From d0521484c854adf082936824c99f9db101363cb8 Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Mon, 4 Mar 2024 10:20:42 -0500 Subject: [PATCH 001/929] [libc++] Remove leftover .fail.cpp matcher in Lit test format (#83583) This should have been removed in 8dcb8ea75cef, which removed support for .fail.cpp tests in the libc++ test suite. --- libcxx/utils/libcxx/test/format.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libcxx/utils/libcxx/test/format.py b/libcxx/utils/libcxx/test/format.py index 229da224994f..7e5281c0b740 100644 --- a/libcxx/utils/libcxx/test/format.py +++ b/libcxx/utils/libcxx/test/format.py @@ -265,7 +265,6 @@ class CxxStandardLibraryTest(lit.formats.FileBasedTest): "[.]sh[.][^.]+$", "[.]gen[.][^.]+$", "[.]verify[.]cpp$", - "[.]fail[.]cpp$", ] sourcePath = testSuite.getSourcePath(pathInSuite) -- GitLab From f010b1bef4dda2c7082cbb41dbabf1f149cce306 Mon Sep 17 00:00:00 2001 From: Mitch Phillips Date: Mon, 4 Mar 2024 17:04:22 +0100 Subject: [PATCH 002/929] Revert "Restore "Implement convergence control in MIR using SelectionDAG (#71785)"" This reverts commit c7fdd8c11e54585dc9d15d63de9742067e0506b9. Reason: Broke the sanitizer buildbots. See the comments at https://github.com/llvm/llvm-project/pull/71785 for more information. --- .../llvm/ADT/GenericConvergenceVerifier.h | 6 -- .../llvm/CodeGen/FunctionLoweringInfo.h | 10 +- llvm/include/llvm/CodeGen/ISDOpcodes.h | 9 -- .../llvm/CodeGen/MachineConvergenceVerifier.h | 28 ------ llvm/include/llvm/CodeGen/SelectionDAGISel.h | 4 - llvm/include/llvm/CodeGen/TargetLowering.h | 6 -- .../llvm/IR/GenericConvergenceVerifierImpl.h | 28 +++--- llvm/include/llvm/Support/TargetOpcodes.def | 5 - llvm/include/llvm/Target/Target.td | 19 ---- .../include/llvm/Target/TargetSelectionDAG.td | 10 -- llvm/lib/CodeGen/CMakeLists.txt | 1 - .../CodeGen/MachineConvergenceVerifier.cpp | 99 ------------------- llvm/lib/CodeGen/MachineVerifier.cpp | 30 ------ .../SelectionDAG/FunctionLoweringInfo.cpp | 10 -- .../lib/CodeGen/SelectionDAG/InstrEmitter.cpp | 44 +-------- .../SelectionDAG/SelectionDAGBuilder.cpp | 50 +--------- .../SelectionDAG/SelectionDAGBuilder.h | 1 - .../SelectionDAG/SelectionDAGDumper.cpp | 10 -- .../CodeGen/SelectionDAG/SelectionDAGISel.cpp | 24 ----- llvm/lib/CodeGen/ValueTypes.cpp | 2 - llvm/lib/IR/ConvergenceVerifier.cpp | 33 +------ llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 27 +---- .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 12 +-- llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h | 1 - llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 24 +---- llvm/lib/Target/AMDGPU/SIInstructions.td | 8 +- .../test/CodeGen/AMDGPU/convergence-tokens.ll | 83 ---------------- .../CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll | 18 ---- .../kernel-vgpr-spill-mubuf-with-voffset.ll | 1 - .../AMDGPU/need-fp-from-vgpr-spills.ll | 18 ++-- .../AMDGPU/no-source-locations-in-prologue.ll | 1 - .../AMDGPU/sgpr-spills-split-regalloc.ll | 15 ++- .../CodeGen/AMDGPU/stacksave_stackrestore.ll | 78 +++++++-------- llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll | 26 ++--- .../CodeGen/AMDGPU/vgpr_constant_to_sgpr.ll | 1 - .../AMDGPU/whole-wave-register-spill.ll | 1 - .../test/CodeGen/AMDGPU/wwm-reserved-spill.ll | 2 - llvm/test/CodeGen/AMDGPU/wwm-reserved.ll | 4 - llvm/test/CodeGen/PowerPC/fmf-propagation.ll | 12 +-- .../convergencectrl/AMDGPU/basic.mir | 37 ------- .../convergencectrl/AMDGPU/cycles.mir | 52 ---------- .../convergencectrl/AMDGPU/lit.local.cfg | 2 - .../convergencectrl/AMDGPU/mixed2.mir | 15 --- .../convergencectrl/AMDGPU/region-nesting.mir | 24 ----- .../builtins/match-table-replacerreg.td | 2 +- .../match-table-imms.td | 30 +++--- .../match-table-intrinsics.td | 2 +- .../match-table-patfrag-root.td | 2 +- .../match-table-variadics.td | 4 +- .../GlobalISelCombinerEmitter/match-table.td | 62 ++++++------ 50 files changed, 161 insertions(+), 832 deletions(-) delete mode 100644 llvm/include/llvm/CodeGen/MachineConvergenceVerifier.h delete mode 100644 llvm/lib/CodeGen/MachineConvergenceVerifier.cpp delete mode 100644 llvm/test/CodeGen/AMDGPU/convergence-tokens.ll delete mode 100644 llvm/test/MachineVerifier/convergencectrl/AMDGPU/basic.mir delete mode 100644 llvm/test/MachineVerifier/convergencectrl/AMDGPU/cycles.mir delete mode 100644 llvm/test/MachineVerifier/convergencectrl/AMDGPU/lit.local.cfg delete mode 100644 llvm/test/MachineVerifier/convergencectrl/AMDGPU/mixed2.mir delete mode 100644 llvm/test/MachineVerifier/convergencectrl/AMDGPU/region-nesting.mir diff --git a/llvm/include/llvm/ADT/GenericConvergenceVerifier.h b/llvm/include/llvm/ADT/GenericConvergenceVerifier.h index dc9495ce57bf..0810a0701322 100644 --- a/llvm/include/llvm/ADT/GenericConvergenceVerifier.h +++ b/llvm/include/llvm/ADT/GenericConvergenceVerifier.h @@ -60,10 +60,6 @@ private: NoConvergence } ConvergenceKind = NoConvergence; - /// The control token operation performed by a convergence control Intrinsic - /// in LLVM IR, or by a CONVERGENCECTRL* instruction in MIR - enum ConvOpKind { CONV_ANCHOR, CONV_ENTRY, CONV_LOOP, CONV_NONE }; - // Cache token uses found so far. Note that we track the unique definitions // and not the token values. DenseMap Tokens; @@ -72,8 +68,6 @@ private: static bool isInsideConvergentFunction(const InstructionT &I); static bool isConvergent(const InstructionT &I); - static ConvOpKind getConvOp(const InstructionT &I); - void checkConvergenceTokenProduced(const InstructionT &I); const InstructionT *findAndCheckConvergenceTokenUsed(const InstructionT &I); void reportFailure(const Twine &Message, ArrayRef Values); diff --git a/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h b/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h index 31af3014afe4..cde7247aeb15 100644 --- a/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h +++ b/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h @@ -215,7 +215,15 @@ public: Register CreateRegs(Type *Ty, bool isDivergent = false); - Register InitializeRegForValue(const Value *V); + Register InitializeRegForValue(const Value *V) { + // Tokens never live in vregs. + if (V->getType()->isTokenTy()) + return 0; + Register &R = ValueMap[V]; + assert(R == 0 && "Already initialized this value register!"); + assert(VirtReg2Value.empty()); + return R = CreateRegs(V); + } /// GetLiveOutRegInfo - Gets LiveOutInfo for a register, returning NULL if the /// register is a PHI destination and the PHI's LiveOutInfo is not valid. diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h index 36b5a60d1830..ef0fec270a43 100644 --- a/llvm/include/llvm/CodeGen/ISDOpcodes.h +++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -1386,15 +1386,6 @@ enum NodeType { #define BEGIN_REGISTER_VP_SDNODE(VPSDID, ...) VPSDID, #include "llvm/IR/VPIntrinsics.def" - // The `llvm.experimental.convergence.*` intrinsics. - CONVERGENCECTRL_ANCHOR, - CONVERGENCECTRL_ENTRY, - CONVERGENCECTRL_LOOP, - // This does not correspond to any convergence control intrinsic. It used to - // glue a convergence control token to a convergent operation in the DAG, - // which is later translated to an implicit use in the MIR. - CONVERGENCECTRL_GLUE, - /// BUILTIN_OP_END - This must be the last enum value in this list. /// The target-specific pre-isel opcode values start here. BUILTIN_OP_END diff --git a/llvm/include/llvm/CodeGen/MachineConvergenceVerifier.h b/llvm/include/llvm/CodeGen/MachineConvergenceVerifier.h deleted file mode 100644 index b2faa30816c6..000000000000 --- a/llvm/include/llvm/CodeGen/MachineConvergenceVerifier.h +++ /dev/null @@ -1,28 +0,0 @@ -//===- MachineConvergenceVerifier.h - Verify convergenctrl ------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// \file -/// -/// This file declares the MIR specialization of the GenericConvergenceVerifier -/// template. -/// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CODEGEN_MACHINECONVERGENCEVERIFIER_H -#define LLVM_CODEGEN_MACHINECONVERGENCEVERIFIER_H - -#include "llvm/ADT/GenericConvergenceVerifier.h" -#include "llvm/CodeGen/MachineSSAContext.h" - -namespace llvm { - -using MachineConvergenceVerifier = - GenericConvergenceVerifier; - -} // namespace llvm - -#endif // LLVM_CODEGEN_MACHINECONVERGENCEVERIFIER_H diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h index 837f8bf7263e..dbd9b391f4a4 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h @@ -459,10 +459,6 @@ private: void Select_ARITH_FENCE(SDNode *N); void Select_MEMBARRIER(SDNode *N); - void Select_CONVERGENCECTRL_ANCHOR(SDNode *N); - void Select_CONVERGENCECTRL_ENTRY(SDNode *N); - void Select_CONVERGENCECTRL_LOOP(SDNode *N); - void pushStackMapLiveVariable(SmallVectorImpl &Ops, SDValue Operand, SDLoc DL); void Select_STACKMAP(SDNode *N); diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index e027779db68e..0438abc7c306 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -4402,7 +4402,6 @@ public: SmallVector Ins; SmallVector InVals; const ConstantInt *CFIType = nullptr; - SDValue ConvergenceControlToken; CallLoweringInfo(SelectionDAG &DAG) : RetSExt(false), RetZExt(false), IsVarArg(false), IsInReg(false), @@ -4536,11 +4535,6 @@ public: return *this; } - CallLoweringInfo &setConvergenceControlToken(SDValue Token) { - ConvergenceControlToken = Token; - return *this; - } - ArgListTy &getArgs() { return Args; } diff --git a/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h b/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h index 979c82320c8c..f6eb5066d553 100644 --- a/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h +++ b/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h @@ -61,11 +61,12 @@ void GenericConvergenceVerifier::visit(const BlockT &BB) { template void GenericConvergenceVerifier::visit(const InstructionT &I) { - ConvOpKind ConvOp = getConvOp(I); - + auto ID = ContextT::getIntrinsicID(I); auto *TokenDef = findAndCheckConvergenceTokenUsed(I); - switch (ConvOp) { - case CONV_ENTRY: + bool IsCtrlIntrinsic = true; + + switch (ID) { + case Intrinsic::experimental_convergence_entry: Check(isInsideConvergentFunction(I), "Entry intrinsic can occur only in a convergent function.", {Context.print(&I)}); @@ -77,13 +78,13 @@ void GenericConvergenceVerifier::visit(const InstructionT &I) { "same basic block.", {Context.print(&I)}); LLVM_FALLTHROUGH; - case CONV_ANCHOR: + case Intrinsic::experimental_convergence_anchor: Check(!TokenDef, "Entry or anchor intrinsic cannot have a convergencectrl token " "operand.", {Context.print(&I)}); break; - case CONV_LOOP: + case Intrinsic::experimental_convergence_loop: Check(TokenDef, "Loop intrinsic must have a convergencectrl token operand.", {Context.print(&I)}); Check(!SeenFirstConvOp, @@ -92,16 +93,14 @@ void GenericConvergenceVerifier::visit(const InstructionT &I) { {Context.print(&I)}); break; default: + IsCtrlIntrinsic = false; break; } - if (ConvOp != CONV_NONE) - checkConvergenceTokenProduced(I); - if (isConvergent(I)) SeenFirstConvOp = true; - if (TokenDef || ConvOp != CONV_NONE) { + if (TokenDef || IsCtrlIntrinsic) { Check(isConvergent(I), "Convergence control token can only be used in a convergent call.", {Context.print(&I)}); @@ -144,10 +143,6 @@ void GenericConvergenceVerifier::verify(const DominatorTreeT &DT) { auto checkToken = [&](const InstructionT *Token, const InstructionT *User, SmallVectorImpl &LiveTokens) { - Check(DT.dominates(Token->getParent(), User->getParent()), - "Convergence control token must dominate all its uses.", - {Context.print(Token), Context.print(User)}); - Check(llvm::is_contained(LiveTokens, Token), "Convergence region is not well-nested.", {Context.print(Token), Context.print(User)}); @@ -166,7 +161,8 @@ void GenericConvergenceVerifier::verify(const DominatorTreeT &DT) { return; } - Check(getConvOp(*User) == CONV_LOOP, + Check(ContextT::getIntrinsicID(*User) == + Intrinsic::experimental_convergence_loop, "Convergence token used by an instruction other than " "llvm.experimental.convergence.loop in a cycle that does " "not contain the token's definition.", @@ -203,7 +199,7 @@ void GenericConvergenceVerifier::verify(const DominatorTreeT &DT) { for (auto &I : *BB) { if (auto *Token = Tokens.lookup(&I)) checkToken(Token, &I, LiveTokens); - if (getConvOp(I) != CONV_NONE) + if (isConvergenceControlIntrinsic(ContextT::getIntrinsicID(I))) LiveTokens.push_back(&I); } diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def index 6aded2ceebe1..42cb854d9505 100644 --- a/llvm/include/llvm/Support/TargetOpcodes.def +++ b/llvm/include/llvm/Support/TargetOpcodes.def @@ -225,11 +225,6 @@ HANDLE_TARGET_OPCODE(MEMBARRIER) // using. HANDLE_TARGET_OPCODE(JUMP_TABLE_DEBUG_INFO) -HANDLE_TARGET_OPCODE(CONVERGENCECTRL_ENTRY) -HANDLE_TARGET_OPCODE(CONVERGENCECTRL_ANCHOR) -HANDLE_TARGET_OPCODE(CONVERGENCECTRL_LOOP) -HANDLE_TARGET_OPCODE(CONVERGENCECTRL_GLUE) - /// The following generic opcodes are not supposed to appear after ISel. /// This is something we might want to relax, but for now, this is convenient /// to produce diagnostics. diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td index 0577c58f8da2..0d97a47190b1 100644 --- a/llvm/include/llvm/Target/Target.td +++ b/llvm/include/llvm/Target/Target.td @@ -1483,25 +1483,6 @@ def JUMP_TABLE_DEBUG_INFO : StandardPseudoInstruction { let isMeta = true; } -let hasSideEffects = false, isMeta = true, isConvergent = true in { -def CONVERGENCECTRL_ANCHOR : StandardPseudoInstruction { - let OutOperandList = (outs unknown:$dst); - let InOperandList = (ins); -} -def CONVERGENCECTRL_ENTRY : StandardPseudoInstruction { - let OutOperandList = (outs unknown:$dst); - let InOperandList = (ins); -} -def CONVERGENCECTRL_LOOP : StandardPseudoInstruction { - let OutOperandList = (outs unknown:$dst); - let InOperandList = (ins unknown:$src); -} -def CONVERGENCECTRL_GLUE : StandardPseudoInstruction { - let OutOperandList = (outs); - let InOperandList = (ins unknown:$src); -} -} - // Generic opcodes used in GlobalISel. include "llvm/Target/GenericOpcodes.td" diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td index d7bf8c35ee10..d84c2d30e447 100644 --- a/llvm/include/llvm/Target/TargetSelectionDAG.td +++ b/llvm/include/llvm/Target/TargetSelectionDAG.td @@ -789,16 +789,6 @@ def assertsext : SDNode<"ISD::AssertSext", SDT_assert>; def assertzext : SDNode<"ISD::AssertZext", SDT_assert>; def assertalign : SDNode<"ISD::AssertAlign", SDT_assert>; -def convergencectrl_anchor : SDNode<"ISD::CONVERGENCECTRL_ANCHOR", - SDTypeProfile<1, 0, [SDTCisVT<0,untyped>]>>; -def convergencectrl_entry : SDNode<"ISD::CONVERGENCECTRL_ENTRY", - SDTypeProfile<1, 0, [SDTCisVT<0,untyped>]>>; -def convergencectrl_loop : SDNode<"ISD::CONVERGENCECTRL_LOOP", - SDTypeProfile<1, 1, - [SDTCisVT<0,untyped>, SDTCisVT<1,untyped>]>>; -def convergencectrl_glue : SDNode<"ISD::CONVERGENCECTRL_GLUE", - SDTypeProfile<0, 1, [SDTCisVT<0, untyped>]>>; - //===----------------------------------------------------------------------===// // Selection DAG Condition Codes diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt index 2c24de60edd4..e02c1d6417e0 100644 --- a/llvm/lib/CodeGen/CMakeLists.txt +++ b/llvm/lib/CodeGen/CMakeLists.txt @@ -110,7 +110,6 @@ add_llvm_component_library(LLVMCodeGen MachineBranchProbabilityInfo.cpp MachineCFGPrinter.cpp MachineCombiner.cpp - MachineConvergenceVerifier.cpp MachineCopyPropagation.cpp MachineCSE.cpp MachineCheckDebugify.cpp diff --git a/llvm/lib/CodeGen/MachineConvergenceVerifier.cpp b/llvm/lib/CodeGen/MachineConvergenceVerifier.cpp deleted file mode 100644 index 9320dfebfe59..000000000000 --- a/llvm/lib/CodeGen/MachineConvergenceVerifier.cpp +++ /dev/null @@ -1,99 +0,0 @@ -//===- ConvergenceVerifier.cpp - Verify convergence control -----*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -//===----------------------------------------------------------------------===// - -#include "llvm/CodeGen/MachineConvergenceVerifier.h" -#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" -#include "llvm/CodeGen/MachineDominators.h" -#include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/MachineSSAContext.h" -#include "llvm/IR/GenericConvergenceVerifierImpl.h" - -using namespace llvm; - -template <> -auto GenericConvergenceVerifier::getConvOp( - const MachineInstr &MI) -> ConvOpKind { - switch (MI.getOpcode()) { - default: - return CONV_NONE; - case TargetOpcode::CONVERGENCECTRL_ENTRY: - return CONV_ENTRY; - case TargetOpcode::CONVERGENCECTRL_ANCHOR: - return CONV_ANCHOR; - case TargetOpcode::CONVERGENCECTRL_LOOP: - return CONV_LOOP; - } -} - -template <> -void GenericConvergenceVerifier< - MachineSSAContext>::checkConvergenceTokenProduced(const MachineInstr &MI) { - Check(!MI.hasImplicitDef(), - "Convergence control tokens are defined explicitly.", - {Context.print(&MI)}); - const MachineOperand &Def = MI.getOperand(0); - const MachineRegisterInfo &MRI = Context.getFunction()->getRegInfo(); - Check(MRI.getUniqueVRegDef(Def.getReg()), - "Convergence control tokens must have unique definitions.", - {Context.print(&MI)}); -} - -template <> -const MachineInstr * -GenericConvergenceVerifier::findAndCheckConvergenceTokenUsed( - const MachineInstr &MI) { - const MachineRegisterInfo &MRI = Context.getFunction()->getRegInfo(); - const MachineInstr *TokenDef = nullptr; - - for (const MachineOperand &MO : MI.uses()) { - if (!MO.isReg()) - continue; - Register OpReg = MO.getReg(); - if (!OpReg.isVirtual()) - continue; - - const MachineInstr *Def = MRI.getUniqueVRegDef(OpReg); - if (!Def) - continue; - if (getConvOp(*Def) == CONV_NONE) - continue; - - CheckOrNull( - MI.isConvergent(), - "Convergence control tokens can only be used by convergent operations.", - {Context.print(OpReg), Context.print(&MI)}); - - CheckOrNull(!TokenDef, - "An operation can use at most one convergence control token.", - {Context.print(OpReg), Context.print(&MI)}); - - TokenDef = Def; - } - - if (TokenDef) - Tokens[&MI] = TokenDef; - - return TokenDef; -} - -template <> -bool GenericConvergenceVerifier::isInsideConvergentFunction( - const MachineInstr &MI) { - // The class MachineFunction does not have any property to indicate whether it - // is convergent. Trivially return true so that the check always passes. - return true; -} - -template <> -bool GenericConvergenceVerifier::isConvergent( - const MachineInstr &MI) { - return MI.isConvergent(); -} - -template class llvm::GenericConvergenceVerifier; diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp index 1d0757c5d7f5..2632b5b9feac 100644 --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -39,8 +39,6 @@ #include "llvm/CodeGen/LiveStacks.h" #include "llvm/CodeGen/LiveVariables.h" #include "llvm/CodeGen/MachineBasicBlock.h" -#include "llvm/CodeGen/MachineConvergenceVerifier.h" -#include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" @@ -222,11 +220,6 @@ namespace { LiveStacks *LiveStks = nullptr; SlotIndexes *Indexes = nullptr; - // This is calculated only when trying to verify convergence control tokens. - // Similar to the LLVM IR verifier, we calculate this locally instead of - // relying on the pass manager. - MachineDomTree DT; - void visitMachineFunctionBefore(); void visitMachineBasicBlockBefore(const MachineBasicBlock *MBB); void visitMachineBundleBefore(const MachineInstr *MI); @@ -2962,30 +2955,7 @@ void MachineVerifier::checkPHIOps(const MachineBasicBlock &MBB) { } } -static void -verifyConvergenceControl(const MachineFunction &MF, MachineDomTree &DT, - std::function FailureCB) { - MachineConvergenceVerifier CV; - CV.initialize(&errs(), FailureCB, MF); - - for (const auto &MBB : MF) { - CV.visit(MBB); - for (const auto &MI : MBB.instrs()) - CV.visit(MI); - } - - if (CV.sawTokens()) { - DT.recalculate(const_cast(MF)); - CV.verify(DT); - } -} - void MachineVerifier::visitMachineFunctionAfter() { - auto FailureCB = [this](const Twine &Message) { - report(Message.str().c_str(), MF); - }; - verifyConvergenceControl(*MF, DT, FailureCB); - calcRegsPassed(); for (const MachineBasicBlock &MBB : *MF) diff --git a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp index e01cd8cbf925..4172fbc96d1e 100644 --- a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp @@ -395,16 +395,6 @@ Register FunctionLoweringInfo::CreateRegs(const Value *V) { !TLI->requiresUniformRegister(*MF, V)); } -Register FunctionLoweringInfo::InitializeRegForValue(const Value *V) { - // Tokens live in vregs only when used for convergence control. - if (V->getType()->isTokenTy() && !isa(V)) - return 0; - Register &R = ValueMap[V]; - assert(R == Register() && "Already initialized this value register!"); - assert(VirtReg2Value.empty()); - return R = CreateRegs(V); -} - /// GetLiveOutRegInfo - Gets LiveOutInfo for a register, returning NULL if the /// register is a PHI destination and the PHI's LiveOutInfo is not valid. If /// the register's LiveOutInfo is for a smaller bit width, it is extended to diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp index 54409cbf91f1..032cff416cda 100644 --- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -285,30 +285,6 @@ Register InstrEmitter::getVR(SDValue Op, return I->second; } -static bool isConvergenceCtrlMachineOp(SDValue Op) { - if (Op->isMachineOpcode()) { - switch (Op->getMachineOpcode()) { - case TargetOpcode::CONVERGENCECTRL_ANCHOR: - case TargetOpcode::CONVERGENCECTRL_ENTRY: - case TargetOpcode::CONVERGENCECTRL_LOOP: - case TargetOpcode::CONVERGENCECTRL_GLUE: - return true; - } - return false; - } - - // We can reach here when CopyFromReg is encountered. But rather than making a - // special case for that, we just make sure we don't reach here in some - // surprising way. - switch (Op->getOpcode()) { - case ISD::CONVERGENCECTRL_ANCHOR: - case ISD::CONVERGENCECTRL_ENTRY: - case ISD::CONVERGENCECTRL_LOOP: - case ISD::CONVERGENCECTRL_GLUE: - llvm_unreachable("Convergence control should have been selected by now."); - } - return false; -} /// AddRegisterOperand - Add the specified register as an operand to the /// specified machine instr. Insert register copies if the register is @@ -370,12 +346,9 @@ InstrEmitter::AddRegisterOperand(MachineInstrBuilder &MIB, // multiple uses. // Tied operands are never killed, so we need to check that. And that // means we need to determine the index of the operand. - // Don't kill convergence control tokens. Initially they are only used in glue - // nodes, and the InstrEmitter later adds implicit uses on the users of the - // glue node. This can sometimes make it seem like there is only one use, - // which is the glue node itself. - bool isKill = Op.hasOneUse() && !isConvergenceCtrlMachineOp(Op) && - Op.getNode()->getOpcode() != ISD::CopyFromReg && !IsDebug && + bool isKill = Op.hasOneUse() && + Op.getNode()->getOpcode() != ISD::CopyFromReg && + !IsDebug && !(IsClone || IsCloned); if (isKill) { unsigned Idx = MIB->getNumOperands(); @@ -1218,17 +1191,6 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, } } - if (SDNode *GluedNode = Node->getGluedNode()) { - // FIXME: Possibly iterate over multiple glue nodes? - if (GluedNode->getOpcode() == - ~(unsigned)TargetOpcode::CONVERGENCECTRL_GLUE) { - Register VReg = getVR(GluedNode->getOperand(0), VRBaseMap); - MachineOperand MO = MachineOperand::CreateReg(VReg, /*isDef=*/false, - /*isImp=*/true); - MIB->addOperand(MO); - } - } - // Run post-isel target hook to adjust this instruction if needed. if (II.hasPostISelHook()) TLI->AdjustInstrPostInstrSelection(*MIB, Node); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 73bfed530999..ab2f42d2024c 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -5078,17 +5078,6 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, // Create the node. SDValue Result; - - if (auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl)) { - auto *Token = Bundle->Inputs[0].get(); - SDValue ConvControlToken = getValue(Token); - assert(Ops.back().getValueType() != MVT::Glue && - "Did not expected another glue node here."); - ConvControlToken = - DAG.getNode(ISD::CONVERGENCECTRL_GLUE, {}, MVT::Glue, ConvControlToken); - Ops.push_back(ConvControlToken); - } - // In some cases, custom collection of operands from CallInst I may be needed. TLI.CollectTargetIntrinsicOperands(I, Ops, DAG); if (IsTgtIntrinsic) { @@ -6089,27 +6078,6 @@ bool SelectionDAGBuilder::visitEntryValueDbgValue( return true; } -/// Lower the call to the specified intrinsic function. -void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I, - unsigned Intrinsic) { - SDLoc sdl = getCurSDLoc(); - switch (Intrinsic) { - case Intrinsic::experimental_convergence_anchor: - setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ANCHOR, sdl, MVT::Untyped)); - break; - case Intrinsic::experimental_convergence_entry: - setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ENTRY, sdl, MVT::Untyped)); - break; - case Intrinsic::experimental_convergence_loop: { - auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl); - auto *Token = Bundle->Inputs[0].get(); - setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_LOOP, sdl, MVT::Untyped, - getValue(Token))); - break; - } - } -} - /// Lower the call to the specified intrinsic function. void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { @@ -7769,10 +7737,6 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, case Intrinsic::experimental_vector_deinterleave2: visitVectorDeinterleave(I); return; - case Intrinsic::experimental_convergence_anchor: - case Intrinsic::experimental_convergence_entry: - case Intrinsic::experimental_convergence_loop: - visitConvergenceControl(I, Intrinsic); } } @@ -8449,14 +8413,6 @@ void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee, } } - SDValue ConvControlToken; - if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) { - auto *Token = Bundle->Inputs[0].get(); - ConvControlToken = getValue(Token); - } else { - ConvControlToken = DAG.getUNDEF(MVT::Untyped); - } - TargetLowering::CallLoweringInfo CLI(DAG); CLI.setDebugLoc(getCurSDLoc()) .setChain(getRoot()) @@ -8465,8 +8421,7 @@ void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee, .setConvergent(CB.isConvergent()) .setIsPreallocated( CB.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0) - .setCFIType(CFIType) - .setConvergenceControlToken(ConvControlToken); + .setCFIType(CFIType); std::pair Result = lowerInvokable(CLI, EHPadBB); if (Result.first.getNode()) { @@ -9018,8 +8973,7 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) { assert(!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated, - LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi, - LLVMContext::OB_convergencectrl}) && + LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi}) && "Cannot lower calls with arbitrary operand bundles!"); SDValue Callee = getValue(I.getCalledOperand()); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h index 9b735672eedf..47657313cb6a 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -618,7 +618,6 @@ private: void visitIntrinsicCall(const CallInst &I, unsigned Intrinsic); void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic); void visitConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI); - void visitConvergenceControl(const CallInst &I, unsigned Intrinsic); void visitVPLoad(const VPIntrinsic &VPIntrin, EVT VT, const SmallVectorImpl &OpValues); void visitVPStore(const VPIntrinsic &VPIntrin, diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp index fa71adc8da3f..18ca17e53dac 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -165,9 +165,6 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { if (cast(this)->isOpaque()) return "OpaqueTargetConstant"; return "TargetConstant"; - - // clang-format off - case ISD::TargetConstantFP: return "TargetConstantFP"; case ISD::TargetGlobalAddress: return "TargetGlobalAddress"; case ISD::TargetGlobalTLSAddress: return "TargetGlobalTLSAddress"; @@ -452,11 +449,6 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::SET_FPMODE: return "set_fpmode"; case ISD::RESET_FPMODE: return "reset_fpmode"; - // Convergence control instructions - case ISD::CONVERGENCECTRL_ANCHOR: return "convergencectrl_anchor"; - case ISD::CONVERGENCECTRL_ENTRY: return "convergencectrl_entry"; - case ISD::CONVERGENCECTRL_LOOP: return "convergencectrl_loop"; - // Bit manipulation case ISD::ABS: return "abs"; case ISD::BITREVERSE: return "bitreverse"; @@ -472,8 +464,6 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::INIT_TRAMPOLINE: return "init_trampoline"; case ISD::ADJUST_TRAMPOLINE: return "adjust_trampoline"; - // clang-format on - case ISD::CONDCODE: switch (cast(this)->get()) { default: llvm_unreachable("Unknown setcc condition!"); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 1c14e4da8e9d..9b5ab4267b80 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -2370,21 +2370,6 @@ void SelectionDAGISel::Select_MEMBARRIER(SDNode *N) { N->getOperand(0)); } -void SelectionDAGISel::Select_CONVERGENCECTRL_ANCHOR(SDNode *N) { - CurDAG->SelectNodeTo(N, TargetOpcode::CONVERGENCECTRL_ANCHOR, - N->getValueType(0)); -} - -void SelectionDAGISel::Select_CONVERGENCECTRL_ENTRY(SDNode *N) { - CurDAG->SelectNodeTo(N, TargetOpcode::CONVERGENCECTRL_ENTRY, - N->getValueType(0)); -} - -void SelectionDAGISel::Select_CONVERGENCECTRL_LOOP(SDNode *N) { - CurDAG->SelectNodeTo(N, TargetOpcode::CONVERGENCECTRL_LOOP, - N->getValueType(0), N->getOperand(0)); -} - void SelectionDAGISel::pushStackMapLiveVariable(SmallVectorImpl &Ops, SDValue OpVal, SDLoc DL) { SDNode *OpNode = OpVal.getNode(); @@ -3132,15 +3117,6 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, case ISD::JUMP_TABLE_DEBUG_INFO: Select_JUMP_TABLE_DEBUG_INFO(NodeToMatch); return; - case ISD::CONVERGENCECTRL_ANCHOR: - Select_CONVERGENCECTRL_ANCHOR(NodeToMatch); - return; - case ISD::CONVERGENCECTRL_ENTRY: - Select_CONVERGENCECTRL_ENTRY(NodeToMatch); - return; - case ISD::CONVERGENCECTRL_LOOP: - Select_CONVERGENCECTRL_LOOP(NodeToMatch); - return; } assert(!NodeToMatch->isMachineOpcode() && "Node already selected!"); diff --git a/llvm/lib/CodeGen/ValueTypes.cpp b/llvm/lib/CodeGen/ValueTypes.cpp index 58db686ec7d5..5dcabdb6a9e0 100644 --- a/llvm/lib/CodeGen/ValueTypes.cpp +++ b/llvm/lib/CodeGen/ValueTypes.cpp @@ -629,8 +629,6 @@ EVT EVT::getEVT(Type *Ty, bool HandleUnknown){ switch (Ty->getTypeID()) { default: return MVT::getVT(Ty, HandleUnknown); - case Type::TokenTyID: - return MVT::Untyped; case Type::IntegerTyID: return getIntegerVT(Ty->getContext(), cast(Ty)->getBitWidth()); case Type::FixedVectorTyID: diff --git a/llvm/lib/IR/ConvergenceVerifier.cpp b/llvm/lib/IR/ConvergenceVerifier.cpp index e73aeaade5f7..336c202b6f94 100644 --- a/llvm/lib/IR/ConvergenceVerifier.cpp +++ b/llvm/lib/IR/ConvergenceVerifier.cpp @@ -14,30 +14,6 @@ using namespace llvm; -template <> -auto GenericConvergenceVerifier::getConvOp(const Instruction &I) - -> ConvOpKind { - const auto *CB = dyn_cast(&I); - if (!CB) - return CONV_NONE; - switch (CB->getIntrinsicID()) { - default: - return CONV_NONE; - case Intrinsic::experimental_convergence_anchor: - return CONV_ANCHOR; - case Intrinsic::experimental_convergence_entry: - return CONV_ENTRY; - case Intrinsic::experimental_convergence_loop: - return CONV_LOOP; - } -} - -template <> -void GenericConvergenceVerifier::checkConvergenceTokenProduced( - const Instruction &I) { - return; -} - template <> const Instruction * GenericConvergenceVerifier::findAndCheckConvergenceTokenUsed( @@ -62,10 +38,11 @@ GenericConvergenceVerifier::findAndCheckConvergenceTokenUsed( auto *Token = Bundle->Inputs[0].get(); auto *Def = dyn_cast(Token); - CheckOrNull(Def && getConvOp(*Def) != CONV_NONE, - "Convergence control tokens can only be produced by calls to the " - "convergence control intrinsics.", - {Context.print(Token), Context.print(&I)}); + CheckOrNull( + Def && isConvergenceControlIntrinsic(SSAContext::getIntrinsicID(*Def)), + "Convergence control tokens can only be produced by calls to the " + "convergence control intrinsics.", + {Context.print(Token), Context.print(&I)}); if (Def) Tokens[&I] = Def; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp index 0edf78cf347e..4896ae8bad9e 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp @@ -2687,18 +2687,7 @@ void AMDGPUDAGToDAGISel::SelectINTRINSIC_W_CHAIN(SDNode *N) { void AMDGPUDAGToDAGISel::SelectINTRINSIC_WO_CHAIN(SDNode *N) { unsigned IntrID = N->getConstantOperandVal(0); - unsigned Opcode = AMDGPU::INSTRUCTION_LIST_END; - SDNode *ConvGlueNode = N->getGluedNode(); - if (ConvGlueNode) { - // FIXME: Possibly iterate over multiple glue nodes? - assert(ConvGlueNode->getOpcode() == ISD::CONVERGENCECTRL_GLUE); - ConvGlueNode = ConvGlueNode->getOperand(0).getNode(); - ConvGlueNode = - CurDAG->getMachineNode(TargetOpcode::CONVERGENCECTRL_GLUE, {}, - MVT::Glue, SDValue(ConvGlueNode, 0)); - } else { - ConvGlueNode = nullptr; - } + unsigned Opcode; switch (IntrID) { case Intrinsic::amdgcn_wqm: Opcode = AMDGPU::WQM; @@ -2730,19 +2719,11 @@ void AMDGPUDAGToDAGISel::SelectINTRINSIC_WO_CHAIN(SDNode *N) { break; default: SelectCode(N); - break; - } - - if (Opcode != AMDGPU::INSTRUCTION_LIST_END) { - SDValue Src = N->getOperand(1); - CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), {Src}); + return; } - if (ConvGlueNode) { - SmallVector NewOps(N->op_begin(), N->op_end()); - NewOps.push_back(SDValue(ConvGlueNode, 0)); - CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), NewOps); - } + SDValue Src = N->getOperand(1); + CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), {Src}); } void AMDGPUDAGToDAGISel::SelectINTRINSIC_VOID(SDNode *N) { diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index 7467c25a84d6..76e843455bab 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -245,13 +245,6 @@ static cl::opt LateCFGStructurize( cl::location(AMDGPUTargetMachine::EnableLateStructurizeCFG), cl::Hidden); -// Disable structurizer-based control-flow lowering in order to test convergence -// control tokens. This should eventually be replaced by the wave-transform. -static cl::opt DisableStructurizer( - "amdgpu-disable-structurizer", - cl::desc("Disable structurizer for experiments; produces unusable code"), - cl::location(AMDGPUTargetMachine::DisableStructurizer), cl::ReallyHidden); - // Enable lib calls simplifications static cl::opt EnableLibCallSimplify( "amdgpu-simplify-libcall", @@ -598,7 +591,6 @@ AMDGPUTargetMachine::AMDGPUTargetMachine(const Target &T, const Triple &TT, bool AMDGPUTargetMachine::EnableLateStructurizeCFG = false; bool AMDGPUTargetMachine::EnableFunctionCalls = false; bool AMDGPUTargetMachine::EnableLowerModuleLDS = true; -bool AMDGPUTargetMachine::DisableStructurizer = false; AMDGPUTargetMachine::~AMDGPUTargetMachine() = default; @@ -1194,7 +1186,7 @@ bool GCNPassConfig::addPreISel() { // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit // regions formed by them. addPass(&AMDGPUUnifyDivergentExitNodesID); - if (!LateCFGStructurize && !DisableStructurizer) { + if (!LateCFGStructurize) { if (EnableStructurizerWorkarounds) { addPass(createFixIrreduciblePass()); addPass(createUnifyLoopExitsPass()); @@ -1202,7 +1194,7 @@ bool GCNPassConfig::addPreISel() { addPass(createStructurizeCFGPass(false)); // true -> SkipUniformRegions } addPass(createAMDGPUAnnotateUniformValues()); - if (!LateCFGStructurize && !DisableStructurizer) { + if (!LateCFGStructurize) { addPass(createSIAnnotateControlFlowPass()); // TODO: Move this right after structurizeCFG to avoid extra divergence // analysis. This depends on stopping SIAnnotateControlFlow from making diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h index 30ab388c7d52..ce2dd2947daf 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h @@ -37,7 +37,6 @@ public: static bool EnableLateStructurizeCFG; static bool EnableFunctionCalls; static bool EnableLowerModuleLDS; - static bool DisableStructurizer; AMDGPUTargetMachine(const Target &T, const Triple &TT, StringRef CPU, StringRef FS, const TargetOptions &Options, diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 9476c33acc34..30a65bb33265 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -98,7 +98,6 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, addRegisterClass(MVT::f64, V64RegClass); addRegisterClass(MVT::v2f32, V64RegClass); - addRegisterClass(MVT::Untyped, V64RegClass); addRegisterClass(MVT::v3i32, &AMDGPU::SGPR_96RegClass); addRegisterClass(MVT::v3f32, TRI->getVGPRClassForBitWidth(96)); @@ -3830,9 +3829,6 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI, Ops.push_back(DAG.getTargetConstant(0, DL, MVT::i64)); } - if (!IsTailCall) - Ops.push_back(CLI.ConvergenceControlToken); - if (IsTailCall) { // Each tail call may have to adjust the stack by a different amount, so // this information must travel along with the operation for eventual @@ -5158,24 +5154,8 @@ MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter( MachineInstrBuilder MIB; MIB = BuildMI(*BB, MI, DL, TII->get(AMDGPU::SI_CALL), ReturnAddrReg); - for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) { - MachineOperand &MO = MI.getOperand(I); - if (I != 2) { - MIB.add(MO); - continue; - } - } - - MachineOperand &MO = MI.getOperand(2); - MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); - // The token operand is always a register, whose definition is IMPLICIT_DEF - // iff there was no token on the call. - if (MachineInstr *Def = MRI.getVRegDef(MO.getReg())) { - if (Def->getOpcode() != TargetOpcode::IMPLICIT_DEF) { - MO.setImplicit(); - MIB.add(MO); - } - } + for (const MachineOperand &MO : MI.operands()) + MIB.add(MO); MIB.cloneMemRefs(MI); MI.eraseFromParent(); diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td index 33c93cdf20c4..565af36bc523 100644 --- a/llvm/lib/Target/AMDGPU/SIInstructions.td +++ b/llvm/lib/Target/AMDGPU/SIInstructions.td @@ -618,8 +618,8 @@ def SI_RETURN : SPseudoInstSI < // This version is only needed so we can fill in the output register // in the custom inserter. def SI_CALL_ISEL : SPseudoInstSI < - (outs), (ins SSrc_b64:$src0, unknown:$callee, unknown:$token), - [(AMDGPUcall i64:$src0, tglobaladdr:$callee, untyped:$token)]> { + (outs), (ins SSrc_b64:$src0, unknown:$callee), + [(AMDGPUcall i64:$src0, tglobaladdr:$callee)]> { let Size = 4; let isCall = 1; let SchedRW = [WriteBranch]; @@ -629,8 +629,8 @@ def SI_CALL_ISEL : SPseudoInstSI < } def : GCNPat< - (AMDGPUcall i64:$src0, (i64 0), untyped:$token), - (SI_CALL_ISEL $src0, (i64 0), untyped:$token) + (AMDGPUcall i64:$src0, (i64 0)), + (SI_CALL_ISEL $src0, (i64 0)) >; // Wrapper around s_swappc_b64 with extra $callee parameter to track diff --git a/llvm/test/CodeGen/AMDGPU/convergence-tokens.ll b/llvm/test/CodeGen/AMDGPU/convergence-tokens.ll deleted file mode 100644 index 2ed6d7fd0f59..000000000000 --- a/llvm/test/CodeGen/AMDGPU/convergence-tokens.ll +++ /dev/null @@ -1,83 +0,0 @@ -; RUN: llc --amdgpu-disable-structurizer -stop-after=amdgpu-isel -mtriple=amdgcn-- -mcpu=gfx900 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=CHECK,ISEL %s -; RUN: llc --amdgpu-disable-structurizer -stop-after=dead-mi-elimination -mtriple=amdgcn-- -mcpu=gfx900 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=CHECK,DEADMI %s - -; CHECK-LABEL: name: basic_call -; CHECK: [[TOKEN:%[0-9]+]]:sreg_64 = CONVERGENCECTRL_ENTRY -; ISEL: {{.*}} SI_CALL_ISEL {{.*}}, @foo, [[TOKEN]], csr_amdgpu, {{.*}} -; DEADMI: {{.*}} SI_CALL {{.*}}, @foo, csr_amdgpu, {{.*}}, implicit [[TOKEN]] -define i32 @basic_call(i32 %src) #0 { - %t = call token @llvm.experimental.convergence.entry() - %r = call i32 @foo(i32 %src) [ "convergencectrl"(token %t) ] - ret i32 %r -} - -; CHECK-LABEL: name: basic_intrinsic -; CHECK: [[TOKEN:%[0-9]+]]:sreg_64 = CONVERGENCECTRL_ANCHOR -; ISEL: CONVERGENCECTRL_GLUE [[TOKEN]] -; DEADMI-NOT: CONVERGENCECTRL_GLUE -; CHECK: {{.*}} = V_READFIRSTLANE_B32 {{.*}}, implicit [[TOKEN]] -define i32 @basic_intrinsic(i32 %src) #0 { - %t = call token @llvm.experimental.convergence.anchor() - %r = call i32 @llvm.amdgcn.readfirstlane(i32 %src) [ "convergencectrl"(token %t) ] - ret i32 %r -} - -; There's nothing to check here. The test is just meant to catch any crashes -; when a convergent call has no token. -define i32 @uncontrolled_call(i32 %src) #0 { - %r = call i32 @foo(i32 %src) - ret i32 %r -} - -; CHECK-LABEL: name: basic_branch -; CHECK: bb.0.entry: -; CHECK: [[TOKEN:%[0-9]+]]:sreg_64 = CONVERGENCECTRL_ANCHOR -; CHECK: bb.1.then: -; ISEL: CONVERGENCECTRL_GLUE [[TOKEN]] -; DEADMI-NOT: CONVERGENCECTRL_GLUE -; CHECK: {{.*}} = V_READFIRSTLANE_B32 {{.*}}, implicit [[TOKEN]] -define i32 @basic_branch(i32 %src, i1 %cond) #0 { -entry: - %t = call token @llvm.experimental.convergence.anchor() - %x = add i32 %src, 1 - br i1 %cond, label %then, label %else - -then: - %r = call i32 @llvm.amdgcn.readfirstlane(i32 %x) [ "convergencectrl"(token %t) ] - br label %else - -else: - %p = phi i32 [%r, %then], [%x, %entry] - ret i32 %p -} - -; CHECK-LABEL: name: basic_loop -; CHECK: [[TOKEN:%[0-9]+]]:sreg_64 = CONVERGENCECTRL_ANCHOR -; CHECK: bb.1.loop: -; CHECK: [[LOOP:%[0-9]+]]:sreg_64 = CONVERGENCECTRL_LOOP [[TOKEN]] -; ISEL: CONVERGENCECTRL_GLUE [[LOOP]] -; DEADMI-NOT: CONVERGENCECTRL_GLUE -; CHECK: {{.*}} = V_READFIRSTLANE_B32 {{.*}}, implicit [[LOOP]] -define i32 @basic_loop(i32 %src, i1 %cond) #0 { - %t1 = call token @llvm.experimental.convergence.anchor() - br label %loop - -loop: - %t2 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t1) ] - %r = call i32 @llvm.amdgcn.readfirstlane(i32 %src) [ "convergencectrl"(token %t2) ] - br i1 %cond, label %loop, label %end - -end: - ret i32 %r -} - -declare i32 @foo(i32 %x) #0 - -declare i32 @llvm.amdgcn.readfirstlane(i32) #0 - -declare token @llvm.experimental.convergence.entry() -declare token @llvm.experimental.convergence.anchor() -declare token @llvm.experimental.convergence.loop() - -attributes #0 = { nounwind readnone convergent } -attributes #1 = { nounwind } diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll index e015095a4884..ab160ffc10ed 100644 --- a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll +++ b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll @@ -92,7 +92,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc(<4 x i32> inreg %a, <4 x i32> %b ; DAGISEL-GFX11-NEXT: $vgpr5 = COPY [[COPY2]] ; DAGISEL-GFX11-NEXT: $vgpr6 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr7 = COPY [[COPY]] - ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -122,7 +121,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc(<4 x i32> inreg %a, <4 x i32> %b ; DAGISEL-GFX10-NEXT: $vgpr5 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr6 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr7 = COPY [[COPY]] - ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -234,7 +232,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_ptr(ptr inreg %a, ptr %b, ptr ad ; DAGISEL-GFX11-NEXT: $vgpr9 = COPY [[COPY2]] ; DAGISEL-GFX11-NEXT: $vgpr10 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr11 = COPY [[COPY]] - ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -272,7 +269,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_ptr(ptr inreg %a, ptr %b, ptr ad ; DAGISEL-GFX10-NEXT: $vgpr9 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr10 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr11 = COPY [[COPY]] - ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -404,7 +400,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_struct( {ptr, i32, <4 x i32>} in ; DAGISEL-GFX11-NEXT: $vgpr11 = COPY [[COPY2]] ; DAGISEL-GFX11-NEXT: $vgpr12 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr13 = COPY [[COPY]] - ; DAGISEL-GFX11-NEXT: [[DEF2:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -454,7 +449,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_struct( {ptr, i32, <4 x i32>} in ; DAGISEL-GFX10-NEXT: $vgpr11 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr12 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr13 = COPY [[COPY]] - ; DAGISEL-GFX10-NEXT: [[DEF2:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -506,7 +500,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_float(float inreg %a, float %b) ; DAGISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4) ; DAGISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY]] - ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -524,7 +517,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_float(float inreg %a, float %b) ; DAGISEL-GFX10-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr1 = COPY [[COPY]] - ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -576,7 +568,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_half(half inreg %a, half %b) { ; DAGISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4) ; DAGISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY]] - ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -594,7 +585,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_half(half inreg %a, half %b) { ; DAGISEL-GFX10-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr1 = COPY [[COPY]] - ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -646,7 +636,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_bfloat(bfloat inreg %a, bfloat % ; DAGISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4) ; DAGISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY]] - ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -664,7 +653,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_bfloat(bfloat inreg %a, bfloat % ; DAGISEL-GFX10-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr1 = COPY [[COPY]] - ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -716,7 +704,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_i16(i16 inreg %a, i16 %b) { ; DAGISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4) ; DAGISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY]] - ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -734,7 +721,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_i16(i16 inreg %a, i16 %b) { ; DAGISEL-GFX10-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr1 = COPY [[COPY]] - ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -870,7 +856,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_v16i16(<16 x i16> inreg %a, <16 ; DAGISEL-GFX11-NEXT: $vgpr13 = COPY [[COPY2]] ; DAGISEL-GFX11-NEXT: $vgpr14 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr15 = COPY [[COPY]] - ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -916,7 +901,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_v16i16(<16 x i16> inreg %a, <16 ; DAGISEL-GFX10-NEXT: $vgpr13 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr14 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr15 = COPY [[COPY]] - ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -2480,7 +2464,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_many_regs(<36 x i32> inreg %a, <128 ; DAGISEL-GFX11-NEXT: $vgpr29 = COPY [[COPY134]] ; DAGISEL-GFX11-NEXT: $vgpr30 = COPY [[COPY133]] ; DAGISEL-GFX11-NEXT: $vgpr31 = COPY [[COPY132]] - ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $vgpr31 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 528, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -2827,7 +2810,6 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_many_regs(<36 x i32> inreg %a, <128 ; DAGISEL-GFX10-NEXT: $vgpr29 = COPY [[COPY134]] ; DAGISEL-GFX10-NEXT: $vgpr30 = COPY [[COPY133]] ; DAGISEL-GFX10-NEXT: $vgpr31 = COPY [[COPY132]] - ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $vgpr31 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 528, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 diff --git a/llvm/test/CodeGen/AMDGPU/kernel-vgpr-spill-mubuf-with-voffset.ll b/llvm/test/CodeGen/AMDGPU/kernel-vgpr-spill-mubuf-with-voffset.ll index 8b6b48bcdba0..6e905542ce53 100644 --- a/llvm/test/CodeGen/AMDGPU/kernel-vgpr-spill-mubuf-with-voffset.ll +++ b/llvm/test/CodeGen/AMDGPU/kernel-vgpr-spill-mubuf-with-voffset.ll @@ -60,7 +60,6 @@ define amdgpu_kernel void @test_kernel(i32 %val) #0 { ; CHECK-NEXT: ; implicit-def: $sgpr15 ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] -; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_waitcnt lgkmcnt(0) ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: s_or_saveexec_b64 s[34:35], -1 diff --git a/llvm/test/CodeGen/AMDGPU/need-fp-from-vgpr-spills.ll b/llvm/test/CodeGen/AMDGPU/need-fp-from-vgpr-spills.ll index 5f507d482eeb..f70441e87a74 100644 --- a/llvm/test/CodeGen/AMDGPU/need-fp-from-vgpr-spills.ll +++ b/llvm/test/CodeGen/AMDGPU/need-fp-from-vgpr-spills.ll @@ -27,7 +27,7 @@ define internal fastcc void @csr_vgpr_spill_fp_callee() #0 { ; CHECK-LABEL: csr_vgpr_spill_fp_callee: ; CHECK: ; %bb.0: ; %bb ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: s_mov_b32 s24, s33 +; CHECK-NEXT: s_mov_b32 s18, s33 ; CHECK-NEXT: s_mov_b32 s33, s32 ; CHECK-NEXT: s_xor_saveexec_b64 s[16:17], -1 ; CHECK-NEXT: buffer_store_dword v1, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill @@ -43,7 +43,6 @@ define internal fastcc void @csr_vgpr_spill_fp_callee() #0 { ; CHECK-NEXT: s_mov_b64 s[20:21], s[0:1] ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] -; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: ;;#ASMSTART ; CHECK-NEXT: ; clobber csr v40 @@ -55,7 +54,7 @@ define internal fastcc void @csr_vgpr_spill_fp_callee() #0 { ; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload ; CHECK-NEXT: s_mov_b64 exec, s[4:5] ; CHECK-NEXT: s_add_i32 s32, s32, 0xfffffc00 -; CHECK-NEXT: s_mov_b32 s33, s24 +; CHECK-NEXT: s_mov_b32 s33, s18 ; CHECK-NEXT: s_waitcnt vmcnt(0) ; CHECK-NEXT: s_setpc_b64 s[30:31] bb: @@ -88,7 +87,6 @@ define amdgpu_kernel void @kernel_call() { ; CHECK-NEXT: ; implicit-def: $sgpr15 ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] -; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: s_endpgm bb: @@ -148,7 +146,6 @@ define amdgpu_kernel void @kernel_tailcall() { ; CHECK-NEXT: ; implicit-def: $sgpr15 ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] -; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: s_endpgm bb: @@ -173,7 +170,7 @@ define hidden i32 @caller_save_vgpr_spill_fp_tail_call() #0 { ; CHECK-LABEL: caller_save_vgpr_spill_fp_tail_call: ; CHECK: ; %bb.0: ; %entry ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: s_mov_b32 s24, s33 +; CHECK-NEXT: s_mov_b32 s18, s33 ; CHECK-NEXT: s_mov_b32 s33, s32 ; CHECK-NEXT: s_xor_saveexec_b64 s[16:17], -1 ; CHECK-NEXT: buffer_store_dword v1, off, s[0:3], s33 ; 4-byte Folded Spill @@ -188,7 +185,6 @@ define hidden i32 @caller_save_vgpr_spill_fp_tail_call() #0 { ; CHECK-NEXT: s_mov_b64 s[20:21], s[0:1] ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] -; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: v_readlane_b32 s31, v1, 1 ; CHECK-NEXT: v_readlane_b32 s30, v1, 0 @@ -196,7 +192,7 @@ define hidden i32 @caller_save_vgpr_spill_fp_tail_call() #0 { ; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s33 ; 4-byte Folded Reload ; CHECK-NEXT: s_mov_b64 exec, s[4:5] ; CHECK-NEXT: s_add_i32 s32, s32, 0xfffffc00 -; CHECK-NEXT: s_mov_b32 s33, s24 +; CHECK-NEXT: s_mov_b32 s33, s18 ; CHECK-NEXT: s_waitcnt vmcnt(0) ; CHECK-NEXT: s_setpc_b64 s[30:31] entry: @@ -208,7 +204,7 @@ define hidden i32 @caller_save_vgpr_spill_fp() #0 { ; CHECK-LABEL: caller_save_vgpr_spill_fp: ; CHECK: ; %bb.0: ; %entry ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: s_mov_b32 s25, s33 +; CHECK-NEXT: s_mov_b32 s19, s33 ; CHECK-NEXT: s_mov_b32 s33, s32 ; CHECK-NEXT: s_xor_saveexec_b64 s[16:17], -1 ; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s33 ; 4-byte Folded Spill @@ -223,7 +219,6 @@ define hidden i32 @caller_save_vgpr_spill_fp() #0 { ; CHECK-NEXT: s_mov_b64 s[20:21], s[0:1] ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] -; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: v_readlane_b32 s31, v2, 1 ; CHECK-NEXT: v_readlane_b32 s30, v2, 0 @@ -231,7 +226,7 @@ define hidden i32 @caller_save_vgpr_spill_fp() #0 { ; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s33 ; 4-byte Folded Reload ; CHECK-NEXT: s_mov_b64 exec, s[4:5] ; CHECK-NEXT: s_add_i32 s32, s32, 0xfffffc00 -; CHECK-NEXT: s_mov_b32 s33, s25 +; CHECK-NEXT: s_mov_b32 s33, s19 ; CHECK-NEXT: s_waitcnt vmcnt(0) ; CHECK-NEXT: s_setpc_b64 s[30:31] entry: @@ -263,7 +258,6 @@ define protected amdgpu_kernel void @kernel() { ; CHECK-NEXT: ; implicit-def: $sgpr15 ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] -; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: s_endpgm entry: diff --git a/llvm/test/CodeGen/AMDGPU/no-source-locations-in-prologue.ll b/llvm/test/CodeGen/AMDGPU/no-source-locations-in-prologue.ll index 34e67d0993fb..9999cb9173b5 100644 --- a/llvm/test/CodeGen/AMDGPU/no-source-locations-in-prologue.ll +++ b/llvm/test/CodeGen/AMDGPU/no-source-locations-in-prologue.ll @@ -32,7 +32,6 @@ define hidden void @_ZL3barv() #0 !dbg !1644 { ; CHECK-NEXT: s_mov_b64 s[20:21], s[0:1] ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] -; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_waitcnt lgkmcnt(0) ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: .Ltmp1: diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spills-split-regalloc.ll b/llvm/test/CodeGen/AMDGPU/sgpr-spills-split-regalloc.ll index 764f4942cbd0..f523b4a2495f 100644 --- a/llvm/test/CodeGen/AMDGPU/sgpr-spills-split-regalloc.ll +++ b/llvm/test/CodeGen/AMDGPU/sgpr-spills-split-regalloc.ll @@ -16,7 +16,7 @@ define void @spill_sgpr_with_no_lower_vgpr_available() #0 { ; GCN-LABEL: spill_sgpr_with_no_lower_vgpr_available: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_mov_b32 s24, s33 +; GCN-NEXT: s_mov_b32 s18, s33 ; GCN-NEXT: s_mov_b32 s33, s32 ; GCN-NEXT: s_or_saveexec_b64 s[16:17], -1 ; GCN-NEXT: buffer_store_dword v255, off, s[0:3], s33 offset:448 ; 4-byte Folded Spill @@ -150,7 +150,6 @@ define void @spill_sgpr_with_no_lower_vgpr_available() #0 { ; GCN-NEXT: s_mov_b64 s[20:21], s[0:1] ; GCN-NEXT: s_mov_b64 s[0:1], s[20:21] ; GCN-NEXT: s_mov_b64 s[2:3], s[22:23] -; GCN-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GCN-NEXT: v_readlane_b32 s31, v255, 1 @@ -270,7 +269,7 @@ define void @spill_sgpr_with_no_lower_vgpr_available() #0 { ; GCN-NEXT: buffer_load_dword v255, off, s[0:3], s33 offset:448 ; 4-byte Folded Reload ; GCN-NEXT: s_mov_b64 exec, s[4:5] ; GCN-NEXT: s_add_i32 s32, s32, 0xffff8c00 -; GCN-NEXT: s_mov_b32 s33, s24 +; GCN-NEXT: s_mov_b32 s33, s18 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_setpc_b64 s[30:31] %alloca = alloca i32, align 4, addrspace(5) @@ -311,7 +310,7 @@ define void @spill_to_lowest_available_vgpr() #0 { ; GCN-LABEL: spill_to_lowest_available_vgpr: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_mov_b32 s24, s33 +; GCN-NEXT: s_mov_b32 s18, s33 ; GCN-NEXT: s_mov_b32 s33, s32 ; GCN-NEXT: s_or_saveexec_b64 s[16:17], -1 ; GCN-NEXT: buffer_store_dword v254, off, s[0:3], s33 offset:444 ; 4-byte Folded Spill @@ -444,7 +443,6 @@ define void @spill_to_lowest_available_vgpr() #0 { ; GCN-NEXT: s_mov_b64 s[20:21], s[0:1] ; GCN-NEXT: s_mov_b64 s[0:1], s[20:21] ; GCN-NEXT: s_mov_b64 s[2:3], s[22:23] -; GCN-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GCN-NEXT: v_readlane_b32 s31, v254, 1 @@ -563,7 +561,7 @@ define void @spill_to_lowest_available_vgpr() #0 { ; GCN-NEXT: buffer_load_dword v254, off, s[0:3], s33 offset:444 ; 4-byte Folded Reload ; GCN-NEXT: s_mov_b64 exec, s[4:5] ; GCN-NEXT: s_add_i32 s32, s32, 0xffff8c00 -; GCN-NEXT: s_mov_b32 s33, s24 +; GCN-NEXT: s_mov_b32 s33, s18 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_setpc_b64 s[30:31] %alloca = alloca i32, align 4, addrspace(5) @@ -1530,7 +1528,7 @@ define void @spill_sgpr_no_free_vgpr_ipra() #0 { ; GCN-LABEL: spill_sgpr_no_free_vgpr_ipra: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_mov_b32 s24, s33 +; GCN-NEXT: s_mov_b32 s18, s33 ; GCN-NEXT: s_mov_b32 s33, s32 ; GCN-NEXT: s_add_i32 s32, s32, 0x7400 ; GCN-NEXT: buffer_store_dword v40, off, s[0:3], s33 offset:444 ; 4-byte Folded Spill @@ -1668,7 +1666,6 @@ define void @spill_sgpr_no_free_vgpr_ipra() #0 { ; GCN-NEXT: s_mov_b64 s[20:21], s[0:1] ; GCN-NEXT: s_mov_b64 s[0:1], s[20:21] ; GCN-NEXT: s_mov_b64 s[2:3], s[22:23] -; GCN-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GCN-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GCN-NEXT: s_mov_b64 s[4:5], exec ; GCN-NEXT: s_mov_b64 exec, 1 @@ -1801,7 +1798,7 @@ define void @spill_sgpr_no_free_vgpr_ipra() #0 { ; GCN-NEXT: buffer_load_dword v41, off, s[0:3], s33 offset:440 ; 4-byte Folded Reload ; GCN-NEXT: buffer_load_dword v40, off, s[0:3], s33 offset:444 ; 4-byte Folded Reload ; GCN-NEXT: s_add_i32 s32, s32, 0xffff8c00 -; GCN-NEXT: s_mov_b32 s33, s24 +; GCN-NEXT: s_mov_b32 s33, s18 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_setpc_b64 s[30:31] call void @child_function_ipra() diff --git a/llvm/test/CodeGen/AMDGPU/stacksave_stackrestore.ll b/llvm/test/CodeGen/AMDGPU/stacksave_stackrestore.ll index 33b5d6c6850b..8c5b89429bcc 100644 --- a/llvm/test/CodeGen/AMDGPU/stacksave_stackrestore.ll +++ b/llvm/test/CodeGen/AMDGPU/stacksave_stackrestore.ll @@ -916,13 +916,13 @@ define amdgpu_kernel void @kernel_stacksave_stackrestore_call_with_stack_objects ; WAVE32-O0-LABEL: kernel_stacksave_stackrestore_call_with_stack_objects: ; WAVE32-O0: ; %bb.0: ; WAVE32-O0-NEXT: s_mov_b32 s32, 0x1200 -; WAVE32-O0-NEXT: s_getpc_b64 s[24:25] -; WAVE32-O0-NEXT: s_mov_b32 s24, s0 -; WAVE32-O0-NEXT: s_load_dwordx4 s[24:27], s[24:25], 0x0 +; WAVE32-O0-NEXT: s_getpc_b64 s[20:21] +; WAVE32-O0-NEXT: s_mov_b32 s20, s0 +; WAVE32-O0-NEXT: s_load_dwordx4 s[20:23], s[20:21], 0x0 ; WAVE32-O0-NEXT: s_waitcnt lgkmcnt(0) -; WAVE32-O0-NEXT: s_bitset0_b32 s27, 21 -; WAVE32-O0-NEXT: s_add_u32 s24, s24, s9 -; WAVE32-O0-NEXT: s_addc_u32 s25, s25, 0 +; WAVE32-O0-NEXT: s_bitset0_b32 s23, 21 +; WAVE32-O0-NEXT: s_add_u32 s20, s20, s9 +; WAVE32-O0-NEXT: s_addc_u32 s21, s21, 0 ; WAVE32-O0-NEXT: ; implicit-def: $vgpr3 : SGPR spill to VGPR lane ; WAVE32-O0-NEXT: s_mov_b32 s14, s8 ; WAVE32-O0-NEXT: s_mov_b32 s13, s7 @@ -934,17 +934,17 @@ define amdgpu_kernel void @kernel_stacksave_stackrestore_call_with_stack_objects ; WAVE32-O0-NEXT: v_writelane_b32 v3, s0, 0 ; WAVE32-O0-NEXT: s_lshr_b32 s0, s0, 5 ; WAVE32-O0-NEXT: v_writelane_b32 v3, s0, 1 -; WAVE32-O0-NEXT: s_or_saveexec_b32 s20, -1 -; WAVE32-O0-NEXT: buffer_store_dword v3, off, s[24:27], 0 offset:128 ; 4-byte Folded Spill -; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s20 +; WAVE32-O0-NEXT: s_or_saveexec_b32 s19, -1 +; WAVE32-O0-NEXT: buffer_store_dword v3, off, s[20:23], 0 offset:128 ; 4-byte Folded Spill +; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s19 ; WAVE32-O0-NEXT: v_mov_b32_e32 v3, 42 -; WAVE32-O0-NEXT: buffer_store_dword v3, off, s[24:27], 0 +; WAVE32-O0-NEXT: buffer_store_dword v3, off, s[20:23], 0 ; WAVE32-O0-NEXT: s_waitcnt_vscnt null, 0x0 -; WAVE32-O0-NEXT: s_mov_b64 s[0:1], s[24:25] -; WAVE32-O0-NEXT: s_mov_b64 s[2:3], s[26:27] +; WAVE32-O0-NEXT: s_mov_b64 s[0:1], s[20:21] +; WAVE32-O0-NEXT: s_mov_b64 s[2:3], s[22:23] ; WAVE32-O0-NEXT: s_mov_b32 s6, s32 ; WAVE32-O0-NEXT: v_mov_b32_e32 v3, 17 -; WAVE32-O0-NEXT: buffer_store_dword v3, off, s[24:27], s6 offset:4 +; WAVE32-O0-NEXT: buffer_store_dword v3, off, s[20:23], s6 offset:4 ; WAVE32-O0-NEXT: s_mov_b32 s6, stack_passed_argument@abs32@hi ; WAVE32-O0-NEXT: s_mov_b32 s16, stack_passed_argument@abs32@lo ; WAVE32-O0-NEXT: ; kill: def $sgpr16 killed $sgpr16 def $sgpr16_sgpr17 @@ -1018,11 +1018,10 @@ define amdgpu_kernel void @kernel_stacksave_stackrestore_call_with_stack_objects ; WAVE32-O0-NEXT: v_mov_b32_e32 v29, s18 ; WAVE32-O0-NEXT: ; implicit-def: $sgpr18 ; WAVE32-O0-NEXT: v_mov_b32_e32 v30, s18 -; WAVE32-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; WAVE32-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] -; WAVE32-O0-NEXT: s_or_saveexec_b32 s20, -1 -; WAVE32-O0-NEXT: buffer_load_dword v0, off, s[24:27], 0 offset:128 ; 4-byte Folded Reload -; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s20 +; WAVE32-O0-NEXT: s_or_saveexec_b32 s19, -1 +; WAVE32-O0-NEXT: buffer_load_dword v0, off, s[20:23], 0 offset:128 ; 4-byte Folded Reload +; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s19 ; WAVE32-O0-NEXT: s_waitcnt vmcnt(0) ; WAVE32-O0-NEXT: v_readlane_b32 s1, v0, 1 ; WAVE32-O0-NEXT: v_readlane_b32 s0, v0, 0 @@ -1137,7 +1136,6 @@ define amdgpu_kernel void @kernel_stacksave_stackrestore_call_with_stack_objects ; WAVE64-O0-NEXT: v_mov_b32_e32 v29, s18 ; WAVE64-O0-NEXT: ; implicit-def: $sgpr18 ; WAVE64-O0-NEXT: v_mov_b32_e32 v30, s18 -; WAVE64-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; WAVE64-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] ; WAVE64-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 ; WAVE64-O0-NEXT: buffer_load_dword v0, off, s[24:27], 0 offset:128 ; 4-byte Folded Reload @@ -1155,13 +1153,13 @@ define amdgpu_kernel void @kernel_stacksave_stackrestore_call_with_stack_objects ; WAVE32-WWM-PREALLOC-LABEL: kernel_stacksave_stackrestore_call_with_stack_objects: ; WAVE32-WWM-PREALLOC: ; %bb.0: ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s32, 0x1200 -; WAVE32-WWM-PREALLOC-NEXT: s_getpc_b64 s[24:25] -; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s24, s0 -; WAVE32-WWM-PREALLOC-NEXT: s_load_dwordx4 s[24:27], s[24:25], 0x0 +; WAVE32-WWM-PREALLOC-NEXT: s_getpc_b64 s[20:21] +; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s20, s0 +; WAVE32-WWM-PREALLOC-NEXT: s_load_dwordx4 s[20:23], s[20:21], 0x0 ; WAVE32-WWM-PREALLOC-NEXT: s_waitcnt lgkmcnt(0) -; WAVE32-WWM-PREALLOC-NEXT: s_bitset0_b32 s27, 21 -; WAVE32-WWM-PREALLOC-NEXT: s_add_u32 s24, s24, s9 -; WAVE32-WWM-PREALLOC-NEXT: s_addc_u32 s25, s25, 0 +; WAVE32-WWM-PREALLOC-NEXT: s_bitset0_b32 s23, 21 +; WAVE32-WWM-PREALLOC-NEXT: s_add_u32 s20, s20, s9 +; WAVE32-WWM-PREALLOC-NEXT: s_addc_u32 s21, s21, 0 ; WAVE32-WWM-PREALLOC-NEXT: ; implicit-def: $vgpr32 : SGPR spill to VGPR lane ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s14, s8 ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s13, s7 @@ -1174,13 +1172,13 @@ define amdgpu_kernel void @kernel_stacksave_stackrestore_call_with_stack_objects ; WAVE32-WWM-PREALLOC-NEXT: s_lshr_b32 s0, s0, 5 ; WAVE32-WWM-PREALLOC-NEXT: v_writelane_b32 v32, s0, 1 ; WAVE32-WWM-PREALLOC-NEXT: v_mov_b32_e32 v3, 42 -; WAVE32-WWM-PREALLOC-NEXT: buffer_store_dword v3, off, s[24:27], 0 +; WAVE32-WWM-PREALLOC-NEXT: buffer_store_dword v3, off, s[20:23], 0 ; WAVE32-WWM-PREALLOC-NEXT: s_waitcnt_vscnt null, 0x0 -; WAVE32-WWM-PREALLOC-NEXT: s_mov_b64 s[0:1], s[24:25] -; WAVE32-WWM-PREALLOC-NEXT: s_mov_b64 s[2:3], s[26:27] +; WAVE32-WWM-PREALLOC-NEXT: s_mov_b64 s[0:1], s[20:21] +; WAVE32-WWM-PREALLOC-NEXT: s_mov_b64 s[2:3], s[22:23] ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s6, s32 ; WAVE32-WWM-PREALLOC-NEXT: v_mov_b32_e32 v3, 17 -; WAVE32-WWM-PREALLOC-NEXT: buffer_store_dword v3, off, s[24:27], s6 offset:4 +; WAVE32-WWM-PREALLOC-NEXT: buffer_store_dword v3, off, s[20:23], s6 offset:4 ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s6, stack_passed_argument@abs32@hi ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s16, stack_passed_argument@abs32@lo ; WAVE32-WWM-PREALLOC-NEXT: ; kill: def $sgpr16 killed $sgpr16 def $sgpr16_sgpr17 @@ -1254,7 +1252,6 @@ define amdgpu_kernel void @kernel_stacksave_stackrestore_call_with_stack_objects ; WAVE32-WWM-PREALLOC-NEXT: v_mov_b32_e32 v29, s18 ; WAVE32-WWM-PREALLOC-NEXT: ; implicit-def: $sgpr18 ; WAVE32-WWM-PREALLOC-NEXT: v_mov_b32_e32 v30, s18 -; WAVE32-WWM-PREALLOC-NEXT: ; implicit-def: $sgpr18_sgpr19 ; WAVE32-WWM-PREALLOC-NEXT: s_swappc_b64 s[30:31], s[16:17] ; WAVE32-WWM-PREALLOC-NEXT: v_readlane_b32 s1, v32, 1 ; WAVE32-WWM-PREALLOC-NEXT: v_readlane_b32 s0, v32, 0 @@ -1347,7 +1344,7 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE32-O0-LABEL: func_stacksave_stackrestore_call_with_stack_objects: ; WAVE32-O0: ; %bb.0: ; WAVE32-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; WAVE32-O0-NEXT: s_mov_b32 s26, s33 +; WAVE32-O0-NEXT: s_mov_b32 s25, s33 ; WAVE32-O0-NEXT: s_mov_b32 s33, s32 ; WAVE32-O0-NEXT: s_xor_saveexec_b32 s16, -1 ; WAVE32-O0-NEXT: buffer_store_dword v32, off, s[0:3], s33 offset:128 ; 4-byte Folded Spill @@ -1361,9 +1358,9 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE32-O0-NEXT: v_writelane_b32 v0, s16, 0 ; WAVE32-O0-NEXT: s_lshr_b32 s16, s16, 5 ; WAVE32-O0-NEXT: v_writelane_b32 v0, s16, 1 -; WAVE32-O0-NEXT: s_or_saveexec_b32 s25, -1 +; WAVE32-O0-NEXT: s_or_saveexec_b32 s24, -1 ; WAVE32-O0-NEXT: buffer_store_dword v0, off, s[0:3], s33 offset:132 ; 4-byte Folded Spill -; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s25 +; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s24 ; WAVE32-O0-NEXT: v_mov_b32_e32 v0, 42 ; WAVE32-O0-NEXT: buffer_store_dword v0, off, s[0:3], s33 ; WAVE32-O0-NEXT: s_waitcnt_vscnt null, 0x0 @@ -1440,11 +1437,10 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE32-O0-NEXT: v_mov_b32_e32 v29, s18 ; WAVE32-O0-NEXT: ; implicit-def: $sgpr18 ; WAVE32-O0-NEXT: v_mov_b32_e32 v30, s18 -; WAVE32-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; WAVE32-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] -; WAVE32-O0-NEXT: s_or_saveexec_b32 s25, -1 +; WAVE32-O0-NEXT: s_or_saveexec_b32 s24, -1 ; WAVE32-O0-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:132 ; 4-byte Folded Reload -; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s25 +; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s24 ; WAVE32-O0-NEXT: s_waitcnt vmcnt(0) ; WAVE32-O0-NEXT: v_readlane_b32 s5, v0, 1 ; WAVE32-O0-NEXT: v_readlane_b32 s4, v0, 0 @@ -1460,14 +1456,14 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE32-O0-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:136 ; 4-byte Folded Reload ; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s4 ; WAVE32-O0-NEXT: s_add_i32 s32, s32, 0xffffee00 -; WAVE32-O0-NEXT: s_mov_b32 s33, s26 +; WAVE32-O0-NEXT: s_mov_b32 s33, s25 ; WAVE32-O0-NEXT: s_waitcnt vmcnt(0) ; WAVE32-O0-NEXT: s_setpc_b64 s[30:31] ; ; WAVE64-O0-LABEL: func_stacksave_stackrestore_call_with_stack_objects: ; WAVE64-O0: ; %bb.0: ; WAVE64-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; WAVE64-O0-NEXT: s_mov_b32 s28, s33 +; WAVE64-O0-NEXT: s_mov_b32 s19, s33 ; WAVE64-O0-NEXT: s_mov_b32 s33, s32 ; WAVE64-O0-NEXT: s_xor_saveexec_b64 s[16:17], -1 ; WAVE64-O0-NEXT: buffer_store_dword v32, off, s[0:3], s33 offset:128 ; 4-byte Folded Spill @@ -1560,7 +1556,6 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE64-O0-NEXT: v_mov_b32_e32 v29, s18 ; WAVE64-O0-NEXT: ; implicit-def: $sgpr18 ; WAVE64-O0-NEXT: v_mov_b32_e32 v30, s18 -; WAVE64-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; WAVE64-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] ; WAVE64-O0-NEXT: s_or_saveexec_b64 s[26:27], -1 ; WAVE64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:132 ; 4-byte Folded Reload @@ -1580,14 +1575,14 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:136 ; 4-byte Folded Reload ; WAVE64-O0-NEXT: s_mov_b64 exec, s[4:5] ; WAVE64-O0-NEXT: s_add_i32 s32, s32, 0xffffdc00 -; WAVE64-O0-NEXT: s_mov_b32 s33, s28 +; WAVE64-O0-NEXT: s_mov_b32 s33, s19 ; WAVE64-O0-NEXT: s_waitcnt vmcnt(0) ; WAVE64-O0-NEXT: s_setpc_b64 s[30:31] ; ; WAVE32-WWM-PREALLOC-LABEL: func_stacksave_stackrestore_call_with_stack_objects: ; WAVE32-WWM-PREALLOC: ; %bb.0: ; WAVE32-WWM-PREALLOC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s25, s33 +; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s24, s33 ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s33, s32 ; WAVE32-WWM-PREALLOC-NEXT: s_xor_saveexec_b32 s16, -1 ; WAVE32-WWM-PREALLOC-NEXT: buffer_store_dword v33, off, s[0:3], s33 offset:128 ; 4-byte Folded Spill @@ -1677,7 +1672,6 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE32-WWM-PREALLOC-NEXT: v_mov_b32_e32 v29, s18 ; WAVE32-WWM-PREALLOC-NEXT: ; implicit-def: $sgpr18 ; WAVE32-WWM-PREALLOC-NEXT: v_mov_b32_e32 v30, s18 -; WAVE32-WWM-PREALLOC-NEXT: ; implicit-def: $sgpr18_sgpr19 ; WAVE32-WWM-PREALLOC-NEXT: s_swappc_b64 s[30:31], s[16:17] ; WAVE32-WWM-PREALLOC-NEXT: v_readlane_b32 s5, v32, 1 ; WAVE32-WWM-PREALLOC-NEXT: v_readlane_b32 s4, v32, 0 @@ -1693,7 +1687,7 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE32-WWM-PREALLOC-NEXT: buffer_load_dword v32, off, s[0:3], s33 offset:132 ; 4-byte Folded Reload ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 exec_lo, s4 ; WAVE32-WWM-PREALLOC-NEXT: s_add_i32 s32, s32, 0xffffee00 -; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s33, s25 +; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s33, s24 ; WAVE32-WWM-PREALLOC-NEXT: s_waitcnt vmcnt(0) ; WAVE32-WWM-PREALLOC-NEXT: s_setpc_b64 s[30:31] %alloca = alloca [32 x i32], addrspace(5) diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll index d2364a61ed68..bfc249e9081d 100644 --- a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll +++ b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll @@ -233,10 +233,10 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun ; SI-NEXT: bb.1.Flow: ; SI-NEXT: successors: %bb.2(0x40000000), %bb.10(0x40000000) ; SI-NEXT: {{ $}} - ; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %49:vgpr_32, %bb.0, %4, %bb.9 - ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY4]], %bb.0, undef %51:vgpr_32, %bb.9 - ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %53:vgpr_32, %bb.9 - ; SI-NEXT: [[PHI3:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %55:vgpr_32, %bb.9 + ; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %47:vgpr_32, %bb.0, %4, %bb.9 + ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY4]], %bb.0, undef %49:vgpr_32, %bb.9 + ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %51:vgpr_32, %bb.9 + ; SI-NEXT: [[PHI3:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %53:vgpr_32, %bb.9 ; SI-NEXT: [[SI_ELSE:%[0-9]+]]:sreg_32 = SI_ELSE killed [[SI_IF]], %bb.10, implicit-def dead $exec, implicit-def dead $scc, implicit $exec ; SI-NEXT: S_BRANCH %bb.2 ; SI-NEXT: {{ $}} @@ -249,8 +249,8 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun ; SI-NEXT: bb.3: ; SI-NEXT: successors: %bb.4(0x80000000) ; SI-NEXT: {{ $}} - ; SI-NEXT: [[PHI4:%[0-9]+]]:vreg_64 = PHI undef %57:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2 - ; SI-NEXT: [[PHI5:%[0-9]+]]:vgpr_32 = PHI undef %59:vgpr_32, %bb.4, [[PHI1]], %bb.2 + ; SI-NEXT: [[PHI4:%[0-9]+]]:vreg_64 = PHI undef %55:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2 + ; SI-NEXT: [[PHI5:%[0-9]+]]:vgpr_32 = PHI undef %57:vgpr_32, %bb.4, [[PHI1]], %bb.2 ; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI4]].sub0, implicit $exec ; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI4]].sub1, implicit $exec ; SI-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_1]], %subreg.sub1 @@ -286,8 +286,8 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun ; SI-NEXT: bb.7: ; SI-NEXT: successors: %bb.8(0x80000000) ; SI-NEXT: {{ $}} - ; SI-NEXT: [[PHI6:%[0-9]+]]:vreg_64 = PHI undef %61:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6 - ; SI-NEXT: [[PHI7:%[0-9]+]]:vgpr_32 = PHI undef %63:vgpr_32, %bb.8, [[COPY4]], %bb.6 + ; SI-NEXT: [[PHI6:%[0-9]+]]:vreg_64 = PHI undef %59:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6 + ; SI-NEXT: [[PHI7:%[0-9]+]]:vgpr_32 = PHI undef %61:vgpr_32, %bb.8, [[COPY4]], %bb.6 ; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI6]].sub0, implicit $exec ; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI6]].sub1, implicit $exec ; SI-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_2]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_3]], %subreg.sub1 @@ -356,9 +356,9 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e ; SI-NEXT: bb.1.Flow: ; SI-NEXT: successors: %bb.2(0x40000000), %bb.10(0x40000000) ; SI-NEXT: {{ $}} - ; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %50:vgpr_32, %bb.0, %4, %bb.9 - ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %52:vgpr_32, %bb.9 - ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %54:vgpr_32, %bb.9 + ; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %48:vgpr_32, %bb.0, %4, %bb.9 + ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %50:vgpr_32, %bb.9 + ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %52:vgpr_32, %bb.9 ; SI-NEXT: [[SI_ELSE:%[0-9]+]]:sreg_32 = SI_ELSE killed [[SI_IF]], %bb.10, implicit-def dead $exec, implicit-def dead $scc, implicit $exec ; SI-NEXT: S_BRANCH %bb.2 ; SI-NEXT: {{ $}} @@ -371,7 +371,7 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e ; SI-NEXT: bb.3: ; SI-NEXT: successors: %bb.4(0x80000000) ; SI-NEXT: {{ $}} - ; SI-NEXT: [[PHI3:%[0-9]+]]:vreg_64 = PHI undef %56:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2 + ; SI-NEXT: [[PHI3:%[0-9]+]]:vreg_64 = PHI undef %54:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2 ; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI3]].sub0, implicit $exec ; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI3]].sub1, implicit $exec ; SI-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_1]], %subreg.sub1 @@ -407,7 +407,7 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e ; SI-NEXT: bb.7: ; SI-NEXT: successors: %bb.8(0x80000000) ; SI-NEXT: {{ $}} - ; SI-NEXT: [[PHI4:%[0-9]+]]:vreg_64 = PHI undef %58:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6 + ; SI-NEXT: [[PHI4:%[0-9]+]]:vreg_64 = PHI undef %56:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6 ; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI4]].sub0, implicit $exec ; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI4]].sub1, implicit $exec ; SI-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_2]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_3]], %subreg.sub1 diff --git a/llvm/test/CodeGen/AMDGPU/vgpr_constant_to_sgpr.ll b/llvm/test/CodeGen/AMDGPU/vgpr_constant_to_sgpr.ll index 364ce82b2e99..7840559c78eb 100644 --- a/llvm/test/CodeGen/AMDGPU/vgpr_constant_to_sgpr.ll +++ b/llvm/test/CodeGen/AMDGPU/vgpr_constant_to_sgpr.ll @@ -47,7 +47,6 @@ define protected amdgpu_kernel void @kern(ptr %addr) !llvm.amdgcn.lds.kernel.id ; CHECK-NEXT: s_mov_b32 s15, 42 ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] -; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_waitcnt lgkmcnt(0) ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: s_endpgm diff --git a/llvm/test/CodeGen/AMDGPU/whole-wave-register-spill.ll b/llvm/test/CodeGen/AMDGPU/whole-wave-register-spill.ll index 3a33194f17c8..7eabe982ff2b 100644 --- a/llvm/test/CodeGen/AMDGPU/whole-wave-register-spill.ll +++ b/llvm/test/CodeGen/AMDGPU/whole-wave-register-spill.ll @@ -101,7 +101,6 @@ define void @test() #0 { ; GCN-O0-NEXT: s_mov_b64 s[20:21], s[0:1] ; GCN-O0-NEXT: s_mov_b64 s[0:1], s[20:21] ; GCN-O0-NEXT: s_mov_b64 s[2:3], s[22:23] -; GCN-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GCN-O0-NEXT: s_waitcnt lgkmcnt(0) ; GCN-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GCN-O0-NEXT: s_or_saveexec_b64 s[28:29], -1 diff --git a/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll b/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll index 11f6a2960776..e79cb66dcd77 100644 --- a/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll +++ b/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll @@ -406,7 +406,6 @@ define amdgpu_gfx void @strict_wwm_call(ptr addrspace(8) inreg %tmp14, i32 inreg ; GFX9-O0-NEXT: s_mov_b64 s[0:1], s[44:45] ; GFX9-O0-NEXT: s_mov_b64 s[2:3], s[46:47] ; GFX9-O0-NEXT: v_mov_b32_e32 v0, v2 -; GFX9-O0-NEXT: ; implicit-def: $sgpr44_sgpr45 ; GFX9-O0-NEXT: s_swappc_b64 s[30:31], s[42:43] ; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0 ; GFX9-O0-NEXT: v_add_u32_e64 v1, v1, v2 @@ -633,7 +632,6 @@ define amdgpu_gfx void @strict_wwm_call_i64(ptr addrspace(8) inreg %tmp14, i64 i ; GFX9-O0-NEXT: s_mov_b64 s[2:3], s[38:39] ; GFX9-O0-NEXT: v_mov_b32_e32 v0, v2 ; GFX9-O0-NEXT: v_mov_b32_e32 v1, v3 -; GFX9-O0-NEXT: ; implicit-def: $sgpr36_sgpr37 ; GFX9-O0-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-O0-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX9-O0-NEXT: s_or_saveexec_b64 s[46:47], -1 diff --git a/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll b/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll index 6ac61410a0e7..47c976d2a5c3 100644 --- a/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll +++ b/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll @@ -413,7 +413,6 @@ define amdgpu_kernel void @call(ptr addrspace(8) inreg %tmp14, i32 inreg %arg) { ; GFX9-O0-NEXT: ; implicit-def: $sgpr15 ; GFX9-O0-NEXT: v_mov_b32_e32 v31, v3 ; GFX9-O0-NEXT: v_mov_b32_e32 v0, v6 -; GFX9-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GFX9-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 ; GFX9-O0-NEXT: buffer_load_dword v1, off, s[24:27], 0 ; 4-byte Folded Reload @@ -657,7 +656,6 @@ define amdgpu_kernel void @call_i64(ptr addrspace(8) inreg %tmp14, i64 inreg %ar ; GFX9-O0-NEXT: v_mov_b32_e32 v31, v3 ; GFX9-O0-NEXT: v_mov_b32_e32 v0, v6 ; GFX9-O0-NEXT: v_mov_b32_e32 v1, v7 -; GFX9-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GFX9-O0-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 @@ -1285,7 +1283,6 @@ define amdgpu_kernel void @strict_wwm_call(ptr addrspace(8) inreg %tmp14, i32 in ; GFX9-O0-NEXT: ; implicit-def: $sgpr15 ; GFX9-O0-NEXT: v_mov_b32_e32 v31, v3 ; GFX9-O0-NEXT: v_mov_b32_e32 v0, v6 -; GFX9-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GFX9-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 ; GFX9-O0-NEXT: buffer_load_dword v1, off, s[24:27], 0 ; 4-byte Folded Reload @@ -1529,7 +1526,6 @@ define amdgpu_kernel void @strict_wwm_call_i64(ptr addrspace(8) inreg %tmp14, i6 ; GFX9-O0-NEXT: v_mov_b32_e32 v31, v3 ; GFX9-O0-NEXT: v_mov_b32_e32 v0, v6 ; GFX9-O0-NEXT: v_mov_b32_e32 v1, v7 -; GFX9-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GFX9-O0-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 diff --git a/llvm/test/CodeGen/PowerPC/fmf-propagation.ll b/llvm/test/CodeGen/PowerPC/fmf-propagation.ll index 4e72a5ac5ede..58b3ee485ea4 100644 --- a/llvm/test/CodeGen/PowerPC/fmf-propagation.ll +++ b/llvm/test/CodeGen/PowerPC/fmf-propagation.ll @@ -577,15 +577,15 @@ define double @fcmp_nnan(double %a, double %y, double %z) { ; FP library calls can have fast-math-flags. ; FMFDEBUG-LABEL: Optimized lowered selection DAG: %bb.0 'log2_approx:' -; FMFDEBUG: ch,glue = PPCISD::CALL_NOP {{t[0-9]+}}, TargetGlobalAddress:i64 -; FMFDEBUG: ch,glue = callseq_end [[T15:t[0-9]+]], TargetConstant:i64<32>, TargetConstant:i64<0>, [[T15]]:1 -; FMFDEBUG: f64,ch,glue = CopyFromReg [[T16:t[0-9]+]], Register:f64 $f1, [[T16]]:1 +; FMFDEBUG: ch,glue = PPCISD::CALL_NOP t11, TargetGlobalAddress:i64 +; FMFDEBUG: ch,glue = callseq_end t15, TargetConstant:i64<32>, TargetConstant:i64<0>, t15:1 +; FMFDEBUG: f64,ch,glue = CopyFromReg t16, Register:f64 $f1, t16:1 ; FMFDEBUG: Type-legalized selection DAG: %bb.0 'log2_approx:' ; GLOBALDEBUG-LABEL: Optimized lowered selection DAG: %bb.0 'log2_approx:' -; GLOBALDEBUG: ch,glue = PPCISD::CALL_NOP {{t[0-9]+}}, TargetGlobalAddress:i64 -; GLOBALDEBUG: ch,glue = callseq_end [[T15:t[0-9]+]], TargetConstant:i64<32>, TargetConstant:i64<0>, [[T15]]:1 -; GLOBALDEBUG: f64,ch,glue = CopyFromReg [[T16:t[0-9]+]], Register:f64 $f1, [[T16]]:1 +; GLOBALDEBUG: ch,glue = PPCISD::CALL_NOP t11, TargetGlobalAddress:i64 +; GLOBALDEBUG: ch,glue = callseq_end t15, TargetConstant:i64<32>, TargetConstant:i64<0>, t15:1 +; GLOBALDEBUG: f64,ch,glue = CopyFromReg t16, Register:f64 $f1, t16:1 ; GLOBALDEBUG: Type-legalized selection DAG: %bb.0 'log2_approx:' declare double @log2(double) diff --git a/llvm/test/MachineVerifier/convergencectrl/AMDGPU/basic.mir b/llvm/test/MachineVerifier/convergencectrl/AMDGPU/basic.mir deleted file mode 100644 index 94d0ddad2594..000000000000 --- a/llvm/test/MachineVerifier/convergencectrl/AMDGPU/basic.mir +++ /dev/null @@ -1,37 +0,0 @@ -# RUN: not --crash llc -march=amdgcn -run-pass=none -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s ---- -name: basic -tracksRegLiveness: true -body: | - bb.0: - successors: %bb.1, %bb.2; - %0:sgpr_64 = CONVERGENCECTRL_ANCHOR - ; CHECK: Entry intrinsic cannot be preceded by a convergent operation in the same basic block. - ; CHECK: CONVERGENCECTRL_ENTRY - %1:sgpr_64 = CONVERGENCECTRL_ENTRY - ; CHECK: Loop intrinsic cannot be preceded by a convergent operation in the same basic block. - ; CHECK: CONVERGENCECTRL_LOOP - %2:sgpr_64 = CONVERGENCECTRL_LOOP %0:sgpr_64 - S_CBRANCH_EXECZ %bb.1, implicit $exec - S_BRANCH %bb.2 - - bb.1: - successors: %bb.2; - ; CHECK: Entry intrinsic can occur only in the entry block. - ; CHECK: CONVERGENCECTRL_ENTRY - %5:sgpr_64 = CONVERGENCECTRL_ENTRY - - bb.2: - ; CHECK: Convergence control tokens can only be used by convergent operations. - ; CHECK: G_PHI - %6:sgpr_64 = G_PHI %0:sgpr_64, %bb.0, %0:sgpr_64, %bb.1 - %7:sgpr_64 = CONVERGENCECTRL_ANCHOR - %8:sgpr_64 = IMPLICIT_DEF - %4:sgpr_64 = SI_CALL %8:sgpr_64, 1, implicit %7:sgpr_64 - ; CHECK: An operation can use at most one convergence control token. - ; CHECK: SI_CALL %{{[0-9]}}:sgpr_64, 2 - %9:sgpr_64 = SI_CALL %8:sgpr_64, 2, implicit %7:sgpr_64, implicit %7:sgpr_64 - ; CHECK: Cannot mix controlled and uncontrolled convergence in the same function. - ; CHECK: SI_CALL %{{[0-9]}}:sgpr_64, 3 - %10:sgpr_64 = SI_CALL %8:sgpr_64, 3 -... diff --git a/llvm/test/MachineVerifier/convergencectrl/AMDGPU/cycles.mir b/llvm/test/MachineVerifier/convergencectrl/AMDGPU/cycles.mir deleted file mode 100644 index 87cf3e604929..000000000000 --- a/llvm/test/MachineVerifier/convergencectrl/AMDGPU/cycles.mir +++ /dev/null @@ -1,52 +0,0 @@ -# RUN: not --crash llc -march=amdgcn -run-pass=none -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s ---- -name: cycles -body: | - bb.0: - %0:sgpr_64 = CONVERGENCECTRL_ANCHOR - %1:sgpr_64 = IMPLICIT_DEF - S_CBRANCH_EXECZ %bb.9, implicit $exec - S_BRANCH %bb.1 - - bb.1: - S_CBRANCH_EXECZ %bb.8, implicit $exec - S_BRANCH %bb.5 - - bb.2: - S_CBRANCH_EXECZ %bb.3, implicit $exec - S_BRANCH %bb.4 - - bb.3: - ; CHECK: Cycle heart must dominate all blocks in the cycle. - ; Irreducible cycle: entries(bb.4 bb.3) - %3:sgpr_64 = CONVERGENCECTRL_LOOP %0:sgpr_64 - S_BRANCH %bb.4 - - bb.4: - S_BRANCH %bb.3 - - bb.5: - S_CBRANCH_EXECZ %bb.6, implicit $exec - S_BRANCH %bb.2 - - bb.6: - S_BRANCH %bb.7 - - bb.7: - ; CHECK: Cycle heart must dominate all blocks in the cycle. - ; Reducible cycle: entries(bb.6) bb.7 - %4:sgpr_64 = CONVERGENCECTRL_LOOP %0:sgpr_64 - S_BRANCH %bb.6 - - bb.8: - ; CHECK: Two static convergence token uses in a cycle that does not contain either token's definition. - %5:sgpr_64 = CONVERGENCECTRL_LOOP %0:sgpr_64 - %6:sgpr_64 = CONVERGENCECTRL_LOOP %0:sgpr_64 - S_BRANCH %bb.8 - - bb.9: - ; CHECK: Convergence token used by an instruction other than llvm.experimental.convergence.loop in a cycle that does not contain the token's definition. - %7:sgpr_64 = G_SI_CALL %1:sgpr_64, 3, implicit %0:sgpr_64 - S_BRANCH %bb.9 - -... diff --git a/llvm/test/MachineVerifier/convergencectrl/AMDGPU/lit.local.cfg b/llvm/test/MachineVerifier/convergencectrl/AMDGPU/lit.local.cfg deleted file mode 100644 index 7c492428aec7..000000000000 --- a/llvm/test/MachineVerifier/convergencectrl/AMDGPU/lit.local.cfg +++ /dev/null @@ -1,2 +0,0 @@ -if not "AMDGPU" in config.root.targets: - config.unsupported = True diff --git a/llvm/test/MachineVerifier/convergencectrl/AMDGPU/mixed2.mir b/llvm/test/MachineVerifier/convergencectrl/AMDGPU/mixed2.mir deleted file mode 100644 index c70a48bf2130..000000000000 --- a/llvm/test/MachineVerifier/convergencectrl/AMDGPU/mixed2.mir +++ /dev/null @@ -1,15 +0,0 @@ -# RUN: not --crash llc -march=amdgcn -run-pass=none -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s ---- -name: mixed2 -body: | - bb.0: - %0:sgpr_64 = IMPLICIT_DEF - %1:sgpr_64 = SI_CALL %0, 1 - ; CHECK: Cannot mix controlled and uncontrolled convergence in the same function. - ; CHECK: CONVERGENCECTRL_ANCHOR - %2:sgpr_64 = CONVERGENCECTRL_ANCHOR - ; CHECK: Cannot mix controlled and uncontrolled convergence in the same function. - ; CHECK: SI_CALL %{{[0-9]}}:sgpr_64, 2 - %3:sgpr_64 = SI_CALL %0, 2, implicit %2:sgpr_64 - -... diff --git a/llvm/test/MachineVerifier/convergencectrl/AMDGPU/region-nesting.mir b/llvm/test/MachineVerifier/convergencectrl/AMDGPU/region-nesting.mir deleted file mode 100644 index 9e869acb3e93..000000000000 --- a/llvm/test/MachineVerifier/convergencectrl/AMDGPU/region-nesting.mir +++ /dev/null @@ -1,24 +0,0 @@ -# RUN: not --crash llc -march=amdgcn -run-pass=none -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s ---- -name: region_nesting -body: | - bb.0: - %0:sgpr_64 = CONVERGENCECTRL_ANCHOR - %1:sgpr_64 = CONVERGENCECTRL_ANCHOR - %2:sgpr_64 = IMPLICIT_DEF - %3:sgpr_64 = SI_CALL %2, 1, implicit %0:sgpr_64 - ; CHECK: Convergence region is not well-nested. - ; CHECK: SI_CALL %{{[0-9]}}:sgpr_64, 2 - %4:sgpr_64 = SI_CALL %2, 2, implicit %1:sgpr_64 - S_CBRANCH_EXECZ %bb.1, implicit $exec - S_BRANCH %bb.2 - - bb.1: - %5:sgpr_64 = SI_CALL %2, 3, implicit %0:sgpr_64 - - bb.2: - ; CHECK: Convergence region is not well-nested. - ; CHECK: SI_CALL %{{[0-9]}}:sgpr_64, 4 - %6:sgpr_64 = SI_CALL %2, 4, implicit %1:sgpr_64 - -... diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td index 40a831d7e9e8..622d1df7b381 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td @@ -28,7 +28,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(69), GIMT_Encode2(186), /*)*//*default:*//*Label 2*/ GIMT_Encode4(562), +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(65), GIMT_Encode2(182), /*)*//*default:*//*Label 2*/ GIMT_Encode4(562), // CHECK-NEXT: /*TargetOpcode::G_UNMERGE_VALUES*//*Label 0*/ GIMT_Encode4(478), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), // CHECK-NEXT: /*TargetOpcode::G_FNEG*//*Label 1*/ GIMT_Encode4(530), // CHECK-NEXT: // Label 0: @478 diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td index 751b1318ecc0..f0ca65a87b76 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td @@ -34,12 +34,12 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2(132), /*)*//*default:*//*Label 3*/ GIMT_Encode4(579), -// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4(462), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_CONSTANT*//*Label 1*/ GIMT_Encode4(493), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 2*/ GIMT_Encode4(539), -// CHECK-NEXT: // Label 0: @462 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(492), // Rule ID 0 // +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2(128), /*)*//*default:*//*Label 3*/ GIMT_Encode4(563), +// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4(446), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_CONSTANT*//*Label 1*/ GIMT_Encode4(477), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 2*/ GIMT_Encode4(523), +// CHECK-NEXT: // Label 0: @446 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(476), // Rule ID 0 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled), // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[0] a @@ -51,10 +51,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddImm8, /*InsnID*/0, /*Imm*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 4: @492 +// CHECK-NEXT: // Label 4: @476 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 1: @493 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4(538), // Rule ID 2 // +// CHECK-NEXT: // Label 1: @477 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4(522), // Rule ID 2 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled), // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[0] a @@ -66,10 +66,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddCImm, /*InsnID*/0, /*Type*/GILLT_s32, /*Imm*/GIMT_Encode8(42), // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 5: @538 +// CHECK-NEXT: // Label 5: @522 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 2: @539 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(578), // Rule ID 1 // +// CHECK-NEXT: // Label 2: @523 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(562), // Rule ID 1 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled), // CHECK-NEXT: // MIs[0] a // CHECK-NEXT: // No operand predicates @@ -83,10 +83,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 6: @578 +// CHECK-NEXT: // Label 6: @562 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 3: @579 +// CHECK-NEXT: // Label 3: @563 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: }; // Size: 580 bytes +// CHECK-NEXT: }; // Size: 564 bytes // CHECK-NEXT: return MatchTable0; // CHECK-NEXT: } diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td index e8e6d3e74f40..a446fb72298c 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td @@ -29,7 +29,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(119), GIMT_Encode2(121), /*)*//*default:*//*Label 2*/ GIMT_Encode4(132), +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(115), GIMT_Encode2(117), /*)*//*default:*//*Label 2*/ GIMT_Encode4(132), // CHECK-NEXT: /*TargetOpcode::G_INTRINSIC*//*Label 0*/ GIMT_Encode4(18), // CHECK-NEXT: /*TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS*//*Label 1*/ GIMT_Encode4(73), // CHECK-NEXT: // Label 0: @18 diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td index 26a0ec6235e3..d3c202c4cb01 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td @@ -28,7 +28,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(124), GIMT_Encode2(187), /*)*//*default:*//*Label 3*/ GIMT_Encode4(380), +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(120), GIMT_Encode2(183), /*)*//*default:*//*Label 3*/ GIMT_Encode4(380), // CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 0*/ GIMT_Encode4(262), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), // CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 1*/ GIMT_Encode4(298), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), // CHECK-NEXT: /*TargetOpcode::G_FPEXT*//*Label 2*/ GIMT_Encode4(344), diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td index 83b77519bc73..cc77bfdd29c3 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td @@ -37,7 +37,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(69), GIMT_Encode2(73), /*)*//*default:*//*Label 2*/ GIMT_Encode4(88), +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(65), GIMT_Encode2(69), /*)*//*default:*//*Label 2*/ GIMT_Encode4(88), // CHECK-NEXT: /*TargetOpcode::G_UNMERGE_VALUES*//*Label 0*/ GIMT_Encode4(26), GIMT_Encode4(0), GIMT_Encode4(0), // CHECK-NEXT: /*TargetOpcode::G_BUILD_VECTOR*//*Label 1*/ GIMT_Encode4(57), // CHECK-NEXT: // Label 0: @26 @@ -98,6 +98,6 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIM_Reject, // CHECK-NEXT: // Label 2: @88 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: }; // Size: 89 bytes +// CHECK-NEXT: }; // CHECK-NEXT: return MatchTable0; // CHECK-NEXT: } diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td index 5cf4e044a0fb..57ad0009b5bd 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td @@ -132,15 +132,15 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // Verify match table. // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2(132), /*)*//*default:*//*Label 6*/ GIMT_Encode4(677), -// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4(462), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_AND*//*Label 1*/ GIMT_Encode4(504), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_STORE*//*Label 2*/ GIMT_Encode4(557), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 3*/ GIMT_Encode4(599), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_SEXT*//*Label 4*/ GIMT_Encode4(624), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 5*/ GIMT_Encode4(637), -// CHECK-NEXT: // Label 0: @462 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(491), // Rule ID 4 // +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2(128), /*)*//*default:*//*Label 6*/ GIMT_Encode4(661), +// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4(446), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_AND*//*Label 1*/ GIMT_Encode4(488), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_STORE*//*Label 2*/ GIMT_Encode4(541), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 3*/ GIMT_Encode4(583), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_SEXT*//*Label 4*/ GIMT_Encode4(608), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 5*/ GIMT_Encode4(621), +// CHECK-NEXT: // Label 0: @446 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(475), // Rule ID 4 // // CHECK-NEXT: GIM_CheckFeatures, GIMT_Encode2(GIFBS_HasAnswerToEverything), // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule3Enabled), // CHECK-NEXT: // MIs[0] a @@ -155,8 +155,8 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: // Combiner Rule #3: InstTest1 // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0), // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 7: @491 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(503), // Rule ID 3 // +// CHECK-NEXT: // Label 7: @475 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(487), // Rule ID 3 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled), // CHECK-NEXT: // MIs[0] a // CHECK-NEXT: // No operand predicates @@ -165,10 +165,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: // Combiner Rule #2: InstTest0 // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner1), // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 8: @503 +// CHECK-NEXT: // Label 8: @487 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 1: @504 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 9*/ GIMT_Encode4(556), // Rule ID 6 // +// CHECK-NEXT: // Label 1: @488 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 9*/ GIMT_Encode4(540), // Rule ID 6 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule5Enabled), // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[0] dst @@ -186,10 +186,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // z // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 9: @556 +// CHECK-NEXT: // Label 9: @540 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 2: @557 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 10*/ GIMT_Encode4(598), // Rule ID 5 // +// CHECK-NEXT: // Label 2: @541 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 10*/ GIMT_Encode4(582), // Rule ID 5 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule4Enabled), // CHECK-NEXT: // MIs[0] tmp // CHECK-NEXT: GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/0, // MIs[1] @@ -207,32 +207,32 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner2), // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 10: @598 +// CHECK-NEXT: // Label 10: @582 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 3: @599 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 11*/ GIMT_Encode4(611), // Rule ID 0 // +// CHECK-NEXT: // Label 3: @583 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 11*/ GIMT_Encode4(595), // Rule ID 0 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled), // CHECK-NEXT: // Combiner Rule #0: WipOpcodeTest0; wip_match_opcode 'G_TRUNC' // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0), // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 11: @611 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 12*/ GIMT_Encode4(623), // Rule ID 1 // +// CHECK-NEXT: // Label 11: @595 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 12*/ GIMT_Encode4(607), // Rule ID 1 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled), // CHECK-NEXT: // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_TRUNC' // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0), // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 12: @623 +// CHECK-NEXT: // Label 12: @607 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 4: @624 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 13*/ GIMT_Encode4(636), // Rule ID 2 // +// CHECK-NEXT: // Label 4: @608 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 13*/ GIMT_Encode4(620), // Rule ID 2 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled), // CHECK-NEXT: // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_SEXT' // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0), // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 13: @636 +// CHECK-NEXT: // Label 13: @620 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 5: @637 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 14*/ GIMT_Encode4(676), // Rule ID 7 // +// CHECK-NEXT: // Label 5: @621 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 14*/ GIMT_Encode4(660), // Rule ID 7 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule6Enabled), // CHECK-NEXT: // MIs[0] dst // CHECK-NEXT: // No operand predicates @@ -247,10 +247,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 14: @676 +// CHECK-NEXT: // Label 14: @660 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 6: @677 +// CHECK-NEXT: // Label 6: @661 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: }; // Size: 678 bytes +// CHECK-NEXT: }; // Size: 662 bytes // CHECK-NEXT: return MatchTable0; // CHECK-NEXT: } -- GitLab From 89e41e2965b2b38a4aa1ad7757684566679ef762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Mon, 4 Mar 2024 07:37:33 +0100 Subject: [PATCH 003/929] [clang][Interp] Implement __builtin_complex --- clang/lib/AST/Interp/ByteCodeExprGen.cpp | 17 ++++++++++++--- clang/lib/AST/Interp/InterpBuiltin.cpp | 27 +++++++++++++++++++++--- clang/test/AST/Interp/complex.cpp | 20 +++++++++++++----- 3 files changed, 53 insertions(+), 11 deletions(-) diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp index b4110d5856d5..efe10458f779 100644 --- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp +++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp @@ -2730,6 +2730,18 @@ bool ByteCodeExprGen::VisitBuiltinCallExpr(const CallExpr *E) { if (!Func) return false; + QualType ReturnType = E->getType(); + std::optional ReturnT = classify(E); + + // Non-primitive return type. Prepare storage. + if (!Initializing && !ReturnT && !ReturnType->isVoidType()) { + std::optional LocalIndex = allocateLocal(E, /*IsExtended=*/false); + if (!LocalIndex) + return false; + if (!this->emitGetPtrLocal(*LocalIndex, E)) + return false; + } + if (!Func->isUnevaluatedBuiltin()) { // Put arguments on the stack. for (const auto *Arg : E->arguments()) { @@ -2741,10 +2753,9 @@ bool ByteCodeExprGen::VisitBuiltinCallExpr(const CallExpr *E) { if (!this->emitCallBI(Func, E, E)) return false; - QualType ReturnType = E->getCallReturnType(Ctx.getASTContext()); if (DiscardResult && !ReturnType->isVoidType()) { - PrimType T = classifyPrim(ReturnType); - return this->emitPop(T, E); + assert(ReturnT); + return this->emitPop(*ReturnT, E); } return true; diff --git a/clang/lib/AST/Interp/InterpBuiltin.cpp b/clang/lib/AST/Interp/InterpBuiltin.cpp index cc457ce41af5..5250d02be85a 100644 --- a/clang/lib/AST/Interp/InterpBuiltin.cpp +++ b/clang/lib/AST/Interp/InterpBuiltin.cpp @@ -900,6 +900,25 @@ static bool interp__builtin_atomic_lock_free(InterpState &S, CodePtr OpPC, return false; } +/// __builtin_complex(Float A, float B); +static bool interp__builtin_complex(InterpState &S, CodePtr OpPC, + const InterpFrame *Frame, + const Function *Func, + const CallExpr *Call) { + const Floating &Arg2 = S.Stk.peek(); + const Floating &Arg1 = S.Stk.peek(align(primSize(PT_Float)) * 2); + Pointer &Result = S.Stk.peek(align(primSize(PT_Float)) * 2 + + align(primSize(PT_Ptr))); + + Result.atIndex(0).deref() = Arg1; + Result.atIndex(0).initialize(); + Result.atIndex(1).deref() = Arg2; + Result.atIndex(1).initialize(); + Result.initialize(); + + return true; +} + bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F, const CallExpr *Call) { InterpFrame *Frame = S.Current; @@ -907,9 +926,6 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F, std::optional ReturnT = S.getContext().classify(Call); - // If classify failed, we assume void. - assert(ReturnT || Call->getType()->isVoidType()); - switch (F->getBuiltinID()) { case Builtin::BI__builtin_is_constant_evaluated: S.Stk.push(Boolean::from(S.inConstantContext())); @@ -1206,6 +1222,11 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F, return false; break; + case Builtin::BI__builtin_complex: + if (!interp__builtin_complex(S, OpPC, Frame, F, Call)) + return false; + break; + default: S.FFDiag(S.Current->getLocation(OpPC), diag::note_invalid_subexpr_in_const_expr) diff --git a/clang/test/AST/Interp/complex.cpp b/clang/test/AST/Interp/complex.cpp index b6091d90867a..8acce7b734d8 100644 --- a/clang/test/AST/Interp/complex.cpp +++ b/clang/test/AST/Interp/complex.cpp @@ -1,8 +1,5 @@ -// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -verify -Wno-unused-value %s -// RUN: %clang_cc1 -verify=ref -Wno-unused-value %s - -// expected-no-diagnostics -// ref-no-diagnostics +// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -verify=both,expected -Wno-unused-value %s +// RUN: %clang_cc1 -verify=both,ref -Wno-unused-value %s constexpr _Complex double z1 = {1.0, 2.0}; static_assert(__real(z1) == 1.0, ""); @@ -256,3 +253,16 @@ namespace DeclRefCopy { } static_assert(localComplexArray() == (24 + 42), ""); } + +namespace Builtin { + constexpr _Complex float A = __builtin_complex(10.0f, 20.0f); + static_assert(__real(A) == 10, ""); + static_assert(__imag(A) == 20, ""); + + constexpr _Complex double B = __builtin_complex(10.0, 20.0); + static_assert(__real(B) == 10, ""); + static_assert(__imag(B) == 20, ""); + + + constexpr _Complex float C = __builtin_complex(10.0f, 20.0); // both-error {{arguments are of different types}} +} -- GitLab From 03f852f704c343abc4a36db194027461b7b35afa Mon Sep 17 00:00:00 2001 From: Graham Hunter Date: Mon, 4 Mar 2024 16:17:01 +0000 Subject: [PATCH 004/929] [AArch64] Improve cost model for legal subvec insert/extract (#81135) Currently we model subvector inserts and extracts as shuffles, potentially going as far as scalarizing. If the types are legal then they can just be simple zip/unzip operations, or possible even no-ops. Change the cost to a relatively small one to ensure that simple loops featuring such operations between fixed and scalable vector types that are effectively the same at a given sve width can be unrolled and further optimized. --- .../AArch64/AArch64TargetTransformInfo.cpp | 38 +++++++++++++++++++ .../CostModel/AArch64/sve-intrinsics.ll | 36 ++++++++++++++++-- .../AArch64/scalable-vec-ins-ext.ll | 32 +++++++++------- 3 files changed, 88 insertions(+), 18 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index 010e569809e2..48f053c39b1b 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -500,6 +500,11 @@ AArch64TTIImpl::getPopcntSupport(unsigned TyWidth) { return TTI::PSK_Software; } +static bool isUnpackedVectorVT(EVT VecVT) { + return VecVT.isScalableVector() && + VecVT.getSizeInBits().getKnownMinValue() < AArch64::SVEBitsPerBlock; +} + InstructionCost AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, TTI::TargetCostKind CostKind) { @@ -568,6 +573,39 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, } return Cost; } + case Intrinsic::vector_extract: + case Intrinsic::vector_insert: { + // If both the vector and subvector types are legal types and the index + // is 0, then this should be a no-op or simple operation; return a + // relatively low cost. + + // If arguments aren't actually supplied, then we cannot determine the + // value of the index. We also want to skip predicate types. + if (ICA.getArgs().size() != ICA.getArgTypes().size() || + ICA.getReturnType()->getScalarType()->isIntegerTy(1)) + break; + + LLVMContext &C = RetTy->getContext(); + EVT VecVT = getTLI()->getValueType(DL, ICA.getArgTypes()[0]); + bool IsExtract = ICA.getID() == Intrinsic::vector_extract; + EVT SubVecVT = IsExtract ? getTLI()->getValueType(DL, RetTy) + : getTLI()->getValueType(DL, ICA.getArgTypes()[1]); + // Skip this if either the vector or subvector types are unpacked + // SVE types; they may get lowered to stack stores and loads. + if (isUnpackedVectorVT(VecVT) || isUnpackedVectorVT(SubVecVT)) + break; + + TargetLoweringBase::LegalizeKind SubVecLK = + getTLI()->getTypeConversion(C, SubVecVT); + TargetLoweringBase::LegalizeKind VecLK = + getTLI()->getTypeConversion(C, VecVT); + const Value *Idx = IsExtract ? ICA.getArgs()[1] : ICA.getArgs()[2]; + const ConstantInt *CIdx = dyn_cast(Idx); + if (SubVecLK.first == TargetLoweringBase::TypeLegal && + VecLK.first == TargetLoweringBase::TypeLegal && CIdx && CIdx->isZero()) + return TTI::TCC_Free; + break; + } case Intrinsic::bitreverse: { static const CostTblEntry BitreverseTbl[] = { {Intrinsic::bitreverse, MVT::i32, 1}, diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll index 025382878d99..937c383aedc7 100644 --- a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll +++ b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll @@ -32,11 +32,15 @@ declare @llvm.vector.insert.nxv16i32.nxv4i32( @llvm.vector.insert.nxv4f32.v4f32( undef, <4 x float> undef, i64 0) -; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %extract_legal_fixed_from_scalable = call <2 x double> @llvm.vector.extract.v2f64.nxv2f64( undef, i64 0) +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %insert_legal_fixed_into_scalable = call @llvm.vector.insert.nxv4f32.v4f32( undef, <4 x float> undef, i64 0) +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %extract_legal_fixed_from_scalable = call <2 x double> @llvm.vector.extract.v2f64.nxv2f64( undef, i64 0) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv16i1_nxv2i1 = call @llvm.vector.insert.nxv16i1.nxv2i1( undef, undef, i64 0) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4i1_nxv16i1 = call @llvm.vector.extract.nxv4i1.nxv16i1( undef, i64 0) ; CHECK-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1( undef, i64 0) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %insert_v2f32_nxv2f32 = call @llvm.vector.insert.nxv2f32.v2f32( undef, <2 x float> undef, i64 0) +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16( undef, i64 0) +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv2f32_nxv4f32 = call @llvm.vector.insert.nxv4f32.nxv2f32( undef, undef, i64 0) +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4f32_nxv8f32 = call @llvm.vector.extract.nxv4f32.nxv8f32( undef, i64 0) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; ; TYPE_BASED_ONLY-LABEL: 'vector_insert_extract_idxzero_128b' @@ -45,6 +49,10 @@ define void @vector_insert_extract_idxzero_128b() #1 { ; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %insert_nxv16i1_nxv2i1 = call @llvm.vector.insert.nxv16i1.nxv2i1( undef, undef, i64 0) ; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %extract_nxv4i1_nxv16i1 = call @llvm.vector.extract.nxv4i1.nxv16i1( undef, i64 0) ; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1( undef, i64 0) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %insert_v2f32_nxv2f32 = call @llvm.vector.insert.nxv2f32.v2f32( undef, <2 x float> undef, i64 0) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16( undef, i64 0) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %insert_nxv2f32_nxv4f32 = call @llvm.vector.insert.nxv4f32.nxv2f32( undef, undef, i64 0) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %extract_nxv4f32_nxv8f32 = call @llvm.vector.extract.nxv4f32.nxv8f32( undef, i64 0) ; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %insert_legal_fixed_into_scalable = call @llvm.vector.insert.nxv4f32.v4f32( undef, <4 x float> undef, i64 0) @@ -52,20 +60,32 @@ define void @vector_insert_extract_idxzero_128b() #1 { %insert_nxv16i1_nxv2i1 = call @llvm.vector.insert.nxv16i1.v2i1( undef, undef, i64 0) %extract_nxv4i1_nxv16i1 = call @llvm.vector.extract.nxv4i1.nxv16i1( undef, i64 0) %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1( undef, i64 0) + %insert_v2f32_nxv2f32 = call @llvm.vector.insert.nxv2f32.v2f32( undef, <2 x float> undef, i64 0) + %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16( undef, i64 0) + %insert_nxv2f32_nxv4f32 = call @llvm.vector.insert.nxv4f32.nxv2f32( undef, undef, i64 0) + %extract_nxv4f32_nxv8f32 = call @llvm.vector.extract.nxv4f32.nxv8f32( undef, i64 0) ret void } declare @llvm.vector.insert.nxv4f32.v4f32(, <4 x float>, i64) declare <2 x double> @llvm.vector.extract.v2f64.nxv2f64(, i64) declare @llvm.vector.insert.nxv16i1.v2i1(, , i64) declare @llvm.vector.extract.nxv4i1.nxv16i1(, i64) +declare @llvm.vector.insert.nxv2f32.v2f32(, <2 x float>, i64) +declare <4 x half> @llvm.vector.extract.v4f16.nxv4f16(, i64) +declare @llvm.vector.insert.nxv4f32.nxv2f32(, , i64) +declare @llvm.vector.extract.nxv4f32.nxv8f32(, i64) define void @vector_insert_extract_idxzero_256b() #2 { ; CHECK-LABEL: 'vector_insert_extract_idxzero_256b' -; CHECK-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %insert_legal_fixed_into_scalable = call @llvm.vector.insert.nxv8i16.v16i16( undef, <16 x i16> undef, i64 0) -; CHECK-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %extract_legal_fixed_from_scalable = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32( undef, i64 0) +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %insert_legal_fixed_into_scalable = call @llvm.vector.insert.nxv8i16.v16i16( undef, <16 x i16> undef, i64 0) +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %extract_legal_fixed_from_scalable = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32( undef, i64 0) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv16i1_nxv2i1 = call @llvm.vector.insert.nxv16i1.nxv2i1( undef, undef, i64 0) ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4i1_nxv16i1 = call @llvm.vector.extract.nxv4i1.nxv16i1( undef, i64 0) ; CHECK-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1( undef, i64 0) +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %insert_v2f32_nxv2f32 = call @llvm.vector.insert.nxv2f32.v2f32( undef, <2 x float> undef, i64 0) +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16( undef, i64 0) +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv2f32_nxv4f32 = call @llvm.vector.insert.nxv4f32.nxv2f32( undef, undef, i64 0) +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4f32_nxv8f32 = call @llvm.vector.extract.nxv4f32.nxv8f32( undef, i64 0) ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; ; TYPE_BASED_ONLY-LABEL: 'vector_insert_extract_idxzero_256b' @@ -74,6 +94,10 @@ define void @vector_insert_extract_idxzero_256b() #2 { ; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %insert_nxv16i1_nxv2i1 = call @llvm.vector.insert.nxv16i1.nxv2i1( undef, undef, i64 0) ; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %extract_nxv4i1_nxv16i1 = call @llvm.vector.extract.nxv4i1.nxv16i1( undef, i64 0) ; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1( undef, i64 0) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %insert_v2f32_nxv2f32 = call @llvm.vector.insert.nxv2f32.v2f32( undef, <2 x float> undef, i64 0) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16( undef, i64 0) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %insert_nxv2f32_nxv4f32 = call @llvm.vector.insert.nxv4f32.nxv2f32( undef, undef, i64 0) +; TYPE_BASED_ONLY-NEXT: Cost Model: Invalid cost for instruction: %extract_nxv4f32_nxv8f32 = call @llvm.vector.extract.nxv4f32.nxv8f32( undef, i64 0) ; TYPE_BASED_ONLY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %insert_legal_fixed_into_scalable = call @llvm.vector.insert.nxv8i16.v16i16( undef, <16 x i16> undef, i64 0) @@ -81,6 +105,10 @@ define void @vector_insert_extract_idxzero_256b() #2 { %insert_nxv16i1_nxv2i1 = call @llvm.vector.insert.nxv16i1.v2i1( undef, undef, i64 0) %extract_nxv4i1_nxv16i1 = call @llvm.vector.extract.nxv4i1.nxv16i1( undef, i64 0) %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1( undef, i64 0) + %insert_v2f32_nxv2f32 = call @llvm.vector.insert.nxv2f32.v2f32( undef, <2 x float> undef, i64 0) + %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16( undef, i64 0) + %insert_nxv2f32_nxv4f32 = call @llvm.vector.insert.nxv4f32.nxv2f32( undef, undef, i64 0) + %extract_nxv4f32_nxv8f32 = call @llvm.vector.extract.nxv4f32.nxv8f32( undef, i64 0) ret void } declare @llvm.vector.insert.nxv8i16.v16i16(, <16 x i16>, i64) diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/scalable-vec-ins-ext.ll b/llvm/test/Transforms/LoopUnroll/AArch64/scalable-vec-ins-ext.ll index 1d5bc84c2763..cf670f5f4f27 100644 --- a/llvm/test/Transforms/LoopUnroll/AArch64/scalable-vec-ins-ext.ll +++ b/llvm/test/Transforms/LoopUnroll/AArch64/scalable-vec-ins-ext.ll @@ -12,26 +12,30 @@ define void @test_ins_ext_cost(ptr readonly %a, ptr readonly %b, ptr readonly %c ; CHECK-LABEL: define void @test_ins_ext_cost( ; CHECK-SAME: ptr readonly [[A:%.*]], ptr readonly [[B:%.*]], ptr readonly [[C:%.*]], ptr noalias [[D:%.*]]) #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: entry: -; CHECK-NEXT: br label [[FOR_BODY:%.*]] -; CHECK: for.body: -; CHECK-NEXT: [[EXIT_COND:%.*]] = phi i1 [ true, [[ENTRY:%.*]] ], [ false, [[FOR_BODY]] ] -; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ 1, [[FOR_BODY]] ] -; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds <8 x float>, ptr [[A]], i64 [[IV]] -; CHECK-NEXT: [[LOAD_A:%.*]] = load <8 x float>, ptr [[GEP_A]], align 16 -; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds <8 x float>, ptr [[B]], i64 [[IV]] -; CHECK-NEXT: [[LOAD_B:%.*]] = load <8 x float>, ptr [[GEP_B]], align 16 -; CHECK-NEXT: [[GEP_C:%.*]] = getelementptr inbounds <8 x float>, ptr [[C]], i64 [[IV]] -; CHECK-NEXT: [[LOAD_C:%.*]] = load <8 x float>, ptr [[GEP_C]], align 16 +; CHECK-NEXT: [[LOAD_A:%.*]] = load <8 x float>, ptr [[A]], align 16 +; CHECK-NEXT: [[LOAD_B:%.*]] = load <8 x float>, ptr [[B]], align 16 +; CHECK-NEXT: [[LOAD_C:%.*]] = load <8 x float>, ptr [[C]], align 16 ; CHECK-NEXT: [[CAST_SCALABLE_B:%.*]] = tail call @llvm.vector.insert.nxv4f32.v8f32( undef, <8 x float> [[LOAD_B]], i64 0) ; CHECK-NEXT: [[CAST_SCALABLE_C:%.*]] = tail call @llvm.vector.insert.nxv4f32.v8f32( undef, <8 x float> [[LOAD_C]], i64 0) ; CHECK-NEXT: [[ADD:%.*]] = fadd [[CAST_SCALABLE_B]], [[CAST_SCALABLE_C]] ; CHECK-NEXT: [[CAST_SCALABLE_A:%.*]] = tail call @llvm.vector.insert.nxv4f32.v8f32( undef, <8 x float> [[LOAD_A]], i64 0) ; CHECK-NEXT: [[MUL:%.*]] = fmul [[CAST_SCALABLE_A]], [[ADD]] ; CHECK-NEXT: [[CAST_FIXED_D:%.*]] = tail call <8 x float> @llvm.vector.extract.v8f32.nxv4f32( [[MUL]], i64 0) -; CHECK-NEXT: [[GEP_D:%.*]] = getelementptr inbounds <8 x float>, ptr [[D]], i64 0, i64 [[IV]] -; CHECK-NEXT: store <8 x float> [[CAST_FIXED_D]], ptr [[GEP_D]], align 16 -; CHECK-NEXT: br i1 [[EXIT_COND]], label [[FOR_BODY]], label [[EXIT:%.*]] -; CHECK: exit: +; CHECK-NEXT: store <8 x float> [[CAST_FIXED_D]], ptr [[D]], align 16 +; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds <8 x float>, ptr [[A]], i64 1 +; CHECK-NEXT: [[LOAD_A_1:%.*]] = load <8 x float>, ptr [[GEP_A_1]], align 16 +; CHECK-NEXT: [[GEP_B_1:%.*]] = getelementptr inbounds <8 x float>, ptr [[B]], i64 1 +; CHECK-NEXT: [[LOAD_B_1:%.*]] = load <8 x float>, ptr [[GEP_B_1]], align 16 +; CHECK-NEXT: [[GEP_C_1:%.*]] = getelementptr inbounds <8 x float>, ptr [[C]], i64 1 +; CHECK-NEXT: [[LOAD_C_1:%.*]] = load <8 x float>, ptr [[GEP_C_1]], align 16 +; CHECK-NEXT: [[CAST_SCALABLE_B_1:%.*]] = tail call @llvm.vector.insert.nxv4f32.v8f32( undef, <8 x float> [[LOAD_B_1]], i64 0) +; CHECK-NEXT: [[CAST_SCALABLE_C_1:%.*]] = tail call @llvm.vector.insert.nxv4f32.v8f32( undef, <8 x float> [[LOAD_C_1]], i64 0) +; CHECK-NEXT: [[ADD_1:%.*]] = fadd [[CAST_SCALABLE_B_1]], [[CAST_SCALABLE_C_1]] +; CHECK-NEXT: [[CAST_SCALABLE_A_1:%.*]] = tail call @llvm.vector.insert.nxv4f32.v8f32( undef, <8 x float> [[LOAD_A_1]], i64 0) +; CHECK-NEXT: [[MUL_1:%.*]] = fmul [[CAST_SCALABLE_A_1]], [[ADD_1]] +; CHECK-NEXT: [[CAST_FIXED_D_1:%.*]] = tail call <8 x float> @llvm.vector.extract.v8f32.nxv4f32( [[MUL_1]], i64 0) +; CHECK-NEXT: [[GEP_D_1:%.*]] = getelementptr inbounds <8 x float>, ptr [[D]], i64 0, i64 1 +; CHECK-NEXT: store <8 x float> [[CAST_FIXED_D_1]], ptr [[GEP_D_1]], align 16 ; CHECK-NEXT: ret void ; entry: -- GitLab From 2a65941a7964644930dcf97559f73f8e5b81738e Mon Sep 17 00:00:00 2001 From: Nico Weber Date: Mon, 4 Mar 2024 11:25:47 -0500 Subject: [PATCH 005/929] [gn] port 8300f30a9234 --- llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn index 63285ef7ff8a..4b8d6193eae4 100644 --- a/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn +++ b/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn @@ -276,6 +276,7 @@ static_library("builtins") { sources += [ "cpu_model/x86.c", "i386/fp_mode.c", + "extendbfsf2.c", "truncdfbf2.c", "truncsfbf2.c", ] -- GitLab From cedb9704bda4f4f9622ba60a18135d70569d0582 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 4 Mar 2024 16:29:19 +0000 Subject: [PATCH 006/929] [VPlan] Verify CFG invariants first (NFCI). Verifying CFG invariants of a block before verifying its contents allows contents verification to rely on the CFG invariants (e.g. that there's a vector loop region that can be retrieved). This avoids extra checks in https://github.com/llvm/llvm-project/pull/76172. --- llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index 5da752f62cfc..7ebdb914fb85 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -135,9 +135,6 @@ static bool hasDuplicates(const SmallVectorImpl &VPBlockVec) { static bool verifyBlock(const VPBlockBase *VPB, const VPDominatorTree &VPDT) { auto *VPBB = dyn_cast(VPB); - if (VPBB && !verifyVPBasicBlock(VPBB, VPDT)) - return false; - // Check block's condition bit. if (VPB->getNumSuccessors() > 1 || (VPBB && VPBB->getParent() && VPBB->isExiting() && @@ -196,7 +193,7 @@ static bool verifyBlock(const VPBlockBase *VPB, const VPDominatorTree &VPDT) { return false; } } - return true; + return !VPBB || verifyVPBasicBlock(VPBB, VPDT); } /// Helper function that verifies the CFG invariants of the VPBlockBases within -- GitLab From c240aca7a8a71f7218724aeb6c040289b51057dc Mon Sep 17 00:00:00 2001 From: "Kazushi (Jam) Marukawa" Date: Tue, 5 Mar 2024 01:34:48 +0900 Subject: [PATCH 007/929] Revert "Reland "[clang][modules] Print library module manifest path." (#82160)" This reverts commit 0c89427b99f6f6d7c217c70ff880ca097340f9a4. --- clang/include/clang/Driver/Driver.h | 10 ----- clang/include/clang/Driver/Options.td | 3 -- clang/lib/Driver/Driver.cpp | 44 ------------------- ...les-print-library-module-manifest-path.cpp | 40 ----------------- 4 files changed, 97 deletions(-) delete mode 100644 clang/test/Driver/modules-print-library-module-manifest-path.cpp diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h index bcf8c1295f2d..c4cab360bab3 100644 --- a/clang/include/clang/Driver/Driver.h +++ b/clang/include/clang/Driver/Driver.h @@ -615,16 +615,6 @@ public: // FIXME: This should be in CompilationInfo. std::string GetProgramPath(StringRef Name, const ToolChain &TC) const; - /// Lookup the path to the Standard library module manifest. - /// - /// \param C - The compilation. - /// \param TC - The tool chain for additional information on - /// directories to search. - // - // FIXME: This should be in CompilationInfo. - std::string GetStdModuleManifestPath(const Compilation &C, - const ToolChain &TC) const; - /// HandleAutocompletions - Handle --autocomplete by searching and printing /// possible flags, descriptions, and its arguments. void HandleAutocompletions(StringRef PassedFlags) const; diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 3e857f4e6faf..bef38738fde8 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -5364,9 +5364,6 @@ def print_resource_dir : Flag<["-", "--"], "print-resource-dir">, def print_search_dirs : Flag<["-", "--"], "print-search-dirs">, HelpText<"Print the paths used for finding libraries and programs">, Visibility<[ClangOption, CLOption]>; -def print_std_module_manifest_path : Flag<["-", "--"], "print-library-module-manifest-path">, - HelpText<"Print the path for the C++ Standard library module manifest">, - Visibility<[ClangOption, CLOption]>; def print_targets : Flag<["-", "--"], "print-targets">, HelpText<"Print the registered targets">, Visibility<[ClangOption, CLOption]>; diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp index b8ec0791dc51..de8ceb2f0898 100644 --- a/clang/lib/Driver/Driver.cpp +++ b/clang/lib/Driver/Driver.cpp @@ -2196,12 +2196,6 @@ bool Driver::HandleImmediateArgs(const Compilation &C) { return false; } - if (C.getArgs().hasArg(options::OPT_print_std_module_manifest_path)) { - llvm::outs() << GetStdModuleManifestPath(C, C.getDefaultToolChain()) - << '\n'; - return false; - } - if (C.getArgs().hasArg(options::OPT_print_runtime_dir)) { if (std::optional RuntimePath = TC.getRuntimePath()) llvm::outs() << *RuntimePath << '\n'; @@ -6174,44 +6168,6 @@ std::string Driver::GetProgramPath(StringRef Name, const ToolChain &TC) const { return std::string(Name); } -std::string Driver::GetStdModuleManifestPath(const Compilation &C, - const ToolChain &TC) const { - std::string error = ""; - - switch (TC.GetCXXStdlibType(C.getArgs())) { - case ToolChain::CST_Libcxx: { - std::string lib = GetFilePath("libc++.so", TC); - - // Note when there are multiple flavours of libc++ the module json needs to - // look at the command-line arguments for the proper json. - // These flavours do not exist at the moment, but there are plans to - // provide a variant that is built with sanitizer instrumentation enabled. - - // For example - // StringRef modules = [&] { - // const SanitizerArgs &Sanitize = TC.getSanitizerArgs(C.getArgs()); - // if (Sanitize.needsAsanRt()) - // return "modules-asan.json"; - // return "modules.json"; - // }(); - - SmallString<128> path(lib.begin(), lib.end()); - llvm::sys::path::remove_filename(path); - llvm::sys::path::append(path, "modules.json"); - if (TC.getVFS().exists(path)) - return static_cast(path); - - return error; - } - - case ToolChain::CST_Libstdcxx: - // libstdc++ does not provide Standard library modules yet. - return error; - } - - return error; -} - std::string Driver::GetTemporaryPath(StringRef Prefix, StringRef Suffix) const { SmallString<128> Path; std::error_code EC = llvm::sys::fs::createTemporaryFile(Prefix, Suffix, Path); diff --git a/clang/test/Driver/modules-print-library-module-manifest-path.cpp b/clang/test/Driver/modules-print-library-module-manifest-path.cpp deleted file mode 100644 index 95bcc59ae23c..000000000000 --- a/clang/test/Driver/modules-print-library-module-manifest-path.cpp +++ /dev/null @@ -1,40 +0,0 @@ -// Test that -print-library-module-manifest-path finds the correct file. - -// FIXME: Enable on all platforms. - -// REQUIRES: x86-registered-target - -// RUN: rm -rf %t && split-file %s %t && cd %t -// RUN: mkdir -p %t/Inputs/usr/lib/x86_64-linux-gnu -// RUN: touch %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.so - -// RUN: %clang -print-library-module-manifest-path \ -// RUN: -stdlib=libc++ \ -// RUN: --sysroot=%t/Inputs \ -// RUN: --target=x86_64-linux-gnu 2>&1 \ -// RUN: | FileCheck libcxx-no-module-json.cpp - -// RUN: touch %t/Inputs/usr/lib/x86_64-linux-gnu/modules.json -// RUN: %clang -print-library-module-manifest-path \ -// RUN: -stdlib=libc++ \ -// RUN: --sysroot=%t/Inputs \ -// RUN: --target=x86_64-linux-gnu 2>&1 \ -// RUN: | FileCheck libcxx.cpp - -// RUN: %clang -print-library-module-manifest-path \ -// RUN: -stdlib=libstdc++ \ -// RUN: --sysroot=%t/Inputs \ -// RUN: --target=x86_64-linux-gnu 2>&1 \ -// RUN: | FileCheck libstdcxx.cpp - -//--- libcxx-no-module-json.cpp - -// CHECK: - -//--- libcxx.cpp - -// CHECK: {{.*}}/Inputs/usr/lib/x86_64-linux-gnu{{/|\\}}modules.json - -//--- libstdcxx.cpp - -// CHECK: -- GitLab From de92615d68ff3ae206d5059f35a6e4ded8b38297 Mon Sep 17 00:00:00 2001 From: Hongyu Chen <46539970+yugier@users.noreply.github.com> Date: Mon, 4 Mar 2024 08:44:36 -0800 Subject: [PATCH 008/929] [ExecutionEngine] Move IntelJITEventsWrapper to its own library. NFC (#81825) Split IntelJITEventsWrapper away from IntelJITEvents and rename it into IntelJITProfiling. NFC --- llvm/lib/ExecutionEngine/CMakeLists.txt | 9 ++-- .../IntelJITEvents/CMakeLists.txt | 43 +---------------- .../IntelJITEvents/IntelJITEventListener.cpp | 2 +- .../IntelJITProfiling/CMakeLists.txt | 47 +++++++++++++++++++ .../IntelJITEventsWrapper.h | 0 .../ittnotify_config.h | 0 .../ittnotify_types.h | 0 .../jitprofiling.c | 0 .../jitprofiling.h | 0 .../llvm-jitlistener/llvm-jitlistener.cpp | 2 +- 10 files changed, 56 insertions(+), 47 deletions(-) create mode 100644 llvm/lib/ExecutionEngine/IntelJITProfiling/CMakeLists.txt rename llvm/lib/ExecutionEngine/{IntelJITEvents => IntelJITProfiling}/IntelJITEventsWrapper.h (100%) rename llvm/lib/ExecutionEngine/{IntelJITEvents => IntelJITProfiling}/ittnotify_config.h (100%) rename llvm/lib/ExecutionEngine/{IntelJITEvents => IntelJITProfiling}/ittnotify_types.h (100%) rename llvm/lib/ExecutionEngine/{IntelJITEvents => IntelJITProfiling}/jitprofiling.c (100%) rename llvm/lib/ExecutionEngine/{IntelJITEvents => IntelJITProfiling}/jitprofiling.h (100%) diff --git a/llvm/lib/ExecutionEngine/CMakeLists.txt b/llvm/lib/ExecutionEngine/CMakeLists.txt index af6be62dd525..02178ff526bf 100644 --- a/llvm/lib/ExecutionEngine/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/CMakeLists.txt @@ -28,6 +28,11 @@ if(BUILD_SHARED_LIBS) target_link_libraries(LLVMExecutionEngine PUBLIC LLVMRuntimeDyld) endif() +if( LLVM_USE_INTEL_JITEVENTS ) + add_subdirectory(IntelJITProfiling) + add_subdirectory(IntelJITEvents) +endif( LLVM_USE_INTEL_JITEVENTS ) + add_subdirectory(Interpreter) add_subdirectory(JITLink) add_subdirectory(MCJIT) @@ -38,10 +43,6 @@ if( LLVM_USE_OPROFILE ) add_subdirectory(OProfileJIT) endif( LLVM_USE_OPROFILE ) -if( LLVM_USE_INTEL_JITEVENTS ) - add_subdirectory(IntelJITEvents) -endif( LLVM_USE_INTEL_JITEVENTS ) - if( LLVM_USE_PERF ) add_subdirectory(PerfJITEvents) endif( LLVM_USE_PERF ) diff --git a/llvm/lib/ExecutionEngine/IntelJITEvents/CMakeLists.txt b/llvm/lib/ExecutionEngine/IntelJITEvents/CMakeLists.txt index 0c5017c359d6..b4fd04d65e26 100644 --- a/llvm/lib/ExecutionEngine/IntelJITEvents/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/IntelJITEvents/CMakeLists.txt @@ -1,48 +1,8 @@ include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/.. ) - -if(NOT DEFINED ITTAPI_GIT_REPOSITORY) - set(ITTAPI_GIT_REPOSITORY https://github.com/intel/ittapi.git) -endif() - -if(NOT DEFINED ITTAPI_GIT_TAG) - set(ITTAPI_GIT_TAG v3.18.12) -endif() - -if(NOT DEFINED ITTAPI_SOURCE_DIR) - set(ITTAPI_SOURCE_DIR ${PROJECT_BINARY_DIR}) -endif() - -if(NOT EXISTS ${ITTAPI_SOURCE_DIR}/ittapi) - execute_process(COMMAND ${GIT_EXECUTABLE} clone ${ITTAPI_GIT_REPOSITORY} - WORKING_DIRECTORY ${ITTAPI_SOURCE_DIR} - RESULT_VARIABLE GIT_CLONE_RESULT) - if(NOT GIT_CLONE_RESULT EQUAL "0") - message(FATAL_ERROR "git clone ${ITTAPI_GIT_REPOSITORY} failed with ${GIT_CLONE_RESULT}, please clone ${ITTAPI_GIT_REPOSITORY}") - endif() -endif() - -execute_process(COMMAND ${GIT_EXECUTABLE} checkout ${ITTAPI_GIT_TAG} - WORKING_DIRECTORY ${ITTAPI_SOURCE_DIR}/ittapi - RESULT_VARIABLE GIT_CHECKOUT_RESULT) -if(NOT GIT_CHECKOUT_RESULT EQUAL "0") - message(FATAL_ERROR "git checkout ${ITTAPI_GIT_TAG} failed with ${GIT_CHECKOUT_RESULT}, please checkout ${ITTAPI_GIT_TAG} at ${ITTAPI_SOURCE_DIR}/ittapi") -endif() - -include_directories( ${ITTAPI_SOURCE_DIR}/ittapi/include/ ) - -if( HAVE_LIBDL ) - set(LLVM_INTEL_JIT_LIBS ${CMAKE_DL_LIBS}) -endif() - -set(LLVM_INTEL_JIT_LIBS ${LLVM_PTHREAD_LIB} ${LLVM_INTEL_JIT_LIBS}) - - +include_directories( ${PROJECT_BINARY_DIR}/ittapi/include/ ) add_llvm_component_library(LLVMIntelJITEvents IntelJITEventListener.cpp - jitprofiling.c - ${ITTAPI_SOURCE_DIR}/ittapi/src/ittnotify/ittnotify_static.c - LINK_LIBS ${LLVM_INTEL_JIT_LIBS} LINK_COMPONENTS CodeGen @@ -51,6 +11,7 @@ add_llvm_component_library(LLVMIntelJITEvents Support Object ExecutionEngine + IntelJITProfiling ) add_dependencies(LLVMIntelJITEvents LLVMCodeGen) diff --git a/llvm/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp b/llvm/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp index 4afcf95e9e8e..b1e99df73b48 100644 --- a/llvm/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp +++ b/llvm/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp @@ -11,7 +11,7 @@ // //===----------------------------------------------------------------------===// -#include "IntelJITEventsWrapper.h" +#include "IntelJITProfiling/IntelJITEventsWrapper.h" #include "ittnotify.h" #include "llvm-c/ExecutionEngine.h" #include "llvm/ADT/DenseMap.h" diff --git a/llvm/lib/ExecutionEngine/IntelJITProfiling/CMakeLists.txt b/llvm/lib/ExecutionEngine/IntelJITProfiling/CMakeLists.txt new file mode 100644 index 000000000000..0aedadc65df0 --- /dev/null +++ b/llvm/lib/ExecutionEngine/IntelJITProfiling/CMakeLists.txt @@ -0,0 +1,47 @@ +include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/.. ) + +if(NOT DEFINED ITTAPI_GIT_REPOSITORY) + set(ITTAPI_GIT_REPOSITORY https://github.com/intel/ittapi.git) +endif() + +if(NOT DEFINED ITTAPI_GIT_TAG) + set(ITTAPI_GIT_TAG v3.18.12) +endif() + +if(NOT DEFINED ITTAPI_SOURCE_DIR) + set(ITTAPI_SOURCE_DIR ${PROJECT_BINARY_DIR}) +endif() + +if(NOT EXISTS ${ITTAPI_SOURCE_DIR}/ittapi) + execute_process(COMMAND ${GIT_EXECUTABLE} clone ${ITTAPI_GIT_REPOSITORY} + WORKING_DIRECTORY ${ITTAPI_SOURCE_DIR} + RESULT_VARIABLE GIT_CLONE_RESULT) + if(NOT GIT_CLONE_RESULT EQUAL "0") + message(FATAL_ERROR "git clone ${ITTAPI_GIT_REPOSITORY} failed with ${GIT_CLONE_RESULT}, please clone ${ITTAPI_GIT_REPOSITORY}") + endif() +endif() + +execute_process(COMMAND ${GIT_EXECUTABLE} checkout ${ITTAPI_GIT_TAG} + WORKING_DIRECTORY ${ITTAPI_SOURCE_DIR}/ittapi + RESULT_VARIABLE GIT_CHECKOUT_RESULT) +if(NOT GIT_CHECKOUT_RESULT EQUAL "0") + message(FATAL_ERROR "git checkout ${ITTAPI_GIT_TAG} failed with ${GIT_CHECKOUT_RESULT}, please checkout ${ITTAPI_GIT_TAG} at ${ITTAPI_SOURCE_DIR}/ittapi") +endif() + +include_directories( ${ITTAPI_SOURCE_DIR}/ittapi/include/ ) + +if( HAVE_LIBDL ) + set(LLVM_INTEL_JIT_LIBS ${CMAKE_DL_LIBS}) +endif() + +set(LLVM_INTEL_JIT_LIBS ${LLVM_PTHREAD_LIB} ${LLVM_INTEL_JIT_LIBS}) + + +add_llvm_component_library(LLVMIntelJITProfiling + jitprofiling.c + ${ITTAPI_SOURCE_DIR}/ittapi/src/ittnotify/ittnotify_static.c + + LINK_LIBS ${LLVM_INTEL_JIT_LIBS} + +) + diff --git a/llvm/lib/ExecutionEngine/IntelJITEvents/IntelJITEventsWrapper.h b/llvm/lib/ExecutionEngine/IntelJITProfiling/IntelJITEventsWrapper.h similarity index 100% rename from llvm/lib/ExecutionEngine/IntelJITEvents/IntelJITEventsWrapper.h rename to llvm/lib/ExecutionEngine/IntelJITProfiling/IntelJITEventsWrapper.h diff --git a/llvm/lib/ExecutionEngine/IntelJITEvents/ittnotify_config.h b/llvm/lib/ExecutionEngine/IntelJITProfiling/ittnotify_config.h similarity index 100% rename from llvm/lib/ExecutionEngine/IntelJITEvents/ittnotify_config.h rename to llvm/lib/ExecutionEngine/IntelJITProfiling/ittnotify_config.h diff --git a/llvm/lib/ExecutionEngine/IntelJITEvents/ittnotify_types.h b/llvm/lib/ExecutionEngine/IntelJITProfiling/ittnotify_types.h similarity index 100% rename from llvm/lib/ExecutionEngine/IntelJITEvents/ittnotify_types.h rename to llvm/lib/ExecutionEngine/IntelJITProfiling/ittnotify_types.h diff --git a/llvm/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c b/llvm/lib/ExecutionEngine/IntelJITProfiling/jitprofiling.c similarity index 100% rename from llvm/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c rename to llvm/lib/ExecutionEngine/IntelJITProfiling/jitprofiling.c diff --git a/llvm/lib/ExecutionEngine/IntelJITEvents/jitprofiling.h b/llvm/lib/ExecutionEngine/IntelJITProfiling/jitprofiling.h similarity index 100% rename from llvm/lib/ExecutionEngine/IntelJITEvents/jitprofiling.h rename to llvm/lib/ExecutionEngine/IntelJITProfiling/jitprofiling.h diff --git a/llvm/tools/llvm-jitlistener/llvm-jitlistener.cpp b/llvm/tools/llvm-jitlistener/llvm-jitlistener.cpp index bd25a000c753..59949435f9ca 100644 --- a/llvm/tools/llvm-jitlistener/llvm-jitlistener.cpp +++ b/llvm/tools/llvm-jitlistener/llvm-jitlistener.cpp @@ -12,7 +12,7 @@ // //===----------------------------------------------------------------------===// -#include "../../lib/ExecutionEngine/IntelJITEvents/IntelJITEventsWrapper.h" +#include "../../lib/ExecutionEngine/IntelJITProfiling/IntelJITEventsWrapper.h" #include "llvm/ExecutionEngine/JITEventListener.h" #include "llvm/ExecutionEngine/MCJIT.h" #include "llvm/ExecutionEngine/SectionMemoryManager.h" -- GitLab From bc9c6c0c65e267a1a84e4eede81099d8ceb37558 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Mon, 4 Mar 2024 07:56:48 +0100 Subject: [PATCH 009/929] [clang][Interp][NFC] Remove ByteCodeGenError Unused. --- clang/lib/AST/CMakeLists.txt | 1 - clang/lib/AST/Interp/ByteCodeEmitter.cpp | 1 - clang/lib/AST/Interp/ByteCodeExprGen.cpp | 1 - clang/lib/AST/Interp/ByteCodeGenError.cpp | 14 ------- clang/lib/AST/Interp/ByteCodeGenError.h | 46 ----------------------- clang/lib/AST/Interp/ByteCodeStmtGen.cpp | 1 - clang/lib/AST/Interp/Context.cpp | 12 ------ clang/lib/AST/Interp/Context.h | 3 -- clang/lib/AST/Interp/EvalEmitter.cpp | 1 - 9 files changed, 80 deletions(-) delete mode 100644 clang/lib/AST/Interp/ByteCodeGenError.cpp delete mode 100644 clang/lib/AST/Interp/ByteCodeGenError.h diff --git a/clang/lib/AST/CMakeLists.txt b/clang/lib/AST/CMakeLists.txt index 6ea1ca3e76cf..3fba052d916c 100644 --- a/clang/lib/AST/CMakeLists.txt +++ b/clang/lib/AST/CMakeLists.txt @@ -66,7 +66,6 @@ add_clang_library(clangAST InheritViz.cpp Interp/ByteCodeEmitter.cpp Interp/ByteCodeExprGen.cpp - Interp/ByteCodeGenError.cpp Interp/ByteCodeStmtGen.cpp Interp/Context.cpp Interp/Descriptor.cpp diff --git a/clang/lib/AST/Interp/ByteCodeEmitter.cpp b/clang/lib/AST/Interp/ByteCodeEmitter.cpp index 4fbfc0930fba..d912c101449d 100644 --- a/clang/lib/AST/Interp/ByteCodeEmitter.cpp +++ b/clang/lib/AST/Interp/ByteCodeEmitter.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "ByteCodeEmitter.h" -#include "ByteCodeGenError.h" #include "Context.h" #include "Floating.h" #include "IntegralAP.h" diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp index efe10458f779..d887170cbc5d 100644 --- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp +++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp @@ -8,7 +8,6 @@ #include "ByteCodeExprGen.h" #include "ByteCodeEmitter.h" -#include "ByteCodeGenError.h" #include "ByteCodeStmtGen.h" #include "Context.h" #include "Floating.h" diff --git a/clang/lib/AST/Interp/ByteCodeGenError.cpp b/clang/lib/AST/Interp/ByteCodeGenError.cpp deleted file mode 100644 index 5fd3d77c3842..000000000000 --- a/clang/lib/AST/Interp/ByteCodeGenError.cpp +++ /dev/null @@ -1,14 +0,0 @@ -//===--- ByteCodeGenError.h - Byte code generation error --------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "ByteCodeGenError.h" - -using namespace clang; -using namespace clang::interp; - -char ByteCodeGenError::ID; diff --git a/clang/lib/AST/Interp/ByteCodeGenError.h b/clang/lib/AST/Interp/ByteCodeGenError.h deleted file mode 100644 index af464b5ed4ab..000000000000 --- a/clang/lib/AST/Interp/ByteCodeGenError.h +++ /dev/null @@ -1,46 +0,0 @@ -//===--- ByteCodeGenError.h - Byte code generation error ----------*- C -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CLANG_AST_INTERP_BYTECODEGENERROR_H -#define LLVM_CLANG_AST_INTERP_BYTECODEGENERROR_H - -#include "clang/AST/Decl.h" -#include "clang/AST/Stmt.h" -#include "clang/Basic/SourceLocation.h" -#include "llvm/Support/Error.h" - -namespace clang { -namespace interp { - -/// Error thrown by the compiler. -struct ByteCodeGenError : public llvm::ErrorInfo { -public: - ByteCodeGenError(SourceRange Range) : Range(Range) {} - ByteCodeGenError(const Stmt *S) : ByteCodeGenError(S->getSourceRange()) {} - ByteCodeGenError(const Decl *D) : ByteCodeGenError(D->getSourceRange()) {} - - void log(raw_ostream &OS) const override { OS << "unimplemented feature"; } - - const SourceRange &getRange() const { return Range; } - - static char ID; - -private: - // Range of the item where the error occurred. - SourceRange Range; - - // Users are not expected to use error_code. - std::error_code convertToErrorCode() const override { - return llvm::inconvertibleErrorCode(); - } -}; - -} // namespace interp -} // namespace clang - -#endif diff --git a/clang/lib/AST/Interp/ByteCodeStmtGen.cpp b/clang/lib/AST/Interp/ByteCodeStmtGen.cpp index d9213b12cbd0..6da3860f98d8 100644 --- a/clang/lib/AST/Interp/ByteCodeStmtGen.cpp +++ b/clang/lib/AST/Interp/ByteCodeStmtGen.cpp @@ -8,7 +8,6 @@ #include "ByteCodeStmtGen.h" #include "ByteCodeEmitter.h" -#include "ByteCodeGenError.h" #include "Context.h" #include "Function.h" #include "PrimType.h" diff --git a/clang/lib/AST/Interp/Context.cpp b/clang/lib/AST/Interp/Context.cpp index 017095352dc2..15a9d46880e9 100644 --- a/clang/lib/AST/Interp/Context.cpp +++ b/clang/lib/AST/Interp/Context.cpp @@ -9,7 +9,6 @@ #include "Context.h" #include "ByteCodeEmitter.h" #include "ByteCodeExprGen.h" -#include "ByteCodeGenError.h" #include "ByteCodeStmtGen.h" #include "EvalEmitter.h" #include "Interp.h" @@ -208,17 +207,6 @@ bool Context::Run(State &Parent, const Function *Func, APValue &Result) { return false; } -bool Context::Check(State &Parent, llvm::Expected &&Flag) { - if (Flag) - return *Flag; - handleAllErrors(Flag.takeError(), [&Parent](ByteCodeGenError &Err) { - Parent.FFDiag(Err.getRange().getBegin(), - diag::err_experimental_clang_interp_failed) - << Err.getRange(); - }); - return false; -} - // TODO: Virtual bases? const CXXMethodDecl * Context::getOverridingFunction(const CXXRecordDecl *DynamicDecl, diff --git a/clang/lib/AST/Interp/Context.h b/clang/lib/AST/Interp/Context.h index dbb63e369181..23c439ad8912 100644 --- a/clang/lib/AST/Interp/Context.h +++ b/clang/lib/AST/Interp/Context.h @@ -108,9 +108,6 @@ private: /// Runs a function. bool Run(State &Parent, const Function *Func, APValue &Result); - /// Checks a result from the interpreter. - bool Check(State &Parent, llvm::Expected &&R); - /// Current compilation context. ASTContext &Ctx; /// Interpreter stack, shared across invocations. diff --git a/clang/lib/AST/Interp/EvalEmitter.cpp b/clang/lib/AST/Interp/EvalEmitter.cpp index bfd502d21b4c..caffb69d83e3 100644 --- a/clang/lib/AST/Interp/EvalEmitter.cpp +++ b/clang/lib/AST/Interp/EvalEmitter.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "EvalEmitter.h" -#include "ByteCodeGenError.h" #include "Context.h" #include "IntegralAP.h" #include "Interp.h" -- GitLab From 17efdad2296a2757813e4f11d0575ee6fb826e39 Mon Sep 17 00:00:00 2001 From: Hongyu Chen <46539970+yugier@users.noreply.github.com> Date: Mon, 4 Mar 2024 08:52:45 -0800 Subject: [PATCH 010/929] [ORC][JITLink] Add Intel VTune support to JITLink (#81826) This patch adds two plugins: VTuneSupportPlugin.cpp and JITLoaderVTune.cpp. The testing is done in a manner similar to llvm-jitlistener. Currently, we only support the old version of Intel VTune API. This pull request is stacked on top of https://github.com/llvm/llvm-project/pull/81825 --- .../Orc/Debugging/VTuneSupportPlugin.h | 145 ++++++++++++ .../Orc/TargetProcess/JITLoaderVTune.h | 31 +++ .../Orc/Debugging/CMakeLists.txt | 1 + .../Orc/Debugging/VTuneSupportPlugin.cpp | 188 +++++++++++++++ .../Orc/TargetProcess/CMakeLists.txt | 9 + .../Orc/TargetProcess/JITLoaderVTune.cpp | 217 ++++++++++++++++++ .../JITLink/x86-64/ELF_vtune.s | 52 +++++ .../JITLink/x86-64/lit.local.cfg | 3 + llvm/tools/llvm-jitlink/llvm-jitlink.cpp | 18 +- 9 files changed, 663 insertions(+), 1 deletion(-) create mode 100644 llvm/include/llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h create mode 100644 llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h create mode 100644 llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp create mode 100644 llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.cpp create mode 100644 llvm/test/ExecutionEngine/JITLink/x86-64/ELF_vtune.s diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h b/llvm/include/llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h new file mode 100644 index 000000000000..5ded714c5f88 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h @@ -0,0 +1,145 @@ +//===--- VTuneSupportPlugin.h -- Support for VTune profiler ---*- C++ -*---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Handles support for registering code with VIntel Tune's Amplifier JIT API. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_AMPLIFIERSUPPORTPLUGIN_H +#define LLVM_EXECUTIONENGINE_ORC_AMPLIFIERSUPPORTPLUGIN_H + +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" + +#include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h" + +namespace llvm { + +namespace orc { + +class VTuneSupportPlugin : public ObjectLinkingLayer::Plugin { +public: + VTuneSupportPlugin(ExecutorProcessControl &EPC, ExecutorAddr RegisterImplAddr, + ExecutorAddr UnregisterImplAddr, bool EmitDebugInfo) + : EPC(EPC), RegisterVTuneImplAddr(RegisterImplAddr), + UnregisterVTuneImplAddr(UnregisterImplAddr), + EmitDebugInfo(EmitDebugInfo) {} + + void modifyPassConfig(MaterializationResponsibility &MR, + jitlink::LinkGraph &G, + jitlink::PassConfiguration &Config) override; + + Error notifyEmitted(MaterializationResponsibility &MR) override; + Error notifyFailed(MaterializationResponsibility &MR) override; + Error notifyRemovingResources(JITDylib &JD, ResourceKey K) override; + void notifyTransferringResources(JITDylib &JD, ResourceKey DstKey, + ResourceKey SrcKey) override; + + static Expected> + Create(ExecutorProcessControl &EPC, JITDylib &JD, bool EmitDebugInfo, + bool TestMode = false); + +private: + ExecutorProcessControl &EPC; + ExecutorAddr RegisterVTuneImplAddr; + ExecutorAddr UnregisterVTuneImplAddr; + std::mutex PluginMutex; + uint64_t NextMethodID{0}; + DenseMap> + PendingMethodIDs; + DenseMap>> + LoadedMethodIDs; + bool EmitDebugInfo; +}; + +typedef std::vector> VTuneLineTable; + +// SI = String Index, 1-indexed into the VTuneMethodBatch::Strings table. +// SI == 0 means replace with nullptr. + +// MI = Method Index, 1-indexed into the VTuneMethodBatch::Methods table. +// MI == 0 means this is a parent method and was not inlined. + +struct VTuneMethodInfo { + VTuneLineTable LineTable; + ExecutorAddr LoadAddr; + uint64_t LoadSize; + uint64_t MethodID; + uint32_t NameSI; + uint32_t ClassFileSI; + uint32_t SourceFileSI; + uint32_t ParentMI; +}; + +typedef std::vector VTuneMethodTable; +typedef std::vector VTuneStringTable; + +struct VTuneMethodBatch { + VTuneMethodTable Methods; + VTuneStringTable Strings; +}; + +typedef std::vector> VTuneUnloadedMethodIDs; + +namespace shared { + +using SPSVTuneLineTable = SPSSequence>; +using SPSVTuneMethodInfo = + SPSTuple; +using SPSVTuneMethodTable = SPSSequence; +using SPSVTuneStringTable = SPSSequence; +using SPSVTuneMethodBatch = SPSTuple; +using SPSVTuneUnloadedMethodIDs = SPSSequence>; + +template <> class SPSSerializationTraits { +public: + static size_t size(const VTuneMethodInfo &MI) { + return SPSVTuneMethodInfo::AsArgList::size( + MI.LineTable, MI.LoadAddr, MI.LoadSize, MI.MethodID, MI.NameSI, + MI.ClassFileSI, MI.SourceFileSI, MI.ParentMI); + } + + static bool deserialize(SPSInputBuffer &IB, VTuneMethodInfo &MI) { + return SPSVTuneMethodInfo::AsArgList::deserialize( + IB, MI.LineTable, MI.LoadAddr, MI.LoadSize, MI.MethodID, MI.NameSI, + MI.ClassFileSI, MI.SourceFileSI, MI.ParentMI); + } + + static bool serialize(SPSOutputBuffer &OB, const VTuneMethodInfo &MI) { + return SPSVTuneMethodInfo::AsArgList::serialize( + OB, MI.LineTable, MI.LoadAddr, MI.LoadSize, MI.MethodID, MI.NameSI, + MI.ClassFileSI, MI.SourceFileSI, MI.ParentMI); + } +}; + +template <> +class SPSSerializationTraits { +public: + static size_t size(const VTuneMethodBatch &MB) { + return SPSVTuneMethodBatch::AsArgList::size(MB.Methods, MB.Strings); + } + + static bool deserialize(SPSInputBuffer &IB, VTuneMethodBatch &MB) { + return SPSVTuneMethodBatch::AsArgList::deserialize(IB, MB.Methods, + MB.Strings); + } + + static bool serialize(SPSOutputBuffer &OB, const VTuneMethodBatch &MB) { + return SPSVTuneMethodBatch::AsArgList::serialize(OB, MB.Methods, + MB.Strings); + } +}; + +} // end namespace shared + +} // end namespace orc + +} // end namespace llvm + +#endif diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h new file mode 100644 index 000000000000..a9fa9967ace9 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h @@ -0,0 +1,31 @@ + +//===------- JITLoaderVTune.h --- Register profiler objects ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Register objects for access by profilers via the perf JIT interface. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_JITLOADERVTUNE_H +#define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_JITLOADERVTUNE_H + +#include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" +#include + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_registerVTuneImpl(const char *Data, uint64_t Size); + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_unregisterVTuneImpl(const char *Data, uint64_t Size); + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_test_registerVTuneImpl(const char *Data, uint64_t Size); + +#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_JITLOADERVTUNE_H + + diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/Debugging/CMakeLists.txt index 5bf23a7ec0bc..ed52692662a8 100644 --- a/llvm/lib/ExecutionEngine/Orc/Debugging/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/Orc/Debugging/CMakeLists.txt @@ -8,6 +8,7 @@ add_llvm_component_library(LLVMOrcDebugging DebuggerSupportPlugin.cpp LLJITUtilsCBindings.cpp PerfSupportPlugin.cpp + VTuneSupportPlugin.cpp ADDITIONAL_HEADER_DIRS ${LLVM_MAIN_INCLUDE_DIR}/llvm/ExecutionEngine/Orc/Debugging/ diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp new file mode 100644 index 000000000000..e21ed4bf84d0 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp @@ -0,0 +1,188 @@ +//===--- VTuneSupportPlugin.cpp -- Support for VTune profiler --*- C++ -*--===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Handles support for registering code with VIntel Tune's Amplfiier JIT API. +// +//===----------------------------------------------------------------------===// +#include "llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h" +#include "llvm/DebugInfo/DWARF/DWARFContext.h" +#include "llvm/ExecutionEngine/Orc/Debugging/DebugInfoSupport.h" + +using namespace llvm; +using namespace llvm::orc; +using namespace llvm::jitlink; + +namespace { + +constexpr StringRef RegisterVTuneImplName = "llvm_orc_registerVTuneImpl"; +constexpr StringRef UnregisterVTuneImplName = "llvm_orc_unregisterVTuneImpl"; +constexpr StringRef RegisterTestVTuneImplName = + "llvm_orc_test_registerVTuneImpl"; + +static VTuneMethodBatch getMethodBatch(LinkGraph &G, bool EmitDebugInfo) { + VTuneMethodBatch Batch; + std::unique_ptr DC; + StringMap> DCBacking; + if (EmitDebugInfo) { + auto EDC = createDWARFContext(G); + if (!EDC) { + EmitDebugInfo = false; + } else { + DC = std::move(EDC->first); + DCBacking = std::move(EDC->second); + } + } + + auto GetStringIdx = [Deduplicator = StringMap(), + &Batch](StringRef S) mutable { + auto I = Deduplicator.find(S); + if (I != Deduplicator.end()) + return I->second; + + Batch.Strings.push_back(S.str()); + return Deduplicator[S] = Batch.Strings.size(); + }; + for (auto Sym : G.defined_symbols()) { + if (!Sym->isCallable()) + continue; + + Batch.Methods.push_back(VTuneMethodInfo()); + auto &Method = Batch.Methods.back(); + Method.MethodID = 0; + Method.ParentMI = 0; + Method.LoadAddr = Sym->getAddress(); + Method.LoadSize = Sym->getSize(); + Method.NameSI = GetStringIdx(Sym->getName()); + Method.ClassFileSI = 0; + Method.SourceFileSI = 0; + + if (!EmitDebugInfo) + continue; + + auto &Section = Sym->getBlock().getSection(); + auto Addr = Sym->getAddress(); + auto SAddr = + object::SectionedAddress{Addr.getValue(), Section.getOrdinal()}; + DILineInfoTable LinesInfo = DC->getLineInfoForAddressRange( + SAddr, Sym->getSize(), + DILineInfoSpecifier::FileLineInfoKind::AbsoluteFilePath); + Method.SourceFileSI = Batch.Strings.size(); + Batch.Strings.push_back(DC->getLineInfoForAddress(SAddr).FileName); + for (auto &LInfo : LinesInfo) { + Method.LineTable.push_back( + std::pair{/*unsigned*/ Sym->getOffset(), + /*DILineInfo*/ LInfo.second.Line}); + } + } + return Batch; +} + +} // namespace + +void VTuneSupportPlugin::modifyPassConfig(MaterializationResponsibility &MR, + LinkGraph &G, + PassConfiguration &Config) { + Config.PostFixupPasses.push_back([this, MR = &MR](LinkGraph &G) { + // the object file is generated but not linked yet + auto Batch = getMethodBatch(G, EmitDebugInfo); + if (Batch.Methods.empty()) { + return Error::success(); + } + { + std::lock_guard Lock(PluginMutex); + uint64_t Allocated = Batch.Methods.size(); + uint64_t Start = NextMethodID; + NextMethodID += Allocated; + for (size_t i = Start; i < NextMethodID; ++i) { + Batch.Methods[i - Start].MethodID = i; + } + this->PendingMethodIDs[MR] = {Start, Allocated}; + } + G.allocActions().push_back( + {cantFail(shared::WrapperFunctionCall::Create< + shared::SPSArgList>( + RegisterVTuneImplAddr, Batch)), + {}}); + return Error::success(); + }); +} + +Error VTuneSupportPlugin::notifyEmitted(MaterializationResponsibility &MR) { + if (auto Err = MR.withResourceKeyDo([this, MR = &MR](ResourceKey K) { + std::lock_guard Lock(PluginMutex); + auto I = PendingMethodIDs.find(MR); + if (I == PendingMethodIDs.end()) + return; + + LoadedMethodIDs[K].push_back(I->second); + PendingMethodIDs.erase(I); + })) { + return Err; + } + return Error::success(); +} + +Error VTuneSupportPlugin::notifyFailed(MaterializationResponsibility &MR) { + std::lock_guard Lock(PluginMutex); + PendingMethodIDs.erase(&MR); + return Error::success(); +} + +Error VTuneSupportPlugin::notifyRemovingResources(JITDylib &JD, ResourceKey K) { + // Unregistration not required if not provided + if (!UnregisterVTuneImplAddr) { + return Error::success(); + } + VTuneUnloadedMethodIDs UnloadedIDs; + { + std::lock_guard Lock(PluginMutex); + auto I = LoadedMethodIDs.find(K); + if (I == LoadedMethodIDs.end()) + return Error::success(); + + UnloadedIDs = std::move(I->second); + LoadedMethodIDs.erase(I); + } + if (auto Err = EPC.callSPSWrapper( + UnregisterVTuneImplAddr, UnloadedIDs)) + return Err; + + return Error::success(); +} + +void VTuneSupportPlugin::notifyTransferringResources(JITDylib &JD, + ResourceKey DstKey, + ResourceKey SrcKey) { + std::lock_guard Lock(PluginMutex); + auto I = LoadedMethodIDs.find(SrcKey); + if (I == LoadedMethodIDs.end()) + return; + + auto &Dest = LoadedMethodIDs[DstKey]; + Dest.insert(Dest.end(), I->second.begin(), I->second.end()); + LoadedMethodIDs.erase(SrcKey); +} + +Expected> +VTuneSupportPlugin::Create(ExecutorProcessControl &EPC, JITDylib &JD, + bool EmitDebugInfo, bool TestMode) { + auto &ES = EPC.getExecutionSession(); + auto RegisterImplName = + ES.intern(TestMode ? RegisterTestVTuneImplName : RegisterVTuneImplName); + auto UnregisterImplName = ES.intern(UnregisterVTuneImplName); + SymbolLookupSet SLS{RegisterImplName, UnregisterImplName}; + auto Res = ES.lookup(makeJITDylibSearchOrder({&JD}), std::move(SLS)); + if (!Res) + return Res.takeError(); + ExecutorAddr RegisterImplAddr( + Res->find(RegisterImplName)->second.getAddress()); + ExecutorAddr UnregisterImplAddr( + Res->find(UnregisterImplName)->second.getAddress()); + return std::make_unique( + EPC, RegisterImplAddr, UnregisterImplAddr, EmitDebugInfo); +} diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt index f2005dc1775e..4ab8dae4a9ba 100644 --- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt @@ -2,10 +2,18 @@ if( CMAKE_HOST_UNIX AND HAVE_LIBRT ) set(rt_lib rt) endif() +set(intel_jit_profiling ) +if( LLVM_USE_INTEL_JITEVENTS ) +set(intel_jit_profiling IntelJITProfiling) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../IntelJITProfiling) +include_directories(${PROJECT_BINARY_DIR}/ittapi/include/ ) +endif() + add_llvm_component_library(LLVMOrcTargetProcess ExecutorSharedMemoryMapperService.cpp JITLoaderGDB.cpp JITLoaderPerf.cpp + JITLoaderVTune.cpp OrcRTBootstrap.cpp RegisterEHFrames.cpp SimpleExecutorDylibManager.cpp @@ -21,6 +29,7 @@ add_llvm_component_library(LLVMOrcTargetProcess ${rt_lib} LINK_COMPONENTS + ${intel_jit_profiling} OrcShared Support TargetParser diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.cpp new file mode 100644 index 000000000000..e109d230bc62 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.cpp @@ -0,0 +1,217 @@ +//===------- JITLoaderVTune.cpp - Register profiler objects -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Register objects for access by profilers via the VTune JIT interface. +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h" +#include "llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h" + +#if LLVM_USE_INTEL_JITEVENTS +#include "IntelJITEventsWrapper.h" +#include "ittnotify.h" + +using namespace llvm; +using namespace llvm::orc; + +static std::unique_ptr Wrapper; + +static Error registerJITLoaderVTuneRegisterImpl(const VTuneMethodBatch &MB) { + const size_t StringsSize = MB.Strings.size(); + + for (size_t i = 0; i < MB.Methods.size(); ++i) { + VTuneMethodInfo MethodInfo = MB.Methods.at(i); + iJIT_Method_Load MethodMessage; + memset(&MethodMessage, 0, sizeof(iJIT_Method_Load)); + + MethodMessage.method_id = MethodInfo.MethodID; + if (MethodInfo.NameSI != 0 && MethodInfo.NameSI < StringsSize) { + MethodMessage.method_name = + const_cast(MB.Strings.at(MethodInfo.NameSI).data()); + } else { + MethodMessage.method_name = NULL; + } + if (MethodInfo.ClassFileSI != 0 && MethodInfo.ClassFileSI < StringsSize) { + MethodMessage.class_file_name = + const_cast(MB.Strings.at(MethodInfo.ClassFileSI).data()); + } else { + MethodMessage.class_file_name = NULL; + } + if (MethodInfo.SourceFileSI != 0 && MethodInfo.SourceFileSI < StringsSize) { + MethodMessage.source_file_name = + const_cast(MB.Strings.at(MethodInfo.SourceFileSI).data()); + } else { + MethodMessage.source_file_name = NULL; + } + + MethodMessage.method_load_address = MethodInfo.LoadAddr.toPtr(); + MethodMessage.method_size = MethodInfo.LoadSize; + MethodMessage.class_id = 0; + + MethodMessage.user_data = NULL; + MethodMessage.user_data_size = 0; + MethodMessage.env = iJDE_JittingAPI; + + std::vector LineInfo; + for (const auto &LInfo : MethodInfo.LineTable) { + LineInfo.push_back(LineNumberInfo{LInfo.first, LInfo.second}); + } + + if (LineInfo.size() == 0) { + MethodMessage.line_number_size = 0; + MethodMessage.line_number_table = 0; + } else { + MethodMessage.line_number_size = LineInfo.size(); + MethodMessage.line_number_table = &*LineInfo.begin(); + } + Wrapper->iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, + &MethodMessage); + } + + return Error::success(); +} + +static void registerJITLoaderVTuneUnregisterImpl( + const std::vector> &UM) { + for (auto &Method : UM) { + Wrapper->iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_UNLOAD_START, + const_cast(&Method.first)); + } +} + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_registerVTuneImpl(const char *Data, uint64_t Size) { + using namespace orc::shared; + if (!Wrapper) + Wrapper.reset(new IntelJITEventsWrapper); + + return WrapperFunction::handle( + Data, Size, registerJITLoaderVTuneRegisterImpl) + .release(); +} + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_unregisterVTuneImpl(const char *Data, uint64_t Size) { + using namespace orc::shared; + return WrapperFunction::handle( + Data, Size, registerJITLoaderVTuneUnregisterImpl) + .release(); +} + +// For Testing: following code comes from llvm-jitlistener.cpp in llvm tools +namespace { +typedef std::vector> SourceLocations; +typedef std::map NativeCodeMap; +NativeCodeMap ReportedDebugFuncs; +} // namespace + +static int NotifyEvent(iJIT_JVM_EVENT EventType, void *EventSpecificData) { + switch (EventType) { + case iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED: { + if (!EventSpecificData) { + errs() << "Error: The JIT event listener did not provide a event data."; + return -1; + } + iJIT_Method_Load *msg = static_cast(EventSpecificData); + + ReportedDebugFuncs[msg->method_id]; + + outs() << "Method load [" << msg->method_id << "]: " << msg->method_name + << ", Size = " << msg->method_size << "\n"; + + for (unsigned int i = 0; i < msg->line_number_size; ++i) { + if (!msg->line_number_table) { + errs() << "A function with a non-zero line count had no line table."; + return -1; + } + std::pair loc( + std::string(msg->source_file_name), + msg->line_number_table[i].LineNumber); + ReportedDebugFuncs[msg->method_id].push_back(loc); + outs() << " Line info @ " << msg->line_number_table[i].Offset << ": " + << msg->source_file_name << ", line " + << msg->line_number_table[i].LineNumber << "\n"; + } + outs() << "\n"; + } break; + case iJVM_EVENT_TYPE_METHOD_UNLOAD_START: { + if (!EventSpecificData) { + errs() << "Error: The JIT event listener did not provide a event data."; + return -1; + } + unsigned int UnloadId = + *reinterpret_cast(EventSpecificData); + assert(1 == ReportedDebugFuncs.erase(UnloadId)); + outs() << "Method unload [" << UnloadId << "]\n"; + } break; + default: + break; + } + return 0; +} + +static iJIT_IsProfilingActiveFlags IsProfilingActive(void) { + // for testing, pretend we have an Intel Parallel Amplifier XE 2011 + // instance attached + return iJIT_SAMPLING_ON; +} + +static unsigned int GetNewMethodID(void) { + static unsigned int id = 0; + return ++id; +} + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_test_registerVTuneImpl(const char *Data, uint64_t Size) { + using namespace orc::shared; + Wrapper.reset(new IntelJITEventsWrapper( + NotifyEvent, NULL, NULL, IsProfilingActive, 0, 0, GetNewMethodID)); + return WrapperFunction::handle( + Data, Size, registerJITLoaderVTuneRegisterImpl) + .release(); +} + +#else + +using namespace llvm; +using namespace llvm::orc; + +static Error unsupportedBatch(const VTuneMethodBatch &MB) { + return llvm::make_error("unsupported for Intel VTune", + inconvertibleErrorCode()); +} + +static void unsuppported(const std::vector> &UM) { + +} + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_registerVTuneImpl(const char *Data, uint64_t Size) { + using namespace orc::shared; + return WrapperFunction::handle( + Data, Size, unsupportedBatch) + .release(); +} + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_unregisterVTuneImpl(const char *Data, uint64_t Size) { + using namespace orc::shared; + return WrapperFunction::handle(Data, Size, + unsuppported) + .release(); +} + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_test_registerVTuneImpl(const char *Data, uint64_t Size) { + using namespace orc::shared; + return WrapperFunction::handle( + Data, Size, unsupportedBatch) + .release(); +} + +#endif diff --git a/llvm/test/ExecutionEngine/JITLink/x86-64/ELF_vtune.s b/llvm/test/ExecutionEngine/JITLink/x86-64/ELF_vtune.s new file mode 100644 index 000000000000..0126b9ff3a61 --- /dev/null +++ b/llvm/test/ExecutionEngine/JITLink/x86-64/ELF_vtune.s @@ -0,0 +1,52 @@ +# REQUIRES: native && x86_64-linux && intel-jitevents + +# RUN: rm -rf %t && mkdir -p %t +# RUN: llvm-mc -triple=x86_64-unknown-linux -position-independent \ +# RUN: -filetype=obj -o %t/ELF_x86-64_vtune.o %s +# RUN: llvm-jitlink -vtune-support %t/ELF_x86-64_vtune.o | \ +# RUN: FileCheck %s + +# CHECK: Method load [0]: {{.*}}, Size = {{[0-9]+}} +# CHECK: Method unload [0] + .file "test.c" + .text + .globl main + .type main, @function +main: +.LFB0: + .cfi_startproc + endbr64 + pushq %rbp + .cfi_def_cfa_offset 16 + .cfi_offset 6, -16 + movq %rsp, %rbp + .cfi_def_cfa_register 6 + movl %edi, -4(%rbp) + movq %rsi, -16(%rbp) + movl -4(%rbp), %ebx + addl $1, %ebx + movl $0, %eax + popq %rbp + .cfi_def_cfa 7, 8 + ret + .cfi_endproc +.LFE0: + .size main, .-main + .ident "GCC: (Ubuntu 9.4.0-1ubuntu1~20.04.2) 9.4.0" + .section .note.GNU-stack,"",@progbits + .section .note.gnu.property,"a" + .align 8 + .long 1f - 0f + .long 4f - 1f + .long 5 +0: + .string "GNU" +1: + .align 8 + .long 0xc0000002 + .long 3f - 2f +2: + .long 0x3 +3: + .align 8 +4: diff --git a/llvm/test/ExecutionEngine/JITLink/x86-64/lit.local.cfg b/llvm/test/ExecutionEngine/JITLink/x86-64/lit.local.cfg index 42bf50dcc13c..d5a1ad626b65 100644 --- a/llvm/test/ExecutionEngine/JITLink/x86-64/lit.local.cfg +++ b/llvm/test/ExecutionEngine/JITLink/x86-64/lit.local.cfg @@ -1,2 +1,5 @@ if not "X86" in config.root.targets: config.unsupported = True + +if config.llvm_use_intel_jitevents: + config.available_features.add("intel-jitevents") diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp index f0b8310a32ef..0217f645a0f6 100644 --- a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp +++ b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp @@ -21,6 +21,7 @@ #include "llvm/ExecutionEngine/Orc/Debugging/DebugInfoSupport.h" #include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.h" #include "llvm/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.h" +#include "llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h" #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h" #include "llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h" #include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h" @@ -34,6 +35,7 @@ #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderPerf.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" @@ -148,6 +150,10 @@ static cl::opt PerfSupport("perf-support", cl::init(false), cl::Hidden, cl::cat(JITLinkCategory)); +static cl::opt VTuneSupport("vtune-support", + cl::desc("Enable vtune profiling support"), + cl::init(false), cl::Hidden, + cl::cat(JITLinkCategory)); static cl::opt NoProcessSymbols("no-process-syms", cl::desc("Do not resolve to llvm-jitlink process symbols"), @@ -264,7 +270,10 @@ static LLVM_ATTRIBUTE_USED void linkComponents() { << (void *)&llvm_orc_registerJITLoaderGDBAllocAction << '\n' << (void *)&llvm_orc_registerJITLoaderPerfStart << '\n' << (void *)&llvm_orc_registerJITLoaderPerfEnd << '\n' - << (void *)&llvm_orc_registerJITLoaderPerfImpl << '\n'; + << (void *)&llvm_orc_registerJITLoaderPerfImpl << '\n' + << (void *)&llvm_orc_registerVTuneImpl << '\n' + << (void *)&llvm_orc_unregisterVTuneImpl << '\n' + << (void *)&llvm_orc_test_registerVTuneImpl << '\n'; } static bool UseTestResultOverride = false; @@ -1004,6 +1013,13 @@ Session::Session(std::unique_ptr EPC, Error &Err) this->ES.getExecutorProcessControl(), *ProcessSymsJD, true, true))); } + if (VTuneSupport && TT.isOSBinFormatELF()) { + ObjLayer.addPlugin(ExitOnErr(DebugInfoPreservationPlugin::Create())); + ObjLayer.addPlugin(ExitOnErr(VTuneSupportPlugin::Create( + this->ES.getExecutorProcessControl(), *ProcessSymsJD, /*EmitDebugInfo=*/true, + /*TestMode=*/true))); + } + // Set up the platform. if (!OrcRuntime.empty()) { assert(ProcessSymsJD && "ProcessSymsJD should have been set"); -- GitLab From 57a720872128ea21e8ed22cdd9ae4c62154e6fb1 Mon Sep 17 00:00:00 2001 From: ykhatav Date: Mon, 4 Mar 2024 11:56:28 -0500 Subject: [PATCH 011/929] Fix a use-after-move bug in DWARFVerifier constructor (#83621) Resolve a use-after-move bug for the parameter "DumpOpts" in the DWARFVerifier constructor. --- llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp b/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp index 520debe513d9..4ef6c80ed028 100644 --- a/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp +++ b/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp @@ -1083,7 +1083,8 @@ DWARFVerifier::DWARFVerifier(raw_ostream &S, DWARFContext &D, DIDumpOptions DumpOpts) : OS(S), DCtx(D), DumpOpts(std::move(DumpOpts)), IsObjectFile(false), IsMachOObject(false) { - ErrorCategory.ShowDetail(DumpOpts.Verbose || !DumpOpts.ShowAggregateErrors); + ErrorCategory.ShowDetail(this->DumpOpts.Verbose || + !this->DumpOpts.ShowAggregateErrors); if (const auto *F = DCtx.getDWARFObj().getFile()) { IsObjectFile = F->isRelocatableObject(); IsMachOObject = F->isMachO(); -- GitLab From f83858f87c5d7a634af717f18a7a53ed5ddde3bd Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Mon, 4 Mar 2024 08:56:54 -0800 Subject: [PATCH 012/929] [nfc][InstrProfiling]Compute a boolean state as a constant and use it everywhere (#83756) --- .../Instrumentation/InstrProfiling.cpp | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp index dbd44bd36e11..d5d55dec6382 100644 --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -168,13 +168,33 @@ cl::opt SkipRetExitBlock( using LoadStorePair = std::pair; +static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag) { + auto *MD = dyn_cast_or_null(M.getModuleFlag(Flag)); + if (!MD) + return 0; + + // If the flag is a ConstantAsMetadata, it should be an integer representable + // in 64-bits. + return cast(MD->getValue())->getZExtValue(); +} + +static bool enablesValueProfiling(const Module &M) { + return isIRPGOFlagSet(&M) || + getIntModuleFlagOrZero(M, "EnableValueProfiling") != 0; +} + +// Conservatively returns true if value profiling is enabled. +static bool profDataReferencedByCode(const Module &M) { + return enablesValueProfiling(M); +} + class InstrLowerer final { public: InstrLowerer(Module &M, const InstrProfOptions &Options, std::function GetTLI, bool IsCS) : M(M), Options(Options), TT(Triple(M.getTargetTriple())), IsCS(IsCS), - GetTLI(GetTLI) {} + GetTLI(GetTLI), DataReferencedByCode(profDataReferencedByCode(M)) {} bool lower(); @@ -186,6 +206,8 @@ private: const bool IsCS; std::function GetTLI; + + const bool DataReferencedByCode; struct PerFunctionProfileData { uint32_t NumValueSites[IPVK_Last + 1] = {}; GlobalVariable *RegionCounters = nullptr; @@ -1057,26 +1079,6 @@ static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix, return (Prefix + Name + "." + Twine(FuncHash)).str(); } -static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag) { - auto *MD = dyn_cast_or_null(M.getModuleFlag(Flag)); - if (!MD) - return 0; - - // If the flag is a ConstantAsMetadata, it should be an integer representable - // in 64-bits. - return cast(MD->getValue())->getZExtValue(); -} - -static bool enablesValueProfiling(const Module &M) { - return isIRPGOFlagSet(&M) || - getIntModuleFlagOrZero(M, "EnableValueProfiling") != 0; -} - -// Conservatively returns true if data variables may be referenced by code. -static bool profDataReferencedByCode(const Module &M) { - return enablesValueProfiling(M); -} - static inline bool shouldRecordFunctionAddr(Function *F) { // Only record function addresses if IR PGO is enabled or if clang value // profiling is enabled. Recording function addresses greatly increases object @@ -1192,7 +1194,6 @@ static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) { void InstrLowerer::maybeSetComdat(GlobalVariable *GV, Function *Fn, StringRef VarName) { - bool DataReferencedByCode = profDataReferencedByCode(M); bool NeedComdat = needsComdatForCounter(*Fn, M); bool UseComdat = (NeedComdat || TT.isOSBinFormatELF()); @@ -1418,7 +1419,6 @@ void InstrLowerer::createDataVariable(InstrProfCntrInstBase *Inc) { Visibility = GlobalValue::DefaultVisibility; } - bool DataReferencedByCode = profDataReferencedByCode(M); bool NeedComdat = needsComdatForCounter(*Fn, M); bool Renamed; @@ -1719,7 +1719,7 @@ void InstrLowerer::emitUses() { // and ensure this GC property as well. Otherwise, we have to conservatively // make all of the sections retained by the linker. if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() || - (TT.isOSBinFormatCOFF() && !profDataReferencedByCode(M))) + (TT.isOSBinFormatCOFF() && !DataReferencedByCode)) appendToCompilerUsed(M, CompilerUsedVars); else appendToUsed(M, CompilerUsedVars); -- GitLab From af9c5ca820499b932e23d7644e90ea5ff04c3c5f Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Mon, 4 Mar 2024 16:57:42 +0000 Subject: [PATCH 013/929] [gn build] Port 17efdad2296a --- .../gn/secondary/llvm/lib/ExecutionEngine/Orc/Debugging/BUILD.gn | 1 + .../llvm/lib/ExecutionEngine/Orc/TargetProcess/BUILD.gn | 1 + 2 files changed, 2 insertions(+) diff --git a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Debugging/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Debugging/BUILD.gn index 1d3fc6cfdfaa..5610679ff333 100644 --- a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Debugging/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Debugging/BUILD.gn @@ -13,6 +13,7 @@ static_library("Debugging") { "DebuggerSupportPlugin.cpp", "LLJITUtilsCBindings.cpp", "PerfSupportPlugin.cpp", + "VTuneSupportPlugin.cpp", ] if (current_os == "linux") { libs = [ "rt" ] diff --git a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/TargetProcess/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/TargetProcess/BUILD.gn index f34855e8e1cc..d62f5042c946 100644 --- a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/TargetProcess/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/TargetProcess/BUILD.gn @@ -9,6 +9,7 @@ static_library("TargetProcess") { "ExecutorSharedMemoryMapperService.cpp", "JITLoaderGDB.cpp", "JITLoaderPerf.cpp", + "JITLoaderVTune.cpp", "OrcRTBootstrap.cpp", "RegisterEHFrames.cpp", "SimpleExecutorDylibManager.cpp", -- GitLab From e6dff54f362f9bcae2a3dd8b0bf7aa6fc9f11017 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Mon, 4 Mar 2024 16:57:43 +0000 Subject: [PATCH 014/929] [gn build] Port bc9c6c0c65e2 --- llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn | 1 - 1 file changed, 1 deletion(-) diff --git a/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn index cfdfd203fc62..a405878fecb2 100644 --- a/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn +++ b/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn @@ -94,7 +94,6 @@ static_library("AST") { "InheritViz.cpp", "Interp/ByteCodeEmitter.cpp", "Interp/ByteCodeExprGen.cpp", - "Interp/ByteCodeGenError.cpp", "Interp/ByteCodeStmtGen.cpp", "Interp/Context.cpp", "Interp/Descriptor.cpp", -- GitLab From 252f3c98db1383ee0e1f25020d488ffb7b4ac392 Mon Sep 17 00:00:00 2001 From: Michael Buch Date: Mon, 4 Mar 2024 17:17:13 +0000 Subject: [PATCH 015/929] [lldb][test] Add test for chained PCH debugging (#83582) Adds a test-case for debugging a program with a pch chain, that is, the main executable depends on a pch that itself included another pch. Currently clang doesn't emit the sekeleton CUs required for LLDB to track all types on the pch chain. Thus this test is XFAILed for now. --- .../API/lang/cpp/gmodules/pch-chain/Makefile | 10 +++ .../cpp/gmodules/pch-chain/TestPchChain.py | 73 +++++++++++++++++++ .../lang/cpp/gmodules/pch-chain/base-pch.h | 9 +++ .../API/lang/cpp/gmodules/pch-chain/main.cpp | 10 +++ .../API/lang/cpp/gmodules/pch-chain/pch.h | 16 ++++ 5 files changed, 118 insertions(+) create mode 100644 lldb/test/API/lang/cpp/gmodules/pch-chain/Makefile create mode 100644 lldb/test/API/lang/cpp/gmodules/pch-chain/TestPchChain.py create mode 100644 lldb/test/API/lang/cpp/gmodules/pch-chain/base-pch.h create mode 100644 lldb/test/API/lang/cpp/gmodules/pch-chain/main.cpp create mode 100644 lldb/test/API/lang/cpp/gmodules/pch-chain/pch.h diff --git a/lldb/test/API/lang/cpp/gmodules/pch-chain/Makefile b/lldb/test/API/lang/cpp/gmodules/pch-chain/Makefile new file mode 100644 index 000000000000..6477c25dedf7 --- /dev/null +++ b/lldb/test/API/lang/cpp/gmodules/pch-chain/Makefile @@ -0,0 +1,10 @@ +include Makefile.rules + +OBJECTS += main.o + +$(EXE): $(BUILDDIR)/main.o + +$(BUILDDIR)/main.o: main.cpp + $(CC) -cc1 -emit-pch -x c++-header -fmodule-format=obj -fmodules -O0 -dwarf-ext-refs -debug-info-kind=standalone $(SRCDIR)/base-pch.h -o base-pch.h.gch + $(CC) -cc1 -emit-pch -x c++-header -fmodule-format=obj -fmodules -O0 -dwarf-ext-refs -debug-info-kind=standalone -include-pch base-pch.h.gch $(SRCDIR)/pch.h -o pch.h.gch + $(CC) -cc1 -emit-obj -x c++ -fmodules -O0 -dwarf-ext-refs -debug-info-kind=standalone -include-pch pch.h.gch $(SRCDIR)/main.cpp -o $(BUILDDIR)/main.o diff --git a/lldb/test/API/lang/cpp/gmodules/pch-chain/TestPchChain.py b/lldb/test/API/lang/cpp/gmodules/pch-chain/TestPchChain.py new file mode 100644 index 000000000000..b08c6caa713b --- /dev/null +++ b/lldb/test/API/lang/cpp/gmodules/pch-chain/TestPchChain.py @@ -0,0 +1,73 @@ +""" +Tests that we correctly track AST layout info +(specifically alignment) when moving AST nodes +between several ClangASTImporter instances +(in this case, from a pch chain to executable +to expression AST). +""" + +import lldb +import os +from lldbsuite.test.decorators import * +from lldbsuite.test.lldbtest import * +from lldbsuite.test import lldbutil + + +class TestPchChain(TestBase): + @add_test_categories(["gmodules"]) + @expectedFailureAll("Chained pch debugging currently not fully supported") + def test_expr(self): + self.build() + exe = self.getBuildArtifact("a.out") + self.target = self.dbg.CreateTarget(exe) + self.assertTrue(self.target, VALID_TARGET) + lldbutil.run_break_set_by_file_and_line( + self, "main.cpp", 9, num_expected_locations=1 + ) + + self.runCmd("run", RUN_SUCCEEDED) + + self.expect( + "frame variable data", + substrs=["row = 1", "col = 2", "row = 3", "col = 4", "stride = 5"], + ) + + @add_test_categories(["gmodules"]) + @expectedFailureAll("Chained pch debugging currently not fully supported") + def test_frame_var(self): + self.build() + exe = self.getBuildArtifact("a.out") + self.target = self.dbg.CreateTarget(exe) + self.assertTrue(self.target, VALID_TARGET) + lldbutil.run_break_set_by_file_and_line( + self, "main.cpp", 9, num_expected_locations=1 + ) + + self.runCmd("run", RUN_SUCCEEDED) + + self.expect_expr( + "data", + result_type="MatrixData", + result_children=[ + ValueCheck( + name="section", + children=[ + ValueCheck( + name="origin", + children=[ + ValueCheck(name="row", value="1"), + ValueCheck(name="col", value="2"), + ], + ), + ValueCheck( + name="size", + children=[ + ValueCheck(name="row", value="3"), + ValueCheck(name="col", value="4"), + ], + ), + ], + ), + ValueCheck(name="stride", value="5"), + ], + ) diff --git a/lldb/test/API/lang/cpp/gmodules/pch-chain/base-pch.h b/lldb/test/API/lang/cpp/gmodules/pch-chain/base-pch.h new file mode 100644 index 000000000000..53dacd796f2c --- /dev/null +++ b/lldb/test/API/lang/cpp/gmodules/pch-chain/base-pch.h @@ -0,0 +1,9 @@ +#ifndef BASE_PCH_H_IN +#define BASE_PCH_H_IN + +struct [[gnu::aligned(128)]] RowCol { + unsigned row; + unsigned col; +}; + +#endif // _H_IN diff --git a/lldb/test/API/lang/cpp/gmodules/pch-chain/main.cpp b/lldb/test/API/lang/cpp/gmodules/pch-chain/main.cpp new file mode 100644 index 000000000000..5481f3fad1ff --- /dev/null +++ b/lldb/test/API/lang/cpp/gmodules/pch-chain/main.cpp @@ -0,0 +1,10 @@ +int main(int argc, const char *argv[]) { + struct MatrixData data = {0}; + data.section.origin.row = 1; + data.section.origin.col = 2; + data.section.size.row = 3; + data.section.size.col = 4; + data.stride = 5; + + return data.section.size.row; +} diff --git a/lldb/test/API/lang/cpp/gmodules/pch-chain/pch.h b/lldb/test/API/lang/cpp/gmodules/pch-chain/pch.h new file mode 100644 index 000000000000..6c373ddb1f13 --- /dev/null +++ b/lldb/test/API/lang/cpp/gmodules/pch-chain/pch.h @@ -0,0 +1,16 @@ +#ifndef PCH_H_IN +#define PCH_H_IN + +static const int kAlignment = 64; + +struct [[gnu::aligned(kAlignment)]] Submatrix { + struct RowCol origin; + struct RowCol size; +}; + +struct [[gnu::aligned(kAlignment)]] MatrixData { + struct Submatrix section; + unsigned stride; +}; + +#endif // _H_IN -- GitLab From bd7bce2c8465e9cb36a823846a52c9f553502575 Mon Sep 17 00:00:00 2001 From: Erich Keane Date: Mon, 4 Mar 2024 09:25:29 -0800 Subject: [PATCH 016/929] Fix null-deref thanks to an attribute on a global declarator chunk (#83611) This was reported (sort of) in a PR: #77703. The problem is that a declarator 'owns' an attributes allocation via an `AttributePool`. However, this example tries to copy a DeclaratorChunk from one Declarator to another, so when the temporary Declarator goes out of scope, it deletes the attribute it has tried to pass on via the chunk. This patch ensures that we copy the 'ownership' of the attribute correctly, and adds an assert to catch any other casess where this happens. Additionally, this was put in as a bug report, so this Fixes #83611 --- clang/docs/ReleaseNotes.rst | 4 +++- clang/include/clang/Sema/DeclSpec.h | 16 ++++++++++++++++ clang/include/clang/Sema/ParsedAttr.h | 6 ++++++ clang/lib/Parse/ParseDecl.cpp | 2 +- clang/lib/Sema/ParsedAttr.cpp | 6 ++++++ .../Parser/cxx-declarator-attribute-crash.cpp | 8 ++++++++ 6 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 clang/test/Parser/cxx-declarator-attribute-crash.cpp diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index c381da7e354c..806dc62cc55d 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -295,7 +295,9 @@ Bug Fixes to C++ Support of templates. Previously we were diagnosing on any non-function template instead of only on class, alias, and variable templates, as last updated by CWG2032. Fixes (#GH83461) - +- Fixed an issue where an attribute on a declarator would cause the attribute to + be destructed prematurely. This fixes a pair of Chromium that were brought to + our attention by an attempt to fix in (#GH77703). Fixes (#GH83611). Bug Fixes to AST Handling ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/include/clang/Sema/DeclSpec.h b/clang/include/clang/Sema/DeclSpec.h index 316e8071169a..a17615970748 100644 --- a/clang/include/clang/Sema/DeclSpec.h +++ b/clang/include/clang/Sema/DeclSpec.h @@ -2359,11 +2359,27 @@ public: SetRangeEnd(EndLoc); } + /// AddTypeInfo - Add a chunk to this declarator. Also extend the range to + /// EndLoc, which should be the last token of the chunk. This overload is for + /// copying a 'chunk' from another declarator, so it takes the pool that the + /// other Declarator owns so that it can 'take' the attributes from it. + void AddTypeInfo(const DeclaratorChunk &TI, AttributePool &OtherPool, + SourceLocation EndLoc) { + DeclTypeInfo.push_back(TI); + getAttributePool().takeFrom(DeclTypeInfo.back().getAttrs(), OtherPool); + + if (!EndLoc.isInvalid()) + SetRangeEnd(EndLoc); + } + /// AddTypeInfo - Add a chunk to this declarator. Also extend the range to /// EndLoc, which should be the last token of the chunk. void AddTypeInfo(const DeclaratorChunk &TI, SourceLocation EndLoc) { DeclTypeInfo.push_back(TI); + assert(TI.AttrList.empty() && + "Cannot add a declarator chunk with attributes with this overload"); + if (!EndLoc.isInvalid()) SetRangeEnd(EndLoc); } diff --git a/clang/include/clang/Sema/ParsedAttr.h b/clang/include/clang/Sema/ParsedAttr.h index 8c3ba39031aa..e3857b2f07d9 100644 --- a/clang/include/clang/Sema/ParsedAttr.h +++ b/clang/include/clang/Sema/ParsedAttr.h @@ -680,6 +680,7 @@ public: ~AttributeFactory(); }; +class ParsedAttributesView; class AttributePool { friend class AttributeFactory; friend class ParsedAttributes; @@ -734,6 +735,10 @@ public: pool.Attrs.clear(); } + /// Removes the attributes from \c List, which are owned by \c Pool, and adds + /// them at the end of this \c AttributePool. + void takeFrom(ParsedAttributesView &List, AttributePool &Pool); + ParsedAttr *create(IdentifierInfo *attrName, SourceRange attrRange, IdentifierInfo *scopeName, SourceLocation scopeLoc, ArgsUnion *args, unsigned numArgs, ParsedAttr::Form form, @@ -816,6 +821,7 @@ public: }; class ParsedAttributesView { + friend class AttributePool; using VecTy = llvm::SmallVector; using SizeType = decltype(std::declval().size()); diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp index edfab11c37cf..ccbfea6a66fb 100644 --- a/clang/lib/Parse/ParseDecl.cpp +++ b/clang/lib/Parse/ParseDecl.cpp @@ -7928,7 +7928,7 @@ void Parser::ParseMisplacedBracketDeclarator(Declarator &D) { // Adding back the bracket info to the end of the Declarator. for (unsigned i = 0, e = TempDeclarator.getNumTypeObjects(); i < e; ++i) { const DeclaratorChunk &Chunk = TempDeclarator.getTypeObject(i); - D.AddTypeInfo(Chunk, SourceLocation()); + D.AddTypeInfo(Chunk, TempDeclarator.getAttributePool(), SourceLocation()); } // The missing identifier would have been diagnosed in ParseDirectDeclarator. diff --git a/clang/lib/Sema/ParsedAttr.cpp b/clang/lib/Sema/ParsedAttr.cpp index 06c213267c7e..6abc90336c99 100644 --- a/clang/lib/Sema/ParsedAttr.cpp +++ b/clang/lib/Sema/ParsedAttr.cpp @@ -100,6 +100,12 @@ void AttributePool::takePool(AttributePool &pool) { pool.Attrs.clear(); } +void AttributePool::takeFrom(ParsedAttributesView &List, AttributePool &Pool) { + assert(&Pool != this && "AttributePool can't take attributes from itself"); + llvm::for_each(List.AttrList, [&Pool](ParsedAttr *A) { Pool.remove(A); }); + Attrs.insert(Attrs.end(), List.AttrList.begin(), List.AttrList.end()); +} + namespace { #include "clang/Sema/AttrParsedAttrImpl.inc" diff --git a/clang/test/Parser/cxx-declarator-attribute-crash.cpp b/clang/test/Parser/cxx-declarator-attribute-crash.cpp new file mode 100644 index 000000000000..3b989a659db5 --- /dev/null +++ b/clang/test/Parser/cxx-declarator-attribute-crash.cpp @@ -0,0 +1,8 @@ +// RUN: %clang_cc1 -fsyntax-only -verify %s + +// expected-error@+5{{brackets are not allowed here}} +// expected-error@+4{{a type specifier is required for all declarations}} +// expected-warning@+3{{unknown attribute 'h' ignored}} +// expected-error@+2{{definition of variable with array type}} +// expected-error@+1{{expected ';'}} +[][[h]]l -- GitLab From 87e7140f72ba46330efb8d9110c2a4a1e6e94900 Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Mon, 4 Mar 2024 09:46:34 -0800 Subject: [PATCH 017/929] [nfc][InstrProfiling]For comdat setting helper function, move comment closer to the code (#83757) --- .../Instrumentation/InstrProfiling.cpp | 50 +++++++++++-------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp index d5d55dec6382..c42c53edd511 100644 --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -208,6 +208,7 @@ private: std::function GetTLI; const bool DataReferencedByCode; + struct PerFunctionProfileData { uint32_t NumValueSites[IPVK_Last + 1] = {}; GlobalVariable *RegionCounters = nullptr; @@ -1193,18 +1194,41 @@ static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) { } void InstrLowerer::maybeSetComdat(GlobalVariable *GV, Function *Fn, - StringRef VarName) { + StringRef CounterGroupName) { + // Place lowered global variables in a comdat group if the associated function + // is a COMDAT. This will make sure that only one copy of global variable + // (e.g. function counters) of the COMDAT function will be emitted after + // linking. bool NeedComdat = needsComdatForCounter(*Fn, M); bool UseComdat = (NeedComdat || TT.isOSBinFormatELF()); if (!UseComdat) return; - StringRef GroupName = - TT.isOSBinFormatCOFF() && DataReferencedByCode ? GV->getName() : VarName; + // Keep in mind that this pass may run before the inliner, so we need to + // create a new comdat group (for counters, profiling data, etc). If we use + // the comdat of the parent function, that will result in relocations against + // discarded sections. + // + // If the data variable is referenced by code, non-counter variables (notably + // profiling data) and counters have to be in different comdats for COFF + // because the Visual C++ linker will report duplicate symbol errors if there + // are multiple external symbols with the same name marked + // IMAGE_COMDAT_SELECT_ASSOCIATIVE. + StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode + ? GV->getName() + : CounterGroupName; Comdat *C = M.getOrInsertComdat(GroupName); - if (!NeedComdat) + + if (!NeedComdat) { + // Object file format must be ELF since `UseComdat && !NeedComdat` is true. + // + // For ELF, when not using COMDAT, put counters, data and values into a + // nodeduplicate COMDAT which is lowered to a zero-flag section group. This + // allows -z start-stop-gc to discard the entire group when the function is + // discarded. C->setSelectionKind(Comdat::NoDeduplicate); + } GV->setComdat(C); // COFF doesn't allow the comdat group leader to have private linkage, so // upgrade private linkage to internal linkage to produce a symbol table @@ -1238,23 +1262,7 @@ GlobalVariable *InstrLowerer::setupProfileSection(InstrProfInstBase *Inc, Linkage = GlobalValue::PrivateLinkage; Visibility = GlobalValue::DefaultVisibility; } - // Move the name variable to the right section. Place them in a COMDAT group - // if the associated function is a COMDAT. This will make sure that only one - // copy of counters of the COMDAT function will be emitted after linking. Keep - // in mind that this pass may run before the inliner, so we need to create a - // new comdat group for the counters and profiling data. If we use the comdat - // of the parent function, that will result in relocations against discarded - // sections. - // - // If the data variable is referenced by code, counters and data have to be - // in different comdats for COFF because the Visual C++ linker will report - // duplicate symbol errors if there are multiple external symbols with the - // same name marked IMAGE_COMDAT_SELECT_ASSOCIATIVE. - // - // For ELF, when not using COMDAT, put counters, data and values into a - // nodeduplicate COMDAT which is lowered to a zero-flag section group. This - // allows -z start-stop-gc to discard the entire group when the function is - // discarded. + // Move the name variable to the right section. bool Renamed; GlobalVariable *Ptr; StringRef VarPrefix; -- GitLab From 2e0693abbcf844c138adfa29fc70ea07681e373f Mon Sep 17 00:00:00 2001 From: Aart Bik <39774503+aartbik@users.noreply.github.com> Date: Mon, 4 Mar 2024 09:49:54 -0800 Subject: [PATCH 018/929] [mlir][sparse][gpu] migration to sparse_tensor.print (#83510) Continuous efforts #83357 for our sparse CUDA tests --- .../GPU/CUDA/sparse-gemm-lib.mlir | 24 ++++++---------- .../GPU/CUDA/sparse-matmul-lib.mlir | 28 ++++--------------- .../GPU/CUDA/sparse-sampled-matmul-lib.mlir | 23 +++++++++------ .../GPU/CUDA/sparse-sddmm-lib.mlir | 23 +++++++++------ 4 files changed, 43 insertions(+), 55 deletions(-) diff --git a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-gemm-lib.mlir b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-gemm-lib.mlir index 73faf7d9b7b9..984b3614eded 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-gemm-lib.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-gemm-lib.mlir @@ -62,23 +62,15 @@ module { tensor<8x8xf32, #CSR>) -> tensor<8x8xf32, #CSR> // - // Verify computed result (expected output, with only 20 nonzeros). + // Verify computed result. // - // CHECK: ( ( 1, 39, 52, 0, 0, 0, 45, 51 ), - // CHECK-SAME: ( 0, 0, 0, 0, 0, 0, 0, 0 ), - // CHECK-SAME: ( 0, 0, 16, 0, 0, 0, 0, 0 ), - // CHECK-SAME: ( 0, 0, 0, 25, 0, 0, 0, 0 ), - // CHECK-SAME: ( 0, 0, 0, 0, 36, 0, 0, 0 ), - // CHECK-SAME: ( 0, 117, 158, 0, 0, 0, 135, 144 ), - // CHECK-SAME: ( 0, 156, 318, 0, 0, 0, 301, 324 ), - // CHECK-SAME: ( 0, 208, 430, 0, 0, 0, 405, 436 ) ) - // CHECK-NEXT: 20 - %d = sparse_tensor.convert %Ccsr : tensor<8x8xf32, #CSR> to tensor<8x8xf32> - %v = vector.transfer_read %d[%c0, %c0], %f0: tensor<8x8xf32>, vector<8x8xf32> - vector.print %v : vector<8x8xf32> - %nnz = sparse_tensor.number_of_entries %Ccsr : tensor<8x8xf32, #CSR> - %x = sparse_tensor.number_of_entries %Ccsr : tensor<8x8xf32, #CSR> - vector.print %nnz : index + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 20 + // CHECK-NEXT: pos[1] : ( 0, 5, 5, 6, 7, 8, 12, 16, 20, + // CHECK-NEXT: crd[1] : ( 0, 1, 2, 6, 7, 2, 3, 4, 1, 2, 6, 7, 1, 2, 6, 7, 1, 2, 6, 7, + // CHECK-NEXT: values : ( 1, 39, 52, 45, 51, 16, 25, 36, 117, 158, 135, 144, 156, 318, 301, 324, 208, 430, 405, 436, + // CHECK-NEXT: ---- + sparse_tensor.print %Ccsr : tensor<8x8xf32, #CSR> llvm.call @mgpuDestroySparseEnv(): () -> () return diff --git a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-matmul-lib.mlir b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-matmul-lib.mlir index 67b6e60ab5c8..0e7c75bc41aa 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-matmul-lib.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-matmul-lib.mlir @@ -66,32 +66,16 @@ module { return %D: tensor<8x8xf32> } + // Helper to dump dense tensor as series of vectors. func.func @dump(%mat: tensor<8x8xf32>) { %f0 = arith.constant 0.0 : f32 %c0 = arith.constant 0 : index %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c3 = arith.constant 3 : index - %c4 = arith.constant 4 : index - %c5 = arith.constant 5 : index - %c6 = arith.constant 6 : index - %c7 = arith.constant 7 : index - %r0 = vector.transfer_read %mat[%c0,%c0], %f0 : tensor<8x8xf32>, vector<8xf32> - vector.print %r0 : vector<8xf32> - %r1 = vector.transfer_read %mat[%c1,%c0], %f0 : tensor<8x8xf32>, vector<8xf32> - vector.print %r1 : vector<8xf32> - %r2 = vector.transfer_read %mat[%c2,%c0], %f0 : tensor<8x8xf32>, vector<8xf32> - vector.print %r2 : vector<8xf32> - %r3 = vector.transfer_read %mat[%c3,%c0], %f0 : tensor<8x8xf32>, vector<8xf32> - vector.print %r3 : vector<8xf32> - %r4 = vector.transfer_read %mat[%c4,%c0], %f0 : tensor<8x8xf32>, vector<8xf32> - vector.print %r4 : vector<8xf32> - %r5 = vector.transfer_read %mat[%c5,%c0], %f0 : tensor<8x8xf32>, vector<8xf32> - vector.print %r5 : vector<8xf32> - %r6 = vector.transfer_read %mat[%c6,%c0], %f0 : tensor<8x8xf32>, vector<8xf32> - vector.print %r6 : vector<8xf32> - %r7 = vector.transfer_read %mat[%c7,%c0], %f0 : tensor<8x8xf32>, vector<8xf32> - vector.print %r7 : vector<8xf32> + %c8 = arith.constant 8 : index + scf.for %i = %c0 to %c8 step %c1 { + %v = vector.transfer_read %mat[%i,%c0], %f0 : tensor<8x8xf32>, vector<8xf32> + vector.print %v : vector<8xf32> + } return } diff --git a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sampled-matmul-lib.mlir b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sampled-matmul-lib.mlir index 9b33f0815123..aad26556b00e 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sampled-matmul-lib.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sampled-matmul-lib.mlir @@ -113,11 +113,13 @@ module { // // Print the result for verification. // - // CHECK: ( 11, 41.4, 42, 102.5, 93, 44.1, 164, 105.2, 255 ) - // - %vm = sparse_tensor.values %0 : tensor to memref - %vv = vector.transfer_read %vm[%c0], %d0 : memref, vector<9xf32> - vector.print %vv : vector<9xf32> + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 9 + // CHECK-NEXT: pos[1] : ( 0, 2, 4, 5, 7, 9, + // CHECK-NEXT: crd[1] : ( 0, 3, 1, 4, 2, 0, 3, 1, 4, + // CHECK-NEXT: values : ( 11, 41.4, 42, 102.5, 93, 44.1, 164, 105.2, 255, + // CHECK-NEXT: ---- + sparse_tensor.print %0 : tensor // Create a much sparser sampling matrix. %t = arith.constant sparse<[[0,0], [0,1], [1,0], [3,4], [7,7]], @@ -137,11 +139,14 @@ module { // // Print the result for verification. // - // CHECK: ( ( 17, 18, 0, 0, 0, 0, 0, 0 ), ( 19, 0, 0, 0, 0, 0, 0, 0 ), ( 0, 0, 0, 0, 0, 0, 0, 0 ), ( 0, 0, 0, 0, 20, 0, 0, 0 ), ( 0, 0, 0, 0, 0, 0, 0, 0 ), ( 0, 0, 0, 0, 0, 0, 0, 0 ), ( 0, 0, 0, 0, 0, 0, 0, 0 ), ( 0, 0, 0, 0, 0, 0, 0, 21 ) ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 5 + // CHECK-NEXT: pos[1] : ( 0, 2, 3, 3, 4, 4, 4, 4, 5, + // CHECK-NEXT: crd[1] : ( 0, 1, 0, 4, 7, + // CHECK-NEXT: values : ( 17, 18, 19, 20, 21, + // CHECK-NEXT: ---- // - %d = sparse_tensor.convert %1 : tensor to tensor - %mm = vector.transfer_read %d[%c0, %c0], %d0 : tensor, vector<8x8xf32> - vector.print %mm : vector<8x8xf32> + sparse_tensor.print %1 : tensor // Release the resources. bufferization.dealloc_tensor %0 : tensor diff --git a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sddmm-lib.mlir b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sddmm-lib.mlir index db5c154e5e43..bd2c72271c61 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sddmm-lib.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sddmm-lib.mlir @@ -166,15 +166,22 @@ module { // the result (which is why the block sparse version has actual results // in the original zero positions). // - // CHECK: ( 5, 10, 24, 19, 53, 42, 55, 56 ) - // CHECK-NEXT: ( 5, 10, 8, 19, 24, 24, 40, 53, 42, 55, 56, 64 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 8 + // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7, 8, + // CHECK-NEXT: crd[1] : ( 0, 1, 4, 1, 5, 2, 3, 2, + // CHECK-NEXT: values : ( 5, 10, 24, 19, 53, 42, 55, 56, + // CHECK-NEXT: ---- // - %v0 = sparse_tensor.values %0 : tensor to memref - %vv0 = vector.transfer_read %v0[%c0], %d0 : memref, vector<8xf32> - vector.print %vv0 : vector<8xf32> - %v1 = sparse_tensor.values %1 : tensor to memref - %vv1 = vector.transfer_read %v1[%c0], %d0 : memref, vector<12xf32> - vector.print %vv1 : vector<12xf32> + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 12 + // CHECK-NEXT: pos[1] : ( 0, 2, 3, + // CHECK-NEXT: crd[1] : ( 0, 2, 1, + // CHECK-NEXT: values : ( 5, 10, 8, 19, 24, 24, 40, 53, 42, 55, 56, 64, + // CHECK-NEXT: ---- + // + sparse_tensor.print %0 : tensor + sparse_tensor.print %1 : tensor // Release the resources. bufferization.dealloc_tensor %0 : tensor -- GitLab From 379e55ea40cbc5ae8b4a16d96236f6a88bd0a649 Mon Sep 17 00:00:00 2001 From: erichkeane Date: Mon, 4 Mar 2024 09:52:25 -0800 Subject: [PATCH 019/929] Fix release note bug reference from bd7bce2c8465e9cb --- clang/docs/ReleaseNotes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 806dc62cc55d..612b43297274 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -297,7 +297,7 @@ Bug Fixes to C++ Support CWG2032. Fixes (#GH83461) - Fixed an issue where an attribute on a declarator would cause the attribute to be destructed prematurely. This fixes a pair of Chromium that were brought to - our attention by an attempt to fix in (#GH77703). Fixes (#GH83611). + our attention by an attempt to fix in (#GH77703). Fixes (#GH83385). Bug Fixes to AST Handling ^^^^^^^^^^^^^^^^^^^^^^^^^ -- GitLab From ac783addc4dd5bda557cc07b90c08bab2d110ec3 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Mon, 4 Mar 2024 10:01:24 -0800 Subject: [PATCH 020/929] [LV] Use SmallVector::resize instead of push_back/emplace_back in a loop. NFC (#83696) This should be more efficient since the vector can know how much additional space to reserve before creating the new elements. --- llvm/lib/Transforms/Vectorize/VPlan.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 16c09a83e777..bc499400bced 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -311,12 +311,12 @@ struct VPTransformState { void set(VPValue *Def, Value *V, const VPIteration &Instance) { auto Iter = Data.PerPartScalars.insert({Def, {}}); auto &PerPartVec = Iter.first->second; - while (PerPartVec.size() <= Instance.Part) - PerPartVec.emplace_back(); + if (PerPartVec.size() <= Instance.Part) + PerPartVec.resize(Instance.Part + 1); auto &Scalars = PerPartVec[Instance.Part]; unsigned CacheIdx = Instance.Lane.mapToCacheIndex(VF); - while (Scalars.size() <= CacheIdx) - Scalars.push_back(nullptr); + if (Scalars.size() <= CacheIdx) + Scalars.resize(CacheIdx + 1); assert(!Scalars[CacheIdx] && "should overwrite existing value"); Scalars[CacheIdx] = V; } -- GitLab From 0097fd2b068e374b02571b391bd4a0a84e8ed3e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix-Antoine=20Constantin?= <60141446+felix642@users.noreply.github.com> Date: Mon, 4 Mar 2024 13:05:06 -0500 Subject: [PATCH 021/929] [clang-tidy] bugprone-unused-return-value config now supports regexes (#82952) The parameter `CheckedFunctions` now supports regexes Fixes #63107 --- .../bugprone/UnusedReturnValueCheck.cpp | 202 +++++++++--------- .../bugprone/UnusedReturnValueCheck.h | 6 +- .../hicpp/IgnoredRemoveResultCheck.cpp | 8 +- clang-tools-extra/docs/ReleaseNotes.rst | 4 + .../checks/bugprone/unused-return-value.rst | 5 +- .../bugprone/unused-return-value-custom.cpp | 2 +- 6 files changed, 117 insertions(+), 110 deletions(-) diff --git a/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp index 05012c7df6a9..b4bf85c912c3 100644 --- a/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp @@ -33,98 +33,98 @@ AST_MATCHER_P(FunctionDecl, isInstantiatedFrom, Matcher, UnusedReturnValueCheck::UnusedReturnValueCheck(llvm::StringRef Name, ClangTidyContext *Context) : ClangTidyCheck(Name, Context), - CheckedFunctions(Options.get("CheckedFunctions", - "::std::async;" - "::std::launder;" - "::std::remove;" - "::std::remove_if;" - "::std::unique;" - "::std::unique_ptr::release;" - "::std::basic_string::empty;" - "::std::vector::empty;" - "::std::back_inserter;" - "::std::distance;" - "::std::find;" - "::std::find_if;" - "::std::inserter;" - "::std::lower_bound;" - "::std::make_pair;" - "::std::map::count;" - "::std::map::find;" - "::std::map::lower_bound;" - "::std::multimap::equal_range;" - "::std::multimap::upper_bound;" - "::std::set::count;" - "::std::set::find;" - "::std::setfill;" - "::std::setprecision;" - "::std::setw;" - "::std::upper_bound;" - "::std::vector::at;" - // C standard library - "::bsearch;" - "::ferror;" - "::feof;" - "::isalnum;" - "::isalpha;" - "::isblank;" - "::iscntrl;" - "::isdigit;" - "::isgraph;" - "::islower;" - "::isprint;" - "::ispunct;" - "::isspace;" - "::isupper;" - "::iswalnum;" - "::iswprint;" - "::iswspace;" - "::isxdigit;" - "::memchr;" - "::memcmp;" - "::strcmp;" - "::strcoll;" - "::strncmp;" - "::strpbrk;" - "::strrchr;" - "::strspn;" - "::strstr;" - "::wcscmp;" - // POSIX - "::access;" - "::bind;" - "::connect;" - "::difftime;" - "::dlsym;" - "::fnmatch;" - "::getaddrinfo;" - "::getopt;" - "::htonl;" - "::htons;" - "::iconv_open;" - "::inet_addr;" - "::isascii;" - "::isatty;" - "::mmap;" - "::newlocale;" - "::openat;" - "::pathconf;" - "::pthread_equal;" - "::pthread_getspecific;" - "::pthread_mutex_trylock;" - "::readdir;" - "::readlink;" - "::recvmsg;" - "::regexec;" - "::scandir;" - "::semget;" - "::setjmp;" - "::shm_open;" - "::shmget;" - "::sigismember;" - "::strcasecmp;" - "::strsignal;" - "::ttyname")), + CheckedFunctions(utils::options::parseStringList( + Options.get("CheckedFunctions", "::std::async;" + "::std::launder;" + "::std::remove;" + "::std::remove_if;" + "::std::unique;" + "::std::unique_ptr::release;" + "::std::basic_string::empty;" + "::std::vector::empty;" + "::std::back_inserter;" + "::std::distance;" + "::std::find;" + "::std::find_if;" + "::std::inserter;" + "::std::lower_bound;" + "::std::make_pair;" + "::std::map::count;" + "::std::map::find;" + "::std::map::lower_bound;" + "::std::multimap::equal_range;" + "::std::multimap::upper_bound;" + "::std::set::count;" + "::std::set::find;" + "::std::setfill;" + "::std::setprecision;" + "::std::setw;" + "::std::upper_bound;" + "::std::vector::at;" + // C standard library + "::bsearch;" + "::ferror;" + "::feof;" + "::isalnum;" + "::isalpha;" + "::isblank;" + "::iscntrl;" + "::isdigit;" + "::isgraph;" + "::islower;" + "::isprint;" + "::ispunct;" + "::isspace;" + "::isupper;" + "::iswalnum;" + "::iswprint;" + "::iswspace;" + "::isxdigit;" + "::memchr;" + "::memcmp;" + "::strcmp;" + "::strcoll;" + "::strncmp;" + "::strpbrk;" + "::strrchr;" + "::strspn;" + "::strstr;" + "::wcscmp;" + // POSIX + "::access;" + "::bind;" + "::connect;" + "::difftime;" + "::dlsym;" + "::fnmatch;" + "::getaddrinfo;" + "::getopt;" + "::htonl;" + "::htons;" + "::iconv_open;" + "::inet_addr;" + "::isascii;" + "::isatty;" + "::mmap;" + "::newlocale;" + "::openat;" + "::pathconf;" + "::pthread_equal;" + "::pthread_getspecific;" + "::pthread_mutex_trylock;" + "::readdir;" + "::readlink;" + "::recvmsg;" + "::regexec;" + "::scandir;" + "::semget;" + "::setjmp;" + "::shm_open;" + "::shmget;" + "::sigismember;" + "::strcasecmp;" + "::strsignal;" + "::ttyname"))), CheckedReturnTypes(utils::options::parseStringList( Options.get("CheckedReturnTypes", "::std::error_code;" "::std::error_condition;" @@ -133,36 +133,36 @@ UnusedReturnValueCheck::UnusedReturnValueCheck(llvm::StringRef Name, "::boost::system::error_code"))), AllowCastToVoid(Options.get("AllowCastToVoid", false)) {} -UnusedReturnValueCheck::UnusedReturnValueCheck(llvm::StringRef Name, - ClangTidyContext *Context, - std::string CheckedFunctions) +UnusedReturnValueCheck::UnusedReturnValueCheck( + llvm::StringRef Name, ClangTidyContext *Context, + std::vector CheckedFunctions) : UnusedReturnValueCheck(Name, Context, std::move(CheckedFunctions), {}, false) {} UnusedReturnValueCheck::UnusedReturnValueCheck( llvm::StringRef Name, ClangTidyContext *Context, - std::string CheckedFunctions, std::vector CheckedReturnTypes, - bool AllowCastToVoid) + std::vector CheckedFunctions, + std::vector CheckedReturnTypes, bool AllowCastToVoid) : ClangTidyCheck(Name, Context), CheckedFunctions(std::move(CheckedFunctions)), CheckedReturnTypes(std::move(CheckedReturnTypes)), AllowCastToVoid(AllowCastToVoid) {} void UnusedReturnValueCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) { - Options.store(Opts, "CheckedFunctions", CheckedFunctions); + Options.store(Opts, "CheckedFunctions", + utils::options::serializeStringList(CheckedFunctions)); Options.store(Opts, "CheckedReturnTypes", utils::options::serializeStringList(CheckedReturnTypes)); Options.store(Opts, "AllowCastToVoid", AllowCastToVoid); } void UnusedReturnValueCheck::registerMatchers(MatchFinder *Finder) { - auto FunVec = utils::options::parseStringList(CheckedFunctions); - auto MatchedDirectCallExpr = expr(callExpr(callee(functionDecl( // Don't match void overloads of checked functions. unless(returns(voidType())), - anyOf(isInstantiatedFrom(hasAnyName(FunVec)), + anyOf(isInstantiatedFrom(matchers::matchesAnyListedName( + CheckedFunctions)), returns(hasCanonicalType(hasDeclaration( namedDecl(matchers::matchesAnyListedName( CheckedReturnTypes))))))))) diff --git a/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.h b/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.h index ab2cc691b894..d65a567e1c46 100644 --- a/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.h +++ b/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.h @@ -29,14 +29,14 @@ public: } private: - std::string CheckedFunctions; + const std::vector CheckedFunctions; const std::vector CheckedReturnTypes; protected: UnusedReturnValueCheck(StringRef Name, ClangTidyContext *Context, - std::string CheckedFunctions); + std::vector CheckedFunctions); UnusedReturnValueCheck(StringRef Name, ClangTidyContext *Context, - std::string CheckedFunctions, + std::vector CheckedFunctions, std::vector CheckedReturnTypes, bool AllowCastToVoid); bool AllowCastToVoid; diff --git a/clang-tools-extra/clang-tidy/hicpp/IgnoredRemoveResultCheck.cpp b/clang-tools-extra/clang-tidy/hicpp/IgnoredRemoveResultCheck.cpp index 3410559d435f..8020f8cd0625 100644 --- a/clang-tools-extra/clang-tidy/hicpp/IgnoredRemoveResultCheck.cpp +++ b/clang-tools-extra/clang-tidy/hicpp/IgnoredRemoveResultCheck.cpp @@ -13,9 +13,11 @@ namespace clang::tidy::hicpp { IgnoredRemoveResultCheck::IgnoredRemoveResultCheck(llvm::StringRef Name, ClangTidyContext *Context) : UnusedReturnValueCheck(Name, Context, - "::std::remove;" - "::std::remove_if;" - "::std::unique") { + { + "::std::remove", + "::std::remove_if", + "::std::unique", + }) { // The constructor for ClangTidyCheck needs to have been called // before we can access options via Options.get(). AllowCastToVoid = Options.get("AllowCastToVoid", true); diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst index 0d2467210fc6..70ae23ba185a 100644 --- a/clang-tools-extra/docs/ReleaseNotes.rst +++ b/clang-tools-extra/docs/ReleaseNotes.rst @@ -140,6 +140,10 @@ Changes in existing checks ` check by ignoring local variable with ``[maybe_unused]`` attribute. +- Improved :doc:`bugprone-unused-return-value + ` check by updating the + parameter `CheckedFunctions` to support regexp. + - Improved :doc:`cppcoreguidelines-missing-std-forward ` check by no longer giving false positives for deleted functions. diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone/unused-return-value.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone/unused-return-value.rst index 1e3c8a326822..8d5eddbe215c 100644 --- a/clang-tools-extra/docs/clang-tidy/checks/bugprone/unused-return-value.rst +++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone/unused-return-value.rst @@ -10,8 +10,9 @@ Options .. option:: CheckedFunctions - Semicolon-separated list of functions to check. The function is checked if - the name and scope matches, with any arguments. + Semicolon-separated list of functions to check. + This parameter supports regexp. The function is checked if the name + and scope matches, with any arguments. By default the following functions are checked: ``std::async, std::launder, std::remove, std::remove_if, std::unique, std::unique_ptr::release, std::basic_string::empty, std::vector::empty, diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/unused-return-value-custom.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/unused-return-value-custom.cpp index d3650b210ab0..3035183573cc 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/unused-return-value-custom.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/unused-return-value-custom.cpp @@ -1,7 +1,7 @@ // RUN: %check_clang_tidy %s bugprone-unused-return-value %t \ // RUN: -config='{CheckOptions: \ // RUN: {bugprone-unused-return-value.CheckedFunctions: \ -// RUN: "::fun;::ns::Outer::Inner::memFun;::ns::Type::staticFun;::ns::ClassTemplate::memFun;::ns::ClassTemplate::staticFun"}}' \ +// RUN: "::fun;::ns::Outer::Inner::memFun;::ns::Type::staticFun;::ns::ClassTemplate::(mem|static)Fun"}}' \ // RUN: -- namespace std { -- GitLab From 98418c27bc17946990e5980d56ef98ae82da8770 Mon Sep 17 00:00:00 2001 From: Jake Egan Date: Mon, 4 Mar 2024 13:06:05 -0500 Subject: [PATCH 022/929] [libc++][AIX] Use input redirection instead of piping for cin tests (#83184) When echo is used for piping, lit uses the system echo rather than the builtin echo. The system echo on AIX doesn't support the `-n` option, which causes these tests to fail. Use input redirection, so the builtin echo can be used. --- .../iostream.objects/narrow.stream.objects/cin.sh.cpp | 6 ++---- .../iostream.objects/wide.stream.objects/wcin-imbue.sh.cpp | 6 ++---- .../iostream.objects/wide.stream.objects/wcin.sh.cpp | 6 ++---- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cin.sh.cpp b/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cin.sh.cpp index ce6a7f37801e..4f23b6b9a5d8 100644 --- a/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cin.sh.cpp +++ b/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cin.sh.cpp @@ -6,9 +6,6 @@ // //===----------------------------------------------------------------------===// -// TODO: Investigate -// UNSUPPORTED: LIBCXX-AIX-FIXME - // QEMU does not detect EOF, when reading from stdin // "echo -n" suppresses any characters after the output and so the test hangs. // https://gitlab.com/qemu-project/qemu/-/issues/1963 @@ -23,7 +20,8 @@ // istream cin; // RUN: %{build} -// RUN: echo -n 1234 | %{exec} %t.exe +// RUN: echo -n 1234 > %t.input +// RUN: %{exec} %t.exe < %t.input #include #include diff --git a/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcin-imbue.sh.cpp b/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcin-imbue.sh.cpp index 027e4fa936f1..6ae1972234b2 100644 --- a/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcin-imbue.sh.cpp +++ b/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcin-imbue.sh.cpp @@ -6,9 +6,6 @@ // //===----------------------------------------------------------------------===// -// TODO: Investigate -// UNSUPPORTED: LIBCXX-AIX-FIXME - // This test hangs on Android devices that lack shell_v2, which was added in // Android N (API 24). // UNSUPPORTED: LIBCXX-ANDROID-FIXME && android-device-api={{2[1-3]}} @@ -20,7 +17,8 @@ // UNSUPPORTED: no-wide-characters // RUN: %{build} -// RUN: echo -n 1234 | %{exec} %t.exe +// RUN: echo -n 1234 > %t.input +// RUN: %{exec} %t.exe < %t.input #include #include diff --git a/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcin.sh.cpp b/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcin.sh.cpp index 30972da2f093..08b74b999bbf 100644 --- a/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcin.sh.cpp +++ b/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcin.sh.cpp @@ -6,9 +6,6 @@ // //===----------------------------------------------------------------------===// -// TODO: Investigate -// UNSUPPORTED: LIBCXX-AIX-FIXME - // This test hangs on Android devices that lack shell_v2, which was added in // Android N (API 24). // UNSUPPORTED: LIBCXX-ANDROID-FIXME && android-device-api={{2[1-3]}} @@ -20,7 +17,8 @@ // UNSUPPORTED: no-wide-characters // RUN: %{build} -// RUN: echo -n 1234 | %{exec} %t.exe +// RUN: echo -n 1234 > %t.input +// RUN: %{exec} %t.exe < %t.input #include #include -- GitLab From 56754478e7184a68e88ccc578ffd31097da85d15 Mon Sep 17 00:00:00 2001 From: Kirill Stoimenov Date: Sun, 3 Mar 2024 16:59:43 +0000 Subject: [PATCH 023/929] Reapply "[HWASAN] Implement selective instrumentation based on profiling information (#83503)" Added REQUIRES: asserts, which should fix the build problem. This reverts commit f6f79d46e580b34b2c98bd9bda7ede3a38f43f77. --- .../Instrumentation/HWAddressSanitizer.cpp | 36 +++++++++++++++++++ .../HWAddressSanitizer/pgo-opt-out-no-ps.ll | 16 +++++++++ .../HWAddressSanitizer/pgo-opt-out.ll | 33 +++++++++++++++++ 3 files changed, 85 insertions(+) create mode 100644 llvm/test/Instrumentation/HWAddressSanitizer/pgo-opt-out-no-ps.ll create mode 100644 llvm/test/Instrumentation/HWAddressSanitizer/pgo-opt-out.ll diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp index 33add6d4cd76..4404382a85b7 100644 --- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -15,11 +15,14 @@ #include "llvm/ADT/MapVector.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/DomTreeUpdater.h" #include "llvm/Analysis/GlobalsModRef.h" #include "llvm/Analysis/PostDominators.h" +#include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/Analysis/StackSafetyAnalysis.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/ValueTracking.h" @@ -177,6 +180,18 @@ static cl::opt ClWithTls( "platforms that support this"), cl::Hidden, cl::init(true)); +static cl::opt + CSkipHotCode("hwasan-skip-hot-code", + cl::desc("Do not instument hot functions based on FDO."), + cl::Hidden, cl::init(false)); + +static cl::opt HotPercentileCutoff("hwasan-percentile-cutoff-hot", + cl::init(0)); + +STATISTIC(NumTotalFuncs, "Number of total funcs HWASAN"); +STATISTIC(NumInstrumentedFuncs, "Number of HWASAN instrumented funcs"); +STATISTIC(NumNoProfileSummaryFuncs, "Number of HWASAN funcs without PS"); + // Mode for selecting how to insert frame record info into the stack ring // buffer. enum RecordStackHistoryMode { @@ -1507,6 +1522,27 @@ void HWAddressSanitizer::sanitizeFunction(Function &F, if (!F.hasFnAttribute(Attribute::SanitizeHWAddress)) return; + if (F.empty()) + return; + + NumTotalFuncs++; + if (CSkipHotCode) { + auto &MAMProxy = FAM.getResult(F); + ProfileSummaryInfo *PSI = + MAMProxy.getCachedResult(*F.getParent()); + if (PSI && PSI->hasProfileSummary()) { + auto &BFI = FAM.getResult(F); + if ((HotPercentileCutoff.getNumOccurrences() && HotPercentileCutoff >= 0) + ? PSI->isFunctionHotInCallGraphNthPercentile(HotPercentileCutoff, + &F, BFI) + : PSI->isFunctionHotInCallGraph(&F, BFI)) + return; + } else { + ++NumNoProfileSummaryFuncs; + } + } + NumInstrumentedFuncs++; + LLVM_DEBUG(dbgs() << "Function: " << F.getName() << "\n"); SmallVector OperandsToInstrument; diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/pgo-opt-out-no-ps.ll b/llvm/test/Instrumentation/HWAddressSanitizer/pgo-opt-out-no-ps.ll new file mode 100644 index 000000000000..2aa218fa1522 --- /dev/null +++ b/llvm/test/Instrumentation/HWAddressSanitizer/pgo-opt-out-no-ps.ll @@ -0,0 +1,16 @@ +; RUN: opt < %s -passes='require,hwasan' -S -stats 2>&1 \ +; RUN: -hwasan-skip-hot-code=0 | FileCheck %s --check-prefix=FULL +; RUN: opt < %s -passes='require,hwasan' -S -stats 2>&1 \ +; RUN: -hwasan-skip-hot-code=1 | FileCheck %s --check-prefix=SELSAN + +; REQUIRES: asserts + +; FULL: 1 hwasan - Number of HWASAN instrumented funcs +; FULL: 1 hwasan - Number of total funcs HWASAN + +; SELSAN: 1 hwasan - Number of HWASAN instrumented funcs +; SELSAN: 1 hwasan - Number of HWASAN funcs without PS +; SELSAN: 1 hwasan - Number of total funcs HWASAN + +define void @not_sanitized() { ret void } +define void @sanitized_no_ps() sanitize_hwaddress { ret void } diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/pgo-opt-out.ll b/llvm/test/Instrumentation/HWAddressSanitizer/pgo-opt-out.ll new file mode 100644 index 000000000000..65a5f8c96896 --- /dev/null +++ b/llvm/test/Instrumentation/HWAddressSanitizer/pgo-opt-out.ll @@ -0,0 +1,33 @@ +; RUN: opt < %s -passes='require,hwasan' -S -stats 2>&1 \ +; RUN: -hwasan-skip-hot-code=1 | FileCheck %s --check-prefix=DEFAULT +; RUN: opt < %s -passes='require,hwasan' -S -stats 2>&1 \ +; RUN: -hwasan-skip-hot-code=1 -hwasan-percentile-cutoff-hot=700000 | FileCheck %s --check-prefix=PERCENT + +; REQUIRES: asserts + +; DEFAULT: 1 hwasan - Number of total funcs HWASAN + +; PERCENT: 1 hwasan - Number of HWASAN instrumented funcs +; PERCENT: 1 hwasan - Number of total funcs HWASAN + +define void @sanitized() sanitize_hwaddress !prof !36 { ret void } + +!llvm.module.flags = !{!6} +!6 = !{i32 1, !"ProfileSummary", !7} +!7 = !{!8, !9, !10, !11, !12, !13, !14, !17} +!8 = !{!"ProfileFormat", !"InstrProf"} +!9 = !{!"TotalCount", i64 30000} +!10 = !{!"MaxCount", i64 10000} +!11 = !{!"MaxInternalCount", i64 10000} +!12 = !{!"MaxFunctionCount", i64 10000} +!13 = !{!"NumCounts", i64 3} +!14 = !{!"NumFunctions", i64 5} +!17 = !{!"DetailedSummary", !18} +!18 = !{!19, !29, !30, !32, !34} +!19 = !{i32 10000, i64 10000, i32 3} +!29 = !{i32 950000, i64 5000, i32 3} +!30 = !{i32 990000, i64 500, i32 4} +!32 = !{i32 999900, i64 250, i32 4} +!34 = !{i32 999999, i64 1, i32 6} + +!36 = !{!"function_entry_count", i64 1000} -- GitLab From c996023f9a288f12f97de8e1a2b8fbb28d2ed0c3 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Mon, 4 Mar 2024 12:23:11 -0600 Subject: [PATCH 024/929] [libc] Provide an implementation of the 'stdint.h' header (#83353) Summary: I've noticed one problem is that the user includes `stdint.h` the compiler will do `#include_next ` potentially into a conflicting implementation on systems with multiple headers installed. The `clang` header is standards compliant and works with `clang` and `gcc` which are both of our targets, so I simply copied it here. This has the effect of including `stdint.h` on clang / LLVM libc behaving the same as `-ffreestanding`. --- libc/config/baremetal/arm/headers.txt | 1 + libc/config/baremetal/riscv/headers.txt | 1 + libc/config/darwin/arm/headers.txt | 1 + libc/config/darwin/x86_64/headers.txt | 1 + libc/config/gpu/headers.txt | 1 + libc/config/linux/aarch64/headers.txt | 1 + libc/config/linux/arm/headers.txt | 1 + libc/config/linux/riscv/headers.txt | 1 + libc/config/linux/x86_64/headers.txt | 1 + libc/include/CMakeLists.txt | 8 + libc/include/llvm-libc-macros/CMakeLists.txt | 6 + libc/include/llvm-libc-macros/stdint-macros.h | 878 ++++++++++++++++++ libc/include/stdint.h.def | 14 + libc/spec/stdc.td | 3 + .../llvm-project-overlay/libc/BUILD.bazel | 5 + 15 files changed, 923 insertions(+) create mode 100644 libc/include/llvm-libc-macros/stdint-macros.h create mode 100644 libc/include/stdint.h.def diff --git a/libc/config/baremetal/arm/headers.txt b/libc/config/baremetal/arm/headers.txt index 4c02ac84018d..bc43abf47bbd 100644 --- a/libc/config/baremetal/arm/headers.txt +++ b/libc/config/baremetal/arm/headers.txt @@ -4,6 +4,7 @@ set(TARGET_PUBLIC_HEADERS libc.include.fenv libc.include.errno libc.include.float + libc.include.stdint libc.include.inttypes libc.include.math libc.include.stdio diff --git a/libc/config/baremetal/riscv/headers.txt b/libc/config/baremetal/riscv/headers.txt index 4c02ac84018d..bc43abf47bbd 100644 --- a/libc/config/baremetal/riscv/headers.txt +++ b/libc/config/baremetal/riscv/headers.txt @@ -4,6 +4,7 @@ set(TARGET_PUBLIC_HEADERS libc.include.fenv libc.include.errno libc.include.float + libc.include.stdint libc.include.inttypes libc.include.math libc.include.stdio diff --git a/libc/config/darwin/arm/headers.txt b/libc/config/darwin/arm/headers.txt index d80628445af5..86e714597232 100644 --- a/libc/config/darwin/arm/headers.txt +++ b/libc/config/darwin/arm/headers.txt @@ -3,6 +3,7 @@ set(TARGET_PUBLIC_HEADERS libc.include.errno libc.include.fenv libc.include.float + libc.include.stdint libc.include.inttypes libc.include.limits libc.include.math diff --git a/libc/config/darwin/x86_64/headers.txt b/libc/config/darwin/x86_64/headers.txt index 1e81e303ddd6..acd240264987 100644 --- a/libc/config/darwin/x86_64/headers.txt +++ b/libc/config/darwin/x86_64/headers.txt @@ -4,6 +4,7 @@ set(TARGET_PUBLIC_HEADERS # Fenv is currently disabled. #libc.include.fenv libc.include.float + libc.include.stdint libc.include.inttypes libc.include.limits libc.include.math diff --git a/libc/config/gpu/headers.txt b/libc/config/gpu/headers.txt index 3b04dd89fafe..dd16938da8a4 100644 --- a/libc/config/gpu/headers.txt +++ b/libc/config/gpu/headers.txt @@ -3,6 +3,7 @@ set(TARGET_PUBLIC_HEADERS libc.include.ctype libc.include.string libc.include.float + libc.include.stdint libc.include.inttypes libc.include.limits libc.include.math diff --git a/libc/config/linux/aarch64/headers.txt b/libc/config/linux/aarch64/headers.txt index 21b880cf2bb9..47db4434b09b 100644 --- a/libc/config/linux/aarch64/headers.txt +++ b/libc/config/linux/aarch64/headers.txt @@ -5,6 +5,7 @@ set(TARGET_PUBLIC_HEADERS libc.include.features libc.include.fenv libc.include.float + libc.include.stdint libc.include.inttypes libc.include.limits libc.include.math diff --git a/libc/config/linux/arm/headers.txt b/libc/config/linux/arm/headers.txt index 268c8c41702a..307bb6b146a4 100644 --- a/libc/config/linux/arm/headers.txt +++ b/libc/config/linux/arm/headers.txt @@ -3,6 +3,7 @@ set(TARGET_PUBLIC_HEADERS libc.include.fenv libc.include.errno libc.include.float + libc.include.stdint libc.include.inttypes libc.include.math libc.include.stdckdint diff --git a/libc/config/linux/riscv/headers.txt b/libc/config/linux/riscv/headers.txt index 3ebea2e2b07b..c858bcc978d9 100644 --- a/libc/config/linux/riscv/headers.txt +++ b/libc/config/linux/riscv/headers.txt @@ -7,6 +7,7 @@ set(TARGET_PUBLIC_HEADERS libc.include.features libc.include.fenv libc.include.float + libc.include.stdint libc.include.inttypes libc.include.limits libc.include.math diff --git a/libc/config/linux/x86_64/headers.txt b/libc/config/linux/x86_64/headers.txt index a887eba6805b..e51c79319427 100644 --- a/libc/config/linux/x86_64/headers.txt +++ b/libc/config/linux/x86_64/headers.txt @@ -7,6 +7,7 @@ set(TARGET_PUBLIC_HEADERS libc.include.features libc.include.fenv libc.include.float + libc.include.stdint libc.include.inttypes libc.include.limits libc.include.math diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt index 9090b3bca01e..34d6839fd789 100644 --- a/libc/include/CMakeLists.txt +++ b/libc/include/CMakeLists.txt @@ -84,6 +84,14 @@ add_gen_header( .llvm-libc-macros.float_macros ) +add_gen_header( + stdint + DEF_FILE stdint.h.def + GEN_HDR stdint.h + DEPENDS + .llvm-libc-macros.stdint_macros +) + add_gen_header( limits DEF_FILE limits.h.def diff --git a/libc/include/llvm-libc-macros/CMakeLists.txt b/libc/include/llvm-libc-macros/CMakeLists.txt index 157b786aa7e8..635ccadfb49e 100644 --- a/libc/include/llvm-libc-macros/CMakeLists.txt +++ b/libc/include/llvm-libc-macros/CMakeLists.txt @@ -67,6 +67,12 @@ add_macro_header( file-seek-macros.h ) +add_macro_header( + stdint_macros + HDR + stdint-macros.h +) + add_macro_header( float_macros HDR diff --git a/libc/include/llvm-libc-macros/stdint-macros.h b/libc/include/llvm-libc-macros/stdint-macros.h new file mode 100644 index 000000000000..1d5da2b783b6 --- /dev/null +++ b/libc/include/llvm-libc-macros/stdint-macros.h @@ -0,0 +1,878 @@ +//===-- Definition of macros from stdint.h --------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_MACROS_STDINT_MACROS_H +#define LLVM_LIBC_MACROS_STDINT_MACROS_H + +// These definitions are copied directly from the clang implementation located +// at 'clang/lib/Headers/stdint.h'. We provide it here again for compatibility. + +/* C99 7.18.1.1 Exact-width integer types. + * C99 7.18.1.2 Minimum-width integer types. + * C99 7.18.1.3 Fastest minimum-width integer types. + * + * The standard requires that exact-width type be defined for 8-, 16-, 32-, and + * 64-bit types if they are implemented. Other exact width types are optional. + * This implementation defines an exact-width types for every integer width + * that is represented in the standard integer types. + * + * The standard also requires minimum-width types be defined for 8-, 16-, 32-, + * and 64-bit widths regardless of whether there are corresponding exact-width + * types. + * + * To accommodate targets that are missing types that are exactly 8, 16, 32, or + * 64 bits wide, this implementation takes an approach of cascading + * redefinitions, redefining __int_leastN_t to successively smaller exact-width + * types. It is therefore important that the types are defined in order of + * descending widths. + * + * We currently assume that the minimum-width types and the fastest + * minimum-width types are the same. This is allowed by the standard, but is + * suboptimal. + * + * In violation of the standard, some targets do not implement a type that is + * wide enough to represent all of the required widths (8-, 16-, 32-, 64-bit). + * To accommodate these targets, a required minimum-width type is only + * defined if there exists an exact-width type of equal or greater width. + */ + +#ifdef __INT64_TYPE__ +#ifndef __int8_t_defined /* glibc sys/types.h also defines int64_t*/ +typedef __INT64_TYPE__ int64_t; +#endif /* __int8_t_defined */ +typedef __UINT64_TYPE__ uint64_t; +#undef __int_least64_t +#define __int_least64_t int64_t +#undef __uint_least64_t +#define __uint_least64_t uint64_t +#undef __int_least32_t +#define __int_least32_t int64_t +#undef __uint_least32_t +#define __uint_least32_t uint64_t +#undef __int_least16_t +#define __int_least16_t int64_t +#undef __uint_least16_t +#define __uint_least16_t uint64_t +#undef __int_least8_t +#define __int_least8_t int64_t +#undef __uint_least8_t +#define __uint_least8_t uint64_t +#endif /* __INT64_TYPE__ */ + +#ifdef __int_least64_t +typedef __int_least64_t int_least64_t; +typedef __uint_least64_t uint_least64_t; +typedef __int_least64_t int_fast64_t; +typedef __uint_least64_t uint_fast64_t; +#endif /* __int_least64_t */ + +#ifdef __INT56_TYPE__ +typedef __INT56_TYPE__ int56_t; +typedef __UINT56_TYPE__ uint56_t; +typedef int56_t int_least56_t; +typedef uint56_t uint_least56_t; +typedef int56_t int_fast56_t; +typedef uint56_t uint_fast56_t; +#undef __int_least32_t +#define __int_least32_t int56_t +#undef __uint_least32_t +#define __uint_least32_t uint56_t +#undef __int_least16_t +#define __int_least16_t int56_t +#undef __uint_least16_t +#define __uint_least16_t uint56_t +#undef __int_least8_t +#define __int_least8_t int56_t +#undef __uint_least8_t +#define __uint_least8_t uint56_t +#endif /* __INT56_TYPE__ */ + +#ifdef __INT48_TYPE__ +typedef __INT48_TYPE__ int48_t; +typedef __UINT48_TYPE__ uint48_t; +typedef int48_t int_least48_t; +typedef uint48_t uint_least48_t; +typedef int48_t int_fast48_t; +typedef uint48_t uint_fast48_t; +#undef __int_least32_t +#define __int_least32_t int48_t +#undef __uint_least32_t +#define __uint_least32_t uint48_t +#undef __int_least16_t +#define __int_least16_t int48_t +#undef __uint_least16_t +#define __uint_least16_t uint48_t +#undef __int_least8_t +#define __int_least8_t int48_t +#undef __uint_least8_t +#define __uint_least8_t uint48_t +#endif /* __INT48_TYPE__ */ + +#ifdef __INT40_TYPE__ +typedef __INT40_TYPE__ int40_t; +typedef __UINT40_TYPE__ uint40_t; +typedef int40_t int_least40_t; +typedef uint40_t uint_least40_t; +typedef int40_t int_fast40_t; +typedef uint40_t uint_fast40_t; +#undef __int_least32_t +#define __int_least32_t int40_t +#undef __uint_least32_t +#define __uint_least32_t uint40_t +#undef __int_least16_t +#define __int_least16_t int40_t +#undef __uint_least16_t +#define __uint_least16_t uint40_t +#undef __int_least8_t +#define __int_least8_t int40_t +#undef __uint_least8_t +#define __uint_least8_t uint40_t +#endif /* __INT40_TYPE__ */ + +#ifdef __INT32_TYPE__ + +#ifndef __int8_t_defined /* glibc sys/types.h also defines int32_t*/ +typedef __INT32_TYPE__ int32_t; +#endif /* __int8_t_defined */ + +#ifndef __uint32_t_defined /* more glibc compatibility */ +#define __uint32_t_defined +typedef __UINT32_TYPE__ uint32_t; +#endif /* __uint32_t_defined */ + +#undef __int_least32_t +#define __int_least32_t int32_t +#undef __uint_least32_t +#define __uint_least32_t uint32_t +#undef __int_least16_t +#define __int_least16_t int32_t +#undef __uint_least16_t +#define __uint_least16_t uint32_t +#undef __int_least8_t +#define __int_least8_t int32_t +#undef __uint_least8_t +#define __uint_least8_t uint32_t +#endif /* __INT32_TYPE__ */ + +#ifdef __int_least32_t +typedef __int_least32_t int_least32_t; +typedef __uint_least32_t uint_least32_t; +typedef __int_least32_t int_fast32_t; +typedef __uint_least32_t uint_fast32_t; +#endif /* __int_least32_t */ + +#ifdef __INT24_TYPE__ +typedef __INT24_TYPE__ int24_t; +typedef __UINT24_TYPE__ uint24_t; +typedef int24_t int_least24_t; +typedef uint24_t uint_least24_t; +typedef int24_t int_fast24_t; +typedef uint24_t uint_fast24_t; +#undef __int_least16_t +#define __int_least16_t int24_t +#undef __uint_least16_t +#define __uint_least16_t uint24_t +#undef __int_least8_t +#define __int_least8_t int24_t +#undef __uint_least8_t +#define __uint_least8_t uint24_t +#endif /* __INT24_TYPE__ */ + +#ifdef __INT16_TYPE__ +#ifndef __int8_t_defined /* glibc sys/types.h also defines int16_t*/ +typedef __INT16_TYPE__ int16_t; +#endif /* __int8_t_defined */ +typedef __UINT16_TYPE__ uint16_t; +#undef __int_least16_t +#define __int_least16_t int16_t +#undef __uint_least16_t +#define __uint_least16_t uint16_t +#undef __int_least8_t +#define __int_least8_t int16_t +#undef __uint_least8_t +#define __uint_least8_t uint16_t +#endif /* __INT16_TYPE__ */ + +#ifdef __int_least16_t +typedef __int_least16_t int_least16_t; +typedef __uint_least16_t uint_least16_t; +typedef __int_least16_t int_fast16_t; +typedef __uint_least16_t uint_fast16_t; +#endif /* __int_least16_t */ + +#ifdef __INT8_TYPE__ +#ifndef __int8_t_defined /* glibc sys/types.h also defines int8_t*/ +typedef __INT8_TYPE__ int8_t; +#endif /* __int8_t_defined */ +typedef __UINT8_TYPE__ uint8_t; +#undef __int_least8_t +#define __int_least8_t int8_t +#undef __uint_least8_t +#define __uint_least8_t uint8_t +#endif /* __INT8_TYPE__ */ + +#ifdef __int_least8_t +typedef __int_least8_t int_least8_t; +typedef __uint_least8_t uint_least8_t; +typedef __int_least8_t int_fast8_t; +typedef __uint_least8_t uint_fast8_t; +#endif /* __int_least8_t */ + +/* prevent glibc sys/types.h from defining conflicting types */ +#ifndef __int8_t_defined +#define __int8_t_defined +#endif /* __int8_t_defined */ + +/* C99 7.18.1.4 Integer types capable of holding object pointers. + */ +#define __stdint_join3(a, b, c) a##b##c + +#ifndef _INTPTR_T +#ifndef __intptr_t_defined +typedef __INTPTR_TYPE__ intptr_t; +#define __intptr_t_defined +#define _INTPTR_T +#endif +#endif + +#ifndef _UINTPTR_T +typedef __UINTPTR_TYPE__ uintptr_t; +#define _UINTPTR_T +#endif + +/* C99 7.18.1.5 Greatest-width integer types. + */ +typedef __INTMAX_TYPE__ intmax_t; +typedef __UINTMAX_TYPE__ uintmax_t; + +/* C99 7.18.4 Macros for minimum-width integer constants. + * + * The standard requires that integer constant macros be defined for all the + * minimum-width types defined above. As 8-, 16-, 32-, and 64-bit minimum-width + * types are required, the corresponding integer constant macros are defined + * here. This implementation also defines minimum-width types for every other + * integer width that the target implements, so corresponding macros are + * defined below, too. + * + * These macros are defined using the same successive-shrinking approach as + * the type definitions above. It is likewise important that macros are defined + * in order of decending width. + * + * Note that C++ should not check __STDC_CONSTANT_MACROS here, contrary to the + * claims of the C standard (see C++ 18.3.1p2, [cstdint.syn]). + */ + +#define __int_c_join(a, b) a##b +#define __int_c(v, suffix) __int_c_join(v, suffix) +#define __uint_c(v, suffix) __int_c_join(v##U, suffix) + +#ifdef __INT64_TYPE__ +#undef __int64_c_suffix +#undef __int32_c_suffix +#undef __int16_c_suffix +#undef __int8_c_suffix +#ifdef __INT64_C_SUFFIX__ +#define __int64_c_suffix __INT64_C_SUFFIX__ +#define __int32_c_suffix __INT64_C_SUFFIX__ +#define __int16_c_suffix __INT64_C_SUFFIX__ +#define __int8_c_suffix __INT64_C_SUFFIX__ +#endif /* __INT64_C_SUFFIX__ */ +#endif /* __INT64_TYPE__ */ + +#ifdef __int_least64_t +#ifdef __int64_c_suffix +#define INT64_C(v) __int_c(v, __int64_c_suffix) +#define UINT64_C(v) __uint_c(v, __int64_c_suffix) +#else +#define INT64_C(v) v +#define UINT64_C(v) v##U +#endif /* __int64_c_suffix */ +#endif /* __int_least64_t */ + +#ifdef __INT56_TYPE__ +#undef __int32_c_suffix +#undef __int16_c_suffix +#undef __int8_c_suffix +#ifdef __INT56_C_SUFFIX__ +#define INT56_C(v) __int_c(v, __INT56_C_SUFFIX__) +#define UINT56_C(v) __uint_c(v, __INT56_C_SUFFIX__) +#define __int32_c_suffix __INT56_C_SUFFIX__ +#define __int16_c_suffix __INT56_C_SUFFIX__ +#define __int8_c_suffix __INT56_C_SUFFIX__ +#else +#define INT56_C(v) v +#define UINT56_C(v) v##U +#endif /* __INT56_C_SUFFIX__ */ +#endif /* __INT56_TYPE__ */ + +#ifdef __INT48_TYPE__ +#undef __int32_c_suffix +#undef __int16_c_suffix +#undef __int8_c_suffix +#ifdef __INT48_C_SUFFIX__ +#define INT48_C(v) __int_c(v, __INT48_C_SUFFIX__) +#define UINT48_C(v) __uint_c(v, __INT48_C_SUFFIX__) +#define __int32_c_suffix __INT48_C_SUFFIX__ +#define __int16_c_suffix __INT48_C_SUFFIX__ +#define __int8_c_suffix __INT48_C_SUFFIX__ +#else +#define INT48_C(v) v +#define UINT48_C(v) v##U +#endif /* __INT48_C_SUFFIX__ */ +#endif /* __INT48_TYPE__ */ + +#ifdef __INT40_TYPE__ +#undef __int32_c_suffix +#undef __int16_c_suffix +#undef __int8_c_suffix +#ifdef __INT40_C_SUFFIX__ +#define INT40_C(v) __int_c(v, __INT40_C_SUFFIX__) +#define UINT40_C(v) __uint_c(v, __INT40_C_SUFFIX__) +#define __int32_c_suffix __INT40_C_SUFFIX__ +#define __int16_c_suffix __INT40_C_SUFFIX__ +#define __int8_c_suffix __INT40_C_SUFFIX__ +#else +#define INT40_C(v) v +#define UINT40_C(v) v##U +#endif /* __INT40_C_SUFFIX__ */ +#endif /* __INT40_TYPE__ */ + +#ifdef __INT32_TYPE__ +#undef __int32_c_suffix +#undef __int16_c_suffix +#undef __int8_c_suffix +#ifdef __INT32_C_SUFFIX__ +#define __int32_c_suffix __INT32_C_SUFFIX__ +#define __int16_c_suffix __INT32_C_SUFFIX__ +#define __int8_c_suffix __INT32_C_SUFFIX__ +#endif /* __INT32_C_SUFFIX__ */ +#endif /* __INT32_TYPE__ */ + +#ifdef __int_least32_t +#ifdef __int32_c_suffix +#define INT32_C(v) __int_c(v, __int32_c_suffix) +#define UINT32_C(v) __uint_c(v, __int32_c_suffix) +#else +#define INT32_C(v) v +#define UINT32_C(v) v##U +#endif /* __int32_c_suffix */ +#endif /* __int_least32_t */ + +#ifdef __INT24_TYPE__ +#undef __int16_c_suffix +#undef __int8_c_suffix +#ifdef __INT24_C_SUFFIX__ +#define INT24_C(v) __int_c(v, __INT24_C_SUFFIX__) +#define UINT24_C(v) __uint_c(v, __INT24_C_SUFFIX__) +#define __int16_c_suffix __INT24_C_SUFFIX__ +#define __int8_c_suffix __INT24_C_SUFFIX__ +#else +#define INT24_C(v) v +#define UINT24_C(v) v##U +#endif /* __INT24_C_SUFFIX__ */ +#endif /* __INT24_TYPE__ */ + +#ifdef __INT16_TYPE__ +#undef __int16_c_suffix +#undef __int8_c_suffix +#ifdef __INT16_C_SUFFIX__ +#define __int16_c_suffix __INT16_C_SUFFIX__ +#define __int8_c_suffix __INT16_C_SUFFIX__ +#endif /* __INT16_C_SUFFIX__ */ +#endif /* __INT16_TYPE__ */ + +#ifdef __int_least16_t +#ifdef __int16_c_suffix +#define INT16_C(v) __int_c(v, __int16_c_suffix) +#define UINT16_C(v) __uint_c(v, __int16_c_suffix) +#else +#define INT16_C(v) v +#define UINT16_C(v) v##U +#endif /* __int16_c_suffix */ +#endif /* __int_least16_t */ + +#ifdef __INT8_TYPE__ +#undef __int8_c_suffix +#ifdef __INT8_C_SUFFIX__ +#define __int8_c_suffix __INT8_C_SUFFIX__ +#endif /* __INT8_C_SUFFIX__ */ +#endif /* __INT8_TYPE__ */ + +#ifdef __int_least8_t +#ifdef __int8_c_suffix +#define INT8_C(v) __int_c(v, __int8_c_suffix) +#define UINT8_C(v) __uint_c(v, __int8_c_suffix) +#else +#define INT8_C(v) v +#define UINT8_C(v) v##U +#endif /* __int8_c_suffix */ +#endif /* __int_least8_t */ + +/* C99 7.18.2.1 Limits of exact-width integer types. + * C99 7.18.2.2 Limits of minimum-width integer types. + * C99 7.18.2.3 Limits of fastest minimum-width integer types. + * + * The presence of limit macros are completely optional in C99. This + * implementation defines limits for all of the types (exact- and + * minimum-width) that it defines above, using the limits of the minimum-width + * type for any types that do not have exact-width representations. + * + * As in the type definitions, this section takes an approach of + * successive-shrinking to determine which limits to use for the standard (8, + * 16, 32, 64) bit widths when they don't have exact representations. It is + * therefore important that the definitions be kept in order of decending + * widths. + * + * Note that C++ should not check __STDC_LIMIT_MACROS here, contrary to the + * claims of the C standard (see C++ 18.3.1p2, [cstdint.syn]). + */ + +#ifdef __INT64_TYPE__ +#define INT64_MAX INT64_C(9223372036854775807) +#define INT64_MIN (-INT64_C(9223372036854775807) - 1) +#define UINT64_MAX UINT64_C(18446744073709551615) + +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +#define UINT64_WIDTH 64 +#define INT64_WIDTH UINT64_WIDTH + +#define __UINT_LEAST64_WIDTH UINT64_WIDTH +#undef __UINT_LEAST32_WIDTH +#define __UINT_LEAST32_WIDTH UINT64_WIDTH +#undef __UINT_LEAST16_WIDTH +#define __UINT_LEAST16_WIDTH UINT64_WIDTH +#undef __UINT_LEAST8_MAX +#define __UINT_LEAST8_MAX UINT64_MAX +#endif /* __STDC_VERSION__ */ + +#define __INT_LEAST64_MIN INT64_MIN +#define __INT_LEAST64_MAX INT64_MAX +#define __UINT_LEAST64_MAX UINT64_MAX +#undef __INT_LEAST32_MIN +#define __INT_LEAST32_MIN INT64_MIN +#undef __INT_LEAST32_MAX +#define __INT_LEAST32_MAX INT64_MAX +#undef __UINT_LEAST32_MAX +#define __UINT_LEAST32_MAX UINT64_MAX +#undef __INT_LEAST16_MIN +#define __INT_LEAST16_MIN INT64_MIN +#undef __INT_LEAST16_MAX +#define __INT_LEAST16_MAX INT64_MAX +#undef __UINT_LEAST16_MAX +#define __UINT_LEAST16_MAX UINT64_MAX +#undef __INT_LEAST8_MIN +#define __INT_LEAST8_MIN INT64_MIN +#undef __INT_LEAST8_MAX +#define __INT_LEAST8_MAX INT64_MAX +#undef __UINT_LEAST8_MAX +#define __UINT_LEAST8_MAX UINT64_MAX +#endif /* __INT64_TYPE__ */ + +#ifdef __INT_LEAST64_MIN +#define INT_LEAST64_MIN __INT_LEAST64_MIN +#define INT_LEAST64_MAX __INT_LEAST64_MAX +#define UINT_LEAST64_MAX __UINT_LEAST64_MAX +#define INT_FAST64_MIN __INT_LEAST64_MIN +#define INT_FAST64_MAX __INT_LEAST64_MAX +#define UINT_FAST64_MAX __UINT_LEAST64_MAX + +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +#define UINT_LEAST64_WIDTH __UINT_LEAST64_WIDTH +#define INT_LEAST64_WIDTH UINT_LEAST64_WIDTH +#define UINT_FAST64_WIDTH __UINT_LEAST64_WIDTH +#define INT_FAST64_WIDTH UINT_FAST64_WIDTH +#endif /* __STDC_VERSION__ */ +#endif /* __INT_LEAST64_MIN */ + +#ifdef __INT56_TYPE__ +#define INT56_MAX INT56_C(36028797018963967) +#define INT56_MIN (-INT56_C(36028797018963967) - 1) +#define UINT56_MAX UINT56_C(72057594037927935) +#define INT_LEAST56_MIN INT56_MIN +#define INT_LEAST56_MAX INT56_MAX +#define UINT_LEAST56_MAX UINT56_MAX +#define INT_FAST56_MIN INT56_MIN +#define INT_FAST56_MAX INT56_MAX +#define UINT_FAST56_MAX UINT56_MAX + +#undef __INT_LEAST32_MIN +#define __INT_LEAST32_MIN INT56_MIN +#undef __INT_LEAST32_MAX +#define __INT_LEAST32_MAX INT56_MAX +#undef __UINT_LEAST32_MAX +#define __UINT_LEAST32_MAX UINT56_MAX +#undef __INT_LEAST16_MIN +#define __INT_LEAST16_MIN INT56_MIN +#undef __INT_LEAST16_MAX +#define __INT_LEAST16_MAX INT56_MAX +#undef __UINT_LEAST16_MAX +#define __UINT_LEAST16_MAX UINT56_MAX +#undef __INT_LEAST8_MIN +#define __INT_LEAST8_MIN INT56_MIN +#undef __INT_LEAST8_MAX +#define __INT_LEAST8_MAX INT56_MAX +#undef __UINT_LEAST8_MAX +#define __UINT_LEAST8_MAX UINT56_MAX + +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +#define UINT56_WIDTH 56 +#define INT56_WIDTH UINT56_WIDTH +#define UINT_LEAST56_WIDTH UINT56_WIDTH +#define INT_LEAST56_WIDTH UINT_LEAST56_WIDTH +#define UINT_FAST56_WIDTH UINT56_WIDTH +#define INT_FAST56_WIDTH UINT_FAST56_WIDTH +#undef __UINT_LEAST32_WIDTH +#define __UINT_LEAST32_WIDTH UINT56_WIDTH +#undef __UINT_LEAST16_WIDTH +#define __UINT_LEAST16_WIDTH UINT56_WIDTH +#undef __UINT_LEAST8_WIDTH +#define __UINT_LEAST8_WIDTH UINT56_WIDTH +#endif /* __STDC_VERSION__ */ +#endif /* __INT56_TYPE__ */ + +#ifdef __INT48_TYPE__ +#define INT48_MAX INT48_C(140737488355327) +#define INT48_MIN (-INT48_C(140737488355327) - 1) +#define UINT48_MAX UINT48_C(281474976710655) +#define INT_LEAST48_MIN INT48_MIN +#define INT_LEAST48_MAX INT48_MAX +#define UINT_LEAST48_MAX UINT48_MAX +#define INT_FAST48_MIN INT48_MIN +#define INT_FAST48_MAX INT48_MAX +#define UINT_FAST48_MAX UINT48_MAX + +#undef __INT_LEAST32_MIN +#define __INT_LEAST32_MIN INT48_MIN +#undef __INT_LEAST32_MAX +#define __INT_LEAST32_MAX INT48_MAX +#undef __UINT_LEAST32_MAX +#define __UINT_LEAST32_MAX UINT48_MAX +#undef __INT_LEAST16_MIN +#define __INT_LEAST16_MIN INT48_MIN +#undef __INT_LEAST16_MAX +#define __INT_LEAST16_MAX INT48_MAX +#undef __UINT_LEAST16_MAX +#define __UINT_LEAST16_MAX UINT48_MAX +#undef __INT_LEAST8_MIN +#define __INT_LEAST8_MIN INT48_MIN +#undef __INT_LEAST8_MAX +#define __INT_LEAST8_MAX INT48_MAX +#undef __UINT_LEAST8_MAX +#define __UINT_LEAST8_MAX UINT48_MAX + +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +#define UINT48_WIDTH 48 +#define INT48_WIDTH UINT48_WIDTH +#define UINT_LEAST48_WIDTH UINT48_WIDTH +#define INT_LEAST48_WIDTH UINT_LEAST48_WIDTH +#define UINT_FAST48_WIDTH UINT48_WIDTH +#define INT_FAST48_WIDTH UINT_FAST48_WIDTH +#undef __UINT_LEAST32_WIDTH +#define __UINT_LEAST32_WIDTH UINT48_WIDTH +#undef __UINT_LEAST16_WIDTH +#define __UINT_LEAST16_WIDTH UINT48_WIDTH +#undef __UINT_LEAST8_WIDTH +#define __UINT_LEAST8_WIDTH UINT48_WIDTH +#endif /* __STDC_VERSION__ */ +#endif /* __INT48_TYPE__ */ + +#ifdef __INT40_TYPE__ +#define INT40_MAX INT40_C(549755813887) +#define INT40_MIN (-INT40_C(549755813887) - 1) +#define UINT40_MAX UINT40_C(1099511627775) +#define INT_LEAST40_MIN INT40_MIN +#define INT_LEAST40_MAX INT40_MAX +#define UINT_LEAST40_MAX UINT40_MAX +#define INT_FAST40_MIN INT40_MIN +#define INT_FAST40_MAX INT40_MAX +#define UINT_FAST40_MAX UINT40_MAX + +#undef __INT_LEAST32_MIN +#define __INT_LEAST32_MIN INT40_MIN +#undef __INT_LEAST32_MAX +#define __INT_LEAST32_MAX INT40_MAX +#undef __UINT_LEAST32_MAX +#define __UINT_LEAST32_MAX UINT40_MAX +#undef __INT_LEAST16_MIN +#define __INT_LEAST16_MIN INT40_MIN +#undef __INT_LEAST16_MAX +#define __INT_LEAST16_MAX INT40_MAX +#undef __UINT_LEAST16_MAX +#define __UINT_LEAST16_MAX UINT40_MAX +#undef __INT_LEAST8_MIN +#define __INT_LEAST8_MIN INT40_MIN +#undef __INT_LEAST8_MAX +#define __INT_LEAST8_MAX INT40_MAX +#undef __UINT_LEAST8_MAX +#define __UINT_LEAST8_MAX UINT40_MAX + +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +#define UINT40_WIDTH 40 +#define INT40_WIDTH UINT40_WIDTH +#define UINT_LEAST40_WIDTH UINT40_WIDTH +#define INT_LEAST40_WIDTH UINT_LEAST40_WIDTH +#define UINT_FAST40_WIDTH UINT40_WIDTH +#define INT_FAST40_WIDTH UINT_FAST40_WIDTH +#undef __UINT_LEAST32_WIDTH +#define __UINT_LEAST32_WIDTH UINT40_WIDTH +#undef __UINT_LEAST16_WIDTH +#define __UINT_LEAST16_WIDTH UINT40_WIDTH +#undef __UINT_LEAST8_WIDTH +#define __UINT_LEAST8_WIDTH UINT40_WIDTH +#endif /* __STDC_VERSION__ */ +#endif /* __INT40_TYPE__ */ + +#ifdef __INT32_TYPE__ +#define INT32_MAX INT32_C(2147483647) +#define INT32_MIN (-INT32_C(2147483647) - 1) +#define UINT32_MAX UINT32_C(4294967295) + +#undef __INT_LEAST32_MIN +#define __INT_LEAST32_MIN INT32_MIN +#undef __INT_LEAST32_MAX +#define __INT_LEAST32_MAX INT32_MAX +#undef __UINT_LEAST32_MAX +#define __UINT_LEAST32_MAX UINT32_MAX +#undef __INT_LEAST16_MIN +#define __INT_LEAST16_MIN INT32_MIN +#undef __INT_LEAST16_MAX +#define __INT_LEAST16_MAX INT32_MAX +#undef __UINT_LEAST16_MAX +#define __UINT_LEAST16_MAX UINT32_MAX +#undef __INT_LEAST8_MIN +#define __INT_LEAST8_MIN INT32_MIN +#undef __INT_LEAST8_MAX +#define __INT_LEAST8_MAX INT32_MAX +#undef __UINT_LEAST8_MAX +#define __UINT_LEAST8_MAX UINT32_MAX + +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +#define UINT32_WIDTH 32 +#define INT32_WIDTH UINT32_WIDTH +#undef __UINT_LEAST32_WIDTH +#define __UINT_LEAST32_WIDTH UINT32_WIDTH +#undef __UINT_LEAST16_WIDTH +#define __UINT_LEAST16_WIDTH UINT32_WIDTH +#undef __UINT_LEAST8_WIDTH +#define __UINT_LEAST8_WIDTH UINT32_WIDTH +#endif /* __STDC_VERSION__ */ +#endif /* __INT32_TYPE__ */ + +#ifdef __INT_LEAST32_MIN +#define INT_LEAST32_MIN __INT_LEAST32_MIN +#define INT_LEAST32_MAX __INT_LEAST32_MAX +#define UINT_LEAST32_MAX __UINT_LEAST32_MAX +#define INT_FAST32_MIN __INT_LEAST32_MIN +#define INT_FAST32_MAX __INT_LEAST32_MAX +#define UINT_FAST32_MAX __UINT_LEAST32_MAX + +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +#define UINT_LEAST32_WIDTH __UINT_LEAST32_WIDTH +#define INT_LEAST32_WIDTH UINT_LEAST32_WIDTH +#define UINT_FAST32_WIDTH __UINT_LEAST32_WIDTH +#define INT_FAST32_WIDTH UINT_FAST32_WIDTH +#endif /* __STDC_VERSION__ */ +#endif /* __INT_LEAST32_MIN */ + +#ifdef __INT24_TYPE__ +#define INT24_MAX INT24_C(8388607) +#define INT24_MIN (-INT24_C(8388607) - 1) +#define UINT24_MAX UINT24_C(16777215) +#define INT_LEAST24_MIN INT24_MIN +#define INT_LEAST24_MAX INT24_MAX +#define UINT_LEAST24_MAX UINT24_MAX +#define INT_FAST24_MIN INT24_MIN +#define INT_FAST24_MAX INT24_MAX +#define UINT_FAST24_MAX UINT24_MAX + +#undef __INT_LEAST16_MIN +#define __INT_LEAST16_MIN INT24_MIN +#undef __INT_LEAST16_MAX +#define __INT_LEAST16_MAX INT24_MAX +#undef __UINT_LEAST16_MAX +#define __UINT_LEAST16_MAX UINT24_MAX +#undef __INT_LEAST8_MIN +#define __INT_LEAST8_MIN INT24_MIN +#undef __INT_LEAST8_MAX +#define __INT_LEAST8_MAX INT24_MAX +#undef __UINT_LEAST8_MAX +#define __UINT_LEAST8_MAX UINT24_MAX + +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +#define UINT24_WIDTH 24 +#define INT24_WIDTH UINT24_WIDTH +#define UINT_LEAST24_WIDTH UINT24_WIDTH +#define INT_LEAST24_WIDTH UINT_LEAST24_WIDTH +#define UINT_FAST24_WIDTH UINT24_WIDTH +#define INT_FAST24_WIDTH UINT_FAST24_WIDTH +#undef __UINT_LEAST16_WIDTH +#define __UINT_LEAST16_WIDTH UINT24_WIDTH +#undef __UINT_LEAST8_WIDTH +#define __UINT_LEAST8_WIDTH UINT24_WIDTH +#endif /* __STDC_VERSION__ */ +#endif /* __INT24_TYPE__ */ + +#ifdef __INT16_TYPE__ +#define INT16_MAX INT16_C(32767) +#define INT16_MIN (-INT16_C(32767) - 1) +#define UINT16_MAX UINT16_C(65535) + +#undef __INT_LEAST16_MIN +#define __INT_LEAST16_MIN INT16_MIN +#undef __INT_LEAST16_MAX +#define __INT_LEAST16_MAX INT16_MAX +#undef __UINT_LEAST16_MAX +#define __UINT_LEAST16_MAX UINT16_MAX +#undef __INT_LEAST8_MIN +#define __INT_LEAST8_MIN INT16_MIN +#undef __INT_LEAST8_MAX +#define __INT_LEAST8_MAX INT16_MAX +#undef __UINT_LEAST8_MAX +#define __UINT_LEAST8_MAX UINT16_MAX + +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +#define UINT16_WIDTH 16 +#define INT16_WIDTH UINT16_WIDTH +#undef __UINT_LEAST16_WIDTH +#define __UINT_LEAST16_WIDTH UINT16_WIDTH +#undef __UINT_LEAST8_WIDTH +#define __UINT_LEAST8_WIDTH UINT16_WIDTH +#endif /* __STDC_VERSION__ */ +#endif /* __INT16_TYPE__ */ + +#ifdef __INT_LEAST16_MIN +#define INT_LEAST16_MIN __INT_LEAST16_MIN +#define INT_LEAST16_MAX __INT_LEAST16_MAX +#define UINT_LEAST16_MAX __UINT_LEAST16_MAX +#define INT_FAST16_MIN __INT_LEAST16_MIN +#define INT_FAST16_MAX __INT_LEAST16_MAX +#define UINT_FAST16_MAX __UINT_LEAST16_MAX + +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +#define UINT_LEAST16_WIDTH __UINT_LEAST16_WIDTH +#define INT_LEAST16_WIDTH UINT_LEAST16_WIDTH +#define UINT_FAST16_WIDTH __UINT_LEAST16_WIDTH +#define INT_FAST16_WIDTH UINT_FAST16_WIDTH +#endif /* __STDC_VERSION__ */ +#endif /* __INT_LEAST16_MIN */ + +#ifdef __INT8_TYPE__ +#define INT8_MAX INT8_C(127) +#define INT8_MIN (-INT8_C(127) - 1) +#define UINT8_MAX UINT8_C(255) + +#undef __INT_LEAST8_MIN +#define __INT_LEAST8_MIN INT8_MIN +#undef __INT_LEAST8_MAX +#define __INT_LEAST8_MAX INT8_MAX +#undef __UINT_LEAST8_MAX +#define __UINT_LEAST8_MAX UINT8_MAX + +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +#define UINT8_WIDTH 8 +#define INT8_WIDTH UINT8_WIDTH +#undef __UINT_LEAST8_WIDTH +#define __UINT_LEAST8_WIDTH UINT8_WIDTH +#endif /* __STDC_VERSION__ */ +#endif /* __INT8_TYPE__ */ + +#ifdef __INT_LEAST8_MIN +#define INT_LEAST8_MIN __INT_LEAST8_MIN +#define INT_LEAST8_MAX __INT_LEAST8_MAX +#define UINT_LEAST8_MAX __UINT_LEAST8_MAX +#define INT_FAST8_MIN __INT_LEAST8_MIN +#define INT_FAST8_MAX __INT_LEAST8_MAX +#define UINT_FAST8_MAX __UINT_LEAST8_MAX + +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +#define UINT_LEAST8_WIDTH __UINT_LEAST8_WIDTH +#define INT_LEAST8_WIDTH UINT_LEAST8_WIDTH +#define UINT_FAST8_WIDTH __UINT_LEAST8_WIDTH +#define INT_FAST8_WIDTH UINT_FAST8_WIDTH +#endif /* __STDC_VERSION__ */ +#endif /* __INT_LEAST8_MIN */ + +/* Some utility macros */ +#define __INTN_MIN(n) __stdint_join3(INT, n, _MIN) +#define __INTN_MAX(n) __stdint_join3(INT, n, _MAX) +#define __UINTN_MAX(n) __stdint_join3(UINT, n, _MAX) +#define __INTN_C(n, v) __stdint_join3(INT, n, _C(v)) +#define __UINTN_C(n, v) __stdint_join3(UINT, n, _C(v)) + +/* C99 7.18.2.4 Limits of integer types capable of holding object pointers. */ +/* C99 7.18.3 Limits of other integer types. */ + +#define INTPTR_MIN (-__INTPTR_MAX__ - 1) +#define INTPTR_MAX __INTPTR_MAX__ +#define UINTPTR_MAX __UINTPTR_MAX__ +#define PTRDIFF_MIN (-__PTRDIFF_MAX__ - 1) +#define PTRDIFF_MAX __PTRDIFF_MAX__ +#define SIZE_MAX __SIZE_MAX__ + +/* C23 7.22.2.4 Width of integer types capable of holding object pointers. */ +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +/* NB: The C standard requires that these be the same value, but the compiler + exposes separate internal width macros. */ +#define INTPTR_WIDTH __INTPTR_WIDTH__ +#define UINTPTR_WIDTH __UINTPTR_WIDTH__ +#endif + +/* ISO9899:2011 7.20 (C11 Annex K): Define RSIZE_MAX if __STDC_WANT_LIB_EXT1__ + * is enabled. */ +#if defined(__STDC_WANT_LIB_EXT1__) && __STDC_WANT_LIB_EXT1__ >= 1 +#define RSIZE_MAX (SIZE_MAX >> 1) +#endif + +/* C99 7.18.2.5 Limits of greatest-width integer types. */ +#define INTMAX_MIN (-__INTMAX_MAX__ - 1) +#define INTMAX_MAX __INTMAX_MAX__ +#define UINTMAX_MAX __UINTMAX_MAX__ + +/* C23 7.22.2.5 Width of greatest-width integer types. */ +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +/* NB: The C standard requires that these be the same value, but the compiler + exposes separate internal width macros. */ +#define INTMAX_WIDTH __INTMAX_WIDTH__ +#define UINTMAX_WIDTH __UINTMAX_WIDTH__ +#endif + +/* C99 7.18.3 Limits of other integer types. */ +#define SIG_ATOMIC_MIN __INTN_MIN(__SIG_ATOMIC_WIDTH__) +#define SIG_ATOMIC_MAX __INTN_MAX(__SIG_ATOMIC_WIDTH__) +#ifdef __WINT_UNSIGNED__ +#define WINT_MIN __UINTN_C(__WINT_WIDTH__, 0) +#define WINT_MAX __UINTN_MAX(__WINT_WIDTH__) +#else +#define WINT_MIN __INTN_MIN(__WINT_WIDTH__) +#define WINT_MAX __INTN_MAX(__WINT_WIDTH__) +#endif + +#ifndef WCHAR_MAX +#define WCHAR_MAX __WCHAR_MAX__ +#endif +#ifndef WCHAR_MIN +#if __WCHAR_MAX__ == __INTN_MAX(__WCHAR_WIDTH__) +#define WCHAR_MIN __INTN_MIN(__WCHAR_WIDTH__) +#else +#define WCHAR_MIN __UINTN_C(__WCHAR_WIDTH__, 0) +#endif +#endif + +/* 7.18.4.2 Macros for greatest-width integer constants. */ +#define INTMAX_C(v) __int_c(v, __INTMAX_C_SUFFIX__) +#define UINTMAX_C(v) __int_c(v, __UINTMAX_C_SUFFIX__) + +/* C23 7.22.3.x Width of other integer types. */ +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +#define PTRDIFF_WIDTH __PTRDIFF_WIDTH__ +#define SIG_ATOMIC_WIDTH __SIG_ATOMIC_WIDTH__ +#define SIZE_WIDTH __SIZE_WIDTH__ +#define WCHAR_WIDTH __WCHAR_WIDTH__ +#define WINT_WIDTH __WINT_WIDTH__ +#endif +#endif // LLVM_LIBC_MACROS_STDINT_MACROS_H diff --git a/libc/include/stdint.h.def b/libc/include/stdint.h.def new file mode 100644 index 000000000000..9e269101acd2 --- /dev/null +++ b/libc/include/stdint.h.def @@ -0,0 +1,14 @@ +//===-- C standard library header stdint.h --------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_STDINT_H +#define LLVM_LIBC_STDINT_H + +#include + +#endif // LLVM_LIBC_STDINT_H diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index 94ac62966f3b..fc5a2f78f7bd 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -963,6 +963,8 @@ def StdC : StandardSpec<"stdc"> { ] >; + HeaderSpec StdInt = HeaderSpec<"StdInt.h">; + HeaderSpec Limits = HeaderSpec<"limits.h">; NamedType SigAtomicT = NamedType<"sig_atomic_t">; @@ -1268,6 +1270,7 @@ def StdC : StandardSpec<"stdc"> { Errno, Fenv, Float, + StdInt, Limits, Math, String, diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index 16ceaadf276f..49a454379e1c 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -997,6 +997,11 @@ libc_support_library( hdrs = ["include/llvm-libc-macros/float-macros.h"], ) +libc_support_library( + name = "llvm_libc_macros_stdint_macros", + hdrs = ["include/llvm-libc-macros/stdint-macros.h"], +) + libc_support_library( name = "llvm_libc_macros_stdfix_macros", hdrs = ["include/llvm-libc-macros/stdfix-macros.h"], -- GitLab From 32e2294b8abba6b70356aa37b65acf155506d457 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Mon, 4 Mar 2024 12:23:52 -0600 Subject: [PATCH 025/929] [libc][NFC] Clean up GPU utilities directory after rework (#83692) Summary: This CMake can be cleaned up now that the AMDGPU and NVPTX builds do not share a CMake invocation. --- libc/src/__support/GPU/CMakeLists.txt | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/libc/src/__support/GPU/CMakeLists.txt b/libc/src/__support/GPU/CMakeLists.txt index d7ebd3cab7ab..c181b2ed43c8 100644 --- a/libc/src/__support/GPU/CMakeLists.txt +++ b/libc/src/__support/GPU/CMakeLists.txt @@ -1,11 +1,9 @@ -if(NOT LIBC_TARGET_OS_IS_GPU) +if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_ARCHITECTURE}) return() endif() -foreach(target nvptx amdgpu generic) - add_subdirectory(${target}) - list(APPEND target_gpu_utils libc.src.__support.GPU.${target}.${target}_utils) -endforeach() +add_subdirectory(${LIBC_TARGET_ARCHITECTURE}) +set(target_gpu_utils libc.src.__support.GPU.${LIBC_TARGET_ARCHITECTURE}.${LIBC_TARGET_ARCHITECTURE}_utils) add_header_library( utils -- GitLab From 1da5db97cbf3451da05788b1d35b8a107a7f6a69 Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Mon, 4 Mar 2024 10:44:17 -0800 Subject: [PATCH 026/929] [lldb] Use sort-ordering for indexes when sorting by size (#83889) Use sort-ordering for indexes when sorting by size. This addresses Jason's post commit review feedback. --- lldb/source/Symbol/Symtab.cpp | 3 ++- .../Shell/SymbolFile/Breakpad/symtab-sorted-by-size.test | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lldb/source/Symbol/Symtab.cpp b/lldb/source/Symbol/Symtab.cpp index b7837892d7e2..c63bbe94fece 100644 --- a/lldb/source/Symbol/Symtab.cpp +++ b/lldb/source/Symbol/Symtab.cpp @@ -142,10 +142,11 @@ void Symtab::Dump(Stream *s, Target *target, SortOrder sort_order, for (const Symbol &symbol : m_symbols) size_map.emplace(symbol.GetByteSize(), &symbol); + size_t idx = 0; for (const auto &size_to_symbol : size_map) { const Symbol *symbol = size_to_symbol.second; s->Indent(); - symbol->Dump(s, target, symbol - &m_symbols[0], name_preference); + symbol->Dump(s, target, idx++, name_preference); } } break; diff --git a/lldb/test/Shell/SymbolFile/Breakpad/symtab-sorted-by-size.test b/lldb/test/Shell/SymbolFile/Breakpad/symtab-sorted-by-size.test index a9b6c0b1ef09..83b80236705e 100644 --- a/lldb/test/Shell/SymbolFile/Breakpad/symtab-sorted-by-size.test +++ b/lldb/test/Shell/SymbolFile/Breakpad/symtab-sorted-by-size.test @@ -4,8 +4,8 @@ # CHECK: num_symbols = 4 (sorted by size): # CHECK: [ 0] 0 SX Code 0x0000000000400000 0x00000000000000b0 0x00000000 ___lldb_unnamed_symbol0 -# CHECK: [ 3] 0 X Code 0x00000000004000d0 0x0000000000000022 0x00000000 _start -# CHECK: [ 1] 0 X Code 0x00000000004000b0 0x0000000000000010 0x00000000 f1 -# CHECK: [ 2] 0 X Code 0x00000000004000c0 0x0000000000000010 0x00000000 f2 +# CHECK: [ 1] 0 X Code 0x00000000004000d0 0x0000000000000022 0x00000000 _start +# CHECK: [ 2] 0 X Code 0x00000000004000b0 0x0000000000000010 0x00000000 f1 +# CHECK: [ 3] 0 X Code 0x00000000004000c0 0x0000000000000010 0x00000000 f2 image dump symtab -s size symtab.out -- GitLab From 53a656d9a4032b842ad0c59489353e17daa54531 Mon Sep 17 00:00:00 2001 From: Alexander Richardson Date: Mon, 4 Mar 2024 10:45:22 -0800 Subject: [PATCH 027/929] [utils/not] Disable coredumps when --crash is passed We are expecting a crash, so it is unlikely that a coredump is going to be particularly useful. For debugging the program can always be run without the `not --crash` wrapper. Reviewed By: jh7370 Pull Request: https://github.com/llvm/llvm-project/pull/83704 --- llvm/utils/not/not.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/llvm/utils/not/not.cpp b/llvm/utils/not/not.cpp index 4fcc0e89fc40..6ba59190d8ad 100644 --- a/llvm/utils/not/not.cpp +++ b/llvm/utils/not/not.cpp @@ -11,6 +11,7 @@ // not --crash cmd // Will return true if cmd crashes (e.g. for testing crash reporting). +#include "llvm/Support/Process.h" #include "llvm/Support/Program.h" #include "llvm/Support/WithColor.h" #include "llvm/Support/raw_ostream.h" @@ -41,6 +42,9 @@ int main(int argc, const char **argv) { setenv("LLVM_DISABLE_CRASH_REPORT", "1", 0); setenv("LLVM_DISABLE_SYMBOLIZATION", "1", 0); #endif + // Try to disable coredumps for expected crashes as well since this can + // noticeably slow down running the test suite. + sys::Process::PreventCoreFiles(); } if (argc == 0) -- GitLab From 081882eb9ca63ae9582399beb8b76c1d135d7062 Mon Sep 17 00:00:00 2001 From: AtariDreams <83477269+AtariDreams@users.noreply.github.com> Date: Mon, 4 Mar 2024 13:45:45 -0500 Subject: [PATCH 028/929] [InstCombine] Remove m_OneUse requirement for max, but not min (#81505) If it is ever determined that min doesn't need one-use, then we can remove the one-use requirement entirely. --- .../Transforms/InstCombine/InstCombineCalls.cpp | 17 ++++++++++++----- llvm/test/Transforms/InstCombine/maximum.ll | 15 ++++++++++++++- llvm/test/Transforms/InstCombine/maxnum.ll | 15 ++++++++++++++- 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index 60b8243d6ba6..50c0f9a913f3 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -2288,11 +2288,18 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { // max X, -X --> fabs X // min X, -X --> -(fabs X) - // TODO: Remove one-use limitation? That is obviously better for max. - // It would be an extra instruction for min (fnabs), but that is - // still likely better for analysis and codegen. - if ((match(Arg0, m_OneUse(m_FNeg(m_Value(X)))) && Arg1 == X) || - (match(Arg1, m_OneUse(m_FNeg(m_Value(X)))) && Arg0 == X)) { + // TODO: Remove one-use limitation? That is obviously better for max, + // hence why we don't check for one-use for that. However, + // it would be an extra instruction for min (fnabs), but + // that is still likely better for analysis and codegen. + auto IsMinMaxOrXNegX = [IID, &X](Value *Op0, Value *Op1) { + if (match(Op0, m_FNeg(m_Value(X))) && match(Op1, m_Specific(X))) + return Op0->hasOneUse() || + (IID != Intrinsic::minimum && IID != Intrinsic::minnum); + return false; + }; + + if (IsMinMaxOrXNegX(Arg0, Arg1) || IsMinMaxOrXNegX(Arg1, Arg0)) { Value *R = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, X, II); if (IID == Intrinsic::minimum || IID == Intrinsic::minnum) R = Builder.CreateFNegFMF(R, II); diff --git a/llvm/test/Transforms/InstCombine/maximum.ll b/llvm/test/Transforms/InstCombine/maximum.ll index 82e4c8794c1c..7455d3b7b924 100644 --- a/llvm/test/Transforms/InstCombine/maximum.ll +++ b/llvm/test/Transforms/InstCombine/maximum.ll @@ -436,7 +436,7 @@ define float @negated_op_extra_use(float %x) { ; CHECK-LABEL: @negated_op_extra_use( ; CHECK-NEXT: [[NEGX:%.*]] = fneg float [[X:%.*]] ; CHECK-NEXT: call void @use(float [[NEGX]]) -; CHECK-NEXT: [[R:%.*]] = call float @llvm.maximum.f32(float [[NEGX]], float [[X]]) +; CHECK-NEXT: [[R:%.*]] = call float @llvm.fabs.f32(float [[X]]) ; CHECK-NEXT: ret float [[R]] ; %negx = fneg float %x @@ -444,3 +444,16 @@ define float @negated_op_extra_use(float %x) { %r = call float @llvm.maximum.f32(float %negx, float %x) ret float %r } + +define float @negated_op_extra_use_comm(float %x) { +; CHECK-LABEL: @negated_op_extra_use_comm( +; CHECK-NEXT: [[NEGX:%.*]] = fneg float [[X:%.*]] +; CHECK-NEXT: call void @use(float [[NEGX]]) +; CHECK-NEXT: [[R:%.*]] = call float @llvm.fabs.f32(float [[X]]) +; CHECK-NEXT: ret float [[R]] +; + %negx = fneg float %x + call void @use(float %negx) + %r = call float @llvm.maximum.f32(float %x, float %negx) + ret float %r +} diff --git a/llvm/test/Transforms/InstCombine/maxnum.ll b/llvm/test/Transforms/InstCombine/maxnum.ll index e140a5b405ea..f26a5300febd 100644 --- a/llvm/test/Transforms/InstCombine/maxnum.ll +++ b/llvm/test/Transforms/InstCombine/maxnum.ll @@ -458,7 +458,7 @@ define float @negated_op_extra_use(float %x) { ; CHECK-LABEL: @negated_op_extra_use( ; CHECK-NEXT: [[NEGX:%.*]] = fneg float [[X:%.*]] ; CHECK-NEXT: call void @use(float [[NEGX]]) -; CHECK-NEXT: [[R:%.*]] = call float @llvm.maxnum.f32(float [[NEGX]], float [[X]]) +; CHECK-NEXT: [[R:%.*]] = call float @llvm.fabs.f32(float [[X]]) ; CHECK-NEXT: ret float [[R]] ; %negx = fneg float %x @@ -466,3 +466,16 @@ define float @negated_op_extra_use(float %x) { %r = call float @llvm.maxnum.f32(float %negx, float %x) ret float %r } + +define float @negated_op_extra_use_comm(float %x) { +; CHECK-LABEL: @negated_op_extra_use_comm( +; CHECK-NEXT: [[NEGX:%.*]] = fneg float [[X:%.*]] +; CHECK-NEXT: call void @use(float [[NEGX]]) +; CHECK-NEXT: [[R:%.*]] = call float @llvm.fabs.f32(float [[X]]) +; CHECK-NEXT: ret float [[R]] +; + %negx = fneg float %x + call void @use(float %negx) + %r = call float @llvm.maxnum.f32(float %x, float %negx) + ret float %r +} -- GitLab From b32845cb94a81e6fd8b01a8631e3d276c9fc9e35 Mon Sep 17 00:00:00 2001 From: AMS21 Date: Mon, 4 Mar 2024 19:56:15 +0100 Subject: [PATCH 029/929] [clang-tidy] Let `bugprone-use-after-move` also handle calls to `std::forward` (#82673) Add support for std::forward. Fixes #82023 --- .../clang-tidy/bugprone/UseAfterMoveCheck.cpp | 57 ++++++++++++++----- clang-tools-extra/docs/ReleaseNotes.rst | 4 ++ .../checks/bugprone/use-after-move.rst | 12 ++++ .../checkers/bugprone/use-after-move.cpp | 37 ++++++++++++ 4 files changed, 95 insertions(+), 15 deletions(-) diff --git a/clang-tools-extra/clang-tidy/bugprone/UseAfterMoveCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/UseAfterMoveCheck.cpp index c5b6b541096c..b91ad0f18229 100644 --- a/clang-tools-extra/clang-tidy/bugprone/UseAfterMoveCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/UseAfterMoveCheck.cpp @@ -330,7 +330,8 @@ void UseAfterMoveFinder::getReinits( traverse(TK_AsIs, DeclRefMatcher), unless(parmVarDecl(hasType( references(qualType(isConstQualified())))))), - unless(callee(functionDecl(hasName("::std::move"))))))) + unless(callee(functionDecl( + hasAnyName("::std::move", "::std::forward"))))))) .bind("reinit"); Stmts->clear(); @@ -359,24 +360,46 @@ void UseAfterMoveFinder::getReinits( } } +enum class MoveType { + Move, // std::move + Forward, // std::forward +}; + +static MoveType determineMoveType(const FunctionDecl *FuncDecl) { + if (FuncDecl->getName() == "move") + return MoveType::Move; + if (FuncDecl->getName() == "forward") + return MoveType::Forward; + + llvm_unreachable("Invalid move type"); +} + static void emitDiagnostic(const Expr *MovingCall, const DeclRefExpr *MoveArg, const UseAfterMove &Use, ClangTidyCheck *Check, - ASTContext *Context) { - SourceLocation UseLoc = Use.DeclRef->getExprLoc(); - SourceLocation MoveLoc = MovingCall->getExprLoc(); + ASTContext *Context, MoveType Type) { + const SourceLocation UseLoc = Use.DeclRef->getExprLoc(); + const SourceLocation MoveLoc = MovingCall->getExprLoc(); + + const bool IsMove = (Type == MoveType::Move); - Check->diag(UseLoc, "'%0' used after it was moved") - << MoveArg->getDecl()->getName(); - Check->diag(MoveLoc, "move occurred here", DiagnosticIDs::Note); + Check->diag(UseLoc, "'%0' used after it was %select{forwarded|moved}1") + << MoveArg->getDecl()->getName() << IsMove; + Check->diag(MoveLoc, "%select{forward|move}0 occurred here", + DiagnosticIDs::Note) + << IsMove; if (Use.EvaluationOrderUndefined) { - Check->diag(UseLoc, - "the use and move are unsequenced, i.e. there is no guarantee " - "about the order in which they are evaluated", - DiagnosticIDs::Note); + Check->diag( + UseLoc, + "the use and %select{forward|move}0 are unsequenced, i.e. " + "there is no guarantee about the order in which they are evaluated", + DiagnosticIDs::Note) + << IsMove; } else if (UseLoc < MoveLoc || Use.DeclRef == MoveArg) { Check->diag(UseLoc, - "the use happens in a later loop iteration than the move", - DiagnosticIDs::Note); + "the use happens in a later loop iteration than the " + "%select{forward|move}0", + DiagnosticIDs::Note) + << IsMove; } } @@ -388,7 +411,9 @@ void UseAfterMoveCheck::registerMatchers(MatchFinder *Finder) { auto TryEmplaceMatcher = cxxMemberCallExpr(callee(cxxMethodDecl(hasName("try_emplace")))); auto CallMoveMatcher = - callExpr(argumentCountIs(1), callee(functionDecl(hasName("::std::move"))), + callExpr(argumentCountIs(1), + callee(functionDecl(hasAnyName("::std::move", "::std::forward")) + .bind("move-decl")), hasArgument(0, declRefExpr().bind("arg")), unless(inDecltypeOrTemplateArg()), unless(hasParent(TryEmplaceMatcher)), expr().bind("call-move"), @@ -436,6 +461,7 @@ void UseAfterMoveCheck::check(const MatchFinder::MatchResult &Result) { const auto *CallMove = Result.Nodes.getNodeAs("call-move"); const auto *MovingCall = Result.Nodes.getNodeAs("moving-call"); const auto *Arg = Result.Nodes.getNodeAs("arg"); + const auto *MoveDecl = Result.Nodes.getNodeAs("move-decl"); if (!MovingCall || !MovingCall->getExprLoc().isValid()) MovingCall = CallMove; @@ -470,7 +496,8 @@ void UseAfterMoveCheck::check(const MatchFinder::MatchResult &Result) { UseAfterMoveFinder Finder(Result.Context); UseAfterMove Use; if (Finder.find(CodeBlock, MovingCall, Arg->getDecl(), &Use)) - emitDiagnostic(MovingCall, Arg, Use, this, Result.Context); + emitDiagnostic(MovingCall, Arg, Use, this, Result.Context, + determineMoveType(MoveDecl)); } } diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst index 70ae23ba185a..1e1949756e0e 100644 --- a/clang-tools-extra/docs/ReleaseNotes.rst +++ b/clang-tools-extra/docs/ReleaseNotes.rst @@ -144,6 +144,10 @@ Changes in existing checks ` check by updating the parameter `CheckedFunctions` to support regexp. +- Improved :doc:`bugprone-use-after-move + ` check to also handle + calls to ``std::forward``. + - Improved :doc:`cppcoreguidelines-missing-std-forward ` check by no longer giving false positives for deleted functions. diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone/use-after-move.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone/use-after-move.rst index 8509292eff99..08bb5374bab1 100644 --- a/clang-tools-extra/docs/clang-tidy/checks/bugprone/use-after-move.rst +++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone/use-after-move.rst @@ -177,6 +177,18 @@ When analyzing the order in which moves, uses and reinitializations happen (see section `Unsequenced moves, uses, and reinitializations`_), the move is assumed to occur in whichever function the result of the ``std::move`` is passed to. +The check also handles perfect-forwarding with ``std::forward`` so the +following code will also trigger a use-after-move warning. + +.. code-block:: c++ + + void consume(int); + + void f(int&& i) { + consume(std::forward(i)); + consume(std::forward(i)); // use-after-move + } + Use --- diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/use-after-move.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/use-after-move.cpp index 00b1da1e727e..7d9f63479a1b 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/use-after-move.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/use-after-move.cpp @@ -111,6 +111,18 @@ constexpr typename std::remove_reference<_Tp>::type &&move(_Tp &&__t) noexcept { return static_cast::type &&>(__t); } +template +constexpr _Tp&& +forward(typename std::remove_reference<_Tp>::type& __t) noexcept { + return static_cast<_Tp&&>(__t); +} + +template +constexpr _Tp&& +forward(typename std::remove_reference<_Tp>::type&& __t) noexcept { + return static_cast<_Tp&&>(__t); +} + } // namespace std class A { @@ -1525,3 +1537,28 @@ public: private: std::string val_; }; + +namespace issue82023 +{ + +struct S { + S(); + S(S&&); +}; + +void consume(S s); + +template +void forward(T&& t) { + consume(std::forward(t)); + consume(std::forward(t)); + // CHECK-NOTES: [[@LINE-1]]:27: warning: 't' used after it was forwarded + // CHECK-NOTES: [[@LINE-3]]:11: note: forward occurred here +} + +void create() { + S s; + forward(std::move(s)); +} + +} // namespace issue82023 -- GitLab From 503075e4d4a9c1b3754b21ee9ec41f176e54fd83 Mon Sep 17 00:00:00 2001 From: Jordan Rupprecht Date: Mon, 4 Mar 2024 11:21:46 -0800 Subject: [PATCH 030/929] [lldb][test][NFC] Narrow scope of `import pexpect` We do not run `pexpect` based tests on Windows, but there are still cases where those tests run `import pexpect` outside of the scope where the test is skipped. By moving the import statement to a different scope, those tests can run even when `pexpect` truly isn't installed. Tangentially related: TestSTTYBeforeAndAfter.py is using a manual `@expectedFailureAll` for windows instead of the common `@skipIfWindows`. If `pexepect` is generally expected to not be available, we should not bother running the test at all. --- lldb/test/API/driver/quit_speed/TestQuitWithProcess.py | 4 ++-- .../progress_reporting/TestTrimmedProgressReporting.py | 1 - lldb/test/API/terminal/TestSTTYBeforeAndAfter.py | 5 +---- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/lldb/test/API/driver/quit_speed/TestQuitWithProcess.py b/lldb/test/API/driver/quit_speed/TestQuitWithProcess.py index 957586d41f6b..42527c88b992 100644 --- a/lldb/test/API/driver/quit_speed/TestQuitWithProcess.py +++ b/lldb/test/API/driver/quit_speed/TestQuitWithProcess.py @@ -2,12 +2,10 @@ Test that killing the target while quitting doesn't stall """ - import lldb from lldbsuite.test.decorators import * from lldbsuite.test.lldbtest import * from lldbsuite.test import lldbutil -import pexpect from lldbsuite.test.lldbpexpect import PExpectTest @@ -16,6 +14,8 @@ class DriverQuitSpeedTest(PExpectTest): def test_run_quit(self): """Test that the lldb driver's batch mode works correctly.""" + import pexpect + self.build() exe = self.getBuildArtifact("a.out") diff --git a/lldb/test/API/functionalities/progress_reporting/TestTrimmedProgressReporting.py b/lldb/test/API/functionalities/progress_reporting/TestTrimmedProgressReporting.py index ee35dbd23b3d..3cf7b9d21008 100644 --- a/lldb/test/API/functionalities/progress_reporting/TestTrimmedProgressReporting.py +++ b/lldb/test/API/functionalities/progress_reporting/TestTrimmedProgressReporting.py @@ -3,7 +3,6 @@ Test trimming long progress report in tiny terminal windows """ import os -import pexpect import tempfile import re diff --git a/lldb/test/API/terminal/TestSTTYBeforeAndAfter.py b/lldb/test/API/terminal/TestSTTYBeforeAndAfter.py index 31b960859fa2..e5663c50c736 100644 --- a/lldb/test/API/terminal/TestSTTYBeforeAndAfter.py +++ b/lldb/test/API/terminal/TestSTTYBeforeAndAfter.py @@ -19,10 +19,7 @@ class TestSTTYBeforeAndAfter(TestBase): cls.RemoveTempFile("child_send2.txt") cls.RemoveTempFile("child_read2.txt") - @expectedFailureAll( - hostoslist=["windows"], - bugnumber="llvm.org/pr22274: need a pexpect replacement for windows", - ) + @skipIfWindows # llvm.org/pr22274: need a pexpect replacement for windows @no_debug_info_test def test_stty_dash_a_before_and_afetr_invoking_lldb_command(self): """Test that 'stty -a' displays the same output before and after running the lldb command.""" -- GitLab From a81a7b9962f093f603d3890822e253120e074d13 Mon Sep 17 00:00:00 2001 From: Mircea Trofin Date: Mon, 4 Mar 2024 11:22:35 -0800 Subject: [PATCH 031/929] [Py Reformat] Exclude `third-party` in `code-format-helper.py` (#83872) Follow-up from PR #83491. `Darker`'s configuration is ignored because of the way we invoke it - with an explicit list of files. We need to filter it in `code-format-helper.py`. --- llvm/utils/git/code-format-helper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/git/code-format-helper.py b/llvm/utils/git/code-format-helper.py index 8a29a57d8d16..1113bf02570b 100755 --- a/llvm/utils/git/code-format-helper.py +++ b/llvm/utils/git/code-format-helper.py @@ -125,6 +125,7 @@ View the diff from {self.name} here. pr.as_issue().create_comment(comment_text) def run(self, changed_files: List[str], args: FormatArgs) -> bool: + changed_files = [arg for arg in changed_files if "third-party" not in arg] diff = self.format_run(changed_files, args) should_update_gh = args.token is not None and args.repo is not None -- GitLab From a4951eca40c070e020aa5d2689c08177fbeb780d Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Fri, 29 Sep 2023 14:27:36 -0500 Subject: [PATCH 032/929] Recommit "[X86] Don't always separate conditions in `(br (and/or cond0, cond1))` into separate branches" (2nd Try) Changes in Recommit: 1) Fix non-determanism by using `SmallMapVector` instead of `SmallPtrSet`. 2) Fix bug in dependency pruning where we discounted the actual `and/or` combining the two conditions. This lead to over pruning. Closes #81689 --- llvm/include/llvm/CodeGen/TargetLowering.h | 36 +++ .../SelectionDAG/SelectionDAGBuilder.cpp | 157 ++++++++- .../SelectionDAG/SelectionDAGBuilder.h | 5 + llvm/lib/Target/X86/X86ISelLowering.cpp | 52 +++ llvm/lib/Target/X86/X86ISelLowering.h | 4 + .../CodeGen/X86/2006-04-27-ISelFoldingBug.ll | 11 +- .../X86/2007-08-09-IllegalX86-64Asm.ll | 109 ++++--- .../test/CodeGen/X86/2007-12-18-LoadCSEBug.ll | 13 +- .../CodeGen/X86/2008-02-18-TailMergingBug.ll | 2 +- llvm/test/CodeGen/X86/avx-cmp.ll | 27 +- llvm/test/CodeGen/X86/block-placement.ll | 1 - .../div-rem-pair-recomposition-unsigned.ll | 306 +++++++++--------- ...iller-impdef-on-implicit-def-regression.ll | 70 ++-- llvm/test/CodeGen/X86/movmsk-cmp.ll | 97 +++--- llvm/test/CodeGen/X86/or-branch.ll | 9 +- .../X86/peephole-na-phys-copy-folding.ll | 34 +- llvm/test/CodeGen/X86/pr33747.ll | 17 +- llvm/test/CodeGen/X86/pr37025.ll | 38 ++- llvm/test/CodeGen/X86/pr38795.ll | 131 ++++---- llvm/test/CodeGen/X86/setcc-logic.ll | 76 ++--- llvm/test/CodeGen/X86/swifterror.ll | 10 - .../X86/tail-dup-merge-loop-headers.ll | 119 +++---- llvm/test/CodeGen/X86/tail-opts.ll | 5 +- llvm/test/CodeGen/X86/test-shrink-bug.ll | 34 +- .../CodeGen/X86/x86-shrink-wrap-unwind.ll | 52 +-- 25 files changed, 793 insertions(+), 622 deletions(-) diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 0438abc7c306..7be5b5e8dd2f 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -596,6 +596,42 @@ public: /// avoided. bool isJumpExpensive() const { return JumpIsExpensive; } + // Costs parameters used by + // SelectionDAGBuilder::shouldKeepJumpConditionsTogether. + // shouldKeepJumpConditionsTogether will use these parameter value to + // determine if two conditions in the form `br (and/or cond1, cond2)` should + // be split into two branches or left as one. + // + // BaseCost is the cost threshold (in latency). If the estimated latency of + // computing both `cond1` and `cond2` is below the cost of just computing + // `cond1` + BaseCost, the two conditions will be kept together. Otherwise + // they will be split. + // + // LikelyBias increases BaseCost if branch probability info indicates that it + // is likely that both `cond1` and `cond2` will be computed. + // + // UnlikelyBias decreases BaseCost if branch probability info indicates that + // it is likely that both `cond1` and `cond2` will be computed. + // + // Set any field to -1 to make it ignored (setting BaseCost to -1 results in + // `shouldKeepJumpConditionsTogether` always returning false). + struct CondMergingParams { + int BaseCost; + int LikelyBias; + int UnlikelyBias; + }; + // Return params for deciding if we should keep two branch conditions merged + // or split them into two separate branches. + // Arg0: The binary op joining the two conditions (and/or). + // Arg1: The first condition (cond1) + // Arg2: The second condition (cond2) + virtual CondMergingParams + getJumpConditionMergingParams(Instruction::BinaryOps, const Value *, + const Value *) const { + // -1 will always result in splitting. + return {-1, -1, -1}; + } + /// Return true if selects are only cheaper than branches if the branch is /// unlikely to be predicted right. bool isPredictableSelectExpensive() const { diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index ab2f42d2024c..48476b0ef970 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -26,6 +26,7 @@ #include "llvm/Analysis/Loads.h" #include "llvm/Analysis/MemoryLocation.h" #include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/Analysis/VectorUtils.h" #include "llvm/CodeGen/Analysis.h" @@ -93,6 +94,7 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/InstructionCost.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetIntrinsicInfo.h" @@ -2446,6 +2448,152 @@ SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond, SL->SwitchCases.push_back(CB); } +// Collect dependencies on V recursively. This is used for the cost analysis in +// `shouldKeepJumpConditionsTogether`. +static bool collectInstructionDeps( + SmallMapVector *Deps, const Value *V, + SmallMapVector *Necessary = nullptr, + unsigned Depth = 0) { + // Return false if we have an incomplete count. + if (Depth >= SelectionDAG::MaxRecursionDepth) + return false; + + auto *I = dyn_cast(V); + if (I == nullptr) + return true; + + if (Necessary != nullptr) { + // This instruction is necessary for the other side of the condition so + // don't count it. + if (Necessary->contains(I)) + return true; + } + + // Already added this dep. + if (!Deps->try_emplace(I, false).second) + return true; + + for (unsigned OpIdx = 0, E = I->getNumOperands(); OpIdx < E; ++OpIdx) + if (!collectInstructionDeps(Deps, I->getOperand(OpIdx), Necessary, + Depth + 1)) + return false; + return true; +} + +bool SelectionDAGBuilder::shouldKeepJumpConditionsTogether( + const FunctionLoweringInfo &FuncInfo, const BranchInst &I, + Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs, + TargetLoweringBase::CondMergingParams Params) const { + if (I.getNumSuccessors() != 2) + return false; + + if (!I.isConditional()) + return false; + + if (Params.BaseCost < 0) + return false; + + // Baseline cost. + InstructionCost CostThresh = Params.BaseCost; + + BranchProbabilityInfo *BPI = nullptr; + if (Params.LikelyBias || Params.UnlikelyBias) + BPI = FuncInfo.BPI; + if (BPI != nullptr) { + // See if we are either likely to get an early out or compute both lhs/rhs + // of the condition. + BasicBlock *IfFalse = I.getSuccessor(0); + BasicBlock *IfTrue = I.getSuccessor(1); + + std::optional Likely; + if (BPI->isEdgeHot(I.getParent(), IfTrue)) + Likely = true; + else if (BPI->isEdgeHot(I.getParent(), IfFalse)) + Likely = false; + + if (Likely) { + if (Opc == (*Likely ? Instruction::And : Instruction::Or)) + // Its likely we will have to compute both lhs and rhs of condition + CostThresh += Params.LikelyBias; + else { + if (Params.UnlikelyBias < 0) + return false; + // Its likely we will get an early out. + CostThresh -= Params.UnlikelyBias; + } + } + } + + if (CostThresh <= 0) + return false; + + // Collect "all" instructions that lhs condition is dependent on. + // Use map for stable iteration (to avoid non-determanism of iteration of + // SmallPtrSet). The `bool` value is just a dummy. + SmallMapVector LhsDeps, RhsDeps; + collectInstructionDeps(&LhsDeps, Lhs); + // Collect "all" instructions that rhs condition is dependent on AND are + // dependencies of lhs. This gives us an estimate on which instructions we + // stand to save by splitting the condition. + if (!collectInstructionDeps(&RhsDeps, Rhs, &LhsDeps)) + return false; + // Add the compare instruction itself unless its a dependency on the LHS. + if (const auto *RhsI = dyn_cast(Rhs)) + if (!LhsDeps.contains(RhsI)) + RhsDeps.try_emplace(RhsI, false); + + const auto &TLI = DAG.getTargetLoweringInfo(); + const auto &TTI = + TLI.getTargetMachine().getTargetTransformInfo(*I.getFunction()); + + InstructionCost CostOfIncluding = 0; + // See if this instruction will need to computed independently of whether RHS + // is. + Value *BrCond = I.getCondition(); + auto ShouldCountInsn = [&RhsDeps, &BrCond](const Instruction *Ins) { + for (const auto *U : Ins->users()) { + // If user is independent of RHS calculation we don't need to count it. + if (auto *UIns = dyn_cast(U)) + if (UIns != BrCond && !RhsDeps.contains(UIns)) + return false; + } + return true; + }; + + // Prune instructions from RHS Deps that are dependencies of unrelated + // instructions. The value (SelectionDAG::MaxRecursionDepth) is fairly + // arbitrary and just meant to cap the how much time we spend in the pruning + // loop. Its highly unlikely to come into affect. + const unsigned MaxPruneIters = SelectionDAG::MaxRecursionDepth; + // Stop after a certain point. No incorrectness from including too many + // instructions. + for (unsigned PruneIters = 0; PruneIters < MaxPruneIters; ++PruneIters) { + const Instruction *ToDrop = nullptr; + for (const auto &InsPair : RhsDeps) { + if (!ShouldCountInsn(InsPair.first)) { + ToDrop = InsPair.first; + break; + } + } + if (ToDrop == nullptr) + break; + RhsDeps.erase(ToDrop); + } + + for (const auto &InsPair : RhsDeps) { + // Finally accumulate latency that we can only attribute to computing the + // RHS condition. Use latency because we are essentially trying to calculate + // the cost of the dependency chain. + // Possible TODO: We could try to estimate ILP and make this more precise. + CostOfIncluding += + TTI.getInstructionCost(InsPair.first, TargetTransformInfo::TCK_Latency); + + if (CostOfIncluding > CostThresh) + return false; + } + return true; +} + void SelectionDAGBuilder::FindMergedConditions(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, @@ -2660,8 +2808,13 @@ void SelectionDAGBuilder::visitBr(const BranchInst &I) { else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1)))) Opcode = Instruction::Or; - if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) && - match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) { + if (Opcode && + !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) && + match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value()))) && + !shouldKeepJumpConditionsTogether( + FuncInfo, I, Opcode, BOp0, BOp1, + DAG.getTargetLoweringInfo().getJumpConditionMergingParams( + Opcode, BOp0, BOp1))) { FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode, getEdgeProbability(BrMBB, Succ0MBB), getEdgeProbability(BrMBB, Succ1MBB), diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h index 47657313cb6a..2084de473b80 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -385,6 +385,11 @@ public: N = NewN; } + bool shouldKeepJumpConditionsTogether( + const FunctionLoweringInfo &FuncInfo, const BranchInst &I, + Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs, + TargetLoweringBase::CondMergingParams Params) const; + void FindMergedConditions(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index b87e3121838d..6eaaec407dbb 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -77,6 +77,40 @@ static cl::opt ExperimentalPrefInnermostLoopAlignment( "alignment set by x86-experimental-pref-loop-alignment."), cl::Hidden); +static cl::opt BrMergingBaseCostThresh( + "x86-br-merging-base-cost", cl::init(2), + cl::desc( + "Sets the cost threshold for when multiple conditionals will be merged " + "into one branch versus be split in multiple branches. Merging " + "conditionals saves branches at the cost of additional instructions. " + "This value sets the instruction cost limit, below which conditionals " + "will be merged, and above which conditionals will be split. Set to -1 " + "to never merge branches."), + cl::Hidden); + +static cl::opt BrMergingLikelyBias( + "x86-br-merging-likely-bias", cl::init(0), + cl::desc("Increases 'x86-br-merging-base-cost' in cases that it is likely " + "that all conditionals will be executed. For example for merging " + "the conditionals (a == b && c > d), if its known that a == b is " + "likely, then it is likely that if the conditionals are split " + "both sides will be executed, so it may be desirable to increase " + "the instruction cost threshold. Set to -1 to never merge likely " + "branches."), + cl::Hidden); + +static cl::opt BrMergingUnlikelyBias( + "x86-br-merging-unlikely-bias", cl::init(-1), + cl::desc( + "Decreases 'x86-br-merging-base-cost' in cases that it is unlikely " + "that all conditionals will be executed. For example for merging " + "the conditionals (a == b && c > d), if its known that a == b is " + "unlikely, then it is unlikely that if the conditionals are split " + "both sides will be executed, so it may be desirable to decrease " + "the instruction cost threshold. Set to -1 to never merge unlikely " + "branches."), + cl::Hidden); + static cl::opt MulConstantOptimization( "mul-constant-optimization", cl::init(true), cl::desc("Replace 'mul x, Const' with more effective instructions like " @@ -3338,6 +3372,24 @@ unsigned X86TargetLowering::preferedOpcodeForCmpEqPiecesOfOperand( return ISD::SRL; } +TargetLoweringBase::CondMergingParams +X86TargetLowering::getJumpConditionMergingParams(Instruction::BinaryOps Opc, + const Value *Lhs, + const Value *Rhs) const { + using namespace llvm::PatternMatch; + int BaseCost = BrMergingBaseCostThresh.getValue(); + // a == b && a == c is a fast pattern on x86. + ICmpInst::Predicate Pred; + if (BaseCost >= 0 && Opc == Instruction::And && + match(Lhs, m_ICmp(Pred, m_Value(), m_Value())) && + Pred == ICmpInst::ICMP_EQ && + match(Rhs, m_ICmp(Pred, m_Value(), m_Value())) && + Pred == ICmpInst::ICMP_EQ) + BaseCost += 1; + return {BaseCost, BrMergingLikelyBias.getValue(), + BrMergingUnlikelyBias.getValue()}; +} + bool X86TargetLowering::preferScalarizeSplat(SDNode *N) const { return N->getOpcode() != ISD::FP_EXTEND; } diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h index f93c54781846..fe1943b57608 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.h +++ b/llvm/lib/Target/X86/X86ISelLowering.h @@ -1150,6 +1150,10 @@ namespace llvm { bool preferScalarizeSplat(SDNode *N) const override; + CondMergingParams + getJumpConditionMergingParams(Instruction::BinaryOps Opc, const Value *Lhs, + const Value *Rhs) const override; + bool shouldFoldConstantShiftPairToMask(const SDNode *N, CombineLevel Level) const override; diff --git a/llvm/test/CodeGen/X86/2006-04-27-ISelFoldingBug.ll b/llvm/test/CodeGen/X86/2006-04-27-ISelFoldingBug.ll index 0044d1c35683..e6f28c2057f7 100644 --- a/llvm/test/CodeGen/X86/2006-04-27-ISelFoldingBug.ll +++ b/llvm/test/CodeGen/X86/2006-04-27-ISelFoldingBug.ll @@ -18,15 +18,16 @@ define i1 @loadAndRLEsource_no_exit_2E_1_label_2E_0(i32 %tmp.21.reload, i32 %tmp ; CHECK-NEXT: movl _block, %esi ; CHECK-NEXT: movb %al, 1(%esi,%edx) ; CHECK-NEXT: cmpl %ecx, _last -; CHECK-NEXT: jge LBB0_3 -; CHECK-NEXT: ## %bb.1: ## %label.0 +; CHECK-NEXT: setl %cl ; CHECK-NEXT: cmpl $257, %eax ## imm = 0x101 -; CHECK-NEXT: je LBB0_3 -; CHECK-NEXT: ## %bb.2: ## %label.0.no_exit.1_crit_edge.exitStub +; CHECK-NEXT: setne %al +; CHECK-NEXT: testb %al, %cl +; CHECK-NEXT: je LBB0_2 +; CHECK-NEXT: ## %bb.1: ## %label.0.no_exit.1_crit_edge.exitStub ; CHECK-NEXT: movb $1, %al ; CHECK-NEXT: popl %esi ; CHECK-NEXT: retl -; CHECK-NEXT: LBB0_3: ## %codeRepl5.exitStub +; CHECK-NEXT: LBB0_2: ## %codeRepl5.exitStub ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: popl %esi ; CHECK-NEXT: retl diff --git a/llvm/test/CodeGen/X86/2007-08-09-IllegalX86-64Asm.ll b/llvm/test/CodeGen/X86/2007-08-09-IllegalX86-64Asm.ll index 7bdc4e19a1cf..28b4541c1bfc 100644 --- a/llvm/test/CodeGen/X86/2007-08-09-IllegalX86-64Asm.ll +++ b/llvm/test/CodeGen/X86/2007-08-09-IllegalX86-64Asm.ll @@ -44,7 +44,7 @@ define ptr @ubyte_divmod(ptr %a, ptr %b) { ; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rsi ; CHECK-NEXT: callq __ubyte_convert_to_ctype ; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: js LBB0_4 +; CHECK-NEXT: js LBB0_6 ; CHECK-NEXT: ## %bb.1: ## %cond_next.i ; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rsi ; CHECK-NEXT: movq %rbx, %rdi @@ -53,81 +53,84 @@ define ptr @ubyte_divmod(ptr %a, ptr %b) { ; CHECK-NEXT: sarl $31, %ecx ; CHECK-NEXT: andl %eax, %ecx ; CHECK-NEXT: cmpl $-2, %ecx -; CHECK-NEXT: je LBB0_8 +; CHECK-NEXT: je LBB0_10 ; CHECK-NEXT: ## %bb.2: ## %cond_next.i ; CHECK-NEXT: cmpl $-1, %ecx -; CHECK-NEXT: jne LBB0_6 -; CHECK-NEXT: LBB0_3: ## %bb4 +; CHECK-NEXT: jne LBB0_3 +; CHECK-NEXT: LBB0_8: ## %bb4 ; CHECK-NEXT: movq _PyArray_API@GOTPCREL(%rip), %rax ; CHECK-NEXT: movq (%rax), %rax ; CHECK-NEXT: movq 16(%rax), %rax -; CHECK-NEXT: jmp LBB0_10 -; CHECK-NEXT: LBB0_4: ## %_ubyte_convert2_to_ctypes.exit +; CHECK-NEXT: jmp LBB0_9 +; CHECK-NEXT: LBB0_6: ## %_ubyte_convert2_to_ctypes.exit ; CHECK-NEXT: cmpl $-2, %eax -; CHECK-NEXT: je LBB0_8 -; CHECK-NEXT: ## %bb.5: ## %_ubyte_convert2_to_ctypes.exit +; CHECK-NEXT: je LBB0_10 +; CHECK-NEXT: ## %bb.7: ## %_ubyte_convert2_to_ctypes.exit ; CHECK-NEXT: cmpl $-1, %eax -; CHECK-NEXT: je LBB0_3 -; CHECK-NEXT: LBB0_6: ## %bb35 +; CHECK-NEXT: je LBB0_8 +; CHECK-NEXT: LBB0_3: ## %bb35 ; CHECK-NEXT: movq _PyUFunc_API@GOTPCREL(%rip), %r14 ; CHECK-NEXT: movq (%r14), %rax ; CHECK-NEXT: callq *216(%rax) ; CHECK-NEXT: movzbl {{[0-9]+}}(%rsp), %edx ; CHECK-NEXT: testb %dl, %dl -; CHECK-NEXT: je LBB0_11 -; CHECK-NEXT: ## %bb.7: ## %cond_false.i +; CHECK-NEXT: je LBB0_4 +; CHECK-NEXT: ## %bb.12: ## %cond_false.i +; CHECK-NEXT: setne %dil ; CHECK-NEXT: movzbl {{[0-9]+}}(%rsp), %esi ; CHECK-NEXT: movzbl %sil, %ecx ; CHECK-NEXT: movl %ecx, %eax ; CHECK-NEXT: divb %dl ; CHECK-NEXT: movl %eax, %r15d ; CHECK-NEXT: testb %cl, %cl -; CHECK-NEXT: jne LBB0_12 -; CHECK-NEXT: jmp LBB0_14 -; CHECK-NEXT: LBB0_8: ## %bb17 +; CHECK-NEXT: setne %al +; CHECK-NEXT: testb %dil, %al +; CHECK-NEXT: jne LBB0_5 +; CHECK-NEXT: LBB0_13: ## %cond_true.i200 +; CHECK-NEXT: testb %dl, %dl +; CHECK-NEXT: jne LBB0_15 +; CHECK-NEXT: ## %bb.14: ## %cond_true14.i +; CHECK-NEXT: movl $4, %edi +; CHECK-NEXT: callq _feraiseexcept +; CHECK-NEXT: LBB0_15: ## %ubyte_ctype_remainder.exit +; CHECK-NEXT: xorl %ebx, %ebx +; CHECK-NEXT: jmp LBB0_16 +; CHECK-NEXT: LBB0_10: ## %bb17 ; CHECK-NEXT: callq _PyErr_Occurred ; CHECK-NEXT: testq %rax, %rax -; CHECK-NEXT: jne LBB0_27 -; CHECK-NEXT: ## %bb.9: ## %cond_next +; CHECK-NEXT: jne LBB0_23 +; CHECK-NEXT: ## %bb.11: ## %cond_next ; CHECK-NEXT: movq _PyArray_API@GOTPCREL(%rip), %rax ; CHECK-NEXT: movq (%rax), %rax ; CHECK-NEXT: movq 80(%rax), %rax -; CHECK-NEXT: LBB0_10: ## %bb4 +; CHECK-NEXT: LBB0_9: ## %bb4 ; CHECK-NEXT: movq 96(%rax), %rax ; CHECK-NEXT: movq %r14, %rdi ; CHECK-NEXT: movq %rbx, %rsi ; CHECK-NEXT: callq *40(%rax) -; CHECK-NEXT: jmp LBB0_28 -; CHECK-NEXT: LBB0_11: ## %cond_true.i +; CHECK-NEXT: jmp LBB0_24 +; CHECK-NEXT: LBB0_4: ## %cond_true.i ; CHECK-NEXT: movl $4, %edi ; CHECK-NEXT: callq _feraiseexcept ; CHECK-NEXT: movzbl {{[0-9]+}}(%rsp), %edx ; CHECK-NEXT: movzbl {{[0-9]+}}(%rsp), %esi -; CHECK-NEXT: xorl %r15d, %r15d ; CHECK-NEXT: testb %sil, %sil -; CHECK-NEXT: je LBB0_14 -; CHECK-NEXT: LBB0_12: ## %cond_false.i +; CHECK-NEXT: sete %al ; CHECK-NEXT: testb %dl, %dl -; CHECK-NEXT: je LBB0_14 -; CHECK-NEXT: ## %bb.13: ## %cond_next17.i +; CHECK-NEXT: sete %cl +; CHECK-NEXT: xorl %r15d, %r15d +; CHECK-NEXT: orb %al, %cl +; CHECK-NEXT: jne LBB0_13 +; CHECK-NEXT: LBB0_5: ## %cond_next17.i ; CHECK-NEXT: movzbl %sil, %eax ; CHECK-NEXT: divb %dl ; CHECK-NEXT: movzbl %ah, %ebx -; CHECK-NEXT: jmp LBB0_18 -; CHECK-NEXT: LBB0_14: ## %cond_true.i200 -; CHECK-NEXT: testb %dl, %dl -; CHECK-NEXT: jne LBB0_17 -; CHECK-NEXT: ## %bb.16: ## %cond_true14.i -; CHECK-NEXT: movl $4, %edi -; CHECK-NEXT: callq _feraiseexcept -; CHECK-NEXT: LBB0_17: ## %ubyte_ctype_remainder.exit -; CHECK-NEXT: xorl %ebx, %ebx -; CHECK-NEXT: LBB0_18: ## %ubyte_ctype_remainder.exit +; CHECK-NEXT: LBB0_16: ## %ubyte_ctype_remainder.exit ; CHECK-NEXT: movq (%r14), %rax ; CHECK-NEXT: callq *224(%rax) ; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je LBB0_21 -; CHECK-NEXT: ## %bb.19: ## %cond_true61 +; CHECK-NEXT: je LBB0_19 +; CHECK-NEXT: ## %bb.17: ## %cond_true61 ; CHECK-NEXT: movl %eax, %ebp ; CHECK-NEXT: movq (%r14), %rax ; CHECK-NEXT: movq _.str5@GOTPCREL(%rip), %rdi @@ -136,8 +139,8 @@ define ptr @ubyte_divmod(ptr %a, ptr %b) { ; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rcx ; CHECK-NEXT: callq *200(%rax) ; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: js LBB0_27 -; CHECK-NEXT: ## %bb.20: ## %cond_next73 +; CHECK-NEXT: js LBB0_23 +; CHECK-NEXT: ## %bb.18: ## %cond_next73 ; CHECK-NEXT: movl $1, {{[0-9]+}}(%rsp) ; CHECK-NEXT: movq (%r14), %rax ; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rsi @@ -146,13 +149,13 @@ define ptr @ubyte_divmod(ptr %a, ptr %b) { ; CHECK-NEXT: movl %ebp, %edx ; CHECK-NEXT: callq *232(%rax) ; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: jne LBB0_27 -; CHECK-NEXT: LBB0_21: ## %cond_next89 +; CHECK-NEXT: jne LBB0_23 +; CHECK-NEXT: LBB0_19: ## %cond_next89 ; CHECK-NEXT: movl $2, %edi ; CHECK-NEXT: callq _PyTuple_New ; CHECK-NEXT: testq %rax, %rax -; CHECK-NEXT: je LBB0_27 -; CHECK-NEXT: ## %bb.22: ## %cond_next97 +; CHECK-NEXT: je LBB0_23 +; CHECK-NEXT: ## %bb.20: ## %cond_next97 ; CHECK-NEXT: movq %rax, %r14 ; CHECK-NEXT: movq _PyArray_API@GOTPCREL(%rip), %r12 ; CHECK-NEXT: movq (%r12), %rax @@ -160,8 +163,8 @@ define ptr @ubyte_divmod(ptr %a, ptr %b) { ; CHECK-NEXT: xorl %esi, %esi ; CHECK-NEXT: callq *304(%rdi) ; CHECK-NEXT: testq %rax, %rax -; CHECK-NEXT: je LBB0_25 -; CHECK-NEXT: ## %bb.23: ## %cond_next135 +; CHECK-NEXT: je LBB0_21 +; CHECK-NEXT: ## %bb.25: ## %cond_next135 ; CHECK-NEXT: movb %r15b, 16(%rax) ; CHECK-NEXT: movq %rax, 24(%r14) ; CHECK-NEXT: movq (%r12), %rax @@ -169,22 +172,22 @@ define ptr @ubyte_divmod(ptr %a, ptr %b) { ; CHECK-NEXT: xorl %esi, %esi ; CHECK-NEXT: callq *304(%rdi) ; CHECK-NEXT: testq %rax, %rax -; CHECK-NEXT: je LBB0_25 -; CHECK-NEXT: ## %bb.24: ## %cond_next182 +; CHECK-NEXT: je LBB0_21 +; CHECK-NEXT: ## %bb.26: ## %cond_next182 ; CHECK-NEXT: movb %bl, 16(%rax) ; CHECK-NEXT: movq %rax, 32(%r14) ; CHECK-NEXT: movq %r14, %rax -; CHECK-NEXT: jmp LBB0_28 -; CHECK-NEXT: LBB0_25: ## %cond_true113 +; CHECK-NEXT: jmp LBB0_24 +; CHECK-NEXT: LBB0_21: ## %cond_true113 ; CHECK-NEXT: decq (%r14) -; CHECK-NEXT: jne LBB0_27 -; CHECK-NEXT: ## %bb.26: ## %cond_true126 +; CHECK-NEXT: jne LBB0_23 +; CHECK-NEXT: ## %bb.22: ## %cond_true126 ; CHECK-NEXT: movq 8(%r14), %rax ; CHECK-NEXT: movq %r14, %rdi ; CHECK-NEXT: callq *48(%rax) -; CHECK-NEXT: LBB0_27: ## %UnifiedReturnBlock +; CHECK-NEXT: LBB0_23: ## %UnifiedReturnBlock ; CHECK-NEXT: xorl %eax, %eax -; CHECK-NEXT: LBB0_28: ## %UnifiedReturnBlock +; CHECK-NEXT: LBB0_24: ## %UnifiedReturnBlock ; CHECK-NEXT: addq $32, %rsp ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: popq %r12 diff --git a/llvm/test/CodeGen/X86/2007-12-18-LoadCSEBug.ll b/llvm/test/CodeGen/X86/2007-12-18-LoadCSEBug.ll index 4482c5aec8e8..d9d4424267d7 100644 --- a/llvm/test/CodeGen/X86/2007-12-18-LoadCSEBug.ll +++ b/llvm/test/CodeGen/X86/2007-12-18-LoadCSEBug.ll @@ -16,15 +16,12 @@ define void @_ada_c34007g() { ; CHECK-NEXT: andl $-8, %esp ; CHECK-NEXT: subl $8, %esp ; CHECK-NEXT: movl (%esp), %eax +; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx +; CHECK-NEXT: orl %eax, %ecx +; CHECK-NEXT: sete %cl ; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB0_3 -; CHECK-NEXT: # %bb.1: # %entry -; CHECK-NEXT: orl {{[0-9]+}}(%esp), %eax -; CHECK-NEXT: jne .LBB0_3 -; CHECK-NEXT: # %bb.2: # %entry -; CHECK-NEXT: movb $1, %al -; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: .LBB0_3: # %bb5507 +; CHECK-NEXT: setne %al +; CHECK-NEXT: testb %cl, %al ; CHECK-NEXT: movl %ebp, %esp ; CHECK-NEXT: popl %ebp ; CHECK-NEXT: .cfi_def_cfa %esp, 4 diff --git a/llvm/test/CodeGen/X86/2008-02-18-TailMergingBug.ll b/llvm/test/CodeGen/X86/2008-02-18-TailMergingBug.ll index dd60e641df25..6ffafc558747 100644 --- a/llvm/test/CodeGen/X86/2008-02-18-TailMergingBug.ll +++ b/llvm/test/CodeGen/X86/2008-02-18-TailMergingBug.ll @@ -217,4 +217,4 @@ bb456: ; preds = %bb448, %bb425, %bb417, %bb395, %bb385, %bb371 ret void } -declare i32 @printf(ptr, ...) nounwind +declare i32 @printf(ptr, ...) nounwind diff --git a/llvm/test/CodeGen/X86/avx-cmp.ll b/llvm/test/CodeGen/X86/avx-cmp.ll index 502bbf3f5d11..4ab9c545ed90 100644 --- a/llvm/test/CodeGen/X86/avx-cmp.ll +++ b/llvm/test/CodeGen/X86/avx-cmp.ll @@ -26,40 +26,33 @@ declare void @scale() nounwind define void @render(double %a0) nounwind { ; CHECK-LABEL: render: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: pushq %rbp ; CHECK-NEXT: pushq %rbx -; CHECK-NEXT: pushq %rax -; CHECK-NEXT: vmovsd %xmm0, (%rsp) # 8-byte Spill +; CHECK-NEXT: subq $16, %rsp +; CHECK-NEXT: vmovsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: jne .LBB2_6 +; CHECK-NEXT: jne .LBB2_5 ; CHECK-NEXT: # %bb.1: # %for.cond5.preheader -; CHECK-NEXT: xorl %ebx, %ebx -; CHECK-NEXT: movb $1, %bpl +; CHECK-NEXT: movb $1, %bl ; CHECK-NEXT: .p2align 4, 0x90 ; CHECK-NEXT: .LBB2_2: # %for.cond5 ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: testb %bl, %bl ; CHECK-NEXT: jne .LBB2_2 -; CHECK-NEXT: # %bb.3: # %for.cond5 -; CHECK-NEXT: # in Loop: Header=BB2_2 Depth=1 -; CHECK-NEXT: testb %bpl, %bpl -; CHECK-NEXT: jne .LBB2_2 -; CHECK-NEXT: # %bb.4: # %for.body33.preheader +; CHECK-NEXT: # %bb.3: # %for.body33.preheader ; CHECK-NEXT: # in Loop: Header=BB2_2 Depth=1 -; CHECK-NEXT: vmovsd (%rsp), %xmm0 # 8-byte Reload +; CHECK-NEXT: vmovsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload ; CHECK-NEXT: # xmm0 = mem[0],zero ; CHECK-NEXT: vucomisd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; CHECK-NEXT: jne .LBB2_5 +; CHECK-NEXT: jne .LBB2_4 ; CHECK-NEXT: jnp .LBB2_2 -; CHECK-NEXT: .LBB2_5: # %if.then +; CHECK-NEXT: .LBB2_4: # %if.then ; CHECK-NEXT: # in Loop: Header=BB2_2 Depth=1 ; CHECK-NEXT: callq scale@PLT ; CHECK-NEXT: jmp .LBB2_2 -; CHECK-NEXT: .LBB2_6: # %for.end52 -; CHECK-NEXT: addq $8, %rsp +; CHECK-NEXT: .LBB2_5: # %for.end52 +; CHECK-NEXT: addq $16, %rsp ; CHECK-NEXT: popq %rbx -; CHECK-NEXT: popq %rbp ; CHECK-NEXT: retq entry: br i1 undef, label %for.cond5, label %for.end52 diff --git a/llvm/test/CodeGen/X86/block-placement.ll b/llvm/test/CodeGen/X86/block-placement.ll index b134d8a36963..e479ad20d6e4 100644 --- a/llvm/test/CodeGen/X86/block-placement.ll +++ b/llvm/test/CodeGen/X86/block-placement.ll @@ -359,7 +359,6 @@ define void @unnatural_cfg2(ptr %p0, i32 %a0) { ; CHECK: %loop.body2 ; CHECK: %loop.body4 ; CHECK: %loop.inner2.begin -; CHECK: %loop.inner2.begin ; CHECK: %loop.body3 ; CHECK: %loop.inner1.begin ; CHECK: %bail diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll index 1372bd804735..fa45afbb634c 100644 --- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll +++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll @@ -178,13 +178,13 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: pushl %edi ; X86-NEXT: pushl %esi ; X86-NEXT: subl $136, %esp -; X86-NEXT: movl {{[0-9]+}}(%esp), %edx -; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx ; X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp +; X86-NEXT: movl {{[0-9]+}}(%esp), %edi ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-NEXT: orl %esi, %eax -; X86-NEXT: orl %edx, %ecx -; X86-NEXT: movl %edx, %edi +; X86-NEXT: orl %edi, %eax +; X86-NEXT: movl %ebp, %ecx +; X86-NEXT: orl %esi, %ecx ; X86-NEXT: orl %eax, %ecx ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ; X86-NEXT: sete %bl @@ -195,30 +195,33 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: sete %al ; X86-NEXT: orb %bl, %al ; X86-NEXT: movb %al, (%esp) # 1-byte Spill -; X86-NEXT: bsrl %esi, %edx +; X86-NEXT: bsrl %edi, %edx ; X86-NEXT: xorl $31, %edx -; X86-NEXT: bsrl %edi, %ecx +; X86-NEXT: bsrl %esi, %ecx ; X86-NEXT: xorl $31, %ecx ; X86-NEXT: addl $32, %ecx -; X86-NEXT: testl %esi, %esi +; X86-NEXT: testl %edi, %edi +; X86-NEXT: movl %edi, %ebx ; X86-NEXT: cmovnel %edx, %ecx ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ; X86-NEXT: bsrl %eax, %edx ; X86-NEXT: xorl $31, %edx -; X86-NEXT: bsrl {{[0-9]+}}(%esp), %ebp -; X86-NEXT: movl %esi, %ebx +; X86-NEXT: bsrl %ebp, %ebp +; X86-NEXT: movl %esi, %edi ; X86-NEXT: movl {{[0-9]+}}(%esp), %esi ; X86-NEXT: xorl $31, %ebp ; X86-NEXT: addl $32, %ebp ; X86-NEXT: testl %eax, %eax ; X86-NEXT: cmovnel %edx, %ebp ; X86-NEXT: addl $64, %ebp -; X86-NEXT: orl %ebx, %edi +; X86-NEXT: movl %edi, %edx +; X86-NEXT: orl %ebx, %edx ; X86-NEXT: cmovnel %ecx, %ebp ; X86-NEXT: bsrl %esi, %edx +; X86-NEXT: movl %esi, %ebx ; X86-NEXT: xorl $31, %edx -; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx -; X86-NEXT: bsrl %ebx, %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: bsrl %eax, %ecx ; X86-NEXT: xorl $31, %ecx ; X86-NEXT: addl $32, %ecx ; X86-NEXT: testl %esi, %esi @@ -230,51 +233,51 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: xorl $31, %edx ; X86-NEXT: addl $32, %edx ; X86-NEXT: testl %edi, %edi -; X86-NEXT: movl %edi, %eax ; X86-NEXT: cmovnel %esi, %edx ; X86-NEXT: addl $64, %edx -; X86-NEXT: movl %ebx, %esi -; X86-NEXT: orl {{[0-9]+}}(%esp), %esi +; X86-NEXT: orl %ebx, %eax ; X86-NEXT: cmovnel %ecx, %edx -; X86-NEXT: xorl %ecx, %ecx ; X86-NEXT: subl %edx, %ebp +; X86-NEXT: movl $0, %edx +; X86-NEXT: sbbl %edx, %edx ; X86-NEXT: movl $0, %esi ; X86-NEXT: sbbl %esi, %esi ; X86-NEXT: movl $0, %edi ; X86-NEXT: sbbl %edi, %edi -; X86-NEXT: movl $0, %ebx -; X86-NEXT: sbbl %ebx, %ebx -; X86-NEXT: movl $127, %edx +; X86-NEXT: movl $127, %ecx +; X86-NEXT: cmpl %ebp, %ecx +; X86-NEXT: movl $0, %ecx +; X86-NEXT: sbbl %edx, %ecx +; X86-NEXT: movl $0, %ecx +; X86-NEXT: sbbl %esi, %ecx +; X86-NEXT: movl $0, %ecx +; X86-NEXT: sbbl %edi, %ecx +; X86-NEXT: setb %cl +; X86-NEXT: orb (%esp), %cl # 1-byte Folded Reload ; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: cmpl %ebp, %edx -; X86-NEXT: movl $0, %edx +; X86-NEXT: movl %ebp, %eax +; X86-NEXT: xorl $127, %eax ; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: sbbl %esi, %edx -; X86-NEXT: movl $0, %edx +; X86-NEXT: orl %esi, %eax +; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: sbbl %edi, %edx -; X86-NEXT: movl $0, %edx -; X86-NEXT: sbbl %ebx, %edx -; X86-NEXT: setb %dl -; X86-NEXT: orb (%esp), %dl # 1-byte Folded Reload -; X86-NEXT: movl {{[0-9]+}}(%esp), %edx -; X86-NEXT: cmovnel %ecx, %edx +; X86-NEXT: orl %edi, %edx +; X86-NEXT: orl %eax, %edx +; X86-NEXT: sete %al +; X86-NEXT: testb %cl, %cl +; X86-NEXT: movl %ebx, %edx +; X86-NEXT: movl $0, %edi +; X86-NEXT: cmovnel %edi, %edx ; X86-NEXT: movl {{[0-9]+}}(%esp), %esi -; X86-NEXT: cmovnel %ecx, %esi -; X86-NEXT: cmovnel %ecx, %eax -; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: cmovnel %edi, %esi ; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp -; X86-NEXT: cmovnel %ecx, %ebp -; X86-NEXT: jne .LBB4_8 -; X86-NEXT: # %bb.1: # %_udiv-special-cases -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload -; X86-NEXT: xorl $127, %eax -; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload -; X86-NEXT: orl %ebx, %ecx -; X86-NEXT: orl %eax, %ecx -; X86-NEXT: je .LBB4_8 -; X86-NEXT: # %bb.2: # %udiv-bb1 +; X86-NEXT: cmovnel %edi, %ebp +; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx +; X86-NEXT: cmovnel %edi, %ebx +; X86-NEXT: orb %cl, %al +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: jne .LBB4_7 +; X86-NEXT: # %bb.1: # %udiv-bb1 ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ; X86-NEXT: movl %eax, {{[0-9]+}}(%esp) ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax @@ -287,9 +290,8 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload -; X86-NEXT: movl %ecx, %eax -; X86-NEXT: movl %ecx, %edi +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload +; X86-NEXT: movl %ebx, %eax ; X86-NEXT: xorb $127, %al ; X86-NEXT: movb %al, %ch ; X86-NEXT: andb $7, %ch @@ -301,7 +303,7 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: movl 132(%esp,%eax), %esi ; X86-NEXT: movb %ch, %cl ; X86-NEXT: shldl %cl, %edx, %esi -; X86-NEXT: movl %esi, (%esp) # 4-byte Spill +; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: shll %cl, %edx ; X86-NEXT: notb %cl ; X86-NEXT: movl 124(%esp,%eax), %ebp @@ -309,68 +311,69 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: shrl %esi ; X86-NEXT: shrl %cl, %esi ; X86-NEXT: orl %edx, %esi -; X86-NEXT: movl %ebp, %edx -; X86-NEXT: movl 120(%esp,%eax), %ebp +; X86-NEXT: movl 120(%esp,%eax), %eax ; X86-NEXT: movb %ch, %cl -; X86-NEXT: shldl %cl, %ebp, %edx -; X86-NEXT: shll %cl, %ebp -; X86-NEXT: addl $1, %edi -; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: shldl %cl, %eax, %ebp +; X86-NEXT: shll %cl, %eax +; X86-NEXT: movl %eax, (%esp) # 4-byte Spill +; X86-NEXT: addl $1, %ebx +; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X86-NEXT: adcl $0, %ecx ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload ; X86-NEXT: adcl $0, %eax +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload ; X86-NEXT: adcl $0, %ebx -; X86-NEXT: jae .LBB4_3 -; X86-NEXT: # %bb.6: +; X86-NEXT: jae .LBB4_2 +; X86-NEXT: # %bb.5: ; X86-NEXT: xorl %eax, %eax ; X86-NEXT: xorl %ecx, %ecx -; X86-NEXT: movl %edx, %ebx -; X86-NEXT: jmp .LBB4_7 -; X86-NEXT: .LBB4_3: # %udiv-preheader -; X86-NEXT: movl {{[0-9]+}}(%esp), %edi -; X86-NEXT: movl %edi, {{[0-9]+}}(%esp) -; X86-NEXT: movl {{[0-9]+}}(%esp), %edi -; X86-NEXT: movl %edi, {{[0-9]+}}(%esp) -; X86-NEXT: movl {{[0-9]+}}(%esp), %edi -; X86-NEXT: movl %edi, {{[0-9]+}}(%esp) -; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: jmp .LBB4_6 +; X86-NEXT: .LBB4_2: # %udiv-preheader +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movl %edx, {{[0-9]+}}(%esp) +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movl %edx, {{[0-9]+}}(%esp) +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movl %edx, {{[0-9]+}}(%esp) ; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp) +; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl %eax, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) ; X86-NEXT: movl $0, {{[0-9]+}}(%esp) +; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload ; X86-NEXT: movb %al, %ch ; X86-NEXT: andb $7, %ch -; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: # kill: def $al killed $al killed $eax ; X86-NEXT: shrb $3, %al ; X86-NEXT: andb $15, %al ; X86-NEXT: movzbl %al, %eax -; X86-NEXT: movl 84(%esp,%eax), %ebp -; X86-NEXT: movl %esi, %edi -; X86-NEXT: movl 80(%esp,%eax), %ebx -; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl %ebx, %esi -; X86-NEXT: movb %ch, %cl -; X86-NEXT: shrdl %cl, %ebp, %esi +; X86-NEXT: movl 84(%esp,%eax), %ebx ; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl 72(%esp,%eax), %esi -; X86-NEXT: movl 76(%esp,%eax), %eax -; X86-NEXT: movl %eax, %edx -; X86-NEXT: shrl %cl, %edx -; X86-NEXT: notb %cl -; X86-NEXT: addl %ebx, %ebx -; X86-NEXT: shll %cl, %ebx -; X86-NEXT: orl %edx, %ebx +; X86-NEXT: movl 80(%esp,%eax), %esi +; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl %esi, %edx ; X86-NEXT: movb %ch, %cl -; X86-NEXT: shrl %cl, %ebp -; X86-NEXT: shrdl %cl, %eax, %esi +; X86-NEXT: shrdl %cl, %ebx, %edx +; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl 72(%esp,%eax), %ebp +; X86-NEXT: movl 76(%esp,%eax), %edx +; X86-NEXT: movl %edx, %eax +; X86-NEXT: shrl %cl, %eax +; X86-NEXT: notb %cl +; X86-NEXT: addl %esi, %esi +; X86-NEXT: shll %cl, %esi +; X86-NEXT: orl %eax, %esi ; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movb %ch, %cl +; X86-NEXT: shrl %cl, %ebx +; X86-NEXT: movl %ebx, %edi +; X86-NEXT: shrdl %cl, %edx, %ebp +; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ; X86-NEXT: addl $-1, %eax ; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill @@ -380,25 +383,25 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx ; X86-NEXT: adcl $-1, %ecx ; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl {{[0-9]+}}(%esp), %esi -; X86-NEXT: adcl $-1, %esi -; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: adcl $-1, %edx +; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: xorl %ecx, %ecx ; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill -; X86-NEXT: movl (%esp), %esi # 4-byte Reload +; X86-NEXT: movl (%esp), %edx # 4-byte Reload ; X86-NEXT: .p2align 4, 0x90 -; X86-NEXT: .LBB4_4: # %udiv-do-while +; X86-NEXT: .LBB4_3: # %udiv-do-while ; X86-NEXT: # =>This Inner Loop Header: Depth=1 ; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload -; X86-NEXT: shldl $1, %edx, %ebp -; X86-NEXT: movl %ebp, (%esp) # 4-byte Spill -; X86-NEXT: shldl $1, %ebx, %edx ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload -; X86-NEXT: shldl $1, %ebp, %ebx -; X86-NEXT: shldl $1, %esi, %ebp +; X86-NEXT: shldl $1, %ebp, %edi +; X86-NEXT: movl %edi, (%esp) # 4-byte Spill +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload +; X86-NEXT: shldl $1, %ebx, %ebp +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload +; X86-NEXT: shldl $1, %esi, %ebx +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload ; X86-NEXT: shldl $1, %edi, %esi -; X86-NEXT: orl %ecx, %esi ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload ; X86-NEXT: shldl $1, %eax, %edi ; X86-NEXT: orl %ecx, %edi @@ -407,14 +410,16 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: shldl $1, %edi, %eax ; X86-NEXT: orl %ecx, %eax ; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: addl %edi, %edi -; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload +; X86-NEXT: shldl $1, %edx, %edi +; X86-NEXT: orl %ecx, %edi ; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: cmpl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload +; X86-NEXT: addl %edx, %edx +; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload +; X86-NEXT: cmpl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X86-NEXT: sbbl %ebx, %ecx ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload -; X86-NEXT: sbbl %edx, %ecx +; X86-NEXT: sbbl %ebp, %ecx ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X86-NEXT: sbbl (%esp), %ecx # 4-byte Folded Reload ; X86-NEXT: sarl $31, %ecx @@ -429,84 +434,81 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: movl %ecx, %eax ; X86-NEXT: andl {{[0-9]+}}(%esp), %eax ; X86-NEXT: andl {{[0-9]+}}(%esp), %ecx -; X86-NEXT: subl %ecx, %ebp -; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: subl %ecx, %esi +; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: sbbl %eax, %ebx ; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: sbbl %edi, %edx -; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload +; X86-NEXT: sbbl %edi, %ebp +; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload ; X86-NEXT: sbbl %eax, (%esp) # 4-byte Folded Spill ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload ; X86-NEXT: addl $-1, %ecx ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload ; X86-NEXT: adcl $-1, %eax -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload -; X86-NEXT: adcl $-1, %edx ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload ; X86-NEXT: adcl $-1, %ebx +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload +; X86-NEXT: adcl $-1, %edi ; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: orl %ebx, %eax -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload +; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: orl %edi, %eax ; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: orl %edx, %ecx -; X86-NEXT: movl (%esp), %ebp # 4-byte Reload +; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: orl %ebx, %ecx +; X86-NEXT: movl (%esp), %edi # 4-byte Reload ; X86-NEXT: orl %eax, %ecx ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload -; X86-NEXT: jne .LBB4_4 -; X86-NEXT: # %bb.5: -; X86-NEXT: movl %esi, (%esp) # 4-byte Spill -; X86-NEXT: movl %edi, %esi -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload +; X86-NEXT: jne .LBB4_3 +; X86-NEXT: # %bb.4: +; X86-NEXT: movl %edx, (%esp) # 4-byte Spill +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload -; X86-NEXT: .LBB4_7: # %udiv-loop-exit -; X86-NEXT: movl (%esp), %edx # 4-byte Reload +; X86-NEXT: .LBB4_6: # %udiv-loop-exit +; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload ; X86-NEXT: shldl $1, %esi, %edx ; X86-NEXT: orl %ecx, %edx -; X86-NEXT: shldl $1, %ebx, %esi +; X86-NEXT: shldl $1, %ebp, %esi ; X86-NEXT: orl %ecx, %esi -; X86-NEXT: shldl $1, %ebp, %ebx -; X86-NEXT: orl %ecx, %ebx -; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: addl %ebp, %ebp -; X86-NEXT: orl %eax, %ebp -; X86-NEXT: .LBB4_8: # %udiv-end +; X86-NEXT: movl (%esp), %ebx # 4-byte Reload +; X86-NEXT: shldl $1, %ebx, %ebp +; X86-NEXT: orl %ecx, %ebp +; X86-NEXT: addl %ebx, %ebx +; X86-NEXT: orl %eax, %ebx ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-NEXT: movl %ebp, (%eax) -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload -; X86-NEXT: movl %ecx, 4(%eax) +; X86-NEXT: .LBB4_7: # %udiv-end +; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl %ebx, (%eax) +; X86-NEXT: movl %ebp, 4(%eax) ; X86-NEXT: movl %esi, 8(%eax) ; X86-NEXT: movl %edx, 12(%eax) +; X86-NEXT: movl %ebx, %ecx +; X86-NEXT: movl %ebx, (%esp) # 4-byte Spill ; X86-NEXT: movl %esi, %ebx ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ; X86-NEXT: movl %eax, %esi -; X86-NEXT: imull %ecx, %esi -; X86-NEXT: movl %ebp, %ecx -; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; X86-NEXT: movl %edx, %ebp +; X86-NEXT: imull %ebp, %esi +; X86-NEXT: movl %edx, %edi ; X86-NEXT: mull %ecx -; X86-NEXT: movl %eax, (%esp) # 4-byte Spill +; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: addl %esi, %edx -; X86-NEXT: movl {{[0-9]+}}(%esp), %edi -; X86-NEXT: imull %ecx, %edi -; X86-NEXT: addl %edx, %edi +; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp +; X86-NEXT: imull %ecx, %ebp +; X86-NEXT: addl %edx, %ebp ; X86-NEXT: movl {{[0-9]+}}(%esp), %esi ; X86-NEXT: movl %esi, %eax ; X86-NEXT: mull %ebx ; X86-NEXT: movl %eax, %ecx -; X86-NEXT: imull %esi, %ebp -; X86-NEXT: addl %edx, %ebp +; X86-NEXT: imull %esi, %edi +; X86-NEXT: addl %edx, %edi ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ; X86-NEXT: imull %eax, %ebx -; X86-NEXT: addl %ebp, %ebx -; X86-NEXT: addl (%esp), %ecx # 4-byte Folded Reload -; X86-NEXT: movl %ecx, (%esp) # 4-byte Spill -; X86-NEXT: adcl %edi, %ebx -; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload +; X86-NEXT: addl %edi, %ebx +; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload +; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: adcl %ebp, %ebx +; X86-NEXT: movl (%esp), %ebp # 4-byte Reload ; X86-NEXT: movl %ebp, %eax ; X86-NEXT: mull %esi ; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill @@ -522,7 +524,7 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: movl {{[0-9]+}}(%esp), %esi ; X86-NEXT: movl %edx, %ebp ; X86-NEXT: addl %ecx, %eax -; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill +; X86-NEXT: movl %eax, (%esp) # 4-byte Spill ; X86-NEXT: adcl %edi, %ebp ; X86-NEXT: setb %cl ; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload @@ -530,11 +532,11 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind { ; X86-NEXT: addl %ebp, %eax ; X86-NEXT: movzbl %cl, %ecx ; X86-NEXT: adcl %ecx, %edx -; X86-NEXT: addl (%esp), %eax # 4-byte Folded Reload +; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload ; X86-NEXT: adcl %ebx, %edx ; X86-NEXT: subl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload ; X86-NEXT: movl {{[0-9]+}}(%esp), %edi -; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload +; X86-NEXT: sbbl (%esp), %edi # 4-byte Folded Reload ; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx ; X86-NEXT: sbbl %eax, %ebx ; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx diff --git a/llvm/test/CodeGen/X86/inline-spiller-impdef-on-implicit-def-regression.ll b/llvm/test/CodeGen/X86/inline-spiller-impdef-on-implicit-def-regression.ll index 0250b1b4a7f8..f42c2f8f1447 100644 --- a/llvm/test/CodeGen/X86/inline-spiller-impdef-on-implicit-def-regression.ll +++ b/llvm/test/CodeGen/X86/inline-spiller-impdef-on-implicit-def-regression.ll @@ -28,78 +28,70 @@ define i32 @decode_sb(ptr %t, i32 %bl, i32 %_msprop1966, i32 %sub.i, i64 %idxpro ; CHECK-NEXT: .cfi_offset %r15, -24 ; CHECK-NEXT: movl %r9d, %ebx ; CHECK-NEXT: # kill: def $edx killed $edx def $rdx -; CHECK-NEXT: movabsq $87960930222080, %r15 # imm = 0x500000000000 -; CHECK-NEXT: movl 0, %r11d -; CHECK-NEXT: movl %esi, %r12d -; CHECK-NEXT: # implicit-def: $r13d +; CHECK-NEXT: movabsq $87960930222080, %r14 # imm = 0x500000000000 +; CHECK-NEXT: movl 0, %r13d +; CHECK-NEXT: movl %esi, %r15d +; CHECK-NEXT: # implicit-def: $r12d ; CHECK-NEXT: testb $1, %bl -; CHECK-NEXT: jne .LBB0_7 +; CHECK-NEXT: jne .LBB0_6 ; CHECK-NEXT: # %bb.1: # %if.else -; CHECK-NEXT: movq %r8, %r14 -; CHECK-NEXT: movl %ecx, %r13d -; CHECK-NEXT: andl $1, %r13d -; CHECK-NEXT: movzbl 544(%r13), %r8d -; CHECK-NEXT: andl $1, %r8d -; CHECK-NEXT: movl %r15d, %r9d +; CHECK-NEXT: movl %ecx, %r12d +; CHECK-NEXT: andl $1, %r12d +; CHECK-NEXT: movzbl 544(%r12), %r9d ; CHECK-NEXT: andl $1, %r9d ; CHECK-NEXT: movl %r14d, %r10d ; CHECK-NEXT: andl $1, %r10d +; CHECK-NEXT: andl $1, %r8d ; CHECK-NEXT: movabsq $17592186044416, %rax # imm = 0x100000000000 -; CHECK-NEXT: orq %r10, %rax -; CHECK-NEXT: movl %esi, %r10d +; CHECK-NEXT: orq %r8, %rax +; CHECK-NEXT: movl %esi, %r8d ; CHECK-NEXT: # kill: def $cl killed $cl killed $ecx -; CHECK-NEXT: shrl %cl, %r10d -; CHECK-NEXT: andl $2, %r10d +; CHECK-NEXT: shrl %cl, %r8d +; CHECK-NEXT: andl $2, %r8d ; CHECK-NEXT: testb $1, %bl -; CHECK-NEXT: cmoveq %r9, %rax -; CHECK-NEXT: orl %r8d, %edx -; CHECK-NEXT: movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; CHECK-NEXT: movq %r11, %rcx +; CHECK-NEXT: cmoveq %r10, %rax +; CHECK-NEXT: orl %r9d, %edx +; CHECK-NEXT: movq %r13, %rcx ; CHECK-NEXT: orq $1, %rcx -; CHECK-NEXT: orl %esi, %r10d +; CHECK-NEXT: orl %esi, %r8d ; CHECK-NEXT: movl $1, %r8d ; CHECK-NEXT: je .LBB0_3 ; CHECK-NEXT: # %bb.2: # %if.else ; CHECK-NEXT: movl (%rax), %r8d ; CHECK-NEXT: .LBB0_3: # %if.else ; CHECK-NEXT: shlq $5, %rdx -; CHECK-NEXT: movq %r12, %rax +; CHECK-NEXT: movq %r15, %rax ; CHECK-NEXT: shlq $7, %rax ; CHECK-NEXT: leaq (%rax,%rdx), %rsi ; CHECK-NEXT: addq $1248, %rsi # imm = 0x4E0 ; CHECK-NEXT: movq %rcx, 0 -; CHECK-NEXT: movq %rdi, %r15 +; CHECK-NEXT: movq %rdi, %r14 ; CHECK-NEXT: movl %r8d, (%rdi) ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: xorl %edi, %edi ; CHECK-NEXT: xorl %edx, %edx ; CHECK-NEXT: callq *%rax -; CHECK-NEXT: xorq $1, %r14 -; CHECK-NEXT: cmpl $0, (%r14) -; CHECK-NEXT: je .LBB0_6 -; CHECK-NEXT: # %bb.4: # %if.else ; CHECK-NEXT: movb $1, %al ; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: je .LBB0_5 -; CHECK-NEXT: .LBB0_6: # %bb19 +; CHECK-NEXT: je .LBB0_4 +; CHECK-NEXT: # %bb.5: # %bb19 ; CHECK-NEXT: testb $1, %bl -; CHECK-NEXT: movq %r15, %rdi -; CHECK-NEXT: movabsq $87960930222080, %r15 # imm = 0x500000000000 -; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload -; CHECK-NEXT: jne .LBB0_8 -; CHECK-NEXT: .LBB0_7: # %if.end69 -; CHECK-NEXT: movl %r11d, 0 +; CHECK-NEXT: movq %r14, %rdi +; CHECK-NEXT: movabsq $87960930222080, %r14 # imm = 0x500000000000 +; CHECK-NEXT: jne .LBB0_7 +; CHECK-NEXT: .LBB0_6: # %if.end69 +; CHECK-NEXT: movl %r13d, 0 ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: xorl %esi, %esi ; CHECK-NEXT: xorl %edx, %edx ; CHECK-NEXT: xorl %ecx, %ecx ; CHECK-NEXT: xorl %r8d, %r8d ; CHECK-NEXT: callq *%rax -; CHECK-NEXT: xorq %r15, %r12 -; CHECK-NEXT: movslq %r13d, %rax -; CHECK-NEXT: movzbl (%r12), %ecx +; CHECK-NEXT: xorq %r14, %r15 +; CHECK-NEXT: movslq %r12d, %rax +; CHECK-NEXT: movzbl (%r15), %ecx ; CHECK-NEXT: movb %cl, 544(%rax) -; CHECK-NEXT: .LBB0_8: # %land.lhs.true56 +; CHECK-NEXT: .LBB0_7: # %land.lhs.true56 ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: addq $8, %rsp ; CHECK-NEXT: popq %rbx @@ -110,7 +102,7 @@ define i32 @decode_sb(ptr %t, i32 %bl, i32 %_msprop1966, i32 %sub.i, i64 %idxpro ; CHECK-NEXT: popq %rbp ; CHECK-NEXT: .cfi_def_cfa %rsp, 8 ; CHECK-NEXT: retq -; CHECK-NEXT: .LBB0_5: # %bb +; CHECK-NEXT: .LBB0_4: # %bb entry: %i = load i32, ptr null, align 8 br i1 %cmp54, label %if.end69, label %if.else diff --git a/llvm/test/CodeGen/X86/movmsk-cmp.ll b/llvm/test/CodeGen/X86/movmsk-cmp.ll index a7564c9622c5..e8b3121ecfb5 100644 --- a/llvm/test/CodeGen/X86/movmsk-cmp.ll +++ b/llvm/test/CodeGen/X86/movmsk-cmp.ll @@ -4440,16 +4440,14 @@ define i32 @pr67287(<2 x i64> %broadcast.splatinsert25) { ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,0,3,2] ; SSE2-NEXT: movmskpd %xmm0, %eax ; SSE2-NEXT: testl %eax, %eax -; SSE2-NEXT: jne .LBB97_2 -; SSE2-NEXT: # %bb.1: # %entry -; SSE2-NEXT: movd %xmm1, %eax -; SSE2-NEXT: testb $1, %al -; SSE2-NEXT: jne .LBB97_2 -; SSE2-NEXT: # %bb.3: # %middle.block -; SSE2-NEXT: xorl %eax, %eax -; SSE2-NEXT: retq -; SSE2-NEXT: .LBB97_2: +; SSE2-NEXT: setne %al +; SSE2-NEXT: movd %xmm1, %ecx +; SSE2-NEXT: orb %al, %cl +; SSE2-NEXT: testb $1, %cl +; SSE2-NEXT: je .LBB97_2 +; SSE2-NEXT: # %bb.1: ; SSE2-NEXT: movw $0, 0 +; SSE2-NEXT: .LBB97_2: # %middle.block ; SSE2-NEXT: xorl %eax, %eax ; SSE2-NEXT: retq ; @@ -4460,16 +4458,14 @@ define i32 @pr67287(<2 x i64> %broadcast.splatinsert25) { ; SSE41-NEXT: pcmpeqq %xmm1, %xmm0 ; SSE41-NEXT: movmskpd %xmm0, %eax ; SSE41-NEXT: testl %eax, %eax -; SSE41-NEXT: jne .LBB97_2 -; SSE41-NEXT: # %bb.1: # %entry -; SSE41-NEXT: movd %xmm0, %eax -; SSE41-NEXT: testb $1, %al -; SSE41-NEXT: jne .LBB97_2 -; SSE41-NEXT: # %bb.3: # %middle.block -; SSE41-NEXT: xorl %eax, %eax -; SSE41-NEXT: retq -; SSE41-NEXT: .LBB97_2: +; SSE41-NEXT: setne %al +; SSE41-NEXT: movd %xmm0, %ecx +; SSE41-NEXT: orb %al, %cl +; SSE41-NEXT: testb $1, %cl +; SSE41-NEXT: je .LBB97_2 +; SSE41-NEXT: # %bb.1: ; SSE41-NEXT: movw $0, 0 +; SSE41-NEXT: .LBB97_2: # %middle.block ; SSE41-NEXT: xorl %eax, %eax ; SSE41-NEXT: retq ; @@ -4479,16 +4475,14 @@ define i32 @pr67287(<2 x i64> %broadcast.splatinsert25) { ; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7] ; AVX1-NEXT: vpcmpeqq %xmm1, %xmm0, %xmm0 ; AVX1-NEXT: vtestpd %xmm0, %xmm0 -; AVX1-NEXT: jne .LBB97_2 -; AVX1-NEXT: # %bb.1: # %entry -; AVX1-NEXT: vmovd %xmm0, %eax -; AVX1-NEXT: testb $1, %al -; AVX1-NEXT: jne .LBB97_2 -; AVX1-NEXT: # %bb.3: # %middle.block -; AVX1-NEXT: xorl %eax, %eax -; AVX1-NEXT: retq -; AVX1-NEXT: .LBB97_2: +; AVX1-NEXT: setne %al +; AVX1-NEXT: vmovd %xmm0, %ecx +; AVX1-NEXT: orb %al, %cl +; AVX1-NEXT: testb $1, %cl +; AVX1-NEXT: je .LBB97_2 +; AVX1-NEXT: # %bb.1: ; AVX1-NEXT: movw $0, 0 +; AVX1-NEXT: .LBB97_2: # %middle.block ; AVX1-NEXT: xorl %eax, %eax ; AVX1-NEXT: retq ; @@ -4498,16 +4492,14 @@ define i32 @pr67287(<2 x i64> %broadcast.splatinsert25) { ; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] ; AVX2-NEXT: vpcmpeqq %xmm1, %xmm0, %xmm0 ; AVX2-NEXT: vtestpd %xmm0, %xmm0 -; AVX2-NEXT: jne .LBB97_2 -; AVX2-NEXT: # %bb.1: # %entry -; AVX2-NEXT: vmovd %xmm0, %eax -; AVX2-NEXT: testb $1, %al -; AVX2-NEXT: jne .LBB97_2 -; AVX2-NEXT: # %bb.3: # %middle.block -; AVX2-NEXT: xorl %eax, %eax -; AVX2-NEXT: retq -; AVX2-NEXT: .LBB97_2: +; AVX2-NEXT: setne %al +; AVX2-NEXT: vmovd %xmm0, %ecx +; AVX2-NEXT: orb %al, %cl +; AVX2-NEXT: testb $1, %cl +; AVX2-NEXT: je .LBB97_2 +; AVX2-NEXT: # %bb.1: ; AVX2-NEXT: movw $0, 0 +; AVX2-NEXT: .LBB97_2: # %middle.block ; AVX2-NEXT: xorl %eax, %eax ; AVX2-NEXT: retq ; @@ -4517,18 +4509,15 @@ define i32 @pr67287(<2 x i64> %broadcast.splatinsert25) { ; KNL-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] ; KNL-NEXT: vptestnmq %zmm0, %zmm0, %k0 ; KNL-NEXT: kmovw %k0, %eax -; KNL-NEXT: testb $3, %al -; KNL-NEXT: jne .LBB97_2 -; KNL-NEXT: # %bb.1: # %entry -; KNL-NEXT: kmovw %k0, %eax +; KNL-NEXT: kmovw %k0, %ecx +; KNL-NEXT: testb $3, %cl +; KNL-NEXT: setne %cl +; KNL-NEXT: orb %cl, %al ; KNL-NEXT: testb $1, %al -; KNL-NEXT: jne .LBB97_2 -; KNL-NEXT: # %bb.3: # %middle.block -; KNL-NEXT: xorl %eax, %eax -; KNL-NEXT: vzeroupper -; KNL-NEXT: retq -; KNL-NEXT: .LBB97_2: +; KNL-NEXT: je .LBB97_2 +; KNL-NEXT: # %bb.1: ; KNL-NEXT: movw $0, 0 +; KNL-NEXT: .LBB97_2: # %middle.block ; KNL-NEXT: xorl %eax, %eax ; KNL-NEXT: vzeroupper ; KNL-NEXT: retq @@ -4539,16 +4528,14 @@ define i32 @pr67287(<2 x i64> %broadcast.splatinsert25) { ; SKX-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] ; SKX-NEXT: vptestnmq %xmm0, %xmm0, %k0 ; SKX-NEXT: kortestb %k0, %k0 -; SKX-NEXT: jne .LBB97_2 -; SKX-NEXT: # %bb.1: # %entry -; SKX-NEXT: kmovd %k0, %eax -; SKX-NEXT: testb $1, %al -; SKX-NEXT: jne .LBB97_2 -; SKX-NEXT: # %bb.3: # %middle.block -; SKX-NEXT: xorl %eax, %eax -; SKX-NEXT: retq -; SKX-NEXT: .LBB97_2: +; SKX-NEXT: setne %al +; SKX-NEXT: kmovd %k0, %ecx +; SKX-NEXT: orb %al, %cl +; SKX-NEXT: testb $1, %cl +; SKX-NEXT: je .LBB97_2 +; SKX-NEXT: # %bb.1: ; SKX-NEXT: movw $0, 0 +; SKX-NEXT: .LBB97_2: # %middle.block ; SKX-NEXT: xorl %eax, %eax ; SKX-NEXT: retq entry: diff --git a/llvm/test/CodeGen/X86/or-branch.ll b/llvm/test/CodeGen/X86/or-branch.ll index 5d5cc2cb32f1..c6df237393e4 100644 --- a/llvm/test/CodeGen/X86/or-branch.ll +++ b/llvm/test/CodeGen/X86/or-branch.ll @@ -5,12 +5,13 @@ define void @foo(i32 %X, i32 %Y, i32 %Z) nounwind { ; JUMP2-LABEL: foo: ; JUMP2: # %bb.0: # %entry -; JUMP2-NEXT: cmpl $5, {{[0-9]+}}(%esp) -; JUMP2-NEXT: jl bar@PLT # TAILCALL -; JUMP2-NEXT: # %bb.1: # %entry ; JUMP2-NEXT: cmpl $0, {{[0-9]+}}(%esp) +; JUMP2-NEXT: setne %al +; JUMP2-NEXT: cmpl $5, {{[0-9]+}}(%esp) +; JUMP2-NEXT: setge %cl +; JUMP2-NEXT: testb %al, %cl ; JUMP2-NEXT: je bar@PLT # TAILCALL -; JUMP2-NEXT: # %bb.2: # %UnifiedReturnBlock +; JUMP2-NEXT: # %bb.1: # %UnifiedReturnBlock ; JUMP2-NEXT: retl ; ; JUMP1-LABEL: foo: diff --git a/llvm/test/CodeGen/X86/peephole-na-phys-copy-folding.ll b/llvm/test/CodeGen/X86/peephole-na-phys-copy-folding.ll index 9069688c8037..f3741dc202dc 100644 --- a/llvm/test/CodeGen/X86/peephole-na-phys-copy-folding.ll +++ b/llvm/test/CodeGen/X86/peephole-na-phys-copy-folding.ll @@ -284,7 +284,7 @@ define i64 @test_two_live_flags(ptr %foo0, i64 %bar0, i64 %baz0, ptr %foo1, i64 ; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx ; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi ; CHECK32-NEXT: lock cmpxchg8b (%esi) -; CHECK32-NEXT: setne {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill +; CHECK32-NEXT: sete {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill ; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %eax ; CHECK32-NEXT: movl %ebp, %edx ; CHECK32-NEXT: movl %edi, %ecx @@ -292,17 +292,15 @@ define i64 @test_two_live_flags(ptr %foo0, i64 %bar0, i64 %baz0, ptr %foo1, i64 ; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi ; CHECK32-NEXT: lock cmpxchg8b (%esi) ; CHECK32-NEXT: sete %al -; CHECK32-NEXT: cmpb $0, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload -; CHECK32-NEXT: jne .LBB5_4 -; CHECK32-NEXT: # %bb.1: # %entry -; CHECK32-NEXT: testb %al, %al -; CHECK32-NEXT: je .LBB5_4 -; CHECK32-NEXT: # %bb.2: # %t +; CHECK32-NEXT: andb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Folded Reload +; CHECK32-NEXT: cmpb $1, %al +; CHECK32-NEXT: jne .LBB5_3 +; CHECK32-NEXT: # %bb.1: # %t ; CHECK32-NEXT: movl $42, %eax -; CHECK32-NEXT: jmp .LBB5_3 -; CHECK32-NEXT: .LBB5_4: # %f +; CHECK32-NEXT: jmp .LBB5_2 +; CHECK32-NEXT: .LBB5_3: # %f ; CHECK32-NEXT: xorl %eax, %eax -; CHECK32-NEXT: .LBB5_3: # %t +; CHECK32-NEXT: .LBB5_2: # %t ; CHECK32-NEXT: xorl %edx, %edx ; CHECK32-NEXT: addl $4, %esp ; CHECK32-NEXT: popl %esi @@ -315,19 +313,17 @@ define i64 @test_two_live_flags(ptr %foo0, i64 %bar0, i64 %baz0, ptr %foo1, i64 ; CHECK64: # %bb.0: # %entry ; CHECK64-NEXT: movq %rsi, %rax ; CHECK64-NEXT: lock cmpxchgq %rdx, (%rdi) -; CHECK64-NEXT: setne %dl +; CHECK64-NEXT: sete %dl ; CHECK64-NEXT: movq %r8, %rax ; CHECK64-NEXT: lock cmpxchgq %r9, (%rcx) ; CHECK64-NEXT: sete %al -; CHECK64-NEXT: testb %dl, %dl -; CHECK64-NEXT: jne .LBB5_3 -; CHECK64-NEXT: # %bb.1: # %entry -; CHECK64-NEXT: testb %al, %al -; CHECK64-NEXT: je .LBB5_3 -; CHECK64-NEXT: # %bb.2: # %t +; CHECK64-NEXT: andb %dl, %al +; CHECK64-NEXT: cmpb $1, %al +; CHECK64-NEXT: jne .LBB5_2 +; CHECK64-NEXT: # %bb.1: # %t ; CHECK64-NEXT: movl $42, %eax ; CHECK64-NEXT: retq -; CHECK64-NEXT: .LBB5_3: # %f +; CHECK64-NEXT: .LBB5_2: # %f ; CHECK64-NEXT: xorl %eax, %eax ; CHECK64-NEXT: retq entry: @@ -353,7 +349,6 @@ define i1 @asm_clobbering_flags(ptr %mem) nounwind { ; CHECK32-NEXT: testl %edx, %edx ; CHECK32-NEXT: setg %al ; CHECK32-NEXT: #APP -; CHECK32-NOT: rep ; CHECK32-NEXT: bsfl %edx, %edx ; CHECK32-NEXT: #NO_APP ; CHECK32-NEXT: movl %edx, (%ecx) @@ -365,7 +360,6 @@ define i1 @asm_clobbering_flags(ptr %mem) nounwind { ; CHECK64-NEXT: testl %ecx, %ecx ; CHECK64-NEXT: setg %al ; CHECK64-NEXT: #APP -; CHECK64-NOT: rep ; CHECK64-NEXT: bsfl %ecx, %ecx ; CHECK64-NEXT: #NO_APP ; CHECK64-NEXT: movl %ecx, (%rdi) diff --git a/llvm/test/CodeGen/X86/pr33747.ll b/llvm/test/CodeGen/X86/pr33747.ll index e261486dd592..c8ba2b2e3a79 100644 --- a/llvm/test/CodeGen/X86/pr33747.ll +++ b/llvm/test/CodeGen/X86/pr33747.ll @@ -5,18 +5,19 @@ define void @PR33747(ptr nocapture) { ; CHECK-LABEL: PR33747: ; CHECK: # %bb.0: ; CHECK-NEXT: movl 24(%rdi), %eax +; CHECK-NEXT: leal 1(%rax), %ecx +; CHECK-NEXT: cmpl $3, %ecx +; CHECK-NEXT: setb %cl ; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB0_3 -; CHECK-NEXT: # %bb.1: -; CHECK-NEXT: incl %eax -; CHECK-NEXT: cmpl $3, %eax -; CHECK-NEXT: jae .LBB0_3 +; CHECK-NEXT: setne %al +; CHECK-NEXT: testb %cl, %al +; CHECK-NEXT: je .LBB0_2 +; CHECK-NEXT: .p2align 4, 0x90 +; CHECK-NEXT: .LBB0_1: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: jmp .LBB0_1 ; CHECK-NEXT: .p2align 4, 0x90 ; CHECK-NEXT: .LBB0_2: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: jmp .LBB0_2 -; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: .LBB0_3: # =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: jmp .LBB0_3 %2 = getelementptr inbounds i32, ptr %0, i64 6 %3 = load i32, ptr %2, align 4 %4 = add i32 %3, 1 diff --git a/llvm/test/CodeGen/X86/pr37025.ll b/llvm/test/CodeGen/X86/pr37025.ll index a758ddc91541..8ac28d6286a6 100644 --- a/llvm/test/CodeGen/X86/pr37025.ll +++ b/llvm/test/CodeGen/X86/pr37025.ll @@ -18,11 +18,13 @@ define void @test_dec_select(ptr nocapture %0, ptr readnone %1) { ; CHECK-LABEL: test_dec_select: ; CHECK: # %bb.0: ; CHECK-NEXT: lock decq (%rdi) -; CHECK-NEXT: jne .LBB0_2 -; CHECK-NEXT: # %bb.1: +; CHECK-NEXT: sete %al ; CHECK-NEXT: testq %rsi, %rsi -; CHECK-NEXT: jne func2 # TAILCALL -; CHECK-NEXT: .LBB0_2: +; CHECK-NEXT: setne %cl +; CHECK-NEXT: andb %al, %cl +; CHECK-NEXT: cmpb $1, %cl +; CHECK-NEXT: je func2 # TAILCALL +; CHECK-NEXT: # %bb.1: ; CHECK-NEXT: retq %3 = atomicrmw sub ptr %0, i64 1 seq_cst %4 = icmp eq i64 %3, 1 @@ -44,11 +46,11 @@ define void @test_dec_select_commute(ptr nocapture %0, ptr readnone %1) { ; CHECK-NEXT: lock decq (%rdi) ; CHECK-NEXT: sete %al ; CHECK-NEXT: testq %rsi, %rsi -; CHECK-NEXT: je .LBB1_2 +; CHECK-NEXT: setne %cl +; CHECK-NEXT: andb %al, %cl +; CHECK-NEXT: cmpb $1, %cl +; CHECK-NEXT: je func2 # TAILCALL ; CHECK-NEXT: # %bb.1: -; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: jne func2 # TAILCALL -; CHECK-NEXT: .LBB1_2: ; CHECK-NEXT: retq %3 = atomicrmw sub ptr %0, i64 1 seq_cst %4 = icmp eq i64 %3, 1 @@ -69,12 +71,13 @@ define void @test_dec_and(ptr nocapture %0, ptr readnone %1) { ; CHECK: # %bb.0: ; CHECK-NEXT: lock decq (%rdi) ; CHECK-NEXT: sete %al +; CHECK-NEXT: notb %al ; CHECK-NEXT: testq %rsi, %rsi -; CHECK-NEXT: je .LBB2_2 +; CHECK-NEXT: sete %cl +; CHECK-NEXT: orb %al, %cl +; CHECK-NEXT: testb $1, %cl +; CHECK-NEXT: je func2 # TAILCALL ; CHECK-NEXT: # %bb.1: -; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: jne func2 # TAILCALL -; CHECK-NEXT: .LBB2_2: ; CHECK-NEXT: retq %3 = atomicrmw sub ptr %0, i64 1 seq_cst %4 = icmp eq i64 %3, 1 @@ -94,11 +97,14 @@ define void @test_dec_and_commute(ptr nocapture %0, ptr readnone %1) { ; CHECK-LABEL: test_dec_and_commute: ; CHECK: # %bb.0: ; CHECK-NEXT: lock decq (%rdi) -; CHECK-NEXT: jne .LBB3_2 -; CHECK-NEXT: # %bb.1: +; CHECK-NEXT: sete %al +; CHECK-NEXT: notb %al ; CHECK-NEXT: testq %rsi, %rsi -; CHECK-NEXT: jne func2 # TAILCALL -; CHECK-NEXT: .LBB3_2: +; CHECK-NEXT: sete %cl +; CHECK-NEXT: orb %al, %cl +; CHECK-NEXT: testb $1, %cl +; CHECK-NEXT: je func2 # TAILCALL +; CHECK-NEXT: # %bb.1: ; CHECK-NEXT: retq %3 = atomicrmw sub ptr %0, i64 1 seq_cst %4 = icmp eq i64 %3, 1 diff --git a/llvm/test/CodeGen/X86/pr38795.ll b/llvm/test/CodeGen/X86/pr38795.ll index 03629a353d84..f64c70e8fc79 100644 --- a/llvm/test/CodeGen/X86/pr38795.ll +++ b/llvm/test/CodeGen/X86/pr38795.ll @@ -25,141 +25,126 @@ define dso_local void @fn() { ; CHECK-NEXT: xorl %ebx, %ebx ; CHECK-NEXT: # implicit-def: $ecx ; CHECK-NEXT: # implicit-def: $edi +; CHECK-NEXT: # implicit-def: $dh ; CHECK-NEXT: # implicit-def: $al ; CHECK-NEXT: # kill: killed $al -; CHECK-NEXT: # implicit-def: $al ; CHECK-NEXT: # implicit-def: $ebp ; CHECK-NEXT: jmp .LBB0_1 ; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: .LBB0_16: # in Loop: Header=BB0_1 Depth=1 -; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill -; CHECK-NEXT: movb %dh, %al +; CHECK-NEXT: .LBB0_15: # %for.inc +; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 +; CHECK-NEXT: movb %dl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill ; CHECK-NEXT: .LBB0_1: # %for.cond ; CHECK-NEXT: # =>This Loop Header: Depth=1 -; CHECK-NEXT: # Child Loop BB0_22 Depth 2 -; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill -; CHECK-NEXT: cmpb $8, %al -; CHECK-NEXT: ja .LBB0_3 -; CHECK-NEXT: # %bb.2: # %for.cond -; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 +; CHECK-NEXT: # Child Loop BB0_19 Depth 2 ; CHECK-NEXT: testb %bl, %bl -; CHECK-NEXT: je .LBB0_3 -; CHECK-NEXT: # %bb.4: # %if.end +; CHECK-NEXT: jne .LBB0_3 +; CHECK-NEXT: # %bb.2: # %if.then +; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 +; CHECK-NEXT: movb %dh, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill +; CHECK-NEXT: movl $.str, (%esp) +; CHECK-NEXT: calll printf +; CHECK-NEXT: # implicit-def: $eax +; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload +; CHECK-NEXT: testl %edi, %edi +; CHECK-NEXT: jne .LBB0_10 +; CHECK-NEXT: jmp .LBB0_6 +; CHECK-NEXT: .p2align 4, 0x90 +; CHECK-NEXT: .LBB0_3: # %if.end ; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 ; CHECK-NEXT: movl %ecx, %eax ; CHECK-NEXT: cltd ; CHECK-NEXT: idivl a -; CHECK-NEXT: movl %eax, %esi -; CHECK-NEXT: movb %cl, %dh +; CHECK-NEXT: movl %ecx, %edx ; CHECK-NEXT: movl $0, h -; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload -; CHECK-NEXT: cmpb $8, %al -; CHECK-NEXT: jg .LBB0_8 -; CHECK-NEXT: # %bb.5: # %if.then13 +; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %dh # 1-byte Reload +; CHECK-NEXT: cmpb $8, %dh +; CHECK-NEXT: jg .LBB0_7 +; CHECK-NEXT: # %bb.4: # %if.then13 ; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 +; CHECK-NEXT: movl %eax, %esi ; CHECK-NEXT: movl $.str, (%esp) -; CHECK-NEXT: movb %dh, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill +; CHECK-NEXT: movb %dl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill ; CHECK-NEXT: calll printf ; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %dh # 1-byte Reload +; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %dl # 1-byte Reload ; CHECK-NEXT: testb %bl, %bl ; CHECK-NEXT: movl %esi, %ecx ; CHECK-NEXT: # implicit-def: $eax -; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %dl # 1-byte Reload -; CHECK-NEXT: movb %dl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill -; CHECK-NEXT: movb %dh, %dl -; CHECK-NEXT: je .LBB0_6 -; CHECK-NEXT: jmp .LBB0_18 +; CHECK-NEXT: movb %dh, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill +; CHECK-NEXT: jne .LBB0_15 ; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: .LBB0_3: # %if.then -; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 -; CHECK-NEXT: movl $.str, (%esp) -; CHECK-NEXT: calll printf -; CHECK-NEXT: # implicit-def: $eax -; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload -; CHECK-NEXT: .LBB0_6: # %for.cond35 +; CHECK-NEXT: # %bb.5: # %for.cond35 ; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 ; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: je .LBB0_7 -; CHECK-NEXT: .LBB0_11: # %af +; CHECK-NEXT: je .LBB0_6 +; CHECK-NEXT: .LBB0_10: # %af ; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 ; CHECK-NEXT: testb %bl, %bl -; CHECK-NEXT: jne .LBB0_12 -; CHECK-NEXT: .LBB0_19: # %if.end39 +; CHECK-NEXT: jne .LBB0_11 +; CHECK-NEXT: .LBB0_16: # %if.end39 ; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 ; CHECK-NEXT: testl %eax, %eax -; CHECK-NEXT: je .LBB0_21 -; CHECK-NEXT: # %bb.20: # %if.then41 +; CHECK-NEXT: je .LBB0_18 +; CHECK-NEXT: # %bb.17: # %if.then41 ; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 ; CHECK-NEXT: movl $0, {{[0-9]+}}(%esp) ; CHECK-NEXT: movl $fn, {{[0-9]+}}(%esp) ; CHECK-NEXT: movl $.str, (%esp) ; CHECK-NEXT: calll printf -; CHECK-NEXT: .LBB0_21: # %for.end46 +; CHECK-NEXT: .LBB0_18: # %for.end46 ; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 -; CHECK-NEXT: # implicit-def: $al ; CHECK-NEXT: # implicit-def: $dh +; CHECK-NEXT: # implicit-def: $dl ; CHECK-NEXT: # implicit-def: $ebp -; CHECK-NEXT: jmp .LBB0_22 +; CHECK-NEXT: jmp .LBB0_19 ; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: .LBB0_8: # %if.end21 +; CHECK-NEXT: .LBB0_7: # %if.end21 ; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 ; CHECK-NEXT: # implicit-def: $ebp -; CHECK-NEXT: jmp .LBB0_9 +; CHECK-NEXT: jmp .LBB0_8 ; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: .LBB0_7: # in Loop: Header=BB0_1 Depth=1 +; CHECK-NEXT: .LBB0_6: # in Loop: Header=BB0_1 Depth=1 ; CHECK-NEXT: xorl %edi, %edi -; CHECK-NEXT: movb %dl, %dh -; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload +; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %dh # 1-byte Reload ; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: .LBB0_22: # %for.cond47 +; CHECK-NEXT: .LBB0_19: # %for.cond47 ; CHECK-NEXT: # Parent Loop BB0_1 Depth=1 ; CHECK-NEXT: # => This Inner Loop Header: Depth=2 ; CHECK-NEXT: testb %bl, %bl -; CHECK-NEXT: jne .LBB0_22 -; CHECK-NEXT: # %bb.23: # %for.cond47 -; CHECK-NEXT: # in Loop: Header=BB0_22 Depth=2 -; CHECK-NEXT: jne .LBB0_22 -; CHECK-NEXT: .LBB0_9: # %ae +; CHECK-NEXT: jne .LBB0_19 +; CHECK-NEXT: .LBB0_8: # %ae ; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 ; CHECK-NEXT: testb %bl, %bl -; CHECK-NEXT: jne .LBB0_10 -; CHECK-NEXT: # %bb.13: # %if.end26 +; CHECK-NEXT: jne .LBB0_9 +; CHECK-NEXT: # %bb.12: # %if.end26 ; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 ; CHECK-NEXT: xorl %ecx, %ecx -; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: je .LBB0_14 -; CHECK-NEXT: # %bb.15: # %if.end26 +; CHECK-NEXT: testb %dh, %dh +; CHECK-NEXT: je .LBB0_15 +; CHECK-NEXT: # %bb.13: # %if.end26 ; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 ; CHECK-NEXT: testl %ebp, %ebp -; CHECK-NEXT: jne .LBB0_16 -; CHECK-NEXT: # %bb.17: # %if.then31 +; CHECK-NEXT: jne .LBB0_15 +; CHECK-NEXT: # %bb.14: # %if.then31 ; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 ; CHECK-NEXT: xorl %ecx, %ecx -; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill ; CHECK-NEXT: xorl %ebp, %ebp -; CHECK-NEXT: .LBB0_18: # %for.inc -; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1 -; CHECK-NEXT: movb %dh, %al -; CHECK-NEXT: jmp .LBB0_1 +; CHECK-NEXT: jmp .LBB0_15 ; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: .LBB0_10: # in Loop: Header=BB0_1 Depth=1 +; CHECK-NEXT: .LBB0_9: # in Loop: Header=BB0_1 Depth=1 ; CHECK-NEXT: # implicit-def: $eax ; CHECK-NEXT: testb %bl, %bl -; CHECK-NEXT: je .LBB0_19 -; CHECK-NEXT: .LBB0_12: # in Loop: Header=BB0_1 Depth=1 +; CHECK-NEXT: je .LBB0_16 +; CHECK-NEXT: .LBB0_11: # in Loop: Header=BB0_1 Depth=1 ; CHECK-NEXT: # implicit-def: $edi ; CHECK-NEXT: # implicit-def: $cl ; CHECK-NEXT: # kill: killed $cl ; CHECK-NEXT: # implicit-def: $dl ; CHECK-NEXT: # implicit-def: $ebp ; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: jne .LBB0_11 -; CHECK-NEXT: jmp .LBB0_7 -; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: .LBB0_14: # in Loop: Header=BB0_1 Depth=1 -; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill -; CHECK-NEXT: movb %dh, %al -; CHECK-NEXT: jmp .LBB0_1 +; CHECK-NEXT: jne .LBB0_10 +; CHECK-NEXT: jmp .LBB0_6 entry: br label %for.cond diff --git a/llvm/test/CodeGen/X86/setcc-logic.ll b/llvm/test/CodeGen/X86/setcc-logic.ll index 3faa493ebccd..c98aae7fbf40 100644 --- a/llvm/test/CodeGen/X86/setcc-logic.ll +++ b/llvm/test/CodeGen/X86/setcc-logic.ll @@ -132,15 +132,12 @@ return: define i32 @all_sign_bits_clear_branch(i32 %P, i32 %Q) nounwind { ; CHECK-LABEL: all_sign_bits_clear_branch: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: js .LBB9_3 -; CHECK-NEXT: # %bb.1: # %entry -; CHECK-NEXT: testl %esi, %esi -; CHECK-NEXT: js .LBB9_3 -; CHECK-NEXT: # %bb.2: # %bb1 +; CHECK-NEXT: orl %esi, %edi +; CHECK-NEXT: js .LBB9_2 +; CHECK-NEXT: # %bb.1: # %bb1 ; CHECK-NEXT: movl $4, %eax ; CHECK-NEXT: retq -; CHECK-NEXT: .LBB9_3: # %return +; CHECK-NEXT: .LBB9_2: # %return ; CHECK-NEXT: movl $192, %eax ; CHECK-NEXT: retq entry: @@ -159,15 +156,13 @@ return: define i32 @all_bits_set_branch(i32 %P, i32 %Q) nounwind { ; CHECK-LABEL: all_bits_set_branch: ; CHECK: # %bb.0: # %entry +; CHECK-NEXT: andl %esi, %edi ; CHECK-NEXT: cmpl $-1, %edi -; CHECK-NEXT: jne .LBB10_3 -; CHECK-NEXT: # %bb.1: # %entry -; CHECK-NEXT: cmpl $-1, %esi -; CHECK-NEXT: jne .LBB10_3 -; CHECK-NEXT: # %bb.2: # %bb1 +; CHECK-NEXT: jne .LBB10_2 +; CHECK-NEXT: # %bb.1: # %bb1 ; CHECK-NEXT: movl $4, %eax ; CHECK-NEXT: retq -; CHECK-NEXT: .LBB10_3: # %return +; CHECK-NEXT: .LBB10_2: # %return ; CHECK-NEXT: movl $192, %eax ; CHECK-NEXT: retq entry: @@ -186,15 +181,12 @@ return: define i32 @all_sign_bits_set_branch(i32 %P, i32 %Q) nounwind { ; CHECK-LABEL: all_sign_bits_set_branch: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: jns .LBB11_3 -; CHECK-NEXT: # %bb.1: # %entry -; CHECK-NEXT: testl %esi, %esi -; CHECK-NEXT: jns .LBB11_3 -; CHECK-NEXT: # %bb.2: # %bb1 +; CHECK-NEXT: testl %esi, %edi +; CHECK-NEXT: jns .LBB11_2 +; CHECK-NEXT: # %bb.1: # %bb1 ; CHECK-NEXT: movl $4, %eax ; CHECK-NEXT: retq -; CHECK-NEXT: .LBB11_3: # %return +; CHECK-NEXT: .LBB11_2: # %return ; CHECK-NEXT: movl $192, %eax ; CHECK-NEXT: retq entry: @@ -238,17 +230,14 @@ return: define i32 @any_sign_bits_set_branch(i32 %P, i32 %Q) nounwind { ; CHECK-LABEL: any_sign_bits_set_branch: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: js .LBB13_2 -; CHECK-NEXT: # %bb.1: # %entry -; CHECK-NEXT: testl %esi, %esi -; CHECK-NEXT: js .LBB13_2 -; CHECK-NEXT: # %bb.3: # %return -; CHECK-NEXT: movl $192, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB13_2: # %bb1 +; CHECK-NEXT: orl %esi, %edi +; CHECK-NEXT: jns .LBB13_2 +; CHECK-NEXT: # %bb.1: # %bb1 ; CHECK-NEXT: movl $4, %eax ; CHECK-NEXT: retq +; CHECK-NEXT: .LBB13_2: # %return +; CHECK-NEXT: movl $192, %eax +; CHECK-NEXT: retq entry: %a = icmp slt i32 %P, 0 %b = icmp slt i32 %Q, 0 @@ -265,17 +254,15 @@ return: define i32 @any_bits_clear_branch(i32 %P, i32 %Q) nounwind { ; CHECK-LABEL: any_bits_clear_branch: ; CHECK: # %bb.0: # %entry +; CHECK-NEXT: andl %esi, %edi ; CHECK-NEXT: cmpl $-1, %edi -; CHECK-NEXT: jne .LBB14_2 -; CHECK-NEXT: # %bb.1: # %entry -; CHECK-NEXT: cmpl $-1, %esi -; CHECK-NEXT: jne .LBB14_2 -; CHECK-NEXT: # %bb.3: # %return -; CHECK-NEXT: movl $192, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB14_2: # %bb1 +; CHECK-NEXT: je .LBB14_2 +; CHECK-NEXT: # %bb.1: # %bb1 ; CHECK-NEXT: movl $4, %eax ; CHECK-NEXT: retq +; CHECK-NEXT: .LBB14_2: # %return +; CHECK-NEXT: movl $192, %eax +; CHECK-NEXT: retq entry: %a = icmp ne i32 %P, -1 %b = icmp ne i32 %Q, -1 @@ -292,17 +279,14 @@ return: define i32 @any_sign_bits_clear_branch(i32 %P, i32 %Q) nounwind { ; CHECK-LABEL: any_sign_bits_clear_branch: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: testl %edi, %edi -; CHECK-NEXT: jns .LBB15_2 -; CHECK-NEXT: # %bb.1: # %entry -; CHECK-NEXT: testl %esi, %esi -; CHECK-NEXT: jns .LBB15_2 -; CHECK-NEXT: # %bb.3: # %return -; CHECK-NEXT: movl $192, %eax -; CHECK-NEXT: retq -; CHECK-NEXT: .LBB15_2: # %bb1 +; CHECK-NEXT: testl %esi, %edi +; CHECK-NEXT: js .LBB15_2 +; CHECK-NEXT: # %bb.1: # %bb1 ; CHECK-NEXT: movl $4, %eax ; CHECK-NEXT: retq +; CHECK-NEXT: .LBB15_2: # %return +; CHECK-NEXT: movl $192, %eax +; CHECK-NEXT: retq entry: %a = icmp sgt i32 %P, -1 %b = icmp sgt i32 %Q, -1 diff --git a/llvm/test/CodeGen/X86/swifterror.ll b/llvm/test/CodeGen/X86/swifterror.ll index 75252309790b..1489b0295e93 100644 --- a/llvm/test/CodeGen/X86/swifterror.ll +++ b/llvm/test/CodeGen/X86/swifterror.ll @@ -1259,12 +1259,7 @@ entry: define swiftcc void @dont_crash_on_new_isel_blocks(ptr nocapture swifterror, i1, ptr) { ; CHECK-APPLE-LABEL: dont_crash_on_new_isel_blocks: ; CHECK-APPLE: ## %bb.0: ## %entry -; CHECK-APPLE-NEXT: xorl %eax, %eax -; CHECK-APPLE-NEXT: testb %al, %al -; CHECK-APPLE-NEXT: jne LBB15_2 -; CHECK-APPLE-NEXT: ## %bb.1: ## %entry ; CHECK-APPLE-NEXT: testb $1, %dil -; CHECK-APPLE-NEXT: LBB15_2: ## %cont ; CHECK-APPLE-NEXT: pushq %rax ; CHECK-APPLE-NEXT: .cfi_def_cfa_offset 16 ; CHECK-APPLE-NEXT: callq *%rax @@ -1290,12 +1285,7 @@ define swiftcc void @dont_crash_on_new_isel_blocks(ptr nocapture swifterror, i1, ; ; CHECK-i386-LABEL: dont_crash_on_new_isel_blocks: ; CHECK-i386: ## %bb.0: ## %entry -; CHECK-i386-NEXT: xorl %eax, %eax -; CHECK-i386-NEXT: testb %al, %al -; CHECK-i386-NEXT: jne LBB15_2 -; CHECK-i386-NEXT: ## %bb.1: ## %entry ; CHECK-i386-NEXT: testb $1, 8(%esp) -; CHECK-i386-NEXT: LBB15_2: ## %cont ; CHECK-i386-NEXT: jmpl *%eax ## TAILCALL entry: %3 = or i1 false, %1 diff --git a/llvm/test/CodeGen/X86/tail-dup-merge-loop-headers.ll b/llvm/test/CodeGen/X86/tail-dup-merge-loop-headers.ll index 9cd373151812..8d84e887d3f2 100644 --- a/llvm/test/CodeGen/X86/tail-dup-merge-loop-headers.ll +++ b/llvm/test/CodeGen/X86/tail-dup-merge-loop-headers.ll @@ -91,116 +91,97 @@ define i32 @loop_shared_header(ptr %exe, i32 %exesz, i32 %headsize, i32 %min, i3 ; CHECK-NEXT: pushq %rbp ; CHECK-NEXT: pushq %r15 ; CHECK-NEXT: pushq %r14 -; CHECK-NEXT: pushq %r13 ; CHECK-NEXT: pushq %r12 ; CHECK-NEXT: pushq %rbx -; CHECK-NEXT: pushq %rax ; CHECK-NEXT: movl $1, %ebx ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: jne .LBB1_24 +; CHECK-NEXT: jne .LBB1_12 ; CHECK-NEXT: # %bb.1: # %if.end19 -; CHECK-NEXT: movl %esi, %ebp -; CHECK-NEXT: movq %rdi, %r15 -; CHECK-NEXT: movl (%rax), %r13d -; CHECK-NEXT: leal (,%r13,4), %ebx -; CHECK-NEXT: movl %ebx, %r12d +; CHECK-NEXT: movl (%rax), %r12d +; CHECK-NEXT: leal (,%r12,4), %ebp +; CHECK-NEXT: movl %ebp, %r15d ; CHECK-NEXT: movl $1, %esi -; CHECK-NEXT: movq %r12, %rdi +; CHECK-NEXT: movq %r15, %rdi ; CHECK-NEXT: callq cli_calloc@PLT -; CHECK-NEXT: testl %ebp, %ebp -; CHECK-NEXT: je .LBB1_23 -; CHECK-NEXT: # %bb.2: # %if.end19 -; CHECK-NEXT: testl %r13d, %r13d -; CHECK-NEXT: je .LBB1_23 -; CHECK-NEXT: # %bb.3: # %if.end19 ; CHECK-NEXT: movq %rax, %r14 -; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movb $1, %al ; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: jne .LBB1_23 -; CHECK-NEXT: # %bb.4: # %if.end19 -; CHECK-NEXT: cmpq %r15, %r14 -; CHECK-NEXT: jb .LBB1_23 -; CHECK-NEXT: # %bb.5: # %if.end50 +; CHECK-NEXT: jne .LBB1_12 +; CHECK-NEXT: # %bb.2: # %if.end50 ; CHECK-NEXT: movq %r14, %rdi -; CHECK-NEXT: movq %r12, %rdx +; CHECK-NEXT: movq %r15, %rdx ; CHECK-NEXT: callq memcpy@PLT -; CHECK-NEXT: cmpl $4, %ebx -; CHECK-NEXT: jb .LBB1_26 -; CHECK-NEXT: # %bb.6: # %shared_preheader +; CHECK-NEXT: cmpl $4, %ebp +; CHECK-NEXT: jb .LBB1_19 +; CHECK-NEXT: # %bb.3: # %shared_preheader ; CHECK-NEXT: movb $32, %cl ; CHECK-NEXT: xorl %eax, %eax -; CHECK-NEXT: jmp .LBB1_8 +; CHECK-NEXT: jmp .LBB1_4 ; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: .LBB1_7: # %merge_predecessor_split -; CHECK-NEXT: # in Loop: Header=BB1_8 Depth=1 +; CHECK-NEXT: .LBB1_15: # %merge_predecessor_split +; CHECK-NEXT: # in Loop: Header=BB1_4 Depth=1 ; CHECK-NEXT: movb $32, %cl -; CHECK-NEXT: .LBB1_8: # %outer_loop_header +; CHECK-NEXT: .LBB1_4: # %outer_loop_header ; CHECK-NEXT: # =>This Loop Header: Depth=1 -; CHECK-NEXT: # Child Loop BB1_9 Depth 2 -; CHECK-NEXT: testl %r13d, %r13d -; CHECK-NEXT: je .LBB1_16 +; CHECK-NEXT: # Child Loop BB1_8 Depth 2 +; CHECK-NEXT: testl %r12d, %r12d +; CHECK-NEXT: je .LBB1_5 ; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: .LBB1_9: # %shared_loop_header -; CHECK-NEXT: # Parent Loop BB1_8 Depth=1 +; CHECK-NEXT: .LBB1_8: # %shared_loop_header +; CHECK-NEXT: # Parent Loop BB1_4 Depth=1 ; CHECK-NEXT: # => This Inner Loop Header: Depth=2 ; CHECK-NEXT: testq %r14, %r14 -; CHECK-NEXT: jne .LBB1_25 -; CHECK-NEXT: # %bb.10: # %inner_loop_body -; CHECK-NEXT: # in Loop: Header=BB1_9 Depth=2 +; CHECK-NEXT: jne .LBB1_18 +; CHECK-NEXT: # %bb.9: # %inner_loop_body +; CHECK-NEXT: # in Loop: Header=BB1_8 Depth=2 ; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: je .LBB1_9 -; CHECK-NEXT: # %bb.11: # %if.end96.i -; CHECK-NEXT: # in Loop: Header=BB1_8 Depth=1 -; CHECK-NEXT: cmpl $3, %r13d -; CHECK-NEXT: jae .LBB1_20 -; CHECK-NEXT: # %bb.12: # %if.end287.i -; CHECK-NEXT: # in Loop: Header=BB1_8 Depth=1 +; CHECK-NEXT: je .LBB1_8 +; CHECK-NEXT: # %bb.10: # %if.end96.i +; CHECK-NEXT: # in Loop: Header=BB1_4 Depth=1 +; CHECK-NEXT: cmpl $3, %r12d +; CHECK-NEXT: jae .LBB1_11 +; CHECK-NEXT: # %bb.13: # %if.end287.i +; CHECK-NEXT: # in Loop: Header=BB1_4 Depth=1 ; CHECK-NEXT: testb %al, %al ; CHECK-NEXT: # implicit-def: $cl -; CHECK-NEXT: jne .LBB1_8 -; CHECK-NEXT: # %bb.13: # %if.end308.i -; CHECK-NEXT: # in Loop: Header=BB1_8 Depth=1 +; CHECK-NEXT: jne .LBB1_4 +; CHECK-NEXT: # %bb.14: # %if.end308.i +; CHECK-NEXT: # in Loop: Header=BB1_4 Depth=1 ; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: je .LBB1_7 -; CHECK-NEXT: # %bb.14: # %if.end335.i -; CHECK-NEXT: # in Loop: Header=BB1_8 Depth=1 +; CHECK-NEXT: je .LBB1_15 +; CHECK-NEXT: # %bb.16: # %if.end335.i +; CHECK-NEXT: # in Loop: Header=BB1_4 Depth=1 ; CHECK-NEXT: xorl %ecx, %ecx ; CHECK-NEXT: testb %cl, %cl -; CHECK-NEXT: jne .LBB1_8 -; CHECK-NEXT: # %bb.15: # %merge_other -; CHECK-NEXT: # in Loop: Header=BB1_8 Depth=1 +; CHECK-NEXT: jne .LBB1_4 +; CHECK-NEXT: # %bb.17: # %merge_other +; CHECK-NEXT: # in Loop: Header=BB1_4 Depth=1 ; CHECK-NEXT: # implicit-def: $cl -; CHECK-NEXT: jmp .LBB1_8 -; CHECK-NEXT: .LBB1_23: -; CHECK-NEXT: movl $1, %ebx -; CHECK-NEXT: jmp .LBB1_24 -; CHECK-NEXT: .LBB1_16: # %while.cond.us1412.i +; CHECK-NEXT: jmp .LBB1_4 +; CHECK-NEXT: .LBB1_5: # %while.cond.us1412.i ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: movl $1, %ebx -; CHECK-NEXT: jne .LBB1_18 -; CHECK-NEXT: # %bb.17: # %while.cond.us1412.i +; CHECK-NEXT: jne .LBB1_7 +; CHECK-NEXT: # %bb.6: # %while.cond.us1412.i ; CHECK-NEXT: decb %cl -; CHECK-NEXT: jne .LBB1_24 -; CHECK-NEXT: .LBB1_18: # %if.end41.us1436.i -; CHECK-NEXT: .LBB1_20: # %if.then99.i +; CHECK-NEXT: jne .LBB1_12 +; CHECK-NEXT: .LBB1_7: # %if.end41.us1436.i +; CHECK-NEXT: .LBB1_11: # %if.then99.i ; CHECK-NEXT: movq .str.6@GOTPCREL(%rip), %rdi ; CHECK-NEXT: xorl %ebx, %ebx ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: callq cli_dbgmsg@PLT -; CHECK-NEXT: .LBB1_24: # %cleanup +; CHECK-NEXT: .LBB1_12: # %cleanup ; CHECK-NEXT: movl %ebx, %eax -; CHECK-NEXT: addq $8, %rsp ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: popq %r12 -; CHECK-NEXT: popq %r13 ; CHECK-NEXT: popq %r14 ; CHECK-NEXT: popq %r15 ; CHECK-NEXT: popq %rbp ; CHECK-NEXT: retq -; CHECK-NEXT: .LBB1_25: # %wunpsect.exit.thread.loopexit389 -; CHECK-NEXT: .LBB1_26: # %wunpsect.exit.thread.loopexit391 +; CHECK-NEXT: .LBB1_18: # %wunpsect.exit.thread.loopexit389 +; CHECK-NEXT: .LBB1_19: # %wunpsect.exit.thread.loopexit391 entry: %0 = load i32, ptr undef, align 4 %mul = shl nsw i32 %0, 2 diff --git a/llvm/test/CodeGen/X86/tail-opts.ll b/llvm/test/CodeGen/X86/tail-opts.ll index d54110d1fa81..d9ab2f7d1f5f 100644 --- a/llvm/test/CodeGen/X86/tail-opts.ll +++ b/llvm/test/CodeGen/X86/tail-opts.ll @@ -300,10 +300,9 @@ define fastcc void @c_expand_expr_stmt(ptr %expr) nounwind { ; CHECK-NEXT: cmpl $23, %ecx ; CHECK-NEXT: jne .LBB3_9 ; CHECK-NEXT: .LBB3_16: # %lvalue_p.exit4 -; CHECK-NEXT: testb %al, %al -; CHECK-NEXT: jne .LBB3_9 -; CHECK-NEXT: # %bb.17: # %lvalue_p.exit4 ; CHECK-NEXT: testb %bl, %bl +; CHECK-NEXT: sete %cl +; CHECK-NEXT: orb %al, %cl entry: %tmp4 = load i8, ptr null, align 8 ; [#uses=3] switch i8 %tmp4, label %bb3 [ diff --git a/llvm/test/CodeGen/X86/test-shrink-bug.ll b/llvm/test/CodeGen/X86/test-shrink-bug.ll index ed43cabbdaee..953a0d65c538 100644 --- a/llvm/test/CodeGen/X86/test-shrink-bug.ll +++ b/llvm/test/CodeGen/X86/test-shrink-bug.ll @@ -48,37 +48,39 @@ define dso_local void @fail(i16 %a, <2 x i8> %b) { ; CHECK-X86: ## %bb.0: ; CHECK-X86-NEXT: subl $12, %esp ; CHECK-X86-NEXT: .cfi_def_cfa_offset 16 -; CHECK-X86-NEXT: movzwl {{[0-9]+}}(%esp), %ecx +; CHECK-X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax ; CHECK-X86-NEXT: cmpb $123, {{[0-9]+}}(%esp) -; CHECK-X86-NEXT: sete %al -; CHECK-X86-NEXT: testl $263, %ecx ## imm = 0x107 -; CHECK-X86-NEXT: je LBB1_3 -; CHECK-X86-NEXT: ## %bb.1: -; CHECK-X86-NEXT: testb %al, %al -; CHECK-X86-NEXT: jne LBB1_3 -; CHECK-X86-NEXT: ## %bb.2: ## %no +; CHECK-X86-NEXT: setne %cl +; CHECK-X86-NEXT: testl $263, %eax ## imm = 0x107 +; CHECK-X86-NEXT: setne %al +; CHECK-X86-NEXT: testb %cl, %al +; CHECK-X86-NEXT: jne LBB1_2 +; CHECK-X86-NEXT: ## %bb.1: ## %yes +; CHECK-X86-NEXT: addl $12, %esp +; CHECK-X86-NEXT: retl +; CHECK-X86-NEXT: LBB1_2: ## %no ; CHECK-X86-NEXT: calll _bar -; CHECK-X86-NEXT: LBB1_3: ## %yes ; CHECK-X86-NEXT: addl $12, %esp ; CHECK-X86-NEXT: retl ; ; CHECK-X64-LABEL: fail: ; CHECK-X64: # %bb.0: -; CHECK-X64-NEXT: testl $263, %edi # imm = 0x107 -; CHECK-X64-NEXT: je .LBB1_3 -; CHECK-X64-NEXT: # %bb.1: ; CHECK-X64-NEXT: pslld $8, %xmm0 ; CHECK-X64-NEXT: pcmpeqb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; CHECK-X64-NEXT: pextrw $1, %xmm0, %eax -; CHECK-X64-NEXT: testb $1, %al -; CHECK-X64-NEXT: jne .LBB1_3 -; CHECK-X64-NEXT: # %bb.2: # %no +; CHECK-X64-NEXT: xorb $1, %al +; CHECK-X64-NEXT: testl $263, %edi # imm = 0x107 +; CHECK-X64-NEXT: setne %cl +; CHECK-X64-NEXT: testb %al, %cl +; CHECK-X64-NEXT: jne .LBB1_2 +; CHECK-X64-NEXT: # %bb.1: # %yes +; CHECK-X64-NEXT: retq +; CHECK-X64-NEXT: .LBB1_2: # %no ; CHECK-X64-NEXT: pushq %rax ; CHECK-X64-NEXT: .cfi_def_cfa_offset 16 ; CHECK-X64-NEXT: callq bar@PLT ; CHECK-X64-NEXT: popq %rax ; CHECK-X64-NEXT: .cfi_def_cfa_offset 8 -; CHECK-X64-NEXT: .LBB1_3: # %yes ; CHECK-X64-NEXT: retq %1 = icmp eq <2 x i8> %b, %2 = extractelement <2 x i1> %1, i32 1 diff --git a/llvm/test/CodeGen/X86/x86-shrink-wrap-unwind.ll b/llvm/test/CodeGen/X86/x86-shrink-wrap-unwind.ll index b9e490888d9b..3349d31cad4b 100644 --- a/llvm/test/CodeGen/X86/x86-shrink-wrap-unwind.ll +++ b/llvm/test/CodeGen/X86/x86-shrink-wrap-unwind.ll @@ -181,38 +181,40 @@ define zeroext i1 @segmentedStack(ptr readonly %vk1, ptr readonly %vk2, i64 %key ; CHECK-LABEL: segmentedStack: ; CHECK: ## %bb.0: ; CHECK-NEXT: cmpq %gs:816, %rsp -; CHECK-NEXT: jbe LBB3_7 +; CHECK-NEXT: jbe LBB3_6 ; CHECK-NEXT: LBB3_1: ## %entry ; CHECK-NEXT: pushq %rax ; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: testq %rdi, %rdi +; CHECK-NEXT: sete %al +; CHECK-NEXT: testq %rsi, %rsi +; CHECK-NEXT: sete %cl +; CHECK-NEXT: orb %al, %cl ; CHECK-NEXT: movq %rdi, %rax ; CHECK-NEXT: orq %rsi, %rax ; CHECK-NEXT: sete %al -; CHECK-NEXT: testq %rdi, %rdi -; CHECK-NEXT: je LBB3_5 -; CHECK-NEXT: ## %bb.2: ## %entry -; CHECK-NEXT: testq %rsi, %rsi -; CHECK-NEXT: je LBB3_5 -; CHECK-NEXT: ## %bb.3: ## %if.end4.i +; CHECK-NEXT: testb %cl, %cl +; CHECK-NEXT: jne LBB3_4 +; CHECK-NEXT: ## %bb.2: ## %if.end4.i ; CHECK-NEXT: movq 8(%rdi), %rdx ; CHECK-NEXT: cmpq 8(%rsi), %rdx -; CHECK-NEXT: jne LBB3_6 -; CHECK-NEXT: ## %bb.4: ## %land.rhs.i.i +; CHECK-NEXT: jne LBB3_5 +; CHECK-NEXT: ## %bb.3: ## %land.rhs.i.i ; CHECK-NEXT: movq (%rsi), %rsi ; CHECK-NEXT: movq (%rdi), %rdi ; CHECK-NEXT: callq _memcmp ; CHECK-NEXT: testl %eax, %eax ; CHECK-NEXT: sete %al -; CHECK-NEXT: LBB3_5: ## %__go_ptr_strings_equal.exit +; CHECK-NEXT: LBB3_4: ## %__go_ptr_strings_equal.exit ; CHECK-NEXT: ## kill: def $al killed $al killed $eax ; CHECK-NEXT: popq %rcx ; CHECK-NEXT: retq -; CHECK-NEXT: LBB3_6: +; CHECK-NEXT: LBB3_5: ; CHECK-NEXT: xorl %eax, %eax ; CHECK-NEXT: ## kill: def $al killed $al killed $eax ; CHECK-NEXT: popq %rcx ; CHECK-NEXT: retq -; CHECK-NEXT: LBB3_7: +; CHECK-NEXT: LBB3_6: ; CHECK-NEXT: movl $8, %r10d ; CHECK-NEXT: movl $0, %r11d ; CHECK-NEXT: callq ___morestack @@ -222,41 +224,43 @@ define zeroext i1 @segmentedStack(ptr readonly %vk1, ptr readonly %vk2, i64 %key ; NOCOMPACTUNWIND-LABEL: segmentedStack: ; NOCOMPACTUNWIND: # %bb.0: ; NOCOMPACTUNWIND-NEXT: cmpq %fs:112, %rsp -; NOCOMPACTUNWIND-NEXT: jbe .LBB3_7 +; NOCOMPACTUNWIND-NEXT: jbe .LBB3_6 ; NOCOMPACTUNWIND-NEXT: .LBB3_1: # %entry ; NOCOMPACTUNWIND-NEXT: pushq %rax ; NOCOMPACTUNWIND-NEXT: .cfi_def_cfa_offset 16 +; NOCOMPACTUNWIND-NEXT: testq %rdi, %rdi +; NOCOMPACTUNWIND-NEXT: sete %al +; NOCOMPACTUNWIND-NEXT: testq %rsi, %rsi +; NOCOMPACTUNWIND-NEXT: sete %cl +; NOCOMPACTUNWIND-NEXT: orb %al, %cl ; NOCOMPACTUNWIND-NEXT: movq %rdi, %rax ; NOCOMPACTUNWIND-NEXT: orq %rsi, %rax ; NOCOMPACTUNWIND-NEXT: sete %al -; NOCOMPACTUNWIND-NEXT: testq %rdi, %rdi -; NOCOMPACTUNWIND-NEXT: je .LBB3_5 -; NOCOMPACTUNWIND-NEXT: # %bb.2: # %entry -; NOCOMPACTUNWIND-NEXT: testq %rsi, %rsi -; NOCOMPACTUNWIND-NEXT: je .LBB3_5 -; NOCOMPACTUNWIND-NEXT: # %bb.3: # %if.end4.i +; NOCOMPACTUNWIND-NEXT: testb %cl, %cl +; NOCOMPACTUNWIND-NEXT: jne .LBB3_4 +; NOCOMPACTUNWIND-NEXT: # %bb.2: # %if.end4.i ; NOCOMPACTUNWIND-NEXT: movq 8(%rdi), %rdx ; NOCOMPACTUNWIND-NEXT: cmpq 8(%rsi), %rdx -; NOCOMPACTUNWIND-NEXT: jne .LBB3_6 -; NOCOMPACTUNWIND-NEXT: # %bb.4: # %land.rhs.i.i +; NOCOMPACTUNWIND-NEXT: jne .LBB3_5 +; NOCOMPACTUNWIND-NEXT: # %bb.3: # %land.rhs.i.i ; NOCOMPACTUNWIND-NEXT: movq (%rsi), %rsi ; NOCOMPACTUNWIND-NEXT: movq (%rdi), %rdi ; NOCOMPACTUNWIND-NEXT: callq memcmp@PLT ; NOCOMPACTUNWIND-NEXT: testl %eax, %eax ; NOCOMPACTUNWIND-NEXT: sete %al -; NOCOMPACTUNWIND-NEXT: .LBB3_5: # %__go_ptr_strings_equal.exit +; NOCOMPACTUNWIND-NEXT: .LBB3_4: # %__go_ptr_strings_equal.exit ; NOCOMPACTUNWIND-NEXT: # kill: def $al killed $al killed $eax ; NOCOMPACTUNWIND-NEXT: popq %rcx ; NOCOMPACTUNWIND-NEXT: .cfi_def_cfa_offset 8 ; NOCOMPACTUNWIND-NEXT: retq -; NOCOMPACTUNWIND-NEXT: .LBB3_6: +; NOCOMPACTUNWIND-NEXT: .LBB3_5: ; NOCOMPACTUNWIND-NEXT: .cfi_def_cfa_offset 16 ; NOCOMPACTUNWIND-NEXT: xorl %eax, %eax ; NOCOMPACTUNWIND-NEXT: # kill: def $al killed $al killed $eax ; NOCOMPACTUNWIND-NEXT: popq %rcx ; NOCOMPACTUNWIND-NEXT: .cfi_def_cfa_offset 8 ; NOCOMPACTUNWIND-NEXT: retq -; NOCOMPACTUNWIND-NEXT: .LBB3_7: +; NOCOMPACTUNWIND-NEXT: .LBB3_6: ; NOCOMPACTUNWIND-NEXT: movl $8, %r10d ; NOCOMPACTUNWIND-NEXT: movl $0, %r11d ; NOCOMPACTUNWIND-NEXT: callq __morestack -- GitLab From df5252165524fbcabbd7287a8d923b5139247114 Mon Sep 17 00:00:00 2001 From: Jeremy Morse Date: Mon, 4 Mar 2024 19:27:36 +0000 Subject: [PATCH 033/929] [RemoveDIs] Follow up to 6b62a9135a2, fix a ICmpInst constructor call Seemingly I either missed this on a buildbot or otherwise didn't cover this :( --- llvm/examples/ParallelJIT/ParallelJIT.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/examples/ParallelJIT/ParallelJIT.cpp b/llvm/examples/ParallelJIT/ParallelJIT.cpp index 7da105d201b9..c736f170e877 100644 --- a/llvm/examples/ParallelJIT/ParallelJIT.cpp +++ b/llvm/examples/ParallelJIT/ParallelJIT.cpp @@ -104,7 +104,7 @@ static Function *CreateFibFunction(Module *M) { BasicBlock *RecurseBB = BasicBlock::Create(Context, "recurse", FibF); // Create the "if (arg < 2) goto exitbb" - Value *CondInst = new ICmpInst(*BB, ICmpInst::ICMP_SLE, ArgX, Two, "cond"); + Value *CondInst = new ICmpInst(BB, ICmpInst::ICMP_SLE, ArgX, Two, "cond"); BranchInst::Create(RetBB, RecurseBB, CondInst, BB); // Create: ret int 1 -- GitLab From f32c6b28b84130dc77e0e69d6d3b692aec933280 Mon Sep 17 00:00:00 2001 From: Med Ismail Bennani Date: Mon, 4 Mar 2024 11:42:04 -0800 Subject: [PATCH 034/929] [lldb/Test] Fix oversight in ProcessEventDataTest::SetUp (NFC) (#83895) This patch addresses an oversight in `ProcessEventDataTest::SetUp` unittest to ensure the Debugger is initialized properly. Signed-off-by: Med Ismail Bennani --- lldb/unittests/Process/ProcessEventDataTest.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lldb/unittests/Process/ProcessEventDataTest.cpp b/lldb/unittests/Process/ProcessEventDataTest.cpp index dd159496cd88..e793c6eae20a 100644 --- a/lldb/unittests/Process/ProcessEventDataTest.cpp +++ b/lldb/unittests/Process/ProcessEventDataTest.cpp @@ -8,6 +8,7 @@ #include "Plugins/Platform/MacOSX/PlatformMacOSX.h" #include "Plugins/Platform/MacOSX/PlatformRemoteMacOSX.h" +#include "TestingSupport/TestUtilities.h" #include "lldb/Core/Debugger.h" #include "lldb/Host/FileSystem.h" #include "lldb/Host/HostInfo.h" @@ -29,6 +30,8 @@ public: FileSystem::Initialize(); HostInfo::Initialize(); PlatformMacOSX::Initialize(); + std::call_once(TestUtilities::g_debugger_initialize_flag, + []() { Debugger::Initialize(nullptr); }); } void TearDown() override { PlatformMacOSX::Terminate(); -- GitLab From abe4677d9f8ab82f722f29dbf57952052accd70b Mon Sep 17 00:00:00 2001 From: Yingwei Zheng Date: Tue, 5 Mar 2024 03:51:06 +0800 Subject: [PATCH 035/929] [InstCombine] Fix infinite loop due to incorrect `DoesConsume` (#82973) When a call to `getFreelyInvertedImpl` with a select/phi node fails, `DoesConsume` should not be changed. Fixes https://github.com/llvm/llvm-project/issues/82877. --- .../InstCombine/InstructionCombining.cpp | 11 ++++-- llvm/test/Transforms/InstCombine/pr82877.ll | 34 +++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 llvm/test/Transforms/InstCombine/pr82877.ll diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index 87c8dca7efed..80ce0c9275b2 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -2341,11 +2341,13 @@ Value *InstCombiner::getFreelyInvertedImpl(Value *V, bool WillInvertAllUses, !shouldAvoidAbsorbingNotIntoSelect(*cast(V)); // Selects/min/max with invertible operands are freely invertible if (IsSelect || match(V, m_MaxOrMin(m_Value(A), m_Value(B)))) { + bool LocalDoesConsume = DoesConsume; if (!getFreelyInvertedImpl(B, B->hasOneUse(), /*Builder*/ nullptr, - DoesConsume, Depth)) + LocalDoesConsume, Depth)) return nullptr; if (Value *NotA = getFreelyInvertedImpl(A, A->hasOneUse(), Builder, - DoesConsume, Depth)) { + LocalDoesConsume, Depth)) { + DoesConsume = LocalDoesConsume; if (Builder != nullptr) { Value *NotB = getFreelyInvertedImpl(B, B->hasOneUse(), Builder, DoesConsume, Depth); @@ -2361,12 +2363,13 @@ Value *InstCombiner::getFreelyInvertedImpl(Value *V, bool WillInvertAllUses, } if (PHINode *PN = dyn_cast(V)) { + bool LocalDoesConsume = DoesConsume; SmallVector, 8> IncomingValues; for (Use &U : PN->operands()) { BasicBlock *IncomingBlock = PN->getIncomingBlock(U); Value *NewIncomingVal = getFreelyInvertedImpl( U.get(), /*WillInvertAllUses=*/false, - /*Builder=*/nullptr, DoesConsume, MaxAnalysisRecursionDepth - 1); + /*Builder=*/nullptr, LocalDoesConsume, MaxAnalysisRecursionDepth - 1); if (NewIncomingVal == nullptr) return nullptr; // Make sure that we can safely erase the original PHI node. @@ -2375,6 +2378,8 @@ Value *InstCombiner::getFreelyInvertedImpl(Value *V, bool WillInvertAllUses, if (Builder != nullptr) IncomingValues.emplace_back(NewIncomingVal, IncomingBlock); } + + DoesConsume = LocalDoesConsume; if (Builder != nullptr) { IRBuilderBase::InsertPointGuard Guard(*Builder); Builder->SetInsertPoint(PN); diff --git a/llvm/test/Transforms/InstCombine/pr82877.ll b/llvm/test/Transforms/InstCombine/pr82877.ll new file mode 100644 index 000000000000..8594bb68b8e4 --- /dev/null +++ b/llvm/test/Transforms/InstCombine/pr82877.ll @@ -0,0 +1,34 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -passes=instcombine < %s | FileCheck %s + +define i64 @func(i32 %p, i1 %cmp1) { +; CHECK-LABEL: define i64 @func( +; CHECK-SAME: i32 [[P:%.*]], i1 [[CMP1:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[NOT:%.*]] = xor i32 [[P]], -1 +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[P0:%.*]] = phi i32 [ [[NOT]], [[ENTRY:%.*]] ], [ [[CONV:%.*]], [[FOR_BODY]] ] +; CHECK-NEXT: [[COND:%.*]] = select i1 [[CMP1]], i32 0, i32 -1231558963 +; CHECK-NEXT: [[XOR:%.*]] = xor i32 [[COND]], [[P0]] +; CHECK-NEXT: [[CMP2:%.*]] = icmp ne i32 [[XOR]], 1 +; CHECK-NEXT: [[CONV]] = zext i1 [[CMP2]] to i32 +; CHECK-NEXT: br i1 [[CMP2]], label [[FOR_BODY]], label [[FOR_EXIT:%.*]] +; CHECK: for.exit: +; CHECK-NEXT: ret i64 0 +; +entry: + %not = xor i32 %p, -1 + br label %for.body + +for.body: + %p0 = phi i32 [ %not, %entry ], [ %conv, %for.body ] + %cond = select i1 %cmp1, i32 0, i32 -1231558963 + %xor = xor i32 %cond, %p0 + %cmp2 = icmp ne i32 %xor, 1 + %conv = zext i1 %cmp2 to i32 + br i1 %cmp2, label %for.body, label %for.exit + +for.exit: + ret i64 0 +} -- GitLab From 79e8f29ab06560d159389d940c35c563e0c8640e Mon Sep 17 00:00:00 2001 From: Alexander M Date: Mon, 4 Mar 2024 20:54:24 +0100 Subject: [PATCH 036/929] [lldb/lua] Fix Lua building on Windows (#83871) Add `liblldb` dependency and use correct extension for compiled Lua module. Replace 'Python' with 'Lua' in install path name. Fixes #55075. --- lldb/bindings/lua/CMakeLists.txt | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/lldb/bindings/lua/CMakeLists.txt b/lldb/bindings/lua/CMakeLists.txt index 1a739a9805ec..2d128cc1864c 100644 --- a/lldb/bindings/lua/CMakeLists.txt +++ b/lldb/bindings/lua/CMakeLists.txt @@ -28,7 +28,7 @@ endfunction() function(finish_swig_lua swig_target lldb_lua_bindings_dir lldb_lua_target_dir) add_custom_target(${swig_target} ALL VERBATIM COMMAND ${CMAKE_COMMAND} -E make_directory ${lldb_lua_target_dir} - DEPENDS swig_wrapper_lua + DEPENDS swig_wrapper_lua liblldb COMMENT "LLDB Lua API") if(LLDB_BUILD_FRAMEWORK) set(LIBLLDB_SYMLINK_DEST "${LLDB_FRAMEWORK_ABSOLUTE_BUILD_DIR}/LLDB.framework/LLDB") @@ -36,11 +36,7 @@ function(finish_swig_lua swig_target lldb_lua_bindings_dir lldb_lua_target_dir) set(LIBLLDB_SYMLINK_DEST "${LLVM_SHLIB_OUTPUT_INTDIR}/liblldb${CMAKE_SHARED_LIBRARY_SUFFIX}") endif() if(WIN32) - if(CMAKE_BUILD_TYPE STREQUAL Debug) - set(LIBLLDB_SYMLINK_OUTPUT_FILE "_lldb_d.pyd") - else() - set(LIBLLDB_SYMLINK_OUTPUT_FILE "_lldb.pyd") - endif() + set(LIBLLDB_SYMLINK_OUTPUT_FILE "lldb.dll") else() set(LIBLLDB_SYMLINK_OUTPUT_FILE "lldb.so") endif() @@ -54,7 +50,7 @@ function(finish_swig_lua swig_target lldb_lua_bindings_dir lldb_lua_target_dir) add_dependencies(lldb ${swig_target}) if(LLDB_BUILD_FRAMEWORK) - set(LLDB_LUA_INSTALL_PATH ${LLDB_FRAMEWORK_INSTALL_DIR}/LLDB.framework/Resources/Python) + set(LLDB_LUA_INSTALL_PATH ${LLDB_FRAMEWORK_INSTALL_DIR}/LLDB.framework/Resources/Lua) else() set(LLDB_LUA_INSTALL_PATH ${LLDB_LUA_RELATIVE_PATH}) endif() -- GitLab From 0597644a6466ae9148b0b41cb8f95d5022e045c2 Mon Sep 17 00:00:00 2001 From: Congcong Cai Date: Tue, 5 Mar 2024 03:58:12 +0800 Subject: [PATCH 037/929] [mlir][transform] replace original op to loop ops (#83537) --- mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp | 1 + .../TilingInterface/lower-to-loops-using-interface.mlir | 1 + 2 files changed, 2 insertions(+) diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp index ef9cd5561665..0ac0a89dcc76 100644 --- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp +++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp @@ -2121,6 +2121,7 @@ DiagnosedSilenceableFailure transform::ConvertToLoopsOp::applyToOne( scf::lowerToLoopsUsingSCFForOp(rewriter, target); if (failed(loops)) return emitDefaultDefiniteFailure(target); + rewriter.eraseOp(target); return DiagnosedSilenceableFailure::success(); } diff --git a/mlir/test/Interfaces/TilingInterface/lower-to-loops-using-interface.mlir b/mlir/test/Interfaces/TilingInterface/lower-to-loops-using-interface.mlir index 7969de0d456b..1b2c553b25de 100644 --- a/mlir/test/Interfaces/TilingInterface/lower-to-loops-using-interface.mlir +++ b/mlir/test/Interfaces/TilingInterface/lower-to-loops-using-interface.mlir @@ -33,6 +33,7 @@ module attributes {transform.with_named_sequence} { // CHECK: %[[MULF:.+]] = arith.mulf %[[LHS]], %[[RHS]] // CHECK: %[[ADDF:.+]] = arith.addf %[[OUT]], %[[MULF]] // CHECK: memref.store %[[ADDF]], %[[ARG2]][%[[IV0]], %[[IV1]]] +// CHECK-NOT: linalg.matmul ins(%arg0, %arg1 : memref, memref) // ----- -- GitLab From 930e7ff9aee24140e8c11f6527ba0e3e2208b55c Mon Sep 17 00:00:00 2001 From: David Majnemer Date: Mon, 4 Mar 2024 17:55:09 +0000 Subject: [PATCH 038/929] [AArch64] Optimize abs, neg and copysign for fp16/bf16 We can use bitwise arithmetic to implement these, making them considerably faster than legalization via promotion. --- .../Target/AArch64/AArch64ISelLowering.cpp | 83 +- llvm/lib/Target/AArch64/AArch64InstrInfo.td | 39 +- llvm/test/CodeGen/AArch64/f16-instructions.ll | 15 +- llvm/test/CodeGen/AArch64/fabs.ll | 189 +---- llvm/test/CodeGen/AArch64/fcopysign.ll | 303 +------- llvm/test/CodeGen/AArch64/fneg.ll | 193 +---- llvm/test/CodeGen/AArch64/vector-fcopysign.ll | 726 +++--------------- 7 files changed, 241 insertions(+), 1307 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 5af89bc5192a..63725f840b6f 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -676,11 +676,13 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, setOperationAction(ISD::FPOW, MVT::f64, Expand); setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); - if (Subtarget->hasFullFP16()) + if (Subtarget->hasFullFP16()) { setOperationAction(ISD::FCOPYSIGN, MVT::f16, Custom); - else + setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Custom); + } else { setOperationAction(ISD::FCOPYSIGN, MVT::f16, Promote); - setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Promote); + setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Promote); + } for (auto Op : {ISD::FREM, ISD::FPOW, ISD::FPOWI, ISD::FCOS, ISD::FSIN, ISD::FSINCOS, @@ -699,23 +701,48 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, } auto LegalizeNarrowFP = [this](MVT ScalarVT) { - for (auto Op : - {ISD::SETCC, ISD::SELECT_CC, - ISD::BR_CC, ISD::FADD, ISD::FSUB, - ISD::FMUL, ISD::FDIV, ISD::FMA, - ISD::FNEG, ISD::FABS, ISD::FCEIL, - ISD::FSQRT, ISD::FFLOOR, ISD::FNEARBYINT, - ISD::FRINT, ISD::FROUND, ISD::FROUNDEVEN, - ISD::FTRUNC, ISD::FMINNUM, ISD::FMAXNUM, - ISD::FMINIMUM, ISD::FMAXIMUM, ISD::STRICT_FADD, - ISD::STRICT_FSUB, ISD::STRICT_FMUL, ISD::STRICT_FDIV, - ISD::STRICT_FMA, ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR, - ISD::STRICT_FSQRT, ISD::STRICT_FRINT, ISD::STRICT_FNEARBYINT, - ISD::STRICT_FROUND, ISD::STRICT_FTRUNC, ISD::STRICT_FROUNDEVEN, - ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM, ISD::STRICT_FMINIMUM, - ISD::STRICT_FMAXIMUM}) + for (auto Op : {ISD::SETCC, + ISD::SELECT_CC, + ISD::BR_CC, + ISD::FADD, + ISD::FSUB, + ISD::FMUL, + ISD::FDIV, + ISD::FMA, + ISD::FCEIL, + ISD::FSQRT, + ISD::FFLOOR, + ISD::FNEARBYINT, + ISD::FRINT, + ISD::FROUND, + ISD::FROUNDEVEN, + ISD::FTRUNC, + ISD::FMINNUM, + ISD::FMAXNUM, + ISD::FMINIMUM, + ISD::FMAXIMUM, + ISD::STRICT_FADD, + ISD::STRICT_FSUB, + ISD::STRICT_FMUL, + ISD::STRICT_FDIV, + ISD::STRICT_FMA, + ISD::STRICT_FCEIL, + ISD::STRICT_FFLOOR, + ISD::STRICT_FSQRT, + ISD::STRICT_FRINT, + ISD::STRICT_FNEARBYINT, + ISD::STRICT_FROUND, + ISD::STRICT_FTRUNC, + ISD::STRICT_FROUNDEVEN, + ISD::STRICT_FMINNUM, + ISD::STRICT_FMAXNUM, + ISD::STRICT_FMINIMUM, + ISD::STRICT_FMAXIMUM}) setOperationAction(Op, ScalarVT, Promote); + for (auto Op : {ISD::FNEG, ISD::FABS}) + setOperationAction(Op, ScalarVT, Legal); + // Round-to-integer need custom lowering for fp16, as Promote doesn't work // because the result type is integer. for (auto Op : {ISD::LROUND, ISD::LLROUND, ISD::LRINT, ISD::LLRINT, @@ -730,8 +757,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, setOperationPromotedToType(ISD::FMUL, V4Narrow, MVT::v4f32); setOperationPromotedToType(ISD::FDIV, V4Narrow, MVT::v4f32); - setOperationAction(ISD::FABS, V4Narrow, Expand); - setOperationAction(ISD::FNEG, V4Narrow, Expand); + setOperationAction(ISD::FABS, V4Narrow, Legal); + setOperationAction(ISD::FNEG, V4Narrow, Legal); setOperationAction(ISD::FROUND, V4Narrow, Expand); setOperationAction(ISD::FROUNDEVEN, V4Narrow, Expand); setOperationAction(ISD::FMA, V4Narrow, Expand); @@ -740,7 +767,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, setOperationAction(ISD::SELECT, V4Narrow, Expand); setOperationAction(ISD::SELECT_CC, V4Narrow, Expand); setOperationAction(ISD::FTRUNC, V4Narrow, Expand); - setOperationAction(ISD::FCOPYSIGN, V4Narrow, Expand); + setOperationAction(ISD::FCOPYSIGN, V4Narrow, Custom); setOperationAction(ISD::FFLOOR, V4Narrow, Expand); setOperationAction(ISD::FCEIL, V4Narrow, Expand); setOperationAction(ISD::FRINT, V4Narrow, Expand); @@ -748,16 +775,16 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, setOperationAction(ISD::FSQRT, V4Narrow, Expand); auto V8Narrow = MVT::getVectorVT(ScalarVT, 8); - setOperationAction(ISD::FABS, V8Narrow, Expand); + setOperationAction(ISD::FABS, V8Narrow, Legal); setOperationAction(ISD::FADD, V8Narrow, Expand); setOperationAction(ISD::FCEIL, V8Narrow, Expand); - setOperationAction(ISD::FCOPYSIGN, V8Narrow, Expand); + setOperationAction(ISD::FCOPYSIGN, V8Narrow, Custom); setOperationAction(ISD::FDIV, V8Narrow, Expand); setOperationAction(ISD::FFLOOR, V8Narrow, Expand); setOperationAction(ISD::FMA, V8Narrow, Expand); setOperationAction(ISD::FMUL, V8Narrow, Expand); setOperationAction(ISD::FNEARBYINT, V8Narrow, Expand); - setOperationAction(ISD::FNEG, V8Narrow, Expand); + setOperationAction(ISD::FNEG, V8Narrow, Legal); setOperationAction(ISD::FROUND, V8Narrow, Expand); setOperationAction(ISD::FROUNDEVEN, V8Narrow, Expand); setOperationAction(ISD::FRINT, V8Narrow, Expand); @@ -1745,7 +1772,9 @@ void AArch64TargetLowering::addTypeForNEON(MVT VT) { // But we do support custom-lowering for FCOPYSIGN. if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64 || - ((VT == MVT::v4f16 || VT == MVT::v8f16) && Subtarget->hasFullFP16())) + ((VT == MVT::v4bf16 || VT == MVT::v8bf16 || VT == MVT::v4f16 || + VT == MVT::v8f16) && + Subtarget->hasFullFP16())) setOperationAction(ISD::FCOPYSIGN, VT, Custom); setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); @@ -9208,7 +9237,7 @@ SDValue AArch64TargetLowering::LowerFCOPYSIGN(SDValue Op, } else if (VT == MVT::f32) { VecVT = MVT::v4i32; SetVecVal(AArch64::ssub); - } else if (VT == MVT::f16) { + } else if (VT == MVT::f16 || VT == MVT::bf16) { VecVT = MVT::v8i16; SetVecVal(AArch64::hsub); } else { @@ -9230,7 +9259,7 @@ SDValue AArch64TargetLowering::LowerFCOPYSIGN(SDValue Op, SDValue BSP = DAG.getNode(AArch64ISD::BSP, DL, VecVT, SignMaskV, VecVal1, VecVal2); - if (VT == MVT::f16) + if (VT == MVT::f16 || VT == MVT::bf16) return DAG.getTargetExtractSubreg(AArch64::hsub, DL, VT, BSP); if (VT == MVT::f32) return DAG.getTargetExtractSubreg(AArch64::ssub, DL, VT, BSP); diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index c153bb3f0145..2f3aaf86d376 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -5028,9 +5028,6 @@ defm FCVTNU : SIMDTwoVectorFPToInt<1,0,0b11010, "fcvtnu",int_aarch64_neon_fcvtnu defm FCVTN : SIMDFPNarrowTwoVector<0, 0, 0b10110, "fcvtn">; def : Pat<(v4i16 (int_aarch64_neon_vcvtfp2hf (v4f32 V128:$Rn))), (FCVTNv4i16 V128:$Rn)>; -//def : Pat<(concat_vectors V64:$Rd, -// (v4bf16 (any_fpround (v4f32 V128:$Rn)))), -// (FCVTNv8bf16 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rd, dsub), V128:$Rn)>; def : Pat<(concat_vectors V64:$Rd, (v4i16 (int_aarch64_neon_vcvtfp2hf (v4f32 V128:$Rn)))), (FCVTNv8i16 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rd, dsub), V128:$Rn)>; @@ -7813,6 +7810,42 @@ def : InstAlias<"uxtl2 $dst.2d, $src1.4s", (USHLLv4i32_shift V128:$dst, V128:$src1, 0)>; } +def abs_f16 : + OutPatFrag<(ops node:$Rn), + (EXTRACT_SUBREG (f32 (COPY_TO_REGCLASS + (i32 (ANDWri + (i32 (COPY_TO_REGCLASS (INSERT_SUBREG (f32 (IMPLICIT_DEF)), + node:$Rn, hsub), GPR32)), + (i32 (logical_imm32_XFORM(i32 0x7fff))))), + FPR32)), hsub)>; + +def : Pat<(f16 (fabs (f16 FPR16:$Rn))), (f16 (abs_f16 (f16 FPR16:$Rn)))>; +def : Pat<(bf16 (fabs (bf16 FPR16:$Rn))), (bf16 (abs_f16 (bf16 FPR16:$Rn)))>; + +def neg_f16 : + OutPatFrag<(ops node:$Rn), + (EXTRACT_SUBREG (f32 (COPY_TO_REGCLASS + (i32 (EORWri + (i32 (COPY_TO_REGCLASS (INSERT_SUBREG (f32 (IMPLICIT_DEF)), + node:$Rn, hsub), GPR32)), + (i32 (logical_imm32_XFORM(i32 0x8000))))), + FPR32)), hsub)>; + +def : Pat<(f16 (fneg (f16 FPR16:$Rn))), (f16 (neg_f16 (f16 FPR16:$Rn)))>; +def : Pat<(bf16 (fneg (bf16 FPR16:$Rn))), (bf16 (neg_f16 (bf16 FPR16:$Rn)))>; + +let Predicates = [HasNEON] in { +def : Pat<(v4f16 (fabs (v4f16 V64:$Rn))), (v4f16 (BICv4i16 (v4f16 V64:$Rn), (i32 128), (i32 8)))>; +def : Pat<(v4bf16 (fabs (v4bf16 V64:$Rn))), (v4bf16 (BICv4i16 (v4bf16 V64:$Rn), (i32 128), (i32 8)))>; +def : Pat<(v8f16 (fabs (v8f16 V128:$Rn))), (v8f16 (BICv8i16 (v8f16 V128:$Rn), (i32 128), (i32 8)))>; +def : Pat<(v8bf16 (fabs (v8bf16 V128:$Rn))), (v8bf16 (BICv8i16 (v8bf16 V128:$Rn), (i32 128), (i32 8)))>; + +def : Pat<(v4f16 (fneg (v4f16 V64:$Rn))), (v4f16 (EORv8i8 (v4f16 V64:$Rn), (MOVIv4i16 (i32 128), (i32 8))))>; +def : Pat<(v4bf16 (fneg (v4bf16 V64:$Rn))), (v4bf16 (EORv8i8 (v4bf16 V64:$Rn), (v4i16 (MOVIv4i16 (i32 0x80), (i32 8)))))>; +def : Pat<(v8f16 (fneg (v8f16 V128:$Rn))), (v8f16 (EORv16i8 (v8f16 V128:$Rn), (MOVIv8i16 (i32 128), (i32 8))))>; +def : Pat<(v8bf16 (fneg (v8bf16 V128:$Rn))), (v8bf16 (EORv16i8 (v8bf16 V128:$Rn), (v8i16 (MOVIv8i16 (i32 0x80), (i32 8)))))>; +} + // If an integer is about to be converted to a floating point value, // just load it on the floating point unit. // These patterns are more complex because floating point loads do not diff --git a/llvm/test/CodeGen/AArch64/f16-instructions.ll b/llvm/test/CodeGen/AArch64/f16-instructions.ll index 0be41f246512..fa362ae798aa 100644 --- a/llvm/test/CodeGen/AArch64/f16-instructions.ll +++ b/llvm/test/CodeGen/AArch64/f16-instructions.ll @@ -1027,9 +1027,9 @@ define half @test_fma(half %a, half %b, half %c) #0 { } ; CHECK-CVT-LABEL: test_fabs: -; CHECK-CVT-NEXT: fcvt s0, h0 -; CHECK-CVT-NEXT: fabs s0, s0 -; CHECK-CVT-NEXT: fcvt h0, s0 +; CHECK-CVT-NEXT: fmov w8, s0 +; CHECK-CVT-NEXT: and w8, w8, #0x7fff +; CHECK-CVT-NEXT: fmov s0, w8 ; CHECK-CVT-NEXT: ret ; CHECK-FP16-LABEL: test_fabs: @@ -1338,3 +1338,12 @@ define half @test_fmuladd(half %a, half %b, half %c) #0 { } attributes #0 = { nounwind } +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; CHECK-COMMON: {{.*}} +; CHECK-CVT: {{.*}} +; CHECK-FP16: {{.*}} +; FALLBACK: {{.*}} +; FALLBACK-FP16: {{.*}} +; GISEL: {{.*}} +; GISEL-CVT: {{.*}} +; GISEL-FP16: {{.*}} diff --git a/llvm/test/CodeGen/AArch64/fabs.ll b/llvm/test/CodeGen/AArch64/fabs.ll index 5462bc65fd34..7c13b49246d2 100644 --- a/llvm/test/CodeGen/AArch64/fabs.ll +++ b/llvm/test/CodeGen/AArch64/fabs.ll @@ -27,9 +27,11 @@ entry: define half @fabs_f16(half %a) { ; CHECK-SD-NOFP16-LABEL: fabs_f16: ; CHECK-SD-NOFP16: // %bb.0: // %entry -; CHECK-SD-NOFP16-NEXT: fcvt s0, h0 -; CHECK-SD-NOFP16-NEXT: fabs s0, s0 -; CHECK-SD-NOFP16-NEXT: fcvt h0, s0 +; CHECK-SD-NOFP16-NEXT: // kill: def $h0 killed $h0 def $s0 +; CHECK-SD-NOFP16-NEXT: fmov w8, s0 +; CHECK-SD-NOFP16-NEXT: and w8, w8, #0x7fff +; CHECK-SD-NOFP16-NEXT: fmov s0, w8 +; CHECK-SD-NOFP16-NEXT: // kill: def $h0 killed $h0 killed $s0 ; CHECK-SD-NOFP16-NEXT: ret ; ; CHECK-SD-FP16-LABEL: fabs_f16: @@ -148,45 +150,7 @@ entry: define <7 x half> @fabs_v7f16(<7 x half> %a) { ; CHECK-SD-NOFP16-LABEL: fabs_v7f16: ; CHECK-SD-NOFP16: // %bb.0: // %entry -; CHECK-SD-NOFP16-NEXT: mov h1, v0.h[1] -; CHECK-SD-NOFP16-NEXT: fcvt s2, h0 -; CHECK-SD-NOFP16-NEXT: mov h3, v0.h[2] -; CHECK-SD-NOFP16-NEXT: mov h4, v0.h[3] -; CHECK-SD-NOFP16-NEXT: mov h6, v0.h[4] -; CHECK-SD-NOFP16-NEXT: fcvt s1, h1 -; CHECK-SD-NOFP16-NEXT: fabs s2, s2 -; CHECK-SD-NOFP16-NEXT: fcvt s3, h3 -; CHECK-SD-NOFP16-NEXT: fcvt s4, h4 -; CHECK-SD-NOFP16-NEXT: fabs s5, s1 -; CHECK-SD-NOFP16-NEXT: fcvt h1, s2 -; CHECK-SD-NOFP16-NEXT: fabs s2, s3 -; CHECK-SD-NOFP16-NEXT: fabs s4, s4 -; CHECK-SD-NOFP16-NEXT: fcvt h3, s5 -; CHECK-SD-NOFP16-NEXT: fcvt s5, h6 -; CHECK-SD-NOFP16-NEXT: fcvt h2, s2 -; CHECK-SD-NOFP16-NEXT: fcvt h4, s4 -; CHECK-SD-NOFP16-NEXT: mov v1.h[1], v3.h[0] -; CHECK-SD-NOFP16-NEXT: mov h3, v0.h[5] -; CHECK-SD-NOFP16-NEXT: fabs s5, s5 -; CHECK-SD-NOFP16-NEXT: mov v1.h[2], v2.h[0] -; CHECK-SD-NOFP16-NEXT: mov h2, v0.h[6] -; CHECK-SD-NOFP16-NEXT: fcvt s3, h3 -; CHECK-SD-NOFP16-NEXT: mov h0, v0.h[7] -; CHECK-SD-NOFP16-NEXT: mov v1.h[3], v4.h[0] -; CHECK-SD-NOFP16-NEXT: fcvt h4, s5 -; CHECK-SD-NOFP16-NEXT: fcvt s2, h2 -; CHECK-SD-NOFP16-NEXT: fabs s3, s3 -; CHECK-SD-NOFP16-NEXT: fcvt s0, h0 -; CHECK-SD-NOFP16-NEXT: mov v1.h[4], v4.h[0] -; CHECK-SD-NOFP16-NEXT: fabs s2, s2 -; CHECK-SD-NOFP16-NEXT: fcvt h3, s3 -; CHECK-SD-NOFP16-NEXT: fabs s0, s0 -; CHECK-SD-NOFP16-NEXT: fcvt h2, s2 -; CHECK-SD-NOFP16-NEXT: mov v1.h[5], v3.h[0] -; CHECK-SD-NOFP16-NEXT: fcvt h0, s0 -; CHECK-SD-NOFP16-NEXT: mov v1.h[6], v2.h[0] -; CHECK-SD-NOFP16-NEXT: mov v1.h[7], v0.h[0] -; CHECK-SD-NOFP16-NEXT: mov v0.16b, v1.16b +; CHECK-SD-NOFP16-NEXT: bic v0.8h, #128, lsl #8 ; CHECK-SD-NOFP16-NEXT: ret ; ; CHECK-SD-FP16-LABEL: fabs_v7f16: @@ -234,26 +198,7 @@ entry: define <4 x half> @fabs_v4f16(<4 x half> %a) { ; CHECK-SD-NOFP16-LABEL: fabs_v4f16: ; CHECK-SD-NOFP16: // %bb.0: // %entry -; CHECK-SD-NOFP16-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-SD-NOFP16-NEXT: mov h1, v0.h[1] -; CHECK-SD-NOFP16-NEXT: fcvt s2, h0 -; CHECK-SD-NOFP16-NEXT: mov h3, v0.h[2] -; CHECK-SD-NOFP16-NEXT: mov h4, v0.h[3] -; CHECK-SD-NOFP16-NEXT: fcvt s1, h1 -; CHECK-SD-NOFP16-NEXT: fabs s0, s2 -; CHECK-SD-NOFP16-NEXT: fcvt s2, h3 -; CHECK-SD-NOFP16-NEXT: fcvt s3, h4 -; CHECK-SD-NOFP16-NEXT: fabs s1, s1 -; CHECK-SD-NOFP16-NEXT: fcvt h0, s0 -; CHECK-SD-NOFP16-NEXT: fabs s2, s2 -; CHECK-SD-NOFP16-NEXT: fcvt h1, s1 -; CHECK-SD-NOFP16-NEXT: mov v0.h[1], v1.h[0] -; CHECK-SD-NOFP16-NEXT: fcvt h1, s2 -; CHECK-SD-NOFP16-NEXT: fabs s2, s3 -; CHECK-SD-NOFP16-NEXT: mov v0.h[2], v1.h[0] -; CHECK-SD-NOFP16-NEXT: fcvt h1, s2 -; CHECK-SD-NOFP16-NEXT: mov v0.h[3], v1.h[0] -; CHECK-SD-NOFP16-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-SD-NOFP16-NEXT: bic v0.4h, #128, lsl #8 ; CHECK-SD-NOFP16-NEXT: ret ; ; CHECK-SD-FP16-LABEL: fabs_v4f16: @@ -280,45 +225,7 @@ entry: define <8 x half> @fabs_v8f16(<8 x half> %a) { ; CHECK-SD-NOFP16-LABEL: fabs_v8f16: ; CHECK-SD-NOFP16: // %bb.0: // %entry -; CHECK-SD-NOFP16-NEXT: mov h1, v0.h[1] -; CHECK-SD-NOFP16-NEXT: fcvt s2, h0 -; CHECK-SD-NOFP16-NEXT: mov h3, v0.h[2] -; CHECK-SD-NOFP16-NEXT: mov h4, v0.h[3] -; CHECK-SD-NOFP16-NEXT: mov h6, v0.h[4] -; CHECK-SD-NOFP16-NEXT: fcvt s1, h1 -; CHECK-SD-NOFP16-NEXT: fabs s2, s2 -; CHECK-SD-NOFP16-NEXT: fcvt s3, h3 -; CHECK-SD-NOFP16-NEXT: fcvt s4, h4 -; CHECK-SD-NOFP16-NEXT: fabs s5, s1 -; CHECK-SD-NOFP16-NEXT: fcvt h1, s2 -; CHECK-SD-NOFP16-NEXT: fabs s2, s3 -; CHECK-SD-NOFP16-NEXT: fabs s4, s4 -; CHECK-SD-NOFP16-NEXT: fcvt h3, s5 -; CHECK-SD-NOFP16-NEXT: fcvt s5, h6 -; CHECK-SD-NOFP16-NEXT: fcvt h2, s2 -; CHECK-SD-NOFP16-NEXT: fcvt h4, s4 -; CHECK-SD-NOFP16-NEXT: mov v1.h[1], v3.h[0] -; CHECK-SD-NOFP16-NEXT: mov h3, v0.h[5] -; CHECK-SD-NOFP16-NEXT: fabs s5, s5 -; CHECK-SD-NOFP16-NEXT: mov v1.h[2], v2.h[0] -; CHECK-SD-NOFP16-NEXT: mov h2, v0.h[6] -; CHECK-SD-NOFP16-NEXT: fcvt s3, h3 -; CHECK-SD-NOFP16-NEXT: mov h0, v0.h[7] -; CHECK-SD-NOFP16-NEXT: mov v1.h[3], v4.h[0] -; CHECK-SD-NOFP16-NEXT: fcvt h4, s5 -; CHECK-SD-NOFP16-NEXT: fcvt s2, h2 -; CHECK-SD-NOFP16-NEXT: fabs s3, s3 -; CHECK-SD-NOFP16-NEXT: fcvt s0, h0 -; CHECK-SD-NOFP16-NEXT: mov v1.h[4], v4.h[0] -; CHECK-SD-NOFP16-NEXT: fabs s2, s2 -; CHECK-SD-NOFP16-NEXT: fcvt h3, s3 -; CHECK-SD-NOFP16-NEXT: fabs s0, s0 -; CHECK-SD-NOFP16-NEXT: fcvt h2, s2 -; CHECK-SD-NOFP16-NEXT: mov v1.h[5], v3.h[0] -; CHECK-SD-NOFP16-NEXT: fcvt h0, s0 -; CHECK-SD-NOFP16-NEXT: mov v1.h[6], v2.h[0] -; CHECK-SD-NOFP16-NEXT: mov v1.h[7], v0.h[0] -; CHECK-SD-NOFP16-NEXT: mov v0.16b, v1.16b +; CHECK-SD-NOFP16-NEXT: bic v0.8h, #128, lsl #8 ; CHECK-SD-NOFP16-NEXT: ret ; ; CHECK-SD-FP16-LABEL: fabs_v8f16: @@ -348,84 +255,8 @@ entry: define <16 x half> @fabs_v16f16(<16 x half> %a) { ; CHECK-SD-NOFP16-LABEL: fabs_v16f16: ; CHECK-SD-NOFP16: // %bb.0: // %entry -; CHECK-SD-NOFP16-NEXT: mov h2, v0.h[1] -; CHECK-SD-NOFP16-NEXT: mov h3, v1.h[1] -; CHECK-SD-NOFP16-NEXT: fcvt s4, h0 -; CHECK-SD-NOFP16-NEXT: mov h5, v0.h[2] -; CHECK-SD-NOFP16-NEXT: fcvt s6, h1 -; CHECK-SD-NOFP16-NEXT: mov h7, v1.h[2] -; CHECK-SD-NOFP16-NEXT: mov h16, v0.h[3] -; CHECK-SD-NOFP16-NEXT: mov h17, v1.h[3] -; CHECK-SD-NOFP16-NEXT: mov h20, v0.h[4] -; CHECK-SD-NOFP16-NEXT: mov h21, v1.h[4] -; CHECK-SD-NOFP16-NEXT: fcvt s2, h2 -; CHECK-SD-NOFP16-NEXT: fcvt s3, h3 -; CHECK-SD-NOFP16-NEXT: fabs s4, s4 -; CHECK-SD-NOFP16-NEXT: fcvt s5, h5 -; CHECK-SD-NOFP16-NEXT: fabs s6, s6 -; CHECK-SD-NOFP16-NEXT: fcvt s7, h7 -; CHECK-SD-NOFP16-NEXT: fcvt s16, h16 -; CHECK-SD-NOFP16-NEXT: fcvt s17, h17 -; CHECK-SD-NOFP16-NEXT: fabs s18, s2 -; CHECK-SD-NOFP16-NEXT: fabs s19, s3 -; CHECK-SD-NOFP16-NEXT: fcvt h2, s4 -; CHECK-SD-NOFP16-NEXT: fabs s4, s5 -; CHECK-SD-NOFP16-NEXT: fcvt h3, s6 -; CHECK-SD-NOFP16-NEXT: fabs s6, s7 -; CHECK-SD-NOFP16-NEXT: fabs s16, s16 -; CHECK-SD-NOFP16-NEXT: fcvt h5, s18 -; CHECK-SD-NOFP16-NEXT: fcvt h7, s19 -; CHECK-SD-NOFP16-NEXT: fcvt s18, h20 -; CHECK-SD-NOFP16-NEXT: fcvt s19, h21 -; CHECK-SD-NOFP16-NEXT: fcvt h4, s4 -; CHECK-SD-NOFP16-NEXT: fcvt h6, s6 -; CHECK-SD-NOFP16-NEXT: fcvt h16, s16 -; CHECK-SD-NOFP16-NEXT: mov v2.h[1], v5.h[0] -; CHECK-SD-NOFP16-NEXT: fabs s5, s17 -; CHECK-SD-NOFP16-NEXT: mov v3.h[1], v7.h[0] -; CHECK-SD-NOFP16-NEXT: mov h7, v0.h[5] -; CHECK-SD-NOFP16-NEXT: mov h17, v1.h[5] -; CHECK-SD-NOFP16-NEXT: fabs s18, s18 -; CHECK-SD-NOFP16-NEXT: mov v2.h[2], v4.h[0] -; CHECK-SD-NOFP16-NEXT: fabs s4, s19 -; CHECK-SD-NOFP16-NEXT: fcvt h5, s5 -; CHECK-SD-NOFP16-NEXT: mov v3.h[2], v6.h[0] -; CHECK-SD-NOFP16-NEXT: mov h6, v0.h[6] -; CHECK-SD-NOFP16-NEXT: fcvt s7, h7 -; CHECK-SD-NOFP16-NEXT: fcvt s17, h17 -; CHECK-SD-NOFP16-NEXT: mov h19, v1.h[6] -; CHECK-SD-NOFP16-NEXT: mov h0, v0.h[7] -; CHECK-SD-NOFP16-NEXT: mov h1, v1.h[7] -; CHECK-SD-NOFP16-NEXT: mov v2.h[3], v16.h[0] -; CHECK-SD-NOFP16-NEXT: fcvt h4, s4 -; CHECK-SD-NOFP16-NEXT: mov v3.h[3], v5.h[0] -; CHECK-SD-NOFP16-NEXT: fcvt h5, s18 -; CHECK-SD-NOFP16-NEXT: fcvt s6, h6 -; CHECK-SD-NOFP16-NEXT: fabs s7, s7 -; CHECK-SD-NOFP16-NEXT: fabs s16, s17 -; CHECK-SD-NOFP16-NEXT: fcvt s17, h19 -; CHECK-SD-NOFP16-NEXT: fcvt s0, h0 -; CHECK-SD-NOFP16-NEXT: fcvt s1, h1 -; CHECK-SD-NOFP16-NEXT: mov v2.h[4], v5.h[0] -; CHECK-SD-NOFP16-NEXT: mov v3.h[4], v4.h[0] -; CHECK-SD-NOFP16-NEXT: fabs s4, s6 -; CHECK-SD-NOFP16-NEXT: fcvt h5, s7 -; CHECK-SD-NOFP16-NEXT: fcvt h6, s16 -; CHECK-SD-NOFP16-NEXT: fabs s7, s17 -; CHECK-SD-NOFP16-NEXT: fabs s0, s0 -; CHECK-SD-NOFP16-NEXT: fabs s1, s1 -; CHECK-SD-NOFP16-NEXT: fcvt h4, s4 -; CHECK-SD-NOFP16-NEXT: mov v2.h[5], v5.h[0] -; CHECK-SD-NOFP16-NEXT: mov v3.h[5], v6.h[0] -; CHECK-SD-NOFP16-NEXT: fcvt h5, s7 -; CHECK-SD-NOFP16-NEXT: fcvt h0, s0 -; CHECK-SD-NOFP16-NEXT: fcvt h1, s1 -; CHECK-SD-NOFP16-NEXT: mov v2.h[6], v4.h[0] -; CHECK-SD-NOFP16-NEXT: mov v3.h[6], v5.h[0] -; CHECK-SD-NOFP16-NEXT: mov v2.h[7], v0.h[0] -; CHECK-SD-NOFP16-NEXT: mov v3.h[7], v1.h[0] -; CHECK-SD-NOFP16-NEXT: mov v0.16b, v2.16b -; CHECK-SD-NOFP16-NEXT: mov v1.16b, v3.16b +; CHECK-SD-NOFP16-NEXT: bic v0.8h, #128, lsl #8 +; CHECK-SD-NOFP16-NEXT: bic v1.8h, #128, lsl #8 ; CHECK-SD-NOFP16-NEXT: ret ; ; CHECK-SD-FP16-LABEL: fabs_v16f16: diff --git a/llvm/test/CodeGen/AArch64/fcopysign.ll b/llvm/test/CodeGen/AArch64/fcopysign.ll index 89e78f714749..78fd38ca9f26 100644 --- a/llvm/test/CodeGen/AArch64/fcopysign.ll +++ b/llvm/test/CodeGen/AArch64/fcopysign.ll @@ -199,63 +199,8 @@ entry: define <7 x half> @copysign_v7f16(<7 x half> %a, <7 x half> %b) { ; CHECK-SD-LABEL: copysign_v7f16: ; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: mov h2, v1.h[1] -; CHECK-SD-NEXT: mov h4, v0.h[1] -; CHECK-SD-NEXT: fcvt s5, h1 -; CHECK-SD-NEXT: fcvt s6, h0 -; CHECK-SD-NEXT: mvni v3.4s, #128, lsl #24 -; CHECK-SD-NEXT: mov h7, v1.h[2] -; CHECK-SD-NEXT: mov h16, v0.h[2] -; CHECK-SD-NEXT: mov h17, v1.h[3] -; CHECK-SD-NEXT: fcvt s2, h2 -; CHECK-SD-NEXT: fcvt s4, h4 -; CHECK-SD-NEXT: bit v5.16b, v6.16b, v3.16b -; CHECK-SD-NEXT: mov h6, v0.h[3] -; CHECK-SD-NEXT: fcvt s7, h7 -; CHECK-SD-NEXT: fcvt s16, h16 -; CHECK-SD-NEXT: fcvt s17, h17 -; CHECK-SD-NEXT: bif v4.16b, v2.16b, v3.16b -; CHECK-SD-NEXT: fcvt h2, s5 -; CHECK-SD-NEXT: mov v5.16b, v3.16b -; CHECK-SD-NEXT: fcvt s6, h6 -; CHECK-SD-NEXT: bsl v5.16b, v16.16b, v7.16b -; CHECK-SD-NEXT: fcvt h4, s4 -; CHECK-SD-NEXT: mov h7, v1.h[4] -; CHECK-SD-NEXT: mov h16, v0.h[4] -; CHECK-SD-NEXT: bif v6.16b, v17.16b, v3.16b -; CHECK-SD-NEXT: mov h17, v0.h[5] -; CHECK-SD-NEXT: fcvt h5, s5 -; CHECK-SD-NEXT: mov v2.h[1], v4.h[0] -; CHECK-SD-NEXT: fcvt s4, h7 -; CHECK-SD-NEXT: fcvt s7, h16 -; CHECK-SD-NEXT: mov h16, v1.h[5] -; CHECK-SD-NEXT: fcvt h6, s6 -; CHECK-SD-NEXT: fcvt s17, h17 -; CHECK-SD-NEXT: mov v2.h[2], v5.h[0] -; CHECK-SD-NEXT: mov h5, v1.h[6] -; CHECK-SD-NEXT: mov h1, v1.h[7] -; CHECK-SD-NEXT: bit v4.16b, v7.16b, v3.16b -; CHECK-SD-NEXT: mov h7, v0.h[6] -; CHECK-SD-NEXT: fcvt s16, h16 -; CHECK-SD-NEXT: mov h0, v0.h[7] -; CHECK-SD-NEXT: mov v2.h[3], v6.h[0] -; CHECK-SD-NEXT: fcvt s5, h5 -; CHECK-SD-NEXT: fcvt s1, h1 -; CHECK-SD-NEXT: fcvt s6, h7 -; CHECK-SD-NEXT: mov v7.16b, v3.16b -; CHECK-SD-NEXT: fcvt h4, s4 -; CHECK-SD-NEXT: fcvt s0, h0 -; CHECK-SD-NEXT: bsl v7.16b, v17.16b, v16.16b -; CHECK-SD-NEXT: bit v5.16b, v6.16b, v3.16b -; CHECK-SD-NEXT: mov v2.h[4], v4.h[0] -; CHECK-SD-NEXT: bif v0.16b, v1.16b, v3.16b -; CHECK-SD-NEXT: fcvt h4, s7 -; CHECK-SD-NEXT: fcvt h0, s0 -; CHECK-SD-NEXT: mov v2.h[5], v4.h[0] -; CHECK-SD-NEXT: fcvt h4, s5 -; CHECK-SD-NEXT: mov v2.h[6], v4.h[0] -; CHECK-SD-NEXT: mov v2.h[7], v0.h[0] -; CHECK-SD-NEXT: mov v0.16b, v2.16b +; CHECK-SD-NEXT: mvni v2.8h, #128, lsl #8 +; CHECK-SD-NEXT: bif v0.16b, v1.16b, v2.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: copysign_v7f16: @@ -290,246 +235,34 @@ entry: } define <4 x half> @copysign_v4f16(<4 x half> %a, <4 x half> %b) { -; CHECK-SD-LABEL: copysign_v4f16: -; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q1 -; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-SD-NEXT: mov h3, v1.h[1] -; CHECK-SD-NEXT: mov h4, v0.h[1] -; CHECK-SD-NEXT: mov h5, v1.h[2] -; CHECK-SD-NEXT: mov h6, v0.h[2] -; CHECK-SD-NEXT: mvni v2.4s, #128, lsl #24 -; CHECK-SD-NEXT: fcvt s7, h1 -; CHECK-SD-NEXT: fcvt s16, h0 -; CHECK-SD-NEXT: mov h1, v1.h[3] -; CHECK-SD-NEXT: fcvt s3, h3 -; CHECK-SD-NEXT: fcvt s4, h4 -; CHECK-SD-NEXT: fcvt s1, h1 -; CHECK-SD-NEXT: bit v3.16b, v4.16b, v2.16b -; CHECK-SD-NEXT: fcvt s4, h5 -; CHECK-SD-NEXT: fcvt s5, h6 -; CHECK-SD-NEXT: mov v6.16b, v2.16b -; CHECK-SD-NEXT: bsl v6.16b, v16.16b, v7.16b -; CHECK-SD-NEXT: mov h7, v0.h[3] -; CHECK-SD-NEXT: bit v4.16b, v5.16b, v2.16b -; CHECK-SD-NEXT: fcvt h3, s3 -; CHECK-SD-NEXT: fcvt h0, s6 -; CHECK-SD-NEXT: fcvt s5, h7 -; CHECK-SD-NEXT: mov v0.h[1], v3.h[0] -; CHECK-SD-NEXT: fcvt h3, s4 -; CHECK-SD-NEXT: bit v1.16b, v5.16b, v2.16b -; CHECK-SD-NEXT: mov v0.h[2], v3.h[0] -; CHECK-SD-NEXT: fcvt h1, s1 -; CHECK-SD-NEXT: mov v0.h[3], v1.h[0] -; CHECK-SD-NEXT: // kill: def $d0 killed $d0 killed $q0 -; CHECK-SD-NEXT: ret -; -; CHECK-GI-LABEL: copysign_v4f16: -; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: mvni v2.4h, #128, lsl #8 -; CHECK-GI-NEXT: bif v0.8b, v1.8b, v2.8b -; CHECK-GI-NEXT: ret +; CHECK-LABEL: copysign_v4f16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: mvni v2.4h, #128, lsl #8 +; CHECK-NEXT: bif v0.8b, v1.8b, v2.8b +; CHECK-NEXT: ret entry: %c = call <4 x half> @llvm.copysign.v4f16(<4 x half> %a, <4 x half> %b) ret <4 x half> %c } define <8 x half> @copysign_v8f16(<8 x half> %a, <8 x half> %b) { -; CHECK-SD-LABEL: copysign_v8f16: -; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: mov h2, v1.h[1] -; CHECK-SD-NEXT: mov h4, v0.h[1] -; CHECK-SD-NEXT: fcvt s5, h1 -; CHECK-SD-NEXT: fcvt s6, h0 -; CHECK-SD-NEXT: mvni v3.4s, #128, lsl #24 -; CHECK-SD-NEXT: mov h7, v1.h[2] -; CHECK-SD-NEXT: mov h16, v0.h[2] -; CHECK-SD-NEXT: mov h17, v1.h[3] -; CHECK-SD-NEXT: fcvt s2, h2 -; CHECK-SD-NEXT: fcvt s4, h4 -; CHECK-SD-NEXT: bit v5.16b, v6.16b, v3.16b -; CHECK-SD-NEXT: mov h6, v0.h[3] -; CHECK-SD-NEXT: fcvt s7, h7 -; CHECK-SD-NEXT: fcvt s16, h16 -; CHECK-SD-NEXT: fcvt s17, h17 -; CHECK-SD-NEXT: bif v4.16b, v2.16b, v3.16b -; CHECK-SD-NEXT: fcvt h2, s5 -; CHECK-SD-NEXT: mov v5.16b, v3.16b -; CHECK-SD-NEXT: fcvt s6, h6 -; CHECK-SD-NEXT: bsl v5.16b, v16.16b, v7.16b -; CHECK-SD-NEXT: fcvt h4, s4 -; CHECK-SD-NEXT: mov h7, v1.h[4] -; CHECK-SD-NEXT: mov h16, v0.h[4] -; CHECK-SD-NEXT: bif v6.16b, v17.16b, v3.16b -; CHECK-SD-NEXT: mov h17, v0.h[5] -; CHECK-SD-NEXT: fcvt h5, s5 -; CHECK-SD-NEXT: mov v2.h[1], v4.h[0] -; CHECK-SD-NEXT: fcvt s4, h7 -; CHECK-SD-NEXT: fcvt s7, h16 -; CHECK-SD-NEXT: mov h16, v1.h[5] -; CHECK-SD-NEXT: fcvt h6, s6 -; CHECK-SD-NEXT: fcvt s17, h17 -; CHECK-SD-NEXT: mov v2.h[2], v5.h[0] -; CHECK-SD-NEXT: mov h5, v1.h[6] -; CHECK-SD-NEXT: mov h1, v1.h[7] -; CHECK-SD-NEXT: bit v4.16b, v7.16b, v3.16b -; CHECK-SD-NEXT: mov h7, v0.h[6] -; CHECK-SD-NEXT: fcvt s16, h16 -; CHECK-SD-NEXT: mov h0, v0.h[7] -; CHECK-SD-NEXT: mov v2.h[3], v6.h[0] -; CHECK-SD-NEXT: fcvt s5, h5 -; CHECK-SD-NEXT: fcvt s1, h1 -; CHECK-SD-NEXT: fcvt s6, h7 -; CHECK-SD-NEXT: mov v7.16b, v3.16b -; CHECK-SD-NEXT: fcvt h4, s4 -; CHECK-SD-NEXT: fcvt s0, h0 -; CHECK-SD-NEXT: bsl v7.16b, v17.16b, v16.16b -; CHECK-SD-NEXT: bit v5.16b, v6.16b, v3.16b -; CHECK-SD-NEXT: mov v2.h[4], v4.h[0] -; CHECK-SD-NEXT: bif v0.16b, v1.16b, v3.16b -; CHECK-SD-NEXT: fcvt h4, s7 -; CHECK-SD-NEXT: fcvt h0, s0 -; CHECK-SD-NEXT: mov v2.h[5], v4.h[0] -; CHECK-SD-NEXT: fcvt h4, s5 -; CHECK-SD-NEXT: mov v2.h[6], v4.h[0] -; CHECK-SD-NEXT: mov v2.h[7], v0.h[0] -; CHECK-SD-NEXT: mov v0.16b, v2.16b -; CHECK-SD-NEXT: ret -; -; CHECK-GI-LABEL: copysign_v8f16: -; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: mvni v2.8h, #128, lsl #8 -; CHECK-GI-NEXT: bif v0.16b, v1.16b, v2.16b -; CHECK-GI-NEXT: ret +; CHECK-LABEL: copysign_v8f16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: mvni v2.8h, #128, lsl #8 +; CHECK-NEXT: bif v0.16b, v1.16b, v2.16b +; CHECK-NEXT: ret entry: %c = call <8 x half> @llvm.copysign.v8f16(<8 x half> %a, <8 x half> %b) ret <8 x half> %c } define <16 x half> @copysign_v16f16(<16 x half> %a, <16 x half> %b) { -; CHECK-SD-LABEL: copysign_v16f16: -; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: mov h4, v2.h[1] -; CHECK-SD-NEXT: mov h5, v0.h[1] -; CHECK-SD-NEXT: mvni v6.4s, #128, lsl #24 -; CHECK-SD-NEXT: mov h7, v3.h[1] -; CHECK-SD-NEXT: mov h16, v1.h[1] -; CHECK-SD-NEXT: fcvt s17, h2 -; CHECK-SD-NEXT: fcvt s18, h0 -; CHECK-SD-NEXT: mov h19, v2.h[2] -; CHECK-SD-NEXT: mov h20, v0.h[2] -; CHECK-SD-NEXT: fcvt s21, h3 -; CHECK-SD-NEXT: fcvt s22, h1 -; CHECK-SD-NEXT: mov h23, v3.h[2] -; CHECK-SD-NEXT: fcvt s4, h4 -; CHECK-SD-NEXT: fcvt s5, h5 -; CHECK-SD-NEXT: mov h24, v1.h[2] -; CHECK-SD-NEXT: fcvt s7, h7 -; CHECK-SD-NEXT: fcvt s16, h16 -; CHECK-SD-NEXT: mov h25, v1.h[3] -; CHECK-SD-NEXT: mov h26, v1.h[6] -; CHECK-SD-NEXT: bit v21.16b, v22.16b, v6.16b -; CHECK-SD-NEXT: fcvt s22, h23 -; CHECK-SD-NEXT: bit v4.16b, v5.16b, v6.16b -; CHECK-SD-NEXT: mov v5.16b, v6.16b -; CHECK-SD-NEXT: fcvt s23, h24 -; CHECK-SD-NEXT: bit v7.16b, v16.16b, v6.16b -; CHECK-SD-NEXT: mov h24, v3.h[3] -; CHECK-SD-NEXT: bsl v5.16b, v18.16b, v17.16b -; CHECK-SD-NEXT: fcvt s18, h19 -; CHECK-SD-NEXT: fcvt s19, h20 -; CHECK-SD-NEXT: mov h20, v0.h[3] -; CHECK-SD-NEXT: mov h17, v2.h[3] -; CHECK-SD-NEXT: fcvt h16, s4 -; CHECK-SD-NEXT: fcvt h7, s7 -; CHECK-SD-NEXT: fcvt h4, s5 -; CHECK-SD-NEXT: bit v18.16b, v19.16b, v6.16b -; CHECK-SD-NEXT: fcvt h5, s21 -; CHECK-SD-NEXT: fcvt s19, h20 -; CHECK-SD-NEXT: mov v20.16b, v6.16b -; CHECK-SD-NEXT: fcvt s17, h17 -; CHECK-SD-NEXT: fcvt s21, h25 -; CHECK-SD-NEXT: mov h25, v0.h[6] -; CHECK-SD-NEXT: bsl v20.16b, v23.16b, v22.16b -; CHECK-SD-NEXT: mov v4.h[1], v16.h[0] -; CHECK-SD-NEXT: fcvt s16, h24 -; CHECK-SD-NEXT: fcvt h18, s18 -; CHECK-SD-NEXT: mov h22, v2.h[4] -; CHECK-SD-NEXT: mov h23, v0.h[4] -; CHECK-SD-NEXT: bit v17.16b, v19.16b, v6.16b -; CHECK-SD-NEXT: mov h19, v3.h[4] -; CHECK-SD-NEXT: mov h24, v1.h[4] -; CHECK-SD-NEXT: mov v5.h[1], v7.h[0] -; CHECK-SD-NEXT: fcvt h7, s20 -; CHECK-SD-NEXT: bit v16.16b, v21.16b, v6.16b -; CHECK-SD-NEXT: mov v4.h[2], v18.h[0] -; CHECK-SD-NEXT: fcvt s18, h22 -; CHECK-SD-NEXT: fcvt s20, h23 -; CHECK-SD-NEXT: fcvt h17, s17 -; CHECK-SD-NEXT: fcvt s19, h19 -; CHECK-SD-NEXT: fcvt s21, h24 -; CHECK-SD-NEXT: mov h22, v2.h[5] -; CHECK-SD-NEXT: mov h23, v0.h[5] -; CHECK-SD-NEXT: mov h24, v1.h[5] -; CHECK-SD-NEXT: mov v5.h[2], v7.h[0] -; CHECK-SD-NEXT: fcvt h7, s16 -; CHECK-SD-NEXT: mov h16, v3.h[5] -; CHECK-SD-NEXT: bit v18.16b, v20.16b, v6.16b -; CHECK-SD-NEXT: mov h20, v2.h[6] -; CHECK-SD-NEXT: mov h2, v2.h[7] -; CHECK-SD-NEXT: bit v19.16b, v21.16b, v6.16b -; CHECK-SD-NEXT: mov h21, v3.h[6] -; CHECK-SD-NEXT: mov v4.h[3], v17.h[0] -; CHECK-SD-NEXT: fcvt s17, h22 -; CHECK-SD-NEXT: fcvt s22, h23 -; CHECK-SD-NEXT: fcvt s23, h25 -; CHECK-SD-NEXT: mov v5.h[3], v7.h[0] -; CHECK-SD-NEXT: fcvt s7, h16 -; CHECK-SD-NEXT: fcvt s16, h24 -; CHECK-SD-NEXT: fcvt h18, s18 -; CHECK-SD-NEXT: fcvt s20, h20 -; CHECK-SD-NEXT: fcvt s24, h26 -; CHECK-SD-NEXT: fcvt h19, s19 -; CHECK-SD-NEXT: fcvt s21, h21 -; CHECK-SD-NEXT: mov h0, v0.h[7] -; CHECK-SD-NEXT: bit v17.16b, v22.16b, v6.16b -; CHECK-SD-NEXT: mov h3, v3.h[7] -; CHECK-SD-NEXT: mov h1, v1.h[7] -; CHECK-SD-NEXT: bit v7.16b, v16.16b, v6.16b -; CHECK-SD-NEXT: mov v16.16b, v6.16b -; CHECK-SD-NEXT: fcvt s2, h2 -; CHECK-SD-NEXT: mov v4.h[4], v18.h[0] -; CHECK-SD-NEXT: mov v18.16b, v6.16b -; CHECK-SD-NEXT: mov v5.h[4], v19.h[0] -; CHECK-SD-NEXT: fcvt s0, h0 -; CHECK-SD-NEXT: bsl v16.16b, v23.16b, v20.16b -; CHECK-SD-NEXT: fcvt h17, s17 -; CHECK-SD-NEXT: fcvt s3, h3 -; CHECK-SD-NEXT: bsl v18.16b, v24.16b, v21.16b -; CHECK-SD-NEXT: fcvt h7, s7 -; CHECK-SD-NEXT: fcvt s1, h1 -; CHECK-SD-NEXT: bif v0.16b, v2.16b, v6.16b -; CHECK-SD-NEXT: mov v4.h[5], v17.h[0] -; CHECK-SD-NEXT: fcvt h2, s16 -; CHECK-SD-NEXT: mov v5.h[5], v7.h[0] -; CHECK-SD-NEXT: fcvt h7, s18 -; CHECK-SD-NEXT: bif v1.16b, v3.16b, v6.16b -; CHECK-SD-NEXT: fcvt h0, s0 -; CHECK-SD-NEXT: mov v4.h[6], v2.h[0] -; CHECK-SD-NEXT: mov v5.h[6], v7.h[0] -; CHECK-SD-NEXT: fcvt h1, s1 -; CHECK-SD-NEXT: mov v4.h[7], v0.h[0] -; CHECK-SD-NEXT: mov v5.h[7], v1.h[0] -; CHECK-SD-NEXT: mov v0.16b, v4.16b -; CHECK-SD-NEXT: mov v1.16b, v5.16b -; CHECK-SD-NEXT: ret -; -; CHECK-GI-LABEL: copysign_v16f16: -; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: mvni v4.8h, #128, lsl #8 -; CHECK-GI-NEXT: bif v0.16b, v2.16b, v4.16b -; CHECK-GI-NEXT: bif v1.16b, v3.16b, v4.16b -; CHECK-GI-NEXT: ret +; CHECK-LABEL: copysign_v16f16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: mvni v4.8h, #128, lsl #8 +; CHECK-NEXT: bif v0.16b, v2.16b, v4.16b +; CHECK-NEXT: bif v1.16b, v3.16b, v4.16b +; CHECK-NEXT: ret entry: %c = call <16 x half> @llvm.copysign.v16f16(<16 x half> %a, <16 x half> %b) ret <16 x half> %c diff --git a/llvm/test/CodeGen/AArch64/fneg.ll b/llvm/test/CodeGen/AArch64/fneg.ll index 7805512cbf45..d5010cf36084 100644 --- a/llvm/test/CodeGen/AArch64/fneg.ll +++ b/llvm/test/CodeGen/AArch64/fneg.ll @@ -27,9 +27,11 @@ entry: define half @fabs_f16(half %a) { ; CHECK-SD-NOFP16-LABEL: fabs_f16: ; CHECK-SD-NOFP16: // %bb.0: // %entry -; CHECK-SD-NOFP16-NEXT: fcvt s0, h0 -; CHECK-SD-NOFP16-NEXT: fneg s0, s0 -; CHECK-SD-NOFP16-NEXT: fcvt h0, s0 +; CHECK-SD-NOFP16-NEXT: // kill: def $h0 killed $h0 def $s0 +; CHECK-SD-NOFP16-NEXT: fmov w8, s0 +; CHECK-SD-NOFP16-NEXT: eor w8, w8, #0x8000 +; CHECK-SD-NOFP16-NEXT: fmov s0, w8 +; CHECK-SD-NOFP16-NEXT: // kill: def $h0 killed $h0 killed $s0 ; CHECK-SD-NOFP16-NEXT: ret ; ; CHECK-SD-FP16-LABEL: fabs_f16: @@ -148,45 +150,8 @@ entry: define <7 x half> @fabs_v7f16(<7 x half> %a) { ; CHECK-SD-NOFP16-LABEL: fabs_v7f16: ; CHECK-SD-NOFP16: // %bb.0: // %entry -; CHECK-SD-NOFP16-NEXT: mov h1, v0.h[1] -; CHECK-SD-NOFP16-NEXT: fcvt s2, h0 -; CHECK-SD-NOFP16-NEXT: mov h3, v0.h[2] -; CHECK-SD-NOFP16-NEXT: mov h4, v0.h[3] -; CHECK-SD-NOFP16-NEXT: mov h6, v0.h[4] -; CHECK-SD-NOFP16-NEXT: fcvt s1, h1 -; CHECK-SD-NOFP16-NEXT: fneg s2, s2 -; CHECK-SD-NOFP16-NEXT: fcvt s3, h3 -; CHECK-SD-NOFP16-NEXT: fcvt s4, h4 -; CHECK-SD-NOFP16-NEXT: fneg s5, s1 -; CHECK-SD-NOFP16-NEXT: fcvt h1, s2 -; CHECK-SD-NOFP16-NEXT: fneg s2, s3 -; CHECK-SD-NOFP16-NEXT: fneg s4, s4 -; CHECK-SD-NOFP16-NEXT: fcvt h3, s5 -; CHECK-SD-NOFP16-NEXT: fcvt s5, h6 -; CHECK-SD-NOFP16-NEXT: fcvt h2, s2 -; CHECK-SD-NOFP16-NEXT: fcvt h4, s4 -; CHECK-SD-NOFP16-NEXT: mov v1.h[1], v3.h[0] -; CHECK-SD-NOFP16-NEXT: mov h3, v0.h[5] -; CHECK-SD-NOFP16-NEXT: fneg s5, s5 -; CHECK-SD-NOFP16-NEXT: mov v1.h[2], v2.h[0] -; CHECK-SD-NOFP16-NEXT: mov h2, v0.h[6] -; CHECK-SD-NOFP16-NEXT: fcvt s3, h3 -; CHECK-SD-NOFP16-NEXT: mov h0, v0.h[7] -; CHECK-SD-NOFP16-NEXT: mov v1.h[3], v4.h[0] -; CHECK-SD-NOFP16-NEXT: fcvt h4, s5 -; CHECK-SD-NOFP16-NEXT: fcvt s2, h2 -; CHECK-SD-NOFP16-NEXT: fneg s3, s3 -; CHECK-SD-NOFP16-NEXT: fcvt s0, h0 -; CHECK-SD-NOFP16-NEXT: mov v1.h[4], v4.h[0] -; CHECK-SD-NOFP16-NEXT: fneg s2, s2 -; CHECK-SD-NOFP16-NEXT: fcvt h3, s3 -; CHECK-SD-NOFP16-NEXT: fneg s0, s0 -; CHECK-SD-NOFP16-NEXT: fcvt h2, s2 -; CHECK-SD-NOFP16-NEXT: mov v1.h[5], v3.h[0] -; CHECK-SD-NOFP16-NEXT: fcvt h0, s0 -; CHECK-SD-NOFP16-NEXT: mov v1.h[6], v2.h[0] -; CHECK-SD-NOFP16-NEXT: mov v1.h[7], v0.h[0] -; CHECK-SD-NOFP16-NEXT: mov v0.16b, v1.16b +; CHECK-SD-NOFP16-NEXT: movi v1.8h, #128, lsl #8 +; CHECK-SD-NOFP16-NEXT: eor v0.16b, v0.16b, v1.16b ; CHECK-SD-NOFP16-NEXT: ret ; ; CHECK-SD-FP16-LABEL: fabs_v7f16: @@ -234,26 +199,8 @@ entry: define <4 x half> @fabs_v4f16(<4 x half> %a) { ; CHECK-SD-NOFP16-LABEL: fabs_v4f16: ; CHECK-SD-NOFP16: // %bb.0: // %entry -; CHECK-SD-NOFP16-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-SD-NOFP16-NEXT: mov h1, v0.h[1] -; CHECK-SD-NOFP16-NEXT: fcvt s2, h0 -; CHECK-SD-NOFP16-NEXT: mov h3, v0.h[2] -; CHECK-SD-NOFP16-NEXT: mov h4, v0.h[3] -; CHECK-SD-NOFP16-NEXT: fcvt s1, h1 -; CHECK-SD-NOFP16-NEXT: fneg s0, s2 -; CHECK-SD-NOFP16-NEXT: fcvt s2, h3 -; CHECK-SD-NOFP16-NEXT: fcvt s3, h4 -; CHECK-SD-NOFP16-NEXT: fneg s1, s1 -; CHECK-SD-NOFP16-NEXT: fcvt h0, s0 -; CHECK-SD-NOFP16-NEXT: fneg s2, s2 -; CHECK-SD-NOFP16-NEXT: fcvt h1, s1 -; CHECK-SD-NOFP16-NEXT: mov v0.h[1], v1.h[0] -; CHECK-SD-NOFP16-NEXT: fcvt h1, s2 -; CHECK-SD-NOFP16-NEXT: fneg s2, s3 -; CHECK-SD-NOFP16-NEXT: mov v0.h[2], v1.h[0] -; CHECK-SD-NOFP16-NEXT: fcvt h1, s2 -; CHECK-SD-NOFP16-NEXT: mov v0.h[3], v1.h[0] -; CHECK-SD-NOFP16-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-SD-NOFP16-NEXT: movi v1.4h, #128, lsl #8 +; CHECK-SD-NOFP16-NEXT: eor v0.8b, v0.8b, v1.8b ; CHECK-SD-NOFP16-NEXT: ret ; ; CHECK-SD-FP16-LABEL: fabs_v4f16: @@ -280,45 +227,8 @@ entry: define <8 x half> @fabs_v8f16(<8 x half> %a) { ; CHECK-SD-NOFP16-LABEL: fabs_v8f16: ; CHECK-SD-NOFP16: // %bb.0: // %entry -; CHECK-SD-NOFP16-NEXT: mov h1, v0.h[1] -; CHECK-SD-NOFP16-NEXT: fcvt s2, h0 -; CHECK-SD-NOFP16-NEXT: mov h3, v0.h[2] -; CHECK-SD-NOFP16-NEXT: mov h4, v0.h[3] -; CHECK-SD-NOFP16-NEXT: mov h6, v0.h[4] -; CHECK-SD-NOFP16-NEXT: fcvt s1, h1 -; CHECK-SD-NOFP16-NEXT: fneg s2, s2 -; CHECK-SD-NOFP16-NEXT: fcvt s3, h3 -; CHECK-SD-NOFP16-NEXT: fcvt s4, h4 -; CHECK-SD-NOFP16-NEXT: fneg s5, s1 -; CHECK-SD-NOFP16-NEXT: fcvt h1, s2 -; CHECK-SD-NOFP16-NEXT: fneg s2, s3 -; CHECK-SD-NOFP16-NEXT: fneg s4, s4 -; CHECK-SD-NOFP16-NEXT: fcvt h3, s5 -; CHECK-SD-NOFP16-NEXT: fcvt s5, h6 -; CHECK-SD-NOFP16-NEXT: fcvt h2, s2 -; CHECK-SD-NOFP16-NEXT: fcvt h4, s4 -; CHECK-SD-NOFP16-NEXT: mov v1.h[1], v3.h[0] -; CHECK-SD-NOFP16-NEXT: mov h3, v0.h[5] -; CHECK-SD-NOFP16-NEXT: fneg s5, s5 -; CHECK-SD-NOFP16-NEXT: mov v1.h[2], v2.h[0] -; CHECK-SD-NOFP16-NEXT: mov h2, v0.h[6] -; CHECK-SD-NOFP16-NEXT: fcvt s3, h3 -; CHECK-SD-NOFP16-NEXT: mov h0, v0.h[7] -; CHECK-SD-NOFP16-NEXT: mov v1.h[3], v4.h[0] -; CHECK-SD-NOFP16-NEXT: fcvt h4, s5 -; CHECK-SD-NOFP16-NEXT: fcvt s2, h2 -; CHECK-SD-NOFP16-NEXT: fneg s3, s3 -; CHECK-SD-NOFP16-NEXT: fcvt s0, h0 -; CHECK-SD-NOFP16-NEXT: mov v1.h[4], v4.h[0] -; CHECK-SD-NOFP16-NEXT: fneg s2, s2 -; CHECK-SD-NOFP16-NEXT: fcvt h3, s3 -; CHECK-SD-NOFP16-NEXT: fneg s0, s0 -; CHECK-SD-NOFP16-NEXT: fcvt h2, s2 -; CHECK-SD-NOFP16-NEXT: mov v1.h[5], v3.h[0] -; CHECK-SD-NOFP16-NEXT: fcvt h0, s0 -; CHECK-SD-NOFP16-NEXT: mov v1.h[6], v2.h[0] -; CHECK-SD-NOFP16-NEXT: mov v1.h[7], v0.h[0] -; CHECK-SD-NOFP16-NEXT: mov v0.16b, v1.16b +; CHECK-SD-NOFP16-NEXT: movi v1.8h, #128, lsl #8 +; CHECK-SD-NOFP16-NEXT: eor v0.16b, v0.16b, v1.16b ; CHECK-SD-NOFP16-NEXT: ret ; ; CHECK-SD-FP16-LABEL: fabs_v8f16: @@ -348,84 +258,9 @@ entry: define <16 x half> @fabs_v16f16(<16 x half> %a) { ; CHECK-SD-NOFP16-LABEL: fabs_v16f16: ; CHECK-SD-NOFP16: // %bb.0: // %entry -; CHECK-SD-NOFP16-NEXT: mov h2, v0.h[1] -; CHECK-SD-NOFP16-NEXT: mov h3, v1.h[1] -; CHECK-SD-NOFP16-NEXT: fcvt s4, h0 -; CHECK-SD-NOFP16-NEXT: mov h5, v0.h[2] -; CHECK-SD-NOFP16-NEXT: fcvt s6, h1 -; CHECK-SD-NOFP16-NEXT: mov h7, v1.h[2] -; CHECK-SD-NOFP16-NEXT: mov h16, v0.h[3] -; CHECK-SD-NOFP16-NEXT: mov h17, v1.h[3] -; CHECK-SD-NOFP16-NEXT: mov h20, v0.h[4] -; CHECK-SD-NOFP16-NEXT: mov h21, v1.h[4] -; CHECK-SD-NOFP16-NEXT: fcvt s2, h2 -; CHECK-SD-NOFP16-NEXT: fcvt s3, h3 -; CHECK-SD-NOFP16-NEXT: fneg s4, s4 -; CHECK-SD-NOFP16-NEXT: fcvt s5, h5 -; CHECK-SD-NOFP16-NEXT: fneg s6, s6 -; CHECK-SD-NOFP16-NEXT: fcvt s7, h7 -; CHECK-SD-NOFP16-NEXT: fcvt s16, h16 -; CHECK-SD-NOFP16-NEXT: fcvt s17, h17 -; CHECK-SD-NOFP16-NEXT: fneg s18, s2 -; CHECK-SD-NOFP16-NEXT: fneg s19, s3 -; CHECK-SD-NOFP16-NEXT: fcvt h2, s4 -; CHECK-SD-NOFP16-NEXT: fneg s4, s5 -; CHECK-SD-NOFP16-NEXT: fcvt h3, s6 -; CHECK-SD-NOFP16-NEXT: fneg s6, s7 -; CHECK-SD-NOFP16-NEXT: fneg s16, s16 -; CHECK-SD-NOFP16-NEXT: fcvt h5, s18 -; CHECK-SD-NOFP16-NEXT: fcvt h7, s19 -; CHECK-SD-NOFP16-NEXT: fcvt s18, h20 -; CHECK-SD-NOFP16-NEXT: fcvt s19, h21 -; CHECK-SD-NOFP16-NEXT: fcvt h4, s4 -; CHECK-SD-NOFP16-NEXT: fcvt h6, s6 -; CHECK-SD-NOFP16-NEXT: fcvt h16, s16 -; CHECK-SD-NOFP16-NEXT: mov v2.h[1], v5.h[0] -; CHECK-SD-NOFP16-NEXT: fneg s5, s17 -; CHECK-SD-NOFP16-NEXT: mov v3.h[1], v7.h[0] -; CHECK-SD-NOFP16-NEXT: mov h7, v0.h[5] -; CHECK-SD-NOFP16-NEXT: mov h17, v1.h[5] -; CHECK-SD-NOFP16-NEXT: fneg s18, s18 -; CHECK-SD-NOFP16-NEXT: mov v2.h[2], v4.h[0] -; CHECK-SD-NOFP16-NEXT: fneg s4, s19 -; CHECK-SD-NOFP16-NEXT: fcvt h5, s5 -; CHECK-SD-NOFP16-NEXT: mov v3.h[2], v6.h[0] -; CHECK-SD-NOFP16-NEXT: mov h6, v0.h[6] -; CHECK-SD-NOFP16-NEXT: fcvt s7, h7 -; CHECK-SD-NOFP16-NEXT: fcvt s17, h17 -; CHECK-SD-NOFP16-NEXT: mov h19, v1.h[6] -; CHECK-SD-NOFP16-NEXT: mov h0, v0.h[7] -; CHECK-SD-NOFP16-NEXT: mov h1, v1.h[7] -; CHECK-SD-NOFP16-NEXT: mov v2.h[3], v16.h[0] -; CHECK-SD-NOFP16-NEXT: fcvt h4, s4 -; CHECK-SD-NOFP16-NEXT: mov v3.h[3], v5.h[0] -; CHECK-SD-NOFP16-NEXT: fcvt h5, s18 -; CHECK-SD-NOFP16-NEXT: fcvt s6, h6 -; CHECK-SD-NOFP16-NEXT: fneg s7, s7 -; CHECK-SD-NOFP16-NEXT: fneg s16, s17 -; CHECK-SD-NOFP16-NEXT: fcvt s17, h19 -; CHECK-SD-NOFP16-NEXT: fcvt s0, h0 -; CHECK-SD-NOFP16-NEXT: fcvt s1, h1 -; CHECK-SD-NOFP16-NEXT: mov v2.h[4], v5.h[0] -; CHECK-SD-NOFP16-NEXT: mov v3.h[4], v4.h[0] -; CHECK-SD-NOFP16-NEXT: fneg s4, s6 -; CHECK-SD-NOFP16-NEXT: fcvt h5, s7 -; CHECK-SD-NOFP16-NEXT: fcvt h6, s16 -; CHECK-SD-NOFP16-NEXT: fneg s7, s17 -; CHECK-SD-NOFP16-NEXT: fneg s0, s0 -; CHECK-SD-NOFP16-NEXT: fneg s1, s1 -; CHECK-SD-NOFP16-NEXT: fcvt h4, s4 -; CHECK-SD-NOFP16-NEXT: mov v2.h[5], v5.h[0] -; CHECK-SD-NOFP16-NEXT: mov v3.h[5], v6.h[0] -; CHECK-SD-NOFP16-NEXT: fcvt h5, s7 -; CHECK-SD-NOFP16-NEXT: fcvt h0, s0 -; CHECK-SD-NOFP16-NEXT: fcvt h1, s1 -; CHECK-SD-NOFP16-NEXT: mov v2.h[6], v4.h[0] -; CHECK-SD-NOFP16-NEXT: mov v3.h[6], v5.h[0] -; CHECK-SD-NOFP16-NEXT: mov v2.h[7], v0.h[0] -; CHECK-SD-NOFP16-NEXT: mov v3.h[7], v1.h[0] -; CHECK-SD-NOFP16-NEXT: mov v0.16b, v2.16b -; CHECK-SD-NOFP16-NEXT: mov v1.16b, v3.16b +; CHECK-SD-NOFP16-NEXT: movi v2.8h, #128, lsl #8 +; CHECK-SD-NOFP16-NEXT: eor v0.16b, v0.16b, v2.16b +; CHECK-SD-NOFP16-NEXT: eor v1.16b, v1.16b, v2.16b ; CHECK-SD-NOFP16-NEXT: ret ; ; CHECK-SD-FP16-LABEL: fabs_v16f16: diff --git a/llvm/test/CodeGen/AArch64/vector-fcopysign.ll b/llvm/test/CodeGen/AArch64/vector-fcopysign.ll index d01ca881545c..c33759331bbc 100644 --- a/llvm/test/CodeGen/AArch64/vector-fcopysign.ll +++ b/llvm/test/CodeGen/AArch64/vector-fcopysign.ll @@ -185,142 +185,42 @@ declare <4 x double> @llvm.copysign.v4f64(<4 x double> %a, <4 x double> %b) #0 ;============ v4f16 define <4 x half> @test_copysign_v4f16_v4f16(<4 x half> %a, <4 x half> %b) #0 { -; NOFP16-LABEL: test_copysign_v4f16_v4f16: -; NOFP16: ; %bb.0: -; NOFP16-NEXT: ; kill: def $d1 killed $d1 def $q1 -; NOFP16-NEXT: ; kill: def $d0 killed $d0 def $q0 -; NOFP16-NEXT: mov h3, v1[1] -; NOFP16-NEXT: mov h4, v0[1] -; NOFP16-NEXT: mov h5, v1[2] -; NOFP16-NEXT: mov h6, v0[2] -; NOFP16-NEXT: mvni.4s v2, #128, lsl #24 -; NOFP16-NEXT: fcvt s7, h1 -; NOFP16-NEXT: fcvt s16, h0 -; NOFP16-NEXT: mov h1, v1[3] -; NOFP16-NEXT: fcvt s3, h3 -; NOFP16-NEXT: fcvt s4, h4 -; NOFP16-NEXT: fcvt s1, h1 -; NOFP16-NEXT: bit.16b v3, v4, v2 -; NOFP16-NEXT: fcvt s4, h5 -; NOFP16-NEXT: fcvt s5, h6 -; NOFP16-NEXT: mov.16b v6, v2 -; NOFP16-NEXT: bsl.16b v6, v16, v7 -; NOFP16-NEXT: mov h7, v0[3] -; NOFP16-NEXT: bit.16b v4, v5, v2 -; NOFP16-NEXT: fcvt h3, s3 -; NOFP16-NEXT: fcvt h0, s6 -; NOFP16-NEXT: fcvt s5, h7 -; NOFP16-NEXT: mov.h v0[1], v3[0] -; NOFP16-NEXT: fcvt h3, s4 -; NOFP16-NEXT: bit.16b v1, v5, v2 -; NOFP16-NEXT: mov.h v0[2], v3[0] -; NOFP16-NEXT: fcvt h1, s1 -; NOFP16-NEXT: mov.h v0[3], v1[0] -; NOFP16-NEXT: ; kill: def $d0 killed $d0 killed $q0 -; NOFP16-NEXT: ret -; -; FP16-LABEL: test_copysign_v4f16_v4f16: -; FP16: ; %bb.0: -; FP16-NEXT: mvni.4h v2, #128, lsl #8 -; FP16-NEXT: bif.8b v0, v1, v2 -; FP16-NEXT: ret +; CHECK-LABEL: test_copysign_v4f16_v4f16: +; CHECK: ; %bb.0: +; CHECK-NEXT: mvni.4h v2, #128, lsl #8 +; CHECK-NEXT: bif.8b v0, v1, v2 +; CHECK-NEXT: ret %r = call <4 x half> @llvm.copysign.v4f16(<4 x half> %a, <4 x half> %b) ret <4 x half> %r } define <4 x half> @test_copysign_v4f16_v4f32(<4 x half> %a, <4 x float> %b) #0 { -; NOFP16-LABEL: test_copysign_v4f16_v4f32: -; NOFP16: ; %bb.0: -; NOFP16-NEXT: fcvtn v1.4h, v1.4s -; NOFP16-NEXT: ; kill: def $d0 killed $d0 def $q0 -; NOFP16-NEXT: mov h3, v0[1] -; NOFP16-NEXT: mov h5, v0[2] -; NOFP16-NEXT: mvni.4s v2, #128, lsl #24 -; NOFP16-NEXT: fcvt s7, h0 -; NOFP16-NEXT: mov h4, v1[1] -; NOFP16-NEXT: fcvt s3, h3 -; NOFP16-NEXT: mov h6, v1[2] -; NOFP16-NEXT: fcvt s16, h1 -; NOFP16-NEXT: mov h1, v1[3] -; NOFP16-NEXT: fcvt s4, h4 -; NOFP16-NEXT: fcvt s1, h1 -; NOFP16-NEXT: bif.16b v3, v4, v2 -; NOFP16-NEXT: fcvt s4, h5 -; NOFP16-NEXT: fcvt s5, h6 -; NOFP16-NEXT: mov.16b v6, v2 -; NOFP16-NEXT: bsl.16b v6, v7, v16 -; NOFP16-NEXT: mov h7, v0[3] -; NOFP16-NEXT: bif.16b v4, v5, v2 -; NOFP16-NEXT: fcvt h3, s3 -; NOFP16-NEXT: fcvt h0, s6 -; NOFP16-NEXT: fcvt s5, h7 -; NOFP16-NEXT: mov.h v0[1], v3[0] -; NOFP16-NEXT: fcvt h3, s4 -; NOFP16-NEXT: bit.16b v1, v5, v2 -; NOFP16-NEXT: mov.h v0[2], v3[0] -; NOFP16-NEXT: fcvt h1, s1 -; NOFP16-NEXT: mov.h v0[3], v1[0] -; NOFP16-NEXT: ; kill: def $d0 killed $d0 killed $q0 -; NOFP16-NEXT: ret -; -; FP16-LABEL: test_copysign_v4f16_v4f32: -; FP16: ; %bb.0: -; FP16-NEXT: fcvtn v1.4h, v1.4s -; FP16-NEXT: mvni.4h v2, #128, lsl #8 -; FP16-NEXT: bif.8b v0, v1, v2 -; FP16-NEXT: ret +; CHECK-LABEL: test_copysign_v4f16_v4f32: +; CHECK: ; %bb.0: +; CHECK-NEXT: fcvtn v1.4h, v1.4s +; CHECK-NEXT: mvni.4h v2, #128, lsl #8 +; CHECK-NEXT: bif.8b v0, v1, v2 +; CHECK-NEXT: ret %tmp0 = fptrunc <4 x float> %b to <4 x half> %r = call <4 x half> @llvm.copysign.v4f16(<4 x half> %a, <4 x half> %tmp0) ret <4 x half> %r } define <4 x half> @test_copysign_v4f16_v4f64(<4 x half> %a, <4 x double> %b) #0 { -; NOFP16-LABEL: test_copysign_v4f16_v4f64: -; NOFP16: ; %bb.0: -; NOFP16-NEXT: ; kill: def $d0 killed $d0 def $q0 -; NOFP16-NEXT: mov d3, v1[1] -; NOFP16-NEXT: mov h4, v0[1] -; NOFP16-NEXT: fcvt s1, d1 -; NOFP16-NEXT: fcvt s5, h0 -; NOFP16-NEXT: mov h7, v0[2] -; NOFP16-NEXT: mvni.4s v6, #128, lsl #24 -; NOFP16-NEXT: fcvt s3, d3 -; NOFP16-NEXT: fcvt s4, h4 -; NOFP16-NEXT: bit.16b v1, v5, v6 -; NOFP16-NEXT: fcvt s7, h7 -; NOFP16-NEXT: mov h5, v0[3] -; NOFP16-NEXT: bit.16b v3, v4, v6 -; NOFP16-NEXT: mov d4, v2[1] -; NOFP16-NEXT: fcvt s2, d2 -; NOFP16-NEXT: fcvt h0, s1 -; NOFP16-NEXT: fcvt h1, s3 -; NOFP16-NEXT: bit.16b v2, v7, v6 -; NOFP16-NEXT: fcvt s3, d4 -; NOFP16-NEXT: fcvt s4, h5 -; NOFP16-NEXT: mov.h v0[1], v1[0] -; NOFP16-NEXT: fcvt h1, s2 -; NOFP16-NEXT: mov.16b v2, v6 -; NOFP16-NEXT: bsl.16b v2, v4, v3 -; NOFP16-NEXT: mov.h v0[2], v1[0] -; NOFP16-NEXT: fcvt h1, s2 -; NOFP16-NEXT: mov.h v0[3], v1[0] -; NOFP16-NEXT: ; kill: def $d0 killed $d0 killed $q0 -; NOFP16-NEXT: ret -; -; FP16-LABEL: test_copysign_v4f16_v4f64: -; FP16: ; %bb.0: -; FP16-NEXT: mov d3, v1[1] -; FP16-NEXT: fcvt h1, d1 -; FP16-NEXT: fcvt h3, d3 -; FP16-NEXT: mov.h v1[1], v3[0] -; FP16-NEXT: fcvt h3, d2 -; FP16-NEXT: mov d2, v2[1] -; FP16-NEXT: mov.h v1[2], v3[0] -; FP16-NEXT: fcvt h2, d2 -; FP16-NEXT: mov.h v1[3], v2[0] -; FP16-NEXT: mvni.4h v2, #128, lsl #8 -; FP16-NEXT: bif.8b v0, v1, v2 -; FP16-NEXT: ret +; CHECK-LABEL: test_copysign_v4f16_v4f64: +; CHECK: ; %bb.0: +; CHECK-NEXT: mov d3, v1[1] +; CHECK-NEXT: fcvt h1, d1 +; CHECK-NEXT: fcvt h3, d3 +; CHECK-NEXT: mov.h v1[1], v3[0] +; CHECK-NEXT: fcvt h3, d2 +; CHECK-NEXT: mov d2, v2[1] +; CHECK-NEXT: mov.h v1[2], v3[0] +; CHECK-NEXT: fcvt h2, d2 +; CHECK-NEXT: mov.h v1[3], v2[0] +; CHECK-NEXT: mvni.4h v2, #128, lsl #8 +; CHECK-NEXT: bif.8b v0, v1, v2 +; CHECK-NEXT: ret %tmp0 = fptrunc <4 x double> %b to <4 x half> %r = call <4 x half> @llvm.copysign.v4f16(<4 x half> %a, <4 x half> %tmp0) ret <4 x half> %r @@ -331,145 +231,23 @@ declare <4 x half> @llvm.copysign.v4f16(<4 x half> %a, <4 x half> %b) #0 ;============ v8f16 define <8 x half> @test_copysign_v8f16_v8f16(<8 x half> %a, <8 x half> %b) #0 { -; NOFP16-LABEL: test_copysign_v8f16_v8f16: -; NOFP16: ; %bb.0: -; NOFP16-NEXT: mov h2, v1[1] -; NOFP16-NEXT: mov h4, v0[1] -; NOFP16-NEXT: fcvt s5, h1 -; NOFP16-NEXT: fcvt s6, h0 -; NOFP16-NEXT: mvni.4s v3, #128, lsl #24 -; NOFP16-NEXT: mov h7, v1[2] -; NOFP16-NEXT: mov h16, v0[2] -; NOFP16-NEXT: mov h17, v1[3] -; NOFP16-NEXT: fcvt s2, h2 -; NOFP16-NEXT: fcvt s4, h4 -; NOFP16-NEXT: bit.16b v5, v6, v3 -; NOFP16-NEXT: mov h6, v0[3] -; NOFP16-NEXT: fcvt s7, h7 -; NOFP16-NEXT: fcvt s16, h16 -; NOFP16-NEXT: fcvt s17, h17 -; NOFP16-NEXT: bif.16b v4, v2, v3 -; NOFP16-NEXT: fcvt h2, s5 -; NOFP16-NEXT: mov.16b v5, v3 -; NOFP16-NEXT: fcvt s6, h6 -; NOFP16-NEXT: bsl.16b v5, v16, v7 -; NOFP16-NEXT: fcvt h4, s4 -; NOFP16-NEXT: mov h7, v1[4] -; NOFP16-NEXT: mov h16, v0[4] -; NOFP16-NEXT: bif.16b v6, v17, v3 -; NOFP16-NEXT: mov h17, v0[5] -; NOFP16-NEXT: fcvt h5, s5 -; NOFP16-NEXT: mov.h v2[1], v4[0] -; NOFP16-NEXT: fcvt s4, h7 -; NOFP16-NEXT: fcvt s7, h16 -; NOFP16-NEXT: mov h16, v1[5] -; NOFP16-NEXT: fcvt h6, s6 -; NOFP16-NEXT: fcvt s17, h17 -; NOFP16-NEXT: mov.h v2[2], v5[0] -; NOFP16-NEXT: mov h5, v1[6] -; NOFP16-NEXT: mov h1, v1[7] -; NOFP16-NEXT: bit.16b v4, v7, v3 -; NOFP16-NEXT: mov h7, v0[6] -; NOFP16-NEXT: fcvt s16, h16 -; NOFP16-NEXT: mov h0, v0[7] -; NOFP16-NEXT: mov.h v2[3], v6[0] -; NOFP16-NEXT: fcvt s5, h5 -; NOFP16-NEXT: fcvt s1, h1 -; NOFP16-NEXT: fcvt s6, h7 -; NOFP16-NEXT: mov.16b v7, v3 -; NOFP16-NEXT: fcvt h4, s4 -; NOFP16-NEXT: fcvt s0, h0 -; NOFP16-NEXT: bsl.16b v7, v17, v16 -; NOFP16-NEXT: bit.16b v5, v6, v3 -; NOFP16-NEXT: mov.h v2[4], v4[0] -; NOFP16-NEXT: bif.16b v0, v1, v3 -; NOFP16-NEXT: fcvt h4, s7 -; NOFP16-NEXT: fcvt h0, s0 -; NOFP16-NEXT: mov.h v2[5], v4[0] -; NOFP16-NEXT: fcvt h4, s5 -; NOFP16-NEXT: mov.h v2[6], v4[0] -; NOFP16-NEXT: mov.h v2[7], v0[0] -; NOFP16-NEXT: mov.16b v0, v2 -; NOFP16-NEXT: ret -; -; FP16-LABEL: test_copysign_v8f16_v8f16: -; FP16: ; %bb.0: -; FP16-NEXT: mvni.8h v2, #128, lsl #8 -; FP16-NEXT: bif.16b v0, v1, v2 -; FP16-NEXT: ret +; CHECK-LABEL: test_copysign_v8f16_v8f16: +; CHECK: ; %bb.0: +; CHECK-NEXT: mvni.8h v2, #128, lsl #8 +; CHECK-NEXT: bif.16b v0, v1, v2 +; CHECK-NEXT: ret %r = call <8 x half> @llvm.copysign.v8f16(<8 x half> %a, <8 x half> %b) ret <8 x half> %r } define <8 x half> @test_copysign_v8f16_v8f32(<8 x half> %a, <8 x float> %b) #0 { -; NOFP16-LABEL: test_copysign_v8f16_v8f32: -; NOFP16: ; %bb.0: -; NOFP16-NEXT: fcvtn v1.4h, v1.4s -; NOFP16-NEXT: mov h4, v0[1] -; NOFP16-NEXT: fcvt s6, h0 -; NOFP16-NEXT: mvni.4s v3, #128, lsl #24 -; NOFP16-NEXT: mov h7, v0[2] -; NOFP16-NEXT: fcvtn v2.4h, v2.4s -; NOFP16-NEXT: mov h5, v1[1] -; NOFP16-NEXT: fcvt s16, h1 -; NOFP16-NEXT: fcvt s4, h4 -; NOFP16-NEXT: mov h17, v1[2] -; NOFP16-NEXT: mov h1, v1[3] -; NOFP16-NEXT: fcvt s7, h7 -; NOFP16-NEXT: fcvt s5, h5 -; NOFP16-NEXT: bif.16b v6, v16, v3 -; NOFP16-NEXT: mov h16, v0[3] -; NOFP16-NEXT: fcvt s17, h17 -; NOFP16-NEXT: fcvt s18, h1 -; NOFP16-NEXT: bif.16b v4, v5, v3 -; NOFP16-NEXT: fcvt h1, s6 -; NOFP16-NEXT: mov.16b v6, v3 -; NOFP16-NEXT: mov h5, v0[4] -; NOFP16-NEXT: fcvt s16, h16 -; NOFP16-NEXT: bsl.16b v6, v7, v17 -; NOFP16-NEXT: mov h7, v0[5] -; NOFP16-NEXT: mov h17, v2[1] -; NOFP16-NEXT: fcvt h4, s4 -; NOFP16-NEXT: fcvt s5, h5 -; NOFP16-NEXT: bif.16b v16, v18, v3 -; NOFP16-NEXT: fcvt h6, s6 -; NOFP16-NEXT: fcvt s7, h7 -; NOFP16-NEXT: fcvt s17, h17 -; NOFP16-NEXT: mov.h v1[1], v4[0] -; NOFP16-NEXT: fcvt s4, h2 -; NOFP16-NEXT: bif.16b v7, v17, v3 -; NOFP16-NEXT: bit.16b v4, v5, v3 -; NOFP16-NEXT: fcvt h5, s16 -; NOFP16-NEXT: mov.h v1[2], v6[0] -; NOFP16-NEXT: mov h6, v0[6] -; NOFP16-NEXT: mov h16, v2[2] -; NOFP16-NEXT: mov h0, v0[7] -; NOFP16-NEXT: mov h2, v2[3] -; NOFP16-NEXT: mov.h v1[3], v5[0] -; NOFP16-NEXT: fcvt h4, s4 -; NOFP16-NEXT: fcvt s5, h6 -; NOFP16-NEXT: fcvt s6, h16 -; NOFP16-NEXT: fcvt s0, h0 -; NOFP16-NEXT: fcvt s2, h2 -; NOFP16-NEXT: mov.h v1[4], v4[0] -; NOFP16-NEXT: fcvt h4, s7 -; NOFP16-NEXT: bif.16b v5, v6, v3 -; NOFP16-NEXT: bif.16b v0, v2, v3 -; NOFP16-NEXT: mov.h v1[5], v4[0] -; NOFP16-NEXT: fcvt h4, s5 -; NOFP16-NEXT: fcvt h0, s0 -; NOFP16-NEXT: mov.h v1[6], v4[0] -; NOFP16-NEXT: mov.h v1[7], v0[0] -; NOFP16-NEXT: mov.16b v0, v1 -; NOFP16-NEXT: ret -; -; FP16-LABEL: test_copysign_v8f16_v8f32: -; FP16: ; %bb.0: -; FP16-NEXT: fcvtn v1.4h, v1.4s -; FP16-NEXT: fcvtn2 v1.8h, v2.4s -; FP16-NEXT: mvni.8h v2, #128, lsl #8 -; FP16-NEXT: bif.16b v0, v1, v2 -; FP16-NEXT: ret +; CHECK-LABEL: test_copysign_v8f16_v8f32: +; CHECK: ; %bb.0: +; CHECK-NEXT: fcvtn v1.4h, v1.4s +; CHECK-NEXT: fcvtn2 v1.8h, v2.4s +; CHECK-NEXT: mvni.8h v2, #128, lsl #8 +; CHECK-NEXT: bif.16b v0, v1, v2 +; CHECK-NEXT: ret %tmp0 = fptrunc <8 x float> %b to <8 x half> %r = call <8 x half> @llvm.copysign.v8f16(<8 x half> %a, <8 x half> %tmp0) ret <8 x half> %r @@ -482,60 +260,8 @@ declare <8 x half> @llvm.copysign.v8f16(<8 x half> %a, <8 x half> %b) #0 define <4 x bfloat> @test_copysign_v4bf16_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) #0 { ; CHECK-LABEL: test_copysign_v4bf16_v4bf16: ; CHECK: ; %bb.0: -; CHECK-NEXT: ; kill: def $d1 killed $d1 def $q1 -; CHECK-NEXT: ; kill: def $d0 killed $d0 def $q0 -; CHECK-NEXT: mov h3, v1[1] -; CHECK-NEXT: mov h4, v0[1] -; CHECK-NEXT: fmov w8, s1 -; CHECK-NEXT: mov h5, v1[2] -; CHECK-NEXT: mov h6, v0[2] -; CHECK-NEXT: fmov w11, s0 -; CHECK-NEXT: mvni.4s v2, #128, lsl #24 -; CHECK-NEXT: mov h1, v1[3] -; CHECK-NEXT: mov h0, v0[3] -; CHECK-NEXT: lsl w8, w8, #16 -; CHECK-NEXT: fmov w9, s3 -; CHECK-NEXT: lsl w11, w11, #16 -; CHECK-NEXT: fmov w10, s4 -; CHECK-NEXT: fmov s7, w8 -; CHECK-NEXT: fmov w8, s5 -; CHECK-NEXT: lsl w9, w9, #16 -; CHECK-NEXT: lsl w10, w10, #16 -; CHECK-NEXT: lsl w8, w8, #16 -; CHECK-NEXT: fmov s3, w9 -; CHECK-NEXT: fmov s4, w10 -; CHECK-NEXT: fmov w9, s6 -; CHECK-NEXT: fmov w10, s1 -; CHECK-NEXT: bit.16b v3, v4, v2 -; CHECK-NEXT: lsl w9, w9, #16 -; CHECK-NEXT: fmov s4, w11 -; CHECK-NEXT: fmov w11, s0 -; CHECK-NEXT: fmov s0, w8 -; CHECK-NEXT: lsl w10, w10, #16 -; CHECK-NEXT: fmov s1, w9 -; CHECK-NEXT: bif.16b v4, v7, v2 -; CHECK-NEXT: fmov w8, s3 -; CHECK-NEXT: lsl w11, w11, #16 -; CHECK-NEXT: bif.16b v1, v0, v2 -; CHECK-NEXT: fmov s5, w11 -; CHECK-NEXT: lsr w8, w8, #16 -; CHECK-NEXT: fmov w9, s4 -; CHECK-NEXT: fmov s4, w10 -; CHECK-NEXT: fmov s3, w8 -; CHECK-NEXT: fmov w8, s1 -; CHECK-NEXT: mov.16b v1, v2 -; CHECK-NEXT: lsr w9, w9, #16 -; CHECK-NEXT: bsl.16b v1, v5, v4 -; CHECK-NEXT: fmov s0, w9 -; CHECK-NEXT: lsr w8, w8, #16 -; CHECK-NEXT: fmov s2, w8 -; CHECK-NEXT: mov.h v0[1], v3[0] -; CHECK-NEXT: fmov w8, s1 -; CHECK-NEXT: lsr w8, w8, #16 -; CHECK-NEXT: mov.h v0[2], v2[0] -; CHECK-NEXT: fmov s1, w8 -; CHECK-NEXT: mov.h v0[3], v1[0] -; CHECK-NEXT: ; kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: mvni.4h v2, #128, lsl #8 +; CHECK-NEXT: bif.8b v0, v1, v2 ; CHECK-NEXT: ret %r = call <4 x bfloat> @llvm.copysign.v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) ret <4 x bfloat> %r @@ -546,66 +272,16 @@ define <4 x bfloat> @test_copysign_v4bf16_v4f32(<4 x bfloat> %a, <4 x float> %b) ; CHECK: ; %bb.0: ; CHECK-NEXT: movi.4s v2, #127, msl #8 ; CHECK-NEXT: movi.4s v3, #1 -; CHECK-NEXT: ; kill: def $d0 killed $d0 def $q0 ; CHECK-NEXT: ushr.4s v4, v1, #16 -; CHECK-NEXT: fmov w9, s0 -; CHECK-NEXT: mov h5, v0[2] -; CHECK-NEXT: mov h6, v0[3] ; CHECK-NEXT: add.4s v2, v1, v2 ; CHECK-NEXT: and.16b v3, v4, v3 -; CHECK-NEXT: fcmeq.4s v4, v1, v1 -; CHECK-NEXT: orr.4s v1, #64, lsl #16 -; CHECK-NEXT: lsl w9, w9, #16 ; CHECK-NEXT: add.4s v2, v3, v2 -; CHECK-NEXT: mov h3, v0[1] -; CHECK-NEXT: bit.16b v1, v2, v4 -; CHECK-NEXT: fmov w8, s3 -; CHECK-NEXT: lsl w8, w8, #16 -; CHECK-NEXT: shrn.4h v2, v1, #16 -; CHECK-NEXT: mvni.4s v1, #128, lsl #24 -; CHECK-NEXT: fmov s3, w8 -; CHECK-NEXT: fmov w8, s5 -; CHECK-NEXT: fmov s5, w9 -; CHECK-NEXT: mov h4, v2[1] -; CHECK-NEXT: mov h0, v2[2] -; CHECK-NEXT: fmov w11, s2 -; CHECK-NEXT: mov h2, v2[3] -; CHECK-NEXT: lsl w8, w8, #16 -; CHECK-NEXT: lsl w11, w11, #16 -; CHECK-NEXT: fmov w10, s4 -; CHECK-NEXT: fmov w9, s0 -; CHECK-NEXT: fmov s0, w8 -; CHECK-NEXT: fmov w8, s2 -; CHECK-NEXT: lsl w10, w10, #16 -; CHECK-NEXT: lsl w9, w9, #16 -; CHECK-NEXT: lsl w8, w8, #16 -; CHECK-NEXT: fmov s4, w10 -; CHECK-NEXT: fmov w10, s6 -; CHECK-NEXT: fmov s2, w9 -; CHECK-NEXT: bif.16b v3, v4, v1 -; CHECK-NEXT: fmov s4, w11 -; CHECK-NEXT: bit.16b v2, v0, v1 -; CHECK-NEXT: lsl w10, w10, #16 -; CHECK-NEXT: bit.16b v4, v5, v1 -; CHECK-NEXT: fmov s5, w8 -; CHECK-NEXT: fmov w9, s3 -; CHECK-NEXT: fmov w8, s2 -; CHECK-NEXT: fmov w11, s4 -; CHECK-NEXT: fmov s4, w10 -; CHECK-NEXT: lsr w9, w9, #16 -; CHECK-NEXT: lsr w8, w8, #16 -; CHECK-NEXT: fmov s3, w9 -; CHECK-NEXT: lsr w11, w11, #16 -; CHECK-NEXT: bsl.16b v1, v4, v5 -; CHECK-NEXT: fmov s2, w8 -; CHECK-NEXT: fmov s0, w11 -; CHECK-NEXT: fmov w8, s1 -; CHECK-NEXT: mov.h v0[1], v3[0] -; CHECK-NEXT: lsr w8, w8, #16 -; CHECK-NEXT: mov.h v0[2], v2[0] -; CHECK-NEXT: fmov s1, w8 -; CHECK-NEXT: mov.h v0[3], v1[0] -; CHECK-NEXT: ; kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: fcmeq.4s v3, v1, v1 +; CHECK-NEXT: orr.4s v1, #64, lsl #16 +; CHECK-NEXT: bit.16b v1, v2, v3 +; CHECK-NEXT: mvni.4h v2, #128, lsl #8 +; CHECK-NEXT: shrn.4h v1, v1, #16 +; CHECK-NEXT: bif.8b v0, v1, v2 ; CHECK-NEXT: ret %tmp0 = fptrunc <4 x float> %b to <4 x bfloat> %r = call <4 x bfloat> @llvm.copysign.v4bf16(<4 x bfloat> %a, <4 x bfloat> %tmp0) @@ -615,51 +291,42 @@ define <4 x bfloat> @test_copysign_v4bf16_v4f32(<4 x bfloat> %a, <4 x float> %b) define <4 x bfloat> @test_copysign_v4bf16_v4f64(<4 x bfloat> %a, <4 x double> %b) #0 { ; CHECK-LABEL: test_copysign_v4bf16_v4f64: ; CHECK: ; %bb.0: -; CHECK-NEXT: ; kill: def $d0 killed $d0 def $q0 -; CHECK-NEXT: fmov w8, s0 -; CHECK-NEXT: mov h4, v0[1] -; CHECK-NEXT: mov h5, v0[2] ; CHECK-NEXT: mov d3, v1[1] -; CHECK-NEXT: fcvt s1, d1 -; CHECK-NEXT: mov h0, v0[3] -; CHECK-NEXT: lsl w8, w8, #16 -; CHECK-NEXT: fmov w9, s4 -; CHECK-NEXT: mvni.4s v4, #128, lsl #24 -; CHECK-NEXT: fmov s6, w8 -; CHECK-NEXT: fmov w8, s5 -; CHECK-NEXT: fcvt s3, d3 -; CHECK-NEXT: fmov w10, s0 -; CHECK-NEXT: lsl w9, w9, #16 -; CHECK-NEXT: bit.16b v1, v6, v4 -; CHECK-NEXT: lsl w8, w8, #16 -; CHECK-NEXT: mov d6, v2[1] -; CHECK-NEXT: fmov s7, w9 -; CHECK-NEXT: fcvt s2, d2 -; CHECK-NEXT: lsl w10, w10, #16 -; CHECK-NEXT: fmov s5, w8 -; CHECK-NEXT: fmov w8, s1 -; CHECK-NEXT: mov.16b v1, v4 -; CHECK-NEXT: bit.16b v3, v7, v4 -; CHECK-NEXT: bsl.16b v1, v5, v2 -; CHECK-NEXT: lsr w8, w8, #16 -; CHECK-NEXT: fcvt s2, d6 +; CHECK-NEXT: fcvtxn s1, d1 +; CHECK-NEXT: mov w8, #32767 ; =0x7fff +; CHECK-NEXT: fcvtxn s3, d3 +; CHECK-NEXT: fmov w10, s1 +; CHECK-NEXT: ubfx w12, w10, #16, #1 +; CHECK-NEXT: add w10, w10, w8 ; CHECK-NEXT: fmov w9, s3 -; CHECK-NEXT: fmov s5, w10 -; CHECK-NEXT: fmov s0, w8 -; CHECK-NEXT: fmov w8, s1 -; CHECK-NEXT: mov.16b v1, v4 +; CHECK-NEXT: fcvtxn s3, d2 +; CHECK-NEXT: mov d2, v2[1] +; CHECK-NEXT: add w10, w12, w10 +; CHECK-NEXT: lsr w10, w10, #16 +; CHECK-NEXT: ubfx w11, w9, #16, #1 +; CHECK-NEXT: add w9, w9, w8 +; CHECK-NEXT: fcvtxn s1, d2 +; CHECK-NEXT: add w9, w11, w9 +; CHECK-NEXT: fmov w11, s3 +; CHECK-NEXT: fmov s3, w10 ; CHECK-NEXT: lsr w9, w9, #16 -; CHECK-NEXT: fmov s3, w9 -; CHECK-NEXT: bsl.16b v1, v5, v2 -; CHECK-NEXT: lsr w8, w8, #16 -; CHECK-NEXT: mov.h v0[1], v3[0] -; CHECK-NEXT: fmov s2, w8 -; CHECK-NEXT: fmov w8, s1 -; CHECK-NEXT: mov.h v0[2], v2[0] +; CHECK-NEXT: ubfx w12, w11, #16, #1 +; CHECK-NEXT: fmov s2, w9 +; CHECK-NEXT: add w9, w11, w8 +; CHECK-NEXT: fmov w10, s1 +; CHECK-NEXT: add w9, w12, w9 +; CHECK-NEXT: lsr w9, w9, #16 +; CHECK-NEXT: mov.h v3[1], v2[0] +; CHECK-NEXT: ubfx w11, w10, #16, #1 +; CHECK-NEXT: add w8, w10, w8 +; CHECK-NEXT: fmov s1, w9 +; CHECK-NEXT: add w8, w11, w8 ; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: mov.h v3[2], v1[0] ; CHECK-NEXT: fmov s1, w8 -; CHECK-NEXT: mov.h v0[3], v1[0] -; CHECK-NEXT: ; kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: mov.h v3[3], v1[0] +; CHECK-NEXT: mvni.4h v1, #128, lsl #8 +; CHECK-NEXT: bif.8b v0, v3, v1 ; CHECK-NEXT: ret %tmp0 = fptrunc <4 x double> %b to <4 x bfloat> %r = call <4 x bfloat> @llvm.copysign.v4bf16(<4 x bfloat> %a, <4 x bfloat> %tmp0) @@ -673,111 +340,8 @@ declare <4 x bfloat> @llvm.copysign.v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) #0 define <8 x bfloat> @test_copysign_v8bf16_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) #0 { ; CHECK-LABEL: test_copysign_v8bf16_v8bf16: ; CHECK: ; %bb.0: -; CHECK-NEXT: fmov w8, s1 -; CHECK-NEXT: mov h2, v1[1] -; CHECK-NEXT: mov h4, v0[1] -; CHECK-NEXT: fmov w9, s0 -; CHECK-NEXT: mov h6, v1[2] -; CHECK-NEXT: mov h7, v0[2] -; CHECK-NEXT: mvni.4s v3, #128, lsl #24 -; CHECK-NEXT: mov h5, v1[3] -; CHECK-NEXT: mov h16, v0[3] -; CHECK-NEXT: lsl w8, w8, #16 -; CHECK-NEXT: mov h17, v1[4] -; CHECK-NEXT: lsl w9, w9, #16 -; CHECK-NEXT: fmov w10, s4 -; CHECK-NEXT: mov h4, v0[4] -; CHECK-NEXT: fmov s18, w8 -; CHECK-NEXT: fmov w8, s2 -; CHECK-NEXT: fmov w11, s7 -; CHECK-NEXT: fmov s2, w9 -; CHECK-NEXT: lsl w9, w10, #16 -; CHECK-NEXT: fmov w10, s6 -; CHECK-NEXT: lsl w8, w8, #16 -; CHECK-NEXT: fmov s7, w9 -; CHECK-NEXT: bif.16b v2, v18, v3 -; CHECK-NEXT: lsl w9, w11, #16 -; CHECK-NEXT: fmov s6, w8 -; CHECK-NEXT: lsl w8, w10, #16 -; CHECK-NEXT: fmov w10, s5 -; CHECK-NEXT: fmov w11, s16 -; CHECK-NEXT: fmov s16, w9 -; CHECK-NEXT: mov h18, v0[5] -; CHECK-NEXT: fmov s5, w8 -; CHECK-NEXT: bit.16b v6, v7, v3 -; CHECK-NEXT: fmov w8, s2 -; CHECK-NEXT: lsl w9, w10, #16 -; CHECK-NEXT: lsl w10, w11, #16 -; CHECK-NEXT: mov h7, v1[5] -; CHECK-NEXT: bit.16b v5, v16, v3 -; CHECK-NEXT: fmov s16, w10 -; CHECK-NEXT: fmov w10, s4 -; CHECK-NEXT: mov.16b v4, v3 -; CHECK-NEXT: fmov w11, s6 -; CHECK-NEXT: fmov s6, w9 -; CHECK-NEXT: fmov w9, s17 -; CHECK-NEXT: lsr w8, w8, #16 -; CHECK-NEXT: lsr w11, w11, #16 -; CHECK-NEXT: fmov s2, w8 -; CHECK-NEXT: lsl w8, w9, #16 -; CHECK-NEXT: bsl.16b v4, v16, v6 -; CHECK-NEXT: lsl w9, w10, #16 -; CHECK-NEXT: fmov w10, s5 -; CHECK-NEXT: fmov s6, w11 -; CHECK-NEXT: fmov s5, w8 -; CHECK-NEXT: lsr w8, w10, #16 -; CHECK-NEXT: fmov w10, s7 -; CHECK-NEXT: mov.h v2[1], v6[0] -; CHECK-NEXT: fmov s6, w9 -; CHECK-NEXT: fmov w9, s18 -; CHECK-NEXT: fmov s7, w8 -; CHECK-NEXT: fmov w8, s4 -; CHECK-NEXT: mov h4, v1[6] -; CHECK-NEXT: lsl w10, w10, #16 -; CHECK-NEXT: mov h1, v1[7] -; CHECK-NEXT: lsl w9, w9, #16 -; CHECK-NEXT: bit.16b v5, v6, v3 -; CHECK-NEXT: mov h6, v0[6] -; CHECK-NEXT: mov.h v2[2], v7[0] -; CHECK-NEXT: fmov s7, w10 -; CHECK-NEXT: lsr w8, w8, #16 -; CHECK-NEXT: fmov s16, w9 -; CHECK-NEXT: fmov w9, s4 -; CHECK-NEXT: mov h0, v0[7] -; CHECK-NEXT: fmov w10, s6 -; CHECK-NEXT: bit.16b v7, v16, v3 -; CHECK-NEXT: fmov s16, w8 -; CHECK-NEXT: fmov w8, s5 -; CHECK-NEXT: lsl w9, w9, #16 -; CHECK-NEXT: lsl w10, w10, #16 -; CHECK-NEXT: lsr w8, w8, #16 -; CHECK-NEXT: mov.h v2[3], v16[0] -; CHECK-NEXT: fmov s5, w9 -; CHECK-NEXT: fmov w9, s1 -; CHECK-NEXT: fmov s4, w8 -; CHECK-NEXT: fmov w8, s7 -; CHECK-NEXT: lsl w9, w9, #16 -; CHECK-NEXT: mov.h v2[4], v4[0] -; CHECK-NEXT: fmov s4, w10 -; CHECK-NEXT: fmov w10, s0 -; CHECK-NEXT: lsr w8, w8, #16 -; CHECK-NEXT: fmov s1, w9 -; CHECK-NEXT: fmov s0, w8 -; CHECK-NEXT: bif.16b v4, v5, v3 -; CHECK-NEXT: lsl w10, w10, #16 -; CHECK-NEXT: fmov s5, w10 -; CHECK-NEXT: mov.h v2[5], v0[0] -; CHECK-NEXT: mov.16b v0, v3 -; CHECK-NEXT: fmov w8, s4 -; CHECK-NEXT: bsl.16b v0, v5, v1 -; CHECK-NEXT: lsr w8, w8, #16 -; CHECK-NEXT: fmov s1, w8 -; CHECK-NEXT: fmov w8, s0 -; CHECK-NEXT: mov.h v2[6], v1[0] -; CHECK-NEXT: lsr w8, w8, #16 -; CHECK-NEXT: fmov s0, w8 -; CHECK-NEXT: mov.h v2[7], v0[0] -; CHECK-NEXT: mov.16b v0, v2 +; CHECK-NEXT: mvni.8h v2, #128, lsl #8 +; CHECK-NEXT: bif.16b v0, v1, v2 ; CHECK-NEXT: ret %r = call <8 x bfloat> @llvm.copysign.v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) ret <8 x bfloat> %r @@ -788,126 +352,23 @@ define <8 x bfloat> @test_copysign_v8bf16_v8f32(<8 x bfloat> %a, <8 x float> %b) ; CHECK: ; %bb.0: ; CHECK-NEXT: movi.4s v3, #127, msl #8 ; CHECK-NEXT: movi.4s v4, #1 -; CHECK-NEXT: ushr.4s v5, v1, #16 -; CHECK-NEXT: fcmeq.4s v7, v1, v1 -; CHECK-NEXT: fmov w9, s0 -; CHECK-NEXT: add.4s v6, v1, v3 +; CHECK-NEXT: ushr.4s v5, v2, #16 +; CHECK-NEXT: ushr.4s v6, v1, #16 +; CHECK-NEXT: add.4s v7, v2, v3 +; CHECK-NEXT: add.4s v3, v1, v3 ; CHECK-NEXT: and.16b v5, v5, v4 -; CHECK-NEXT: orr.4s v1, #64, lsl #16 -; CHECK-NEXT: lsl w9, w9, #16 -; CHECK-NEXT: add.4s v5, v5, v6 -; CHECK-NEXT: ushr.4s v6, v2, #16 ; CHECK-NEXT: and.16b v4, v6, v4 -; CHECK-NEXT: mov h6, v0[2] -; CHECK-NEXT: bit.16b v1, v5, v7 -; CHECK-NEXT: add.4s v7, v2, v3 -; CHECK-NEXT: mov h5, v0[1] -; CHECK-NEXT: fcmeq.4s v3, v2, v2 +; CHECK-NEXT: fcmeq.4s v6, v2, v2 ; CHECK-NEXT: orr.4s v2, #64, lsl #16 -; CHECK-NEXT: shrn.4h v1, v1, #16 -; CHECK-NEXT: add.4s v4, v4, v7 -; CHECK-NEXT: fmov w8, s5 -; CHECK-NEXT: mov h7, v0[3] -; CHECK-NEXT: mov h5, v0[4] -; CHECK-NEXT: mov h16, v1[1] -; CHECK-NEXT: fmov w10, s1 -; CHECK-NEXT: lsl w8, w8, #16 -; CHECK-NEXT: bsl.16b v3, v4, v2 -; CHECK-NEXT: mov h4, v1[2] -; CHECK-NEXT: mov h17, v1[3] -; CHECK-NEXT: mvni.4s v2, #128, lsl #24 -; CHECK-NEXT: fmov s1, w9 -; CHECK-NEXT: fmov w9, s6 -; CHECK-NEXT: lsl w10, w10, #16 -; CHECK-NEXT: fmov s6, w8 -; CHECK-NEXT: fmov w8, s7 -; CHECK-NEXT: fmov w11, s16 -; CHECK-NEXT: fmov s7, w10 -; CHECK-NEXT: fmov w10, s4 -; CHECK-NEXT: mov.16b v4, v2 -; CHECK-NEXT: lsl w9, w9, #16 -; CHECK-NEXT: lsl w8, w8, #16 -; CHECK-NEXT: shrn.4h v3, v3, #16 -; CHECK-NEXT: lsl w11, w11, #16 -; CHECK-NEXT: bif.16b v1, v7, v2 -; CHECK-NEXT: fmov s16, w8 -; CHECK-NEXT: fmov s7, w11 -; CHECK-NEXT: bsl.16b v4, v6, v7 -; CHECK-NEXT: fmov s7, w9 -; CHECK-NEXT: lsl w9, w10, #16 -; CHECK-NEXT: fmov w10, s17 -; CHECK-NEXT: mov h6, v0[5] -; CHECK-NEXT: lsl w8, w10, #16 -; CHECK-NEXT: fmov w10, s1 -; CHECK-NEXT: fmov s1, w9 -; CHECK-NEXT: lsr w9, w10, #16 -; CHECK-NEXT: fmov w10, s4 -; CHECK-NEXT: fmov s4, w8 -; CHECK-NEXT: bif.16b v7, v1, v2 -; CHECK-NEXT: fmov w8, s5 -; CHECK-NEXT: mov h5, v3[1] -; CHECK-NEXT: fmov s1, w9 -; CHECK-NEXT: fmov w9, s3 -; CHECK-NEXT: lsr w10, w10, #16 -; CHECK-NEXT: bit.16b v4, v16, v2 -; CHECK-NEXT: lsl w8, w8, #16 -; CHECK-NEXT: fmov s16, w10 -; CHECK-NEXT: lsl w9, w9, #16 -; CHECK-NEXT: fmov w10, s7 -; CHECK-NEXT: mov h7, v0[6] -; CHECK-NEXT: mov h0, v0[7] -; CHECK-NEXT: mov.h v1[1], v16[0] -; CHECK-NEXT: fmov s16, w8 -; CHECK-NEXT: fmov w8, s6 -; CHECK-NEXT: fmov s6, w9 -; CHECK-NEXT: fmov w9, s5 -; CHECK-NEXT: lsr w10, w10, #16 -; CHECK-NEXT: lsl w8, w8, #16 -; CHECK-NEXT: lsl w9, w9, #16 -; CHECK-NEXT: bit.16b v6, v16, v2 -; CHECK-NEXT: fmov s16, w10 -; CHECK-NEXT: fmov w10, s4 -; CHECK-NEXT: fmov s5, w8 -; CHECK-NEXT: fmov w8, s7 -; CHECK-NEXT: fmov s7, w9 -; CHECK-NEXT: mov h4, v3[2] -; CHECK-NEXT: mov h3, v3[3] -; CHECK-NEXT: mov.h v1[2], v16[0] -; CHECK-NEXT: lsr w10, w10, #16 -; CHECK-NEXT: fmov w9, s6 -; CHECK-NEXT: lsl w8, w8, #16 -; CHECK-NEXT: bif.16b v5, v7, v2 -; CHECK-NEXT: fmov s16, w10 -; CHECK-NEXT: fmov w10, s4 -; CHECK-NEXT: fmov s4, w8 -; CHECK-NEXT: lsr w9, w9, #16 -; CHECK-NEXT: mov.h v1[3], v16[0] -; CHECK-NEXT: fmov w8, s5 -; CHECK-NEXT: lsl w10, w10, #16 -; CHECK-NEXT: fmov s6, w9 -; CHECK-NEXT: fmov w9, s0 -; CHECK-NEXT: fmov s5, w10 -; CHECK-NEXT: fmov w10, s3 -; CHECK-NEXT: lsr w8, w8, #16 -; CHECK-NEXT: mov.h v1[4], v6[0] -; CHECK-NEXT: lsl w9, w9, #16 -; CHECK-NEXT: fmov s0, w8 -; CHECK-NEXT: bif.16b v4, v5, v2 -; CHECK-NEXT: lsl w10, w10, #16 -; CHECK-NEXT: fmov s3, w9 -; CHECK-NEXT: fmov s5, w10 -; CHECK-NEXT: mov.h v1[5], v0[0] -; CHECK-NEXT: mov.16b v0, v2 -; CHECK-NEXT: fmov w8, s4 -; CHECK-NEXT: bsl.16b v0, v3, v5 -; CHECK-NEXT: lsr w8, w8, #16 -; CHECK-NEXT: fmov s2, w8 -; CHECK-NEXT: fmov w8, s0 -; CHECK-NEXT: mov.h v1[6], v2[0] -; CHECK-NEXT: lsr w8, w8, #16 -; CHECK-NEXT: fmov s0, w8 -; CHECK-NEXT: mov.h v1[7], v0[0] -; CHECK-NEXT: mov.16b v0, v1 +; CHECK-NEXT: add.4s v5, v5, v7 +; CHECK-NEXT: add.4s v3, v4, v3 +; CHECK-NEXT: fcmeq.4s v4, v1, v1 +; CHECK-NEXT: orr.4s v1, #64, lsl #16 +; CHECK-NEXT: bit.16b v2, v5, v6 +; CHECK-NEXT: bit.16b v1, v3, v4 +; CHECK-NEXT: uzp2.8h v1, v1, v2 +; CHECK-NEXT: mvni.8h v2, #128, lsl #8 +; CHECK-NEXT: bif.16b v0, v1, v2 ; CHECK-NEXT: ret %tmp0 = fptrunc <8 x float> %b to <8 x bfloat> %r = call <8 x bfloat> @llvm.copysign.v8bf16(<8 x bfloat> %a, <8 x bfloat> %tmp0) @@ -917,3 +378,6 @@ define <8 x bfloat> @test_copysign_v8bf16_v8f32(<8 x bfloat> %a, <8 x float> %b) declare <8 x bfloat> @llvm.copysign.v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) #0 attributes #0 = { nounwind } +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; FP16: {{.*}} +; NOFP16: {{.*}} -- GitLab From 1c9125cdd5eb8f3e949313a57ff736d87b540546 Mon Sep 17 00:00:00 2001 From: Changpeng Fang Date: Mon, 4 Mar 2024 12:06:49 -0800 Subject: [PATCH 039/929] AMDGPU: Proper use of HasImageInsts in vimage inst definitions, NFC (#83884) This work corrects a few inappropriate uses of HasImageInsts predicate in vimage instruction definitions. In MnemonicAlias for VIMAGE_Atomic_gfx12_Renamed, we also need HasImageInsts to be in the "Requires" predicate list for the alias to depend on whether or not the GPU has image instruction support. For nested uses of "let OtherPredicates = ..." around vimage instruction definitions, the inner assignment will override the outer one. This makes the outermost "let OtherPredicates = [HasImageInsts]" unused when we have an inner assignment. As a result, HasImageInsts is not actually used for some vimage instructions. To resove this issue, we propogate the predicates in an outer assignment into the inner one. We should avoid using nested "let SubtargetPredicate = ...". However, we can always put the predicate into OtherPtredicates list. --- llvm/lib/Target/AMDGPU/MIMGInstructions.td | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/llvm/lib/Target/AMDGPU/MIMGInstructions.td index cc374fbae7cc..595ef39ce03e 100644 --- a/llvm/lib/Target/AMDGPU/MIMGInstructions.td +++ b/llvm/lib/Target/AMDGPU/MIMGInstructions.td @@ -966,7 +966,7 @@ class VIMAGE_Atomic_gfx12_Renamed : VIMAGE_Atomic_gfx12, - MnemonicAlias, Requires<[isGFX12Plus]>; + MnemonicAlias, Requires<[isGFX12Plus, HasImageInsts]>; multiclass MIMG_Atomic_Addr_Helper_m , "image_atomic_max_num_flt", 0, 1, "image_atomic_max_flt">; defm IMAGE_SAMPLE : MIMG_Sampler_WQM , AMDGPUSample>; -let OtherPredicates = [HasExtendedImageInsts] in { +let OtherPredicates = [HasImageInsts, HasExtendedImageInsts] in { defm IMAGE_SAMPLE_CL : MIMG_Sampler_WQM , AMDGPUSample_cl>; defm IMAGE_SAMPLE_D : MIMG_Sampler , AMDGPUSample_d>; defm IMAGE_SAMPLE_D_CL : MIMG_Sampler , AMDGPUSample_d_cl>; @@ -1617,7 +1617,7 @@ defm IMAGE_GATHER4_C_B_O : MIMG_Gather_WQM , AMDGPUSample_c_b_cl_o>; defm IMAGE_GATHER4_C_LZ_O : MIMG_Gather , AMDGPUSample_c_lz_o>; -let SubtargetPredicate = isGFX9Plus in +let OtherPredicates = [HasImageInsts, HasExtendedImageInsts, isGFX9Plus] in defm IMAGE_GATHER4H : MIMG_Gather , AMDGPUSample, 1, "image_gather4h">; defm IMAGE_GET_LOD : MIMG_Sampler , AMDGPUSample, 1, 0, 1, "image_get_lod">; @@ -1630,9 +1630,9 @@ defm IMAGE_SAMPLE_CD_O : MIMG_Sampler , AMDGPUSample_cd_cl_o>; defm IMAGE_SAMPLE_C_CD_O : MIMG_Sampler , AMDGPUSample_c_cd_o>; defm IMAGE_SAMPLE_C_CD_CL_O : MIMG_Sampler , AMDGPUSample_c_cd_cl_o>; -} // End OtherPredicates = [HasExtendedImageInsts] +} // End OtherPredicates = [HasImageInsts, HasExtendedImageInsts] -let OtherPredicates = [HasExtendedImageInsts,HasG16] in { +let OtherPredicates = [HasImageInsts, HasExtendedImageInsts, HasG16] in { defm IMAGE_SAMPLE_D_G16 : MIMG_Sampler , AMDGPUSample_d, 0, 1>; defm IMAGE_SAMPLE_D_CL_G16 : MIMG_Sampler , AMDGPUSample_d_cl, 0, 1>; defm IMAGE_SAMPLE_C_D_G16 : MIMG_Sampler , AMDGPUSample_c_d, 0, 1>; @@ -1649,23 +1649,22 @@ defm IMAGE_SAMPLE_CD_O_G16 : MIMG_Sampler , AMDGPUSample_cd_cl_o, 0, 1>; defm IMAGE_SAMPLE_C_CD_O_G16 : MIMG_Sampler , AMDGPUSample_c_cd_o, 0, 1>; defm IMAGE_SAMPLE_C_CD_CL_O_G16 : MIMG_Sampler , AMDGPUSample_c_cd_cl_o, 0, 1>; -} // End OtherPredicates = [HasExtendedImageInsts,HasG16] +} // End OtherPredicates = [HasImageInsts, HasExtendedImageInsts, HasG16] //def IMAGE_RSRC256 : MIMG_NoPattern_RSRC256 <"image_rsrc256", mimgopc<0x7e>>; //def IMAGE_SAMPLER : MIMG_NoPattern_ <"image_sampler", mimgopc<0x7f>>; -let SubtargetPredicate = isGFX10Only, OtherPredicates = [HasGFX10_AEncoding] in +let OtherPredicates = [HasImageInsts, HasGFX10_AEncoding, isGFX10Only] in defm IMAGE_MSAA_LOAD_X : MIMG_NoSampler , "image_msaa_load", 1, 0, 0, 1>; -let OtherPredicates = [HasGFX10_AEncoding] in +let OtherPredicates = [HasImageInsts, HasGFX10_AEncoding] in { defm IMAGE_MSAA_LOAD : MIMG_MSAA_Load , "image_msaa_load">; -let OtherPredicates = [HasGFX10_AEncoding] in { defm IMAGE_BVH_INTERSECT_RAY : MIMG_IntersectRay, "image_bvh_intersect_ray", 0, 0>; defm IMAGE_BVH_INTERSECT_RAY_a16 : MIMG_IntersectRay, "image_bvh_intersect_ray", 0, 1>; defm IMAGE_BVH64_INTERSECT_RAY : MIMG_IntersectRay, "image_bvh64_intersect_ray", 1, 0>; defm IMAGE_BVH64_INTERSECT_RAY_a16 : MIMG_IntersectRay, "image_bvh64_intersect_ray", 1, 1>; -} // End OtherPredicates = [HasGFX10_AEncoding] +} // End OtherPredicates = [HasImageInsts, HasGFX10_AEncoding] } // End let OtherPredicates = [HasImageInsts] -- GitLab From 3b5965e633bf6f945436dc8ac33bbb37ace4acf1 Mon Sep 17 00:00:00 2001 From: PeterChou1 Date: Mon, 4 Mar 2024 15:15:17 -0500 Subject: [PATCH 040/929] [clang-format][doc] fix documentation for clang-format (#83415) Fixes typo in documentation for clang-format Fixes #83207. --- clang/docs/ClangFormatStyleOptions.rst | 20 ++++++++++---------- clang/docs/tools/dump_format_style.py | 1 + clang/include/clang/Format/Format.h | 4 ++-- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/clang/docs/ClangFormatStyleOptions.rst b/clang/docs/ClangFormatStyleOptions.rst index df399a229d8d..5b00a8f4c00f 100644 --- a/clang/docs/ClangFormatStyleOptions.rst +++ b/clang/docs/ClangFormatStyleOptions.rst @@ -318,9 +318,9 @@ the configuration (without a prefix: ``Auto``). .. code-block:: c++ - AlignConsecutiveMacros: AcrossEmptyLines + AlignConsecutiveAssignments: AcrossEmptyLines - AlignConsecutiveMacros: + AlignConsecutiveAssignments: Enabled: true AcrossEmptyLines: true AcrossComments: false @@ -460,9 +460,9 @@ the configuration (without a prefix: ``Auto``). .. code-block:: c++ - AlignConsecutiveMacros: AcrossEmptyLines + AlignConsecutiveBitFields: AcrossEmptyLines - AlignConsecutiveMacros: + AlignConsecutiveBitFields: Enabled: true AcrossEmptyLines: true AcrossComments: false @@ -602,9 +602,9 @@ the configuration (without a prefix: ``Auto``). .. code-block:: c++ - AlignConsecutiveMacros: AcrossEmptyLines + AlignConsecutiveDeclarations: AcrossEmptyLines - AlignConsecutiveMacros: + AlignConsecutiveDeclarations: Enabled: true AcrossEmptyLines: true AcrossComments: false @@ -983,9 +983,9 @@ the configuration (without a prefix: ``Auto``). .. code-block:: c++ - AlignConsecutiveMacros: AcrossEmptyLines + AlignConsecutiveTableGenCondOperatorColons: AcrossEmptyLines - AlignConsecutiveMacros: + AlignConsecutiveTableGenCondOperatorColons: Enabled: true AcrossEmptyLines: true AcrossComments: false @@ -1123,9 +1123,9 @@ the configuration (without a prefix: ``Auto``). .. code-block:: c++ - AlignConsecutiveMacros: AcrossEmptyLines + AlignConsecutiveTableGenDefinitionColons: AcrossEmptyLines - AlignConsecutiveMacros: + AlignConsecutiveTableGenDefinitionColons: Enabled: true AcrossEmptyLines: true AcrossComments: false diff --git a/clang/docs/tools/dump_format_style.py b/clang/docs/tools/dump_format_style.py index af0124b94eca..af0e658fcdc5 100755 --- a/clang/docs/tools/dump_format_style.py +++ b/clang/docs/tools/dump_format_style.py @@ -129,6 +129,7 @@ class Option(object): s += indent( "\n\nNested configuration flags:\n\n%s\n" % self.nested_struct, 2 ) + s = s.replace("", self.name) return s diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h index 613f1fd16846..590297fd89a3 100644 --- a/clang/include/clang/Format/Format.h +++ b/clang/include/clang/Format/Format.h @@ -154,9 +154,9 @@ struct FormatStyle { /// For example, to align across empty lines and not across comments, either /// of these work. /// \code - /// AlignConsecutiveMacros: AcrossEmptyLines + /// : AcrossEmptyLines /// - /// AlignConsecutiveMacros: + /// : /// Enabled: true /// AcrossEmptyLines: true /// AcrossComments: false -- GitLab From 72cf95df2f69a1c023ef404298f3cce2e6b44d86 Mon Sep 17 00:00:00 2001 From: Michal Paszkowski Date: Mon, 4 Mar 2024 12:15:38 -0800 Subject: [PATCH 041/929] [SPIR-V] Fix warning -Wsometimes-uninitialized (#83901) --- llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index 6a700014ea93..c83537bc7ae8 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -620,7 +620,7 @@ void SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(Instruction *I) { EltTyConst = UndefValue::get(AI->getAllocatedType()); else if (auto *GEP = dyn_cast(I)) EltTyConst = UndefValue::get(GEP->getResultElementType()); - else if (I->getType()->isPointerTy()) + else EltTyConst = UndefValue::get(IntegerType::getInt8Ty(I->getContext())); buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {I->getType()}, EltTyConst, I, -- GitLab From 93e423f19fc5317ee208f77d8f36e677db124cc8 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Mon, 4 Mar 2024 12:35:57 -0800 Subject: [PATCH 042/929] [libc][test] update constants used in stdbit test (#83893) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Either: - I forgot my alphabet (that E comes before F). - My juvenile inner brain finds unsigned literal constants with the sequence FU funny. ¿Por qué no los dos? --- libc/test/include/stdbit_test.cpp | 40 +++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/libc/test/include/stdbit_test.cpp b/libc/test/include/stdbit_test.cpp index acb79ca0f3ff..16dcd8e085f4 100644 --- a/libc/test/include/stdbit_test.cpp +++ b/libc/test/include/stdbit_test.cpp @@ -27,60 +27,60 @@ unsigned stdc_leading_zeros_uc(unsigned char) noexcept { return 0xAAU; } unsigned stdc_leading_zeros_us(unsigned short) noexcept { return 0xABU; } unsigned stdc_leading_zeros_ui(unsigned) noexcept { return 0xACU; } unsigned stdc_leading_zeros_ul(unsigned long) noexcept { return 0xADU; } -unsigned stdc_leading_zeros_ull(unsigned long long) noexcept { return 0xAFU; } +unsigned stdc_leading_zeros_ull(unsigned long long) noexcept { return 0xAEU; } unsigned stdc_leading_ones_uc(unsigned char) noexcept { return 0xBAU; } unsigned stdc_leading_ones_us(unsigned short) noexcept { return 0xBBU; } unsigned stdc_leading_ones_ui(unsigned) noexcept { return 0xBCU; } unsigned stdc_leading_ones_ul(unsigned long) noexcept { return 0xBDU; } -unsigned stdc_leading_ones_ull(unsigned long long) noexcept { return 0xBFU; } +unsigned stdc_leading_ones_ull(unsigned long long) noexcept { return 0xBEU; } unsigned stdc_trailing_zeros_uc(unsigned char) noexcept { return 0xCAU; } unsigned stdc_trailing_zeros_us(unsigned short) noexcept { return 0xCBU; } unsigned stdc_trailing_zeros_ui(unsigned) noexcept { return 0xCCU; } unsigned stdc_trailing_zeros_ul(unsigned long) noexcept { return 0xCDU; } -unsigned stdc_trailing_zeros_ull(unsigned long long) noexcept { return 0xCFU; } +unsigned stdc_trailing_zeros_ull(unsigned long long) noexcept { return 0xCEU; } unsigned stdc_trailing_ones_uc(unsigned char) noexcept { return 0xDAU; } unsigned stdc_trailing_ones_us(unsigned short) noexcept { return 0xDBU; } unsigned stdc_trailing_ones_ui(unsigned) noexcept { return 0xDCU; } unsigned stdc_trailing_ones_ul(unsigned long) noexcept { return 0xDDU; } -unsigned stdc_trailing_ones_ull(unsigned long long) noexcept { return 0xDFU; } +unsigned stdc_trailing_ones_ull(unsigned long long) noexcept { return 0xDEU; } unsigned stdc_first_leading_zero_uc(unsigned char) noexcept { return 0xEAU; } unsigned stdc_first_leading_zero_us(unsigned short) noexcept { return 0xEBU; } unsigned stdc_first_leading_zero_ui(unsigned) noexcept { return 0xECU; } unsigned stdc_first_leading_zero_ul(unsigned long) noexcept { return 0xEDU; } unsigned stdc_first_leading_zero_ull(unsigned long long) noexcept { - return 0xEFU; + return 0xEEU; } unsigned stdc_first_leading_one_uc(unsigned char) noexcept { return 0xFAU; } unsigned stdc_first_leading_one_us(unsigned short) noexcept { return 0xFBU; } unsigned stdc_first_leading_one_ui(unsigned) noexcept { return 0xFCU; } unsigned stdc_first_leading_one_ul(unsigned long) noexcept { return 0xFDU; } unsigned stdc_first_leading_one_ull(unsigned long long) noexcept { - return 0xFFU; + return 0xFEU; } unsigned stdc_first_trailing_zero_uc(unsigned char) noexcept { return 0x0AU; } unsigned stdc_first_trailing_zero_us(unsigned short) noexcept { return 0x0BU; } unsigned stdc_first_trailing_zero_ui(unsigned) noexcept { return 0x0CU; } unsigned stdc_first_trailing_zero_ul(unsigned long) noexcept { return 0x0DU; } unsigned stdc_first_trailing_zero_ull(unsigned long long) noexcept { - return 0x0FU; + return 0x0EU; } unsigned stdc_first_trailing_one_uc(unsigned char) noexcept { return 0x1AU; } unsigned stdc_first_trailing_one_us(unsigned short) noexcept { return 0x1BU; } unsigned stdc_first_trailing_one_ui(unsigned) noexcept { return 0x1CU; } unsigned stdc_first_trailing_one_ul(unsigned long) noexcept { return 0x1DU; } unsigned stdc_first_trailing_one_ull(unsigned long long) noexcept { - return 0x1FU; + return 0x1EU; } unsigned stdc_count_zeros_uc(unsigned char) noexcept { return 0x2AU; } unsigned stdc_count_zeros_us(unsigned short) noexcept { return 0x2BU; } unsigned stdc_count_zeros_ui(unsigned) noexcept { return 0x2CU; } unsigned stdc_count_zeros_ul(unsigned long) noexcept { return 0x2DU; } -unsigned stdc_count_zeros_ull(unsigned long long) noexcept { return 0x2FU; } +unsigned stdc_count_zeros_ull(unsigned long long) noexcept { return 0x2EU; } unsigned stdc_count_ones_uc(unsigned char) noexcept { return 0x3AU; } unsigned stdc_count_ones_us(unsigned short) noexcept { return 0x3BU; } unsigned stdc_count_ones_ui(unsigned) noexcept { return 0x3CU; } unsigned stdc_count_ones_ul(unsigned long) noexcept { return 0x3DU; } -unsigned stdc_count_ones_ull(unsigned long long) noexcept { return 0x3FU; } +unsigned stdc_count_ones_ull(unsigned long long) noexcept { return 0x3EU; } bool stdc_has_single_bit_uc(unsigned char) noexcept { return false; } bool stdc_has_single_bit_us(unsigned short) noexcept { return false; } bool stdc_has_single_bit_ui(unsigned) noexcept { return false; } @@ -95,7 +95,7 @@ TEST(LlvmLibcStdbitTest, TypeGenericMacroLeadingZeros) { EXPECT_EQ(stdc_leading_zeros(static_cast(0U)), 0xABU); EXPECT_EQ(stdc_leading_zeros(0U), 0xACU); EXPECT_EQ(stdc_leading_zeros(0UL), 0xADU); - EXPECT_EQ(stdc_leading_zeros(0ULL), 0xAFU); + EXPECT_EQ(stdc_leading_zeros(0ULL), 0xAEU); } TEST(LlvmLibcStdbitTest, TypeGenericMacroLeadingOnes) { @@ -103,7 +103,7 @@ TEST(LlvmLibcStdbitTest, TypeGenericMacroLeadingOnes) { EXPECT_EQ(stdc_leading_ones(static_cast(0U)), 0xBBU); EXPECT_EQ(stdc_leading_ones(0U), 0xBCU); EXPECT_EQ(stdc_leading_ones(0UL), 0xBDU); - EXPECT_EQ(stdc_leading_ones(0ULL), 0xBFU); + EXPECT_EQ(stdc_leading_ones(0ULL), 0xBEU); } TEST(LlvmLibcStdbitTest, TypeGenericMacroTrailingZeros) { @@ -111,7 +111,7 @@ TEST(LlvmLibcStdbitTest, TypeGenericMacroTrailingZeros) { EXPECT_EQ(stdc_trailing_zeros(static_cast(0U)), 0xCBU); EXPECT_EQ(stdc_trailing_zeros(0U), 0xCCU); EXPECT_EQ(stdc_trailing_zeros(0UL), 0xCDU); - EXPECT_EQ(stdc_trailing_zeros(0ULL), 0xCFU); + EXPECT_EQ(stdc_trailing_zeros(0ULL), 0xCEU); } TEST(LlvmLibcStdbitTest, TypeGenericMacroTrailingOnes) { @@ -119,7 +119,7 @@ TEST(LlvmLibcStdbitTest, TypeGenericMacroTrailingOnes) { EXPECT_EQ(stdc_trailing_ones(static_cast(0U)), 0xDBU); EXPECT_EQ(stdc_trailing_ones(0U), 0xDCU); EXPECT_EQ(stdc_trailing_ones(0UL), 0xDDU); - EXPECT_EQ(stdc_trailing_ones(0ULL), 0xDFU); + EXPECT_EQ(stdc_trailing_ones(0ULL), 0xDEU); } TEST(LlvmLibcStdbitTest, TypeGenericMacroFirstLeadingZero) { @@ -127,7 +127,7 @@ TEST(LlvmLibcStdbitTest, TypeGenericMacroFirstLeadingZero) { EXPECT_EQ(stdc_first_leading_zero(static_cast(0U)), 0xEBU); EXPECT_EQ(stdc_first_leading_zero(0U), 0xECU); EXPECT_EQ(stdc_first_leading_zero(0UL), 0xEDU); - EXPECT_EQ(stdc_first_leading_zero(0ULL), 0xEFU); + EXPECT_EQ(stdc_first_leading_zero(0ULL), 0xEEU); } TEST(LlvmLibcStdbitTest, TypeGenericMacroFirstLeadingOne) { @@ -135,7 +135,7 @@ TEST(LlvmLibcStdbitTest, TypeGenericMacroFirstLeadingOne) { EXPECT_EQ(stdc_first_leading_one(static_cast(0U)), 0xFBU); EXPECT_EQ(stdc_first_leading_one(0U), 0xFCU); EXPECT_EQ(stdc_first_leading_one(0UL), 0xFDU); - EXPECT_EQ(stdc_first_leading_one(0ULL), 0xFFU); + EXPECT_EQ(stdc_first_leading_one(0ULL), 0xFEU); } TEST(LlvmLibcStdbitTest, TypeGenericMacroFirstTrailingZero) { @@ -143,7 +143,7 @@ TEST(LlvmLibcStdbitTest, TypeGenericMacroFirstTrailingZero) { EXPECT_EQ(stdc_first_trailing_zero(static_cast(0U)), 0x0BU); EXPECT_EQ(stdc_first_trailing_zero(0U), 0x0CU); EXPECT_EQ(stdc_first_trailing_zero(0UL), 0x0DU); - EXPECT_EQ(stdc_first_trailing_zero(0ULL), 0x0FU); + EXPECT_EQ(stdc_first_trailing_zero(0ULL), 0x0EU); } TEST(LlvmLibcStdbitTest, TypeGenericMacroFirstTrailingOne) { @@ -151,7 +151,7 @@ TEST(LlvmLibcStdbitTest, TypeGenericMacroFirstTrailingOne) { EXPECT_EQ(stdc_first_trailing_one(static_cast(0U)), 0x1BU); EXPECT_EQ(stdc_first_trailing_one(0U), 0x1CU); EXPECT_EQ(stdc_first_trailing_one(0UL), 0x1DU); - EXPECT_EQ(stdc_first_trailing_one(0ULL), 0x1FU); + EXPECT_EQ(stdc_first_trailing_one(0ULL), 0x1EU); } TEST(LlvmLibcStdbitTest, TypeGenericMacroCountZeros) { @@ -159,7 +159,7 @@ TEST(LlvmLibcStdbitTest, TypeGenericMacroCountZeros) { EXPECT_EQ(stdc_count_zeros(static_cast(0U)), 0x2BU); EXPECT_EQ(stdc_count_zeros(0U), 0x2CU); EXPECT_EQ(stdc_count_zeros(0UL), 0x2DU); - EXPECT_EQ(stdc_count_zeros(0ULL), 0x2FU); + EXPECT_EQ(stdc_count_zeros(0ULL), 0x2EU); } TEST(LlvmLibcStdbitTest, TypeGenericMacroCountOnes) { @@ -167,7 +167,7 @@ TEST(LlvmLibcStdbitTest, TypeGenericMacroCountOnes) { EXPECT_EQ(stdc_count_ones(static_cast(0U)), 0x3BU); EXPECT_EQ(stdc_count_ones(0U), 0x3CU); EXPECT_EQ(stdc_count_ones(0UL), 0x3DU); - EXPECT_EQ(stdc_count_ones(0ULL), 0x3FU); + EXPECT_EQ(stdc_count_ones(0ULL), 0x3EU); } TEST(LlvmLibcStdbitTest, TypeGenericMacroHasSingleBit) { -- GitLab From be3eeea7beb7dc701554a67773669c91367e7a81 Mon Sep 17 00:00:00 2001 From: David Majnemer Date: Mon, 4 Mar 2024 20:42:14 +0000 Subject: [PATCH 043/929] [AArch64] Use SHLLv4i16 to shift+widen BF16 to F32. --- llvm/lib/Target/AArch64/AArch64InstrInfo.td | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index 2f3aaf86d376..f8c6d9019ef6 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -7730,8 +7730,7 @@ def : Pat<(v2i64 (sext (v2i32 V64:$Rn))), (SSHLLv2i32_shift V64:$Rn, (i32 0))> def : Pat<(v2i64 (zext (v2i32 V64:$Rn))), (USHLLv2i32_shift V64:$Rn, (i32 0))>; def : Pat<(v2i64 (anyext (v2i32 V64:$Rn))), (USHLLv2i32_shift V64:$Rn, (i32 0))>; // Vector bf16 -> fp32 is implemented morally as a zext + shift. -def : Pat<(v4f32 (any_fpextend (v4bf16 V64:$Rn))), - (USHLLv4i16_shift V64:$Rn, (i32 16))>; +def : Pat<(v4f32 (any_fpextend (v4bf16 V64:$Rn))), (SHLLv4i16 V64:$Rn)>; // Also match an extend from the upper half of a 128 bit source register. def : Pat<(v8i16 (anyext (v8i8 (extract_high_v16i8 (v16i8 V128:$Rn)) ))), (USHLLv16i8_shift V128:$Rn, (i32 0))>; -- GitLab From bb97c992834a24632629dec4ec79ea7ffd1261fa Mon Sep 17 00:00:00 2001 From: erichkeane Date: Mon, 4 Mar 2024 12:32:09 -0800 Subject: [PATCH 044/929] [OpenACC] Enable serial/kernels Compute Constructs So far, all the work we've done for compute constructs has only used 'parallel'. This patch does the work to enable the same logic for 'serial' and 'kernels' constructs as well, since they are the same semantic behavior. --- clang/lib/Parse/ParseOpenACC.cpp | 4 + clang/lib/Sema/SemaOpenACC.cpp | 8 + clang/test/ParserOpenACC/parse-clauses.c | 841 +++++++++--------- clang/test/ParserOpenACC/parse-constructs.c | 21 +- .../SemaOpenACC/compute-construct-ast.cpp | 50 ++ clang/test/SemaOpenACC/no-branch-in-out.c | 66 ++ clang/test/SemaOpenACC/no-branch-in-out.cpp | 11 + .../SemaOpenACC/parallel-assoc-stmt-inst.cpp | 8 +- .../test/SemaOpenACC/parallel-loc-and-stmt.c | 27 + 9 files changed, 585 insertions(+), 451 deletions(-) diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp index 4946a61fca00..50e3c39f6091 100644 --- a/clang/lib/Parse/ParseOpenACC.cpp +++ b/clang/lib/Parse/ParseOpenACC.cpp @@ -555,6 +555,8 @@ bool doesDirectiveHaveAssociatedStmt(OpenACCDirectiveKind DirKind) { default: return false; case OpenACCDirectiveKind::Parallel: + case OpenACCDirectiveKind::Serial: + case OpenACCDirectiveKind::Kernels: return true; } llvm_unreachable("Unhandled directive->assoc stmt"); @@ -563,6 +565,8 @@ bool doesDirectiveHaveAssociatedStmt(OpenACCDirectiveKind DirKind) { unsigned getOpenACCScopeFlags(OpenACCDirectiveKind DirKind) { switch (DirKind) { case OpenACCDirectiveKind::Parallel: + case OpenACCDirectiveKind::Serial: + case OpenACCDirectiveKind::Kernels: // Mark this as a BreakScope/ContinueScope as well as a compute construct // so that we can diagnose trying to 'break'/'continue' inside of one. return Scope::BreakScope | Scope::ContinueScope | diff --git a/clang/lib/Sema/SemaOpenACC.cpp b/clang/lib/Sema/SemaOpenACC.cpp index d365a5151a45..d3a602d1c382 100644 --- a/clang/lib/Sema/SemaOpenACC.cpp +++ b/clang/lib/Sema/SemaOpenACC.cpp @@ -27,6 +27,8 @@ bool diagnoseConstructAppertainment(Sema &S, OpenACCDirectiveKind K, // do anything. break; case OpenACCDirectiveKind::Parallel: + case OpenACCDirectiveKind::Serial: + case OpenACCDirectiveKind::Kernels: if (!IsStmt) return S.Diag(StartLoc, diag::err_acc_construct_appertainment) << K; break; @@ -55,6 +57,8 @@ void Sema::ActOnOpenACCConstruct(OpenACCDirectiveKind K, // rules anywhere. break; case OpenACCDirectiveKind::Parallel: + case OpenACCDirectiveKind::Serial: + case OpenACCDirectiveKind::Kernels: // Nothing to do here, there is no real legalization that needs to happen // here as these constructs do not take any arguments. break; @@ -79,6 +83,8 @@ StmtResult Sema::ActOnEndOpenACCStmtDirective(OpenACCDirectiveKind K, case OpenACCDirectiveKind::Invalid: return StmtError(); case OpenACCDirectiveKind::Parallel: + case OpenACCDirectiveKind::Serial: + case OpenACCDirectiveKind::Kernels: return OpenACCComputeConstruct::Create( getASTContext(), K, StartLoc, EndLoc, AssocStmt.isUsable() ? AssocStmt.get() : nullptr); @@ -92,6 +98,8 @@ StmtResult Sema::ActOnOpenACCAssociatedStmt(OpenACCDirectiveKind K, default: llvm_unreachable("Unimplemented associated statement application"); case OpenACCDirectiveKind::Parallel: + case OpenACCDirectiveKind::Serial: + case OpenACCDirectiveKind::Kernels: // There really isn't any checking here that could happen. As long as we // have a statement to associate, this should be fine. // OpenACC 3.3 Section 6: diff --git a/clang/test/ParserOpenACC/parse-clauses.c b/clang/test/ParserOpenACC/parse-clauses.c index fb2f51890d7d..a82c3662f2ad 100644 --- a/clang/test/ParserOpenACC/parse-clauses.c +++ b/clang/test/ParserOpenACC/parse-clauses.c @@ -60,12 +60,14 @@ void func() { // expected-warning@+2{{OpenACC clause 'auto' not yet implemented, clause ignored}} // expected-warning@+1{{OpenACC construct 'kernels loop' not yet implemented, pragma ignored}} #pragma acc kernels loop seq independent auto + for(;;){} // expected-warning@+4{{OpenACC clause 'seq' not yet implemented, clause ignored}} // expected-warning@+3{{OpenACC clause 'independent' not yet implemented, clause ignored}} // expected-warning@+2{{OpenACC clause 'auto' not yet implemented, clause ignored}} // expected-warning@+1{{OpenACC construct 'serial loop' not yet implemented, pragma ignored}} #pragma acc serial loop seq, independent auto + {} // expected-warning@+4{{OpenACC clause 'seq' not yet implemented, clause ignored}} // expected-warning@+3{{OpenACC clause 'independent' not yet implemented, clause ignored}} @@ -140,105 +142,90 @@ void DefaultClause() { #pragma acc serial loop default for(;;){} - // expected-error@+3{{expected '('}} - // expected-warning@+2{{OpenACC clause 'default' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+2{{expected '('}} + // expected-warning@+1{{OpenACC clause 'default' not yet implemented, clause ignored}} #pragma acc serial default seq for(;;){} - // expected-error@+4{{expected '('}} - // expected-warning@+3{{OpenACC clause 'default' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected '('}} + // expected-warning@+2{{OpenACC clause 'default' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial default, seq for(;;){} - // expected-error@+5{{expected identifier}} - // expected-error@+4{{expected ')'}} - // expected-note@+3{{to match this '('}} - // expected-warning@+2{{OpenACC clause 'default' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+4{{expected identifier}} + // expected-error@+3{{expected ')'}} + // expected-note@+2{{to match this '('}} + // expected-warning@+1{{OpenACC clause 'default' not yet implemented, clause ignored}} #pragma acc serial default( for(;;){} - // expected-error@+5{{invalid value for 'default' clause; expected 'present' or 'none'}} - // expected-error@+4{{expected ')'}} - // expected-note@+3{{to match this '('}} - // expected-warning@+2{{OpenACC clause 'default' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+4{{invalid value for 'default' clause; expected 'present' or 'none'}} + // expected-error@+3{{expected ')'}} + // expected-note@+2{{to match this '('}} + // expected-warning@+1{{OpenACC clause 'default' not yet implemented, clause ignored}} #pragma acc serial default( seq for(;;){} - // expected-error@+5{{expected identifier}} - // expected-error@+4{{expected ')'}} - // expected-note@+3{{to match this '('}} - // expected-warning@+2{{OpenACC clause 'default' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+4{{expected identifier}} + // expected-error@+3{{expected ')'}} + // expected-note@+2{{to match this '('}} + // expected-warning@+1{{OpenACC clause 'default' not yet implemented, clause ignored}} #pragma acc serial default(, seq for(;;){} - // expected-error@+4{{expected '('}} - // expected-error@+3{{expected identifier}} - // expected-warning@+2{{OpenACC clause 'default' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected '('}} + // expected-error@+2{{expected identifier}} + // expected-warning@+1{{OpenACC clause 'default' not yet implemented, clause ignored}} #pragma acc serial default) for(;;){} - // expected-error@+4{{expected '('}} - // expected-error@+3{{expected identifier}} - // expected-warning@+2{{OpenACC clause 'default' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected '('}} + // expected-error@+2{{expected identifier}} + // expected-warning@+1{{OpenACC clause 'default' not yet implemented, clause ignored}} #pragma acc serial default), seq for(;;){} - // expected-error@+3{{expected identifier}} - // expected-warning@+2{{OpenACC clause 'default' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+2{{expected identifier}} + // expected-warning@+1{{OpenACC clause 'default' not yet implemented, clause ignored}} #pragma acc serial default() for(;;){} - // expected-error@+4{{expected identifier}} - // expected-warning@+3{{OpenACC clause 'default' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected identifier}} + // expected-warning@+2{{OpenACC clause 'default' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial default() seq for(;;){} - // expected-error@+4{{expected identifier}} - // expected-warning@+3{{OpenACC clause 'default' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected identifier}} + // expected-warning@+2{{OpenACC clause 'default' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial default(), seq for(;;){} - // expected-error@+3{{invalid value for 'default' clause; expected 'present' or 'none'}} - // expected-warning@+2{{OpenACC clause 'default' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+2{{invalid value for 'default' clause; expected 'present' or 'none'}} + // expected-warning@+1{{OpenACC clause 'default' not yet implemented, clause ignored}} #pragma acc serial default(invalid) for(;;){} - // expected-error@+4{{invalid value for 'default' clause; expected 'present' or 'none'}} - // expected-warning@+3{{OpenACC clause 'default' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{invalid value for 'default' clause; expected 'present' or 'none'}} + // expected-warning@+2{{OpenACC clause 'default' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial default(auto) seq for(;;){} - // expected-error@+4{{invalid value for 'default' clause; expected 'present' or 'none'}} - // expected-warning@+3{{OpenACC clause 'default' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{invalid value for 'default' clause; expected 'present' or 'none'}} + // expected-warning@+2{{OpenACC clause 'default' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial default(invalid), seq for(;;){} - // expected-warning@+2{{OpenACC clause 'default' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'default' not yet implemented, clause ignored}} #pragma acc serial default(none) for(;;){} - // expected-warning@+3{{OpenACC clause 'default' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'default' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial default(present), seq for(;;){} } @@ -250,108 +237,93 @@ void IfClause() { #pragma acc serial loop if for(;;){} - // expected-error@+3{{expected '('}} - // expected-warning@+2{{OpenACC clause 'if' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+2{{expected '('}} + // expected-warning@+1{{OpenACC clause 'if' not yet implemented, clause ignored}} #pragma acc serial if seq for(;;){} - // expected-error@+4{{expected '('}} - // expected-warning@+3{{OpenACC clause 'if' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected '('}} + // expected-warning@+2{{OpenACC clause 'if' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial if, seq for(;;){} - // expected-error@+5{{expected expression}} - // expected-error@+4{{expected ')'}} - // expected-note@+3{{to match this '('}} - // expected-warning@+2{{OpenACC clause 'if' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+4{{expected expression}} + // expected-error@+3{{expected ')'}} + // expected-note@+2{{to match this '('}} + // expected-warning@+1{{OpenACC clause 'if' not yet implemented, clause ignored}} #pragma acc serial if( for(;;){} - // expected-error@+5{{use of undeclared identifier 'seq'}} - // expected-error@+4{{expected ')'}} - // expected-note@+3{{to match this '('}} - // expected-warning@+2{{OpenACC clause 'if' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+4{{use of undeclared identifier 'seq'}} + // expected-error@+3{{expected ')'}} + // expected-note@+2{{to match this '('}} + // expected-warning@+1{{OpenACC clause 'if' not yet implemented, clause ignored}} #pragma acc serial if( seq for(;;){} - // expected-error@+6{{expected expression}} - // expected-error@+5{{use of undeclared identifier 'seq'}} - // expected-error@+4{{expected ')'}} - // expected-note@+3{{to match this '('}} - // expected-warning@+2{{OpenACC clause 'if' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+5{{expected expression}} + // expected-error@+4{{use of undeclared identifier 'seq'}} + // expected-error@+3{{expected ')'}} + // expected-note@+2{{to match this '('}} + // expected-warning@+1{{OpenACC clause 'if' not yet implemented, clause ignored}} #pragma acc serial if(, seq for(;;){} - // expected-error@+4{{expected '('}} - // expected-error@+3{{expected identifier}} - // expected-warning@+2{{OpenACC clause 'if' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected '('}} + // expected-error@+2{{expected identifier}} + // expected-warning@+1{{OpenACC clause 'if' not yet implemented, clause ignored}} #pragma acc serial if) for(;;){} - // expected-error@+4{{expected '('}} - // expected-error@+3{{expected identifier}} - // expected-warning@+2{{OpenACC clause 'if' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected '('}} + // expected-error@+2{{expected identifier}} + // expected-warning@+1{{OpenACC clause 'if' not yet implemented, clause ignored}} #pragma acc serial if) seq for(;;){} - // expected-error@+4{{expected '('}} - // expected-error@+3{{expected identifier}} - // expected-warning@+2{{OpenACC clause 'if' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected '('}} + // expected-error@+2{{expected identifier}} + // expected-warning@+1{{OpenACC clause 'if' not yet implemented, clause ignored}} #pragma acc serial if), seq for(;;){} - // expected-error@+3{{expected expression}} - // expected-warning@+2{{OpenACC clause 'if' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+2{{expected expression}} + // expected-warning@+1{{OpenACC clause 'if' not yet implemented, clause ignored}} #pragma acc serial if() for(;;){} - // expected-error@+4{{expected expression}} - // expected-warning@+3{{OpenACC clause 'if' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected expression}} + // expected-warning@+2{{OpenACC clause 'if' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial if() seq for(;;){} - // expected-error@+4{{expected expression}} - // expected-warning@+3{{OpenACC clause 'if' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected expression}} + // expected-warning@+2{{OpenACC clause 'if' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial if(), seq for(;;){} - // expected-error@+3{{use of undeclared identifier 'invalid_expr'}} - // expected-warning@+2{{OpenACC clause 'if' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+2{{use of undeclared identifier 'invalid_expr'}} + // expected-warning@+1{{OpenACC clause 'if' not yet implemented, clause ignored}} #pragma acc serial if(invalid_expr) for(;;){} - // expected-error@+4{{expected expression}} - // expected-warning@+3{{OpenACC clause 'if' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected expression}} + // expected-warning@+2{{OpenACC clause 'if' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial if() seq for(;;){} int i, j; - // expected-warning@+2{{OpenACC clause 'if' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'if' not yet implemented, clause ignored}} #pragma acc serial if(i > j) for(;;){} - // expected-warning@+3{{OpenACC clause 'if' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'if' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial if(1+5>3), seq for(;;){} } @@ -436,35 +408,30 @@ void SelfClause() { int i, j; - // expected-error@+4{{expected ')'}} - // expected-note@+3{{to match this '('}} - // expected-warning@+2{{OpenACC clause 'self' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected ')'}} + // expected-note@+2{{to match this '('}} + // expected-warning@+1{{OpenACC clause 'self' not yet implemented, clause ignored}} #pragma acc serial self(i > j for(;;){} - // expected-error@+5{{use of undeclared identifier 'seq'}} - // expected-error@+4{{expected ')'}} - // expected-note@+3{{to match this '('}} - // expected-warning@+2{{OpenACC clause 'self' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+4{{use of undeclared identifier 'seq'}} + // expected-error@+3{{expected ')'}} + // expected-note@+2{{to match this '('}} + // expected-warning@+1{{OpenACC clause 'self' not yet implemented, clause ignored}} #pragma acc serial self(i > j, seq for(;;){} - // expected-warning@+3{{left operand of comma operator has no effect}} - // expected-warning@+2{{OpenACC clause 'self' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{left operand of comma operator has no effect}} + // expected-warning@+1{{OpenACC clause 'self' not yet implemented, clause ignored}} #pragma acc serial self(i, j) for(;;){} - // expected-warning@+2{{OpenACC clause 'self' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'self' not yet implemented, clause ignored}} #pragma acc serial self(i > j) for(;;){} - // expected-warning@+3{{OpenACC clause 'self' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'self' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial self(1+5>3), seq for(;;){} } @@ -502,478 +469,478 @@ void SelfUpdate() { } void VarListClauses() { - // expected-error@+3{{expected '('}} - // expected-warning@+2{{OpenACC clause 'copy' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+2{{expected '('}} + // expected-warning@+1{{OpenACC clause 'copy' not yet implemented, clause ignored}} #pragma acc serial copy + for(;;){} - // expected-error@+4{{expected '('}} - // expected-warning@+3{{OpenACC clause 'copy' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected '('}} + // expected-warning@+2{{OpenACC clause 'copy' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copy, seq + for(;;){} - // expected-error@+4{{expected '('}} - // expected-error@+3{{expected identifier}} - // expected-warning@+2{{OpenACC clause 'copy' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected '('}} + // expected-error@+2{{expected identifier}} + // expected-warning@+1{{OpenACC clause 'copy' not yet implemented, clause ignored}} #pragma acc serial copy) + for(;;){} - // expected-error@+4{{expected '('}} - // expected-error@+3{{expected identifier}} - // expected-warning@+2{{OpenACC clause 'copy' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected '('}} + // expected-error@+2{{expected identifier}} + // expected-warning@+1{{OpenACC clause 'copy' not yet implemented, clause ignored}} #pragma acc serial copy), seq + for(;;){} - // expected-error@+5{{expected expression}} - // expected-error@+4{{expected ')'}} - // expected-note@+3{{to match this '('}} - // expected-warning@+2{{OpenACC clause 'copy' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+4{{expected expression}} + // expected-error@+3{{expected ')'}} + // expected-note@+2{{to match this '('}} + // expected-warning@+1{{OpenACC clause 'copy' not yet implemented, clause ignored}} #pragma acc serial copy( + for(;;){} - // expected-error@+5{{expected expression}} - // expected-error@+4{{expected ')'}} - // expected-note@+3{{to match this '('}} - // expected-warning@+2{{OpenACC clause 'copy' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+4{{expected expression}} + // expected-error@+3{{expected ')'}} + // expected-note@+2{{to match this '('}} + // expected-warning@+1{{OpenACC clause 'copy' not yet implemented, clause ignored}} #pragma acc serial copy(, seq + for(;;){} - // expected-error@+3{{expected expression}} - // expected-warning@+2{{OpenACC clause 'copy' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+2{{expected expression}} + // expected-warning@+1{{OpenACC clause 'copy' not yet implemented, clause ignored}} #pragma acc serial copy() + for(;;){} - // expected-error@+4{{expected expression}} - // expected-warning@+3{{OpenACC clause 'copy' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected expression}} + // expected-warning@+2{{OpenACC clause 'copy' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copy(), seq + for(;;){} struct Members s; struct HasMembersArray HasMem; - // expected-warning@+3{{OpenACC clause 'copy' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'copy' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copy(s.array[s.value]), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'copy' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'copy' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copy(s.array[s.value], s.array[s.value :5] ), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'copy' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'copy' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copy(HasMem.MemArr[3].array[1]), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'copy' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'copy' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copy(HasMem.MemArr[3].array[1:4]), seq + for(;;){} - // expected-error@+4{{OpenMP array section is not allowed here}} - // expected-warning@+3{{OpenACC clause 'copy' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{OpenMP array section is not allowed here}} + // expected-warning@+2{{OpenACC clause 'copy' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copy(HasMem.MemArr[1:3].array[1]), seq + for(;;){} - // expected-error@+4{{OpenMP array section is not allowed here}} - // expected-warning@+3{{OpenACC clause 'copy' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{OpenMP array section is not allowed here}} + // expected-warning@+2{{OpenACC clause 'copy' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copy(HasMem.MemArr[1:3].array[1:2]), seq + for(;;){} - // expected-error@+4{{expected expression}} - // expected-warning@+3{{OpenACC clause 'copy' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected expression}} + // expected-warning@+2{{OpenACC clause 'copy' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copy(HasMem.MemArr[:]), seq + for(;;){} - // expected-error@+4{{expected expression}} - // expected-warning@+3{{OpenACC clause 'copy' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected expression}} + // expected-warning@+2{{OpenACC clause 'copy' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copy(HasMem.MemArr[::]), seq + for(;;){} - // expected-error@+6{{expected expression}} - // expected-error@+5{{expected ']'}} - // expected-note@+4{{to match this '['}} - // expected-warning@+3{{OpenACC clause 'copy' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+5{{expected expression}} + // expected-error@+4{{expected ']'}} + // expected-note@+3{{to match this '['}} + // expected-warning@+2{{OpenACC clause 'copy' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copy(HasMem.MemArr[: :]), seq + for(;;){} - // expected-error@+4{{expected expression}} - // expected-warning@+3{{OpenACC clause 'copy' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected expression}} + // expected-warning@+2{{OpenACC clause 'copy' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copy(HasMem.MemArr[3:]), seq + for(;;){} - // expected-error@+4{{expected ','}} - // expected-warning@+3{{OpenACC clause 'use_device' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected ','}} + // expected-warning@+2{{OpenACC clause 'use_device' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial use_device(s.array[s.value] s.array[s.value :5] ), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'use_device' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'use_device' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial use_device(s.array[s.value : 5]), seq + for(;;){} - // expected-error@+4{{expected ','}} - // expected-warning@+3{{OpenACC clause 'no_create' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected ','}} + // expected-warning@+2{{OpenACC clause 'no_create' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial no_create(s.array[s.value] s.array[s.value :5] ), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'no_create' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'no_create' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial no_create(s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{expected ','}} - // expected-warning@+3{{OpenACC clause 'present' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected ','}} + // expected-warning@+2{{OpenACC clause 'present' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial present(s.array[s.value] s.array[s.value :5] ), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'present' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'present' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial present(s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{expected ','}} - // expected-warning@+3{{OpenACC clause 'deviceptr' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected ','}} + // expected-warning@+2{{OpenACC clause 'deviceptr' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial deviceptr(s.array[s.value] s.array[s.value :5] ), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'deviceptr' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'deviceptr' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial deviceptr(s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{expected ','}} - // expected-warning@+3{{OpenACC clause 'attach' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected ','}} + // expected-warning@+2{{OpenACC clause 'attach' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial attach(s.array[s.value] s.array[s.value :5] ), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'attach' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'attach' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial attach(s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{expected ','}} - // expected-warning@+3{{OpenACC clause 'detach' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected ','}} + // expected-warning@+2{{OpenACC clause 'detach' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial detach(s.array[s.value] s.array[s.value :5] ), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'detach' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'detach' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial detach(s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{expected ','}} - // expected-warning@+3{{OpenACC clause 'private' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected ','}} + // expected-warning@+2{{OpenACC clause 'private' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial private(s.array[s.value] s.array[s.value :5] ), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'private' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'private' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial private(s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{expected ','}} - // expected-warning@+3{{OpenACC clause 'firstprivate' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected ','}} + // expected-warning@+2{{OpenACC clause 'firstprivate' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial firstprivate(s.array[s.value] s.array[s.value :5] ), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'firstprivate' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'firstprivate' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial firstprivate(s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{expected ','}} - // expected-warning@+3{{OpenACC clause 'delete' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected ','}} + // expected-warning@+2{{OpenACC clause 'delete' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial delete(s.array[s.value] s.array[s.value :5] ), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'delete' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'delete' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial delete(s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{expected ','}} - // expected-warning@+3{{OpenACC clause 'use_device' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected ','}} + // expected-warning@+2{{OpenACC clause 'use_device' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial use_device(s.array[s.value] s.array[s.value :5] ), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'use_device' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'use_device' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial use_device(s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{expected ','}} - // expected-warning@+3{{OpenACC clause 'device_resident' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected ','}} + // expected-warning@+2{{OpenACC clause 'device_resident' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial device_resident(s.array[s.value] s.array[s.value :5] ), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'device_resident' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'device_resident' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial device_resident(s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{expected ','}} - // expected-warning@+3{{OpenACC clause 'link' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected ','}} + // expected-warning@+2{{OpenACC clause 'link' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial link(s.array[s.value] s.array[s.value :5] ), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'link' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'link' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial link(s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{expected ','}} - // expected-warning@+3{{OpenACC clause 'host' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected ','}} + // expected-warning@+2{{OpenACC clause 'host' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial host(s.array[s.value] s.array[s.value :5] ), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'host' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'host' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial host(s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{expected ','}} - // expected-warning@+3{{OpenACC clause 'device' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected ','}} + // expected-warning@+2{{OpenACC clause 'device' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial device(s.array[s.value] s.array[s.value :5] ), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'device' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'device' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial device(s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{expected ','}} - // expected-warning@+3{{OpenACC clause 'copyout' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected ','}} + // expected-warning@+2{{OpenACC clause 'copyout' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copyout(s.array[s.value] s.array[s.value :5] ), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'copyout' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'copyout' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copyout(s.array[s.value : 5], s.value), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'copyout' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'copyout' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copyout(zero:s.array[s.value : 5], s.value), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'copyout' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'copyout' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copyout(zero : s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{use of undeclared identifier 'zero'}} - // expected-warning@+3{{OpenACC clause 'copyout' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{use of undeclared identifier 'zero'}} + // expected-warning@+2{{OpenACC clause 'copyout' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copyout(zero s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{invalid tag 'readonly' on 'copyout' clause}} - // expected-warning@+3{{OpenACC clause 'copyout' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{invalid tag 'readonly' on 'copyout' clause}} + // expected-warning@+2{{OpenACC clause 'copyout' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copyout(readonly:s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{invalid tag 'invalid' on 'copyout' clause}} - // expected-warning@+3{{OpenACC clause 'copyout' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{invalid tag 'invalid' on 'copyout' clause}} + // expected-warning@+2{{OpenACC clause 'copyout' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copyout(invalid:s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{invalid tag 'invalid' on 'copyout' clause}} - // expected-warning@+3{{OpenACC clause 'copyout' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{invalid tag 'invalid' on 'copyout' clause}} + // expected-warning@+2{{OpenACC clause 'copyout' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copyout(invalid:s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{use of undeclared identifier 'invalid'}} - // expected-warning@+3{{OpenACC clause 'copyout' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{use of undeclared identifier 'invalid'}} + // expected-warning@+2{{OpenACC clause 'copyout' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copyout(invalid s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{expected ','}} - // expected-warning@+3{{OpenACC clause 'create' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected ','}} + // expected-warning@+2{{OpenACC clause 'create' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial create(s.array[s.value] s.array[s.value :5] ), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'create' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'create' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial create(s.array[s.value : 5], s.value), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'create' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'create' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial create(zero:s.array[s.value : 5], s.value), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'create' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'create' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial create(zero : s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{use of undeclared identifier 'zero'}} - // expected-warning@+3{{OpenACC clause 'create' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{use of undeclared identifier 'zero'}} + // expected-warning@+2{{OpenACC clause 'create' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial create(zero s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{invalid tag 'readonly' on 'create' clause}} - // expected-warning@+3{{OpenACC clause 'create' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{invalid tag 'readonly' on 'create' clause}} + // expected-warning@+2{{OpenACC clause 'create' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial create(readonly:s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{invalid tag 'invalid' on 'create' clause}} - // expected-warning@+3{{OpenACC clause 'create' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{invalid tag 'invalid' on 'create' clause}} + // expected-warning@+2{{OpenACC clause 'create' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial create(invalid:s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{invalid tag 'invalid' on 'create' clause}} - // expected-warning@+3{{OpenACC clause 'create' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{invalid tag 'invalid' on 'create' clause}} + // expected-warning@+2{{OpenACC clause 'create' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial create(invalid:s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{use of undeclared identifier 'invalid'}} - // expected-warning@+3{{OpenACC clause 'create' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{use of undeclared identifier 'invalid'}} + // expected-warning@+2{{OpenACC clause 'create' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial create(invalid s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{expected ','}} - // expected-warning@+3{{OpenACC clause 'copyin' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected ','}} + // expected-warning@+2{{OpenACC clause 'copyin' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copyin(s.array[s.value] s.array[s.value :5] ), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'copyin' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'copyin' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copyin(s.array[s.value : 5], s.value), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'copyin' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'copyin' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copyin(readonly:s.array[s.value : 5], s.value), seq + for(;;){} - // expected-warning@+3{{OpenACC clause 'copyin' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+2{{OpenACC clause 'copyin' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copyin(readonly : s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{use of undeclared identifier 'readonly'}} - // expected-warning@+3{{OpenACC clause 'copyin' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{use of undeclared identifier 'readonly'}} + // expected-warning@+2{{OpenACC clause 'copyin' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copyin(readonly s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{invalid tag 'zero' on 'copyin' clause}} - // expected-warning@+3{{OpenACC clause 'copyin' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{invalid tag 'zero' on 'copyin' clause}} + // expected-warning@+2{{OpenACC clause 'copyin' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copyin(zero :s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{invalid tag 'invalid' on 'copyin' clause}} - // expected-warning@+3{{OpenACC clause 'copyin' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{invalid tag 'invalid' on 'copyin' clause}} + // expected-warning@+2{{OpenACC clause 'copyin' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copyin(invalid:s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{invalid tag 'invalid' on 'copyin' clause}} - // expected-warning@+3{{OpenACC clause 'copyin' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{invalid tag 'invalid' on 'copyin' clause}} + // expected-warning@+2{{OpenACC clause 'copyin' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copyin(invalid:s.array[s.value : 5], s.value), seq + for(;;){} - // expected-error@+4{{use of undeclared identifier 'invalid'}} - // expected-warning@+3{{OpenACC clause 'copyin' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{use of undeclared identifier 'invalid'}} + // expected-warning@+2{{OpenACC clause 'copyin' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial copyin(invalid s.array[s.value : 5], s.value), seq + for(;;){} } void ReductionClauseParsing() { char *Begin, *End; - // expected-error@+3{{expected '('}} - // expected-warning@+2{{OpenACC clause 'reduction' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+2{{expected '('}} + // expected-warning@+1{{OpenACC clause 'reduction' not yet implemented, clause ignored}} #pragma acc serial reduction - // expected-error@+4{{missing reduction operator, expected '+', '*', 'max', 'min', '&', '|', '^', '&&', or '||', follwed by a ':'}} - // expected-error@+3{{expected expression}} - // expected-warning@+2{{OpenACC clause 'reduction' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} -#pragma acc serial reduction() + for(;;){} // expected-error@+3{{missing reduction operator, expected '+', '*', 'max', 'min', '&', '|', '^', '&&', or '||', follwed by a ':'}} - // expected-warning@+2{{OpenACC clause 'reduction' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+2{{expected expression}} + // expected-warning@+1{{OpenACC clause 'reduction' not yet implemented, clause ignored}} +#pragma acc serial reduction() + for(;;){} + // expected-error@+2{{missing reduction operator, expected '+', '*', 'max', 'min', '&', '|', '^', '&&', or '||', follwed by a ':'}} + // expected-warning@+1{{OpenACC clause 'reduction' not yet implemented, clause ignored}} #pragma acc serial reduction(Begin) - // expected-error@+3{{missing reduction operator, expected '+', '*', 'max', 'min', '&', '|', '^', '&&', or '||', follwed by a ':'}} - // expected-warning@+2{{OpenACC clause 'reduction' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + for(;;){} + // expected-error@+2{{missing reduction operator, expected '+', '*', 'max', 'min', '&', '|', '^', '&&', or '||', follwed by a ':'}} + // expected-warning@+1{{OpenACC clause 'reduction' not yet implemented, clause ignored}} #pragma acc serial reduction(Begin, End) - // expected-error@+3{{missing reduction operator, expected '+', '*', 'max', 'min', '&', '|', '^', '&&', or '||', follwed by a ':'}} - // expected-warning@+2{{OpenACC clause 'reduction' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + for(;;){} + // expected-error@+2{{missing reduction operator, expected '+', '*', 'max', 'min', '&', '|', '^', '&&', or '||', follwed by a ':'}} + // expected-warning@+1{{OpenACC clause 'reduction' not yet implemented, clause ignored}} #pragma acc serial reduction(Begin, End) - // expected-warning@+2{{OpenACC clause 'reduction' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + for(;;){} + // expected-warning@+1{{OpenACC clause 'reduction' not yet implemented, clause ignored}} #pragma acc serial reduction(+:Begin) - // expected-warning@+2{{OpenACC clause 'reduction' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + for(;;){} + // expected-warning@+1{{OpenACC clause 'reduction' not yet implemented, clause ignored}} #pragma acc serial reduction(+:Begin, End) - // expected-warning@+2{{OpenACC clause 'reduction' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + for(;;){} + // expected-warning@+1{{OpenACC clause 'reduction' not yet implemented, clause ignored}} #pragma acc serial reduction(*: Begin, End) - // expected-warning@+2{{OpenACC clause 'reduction' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + for(;;){} + // expected-warning@+1{{OpenACC clause 'reduction' not yet implemented, clause ignored}} #pragma acc serial reduction(max : Begin, End) - // expected-warning@+2{{OpenACC clause 'reduction' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + for(;;){} + // expected-warning@+1{{OpenACC clause 'reduction' not yet implemented, clause ignored}} #pragma acc serial reduction(min: Begin, End) - // expected-warning@+2{{OpenACC clause 'reduction' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + for(;;){} + // expected-warning@+1{{OpenACC clause 'reduction' not yet implemented, clause ignored}} #pragma acc serial reduction(&: Begin, End) - // expected-warning@+2{{OpenACC clause 'reduction' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + for(;;){} + // expected-warning@+1{{OpenACC clause 'reduction' not yet implemented, clause ignored}} #pragma acc serial reduction(|: Begin, End) - // expected-warning@+2{{OpenACC clause 'reduction' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + for(;;){} + // expected-warning@+1{{OpenACC clause 'reduction' not yet implemented, clause ignored}} #pragma acc serial reduction(^: Begin, End) - // expected-warning@+3{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+2{{OpenACC clause 'reduction' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} -#pragma acc serial seq, reduction(&&: Begin, End) - // expected-warning@+3{{OpenACC clause 'reduction' not yet implemented, clause ignored}} + for(;;){} // expected-warning@+2{{OpenACC clause 'seq' not yet implemented, clause ignored}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-warning@+1{{OpenACC clause 'reduction' not yet implemented, clause ignored}} +#pragma acc serial seq, reduction(&&: Begin, End) + for(;;){} + // expected-warning@+2{{OpenACC clause 'reduction' not yet implemented, clause ignored}} + // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}} #pragma acc serial reduction(||: Begin, End), seq + for(;;){} } int returns_int(); diff --git a/clang/test/ParserOpenACC/parse-constructs.c b/clang/test/ParserOpenACC/parse-constructs.c index adb5e3c7c755..ecedfd9e9e6d 100644 --- a/clang/test/ParserOpenACC/parse-constructs.c +++ b/clang/test/ParserOpenACC/parse-constructs.c @@ -39,23 +39,19 @@ void func() { // expected-note@+1{{to match this '('}} #pragma acc parallel( clause list for(;;){} - // expected-error@+3{{expected clause-list or newline in OpenACC directive}} - // expected-error@+2{{invalid OpenACC clause 'clause'}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+2{{expected clause-list or newline in OpenACC directive}} + // expected-error@+1{{invalid OpenACC clause 'clause'}} #pragma acc serial() clause list for(;;){} - // expected-error@+4{{expected clause-list or newline in OpenACC directive}} - // expected-error@+3{{expected ')'}} - // expected-note@+2{{to match this '('}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+3{{expected clause-list or newline in OpenACC directive}} + // expected-error@+2{{expected ')'}} + // expected-note@+1{{to match this '('}} #pragma acc serial( clause list for(;;){} - // expected-error@+2{{invalid OpenACC clause 'clause'}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+1{{invalid OpenACC clause 'clause'}} #pragma acc serial clause list for(;;){} - // expected-error@+2{{invalid OpenACC clause 'clause'}} - // expected-warning@+1{{OpenACC construct 'kernels' not yet implemented, pragma ignored}} + // expected-error@+1{{invalid OpenACC clause 'clause'}} #pragma acc kernels clause list for(;;){} // expected-error@+2{{invalid OpenACC clause 'clause'}} @@ -93,8 +89,7 @@ void func() { // expected-error@+1{{invalid OpenACC clause 'invalid'}} #pragma acc parallel invalid clause list for(;;){} - // expected-error@+2{{invalid OpenACC clause 'invalid'}} - // expected-warning@+1{{OpenACC construct 'serial' not yet implemented, pragma ignored}} + // expected-error@+1{{invalid OpenACC clause 'invalid'}} #pragma acc serial invalid clause list for(;;){} // expected-error@+2{{invalid OpenACC clause 'clause'}} diff --git a/clang/test/SemaOpenACC/compute-construct-ast.cpp b/clang/test/SemaOpenACC/compute-construct-ast.cpp index 351fd1c3e0ee..0d1d0b266573 100644 --- a/clang/test/SemaOpenACC/compute-construct-ast.cpp +++ b/clang/test/SemaOpenACC/compute-construct-ast.cpp @@ -15,6 +15,30 @@ void NormalFunc() { #pragma acc parallel {} } + // FIXME: Add a test once we have clauses for this. + // CHECK-NEXT: OpenACCComputeConstruct {{.*}}serial + // CHECK-NEXT: CompoundStmt +#pragma acc serial + { +#pragma acc serial + // CHECK-NEXT: OpenACCComputeConstruct {{.*}}serial + // CHECK-NEXT: OpenACCComputeConstruct {{.*}}serial + // CHECK-NEXT: CompoundStmt +#pragma acc serial + {} + } + // FIXME: Add a test once we have clauses for this. + // CHECK-NEXT: OpenACCComputeConstruct {{.*}}kernels + // CHECK-NEXT: CompoundStmt +#pragma acc kernels + { +#pragma acc kernels + // CHECK-NEXT: OpenACCComputeConstruct {{.*}}kernels + // CHECK-NEXT: OpenACCComputeConstruct {{.*}}kernels + // CHECK-NEXT: CompoundStmt +#pragma acc kernels + {} + } } template @@ -24,6 +48,16 @@ void TemplFunc() { typename T::type I; } +#pragma acc serial + { + typename T::type I; + } + +#pragma acc kernels + { + typename T::type I; + } + // CHECK-LABEL: FunctionTemplateDecl {{.*}}TemplFunc // CHECK-NEXT: TemplateTypeParmDecl @@ -34,6 +68,14 @@ void TemplFunc() { // CHECK-NEXT: CompoundStmt // CHECK-NEXT: DeclStmt // CHECK-NEXT: VarDecl{{.*}} I 'typename T::type' + // CHECK-NEXT: OpenACCComputeConstruct {{.*}}serial + // CHECK-NEXT: CompoundStmt + // CHECK-NEXT: DeclStmt + // CHECK-NEXT: VarDecl{{.*}} I 'typename T::type' + // CHECK-NEXT: OpenACCComputeConstruct {{.*}}kernels + // CHECK-NEXT: CompoundStmt + // CHECK-NEXT: DeclStmt + // CHECK-NEXT: VarDecl{{.*}} I 'typename T::type' // Check instantiation. // CHECK-LABEL: FunctionDecl{{.*}} used TemplFunc 'void ()' implicit_instantiation @@ -45,6 +87,14 @@ void TemplFunc() { // CHECK-NEXT: CompoundStmt // CHECK-NEXT: DeclStmt // CHECK-NEXT: VarDecl{{.*}} I 'typename S::type':'int' + // CHECK-NEXT: OpenACCComputeConstruct {{.*}}serial + // CHECK-NEXT: CompoundStmt + // CHECK-NEXT: DeclStmt + // CHECK-NEXT: VarDecl{{.*}} I 'typename S::type':'int' + // CHECK-NEXT: OpenACCComputeConstruct {{.*}}kernels + // CHECK-NEXT: CompoundStmt + // CHECK-NEXT: DeclStmt + // CHECK-NEXT: VarDecl{{.*}} I 'typename S::type':'int' } struct S { diff --git a/clang/test/SemaOpenACC/no-branch-in-out.c b/clang/test/SemaOpenACC/no-branch-in-out.c index eccc64324500..5fff18eb7cdd 100644 --- a/clang/test/SemaOpenACC/no-branch-in-out.c +++ b/clang/test/SemaOpenACC/no-branch-in-out.c @@ -37,6 +37,18 @@ void BreakContinue() { break; // expected-error{{invalid branch out of OpenACC Compute Construct}} } +#pragma acc serial + for(int i = 0; i < 5; ++i) { + if (i > 1) + break; // expected-error{{invalid branch out of OpenACC Compute Construct}} + } + +#pragma acc kernels + for(int i = 0; i < 5; ++i) { + if (i > 1) + break; // expected-error{{invalid branch out of OpenACC Compute Construct}} + } + #pragma acc parallel switch(j) { case 1: @@ -99,6 +111,16 @@ void Return() { return;// expected-error{{invalid return out of OpenACC Compute Construct}} } +#pragma acc serial + { + return;// expected-error{{invalid return out of OpenACC Compute Construct}} + } + +#pragma acc kernels + { + return;// expected-error{{invalid return out of OpenACC Compute Construct}} + } + #pragma acc parallel { { @@ -255,6 +277,34 @@ LABEL13:{} LABEL14:{} ({goto LABEL14;}); } + + + + ({goto LABEL15;});// expected-error{{cannot jump from this goto statement to its label}} +#pragma acc serial// expected-note{{invalid branch into OpenACC Compute Construct}} + { +LABEL15:{} + } + +LABEL16:{} +#pragma acc serial// expected-note{{invalid branch out of OpenACC Compute Construct}} + { + ({goto LABEL16;});// expected-error{{cannot jump from this goto statement to its label}} + } + + + ({goto LABEL17;});// expected-error{{cannot jump from this goto statement to its label}} +#pragma acc kernels// expected-note{{invalid branch into OpenACC Compute Construct}} + { +LABEL17:{} + } + +LABEL18:{} +#pragma acc kernels// expected-note{{invalid branch out of OpenACC Compute Construct}} + { + ({goto LABEL18;});// expected-error{{cannot jump from this goto statement to its label}} + } + } void IndirectGoto1() { @@ -329,6 +379,14 @@ void DuffsDevice() { } } + switch (j) { +#pragma acc kernels + for(int i =0; i < 5; ++i) { + default: // expected-error{{invalid branch into OpenACC Compute Construct}} + {} + } + } + switch (j) { #pragma acc parallel for(int i =0; i < 5; ++i) { @@ -336,4 +394,12 @@ void DuffsDevice() { {} } } + + switch (j) { +#pragma acc serial + for(int i =0; i < 5; ++i) { + case 'a' ... 'z': // expected-error{{invalid branch into OpenACC Compute Construct}} + {} + } + } } diff --git a/clang/test/SemaOpenACC/no-branch-in-out.cpp b/clang/test/SemaOpenACC/no-branch-in-out.cpp index 6ee4553cd303..bc559f1898f1 100644 --- a/clang/test/SemaOpenACC/no-branch-in-out.cpp +++ b/clang/test/SemaOpenACC/no-branch-in-out.cpp @@ -147,6 +147,17 @@ void Exceptions() { throw; // expected-error{{invalid throw out of OpenACC Compute Construct}} } +#pragma acc serial + for(int i = 0; i < 5; ++i) { + throw; // expected-error{{invalid throw out of OpenACC Compute Construct}} + } + +#pragma acc kernels + for(int i = 0; i < 5; ++i) { + throw; // expected-error{{invalid throw out of OpenACC Compute Construct}} + } + + #pragma acc parallel for(int i = 0; i < 5; ++i) { try { diff --git a/clang/test/SemaOpenACC/parallel-assoc-stmt-inst.cpp b/clang/test/SemaOpenACC/parallel-assoc-stmt-inst.cpp index 0464e164a754..f533baa4c0e6 100644 --- a/clang/test/SemaOpenACC/parallel-assoc-stmt-inst.cpp +++ b/clang/test/SemaOpenACC/parallel-assoc-stmt-inst.cpp @@ -4,6 +4,10 @@ template void Func() { #pragma acc parallel typename T::type I; //#ILOC +#pragma acc serial + typename T::type IS; //#ILOCSERIAL +#pragma acc kernels + typename T::type IK; //#ILOCKERNELS } struct S { @@ -13,6 +17,8 @@ struct S { void use() { Func(); // expected-error@#ILOC{{type 'int' cannot be used prior to '::' because it has no members}} - // expected-note@+1{{in instantiation of function template specialization 'Func' requested here}} + // expected-note@+3{{in instantiation of function template specialization 'Func' requested here}} + // expected-error@#ILOCSERIAL{{type 'int' cannot be used prior to '::' because it has no members}} + // expected-error@#ILOCKERNELS{{type 'int' cannot be used prior to '::' because it has no members}} Func(); } diff --git a/clang/test/SemaOpenACC/parallel-loc-and-stmt.c b/clang/test/SemaOpenACC/parallel-loc-and-stmt.c index 5189a6aa44f0..ba29f6da8ba2 100644 --- a/clang/test/SemaOpenACC/parallel-loc-and-stmt.c +++ b/clang/test/SemaOpenACC/parallel-loc-and-stmt.c @@ -3,14 +3,32 @@ // expected-error@+1{{OpenACC construct 'parallel' cannot be used here; it can only be used in a statement context}} #pragma acc parallel +// expected-error@+1{{OpenACC construct 'serial' cannot be used here; it can only be used in a statement context}} +#pragma acc serial + +// expected-error@+1{{OpenACC construct 'kernels' cannot be used here; it can only be used in a statement context}} +#pragma acc kernels + // expected-error@+1{{OpenACC construct 'parallel' cannot be used here; it can only be used in a statement context}} #pragma acc parallel int foo; +// expected-error@+1{{OpenACC construct 'serial' cannot be used here; it can only be used in a statement context}} +#pragma acc serial +int foo2; +// expected-error@+1{{OpenACC construct 'kernels' cannot be used here; it can only be used in a statement context}} +#pragma acc kernels +int foo3; struct S { // expected-error@+1{{OpenACC construct 'parallel' cannot be used here; it can only be used in a statement context}} #pragma acc parallel int foo; +// expected-error@+1{{OpenACC construct 'serial' cannot be used here; it can only be used in a statement context}} +#pragma acc serial +int foo2; +// expected-error@+1{{OpenACC construct 'kernels' cannot be used here; it can only be used in a statement context}} +#pragma acc kernels +int foo3; }; void func() { @@ -31,6 +49,15 @@ void func() { #pragma acc parallel } + { +// expected-error@+2{{expected statement}} +#pragma acc serial + } + { +// expected-error@+2{{expected statement}} +#pragma acc kernels + } + #pragma acc parallel while(0){} -- GitLab From e10dc60ad742f37e61fd6cf13e1b7d48d7f341d9 Mon Sep 17 00:00:00 2001 From: Aart Bik <39774503+aartbik@users.noreply.github.com> Date: Mon, 4 Mar 2024 12:57:48 -0800 Subject: [PATCH 045/929] [mlir][sparse][nfc] fixed typo in "translate" (#83891) --- .../mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td | 2 +- .../mlir/Dialect/SparseTensor/IR/SparseTensorType.h | 8 ++++---- mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td index 5d1db2323f95..d3be8a3009ba 100644 --- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td +++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td @@ -431,7 +431,7 @@ def SparseTensorEncodingAttr : SparseTensor_Attr<"SparseTensorEncoding", // // Helper function to translate between level/dimension space. // - SmallVector tranlateShape(::mlir::ArrayRef srcShape, ::mlir::sparse_tensor::CrdTransDirectionKind) const; + SmallVector translateShape(::mlir::ArrayRef srcShape, ::mlir::sparse_tensor::CrdTransDirectionKind) const; ValueRange translateCrds(::mlir::OpBuilder &builder, ::mlir::Location loc, ::mlir::ValueRange crds, ::mlir::sparse_tensor::CrdTransDirectionKind) const; // diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorType.h b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorType.h index c93a4fcd922c..bd2c3c1dd551 100644 --- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorType.h +++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorType.h @@ -249,14 +249,14 @@ public: /// Returns the Level-shape. SmallVector getLvlShape() const { - return getEncoding().tranlateShape(getDimShape(), - CrdTransDirectionKind::dim2lvl); + return getEncoding().translateShape(getDimShape(), + CrdTransDirectionKind::dim2lvl); } /// Returns the Level-shape. SmallVector getBatchLvlShape() const { - auto lvlShape = getEncoding().tranlateShape(getDimShape(), - CrdTransDirectionKind::dim2lvl); + auto lvlShape = getEncoding().translateShape( + getDimShape(), CrdTransDirectionKind::dim2lvl); lvlShape.truncate(getEncoding().getBatchLvlRank()); return lvlShape; } diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp index 232635ca84a4..244a082d0487 100644 --- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp +++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp @@ -413,8 +413,8 @@ SparseTensorEncodingAttr::getStaticLvlSliceStride(Level lvl) const { } SmallVector -SparseTensorEncodingAttr::tranlateShape(ArrayRef srcShape, - CrdTransDirectionKind dir) const { +SparseTensorEncodingAttr::translateShape(ArrayRef srcShape, + CrdTransDirectionKind dir) const { if (isIdentity()) return SmallVector(srcShape); @@ -1392,7 +1392,7 @@ void ReinterpretMapOp::build(OpBuilder &odsBuilder, OperationState &odsState, auto srcStt = getSparseTensorType(source); SmallVector srcLvlShape = srcStt.getLvlShape(); SmallVector dstDimShape = - dstEnc.tranlateShape(srcLvlShape, CrdTransDirectionKind::lvl2dim); + dstEnc.translateShape(srcLvlShape, CrdTransDirectionKind::lvl2dim); auto dstTp = RankedTensorType::get(dstDimShape, srcStt.getElementType(), dstEnc); return build(odsBuilder, odsState, dstTp, source); -- GitLab From 2e93ee6a2389cc65faf9739cd7a6c438d5e05a9e Mon Sep 17 00:00:00 2001 From: Mircea Trofin Date: Mon, 4 Mar 2024 12:59:36 -0800 Subject: [PATCH 046/929] Update Benchmark (#83488) Addresses the `third-party/benchmark` part of #81859 (by happening to remove `requirements.txt`) --- third-party/benchmark/.ycm_extra_conf.py | 8 +- third-party/benchmark/AUTHORS | 12 +- third-party/benchmark/CMakeLists.txt | 100 ++-- third-party/benchmark/CONTRIBUTORS | 14 +- third-party/benchmark/README.md | 13 +- third-party/benchmark/WORKSPACE | 54 +- .../benchmark/bindings/python/build_defs.bzl | 8 +- .../python/google_benchmark/__init__.py | 58 +- .../python/google_benchmark/benchmark.cc | 149 ++--- .../python/google_benchmark/example.py | 6 +- .../benchmark/bindings/python/pybind11.BUILD | 20 - .../bindings/python/python_headers.BUILD | 4 + .../bindings/python/requirements.txt | 2 - .../benchmark/cmake/CXXFeatureCheck.cmake | 29 +- .../benchmark/cmake/GetGitVersion.cmake | 30 +- third-party/benchmark/cmake/GoogleTest.cmake | 16 +- .../benchmark/cmake/Modules/FindPFM.cmake | 38 +- third-party/benchmark/cmake/benchmark.pc.in | 4 +- third-party/benchmark/docs/AssemblyTests.md | 2 + third-party/benchmark/docs/_config.yml | 4 +- third-party/benchmark/docs/dependencies.md | 22 +- third-party/benchmark/docs/index.md | 4 +- third-party/benchmark/docs/perf_counters.md | 13 +- third-party/benchmark/docs/releasing.md | 24 +- third-party/benchmark/docs/tools.md | 140 +++++ third-party/benchmark/docs/user_guide.md | 158 ++++- .../benchmark/include/benchmark/benchmark.h | 563 +++++++++++++----- third-party/benchmark/requirements.txt | 3 - third-party/benchmark/setup.py | 193 +++--- third-party/benchmark/src/CMakeLists.txt | 28 +- third-party/benchmark/src/benchmark.cc | 289 +++++++-- .../benchmark/src/benchmark_api_internal.cc | 20 +- .../benchmark/src/benchmark_api_internal.h | 3 + third-party/benchmark/src/benchmark_main.cc | 1 + third-party/benchmark/src/benchmark_name.cc | 5 +- .../benchmark/src/benchmark_register.cc | 52 +- .../benchmark/src/benchmark_register.h | 5 +- third-party/benchmark/src/benchmark_runner.cc | 201 ++++++- third-party/benchmark/src/benchmark_runner.h | 33 +- third-party/benchmark/src/check.h | 31 +- third-party/benchmark/src/colorprint.cc | 46 +- third-party/benchmark/src/commandlineflags.cc | 19 +- third-party/benchmark/src/commandlineflags.h | 39 +- third-party/benchmark/src/complexity.cc | 43 +- third-party/benchmark/src/complexity.h | 2 +- third-party/benchmark/src/console_reporter.cc | 34 +- third-party/benchmark/src/counter.cc | 4 +- third-party/benchmark/src/csv_reporter.cc | 23 +- third-party/benchmark/src/cycleclock.h | 40 +- third-party/benchmark/src/internal_macros.h | 17 +- third-party/benchmark/src/json_reporter.cc | 36 +- third-party/benchmark/src/log.h | 26 +- third-party/benchmark/src/perf_counters.cc | 242 ++++++-- third-party/benchmark/src/perf_counters.h | 108 ++-- third-party/benchmark/src/re.h | 2 +- third-party/benchmark/src/reporter.cc | 14 +- third-party/benchmark/src/sleep.cc | 66 -- third-party/benchmark/src/sleep.h | 15 - third-party/benchmark/src/statistics.cc | 38 +- third-party/benchmark/src/statistics.h | 12 +- third-party/benchmark/src/string_util.cc | 65 +- third-party/benchmark/src/string_util.h | 11 +- third-party/benchmark/src/sysinfo.cc | 443 +++++++++----- third-party/benchmark/src/thread_manager.h | 4 +- third-party/benchmark/src/timers.cc | 24 +- .../benchmark/test/AssemblyTests.cmake | 21 + third-party/benchmark/test/CMakeLists.txt | 127 ++-- .../benchmark/test/args_product_test.cc | 4 +- third-party/benchmark/test/basic_test.cc | 9 +- third-party/benchmark/test/benchmark_gtest.cc | 14 +- .../benchmark/test/benchmark_name_gtest.cc | 8 + .../benchmark_random_interleaving_gtest.cc | 5 +- .../test/benchmark_setup_teardown_test.cc | 18 +- third-party/benchmark/test/benchmark_test.cc | 65 +- .../test/clobber_memory_assembly_test.cc | 1 + third-party/benchmark/test/complexity_test.cc | 158 +++-- .../benchmark/test/diagnostics_test.cc | 15 +- .../test/donotoptimize_assembly_test.cc | 40 ++ .../benchmark/test/donotoptimize_test.cc | 28 +- third-party/benchmark/test/filter_test.cc | 31 +- third-party/benchmark/test/fixture_test.cc | 6 +- third-party/benchmark/test/link_main_test.cc | 3 +- third-party/benchmark/test/map_test.cc | 10 +- .../benchmark/test/memory_manager_test.cc | 11 +- .../benchmark/test/multiple_ranges_test.cc | 4 +- third-party/benchmark/test/options_test.cc | 6 +- third-party/benchmark/test/output_test.h | 10 +- .../benchmark/test/output_test_helper.cc | 47 +- .../benchmark/test/perf_counters_gtest.cc | 266 +++++++-- .../benchmark/test/perf_counters_test.cc | 71 ++- .../benchmark/test/register_benchmark_test.cc | 26 +- .../benchmark/test/reporter_output_test.cc | 18 +- .../benchmark/test/skip_with_error_test.cc | 17 +- third-party/benchmark/test/spec_arg_test.cc | 16 +- .../benchmark/test/statistics_gtest.cc | 4 +- .../benchmark/test/string_util_gtest.cc | 55 +- .../test/user_counters_tabular_test.cc | 10 +- .../benchmark/test/user_counters_test.cc | 28 +- .../test/user_counters_thousands_test.cc | 32 +- third-party/benchmark/tools/compare.py | 62 +- .../tools/gbench/Inputs/test1_run1.json | 8 + .../tools/gbench/Inputs/test1_run2.json | 8 + .../benchmark/tools/gbench/__init__.py | 2 +- third-party/benchmark/tools/gbench/report.py | 271 ++++++++- third-party/benchmark/tools/gbench/util.py | 52 +- third-party/benchmark/tools/requirements.txt | 3 +- third-party/benchmark/tools/strip_asm.py | 73 ++- 107 files changed, 3773 insertions(+), 1622 deletions(-) delete mode 100644 third-party/benchmark/bindings/python/pybind11.BUILD delete mode 100644 third-party/benchmark/bindings/python/requirements.txt delete mode 100644 third-party/benchmark/requirements.txt delete mode 100644 third-party/benchmark/src/sleep.cc delete mode 100644 third-party/benchmark/src/sleep.h diff --git a/third-party/benchmark/.ycm_extra_conf.py b/third-party/benchmark/.ycm_extra_conf.py index 1482c7b00202..caf257f0540e 100644 --- a/third-party/benchmark/.ycm_extra_conf.py +++ b/third-party/benchmark/.ycm_extra_conf.py @@ -1,4 +1,5 @@ import os + import ycm_core # These are the compilation flags that will be used in case there's no @@ -91,7 +92,9 @@ def GetCompilationInfoForFile(filename): for extension in SOURCE_EXTENSIONS: replacement_file = basename + extension if os.path.exists(replacement_file): - compilation_info = database.GetCompilationInfoForFile(replacement_file) + compilation_info = database.GetCompilationInfoForFile( + replacement_file + ) if compilation_info.compiler_flags_: return compilation_info return None @@ -107,7 +110,8 @@ def FlagsForFile(filename, **kwargs): return None final_flags = MakeRelativePathsInFlagsAbsolute( - compilation_info.compiler_flags_, compilation_info.compiler_working_dir_ + compilation_info.compiler_flags_, + compilation_info.compiler_working_dir_, ) else: relative_to = DirectoryOfThisScript() diff --git a/third-party/benchmark/AUTHORS b/third-party/benchmark/AUTHORS index 54770f35499c..2170e46fd4a0 100644 --- a/third-party/benchmark/AUTHORS +++ b/third-party/benchmark/AUTHORS @@ -13,6 +13,7 @@ Alex Steele Andriy Berestovskyy Arne Beer Carto +Cezary Skrzyński Christian Wassermann Christopher Seymour Colin Braley @@ -27,10 +28,13 @@ Eric Backus Eric Fiselier Eugene Zhuk Evgeny Safronov +Fabien Pichot Federico Ficarelli Felix Homann +Gergely Meszaros Gergő Szitár Google Inc. +Henrique Bucher International Business Machines Corporation Ismael Jimenez Martinez Jern-Kuan Leong @@ -41,8 +45,11 @@ Jussi Knuuttila Kaito Udagawa Kishan Kumar Lei Xu +Marcel Jacobse Matt Clarkson Maxim Vafin +Mike Apodaca +Min-Yih Hsu MongoDB Inc. Nick Hutchinson Norman Heino @@ -50,13 +57,16 @@ Oleksandr Sochka Ori Livneh Paul Redmond Radoslav Yovchev +Raghu Raja +Rainer Orth Roman Lebedev Sayan Bhattacharjee +Shapr3D Shuo Chen +Staffan Tjernstrom Steinar H. Gunderson Stripe, Inc. Tobias Schmidt Yixuan Qiu Yusuke Suzuki Zbigniew Skowron -Min-Yih Hsu diff --git a/third-party/benchmark/CMakeLists.txt b/third-party/benchmark/CMakeLists.txt index 8af49406d052..d9bcc6a4939b 100644 --- a/third-party/benchmark/CMakeLists.txt +++ b/third-party/benchmark/CMakeLists.txt @@ -1,19 +1,7 @@ -cmake_minimum_required (VERSION 3.5.1) - -foreach(p - CMP0048 # OK to clear PROJECT_VERSION on project() - CMP0054 # CMake 3.1 - CMP0056 # export EXE_LINKER_FLAGS to try_run - CMP0057 # Support no if() IN_LIST operator - CMP0063 # Honor visibility properties for all targets - CMP0077 # Allow option() overrides in importing projects - ) - if(POLICY ${p}) - cmake_policy(SET ${p} NEW) - endif() -endforeach() +# Require CMake 3.10. If available, use the policies up to CMake 3.22. +cmake_minimum_required (VERSION 3.10...3.22) -project (benchmark VERSION 1.6.0 LANGUAGES CXX) +project (benchmark VERSION 1.8.3 LANGUAGES CXX) option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." ON) option(BENCHMARK_ENABLE_EXCEPTIONS "Enable the use of exceptions in the benchmark library." ON) @@ -26,11 +14,14 @@ if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "PGI") # PGC++ maybe reporting false positives. set(BENCHMARK_ENABLE_WERROR OFF) endif() +if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "NVHPC") + set(BENCHMARK_ENABLE_WERROR OFF) +endif() if(BENCHMARK_FORCE_WERROR) set(BENCHMARK_ENABLE_WERROR ON) endif(BENCHMARK_FORCE_WERROR) -if(NOT MSVC) +if(NOT (MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC")) option(BENCHMARK_BUILD_32_BITS "Build a 32 bit version of the library." OFF) else() set(BENCHMARK_BUILD_32_BITS OFF CACHE BOOL "Build a 32 bit version of the library - unsupported when using MSVC)" FORCE) @@ -50,8 +41,11 @@ option(BENCHMARK_USE_BUNDLED_GTEST "Use bundled GoogleTest. If disabled, the fin option(BENCHMARK_ENABLE_LIBPFM "Enable performance counters provided by libpfm" OFF) -set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) -if(MSVC) +# Export only public symbols +set(CMAKE_CXX_VISIBILITY_PRESET hidden) +set(CMAKE_VISIBILITY_INLINES_HIDDEN ON) + +if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") # As of CMake 3.18, CMAKE_SYSTEM_PROCESSOR is not set properly for MSVC and # cross-compilation (e.g. Host=x86_64, target=aarch64) requires using the # undocumented, but working variable. @@ -72,7 +66,7 @@ function(should_enable_assembly_tests) return() endif() endif() - if (MSVC) + if (MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC") return() elseif(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64") return() @@ -111,22 +105,32 @@ get_git_version(GIT_VERSION) # If no git version can be determined, use the version # from the project() command if ("${GIT_VERSION}" STREQUAL "0.0.0") - set(VERSION "${benchmark_VERSION}") + set(VERSION "v${benchmark_VERSION}") else() set(VERSION "${GIT_VERSION}") endif() + +# Normalize version: drop "v" prefix, replace first "-" with ".", +# drop everything after second "-" (including said "-"). +string(STRIP ${VERSION} VERSION) +if(VERSION MATCHES v[^-]*-) + string(REGEX REPLACE "v([^-]*)-([0-9]+)-.*" "\\1.\\2" NORMALIZED_VERSION ${VERSION}) +else() + string(REGEX REPLACE "v(.*)" "\\1" NORMALIZED_VERSION ${VERSION}) +endif() + # Tell the user what versions we are using -message(STATUS "Version: ${VERSION}") +message(STATUS "Google Benchmark version: ${VERSION}, normalized to ${NORMALIZED_VERSION}") # The version of the libraries -set(GENERIC_LIB_VERSION ${VERSION}) -string(SUBSTRING ${VERSION} 0 1 GENERIC_LIB_SOVERSION) +set(GENERIC_LIB_VERSION ${NORMALIZED_VERSION}) +string(SUBSTRING ${NORMALIZED_VERSION} 0 1 GENERIC_LIB_SOVERSION) # Import our CMake modules -include(CheckCXXCompilerFlag) include(AddCXXCompilerFlag) -include(CXXFeatureCheck) +include(CheckCXXCompilerFlag) include(CheckLibraryExists) +include(CXXFeatureCheck) check_library_exists(rt shm_open "" HAVE_LIB_RT) @@ -134,6 +138,16 @@ if (BENCHMARK_BUILD_32_BITS) add_required_cxx_compiler_flag(-m32) endif() +if (MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC") + set(BENCHMARK_CXX_STANDARD 14) +else() + set(BENCHMARK_CXX_STANDARD 11) +endif() + +set(CMAKE_CXX_STANDARD ${BENCHMARK_CXX_STANDARD}) +set(CMAKE_CXX_STANDARD_REQUIRED YES) +set(CMAKE_CXX_EXTENSIONS OFF) + if (MSVC) # Turn compiler warnings up to 11 string(REGEX REPLACE "[-/]W[1-4]" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") @@ -166,21 +180,18 @@ if (MSVC) set(CMAKE_EXE_LINKER_FLAGS_MINSIZEREL "${CMAKE_EXE_LINKER_FLAGS_MINSIZEREL} /LTCG") endif() else() - # Try and enable C++11. Don't use C++14 because it doesn't work in some - # configurations. - add_cxx_compiler_flag(-std=c++11) - if (NOT HAVE_CXX_FLAG_STD_CXX11) - add_cxx_compiler_flag(-std=c++0x) - endif() - + # Turn on Large-file Support + add_definitions(-D_FILE_OFFSET_BITS=64) + add_definitions(-D_LARGEFILE64_SOURCE) + add_definitions(-D_LARGEFILE_SOURCE) # Turn compiler warnings up to 11 add_cxx_compiler_flag(-Wall) add_cxx_compiler_flag(-Wextra) add_cxx_compiler_flag(-Wshadow) + add_cxx_compiler_flag(-Wfloat-equal) + add_cxx_compiler_flag(-Wold-style-cast) if(BENCHMARK_ENABLE_WERROR) - add_cxx_compiler_flag(-Werror RELEASE) - add_cxx_compiler_flag(-Werror RELWITHDEBINFO) - add_cxx_compiler_flag(-Werror MINSIZEREL) + add_cxx_compiler_flag(-Werror) endif() if (NOT BENCHMARK_ENABLE_TESTING) # Disable warning when compiling tests as gtest does not use 'override'. @@ -193,24 +204,23 @@ else() # Disable warnings regarding deprecated parts of the library while building # and testing those parts of the library. add_cxx_compiler_flag(-Wno-deprecated-declarations) - if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel" OR CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") # Intel silently ignores '-Wno-deprecated-declarations', # warning no. 1786 must be explicitly disabled. # See #631 for rationale. add_cxx_compiler_flag(-wd1786) + add_cxx_compiler_flag(-fno-finite-math-only) endif() # Disable deprecation warnings for release builds (when -Werror is enabled). if(BENCHMARK_ENABLE_WERROR) - add_cxx_compiler_flag(-Wno-deprecated RELEASE) - add_cxx_compiler_flag(-Wno-deprecated RELWITHDEBINFO) - add_cxx_compiler_flag(-Wno-deprecated MINSIZEREL) + add_cxx_compiler_flag(-Wno-deprecated) endif() if (NOT BENCHMARK_ENABLE_EXCEPTIONS) add_cxx_compiler_flag(-fno-exceptions) endif() if (HAVE_CXX_FLAG_FSTRICT_ALIASING) - if (NOT CMAKE_CXX_COMPILER_ID STREQUAL "Intel") #ICC17u2: Many false positives for Wstrict-aliasing + if (NOT CMAKE_CXX_COMPILER_ID STREQUAL "Intel" AND NOT CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") #ICC17u2: Many false positives for Wstrict-aliasing add_cxx_compiler_flag(-Wstrict-aliasing) endif() endif() @@ -219,12 +229,12 @@ else() add_cxx_compiler_flag(-wd654) add_cxx_compiler_flag(-Wthread-safety) if (HAVE_CXX_FLAG_WTHREAD_SAFETY) - cxx_feature_check(THREAD_SAFETY_ATTRIBUTES) + cxx_feature_check(THREAD_SAFETY_ATTRIBUTES "-DINCLUDE_DIRECTORIES=${PROJECT_SOURCE_DIR}/include") endif() # On most UNIX like platforms g++ and clang++ define _GNU_SOURCE as a # predefined macro, which turns on all of the wonderful libc extensions. - # However g++ doesn't do this in Cygwin so we have to define it ourselfs + # However g++ doesn't do this in Cygwin so we have to define it ourselves # since we depend on GNU/POSIX/BSD extensions. if (CYGWIN) add_definitions(-D_GNU_SOURCE=1) @@ -275,7 +285,8 @@ if (BENCHMARK_USE_LIBCXX) if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") add_cxx_compiler_flag(-stdlib=libc++) elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" OR - "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") + "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel" OR + "${CMAKE_CXX_COMPILER_ID}" STREQUAL "IntelLLVM") add_cxx_compiler_flag(-nostdinc++) message(WARNING "libc++ header path must be manually specified using CMAKE_CXX_FLAGS") # Adding -nodefaultlibs directly to CMAKE__LINKER_FLAGS will break @@ -312,9 +323,10 @@ cxx_feature_check(STEADY_CLOCK) # Ensure we have pthreads set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) +cxx_feature_check(PTHREAD_AFFINITY) if (BENCHMARK_ENABLE_LIBPFM) - find_package(PFM) + find_package(PFM REQUIRED) endif() # Set up directories diff --git a/third-party/benchmark/CONTRIBUTORS b/third-party/benchmark/CONTRIBUTORS index 651fbeafe66a..9ca2caa3ee78 100644 --- a/third-party/benchmark/CONTRIBUTORS +++ b/third-party/benchmark/CONTRIBUTORS @@ -27,7 +27,9 @@ Albert Pretorius Alex Steele Andriy Berestovskyy Arne Beer +Bátor Tallér Billy Robert O'Neal III +Cezary Skrzyński Chris Kennelly Christian Wassermann Christopher Seymour @@ -44,25 +46,32 @@ Eric Backus Eric Fiselier Eugene Zhuk Evgeny Safronov +Fabien Pichot Fanbo Meng Federico Ficarelli Felix Homann Geoffrey Martin-Noble +Gergely Meszaros Gergő Szitár Hannes Hauswedell +Henrique Bucher Ismael Jimenez Martinez +Iakov Sergeev Jern-Kuan Leong JianXiong Zhou Joao Paulo Magalhaes John Millikin Jordan Williams Jussi Knuuttila -Kai Wolf Kaito Udagawa +Kai Wolf Kishan Kumar Lei Xu +Marcel Jacobse Matt Clarkson Maxim Vafin +Mike Apodaca +Min-Yih Hsu Nick Hutchinson Norman Heino Oleksandr Sochka @@ -71,6 +80,8 @@ Pascal Leroy Paul Redmond Pierre Phaneuf Radoslav Yovchev +Raghu Raja +Rainer Orth Raul Marin Ray Glover Robert Guo @@ -84,4 +95,3 @@ Tom Madams Yixuan Qiu Yusuke Suzuki Zbigniew Skowron -Min-Yih Hsu diff --git a/third-party/benchmark/README.md b/third-party/benchmark/README.md index 7b81d960fc1d..a5e5d392d826 100644 --- a/third-party/benchmark/README.md +++ b/third-party/benchmark/README.md @@ -4,10 +4,9 @@ [![bazel](https://github.com/google/benchmark/actions/workflows/bazel.yml/badge.svg)](https://github.com/google/benchmark/actions/workflows/bazel.yml) [![pylint](https://github.com/google/benchmark/workflows/pylint/badge.svg)](https://github.com/google/benchmark/actions?query=workflow%3Apylint) [![test-bindings](https://github.com/google/benchmark/workflows/test-bindings/badge.svg)](https://github.com/google/benchmark/actions?query=workflow%3Atest-bindings) - -[![Build Status](https://travis-ci.org/google/benchmark.svg?branch=master)](https://travis-ci.org/google/benchmark) [![Coverage Status](https://coveralls.io/repos/google/benchmark/badge.svg)](https://coveralls.io/r/google/benchmark) +[![Discord](https://discordapp.com/api/guilds/1125694995928719494/widget.png?style=shield)](https://discord.gg/cz7UX7wKC2) A library to benchmark code snippets, similar to unit tests. Example: @@ -33,7 +32,7 @@ To get started, see [Requirements](#requirements) and [Installation](#installation). See [Usage](#usage) for a full example and the [User Guide](docs/user_guide.md) for a more comprehensive feature overview. -It may also help to read the [Google Test documentation](https://github.com/google/googletest/blob/master/docs/primer.md) +It may also help to read the [Google Test documentation](https://github.com/google/googletest/blob/main/docs/primer.md) as some of the structural aspects of the APIs are similar. ## Resources @@ -47,6 +46,8 @@ IRC channels: [Assembly Testing Documentation](docs/AssemblyTests.md) +[Building and installing Python bindings](docs/python_bindings.md) + ## Requirements The library can be used with C++03. However, it requires C++11 to build, @@ -137,6 +138,12 @@ cache variables, if autodetection fails. If you are using clang, you may need to set `LLVMAR_EXECUTABLE`, `LLVMNM_EXECUTABLE` and `LLVMRANLIB_EXECUTABLE` cmake cache variables. +To enable sanitizer checks (eg., `asan` and `tsan`), add: +``` + -DCMAKE_C_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=address -fsanitize=thread -fno-sanitize-recover=all" + -DCMAKE_CXX_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=address -fsanitize=thread -fno-sanitize-recover=all " +``` + ### Stable and Experimental Library Versions The main branch contains the latest stable version of the benchmarking library; diff --git a/third-party/benchmark/WORKSPACE b/third-party/benchmark/WORKSPACE index 949eb98bc5d9..256207022597 100644 --- a/third-party/benchmark/WORKSPACE +++ b/third-party/benchmark/WORKSPACE @@ -1,44 +1,30 @@ workspace(name = "com_github_google_benchmark") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") - -http_archive( - name = "com_google_absl", - sha256 = "f41868f7a938605c92936230081175d1eae87f6ea2c248f41077c8f88316f111", - strip_prefix = "abseil-cpp-20200225.2", - urls = ["https://github.com/abseil/abseil-cpp/archive/20200225.2.tar.gz"], -) +load("//:bazel/benchmark_deps.bzl", "benchmark_deps") -git_repository( - name = "com_google_googletest", - remote = "https://github.com/google/googletest.git", - tag = "release-1.11.0", -) +benchmark_deps() -http_archive( - name = "pybind11", - build_file = "@//bindings/python:pybind11.BUILD", - sha256 = "1eed57bc6863190e35637290f97a20c81cfe4d9090ac0a24f3bbf08f265eb71d", - strip_prefix = "pybind11-2.4.3", - urls = ["https://github.com/pybind/pybind11/archive/v2.4.3.tar.gz"], -) +load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies") -new_local_repository( - name = "python_headers", - build_file = "@//bindings/python:python_headers.BUILD", - path = "/usr/include/python3.6", # May be overwritten by setup.py. -) +rules_foreign_cc_dependencies() -http_archive( - name = "rules_python", - url = "https://github.com/bazelbuild/rules_python/releases/download/0.1.0/rules_python-0.1.0.tar.gz", - sha256 = "b6d46438523a3ec0f3cead544190ee13223a52f6a6765a29eae7b7cc24cc83a0", +load("@rules_python//python:repositories.bzl", "py_repositories") + +py_repositories() + +load("@rules_python//python:pip.bzl", "pip_parse") + +pip_parse( + name = "tools_pip_deps", + requirements_lock = "//tools:requirements.txt", ) -load("@rules_python//python:pip.bzl", pip3_install="pip_install") +load("@tools_pip_deps//:requirements.bzl", "install_deps") + +install_deps() -pip3_install( - name = "py_deps", - requirements = "//:requirements.txt", +new_local_repository( + name = "python_headers", + build_file = "@//bindings/python:python_headers.BUILD", + path = "", # May be overwritten by setup.py. ) diff --git a/third-party/benchmark/bindings/python/build_defs.bzl b/third-party/benchmark/bindings/python/build_defs.bzl index 45907aaa5e2d..b0c1b0f5807e 100644 --- a/third-party/benchmark/bindings/python/build_defs.bzl +++ b/third-party/benchmark/bindings/python/build_defs.bzl @@ -1,3 +1,7 @@ +""" +This file contains some build definitions for C++ extensions used in the Google Benchmark Python bindings. +""" + _SHARED_LIB_SUFFIX = { "//conditions:default": ".so", "//:windows": ".dll", @@ -8,8 +12,8 @@ def py_extension(name, srcs, hdrs = [], copts = [], features = [], deps = []): shared_lib_name = name + shared_lib_suffix native.cc_binary( name = shared_lib_name, - linkshared = 1, - linkstatic = 1, + linkshared = True, + linkstatic = True, srcs = srcs + hdrs, copts = copts, features = features, diff --git a/third-party/benchmark/bindings/python/google_benchmark/__init__.py b/third-party/benchmark/bindings/python/google_benchmark/__init__.py index 1055bf241856..e14769f451fe 100644 --- a/third-party/benchmark/bindings/python/google_benchmark/__init__.py +++ b/third-party/benchmark/bindings/python/google_benchmark/__init__.py @@ -26,47 +26,29 @@ Example usage: if __name__ == '__main__': benchmark.main() """ +import atexit from absl import app + from google_benchmark import _benchmark from google_benchmark._benchmark import ( - Counter, - kNanosecond, - kMicrosecond, - kMillisecond, - kSecond, - oNone, - o1, - oN, - oNSquared, - oNCubed, - oLogN, - oNLogN, - oAuto, - oLambda, + Counter as Counter, + State as State, + kMicrosecond as kMicrosecond, + kMillisecond as kMillisecond, + kNanosecond as kNanosecond, + kSecond as kSecond, + o1 as o1, + oAuto as oAuto, + oLambda as oLambda, + oLogN as oLogN, + oN as oN, + oNCubed as oNCubed, + oNLogN as oNLogN, + oNone as oNone, + oNSquared as oNSquared, ) - - -__all__ = [ - "register", - "main", - "Counter", - "kNanosecond", - "kMicrosecond", - "kMillisecond", - "kSecond", - "oNone", - "o1", - "oN", - "oNSquared", - "oNCubed", - "oLogN", - "oNLogN", - "oAuto", - "oLambda", -] - -__version__ = "0.2.0" +from google_benchmark.version import __version__ as __version__ class __OptionMaker: @@ -94,14 +76,13 @@ class __OptionMaker: # The function that get returned on @option.range(start=0, limit=1<<5). def __builder_method(*args, **kwargs): - # The decorator that get called, either with the benchmared function # or the previous Options def __decorator(func_or_options): options = self.make(func_or_options) options.builder_calls.append((builder_name, args, kwargs)) # The decorator returns Options so it is not technically a decorator - # and needs a final call to @regiser + # and needs a final call to @register return options return __decorator @@ -156,3 +137,4 @@ def main(argv=None): # Methods for use with custom main function. initialize = _benchmark.Initialize run_benchmarks = _benchmark.RunSpecifiedBenchmarks +atexit.register(_benchmark.ClearRegisteredBenchmarks) diff --git a/third-party/benchmark/bindings/python/google_benchmark/benchmark.cc b/third-party/benchmark/bindings/python/google_benchmark/benchmark.cc index 02b6ed7ed590..f44476901cae 100644 --- a/third-party/benchmark/bindings/python/google_benchmark/benchmark.cc +++ b/third-party/benchmark/bindings/python/google_benchmark/benchmark.cc @@ -1,20 +1,17 @@ // Benchmark for Python. -#include -#include -#include - -#include "pybind11/operators.h" -#include "pybind11/pybind11.h" -#include "pybind11/stl.h" -#include "pybind11/stl_bind.h" - #include "benchmark/benchmark.h" -PYBIND11_MAKE_OPAQUE(benchmark::UserCounters); +#include "nanobind/nanobind.h" +#include "nanobind/operators.h" +#include "nanobind/stl/bind_map.h" +#include "nanobind/stl/string.h" +#include "nanobind/stl/vector.h" + +NB_MAKE_OPAQUE(benchmark::UserCounters); namespace { -namespace py = ::pybind11; +namespace nb = nanobind; std::vector Initialize(const std::vector& argv) { // The `argv` pointers here become invalid when this function returns, but @@ -37,15 +34,16 @@ std::vector Initialize(const std::vector& argv) { return remaining_argv; } -benchmark::internal::Benchmark* RegisterBenchmark(const char* name, - py::function f) { +benchmark::internal::Benchmark* RegisterBenchmark(const std::string& name, + nb::callable f) { return benchmark::RegisterBenchmark( name, [f](benchmark::State& state) { f(&state); }); } -PYBIND11_MODULE(_benchmark, m) { +NB_MODULE(_benchmark, m) { + using benchmark::TimeUnit; - py::enum_(m, "TimeUnit") + nb::enum_(m, "TimeUnit") .value("kNanosecond", TimeUnit::kNanosecond) .value("kMicrosecond", TimeUnit::kMicrosecond) .value("kMillisecond", TimeUnit::kMillisecond) @@ -53,72 +51,74 @@ PYBIND11_MODULE(_benchmark, m) { .export_values(); using benchmark::BigO; - py::enum_(m, "BigO") + nb::enum_(m, "BigO") .value("oNone", BigO::oNone) .value("o1", BigO::o1) .value("oN", BigO::oN) .value("oNSquared", BigO::oNSquared) .value("oNCubed", BigO::oNCubed) .value("oLogN", BigO::oLogN) - .value("oNLogN", BigO::oLogN) + .value("oNLogN", BigO::oNLogN) .value("oAuto", BigO::oAuto) .value("oLambda", BigO::oLambda) .export_values(); using benchmark::internal::Benchmark; - py::class_(m, "Benchmark") - // For methods returning a pointer tor the current object, reference - // return policy is used to ask pybind not to take ownership oof the + nb::class_(m, "Benchmark") + // For methods returning a pointer to the current object, reference + // return policy is used to ask nanobind not to take ownership of the // returned object and avoid calling delete on it. // https://pybind11.readthedocs.io/en/stable/advanced/functions.html#return-value-policies // // For methods taking a const std::vector<...>&, a copy is created // because a it is bound to a Python list. // https://pybind11.readthedocs.io/en/stable/advanced/cast/stl.html - .def("unit", &Benchmark::Unit, py::return_value_policy::reference) - .def("arg", &Benchmark::Arg, py::return_value_policy::reference) - .def("args", &Benchmark::Args, py::return_value_policy::reference) - .def("range", &Benchmark::Range, py::return_value_policy::reference, - py::arg("start"), py::arg("limit")) + .def("unit", &Benchmark::Unit, nb::rv_policy::reference) + .def("arg", &Benchmark::Arg, nb::rv_policy::reference) + .def("args", &Benchmark::Args, nb::rv_policy::reference) + .def("range", &Benchmark::Range, nb::rv_policy::reference, + nb::arg("start"), nb::arg("limit")) .def("dense_range", &Benchmark::DenseRange, - py::return_value_policy::reference, py::arg("start"), - py::arg("limit"), py::arg("step") = 1) - .def("ranges", &Benchmark::Ranges, py::return_value_policy::reference) + nb::rv_policy::reference, nb::arg("start"), + nb::arg("limit"), nb::arg("step") = 1) + .def("ranges", &Benchmark::Ranges, nb::rv_policy::reference) .def("args_product", &Benchmark::ArgsProduct, - py::return_value_policy::reference) - .def("arg_name", &Benchmark::ArgName, py::return_value_policy::reference) + nb::rv_policy::reference) + .def("arg_name", &Benchmark::ArgName, nb::rv_policy::reference) .def("arg_names", &Benchmark::ArgNames, - py::return_value_policy::reference) + nb::rv_policy::reference) .def("range_pair", &Benchmark::RangePair, - py::return_value_policy::reference, py::arg("lo1"), py::arg("hi1"), - py::arg("lo2"), py::arg("hi2")) + nb::rv_policy::reference, nb::arg("lo1"), nb::arg("hi1"), + nb::arg("lo2"), nb::arg("hi2")) .def("range_multiplier", &Benchmark::RangeMultiplier, - py::return_value_policy::reference) - .def("min_time", &Benchmark::MinTime, py::return_value_policy::reference) + nb::rv_policy::reference) + .def("min_time", &Benchmark::MinTime, nb::rv_policy::reference) + .def("min_warmup_time", &Benchmark::MinWarmUpTime, + nb::rv_policy::reference) .def("iterations", &Benchmark::Iterations, - py::return_value_policy::reference) + nb::rv_policy::reference) .def("repetitions", &Benchmark::Repetitions, - py::return_value_policy::reference) + nb::rv_policy::reference) .def("report_aggregates_only", &Benchmark::ReportAggregatesOnly, - py::return_value_policy::reference, py::arg("value") = true) + nb::rv_policy::reference, nb::arg("value") = true) .def("display_aggregates_only", &Benchmark::DisplayAggregatesOnly, - py::return_value_policy::reference, py::arg("value") = true) + nb::rv_policy::reference, nb::arg("value") = true) .def("measure_process_cpu_time", &Benchmark::MeasureProcessCPUTime, - py::return_value_policy::reference) + nb::rv_policy::reference) .def("use_real_time", &Benchmark::UseRealTime, - py::return_value_policy::reference) + nb::rv_policy::reference) .def("use_manual_time", &Benchmark::UseManualTime, - py::return_value_policy::reference) + nb::rv_policy::reference) .def( "complexity", (Benchmark * (Benchmark::*)(benchmark::BigO)) & Benchmark::Complexity, - py::return_value_policy::reference, - py::arg("complexity") = benchmark::oAuto); + nb::rv_policy::reference, + nb::arg("complexity") = benchmark::oAuto); using benchmark::Counter; - py::class_ py_counter(m, "Counter"); + nb::class_ py_counter(m, "Counter"); - py::enum_(py_counter, "Flags") + nb::enum_(py_counter, "Flags") .value("kDefaults", Counter::Flags::kDefaults) .value("kIsRate", Counter::Flags::kIsRate) .value("kAvgThreads", Counter::Flags::kAvgThreads) @@ -130,52 +130,55 @@ PYBIND11_MODULE(_benchmark, m) { .value("kAvgIterationsRate", Counter::Flags::kAvgIterationsRate) .value("kInvert", Counter::Flags::kInvert) .export_values() - .def(py::self | py::self); + .def(nb::self | nb::self); - py::enum_(py_counter, "OneK") + nb::enum_(py_counter, "OneK") .value("kIs1000", Counter::OneK::kIs1000) .value("kIs1024", Counter::OneK::kIs1024) .export_values(); py_counter - .def(py::init(), - py::arg("value") = 0., py::arg("flags") = Counter::kDefaults, - py::arg("k") = Counter::kIs1000) - .def(py::init([](double value) { return Counter(value); })) - .def_readwrite("value", &Counter::value) - .def_readwrite("flags", &Counter::flags) - .def_readwrite("oneK", &Counter::oneK); - py::implicitly_convertible(); - py::implicitly_convertible(); - - py::bind_map(m, "UserCounters"); + .def(nb::init(), + nb::arg("value") = 0., nb::arg("flags") = Counter::kDefaults, + nb::arg("k") = Counter::kIs1000) + .def("__init__", ([](Counter *c, double value) { new (c) Counter(value); })) + .def_rw("value", &Counter::value) + .def_rw("flags", &Counter::flags) + .def_rw("oneK", &Counter::oneK) + .def(nb::init_implicit()); + + nb::implicitly_convertible(); + + nb::bind_map(m, "UserCounters"); using benchmark::State; - py::class_(m, "State") + nb::class_(m, "State") .def("__bool__", &State::KeepRunning) - .def_property_readonly("keep_running", &State::KeepRunning) + .def_prop_ro("keep_running", &State::KeepRunning) .def("pause_timing", &State::PauseTiming) .def("resume_timing", &State::ResumeTiming) .def("skip_with_error", &State::SkipWithError) - .def_property_readonly("error_occurred", &State::error_occurred) + .def_prop_ro("error_occurred", &State::error_occurred) .def("set_iteration_time", &State::SetIterationTime) - .def_property("bytes_processed", &State::bytes_processed, + .def_prop_rw("bytes_processed", &State::bytes_processed, &State::SetBytesProcessed) - .def_property("complexity_n", &State::complexity_length_n, + .def_prop_rw("complexity_n", &State::complexity_length_n, &State::SetComplexityN) - .def_property("items_processed", &State::items_processed, - &State::SetItemsProcessed) - .def("set_label", (void(State::*)(const char*)) & State::SetLabel) - .def("range", &State::range, py::arg("pos") = 0) - .def_property_readonly("iterations", &State::iterations) - .def_readwrite("counters", &State::counters) - .def_property_readonly("thread_index", &State::thread_index) - .def_property_readonly("threads", &State::threads); + .def_prop_rw("items_processed", &State::items_processed, + &State::SetItemsProcessed) + .def("set_label", &State::SetLabel) + .def("range", &State::range, nb::arg("pos") = 0) + .def_prop_ro("iterations", &State::iterations) + .def_prop_ro("name", &State::name) + .def_rw("counters", &State::counters) + .def_prop_ro("thread_index", &State::thread_index) + .def_prop_ro("threads", &State::threads); m.def("Initialize", Initialize); m.def("RegisterBenchmark", RegisterBenchmark, - py::return_value_policy::reference); + nb::rv_policy::reference); m.def("RunSpecifiedBenchmarks", []() { benchmark::RunSpecifiedBenchmarks(); }); + m.def("ClearRegisteredBenchmarks", benchmark::ClearRegisteredBenchmarks); }; } // namespace diff --git a/third-party/benchmark/bindings/python/google_benchmark/example.py b/third-party/benchmark/bindings/python/google_benchmark/example.py index fb0234b8fd7e..b5b2f88ff306 100644 --- a/third-party/benchmark/bindings/python/google_benchmark/example.py +++ b/third-party/benchmark/bindings/python/google_benchmark/example.py @@ -73,7 +73,7 @@ def manual_timing(state): @benchmark.register def custom_counters(state): - """Collect cutom metric using benchmark.Counter.""" + """Collect custom metric using benchmark.Counter.""" num_foo = 0.0 while state: # Benchmark some code here @@ -86,7 +86,9 @@ def custom_counters(state): # Set a counter as a rate. state.counters["foo_rate"] = Counter(num_foo, Counter.kIsRate) # Set a counter as an inverse of rate. - state.counters["foo_inv_rate"] = Counter(num_foo, Counter.kIsRate | Counter.kInvert) + state.counters["foo_inv_rate"] = Counter( + num_foo, Counter.kIsRate | Counter.kInvert + ) # Set a counter as a thread-average quantity. state.counters["foo_avg"] = Counter(num_foo, Counter.kAvgThreads) # There's also a combined flag: diff --git a/third-party/benchmark/bindings/python/pybind11.BUILD b/third-party/benchmark/bindings/python/pybind11.BUILD deleted file mode 100644 index bc833500383a..000000000000 --- a/third-party/benchmark/bindings/python/pybind11.BUILD +++ /dev/null @@ -1,20 +0,0 @@ -cc_library( - name = "pybind11", - hdrs = glob( - include = [ - "include/pybind11/*.h", - "include/pybind11/detail/*.h", - ], - exclude = [ - "include/pybind11/common.h", - "include/pybind11/eigen.h", - ], - ), - copts = [ - "-fexceptions", - "-Wno-undefined-inline", - "-Wno-pragma-once-outside-header", - ], - includes = ["include"], - visibility = ["//visibility:public"], -) diff --git a/third-party/benchmark/bindings/python/python_headers.BUILD b/third-party/benchmark/bindings/python/python_headers.BUILD index 9c34cf6ca4bd..8f139f8621e0 100644 --- a/third-party/benchmark/bindings/python/python_headers.BUILD +++ b/third-party/benchmark/bindings/python/python_headers.BUILD @@ -1,3 +1,7 @@ +licenses(["notice"]) + +package(default_visibility = ["//visibility:public"]) + cc_library( name = "python_headers", hdrs = glob(["**/*.h"]), diff --git a/third-party/benchmark/bindings/python/requirements.txt b/third-party/benchmark/bindings/python/requirements.txt deleted file mode 100644 index f5bbe7eca5ce..000000000000 --- a/third-party/benchmark/bindings/python/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -absl-py>=0.7.1 - diff --git a/third-party/benchmark/cmake/CXXFeatureCheck.cmake b/third-party/benchmark/cmake/CXXFeatureCheck.cmake index 62e6741fe3de..e51482659b0f 100644 --- a/third-party/benchmark/cmake/CXXFeatureCheck.cmake +++ b/third-party/benchmark/cmake/CXXFeatureCheck.cmake @@ -17,6 +17,8 @@ if(__cxx_feature_check) endif() set(__cxx_feature_check INCLUDED) +option(CXXFEATURECHECK_DEBUG OFF) + function(cxx_feature_check FILE) string(TOLOWER ${FILE} FILE) string(TOUPPER ${FILE} VAR) @@ -27,18 +29,22 @@ function(cxx_feature_check FILE) return() endif() + set(FEATURE_CHECK_CMAKE_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS}) if (ARGC GREATER 1) message(STATUS "Enabling additional flags: ${ARGV1}") - list(APPEND BENCHMARK_CXX_LINKER_FLAGS ${ARGV1}) + list(APPEND FEATURE_CHECK_CMAKE_FLAGS ${ARGV1}) endif() if (NOT DEFINED COMPILE_${FEATURE}) - message(STATUS "Performing Test ${FEATURE}") if(CMAKE_CROSSCOMPILING) + message(STATUS "Cross-compiling to test ${FEATURE}") try_compile(COMPILE_${FEATURE} ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/${FILE}.cpp - CMAKE_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS} - LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES}) + CXX_STANDARD 11 + CXX_STANDARD_REQUIRED ON + CMAKE_FLAGS ${FEATURE_CHECK_CMAKE_FLAGS} + LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES} + OUTPUT_VARIABLE COMPILE_OUTPUT_VAR) if(COMPILE_${FEATURE}) message(WARNING "If you see build failures due to cross compilation, try setting HAVE_${VAR} to 0") @@ -47,11 +53,14 @@ function(cxx_feature_check FILE) set(RUN_${FEATURE} 1 CACHE INTERNAL "") endif() else() - message(STATUS "Performing Test ${FEATURE}") + message(STATUS "Compiling and running to test ${FEATURE}") try_run(RUN_${FEATURE} COMPILE_${FEATURE} ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/${FILE}.cpp - CMAKE_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS} - LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES}) + CXX_STANDARD 11 + CXX_STANDARD_REQUIRED ON + CMAKE_FLAGS ${FEATURE_CHECK_CMAKE_FLAGS} + LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES} + COMPILE_OUTPUT_VARIABLE COMPILE_OUTPUT_VAR) endif() endif() @@ -61,7 +70,11 @@ function(cxx_feature_check FILE) add_definitions(-DHAVE_${VAR}) else() if(NOT COMPILE_${FEATURE}) - message(STATUS "Performing Test ${FEATURE} -- failed to compile") + if(CXXFEATURECHECK_DEBUG) + message(STATUS "Performing Test ${FEATURE} -- failed to compile: ${COMPILE_OUTPUT_VAR}") + else() + message(STATUS "Performing Test ${FEATURE} -- failed to compile") + endif() else() message(STATUS "Performing Test ${FEATURE} -- compiled but failed to run") endif() diff --git a/third-party/benchmark/cmake/GetGitVersion.cmake b/third-party/benchmark/cmake/GetGitVersion.cmake index 04a1f9b70d68..b0210103b2cc 100644 --- a/third-party/benchmark/cmake/GetGitVersion.cmake +++ b/third-party/benchmark/cmake/GetGitVersion.cmake @@ -20,38 +20,16 @@ set(__get_git_version INCLUDED) function(get_git_version var) if(GIT_EXECUTABLE) - execute_process(COMMAND ${GIT_EXECUTABLE} describe --tags --match "v[0-9]*.[0-9]*.[0-9]*" --abbrev=8 + execute_process(COMMAND ${GIT_EXECUTABLE} describe --tags --match "v[0-9]*.[0-9]*.[0-9]*" --abbrev=8 --dirty WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} RESULT_VARIABLE status - OUTPUT_VARIABLE GIT_DESCRIBE_VERSION + OUTPUT_VARIABLE GIT_VERSION ERROR_QUIET) if(status) - set(GIT_DESCRIBE_VERSION "v0.0.0") + set(GIT_VERSION "v0.0.0") endif() - - string(STRIP ${GIT_DESCRIBE_VERSION} GIT_DESCRIBE_VERSION) - if(GIT_DESCRIBE_VERSION MATCHES v[^-]*-) - string(REGEX REPLACE "v([^-]*)-([0-9]+)-.*" "\\1.\\2" GIT_VERSION ${GIT_DESCRIBE_VERSION}) - else() - string(REGEX REPLACE "v(.*)" "\\1" GIT_VERSION ${GIT_DESCRIBE_VERSION}) - endif() - - # Work out if the repository is dirty - execute_process(COMMAND ${GIT_EXECUTABLE} update-index -q --refresh - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} - OUTPUT_QUIET - ERROR_QUIET) - execute_process(COMMAND ${GIT_EXECUTABLE} diff-index --name-only HEAD -- - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} - OUTPUT_VARIABLE GIT_DIFF_INDEX - ERROR_QUIET) - string(COMPARE NOTEQUAL "${GIT_DIFF_INDEX}" "" GIT_DIRTY) - if (${GIT_DIRTY}) - set(GIT_DESCRIBE_VERSION "${GIT_DESCRIBE_VERSION}-dirty") - endif() - message(STATUS "git version: ${GIT_DESCRIBE_VERSION} normalized to ${GIT_VERSION}") else() - set(GIT_VERSION "0.0.0") + set(GIT_VERSION "v0.0.0") endif() set(${var} ${GIT_VERSION} PARENT_SCOPE) diff --git a/third-party/benchmark/cmake/GoogleTest.cmake b/third-party/benchmark/cmake/GoogleTest.cmake index 66cb91008b73..e66e9d1a2076 100644 --- a/third-party/benchmark/cmake/GoogleTest.cmake +++ b/third-party/benchmark/cmake/GoogleTest.cmake @@ -29,15 +29,25 @@ set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) include(${GOOGLETEST_PREFIX}/googletest-paths.cmake) -# googletest doesn't seem to want to stay build warning clean so let's not hurt ourselves. -add_compile_options(-w) - # Add googletest directly to our build. This defines # the gtest and gtest_main targets. add_subdirectory(${GOOGLETEST_SOURCE_DIR} ${GOOGLETEST_BINARY_DIR} EXCLUDE_FROM_ALL) +# googletest doesn't seem to want to stay build warning clean so let's not hurt ourselves. +if (MSVC) + target_compile_options(gtest PRIVATE "/wd4244" "/wd4722") + target_compile_options(gtest_main PRIVATE "/wd4244" "/wd4722") + target_compile_options(gmock PRIVATE "/wd4244" "/wd4722") + target_compile_options(gmock_main PRIVATE "/wd4244" "/wd4722") +else() + target_compile_options(gtest PRIVATE "-w") + target_compile_options(gtest_main PRIVATE "-w") + target_compile_options(gmock PRIVATE "-w") + target_compile_options(gmock_main PRIVATE "-w") +endif() + if(NOT DEFINED GTEST_COMPILE_COMMANDS) set(GTEST_COMPILE_COMMANDS ON) endif() diff --git a/third-party/benchmark/cmake/Modules/FindPFM.cmake b/third-party/benchmark/cmake/Modules/FindPFM.cmake index cf807a1ee9e9..4c1ce938f9f7 100644 --- a/third-party/benchmark/cmake/Modules/FindPFM.cmake +++ b/third-party/benchmark/cmake/Modules/FindPFM.cmake @@ -1,26 +1,28 @@ # If successful, the following variables will be defined: -# HAVE_LIBPFM. -# Set BENCHMARK_ENABLE_LIBPFM to 0 to disable, regardless of libpfm presence. -include(CheckIncludeFile) -include(CheckLibraryExists) +# PFM_FOUND. +# PFM_LIBRARIES +# PFM_INCLUDE_DIRS +# the following target will be defined: +# PFM::libpfm + include(FeatureSummary) -enable_language(C) +include(FindPackageHandleStandardArgs) set_package_properties(PFM PROPERTIES URL http://perfmon2.sourceforge.net/ - DESCRIPTION "a helper library to develop monitoring tools" + DESCRIPTION "A helper library to develop monitoring tools" PURPOSE "Used to program specific performance monitoring events") -check_library_exists(libpfm.a pfm_initialize "" HAVE_LIBPFM_INITIALIZE) -if(HAVE_LIBPFM_INITIALIZE) - check_include_file(perfmon/perf_event.h HAVE_PERFMON_PERF_EVENT_H) - check_include_file(perfmon/pfmlib.h HAVE_PERFMON_PFMLIB_H) - check_include_file(perfmon/pfmlib_perf_event.h HAVE_PERFMON_PFMLIB_PERF_EVENT_H) - if(HAVE_PERFMON_PERF_EVENT_H AND HAVE_PERFMON_PFMLIB_H AND HAVE_PERFMON_PFMLIB_PERF_EVENT_H) - message("Using Perf Counters.") - set(HAVE_LIBPFM 1) - set(PFM_FOUND 1) - endif() -else() - message("Perf Counters support requested, but was unable to find libpfm.") +find_library(PFM_LIBRARY NAMES pfm) +find_path(PFM_INCLUDE_DIR NAMES perfmon/pfmlib.h) + +find_package_handle_standard_args(PFM REQUIRED_VARS PFM_LIBRARY PFM_INCLUDE_DIR) + +if (PFM_FOUND AND NOT TARGET PFM::libpfm) + add_library(PFM::libpfm UNKNOWN IMPORTED) + set_target_properties(PFM::libpfm PROPERTIES + IMPORTED_LOCATION "${PFM_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${PFM_INCLUDE_DIR}") endif() + +mark_as_advanced(PFM_LIBRARY PFM_INCLUDE_DIR) diff --git a/third-party/benchmark/cmake/benchmark.pc.in b/third-party/benchmark/cmake/benchmark.pc.in index 34beb012eef1..9dae881c79f9 100644 --- a/third-party/benchmark/cmake/benchmark.pc.in +++ b/third-party/benchmark/cmake/benchmark.pc.in @@ -1,7 +1,7 @@ prefix=@CMAKE_INSTALL_PREFIX@ exec_prefix=${prefix} -libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ -includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +libdir=@CMAKE_INSTALL_FULL_LIBDIR@ +includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ Name: @PROJECT_NAME@ Description: Google microbenchmark framework diff --git a/third-party/benchmark/docs/AssemblyTests.md b/third-party/benchmark/docs/AssemblyTests.md index 1fbdc269b53d..89df7ca520df 100644 --- a/third-party/benchmark/docs/AssemblyTests.md +++ b/third-party/benchmark/docs/AssemblyTests.md @@ -111,6 +111,7 @@ between compilers or compiler versions. A common example of this is matching stack frame addresses. In this case regular expressions can be used to match the differing bits of output. For example: + ```c++ int ExternInt; struct Point { int x, y, z; }; @@ -127,6 +128,7 @@ extern "C" void test_store_point() { // CHECK: ret } ``` + ## Current Requirements and Limitations diff --git a/third-party/benchmark/docs/_config.yml b/third-party/benchmark/docs/_config.yml index 2f7efbeab578..32f9f2e0ddc6 100644 --- a/third-party/benchmark/docs/_config.yml +++ b/third-party/benchmark/docs/_config.yml @@ -1 +1,3 @@ -theme: jekyll-theme-minimal \ No newline at end of file +theme: jekyll-theme-minimal +logo: /assets/images/icon_black.png +show_downloads: true diff --git a/third-party/benchmark/docs/dependencies.md b/third-party/benchmark/docs/dependencies.md index 7af52b95bd86..07760e10e370 100644 --- a/third-party/benchmark/docs/dependencies.md +++ b/third-party/benchmark/docs/dependencies.md @@ -1,19 +1,13 @@ # Build tool dependency policy -To ensure the broadest compatibility when building the benchmark library, but -still allow forward progress, we require any build tooling to be available for: +We follow the [Foundational C++ support policy](https://opensource.google/documentation/policies/cplusplus-support) for our build tools. In +particular the ["Build Systems" section](https://opensource.google/documentation/policies/cplusplus-support#build-systems). -* Debian stable _and_ -* The last two Ubuntu LTS releases +## CMake -Currently, this means using build tool versions that are available for Ubuntu -18.04 (Bionic Beaver), Ubuntu 20.04 (Focal Fossa), and Debian 11 (bullseye). +The current supported version is CMake 3.10 as of 2023-08-10. Most modern +distributions include newer versions, for example: -_Note, CI also runs ubuntu-16.04 and ubuntu-14.04 to ensure best effort support -for older versions._ - -## cmake -The current supported version is cmake 3.5.1 as of 2018-06-06. - -_Note, this version is also available for Ubuntu 14.04, an older Ubuntu LTS -release, as `cmake3`._ +* Ubuntu 20.04 provides CMake 3.16.3 +* Debian 11.4 provides CMake 3.18.4 +* Ubuntu 22.04 provides CMake 3.22.1 diff --git a/third-party/benchmark/docs/index.md b/third-party/benchmark/docs/index.md index eb82eff9eee0..9cada9688b1d 100644 --- a/third-party/benchmark/docs/index.md +++ b/third-party/benchmark/docs/index.md @@ -4,7 +4,9 @@ * [Dependencies](dependencies.md) * [Perf Counters](perf_counters.md) * [Platform Specific Build Instructions](platform_specific_build_instructions.md) +* [Python Bindings](python_bindings.md) * [Random Interleaving](random_interleaving.md) +* [Reducing Variance](reducing_variance.md) * [Releasing](releasing.md) * [Tools](tools.md) -* [User Guide](user_guide.md) \ No newline at end of file +* [User Guide](user_guide.md) diff --git a/third-party/benchmark/docs/perf_counters.md b/third-party/benchmark/docs/perf_counters.md index 74560e966971..f342092c9970 100644 --- a/third-party/benchmark/docs/perf_counters.md +++ b/third-party/benchmark/docs/perf_counters.md @@ -12,16 +12,17 @@ This feature is available if: * The benchmark is run on an architecture featuring a Performance Monitoring Unit (PMU), * The benchmark is compiled with support for collecting counters. Currently, - this requires [libpfm](http://perfmon2.sourceforge.net/) be available at build - time + this requires [libpfm](http://perfmon2.sourceforge.net/), which is built as a + dependency via Bazel. The feature does not require modifying benchmark code. Counter collection is handled at the boundaries where timer collection is also handled. To opt-in: - -* Install `libpfm4-dev`, e.g. `apt-get install libpfm4-dev`. -* Enable the cmake flag BENCHMARK_ENABLE_LIBPFM. +* If using a Bazel build, add `--define pfm=1` to your build flags +* If using CMake: + * Install `libpfm4-dev`, e.g. `apt-get install libpfm4-dev`. + * Enable the CMake flag `BENCHMARK_ENABLE_LIBPFM` in `CMakeLists.txt`. To use, pass a comma-separated list of counter names through the `--benchmark_perf_counters` flag. The names are decoded through libpfm - meaning, @@ -31,4 +32,4 @@ mapped by libpfm to platform-specifics - see libpfm The counter values are reported back through the [User Counters](../README.md#custom-counters) mechanism, meaning, they are available in all the formats (e.g. JSON) supported -by User Counters. \ No newline at end of file +by User Counters. diff --git a/third-party/benchmark/docs/releasing.md b/third-party/benchmark/docs/releasing.md index 334f93539381..09bf93764d00 100644 --- a/third-party/benchmark/docs/releasing.md +++ b/third-party/benchmark/docs/releasing.md @@ -1,30 +1,23 @@ # How to release * Make sure you're on main and synced to HEAD -* Ensure the project builds and tests run (sanity check only, obviously) +* Ensure the project builds and tests run * `parallel -j0 exec ::: test/*_test` can help ensure everything at least passes * Prepare release notes * `git log $(git describe --abbrev=0 --tags)..HEAD` gives you the list of commits between the last annotated tag and HEAD * Pick the most interesting. -* Create one last commit that updates the version saved in `CMakeLists.txt` and the - `__version__` variable in `bindings/python/google_benchmark/__init__.py`to the release - version you're creating. (This version will be used if benchmark is installed from the - archive you'll be creating in the next step.) +* Create one last commit that updates the version saved in `CMakeLists.txt` and `MODULE.bazel` + to the release version you're creating. (This version will be used if benchmark is installed + from the archive you'll be creating in the next step.) ``` -project (benchmark VERSION 1.6.0 LANGUAGES CXX) +project (benchmark VERSION 1.8.0 LANGUAGES CXX) ``` -```python -# bindings/python/google_benchmark/__init__.py - -# ... - -__version__ = "1.6.0" # <-- change this to the release version you are creating - -# ... +``` +module(name = "com_github_google_benchmark", version="1.8.0") ``` * Create a release through github's interface @@ -33,3 +26,6 @@ __version__ = "1.6.0" # <-- change this to the release version you are creating * `git pull --tags` * `git tag -a -f ` * `git push --force --tags origin` +* Confirm that the "Build and upload Python wheels" action runs to completion + * Run it manually if it hasn't run. + * IMPORTANT: When re-running manually, make sure to select the newly created `` as the workflow version in the "Run workflow" tab on the GitHub Actions page. diff --git a/third-party/benchmark/docs/tools.md b/third-party/benchmark/docs/tools.md index f2d0c497f3fc..411f41d405ff 100644 --- a/third-party/benchmark/docs/tools.md +++ b/third-party/benchmark/docs/tools.md @@ -186,6 +186,146 @@ Benchmark Time CPU Time Old This is a mix of the previous two modes, two (potentially different) benchmark binaries are run, and a different filter is applied to each one. As you can note, the values in `Time` and `CPU` columns are calculated as `(new - old) / |old|`. +### Note: Interpreting the output + +Performance measurements are an art, and performance comparisons are doubly so. +Results are often noisy and don't necessarily have large absolute differences to +them, so just by visual inspection, it is not at all apparent if two +measurements are actually showing a performance change or not. It is even more +confusing with multiple benchmark repetitions. + +Thankfully, what we can do, is use statistical tests on the results to determine +whether the performance has statistically-significantly changed. `compare.py` +uses [Mann–Whitney U +test](https://en.wikipedia.org/wiki/Mann%E2%80%93Whitney_U_test), with a null +hypothesis being that there's no difference in performance. + +**The below output is a summary of a benchmark comparison with statistics +provided for a multi-threaded process.** +``` +Benchmark Time CPU Time Old Time New CPU Old CPU New +----------------------------------------------------------------------------------------------------------------------------- +benchmark/threads:1/process_time/real_time_pvalue 0.0000 0.0000 U Test, Repetitions: 27 vs 27 +benchmark/threads:1/process_time/real_time_mean -0.1442 -0.1442 90 77 90 77 +benchmark/threads:1/process_time/real_time_median -0.1444 -0.1444 90 77 90 77 +benchmark/threads:1/process_time/real_time_stddev +0.3974 +0.3933 0 0 0 0 +benchmark/threads:1/process_time/real_time_cv +0.6329 +0.6280 0 0 0 0 +OVERALL_GEOMEAN -0.1442 -0.1442 0 0 0 0 +``` +-------------------------------------------- +Here's a breakdown of each row: + +**benchmark/threads:1/process_time/real_time_pvalue**: This shows the _p-value_ for +the statistical test comparing the performance of the process running with one +thread. A value of 0.0000 suggests a statistically significant difference in +performance. The comparison was conducted using the U Test (Mann-Whitney +U Test) with 27 repetitions for each case. + +**benchmark/threads:1/process_time/real_time_mean**: This shows the relative +difference in mean execution time between two different cases. The negative +value (-0.1442) implies that the new process is faster by about 14.42%. The old +time was 90 units, while the new time is 77 units. + +**benchmark/threads:1/process_time/real_time_median**: Similarly, this shows the +relative difference in the median execution time. Again, the new process is +faster by 14.44%. + +**benchmark/threads:1/process_time/real_time_stddev**: This is the relative +difference in the standard deviation of the execution time, which is a measure +of how much variation or dispersion there is from the mean. A positive value +(+0.3974) implies there is more variance in the execution time in the new +process. + +**benchmark/threads:1/process_time/real_time_cv**: CV stands for Coefficient of +Variation. It is the ratio of the standard deviation to the mean. It provides a +standardized measure of dispersion. An increase (+0.6329) indicates more +relative variability in the new process. + +**OVERALL_GEOMEAN**: Geomean stands for geometric mean, a type of average that is +less influenced by outliers. The negative value indicates a general improvement +in the new process. However, given the values are all zero for the old and new +times, this seems to be a mistake or placeholder in the output. + +----------------------------------------- + + + +Let's first try to see what the different columns represent in the above +`compare.py` benchmarking output: + + 1. **Benchmark:** The name of the function being benchmarked, along with the + size of the input (after the slash). + + 2. **Time:** The average time per operation, across all iterations. + + 3. **CPU:** The average CPU time per operation, across all iterations. + + 4. **Iterations:** The number of iterations the benchmark was run to get a + stable estimate. + + 5. **Time Old and Time New:** These represent the average time it takes for a + function to run in two different scenarios or versions. For example, you + might be comparing how fast a function runs before and after you make some + changes to it. + + 6. **CPU Old and CPU New:** These show the average amount of CPU time that the + function uses in two different scenarios or versions. This is similar to + Time Old and Time New, but focuses on CPU usage instead of overall time. + +In the comparison section, the relative differences in both time and CPU time +are displayed for each input size. + + +A statistically-significant difference is determined by a **p-value**, which is +a measure of the probability that the observed difference could have occurred +just by random chance. A smaller p-value indicates stronger evidence against the +null hypothesis. + +**Therefore:** + 1. If the p-value is less than the chosen significance level (alpha), we + reject the null hypothesis and conclude the benchmarks are significantly + different. + 2. If the p-value is greater than or equal to alpha, we fail to reject the + null hypothesis and treat the two benchmarks as similar. + + + +The result of said the statistical test is additionally communicated through color coding: +```diff ++ Green: +``` + The benchmarks are _**statistically different**_. This could mean the + performance has either **significantly improved** or **significantly + deteriorated**. You should look at the actual performance numbers to see which + is the case. +```diff +- Red: +``` + The benchmarks are _**statistically similar**_. This means the performance + **hasn't significantly changed**. + +In statistical terms, **'green'** means we reject the null hypothesis that +there's no difference in performance, and **'red'** means we fail to reject the +null hypothesis. This might seem counter-intuitive if you're expecting 'green' +to mean 'improved performance' and 'red' to mean 'worsened performance'. +```bash + But remember, in this context: + + 'Success' means 'successfully finding a difference'. + 'Failure' means 'failing to find a difference'. +``` + + +Also, please note that **even if** we determine that there **is** a +statistically-significant difference between the two measurements, it does not +_necessarily_ mean that the actual benchmarks that were measured **are** +different, or vice versa, even if we determine that there is **no** +statistically-significant difference between the two measurements, it does not +necessarily mean that the actual benchmarks that were measured **are not** +different. + + + ### U test If there is a sufficient repetition count of the benchmarks, the tool can do diff --git a/third-party/benchmark/docs/user_guide.md b/third-party/benchmark/docs/user_guide.md index 34bea6904240..d22a9069091f 100644 --- a/third-party/benchmark/docs/user_guide.md +++ b/third-party/benchmark/docs/user_guide.md @@ -28,6 +28,8 @@ [Templated Benchmarks](#templated-benchmarks) +[Templated Benchmarks that take arguments](#templated-benchmarks-with-arguments) + [Fixtures](#fixtures) [Custom Counters](#custom-counters) @@ -50,14 +52,19 @@ [Custom Statistics](#custom-statistics) +[Memory Usage](#memory-usage) + [Using RegisterBenchmark](#using-register-benchmark) [Exiting with an Error](#exiting-with-an-error) -[A Faster KeepRunning Loop](#a-faster-keep-running-loop) +[A Faster `KeepRunning` Loop](#a-faster-keep-running-loop) + +## Benchmarking Tips [Disabling CPU Frequency Scaling](#disabling-cpu-frequency-scaling) +[Reducing Variance in Benchmarks](reducing_variance.md) @@ -180,6 +187,12 @@ BM_memcpy/32 12 ns 12 ns 54687500 BM_memcpy/32k 1834 ns 1837 ns 357143 ``` +## Disabling Benchmarks + +It is possible to temporarily disable benchmarks by renaming the benchmark +function to have the prefix "DISABLED_". This will cause the benchmark to +be skipped at runtime. + ## Result comparison @@ -232,6 +245,19 @@ iterations is at least one, not more than 1e9, until CPU time is greater than the minimum time, or the wallclock time is 5x minimum time. The minimum time is set per benchmark by calling `MinTime` on the registered benchmark object. +Furthermore warming up a benchmark might be necessary in order to get +stable results because of e.g caching effects of the code under benchmark. +Warming up means running the benchmark a given amount of time, before +results are actually taken into account. The amount of time for which +the warmup should be run can be set per benchmark by calling +`MinWarmUpTime` on the registered benchmark object or for all benchmarks +using the `--benchmark_min_warmup_time` command-line option. Note that +`MinWarmUpTime` will overwrite the value of `--benchmark_min_warmup_time` +for the single benchmark. How many iterations the warmup run of each +benchmark takes is determined the same way as described in the paragraph +above. Per default the warmup phase is set to 0 seconds and is therefore +disabled. + Average timings are then reported over the iterations run. If multiple repetitions are requested using the `--benchmark_repetitions` command-line option, or at registration time, the benchmark function will be run several @@ -247,10 +273,12 @@ information about the machine on which the benchmarks are run. Global setup/teardown specific to each benchmark can be done by passing a callback to Setup/Teardown: -The setup/teardown callbacks will be invoked once for each benchmark. -If the benchmark is multi-threaded (will run in k threads), they will be invoked exactly once before -each run with k threads. -If the benchmark uses different size groups of threads, the above will be true for each size group. +The setup/teardown callbacks will be invoked once for each benchmark. If the +benchmark is multi-threaded (will run in k threads), they will be invoked +exactly once before each run with k threads. + +If the benchmark uses different size groups of threads, the above will be true +for each size group. Eg., @@ -293,7 +321,7 @@ static void BM_memcpy(benchmark::State& state) { delete[] src; delete[] dst; } -BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(1<<10)->Arg(8<<10); +BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(4<<10)->Arg(8<<10); ``` The preceding code is quite repetitive, and can be replaced with the following @@ -322,7 +350,8 @@ the performance of `std::vector` initialization for uniformly increasing sizes. static void BM_DenseRange(benchmark::State& state) { for(auto _ : state) { std::vector v(state.range(0), state.range(0)); - benchmark::DoNotOptimize(v.data()); + auto data = v.data(); + benchmark::DoNotOptimize(data); benchmark::ClobberMemory(); } } @@ -362,17 +391,17 @@ short-hand. The following macro will pick a few appropriate arguments in the product of the two specified ranges and will generate a benchmark for each such pair. -{% raw %} + ```c++ BENCHMARK(BM_SetInsert)->Ranges({{1<<10, 8<<10}, {128, 512}}); ``` -{% endraw %} + Some benchmarks may require specific argument values that cannot be expressed with `Ranges`. In this case, `ArgsProduct` offers the ability to generate a benchmark input for each combination in the product of the supplied vectors. -{% raw %} + ```c++ BENCHMARK(BM_SetInsert) ->ArgsProduct({{1<<10, 3<<10, 8<<10}, {20, 40, 60, 80}}) @@ -391,7 +420,7 @@ BENCHMARK(BM_SetInsert) ->Args({3<<10, 80}) ->Args({8<<10, 80}); ``` -{% endraw %} + For the most common scenarios, helper methods for creating a list of integers for a given sparse or dense range are provided. @@ -434,13 +463,22 @@ The `test_case_name` is appended to the name of the benchmark and should describe the values passed. ```c++ -template -void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) { - [...] +template +void BM_takes_args(benchmark::State& state, Args&&... args) { + auto args_tuple = std::make_tuple(std::move(args)...); + for (auto _ : state) { + std::cout << std::get<0>(args_tuple) << ": " << std::get<1>(args_tuple) + << '\n'; + [...] + } } // Registers a benchmark named "BM_takes_args/int_string_test" that passes -// the specified values to `extra_args`. +// the specified values to `args`. BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); + +// Registers the same benchmark "BM_takes_args/int_test" that passes +// the specified values to `args`. +BENCHMARK_CAPTURE(BM_takes_args, int_test, 42, 43); ``` Note that elements of `...args` may refer to global variables. Users should @@ -459,7 +497,8 @@ static void BM_StringCompare(benchmark::State& state) { std::string s1(state.range(0), '-'); std::string s2(state.range(0), '-'); for (auto _ : state) { - benchmark::DoNotOptimize(s1.compare(s2)); + auto comparison_result = s1.compare(s2); + benchmark::DoNotOptimize(comparison_result); } state.SetComplexityN(state.range(0)); } @@ -537,6 +576,30 @@ Three macros are provided for adding benchmark templates. #define BENCHMARK_TEMPLATE2(func, arg1, arg2) ``` + + +## Templated Benchmarks that take arguments + +Sometimes there is a need to template benchmarks, and provide arguments to them. + +```c++ +template void BM_Sequential_With_Step(benchmark::State& state, int step) { + Q q; + typename Q::value_type v; + for (auto _ : state) { + for (int i = state.range(0); i-=step; ) + q.push(v); + for (int e = state.range(0); e-=step; ) + q.Wait(&v); + } + // actually messages, not bytes: + state.SetBytesProcessed( + static_cast(state.iterations())*state.range(0)); +} + +BENCHMARK_TEMPLATE1_CAPTURE(BM_Sequential, WaitQueue, Step1, 1)->Range(1<<0, 1<<10); +``` + ## Fixtures @@ -554,10 +617,10 @@ For Example: ```c++ class MyFixture : public benchmark::Fixture { public: - void SetUp(const ::benchmark::State& state) { + void SetUp(::benchmark::State& state) { } - void TearDown(const ::benchmark::State& state) { + void TearDown(::benchmark::State& state) { } }; @@ -668,7 +731,7 @@ is 1k a 1000 (default, `benchmark::Counter::OneK::kIs1000`), or 1024 When you're compiling in C++11 mode or later you can use `insert()` with `std::initializer_list`: -{% raw %} + ```c++ // With C++11, this can be done: state.counters.insert({{"Foo", numFoos}, {"Bar", numBars}, {"Baz", numBazs}}); @@ -677,7 +740,7 @@ When you're compiling in C++11 mode or later you can use `insert()` with state.counters["Bar"] = numBars; state.counters["Baz"] = numBazs; ``` -{% endraw %} + ### Counter Reporting @@ -773,6 +836,16 @@ static void BM_MultiThreaded(benchmark::State& state) { BENCHMARK(BM_MultiThreaded)->Threads(2); ``` +To run the benchmark across a range of thread counts, instead of `Threads`, use +`ThreadRange`. This takes two parameters (`min_threads` and `max_threads`) and +runs the benchmark once for values in the inclusive range. For example: + +```c++ +BENCHMARK(BM_MultiThreaded)->ThreadRange(1, 8); +``` + +will run `BM_MultiThreaded` with thread counts 1, 2, 4, and 8. + If the benchmarked code itself uses threads and you want to compare it to single-threaded code, you may want to use real-time ("wallclock") measurements for latency comparisons: @@ -814,7 +887,7 @@ BENCHMARK(BM_OpenMP)->Range(8, 8<<10); // Measure the user-visible time, the wall clock (literally, the time that // has passed on the clock on the wall), use it to decide for how long to -// run the benchmark loop. This will always be meaningful, an will match the +// run the benchmark loop. This will always be meaningful, and will match the // time spent by the main thread in single-threaded case, in general decreasing // with the number of internal threads doing the work. BENCHMARK(BM_OpenMP)->Range(8, 8<<10)->UseRealTime(); @@ -836,7 +909,7 @@ is measured. But sometimes, it is necessary to do some work inside of that loop, every iteration, but without counting that time to the benchmark time. That is possible, although it is not recommended, since it has high overhead. -{% raw %} + ```c++ static void BM_SetInsert_With_Timer_Control(benchmark::State& state) { std::set data; @@ -851,7 +924,7 @@ static void BM_SetInsert_With_Timer_Control(benchmark::State& state) { } BENCHMARK(BM_SetInsert_With_Timer_Control)->Ranges({{1<<10, 8<<10}, {128, 512}}); ``` -{% endraw %} + @@ -906,6 +979,10 @@ order to manually set the time unit, you can specify it manually: BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); ``` +Additionally the default time unit can be set globally with the +`--benchmark_time_unit={ns|us|ms|s}` command line argument. The argument only +affects benchmarks where the time unit is not set explicitly. + ## Preventing Optimization @@ -958,7 +1035,8 @@ static void BM_vector_push_back(benchmark::State& state) { for (auto _ : state) { std::vector v; v.reserve(1); - benchmark::DoNotOptimize(v.data()); // Allow v.data() to be clobbered. + auto data = v.data(); // Allow v.data() to be clobbered. Pass as non-const + benchmark::DoNotOptimize(data); // lvalue to avoid undesired compiler optimizations v.push_back(42); benchmark::ClobberMemory(); // Force 42 to be written to memory. } @@ -1037,10 +1115,25 @@ void BM_spin_empty(benchmark::State& state) { BENCHMARK(BM_spin_empty) ->ComputeStatistics("ratio", [](const std::vector& v) -> double { return std::begin(v) / std::end(v); - }, benchmark::StatisticUnit::Percentage) + }, benchmark::StatisticUnit::kPercentage) ->Arg(512); ``` + + +## Memory Usage + +It's often useful to also track memory usage for benchmarks, alongside CPU +performance. For this reason, benchmark offers the `RegisterMemoryManager` +method that allows a custom `MemoryManager` to be injected. + +If set, the `MemoryManager::Start` and `MemoryManager::Stop` methods will be +called at the start and end of benchmark runs to allow user code to fill out +a report on the number of allocations, bytes used, etc. + +This data will then be reported alongside other performance data, currently +only when using JSON output. + ## Using RegisterBenchmark(name, fn, args...) @@ -1077,7 +1170,7 @@ int main(int argc, char** argv) { When errors caused by external influences, such as file I/O and network communication, occur within a benchmark the -`State::SkipWithError(const char* msg)` function can be used to skip that run +`State::SkipWithError(const std::string& msg)` function can be used to skip that run of benchmark and report the error. Note that only future iterations of the `KeepRunning()` are skipped. For the ranged-for version of the benchmark loop Users must explicitly exit the loop, otherwise all iterations will be performed. @@ -1188,13 +1281,12 @@ the benchmark loop should be preferred. If you see this error: ``` -***WARNING*** CPU scaling is enabled, the benchmark real time measurements may be noisy and will incur extra overhead. +***WARNING*** CPU scaling is enabled, the benchmark real time measurements may +be noisy and will incur extra overhead. ``` -you might want to disable the CPU frequency scaling while running the benchmark: +you might want to disable the CPU frequency scaling while running the +benchmark, as well as consider other ways to stabilize the performance of +your system while benchmarking. -```bash -sudo cpupower frequency-set --governor performance -./mybench -sudo cpupower frequency-set --governor powersave -``` +See [Reducing Variance](reducing_variance.md) for more information. diff --git a/third-party/benchmark/include/benchmark/benchmark.h b/third-party/benchmark/include/benchmark/benchmark.h index 6287c0afbdcf..08cfe29da344 100644 --- a/third-party/benchmark/include/benchmark/benchmark.h +++ b/third-party/benchmark/include/benchmark/benchmark.h @@ -187,6 +187,8 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #include #include +#include "benchmark/export.h" + #if defined(BENCHMARK_HAS_CXX11) #include #include @@ -216,37 +218,45 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #define BENCHMARK_UNUSED #endif +// Used to annotate functions, methods and classes so they +// are not optimized by the compiler. Useful for tests +// where you expect loops to stay in place churning cycles +#if defined(__clang__) +#define BENCHMARK_DONT_OPTIMIZE __attribute__((optnone)) +#elif defined(__GNUC__) || defined(__GNUG__) +#define BENCHMARK_DONT_OPTIMIZE __attribute__((optimize(0))) +#else +// MSVC & Intel do not have a no-optimize attribute, only line pragmas +#define BENCHMARK_DONT_OPTIMIZE +#endif + #if defined(__GNUC__) || defined(__clang__) #define BENCHMARK_ALWAYS_INLINE __attribute__((always_inline)) -#define BENCHMARK_NOEXCEPT noexcept -#define BENCHMARK_NOEXCEPT_OP(x) noexcept(x) #elif defined(_MSC_VER) && !defined(__clang__) #define BENCHMARK_ALWAYS_INLINE __forceinline -#if _MSC_VER >= 1900 -#define BENCHMARK_NOEXCEPT noexcept -#define BENCHMARK_NOEXCEPT_OP(x) noexcept(x) -#else -#define BENCHMARK_NOEXCEPT -#define BENCHMARK_NOEXCEPT_OP(x) -#endif #define __func__ __FUNCTION__ #else #define BENCHMARK_ALWAYS_INLINE -#define BENCHMARK_NOEXCEPT -#define BENCHMARK_NOEXCEPT_OP(x) #endif #define BENCHMARK_INTERNAL_TOSTRING2(x) #x #define BENCHMARK_INTERNAL_TOSTRING(x) BENCHMARK_INTERNAL_TOSTRING2(x) // clang-format off -#if defined(__GNUC__) || defined(__clang__) +#if (defined(__GNUC__) && !defined(__NVCC__) && !defined(__NVCOMPILER)) || defined(__clang__) #define BENCHMARK_BUILTIN_EXPECT(x, y) __builtin_expect(x, y) #define BENCHMARK_DEPRECATED_MSG(msg) __attribute__((deprecated(msg))) #define BENCHMARK_DISABLE_DEPRECATED_WARNING \ _Pragma("GCC diagnostic push") \ _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") #define BENCHMARK_RESTORE_DEPRECATED_WARNING _Pragma("GCC diagnostic pop") +#elif defined(__NVCOMPILER) +#define BENCHMARK_BUILTIN_EXPECT(x, y) __builtin_expect(x, y) +#define BENCHMARK_DEPRECATED_MSG(msg) __attribute__((deprecated(msg))) +#define BENCHMARK_DISABLE_DEPRECATED_WARNING \ + _Pragma("diagnostic push") \ + _Pragma("diag_suppress deprecated_entity_with_custom_message") +#define BENCHMARK_RESTORE_DEPRECATED_WARNING _Pragma("diagnostic pop") #else #define BENCHMARK_BUILTIN_EXPECT(x, y) x #define BENCHMARK_DEPRECATED_MSG(msg) @@ -280,18 +290,47 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #define BENCHMARK_OVERRIDE #endif +#if defined(_MSC_VER) +#pragma warning(push) +// C4251: needs to have dll-interface to be used by clients of class +#pragma warning(disable : 4251) +#endif + namespace benchmark { class BenchmarkReporter; -void Initialize(int* argc, char** argv); -void Shutdown(); +// Default number of minimum benchmark running time in seconds. +const char kDefaultMinTimeStr[] = "0.5s"; + +// Returns the version of the library. +BENCHMARK_EXPORT std::string GetBenchmarkVersion(); + +BENCHMARK_EXPORT void PrintDefaultHelp(); + +BENCHMARK_EXPORT void Initialize(int* argc, char** argv, + void (*HelperPrinterf)() = PrintDefaultHelp); +BENCHMARK_EXPORT void Shutdown(); // Report to stdout all arguments in 'argv' as unrecognized except the first. // Returns true there is at least on unrecognized argument (i.e. 'argc' > 1). -bool ReportUnrecognizedArguments(int argc, char** argv); +BENCHMARK_EXPORT bool ReportUnrecognizedArguments(int argc, char** argv); // Returns the current value of --benchmark_filter. -std::string GetBenchmarkFilter(); +BENCHMARK_EXPORT std::string GetBenchmarkFilter(); + +// Sets a new value to --benchmark_filter. (This will override this flag's +// current value). +// Should be called after `benchmark::Initialize()`, as +// `benchmark::Initialize()` will override the flag's value. +BENCHMARK_EXPORT void SetBenchmarkFilter(std::string value); + +// Returns the current value of --v (command line value for verbosity). +BENCHMARK_EXPORT int32_t GetBenchmarkVerbosity(); + +// Creates a default display reporter. Used by the library when no display +// reporter is provided, but also made available for external use in case a +// custom reporter should respect the `--benchmark_format` flag as a fallback +BENCHMARK_EXPORT BenchmarkReporter* CreateDefaultDisplayReporter(); // Generate a list of benchmarks matching the specified --benchmark_filter flag // and if --benchmark_list_tests is specified return after printing the name @@ -305,22 +344,33 @@ std::string GetBenchmarkFilter(); // The second and third overload use the specified 'display_reporter' and // 'file_reporter' respectively. 'file_reporter' will write to the file // specified -// by '--benchmark_output'. If '--benchmark_output' is not given the +// by '--benchmark_out'. If '--benchmark_out' is not given the // 'file_reporter' is ignored. // // RETURNS: The number of matching benchmarks. -size_t RunSpecifiedBenchmarks(); -size_t RunSpecifiedBenchmarks(std::string spec); +BENCHMARK_EXPORT size_t RunSpecifiedBenchmarks(); +BENCHMARK_EXPORT size_t RunSpecifiedBenchmarks(std::string spec); + +BENCHMARK_EXPORT size_t +RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter); +BENCHMARK_EXPORT size_t +RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, std::string spec); -size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter); -size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, - std::string spec); +BENCHMARK_EXPORT size_t RunSpecifiedBenchmarks( + BenchmarkReporter* display_reporter, BenchmarkReporter* file_reporter); +BENCHMARK_EXPORT size_t +RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, + BenchmarkReporter* file_reporter, std::string spec); -size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, - BenchmarkReporter* file_reporter); -size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, - BenchmarkReporter* file_reporter, - std::string spec); +// TimeUnit is passed to a benchmark in order to specify the order of magnitude +// for the measured time. +enum TimeUnit { kNanosecond, kMicrosecond, kMillisecond, kSecond }; + +BENCHMARK_EXPORT TimeUnit GetDefaultTimeUnit(); + +// Sets the default time unit the benchmarks use +// Has to be called before the benchmark loop to take effect +BENCHMARK_EXPORT void SetDefaultTimeUnit(TimeUnit unit); // If a MemoryManager is registered (via RegisterMemoryManager()), // it can be used to collect and report allocation metrics for a run of the @@ -358,20 +408,16 @@ class MemoryManager { virtual void Start() = 0; // Implement this to stop recording and fill out the given Result structure. - BENCHMARK_DEPRECATED_MSG("Use Stop(Result&) instead") - virtual void Stop(Result* result) = 0; - - // FIXME(vyng): Make this pure virtual once we've migrated current users. - BENCHMARK_DISABLE_DEPRECATED_WARNING - virtual void Stop(Result& result) { Stop(&result); } - BENCHMARK_RESTORE_DEPRECATED_WARNING + virtual void Stop(Result& result) = 0; }; // Register a MemoryManager instance that will be used to collect and report // allocation measurements for benchmark runs. +BENCHMARK_EXPORT void RegisterMemoryManager(MemoryManager* memory_manager); // Add a key-value pair to output as part of the context stanza in the report. +BENCHMARK_EXPORT void AddCustomContext(const std::string& key, const std::string& value); namespace internal { @@ -379,14 +425,17 @@ class Benchmark; class BenchmarkImp; class BenchmarkFamilies; +BENCHMARK_EXPORT std::map*& GetGlobalContext(); + +BENCHMARK_EXPORT void UseCharPointer(char const volatile*); // Take ownership of the pointer and register the benchmark. Return the // registered benchmark. -Benchmark* RegisterBenchmarkInternal(Benchmark*); +BENCHMARK_EXPORT Benchmark* RegisterBenchmarkInternal(Benchmark*); // Ensure that the standard streams are properly initialized in every TU. -int InitializeStreams(); +BENCHMARK_EXPORT int InitializeStreams(); BENCHMARK_UNUSED static int stream_init_anchor = InitializeStreams(); } // namespace internal @@ -409,7 +458,11 @@ inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { // intended to add little to no overhead. // See: https://youtu.be/nXaxk27zwlk?t=2441 #ifndef BENCHMARK_HAS_NO_INLINE_ASSEMBLY +#if !defined(__GNUC__) || defined(__llvm__) || defined(__INTEL_COMPILER) template +BENCHMARK_DEPRECATED_MSG( + "The const-ref version of this method can permit " + "undesired compiler optimizations in benchmarks") inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { asm volatile("" : : "r,m"(value) : "memory"); } @@ -423,6 +476,98 @@ inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp& value) { #endif } +#ifdef BENCHMARK_HAS_CXX11 +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp&& value) { +#if defined(__clang__) + asm volatile("" : "+r,m"(value) : : "memory"); +#else + asm volatile("" : "+m,r"(value) : : "memory"); +#endif +} +#endif +#elif defined(BENCHMARK_HAS_CXX11) && (__GNUC__ >= 5) +// Workaround for a bug with full argument copy overhead with GCC. +// See: #1340 and https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105519 +template +BENCHMARK_DEPRECATED_MSG( + "The const-ref version of this method can permit " + "undesired compiler optimizations in benchmarks") +inline BENCHMARK_ALWAYS_INLINE + typename std::enable_if::value && + (sizeof(Tp) <= sizeof(Tp*))>::type + DoNotOptimize(Tp const& value) { + asm volatile("" : : "r,m"(value) : "memory"); +} + +template +BENCHMARK_DEPRECATED_MSG( + "The const-ref version of this method can permit " + "undesired compiler optimizations in benchmarks") +inline BENCHMARK_ALWAYS_INLINE + typename std::enable_if::value || + (sizeof(Tp) > sizeof(Tp*))>::type + DoNotOptimize(Tp const& value) { + asm volatile("" : : "m"(value) : "memory"); +} + +template +inline BENCHMARK_ALWAYS_INLINE + typename std::enable_if::value && + (sizeof(Tp) <= sizeof(Tp*))>::type + DoNotOptimize(Tp& value) { + asm volatile("" : "+m,r"(value) : : "memory"); +} + +template +inline BENCHMARK_ALWAYS_INLINE + typename std::enable_if::value || + (sizeof(Tp) > sizeof(Tp*))>::type + DoNotOptimize(Tp& value) { + asm volatile("" : "+m"(value) : : "memory"); +} + +template +inline BENCHMARK_ALWAYS_INLINE + typename std::enable_if::value && + (sizeof(Tp) <= sizeof(Tp*))>::type + DoNotOptimize(Tp&& value) { + asm volatile("" : "+m,r"(value) : : "memory"); +} + +template +inline BENCHMARK_ALWAYS_INLINE + typename std::enable_if::value || + (sizeof(Tp) > sizeof(Tp*))>::type + DoNotOptimize(Tp&& value) { + asm volatile("" : "+m"(value) : : "memory"); +} + +#else +// Fallback for GCC < 5. Can add some overhead because the compiler is forced +// to use memory operations instead of operations with registers. +// TODO: Remove if GCC < 5 will be unsupported. +template +BENCHMARK_DEPRECATED_MSG( + "The const-ref version of this method can permit " + "undesired compiler optimizations in benchmarks") +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { + asm volatile("" : : "m"(value) : "memory"); +} + +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp& value) { + asm volatile("" : "+m"(value) : : "memory"); +} + +#ifdef BENCHMARK_HAS_CXX11 +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp&& value) { + asm volatile("" : "+m"(value) : : "memory"); +} +#endif +#endif + #ifndef BENCHMARK_HAS_CXX11 inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { asm volatile("" : : : "memory"); @@ -430,6 +575,9 @@ inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { #endif #elif defined(_MSC_VER) template +BENCHMARK_DEPRECATED_MSG( + "The const-ref version of this method can permit " + "undesired compiler optimizations in benchmarks") inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { internal::UseCharPointer(&reinterpret_cast(value)); _ReadWriteBarrier(); @@ -439,10 +587,25 @@ inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { _ReadWriteBarrier(); } #endif #else +#ifdef BENCHMARK_HAS_CXX11 +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp&& value) { + internal::UseCharPointer(&reinterpret_cast(value)); +} +#else template +BENCHMARK_DEPRECATED_MSG( + "The const-ref version of this method can permit " + "undesired compiler optimizations in benchmarks") inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { internal::UseCharPointer(&reinterpret_cast(value)); } + +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp& value) { + internal::UseCharPointer(&reinterpret_cast(value)); +} +#endif // FIXME Add ClobberMemory() for non-gnu and non-msvc compilers, before C++11. #endif @@ -506,23 +669,21 @@ Counter::Flags inline operator|(const Counter::Flags& LHS, // This is the container for the user-defined counters. typedef std::map UserCounters; -// TimeUnit is passed to a benchmark in order to specify the order of magnitude -// for the measured time. -enum TimeUnit { kNanosecond, kMicrosecond, kMillisecond, kSecond }; - // BigO is passed to a benchmark in order to specify the asymptotic // computational // complexity for the benchmark. In case oAuto is selected, complexity will be // calculated automatically to the best fit. enum BigO { oNone, o1, oN, oNSquared, oNCubed, oLogN, oNLogN, oAuto, oLambda }; -typedef uint64_t IterationCount; +typedef int64_t ComplexityN; + +typedef int64_t IterationCount; enum StatisticUnit { kTime, kPercentage }; // BigOFunc is passed to a benchmark in order to specify the asymptotic // computational complexity for the benchmark. -typedef double(BigOFunc)(IterationCount); +typedef double(BigOFunc)(ComplexityN); // StatisticsFunc is passed to a benchmark in order to compute some descriptive // statistics over all the measurements of some type @@ -564,11 +725,21 @@ enum AggregationReportMode ARM_FileReportAggregatesOnly | ARM_DisplayReportAggregatesOnly }; +enum Skipped +#if defined(BENCHMARK_HAS_CXX11) + : unsigned +#endif +{ + NotSkipped = 0, + SkippedWithMessage, + SkippedWithError +}; + } // namespace internal // State is passed to a running Benchmark and contains state for the // benchmark to use. -class State { +class BENCHMARK_EXPORT State { public: struct StateIterator; friend struct StateIterator; @@ -580,13 +751,13 @@ class State { // have been called previously. // // NOTE: KeepRunning may not be used after calling either of these functions. - BENCHMARK_ALWAYS_INLINE StateIterator begin(); - BENCHMARK_ALWAYS_INLINE StateIterator end(); + inline BENCHMARK_ALWAYS_INLINE StateIterator begin(); + inline BENCHMARK_ALWAYS_INLINE StateIterator end(); // Returns true if the benchmark should continue through another iteration. // NOTE: A benchmark may not return from the test until KeepRunning() has // returned false. - bool KeepRunning(); + inline bool KeepRunning(); // Returns true iff the benchmark should run n more iterations. // REQUIRES: 'n' > 0. @@ -598,10 +769,10 @@ class State { // while (state.KeepRunningBatch(1000)) { // // process 1000 elements // } - bool KeepRunningBatch(IterationCount n); + inline bool KeepRunningBatch(IterationCount n); - // REQUIRES: timer is running and 'SkipWithError(...)' has not been called - // by the current thread. + // REQUIRES: timer is running and 'SkipWithMessage(...)' or + // 'SkipWithError(...)' has not been called by the current thread. // Stop the benchmark timer. If not called, the timer will be // automatically stopped after the last iteration of the benchmark loop. // @@ -616,8 +787,8 @@ class State { // within each benchmark iteration, if possible. void PauseTiming(); - // REQUIRES: timer is not running and 'SkipWithError(...)' has not been called - // by the current thread. + // REQUIRES: timer is not running and 'SkipWithMessage(...)' or + // 'SkipWithError(...)' has not been called by the current thread. // Start the benchmark timer. The timer is NOT running on entrance to the // benchmark function. It begins running after control flow enters the // benchmark loop. @@ -627,8 +798,30 @@ class State { // within each benchmark iteration, if possible. void ResumeTiming(); - // REQUIRES: 'SkipWithError(...)' has not been called previously by the - // current thread. + // REQUIRES: 'SkipWithMessage(...)' or 'SkipWithError(...)' has not been + // called previously by the current thread. + // Report the benchmark as resulting in being skipped with the specified + // 'msg'. + // After this call the user may explicitly 'return' from the benchmark. + // + // If the ranged-for style of benchmark loop is used, the user must explicitly + // break from the loop, otherwise all future iterations will be run. + // If the 'KeepRunning()' loop is used the current thread will automatically + // exit the loop at the end of the current iteration. + // + // For threaded benchmarks only the current thread stops executing and future + // calls to `KeepRunning()` will block until all threads have completed + // the `KeepRunning()` loop. If multiple threads report being skipped only the + // first skip message is used. + // + // NOTE: Calling 'SkipWithMessage(...)' does not cause the benchmark to exit + // the current scope immediately. If the function is called from within + // the 'KeepRunning()' loop the current iteration will finish. It is the users + // responsibility to exit the scope as needed. + void SkipWithMessage(const std::string& msg); + + // REQUIRES: 'SkipWithMessage(...)' or 'SkipWithError(...)' has not been + // called previously by the current thread. // Report the benchmark as resulting in an error with the specified 'msg'. // After this call the user may explicitly 'return' from the benchmark. // @@ -646,10 +839,13 @@ class State { // the current scope immediately. If the function is called from within // the 'KeepRunning()' loop the current iteration will finish. It is the users // responsibility to exit the scope as needed. - void SkipWithError(const char* msg); + void SkipWithError(const std::string& msg); + + // Returns true if 'SkipWithMessage(...)' or 'SkipWithError(...)' was called. + bool skipped() const { return internal::NotSkipped != skipped_; } // Returns true if an error has been reported with 'SkipWithError(...)'. - bool error_occurred() const { return error_occurred_; } + bool error_occurred() const { return internal::SkippedWithError == skipped_; } // REQUIRES: called exactly once per iteration of the benchmarking loop. // Set the manually measured time for this benchmark iteration, which @@ -684,10 +880,12 @@ class State { // and complexity_n will // represent the length of N. BENCHMARK_ALWAYS_INLINE - void SetComplexityN(int64_t complexity_n) { complexity_n_ = complexity_n; } + void SetComplexityN(ComplexityN complexity_n) { + complexity_n_ = complexity_n; + } BENCHMARK_ALWAYS_INLINE - int64_t complexity_length_n() const { return complexity_n_; } + ComplexityN complexity_length_n() const { return complexity_n_; } // If this routine is called with items > 0, then an items/s // label is printed on the benchmark report line for the currently @@ -720,11 +918,7 @@ class State { // BM_Compress 50 50 14115038 compress:27.3% // // REQUIRES: a benchmark has exited its benchmarking loop. - void SetLabel(const char* label); - - void BENCHMARK_ALWAYS_INLINE SetLabel(const std::string& str) { - this->SetLabel(str.c_str()); - } + void SetLabel(const std::string& label); // Range arguments for this run. CHECKs if the argument has been set. BENCHMARK_ALWAYS_INLINE @@ -755,6 +949,9 @@ class State { return max_iterations - total_iterations_ + batch_leftover_; } + BENCHMARK_ALWAYS_INLINE + std::string name() const { return name_; } + private: // items we expect on the first cache line (ie 64 bytes of the struct) // When total_iterations_ is 0, KeepRunning() and friends will return false. @@ -772,29 +969,30 @@ class State { private: bool started_; bool finished_; - bool error_occurred_; + internal::Skipped skipped_; // items we don't need on the first cache line std::vector range_; - int64_t complexity_n_; + ComplexityN complexity_n_; public: // Container for user-defined counters. UserCounters counters; private: - State(IterationCount max_iters, const std::vector& ranges, - int thread_i, int n_threads, internal::ThreadTimer* timer, - internal::ThreadManager* manager, + State(std::string name, IterationCount max_iters, + const std::vector& ranges, int thread_i, int n_threads, + internal::ThreadTimer* timer, internal::ThreadManager* manager, internal::PerfCountersMeasurement* perf_counters_measurement); void StartKeepRunning(); // Implementation of KeepRunning() and KeepRunningBatch(). // is_batch must be true unless n is 1. - bool KeepRunningInternal(IterationCount n, bool is_batch); + inline bool KeepRunningInternal(IterationCount n, bool is_batch); void FinishKeepRunning(); + const std::string name_; const int thread_index_; const int threads_; @@ -826,7 +1024,7 @@ inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningInternal(IterationCount n, } if (!started_) { StartKeepRunning(); - if (!error_occurred_ && total_iterations_ >= n) { + if (!skipped() && total_iterations_ >= n) { total_iterations_ -= n; return true; } @@ -856,7 +1054,7 @@ struct State::StateIterator { BENCHMARK_ALWAYS_INLINE explicit StateIterator(State* st) - : cached_(st->error_occurred_ ? 0 : st->max_iterations), parent_(st) {} + : cached_(st->skipped() ? 0 : st->max_iterations), parent_(st) {} public: BENCHMARK_ALWAYS_INLINE @@ -899,7 +1097,7 @@ typedef void(Function)(State&); // be called on this object to change the properties of the benchmark. // Each method returns "this" so that multiple method calls can // chained into one expression. -class Benchmark { +class BENCHMARK_EXPORT Benchmark { public: virtual ~Benchmark(); @@ -971,7 +1169,7 @@ class Benchmark { // Have "setup" and/or "teardown" invoked once for every benchmark run. // If the benchmark is multi-threaded (will run in k threads concurrently), - // the setup callback will be invoked exactly once (not k times) before + // the setup callback will be be invoked exactly once (not k times) before // each run with k threads. Time allowing (e.g. for a short benchmark), there // may be multiple such runs per benchmark, each run with its own // "setup"/"teardown". @@ -1000,12 +1198,19 @@ class Benchmark { // REQUIRES: `t > 0` and `Iterations` has not been called on this benchmark. Benchmark* MinTime(double t); + // Set the minimum amount of time to run the benchmark before taking runtimes + // of this benchmark into account. This + // option overrides the `benchmark_min_warmup_time` flag. + // REQUIRES: `t >= 0` and `Iterations` has not been called on this benchmark. + Benchmark* MinWarmUpTime(double t); + // Specify the amount of iterations that should be run by this benchmark. + // This option overrides the `benchmark_min_time` flag. // REQUIRES: 'n > 0' and `MinTime` has not been called on this benchmark. // // NOTE: This function should only be used when *exact* iteration control is // needed and never to control or limit how long a benchmark runs, where - // `--benchmark_min_time=N` or `MinTime(...)` should be used instead. + // `--benchmark_min_time=s` or `MinTime(...)` should be used instead. Benchmark* Iterations(IterationCount n); // Specify the amount of times to repeat this benchmark. This option overrides @@ -1025,7 +1230,7 @@ class Benchmark { // By default, the CPU time is measured only for the main thread, which may // be unrepresentative if the benchmark uses threads internally. If called, // the total CPU time spent by all the threads will be measured instead. - // By default, the only the main thread CPU time will be measured. + // By default, only the main thread CPU time will be measured. Benchmark* MeasureProcessCPUTime(); // If a particular benchmark should use the Wall clock instead of the CPU time @@ -1090,12 +1295,16 @@ class Benchmark { virtual void Run(State& state) = 0; + TimeUnit GetTimeUnit() const; + protected: - explicit Benchmark(const char* name); - Benchmark(Benchmark const&); - void SetName(const char* name); + explicit Benchmark(const std::string& name); + void SetName(const std::string& name); + public: + const char* GetName() const; int ArgsCnt() const; + const char* GetArgName(int arg) const; private: friend class BenchmarkFamilies; @@ -1105,9 +1314,13 @@ class Benchmark { AggregationReportMode aggregation_report_mode_; std::vector arg_names_; // Args for all benchmark runs std::vector > args_; // Args for all benchmark runs + TimeUnit time_unit_; + bool use_default_time_unit_; + int range_multiplier_; double min_time_; + double min_warmup_time_; IterationCount iterations_; int repetitions_; bool measure_process_cpu_time_; @@ -1122,7 +1335,17 @@ class Benchmark { callback_function setup_; callback_function teardown_; - Benchmark& operator=(Benchmark const&); + Benchmark(Benchmark const&) +#if defined(BENCHMARK_HAS_CXX11) + = delete +#endif + ; + + Benchmark& operator=(Benchmark const&) +#if defined(BENCHMARK_HAS_CXX11) + = delete +#endif + ; }; } // namespace internal @@ -1131,27 +1354,27 @@ class Benchmark { // the specified functor 'fn'. // // RETURNS: A pointer to the registered benchmark. -internal::Benchmark* RegisterBenchmark(const char* name, +internal::Benchmark* RegisterBenchmark(const std::string& name, internal::Function* fn); #if defined(BENCHMARK_HAS_CXX11) template -internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn); +internal::Benchmark* RegisterBenchmark(const std::string& name, Lambda&& fn); #endif // Remove all registered benchmarks. All pointers to previously registered // benchmarks are invalidated. -void ClearRegisteredBenchmarks(); +BENCHMARK_EXPORT void ClearRegisteredBenchmarks(); namespace internal { // The class used to hold all Benchmarks created from static function. // (ie those created using the BENCHMARK(...) macros. -class FunctionBenchmark : public Benchmark { +class BENCHMARK_EXPORT FunctionBenchmark : public Benchmark { public: - FunctionBenchmark(const char* name, Function* func) + FunctionBenchmark(const std::string& name, Function* func) : Benchmark(name), func_(func) {} - virtual void Run(State& st) BENCHMARK_OVERRIDE; + void Run(State& st) BENCHMARK_OVERRIDE; private: Function* func_; @@ -1161,35 +1384,38 @@ class FunctionBenchmark : public Benchmark { template class LambdaBenchmark : public Benchmark { public: - virtual void Run(State& st) BENCHMARK_OVERRIDE { lambda_(st); } + void Run(State& st) BENCHMARK_OVERRIDE { lambda_(st); } private: template - LambdaBenchmark(const char* name, OLambda&& lam) + LambdaBenchmark(const std::string& name, OLambda&& lam) : Benchmark(name), lambda_(std::forward(lam)) {} LambdaBenchmark(LambdaBenchmark const&) = delete; template // NOLINTNEXTLINE(readability-redundant-declaration) - friend Benchmark* ::benchmark::RegisterBenchmark(const char*, Lam&&); + friend Benchmark* ::benchmark::RegisterBenchmark(const std::string&, Lam&&); Lambda lambda_; }; #endif - } // namespace internal -inline internal::Benchmark* RegisterBenchmark(const char* name, +inline internal::Benchmark* RegisterBenchmark(const std::string& name, internal::Function* fn) { + // FIXME: this should be a `std::make_unique<>()` but we don't have C++14. + // codechecker_intentional [cplusplus.NewDeleteLeaks] return internal::RegisterBenchmarkInternal( ::new internal::FunctionBenchmark(name, fn)); } #ifdef BENCHMARK_HAS_CXX11 template -internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn) { +internal::Benchmark* RegisterBenchmark(const std::string& name, Lambda&& fn) { using BenchType = internal::LambdaBenchmark::type>; + // FIXME: this should be a `std::make_unique<>()` but we don't have C++14. + // codechecker_intentional [cplusplus.NewDeleteLeaks] return internal::RegisterBenchmarkInternal( ::new BenchType(name, std::forward(fn))); } @@ -1198,7 +1424,7 @@ internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn) { #if defined(BENCHMARK_HAS_CXX11) && \ (!defined(BENCHMARK_GCC_VERSION) || BENCHMARK_GCC_VERSION >= 409) template -internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn, +internal::Benchmark* RegisterBenchmark(const std::string& name, Lambda&& fn, Args&&... args) { return benchmark::RegisterBenchmark( name, [=](benchmark::State& st) { fn(st, args...); }); @@ -1212,7 +1438,7 @@ class Fixture : public internal::Benchmark { public: Fixture() : internal::Benchmark("") {} - virtual void Run(State& st) BENCHMARK_OVERRIDE { + void Run(State& st) BENCHMARK_OVERRIDE { this->SetUp(st); this->BenchmarkCase(st); this->TearDown(st); @@ -1228,7 +1454,6 @@ class Fixture : public internal::Benchmark { protected: virtual void BenchmarkCase(State&) = 0; }; - } // namespace benchmark // ------------------------------------------------------ @@ -1268,7 +1493,7 @@ class Fixture : public internal::Benchmark { BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ (::benchmark::internal::RegisterBenchmarkInternal( \ new ::benchmark::internal::FunctionBenchmark(#__VA_ARGS__, \ - &__VA_ARGS__))) + __VA_ARGS__))) #else #define BENCHMARK(n) \ BENCHMARK_PRIVATE_DECLARE(n) = \ @@ -1298,7 +1523,7 @@ class Fixture : public internal::Benchmark { // /* Registers a benchmark named "BM_takes_args/int_string_test` */ // BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); #define BENCHMARK_CAPTURE(func, test_case_name, ...) \ - BENCHMARK_PRIVATE_DECLARE(func) = \ + BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ (::benchmark::internal::RegisterBenchmarkInternal( \ new ::benchmark::internal::FunctionBenchmark( \ #func "/" #test_case_name, \ @@ -1335,37 +1560,62 @@ class Fixture : public internal::Benchmark { #define BENCHMARK_TEMPLATE(n, a) BENCHMARK_TEMPLATE1(n, a) #endif -#define BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ - class BaseClass##_##Method##_Benchmark : public BaseClass { \ - public: \ - BaseClass##_##Method##_Benchmark() { \ - this->SetName(#BaseClass "/" #Method); \ - } \ - \ - protected: \ - virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ +#ifdef BENCHMARK_HAS_CXX11 +// This will register a benchmark for a templatized function, +// with the additional arguments specified by `...`. +// +// For example: +// +// template ` +// void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) { +// [...] +//} +// /* Registers a benchmark named "BM_takes_args/int_string_test` */ +// BENCHMARK_TEMPLATE1_CAPTURE(BM_takes_args, void, int_string_test, 42, +// std::string("abc")); +#define BENCHMARK_TEMPLATE1_CAPTURE(func, a, test_case_name, ...) \ + BENCHMARK_CAPTURE(func, test_case_name, __VA_ARGS__) + +#define BENCHMARK_TEMPLATE2_CAPTURE(func, a, b, test_case_name, ...) \ + BENCHMARK_PRIVATE_DECLARE(func) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + new ::benchmark::internal::FunctionBenchmark( \ + #func "<" #a "," #b ">" \ + "/" #test_case_name, \ + [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) +#endif // BENCHMARK_HAS_CXX11 + +#define BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ + class BaseClass##_##Method##_Benchmark : public BaseClass { \ + public: \ + BaseClass##_##Method##_Benchmark() { \ + this->SetName(#BaseClass "/" #Method); \ + } \ + \ + protected: \ + void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ }; -#define BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ - class BaseClass##_##Method##_Benchmark : public BaseClass { \ - public: \ - BaseClass##_##Method##_Benchmark() { \ - this->SetName(#BaseClass "<" #a ">/" #Method); \ - } \ - \ - protected: \ - virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ +#define BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ + class BaseClass##_##Method##_Benchmark : public BaseClass { \ + public: \ + BaseClass##_##Method##_Benchmark() { \ + this->SetName(#BaseClass "<" #a ">/" #Method); \ + } \ + \ + protected: \ + void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ }; -#define BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ - class BaseClass##_##Method##_Benchmark : public BaseClass { \ - public: \ - BaseClass##_##Method##_Benchmark() { \ - this->SetName(#BaseClass "<" #a "," #b ">/" #Method); \ - } \ - \ - protected: \ - virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ +#define BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ + class BaseClass##_##Method##_Benchmark : public BaseClass { \ + public: \ + BaseClass##_##Method##_Benchmark() { \ + this->SetName(#BaseClass "<" #a "," #b ">/" #Method); \ + } \ + \ + protected: \ + void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ }; #ifdef BENCHMARK_HAS_CXX11 @@ -1377,7 +1627,7 @@ class Fixture : public internal::Benchmark { } \ \ protected: \ - virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ + void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ }; #else #define BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(n, a) \ @@ -1439,8 +1689,15 @@ class Fixture : public internal::Benchmark { #endif // Helper macro to create a main routine in a test that runs the benchmarks +// Note the workaround for Hexagon simulator passing argc != 0, argv = NULL. #define BENCHMARK_MAIN() \ int main(int argc, char** argv) { \ + char arg0_default[] = "benchmark"; \ + char* args_default = arg0_default; \ + if (!argv) { \ + argc = 1; \ + argv = &args_default; \ + } \ ::benchmark::Initialize(&argc, argv); \ if (::benchmark::ReportUnrecognizedArguments(argc, argv)) return 1; \ ::benchmark::RunSpecifiedBenchmarks(); \ @@ -1454,7 +1711,7 @@ class Fixture : public internal::Benchmark { namespace benchmark { -struct CPUInfo { +struct BENCHMARK_EXPORT CPUInfo { struct CacheInfo { std::string type; int level; @@ -1478,7 +1735,7 @@ struct CPUInfo { }; // Adding Struct for System Information -struct SystemInfo { +struct BENCHMARK_EXPORT SystemInfo { std::string name; static const SystemInfo& Get(); @@ -1490,10 +1747,11 @@ struct SystemInfo { // BenchmarkName contains the components of the Benchmark's name // which allows individual fields to be modified or cleared before // building the final name using 'str()'. -struct BenchmarkName { +struct BENCHMARK_EXPORT BenchmarkName { std::string function_name; std::string args; std::string min_time; + std::string min_warmup_time; std::string iterations; std::string repetitions; std::string time_type; @@ -1509,7 +1767,7 @@ struct BenchmarkName { // can control the destination of the reports by calling // RunSpecifiedBenchmarks and passing it a custom reporter object. // The reporter object must implement the following interface. -class BenchmarkReporter { +class BENCHMARK_EXPORT BenchmarkReporter { public: struct Context { CPUInfo const& cpu_info; @@ -1520,20 +1778,21 @@ class BenchmarkReporter { Context(); }; - struct Run { + struct BENCHMARK_EXPORT Run { static const int64_t no_repetition_index = -1; enum RunType { RT_Iteration, RT_Aggregate }; Run() : run_type(RT_Iteration), aggregate_unit(kTime), - error_occurred(false), + skipped(internal::NotSkipped), iterations(1), threads(1), - time_unit(kNanosecond), + time_unit(GetDefaultTimeUnit()), real_accumulated_time(0), cpu_accumulated_time(0), max_heapbytes_used(0), + use_real_time_for_initial_big_o(false), complexity(oNone), complexity_lambda(), complexity_n(0), @@ -1550,8 +1809,8 @@ class BenchmarkReporter { std::string aggregate_name; StatisticUnit aggregate_unit; std::string report_label; // Empty if not set by benchmark. - bool error_occurred; - std::string error_message; + internal::Skipped skipped; + std::string skip_message; IterationCount iterations; int64_t threads; @@ -1576,10 +1835,14 @@ class BenchmarkReporter { // This is set to 0.0 if memory tracing is not enabled. double max_heapbytes_used; + // By default Big-O is computed for CPU time, but that is not what you want + // to happen when manual time was requested, which is stored as real time. + bool use_real_time_for_initial_big_o; + // Keep track of arguments to compute asymptotic complexity BigO complexity; BigOFunc* complexity_lambda; - int64_t complexity_n; + ComplexityN complexity_n; // what statistics to compute from the measurements const std::vector* statistics; @@ -1620,6 +1883,12 @@ class BenchmarkReporter { // to skip runs based on the context information. virtual bool ReportContext(const Context& context) = 0; + // Called once for each group of benchmark runs, gives information about + // the configurations of the runs. + virtual void ReportRunsConfig(double /*min_time*/, + bool /*has_explicit_iters*/, + IterationCount /*iters*/) {} + // Called once for each group of benchmark runs, gives information about // cpu-time and heap memory usage during the benchmark run. If the group // of runs contained more than two entries then 'report' contains additional @@ -1665,7 +1934,7 @@ class BenchmarkReporter { // Simple reporter that outputs benchmark data to the console. This is the // default reporter used by RunSpecifiedBenchmarks(). -class ConsoleReporter : public BenchmarkReporter { +class BENCHMARK_EXPORT ConsoleReporter : public BenchmarkReporter { public: enum OutputOptions { OO_None = 0, @@ -1677,8 +1946,8 @@ class ConsoleReporter : public BenchmarkReporter { explicit ConsoleReporter(OutputOptions opts_ = OO_Defaults) : output_options_(opts_), name_field_width_(0), printed_header_(false) {} - virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; - virtual void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; + bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; + void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; protected: virtual void PrintRunData(const Run& report); @@ -1690,12 +1959,12 @@ class ConsoleReporter : public BenchmarkReporter { bool printed_header_; }; -class JSONReporter : public BenchmarkReporter { +class BENCHMARK_EXPORT JSONReporter : public BenchmarkReporter { public: JSONReporter() : first_report_(true) {} - virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; - virtual void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; - virtual void Finalize() BENCHMARK_OVERRIDE; + bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; + void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; + void Finalize() BENCHMARK_OVERRIDE; private: void PrintRunData(const Run& report); @@ -1703,13 +1972,13 @@ class JSONReporter : public BenchmarkReporter { bool first_report_; }; -class BENCHMARK_DEPRECATED_MSG( +class BENCHMARK_EXPORT BENCHMARK_DEPRECATED_MSG( "The CSV Reporter will be removed in a future release") CSVReporter : public BenchmarkReporter { public: CSVReporter() : printed_header_(false) {} - virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; - virtual void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; + bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; + void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; private: void PrintRunData(const Run& report); @@ -1748,18 +2017,24 @@ inline double GetTimeUnitMultiplier(TimeUnit unit) { // Creates a list of integer values for the given range and multiplier. // This can be used together with ArgsProduct() to allow multiple ranges -// with different multiplers. +// with different multipliers. // Example: // ArgsProduct({ // CreateRange(0, 1024, /*multi=*/32), // CreateRange(0, 100, /*multi=*/4), // CreateDenseRange(0, 4, /*step=*/1), // }); +BENCHMARK_EXPORT std::vector CreateRange(int64_t lo, int64_t hi, int multi); // Creates a list of integer values for the given range and step. +BENCHMARK_EXPORT std::vector CreateDenseRange(int64_t start, int64_t limit, int step); } // namespace benchmark +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + #endif // BENCHMARK_BENCHMARK_H_ diff --git a/third-party/benchmark/requirements.txt b/third-party/benchmark/requirements.txt deleted file mode 100644 index e451894e2356..000000000000 --- a/third-party/benchmark/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -numpy == 1.19.4 -scipy == 1.5.4 -pandas == 1.1.5 diff --git a/third-party/benchmark/setup.py b/third-party/benchmark/setup.py index 83069e566839..cb20042da512 100644 --- a/third-party/benchmark/setup.py +++ b/third-party/benchmark/setup.py @@ -1,56 +1,50 @@ +import contextlib import os -import posixpath import platform -import re import shutil -import sys +import sysconfig +from pathlib import Path +from typing import Generator -from distutils import sysconfig import setuptools from setuptools.command import build_ext +PYTHON_INCLUDE_PATH_PLACEHOLDER = "" -HERE = os.path.dirname(os.path.abspath(__file__)) +IS_WINDOWS = platform.system() == "Windows" +IS_MAC = platform.system() == "Darwin" -IS_WINDOWS = sys.platform.startswith("win") - - -def _get_version(): - """Parse the version string from __init__.py.""" - with open( - os.path.join(HERE, "bindings", "python", "google_benchmark", "__init__.py") - ) as init_file: +@contextlib.contextmanager +def temp_fill_include_path(fp: str) -> Generator[None, None, None]: + """Temporarily set the Python include path in a file.""" + with open(fp, "r+") as f: try: - version_line = next( - line for line in init_file if line.startswith("__version__") + content = f.read() + replaced = content.replace( + PYTHON_INCLUDE_PATH_PLACEHOLDER, + Path(sysconfig.get_paths()["include"]).as_posix(), ) - except StopIteration: - raise ValueError("__version__ not defined in __init__.py") - else: - namespace = {} - exec(version_line, namespace) # pylint: disable=exec-used - return namespace["__version__"] - - -def _parse_requirements(path): - with open(os.path.join(HERE, path)) as requirements: - return [ - line.rstrip() - for line in requirements - if not (line.isspace() or line.startswith("#")) - ] + f.seek(0) + f.write(replaced) + f.truncate() + yield + finally: + # revert to the original content after exit + f.seek(0) + f.write(content) + f.truncate() class BazelExtension(setuptools.Extension): """A C/C++ extension that is defined as a Bazel BUILD target.""" - def __init__(self, name, bazel_target): + def __init__(self, name: str, bazel_target: str): + super().__init__(name=name, sources=[]) + self.bazel_target = bazel_target - self.relpath, self.target_name = posixpath.relpath(bazel_target, "//").split( - ":" - ) - setuptools.Extension.__init__(self, name, sources=[]) + stripped_target = bazel_target.split("//")[-1] + self.relpath, self.target_name = stripped_target.split(":") class BuildBazelExtension(build_ext.build_ext): @@ -59,88 +53,71 @@ class BuildBazelExtension(build_ext.build_ext): def run(self): for ext in self.extensions: self.bazel_build(ext) - build_ext.build_ext.run(self) - - def bazel_build(self, ext): + super().run() + # explicitly call `bazel shutdown` for graceful exit + self.spawn(["bazel", "shutdown"]) + + def copy_extensions_to_source(self): + """ + Copy generated extensions into the source tree. + This is done in the ``bazel_build`` method, so it's not necessary to + do again in the `build_ext` base class. + """ + pass + + def bazel_build(self, ext: BazelExtension) -> None: """Runs the bazel build to create the package.""" - with open("WORKSPACE", "r") as workspace: - workspace_contents = workspace.read() - - with open("WORKSPACE", "w") as workspace: - workspace.write( - re.sub( - r'(?<=path = ").*(?=", # May be overwritten by setup\.py\.)', - sysconfig.get_python_inc().replace(os.path.sep, posixpath.sep), - workspace_contents, - ) + with temp_fill_include_path("WORKSPACE"): + temp_path = Path(self.build_temp) + + bazel_argv = [ + "bazel", + "build", + ext.bazel_target, + "--enable_bzlmod=false", + f"--symlink_prefix={temp_path / 'bazel-'}", + f"--compilation_mode={'dbg' if self.debug else 'opt'}", + # C++17 is required by nanobind + f"--cxxopt={'/std:c++17' if IS_WINDOWS else '-std=c++17'}", + ] + + if IS_WINDOWS: + # Link with python*.lib. + for library_dir in self.library_dirs: + bazel_argv.append("--linkopt=/LIBPATH:" + library_dir) + elif IS_MAC: + if platform.machine() == "x86_64": + # C++17 needs macOS 10.14 at minimum + bazel_argv.append("--macos_minimum_os=10.14") + + # cross-compilation for Mac ARM64 on GitHub Mac x86 runners. + # ARCHFLAGS is set by cibuildwheel before macOS wheel builds. + archflags = os.getenv("ARCHFLAGS", "") + if "arm64" in archflags: + bazel_argv.append("--cpu=darwin_arm64") + bazel_argv.append("--macos_cpus=arm64") + + elif platform.machine() == "arm64": + bazel_argv.append("--macos_minimum_os=11.0") + + self.spawn(bazel_argv) + + shared_lib_suffix = ".dll" if IS_WINDOWS else ".so" + ext_name = ext.target_name + shared_lib_suffix + ext_bazel_bin_path = ( + temp_path / "bazel-bin" / ext.relpath / ext_name ) - if not os.path.exists(self.build_temp): - os.makedirs(self.build_temp) - - bazel_argv = [ - "bazel", - "build", - ext.bazel_target, - "--symlink_prefix=" + os.path.join(self.build_temp, "bazel-"), - "--compilation_mode=" + ("dbg" if self.debug else "opt"), - ] - - if IS_WINDOWS: - # Link with python*.lib. - for library_dir in self.library_dirs: - bazel_argv.append("--linkopt=/LIBPATH:" + library_dir) - elif sys.platform == "darwin" and platform.machine() == "x86_64": - bazel_argv.append("--macos_minimum_os=10.9") - - self.spawn(bazel_argv) - - shared_lib_suffix = ".dll" if IS_WINDOWS else ".so" - ext_bazel_bin_path = os.path.join( - self.build_temp, - "bazel-bin", - ext.relpath, - ext.target_name + shared_lib_suffix, - ) - - ext_dest_path = self.get_ext_fullpath(ext.name) - ext_dest_dir = os.path.dirname(ext_dest_path) - if not os.path.exists(ext_dest_dir): - os.makedirs(ext_dest_dir) - shutil.copyfile(ext_bazel_bin_path, ext_dest_path) + ext_dest_path = Path(self.get_ext_fullpath(ext.name)) + shutil.copyfile(ext_bazel_bin_path, ext_dest_path) setuptools.setup( - name="google_benchmark", - version=_get_version(), - url="https://github.com/google/benchmark", - description="A library to benchmark code snippets.", - author="Google", - author_email="benchmark-py@google.com", - # Contained modules and scripts. - package_dir={"": "bindings/python"}, - packages=setuptools.find_packages("bindings/python"), - install_requires=_parse_requirements("bindings/python/requirements.txt"), cmdclass=dict(build_ext=BuildBazelExtension), ext_modules=[ BazelExtension( - "google_benchmark._benchmark", - "//bindings/python/google_benchmark:_benchmark", + name="google_benchmark._benchmark", + bazel_target="//bindings/python/google_benchmark:_benchmark", ) ], - zip_safe=False, - # PyPI package information. - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Topic :: Software Development :: Testing", - "Topic :: System :: Benchmark", - ], - license="Apache 2.0", - keywords="benchmark", ) diff --git a/third-party/benchmark/src/CMakeLists.txt b/third-party/benchmark/src/CMakeLists.txt index e814a4e00f7c..943594b70bcd 100644 --- a/third-party/benchmark/src/CMakeLists.txt +++ b/third-party/benchmark/src/CMakeLists.txt @@ -25,12 +25,25 @@ set_target_properties(benchmark PROPERTIES SOVERSION ${GENERIC_LIB_SOVERSION} ) target_include_directories(benchmark PUBLIC - $) + $ +) + +set_property( + SOURCE benchmark.cc + APPEND + PROPERTY COMPILE_DEFINITIONS + BENCHMARK_VERSION="${VERSION}" +) # libpfm, if available -if (HAVE_LIBPFM) - target_link_libraries(benchmark PRIVATE pfm) - add_definitions(-DHAVE_LIBPFM) +if (PFM_FOUND) + target_link_libraries(benchmark PRIVATE PFM::libpfm) + target_compile_definitions(benchmark PRIVATE -DHAVE_LIBPFM) +endif() + +# pthread affinity, if available +if(HAVE_PTHREAD_AFFINITY) + target_compile_definitions(benchmark PRIVATE -DBENCHMARK_HAS_PTHREAD_AFFINITY) endif() # Link threads. @@ -53,6 +66,10 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "SunOS") target_link_libraries(benchmark PRIVATE kstat) endif() +if (NOT BUILD_SHARED_LIBS) + target_compile_definitions(benchmark PUBLIC -DBENCHMARK_STATIC_DEFINE) +endif() + # Benchmark main library add_library(benchmark_main "benchmark_main.cc") add_library(benchmark::benchmark_main ALIAS benchmark_main) @@ -60,10 +77,10 @@ set_target_properties(benchmark_main PROPERTIES OUTPUT_NAME "benchmark_main" VERSION ${GENERIC_LIB_VERSION} SOVERSION ${GENERIC_LIB_SOVERSION} + DEFINE_SYMBOL benchmark_EXPORTS ) target_link_libraries(benchmark_main PUBLIC benchmark::benchmark) - set(generated_dir "${PROJECT_BINARY_DIR}") set(version_config "${generated_dir}/${PROJECT_NAME}ConfigVersion.cmake") @@ -107,6 +124,7 @@ if (BENCHMARK_ENABLE_INSTALL) install( DIRECTORY "${PROJECT_SOURCE_DIR}/include/benchmark" + "${PROJECT_BINARY_DIR}/include/benchmark" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} FILES_MATCHING PATTERN "*.*h") diff --git a/third-party/benchmark/src/benchmark.cc b/third-party/benchmark/src/benchmark.cc index 473151136202..495944db29ff 100644 --- a/third-party/benchmark/src/benchmark.cc +++ b/third-party/benchmark/src/benchmark.cc @@ -19,7 +19,7 @@ #include "internal_macros.h" #ifndef BENCHMARK_OS_WINDOWS -#ifndef BENCHMARK_OS_FUCHSIA +#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) #include #endif #include @@ -65,12 +65,28 @@ BM_DEFINE_bool(benchmark_list_tests, false); // linked into the binary are run. BM_DEFINE_string(benchmark_filter, ""); -// Minimum number of seconds we should run benchmark before results are -// considered significant. For cpu-time based tests, this is the lower bound +// Specification of how long to run the benchmark. +// +// It can be either an exact number of iterations (specified as `x`), +// or a minimum number of seconds (specified as `s`). If the latter +// format (ie., min seconds) is used, the system may run the benchmark longer +// until the results are considered significant. +// +// For backward compatibility, the `s` suffix may be omitted, in which case, +// the specified number is interpreted as the number of seconds. +// +// For cpu-time based tests, this is the lower bound // on the total cpu time used by all threads that make up the test. For // real-time based tests, this is the lower bound on the elapsed time of the // benchmark execution, regardless of number of threads. -BM_DEFINE_double(benchmark_min_time, 0.5); +BM_DEFINE_string(benchmark_min_time, kDefaultMinTimeStr); + +// Minimum number of seconds a benchmark should be run before results should be +// taken into account. This e.g can be necessary for benchmarks of code which +// needs to fill some form of cache before performance is of interest. +// Note: results gathered within this period are discarded and not used for +// reported result. +BM_DEFINE_double(benchmark_min_warmup_time, 0.0); // The number of runs of each benchmark. If greater than 1, the mean and // standard deviation of the runs will be reported. @@ -121,6 +137,10 @@ BM_DEFINE_string(benchmark_perf_counters, ""); // pairs. Kept internal as it's only used for parsing from env/command line. BM_DEFINE_kvpairs(benchmark_context, {}); +// Set the default time unit to use for reports +// Valid values are 'ns', 'us', 'ms' or 's' +BM_DEFINE_string(benchmark_time_unit, ""); + // The level of verbose logging to output BM_DEFINE_int32(v, 0); @@ -128,23 +148,28 @@ namespace internal { std::map* global_context = nullptr; +BENCHMARK_EXPORT std::map*& GetGlobalContext() { + return global_context; +} + // FIXME: wouldn't LTO mess this up? void UseCharPointer(char const volatile*) {} } // namespace internal -State::State(IterationCount max_iters, const std::vector& ranges, - int thread_i, int n_threads, internal::ThreadTimer* timer, - internal::ThreadManager* manager, +State::State(std::string name, IterationCount max_iters, + const std::vector& ranges, int thread_i, int n_threads, + internal::ThreadTimer* timer, internal::ThreadManager* manager, internal::PerfCountersMeasurement* perf_counters_measurement) : total_iterations_(0), batch_leftover_(0), max_iterations(max_iters), started_(false), finished_(false), - error_occurred_(false), + skipped_(internal::NotSkipped), range_(ranges), complexity_n_(0), + name_(std::move(name)), thread_index_(thread_i), threads_(n_threads), timer_(timer), @@ -154,6 +179,17 @@ State::State(IterationCount max_iters, const std::vector& ranges, BM_CHECK_LT(thread_index_, threads_) << "thread_index must be less than threads"; + // Add counters with correct flag now. If added with `counters[name]` in + // `PauseTiming`, a new `Counter` will be inserted the first time, which + // won't have the flag. Inserting them now also reduces the allocations + // during the benchmark. + if (perf_counters_measurement_) { + for (const std::string& counter_name : + perf_counters_measurement_->names()) { + counters[counter_name] = Counter(0.0, Counter::kAvgIterations); + } + } + // Note: The use of offsetof below is technically undefined until C++17 // because State is not a standard layout type. However, all compilers // currently provide well-defined behavior as an extension (which is @@ -169,12 +205,19 @@ State::State(IterationCount max_iters, const std::vector& ranges, #elif defined(__clang__) #pragma clang diagnostic push #pragma clang diagnostic ignored "-Winvalid-offsetof" +#endif +#if defined(__NVCC__) +#pragma nv_diagnostic push +#pragma nv_diag_suppress 1427 +#endif +#if defined(__NVCOMPILER) +#pragma diagnostic push +#pragma diag_suppress offset_in_non_POD_nonstandard #endif // Offset tests to ensure commonly accessed data is on the first cache line. const int cache_line_size = 64; - static_assert(offsetof(State, error_occurred_) <= - (cache_line_size - sizeof(error_occurred_)), - ""); + static_assert( + offsetof(State, skipped_) <= (cache_line_size - sizeof(skipped_)), ""); #if defined(__INTEL_COMPILER) #pragma warning pop #elif defined(__GNUC__) @@ -182,39 +225,61 @@ State::State(IterationCount max_iters, const std::vector& ranges, #elif defined(__clang__) #pragma clang diagnostic pop #endif +#if defined(__NVCC__) +#pragma nv_diagnostic pop +#endif +#if defined(__NVCOMPILER) +#pragma diagnostic pop +#endif } void State::PauseTiming() { // Add in time accumulated so far - BM_CHECK(started_ && !finished_ && !error_occurred_); + BM_CHECK(started_ && !finished_ && !skipped()); timer_->StopTimer(); if (perf_counters_measurement_) { - auto measurements = perf_counters_measurement_->StopAndGetMeasurements(); + std::vector> measurements; + if (!perf_counters_measurement_->Stop(measurements)) { + BM_CHECK(false) << "Perf counters read the value failed."; + } for (const auto& name_and_measurement : measurements) { - auto name = name_and_measurement.first; - auto measurement = name_and_measurement.second; - BM_CHECK_EQ(counters[name], 0.0); - counters[name] = Counter(measurement, Counter::kAvgIterations); + const std::string& name = name_and_measurement.first; + const double measurement = name_and_measurement.second; + // Counter was inserted with `kAvgIterations` flag by the constructor. + assert(counters.find(name) != counters.end()); + counters[name].value += measurement; } } } void State::ResumeTiming() { - BM_CHECK(started_ && !finished_ && !error_occurred_); + BM_CHECK(started_ && !finished_ && !skipped()); timer_->StartTimer(); if (perf_counters_measurement_) { perf_counters_measurement_->Start(); } } -void State::SkipWithError(const char* msg) { - BM_CHECK(msg); - error_occurred_ = true; +void State::SkipWithMessage(const std::string& msg) { + skipped_ = internal::SkippedWithMessage; + { + MutexLock l(manager_->GetBenchmarkMutex()); + if (internal::NotSkipped == manager_->results.skipped_) { + manager_->results.skip_message_ = msg; + manager_->results.skipped_ = skipped_; + } + } + total_iterations_ = 0; + if (timer_->running()) timer_->StopTimer(); +} + +void State::SkipWithError(const std::string& msg) { + skipped_ = internal::SkippedWithError; { MutexLock l(manager_->GetBenchmarkMutex()); - if (manager_->results.has_error_ == false) { - manager_->results.error_message_ = msg; - manager_->results.has_error_ = true; + if (internal::NotSkipped == manager_->results.skipped_) { + manager_->results.skip_message_ = msg; + manager_->results.skipped_ = skipped_; } } total_iterations_ = 0; @@ -225,7 +290,7 @@ void State::SetIterationTime(double seconds) { timer_->SetIterationTime(seconds); } -void State::SetLabel(const char* label) { +void State::SetLabel(const std::string& label) { MutexLock l(manager_->GetBenchmarkMutex()); manager_->results.report_label_ = label; } @@ -233,14 +298,14 @@ void State::SetLabel(const char* label) { void State::StartKeepRunning() { BM_CHECK(!started_ && !finished_); started_ = true; - total_iterations_ = error_occurred_ ? 0 : max_iterations; + total_iterations_ = skipped() ? 0 : max_iterations; manager_->StartStopBarrier(); - if (!error_occurred_) ResumeTiming(); + if (!skipped()) ResumeTiming(); } void State::FinishKeepRunning() { - BM_CHECK(started_ && (!finished_ || error_occurred_)); - if (!error_occurred_) { + BM_CHECK(started_ && (!finished_ || skipped())); + if (!skipped()) { PauseTiming(); } // Total iterations has now wrapped around past 0. Fix this. @@ -318,14 +383,26 @@ void RunBenchmarks(const std::vector& benchmarks, size_t num_repetitions_total = 0; + // This perfcounters object needs to be created before the runners vector + // below so it outlasts their lifetime. + PerfCountersMeasurement perfcounters( + StrSplit(FLAGS_benchmark_perf_counters, ',')); + + // Vector of benchmarks to run std::vector runners; runners.reserve(benchmarks.size()); + + // Count the number of benchmarks with threads to warn the user in case + // performance counters are used. + int benchmarks_with_threads = 0; + + // Loop through all benchmarks for (const BenchmarkInstance& benchmark : benchmarks) { BenchmarkReporter::PerFamilyRunReports* reports_for_family = nullptr; if (benchmark.complexity() != oNone) reports_for_family = &per_family_reports[benchmark.family_index()]; - - runners.emplace_back(benchmark, reports_for_family); + benchmarks_with_threads += (benchmark.threads() > 1); + runners.emplace_back(benchmark, &perfcounters, reports_for_family); int num_repeats_of_this_instance = runners.back().GetNumRepeats(); num_repetitions_total += num_repeats_of_this_instance; if (reports_for_family) @@ -333,6 +410,17 @@ void RunBenchmarks(const std::vector& benchmarks, } assert(runners.size() == benchmarks.size() && "Unexpected runner count."); + // The use of performance counters with threads would be unintuitive for + // the average user so we need to warn them about this case + if ((benchmarks_with_threads > 0) && (perfcounters.num_counters() > 0)) { + GetErrorLogInstance() + << "***WARNING*** There are " << benchmarks_with_threads + << " benchmarks with threads and " << perfcounters.num_counters() + << " performance counters were requested. Beware counters will " + "reflect the combined usage across all " + "threads.\n"; + } + std::vector repetition_indices; repetition_indices.reserve(num_repetitions_total); for (size_t runner_index = 0, num_runners = runners.size(); @@ -356,6 +444,12 @@ void RunBenchmarks(const std::vector& benchmarks, if (runner.HasRepeatsRemaining()) continue; // FIXME: report each repetition separately, not all of them in bulk. + display_reporter->ReportRunsConfig( + runner.GetMinTime(), runner.HasExplicitIters(), runner.GetIters()); + if (file_reporter) + file_reporter->ReportRunsConfig( + runner.GetMinTime(), runner.HasExplicitIters(), runner.GetIters()); + RunResults run_results = runner.GetResults(); // Maybe calculate complexity report @@ -389,14 +483,15 @@ std::unique_ptr CreateReporter( typedef std::unique_ptr PtrType; if (name == "console") { return PtrType(new ConsoleReporter(output_opts)); - } else if (name == "json") { - return PtrType(new JSONReporter); - } else if (name == "csv") { - return PtrType(new CSVReporter); - } else { - std::cerr << "Unexpected format: '" << name << "'\n"; - std::exit(1); } + if (name == "json") { + return PtrType(new JSONReporter()); + } + if (name == "csv") { + return PtrType(new CSVReporter()); + } + std::cerr << "Unexpected format: '" << name << "'\n"; + std::exit(1); } BENCHMARK_RESTORE_DEPRECATED_WARNING @@ -433,6 +528,14 @@ ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color) { } // end namespace internal +BenchmarkReporter* CreateDefaultDisplayReporter() { + static auto default_display_reporter = + internal::CreateReporter(FLAGS_benchmark_format, + internal::GetOutputOptions()) + .release(); + return default_display_reporter; +} + size_t RunSpecifiedBenchmarks() { return RunSpecifiedBenchmarks(nullptr, nullptr, FLAGS_benchmark_filter); } @@ -468,8 +571,7 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, std::unique_ptr default_display_reporter; std::unique_ptr default_file_reporter; if (!display_reporter) { - default_display_reporter = internal::CreateReporter( - FLAGS_benchmark_format, internal::GetOutputOptions()); + default_display_reporter.reset(CreateDefaultDisplayReporter()); display_reporter = default_display_reporter.get(); } auto& Out = display_reporter->GetOutputStream(); @@ -480,17 +582,23 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, Err << "A custom file reporter was provided but " "--benchmark_out= was not specified." << std::endl; + Out.flush(); + Err.flush(); std::exit(1); } if (!fname.empty()) { output_file.open(fname); if (!output_file.is_open()) { Err << "invalid file name: '" << fname << "'" << std::endl; + Out.flush(); + Err.flush(); std::exit(1); } if (!file_reporter) { default_file_reporter = internal::CreateReporter( - FLAGS_benchmark_out_format, ConsoleReporter::OO_None); + FLAGS_benchmark_out_format, FLAGS_benchmark_counters_tabular + ? ConsoleReporter::OO_Tabular + : ConsoleReporter::OO_None); file_reporter = default_file_reporter.get(); } file_reporter->SetOutputStream(&output_file); @@ -498,10 +606,16 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, } std::vector benchmarks; - if (!FindBenchmarksInternal(spec, &benchmarks, &Err)) return 0; + if (!FindBenchmarksInternal(spec, &benchmarks, &Err)) { + Out.flush(); + Err.flush(); + return 0; + } if (benchmarks.empty()) { Err << "Failed to match any benchmarks against regex: " << spec << "\n"; + Out.flush(); + Err.flush(); return 0; } @@ -512,11 +626,28 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, internal::RunBenchmarks(benchmarks, display_reporter, file_reporter); } + Out.flush(); + Err.flush(); return benchmarks.size(); } +namespace { +// stores the time unit benchmarks use by default +TimeUnit default_time_unit = kNanosecond; +} // namespace + +TimeUnit GetDefaultTimeUnit() { return default_time_unit; } + +void SetDefaultTimeUnit(TimeUnit unit) { default_time_unit = unit; } + std::string GetBenchmarkFilter() { return FLAGS_benchmark_filter; } +void SetBenchmarkFilter(std::string value) { + FLAGS_benchmark_filter = std::move(value); +} + +int32_t GetBenchmarkVerbosity() { return FLAGS_v; } + void RegisterMemoryManager(MemoryManager* manager) { internal::memory_manager = manager; } @@ -533,27 +664,31 @@ void AddCustomContext(const std::string& key, const std::string& value) { namespace internal { +void (*HelperPrintf)(); + void PrintUsageAndExit() { - fprintf(stdout, - "benchmark" - " [--benchmark_list_tests={true|false}]\n" - " [--benchmark_filter=]\n" - " [--benchmark_min_time=]\n" - " [--benchmark_repetitions=]\n" - " [--benchmark_enable_random_interleaving={true|false}]\n" - " [--benchmark_report_aggregates_only={true|false}]\n" - " [--benchmark_display_aggregates_only={true|false}]\n" - " [--benchmark_format=]\n" - " [--benchmark_out=]\n" - " [--benchmark_out_format=]\n" - " [--benchmark_color={auto|true|false}]\n" - " [--benchmark_counters_tabular={true|false}]\n" - " [--benchmark_perf_counters=,...]\n" - " [--benchmark_context==,...]\n" - " [--v=]\n"); + HelperPrintf(); exit(0); } +void SetDefaultTimeUnitFromFlag(const std::string& time_unit_flag) { + if (time_unit_flag == "s") { + return SetDefaultTimeUnit(kSecond); + } + if (time_unit_flag == "ms") { + return SetDefaultTimeUnit(kMillisecond); + } + if (time_unit_flag == "us") { + return SetDefaultTimeUnit(kMicrosecond); + } + if (time_unit_flag == "ns") { + return SetDefaultTimeUnit(kNanosecond); + } + if (!time_unit_flag.empty()) { + PrintUsageAndExit(); + } +} + void ParseCommandLineFlags(int* argc, char** argv) { using namespace benchmark; BenchmarkReporter::Context::executable_name = @@ -562,8 +697,10 @@ void ParseCommandLineFlags(int* argc, char** argv) { if (ParseBoolFlag(argv[i], "benchmark_list_tests", &FLAGS_benchmark_list_tests) || ParseStringFlag(argv[i], "benchmark_filter", &FLAGS_benchmark_filter) || - ParseDoubleFlag(argv[i], "benchmark_min_time", + ParseStringFlag(argv[i], "benchmark_min_time", &FLAGS_benchmark_min_time) || + ParseDoubleFlag(argv[i], "benchmark_min_warmup_time", + &FLAGS_benchmark_min_warmup_time) || ParseInt32Flag(argv[i], "benchmark_repetitions", &FLAGS_benchmark_repetitions) || ParseBoolFlag(argv[i], "benchmark_enable_random_interleaving", @@ -583,6 +720,8 @@ void ParseCommandLineFlags(int* argc, char** argv) { &FLAGS_benchmark_perf_counters) || ParseKeyValueFlag(argv[i], "benchmark_context", &FLAGS_benchmark_context) || + ParseStringFlag(argv[i], "benchmark_time_unit", + &FLAGS_benchmark_time_unit) || ParseInt32Flag(argv[i], "v", &FLAGS_v)) { for (int j = i; j != *argc - 1; ++j) argv[j] = argv[j + 1]; @@ -598,6 +737,7 @@ void ParseCommandLineFlags(int* argc, char** argv) { PrintUsageAndExit(); } } + SetDefaultTimeUnitFromFlag(FLAGS_benchmark_time_unit); if (FLAGS_benchmark_color.empty()) { PrintUsageAndExit(); } @@ -613,7 +753,34 @@ int InitializeStreams() { } // end namespace internal -void Initialize(int* argc, char** argv) { +std::string GetBenchmarkVersion() { return {BENCHMARK_VERSION}; } + +void PrintDefaultHelp() { + fprintf(stdout, + "benchmark" + " [--benchmark_list_tests={true|false}]\n" + " [--benchmark_filter=]\n" + " [--benchmark_min_time=`x` OR `s` ]\n" + " [--benchmark_min_warmup_time=]\n" + " [--benchmark_repetitions=]\n" + " [--benchmark_enable_random_interleaving={true|false}]\n" + " [--benchmark_report_aggregates_only={true|false}]\n" + " [--benchmark_display_aggregates_only={true|false}]\n" + " [--benchmark_format=]\n" + " [--benchmark_out=]\n" + " [--benchmark_out_format=]\n" + " [--benchmark_color={auto|true|false}]\n" + " [--benchmark_counters_tabular={true|false}]\n" +#if defined HAVE_LIBPFM + " [--benchmark_perf_counters=,...]\n" +#endif + " [--benchmark_context==,...]\n" + " [--benchmark_time_unit={ns|us|ms|s}]\n" + " [--v=]\n"); +} + +void Initialize(int* argc, char** argv, void (*HelperPrintf)()) { + internal::HelperPrintf = HelperPrintf; internal::ParseCommandLineFlags(argc, argv); internal::LogLevel() = FLAGS_v; } diff --git a/third-party/benchmark/src/benchmark_api_internal.cc b/third-party/benchmark/src/benchmark_api_internal.cc index 4de36e3c8ba3..286f98653085 100644 --- a/third-party/benchmark/src/benchmark_api_internal.cc +++ b/third-party/benchmark/src/benchmark_api_internal.cc @@ -16,7 +16,7 @@ BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, int family_idx, per_family_instance_index_(per_family_instance_idx), aggregation_report_mode_(benchmark_.aggregation_report_mode_), args_(args), - time_unit_(benchmark_.time_unit_), + time_unit_(benchmark_.GetTimeUnit()), measure_process_cpu_time_(benchmark_.measure_process_cpu_time_), use_real_time_(benchmark_.use_real_time_), use_manual_time_(benchmark_.use_manual_time_), @@ -25,6 +25,7 @@ BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, int family_idx, statistics_(benchmark_.statistics_), repetitions_(benchmark_.repetitions_), min_time_(benchmark_.min_time_), + min_warmup_time_(benchmark_.min_warmup_time_), iterations_(benchmark_.iterations_), threads_(thread_count) { name_.function_name = benchmark_.name_; @@ -50,6 +51,11 @@ BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, int family_idx, name_.min_time = StrFormat("min_time:%0.3f", benchmark_.min_time_); } + if (!IsZero(benchmark->min_warmup_time_)) { + name_.min_warmup_time = + StrFormat("min_warmup_time:%0.3f", benchmark_.min_warmup_time_); + } + if (benchmark_.iterations_ != 0) { name_.iterations = StrFormat( "iterations:%lu", static_cast(benchmark_.iterations_)); @@ -87,24 +93,24 @@ State BenchmarkInstance::Run( IterationCount iters, int thread_id, internal::ThreadTimer* timer, internal::ThreadManager* manager, internal::PerfCountersMeasurement* perf_counters_measurement) const { - State st(iters, args_, thread_id, threads_, timer, manager, - perf_counters_measurement); + State st(name_.function_name, iters, args_, thread_id, threads_, timer, + manager, perf_counters_measurement); benchmark_.Run(st); return st; } void BenchmarkInstance::Setup() const { if (setup_) { - State st(/*iters*/ 1, args_, /*thread_id*/ 0, threads_, nullptr, nullptr, - nullptr); + State st(name_.function_name, /*iters*/ 1, args_, /*thread_id*/ 0, threads_, + nullptr, nullptr, nullptr); setup_(st); } } void BenchmarkInstance::Teardown() const { if (teardown_) { - State st(/*iters*/ 1, args_, /*thread_id*/ 0, threads_, nullptr, nullptr, - nullptr); + State st(name_.function_name, /*iters*/ 1, args_, /*thread_id*/ 0, threads_, + nullptr, nullptr, nullptr); teardown_(st); } } diff --git a/third-party/benchmark/src/benchmark_api_internal.h b/third-party/benchmark/src/benchmark_api_internal.h index 94c2b2972bb1..94f516531bc4 100644 --- a/third-party/benchmark/src/benchmark_api_internal.h +++ b/third-party/benchmark/src/benchmark_api_internal.h @@ -36,6 +36,7 @@ class BenchmarkInstance { const std::vector& statistics() const { return statistics_; } int repetitions() const { return repetitions_; } double min_time() const { return min_time_; } + double min_warmup_time() const { return min_warmup_time_; } IterationCount iterations() const { return iterations_; } int threads() const { return threads_; } void Setup() const; @@ -62,6 +63,7 @@ class BenchmarkInstance { const std::vector& statistics_; int repetitions_; double min_time_; + double min_warmup_time_; IterationCount iterations_; int threads_; // Number of concurrent threads to us @@ -76,6 +78,7 @@ bool FindBenchmarksInternal(const std::string& re, bool IsZero(double n); +BENCHMARK_EXPORT ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color = false); } // end namespace internal diff --git a/third-party/benchmark/src/benchmark_main.cc b/third-party/benchmark/src/benchmark_main.cc index b3b247831496..cd61cd2ad506 100644 --- a/third-party/benchmark/src/benchmark_main.cc +++ b/third-party/benchmark/src/benchmark_main.cc @@ -14,4 +14,5 @@ #include "benchmark/benchmark.h" +BENCHMARK_EXPORT int main(int, char**); BENCHMARK_MAIN(); diff --git a/third-party/benchmark/src/benchmark_name.cc b/third-party/benchmark/src/benchmark_name.cc index 2a17ebce277f..01676bbc84df 100644 --- a/third-party/benchmark/src/benchmark_name.cc +++ b/third-party/benchmark/src/benchmark_name.cc @@ -51,8 +51,9 @@ std::string join(char delimiter, const Ts&... ts) { } } // namespace +BENCHMARK_EXPORT std::string BenchmarkName::str() const { - return join('/', function_name, args, min_time, iterations, repetitions, - time_type, threads); + return join('/', function_name, args, min_time, min_warmup_time, iterations, + repetitions, time_type, threads); } } // namespace benchmark diff --git a/third-party/benchmark/src/benchmark_register.cc b/third-party/benchmark/src/benchmark_register.cc index 61a0c26178e3..e447c9a2d39b 100644 --- a/third-party/benchmark/src/benchmark_register.cc +++ b/third-party/benchmark/src/benchmark_register.cc @@ -15,7 +15,7 @@ #include "benchmark_register.h" #ifndef BENCHMARK_OS_WINDOWS -#ifndef BENCHMARK_OS_FUCHSIA +#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) #include #endif #include @@ -53,10 +53,13 @@ namespace benchmark { namespace { // For non-dense Range, intermediate values are powers of kRangeMultiplier. -static const int kRangeMultiplier = 8; +static constexpr int kRangeMultiplier = 8; + // The size of a benchmark family determines is the number of inputs to repeat // the benchmark on. If this is "large" then warn the user during configuration. -static const size_t kMaxFamilySize = 100; +static constexpr size_t kMaxFamilySize = 100; + +static constexpr char kDisabledPrefix[] = "DISABLED_"; } // end namespace namespace internal { @@ -116,10 +119,10 @@ bool BenchmarkFamilies::FindBenchmarks( // Make regular expression out of command-line flag std::string error_msg; Regex re; - bool isNegativeFilter = false; + bool is_negative_filter = false; if (spec[0] == '-') { spec.replace(0, 1, ""); - isNegativeFilter = true; + is_negative_filter = true; } if (!re.Init(spec, &error_msg)) { Err << "Could not compile benchmark re: " << error_msg << std::endl; @@ -154,7 +157,8 @@ bool BenchmarkFamilies::FindBenchmarks( << " will be repeated at least " << family_size << " times.\n"; } // reserve in the special case the regex ".", since we know the final - // family size. + // family size. this doesn't take into account any disabled benchmarks + // so worst case we reserve more than we need. if (spec == ".") benchmarks->reserve(benchmarks->size() + family_size); for (auto const& args : family->args_) { @@ -164,8 +168,9 @@ bool BenchmarkFamilies::FindBenchmarks( num_threads); const auto full_name = instance.name().str(); - if ((re.Match(full_name) && !isNegativeFilter) || - (!re.Match(full_name) && isNegativeFilter)) { + if (full_name.rfind(kDisabledPrefix, 0) != 0 && + ((re.Match(full_name) && !is_negative_filter) || + (!re.Match(full_name) && is_negative_filter))) { benchmarks->push_back(std::move(instance)); ++per_family_instance_index; @@ -199,12 +204,14 @@ bool FindBenchmarksInternal(const std::string& re, // Benchmark //=============================================================================// -Benchmark::Benchmark(const char* name) +Benchmark::Benchmark(const std::string& name) : name_(name), aggregation_report_mode_(ARM_Unspecified), - time_unit_(kNanosecond), + time_unit_(GetDefaultTimeUnit()), + use_default_time_unit_(true), range_multiplier_(kRangeMultiplier), min_time_(0), + min_warmup_time_(0), iterations_(0), repetitions_(0), measure_process_cpu_time_(false), @@ -223,7 +230,7 @@ Benchmark::Benchmark(const char* name) Benchmark::~Benchmark() {} Benchmark* Benchmark::Name(const std::string& name) { - SetName(name.c_str()); + SetName(name); return this; } @@ -235,6 +242,7 @@ Benchmark* Benchmark::Arg(int64_t x) { Benchmark* Benchmark::Unit(TimeUnit unit) { time_unit_ = unit; + use_default_time_unit_ = false; return this; } @@ -348,9 +356,17 @@ Benchmark* Benchmark::MinTime(double t) { return this; } +Benchmark* Benchmark::MinWarmUpTime(double t) { + BM_CHECK(t >= 0.0); + BM_CHECK(iterations_ == 0); + min_warmup_time_ = t; + return this; +} + Benchmark* Benchmark::Iterations(IterationCount n) { BM_CHECK(n > 0); BM_CHECK(IsZero(min_time_)); + BM_CHECK(IsZero(min_warmup_time_)); iterations_ = n; return this; } @@ -452,7 +468,9 @@ Benchmark* Benchmark::ThreadPerCpu() { return this; } -void Benchmark::SetName(const char* name) { name_ = name; } +void Benchmark::SetName(const std::string& name) { name_ = name; } + +const char* Benchmark::GetName() const { return name_.c_str(); } int Benchmark::ArgsCnt() const { if (args_.empty()) { @@ -462,6 +480,16 @@ int Benchmark::ArgsCnt() const { return static_cast(args_.front().size()); } +const char* Benchmark::GetArgName(int arg) const { + BM_CHECK_GE(arg, 0); + BM_CHECK_LT(arg, static_cast(arg_names_.size())); + return arg_names_[arg].c_str(); +} + +TimeUnit Benchmark::GetTimeUnit() const { + return use_default_time_unit_ ? GetDefaultTimeUnit() : time_unit_; +} + //=============================================================================// // FunctionBenchmark //=============================================================================// diff --git a/third-party/benchmark/src/benchmark_register.h b/third-party/benchmark/src/benchmark_register.h index d3f4974e9074..53367c707cf4 100644 --- a/third-party/benchmark/src/benchmark_register.h +++ b/third-party/benchmark/src/benchmark_register.h @@ -1,6 +1,7 @@ #ifndef BENCHMARK_REGISTER_H #define BENCHMARK_REGISTER_H +#include #include #include @@ -23,7 +24,7 @@ typename std::vector::iterator AddPowers(std::vector* dst, T lo, T hi, static const T kmax = std::numeric_limits::max(); // Space out the values in multiples of "mult" - for (T i = static_cast(1); i <= hi; i *= mult) { + for (T i = static_cast(1); i <= hi; i *= static_cast(mult)) { if (i >= lo) { dst->push_back(i); } @@ -32,7 +33,7 @@ typename std::vector::iterator AddPowers(std::vector* dst, T lo, T hi, if (i > kmax / mult) break; } - return dst->begin() + start_offset; + return dst->begin() + static_cast(start_offset); } template diff --git a/third-party/benchmark/src/benchmark_runner.cc b/third-party/benchmark/src/benchmark_runner.cc index eac807b066f1..dcddb437e37d 100644 --- a/third-party/benchmark/src/benchmark_runner.cc +++ b/third-party/benchmark/src/benchmark_runner.cc @@ -19,7 +19,7 @@ #include "internal_macros.h" #ifndef BENCHMARK_OS_WINDOWS -#ifndef BENCHMARK_OS_FUCHSIA +#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) #include #endif #include @@ -28,11 +28,14 @@ #include #include +#include +#include #include #include #include #include #include +#include #include #include #include @@ -61,7 +64,9 @@ MemoryManager* memory_manager = nullptr; namespace { -static constexpr IterationCount kMaxIterations = 1000000000; +static constexpr IterationCount kMaxIterations = 1000000000000; +const double kDefaultMinTime = + std::strtod(::benchmark::kDefaultMinTimeStr, /*p_end*/ nullptr); BenchmarkReporter::Run CreateRunReport( const benchmark::internal::BenchmarkInstance& b, @@ -75,8 +80,8 @@ BenchmarkReporter::Run CreateRunReport( report.run_name = b.name(); report.family_index = b.family_index(); report.per_family_instance_index = b.per_family_instance_index(); - report.error_occurred = results.has_error_; - report.error_message = results.error_message_; + report.skipped = results.skipped_; + report.skip_message = results.skip_message_; report.report_label = results.report_label_; // This is the total iterations across all threads. report.iterations = results.iterations; @@ -85,12 +90,13 @@ BenchmarkReporter::Run CreateRunReport( report.repetition_index = repetition_index; report.repetitions = repeats; - if (!report.error_occurred) { + if (!report.skipped) { if (b.use_manual_time()) { report.real_accumulated_time = results.manual_time_used; } else { report.real_accumulated_time = results.real_time_used; } + report.use_real_time_for_initial_big_o = b.use_manual_time(); report.cpu_accumulated_time = results.cpu_time_used; report.complexity_n = results.complexity_n; report.complexity = b.complexity(); @@ -103,7 +109,7 @@ BenchmarkReporter::Run CreateRunReport( report.memory_result = memory_result; report.allocs_per_iter = memory_iterations ? static_cast(memory_result->num_allocs) / - memory_iterations + static_cast(memory_iterations) : 0; } @@ -122,9 +128,10 @@ void RunInThread(const BenchmarkInstance* b, IterationCount iters, b->measure_process_cpu_time() ? internal::ThreadTimer::CreateProcessCpuTime() : internal::ThreadTimer::Create()); + State st = b->Run(iters, thread_id, &timer, manager, perf_counters_measurement); - BM_CHECK(st.error_occurred() || st.iterations() >= st.max_iterations) + BM_CHECK(st.skipped() || st.iterations() >= st.max_iterations) << "Benchmark returned before State::KeepRunning() returned false!"; { MutexLock l(manager->GetBenchmarkMutex()); @@ -139,24 +146,100 @@ void RunInThread(const BenchmarkInstance* b, IterationCount iters, manager->NotifyThreadComplete(); } +double ComputeMinTime(const benchmark::internal::BenchmarkInstance& b, + const BenchTimeType& iters_or_time) { + if (!IsZero(b.min_time())) return b.min_time(); + // If the flag was used to specify number of iters, then return the default + // min_time. + if (iters_or_time.tag == BenchTimeType::ITERS) return kDefaultMinTime; + + return iters_or_time.time; +} + +IterationCount ComputeIters(const benchmark::internal::BenchmarkInstance& b, + const BenchTimeType& iters_or_time) { + if (b.iterations() != 0) return b.iterations(); + + // We've already concluded that this flag is currently used to pass + // iters but do a check here again anyway. + BM_CHECK(iters_or_time.tag == BenchTimeType::ITERS); + return iters_or_time.iters; +} + } // end namespace +BenchTimeType ParseBenchMinTime(const std::string& value) { + BenchTimeType ret; + + if (value.empty()) { + ret.tag = BenchTimeType::TIME; + ret.time = 0.0; + return ret; + } + + if (value.back() == 'x') { + char* p_end; + // Reset errno before it's changed by strtol. + errno = 0; + IterationCount num_iters = std::strtol(value.c_str(), &p_end, 10); + + // After a valid parse, p_end should have been set to + // point to the 'x' suffix. + BM_CHECK(errno == 0 && p_end != nullptr && *p_end == 'x') + << "Malformed iters value passed to --benchmark_min_time: `" << value + << "`. Expected --benchmark_min_time=x."; + + ret.tag = BenchTimeType::ITERS; + ret.iters = num_iters; + return ret; + } + + bool has_suffix = value.back() == 's'; + if (!has_suffix) { + BM_VLOG(0) << "Value passed to --benchmark_min_time should have a suffix. " + "Eg., `30s` for 30-seconds."; + } + + char* p_end; + // Reset errno before it's changed by strtod. + errno = 0; + double min_time = std::strtod(value.c_str(), &p_end); + + // After a successful parse, p_end should point to the suffix 's', + // or the end of the string if the suffix was omitted. + BM_CHECK(errno == 0 && p_end != nullptr && + ((has_suffix && *p_end == 's') || *p_end == '\0')) + << "Malformed seconds value passed to --benchmark_min_time: `" << value + << "`. Expected --benchmark_min_time=x."; + + ret.tag = BenchTimeType::TIME; + ret.time = min_time; + + return ret; +} + BenchmarkRunner::BenchmarkRunner( const benchmark::internal::BenchmarkInstance& b_, + PerfCountersMeasurement* pcm_, BenchmarkReporter::PerFamilyRunReports* reports_for_family_) : b(b_), reports_for_family(reports_for_family_), - min_time(!IsZero(b.min_time()) ? b.min_time() : FLAGS_benchmark_min_time), + parsed_benchtime_flag(ParseBenchMinTime(FLAGS_benchmark_min_time)), + min_time(ComputeMinTime(b_, parsed_benchtime_flag)), + min_warmup_time((!IsZero(b.min_time()) && b.min_warmup_time() > 0.0) + ? b.min_warmup_time() + : FLAGS_benchmark_min_warmup_time), + warmup_done(!(min_warmup_time > 0.0)), repeats(b.repetitions() != 0 ? b.repetitions() : FLAGS_benchmark_repetitions), - has_explicit_iteration_count(b.iterations() != 0), + has_explicit_iteration_count(b.iterations() != 0 || + parsed_benchtime_flag.tag == + BenchTimeType::ITERS), pool(b.threads() - 1), - iters(has_explicit_iteration_count ? b.iterations() : 1), - perf_counters_measurement( - PerfCounters::Create(StrSplit(FLAGS_benchmark_perf_counters, ','))), - perf_counters_measurement_ptr(perf_counters_measurement.IsValid() - ? &perf_counters_measurement - : nullptr) { + iters(has_explicit_iteration_count + ? ComputeIters(b_, parsed_benchtime_flag) + : 1), + perf_counters_measurement_ptr(pcm_) { run_results.display_report_aggregates_only = (FLAGS_benchmark_report_aggregates_only || FLAGS_benchmark_display_aggregates_only); @@ -169,7 +252,7 @@ BenchmarkRunner::BenchmarkRunner( run_results.file_report_aggregates_only = (b.aggregation_report_mode() & internal::ARM_FileReportAggregatesOnly); BM_CHECK(FLAGS_benchmark_perf_counters.empty() || - perf_counters_measurement.IsValid()) + (perf_counters_measurement_ptr->num_counters() == 0)) << "Perf counters were requested but could not be set up."; } } @@ -232,20 +315,20 @@ IterationCount BenchmarkRunner::PredictNumItersNeeded( const IterationResults& i) const { // See how much iterations should be increased by. // Note: Avoid division by zero with max(seconds, 1ns). - double multiplier = min_time * 1.4 / std::max(i.seconds, 1e-9); + double multiplier = GetMinTimeToApply() * 1.4 / std::max(i.seconds, 1e-9); // If our last run was at least 10% of FLAGS_benchmark_min_time then we // use the multiplier directly. // Otherwise we use at most 10 times expansion. // NOTE: When the last run was at least 10% of the min time the max // expansion should be 14x. - bool is_significant = (i.seconds / min_time) > 0.1; + const bool is_significant = (i.seconds / GetMinTimeToApply()) > 0.1; multiplier = is_significant ? multiplier : 10.0; // So what seems to be the sufficiently-large iteration count? Round up. const IterationCount max_next_iters = static_cast( - std::lround(std::max(multiplier * static_cast(i.iters), - static_cast(i.iters) + 1.0))); - // But we do have *some* sanity limits though.. + std::llround(std::max(multiplier * static_cast(i.iters), + static_cast(i.iters) + 1.0))); + // But we do have *some* limits though.. const IterationCount next_iters = std::min(max_next_iters, kMaxIterations); BM_VLOG(3) << "Next iters: " << next_iters << ", " << multiplier << "\n"; @@ -257,21 +340,80 @@ bool BenchmarkRunner::ShouldReportIterationResults( // Determine if this run should be reported; // Either it has run for a sufficient amount of time // or because an error was reported. - return i.results.has_error_ || + return i.results.skipped_ || i.iters >= kMaxIterations || // Too many iterations already. - i.seconds >= min_time || // The elapsed time is large enough. + i.seconds >= + GetMinTimeToApply() || // The elapsed time is large enough. // CPU time is specified but the elapsed real time greatly exceeds // the minimum time. - // Note that user provided timers are except from this sanity check. - ((i.results.real_time_used >= 5 * min_time) && !b.use_manual_time()); + // Note that user provided timers are except from this test. + ((i.results.real_time_used >= 5 * GetMinTimeToApply()) && + !b.use_manual_time()); +} + +double BenchmarkRunner::GetMinTimeToApply() const { + // In order to re-use functionality to run and measure benchmarks for running + // a warmup phase of the benchmark, we need a way of telling whether to apply + // min_time or min_warmup_time. This function will figure out if we are in the + // warmup phase and therefore need to apply min_warmup_time or if we already + // in the benchmarking phase and min_time needs to be applied. + return warmup_done ? min_time : min_warmup_time; +} + +void BenchmarkRunner::FinishWarmUp(const IterationCount& i) { + warmup_done = true; + iters = i; +} + +void BenchmarkRunner::RunWarmUp() { + // Use the same mechanisms for warming up the benchmark as used for actually + // running and measuring the benchmark. + IterationResults i_warmup; + // Dont use the iterations determined in the warmup phase for the actual + // measured benchmark phase. While this may be a good starting point for the + // benchmark and it would therefore get rid of the need to figure out how many + // iterations are needed if min_time is set again, this may also be a complete + // wrong guess since the warmup loops might be considerably slower (e.g + // because of caching effects). + const IterationCount i_backup = iters; + + for (;;) { + b.Setup(); + i_warmup = DoNIterations(); + b.Teardown(); + + const bool finish = ShouldReportIterationResults(i_warmup); + + if (finish) { + FinishWarmUp(i_backup); + break; + } + + // Although we are running "only" a warmup phase where running enough + // iterations at once without measuring time isn't as important as it is for + // the benchmarking phase, we still do it the same way as otherwise it is + // very confusing for the user to know how to choose a proper value for + // min_warmup_time if a different approach on running it is used. + iters = PredictNumItersNeeded(i_warmup); + assert(iters > i_warmup.iters && + "if we did more iterations than we want to do the next time, " + "then we should have accepted the current iteration run."); + } } void BenchmarkRunner::DoOneRepetition() { assert(HasRepeatsRemaining() && "Already done all repetitions?"); const bool is_the_first_repetition = num_repetitions_done == 0; - IterationResults i; + // In case a warmup phase is requested by the benchmark, run it now. + // After running the warmup phase the BenchmarkRunner should be in a state as + // this warmup never happened except the fact that warmup_done is set. Every + // other manipulation of the BenchmarkRunner instance would be a bug! Please + // fix it. + if (!warmup_done) RunWarmUp(); + + IterationResults i; // We *may* be gradually increasing the length (iteration count) // of the benchmark until we decide the results are significant. // And once we do, we report those last results and exit. @@ -324,10 +466,7 @@ void BenchmarkRunner::DoOneRepetition() { manager->WaitForAllThreads(); manager.reset(); b.Teardown(); - - BENCHMARK_DISABLE_DEPRECATED_WARNING - memory_manager->Stop(memory_result); - BENCHMARK_RESTORE_DEPRECATED_WARNING + memory_manager->Stop(*memory_result); } // Ok, now actually report. @@ -337,7 +476,7 @@ void BenchmarkRunner::DoOneRepetition() { if (reports_for_family) { ++reports_for_family->num_runs_done; - if (!report.error_occurred) reports_for_family->Runs.push_back(report); + if (!report.skipped) reports_for_family->Runs.push_back(report); } run_results.non_aggregates.push_back(report); diff --git a/third-party/benchmark/src/benchmark_runner.h b/third-party/benchmark/src/benchmark_runner.h index 752eefdc26fa..db2fa04396c5 100644 --- a/third-party/benchmark/src/benchmark_runner.h +++ b/third-party/benchmark/src/benchmark_runner.h @@ -25,7 +25,8 @@ namespace benchmark { -BM_DECLARE_double(benchmark_min_time); +BM_DECLARE_string(benchmark_min_time); +BM_DECLARE_double(benchmark_min_warmup_time); BM_DECLARE_int32(benchmark_repetitions); BM_DECLARE_bool(benchmark_report_aggregates_only); BM_DECLARE_bool(benchmark_display_aggregates_only); @@ -43,9 +44,21 @@ struct RunResults { bool file_report_aggregates_only = false; }; +struct BENCHMARK_EXPORT BenchTimeType { + enum { ITERS, TIME } tag; + union { + IterationCount iters; + double time; + }; +}; + +BENCHMARK_EXPORT +BenchTimeType ParseBenchMinTime(const std::string& value); + class BenchmarkRunner { public: BenchmarkRunner(const benchmark::internal::BenchmarkInstance& b_, + benchmark::internal::PerfCountersMeasurement* pmc_, BenchmarkReporter::PerFamilyRunReports* reports_for_family); int GetNumRepeats() const { return repeats; } @@ -62,13 +75,22 @@ class BenchmarkRunner { return reports_for_family; } + double GetMinTime() const { return min_time; } + + bool HasExplicitIters() const { return has_explicit_iteration_count; } + + IterationCount GetIters() const { return iters; } + private: RunResults run_results; const benchmark::internal::BenchmarkInstance& b; BenchmarkReporter::PerFamilyRunReports* reports_for_family; + BenchTimeType parsed_benchtime_flag; const double min_time; + const double min_warmup_time; + bool warmup_done; const int repeats; const bool has_explicit_iteration_count; @@ -82,8 +104,7 @@ class BenchmarkRunner { // So only the first repetition has to find/calculate it, // the other repetitions will just use that precomputed iteration count. - PerfCountersMeasurement perf_counters_measurement; - PerfCountersMeasurement* const perf_counters_measurement_ptr; + PerfCountersMeasurement* const perf_counters_measurement_ptr = nullptr; struct IterationResults { internal::ThreadManager::Result results; @@ -95,6 +116,12 @@ class BenchmarkRunner { IterationCount PredictNumItersNeeded(const IterationResults& i) const; bool ShouldReportIterationResults(const IterationResults& i) const; + + double GetMinTimeToApply() const; + + void FinishWarmUp(const IterationCount& i); + + void RunWarmUp(); }; } // namespace internal diff --git a/third-party/benchmark/src/check.h b/third-party/benchmark/src/check.h index 0efd13ff4db6..c1cd5e85e44c 100644 --- a/third-party/benchmark/src/check.h +++ b/third-party/benchmark/src/check.h @@ -5,18 +5,34 @@ #include #include +#include "benchmark/export.h" #include "internal_macros.h" #include "log.h" +#if defined(__GNUC__) || defined(__clang__) +#define BENCHMARK_NOEXCEPT noexcept +#define BENCHMARK_NOEXCEPT_OP(x) noexcept(x) +#elif defined(_MSC_VER) && !defined(__clang__) +#if _MSC_VER >= 1900 +#define BENCHMARK_NOEXCEPT noexcept +#define BENCHMARK_NOEXCEPT_OP(x) noexcept(x) +#else +#define BENCHMARK_NOEXCEPT +#define BENCHMARK_NOEXCEPT_OP(x) +#endif +#define __func__ __FUNCTION__ +#else +#define BENCHMARK_NOEXCEPT +#define BENCHMARK_NOEXCEPT_OP(x) +#endif + namespace benchmark { namespace internal { typedef void(AbortHandlerT)(); -inline AbortHandlerT*& GetAbortHandler() { - static AbortHandlerT* handler = &std::abort; - return handler; -} +BENCHMARK_EXPORT +AbortHandlerT*& GetAbortHandler(); BENCHMARK_NORETURN inline void CallAbortHandler() { GetAbortHandler()(); @@ -36,10 +52,17 @@ class CheckHandler { LogType& GetLog() { return log_; } +#if defined(COMPILER_MSVC) +#pragma warning(push) +#pragma warning(disable : 4722) +#endif BENCHMARK_NORETURN ~CheckHandler() BENCHMARK_NOEXCEPT_OP(false) { log_ << std::endl; CallAbortHandler(); } +#if defined(COMPILER_MSVC) +#pragma warning(pop) +#endif CheckHandler& operator=(const CheckHandler&) = delete; CheckHandler(const CheckHandler&) = delete; diff --git a/third-party/benchmark/src/colorprint.cc b/third-party/benchmark/src/colorprint.cc index 1a000a063736..abc71492f77a 100644 --- a/third-party/benchmark/src/colorprint.cc +++ b/third-party/benchmark/src/colorprint.cc @@ -96,18 +96,18 @@ std::string FormatString(const char* msg, va_list args) { // currently there is no error handling for failure, so this is hack. BM_CHECK(ret >= 0); - if (ret == 0) // handle empty expansion + if (ret == 0) { // handle empty expansion return {}; - else if (static_cast(ret) < size) + } + if (static_cast(ret) < size) { return local_buff; - else { - // we did not provide a long enough buffer on our first attempt. - size = static_cast(ret) + 1; // + 1 for the null byte - std::unique_ptr buff(new char[size]); - ret = vsnprintf(buff.get(), size, msg, args); - BM_CHECK(ret > 0 && (static_cast(ret)) < size); - return buff.get(); } + // we did not provide a long enough buffer on our first attempt. + size = static_cast(ret) + 1; // + 1 for the null byte + std::unique_ptr buff(new char[size]); + ret = vsnprintf(buff.get(), size, msg, args); + BM_CHECK(ret > 0 && (static_cast(ret)) < size); + return buff.get(); } std::string FormatString(const char* msg, ...) { @@ -140,12 +140,12 @@ void ColorPrintf(std::ostream& out, LogColor color, const char* fmt, // We need to flush the stream buffers into the console before each // SetConsoleTextAttribute call lest it affect the text that is already // printed but has not yet reached the console. - fflush(stdout); + out.flush(); SetConsoleTextAttribute(stdout_handle, GetPlatformColorCode(color) | FOREGROUND_INTENSITY); - vprintf(fmt, args); + out << FormatString(fmt, args); - fflush(stdout); + out.flush(); // Restores the text color. SetConsoleTextAttribute(stdout_handle, old_color_attrs); #else @@ -163,12 +163,24 @@ bool IsColorTerminal() { #else // On non-Windows platforms, we rely on the TERM variable. This list of // supported TERM values is copied from Google Test: - // . + // . const char* const SUPPORTED_TERM_VALUES[] = { - "xterm", "xterm-color", "xterm-256color", - "screen", "screen-256color", "tmux", - "tmux-256color", "rxvt-unicode", "rxvt-unicode-256color", - "linux", "cygwin", + "xterm", + "xterm-color", + "xterm-256color", + "screen", + "screen-256color", + "tmux", + "tmux-256color", + "rxvt-unicode", + "rxvt-unicode-256color", + "linux", + "cygwin", + "xterm-kitty", + "alacritty", + "foot", + "foot-extra", + "wezterm", }; const char* const term = getenv("TERM"); diff --git a/third-party/benchmark/src/commandlineflags.cc b/third-party/benchmark/src/commandlineflags.cc index 9615e351ffae..dcb414959df4 100644 --- a/third-party/benchmark/src/commandlineflags.cc +++ b/third-party/benchmark/src/commandlineflags.cc @@ -121,12 +121,14 @@ static std::string FlagToEnvVar(const char* flag) { } // namespace +BENCHMARK_EXPORT bool BoolFromEnv(const char* flag, bool default_val) { const std::string env_var = FlagToEnvVar(flag); const char* const value_str = getenv(env_var.c_str()); return value_str == nullptr ? default_val : IsTruthyFlagValue(value_str); } +BENCHMARK_EXPORT int32_t Int32FromEnv(const char* flag, int32_t default_val) { const std::string env_var = FlagToEnvVar(flag); const char* const value_str = getenv(env_var.c_str()); @@ -139,6 +141,7 @@ int32_t Int32FromEnv(const char* flag, int32_t default_val) { return value; } +BENCHMARK_EXPORT double DoubleFromEnv(const char* flag, double default_val) { const std::string env_var = FlagToEnvVar(flag); const char* const value_str = getenv(env_var.c_str()); @@ -151,12 +154,14 @@ double DoubleFromEnv(const char* flag, double default_val) { return value; } +BENCHMARK_EXPORT const char* StringFromEnv(const char* flag, const char* default_val) { const std::string env_var = FlagToEnvVar(flag); const char* const value = getenv(env_var.c_str()); return value == nullptr ? default_val : value; } +BENCHMARK_EXPORT std::map KvPairsFromEnv( const char* flag, std::map default_val) { const std::string env_var = FlagToEnvVar(flag); @@ -201,6 +206,7 @@ const char* ParseFlagValue(const char* str, const char* flag, return flag_end + 1; } +BENCHMARK_EXPORT bool ParseBoolFlag(const char* str, const char* flag, bool* value) { // Gets the value of the flag as a string. const char* const value_str = ParseFlagValue(str, flag, true); @@ -213,6 +219,7 @@ bool ParseBoolFlag(const char* str, const char* flag, bool* value) { return true; } +BENCHMARK_EXPORT bool ParseInt32Flag(const char* str, const char* flag, int32_t* value) { // Gets the value of the flag as a string. const char* const value_str = ParseFlagValue(str, flag, false); @@ -225,6 +232,7 @@ bool ParseInt32Flag(const char* str, const char* flag, int32_t* value) { value); } +BENCHMARK_EXPORT bool ParseDoubleFlag(const char* str, const char* flag, double* value) { // Gets the value of the flag as a string. const char* const value_str = ParseFlagValue(str, flag, false); @@ -237,6 +245,7 @@ bool ParseDoubleFlag(const char* str, const char* flag, double* value) { value); } +BENCHMARK_EXPORT bool ParseStringFlag(const char* str, const char* flag, std::string* value) { // Gets the value of the flag as a string. const char* const value_str = ParseFlagValue(str, flag, false); @@ -248,6 +257,7 @@ bool ParseStringFlag(const char* str, const char* flag, std::string* value) { return true; } +BENCHMARK_EXPORT bool ParseKeyValueFlag(const char* str, const char* flag, std::map* value) { const char* const value_str = ParseFlagValue(str, flag, false); @@ -263,23 +273,26 @@ bool ParseKeyValueFlag(const char* str, const char* flag, return true; } +BENCHMARK_EXPORT bool IsFlag(const char* str, const char* flag) { return (ParseFlagValue(str, flag, true) != nullptr); } +BENCHMARK_EXPORT bool IsTruthyFlagValue(const std::string& value) { if (value.size() == 1) { char v = value[0]; return isalnum(v) && !(v == '0' || v == 'f' || v == 'F' || v == 'n' || v == 'N'); - } else if (!value.empty()) { + } + if (!value.empty()) { std::string value_lower(value); std::transform(value_lower.begin(), value_lower.end(), value_lower.begin(), [](char c) { return static_cast(::tolower(c)); }); return !(value_lower == "false" || value_lower == "no" || value_lower == "off"); - } else - return true; + } + return true; } } // end namespace benchmark diff --git a/third-party/benchmark/src/commandlineflags.h b/third-party/benchmark/src/commandlineflags.h index 5baaf11784df..7882628975ea 100644 --- a/third-party/benchmark/src/commandlineflags.h +++ b/third-party/benchmark/src/commandlineflags.h @@ -5,28 +5,33 @@ #include #include +#include "benchmark/export.h" + // Macro for referencing flags. #define FLAG(name) FLAGS_##name // Macros for declaring flags. -#define BM_DECLARE_bool(name) extern bool FLAG(name) -#define BM_DECLARE_int32(name) extern int32_t FLAG(name) -#define BM_DECLARE_double(name) extern double FLAG(name) -#define BM_DECLARE_string(name) extern std::string FLAG(name) +#define BM_DECLARE_bool(name) BENCHMARK_EXPORT extern bool FLAG(name) +#define BM_DECLARE_int32(name) BENCHMARK_EXPORT extern int32_t FLAG(name) +#define BM_DECLARE_double(name) BENCHMARK_EXPORT extern double FLAG(name) +#define BM_DECLARE_string(name) BENCHMARK_EXPORT extern std::string FLAG(name) #define BM_DECLARE_kvpairs(name) \ - extern std::map FLAG(name) + BENCHMARK_EXPORT extern std::map FLAG(name) // Macros for defining flags. #define BM_DEFINE_bool(name, default_val) \ - bool FLAG(name) = benchmark::BoolFromEnv(#name, default_val) + BENCHMARK_EXPORT bool FLAG(name) = benchmark::BoolFromEnv(#name, default_val) #define BM_DEFINE_int32(name, default_val) \ - int32_t FLAG(name) = benchmark::Int32FromEnv(#name, default_val) + BENCHMARK_EXPORT int32_t FLAG(name) = \ + benchmark::Int32FromEnv(#name, default_val) #define BM_DEFINE_double(name, default_val) \ - double FLAG(name) = benchmark::DoubleFromEnv(#name, default_val) + BENCHMARK_EXPORT double FLAG(name) = \ + benchmark::DoubleFromEnv(#name, default_val) #define BM_DEFINE_string(name, default_val) \ - std::string FLAG(name) = benchmark::StringFromEnv(#name, default_val) -#define BM_DEFINE_kvpairs(name, default_val) \ - std::map FLAG(name) = \ + BENCHMARK_EXPORT std::string FLAG(name) = \ + benchmark::StringFromEnv(#name, default_val) +#define BM_DEFINE_kvpairs(name, default_val) \ + BENCHMARK_EXPORT std::map FLAG(name) = \ benchmark::KvPairsFromEnv(#name, default_val) namespace benchmark { @@ -35,6 +40,7 @@ namespace benchmark { // // If the variable exists, returns IsTruthyFlagValue() value; if not, // returns the given default value. +BENCHMARK_EXPORT bool BoolFromEnv(const char* flag, bool default_val); // Parses an Int32 from the environment variable corresponding to the given @@ -42,6 +48,7 @@ bool BoolFromEnv(const char* flag, bool default_val); // // If the variable exists, returns ParseInt32() value; if not, returns // the given default value. +BENCHMARK_EXPORT int32_t Int32FromEnv(const char* flag, int32_t default_val); // Parses an Double from the environment variable corresponding to the given @@ -49,6 +56,7 @@ int32_t Int32FromEnv(const char* flag, int32_t default_val); // // If the variable exists, returns ParseDouble(); if not, returns // the given default value. +BENCHMARK_EXPORT double DoubleFromEnv(const char* flag, double default_val); // Parses a string from the environment variable corresponding to the given @@ -56,6 +64,7 @@ double DoubleFromEnv(const char* flag, double default_val); // // If variable exists, returns its value; if not, returns // the given default value. +BENCHMARK_EXPORT const char* StringFromEnv(const char* flag, const char* default_val); // Parses a set of kvpairs from the environment variable corresponding to the @@ -63,6 +72,7 @@ const char* StringFromEnv(const char* flag, const char* default_val); // // If variable exists, returns its value; if not, returns // the given default value. +BENCHMARK_EXPORT std::map KvPairsFromEnv( const char* flag, std::map default_val); @@ -75,40 +85,47 @@ std::map KvPairsFromEnv( // // On success, stores the value of the flag in *value, and returns // true. On failure, returns false without changing *value. +BENCHMARK_EXPORT bool ParseBoolFlag(const char* str, const char* flag, bool* value); // Parses a string for an Int32 flag, in the form of "--flag=value". // // On success, stores the value of the flag in *value, and returns // true. On failure, returns false without changing *value. +BENCHMARK_EXPORT bool ParseInt32Flag(const char* str, const char* flag, int32_t* value); // Parses a string for a Double flag, in the form of "--flag=value". // // On success, stores the value of the flag in *value, and returns // true. On failure, returns false without changing *value. +BENCHMARK_EXPORT bool ParseDoubleFlag(const char* str, const char* flag, double* value); // Parses a string for a string flag, in the form of "--flag=value". // // On success, stores the value of the flag in *value, and returns // true. On failure, returns false without changing *value. +BENCHMARK_EXPORT bool ParseStringFlag(const char* str, const char* flag, std::string* value); // Parses a string for a kvpairs flag in the form "--flag=key=value,key=value" // // On success, stores the value of the flag in *value and returns true. On // failure returns false, though *value may have been mutated. +BENCHMARK_EXPORT bool ParseKeyValueFlag(const char* str, const char* flag, std::map* value); // Returns true if the string matches the flag. +BENCHMARK_EXPORT bool IsFlag(const char* str, const char* flag); // Returns true unless value starts with one of: '0', 'f', 'F', 'n' or 'N', or // some non-alphanumeric character. Also returns false if the value matches // one of 'no', 'false', 'off' (case-insensitive). As a special case, also // returns true if value is the empty string. +BENCHMARK_EXPORT bool IsTruthyFlagValue(const std::string& value); } // end namespace benchmark diff --git a/third-party/benchmark/src/complexity.cc b/third-party/benchmark/src/complexity.cc index 825c57394a8c..eee3122646f9 100644 --- a/third-party/benchmark/src/complexity.cc +++ b/third-party/benchmark/src/complexity.cc @@ -37,12 +37,14 @@ BigOFunc* FittingCurve(BigO complexity) { return [](IterationCount n) -> double { return std::pow(n, 3); }; case oLogN: /* Note: can't use log2 because Android's GNU STL lacks it */ - return - [](IterationCount n) { return kLog2E * log(static_cast(n)); }; + return [](IterationCount n) { + return kLog2E * std::log(static_cast(n)); + }; case oNLogN: /* Note: can't use log2 because Android's GNU STL lacks it */ return [](IterationCount n) { - return kLog2E * n * log(static_cast(n)); + return kLog2E * static_cast(n) * + std::log(static_cast(n)); }; case o1: default: @@ -75,12 +77,12 @@ std::string GetBigOString(BigO complexity) { // given by the lambda expression. // - n : Vector containing the size of the benchmark tests. // - time : Vector containing the times for the benchmark tests. -// - fitting_curve : lambda expression (e.g. [](int64_t n) {return n; };). +// - fitting_curve : lambda expression (e.g. [](ComplexityN n) {return n; };). // For a deeper explanation on the algorithm logic, please refer to // https://en.wikipedia.org/wiki/Least_squares#Least_squares,_regression_analysis_and_statistics -LeastSq MinimalLeastSq(const std::vector& n, +LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, BigOFunc* fitting_curve) { double sigma_gn_squared = 0.0; @@ -105,12 +107,12 @@ LeastSq MinimalLeastSq(const std::vector& n, double rms = 0.0; for (size_t i = 0; i < n.size(); ++i) { double fit = result.coef * fitting_curve(n[i]); - rms += pow((time[i] - fit), 2); + rms += std::pow((time[i] - fit), 2); } // Normalized RMS by the mean of the observed values - double mean = sigma_time / n.size(); - result.rms = sqrt(rms / n.size()) / mean; + double mean = sigma_time / static_cast(n.size()); + result.rms = std::sqrt(rms / static_cast(n.size())) / mean; return result; } @@ -122,7 +124,7 @@ LeastSq MinimalLeastSq(const std::vector& n, // - complexity : If different than oAuto, the fitting curve will stick to // this one. If it is oAuto, it will be calculated the best // fitting curve. -LeastSq MinimalLeastSq(const std::vector& n, +LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, const BigO complexity) { BM_CHECK_EQ(n.size(), time.size()); BM_CHECK_GE(n.size(), 2); // Do not compute fitting curve is less than two @@ -162,7 +164,7 @@ std::vector ComputeBigO( if (reports.size() < 2) return results; // Accumulators. - std::vector n; + std::vector n; std::vector real_time; std::vector cpu_time; @@ -171,8 +173,10 @@ std::vector ComputeBigO( BM_CHECK_GT(run.complexity_n, 0) << "Did you forget to call SetComplexityN?"; n.push_back(run.complexity_n); - real_time.push_back(run.real_accumulated_time / run.iterations); - cpu_time.push_back(run.cpu_accumulated_time / run.iterations); + real_time.push_back(run.real_accumulated_time / + static_cast(run.iterations)); + cpu_time.push_back(run.cpu_accumulated_time / + static_cast(run.iterations)); } LeastSq result_cpu; @@ -182,8 +186,19 @@ std::vector ComputeBigO( result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity_lambda); result_real = MinimalLeastSq(n, real_time, reports[0].complexity_lambda); } else { - result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity); - result_real = MinimalLeastSq(n, real_time, result_cpu.complexity); + const BigO* InitialBigO = &reports[0].complexity; + const bool use_real_time_for_initial_big_o = + reports[0].use_real_time_for_initial_big_o; + if (use_real_time_for_initial_big_o) { + result_real = MinimalLeastSq(n, real_time, *InitialBigO); + InitialBigO = &result_real.complexity; + // The Big-O complexity for CPU time must have the same Big-O function! + } + result_cpu = MinimalLeastSq(n, cpu_time, *InitialBigO); + InitialBigO = &result_cpu.complexity; + if (!use_real_time_for_initial_big_o) { + result_real = MinimalLeastSq(n, real_time, *InitialBigO); + } } // Drop the 'args' when reporting complexity. diff --git a/third-party/benchmark/src/complexity.h b/third-party/benchmark/src/complexity.h index df29b48d29b4..0a0679b48bc8 100644 --- a/third-party/benchmark/src/complexity.h +++ b/third-party/benchmark/src/complexity.h @@ -31,7 +31,7 @@ std::vector ComputeBigO( const std::vector& reports); // This data structure will contain the result returned by MinimalLeastSq -// - coef : Estimated coeficient for the high-order term as +// - coef : Estimated coefficient for the high-order term as // interpolated from data. // - rms : Normalized Root Mean Squared Error. // - complexity : Scalability form (e.g. oN, oNLogN). In case a scalability diff --git a/third-party/benchmark/src/console_reporter.cc b/third-party/benchmark/src/console_reporter.cc index 04cc0b74e58e..35c3de2a4dba 100644 --- a/third-party/benchmark/src/console_reporter.cc +++ b/third-party/benchmark/src/console_reporter.cc @@ -33,6 +33,7 @@ namespace benchmark { +BENCHMARK_EXPORT bool ConsoleReporter::ReportContext(const Context& context) { name_field_width_ = context.name_field_width; printed_header_ = false; @@ -41,17 +42,22 @@ bool ConsoleReporter::ReportContext(const Context& context) { PrintBasicContext(&GetErrorStream(), context); #ifdef BENCHMARK_OS_WINDOWS - if ((output_options_ & OO_Color) && &std::cout != &GetOutputStream()) { - GetErrorStream() - << "Color printing is only supported for stdout on windows." - " Disabling color printing\n"; - output_options_ = static_cast(output_options_ & ~OO_Color); + if ((output_options_ & OO_Color)) { + auto stdOutBuf = std::cout.rdbuf(); + auto outStreamBuf = GetOutputStream().rdbuf(); + if (stdOutBuf != outStreamBuf) { + GetErrorStream() + << "Color printing is only supported for stdout on windows." + " Disabling color printing\n"; + output_options_ = static_cast(output_options_ & ~OO_Color); + } } #endif return true; } +BENCHMARK_EXPORT void ConsoleReporter::PrintHeader(const Run& run) { std::string str = FormatString("%-*s %13s %15s %12s", static_cast(name_field_width_), @@ -69,6 +75,7 @@ void ConsoleReporter::PrintHeader(const Run& run) { GetOutputStream() << line << "\n" << str << "\n" << line << "\n"; } +BENCHMARK_EXPORT void ConsoleReporter::ReportRuns(const std::vector& reports) { for (const auto& run : reports) { // print the header: @@ -99,6 +106,9 @@ static void IgnoreColorPrint(std::ostream& out, LogColor, const char* fmt, } static std::string FormatTime(double time) { + // For the time columns of the console printer 13 digits are reserved. One of + // them is a space and max two of them are the time unit (e.g ns). That puts + // us at 10 digits usable for the number. // Align decimal places... if (time < 1.0) { return FormatString("%10.3f", time); @@ -109,9 +119,15 @@ static std::string FormatTime(double time) { if (time < 100.0) { return FormatString("%10.1f", time); } + // Assuming the time is at max 9.9999e+99 and we have 10 digits for the + // number, we get 10-1(.)-1(e)-1(sign)-2(exponent) = 5 digits to print. + if (time > 9999999999 /*max 10 digit number*/) { + return FormatString("%1.4e", time); + } return FormatString("%10.0f", time); } +BENCHMARK_EXPORT void ConsoleReporter::PrintRunData(const Run& result) { typedef void(PrinterFn)(std::ostream&, LogColor, const char*, ...); auto& Out = GetOutputStream(); @@ -123,9 +139,13 @@ void ConsoleReporter::PrintRunData(const Run& result) { printer(Out, name_color, "%-*s ", name_field_width_, result.benchmark_name().c_str()); - if (result.error_occurred) { + if (internal::SkippedWithError == result.skipped) { printer(Out, COLOR_RED, "ERROR OCCURRED: \'%s\'", - result.error_message.c_str()); + result.skip_message.c_str()); + printer(Out, COLOR_DEFAULT, "\n"); + return; + } else if (internal::SkippedWithMessage == result.skipped) { + printer(Out, COLOR_WHITE, "SKIPPED: \'%s\'", result.skip_message.c_str()); printer(Out, COLOR_DEFAULT, "\n"); return; } diff --git a/third-party/benchmark/src/counter.cc b/third-party/benchmark/src/counter.cc index cf5b78ee3ac6..aa14cd8092f9 100644 --- a/third-party/benchmark/src/counter.cc +++ b/third-party/benchmark/src/counter.cc @@ -27,10 +27,10 @@ double Finish(Counter const& c, IterationCount iterations, double cpu_time, v /= num_threads; } if (c.flags & Counter::kIsIterationInvariant) { - v *= iterations; + v *= static_cast(iterations); } if (c.flags & Counter::kAvgIterations) { - v /= iterations; + v /= static_cast(iterations); } if (c.flags & Counter::kInvert) { // Invert is *always* last. diff --git a/third-party/benchmark/src/csv_reporter.cc b/third-party/benchmark/src/csv_reporter.cc index 1c5e9fa6689c..4b39e2c52fb9 100644 --- a/third-party/benchmark/src/csv_reporter.cc +++ b/third-party/benchmark/src/csv_reporter.cc @@ -52,11 +52,13 @@ std::string CsvEscape(const std::string& s) { return '"' + tmp + '"'; } +BENCHMARK_EXPORT bool CSVReporter::ReportContext(const Context& context) { PrintBasicContext(&GetErrorStream(), context); return true; } +BENCHMARK_EXPORT void CSVReporter::ReportRuns(const std::vector& reports) { std::ostream& Out = GetOutputStream(); @@ -103,13 +105,14 @@ void CSVReporter::ReportRuns(const std::vector& reports) { } } +BENCHMARK_EXPORT void CSVReporter::PrintRunData(const Run& run) { std::ostream& Out = GetOutputStream(); Out << CsvEscape(run.benchmark_name()) << ","; - if (run.error_occurred) { + if (run.skipped) { Out << std::string(elements.size() - 3, ','); - Out << "true,"; - Out << CsvEscape(run.error_message) << "\n"; + Out << std::boolalpha << (internal::SkippedWithError == run.skipped) << ","; + Out << CsvEscape(run.skip_message) << "\n"; return; } @@ -119,13 +122,21 @@ void CSVReporter::PrintRunData(const Run& run) { } Out << ","; - Out << run.GetAdjustedRealTime() << ","; - Out << run.GetAdjustedCPUTime() << ","; + if (run.run_type != Run::RT_Aggregate || + run.aggregate_unit == StatisticUnit::kTime) { + Out << run.GetAdjustedRealTime() << ","; + Out << run.GetAdjustedCPUTime() << ","; + } else { + assert(run.aggregate_unit == StatisticUnit::kPercentage); + Out << run.real_accumulated_time << ","; + Out << run.cpu_accumulated_time << ","; + } // Do not print timeLabel on bigO and RMS report if (run.report_big_o) { Out << GetBigOString(run.complexity); - } else if (!run.report_rms) { + } else if (!run.report_rms && + run.aggregate_unit != StatisticUnit::kPercentage) { Out << GetTimeUnitString(run.time_unit); } Out << ","; diff --git a/third-party/benchmark/src/cycleclock.h b/third-party/benchmark/src/cycleclock.h index d65d32a39d3d..eff563e7fac2 100644 --- a/third-party/benchmark/src/cycleclock.h +++ b/third-party/benchmark/src/cycleclock.h @@ -36,7 +36,8 @@ // declarations of some other intrinsics, breaking compilation. // Therefore, we simply declare __rdtsc ourselves. See also // http://connect.microsoft.com/VisualStudio/feedback/details/262047 -#if defined(COMPILER_MSVC) && !defined(_M_IX86) && !defined(_M_ARM64) +#if defined(COMPILER_MSVC) && !defined(_M_IX86) && !defined(_M_ARM64) && \ + !defined(_M_ARM64EC) extern "C" uint64_t __rdtsc(); #pragma intrinsic(__rdtsc) #endif @@ -114,7 +115,7 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { // when I know it will work. Otherwise, I'll use __rdtsc and hope // the code is being compiled with a non-ancient compiler. _asm rdtsc -#elif defined(COMPILER_MSVC) && defined(_M_ARM64) +#elif defined(COMPILER_MSVC) && (defined(_M_ARM64) || defined(_M_ARM64EC)) // See // https://docs.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics // and https://reviews.llvm.org/D53115 int64_t virtual_timer_value; @@ -132,7 +133,7 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { // Native Client does not provide any API to access cycle counter. // Use clock_gettime(CLOCK_MONOTONIC, ...) instead of gettimeofday - // because is provides nanosecond resolution (which is noticable at + // because is provides nanosecond resolution (which is noticeable at // least for PNaCl modules running on x86 Mac & Linux). // Initialize to always return 0 if clock_gettime fails. struct timespec ts = {0, 0}; @@ -173,7 +174,7 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { struct timeval tv; gettimeofday(&tv, nullptr); return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; -#elif defined(__loongarch__) +#elif defined(__loongarch__) || defined(__csky__) struct timeval tv; gettimeofday(&tv, nullptr); return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; @@ -188,15 +189,16 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { #endif return tsc; #elif defined(__riscv) // RISC-V - // Use RDCYCLE (and RDCYCLEH on riscv32) + // Use RDTIME (and RDTIMEH on riscv32). + // RDCYCLE is a privileged instruction since Linux 6.6. #if __riscv_xlen == 32 uint32_t cycles_lo, cycles_hi0, cycles_hi1; // This asm also includes the PowerPC overflow handling strategy, as above. // Implemented in assembly because Clang insisted on branching. asm volatile( - "rdcycleh %0\n" - "rdcycle %1\n" - "rdcycleh %2\n" + "rdtimeh %0\n" + "rdtime %1\n" + "rdtimeh %2\n" "sub %0, %0, %2\n" "seqz %0, %0\n" "sub %0, zero, %0\n" @@ -205,17 +207,31 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { return (static_cast(cycles_hi1) << 32) | cycles_lo; #else uint64_t cycles; - asm volatile("rdcycle %0" : "=r"(cycles)); + asm volatile("rdtime %0" : "=r"(cycles)); return cycles; #endif #elif defined(__e2k__) || defined(__elbrus__) struct timeval tv; gettimeofday(&tv, nullptr); return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; +#elif defined(__hexagon__) + uint64_t pcycle; + asm volatile("%0 = C15:14" : "=r"(pcycle)); + return static_cast(pcycle); +#elif defined(__alpha__) + // Alpha has a cycle counter, the PCC register, but it is an unsigned 32-bit + // integer and thus wraps every ~4s, making using it for tick counts + // unreliable beyond this time range. The real-time clock is low-precision, + // roughtly ~1ms, but it is the only option that can reasonable count + // indefinitely. + struct timeval tv; + gettimeofday(&tv, nullptr); + return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; #else -// The soft failover to a generic implementation is automatic only for ARM. -// For other platforms the developer is expected to make an attempt to create -// a fast implementation and use generic version if nothing better is available. + // The soft failover to a generic implementation is automatic only for ARM. + // For other platforms the developer is expected to make an attempt to create + // a fast implementation and use generic version if nothing better is + // available. #error You need to define CycleTimer for your OS and CPU #endif } diff --git a/third-party/benchmark/src/internal_macros.h b/third-party/benchmark/src/internal_macros.h index 91f367b894bc..8dd7d0c6502e 100644 --- a/third-party/benchmark/src/internal_macros.h +++ b/third-party/benchmark/src/internal_macros.h @@ -1,8 +1,6 @@ #ifndef BENCHMARK_INTERNAL_MACROS_H_ #define BENCHMARK_INTERNAL_MACROS_H_ -#include "benchmark/benchmark.h" - /* Needed to detect STL */ #include @@ -44,6 +42,19 @@ #define BENCHMARK_OS_CYGWIN 1 #elif defined(_WIN32) #define BENCHMARK_OS_WINDOWS 1 + // WINAPI_FAMILY_PARTITION is defined in winapifamily.h. + // We include windows.h which implicitly includes winapifamily.h for compatibility. + #ifndef NOMINMAX + #define NOMINMAX + #endif + #include + #if defined(WINAPI_FAMILY_PARTITION) + #if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) + #define BENCHMARK_OS_WINDOWS_WIN32 1 + #elif WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) + #define BENCHMARK_OS_WINDOWS_RT 1 + #endif + #endif #if defined(__MINGW32__) #define BENCHMARK_OS_MINGW 1 #endif @@ -80,6 +91,8 @@ #define BENCHMARK_OS_QNX 1 #elif defined(__MVS__) #define BENCHMARK_OS_ZOS 1 +#elif defined(__hexagon__) +#define BENCHMARK_OS_QURT 1 #endif #if defined(__ANDROID__) && defined(__GLIBCXX__) diff --git a/third-party/benchmark/src/json_reporter.cc b/third-party/benchmark/src/json_reporter.cc index e84a4ed24f9d..b8c8c94c08a0 100644 --- a/third-party/benchmark/src/json_reporter.cc +++ b/third-party/benchmark/src/json_reporter.cc @@ -28,10 +28,6 @@ #include "timers.h" namespace benchmark { -namespace internal { -extern std::map* global_context; -} - namespace { std::string StrEscape(const std::string& s) { @@ -89,12 +85,6 @@ std::string FormatKV(std::string const& key, int64_t value) { return ss.str(); } -std::string FormatKV(std::string const& key, IterationCount value) { - std::stringstream ss; - ss << '"' << StrEscape(key) << "\": " << value; - return ss.str(); -} - std::string FormatKV(std::string const& key, double value) { std::stringstream ss; ss << '"' << StrEscape(key) << "\": "; @@ -177,15 +167,25 @@ bool JSONReporter::ReportContext(const Context& context) { } out << "],\n"; + out << indent << FormatKV("library_version", GetBenchmarkVersion()); + out << ",\n"; + #if defined(NDEBUG) const char build_type[] = "release"; #else const char build_type[] = "debug"; #endif out << indent << FormatKV("library_build_type", build_type); + out << ",\n"; + + // NOTE: our json schema is not strictly tied to the library version! + out << indent << FormatKV("json_schema_version", int64_t(1)); + + std::map* global_context = + internal::GetGlobalContext(); - if (internal::global_context != nullptr) { - for (const auto& kv : *internal::global_context) { + if (global_context != nullptr) { + for (const auto& kv : *global_context) { out << ",\n"; out << indent << FormatKV(kv.first, kv.second); } @@ -261,9 +261,12 @@ void JSONReporter::PrintRunData(Run const& run) { BENCHMARK_UNREACHABLE(); }()) << ",\n"; } - if (run.error_occurred) { - out << indent << FormatKV("error_occurred", run.error_occurred) << ",\n"; - out << indent << FormatKV("error_message", run.error_message) << ",\n"; + if (internal::SkippedWithError == run.skipped) { + out << indent << FormatKV("error_occurred", true) << ",\n"; + out << indent << FormatKV("error_message", run.skip_message) << ",\n"; + } else if (internal::SkippedWithMessage == run.skipped) { + out << indent << FormatKV("skipped", true) << ",\n"; + out << indent << FormatKV("skip_message", run.skip_message) << ",\n"; } if (!run.report_big_o && !run.report_rms) { out << indent << FormatKV("iterations", run.iterations) << ",\n"; @@ -301,7 +304,8 @@ void JSONReporter::PrintRunData(Run const& run) { out << ",\n" << indent << FormatKV("max_bytes_used", memory_result.max_bytes_used); - auto report_if_present = [&out, &indent](const char* label, int64_t val) { + auto report_if_present = [&out, &indent](const std::string& label, + int64_t val) { if (val != MemoryManager::TombstoneValue) out << ",\n" << indent << FormatKV(label, val); }; diff --git a/third-party/benchmark/src/log.h b/third-party/benchmark/src/log.h index 48c071aded8f..9a21400b096d 100644 --- a/third-party/benchmark/src/log.h +++ b/third-party/benchmark/src/log.h @@ -4,7 +4,12 @@ #include #include -#include "benchmark/benchmark.h" +// NOTE: this is also defined in benchmark.h but we're trying to avoid a +// dependency. +// The _MSVC_LANG check should detect Visual Studio 2015 Update 3 and newer. +#if __cplusplus >= 201103L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201103L) +#define BENCHMARK_HAS_CXX11 +#endif namespace benchmark { namespace internal { @@ -23,7 +28,16 @@ class LogType { private: LogType(std::ostream* out) : out_(out) {} std::ostream* out_; - BENCHMARK_DISALLOW_COPY_AND_ASSIGN(LogType); + + // NOTE: we could use BENCHMARK_DISALLOW_COPY_AND_ASSIGN but we shouldn't have + // a dependency on benchmark.h from here. +#ifndef BENCHMARK_HAS_CXX11 + LogType(const LogType&); + LogType& operator=(const LogType&); +#else + LogType(const LogType&) = delete; + LogType& operator=(const LogType&) = delete; +#endif }; template @@ -47,13 +61,13 @@ inline int& LogLevel() { } inline LogType& GetNullLogInstance() { - static LogType log(nullptr); - return log; + static LogType null_log(static_cast(nullptr)); + return null_log; } inline LogType& GetErrorLogInstance() { - static LogType log(&std::clog); - return log; + static LogType error_log(&std::clog); + return error_log; } inline LogType& GetLogInstanceForLevel(int level) { diff --git a/third-party/benchmark/src/perf_counters.cc b/third-party/benchmark/src/perf_counters.cc index b2ac7687efef..d466e27e86f9 100644 --- a/third-party/benchmark/src/perf_counters.cc +++ b/third-party/benchmark/src/perf_counters.cc @@ -15,6 +15,7 @@ #include "perf_counters.h" #include +#include #include #if defined HAVE_LIBPFM @@ -28,105 +29,254 @@ namespace internal { constexpr size_t PerfCounterValues::kMaxCounters; #if defined HAVE_LIBPFM + +size_t PerfCounterValues::Read(const std::vector& leaders) { + // Create a pointer for multiple reads + const size_t bufsize = values_.size() * sizeof(values_[0]); + char* ptr = reinterpret_cast(values_.data()); + size_t size = bufsize; + for (int lead : leaders) { + auto read_bytes = ::read(lead, ptr, size); + if (read_bytes >= ssize_t(sizeof(uint64_t))) { + // Actual data bytes are all bytes minus initial padding + std::size_t data_bytes = read_bytes - sizeof(uint64_t); + // This should be very cheap since it's in hot cache + std::memmove(ptr, ptr + sizeof(uint64_t), data_bytes); + // Increment our counters + ptr += data_bytes; + size -= data_bytes; + } else { + int err = errno; + GetErrorLogInstance() << "Error reading lead " << lead << " errno:" << err + << " " << ::strerror(err) << "\n"; + return 0; + } + } + return (bufsize - size) / sizeof(uint64_t); +} + const bool PerfCounters::kSupported = true; -bool PerfCounters::Initialize() { return pfm_initialize() == PFM_SUCCESS; } +// Initializes libpfm only on the first call. Returns whether that single +// initialization was successful. +bool PerfCounters::Initialize() { + // Function-scope static gets initialized only once on first call. + static const bool success = []() { + return pfm_initialize() == PFM_SUCCESS; + }(); + return success; +} + +bool PerfCounters::IsCounterSupported(const std::string& name) { + Initialize(); + perf_event_attr_t attr; + std::memset(&attr, 0, sizeof(attr)); + pfm_perf_encode_arg_t arg; + std::memset(&arg, 0, sizeof(arg)); + arg.attr = &attr; + const int mode = PFM_PLM3; // user mode only + int ret = pfm_get_os_event_encoding(name.c_str(), mode, PFM_OS_PERF_EVENT_EXT, + &arg); + return (ret == PFM_SUCCESS); +} PerfCounters PerfCounters::Create( const std::vector& counter_names) { - if (counter_names.empty()) { - return NoCounters(); - } - if (counter_names.size() > PerfCounterValues::kMaxCounters) { - GetErrorLogInstance() - << counter_names.size() - << " counters were requested. The minimum is 1, the maximum is " - << PerfCounterValues::kMaxCounters << "\n"; - return NoCounters(); + if (!counter_names.empty()) { + Initialize(); } - std::vector counter_ids(counter_names.size()); - const int mode = PFM_PLM3; // user mode only + // Valid counters will populate these arrays but we start empty + std::vector valid_names; + std::vector counter_ids; + std::vector leader_ids; + + // Resize to the maximum possible + valid_names.reserve(counter_names.size()); + counter_ids.reserve(counter_names.size()); + + const int kCounterMode = PFM_PLM3; // user mode only + + // Group leads will be assigned on demand. The idea is that once we cannot + // create a counter descriptor, the reason is that this group has maxed out + // so we set the group_id again to -1 and retry - giving the algorithm a + // chance to create a new group leader to hold the next set of counters. + int group_id = -1; + + // Loop through all performance counters for (size_t i = 0; i < counter_names.size(); ++i) { - const bool is_first = i == 0; - struct perf_event_attr attr {}; - attr.size = sizeof(attr); - const int group_id = !is_first ? counter_ids[0] : -1; + // we are about to push into the valid names vector + // check if we did not reach the maximum + if (valid_names.size() == PerfCounterValues::kMaxCounters) { + // Log a message if we maxed out and stop adding + GetErrorLogInstance() + << counter_names.size() << " counters were requested. The maximum is " + << PerfCounterValues::kMaxCounters << " and " << valid_names.size() + << " were already added. All remaining counters will be ignored\n"; + // stop the loop and return what we have already + break; + } + + // Check if this name is empty const auto& name = counter_names[i]; if (name.empty()) { - GetErrorLogInstance() << "A counter name was the empty string\n"; - return NoCounters(); + GetErrorLogInstance() + << "A performance counter name was the empty string\n"; + continue; } + + // Here first means first in group, ie the group leader + const bool is_first = (group_id < 0); + + // This struct will be populated by libpfm from the counter string + // and then fed into the syscall perf_event_open + struct perf_event_attr attr {}; + attr.size = sizeof(attr); + + // This is the input struct to libpfm. pfm_perf_encode_arg_t arg{}; arg.attr = &attr; - - const int pfm_get = - pfm_get_os_event_encoding(name.c_str(), mode, PFM_OS_PERF_EVENT, &arg); + const int pfm_get = pfm_get_os_event_encoding(name.c_str(), kCounterMode, + PFM_OS_PERF_EVENT, &arg); if (pfm_get != PFM_SUCCESS) { - GetErrorLogInstance() << "Unknown counter name: " << name << "\n"; - return NoCounters(); + GetErrorLogInstance() + << "Unknown performance counter name: " << name << "\n"; + continue; } - attr.disabled = is_first; - // Note: the man page for perf_event_create suggests inerit = true and + + // We then proceed to populate the remaining fields in our attribute struct + // Note: the man page for perf_event_create suggests inherit = true and // read_format = PERF_FORMAT_GROUP don't work together, but that's not the // case. + attr.disabled = is_first; attr.inherit = true; attr.pinned = is_first; attr.exclude_kernel = true; attr.exclude_user = false; attr.exclude_hv = true; - // Read all counters in one read. + + // Read all counters in a group in one read. attr.read_format = PERF_FORMAT_GROUP; int id = -1; - static constexpr size_t kNrOfSyscallRetries = 5; - // Retry syscall as it was interrupted often (b/64774091). - for (size_t num_retries = 0; num_retries < kNrOfSyscallRetries; - ++num_retries) { - id = perf_event_open(&attr, 0, -1, group_id, 0); - if (id >= 0 || errno != EINTR) { - break; + while (id < 0) { + static constexpr size_t kNrOfSyscallRetries = 5; + // Retry syscall as it was interrupted often (b/64774091). + for (size_t num_retries = 0; num_retries < kNrOfSyscallRetries; + ++num_retries) { + id = perf_event_open(&attr, 0, -1, group_id, 0); + if (id >= 0 || errno != EINTR) { + break; + } + } + if (id < 0) { + // If the file descriptor is negative we might have reached a limit + // in the current group. Set the group_id to -1 and retry + if (group_id >= 0) { + // Create a new group + group_id = -1; + } else { + // At this point we have already retried to set a new group id and + // failed. We then give up. + break; + } } } + + // We failed to get a new file descriptor. We might have reached a hard + // hardware limit that cannot be resolved even with group multiplexing if (id < 0) { - GetErrorLogInstance() - << "Failed to get a file descriptor for " << name << "\n"; - return NoCounters(); - } + GetErrorLogInstance() << "***WARNING** Failed to get a file descriptor " + "for performance counter " + << name << ". Ignoring\n"; - counter_ids[i] = id; + // We give up on this counter but try to keep going + // as the others would be fine + continue; + } + if (group_id < 0) { + // This is a leader, store and assign it to the current file descriptor + leader_ids.push_back(id); + group_id = id; + } + // This is a valid counter, add it to our descriptor's list + counter_ids.push_back(id); + valid_names.push_back(name); } - if (ioctl(counter_ids[0], PERF_EVENT_IOC_ENABLE) != 0) { - GetErrorLogInstance() << "Failed to start counters\n"; - return NoCounters(); + + // Loop through all group leaders activating them + // There is another option of starting ALL counters in a process but + // that would be far reaching an intrusion. If the user is using PMCs + // by themselves then this would have a side effect on them. It is + // friendlier to loop through all groups individually. + for (int lead : leader_ids) { + if (ioctl(lead, PERF_EVENT_IOC_ENABLE) != 0) { + // This should never happen but if it does, we give up on the + // entire batch as recovery would be a mess. + GetErrorLogInstance() << "***WARNING*** Failed to start counters. " + "Claring out all counters.\n"; + + // Close all peformance counters + for (int id : counter_ids) { + ::close(id); + } + + // Return an empty object so our internal state is still good and + // the process can continue normally without impact + return NoCounters(); + } } - return PerfCounters(counter_names, std::move(counter_ids)); + return PerfCounters(std::move(valid_names), std::move(counter_ids), + std::move(leader_ids)); } -PerfCounters::~PerfCounters() { +void PerfCounters::CloseCounters() const { if (counter_ids_.empty()) { return; } - ioctl(counter_ids_[0], PERF_EVENT_IOC_DISABLE); + for (int lead : leader_ids_) { + ioctl(lead, PERF_EVENT_IOC_DISABLE); + } for (int fd : counter_ids_) { close(fd); } } #else // defined HAVE_LIBPFM +size_t PerfCounterValues::Read(const std::vector&) { return 0; } + const bool PerfCounters::kSupported = false; bool PerfCounters::Initialize() { return false; } +bool PerfCounters::IsCounterSupported(const std::string&) { return false; } + PerfCounters PerfCounters::Create( const std::vector& counter_names) { if (!counter_names.empty()) { - GetErrorLogInstance() << "Performance counters not supported."; + GetErrorLogInstance() << "Performance counters not supported.\n"; } return NoCounters(); } -PerfCounters::~PerfCounters() = default; +void PerfCounters::CloseCounters() const {} #endif // defined HAVE_LIBPFM + +PerfCountersMeasurement::PerfCountersMeasurement( + const std::vector& counter_names) + : start_values_(counter_names.size()), end_values_(counter_names.size()) { + counters_ = PerfCounters::Create(counter_names); +} + +PerfCounters& PerfCounters::operator=(PerfCounters&& other) noexcept { + if (this != &other) { + CloseCounters(); + + counter_ids_ = std::move(other.counter_ids_); + leader_ids_ = std::move(other.leader_ids_); + counter_names_ = std::move(other.counter_names_); + } + return *this; +} } // namespace internal } // namespace benchmark diff --git a/third-party/benchmark/src/perf_counters.h b/third-party/benchmark/src/perf_counters.h index 47ca1385e24d..bf5eb6bc3aec 100644 --- a/third-party/benchmark/src/perf_counters.h +++ b/third-party/benchmark/src/perf_counters.h @@ -17,16 +17,25 @@ #include #include +#include +#include #include #include "benchmark/benchmark.h" #include "check.h" #include "log.h" +#include "mutex.h" #ifndef BENCHMARK_OS_WINDOWS #include #endif +#if defined(_MSC_VER) +#pragma warning(push) +// C4251: needs to have dll-interface to be used by clients of class +#pragma warning(disable : 4251) +#endif + namespace benchmark { namespace internal { @@ -36,18 +45,21 @@ namespace internal { // The implementation ensures the storage is inlined, and allows 0-based // indexing into the counter values. // The object is used in conjunction with a PerfCounters object, by passing it -// to Snapshot(). The values are populated such that -// perfCounters->names()[i]'s value is obtained at position i (as given by -// operator[]) of this object. -class PerfCounterValues { +// to Snapshot(). The Read() method relocates individual reads, discarding +// the initial padding from each group leader in the values buffer such that +// all user accesses through the [] operator are correct. +class BENCHMARK_EXPORT PerfCounterValues { public: explicit PerfCounterValues(size_t nr_counters) : nr_counters_(nr_counters) { BM_CHECK_LE(nr_counters_, kMaxCounters); } - uint64_t operator[](size_t pos) const { return values_[kPadding + pos]; } + // We are reading correctly now so the values don't need to skip padding + uint64_t operator[](size_t pos) const { return values_[pos]; } - static constexpr size_t kMaxCounters = 3; + // Increased the maximum to 32 only since the buffer + // is std::array<> backed + static constexpr size_t kMaxCounters = 32; private: friend class PerfCounters; @@ -58,7 +70,14 @@ class PerfCounterValues { sizeof(uint64_t) * (kPadding + nr_counters_)}; } - static constexpr size_t kPadding = 1; + // This reading is complex and as the goal of this class is to + // abstract away the intrincacies of the reading process, this is + // a better place for it + size_t Read(const std::vector& leaders); + + // Move the padding to 2 due to the reading algorithm (1st padding plus a + // current read padding) + static constexpr size_t kPadding = 2; std::array values_; const size_t nr_counters_; }; @@ -66,27 +85,34 @@ class PerfCounterValues { // Collect PMU counters. The object, once constructed, is ready to be used by // calling read(). PMU counter collection is enabled from the time create() is // called, to obtain the object, until the object's destructor is called. -class PerfCounters final { +class BENCHMARK_EXPORT PerfCounters final { public: // True iff this platform supports performance counters. static const bool kSupported; - bool IsValid() const { return is_valid_; } + // Returns an empty object static PerfCounters NoCounters() { return PerfCounters(); } - ~PerfCounters(); + ~PerfCounters() { CloseCounters(); } + PerfCounters() = default; PerfCounters(PerfCounters&&) = default; PerfCounters(const PerfCounters&) = delete; + PerfCounters& operator=(PerfCounters&&) noexcept; + PerfCounters& operator=(const PerfCounters&) = delete; // Platform-specific implementations may choose to do some library // initialization here. static bool Initialize(); + // Check if the given counter is supported, if the app wants to + // check before passing + static bool IsCounterSupported(const std::string& name); + // Return a PerfCounters object ready to read the counters with the names // specified. The values are user-mode only. The counter name format is // implementation and OS specific. - // TODO: once we move to C++-17, this should be a std::optional, and then the - // IsValid() boolean can be dropped. + // In case of failure, this method will in the worst case return an + // empty object whose state will still be valid. static PerfCounters Create(const std::vector& counter_names); // Take a snapshot of the current value of the counters into the provided @@ -95,10 +121,7 @@ class PerfCounters final { BENCHMARK_ALWAYS_INLINE bool Snapshot(PerfCounterValues* values) const { #ifndef BENCHMARK_OS_WINDOWS assert(values != nullptr); - assert(IsValid()); - auto buffer = values->get_data_buffer(); - auto read_bytes = ::read(counter_ids_[0], buffer.first, buffer.second); - return static_cast(read_bytes) == buffer.second; + return values->Read(leader_ids_) == counter_ids_.size(); #else (void)values; return false; @@ -110,63 +133,68 @@ class PerfCounters final { private: PerfCounters(const std::vector& counter_names, - std::vector&& counter_ids) + std::vector&& counter_ids, std::vector&& leader_ids) : counter_ids_(std::move(counter_ids)), - counter_names_(counter_names), - is_valid_(true) {} - PerfCounters() : is_valid_(false) {} + leader_ids_(std::move(leader_ids)), + counter_names_(counter_names) {} + + void CloseCounters() const; std::vector counter_ids_; - const std::vector counter_names_; - const bool is_valid_; + std::vector leader_ids_; + std::vector counter_names_; }; // Typical usage of the above primitives. -class PerfCountersMeasurement final { +class BENCHMARK_EXPORT PerfCountersMeasurement final { public: - PerfCountersMeasurement(PerfCounters&& c) - : counters_(std::move(c)), - start_values_(counters_.IsValid() ? counters_.names().size() : 0), - end_values_(counters_.IsValid() ? counters_.names().size() : 0) {} + PerfCountersMeasurement(const std::vector& counter_names); + + size_t num_counters() const { return counters_.num_counters(); } - bool IsValid() const { return counters_.IsValid(); } + std::vector names() const { return counters_.names(); } - BENCHMARK_ALWAYS_INLINE void Start() { - assert(IsValid()); + BENCHMARK_ALWAYS_INLINE bool Start() { + if (num_counters() == 0) return true; // Tell the compiler to not move instructions above/below where we take // the snapshot. ClobberMemory(); - counters_.Snapshot(&start_values_); + valid_read_ &= counters_.Snapshot(&start_values_); ClobberMemory(); + + return valid_read_; } - BENCHMARK_ALWAYS_INLINE std::vector> - StopAndGetMeasurements() { - assert(IsValid()); + BENCHMARK_ALWAYS_INLINE bool Stop( + std::vector>& measurements) { + if (num_counters() == 0) return true; // Tell the compiler to not move instructions above/below where we take // the snapshot. ClobberMemory(); - counters_.Snapshot(&end_values_); + valid_read_ &= counters_.Snapshot(&end_values_); ClobberMemory(); - std::vector> ret; for (size_t i = 0; i < counters_.names().size(); ++i) { double measurement = static_cast(end_values_[i]) - static_cast(start_values_[i]); - ret.push_back({counters_.names()[i], measurement}); + measurements.push_back({counters_.names()[i], measurement}); } - return ret; + + return valid_read_; } private: PerfCounters counters_; + bool valid_read_ = true; PerfCounterValues start_values_; PerfCounterValues end_values_; }; -BENCHMARK_UNUSED static bool perf_init_anchor = PerfCounters::Initialize(); - } // namespace internal } // namespace benchmark +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + #endif // BENCHMARK_PERF_COUNTERS_H diff --git a/third-party/benchmark/src/re.h b/third-party/benchmark/src/re.h index 630046782dc4..9afb869bea27 100644 --- a/third-party/benchmark/src/re.h +++ b/third-party/benchmark/src/re.h @@ -33,7 +33,7 @@ // Prefer C regex libraries when compiling w/o exceptions so that we can // correctly report errors. #if defined(BENCHMARK_HAS_NO_EXCEPTIONS) && \ - defined(BENCHMARK_HAVE_STD_REGEX) && \ + defined(HAVE_STD_REGEX) && \ (defined(HAVE_GNU_POSIX_REGEX) || defined(HAVE_POSIX_REGEX)) #undef HAVE_STD_REGEX #endif diff --git a/third-party/benchmark/src/reporter.cc b/third-party/benchmark/src/reporter.cc index 1d2df17b90f0..076bc31a2ecc 100644 --- a/third-party/benchmark/src/reporter.cc +++ b/third-party/benchmark/src/reporter.cc @@ -25,9 +25,6 @@ #include "timers.h" namespace benchmark { -namespace internal { -extern std::map *global_context; -} BenchmarkReporter::BenchmarkReporter() : output_stream_(&std::cout), error_stream_(&std::cerr) {} @@ -39,7 +36,11 @@ void BenchmarkReporter::PrintBasicContext(std::ostream *out, BM_CHECK(out) << "cannot be null"; auto &Out = *out; +#ifndef BENCHMARK_OS_QURT + // Date/time information is not available on QuRT. + // Attempting to get it via this call cause the binary to crash. Out << LocalDateTimeString() << "\n"; +#endif if (context.executable_name) Out << "Running " << context.executable_name << "\n"; @@ -67,8 +68,11 @@ void BenchmarkReporter::PrintBasicContext(std::ostream *out, Out << "\n"; } - if (internal::global_context != nullptr) { - for (const auto &kv : *internal::global_context) { + std::map *global_context = + internal::GetGlobalContext(); + + if (global_context != nullptr) { + for (const auto &kv : *global_context) { Out << kv.first << ": " << kv.second << "\n"; } } diff --git a/third-party/benchmark/src/sleep.cc b/third-party/benchmark/src/sleep.cc deleted file mode 100644 index ab59000f24ad..000000000000 --- a/third-party/benchmark/src/sleep.cc +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2015 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "sleep.h" - -#include -#include -#include - -#include "internal_macros.h" - -#ifdef BENCHMARK_OS_WINDOWS -#include -#endif - -#ifdef BENCHMARK_OS_ZOS -#include -#endif - -namespace benchmark { -#ifdef BENCHMARK_OS_WINDOWS -// Window's Sleep takes milliseconds argument. -void SleepForMilliseconds(int milliseconds) { Sleep(milliseconds); } -void SleepForSeconds(double seconds) { - SleepForMilliseconds(static_cast(kNumMillisPerSecond * seconds)); -} -#else // BENCHMARK_OS_WINDOWS -void SleepForMicroseconds(int microseconds) { -#ifdef BENCHMARK_OS_ZOS - // z/OS does not support nanosleep. Instead call sleep() and then usleep() to - // sleep for the remaining microseconds because usleep() will fail if its - // argument is greater than 1000000. - div_t sleepTime = div(microseconds, kNumMicrosPerSecond); - int seconds = sleepTime.quot; - while (seconds != 0) seconds = sleep(seconds); - while (usleep(sleepTime.rem) == -1 && errno == EINTR) - ; -#else - struct timespec sleep_time; - sleep_time.tv_sec = microseconds / kNumMicrosPerSecond; - sleep_time.tv_nsec = (microseconds % kNumMicrosPerSecond) * kNumNanosPerMicro; - while (nanosleep(&sleep_time, &sleep_time) != 0 && errno == EINTR) - ; // Ignore signals and wait for the full interval to elapse. -#endif -} - -void SleepForMilliseconds(int milliseconds) { - SleepForMicroseconds(milliseconds * kNumMicrosPerMilli); -} - -void SleepForSeconds(double seconds) { - SleepForMicroseconds(static_cast(seconds * kNumMicrosPerSecond)); -} -#endif // BENCHMARK_OS_WINDOWS -} // end namespace benchmark diff --git a/third-party/benchmark/src/sleep.h b/third-party/benchmark/src/sleep.h deleted file mode 100644 index f98551afe284..000000000000 --- a/third-party/benchmark/src/sleep.h +++ /dev/null @@ -1,15 +0,0 @@ -#ifndef BENCHMARK_SLEEP_H_ -#define BENCHMARK_SLEEP_H_ - -namespace benchmark { -const int kNumMillisPerSecond = 1000; -const int kNumMicrosPerMilli = 1000; -const int kNumMicrosPerSecond = kNumMillisPerSecond * 1000; -const int kNumNanosPerMicro = 1000; -const int kNumNanosPerSecond = kNumNanosPerMicro * kNumMicrosPerSecond; - -void SleepForMilliseconds(int milliseconds); -void SleepForSeconds(double seconds); -} // end namespace benchmark - -#endif // BENCHMARK_SLEEP_H_ diff --git a/third-party/benchmark/src/statistics.cc b/third-party/benchmark/src/statistics.cc index 3e5ef0993971..261dcb299a67 100644 --- a/third-party/benchmark/src/statistics.cc +++ b/third-party/benchmark/src/statistics.cc @@ -32,7 +32,7 @@ auto StatisticsSum = [](const std::vector& v) { double StatisticsMean(const std::vector& v) { if (v.empty()) return 0.0; - return StatisticsSum(v) * (1.0 / v.size()); + return StatisticsSum(v) * (1.0 / static_cast(v.size())); } double StatisticsMedian(const std::vector& v) { @@ -42,13 +42,13 @@ double StatisticsMedian(const std::vector& v) { auto center = copy.begin() + v.size() / 2; std::nth_element(copy.begin(), center, copy.end()); - // did we have an odd number of samples? - // if yes, then center is the median - // it no, then we are looking for the average between center and the value - // before + // Did we have an odd number of samples? If yes, then center is the median. + // If not, then we are looking for the average between center and the value + // before. Instead of resorting, we just look for the max value before it, + // which is not necessarily the element immediately preceding `center` Since + // `copy` is only partially sorted by `nth_element`. if (v.size() % 2 == 1) return *center; - auto center2 = copy.begin() + v.size() / 2 - 1; - std::nth_element(copy.begin(), center2, copy.end()); + auto center2 = std::max_element(copy.begin(), center); return (*center + *center2) / 2.0; } @@ -71,8 +71,11 @@ double StatisticsStdDev(const std::vector& v) { // Sample standard deviation is undefined for n = 1 if (v.size() == 1) return 0.0; - const double avg_squares = SumSquares(v) * (1.0 / v.size()); - return Sqrt(v.size() / (v.size() - 1.0) * (avg_squares - Sqr(mean))); + const double avg_squares = + SumSquares(v) * (1.0 / static_cast(v.size())); + return Sqrt(static_cast(v.size()) / + (static_cast(v.size()) - 1.0) * + (avg_squares - Sqr(mean))); } double StatisticsCV(const std::vector& v) { @@ -81,6 +84,8 @@ double StatisticsCV(const std::vector& v) { const auto stddev = StatisticsStdDev(v); const auto mean = StatisticsMean(v); + if (std::fpclassify(mean) == FP_ZERO) return 0.0; + return stddev / mean; } @@ -89,9 +94,8 @@ std::vector ComputeStats( typedef BenchmarkReporter::Run Run; std::vector results; - auto error_count = - std::count_if(reports.begin(), reports.end(), - [](Run const& run) { return run.error_occurred; }); + auto error_count = std::count_if(reports.begin(), reports.end(), + [](Run const& run) { return run.skipped; }); if (reports.size() - error_count < 2) { // We don't report aggregated data if there was a single run. @@ -118,11 +122,13 @@ std::vector ComputeStats( for (auto const& cnt : r.counters) { auto it = counter_stats.find(cnt.first); if (it == counter_stats.end()) { - counter_stats.insert({cnt.first, {cnt.second, std::vector{}}}); - it = counter_stats.find(cnt.first); + it = counter_stats + .emplace(cnt.first, + CounterStat{cnt.second, std::vector{}}) + .first; it->second.s.reserve(reports.size()); } else { - BM_CHECK_EQ(counter_stats[cnt.first].c.flags, cnt.second.flags); + BM_CHECK_EQ(it->second.c.flags, cnt.second.flags); } } } @@ -131,7 +137,7 @@ std::vector ComputeStats( for (Run const& run : reports) { BM_CHECK_EQ(reports[0].benchmark_name(), run.benchmark_name()); BM_CHECK_EQ(run_iterations, run.iterations); - if (run.error_occurred) continue; + if (run.skipped) continue; real_accumulated_time_stat.emplace_back(run.real_accumulated_time); cpu_accumulated_time_stat.emplace_back(run.cpu_accumulated_time); // user counters diff --git a/third-party/benchmark/src/statistics.h b/third-party/benchmark/src/statistics.h index a9545a58c648..6e5560e8f19f 100644 --- a/third-party/benchmark/src/statistics.h +++ b/third-party/benchmark/src/statistics.h @@ -22,15 +22,21 @@ namespace benchmark { -// Return a vector containing the mean, median and standard devation information -// (and any user-specified info) for the specified list of reports. If 'reports' -// contains less than two non-errored runs an empty vector is returned +// Return a vector containing the mean, median and standard deviation +// information (and any user-specified info) for the specified list of reports. +// If 'reports' contains less than two non-errored runs an empty vector is +// returned +BENCHMARK_EXPORT std::vector ComputeStats( const std::vector& reports); +BENCHMARK_EXPORT double StatisticsMean(const std::vector& v); +BENCHMARK_EXPORT double StatisticsMedian(const std::vector& v); +BENCHMARK_EXPORT double StatisticsStdDev(const std::vector& v); +BENCHMARK_EXPORT double StatisticsCV(const std::vector& v); } // end namespace benchmark diff --git a/third-party/benchmark/src/string_util.cc b/third-party/benchmark/src/string_util.cc index 401fa13df7af..c69e40a8133c 100644 --- a/third-party/benchmark/src/string_util.cc +++ b/third-party/benchmark/src/string_util.cc @@ -11,16 +11,17 @@ #include #include "arraysize.h" +#include "benchmark/benchmark.h" namespace benchmark { namespace { - // kilo, Mega, Giga, Tera, Peta, Exa, Zetta, Yotta. -const char kBigSIUnits[] = "kMGTPEZY"; +const char* const kBigSIUnits[] = {"k", "M", "G", "T", "P", "E", "Z", "Y"}; // Kibi, Mebi, Gibi, Tebi, Pebi, Exbi, Zebi, Yobi. -const char kBigIECUnits[] = "KMGTPEZY"; +const char* const kBigIECUnits[] = {"Ki", "Mi", "Gi", "Ti", + "Pi", "Ei", "Zi", "Yi"}; // milli, micro, nano, pico, femto, atto, zepto, yocto. -const char kSmallSIUnits[] = "munpfazy"; +const char* const kSmallSIUnits[] = {"m", "u", "n", "p", "f", "a", "z", "y"}; // We require that all three arrays have the same size. static_assert(arraysize(kBigSIUnits) == arraysize(kBigIECUnits), @@ -30,9 +31,8 @@ static_assert(arraysize(kSmallSIUnits) == arraysize(kBigSIUnits), static const int64_t kUnitsSize = arraysize(kBigSIUnits); -void ToExponentAndMantissa(double val, double thresh, int precision, - double one_k, std::string* mantissa, - int64_t* exponent) { +void ToExponentAndMantissa(double val, int precision, double one_k, + std::string* mantissa, int64_t* exponent) { std::stringstream mantissa_stream; if (val < 0) { @@ -43,8 +43,8 @@ void ToExponentAndMantissa(double val, double thresh, int precision, // Adjust threshold so that it never excludes things which can't be rendered // in 'precision' digits. const double adjusted_threshold = - std::max(thresh, 1.0 / std::pow(10.0, precision)); - const double big_threshold = adjusted_threshold * one_k; + std::max(1.0, 1.0 / std::pow(10.0, precision)); + const double big_threshold = (adjusted_threshold * one_k) - 1; const double small_threshold = adjusted_threshold; // Values in ]simple_threshold,small_threshold[ will be printed as-is const double simple_threshold = 0.01; @@ -92,37 +92,20 @@ std::string ExponentToPrefix(int64_t exponent, bool iec) { const int64_t index = (exponent > 0 ? exponent - 1 : -exponent - 1); if (index >= kUnitsSize) return ""; - const char* array = + const char* const* array = (exponent > 0 ? (iec ? kBigIECUnits : kBigSIUnits) : kSmallSIUnits); - if (iec) - return array[index] + std::string("i"); - else - return std::string(1, array[index]); + + return std::string(array[index]); } -std::string ToBinaryStringFullySpecified(double value, double threshold, - int precision, double one_k = 1024.0) { +std::string ToBinaryStringFullySpecified(double value, int precision, + Counter::OneK one_k) { std::string mantissa; int64_t exponent; - ToExponentAndMantissa(value, threshold, precision, one_k, &mantissa, + ToExponentAndMantissa(value, precision, + one_k == Counter::kIs1024 ? 1024.0 : 1000.0, &mantissa, &exponent); - return mantissa + ExponentToPrefix(exponent, false); -} - -} // end namespace - -void AppendHumanReadable(int n, std::string* str) { - std::stringstream ss; - // Round down to the nearest SI prefix. - ss << ToBinaryStringFullySpecified(n, 1.0, 0); - *str += ss.str(); -} - -std::string HumanReadableNumber(double n, double one_k) { - // 1.1 means that figures up to 1.1k should be shown with the next unit down; - // this softens edge effects. - // 1 means that we should show one decimal place of precision. - return ToBinaryStringFullySpecified(n, 1.1, 1, one_k); + return mantissa + ExponentToPrefix(exponent, one_k == Counter::kIs1024); } std::string StrFormatImp(const char* msg, va_list args) { @@ -133,21 +116,21 @@ std::string StrFormatImp(const char* msg, va_list args) { // TODO(ericwf): use std::array for first attempt to avoid one memory // allocation guess what the size might be std::array local_buff; - std::size_t size = local_buff.size(); + // 2015-10-08: vsnprintf is used instead of snd::vsnprintf due to a limitation // in the android-ndk - auto ret = vsnprintf(local_buff.data(), size, msg, args_cp); + auto ret = vsnprintf(local_buff.data(), local_buff.size(), msg, args_cp); va_end(args_cp); // handle empty expansion if (ret == 0) return std::string{}; - if (static_cast(ret) < size) + if (static_cast(ret) < local_buff.size()) return std::string(local_buff.data()); // we did not provide a long enough buffer on our first attempt. // add 1 to size to account for null-byte in size cast to prevent overflow - size = static_cast(ret) + 1; + std::size_t size = static_cast(ret) + 1; auto buff_ptr = std::unique_ptr(new char[size]); // 2015-10-08: vsnprintf is used instead of snd::vsnprintf due to a limitation // in the android-ndk @@ -155,6 +138,12 @@ std::string StrFormatImp(const char* msg, va_list args) { return std::string(buff_ptr.get()); } +} // end namespace + +std::string HumanReadableNumber(double n, Counter::OneK one_k) { + return ToBinaryStringFullySpecified(n, 1, one_k); +} + std::string StrFormat(const char* format, ...) { va_list args; va_start(args, format); diff --git a/third-party/benchmark/src/string_util.h b/third-party/benchmark/src/string_util.h index ff3b7da47d6c..731aa2c04c3e 100644 --- a/third-party/benchmark/src/string_util.h +++ b/third-party/benchmark/src/string_util.h @@ -4,15 +4,19 @@ #include #include #include +#include +#include "benchmark/benchmark.h" +#include "benchmark/export.h" +#include "check.h" #include "internal_macros.h" namespace benchmark { -void AppendHumanReadable(int n, std::string* str); - -std::string HumanReadableNumber(double n, double one_k = 1024.0); +BENCHMARK_EXPORT +std::string HumanReadableNumber(double n, Counter::OneK one_k); +BENCHMARK_EXPORT #if defined(__MINGW32__) __attribute__((format(__MINGW_PRINTF_FORMAT, 1, 2))) #elif defined(__GNUC__) @@ -38,6 +42,7 @@ inline std::string StrCat(Args&&... args) { return ss.str(); } +BENCHMARK_EXPORT std::vector StrSplit(const std::string& str, char delim); // Disable lint checking for this block since it re-implements C functions. diff --git a/third-party/benchmark/src/sysinfo.cc b/third-party/benchmark/src/sysinfo.cc index 3a56e8cace48..46df973b069a 100644 --- a/third-party/benchmark/src/sysinfo.cc +++ b/third-party/benchmark/src/sysinfo.cc @@ -22,6 +22,10 @@ #include "internal_macros.h" #ifdef BENCHMARK_OS_WINDOWS +#if !defined(WINVER) || WINVER < 0x0600 +#undef WINVER +#define WINVER 0x0600 +#endif // WINVER handling #include #undef StrCat // Don't let StrCat in string_util.h be renamed to lstrcatA #include @@ -30,7 +34,7 @@ #include #else #include -#ifndef BENCHMARK_OS_FUCHSIA +#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) #include #endif #include @@ -45,10 +49,17 @@ #endif #if defined(BENCHMARK_OS_SOLARIS) #include +#include #endif #if defined(BENCHMARK_OS_QNX) #include #endif +#if defined(BENCHMARK_OS_QURT) +#include +#endif +#if defined(BENCHMARK_HAS_PTHREAD_AFFINITY) +#include +#endif #include #include @@ -65,15 +76,17 @@ #include #include #include +#include #include #include +#include "benchmark/benchmark.h" #include "check.h" #include "cycleclock.h" #include "internal_macros.h" #include "log.h" -#include "sleep.h" #include "string_util.h" +#include "timers.h" namespace benchmark { namespace { @@ -98,67 +111,59 @@ BENCHMARK_NORETURN void PrintErrorAndDie(Args&&... args) { /// `sysctl` with the result type it's to be interpreted as. struct ValueUnion { union DataT { - uint32_t uint32_value; - uint64_t uint64_value; + int32_t int32_value; + int64_t int64_value; // For correct aliasing of union members from bytes. char bytes[8]; }; using DataPtr = std::unique_ptr; // The size of the data union member + its trailing array size. - size_t Size; - DataPtr Buff; + std::size_t size; + DataPtr buff; public: - ValueUnion() : Size(0), Buff(nullptr, &std::free) {} + ValueUnion() : size(0), buff(nullptr, &std::free) {} - explicit ValueUnion(size_t BuffSize) - : Size(sizeof(DataT) + BuffSize), - Buff(::new (std::malloc(Size)) DataT(), &std::free) {} + explicit ValueUnion(std::size_t buff_size) + : size(sizeof(DataT) + buff_size), + buff(::new (std::malloc(size)) DataT(), &std::free) {} ValueUnion(ValueUnion&& other) = default; - explicit operator bool() const { return bool(Buff); } + explicit operator bool() const { return bool(buff); } - char* data() const { return Buff->bytes; } + char* data() const { return buff->bytes; } std::string GetAsString() const { return std::string(data()); } int64_t GetAsInteger() const { - if (Size == sizeof(Buff->uint32_value)) - return static_cast(Buff->uint32_value); - else if (Size == sizeof(Buff->uint64_value)) - return static_cast(Buff->uint64_value); - BENCHMARK_UNREACHABLE(); - } - - uint64_t GetAsUnsigned() const { - if (Size == sizeof(Buff->uint32_value)) - return Buff->uint32_value; - else if (Size == sizeof(Buff->uint64_value)) - return Buff->uint64_value; + if (size == sizeof(buff->int32_value)) + return buff->int32_value; + else if (size == sizeof(buff->int64_value)) + return buff->int64_value; BENCHMARK_UNREACHABLE(); } template std::array GetAsArray() { - const int ArrSize = sizeof(T) * N; - BM_CHECK_LE(ArrSize, Size); - std::array Arr; - std::memcpy(Arr.data(), data(), ArrSize); - return Arr; + const int arr_size = sizeof(T) * N; + BM_CHECK_LE(arr_size, size); + std::array arr; + std::memcpy(arr.data(), data(), arr_size); + return arr; } }; -ValueUnion GetSysctlImp(std::string const& Name) { +ValueUnion GetSysctlImp(std::string const& name) { #if defined BENCHMARK_OS_OPENBSD int mib[2]; mib[0] = CTL_HW; - if ((Name == "hw.ncpu") || (Name == "hw.cpuspeed")) { + if ((name == "hw.ncpu") || (name == "hw.cpuspeed")) { ValueUnion buff(sizeof(int)); - if (Name == "hw.ncpu") { + if (name == "hw.ncpu") { mib[1] = HW_NCPU; } else { mib[1] = HW_CPUSPEED; @@ -171,41 +176,41 @@ ValueUnion GetSysctlImp(std::string const& Name) { } return ValueUnion(); #else - size_t CurBuffSize = 0; - if (sysctlbyname(Name.c_str(), nullptr, &CurBuffSize, nullptr, 0) == -1) + std::size_t cur_buff_size = 0; + if (sysctlbyname(name.c_str(), nullptr, &cur_buff_size, nullptr, 0) == -1) return ValueUnion(); - ValueUnion buff(CurBuffSize); - if (sysctlbyname(Name.c_str(), buff.data(), &buff.Size, nullptr, 0) == 0) + ValueUnion buff(cur_buff_size); + if (sysctlbyname(name.c_str(), buff.data(), &buff.size, nullptr, 0) == 0) return buff; return ValueUnion(); #endif } BENCHMARK_MAYBE_UNUSED -bool GetSysctl(std::string const& Name, std::string* Out) { - Out->clear(); - auto Buff = GetSysctlImp(Name); - if (!Buff) return false; - Out->assign(Buff.data()); +bool GetSysctl(std::string const& name, std::string* out) { + out->clear(); + auto buff = GetSysctlImp(name); + if (!buff) return false; + out->assign(buff.data()); return true; } template ::value>::type> -bool GetSysctl(std::string const& Name, Tp* Out) { - *Out = 0; - auto Buff = GetSysctlImp(Name); - if (!Buff) return false; - *Out = static_cast(Buff.GetAsUnsigned()); +bool GetSysctl(std::string const& name, Tp* out) { + *out = 0; + auto buff = GetSysctlImp(name); + if (!buff) return false; + *out = static_cast(buff.GetAsInteger()); return true; } template -bool GetSysctl(std::string const& Name, std::array* Out) { - auto Buff = GetSysctlImp(Name); - if (!Buff) return false; - *Out = Buff.GetAsArray(); +bool GetSysctl(std::string const& name, std::array* out) { + auto buff = GetSysctlImp(name); + if (!buff) return false; + *out = buff.GetAsArray(); return true; } #endif @@ -241,21 +246,21 @@ CPUInfo::Scaling CpuScaling(int num_cpus) { #endif } -int CountSetBitsInCPUMap(std::string Val) { - auto CountBits = [](std::string Part) { +int CountSetBitsInCPUMap(std::string val) { + auto CountBits = [](std::string part) { using CPUMask = std::bitset; - Part = "0x" + Part; - CPUMask Mask(benchmark::stoul(Part, nullptr, 16)); - return static_cast(Mask.count()); + part = "0x" + part; + CPUMask mask(benchmark::stoul(part, nullptr, 16)); + return static_cast(mask.count()); }; - size_t Pos; + std::size_t pos; int total = 0; - while ((Pos = Val.find(',')) != std::string::npos) { - total += CountBits(Val.substr(0, Pos)); - Val = Val.substr(Pos + 1); + while ((pos = val.find(',')) != std::string::npos) { + total += CountBits(val.substr(0, pos)); + val = val.substr(pos + 1); } - if (!Val.empty()) { - total += CountBits(Val); + if (!val.empty()) { + total += CountBits(val); } return total; } @@ -264,16 +269,16 @@ BENCHMARK_MAYBE_UNUSED std::vector GetCacheSizesFromKVFS() { std::vector res; std::string dir = "/sys/devices/system/cpu/cpu0/cache/"; - int Idx = 0; + int idx = 0; while (true) { CPUInfo::CacheInfo info; - std::string FPath = StrCat(dir, "index", Idx++, "/"); - std::ifstream f(StrCat(FPath, "size").c_str()); + std::string fpath = StrCat(dir, "index", idx++, "/"); + std::ifstream f(StrCat(fpath, "size").c_str()); if (!f.is_open()) break; std::string suffix; f >> info.size; if (f.fail()) - PrintErrorAndDie("Failed while reading file '", FPath, "size'"); + PrintErrorAndDie("Failed while reading file '", fpath, "size'"); if (f.good()) { f >> suffix; if (f.bad()) @@ -284,13 +289,13 @@ std::vector GetCacheSizesFromKVFS() { else if (suffix == "K") info.size *= 1024; } - if (!ReadFromFile(StrCat(FPath, "type"), &info.type)) - PrintErrorAndDie("Failed to read from file ", FPath, "type"); - if (!ReadFromFile(StrCat(FPath, "level"), &info.level)) - PrintErrorAndDie("Failed to read from file ", FPath, "level"); + if (!ReadFromFile(StrCat(fpath, "type"), &info.type)) + PrintErrorAndDie("Failed to read from file ", fpath, "type"); + if (!ReadFromFile(StrCat(fpath, "level"), &info.level)) + PrintErrorAndDie("Failed to read from file ", fpath, "level"); std::string map_str; - if (!ReadFromFile(StrCat(FPath, "shared_cpu_map"), &map_str)) - PrintErrorAndDie("Failed to read from file ", FPath, "shared_cpu_map"); + if (!ReadFromFile(StrCat(fpath, "shared_cpu_map"), &map_str)) + PrintErrorAndDie("Failed to read from file ", fpath, "shared_cpu_map"); info.num_sharing = CountSetBitsInCPUMap(map_str); res.push_back(info); } @@ -301,26 +306,26 @@ std::vector GetCacheSizesFromKVFS() { #ifdef BENCHMARK_OS_MACOSX std::vector GetCacheSizesMacOSX() { std::vector res; - std::array CacheCounts{{0, 0, 0, 0}}; - GetSysctl("hw.cacheconfig", &CacheCounts); + std::array cache_counts{{0, 0, 0, 0}}; + GetSysctl("hw.cacheconfig", &cache_counts); struct { std::string name; std::string type; int level; - uint64_t num_sharing; - } Cases[] = {{"hw.l1dcachesize", "Data", 1, CacheCounts[1]}, - {"hw.l1icachesize", "Instruction", 1, CacheCounts[1]}, - {"hw.l2cachesize", "Unified", 2, CacheCounts[2]}, - {"hw.l3cachesize", "Unified", 3, CacheCounts[3]}}; - for (auto& C : Cases) { + int num_sharing; + } cases[] = {{"hw.l1dcachesize", "Data", 1, cache_counts[1]}, + {"hw.l1icachesize", "Instruction", 1, cache_counts[1]}, + {"hw.l2cachesize", "Unified", 2, cache_counts[2]}, + {"hw.l3cachesize", "Unified", 3, cache_counts[3]}}; + for (auto& c : cases) { int val; - if (!GetSysctl(C.name, &val)) continue; + if (!GetSysctl(c.name, &val)) continue; CPUInfo::CacheInfo info; - info.type = C.type; - info.level = C.level; + info.type = c.type; + info.level = c.level; info.size = val; - info.num_sharing = static_cast(C.num_sharing); + info.num_sharing = c.num_sharing; res.push_back(std::move(info)); } return res; @@ -334,7 +339,7 @@ std::vector GetCacheSizesWindows() { using UPtr = std::unique_ptr; GetLogicalProcessorInformation(nullptr, &buffer_size); - UPtr buff((PInfo*)malloc(buffer_size), &std::free); + UPtr buff(static_cast(std::malloc(buffer_size)), &std::free); if (!GetLogicalProcessorInformation(buff.get(), &buffer_size)) PrintErrorAndDie("Failed during call to GetLogicalProcessorInformation: ", GetLastError()); @@ -345,16 +350,16 @@ std::vector GetCacheSizesWindows() { for (; it != end; ++it) { if (it->Relationship != RelationCache) continue; using BitSet = std::bitset; - BitSet B(it->ProcessorMask); + BitSet b(it->ProcessorMask); // To prevent duplicates, only consider caches where CPU 0 is specified - if (!B.test(0)) continue; - CInfo* Cache = &it->Cache; + if (!b.test(0)) continue; + const CInfo& cache = it->Cache; CPUInfo::CacheInfo C; - C.num_sharing = static_cast(B.count()); - C.level = Cache->Level; - C.size = Cache->Size; + C.num_sharing = static_cast(b.count()); + C.level = cache.Level; + C.size = cache.Size; C.type = "Unknown"; - switch (Cache->Type) { + switch (cache.Type) { case CacheUnified: C.type = "Unified"; break; @@ -417,6 +422,8 @@ std::vector GetCacheSizes() { return GetCacheSizesWindows(); #elif defined(BENCHMARK_OS_QNX) return GetCacheSizesQNX(); +#elif defined(BENCHMARK_OS_QURT) + return std::vector(); #else return GetCacheSizesFromKVFS(); #endif @@ -425,23 +432,32 @@ std::vector GetCacheSizes() { std::string GetSystemName() { #if defined(BENCHMARK_OS_WINDOWS) std::string str; - const unsigned COUNT = MAX_COMPUTERNAME_LENGTH + 1; + static constexpr int COUNT = MAX_COMPUTERNAME_LENGTH + 1; TCHAR hostname[COUNT] = {'\0'}; DWORD DWCOUNT = COUNT; if (!GetComputerName(hostname, &DWCOUNT)) return std::string(""); #ifndef UNICODE str = std::string(hostname, DWCOUNT); #else - // Using wstring_convert, Is deprecated in C++17 - using convert_type = std::codecvt_utf8; - std::wstring_convert converter; - std::wstring wStr(hostname, DWCOUNT); - str = converter.to_bytes(wStr); + // `WideCharToMultiByte` returns `0` when conversion fails. + int len = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, hostname, + DWCOUNT, NULL, 0, NULL, NULL); + str.resize(len); + WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, hostname, DWCOUNT, &str[0], + str.size(), NULL, NULL); #endif return str; -#else // defined(BENCHMARK_OS_WINDOWS) +#elif defined(BENCHMARK_OS_QURT) + std::string str = "Hexagon DSP"; + qurt_arch_version_t arch_version_struct; + if (qurt_sysenv_get_arch_version(&arch_version_struct) == QURT_EOK) { + str += " v"; + str += std::to_string(arch_version_struct.arch_version); + } + return str; +#else #ifndef HOST_NAME_MAX -#ifdef BENCHMARK_HAS_SYSCTL // BSD/Mac Doesnt have HOST_NAME_MAX defined +#ifdef BENCHMARK_HAS_SYSCTL // BSD/Mac doesn't have HOST_NAME_MAX defined #define HOST_NAME_MAX 64 #elif defined(BENCHMARK_OS_NACL) #define HOST_NAME_MAX 64 @@ -449,6 +465,8 @@ std::string GetSystemName() { #define HOST_NAME_MAX 154 #elif defined(BENCHMARK_OS_RTEMS) #define HOST_NAME_MAX 256 +#elif defined(BENCHMARK_OS_SOLARIS) +#define HOST_NAME_MAX MAXHOSTNAMELEN #elif defined(BENCHMARK_OS_ZOS) #define HOST_NAME_MAX _POSIX_HOST_NAME_MAX #else @@ -463,12 +481,11 @@ std::string GetSystemName() { #endif // Catch-all POSIX block. } -int GetNumCPUs() { +int GetNumCPUsImpl() { #ifdef BENCHMARK_HAS_SYSCTL - int NumCPU = -1; - if (GetSysctl("hw.ncpu", &NumCPU)) return NumCPU; - fprintf(stderr, "Err: %s\n", strerror(errno)); - std::exit(EXIT_FAILURE); + int num_cpu = -1; + if (GetSysctl("hw.ncpu", &num_cpu)) return num_cpu; + PrintErrorAndDie("Err: ", strerror(errno)); #elif defined(BENCHMARK_OS_WINDOWS) SYSTEM_INFO sysinfo; // Use memset as opposed to = {} to avoid GCC missing initializer false @@ -480,64 +497,155 @@ int GetNumCPUs() { // group #elif defined(BENCHMARK_OS_SOLARIS) // Returns -1 in case of a failure. - int NumCPU = sysconf(_SC_NPROCESSORS_ONLN); - if (NumCPU < 0) { - fprintf(stderr, "sysconf(_SC_NPROCESSORS_ONLN) failed with error: %s\n", - strerror(errno)); + long num_cpu = sysconf(_SC_NPROCESSORS_ONLN); + if (num_cpu < 0) { + PrintErrorAndDie("sysconf(_SC_NPROCESSORS_ONLN) failed with error: ", + strerror(errno)); } - return NumCPU; + return (int)num_cpu; #elif defined(BENCHMARK_OS_QNX) return static_cast(_syspage_ptr->num_cpu); +#elif defined(BENCHMARK_OS_QURT) + qurt_sysenv_max_hthreads_t hardware_threads; + if (qurt_sysenv_get_max_hw_threads(&hardware_threads) != QURT_EOK) { + hardware_threads.max_hthreads = 1; + } + return hardware_threads.max_hthreads; #else - int NumCPUs = 0; - int MaxID = -1; + int num_cpus = 0; + int max_id = -1; std::ifstream f("/proc/cpuinfo"); if (!f.is_open()) { - std::cerr << "failed to open /proc/cpuinfo\n"; - return -1; + PrintErrorAndDie("Failed to open /proc/cpuinfo"); } +#if defined(__alpha__) + const std::string Key = "cpus detected"; +#else const std::string Key = "processor"; +#endif std::string ln; while (std::getline(f, ln)) { if (ln.empty()) continue; - size_t SplitIdx = ln.find(':'); + std::size_t split_idx = ln.find(':'); std::string value; #if defined(__s390__) // s390 has another format in /proc/cpuinfo // it needs to be parsed differently - if (SplitIdx != std::string::npos) - value = ln.substr(Key.size() + 1, SplitIdx - Key.size() - 1); + if (split_idx != std::string::npos) + value = ln.substr(Key.size() + 1, split_idx - Key.size() - 1); #else - if (SplitIdx != std::string::npos) value = ln.substr(SplitIdx + 1); + if (split_idx != std::string::npos) value = ln.substr(split_idx + 1); #endif if (ln.size() >= Key.size() && ln.compare(0, Key.size(), Key) == 0) { - NumCPUs++; + num_cpus++; if (!value.empty()) { - int CurID = benchmark::stoi(value); - MaxID = std::max(CurID, MaxID); + const int cur_id = benchmark::stoi(value); + max_id = std::max(cur_id, max_id); } } } if (f.bad()) { - std::cerr << "Failure reading /proc/cpuinfo\n"; - return -1; + PrintErrorAndDie("Failure reading /proc/cpuinfo"); } if (!f.eof()) { - std::cerr << "Failed to read to end of /proc/cpuinfo\n"; - return -1; + PrintErrorAndDie("Failed to read to end of /proc/cpuinfo"); } f.close(); - if ((MaxID + 1) != NumCPUs) { + if ((max_id + 1) != num_cpus) { fprintf(stderr, "CPU ID assignments in /proc/cpuinfo seem messed up." " This is usually caused by a bad BIOS.\n"); } - return NumCPUs; + return num_cpus; #endif BENCHMARK_UNREACHABLE(); } +int GetNumCPUs() { + const int num_cpus = GetNumCPUsImpl(); + if (num_cpus < 1) { + PrintErrorAndDie( + "Unable to extract number of CPUs. If your platform uses " + "/proc/cpuinfo, custom support may need to be added."); + } + return num_cpus; +} + +class ThreadAffinityGuard final { + public: + ThreadAffinityGuard() : reset_affinity(SetAffinity()) { + if (!reset_affinity) + std::cerr << "***WARNING*** Failed to set thread affinity. Estimated CPU " + "frequency may be incorrect." + << std::endl; + } + + ~ThreadAffinityGuard() { + if (!reset_affinity) return; + +#if defined(BENCHMARK_HAS_PTHREAD_AFFINITY) + int ret = pthread_setaffinity_np(self, sizeof(previous_affinity), + &previous_affinity); + if (ret == 0) return; +#elif defined(BENCHMARK_OS_WINDOWS_WIN32) + DWORD_PTR ret = SetThreadAffinityMask(self, previous_affinity); + if (ret != 0) return; +#endif // def BENCHMARK_HAS_PTHREAD_AFFINITY + PrintErrorAndDie("Failed to reset thread affinity"); + } + + ThreadAffinityGuard(ThreadAffinityGuard&&) = delete; + ThreadAffinityGuard(const ThreadAffinityGuard&) = delete; + ThreadAffinityGuard& operator=(ThreadAffinityGuard&&) = delete; + ThreadAffinityGuard& operator=(const ThreadAffinityGuard&) = delete; + + private: + bool SetAffinity() { +#if defined(BENCHMARK_HAS_PTHREAD_AFFINITY) + int ret; + self = pthread_self(); + ret = pthread_getaffinity_np(self, sizeof(previous_affinity), + &previous_affinity); + if (ret != 0) return false; + + cpu_set_t affinity; + memcpy(&affinity, &previous_affinity, sizeof(affinity)); + + bool is_first_cpu = true; + + for (int i = 0; i < CPU_SETSIZE; ++i) + if (CPU_ISSET(i, &affinity)) { + if (is_first_cpu) + is_first_cpu = false; + else + CPU_CLR(i, &affinity); + } + + if (is_first_cpu) return false; + + ret = pthread_setaffinity_np(self, sizeof(affinity), &affinity); + return ret == 0; +#elif defined(BENCHMARK_OS_WINDOWS_WIN32) + self = GetCurrentThread(); + DWORD_PTR mask = static_cast(1) << GetCurrentProcessorNumber(); + previous_affinity = SetThreadAffinityMask(self, mask); + return previous_affinity != 0; +#else + return false; +#endif // def BENCHMARK_HAS_PTHREAD_AFFINITY + } + +#if defined(BENCHMARK_HAS_PTHREAD_AFFINITY) + pthread_t self; + cpu_set_t previous_affinity; +#elif defined(BENCHMARK_OS_WINDOWS_WIN32) + HANDLE self; + DWORD_PTR previous_affinity; +#endif // def BENCHMARK_HAS_PTHREAD_AFFINITY + bool reset_affinity; +}; + double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { // Currently, scaling is only used on linux path here, // suppress diagnostics about it being unused on other paths. @@ -566,7 +674,7 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { &freq)) { // The value is in kHz (as the file name suggests). For example, on a // 2GHz warpstation, the file contains the value "2000000". - return freq * 1000.0; + return static_cast(freq) * 1000.0; } const double error_value = -1; @@ -578,7 +686,7 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { return error_value; } - auto startsWithKey = [](std::string const& Value, std::string const& Key) { + auto StartsWithKey = [](std::string const& Value, std::string const& Key) { if (Key.size() > Value.size()) return false; auto Cmp = [&](char X, char Y) { return std::tolower(X) == std::tolower(Y); @@ -589,18 +697,18 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { std::string ln; while (std::getline(f, ln)) { if (ln.empty()) continue; - size_t SplitIdx = ln.find(':'); + std::size_t split_idx = ln.find(':'); std::string value; - if (SplitIdx != std::string::npos) value = ln.substr(SplitIdx + 1); + if (split_idx != std::string::npos) value = ln.substr(split_idx + 1); // When parsing the "cpu MHz" and "bogomips" (fallback) entries, we only // accept positive values. Some environments (virtual machines) report zero, // which would cause infinite looping in WallTime_Init. - if (startsWithKey(ln, "cpu MHz")) { + if (StartsWithKey(ln, "cpu MHz")) { if (!value.empty()) { double cycles_per_second = benchmark::stod(value) * 1000000.0; if (cycles_per_second > 0) return cycles_per_second; } - } else if (startsWithKey(ln, "bogomips")) { + } else if (StartsWithKey(ln, "bogomips")) { if (!value.empty()) { bogo_clock = benchmark::stod(value) * 1000000.0; if (bogo_clock < 0.0) bogo_clock = error_value; @@ -622,7 +730,7 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { if (bogo_clock >= 0.0) return bogo_clock; #elif defined BENCHMARK_HAS_SYSCTL - constexpr auto* FreqStr = + constexpr auto* freqStr = #if defined(BENCHMARK_OS_FREEBSD) || defined(BENCHMARK_OS_NETBSD) "machdep.tsc_freq"; #elif defined BENCHMARK_OS_OPENBSD @@ -634,14 +742,17 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { #endif unsigned long long hz = 0; #if defined BENCHMARK_OS_OPENBSD - if (GetSysctl(FreqStr, &hz)) return hz * 1000000; + if (GetSysctl(freqStr, &hz)) return hz * 1000000; #else - if (GetSysctl(FreqStr, &hz)) return hz; + if (GetSysctl(freqStr, &hz)) return hz; #endif fprintf(stderr, "Unable to determine clock rate from sysctl: %s: %s\n", - FreqStr, strerror(errno)); + freqStr, strerror(errno)); + fprintf(stderr, + "This does not affect benchmark measurements, only the " + "metadata output.\n"); -#elif defined BENCHMARK_OS_WINDOWS +#elif defined BENCHMARK_OS_WINDOWS_WIN32 // In NT, read MHz from the registry. If we fail to do so or we're in win9x // then make a crude estimate. DWORD data, data_size = sizeof(data); @@ -650,15 +761,16 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { SHGetValueA(HKEY_LOCAL_MACHINE, "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", "~MHz", nullptr, &data, &data_size))) - return static_cast((int64_t)data * - (int64_t)(1000 * 1000)); // was mhz + return static_cast(static_cast(data) * + static_cast(1000 * 1000)); // was mhz #elif defined(BENCHMARK_OS_SOLARIS) kstat_ctl_t* kc = kstat_open(); if (!kc) { std::cerr << "failed to open /dev/kstat\n"; return -1; } - kstat_t* ksp = kstat_lookup(kc, (char*)"cpu_info", -1, (char*)"cpu_info0"); + kstat_t* ksp = kstat_lookup(kc, const_cast("cpu_info"), -1, + const_cast("cpu_info0")); if (!ksp) { std::cerr << "failed to lookup in /dev/kstat\n"; return -1; @@ -667,8 +779,8 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { std::cerr << "failed to read from /dev/kstat\n"; return -1; } - kstat_named_t* knp = - (kstat_named_t*)kstat_data_lookup(ksp, (char*)"current_clock_Hz"); + kstat_named_t* knp = (kstat_named_t*)kstat_data_lookup( + ksp, const_cast("current_clock_Hz")); if (!knp) { std::cerr << "failed to lookup data in /dev/kstat\n"; return -1; @@ -682,22 +794,55 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { kstat_close(kc); return clock_hz; #elif defined(BENCHMARK_OS_QNX) - return static_cast((int64_t)(SYSPAGE_ENTRY(cpuinfo)->speed) * - (int64_t)(1000 * 1000)); + return static_cast( + static_cast(SYSPAGE_ENTRY(cpuinfo)->speed) * + static_cast(1000 * 1000)); +#elif defined(BENCHMARK_OS_QURT) + // QuRT doesn't provide any API to query Hexagon frequency. + return 1000000000; #endif // If we've fallen through, attempt to roughly estimate the CPU clock rate. - const int estimate_time_ms = 1000; + + // Make sure to use the same cycle counter when starting and stopping the + // cycle timer. We just pin the current thread to a cpu in the previous + // affinity set. + ThreadAffinityGuard affinity_guard; + + static constexpr double estimate_time_s = 1.0; + const double start_time = ChronoClockNow(); const auto start_ticks = cycleclock::Now(); - SleepForMilliseconds(estimate_time_ms); - return static_cast(cycleclock::Now() - start_ticks); + + // Impose load instead of calling sleep() to make sure the cycle counter + // works. + using PRNG = std::minstd_rand; + using Result = PRNG::result_type; + PRNG rng(static_cast(start_ticks)); + + Result state = 0; + + do { + static constexpr size_t batch_size = 10000; + rng.discard(batch_size); + state += rng(); + + } while (ChronoClockNow() - start_time < estimate_time_s); + + DoNotOptimize(state); + + const auto end_ticks = cycleclock::Now(); + const double end_time = ChronoClockNow(); + + return static_cast(end_ticks - start_ticks) / (end_time - start_time); + // Reset the affinity of current thread when the lifetime of affinity_guard + // ends. } std::vector GetLoadAvg() { #if (defined BENCHMARK_OS_FREEBSD || defined(BENCHMARK_OS_LINUX) || \ defined BENCHMARK_OS_MACOSX || defined BENCHMARK_OS_NETBSD || \ defined BENCHMARK_OS_OPENBSD || defined BENCHMARK_OS_DRAGONFLY) && \ - !defined(__ANDROID__) - constexpr int kMaxSamples = 3; + !(defined(__ANDROID__) && __ANDROID_API__ < 29) + static constexpr int kMaxSamples = 3; std::vector res(kMaxSamples, 0.0); const int nelem = getloadavg(res.data(), kMaxSamples); if (nelem < 1) { diff --git a/third-party/benchmark/src/thread_manager.h b/third-party/benchmark/src/thread_manager.h index 468028508940..819b3c44db66 100644 --- a/third-party/benchmark/src/thread_manager.h +++ b/third-party/benchmark/src/thread_manager.h @@ -43,8 +43,8 @@ class ThreadManager { double manual_time_used = 0; int64_t complexity_n = 0; std::string report_label_; - std::string error_message_; - bool has_error_ = false; + std::string skip_message_; + internal::Skipped skipped_ = internal::NotSkipped; UserCounters counters; }; GUARDED_BY(GetBenchmarkMutex()) Result results; diff --git a/third-party/benchmark/src/timers.cc b/third-party/benchmark/src/timers.cc index ed35c01f5540..667e7b2eef3c 100644 --- a/third-party/benchmark/src/timers.cc +++ b/third-party/benchmark/src/timers.cc @@ -23,7 +23,7 @@ #include #else #include -#ifndef BENCHMARK_OS_FUCHSIA +#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) #include #endif #include @@ -38,6 +38,9 @@ #include #include #endif +#if defined(BENCHMARK_OS_QURT) +#include +#endif #endif #ifdef BENCHMARK_OS_EMSCRIPTEN @@ -56,7 +59,6 @@ #include "check.h" #include "log.h" -#include "sleep.h" #include "string_util.h" namespace benchmark { @@ -65,6 +67,9 @@ namespace benchmark { #if defined(__GNUC__) #pragma GCC diagnostic ignored "-Wunused-function" #endif +#if defined(__NVCOMPILER) +#pragma diag_suppress declared_but_not_referenced +#endif namespace { #if defined(BENCHMARK_OS_WINDOWS) @@ -79,7 +84,7 @@ double MakeTime(FILETIME const& kernel_time, FILETIME const& user_time) { static_cast(user.QuadPart)) * 1e-7; } -#elif !defined(BENCHMARK_OS_FUCHSIA) +#elif !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) double MakeTime(struct rusage const& ru) { return (static_cast(ru.ru_utime.tv_sec) + static_cast(ru.ru_utime.tv_usec) * 1e-6 + @@ -97,7 +102,8 @@ double MakeTime(thread_basic_info_data_t const& info) { #endif #if defined(CLOCK_PROCESS_CPUTIME_ID) || defined(CLOCK_THREAD_CPUTIME_ID) double MakeTime(struct timespec const& ts) { - return ts.tv_sec + (static_cast(ts.tv_nsec) * 1e-9); + return static_cast(ts.tv_sec) + + (static_cast(ts.tv_nsec) * 1e-9); } #endif @@ -119,11 +125,15 @@ double ProcessCPUUsage() { &user_time)) return MakeTime(kernel_time, user_time); DiagnoseAndExit("GetProccessTimes() failed"); +#elif defined(BENCHMARK_OS_QURT) + return static_cast( + qurt_timer_timetick_to_us(qurt_timer_get_ticks())) * + 1.0e-6; #elif defined(BENCHMARK_OS_EMSCRIPTEN) // clock_gettime(CLOCK_PROCESS_CPUTIME_ID, ...) returns 0 on Emscripten. // Use Emscripten-specific API. Reported CPU time would be exactly the // same as total time, but this is ok because there aren't long-latency - // syncronous system calls in Emscripten. + // synchronous system calls in Emscripten. return emscripten_get_now() * 1e-3; #elif defined(CLOCK_PROCESS_CPUTIME_ID) && !defined(BENCHMARK_OS_MACOSX) // FIXME We want to use clock_gettime, but its not available in MacOS 10.11. @@ -149,6 +159,10 @@ double ThreadCPUUsage() { GetThreadTimes(this_thread, &creation_time, &exit_time, &kernel_time, &user_time); return MakeTime(kernel_time, user_time); +#elif defined(BENCHMARK_OS_QURT) + return static_cast( + qurt_timer_timetick_to_us(qurt_timer_get_ticks())) * + 1.0e-6; #elif defined(BENCHMARK_OS_MACOSX) // FIXME We want to use clock_gettime, but its not available in MacOS 10.11. // See https://github.com/google/benchmark/pull/292 diff --git a/third-party/benchmark/test/AssemblyTests.cmake b/third-party/benchmark/test/AssemblyTests.cmake index 3d078586f1de..c43c711faf87 100644 --- a/third-party/benchmark/test/AssemblyTests.cmake +++ b/third-party/benchmark/test/AssemblyTests.cmake @@ -1,3 +1,23 @@ +set(CLANG_SUPPORTED_VERSION "5.0.0") +set(GCC_SUPPORTED_VERSION "5.5.0") + +if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") + if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL ${CLANG_SUPPORTED_VERSION}) + message (WARNING + "Unsupported Clang version " ${CMAKE_CXX_COMPILER_VERSION} + ". Expected is " ${CLANG_SUPPORTED_VERSION} + ". Assembly tests may be broken.") + endif() +elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU") + if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL ${GCC_SUPPORTED_VERSION}) + message (WARNING + "Unsupported GCC version " ${CMAKE_CXX_COMPILER_VERSION} + ". Expected is " ${GCC_SUPPORTED_VERSION} + ". Assembly tests may be broken.") + endif() +else() + message (WARNING "Unsupported compiler. Assembly tests may be broken.") +endif() include(split_list) @@ -23,6 +43,7 @@ string(TOUPPER "${CMAKE_CXX_COMPILER_ID}" ASM_TEST_COMPILER) macro(add_filecheck_test name) cmake_parse_arguments(ARG "" "" "CHECK_PREFIXES" ${ARGV}) add_library(${name} OBJECT ${name}.cc) + target_link_libraries(${name} PRIVATE benchmark::benchmark) set_target_properties(${name} PROPERTIES COMPILE_FLAGS "-S ${ASM_TEST_FLAGS}") set(ASM_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/${name}.s") add_custom_target(copy_${name} ALL diff --git a/third-party/benchmark/test/CMakeLists.txt b/third-party/benchmark/test/CMakeLists.txt index 162af53f80f7..1de175f98d34 100644 --- a/third-party/benchmark/test/CMakeLists.txt +++ b/third-party/benchmark/test/CMakeLists.txt @@ -1,8 +1,12 @@ # Enable the tests +set(THREADS_PREFER_PTHREAD_FLAG ON) + find_package(Threads REQUIRED) include(CheckCXXCompilerFlag) +add_cxx_compiler_flag(-Wno-unused-variable) + # NOTE: Some tests use `` to perform the test. Therefore we must # strip -DNDEBUG from the default CMake flags in DEBUG mode. string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) @@ -22,6 +26,10 @@ if( NOT uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG" ) endforeach() endif() +if (NOT BUILD_SHARED_LIBS) + add_definitions(-DBENCHMARK_STATIC_DEFINE) +endif() + check_cxx_compiler_flag(-O3 BENCHMARK_HAS_O3_FLAG) set(BENCHMARK_O3_FLAG "") if (BENCHMARK_HAS_O3_FLAG) @@ -35,10 +43,14 @@ if (DEFINED BENCHMARK_CXX_LINKER_FLAGS) endif() add_library(output_test_helper STATIC output_test_helper.cc output_test.h) +target_link_libraries(output_test_helper PRIVATE benchmark::benchmark) macro(compile_benchmark_test name) add_executable(${name} "${name}.cc") target_link_libraries(${name} benchmark::benchmark ${CMAKE_THREAD_LIBS_INIT}) + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "NVHPC") + target_compile_options( ${name} PRIVATE --diag_suppress partial_override ) + endif() endmacro(compile_benchmark_test) macro(compile_benchmark_test_with_main name) @@ -48,26 +60,43 @@ endmacro(compile_benchmark_test_with_main) macro(compile_output_test name) add_executable(${name} "${name}.cc" output_test.h) - target_link_libraries(${name} output_test_helper benchmark::benchmark + target_link_libraries(${name} output_test_helper benchmark::benchmark_main ${BENCHMARK_CXX_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) endmacro(compile_output_test) +macro(benchmark_add_test) + add_test(${ARGV}) + if(WIN32 AND BUILD_SHARED_LIBS) + cmake_parse_arguments(TEST "" "NAME" "" ${ARGN}) + set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + endif() +endmacro(benchmark_add_test) + # Demonstration executable compile_benchmark_test(benchmark_test) -add_test(NAME benchmark COMMAND benchmark_test --benchmark_min_time=0.01) +benchmark_add_test(NAME benchmark COMMAND benchmark_test --benchmark_min_time=0.01s) compile_benchmark_test(spec_arg_test) -add_test(NAME spec_arg COMMAND spec_arg_test --benchmark_filter=BM_NotChosen) +benchmark_add_test(NAME spec_arg COMMAND spec_arg_test --benchmark_filter=BM_NotChosen) + +compile_benchmark_test(spec_arg_verbosity_test) +benchmark_add_test(NAME spec_arg_verbosity COMMAND spec_arg_verbosity_test --v=42) compile_benchmark_test(benchmark_setup_teardown_test) -add_test(NAME benchmark_setup_teardown COMMAND benchmark_setup_teardown_test) +benchmark_add_test(NAME benchmark_setup_teardown COMMAND benchmark_setup_teardown_test) compile_benchmark_test(filter_test) macro(add_filter_test name filter expect) - add_test(NAME ${name} COMMAND filter_test --benchmark_min_time=0.01 --benchmark_filter=${filter} ${expect}) - add_test(NAME ${name}_list_only COMMAND filter_test --benchmark_list_tests --benchmark_filter=${filter} ${expect}) + benchmark_add_test(NAME ${name} COMMAND filter_test --benchmark_min_time=0.01s --benchmark_filter=${filter} ${expect}) + benchmark_add_test(NAME ${name}_list_only COMMAND filter_test --benchmark_list_tests --benchmark_filter=${filter} ${expect}) endmacro(add_filter_test) +compile_benchmark_test(benchmark_min_time_flag_time_test) +benchmark_add_test(NAME min_time_flag_time COMMAND benchmark_min_time_flag_time_test) + +compile_benchmark_test(benchmark_min_time_flag_iters_test) +benchmark_add_test(NAME min_time_flag_iters COMMAND benchmark_min_time_flag_iters_test) + add_filter_test(filter_simple "Foo" 3) add_filter_test(filter_simple_negative "-Foo" 2) add_filter_test(filter_suffix "BM_.*" 4) @@ -88,78 +117,83 @@ add_filter_test(filter_regex_end ".*Ba$" 1) add_filter_test(filter_regex_end_negative "-.*Ba$" 4) compile_benchmark_test(options_test) -add_test(NAME options_benchmarks COMMAND options_test --benchmark_min_time=0.01) +benchmark_add_test(NAME options_benchmarks COMMAND options_test --benchmark_min_time=0.01s) compile_benchmark_test(basic_test) -add_test(NAME basic_benchmark COMMAND basic_test --benchmark_min_time=0.01) +benchmark_add_test(NAME basic_benchmark COMMAND basic_test --benchmark_min_time=0.01s) compile_output_test(repetitions_test) -add_test(NAME repetitions_benchmark COMMAND repetitions_test --benchmark_min_time=0.01 --benchmark_repetitions=3) +benchmark_add_test(NAME repetitions_benchmark COMMAND repetitions_test --benchmark_min_time=0.01s --benchmark_repetitions=3) compile_benchmark_test(diagnostics_test) -add_test(NAME diagnostics_test COMMAND diagnostics_test --benchmark_min_time=0.01) +benchmark_add_test(NAME diagnostics_test COMMAND diagnostics_test --benchmark_min_time=0.01s) compile_benchmark_test(skip_with_error_test) -add_test(NAME skip_with_error_test COMMAND skip_with_error_test --benchmark_min_time=0.01) +benchmark_add_test(NAME skip_with_error_test COMMAND skip_with_error_test --benchmark_min_time=0.01s) compile_benchmark_test(donotoptimize_test) +# Enable errors for deprecated deprecations (DoNotOptimize(Tp const& value)). +check_cxx_compiler_flag(-Werror=deprecated-declarations BENCHMARK_HAS_DEPRECATED_DECLARATIONS_FLAG) +if (BENCHMARK_HAS_DEPRECATED_DECLARATIONS_FLAG) + target_compile_options (donotoptimize_test PRIVATE "-Werror=deprecated-declarations") +endif() # Some of the issues with DoNotOptimize only occur when optimization is enabled check_cxx_compiler_flag(-O3 BENCHMARK_HAS_O3_FLAG) if (BENCHMARK_HAS_O3_FLAG) set_target_properties(donotoptimize_test PROPERTIES COMPILE_FLAGS "-O3") endif() -add_test(NAME donotoptimize_test COMMAND donotoptimize_test --benchmark_min_time=0.01) +benchmark_add_test(NAME donotoptimize_test COMMAND donotoptimize_test --benchmark_min_time=0.01s) compile_benchmark_test(fixture_test) -add_test(NAME fixture_test COMMAND fixture_test --benchmark_min_time=0.01) +benchmark_add_test(NAME fixture_test COMMAND fixture_test --benchmark_min_time=0.01s) compile_benchmark_test(register_benchmark_test) -add_test(NAME register_benchmark_test COMMAND register_benchmark_test --benchmark_min_time=0.01) +benchmark_add_test(NAME register_benchmark_test COMMAND register_benchmark_test --benchmark_min_time=0.01s) compile_benchmark_test(map_test) -add_test(NAME map_test COMMAND map_test --benchmark_min_time=0.01) +benchmark_add_test(NAME map_test COMMAND map_test --benchmark_min_time=0.01s) compile_benchmark_test(multiple_ranges_test) -add_test(NAME multiple_ranges_test COMMAND multiple_ranges_test --benchmark_min_time=0.01) +benchmark_add_test(NAME multiple_ranges_test COMMAND multiple_ranges_test --benchmark_min_time=0.01s) compile_benchmark_test(args_product_test) -add_test(NAME args_product_test COMMAND args_product_test --benchmark_min_time=0.01) +benchmark_add_test(NAME args_product_test COMMAND args_product_test --benchmark_min_time=0.01s) compile_benchmark_test_with_main(link_main_test) -add_test(NAME link_main_test COMMAND link_main_test --benchmark_min_time=0.01) +benchmark_add_test(NAME link_main_test COMMAND link_main_test --benchmark_min_time=0.01s) compile_output_test(reporter_output_test) -add_test(NAME reporter_output_test COMMAND reporter_output_test --benchmark_min_time=0.01) +benchmark_add_test(NAME reporter_output_test COMMAND reporter_output_test --benchmark_min_time=0.01s) compile_output_test(templated_fixture_test) -add_test(NAME templated_fixture_test COMMAND templated_fixture_test --benchmark_min_time=0.01) +benchmark_add_test(NAME templated_fixture_test COMMAND templated_fixture_test --benchmark_min_time=0.01s) compile_output_test(user_counters_test) -add_test(NAME user_counters_test COMMAND user_counters_test --benchmark_min_time=0.01) +benchmark_add_test(NAME user_counters_test COMMAND user_counters_test --benchmark_min_time=0.01s) compile_output_test(perf_counters_test) -add_test(NAME perf_counters_test COMMAND perf_counters_test --benchmark_min_time=0.01 --benchmark_perf_counters=CYCLES,BRANCHES) +benchmark_add_test(NAME perf_counters_test COMMAND perf_counters_test --benchmark_min_time=0.01s --benchmark_perf_counters=CYCLES,INSTRUCTIONS) compile_output_test(internal_threading_test) -add_test(NAME internal_threading_test COMMAND internal_threading_test --benchmark_min_time=0.01) +benchmark_add_test(NAME internal_threading_test COMMAND internal_threading_test --benchmark_min_time=0.01s) compile_output_test(report_aggregates_only_test) -add_test(NAME report_aggregates_only_test COMMAND report_aggregates_only_test --benchmark_min_time=0.01) +benchmark_add_test(NAME report_aggregates_only_test COMMAND report_aggregates_only_test --benchmark_min_time=0.01s) compile_output_test(display_aggregates_only_test) -add_test(NAME display_aggregates_only_test COMMAND display_aggregates_only_test --benchmark_min_time=0.01) +benchmark_add_test(NAME display_aggregates_only_test COMMAND display_aggregates_only_test --benchmark_min_time=0.01s) compile_output_test(user_counters_tabular_test) -add_test(NAME user_counters_tabular_test COMMAND user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01) +benchmark_add_test(NAME user_counters_tabular_test COMMAND user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01s) compile_output_test(user_counters_thousands_test) -add_test(NAME user_counters_thousands_test COMMAND user_counters_thousands_test --benchmark_min_time=0.01) +benchmark_add_test(NAME user_counters_thousands_test COMMAND user_counters_thousands_test --benchmark_min_time=0.01s) compile_output_test(memory_manager_test) -add_test(NAME memory_manager_test COMMAND memory_manager_test --benchmark_min_time=0.01) +benchmark_add_test(NAME memory_manager_test COMMAND memory_manager_test --benchmark_min_time=0.01s) -check_cxx_compiler_flag(-std=c++03 BENCHMARK_HAS_CXX03_FLAG) -if (BENCHMARK_HAS_CXX03_FLAG) +# MSVC does not allow to set the language standard to C++98/03. +if(NOT (MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC")) compile_benchmark_test(cxx03_test) set_target_properties(cxx03_test PROPERTIES @@ -170,22 +204,22 @@ if (BENCHMARK_HAS_CXX03_FLAG) # causing the test to fail to compile. To prevent this we explicitly disable # the warning. check_cxx_compiler_flag(-Wno-odr BENCHMARK_HAS_WNO_ODR) - if (BENCHMARK_ENABLE_LTO AND BENCHMARK_HAS_WNO_ODR) - set_target_properties(cxx03_test - PROPERTIES - LINK_FLAGS "-Wno-odr") + check_cxx_compiler_flag(-Wno-lto-type-mismatch BENCHMARK_HAS_WNO_LTO_TYPE_MISMATCH) + # Cannot set_target_properties multiple times here because the warnings will + # be overwritten on each call + set (DISABLE_LTO_WARNINGS "") + if (BENCHMARK_HAS_WNO_ODR) + set(DISABLE_LTO_WARNINGS "${DISABLE_LTO_WARNINGS} -Wno-odr") endif() - add_test(NAME cxx03 COMMAND cxx03_test --benchmark_min_time=0.01) + if (BENCHMARK_HAS_WNO_LTO_TYPE_MISMATCH) + set(DISABLE_LTO_WARNINGS "${DISABLE_LTO_WARNINGS} -Wno-lto-type-mismatch") + endif() + set_target_properties(cxx03_test PROPERTIES LINK_FLAGS "${DISABLE_LTO_WARNINGS}") + benchmark_add_test(NAME cxx03 COMMAND cxx03_test --benchmark_min_time=0.01s) endif() -# Attempt to work around flaky test failures when running on Appveyor servers. -if (DEFINED ENV{APPVEYOR}) - set(COMPLEXITY_MIN_TIME "0.5") -else() - set(COMPLEXITY_MIN_TIME "0.01") -endif() compile_output_test(complexity_test) -add_test(NAME complexity_benchmark COMMAND complexity_test --benchmark_min_time=${COMPLEXITY_MIN_TIME}) +benchmark_add_test(NAME complexity_benchmark COMMAND complexity_test --benchmark_min_time=1000000x) ############################################################################### # GoogleTest Unit Tests @@ -200,7 +234,12 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) macro(add_gtest name) compile_gtest(${name}) - add_test(NAME ${name} COMMAND ${name}) + benchmark_add_test(NAME ${name} COMMAND ${name}) + if(WIN32 AND BUILD_SHARED_LIBS) + set_tests_properties(${name} PROPERTIES + ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$" + ) + endif() endmacro() add_gtest(benchmark_gtest) @@ -210,6 +249,8 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) add_gtest(statistics_gtest) add_gtest(string_util_gtest) add_gtest(perf_counters_gtest) + add_gtest(time_unit_gtest) + add_gtest(min_time_parse_gtest) endif(BENCHMARK_ENABLE_GTEST_TESTS) ############################################################################### diff --git a/third-party/benchmark/test/args_product_test.cc b/third-party/benchmark/test/args_product_test.cc index d44f391f7480..63b8b71e45a0 100644 --- a/third-party/benchmark/test/args_product_test.cc +++ b/third-party/benchmark/test/args_product_test.cc @@ -23,7 +23,7 @@ class ArgsProductFixture : public ::benchmark::Fixture { {2, 15, 10, 9}, {4, 5, 6, 11}}) {} - void SetUp(const ::benchmark::State& state) BENCHMARK_OVERRIDE { + void SetUp(const ::benchmark::State& state) override { std::vector ranges = {state.range(0), state.range(1), state.range(2), state.range(3)}; @@ -34,7 +34,7 @@ class ArgsProductFixture : public ::benchmark::Fixture { // NOTE: This is not TearDown as we want to check after _all_ runs are // complete. - virtual ~ArgsProductFixture() { + ~ArgsProductFixture() override { if (actualValues != expectedValues) { std::cout << "EXPECTED\n"; for (const auto& v : expectedValues) { diff --git a/third-party/benchmark/test/basic_test.cc b/third-party/benchmark/test/basic_test.cc index 3a8fd42a8cdd..c25bec7ddd58 100644 --- a/third-party/benchmark/test/basic_test.cc +++ b/third-party/benchmark/test/basic_test.cc @@ -5,7 +5,8 @@ void BM_empty(benchmark::State& state) { for (auto _ : state) { - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } } BENCHMARK(BM_empty); @@ -147,7 +148,7 @@ void BM_OneTemplateFunc(benchmark::State& state) { auto arg = state.range(0); T sum = 0; for (auto _ : state) { - sum += arg; + sum += static_cast(arg); } } BENCHMARK(BM_OneTemplateFunc)->Arg(1); @@ -159,8 +160,8 @@ void BM_TwoTemplateFunc(benchmark::State& state) { A sum = 0; B prod = 1; for (auto _ : state) { - sum += arg; - prod *= arg; + sum += static_cast(arg); + prod *= static_cast(arg); } } BENCHMARK(BM_TwoTemplateFunc)->Arg(1); diff --git a/third-party/benchmark/test/benchmark_gtest.cc b/third-party/benchmark/test/benchmark_gtest.cc index 14a885ba46da..2c9e555d92dc 100644 --- a/third-party/benchmark/test/benchmark_gtest.cc +++ b/third-party/benchmark/test/benchmark_gtest.cc @@ -3,12 +3,12 @@ #include #include "../src/benchmark_register.h" +#include "benchmark/benchmark.h" #include "gmock/gmock.h" #include "gtest/gtest.h" namespace benchmark { namespace internal { -extern std::map* global_context; namespace { @@ -38,8 +38,9 @@ TEST(AddRangeTest, Advanced64) { TEST(AddRangeTest, FullRange8) { std::vector dst; - AddRange(&dst, int8_t{1}, std::numeric_limits::max(), 8); - EXPECT_THAT(dst, testing::ElementsAre(1, 8, 64, 127)); + AddRange(&dst, int8_t{1}, std::numeric_limits::max(), int8_t{8}); + EXPECT_THAT( + dst, testing::ElementsAre(int8_t{1}, int8_t{8}, int8_t{64}, int8_t{127})); } TEST(AddRangeTest, FullRange64) { @@ -129,11 +130,13 @@ TEST(AddRangeTest, FullNegativeRange64) { TEST(AddRangeTest, Simple8) { std::vector dst; - AddRange(&dst, 1, 8, 2); - EXPECT_THAT(dst, testing::ElementsAre(1, 2, 4, 8)); + AddRange(&dst, int8_t{1}, int8_t{8}, int8_t{2}); + EXPECT_THAT(dst, + testing::ElementsAre(int8_t{1}, int8_t{2}, int8_t{4}, int8_t{8})); } TEST(AddCustomContext, Simple) { + std::map *&global_context = GetGlobalContext(); EXPECT_THAT(global_context, nullptr); AddCustomContext("foo", "bar"); @@ -148,6 +151,7 @@ TEST(AddCustomContext, Simple) { } TEST(AddCustomContext, DuplicateKey) { + std::map *&global_context = GetGlobalContext(); EXPECT_THAT(global_context, nullptr); AddCustomContext("foo", "bar"); diff --git a/third-party/benchmark/test/benchmark_name_gtest.cc b/third-party/benchmark/test/benchmark_name_gtest.cc index afb401c1f532..0a6746d04df7 100644 --- a/third-party/benchmark/test/benchmark_name_gtest.cc +++ b/third-party/benchmark/test/benchmark_name_gtest.cc @@ -32,6 +32,14 @@ TEST(BenchmarkNameTest, MinTime) { EXPECT_EQ(name.str(), "function_name/some_args:3/4/min_time:3.4s"); } +TEST(BenchmarkNameTest, MinWarmUpTime) { + auto name = BenchmarkName(); + name.function_name = "function_name"; + name.args = "some_args:3/4"; + name.min_warmup_time = "min_warmup_time:3.5s"; + EXPECT_EQ(name.str(), "function_name/some_args:3/4/min_warmup_time:3.5s"); +} + TEST(BenchmarkNameTest, Iterations) { auto name = BenchmarkName(); name.function_name = "function_name"; diff --git a/third-party/benchmark/test/benchmark_random_interleaving_gtest.cc b/third-party/benchmark/test/benchmark_random_interleaving_gtest.cc index d04befa8e381..7f2086750d53 100644 --- a/third-party/benchmark/test/benchmark_random_interleaving_gtest.cc +++ b/third-party/benchmark/test/benchmark_random_interleaving_gtest.cc @@ -51,10 +51,9 @@ class BenchmarkTest : public testing::Test { void Execute(const std::string& pattern) { queue->Clear(); - BenchmarkReporter* reporter = new NullReporter; + std::unique_ptr reporter(new NullReporter()); FLAGS_benchmark_filter = pattern; - RunSpecifiedBenchmarks(reporter); - delete reporter; + RunSpecifiedBenchmarks(reporter.get()); queue->Put("DONE"); // End marker } diff --git a/third-party/benchmark/test/benchmark_setup_teardown_test.cc b/third-party/benchmark/test/benchmark_setup_teardown_test.cc index efa34e15c129..6c3cc2e58fbd 100644 --- a/third-party/benchmark/test/benchmark_setup_teardown_test.cc +++ b/third-party/benchmark/test/benchmark_setup_teardown_test.cc @@ -10,19 +10,19 @@ // Test that Setup() and Teardown() are called exactly once // for each benchmark run (single-threaded). -namespace single { +namespace singlethreaded { static int setup_call = 0; static int teardown_call = 0; -} // namespace single +} // namespace singlethreaded static void DoSetup1(const benchmark::State& state) { - ++single::setup_call; + ++singlethreaded::setup_call; // Setup/Teardown should never be called with any thread_idx != 0. assert(state.thread_index() == 0); } static void DoTeardown1(const benchmark::State& state) { - ++single::teardown_call; + ++singlethreaded::teardown_call; assert(state.thread_index() == 0); } @@ -80,11 +80,11 @@ int fixture_setup = 0; class FIXTURE_BECHMARK_NAME : public ::benchmark::Fixture { public: - void SetUp(const ::benchmark::State&) BENCHMARK_OVERRIDE { + void SetUp(const ::benchmark::State&) override { fixture_interaction::fixture_setup++; } - ~FIXTURE_BECHMARK_NAME() {} + ~FIXTURE_BECHMARK_NAME() override {} }; BENCHMARK_F(FIXTURE_BECHMARK_NAME, BM_WithFixture)(benchmark::State& st) { @@ -134,8 +134,8 @@ int main(int argc, char** argv) { assert(ret > 0); // Setup/Teardown is called once for each arg group (1,3,5,7). - assert(single::setup_call == 4); - assert(single::teardown_call == 4); + assert(singlethreaded::setup_call == 4); + assert(singlethreaded::teardown_call == 4); // 3 group of threads calling this function (3,5,10). assert(concurrent::setup_call.load(std::memory_order_relaxed) == 3); @@ -145,7 +145,7 @@ int main(int argc, char** argv) { // Setup is called 4 times, once for each arg group (1,3,5,7) assert(fixture_interaction::setup == 4); - // Fixture::Setup is called everytime the bm routine is run. + // Fixture::Setup is called every time the bm routine is run. // The exact number is indeterministic, so we just assert that // it's more than setup. assert(fixture_interaction::fixture_setup > fixture_interaction::setup); diff --git a/third-party/benchmark/test/benchmark_test.cc b/third-party/benchmark/test/benchmark_test.cc index 2906cdcde997..8b14017d03a5 100644 --- a/third-party/benchmark/test/benchmark_test.cc +++ b/third-party/benchmark/test/benchmark_test.cc @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -15,6 +16,7 @@ #include #include #include +#include #include #include @@ -26,7 +28,7 @@ namespace { -int BENCHMARK_NOINLINE Factorial(uint32_t n) { +int BENCHMARK_NOINLINE Factorial(int n) { return (n == 1) ? 1 : n * Factorial(n - 1); } @@ -74,7 +76,8 @@ BENCHMARK_RANGE(BM_CalculatePiRange, 1, 1024 * 1024); static void BM_CalculatePi(benchmark::State& state) { static const int depth = 1024; for (auto _ : state) { - benchmark::DoNotOptimize(CalculatePi(static_cast(depth))); + double pi = CalculatePi(static_cast(depth)); + benchmark::DoNotOptimize(pi); } } BENCHMARK(BM_CalculatePi)->Threads(8); @@ -90,7 +93,8 @@ static void BM_SetInsert(benchmark::State& state) { for (int j = 0; j < state.range(1); ++j) data.insert(rand()); } state.SetItemsProcessed(state.iterations() * state.range(1)); - state.SetBytesProcessed(state.iterations() * state.range(1) * sizeof(int)); + state.SetBytesProcessed(state.iterations() * state.range(1) * + static_cast(sizeof(int))); } // Test many inserts at once to reduce the total iterations needed. Otherwise, @@ -108,7 +112,7 @@ static void BM_Sequential(benchmark::State& state) { } const int64_t items_processed = state.iterations() * state.range(0); state.SetItemsProcessed(items_processed); - state.SetBytesProcessed(items_processed * sizeof(v)); + state.SetBytesProcessed(items_processed * static_cast(sizeof(v))); } BENCHMARK_TEMPLATE2(BM_Sequential, std::vector, int) ->Range(1 << 0, 1 << 10); @@ -122,7 +126,10 @@ static void BM_StringCompare(benchmark::State& state) { size_t len = static_cast(state.range(0)); std::string s1(len, '-'); std::string s2(len, '-'); - for (auto _ : state) benchmark::DoNotOptimize(s1.compare(s2)); + for (auto _ : state) { + auto comp = s1.compare(s2); + benchmark::DoNotOptimize(comp); + } } BENCHMARK(BM_StringCompare)->Range(1, 1 << 20); @@ -169,7 +176,7 @@ static void BM_ParallelMemset(benchmark::State& state) { for (int i = from; i < to; i++) { // No need to lock test_vector_mu as ranges // do not overlap between threads. - benchmark::DoNotOptimize(test_vector->at(i) = 1); + benchmark::DoNotOptimize(test_vector->at(static_cast(i)) = 1); } } @@ -220,6 +227,31 @@ void BM_non_template_args(benchmark::State& state, int, double) { } BENCHMARK_CAPTURE(BM_non_template_args, basic_test, 0, 0); +template +void BM_template2_capture(benchmark::State& state, ExtraArgs&&... extra_args) { + static_assert(std::is_same::value, ""); + static_assert(std::is_same::value, ""); + static_assert(std::is_same::value, ""); + unsigned int dummy[sizeof...(ExtraArgs)] = {extra_args...}; + assert(dummy[0] == 42); + for (auto _ : state) { + } +} +BENCHMARK_TEMPLATE2_CAPTURE(BM_template2_capture, void, char*, foo, 42U); +BENCHMARK_CAPTURE((BM_template2_capture), foo, 42U); + +template +void BM_template1_capture(benchmark::State& state, ExtraArgs&&... extra_args) { + static_assert(std::is_same::value, ""); + static_assert(std::is_same::value, ""); + unsigned long dummy[sizeof...(ExtraArgs)] = {extra_args...}; + assert(dummy[0] == 24); + for (auto _ : state) { + } +} +BENCHMARK_TEMPLATE1_CAPTURE(BM_template1_capture, void, foo, 24UL); +BENCHMARK_CAPTURE(BM_template1_capture, foo, 24UL); + #endif // BENCHMARK_HAS_CXX11 static void BM_DenseThreadRanges(benchmark::State& st) { @@ -244,4 +276,25 @@ BENCHMARK(BM_DenseThreadRanges)->Arg(1)->DenseThreadRange(1, 3); BENCHMARK(BM_DenseThreadRanges)->Arg(2)->DenseThreadRange(1, 4, 2); BENCHMARK(BM_DenseThreadRanges)->Arg(3)->DenseThreadRange(5, 14, 3); +static void BM_BenchmarkName(benchmark::State& state) { + for (auto _ : state) { + } + + // Check that the benchmark name is passed correctly to `state`. + assert("BM_BenchmarkName" == state.name()); +} +BENCHMARK(BM_BenchmarkName); + +// regression test for #1446 +template +static void BM_templated_test(benchmark::State& state) { + for (auto _ : state) { + type created_string; + benchmark::DoNotOptimize(created_string); + } +} + +static auto BM_templated_test_double = BM_templated_test>; +BENCHMARK(BM_templated_test_double); + BENCHMARK_MAIN(); diff --git a/third-party/benchmark/test/clobber_memory_assembly_test.cc b/third-party/benchmark/test/clobber_memory_assembly_test.cc index ab269130cd5c..54e26ccdadf4 100644 --- a/third-party/benchmark/test/clobber_memory_assembly_test.cc +++ b/third-party/benchmark/test/clobber_memory_assembly_test.cc @@ -3,6 +3,7 @@ #ifdef __clang__ #pragma clang diagnostic ignored "-Wreturn-type" #endif +BENCHMARK_DISABLE_DEPRECATED_WARNING extern "C" { diff --git a/third-party/benchmark/test/complexity_test.cc b/third-party/benchmark/test/complexity_test.cc index ea268b545988..fb4ad1ad53a9 100644 --- a/third-party/benchmark/test/complexity_test.cc +++ b/third-party/benchmark/test/complexity_test.cc @@ -26,7 +26,7 @@ int AddComplexityTest(const std::string &test_name, AddCases( TC_ConsoleOut, {{"^%bigo_name %bigo_str %bigo_str[ ]*$"}, - {"^%bigo_name", MR_Not}, // Assert we didn't only matched a name. + {"^%bigo_name", MR_Not}, // Assert we we didn't only matched a name. {"^%rms_name %rms %rms[ ]*$", MR_Next}}); AddCases( TC_JSONOut, @@ -69,35 +69,44 @@ int AddComplexityTest(const std::string &test_name, void BM_Complexity_O1(benchmark::State &state) { for (auto _ : state) { - for (int i = 0; i < 1024; ++i) { - benchmark::DoNotOptimize(&i); + // This test requires a non-zero CPU time to avoid divide-by-zero + benchmark::DoNotOptimize(state.iterations()); + long tmp = state.iterations(); + benchmark::DoNotOptimize(tmp); + for (benchmark::IterationCount i = 0; i < state.iterations(); ++i) { + benchmark::DoNotOptimize(state.iterations()); + tmp *= state.iterations(); + benchmark::DoNotOptimize(tmp); } + + // always 1ns per iteration + state.SetIterationTime(42 * 1e-9); } state.SetComplexityN(state.range(0)); } -BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1); -BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(); BENCHMARK(BM_Complexity_O1) ->Range(1, 1 << 18) + ->UseManualTime() + ->Complexity(benchmark::o1); +BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->UseManualTime()->Complexity(); +BENCHMARK(BM_Complexity_O1) + ->Range(1, 1 << 18) + ->UseManualTime() ->Complexity([](benchmark::IterationCount) { return 1.0; }); -const char *one_test_name = "BM_Complexity_O1"; -const char *big_o_1_test_name = "BM_Complexity_O1_BigO"; -const char *rms_o_1_test_name = "BM_Complexity_O1_RMS"; -const char *enum_big_o_1 = "\\([0-9]+\\)"; -// FIXME: Tolerate both '(1)' and 'lgN' as output when the complexity is auto -// deduced. -// See https://github.com/google/benchmark/issues/272 -const char *auto_big_o_1 = "(\\([0-9]+\\))|(lgN)"; +const char *one_test_name = "BM_Complexity_O1/manual_time"; +const char *big_o_1_test_name = "BM_Complexity_O1/manual_time_BigO"; +const char *rms_o_1_test_name = "BM_Complexity_O1/manual_time_RMS"; +const char *enum_auto_big_o_1 = "\\([0-9]+\\)"; const char *lambda_big_o_1 = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - enum_big_o_1, /*family_index=*/0); + enum_auto_big_o_1, /*family_index=*/0); -// Add auto enum tests +// Add auto tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - auto_big_o_1, /*family_index=*/1); + enum_auto_big_o_1, /*family_index=*/1); // Add lambda tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, @@ -107,42 +116,44 @@ ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, // --------------------------- Testing BigO O(N) --------------------------- // // ========================================================================= // -std::vector ConstructRandomVector(int64_t size) { - std::vector v; - v.reserve(static_cast(size)); - for (int i = 0; i < size; ++i) { - v.push_back(static_cast(std::rand() % size)); - } - return v; -} - void BM_Complexity_O_N(benchmark::State &state) { - auto v = ConstructRandomVector(state.range(0)); - // Test worst case scenario (item not in vector) - const int64_t item_not_in_vector = state.range(0) * 2; for (auto _ : state) { - benchmark::DoNotOptimize(std::find(v.begin(), v.end(), item_not_in_vector)); + // This test requires a non-zero CPU time to avoid divide-by-zero + benchmark::DoNotOptimize(state.iterations()); + long tmp = state.iterations(); + benchmark::DoNotOptimize(tmp); + for (benchmark::IterationCount i = 0; i < state.iterations(); ++i) { + benchmark::DoNotOptimize(state.iterations()); + tmp *= state.iterations(); + benchmark::DoNotOptimize(tmp); + } + + // 1ns per iteration per entry + state.SetIterationTime(static_cast(state.range(0)) * 42.0 * 1e-9); } state.SetComplexityN(state.range(0)); } BENCHMARK(BM_Complexity_O_N) ->RangeMultiplier(2) - ->Range(1 << 10, 1 << 16) + ->Range(1 << 10, 1 << 20) + ->UseManualTime() ->Complexity(benchmark::oN); BENCHMARK(BM_Complexity_O_N) ->RangeMultiplier(2) - ->Range(1 << 10, 1 << 16) + ->Range(1 << 10, 1 << 20) + ->UseManualTime() + ->Complexity(); +BENCHMARK(BM_Complexity_O_N) + ->RangeMultiplier(2) + ->Range(1 << 10, 1 << 20) + ->UseManualTime() ->Complexity([](benchmark::IterationCount n) -> double { return static_cast(n); }); -BENCHMARK(BM_Complexity_O_N) - ->RangeMultiplier(2) - ->Range(1 << 10, 1 << 16) - ->Complexity(); -const char *n_test_name = "BM_Complexity_O_N"; -const char *big_o_n_test_name = "BM_Complexity_O_N_BigO"; -const char *rms_o_n_test_name = "BM_Complexity_O_N_RMS"; +const char *n_test_name = "BM_Complexity_O_N/manual_time"; +const char *big_o_n_test_name = "BM_Complexity_O_N/manual_time_BigO"; +const char *rms_o_n_test_name = "BM_Complexity_O_N/manual_time_RMS"; const char *enum_auto_big_o_n = "N"; const char *lambda_big_o_n = "f\\(N\\)"; @@ -150,40 +161,57 @@ const char *lambda_big_o_n = "f\\(N\\)"; ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, enum_auto_big_o_n, /*family_index=*/3); +// Add auto tests +ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, + enum_auto_big_o_n, /*family_index=*/4); + // Add lambda tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - lambda_big_o_n, /*family_index=*/4); + lambda_big_o_n, /*family_index=*/5); // ========================================================================= // -// ------------------------- Testing BigO O(N*lgN) ------------------------- // +// ------------------------- Testing BigO O(NlgN) ------------------------- // // ========================================================================= // +static const double kLog2E = 1.44269504088896340736; static void BM_Complexity_O_N_log_N(benchmark::State &state) { - auto v = ConstructRandomVector(state.range(0)); for (auto _ : state) { - std::sort(v.begin(), v.end()); + // This test requires a non-zero CPU time to avoid divide-by-zero + benchmark::DoNotOptimize(state.iterations()); + long tmp = state.iterations(); + benchmark::DoNotOptimize(tmp); + for (benchmark::IterationCount i = 0; i < state.iterations(); ++i) { + benchmark::DoNotOptimize(state.iterations()); + tmp *= state.iterations(); + benchmark::DoNotOptimize(tmp); + } + + state.SetIterationTime(static_cast(state.range(0)) * kLog2E * + std::log(state.range(0)) * 42.0 * 1e-9); } state.SetComplexityN(state.range(0)); } -static const double kLog2E = 1.44269504088896340736; BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2) - ->Range(1 << 10, 1 << 16) + ->Range(1 << 10, 1U << 24) + ->UseManualTime() ->Complexity(benchmark::oNLogN); BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2) - ->Range(1 << 10, 1 << 16) - ->Complexity([](benchmark::IterationCount n) { - return kLog2E * n * log(static_cast(n)); - }); + ->Range(1 << 10, 1U << 24) + ->UseManualTime() + ->Complexity(); BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2) - ->Range(1 << 10, 1 << 16) - ->Complexity(); + ->Range(1 << 10, 1U << 24) + ->UseManualTime() + ->Complexity([](benchmark::IterationCount n) { + return kLog2E * static_cast(n) * std::log(static_cast(n)); + }); -const char *n_lg_n_test_name = "BM_Complexity_O_N_log_N"; -const char *big_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_BigO"; -const char *rms_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_RMS"; +const char *n_lg_n_test_name = "BM_Complexity_O_N_log_N/manual_time"; +const char *big_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N/manual_time_BigO"; +const char *rms_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N/manual_time_RMS"; const char *enum_auto_big_o_n_lg_n = "NlgN"; const char *lambda_big_o_n_lg_n = "f\\(N\\)"; @@ -192,11 +220,16 @@ ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n, /*family_index=*/6); -// Add lambda tests +// NOTE: auto big-o is wron.g ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, - rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n, + rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n, /*family_index=*/7); +//// Add lambda tests +ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, + rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n, + /*family_index=*/8); + // ========================================================================= // // -------- Testing formatting of Complexity with captured args ------------ // // ========================================================================= // @@ -205,19 +238,30 @@ void BM_ComplexityCaptureArgs(benchmark::State &state, int n) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero benchmark::DoNotOptimize(state.iterations()); + long tmp = state.iterations(); + benchmark::DoNotOptimize(tmp); + for (benchmark::IterationCount i = 0; i < state.iterations(); ++i) { + benchmark::DoNotOptimize(state.iterations()); + tmp *= state.iterations(); + benchmark::DoNotOptimize(tmp); + } + + state.SetIterationTime(static_cast(state.range(0)) * 42.0 * 1e-9); } state.SetComplexityN(n); } BENCHMARK_CAPTURE(BM_ComplexityCaptureArgs, capture_test, 100) + ->UseManualTime() ->Complexity(benchmark::oN) ->Ranges({{1, 2}, {3, 4}}); const std::string complexity_capture_name = - "BM_ComplexityCaptureArgs/capture_test"; + "BM_ComplexityCaptureArgs/capture_test/manual_time"; ADD_COMPLEXITY_CASES(complexity_capture_name, complexity_capture_name + "_BigO", - complexity_capture_name + "_RMS", "N", /*family_index=*/9); + complexity_capture_name + "_RMS", "N", + /*family_index=*/9); // ========================================================================= // // --------------------------- TEST CASES END ------------------------------ // diff --git a/third-party/benchmark/test/diagnostics_test.cc b/third-party/benchmark/test/diagnostics_test.cc index c54d5b0d708a..7c68a98929d8 100644 --- a/third-party/benchmark/test/diagnostics_test.cc +++ b/third-party/benchmark/test/diagnostics_test.cc @@ -49,7 +49,8 @@ void BM_diagnostic_test(benchmark::State& state) { if (called_once == false) try_invalid_pause_resume(state); for (auto _ : state) { - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } if (called_once == false) try_invalid_pause_resume(state); @@ -64,7 +65,8 @@ void BM_diagnostic_test_keep_running(benchmark::State& state) { if (called_once == false) try_invalid_pause_resume(state); while (state.KeepRunning()) { - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } if (called_once == false) try_invalid_pause_resume(state); @@ -74,7 +76,16 @@ void BM_diagnostic_test_keep_running(benchmark::State& state) { BENCHMARK(BM_diagnostic_test_keep_running); int main(int argc, char* argv[]) { +#ifdef NDEBUG + // This test is exercising functionality for debug builds, which are not + // available in release builds. Skip the test if we are in that environment + // to avoid a test failure. + std::cout << "Diagnostic test disabled in release build" << std::endl; + (void)argc; + (void)argv; +#else benchmark::internal::GetAbortHandler() = &TestHandler; benchmark::Initialize(&argc, argv); benchmark::RunSpecifiedBenchmarks(); +#endif } diff --git a/third-party/benchmark/test/donotoptimize_assembly_test.cc b/third-party/benchmark/test/donotoptimize_assembly_test.cc index 2e86a51e2234..dc286f53e20f 100644 --- a/third-party/benchmark/test/donotoptimize_assembly_test.cc +++ b/third-party/benchmark/test/donotoptimize_assembly_test.cc @@ -3,12 +3,16 @@ #ifdef __clang__ #pragma clang diagnostic ignored "-Wreturn-type" #endif +BENCHMARK_DISABLE_DEPRECATED_WARNING extern "C" { extern int ExternInt; extern int ExternInt2; extern int ExternInt3; +extern int BigArray[2049]; + +const int ConstBigArray[2049]{}; inline int Add42(int x) { return x + 42; } @@ -23,7 +27,15 @@ struct Large { int value; int data[2]; }; + +struct ExtraLarge { + int arr[2049]; +}; } + +extern ExtraLarge ExtraLargeObj; +const ExtraLarge ConstExtraLargeObj{}; + // CHECK-LABEL: test_with_rvalue: extern "C" void test_with_rvalue() { benchmark::DoNotOptimize(Add42(0)); @@ -68,6 +80,22 @@ extern "C" void test_with_large_lvalue() { // CHECK: ret } +// CHECK-LABEL: test_with_extra_large_lvalue_with_op: +extern "C" void test_with_extra_large_lvalue_with_op() { + ExtraLargeObj.arr[16] = 42; + benchmark::DoNotOptimize(ExtraLargeObj); + // CHECK: movl $42, ExtraLargeObj+64(%rip) + // CHECK: ret +} + +// CHECK-LABEL: test_with_big_array_with_op +extern "C" void test_with_big_array_with_op() { + BigArray[16] = 42; + benchmark::DoNotOptimize(BigArray); + // CHECK: movl $42, BigArray+64(%rip) + // CHECK: ret +} + // CHECK-LABEL: test_with_non_trivial_lvalue: extern "C" void test_with_non_trivial_lvalue() { NotTriviallyCopyable NTC(ExternInt); @@ -96,6 +124,18 @@ extern "C" void test_with_large_const_lvalue() { // CHECK: ret } +// CHECK-LABEL: test_with_const_extra_large_obj: +extern "C" void test_with_const_extra_large_obj() { + benchmark::DoNotOptimize(ConstExtraLargeObj); + // CHECK: ret +} + +// CHECK-LABEL: test_with_const_big_array +extern "C" void test_with_const_big_array() { + benchmark::DoNotOptimize(ConstBigArray); + // CHECK: ret +} + // CHECK-LABEL: test_with_non_trivial_const_lvalue: extern "C" void test_with_non_trivial_const_lvalue() { const NotTriviallyCopyable Obj(ExternInt); diff --git a/third-party/benchmark/test/donotoptimize_test.cc b/third-party/benchmark/test/donotoptimize_test.cc index c321f156a1e0..04ec9386a3b4 100644 --- a/third-party/benchmark/test/donotoptimize_test.cc +++ b/third-party/benchmark/test/donotoptimize_test.cc @@ -4,9 +4,9 @@ namespace { #if defined(__GNUC__) -std::uint64_t double_up(const std::uint64_t x) __attribute__((const)); +std::int64_t double_up(const std::int64_t x) __attribute__((const)); #endif -std::uint64_t double_up(const std::uint64_t x) { return x * 2; } +std::int64_t double_up(const std::int64_t x) { return x * 2; } } // namespace // Using DoNotOptimize on types like BitRef seem to cause a lot of problems @@ -29,6 +29,15 @@ struct BitRef { int main(int, char*[]) { // this test verifies compilation of DoNotOptimize() for some types + char buffer1[1] = ""; + benchmark::DoNotOptimize(buffer1); + + char buffer2[2] = ""; + benchmark::DoNotOptimize(buffer2); + + char buffer3[3] = ""; + benchmark::DoNotOptimize(buffer3); + char buffer8[8] = ""; benchmark::DoNotOptimize(buffer8); @@ -37,17 +46,24 @@ int main(int, char*[]) { char buffer1024[1024] = ""; benchmark::DoNotOptimize(buffer1024); - benchmark::DoNotOptimize(&buffer1024[0]); + char* bptr = &buffer1024[0]; + benchmark::DoNotOptimize(bptr); int x = 123; benchmark::DoNotOptimize(x); - benchmark::DoNotOptimize(&x); + int* xp = &x; + benchmark::DoNotOptimize(xp); benchmark::DoNotOptimize(x += 42); - benchmark::DoNotOptimize(double_up(x)); + std::int64_t y = double_up(x); + benchmark::DoNotOptimize(y); // These tests are to e - benchmark::DoNotOptimize(BitRef::Make()); BitRef lval = BitRef::Make(); benchmark::DoNotOptimize(lval); + +#ifdef BENCHMARK_HAS_CXX11 + // Check that accept rvalue. + benchmark::DoNotOptimize(BitRef::Make()); +#endif } diff --git a/third-party/benchmark/test/filter_test.cc b/third-party/benchmark/test/filter_test.cc index a567de2dd58e..4c8b8ea488ad 100644 --- a/third-party/benchmark/test/filter_test.cc +++ b/third-party/benchmark/test/filter_test.cc @@ -14,28 +14,27 @@ namespace { class TestReporter : public benchmark::ConsoleReporter { public: - virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { + bool ReportContext(const Context& context) override { return ConsoleReporter::ReportContext(context); }; - virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { + void ReportRuns(const std::vector& report) override { ++count_; - max_family_index_ = - std::max(max_family_index_, report[0].family_index); + max_family_index_ = std::max(max_family_index_, report[0].family_index); ConsoleReporter::ReportRuns(report); }; TestReporter() : count_(0), max_family_index_(0) {} - virtual ~TestReporter() {} + ~TestReporter() override {} - size_t GetCount() const { return count_; } + int GetCount() const { return count_; } - size_t GetMaxFamilyIndex() const { return max_family_index_; } + int64_t GetMaxFamilyIndex() const { return max_family_index_; } private: - mutable size_t count_; - mutable size_t max_family_index_; + mutable int count_; + mutable int64_t max_family_index_; }; } // end namespace @@ -79,13 +78,13 @@ int main(int argc, char** argv) { benchmark::Initialize(&argc, argv); TestReporter test_reporter; - const size_t returned_count = - benchmark::RunSpecifiedBenchmarks(&test_reporter); + const int64_t returned_count = + static_cast(benchmark::RunSpecifiedBenchmarks(&test_reporter)); if (argc == 2) { // Make sure we ran all of the tests std::stringstream ss(argv[1]); - size_t expected_return; + int64_t expected_return; ss >> expected_return; if (returned_count != expected_return) { @@ -95,8 +94,8 @@ int main(int argc, char** argv) { return -1; } - const size_t expected_reports = list_only ? 0 : expected_return; - const size_t reports_count = test_reporter.GetCount(); + const int64_t expected_reports = list_only ? 0 : expected_return; + const int64_t reports_count = test_reporter.GetCount(); if (reports_count != expected_reports) { std::cerr << "ERROR: Expected " << expected_reports << " tests to be run but reported_count = " << reports_count @@ -104,8 +103,8 @@ int main(int argc, char** argv) { return -1; } - const size_t max_family_index = test_reporter.GetMaxFamilyIndex(); - const size_t num_families = reports_count == 0 ? 0 : 1 + max_family_index; + const int64_t max_family_index = test_reporter.GetMaxFamilyIndex(); + const int64_t num_families = reports_count == 0 ? 0 : 1 + max_family_index; if (num_families != expected_reports) { std::cerr << "ERROR: Expected " << expected_reports << " test families to be run but num_families = " diff --git a/third-party/benchmark/test/fixture_test.cc b/third-party/benchmark/test/fixture_test.cc index af650dbd0661..d1093ebf52fc 100644 --- a/third-party/benchmark/test/fixture_test.cc +++ b/third-party/benchmark/test/fixture_test.cc @@ -8,21 +8,21 @@ class FIXTURE_BECHMARK_NAME : public ::benchmark::Fixture { public: - void SetUp(const ::benchmark::State& state) BENCHMARK_OVERRIDE { + void SetUp(const ::benchmark::State& state) override { if (state.thread_index() == 0) { assert(data.get() == nullptr); data.reset(new int(42)); } } - void TearDown(const ::benchmark::State& state) BENCHMARK_OVERRIDE { + void TearDown(const ::benchmark::State& state) override { if (state.thread_index() == 0) { assert(data.get() != nullptr); data.reset(); } } - ~FIXTURE_BECHMARK_NAME() { assert(data == nullptr); } + ~FIXTURE_BECHMARK_NAME() override { assert(data == nullptr); } std::unique_ptr data; }; diff --git a/third-party/benchmark/test/link_main_test.cc b/third-party/benchmark/test/link_main_test.cc index 241ad5c3905e..131937eebca9 100644 --- a/third-party/benchmark/test/link_main_test.cc +++ b/third-party/benchmark/test/link_main_test.cc @@ -2,7 +2,8 @@ void BM_empty(benchmark::State& state) { for (auto _ : state) { - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } } BENCHMARK(BM_empty); diff --git a/third-party/benchmark/test/map_test.cc b/third-party/benchmark/test/map_test.cc index 509613457c1b..0fdba7c87c4f 100644 --- a/third-party/benchmark/test/map_test.cc +++ b/third-party/benchmark/test/map_test.cc @@ -24,7 +24,8 @@ static void BM_MapLookup(benchmark::State& state) { m = ConstructRandomMap(size); state.ResumeTiming(); for (int i = 0; i < size; ++i) { - benchmark::DoNotOptimize(m.find(std::rand() % size)); + auto it = m.find(std::rand() % size); + benchmark::DoNotOptimize(it); } } state.SetItemsProcessed(state.iterations() * size); @@ -34,11 +35,11 @@ BENCHMARK(BM_MapLookup)->Range(1 << 3, 1 << 12); // Using fixtures. class MapFixture : public ::benchmark::Fixture { public: - void SetUp(const ::benchmark::State& st) BENCHMARK_OVERRIDE { + void SetUp(const ::benchmark::State& st) override { m = ConstructRandomMap(static_cast(st.range(0))); } - void TearDown(const ::benchmark::State&) BENCHMARK_OVERRIDE { m.clear(); } + void TearDown(const ::benchmark::State&) override { m.clear(); } std::map m; }; @@ -47,7 +48,8 @@ BENCHMARK_DEFINE_F(MapFixture, Lookup)(benchmark::State& state) { const int size = static_cast(state.range(0)); for (auto _ : state) { for (int i = 0; i < size; ++i) { - benchmark::DoNotOptimize(m.find(std::rand() % size)); + auto it = m.find(std::rand() % size); + benchmark::DoNotOptimize(it); } } state.SetItemsProcessed(state.iterations() * size); diff --git a/third-party/benchmark/test/memory_manager_test.cc b/third-party/benchmark/test/memory_manager_test.cc index f0c192fcbd00..4df674d586ed 100644 --- a/third-party/benchmark/test/memory_manager_test.cc +++ b/third-party/benchmark/test/memory_manager_test.cc @@ -5,16 +5,17 @@ #include "output_test.h" class TestMemoryManager : public benchmark::MemoryManager { - void Start() BENCHMARK_OVERRIDE {} - void Stop(Result* result) BENCHMARK_OVERRIDE { - result->num_allocs = 42; - result->max_bytes_used = 42000; + void Start() override {} + void Stop(Result& result) override { + result.num_allocs = 42; + result.max_bytes_used = 42000; } }; void BM_empty(benchmark::State& state) { for (auto _ : state) { - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } } BENCHMARK(BM_empty); diff --git a/third-party/benchmark/test/multiple_ranges_test.cc b/third-party/benchmark/test/multiple_ranges_test.cc index 7618c4da0892..5300a96036c1 100644 --- a/third-party/benchmark/test/multiple_ranges_test.cc +++ b/third-party/benchmark/test/multiple_ranges_test.cc @@ -28,7 +28,7 @@ class MultipleRangesFixture : public ::benchmark::Fixture { {2, 7, 15}, {7, 6, 3}}) {} - void SetUp(const ::benchmark::State& state) BENCHMARK_OVERRIDE { + void SetUp(const ::benchmark::State& state) override { std::vector ranges = {state.range(0), state.range(1), state.range(2)}; @@ -39,7 +39,7 @@ class MultipleRangesFixture : public ::benchmark::Fixture { // NOTE: This is not TearDown as we want to check after _all_ runs are // complete. - virtual ~MultipleRangesFixture() { + ~MultipleRangesFixture() override { if (actualValues != expectedValues) { std::cout << "EXPECTED\n"; for (const auto& v : expectedValues) { diff --git a/third-party/benchmark/test/options_test.cc b/third-party/benchmark/test/options_test.cc index d424d40b9518..a1b209f3eb33 100644 --- a/third-party/benchmark/test/options_test.cc +++ b/third-party/benchmark/test/options_test.cc @@ -33,6 +33,8 @@ BENCHMARK(BM_basic)->DenseRange(10, 15); BENCHMARK(BM_basic)->Args({42, 42}); BENCHMARK(BM_basic)->Ranges({{64, 512}, {64, 512}}); BENCHMARK(BM_basic)->MinTime(0.7); +BENCHMARK(BM_basic)->MinWarmUpTime(0.8); +BENCHMARK(BM_basic)->MinTime(0.1)->MinWarmUpTime(0.2); BENCHMARK(BM_basic)->UseRealTime(); BENCHMARK(BM_basic)->ThreadRange(2, 4); BENCHMARK(BM_basic)->ThreadPerCpu(); @@ -65,8 +67,8 @@ void BM_explicit_iteration_count(benchmark::State& state) { // Test that the requested iteration count is respected. assert(state.max_iterations == 42); - size_t actual_iterations = 0; - for (auto _ : state) ++actual_iterations; + for (auto _ : state) { + } assert(state.iterations() == state.max_iterations); assert(state.iterations() == 42); } diff --git a/third-party/benchmark/test/output_test.h b/third-party/benchmark/test/output_test.h index c6ff8ef2d303..c08fe1d87e6c 100644 --- a/third-party/benchmark/test/output_test.h +++ b/third-party/benchmark/test/output_test.h @@ -85,7 +85,7 @@ std::string GetFileReporterOutput(int argc, char* argv[]); struct Results; typedef std::function ResultsCheckFn; -size_t AddChecker(const char* bm_name_pattern, const ResultsCheckFn& fn); +size_t AddChecker(const std::string& bm_name_pattern, const ResultsCheckFn& fn); // Class holding the results of a benchmark. // It is passed in calls to checker functions. @@ -117,7 +117,7 @@ struct Results { // get the string for a result by name, or nullptr if the name // is not found - const std::string* Get(const char* entry_name) const { + const std::string* Get(const std::string& entry_name) const { auto it = values.find(entry_name); if (it == values.end()) return nullptr; return &it->second; @@ -126,12 +126,12 @@ struct Results { // get a result by name, parsed as a specific type. // NOTE: for counters, use GetCounterAs instead. template - T GetAs(const char* entry_name) const; + T GetAs(const std::string& entry_name) const; // counters are written as doubles, so they have to be read first // as a double, and only then converted to the asked type. template - T GetCounterAs(const char* entry_name) const { + T GetCounterAs(const std::string& entry_name) const { double dval = GetAs(entry_name); T tval = static_cast(dval); return tval; @@ -139,7 +139,7 @@ struct Results { }; template -T Results::GetAs(const char* entry_name) const { +T Results::GetAs(const std::string& entry_name) const { auto* sv = Get(entry_name); BM_CHECK(sv != nullptr && !sv->empty()); std::stringstream ss; diff --git a/third-party/benchmark/test/output_test_helper.cc b/third-party/benchmark/test/output_test_helper.cc index 81584cbf778b..265f28aae7c7 100644 --- a/third-party/benchmark/test/output_test_helper.cc +++ b/third-party/benchmark/test/output_test_helper.cc @@ -45,7 +45,7 @@ SubMap& GetSubstitutions() { static SubMap map = { {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"}, // human-readable float - {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"}, + {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kKMGTPEZYmunpfazy]?i?"}, {"%percentage", percentage_re}, {"%int", "[ ]*[0-9]+"}, {" %s ", "[ ]+"}, @@ -65,6 +65,7 @@ SubMap& GetSubstitutions() { {"%csv_us_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",us,,,,,"}, {"%csv_ms_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ms,,,,,"}, {"%csv_s_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",s,,,,,"}, + {"%csv_cv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",,,,,,"}, {"%csv_bytes_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + ",,,,"}, {"%csv_items_report", @@ -143,7 +144,7 @@ class TestReporter : public benchmark::BenchmarkReporter { TestReporter(std::vector reps) : reporters_(std::move(reps)) {} - virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { + bool ReportContext(const Context& context) override { bool last_ret = false; bool first = true; for (auto rep : reporters_) { @@ -157,10 +158,10 @@ class TestReporter : public benchmark::BenchmarkReporter { return last_ret; } - void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { + void ReportRuns(const std::vector& report) override { for (auto rep : reporters_) rep->ReportRuns(report); } - void Finalize() BENCHMARK_OVERRIDE { + void Finalize() override { for (auto rep : reporters_) rep->Finalize(); } @@ -248,9 +249,8 @@ void ResultsChecker::CheckResults(std::stringstream& output) { if (!p.regex->Match(r.name)) { BM_VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n"; continue; - } else { - BM_VLOG(2) << p.regex_str << " is matched by " << r.name << "\n"; } + BM_VLOG(2) << p.regex_str << " is matched by " << r.name << "\n"; BM_VLOG(1) << "Checking results of " << r.name << ": ... \n"; p.fn(r); BM_VLOG(1) << "Checking results of " << r.name << ": OK.\n"; @@ -300,7 +300,7 @@ std::vector ResultsChecker::SplitCsv_(const std::string& line) { } // end namespace internal -size_t AddChecker(const char* bm_name, const ResultsCheckFn& fn) { +size_t AddChecker(const std::string& bm_name, const ResultsCheckFn& fn) { auto& rc = internal::GetResultsChecker(); rc.Add(bm_name, fn); return rc.results.size(); @@ -328,16 +328,18 @@ double Results::GetTime(BenchmarkTime which) const { BM_CHECK(unit); if (*unit == "ns") { return val * 1.e-9; - } else if (*unit == "us") { + } + if (*unit == "us") { return val * 1.e-6; - } else if (*unit == "ms") { + } + if (*unit == "ms") { return val * 1.e-3; - } else if (*unit == "s") { + } + if (*unit == "s") { return val; - } else { - BM_CHECK(1 == 0) << "unknown time unit: " << *unit; - return 0; } + BM_CHECK(1 == 0) << "unknown time unit: " << *unit; + return 0; } // ========================================================================= // @@ -393,14 +395,14 @@ void RunOutputTests(int argc, char* argv[]) { benchmark::JSONReporter JR; benchmark::CSVReporter CSVR; struct ReporterTest { - const char* name; + std::string name; std::vector& output_cases; std::vector& error_cases; benchmark::BenchmarkReporter& reporter; std::stringstream out_stream; std::stringstream err_stream; - ReporterTest(const char* n, std::vector& out_tc, + ReporterTest(const std::string& n, std::vector& out_tc, std::vector& err_tc, benchmark::BenchmarkReporter& br) : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) { @@ -408,12 +410,12 @@ void RunOutputTests(int argc, char* argv[]) { reporter.SetErrorStream(&err_stream); } } TestCases[] = { - {"ConsoleReporter", GetTestCaseList(TC_ConsoleOut), + {std::string("ConsoleReporter"), GetTestCaseList(TC_ConsoleOut), GetTestCaseList(TC_ConsoleErr), CR}, - {"JSONReporter", GetTestCaseList(TC_JSONOut), GetTestCaseList(TC_JSONErr), - JR}, - {"CSVReporter", GetTestCaseList(TC_CSVOut), GetTestCaseList(TC_CSVErr), - CSVR}, + {std::string("JSONReporter"), GetTestCaseList(TC_JSONOut), + GetTestCaseList(TC_JSONErr), JR}, + {std::string("CSVReporter"), GetTestCaseList(TC_CSVOut), + GetTestCaseList(TC_CSVErr), CSVR}, }; // Create the test reporter and run the benchmarks. @@ -422,7 +424,8 @@ void RunOutputTests(int argc, char* argv[]) { benchmark::RunSpecifiedBenchmarks(&test_rep); for (auto& rep_test : TestCases) { - std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n"; + std::string msg = + std::string("\nTesting ") + rep_test.name + std::string(" Output\n"); std::string banner(msg.size() - 1, '-'); std::cout << banner << msg << banner << "\n"; @@ -439,7 +442,7 @@ void RunOutputTests(int argc, char* argv[]) { // the checks to subscribees. auto& csv = TestCases[2]; // would use == but gcc spits a warning - BM_CHECK(std::strcmp(csv.name, "CSVReporter") == 0); + BM_CHECK(csv.name == std::string("CSVReporter")); internal::GetResultsChecker().CheckResults(csv.out_stream); } diff --git a/third-party/benchmark/test/perf_counters_gtest.cc b/third-party/benchmark/test/perf_counters_gtest.cc index 3eac62463bc5..2e63049285d7 100644 --- a/third-party/benchmark/test/perf_counters_gtest.cc +++ b/third-party/benchmark/test/perf_counters_gtest.cc @@ -1,6 +1,8 @@ +#include #include #include "../src/perf_counters.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" #ifndef GTEST_SKIP @@ -11,12 +13,15 @@ struct MsgHandler { #endif using benchmark::internal::PerfCounters; +using benchmark::internal::PerfCountersMeasurement; using benchmark::internal::PerfCounterValues; +using ::testing::AllOf; +using ::testing::Gt; +using ::testing::Lt; namespace { const char kGenericPerfEvent1[] = "CYCLES"; -const char kGenericPerfEvent2[] = "BRANCHES"; -const char kGenericPerfEvent3[] = "INSTRUCTIONS"; +const char kGenericPerfEvent2[] = "INSTRUCTIONS"; TEST(PerfCountersTest, Init) { EXPECT_EQ(PerfCounters::Initialize(), PerfCounters::kSupported); @@ -27,7 +32,7 @@ TEST(PerfCountersTest, OneCounter) { GTEST_SKIP() << "Performance counters not supported.\n"; } EXPECT_TRUE(PerfCounters::Initialize()); - EXPECT_TRUE(PerfCounters::Create({kGenericPerfEvent1}).IsValid()); + EXPECT_EQ(PerfCounters::Create({kGenericPerfEvent1}).num_counters(), 1); } TEST(PerfCountersTest, NegativeTest) { @@ -36,29 +41,44 @@ TEST(PerfCountersTest, NegativeTest) { return; } EXPECT_TRUE(PerfCounters::Initialize()); - EXPECT_FALSE(PerfCounters::Create({}).IsValid()); - EXPECT_FALSE(PerfCounters::Create({""}).IsValid()); - EXPECT_FALSE(PerfCounters::Create({"not a counter name"}).IsValid()); + // Safety checks + // Create() will always create a valid object, even if passed no or + // wrong arguments as the new behavior is to warn and drop unsupported + // counters + EXPECT_EQ(PerfCounters::Create({}).num_counters(), 0); + EXPECT_EQ(PerfCounters::Create({""}).num_counters(), 0); + EXPECT_EQ(PerfCounters::Create({"not a counter name"}).num_counters(), 0); { - EXPECT_TRUE(PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2, - kGenericPerfEvent3}) - .IsValid()); - } - EXPECT_FALSE( - PerfCounters::Create({kGenericPerfEvent2, "", kGenericPerfEvent1}) - .IsValid()); - EXPECT_FALSE(PerfCounters::Create({kGenericPerfEvent3, "not a counter name", - kGenericPerfEvent1}) - .IsValid()); + // Try sneaking in a bad egg to see if it is filtered out. The + // number of counters has to be two, not zero + auto counter = + PerfCounters::Create({kGenericPerfEvent2, "", kGenericPerfEvent1}); + EXPECT_EQ(counter.num_counters(), 2); + EXPECT_EQ(counter.names(), std::vector( + {kGenericPerfEvent2, kGenericPerfEvent1})); + } + { + // Try sneaking in an outrageous counter, like a fat finger mistake + auto counter = PerfCounters::Create( + {kGenericPerfEvent2, "not a counter name", kGenericPerfEvent1}); + EXPECT_EQ(counter.num_counters(), 2); + EXPECT_EQ(counter.names(), std::vector( + {kGenericPerfEvent2, kGenericPerfEvent1})); + } { - EXPECT_TRUE(PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2, - kGenericPerfEvent3}) - .IsValid()); - } - EXPECT_FALSE( - PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2, - kGenericPerfEvent3, "MISPREDICTED_BRANCH_RETIRED"}) - .IsValid()); + // Finally try a golden input - it should like both of them + EXPECT_EQ(PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2}) + .num_counters(), + 2); + } + { + // Add a bad apple in the end of the chain to check the edges + auto counter = PerfCounters::Create( + {kGenericPerfEvent1, kGenericPerfEvent2, "bad event name"}); + EXPECT_EQ(counter.num_counters(), 2); + EXPECT_EQ(counter.names(), std::vector( + {kGenericPerfEvent1, kGenericPerfEvent2})); + } } TEST(PerfCountersTest, Read1Counter) { @@ -67,7 +87,7 @@ TEST(PerfCountersTest, Read1Counter) { } EXPECT_TRUE(PerfCounters::Initialize()); auto counters = PerfCounters::Create({kGenericPerfEvent1}); - EXPECT_TRUE(counters.IsValid()); + EXPECT_EQ(counters.num_counters(), 1); PerfCounterValues values1(1); EXPECT_TRUE(counters.Snapshot(&values1)); EXPECT_GT(values1[0], 0); @@ -84,7 +104,7 @@ TEST(PerfCountersTest, Read2Counters) { EXPECT_TRUE(PerfCounters::Initialize()); auto counters = PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2}); - EXPECT_TRUE(counters.IsValid()); + EXPECT_EQ(counters.num_counters(), 2); PerfCounterValues values1(2); EXPECT_TRUE(counters.Snapshot(&values1)); EXPECT_GT(values1[0], 0); @@ -95,30 +115,121 @@ TEST(PerfCountersTest, Read2Counters) { EXPECT_GT(values2[1], 0); } -size_t do_work() { - size_t res = 0; - for (size_t i = 0; i < 100000000; ++i) res += i * i; - return res; +TEST(PerfCountersTest, ReopenExistingCounters) { + // This test works in recent and old Intel hardware, Pixel 3, and Pixel 6. + // However we cannot make assumptions beyond 2 HW counters due to Pixel 6. + if (!PerfCounters::kSupported) { + GTEST_SKIP() << "Test skipped because libpfm is not supported.\n"; + } + EXPECT_TRUE(PerfCounters::Initialize()); + std::vector kMetrics({kGenericPerfEvent1}); + std::vector counters(2); + for (auto& counter : counters) { + counter = PerfCounters::Create(kMetrics); + } + PerfCounterValues values(1); + EXPECT_TRUE(counters[0].Snapshot(&values)); + EXPECT_TRUE(counters[1].Snapshot(&values)); } -void measure(size_t threadcount, PerfCounterValues* values1, - PerfCounterValues* values2) { - BM_CHECK_NE(values1, nullptr); - BM_CHECK_NE(values2, nullptr); +TEST(PerfCountersTest, CreateExistingMeasurements) { + // The test works (i.e. causes read to fail) for the assumptions + // about hardware capabilities (i.e. small number (2) hardware + // counters) at this date, + // the same as previous test ReopenExistingCounters. + if (!PerfCounters::kSupported) { + GTEST_SKIP() << "Test skipped because libpfm is not supported.\n"; + } + EXPECT_TRUE(PerfCounters::Initialize()); + + // This means we will try 10 counters but we can only guarantee + // for sure at this time that only 3 will work. Perhaps in the future + // we could use libpfm to query for the hardware limits on this + // particular platform. + const int kMaxCounters = 10; + const int kMinValidCounters = 2; + + // Let's use a ubiquitous counter that is guaranteed to work + // on all platforms + const std::vector kMetrics{"cycles"}; + + // Cannot create a vector of actual objects because the + // copy constructor of PerfCounters is deleted - and so is + // implicitly deleted on PerfCountersMeasurement too + std::vector> + perf_counter_measurements; + + perf_counter_measurements.reserve(kMaxCounters); + for (int j = 0; j < kMaxCounters; ++j) { + perf_counter_measurements.emplace_back( + new PerfCountersMeasurement(kMetrics)); + } + + std::vector> measurements; + + // Start all counters together to see if they hold + size_t max_counters = kMaxCounters; + for (size_t i = 0; i < kMaxCounters; ++i) { + auto& counter(*perf_counter_measurements[i]); + EXPECT_EQ(counter.num_counters(), 1); + if (!counter.Start()) { + max_counters = i; + break; + }; + } + + ASSERT_GE(max_counters, kMinValidCounters); + + // Start all together + for (size_t i = 0; i < max_counters; ++i) { + auto& counter(*perf_counter_measurements[i]); + EXPECT_TRUE(counter.Stop(measurements) || (i >= kMinValidCounters)); + } + + // Start/stop individually + for (size_t i = 0; i < max_counters; ++i) { + auto& counter(*perf_counter_measurements[i]); + measurements.clear(); + counter.Start(); + EXPECT_TRUE(counter.Stop(measurements) || (i >= kMinValidCounters)); + } +} + +// We try to do some meaningful work here but the compiler +// insists in optimizing away our loop so we had to add a +// no-optimize macro. In case it fails, we added some entropy +// to this pool as well. + +BENCHMARK_DONT_OPTIMIZE size_t do_work() { + static std::mt19937 rd{std::random_device{}()}; + static std::uniform_int_distribution mrand(0, 10); + const size_t kNumLoops = 1000000; + size_t sum = 0; + for (size_t j = 0; j < kNumLoops; ++j) { + sum += mrand(rd); + } + benchmark::DoNotOptimize(sum); + return sum; +} + +void measure(size_t threadcount, PerfCounterValues* before, + PerfCounterValues* after) { + BM_CHECK_NE(before, nullptr); + BM_CHECK_NE(after, nullptr); std::vector threads(threadcount); auto work = [&]() { BM_CHECK(do_work() > 1000); }; // We need to first set up the counters, then start the threads, so the - // threads would inherit the counters. But later, we need to first destroy the - // thread pool (so all the work finishes), then measure the counters. So the - // scopes overlap, and we need to explicitly control the scope of the + // threads would inherit the counters. But later, we need to first destroy + // the thread pool (so all the work finishes), then measure the counters. So + // the scopes overlap, and we need to explicitly control the scope of the // threadpool. auto counters = - PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent3}); + PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2}); for (auto& t : threads) t = std::thread(work); - counters.Snapshot(values1); + counters.Snapshot(before); for (auto& t : threads) t.join(); - counters.Snapshot(values2); + counters.Snapshot(after); } TEST(PerfCountersTest, MultiThreaded) { @@ -126,20 +237,71 @@ TEST(PerfCountersTest, MultiThreaded) { GTEST_SKIP() << "Test skipped because libpfm is not supported."; } EXPECT_TRUE(PerfCounters::Initialize()); - PerfCounterValues values1(2); - PerfCounterValues values2(2); + PerfCounterValues before(2); + PerfCounterValues after(2); - measure(2, &values1, &values2); - std::vector D1{static_cast(values2[0] - values1[0]), - static_cast(values2[1] - values1[1])}; + // Notice that this test will work even if we taskset it to a single CPU + // In this case the threads will run sequentially + // Start two threads and measure the number of combined cycles and + // instructions + measure(2, &before, &after); + std::vector Elapsed2Threads{ + static_cast(after[0] - before[0]), + static_cast(after[1] - before[1])}; - measure(4, &values1, &values2); - std::vector D2{static_cast(values2[0] - values1[0]), - static_cast(values2[1] - values1[1])}; + // Start four threads and measure the number of combined cycles and + // instructions + measure(4, &before, &after); + std::vector Elapsed4Threads{ + static_cast(after[0] - before[0]), + static_cast(after[1] - before[1])}; - // Some extra work will happen on the main thread - like joining the threads - // - so the ratio won't be quite 2.0, but very close. - EXPECT_GE(D2[0], 1.9 * D1[0]); - EXPECT_GE(D2[1], 1.9 * D1[1]); + // The following expectations fail (at least on a beefy workstation with lots + // of cpus) - it seems that in some circumstances the runtime of 4 threads + // can even be better than with 2. + // So instead of expecting 4 threads to be slower, let's just make sure they + // do not differ too much in general (one is not more than 10x than the + // other). + EXPECT_THAT(Elapsed4Threads[0] / Elapsed2Threads[0], AllOf(Gt(0.1), Lt(10))); + EXPECT_THAT(Elapsed4Threads[1] / Elapsed2Threads[1], AllOf(Gt(0.1), Lt(10))); } + +TEST(PerfCountersTest, HardwareLimits) { + // The test works (i.e. causes read to fail) for the assumptions + // about hardware capabilities (i.e. small number (3-4) hardware + // counters) at this date, + // the same as previous test ReopenExistingCounters. + if (!PerfCounters::kSupported) { + GTEST_SKIP() << "Test skipped because libpfm is not supported.\n"; + } + EXPECT_TRUE(PerfCounters::Initialize()); + + // Taken from `perf list`, but focusses only on those HW events that actually + // were reported when running `sudo perf stat -a sleep 10`, intersected over + // several platforms. All HW events listed in the first command not reported + // in the second seem to not work. This is sad as we don't really get to test + // the grouping here (groups can contain up to 6 members)... + std::vector counter_names{ + "cycles", // leader + "instructions", // + "branch-misses", // + }; + + // In the off-chance that some of these values are not supported, + // we filter them out so the test will complete without failure + // albeit it might not actually test the grouping on that platform + std::vector valid_names; + for (const std::string& name : counter_names) { + if (PerfCounters::IsCounterSupported(name)) { + valid_names.push_back(name); + } + } + PerfCountersMeasurement counter(valid_names); + + std::vector> measurements; + + counter.Start(); + EXPECT_TRUE(counter.Stop(measurements)); +} + } // namespace diff --git a/third-party/benchmark/test/perf_counters_test.cc b/third-party/benchmark/test/perf_counters_test.cc index 3017a452fe27..3cc593e629d8 100644 --- a/third-party/benchmark/test/perf_counters_test.cc +++ b/third-party/benchmark/test/perf_counters_test.cc @@ -1,27 +1,92 @@ +#include #undef NDEBUG +#include "../src/commandlineflags.h" #include "../src/perf_counters.h" - #include "benchmark/benchmark.h" #include "output_test.h" +namespace benchmark { + +BM_DECLARE_string(benchmark_perf_counters); + +} // namespace benchmark + static void BM_Simple(benchmark::State& state) { for (auto _ : state) { - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } } BENCHMARK(BM_Simple); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Simple\",$"}}); +const int kIters = 1000000; + +void BM_WithoutPauseResume(benchmark::State& state) { + int n = 0; + + for (auto _ : state) { + for (auto i = 0; i < kIters; ++i) { + n = 1 - n; + benchmark::DoNotOptimize(n); + } + } +} + +BENCHMARK(BM_WithoutPauseResume); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_WithoutPauseResume\",$"}}); + +void BM_WithPauseResume(benchmark::State& state) { + int m = 0, n = 0; + + for (auto _ : state) { + for (auto i = 0; i < kIters; ++i) { + n = 1 - n; + benchmark::DoNotOptimize(n); + } + + state.PauseTiming(); + for (auto j = 0; j < kIters; ++j) { + m = 1 - m; + benchmark::DoNotOptimize(m); + } + state.ResumeTiming(); + } +} + +BENCHMARK(BM_WithPauseResume); + +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_WithPauseResume\",$"}}); + static void CheckSimple(Results const& e) { CHECK_COUNTER_VALUE(e, double, "CYCLES", GT, 0); - CHECK_COUNTER_VALUE(e, double, "BRANCHES", GT, 0.0); } + +double withoutPauseResumeInstrCount = 0.0; +double withPauseResumeInstrCount = 0.0; + +static void SaveInstrCountWithoutResume(Results const& e) { + withoutPauseResumeInstrCount = e.GetAs("INSTRUCTIONS"); +} + +static void SaveInstrCountWithResume(Results const& e) { + withPauseResumeInstrCount = e.GetAs("INSTRUCTIONS"); +} + CHECK_BENCHMARK_RESULTS("BM_Simple", &CheckSimple); +CHECK_BENCHMARK_RESULTS("BM_WithoutPauseResume", &SaveInstrCountWithoutResume); +CHECK_BENCHMARK_RESULTS("BM_WithPauseResume", &SaveInstrCountWithResume); int main(int argc, char* argv[]) { if (!benchmark::internal::PerfCounters::kSupported) { return 0; } + benchmark::FLAGS_benchmark_perf_counters = "CYCLES,INSTRUCTIONS"; + benchmark::internal::PerfCounters::Initialize(); RunOutputTests(argc, argv); + + BM_CHECK_GT(withPauseResumeInstrCount, kIters); + BM_CHECK_GT(withoutPauseResumeInstrCount, kIters); + BM_CHECK_LT(withPauseResumeInstrCount, 1.5 * withoutPauseResumeInstrCount); } diff --git a/third-party/benchmark/test/register_benchmark_test.cc b/third-party/benchmark/test/register_benchmark_test.cc index 602405b67e8d..d69d144a4e17 100644 --- a/third-party/benchmark/test/register_benchmark_test.cc +++ b/third-party/benchmark/test/register_benchmark_test.cc @@ -10,7 +10,7 @@ namespace { class TestReporter : public benchmark::ConsoleReporter { public: - virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { + void ReportRuns(const std::vector& report) override { all_runs_.insert(all_runs_.end(), begin(report), end(report)); ConsoleReporter::ReportRuns(report); } @@ -19,11 +19,11 @@ class TestReporter : public benchmark::ConsoleReporter { }; struct TestCase { - std::string name; - const char* label; + const std::string name; + const std::string label; // Note: not explicit as we rely on it being converted through ADD_CASES. - TestCase(const char* xname) : TestCase(xname, nullptr) {} - TestCase(const char* xname, const char* xlabel) + TestCase(const std::string& xname) : TestCase(xname, "") {} + TestCase(const std::string& xname, const std::string& xlabel) : name(xname), label(xlabel) {} typedef benchmark::BenchmarkReporter::Run Run; @@ -32,7 +32,7 @@ struct TestCase { // clang-format off BM_CHECK(name == run.benchmark_name()) << "expected " << name << " got " << run.benchmark_name(); - if (label) { + if (!label.empty()) { BM_CHECK(run.report_label == label) << "expected " << label << " got " << run.report_label; } else { @@ -95,6 +95,18 @@ ADD_CASES({"test1", "One"}, {"test2", "Two"}, {"test3", "Three"}); #endif // BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK +//----------------------------------------------------------------------------// +// Test RegisterBenchmark with DISABLED_ benchmark +//----------------------------------------------------------------------------// +void DISABLED_BM_function(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(DISABLED_BM_function); +ReturnVal dummy3 = benchmark::RegisterBenchmark("DISABLED_BM_function_manual", + DISABLED_BM_function); +// No need to add cases because we don't expect them to run. + //----------------------------------------------------------------------------// // Test RegisterBenchmark with different callable types //----------------------------------------------------------------------------// @@ -111,7 +123,7 @@ void TestRegistrationAtRuntime() { { CustomFixture fx; benchmark::RegisterBenchmark("custom_fixture", fx); - AddCases({"custom_fixture"}); + AddCases({std::string("custom_fixture")}); } #endif #ifndef BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK diff --git a/third-party/benchmark/test/reporter_output_test.cc b/third-party/benchmark/test/reporter_output_test.cc index 2b6e6543dd2e..7867165d1f3d 100644 --- a/third-party/benchmark/test/reporter_output_test.cc +++ b/third-party/benchmark/test/reporter_output_test.cc @@ -17,7 +17,7 @@ static int AddContextCases() { AddCases(TC_ConsoleErr, { {"^%int-%int-%intT%int:%int:%int[-+]%int:%int$", MR_Default}, - {"Running .*/reporter_output_test(\\.exe)?$", MR_Next}, + {"Running .*(/|\\\\)reporter_output_test(\\.exe)?$", MR_Next}, {"Run on \\(%int X %float MHz CPU s?\\)", MR_Next}, }); AddCases(TC_JSONOut, @@ -55,6 +55,9 @@ static int AddContextCases() { {{"Load Average: (%float, ){0,2}%float$", MR_Next}}); } AddCases(TC_JSONOut, {{"\"load_avg\": \\[(%float,?){0,3}],$", MR_Next}}); + AddCases(TC_JSONOut, {{"\"library_version\": \".*\",$", MR_Next}}); + AddCases(TC_JSONOut, {{"\"library_build_type\": \".*\",$", MR_Next}}); + AddCases(TC_JSONOut, {{"\"json_schema_version\": 1$", MR_Next}}); return 0; } int dummy_register = AddContextCases(); @@ -93,7 +96,8 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}}); void BM_bytes_per_second(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } state.SetBytesProcessed(1); } @@ -124,7 +128,8 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}}); void BM_items_per_second(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } state.SetItemsProcessed(1); } @@ -318,7 +323,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}, ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}}); // ========================================================================= // -// ------------------------ Testing Arg Name Output ----------------------- // +// ------------------------ Testing Arg Name Output ------------------------ // // ========================================================================= // void BM_arg_name(benchmark::State& state) { @@ -404,7 +409,8 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_BigArgs/1073741824 %console_report$"}, void BM_Complexity_O1(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } state.SetComplexityN(state.range(0)); } @@ -1085,7 +1091,7 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_UserPercentStats/iterations:5/repeats:3/" {"^\"BM_UserPercentStats/iterations:5/repeats:3/" "manual_time_stddev\",%csv_report$"}, {"^\"BM_UserPercentStats/iterations:5/repeats:3/" - "manual_time_\",%csv_report$"}}); + "manual_time_\",%csv_cv_report$"}}); // ========================================================================= // // ------------------------- Testing StrEscape JSON ------------------------ // diff --git a/third-party/benchmark/test/skip_with_error_test.cc b/third-party/benchmark/test/skip_with_error_test.cc index 026d47913350..2139a19e2507 100644 --- a/third-party/benchmark/test/skip_with_error_test.cc +++ b/third-party/benchmark/test/skip_with_error_test.cc @@ -10,17 +10,17 @@ namespace { class TestReporter : public benchmark::ConsoleReporter { public: - virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { + bool ReportContext(const Context& context) override { return ConsoleReporter::ReportContext(context); }; - virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { + void ReportRuns(const std::vector& report) override { all_runs_.insert(all_runs_.end(), begin(report), end(report)); ConsoleReporter::ReportRuns(report); } TestReporter() {} - virtual ~TestReporter() {} + ~TestReporter() override {} mutable std::vector all_runs_; }; @@ -35,8 +35,9 @@ struct TestCase { void CheckRun(Run const& run) const { BM_CHECK(name == run.benchmark_name()) << "expected " << name << " got " << run.benchmark_name(); - BM_CHECK(error_occurred == run.error_occurred); - BM_CHECK(error_message == run.error_message); + BM_CHECK_EQ(error_occurred, + benchmark::internal::SkippedWithError == run.skipped); + BM_CHECK(error_message == run.skip_message); if (error_occurred) { // BM_CHECK(run.iterations == 0); } else { @@ -47,7 +48,8 @@ struct TestCase { std::vector ExpectedResults; -int AddCases(const char* base_name, std::initializer_list const& v) { +int AddCases(const std::string& base_name, + std::initializer_list const& v) { for (auto TC : v) { TC.name = base_name + TC.name; ExpectedResults.push_back(std::move(TC)); @@ -141,7 +143,8 @@ ADD_CASES("BM_error_during_running_ranged_for", void BM_error_after_running(benchmark::State& state) { for (auto _ : state) { - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } if (state.thread_index() <= (state.threads() / 2)) state.SkipWithError("error message"); diff --git a/third-party/benchmark/test/spec_arg_test.cc b/third-party/benchmark/test/spec_arg_test.cc index 043db1be47a2..06aafbeb9b5e 100644 --- a/third-party/benchmark/test/spec_arg_test.cc +++ b/third-party/benchmark/test/spec_arg_test.cc @@ -17,11 +17,11 @@ namespace { class TestReporter : public benchmark::ConsoleReporter { public: - virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { + bool ReportContext(const Context& context) override { return ConsoleReporter::ReportContext(context); }; - virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { + void ReportRuns(const std::vector& report) override { assert(report.size() == 1); matched_functions.push_back(report[0].run_name.function_name); ConsoleReporter::ReportRuns(report); @@ -29,7 +29,7 @@ class TestReporter : public benchmark::ConsoleReporter { TestReporter() {} - virtual ~TestReporter() {} + ~TestReporter() override {} const std::vector& GetMatchedFunctions() const { return matched_functions; @@ -91,5 +91,15 @@ int main(int argc, char** argv) { << matched_functions.front() << "]\n"; return 2; } + + // Test that SetBenchmarkFilter works. + const std::string golden_value = "golden_value"; + benchmark::SetBenchmarkFilter(golden_value); + std::string current_value = benchmark::GetBenchmarkFilter(); + if (golden_value != current_value) { + std::cerr << "Expected [" << golden_value + << "] for --benchmark_filter but got [" << current_value << "]\n"; + return 3; + } return 0; } diff --git a/third-party/benchmark/test/statistics_gtest.cc b/third-party/benchmark/test/statistics_gtest.cc index 1de2d87d4ba5..48c77260fd53 100644 --- a/third-party/benchmark/test/statistics_gtest.cc +++ b/third-party/benchmark/test/statistics_gtest.cc @@ -28,8 +28,8 @@ TEST(StatisticsTest, StdDev) { TEST(StatisticsTest, CV) { EXPECT_DOUBLE_EQ(benchmark::StatisticsCV({101, 101, 101, 101}), 0.0); EXPECT_DOUBLE_EQ(benchmark::StatisticsCV({1, 2, 3}), 1. / 2.); - EXPECT_DOUBLE_EQ(benchmark::StatisticsCV({2.5, 2.4, 3.3, 4.2, 5.1}), - 0.32888184094918121); + ASSERT_NEAR(benchmark::StatisticsCV({2.5, 2.4, 3.3, 4.2, 5.1}), + 0.32888184094918121, 1e-15); } } // end namespace diff --git a/third-party/benchmark/test/string_util_gtest.cc b/third-party/benchmark/test/string_util_gtest.cc index 698f2d43eb88..67b4bc0c24f2 100644 --- a/third-party/benchmark/test/string_util_gtest.cc +++ b/third-party/benchmark/test/string_util_gtest.cc @@ -1,9 +1,12 @@ //===---------------------------------------------------------------------===// -// statistics_test - Unit tests for src/statistics.cc +// string_util_test - Unit tests for src/string_util.cc //===---------------------------------------------------------------------===// +#include + #include "../src/internal_macros.h" #include "../src/string_util.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" namespace { @@ -63,7 +66,10 @@ TEST(StringUtilTest, stoul) { EXPECT_EQ(4ul, pos); } #ifndef BENCHMARK_HAS_NO_EXCEPTIONS - { ASSERT_THROW(benchmark::stoul("this is a test"), std::invalid_argument); } + { + ASSERT_THROW(std::ignore = benchmark::stoul("this is a test"), + std::invalid_argument); + } #endif } @@ -107,7 +113,10 @@ EXPECT_EQ(1ul, pos); EXPECT_EQ(4ul, pos); } #ifndef BENCHMARK_HAS_NO_EXCEPTIONS -{ ASSERT_THROW(benchmark::stoi("this is a test"), std::invalid_argument); } +{ + ASSERT_THROW(std::ignore = benchmark::stoi("this is a test"), + std::invalid_argument); +} #endif } @@ -137,7 +146,10 @@ EXPECT_EQ(1ul, pos); EXPECT_EQ(8ul, pos); } #ifndef BENCHMARK_HAS_NO_EXCEPTIONS -{ ASSERT_THROW(benchmark::stod("this is a test"), std::invalid_argument); } +{ + ASSERT_THROW(std::ignore = benchmark::stod("this is a test"), + std::invalid_argument); +} #endif } @@ -149,4 +161,39 @@ TEST(StringUtilTest, StrSplit) { std::vector({"hello", "there", "is", "more"})); } +using HumanReadableFixture = ::testing::TestWithParam< + std::tuple>; + +INSTANTIATE_TEST_SUITE_P( + HumanReadableTests, HumanReadableFixture, + ::testing::Values( + std::make_tuple(0.0, benchmark::Counter::kIs1024, "0"), + std::make_tuple(999.0, benchmark::Counter::kIs1024, "999"), + std::make_tuple(1000.0, benchmark::Counter::kIs1024, "1000"), + std::make_tuple(1024.0, benchmark::Counter::kIs1024, "1Ki"), + std::make_tuple(1000 * 1000.0, benchmark::Counter::kIs1024, + "976\\.56.Ki"), + std::make_tuple(1024 * 1024.0, benchmark::Counter::kIs1024, "1Mi"), + std::make_tuple(1000 * 1000 * 1000.0, benchmark::Counter::kIs1024, + "953\\.674Mi"), + std::make_tuple(1024 * 1024 * 1024.0, benchmark::Counter::kIs1024, + "1Gi"), + std::make_tuple(0.0, benchmark::Counter::kIs1000, "0"), + std::make_tuple(999.0, benchmark::Counter::kIs1000, "999"), + std::make_tuple(1000.0, benchmark::Counter::kIs1000, "1k"), + std::make_tuple(1024.0, benchmark::Counter::kIs1000, "1.024k"), + std::make_tuple(1000 * 1000.0, benchmark::Counter::kIs1000, "1M"), + std::make_tuple(1024 * 1024.0, benchmark::Counter::kIs1000, + "1\\.04858M"), + std::make_tuple(1000 * 1000 * 1000.0, benchmark::Counter::kIs1000, + "1G"), + std::make_tuple(1024 * 1024 * 1024.0, benchmark::Counter::kIs1000, + "1\\.07374G"))); + +TEST_P(HumanReadableFixture, HumanReadableNumber) { + std::string str = benchmark::HumanReadableNumber(std::get<0>(GetParam()), + std::get<1>(GetParam())); + ASSERT_THAT(str, ::testing::MatchesRegex(std::get<2>(GetParam()))); +} + } // end namespace diff --git a/third-party/benchmark/test/user_counters_tabular_test.cc b/third-party/benchmark/test/user_counters_tabular_test.cc index 45ac043d5193..cfc1ab069c78 100644 --- a/third-party/benchmark/test/user_counters_tabular_test.cc +++ b/third-party/benchmark/test/user_counters_tabular_test.cc @@ -63,6 +63,9 @@ ADD_CASES(TC_CSVOut, {{"%csv_header," void BM_Counters_Tabular(benchmark::State& state) { for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters.insert({ @@ -330,7 +333,7 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/repeats:2/threads:1_stddev\",%csv_report," "%float,%float,%float,%float,%float,%float$"}}); ADD_CASES(TC_CSVOut, - {{"^\"BM_Counters_Tabular/repeats:2/threads:1_cv\",%csv_report," + {{"^\"BM_Counters_Tabular/repeats:2/threads:1_cv\",%csv_cv_report," "%float,%float,%float,%float,%float,%float$"}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/repeats:2/threads:2\",%csv_report," @@ -348,7 +351,7 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/repeats:2/threads:2_stddev\",%csv_report," "%float,%float,%float,%float,%float,%float$"}}); ADD_CASES(TC_CSVOut, - {{"^\"BM_Counters_Tabular/repeats:2/threads:2_cv\",%csv_report," + {{"^\"BM_Counters_Tabular/repeats:2/threads:2_cv\",%csv_cv_report," "%float,%float,%float,%float,%float,%float$"}}); // VS2013 does not allow this function to be passed as a lambda argument // to CHECK_BENCHMARK_RESULTS() @@ -372,7 +375,8 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/repeats:2/threads:2$", void BM_CounterRates_Tabular(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters.insert({ diff --git a/third-party/benchmark/test/user_counters_test.cc b/third-party/benchmark/test/user_counters_test.cc index 1cc74552a1bd..22252acbf6a2 100644 --- a/third-party/benchmark/test/user_counters_test.cc +++ b/third-party/benchmark/test/user_counters_test.cc @@ -67,7 +67,8 @@ int num_calls1 = 0; void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } state.counters["foo"] = 1; state.counters["bar"] = ++num_calls1; @@ -118,7 +119,8 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_WithBytesAndItemsPSec", void BM_Counters_Rate(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters["foo"] = bm::Counter{1, bm::Counter::kIsRate}; @@ -161,7 +163,8 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_Rate", &CheckRate); void BM_Invert(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters["foo"] = bm::Counter{0.0001, bm::Counter::kInvert}; @@ -195,14 +198,14 @@ void CheckInvert(Results const& e) { CHECK_BENCHMARK_RESULTS("BM_Invert", &CheckInvert); // ========================================================================= // -// ------------------------- InvertedRate Counters Output -// -------------------------- // +// --------------------- InvertedRate Counters Output ---------------------- // // ========================================================================= // void BM_Counters_InvertedRate(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters["foo"] = @@ -330,7 +333,8 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_AvgThreads/threads:%int", void BM_Counters_AvgThreadsRate(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgThreadsRate}; @@ -417,7 +421,8 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_IterationInvariant", void BM_Counters_kIsIterationInvariantRate(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters["foo"] = @@ -460,7 +465,7 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_kIsIterationInvariantRate", &CheckIsIterationInvariantRate); // ========================================================================= // -// ------------------- AvgIterations Counters Output ------------------ // +// --------------------- AvgIterations Counters Output --------------------- // // ========================================================================= // void BM_Counters_AvgIterations(benchmark::State& state) { @@ -502,13 +507,14 @@ void CheckAvgIterations(Results const& e) { CHECK_BENCHMARK_RESULTS("BM_Counters_AvgIterations", &CheckAvgIterations); // ========================================================================= // -// ----------------- AvgIterationsRate Counters Output ---------------- // +// ------------------- AvgIterationsRate Counters Output ------------------- // // ========================================================================= // void BM_Counters_kAvgIterationsRate(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgIterationsRate}; diff --git a/third-party/benchmark/test/user_counters_thousands_test.cc b/third-party/benchmark/test/user_counters_thousands_test.cc index a42683b32fa7..fc153835f800 100644 --- a/third-party/benchmark/test/user_counters_thousands_test.cc +++ b/third-party/benchmark/test/user_counters_thousands_test.cc @@ -16,13 +16,13 @@ void BM_Counters_Thousands(benchmark::State& state) { {"t0_1000000DefaultBase", bm::Counter(1000 * 1000, bm::Counter::kDefaults)}, {"t1_1000000Base1000", bm::Counter(1000 * 1000, bm::Counter::kDefaults, - benchmark::Counter::OneK::kIs1000)}, + bm::Counter::OneK::kIs1000)}, {"t2_1000000Base1024", bm::Counter(1000 * 1000, bm::Counter::kDefaults, - benchmark::Counter::OneK::kIs1024)}, + bm::Counter::OneK::kIs1024)}, {"t3_1048576Base1000", bm::Counter(1024 * 1024, bm::Counter::kDefaults, - benchmark::Counter::OneK::kIs1000)}, + bm::Counter::OneK::kIs1000)}, {"t4_1048576Base1024", bm::Counter(1024 * 1024, bm::Counter::kDefaults, - benchmark::Counter::OneK::kIs1024)}, + bm::Counter::OneK::kIs1024)}, }); } BENCHMARK(BM_Counters_Thousands)->Repetitions(2); @@ -30,21 +30,21 @@ ADD_CASES( TC_ConsoleOut, { {"^BM_Counters_Thousands/repeats:2 %console_report " - "t0_1000000DefaultBase=1000k " - "t1_1000000Base1000=1000k t2_1000000Base1024=976.56[23]k " - "t3_1048576Base1000=1048.58k t4_1048576Base1024=1024k$"}, + "t0_1000000DefaultBase=1M " + "t1_1000000Base1000=1M t2_1000000Base1024=976.56[23]Ki " + "t3_1048576Base1000=1.04858M t4_1048576Base1024=1Mi$"}, {"^BM_Counters_Thousands/repeats:2 %console_report " - "t0_1000000DefaultBase=1000k " - "t1_1000000Base1000=1000k t2_1000000Base1024=976.56[23]k " - "t3_1048576Base1000=1048.58k t4_1048576Base1024=1024k$"}, + "t0_1000000DefaultBase=1M " + "t1_1000000Base1000=1M t2_1000000Base1024=976.56[23]Ki " + "t3_1048576Base1000=1.04858M t4_1048576Base1024=1Mi$"}, {"^BM_Counters_Thousands/repeats:2_mean %console_report " - "t0_1000000DefaultBase=1000k t1_1000000Base1000=1000k " - "t2_1000000Base1024=976.56[23]k t3_1048576Base1000=1048.58k " - "t4_1048576Base1024=1024k$"}, + "t0_1000000DefaultBase=1M t1_1000000Base1000=1M " + "t2_1000000Base1024=976.56[23]Ki t3_1048576Base1000=1.04858M " + "t4_1048576Base1024=1Mi$"}, {"^BM_Counters_Thousands/repeats:2_median %console_report " - "t0_1000000DefaultBase=1000k t1_1000000Base1000=1000k " - "t2_1000000Base1024=976.56[23]k t3_1048576Base1000=1048.58k " - "t4_1048576Base1024=1024k$"}, + "t0_1000000DefaultBase=1M t1_1000000Base1000=1M " + "t2_1000000Base1024=976.56[23]Ki t3_1048576Base1000=1.04858M " + "t4_1048576Base1024=1Mi$"}, {"^BM_Counters_Thousands/repeats:2_stddev %console_time_only_report [ " "]*2 t0_1000000DefaultBase=0 t1_1000000Base1000=0 " "t2_1000000Base1024=0 t3_1048576Base1000=0 t4_1048576Base1024=0$"}, diff --git a/third-party/benchmark/tools/compare.py b/third-party/benchmark/tools/compare.py index f1504c96fa2b..7572520cc0ca 100755 --- a/third-party/benchmark/tools/compare.py +++ b/third-party/benchmark/tools/compare.py @@ -1,29 +1,35 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -import unittest +# type: ignore """ compare.py - versatile benchmark output compare tool """ import argparse -from argparse import ArgumentParser import json +import os import sys +import unittest +from argparse import ArgumentParser + import gbench -from gbench import util, report -from gbench.util import * +from gbench import report, util def check_inputs(in1, in2, flags): """ Perform checking on the user provided inputs and diagnose any abnormalities """ - in1_kind, in1_err = classify_input_file(in1) - in2_kind, in2_err = classify_input_file(in2) - output_file = find_benchmark_flag("--benchmark_out=", flags) - output_type = find_benchmark_flag("--benchmark_out_format=", flags) - if in1_kind == IT_Executable and in2_kind == IT_Executable and output_file: + in1_kind, in1_err = util.classify_input_file(in1) + in2_kind, in2_err = util.classify_input_file(in2) + output_file = util.find_benchmark_flag("--benchmark_out=", flags) + output_type = util.find_benchmark_flag("--benchmark_out_format=", flags) + if ( + in1_kind == util.IT_Executable + and in2_kind == util.IT_Executable + and output_file + ): print( ( "WARNING: '--benchmark_out=%s' will be passed to both " @@ -31,11 +37,14 @@ def check_inputs(in1, in2, flags): ) % output_file ) - if in1_kind == IT_JSON and in2_kind == IT_JSON and len(flags) > 0: - print( - "WARNING: passing optional flags has no effect since both " - "inputs are JSON" - ) + if in1_kind == util.IT_JSON and in2_kind == util.IT_JSON: + # When both sides are JSON the only supported flag is + # --benchmark_filter= + for flag in util.remove_benchmark_flags("--benchmark_filter=", flags): + print( + "WARNING: passing %s has no effect since both " + "inputs are JSON" % flag + ) if output_type is not None and output_type != "json": print( ( @@ -48,7 +57,9 @@ def check_inputs(in1, in2, flags): def create_parser(): - parser = ArgumentParser(description="versatile benchmark output compare tool") + parser = ArgumentParser( + description="versatile benchmark output compare tool" + ) parser.add_argument( "-a", @@ -294,7 +305,9 @@ def main(): # Now, filter the benchmarks so that the difference report can work if filter_baseline and filter_contender: replacement = "[%s vs. %s]" % (filter_baseline, filter_contender) - json1 = gbench.report.filter_benchmark(json1_orig, filter_baseline, replacement) + json1 = gbench.report.filter_benchmark( + json1_orig, filter_baseline, replacement + ) json2 = gbench.report.filter_benchmark( json2_orig, filter_contender, replacement ) @@ -314,7 +327,7 @@ def main(): # Optionally, diff and output to JSON if args.dump_to_json is not None: with open(args.dump_to_json, "w") as f_json: - json.dump(diff_report, f_json) + json.dump(diff_report, f_json, indent=1) class TestParser(unittest.TestCase): @@ -423,7 +436,9 @@ class TestParser(unittest.TestCase): self.assertFalse(parsed.benchmark_options) def test_filters_with_remainder(self): - parsed = self.parser.parse_args(["filters", self.testInput0, "c", "d", "e"]) + parsed = self.parser.parse_args( + ["filters", self.testInput0, "c", "d", "e"] + ) self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) self.assertEqual(parsed.mode, "filters") @@ -459,7 +474,14 @@ class TestParser(unittest.TestCase): def test_benchmarksfiltered_with_remainder(self): parsed = self.parser.parse_args( - ["benchmarksfiltered", self.testInput0, "c", self.testInput1, "e", "f"] + [ + "benchmarksfiltered", + self.testInput0, + "c", + self.testInput1, + "e", + "f", + ] ) self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) diff --git a/third-party/benchmark/tools/gbench/Inputs/test1_run1.json b/third-party/benchmark/tools/gbench/Inputs/test1_run1.json index 601e327aefb5..9daed0bcc6c4 100644 --- a/third-party/benchmark/tools/gbench/Inputs/test1_run1.json +++ b/third-party/benchmark/tools/gbench/Inputs/test1_run1.json @@ -114,6 +114,14 @@ "real_time": 1, "cpu_time": 1, "time_unit": "s" + }, + { + "name": "BM_hasLabel", + "label": "a label", + "iterations": 1, + "real_time": 1, + "cpu_time": 1, + "time_unit": "s" } ] } diff --git a/third-party/benchmark/tools/gbench/Inputs/test1_run2.json b/third-party/benchmark/tools/gbench/Inputs/test1_run2.json index 3cbcf39b0c93..dc52970abf8b 100644 --- a/third-party/benchmark/tools/gbench/Inputs/test1_run2.json +++ b/third-party/benchmark/tools/gbench/Inputs/test1_run2.json @@ -114,6 +114,14 @@ "real_time": 1, "cpu_time": 1, "time_unit": "ns" + }, + { + "name": "BM_hasLabel", + "label": "a label", + "iterations": 1, + "real_time": 1, + "cpu_time": 1, + "time_unit": "s" } ] } diff --git a/third-party/benchmark/tools/gbench/__init__.py b/third-party/benchmark/tools/gbench/__init__.py index ffca396b4c3f..921256881491 100644 --- a/third-party/benchmark/tools/gbench/__init__.py +++ b/third-party/benchmark/tools/gbench/__init__.py @@ -5,4 +5,4 @@ __email__ = "eric@efcs.ca" __versioninfo__ = (0, 5, 0) __version__ = ".".join(str(v) for v in __versioninfo__) + "dev" -__all__ = [] +__all__ = [] # type: ignore diff --git a/third-party/benchmark/tools/gbench/report.py b/third-party/benchmark/tools/gbench/report.py index 5092b0bf1469..7158fd1654cb 100644 --- a/third-party/benchmark/tools/gbench/report.py +++ b/third-party/benchmark/tools/gbench/report.py @@ -1,15 +1,17 @@ -"""report.py - Utilities for reporting statistics about benchmark results +# type: ignore + +""" +report.py - Utilities for reporting statistics about benchmark results """ -import unittest -import os -import re import copy +import os import random +import re +import unittest -from scipy.stats import mannwhitneyu, gmean from numpy import array -from pandas import Timedelta +from scipy.stats import gmean, mannwhitneyu class BenchmarkColor(object): @@ -42,6 +44,13 @@ UTEST_MIN_REPETITIONS = 2 UTEST_OPTIMAL_REPETITIONS = 9 # Lowest reasonable number, More is better. UTEST_COL_NAME = "_pvalue" +_TIME_UNIT_TO_SECONDS_MULTIPLIER = { + "s": 1.0, + "ms": 1e-3, + "us": 1e-6, + "ns": 1e-9, +} + def color_format(use_color, fmt_str, *args, **kwargs): """ @@ -52,7 +61,10 @@ def color_format(use_color, fmt_str, *args, **kwargs): """ assert use_color is True or use_color is False if not use_color: - args = [arg if not isinstance(arg, BenchmarkColor) else BC_NONE for arg in args] + args = [ + arg if not isinstance(arg, BenchmarkColor) else BC_NONE + for arg in args + ] kwargs = { key: arg if not isinstance(arg, BenchmarkColor) else BC_NONE for key, arg in kwargs.items() @@ -165,9 +177,9 @@ def get_timedelta_field_as_seconds(benchmark, field_name): Get value of field_name field of benchmark, which is time with time unit time_unit, as time in seconds. """ - time_unit = benchmark["time_unit"] if "time_unit" in benchmark else "s" - dt = Timedelta(benchmark[field_name], time_unit) - return dt / Timedelta(1, "s") + timedelta = benchmark[field_name] + time_unit = benchmark.get("time_unit", "s") + return timedelta * _TIME_UNIT_TO_SECONDS_MULTIPLIER.get(time_unit) def calculate_geomean(json): @@ -273,6 +285,7 @@ def get_difference_report(json1, json2, utest=False): partitions = partition_benchmarks(json1, json2) for partition in partitions: benchmark_name = partition[0][0]["name"] + label = partition[0][0]["label"] if "label" in partition[0][0] else "" time_unit = partition[0][0]["time_unit"] measurements = [] utest_results = {} @@ -286,8 +299,12 @@ def get_difference_report(json1, json2, utest=False): "cpu_time": bn["cpu_time"], "real_time_other": other_bench["real_time"], "cpu_time_other": other_bench["cpu_time"], - "time": calculate_change(bn["real_time"], other_bench["real_time"]), - "cpu": calculate_change(bn["cpu_time"], other_bench["cpu_time"]), + "time": calculate_change( + bn["real_time"], other_bench["real_time"] + ), + "cpu": calculate_change( + bn["cpu_time"], other_bench["cpu_time"] + ), } ) @@ -298,7 +315,7 @@ def get_difference_report(json1, json2, utest=False): have_optimal_repetitions, cpu_pvalue, time_pvalue = calc_utest( timings_cpu, timings_time ) - if cpu_pvalue and time_pvalue: + if cpu_pvalue is not None and time_pvalue is not None: utest_results = { "have_optimal_repetitions": have_optimal_repetitions, "cpu_pvalue": cpu_pvalue, @@ -313,16 +330,20 @@ def get_difference_report(json1, json2, utest=False): # benchmark suite. if measurements: run_type = ( - partition[0][0]["run_type"] if "run_type" in partition[0][0] else "" + partition[0][0]["run_type"] + if "run_type" in partition[0][0] + else "" ) aggregate_name = ( partition[0][0]["aggregate_name"] - if run_type == "aggregate" and "aggregate_name" in partition[0][0] + if run_type == "aggregate" + and "aggregate_name" in partition[0][0] else "" ) diff_report.append( { "name": benchmark_name, + "label": label, "measurements": measurements, "time_unit": time_unit, "run_type": run_type, @@ -337,6 +358,7 @@ def get_difference_report(json1, json2, utest=False): diff_report.append( { "name": "OVERALL_GEOMEAN", + "label": "", "measurements": [ { "real_time": lhs_gmean[0], @@ -392,7 +414,7 @@ def print_difference_report( # and if it is non-aggregate, then don't print it. if ( not include_aggregates_only - or not "run_type" in benchmark + or "run_type" not in benchmark or benchmark["run_type"] == "aggregate" ): for measurement in benchmark["measurements"]: @@ -438,7 +460,9 @@ class TestGetUniqueBenchmarkNames(unittest.TestCase): def load_results(self): import json - testInputs = os.path.join(os.path.dirname(os.path.realpath(__file__)), "Inputs") + testInputs = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "Inputs" + ) testOutput = os.path.join(testInputs, "test3_run0.json") with open(testOutput, "r") as f: json = json.load(f) @@ -485,16 +509,73 @@ class TestReportDifference(unittest.TestCase): ["BM_SameTimes", "+0.0000", "+0.0000", "10", "10", "10", "10"], ["BM_2xFaster", "-0.5000", "-0.5000", "50", "25", "50", "25"], ["BM_2xSlower", "+1.0000", "+1.0000", "50", "100", "50", "100"], - ["BM_1PercentFaster", "-0.0100", "-0.0100", "100", "99", "100", "99"], - ["BM_1PercentSlower", "+0.0100", "+0.0100", "100", "101", "100", "101"], - ["BM_10PercentFaster", "-0.1000", "-0.1000", "100", "90", "100", "90"], - ["BM_10PercentSlower", "+0.1000", "+0.1000", "100", "110", "100", "110"], - ["BM_100xSlower", "+99.0000", "+99.0000", "100", "10000", "100", "10000"], - ["BM_100xFaster", "-0.9900", "-0.9900", "10000", "100", "10000", "100"], - ["BM_10PercentCPUToTime", "+0.1000", "-0.1000", "100", "110", "100", "90"], + [ + "BM_1PercentFaster", + "-0.0100", + "-0.0100", + "100", + "99", + "100", + "99", + ], + [ + "BM_1PercentSlower", + "+0.0100", + "+0.0100", + "100", + "101", + "100", + "101", + ], + [ + "BM_10PercentFaster", + "-0.1000", + "-0.1000", + "100", + "90", + "100", + "90", + ], + [ + "BM_10PercentSlower", + "+0.1000", + "+0.1000", + "100", + "110", + "100", + "110", + ], + [ + "BM_100xSlower", + "+99.0000", + "+99.0000", + "100", + "10000", + "100", + "10000", + ], + [ + "BM_100xFaster", + "-0.9900", + "-0.9900", + "10000", + "100", + "10000", + "100", + ], + [ + "BM_10PercentCPUToTime", + "+0.1000", + "-0.1000", + "100", + "110", + "100", + "90", + ], ["BM_ThirdFaster", "-0.3333", "-0.3334", "100", "67", "100", "67"], ["BM_NotBadTimeUnit", "-0.9000", "+0.2000", "0", "0", "0", "1"], - ["OVERALL_GEOMEAN", "-0.8344", "-0.8026", "0", "0", "0", "0"], + ["BM_hasLabel", "+0.0000", "+0.0000", "1", "1", "1", "1"], + ["OVERALL_GEOMEAN", "-0.8113", "-0.7779", "0", "0", "0", "0"], ] output_lines_with_header = print_difference_report( self.json_diff_report, use_color=False @@ -512,6 +593,7 @@ class TestReportDifference(unittest.TestCase): expected_output = [ { "name": "BM_SameTimes", + "label": "", "measurements": [ { "time": 0.0000, @@ -527,6 +609,7 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_2xFaster", + "label": "", "measurements": [ { "time": -0.5000, @@ -542,6 +625,7 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_2xSlower", + "label": "", "measurements": [ { "time": 1.0000, @@ -557,6 +641,7 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_1PercentFaster", + "label": "", "measurements": [ { "time": -0.0100, @@ -572,6 +657,7 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_1PercentSlower", + "label": "", "measurements": [ { "time": 0.0100, @@ -587,6 +673,7 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_10PercentFaster", + "label": "", "measurements": [ { "time": -0.1000, @@ -602,6 +689,7 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_10PercentSlower", + "label": "", "measurements": [ { "time": 0.1000, @@ -617,6 +705,7 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_100xSlower", + "label": "", "measurements": [ { "time": 99.0000, @@ -632,6 +721,7 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_100xFaster", + "label": "", "measurements": [ { "time": -0.9900, @@ -647,6 +737,7 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_10PercentCPUToTime", + "label": "", "measurements": [ { "time": 0.1000, @@ -662,6 +753,7 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_ThirdFaster", + "label": "", "measurements": [ { "time": -0.3333, @@ -677,6 +769,7 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_NotBadTimeUnit", + "label": "", "measurements": [ { "time": -0.9000, @@ -690,16 +783,33 @@ class TestReportDifference(unittest.TestCase): "time_unit": "s", "utest": {}, }, + { + "name": "BM_hasLabel", + "label": "a label", + "measurements": [ + { + "time": 0.0000, + "cpu": 0.0000, + "real_time": 1, + "real_time_other": 1, + "cpu_time": 1, + "cpu_time_other": 1, + } + ], + "time_unit": "s", + "utest": {}, + }, { "name": "OVERALL_GEOMEAN", + "label": "", "measurements": [ { - "real_time": 1.193776641714438e-06, - "cpu_time": 1.2144445585302297e-06, + "real_time": 3.1622776601683826e-06, + "cpu_time": 3.2130844755623912e-06, "real_time_other": 1.9768988699420897e-07, "cpu_time_other": 2.397447755209533e-07, - "time": -0.834399601997324, - "cpu": -0.8025889499549471, + "time": -0.8112976497120911, + "cpu": -0.7778551721181174, } ], "time_unit": "s", @@ -711,6 +821,7 @@ class TestReportDifference(unittest.TestCase): self.assertEqual(len(self.json_diff_report), len(expected_output)) for out, expected in zip(self.json_diff_report, expected_output): self.assertEqual(out["name"], expected["name"]) + self.assertEqual(out["label"], expected["label"]) self.assertEqual(out["time_unit"], expected["time_unit"]) assert_utest(self, out, expected) assert_measurements(self, out, expected) @@ -1086,7 +1197,9 @@ class TestReportDifferenceWithUTest(unittest.TestCase): assert_measurements(self, out, expected) -class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly(unittest.TestCase): +class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly( + unittest.TestCase +): @classmethod def setUpClass(cls): def load_results(): @@ -1369,12 +1482,108 @@ class TestReportSorting(unittest.TestCase): for n in range(len(self.json["benchmarks"]) ** 2): random.shuffle(self.json["benchmarks"]) - sorted_benchmarks = util.sort_benchmark_results(self.json)["benchmarks"] + sorted_benchmarks = util.sort_benchmark_results(self.json)[ + "benchmarks" + ] self.assertEqual(len(expected_names), len(sorted_benchmarks)) for out, expected in zip(sorted_benchmarks, expected_names): self.assertEqual(out["name"], expected) +class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly2( + unittest.TestCase +): + @classmethod + def setUpClass(cls): + def load_results(): + import json + + testInputs = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "Inputs" + ) + testOutput1 = os.path.join(testInputs, "test5_run0.json") + testOutput2 = os.path.join(testInputs, "test5_run1.json") + with open(testOutput1, "r") as f: + json1 = json.load(f) + json1["benchmarks"] = [ + json1["benchmarks"][0] for i in range(1000) + ] + with open(testOutput2, "r") as f: + json2 = json.load(f) + json2["benchmarks"] = [ + json2["benchmarks"][0] for i in range(1000) + ] + return json1, json2 + + json1, json2 = load_results() + cls.json_diff_report = get_difference_report(json1, json2, utest=True) + + def test_json_diff_report_pretty_printing(self): + expect_line = [ + "BM_ManyRepetitions_pvalue", + "0.0000", + "0.0000", + "U", + "Test,", + "Repetitions:", + "1000", + "vs", + "1000", + ] + output_lines_with_header = print_difference_report( + self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False + ) + output_lines = output_lines_with_header[2:] + found = False + for i in range(0, len(output_lines)): + parts = [x for x in output_lines[i].split(" ") if x] + found = expect_line == parts + if found: + break + self.assertTrue(found) + + def test_json_diff_report(self): + expected_output = [ + { + "name": "BM_ManyRepetitions", + "label": "", + "time_unit": "s", + "run_type": "", + "aggregate_name": "", + "utest": { + "have_optimal_repetitions": True, + "cpu_pvalue": 0.0, + "time_pvalue": 0.0, + "nr_of_repetitions": 1000, + "nr_of_repetitions_other": 1000, + }, + }, + { + "name": "OVERALL_GEOMEAN", + "label": "", + "measurements": [ + { + "real_time": 1.0, + "cpu_time": 1000.000000000069, + "real_time_other": 1000.000000000069, + "cpu_time_other": 1.0, + "time": 999.000000000069, + "cpu": -0.9990000000000001, + } + ], + "time_unit": "s", + "run_type": "aggregate", + "aggregate_name": "geomean", + "utest": {}, + }, + ] + self.assertEqual(len(self.json_diff_report), len(expected_output)) + for out, expected in zip(self.json_diff_report, expected_output): + self.assertEqual(out["name"], expected["name"]) + self.assertEqual(out["time_unit"], expected["time_unit"]) + assert_utest(self, out, expected) + + def assert_utest(unittest_instance, lhs, rhs): if lhs["utest"]: unittest_instance.assertAlmostEqual( diff --git a/third-party/benchmark/tools/gbench/util.py b/third-party/benchmark/tools/gbench/util.py index a46903a6d248..4d061a3a1e34 100644 --- a/third-party/benchmark/tools/gbench/util.py +++ b/third-party/benchmark/tools/gbench/util.py @@ -2,10 +2,10 @@ """ import json import os -import tempfile +import re import subprocess import sys -import functools +import tempfile # Input file type enumeration IT_Invalid = 0 @@ -58,7 +58,7 @@ def classify_input_file(filename): """ Return a tuple (type, msg) where 'type' specifies the classified type of 'filename'. If 'type' is 'IT_Invalid' then 'msg' is a human readable - string represeting the error. + string representing the error. """ ftype = IT_Invalid err_msg = None @@ -72,7 +72,8 @@ def classify_input_file(filename): ftype = IT_JSON else: err_msg = ( - "'%s' does not name a valid benchmark executable or JSON file" % filename + "'%s' does not name a valid benchmark executable or JSON file" + % filename ) return ftype, err_msg @@ -113,13 +114,41 @@ def remove_benchmark_flags(prefix, benchmark_flags): return [f for f in benchmark_flags if not f.startswith(prefix)] -def load_benchmark_results(fname): +def load_benchmark_results(fname, benchmark_filter): """ Read benchmark output from a file and return the JSON object. + + Apply benchmark_filter, a regular expression, with nearly the same + semantics of the --benchmark_filter argument. May be None. + Note: the Python regular expression engine is used instead of the + one used by the C++ code, which may produce different results + in complex cases. + REQUIRES: 'fname' names a file containing JSON benchmark output. """ + + def benchmark_wanted(benchmark): + if benchmark_filter is None: + return True + name = benchmark.get("run_name", None) or benchmark["name"] + return re.search(benchmark_filter, name) is not None + with open(fname, "r") as f: - return json.load(f) + results = json.load(f) + if "context" in results: + if "json_schema_version" in results["context"]: + json_schema_version = results["context"]["json_schema_version"] + if json_schema_version != 1: + print( + "In %s, got unnsupported JSON schema version: %i, expected 1" + % (fname, json_schema_version) + ) + sys.exit(1) + if "benchmarks" in results: + results["benchmarks"] = list( + filter(benchmark_wanted, results["benchmarks"]) + ) + return results def sort_benchmark_results(result): @@ -168,7 +197,9 @@ def run_benchmark(exe_name, benchmark_flags): is_temp_output = True thandle, output_name = tempfile.mkstemp() os.close(thandle) - benchmark_flags = list(benchmark_flags) + ["--benchmark_out=%s" % output_name] + benchmark_flags = list(benchmark_flags) + [ + "--benchmark_out=%s" % output_name + ] cmd = [exe_name] + benchmark_flags print("RUNNING: %s" % " ".join(cmd)) @@ -176,7 +207,7 @@ def run_benchmark(exe_name, benchmark_flags): if exitCode != 0: print("TEST FAILED...") sys.exit(exitCode) - json_res = load_benchmark_results(output_name) + json_res = load_benchmark_results(output_name, None) if is_temp_output: os.unlink(output_name) return json_res @@ -191,7 +222,10 @@ def run_or_load_benchmark(filename, benchmark_flags): """ ftype = check_input_file(filename) if ftype == IT_JSON: - return load_benchmark_results(filename) + benchmark_filter = find_benchmark_flag( + "--benchmark_filter=", benchmark_flags + ) + return load_benchmark_results(filename, benchmark_filter) if ftype == IT_Executable: return run_benchmark(filename, benchmark_flags) raise ValueError("Unknown file type %s" % ftype) diff --git a/third-party/benchmark/tools/requirements.txt b/third-party/benchmark/tools/requirements.txt index 3b3331b5af12..f32f35b8fbfd 100644 --- a/third-party/benchmark/tools/requirements.txt +++ b/third-party/benchmark/tools/requirements.txt @@ -1 +1,2 @@ -scipy>=1.5.0 \ No newline at end of file +numpy == 1.25 +scipy == 1.10.0 diff --git a/third-party/benchmark/tools/strip_asm.py b/third-party/benchmark/tools/strip_asm.py index 086255dc6577..bc3a774a7932 100755 --- a/third-party/benchmark/tools/strip_asm.py +++ b/third-party/benchmark/tools/strip_asm.py @@ -1,20 +1,20 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ strip_asm.py - Cleanup ASM output for the specified file """ -from argparse import ArgumentParser -import sys import os import re +import sys +from argparse import ArgumentParser def find_used_labels(asm): found = set() - label_re = re.compile("\s*j[a-z]+\s+\.L([a-zA-Z0-9][a-zA-Z0-9_]*)") - for l in asm.splitlines(): - m = label_re.match(l) + label_re = re.compile(r"\s*j[a-z]+\s+\.L([a-zA-Z0-9][a-zA-Z0-9_]*)") + for line in asm.splitlines(): + m = label_re.match(line) if m: found.add(".L%s" % m.group(1)) return found @@ -23,8 +23,8 @@ def find_used_labels(asm): def normalize_labels(asm): decls = set() label_decl = re.compile("^[.]{0,1}L([a-zA-Z0-9][a-zA-Z0-9_]*)(?=:)") - for l in asm.splitlines(): - m = label_decl.match(l) + for line in asm.splitlines(): + m = label_decl.match(line) if m: decls.add(m.group(0)) if len(decls) == 0: @@ -33,7 +33,7 @@ def normalize_labels(asm): if not needs_dot: return asm for ld in decls: - asm = re.sub("(^|\s+)" + ld + "(?=:|\s)", "\\1." + ld, asm) + asm = re.sub(r"(^|\s+)" + ld + r"(?=:|\s)", "\\1." + ld, asm) return asm @@ -41,11 +41,11 @@ def transform_labels(asm): asm = normalize_labels(asm) used_decls = find_used_labels(asm) new_asm = "" - label_decl = re.compile("^\.L([a-zA-Z0-9][a-zA-Z0-9_]*)(?=:)") - for l in asm.splitlines(): - m = label_decl.match(l) + label_decl = re.compile(r"^\.L([a-zA-Z0-9][a-zA-Z0-9_]*)(?=:)") + for line in asm.splitlines(): + m = label_decl.match(line) if not m or m.group(0) in used_decls: - new_asm += l + new_asm += line new_asm += "\n" return new_asm @@ -63,21 +63,24 @@ def is_identifier(tk): return True -def process_identifiers(l): +def process_identifiers(line): """ process_identifiers - process all identifiers and modify them to have consistent names across all platforms; specifically across ELF and MachO. For example, MachO inserts an additional understore at the beginning of names. This function removes that. """ - parts = re.split(r"([a-zA-Z0-9_]+)", l) + parts = re.split(r"([a-zA-Z0-9_]+)", line) new_line = "" for tk in parts: if is_identifier(tk): if tk.startswith("__Z"): tk = tk[1:] elif ( - tk.startswith("_") and len(tk) > 1 and tk[1].isalpha() and tk[1] != "Z" + tk.startswith("_") + and len(tk) > 1 + and tk[1].isalpha() + and tk[1] != "Z" ): tk = tk[1:] new_line += tk @@ -93,33 +96,35 @@ def process_asm(asm): # TODO: Add more things we want to remove discard_regexes = [ - re.compile("\s+\..*$"), # directive - re.compile("\s*#(NO_APP|APP)$"), # inline ASM - re.compile("\s*#.*$"), # comment line - re.compile("\s*\.globa?l\s*([.a-zA-Z_][a-zA-Z0-9$_.]*)"), # global directive + re.compile(r"\s+\..*$"), # directive + re.compile(r"\s*#(NO_APP|APP)$"), # inline ASM + re.compile(r"\s*#.*$"), # comment line + re.compile( + r"\s*\.globa?l\s*([.a-zA-Z_][a-zA-Z0-9$_.]*)" + ), # global directive re.compile( - "\s*\.(string|asciz|ascii|[1248]?byte|short|word|long|quad|value|zero)" + r"\s*\.(string|asciz|ascii|[1248]?byte|short|word|long|quad|value|zero)" ), ] - keep_regexes = [] + keep_regexes: list[re.Pattern] = [] fn_label_def = re.compile("^[a-zA-Z_][a-zA-Z0-9_.]*:") - for l in asm.splitlines(): + for line in asm.splitlines(): # Remove Mach-O attribute - l = l.replace("@GOTPCREL", "") + line = line.replace("@GOTPCREL", "") add_line = True for reg in discard_regexes: - if reg.match(l) is not None: + if reg.match(line) is not None: add_line = False break for reg in keep_regexes: - if reg.match(l) is not None: + if reg.match(line) is not None: add_line = True break if add_line: - if fn_label_def.match(l) and len(new_contents) != 0: + if fn_label_def.match(line) and len(new_contents) != 0: new_contents += "\n" - l = process_identifiers(l) - new_contents += l + line = process_identifiers(line) + new_contents += line new_contents += "\n" return new_contents @@ -127,7 +132,11 @@ def process_asm(asm): def main(): parser = ArgumentParser(description="generate a stripped assembly file") parser.add_argument( - "input", metavar="input", type=str, nargs=1, help="An input assembly file" + "input", + metavar="input", + type=str, + nargs=1, + help="An input assembly file", ) parser.add_argument( "out", metavar="output", type=str, nargs=1, help="The output file" @@ -136,9 +145,9 @@ def main(): input = args.input[0] output = args.out[0] if not os.path.isfile(input): - print(("ERROR: input file '%s' does not exist") % input) + print("ERROR: input file '%s' does not exist" % input) sys.exit(1) - contents = None + with open(input, "r") as f: contents = f.read() new_contents = process_asm(contents) -- GitLab From d95e6d027486876559f1a2a96c33b8ad93cc0ae4 Mon Sep 17 00:00:00 2001 From: Guray Ozen Date: Mon, 4 Mar 2024 21:03:59 +0000 Subject: [PATCH 047/929] [mlir] GEMM Hopper Tensor Core Integration Test (#81478) --- .../GPU/CUDA/sm90/python/lit.local.cfg | 2 + .../GPU/CUDA/sm90/python/matmul.py | 341 +++++ .../GPU/CUDA/sm90/python/tools/lit.local.cfg | 3 + .../CUDA/sm90/python/tools/matmulBuilder.py | 1156 +++++++++++++++++ .../CUDA/sm90/python/tools/nvgpucompiler.py | 45 + 5 files changed, 1547 insertions(+) create mode 100644 mlir/test/Integration/GPU/CUDA/sm90/python/lit.local.cfg create mode 100644 mlir/test/Integration/GPU/CUDA/sm90/python/matmul.py create mode 100644 mlir/test/Integration/GPU/CUDA/sm90/python/tools/lit.local.cfg create mode 100644 mlir/test/Integration/GPU/CUDA/sm90/python/tools/matmulBuilder.py create mode 100644 mlir/test/Integration/GPU/CUDA/sm90/python/tools/nvgpucompiler.py diff --git a/mlir/test/Integration/GPU/CUDA/sm90/python/lit.local.cfg b/mlir/test/Integration/GPU/CUDA/sm90/python/lit.local.cfg new file mode 100644 index 000000000000..2d5a9d00e732 --- /dev/null +++ b/mlir/test/Integration/GPU/CUDA/sm90/python/lit.local.cfg @@ -0,0 +1,2 @@ +if not config.enable_cuda_runner or not config.mlir_run_cuda_sm90_tests: + config.unsupported = True diff --git a/mlir/test/Integration/GPU/CUDA/sm90/python/matmul.py b/mlir/test/Integration/GPU/CUDA/sm90/python/matmul.py new file mode 100644 index 000000000000..cb7248ef23cd --- /dev/null +++ b/mlir/test/Integration/GPU/CUDA/sm90/python/matmul.py @@ -0,0 +1,341 @@ +# RUN: env SUPPORT_LIB=%mlir_cuda_runtime \ +# RUN: %PYTHON %s | FileCheck %s + + +# ===--- GEMM Hopper Tensor Core Integration Test ---=== +# +# This test aims to validate the correctness of the supported GEMM kernels in +# NVGPU dialects, with current support for Multistage and Warp Specialization +# kernels. +# The test constructs and metaprograms IR using Python bindings, allowing +# generic IR building. This flexibility enables changes to the shape, +# tile size, or data type of the GEMM for testing purposes. +# The entry function is `matmul`, where one can specify GEMM shape, tile size, +# data type, GEMM algorithm (Multistage or Warp Specialization), and the maximum +# number of stages. +# Verification is done via numpy's matmul operation. +# +# Example: +# matmul(input_type=np.float16, # input types +# output_type=np.float32, # output type +# M=4096, N=4096, K=4096, # Shape +# BLOCK_M=128, BLOCK_N=128, BLOCK_K=64, # Tile Size +# use_warp_specialization=True, # Enable Warp Specialization +# max_num_stages=3) # Number of stages in shared memory +# +# ===--- Parallelism Across CTAs ---=== +# +# GEMM includes three loops defining the shape of the GEMM, specified in the +# `matmul` function. +# The program builds IR using the following loop structure, tiling the loops +# with the given tile size and parallelizing the two outermost loops into the +# first and second dimensions of CTAs. +# +# for(bi = 0; i < M; i += BLOCK_M) # parallelize across blockIdx.x +# for(bj = 0; j < N; j += BLOCK_N) # parallelize across blockIdx.y +# for(bk = 0; k < K; K += BLOCK_K) +# for(i = bi; i < (bi + BLOCK_M); ++i) +# for(j = bj; j < (bj + BLOCK_N); ++j) +# for(k = bk; k < (bk + BLOCK_K); ++k) +# +# ===--- Multistage Kernel ---=== +# +# This kernel launches a single warp group (128 threads). The primary thread +# (pthread) requests load from TMA. Threads collectively wait for the data and +# perform mma operations. After completing the shape, threads together store +# first fragmented registers to shared memory, then from shared memory to global +# memory; this part is called the epilogue. +# +# Execution Timeline of Multistage Kernel with 3 stages: +# +-------+----------------+--------------------+--------------------+--------------------+-----+-----------------------+ +# | |Prologue ----> |MainLoop ----> |Epilogue | +# +-------+----------------+--------------------+--------------------+--------------------+-----+-----------------------+ +# |pthread|[tma-0,1,2] |[wait-0][mma][tma-2]|[wait-1][mma][tma-0]|[wait-2][mma][tma-1]| ... | [mma-wait] |[epilogue]| +# |wgroup | ........ |[wait-0][mma] |[wait-1][mma] |[wait-2][mma] | ... | [mma-wait] |[epilogue]| +# +-------+----------------+--------------------+--------------------+--------------------+-----+-----------------------+ +# +# ===--- Warp Specialization Kernel ---=== +# +# This kernel launches 2 warp groups (2x128 threads) per CTA, specializing one +# as `producer warp group` and another as `consumer warp group`. The +# `producer warp group` is responsible for requesting TMA load, while the +# `consumer warp group` performs the mma operation. The epilogue section is +# handled by the `consumer warp group` as its threads own the fragmented registers. +# +# Execution Timeline of Warp Specialization Kernel with 2 stages: +# +--------+--------+---------+---------+---------+-----------------------+---+--------------+-----------------+ +# | |MainLoop ----> | 1st Epilogue | 2nd Epilogue | +# +--------+--------+---------+---------+---------+-----------------------+---+--------------+-----------------+ +# |pthread1|[tma-0] | [tma-1] | [tma-0] | [tma-1] | ..........................| ........... | [shmem->global] | +# |wgroup1 | .......| | | | | | [shmem->global] | +# +--------+--------+---------+---------+---------+-----------------------+---+--------------+-----------------+ +# |wgroup2 |[wait-0][mma], [wait-1][mma], [wait-0][mma], [wait-1][mma], ......| [reg->shmem] | [shmem->global]| +# +--------+--------+---------+---------+---------+-----------------------+---+--------------+-----------------+ + +import errno +import numpy as np +import subprocess +import ctypes +from tools import nvgpucompiler +from tools import matmulBuilder +import contextlib +import os +import sys +import pathlib +import ctypes +from mlir import runtime as rt + + +def generate_matmul( + input_type=np.float16, + output_type=np.float32, + M=4096, + N=4096, + K=4096, + BLOCK_M=128, + BLOCK_N=128, + BLOCK_K=64, + use_warp_specialization=True, + saveIR=False, + max_num_stages=3, + options=f"cubin-chip=sm_90a cubin-features=+ptx80 opt-level=3", +): + with matmulBuilder.ir.Context() as ctx, matmulBuilder.ir.Location.unknown(): + if use_warp_specialization: + mlir_nvgpu_module = matmulBuilder.generate_matmul_ws( + input_type, + output_type, + M, + N, + K, + BLOCK_M, + BLOCK_N, + BLOCK_K, + max_num_stages, + ) + else: + mlir_nvgpu_module = matmulBuilder.generate_matmul_multistage( + input_type, + output_type, + M, + N, + K, + BLOCK_M, + BLOCK_N, + BLOCK_K, + max_num_stages, + ) + + mlir_nvgpu_module.operation.verify() + + # Save generated IR + if saveIR: + # print(mlir_nvgpu_module) + original_stdout = sys.stdout + with open("gemm.mlir", "w") as f: + sys.stdout = f + print(mlir_nvgpu_module) + sys.stdout = original_stdout + + # Get compiler + support_lib = os.getenv("SUPPORT_LIB") + if not os.path.exists(support_lib): + raise FileNotFoundError( + errno.ENOENT, os.strerror(errno.ENOENT), support_lib + ) + compiler = nvgpucompiler.NvgpuCompiler( + options, opt_level=3, shared_libs=[support_lib] + ) + + # Compile + engine = compiler.compile_and_jit(mlir_nvgpu_module) + return engine + + +def matmul( + input_type=np.float16, + output_type=np.float32, + M=128, + N=128, + K=128, + BLOCK_M=128, + BLOCK_N=128, + BLOCK_K=64, + use_warp_specialization=True, + saveIR=False, + max_num_stages=3, + print_results=False, + no_verify=False, +): + # Print the configuration + required_stages = (M * K + K * N) // (BLOCK_M * BLOCK_K + BLOCK_K * BLOCK_N) + num_stages = min(required_stages, max_num_stages) + ity = "f16" if input_type == np.float16 else "f32" + oty = "f16" if output_type == np.float16 else "f32" + gemmty = "Warp specialization" if use_warp_specialization else "Multistage" + print( + "===-- Running GEMM " + + gemmty + + " " + + oty + + " += " + + ity + + " * " + + ity + + ", Size " + + str(M) + + "x" + + str(N) + + "x" + + str(K) + + ", Tile " + + str(BLOCK_M) + + "x" + + str(BLOCK_N) + + "x" + + str(BLOCK_K) + + ", stages " + + str(num_stages) + + " --===" + ) + + # Build IR and compile + engine = generate_matmul( + input_type, + output_type, + M, + N, + K, + BLOCK_M, + BLOCK_N, + BLOCK_K, + use_warp_specialization, + saveIR, + num_stages, + ) + + # Allocate matrices and invoke the matmul + c = np.zeros((M, N), output_type) + a = np.random.randn(M, K).astype(input_type) + b = np.random.randn(K, N).astype(input_type) + mem_a = ctypes.pointer(ctypes.pointer(rt.get_ranked_memref_descriptor(a))) + mem_b = ctypes.pointer(ctypes.pointer(rt.get_ranked_memref_descriptor(b))) + mem_c = ctypes.pointer(ctypes.pointer(rt.get_ranked_memref_descriptor(c))) + kernelName = matmulBuilder.make_kernel_name( + input_type, + output_type, + M, + N, + K, + BLOCK_M, + BLOCK_N, + BLOCK_K, + num_stages, + use_warp_specialization, + ) + + # Launch the MLIR generated kernel + engine.invoke(kernelName, mem_a, mem_b, mem_c) + + float_formatter = "{:.2f}".format + np.set_printoptions(formatter={"float_kind": float_formatter}) + + if print_results: + print(c) + + # Verify the results + if not no_verify: + ref = a.astype(input_type) @ b.astype(input_type) + if print_results: + print(ref) + np.testing.assert_allclose(c, ref, rtol=5e-03, atol=1e-01) + + print("PASS ") + + +# Takes longer time to run +def test_long(): + for stages in range(1, 7): + for M in [128, 512, 1024, 4096, 8192]: + for N in [128, 512, 1024, 4096, 8192]: + for K in [64, 128, 512, 1024, 4096, 8192]: + matmul( + np.float16, + np.float32, + M, + N, + K, + max_num_stages=stages, + use_warp_specialization=False, + no_verify=True, + ) + matmul( + np.float16, + np.float32, + M, + N, + K, + max_num_stages=stages, + use_warp_specialization=True, + ) + + +def test_short(): + for stages in [1, 3]: + for M in [128, 512]: + for N in [128]: + for K in [64, 256]: + matmul( + np.float16, + np.float32, + M, + N, + K, + max_num_stages=stages, + use_warp_specialization=False, + ) + matmul( + np.float16, + np.float32, + M, + N, + K, + max_num_stages=stages, + use_warp_specialization=True, + ) + + +# CHECK: ===-- Running GEMM Multistage f32 += f16 * f16, Size 128x128x64, Tile 128x128x64, stages 1 --=== +# CHECK: PASS +# CHECK: ===-- Running GEMM Warp specialization f32 += f16 * f16, Size 128x128x64, Tile 128x128x64, stages 1 --=== +# CHECK: PASS +# CHECK: ===-- Running GEMM Multistage f32 += f16 * f16, Size 128x128x256, Tile 128x128x64, stages 1 --=== +# CHECK: PASS +# CHECK: ===-- Running GEMM Warp specialization f32 += f16 * f16, Size 128x128x256, Tile 128x128x64, stages 1 --=== +# CHECK: PASS +# CHECK: ===-- Running GEMM Multistage f32 += f16 * f16, Size 512x128x64, Tile 128x128x64, stages 1 --=== +# CHECK: PASS +# CHECK: ===-- Running GEMM Warp specialization f32 += f16 * f16, Size 512x128x64, Tile 128x128x64, stages 1 --=== +# CHECK: PASS +# CHECK: ===-- Running GEMM Multistage f32 += f16 * f16, Size 512x128x256, Tile 128x128x64, stages 1 --=== +# CHECK: PASS +# CHECK: ===-- Running GEMM Warp specialization f32 += f16 * f16, Size 512x128x256, Tile 128x128x64, stages 1 --=== +# CHECK: PASS +# CHECK: ===-- Running GEMM Multistage f32 += f16 * f16, Size 128x128x64, Tile 128x128x64, stages 1 --=== +# CHECK: PASS +# CHECK: ===-- Running GEMM Warp specialization f32 += f16 * f16, Size 128x128x64, Tile 128x128x64, stages 1 --=== +# CHECK: PASS +# CHECK: ===-- Running GEMM Multistage f32 += f16 * f16, Size 128x128x256, Tile 128x128x64, stages 3 --=== +# CHECK: PASS +# CHECK: ===-- Running GEMM Warp specialization f32 += f16 * f16, Size 128x128x256, Tile 128x128x64, stages 3 --=== +# CHECK: PASS +# CHECK: ===-- Running GEMM Multistage f32 += f16 * f16, Size 512x128x64, Tile 128x128x64, stages 2 --=== +# CHECK: PASS +# CHECK: ===-- Running GEMM Warp specialization f32 += f16 * f16, Size 512x128x64, Tile 128x128x64, stages 2 --=== +# CHECK: PASS +# CHECK: ===-- Running GEMM Multistage f32 += f16 * f16, Size 512x128x256, Tile 128x128x64, stages 3 --=== +# CHECK: PASS +# CHECK: ===-- Running GEMM Warp specialization f32 += f16 * f16, Size 512x128x256, Tile 128x128x64, stages 3 --=== +# CHECK: PASS + +test_short() diff --git a/mlir/test/Integration/GPU/CUDA/sm90/python/tools/lit.local.cfg b/mlir/test/Integration/GPU/CUDA/sm90/python/tools/lit.local.cfg new file mode 100644 index 000000000000..d9f34f219c4d --- /dev/null +++ b/mlir/test/Integration/GPU/CUDA/sm90/python/tools/lit.local.cfg @@ -0,0 +1,3 @@ +# Files in this directory are tools, not tests. +config.unsupported = True + diff --git a/mlir/test/Integration/GPU/CUDA/sm90/python/tools/matmulBuilder.py b/mlir/test/Integration/GPU/CUDA/sm90/python/tools/matmulBuilder.py new file mode 100644 index 000000000000..fac138dce605 --- /dev/null +++ b/mlir/test/Integration/GPU/CUDA/sm90/python/tools/matmulBuilder.py @@ -0,0 +1,1156 @@ +import numpy as np +from mlir import ir +from mlir.dialects import arith +from mlir.dialects import func +from mlir.dialects import gpu +from mlir.dialects import memref +from mlir.dialects import nvgpu +from mlir.dialects import nvvm +from mlir.dialects import llvm +from mlir.dialects import builtin +from mlir.dialects import scf +from mlir.dialects import vector +from mlir.extras import types as T + +TMA_LAST_DIM_F16 = 64 # 128B flaot16 +WARP_SIZE = 32 +WARP_GROUP_SIZE = WARP_SIZE * 4 + +PRODUCER_REGISTER_SIZE = 40 +CONSUMER_REGISTER_SIZE = 232 + +PRODUCER_PRIMARY_THREAD = 128 +CONSUMER_PRIMARY_THREAD = 0 + +# C++ uses this value to understand whether it's dynamic or not. +MLIR_DYNAMIC = -9223372036854775808 + +DEBUG = False + + +def debug_print(fmt, *args, predicate=None, threadNumber=-1, forcePrint=False): + if not DEBUG and not forcePrint: + return + type_formats = [] + for arg in args: + ty_format = None + if ir.IndexType.isinstance(arg.type): + ty_format = "%llu" + if ir.IntegerType.isinstance(arg.type): + width = ir.IntegerType(arg.type).width + if width == 64: + ty_format = "%llu" + elif width == 32: + ty_format = "%d" + elif width == 1: + ty_format = "%i" + if ir.F32Type.isinstance(arg.type): + ty_format = "%f" + if ty_format is None: + raise NotImplementedError(arg.type) + type_formats.append(ty_format) + if threadNumber != -1: + tidx = gpu.thread_id(gpu.Dimension.x) + predicate = arith.cmpi(arith.CmpIPredicate.eq, tidx, c(threadNumber)) + scf.yield_([]) + if_op = scf.IfOp(predicate) + with ir.InsertionPoint(if_op.then_block): + gpu.printf(fmt.format(*type_formats) + "\n", args) + scf.yield_([]) + + +def get_type_size(ty): + if ir.FloatType.isinstance(ty): + return ir.FloatType(ty).width // 8 + if ir.IntegerType.isinstance(ty): + return ir.IntegerType(ty).width // 8 + raise NotImplementedError(ty) + + +def get_mlir_ty(dtype): + if dtype == np.float16: + return T.f16() + if dtype == np.float32: + return T.f32() + if dtype == np.float64: + return T.f64() + if dtype == np.int32: + return T.i32() + if dtype == np.int64: + return T.i64() + raise NotImplementedError(dtype) + + +def c(value, ty=None): + ty = T.index() if ty is None else ty + return arith.constant(ty, value) + + +def make_kernel_name( + input_type=np.float16, + output_type=np.float32, + M=4096, + N=4096, + K=4096, + BLOCK_M=128, + BLOCK_N=128, + BLOCK_K=128, + num_stages=3, + use_warp_specialization=False, +): + kernelName = "warpspecialized" if use_warp_specialization else "multistage" + return ( + kernelName + + "_" + + str(M) + + "x" + + str(N) + + "x" + + str(K) + + "_" + + str(BLOCK_M) + + "x" + + str(BLOCK_N) + + "x" + + str(BLOCK_K) + + "_" + + str(num_stages) + ) + + +def generate_matmul_ws( + input_type=np.float16, + output_type=np.float32, + M=4096, + N=4096, + K=4096, + BLOCK_M=128, + BLOCK_N=128, + BLOCK_K=128, + num_stages=3, +): + # Limitaitons for now + assert input_type == np.float16 + assert output_type == np.float32 + assert BLOCK_M == 128 + assert BLOCK_N == 128 + assert BLOCK_K == 64 + assert M % BLOCK_M == 0 + assert N % BLOCK_N == 0 + assert K % BLOCK_K == 0 + + module = ir.Module.create() + token_ty = ir.Type.parse("!gpu.async.token") + a_elem_ty = get_mlir_ty(input_type) + b_elem_ty = get_mlir_ty(input_type) + c_elem_ty = get_mlir_ty(output_type) + a_ty = ir.MemRefType.get([M, K], a_elem_ty) + b_ty = ir.MemRefType.get((K, N), b_elem_ty) + c_ty = ir.MemRefType.get((M, N), c_elem_ty) + a_tile_shape = a_tma_shape = (BLOCK_M, TMA_LAST_DIM_F16) + b_tma_shape = (BLOCK_K, TMA_LAST_DIM_F16) + b_tile_shape = (BLOCK_K, BLOCK_N) + txcount = (b_tile_shape[0] * b_tile_shape[1] * get_type_size(a_elem_ty)) + ( + a_tile_shape[0] * a_tile_shape[1] * get_type_size(b_elem_ty) + ) + smem_space_str = "#gpu.address_space" + smem_space = ir.Attribute.parse(smem_space_str) + mbar_ty = ir.Type.parse( + "!nvgpu.mbarrier.group" + ) + a_tma_desc_ty = ir.Type.parse( + "!nvgpu.tensormap.descriptor, swizzle = swizzle_128b, l2promo=none, oob=zero, interleave=none>" + ) + b_tma_desc_ty = ir.Type.parse( + "!nvgpu.tensormap.descriptor, swizzle = swizzle_128b, l2promo=none, oob=zero, interleave=none>" + ) + acc_ty = ir.Type.parse( + "!nvgpu.warpgroup.accumulator>" + ) + a_wgmma_ty = ir.Type.parse( + "!nvgpu.warpgroup.descriptor>" + ) + b_wgmma_ty = ir.Type.parse( + "!nvgpu.warpgroup.descriptor>" + ) + kernelName = make_kernel_name( + input_type, output_type, M, N, K, BLOCK_M, BLOCK_N, BLOCK_K, num_stages, True + ) + with ir.InsertionPoint(module.body): + fop = func.FuncOp(kernelName, ([a_ty, b_ty, c_ty], [])) + with ir.InsertionPoint(fop.add_entry_block()): + a_host = fop.arguments[0] + b_host = fop.arguments[1] + c_host = fop.arguments[2] + lhs_tile_bytes = BLOCK_M * BLOCK_K * get_type_size(a_elem_ty) + rhs_tile_bytes = BLOCK_N * BLOCK_K * get_type_size(b_elem_ty) + smem_size_input = (lhs_tile_bytes + rhs_tile_bytes) * num_stages + smem_size_output = BLOCK_M * BLOCK_N * get_type_size(c_elem_ty) + smem_size = max(smem_size_input, smem_size_output) + + # Step 1. Allocate device memory and memcpy + t1 = gpu.wait(token_ty, []) + a_device, t2 = gpu.alloc(a_ty, token_ty, [t1], [], []) + b_device, t3 = gpu.alloc(b_ty, token_ty, [t2], [], []) + c_device, t4 = gpu.alloc(c_ty, token_ty, [t3], [], []) + t5 = gpu.memcpy(token_ty, [t4], a_device, a_host) + t6 = gpu.memcpy(token_ty, [t5], b_device, b_host) + t7 = gpu.wait(token_ty, [t6]) + + # Step 2. Create TMA Descriptors + tma_specs = [ + (a_device, a_tma_desc_ty, a_tma_shape), + (b_device, b_tma_desc_ty, b_tma_shape), + ] + tma_descs = [] + for x_device, tensor_map_ty, tile_shape in tma_specs: + x_unranked = memref.cast( + ir.UnrankedMemRefType.get(a_elem_ty, a_ty.memory_space), x_device + ) + tma_descs.append( + nvgpu.TmaCreateDescriptorOp( + tensor_map_ty, x_unranked, map(c, tile_shape) + ).result + ) + a_tma_desc, b_tma_desc = tma_descs + + # Step 3. Launch Kernel with 2 Warpgroups : 1 Producer, 1 Consumer + cta_m = M // BLOCK_M + cta_n = N // BLOCK_N + assert M % BLOCK_M == 0 and N % BLOCK_N == 0 + grid = (cta_m, cta_n, 1) + block = (WARP_GROUP_SIZE * 2, 1, 1) + launch_op = gpu.LaunchOp( + token_ty, + [t7], + *map(c, grid), + *map(c, block), + dynamicSharedMemorySize=c(smem_size, ty=T.i32()) + ) + launch_op.body.blocks.append(*([T.index()] * 12)) + with ir.InsertionPoint(launch_op.body.blocks[0]): + # GPU Step 0. This is need for vectorized ld/st + memref.assume_alignment(c_device, 16) + dynamic_smem = gpu.dynamic_shared_memory( + ir.MemRefType.get((MLIR_DYNAMIC,), T.i8(), memory_space=smem_space) + ) + ticks = c(10000000) + + # GPU Step 1. Bootstrapping: find the primary thread, warps, warp groups and etc. + tidx = gpu.thread_id(gpu.Dimension.x) + wgPrimaryThread = arith.cmpi( + arith.CmpIPredicate.eq, arith.remui(tidx, c(WARP_GROUP_SIZE)), c(0) + ) + warp_id = arith.divui(tidx, c(32)) + warpgroup_id = arith.divui(warp_id, c(4)) + is_producer = arith.cmpi( + arith.CmpIPredicate.eq, + warpgroup_id, + c(1 if PRODUCER_PRIMARY_THREAD == 128 else 0), + ) + is_consumer = arith.cmpi( + arith.CmpIPredicate.eq, + warpgroup_id, + c(0 if CONSUMER_PRIMARY_THREAD == 0 else 1), + ) + producerPrimaryThread = arith.cmpi( + arith.CmpIPredicate.eq, tidx, c(PRODUCER_PRIMARY_THREAD) + ) + consumerPrimaryThread = arith.cmpi( + arith.CmpIPredicate.eq, tidx, c(CONSUMER_PRIMARY_THREAD) + ) + bidx = gpu.block_id(gpu.Dimension.x) + bidy = gpu.block_id(gpu.Dimension.y) + dimX = arith.muli(bidx, c(BLOCK_M)) + dimY = arith.muli(bidy, c(BLOCK_N)) + + # GPU Step 2. Initialize mbarrier groups + mbarTMA = nvgpu.mbarrier_create(mbar_ty) + mbarDONE = nvgpu.mbarrier_create(mbar_ty) + for i in range(num_stages): + nvgpu.mbarrier_init(mbarTMA, c(1), c(i), predicate=wgPrimaryThread) + nvgpu.mbarrier_init(mbarDONE, c(1), c(i), predicate=wgPrimaryThread) + gpu.barrier() + + # GPU Step 3. Prefetch TMA descriptors + nvgpu.tma_prefetch_descriptor(a_tma_desc, predicate=wgPrimaryThread) + nvgpu.tma_prefetch_descriptor(b_tma_desc, predicate=wgPrimaryThread) + + ns = num_stages if num_stages == 1 else num_stages - 1 + # GPU Step 5. Producer Warpgroup (TMA Warpgroup) + with ir.InsertionPoint(scf.IfOp(is_producer).then_block): + # Step 5.1. Reduce register size + nvvm.setmaxregister( + PRODUCER_REGISTER_SIZE, nvvm.SetMaxRegisterAction.decrease + ) + + # Step 5.2. TMA Main Loop + for_op = scf.ForOp( + c(0), c(K // BLOCK_K), c(1), [arith.constant(T.bool(), 1)] + ) + with ir.InsertionPoint(for_op.body): + phaseParity = for_op.inner_iter_args[0] + iv = for_op.induction_variable + stage = arith.remui(iv, c(num_stages)) + + # Step 5.2.1. Wait mbarDONE + debug_print( + "[prod] iv={} | mbarDONE[{}] try_wait phase={}", + iv, + stage, + phaseParity, + predicate=producerPrimaryThread, + ) + nvgpu.MBarrierTryWaitParityOp( + mbarDONE, phaseParity, ticks, mbarId=stage + ) + debug_print( + "[prod] iv={} | mbarDONE[{}] try_wait phase={} [done]", + iv, + stage, + phaseParity, + predicate=producerPrimaryThread, + ) + p = arith.cmpi(arith.CmpIPredicate.eq, stage, c(num_stages - 1)) + phaseParity = arith.select( + p, + arith.xori(phaseParity, arith.constant(T.bool(), 1)), + phaseParity, + ) + + # Step 5.2.2. Load TMA + a_offset = arith.muli(stage, c(lhs_tile_bytes)) + a_tma_slice = memref.view( + ir.MemRefType.get( + a_tma_shape, a_elem_ty, memory_space=smem_space + ), + dynamic_smem, + a_offset, + [], + ) + b_offset = arith.addi( + arith.muli(stage, c(rhs_tile_bytes)), + c(lhs_tile_bytes * num_stages), + ) + b_tma_slice_1 = memref.view( + ir.MemRefType.get( + b_tma_shape, b_elem_ty, memory_space=smem_space + ), + dynamic_smem, + b_offset, + [], + ) + b_offset2 = arith.addi( + b_offset, + c(BLOCK_K * TMA_LAST_DIM_F16 * get_type_size(b_elem_ty)), + ) + b_tma_slice_2 = memref.view( + ir.MemRefType.get( + b_tma_shape, b_elem_ty, memory_space=smem_space + ), + dynamic_smem, + b_offset2, + [], + ) + debug_print( + "[prod] a_offset={} b_offset={} b_offset2={}", + a_offset, + b_offset, + b_offset2, + predicate=producerPrimaryThread, + ) + coord = arith.muli(c(64), iv) + nvgpu.TmaAsyncLoadOp( + a_tma_slice, + mbarTMA, + a_tma_desc, + coordinates=[coord, dimX], + mbarId=stage, + predicate=producerPrimaryThread, + ) + nvgpu.TmaAsyncLoadOp( + b_tma_slice_1, + mbarTMA, + b_tma_desc, + coordinates=[dimY, coord], + mbarId=stage, + predicate=producerPrimaryThread, + ) + dimY2 = arith.addi(dimY, c(64)) + nvgpu.TmaAsyncLoadOp( + b_tma_slice_2, + mbarTMA, + b_tma_desc, + coordinates=[dimY2, coord], + mbarId=stage, + predicate=producerPrimaryThread, + ) + + # Step 5.2.3. Arrive mbarTMA + debug_print( + "[prod] iv={} | mbarTMA[{}] arrive", + iv, + stage, + predicate=producerPrimaryThread, + ) + nvgpu.mbarrier_arrive_expect_tx( + mbarTMA, c(txcount), stage, predicate=producerPrimaryThread + ) + debug_print( + "[prod] iv={} | mbarTMA[{}] arrive [done]", + iv, + stage, + predicate=producerPrimaryThread, + ) + scf.yield_([phaseParity]) + scf.yield_([]) + + # GPU Step 6. Consumer Warpgroup (MMA Warpgroup) + if_op = scf.IfOp(is_consumer) + with ir.InsertionPoint(if_op.then_block): + # Step 6.1. Increase register size + nvvm.setmaxregister( + CONSUMER_REGISTER_SIZE, nvvm.SetMaxRegisterAction.increase + ) + + # GPU Step 6.2. Initialize MMA registers + acc = nvgpu.warpgroup_mma_init_accumulator(acc_ty) + + # Step 6.3. MMA Main Loop + for_op = scf.ForOp( + c(0), c(K // BLOCK_K), c(1), [acc, arith.constant(T.bool(), 0)] + ) + with ir.InsertionPoint(for_op.body): + # Step 6.3.1. Wait mbar1 + phaseParity = for_op.inner_iter_args[1] + iv = for_op.induction_variable + stage = arith.remui(iv, c(num_stages)) + debug_print( + "[cons] iv={} | mbarTMA[{}] try_wait phase={}", + iv, + stage, + phaseParity, + predicate=consumerPrimaryThread, + ) + nvgpu.MBarrierTryWaitParityOp( + mbarTMA, phaseParity, ticks, mbarId=stage + ) + debug_print( + "[cons] iv={} | mbarTMA[{}] try_wait phase={} [done]", + iv, + stage, + phaseParity, + predicate=consumerPrimaryThread, + ) + + # Step 6.3.2. Create WGMMA Descriptors + a_offset = arith.muli(stage, c(lhs_tile_bytes)) + a_tile_slice = memref.view( + ir.MemRefType.get( + a_tile_shape, a_elem_ty, memory_space=smem_space + ), + dynamic_smem, + a_offset, + [], + ) + b_offset = arith.addi( + arith.muli(stage, c(rhs_tile_bytes)), + c(lhs_tile_bytes * num_stages), + ) + b_tile_slice = memref.view( + ir.MemRefType.get( + b_tile_shape, b_elem_ty, memory_space=smem_space + ), + dynamic_smem, + b_offset, + [], + ) + debug_print( + "[cons] a_offset={} b_offset={}", + a_offset, + b_offset, + predicate=consumerPrimaryThread, + ) + da = nvgpu.WarpgroupGenerateDescriptorOp( + a_wgmma_ty, a_tile_slice, a_tma_desc + ) + db = nvgpu.WarpgroupGenerateDescriptorOp( + b_wgmma_ty, b_tile_slice, b_tma_desc + ) + + # Step 6.3.3. MMA + carry_acc = for_op.inner_iter_args[0] + new_acc = nvgpu.WarpgroupMmaOp( + acc.type, da, db, carry_acc, transposeB=True + ) + + # Step 6.3.4. Arrive mbarDONE + if num_stages == 1: + p_arrive = consumerPrimaryThread + else: + p1 = arith.cmpi(arith.CmpIPredicate.sgt, iv, c(0)) + p_arrive = arith.andi(consumerPrimaryThread, p1) + with ir.InsertionPoint(scf.IfOp(p_arrive).then_block): + p = arith.cmpi(arith.CmpIPredicate.eq, stage, c(0)) + barId = arith.select( + p, c(num_stages - 1), arith.subi(stage, c(1)) + ) + debug_print( + "[cons] iv={} | mbarDONE[{}] arrive ", + iv, + barId, + predicate=consumerPrimaryThread, + ) + nvgpu.mbarrier_arrive( + ir.Type.parse("!nvgpu.mbarrier.token"), mbarDONE, barId + ) + debug_print( + "[cons] iv={} | mbarDONE[{}] arrive [done]", + iv, + barId, + predicate=consumerPrimaryThread, + ) + scf.yield_([]) + + p = arith.cmpi(arith.CmpIPredicate.eq, stage, c(num_stages - 1)) + phaseParity = arith.select( + p, + arith.xori(phaseParity, arith.constant(T.bool(), 1)), + phaseParity, + ) + + # Step 6.3.5. Yield + scf.yield_([new_acc, phaseParity]) + + # Step 6.3. Wait All WGMMA + nvvm.WgmmaWaitGroupSyncOp(0) + + with ir.InsertionPoint(scf.IfOp(consumerPrimaryThread).then_block): + barId = c((K // BLOCK_K) % num_stages) + nvgpu.mbarrier_arrive( + ir.Type.parse("!nvgpu.mbarrier.token"), mbarDONE, barId + ) + scf.yield_([]) + + # Step 6.4. Epilogue (registers --> shared memory) + acc_smem_ty = ir.MemRefType.get( + (BLOCK_M, BLOCK_N), c_elem_ty, memory_space=smem_space + ) + acc_smem = memref.view(acc_smem_ty, dynamic_smem, c(0), []) + debug_print("[cons] | Storing", predicate=consumerPrimaryThread) + nvgpu.WarpgroupMmaStoreOp(for_op.results[0], acc_smem) + scf.yield_([]) + gpu.barrier() + + # GPU Step 9. Epilogue (shared memory --> global memory) + fd = ir.MemRefType.get( + [BLOCK_M * BLOCK_N], c_elem_ty, memory_space=smem_space + ) + collapsed_smem = memref.view(fd, dynamic_smem, c(0), []) + rty = ir.MemRefType.get( + (BLOCK_M, BLOCK_N), + c_elem_ty, + ir.Attribute.parse("strided<[" + str(N) + ", 1], offset: ?>"), + ) + c_device_per_block = memref.SubViewOp( + rty, + c_device, + [dimX, dimY], + [], + [], + [MLIR_DYNAMIC, MLIR_DYNAMIC], + [BLOCK_M, BLOCK_N], + [1, 1], + ) + vlen = 1 + for_op = scf.ForOp( + tidx, c(BLOCK_M * BLOCK_N), c(vlen * WARP_GROUP_SIZE * 2) + ) + with ir.InsertionPoint(for_op.body): + x = arith.divui(for_op.induction_variable, c(BLOCK_M)) + y = arith.remui(for_op.induction_variable, c(BLOCK_N)) + vdata = vector.load( + ir.VectorType.get((vlen,), c_elem_ty), + collapsed_smem, + [for_op.induction_variable], + ) + vector.store(vdata, c_device_per_block, [x, y]) + scf.yield_([]) + + gpu.terminator() + + # Step 4. Copy back to host + t8 = gpu.wait(token_ty, [launch_op]) + t9 = gpu.memcpy(token_ty, [t8], c_host, c_device) + gpu.dealloc(token_ty, [t8], a_device) + gpu.dealloc(token_ty, [t8], b_device) + gpu.wait(token_ty, [t9]) + gpu.dealloc(token_ty, [t8], c_device) + func.ReturnOp([]) + + fop.attributes["llvm.emit_c_interface"] = ir.UnitAttr.get() + module.operation.verify() + return module + + +def generate_matmul_multistage( + input_type=np.float16, + output_type=np.float32, + M=4096, + N=4096, + K=4096, + BLOCK_M=128, + BLOCK_N=128, + BLOCK_K=64, + num_stages=3, +): + # Limitaitons for now + assert input_type == np.float16 + assert output_type == np.float32 + assert BLOCK_M == 128 + assert BLOCK_N == 128 + assert BLOCK_K == 64 + assert M % BLOCK_M == 0 + assert N % BLOCK_N == 0 + assert K % BLOCK_K == 0 + + module = ir.Module.create() + token_ty = ir.Type.parse("!gpu.async.token") + a_elem_ty = get_mlir_ty(input_type) + b_elem_ty = get_mlir_ty(input_type) + c_elem_ty = get_mlir_ty(output_type) + a_ty = ir.MemRefType.get([M, K], a_elem_ty) + b_ty = ir.MemRefType.get((K, N), b_elem_ty) + c_ty = ir.MemRefType.get((M, N), c_elem_ty) + a_tile_shape = a_tma_shape = (BLOCK_M, TMA_LAST_DIM_F16) + b_tma_shape = (BLOCK_K, TMA_LAST_DIM_F16) + b_tile_shape = (BLOCK_K, BLOCK_N) + txcount = (b_tile_shape[0] * b_tile_shape[1] * get_type_size(a_elem_ty)) + ( + a_tile_shape[0] * a_tile_shape[1] * get_type_size(b_elem_ty) + ) + smem_space_str = "#gpu.address_space" + smem_space = ir.Attribute.parse(smem_space_str) + mbar_ty = ir.Type.parse( + "!nvgpu.mbarrier.group" + ) + a_tma_desc_ty = ir.Type.parse( + "!nvgpu.tensormap.descriptor, swizzle = swizzle_128b, l2promo=none, oob=zero, interleave=none>" + ) + b_tma_desc_ty = ir.Type.parse( + "!nvgpu.tensormap.descriptor, swizzle = swizzle_128b, l2promo=none, oob=zero, interleave=none>" + ) + acc_ty = ir.Type.parse( + "!nvgpu.warpgroup.accumulator>" + ) + a_wgmma_ty = ir.Type.parse( + "!nvgpu.warpgroup.descriptor>" + ) + b_wgmma_ty = ir.Type.parse( + "!nvgpu.warpgroup.descriptor>" + ) + + with ir.InsertionPoint(module.body): + kernelName = make_kernel_name( + input_type, + output_type, + M, + N, + K, + BLOCK_M, + BLOCK_N, + BLOCK_K, + num_stages, + False, + ) + fop = func.FuncOp(kernelName, ([a_ty, b_ty, c_ty], [])) + with ir.InsertionPoint(fop.add_entry_block()): + a_host = fop.arguments[0] + b_host = fop.arguments[1] + c_host = fop.arguments[2] + lhs_tile_bytes = BLOCK_M * BLOCK_K * get_type_size(a_elem_ty) + rhs_tile_bytes = BLOCK_N * BLOCK_K * get_type_size(b_elem_ty) + smem_size_input = (lhs_tile_bytes + rhs_tile_bytes) * num_stages + smem_size_output = BLOCK_M * BLOCK_N * get_type_size(c_elem_ty) + smem_size = max(smem_size_input, smem_size_output) + + # Step 1. Allocate device memory and memcpy + t1 = gpu.wait(token_ty, []) + a_device, t2 = gpu.alloc(a_ty, token_ty, [t1], [], []) + b_device, t3 = gpu.alloc(b_ty, token_ty, [t2], [], []) + c_device, t4 = gpu.alloc(c_ty, token_ty, [t3], [], []) + t5 = gpu.memcpy(token_ty, [t4], a_device, a_host) + t6 = gpu.memcpy(token_ty, [t5], b_device, b_host) + t7 = gpu.wait(token_ty, [t6]) + + # Step 2. Create TMA Descriptors + tma_specs = [ + (a_device, a_tma_desc_ty, a_tma_shape), + (b_device, b_tma_desc_ty, b_tma_shape), + ] + tma_descs = [] + for x_device, tensor_map_ty, tile_shape in tma_specs: + x_unranked = memref.cast( + ir.UnrankedMemRefType.get(a_elem_ty, a_ty.memory_space), x_device + ) + tma_descs.append( + nvgpu.TmaCreateDescriptorOp( + tensor_map_ty, x_unranked, map(c, tile_shape) + ).result + ) + a_tma_desc, b_tma_desc = tma_descs + + # Step 3. Launch Kernel with 1 Warpgroup + cta_m = M // BLOCK_M + cta_n = N // BLOCK_N + assert M % BLOCK_M == 0 and N % BLOCK_N == 0 + grid = (cta_m, cta_n, 1) + block = (WARP_GROUP_SIZE, 1, 1) + launch_op = gpu.LaunchOp( + token_ty, + [t7], + *map(c, grid), + *map(c, block), + dynamicSharedMemorySize=c(smem_size, ty=T.i32()) + ) + launch_op.body.blocks.append(*([T.index()] * 12)) + with ir.InsertionPoint(launch_op.body.blocks[0]): + # GPU Step 0. Bootstrapping + memref.assume_alignment(c_device, 16) + dynamic_smem = gpu.dynamic_shared_memory( + ir.MemRefType.get((MLIR_DYNAMIC,), T.i8(), memory_space=smem_space) + ) + ticks = c(10000000) + tidx = gpu.thread_id(gpu.Dimension.x) + primaryThread = arith.cmpi(arith.CmpIPredicate.eq, tidx, c(0)) + warpId = arith.divui(tidx, c(32)) + bidx = gpu.block_id(gpu.Dimension.x) + bidy = gpu.block_id(gpu.Dimension.y) + dimX = arith.muli(bidx, c(BLOCK_M)) + dimY = arith.muli(bidy, c(BLOCK_N)) + + # GPU Step 1. Initialize mbarrier groups + mbarTMA = nvgpu.mbarrier_create(mbar_ty) + for i in range(num_stages): + nvgpu.mbarrier_init(mbarTMA, c(1), c(i), predicate=primaryThread) + gpu.barrier() + + # GPU Step 2. Prefetch TMA descriptors + nvgpu.tma_prefetch_descriptor(a_tma_desc, predicate=primaryThread) + nvgpu.tma_prefetch_descriptor(b_tma_desc, predicate=primaryThread) + + # GPU Step 3. Prologue (global memory --> shared memory) + ns = num_stages if num_stages == 1 else num_stages - 1 + for_op = scf.ForOp(c(0), c(ns), c(1)) + with ir.InsertionPoint(for_op.body): + iv = for_op.induction_variable + + # Step 3.1. Calculate offsets + a_offset = arith.muli(iv, c(lhs_tile_bytes)) + a_tma_slice = memref.view( + ir.MemRefType.get( + a_tma_shape, a_elem_ty, memory_space=smem_space + ), + dynamic_smem, + a_offset, + [], + ) + b_offset = arith.addi( + arith.muli(iv, c(rhs_tile_bytes)), + c(lhs_tile_bytes * num_stages), + ) + b_tma_slice_1 = memref.view( + ir.MemRefType.get( + b_tma_shape, b_elem_ty, memory_space=smem_space + ), + dynamic_smem, + b_offset, + [], + ) + b_offset2 = arith.addi( + b_offset, + c(BLOCK_K * TMA_LAST_DIM_F16 * get_type_size(b_elem_ty)), + ) + b_tma_slice_2 = memref.view( + ir.MemRefType.get( + b_tma_shape, b_elem_ty, memory_space=smem_space + ), + dynamic_smem, + b_offset2, + [], + ) + + # Step 3.2. TMA Load + coord = arith.muli(c(64), iv) + dimY2 = arith.addi(dimY, c(64)) + debug_print( + "[Prologue] TMA Load a_offset={} b_offset={} b_offset2={} @ a=({},{}) b=({},{})", + a_offset, + b_offset, + b_offset2, + coord, + dimX, + dimY, + coord, + predicate=primaryThread, + ) + nvgpu.TmaAsyncLoadOp( + a_tma_slice, + mbarTMA, + a_tma_desc, + coordinates=[coord, dimX], + mbarId=iv, + predicate=primaryThread, + ) + nvgpu.TmaAsyncLoadOp( + b_tma_slice_1, + mbarTMA, + b_tma_desc, + coordinates=[dimY, coord], + mbarId=iv, + predicate=primaryThread, + ) + nvgpu.TmaAsyncLoadOp( + b_tma_slice_2, + mbarTMA, + b_tma_desc, + coordinates=[dimY2, coord], + mbarId=iv, + predicate=primaryThread, + ) + + # Step 3.2. mbarTMA arrive + debug_print( + "[Prologue] mbarTMA[{}] arrive", iv, predicate=primaryThread + ) + nvgpu.mbarrier_arrive_expect_tx( + mbarTMA, c(txcount), iv, predicate=primaryThread + ) + debug_print( + "[Prologue] mbarTMA[{}] arrive [done]", + iv, + predicate=primaryThread, + ) + scf.yield_([]) + + # GPU Step 4. Main Loop + acc = nvgpu.warpgroup_mma_init_accumulator(acc_ty) + for_op = scf.ForOp( + c(0), c(K // BLOCK_K), c(1), [acc, arith.constant(T.bool(), 0)] + ) + with ir.InsertionPoint(for_op.body): + # Step 4.1. Wait mbarTMA + phaseParity = for_op.inner_iter_args[1] + iv = for_op.induction_variable + stage = arith.remui(iv, c(num_stages)) + debug_print( + "[MainLoop] mbarTMA[{}] try_wait phase={}", + stage, + phaseParity, + predicate=primaryThread, + ) + nvgpu.MBarrierTryWaitParityOp( + mbarTMA, phaseParity, ticks, mbarId=stage + ) + debug_print( + "[MainLoop] mbarTMA[{}] try_wait phase={} [done]", + stage, + phaseParity, + predicate=primaryThread, + ) + + # Step 4.2. Create WGMMA Descriptors + a_offset = arith.muli(stage, c(lhs_tile_bytes)) + a_tile_slice = memref.view( + ir.MemRefType.get( + a_tile_shape, a_elem_ty, memory_space=smem_space + ), + dynamic_smem, + a_offset, + [], + ) + b_offset = arith.addi( + arith.muli(stage, c(rhs_tile_bytes)), + c(lhs_tile_bytes * num_stages), + ) + b_tile_slice = memref.view( + ir.MemRefType.get( + b_tile_shape, b_elem_ty, memory_space=smem_space + ), + dynamic_smem, + b_offset, + [], + ) + debug_print( + "[MainLoop] iv={} MMA a_offset={} b_offset={}", + iv, + a_offset, + b_offset, + predicate=primaryThread, + ) + da = nvgpu.WarpgroupGenerateDescriptorOp( + a_wgmma_ty, a_tile_slice, a_tma_desc + ) + db = nvgpu.WarpgroupGenerateDescriptorOp( + b_wgmma_ty, b_tile_slice, b_tma_desc + ) + + # Step 4.3. MMA + carry_acc = for_op.inner_iter_args[0] + new_acc = nvgpu.WarpgroupMmaOp( + acc.type, da, db, carry_acc, transposeB=True + ) + if num_stages == 1: + nvvm.WgmmaWaitGroupSyncOp(0) + + # Step 4.4. Load TMA for next stage + p1 = arith.cmpi( + arith.CmpIPredicate.ult, + arith.addi(iv, c(ns)), + c(K // BLOCK_K), + ) + p = arith.andi(primaryThread, p1) + nextStage = arith.addi(iv, c(ns)) + nextSlot = arith.remui(nextStage, c(num_stages)) + a_offset = arith.muli(nextSlot, c(lhs_tile_bytes)) + + debug_print( + "[MainLoop] mbarTMA[{}] arrive", + nextSlot, + predicate=p, + ) + nvgpu.mbarrier_arrive_expect_tx( + mbarTMA, c(txcount), nextSlot, predicate=p + ) + debug_print( + "[MainLoop] mbarTMA[{}] arrive [done]", + nextSlot, + predicate=p, + ) + + a_tma_slice = memref.view( + ir.MemRefType.get( + a_tma_shape, a_elem_ty, memory_space=smem_space + ), + dynamic_smem, + a_offset, + [], + ) + b_offset = arith.addi( + arith.muli(nextSlot, c(rhs_tile_bytes)), + c(lhs_tile_bytes * num_stages), + ) + b_tma_slice_1 = memref.view( + ir.MemRefType.get( + b_tma_shape, b_elem_ty, memory_space=smem_space + ), + dynamic_smem, + b_offset, + [], + ) + b_offset2 = arith.addi( + b_offset, + c(BLOCK_K * TMA_LAST_DIM_F16 * get_type_size(b_elem_ty)), + ) + b_tma_slice_2 = memref.view( + ir.MemRefType.get( + b_tma_shape, b_elem_ty, memory_space=smem_space + ), + dynamic_smem, + b_offset2, + [], + ) + + coord = arith.muli(c(64), nextStage) + debug_print( + "[MainLoop] iv={} TMA Load a_offset={} b_offset={} b_offset2={} @ a=({},{}) b=({},{})", + iv, + a_offset, + b_offset, + b_offset2, + coord, + dimX, + dimY, + coord, + predicate=p, + ) + nvgpu.TmaAsyncLoadOp( + a_tma_slice, + mbarTMA, + a_tma_desc, + coordinates=[coord, dimX], + mbarId=nextSlot, + predicate=p, + ) + nvgpu.TmaAsyncLoadOp( + b_tma_slice_1, + mbarTMA, + b_tma_desc, + coordinates=[dimY, coord], + mbarId=nextSlot, + predicate=p, + ) + dimY2 = arith.addi(dimY, c(64)) + nvgpu.TmaAsyncLoadOp( + b_tma_slice_2, + mbarTMA, + b_tma_desc, + coordinates=[dimY2, coord], + mbarId=nextSlot, + predicate=p, + ) + # Step 4.5. Change the phaseParity + p = arith.cmpi(arith.CmpIPredicate.eq, stage, c(num_stages - 1)) + phaseParity = arith.select( + p, + arith.xori(phaseParity, arith.constant(T.bool(), 1)), + phaseParity, + ) + + # Step 4.5. Yield + scf.yield_([new_acc, phaseParity]) + + # Step 5. Wait All WGMMA groups + nvvm.WgmmaWaitGroupSyncOp(0) + + # Step 6. Epilogue (registers --> shared memory) + acc_smem_ty = ir.MemRefType.get( + (BLOCK_M, BLOCK_N), c_elem_ty, memory_space=smem_space + ) + acc_smem = memref.view(acc_smem_ty, dynamic_smem, c(0), []) + debug_print("Storing", predicate=primaryThread) + nvgpu.WarpgroupMmaStoreOp(for_op.results[0], acc_smem) + gpu.barrier() + + # GPU Step 7. Epilogue (shared memory --> global memory) + fd = ir.MemRefType.get( + [BLOCK_M * BLOCK_N], c_elem_ty, memory_space=smem_space + ) + collapsed_smem = memref.view(fd, dynamic_smem, c(0), []) + rty = ir.MemRefType.get( + (BLOCK_M, BLOCK_N), + c_elem_ty, + ir.Attribute.parse("strided<[" + str(N) + ", 1], offset: ?>"), + ) + c_device_per_block = memref.SubViewOp( + rty, + c_device, + [dimX, dimY], + [], + [], + [MLIR_DYNAMIC, MLIR_DYNAMIC], + [BLOCK_M, BLOCK_N], + [1, 1], + ) + vlen = 1 + for_op = scf.ForOp( + tidx, c(BLOCK_M * BLOCK_N), c(vlen * WARP_GROUP_SIZE) + ) + with ir.InsertionPoint(for_op.body): + x = arith.divui(for_op.induction_variable, c(BLOCK_M)) + y = arith.remui(for_op.induction_variable, c(BLOCK_N)) + vdata = vector.load( + ir.VectorType.get((vlen,), c_elem_ty), + collapsed_smem, + [for_op.induction_variable], + ) + vector.store(vdata, c_device_per_block, [x, y]) + scf.yield_([]) + + gpu.terminator() + + # Step 4. Copy back to host + t8 = gpu.wait(token_ty, [launch_op]) + t9 = gpu.memcpy(token_ty, [t8], c_host, c_device) + gpu.dealloc(token_ty, [t8], a_device) + gpu.dealloc(token_ty, [t8], b_device) + gpu.wait(token_ty, [t9]) + gpu.dealloc(token_ty, [t8], c_device) + func.ReturnOp([]) + + fop.attributes["llvm.emit_c_interface"] = ir.UnitAttr.get() + module.operation.verify() + return module diff --git a/mlir/test/Integration/GPU/CUDA/sm90/python/tools/nvgpucompiler.py b/mlir/test/Integration/GPU/CUDA/sm90/python/tools/nvgpucompiler.py new file mode 100644 index 000000000000..1c9cc74fcd16 --- /dev/null +++ b/mlir/test/Integration/GPU/CUDA/sm90/python/tools/nvgpucompiler.py @@ -0,0 +1,45 @@ +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# This file contains the Nvgpu class. + +from mlir import execution_engine +from mlir import ir +from mlir import passmanager +from typing import Sequence +import errno +import os +import sys + +_SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(_SCRIPT_PATH) + + +class NvgpuCompiler: + """Nvgpu class for compiling and building MLIR modules.""" + + def __init__(self, options: str, opt_level: int, shared_libs: Sequence[str]): + pipeline = f"builtin.module(gpu-lower-to-nvvm-pipeline{{{options}}})" + self.pipeline = pipeline + self.shared_libs = shared_libs + self.opt_level = opt_level + + def __call__(self, module: ir.Module): + """Convenience application method.""" + self.compile(module) + + def compile(self, module: ir.Module): + """Compiles the module by invoking the nvgpu pipeline.""" + passmanager.PassManager.parse(self.pipeline).run(module.operation) + + def jit(self, module: ir.Module) -> execution_engine.ExecutionEngine: + """Wraps the module in a JIT execution engine.""" + return execution_engine.ExecutionEngine( + module, opt_level=self.opt_level, shared_libs=self.shared_libs + ) + + def compile_and_jit(self, module: ir.Module) -> execution_engine.ExecutionEngine: + """Compiles and jits the module.""" + self.compile(module) + return self.jit(module) -- GitLab From 922a431e10fa28519eb5d62e094f19b1008612a1 Mon Sep 17 00:00:00 2001 From: Alan Zhao Date: Mon, 4 Mar 2024 13:15:50 -0800 Subject: [PATCH 048/929] [profdata][nfc] Disable several tests on Windows (#83907) Several profdata tests pass the byte 012 to printf. This causes these tests to fail when using GnuWin32's version of printf because printf will detect that 012 is the LF character and will prepend the byte 015 (CR) in front of LF. This change is required after https://github.com/llvm/llvm-project/pull/82711 which bumped the version number. --- llvm/test/tools/llvm-profdata/binary-ids-padding.test | 3 +++ llvm/test/tools/llvm-profdata/large-binary-id-size.test | 3 +++ .../llvm-profdata/malformed-not-space-for-another-header.test | 3 +++ llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test | 3 +++ .../tools/llvm-profdata/malformed-ptr-to-counter-array.test | 3 +++ llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test | 3 +++ llvm/test/tools/llvm-profdata/raw-32-bits-be.test | 3 +++ llvm/test/tools/llvm-profdata/raw-32-bits-le.test | 3 +++ llvm/test/tools/llvm-profdata/raw-64-bits-be.test | 3 +++ llvm/test/tools/llvm-profdata/raw-64-bits-le.test | 3 +++ llvm/test/tools/llvm-profdata/raw-two-profiles.test | 3 +++ 11 files changed, 33 insertions(+) diff --git a/llvm/test/tools/llvm-profdata/binary-ids-padding.test b/llvm/test/tools/llvm-profdata/binary-ids-padding.test index 61881b69cfd5..292c582b45c5 100644 --- a/llvm/test/tools/llvm-profdata/binary-ids-padding.test +++ b/llvm/test/tools/llvm-profdata/binary-ids-padding.test @@ -14,6 +14,9 @@ // INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) +// gnuwin32 printf does not work for this test because it will print \15 (CR) +// whenever \12 (LF) is in the input string. +UNSUPPORTED: system-windows RUN: printf '\201rforpl\377' > %t.profraw RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw // There will be 2 20-byte binary IDs, so the total Binary IDs size will be 64 bytes. diff --git a/llvm/test/tools/llvm-profdata/large-binary-id-size.test b/llvm/test/tools/llvm-profdata/large-binary-id-size.test index 316a9a4c9df4..b62bdad4ddb2 100644 --- a/llvm/test/tools/llvm-profdata/large-binary-id-size.test +++ b/llvm/test/tools/llvm-profdata/large-binary-id-size.test @@ -1,3 +1,6 @@ +// gnuwin32 printf does not work for this test because it will print \15 (CR) +// whenever \12 (LF) is in the input string. +UNSUPPORTED: system-windows RUN: printf '\201rforpl\377' > %t.profraw RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\40\0\0\0\0\0\0\0' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test b/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test index 8b686d5c50cb..705e5efaf587 100644 --- a/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test +++ b/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test @@ -14,6 +14,9 @@ // INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) +// gnuwin32 printf does not work for this test because it will print \15 (CR) +// whenever \12 (LF) is in the input string. +UNSUPPORTED: system-windows RUN: printf '\201rforpl\377' > %t.profraw RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test b/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test index 089afad42062..157c13b926a7 100644 --- a/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test +++ b/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test @@ -14,6 +14,9 @@ // INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) +// gnuwin32 printf does not work for this test because it will print \15 (CR) +// whenever \12 (LF) is in the input string. +UNSUPPORTED: system-windows RUN: printf '\201rforpl\377' > %t.profraw RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test b/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test index e404ba4210cc..83cf76f68fb6 100644 --- a/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test +++ b/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test @@ -14,6 +14,9 @@ // INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) +// gnuwin32 printf does not work for this test because it will print \15 (CR) +// whenever \12 (LF) is in the input string. +UNSUPPORTED: system-windows RUN: printf '\201rforpl\377' > %t.profraw RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test b/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test index ee54bfb97856..0f20a1b0b369 100644 --- a/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test +++ b/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test @@ -1,3 +1,6 @@ +// gnuwin32 printf does not work for this test because it will print \15 (CR) +// whenever \12 (LF) is in the input string. +UNSUPPORTED: system-windows RUN: printf '\201rforpl\377' > %t.profraw RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw // We should fail on this because the binary IDs is not a multiple of 8 bytes. diff --git a/llvm/test/tools/llvm-profdata/raw-32-bits-be.test b/llvm/test/tools/llvm-profdata/raw-32-bits-be.test index 63782c8b94d4..fbd73ae30a5b 100644 --- a/llvm/test/tools/llvm-profdata/raw-32-bits-be.test +++ b/llvm/test/tools/llvm-profdata/raw-32-bits-be.test @@ -1,3 +1,6 @@ +// gnuwin32 printf does not work for this test because it will print \15 (CR) +// whenever \12 (LF) is in the input string. +UNSUPPORTED: system-windows // Header RUN: printf '\377lprofR\201' > %t RUN: printf '\0\0\0\0\0\0\0\12' >> %t diff --git a/llvm/test/tools/llvm-profdata/raw-32-bits-le.test b/llvm/test/tools/llvm-profdata/raw-32-bits-le.test index e9569bec1178..91f10fc87458 100644 --- a/llvm/test/tools/llvm-profdata/raw-32-bits-le.test +++ b/llvm/test/tools/llvm-profdata/raw-32-bits-le.test @@ -1,3 +1,6 @@ +// gnuwin32 printf does not work for this test because it will print \15 (CR) +// whenever \12 (LF) is in the input string. +UNSUPPORTED: system-windows RUN: printf '\201Rforpl\377' > %t RUN: printf '\12\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t diff --git a/llvm/test/tools/llvm-profdata/raw-64-bits-be.test b/llvm/test/tools/llvm-profdata/raw-64-bits-be.test index 0bc579eec58a..e7694a1b5a73 100644 --- a/llvm/test/tools/llvm-profdata/raw-64-bits-be.test +++ b/llvm/test/tools/llvm-profdata/raw-64-bits-be.test @@ -1,3 +1,6 @@ +// gnuwin32 printf does not work for this test because it will print \15 (CR) +// whenever \12 (LF) is in the input string. +UNSUPPORTED: system-windows RUN: printf '\377lprofr\201' > %t RUN: printf '\0\0\0\0\0\0\0\12' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t diff --git a/llvm/test/tools/llvm-profdata/raw-64-bits-le.test b/llvm/test/tools/llvm-profdata/raw-64-bits-le.test index ca9ea54c3f01..99d486b8a517 100644 --- a/llvm/test/tools/llvm-profdata/raw-64-bits-le.test +++ b/llvm/test/tools/llvm-profdata/raw-64-bits-le.test @@ -1,3 +1,6 @@ +// gnuwin32 printf does not work for this test because it will print \15 (CR) +// whenever \12 (LF) is in the input string. +UNSUPPORTED: system-windows RUN: printf '\201rforpl\377' > %t RUN: printf '\12\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t diff --git a/llvm/test/tools/llvm-profdata/raw-two-profiles.test b/llvm/test/tools/llvm-profdata/raw-two-profiles.test index 70a4210dea9f..47cc6fa4fd7f 100644 --- a/llvm/test/tools/llvm-profdata/raw-two-profiles.test +++ b/llvm/test/tools/llvm-profdata/raw-two-profiles.test @@ -1,3 +1,6 @@ +// gnuwin32 printf does not work for this test because it will print \15 (CR) +// whenever \12 (LF) is in the input string. +UNSUPPORTED: system-windows RUN: printf '\201rforpl\377' > %t-foo.profraw RUN: printf '\12\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw -- GitLab From aec6a04b8e99b42eca431fc0b56947937d3a14c2 Mon Sep 17 00:00:00 2001 From: Mircea Trofin Date: Mon, 4 Mar 2024 13:33:58 -0800 Subject: [PATCH 049/929] Revert "Update Benchmark (#83488)" This reverts commit 2e93ee6a2389cc65faf9739cd7a6c438d5e05a9e. buildbot failures, e.g. `/third-party/benchmark/cmake/pthread_affinity.cpp` --- third-party/benchmark/.ycm_extra_conf.py | 8 +- third-party/benchmark/AUTHORS | 12 +- third-party/benchmark/CMakeLists.txt | 100 ++-- third-party/benchmark/CONTRIBUTORS | 14 +- third-party/benchmark/README.md | 13 +- third-party/benchmark/WORKSPACE | 54 +- .../benchmark/bindings/python/build_defs.bzl | 8 +- .../python/google_benchmark/__init__.py | 58 +- .../python/google_benchmark/benchmark.cc | 149 +++-- .../python/google_benchmark/example.py | 6 +- .../benchmark/bindings/python/pybind11.BUILD | 20 + .../bindings/python/python_headers.BUILD | 4 - .../bindings/python/requirements.txt | 2 + .../benchmark/cmake/CXXFeatureCheck.cmake | 29 +- .../benchmark/cmake/GetGitVersion.cmake | 30 +- third-party/benchmark/cmake/GoogleTest.cmake | 16 +- .../benchmark/cmake/Modules/FindPFM.cmake | 38 +- third-party/benchmark/cmake/benchmark.pc.in | 4 +- third-party/benchmark/docs/AssemblyTests.md | 2 - third-party/benchmark/docs/_config.yml | 4 +- third-party/benchmark/docs/dependencies.md | 22 +- third-party/benchmark/docs/index.md | 4 +- third-party/benchmark/docs/perf_counters.md | 13 +- third-party/benchmark/docs/releasing.md | 24 +- third-party/benchmark/docs/tools.md | 140 ----- third-party/benchmark/docs/user_guide.md | 158 +---- .../benchmark/include/benchmark/benchmark.h | 563 +++++------------- third-party/benchmark/requirements.txt | 3 + third-party/benchmark/setup.py | 193 +++--- third-party/benchmark/src/CMakeLists.txt | 28 +- third-party/benchmark/src/benchmark.cc | 289 ++------- .../benchmark/src/benchmark_api_internal.cc | 20 +- .../benchmark/src/benchmark_api_internal.h | 3 - third-party/benchmark/src/benchmark_main.cc | 1 - third-party/benchmark/src/benchmark_name.cc | 5 +- .../benchmark/src/benchmark_register.cc | 52 +- .../benchmark/src/benchmark_register.h | 5 +- third-party/benchmark/src/benchmark_runner.cc | 201 +------ third-party/benchmark/src/benchmark_runner.h | 33 +- third-party/benchmark/src/check.h | 31 +- third-party/benchmark/src/colorprint.cc | 46 +- third-party/benchmark/src/commandlineflags.cc | 19 +- third-party/benchmark/src/commandlineflags.h | 39 +- third-party/benchmark/src/complexity.cc | 43 +- third-party/benchmark/src/complexity.h | 2 +- third-party/benchmark/src/console_reporter.cc | 34 +- third-party/benchmark/src/counter.cc | 4 +- third-party/benchmark/src/csv_reporter.cc | 23 +- third-party/benchmark/src/cycleclock.h | 40 +- third-party/benchmark/src/internal_macros.h | 17 +- third-party/benchmark/src/json_reporter.cc | 36 +- third-party/benchmark/src/log.h | 26 +- third-party/benchmark/src/perf_counters.cc | 242 ++------ third-party/benchmark/src/perf_counters.h | 108 ++-- third-party/benchmark/src/re.h | 2 +- third-party/benchmark/src/reporter.cc | 14 +- third-party/benchmark/src/sleep.cc | 66 ++ third-party/benchmark/src/sleep.h | 15 + third-party/benchmark/src/statistics.cc | 38 +- third-party/benchmark/src/statistics.h | 12 +- third-party/benchmark/src/string_util.cc | 65 +- third-party/benchmark/src/string_util.h | 11 +- third-party/benchmark/src/sysinfo.cc | 443 +++++--------- third-party/benchmark/src/thread_manager.h | 4 +- third-party/benchmark/src/timers.cc | 24 +- .../benchmark/test/AssemblyTests.cmake | 21 - third-party/benchmark/test/CMakeLists.txt | 127 ++-- .../benchmark/test/args_product_test.cc | 4 +- third-party/benchmark/test/basic_test.cc | 9 +- third-party/benchmark/test/benchmark_gtest.cc | 14 +- .../benchmark/test/benchmark_name_gtest.cc | 8 - .../benchmark_random_interleaving_gtest.cc | 5 +- .../test/benchmark_setup_teardown_test.cc | 18 +- third-party/benchmark/test/benchmark_test.cc | 65 +- .../test/clobber_memory_assembly_test.cc | 1 - third-party/benchmark/test/complexity_test.cc | 158 ++--- .../benchmark/test/diagnostics_test.cc | 15 +- .../test/donotoptimize_assembly_test.cc | 40 -- .../benchmark/test/donotoptimize_test.cc | 28 +- third-party/benchmark/test/filter_test.cc | 31 +- third-party/benchmark/test/fixture_test.cc | 6 +- third-party/benchmark/test/link_main_test.cc | 3 +- third-party/benchmark/test/map_test.cc | 10 +- .../benchmark/test/memory_manager_test.cc | 11 +- .../benchmark/test/multiple_ranges_test.cc | 4 +- third-party/benchmark/test/options_test.cc | 6 +- third-party/benchmark/test/output_test.h | 10 +- .../benchmark/test/output_test_helper.cc | 47 +- .../benchmark/test/perf_counters_gtest.cc | 266 ++------- .../benchmark/test/perf_counters_test.cc | 71 +-- .../benchmark/test/register_benchmark_test.cc | 26 +- .../benchmark/test/reporter_output_test.cc | 18 +- .../benchmark/test/skip_with_error_test.cc | 17 +- third-party/benchmark/test/spec_arg_test.cc | 16 +- .../benchmark/test/statistics_gtest.cc | 4 +- .../benchmark/test/string_util_gtest.cc | 55 +- .../test/user_counters_tabular_test.cc | 10 +- .../benchmark/test/user_counters_test.cc | 28 +- .../test/user_counters_thousands_test.cc | 32 +- third-party/benchmark/tools/compare.py | 62 +- .../tools/gbench/Inputs/test1_run1.json | 8 - .../tools/gbench/Inputs/test1_run2.json | 8 - .../benchmark/tools/gbench/__init__.py | 2 +- third-party/benchmark/tools/gbench/report.py | 271 +-------- third-party/benchmark/tools/gbench/util.py | 52 +- third-party/benchmark/tools/requirements.txt | 3 +- third-party/benchmark/tools/strip_asm.py | 73 +-- 107 files changed, 1622 insertions(+), 3773 deletions(-) create mode 100644 third-party/benchmark/bindings/python/pybind11.BUILD create mode 100644 third-party/benchmark/bindings/python/requirements.txt create mode 100644 third-party/benchmark/requirements.txt create mode 100644 third-party/benchmark/src/sleep.cc create mode 100644 third-party/benchmark/src/sleep.h diff --git a/third-party/benchmark/.ycm_extra_conf.py b/third-party/benchmark/.ycm_extra_conf.py index caf257f0540e..1482c7b00202 100644 --- a/third-party/benchmark/.ycm_extra_conf.py +++ b/third-party/benchmark/.ycm_extra_conf.py @@ -1,5 +1,4 @@ import os - import ycm_core # These are the compilation flags that will be used in case there's no @@ -92,9 +91,7 @@ def GetCompilationInfoForFile(filename): for extension in SOURCE_EXTENSIONS: replacement_file = basename + extension if os.path.exists(replacement_file): - compilation_info = database.GetCompilationInfoForFile( - replacement_file - ) + compilation_info = database.GetCompilationInfoForFile(replacement_file) if compilation_info.compiler_flags_: return compilation_info return None @@ -110,8 +107,7 @@ def FlagsForFile(filename, **kwargs): return None final_flags = MakeRelativePathsInFlagsAbsolute( - compilation_info.compiler_flags_, - compilation_info.compiler_working_dir_, + compilation_info.compiler_flags_, compilation_info.compiler_working_dir_ ) else: relative_to = DirectoryOfThisScript() diff --git a/third-party/benchmark/AUTHORS b/third-party/benchmark/AUTHORS index 2170e46fd4a0..54770f35499c 100644 --- a/third-party/benchmark/AUTHORS +++ b/third-party/benchmark/AUTHORS @@ -13,7 +13,6 @@ Alex Steele Andriy Berestovskyy Arne Beer Carto -Cezary Skrzyński Christian Wassermann Christopher Seymour Colin Braley @@ -28,13 +27,10 @@ Eric Backus Eric Fiselier Eugene Zhuk Evgeny Safronov -Fabien Pichot Federico Ficarelli Felix Homann -Gergely Meszaros Gergő Szitár Google Inc. -Henrique Bucher International Business Machines Corporation Ismael Jimenez Martinez Jern-Kuan Leong @@ -45,11 +41,8 @@ Jussi Knuuttila Kaito Udagawa Kishan Kumar Lei Xu -Marcel Jacobse Matt Clarkson Maxim Vafin -Mike Apodaca -Min-Yih Hsu MongoDB Inc. Nick Hutchinson Norman Heino @@ -57,16 +50,13 @@ Oleksandr Sochka Ori Livneh Paul Redmond Radoslav Yovchev -Raghu Raja -Rainer Orth Roman Lebedev Sayan Bhattacharjee -Shapr3D Shuo Chen -Staffan Tjernstrom Steinar H. Gunderson Stripe, Inc. Tobias Schmidt Yixuan Qiu Yusuke Suzuki Zbigniew Skowron +Min-Yih Hsu diff --git a/third-party/benchmark/CMakeLists.txt b/third-party/benchmark/CMakeLists.txt index d9bcc6a4939b..8af49406d052 100644 --- a/third-party/benchmark/CMakeLists.txt +++ b/third-party/benchmark/CMakeLists.txt @@ -1,7 +1,19 @@ -# Require CMake 3.10. If available, use the policies up to CMake 3.22. -cmake_minimum_required (VERSION 3.10...3.22) +cmake_minimum_required (VERSION 3.5.1) + +foreach(p + CMP0048 # OK to clear PROJECT_VERSION on project() + CMP0054 # CMake 3.1 + CMP0056 # export EXE_LINKER_FLAGS to try_run + CMP0057 # Support no if() IN_LIST operator + CMP0063 # Honor visibility properties for all targets + CMP0077 # Allow option() overrides in importing projects + ) + if(POLICY ${p}) + cmake_policy(SET ${p} NEW) + endif() +endforeach() -project (benchmark VERSION 1.8.3 LANGUAGES CXX) +project (benchmark VERSION 1.6.0 LANGUAGES CXX) option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." ON) option(BENCHMARK_ENABLE_EXCEPTIONS "Enable the use of exceptions in the benchmark library." ON) @@ -14,14 +26,11 @@ if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "PGI") # PGC++ maybe reporting false positives. set(BENCHMARK_ENABLE_WERROR OFF) endif() -if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "NVHPC") - set(BENCHMARK_ENABLE_WERROR OFF) -endif() if(BENCHMARK_FORCE_WERROR) set(BENCHMARK_ENABLE_WERROR ON) endif(BENCHMARK_FORCE_WERROR) -if(NOT (MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC")) +if(NOT MSVC) option(BENCHMARK_BUILD_32_BITS "Build a 32 bit version of the library." OFF) else() set(BENCHMARK_BUILD_32_BITS OFF CACHE BOOL "Build a 32 bit version of the library - unsupported when using MSVC)" FORCE) @@ -41,11 +50,8 @@ option(BENCHMARK_USE_BUNDLED_GTEST "Use bundled GoogleTest. If disabled, the fin option(BENCHMARK_ENABLE_LIBPFM "Enable performance counters provided by libpfm" OFF) -# Export only public symbols -set(CMAKE_CXX_VISIBILITY_PRESET hidden) -set(CMAKE_VISIBILITY_INLINES_HIDDEN ON) - -if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") +set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) +if(MSVC) # As of CMake 3.18, CMAKE_SYSTEM_PROCESSOR is not set properly for MSVC and # cross-compilation (e.g. Host=x86_64, target=aarch64) requires using the # undocumented, but working variable. @@ -66,7 +72,7 @@ function(should_enable_assembly_tests) return() endif() endif() - if (MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC") + if (MSVC) return() elseif(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64") return() @@ -105,32 +111,22 @@ get_git_version(GIT_VERSION) # If no git version can be determined, use the version # from the project() command if ("${GIT_VERSION}" STREQUAL "0.0.0") - set(VERSION "v${benchmark_VERSION}") + set(VERSION "${benchmark_VERSION}") else() set(VERSION "${GIT_VERSION}") endif() - -# Normalize version: drop "v" prefix, replace first "-" with ".", -# drop everything after second "-" (including said "-"). -string(STRIP ${VERSION} VERSION) -if(VERSION MATCHES v[^-]*-) - string(REGEX REPLACE "v([^-]*)-([0-9]+)-.*" "\\1.\\2" NORMALIZED_VERSION ${VERSION}) -else() - string(REGEX REPLACE "v(.*)" "\\1" NORMALIZED_VERSION ${VERSION}) -endif() - # Tell the user what versions we are using -message(STATUS "Google Benchmark version: ${VERSION}, normalized to ${NORMALIZED_VERSION}") +message(STATUS "Version: ${VERSION}") # The version of the libraries -set(GENERIC_LIB_VERSION ${NORMALIZED_VERSION}) -string(SUBSTRING ${NORMALIZED_VERSION} 0 1 GENERIC_LIB_SOVERSION) +set(GENERIC_LIB_VERSION ${VERSION}) +string(SUBSTRING ${VERSION} 0 1 GENERIC_LIB_SOVERSION) # Import our CMake modules -include(AddCXXCompilerFlag) include(CheckCXXCompilerFlag) -include(CheckLibraryExists) +include(AddCXXCompilerFlag) include(CXXFeatureCheck) +include(CheckLibraryExists) check_library_exists(rt shm_open "" HAVE_LIB_RT) @@ -138,16 +134,6 @@ if (BENCHMARK_BUILD_32_BITS) add_required_cxx_compiler_flag(-m32) endif() -if (MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC") - set(BENCHMARK_CXX_STANDARD 14) -else() - set(BENCHMARK_CXX_STANDARD 11) -endif() - -set(CMAKE_CXX_STANDARD ${BENCHMARK_CXX_STANDARD}) -set(CMAKE_CXX_STANDARD_REQUIRED YES) -set(CMAKE_CXX_EXTENSIONS OFF) - if (MSVC) # Turn compiler warnings up to 11 string(REGEX REPLACE "[-/]W[1-4]" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") @@ -180,18 +166,21 @@ if (MSVC) set(CMAKE_EXE_LINKER_FLAGS_MINSIZEREL "${CMAKE_EXE_LINKER_FLAGS_MINSIZEREL} /LTCG") endif() else() - # Turn on Large-file Support - add_definitions(-D_FILE_OFFSET_BITS=64) - add_definitions(-D_LARGEFILE64_SOURCE) - add_definitions(-D_LARGEFILE_SOURCE) + # Try and enable C++11. Don't use C++14 because it doesn't work in some + # configurations. + add_cxx_compiler_flag(-std=c++11) + if (NOT HAVE_CXX_FLAG_STD_CXX11) + add_cxx_compiler_flag(-std=c++0x) + endif() + # Turn compiler warnings up to 11 add_cxx_compiler_flag(-Wall) add_cxx_compiler_flag(-Wextra) add_cxx_compiler_flag(-Wshadow) - add_cxx_compiler_flag(-Wfloat-equal) - add_cxx_compiler_flag(-Wold-style-cast) if(BENCHMARK_ENABLE_WERROR) - add_cxx_compiler_flag(-Werror) + add_cxx_compiler_flag(-Werror RELEASE) + add_cxx_compiler_flag(-Werror RELWITHDEBINFO) + add_cxx_compiler_flag(-Werror MINSIZEREL) endif() if (NOT BENCHMARK_ENABLE_TESTING) # Disable warning when compiling tests as gtest does not use 'override'. @@ -204,23 +193,24 @@ else() # Disable warnings regarding deprecated parts of the library while building # and testing those parts of the library. add_cxx_compiler_flag(-Wno-deprecated-declarations) - if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel" OR CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") + if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") # Intel silently ignores '-Wno-deprecated-declarations', # warning no. 1786 must be explicitly disabled. # See #631 for rationale. add_cxx_compiler_flag(-wd1786) - add_cxx_compiler_flag(-fno-finite-math-only) endif() # Disable deprecation warnings for release builds (when -Werror is enabled). if(BENCHMARK_ENABLE_WERROR) - add_cxx_compiler_flag(-Wno-deprecated) + add_cxx_compiler_flag(-Wno-deprecated RELEASE) + add_cxx_compiler_flag(-Wno-deprecated RELWITHDEBINFO) + add_cxx_compiler_flag(-Wno-deprecated MINSIZEREL) endif() if (NOT BENCHMARK_ENABLE_EXCEPTIONS) add_cxx_compiler_flag(-fno-exceptions) endif() if (HAVE_CXX_FLAG_FSTRICT_ALIASING) - if (NOT CMAKE_CXX_COMPILER_ID STREQUAL "Intel" AND NOT CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") #ICC17u2: Many false positives for Wstrict-aliasing + if (NOT CMAKE_CXX_COMPILER_ID STREQUAL "Intel") #ICC17u2: Many false positives for Wstrict-aliasing add_cxx_compiler_flag(-Wstrict-aliasing) endif() endif() @@ -229,12 +219,12 @@ else() add_cxx_compiler_flag(-wd654) add_cxx_compiler_flag(-Wthread-safety) if (HAVE_CXX_FLAG_WTHREAD_SAFETY) - cxx_feature_check(THREAD_SAFETY_ATTRIBUTES "-DINCLUDE_DIRECTORIES=${PROJECT_SOURCE_DIR}/include") + cxx_feature_check(THREAD_SAFETY_ATTRIBUTES) endif() # On most UNIX like platforms g++ and clang++ define _GNU_SOURCE as a # predefined macro, which turns on all of the wonderful libc extensions. - # However g++ doesn't do this in Cygwin so we have to define it ourselves + # However g++ doesn't do this in Cygwin so we have to define it ourselfs # since we depend on GNU/POSIX/BSD extensions. if (CYGWIN) add_definitions(-D_GNU_SOURCE=1) @@ -285,8 +275,7 @@ if (BENCHMARK_USE_LIBCXX) if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") add_cxx_compiler_flag(-stdlib=libc++) elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" OR - "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel" OR - "${CMAKE_CXX_COMPILER_ID}" STREQUAL "IntelLLVM") + "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") add_cxx_compiler_flag(-nostdinc++) message(WARNING "libc++ header path must be manually specified using CMAKE_CXX_FLAGS") # Adding -nodefaultlibs directly to CMAKE__LINKER_FLAGS will break @@ -323,10 +312,9 @@ cxx_feature_check(STEADY_CLOCK) # Ensure we have pthreads set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -cxx_feature_check(PTHREAD_AFFINITY) if (BENCHMARK_ENABLE_LIBPFM) - find_package(PFM REQUIRED) + find_package(PFM) endif() # Set up directories diff --git a/third-party/benchmark/CONTRIBUTORS b/third-party/benchmark/CONTRIBUTORS index 9ca2caa3ee78..651fbeafe66a 100644 --- a/third-party/benchmark/CONTRIBUTORS +++ b/third-party/benchmark/CONTRIBUTORS @@ -27,9 +27,7 @@ Albert Pretorius Alex Steele Andriy Berestovskyy Arne Beer -Bátor Tallér Billy Robert O'Neal III -Cezary Skrzyński Chris Kennelly Christian Wassermann Christopher Seymour @@ -46,32 +44,25 @@ Eric Backus Eric Fiselier Eugene Zhuk Evgeny Safronov -Fabien Pichot Fanbo Meng Federico Ficarelli Felix Homann Geoffrey Martin-Noble -Gergely Meszaros Gergő Szitár Hannes Hauswedell -Henrique Bucher Ismael Jimenez Martinez -Iakov Sergeev Jern-Kuan Leong JianXiong Zhou Joao Paulo Magalhaes John Millikin Jordan Williams Jussi Knuuttila -Kaito Udagawa Kai Wolf +Kaito Udagawa Kishan Kumar Lei Xu -Marcel Jacobse Matt Clarkson Maxim Vafin -Mike Apodaca -Min-Yih Hsu Nick Hutchinson Norman Heino Oleksandr Sochka @@ -80,8 +71,6 @@ Pascal Leroy Paul Redmond Pierre Phaneuf Radoslav Yovchev -Raghu Raja -Rainer Orth Raul Marin Ray Glover Robert Guo @@ -95,3 +84,4 @@ Tom Madams Yixuan Qiu Yusuke Suzuki Zbigniew Skowron +Min-Yih Hsu diff --git a/third-party/benchmark/README.md b/third-party/benchmark/README.md index a5e5d392d826..7b81d960fc1d 100644 --- a/third-party/benchmark/README.md +++ b/third-party/benchmark/README.md @@ -4,9 +4,10 @@ [![bazel](https://github.com/google/benchmark/actions/workflows/bazel.yml/badge.svg)](https://github.com/google/benchmark/actions/workflows/bazel.yml) [![pylint](https://github.com/google/benchmark/workflows/pylint/badge.svg)](https://github.com/google/benchmark/actions?query=workflow%3Apylint) [![test-bindings](https://github.com/google/benchmark/workflows/test-bindings/badge.svg)](https://github.com/google/benchmark/actions?query=workflow%3Atest-bindings) + +[![Build Status](https://travis-ci.org/google/benchmark.svg?branch=master)](https://travis-ci.org/google/benchmark) [![Coverage Status](https://coveralls.io/repos/google/benchmark/badge.svg)](https://coveralls.io/r/google/benchmark) -[![Discord](https://discordapp.com/api/guilds/1125694995928719494/widget.png?style=shield)](https://discord.gg/cz7UX7wKC2) A library to benchmark code snippets, similar to unit tests. Example: @@ -32,7 +33,7 @@ To get started, see [Requirements](#requirements) and [Installation](#installation). See [Usage](#usage) for a full example and the [User Guide](docs/user_guide.md) for a more comprehensive feature overview. -It may also help to read the [Google Test documentation](https://github.com/google/googletest/blob/main/docs/primer.md) +It may also help to read the [Google Test documentation](https://github.com/google/googletest/blob/master/docs/primer.md) as some of the structural aspects of the APIs are similar. ## Resources @@ -46,8 +47,6 @@ IRC channels: [Assembly Testing Documentation](docs/AssemblyTests.md) -[Building and installing Python bindings](docs/python_bindings.md) - ## Requirements The library can be used with C++03. However, it requires C++11 to build, @@ -138,12 +137,6 @@ cache variables, if autodetection fails. If you are using clang, you may need to set `LLVMAR_EXECUTABLE`, `LLVMNM_EXECUTABLE` and `LLVMRANLIB_EXECUTABLE` cmake cache variables. -To enable sanitizer checks (eg., `asan` and `tsan`), add: -``` - -DCMAKE_C_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=address -fsanitize=thread -fno-sanitize-recover=all" - -DCMAKE_CXX_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=address -fsanitize=thread -fno-sanitize-recover=all " -``` - ### Stable and Experimental Library Versions The main branch contains the latest stable version of the benchmarking library; diff --git a/third-party/benchmark/WORKSPACE b/third-party/benchmark/WORKSPACE index 256207022597..949eb98bc5d9 100644 --- a/third-party/benchmark/WORKSPACE +++ b/third-party/benchmark/WORKSPACE @@ -1,30 +1,44 @@ workspace(name = "com_github_google_benchmark") -load("//:bazel/benchmark_deps.bzl", "benchmark_deps") - -benchmark_deps() - -load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies") - -rules_foreign_cc_dependencies() - -load("@rules_python//python:repositories.bzl", "py_repositories") - -py_repositories() - -load("@rules_python//python:pip.bzl", "pip_parse") - -pip_parse( - name = "tools_pip_deps", - requirements_lock = "//tools:requirements.txt", +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +http_archive( + name = "com_google_absl", + sha256 = "f41868f7a938605c92936230081175d1eae87f6ea2c248f41077c8f88316f111", + strip_prefix = "abseil-cpp-20200225.2", + urls = ["https://github.com/abseil/abseil-cpp/archive/20200225.2.tar.gz"], ) -load("@tools_pip_deps//:requirements.bzl", "install_deps") +git_repository( + name = "com_google_googletest", + remote = "https://github.com/google/googletest.git", + tag = "release-1.11.0", +) -install_deps() +http_archive( + name = "pybind11", + build_file = "@//bindings/python:pybind11.BUILD", + sha256 = "1eed57bc6863190e35637290f97a20c81cfe4d9090ac0a24f3bbf08f265eb71d", + strip_prefix = "pybind11-2.4.3", + urls = ["https://github.com/pybind/pybind11/archive/v2.4.3.tar.gz"], +) new_local_repository( name = "python_headers", build_file = "@//bindings/python:python_headers.BUILD", - path = "", # May be overwritten by setup.py. + path = "/usr/include/python3.6", # May be overwritten by setup.py. +) + +http_archive( + name = "rules_python", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.1.0/rules_python-0.1.0.tar.gz", + sha256 = "b6d46438523a3ec0f3cead544190ee13223a52f6a6765a29eae7b7cc24cc83a0", +) + +load("@rules_python//python:pip.bzl", pip3_install="pip_install") + +pip3_install( + name = "py_deps", + requirements = "//:requirements.txt", ) diff --git a/third-party/benchmark/bindings/python/build_defs.bzl b/third-party/benchmark/bindings/python/build_defs.bzl index b0c1b0f5807e..45907aaa5e2d 100644 --- a/third-party/benchmark/bindings/python/build_defs.bzl +++ b/third-party/benchmark/bindings/python/build_defs.bzl @@ -1,7 +1,3 @@ -""" -This file contains some build definitions for C++ extensions used in the Google Benchmark Python bindings. -""" - _SHARED_LIB_SUFFIX = { "//conditions:default": ".so", "//:windows": ".dll", @@ -12,8 +8,8 @@ def py_extension(name, srcs, hdrs = [], copts = [], features = [], deps = []): shared_lib_name = name + shared_lib_suffix native.cc_binary( name = shared_lib_name, - linkshared = True, - linkstatic = True, + linkshared = 1, + linkstatic = 1, srcs = srcs + hdrs, copts = copts, features = features, diff --git a/third-party/benchmark/bindings/python/google_benchmark/__init__.py b/third-party/benchmark/bindings/python/google_benchmark/__init__.py index e14769f451fe..1055bf241856 100644 --- a/third-party/benchmark/bindings/python/google_benchmark/__init__.py +++ b/third-party/benchmark/bindings/python/google_benchmark/__init__.py @@ -26,29 +26,47 @@ Example usage: if __name__ == '__main__': benchmark.main() """ -import atexit from absl import app - from google_benchmark import _benchmark from google_benchmark._benchmark import ( - Counter as Counter, - State as State, - kMicrosecond as kMicrosecond, - kMillisecond as kMillisecond, - kNanosecond as kNanosecond, - kSecond as kSecond, - o1 as o1, - oAuto as oAuto, - oLambda as oLambda, - oLogN as oLogN, - oN as oN, - oNCubed as oNCubed, - oNLogN as oNLogN, - oNone as oNone, - oNSquared as oNSquared, + Counter, + kNanosecond, + kMicrosecond, + kMillisecond, + kSecond, + oNone, + o1, + oN, + oNSquared, + oNCubed, + oLogN, + oNLogN, + oAuto, + oLambda, ) -from google_benchmark.version import __version__ as __version__ + + +__all__ = [ + "register", + "main", + "Counter", + "kNanosecond", + "kMicrosecond", + "kMillisecond", + "kSecond", + "oNone", + "o1", + "oN", + "oNSquared", + "oNCubed", + "oLogN", + "oNLogN", + "oAuto", + "oLambda", +] + +__version__ = "0.2.0" class __OptionMaker: @@ -76,13 +94,14 @@ class __OptionMaker: # The function that get returned on @option.range(start=0, limit=1<<5). def __builder_method(*args, **kwargs): + # The decorator that get called, either with the benchmared function # or the previous Options def __decorator(func_or_options): options = self.make(func_or_options) options.builder_calls.append((builder_name, args, kwargs)) # The decorator returns Options so it is not technically a decorator - # and needs a final call to @register + # and needs a final call to @regiser return options return __decorator @@ -137,4 +156,3 @@ def main(argv=None): # Methods for use with custom main function. initialize = _benchmark.Initialize run_benchmarks = _benchmark.RunSpecifiedBenchmarks -atexit.register(_benchmark.ClearRegisteredBenchmarks) diff --git a/third-party/benchmark/bindings/python/google_benchmark/benchmark.cc b/third-party/benchmark/bindings/python/google_benchmark/benchmark.cc index f44476901cae..02b6ed7ed590 100644 --- a/third-party/benchmark/bindings/python/google_benchmark/benchmark.cc +++ b/third-party/benchmark/bindings/python/google_benchmark/benchmark.cc @@ -1,17 +1,20 @@ // Benchmark for Python. -#include "benchmark/benchmark.h" +#include +#include +#include + +#include "pybind11/operators.h" +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" +#include "pybind11/stl_bind.h" -#include "nanobind/nanobind.h" -#include "nanobind/operators.h" -#include "nanobind/stl/bind_map.h" -#include "nanobind/stl/string.h" -#include "nanobind/stl/vector.h" +#include "benchmark/benchmark.h" -NB_MAKE_OPAQUE(benchmark::UserCounters); +PYBIND11_MAKE_OPAQUE(benchmark::UserCounters); namespace { -namespace nb = nanobind; +namespace py = ::pybind11; std::vector Initialize(const std::vector& argv) { // The `argv` pointers here become invalid when this function returns, but @@ -34,16 +37,15 @@ std::vector Initialize(const std::vector& argv) { return remaining_argv; } -benchmark::internal::Benchmark* RegisterBenchmark(const std::string& name, - nb::callable f) { +benchmark::internal::Benchmark* RegisterBenchmark(const char* name, + py::function f) { return benchmark::RegisterBenchmark( name, [f](benchmark::State& state) { f(&state); }); } -NB_MODULE(_benchmark, m) { - +PYBIND11_MODULE(_benchmark, m) { using benchmark::TimeUnit; - nb::enum_(m, "TimeUnit") + py::enum_(m, "TimeUnit") .value("kNanosecond", TimeUnit::kNanosecond) .value("kMicrosecond", TimeUnit::kMicrosecond) .value("kMillisecond", TimeUnit::kMillisecond) @@ -51,74 +53,72 @@ NB_MODULE(_benchmark, m) { .export_values(); using benchmark::BigO; - nb::enum_(m, "BigO") + py::enum_(m, "BigO") .value("oNone", BigO::oNone) .value("o1", BigO::o1) .value("oN", BigO::oN) .value("oNSquared", BigO::oNSquared) .value("oNCubed", BigO::oNCubed) .value("oLogN", BigO::oLogN) - .value("oNLogN", BigO::oNLogN) + .value("oNLogN", BigO::oLogN) .value("oAuto", BigO::oAuto) .value("oLambda", BigO::oLambda) .export_values(); using benchmark::internal::Benchmark; - nb::class_(m, "Benchmark") - // For methods returning a pointer to the current object, reference - // return policy is used to ask nanobind not to take ownership of the + py::class_(m, "Benchmark") + // For methods returning a pointer tor the current object, reference + // return policy is used to ask pybind not to take ownership oof the // returned object and avoid calling delete on it. // https://pybind11.readthedocs.io/en/stable/advanced/functions.html#return-value-policies // // For methods taking a const std::vector<...>&, a copy is created // because a it is bound to a Python list. // https://pybind11.readthedocs.io/en/stable/advanced/cast/stl.html - .def("unit", &Benchmark::Unit, nb::rv_policy::reference) - .def("arg", &Benchmark::Arg, nb::rv_policy::reference) - .def("args", &Benchmark::Args, nb::rv_policy::reference) - .def("range", &Benchmark::Range, nb::rv_policy::reference, - nb::arg("start"), nb::arg("limit")) + .def("unit", &Benchmark::Unit, py::return_value_policy::reference) + .def("arg", &Benchmark::Arg, py::return_value_policy::reference) + .def("args", &Benchmark::Args, py::return_value_policy::reference) + .def("range", &Benchmark::Range, py::return_value_policy::reference, + py::arg("start"), py::arg("limit")) .def("dense_range", &Benchmark::DenseRange, - nb::rv_policy::reference, nb::arg("start"), - nb::arg("limit"), nb::arg("step") = 1) - .def("ranges", &Benchmark::Ranges, nb::rv_policy::reference) + py::return_value_policy::reference, py::arg("start"), + py::arg("limit"), py::arg("step") = 1) + .def("ranges", &Benchmark::Ranges, py::return_value_policy::reference) .def("args_product", &Benchmark::ArgsProduct, - nb::rv_policy::reference) - .def("arg_name", &Benchmark::ArgName, nb::rv_policy::reference) + py::return_value_policy::reference) + .def("arg_name", &Benchmark::ArgName, py::return_value_policy::reference) .def("arg_names", &Benchmark::ArgNames, - nb::rv_policy::reference) + py::return_value_policy::reference) .def("range_pair", &Benchmark::RangePair, - nb::rv_policy::reference, nb::arg("lo1"), nb::arg("hi1"), - nb::arg("lo2"), nb::arg("hi2")) + py::return_value_policy::reference, py::arg("lo1"), py::arg("hi1"), + py::arg("lo2"), py::arg("hi2")) .def("range_multiplier", &Benchmark::RangeMultiplier, - nb::rv_policy::reference) - .def("min_time", &Benchmark::MinTime, nb::rv_policy::reference) - .def("min_warmup_time", &Benchmark::MinWarmUpTime, - nb::rv_policy::reference) + py::return_value_policy::reference) + .def("min_time", &Benchmark::MinTime, py::return_value_policy::reference) .def("iterations", &Benchmark::Iterations, - nb::rv_policy::reference) + py::return_value_policy::reference) .def("repetitions", &Benchmark::Repetitions, - nb::rv_policy::reference) + py::return_value_policy::reference) .def("report_aggregates_only", &Benchmark::ReportAggregatesOnly, - nb::rv_policy::reference, nb::arg("value") = true) + py::return_value_policy::reference, py::arg("value") = true) .def("display_aggregates_only", &Benchmark::DisplayAggregatesOnly, - nb::rv_policy::reference, nb::arg("value") = true) + py::return_value_policy::reference, py::arg("value") = true) .def("measure_process_cpu_time", &Benchmark::MeasureProcessCPUTime, - nb::rv_policy::reference) + py::return_value_policy::reference) .def("use_real_time", &Benchmark::UseRealTime, - nb::rv_policy::reference) + py::return_value_policy::reference) .def("use_manual_time", &Benchmark::UseManualTime, - nb::rv_policy::reference) + py::return_value_policy::reference) .def( "complexity", (Benchmark * (Benchmark::*)(benchmark::BigO)) & Benchmark::Complexity, - nb::rv_policy::reference, - nb::arg("complexity") = benchmark::oAuto); + py::return_value_policy::reference, + py::arg("complexity") = benchmark::oAuto); using benchmark::Counter; - nb::class_ py_counter(m, "Counter"); + py::class_ py_counter(m, "Counter"); - nb::enum_(py_counter, "Flags") + py::enum_(py_counter, "Flags") .value("kDefaults", Counter::Flags::kDefaults) .value("kIsRate", Counter::Flags::kIsRate) .value("kAvgThreads", Counter::Flags::kAvgThreads) @@ -130,55 +130,52 @@ NB_MODULE(_benchmark, m) { .value("kAvgIterationsRate", Counter::Flags::kAvgIterationsRate) .value("kInvert", Counter::Flags::kInvert) .export_values() - .def(nb::self | nb::self); + .def(py::self | py::self); - nb::enum_(py_counter, "OneK") + py::enum_(py_counter, "OneK") .value("kIs1000", Counter::OneK::kIs1000) .value("kIs1024", Counter::OneK::kIs1024) .export_values(); py_counter - .def(nb::init(), - nb::arg("value") = 0., nb::arg("flags") = Counter::kDefaults, - nb::arg("k") = Counter::kIs1000) - .def("__init__", ([](Counter *c, double value) { new (c) Counter(value); })) - .def_rw("value", &Counter::value) - .def_rw("flags", &Counter::flags) - .def_rw("oneK", &Counter::oneK) - .def(nb::init_implicit()); - - nb::implicitly_convertible(); - - nb::bind_map(m, "UserCounters"); + .def(py::init(), + py::arg("value") = 0., py::arg("flags") = Counter::kDefaults, + py::arg("k") = Counter::kIs1000) + .def(py::init([](double value) { return Counter(value); })) + .def_readwrite("value", &Counter::value) + .def_readwrite("flags", &Counter::flags) + .def_readwrite("oneK", &Counter::oneK); + py::implicitly_convertible(); + py::implicitly_convertible(); + + py::bind_map(m, "UserCounters"); using benchmark::State; - nb::class_(m, "State") + py::class_(m, "State") .def("__bool__", &State::KeepRunning) - .def_prop_ro("keep_running", &State::KeepRunning) + .def_property_readonly("keep_running", &State::KeepRunning) .def("pause_timing", &State::PauseTiming) .def("resume_timing", &State::ResumeTiming) .def("skip_with_error", &State::SkipWithError) - .def_prop_ro("error_occurred", &State::error_occurred) + .def_property_readonly("error_occurred", &State::error_occurred) .def("set_iteration_time", &State::SetIterationTime) - .def_prop_rw("bytes_processed", &State::bytes_processed, + .def_property("bytes_processed", &State::bytes_processed, &State::SetBytesProcessed) - .def_prop_rw("complexity_n", &State::complexity_length_n, + .def_property("complexity_n", &State::complexity_length_n, &State::SetComplexityN) - .def_prop_rw("items_processed", &State::items_processed, - &State::SetItemsProcessed) - .def("set_label", &State::SetLabel) - .def("range", &State::range, nb::arg("pos") = 0) - .def_prop_ro("iterations", &State::iterations) - .def_prop_ro("name", &State::name) - .def_rw("counters", &State::counters) - .def_prop_ro("thread_index", &State::thread_index) - .def_prop_ro("threads", &State::threads); + .def_property("items_processed", &State::items_processed, + &State::SetItemsProcessed) + .def("set_label", (void(State::*)(const char*)) & State::SetLabel) + .def("range", &State::range, py::arg("pos") = 0) + .def_property_readonly("iterations", &State::iterations) + .def_readwrite("counters", &State::counters) + .def_property_readonly("thread_index", &State::thread_index) + .def_property_readonly("threads", &State::threads); m.def("Initialize", Initialize); m.def("RegisterBenchmark", RegisterBenchmark, - nb::rv_policy::reference); + py::return_value_policy::reference); m.def("RunSpecifiedBenchmarks", []() { benchmark::RunSpecifiedBenchmarks(); }); - m.def("ClearRegisteredBenchmarks", benchmark::ClearRegisteredBenchmarks); }; } // namespace diff --git a/third-party/benchmark/bindings/python/google_benchmark/example.py b/third-party/benchmark/bindings/python/google_benchmark/example.py index b5b2f88ff306..fb0234b8fd7e 100644 --- a/third-party/benchmark/bindings/python/google_benchmark/example.py +++ b/third-party/benchmark/bindings/python/google_benchmark/example.py @@ -73,7 +73,7 @@ def manual_timing(state): @benchmark.register def custom_counters(state): - """Collect custom metric using benchmark.Counter.""" + """Collect cutom metric using benchmark.Counter.""" num_foo = 0.0 while state: # Benchmark some code here @@ -86,9 +86,7 @@ def custom_counters(state): # Set a counter as a rate. state.counters["foo_rate"] = Counter(num_foo, Counter.kIsRate) # Set a counter as an inverse of rate. - state.counters["foo_inv_rate"] = Counter( - num_foo, Counter.kIsRate | Counter.kInvert - ) + state.counters["foo_inv_rate"] = Counter(num_foo, Counter.kIsRate | Counter.kInvert) # Set a counter as a thread-average quantity. state.counters["foo_avg"] = Counter(num_foo, Counter.kAvgThreads) # There's also a combined flag: diff --git a/third-party/benchmark/bindings/python/pybind11.BUILD b/third-party/benchmark/bindings/python/pybind11.BUILD new file mode 100644 index 000000000000..bc833500383a --- /dev/null +++ b/third-party/benchmark/bindings/python/pybind11.BUILD @@ -0,0 +1,20 @@ +cc_library( + name = "pybind11", + hdrs = glob( + include = [ + "include/pybind11/*.h", + "include/pybind11/detail/*.h", + ], + exclude = [ + "include/pybind11/common.h", + "include/pybind11/eigen.h", + ], + ), + copts = [ + "-fexceptions", + "-Wno-undefined-inline", + "-Wno-pragma-once-outside-header", + ], + includes = ["include"], + visibility = ["//visibility:public"], +) diff --git a/third-party/benchmark/bindings/python/python_headers.BUILD b/third-party/benchmark/bindings/python/python_headers.BUILD index 8f139f8621e0..9c34cf6ca4bd 100644 --- a/third-party/benchmark/bindings/python/python_headers.BUILD +++ b/third-party/benchmark/bindings/python/python_headers.BUILD @@ -1,7 +1,3 @@ -licenses(["notice"]) - -package(default_visibility = ["//visibility:public"]) - cc_library( name = "python_headers", hdrs = glob(["**/*.h"]), diff --git a/third-party/benchmark/bindings/python/requirements.txt b/third-party/benchmark/bindings/python/requirements.txt new file mode 100644 index 000000000000..f5bbe7eca5ce --- /dev/null +++ b/third-party/benchmark/bindings/python/requirements.txt @@ -0,0 +1,2 @@ +absl-py>=0.7.1 + diff --git a/third-party/benchmark/cmake/CXXFeatureCheck.cmake b/third-party/benchmark/cmake/CXXFeatureCheck.cmake index e51482659b0f..62e6741fe3de 100644 --- a/third-party/benchmark/cmake/CXXFeatureCheck.cmake +++ b/third-party/benchmark/cmake/CXXFeatureCheck.cmake @@ -17,8 +17,6 @@ if(__cxx_feature_check) endif() set(__cxx_feature_check INCLUDED) -option(CXXFEATURECHECK_DEBUG OFF) - function(cxx_feature_check FILE) string(TOLOWER ${FILE} FILE) string(TOUPPER ${FILE} VAR) @@ -29,22 +27,18 @@ function(cxx_feature_check FILE) return() endif() - set(FEATURE_CHECK_CMAKE_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS}) if (ARGC GREATER 1) message(STATUS "Enabling additional flags: ${ARGV1}") - list(APPEND FEATURE_CHECK_CMAKE_FLAGS ${ARGV1}) + list(APPEND BENCHMARK_CXX_LINKER_FLAGS ${ARGV1}) endif() if (NOT DEFINED COMPILE_${FEATURE}) + message(STATUS "Performing Test ${FEATURE}") if(CMAKE_CROSSCOMPILING) - message(STATUS "Cross-compiling to test ${FEATURE}") try_compile(COMPILE_${FEATURE} ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/${FILE}.cpp - CXX_STANDARD 11 - CXX_STANDARD_REQUIRED ON - CMAKE_FLAGS ${FEATURE_CHECK_CMAKE_FLAGS} - LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES} - OUTPUT_VARIABLE COMPILE_OUTPUT_VAR) + CMAKE_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS} + LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES}) if(COMPILE_${FEATURE}) message(WARNING "If you see build failures due to cross compilation, try setting HAVE_${VAR} to 0") @@ -53,14 +47,11 @@ function(cxx_feature_check FILE) set(RUN_${FEATURE} 1 CACHE INTERNAL "") endif() else() - message(STATUS "Compiling and running to test ${FEATURE}") + message(STATUS "Performing Test ${FEATURE}") try_run(RUN_${FEATURE} COMPILE_${FEATURE} ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/${FILE}.cpp - CXX_STANDARD 11 - CXX_STANDARD_REQUIRED ON - CMAKE_FLAGS ${FEATURE_CHECK_CMAKE_FLAGS} - LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES} - COMPILE_OUTPUT_VARIABLE COMPILE_OUTPUT_VAR) + CMAKE_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS} + LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES}) endif() endif() @@ -70,11 +61,7 @@ function(cxx_feature_check FILE) add_definitions(-DHAVE_${VAR}) else() if(NOT COMPILE_${FEATURE}) - if(CXXFEATURECHECK_DEBUG) - message(STATUS "Performing Test ${FEATURE} -- failed to compile: ${COMPILE_OUTPUT_VAR}") - else() - message(STATUS "Performing Test ${FEATURE} -- failed to compile") - endif() + message(STATUS "Performing Test ${FEATURE} -- failed to compile") else() message(STATUS "Performing Test ${FEATURE} -- compiled but failed to run") endif() diff --git a/third-party/benchmark/cmake/GetGitVersion.cmake b/third-party/benchmark/cmake/GetGitVersion.cmake index b0210103b2cc..04a1f9b70d68 100644 --- a/third-party/benchmark/cmake/GetGitVersion.cmake +++ b/third-party/benchmark/cmake/GetGitVersion.cmake @@ -20,16 +20,38 @@ set(__get_git_version INCLUDED) function(get_git_version var) if(GIT_EXECUTABLE) - execute_process(COMMAND ${GIT_EXECUTABLE} describe --tags --match "v[0-9]*.[0-9]*.[0-9]*" --abbrev=8 --dirty + execute_process(COMMAND ${GIT_EXECUTABLE} describe --tags --match "v[0-9]*.[0-9]*.[0-9]*" --abbrev=8 WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} RESULT_VARIABLE status - OUTPUT_VARIABLE GIT_VERSION + OUTPUT_VARIABLE GIT_DESCRIBE_VERSION ERROR_QUIET) if(status) - set(GIT_VERSION "v0.0.0") + set(GIT_DESCRIBE_VERSION "v0.0.0") endif() + + string(STRIP ${GIT_DESCRIBE_VERSION} GIT_DESCRIBE_VERSION) + if(GIT_DESCRIBE_VERSION MATCHES v[^-]*-) + string(REGEX REPLACE "v([^-]*)-([0-9]+)-.*" "\\1.\\2" GIT_VERSION ${GIT_DESCRIBE_VERSION}) + else() + string(REGEX REPLACE "v(.*)" "\\1" GIT_VERSION ${GIT_DESCRIBE_VERSION}) + endif() + + # Work out if the repository is dirty + execute_process(COMMAND ${GIT_EXECUTABLE} update-index -q --refresh + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + OUTPUT_QUIET + ERROR_QUIET) + execute_process(COMMAND ${GIT_EXECUTABLE} diff-index --name-only HEAD -- + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + OUTPUT_VARIABLE GIT_DIFF_INDEX + ERROR_QUIET) + string(COMPARE NOTEQUAL "${GIT_DIFF_INDEX}" "" GIT_DIRTY) + if (${GIT_DIRTY}) + set(GIT_DESCRIBE_VERSION "${GIT_DESCRIBE_VERSION}-dirty") + endif() + message(STATUS "git version: ${GIT_DESCRIBE_VERSION} normalized to ${GIT_VERSION}") else() - set(GIT_VERSION "v0.0.0") + set(GIT_VERSION "0.0.0") endif() set(${var} ${GIT_VERSION} PARENT_SCOPE) diff --git a/third-party/benchmark/cmake/GoogleTest.cmake b/third-party/benchmark/cmake/GoogleTest.cmake index e66e9d1a2076..66cb91008b73 100644 --- a/third-party/benchmark/cmake/GoogleTest.cmake +++ b/third-party/benchmark/cmake/GoogleTest.cmake @@ -29,25 +29,15 @@ set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) include(${GOOGLETEST_PREFIX}/googletest-paths.cmake) +# googletest doesn't seem to want to stay build warning clean so let's not hurt ourselves. +add_compile_options(-w) + # Add googletest directly to our build. This defines # the gtest and gtest_main targets. add_subdirectory(${GOOGLETEST_SOURCE_DIR} ${GOOGLETEST_BINARY_DIR} EXCLUDE_FROM_ALL) -# googletest doesn't seem to want to stay build warning clean so let's not hurt ourselves. -if (MSVC) - target_compile_options(gtest PRIVATE "/wd4244" "/wd4722") - target_compile_options(gtest_main PRIVATE "/wd4244" "/wd4722") - target_compile_options(gmock PRIVATE "/wd4244" "/wd4722") - target_compile_options(gmock_main PRIVATE "/wd4244" "/wd4722") -else() - target_compile_options(gtest PRIVATE "-w") - target_compile_options(gtest_main PRIVATE "-w") - target_compile_options(gmock PRIVATE "-w") - target_compile_options(gmock_main PRIVATE "-w") -endif() - if(NOT DEFINED GTEST_COMPILE_COMMANDS) set(GTEST_COMPILE_COMMANDS ON) endif() diff --git a/third-party/benchmark/cmake/Modules/FindPFM.cmake b/third-party/benchmark/cmake/Modules/FindPFM.cmake index 4c1ce938f9f7..cf807a1ee9e9 100644 --- a/third-party/benchmark/cmake/Modules/FindPFM.cmake +++ b/third-party/benchmark/cmake/Modules/FindPFM.cmake @@ -1,28 +1,26 @@ # If successful, the following variables will be defined: -# PFM_FOUND. -# PFM_LIBRARIES -# PFM_INCLUDE_DIRS -# the following target will be defined: -# PFM::libpfm - +# HAVE_LIBPFM. +# Set BENCHMARK_ENABLE_LIBPFM to 0 to disable, regardless of libpfm presence. +include(CheckIncludeFile) +include(CheckLibraryExists) include(FeatureSummary) -include(FindPackageHandleStandardArgs) +enable_language(C) set_package_properties(PFM PROPERTIES URL http://perfmon2.sourceforge.net/ - DESCRIPTION "A helper library to develop monitoring tools" + DESCRIPTION "a helper library to develop monitoring tools" PURPOSE "Used to program specific performance monitoring events") -find_library(PFM_LIBRARY NAMES pfm) -find_path(PFM_INCLUDE_DIR NAMES perfmon/pfmlib.h) - -find_package_handle_standard_args(PFM REQUIRED_VARS PFM_LIBRARY PFM_INCLUDE_DIR) - -if (PFM_FOUND AND NOT TARGET PFM::libpfm) - add_library(PFM::libpfm UNKNOWN IMPORTED) - set_target_properties(PFM::libpfm PROPERTIES - IMPORTED_LOCATION "${PFM_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "${PFM_INCLUDE_DIR}") +check_library_exists(libpfm.a pfm_initialize "" HAVE_LIBPFM_INITIALIZE) +if(HAVE_LIBPFM_INITIALIZE) + check_include_file(perfmon/perf_event.h HAVE_PERFMON_PERF_EVENT_H) + check_include_file(perfmon/pfmlib.h HAVE_PERFMON_PFMLIB_H) + check_include_file(perfmon/pfmlib_perf_event.h HAVE_PERFMON_PFMLIB_PERF_EVENT_H) + if(HAVE_PERFMON_PERF_EVENT_H AND HAVE_PERFMON_PFMLIB_H AND HAVE_PERFMON_PFMLIB_PERF_EVENT_H) + message("Using Perf Counters.") + set(HAVE_LIBPFM 1) + set(PFM_FOUND 1) + endif() +else() + message("Perf Counters support requested, but was unable to find libpfm.") endif() - -mark_as_advanced(PFM_LIBRARY PFM_INCLUDE_DIR) diff --git a/third-party/benchmark/cmake/benchmark.pc.in b/third-party/benchmark/cmake/benchmark.pc.in index 9dae881c79f9..34beb012eef1 100644 --- a/third-party/benchmark/cmake/benchmark.pc.in +++ b/third-party/benchmark/cmake/benchmark.pc.in @@ -1,7 +1,7 @@ prefix=@CMAKE_INSTALL_PREFIX@ exec_prefix=${prefix} -libdir=@CMAKE_INSTALL_FULL_LIBDIR@ -includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ +libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ +includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ Name: @PROJECT_NAME@ Description: Google microbenchmark framework diff --git a/third-party/benchmark/docs/AssemblyTests.md b/third-party/benchmark/docs/AssemblyTests.md index 89df7ca520df..1fbdc269b53d 100644 --- a/third-party/benchmark/docs/AssemblyTests.md +++ b/third-party/benchmark/docs/AssemblyTests.md @@ -111,7 +111,6 @@ between compilers or compiler versions. A common example of this is matching stack frame addresses. In this case regular expressions can be used to match the differing bits of output. For example: - ```c++ int ExternInt; struct Point { int x, y, z; }; @@ -128,7 +127,6 @@ extern "C" void test_store_point() { // CHECK: ret } ``` - ## Current Requirements and Limitations diff --git a/third-party/benchmark/docs/_config.yml b/third-party/benchmark/docs/_config.yml index 32f9f2e0ddc6..2f7efbeab578 100644 --- a/third-party/benchmark/docs/_config.yml +++ b/third-party/benchmark/docs/_config.yml @@ -1,3 +1 @@ -theme: jekyll-theme-minimal -logo: /assets/images/icon_black.png -show_downloads: true +theme: jekyll-theme-minimal \ No newline at end of file diff --git a/third-party/benchmark/docs/dependencies.md b/third-party/benchmark/docs/dependencies.md index 07760e10e370..7af52b95bd86 100644 --- a/third-party/benchmark/docs/dependencies.md +++ b/third-party/benchmark/docs/dependencies.md @@ -1,13 +1,19 @@ # Build tool dependency policy -We follow the [Foundational C++ support policy](https://opensource.google/documentation/policies/cplusplus-support) for our build tools. In -particular the ["Build Systems" section](https://opensource.google/documentation/policies/cplusplus-support#build-systems). +To ensure the broadest compatibility when building the benchmark library, but +still allow forward progress, we require any build tooling to be available for: -## CMake +* Debian stable _and_ +* The last two Ubuntu LTS releases -The current supported version is CMake 3.10 as of 2023-08-10. Most modern -distributions include newer versions, for example: +Currently, this means using build tool versions that are available for Ubuntu +18.04 (Bionic Beaver), Ubuntu 20.04 (Focal Fossa), and Debian 11 (bullseye). -* Ubuntu 20.04 provides CMake 3.16.3 -* Debian 11.4 provides CMake 3.18.4 -* Ubuntu 22.04 provides CMake 3.22.1 +_Note, CI also runs ubuntu-16.04 and ubuntu-14.04 to ensure best effort support +for older versions._ + +## cmake +The current supported version is cmake 3.5.1 as of 2018-06-06. + +_Note, this version is also available for Ubuntu 14.04, an older Ubuntu LTS +release, as `cmake3`._ diff --git a/third-party/benchmark/docs/index.md b/third-party/benchmark/docs/index.md index 9cada9688b1d..eb82eff9eee0 100644 --- a/third-party/benchmark/docs/index.md +++ b/third-party/benchmark/docs/index.md @@ -4,9 +4,7 @@ * [Dependencies](dependencies.md) * [Perf Counters](perf_counters.md) * [Platform Specific Build Instructions](platform_specific_build_instructions.md) -* [Python Bindings](python_bindings.md) * [Random Interleaving](random_interleaving.md) -* [Reducing Variance](reducing_variance.md) * [Releasing](releasing.md) * [Tools](tools.md) -* [User Guide](user_guide.md) +* [User Guide](user_guide.md) \ No newline at end of file diff --git a/third-party/benchmark/docs/perf_counters.md b/third-party/benchmark/docs/perf_counters.md index f342092c9970..74560e966971 100644 --- a/third-party/benchmark/docs/perf_counters.md +++ b/third-party/benchmark/docs/perf_counters.md @@ -12,17 +12,16 @@ This feature is available if: * The benchmark is run on an architecture featuring a Performance Monitoring Unit (PMU), * The benchmark is compiled with support for collecting counters. Currently, - this requires [libpfm](http://perfmon2.sourceforge.net/), which is built as a - dependency via Bazel. + this requires [libpfm](http://perfmon2.sourceforge.net/) be available at build + time The feature does not require modifying benchmark code. Counter collection is handled at the boundaries where timer collection is also handled. To opt-in: -* If using a Bazel build, add `--define pfm=1` to your build flags -* If using CMake: - * Install `libpfm4-dev`, e.g. `apt-get install libpfm4-dev`. - * Enable the CMake flag `BENCHMARK_ENABLE_LIBPFM` in `CMakeLists.txt`. + +* Install `libpfm4-dev`, e.g. `apt-get install libpfm4-dev`. +* Enable the cmake flag BENCHMARK_ENABLE_LIBPFM. To use, pass a comma-separated list of counter names through the `--benchmark_perf_counters` flag. The names are decoded through libpfm - meaning, @@ -32,4 +31,4 @@ mapped by libpfm to platform-specifics - see libpfm The counter values are reported back through the [User Counters](../README.md#custom-counters) mechanism, meaning, they are available in all the formats (e.g. JSON) supported -by User Counters. +by User Counters. \ No newline at end of file diff --git a/third-party/benchmark/docs/releasing.md b/third-party/benchmark/docs/releasing.md index 09bf93764d00..334f93539381 100644 --- a/third-party/benchmark/docs/releasing.md +++ b/third-party/benchmark/docs/releasing.md @@ -1,23 +1,30 @@ # How to release * Make sure you're on main and synced to HEAD -* Ensure the project builds and tests run +* Ensure the project builds and tests run (sanity check only, obviously) * `parallel -j0 exec ::: test/*_test` can help ensure everything at least passes * Prepare release notes * `git log $(git describe --abbrev=0 --tags)..HEAD` gives you the list of commits between the last annotated tag and HEAD * Pick the most interesting. -* Create one last commit that updates the version saved in `CMakeLists.txt` and `MODULE.bazel` - to the release version you're creating. (This version will be used if benchmark is installed - from the archive you'll be creating in the next step.) +* Create one last commit that updates the version saved in `CMakeLists.txt` and the + `__version__` variable in `bindings/python/google_benchmark/__init__.py`to the release + version you're creating. (This version will be used if benchmark is installed from the + archive you'll be creating in the next step.) ``` -project (benchmark VERSION 1.8.0 LANGUAGES CXX) +project (benchmark VERSION 1.6.0 LANGUAGES CXX) ``` -``` -module(name = "com_github_google_benchmark", version="1.8.0") +```python +# bindings/python/google_benchmark/__init__.py + +# ... + +__version__ = "1.6.0" # <-- change this to the release version you are creating + +# ... ``` * Create a release through github's interface @@ -26,6 +33,3 @@ module(name = "com_github_google_benchmark", version="1.8.0") * `git pull --tags` * `git tag -a -f ` * `git push --force --tags origin` -* Confirm that the "Build and upload Python wheels" action runs to completion - * Run it manually if it hasn't run. - * IMPORTANT: When re-running manually, make sure to select the newly created `` as the workflow version in the "Run workflow" tab on the GitHub Actions page. diff --git a/third-party/benchmark/docs/tools.md b/third-party/benchmark/docs/tools.md index 411f41d405ff..f2d0c497f3fc 100644 --- a/third-party/benchmark/docs/tools.md +++ b/third-party/benchmark/docs/tools.md @@ -186,146 +186,6 @@ Benchmark Time CPU Time Old This is a mix of the previous two modes, two (potentially different) benchmark binaries are run, and a different filter is applied to each one. As you can note, the values in `Time` and `CPU` columns are calculated as `(new - old) / |old|`. -### Note: Interpreting the output - -Performance measurements are an art, and performance comparisons are doubly so. -Results are often noisy and don't necessarily have large absolute differences to -them, so just by visual inspection, it is not at all apparent if two -measurements are actually showing a performance change or not. It is even more -confusing with multiple benchmark repetitions. - -Thankfully, what we can do, is use statistical tests on the results to determine -whether the performance has statistically-significantly changed. `compare.py` -uses [Mann–Whitney U -test](https://en.wikipedia.org/wiki/Mann%E2%80%93Whitney_U_test), with a null -hypothesis being that there's no difference in performance. - -**The below output is a summary of a benchmark comparison with statistics -provided for a multi-threaded process.** -``` -Benchmark Time CPU Time Old Time New CPU Old CPU New ------------------------------------------------------------------------------------------------------------------------------ -benchmark/threads:1/process_time/real_time_pvalue 0.0000 0.0000 U Test, Repetitions: 27 vs 27 -benchmark/threads:1/process_time/real_time_mean -0.1442 -0.1442 90 77 90 77 -benchmark/threads:1/process_time/real_time_median -0.1444 -0.1444 90 77 90 77 -benchmark/threads:1/process_time/real_time_stddev +0.3974 +0.3933 0 0 0 0 -benchmark/threads:1/process_time/real_time_cv +0.6329 +0.6280 0 0 0 0 -OVERALL_GEOMEAN -0.1442 -0.1442 0 0 0 0 -``` --------------------------------------------- -Here's a breakdown of each row: - -**benchmark/threads:1/process_time/real_time_pvalue**: This shows the _p-value_ for -the statistical test comparing the performance of the process running with one -thread. A value of 0.0000 suggests a statistically significant difference in -performance. The comparison was conducted using the U Test (Mann-Whitney -U Test) with 27 repetitions for each case. - -**benchmark/threads:1/process_time/real_time_mean**: This shows the relative -difference in mean execution time between two different cases. The negative -value (-0.1442) implies that the new process is faster by about 14.42%. The old -time was 90 units, while the new time is 77 units. - -**benchmark/threads:1/process_time/real_time_median**: Similarly, this shows the -relative difference in the median execution time. Again, the new process is -faster by 14.44%. - -**benchmark/threads:1/process_time/real_time_stddev**: This is the relative -difference in the standard deviation of the execution time, which is a measure -of how much variation or dispersion there is from the mean. A positive value -(+0.3974) implies there is more variance in the execution time in the new -process. - -**benchmark/threads:1/process_time/real_time_cv**: CV stands for Coefficient of -Variation. It is the ratio of the standard deviation to the mean. It provides a -standardized measure of dispersion. An increase (+0.6329) indicates more -relative variability in the new process. - -**OVERALL_GEOMEAN**: Geomean stands for geometric mean, a type of average that is -less influenced by outliers. The negative value indicates a general improvement -in the new process. However, given the values are all zero for the old and new -times, this seems to be a mistake or placeholder in the output. - ------------------------------------------ - - - -Let's first try to see what the different columns represent in the above -`compare.py` benchmarking output: - - 1. **Benchmark:** The name of the function being benchmarked, along with the - size of the input (after the slash). - - 2. **Time:** The average time per operation, across all iterations. - - 3. **CPU:** The average CPU time per operation, across all iterations. - - 4. **Iterations:** The number of iterations the benchmark was run to get a - stable estimate. - - 5. **Time Old and Time New:** These represent the average time it takes for a - function to run in two different scenarios or versions. For example, you - might be comparing how fast a function runs before and after you make some - changes to it. - - 6. **CPU Old and CPU New:** These show the average amount of CPU time that the - function uses in two different scenarios or versions. This is similar to - Time Old and Time New, but focuses on CPU usage instead of overall time. - -In the comparison section, the relative differences in both time and CPU time -are displayed for each input size. - - -A statistically-significant difference is determined by a **p-value**, which is -a measure of the probability that the observed difference could have occurred -just by random chance. A smaller p-value indicates stronger evidence against the -null hypothesis. - -**Therefore:** - 1. If the p-value is less than the chosen significance level (alpha), we - reject the null hypothesis and conclude the benchmarks are significantly - different. - 2. If the p-value is greater than or equal to alpha, we fail to reject the - null hypothesis and treat the two benchmarks as similar. - - - -The result of said the statistical test is additionally communicated through color coding: -```diff -+ Green: -``` - The benchmarks are _**statistically different**_. This could mean the - performance has either **significantly improved** or **significantly - deteriorated**. You should look at the actual performance numbers to see which - is the case. -```diff -- Red: -``` - The benchmarks are _**statistically similar**_. This means the performance - **hasn't significantly changed**. - -In statistical terms, **'green'** means we reject the null hypothesis that -there's no difference in performance, and **'red'** means we fail to reject the -null hypothesis. This might seem counter-intuitive if you're expecting 'green' -to mean 'improved performance' and 'red' to mean 'worsened performance'. -```bash - But remember, in this context: - - 'Success' means 'successfully finding a difference'. - 'Failure' means 'failing to find a difference'. -``` - - -Also, please note that **even if** we determine that there **is** a -statistically-significant difference between the two measurements, it does not -_necessarily_ mean that the actual benchmarks that were measured **are** -different, or vice versa, even if we determine that there is **no** -statistically-significant difference between the two measurements, it does not -necessarily mean that the actual benchmarks that were measured **are not** -different. - - - ### U test If there is a sufficient repetition count of the benchmarks, the tool can do diff --git a/third-party/benchmark/docs/user_guide.md b/third-party/benchmark/docs/user_guide.md index d22a9069091f..34bea6904240 100644 --- a/third-party/benchmark/docs/user_guide.md +++ b/third-party/benchmark/docs/user_guide.md @@ -28,8 +28,6 @@ [Templated Benchmarks](#templated-benchmarks) -[Templated Benchmarks that take arguments](#templated-benchmarks-with-arguments) - [Fixtures](#fixtures) [Custom Counters](#custom-counters) @@ -52,19 +50,14 @@ [Custom Statistics](#custom-statistics) -[Memory Usage](#memory-usage) - [Using RegisterBenchmark](#using-register-benchmark) [Exiting with an Error](#exiting-with-an-error) -[A Faster `KeepRunning` Loop](#a-faster-keep-running-loop) - -## Benchmarking Tips +[A Faster KeepRunning Loop](#a-faster-keep-running-loop) [Disabling CPU Frequency Scaling](#disabling-cpu-frequency-scaling) -[Reducing Variance in Benchmarks](reducing_variance.md) @@ -187,12 +180,6 @@ BM_memcpy/32 12 ns 12 ns 54687500 BM_memcpy/32k 1834 ns 1837 ns 357143 ``` -## Disabling Benchmarks - -It is possible to temporarily disable benchmarks by renaming the benchmark -function to have the prefix "DISABLED_". This will cause the benchmark to -be skipped at runtime. - ## Result comparison @@ -245,19 +232,6 @@ iterations is at least one, not more than 1e9, until CPU time is greater than the minimum time, or the wallclock time is 5x minimum time. The minimum time is set per benchmark by calling `MinTime` on the registered benchmark object. -Furthermore warming up a benchmark might be necessary in order to get -stable results because of e.g caching effects of the code under benchmark. -Warming up means running the benchmark a given amount of time, before -results are actually taken into account. The amount of time for which -the warmup should be run can be set per benchmark by calling -`MinWarmUpTime` on the registered benchmark object or for all benchmarks -using the `--benchmark_min_warmup_time` command-line option. Note that -`MinWarmUpTime` will overwrite the value of `--benchmark_min_warmup_time` -for the single benchmark. How many iterations the warmup run of each -benchmark takes is determined the same way as described in the paragraph -above. Per default the warmup phase is set to 0 seconds and is therefore -disabled. - Average timings are then reported over the iterations run. If multiple repetitions are requested using the `--benchmark_repetitions` command-line option, or at registration time, the benchmark function will be run several @@ -273,12 +247,10 @@ information about the machine on which the benchmarks are run. Global setup/teardown specific to each benchmark can be done by passing a callback to Setup/Teardown: -The setup/teardown callbacks will be invoked once for each benchmark. If the -benchmark is multi-threaded (will run in k threads), they will be invoked -exactly once before each run with k threads. - -If the benchmark uses different size groups of threads, the above will be true -for each size group. +The setup/teardown callbacks will be invoked once for each benchmark. +If the benchmark is multi-threaded (will run in k threads), they will be invoked exactly once before +each run with k threads. +If the benchmark uses different size groups of threads, the above will be true for each size group. Eg., @@ -321,7 +293,7 @@ static void BM_memcpy(benchmark::State& state) { delete[] src; delete[] dst; } -BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(4<<10)->Arg(8<<10); +BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(1<<10)->Arg(8<<10); ``` The preceding code is quite repetitive, and can be replaced with the following @@ -350,8 +322,7 @@ the performance of `std::vector` initialization for uniformly increasing sizes. static void BM_DenseRange(benchmark::State& state) { for(auto _ : state) { std::vector v(state.range(0), state.range(0)); - auto data = v.data(); - benchmark::DoNotOptimize(data); + benchmark::DoNotOptimize(v.data()); benchmark::ClobberMemory(); } } @@ -391,17 +362,17 @@ short-hand. The following macro will pick a few appropriate arguments in the product of the two specified ranges and will generate a benchmark for each such pair. - +{% raw %} ```c++ BENCHMARK(BM_SetInsert)->Ranges({{1<<10, 8<<10}, {128, 512}}); ``` - +{% endraw %} Some benchmarks may require specific argument values that cannot be expressed with `Ranges`. In this case, `ArgsProduct` offers the ability to generate a benchmark input for each combination in the product of the supplied vectors. - +{% raw %} ```c++ BENCHMARK(BM_SetInsert) ->ArgsProduct({{1<<10, 3<<10, 8<<10}, {20, 40, 60, 80}}) @@ -420,7 +391,7 @@ BENCHMARK(BM_SetInsert) ->Args({3<<10, 80}) ->Args({8<<10, 80}); ``` - +{% endraw %} For the most common scenarios, helper methods for creating a list of integers for a given sparse or dense range are provided. @@ -463,22 +434,13 @@ The `test_case_name` is appended to the name of the benchmark and should describe the values passed. ```c++ -template -void BM_takes_args(benchmark::State& state, Args&&... args) { - auto args_tuple = std::make_tuple(std::move(args)...); - for (auto _ : state) { - std::cout << std::get<0>(args_tuple) << ": " << std::get<1>(args_tuple) - << '\n'; - [...] - } +template +void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) { + [...] } // Registers a benchmark named "BM_takes_args/int_string_test" that passes -// the specified values to `args`. +// the specified values to `extra_args`. BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); - -// Registers the same benchmark "BM_takes_args/int_test" that passes -// the specified values to `args`. -BENCHMARK_CAPTURE(BM_takes_args, int_test, 42, 43); ``` Note that elements of `...args` may refer to global variables. Users should @@ -497,8 +459,7 @@ static void BM_StringCompare(benchmark::State& state) { std::string s1(state.range(0), '-'); std::string s2(state.range(0), '-'); for (auto _ : state) { - auto comparison_result = s1.compare(s2); - benchmark::DoNotOptimize(comparison_result); + benchmark::DoNotOptimize(s1.compare(s2)); } state.SetComplexityN(state.range(0)); } @@ -576,30 +537,6 @@ Three macros are provided for adding benchmark templates. #define BENCHMARK_TEMPLATE2(func, arg1, arg2) ``` - - -## Templated Benchmarks that take arguments - -Sometimes there is a need to template benchmarks, and provide arguments to them. - -```c++ -template void BM_Sequential_With_Step(benchmark::State& state, int step) { - Q q; - typename Q::value_type v; - for (auto _ : state) { - for (int i = state.range(0); i-=step; ) - q.push(v); - for (int e = state.range(0); e-=step; ) - q.Wait(&v); - } - // actually messages, not bytes: - state.SetBytesProcessed( - static_cast(state.iterations())*state.range(0)); -} - -BENCHMARK_TEMPLATE1_CAPTURE(BM_Sequential, WaitQueue, Step1, 1)->Range(1<<0, 1<<10); -``` - ## Fixtures @@ -617,10 +554,10 @@ For Example: ```c++ class MyFixture : public benchmark::Fixture { public: - void SetUp(::benchmark::State& state) { + void SetUp(const ::benchmark::State& state) { } - void TearDown(::benchmark::State& state) { + void TearDown(const ::benchmark::State& state) { } }; @@ -731,7 +668,7 @@ is 1k a 1000 (default, `benchmark::Counter::OneK::kIs1000`), or 1024 When you're compiling in C++11 mode or later you can use `insert()` with `std::initializer_list`: - +{% raw %} ```c++ // With C++11, this can be done: state.counters.insert({{"Foo", numFoos}, {"Bar", numBars}, {"Baz", numBazs}}); @@ -740,7 +677,7 @@ When you're compiling in C++11 mode or later you can use `insert()` with state.counters["Bar"] = numBars; state.counters["Baz"] = numBazs; ``` - +{% endraw %} ### Counter Reporting @@ -836,16 +773,6 @@ static void BM_MultiThreaded(benchmark::State& state) { BENCHMARK(BM_MultiThreaded)->Threads(2); ``` -To run the benchmark across a range of thread counts, instead of `Threads`, use -`ThreadRange`. This takes two parameters (`min_threads` and `max_threads`) and -runs the benchmark once for values in the inclusive range. For example: - -```c++ -BENCHMARK(BM_MultiThreaded)->ThreadRange(1, 8); -``` - -will run `BM_MultiThreaded` with thread counts 1, 2, 4, and 8. - If the benchmarked code itself uses threads and you want to compare it to single-threaded code, you may want to use real-time ("wallclock") measurements for latency comparisons: @@ -887,7 +814,7 @@ BENCHMARK(BM_OpenMP)->Range(8, 8<<10); // Measure the user-visible time, the wall clock (literally, the time that // has passed on the clock on the wall), use it to decide for how long to -// run the benchmark loop. This will always be meaningful, and will match the +// run the benchmark loop. This will always be meaningful, an will match the // time spent by the main thread in single-threaded case, in general decreasing // with the number of internal threads doing the work. BENCHMARK(BM_OpenMP)->Range(8, 8<<10)->UseRealTime(); @@ -909,7 +836,7 @@ is measured. But sometimes, it is necessary to do some work inside of that loop, every iteration, but without counting that time to the benchmark time. That is possible, although it is not recommended, since it has high overhead. - +{% raw %} ```c++ static void BM_SetInsert_With_Timer_Control(benchmark::State& state) { std::set data; @@ -924,7 +851,7 @@ static void BM_SetInsert_With_Timer_Control(benchmark::State& state) { } BENCHMARK(BM_SetInsert_With_Timer_Control)->Ranges({{1<<10, 8<<10}, {128, 512}}); ``` - +{% endraw %} @@ -979,10 +906,6 @@ order to manually set the time unit, you can specify it manually: BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); ``` -Additionally the default time unit can be set globally with the -`--benchmark_time_unit={ns|us|ms|s}` command line argument. The argument only -affects benchmarks where the time unit is not set explicitly. - ## Preventing Optimization @@ -1035,8 +958,7 @@ static void BM_vector_push_back(benchmark::State& state) { for (auto _ : state) { std::vector v; v.reserve(1); - auto data = v.data(); // Allow v.data() to be clobbered. Pass as non-const - benchmark::DoNotOptimize(data); // lvalue to avoid undesired compiler optimizations + benchmark::DoNotOptimize(v.data()); // Allow v.data() to be clobbered. v.push_back(42); benchmark::ClobberMemory(); // Force 42 to be written to memory. } @@ -1115,25 +1037,10 @@ void BM_spin_empty(benchmark::State& state) { BENCHMARK(BM_spin_empty) ->ComputeStatistics("ratio", [](const std::vector& v) -> double { return std::begin(v) / std::end(v); - }, benchmark::StatisticUnit::kPercentage) + }, benchmark::StatisticUnit::Percentage) ->Arg(512); ``` - - -## Memory Usage - -It's often useful to also track memory usage for benchmarks, alongside CPU -performance. For this reason, benchmark offers the `RegisterMemoryManager` -method that allows a custom `MemoryManager` to be injected. - -If set, the `MemoryManager::Start` and `MemoryManager::Stop` methods will be -called at the start and end of benchmark runs to allow user code to fill out -a report on the number of allocations, bytes used, etc. - -This data will then be reported alongside other performance data, currently -only when using JSON output. - ## Using RegisterBenchmark(name, fn, args...) @@ -1170,7 +1077,7 @@ int main(int argc, char** argv) { When errors caused by external influences, such as file I/O and network communication, occur within a benchmark the -`State::SkipWithError(const std::string& msg)` function can be used to skip that run +`State::SkipWithError(const char* msg)` function can be used to skip that run of benchmark and report the error. Note that only future iterations of the `KeepRunning()` are skipped. For the ranged-for version of the benchmark loop Users must explicitly exit the loop, otherwise all iterations will be performed. @@ -1281,12 +1188,13 @@ the benchmark loop should be preferred. If you see this error: ``` -***WARNING*** CPU scaling is enabled, the benchmark real time measurements may -be noisy and will incur extra overhead. +***WARNING*** CPU scaling is enabled, the benchmark real time measurements may be noisy and will incur extra overhead. ``` -you might want to disable the CPU frequency scaling while running the -benchmark, as well as consider other ways to stabilize the performance of -your system while benchmarking. +you might want to disable the CPU frequency scaling while running the benchmark: -See [Reducing Variance](reducing_variance.md) for more information. +```bash +sudo cpupower frequency-set --governor performance +./mybench +sudo cpupower frequency-set --governor powersave +``` diff --git a/third-party/benchmark/include/benchmark/benchmark.h b/third-party/benchmark/include/benchmark/benchmark.h index 08cfe29da344..6287c0afbdcf 100644 --- a/third-party/benchmark/include/benchmark/benchmark.h +++ b/third-party/benchmark/include/benchmark/benchmark.h @@ -187,8 +187,6 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #include #include -#include "benchmark/export.h" - #if defined(BENCHMARK_HAS_CXX11) #include #include @@ -218,45 +216,37 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #define BENCHMARK_UNUSED #endif -// Used to annotate functions, methods and classes so they -// are not optimized by the compiler. Useful for tests -// where you expect loops to stay in place churning cycles -#if defined(__clang__) -#define BENCHMARK_DONT_OPTIMIZE __attribute__((optnone)) -#elif defined(__GNUC__) || defined(__GNUG__) -#define BENCHMARK_DONT_OPTIMIZE __attribute__((optimize(0))) -#else -// MSVC & Intel do not have a no-optimize attribute, only line pragmas -#define BENCHMARK_DONT_OPTIMIZE -#endif - #if defined(__GNUC__) || defined(__clang__) #define BENCHMARK_ALWAYS_INLINE __attribute__((always_inline)) +#define BENCHMARK_NOEXCEPT noexcept +#define BENCHMARK_NOEXCEPT_OP(x) noexcept(x) #elif defined(_MSC_VER) && !defined(__clang__) #define BENCHMARK_ALWAYS_INLINE __forceinline +#if _MSC_VER >= 1900 +#define BENCHMARK_NOEXCEPT noexcept +#define BENCHMARK_NOEXCEPT_OP(x) noexcept(x) +#else +#define BENCHMARK_NOEXCEPT +#define BENCHMARK_NOEXCEPT_OP(x) +#endif #define __func__ __FUNCTION__ #else #define BENCHMARK_ALWAYS_INLINE +#define BENCHMARK_NOEXCEPT +#define BENCHMARK_NOEXCEPT_OP(x) #endif #define BENCHMARK_INTERNAL_TOSTRING2(x) #x #define BENCHMARK_INTERNAL_TOSTRING(x) BENCHMARK_INTERNAL_TOSTRING2(x) // clang-format off -#if (defined(__GNUC__) && !defined(__NVCC__) && !defined(__NVCOMPILER)) || defined(__clang__) +#if defined(__GNUC__) || defined(__clang__) #define BENCHMARK_BUILTIN_EXPECT(x, y) __builtin_expect(x, y) #define BENCHMARK_DEPRECATED_MSG(msg) __attribute__((deprecated(msg))) #define BENCHMARK_DISABLE_DEPRECATED_WARNING \ _Pragma("GCC diagnostic push") \ _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") #define BENCHMARK_RESTORE_DEPRECATED_WARNING _Pragma("GCC diagnostic pop") -#elif defined(__NVCOMPILER) -#define BENCHMARK_BUILTIN_EXPECT(x, y) __builtin_expect(x, y) -#define BENCHMARK_DEPRECATED_MSG(msg) __attribute__((deprecated(msg))) -#define BENCHMARK_DISABLE_DEPRECATED_WARNING \ - _Pragma("diagnostic push") \ - _Pragma("diag_suppress deprecated_entity_with_custom_message") -#define BENCHMARK_RESTORE_DEPRECATED_WARNING _Pragma("diagnostic pop") #else #define BENCHMARK_BUILTIN_EXPECT(x, y) x #define BENCHMARK_DEPRECATED_MSG(msg) @@ -290,47 +280,18 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #define BENCHMARK_OVERRIDE #endif -#if defined(_MSC_VER) -#pragma warning(push) -// C4251: needs to have dll-interface to be used by clients of class -#pragma warning(disable : 4251) -#endif - namespace benchmark { class BenchmarkReporter; -// Default number of minimum benchmark running time in seconds. -const char kDefaultMinTimeStr[] = "0.5s"; - -// Returns the version of the library. -BENCHMARK_EXPORT std::string GetBenchmarkVersion(); - -BENCHMARK_EXPORT void PrintDefaultHelp(); - -BENCHMARK_EXPORT void Initialize(int* argc, char** argv, - void (*HelperPrinterf)() = PrintDefaultHelp); -BENCHMARK_EXPORT void Shutdown(); +void Initialize(int* argc, char** argv); +void Shutdown(); // Report to stdout all arguments in 'argv' as unrecognized except the first. // Returns true there is at least on unrecognized argument (i.e. 'argc' > 1). -BENCHMARK_EXPORT bool ReportUnrecognizedArguments(int argc, char** argv); +bool ReportUnrecognizedArguments(int argc, char** argv); // Returns the current value of --benchmark_filter. -BENCHMARK_EXPORT std::string GetBenchmarkFilter(); - -// Sets a new value to --benchmark_filter. (This will override this flag's -// current value). -// Should be called after `benchmark::Initialize()`, as -// `benchmark::Initialize()` will override the flag's value. -BENCHMARK_EXPORT void SetBenchmarkFilter(std::string value); - -// Returns the current value of --v (command line value for verbosity). -BENCHMARK_EXPORT int32_t GetBenchmarkVerbosity(); - -// Creates a default display reporter. Used by the library when no display -// reporter is provided, but also made available for external use in case a -// custom reporter should respect the `--benchmark_format` flag as a fallback -BENCHMARK_EXPORT BenchmarkReporter* CreateDefaultDisplayReporter(); +std::string GetBenchmarkFilter(); // Generate a list of benchmarks matching the specified --benchmark_filter flag // and if --benchmark_list_tests is specified return after printing the name @@ -344,33 +305,22 @@ BENCHMARK_EXPORT BenchmarkReporter* CreateDefaultDisplayReporter(); // The second and third overload use the specified 'display_reporter' and // 'file_reporter' respectively. 'file_reporter' will write to the file // specified -// by '--benchmark_out'. If '--benchmark_out' is not given the +// by '--benchmark_output'. If '--benchmark_output' is not given the // 'file_reporter' is ignored. // // RETURNS: The number of matching benchmarks. -BENCHMARK_EXPORT size_t RunSpecifiedBenchmarks(); -BENCHMARK_EXPORT size_t RunSpecifiedBenchmarks(std::string spec); - -BENCHMARK_EXPORT size_t -RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter); -BENCHMARK_EXPORT size_t -RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, std::string spec); +size_t RunSpecifiedBenchmarks(); +size_t RunSpecifiedBenchmarks(std::string spec); -BENCHMARK_EXPORT size_t RunSpecifiedBenchmarks( - BenchmarkReporter* display_reporter, BenchmarkReporter* file_reporter); -BENCHMARK_EXPORT size_t -RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, - BenchmarkReporter* file_reporter, std::string spec); +size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter); +size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, + std::string spec); -// TimeUnit is passed to a benchmark in order to specify the order of magnitude -// for the measured time. -enum TimeUnit { kNanosecond, kMicrosecond, kMillisecond, kSecond }; - -BENCHMARK_EXPORT TimeUnit GetDefaultTimeUnit(); - -// Sets the default time unit the benchmarks use -// Has to be called before the benchmark loop to take effect -BENCHMARK_EXPORT void SetDefaultTimeUnit(TimeUnit unit); +size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, + BenchmarkReporter* file_reporter); +size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, + BenchmarkReporter* file_reporter, + std::string spec); // If a MemoryManager is registered (via RegisterMemoryManager()), // it can be used to collect and report allocation metrics for a run of the @@ -408,16 +358,20 @@ class MemoryManager { virtual void Start() = 0; // Implement this to stop recording and fill out the given Result structure. - virtual void Stop(Result& result) = 0; + BENCHMARK_DEPRECATED_MSG("Use Stop(Result&) instead") + virtual void Stop(Result* result) = 0; + + // FIXME(vyng): Make this pure virtual once we've migrated current users. + BENCHMARK_DISABLE_DEPRECATED_WARNING + virtual void Stop(Result& result) { Stop(&result); } + BENCHMARK_RESTORE_DEPRECATED_WARNING }; // Register a MemoryManager instance that will be used to collect and report // allocation measurements for benchmark runs. -BENCHMARK_EXPORT void RegisterMemoryManager(MemoryManager* memory_manager); // Add a key-value pair to output as part of the context stanza in the report. -BENCHMARK_EXPORT void AddCustomContext(const std::string& key, const std::string& value); namespace internal { @@ -425,17 +379,14 @@ class Benchmark; class BenchmarkImp; class BenchmarkFamilies; -BENCHMARK_EXPORT std::map*& GetGlobalContext(); - -BENCHMARK_EXPORT void UseCharPointer(char const volatile*); // Take ownership of the pointer and register the benchmark. Return the // registered benchmark. -BENCHMARK_EXPORT Benchmark* RegisterBenchmarkInternal(Benchmark*); +Benchmark* RegisterBenchmarkInternal(Benchmark*); // Ensure that the standard streams are properly initialized in every TU. -BENCHMARK_EXPORT int InitializeStreams(); +int InitializeStreams(); BENCHMARK_UNUSED static int stream_init_anchor = InitializeStreams(); } // namespace internal @@ -458,11 +409,7 @@ inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { // intended to add little to no overhead. // See: https://youtu.be/nXaxk27zwlk?t=2441 #ifndef BENCHMARK_HAS_NO_INLINE_ASSEMBLY -#if !defined(__GNUC__) || defined(__llvm__) || defined(__INTEL_COMPILER) template -BENCHMARK_DEPRECATED_MSG( - "The const-ref version of this method can permit " - "undesired compiler optimizations in benchmarks") inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { asm volatile("" : : "r,m"(value) : "memory"); } @@ -476,98 +423,6 @@ inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp& value) { #endif } -#ifdef BENCHMARK_HAS_CXX11 -template -inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp&& value) { -#if defined(__clang__) - asm volatile("" : "+r,m"(value) : : "memory"); -#else - asm volatile("" : "+m,r"(value) : : "memory"); -#endif -} -#endif -#elif defined(BENCHMARK_HAS_CXX11) && (__GNUC__ >= 5) -// Workaround for a bug with full argument copy overhead with GCC. -// See: #1340 and https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105519 -template -BENCHMARK_DEPRECATED_MSG( - "The const-ref version of this method can permit " - "undesired compiler optimizations in benchmarks") -inline BENCHMARK_ALWAYS_INLINE - typename std::enable_if::value && - (sizeof(Tp) <= sizeof(Tp*))>::type - DoNotOptimize(Tp const& value) { - asm volatile("" : : "r,m"(value) : "memory"); -} - -template -BENCHMARK_DEPRECATED_MSG( - "The const-ref version of this method can permit " - "undesired compiler optimizations in benchmarks") -inline BENCHMARK_ALWAYS_INLINE - typename std::enable_if::value || - (sizeof(Tp) > sizeof(Tp*))>::type - DoNotOptimize(Tp const& value) { - asm volatile("" : : "m"(value) : "memory"); -} - -template -inline BENCHMARK_ALWAYS_INLINE - typename std::enable_if::value && - (sizeof(Tp) <= sizeof(Tp*))>::type - DoNotOptimize(Tp& value) { - asm volatile("" : "+m,r"(value) : : "memory"); -} - -template -inline BENCHMARK_ALWAYS_INLINE - typename std::enable_if::value || - (sizeof(Tp) > sizeof(Tp*))>::type - DoNotOptimize(Tp& value) { - asm volatile("" : "+m"(value) : : "memory"); -} - -template -inline BENCHMARK_ALWAYS_INLINE - typename std::enable_if::value && - (sizeof(Tp) <= sizeof(Tp*))>::type - DoNotOptimize(Tp&& value) { - asm volatile("" : "+m,r"(value) : : "memory"); -} - -template -inline BENCHMARK_ALWAYS_INLINE - typename std::enable_if::value || - (sizeof(Tp) > sizeof(Tp*))>::type - DoNotOptimize(Tp&& value) { - asm volatile("" : "+m"(value) : : "memory"); -} - -#else -// Fallback for GCC < 5. Can add some overhead because the compiler is forced -// to use memory operations instead of operations with registers. -// TODO: Remove if GCC < 5 will be unsupported. -template -BENCHMARK_DEPRECATED_MSG( - "The const-ref version of this method can permit " - "undesired compiler optimizations in benchmarks") -inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { - asm volatile("" : : "m"(value) : "memory"); -} - -template -inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp& value) { - asm volatile("" : "+m"(value) : : "memory"); -} - -#ifdef BENCHMARK_HAS_CXX11 -template -inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp&& value) { - asm volatile("" : "+m"(value) : : "memory"); -} -#endif -#endif - #ifndef BENCHMARK_HAS_CXX11 inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { asm volatile("" : : : "memory"); @@ -575,9 +430,6 @@ inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { #endif #elif defined(_MSC_VER) template -BENCHMARK_DEPRECATED_MSG( - "The const-ref version of this method can permit " - "undesired compiler optimizations in benchmarks") inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { internal::UseCharPointer(&reinterpret_cast(value)); _ReadWriteBarrier(); @@ -587,25 +439,10 @@ inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { _ReadWriteBarrier(); } #endif #else -#ifdef BENCHMARK_HAS_CXX11 -template -inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp&& value) { - internal::UseCharPointer(&reinterpret_cast(value)); -} -#else template -BENCHMARK_DEPRECATED_MSG( - "The const-ref version of this method can permit " - "undesired compiler optimizations in benchmarks") inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { internal::UseCharPointer(&reinterpret_cast(value)); } - -template -inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp& value) { - internal::UseCharPointer(&reinterpret_cast(value)); -} -#endif // FIXME Add ClobberMemory() for non-gnu and non-msvc compilers, before C++11. #endif @@ -669,21 +506,23 @@ Counter::Flags inline operator|(const Counter::Flags& LHS, // This is the container for the user-defined counters. typedef std::map UserCounters; +// TimeUnit is passed to a benchmark in order to specify the order of magnitude +// for the measured time. +enum TimeUnit { kNanosecond, kMicrosecond, kMillisecond, kSecond }; + // BigO is passed to a benchmark in order to specify the asymptotic // computational // complexity for the benchmark. In case oAuto is selected, complexity will be // calculated automatically to the best fit. enum BigO { oNone, o1, oN, oNSquared, oNCubed, oLogN, oNLogN, oAuto, oLambda }; -typedef int64_t ComplexityN; - -typedef int64_t IterationCount; +typedef uint64_t IterationCount; enum StatisticUnit { kTime, kPercentage }; // BigOFunc is passed to a benchmark in order to specify the asymptotic // computational complexity for the benchmark. -typedef double(BigOFunc)(ComplexityN); +typedef double(BigOFunc)(IterationCount); // StatisticsFunc is passed to a benchmark in order to compute some descriptive // statistics over all the measurements of some type @@ -725,21 +564,11 @@ enum AggregationReportMode ARM_FileReportAggregatesOnly | ARM_DisplayReportAggregatesOnly }; -enum Skipped -#if defined(BENCHMARK_HAS_CXX11) - : unsigned -#endif -{ - NotSkipped = 0, - SkippedWithMessage, - SkippedWithError -}; - } // namespace internal // State is passed to a running Benchmark and contains state for the // benchmark to use. -class BENCHMARK_EXPORT State { +class State { public: struct StateIterator; friend struct StateIterator; @@ -751,13 +580,13 @@ class BENCHMARK_EXPORT State { // have been called previously. // // NOTE: KeepRunning may not be used after calling either of these functions. - inline BENCHMARK_ALWAYS_INLINE StateIterator begin(); - inline BENCHMARK_ALWAYS_INLINE StateIterator end(); + BENCHMARK_ALWAYS_INLINE StateIterator begin(); + BENCHMARK_ALWAYS_INLINE StateIterator end(); // Returns true if the benchmark should continue through another iteration. // NOTE: A benchmark may not return from the test until KeepRunning() has // returned false. - inline bool KeepRunning(); + bool KeepRunning(); // Returns true iff the benchmark should run n more iterations. // REQUIRES: 'n' > 0. @@ -769,10 +598,10 @@ class BENCHMARK_EXPORT State { // while (state.KeepRunningBatch(1000)) { // // process 1000 elements // } - inline bool KeepRunningBatch(IterationCount n); + bool KeepRunningBatch(IterationCount n); - // REQUIRES: timer is running and 'SkipWithMessage(...)' or - // 'SkipWithError(...)' has not been called by the current thread. + // REQUIRES: timer is running and 'SkipWithError(...)' has not been called + // by the current thread. // Stop the benchmark timer. If not called, the timer will be // automatically stopped after the last iteration of the benchmark loop. // @@ -787,8 +616,8 @@ class BENCHMARK_EXPORT State { // within each benchmark iteration, if possible. void PauseTiming(); - // REQUIRES: timer is not running and 'SkipWithMessage(...)' or - // 'SkipWithError(...)' has not been called by the current thread. + // REQUIRES: timer is not running and 'SkipWithError(...)' has not been called + // by the current thread. // Start the benchmark timer. The timer is NOT running on entrance to the // benchmark function. It begins running after control flow enters the // benchmark loop. @@ -798,30 +627,8 @@ class BENCHMARK_EXPORT State { // within each benchmark iteration, if possible. void ResumeTiming(); - // REQUIRES: 'SkipWithMessage(...)' or 'SkipWithError(...)' has not been - // called previously by the current thread. - // Report the benchmark as resulting in being skipped with the specified - // 'msg'. - // After this call the user may explicitly 'return' from the benchmark. - // - // If the ranged-for style of benchmark loop is used, the user must explicitly - // break from the loop, otherwise all future iterations will be run. - // If the 'KeepRunning()' loop is used the current thread will automatically - // exit the loop at the end of the current iteration. - // - // For threaded benchmarks only the current thread stops executing and future - // calls to `KeepRunning()` will block until all threads have completed - // the `KeepRunning()` loop. If multiple threads report being skipped only the - // first skip message is used. - // - // NOTE: Calling 'SkipWithMessage(...)' does not cause the benchmark to exit - // the current scope immediately. If the function is called from within - // the 'KeepRunning()' loop the current iteration will finish. It is the users - // responsibility to exit the scope as needed. - void SkipWithMessage(const std::string& msg); - - // REQUIRES: 'SkipWithMessage(...)' or 'SkipWithError(...)' has not been - // called previously by the current thread. + // REQUIRES: 'SkipWithError(...)' has not been called previously by the + // current thread. // Report the benchmark as resulting in an error with the specified 'msg'. // After this call the user may explicitly 'return' from the benchmark. // @@ -839,13 +646,10 @@ class BENCHMARK_EXPORT State { // the current scope immediately. If the function is called from within // the 'KeepRunning()' loop the current iteration will finish. It is the users // responsibility to exit the scope as needed. - void SkipWithError(const std::string& msg); - - // Returns true if 'SkipWithMessage(...)' or 'SkipWithError(...)' was called. - bool skipped() const { return internal::NotSkipped != skipped_; } + void SkipWithError(const char* msg); // Returns true if an error has been reported with 'SkipWithError(...)'. - bool error_occurred() const { return internal::SkippedWithError == skipped_; } + bool error_occurred() const { return error_occurred_; } // REQUIRES: called exactly once per iteration of the benchmarking loop. // Set the manually measured time for this benchmark iteration, which @@ -880,12 +684,10 @@ class BENCHMARK_EXPORT State { // and complexity_n will // represent the length of N. BENCHMARK_ALWAYS_INLINE - void SetComplexityN(ComplexityN complexity_n) { - complexity_n_ = complexity_n; - } + void SetComplexityN(int64_t complexity_n) { complexity_n_ = complexity_n; } BENCHMARK_ALWAYS_INLINE - ComplexityN complexity_length_n() const { return complexity_n_; } + int64_t complexity_length_n() const { return complexity_n_; } // If this routine is called with items > 0, then an items/s // label is printed on the benchmark report line for the currently @@ -918,7 +720,11 @@ class BENCHMARK_EXPORT State { // BM_Compress 50 50 14115038 compress:27.3% // // REQUIRES: a benchmark has exited its benchmarking loop. - void SetLabel(const std::string& label); + void SetLabel(const char* label); + + void BENCHMARK_ALWAYS_INLINE SetLabel(const std::string& str) { + this->SetLabel(str.c_str()); + } // Range arguments for this run. CHECKs if the argument has been set. BENCHMARK_ALWAYS_INLINE @@ -949,9 +755,6 @@ class BENCHMARK_EXPORT State { return max_iterations - total_iterations_ + batch_leftover_; } - BENCHMARK_ALWAYS_INLINE - std::string name() const { return name_; } - private: // items we expect on the first cache line (ie 64 bytes of the struct) // When total_iterations_ is 0, KeepRunning() and friends will return false. @@ -969,30 +772,29 @@ class BENCHMARK_EXPORT State { private: bool started_; bool finished_; - internal::Skipped skipped_; + bool error_occurred_; // items we don't need on the first cache line std::vector range_; - ComplexityN complexity_n_; + int64_t complexity_n_; public: // Container for user-defined counters. UserCounters counters; private: - State(std::string name, IterationCount max_iters, - const std::vector& ranges, int thread_i, int n_threads, - internal::ThreadTimer* timer, internal::ThreadManager* manager, + State(IterationCount max_iters, const std::vector& ranges, + int thread_i, int n_threads, internal::ThreadTimer* timer, + internal::ThreadManager* manager, internal::PerfCountersMeasurement* perf_counters_measurement); void StartKeepRunning(); // Implementation of KeepRunning() and KeepRunningBatch(). // is_batch must be true unless n is 1. - inline bool KeepRunningInternal(IterationCount n, bool is_batch); + bool KeepRunningInternal(IterationCount n, bool is_batch); void FinishKeepRunning(); - const std::string name_; const int thread_index_; const int threads_; @@ -1024,7 +826,7 @@ inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningInternal(IterationCount n, } if (!started_) { StartKeepRunning(); - if (!skipped() && total_iterations_ >= n) { + if (!error_occurred_ && total_iterations_ >= n) { total_iterations_ -= n; return true; } @@ -1054,7 +856,7 @@ struct State::StateIterator { BENCHMARK_ALWAYS_INLINE explicit StateIterator(State* st) - : cached_(st->skipped() ? 0 : st->max_iterations), parent_(st) {} + : cached_(st->error_occurred_ ? 0 : st->max_iterations), parent_(st) {} public: BENCHMARK_ALWAYS_INLINE @@ -1097,7 +899,7 @@ typedef void(Function)(State&); // be called on this object to change the properties of the benchmark. // Each method returns "this" so that multiple method calls can // chained into one expression. -class BENCHMARK_EXPORT Benchmark { +class Benchmark { public: virtual ~Benchmark(); @@ -1169,7 +971,7 @@ class BENCHMARK_EXPORT Benchmark { // Have "setup" and/or "teardown" invoked once for every benchmark run. // If the benchmark is multi-threaded (will run in k threads concurrently), - // the setup callback will be be invoked exactly once (not k times) before + // the setup callback will be invoked exactly once (not k times) before // each run with k threads. Time allowing (e.g. for a short benchmark), there // may be multiple such runs per benchmark, each run with its own // "setup"/"teardown". @@ -1198,19 +1000,12 @@ class BENCHMARK_EXPORT Benchmark { // REQUIRES: `t > 0` and `Iterations` has not been called on this benchmark. Benchmark* MinTime(double t); - // Set the minimum amount of time to run the benchmark before taking runtimes - // of this benchmark into account. This - // option overrides the `benchmark_min_warmup_time` flag. - // REQUIRES: `t >= 0` and `Iterations` has not been called on this benchmark. - Benchmark* MinWarmUpTime(double t); - // Specify the amount of iterations that should be run by this benchmark. - // This option overrides the `benchmark_min_time` flag. // REQUIRES: 'n > 0' and `MinTime` has not been called on this benchmark. // // NOTE: This function should only be used when *exact* iteration control is // needed and never to control or limit how long a benchmark runs, where - // `--benchmark_min_time=s` or `MinTime(...)` should be used instead. + // `--benchmark_min_time=N` or `MinTime(...)` should be used instead. Benchmark* Iterations(IterationCount n); // Specify the amount of times to repeat this benchmark. This option overrides @@ -1230,7 +1025,7 @@ class BENCHMARK_EXPORT Benchmark { // By default, the CPU time is measured only for the main thread, which may // be unrepresentative if the benchmark uses threads internally. If called, // the total CPU time spent by all the threads will be measured instead. - // By default, only the main thread CPU time will be measured. + // By default, the only the main thread CPU time will be measured. Benchmark* MeasureProcessCPUTime(); // If a particular benchmark should use the Wall clock instead of the CPU time @@ -1295,16 +1090,12 @@ class BENCHMARK_EXPORT Benchmark { virtual void Run(State& state) = 0; - TimeUnit GetTimeUnit() const; - protected: - explicit Benchmark(const std::string& name); - void SetName(const std::string& name); + explicit Benchmark(const char* name); + Benchmark(Benchmark const&); + void SetName(const char* name); - public: - const char* GetName() const; int ArgsCnt() const; - const char* GetArgName(int arg) const; private: friend class BenchmarkFamilies; @@ -1314,13 +1105,9 @@ class BENCHMARK_EXPORT Benchmark { AggregationReportMode aggregation_report_mode_; std::vector arg_names_; // Args for all benchmark runs std::vector > args_; // Args for all benchmark runs - TimeUnit time_unit_; - bool use_default_time_unit_; - int range_multiplier_; double min_time_; - double min_warmup_time_; IterationCount iterations_; int repetitions_; bool measure_process_cpu_time_; @@ -1335,17 +1122,7 @@ class BENCHMARK_EXPORT Benchmark { callback_function setup_; callback_function teardown_; - Benchmark(Benchmark const&) -#if defined(BENCHMARK_HAS_CXX11) - = delete -#endif - ; - - Benchmark& operator=(Benchmark const&) -#if defined(BENCHMARK_HAS_CXX11) - = delete -#endif - ; + Benchmark& operator=(Benchmark const&); }; } // namespace internal @@ -1354,27 +1131,27 @@ class BENCHMARK_EXPORT Benchmark { // the specified functor 'fn'. // // RETURNS: A pointer to the registered benchmark. -internal::Benchmark* RegisterBenchmark(const std::string& name, +internal::Benchmark* RegisterBenchmark(const char* name, internal::Function* fn); #if defined(BENCHMARK_HAS_CXX11) template -internal::Benchmark* RegisterBenchmark(const std::string& name, Lambda&& fn); +internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn); #endif // Remove all registered benchmarks. All pointers to previously registered // benchmarks are invalidated. -BENCHMARK_EXPORT void ClearRegisteredBenchmarks(); +void ClearRegisteredBenchmarks(); namespace internal { // The class used to hold all Benchmarks created from static function. // (ie those created using the BENCHMARK(...) macros. -class BENCHMARK_EXPORT FunctionBenchmark : public Benchmark { +class FunctionBenchmark : public Benchmark { public: - FunctionBenchmark(const std::string& name, Function* func) + FunctionBenchmark(const char* name, Function* func) : Benchmark(name), func_(func) {} - void Run(State& st) BENCHMARK_OVERRIDE; + virtual void Run(State& st) BENCHMARK_OVERRIDE; private: Function* func_; @@ -1384,38 +1161,35 @@ class BENCHMARK_EXPORT FunctionBenchmark : public Benchmark { template class LambdaBenchmark : public Benchmark { public: - void Run(State& st) BENCHMARK_OVERRIDE { lambda_(st); } + virtual void Run(State& st) BENCHMARK_OVERRIDE { lambda_(st); } private: template - LambdaBenchmark(const std::string& name, OLambda&& lam) + LambdaBenchmark(const char* name, OLambda&& lam) : Benchmark(name), lambda_(std::forward(lam)) {} LambdaBenchmark(LambdaBenchmark const&) = delete; template // NOLINTNEXTLINE(readability-redundant-declaration) - friend Benchmark* ::benchmark::RegisterBenchmark(const std::string&, Lam&&); + friend Benchmark* ::benchmark::RegisterBenchmark(const char*, Lam&&); Lambda lambda_; }; #endif + } // namespace internal -inline internal::Benchmark* RegisterBenchmark(const std::string& name, +inline internal::Benchmark* RegisterBenchmark(const char* name, internal::Function* fn) { - // FIXME: this should be a `std::make_unique<>()` but we don't have C++14. - // codechecker_intentional [cplusplus.NewDeleteLeaks] return internal::RegisterBenchmarkInternal( ::new internal::FunctionBenchmark(name, fn)); } #ifdef BENCHMARK_HAS_CXX11 template -internal::Benchmark* RegisterBenchmark(const std::string& name, Lambda&& fn) { +internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn) { using BenchType = internal::LambdaBenchmark::type>; - // FIXME: this should be a `std::make_unique<>()` but we don't have C++14. - // codechecker_intentional [cplusplus.NewDeleteLeaks] return internal::RegisterBenchmarkInternal( ::new BenchType(name, std::forward(fn))); } @@ -1424,7 +1198,7 @@ internal::Benchmark* RegisterBenchmark(const std::string& name, Lambda&& fn) { #if defined(BENCHMARK_HAS_CXX11) && \ (!defined(BENCHMARK_GCC_VERSION) || BENCHMARK_GCC_VERSION >= 409) template -internal::Benchmark* RegisterBenchmark(const std::string& name, Lambda&& fn, +internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn, Args&&... args) { return benchmark::RegisterBenchmark( name, [=](benchmark::State& st) { fn(st, args...); }); @@ -1438,7 +1212,7 @@ class Fixture : public internal::Benchmark { public: Fixture() : internal::Benchmark("") {} - void Run(State& st) BENCHMARK_OVERRIDE { + virtual void Run(State& st) BENCHMARK_OVERRIDE { this->SetUp(st); this->BenchmarkCase(st); this->TearDown(st); @@ -1454,6 +1228,7 @@ class Fixture : public internal::Benchmark { protected: virtual void BenchmarkCase(State&) = 0; }; + } // namespace benchmark // ------------------------------------------------------ @@ -1493,7 +1268,7 @@ class Fixture : public internal::Benchmark { BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ (::benchmark::internal::RegisterBenchmarkInternal( \ new ::benchmark::internal::FunctionBenchmark(#__VA_ARGS__, \ - __VA_ARGS__))) + &__VA_ARGS__))) #else #define BENCHMARK(n) \ BENCHMARK_PRIVATE_DECLARE(n) = \ @@ -1523,7 +1298,7 @@ class Fixture : public internal::Benchmark { // /* Registers a benchmark named "BM_takes_args/int_string_test` */ // BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); #define BENCHMARK_CAPTURE(func, test_case_name, ...) \ - BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ + BENCHMARK_PRIVATE_DECLARE(func) = \ (::benchmark::internal::RegisterBenchmarkInternal( \ new ::benchmark::internal::FunctionBenchmark( \ #func "/" #test_case_name, \ @@ -1560,62 +1335,37 @@ class Fixture : public internal::Benchmark { #define BENCHMARK_TEMPLATE(n, a) BENCHMARK_TEMPLATE1(n, a) #endif -#ifdef BENCHMARK_HAS_CXX11 -// This will register a benchmark for a templatized function, -// with the additional arguments specified by `...`. -// -// For example: -// -// template ` -// void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) { -// [...] -//} -// /* Registers a benchmark named "BM_takes_args/int_string_test` */ -// BENCHMARK_TEMPLATE1_CAPTURE(BM_takes_args, void, int_string_test, 42, -// std::string("abc")); -#define BENCHMARK_TEMPLATE1_CAPTURE(func, a, test_case_name, ...) \ - BENCHMARK_CAPTURE(func, test_case_name, __VA_ARGS__) - -#define BENCHMARK_TEMPLATE2_CAPTURE(func, a, b, test_case_name, ...) \ - BENCHMARK_PRIVATE_DECLARE(func) = \ - (::benchmark::internal::RegisterBenchmarkInternal( \ - new ::benchmark::internal::FunctionBenchmark( \ - #func "<" #a "," #b ">" \ - "/" #test_case_name, \ - [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) -#endif // BENCHMARK_HAS_CXX11 - -#define BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ - class BaseClass##_##Method##_Benchmark : public BaseClass { \ - public: \ - BaseClass##_##Method##_Benchmark() { \ - this->SetName(#BaseClass "/" #Method); \ - } \ - \ - protected: \ - void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ +#define BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ + class BaseClass##_##Method##_Benchmark : public BaseClass { \ + public: \ + BaseClass##_##Method##_Benchmark() { \ + this->SetName(#BaseClass "/" #Method); \ + } \ + \ + protected: \ + virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ }; -#define BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ - class BaseClass##_##Method##_Benchmark : public BaseClass { \ - public: \ - BaseClass##_##Method##_Benchmark() { \ - this->SetName(#BaseClass "<" #a ">/" #Method); \ - } \ - \ - protected: \ - void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ +#define BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ + class BaseClass##_##Method##_Benchmark : public BaseClass { \ + public: \ + BaseClass##_##Method##_Benchmark() { \ + this->SetName(#BaseClass "<" #a ">/" #Method); \ + } \ + \ + protected: \ + virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ }; -#define BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ - class BaseClass##_##Method##_Benchmark : public BaseClass { \ - public: \ - BaseClass##_##Method##_Benchmark() { \ - this->SetName(#BaseClass "<" #a "," #b ">/" #Method); \ - } \ - \ - protected: \ - void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ +#define BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ + class BaseClass##_##Method##_Benchmark : public BaseClass { \ + public: \ + BaseClass##_##Method##_Benchmark() { \ + this->SetName(#BaseClass "<" #a "," #b ">/" #Method); \ + } \ + \ + protected: \ + virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ }; #ifdef BENCHMARK_HAS_CXX11 @@ -1627,7 +1377,7 @@ class Fixture : public internal::Benchmark { } \ \ protected: \ - void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ + virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ }; #else #define BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(n, a) \ @@ -1689,15 +1439,8 @@ class Fixture : public internal::Benchmark { #endif // Helper macro to create a main routine in a test that runs the benchmarks -// Note the workaround for Hexagon simulator passing argc != 0, argv = NULL. #define BENCHMARK_MAIN() \ int main(int argc, char** argv) { \ - char arg0_default[] = "benchmark"; \ - char* args_default = arg0_default; \ - if (!argv) { \ - argc = 1; \ - argv = &args_default; \ - } \ ::benchmark::Initialize(&argc, argv); \ if (::benchmark::ReportUnrecognizedArguments(argc, argv)) return 1; \ ::benchmark::RunSpecifiedBenchmarks(); \ @@ -1711,7 +1454,7 @@ class Fixture : public internal::Benchmark { namespace benchmark { -struct BENCHMARK_EXPORT CPUInfo { +struct CPUInfo { struct CacheInfo { std::string type; int level; @@ -1735,7 +1478,7 @@ struct BENCHMARK_EXPORT CPUInfo { }; // Adding Struct for System Information -struct BENCHMARK_EXPORT SystemInfo { +struct SystemInfo { std::string name; static const SystemInfo& Get(); @@ -1747,11 +1490,10 @@ struct BENCHMARK_EXPORT SystemInfo { // BenchmarkName contains the components of the Benchmark's name // which allows individual fields to be modified or cleared before // building the final name using 'str()'. -struct BENCHMARK_EXPORT BenchmarkName { +struct BenchmarkName { std::string function_name; std::string args; std::string min_time; - std::string min_warmup_time; std::string iterations; std::string repetitions; std::string time_type; @@ -1767,7 +1509,7 @@ struct BENCHMARK_EXPORT BenchmarkName { // can control the destination of the reports by calling // RunSpecifiedBenchmarks and passing it a custom reporter object. // The reporter object must implement the following interface. -class BENCHMARK_EXPORT BenchmarkReporter { +class BenchmarkReporter { public: struct Context { CPUInfo const& cpu_info; @@ -1778,21 +1520,20 @@ class BENCHMARK_EXPORT BenchmarkReporter { Context(); }; - struct BENCHMARK_EXPORT Run { + struct Run { static const int64_t no_repetition_index = -1; enum RunType { RT_Iteration, RT_Aggregate }; Run() : run_type(RT_Iteration), aggregate_unit(kTime), - skipped(internal::NotSkipped), + error_occurred(false), iterations(1), threads(1), - time_unit(GetDefaultTimeUnit()), + time_unit(kNanosecond), real_accumulated_time(0), cpu_accumulated_time(0), max_heapbytes_used(0), - use_real_time_for_initial_big_o(false), complexity(oNone), complexity_lambda(), complexity_n(0), @@ -1809,8 +1550,8 @@ class BENCHMARK_EXPORT BenchmarkReporter { std::string aggregate_name; StatisticUnit aggregate_unit; std::string report_label; // Empty if not set by benchmark. - internal::Skipped skipped; - std::string skip_message; + bool error_occurred; + std::string error_message; IterationCount iterations; int64_t threads; @@ -1835,14 +1576,10 @@ class BENCHMARK_EXPORT BenchmarkReporter { // This is set to 0.0 if memory tracing is not enabled. double max_heapbytes_used; - // By default Big-O is computed for CPU time, but that is not what you want - // to happen when manual time was requested, which is stored as real time. - bool use_real_time_for_initial_big_o; - // Keep track of arguments to compute asymptotic complexity BigO complexity; BigOFunc* complexity_lambda; - ComplexityN complexity_n; + int64_t complexity_n; // what statistics to compute from the measurements const std::vector* statistics; @@ -1883,12 +1620,6 @@ class BENCHMARK_EXPORT BenchmarkReporter { // to skip runs based on the context information. virtual bool ReportContext(const Context& context) = 0; - // Called once for each group of benchmark runs, gives information about - // the configurations of the runs. - virtual void ReportRunsConfig(double /*min_time*/, - bool /*has_explicit_iters*/, - IterationCount /*iters*/) {} - // Called once for each group of benchmark runs, gives information about // cpu-time and heap memory usage during the benchmark run. If the group // of runs contained more than two entries then 'report' contains additional @@ -1934,7 +1665,7 @@ class BENCHMARK_EXPORT BenchmarkReporter { // Simple reporter that outputs benchmark data to the console. This is the // default reporter used by RunSpecifiedBenchmarks(). -class BENCHMARK_EXPORT ConsoleReporter : public BenchmarkReporter { +class ConsoleReporter : public BenchmarkReporter { public: enum OutputOptions { OO_None = 0, @@ -1946,8 +1677,8 @@ class BENCHMARK_EXPORT ConsoleReporter : public BenchmarkReporter { explicit ConsoleReporter(OutputOptions opts_ = OO_Defaults) : output_options_(opts_), name_field_width_(0), printed_header_(false) {} - bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; - void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; + virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; + virtual void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; protected: virtual void PrintRunData(const Run& report); @@ -1959,12 +1690,12 @@ class BENCHMARK_EXPORT ConsoleReporter : public BenchmarkReporter { bool printed_header_; }; -class BENCHMARK_EXPORT JSONReporter : public BenchmarkReporter { +class JSONReporter : public BenchmarkReporter { public: JSONReporter() : first_report_(true) {} - bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; - void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; - void Finalize() BENCHMARK_OVERRIDE; + virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; + virtual void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; + virtual void Finalize() BENCHMARK_OVERRIDE; private: void PrintRunData(const Run& report); @@ -1972,13 +1703,13 @@ class BENCHMARK_EXPORT JSONReporter : public BenchmarkReporter { bool first_report_; }; -class BENCHMARK_EXPORT BENCHMARK_DEPRECATED_MSG( +class BENCHMARK_DEPRECATED_MSG( "The CSV Reporter will be removed in a future release") CSVReporter : public BenchmarkReporter { public: CSVReporter() : printed_header_(false) {} - bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; - void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; + virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; + virtual void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; private: void PrintRunData(const Run& report); @@ -2017,24 +1748,18 @@ inline double GetTimeUnitMultiplier(TimeUnit unit) { // Creates a list of integer values for the given range and multiplier. // This can be used together with ArgsProduct() to allow multiple ranges -// with different multipliers. +// with different multiplers. // Example: // ArgsProduct({ // CreateRange(0, 1024, /*multi=*/32), // CreateRange(0, 100, /*multi=*/4), // CreateDenseRange(0, 4, /*step=*/1), // }); -BENCHMARK_EXPORT std::vector CreateRange(int64_t lo, int64_t hi, int multi); // Creates a list of integer values for the given range and step. -BENCHMARK_EXPORT std::vector CreateDenseRange(int64_t start, int64_t limit, int step); } // namespace benchmark -#if defined(_MSC_VER) -#pragma warning(pop) -#endif - #endif // BENCHMARK_BENCHMARK_H_ diff --git a/third-party/benchmark/requirements.txt b/third-party/benchmark/requirements.txt new file mode 100644 index 000000000000..e451894e2356 --- /dev/null +++ b/third-party/benchmark/requirements.txt @@ -0,0 +1,3 @@ +numpy == 1.19.4 +scipy == 1.5.4 +pandas == 1.1.5 diff --git a/third-party/benchmark/setup.py b/third-party/benchmark/setup.py index cb20042da512..83069e566839 100644 --- a/third-party/benchmark/setup.py +++ b/third-party/benchmark/setup.py @@ -1,50 +1,56 @@ -import contextlib import os +import posixpath import platform +import re import shutil -import sysconfig -from pathlib import Path -from typing import Generator +import sys +from distutils import sysconfig import setuptools from setuptools.command import build_ext -PYTHON_INCLUDE_PATH_PLACEHOLDER = "" -IS_WINDOWS = platform.system() == "Windows" -IS_MAC = platform.system() == "Darwin" +HERE = os.path.dirname(os.path.abspath(__file__)) -@contextlib.contextmanager -def temp_fill_include_path(fp: str) -> Generator[None, None, None]: - """Temporarily set the Python include path in a file.""" - with open(fp, "r+") as f: +IS_WINDOWS = sys.platform.startswith("win") + + +def _get_version(): + """Parse the version string from __init__.py.""" + with open( + os.path.join(HERE, "bindings", "python", "google_benchmark", "__init__.py") + ) as init_file: try: - content = f.read() - replaced = content.replace( - PYTHON_INCLUDE_PATH_PLACEHOLDER, - Path(sysconfig.get_paths()["include"]).as_posix(), + version_line = next( + line for line in init_file if line.startswith("__version__") ) - f.seek(0) - f.write(replaced) - f.truncate() - yield - finally: - # revert to the original content after exit - f.seek(0) - f.write(content) - f.truncate() + except StopIteration: + raise ValueError("__version__ not defined in __init__.py") + else: + namespace = {} + exec(version_line, namespace) # pylint: disable=exec-used + return namespace["__version__"] + + +def _parse_requirements(path): + with open(os.path.join(HERE, path)) as requirements: + return [ + line.rstrip() + for line in requirements + if not (line.isspace() or line.startswith("#")) + ] class BazelExtension(setuptools.Extension): """A C/C++ extension that is defined as a Bazel BUILD target.""" - def __init__(self, name: str, bazel_target: str): - super().__init__(name=name, sources=[]) - + def __init__(self, name, bazel_target): self.bazel_target = bazel_target - stripped_target = bazel_target.split("//")[-1] - self.relpath, self.target_name = stripped_target.split(":") + self.relpath, self.target_name = posixpath.relpath(bazel_target, "//").split( + ":" + ) + setuptools.Extension.__init__(self, name, sources=[]) class BuildBazelExtension(build_ext.build_ext): @@ -53,71 +59,88 @@ class BuildBazelExtension(build_ext.build_ext): def run(self): for ext in self.extensions: self.bazel_build(ext) - super().run() - # explicitly call `bazel shutdown` for graceful exit - self.spawn(["bazel", "shutdown"]) - - def copy_extensions_to_source(self): - """ - Copy generated extensions into the source tree. - This is done in the ``bazel_build`` method, so it's not necessary to - do again in the `build_ext` base class. - """ - pass - - def bazel_build(self, ext: BazelExtension) -> None: + build_ext.build_ext.run(self) + + def bazel_build(self, ext): """Runs the bazel build to create the package.""" - with temp_fill_include_path("WORKSPACE"): - temp_path = Path(self.build_temp) - - bazel_argv = [ - "bazel", - "build", - ext.bazel_target, - "--enable_bzlmod=false", - f"--symlink_prefix={temp_path / 'bazel-'}", - f"--compilation_mode={'dbg' if self.debug else 'opt'}", - # C++17 is required by nanobind - f"--cxxopt={'/std:c++17' if IS_WINDOWS else '-std=c++17'}", - ] - - if IS_WINDOWS: - # Link with python*.lib. - for library_dir in self.library_dirs: - bazel_argv.append("--linkopt=/LIBPATH:" + library_dir) - elif IS_MAC: - if platform.machine() == "x86_64": - # C++17 needs macOS 10.14 at minimum - bazel_argv.append("--macos_minimum_os=10.14") - - # cross-compilation for Mac ARM64 on GitHub Mac x86 runners. - # ARCHFLAGS is set by cibuildwheel before macOS wheel builds. - archflags = os.getenv("ARCHFLAGS", "") - if "arm64" in archflags: - bazel_argv.append("--cpu=darwin_arm64") - bazel_argv.append("--macos_cpus=arm64") - - elif platform.machine() == "arm64": - bazel_argv.append("--macos_minimum_os=11.0") - - self.spawn(bazel_argv) - - shared_lib_suffix = ".dll" if IS_WINDOWS else ".so" - ext_name = ext.target_name + shared_lib_suffix - ext_bazel_bin_path = ( - temp_path / "bazel-bin" / ext.relpath / ext_name + with open("WORKSPACE", "r") as workspace: + workspace_contents = workspace.read() + + with open("WORKSPACE", "w") as workspace: + workspace.write( + re.sub( + r'(?<=path = ").*(?=", # May be overwritten by setup\.py\.)', + sysconfig.get_python_inc().replace(os.path.sep, posixpath.sep), + workspace_contents, + ) ) - ext_dest_path = Path(self.get_ext_fullpath(ext.name)) - shutil.copyfile(ext_bazel_bin_path, ext_dest_path) + if not os.path.exists(self.build_temp): + os.makedirs(self.build_temp) + + bazel_argv = [ + "bazel", + "build", + ext.bazel_target, + "--symlink_prefix=" + os.path.join(self.build_temp, "bazel-"), + "--compilation_mode=" + ("dbg" if self.debug else "opt"), + ] + + if IS_WINDOWS: + # Link with python*.lib. + for library_dir in self.library_dirs: + bazel_argv.append("--linkopt=/LIBPATH:" + library_dir) + elif sys.platform == "darwin" and platform.machine() == "x86_64": + bazel_argv.append("--macos_minimum_os=10.9") + + self.spawn(bazel_argv) + + shared_lib_suffix = ".dll" if IS_WINDOWS else ".so" + ext_bazel_bin_path = os.path.join( + self.build_temp, + "bazel-bin", + ext.relpath, + ext.target_name + shared_lib_suffix, + ) + + ext_dest_path = self.get_ext_fullpath(ext.name) + ext_dest_dir = os.path.dirname(ext_dest_path) + if not os.path.exists(ext_dest_dir): + os.makedirs(ext_dest_dir) + shutil.copyfile(ext_bazel_bin_path, ext_dest_path) setuptools.setup( + name="google_benchmark", + version=_get_version(), + url="https://github.com/google/benchmark", + description="A library to benchmark code snippets.", + author="Google", + author_email="benchmark-py@google.com", + # Contained modules and scripts. + package_dir={"": "bindings/python"}, + packages=setuptools.find_packages("bindings/python"), + install_requires=_parse_requirements("bindings/python/requirements.txt"), cmdclass=dict(build_ext=BuildBazelExtension), ext_modules=[ BazelExtension( - name="google_benchmark._benchmark", - bazel_target="//bindings/python/google_benchmark:_benchmark", + "google_benchmark._benchmark", + "//bindings/python/google_benchmark:_benchmark", ) ], + zip_safe=False, + # PyPI package information. + classifiers=[ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Topic :: Software Development :: Testing", + "Topic :: System :: Benchmark", + ], + license="Apache 2.0", + keywords="benchmark", ) diff --git a/third-party/benchmark/src/CMakeLists.txt b/third-party/benchmark/src/CMakeLists.txt index 943594b70bcd..e814a4e00f7c 100644 --- a/third-party/benchmark/src/CMakeLists.txt +++ b/third-party/benchmark/src/CMakeLists.txt @@ -25,25 +25,12 @@ set_target_properties(benchmark PROPERTIES SOVERSION ${GENERIC_LIB_SOVERSION} ) target_include_directories(benchmark PUBLIC - $ -) - -set_property( - SOURCE benchmark.cc - APPEND - PROPERTY COMPILE_DEFINITIONS - BENCHMARK_VERSION="${VERSION}" -) + $) # libpfm, if available -if (PFM_FOUND) - target_link_libraries(benchmark PRIVATE PFM::libpfm) - target_compile_definitions(benchmark PRIVATE -DHAVE_LIBPFM) -endif() - -# pthread affinity, if available -if(HAVE_PTHREAD_AFFINITY) - target_compile_definitions(benchmark PRIVATE -DBENCHMARK_HAS_PTHREAD_AFFINITY) +if (HAVE_LIBPFM) + target_link_libraries(benchmark PRIVATE pfm) + add_definitions(-DHAVE_LIBPFM) endif() # Link threads. @@ -66,10 +53,6 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "SunOS") target_link_libraries(benchmark PRIVATE kstat) endif() -if (NOT BUILD_SHARED_LIBS) - target_compile_definitions(benchmark PUBLIC -DBENCHMARK_STATIC_DEFINE) -endif() - # Benchmark main library add_library(benchmark_main "benchmark_main.cc") add_library(benchmark::benchmark_main ALIAS benchmark_main) @@ -77,10 +60,10 @@ set_target_properties(benchmark_main PROPERTIES OUTPUT_NAME "benchmark_main" VERSION ${GENERIC_LIB_VERSION} SOVERSION ${GENERIC_LIB_SOVERSION} - DEFINE_SYMBOL benchmark_EXPORTS ) target_link_libraries(benchmark_main PUBLIC benchmark::benchmark) + set(generated_dir "${PROJECT_BINARY_DIR}") set(version_config "${generated_dir}/${PROJECT_NAME}ConfigVersion.cmake") @@ -124,7 +107,6 @@ if (BENCHMARK_ENABLE_INSTALL) install( DIRECTORY "${PROJECT_SOURCE_DIR}/include/benchmark" - "${PROJECT_BINARY_DIR}/include/benchmark" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} FILES_MATCHING PATTERN "*.*h") diff --git a/third-party/benchmark/src/benchmark.cc b/third-party/benchmark/src/benchmark.cc index 495944db29ff..473151136202 100644 --- a/third-party/benchmark/src/benchmark.cc +++ b/third-party/benchmark/src/benchmark.cc @@ -19,7 +19,7 @@ #include "internal_macros.h" #ifndef BENCHMARK_OS_WINDOWS -#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) +#ifndef BENCHMARK_OS_FUCHSIA #include #endif #include @@ -65,28 +65,12 @@ BM_DEFINE_bool(benchmark_list_tests, false); // linked into the binary are run. BM_DEFINE_string(benchmark_filter, ""); -// Specification of how long to run the benchmark. -// -// It can be either an exact number of iterations (specified as `x`), -// or a minimum number of seconds (specified as `s`). If the latter -// format (ie., min seconds) is used, the system may run the benchmark longer -// until the results are considered significant. -// -// For backward compatibility, the `s` suffix may be omitted, in which case, -// the specified number is interpreted as the number of seconds. -// -// For cpu-time based tests, this is the lower bound +// Minimum number of seconds we should run benchmark before results are +// considered significant. For cpu-time based tests, this is the lower bound // on the total cpu time used by all threads that make up the test. For // real-time based tests, this is the lower bound on the elapsed time of the // benchmark execution, regardless of number of threads. -BM_DEFINE_string(benchmark_min_time, kDefaultMinTimeStr); - -// Minimum number of seconds a benchmark should be run before results should be -// taken into account. This e.g can be necessary for benchmarks of code which -// needs to fill some form of cache before performance is of interest. -// Note: results gathered within this period are discarded and not used for -// reported result. -BM_DEFINE_double(benchmark_min_warmup_time, 0.0); +BM_DEFINE_double(benchmark_min_time, 0.5); // The number of runs of each benchmark. If greater than 1, the mean and // standard deviation of the runs will be reported. @@ -137,10 +121,6 @@ BM_DEFINE_string(benchmark_perf_counters, ""); // pairs. Kept internal as it's only used for parsing from env/command line. BM_DEFINE_kvpairs(benchmark_context, {}); -// Set the default time unit to use for reports -// Valid values are 'ns', 'us', 'ms' or 's' -BM_DEFINE_string(benchmark_time_unit, ""); - // The level of verbose logging to output BM_DEFINE_int32(v, 0); @@ -148,28 +128,23 @@ namespace internal { std::map* global_context = nullptr; -BENCHMARK_EXPORT std::map*& GetGlobalContext() { - return global_context; -} - // FIXME: wouldn't LTO mess this up? void UseCharPointer(char const volatile*) {} } // namespace internal -State::State(std::string name, IterationCount max_iters, - const std::vector& ranges, int thread_i, int n_threads, - internal::ThreadTimer* timer, internal::ThreadManager* manager, +State::State(IterationCount max_iters, const std::vector& ranges, + int thread_i, int n_threads, internal::ThreadTimer* timer, + internal::ThreadManager* manager, internal::PerfCountersMeasurement* perf_counters_measurement) : total_iterations_(0), batch_leftover_(0), max_iterations(max_iters), started_(false), finished_(false), - skipped_(internal::NotSkipped), + error_occurred_(false), range_(ranges), complexity_n_(0), - name_(std::move(name)), thread_index_(thread_i), threads_(n_threads), timer_(timer), @@ -179,17 +154,6 @@ State::State(std::string name, IterationCount max_iters, BM_CHECK_LT(thread_index_, threads_) << "thread_index must be less than threads"; - // Add counters with correct flag now. If added with `counters[name]` in - // `PauseTiming`, a new `Counter` will be inserted the first time, which - // won't have the flag. Inserting them now also reduces the allocations - // during the benchmark. - if (perf_counters_measurement_) { - for (const std::string& counter_name : - perf_counters_measurement_->names()) { - counters[counter_name] = Counter(0.0, Counter::kAvgIterations); - } - } - // Note: The use of offsetof below is technically undefined until C++17 // because State is not a standard layout type. However, all compilers // currently provide well-defined behavior as an extension (which is @@ -205,19 +169,12 @@ State::State(std::string name, IterationCount max_iters, #elif defined(__clang__) #pragma clang diagnostic push #pragma clang diagnostic ignored "-Winvalid-offsetof" -#endif -#if defined(__NVCC__) -#pragma nv_diagnostic push -#pragma nv_diag_suppress 1427 -#endif -#if defined(__NVCOMPILER) -#pragma diagnostic push -#pragma diag_suppress offset_in_non_POD_nonstandard #endif // Offset tests to ensure commonly accessed data is on the first cache line. const int cache_line_size = 64; - static_assert( - offsetof(State, skipped_) <= (cache_line_size - sizeof(skipped_)), ""); + static_assert(offsetof(State, error_occurred_) <= + (cache_line_size - sizeof(error_occurred_)), + ""); #if defined(__INTEL_COMPILER) #pragma warning pop #elif defined(__GNUC__) @@ -225,61 +182,39 @@ State::State(std::string name, IterationCount max_iters, #elif defined(__clang__) #pragma clang diagnostic pop #endif -#if defined(__NVCC__) -#pragma nv_diagnostic pop -#endif -#if defined(__NVCOMPILER) -#pragma diagnostic pop -#endif } void State::PauseTiming() { // Add in time accumulated so far - BM_CHECK(started_ && !finished_ && !skipped()); + BM_CHECK(started_ && !finished_ && !error_occurred_); timer_->StopTimer(); if (perf_counters_measurement_) { - std::vector> measurements; - if (!perf_counters_measurement_->Stop(measurements)) { - BM_CHECK(false) << "Perf counters read the value failed."; - } + auto measurements = perf_counters_measurement_->StopAndGetMeasurements(); for (const auto& name_and_measurement : measurements) { - const std::string& name = name_and_measurement.first; - const double measurement = name_and_measurement.second; - // Counter was inserted with `kAvgIterations` flag by the constructor. - assert(counters.find(name) != counters.end()); - counters[name].value += measurement; + auto name = name_and_measurement.first; + auto measurement = name_and_measurement.second; + BM_CHECK_EQ(counters[name], 0.0); + counters[name] = Counter(measurement, Counter::kAvgIterations); } } } void State::ResumeTiming() { - BM_CHECK(started_ && !finished_ && !skipped()); + BM_CHECK(started_ && !finished_ && !error_occurred_); timer_->StartTimer(); if (perf_counters_measurement_) { perf_counters_measurement_->Start(); } } -void State::SkipWithMessage(const std::string& msg) { - skipped_ = internal::SkippedWithMessage; - { - MutexLock l(manager_->GetBenchmarkMutex()); - if (internal::NotSkipped == manager_->results.skipped_) { - manager_->results.skip_message_ = msg; - manager_->results.skipped_ = skipped_; - } - } - total_iterations_ = 0; - if (timer_->running()) timer_->StopTimer(); -} - -void State::SkipWithError(const std::string& msg) { - skipped_ = internal::SkippedWithError; +void State::SkipWithError(const char* msg) { + BM_CHECK(msg); + error_occurred_ = true; { MutexLock l(manager_->GetBenchmarkMutex()); - if (internal::NotSkipped == manager_->results.skipped_) { - manager_->results.skip_message_ = msg; - manager_->results.skipped_ = skipped_; + if (manager_->results.has_error_ == false) { + manager_->results.error_message_ = msg; + manager_->results.has_error_ = true; } } total_iterations_ = 0; @@ -290,7 +225,7 @@ void State::SetIterationTime(double seconds) { timer_->SetIterationTime(seconds); } -void State::SetLabel(const std::string& label) { +void State::SetLabel(const char* label) { MutexLock l(manager_->GetBenchmarkMutex()); manager_->results.report_label_ = label; } @@ -298,14 +233,14 @@ void State::SetLabel(const std::string& label) { void State::StartKeepRunning() { BM_CHECK(!started_ && !finished_); started_ = true; - total_iterations_ = skipped() ? 0 : max_iterations; + total_iterations_ = error_occurred_ ? 0 : max_iterations; manager_->StartStopBarrier(); - if (!skipped()) ResumeTiming(); + if (!error_occurred_) ResumeTiming(); } void State::FinishKeepRunning() { - BM_CHECK(started_ && (!finished_ || skipped())); - if (!skipped()) { + BM_CHECK(started_ && (!finished_ || error_occurred_)); + if (!error_occurred_) { PauseTiming(); } // Total iterations has now wrapped around past 0. Fix this. @@ -383,26 +318,14 @@ void RunBenchmarks(const std::vector& benchmarks, size_t num_repetitions_total = 0; - // This perfcounters object needs to be created before the runners vector - // below so it outlasts their lifetime. - PerfCountersMeasurement perfcounters( - StrSplit(FLAGS_benchmark_perf_counters, ',')); - - // Vector of benchmarks to run std::vector runners; runners.reserve(benchmarks.size()); - - // Count the number of benchmarks with threads to warn the user in case - // performance counters are used. - int benchmarks_with_threads = 0; - - // Loop through all benchmarks for (const BenchmarkInstance& benchmark : benchmarks) { BenchmarkReporter::PerFamilyRunReports* reports_for_family = nullptr; if (benchmark.complexity() != oNone) reports_for_family = &per_family_reports[benchmark.family_index()]; - benchmarks_with_threads += (benchmark.threads() > 1); - runners.emplace_back(benchmark, &perfcounters, reports_for_family); + + runners.emplace_back(benchmark, reports_for_family); int num_repeats_of_this_instance = runners.back().GetNumRepeats(); num_repetitions_total += num_repeats_of_this_instance; if (reports_for_family) @@ -410,17 +333,6 @@ void RunBenchmarks(const std::vector& benchmarks, } assert(runners.size() == benchmarks.size() && "Unexpected runner count."); - // The use of performance counters with threads would be unintuitive for - // the average user so we need to warn them about this case - if ((benchmarks_with_threads > 0) && (perfcounters.num_counters() > 0)) { - GetErrorLogInstance() - << "***WARNING*** There are " << benchmarks_with_threads - << " benchmarks with threads and " << perfcounters.num_counters() - << " performance counters were requested. Beware counters will " - "reflect the combined usage across all " - "threads.\n"; - } - std::vector repetition_indices; repetition_indices.reserve(num_repetitions_total); for (size_t runner_index = 0, num_runners = runners.size(); @@ -444,12 +356,6 @@ void RunBenchmarks(const std::vector& benchmarks, if (runner.HasRepeatsRemaining()) continue; // FIXME: report each repetition separately, not all of them in bulk. - display_reporter->ReportRunsConfig( - runner.GetMinTime(), runner.HasExplicitIters(), runner.GetIters()); - if (file_reporter) - file_reporter->ReportRunsConfig( - runner.GetMinTime(), runner.HasExplicitIters(), runner.GetIters()); - RunResults run_results = runner.GetResults(); // Maybe calculate complexity report @@ -483,15 +389,14 @@ std::unique_ptr CreateReporter( typedef std::unique_ptr PtrType; if (name == "console") { return PtrType(new ConsoleReporter(output_opts)); + } else if (name == "json") { + return PtrType(new JSONReporter); + } else if (name == "csv") { + return PtrType(new CSVReporter); + } else { + std::cerr << "Unexpected format: '" << name << "'\n"; + std::exit(1); } - if (name == "json") { - return PtrType(new JSONReporter()); - } - if (name == "csv") { - return PtrType(new CSVReporter()); - } - std::cerr << "Unexpected format: '" << name << "'\n"; - std::exit(1); } BENCHMARK_RESTORE_DEPRECATED_WARNING @@ -528,14 +433,6 @@ ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color) { } // end namespace internal -BenchmarkReporter* CreateDefaultDisplayReporter() { - static auto default_display_reporter = - internal::CreateReporter(FLAGS_benchmark_format, - internal::GetOutputOptions()) - .release(); - return default_display_reporter; -} - size_t RunSpecifiedBenchmarks() { return RunSpecifiedBenchmarks(nullptr, nullptr, FLAGS_benchmark_filter); } @@ -571,7 +468,8 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, std::unique_ptr default_display_reporter; std::unique_ptr default_file_reporter; if (!display_reporter) { - default_display_reporter.reset(CreateDefaultDisplayReporter()); + default_display_reporter = internal::CreateReporter( + FLAGS_benchmark_format, internal::GetOutputOptions()); display_reporter = default_display_reporter.get(); } auto& Out = display_reporter->GetOutputStream(); @@ -582,23 +480,17 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, Err << "A custom file reporter was provided but " "--benchmark_out= was not specified." << std::endl; - Out.flush(); - Err.flush(); std::exit(1); } if (!fname.empty()) { output_file.open(fname); if (!output_file.is_open()) { Err << "invalid file name: '" << fname << "'" << std::endl; - Out.flush(); - Err.flush(); std::exit(1); } if (!file_reporter) { default_file_reporter = internal::CreateReporter( - FLAGS_benchmark_out_format, FLAGS_benchmark_counters_tabular - ? ConsoleReporter::OO_Tabular - : ConsoleReporter::OO_None); + FLAGS_benchmark_out_format, ConsoleReporter::OO_None); file_reporter = default_file_reporter.get(); } file_reporter->SetOutputStream(&output_file); @@ -606,16 +498,10 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, } std::vector benchmarks; - if (!FindBenchmarksInternal(spec, &benchmarks, &Err)) { - Out.flush(); - Err.flush(); - return 0; - } + if (!FindBenchmarksInternal(spec, &benchmarks, &Err)) return 0; if (benchmarks.empty()) { Err << "Failed to match any benchmarks against regex: " << spec << "\n"; - Out.flush(); - Err.flush(); return 0; } @@ -626,28 +512,11 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, internal::RunBenchmarks(benchmarks, display_reporter, file_reporter); } - Out.flush(); - Err.flush(); return benchmarks.size(); } -namespace { -// stores the time unit benchmarks use by default -TimeUnit default_time_unit = kNanosecond; -} // namespace - -TimeUnit GetDefaultTimeUnit() { return default_time_unit; } - -void SetDefaultTimeUnit(TimeUnit unit) { default_time_unit = unit; } - std::string GetBenchmarkFilter() { return FLAGS_benchmark_filter; } -void SetBenchmarkFilter(std::string value) { - FLAGS_benchmark_filter = std::move(value); -} - -int32_t GetBenchmarkVerbosity() { return FLAGS_v; } - void RegisterMemoryManager(MemoryManager* manager) { internal::memory_manager = manager; } @@ -664,31 +533,27 @@ void AddCustomContext(const std::string& key, const std::string& value) { namespace internal { -void (*HelperPrintf)(); - void PrintUsageAndExit() { - HelperPrintf(); + fprintf(stdout, + "benchmark" + " [--benchmark_list_tests={true|false}]\n" + " [--benchmark_filter=]\n" + " [--benchmark_min_time=]\n" + " [--benchmark_repetitions=]\n" + " [--benchmark_enable_random_interleaving={true|false}]\n" + " [--benchmark_report_aggregates_only={true|false}]\n" + " [--benchmark_display_aggregates_only={true|false}]\n" + " [--benchmark_format=]\n" + " [--benchmark_out=]\n" + " [--benchmark_out_format=]\n" + " [--benchmark_color={auto|true|false}]\n" + " [--benchmark_counters_tabular={true|false}]\n" + " [--benchmark_perf_counters=,...]\n" + " [--benchmark_context==,...]\n" + " [--v=]\n"); exit(0); } -void SetDefaultTimeUnitFromFlag(const std::string& time_unit_flag) { - if (time_unit_flag == "s") { - return SetDefaultTimeUnit(kSecond); - } - if (time_unit_flag == "ms") { - return SetDefaultTimeUnit(kMillisecond); - } - if (time_unit_flag == "us") { - return SetDefaultTimeUnit(kMicrosecond); - } - if (time_unit_flag == "ns") { - return SetDefaultTimeUnit(kNanosecond); - } - if (!time_unit_flag.empty()) { - PrintUsageAndExit(); - } -} - void ParseCommandLineFlags(int* argc, char** argv) { using namespace benchmark; BenchmarkReporter::Context::executable_name = @@ -697,10 +562,8 @@ void ParseCommandLineFlags(int* argc, char** argv) { if (ParseBoolFlag(argv[i], "benchmark_list_tests", &FLAGS_benchmark_list_tests) || ParseStringFlag(argv[i], "benchmark_filter", &FLAGS_benchmark_filter) || - ParseStringFlag(argv[i], "benchmark_min_time", + ParseDoubleFlag(argv[i], "benchmark_min_time", &FLAGS_benchmark_min_time) || - ParseDoubleFlag(argv[i], "benchmark_min_warmup_time", - &FLAGS_benchmark_min_warmup_time) || ParseInt32Flag(argv[i], "benchmark_repetitions", &FLAGS_benchmark_repetitions) || ParseBoolFlag(argv[i], "benchmark_enable_random_interleaving", @@ -720,8 +583,6 @@ void ParseCommandLineFlags(int* argc, char** argv) { &FLAGS_benchmark_perf_counters) || ParseKeyValueFlag(argv[i], "benchmark_context", &FLAGS_benchmark_context) || - ParseStringFlag(argv[i], "benchmark_time_unit", - &FLAGS_benchmark_time_unit) || ParseInt32Flag(argv[i], "v", &FLAGS_v)) { for (int j = i; j != *argc - 1; ++j) argv[j] = argv[j + 1]; @@ -737,7 +598,6 @@ void ParseCommandLineFlags(int* argc, char** argv) { PrintUsageAndExit(); } } - SetDefaultTimeUnitFromFlag(FLAGS_benchmark_time_unit); if (FLAGS_benchmark_color.empty()) { PrintUsageAndExit(); } @@ -753,34 +613,7 @@ int InitializeStreams() { } // end namespace internal -std::string GetBenchmarkVersion() { return {BENCHMARK_VERSION}; } - -void PrintDefaultHelp() { - fprintf(stdout, - "benchmark" - " [--benchmark_list_tests={true|false}]\n" - " [--benchmark_filter=]\n" - " [--benchmark_min_time=`x` OR `s` ]\n" - " [--benchmark_min_warmup_time=]\n" - " [--benchmark_repetitions=]\n" - " [--benchmark_enable_random_interleaving={true|false}]\n" - " [--benchmark_report_aggregates_only={true|false}]\n" - " [--benchmark_display_aggregates_only={true|false}]\n" - " [--benchmark_format=]\n" - " [--benchmark_out=]\n" - " [--benchmark_out_format=]\n" - " [--benchmark_color={auto|true|false}]\n" - " [--benchmark_counters_tabular={true|false}]\n" -#if defined HAVE_LIBPFM - " [--benchmark_perf_counters=,...]\n" -#endif - " [--benchmark_context==,...]\n" - " [--benchmark_time_unit={ns|us|ms|s}]\n" - " [--v=]\n"); -} - -void Initialize(int* argc, char** argv, void (*HelperPrintf)()) { - internal::HelperPrintf = HelperPrintf; +void Initialize(int* argc, char** argv) { internal::ParseCommandLineFlags(argc, argv); internal::LogLevel() = FLAGS_v; } diff --git a/third-party/benchmark/src/benchmark_api_internal.cc b/third-party/benchmark/src/benchmark_api_internal.cc index 286f98653085..4de36e3c8ba3 100644 --- a/third-party/benchmark/src/benchmark_api_internal.cc +++ b/third-party/benchmark/src/benchmark_api_internal.cc @@ -16,7 +16,7 @@ BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, int family_idx, per_family_instance_index_(per_family_instance_idx), aggregation_report_mode_(benchmark_.aggregation_report_mode_), args_(args), - time_unit_(benchmark_.GetTimeUnit()), + time_unit_(benchmark_.time_unit_), measure_process_cpu_time_(benchmark_.measure_process_cpu_time_), use_real_time_(benchmark_.use_real_time_), use_manual_time_(benchmark_.use_manual_time_), @@ -25,7 +25,6 @@ BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, int family_idx, statistics_(benchmark_.statistics_), repetitions_(benchmark_.repetitions_), min_time_(benchmark_.min_time_), - min_warmup_time_(benchmark_.min_warmup_time_), iterations_(benchmark_.iterations_), threads_(thread_count) { name_.function_name = benchmark_.name_; @@ -51,11 +50,6 @@ BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, int family_idx, name_.min_time = StrFormat("min_time:%0.3f", benchmark_.min_time_); } - if (!IsZero(benchmark->min_warmup_time_)) { - name_.min_warmup_time = - StrFormat("min_warmup_time:%0.3f", benchmark_.min_warmup_time_); - } - if (benchmark_.iterations_ != 0) { name_.iterations = StrFormat( "iterations:%lu", static_cast(benchmark_.iterations_)); @@ -93,24 +87,24 @@ State BenchmarkInstance::Run( IterationCount iters, int thread_id, internal::ThreadTimer* timer, internal::ThreadManager* manager, internal::PerfCountersMeasurement* perf_counters_measurement) const { - State st(name_.function_name, iters, args_, thread_id, threads_, timer, - manager, perf_counters_measurement); + State st(iters, args_, thread_id, threads_, timer, manager, + perf_counters_measurement); benchmark_.Run(st); return st; } void BenchmarkInstance::Setup() const { if (setup_) { - State st(name_.function_name, /*iters*/ 1, args_, /*thread_id*/ 0, threads_, - nullptr, nullptr, nullptr); + State st(/*iters*/ 1, args_, /*thread_id*/ 0, threads_, nullptr, nullptr, + nullptr); setup_(st); } } void BenchmarkInstance::Teardown() const { if (teardown_) { - State st(name_.function_name, /*iters*/ 1, args_, /*thread_id*/ 0, threads_, - nullptr, nullptr, nullptr); + State st(/*iters*/ 1, args_, /*thread_id*/ 0, threads_, nullptr, nullptr, + nullptr); teardown_(st); } } diff --git a/third-party/benchmark/src/benchmark_api_internal.h b/third-party/benchmark/src/benchmark_api_internal.h index 94f516531bc4..94c2b2972bb1 100644 --- a/third-party/benchmark/src/benchmark_api_internal.h +++ b/third-party/benchmark/src/benchmark_api_internal.h @@ -36,7 +36,6 @@ class BenchmarkInstance { const std::vector& statistics() const { return statistics_; } int repetitions() const { return repetitions_; } double min_time() const { return min_time_; } - double min_warmup_time() const { return min_warmup_time_; } IterationCount iterations() const { return iterations_; } int threads() const { return threads_; } void Setup() const; @@ -63,7 +62,6 @@ class BenchmarkInstance { const std::vector& statistics_; int repetitions_; double min_time_; - double min_warmup_time_; IterationCount iterations_; int threads_; // Number of concurrent threads to us @@ -78,7 +76,6 @@ bool FindBenchmarksInternal(const std::string& re, bool IsZero(double n); -BENCHMARK_EXPORT ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color = false); } // end namespace internal diff --git a/third-party/benchmark/src/benchmark_main.cc b/third-party/benchmark/src/benchmark_main.cc index cd61cd2ad506..b3b247831496 100644 --- a/third-party/benchmark/src/benchmark_main.cc +++ b/third-party/benchmark/src/benchmark_main.cc @@ -14,5 +14,4 @@ #include "benchmark/benchmark.h" -BENCHMARK_EXPORT int main(int, char**); BENCHMARK_MAIN(); diff --git a/third-party/benchmark/src/benchmark_name.cc b/third-party/benchmark/src/benchmark_name.cc index 01676bbc84df..2a17ebce277f 100644 --- a/third-party/benchmark/src/benchmark_name.cc +++ b/third-party/benchmark/src/benchmark_name.cc @@ -51,9 +51,8 @@ std::string join(char delimiter, const Ts&... ts) { } } // namespace -BENCHMARK_EXPORT std::string BenchmarkName::str() const { - return join('/', function_name, args, min_time, min_warmup_time, iterations, - repetitions, time_type, threads); + return join('/', function_name, args, min_time, iterations, repetitions, + time_type, threads); } } // namespace benchmark diff --git a/third-party/benchmark/src/benchmark_register.cc b/third-party/benchmark/src/benchmark_register.cc index e447c9a2d39b..61a0c26178e3 100644 --- a/third-party/benchmark/src/benchmark_register.cc +++ b/third-party/benchmark/src/benchmark_register.cc @@ -15,7 +15,7 @@ #include "benchmark_register.h" #ifndef BENCHMARK_OS_WINDOWS -#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) +#ifndef BENCHMARK_OS_FUCHSIA #include #endif #include @@ -53,13 +53,10 @@ namespace benchmark { namespace { // For non-dense Range, intermediate values are powers of kRangeMultiplier. -static constexpr int kRangeMultiplier = 8; - +static const int kRangeMultiplier = 8; // The size of a benchmark family determines is the number of inputs to repeat // the benchmark on. If this is "large" then warn the user during configuration. -static constexpr size_t kMaxFamilySize = 100; - -static constexpr char kDisabledPrefix[] = "DISABLED_"; +static const size_t kMaxFamilySize = 100; } // end namespace namespace internal { @@ -119,10 +116,10 @@ bool BenchmarkFamilies::FindBenchmarks( // Make regular expression out of command-line flag std::string error_msg; Regex re; - bool is_negative_filter = false; + bool isNegativeFilter = false; if (spec[0] == '-') { spec.replace(0, 1, ""); - is_negative_filter = true; + isNegativeFilter = true; } if (!re.Init(spec, &error_msg)) { Err << "Could not compile benchmark re: " << error_msg << std::endl; @@ -157,8 +154,7 @@ bool BenchmarkFamilies::FindBenchmarks( << " will be repeated at least " << family_size << " times.\n"; } // reserve in the special case the regex ".", since we know the final - // family size. this doesn't take into account any disabled benchmarks - // so worst case we reserve more than we need. + // family size. if (spec == ".") benchmarks->reserve(benchmarks->size() + family_size); for (auto const& args : family->args_) { @@ -168,9 +164,8 @@ bool BenchmarkFamilies::FindBenchmarks( num_threads); const auto full_name = instance.name().str(); - if (full_name.rfind(kDisabledPrefix, 0) != 0 && - ((re.Match(full_name) && !is_negative_filter) || - (!re.Match(full_name) && is_negative_filter))) { + if ((re.Match(full_name) && !isNegativeFilter) || + (!re.Match(full_name) && isNegativeFilter)) { benchmarks->push_back(std::move(instance)); ++per_family_instance_index; @@ -204,14 +199,12 @@ bool FindBenchmarksInternal(const std::string& re, // Benchmark //=============================================================================// -Benchmark::Benchmark(const std::string& name) +Benchmark::Benchmark(const char* name) : name_(name), aggregation_report_mode_(ARM_Unspecified), - time_unit_(GetDefaultTimeUnit()), - use_default_time_unit_(true), + time_unit_(kNanosecond), range_multiplier_(kRangeMultiplier), min_time_(0), - min_warmup_time_(0), iterations_(0), repetitions_(0), measure_process_cpu_time_(false), @@ -230,7 +223,7 @@ Benchmark::Benchmark(const std::string& name) Benchmark::~Benchmark() {} Benchmark* Benchmark::Name(const std::string& name) { - SetName(name); + SetName(name.c_str()); return this; } @@ -242,7 +235,6 @@ Benchmark* Benchmark::Arg(int64_t x) { Benchmark* Benchmark::Unit(TimeUnit unit) { time_unit_ = unit; - use_default_time_unit_ = false; return this; } @@ -356,17 +348,9 @@ Benchmark* Benchmark::MinTime(double t) { return this; } -Benchmark* Benchmark::MinWarmUpTime(double t) { - BM_CHECK(t >= 0.0); - BM_CHECK(iterations_ == 0); - min_warmup_time_ = t; - return this; -} - Benchmark* Benchmark::Iterations(IterationCount n) { BM_CHECK(n > 0); BM_CHECK(IsZero(min_time_)); - BM_CHECK(IsZero(min_warmup_time_)); iterations_ = n; return this; } @@ -468,9 +452,7 @@ Benchmark* Benchmark::ThreadPerCpu() { return this; } -void Benchmark::SetName(const std::string& name) { name_ = name; } - -const char* Benchmark::GetName() const { return name_.c_str(); } +void Benchmark::SetName(const char* name) { name_ = name; } int Benchmark::ArgsCnt() const { if (args_.empty()) { @@ -480,16 +462,6 @@ int Benchmark::ArgsCnt() const { return static_cast(args_.front().size()); } -const char* Benchmark::GetArgName(int arg) const { - BM_CHECK_GE(arg, 0); - BM_CHECK_LT(arg, static_cast(arg_names_.size())); - return arg_names_[arg].c_str(); -} - -TimeUnit Benchmark::GetTimeUnit() const { - return use_default_time_unit_ ? GetDefaultTimeUnit() : time_unit_; -} - //=============================================================================// // FunctionBenchmark //=============================================================================// diff --git a/third-party/benchmark/src/benchmark_register.h b/third-party/benchmark/src/benchmark_register.h index 53367c707cf4..d3f4974e9074 100644 --- a/third-party/benchmark/src/benchmark_register.h +++ b/third-party/benchmark/src/benchmark_register.h @@ -1,7 +1,6 @@ #ifndef BENCHMARK_REGISTER_H #define BENCHMARK_REGISTER_H -#include #include #include @@ -24,7 +23,7 @@ typename std::vector::iterator AddPowers(std::vector* dst, T lo, T hi, static const T kmax = std::numeric_limits::max(); // Space out the values in multiples of "mult" - for (T i = static_cast(1); i <= hi; i *= static_cast(mult)) { + for (T i = static_cast(1); i <= hi; i *= mult) { if (i >= lo) { dst->push_back(i); } @@ -33,7 +32,7 @@ typename std::vector::iterator AddPowers(std::vector* dst, T lo, T hi, if (i > kmax / mult) break; } - return dst->begin() + static_cast(start_offset); + return dst->begin() + start_offset; } template diff --git a/third-party/benchmark/src/benchmark_runner.cc b/third-party/benchmark/src/benchmark_runner.cc index dcddb437e37d..eac807b066f1 100644 --- a/third-party/benchmark/src/benchmark_runner.cc +++ b/third-party/benchmark/src/benchmark_runner.cc @@ -19,7 +19,7 @@ #include "internal_macros.h" #ifndef BENCHMARK_OS_WINDOWS -#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) +#ifndef BENCHMARK_OS_FUCHSIA #include #endif #include @@ -28,14 +28,11 @@ #include #include -#include -#include #include #include #include #include #include -#include #include #include #include @@ -64,9 +61,7 @@ MemoryManager* memory_manager = nullptr; namespace { -static constexpr IterationCount kMaxIterations = 1000000000000; -const double kDefaultMinTime = - std::strtod(::benchmark::kDefaultMinTimeStr, /*p_end*/ nullptr); +static constexpr IterationCount kMaxIterations = 1000000000; BenchmarkReporter::Run CreateRunReport( const benchmark::internal::BenchmarkInstance& b, @@ -80,8 +75,8 @@ BenchmarkReporter::Run CreateRunReport( report.run_name = b.name(); report.family_index = b.family_index(); report.per_family_instance_index = b.per_family_instance_index(); - report.skipped = results.skipped_; - report.skip_message = results.skip_message_; + report.error_occurred = results.has_error_; + report.error_message = results.error_message_; report.report_label = results.report_label_; // This is the total iterations across all threads. report.iterations = results.iterations; @@ -90,13 +85,12 @@ BenchmarkReporter::Run CreateRunReport( report.repetition_index = repetition_index; report.repetitions = repeats; - if (!report.skipped) { + if (!report.error_occurred) { if (b.use_manual_time()) { report.real_accumulated_time = results.manual_time_used; } else { report.real_accumulated_time = results.real_time_used; } - report.use_real_time_for_initial_big_o = b.use_manual_time(); report.cpu_accumulated_time = results.cpu_time_used; report.complexity_n = results.complexity_n; report.complexity = b.complexity(); @@ -109,7 +103,7 @@ BenchmarkReporter::Run CreateRunReport( report.memory_result = memory_result; report.allocs_per_iter = memory_iterations ? static_cast(memory_result->num_allocs) / - static_cast(memory_iterations) + memory_iterations : 0; } @@ -128,10 +122,9 @@ void RunInThread(const BenchmarkInstance* b, IterationCount iters, b->measure_process_cpu_time() ? internal::ThreadTimer::CreateProcessCpuTime() : internal::ThreadTimer::Create()); - State st = b->Run(iters, thread_id, &timer, manager, perf_counters_measurement); - BM_CHECK(st.skipped() || st.iterations() >= st.max_iterations) + BM_CHECK(st.error_occurred() || st.iterations() >= st.max_iterations) << "Benchmark returned before State::KeepRunning() returned false!"; { MutexLock l(manager->GetBenchmarkMutex()); @@ -146,100 +139,24 @@ void RunInThread(const BenchmarkInstance* b, IterationCount iters, manager->NotifyThreadComplete(); } -double ComputeMinTime(const benchmark::internal::BenchmarkInstance& b, - const BenchTimeType& iters_or_time) { - if (!IsZero(b.min_time())) return b.min_time(); - // If the flag was used to specify number of iters, then return the default - // min_time. - if (iters_or_time.tag == BenchTimeType::ITERS) return kDefaultMinTime; - - return iters_or_time.time; -} - -IterationCount ComputeIters(const benchmark::internal::BenchmarkInstance& b, - const BenchTimeType& iters_or_time) { - if (b.iterations() != 0) return b.iterations(); - - // We've already concluded that this flag is currently used to pass - // iters but do a check here again anyway. - BM_CHECK(iters_or_time.tag == BenchTimeType::ITERS); - return iters_or_time.iters; -} - } // end namespace -BenchTimeType ParseBenchMinTime(const std::string& value) { - BenchTimeType ret; - - if (value.empty()) { - ret.tag = BenchTimeType::TIME; - ret.time = 0.0; - return ret; - } - - if (value.back() == 'x') { - char* p_end; - // Reset errno before it's changed by strtol. - errno = 0; - IterationCount num_iters = std::strtol(value.c_str(), &p_end, 10); - - // After a valid parse, p_end should have been set to - // point to the 'x' suffix. - BM_CHECK(errno == 0 && p_end != nullptr && *p_end == 'x') - << "Malformed iters value passed to --benchmark_min_time: `" << value - << "`. Expected --benchmark_min_time=x."; - - ret.tag = BenchTimeType::ITERS; - ret.iters = num_iters; - return ret; - } - - bool has_suffix = value.back() == 's'; - if (!has_suffix) { - BM_VLOG(0) << "Value passed to --benchmark_min_time should have a suffix. " - "Eg., `30s` for 30-seconds."; - } - - char* p_end; - // Reset errno before it's changed by strtod. - errno = 0; - double min_time = std::strtod(value.c_str(), &p_end); - - // After a successful parse, p_end should point to the suffix 's', - // or the end of the string if the suffix was omitted. - BM_CHECK(errno == 0 && p_end != nullptr && - ((has_suffix && *p_end == 's') || *p_end == '\0')) - << "Malformed seconds value passed to --benchmark_min_time: `" << value - << "`. Expected --benchmark_min_time=x."; - - ret.tag = BenchTimeType::TIME; - ret.time = min_time; - - return ret; -} - BenchmarkRunner::BenchmarkRunner( const benchmark::internal::BenchmarkInstance& b_, - PerfCountersMeasurement* pcm_, BenchmarkReporter::PerFamilyRunReports* reports_for_family_) : b(b_), reports_for_family(reports_for_family_), - parsed_benchtime_flag(ParseBenchMinTime(FLAGS_benchmark_min_time)), - min_time(ComputeMinTime(b_, parsed_benchtime_flag)), - min_warmup_time((!IsZero(b.min_time()) && b.min_warmup_time() > 0.0) - ? b.min_warmup_time() - : FLAGS_benchmark_min_warmup_time), - warmup_done(!(min_warmup_time > 0.0)), + min_time(!IsZero(b.min_time()) ? b.min_time() : FLAGS_benchmark_min_time), repeats(b.repetitions() != 0 ? b.repetitions() : FLAGS_benchmark_repetitions), - has_explicit_iteration_count(b.iterations() != 0 || - parsed_benchtime_flag.tag == - BenchTimeType::ITERS), + has_explicit_iteration_count(b.iterations() != 0), pool(b.threads() - 1), - iters(has_explicit_iteration_count - ? ComputeIters(b_, parsed_benchtime_flag) - : 1), - perf_counters_measurement_ptr(pcm_) { + iters(has_explicit_iteration_count ? b.iterations() : 1), + perf_counters_measurement( + PerfCounters::Create(StrSplit(FLAGS_benchmark_perf_counters, ','))), + perf_counters_measurement_ptr(perf_counters_measurement.IsValid() + ? &perf_counters_measurement + : nullptr) { run_results.display_report_aggregates_only = (FLAGS_benchmark_report_aggregates_only || FLAGS_benchmark_display_aggregates_only); @@ -252,7 +169,7 @@ BenchmarkRunner::BenchmarkRunner( run_results.file_report_aggregates_only = (b.aggregation_report_mode() & internal::ARM_FileReportAggregatesOnly); BM_CHECK(FLAGS_benchmark_perf_counters.empty() || - (perf_counters_measurement_ptr->num_counters() == 0)) + perf_counters_measurement.IsValid()) << "Perf counters were requested but could not be set up."; } } @@ -315,20 +232,20 @@ IterationCount BenchmarkRunner::PredictNumItersNeeded( const IterationResults& i) const { // See how much iterations should be increased by. // Note: Avoid division by zero with max(seconds, 1ns). - double multiplier = GetMinTimeToApply() * 1.4 / std::max(i.seconds, 1e-9); + double multiplier = min_time * 1.4 / std::max(i.seconds, 1e-9); // If our last run was at least 10% of FLAGS_benchmark_min_time then we // use the multiplier directly. // Otherwise we use at most 10 times expansion. // NOTE: When the last run was at least 10% of the min time the max // expansion should be 14x. - const bool is_significant = (i.seconds / GetMinTimeToApply()) > 0.1; + bool is_significant = (i.seconds / min_time) > 0.1; multiplier = is_significant ? multiplier : 10.0; // So what seems to be the sufficiently-large iteration count? Round up. const IterationCount max_next_iters = static_cast( - std::llround(std::max(multiplier * static_cast(i.iters), - static_cast(i.iters) + 1.0))); - // But we do have *some* limits though.. + std::lround(std::max(multiplier * static_cast(i.iters), + static_cast(i.iters) + 1.0))); + // But we do have *some* sanity limits though.. const IterationCount next_iters = std::min(max_next_iters, kMaxIterations); BM_VLOG(3) << "Next iters: " << next_iters << ", " << multiplier << "\n"; @@ -340,80 +257,21 @@ bool BenchmarkRunner::ShouldReportIterationResults( // Determine if this run should be reported; // Either it has run for a sufficient amount of time // or because an error was reported. - return i.results.skipped_ || + return i.results.has_error_ || i.iters >= kMaxIterations || // Too many iterations already. - i.seconds >= - GetMinTimeToApply() || // The elapsed time is large enough. + i.seconds >= min_time || // The elapsed time is large enough. // CPU time is specified but the elapsed real time greatly exceeds // the minimum time. - // Note that user provided timers are except from this test. - ((i.results.real_time_used >= 5 * GetMinTimeToApply()) && - !b.use_manual_time()); -} - -double BenchmarkRunner::GetMinTimeToApply() const { - // In order to re-use functionality to run and measure benchmarks for running - // a warmup phase of the benchmark, we need a way of telling whether to apply - // min_time or min_warmup_time. This function will figure out if we are in the - // warmup phase and therefore need to apply min_warmup_time or if we already - // in the benchmarking phase and min_time needs to be applied. - return warmup_done ? min_time : min_warmup_time; -} - -void BenchmarkRunner::FinishWarmUp(const IterationCount& i) { - warmup_done = true; - iters = i; -} - -void BenchmarkRunner::RunWarmUp() { - // Use the same mechanisms for warming up the benchmark as used for actually - // running and measuring the benchmark. - IterationResults i_warmup; - // Dont use the iterations determined in the warmup phase for the actual - // measured benchmark phase. While this may be a good starting point for the - // benchmark and it would therefore get rid of the need to figure out how many - // iterations are needed if min_time is set again, this may also be a complete - // wrong guess since the warmup loops might be considerably slower (e.g - // because of caching effects). - const IterationCount i_backup = iters; - - for (;;) { - b.Setup(); - i_warmup = DoNIterations(); - b.Teardown(); - - const bool finish = ShouldReportIterationResults(i_warmup); - - if (finish) { - FinishWarmUp(i_backup); - break; - } - - // Although we are running "only" a warmup phase where running enough - // iterations at once without measuring time isn't as important as it is for - // the benchmarking phase, we still do it the same way as otherwise it is - // very confusing for the user to know how to choose a proper value for - // min_warmup_time if a different approach on running it is used. - iters = PredictNumItersNeeded(i_warmup); - assert(iters > i_warmup.iters && - "if we did more iterations than we want to do the next time, " - "then we should have accepted the current iteration run."); - } + // Note that user provided timers are except from this sanity check. + ((i.results.real_time_used >= 5 * min_time) && !b.use_manual_time()); } void BenchmarkRunner::DoOneRepetition() { assert(HasRepeatsRemaining() && "Already done all repetitions?"); const bool is_the_first_repetition = num_repetitions_done == 0; - - // In case a warmup phase is requested by the benchmark, run it now. - // After running the warmup phase the BenchmarkRunner should be in a state as - // this warmup never happened except the fact that warmup_done is set. Every - // other manipulation of the BenchmarkRunner instance would be a bug! Please - // fix it. - if (!warmup_done) RunWarmUp(); - IterationResults i; + // We *may* be gradually increasing the length (iteration count) // of the benchmark until we decide the results are significant. // And once we do, we report those last results and exit. @@ -466,7 +324,10 @@ void BenchmarkRunner::DoOneRepetition() { manager->WaitForAllThreads(); manager.reset(); b.Teardown(); - memory_manager->Stop(*memory_result); + + BENCHMARK_DISABLE_DEPRECATED_WARNING + memory_manager->Stop(memory_result); + BENCHMARK_RESTORE_DEPRECATED_WARNING } // Ok, now actually report. @@ -476,7 +337,7 @@ void BenchmarkRunner::DoOneRepetition() { if (reports_for_family) { ++reports_for_family->num_runs_done; - if (!report.skipped) reports_for_family->Runs.push_back(report); + if (!report.error_occurred) reports_for_family->Runs.push_back(report); } run_results.non_aggregates.push_back(report); diff --git a/third-party/benchmark/src/benchmark_runner.h b/third-party/benchmark/src/benchmark_runner.h index db2fa04396c5..752eefdc26fa 100644 --- a/third-party/benchmark/src/benchmark_runner.h +++ b/third-party/benchmark/src/benchmark_runner.h @@ -25,8 +25,7 @@ namespace benchmark { -BM_DECLARE_string(benchmark_min_time); -BM_DECLARE_double(benchmark_min_warmup_time); +BM_DECLARE_double(benchmark_min_time); BM_DECLARE_int32(benchmark_repetitions); BM_DECLARE_bool(benchmark_report_aggregates_only); BM_DECLARE_bool(benchmark_display_aggregates_only); @@ -44,21 +43,9 @@ struct RunResults { bool file_report_aggregates_only = false; }; -struct BENCHMARK_EXPORT BenchTimeType { - enum { ITERS, TIME } tag; - union { - IterationCount iters; - double time; - }; -}; - -BENCHMARK_EXPORT -BenchTimeType ParseBenchMinTime(const std::string& value); - class BenchmarkRunner { public: BenchmarkRunner(const benchmark::internal::BenchmarkInstance& b_, - benchmark::internal::PerfCountersMeasurement* pmc_, BenchmarkReporter::PerFamilyRunReports* reports_for_family); int GetNumRepeats() const { return repeats; } @@ -75,22 +62,13 @@ class BenchmarkRunner { return reports_for_family; } - double GetMinTime() const { return min_time; } - - bool HasExplicitIters() const { return has_explicit_iteration_count; } - - IterationCount GetIters() const { return iters; } - private: RunResults run_results; const benchmark::internal::BenchmarkInstance& b; BenchmarkReporter::PerFamilyRunReports* reports_for_family; - BenchTimeType parsed_benchtime_flag; const double min_time; - const double min_warmup_time; - bool warmup_done; const int repeats; const bool has_explicit_iteration_count; @@ -104,7 +82,8 @@ class BenchmarkRunner { // So only the first repetition has to find/calculate it, // the other repetitions will just use that precomputed iteration count. - PerfCountersMeasurement* const perf_counters_measurement_ptr = nullptr; + PerfCountersMeasurement perf_counters_measurement; + PerfCountersMeasurement* const perf_counters_measurement_ptr; struct IterationResults { internal::ThreadManager::Result results; @@ -116,12 +95,6 @@ class BenchmarkRunner { IterationCount PredictNumItersNeeded(const IterationResults& i) const; bool ShouldReportIterationResults(const IterationResults& i) const; - - double GetMinTimeToApply() const; - - void FinishWarmUp(const IterationCount& i); - - void RunWarmUp(); }; } // namespace internal diff --git a/third-party/benchmark/src/check.h b/third-party/benchmark/src/check.h index c1cd5e85e44c..0efd13ff4db6 100644 --- a/third-party/benchmark/src/check.h +++ b/third-party/benchmark/src/check.h @@ -5,34 +5,18 @@ #include #include -#include "benchmark/export.h" #include "internal_macros.h" #include "log.h" -#if defined(__GNUC__) || defined(__clang__) -#define BENCHMARK_NOEXCEPT noexcept -#define BENCHMARK_NOEXCEPT_OP(x) noexcept(x) -#elif defined(_MSC_VER) && !defined(__clang__) -#if _MSC_VER >= 1900 -#define BENCHMARK_NOEXCEPT noexcept -#define BENCHMARK_NOEXCEPT_OP(x) noexcept(x) -#else -#define BENCHMARK_NOEXCEPT -#define BENCHMARK_NOEXCEPT_OP(x) -#endif -#define __func__ __FUNCTION__ -#else -#define BENCHMARK_NOEXCEPT -#define BENCHMARK_NOEXCEPT_OP(x) -#endif - namespace benchmark { namespace internal { typedef void(AbortHandlerT)(); -BENCHMARK_EXPORT -AbortHandlerT*& GetAbortHandler(); +inline AbortHandlerT*& GetAbortHandler() { + static AbortHandlerT* handler = &std::abort; + return handler; +} BENCHMARK_NORETURN inline void CallAbortHandler() { GetAbortHandler()(); @@ -52,17 +36,10 @@ class CheckHandler { LogType& GetLog() { return log_; } -#if defined(COMPILER_MSVC) -#pragma warning(push) -#pragma warning(disable : 4722) -#endif BENCHMARK_NORETURN ~CheckHandler() BENCHMARK_NOEXCEPT_OP(false) { log_ << std::endl; CallAbortHandler(); } -#if defined(COMPILER_MSVC) -#pragma warning(pop) -#endif CheckHandler& operator=(const CheckHandler&) = delete; CheckHandler(const CheckHandler&) = delete; diff --git a/third-party/benchmark/src/colorprint.cc b/third-party/benchmark/src/colorprint.cc index abc71492f77a..1a000a063736 100644 --- a/third-party/benchmark/src/colorprint.cc +++ b/third-party/benchmark/src/colorprint.cc @@ -96,18 +96,18 @@ std::string FormatString(const char* msg, va_list args) { // currently there is no error handling for failure, so this is hack. BM_CHECK(ret >= 0); - if (ret == 0) { // handle empty expansion + if (ret == 0) // handle empty expansion return {}; - } - if (static_cast(ret) < size) { + else if (static_cast(ret) < size) return local_buff; + else { + // we did not provide a long enough buffer on our first attempt. + size = static_cast(ret) + 1; // + 1 for the null byte + std::unique_ptr buff(new char[size]); + ret = vsnprintf(buff.get(), size, msg, args); + BM_CHECK(ret > 0 && (static_cast(ret)) < size); + return buff.get(); } - // we did not provide a long enough buffer on our first attempt. - size = static_cast(ret) + 1; // + 1 for the null byte - std::unique_ptr buff(new char[size]); - ret = vsnprintf(buff.get(), size, msg, args); - BM_CHECK(ret > 0 && (static_cast(ret)) < size); - return buff.get(); } std::string FormatString(const char* msg, ...) { @@ -140,12 +140,12 @@ void ColorPrintf(std::ostream& out, LogColor color, const char* fmt, // We need to flush the stream buffers into the console before each // SetConsoleTextAttribute call lest it affect the text that is already // printed but has not yet reached the console. - out.flush(); + fflush(stdout); SetConsoleTextAttribute(stdout_handle, GetPlatformColorCode(color) | FOREGROUND_INTENSITY); - out << FormatString(fmt, args); + vprintf(fmt, args); - out.flush(); + fflush(stdout); // Restores the text color. SetConsoleTextAttribute(stdout_handle, old_color_attrs); #else @@ -163,24 +163,12 @@ bool IsColorTerminal() { #else // On non-Windows platforms, we rely on the TERM variable. This list of // supported TERM values is copied from Google Test: - // . + // . const char* const SUPPORTED_TERM_VALUES[] = { - "xterm", - "xterm-color", - "xterm-256color", - "screen", - "screen-256color", - "tmux", - "tmux-256color", - "rxvt-unicode", - "rxvt-unicode-256color", - "linux", - "cygwin", - "xterm-kitty", - "alacritty", - "foot", - "foot-extra", - "wezterm", + "xterm", "xterm-color", "xterm-256color", + "screen", "screen-256color", "tmux", + "tmux-256color", "rxvt-unicode", "rxvt-unicode-256color", + "linux", "cygwin", }; const char* const term = getenv("TERM"); diff --git a/third-party/benchmark/src/commandlineflags.cc b/third-party/benchmark/src/commandlineflags.cc index dcb414959df4..9615e351ffae 100644 --- a/third-party/benchmark/src/commandlineflags.cc +++ b/third-party/benchmark/src/commandlineflags.cc @@ -121,14 +121,12 @@ static std::string FlagToEnvVar(const char* flag) { } // namespace -BENCHMARK_EXPORT bool BoolFromEnv(const char* flag, bool default_val) { const std::string env_var = FlagToEnvVar(flag); const char* const value_str = getenv(env_var.c_str()); return value_str == nullptr ? default_val : IsTruthyFlagValue(value_str); } -BENCHMARK_EXPORT int32_t Int32FromEnv(const char* flag, int32_t default_val) { const std::string env_var = FlagToEnvVar(flag); const char* const value_str = getenv(env_var.c_str()); @@ -141,7 +139,6 @@ int32_t Int32FromEnv(const char* flag, int32_t default_val) { return value; } -BENCHMARK_EXPORT double DoubleFromEnv(const char* flag, double default_val) { const std::string env_var = FlagToEnvVar(flag); const char* const value_str = getenv(env_var.c_str()); @@ -154,14 +151,12 @@ double DoubleFromEnv(const char* flag, double default_val) { return value; } -BENCHMARK_EXPORT const char* StringFromEnv(const char* flag, const char* default_val) { const std::string env_var = FlagToEnvVar(flag); const char* const value = getenv(env_var.c_str()); return value == nullptr ? default_val : value; } -BENCHMARK_EXPORT std::map KvPairsFromEnv( const char* flag, std::map default_val) { const std::string env_var = FlagToEnvVar(flag); @@ -206,7 +201,6 @@ const char* ParseFlagValue(const char* str, const char* flag, return flag_end + 1; } -BENCHMARK_EXPORT bool ParseBoolFlag(const char* str, const char* flag, bool* value) { // Gets the value of the flag as a string. const char* const value_str = ParseFlagValue(str, flag, true); @@ -219,7 +213,6 @@ bool ParseBoolFlag(const char* str, const char* flag, bool* value) { return true; } -BENCHMARK_EXPORT bool ParseInt32Flag(const char* str, const char* flag, int32_t* value) { // Gets the value of the flag as a string. const char* const value_str = ParseFlagValue(str, flag, false); @@ -232,7 +225,6 @@ bool ParseInt32Flag(const char* str, const char* flag, int32_t* value) { value); } -BENCHMARK_EXPORT bool ParseDoubleFlag(const char* str, const char* flag, double* value) { // Gets the value of the flag as a string. const char* const value_str = ParseFlagValue(str, flag, false); @@ -245,7 +237,6 @@ bool ParseDoubleFlag(const char* str, const char* flag, double* value) { value); } -BENCHMARK_EXPORT bool ParseStringFlag(const char* str, const char* flag, std::string* value) { // Gets the value of the flag as a string. const char* const value_str = ParseFlagValue(str, flag, false); @@ -257,7 +248,6 @@ bool ParseStringFlag(const char* str, const char* flag, std::string* value) { return true; } -BENCHMARK_EXPORT bool ParseKeyValueFlag(const char* str, const char* flag, std::map* value) { const char* const value_str = ParseFlagValue(str, flag, false); @@ -273,26 +263,23 @@ bool ParseKeyValueFlag(const char* str, const char* flag, return true; } -BENCHMARK_EXPORT bool IsFlag(const char* str, const char* flag) { return (ParseFlagValue(str, flag, true) != nullptr); } -BENCHMARK_EXPORT bool IsTruthyFlagValue(const std::string& value) { if (value.size() == 1) { char v = value[0]; return isalnum(v) && !(v == '0' || v == 'f' || v == 'F' || v == 'n' || v == 'N'); - } - if (!value.empty()) { + } else if (!value.empty()) { std::string value_lower(value); std::transform(value_lower.begin(), value_lower.end(), value_lower.begin(), [](char c) { return static_cast(::tolower(c)); }); return !(value_lower == "false" || value_lower == "no" || value_lower == "off"); - } - return true; + } else + return true; } } // end namespace benchmark diff --git a/third-party/benchmark/src/commandlineflags.h b/third-party/benchmark/src/commandlineflags.h index 7882628975ea..5baaf11784df 100644 --- a/third-party/benchmark/src/commandlineflags.h +++ b/third-party/benchmark/src/commandlineflags.h @@ -5,33 +5,28 @@ #include #include -#include "benchmark/export.h" - // Macro for referencing flags. #define FLAG(name) FLAGS_##name // Macros for declaring flags. -#define BM_DECLARE_bool(name) BENCHMARK_EXPORT extern bool FLAG(name) -#define BM_DECLARE_int32(name) BENCHMARK_EXPORT extern int32_t FLAG(name) -#define BM_DECLARE_double(name) BENCHMARK_EXPORT extern double FLAG(name) -#define BM_DECLARE_string(name) BENCHMARK_EXPORT extern std::string FLAG(name) +#define BM_DECLARE_bool(name) extern bool FLAG(name) +#define BM_DECLARE_int32(name) extern int32_t FLAG(name) +#define BM_DECLARE_double(name) extern double FLAG(name) +#define BM_DECLARE_string(name) extern std::string FLAG(name) #define BM_DECLARE_kvpairs(name) \ - BENCHMARK_EXPORT extern std::map FLAG(name) + extern std::map FLAG(name) // Macros for defining flags. #define BM_DEFINE_bool(name, default_val) \ - BENCHMARK_EXPORT bool FLAG(name) = benchmark::BoolFromEnv(#name, default_val) + bool FLAG(name) = benchmark::BoolFromEnv(#name, default_val) #define BM_DEFINE_int32(name, default_val) \ - BENCHMARK_EXPORT int32_t FLAG(name) = \ - benchmark::Int32FromEnv(#name, default_val) + int32_t FLAG(name) = benchmark::Int32FromEnv(#name, default_val) #define BM_DEFINE_double(name, default_val) \ - BENCHMARK_EXPORT double FLAG(name) = \ - benchmark::DoubleFromEnv(#name, default_val) + double FLAG(name) = benchmark::DoubleFromEnv(#name, default_val) #define BM_DEFINE_string(name, default_val) \ - BENCHMARK_EXPORT std::string FLAG(name) = \ - benchmark::StringFromEnv(#name, default_val) -#define BM_DEFINE_kvpairs(name, default_val) \ - BENCHMARK_EXPORT std::map FLAG(name) = \ + std::string FLAG(name) = benchmark::StringFromEnv(#name, default_val) +#define BM_DEFINE_kvpairs(name, default_val) \ + std::map FLAG(name) = \ benchmark::KvPairsFromEnv(#name, default_val) namespace benchmark { @@ -40,7 +35,6 @@ namespace benchmark { // // If the variable exists, returns IsTruthyFlagValue() value; if not, // returns the given default value. -BENCHMARK_EXPORT bool BoolFromEnv(const char* flag, bool default_val); // Parses an Int32 from the environment variable corresponding to the given @@ -48,7 +42,6 @@ bool BoolFromEnv(const char* flag, bool default_val); // // If the variable exists, returns ParseInt32() value; if not, returns // the given default value. -BENCHMARK_EXPORT int32_t Int32FromEnv(const char* flag, int32_t default_val); // Parses an Double from the environment variable corresponding to the given @@ -56,7 +49,6 @@ int32_t Int32FromEnv(const char* flag, int32_t default_val); // // If the variable exists, returns ParseDouble(); if not, returns // the given default value. -BENCHMARK_EXPORT double DoubleFromEnv(const char* flag, double default_val); // Parses a string from the environment variable corresponding to the given @@ -64,7 +56,6 @@ double DoubleFromEnv(const char* flag, double default_val); // // If variable exists, returns its value; if not, returns // the given default value. -BENCHMARK_EXPORT const char* StringFromEnv(const char* flag, const char* default_val); // Parses a set of kvpairs from the environment variable corresponding to the @@ -72,7 +63,6 @@ const char* StringFromEnv(const char* flag, const char* default_val); // // If variable exists, returns its value; if not, returns // the given default value. -BENCHMARK_EXPORT std::map KvPairsFromEnv( const char* flag, std::map default_val); @@ -85,47 +75,40 @@ std::map KvPairsFromEnv( // // On success, stores the value of the flag in *value, and returns // true. On failure, returns false without changing *value. -BENCHMARK_EXPORT bool ParseBoolFlag(const char* str, const char* flag, bool* value); // Parses a string for an Int32 flag, in the form of "--flag=value". // // On success, stores the value of the flag in *value, and returns // true. On failure, returns false without changing *value. -BENCHMARK_EXPORT bool ParseInt32Flag(const char* str, const char* flag, int32_t* value); // Parses a string for a Double flag, in the form of "--flag=value". // // On success, stores the value of the flag in *value, and returns // true. On failure, returns false without changing *value. -BENCHMARK_EXPORT bool ParseDoubleFlag(const char* str, const char* flag, double* value); // Parses a string for a string flag, in the form of "--flag=value". // // On success, stores the value of the flag in *value, and returns // true. On failure, returns false without changing *value. -BENCHMARK_EXPORT bool ParseStringFlag(const char* str, const char* flag, std::string* value); // Parses a string for a kvpairs flag in the form "--flag=key=value,key=value" // // On success, stores the value of the flag in *value and returns true. On // failure returns false, though *value may have been mutated. -BENCHMARK_EXPORT bool ParseKeyValueFlag(const char* str, const char* flag, std::map* value); // Returns true if the string matches the flag. -BENCHMARK_EXPORT bool IsFlag(const char* str, const char* flag); // Returns true unless value starts with one of: '0', 'f', 'F', 'n' or 'N', or // some non-alphanumeric character. Also returns false if the value matches // one of 'no', 'false', 'off' (case-insensitive). As a special case, also // returns true if value is the empty string. -BENCHMARK_EXPORT bool IsTruthyFlagValue(const std::string& value); } // end namespace benchmark diff --git a/third-party/benchmark/src/complexity.cc b/third-party/benchmark/src/complexity.cc index eee3122646f9..825c57394a8c 100644 --- a/third-party/benchmark/src/complexity.cc +++ b/third-party/benchmark/src/complexity.cc @@ -37,14 +37,12 @@ BigOFunc* FittingCurve(BigO complexity) { return [](IterationCount n) -> double { return std::pow(n, 3); }; case oLogN: /* Note: can't use log2 because Android's GNU STL lacks it */ - return [](IterationCount n) { - return kLog2E * std::log(static_cast(n)); - }; + return + [](IterationCount n) { return kLog2E * log(static_cast(n)); }; case oNLogN: /* Note: can't use log2 because Android's GNU STL lacks it */ return [](IterationCount n) { - return kLog2E * static_cast(n) * - std::log(static_cast(n)); + return kLog2E * n * log(static_cast(n)); }; case o1: default: @@ -77,12 +75,12 @@ std::string GetBigOString(BigO complexity) { // given by the lambda expression. // - n : Vector containing the size of the benchmark tests. // - time : Vector containing the times for the benchmark tests. -// - fitting_curve : lambda expression (e.g. [](ComplexityN n) {return n; };). +// - fitting_curve : lambda expression (e.g. [](int64_t n) {return n; };). // For a deeper explanation on the algorithm logic, please refer to // https://en.wikipedia.org/wiki/Least_squares#Least_squares,_regression_analysis_and_statistics -LeastSq MinimalLeastSq(const std::vector& n, +LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, BigOFunc* fitting_curve) { double sigma_gn_squared = 0.0; @@ -107,12 +105,12 @@ LeastSq MinimalLeastSq(const std::vector& n, double rms = 0.0; for (size_t i = 0; i < n.size(); ++i) { double fit = result.coef * fitting_curve(n[i]); - rms += std::pow((time[i] - fit), 2); + rms += pow((time[i] - fit), 2); } // Normalized RMS by the mean of the observed values - double mean = sigma_time / static_cast(n.size()); - result.rms = std::sqrt(rms / static_cast(n.size())) / mean; + double mean = sigma_time / n.size(); + result.rms = sqrt(rms / n.size()) / mean; return result; } @@ -124,7 +122,7 @@ LeastSq MinimalLeastSq(const std::vector& n, // - complexity : If different than oAuto, the fitting curve will stick to // this one. If it is oAuto, it will be calculated the best // fitting curve. -LeastSq MinimalLeastSq(const std::vector& n, +LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, const BigO complexity) { BM_CHECK_EQ(n.size(), time.size()); BM_CHECK_GE(n.size(), 2); // Do not compute fitting curve is less than two @@ -164,7 +162,7 @@ std::vector ComputeBigO( if (reports.size() < 2) return results; // Accumulators. - std::vector n; + std::vector n; std::vector real_time; std::vector cpu_time; @@ -173,10 +171,8 @@ std::vector ComputeBigO( BM_CHECK_GT(run.complexity_n, 0) << "Did you forget to call SetComplexityN?"; n.push_back(run.complexity_n); - real_time.push_back(run.real_accumulated_time / - static_cast(run.iterations)); - cpu_time.push_back(run.cpu_accumulated_time / - static_cast(run.iterations)); + real_time.push_back(run.real_accumulated_time / run.iterations); + cpu_time.push_back(run.cpu_accumulated_time / run.iterations); } LeastSq result_cpu; @@ -186,19 +182,8 @@ std::vector ComputeBigO( result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity_lambda); result_real = MinimalLeastSq(n, real_time, reports[0].complexity_lambda); } else { - const BigO* InitialBigO = &reports[0].complexity; - const bool use_real_time_for_initial_big_o = - reports[0].use_real_time_for_initial_big_o; - if (use_real_time_for_initial_big_o) { - result_real = MinimalLeastSq(n, real_time, *InitialBigO); - InitialBigO = &result_real.complexity; - // The Big-O complexity for CPU time must have the same Big-O function! - } - result_cpu = MinimalLeastSq(n, cpu_time, *InitialBigO); - InitialBigO = &result_cpu.complexity; - if (!use_real_time_for_initial_big_o) { - result_real = MinimalLeastSq(n, real_time, *InitialBigO); - } + result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity); + result_real = MinimalLeastSq(n, real_time, result_cpu.complexity); } // Drop the 'args' when reporting complexity. diff --git a/third-party/benchmark/src/complexity.h b/third-party/benchmark/src/complexity.h index 0a0679b48bc8..df29b48d29b4 100644 --- a/third-party/benchmark/src/complexity.h +++ b/third-party/benchmark/src/complexity.h @@ -31,7 +31,7 @@ std::vector ComputeBigO( const std::vector& reports); // This data structure will contain the result returned by MinimalLeastSq -// - coef : Estimated coefficient for the high-order term as +// - coef : Estimated coeficient for the high-order term as // interpolated from data. // - rms : Normalized Root Mean Squared Error. // - complexity : Scalability form (e.g. oN, oNLogN). In case a scalability diff --git a/third-party/benchmark/src/console_reporter.cc b/third-party/benchmark/src/console_reporter.cc index 35c3de2a4dba..04cc0b74e58e 100644 --- a/third-party/benchmark/src/console_reporter.cc +++ b/third-party/benchmark/src/console_reporter.cc @@ -33,7 +33,6 @@ namespace benchmark { -BENCHMARK_EXPORT bool ConsoleReporter::ReportContext(const Context& context) { name_field_width_ = context.name_field_width; printed_header_ = false; @@ -42,22 +41,17 @@ bool ConsoleReporter::ReportContext(const Context& context) { PrintBasicContext(&GetErrorStream(), context); #ifdef BENCHMARK_OS_WINDOWS - if ((output_options_ & OO_Color)) { - auto stdOutBuf = std::cout.rdbuf(); - auto outStreamBuf = GetOutputStream().rdbuf(); - if (stdOutBuf != outStreamBuf) { - GetErrorStream() - << "Color printing is only supported for stdout on windows." - " Disabling color printing\n"; - output_options_ = static_cast(output_options_ & ~OO_Color); - } + if ((output_options_ & OO_Color) && &std::cout != &GetOutputStream()) { + GetErrorStream() + << "Color printing is only supported for stdout on windows." + " Disabling color printing\n"; + output_options_ = static_cast(output_options_ & ~OO_Color); } #endif return true; } -BENCHMARK_EXPORT void ConsoleReporter::PrintHeader(const Run& run) { std::string str = FormatString("%-*s %13s %15s %12s", static_cast(name_field_width_), @@ -75,7 +69,6 @@ void ConsoleReporter::PrintHeader(const Run& run) { GetOutputStream() << line << "\n" << str << "\n" << line << "\n"; } -BENCHMARK_EXPORT void ConsoleReporter::ReportRuns(const std::vector& reports) { for (const auto& run : reports) { // print the header: @@ -106,9 +99,6 @@ static void IgnoreColorPrint(std::ostream& out, LogColor, const char* fmt, } static std::string FormatTime(double time) { - // For the time columns of the console printer 13 digits are reserved. One of - // them is a space and max two of them are the time unit (e.g ns). That puts - // us at 10 digits usable for the number. // Align decimal places... if (time < 1.0) { return FormatString("%10.3f", time); @@ -119,15 +109,9 @@ static std::string FormatTime(double time) { if (time < 100.0) { return FormatString("%10.1f", time); } - // Assuming the time is at max 9.9999e+99 and we have 10 digits for the - // number, we get 10-1(.)-1(e)-1(sign)-2(exponent) = 5 digits to print. - if (time > 9999999999 /*max 10 digit number*/) { - return FormatString("%1.4e", time); - } return FormatString("%10.0f", time); } -BENCHMARK_EXPORT void ConsoleReporter::PrintRunData(const Run& result) { typedef void(PrinterFn)(std::ostream&, LogColor, const char*, ...); auto& Out = GetOutputStream(); @@ -139,13 +123,9 @@ void ConsoleReporter::PrintRunData(const Run& result) { printer(Out, name_color, "%-*s ", name_field_width_, result.benchmark_name().c_str()); - if (internal::SkippedWithError == result.skipped) { + if (result.error_occurred) { printer(Out, COLOR_RED, "ERROR OCCURRED: \'%s\'", - result.skip_message.c_str()); - printer(Out, COLOR_DEFAULT, "\n"); - return; - } else if (internal::SkippedWithMessage == result.skipped) { - printer(Out, COLOR_WHITE, "SKIPPED: \'%s\'", result.skip_message.c_str()); + result.error_message.c_str()); printer(Out, COLOR_DEFAULT, "\n"); return; } diff --git a/third-party/benchmark/src/counter.cc b/third-party/benchmark/src/counter.cc index aa14cd8092f9..cf5b78ee3ac6 100644 --- a/third-party/benchmark/src/counter.cc +++ b/third-party/benchmark/src/counter.cc @@ -27,10 +27,10 @@ double Finish(Counter const& c, IterationCount iterations, double cpu_time, v /= num_threads; } if (c.flags & Counter::kIsIterationInvariant) { - v *= static_cast(iterations); + v *= iterations; } if (c.flags & Counter::kAvgIterations) { - v /= static_cast(iterations); + v /= iterations; } if (c.flags & Counter::kInvert) { // Invert is *always* last. diff --git a/third-party/benchmark/src/csv_reporter.cc b/third-party/benchmark/src/csv_reporter.cc index 4b39e2c52fb9..1c5e9fa6689c 100644 --- a/third-party/benchmark/src/csv_reporter.cc +++ b/third-party/benchmark/src/csv_reporter.cc @@ -52,13 +52,11 @@ std::string CsvEscape(const std::string& s) { return '"' + tmp + '"'; } -BENCHMARK_EXPORT bool CSVReporter::ReportContext(const Context& context) { PrintBasicContext(&GetErrorStream(), context); return true; } -BENCHMARK_EXPORT void CSVReporter::ReportRuns(const std::vector& reports) { std::ostream& Out = GetOutputStream(); @@ -105,14 +103,13 @@ void CSVReporter::ReportRuns(const std::vector& reports) { } } -BENCHMARK_EXPORT void CSVReporter::PrintRunData(const Run& run) { std::ostream& Out = GetOutputStream(); Out << CsvEscape(run.benchmark_name()) << ","; - if (run.skipped) { + if (run.error_occurred) { Out << std::string(elements.size() - 3, ','); - Out << std::boolalpha << (internal::SkippedWithError == run.skipped) << ","; - Out << CsvEscape(run.skip_message) << "\n"; + Out << "true,"; + Out << CsvEscape(run.error_message) << "\n"; return; } @@ -122,21 +119,13 @@ void CSVReporter::PrintRunData(const Run& run) { } Out << ","; - if (run.run_type != Run::RT_Aggregate || - run.aggregate_unit == StatisticUnit::kTime) { - Out << run.GetAdjustedRealTime() << ","; - Out << run.GetAdjustedCPUTime() << ","; - } else { - assert(run.aggregate_unit == StatisticUnit::kPercentage); - Out << run.real_accumulated_time << ","; - Out << run.cpu_accumulated_time << ","; - } + Out << run.GetAdjustedRealTime() << ","; + Out << run.GetAdjustedCPUTime() << ","; // Do not print timeLabel on bigO and RMS report if (run.report_big_o) { Out << GetBigOString(run.complexity); - } else if (!run.report_rms && - run.aggregate_unit != StatisticUnit::kPercentage) { + } else if (!run.report_rms) { Out << GetTimeUnitString(run.time_unit); } Out << ","; diff --git a/third-party/benchmark/src/cycleclock.h b/third-party/benchmark/src/cycleclock.h index eff563e7fac2..d65d32a39d3d 100644 --- a/third-party/benchmark/src/cycleclock.h +++ b/third-party/benchmark/src/cycleclock.h @@ -36,8 +36,7 @@ // declarations of some other intrinsics, breaking compilation. // Therefore, we simply declare __rdtsc ourselves. See also // http://connect.microsoft.com/VisualStudio/feedback/details/262047 -#if defined(COMPILER_MSVC) && !defined(_M_IX86) && !defined(_M_ARM64) && \ - !defined(_M_ARM64EC) +#if defined(COMPILER_MSVC) && !defined(_M_IX86) && !defined(_M_ARM64) extern "C" uint64_t __rdtsc(); #pragma intrinsic(__rdtsc) #endif @@ -115,7 +114,7 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { // when I know it will work. Otherwise, I'll use __rdtsc and hope // the code is being compiled with a non-ancient compiler. _asm rdtsc -#elif defined(COMPILER_MSVC) && (defined(_M_ARM64) || defined(_M_ARM64EC)) +#elif defined(COMPILER_MSVC) && defined(_M_ARM64) // See // https://docs.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics // and https://reviews.llvm.org/D53115 int64_t virtual_timer_value; @@ -133,7 +132,7 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { // Native Client does not provide any API to access cycle counter. // Use clock_gettime(CLOCK_MONOTONIC, ...) instead of gettimeofday - // because is provides nanosecond resolution (which is noticeable at + // because is provides nanosecond resolution (which is noticable at // least for PNaCl modules running on x86 Mac & Linux). // Initialize to always return 0 if clock_gettime fails. struct timespec ts = {0, 0}; @@ -174,7 +173,7 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { struct timeval tv; gettimeofday(&tv, nullptr); return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; -#elif defined(__loongarch__) || defined(__csky__) +#elif defined(__loongarch__) struct timeval tv; gettimeofday(&tv, nullptr); return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; @@ -189,16 +188,15 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { #endif return tsc; #elif defined(__riscv) // RISC-V - // Use RDTIME (and RDTIMEH on riscv32). - // RDCYCLE is a privileged instruction since Linux 6.6. + // Use RDCYCLE (and RDCYCLEH on riscv32) #if __riscv_xlen == 32 uint32_t cycles_lo, cycles_hi0, cycles_hi1; // This asm also includes the PowerPC overflow handling strategy, as above. // Implemented in assembly because Clang insisted on branching. asm volatile( - "rdtimeh %0\n" - "rdtime %1\n" - "rdtimeh %2\n" + "rdcycleh %0\n" + "rdcycle %1\n" + "rdcycleh %2\n" "sub %0, %0, %2\n" "seqz %0, %0\n" "sub %0, zero, %0\n" @@ -207,31 +205,17 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { return (static_cast(cycles_hi1) << 32) | cycles_lo; #else uint64_t cycles; - asm volatile("rdtime %0" : "=r"(cycles)); + asm volatile("rdcycle %0" : "=r"(cycles)); return cycles; #endif #elif defined(__e2k__) || defined(__elbrus__) struct timeval tv; gettimeofday(&tv, nullptr); return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; -#elif defined(__hexagon__) - uint64_t pcycle; - asm volatile("%0 = C15:14" : "=r"(pcycle)); - return static_cast(pcycle); -#elif defined(__alpha__) - // Alpha has a cycle counter, the PCC register, but it is an unsigned 32-bit - // integer and thus wraps every ~4s, making using it for tick counts - // unreliable beyond this time range. The real-time clock is low-precision, - // roughtly ~1ms, but it is the only option that can reasonable count - // indefinitely. - struct timeval tv; - gettimeofday(&tv, nullptr); - return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; #else - // The soft failover to a generic implementation is automatic only for ARM. - // For other platforms the developer is expected to make an attempt to create - // a fast implementation and use generic version if nothing better is - // available. +// The soft failover to a generic implementation is automatic only for ARM. +// For other platforms the developer is expected to make an attempt to create +// a fast implementation and use generic version if nothing better is available. #error You need to define CycleTimer for your OS and CPU #endif } diff --git a/third-party/benchmark/src/internal_macros.h b/third-party/benchmark/src/internal_macros.h index 8dd7d0c6502e..91f367b894bc 100644 --- a/third-party/benchmark/src/internal_macros.h +++ b/third-party/benchmark/src/internal_macros.h @@ -1,6 +1,8 @@ #ifndef BENCHMARK_INTERNAL_MACROS_H_ #define BENCHMARK_INTERNAL_MACROS_H_ +#include "benchmark/benchmark.h" + /* Needed to detect STL */ #include @@ -42,19 +44,6 @@ #define BENCHMARK_OS_CYGWIN 1 #elif defined(_WIN32) #define BENCHMARK_OS_WINDOWS 1 - // WINAPI_FAMILY_PARTITION is defined in winapifamily.h. - // We include windows.h which implicitly includes winapifamily.h for compatibility. - #ifndef NOMINMAX - #define NOMINMAX - #endif - #include - #if defined(WINAPI_FAMILY_PARTITION) - #if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) - #define BENCHMARK_OS_WINDOWS_WIN32 1 - #elif WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) - #define BENCHMARK_OS_WINDOWS_RT 1 - #endif - #endif #if defined(__MINGW32__) #define BENCHMARK_OS_MINGW 1 #endif @@ -91,8 +80,6 @@ #define BENCHMARK_OS_QNX 1 #elif defined(__MVS__) #define BENCHMARK_OS_ZOS 1 -#elif defined(__hexagon__) -#define BENCHMARK_OS_QURT 1 #endif #if defined(__ANDROID__) && defined(__GLIBCXX__) diff --git a/third-party/benchmark/src/json_reporter.cc b/third-party/benchmark/src/json_reporter.cc index b8c8c94c08a0..e84a4ed24f9d 100644 --- a/third-party/benchmark/src/json_reporter.cc +++ b/third-party/benchmark/src/json_reporter.cc @@ -28,6 +28,10 @@ #include "timers.h" namespace benchmark { +namespace internal { +extern std::map* global_context; +} + namespace { std::string StrEscape(const std::string& s) { @@ -85,6 +89,12 @@ std::string FormatKV(std::string const& key, int64_t value) { return ss.str(); } +std::string FormatKV(std::string const& key, IterationCount value) { + std::stringstream ss; + ss << '"' << StrEscape(key) << "\": " << value; + return ss.str(); +} + std::string FormatKV(std::string const& key, double value) { std::stringstream ss; ss << '"' << StrEscape(key) << "\": "; @@ -167,25 +177,15 @@ bool JSONReporter::ReportContext(const Context& context) { } out << "],\n"; - out << indent << FormatKV("library_version", GetBenchmarkVersion()); - out << ",\n"; - #if defined(NDEBUG) const char build_type[] = "release"; #else const char build_type[] = "debug"; #endif out << indent << FormatKV("library_build_type", build_type); - out << ",\n"; - - // NOTE: our json schema is not strictly tied to the library version! - out << indent << FormatKV("json_schema_version", int64_t(1)); - - std::map* global_context = - internal::GetGlobalContext(); - if (global_context != nullptr) { - for (const auto& kv : *global_context) { + if (internal::global_context != nullptr) { + for (const auto& kv : *internal::global_context) { out << ",\n"; out << indent << FormatKV(kv.first, kv.second); } @@ -261,12 +261,9 @@ void JSONReporter::PrintRunData(Run const& run) { BENCHMARK_UNREACHABLE(); }()) << ",\n"; } - if (internal::SkippedWithError == run.skipped) { - out << indent << FormatKV("error_occurred", true) << ",\n"; - out << indent << FormatKV("error_message", run.skip_message) << ",\n"; - } else if (internal::SkippedWithMessage == run.skipped) { - out << indent << FormatKV("skipped", true) << ",\n"; - out << indent << FormatKV("skip_message", run.skip_message) << ",\n"; + if (run.error_occurred) { + out << indent << FormatKV("error_occurred", run.error_occurred) << ",\n"; + out << indent << FormatKV("error_message", run.error_message) << ",\n"; } if (!run.report_big_o && !run.report_rms) { out << indent << FormatKV("iterations", run.iterations) << ",\n"; @@ -304,8 +301,7 @@ void JSONReporter::PrintRunData(Run const& run) { out << ",\n" << indent << FormatKV("max_bytes_used", memory_result.max_bytes_used); - auto report_if_present = [&out, &indent](const std::string& label, - int64_t val) { + auto report_if_present = [&out, &indent](const char* label, int64_t val) { if (val != MemoryManager::TombstoneValue) out << ",\n" << indent << FormatKV(label, val); }; diff --git a/third-party/benchmark/src/log.h b/third-party/benchmark/src/log.h index 9a21400b096d..48c071aded8f 100644 --- a/third-party/benchmark/src/log.h +++ b/third-party/benchmark/src/log.h @@ -4,12 +4,7 @@ #include #include -// NOTE: this is also defined in benchmark.h but we're trying to avoid a -// dependency. -// The _MSVC_LANG check should detect Visual Studio 2015 Update 3 and newer. -#if __cplusplus >= 201103L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201103L) -#define BENCHMARK_HAS_CXX11 -#endif +#include "benchmark/benchmark.h" namespace benchmark { namespace internal { @@ -28,16 +23,7 @@ class LogType { private: LogType(std::ostream* out) : out_(out) {} std::ostream* out_; - - // NOTE: we could use BENCHMARK_DISALLOW_COPY_AND_ASSIGN but we shouldn't have - // a dependency on benchmark.h from here. -#ifndef BENCHMARK_HAS_CXX11 - LogType(const LogType&); - LogType& operator=(const LogType&); -#else - LogType(const LogType&) = delete; - LogType& operator=(const LogType&) = delete; -#endif + BENCHMARK_DISALLOW_COPY_AND_ASSIGN(LogType); }; template @@ -61,13 +47,13 @@ inline int& LogLevel() { } inline LogType& GetNullLogInstance() { - static LogType null_log(static_cast(nullptr)); - return null_log; + static LogType log(nullptr); + return log; } inline LogType& GetErrorLogInstance() { - static LogType error_log(&std::clog); - return error_log; + static LogType log(&std::clog); + return log; } inline LogType& GetLogInstanceForLevel(int level) { diff --git a/third-party/benchmark/src/perf_counters.cc b/third-party/benchmark/src/perf_counters.cc index d466e27e86f9..b2ac7687efef 100644 --- a/third-party/benchmark/src/perf_counters.cc +++ b/third-party/benchmark/src/perf_counters.cc @@ -15,7 +15,6 @@ #include "perf_counters.h" #include -#include #include #if defined HAVE_LIBPFM @@ -29,254 +28,105 @@ namespace internal { constexpr size_t PerfCounterValues::kMaxCounters; #if defined HAVE_LIBPFM - -size_t PerfCounterValues::Read(const std::vector& leaders) { - // Create a pointer for multiple reads - const size_t bufsize = values_.size() * sizeof(values_[0]); - char* ptr = reinterpret_cast(values_.data()); - size_t size = bufsize; - for (int lead : leaders) { - auto read_bytes = ::read(lead, ptr, size); - if (read_bytes >= ssize_t(sizeof(uint64_t))) { - // Actual data bytes are all bytes minus initial padding - std::size_t data_bytes = read_bytes - sizeof(uint64_t); - // This should be very cheap since it's in hot cache - std::memmove(ptr, ptr + sizeof(uint64_t), data_bytes); - // Increment our counters - ptr += data_bytes; - size -= data_bytes; - } else { - int err = errno; - GetErrorLogInstance() << "Error reading lead " << lead << " errno:" << err - << " " << ::strerror(err) << "\n"; - return 0; - } - } - return (bufsize - size) / sizeof(uint64_t); -} - const bool PerfCounters::kSupported = true; -// Initializes libpfm only on the first call. Returns whether that single -// initialization was successful. -bool PerfCounters::Initialize() { - // Function-scope static gets initialized only once on first call. - static const bool success = []() { - return pfm_initialize() == PFM_SUCCESS; - }(); - return success; -} - -bool PerfCounters::IsCounterSupported(const std::string& name) { - Initialize(); - perf_event_attr_t attr; - std::memset(&attr, 0, sizeof(attr)); - pfm_perf_encode_arg_t arg; - std::memset(&arg, 0, sizeof(arg)); - arg.attr = &attr; - const int mode = PFM_PLM3; // user mode only - int ret = pfm_get_os_event_encoding(name.c_str(), mode, PFM_OS_PERF_EVENT_EXT, - &arg); - return (ret == PFM_SUCCESS); -} +bool PerfCounters::Initialize() { return pfm_initialize() == PFM_SUCCESS; } PerfCounters PerfCounters::Create( const std::vector& counter_names) { - if (!counter_names.empty()) { - Initialize(); + if (counter_names.empty()) { + return NoCounters(); } + if (counter_names.size() > PerfCounterValues::kMaxCounters) { + GetErrorLogInstance() + << counter_names.size() + << " counters were requested. The minimum is 1, the maximum is " + << PerfCounterValues::kMaxCounters << "\n"; + return NoCounters(); + } + std::vector counter_ids(counter_names.size()); - // Valid counters will populate these arrays but we start empty - std::vector valid_names; - std::vector counter_ids; - std::vector leader_ids; - - // Resize to the maximum possible - valid_names.reserve(counter_names.size()); - counter_ids.reserve(counter_names.size()); - - const int kCounterMode = PFM_PLM3; // user mode only - - // Group leads will be assigned on demand. The idea is that once we cannot - // create a counter descriptor, the reason is that this group has maxed out - // so we set the group_id again to -1 and retry - giving the algorithm a - // chance to create a new group leader to hold the next set of counters. - int group_id = -1; - - // Loop through all performance counters + const int mode = PFM_PLM3; // user mode only for (size_t i = 0; i < counter_names.size(); ++i) { - // we are about to push into the valid names vector - // check if we did not reach the maximum - if (valid_names.size() == PerfCounterValues::kMaxCounters) { - // Log a message if we maxed out and stop adding - GetErrorLogInstance() - << counter_names.size() << " counters were requested. The maximum is " - << PerfCounterValues::kMaxCounters << " and " << valid_names.size() - << " were already added. All remaining counters will be ignored\n"; - // stop the loop and return what we have already - break; - } - - // Check if this name is empty + const bool is_first = i == 0; + struct perf_event_attr attr {}; + attr.size = sizeof(attr); + const int group_id = !is_first ? counter_ids[0] : -1; const auto& name = counter_names[i]; if (name.empty()) { - GetErrorLogInstance() - << "A performance counter name was the empty string\n"; - continue; + GetErrorLogInstance() << "A counter name was the empty string\n"; + return NoCounters(); } - - // Here first means first in group, ie the group leader - const bool is_first = (group_id < 0); - - // This struct will be populated by libpfm from the counter string - // and then fed into the syscall perf_event_open - struct perf_event_attr attr {}; - attr.size = sizeof(attr); - - // This is the input struct to libpfm. pfm_perf_encode_arg_t arg{}; arg.attr = &attr; - const int pfm_get = pfm_get_os_event_encoding(name.c_str(), kCounterMode, - PFM_OS_PERF_EVENT, &arg); + + const int pfm_get = + pfm_get_os_event_encoding(name.c_str(), mode, PFM_OS_PERF_EVENT, &arg); if (pfm_get != PFM_SUCCESS) { - GetErrorLogInstance() - << "Unknown performance counter name: " << name << "\n"; - continue; + GetErrorLogInstance() << "Unknown counter name: " << name << "\n"; + return NoCounters(); } - - // We then proceed to populate the remaining fields in our attribute struct - // Note: the man page for perf_event_create suggests inherit = true and + attr.disabled = is_first; + // Note: the man page for perf_event_create suggests inerit = true and // read_format = PERF_FORMAT_GROUP don't work together, but that's not the // case. - attr.disabled = is_first; attr.inherit = true; attr.pinned = is_first; attr.exclude_kernel = true; attr.exclude_user = false; attr.exclude_hv = true; - - // Read all counters in a group in one read. + // Read all counters in one read. attr.read_format = PERF_FORMAT_GROUP; int id = -1; - while (id < 0) { - static constexpr size_t kNrOfSyscallRetries = 5; - // Retry syscall as it was interrupted often (b/64774091). - for (size_t num_retries = 0; num_retries < kNrOfSyscallRetries; - ++num_retries) { - id = perf_event_open(&attr, 0, -1, group_id, 0); - if (id >= 0 || errno != EINTR) { - break; - } - } - if (id < 0) { - // If the file descriptor is negative we might have reached a limit - // in the current group. Set the group_id to -1 and retry - if (group_id >= 0) { - // Create a new group - group_id = -1; - } else { - // At this point we have already retried to set a new group id and - // failed. We then give up. - break; - } + static constexpr size_t kNrOfSyscallRetries = 5; + // Retry syscall as it was interrupted often (b/64774091). + for (size_t num_retries = 0; num_retries < kNrOfSyscallRetries; + ++num_retries) { + id = perf_event_open(&attr, 0, -1, group_id, 0); + if (id >= 0 || errno != EINTR) { + break; } } - - // We failed to get a new file descriptor. We might have reached a hard - // hardware limit that cannot be resolved even with group multiplexing if (id < 0) { - GetErrorLogInstance() << "***WARNING** Failed to get a file descriptor " - "for performance counter " - << name << ". Ignoring\n"; - - // We give up on this counter but try to keep going - // as the others would be fine - continue; - } - if (group_id < 0) { - // This is a leader, store and assign it to the current file descriptor - leader_ids.push_back(id); - group_id = id; - } - // This is a valid counter, add it to our descriptor's list - counter_ids.push_back(id); - valid_names.push_back(name); - } - - // Loop through all group leaders activating them - // There is another option of starting ALL counters in a process but - // that would be far reaching an intrusion. If the user is using PMCs - // by themselves then this would have a side effect on them. It is - // friendlier to loop through all groups individually. - for (int lead : leader_ids) { - if (ioctl(lead, PERF_EVENT_IOC_ENABLE) != 0) { - // This should never happen but if it does, we give up on the - // entire batch as recovery would be a mess. - GetErrorLogInstance() << "***WARNING*** Failed to start counters. " - "Claring out all counters.\n"; - - // Close all peformance counters - for (int id : counter_ids) { - ::close(id); - } - - // Return an empty object so our internal state is still good and - // the process can continue normally without impact + GetErrorLogInstance() + << "Failed to get a file descriptor for " << name << "\n"; return NoCounters(); } + + counter_ids[i] = id; + } + if (ioctl(counter_ids[0], PERF_EVENT_IOC_ENABLE) != 0) { + GetErrorLogInstance() << "Failed to start counters\n"; + return NoCounters(); } - return PerfCounters(std::move(valid_names), std::move(counter_ids), - std::move(leader_ids)); + return PerfCounters(counter_names, std::move(counter_ids)); } -void PerfCounters::CloseCounters() const { +PerfCounters::~PerfCounters() { if (counter_ids_.empty()) { return; } - for (int lead : leader_ids_) { - ioctl(lead, PERF_EVENT_IOC_DISABLE); - } + ioctl(counter_ids_[0], PERF_EVENT_IOC_DISABLE); for (int fd : counter_ids_) { close(fd); } } #else // defined HAVE_LIBPFM -size_t PerfCounterValues::Read(const std::vector&) { return 0; } - const bool PerfCounters::kSupported = false; bool PerfCounters::Initialize() { return false; } -bool PerfCounters::IsCounterSupported(const std::string&) { return false; } - PerfCounters PerfCounters::Create( const std::vector& counter_names) { if (!counter_names.empty()) { - GetErrorLogInstance() << "Performance counters not supported.\n"; + GetErrorLogInstance() << "Performance counters not supported."; } return NoCounters(); } -void PerfCounters::CloseCounters() const {} +PerfCounters::~PerfCounters() = default; #endif // defined HAVE_LIBPFM - -PerfCountersMeasurement::PerfCountersMeasurement( - const std::vector& counter_names) - : start_values_(counter_names.size()), end_values_(counter_names.size()) { - counters_ = PerfCounters::Create(counter_names); -} - -PerfCounters& PerfCounters::operator=(PerfCounters&& other) noexcept { - if (this != &other) { - CloseCounters(); - - counter_ids_ = std::move(other.counter_ids_); - leader_ids_ = std::move(other.leader_ids_); - counter_names_ = std::move(other.counter_names_); - } - return *this; -} } // namespace internal } // namespace benchmark diff --git a/third-party/benchmark/src/perf_counters.h b/third-party/benchmark/src/perf_counters.h index bf5eb6bc3aec..47ca1385e24d 100644 --- a/third-party/benchmark/src/perf_counters.h +++ b/third-party/benchmark/src/perf_counters.h @@ -17,25 +17,16 @@ #include #include -#include -#include #include #include "benchmark/benchmark.h" #include "check.h" #include "log.h" -#include "mutex.h" #ifndef BENCHMARK_OS_WINDOWS #include #endif -#if defined(_MSC_VER) -#pragma warning(push) -// C4251: needs to have dll-interface to be used by clients of class -#pragma warning(disable : 4251) -#endif - namespace benchmark { namespace internal { @@ -45,21 +36,18 @@ namespace internal { // The implementation ensures the storage is inlined, and allows 0-based // indexing into the counter values. // The object is used in conjunction with a PerfCounters object, by passing it -// to Snapshot(). The Read() method relocates individual reads, discarding -// the initial padding from each group leader in the values buffer such that -// all user accesses through the [] operator are correct. -class BENCHMARK_EXPORT PerfCounterValues { +// to Snapshot(). The values are populated such that +// perfCounters->names()[i]'s value is obtained at position i (as given by +// operator[]) of this object. +class PerfCounterValues { public: explicit PerfCounterValues(size_t nr_counters) : nr_counters_(nr_counters) { BM_CHECK_LE(nr_counters_, kMaxCounters); } - // We are reading correctly now so the values don't need to skip padding - uint64_t operator[](size_t pos) const { return values_[pos]; } + uint64_t operator[](size_t pos) const { return values_[kPadding + pos]; } - // Increased the maximum to 32 only since the buffer - // is std::array<> backed - static constexpr size_t kMaxCounters = 32; + static constexpr size_t kMaxCounters = 3; private: friend class PerfCounters; @@ -70,14 +58,7 @@ class BENCHMARK_EXPORT PerfCounterValues { sizeof(uint64_t) * (kPadding + nr_counters_)}; } - // This reading is complex and as the goal of this class is to - // abstract away the intrincacies of the reading process, this is - // a better place for it - size_t Read(const std::vector& leaders); - - // Move the padding to 2 due to the reading algorithm (1st padding plus a - // current read padding) - static constexpr size_t kPadding = 2; + static constexpr size_t kPadding = 1; std::array values_; const size_t nr_counters_; }; @@ -85,34 +66,27 @@ class BENCHMARK_EXPORT PerfCounterValues { // Collect PMU counters. The object, once constructed, is ready to be used by // calling read(). PMU counter collection is enabled from the time create() is // called, to obtain the object, until the object's destructor is called. -class BENCHMARK_EXPORT PerfCounters final { +class PerfCounters final { public: // True iff this platform supports performance counters. static const bool kSupported; - // Returns an empty object + bool IsValid() const { return is_valid_; } static PerfCounters NoCounters() { return PerfCounters(); } - ~PerfCounters() { CloseCounters(); } - PerfCounters() = default; + ~PerfCounters(); PerfCounters(PerfCounters&&) = default; PerfCounters(const PerfCounters&) = delete; - PerfCounters& operator=(PerfCounters&&) noexcept; - PerfCounters& operator=(const PerfCounters&) = delete; // Platform-specific implementations may choose to do some library // initialization here. static bool Initialize(); - // Check if the given counter is supported, if the app wants to - // check before passing - static bool IsCounterSupported(const std::string& name); - // Return a PerfCounters object ready to read the counters with the names // specified. The values are user-mode only. The counter name format is // implementation and OS specific. - // In case of failure, this method will in the worst case return an - // empty object whose state will still be valid. + // TODO: once we move to C++-17, this should be a std::optional, and then the + // IsValid() boolean can be dropped. static PerfCounters Create(const std::vector& counter_names); // Take a snapshot of the current value of the counters into the provided @@ -121,7 +95,10 @@ class BENCHMARK_EXPORT PerfCounters final { BENCHMARK_ALWAYS_INLINE bool Snapshot(PerfCounterValues* values) const { #ifndef BENCHMARK_OS_WINDOWS assert(values != nullptr); - return values->Read(leader_ids_) == counter_ids_.size(); + assert(IsValid()); + auto buffer = values->get_data_buffer(); + auto read_bytes = ::read(counter_ids_[0], buffer.first, buffer.second); + return static_cast(read_bytes) == buffer.second; #else (void)values; return false; @@ -133,68 +110,63 @@ class BENCHMARK_EXPORT PerfCounters final { private: PerfCounters(const std::vector& counter_names, - std::vector&& counter_ids, std::vector&& leader_ids) + std::vector&& counter_ids) : counter_ids_(std::move(counter_ids)), - leader_ids_(std::move(leader_ids)), - counter_names_(counter_names) {} - - void CloseCounters() const; + counter_names_(counter_names), + is_valid_(true) {} + PerfCounters() : is_valid_(false) {} std::vector counter_ids_; - std::vector leader_ids_; - std::vector counter_names_; + const std::vector counter_names_; + const bool is_valid_; }; // Typical usage of the above primitives. -class BENCHMARK_EXPORT PerfCountersMeasurement final { +class PerfCountersMeasurement final { public: - PerfCountersMeasurement(const std::vector& counter_names); - - size_t num_counters() const { return counters_.num_counters(); } + PerfCountersMeasurement(PerfCounters&& c) + : counters_(std::move(c)), + start_values_(counters_.IsValid() ? counters_.names().size() : 0), + end_values_(counters_.IsValid() ? counters_.names().size() : 0) {} - std::vector names() const { return counters_.names(); } + bool IsValid() const { return counters_.IsValid(); } - BENCHMARK_ALWAYS_INLINE bool Start() { - if (num_counters() == 0) return true; + BENCHMARK_ALWAYS_INLINE void Start() { + assert(IsValid()); // Tell the compiler to not move instructions above/below where we take // the snapshot. ClobberMemory(); - valid_read_ &= counters_.Snapshot(&start_values_); + counters_.Snapshot(&start_values_); ClobberMemory(); - - return valid_read_; } - BENCHMARK_ALWAYS_INLINE bool Stop( - std::vector>& measurements) { - if (num_counters() == 0) return true; + BENCHMARK_ALWAYS_INLINE std::vector> + StopAndGetMeasurements() { + assert(IsValid()); // Tell the compiler to not move instructions above/below where we take // the snapshot. ClobberMemory(); - valid_read_ &= counters_.Snapshot(&end_values_); + counters_.Snapshot(&end_values_); ClobberMemory(); + std::vector> ret; for (size_t i = 0; i < counters_.names().size(); ++i) { double measurement = static_cast(end_values_[i]) - static_cast(start_values_[i]); - measurements.push_back({counters_.names()[i], measurement}); + ret.push_back({counters_.names()[i], measurement}); } - - return valid_read_; + return ret; } private: PerfCounters counters_; - bool valid_read_ = true; PerfCounterValues start_values_; PerfCounterValues end_values_; }; +BENCHMARK_UNUSED static bool perf_init_anchor = PerfCounters::Initialize(); + } // namespace internal } // namespace benchmark -#if defined(_MSC_VER) -#pragma warning(pop) -#endif - #endif // BENCHMARK_PERF_COUNTERS_H diff --git a/third-party/benchmark/src/re.h b/third-party/benchmark/src/re.h index 9afb869bea27..630046782dc4 100644 --- a/third-party/benchmark/src/re.h +++ b/third-party/benchmark/src/re.h @@ -33,7 +33,7 @@ // Prefer C regex libraries when compiling w/o exceptions so that we can // correctly report errors. #if defined(BENCHMARK_HAS_NO_EXCEPTIONS) && \ - defined(HAVE_STD_REGEX) && \ + defined(BENCHMARK_HAVE_STD_REGEX) && \ (defined(HAVE_GNU_POSIX_REGEX) || defined(HAVE_POSIX_REGEX)) #undef HAVE_STD_REGEX #endif diff --git a/third-party/benchmark/src/reporter.cc b/third-party/benchmark/src/reporter.cc index 076bc31a2ecc..1d2df17b90f0 100644 --- a/third-party/benchmark/src/reporter.cc +++ b/third-party/benchmark/src/reporter.cc @@ -25,6 +25,9 @@ #include "timers.h" namespace benchmark { +namespace internal { +extern std::map *global_context; +} BenchmarkReporter::BenchmarkReporter() : output_stream_(&std::cout), error_stream_(&std::cerr) {} @@ -36,11 +39,7 @@ void BenchmarkReporter::PrintBasicContext(std::ostream *out, BM_CHECK(out) << "cannot be null"; auto &Out = *out; -#ifndef BENCHMARK_OS_QURT - // Date/time information is not available on QuRT. - // Attempting to get it via this call cause the binary to crash. Out << LocalDateTimeString() << "\n"; -#endif if (context.executable_name) Out << "Running " << context.executable_name << "\n"; @@ -68,11 +67,8 @@ void BenchmarkReporter::PrintBasicContext(std::ostream *out, Out << "\n"; } - std::map *global_context = - internal::GetGlobalContext(); - - if (global_context != nullptr) { - for (const auto &kv : *global_context) { + if (internal::global_context != nullptr) { + for (const auto &kv : *internal::global_context) { Out << kv.first << ": " << kv.second << "\n"; } } diff --git a/third-party/benchmark/src/sleep.cc b/third-party/benchmark/src/sleep.cc new file mode 100644 index 000000000000..ab59000f24ad --- /dev/null +++ b/third-party/benchmark/src/sleep.cc @@ -0,0 +1,66 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "sleep.h" + +#include +#include +#include + +#include "internal_macros.h" + +#ifdef BENCHMARK_OS_WINDOWS +#include +#endif + +#ifdef BENCHMARK_OS_ZOS +#include +#endif + +namespace benchmark { +#ifdef BENCHMARK_OS_WINDOWS +// Window's Sleep takes milliseconds argument. +void SleepForMilliseconds(int milliseconds) { Sleep(milliseconds); } +void SleepForSeconds(double seconds) { + SleepForMilliseconds(static_cast(kNumMillisPerSecond * seconds)); +} +#else // BENCHMARK_OS_WINDOWS +void SleepForMicroseconds(int microseconds) { +#ifdef BENCHMARK_OS_ZOS + // z/OS does not support nanosleep. Instead call sleep() and then usleep() to + // sleep for the remaining microseconds because usleep() will fail if its + // argument is greater than 1000000. + div_t sleepTime = div(microseconds, kNumMicrosPerSecond); + int seconds = sleepTime.quot; + while (seconds != 0) seconds = sleep(seconds); + while (usleep(sleepTime.rem) == -1 && errno == EINTR) + ; +#else + struct timespec sleep_time; + sleep_time.tv_sec = microseconds / kNumMicrosPerSecond; + sleep_time.tv_nsec = (microseconds % kNumMicrosPerSecond) * kNumNanosPerMicro; + while (nanosleep(&sleep_time, &sleep_time) != 0 && errno == EINTR) + ; // Ignore signals and wait for the full interval to elapse. +#endif +} + +void SleepForMilliseconds(int milliseconds) { + SleepForMicroseconds(milliseconds * kNumMicrosPerMilli); +} + +void SleepForSeconds(double seconds) { + SleepForMicroseconds(static_cast(seconds * kNumMicrosPerSecond)); +} +#endif // BENCHMARK_OS_WINDOWS +} // end namespace benchmark diff --git a/third-party/benchmark/src/sleep.h b/third-party/benchmark/src/sleep.h new file mode 100644 index 000000000000..f98551afe284 --- /dev/null +++ b/third-party/benchmark/src/sleep.h @@ -0,0 +1,15 @@ +#ifndef BENCHMARK_SLEEP_H_ +#define BENCHMARK_SLEEP_H_ + +namespace benchmark { +const int kNumMillisPerSecond = 1000; +const int kNumMicrosPerMilli = 1000; +const int kNumMicrosPerSecond = kNumMillisPerSecond * 1000; +const int kNumNanosPerMicro = 1000; +const int kNumNanosPerSecond = kNumNanosPerMicro * kNumMicrosPerSecond; + +void SleepForMilliseconds(int milliseconds); +void SleepForSeconds(double seconds); +} // end namespace benchmark + +#endif // BENCHMARK_SLEEP_H_ diff --git a/third-party/benchmark/src/statistics.cc b/third-party/benchmark/src/statistics.cc index 261dcb299a67..3e5ef0993971 100644 --- a/third-party/benchmark/src/statistics.cc +++ b/third-party/benchmark/src/statistics.cc @@ -32,7 +32,7 @@ auto StatisticsSum = [](const std::vector& v) { double StatisticsMean(const std::vector& v) { if (v.empty()) return 0.0; - return StatisticsSum(v) * (1.0 / static_cast(v.size())); + return StatisticsSum(v) * (1.0 / v.size()); } double StatisticsMedian(const std::vector& v) { @@ -42,13 +42,13 @@ double StatisticsMedian(const std::vector& v) { auto center = copy.begin() + v.size() / 2; std::nth_element(copy.begin(), center, copy.end()); - // Did we have an odd number of samples? If yes, then center is the median. - // If not, then we are looking for the average between center and the value - // before. Instead of resorting, we just look for the max value before it, - // which is not necessarily the element immediately preceding `center` Since - // `copy` is only partially sorted by `nth_element`. + // did we have an odd number of samples? + // if yes, then center is the median + // it no, then we are looking for the average between center and the value + // before if (v.size() % 2 == 1) return *center; - auto center2 = std::max_element(copy.begin(), center); + auto center2 = copy.begin() + v.size() / 2 - 1; + std::nth_element(copy.begin(), center2, copy.end()); return (*center + *center2) / 2.0; } @@ -71,11 +71,8 @@ double StatisticsStdDev(const std::vector& v) { // Sample standard deviation is undefined for n = 1 if (v.size() == 1) return 0.0; - const double avg_squares = - SumSquares(v) * (1.0 / static_cast(v.size())); - return Sqrt(static_cast(v.size()) / - (static_cast(v.size()) - 1.0) * - (avg_squares - Sqr(mean))); + const double avg_squares = SumSquares(v) * (1.0 / v.size()); + return Sqrt(v.size() / (v.size() - 1.0) * (avg_squares - Sqr(mean))); } double StatisticsCV(const std::vector& v) { @@ -84,8 +81,6 @@ double StatisticsCV(const std::vector& v) { const auto stddev = StatisticsStdDev(v); const auto mean = StatisticsMean(v); - if (std::fpclassify(mean) == FP_ZERO) return 0.0; - return stddev / mean; } @@ -94,8 +89,9 @@ std::vector ComputeStats( typedef BenchmarkReporter::Run Run; std::vector results; - auto error_count = std::count_if(reports.begin(), reports.end(), - [](Run const& run) { return run.skipped; }); + auto error_count = + std::count_if(reports.begin(), reports.end(), + [](Run const& run) { return run.error_occurred; }); if (reports.size() - error_count < 2) { // We don't report aggregated data if there was a single run. @@ -122,13 +118,11 @@ std::vector ComputeStats( for (auto const& cnt : r.counters) { auto it = counter_stats.find(cnt.first); if (it == counter_stats.end()) { - it = counter_stats - .emplace(cnt.first, - CounterStat{cnt.second, std::vector{}}) - .first; + counter_stats.insert({cnt.first, {cnt.second, std::vector{}}}); + it = counter_stats.find(cnt.first); it->second.s.reserve(reports.size()); } else { - BM_CHECK_EQ(it->second.c.flags, cnt.second.flags); + BM_CHECK_EQ(counter_stats[cnt.first].c.flags, cnt.second.flags); } } } @@ -137,7 +131,7 @@ std::vector ComputeStats( for (Run const& run : reports) { BM_CHECK_EQ(reports[0].benchmark_name(), run.benchmark_name()); BM_CHECK_EQ(run_iterations, run.iterations); - if (run.skipped) continue; + if (run.error_occurred) continue; real_accumulated_time_stat.emplace_back(run.real_accumulated_time); cpu_accumulated_time_stat.emplace_back(run.cpu_accumulated_time); // user counters diff --git a/third-party/benchmark/src/statistics.h b/third-party/benchmark/src/statistics.h index 6e5560e8f19f..a9545a58c648 100644 --- a/third-party/benchmark/src/statistics.h +++ b/third-party/benchmark/src/statistics.h @@ -22,21 +22,15 @@ namespace benchmark { -// Return a vector containing the mean, median and standard deviation -// information (and any user-specified info) for the specified list of reports. -// If 'reports' contains less than two non-errored runs an empty vector is -// returned -BENCHMARK_EXPORT +// Return a vector containing the mean, median and standard devation information +// (and any user-specified info) for the specified list of reports. If 'reports' +// contains less than two non-errored runs an empty vector is returned std::vector ComputeStats( const std::vector& reports); -BENCHMARK_EXPORT double StatisticsMean(const std::vector& v); -BENCHMARK_EXPORT double StatisticsMedian(const std::vector& v); -BENCHMARK_EXPORT double StatisticsStdDev(const std::vector& v); -BENCHMARK_EXPORT double StatisticsCV(const std::vector& v); } // end namespace benchmark diff --git a/third-party/benchmark/src/string_util.cc b/third-party/benchmark/src/string_util.cc index c69e40a8133c..401fa13df7af 100644 --- a/third-party/benchmark/src/string_util.cc +++ b/third-party/benchmark/src/string_util.cc @@ -11,17 +11,16 @@ #include #include "arraysize.h" -#include "benchmark/benchmark.h" namespace benchmark { namespace { + // kilo, Mega, Giga, Tera, Peta, Exa, Zetta, Yotta. -const char* const kBigSIUnits[] = {"k", "M", "G", "T", "P", "E", "Z", "Y"}; +const char kBigSIUnits[] = "kMGTPEZY"; // Kibi, Mebi, Gibi, Tebi, Pebi, Exbi, Zebi, Yobi. -const char* const kBigIECUnits[] = {"Ki", "Mi", "Gi", "Ti", - "Pi", "Ei", "Zi", "Yi"}; +const char kBigIECUnits[] = "KMGTPEZY"; // milli, micro, nano, pico, femto, atto, zepto, yocto. -const char* const kSmallSIUnits[] = {"m", "u", "n", "p", "f", "a", "z", "y"}; +const char kSmallSIUnits[] = "munpfazy"; // We require that all three arrays have the same size. static_assert(arraysize(kBigSIUnits) == arraysize(kBigIECUnits), @@ -31,8 +30,9 @@ static_assert(arraysize(kSmallSIUnits) == arraysize(kBigSIUnits), static const int64_t kUnitsSize = arraysize(kBigSIUnits); -void ToExponentAndMantissa(double val, int precision, double one_k, - std::string* mantissa, int64_t* exponent) { +void ToExponentAndMantissa(double val, double thresh, int precision, + double one_k, std::string* mantissa, + int64_t* exponent) { std::stringstream mantissa_stream; if (val < 0) { @@ -43,8 +43,8 @@ void ToExponentAndMantissa(double val, int precision, double one_k, // Adjust threshold so that it never excludes things which can't be rendered // in 'precision' digits. const double adjusted_threshold = - std::max(1.0, 1.0 / std::pow(10.0, precision)); - const double big_threshold = (adjusted_threshold * one_k) - 1; + std::max(thresh, 1.0 / std::pow(10.0, precision)); + const double big_threshold = adjusted_threshold * one_k; const double small_threshold = adjusted_threshold; // Values in ]simple_threshold,small_threshold[ will be printed as-is const double simple_threshold = 0.01; @@ -92,20 +92,37 @@ std::string ExponentToPrefix(int64_t exponent, bool iec) { const int64_t index = (exponent > 0 ? exponent - 1 : -exponent - 1); if (index >= kUnitsSize) return ""; - const char* const* array = + const char* array = (exponent > 0 ? (iec ? kBigIECUnits : kBigSIUnits) : kSmallSIUnits); - - return std::string(array[index]); + if (iec) + return array[index] + std::string("i"); + else + return std::string(1, array[index]); } -std::string ToBinaryStringFullySpecified(double value, int precision, - Counter::OneK one_k) { +std::string ToBinaryStringFullySpecified(double value, double threshold, + int precision, double one_k = 1024.0) { std::string mantissa; int64_t exponent; - ToExponentAndMantissa(value, precision, - one_k == Counter::kIs1024 ? 1024.0 : 1000.0, &mantissa, + ToExponentAndMantissa(value, threshold, precision, one_k, &mantissa, &exponent); - return mantissa + ExponentToPrefix(exponent, one_k == Counter::kIs1024); + return mantissa + ExponentToPrefix(exponent, false); +} + +} // end namespace + +void AppendHumanReadable(int n, std::string* str) { + std::stringstream ss; + // Round down to the nearest SI prefix. + ss << ToBinaryStringFullySpecified(n, 1.0, 0); + *str += ss.str(); +} + +std::string HumanReadableNumber(double n, double one_k) { + // 1.1 means that figures up to 1.1k should be shown with the next unit down; + // this softens edge effects. + // 1 means that we should show one decimal place of precision. + return ToBinaryStringFullySpecified(n, 1.1, 1, one_k); } std::string StrFormatImp(const char* msg, va_list args) { @@ -116,21 +133,21 @@ std::string StrFormatImp(const char* msg, va_list args) { // TODO(ericwf): use std::array for first attempt to avoid one memory // allocation guess what the size might be std::array local_buff; - + std::size_t size = local_buff.size(); // 2015-10-08: vsnprintf is used instead of snd::vsnprintf due to a limitation // in the android-ndk - auto ret = vsnprintf(local_buff.data(), local_buff.size(), msg, args_cp); + auto ret = vsnprintf(local_buff.data(), size, msg, args_cp); va_end(args_cp); // handle empty expansion if (ret == 0) return std::string{}; - if (static_cast(ret) < local_buff.size()) + if (static_cast(ret) < size) return std::string(local_buff.data()); // we did not provide a long enough buffer on our first attempt. // add 1 to size to account for null-byte in size cast to prevent overflow - std::size_t size = static_cast(ret) + 1; + size = static_cast(ret) + 1; auto buff_ptr = std::unique_ptr(new char[size]); // 2015-10-08: vsnprintf is used instead of snd::vsnprintf due to a limitation // in the android-ndk @@ -138,12 +155,6 @@ std::string StrFormatImp(const char* msg, va_list args) { return std::string(buff_ptr.get()); } -} // end namespace - -std::string HumanReadableNumber(double n, Counter::OneK one_k) { - return ToBinaryStringFullySpecified(n, 1, one_k); -} - std::string StrFormat(const char* format, ...) { va_list args; va_start(args, format); diff --git a/third-party/benchmark/src/string_util.h b/third-party/benchmark/src/string_util.h index 731aa2c04c3e..ff3b7da47d6c 100644 --- a/third-party/benchmark/src/string_util.h +++ b/third-party/benchmark/src/string_util.h @@ -4,19 +4,15 @@ #include #include #include -#include -#include "benchmark/benchmark.h" -#include "benchmark/export.h" -#include "check.h" #include "internal_macros.h" namespace benchmark { -BENCHMARK_EXPORT -std::string HumanReadableNumber(double n, Counter::OneK one_k); +void AppendHumanReadable(int n, std::string* str); + +std::string HumanReadableNumber(double n, double one_k = 1024.0); -BENCHMARK_EXPORT #if defined(__MINGW32__) __attribute__((format(__MINGW_PRINTF_FORMAT, 1, 2))) #elif defined(__GNUC__) @@ -42,7 +38,6 @@ inline std::string StrCat(Args&&... args) { return ss.str(); } -BENCHMARK_EXPORT std::vector StrSplit(const std::string& str, char delim); // Disable lint checking for this block since it re-implements C functions. diff --git a/third-party/benchmark/src/sysinfo.cc b/third-party/benchmark/src/sysinfo.cc index 46df973b069a..3a56e8cace48 100644 --- a/third-party/benchmark/src/sysinfo.cc +++ b/third-party/benchmark/src/sysinfo.cc @@ -22,10 +22,6 @@ #include "internal_macros.h" #ifdef BENCHMARK_OS_WINDOWS -#if !defined(WINVER) || WINVER < 0x0600 -#undef WINVER -#define WINVER 0x0600 -#endif // WINVER handling #include #undef StrCat // Don't let StrCat in string_util.h be renamed to lstrcatA #include @@ -34,7 +30,7 @@ #include #else #include -#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) +#ifndef BENCHMARK_OS_FUCHSIA #include #endif #include @@ -49,17 +45,10 @@ #endif #if defined(BENCHMARK_OS_SOLARIS) #include -#include #endif #if defined(BENCHMARK_OS_QNX) #include #endif -#if defined(BENCHMARK_OS_QURT) -#include -#endif -#if defined(BENCHMARK_HAS_PTHREAD_AFFINITY) -#include -#endif #include #include @@ -76,17 +65,15 @@ #include #include #include -#include #include #include -#include "benchmark/benchmark.h" #include "check.h" #include "cycleclock.h" #include "internal_macros.h" #include "log.h" +#include "sleep.h" #include "string_util.h" -#include "timers.h" namespace benchmark { namespace { @@ -111,59 +98,67 @@ BENCHMARK_NORETURN void PrintErrorAndDie(Args&&... args) { /// `sysctl` with the result type it's to be interpreted as. struct ValueUnion { union DataT { - int32_t int32_value; - int64_t int64_value; + uint32_t uint32_value; + uint64_t uint64_value; // For correct aliasing of union members from bytes. char bytes[8]; }; using DataPtr = std::unique_ptr; // The size of the data union member + its trailing array size. - std::size_t size; - DataPtr buff; + size_t Size; + DataPtr Buff; public: - ValueUnion() : size(0), buff(nullptr, &std::free) {} + ValueUnion() : Size(0), Buff(nullptr, &std::free) {} - explicit ValueUnion(std::size_t buff_size) - : size(sizeof(DataT) + buff_size), - buff(::new (std::malloc(size)) DataT(), &std::free) {} + explicit ValueUnion(size_t BuffSize) + : Size(sizeof(DataT) + BuffSize), + Buff(::new (std::malloc(Size)) DataT(), &std::free) {} ValueUnion(ValueUnion&& other) = default; - explicit operator bool() const { return bool(buff); } + explicit operator bool() const { return bool(Buff); } - char* data() const { return buff->bytes; } + char* data() const { return Buff->bytes; } std::string GetAsString() const { return std::string(data()); } int64_t GetAsInteger() const { - if (size == sizeof(buff->int32_value)) - return buff->int32_value; - else if (size == sizeof(buff->int64_value)) - return buff->int64_value; + if (Size == sizeof(Buff->uint32_value)) + return static_cast(Buff->uint32_value); + else if (Size == sizeof(Buff->uint64_value)) + return static_cast(Buff->uint64_value); + BENCHMARK_UNREACHABLE(); + } + + uint64_t GetAsUnsigned() const { + if (Size == sizeof(Buff->uint32_value)) + return Buff->uint32_value; + else if (Size == sizeof(Buff->uint64_value)) + return Buff->uint64_value; BENCHMARK_UNREACHABLE(); } template std::array GetAsArray() { - const int arr_size = sizeof(T) * N; - BM_CHECK_LE(arr_size, size); - std::array arr; - std::memcpy(arr.data(), data(), arr_size); - return arr; + const int ArrSize = sizeof(T) * N; + BM_CHECK_LE(ArrSize, Size); + std::array Arr; + std::memcpy(Arr.data(), data(), ArrSize); + return Arr; } }; -ValueUnion GetSysctlImp(std::string const& name) { +ValueUnion GetSysctlImp(std::string const& Name) { #if defined BENCHMARK_OS_OPENBSD int mib[2]; mib[0] = CTL_HW; - if ((name == "hw.ncpu") || (name == "hw.cpuspeed")) { + if ((Name == "hw.ncpu") || (Name == "hw.cpuspeed")) { ValueUnion buff(sizeof(int)); - if (name == "hw.ncpu") { + if (Name == "hw.ncpu") { mib[1] = HW_NCPU; } else { mib[1] = HW_CPUSPEED; @@ -176,41 +171,41 @@ ValueUnion GetSysctlImp(std::string const& name) { } return ValueUnion(); #else - std::size_t cur_buff_size = 0; - if (sysctlbyname(name.c_str(), nullptr, &cur_buff_size, nullptr, 0) == -1) + size_t CurBuffSize = 0; + if (sysctlbyname(Name.c_str(), nullptr, &CurBuffSize, nullptr, 0) == -1) return ValueUnion(); - ValueUnion buff(cur_buff_size); - if (sysctlbyname(name.c_str(), buff.data(), &buff.size, nullptr, 0) == 0) + ValueUnion buff(CurBuffSize); + if (sysctlbyname(Name.c_str(), buff.data(), &buff.Size, nullptr, 0) == 0) return buff; return ValueUnion(); #endif } BENCHMARK_MAYBE_UNUSED -bool GetSysctl(std::string const& name, std::string* out) { - out->clear(); - auto buff = GetSysctlImp(name); - if (!buff) return false; - out->assign(buff.data()); +bool GetSysctl(std::string const& Name, std::string* Out) { + Out->clear(); + auto Buff = GetSysctlImp(Name); + if (!Buff) return false; + Out->assign(Buff.data()); return true; } template ::value>::type> -bool GetSysctl(std::string const& name, Tp* out) { - *out = 0; - auto buff = GetSysctlImp(name); - if (!buff) return false; - *out = static_cast(buff.GetAsInteger()); +bool GetSysctl(std::string const& Name, Tp* Out) { + *Out = 0; + auto Buff = GetSysctlImp(Name); + if (!Buff) return false; + *Out = static_cast(Buff.GetAsUnsigned()); return true; } template -bool GetSysctl(std::string const& name, std::array* out) { - auto buff = GetSysctlImp(name); - if (!buff) return false; - *out = buff.GetAsArray(); +bool GetSysctl(std::string const& Name, std::array* Out) { + auto Buff = GetSysctlImp(Name); + if (!Buff) return false; + *Out = Buff.GetAsArray(); return true; } #endif @@ -246,21 +241,21 @@ CPUInfo::Scaling CpuScaling(int num_cpus) { #endif } -int CountSetBitsInCPUMap(std::string val) { - auto CountBits = [](std::string part) { +int CountSetBitsInCPUMap(std::string Val) { + auto CountBits = [](std::string Part) { using CPUMask = std::bitset; - part = "0x" + part; - CPUMask mask(benchmark::stoul(part, nullptr, 16)); - return static_cast(mask.count()); + Part = "0x" + Part; + CPUMask Mask(benchmark::stoul(Part, nullptr, 16)); + return static_cast(Mask.count()); }; - std::size_t pos; + size_t Pos; int total = 0; - while ((pos = val.find(',')) != std::string::npos) { - total += CountBits(val.substr(0, pos)); - val = val.substr(pos + 1); + while ((Pos = Val.find(',')) != std::string::npos) { + total += CountBits(Val.substr(0, Pos)); + Val = Val.substr(Pos + 1); } - if (!val.empty()) { - total += CountBits(val); + if (!Val.empty()) { + total += CountBits(Val); } return total; } @@ -269,16 +264,16 @@ BENCHMARK_MAYBE_UNUSED std::vector GetCacheSizesFromKVFS() { std::vector res; std::string dir = "/sys/devices/system/cpu/cpu0/cache/"; - int idx = 0; + int Idx = 0; while (true) { CPUInfo::CacheInfo info; - std::string fpath = StrCat(dir, "index", idx++, "/"); - std::ifstream f(StrCat(fpath, "size").c_str()); + std::string FPath = StrCat(dir, "index", Idx++, "/"); + std::ifstream f(StrCat(FPath, "size").c_str()); if (!f.is_open()) break; std::string suffix; f >> info.size; if (f.fail()) - PrintErrorAndDie("Failed while reading file '", fpath, "size'"); + PrintErrorAndDie("Failed while reading file '", FPath, "size'"); if (f.good()) { f >> suffix; if (f.bad()) @@ -289,13 +284,13 @@ std::vector GetCacheSizesFromKVFS() { else if (suffix == "K") info.size *= 1024; } - if (!ReadFromFile(StrCat(fpath, "type"), &info.type)) - PrintErrorAndDie("Failed to read from file ", fpath, "type"); - if (!ReadFromFile(StrCat(fpath, "level"), &info.level)) - PrintErrorAndDie("Failed to read from file ", fpath, "level"); + if (!ReadFromFile(StrCat(FPath, "type"), &info.type)) + PrintErrorAndDie("Failed to read from file ", FPath, "type"); + if (!ReadFromFile(StrCat(FPath, "level"), &info.level)) + PrintErrorAndDie("Failed to read from file ", FPath, "level"); std::string map_str; - if (!ReadFromFile(StrCat(fpath, "shared_cpu_map"), &map_str)) - PrintErrorAndDie("Failed to read from file ", fpath, "shared_cpu_map"); + if (!ReadFromFile(StrCat(FPath, "shared_cpu_map"), &map_str)) + PrintErrorAndDie("Failed to read from file ", FPath, "shared_cpu_map"); info.num_sharing = CountSetBitsInCPUMap(map_str); res.push_back(info); } @@ -306,26 +301,26 @@ std::vector GetCacheSizesFromKVFS() { #ifdef BENCHMARK_OS_MACOSX std::vector GetCacheSizesMacOSX() { std::vector res; - std::array cache_counts{{0, 0, 0, 0}}; - GetSysctl("hw.cacheconfig", &cache_counts); + std::array CacheCounts{{0, 0, 0, 0}}; + GetSysctl("hw.cacheconfig", &CacheCounts); struct { std::string name; std::string type; int level; - int num_sharing; - } cases[] = {{"hw.l1dcachesize", "Data", 1, cache_counts[1]}, - {"hw.l1icachesize", "Instruction", 1, cache_counts[1]}, - {"hw.l2cachesize", "Unified", 2, cache_counts[2]}, - {"hw.l3cachesize", "Unified", 3, cache_counts[3]}}; - for (auto& c : cases) { + uint64_t num_sharing; + } Cases[] = {{"hw.l1dcachesize", "Data", 1, CacheCounts[1]}, + {"hw.l1icachesize", "Instruction", 1, CacheCounts[1]}, + {"hw.l2cachesize", "Unified", 2, CacheCounts[2]}, + {"hw.l3cachesize", "Unified", 3, CacheCounts[3]}}; + for (auto& C : Cases) { int val; - if (!GetSysctl(c.name, &val)) continue; + if (!GetSysctl(C.name, &val)) continue; CPUInfo::CacheInfo info; - info.type = c.type; - info.level = c.level; + info.type = C.type; + info.level = C.level; info.size = val; - info.num_sharing = c.num_sharing; + info.num_sharing = static_cast(C.num_sharing); res.push_back(std::move(info)); } return res; @@ -339,7 +334,7 @@ std::vector GetCacheSizesWindows() { using UPtr = std::unique_ptr; GetLogicalProcessorInformation(nullptr, &buffer_size); - UPtr buff(static_cast(std::malloc(buffer_size)), &std::free); + UPtr buff((PInfo*)malloc(buffer_size), &std::free); if (!GetLogicalProcessorInformation(buff.get(), &buffer_size)) PrintErrorAndDie("Failed during call to GetLogicalProcessorInformation: ", GetLastError()); @@ -350,16 +345,16 @@ std::vector GetCacheSizesWindows() { for (; it != end; ++it) { if (it->Relationship != RelationCache) continue; using BitSet = std::bitset; - BitSet b(it->ProcessorMask); + BitSet B(it->ProcessorMask); // To prevent duplicates, only consider caches where CPU 0 is specified - if (!b.test(0)) continue; - const CInfo& cache = it->Cache; + if (!B.test(0)) continue; + CInfo* Cache = &it->Cache; CPUInfo::CacheInfo C; - C.num_sharing = static_cast(b.count()); - C.level = cache.Level; - C.size = cache.Size; + C.num_sharing = static_cast(B.count()); + C.level = Cache->Level; + C.size = Cache->Size; C.type = "Unknown"; - switch (cache.Type) { + switch (Cache->Type) { case CacheUnified: C.type = "Unified"; break; @@ -422,8 +417,6 @@ std::vector GetCacheSizes() { return GetCacheSizesWindows(); #elif defined(BENCHMARK_OS_QNX) return GetCacheSizesQNX(); -#elif defined(BENCHMARK_OS_QURT) - return std::vector(); #else return GetCacheSizesFromKVFS(); #endif @@ -432,32 +425,23 @@ std::vector GetCacheSizes() { std::string GetSystemName() { #if defined(BENCHMARK_OS_WINDOWS) std::string str; - static constexpr int COUNT = MAX_COMPUTERNAME_LENGTH + 1; + const unsigned COUNT = MAX_COMPUTERNAME_LENGTH + 1; TCHAR hostname[COUNT] = {'\0'}; DWORD DWCOUNT = COUNT; if (!GetComputerName(hostname, &DWCOUNT)) return std::string(""); #ifndef UNICODE str = std::string(hostname, DWCOUNT); #else - // `WideCharToMultiByte` returns `0` when conversion fails. - int len = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, hostname, - DWCOUNT, NULL, 0, NULL, NULL); - str.resize(len); - WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, hostname, DWCOUNT, &str[0], - str.size(), NULL, NULL); + // Using wstring_convert, Is deprecated in C++17 + using convert_type = std::codecvt_utf8; + std::wstring_convert converter; + std::wstring wStr(hostname, DWCOUNT); + str = converter.to_bytes(wStr); #endif return str; -#elif defined(BENCHMARK_OS_QURT) - std::string str = "Hexagon DSP"; - qurt_arch_version_t arch_version_struct; - if (qurt_sysenv_get_arch_version(&arch_version_struct) == QURT_EOK) { - str += " v"; - str += std::to_string(arch_version_struct.arch_version); - } - return str; -#else +#else // defined(BENCHMARK_OS_WINDOWS) #ifndef HOST_NAME_MAX -#ifdef BENCHMARK_HAS_SYSCTL // BSD/Mac doesn't have HOST_NAME_MAX defined +#ifdef BENCHMARK_HAS_SYSCTL // BSD/Mac Doesnt have HOST_NAME_MAX defined #define HOST_NAME_MAX 64 #elif defined(BENCHMARK_OS_NACL) #define HOST_NAME_MAX 64 @@ -465,8 +449,6 @@ std::string GetSystemName() { #define HOST_NAME_MAX 154 #elif defined(BENCHMARK_OS_RTEMS) #define HOST_NAME_MAX 256 -#elif defined(BENCHMARK_OS_SOLARIS) -#define HOST_NAME_MAX MAXHOSTNAMELEN #elif defined(BENCHMARK_OS_ZOS) #define HOST_NAME_MAX _POSIX_HOST_NAME_MAX #else @@ -481,11 +463,12 @@ std::string GetSystemName() { #endif // Catch-all POSIX block. } -int GetNumCPUsImpl() { +int GetNumCPUs() { #ifdef BENCHMARK_HAS_SYSCTL - int num_cpu = -1; - if (GetSysctl("hw.ncpu", &num_cpu)) return num_cpu; - PrintErrorAndDie("Err: ", strerror(errno)); + int NumCPU = -1; + if (GetSysctl("hw.ncpu", &NumCPU)) return NumCPU; + fprintf(stderr, "Err: %s\n", strerror(errno)); + std::exit(EXIT_FAILURE); #elif defined(BENCHMARK_OS_WINDOWS) SYSTEM_INFO sysinfo; // Use memset as opposed to = {} to avoid GCC missing initializer false @@ -497,155 +480,64 @@ int GetNumCPUsImpl() { // group #elif defined(BENCHMARK_OS_SOLARIS) // Returns -1 in case of a failure. - long num_cpu = sysconf(_SC_NPROCESSORS_ONLN); - if (num_cpu < 0) { - PrintErrorAndDie("sysconf(_SC_NPROCESSORS_ONLN) failed with error: ", - strerror(errno)); + int NumCPU = sysconf(_SC_NPROCESSORS_ONLN); + if (NumCPU < 0) { + fprintf(stderr, "sysconf(_SC_NPROCESSORS_ONLN) failed with error: %s\n", + strerror(errno)); } - return (int)num_cpu; + return NumCPU; #elif defined(BENCHMARK_OS_QNX) return static_cast(_syspage_ptr->num_cpu); -#elif defined(BENCHMARK_OS_QURT) - qurt_sysenv_max_hthreads_t hardware_threads; - if (qurt_sysenv_get_max_hw_threads(&hardware_threads) != QURT_EOK) { - hardware_threads.max_hthreads = 1; - } - return hardware_threads.max_hthreads; #else - int num_cpus = 0; - int max_id = -1; + int NumCPUs = 0; + int MaxID = -1; std::ifstream f("/proc/cpuinfo"); if (!f.is_open()) { - PrintErrorAndDie("Failed to open /proc/cpuinfo"); + std::cerr << "failed to open /proc/cpuinfo\n"; + return -1; } -#if defined(__alpha__) - const std::string Key = "cpus detected"; -#else const std::string Key = "processor"; -#endif std::string ln; while (std::getline(f, ln)) { if (ln.empty()) continue; - std::size_t split_idx = ln.find(':'); + size_t SplitIdx = ln.find(':'); std::string value; #if defined(__s390__) // s390 has another format in /proc/cpuinfo // it needs to be parsed differently - if (split_idx != std::string::npos) - value = ln.substr(Key.size() + 1, split_idx - Key.size() - 1); + if (SplitIdx != std::string::npos) + value = ln.substr(Key.size() + 1, SplitIdx - Key.size() - 1); #else - if (split_idx != std::string::npos) value = ln.substr(split_idx + 1); + if (SplitIdx != std::string::npos) value = ln.substr(SplitIdx + 1); #endif if (ln.size() >= Key.size() && ln.compare(0, Key.size(), Key) == 0) { - num_cpus++; + NumCPUs++; if (!value.empty()) { - const int cur_id = benchmark::stoi(value); - max_id = std::max(cur_id, max_id); + int CurID = benchmark::stoi(value); + MaxID = std::max(CurID, MaxID); } } } if (f.bad()) { - PrintErrorAndDie("Failure reading /proc/cpuinfo"); + std::cerr << "Failure reading /proc/cpuinfo\n"; + return -1; } if (!f.eof()) { - PrintErrorAndDie("Failed to read to end of /proc/cpuinfo"); + std::cerr << "Failed to read to end of /proc/cpuinfo\n"; + return -1; } f.close(); - if ((max_id + 1) != num_cpus) { + if ((MaxID + 1) != NumCPUs) { fprintf(stderr, "CPU ID assignments in /proc/cpuinfo seem messed up." " This is usually caused by a bad BIOS.\n"); } - return num_cpus; + return NumCPUs; #endif BENCHMARK_UNREACHABLE(); } -int GetNumCPUs() { - const int num_cpus = GetNumCPUsImpl(); - if (num_cpus < 1) { - PrintErrorAndDie( - "Unable to extract number of CPUs. If your platform uses " - "/proc/cpuinfo, custom support may need to be added."); - } - return num_cpus; -} - -class ThreadAffinityGuard final { - public: - ThreadAffinityGuard() : reset_affinity(SetAffinity()) { - if (!reset_affinity) - std::cerr << "***WARNING*** Failed to set thread affinity. Estimated CPU " - "frequency may be incorrect." - << std::endl; - } - - ~ThreadAffinityGuard() { - if (!reset_affinity) return; - -#if defined(BENCHMARK_HAS_PTHREAD_AFFINITY) - int ret = pthread_setaffinity_np(self, sizeof(previous_affinity), - &previous_affinity); - if (ret == 0) return; -#elif defined(BENCHMARK_OS_WINDOWS_WIN32) - DWORD_PTR ret = SetThreadAffinityMask(self, previous_affinity); - if (ret != 0) return; -#endif // def BENCHMARK_HAS_PTHREAD_AFFINITY - PrintErrorAndDie("Failed to reset thread affinity"); - } - - ThreadAffinityGuard(ThreadAffinityGuard&&) = delete; - ThreadAffinityGuard(const ThreadAffinityGuard&) = delete; - ThreadAffinityGuard& operator=(ThreadAffinityGuard&&) = delete; - ThreadAffinityGuard& operator=(const ThreadAffinityGuard&) = delete; - - private: - bool SetAffinity() { -#if defined(BENCHMARK_HAS_PTHREAD_AFFINITY) - int ret; - self = pthread_self(); - ret = pthread_getaffinity_np(self, sizeof(previous_affinity), - &previous_affinity); - if (ret != 0) return false; - - cpu_set_t affinity; - memcpy(&affinity, &previous_affinity, sizeof(affinity)); - - bool is_first_cpu = true; - - for (int i = 0; i < CPU_SETSIZE; ++i) - if (CPU_ISSET(i, &affinity)) { - if (is_first_cpu) - is_first_cpu = false; - else - CPU_CLR(i, &affinity); - } - - if (is_first_cpu) return false; - - ret = pthread_setaffinity_np(self, sizeof(affinity), &affinity); - return ret == 0; -#elif defined(BENCHMARK_OS_WINDOWS_WIN32) - self = GetCurrentThread(); - DWORD_PTR mask = static_cast(1) << GetCurrentProcessorNumber(); - previous_affinity = SetThreadAffinityMask(self, mask); - return previous_affinity != 0; -#else - return false; -#endif // def BENCHMARK_HAS_PTHREAD_AFFINITY - } - -#if defined(BENCHMARK_HAS_PTHREAD_AFFINITY) - pthread_t self; - cpu_set_t previous_affinity; -#elif defined(BENCHMARK_OS_WINDOWS_WIN32) - HANDLE self; - DWORD_PTR previous_affinity; -#endif // def BENCHMARK_HAS_PTHREAD_AFFINITY - bool reset_affinity; -}; - double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { // Currently, scaling is only used on linux path here, // suppress diagnostics about it being unused on other paths. @@ -674,7 +566,7 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { &freq)) { // The value is in kHz (as the file name suggests). For example, on a // 2GHz warpstation, the file contains the value "2000000". - return static_cast(freq) * 1000.0; + return freq * 1000.0; } const double error_value = -1; @@ -686,7 +578,7 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { return error_value; } - auto StartsWithKey = [](std::string const& Value, std::string const& Key) { + auto startsWithKey = [](std::string const& Value, std::string const& Key) { if (Key.size() > Value.size()) return false; auto Cmp = [&](char X, char Y) { return std::tolower(X) == std::tolower(Y); @@ -697,18 +589,18 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { std::string ln; while (std::getline(f, ln)) { if (ln.empty()) continue; - std::size_t split_idx = ln.find(':'); + size_t SplitIdx = ln.find(':'); std::string value; - if (split_idx != std::string::npos) value = ln.substr(split_idx + 1); + if (SplitIdx != std::string::npos) value = ln.substr(SplitIdx + 1); // When parsing the "cpu MHz" and "bogomips" (fallback) entries, we only // accept positive values. Some environments (virtual machines) report zero, // which would cause infinite looping in WallTime_Init. - if (StartsWithKey(ln, "cpu MHz")) { + if (startsWithKey(ln, "cpu MHz")) { if (!value.empty()) { double cycles_per_second = benchmark::stod(value) * 1000000.0; if (cycles_per_second > 0) return cycles_per_second; } - } else if (StartsWithKey(ln, "bogomips")) { + } else if (startsWithKey(ln, "bogomips")) { if (!value.empty()) { bogo_clock = benchmark::stod(value) * 1000000.0; if (bogo_clock < 0.0) bogo_clock = error_value; @@ -730,7 +622,7 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { if (bogo_clock >= 0.0) return bogo_clock; #elif defined BENCHMARK_HAS_SYSCTL - constexpr auto* freqStr = + constexpr auto* FreqStr = #if defined(BENCHMARK_OS_FREEBSD) || defined(BENCHMARK_OS_NETBSD) "machdep.tsc_freq"; #elif defined BENCHMARK_OS_OPENBSD @@ -742,17 +634,14 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { #endif unsigned long long hz = 0; #if defined BENCHMARK_OS_OPENBSD - if (GetSysctl(freqStr, &hz)) return hz * 1000000; + if (GetSysctl(FreqStr, &hz)) return hz * 1000000; #else - if (GetSysctl(freqStr, &hz)) return hz; + if (GetSysctl(FreqStr, &hz)) return hz; #endif fprintf(stderr, "Unable to determine clock rate from sysctl: %s: %s\n", - freqStr, strerror(errno)); - fprintf(stderr, - "This does not affect benchmark measurements, only the " - "metadata output.\n"); + FreqStr, strerror(errno)); -#elif defined BENCHMARK_OS_WINDOWS_WIN32 +#elif defined BENCHMARK_OS_WINDOWS // In NT, read MHz from the registry. If we fail to do so or we're in win9x // then make a crude estimate. DWORD data, data_size = sizeof(data); @@ -761,16 +650,15 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { SHGetValueA(HKEY_LOCAL_MACHINE, "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", "~MHz", nullptr, &data, &data_size))) - return static_cast(static_cast(data) * - static_cast(1000 * 1000)); // was mhz + return static_cast((int64_t)data * + (int64_t)(1000 * 1000)); // was mhz #elif defined(BENCHMARK_OS_SOLARIS) kstat_ctl_t* kc = kstat_open(); if (!kc) { std::cerr << "failed to open /dev/kstat\n"; return -1; } - kstat_t* ksp = kstat_lookup(kc, const_cast("cpu_info"), -1, - const_cast("cpu_info0")); + kstat_t* ksp = kstat_lookup(kc, (char*)"cpu_info", -1, (char*)"cpu_info0"); if (!ksp) { std::cerr << "failed to lookup in /dev/kstat\n"; return -1; @@ -779,8 +667,8 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { std::cerr << "failed to read from /dev/kstat\n"; return -1; } - kstat_named_t* knp = (kstat_named_t*)kstat_data_lookup( - ksp, const_cast("current_clock_Hz")); + kstat_named_t* knp = + (kstat_named_t*)kstat_data_lookup(ksp, (char*)"current_clock_Hz"); if (!knp) { std::cerr << "failed to lookup data in /dev/kstat\n"; return -1; @@ -794,55 +682,22 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { kstat_close(kc); return clock_hz; #elif defined(BENCHMARK_OS_QNX) - return static_cast( - static_cast(SYSPAGE_ENTRY(cpuinfo)->speed) * - static_cast(1000 * 1000)); -#elif defined(BENCHMARK_OS_QURT) - // QuRT doesn't provide any API to query Hexagon frequency. - return 1000000000; + return static_cast((int64_t)(SYSPAGE_ENTRY(cpuinfo)->speed) * + (int64_t)(1000 * 1000)); #endif // If we've fallen through, attempt to roughly estimate the CPU clock rate. - - // Make sure to use the same cycle counter when starting and stopping the - // cycle timer. We just pin the current thread to a cpu in the previous - // affinity set. - ThreadAffinityGuard affinity_guard; - - static constexpr double estimate_time_s = 1.0; - const double start_time = ChronoClockNow(); + const int estimate_time_ms = 1000; const auto start_ticks = cycleclock::Now(); - - // Impose load instead of calling sleep() to make sure the cycle counter - // works. - using PRNG = std::minstd_rand; - using Result = PRNG::result_type; - PRNG rng(static_cast(start_ticks)); - - Result state = 0; - - do { - static constexpr size_t batch_size = 10000; - rng.discard(batch_size); - state += rng(); - - } while (ChronoClockNow() - start_time < estimate_time_s); - - DoNotOptimize(state); - - const auto end_ticks = cycleclock::Now(); - const double end_time = ChronoClockNow(); - - return static_cast(end_ticks - start_ticks) / (end_time - start_time); - // Reset the affinity of current thread when the lifetime of affinity_guard - // ends. + SleepForMilliseconds(estimate_time_ms); + return static_cast(cycleclock::Now() - start_ticks); } std::vector GetLoadAvg() { #if (defined BENCHMARK_OS_FREEBSD || defined(BENCHMARK_OS_LINUX) || \ defined BENCHMARK_OS_MACOSX || defined BENCHMARK_OS_NETBSD || \ defined BENCHMARK_OS_OPENBSD || defined BENCHMARK_OS_DRAGONFLY) && \ - !(defined(__ANDROID__) && __ANDROID_API__ < 29) - static constexpr int kMaxSamples = 3; + !defined(__ANDROID__) + constexpr int kMaxSamples = 3; std::vector res(kMaxSamples, 0.0); const int nelem = getloadavg(res.data(), kMaxSamples); if (nelem < 1) { diff --git a/third-party/benchmark/src/thread_manager.h b/third-party/benchmark/src/thread_manager.h index 819b3c44db66..468028508940 100644 --- a/third-party/benchmark/src/thread_manager.h +++ b/third-party/benchmark/src/thread_manager.h @@ -43,8 +43,8 @@ class ThreadManager { double manual_time_used = 0; int64_t complexity_n = 0; std::string report_label_; - std::string skip_message_; - internal::Skipped skipped_ = internal::NotSkipped; + std::string error_message_; + bool has_error_ = false; UserCounters counters; }; GUARDED_BY(GetBenchmarkMutex()) Result results; diff --git a/third-party/benchmark/src/timers.cc b/third-party/benchmark/src/timers.cc index 667e7b2eef3c..ed35c01f5540 100644 --- a/third-party/benchmark/src/timers.cc +++ b/third-party/benchmark/src/timers.cc @@ -23,7 +23,7 @@ #include #else #include -#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) +#ifndef BENCHMARK_OS_FUCHSIA #include #endif #include @@ -38,9 +38,6 @@ #include #include #endif -#if defined(BENCHMARK_OS_QURT) -#include -#endif #endif #ifdef BENCHMARK_OS_EMSCRIPTEN @@ -59,6 +56,7 @@ #include "check.h" #include "log.h" +#include "sleep.h" #include "string_util.h" namespace benchmark { @@ -67,9 +65,6 @@ namespace benchmark { #if defined(__GNUC__) #pragma GCC diagnostic ignored "-Wunused-function" #endif -#if defined(__NVCOMPILER) -#pragma diag_suppress declared_but_not_referenced -#endif namespace { #if defined(BENCHMARK_OS_WINDOWS) @@ -84,7 +79,7 @@ double MakeTime(FILETIME const& kernel_time, FILETIME const& user_time) { static_cast(user.QuadPart)) * 1e-7; } -#elif !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) +#elif !defined(BENCHMARK_OS_FUCHSIA) double MakeTime(struct rusage const& ru) { return (static_cast(ru.ru_utime.tv_sec) + static_cast(ru.ru_utime.tv_usec) * 1e-6 + @@ -102,8 +97,7 @@ double MakeTime(thread_basic_info_data_t const& info) { #endif #if defined(CLOCK_PROCESS_CPUTIME_ID) || defined(CLOCK_THREAD_CPUTIME_ID) double MakeTime(struct timespec const& ts) { - return static_cast(ts.tv_sec) + - (static_cast(ts.tv_nsec) * 1e-9); + return ts.tv_sec + (static_cast(ts.tv_nsec) * 1e-9); } #endif @@ -125,15 +119,11 @@ double ProcessCPUUsage() { &user_time)) return MakeTime(kernel_time, user_time); DiagnoseAndExit("GetProccessTimes() failed"); -#elif defined(BENCHMARK_OS_QURT) - return static_cast( - qurt_timer_timetick_to_us(qurt_timer_get_ticks())) * - 1.0e-6; #elif defined(BENCHMARK_OS_EMSCRIPTEN) // clock_gettime(CLOCK_PROCESS_CPUTIME_ID, ...) returns 0 on Emscripten. // Use Emscripten-specific API. Reported CPU time would be exactly the // same as total time, but this is ok because there aren't long-latency - // synchronous system calls in Emscripten. + // syncronous system calls in Emscripten. return emscripten_get_now() * 1e-3; #elif defined(CLOCK_PROCESS_CPUTIME_ID) && !defined(BENCHMARK_OS_MACOSX) // FIXME We want to use clock_gettime, but its not available in MacOS 10.11. @@ -159,10 +149,6 @@ double ThreadCPUUsage() { GetThreadTimes(this_thread, &creation_time, &exit_time, &kernel_time, &user_time); return MakeTime(kernel_time, user_time); -#elif defined(BENCHMARK_OS_QURT) - return static_cast( - qurt_timer_timetick_to_us(qurt_timer_get_ticks())) * - 1.0e-6; #elif defined(BENCHMARK_OS_MACOSX) // FIXME We want to use clock_gettime, but its not available in MacOS 10.11. // See https://github.com/google/benchmark/pull/292 diff --git a/third-party/benchmark/test/AssemblyTests.cmake b/third-party/benchmark/test/AssemblyTests.cmake index c43c711faf87..3d078586f1de 100644 --- a/third-party/benchmark/test/AssemblyTests.cmake +++ b/third-party/benchmark/test/AssemblyTests.cmake @@ -1,23 +1,3 @@ -set(CLANG_SUPPORTED_VERSION "5.0.0") -set(GCC_SUPPORTED_VERSION "5.5.0") - -if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") - if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL ${CLANG_SUPPORTED_VERSION}) - message (WARNING - "Unsupported Clang version " ${CMAKE_CXX_COMPILER_VERSION} - ". Expected is " ${CLANG_SUPPORTED_VERSION} - ". Assembly tests may be broken.") - endif() -elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU") - if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL ${GCC_SUPPORTED_VERSION}) - message (WARNING - "Unsupported GCC version " ${CMAKE_CXX_COMPILER_VERSION} - ". Expected is " ${GCC_SUPPORTED_VERSION} - ". Assembly tests may be broken.") - endif() -else() - message (WARNING "Unsupported compiler. Assembly tests may be broken.") -endif() include(split_list) @@ -43,7 +23,6 @@ string(TOUPPER "${CMAKE_CXX_COMPILER_ID}" ASM_TEST_COMPILER) macro(add_filecheck_test name) cmake_parse_arguments(ARG "" "" "CHECK_PREFIXES" ${ARGV}) add_library(${name} OBJECT ${name}.cc) - target_link_libraries(${name} PRIVATE benchmark::benchmark) set_target_properties(${name} PROPERTIES COMPILE_FLAGS "-S ${ASM_TEST_FLAGS}") set(ASM_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/${name}.s") add_custom_target(copy_${name} ALL diff --git a/third-party/benchmark/test/CMakeLists.txt b/third-party/benchmark/test/CMakeLists.txt index 1de175f98d34..162af53f80f7 100644 --- a/third-party/benchmark/test/CMakeLists.txt +++ b/third-party/benchmark/test/CMakeLists.txt @@ -1,12 +1,8 @@ # Enable the tests -set(THREADS_PREFER_PTHREAD_FLAG ON) - find_package(Threads REQUIRED) include(CheckCXXCompilerFlag) -add_cxx_compiler_flag(-Wno-unused-variable) - # NOTE: Some tests use `` to perform the test. Therefore we must # strip -DNDEBUG from the default CMake flags in DEBUG mode. string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) @@ -26,10 +22,6 @@ if( NOT uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG" ) endforeach() endif() -if (NOT BUILD_SHARED_LIBS) - add_definitions(-DBENCHMARK_STATIC_DEFINE) -endif() - check_cxx_compiler_flag(-O3 BENCHMARK_HAS_O3_FLAG) set(BENCHMARK_O3_FLAG "") if (BENCHMARK_HAS_O3_FLAG) @@ -43,14 +35,10 @@ if (DEFINED BENCHMARK_CXX_LINKER_FLAGS) endif() add_library(output_test_helper STATIC output_test_helper.cc output_test.h) -target_link_libraries(output_test_helper PRIVATE benchmark::benchmark) macro(compile_benchmark_test name) add_executable(${name} "${name}.cc") target_link_libraries(${name} benchmark::benchmark ${CMAKE_THREAD_LIBS_INIT}) - if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "NVHPC") - target_compile_options( ${name} PRIVATE --diag_suppress partial_override ) - endif() endmacro(compile_benchmark_test) macro(compile_benchmark_test_with_main name) @@ -60,43 +48,26 @@ endmacro(compile_benchmark_test_with_main) macro(compile_output_test name) add_executable(${name} "${name}.cc" output_test.h) - target_link_libraries(${name} output_test_helper benchmark::benchmark_main + target_link_libraries(${name} output_test_helper benchmark::benchmark ${BENCHMARK_CXX_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) endmacro(compile_output_test) -macro(benchmark_add_test) - add_test(${ARGV}) - if(WIN32 AND BUILD_SHARED_LIBS) - cmake_parse_arguments(TEST "" "NAME" "" ${ARGN}) - set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") - endif() -endmacro(benchmark_add_test) - # Demonstration executable compile_benchmark_test(benchmark_test) -benchmark_add_test(NAME benchmark COMMAND benchmark_test --benchmark_min_time=0.01s) +add_test(NAME benchmark COMMAND benchmark_test --benchmark_min_time=0.01) compile_benchmark_test(spec_arg_test) -benchmark_add_test(NAME spec_arg COMMAND spec_arg_test --benchmark_filter=BM_NotChosen) - -compile_benchmark_test(spec_arg_verbosity_test) -benchmark_add_test(NAME spec_arg_verbosity COMMAND spec_arg_verbosity_test --v=42) +add_test(NAME spec_arg COMMAND spec_arg_test --benchmark_filter=BM_NotChosen) compile_benchmark_test(benchmark_setup_teardown_test) -benchmark_add_test(NAME benchmark_setup_teardown COMMAND benchmark_setup_teardown_test) +add_test(NAME benchmark_setup_teardown COMMAND benchmark_setup_teardown_test) compile_benchmark_test(filter_test) macro(add_filter_test name filter expect) - benchmark_add_test(NAME ${name} COMMAND filter_test --benchmark_min_time=0.01s --benchmark_filter=${filter} ${expect}) - benchmark_add_test(NAME ${name}_list_only COMMAND filter_test --benchmark_list_tests --benchmark_filter=${filter} ${expect}) + add_test(NAME ${name} COMMAND filter_test --benchmark_min_time=0.01 --benchmark_filter=${filter} ${expect}) + add_test(NAME ${name}_list_only COMMAND filter_test --benchmark_list_tests --benchmark_filter=${filter} ${expect}) endmacro(add_filter_test) -compile_benchmark_test(benchmark_min_time_flag_time_test) -benchmark_add_test(NAME min_time_flag_time COMMAND benchmark_min_time_flag_time_test) - -compile_benchmark_test(benchmark_min_time_flag_iters_test) -benchmark_add_test(NAME min_time_flag_iters COMMAND benchmark_min_time_flag_iters_test) - add_filter_test(filter_simple "Foo" 3) add_filter_test(filter_simple_negative "-Foo" 2) add_filter_test(filter_suffix "BM_.*" 4) @@ -117,83 +88,78 @@ add_filter_test(filter_regex_end ".*Ba$" 1) add_filter_test(filter_regex_end_negative "-.*Ba$" 4) compile_benchmark_test(options_test) -benchmark_add_test(NAME options_benchmarks COMMAND options_test --benchmark_min_time=0.01s) +add_test(NAME options_benchmarks COMMAND options_test --benchmark_min_time=0.01) compile_benchmark_test(basic_test) -benchmark_add_test(NAME basic_benchmark COMMAND basic_test --benchmark_min_time=0.01s) +add_test(NAME basic_benchmark COMMAND basic_test --benchmark_min_time=0.01) compile_output_test(repetitions_test) -benchmark_add_test(NAME repetitions_benchmark COMMAND repetitions_test --benchmark_min_time=0.01s --benchmark_repetitions=3) +add_test(NAME repetitions_benchmark COMMAND repetitions_test --benchmark_min_time=0.01 --benchmark_repetitions=3) compile_benchmark_test(diagnostics_test) -benchmark_add_test(NAME diagnostics_test COMMAND diagnostics_test --benchmark_min_time=0.01s) +add_test(NAME diagnostics_test COMMAND diagnostics_test --benchmark_min_time=0.01) compile_benchmark_test(skip_with_error_test) -benchmark_add_test(NAME skip_with_error_test COMMAND skip_with_error_test --benchmark_min_time=0.01s) +add_test(NAME skip_with_error_test COMMAND skip_with_error_test --benchmark_min_time=0.01) compile_benchmark_test(donotoptimize_test) -# Enable errors for deprecated deprecations (DoNotOptimize(Tp const& value)). -check_cxx_compiler_flag(-Werror=deprecated-declarations BENCHMARK_HAS_DEPRECATED_DECLARATIONS_FLAG) -if (BENCHMARK_HAS_DEPRECATED_DECLARATIONS_FLAG) - target_compile_options (donotoptimize_test PRIVATE "-Werror=deprecated-declarations") -endif() # Some of the issues with DoNotOptimize only occur when optimization is enabled check_cxx_compiler_flag(-O3 BENCHMARK_HAS_O3_FLAG) if (BENCHMARK_HAS_O3_FLAG) set_target_properties(donotoptimize_test PROPERTIES COMPILE_FLAGS "-O3") endif() -benchmark_add_test(NAME donotoptimize_test COMMAND donotoptimize_test --benchmark_min_time=0.01s) +add_test(NAME donotoptimize_test COMMAND donotoptimize_test --benchmark_min_time=0.01) compile_benchmark_test(fixture_test) -benchmark_add_test(NAME fixture_test COMMAND fixture_test --benchmark_min_time=0.01s) +add_test(NAME fixture_test COMMAND fixture_test --benchmark_min_time=0.01) compile_benchmark_test(register_benchmark_test) -benchmark_add_test(NAME register_benchmark_test COMMAND register_benchmark_test --benchmark_min_time=0.01s) +add_test(NAME register_benchmark_test COMMAND register_benchmark_test --benchmark_min_time=0.01) compile_benchmark_test(map_test) -benchmark_add_test(NAME map_test COMMAND map_test --benchmark_min_time=0.01s) +add_test(NAME map_test COMMAND map_test --benchmark_min_time=0.01) compile_benchmark_test(multiple_ranges_test) -benchmark_add_test(NAME multiple_ranges_test COMMAND multiple_ranges_test --benchmark_min_time=0.01s) +add_test(NAME multiple_ranges_test COMMAND multiple_ranges_test --benchmark_min_time=0.01) compile_benchmark_test(args_product_test) -benchmark_add_test(NAME args_product_test COMMAND args_product_test --benchmark_min_time=0.01s) +add_test(NAME args_product_test COMMAND args_product_test --benchmark_min_time=0.01) compile_benchmark_test_with_main(link_main_test) -benchmark_add_test(NAME link_main_test COMMAND link_main_test --benchmark_min_time=0.01s) +add_test(NAME link_main_test COMMAND link_main_test --benchmark_min_time=0.01) compile_output_test(reporter_output_test) -benchmark_add_test(NAME reporter_output_test COMMAND reporter_output_test --benchmark_min_time=0.01s) +add_test(NAME reporter_output_test COMMAND reporter_output_test --benchmark_min_time=0.01) compile_output_test(templated_fixture_test) -benchmark_add_test(NAME templated_fixture_test COMMAND templated_fixture_test --benchmark_min_time=0.01s) +add_test(NAME templated_fixture_test COMMAND templated_fixture_test --benchmark_min_time=0.01) compile_output_test(user_counters_test) -benchmark_add_test(NAME user_counters_test COMMAND user_counters_test --benchmark_min_time=0.01s) +add_test(NAME user_counters_test COMMAND user_counters_test --benchmark_min_time=0.01) compile_output_test(perf_counters_test) -benchmark_add_test(NAME perf_counters_test COMMAND perf_counters_test --benchmark_min_time=0.01s --benchmark_perf_counters=CYCLES,INSTRUCTIONS) +add_test(NAME perf_counters_test COMMAND perf_counters_test --benchmark_min_time=0.01 --benchmark_perf_counters=CYCLES,BRANCHES) compile_output_test(internal_threading_test) -benchmark_add_test(NAME internal_threading_test COMMAND internal_threading_test --benchmark_min_time=0.01s) +add_test(NAME internal_threading_test COMMAND internal_threading_test --benchmark_min_time=0.01) compile_output_test(report_aggregates_only_test) -benchmark_add_test(NAME report_aggregates_only_test COMMAND report_aggregates_only_test --benchmark_min_time=0.01s) +add_test(NAME report_aggregates_only_test COMMAND report_aggregates_only_test --benchmark_min_time=0.01) compile_output_test(display_aggregates_only_test) -benchmark_add_test(NAME display_aggregates_only_test COMMAND display_aggregates_only_test --benchmark_min_time=0.01s) +add_test(NAME display_aggregates_only_test COMMAND display_aggregates_only_test --benchmark_min_time=0.01) compile_output_test(user_counters_tabular_test) -benchmark_add_test(NAME user_counters_tabular_test COMMAND user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01s) +add_test(NAME user_counters_tabular_test COMMAND user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01) compile_output_test(user_counters_thousands_test) -benchmark_add_test(NAME user_counters_thousands_test COMMAND user_counters_thousands_test --benchmark_min_time=0.01s) +add_test(NAME user_counters_thousands_test COMMAND user_counters_thousands_test --benchmark_min_time=0.01) compile_output_test(memory_manager_test) -benchmark_add_test(NAME memory_manager_test COMMAND memory_manager_test --benchmark_min_time=0.01s) +add_test(NAME memory_manager_test COMMAND memory_manager_test --benchmark_min_time=0.01) -# MSVC does not allow to set the language standard to C++98/03. -if(NOT (MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC")) +check_cxx_compiler_flag(-std=c++03 BENCHMARK_HAS_CXX03_FLAG) +if (BENCHMARK_HAS_CXX03_FLAG) compile_benchmark_test(cxx03_test) set_target_properties(cxx03_test PROPERTIES @@ -204,22 +170,22 @@ if(NOT (MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC")) # causing the test to fail to compile. To prevent this we explicitly disable # the warning. check_cxx_compiler_flag(-Wno-odr BENCHMARK_HAS_WNO_ODR) - check_cxx_compiler_flag(-Wno-lto-type-mismatch BENCHMARK_HAS_WNO_LTO_TYPE_MISMATCH) - # Cannot set_target_properties multiple times here because the warnings will - # be overwritten on each call - set (DISABLE_LTO_WARNINGS "") - if (BENCHMARK_HAS_WNO_ODR) - set(DISABLE_LTO_WARNINGS "${DISABLE_LTO_WARNINGS} -Wno-odr") + if (BENCHMARK_ENABLE_LTO AND BENCHMARK_HAS_WNO_ODR) + set_target_properties(cxx03_test + PROPERTIES + LINK_FLAGS "-Wno-odr") endif() - if (BENCHMARK_HAS_WNO_LTO_TYPE_MISMATCH) - set(DISABLE_LTO_WARNINGS "${DISABLE_LTO_WARNINGS} -Wno-lto-type-mismatch") - endif() - set_target_properties(cxx03_test PROPERTIES LINK_FLAGS "${DISABLE_LTO_WARNINGS}") - benchmark_add_test(NAME cxx03 COMMAND cxx03_test --benchmark_min_time=0.01s) + add_test(NAME cxx03 COMMAND cxx03_test --benchmark_min_time=0.01) endif() +# Attempt to work around flaky test failures when running on Appveyor servers. +if (DEFINED ENV{APPVEYOR}) + set(COMPLEXITY_MIN_TIME "0.5") +else() + set(COMPLEXITY_MIN_TIME "0.01") +endif() compile_output_test(complexity_test) -benchmark_add_test(NAME complexity_benchmark COMMAND complexity_test --benchmark_min_time=1000000x) +add_test(NAME complexity_benchmark COMMAND complexity_test --benchmark_min_time=${COMPLEXITY_MIN_TIME}) ############################################################################### # GoogleTest Unit Tests @@ -234,12 +200,7 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) macro(add_gtest name) compile_gtest(${name}) - benchmark_add_test(NAME ${name} COMMAND ${name}) - if(WIN32 AND BUILD_SHARED_LIBS) - set_tests_properties(${name} PROPERTIES - ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$" - ) - endif() + add_test(NAME ${name} COMMAND ${name}) endmacro() add_gtest(benchmark_gtest) @@ -249,8 +210,6 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) add_gtest(statistics_gtest) add_gtest(string_util_gtest) add_gtest(perf_counters_gtest) - add_gtest(time_unit_gtest) - add_gtest(min_time_parse_gtest) endif(BENCHMARK_ENABLE_GTEST_TESTS) ############################################################################### diff --git a/third-party/benchmark/test/args_product_test.cc b/third-party/benchmark/test/args_product_test.cc index 63b8b71e45a0..d44f391f7480 100644 --- a/third-party/benchmark/test/args_product_test.cc +++ b/third-party/benchmark/test/args_product_test.cc @@ -23,7 +23,7 @@ class ArgsProductFixture : public ::benchmark::Fixture { {2, 15, 10, 9}, {4, 5, 6, 11}}) {} - void SetUp(const ::benchmark::State& state) override { + void SetUp(const ::benchmark::State& state) BENCHMARK_OVERRIDE { std::vector ranges = {state.range(0), state.range(1), state.range(2), state.range(3)}; @@ -34,7 +34,7 @@ class ArgsProductFixture : public ::benchmark::Fixture { // NOTE: This is not TearDown as we want to check after _all_ runs are // complete. - ~ArgsProductFixture() override { + virtual ~ArgsProductFixture() { if (actualValues != expectedValues) { std::cout << "EXPECTED\n"; for (const auto& v : expectedValues) { diff --git a/third-party/benchmark/test/basic_test.cc b/third-party/benchmark/test/basic_test.cc index c25bec7ddd58..3a8fd42a8cdd 100644 --- a/third-party/benchmark/test/basic_test.cc +++ b/third-party/benchmark/test/basic_test.cc @@ -5,8 +5,7 @@ void BM_empty(benchmark::State& state) { for (auto _ : state) { - auto iterations = double(state.iterations()) * double(state.iterations()); - benchmark::DoNotOptimize(iterations); + benchmark::DoNotOptimize(state.iterations()); } } BENCHMARK(BM_empty); @@ -148,7 +147,7 @@ void BM_OneTemplateFunc(benchmark::State& state) { auto arg = state.range(0); T sum = 0; for (auto _ : state) { - sum += static_cast(arg); + sum += arg; } } BENCHMARK(BM_OneTemplateFunc)->Arg(1); @@ -160,8 +159,8 @@ void BM_TwoTemplateFunc(benchmark::State& state) { A sum = 0; B prod = 1; for (auto _ : state) { - sum += static_cast(arg); - prod *= static_cast(arg); + sum += arg; + prod *= arg; } } BENCHMARK(BM_TwoTemplateFunc)->Arg(1); diff --git a/third-party/benchmark/test/benchmark_gtest.cc b/third-party/benchmark/test/benchmark_gtest.cc index 2c9e555d92dc..14a885ba46da 100644 --- a/third-party/benchmark/test/benchmark_gtest.cc +++ b/third-party/benchmark/test/benchmark_gtest.cc @@ -3,12 +3,12 @@ #include #include "../src/benchmark_register.h" -#include "benchmark/benchmark.h" #include "gmock/gmock.h" #include "gtest/gtest.h" namespace benchmark { namespace internal { +extern std::map* global_context; namespace { @@ -38,9 +38,8 @@ TEST(AddRangeTest, Advanced64) { TEST(AddRangeTest, FullRange8) { std::vector dst; - AddRange(&dst, int8_t{1}, std::numeric_limits::max(), int8_t{8}); - EXPECT_THAT( - dst, testing::ElementsAre(int8_t{1}, int8_t{8}, int8_t{64}, int8_t{127})); + AddRange(&dst, int8_t{1}, std::numeric_limits::max(), 8); + EXPECT_THAT(dst, testing::ElementsAre(1, 8, 64, 127)); } TEST(AddRangeTest, FullRange64) { @@ -130,13 +129,11 @@ TEST(AddRangeTest, FullNegativeRange64) { TEST(AddRangeTest, Simple8) { std::vector dst; - AddRange(&dst, int8_t{1}, int8_t{8}, int8_t{2}); - EXPECT_THAT(dst, - testing::ElementsAre(int8_t{1}, int8_t{2}, int8_t{4}, int8_t{8})); + AddRange(&dst, 1, 8, 2); + EXPECT_THAT(dst, testing::ElementsAre(1, 2, 4, 8)); } TEST(AddCustomContext, Simple) { - std::map *&global_context = GetGlobalContext(); EXPECT_THAT(global_context, nullptr); AddCustomContext("foo", "bar"); @@ -151,7 +148,6 @@ TEST(AddCustomContext, Simple) { } TEST(AddCustomContext, DuplicateKey) { - std::map *&global_context = GetGlobalContext(); EXPECT_THAT(global_context, nullptr); AddCustomContext("foo", "bar"); diff --git a/third-party/benchmark/test/benchmark_name_gtest.cc b/third-party/benchmark/test/benchmark_name_gtest.cc index 0a6746d04df7..afb401c1f532 100644 --- a/third-party/benchmark/test/benchmark_name_gtest.cc +++ b/third-party/benchmark/test/benchmark_name_gtest.cc @@ -32,14 +32,6 @@ TEST(BenchmarkNameTest, MinTime) { EXPECT_EQ(name.str(), "function_name/some_args:3/4/min_time:3.4s"); } -TEST(BenchmarkNameTest, MinWarmUpTime) { - auto name = BenchmarkName(); - name.function_name = "function_name"; - name.args = "some_args:3/4"; - name.min_warmup_time = "min_warmup_time:3.5s"; - EXPECT_EQ(name.str(), "function_name/some_args:3/4/min_warmup_time:3.5s"); -} - TEST(BenchmarkNameTest, Iterations) { auto name = BenchmarkName(); name.function_name = "function_name"; diff --git a/third-party/benchmark/test/benchmark_random_interleaving_gtest.cc b/third-party/benchmark/test/benchmark_random_interleaving_gtest.cc index 7f2086750d53..d04befa8e381 100644 --- a/third-party/benchmark/test/benchmark_random_interleaving_gtest.cc +++ b/third-party/benchmark/test/benchmark_random_interleaving_gtest.cc @@ -51,9 +51,10 @@ class BenchmarkTest : public testing::Test { void Execute(const std::string& pattern) { queue->Clear(); - std::unique_ptr reporter(new NullReporter()); + BenchmarkReporter* reporter = new NullReporter; FLAGS_benchmark_filter = pattern; - RunSpecifiedBenchmarks(reporter.get()); + RunSpecifiedBenchmarks(reporter); + delete reporter; queue->Put("DONE"); // End marker } diff --git a/third-party/benchmark/test/benchmark_setup_teardown_test.cc b/third-party/benchmark/test/benchmark_setup_teardown_test.cc index 6c3cc2e58fbd..efa34e15c129 100644 --- a/third-party/benchmark/test/benchmark_setup_teardown_test.cc +++ b/third-party/benchmark/test/benchmark_setup_teardown_test.cc @@ -10,19 +10,19 @@ // Test that Setup() and Teardown() are called exactly once // for each benchmark run (single-threaded). -namespace singlethreaded { +namespace single { static int setup_call = 0; static int teardown_call = 0; -} // namespace singlethreaded +} // namespace single static void DoSetup1(const benchmark::State& state) { - ++singlethreaded::setup_call; + ++single::setup_call; // Setup/Teardown should never be called with any thread_idx != 0. assert(state.thread_index() == 0); } static void DoTeardown1(const benchmark::State& state) { - ++singlethreaded::teardown_call; + ++single::teardown_call; assert(state.thread_index() == 0); } @@ -80,11 +80,11 @@ int fixture_setup = 0; class FIXTURE_BECHMARK_NAME : public ::benchmark::Fixture { public: - void SetUp(const ::benchmark::State&) override { + void SetUp(const ::benchmark::State&) BENCHMARK_OVERRIDE { fixture_interaction::fixture_setup++; } - ~FIXTURE_BECHMARK_NAME() override {} + ~FIXTURE_BECHMARK_NAME() {} }; BENCHMARK_F(FIXTURE_BECHMARK_NAME, BM_WithFixture)(benchmark::State& st) { @@ -134,8 +134,8 @@ int main(int argc, char** argv) { assert(ret > 0); // Setup/Teardown is called once for each arg group (1,3,5,7). - assert(singlethreaded::setup_call == 4); - assert(singlethreaded::teardown_call == 4); + assert(single::setup_call == 4); + assert(single::teardown_call == 4); // 3 group of threads calling this function (3,5,10). assert(concurrent::setup_call.load(std::memory_order_relaxed) == 3); @@ -145,7 +145,7 @@ int main(int argc, char** argv) { // Setup is called 4 times, once for each arg group (1,3,5,7) assert(fixture_interaction::setup == 4); - // Fixture::Setup is called every time the bm routine is run. + // Fixture::Setup is called everytime the bm routine is run. // The exact number is indeterministic, so we just assert that // it's more than setup. assert(fixture_interaction::fixture_setup > fixture_interaction::setup); diff --git a/third-party/benchmark/test/benchmark_test.cc b/third-party/benchmark/test/benchmark_test.cc index 8b14017d03a5..2906cdcde997 100644 --- a/third-party/benchmark/test/benchmark_test.cc +++ b/third-party/benchmark/test/benchmark_test.cc @@ -5,7 +5,6 @@ #include #include -#include #include #include #include @@ -16,7 +15,6 @@ #include #include #include -#include #include #include @@ -28,7 +26,7 @@ namespace { -int BENCHMARK_NOINLINE Factorial(int n) { +int BENCHMARK_NOINLINE Factorial(uint32_t n) { return (n == 1) ? 1 : n * Factorial(n - 1); } @@ -76,8 +74,7 @@ BENCHMARK_RANGE(BM_CalculatePiRange, 1, 1024 * 1024); static void BM_CalculatePi(benchmark::State& state) { static const int depth = 1024; for (auto _ : state) { - double pi = CalculatePi(static_cast(depth)); - benchmark::DoNotOptimize(pi); + benchmark::DoNotOptimize(CalculatePi(static_cast(depth))); } } BENCHMARK(BM_CalculatePi)->Threads(8); @@ -93,8 +90,7 @@ static void BM_SetInsert(benchmark::State& state) { for (int j = 0; j < state.range(1); ++j) data.insert(rand()); } state.SetItemsProcessed(state.iterations() * state.range(1)); - state.SetBytesProcessed(state.iterations() * state.range(1) * - static_cast(sizeof(int))); + state.SetBytesProcessed(state.iterations() * state.range(1) * sizeof(int)); } // Test many inserts at once to reduce the total iterations needed. Otherwise, @@ -112,7 +108,7 @@ static void BM_Sequential(benchmark::State& state) { } const int64_t items_processed = state.iterations() * state.range(0); state.SetItemsProcessed(items_processed); - state.SetBytesProcessed(items_processed * static_cast(sizeof(v))); + state.SetBytesProcessed(items_processed * sizeof(v)); } BENCHMARK_TEMPLATE2(BM_Sequential, std::vector, int) ->Range(1 << 0, 1 << 10); @@ -126,10 +122,7 @@ static void BM_StringCompare(benchmark::State& state) { size_t len = static_cast(state.range(0)); std::string s1(len, '-'); std::string s2(len, '-'); - for (auto _ : state) { - auto comp = s1.compare(s2); - benchmark::DoNotOptimize(comp); - } + for (auto _ : state) benchmark::DoNotOptimize(s1.compare(s2)); } BENCHMARK(BM_StringCompare)->Range(1, 1 << 20); @@ -176,7 +169,7 @@ static void BM_ParallelMemset(benchmark::State& state) { for (int i = from; i < to; i++) { // No need to lock test_vector_mu as ranges // do not overlap between threads. - benchmark::DoNotOptimize(test_vector->at(static_cast(i)) = 1); + benchmark::DoNotOptimize(test_vector->at(i) = 1); } } @@ -227,31 +220,6 @@ void BM_non_template_args(benchmark::State& state, int, double) { } BENCHMARK_CAPTURE(BM_non_template_args, basic_test, 0, 0); -template -void BM_template2_capture(benchmark::State& state, ExtraArgs&&... extra_args) { - static_assert(std::is_same::value, ""); - static_assert(std::is_same::value, ""); - static_assert(std::is_same::value, ""); - unsigned int dummy[sizeof...(ExtraArgs)] = {extra_args...}; - assert(dummy[0] == 42); - for (auto _ : state) { - } -} -BENCHMARK_TEMPLATE2_CAPTURE(BM_template2_capture, void, char*, foo, 42U); -BENCHMARK_CAPTURE((BM_template2_capture), foo, 42U); - -template -void BM_template1_capture(benchmark::State& state, ExtraArgs&&... extra_args) { - static_assert(std::is_same::value, ""); - static_assert(std::is_same::value, ""); - unsigned long dummy[sizeof...(ExtraArgs)] = {extra_args...}; - assert(dummy[0] == 24); - for (auto _ : state) { - } -} -BENCHMARK_TEMPLATE1_CAPTURE(BM_template1_capture, void, foo, 24UL); -BENCHMARK_CAPTURE(BM_template1_capture, foo, 24UL); - #endif // BENCHMARK_HAS_CXX11 static void BM_DenseThreadRanges(benchmark::State& st) { @@ -276,25 +244,4 @@ BENCHMARK(BM_DenseThreadRanges)->Arg(1)->DenseThreadRange(1, 3); BENCHMARK(BM_DenseThreadRanges)->Arg(2)->DenseThreadRange(1, 4, 2); BENCHMARK(BM_DenseThreadRanges)->Arg(3)->DenseThreadRange(5, 14, 3); -static void BM_BenchmarkName(benchmark::State& state) { - for (auto _ : state) { - } - - // Check that the benchmark name is passed correctly to `state`. - assert("BM_BenchmarkName" == state.name()); -} -BENCHMARK(BM_BenchmarkName); - -// regression test for #1446 -template -static void BM_templated_test(benchmark::State& state) { - for (auto _ : state) { - type created_string; - benchmark::DoNotOptimize(created_string); - } -} - -static auto BM_templated_test_double = BM_templated_test>; -BENCHMARK(BM_templated_test_double); - BENCHMARK_MAIN(); diff --git a/third-party/benchmark/test/clobber_memory_assembly_test.cc b/third-party/benchmark/test/clobber_memory_assembly_test.cc index 54e26ccdadf4..ab269130cd5c 100644 --- a/third-party/benchmark/test/clobber_memory_assembly_test.cc +++ b/third-party/benchmark/test/clobber_memory_assembly_test.cc @@ -3,7 +3,6 @@ #ifdef __clang__ #pragma clang diagnostic ignored "-Wreturn-type" #endif -BENCHMARK_DISABLE_DEPRECATED_WARNING extern "C" { diff --git a/third-party/benchmark/test/complexity_test.cc b/third-party/benchmark/test/complexity_test.cc index fb4ad1ad53a9..ea268b545988 100644 --- a/third-party/benchmark/test/complexity_test.cc +++ b/third-party/benchmark/test/complexity_test.cc @@ -26,7 +26,7 @@ int AddComplexityTest(const std::string &test_name, AddCases( TC_ConsoleOut, {{"^%bigo_name %bigo_str %bigo_str[ ]*$"}, - {"^%bigo_name", MR_Not}, // Assert we we didn't only matched a name. + {"^%bigo_name", MR_Not}, // Assert we didn't only matched a name. {"^%rms_name %rms %rms[ ]*$", MR_Next}}); AddCases( TC_JSONOut, @@ -69,44 +69,35 @@ int AddComplexityTest(const std::string &test_name, void BM_Complexity_O1(benchmark::State &state) { for (auto _ : state) { - // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); - long tmp = state.iterations(); - benchmark::DoNotOptimize(tmp); - for (benchmark::IterationCount i = 0; i < state.iterations(); ++i) { - benchmark::DoNotOptimize(state.iterations()); - tmp *= state.iterations(); - benchmark::DoNotOptimize(tmp); + for (int i = 0; i < 1024; ++i) { + benchmark::DoNotOptimize(&i); } - - // always 1ns per iteration - state.SetIterationTime(42 * 1e-9); } state.SetComplexityN(state.range(0)); } +BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1); +BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(); BENCHMARK(BM_Complexity_O1) ->Range(1, 1 << 18) - ->UseManualTime() - ->Complexity(benchmark::o1); -BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->UseManualTime()->Complexity(); -BENCHMARK(BM_Complexity_O1) - ->Range(1, 1 << 18) - ->UseManualTime() ->Complexity([](benchmark::IterationCount) { return 1.0; }); -const char *one_test_name = "BM_Complexity_O1/manual_time"; -const char *big_o_1_test_name = "BM_Complexity_O1/manual_time_BigO"; -const char *rms_o_1_test_name = "BM_Complexity_O1/manual_time_RMS"; -const char *enum_auto_big_o_1 = "\\([0-9]+\\)"; +const char *one_test_name = "BM_Complexity_O1"; +const char *big_o_1_test_name = "BM_Complexity_O1_BigO"; +const char *rms_o_1_test_name = "BM_Complexity_O1_RMS"; +const char *enum_big_o_1 = "\\([0-9]+\\)"; +// FIXME: Tolerate both '(1)' and 'lgN' as output when the complexity is auto +// deduced. +// See https://github.com/google/benchmark/issues/272 +const char *auto_big_o_1 = "(\\([0-9]+\\))|(lgN)"; const char *lambda_big_o_1 = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - enum_auto_big_o_1, /*family_index=*/0); + enum_big_o_1, /*family_index=*/0); -// Add auto tests +// Add auto enum tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - enum_auto_big_o_1, /*family_index=*/1); + auto_big_o_1, /*family_index=*/1); // Add lambda tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, @@ -116,44 +107,42 @@ ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, // --------------------------- Testing BigO O(N) --------------------------- // // ========================================================================= // +std::vector ConstructRandomVector(int64_t size) { + std::vector v; + v.reserve(static_cast(size)); + for (int i = 0; i < size; ++i) { + v.push_back(static_cast(std::rand() % size)); + } + return v; +} + void BM_Complexity_O_N(benchmark::State &state) { + auto v = ConstructRandomVector(state.range(0)); + // Test worst case scenario (item not in vector) + const int64_t item_not_in_vector = state.range(0) * 2; for (auto _ : state) { - // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); - long tmp = state.iterations(); - benchmark::DoNotOptimize(tmp); - for (benchmark::IterationCount i = 0; i < state.iterations(); ++i) { - benchmark::DoNotOptimize(state.iterations()); - tmp *= state.iterations(); - benchmark::DoNotOptimize(tmp); - } - - // 1ns per iteration per entry - state.SetIterationTime(static_cast(state.range(0)) * 42.0 * 1e-9); + benchmark::DoNotOptimize(std::find(v.begin(), v.end(), item_not_in_vector)); } state.SetComplexityN(state.range(0)); } BENCHMARK(BM_Complexity_O_N) ->RangeMultiplier(2) - ->Range(1 << 10, 1 << 20) - ->UseManualTime() + ->Range(1 << 10, 1 << 16) ->Complexity(benchmark::oN); BENCHMARK(BM_Complexity_O_N) ->RangeMultiplier(2) - ->Range(1 << 10, 1 << 20) - ->UseManualTime() - ->Complexity(); -BENCHMARK(BM_Complexity_O_N) - ->RangeMultiplier(2) - ->Range(1 << 10, 1 << 20) - ->UseManualTime() + ->Range(1 << 10, 1 << 16) ->Complexity([](benchmark::IterationCount n) -> double { return static_cast(n); }); +BENCHMARK(BM_Complexity_O_N) + ->RangeMultiplier(2) + ->Range(1 << 10, 1 << 16) + ->Complexity(); -const char *n_test_name = "BM_Complexity_O_N/manual_time"; -const char *big_o_n_test_name = "BM_Complexity_O_N/manual_time_BigO"; -const char *rms_o_n_test_name = "BM_Complexity_O_N/manual_time_RMS"; +const char *n_test_name = "BM_Complexity_O_N"; +const char *big_o_n_test_name = "BM_Complexity_O_N_BigO"; +const char *rms_o_n_test_name = "BM_Complexity_O_N_RMS"; const char *enum_auto_big_o_n = "N"; const char *lambda_big_o_n = "f\\(N\\)"; @@ -161,57 +150,40 @@ const char *lambda_big_o_n = "f\\(N\\)"; ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, enum_auto_big_o_n, /*family_index=*/3); -// Add auto tests -ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - enum_auto_big_o_n, /*family_index=*/4); - // Add lambda tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - lambda_big_o_n, /*family_index=*/5); + lambda_big_o_n, /*family_index=*/4); // ========================================================================= // -// ------------------------- Testing BigO O(NlgN) ------------------------- // +// ------------------------- Testing BigO O(N*lgN) ------------------------- // // ========================================================================= // -static const double kLog2E = 1.44269504088896340736; static void BM_Complexity_O_N_log_N(benchmark::State &state) { + auto v = ConstructRandomVector(state.range(0)); for (auto _ : state) { - // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); - long tmp = state.iterations(); - benchmark::DoNotOptimize(tmp); - for (benchmark::IterationCount i = 0; i < state.iterations(); ++i) { - benchmark::DoNotOptimize(state.iterations()); - tmp *= state.iterations(); - benchmark::DoNotOptimize(tmp); - } - - state.SetIterationTime(static_cast(state.range(0)) * kLog2E * - std::log(state.range(0)) * 42.0 * 1e-9); + std::sort(v.begin(), v.end()); } state.SetComplexityN(state.range(0)); } +static const double kLog2E = 1.44269504088896340736; BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2) - ->Range(1 << 10, 1U << 24) - ->UseManualTime() + ->Range(1 << 10, 1 << 16) ->Complexity(benchmark::oNLogN); BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2) - ->Range(1 << 10, 1U << 24) - ->UseManualTime() - ->Complexity(); -BENCHMARK(BM_Complexity_O_N_log_N) - ->RangeMultiplier(2) - ->Range(1 << 10, 1U << 24) - ->UseManualTime() + ->Range(1 << 10, 1 << 16) ->Complexity([](benchmark::IterationCount n) { - return kLog2E * static_cast(n) * std::log(static_cast(n)); + return kLog2E * n * log(static_cast(n)); }); +BENCHMARK(BM_Complexity_O_N_log_N) + ->RangeMultiplier(2) + ->Range(1 << 10, 1 << 16) + ->Complexity(); -const char *n_lg_n_test_name = "BM_Complexity_O_N_log_N/manual_time"; -const char *big_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N/manual_time_BigO"; -const char *rms_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N/manual_time_RMS"; +const char *n_lg_n_test_name = "BM_Complexity_O_N_log_N"; +const char *big_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_BigO"; +const char *rms_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_RMS"; const char *enum_auto_big_o_n_lg_n = "NlgN"; const char *lambda_big_o_n_lg_n = "f\\(N\\)"; @@ -220,15 +192,10 @@ ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n, /*family_index=*/6); -// NOTE: auto big-o is wron.g -ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, - rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n, - /*family_index=*/7); - -//// Add lambda tests +// Add lambda tests ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n, - /*family_index=*/8); + /*family_index=*/7); // ========================================================================= // // -------- Testing formatting of Complexity with captured args ------------ // @@ -238,30 +205,19 @@ void BM_ComplexityCaptureArgs(benchmark::State &state, int n) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero benchmark::DoNotOptimize(state.iterations()); - long tmp = state.iterations(); - benchmark::DoNotOptimize(tmp); - for (benchmark::IterationCount i = 0; i < state.iterations(); ++i) { - benchmark::DoNotOptimize(state.iterations()); - tmp *= state.iterations(); - benchmark::DoNotOptimize(tmp); - } - - state.SetIterationTime(static_cast(state.range(0)) * 42.0 * 1e-9); } state.SetComplexityN(n); } BENCHMARK_CAPTURE(BM_ComplexityCaptureArgs, capture_test, 100) - ->UseManualTime() ->Complexity(benchmark::oN) ->Ranges({{1, 2}, {3, 4}}); const std::string complexity_capture_name = - "BM_ComplexityCaptureArgs/capture_test/manual_time"; + "BM_ComplexityCaptureArgs/capture_test"; ADD_COMPLEXITY_CASES(complexity_capture_name, complexity_capture_name + "_BigO", - complexity_capture_name + "_RMS", "N", - /*family_index=*/9); + complexity_capture_name + "_RMS", "N", /*family_index=*/9); // ========================================================================= // // --------------------------- TEST CASES END ------------------------------ // diff --git a/third-party/benchmark/test/diagnostics_test.cc b/third-party/benchmark/test/diagnostics_test.cc index 7c68a98929d8..c54d5b0d708a 100644 --- a/third-party/benchmark/test/diagnostics_test.cc +++ b/third-party/benchmark/test/diagnostics_test.cc @@ -49,8 +49,7 @@ void BM_diagnostic_test(benchmark::State& state) { if (called_once == false) try_invalid_pause_resume(state); for (auto _ : state) { - auto iterations = double(state.iterations()) * double(state.iterations()); - benchmark::DoNotOptimize(iterations); + benchmark::DoNotOptimize(state.iterations()); } if (called_once == false) try_invalid_pause_resume(state); @@ -65,8 +64,7 @@ void BM_diagnostic_test_keep_running(benchmark::State& state) { if (called_once == false) try_invalid_pause_resume(state); while (state.KeepRunning()) { - auto iterations = double(state.iterations()) * double(state.iterations()); - benchmark::DoNotOptimize(iterations); + benchmark::DoNotOptimize(state.iterations()); } if (called_once == false) try_invalid_pause_resume(state); @@ -76,16 +74,7 @@ void BM_diagnostic_test_keep_running(benchmark::State& state) { BENCHMARK(BM_diagnostic_test_keep_running); int main(int argc, char* argv[]) { -#ifdef NDEBUG - // This test is exercising functionality for debug builds, which are not - // available in release builds. Skip the test if we are in that environment - // to avoid a test failure. - std::cout << "Diagnostic test disabled in release build" << std::endl; - (void)argc; - (void)argv; -#else benchmark::internal::GetAbortHandler() = &TestHandler; benchmark::Initialize(&argc, argv); benchmark::RunSpecifiedBenchmarks(); -#endif } diff --git a/third-party/benchmark/test/donotoptimize_assembly_test.cc b/third-party/benchmark/test/donotoptimize_assembly_test.cc index dc286f53e20f..2e86a51e2234 100644 --- a/third-party/benchmark/test/donotoptimize_assembly_test.cc +++ b/third-party/benchmark/test/donotoptimize_assembly_test.cc @@ -3,16 +3,12 @@ #ifdef __clang__ #pragma clang diagnostic ignored "-Wreturn-type" #endif -BENCHMARK_DISABLE_DEPRECATED_WARNING extern "C" { extern int ExternInt; extern int ExternInt2; extern int ExternInt3; -extern int BigArray[2049]; - -const int ConstBigArray[2049]{}; inline int Add42(int x) { return x + 42; } @@ -27,15 +23,7 @@ struct Large { int value; int data[2]; }; - -struct ExtraLarge { - int arr[2049]; -}; } - -extern ExtraLarge ExtraLargeObj; -const ExtraLarge ConstExtraLargeObj{}; - // CHECK-LABEL: test_with_rvalue: extern "C" void test_with_rvalue() { benchmark::DoNotOptimize(Add42(0)); @@ -80,22 +68,6 @@ extern "C" void test_with_large_lvalue() { // CHECK: ret } -// CHECK-LABEL: test_with_extra_large_lvalue_with_op: -extern "C" void test_with_extra_large_lvalue_with_op() { - ExtraLargeObj.arr[16] = 42; - benchmark::DoNotOptimize(ExtraLargeObj); - // CHECK: movl $42, ExtraLargeObj+64(%rip) - // CHECK: ret -} - -// CHECK-LABEL: test_with_big_array_with_op -extern "C" void test_with_big_array_with_op() { - BigArray[16] = 42; - benchmark::DoNotOptimize(BigArray); - // CHECK: movl $42, BigArray+64(%rip) - // CHECK: ret -} - // CHECK-LABEL: test_with_non_trivial_lvalue: extern "C" void test_with_non_trivial_lvalue() { NotTriviallyCopyable NTC(ExternInt); @@ -124,18 +96,6 @@ extern "C" void test_with_large_const_lvalue() { // CHECK: ret } -// CHECK-LABEL: test_with_const_extra_large_obj: -extern "C" void test_with_const_extra_large_obj() { - benchmark::DoNotOptimize(ConstExtraLargeObj); - // CHECK: ret -} - -// CHECK-LABEL: test_with_const_big_array -extern "C" void test_with_const_big_array() { - benchmark::DoNotOptimize(ConstBigArray); - // CHECK: ret -} - // CHECK-LABEL: test_with_non_trivial_const_lvalue: extern "C" void test_with_non_trivial_const_lvalue() { const NotTriviallyCopyable Obj(ExternInt); diff --git a/third-party/benchmark/test/donotoptimize_test.cc b/third-party/benchmark/test/donotoptimize_test.cc index 04ec9386a3b4..c321f156a1e0 100644 --- a/third-party/benchmark/test/donotoptimize_test.cc +++ b/third-party/benchmark/test/donotoptimize_test.cc @@ -4,9 +4,9 @@ namespace { #if defined(__GNUC__) -std::int64_t double_up(const std::int64_t x) __attribute__((const)); +std::uint64_t double_up(const std::uint64_t x) __attribute__((const)); #endif -std::int64_t double_up(const std::int64_t x) { return x * 2; } +std::uint64_t double_up(const std::uint64_t x) { return x * 2; } } // namespace // Using DoNotOptimize on types like BitRef seem to cause a lot of problems @@ -29,15 +29,6 @@ struct BitRef { int main(int, char*[]) { // this test verifies compilation of DoNotOptimize() for some types - char buffer1[1] = ""; - benchmark::DoNotOptimize(buffer1); - - char buffer2[2] = ""; - benchmark::DoNotOptimize(buffer2); - - char buffer3[3] = ""; - benchmark::DoNotOptimize(buffer3); - char buffer8[8] = ""; benchmark::DoNotOptimize(buffer8); @@ -46,24 +37,17 @@ int main(int, char*[]) { char buffer1024[1024] = ""; benchmark::DoNotOptimize(buffer1024); - char* bptr = &buffer1024[0]; - benchmark::DoNotOptimize(bptr); + benchmark::DoNotOptimize(&buffer1024[0]); int x = 123; benchmark::DoNotOptimize(x); - int* xp = &x; - benchmark::DoNotOptimize(xp); + benchmark::DoNotOptimize(&x); benchmark::DoNotOptimize(x += 42); - std::int64_t y = double_up(x); - benchmark::DoNotOptimize(y); + benchmark::DoNotOptimize(double_up(x)); // These tests are to e + benchmark::DoNotOptimize(BitRef::Make()); BitRef lval = BitRef::Make(); benchmark::DoNotOptimize(lval); - -#ifdef BENCHMARK_HAS_CXX11 - // Check that accept rvalue. - benchmark::DoNotOptimize(BitRef::Make()); -#endif } diff --git a/third-party/benchmark/test/filter_test.cc b/third-party/benchmark/test/filter_test.cc index 4c8b8ea488ad..a567de2dd58e 100644 --- a/third-party/benchmark/test/filter_test.cc +++ b/third-party/benchmark/test/filter_test.cc @@ -14,27 +14,28 @@ namespace { class TestReporter : public benchmark::ConsoleReporter { public: - bool ReportContext(const Context& context) override { + virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { return ConsoleReporter::ReportContext(context); }; - void ReportRuns(const std::vector& report) override { + virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { ++count_; - max_family_index_ = std::max(max_family_index_, report[0].family_index); + max_family_index_ = + std::max(max_family_index_, report[0].family_index); ConsoleReporter::ReportRuns(report); }; TestReporter() : count_(0), max_family_index_(0) {} - ~TestReporter() override {} + virtual ~TestReporter() {} - int GetCount() const { return count_; } + size_t GetCount() const { return count_; } - int64_t GetMaxFamilyIndex() const { return max_family_index_; } + size_t GetMaxFamilyIndex() const { return max_family_index_; } private: - mutable int count_; - mutable int64_t max_family_index_; + mutable size_t count_; + mutable size_t max_family_index_; }; } // end namespace @@ -78,13 +79,13 @@ int main(int argc, char** argv) { benchmark::Initialize(&argc, argv); TestReporter test_reporter; - const int64_t returned_count = - static_cast(benchmark::RunSpecifiedBenchmarks(&test_reporter)); + const size_t returned_count = + benchmark::RunSpecifiedBenchmarks(&test_reporter); if (argc == 2) { // Make sure we ran all of the tests std::stringstream ss(argv[1]); - int64_t expected_return; + size_t expected_return; ss >> expected_return; if (returned_count != expected_return) { @@ -94,8 +95,8 @@ int main(int argc, char** argv) { return -1; } - const int64_t expected_reports = list_only ? 0 : expected_return; - const int64_t reports_count = test_reporter.GetCount(); + const size_t expected_reports = list_only ? 0 : expected_return; + const size_t reports_count = test_reporter.GetCount(); if (reports_count != expected_reports) { std::cerr << "ERROR: Expected " << expected_reports << " tests to be run but reported_count = " << reports_count @@ -103,8 +104,8 @@ int main(int argc, char** argv) { return -1; } - const int64_t max_family_index = test_reporter.GetMaxFamilyIndex(); - const int64_t num_families = reports_count == 0 ? 0 : 1 + max_family_index; + const size_t max_family_index = test_reporter.GetMaxFamilyIndex(); + const size_t num_families = reports_count == 0 ? 0 : 1 + max_family_index; if (num_families != expected_reports) { std::cerr << "ERROR: Expected " << expected_reports << " test families to be run but num_families = " diff --git a/third-party/benchmark/test/fixture_test.cc b/third-party/benchmark/test/fixture_test.cc index d1093ebf52fc..af650dbd0661 100644 --- a/third-party/benchmark/test/fixture_test.cc +++ b/third-party/benchmark/test/fixture_test.cc @@ -8,21 +8,21 @@ class FIXTURE_BECHMARK_NAME : public ::benchmark::Fixture { public: - void SetUp(const ::benchmark::State& state) override { + void SetUp(const ::benchmark::State& state) BENCHMARK_OVERRIDE { if (state.thread_index() == 0) { assert(data.get() == nullptr); data.reset(new int(42)); } } - void TearDown(const ::benchmark::State& state) override { + void TearDown(const ::benchmark::State& state) BENCHMARK_OVERRIDE { if (state.thread_index() == 0) { assert(data.get() != nullptr); data.reset(); } } - ~FIXTURE_BECHMARK_NAME() override { assert(data == nullptr); } + ~FIXTURE_BECHMARK_NAME() { assert(data == nullptr); } std::unique_ptr data; }; diff --git a/third-party/benchmark/test/link_main_test.cc b/third-party/benchmark/test/link_main_test.cc index 131937eebca9..241ad5c3905e 100644 --- a/third-party/benchmark/test/link_main_test.cc +++ b/third-party/benchmark/test/link_main_test.cc @@ -2,8 +2,7 @@ void BM_empty(benchmark::State& state) { for (auto _ : state) { - auto iterations = double(state.iterations()) * double(state.iterations()); - benchmark::DoNotOptimize(iterations); + benchmark::DoNotOptimize(state.iterations()); } } BENCHMARK(BM_empty); diff --git a/third-party/benchmark/test/map_test.cc b/third-party/benchmark/test/map_test.cc index 0fdba7c87c4f..509613457c1b 100644 --- a/third-party/benchmark/test/map_test.cc +++ b/third-party/benchmark/test/map_test.cc @@ -24,8 +24,7 @@ static void BM_MapLookup(benchmark::State& state) { m = ConstructRandomMap(size); state.ResumeTiming(); for (int i = 0; i < size; ++i) { - auto it = m.find(std::rand() % size); - benchmark::DoNotOptimize(it); + benchmark::DoNotOptimize(m.find(std::rand() % size)); } } state.SetItemsProcessed(state.iterations() * size); @@ -35,11 +34,11 @@ BENCHMARK(BM_MapLookup)->Range(1 << 3, 1 << 12); // Using fixtures. class MapFixture : public ::benchmark::Fixture { public: - void SetUp(const ::benchmark::State& st) override { + void SetUp(const ::benchmark::State& st) BENCHMARK_OVERRIDE { m = ConstructRandomMap(static_cast(st.range(0))); } - void TearDown(const ::benchmark::State&) override { m.clear(); } + void TearDown(const ::benchmark::State&) BENCHMARK_OVERRIDE { m.clear(); } std::map m; }; @@ -48,8 +47,7 @@ BENCHMARK_DEFINE_F(MapFixture, Lookup)(benchmark::State& state) { const int size = static_cast(state.range(0)); for (auto _ : state) { for (int i = 0; i < size; ++i) { - auto it = m.find(std::rand() % size); - benchmark::DoNotOptimize(it); + benchmark::DoNotOptimize(m.find(std::rand() % size)); } } state.SetItemsProcessed(state.iterations() * size); diff --git a/third-party/benchmark/test/memory_manager_test.cc b/third-party/benchmark/test/memory_manager_test.cc index 4df674d586ed..f0c192fcbd00 100644 --- a/third-party/benchmark/test/memory_manager_test.cc +++ b/third-party/benchmark/test/memory_manager_test.cc @@ -5,17 +5,16 @@ #include "output_test.h" class TestMemoryManager : public benchmark::MemoryManager { - void Start() override {} - void Stop(Result& result) override { - result.num_allocs = 42; - result.max_bytes_used = 42000; + void Start() BENCHMARK_OVERRIDE {} + void Stop(Result* result) BENCHMARK_OVERRIDE { + result->num_allocs = 42; + result->max_bytes_used = 42000; } }; void BM_empty(benchmark::State& state) { for (auto _ : state) { - auto iterations = double(state.iterations()) * double(state.iterations()); - benchmark::DoNotOptimize(iterations); + benchmark::DoNotOptimize(state.iterations()); } } BENCHMARK(BM_empty); diff --git a/third-party/benchmark/test/multiple_ranges_test.cc b/third-party/benchmark/test/multiple_ranges_test.cc index 5300a96036c1..7618c4da0892 100644 --- a/third-party/benchmark/test/multiple_ranges_test.cc +++ b/third-party/benchmark/test/multiple_ranges_test.cc @@ -28,7 +28,7 @@ class MultipleRangesFixture : public ::benchmark::Fixture { {2, 7, 15}, {7, 6, 3}}) {} - void SetUp(const ::benchmark::State& state) override { + void SetUp(const ::benchmark::State& state) BENCHMARK_OVERRIDE { std::vector ranges = {state.range(0), state.range(1), state.range(2)}; @@ -39,7 +39,7 @@ class MultipleRangesFixture : public ::benchmark::Fixture { // NOTE: This is not TearDown as we want to check after _all_ runs are // complete. - ~MultipleRangesFixture() override { + virtual ~MultipleRangesFixture() { if (actualValues != expectedValues) { std::cout << "EXPECTED\n"; for (const auto& v : expectedValues) { diff --git a/third-party/benchmark/test/options_test.cc b/third-party/benchmark/test/options_test.cc index a1b209f3eb33..d424d40b9518 100644 --- a/third-party/benchmark/test/options_test.cc +++ b/third-party/benchmark/test/options_test.cc @@ -33,8 +33,6 @@ BENCHMARK(BM_basic)->DenseRange(10, 15); BENCHMARK(BM_basic)->Args({42, 42}); BENCHMARK(BM_basic)->Ranges({{64, 512}, {64, 512}}); BENCHMARK(BM_basic)->MinTime(0.7); -BENCHMARK(BM_basic)->MinWarmUpTime(0.8); -BENCHMARK(BM_basic)->MinTime(0.1)->MinWarmUpTime(0.2); BENCHMARK(BM_basic)->UseRealTime(); BENCHMARK(BM_basic)->ThreadRange(2, 4); BENCHMARK(BM_basic)->ThreadPerCpu(); @@ -67,8 +65,8 @@ void BM_explicit_iteration_count(benchmark::State& state) { // Test that the requested iteration count is respected. assert(state.max_iterations == 42); - for (auto _ : state) { - } + size_t actual_iterations = 0; + for (auto _ : state) ++actual_iterations; assert(state.iterations() == state.max_iterations); assert(state.iterations() == 42); } diff --git a/third-party/benchmark/test/output_test.h b/third-party/benchmark/test/output_test.h index c08fe1d87e6c..c6ff8ef2d303 100644 --- a/third-party/benchmark/test/output_test.h +++ b/third-party/benchmark/test/output_test.h @@ -85,7 +85,7 @@ std::string GetFileReporterOutput(int argc, char* argv[]); struct Results; typedef std::function ResultsCheckFn; -size_t AddChecker(const std::string& bm_name_pattern, const ResultsCheckFn& fn); +size_t AddChecker(const char* bm_name_pattern, const ResultsCheckFn& fn); // Class holding the results of a benchmark. // It is passed in calls to checker functions. @@ -117,7 +117,7 @@ struct Results { // get the string for a result by name, or nullptr if the name // is not found - const std::string* Get(const std::string& entry_name) const { + const std::string* Get(const char* entry_name) const { auto it = values.find(entry_name); if (it == values.end()) return nullptr; return &it->second; @@ -126,12 +126,12 @@ struct Results { // get a result by name, parsed as a specific type. // NOTE: for counters, use GetCounterAs instead. template - T GetAs(const std::string& entry_name) const; + T GetAs(const char* entry_name) const; // counters are written as doubles, so they have to be read first // as a double, and only then converted to the asked type. template - T GetCounterAs(const std::string& entry_name) const { + T GetCounterAs(const char* entry_name) const { double dval = GetAs(entry_name); T tval = static_cast(dval); return tval; @@ -139,7 +139,7 @@ struct Results { }; template -T Results::GetAs(const std::string& entry_name) const { +T Results::GetAs(const char* entry_name) const { auto* sv = Get(entry_name); BM_CHECK(sv != nullptr && !sv->empty()); std::stringstream ss; diff --git a/third-party/benchmark/test/output_test_helper.cc b/third-party/benchmark/test/output_test_helper.cc index 265f28aae7c7..81584cbf778b 100644 --- a/third-party/benchmark/test/output_test_helper.cc +++ b/third-party/benchmark/test/output_test_helper.cc @@ -45,7 +45,7 @@ SubMap& GetSubstitutions() { static SubMap map = { {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"}, // human-readable float - {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kKMGTPEZYmunpfazy]?i?"}, + {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"}, {"%percentage", percentage_re}, {"%int", "[ ]*[0-9]+"}, {" %s ", "[ ]+"}, @@ -65,7 +65,6 @@ SubMap& GetSubstitutions() { {"%csv_us_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",us,,,,,"}, {"%csv_ms_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ms,,,,,"}, {"%csv_s_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",s,,,,,"}, - {"%csv_cv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",,,,,,"}, {"%csv_bytes_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + ",,,,"}, {"%csv_items_report", @@ -144,7 +143,7 @@ class TestReporter : public benchmark::BenchmarkReporter { TestReporter(std::vector reps) : reporters_(std::move(reps)) {} - bool ReportContext(const Context& context) override { + virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { bool last_ret = false; bool first = true; for (auto rep : reporters_) { @@ -158,10 +157,10 @@ class TestReporter : public benchmark::BenchmarkReporter { return last_ret; } - void ReportRuns(const std::vector& report) override { + void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { for (auto rep : reporters_) rep->ReportRuns(report); } - void Finalize() override { + void Finalize() BENCHMARK_OVERRIDE { for (auto rep : reporters_) rep->Finalize(); } @@ -249,8 +248,9 @@ void ResultsChecker::CheckResults(std::stringstream& output) { if (!p.regex->Match(r.name)) { BM_VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n"; continue; + } else { + BM_VLOG(2) << p.regex_str << " is matched by " << r.name << "\n"; } - BM_VLOG(2) << p.regex_str << " is matched by " << r.name << "\n"; BM_VLOG(1) << "Checking results of " << r.name << ": ... \n"; p.fn(r); BM_VLOG(1) << "Checking results of " << r.name << ": OK.\n"; @@ -300,7 +300,7 @@ std::vector ResultsChecker::SplitCsv_(const std::string& line) { } // end namespace internal -size_t AddChecker(const std::string& bm_name, const ResultsCheckFn& fn) { +size_t AddChecker(const char* bm_name, const ResultsCheckFn& fn) { auto& rc = internal::GetResultsChecker(); rc.Add(bm_name, fn); return rc.results.size(); @@ -328,18 +328,16 @@ double Results::GetTime(BenchmarkTime which) const { BM_CHECK(unit); if (*unit == "ns") { return val * 1.e-9; - } - if (*unit == "us") { + } else if (*unit == "us") { return val * 1.e-6; - } - if (*unit == "ms") { + } else if (*unit == "ms") { return val * 1.e-3; - } - if (*unit == "s") { + } else if (*unit == "s") { return val; + } else { + BM_CHECK(1 == 0) << "unknown time unit: " << *unit; + return 0; } - BM_CHECK(1 == 0) << "unknown time unit: " << *unit; - return 0; } // ========================================================================= // @@ -395,14 +393,14 @@ void RunOutputTests(int argc, char* argv[]) { benchmark::JSONReporter JR; benchmark::CSVReporter CSVR; struct ReporterTest { - std::string name; + const char* name; std::vector& output_cases; std::vector& error_cases; benchmark::BenchmarkReporter& reporter; std::stringstream out_stream; std::stringstream err_stream; - ReporterTest(const std::string& n, std::vector& out_tc, + ReporterTest(const char* n, std::vector& out_tc, std::vector& err_tc, benchmark::BenchmarkReporter& br) : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) { @@ -410,12 +408,12 @@ void RunOutputTests(int argc, char* argv[]) { reporter.SetErrorStream(&err_stream); } } TestCases[] = { - {std::string("ConsoleReporter"), GetTestCaseList(TC_ConsoleOut), + {"ConsoleReporter", GetTestCaseList(TC_ConsoleOut), GetTestCaseList(TC_ConsoleErr), CR}, - {std::string("JSONReporter"), GetTestCaseList(TC_JSONOut), - GetTestCaseList(TC_JSONErr), JR}, - {std::string("CSVReporter"), GetTestCaseList(TC_CSVOut), - GetTestCaseList(TC_CSVErr), CSVR}, + {"JSONReporter", GetTestCaseList(TC_JSONOut), GetTestCaseList(TC_JSONErr), + JR}, + {"CSVReporter", GetTestCaseList(TC_CSVOut), GetTestCaseList(TC_CSVErr), + CSVR}, }; // Create the test reporter and run the benchmarks. @@ -424,8 +422,7 @@ void RunOutputTests(int argc, char* argv[]) { benchmark::RunSpecifiedBenchmarks(&test_rep); for (auto& rep_test : TestCases) { - std::string msg = - std::string("\nTesting ") + rep_test.name + std::string(" Output\n"); + std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n"; std::string banner(msg.size() - 1, '-'); std::cout << banner << msg << banner << "\n"; @@ -442,7 +439,7 @@ void RunOutputTests(int argc, char* argv[]) { // the checks to subscribees. auto& csv = TestCases[2]; // would use == but gcc spits a warning - BM_CHECK(csv.name == std::string("CSVReporter")); + BM_CHECK(std::strcmp(csv.name, "CSVReporter") == 0); internal::GetResultsChecker().CheckResults(csv.out_stream); } diff --git a/third-party/benchmark/test/perf_counters_gtest.cc b/third-party/benchmark/test/perf_counters_gtest.cc index 2e63049285d7..3eac62463bc5 100644 --- a/third-party/benchmark/test/perf_counters_gtest.cc +++ b/third-party/benchmark/test/perf_counters_gtest.cc @@ -1,8 +1,6 @@ -#include #include #include "../src/perf_counters.h" -#include "gmock/gmock.h" #include "gtest/gtest.h" #ifndef GTEST_SKIP @@ -13,15 +11,12 @@ struct MsgHandler { #endif using benchmark::internal::PerfCounters; -using benchmark::internal::PerfCountersMeasurement; using benchmark::internal::PerfCounterValues; -using ::testing::AllOf; -using ::testing::Gt; -using ::testing::Lt; namespace { const char kGenericPerfEvent1[] = "CYCLES"; -const char kGenericPerfEvent2[] = "INSTRUCTIONS"; +const char kGenericPerfEvent2[] = "BRANCHES"; +const char kGenericPerfEvent3[] = "INSTRUCTIONS"; TEST(PerfCountersTest, Init) { EXPECT_EQ(PerfCounters::Initialize(), PerfCounters::kSupported); @@ -32,7 +27,7 @@ TEST(PerfCountersTest, OneCounter) { GTEST_SKIP() << "Performance counters not supported.\n"; } EXPECT_TRUE(PerfCounters::Initialize()); - EXPECT_EQ(PerfCounters::Create({kGenericPerfEvent1}).num_counters(), 1); + EXPECT_TRUE(PerfCounters::Create({kGenericPerfEvent1}).IsValid()); } TEST(PerfCountersTest, NegativeTest) { @@ -41,44 +36,29 @@ TEST(PerfCountersTest, NegativeTest) { return; } EXPECT_TRUE(PerfCounters::Initialize()); - // Safety checks - // Create() will always create a valid object, even if passed no or - // wrong arguments as the new behavior is to warn and drop unsupported - // counters - EXPECT_EQ(PerfCounters::Create({}).num_counters(), 0); - EXPECT_EQ(PerfCounters::Create({""}).num_counters(), 0); - EXPECT_EQ(PerfCounters::Create({"not a counter name"}).num_counters(), 0); + EXPECT_FALSE(PerfCounters::Create({}).IsValid()); + EXPECT_FALSE(PerfCounters::Create({""}).IsValid()); + EXPECT_FALSE(PerfCounters::Create({"not a counter name"}).IsValid()); { - // Try sneaking in a bad egg to see if it is filtered out. The - // number of counters has to be two, not zero - auto counter = - PerfCounters::Create({kGenericPerfEvent2, "", kGenericPerfEvent1}); - EXPECT_EQ(counter.num_counters(), 2); - EXPECT_EQ(counter.names(), std::vector( - {kGenericPerfEvent2, kGenericPerfEvent1})); - } - { - // Try sneaking in an outrageous counter, like a fat finger mistake - auto counter = PerfCounters::Create( - {kGenericPerfEvent2, "not a counter name", kGenericPerfEvent1}); - EXPECT_EQ(counter.num_counters(), 2); - EXPECT_EQ(counter.names(), std::vector( - {kGenericPerfEvent2, kGenericPerfEvent1})); - } + EXPECT_TRUE(PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2, + kGenericPerfEvent3}) + .IsValid()); + } + EXPECT_FALSE( + PerfCounters::Create({kGenericPerfEvent2, "", kGenericPerfEvent1}) + .IsValid()); + EXPECT_FALSE(PerfCounters::Create({kGenericPerfEvent3, "not a counter name", + kGenericPerfEvent1}) + .IsValid()); { - // Finally try a golden input - it should like both of them - EXPECT_EQ(PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2}) - .num_counters(), - 2); - } - { - // Add a bad apple in the end of the chain to check the edges - auto counter = PerfCounters::Create( - {kGenericPerfEvent1, kGenericPerfEvent2, "bad event name"}); - EXPECT_EQ(counter.num_counters(), 2); - EXPECT_EQ(counter.names(), std::vector( - {kGenericPerfEvent1, kGenericPerfEvent2})); - } + EXPECT_TRUE(PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2, + kGenericPerfEvent3}) + .IsValid()); + } + EXPECT_FALSE( + PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2, + kGenericPerfEvent3, "MISPREDICTED_BRANCH_RETIRED"}) + .IsValid()); } TEST(PerfCountersTest, Read1Counter) { @@ -87,7 +67,7 @@ TEST(PerfCountersTest, Read1Counter) { } EXPECT_TRUE(PerfCounters::Initialize()); auto counters = PerfCounters::Create({kGenericPerfEvent1}); - EXPECT_EQ(counters.num_counters(), 1); + EXPECT_TRUE(counters.IsValid()); PerfCounterValues values1(1); EXPECT_TRUE(counters.Snapshot(&values1)); EXPECT_GT(values1[0], 0); @@ -104,7 +84,7 @@ TEST(PerfCountersTest, Read2Counters) { EXPECT_TRUE(PerfCounters::Initialize()); auto counters = PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2}); - EXPECT_EQ(counters.num_counters(), 2); + EXPECT_TRUE(counters.IsValid()); PerfCounterValues values1(2); EXPECT_TRUE(counters.Snapshot(&values1)); EXPECT_GT(values1[0], 0); @@ -115,121 +95,30 @@ TEST(PerfCountersTest, Read2Counters) { EXPECT_GT(values2[1], 0); } -TEST(PerfCountersTest, ReopenExistingCounters) { - // This test works in recent and old Intel hardware, Pixel 3, and Pixel 6. - // However we cannot make assumptions beyond 2 HW counters due to Pixel 6. - if (!PerfCounters::kSupported) { - GTEST_SKIP() << "Test skipped because libpfm is not supported.\n"; - } - EXPECT_TRUE(PerfCounters::Initialize()); - std::vector kMetrics({kGenericPerfEvent1}); - std::vector counters(2); - for (auto& counter : counters) { - counter = PerfCounters::Create(kMetrics); - } - PerfCounterValues values(1); - EXPECT_TRUE(counters[0].Snapshot(&values)); - EXPECT_TRUE(counters[1].Snapshot(&values)); +size_t do_work() { + size_t res = 0; + for (size_t i = 0; i < 100000000; ++i) res += i * i; + return res; } -TEST(PerfCountersTest, CreateExistingMeasurements) { - // The test works (i.e. causes read to fail) for the assumptions - // about hardware capabilities (i.e. small number (2) hardware - // counters) at this date, - // the same as previous test ReopenExistingCounters. - if (!PerfCounters::kSupported) { - GTEST_SKIP() << "Test skipped because libpfm is not supported.\n"; - } - EXPECT_TRUE(PerfCounters::Initialize()); - - // This means we will try 10 counters but we can only guarantee - // for sure at this time that only 3 will work. Perhaps in the future - // we could use libpfm to query for the hardware limits on this - // particular platform. - const int kMaxCounters = 10; - const int kMinValidCounters = 2; - - // Let's use a ubiquitous counter that is guaranteed to work - // on all platforms - const std::vector kMetrics{"cycles"}; - - // Cannot create a vector of actual objects because the - // copy constructor of PerfCounters is deleted - and so is - // implicitly deleted on PerfCountersMeasurement too - std::vector> - perf_counter_measurements; - - perf_counter_measurements.reserve(kMaxCounters); - for (int j = 0; j < kMaxCounters; ++j) { - perf_counter_measurements.emplace_back( - new PerfCountersMeasurement(kMetrics)); - } - - std::vector> measurements; - - // Start all counters together to see if they hold - size_t max_counters = kMaxCounters; - for (size_t i = 0; i < kMaxCounters; ++i) { - auto& counter(*perf_counter_measurements[i]); - EXPECT_EQ(counter.num_counters(), 1); - if (!counter.Start()) { - max_counters = i; - break; - }; - } - - ASSERT_GE(max_counters, kMinValidCounters); - - // Start all together - for (size_t i = 0; i < max_counters; ++i) { - auto& counter(*perf_counter_measurements[i]); - EXPECT_TRUE(counter.Stop(measurements) || (i >= kMinValidCounters)); - } - - // Start/stop individually - for (size_t i = 0; i < max_counters; ++i) { - auto& counter(*perf_counter_measurements[i]); - measurements.clear(); - counter.Start(); - EXPECT_TRUE(counter.Stop(measurements) || (i >= kMinValidCounters)); - } -} - -// We try to do some meaningful work here but the compiler -// insists in optimizing away our loop so we had to add a -// no-optimize macro. In case it fails, we added some entropy -// to this pool as well. - -BENCHMARK_DONT_OPTIMIZE size_t do_work() { - static std::mt19937 rd{std::random_device{}()}; - static std::uniform_int_distribution mrand(0, 10); - const size_t kNumLoops = 1000000; - size_t sum = 0; - for (size_t j = 0; j < kNumLoops; ++j) { - sum += mrand(rd); - } - benchmark::DoNotOptimize(sum); - return sum; -} - -void measure(size_t threadcount, PerfCounterValues* before, - PerfCounterValues* after) { - BM_CHECK_NE(before, nullptr); - BM_CHECK_NE(after, nullptr); +void measure(size_t threadcount, PerfCounterValues* values1, + PerfCounterValues* values2) { + BM_CHECK_NE(values1, nullptr); + BM_CHECK_NE(values2, nullptr); std::vector threads(threadcount); auto work = [&]() { BM_CHECK(do_work() > 1000); }; // We need to first set up the counters, then start the threads, so the - // threads would inherit the counters. But later, we need to first destroy - // the thread pool (so all the work finishes), then measure the counters. So - // the scopes overlap, and we need to explicitly control the scope of the + // threads would inherit the counters. But later, we need to first destroy the + // thread pool (so all the work finishes), then measure the counters. So the + // scopes overlap, and we need to explicitly control the scope of the // threadpool. auto counters = - PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2}); + PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent3}); for (auto& t : threads) t = std::thread(work); - counters.Snapshot(before); + counters.Snapshot(values1); for (auto& t : threads) t.join(); - counters.Snapshot(after); + counters.Snapshot(values2); } TEST(PerfCountersTest, MultiThreaded) { @@ -237,71 +126,20 @@ TEST(PerfCountersTest, MultiThreaded) { GTEST_SKIP() << "Test skipped because libpfm is not supported."; } EXPECT_TRUE(PerfCounters::Initialize()); - PerfCounterValues before(2); - PerfCounterValues after(2); - - // Notice that this test will work even if we taskset it to a single CPU - // In this case the threads will run sequentially - // Start two threads and measure the number of combined cycles and - // instructions - measure(2, &before, &after); - std::vector Elapsed2Threads{ - static_cast(after[0] - before[0]), - static_cast(after[1] - before[1])}; - - // Start four threads and measure the number of combined cycles and - // instructions - measure(4, &before, &after); - std::vector Elapsed4Threads{ - static_cast(after[0] - before[0]), - static_cast(after[1] - before[1])}; - - // The following expectations fail (at least on a beefy workstation with lots - // of cpus) - it seems that in some circumstances the runtime of 4 threads - // can even be better than with 2. - // So instead of expecting 4 threads to be slower, let's just make sure they - // do not differ too much in general (one is not more than 10x than the - // other). - EXPECT_THAT(Elapsed4Threads[0] / Elapsed2Threads[0], AllOf(Gt(0.1), Lt(10))); - EXPECT_THAT(Elapsed4Threads[1] / Elapsed2Threads[1], AllOf(Gt(0.1), Lt(10))); -} - -TEST(PerfCountersTest, HardwareLimits) { - // The test works (i.e. causes read to fail) for the assumptions - // about hardware capabilities (i.e. small number (3-4) hardware - // counters) at this date, - // the same as previous test ReopenExistingCounters. - if (!PerfCounters::kSupported) { - GTEST_SKIP() << "Test skipped because libpfm is not supported.\n"; - } - EXPECT_TRUE(PerfCounters::Initialize()); - - // Taken from `perf list`, but focusses only on those HW events that actually - // were reported when running `sudo perf stat -a sleep 10`, intersected over - // several platforms. All HW events listed in the first command not reported - // in the second seem to not work. This is sad as we don't really get to test - // the grouping here (groups can contain up to 6 members)... - std::vector counter_names{ - "cycles", // leader - "instructions", // - "branch-misses", // - }; + PerfCounterValues values1(2); + PerfCounterValues values2(2); - // In the off-chance that some of these values are not supported, - // we filter them out so the test will complete without failure - // albeit it might not actually test the grouping on that platform - std::vector valid_names; - for (const std::string& name : counter_names) { - if (PerfCounters::IsCounterSupported(name)) { - valid_names.push_back(name); - } - } - PerfCountersMeasurement counter(valid_names); + measure(2, &values1, &values2); + std::vector D1{static_cast(values2[0] - values1[0]), + static_cast(values2[1] - values1[1])}; - std::vector> measurements; + measure(4, &values1, &values2); + std::vector D2{static_cast(values2[0] - values1[0]), + static_cast(values2[1] - values1[1])}; - counter.Start(); - EXPECT_TRUE(counter.Stop(measurements)); + // Some extra work will happen on the main thread - like joining the threads + // - so the ratio won't be quite 2.0, but very close. + EXPECT_GE(D2[0], 1.9 * D1[0]); + EXPECT_GE(D2[1], 1.9 * D1[1]); } - } // namespace diff --git a/third-party/benchmark/test/perf_counters_test.cc b/third-party/benchmark/test/perf_counters_test.cc index 3cc593e629d8..3017a452fe27 100644 --- a/third-party/benchmark/test/perf_counters_test.cc +++ b/third-party/benchmark/test/perf_counters_test.cc @@ -1,92 +1,27 @@ -#include #undef NDEBUG -#include "../src/commandlineflags.h" #include "../src/perf_counters.h" + #include "benchmark/benchmark.h" #include "output_test.h" -namespace benchmark { - -BM_DECLARE_string(benchmark_perf_counters); - -} // namespace benchmark - static void BM_Simple(benchmark::State& state) { for (auto _ : state) { - auto iterations = double(state.iterations()) * double(state.iterations()); - benchmark::DoNotOptimize(iterations); + benchmark::DoNotOptimize(state.iterations()); } } BENCHMARK(BM_Simple); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Simple\",$"}}); -const int kIters = 1000000; - -void BM_WithoutPauseResume(benchmark::State& state) { - int n = 0; - - for (auto _ : state) { - for (auto i = 0; i < kIters; ++i) { - n = 1 - n; - benchmark::DoNotOptimize(n); - } - } -} - -BENCHMARK(BM_WithoutPauseResume); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_WithoutPauseResume\",$"}}); - -void BM_WithPauseResume(benchmark::State& state) { - int m = 0, n = 0; - - for (auto _ : state) { - for (auto i = 0; i < kIters; ++i) { - n = 1 - n; - benchmark::DoNotOptimize(n); - } - - state.PauseTiming(); - for (auto j = 0; j < kIters; ++j) { - m = 1 - m; - benchmark::DoNotOptimize(m); - } - state.ResumeTiming(); - } -} - -BENCHMARK(BM_WithPauseResume); - -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_WithPauseResume\",$"}}); - static void CheckSimple(Results const& e) { CHECK_COUNTER_VALUE(e, double, "CYCLES", GT, 0); + CHECK_COUNTER_VALUE(e, double, "BRANCHES", GT, 0.0); } - -double withoutPauseResumeInstrCount = 0.0; -double withPauseResumeInstrCount = 0.0; - -static void SaveInstrCountWithoutResume(Results const& e) { - withoutPauseResumeInstrCount = e.GetAs("INSTRUCTIONS"); -} - -static void SaveInstrCountWithResume(Results const& e) { - withPauseResumeInstrCount = e.GetAs("INSTRUCTIONS"); -} - CHECK_BENCHMARK_RESULTS("BM_Simple", &CheckSimple); -CHECK_BENCHMARK_RESULTS("BM_WithoutPauseResume", &SaveInstrCountWithoutResume); -CHECK_BENCHMARK_RESULTS("BM_WithPauseResume", &SaveInstrCountWithResume); int main(int argc, char* argv[]) { if (!benchmark::internal::PerfCounters::kSupported) { return 0; } - benchmark::FLAGS_benchmark_perf_counters = "CYCLES,INSTRUCTIONS"; - benchmark::internal::PerfCounters::Initialize(); RunOutputTests(argc, argv); - - BM_CHECK_GT(withPauseResumeInstrCount, kIters); - BM_CHECK_GT(withoutPauseResumeInstrCount, kIters); - BM_CHECK_LT(withPauseResumeInstrCount, 1.5 * withoutPauseResumeInstrCount); } diff --git a/third-party/benchmark/test/register_benchmark_test.cc b/third-party/benchmark/test/register_benchmark_test.cc index d69d144a4e17..602405b67e8d 100644 --- a/third-party/benchmark/test/register_benchmark_test.cc +++ b/third-party/benchmark/test/register_benchmark_test.cc @@ -10,7 +10,7 @@ namespace { class TestReporter : public benchmark::ConsoleReporter { public: - void ReportRuns(const std::vector& report) override { + virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { all_runs_.insert(all_runs_.end(), begin(report), end(report)); ConsoleReporter::ReportRuns(report); } @@ -19,11 +19,11 @@ class TestReporter : public benchmark::ConsoleReporter { }; struct TestCase { - const std::string name; - const std::string label; + std::string name; + const char* label; // Note: not explicit as we rely on it being converted through ADD_CASES. - TestCase(const std::string& xname) : TestCase(xname, "") {} - TestCase(const std::string& xname, const std::string& xlabel) + TestCase(const char* xname) : TestCase(xname, nullptr) {} + TestCase(const char* xname, const char* xlabel) : name(xname), label(xlabel) {} typedef benchmark::BenchmarkReporter::Run Run; @@ -32,7 +32,7 @@ struct TestCase { // clang-format off BM_CHECK(name == run.benchmark_name()) << "expected " << name << " got " << run.benchmark_name(); - if (!label.empty()) { + if (label) { BM_CHECK(run.report_label == label) << "expected " << label << " got " << run.report_label; } else { @@ -95,18 +95,6 @@ ADD_CASES({"test1", "One"}, {"test2", "Two"}, {"test3", "Three"}); #endif // BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK -//----------------------------------------------------------------------------// -// Test RegisterBenchmark with DISABLED_ benchmark -//----------------------------------------------------------------------------// -void DISABLED_BM_function(benchmark::State& state) { - for (auto _ : state) { - } -} -BENCHMARK(DISABLED_BM_function); -ReturnVal dummy3 = benchmark::RegisterBenchmark("DISABLED_BM_function_manual", - DISABLED_BM_function); -// No need to add cases because we don't expect them to run. - //----------------------------------------------------------------------------// // Test RegisterBenchmark with different callable types //----------------------------------------------------------------------------// @@ -123,7 +111,7 @@ void TestRegistrationAtRuntime() { { CustomFixture fx; benchmark::RegisterBenchmark("custom_fixture", fx); - AddCases({std::string("custom_fixture")}); + AddCases({"custom_fixture"}); } #endif #ifndef BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK diff --git a/third-party/benchmark/test/reporter_output_test.cc b/third-party/benchmark/test/reporter_output_test.cc index 7867165d1f3d..2b6e6543dd2e 100644 --- a/third-party/benchmark/test/reporter_output_test.cc +++ b/third-party/benchmark/test/reporter_output_test.cc @@ -17,7 +17,7 @@ static int AddContextCases() { AddCases(TC_ConsoleErr, { {"^%int-%int-%intT%int:%int:%int[-+]%int:%int$", MR_Default}, - {"Running .*(/|\\\\)reporter_output_test(\\.exe)?$", MR_Next}, + {"Running .*/reporter_output_test(\\.exe)?$", MR_Next}, {"Run on \\(%int X %float MHz CPU s?\\)", MR_Next}, }); AddCases(TC_JSONOut, @@ -55,9 +55,6 @@ static int AddContextCases() { {{"Load Average: (%float, ){0,2}%float$", MR_Next}}); } AddCases(TC_JSONOut, {{"\"load_avg\": \\[(%float,?){0,3}],$", MR_Next}}); - AddCases(TC_JSONOut, {{"\"library_version\": \".*\",$", MR_Next}}); - AddCases(TC_JSONOut, {{"\"library_build_type\": \".*\",$", MR_Next}}); - AddCases(TC_JSONOut, {{"\"json_schema_version\": 1$", MR_Next}}); return 0; } int dummy_register = AddContextCases(); @@ -96,8 +93,7 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}}); void BM_bytes_per_second(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - auto iterations = double(state.iterations()) * double(state.iterations()); - benchmark::DoNotOptimize(iterations); + benchmark::DoNotOptimize(state.iterations()); } state.SetBytesProcessed(1); } @@ -128,8 +124,7 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}}); void BM_items_per_second(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - auto iterations = double(state.iterations()) * double(state.iterations()); - benchmark::DoNotOptimize(iterations); + benchmark::DoNotOptimize(state.iterations()); } state.SetItemsProcessed(1); } @@ -323,7 +318,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}, ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}}); // ========================================================================= // -// ------------------------ Testing Arg Name Output ------------------------ // +// ------------------------ Testing Arg Name Output ----------------------- // // ========================================================================= // void BM_arg_name(benchmark::State& state) { @@ -409,8 +404,7 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_BigArgs/1073741824 %console_report$"}, void BM_Complexity_O1(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - auto iterations = double(state.iterations()) * double(state.iterations()); - benchmark::DoNotOptimize(iterations); + benchmark::DoNotOptimize(state.iterations()); } state.SetComplexityN(state.range(0)); } @@ -1091,7 +1085,7 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_UserPercentStats/iterations:5/repeats:3/" {"^\"BM_UserPercentStats/iterations:5/repeats:3/" "manual_time_stddev\",%csv_report$"}, {"^\"BM_UserPercentStats/iterations:5/repeats:3/" - "manual_time_\",%csv_cv_report$"}}); + "manual_time_\",%csv_report$"}}); // ========================================================================= // // ------------------------- Testing StrEscape JSON ------------------------ // diff --git a/third-party/benchmark/test/skip_with_error_test.cc b/third-party/benchmark/test/skip_with_error_test.cc index 2139a19e2507..026d47913350 100644 --- a/third-party/benchmark/test/skip_with_error_test.cc +++ b/third-party/benchmark/test/skip_with_error_test.cc @@ -10,17 +10,17 @@ namespace { class TestReporter : public benchmark::ConsoleReporter { public: - bool ReportContext(const Context& context) override { + virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { return ConsoleReporter::ReportContext(context); }; - void ReportRuns(const std::vector& report) override { + virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { all_runs_.insert(all_runs_.end(), begin(report), end(report)); ConsoleReporter::ReportRuns(report); } TestReporter() {} - ~TestReporter() override {} + virtual ~TestReporter() {} mutable std::vector all_runs_; }; @@ -35,9 +35,8 @@ struct TestCase { void CheckRun(Run const& run) const { BM_CHECK(name == run.benchmark_name()) << "expected " << name << " got " << run.benchmark_name(); - BM_CHECK_EQ(error_occurred, - benchmark::internal::SkippedWithError == run.skipped); - BM_CHECK(error_message == run.skip_message); + BM_CHECK(error_occurred == run.error_occurred); + BM_CHECK(error_message == run.error_message); if (error_occurred) { // BM_CHECK(run.iterations == 0); } else { @@ -48,8 +47,7 @@ struct TestCase { std::vector ExpectedResults; -int AddCases(const std::string& base_name, - std::initializer_list const& v) { +int AddCases(const char* base_name, std::initializer_list const& v) { for (auto TC : v) { TC.name = base_name + TC.name; ExpectedResults.push_back(std::move(TC)); @@ -143,8 +141,7 @@ ADD_CASES("BM_error_during_running_ranged_for", void BM_error_after_running(benchmark::State& state) { for (auto _ : state) { - auto iterations = double(state.iterations()) * double(state.iterations()); - benchmark::DoNotOptimize(iterations); + benchmark::DoNotOptimize(state.iterations()); } if (state.thread_index() <= (state.threads() / 2)) state.SkipWithError("error message"); diff --git a/third-party/benchmark/test/spec_arg_test.cc b/third-party/benchmark/test/spec_arg_test.cc index 06aafbeb9b5e..043db1be47a2 100644 --- a/third-party/benchmark/test/spec_arg_test.cc +++ b/third-party/benchmark/test/spec_arg_test.cc @@ -17,11 +17,11 @@ namespace { class TestReporter : public benchmark::ConsoleReporter { public: - bool ReportContext(const Context& context) override { + virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { return ConsoleReporter::ReportContext(context); }; - void ReportRuns(const std::vector& report) override { + virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { assert(report.size() == 1); matched_functions.push_back(report[0].run_name.function_name); ConsoleReporter::ReportRuns(report); @@ -29,7 +29,7 @@ class TestReporter : public benchmark::ConsoleReporter { TestReporter() {} - ~TestReporter() override {} + virtual ~TestReporter() {} const std::vector& GetMatchedFunctions() const { return matched_functions; @@ -91,15 +91,5 @@ int main(int argc, char** argv) { << matched_functions.front() << "]\n"; return 2; } - - // Test that SetBenchmarkFilter works. - const std::string golden_value = "golden_value"; - benchmark::SetBenchmarkFilter(golden_value); - std::string current_value = benchmark::GetBenchmarkFilter(); - if (golden_value != current_value) { - std::cerr << "Expected [" << golden_value - << "] for --benchmark_filter but got [" << current_value << "]\n"; - return 3; - } return 0; } diff --git a/third-party/benchmark/test/statistics_gtest.cc b/third-party/benchmark/test/statistics_gtest.cc index 48c77260fd53..1de2d87d4ba5 100644 --- a/third-party/benchmark/test/statistics_gtest.cc +++ b/third-party/benchmark/test/statistics_gtest.cc @@ -28,8 +28,8 @@ TEST(StatisticsTest, StdDev) { TEST(StatisticsTest, CV) { EXPECT_DOUBLE_EQ(benchmark::StatisticsCV({101, 101, 101, 101}), 0.0); EXPECT_DOUBLE_EQ(benchmark::StatisticsCV({1, 2, 3}), 1. / 2.); - ASSERT_NEAR(benchmark::StatisticsCV({2.5, 2.4, 3.3, 4.2, 5.1}), - 0.32888184094918121, 1e-15); + EXPECT_DOUBLE_EQ(benchmark::StatisticsCV({2.5, 2.4, 3.3, 4.2, 5.1}), + 0.32888184094918121); } } // end namespace diff --git a/third-party/benchmark/test/string_util_gtest.cc b/third-party/benchmark/test/string_util_gtest.cc index 67b4bc0c24f2..698f2d43eb88 100644 --- a/third-party/benchmark/test/string_util_gtest.cc +++ b/third-party/benchmark/test/string_util_gtest.cc @@ -1,12 +1,9 @@ //===---------------------------------------------------------------------===// -// string_util_test - Unit tests for src/string_util.cc +// statistics_test - Unit tests for src/statistics.cc //===---------------------------------------------------------------------===// -#include - #include "../src/internal_macros.h" #include "../src/string_util.h" -#include "gmock/gmock.h" #include "gtest/gtest.h" namespace { @@ -66,10 +63,7 @@ TEST(StringUtilTest, stoul) { EXPECT_EQ(4ul, pos); } #ifndef BENCHMARK_HAS_NO_EXCEPTIONS - { - ASSERT_THROW(std::ignore = benchmark::stoul("this is a test"), - std::invalid_argument); - } + { ASSERT_THROW(benchmark::stoul("this is a test"), std::invalid_argument); } #endif } @@ -113,10 +107,7 @@ EXPECT_EQ(1ul, pos); EXPECT_EQ(4ul, pos); } #ifndef BENCHMARK_HAS_NO_EXCEPTIONS -{ - ASSERT_THROW(std::ignore = benchmark::stoi("this is a test"), - std::invalid_argument); -} +{ ASSERT_THROW(benchmark::stoi("this is a test"), std::invalid_argument); } #endif } @@ -146,10 +137,7 @@ EXPECT_EQ(1ul, pos); EXPECT_EQ(8ul, pos); } #ifndef BENCHMARK_HAS_NO_EXCEPTIONS -{ - ASSERT_THROW(std::ignore = benchmark::stod("this is a test"), - std::invalid_argument); -} +{ ASSERT_THROW(benchmark::stod("this is a test"), std::invalid_argument); } #endif } @@ -161,39 +149,4 @@ TEST(StringUtilTest, StrSplit) { std::vector({"hello", "there", "is", "more"})); } -using HumanReadableFixture = ::testing::TestWithParam< - std::tuple>; - -INSTANTIATE_TEST_SUITE_P( - HumanReadableTests, HumanReadableFixture, - ::testing::Values( - std::make_tuple(0.0, benchmark::Counter::kIs1024, "0"), - std::make_tuple(999.0, benchmark::Counter::kIs1024, "999"), - std::make_tuple(1000.0, benchmark::Counter::kIs1024, "1000"), - std::make_tuple(1024.0, benchmark::Counter::kIs1024, "1Ki"), - std::make_tuple(1000 * 1000.0, benchmark::Counter::kIs1024, - "976\\.56.Ki"), - std::make_tuple(1024 * 1024.0, benchmark::Counter::kIs1024, "1Mi"), - std::make_tuple(1000 * 1000 * 1000.0, benchmark::Counter::kIs1024, - "953\\.674Mi"), - std::make_tuple(1024 * 1024 * 1024.0, benchmark::Counter::kIs1024, - "1Gi"), - std::make_tuple(0.0, benchmark::Counter::kIs1000, "0"), - std::make_tuple(999.0, benchmark::Counter::kIs1000, "999"), - std::make_tuple(1000.0, benchmark::Counter::kIs1000, "1k"), - std::make_tuple(1024.0, benchmark::Counter::kIs1000, "1.024k"), - std::make_tuple(1000 * 1000.0, benchmark::Counter::kIs1000, "1M"), - std::make_tuple(1024 * 1024.0, benchmark::Counter::kIs1000, - "1\\.04858M"), - std::make_tuple(1000 * 1000 * 1000.0, benchmark::Counter::kIs1000, - "1G"), - std::make_tuple(1024 * 1024 * 1024.0, benchmark::Counter::kIs1000, - "1\\.07374G"))); - -TEST_P(HumanReadableFixture, HumanReadableNumber) { - std::string str = benchmark::HumanReadableNumber(std::get<0>(GetParam()), - std::get<1>(GetParam())); - ASSERT_THAT(str, ::testing::MatchesRegex(std::get<2>(GetParam()))); -} - } // end namespace diff --git a/third-party/benchmark/test/user_counters_tabular_test.cc b/third-party/benchmark/test/user_counters_tabular_test.cc index cfc1ab069c78..45ac043d5193 100644 --- a/third-party/benchmark/test/user_counters_tabular_test.cc +++ b/third-party/benchmark/test/user_counters_tabular_test.cc @@ -63,9 +63,6 @@ ADD_CASES(TC_CSVOut, {{"%csv_header," void BM_Counters_Tabular(benchmark::State& state) { for (auto _ : state) { - // This test requires a non-zero CPU time to avoid divide-by-zero - auto iterations = double(state.iterations()) * double(state.iterations()); - benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters.insert({ @@ -333,7 +330,7 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/repeats:2/threads:1_stddev\",%csv_report," "%float,%float,%float,%float,%float,%float$"}}); ADD_CASES(TC_CSVOut, - {{"^\"BM_Counters_Tabular/repeats:2/threads:1_cv\",%csv_cv_report," + {{"^\"BM_Counters_Tabular/repeats:2/threads:1_cv\",%csv_report," "%float,%float,%float,%float,%float,%float$"}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/repeats:2/threads:2\",%csv_report," @@ -351,7 +348,7 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/repeats:2/threads:2_stddev\",%csv_report," "%float,%float,%float,%float,%float,%float$"}}); ADD_CASES(TC_CSVOut, - {{"^\"BM_Counters_Tabular/repeats:2/threads:2_cv\",%csv_cv_report," + {{"^\"BM_Counters_Tabular/repeats:2/threads:2_cv\",%csv_report," "%float,%float,%float,%float,%float,%float$"}}); // VS2013 does not allow this function to be passed as a lambda argument // to CHECK_BENCHMARK_RESULTS() @@ -375,8 +372,7 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/repeats:2/threads:2$", void BM_CounterRates_Tabular(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - auto iterations = double(state.iterations()) * double(state.iterations()); - benchmark::DoNotOptimize(iterations); + benchmark::DoNotOptimize(state.iterations()); } namespace bm = benchmark; state.counters.insert({ diff --git a/third-party/benchmark/test/user_counters_test.cc b/third-party/benchmark/test/user_counters_test.cc index 22252acbf6a2..1cc74552a1bd 100644 --- a/third-party/benchmark/test/user_counters_test.cc +++ b/third-party/benchmark/test/user_counters_test.cc @@ -67,8 +67,7 @@ int num_calls1 = 0; void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - auto iterations = double(state.iterations()) * double(state.iterations()); - benchmark::DoNotOptimize(iterations); + benchmark::DoNotOptimize(state.iterations()); } state.counters["foo"] = 1; state.counters["bar"] = ++num_calls1; @@ -119,8 +118,7 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_WithBytesAndItemsPSec", void BM_Counters_Rate(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - auto iterations = double(state.iterations()) * double(state.iterations()); - benchmark::DoNotOptimize(iterations); + benchmark::DoNotOptimize(state.iterations()); } namespace bm = benchmark; state.counters["foo"] = bm::Counter{1, bm::Counter::kIsRate}; @@ -163,8 +161,7 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_Rate", &CheckRate); void BM_Invert(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - auto iterations = double(state.iterations()) * double(state.iterations()); - benchmark::DoNotOptimize(iterations); + benchmark::DoNotOptimize(state.iterations()); } namespace bm = benchmark; state.counters["foo"] = bm::Counter{0.0001, bm::Counter::kInvert}; @@ -198,14 +195,14 @@ void CheckInvert(Results const& e) { CHECK_BENCHMARK_RESULTS("BM_Invert", &CheckInvert); // ========================================================================= // -// --------------------- InvertedRate Counters Output ---------------------- // +// ------------------------- InvertedRate Counters Output +// -------------------------- // // ========================================================================= // void BM_Counters_InvertedRate(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - auto iterations = double(state.iterations()) * double(state.iterations()); - benchmark::DoNotOptimize(iterations); + benchmark::DoNotOptimize(state.iterations()); } namespace bm = benchmark; state.counters["foo"] = @@ -333,8 +330,7 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_AvgThreads/threads:%int", void BM_Counters_AvgThreadsRate(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - auto iterations = double(state.iterations()) * double(state.iterations()); - benchmark::DoNotOptimize(iterations); + benchmark::DoNotOptimize(state.iterations()); } namespace bm = benchmark; state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgThreadsRate}; @@ -421,8 +417,7 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_IterationInvariant", void BM_Counters_kIsIterationInvariantRate(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - auto iterations = double(state.iterations()) * double(state.iterations()); - benchmark::DoNotOptimize(iterations); + benchmark::DoNotOptimize(state.iterations()); } namespace bm = benchmark; state.counters["foo"] = @@ -465,7 +460,7 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_kIsIterationInvariantRate", &CheckIsIterationInvariantRate); // ========================================================================= // -// --------------------- AvgIterations Counters Output --------------------- // +// ------------------- AvgIterations Counters Output ------------------ // // ========================================================================= // void BM_Counters_AvgIterations(benchmark::State& state) { @@ -507,14 +502,13 @@ void CheckAvgIterations(Results const& e) { CHECK_BENCHMARK_RESULTS("BM_Counters_AvgIterations", &CheckAvgIterations); // ========================================================================= // -// ------------------- AvgIterationsRate Counters Output ------------------- // +// ----------------- AvgIterationsRate Counters Output ---------------- // // ========================================================================= // void BM_Counters_kAvgIterationsRate(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - auto iterations = double(state.iterations()) * double(state.iterations()); - benchmark::DoNotOptimize(iterations); + benchmark::DoNotOptimize(state.iterations()); } namespace bm = benchmark; state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgIterationsRate}; diff --git a/third-party/benchmark/test/user_counters_thousands_test.cc b/third-party/benchmark/test/user_counters_thousands_test.cc index fc153835f800..a42683b32fa7 100644 --- a/third-party/benchmark/test/user_counters_thousands_test.cc +++ b/third-party/benchmark/test/user_counters_thousands_test.cc @@ -16,13 +16,13 @@ void BM_Counters_Thousands(benchmark::State& state) { {"t0_1000000DefaultBase", bm::Counter(1000 * 1000, bm::Counter::kDefaults)}, {"t1_1000000Base1000", bm::Counter(1000 * 1000, bm::Counter::kDefaults, - bm::Counter::OneK::kIs1000)}, + benchmark::Counter::OneK::kIs1000)}, {"t2_1000000Base1024", bm::Counter(1000 * 1000, bm::Counter::kDefaults, - bm::Counter::OneK::kIs1024)}, + benchmark::Counter::OneK::kIs1024)}, {"t3_1048576Base1000", bm::Counter(1024 * 1024, bm::Counter::kDefaults, - bm::Counter::OneK::kIs1000)}, + benchmark::Counter::OneK::kIs1000)}, {"t4_1048576Base1024", bm::Counter(1024 * 1024, bm::Counter::kDefaults, - bm::Counter::OneK::kIs1024)}, + benchmark::Counter::OneK::kIs1024)}, }); } BENCHMARK(BM_Counters_Thousands)->Repetitions(2); @@ -30,21 +30,21 @@ ADD_CASES( TC_ConsoleOut, { {"^BM_Counters_Thousands/repeats:2 %console_report " - "t0_1000000DefaultBase=1M " - "t1_1000000Base1000=1M t2_1000000Base1024=976.56[23]Ki " - "t3_1048576Base1000=1.04858M t4_1048576Base1024=1Mi$"}, + "t0_1000000DefaultBase=1000k " + "t1_1000000Base1000=1000k t2_1000000Base1024=976.56[23]k " + "t3_1048576Base1000=1048.58k t4_1048576Base1024=1024k$"}, {"^BM_Counters_Thousands/repeats:2 %console_report " - "t0_1000000DefaultBase=1M " - "t1_1000000Base1000=1M t2_1000000Base1024=976.56[23]Ki " - "t3_1048576Base1000=1.04858M t4_1048576Base1024=1Mi$"}, + "t0_1000000DefaultBase=1000k " + "t1_1000000Base1000=1000k t2_1000000Base1024=976.56[23]k " + "t3_1048576Base1000=1048.58k t4_1048576Base1024=1024k$"}, {"^BM_Counters_Thousands/repeats:2_mean %console_report " - "t0_1000000DefaultBase=1M t1_1000000Base1000=1M " - "t2_1000000Base1024=976.56[23]Ki t3_1048576Base1000=1.04858M " - "t4_1048576Base1024=1Mi$"}, + "t0_1000000DefaultBase=1000k t1_1000000Base1000=1000k " + "t2_1000000Base1024=976.56[23]k t3_1048576Base1000=1048.58k " + "t4_1048576Base1024=1024k$"}, {"^BM_Counters_Thousands/repeats:2_median %console_report " - "t0_1000000DefaultBase=1M t1_1000000Base1000=1M " - "t2_1000000Base1024=976.56[23]Ki t3_1048576Base1000=1.04858M " - "t4_1048576Base1024=1Mi$"}, + "t0_1000000DefaultBase=1000k t1_1000000Base1000=1000k " + "t2_1000000Base1024=976.56[23]k t3_1048576Base1000=1048.58k " + "t4_1048576Base1024=1024k$"}, {"^BM_Counters_Thousands/repeats:2_stddev %console_time_only_report [ " "]*2 t0_1000000DefaultBase=0 t1_1000000Base1000=0 " "t2_1000000Base1024=0 t3_1048576Base1000=0 t4_1048576Base1024=0$"}, diff --git a/third-party/benchmark/tools/compare.py b/third-party/benchmark/tools/compare.py index 7572520cc0ca..f1504c96fa2b 100755 --- a/third-party/benchmark/tools/compare.py +++ b/third-party/benchmark/tools/compare.py @@ -1,35 +1,29 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python -# type: ignore +import unittest """ compare.py - versatile benchmark output compare tool """ import argparse +from argparse import ArgumentParser import json -import os import sys -import unittest -from argparse import ArgumentParser - import gbench -from gbench import report, util +from gbench import util, report +from gbench.util import * def check_inputs(in1, in2, flags): """ Perform checking on the user provided inputs and diagnose any abnormalities """ - in1_kind, in1_err = util.classify_input_file(in1) - in2_kind, in2_err = util.classify_input_file(in2) - output_file = util.find_benchmark_flag("--benchmark_out=", flags) - output_type = util.find_benchmark_flag("--benchmark_out_format=", flags) - if ( - in1_kind == util.IT_Executable - and in2_kind == util.IT_Executable - and output_file - ): + in1_kind, in1_err = classify_input_file(in1) + in2_kind, in2_err = classify_input_file(in2) + output_file = find_benchmark_flag("--benchmark_out=", flags) + output_type = find_benchmark_flag("--benchmark_out_format=", flags) + if in1_kind == IT_Executable and in2_kind == IT_Executable and output_file: print( ( "WARNING: '--benchmark_out=%s' will be passed to both " @@ -37,14 +31,11 @@ def check_inputs(in1, in2, flags): ) % output_file ) - if in1_kind == util.IT_JSON and in2_kind == util.IT_JSON: - # When both sides are JSON the only supported flag is - # --benchmark_filter= - for flag in util.remove_benchmark_flags("--benchmark_filter=", flags): - print( - "WARNING: passing %s has no effect since both " - "inputs are JSON" % flag - ) + if in1_kind == IT_JSON and in2_kind == IT_JSON and len(flags) > 0: + print( + "WARNING: passing optional flags has no effect since both " + "inputs are JSON" + ) if output_type is not None and output_type != "json": print( ( @@ -57,9 +48,7 @@ def check_inputs(in1, in2, flags): def create_parser(): - parser = ArgumentParser( - description="versatile benchmark output compare tool" - ) + parser = ArgumentParser(description="versatile benchmark output compare tool") parser.add_argument( "-a", @@ -305,9 +294,7 @@ def main(): # Now, filter the benchmarks so that the difference report can work if filter_baseline and filter_contender: replacement = "[%s vs. %s]" % (filter_baseline, filter_contender) - json1 = gbench.report.filter_benchmark( - json1_orig, filter_baseline, replacement - ) + json1 = gbench.report.filter_benchmark(json1_orig, filter_baseline, replacement) json2 = gbench.report.filter_benchmark( json2_orig, filter_contender, replacement ) @@ -327,7 +314,7 @@ def main(): # Optionally, diff and output to JSON if args.dump_to_json is not None: with open(args.dump_to_json, "w") as f_json: - json.dump(diff_report, f_json, indent=1) + json.dump(diff_report, f_json) class TestParser(unittest.TestCase): @@ -436,9 +423,7 @@ class TestParser(unittest.TestCase): self.assertFalse(parsed.benchmark_options) def test_filters_with_remainder(self): - parsed = self.parser.parse_args( - ["filters", self.testInput0, "c", "d", "e"] - ) + parsed = self.parser.parse_args(["filters", self.testInput0, "c", "d", "e"]) self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) self.assertEqual(parsed.mode, "filters") @@ -474,14 +459,7 @@ class TestParser(unittest.TestCase): def test_benchmarksfiltered_with_remainder(self): parsed = self.parser.parse_args( - [ - "benchmarksfiltered", - self.testInput0, - "c", - self.testInput1, - "e", - "f", - ] + ["benchmarksfiltered", self.testInput0, "c", self.testInput1, "e", "f"] ) self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) diff --git a/third-party/benchmark/tools/gbench/Inputs/test1_run1.json b/third-party/benchmark/tools/gbench/Inputs/test1_run1.json index 9daed0bcc6c4..601e327aefb5 100644 --- a/third-party/benchmark/tools/gbench/Inputs/test1_run1.json +++ b/third-party/benchmark/tools/gbench/Inputs/test1_run1.json @@ -114,14 +114,6 @@ "real_time": 1, "cpu_time": 1, "time_unit": "s" - }, - { - "name": "BM_hasLabel", - "label": "a label", - "iterations": 1, - "real_time": 1, - "cpu_time": 1, - "time_unit": "s" } ] } diff --git a/third-party/benchmark/tools/gbench/Inputs/test1_run2.json b/third-party/benchmark/tools/gbench/Inputs/test1_run2.json index dc52970abf8b..3cbcf39b0c93 100644 --- a/third-party/benchmark/tools/gbench/Inputs/test1_run2.json +++ b/third-party/benchmark/tools/gbench/Inputs/test1_run2.json @@ -114,14 +114,6 @@ "real_time": 1, "cpu_time": 1, "time_unit": "ns" - }, - { - "name": "BM_hasLabel", - "label": "a label", - "iterations": 1, - "real_time": 1, - "cpu_time": 1, - "time_unit": "s" } ] } diff --git a/third-party/benchmark/tools/gbench/__init__.py b/third-party/benchmark/tools/gbench/__init__.py index 921256881491..ffca396b4c3f 100644 --- a/third-party/benchmark/tools/gbench/__init__.py +++ b/third-party/benchmark/tools/gbench/__init__.py @@ -5,4 +5,4 @@ __email__ = "eric@efcs.ca" __versioninfo__ = (0, 5, 0) __version__ = ".".join(str(v) for v in __versioninfo__) + "dev" -__all__ = [] # type: ignore +__all__ = [] diff --git a/third-party/benchmark/tools/gbench/report.py b/third-party/benchmark/tools/gbench/report.py index 7158fd1654cb..5092b0bf1469 100644 --- a/third-party/benchmark/tools/gbench/report.py +++ b/third-party/benchmark/tools/gbench/report.py @@ -1,17 +1,15 @@ -# type: ignore - -""" -report.py - Utilities for reporting statistics about benchmark results +"""report.py - Utilities for reporting statistics about benchmark results """ -import copy +import unittest import os -import random import re -import unittest +import copy +import random +from scipy.stats import mannwhitneyu, gmean from numpy import array -from scipy.stats import gmean, mannwhitneyu +from pandas import Timedelta class BenchmarkColor(object): @@ -44,13 +42,6 @@ UTEST_MIN_REPETITIONS = 2 UTEST_OPTIMAL_REPETITIONS = 9 # Lowest reasonable number, More is better. UTEST_COL_NAME = "_pvalue" -_TIME_UNIT_TO_SECONDS_MULTIPLIER = { - "s": 1.0, - "ms": 1e-3, - "us": 1e-6, - "ns": 1e-9, -} - def color_format(use_color, fmt_str, *args, **kwargs): """ @@ -61,10 +52,7 @@ def color_format(use_color, fmt_str, *args, **kwargs): """ assert use_color is True or use_color is False if not use_color: - args = [ - arg if not isinstance(arg, BenchmarkColor) else BC_NONE - for arg in args - ] + args = [arg if not isinstance(arg, BenchmarkColor) else BC_NONE for arg in args] kwargs = { key: arg if not isinstance(arg, BenchmarkColor) else BC_NONE for key, arg in kwargs.items() @@ -177,9 +165,9 @@ def get_timedelta_field_as_seconds(benchmark, field_name): Get value of field_name field of benchmark, which is time with time unit time_unit, as time in seconds. """ - timedelta = benchmark[field_name] - time_unit = benchmark.get("time_unit", "s") - return timedelta * _TIME_UNIT_TO_SECONDS_MULTIPLIER.get(time_unit) + time_unit = benchmark["time_unit"] if "time_unit" in benchmark else "s" + dt = Timedelta(benchmark[field_name], time_unit) + return dt / Timedelta(1, "s") def calculate_geomean(json): @@ -285,7 +273,6 @@ def get_difference_report(json1, json2, utest=False): partitions = partition_benchmarks(json1, json2) for partition in partitions: benchmark_name = partition[0][0]["name"] - label = partition[0][0]["label"] if "label" in partition[0][0] else "" time_unit = partition[0][0]["time_unit"] measurements = [] utest_results = {} @@ -299,12 +286,8 @@ def get_difference_report(json1, json2, utest=False): "cpu_time": bn["cpu_time"], "real_time_other": other_bench["real_time"], "cpu_time_other": other_bench["cpu_time"], - "time": calculate_change( - bn["real_time"], other_bench["real_time"] - ), - "cpu": calculate_change( - bn["cpu_time"], other_bench["cpu_time"] - ), + "time": calculate_change(bn["real_time"], other_bench["real_time"]), + "cpu": calculate_change(bn["cpu_time"], other_bench["cpu_time"]), } ) @@ -315,7 +298,7 @@ def get_difference_report(json1, json2, utest=False): have_optimal_repetitions, cpu_pvalue, time_pvalue = calc_utest( timings_cpu, timings_time ) - if cpu_pvalue is not None and time_pvalue is not None: + if cpu_pvalue and time_pvalue: utest_results = { "have_optimal_repetitions": have_optimal_repetitions, "cpu_pvalue": cpu_pvalue, @@ -330,20 +313,16 @@ def get_difference_report(json1, json2, utest=False): # benchmark suite. if measurements: run_type = ( - partition[0][0]["run_type"] - if "run_type" in partition[0][0] - else "" + partition[0][0]["run_type"] if "run_type" in partition[0][0] else "" ) aggregate_name = ( partition[0][0]["aggregate_name"] - if run_type == "aggregate" - and "aggregate_name" in partition[0][0] + if run_type == "aggregate" and "aggregate_name" in partition[0][0] else "" ) diff_report.append( { "name": benchmark_name, - "label": label, "measurements": measurements, "time_unit": time_unit, "run_type": run_type, @@ -358,7 +337,6 @@ def get_difference_report(json1, json2, utest=False): diff_report.append( { "name": "OVERALL_GEOMEAN", - "label": "", "measurements": [ { "real_time": lhs_gmean[0], @@ -414,7 +392,7 @@ def print_difference_report( # and if it is non-aggregate, then don't print it. if ( not include_aggregates_only - or "run_type" not in benchmark + or not "run_type" in benchmark or benchmark["run_type"] == "aggregate" ): for measurement in benchmark["measurements"]: @@ -460,9 +438,7 @@ class TestGetUniqueBenchmarkNames(unittest.TestCase): def load_results(self): import json - testInputs = os.path.join( - os.path.dirname(os.path.realpath(__file__)), "Inputs" - ) + testInputs = os.path.join(os.path.dirname(os.path.realpath(__file__)), "Inputs") testOutput = os.path.join(testInputs, "test3_run0.json") with open(testOutput, "r") as f: json = json.load(f) @@ -509,73 +485,16 @@ class TestReportDifference(unittest.TestCase): ["BM_SameTimes", "+0.0000", "+0.0000", "10", "10", "10", "10"], ["BM_2xFaster", "-0.5000", "-0.5000", "50", "25", "50", "25"], ["BM_2xSlower", "+1.0000", "+1.0000", "50", "100", "50", "100"], - [ - "BM_1PercentFaster", - "-0.0100", - "-0.0100", - "100", - "99", - "100", - "99", - ], - [ - "BM_1PercentSlower", - "+0.0100", - "+0.0100", - "100", - "101", - "100", - "101", - ], - [ - "BM_10PercentFaster", - "-0.1000", - "-0.1000", - "100", - "90", - "100", - "90", - ], - [ - "BM_10PercentSlower", - "+0.1000", - "+0.1000", - "100", - "110", - "100", - "110", - ], - [ - "BM_100xSlower", - "+99.0000", - "+99.0000", - "100", - "10000", - "100", - "10000", - ], - [ - "BM_100xFaster", - "-0.9900", - "-0.9900", - "10000", - "100", - "10000", - "100", - ], - [ - "BM_10PercentCPUToTime", - "+0.1000", - "-0.1000", - "100", - "110", - "100", - "90", - ], + ["BM_1PercentFaster", "-0.0100", "-0.0100", "100", "99", "100", "99"], + ["BM_1PercentSlower", "+0.0100", "+0.0100", "100", "101", "100", "101"], + ["BM_10PercentFaster", "-0.1000", "-0.1000", "100", "90", "100", "90"], + ["BM_10PercentSlower", "+0.1000", "+0.1000", "100", "110", "100", "110"], + ["BM_100xSlower", "+99.0000", "+99.0000", "100", "10000", "100", "10000"], + ["BM_100xFaster", "-0.9900", "-0.9900", "10000", "100", "10000", "100"], + ["BM_10PercentCPUToTime", "+0.1000", "-0.1000", "100", "110", "100", "90"], ["BM_ThirdFaster", "-0.3333", "-0.3334", "100", "67", "100", "67"], ["BM_NotBadTimeUnit", "-0.9000", "+0.2000", "0", "0", "0", "1"], - ["BM_hasLabel", "+0.0000", "+0.0000", "1", "1", "1", "1"], - ["OVERALL_GEOMEAN", "-0.8113", "-0.7779", "0", "0", "0", "0"], + ["OVERALL_GEOMEAN", "-0.8344", "-0.8026", "0", "0", "0", "0"], ] output_lines_with_header = print_difference_report( self.json_diff_report, use_color=False @@ -593,7 +512,6 @@ class TestReportDifference(unittest.TestCase): expected_output = [ { "name": "BM_SameTimes", - "label": "", "measurements": [ { "time": 0.0000, @@ -609,7 +527,6 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_2xFaster", - "label": "", "measurements": [ { "time": -0.5000, @@ -625,7 +542,6 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_2xSlower", - "label": "", "measurements": [ { "time": 1.0000, @@ -641,7 +557,6 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_1PercentFaster", - "label": "", "measurements": [ { "time": -0.0100, @@ -657,7 +572,6 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_1PercentSlower", - "label": "", "measurements": [ { "time": 0.0100, @@ -673,7 +587,6 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_10PercentFaster", - "label": "", "measurements": [ { "time": -0.1000, @@ -689,7 +602,6 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_10PercentSlower", - "label": "", "measurements": [ { "time": 0.1000, @@ -705,7 +617,6 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_100xSlower", - "label": "", "measurements": [ { "time": 99.0000, @@ -721,7 +632,6 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_100xFaster", - "label": "", "measurements": [ { "time": -0.9900, @@ -737,7 +647,6 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_10PercentCPUToTime", - "label": "", "measurements": [ { "time": 0.1000, @@ -753,7 +662,6 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_ThirdFaster", - "label": "", "measurements": [ { "time": -0.3333, @@ -769,7 +677,6 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_NotBadTimeUnit", - "label": "", "measurements": [ { "time": -0.9000, @@ -783,33 +690,16 @@ class TestReportDifference(unittest.TestCase): "time_unit": "s", "utest": {}, }, - { - "name": "BM_hasLabel", - "label": "a label", - "measurements": [ - { - "time": 0.0000, - "cpu": 0.0000, - "real_time": 1, - "real_time_other": 1, - "cpu_time": 1, - "cpu_time_other": 1, - } - ], - "time_unit": "s", - "utest": {}, - }, { "name": "OVERALL_GEOMEAN", - "label": "", "measurements": [ { - "real_time": 3.1622776601683826e-06, - "cpu_time": 3.2130844755623912e-06, + "real_time": 1.193776641714438e-06, + "cpu_time": 1.2144445585302297e-06, "real_time_other": 1.9768988699420897e-07, "cpu_time_other": 2.397447755209533e-07, - "time": -0.8112976497120911, - "cpu": -0.7778551721181174, + "time": -0.834399601997324, + "cpu": -0.8025889499549471, } ], "time_unit": "s", @@ -821,7 +711,6 @@ class TestReportDifference(unittest.TestCase): self.assertEqual(len(self.json_diff_report), len(expected_output)) for out, expected in zip(self.json_diff_report, expected_output): self.assertEqual(out["name"], expected["name"]) - self.assertEqual(out["label"], expected["label"]) self.assertEqual(out["time_unit"], expected["time_unit"]) assert_utest(self, out, expected) assert_measurements(self, out, expected) @@ -1197,9 +1086,7 @@ class TestReportDifferenceWithUTest(unittest.TestCase): assert_measurements(self, out, expected) -class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly( - unittest.TestCase -): +class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly(unittest.TestCase): @classmethod def setUpClass(cls): def load_results(): @@ -1482,108 +1369,12 @@ class TestReportSorting(unittest.TestCase): for n in range(len(self.json["benchmarks"]) ** 2): random.shuffle(self.json["benchmarks"]) - sorted_benchmarks = util.sort_benchmark_results(self.json)[ - "benchmarks" - ] + sorted_benchmarks = util.sort_benchmark_results(self.json)["benchmarks"] self.assertEqual(len(expected_names), len(sorted_benchmarks)) for out, expected in zip(sorted_benchmarks, expected_names): self.assertEqual(out["name"], expected) -class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly2( - unittest.TestCase -): - @classmethod - def setUpClass(cls): - def load_results(): - import json - - testInputs = os.path.join( - os.path.dirname(os.path.realpath(__file__)), "Inputs" - ) - testOutput1 = os.path.join(testInputs, "test5_run0.json") - testOutput2 = os.path.join(testInputs, "test5_run1.json") - with open(testOutput1, "r") as f: - json1 = json.load(f) - json1["benchmarks"] = [ - json1["benchmarks"][0] for i in range(1000) - ] - with open(testOutput2, "r") as f: - json2 = json.load(f) - json2["benchmarks"] = [ - json2["benchmarks"][0] for i in range(1000) - ] - return json1, json2 - - json1, json2 = load_results() - cls.json_diff_report = get_difference_report(json1, json2, utest=True) - - def test_json_diff_report_pretty_printing(self): - expect_line = [ - "BM_ManyRepetitions_pvalue", - "0.0000", - "0.0000", - "U", - "Test,", - "Repetitions:", - "1000", - "vs", - "1000", - ] - output_lines_with_header = print_difference_report( - self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False - ) - output_lines = output_lines_with_header[2:] - found = False - for i in range(0, len(output_lines)): - parts = [x for x in output_lines[i].split(" ") if x] - found = expect_line == parts - if found: - break - self.assertTrue(found) - - def test_json_diff_report(self): - expected_output = [ - { - "name": "BM_ManyRepetitions", - "label": "", - "time_unit": "s", - "run_type": "", - "aggregate_name": "", - "utest": { - "have_optimal_repetitions": True, - "cpu_pvalue": 0.0, - "time_pvalue": 0.0, - "nr_of_repetitions": 1000, - "nr_of_repetitions_other": 1000, - }, - }, - { - "name": "OVERALL_GEOMEAN", - "label": "", - "measurements": [ - { - "real_time": 1.0, - "cpu_time": 1000.000000000069, - "real_time_other": 1000.000000000069, - "cpu_time_other": 1.0, - "time": 999.000000000069, - "cpu": -0.9990000000000001, - } - ], - "time_unit": "s", - "run_type": "aggregate", - "aggregate_name": "geomean", - "utest": {}, - }, - ] - self.assertEqual(len(self.json_diff_report), len(expected_output)) - for out, expected in zip(self.json_diff_report, expected_output): - self.assertEqual(out["name"], expected["name"]) - self.assertEqual(out["time_unit"], expected["time_unit"]) - assert_utest(self, out, expected) - - def assert_utest(unittest_instance, lhs, rhs): if lhs["utest"]: unittest_instance.assertAlmostEqual( diff --git a/third-party/benchmark/tools/gbench/util.py b/third-party/benchmark/tools/gbench/util.py index 4d061a3a1e34..a46903a6d248 100644 --- a/third-party/benchmark/tools/gbench/util.py +++ b/third-party/benchmark/tools/gbench/util.py @@ -2,10 +2,10 @@ """ import json import os -import re +import tempfile import subprocess import sys -import tempfile +import functools # Input file type enumeration IT_Invalid = 0 @@ -58,7 +58,7 @@ def classify_input_file(filename): """ Return a tuple (type, msg) where 'type' specifies the classified type of 'filename'. If 'type' is 'IT_Invalid' then 'msg' is a human readable - string representing the error. + string represeting the error. """ ftype = IT_Invalid err_msg = None @@ -72,8 +72,7 @@ def classify_input_file(filename): ftype = IT_JSON else: err_msg = ( - "'%s' does not name a valid benchmark executable or JSON file" - % filename + "'%s' does not name a valid benchmark executable or JSON file" % filename ) return ftype, err_msg @@ -114,41 +113,13 @@ def remove_benchmark_flags(prefix, benchmark_flags): return [f for f in benchmark_flags if not f.startswith(prefix)] -def load_benchmark_results(fname, benchmark_filter): +def load_benchmark_results(fname): """ Read benchmark output from a file and return the JSON object. - - Apply benchmark_filter, a regular expression, with nearly the same - semantics of the --benchmark_filter argument. May be None. - Note: the Python regular expression engine is used instead of the - one used by the C++ code, which may produce different results - in complex cases. - REQUIRES: 'fname' names a file containing JSON benchmark output. """ - - def benchmark_wanted(benchmark): - if benchmark_filter is None: - return True - name = benchmark.get("run_name", None) or benchmark["name"] - return re.search(benchmark_filter, name) is not None - with open(fname, "r") as f: - results = json.load(f) - if "context" in results: - if "json_schema_version" in results["context"]: - json_schema_version = results["context"]["json_schema_version"] - if json_schema_version != 1: - print( - "In %s, got unnsupported JSON schema version: %i, expected 1" - % (fname, json_schema_version) - ) - sys.exit(1) - if "benchmarks" in results: - results["benchmarks"] = list( - filter(benchmark_wanted, results["benchmarks"]) - ) - return results + return json.load(f) def sort_benchmark_results(result): @@ -197,9 +168,7 @@ def run_benchmark(exe_name, benchmark_flags): is_temp_output = True thandle, output_name = tempfile.mkstemp() os.close(thandle) - benchmark_flags = list(benchmark_flags) + [ - "--benchmark_out=%s" % output_name - ] + benchmark_flags = list(benchmark_flags) + ["--benchmark_out=%s" % output_name] cmd = [exe_name] + benchmark_flags print("RUNNING: %s" % " ".join(cmd)) @@ -207,7 +176,7 @@ def run_benchmark(exe_name, benchmark_flags): if exitCode != 0: print("TEST FAILED...") sys.exit(exitCode) - json_res = load_benchmark_results(output_name, None) + json_res = load_benchmark_results(output_name) if is_temp_output: os.unlink(output_name) return json_res @@ -222,10 +191,7 @@ def run_or_load_benchmark(filename, benchmark_flags): """ ftype = check_input_file(filename) if ftype == IT_JSON: - benchmark_filter = find_benchmark_flag( - "--benchmark_filter=", benchmark_flags - ) - return load_benchmark_results(filename, benchmark_filter) + return load_benchmark_results(filename) if ftype == IT_Executable: return run_benchmark(filename, benchmark_flags) raise ValueError("Unknown file type %s" % ftype) diff --git a/third-party/benchmark/tools/requirements.txt b/third-party/benchmark/tools/requirements.txt index f32f35b8fbfd..3b3331b5af12 100644 --- a/third-party/benchmark/tools/requirements.txt +++ b/third-party/benchmark/tools/requirements.txt @@ -1,2 +1 @@ -numpy == 1.25 -scipy == 1.10.0 +scipy>=1.5.0 \ No newline at end of file diff --git a/third-party/benchmark/tools/strip_asm.py b/third-party/benchmark/tools/strip_asm.py index bc3a774a7932..086255dc6577 100755 --- a/third-party/benchmark/tools/strip_asm.py +++ b/third-party/benchmark/tools/strip_asm.py @@ -1,20 +1,20 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python """ strip_asm.py - Cleanup ASM output for the specified file """ +from argparse import ArgumentParser +import sys import os import re -import sys -from argparse import ArgumentParser def find_used_labels(asm): found = set() - label_re = re.compile(r"\s*j[a-z]+\s+\.L([a-zA-Z0-9][a-zA-Z0-9_]*)") - for line in asm.splitlines(): - m = label_re.match(line) + label_re = re.compile("\s*j[a-z]+\s+\.L([a-zA-Z0-9][a-zA-Z0-9_]*)") + for l in asm.splitlines(): + m = label_re.match(l) if m: found.add(".L%s" % m.group(1)) return found @@ -23,8 +23,8 @@ def find_used_labels(asm): def normalize_labels(asm): decls = set() label_decl = re.compile("^[.]{0,1}L([a-zA-Z0-9][a-zA-Z0-9_]*)(?=:)") - for line in asm.splitlines(): - m = label_decl.match(line) + for l in asm.splitlines(): + m = label_decl.match(l) if m: decls.add(m.group(0)) if len(decls) == 0: @@ -33,7 +33,7 @@ def normalize_labels(asm): if not needs_dot: return asm for ld in decls: - asm = re.sub(r"(^|\s+)" + ld + r"(?=:|\s)", "\\1." + ld, asm) + asm = re.sub("(^|\s+)" + ld + "(?=:|\s)", "\\1." + ld, asm) return asm @@ -41,11 +41,11 @@ def transform_labels(asm): asm = normalize_labels(asm) used_decls = find_used_labels(asm) new_asm = "" - label_decl = re.compile(r"^\.L([a-zA-Z0-9][a-zA-Z0-9_]*)(?=:)") - for line in asm.splitlines(): - m = label_decl.match(line) + label_decl = re.compile("^\.L([a-zA-Z0-9][a-zA-Z0-9_]*)(?=:)") + for l in asm.splitlines(): + m = label_decl.match(l) if not m or m.group(0) in used_decls: - new_asm += line + new_asm += l new_asm += "\n" return new_asm @@ -63,24 +63,21 @@ def is_identifier(tk): return True -def process_identifiers(line): +def process_identifiers(l): """ process_identifiers - process all identifiers and modify them to have consistent names across all platforms; specifically across ELF and MachO. For example, MachO inserts an additional understore at the beginning of names. This function removes that. """ - parts = re.split(r"([a-zA-Z0-9_]+)", line) + parts = re.split(r"([a-zA-Z0-9_]+)", l) new_line = "" for tk in parts: if is_identifier(tk): if tk.startswith("__Z"): tk = tk[1:] elif ( - tk.startswith("_") - and len(tk) > 1 - and tk[1].isalpha() - and tk[1] != "Z" + tk.startswith("_") and len(tk) > 1 and tk[1].isalpha() and tk[1] != "Z" ): tk = tk[1:] new_line += tk @@ -96,35 +93,33 @@ def process_asm(asm): # TODO: Add more things we want to remove discard_regexes = [ - re.compile(r"\s+\..*$"), # directive - re.compile(r"\s*#(NO_APP|APP)$"), # inline ASM - re.compile(r"\s*#.*$"), # comment line - re.compile( - r"\s*\.globa?l\s*([.a-zA-Z_][a-zA-Z0-9$_.]*)" - ), # global directive + re.compile("\s+\..*$"), # directive + re.compile("\s*#(NO_APP|APP)$"), # inline ASM + re.compile("\s*#.*$"), # comment line + re.compile("\s*\.globa?l\s*([.a-zA-Z_][a-zA-Z0-9$_.]*)"), # global directive re.compile( - r"\s*\.(string|asciz|ascii|[1248]?byte|short|word|long|quad|value|zero)" + "\s*\.(string|asciz|ascii|[1248]?byte|short|word|long|quad|value|zero)" ), ] - keep_regexes: list[re.Pattern] = [] + keep_regexes = [] fn_label_def = re.compile("^[a-zA-Z_][a-zA-Z0-9_.]*:") - for line in asm.splitlines(): + for l in asm.splitlines(): # Remove Mach-O attribute - line = line.replace("@GOTPCREL", "") + l = l.replace("@GOTPCREL", "") add_line = True for reg in discard_regexes: - if reg.match(line) is not None: + if reg.match(l) is not None: add_line = False break for reg in keep_regexes: - if reg.match(line) is not None: + if reg.match(l) is not None: add_line = True break if add_line: - if fn_label_def.match(line) and len(new_contents) != 0: + if fn_label_def.match(l) and len(new_contents) != 0: new_contents += "\n" - line = process_identifiers(line) - new_contents += line + l = process_identifiers(l) + new_contents += l new_contents += "\n" return new_contents @@ -132,11 +127,7 @@ def process_asm(asm): def main(): parser = ArgumentParser(description="generate a stripped assembly file") parser.add_argument( - "input", - metavar="input", - type=str, - nargs=1, - help="An input assembly file", + "input", metavar="input", type=str, nargs=1, help="An input assembly file" ) parser.add_argument( "out", metavar="output", type=str, nargs=1, help="The output file" @@ -145,9 +136,9 @@ def main(): input = args.input[0] output = args.out[0] if not os.path.isfile(input): - print("ERROR: input file '%s' does not exist" % input) + print(("ERROR: input file '%s' does not exist") % input) sys.exit(1) - + contents = None with open(input, "r") as f: contents = f.read() new_contents = process_asm(contents) -- GitLab From 6325dd57318bc4640dded8f59cadd315aa8185ae Mon Sep 17 00:00:00 2001 From: Natalie Chouinard Date: Mon, 4 Mar 2024 16:45:23 -0500 Subject: [PATCH 050/929] [HLSL][SPIR-V] Add SV_DispatchThreadID semantic support (#82536) Add SPIR-V backend support for the HLSL SV_DispatchThreadID semantic attribute, which is lowered to a @llvm.dx.thread.id intrinsic in LLVM IR. In the SPIR-V backend, this is now correctly translated to a `GlobalInvocationId` builtin variable. Fixes #82534 --- clang/lib/CodeGen/CGHLSLRuntime.cpp | 16 +++- .../semantics/DispatchThreadID.hlsl | 35 ++++----- llvm/include/llvm/IR/IntrinsicsSPIRV.td | 1 + llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp | 4 +- .../Target/SPIRV/SPIRVInstructionSelector.cpp | 68 +++++++++++++++++ .../hlsl-intrinsics/SV_DispatchThreadID.ll | 76 +++++++++++++++++++ 6 files changed, 178 insertions(+), 22 deletions(-) create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/SV_DispatchThreadID.ll diff --git a/clang/lib/CodeGen/CGHLSLRuntime.cpp b/clang/lib/CodeGen/CGHLSLRuntime.cpp index e887d35198b3..794d93358b0a 100644 --- a/clang/lib/CodeGen/CGHLSLRuntime.cpp +++ b/clang/lib/CodeGen/CGHLSLRuntime.cpp @@ -18,6 +18,7 @@ #include "clang/AST/Decl.h" #include "clang/Basic/TargetOptions.h" #include "llvm/IR/IntrinsicsDirectX.h" +#include "llvm/IR/IntrinsicsSPIRV.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/Support/FormatVariadic.h" @@ -342,8 +343,19 @@ llvm::Value *CGHLSLRuntime::emitInputSemantic(IRBuilder<> &B, return B.CreateCall(FunctionCallee(DxGroupIndex)); } if (D.hasAttr()) { - llvm::Function *DxThreadID = CGM.getIntrinsic(Intrinsic::dx_thread_id); - return buildVectorInput(B, DxThreadID, Ty); + llvm::Function *ThreadIDIntrinsic; + switch (CGM.getTarget().getTriple().getArch()) { + case llvm::Triple::dxil: + ThreadIDIntrinsic = CGM.getIntrinsic(Intrinsic::dx_thread_id); + break; + case llvm::Triple::spirv: + ThreadIDIntrinsic = CGM.getIntrinsic(Intrinsic::spv_thread_id); + break; + default: + llvm_unreachable("Input semantic not supported by target"); + break; + } + return buildVectorInput(B, ThreadIDIntrinsic, Ty); } assert(false && "Unhandled parameter attribute"); return nullptr; diff --git a/clang/test/CodeGenHLSL/semantics/DispatchThreadID.hlsl b/clang/test/CodeGenHLSL/semantics/DispatchThreadID.hlsl index e285f74a822d..3efc36baa35b 100644 --- a/clang/test/CodeGenHLSL/semantics/DispatchThreadID.hlsl +++ b/clang/test/CodeGenHLSL/semantics/DispatchThreadID.hlsl @@ -1,28 +1,25 @@ -// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -emit-llvm -finclude-default-header -disable-llvm-passes -o - %s +// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -emit-llvm -finclude-default-header -disable-llvm-passes -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-DXIL +// RUN: %clang_cc1 -triple spirv-linux-vulkan-library -x hlsl -emit-llvm -finclude-default-header -disable-llvm-passes -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-SPIRV // Make sure SV_DispatchThreadID translated into dx.thread.id. -const RWBuffer In; -RWBuffer Out; - -// CHECK: define void @foo() -// CHECK: %[[ID:[0-9a-zA-Z]+]] = call i32 @llvm.dx.thread.id(i32 0) -// CHECK: call void @"?foo@@YAXH@Z"(i32 %[[ID]]) +// CHECK: define void @foo() +// CHECK-DXIL: %[[#ID:]] = call i32 @llvm.dx.thread.id(i32 0) +// CHECK-SPIRV: %[[#ID:]] = call i32 @llvm.spv.thread.id(i32 0) +// CHECK: call void @{{.*}}foo{{.*}}(i32 %[[#ID]]) [shader("compute")] [numthreads(8,8,1)] -void foo(uint Idx : SV_DispatchThreadID) { - Out[Idx] = In[Idx]; -} +void foo(uint Idx : SV_DispatchThreadID) {} -// CHECK: define void @bar() -// CHECK: %[[ID_X:[0-9a-zA-Z]+]] = call i32 @llvm.dx.thread.id(i32 0) -// CHECK: %[[ID_X_:[0-9a-zA-Z]+]] = insertelement <2 x i32> poison, i32 %[[ID_X]], i64 0 -// CHECK: %[[ID_Y:[0-9a-zA-Z]+]] = call i32 @llvm.dx.thread.id(i32 1) -// CHECK: %[[ID_XY:[0-9a-zA-Z]+]] = insertelement <2 x i32> %[[ID_X_]], i32 %[[ID_Y]], i64 1 -// CHECK: call void @"?bar@@YAXT?$__vector@H$01@__clang@@@Z"(<2 x i32> %[[ID_XY]]) +// CHECK: define void @bar() +// CHECK-DXIL: %[[#ID_X:]] = call i32 @llvm.dx.thread.id(i32 0) +// CHECK-SPIRV: %[[#ID_X:]] = call i32 @llvm.spv.thread.id(i32 0) +// CHECK: %[[#ID_X_:]] = insertelement <2 x i32> poison, i32 %[[#ID_X]], i64 0 +// CHECK-DXIL: %[[#ID_Y:]] = call i32 @llvm.dx.thread.id(i32 1) +// CHECK-SPIRV: %[[#ID_Y:]] = call i32 @llvm.spv.thread.id(i32 1) +// CHECK: %[[#ID_XY:]] = insertelement <2 x i32> %[[#ID_X_]], i32 %[[#ID_Y]], i64 1 +// CHECK-DXIL: call void @{{.*}}bar{{.*}}(<2 x i32> %[[#ID_XY]]) [shader("compute")] [numthreads(8,8,1)] -void bar(uint2 Idx : SV_DispatchThreadID) { - Out[Idx.y] = In[Idx.x]; -} +void bar(uint2 Idx : SV_DispatchThreadID) {} diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td index d6eabc5d2407..766b3b542a9e 100644 --- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td +++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td @@ -41,6 +41,7 @@ let TargetPrefix = "spv" in { def int_spv_expect : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>]>; // The following intrinsic(s) are mirrored from IntrinsicsDirectX.td for HLSL support. + def int_spv_thread_id : Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem, IntrWillReturn]>; def int_spv_create_handle : ClangBuiltin<"__builtin_hlsl_create_handle">, Intrinsic<[ llvm_ptr_ty ], [llvm_i8_ty], [IntrWillReturn]>; } diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp index 74465c019131..e88298f52fbe 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -525,7 +525,9 @@ Register SPIRVGlobalRegistry::buildGlobalVariable( // Output decorations for the GV. // TODO: maybe move to GenerateDecorations pass. - if (IsConst) + const SPIRVSubtarget &ST = + cast(MIRBuilder.getMF().getSubtarget()); + if (IsConst && ST.isOpenCLEnv()) buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::Constant, {}); if (GVar && GVar->getAlign().valueOrOne().value() != 1) { diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index 3e01a6ac71f6..74df8de6eb90 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -194,6 +194,9 @@ private: bool selectLog10(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; + bool selectSpvThreadId(Register ResVReg, const SPIRVType *ResType, + MachineInstr &I) const; + bool selectUnmergeValues(MachineInstr &I) const; Register buildI32Constant(uint32_t Val, MachineInstr &I, @@ -301,6 +304,7 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg, case TargetOpcode::G_FREEZE: return selectFreeze(ResVReg, ResType, I); + case TargetOpcode::G_INTRINSIC: case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: return selectIntrinsic(ResVReg, ResType, I); @@ -1614,6 +1618,8 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, .addUse(I.getOperand(2).getReg()) .addUse(I.getOperand(3).getReg()); break; + case Intrinsic::spv_thread_id: + return selectSpvThreadId(ResVReg, ResType, I); default: llvm_unreachable("Intrinsic selection not implemented"); } @@ -1864,6 +1870,68 @@ bool SPIRVInstructionSelector::selectLog10(Register ResVReg, return Result; } +bool SPIRVInstructionSelector::selectSpvThreadId(Register ResVReg, + const SPIRVType *ResType, + MachineInstr &I) const { + // DX intrinsic: @llvm.dx.thread.id(i32) + // ID Name Description + // 93 ThreadId reads the thread ID + + MachineIRBuilder MIRBuilder(I); + const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); + const SPIRVType *Vec3Ty = + GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder); + const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType( + Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); + + // Create new register for GlobalInvocationID builtin variable. + Register NewRegister = + MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::IDRegClass); + MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 32)); + GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); + + // Build GlobalInvocationID global variable with the necessary decorations. + Register Variable = GR.buildGlobalVariable( + NewRegister, PtrType, + getLinkStringForBuiltIn(SPIRV::BuiltIn::GlobalInvocationId), nullptr, + SPIRV::StorageClass::Input, nullptr, true, true, + SPIRV::LinkageType::Import, MIRBuilder, false); + + // Create new register for loading value. + MachineRegisterInfo *MRI = MIRBuilder.getMRI(); + Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::IDRegClass); + MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 32)); + GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF()); + + // Load v3uint value from the global variable. + BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) + .addDef(LoadedRegister) + .addUse(GR.getSPIRVTypeID(Vec3Ty)) + .addUse(Variable); + + // Get Thread ID index. Expecting operand is a constant immediate value, + // wrapped in a type assignment. + assert(I.getOperand(2).isReg()); + Register ThreadIdReg = I.getOperand(2).getReg(); + SPIRVType *ConstTy = this->MRI->getVRegDef(ThreadIdReg); + assert(ConstTy && ConstTy->getOpcode() == SPIRV::ASSIGN_TYPE && + ConstTy->getOperand(1).isReg()); + Register ConstReg = ConstTy->getOperand(1).getReg(); + const MachineInstr *Const = this->MRI->getVRegDef(ConstReg); + assert(Const && Const->getOpcode() == TargetOpcode::G_CONSTANT); + const llvm::APInt &Val = Const->getOperand(1).getCImm()->getValue(); + const uint32_t ThreadId = Val.getZExtValue(); + + // Extract the thread ID from the loaded vector value. + MachineBasicBlock &BB = *I.getParent(); + auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) + .addDef(ResVReg) + .addUse(GR.getSPIRVTypeID(ResType)) + .addUse(LoadedRegister) + .addImm(ThreadId); + return MIB.constrainAllUses(TII, TRI, RBI); +} + namespace llvm { InstructionSelector * createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/SV_DispatchThreadID.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/SV_DispatchThreadID.ll new file mode 100644 index 000000000000..e93271b703f7 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/SV_DispatchThreadID.ll @@ -0,0 +1,76 @@ +; RUN: llc -O0 -mtriple=spirv-vulkan-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-unknown %s -o - -filetype=obj | spirv-val %} + +; This file generated from the following command: +; clang -cc1 -triple spirv-vulkan-library -x hlsl -emit-llvm -disable-llvm-passes -finclude-default-header - -o - < noundef %ID) #0 { +entry: + %ID.addr = alloca <3 x i32>, align 16 + store <3 x i32> %ID, ptr %ID.addr, align 16 + ret void +} + +; Function Attrs: norecurse +define void @main.1() #1 { +entry: + +; CHECK: %[[#load:]] = OpLoad %[[#v3int]] %[[#GlobalInvocationId]] +; CHECK: %[[#load0:]] = OpCompositeExtract %[[#int]] %[[#load]] 0 + %0 = call i32 @llvm.spv.thread.id(i32 0) + +; CHECK: %[[#tempvar:]] = OpCompositeInsert %[[#v3int]] %[[#load0]] %[[#tempvar]] 0 + %1 = insertelement <3 x i32> poison, i32 %0, i64 0 + +; CHECK: %[[#load:]] = OpLoad %[[#v3int]] %[[#GlobalInvocationId]] +; CHECK: %[[#load1:]] = OpCompositeExtract %[[#int]] %[[#load]] 1 + %2 = call i32 @llvm.spv.thread.id(i32 1) + +; CHECK: %[[#tempvar:]] = OpCompositeInsert %[[#v3int]] %[[#load1]] %[[#tempvar]] 1 + %3 = insertelement <3 x i32> %1, i32 %2, i64 1 + +; CHECK: %[[#load:]] = OpLoad %[[#v3int]] %[[#GlobalInvocationId]] +; CHECK: %[[#load2:]] = OpCompositeExtract %[[#int]] %[[#load]] 2 + %4 = call i32 @llvm.spv.thread.id(i32 2) + +; CHECK: %[[#tempvar:]] = OpCompositeInsert %[[#v3int]] %[[#load2]] %[[#tempvar]] 2 + %5 = insertelement <3 x i32> %3, i32 %4, i64 2 + + call void @main(<3 x i32> %5) + ret void +} + +; Function Attrs: nounwind willreturn memory(none) +declare i32 @llvm.spv.thread.id(i32) #2 + +attributes #0 = { noinline norecurse nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { norecurse "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #2 = { nounwind willreturn memory(none) } + +!llvm.module.flags = !{!0, !1} +!llvm.ident = !{!2} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{i32 4, !"dx.disable_optimizations", i32 1} +!2 = !{!"clang version 19.0.0git (git@github.com:llvm/llvm-project.git 91600507765679e92434ec7c5edb883bf01f847f)"} -- GitLab From df9ba13579e298fcb57aa59c5c187ce6729881d6 Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Mon, 4 Mar 2024 13:49:35 -0800 Subject: [PATCH 051/929] [LV] Handle scalable VFs in optimizeForVFAndUF (#82669) Given a scalable VF of the form , this patch adds the ability to discharge a backedge test for a loop whose trip count is between (NumElts, MinVScale*NumElts). A couple of notes on this: * Annoyingly, I could not figure out to write a test for this case. My attempt is checked in as test32_i8 in f67ef1a, but LV uses a fixed vector in that case, and ignored the force flags. * This depends on 9eb5f94f to avoid appearing like a regression. Since SCEV doesn't know any upper bound on vscale without the vscale_range attribute (it doesn't query TTI), the ranges overflow on the multiply. Arguably, this is fixing a bug in the current LV code since in theory vscale can be large enough to overflow for real, but no actual target is going to see that case. --- llvm/include/llvm/Analysis/ScalarEvolution.h | 1 + llvm/lib/Analysis/ScalarEvolution.cpp | 7 +++++++ llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 4 ++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h index 0880f9c65aa4..5828cc156cc7 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolution.h +++ b/llvm/include/llvm/Analysis/ScalarEvolution.h @@ -570,6 +570,7 @@ public: const SCEV *getPtrToIntExpr(const SCEV *Op, Type *Ty); const SCEV *getTruncateExpr(const SCEV *Op, Type *Ty, unsigned Depth = 0); const SCEV *getVScale(Type *Ty); + const SCEV *getElementCount(Type *Ty, ElementCount EC); const SCEV *getZeroExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth = 0); const SCEV *getZeroExtendExprImpl(const SCEV *Op, Type *Ty, unsigned Depth = 0); diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 4b2db80bc1ec..e1e6742e50ef 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -509,6 +509,13 @@ const SCEV *ScalarEvolution::getVScale(Type *Ty) { return S; } +const SCEV *ScalarEvolution::getElementCount(Type *Ty, ElementCount EC) { + const SCEV *Res = getConstant(Ty, EC.getKnownMinValue()); + if (EC.isScalable()) + Res = getMulExpr(Res, getVScale(Ty)); + return Res; +} + SCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy, const SCEV *op, Type *ty) : SCEV(ID, SCEVTy, computeExpressionSize(op)), Op(op), Ty(ty) {} diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 9d6deb802e20..f6b564ad931c 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -618,8 +618,8 @@ void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF, Plan.getCanonicalIV()->getStartValue()->getLiveInIRValue()->getType(); const SCEV *TripCount = createTripCountSCEV(IdxTy, PSE); ScalarEvolution &SE = *PSE.getSE(); - const SCEV *C = - SE.getConstant(TripCount->getType(), BestVF.getKnownMinValue() * BestUF); + ElementCount NumElements = BestVF.multiplyCoefficientBy(BestUF); + const SCEV *C = SE.getElementCount(TripCount->getType(), NumElements); if (TripCount->isZero() || !SE.isKnownPredicate(CmpInst::ICMP_ULE, TripCount, C)) return; -- GitLab From eaa0d3b1336b82df01ff50dfbe50bb8cdd7ada87 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Mon, 4 Mar 2024 13:57:13 -0800 Subject: [PATCH 052/929] [libc][test][stdbit] fix has_single_bit test names (#83904) This was copy+pasted from count_ones without updating the test name completely. --- libc/test/src/stdbit/stdc_has_single_bit_uc_test.cpp | 4 ++-- libc/test/src/stdbit/stdc_has_single_bit_ui_test.cpp | 4 ++-- libc/test/src/stdbit/stdc_has_single_bit_ul_test.cpp | 4 ++-- libc/test/src/stdbit/stdc_has_single_bit_ull_test.cpp | 4 ++-- libc/test/src/stdbit/stdc_has_single_bit_us_test.cpp | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/libc/test/src/stdbit/stdc_has_single_bit_uc_test.cpp b/libc/test/src/stdbit/stdc_has_single_bit_uc_test.cpp index 6212b1ec765a..1bc189cf0b66 100644 --- a/libc/test/src/stdbit/stdc_has_single_bit_uc_test.cpp +++ b/libc/test/src/stdbit/stdc_has_single_bit_uc_test.cpp @@ -10,11 +10,11 @@ #include "src/stdbit/stdc_has_single_bit_uc.h" #include "test/UnitTest/Test.h" -TEST(LlvmLibcStdcCountOnesUcTest, Zero) { +TEST(LlvmLibcStdcHasSingleBitUcTest, Zero) { EXPECT_EQ(LIBC_NAMESPACE::stdc_has_single_bit_uc(0U), false); } -TEST(LlvmLibcStdcCountOnesUcTest, OneHot) { +TEST(LlvmLibcStdcHasSingleBitUcTest, OneHot) { for (unsigned i = 0U; i != UCHAR_WIDTH; ++i) EXPECT_EQ(LIBC_NAMESPACE::stdc_has_single_bit_uc(1U << i), true); } diff --git a/libc/test/src/stdbit/stdc_has_single_bit_ui_test.cpp b/libc/test/src/stdbit/stdc_has_single_bit_ui_test.cpp index 2e00507aa025..c0b6abcf8fdc 100644 --- a/libc/test/src/stdbit/stdc_has_single_bit_ui_test.cpp +++ b/libc/test/src/stdbit/stdc_has_single_bit_ui_test.cpp @@ -10,11 +10,11 @@ #include "src/stdbit/stdc_has_single_bit_ui.h" #include "test/UnitTest/Test.h" -TEST(LlvmLibcStdcCountOnesUiTest, Zero) { +TEST(LlvmLibcStdcHasSingleBitUiTest, Zero) { EXPECT_EQ(LIBC_NAMESPACE::stdc_has_single_bit_ui(0U), false); } -TEST(LlvmLibcStdcCountOnesUiTest, OneHot) { +TEST(LlvmLibcStdcHasSingleBitUiTest, OneHot) { for (unsigned i = 0U; i != UINT_WIDTH; ++i) EXPECT_EQ(LIBC_NAMESPACE::stdc_has_single_bit_ui(1U << i), true); } diff --git a/libc/test/src/stdbit/stdc_has_single_bit_ul_test.cpp b/libc/test/src/stdbit/stdc_has_single_bit_ul_test.cpp index 8c0178998bbe..4c29fff748e9 100644 --- a/libc/test/src/stdbit/stdc_has_single_bit_ul_test.cpp +++ b/libc/test/src/stdbit/stdc_has_single_bit_ul_test.cpp @@ -10,11 +10,11 @@ #include "src/stdbit/stdc_has_single_bit_ul.h" #include "test/UnitTest/Test.h" -TEST(LlvmLibcStdcCountOnesUlTest, Zero) { +TEST(LlvmLibcStdcHasSingleBitUlTest, Zero) { EXPECT_EQ(LIBC_NAMESPACE::stdc_has_single_bit_ul(0U), false); } -TEST(LlvmLibcStdcCountOnesUlTest, OneHot) { +TEST(LlvmLibcStdcHasSingleBitUlTest, OneHot) { for (unsigned i = 0U; i != ULONG_WIDTH; ++i) EXPECT_EQ(LIBC_NAMESPACE::stdc_has_single_bit_ul(1UL << i), true); } diff --git a/libc/test/src/stdbit/stdc_has_single_bit_ull_test.cpp b/libc/test/src/stdbit/stdc_has_single_bit_ull_test.cpp index 1d9f976b6d63..59716468cc70 100644 --- a/libc/test/src/stdbit/stdc_has_single_bit_ull_test.cpp +++ b/libc/test/src/stdbit/stdc_has_single_bit_ull_test.cpp @@ -10,11 +10,11 @@ #include "src/stdbit/stdc_has_single_bit_ull.h" #include "test/UnitTest/Test.h" -TEST(LlvmLibcStdcCountOnesUllTest, Zero) { +TEST(LlvmLibcStdcHasSingleBitUllTest, Zero) { EXPECT_EQ(LIBC_NAMESPACE::stdc_has_single_bit_ull(0U), false); } -TEST(LlvmLibcStdcCountOnesUllTest, OneHot) { +TEST(LlvmLibcStdcHasSingleBitUllTest, OneHot) { for (unsigned i = 0U; i != ULLONG_WIDTH; ++i) EXPECT_EQ(LIBC_NAMESPACE::stdc_has_single_bit_ull(1ULL << i), true); } diff --git a/libc/test/src/stdbit/stdc_has_single_bit_us_test.cpp b/libc/test/src/stdbit/stdc_has_single_bit_us_test.cpp index 52c4de881044..a038f6fac012 100644 --- a/libc/test/src/stdbit/stdc_has_single_bit_us_test.cpp +++ b/libc/test/src/stdbit/stdc_has_single_bit_us_test.cpp @@ -10,11 +10,11 @@ #include "src/stdbit/stdc_has_single_bit_us.h" #include "test/UnitTest/Test.h" -TEST(LlvmLibcStdcCountOnesUsTest, Zero) { +TEST(LlvmLibcStdcHasSingleBitUsTest, Zero) { EXPECT_EQ(LIBC_NAMESPACE::stdc_has_single_bit_us(0U), false); } -TEST(LlvmLibcStdcCountOnesUsTest, OneHot) { +TEST(LlvmLibcStdcHasSingleBitUsTest, OneHot) { for (unsigned i = 0U; i != USHRT_WIDTH; ++i) EXPECT_EQ(LIBC_NAMESPACE::stdc_has_single_bit_us(1U << i), true); } -- GitLab From a5b797172cc902db166e9a695716fb81405f86e4 Mon Sep 17 00:00:00 2001 From: Mircea Trofin Date: Mon, 4 Mar 2024 14:11:30 -0800 Subject: [PATCH 053/929] Reapply "Update Benchmark (#83488)" (#83916) This reverts commit aec6a04b8e99b42eca431fc0b56947937d3a14c2. (google/benchmark still at hash 1576991177ba97a4b2ff6c45950f1fa6e9aa678c as it was in #83488. Also reapplied same extra local diffs) Verified locally. --- third-party/benchmark/.pre-commit-config.yaml | 18 + third-party/benchmark/.ycm_extra_conf.py | 8 +- third-party/benchmark/AUTHORS | 12 +- third-party/benchmark/CMakeLists.txt | 100 ++-- third-party/benchmark/CONTRIBUTORS | 14 +- third-party/benchmark/MODULE.bazel | 32 + third-party/benchmark/README.md | 13 +- third-party/benchmark/WORKSPACE | 54 +- third-party/benchmark/WORKSPACE.bzlmod | 2 + .../benchmark/bazel/benchmark_deps.bzl | 62 ++ .../benchmark/bindings/python/build_defs.bzl | 8 +- .../python/google_benchmark/__init__.py | 58 +- .../python/google_benchmark/benchmark.cc | 149 ++--- .../python/google_benchmark/example.py | 6 +- .../python/google_benchmark/version.py | 7 + .../benchmark/bindings/python/nanobind.BUILD | 59 ++ .../benchmark/bindings/python/pybind11.BUILD | 20 - .../bindings/python/python_headers.BUILD | 4 + .../bindings/python/requirements.txt | 2 - .../benchmark/cmake/CXXFeatureCheck.cmake | 29 +- .../benchmark/cmake/GetGitVersion.cmake | 30 +- third-party/benchmark/cmake/GoogleTest.cmake | 16 +- .../benchmark/cmake/Modules/FindPFM.cmake | 38 +- third-party/benchmark/cmake/benchmark.pc.in | 4 +- .../benchmark/cmake/pthread_affinity.cpp | 16 + third-party/benchmark/docs/AssemblyTests.md | 2 + third-party/benchmark/docs/_config.yml | 4 +- .../benchmark/docs/assets/images/icon.png | Bin 0 -> 11106 bytes .../benchmark/docs/assets/images/icon.xcf | Bin 0 -> 25934 bytes .../docs/assets/images/icon_black.png | Bin 0 -> 11559 bytes .../docs/assets/images/icon_black.xcf | Bin 0 -> 36322 bytes third-party/benchmark/docs/dependencies.md | 22 +- third-party/benchmark/docs/index.md | 4 +- third-party/benchmark/docs/perf_counters.md | 13 +- third-party/benchmark/docs/python_bindings.md | 34 ++ .../benchmark/docs/reducing_variance.md | 98 +++ third-party/benchmark/docs/releasing.md | 24 +- third-party/benchmark/docs/tools.md | 140 +++++ third-party/benchmark/docs/user_guide.md | 158 ++++- .../benchmark/include/benchmark/benchmark.h | 563 +++++++++++++----- .../benchmark/include/benchmark/export.h | 47 ++ third-party/benchmark/pyproject.toml | 85 +++ third-party/benchmark/requirements.txt | 3 - third-party/benchmark/setup.py | 193 +++--- third-party/benchmark/src/CMakeLists.txt | 28 +- third-party/benchmark/src/benchmark.cc | 289 +++++++-- .../benchmark/src/benchmark_api_internal.cc | 20 +- .../benchmark/src/benchmark_api_internal.h | 3 + third-party/benchmark/src/benchmark_main.cc | 1 + third-party/benchmark/src/benchmark_name.cc | 5 +- .../benchmark/src/benchmark_register.cc | 52 +- .../benchmark/src/benchmark_register.h | 5 +- third-party/benchmark/src/benchmark_runner.cc | 201 ++++++- third-party/benchmark/src/benchmark_runner.h | 33 +- third-party/benchmark/src/check.cc | 11 + third-party/benchmark/src/check.h | 31 +- third-party/benchmark/src/colorprint.cc | 46 +- third-party/benchmark/src/commandlineflags.cc | 19 +- third-party/benchmark/src/commandlineflags.h | 39 +- third-party/benchmark/src/complexity.cc | 43 +- third-party/benchmark/src/complexity.h | 2 +- third-party/benchmark/src/console_reporter.cc | 34 +- third-party/benchmark/src/counter.cc | 4 +- third-party/benchmark/src/csv_reporter.cc | 23 +- third-party/benchmark/src/cycleclock.h | 40 +- third-party/benchmark/src/internal_macros.h | 17 +- third-party/benchmark/src/json_reporter.cc | 36 +- third-party/benchmark/src/log.h | 26 +- third-party/benchmark/src/perf_counters.cc | 242 ++++++-- third-party/benchmark/src/perf_counters.h | 108 ++-- third-party/benchmark/src/re.h | 2 +- third-party/benchmark/src/reporter.cc | 14 +- third-party/benchmark/src/sleep.cc | 66 -- third-party/benchmark/src/sleep.h | 15 - third-party/benchmark/src/statistics.cc | 38 +- third-party/benchmark/src/statistics.h | 12 +- third-party/benchmark/src/string_util.cc | 65 +- third-party/benchmark/src/string_util.h | 11 +- third-party/benchmark/src/sysinfo.cc | 443 +++++++++----- third-party/benchmark/src/thread_manager.h | 4 +- third-party/benchmark/src/timers.cc | 24 +- .../benchmark/test/AssemblyTests.cmake | 21 + third-party/benchmark/test/CMakeLists.txt | 127 ++-- .../benchmark/test/args_product_test.cc | 4 +- third-party/benchmark/test/basic_test.cc | 9 +- third-party/benchmark/test/benchmark_gtest.cc | 14 +- .../benchmark_min_time_flag_iters_test.cc | 66 ++ .../test/benchmark_min_time_flag_time_test.cc | 90 +++ .../benchmark/test/benchmark_name_gtest.cc | 8 + .../benchmark_random_interleaving_gtest.cc | 5 +- .../test/benchmark_setup_teardown_test.cc | 18 +- third-party/benchmark/test/benchmark_test.cc | 65 +- .../test/clobber_memory_assembly_test.cc | 1 + third-party/benchmark/test/complexity_test.cc | 158 +++-- .../benchmark/test/diagnostics_test.cc | 15 +- .../test/donotoptimize_assembly_test.cc | 40 ++ .../benchmark/test/donotoptimize_test.cc | 28 +- third-party/benchmark/test/filter_test.cc | 31 +- third-party/benchmark/test/fixture_test.cc | 6 +- third-party/benchmark/test/link_main_test.cc | 3 +- third-party/benchmark/test/map_test.cc | 10 +- .../benchmark/test/memory_manager_test.cc | 11 +- .../benchmark/test/min_time_parse_gtest.cc | 30 + .../benchmark/test/multiple_ranges_test.cc | 4 +- third-party/benchmark/test/options_test.cc | 6 +- third-party/benchmark/test/output_test.h | 10 +- .../benchmark/test/output_test_helper.cc | 47 +- .../benchmark/test/perf_counters_gtest.cc | 266 +++++++-- .../benchmark/test/perf_counters_test.cc | 71 ++- .../benchmark/test/register_benchmark_test.cc | 26 +- .../benchmark/test/reporter_output_test.cc | 18 +- .../benchmark/test/skip_with_error_test.cc | 17 +- third-party/benchmark/test/spec_arg_test.cc | 16 +- .../benchmark/test/spec_arg_verbosity_test.cc | 43 ++ .../benchmark/test/statistics_gtest.cc | 4 +- .../benchmark/test/string_util_gtest.cc | 55 +- third-party/benchmark/test/time_unit_gtest.cc | 37 ++ .../test/user_counters_tabular_test.cc | 10 +- .../benchmark/test/user_counters_test.cc | 28 +- .../test/user_counters_thousands_test.cc | 32 +- third-party/benchmark/tools/compare.py | 62 +- .../tools/gbench/Inputs/test1_run1.json | 8 + .../tools/gbench/Inputs/test1_run2.json | 8 + .../tools/gbench/Inputs/test5_run0.json | 18 + .../tools/gbench/Inputs/test5_run1.json | 18 + .../benchmark/tools/gbench/__init__.py | 2 +- third-party/benchmark/tools/gbench/report.py | 271 ++++++++- third-party/benchmark/tools/gbench/util.py | 52 +- .../benchmark/tools/libpfm.BUILD.bazel | 22 + third-party/benchmark/tools/requirements.txt | 3 +- third-party/benchmark/tools/strip_asm.py | 73 ++- 131 files changed, 4568 insertions(+), 1622 deletions(-) create mode 100644 third-party/benchmark/.pre-commit-config.yaml create mode 100644 third-party/benchmark/MODULE.bazel create mode 100644 third-party/benchmark/WORKSPACE.bzlmod create mode 100644 third-party/benchmark/bazel/benchmark_deps.bzl create mode 100644 third-party/benchmark/bindings/python/google_benchmark/version.py create mode 100644 third-party/benchmark/bindings/python/nanobind.BUILD delete mode 100644 third-party/benchmark/bindings/python/pybind11.BUILD delete mode 100644 third-party/benchmark/bindings/python/requirements.txt create mode 100644 third-party/benchmark/cmake/pthread_affinity.cpp create mode 100644 third-party/benchmark/docs/assets/images/icon.png create mode 100644 third-party/benchmark/docs/assets/images/icon.xcf create mode 100644 third-party/benchmark/docs/assets/images/icon_black.png create mode 100644 third-party/benchmark/docs/assets/images/icon_black.xcf create mode 100644 third-party/benchmark/docs/python_bindings.md create mode 100644 third-party/benchmark/docs/reducing_variance.md create mode 100644 third-party/benchmark/include/benchmark/export.h create mode 100644 third-party/benchmark/pyproject.toml delete mode 100644 third-party/benchmark/requirements.txt create mode 100644 third-party/benchmark/src/check.cc delete mode 100644 third-party/benchmark/src/sleep.cc delete mode 100644 third-party/benchmark/src/sleep.h create mode 100644 third-party/benchmark/test/benchmark_min_time_flag_iters_test.cc create mode 100644 third-party/benchmark/test/benchmark_min_time_flag_time_test.cc create mode 100644 third-party/benchmark/test/min_time_parse_gtest.cc create mode 100644 third-party/benchmark/test/spec_arg_verbosity_test.cc create mode 100644 third-party/benchmark/test/time_unit_gtest.cc create mode 100644 third-party/benchmark/tools/gbench/Inputs/test5_run0.json create mode 100644 third-party/benchmark/tools/gbench/Inputs/test5_run1.json create mode 100644 third-party/benchmark/tools/libpfm.BUILD.bazel diff --git a/third-party/benchmark/.pre-commit-config.yaml b/third-party/benchmark/.pre-commit-config.yaml new file mode 100644 index 000000000000..0247d1b062b7 --- /dev/null +++ b/third-party/benchmark/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +repos: + - repo: https://github.com/keith/pre-commit-buildifier + rev: 6.4.0 + hooks: + - id: buildifier + - id: buildifier-lint + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.8.0 + hooks: + - id: mypy + types_or: [ python, pyi ] + args: [ "--ignore-missing-imports", "--scripts-are-modules" ] + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.13 + hooks: + - id: ruff + args: [ --fix, --exit-non-zero-on-fix ] + - id: ruff-format \ No newline at end of file diff --git a/third-party/benchmark/.ycm_extra_conf.py b/third-party/benchmark/.ycm_extra_conf.py index 1482c7b00202..caf257f0540e 100644 --- a/third-party/benchmark/.ycm_extra_conf.py +++ b/third-party/benchmark/.ycm_extra_conf.py @@ -1,4 +1,5 @@ import os + import ycm_core # These are the compilation flags that will be used in case there's no @@ -91,7 +92,9 @@ def GetCompilationInfoForFile(filename): for extension in SOURCE_EXTENSIONS: replacement_file = basename + extension if os.path.exists(replacement_file): - compilation_info = database.GetCompilationInfoForFile(replacement_file) + compilation_info = database.GetCompilationInfoForFile( + replacement_file + ) if compilation_info.compiler_flags_: return compilation_info return None @@ -107,7 +110,8 @@ def FlagsForFile(filename, **kwargs): return None final_flags = MakeRelativePathsInFlagsAbsolute( - compilation_info.compiler_flags_, compilation_info.compiler_working_dir_ + compilation_info.compiler_flags_, + compilation_info.compiler_working_dir_, ) else: relative_to = DirectoryOfThisScript() diff --git a/third-party/benchmark/AUTHORS b/third-party/benchmark/AUTHORS index 54770f35499c..2170e46fd4a0 100644 --- a/third-party/benchmark/AUTHORS +++ b/third-party/benchmark/AUTHORS @@ -13,6 +13,7 @@ Alex Steele Andriy Berestovskyy Arne Beer Carto +Cezary Skrzyński Christian Wassermann Christopher Seymour Colin Braley @@ -27,10 +28,13 @@ Eric Backus Eric Fiselier Eugene Zhuk Evgeny Safronov +Fabien Pichot Federico Ficarelli Felix Homann +Gergely Meszaros Gergő Szitár Google Inc. +Henrique Bucher International Business Machines Corporation Ismael Jimenez Martinez Jern-Kuan Leong @@ -41,8 +45,11 @@ Jussi Knuuttila Kaito Udagawa Kishan Kumar Lei Xu +Marcel Jacobse Matt Clarkson Maxim Vafin +Mike Apodaca +Min-Yih Hsu MongoDB Inc. Nick Hutchinson Norman Heino @@ -50,13 +57,16 @@ Oleksandr Sochka Ori Livneh Paul Redmond Radoslav Yovchev +Raghu Raja +Rainer Orth Roman Lebedev Sayan Bhattacharjee +Shapr3D Shuo Chen +Staffan Tjernstrom Steinar H. Gunderson Stripe, Inc. Tobias Schmidt Yixuan Qiu Yusuke Suzuki Zbigniew Skowron -Min-Yih Hsu diff --git a/third-party/benchmark/CMakeLists.txt b/third-party/benchmark/CMakeLists.txt index 8af49406d052..d9bcc6a4939b 100644 --- a/third-party/benchmark/CMakeLists.txt +++ b/third-party/benchmark/CMakeLists.txt @@ -1,19 +1,7 @@ -cmake_minimum_required (VERSION 3.5.1) - -foreach(p - CMP0048 # OK to clear PROJECT_VERSION on project() - CMP0054 # CMake 3.1 - CMP0056 # export EXE_LINKER_FLAGS to try_run - CMP0057 # Support no if() IN_LIST operator - CMP0063 # Honor visibility properties for all targets - CMP0077 # Allow option() overrides in importing projects - ) - if(POLICY ${p}) - cmake_policy(SET ${p} NEW) - endif() -endforeach() +# Require CMake 3.10. If available, use the policies up to CMake 3.22. +cmake_minimum_required (VERSION 3.10...3.22) -project (benchmark VERSION 1.6.0 LANGUAGES CXX) +project (benchmark VERSION 1.8.3 LANGUAGES CXX) option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." ON) option(BENCHMARK_ENABLE_EXCEPTIONS "Enable the use of exceptions in the benchmark library." ON) @@ -26,11 +14,14 @@ if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "PGI") # PGC++ maybe reporting false positives. set(BENCHMARK_ENABLE_WERROR OFF) endif() +if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "NVHPC") + set(BENCHMARK_ENABLE_WERROR OFF) +endif() if(BENCHMARK_FORCE_WERROR) set(BENCHMARK_ENABLE_WERROR ON) endif(BENCHMARK_FORCE_WERROR) -if(NOT MSVC) +if(NOT (MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC")) option(BENCHMARK_BUILD_32_BITS "Build a 32 bit version of the library." OFF) else() set(BENCHMARK_BUILD_32_BITS OFF CACHE BOOL "Build a 32 bit version of the library - unsupported when using MSVC)" FORCE) @@ -50,8 +41,11 @@ option(BENCHMARK_USE_BUNDLED_GTEST "Use bundled GoogleTest. If disabled, the fin option(BENCHMARK_ENABLE_LIBPFM "Enable performance counters provided by libpfm" OFF) -set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) -if(MSVC) +# Export only public symbols +set(CMAKE_CXX_VISIBILITY_PRESET hidden) +set(CMAKE_VISIBILITY_INLINES_HIDDEN ON) + +if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") # As of CMake 3.18, CMAKE_SYSTEM_PROCESSOR is not set properly for MSVC and # cross-compilation (e.g. Host=x86_64, target=aarch64) requires using the # undocumented, but working variable. @@ -72,7 +66,7 @@ function(should_enable_assembly_tests) return() endif() endif() - if (MSVC) + if (MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC") return() elseif(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64") return() @@ -111,22 +105,32 @@ get_git_version(GIT_VERSION) # If no git version can be determined, use the version # from the project() command if ("${GIT_VERSION}" STREQUAL "0.0.0") - set(VERSION "${benchmark_VERSION}") + set(VERSION "v${benchmark_VERSION}") else() set(VERSION "${GIT_VERSION}") endif() + +# Normalize version: drop "v" prefix, replace first "-" with ".", +# drop everything after second "-" (including said "-"). +string(STRIP ${VERSION} VERSION) +if(VERSION MATCHES v[^-]*-) + string(REGEX REPLACE "v([^-]*)-([0-9]+)-.*" "\\1.\\2" NORMALIZED_VERSION ${VERSION}) +else() + string(REGEX REPLACE "v(.*)" "\\1" NORMALIZED_VERSION ${VERSION}) +endif() + # Tell the user what versions we are using -message(STATUS "Version: ${VERSION}") +message(STATUS "Google Benchmark version: ${VERSION}, normalized to ${NORMALIZED_VERSION}") # The version of the libraries -set(GENERIC_LIB_VERSION ${VERSION}) -string(SUBSTRING ${VERSION} 0 1 GENERIC_LIB_SOVERSION) +set(GENERIC_LIB_VERSION ${NORMALIZED_VERSION}) +string(SUBSTRING ${NORMALIZED_VERSION} 0 1 GENERIC_LIB_SOVERSION) # Import our CMake modules -include(CheckCXXCompilerFlag) include(AddCXXCompilerFlag) -include(CXXFeatureCheck) +include(CheckCXXCompilerFlag) include(CheckLibraryExists) +include(CXXFeatureCheck) check_library_exists(rt shm_open "" HAVE_LIB_RT) @@ -134,6 +138,16 @@ if (BENCHMARK_BUILD_32_BITS) add_required_cxx_compiler_flag(-m32) endif() +if (MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC") + set(BENCHMARK_CXX_STANDARD 14) +else() + set(BENCHMARK_CXX_STANDARD 11) +endif() + +set(CMAKE_CXX_STANDARD ${BENCHMARK_CXX_STANDARD}) +set(CMAKE_CXX_STANDARD_REQUIRED YES) +set(CMAKE_CXX_EXTENSIONS OFF) + if (MSVC) # Turn compiler warnings up to 11 string(REGEX REPLACE "[-/]W[1-4]" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") @@ -166,21 +180,18 @@ if (MSVC) set(CMAKE_EXE_LINKER_FLAGS_MINSIZEREL "${CMAKE_EXE_LINKER_FLAGS_MINSIZEREL} /LTCG") endif() else() - # Try and enable C++11. Don't use C++14 because it doesn't work in some - # configurations. - add_cxx_compiler_flag(-std=c++11) - if (NOT HAVE_CXX_FLAG_STD_CXX11) - add_cxx_compiler_flag(-std=c++0x) - endif() - + # Turn on Large-file Support + add_definitions(-D_FILE_OFFSET_BITS=64) + add_definitions(-D_LARGEFILE64_SOURCE) + add_definitions(-D_LARGEFILE_SOURCE) # Turn compiler warnings up to 11 add_cxx_compiler_flag(-Wall) add_cxx_compiler_flag(-Wextra) add_cxx_compiler_flag(-Wshadow) + add_cxx_compiler_flag(-Wfloat-equal) + add_cxx_compiler_flag(-Wold-style-cast) if(BENCHMARK_ENABLE_WERROR) - add_cxx_compiler_flag(-Werror RELEASE) - add_cxx_compiler_flag(-Werror RELWITHDEBINFO) - add_cxx_compiler_flag(-Werror MINSIZEREL) + add_cxx_compiler_flag(-Werror) endif() if (NOT BENCHMARK_ENABLE_TESTING) # Disable warning when compiling tests as gtest does not use 'override'. @@ -193,24 +204,23 @@ else() # Disable warnings regarding deprecated parts of the library while building # and testing those parts of the library. add_cxx_compiler_flag(-Wno-deprecated-declarations) - if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel" OR CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") # Intel silently ignores '-Wno-deprecated-declarations', # warning no. 1786 must be explicitly disabled. # See #631 for rationale. add_cxx_compiler_flag(-wd1786) + add_cxx_compiler_flag(-fno-finite-math-only) endif() # Disable deprecation warnings for release builds (when -Werror is enabled). if(BENCHMARK_ENABLE_WERROR) - add_cxx_compiler_flag(-Wno-deprecated RELEASE) - add_cxx_compiler_flag(-Wno-deprecated RELWITHDEBINFO) - add_cxx_compiler_flag(-Wno-deprecated MINSIZEREL) + add_cxx_compiler_flag(-Wno-deprecated) endif() if (NOT BENCHMARK_ENABLE_EXCEPTIONS) add_cxx_compiler_flag(-fno-exceptions) endif() if (HAVE_CXX_FLAG_FSTRICT_ALIASING) - if (NOT CMAKE_CXX_COMPILER_ID STREQUAL "Intel") #ICC17u2: Many false positives for Wstrict-aliasing + if (NOT CMAKE_CXX_COMPILER_ID STREQUAL "Intel" AND NOT CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") #ICC17u2: Many false positives for Wstrict-aliasing add_cxx_compiler_flag(-Wstrict-aliasing) endif() endif() @@ -219,12 +229,12 @@ else() add_cxx_compiler_flag(-wd654) add_cxx_compiler_flag(-Wthread-safety) if (HAVE_CXX_FLAG_WTHREAD_SAFETY) - cxx_feature_check(THREAD_SAFETY_ATTRIBUTES) + cxx_feature_check(THREAD_SAFETY_ATTRIBUTES "-DINCLUDE_DIRECTORIES=${PROJECT_SOURCE_DIR}/include") endif() # On most UNIX like platforms g++ and clang++ define _GNU_SOURCE as a # predefined macro, which turns on all of the wonderful libc extensions. - # However g++ doesn't do this in Cygwin so we have to define it ourselfs + # However g++ doesn't do this in Cygwin so we have to define it ourselves # since we depend on GNU/POSIX/BSD extensions. if (CYGWIN) add_definitions(-D_GNU_SOURCE=1) @@ -275,7 +285,8 @@ if (BENCHMARK_USE_LIBCXX) if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") add_cxx_compiler_flag(-stdlib=libc++) elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" OR - "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") + "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel" OR + "${CMAKE_CXX_COMPILER_ID}" STREQUAL "IntelLLVM") add_cxx_compiler_flag(-nostdinc++) message(WARNING "libc++ header path must be manually specified using CMAKE_CXX_FLAGS") # Adding -nodefaultlibs directly to CMAKE__LINKER_FLAGS will break @@ -312,9 +323,10 @@ cxx_feature_check(STEADY_CLOCK) # Ensure we have pthreads set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) +cxx_feature_check(PTHREAD_AFFINITY) if (BENCHMARK_ENABLE_LIBPFM) - find_package(PFM) + find_package(PFM REQUIRED) endif() # Set up directories diff --git a/third-party/benchmark/CONTRIBUTORS b/third-party/benchmark/CONTRIBUTORS index 651fbeafe66a..9ca2caa3ee78 100644 --- a/third-party/benchmark/CONTRIBUTORS +++ b/third-party/benchmark/CONTRIBUTORS @@ -27,7 +27,9 @@ Albert Pretorius Alex Steele Andriy Berestovskyy Arne Beer +Bátor Tallér Billy Robert O'Neal III +Cezary Skrzyński Chris Kennelly Christian Wassermann Christopher Seymour @@ -44,25 +46,32 @@ Eric Backus Eric Fiselier Eugene Zhuk Evgeny Safronov +Fabien Pichot Fanbo Meng Federico Ficarelli Felix Homann Geoffrey Martin-Noble +Gergely Meszaros Gergő Szitár Hannes Hauswedell +Henrique Bucher Ismael Jimenez Martinez +Iakov Sergeev Jern-Kuan Leong JianXiong Zhou Joao Paulo Magalhaes John Millikin Jordan Williams Jussi Knuuttila -Kai Wolf Kaito Udagawa +Kai Wolf Kishan Kumar Lei Xu +Marcel Jacobse Matt Clarkson Maxim Vafin +Mike Apodaca +Min-Yih Hsu Nick Hutchinson Norman Heino Oleksandr Sochka @@ -71,6 +80,8 @@ Pascal Leroy Paul Redmond Pierre Phaneuf Radoslav Yovchev +Raghu Raja +Rainer Orth Raul Marin Ray Glover Robert Guo @@ -84,4 +95,3 @@ Tom Madams Yixuan Qiu Yusuke Suzuki Zbigniew Skowron -Min-Yih Hsu diff --git a/third-party/benchmark/MODULE.bazel b/third-party/benchmark/MODULE.bazel new file mode 100644 index 000000000000..7e0e0161235b --- /dev/null +++ b/third-party/benchmark/MODULE.bazel @@ -0,0 +1,32 @@ +module( + name = "google_benchmark", + version = "1.8.3", +) + +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "platforms", version = "0.0.7") +bazel_dep(name = "rules_foreign_cc", version = "0.10.1") +bazel_dep(name = "rules_cc", version = "0.0.9") + +bazel_dep(name = "rules_python", version = "0.27.1", dev_dependency = True) +bazel_dep(name = "googletest", version = "1.12.1", dev_dependency = True, repo_name = "com_google_googletest") + +bazel_dep(name = "libpfm", version = "4.11.0") + +# Register a toolchain for Python 3.9 to be able to build numpy. Python +# versions >=3.10 are problematic. +# A second reason for this is to be able to build Python hermetically instead +# of relying on the changing default version from rules_python. + +python = use_extension("@rules_python//python/extensions:python.bzl", "python", dev_dependency = True) +python.toolchain(python_version = "3.9") + +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip", dev_dependency = True) +pip.parse( + hub_name = "tools_pip_deps", + python_version = "3.9", + requirements_lock = "//tools:requirements.txt", +) +use_repo(pip, "tools_pip_deps") + +# -- bazel_dep definitions -- # diff --git a/third-party/benchmark/README.md b/third-party/benchmark/README.md index 7b81d960fc1d..a5e5d392d826 100644 --- a/third-party/benchmark/README.md +++ b/third-party/benchmark/README.md @@ -4,10 +4,9 @@ [![bazel](https://github.com/google/benchmark/actions/workflows/bazel.yml/badge.svg)](https://github.com/google/benchmark/actions/workflows/bazel.yml) [![pylint](https://github.com/google/benchmark/workflows/pylint/badge.svg)](https://github.com/google/benchmark/actions?query=workflow%3Apylint) [![test-bindings](https://github.com/google/benchmark/workflows/test-bindings/badge.svg)](https://github.com/google/benchmark/actions?query=workflow%3Atest-bindings) - -[![Build Status](https://travis-ci.org/google/benchmark.svg?branch=master)](https://travis-ci.org/google/benchmark) [![Coverage Status](https://coveralls.io/repos/google/benchmark/badge.svg)](https://coveralls.io/r/google/benchmark) +[![Discord](https://discordapp.com/api/guilds/1125694995928719494/widget.png?style=shield)](https://discord.gg/cz7UX7wKC2) A library to benchmark code snippets, similar to unit tests. Example: @@ -33,7 +32,7 @@ To get started, see [Requirements](#requirements) and [Installation](#installation). See [Usage](#usage) for a full example and the [User Guide](docs/user_guide.md) for a more comprehensive feature overview. -It may also help to read the [Google Test documentation](https://github.com/google/googletest/blob/master/docs/primer.md) +It may also help to read the [Google Test documentation](https://github.com/google/googletest/blob/main/docs/primer.md) as some of the structural aspects of the APIs are similar. ## Resources @@ -47,6 +46,8 @@ IRC channels: [Assembly Testing Documentation](docs/AssemblyTests.md) +[Building and installing Python bindings](docs/python_bindings.md) + ## Requirements The library can be used with C++03. However, it requires C++11 to build, @@ -137,6 +138,12 @@ cache variables, if autodetection fails. If you are using clang, you may need to set `LLVMAR_EXECUTABLE`, `LLVMNM_EXECUTABLE` and `LLVMRANLIB_EXECUTABLE` cmake cache variables. +To enable sanitizer checks (eg., `asan` and `tsan`), add: +``` + -DCMAKE_C_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=address -fsanitize=thread -fno-sanitize-recover=all" + -DCMAKE_CXX_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=address -fsanitize=thread -fno-sanitize-recover=all " +``` + ### Stable and Experimental Library Versions The main branch contains the latest stable version of the benchmarking library; diff --git a/third-party/benchmark/WORKSPACE b/third-party/benchmark/WORKSPACE index 949eb98bc5d9..256207022597 100644 --- a/third-party/benchmark/WORKSPACE +++ b/third-party/benchmark/WORKSPACE @@ -1,44 +1,30 @@ workspace(name = "com_github_google_benchmark") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") - -http_archive( - name = "com_google_absl", - sha256 = "f41868f7a938605c92936230081175d1eae87f6ea2c248f41077c8f88316f111", - strip_prefix = "abseil-cpp-20200225.2", - urls = ["https://github.com/abseil/abseil-cpp/archive/20200225.2.tar.gz"], -) +load("//:bazel/benchmark_deps.bzl", "benchmark_deps") -git_repository( - name = "com_google_googletest", - remote = "https://github.com/google/googletest.git", - tag = "release-1.11.0", -) +benchmark_deps() -http_archive( - name = "pybind11", - build_file = "@//bindings/python:pybind11.BUILD", - sha256 = "1eed57bc6863190e35637290f97a20c81cfe4d9090ac0a24f3bbf08f265eb71d", - strip_prefix = "pybind11-2.4.3", - urls = ["https://github.com/pybind/pybind11/archive/v2.4.3.tar.gz"], -) +load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies") -new_local_repository( - name = "python_headers", - build_file = "@//bindings/python:python_headers.BUILD", - path = "/usr/include/python3.6", # May be overwritten by setup.py. -) +rules_foreign_cc_dependencies() -http_archive( - name = "rules_python", - url = "https://github.com/bazelbuild/rules_python/releases/download/0.1.0/rules_python-0.1.0.tar.gz", - sha256 = "b6d46438523a3ec0f3cead544190ee13223a52f6a6765a29eae7b7cc24cc83a0", +load("@rules_python//python:repositories.bzl", "py_repositories") + +py_repositories() + +load("@rules_python//python:pip.bzl", "pip_parse") + +pip_parse( + name = "tools_pip_deps", + requirements_lock = "//tools:requirements.txt", ) -load("@rules_python//python:pip.bzl", pip3_install="pip_install") +load("@tools_pip_deps//:requirements.bzl", "install_deps") + +install_deps() -pip3_install( - name = "py_deps", - requirements = "//:requirements.txt", +new_local_repository( + name = "python_headers", + build_file = "@//bindings/python:python_headers.BUILD", + path = "", # May be overwritten by setup.py. ) diff --git a/third-party/benchmark/WORKSPACE.bzlmod b/third-party/benchmark/WORKSPACE.bzlmod new file mode 100644 index 000000000000..9526376d77c4 --- /dev/null +++ b/third-party/benchmark/WORKSPACE.bzlmod @@ -0,0 +1,2 @@ +# This file marks the root of the Bazel workspace. +# See MODULE.bazel for dependencies and setup. diff --git a/third-party/benchmark/bazel/benchmark_deps.bzl b/third-party/benchmark/bazel/benchmark_deps.bzl new file mode 100644 index 000000000000..4fb45a538d4a --- /dev/null +++ b/third-party/benchmark/bazel/benchmark_deps.bzl @@ -0,0 +1,62 @@ +""" +This file contains the Bazel build dependencies for Google Benchmark (both C++ source and Python bindings). +""" + +load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +def benchmark_deps(): + """Loads dependencies required to build Google Benchmark.""" + + if "bazel_skylib" not in native.existing_rules(): + http_archive( + name = "bazel_skylib", + sha256 = "cd55a062e763b9349921f0f5db8c3933288dc8ba4f76dd9416aac68acee3cb94", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz", + ], + ) + + if "rules_foreign_cc" not in native.existing_rules(): + http_archive( + name = "rules_foreign_cc", + sha256 = "476303bd0f1b04cc311fc258f1708a5f6ef82d3091e53fd1977fa20383425a6a", + strip_prefix = "rules_foreign_cc-0.10.1", + url = "https://github.com/bazelbuild/rules_foreign_cc/releases/download/0.10.1/rules_foreign_cc-0.10.1.tar.gz", + ) + + if "rules_python" not in native.existing_rules(): + http_archive( + name = "rules_python", + sha256 = "e85ae30de33625a63eca7fc40a94fea845e641888e52f32b6beea91e8b1b2793", + strip_prefix = "rules_python-0.27.1", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.27.1/rules_python-0.27.1.tar.gz", + ) + + if "com_google_googletest" not in native.existing_rules(): + new_git_repository( + name = "com_google_googletest", + remote = "https://github.com/google/googletest.git", + tag = "release-1.12.1", + ) + + if "nanobind" not in native.existing_rules(): + new_git_repository( + name = "nanobind", + remote = "https://github.com/wjakob/nanobind.git", + tag = "v1.8.0", + build_file = "@//bindings/python:nanobind.BUILD", + recursive_init_submodules = True, + ) + + if "libpfm" not in native.existing_rules(): + # Downloaded from v4.9.0 tag at https://sourceforge.net/p/perfmon2/libpfm4/ref/master/tags/ + http_archive( + name = "libpfm", + build_file = str(Label("//tools:libpfm.BUILD.bazel")), + sha256 = "5da5f8872bde14b3634c9688d980f68bda28b510268723cc12973eedbab9fecc", + type = "tar.gz", + strip_prefix = "libpfm-4.11.0", + urls = ["https://sourceforge.net/projects/perfmon2/files/libpfm4/libpfm-4.11.0.tar.gz/download"], + ) diff --git a/third-party/benchmark/bindings/python/build_defs.bzl b/third-party/benchmark/bindings/python/build_defs.bzl index 45907aaa5e2d..b0c1b0f5807e 100644 --- a/third-party/benchmark/bindings/python/build_defs.bzl +++ b/third-party/benchmark/bindings/python/build_defs.bzl @@ -1,3 +1,7 @@ +""" +This file contains some build definitions for C++ extensions used in the Google Benchmark Python bindings. +""" + _SHARED_LIB_SUFFIX = { "//conditions:default": ".so", "//:windows": ".dll", @@ -8,8 +12,8 @@ def py_extension(name, srcs, hdrs = [], copts = [], features = [], deps = []): shared_lib_name = name + shared_lib_suffix native.cc_binary( name = shared_lib_name, - linkshared = 1, - linkstatic = 1, + linkshared = True, + linkstatic = True, srcs = srcs + hdrs, copts = copts, features = features, diff --git a/third-party/benchmark/bindings/python/google_benchmark/__init__.py b/third-party/benchmark/bindings/python/google_benchmark/__init__.py index 1055bf241856..e14769f451fe 100644 --- a/third-party/benchmark/bindings/python/google_benchmark/__init__.py +++ b/third-party/benchmark/bindings/python/google_benchmark/__init__.py @@ -26,47 +26,29 @@ Example usage: if __name__ == '__main__': benchmark.main() """ +import atexit from absl import app + from google_benchmark import _benchmark from google_benchmark._benchmark import ( - Counter, - kNanosecond, - kMicrosecond, - kMillisecond, - kSecond, - oNone, - o1, - oN, - oNSquared, - oNCubed, - oLogN, - oNLogN, - oAuto, - oLambda, + Counter as Counter, + State as State, + kMicrosecond as kMicrosecond, + kMillisecond as kMillisecond, + kNanosecond as kNanosecond, + kSecond as kSecond, + o1 as o1, + oAuto as oAuto, + oLambda as oLambda, + oLogN as oLogN, + oN as oN, + oNCubed as oNCubed, + oNLogN as oNLogN, + oNone as oNone, + oNSquared as oNSquared, ) - - -__all__ = [ - "register", - "main", - "Counter", - "kNanosecond", - "kMicrosecond", - "kMillisecond", - "kSecond", - "oNone", - "o1", - "oN", - "oNSquared", - "oNCubed", - "oLogN", - "oNLogN", - "oAuto", - "oLambda", -] - -__version__ = "0.2.0" +from google_benchmark.version import __version__ as __version__ class __OptionMaker: @@ -94,14 +76,13 @@ class __OptionMaker: # The function that get returned on @option.range(start=0, limit=1<<5). def __builder_method(*args, **kwargs): - # The decorator that get called, either with the benchmared function # or the previous Options def __decorator(func_or_options): options = self.make(func_or_options) options.builder_calls.append((builder_name, args, kwargs)) # The decorator returns Options so it is not technically a decorator - # and needs a final call to @regiser + # and needs a final call to @register return options return __decorator @@ -156,3 +137,4 @@ def main(argv=None): # Methods for use with custom main function. initialize = _benchmark.Initialize run_benchmarks = _benchmark.RunSpecifiedBenchmarks +atexit.register(_benchmark.ClearRegisteredBenchmarks) diff --git a/third-party/benchmark/bindings/python/google_benchmark/benchmark.cc b/third-party/benchmark/bindings/python/google_benchmark/benchmark.cc index 02b6ed7ed590..f44476901cae 100644 --- a/third-party/benchmark/bindings/python/google_benchmark/benchmark.cc +++ b/third-party/benchmark/bindings/python/google_benchmark/benchmark.cc @@ -1,20 +1,17 @@ // Benchmark for Python. -#include -#include -#include - -#include "pybind11/operators.h" -#include "pybind11/pybind11.h" -#include "pybind11/stl.h" -#include "pybind11/stl_bind.h" - #include "benchmark/benchmark.h" -PYBIND11_MAKE_OPAQUE(benchmark::UserCounters); +#include "nanobind/nanobind.h" +#include "nanobind/operators.h" +#include "nanobind/stl/bind_map.h" +#include "nanobind/stl/string.h" +#include "nanobind/stl/vector.h" + +NB_MAKE_OPAQUE(benchmark::UserCounters); namespace { -namespace py = ::pybind11; +namespace nb = nanobind; std::vector Initialize(const std::vector& argv) { // The `argv` pointers here become invalid when this function returns, but @@ -37,15 +34,16 @@ std::vector Initialize(const std::vector& argv) { return remaining_argv; } -benchmark::internal::Benchmark* RegisterBenchmark(const char* name, - py::function f) { +benchmark::internal::Benchmark* RegisterBenchmark(const std::string& name, + nb::callable f) { return benchmark::RegisterBenchmark( name, [f](benchmark::State& state) { f(&state); }); } -PYBIND11_MODULE(_benchmark, m) { +NB_MODULE(_benchmark, m) { + using benchmark::TimeUnit; - py::enum_(m, "TimeUnit") + nb::enum_(m, "TimeUnit") .value("kNanosecond", TimeUnit::kNanosecond) .value("kMicrosecond", TimeUnit::kMicrosecond) .value("kMillisecond", TimeUnit::kMillisecond) @@ -53,72 +51,74 @@ PYBIND11_MODULE(_benchmark, m) { .export_values(); using benchmark::BigO; - py::enum_(m, "BigO") + nb::enum_(m, "BigO") .value("oNone", BigO::oNone) .value("o1", BigO::o1) .value("oN", BigO::oN) .value("oNSquared", BigO::oNSquared) .value("oNCubed", BigO::oNCubed) .value("oLogN", BigO::oLogN) - .value("oNLogN", BigO::oLogN) + .value("oNLogN", BigO::oNLogN) .value("oAuto", BigO::oAuto) .value("oLambda", BigO::oLambda) .export_values(); using benchmark::internal::Benchmark; - py::class_(m, "Benchmark") - // For methods returning a pointer tor the current object, reference - // return policy is used to ask pybind not to take ownership oof the + nb::class_(m, "Benchmark") + // For methods returning a pointer to the current object, reference + // return policy is used to ask nanobind not to take ownership of the // returned object and avoid calling delete on it. // https://pybind11.readthedocs.io/en/stable/advanced/functions.html#return-value-policies // // For methods taking a const std::vector<...>&, a copy is created // because a it is bound to a Python list. // https://pybind11.readthedocs.io/en/stable/advanced/cast/stl.html - .def("unit", &Benchmark::Unit, py::return_value_policy::reference) - .def("arg", &Benchmark::Arg, py::return_value_policy::reference) - .def("args", &Benchmark::Args, py::return_value_policy::reference) - .def("range", &Benchmark::Range, py::return_value_policy::reference, - py::arg("start"), py::arg("limit")) + .def("unit", &Benchmark::Unit, nb::rv_policy::reference) + .def("arg", &Benchmark::Arg, nb::rv_policy::reference) + .def("args", &Benchmark::Args, nb::rv_policy::reference) + .def("range", &Benchmark::Range, nb::rv_policy::reference, + nb::arg("start"), nb::arg("limit")) .def("dense_range", &Benchmark::DenseRange, - py::return_value_policy::reference, py::arg("start"), - py::arg("limit"), py::arg("step") = 1) - .def("ranges", &Benchmark::Ranges, py::return_value_policy::reference) + nb::rv_policy::reference, nb::arg("start"), + nb::arg("limit"), nb::arg("step") = 1) + .def("ranges", &Benchmark::Ranges, nb::rv_policy::reference) .def("args_product", &Benchmark::ArgsProduct, - py::return_value_policy::reference) - .def("arg_name", &Benchmark::ArgName, py::return_value_policy::reference) + nb::rv_policy::reference) + .def("arg_name", &Benchmark::ArgName, nb::rv_policy::reference) .def("arg_names", &Benchmark::ArgNames, - py::return_value_policy::reference) + nb::rv_policy::reference) .def("range_pair", &Benchmark::RangePair, - py::return_value_policy::reference, py::arg("lo1"), py::arg("hi1"), - py::arg("lo2"), py::arg("hi2")) + nb::rv_policy::reference, nb::arg("lo1"), nb::arg("hi1"), + nb::arg("lo2"), nb::arg("hi2")) .def("range_multiplier", &Benchmark::RangeMultiplier, - py::return_value_policy::reference) - .def("min_time", &Benchmark::MinTime, py::return_value_policy::reference) + nb::rv_policy::reference) + .def("min_time", &Benchmark::MinTime, nb::rv_policy::reference) + .def("min_warmup_time", &Benchmark::MinWarmUpTime, + nb::rv_policy::reference) .def("iterations", &Benchmark::Iterations, - py::return_value_policy::reference) + nb::rv_policy::reference) .def("repetitions", &Benchmark::Repetitions, - py::return_value_policy::reference) + nb::rv_policy::reference) .def("report_aggregates_only", &Benchmark::ReportAggregatesOnly, - py::return_value_policy::reference, py::arg("value") = true) + nb::rv_policy::reference, nb::arg("value") = true) .def("display_aggregates_only", &Benchmark::DisplayAggregatesOnly, - py::return_value_policy::reference, py::arg("value") = true) + nb::rv_policy::reference, nb::arg("value") = true) .def("measure_process_cpu_time", &Benchmark::MeasureProcessCPUTime, - py::return_value_policy::reference) + nb::rv_policy::reference) .def("use_real_time", &Benchmark::UseRealTime, - py::return_value_policy::reference) + nb::rv_policy::reference) .def("use_manual_time", &Benchmark::UseManualTime, - py::return_value_policy::reference) + nb::rv_policy::reference) .def( "complexity", (Benchmark * (Benchmark::*)(benchmark::BigO)) & Benchmark::Complexity, - py::return_value_policy::reference, - py::arg("complexity") = benchmark::oAuto); + nb::rv_policy::reference, + nb::arg("complexity") = benchmark::oAuto); using benchmark::Counter; - py::class_ py_counter(m, "Counter"); + nb::class_ py_counter(m, "Counter"); - py::enum_(py_counter, "Flags") + nb::enum_(py_counter, "Flags") .value("kDefaults", Counter::Flags::kDefaults) .value("kIsRate", Counter::Flags::kIsRate) .value("kAvgThreads", Counter::Flags::kAvgThreads) @@ -130,52 +130,55 @@ PYBIND11_MODULE(_benchmark, m) { .value("kAvgIterationsRate", Counter::Flags::kAvgIterationsRate) .value("kInvert", Counter::Flags::kInvert) .export_values() - .def(py::self | py::self); + .def(nb::self | nb::self); - py::enum_(py_counter, "OneK") + nb::enum_(py_counter, "OneK") .value("kIs1000", Counter::OneK::kIs1000) .value("kIs1024", Counter::OneK::kIs1024) .export_values(); py_counter - .def(py::init(), - py::arg("value") = 0., py::arg("flags") = Counter::kDefaults, - py::arg("k") = Counter::kIs1000) - .def(py::init([](double value) { return Counter(value); })) - .def_readwrite("value", &Counter::value) - .def_readwrite("flags", &Counter::flags) - .def_readwrite("oneK", &Counter::oneK); - py::implicitly_convertible(); - py::implicitly_convertible(); - - py::bind_map(m, "UserCounters"); + .def(nb::init(), + nb::arg("value") = 0., nb::arg("flags") = Counter::kDefaults, + nb::arg("k") = Counter::kIs1000) + .def("__init__", ([](Counter *c, double value) { new (c) Counter(value); })) + .def_rw("value", &Counter::value) + .def_rw("flags", &Counter::flags) + .def_rw("oneK", &Counter::oneK) + .def(nb::init_implicit()); + + nb::implicitly_convertible(); + + nb::bind_map(m, "UserCounters"); using benchmark::State; - py::class_(m, "State") + nb::class_(m, "State") .def("__bool__", &State::KeepRunning) - .def_property_readonly("keep_running", &State::KeepRunning) + .def_prop_ro("keep_running", &State::KeepRunning) .def("pause_timing", &State::PauseTiming) .def("resume_timing", &State::ResumeTiming) .def("skip_with_error", &State::SkipWithError) - .def_property_readonly("error_occurred", &State::error_occurred) + .def_prop_ro("error_occurred", &State::error_occurred) .def("set_iteration_time", &State::SetIterationTime) - .def_property("bytes_processed", &State::bytes_processed, + .def_prop_rw("bytes_processed", &State::bytes_processed, &State::SetBytesProcessed) - .def_property("complexity_n", &State::complexity_length_n, + .def_prop_rw("complexity_n", &State::complexity_length_n, &State::SetComplexityN) - .def_property("items_processed", &State::items_processed, - &State::SetItemsProcessed) - .def("set_label", (void(State::*)(const char*)) & State::SetLabel) - .def("range", &State::range, py::arg("pos") = 0) - .def_property_readonly("iterations", &State::iterations) - .def_readwrite("counters", &State::counters) - .def_property_readonly("thread_index", &State::thread_index) - .def_property_readonly("threads", &State::threads); + .def_prop_rw("items_processed", &State::items_processed, + &State::SetItemsProcessed) + .def("set_label", &State::SetLabel) + .def("range", &State::range, nb::arg("pos") = 0) + .def_prop_ro("iterations", &State::iterations) + .def_prop_ro("name", &State::name) + .def_rw("counters", &State::counters) + .def_prop_ro("thread_index", &State::thread_index) + .def_prop_ro("threads", &State::threads); m.def("Initialize", Initialize); m.def("RegisterBenchmark", RegisterBenchmark, - py::return_value_policy::reference); + nb::rv_policy::reference); m.def("RunSpecifiedBenchmarks", []() { benchmark::RunSpecifiedBenchmarks(); }); + m.def("ClearRegisteredBenchmarks", benchmark::ClearRegisteredBenchmarks); }; } // namespace diff --git a/third-party/benchmark/bindings/python/google_benchmark/example.py b/third-party/benchmark/bindings/python/google_benchmark/example.py index fb0234b8fd7e..b5b2f88ff306 100644 --- a/third-party/benchmark/bindings/python/google_benchmark/example.py +++ b/third-party/benchmark/bindings/python/google_benchmark/example.py @@ -73,7 +73,7 @@ def manual_timing(state): @benchmark.register def custom_counters(state): - """Collect cutom metric using benchmark.Counter.""" + """Collect custom metric using benchmark.Counter.""" num_foo = 0.0 while state: # Benchmark some code here @@ -86,7 +86,9 @@ def custom_counters(state): # Set a counter as a rate. state.counters["foo_rate"] = Counter(num_foo, Counter.kIsRate) # Set a counter as an inverse of rate. - state.counters["foo_inv_rate"] = Counter(num_foo, Counter.kIsRate | Counter.kInvert) + state.counters["foo_inv_rate"] = Counter( + num_foo, Counter.kIsRate | Counter.kInvert + ) # Set a counter as a thread-average quantity. state.counters["foo_avg"] = Counter(num_foo, Counter.kAvgThreads) # There's also a combined flag: diff --git a/third-party/benchmark/bindings/python/google_benchmark/version.py b/third-party/benchmark/bindings/python/google_benchmark/version.py new file mode 100644 index 000000000000..a324693e2d7b --- /dev/null +++ b/third-party/benchmark/bindings/python/google_benchmark/version.py @@ -0,0 +1,7 @@ +from importlib.metadata import PackageNotFoundError, version + +try: + __version__ = version("google-benchmark") +except PackageNotFoundError: + # package is not installed + pass diff --git a/third-party/benchmark/bindings/python/nanobind.BUILD b/third-party/benchmark/bindings/python/nanobind.BUILD new file mode 100644 index 000000000000..9874b80d1f5a --- /dev/null +++ b/third-party/benchmark/bindings/python/nanobind.BUILD @@ -0,0 +1,59 @@ +load("@bazel_skylib//lib:selects.bzl", "selects") + +licenses(["notice"]) + +package(default_visibility = ["//visibility:public"]) + +config_setting( + name = "msvc_compiler", + flag_values = {"@bazel_tools//tools/cpp:compiler": "msvc-cl"}, +) + +selects.config_setting_group( + name = "winplusmsvc", + match_all = [ + "@platforms//os:windows", + ":msvc_compiler", + ], +) + +cc_library( + name = "nanobind", + srcs = glob([ + "src/*.cpp", + ]), + additional_linker_inputs = select({ + "@platforms//os:macos": [":cmake/darwin-ld-cpython.sym"], + "//conditions:default": [], + }), + copts = select({ + ":msvc_compiler": [ + "/EHsc", # exceptions + "/Os", # size optimizations + "/GL", # LTO / whole program optimization + ], + # these should work on both clang and gcc. + "//conditions:default": [ + "-fexceptions", + "-flto", + "-Os", + ], + }), + includes = [ + "ext/robin_map/include", + "include", + ], + linkopts = select({ + ":winplusmsvc": ["/LTGC"], # Windows + MSVC. + "@platforms//os:macos": ["-Wl,@$(location :cmake/darwin-ld-cpython.sym)"], # Apple. + "//conditions:default": [], + }), + textual_hdrs = glob( + [ + "include/**/*.h", + "src/*.h", + "ext/robin_map/include/tsl/*.h", + ], + ), + deps = ["@python_headers"], +) diff --git a/third-party/benchmark/bindings/python/pybind11.BUILD b/third-party/benchmark/bindings/python/pybind11.BUILD deleted file mode 100644 index bc833500383a..000000000000 --- a/third-party/benchmark/bindings/python/pybind11.BUILD +++ /dev/null @@ -1,20 +0,0 @@ -cc_library( - name = "pybind11", - hdrs = glob( - include = [ - "include/pybind11/*.h", - "include/pybind11/detail/*.h", - ], - exclude = [ - "include/pybind11/common.h", - "include/pybind11/eigen.h", - ], - ), - copts = [ - "-fexceptions", - "-Wno-undefined-inline", - "-Wno-pragma-once-outside-header", - ], - includes = ["include"], - visibility = ["//visibility:public"], -) diff --git a/third-party/benchmark/bindings/python/python_headers.BUILD b/third-party/benchmark/bindings/python/python_headers.BUILD index 9c34cf6ca4bd..8f139f8621e0 100644 --- a/third-party/benchmark/bindings/python/python_headers.BUILD +++ b/third-party/benchmark/bindings/python/python_headers.BUILD @@ -1,3 +1,7 @@ +licenses(["notice"]) + +package(default_visibility = ["//visibility:public"]) + cc_library( name = "python_headers", hdrs = glob(["**/*.h"]), diff --git a/third-party/benchmark/bindings/python/requirements.txt b/third-party/benchmark/bindings/python/requirements.txt deleted file mode 100644 index f5bbe7eca5ce..000000000000 --- a/third-party/benchmark/bindings/python/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -absl-py>=0.7.1 - diff --git a/third-party/benchmark/cmake/CXXFeatureCheck.cmake b/third-party/benchmark/cmake/CXXFeatureCheck.cmake index 62e6741fe3de..e51482659b0f 100644 --- a/third-party/benchmark/cmake/CXXFeatureCheck.cmake +++ b/third-party/benchmark/cmake/CXXFeatureCheck.cmake @@ -17,6 +17,8 @@ if(__cxx_feature_check) endif() set(__cxx_feature_check INCLUDED) +option(CXXFEATURECHECK_DEBUG OFF) + function(cxx_feature_check FILE) string(TOLOWER ${FILE} FILE) string(TOUPPER ${FILE} VAR) @@ -27,18 +29,22 @@ function(cxx_feature_check FILE) return() endif() + set(FEATURE_CHECK_CMAKE_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS}) if (ARGC GREATER 1) message(STATUS "Enabling additional flags: ${ARGV1}") - list(APPEND BENCHMARK_CXX_LINKER_FLAGS ${ARGV1}) + list(APPEND FEATURE_CHECK_CMAKE_FLAGS ${ARGV1}) endif() if (NOT DEFINED COMPILE_${FEATURE}) - message(STATUS "Performing Test ${FEATURE}") if(CMAKE_CROSSCOMPILING) + message(STATUS "Cross-compiling to test ${FEATURE}") try_compile(COMPILE_${FEATURE} ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/${FILE}.cpp - CMAKE_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS} - LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES}) + CXX_STANDARD 11 + CXX_STANDARD_REQUIRED ON + CMAKE_FLAGS ${FEATURE_CHECK_CMAKE_FLAGS} + LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES} + OUTPUT_VARIABLE COMPILE_OUTPUT_VAR) if(COMPILE_${FEATURE}) message(WARNING "If you see build failures due to cross compilation, try setting HAVE_${VAR} to 0") @@ -47,11 +53,14 @@ function(cxx_feature_check FILE) set(RUN_${FEATURE} 1 CACHE INTERNAL "") endif() else() - message(STATUS "Performing Test ${FEATURE}") + message(STATUS "Compiling and running to test ${FEATURE}") try_run(RUN_${FEATURE} COMPILE_${FEATURE} ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/${FILE}.cpp - CMAKE_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS} - LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES}) + CXX_STANDARD 11 + CXX_STANDARD_REQUIRED ON + CMAKE_FLAGS ${FEATURE_CHECK_CMAKE_FLAGS} + LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES} + COMPILE_OUTPUT_VARIABLE COMPILE_OUTPUT_VAR) endif() endif() @@ -61,7 +70,11 @@ function(cxx_feature_check FILE) add_definitions(-DHAVE_${VAR}) else() if(NOT COMPILE_${FEATURE}) - message(STATUS "Performing Test ${FEATURE} -- failed to compile") + if(CXXFEATURECHECK_DEBUG) + message(STATUS "Performing Test ${FEATURE} -- failed to compile: ${COMPILE_OUTPUT_VAR}") + else() + message(STATUS "Performing Test ${FEATURE} -- failed to compile") + endif() else() message(STATUS "Performing Test ${FEATURE} -- compiled but failed to run") endif() diff --git a/third-party/benchmark/cmake/GetGitVersion.cmake b/third-party/benchmark/cmake/GetGitVersion.cmake index 04a1f9b70d68..b0210103b2cc 100644 --- a/third-party/benchmark/cmake/GetGitVersion.cmake +++ b/third-party/benchmark/cmake/GetGitVersion.cmake @@ -20,38 +20,16 @@ set(__get_git_version INCLUDED) function(get_git_version var) if(GIT_EXECUTABLE) - execute_process(COMMAND ${GIT_EXECUTABLE} describe --tags --match "v[0-9]*.[0-9]*.[0-9]*" --abbrev=8 + execute_process(COMMAND ${GIT_EXECUTABLE} describe --tags --match "v[0-9]*.[0-9]*.[0-9]*" --abbrev=8 --dirty WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} RESULT_VARIABLE status - OUTPUT_VARIABLE GIT_DESCRIBE_VERSION + OUTPUT_VARIABLE GIT_VERSION ERROR_QUIET) if(status) - set(GIT_DESCRIBE_VERSION "v0.0.0") + set(GIT_VERSION "v0.0.0") endif() - - string(STRIP ${GIT_DESCRIBE_VERSION} GIT_DESCRIBE_VERSION) - if(GIT_DESCRIBE_VERSION MATCHES v[^-]*-) - string(REGEX REPLACE "v([^-]*)-([0-9]+)-.*" "\\1.\\2" GIT_VERSION ${GIT_DESCRIBE_VERSION}) - else() - string(REGEX REPLACE "v(.*)" "\\1" GIT_VERSION ${GIT_DESCRIBE_VERSION}) - endif() - - # Work out if the repository is dirty - execute_process(COMMAND ${GIT_EXECUTABLE} update-index -q --refresh - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} - OUTPUT_QUIET - ERROR_QUIET) - execute_process(COMMAND ${GIT_EXECUTABLE} diff-index --name-only HEAD -- - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} - OUTPUT_VARIABLE GIT_DIFF_INDEX - ERROR_QUIET) - string(COMPARE NOTEQUAL "${GIT_DIFF_INDEX}" "" GIT_DIRTY) - if (${GIT_DIRTY}) - set(GIT_DESCRIBE_VERSION "${GIT_DESCRIBE_VERSION}-dirty") - endif() - message(STATUS "git version: ${GIT_DESCRIBE_VERSION} normalized to ${GIT_VERSION}") else() - set(GIT_VERSION "0.0.0") + set(GIT_VERSION "v0.0.0") endif() set(${var} ${GIT_VERSION} PARENT_SCOPE) diff --git a/third-party/benchmark/cmake/GoogleTest.cmake b/third-party/benchmark/cmake/GoogleTest.cmake index 66cb91008b73..e66e9d1a2076 100644 --- a/third-party/benchmark/cmake/GoogleTest.cmake +++ b/third-party/benchmark/cmake/GoogleTest.cmake @@ -29,15 +29,25 @@ set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) include(${GOOGLETEST_PREFIX}/googletest-paths.cmake) -# googletest doesn't seem to want to stay build warning clean so let's not hurt ourselves. -add_compile_options(-w) - # Add googletest directly to our build. This defines # the gtest and gtest_main targets. add_subdirectory(${GOOGLETEST_SOURCE_DIR} ${GOOGLETEST_BINARY_DIR} EXCLUDE_FROM_ALL) +# googletest doesn't seem to want to stay build warning clean so let's not hurt ourselves. +if (MSVC) + target_compile_options(gtest PRIVATE "/wd4244" "/wd4722") + target_compile_options(gtest_main PRIVATE "/wd4244" "/wd4722") + target_compile_options(gmock PRIVATE "/wd4244" "/wd4722") + target_compile_options(gmock_main PRIVATE "/wd4244" "/wd4722") +else() + target_compile_options(gtest PRIVATE "-w") + target_compile_options(gtest_main PRIVATE "-w") + target_compile_options(gmock PRIVATE "-w") + target_compile_options(gmock_main PRIVATE "-w") +endif() + if(NOT DEFINED GTEST_COMPILE_COMMANDS) set(GTEST_COMPILE_COMMANDS ON) endif() diff --git a/third-party/benchmark/cmake/Modules/FindPFM.cmake b/third-party/benchmark/cmake/Modules/FindPFM.cmake index cf807a1ee9e9..4c1ce938f9f7 100644 --- a/third-party/benchmark/cmake/Modules/FindPFM.cmake +++ b/third-party/benchmark/cmake/Modules/FindPFM.cmake @@ -1,26 +1,28 @@ # If successful, the following variables will be defined: -# HAVE_LIBPFM. -# Set BENCHMARK_ENABLE_LIBPFM to 0 to disable, regardless of libpfm presence. -include(CheckIncludeFile) -include(CheckLibraryExists) +# PFM_FOUND. +# PFM_LIBRARIES +# PFM_INCLUDE_DIRS +# the following target will be defined: +# PFM::libpfm + include(FeatureSummary) -enable_language(C) +include(FindPackageHandleStandardArgs) set_package_properties(PFM PROPERTIES URL http://perfmon2.sourceforge.net/ - DESCRIPTION "a helper library to develop monitoring tools" + DESCRIPTION "A helper library to develop monitoring tools" PURPOSE "Used to program specific performance monitoring events") -check_library_exists(libpfm.a pfm_initialize "" HAVE_LIBPFM_INITIALIZE) -if(HAVE_LIBPFM_INITIALIZE) - check_include_file(perfmon/perf_event.h HAVE_PERFMON_PERF_EVENT_H) - check_include_file(perfmon/pfmlib.h HAVE_PERFMON_PFMLIB_H) - check_include_file(perfmon/pfmlib_perf_event.h HAVE_PERFMON_PFMLIB_PERF_EVENT_H) - if(HAVE_PERFMON_PERF_EVENT_H AND HAVE_PERFMON_PFMLIB_H AND HAVE_PERFMON_PFMLIB_PERF_EVENT_H) - message("Using Perf Counters.") - set(HAVE_LIBPFM 1) - set(PFM_FOUND 1) - endif() -else() - message("Perf Counters support requested, but was unable to find libpfm.") +find_library(PFM_LIBRARY NAMES pfm) +find_path(PFM_INCLUDE_DIR NAMES perfmon/pfmlib.h) + +find_package_handle_standard_args(PFM REQUIRED_VARS PFM_LIBRARY PFM_INCLUDE_DIR) + +if (PFM_FOUND AND NOT TARGET PFM::libpfm) + add_library(PFM::libpfm UNKNOWN IMPORTED) + set_target_properties(PFM::libpfm PROPERTIES + IMPORTED_LOCATION "${PFM_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${PFM_INCLUDE_DIR}") endif() + +mark_as_advanced(PFM_LIBRARY PFM_INCLUDE_DIR) diff --git a/third-party/benchmark/cmake/benchmark.pc.in b/third-party/benchmark/cmake/benchmark.pc.in index 34beb012eef1..9dae881c79f9 100644 --- a/third-party/benchmark/cmake/benchmark.pc.in +++ b/third-party/benchmark/cmake/benchmark.pc.in @@ -1,7 +1,7 @@ prefix=@CMAKE_INSTALL_PREFIX@ exec_prefix=${prefix} -libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ -includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +libdir=@CMAKE_INSTALL_FULL_LIBDIR@ +includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ Name: @PROJECT_NAME@ Description: Google microbenchmark framework diff --git a/third-party/benchmark/cmake/pthread_affinity.cpp b/third-party/benchmark/cmake/pthread_affinity.cpp new file mode 100644 index 000000000000..7b143bc02128 --- /dev/null +++ b/third-party/benchmark/cmake/pthread_affinity.cpp @@ -0,0 +1,16 @@ +#include +int main() { + cpu_set_t set; + CPU_ZERO(&set); + for (int i = 0; i < CPU_SETSIZE; ++i) { + CPU_SET(i, &set); + CPU_CLR(i, &set); + } + pthread_t self = pthread_self(); + int ret; + ret = pthread_getaffinity_np(self, sizeof(set), &set); + if (ret != 0) return ret; + ret = pthread_setaffinity_np(self, sizeof(set), &set); + if (ret != 0) return ret; + return 0; +} diff --git a/third-party/benchmark/docs/AssemblyTests.md b/third-party/benchmark/docs/AssemblyTests.md index 1fbdc269b53d..89df7ca520df 100644 --- a/third-party/benchmark/docs/AssemblyTests.md +++ b/third-party/benchmark/docs/AssemblyTests.md @@ -111,6 +111,7 @@ between compilers or compiler versions. A common example of this is matching stack frame addresses. In this case regular expressions can be used to match the differing bits of output. For example: + ```c++ int ExternInt; struct Point { int x, y, z; }; @@ -127,6 +128,7 @@ extern "C" void test_store_point() { // CHECK: ret } ``` + ## Current Requirements and Limitations diff --git a/third-party/benchmark/docs/_config.yml b/third-party/benchmark/docs/_config.yml index 2f7efbeab578..32f9f2e0ddc6 100644 --- a/third-party/benchmark/docs/_config.yml +++ b/third-party/benchmark/docs/_config.yml @@ -1 +1,3 @@ -theme: jekyll-theme-minimal \ No newline at end of file +theme: jekyll-theme-minimal +logo: /assets/images/icon_black.png +show_downloads: true diff --git a/third-party/benchmark/docs/assets/images/icon.png b/third-party/benchmark/docs/assets/images/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..b98260486e421e2f1caad425148160910778639e GIT binary patch literal 11106 zcmch7^;?u*^z8sD-6bWcNJvXJNJvVDfPg3Vm%GMl#V2SdL5?APuqU|)%>bze@piAV`qn@^zedj|Rj$9#; zb+9g-3g^XZCVa*4h@3?O$G^|3))$ebB|#+&SbhPGO+u_q3U9BJaX8Gxg|@q!y(635 z1Jt>A2iL>VlI>H9WQ_)1=yj&+UE_X3}LbRiXXifMhT z=em6SNGoJm@meg`3o%_`2dD8qo$=4i{?E7zD%f4pR#`~&zmYJ8sjJIjn@@&~6vvp| zmOI-exFnw7ic-F`ww9HOpMg_qBQ_iy&e!m4F(@ZERHraF-2;}GWdzLi_D0F}JQpwCwX)*k&7=F; z7&{F5rb-~J*6({_(9)v+y548J^LJay+uhK6de8oBHA&wq7JeC_~%8z8>C%jH% zn*HGiwxhxm7X*Uv8S0HzC|O_yA7Z*Hs>)&h#v;eZClmaj9fv?LAQWY#wLPY{<~+6a z%rDz_LzDgZ`Nvr8$cT?$E3@3WpPOxJot@Yio1FhlsXTAet-6YboZpJ3YQS#RkyWmP zpI<{+edG zVZ-*smBjt~O0!-YWyp9#d*Y*c?~2B)`We|Ykyn@dzqhwJKex05cPzbhUGP0x8Qnv4 zqTMJdE!`jN>k~;!OXH}nsgd*ZYh__)Cu3Gi=X>EF9v&{c(CQ0Ah*VPq$T^A|i(AdOJ1|w@sGUK|ybw z0+0dg_gPt^n%#FOtgNh%?SUA`jf%GVIUcu-aaGsNNsTulA>rF|&7}WeQWMeX>74F6 z^Bnc|bN3H-7IvJw>2KZQY_uuGu&N<{SpK3OBA_=PBM`crm@U_REG%U4T>r@$ehsljC+sZiwmq8^HU8Du@5CJE8p19%F0Qll z+c)m&X<1v_(sz-O1qz75PYnxxDZ@YW4eG4#NuF);hF=RPR#sMyxxBo5m&xfo+EiQ1 z7gZQaUU?@kkV!R}@fQ;L5{t6h>%e-qgHSSz=XuvhZ$x>$@0r`$*4AKAEw1}WK)c(4Kln#mhnzkd{|a;4(_ZKT@K{#mW1{QS5O+P%tiKAKbg~ zC(qg08E$CDZCZj*M|XG1SL;8CUu}$&quxhz?ir20)4d^_XPvs~kQM$HHpg*S+bIsM zimedpsBkYVEhSr7S<$-?PJ67Oug`*miyJ(;=9+Y9J6&6(prk|s(YegRLf}S3N5_%C z`P6#m7cKz-1uAa;Cw8;tA}F%`E>8Q9EL>b_eisK5cSJ=EUszg3RJ2_hq>8-itKUjX zOk}x%Pc55{uB&5gEHQ1*R_NvB)d!<`-RC_Y-4b$aubub`+fFLzFwf2m+bkm^W9Ob^ zHD3B;HWX3HA}MKddU7&l#(;x^6CJ~(8eLRW6o2bubWF@ju42rKf2)Ib!(1JqgauLv z-Mvmq69qLjHDyAv7Vo3@65X=sxVX4hx6K?<=EprxYAkvopOlZZJUoQ1qgTF$W$3$Q zu8o(a7`6I*vcvs)u)Mn3w>DbPFaMtUN%aCcI{FJmMQk~A?h@G4C-$aACT*>$d{!hT z_s=r+_g#g(_FwiN@G#!f({rEIlmeH6y>b9SLPf=Tw6idT(8R~a#(u!S&`xq;SYa%l6c;CVg)W~W z>Rx{D#`}VFFQs%b&uHl!3IT_C+qX26u-*h=Vc~uL_Pku0A+iU&yi_cFwUDKwl-RKT zq?8opHjLu3CpBF^gJkse6IImIEFFIgN8eL^(c*QW;!F5C5)%!_Znkl_7ULnLzWKw4 z=oqgbY85=@w-e*DpVdznb^qXj*sr%7x;0904L&y~;#M#=PRAo86x{MA+9%4)dEWh@ zH#U}R^q#+h0+t|y>Kz^)4FY(oX!Dg%Ke_yl-e+=I`WhTA z9dsj!Q#U&&=PkLE&DKnVpiVaC_3ORAx+7yYnKI-O?ec|PR#nVy{7`E1yHJT+tY|&G zk8MwUa&l4)H_Q*usI!?Mk(9C^34ha}Tq>+NQk(jFIB-Ew+WH|CC}N=8AEyjm0kcoF_ub$Mka$s#Skr$+&aQ&>J; zW|(p(aKU>=oMbZO&Ye3d#3of$Ro^Jpb$)fd=aoa+&woz(R#ydi^_|SA-hIayiPuQQ z>(g5X(Y*I}se`c;*W=%yAbzAEpyakBLeLpjjFUsD`EPH}%*u|k2B4z@SN7`i;;rza zm)F6%`ODXPMUN)TL(~SUP47D5LooC5@uUbkd|yxE{EnBvhl_&~CcTK0lOgUfl8YUT z_7LkfAe+@oc6nJ@qR@77h;o7bTNqpoOGgry$IoSOzZL70ggt8SNeR4l>(=b^cK$N` zs;@uX=~2)k&DCxm7az|Q5*$nw|HdZ>O>l4A(1D5~MNRyGtO|RA-|QP!N3_Gy_8cR< zz4hi~)i;*R5j_)MbNvwxqsEW9GI0lMqurIve_r}FrvCV$E<>_Se|3zB9cs;MUDbN> z?>A|MBOSf(e$?%^u#InP zW4tUR^`}azP*e={q=}JH>TT@!s3_^zS>yRCNoES?dp-QNLt&5feh;QRGLz;_6Lr_U z)dBI(9P-R$v_MTSU0d91pP?wTh}ZIa55(h2RVqY3dxzhBHa1@!W^IUmc2?HgS(%>> zHYV7(2-GA_7_0EkJh$p5Vwp$dISdt;{#f*UyxsR)L{u~-F{H&Y{(3Vj2Zt)Hgiqs0 z<1H}{ZYTUR^%i-2+yv=3of6&MWfFBQV`F2b7|WHuI7Y%)jh5x}e?yXLJ{_mpT)lGu z;3SwzE5CoUWr%C*>Yk45Ro23Ou&E{w45)--#HNUOR*wipMnv5G{_N@5;g(i1cg~|o zre~%%-QC^ItL7`8$(YfN^Sl00&+80yr#59AGl`sOdd0R&uu%m3LMMp@R?4Vh+6Y)? z&a=?qU}AP_oRpq^sEOB6SW!_?$&#=Wid`%2r%nRQDxVW)^O?P)BW7z;oJh&bM#E;e ztynvU&Tw)*k=zqn}HFnuXs@HlNYncs#fl1?udnBe6n#p9O&GNz!~6%luez{M;b;q0 zMLfK{Cr+bBM)H*3KG0SKRw(50*R*V?3zL9W!6{&aJQM?yfMx1&g>ehtNg7n7Y&<13 zCg1}p;dv`MD|c83@^N~?!om!$y1L%em@6saXoTE1{OnQ1)HYgUsdn$IUz3^)GvPh~ zkf@R6Pye9G5kD&&Rt%lYd|Oq((kw0h^F;!q=pxre=*hJZQJNx;oaj z0D!#i@;!h*>C?ih<>ePvAwsQ5pFfL^Yg1qkKUQCi}>^Ky;YT4 zR5+61&5z|>Lv>meNteRH!XMdTiUp&Ju324fz<4(n{>-;DN1qWG8DtCH-2R>6KkiFo zz%EWiM6~F(|3{l>bmhrt{ld($2m(HaRo1Y^JuUYIEgq4C`+X zl-=~SwOyNcTF+b;9-EqG!cGCrDfvD%>OU&{^^kHK zk7RVnLSvZKMzY8-f02=qF_y3p0DEe>5V#)nFw^q3I^9rPzp6+Q(lix{(r~^i&0NDT zN1<8mTa<8t=>gIaf^2DZRsOdY?Yk`GjfyAR`0Cs&*L3d_JjBv4GGZ4O6DuSl!#vPd zd78p!Wj4KnV7W>5I{EM3o^y=m`emWjvuC%ZA3AY|gfEU-PvPO*2yRzF<$$w`3y1&S zjqncsG)ez-q|}#8qC~!Oon>Ygf(gJTd8bTMbu1O?3E+Y6^2sot1C8Hk#RmqFu;$TB ziKoYLZ34NqrRo&m8xe%_o|@lzt$?G{0A?nr+hG4R9AmR6H&?b;N?A`eL&7&V32H`A zYja?3F5}$U;!-GmKA~+Fi4B`o)y?@q4-XF_d>tZU;@d#;U|{4?lmxFo+y66cw*Iq-EX<}*tKh@-Orz@>3BIb6`c6TH--s`b z|Jlq!Sja}tz(6cS5@k|SQvUpq)oA`(_;)h;E~fi}Zv%%j8Cj_0$&mkL&Y6IH+l5>9 z5;fFMR=7uJ60aqDRU4abDbdqrH(-fzt^4y8RZokK1oRExB28zIfBmnH{AD}n*^NKz zAO0Eca6W4RHZ)xIyn8e_IC$XB8=`~tv1XGUJ2G*0r6GZ9d&v=|W06qw{D0nH#G!$Pj8#5^if1?-YnB6DB|uVG?F02x(hK!{R=8alM- z9N7UQomvUXDlKK_lj5vS_X4 zJr%D7ck;s)D4ugBSA@8@F1`H(8xs`?<@vBRdtZZb$;(H&p{6u76g4-CCTt}NPr9rw zhC6t4b$9opVcck(b)pyZP=E90&1^zS$_5F>G*p_xGXan3NTewid0>Ir1rslopnYnp zkW+U5jZ0wIa$a6d$51uwOc=CFy=9#OZUep7kJFKXV*c@X@DaW5i;Y%5$mVUk$yQ%) zPBtDF^`Y+WP-YcAb;t+9273;7AVR0U_#jD!B(FY4+u`35n;efgEIJO)4)45>a^hA2 z{%MC923tr4h~9d-&ZfNSASV}`2qo|FkNePM?_>g{-Od|O;SRwMT?h%^v1n+ z?_T}pGM%VfPe1@-$yFrImh1lg;_)`8m6}Q@u=ZQibz0R6$_bq0#Kgq)Y(&3@GA4X! z;K8&ZI=>Lz&o6`csNY!3Q%>j$3&S#>N#S`O@@J~1AiwRTmpP*psNCD)hxb8UyxW(=ox$tWo!fG3C9&D1|>mEv=lH#)K_+@w*bEl2+P^*my&VZN`w{}JS7 zhx1@Ee_#FD&JLG={p{qQzBWBw>wk7hk`zgg(?mYOj2>HgAFhun4`fI#j9fQ6*<1Oq zXmBTgtwc;*Jh3w|F;ON&FHp;f2Uu5HAO^Xb;D28Dz4BQH-^mt?H?EPi7)Z=Ie;SsU zcvHEn>q`)(=Oe7c!;{|`q|1U-*Tlrd;|P;O3FuC>Qsm_2t@ma?E=_DjLNeZ$9ET9p z=ait}G9GABsys5kr!zVZ(!Kw>m!@X4?L>Lno*F+;_b0VB6W_Ni5`t=*O832SDLGxW zF0e;|6h?gfNaVR9|A`!wE1(~Ff{(OQqlT0EUc9TWZhW2yw}gbT#Np)gz6Mc;d9h)j zFAeJi>tjwe-jqP5oNdO6<3@fx2BCNR_HAiY4t`TaRf_GIzXu2AjCn#gl{=+O;{?Nk ziOkNAcSG3IkNVgu!hri2HR$_aT^uzH(UwDyHv^h$dYuD(ApuTI3NkmJiwGDlgciz? z$mQ16){fntP&2H3`Rpz4SHJ?AYK5md<;^yAWw&KQw2QdFa(Yrb)b2I~OzMc%w6tN~ zOIfipAt51Bz=kF^Hns;ClFW^-p|`vWm4X6vXG*?nQi!^3a+gXaz8Y@5j!VHJd>nJIjZBQz8X31hxQwlb`d^(bgph}6{NDY-|xWV2@;NH9YSo=rAh;J5lLFRbyi!j1+SH^D#P;Uc|L9OPZdT_}QO! z0q5l&YPu_+to(D4pFe-5KI#DM!+)Z=d_=)v@IryM$V(xR7&|}v&foW^ll@aQmgYb? zvM>znT0;&;l)0>$xF!smi$q$__mr*p#v+^Sr)w2^o#f=?G-z!Hq8C1H&wi?bINRGJ znJ^^R=7J(v?R$23hnIJ>vK}7`i%8UcyL4Cco&iT%*4R#+-ORWhi2abR4WQ27M%5(C z*yBk^N#kq_WI_qvWpXD>PU_F3Q5AWG{*tHWd&vM)-1^l3eBqdD2G}tYuAEwEY+f!KaC(#XO=hH7K^{TfZnwAC4?G-d!umuy#&`Ku_MD*k2pvxAQNv6yVnY zvgvvJ_t{gV(pb(sDU(X3PR-7VoQN0d0Tq5rgfE1QMvqBh_*^ZIgT}W&BLgc9MG# zGt1@ocdB~%xb%dAQfE}3>AUEDi%>tD5bKMMhO0~C(?%3*PAde+fIlw8Itf?iVAvyD}p3)0^Q z27daOT%4b~(uKhaE%YV#Hzw3-o}-+{xUGhyHg|Na+i$?x+t-OB0tT9VuR3aImXKb; zsNwf02>X(gOK{)T{o+N;`^aQYvy1yP7gX%+%d36CE*7l0p&!)8n~%ycl8&LFp^s>5 zi(i78?zsbwnmhOvd!4FAp6R1X(_wQ{$v%3BqJn~Y{LQ7MrQ~~EK)K65a|8EZOP*AY zXRq3i;Q=Jj9U&wLm8p4tpOaIUTIQWlB^leil(uh|pyhYkIa0%lescp!B_DBea`HLg ztNGq0o6~j5HNQL$rz|-xTxl{(0AN^^Kcd2kRJOB#w)-7b4r3A6gt`5fmM>ng(8L07 zKzUm1!<5C6G-seYHE2yaiWAJ|Yb+xz6r)ktM4lZWw z+uuOj#C3|&|e{DUciYF5=R>tgrGBgOW@RxOKwKq!$m0YPZ0t&6vj)aeEe z#V;BH2Bei29!yq~dgu4|v>c)KCT*sO@u7tbac^DggEX9iQC65_u~^bUa*LICrEWLe z$EM}VfJ-pzbcPj;9o$2%;f6aO)2a#H3tRO6~G>T)zstjYyct8%X7~`QdS|MMx`15<)}w^YDnPH>*K9nMhz83 z#q8dDryCVEt!-?2<=TED9V4@qbYondYrC@HL|0Ge&ny;R*biq1vyuq$^XoL!zjAjM z9Al^hzinfC1DES_=1Hsd38SG7zx{0DSA@6bCxZ=gpaVa}vQTa$+riHcH>diZH+ueCCA%>>^E^^KU#!ZY!M@DU3{>{k#1^>hz1rC4;E(*7E(b)< zr>$&q-{CazTp@pYI2hKSqp6gXm4;m32?`Qql)4B!P9psAWXP}(b}3iw=wmBigco6)a@BRdx0<$*Q<%O3~fRMv77UQSb&xjbv242^7x**3WmrumOkqUp>ngJ zprf5X}dz`($IHi8KQlHh|F1O||S{-uskn&d?2L{zJq z<>%$C%^Mr|*UTuOAPf-1;pMqFa7D5kiw3}zS>7LR)(lPGk*+&EtRtsfC@F`E)DOr_ zP3;agT6JmZy9DZh4Jv-?*rn?ahh8OWYaHCBE@{w`mJURAypIwbp2kvwGVx|J_v29TQ**047ihzL0Itx>6RJv)>sZ1Ja94r{cz7N zHYVnI@8MLfRSc^cyO0osy>>UUGeR}$Gd%j(wY9Z)+i`|7t06B1|n+=-KJFblB-lpfJEO$Mfit1syo= z-o0z?>|DJzV+P`SMCJ5g1+ z4sJ#`m^P15-CLittp+6E7^&ptanAAO*-oeU(d^yrrVUQ$g}H}$@3b6E_`5;NOhQ89 z>%@c}-^(BE54!2d$?xfv8(Ae^U;}dkhi+LYlRRg_q$$3e-aqw~ql(YNB}X9@a<26B z^lD&yPcSm;0MV1mrE+s~d!Q8-^_R))C$qQ%6EAO!5Ze9&`&l6^CO=u(YmxV;ZD;E3 zMnI>>?pC?{er$&E!w_YdJQ++Cj**tC0>#%0xSb@xL_$ImhAkBbbRgM{`{iWiZH(7< zPt4aIGSm6bp4BiifBQ;s!6&8_^PO3K!g4vvVMF*^;j0?t=%>NwWCRsO=$<10*x?J5GnzX2A>4NpvcDzYcB67@#1|7pDi{ zCXP%HTsZG7D+;@B8%=?X=s$`CND%Y*%MgrKNzKbSt~YbUgwh_C8NL=tXhn^U`p}E+ zL$z=osL|*IXJljy3=I`iP8&g+O3OA~^P11*M;MRvSFh8<^7~f%GQK)@6{IJ%3 zZu-Y$QDNbqcyNnSp~VlV&qzQk#IX18pWVbZTux5!f9`Y)thX%%52N$TSD33g7!7h> zO&59-sFL*e=!g>h^{-2gSoHMt5KiBq^c5-^J^k##bFn#9BMr_$c4np&c!*)}*x#n6 z(mDZSoN3qr@113JAW7!KYosYNYW-^?d7N`M07SCw=bExXha>A}>`76l6o?*%iSB&X zn)N%|Le4m}k++@iMH~Z6e1m>B8|Tv|Mo61Z3_>vms0c<)TPP?`_t{fiIZ1*w4P{7j zp{_x>Wlq1-+;PARgn8a%LzFQxGAd+>B=cF_sR0{z&O|{)B^C(8N7ZD$q7fNft$dZJ zM;~aHE-}>&;ov~ICK3%*_@kwdfY0v%F)JHuJv}@7*LpoO82GTfq)|dR*LfVM({ekF ziKat_LJYJi zrtRm0^6;e)b*Of{9axR;;yD>1xTo5M!6oka-~tky3@fu9yyY*Xsse(h?FdhJ5at-G zu%mXzJpRxrAj6Wvg*bngEihX8gco|8kMn|-vt&Y5T+Rm=5bsgu+U{hDxV-?)fpmJl zYD#i3N1`->TEuxdY;&xHGN<&qDv0FDFWj7*-(&3_1|s;}{w_)bed-OfK?A-M4>U4_ z;I)NcWQej#00B6O%%ZQ>zhT)v1R?Yh2qdr1BC12YHWN;DEi(q}e>kJj8D@*l!^5)& z!IV$6353J$t=sb(Xn#5BxW)uTN>WTD4Oo{4P2I8GYHkBADX$-tv6A43oTn= zaAQzsK4$PAx~(x=5UnoBlP-oss|gSOdxz%)t%K36g0{9cEF2sXP^a0z z=a5;Uw@9|(01){MD=X^*cJ={*?i!m3PLz$&1?ENRGEl{8R2EZEgtJ|FL)UMP98lg82NlaCPYpdG#hTa;U-|B$9`xCvgX|O^pp^Fu9(XC=D)*3_Rl);A`2R zKO3EcS&D??1xPr;LHrQwzY{1Key57FzDILNF8`B=MrhUjSLfakA1i(|d)p1R%^BRKR71|Gml6GBMDhPF%GK0G6SAnKC zvc@_1&`b$|ZPk`Va={@U?jS?}HTS=S#YAat7li3YFsjE6yL(XImI=l?XO#p6FBYQk zMoGmFr9cEC?elII4W|}E*CL*Ogmn^}m+un?J2;G_qjOT#4oHy^?+ z{^wp}{EmxK39Llkhnrb&f8H?3{+=GxOnAde_-!ZE0r)$If5Ge;m7shliX#vSiIb&W7BK{y72JLtH^ygyz8=0Npr%%I*@(FuXDAo7RN<|*h-xJx)vafFiRKepa7 zr&n(6LY5ErBHHng)=!&UmqthsKSQ4GNdKroeWVs15n-pn(D)nC&h7RYQ$t((MZ~X6 zAq4kNU`BHx1a#IZzfKNk99Rkd`vS?UbKXugM546O)^_twJ};V-jjn(A8ib=*)is1y z<(`(x{w_HxbN*n+_8(z))UqHOj=vh8A#Nw-FP#8Y>=Zsnbw; z6kr8+6HiFAG&G3+yTSP?6cnIzu<8Y%PXS*MsPXZssVe`iwJnL?h9VdZ9!DGa{bn(65yt?Mic{poz{=zwGIo$ocfjR<`J)HmN444;XvOq*rJ`67iXQo zW;@|Xf*inm#g#vhgvmUH zSJ$btbTN6#v>Re?olqQmOJ-Jhzmp zlPpNTY4Z4Sv6M*^x$Z1n1+%!FQ{3I?mZQM=Q zYs02Ym|l``!^C2(@9+6hb5Q$Epg83VXOYsske2JYT6uWDa5Um08}Y3h@v)8gfJS_q zMts{we0zAkp7*Z>j9y#MpXlKMBSCrL4~%cbd!N7eo;aj`{#V8ZZa^YZ6E$Hqx1 z9?bRN02?(u^&9x6UG4IwJ#B3EjE&7dwXtQY2lG8R)PtTjwEoJ@2U?xaSpLk;cY4ak zu=hQ<(Sti}jM{2r!e$%OF10b+bDf^rzx?%#@Svw1z5ZzD`yaM($QT<(EVuEpaUQ(U zgLivyo{g8gZTyYrey^&r^JAPgj`g&2tf!r0J;#eY{aX}j*DFe}as0zJPF&`}N)LYG z!5tnvYU3o&vz!#{$vbQ`3}>JrjmU5YxWwhED$CCkSM7^eROgBNt&ndcam>YrZeZx_%`4OGxS->?7s}sc^?shQIVO znn|wFQnHNz$uinXh7l-fMzEw9p^{{DlrBbRNif32VMtko#2HZ%W5h_b5i5~~Ln4fL zakgG3N8LMDFPV4O^;Zte&U84!{40;eI1;k+hK{-M-p5~AcPK*Y554#3eZ3JO z;)pQBkt9JzcWGhdOB2J{w4XTJ7~)R6`Qt)yMVG8L`-(g1!M)t}uyPl?W6se+F-vxk zjM+po@(YsDUy|H>n8T%Wsbprpt4H?1EnWzBVRfO%!suV>nHh_H7>m7(79v%-Mr+A2 z+DLb!y>v5zB;5#+RHK6=Ga{21kzE*(3EH>i4n|}gBQk~&8U6h^;LJI}Z{|8t?MRx?5Rb;Ls7 z9whN&nE6Y-(Zr(OtQ&e6Qm$z;$upW;)RRG|q!Xi352F>sQ5>4;z-(LSaKcXmKmM-SNXWEMVHC5c5XMWjE*882q9jUC0@6AL%b4Og4z5HtJp zPBd%;+^zMUMed0cGwSUykqVgZ+iCgsqFCmoTDaP*Sdl;QMos}=rL~9;U{g8K7* ztOtuht#B^~i;luP2orW3<^hfz0&fKMawEXkIT-08Gb@nSJfX_h`@Y`))sB55Q92u4 zq@y7RyGjQmSwf8z2{BSxsz`B~1R5FA-smRnj7({3VLD}L98HyAtET=UiC=c*97o^>;shqFBMa(Cuvou{*zv&EsQJ%hKh=JXz(`8|+U zJN<&?c*=#>-7~-ZopoEc?(#o)eEZffK74)g+&jngjgFOu>~VA6`X)^3>$kkJ#I^6$nSI)cD|_Z@cN(fj#mtlz;=X9{p&W5e*9 z8d6yG-+Qen@gJBe;XK_$q-rS7rVag`gIpCyZ)9#?1TlSJ6bgc4%Jn7_cdZio3KLd@mWG&=!(#@vz)H_Q!pbO;i!G?H4^!@LtEa(@iBbqHoj zjL2ot+z+@2Tn26fS3*PJUn&%N7J2}>mb-KfCpnry9lk zD_7Ec3lL{ZX7t5OEv4Twom|7T@)ntt1~T(u7Vn3dPU%*#2;2l-1#SSZ^pxKKs$yKm z4aEwGF% zYG|>wUnx`lx3^_>QPYd9{pNYJ-=3b;k!rx=P#xGN?;6UxvQqR83PtZwQS`3J8X0EQ zNNY4Q!l*%p*F(PHF)o6TZxV%urV{XZ=z1uVd@cAf^b~XhGyqcL3#$fM3%vyW9@1fP z5!3^UyHkWUgzSMnh4lAoTzLmt3B3xv2)zvb4XS`t&aQ#hLEE5XPGgoxI9xiUR~-w@ zf}Vj^LE9l015w4U3RP7+_3XPT>kM8BJq&#a)l*gomO`|~bVp%t_1*lh^~qbxj4CuX zdFSNDsB`t1lx)brP^_V95&dCO8~(paKXz{K@iSZJ%{RUa<-K?5MU^^Od$+acHRzfu zlE*UiWnPBggMGnjOj;YkUgU2C@1{cVAFjOj-wjtroyh@1!LqnHUX61u*}0!*E#GhErvT zxitwR;SqXcO%yHvfR0{@GPNKDOEeb3V$>(sR4jyPC{HyPp>iKUOWKpu5n~xj(#Duh zXcy>6Z&zah%s~%&79-!cXgH=b63HLUBN-EgaXU&iA#&9x(TR3;i=}_joYqIOa2ZWU zuB4M|In@&^m|noro9yC5`GTYSdE)PX4I{t}b$aI;lIzb_F%V@#A&)*lr%xc2xK2hEcSg$9)rQ zX-|4}m4PZ>rC8genwM%}s*TA(ZRKfic|0vw!>Afa)mR!M&Sw2^@)Dm=lV7rEqbXuZ zCM1iM4e&@6B#@P7(mXz#`F91js-r`gP8VX*+NpLfrx~I%Efe|JI%f8jn8^klVyb@v zUs1~THagiN4?2I`8tHIlcg#>N32o?u5P76ImZmPqmlAgqVjK$cq~Vg|%+L8^_C%dC z9jdJ@o`fT4dV#p(+?OIdt_h+YxtQNdp9>UMgX~M_D(-ReNT4+AMZH^`D6W2WayKG- z6H>5})~6g|!A}0~O|YinOA6c0fiV=khIGTHBp-5M82LpAxoZ)?uTb8b{G4WZQO`%X zJi>t_3htu4N4p_^ev1@j;Ve^IARD7_;#}^gipkB7LN0Eghk6{R$`J~NARt%JQ;u(S zsX;+5g7S}{wEuIeuB9LWfq4r8@EvMakY5}?k6jJ3gqnXvM7k+Bkxz$~QFA^82k6uN zUD~q7K*Zil!Irj&*(j7F7CWE?2jsq)cvP!Jay@z zA354v&c5yvVB`p!rWnb|l>xXd@MG7^}U1ea>^h1sH z=ag=O@`AZgBDr9G91kBX%`jay1)Cc&@WIjovt@G&BjJO&TsRagt?&WL!4@cttub7- z1OpJVt-v-`SqRM!qKqlKm?)~U_LPTG)(#5ATiFH*#d8^8VIX#^_T+=GY6XJ9h}{oW>rt*6r%Fax#)Y$ zGtM4_S0JXw*K>h6t&nI(yNk?Z^rcs4iTnl`7rr(VDcV+Kc{h||q}hCK_gJ8=B3K`+ zL2O-Ki*oryFpl-T43BB7PU1InAN^J8u4cjFu4KXV=R`|Vi<70yZ1L1!F% zl;;zN7TN7<76V_PSVpXmCb)#m3Kjzwpjs|u4RC^`gwv!8UqL$^iw93ep|1`&#Hmkk zjg;-EvM-|3cK(0|1>Q=guOr)Yd> z2lPt~-cyH%*EMo3l!NL(`99?#puB;F8dRtOh3ZJrbE@OS>mkX(HIRz5*P*W|SCQ5inhw1H?V`*9 zs;lL3Xp1L5jW8wO{r*=!U#~mS4ypBesqHt%4z2Ziu=P#ydbPicr@bv+Kg7Qu&(#(? zjO}!rz4vhD{dw=(`6Kb{EX4Hfc@EmoyL&$P6m$q)ZOx0IWAKx}kHAduQRoQ#46qs; z1+E1Lfm%m@r+J+xHh@#G3mqsWGY6(273NJ!$B@|{Mdo=jyy)hxA~G+N(QDN>o39dI z+y2FVJ2jG}%sW05(bX*5r@jT=I1NQ_tcEursA6 zh?VPmvK5*InZ5Xq685S>VUl}*<$ci>jAj)m5GRu&Q6QMa2Vo0a#TtJpO4NJs<5}j< zp>_eMJ{*S|X)*?~w>j_umc8LFuAHaFzk6s)b`7HQIqJF3qd0aCX49zSWOtTwf$C5B zt&`s_=Vb42(yj3Kqrxmg@91xmQH$Q)wBlKQTtAprUS$&6SNaXvqdDBqdG*m^UO-KA zZz|Tvw8Pxz0u-EmSxvEQR-?jfB6qkqCe1bIC0Qpzq@iXYN($P}1R@h_2S3&o-H0~2 z=qh3n%4PfOrxNZ+4|J*g_vqpMIZ0v;tK>G14M-Mu)}jWww`LGuA;jLh zOP}KKG;t4KR^N_uPT;Ml<29O7rv|f$y9MT;oU-D_6k=@tuPo3SJPZn!kln*g)HmkH7}n z1}kJ3Hqf@@J7WWFOTHsE&~{)44~Am-Y)@Y8p@Co^*3du#N!wxh3gSaDZHD)gN>wrA zEfrJMOgr%PbhxTi*&(c5EvIH@E37pcld=ZEYWIXe9g@i>{qn`oQA%cRT$*mHNGNQYXIY$`$v4C(myxu7dF|9Lf~; z7xM-Oh^tTOyT_BnwX6L4c%De^4U5;+5hHbQ)g#wnrmai;&8-XG*o+OCO#( z1W)vl@QnWFj~qF+_^OeY49rg|aQd+v@jX}HbM?KiznJ$`r9~*ScN1o7MSm!i8QqV( zN)xfr8w6_Oj|ZE2KS=m4swDbh;(M|xHG`_WYD{Vt{XQij6PecdO%sf+QjG5D$V6@? zQ#Xd-*+lM*ukrk@lxX5%kR9LPy3z=Y`@xs8F#-$xNhh zIigBG*xpR-1>kI&GL&h1Iw!oqO~Fb$W9|`zm!uNRVm72Pjn?+U30KO;Z}Kpz>AnP_ zRNf`CJzC67i(4WdIOpIP6zCQVHTXZF7cOar-4%^(Cn}cQ(?rZ`RfTmAz$|bG_lv79dJp@ylOsY8Ogs%}gZRdYI+WFe}kezZ`*lTtvHToCww5^F?6Yqn+XI z=QF|jMe*F!oj9sbER09A}=NX#&cYl4&xLJw_<9;rHMGgbmY$kpa(p21)XNOe}# zT|>ZDmQEW2O1Wyip=hxI);b~pH)sgIZi5jygtN6pTMY%@M-7( zyndIaPShjt>S%cvdFn99sK0YJcm=oyRIlo59PH{Q^E6CC1B~uRl%gf&FxmP$-E`TU;}6vpWl-+;zYxYyajUrrw1P_gWd)!W@{M^HX4k*x^|?p#+W?$jqwVt0)$$L>08 zU#hRc|MnhMA70~6BUx7#e}9__d#eiXHbXh5KDx9wBFg1uv3)h(I0SFRfqurRWczCD zIC%ZwQwjIEE_PcqVPN*K+Mu2z^*yp|EKmLZu34vd$g)1pbe`dLWF7bQ5XMb;0VC*O zG29BZ;n&ZF+h0PmZYar`VI=SMC8-)s@)ZZ%`yWcuJU?+CS};Hh7E#N!>&eTjIYC3E zGhmhcqR-AmV1K1817oZS%Kk1}!~9fjQKd%}CH)#sz0!PBP}a`+p4#4C3RR`7x%DJ1 z4_m-0!`Z|XSK_qWb9;)ba@eC^Ho+pcdhWvnHgRk5;8oi4J-hmtiq?hA1;j8xX?gT`bOfZ)9=qo(NawQuPYI*yUfNVmyfUk-QQ$ zbmgD+ttNZ7A0o!9+H|;2IaKfs4nxUP_h&GlGRDvadT@>05*3N_s~~d zC?WGvIU;6mL(#aJPa~?wz2Qc`=-y5@^{Y!`uI8(O*H2`K6;i|GD`p{upzhA0(%>=Z zbIQ~$uio_VF>i}<>$Q74_A7k%Azx-WhLcdqWzSKqt$s@HY+Q{0AjWU6f}+kC%?(%w>?lib2wKv=j_h8v$#ek*ZRIoQa8bC?3Q!}uz)`6JaDf0;f(UP z$**M!h}7>chLd?Ijm@KfK_{iDeJw@8yK{_p&lpug&KA_{-33Nvg{m5M*qW5tftVVA z>HHBD<4l#~$MUT1&KSMcc>j6Ie)v@OH74_1S)G}y6WE{mqUM!%ufKmpw$OLrl+HLi2-hozdAx|bU7gQeeYd-o*KVW}~= z-bhM~p!7yf>Xt{kV9~{jvt>VOnz41f$7<2Z;F*5{&Vu^Go8T_!AJChST7YzY{~4qK z?#ZBjuk{Lsn(C7j+(c#vcryjzY7?q}WXuX*w$IiyAsyfPCw^R6HRjl s$!*`C-I~r4=V@XC_@6g_r&#~=e?nLdedzfw5%j-unCRg%!S2fcABXx#lmGw# literal 0 HcmV?d00001 diff --git a/third-party/benchmark/docs/assets/images/icon_black.png b/third-party/benchmark/docs/assets/images/icon_black.png new file mode 100644 index 0000000000000000000000000000000000000000..656ae797cf4652de02de79fb5c24b37c94860609 GIT binary patch literal 11559 zcmcI~Wmr{P)b&9S5NSoaTN>$JSLhXYh$dM+IllA7W{O z{{-1cNvSDFNl`gM9V~6$SwJ96F}^XP^6f9lI*l|sY1pteab3IB(sz~#BdaiE3UF;d zX_70m-v%;ZDTIgTEb2NQn3ZoWE*BMkETlsB3#hFZV6K-By;s6uGZhiogEe?ZHMj>1 zF}>*D3PVaYtIhEQXg=~^aoF)Ao{M3#DATzC<#>CCU+kx}znakVny z!;(!jL-|9j_zF?ieU~{|dY>N(9@Z)l()q2{FCUSv2ev;L#7x^96LCt`W_2kW`CVjf z+o92c_jZ)osJpfZI=@_$U{~a-de>(B%1Gz0P(L1BSqQy6?zgO9{N4f@#llx%pPuf* zL~o=bjcAf2OYn&VpQ(L7oj&|{|9k&^@y%*b+S4z{a~##TTo@-i5Blrv{V8YNj!_Q8 zk+r;Ljcg#lGQUsG>aL=1%%ZZF%%)bL>F+^4ss_L7japs2~c`5^p?acIP}jp2=ji z9(p-b(or`q$p=ISNks;QP(kZksj{R|<#e9@Cf8xWh?H6T-8LJV`fN`_j#D8_gP4GHqBv_|4lpB`edF8sS}*7CS3lJ*XwnpMa84zV+&i`pJZY_wJvq~>gY>cctk`@ut;(i4GoR8Nh5!@A6O8PnP4o^ z)v+HM7PHm%qR)x-IlRtxm@_?=A9*geBH=%T3`Qp;&|CE-Q%gxnDWIr?Y;0Jk#KzJb zA05d*g?v5l-uU*JBG$yj#O0zmWWLE8u4U*clOg6Sl&Sl$wOX@Cn;H$9OdeL^d~tc% z^Zh$rN=(de>jwd%WP(s>v6CSwZUKShNPUTp&d#1(rKDIoh_;5IVa8W(i?EZkv%FD0 zYDp<63q{4SqM{o zm^JdcbL67H%?dnyT(*mAMv!Lde0+V^z7W4yty%C>iiKpglr~=}Mw0M`VNZ5NrDcHk z#Ychoj*XA!=6(NukUdQe*55)`mqg*gOL)^!FN?Civ-Ka3m58HXgJQG_5bJud5WRl4 zSHq!RFO^fcWhN#jEY-G=^hX;#k3E7`O@>PhgzHQ@F=X)|e(%B5R)8>aNu(fi|Ks$z zsGxvK0dnEb%*^Z(*@5$-m{12t)7aEAF zp-fiG8axSJ=e!=a-+VnlJHXG$sn*fe_5XsChL-lz?Cvu>JUkj28p)u~pTmcS)To1S zk&uvRI5`ih!|%_)MiC@r*T+SCz~=dPBNY_juC1+QH#X|O(b5WkOvvsbopQD_tzm9y z85R_TGQ%W~{fg81uX3I3_;a}oy_YWoHYQ6mHx^o25FdGYdECRp!)%TGSG`-W6%;~m z;6`^e7>^&zn3#OKKpZP7DSe6`@FsaeM#idHtV^HwDqT_d;m*(IX0d4-mIAkpA&FNR zB0B6k71DdtzhD(j4)*r;U3n^Lt6=t5L-1(L!H{MpAK+`T;m_EXMNy39a&vPl-1p}r zJ*b5o7heiG%!{NXC0T7_Gcz*E4W#j_p6$*~;wdaow%qYPdQcWOD1NUGhWTBc9@)~; z(s(=|ytXx7Xklh%_Vr2Wa15>7YPL+cxv6P(@skewnmO4($PN_?ixMg->I`(fqTCGD z>pt%ZD}S_P8x|hEc04G~(cr%S9*%<8+;+CM^7sL7o9^$fMr|Nuv^^Nhg+c4Jo?hDFh!RIncenhk{^}?iD#o8X6LY z@5R#R5f}^>OD>tjZeZ@^B}5)5`S=-Y*$(Cog{W8e6UJ9+l#ra=sq*&NYfc1GJbe6Y z=uz_b@84IKgLsq%J}zP)pb1?Vts9htuANnTciDZLEOiav8L(>LzH6JT@J%-bY z&dbZo&A`y5jwL1_pv|sV{YBP+3rwzUvXJY~HT(_`%s*HH=7TYbO@9Q7fjIEII4~h; z=#<0LB(-7JuZcWj182EkB`^3Ob38Nw?#0dkBo|0*1`S`0Mm1$6?>Rtg0>2oiR?DM?9l9UWp<+}eWdY{}rDpyUZA@3S2R z(*Rt_ke!`(8c%}G2nZn5LA4-SReWKst9&JK;UH`~p;`uM&^F9R&jheAlyTM*R4&A#3mB*xm>MBbc>seb1T3n^|Z+`d;XTMQzT zA92L272A(bPZKhIjvhteOr9RJW5+-kO=2!BnJVtMWXy)S&nTSVK}8iGUHvtw&u1Af-OWz_2#9uCmJT z?25->x>zNNwCc()oEaetV3&;`$!)~GznW~x!)~w3@W1n1i?F@9IOKW0v2}5{!Yv}A zf7zu0_R*MyfGPrAwou2x6zFJZNPP_4nI!J+bq;%^d#PHHlQ`@ z2bxLC$+6hK+;2Lkh(PNtFofQf!zgn6+V9?Q^oV+$Sd(DKKWAf000}_R{fphZcVBW= zP3%BmstK)%c^pvnAoccViW_aJMDwLQEz37(aAUUT?CI&@AxdW8=2q7+*dNJ#>`H#- zdVhP)V#oUJ+qXWOfFJX|=LWg_Qfd4SdCl#9B&4J(fq{YXyz!qN6OO1E*LQSCTe#jO zwcOCVQk=ylCgv5zSTCd^=fV5>-e~Uo`uLc;y7IbG{*f?XGxS^|AVFs}J6QbPyLAU3 zxZt`h3W3!+D?^449we5#ZfjbcinFsvUg2EJD=5645U8A<49NWhc4Y5NfZGdHZKL&p z^b|~#s66P=`bw1eD`>#!?w`6iStN8GTmm{j4#|8i`+6mVHId zvaR7Kd9Xvn&#yI&rSbYT-fr8H&yLsP^z^i%pVdT>&T`#)hTZAHkAi~es;a6n0sY}D ziI4ilx1=@mUWJ9KnPL^SwNqTXIM#`AjH-$p931WYk^1aOfYpw!VQ&VLzE^85cTo3p&?k4Shw5wLx9c*x~bTf`JNRbu}=Ctj~R8h#EYg&&2_FK!EzM8-{4C1~Ui%H66h?z3>9G zrp}lS?f6HnQfNpBE?|)FPp^Zc#P5=4K0O;X+od7sS)#nZ_&l_7tQSqjDh?&$dzVb1 zpscB>sopST3RHu`=>ie|`>${q?4pX-vvO6rS~uaX3_Y2MM_G4oukH8(;8u=6@AL!) zK5*h{zBy8I+WRADOI89@hH?iC7XASl?G8b{itieHuTHoc=c38vvGZ)Lt%+s_-2wLKzqm8E) z7)+wkzbK>p+&)&FVIWKee+5~N9!n5p;@qP~c_O+84UC_|!Z6sydVY&Me{SsK(@1aW z6U!-&;--kbKA1^3K0aRIygtBgi`pY0NFX(tA?mH`CJ4bt4b7qlNz4Gi(b3K2aW&=i z$5mt9weZl;^6^euFlwr9DGsIByX!RmXtI)x)t)#yG;mRCc8`S2HA>G?*w)sTJT@IB z{s)UCpqZQ&9bS(EjMn~h;`Chw~%<+QIAn_x=+swQR zwS$~C-(A_Yv$(%`^E9F15h9b<^*j0a_^gHzu+;i##iXU#UGRdfl7I7U9;^sv=^rq= z`A{P%pjefal`D7lY}k{O*tN?7?5s!n-=6QyF?$!(fN*JNIWIG5FZiS7bg(EfovTEs z_pbbsbvd2-70gDwvd(EG5N68R_vwnF984dynD^Rd+gw>O3&(3;8wR^j$Ix(q z_crF+8u_zlvN}>tfF%;or&t>;zRVGAsd07L8dI>{FEn=4K|?`N8&GoqBsJKb&V9j` zKS9qKx7L#J+7oE# z={w^}%fDI70yP}VF$VAH;c8qV5_T(sY3?DU8sPWBLf|PH_Uhp)CC&y020w?_w%i&N zu+>yMF$V#o_cC8IZ1KD8ObVrVQu7BIF5vd;3^-EYZ(#ytv?|oTz~B@ z29V>dV7M(I8>CGr$U>1Vd)Mdt{Ee&*wM(e7*lMLkMSTaoR4y;Kc3Hf z4<2xT1uC!y>95-1$eUUIwis-3A-bHL9Q)&LMm|1G4C;~Qm8xJTcJI28-jM={7}?P3&~jV-mB%Jl!27HycES7Za=ml6W?_D- z<#^QI1W>H{W*|KjBgKV<_2}s6x&chU=t3(hxZC}uIyJwy5=0Qc=sNgzNO^aFCO60X z(@0=(IF%r&0&{VGuWU7Vu}YlK`60+u>op zjV7t<8Qud27=G*N?laASK&mm?r8Tk42`}ZRZHt^unbqIS0W0Qa$E!%M=2g8uyT$t7 z-Cec?j}@$j$;a*R9u0=4ZGrQi>*Ne=1~8!I%=hP;5*UKJySp>(e;&6Y<7^z&@H@== zbZ#F41z%xp^aLa=)t>y;6E)5VzPJQMzlc{^AgC0D`?9jAE;|X0Zi@P1V`E(|uQxY0 zwVEhE5-dAHaT*jopDOWdJe|-hDnzjw-Ft-<1wwwNB-;w)D{y5ld2UTj&7WGQ6#}>| z5Rvsa!DqgAZzD+gUwH-+3At?0bMzMP4a-sTHY!(cVA28J_K*G;+@d0{<0vK)z7TIiq-;_=^hI6JTmw<7{ z!zN*UZLS?$T~Yka30P9GWX4$OLayA6ZFW)itSi7bOz`vb%X9Gn{~nF1irYgyS65dz z>NWI~dba@w;b)ToGq1bM1qa4Y!q?L3byQ<9g6(T>Pfs076WPr@Xs1zkBdr57zA5MD%VIb zadwA;Y$`cimBV#@SypY_0H>&6+X+b9$_pVfVK?Q=SQ>FGF!u;IOk*o1jQH5iC>#^} zd-k#&hkruh&!4fDeOCl*H$f-R`@8-7lI?PRo87#>Q!V~2N4YXVJ(=~k@Khp-nMT@H zp*L&_zEy7}BB1}JKaD3+c8o~2q<7l3nE|KUlW7$lA)Gc*7H_>ht&f=?+a61w2+7>7 z;?vXFfPPipZOzWk25Q-&aNrW?jocF0Nxt~4LXusU>4UXX)4pV`q#bXQcwmCjuaBn6 z`2|p8lL=()83&F?WjzhSmYHaXW^Hn@o`3K|-`sEE~Ia2Jpr z^->v}QbG{5^n$Ial~%)w3>`Mx4>$8lO4wb2>`j!%R29FiELqPx&mf)A$1zA3aB8dM z{n=qb%CLDkwX>FFqzU~4xD_3$wiY5G{#-w~KQ+3~)iV?DxhUJ2Z)%{&m(ob;c+X%E z^?DZi#@&+b*w^z96wX$kp7~S3e+EUkwf>F$6Ji1aaLo|kPkzxa+MFtS8uI7%f)PK( z0q^alXxC>0qB<9T3EUm2X|7V;s<2gvo%i)_O#*KS$W*FHs7@fM`25<}_CDD#=@#pr znS50rT)kFcQW-fApDf~8@qXP6WPB#Qkm26zbO`Y65fbuEv1V#?be5hJ-*MVC0zy+eKSV;)~MwX^1NC9S3!+wCw}(?!&V*uwY?-=z~sx*YoDrf&L7FG z1^ngvgeR%xT5p0Zc?&*BDC$r)Q&UqH;e5arew-7U2A0MzKK+^a7+0wI$=MgIE*l4@%a$NFClxr~xl5+fK^!u#*r= zZ~$=8=k`N@x7ARum%pDAS|cA#WYhZ&l1B;01aMD_XFUi(+81J3YFBaQ~xPNeDsGZQ&^BNHCBnzeK9=d5z>@6KDaQi&?rJdv(7 zs{HplhcbR`|I9V|w$a4(>M@ae*9@v?-p}IVegN)rzhoV~{I38(W}mIJ$*J@!oR=f# z{Bow!IO1_W=kDt8a)v=lCGvW&VQ@^gsHv>7^G|Hzb9VN`-I*$ldNnhuhx2vf3~=&Y88bw$cYf7VmL|T7 zM^$oecRJf&D0P{mF&RQ-%2&-CXqnIGn}I8g0{TU)Sq|b zRGk-mhK}V(kx*D=h`HO~VK5Gg7nnqxi$BvXWn@0llNm&@J1>SZ>OYt}ujLmgTD%9gnib%9=RZ3*#s?=v;VPTs#K~NE~%hkAGve~=a z>)fCHMSdi}+@(Vk&;yzNTlkCt+Z{lr@v<&nzZ;LstL;*~l0&iElhNSBAJq?kzzV=> zaN4SWD;x6C53sQu%&_lu`|8kin)vEmCPzG*V2R)vJL{QOC)kU=#gqs*c!B^>IY zlhf02SAtiCqZ12UtbFgL7P%~J#+r#H!vX##bwCEQcLVc8K+0e}=8YNtlA*WEObZ;Q z3FiA~07A&f$QGWSf;~tX>t4qXgV2;0xns|YX3FGMR9t-o|Ni~^d~RBWs5XSom~^JO zDX$~w(G#+TgD>-YZ@kBrwfl8arS^wP$QSD zdMYVzFBXvV6|k~}4d@F~8y6Pqt>rM;AP`)gPiN{yBAtK(-VSHw;u?19 zSvsPBjk3*%^cRes2#P{^e9DYX#&ZqsrMqW4d>T?x?NdN|BY-z!7dyIYlF_jFjPw0{q4 zi?DuITvVidwlxkPyZXDfM(@5e^}7J6mcpJzT0<($;GG^2SBkv~vteV2AiOz*-FB?K z3|`ni)M<4BWlfc|gsaio!!A+_grX%asbN zPC4wIbFreLqPdOD4<-%{LPS-&)c>|nz8;IK*K`;oxuZzvx>Bav;0fIuD-A$kES`aK7exui-An`XFSzy#t53k$Pe z2{1$iRc|oMWSG(ImR8Y6vn5hQ1yN}{36S|-knv=(0_LBw>6-uOVh@U5T3Uj3h7s_1 zJWkvXSI>VP)jS2LVRevhIIA9MjcfQlp(;gPBl6U~DabjX()s2P{5)#}m zUc_}kraFO3&RX+K!#M}wG4yX=3iS5sw4%J6Yv9r38pvSK=?V)ds7N46krR{oKyE@8 zntka|Ti+vAJ#<(0)a!)Dv;+MTRHC{E#jgAK$kkn4tNOaTqY-5&`({2{#gn}`y~M=C zSRzP6Z-0N)m>;eeLpV?ya2SQRGKeN!zteFoKxl#WX6C;4hxz^jCf>U_nh&3v(uO58 z;7UePi0bkoO5Uy#yc1rNI=~!m1UXYiLgGWt;3&|5CtKr6C)<;%pidwa(OE%C$^!US z2NwUIETtwPP#^Mp`*yq)*p9C?H5qz)dpm$kVFA%A4FFMj6M6}TjBM;&l{$-`=2@GQk9WGj0DY1*ij}>5uNs*G))|Y~yl6na^toCv_ zjj=aqps0P{PK!<*)QA|wr5>owI8dU90p>NE*aT=o;97VvB&Dc`K~Ci~o<93#=Rgaj zI1#G>G6VM#K#dX+sq|46FNlcT+1)%TI=>La;(&^AyYM8Ctv$dSBB~!U14VSyVlwM* zE3hc|qbCeQTwWd**Vk}DLl21_@^jGmP-q@&3Fg2;LINoV9o&zb+Z-4~n@pmdfC4at zhlRzvWdE}4SND1kDnCbYs$$d3f&CAPszAXUmVmDJ@ef+nd z4n3=?+F@aUPz5{6d#_H8LflVZ=ksMROKGxKGs5@<;)zBA;w7)76o+Ut$-@p4^S$Z@ z702B4RYD^llh9A7#1asppFpUpC@2h{I!m>ET^s73iOvVz6p74evbX^Q%~xA2f}qeT zq?;xKEZtRLCaY{?R=JZTV&J0~2Y{V{=uHtaB=t#6{rZ&^ROpp=QHq74g5{J0B!GOO z2V6o^$n^0%f%w%%&QV9`lRS4Hcp}>`A04hh2xcs(+JhDuli6bSbbRoT-1PJ^ns49g zSK~6|a?2V6qN=!zkY*ow&09eI)*oRFnlB|5(IvfwhxD3=C$LN(3 zyFnMts3+-D(2fLM?g)h<*wXIMJaCm}n{rH0keGZ{N`rjYQ5F8_GZqf;ErqCbP$RNX zSI6flFU3b71~YTd#5F$X1DUP90dvYw2zUSjSI+-9N4=Y^E`~Y4E;n+bdco}kvo$KN z(PO}KQ`gniLGNxZfhPZ*s2vXLM#sTPQczGhs+u&6TkyLsej-dgYeWkMlmV2qr?Qeq z(4zP0?7lW3$V2(cDKw)aBj$d7%`3uWpuh#|nwz=^!uy4)5%|M+6%nNxEP+hXQ+c%axz?KRf{K4Y*ufTr_-q zXP-ktL-B@(hsR6~?~Pb|FV~VH1u6ag{mX&icLLrZ)K>x9F&6YNbYES0y*4u9xI6d1 z*V3ngXq8sSpqI0Xlwd}D+%<6kuo|?-P~RcQ%_zBpM9!+ zgIjUVKqwvxq;h1tv}JE<{Jkd{5`@hK@HCj8O_!E3WC(+Q-XkaH9BzyP{!j*zgxug; z5Eu|P^Ycq)5NfRu(Deen8DRv+Sr0)6+8Um>+lxR*fMs`o|7i$;hXO8SNSUMQBv!7d z1WwBUvCfmD@a&+)4+6a|e<~~qL-_z*msF^ZFo%jSs81nA^~b|M9!I4LNn@`Mf}LwG*tim>&?#X5 zJ0>t%NdFxaW<6WY7K#LBJgKYYcD5yk4AP3J>D#fo zzRn1g)PG;{SPo^B$3#3N3=ljTu75^mY+74gjV6Nx;B`cmTMyHK0i&t>zD)nJiX(!w z_EG?^Avctop8it@1mbo8dW#f5q_&Ie_=41}fC({X1Xb_~VATiH1%r&!b93VZ?l3jJ z+!+pX$^<>L%g)s6yxiQFr;t`=AkG;a>NoQ&8Bvnp^q(<_X%>Lx8l{54{u2k1&%(?c zlj(Q$W2(u!24OT6fOayFRS|ueZ7`S|5aEdQ?sdM$CL2k{-1UW+7J%g@CDQ+C%jp0A b=CYUK(ob{wS0;((5noV{c_m#UX&m@JoO(ZO literal 0 HcmV?d00001 diff --git a/third-party/benchmark/docs/assets/images/icon_black.xcf b/third-party/benchmark/docs/assets/images/icon_black.xcf new file mode 100644 index 0000000000000000000000000000000000000000..430e7bafe579b3d563b40f4bbf2faee7ead16848 GIT binary patch literal 36322 zcmeHw34B!5_5YnEne53VlRzLOAqfyjLIQ!vj#@0JXsw!`Hi{Nx5f@Og#-%!L)VNh_ zt%_AEO1IRi1za%IDzw%mR{zx&l}15DD9@*7c~qUW{mM%hxR>m_;)*65Y6KOC_qa){CQ*8@E1b! zqj}Ma;(Z=hh{hxG=HS$IbI+YVXI9;u3+B$mF2m0stX^>8CFjg4Uoiiixibf6RL`Ds z#{6^UUr=7HsP2L}Gp3g}=>M?6_;|)=d_1gN|M~I73(lK<#@u;x%7-;z>yl4x#W(7{ zHE-^P<#XqqaptTA{Kov*vlq;|uw0K%=sVp{AIxh``14Fo*g*G^**)9)2afkxa_-zS zE|_0FV-|k&-02sb%?q4+#{79h&zq@V;h6C+6^&ykX#LrrcqJgl8IAjSuJm{x$W$cs zOA`7S3H|bfer7^HE1{pA(9cDmzvuo1(BW_M`~%{BAXCM@;19Smp`SPpu-pCfp5gXz z8}6jydGY7H!@-vv9>>SVaf^cq_kRa{f4#HcKheRdoNxFG%yDpfZ5)q_fmF~ zb8zzn2cNdq!81;b;wEisQTE_`x{- zwS&L)j)O0LF^=Dg8$7399OgPU-LNnpas6?ZkZDU7CboYZN z<$29WiSAuCEhoh^O008EZklP3*tW%unP#ZO-kg(}ZE7X9y4ja&220HTa%8@#7TeJn zG6Th~o={|}#J+N9Z&M+$&GUT4rd(qCZp`g#%EUfb)6evi*sfMzsVNcrSXsI0BX;+^ zisVHkj6}?MWwc3=CX*(OCR2u)Y#D0uq}Jri5EGJWQzQdTZ>hrIRTydOEB!I5Wfi>_1e(Sw><5E_`X6cull)-fBBa zQcSG+#*IxlD4II8J=!Q4CRTCF_FcmnMH6t705h#b^@{ z>1;9?GQwoZaFZ+bCMb2LKx#~(49421#@ZN&wNb@K*58j`n8>9gTUhKoq=j_JG*zr5FBhjo=}7D+qbK0aZ3C9qB{JNU6x)0sygXM*RUMFk30iR%ro`X0f0c=qEV1Z5m0Ahn9Nvv<`E#(+*wL@!;yx^|FtN;AW9aT4Rbg!78@;f!kL!<< zUfkVVhP5tJA55_>q6*F+l}7tVP(epz6WA77oE&sm4JhSsP}%UA8C8( zTcxD*{kzJ*y6un;%1QnQe9)iFpmbe@6dT*OtCAcd_w_32Wo*i?2LQ>P1HnDyl0i~v z?4;-*uoPRisagt*O@DbXklZ!|?8Z*EL%_#u;fFOqvaA-YNWM`gxyBZJL?rhQ1(%Y; z>LnW&`6`jjZU6(bIUf?qwZkOS*btanrjZ+mgT>k6T|{z2qof;~y_HDLA0cTtK^Ku6 zHP$ZOxZbch7aR_b zNUqoj4k;Cc4!=xS4mvWxIW2SakVm z6H1{Kcc(Q@zvb6|EtY-VYaa|BU+D>%NYQaO{cdlSL^l1ZbzEAOiTD~@SI26htar2y zNj1^HF%N7V0mXb_z0b(jF%n&KRi&8#895A6KC$!5^PBy|JE;L|vlLHQ+Cz9|S$G~J zAl*pC7)U-7AP!+n%tXuz+q|U&YJn|%wlC&J%y*3~hQ45r-qIUiwgX=41r5Sx{=5j2 z#CjG|udv_jF2Dq_M{LW-{IGQ&Ladmbjvf8Gf)d;GonjnPwl*KejwN-~#`gMS0c4wB z)#Ll2*F)$otH&9O))Y!Cc6TrQM%nrziR}!h;a96ZfHbl82=wYc=`FTB1hM1Ledv93 zQl*g@Fp;zeq3BbzZBhu+i}dfTUz8@U=r zmgw0KI=oN!6MH?Sj?%5@E%lcgIiZw-6FN)KSr!0I%6`*RN`(CgRYgc~3buTR;K)Wea zfMqQYkOasv`GEHAv>;ZzTuNwn6styZAEAxatKRYup|wq_*E~v6A;VSQSxRUlrK%6z zL}<6BtD-bb;i^p2zf-nj5f9EV$%meN=*gA-WmxGTSCdLWq56V6^#N^?NxeahdVv%b zfdmzT{)9kx3P53kAS_Ue+Uu}Yt9$t=kK)n2@U3EyvtlD#2I!f1HlCFRR$2)wA#7B{ z8-BH-A4p~C{#LF=*>W%>8fYtE@~(s!LbEN~08^}H+(FQS*=W`stbz!c7Y)6RC{mz? zUsri{m1$SWc9m|b;ZBnZn#I_7Zsgt@pJX#E>#AeDuLw!#jlLiu}z=M&mKF&a)~wZd~1mJFjl`Z{Gpxv3#}EB~%f1v6}d zv9g7Rrtxa1rLopAHc1=JO=Hw_OY^N`Y#J3eHXRZdk7;m{3Ywgp<_giSXie>GtJb3uhyjZLqHd=hduplQQwAO8f(aIP(Nw|W~HTjg%R zvi{A%?glL9R}OVIAnx4P=x#uGdS;Zw@Iw%=x>U^CF_w~7fRdN=Q8-lL7=?!^oUHIz zg(oOHPT^4sCn+4Sa0Gmiix5sU8nk*Yej^7y$L*8xv-{v^+%OFC+J|s5{;?FY+8^Lx zd^{U++Cy+I-jf1(@Vn5IZ5YDXJm}ARTOf!XQx7-G#4I>+plWwj!x`Y&21@;W5nKe% zfuui_0p@xq7~&!?1m@XbU(s_g6t&RucY?uL&&RL~kQ^v6Sm zyWojz_*=4f!$NWwoHVc{@XAXOc9onrIuDZbCOL0F%aZiwUrTS&;Fh3kVyHG6BtN@f z4|j%pOuZgMA)NM7+5xEZlRHv-r;~v0_p$r+CF)Fnf1@+sapb4KZST749Vb2Ye_8|T zL8^1&&Uc!vX~A~n2Fe67o7Y2I1nm;EPskQtWltzsC9a%K7CKz$d?}M~%0v^*#e{1z z+jGaxTk^UUaky!%>2`Kxx;(JLbZ6-ypxL{Qh2c8;aTv9?Pl0*czZkCc_v`734_mn6 zUoC;n{PcaWsXvhooBH8f!OrjUgMANKmqmhKPk(s^c-6_nVDw&=CU*0~L zWLFam_gDWo*F-!K*;Iv)f$8%7JQLX{vIgIZ9=cDi0_$7j6}u`>X(B_mikycXzr*kS z%!8XMUWbsl)39ebLTj&ql{w=DpU70~Sds#ZIXIi|5qvTRJ8nsVZuC_!z3=0bYV5cI zm%qIp?C@J~Qb0G7a{^%A{lF?ujNw?Df?X}R+1GO6_#e3o2WXD zy@NeX@I3txT)5y3XwffYM^&cCMR5P8KAkSofgSmoINd3Exb3Al_U8x*@W!xf&sgyH zHXM9=zLAeH)SCyw)G`x}lWVbKEn+LWieQ?V62YDY*z*EHEMCljF{k#EY(zR?&*OMf zzwpBR)92kBk*{ITk0Gqx0REi)DkQG)*zx_K$T_ehrL4p!HP~?#o^$kTUXz?m$;p(Q zOi9Z0??{=#FoQS2?pzBu09hbo7#NGdxC8`FR^)2L0u9$1^3>J@RhHIrs(P@qZ!uxd zAS7B$fVQU=_jZ_kpzp>y4UB#s==)xrGFD;G^MSv5f&GvYe84I2Aozh(!EaN51MnWC zDlS1kALjKwX!)?IBlK)b09-u-{)05&OjrWafwOd90WLp?eFfn3(CpfBu|I@;IcOn- zT4bSxAg^R9&V%hW7u!KtX7hmaAzwfTyBIY4Ixynk4iNgcJy4{7fITCDYk_ZsLj63* z{5s%S*dBr-bM|1z0_=DgxC6Ka-`tytPqO!7&m`>W01g8$0Dcm92#(m{uQm~tAaI_# zN>`c)iJXGC2uHLiM(}L0yHP6gXvp1wKjz(BcLUm~O-pyT5cyi2Vj?PuAgp#u3c_>c zOoi7e{Gq~!6s}VECx!1S{7B&^3il|CDeP9bL*ZtH8x(dy^?n8+zc0dB68I^?eIKK) z8b+w^{qT#-=*A`70tPx|JK}j;;R&f*k6_;aqW|ON29c&LW3yJ&i-Z`DcROa<4lwO` zF&L+x2h%=rH;mDDfw7O?i1~CbqWvo0gbn#H^8VFyyu;&;Uxm->HAoDeaFxg%kS1nD z;JiN-5rvbt!%fv65r*~em5ICsJ4fGFF^=DZl#=~;KanF5EZ|=ZMSM?2xyoi3@kb!9 zosMt>X+ea-;aefN?a4s^!;p0dop})P4ZUB&Gdey4*-y#IlAJ8b$)Z#sNm>2}DN99M z*>)7_Jj4kyX3$Z<5AOGTMrS`m02m8Mr+XpPU&f`=x#+qfKM%_mIx7!uqrgbrn0dy_ zGgh8XOP-TXO4lLD-(lQf5N2;i6oN)}I@u|7_cP4Q$xx|;+FhYaH=`CAy~rp=D&7NB z>W~VCA~-<{nPA-V6?(&M>JA1=$})u)DI7y+1A7=zDXSH>Dg3&^DnjesK`VwlukeQo z=P8_`a1fzQvy8x$4GMp!aH+zZ6kehb(VIBt1ceh7)+(%2Sfa26uzf1??D}q z2@2;c{Dr~|8Vg^cqd86C9SYZKSbT+sA)Ko)`4ITBo&%A|Z5Rm_Chryq~UL@L!KV?^>LY@1ARR@8Q7vbMHG0Qku}9^L;vf5aRo(8qfM; zjc48L*Lc>iD!hl#ZcWvw)}s_IA+$SkG`Mw^!WXld>(`(WuD5A~>$V(?ZCyZU*QaDK zRj@_j%NfieJR+S5g)e&2m|b`jp?%pCVBX=^3GHkCRAwa3CA1$GsI+nmp^en|nZNj? zpDB%}`2g+aA}_NXf8u50g|;)C`ivy+Q{Z_14|<;aSF5zSYk(ML z4~R8Rb0ZIl-RMI%F_73viO7s6LuAC`y3+Tj7dq*NGTRZrTv5x36_Yk6cFgsMFs=+% zd9rcaYJyc^twaz73P|8Nm7rCD9_azyW*ABDTo{3+&DicDE+=s)vC&Oy+ z4a;p?@K@~EJ_7grmF_ICWP2ozw;O9 zpAw;A%+9csR^=Q9+{$H?UXc$8IaXRJ(b}_O^eEXgA}}uuLg$E2kd@FGK(FC_c%1IS zV=C!HXC^vH74pmc3C(49o=WZLREmB3t|1jVLu5lKgg~MF6 zYa9&O-tL(}kD6VQJ`koj`_cd=8KJTToon`?f{Ns4u6B%ut=mt$WoG_WdrEt9v_k-(&9JDW7FIixe#c;i3W4kw8FJz=e zaJ`U$7RGT14wqbP(+!gcoCnz<5BxL-QbG_o3*taNa0b{uv2>E(7qHhJKraMln*t6W z!o~VWuYf2#m;Yf$ZJj!lh2maqG3a7{egO@-lk5#48Qhmcc@@jWkJ zBDB_A#f@10^m&G>2SI2gW&Qt;17Vu z10%9tYJgV*7Xu@fUIM`PB35xHG8S(Fei!&M;8%g?0zVCW67Yk-M*!ai+z8wT+#k3V zI2U*pf*vCtbWVWBWh11*!+_rat_OY&xF7He;7s8A5em5n4w*&39{^tgjFJv=4)7}A z7T|}0rvTqmCDCgRE%W7@Xxfme9fM@^zO^eCw|#rT?3UvvH-GcY)~j!L;I|)yy?>JK zjknayd@|Z3(U0%`s^4d#g)@G!wLzk9-8wlX-9-H3uKvxgYKd-WKf5w$y8TV(J@CrD zQi*JN=KH7i?_>6+)cn`QcdmRB&b_^M;~yTn`J5>wgUr@IpP`3+wYmB9+23gX`lPYd zg`O}4;^cErKKJB$|I(~?a-$HqgAd$**o#K4<-s<&l_*Y31aSNte>>s8O?Uh>Ca3$` z4FP8`aTonpM}BTDX8H0LZ>|r2OI@OUoMBmAoUF#w#JKU zZ(`wHrx8MCWMhErJ`_N-<-76mhCnt(&TbhD_1W_z4mc|ZBWmC2g#nxcZE}4e2kCJL zsskS^-;U2`=0Yy8D|}Gq{j6Q|W=bxcRdOdTcp>=b(Rql^kohnT92vvWPcU4=#*Tx< zpk^D6j?@O>lCrx;BcPxF

0MSqK;0R0YdN>ThtL>$378lh`%+NCUVZks*@{ApY2A z0>~4%0-N`TAi3D39>f!$5AF8-f)J!1xyEPgnY1j#4ljg+6FJkEGh;aVx*}}Fy1#l6 zIQ@$#>4|WTz36T#Wz39?O=g)6X)T6ef>_C7)(%(|=mRGge!edxFuS3m#Gur`Ce(#Q zKESpS7~lfr18f_EQC|u(!In|zd>x&?4uvJ5?`BvB-WiCV_bK#V?1d+J5ju}#K;{^+ z51RH(KJ-HCkT$U6B$yMtkE8ciEdmp+LuY9YYzW6$sOQ_J!G=)#33{th*vak4>WQ;Dm`>BsT^eFas>74 zSzL)^5`@acHnsJEHL5jNV$aRUz>M-N%44|*-`zp@9&N@$`Iv&X==lOjdA9oP5bH}E zwxtlW-<}^Sf{W5-{g)|bvuYq~BJK|Gfz6a}4I2j+(o|yL`lUPU?p@MC0VJ!#K z&|e9gzgu{#3T6d*1Q;C4@?H#7smAZI03pi{*3jnCUW=<>P7FtE#O@5!m!e0D7 zYY0(JAcnp?n=pkQlt;A~VcIS&MtG!Fg}7Hs5~hBvB?*tw3KC1TFk#v@Elk*=btZnJ zv8AX%0*PSCw-r z%eggJM`*MeBhH*EtxZ_?jw{uTWOHNISagHUaiuzGW-QOa#(Evle*!)%vXOcXbm}tv z0V+d#&B1k&4+v}s9}xHeDg3=Dq$ak#kahR$mf58g@E;#QG7{@vluN?Fg78`_?3JTg zTgN^>mG(vZ>;zVmh;>f$lSaheo|nluh}hCOIr!fGy6lV zww(oiOf2o(4-m|MYq5#t&U+tBdsr{zI4;@=r^lOxCh9x-ktnqA+Y3yrZ|gfSd?O1m znm2uA490hq(TiqGZQqBII6)XS9z*P?wLYyVzE^AOSr|8l>4$*Bwcn!f2MXWd0+j|0 zNV-J>l6DqoDAGbgT(ZU@t@3Lu(s$ArgmktBAw`-s{3zO@fk)961|HdHi^d#9TNrZ` zjkajG5fD~*v<1908U}Yx4qIr5y88WFCHLaEO z=&_6oIqZR4KT!g3`feE2XN6F)P!{<;tF|*}iaegF;0%@8A10N0s6nVt-leSVLdD2#DBu`*D?*oVEJ17Fn`{RzV#hb#{^;Hza%Y zu$+p6O<$>1R0GSIK8`KkE{@b%s_84KZ(vJnsiv>gTB?ENg|vL7uVfZFKGTw_=_|FQ zYG65I4v(egsyqDiJocrf6V=Ruc`Z&^n@l6Dg;oR&tf0LIC)1Rt^rtkzDf?wLn3QbRgq~wLGV;q>=qbtTfKr|~ z9;)TL1vzO2aLaJgQt(;1Yi?mpN+`4kIHBLt<|J$|Q6u~Xc7Oxz0$;QZ{U_3_v9eKQ z7sE*xX=a&zEOb`v$5wyXP_e~2{@7B}LbQd|{jsI>$)hc_Fks6mP2QrV$s^5Lnmme- z8b;_eYf}_z<`Fty3VY;bpd8p=MY1K{Y^_sl6SVK)Fy8CFkl(gXYER!8mN{_lALo*qYAaU@5U)iz1C+ zGf~f|JNAr}ea7Ew)Hm;WC!BT(Pn^7+f7_=+8t=XX<-oF|pI2n`z~kKSC%#9;O{^nV zarIwv8)i+@j-86Be#yby#eb!Qdm3KD_&>XSx#&ONjJg`_1$OW^9WDM^SJQxg{8h92 zUuTyec z_}k;+IDVdQ-xuFr8pl72;|Cpl`H>F3;-omf#=+rR9eic{g|%0f#wq| zIDR6I<1Yd>j&CM22Ja)Kd`kV)0;myCL!h?cI9Hu_C9(biEWxdPAtm*L+~m9!$$cYI zIOA1jZFS-0bY0b8O2ff;XiW

e)9;h6VHi3Uk`jZLZ}Q2L=>K&Lm|Hm)u}HtDcP zX?7SS%LvlByk7>S?1)Gh)=(x`OP}W&^N?}x6Xnf|S(JW2%MU!W{2&u?Wfa66w&8g4ml#ihL;5I643_?U@#NZ%R=}6j0 z#t>+?xK_pC9s))}MoSLOIJDvLNLp`bx}n{MjWPhwNzvAAPOU|!-eQmcoyIlV*=TZe zUf|nPd6AYn8trJmlV(El53+5F0u&0EhOX*RkdXS3bFa?5Q+uK!<&2OsTIwS2B6P-) zDj=IP%6Yizu&1B#b~Hd5FNO3Y%^{o@GXih;gB8AudL2yiCJYz-M3(;P=y|&ZVhuVz zTlR6}9*kx)U)Wl!5)q4X@XZ%|0|wP+N=|Tr1ex~WG`=%~ukYeCt%M@LSGqc{b(JnB z9fBzl2d70OD%j4OKzfwy#P@-~5rVsO-wMJpbyaRx?RHgfSN(QXaA&Sbmpiu1a?dnj zgjTZhmo-67^6|QgxRRWSF0GvEju!4};;uICYUIvikS^z;xca%Pqq~pL{aIpX(mNOA zK6YJsSarOe#O6#JcltP(w|`*6z;sYc;)MI#jjvYuXWJK@+x`5&t8=#xyqm;3bsuCJ z7OL|eH|n&B? zuu9irv5_Q^+>vaNgw~4p%0aj*s80-4 z2&y$S+>2Ajx)>bp!}|3C`!ReBB2U2@^#iBkLQ;SOpm(W?OVH29&-FoLyfQLif{bU% z0Od;q&crP<&M8ax72pE-W*aC#ixD!>nv4AmQ_evPfhuRAh43IV73YDj<2d#qic2f zoxA+q1c=j-4I78&LLfn;bOPC zaU$;FlrG0~@TZB(fy?h-v;BG3;_$&l4)65c`*N{+-!6&lxg%*!yk{(k{c?jVOC(Z% zSLz^kB@tILaf!DpyEw9pEOus!I~N>gAgvB_aLEW6hw@iQUTONY5SY_SNrXJK0&9-x zhSCTp9GQ3!58H#s8N~YcD(;6Rmxb-VSaQrnEXI<{#&&NkIc6jlVaeqH7shc23oRGh zTz+}Ld02dTVA45QXhGmCEHZ{5W?+#KOJf?AR{?v?%!~|SI;H4BE5riJ!+8sF3qj=z z6kjMG{=KFY*2L;*edA&PC$Ka+;sGmYu~B4T)(IywhbN@^E7>rYwzIsQs3u3RO|;Xy zHhNxV2=~F#o5Wr;-VKX%@-9TN%Y_{L#oUep#=*@Uh0n(jwHmpYX>v+zT-*n@YCH>O z0ZO9}TtpTlK(wm?&x<9;5fjoe0QV-1$a67exC>4lr#WxqhU|v~wy!H*}&S&@P<3E~%0@$B<~18AHzoIMO)WfcqNyCUPJne#4$dBhT= zAHGBZegz`ZbcJixqnX?8{+k+)HMak!f_C-OMyD93Dp0NP5C zx~Gx%(f96(-Bp+^tj$1vN8z3%uo}NN7}qic*U2}GOEX_EPX5nPVi_VD>^QfO-O)j@ zp5J%EXe$->!kE*$pQ3~`Q3ivky!c?Dn*`kzxrsk5cGDr^5DAx=xa@-RBAY`b9GSx@ zEQ>f(5{gE|^5H5*D!U-bA|Zq+uerw2zo>h`J>Uq?6qTi;Ovlp^lBLzDH^&*v8BSS) zBoPuwYmO=oj?#rW)|#rDN^BaLW&vi%eR>Gb>(%KLypptIHI5KyM*_%h%N~($B|5!WR_z`d5gq#KVeNh zRDaFYUtJpYziH}aOef8~pB>Y|xQ + +## Disabling CPU Frequency Scaling + +If you see this error: + +``` +***WARNING*** CPU scaling is enabled, the benchmark real time measurements may be noisy and will incur extra overhead. +``` + +you might want to disable the CPU frequency scaling while running the +benchmark, as well as consider other ways to stabilize the performance of +your system while benchmarking. + +Exactly how to do this depends on the Linux distribution, +desktop environment, and installed programs. Specific details are a moving +target, so we will not attempt to exhaustively document them here. + +One simple option is to use the `cpupower` program to change the +performance governor to "performance". This tool is maintained along with +the Linux kernel and provided by your distribution. + +It must be run as root, like this: + +```bash +sudo cpupower frequency-set --governor performance +``` + +After this you can verify that all CPUs are using the performance governor +by running this command: + +```bash +cpupower frequency-info -o proc +``` + +The benchmarks you subsequently run will have less variance. + + + +## Reducing Variance in Benchmarks + +The Linux CPU frequency governor [discussed +above](user_guide#disabling-cpu-frequency-scaling) is not the only source +of noise in benchmarks. Some, but not all, of the sources of variance +include: + +1. On multi-core machines not all CPUs/CPU cores/CPU threads run the same + speed, so running a benchmark one time and then again may give a + different result depending on which CPU it ran on. +2. CPU scaling features that run on the CPU, like Intel's Turbo Boost and + AMD Turbo Core and Precision Boost, can temporarily change the CPU + frequency even when the using the "performance" governor on Linux. +3. Context switching between CPUs, or scheduling competition on the CPU the + benchmark is running on. +4. Intel Hyperthreading or AMD SMT causing the same issue as above. +5. Cache effects caused by code running on other CPUs. +6. Non-uniform memory architectures (NUMA). + +These can cause variance in benchmarks results within a single run +(`--benchmark_repetitions=N`) or across multiple runs of the benchmark +program. + +Reducing sources of variance is OS and architecture dependent, which is one +reason some companies maintain machines dedicated to performance testing. + +Some of the easier and effective ways of reducing variance on a typical +Linux workstation are: + +1. Use the performance governor as [discussed +above](user_guide#disabling-cpu-frequency-scaling). +1. Disable processor boosting by: + ```sh + echo 0 | sudo tee /sys/devices/system/cpu/cpufreq/boost + ``` + See the Linux kernel's + [boost.txt](https://www.kernel.org/doc/Documentation/cpu-freq/boost.txt) + for more information. +2. Set the benchmark program's task affinity to a fixed cpu. For example: + ```sh + taskset -c 0 ./mybenchmark + ``` +3. Disabling Hyperthreading/SMT. This can be done in the Bios or using the + `/sys` file system (see the LLVM project's [Benchmarking + tips](https://llvm.org/docs/Benchmarking.html)). +4. Close other programs that do non-trivial things based on timers, such as + your web browser, desktop environment, etc. +5. Reduce the working set of your benchmark to fit within the L1 cache, but + do be aware that this may lead you to optimize for an unrealistic + situation. + +Further resources on this topic: + +1. The LLVM project's [Benchmarking + tips](https://llvm.org/docs/Benchmarking.html). +1. The Arch Wiki [Cpu frequency +scaling](https://wiki.archlinux.org/title/CPU_frequency_scaling) page. diff --git a/third-party/benchmark/docs/releasing.md b/third-party/benchmark/docs/releasing.md index 334f93539381..09bf93764d00 100644 --- a/third-party/benchmark/docs/releasing.md +++ b/third-party/benchmark/docs/releasing.md @@ -1,30 +1,23 @@ # How to release * Make sure you're on main and synced to HEAD -* Ensure the project builds and tests run (sanity check only, obviously) +* Ensure the project builds and tests run * `parallel -j0 exec ::: test/*_test` can help ensure everything at least passes * Prepare release notes * `git log $(git describe --abbrev=0 --tags)..HEAD` gives you the list of commits between the last annotated tag and HEAD * Pick the most interesting. -* Create one last commit that updates the version saved in `CMakeLists.txt` and the - `__version__` variable in `bindings/python/google_benchmark/__init__.py`to the release - version you're creating. (This version will be used if benchmark is installed from the - archive you'll be creating in the next step.) +* Create one last commit that updates the version saved in `CMakeLists.txt` and `MODULE.bazel` + to the release version you're creating. (This version will be used if benchmark is installed + from the archive you'll be creating in the next step.) ``` -project (benchmark VERSION 1.6.0 LANGUAGES CXX) +project (benchmark VERSION 1.8.0 LANGUAGES CXX) ``` -```python -# bindings/python/google_benchmark/__init__.py - -# ... - -__version__ = "1.6.0" # <-- change this to the release version you are creating - -# ... +``` +module(name = "com_github_google_benchmark", version="1.8.0") ``` * Create a release through github's interface @@ -33,3 +26,6 @@ __version__ = "1.6.0" # <-- change this to the release version you are creating * `git pull --tags` * `git tag -a -f ` * `git push --force --tags origin` +* Confirm that the "Build and upload Python wheels" action runs to completion + * Run it manually if it hasn't run. + * IMPORTANT: When re-running manually, make sure to select the newly created `` as the workflow version in the "Run workflow" tab on the GitHub Actions page. diff --git a/third-party/benchmark/docs/tools.md b/third-party/benchmark/docs/tools.md index f2d0c497f3fc..411f41d405ff 100644 --- a/third-party/benchmark/docs/tools.md +++ b/third-party/benchmark/docs/tools.md @@ -186,6 +186,146 @@ Benchmark Time CPU Time Old This is a mix of the previous two modes, two (potentially different) benchmark binaries are run, and a different filter is applied to each one. As you can note, the values in `Time` and `CPU` columns are calculated as `(new - old) / |old|`. +### Note: Interpreting the output + +Performance measurements are an art, and performance comparisons are doubly so. +Results are often noisy and don't necessarily have large absolute differences to +them, so just by visual inspection, it is not at all apparent if two +measurements are actually showing a performance change or not. It is even more +confusing with multiple benchmark repetitions. + +Thankfully, what we can do, is use statistical tests on the results to determine +whether the performance has statistically-significantly changed. `compare.py` +uses [Mann–Whitney U +test](https://en.wikipedia.org/wiki/Mann%E2%80%93Whitney_U_test), with a null +hypothesis being that there's no difference in performance. + +**The below output is a summary of a benchmark comparison with statistics +provided for a multi-threaded process.** +``` +Benchmark Time CPU Time Old Time New CPU Old CPU New +----------------------------------------------------------------------------------------------------------------------------- +benchmark/threads:1/process_time/real_time_pvalue 0.0000 0.0000 U Test, Repetitions: 27 vs 27 +benchmark/threads:1/process_time/real_time_mean -0.1442 -0.1442 90 77 90 77 +benchmark/threads:1/process_time/real_time_median -0.1444 -0.1444 90 77 90 77 +benchmark/threads:1/process_time/real_time_stddev +0.3974 +0.3933 0 0 0 0 +benchmark/threads:1/process_time/real_time_cv +0.6329 +0.6280 0 0 0 0 +OVERALL_GEOMEAN -0.1442 -0.1442 0 0 0 0 +``` +-------------------------------------------- +Here's a breakdown of each row: + +**benchmark/threads:1/process_time/real_time_pvalue**: This shows the _p-value_ for +the statistical test comparing the performance of the process running with one +thread. A value of 0.0000 suggests a statistically significant difference in +performance. The comparison was conducted using the U Test (Mann-Whitney +U Test) with 27 repetitions for each case. + +**benchmark/threads:1/process_time/real_time_mean**: This shows the relative +difference in mean execution time between two different cases. The negative +value (-0.1442) implies that the new process is faster by about 14.42%. The old +time was 90 units, while the new time is 77 units. + +**benchmark/threads:1/process_time/real_time_median**: Similarly, this shows the +relative difference in the median execution time. Again, the new process is +faster by 14.44%. + +**benchmark/threads:1/process_time/real_time_stddev**: This is the relative +difference in the standard deviation of the execution time, which is a measure +of how much variation or dispersion there is from the mean. A positive value +(+0.3974) implies there is more variance in the execution time in the new +process. + +**benchmark/threads:1/process_time/real_time_cv**: CV stands for Coefficient of +Variation. It is the ratio of the standard deviation to the mean. It provides a +standardized measure of dispersion. An increase (+0.6329) indicates more +relative variability in the new process. + +**OVERALL_GEOMEAN**: Geomean stands for geometric mean, a type of average that is +less influenced by outliers. The negative value indicates a general improvement +in the new process. However, given the values are all zero for the old and new +times, this seems to be a mistake or placeholder in the output. + +----------------------------------------- + + + +Let's first try to see what the different columns represent in the above +`compare.py` benchmarking output: + + 1. **Benchmark:** The name of the function being benchmarked, along with the + size of the input (after the slash). + + 2. **Time:** The average time per operation, across all iterations. + + 3. **CPU:** The average CPU time per operation, across all iterations. + + 4. **Iterations:** The number of iterations the benchmark was run to get a + stable estimate. + + 5. **Time Old and Time New:** These represent the average time it takes for a + function to run in two different scenarios or versions. For example, you + might be comparing how fast a function runs before and after you make some + changes to it. + + 6. **CPU Old and CPU New:** These show the average amount of CPU time that the + function uses in two different scenarios or versions. This is similar to + Time Old and Time New, but focuses on CPU usage instead of overall time. + +In the comparison section, the relative differences in both time and CPU time +are displayed for each input size. + + +A statistically-significant difference is determined by a **p-value**, which is +a measure of the probability that the observed difference could have occurred +just by random chance. A smaller p-value indicates stronger evidence against the +null hypothesis. + +**Therefore:** + 1. If the p-value is less than the chosen significance level (alpha), we + reject the null hypothesis and conclude the benchmarks are significantly + different. + 2. If the p-value is greater than or equal to alpha, we fail to reject the + null hypothesis and treat the two benchmarks as similar. + + + +The result of said the statistical test is additionally communicated through color coding: +```diff ++ Green: +``` + The benchmarks are _**statistically different**_. This could mean the + performance has either **significantly improved** or **significantly + deteriorated**. You should look at the actual performance numbers to see which + is the case. +```diff +- Red: +``` + The benchmarks are _**statistically similar**_. This means the performance + **hasn't significantly changed**. + +In statistical terms, **'green'** means we reject the null hypothesis that +there's no difference in performance, and **'red'** means we fail to reject the +null hypothesis. This might seem counter-intuitive if you're expecting 'green' +to mean 'improved performance' and 'red' to mean 'worsened performance'. +```bash + But remember, in this context: + + 'Success' means 'successfully finding a difference'. + 'Failure' means 'failing to find a difference'. +``` + + +Also, please note that **even if** we determine that there **is** a +statistically-significant difference between the two measurements, it does not +_necessarily_ mean that the actual benchmarks that were measured **are** +different, or vice versa, even if we determine that there is **no** +statistically-significant difference between the two measurements, it does not +necessarily mean that the actual benchmarks that were measured **are not** +different. + + + ### U test If there is a sufficient repetition count of the benchmarks, the tool can do diff --git a/third-party/benchmark/docs/user_guide.md b/third-party/benchmark/docs/user_guide.md index 34bea6904240..d22a9069091f 100644 --- a/third-party/benchmark/docs/user_guide.md +++ b/third-party/benchmark/docs/user_guide.md @@ -28,6 +28,8 @@ [Templated Benchmarks](#templated-benchmarks) +[Templated Benchmarks that take arguments](#templated-benchmarks-with-arguments) + [Fixtures](#fixtures) [Custom Counters](#custom-counters) @@ -50,14 +52,19 @@ [Custom Statistics](#custom-statistics) +[Memory Usage](#memory-usage) + [Using RegisterBenchmark](#using-register-benchmark) [Exiting with an Error](#exiting-with-an-error) -[A Faster KeepRunning Loop](#a-faster-keep-running-loop) +[A Faster `KeepRunning` Loop](#a-faster-keep-running-loop) + +## Benchmarking Tips [Disabling CPU Frequency Scaling](#disabling-cpu-frequency-scaling) +[Reducing Variance in Benchmarks](reducing_variance.md) @@ -180,6 +187,12 @@ BM_memcpy/32 12 ns 12 ns 54687500 BM_memcpy/32k 1834 ns 1837 ns 357143 ``` +## Disabling Benchmarks + +It is possible to temporarily disable benchmarks by renaming the benchmark +function to have the prefix "DISABLED_". This will cause the benchmark to +be skipped at runtime. + ## Result comparison @@ -232,6 +245,19 @@ iterations is at least one, not more than 1e9, until CPU time is greater than the minimum time, or the wallclock time is 5x minimum time. The minimum time is set per benchmark by calling `MinTime` on the registered benchmark object. +Furthermore warming up a benchmark might be necessary in order to get +stable results because of e.g caching effects of the code under benchmark. +Warming up means running the benchmark a given amount of time, before +results are actually taken into account. The amount of time for which +the warmup should be run can be set per benchmark by calling +`MinWarmUpTime` on the registered benchmark object or for all benchmarks +using the `--benchmark_min_warmup_time` command-line option. Note that +`MinWarmUpTime` will overwrite the value of `--benchmark_min_warmup_time` +for the single benchmark. How many iterations the warmup run of each +benchmark takes is determined the same way as described in the paragraph +above. Per default the warmup phase is set to 0 seconds and is therefore +disabled. + Average timings are then reported over the iterations run. If multiple repetitions are requested using the `--benchmark_repetitions` command-line option, or at registration time, the benchmark function will be run several @@ -247,10 +273,12 @@ information about the machine on which the benchmarks are run. Global setup/teardown specific to each benchmark can be done by passing a callback to Setup/Teardown: -The setup/teardown callbacks will be invoked once for each benchmark. -If the benchmark is multi-threaded (will run in k threads), they will be invoked exactly once before -each run with k threads. -If the benchmark uses different size groups of threads, the above will be true for each size group. +The setup/teardown callbacks will be invoked once for each benchmark. If the +benchmark is multi-threaded (will run in k threads), they will be invoked +exactly once before each run with k threads. + +If the benchmark uses different size groups of threads, the above will be true +for each size group. Eg., @@ -293,7 +321,7 @@ static void BM_memcpy(benchmark::State& state) { delete[] src; delete[] dst; } -BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(1<<10)->Arg(8<<10); +BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(4<<10)->Arg(8<<10); ``` The preceding code is quite repetitive, and can be replaced with the following @@ -322,7 +350,8 @@ the performance of `std::vector` initialization for uniformly increasing sizes. static void BM_DenseRange(benchmark::State& state) { for(auto _ : state) { std::vector v(state.range(0), state.range(0)); - benchmark::DoNotOptimize(v.data()); + auto data = v.data(); + benchmark::DoNotOptimize(data); benchmark::ClobberMemory(); } } @@ -362,17 +391,17 @@ short-hand. The following macro will pick a few appropriate arguments in the product of the two specified ranges and will generate a benchmark for each such pair. -{% raw %} + ```c++ BENCHMARK(BM_SetInsert)->Ranges({{1<<10, 8<<10}, {128, 512}}); ``` -{% endraw %} + Some benchmarks may require specific argument values that cannot be expressed with `Ranges`. In this case, `ArgsProduct` offers the ability to generate a benchmark input for each combination in the product of the supplied vectors. -{% raw %} + ```c++ BENCHMARK(BM_SetInsert) ->ArgsProduct({{1<<10, 3<<10, 8<<10}, {20, 40, 60, 80}}) @@ -391,7 +420,7 @@ BENCHMARK(BM_SetInsert) ->Args({3<<10, 80}) ->Args({8<<10, 80}); ``` -{% endraw %} + For the most common scenarios, helper methods for creating a list of integers for a given sparse or dense range are provided. @@ -434,13 +463,22 @@ The `test_case_name` is appended to the name of the benchmark and should describe the values passed. ```c++ -template -void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) { - [...] +template +void BM_takes_args(benchmark::State& state, Args&&... args) { + auto args_tuple = std::make_tuple(std::move(args)...); + for (auto _ : state) { + std::cout << std::get<0>(args_tuple) << ": " << std::get<1>(args_tuple) + << '\n'; + [...] + } } // Registers a benchmark named "BM_takes_args/int_string_test" that passes -// the specified values to `extra_args`. +// the specified values to `args`. BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); + +// Registers the same benchmark "BM_takes_args/int_test" that passes +// the specified values to `args`. +BENCHMARK_CAPTURE(BM_takes_args, int_test, 42, 43); ``` Note that elements of `...args` may refer to global variables. Users should @@ -459,7 +497,8 @@ static void BM_StringCompare(benchmark::State& state) { std::string s1(state.range(0), '-'); std::string s2(state.range(0), '-'); for (auto _ : state) { - benchmark::DoNotOptimize(s1.compare(s2)); + auto comparison_result = s1.compare(s2); + benchmark::DoNotOptimize(comparison_result); } state.SetComplexityN(state.range(0)); } @@ -537,6 +576,30 @@ Three macros are provided for adding benchmark templates. #define BENCHMARK_TEMPLATE2(func, arg1, arg2) ``` + + +## Templated Benchmarks that take arguments + +Sometimes there is a need to template benchmarks, and provide arguments to them. + +```c++ +template void BM_Sequential_With_Step(benchmark::State& state, int step) { + Q q; + typename Q::value_type v; + for (auto _ : state) { + for (int i = state.range(0); i-=step; ) + q.push(v); + for (int e = state.range(0); e-=step; ) + q.Wait(&v); + } + // actually messages, not bytes: + state.SetBytesProcessed( + static_cast(state.iterations())*state.range(0)); +} + +BENCHMARK_TEMPLATE1_CAPTURE(BM_Sequential, WaitQueue, Step1, 1)->Range(1<<0, 1<<10); +``` + ## Fixtures @@ -554,10 +617,10 @@ For Example: ```c++ class MyFixture : public benchmark::Fixture { public: - void SetUp(const ::benchmark::State& state) { + void SetUp(::benchmark::State& state) { } - void TearDown(const ::benchmark::State& state) { + void TearDown(::benchmark::State& state) { } }; @@ -668,7 +731,7 @@ is 1k a 1000 (default, `benchmark::Counter::OneK::kIs1000`), or 1024 When you're compiling in C++11 mode or later you can use `insert()` with `std::initializer_list`: -{% raw %} + ```c++ // With C++11, this can be done: state.counters.insert({{"Foo", numFoos}, {"Bar", numBars}, {"Baz", numBazs}}); @@ -677,7 +740,7 @@ When you're compiling in C++11 mode or later you can use `insert()` with state.counters["Bar"] = numBars; state.counters["Baz"] = numBazs; ``` -{% endraw %} + ### Counter Reporting @@ -773,6 +836,16 @@ static void BM_MultiThreaded(benchmark::State& state) { BENCHMARK(BM_MultiThreaded)->Threads(2); ``` +To run the benchmark across a range of thread counts, instead of `Threads`, use +`ThreadRange`. This takes two parameters (`min_threads` and `max_threads`) and +runs the benchmark once for values in the inclusive range. For example: + +```c++ +BENCHMARK(BM_MultiThreaded)->ThreadRange(1, 8); +``` + +will run `BM_MultiThreaded` with thread counts 1, 2, 4, and 8. + If the benchmarked code itself uses threads and you want to compare it to single-threaded code, you may want to use real-time ("wallclock") measurements for latency comparisons: @@ -814,7 +887,7 @@ BENCHMARK(BM_OpenMP)->Range(8, 8<<10); // Measure the user-visible time, the wall clock (literally, the time that // has passed on the clock on the wall), use it to decide for how long to -// run the benchmark loop. This will always be meaningful, an will match the +// run the benchmark loop. This will always be meaningful, and will match the // time spent by the main thread in single-threaded case, in general decreasing // with the number of internal threads doing the work. BENCHMARK(BM_OpenMP)->Range(8, 8<<10)->UseRealTime(); @@ -836,7 +909,7 @@ is measured. But sometimes, it is necessary to do some work inside of that loop, every iteration, but without counting that time to the benchmark time. That is possible, although it is not recommended, since it has high overhead. -{% raw %} + ```c++ static void BM_SetInsert_With_Timer_Control(benchmark::State& state) { std::set data; @@ -851,7 +924,7 @@ static void BM_SetInsert_With_Timer_Control(benchmark::State& state) { } BENCHMARK(BM_SetInsert_With_Timer_Control)->Ranges({{1<<10, 8<<10}, {128, 512}}); ``` -{% endraw %} + @@ -906,6 +979,10 @@ order to manually set the time unit, you can specify it manually: BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); ``` +Additionally the default time unit can be set globally with the +`--benchmark_time_unit={ns|us|ms|s}` command line argument. The argument only +affects benchmarks where the time unit is not set explicitly. + ## Preventing Optimization @@ -958,7 +1035,8 @@ static void BM_vector_push_back(benchmark::State& state) { for (auto _ : state) { std::vector v; v.reserve(1); - benchmark::DoNotOptimize(v.data()); // Allow v.data() to be clobbered. + auto data = v.data(); // Allow v.data() to be clobbered. Pass as non-const + benchmark::DoNotOptimize(data); // lvalue to avoid undesired compiler optimizations v.push_back(42); benchmark::ClobberMemory(); // Force 42 to be written to memory. } @@ -1037,10 +1115,25 @@ void BM_spin_empty(benchmark::State& state) { BENCHMARK(BM_spin_empty) ->ComputeStatistics("ratio", [](const std::vector& v) -> double { return std::begin(v) / std::end(v); - }, benchmark::StatisticUnit::Percentage) + }, benchmark::StatisticUnit::kPercentage) ->Arg(512); ``` + + +## Memory Usage + +It's often useful to also track memory usage for benchmarks, alongside CPU +performance. For this reason, benchmark offers the `RegisterMemoryManager` +method that allows a custom `MemoryManager` to be injected. + +If set, the `MemoryManager::Start` and `MemoryManager::Stop` methods will be +called at the start and end of benchmark runs to allow user code to fill out +a report on the number of allocations, bytes used, etc. + +This data will then be reported alongside other performance data, currently +only when using JSON output. + ## Using RegisterBenchmark(name, fn, args...) @@ -1077,7 +1170,7 @@ int main(int argc, char** argv) { When errors caused by external influences, such as file I/O and network communication, occur within a benchmark the -`State::SkipWithError(const char* msg)` function can be used to skip that run +`State::SkipWithError(const std::string& msg)` function can be used to skip that run of benchmark and report the error. Note that only future iterations of the `KeepRunning()` are skipped. For the ranged-for version of the benchmark loop Users must explicitly exit the loop, otherwise all iterations will be performed. @@ -1188,13 +1281,12 @@ the benchmark loop should be preferred. If you see this error: ``` -***WARNING*** CPU scaling is enabled, the benchmark real time measurements may be noisy and will incur extra overhead. +***WARNING*** CPU scaling is enabled, the benchmark real time measurements may +be noisy and will incur extra overhead. ``` -you might want to disable the CPU frequency scaling while running the benchmark: +you might want to disable the CPU frequency scaling while running the +benchmark, as well as consider other ways to stabilize the performance of +your system while benchmarking. -```bash -sudo cpupower frequency-set --governor performance -./mybench -sudo cpupower frequency-set --governor powersave -``` +See [Reducing Variance](reducing_variance.md) for more information. diff --git a/third-party/benchmark/include/benchmark/benchmark.h b/third-party/benchmark/include/benchmark/benchmark.h index 6287c0afbdcf..08cfe29da344 100644 --- a/third-party/benchmark/include/benchmark/benchmark.h +++ b/third-party/benchmark/include/benchmark/benchmark.h @@ -187,6 +187,8 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #include #include +#include "benchmark/export.h" + #if defined(BENCHMARK_HAS_CXX11) #include #include @@ -216,37 +218,45 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #define BENCHMARK_UNUSED #endif +// Used to annotate functions, methods and classes so they +// are not optimized by the compiler. Useful for tests +// where you expect loops to stay in place churning cycles +#if defined(__clang__) +#define BENCHMARK_DONT_OPTIMIZE __attribute__((optnone)) +#elif defined(__GNUC__) || defined(__GNUG__) +#define BENCHMARK_DONT_OPTIMIZE __attribute__((optimize(0))) +#else +// MSVC & Intel do not have a no-optimize attribute, only line pragmas +#define BENCHMARK_DONT_OPTIMIZE +#endif + #if defined(__GNUC__) || defined(__clang__) #define BENCHMARK_ALWAYS_INLINE __attribute__((always_inline)) -#define BENCHMARK_NOEXCEPT noexcept -#define BENCHMARK_NOEXCEPT_OP(x) noexcept(x) #elif defined(_MSC_VER) && !defined(__clang__) #define BENCHMARK_ALWAYS_INLINE __forceinline -#if _MSC_VER >= 1900 -#define BENCHMARK_NOEXCEPT noexcept -#define BENCHMARK_NOEXCEPT_OP(x) noexcept(x) -#else -#define BENCHMARK_NOEXCEPT -#define BENCHMARK_NOEXCEPT_OP(x) -#endif #define __func__ __FUNCTION__ #else #define BENCHMARK_ALWAYS_INLINE -#define BENCHMARK_NOEXCEPT -#define BENCHMARK_NOEXCEPT_OP(x) #endif #define BENCHMARK_INTERNAL_TOSTRING2(x) #x #define BENCHMARK_INTERNAL_TOSTRING(x) BENCHMARK_INTERNAL_TOSTRING2(x) // clang-format off -#if defined(__GNUC__) || defined(__clang__) +#if (defined(__GNUC__) && !defined(__NVCC__) && !defined(__NVCOMPILER)) || defined(__clang__) #define BENCHMARK_BUILTIN_EXPECT(x, y) __builtin_expect(x, y) #define BENCHMARK_DEPRECATED_MSG(msg) __attribute__((deprecated(msg))) #define BENCHMARK_DISABLE_DEPRECATED_WARNING \ _Pragma("GCC diagnostic push") \ _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") #define BENCHMARK_RESTORE_DEPRECATED_WARNING _Pragma("GCC diagnostic pop") +#elif defined(__NVCOMPILER) +#define BENCHMARK_BUILTIN_EXPECT(x, y) __builtin_expect(x, y) +#define BENCHMARK_DEPRECATED_MSG(msg) __attribute__((deprecated(msg))) +#define BENCHMARK_DISABLE_DEPRECATED_WARNING \ + _Pragma("diagnostic push") \ + _Pragma("diag_suppress deprecated_entity_with_custom_message") +#define BENCHMARK_RESTORE_DEPRECATED_WARNING _Pragma("diagnostic pop") #else #define BENCHMARK_BUILTIN_EXPECT(x, y) x #define BENCHMARK_DEPRECATED_MSG(msg) @@ -280,18 +290,47 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #define BENCHMARK_OVERRIDE #endif +#if defined(_MSC_VER) +#pragma warning(push) +// C4251: needs to have dll-interface to be used by clients of class +#pragma warning(disable : 4251) +#endif + namespace benchmark { class BenchmarkReporter; -void Initialize(int* argc, char** argv); -void Shutdown(); +// Default number of minimum benchmark running time in seconds. +const char kDefaultMinTimeStr[] = "0.5s"; + +// Returns the version of the library. +BENCHMARK_EXPORT std::string GetBenchmarkVersion(); + +BENCHMARK_EXPORT void PrintDefaultHelp(); + +BENCHMARK_EXPORT void Initialize(int* argc, char** argv, + void (*HelperPrinterf)() = PrintDefaultHelp); +BENCHMARK_EXPORT void Shutdown(); // Report to stdout all arguments in 'argv' as unrecognized except the first. // Returns true there is at least on unrecognized argument (i.e. 'argc' > 1). -bool ReportUnrecognizedArguments(int argc, char** argv); +BENCHMARK_EXPORT bool ReportUnrecognizedArguments(int argc, char** argv); // Returns the current value of --benchmark_filter. -std::string GetBenchmarkFilter(); +BENCHMARK_EXPORT std::string GetBenchmarkFilter(); + +// Sets a new value to --benchmark_filter. (This will override this flag's +// current value). +// Should be called after `benchmark::Initialize()`, as +// `benchmark::Initialize()` will override the flag's value. +BENCHMARK_EXPORT void SetBenchmarkFilter(std::string value); + +// Returns the current value of --v (command line value for verbosity). +BENCHMARK_EXPORT int32_t GetBenchmarkVerbosity(); + +// Creates a default display reporter. Used by the library when no display +// reporter is provided, but also made available for external use in case a +// custom reporter should respect the `--benchmark_format` flag as a fallback +BENCHMARK_EXPORT BenchmarkReporter* CreateDefaultDisplayReporter(); // Generate a list of benchmarks matching the specified --benchmark_filter flag // and if --benchmark_list_tests is specified return after printing the name @@ -305,22 +344,33 @@ std::string GetBenchmarkFilter(); // The second and third overload use the specified 'display_reporter' and // 'file_reporter' respectively. 'file_reporter' will write to the file // specified -// by '--benchmark_output'. If '--benchmark_output' is not given the +// by '--benchmark_out'. If '--benchmark_out' is not given the // 'file_reporter' is ignored. // // RETURNS: The number of matching benchmarks. -size_t RunSpecifiedBenchmarks(); -size_t RunSpecifiedBenchmarks(std::string spec); +BENCHMARK_EXPORT size_t RunSpecifiedBenchmarks(); +BENCHMARK_EXPORT size_t RunSpecifiedBenchmarks(std::string spec); + +BENCHMARK_EXPORT size_t +RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter); +BENCHMARK_EXPORT size_t +RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, std::string spec); -size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter); -size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, - std::string spec); +BENCHMARK_EXPORT size_t RunSpecifiedBenchmarks( + BenchmarkReporter* display_reporter, BenchmarkReporter* file_reporter); +BENCHMARK_EXPORT size_t +RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, + BenchmarkReporter* file_reporter, std::string spec); -size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, - BenchmarkReporter* file_reporter); -size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, - BenchmarkReporter* file_reporter, - std::string spec); +// TimeUnit is passed to a benchmark in order to specify the order of magnitude +// for the measured time. +enum TimeUnit { kNanosecond, kMicrosecond, kMillisecond, kSecond }; + +BENCHMARK_EXPORT TimeUnit GetDefaultTimeUnit(); + +// Sets the default time unit the benchmarks use +// Has to be called before the benchmark loop to take effect +BENCHMARK_EXPORT void SetDefaultTimeUnit(TimeUnit unit); // If a MemoryManager is registered (via RegisterMemoryManager()), // it can be used to collect and report allocation metrics for a run of the @@ -358,20 +408,16 @@ class MemoryManager { virtual void Start() = 0; // Implement this to stop recording and fill out the given Result structure. - BENCHMARK_DEPRECATED_MSG("Use Stop(Result&) instead") - virtual void Stop(Result* result) = 0; - - // FIXME(vyng): Make this pure virtual once we've migrated current users. - BENCHMARK_DISABLE_DEPRECATED_WARNING - virtual void Stop(Result& result) { Stop(&result); } - BENCHMARK_RESTORE_DEPRECATED_WARNING + virtual void Stop(Result& result) = 0; }; // Register a MemoryManager instance that will be used to collect and report // allocation measurements for benchmark runs. +BENCHMARK_EXPORT void RegisterMemoryManager(MemoryManager* memory_manager); // Add a key-value pair to output as part of the context stanza in the report. +BENCHMARK_EXPORT void AddCustomContext(const std::string& key, const std::string& value); namespace internal { @@ -379,14 +425,17 @@ class Benchmark; class BenchmarkImp; class BenchmarkFamilies; +BENCHMARK_EXPORT std::map*& GetGlobalContext(); + +BENCHMARK_EXPORT void UseCharPointer(char const volatile*); // Take ownership of the pointer and register the benchmark. Return the // registered benchmark. -Benchmark* RegisterBenchmarkInternal(Benchmark*); +BENCHMARK_EXPORT Benchmark* RegisterBenchmarkInternal(Benchmark*); // Ensure that the standard streams are properly initialized in every TU. -int InitializeStreams(); +BENCHMARK_EXPORT int InitializeStreams(); BENCHMARK_UNUSED static int stream_init_anchor = InitializeStreams(); } // namespace internal @@ -409,7 +458,11 @@ inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { // intended to add little to no overhead. // See: https://youtu.be/nXaxk27zwlk?t=2441 #ifndef BENCHMARK_HAS_NO_INLINE_ASSEMBLY +#if !defined(__GNUC__) || defined(__llvm__) || defined(__INTEL_COMPILER) template +BENCHMARK_DEPRECATED_MSG( + "The const-ref version of this method can permit " + "undesired compiler optimizations in benchmarks") inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { asm volatile("" : : "r,m"(value) : "memory"); } @@ -423,6 +476,98 @@ inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp& value) { #endif } +#ifdef BENCHMARK_HAS_CXX11 +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp&& value) { +#if defined(__clang__) + asm volatile("" : "+r,m"(value) : : "memory"); +#else + asm volatile("" : "+m,r"(value) : : "memory"); +#endif +} +#endif +#elif defined(BENCHMARK_HAS_CXX11) && (__GNUC__ >= 5) +// Workaround for a bug with full argument copy overhead with GCC. +// See: #1340 and https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105519 +template +BENCHMARK_DEPRECATED_MSG( + "The const-ref version of this method can permit " + "undesired compiler optimizations in benchmarks") +inline BENCHMARK_ALWAYS_INLINE + typename std::enable_if::value && + (sizeof(Tp) <= sizeof(Tp*))>::type + DoNotOptimize(Tp const& value) { + asm volatile("" : : "r,m"(value) : "memory"); +} + +template +BENCHMARK_DEPRECATED_MSG( + "The const-ref version of this method can permit " + "undesired compiler optimizations in benchmarks") +inline BENCHMARK_ALWAYS_INLINE + typename std::enable_if::value || + (sizeof(Tp) > sizeof(Tp*))>::type + DoNotOptimize(Tp const& value) { + asm volatile("" : : "m"(value) : "memory"); +} + +template +inline BENCHMARK_ALWAYS_INLINE + typename std::enable_if::value && + (sizeof(Tp) <= sizeof(Tp*))>::type + DoNotOptimize(Tp& value) { + asm volatile("" : "+m,r"(value) : : "memory"); +} + +template +inline BENCHMARK_ALWAYS_INLINE + typename std::enable_if::value || + (sizeof(Tp) > sizeof(Tp*))>::type + DoNotOptimize(Tp& value) { + asm volatile("" : "+m"(value) : : "memory"); +} + +template +inline BENCHMARK_ALWAYS_INLINE + typename std::enable_if::value && + (sizeof(Tp) <= sizeof(Tp*))>::type + DoNotOptimize(Tp&& value) { + asm volatile("" : "+m,r"(value) : : "memory"); +} + +template +inline BENCHMARK_ALWAYS_INLINE + typename std::enable_if::value || + (sizeof(Tp) > sizeof(Tp*))>::type + DoNotOptimize(Tp&& value) { + asm volatile("" : "+m"(value) : : "memory"); +} + +#else +// Fallback for GCC < 5. Can add some overhead because the compiler is forced +// to use memory operations instead of operations with registers. +// TODO: Remove if GCC < 5 will be unsupported. +template +BENCHMARK_DEPRECATED_MSG( + "The const-ref version of this method can permit " + "undesired compiler optimizations in benchmarks") +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { + asm volatile("" : : "m"(value) : "memory"); +} + +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp& value) { + asm volatile("" : "+m"(value) : : "memory"); +} + +#ifdef BENCHMARK_HAS_CXX11 +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp&& value) { + asm volatile("" : "+m"(value) : : "memory"); +} +#endif +#endif + #ifndef BENCHMARK_HAS_CXX11 inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { asm volatile("" : : : "memory"); @@ -430,6 +575,9 @@ inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { #endif #elif defined(_MSC_VER) template +BENCHMARK_DEPRECATED_MSG( + "The const-ref version of this method can permit " + "undesired compiler optimizations in benchmarks") inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { internal::UseCharPointer(&reinterpret_cast(value)); _ReadWriteBarrier(); @@ -439,10 +587,25 @@ inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { _ReadWriteBarrier(); } #endif #else +#ifdef BENCHMARK_HAS_CXX11 +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp&& value) { + internal::UseCharPointer(&reinterpret_cast(value)); +} +#else template +BENCHMARK_DEPRECATED_MSG( + "The const-ref version of this method can permit " + "undesired compiler optimizations in benchmarks") inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { internal::UseCharPointer(&reinterpret_cast(value)); } + +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp& value) { + internal::UseCharPointer(&reinterpret_cast(value)); +} +#endif // FIXME Add ClobberMemory() for non-gnu and non-msvc compilers, before C++11. #endif @@ -506,23 +669,21 @@ Counter::Flags inline operator|(const Counter::Flags& LHS, // This is the container for the user-defined counters. typedef std::map UserCounters; -// TimeUnit is passed to a benchmark in order to specify the order of magnitude -// for the measured time. -enum TimeUnit { kNanosecond, kMicrosecond, kMillisecond, kSecond }; - // BigO is passed to a benchmark in order to specify the asymptotic // computational // complexity for the benchmark. In case oAuto is selected, complexity will be // calculated automatically to the best fit. enum BigO { oNone, o1, oN, oNSquared, oNCubed, oLogN, oNLogN, oAuto, oLambda }; -typedef uint64_t IterationCount; +typedef int64_t ComplexityN; + +typedef int64_t IterationCount; enum StatisticUnit { kTime, kPercentage }; // BigOFunc is passed to a benchmark in order to specify the asymptotic // computational complexity for the benchmark. -typedef double(BigOFunc)(IterationCount); +typedef double(BigOFunc)(ComplexityN); // StatisticsFunc is passed to a benchmark in order to compute some descriptive // statistics over all the measurements of some type @@ -564,11 +725,21 @@ enum AggregationReportMode ARM_FileReportAggregatesOnly | ARM_DisplayReportAggregatesOnly }; +enum Skipped +#if defined(BENCHMARK_HAS_CXX11) + : unsigned +#endif +{ + NotSkipped = 0, + SkippedWithMessage, + SkippedWithError +}; + } // namespace internal // State is passed to a running Benchmark and contains state for the // benchmark to use. -class State { +class BENCHMARK_EXPORT State { public: struct StateIterator; friend struct StateIterator; @@ -580,13 +751,13 @@ class State { // have been called previously. // // NOTE: KeepRunning may not be used after calling either of these functions. - BENCHMARK_ALWAYS_INLINE StateIterator begin(); - BENCHMARK_ALWAYS_INLINE StateIterator end(); + inline BENCHMARK_ALWAYS_INLINE StateIterator begin(); + inline BENCHMARK_ALWAYS_INLINE StateIterator end(); // Returns true if the benchmark should continue through another iteration. // NOTE: A benchmark may not return from the test until KeepRunning() has // returned false. - bool KeepRunning(); + inline bool KeepRunning(); // Returns true iff the benchmark should run n more iterations. // REQUIRES: 'n' > 0. @@ -598,10 +769,10 @@ class State { // while (state.KeepRunningBatch(1000)) { // // process 1000 elements // } - bool KeepRunningBatch(IterationCount n); + inline bool KeepRunningBatch(IterationCount n); - // REQUIRES: timer is running and 'SkipWithError(...)' has not been called - // by the current thread. + // REQUIRES: timer is running and 'SkipWithMessage(...)' or + // 'SkipWithError(...)' has not been called by the current thread. // Stop the benchmark timer. If not called, the timer will be // automatically stopped after the last iteration of the benchmark loop. // @@ -616,8 +787,8 @@ class State { // within each benchmark iteration, if possible. void PauseTiming(); - // REQUIRES: timer is not running and 'SkipWithError(...)' has not been called - // by the current thread. + // REQUIRES: timer is not running and 'SkipWithMessage(...)' or + // 'SkipWithError(...)' has not been called by the current thread. // Start the benchmark timer. The timer is NOT running on entrance to the // benchmark function. It begins running after control flow enters the // benchmark loop. @@ -627,8 +798,30 @@ class State { // within each benchmark iteration, if possible. void ResumeTiming(); - // REQUIRES: 'SkipWithError(...)' has not been called previously by the - // current thread. + // REQUIRES: 'SkipWithMessage(...)' or 'SkipWithError(...)' has not been + // called previously by the current thread. + // Report the benchmark as resulting in being skipped with the specified + // 'msg'. + // After this call the user may explicitly 'return' from the benchmark. + // + // If the ranged-for style of benchmark loop is used, the user must explicitly + // break from the loop, otherwise all future iterations will be run. + // If the 'KeepRunning()' loop is used the current thread will automatically + // exit the loop at the end of the current iteration. + // + // For threaded benchmarks only the current thread stops executing and future + // calls to `KeepRunning()` will block until all threads have completed + // the `KeepRunning()` loop. If multiple threads report being skipped only the + // first skip message is used. + // + // NOTE: Calling 'SkipWithMessage(...)' does not cause the benchmark to exit + // the current scope immediately. If the function is called from within + // the 'KeepRunning()' loop the current iteration will finish. It is the users + // responsibility to exit the scope as needed. + void SkipWithMessage(const std::string& msg); + + // REQUIRES: 'SkipWithMessage(...)' or 'SkipWithError(...)' has not been + // called previously by the current thread. // Report the benchmark as resulting in an error with the specified 'msg'. // After this call the user may explicitly 'return' from the benchmark. // @@ -646,10 +839,13 @@ class State { // the current scope immediately. If the function is called from within // the 'KeepRunning()' loop the current iteration will finish. It is the users // responsibility to exit the scope as needed. - void SkipWithError(const char* msg); + void SkipWithError(const std::string& msg); + + // Returns true if 'SkipWithMessage(...)' or 'SkipWithError(...)' was called. + bool skipped() const { return internal::NotSkipped != skipped_; } // Returns true if an error has been reported with 'SkipWithError(...)'. - bool error_occurred() const { return error_occurred_; } + bool error_occurred() const { return internal::SkippedWithError == skipped_; } // REQUIRES: called exactly once per iteration of the benchmarking loop. // Set the manually measured time for this benchmark iteration, which @@ -684,10 +880,12 @@ class State { // and complexity_n will // represent the length of N. BENCHMARK_ALWAYS_INLINE - void SetComplexityN(int64_t complexity_n) { complexity_n_ = complexity_n; } + void SetComplexityN(ComplexityN complexity_n) { + complexity_n_ = complexity_n; + } BENCHMARK_ALWAYS_INLINE - int64_t complexity_length_n() const { return complexity_n_; } + ComplexityN complexity_length_n() const { return complexity_n_; } // If this routine is called with items > 0, then an items/s // label is printed on the benchmark report line for the currently @@ -720,11 +918,7 @@ class State { // BM_Compress 50 50 14115038 compress:27.3% // // REQUIRES: a benchmark has exited its benchmarking loop. - void SetLabel(const char* label); - - void BENCHMARK_ALWAYS_INLINE SetLabel(const std::string& str) { - this->SetLabel(str.c_str()); - } + void SetLabel(const std::string& label); // Range arguments for this run. CHECKs if the argument has been set. BENCHMARK_ALWAYS_INLINE @@ -755,6 +949,9 @@ class State { return max_iterations - total_iterations_ + batch_leftover_; } + BENCHMARK_ALWAYS_INLINE + std::string name() const { return name_; } + private: // items we expect on the first cache line (ie 64 bytes of the struct) // When total_iterations_ is 0, KeepRunning() and friends will return false. @@ -772,29 +969,30 @@ class State { private: bool started_; bool finished_; - bool error_occurred_; + internal::Skipped skipped_; // items we don't need on the first cache line std::vector range_; - int64_t complexity_n_; + ComplexityN complexity_n_; public: // Container for user-defined counters. UserCounters counters; private: - State(IterationCount max_iters, const std::vector& ranges, - int thread_i, int n_threads, internal::ThreadTimer* timer, - internal::ThreadManager* manager, + State(std::string name, IterationCount max_iters, + const std::vector& ranges, int thread_i, int n_threads, + internal::ThreadTimer* timer, internal::ThreadManager* manager, internal::PerfCountersMeasurement* perf_counters_measurement); void StartKeepRunning(); // Implementation of KeepRunning() and KeepRunningBatch(). // is_batch must be true unless n is 1. - bool KeepRunningInternal(IterationCount n, bool is_batch); + inline bool KeepRunningInternal(IterationCount n, bool is_batch); void FinishKeepRunning(); + const std::string name_; const int thread_index_; const int threads_; @@ -826,7 +1024,7 @@ inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningInternal(IterationCount n, } if (!started_) { StartKeepRunning(); - if (!error_occurred_ && total_iterations_ >= n) { + if (!skipped() && total_iterations_ >= n) { total_iterations_ -= n; return true; } @@ -856,7 +1054,7 @@ struct State::StateIterator { BENCHMARK_ALWAYS_INLINE explicit StateIterator(State* st) - : cached_(st->error_occurred_ ? 0 : st->max_iterations), parent_(st) {} + : cached_(st->skipped() ? 0 : st->max_iterations), parent_(st) {} public: BENCHMARK_ALWAYS_INLINE @@ -899,7 +1097,7 @@ typedef void(Function)(State&); // be called on this object to change the properties of the benchmark. // Each method returns "this" so that multiple method calls can // chained into one expression. -class Benchmark { +class BENCHMARK_EXPORT Benchmark { public: virtual ~Benchmark(); @@ -971,7 +1169,7 @@ class Benchmark { // Have "setup" and/or "teardown" invoked once for every benchmark run. // If the benchmark is multi-threaded (will run in k threads concurrently), - // the setup callback will be invoked exactly once (not k times) before + // the setup callback will be be invoked exactly once (not k times) before // each run with k threads. Time allowing (e.g. for a short benchmark), there // may be multiple such runs per benchmark, each run with its own // "setup"/"teardown". @@ -1000,12 +1198,19 @@ class Benchmark { // REQUIRES: `t > 0` and `Iterations` has not been called on this benchmark. Benchmark* MinTime(double t); + // Set the minimum amount of time to run the benchmark before taking runtimes + // of this benchmark into account. This + // option overrides the `benchmark_min_warmup_time` flag. + // REQUIRES: `t >= 0` and `Iterations` has not been called on this benchmark. + Benchmark* MinWarmUpTime(double t); + // Specify the amount of iterations that should be run by this benchmark. + // This option overrides the `benchmark_min_time` flag. // REQUIRES: 'n > 0' and `MinTime` has not been called on this benchmark. // // NOTE: This function should only be used when *exact* iteration control is // needed and never to control or limit how long a benchmark runs, where - // `--benchmark_min_time=N` or `MinTime(...)` should be used instead. + // `--benchmark_min_time=s` or `MinTime(...)` should be used instead. Benchmark* Iterations(IterationCount n); // Specify the amount of times to repeat this benchmark. This option overrides @@ -1025,7 +1230,7 @@ class Benchmark { // By default, the CPU time is measured only for the main thread, which may // be unrepresentative if the benchmark uses threads internally. If called, // the total CPU time spent by all the threads will be measured instead. - // By default, the only the main thread CPU time will be measured. + // By default, only the main thread CPU time will be measured. Benchmark* MeasureProcessCPUTime(); // If a particular benchmark should use the Wall clock instead of the CPU time @@ -1090,12 +1295,16 @@ class Benchmark { virtual void Run(State& state) = 0; + TimeUnit GetTimeUnit() const; + protected: - explicit Benchmark(const char* name); - Benchmark(Benchmark const&); - void SetName(const char* name); + explicit Benchmark(const std::string& name); + void SetName(const std::string& name); + public: + const char* GetName() const; int ArgsCnt() const; + const char* GetArgName(int arg) const; private: friend class BenchmarkFamilies; @@ -1105,9 +1314,13 @@ class Benchmark { AggregationReportMode aggregation_report_mode_; std::vector arg_names_; // Args for all benchmark runs std::vector > args_; // Args for all benchmark runs + TimeUnit time_unit_; + bool use_default_time_unit_; + int range_multiplier_; double min_time_; + double min_warmup_time_; IterationCount iterations_; int repetitions_; bool measure_process_cpu_time_; @@ -1122,7 +1335,17 @@ class Benchmark { callback_function setup_; callback_function teardown_; - Benchmark& operator=(Benchmark const&); + Benchmark(Benchmark const&) +#if defined(BENCHMARK_HAS_CXX11) + = delete +#endif + ; + + Benchmark& operator=(Benchmark const&) +#if defined(BENCHMARK_HAS_CXX11) + = delete +#endif + ; }; } // namespace internal @@ -1131,27 +1354,27 @@ class Benchmark { // the specified functor 'fn'. // // RETURNS: A pointer to the registered benchmark. -internal::Benchmark* RegisterBenchmark(const char* name, +internal::Benchmark* RegisterBenchmark(const std::string& name, internal::Function* fn); #if defined(BENCHMARK_HAS_CXX11) template -internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn); +internal::Benchmark* RegisterBenchmark(const std::string& name, Lambda&& fn); #endif // Remove all registered benchmarks. All pointers to previously registered // benchmarks are invalidated. -void ClearRegisteredBenchmarks(); +BENCHMARK_EXPORT void ClearRegisteredBenchmarks(); namespace internal { // The class used to hold all Benchmarks created from static function. // (ie those created using the BENCHMARK(...) macros. -class FunctionBenchmark : public Benchmark { +class BENCHMARK_EXPORT FunctionBenchmark : public Benchmark { public: - FunctionBenchmark(const char* name, Function* func) + FunctionBenchmark(const std::string& name, Function* func) : Benchmark(name), func_(func) {} - virtual void Run(State& st) BENCHMARK_OVERRIDE; + void Run(State& st) BENCHMARK_OVERRIDE; private: Function* func_; @@ -1161,35 +1384,38 @@ class FunctionBenchmark : public Benchmark { template class LambdaBenchmark : public Benchmark { public: - virtual void Run(State& st) BENCHMARK_OVERRIDE { lambda_(st); } + void Run(State& st) BENCHMARK_OVERRIDE { lambda_(st); } private: template - LambdaBenchmark(const char* name, OLambda&& lam) + LambdaBenchmark(const std::string& name, OLambda&& lam) : Benchmark(name), lambda_(std::forward(lam)) {} LambdaBenchmark(LambdaBenchmark const&) = delete; template // NOLINTNEXTLINE(readability-redundant-declaration) - friend Benchmark* ::benchmark::RegisterBenchmark(const char*, Lam&&); + friend Benchmark* ::benchmark::RegisterBenchmark(const std::string&, Lam&&); Lambda lambda_; }; #endif - } // namespace internal -inline internal::Benchmark* RegisterBenchmark(const char* name, +inline internal::Benchmark* RegisterBenchmark(const std::string& name, internal::Function* fn) { + // FIXME: this should be a `std::make_unique<>()` but we don't have C++14. + // codechecker_intentional [cplusplus.NewDeleteLeaks] return internal::RegisterBenchmarkInternal( ::new internal::FunctionBenchmark(name, fn)); } #ifdef BENCHMARK_HAS_CXX11 template -internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn) { +internal::Benchmark* RegisterBenchmark(const std::string& name, Lambda&& fn) { using BenchType = internal::LambdaBenchmark::type>; + // FIXME: this should be a `std::make_unique<>()` but we don't have C++14. + // codechecker_intentional [cplusplus.NewDeleteLeaks] return internal::RegisterBenchmarkInternal( ::new BenchType(name, std::forward(fn))); } @@ -1198,7 +1424,7 @@ internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn) { #if defined(BENCHMARK_HAS_CXX11) && \ (!defined(BENCHMARK_GCC_VERSION) || BENCHMARK_GCC_VERSION >= 409) template -internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn, +internal::Benchmark* RegisterBenchmark(const std::string& name, Lambda&& fn, Args&&... args) { return benchmark::RegisterBenchmark( name, [=](benchmark::State& st) { fn(st, args...); }); @@ -1212,7 +1438,7 @@ class Fixture : public internal::Benchmark { public: Fixture() : internal::Benchmark("") {} - virtual void Run(State& st) BENCHMARK_OVERRIDE { + void Run(State& st) BENCHMARK_OVERRIDE { this->SetUp(st); this->BenchmarkCase(st); this->TearDown(st); @@ -1228,7 +1454,6 @@ class Fixture : public internal::Benchmark { protected: virtual void BenchmarkCase(State&) = 0; }; - } // namespace benchmark // ------------------------------------------------------ @@ -1268,7 +1493,7 @@ class Fixture : public internal::Benchmark { BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ (::benchmark::internal::RegisterBenchmarkInternal( \ new ::benchmark::internal::FunctionBenchmark(#__VA_ARGS__, \ - &__VA_ARGS__))) + __VA_ARGS__))) #else #define BENCHMARK(n) \ BENCHMARK_PRIVATE_DECLARE(n) = \ @@ -1298,7 +1523,7 @@ class Fixture : public internal::Benchmark { // /* Registers a benchmark named "BM_takes_args/int_string_test` */ // BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); #define BENCHMARK_CAPTURE(func, test_case_name, ...) \ - BENCHMARK_PRIVATE_DECLARE(func) = \ + BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ (::benchmark::internal::RegisterBenchmarkInternal( \ new ::benchmark::internal::FunctionBenchmark( \ #func "/" #test_case_name, \ @@ -1335,37 +1560,62 @@ class Fixture : public internal::Benchmark { #define BENCHMARK_TEMPLATE(n, a) BENCHMARK_TEMPLATE1(n, a) #endif -#define BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ - class BaseClass##_##Method##_Benchmark : public BaseClass { \ - public: \ - BaseClass##_##Method##_Benchmark() { \ - this->SetName(#BaseClass "/" #Method); \ - } \ - \ - protected: \ - virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ +#ifdef BENCHMARK_HAS_CXX11 +// This will register a benchmark for a templatized function, +// with the additional arguments specified by `...`. +// +// For example: +// +// template ` +// void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) { +// [...] +//} +// /* Registers a benchmark named "BM_takes_args/int_string_test` */ +// BENCHMARK_TEMPLATE1_CAPTURE(BM_takes_args, void, int_string_test, 42, +// std::string("abc")); +#define BENCHMARK_TEMPLATE1_CAPTURE(func, a, test_case_name, ...) \ + BENCHMARK_CAPTURE(func, test_case_name, __VA_ARGS__) + +#define BENCHMARK_TEMPLATE2_CAPTURE(func, a, b, test_case_name, ...) \ + BENCHMARK_PRIVATE_DECLARE(func) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + new ::benchmark::internal::FunctionBenchmark( \ + #func "<" #a "," #b ">" \ + "/" #test_case_name, \ + [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) +#endif // BENCHMARK_HAS_CXX11 + +#define BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ + class BaseClass##_##Method##_Benchmark : public BaseClass { \ + public: \ + BaseClass##_##Method##_Benchmark() { \ + this->SetName(#BaseClass "/" #Method); \ + } \ + \ + protected: \ + void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ }; -#define BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ - class BaseClass##_##Method##_Benchmark : public BaseClass { \ - public: \ - BaseClass##_##Method##_Benchmark() { \ - this->SetName(#BaseClass "<" #a ">/" #Method); \ - } \ - \ - protected: \ - virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ +#define BENCHMARK_TEMPLATE1_PRIVATE_DECLARE_F(BaseClass, Method, a) \ + class BaseClass##_##Method##_Benchmark : public BaseClass { \ + public: \ + BaseClass##_##Method##_Benchmark() { \ + this->SetName(#BaseClass "<" #a ">/" #Method); \ + } \ + \ + protected: \ + void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ }; -#define BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ - class BaseClass##_##Method##_Benchmark : public BaseClass { \ - public: \ - BaseClass##_##Method##_Benchmark() { \ - this->SetName(#BaseClass "<" #a "," #b ">/" #Method); \ - } \ - \ - protected: \ - virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ +#define BENCHMARK_TEMPLATE2_PRIVATE_DECLARE_F(BaseClass, Method, a, b) \ + class BaseClass##_##Method##_Benchmark : public BaseClass { \ + public: \ + BaseClass##_##Method##_Benchmark() { \ + this->SetName(#BaseClass "<" #a "," #b ">/" #Method); \ + } \ + \ + protected: \ + void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ }; #ifdef BENCHMARK_HAS_CXX11 @@ -1377,7 +1627,7 @@ class Fixture : public internal::Benchmark { } \ \ protected: \ - virtual void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ + void BenchmarkCase(::benchmark::State&) BENCHMARK_OVERRIDE; \ }; #else #define BENCHMARK_TEMPLATE_PRIVATE_DECLARE_F(n, a) \ @@ -1439,8 +1689,15 @@ class Fixture : public internal::Benchmark { #endif // Helper macro to create a main routine in a test that runs the benchmarks +// Note the workaround for Hexagon simulator passing argc != 0, argv = NULL. #define BENCHMARK_MAIN() \ int main(int argc, char** argv) { \ + char arg0_default[] = "benchmark"; \ + char* args_default = arg0_default; \ + if (!argv) { \ + argc = 1; \ + argv = &args_default; \ + } \ ::benchmark::Initialize(&argc, argv); \ if (::benchmark::ReportUnrecognizedArguments(argc, argv)) return 1; \ ::benchmark::RunSpecifiedBenchmarks(); \ @@ -1454,7 +1711,7 @@ class Fixture : public internal::Benchmark { namespace benchmark { -struct CPUInfo { +struct BENCHMARK_EXPORT CPUInfo { struct CacheInfo { std::string type; int level; @@ -1478,7 +1735,7 @@ struct CPUInfo { }; // Adding Struct for System Information -struct SystemInfo { +struct BENCHMARK_EXPORT SystemInfo { std::string name; static const SystemInfo& Get(); @@ -1490,10 +1747,11 @@ struct SystemInfo { // BenchmarkName contains the components of the Benchmark's name // which allows individual fields to be modified or cleared before // building the final name using 'str()'. -struct BenchmarkName { +struct BENCHMARK_EXPORT BenchmarkName { std::string function_name; std::string args; std::string min_time; + std::string min_warmup_time; std::string iterations; std::string repetitions; std::string time_type; @@ -1509,7 +1767,7 @@ struct BenchmarkName { // can control the destination of the reports by calling // RunSpecifiedBenchmarks and passing it a custom reporter object. // The reporter object must implement the following interface. -class BenchmarkReporter { +class BENCHMARK_EXPORT BenchmarkReporter { public: struct Context { CPUInfo const& cpu_info; @@ -1520,20 +1778,21 @@ class BenchmarkReporter { Context(); }; - struct Run { + struct BENCHMARK_EXPORT Run { static const int64_t no_repetition_index = -1; enum RunType { RT_Iteration, RT_Aggregate }; Run() : run_type(RT_Iteration), aggregate_unit(kTime), - error_occurred(false), + skipped(internal::NotSkipped), iterations(1), threads(1), - time_unit(kNanosecond), + time_unit(GetDefaultTimeUnit()), real_accumulated_time(0), cpu_accumulated_time(0), max_heapbytes_used(0), + use_real_time_for_initial_big_o(false), complexity(oNone), complexity_lambda(), complexity_n(0), @@ -1550,8 +1809,8 @@ class BenchmarkReporter { std::string aggregate_name; StatisticUnit aggregate_unit; std::string report_label; // Empty if not set by benchmark. - bool error_occurred; - std::string error_message; + internal::Skipped skipped; + std::string skip_message; IterationCount iterations; int64_t threads; @@ -1576,10 +1835,14 @@ class BenchmarkReporter { // This is set to 0.0 if memory tracing is not enabled. double max_heapbytes_used; + // By default Big-O is computed for CPU time, but that is not what you want + // to happen when manual time was requested, which is stored as real time. + bool use_real_time_for_initial_big_o; + // Keep track of arguments to compute asymptotic complexity BigO complexity; BigOFunc* complexity_lambda; - int64_t complexity_n; + ComplexityN complexity_n; // what statistics to compute from the measurements const std::vector* statistics; @@ -1620,6 +1883,12 @@ class BenchmarkReporter { // to skip runs based on the context information. virtual bool ReportContext(const Context& context) = 0; + // Called once for each group of benchmark runs, gives information about + // the configurations of the runs. + virtual void ReportRunsConfig(double /*min_time*/, + bool /*has_explicit_iters*/, + IterationCount /*iters*/) {} + // Called once for each group of benchmark runs, gives information about // cpu-time and heap memory usage during the benchmark run. If the group // of runs contained more than two entries then 'report' contains additional @@ -1665,7 +1934,7 @@ class BenchmarkReporter { // Simple reporter that outputs benchmark data to the console. This is the // default reporter used by RunSpecifiedBenchmarks(). -class ConsoleReporter : public BenchmarkReporter { +class BENCHMARK_EXPORT ConsoleReporter : public BenchmarkReporter { public: enum OutputOptions { OO_None = 0, @@ -1677,8 +1946,8 @@ class ConsoleReporter : public BenchmarkReporter { explicit ConsoleReporter(OutputOptions opts_ = OO_Defaults) : output_options_(opts_), name_field_width_(0), printed_header_(false) {} - virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; - virtual void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; + bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; + void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; protected: virtual void PrintRunData(const Run& report); @@ -1690,12 +1959,12 @@ class ConsoleReporter : public BenchmarkReporter { bool printed_header_; }; -class JSONReporter : public BenchmarkReporter { +class BENCHMARK_EXPORT JSONReporter : public BenchmarkReporter { public: JSONReporter() : first_report_(true) {} - virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; - virtual void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; - virtual void Finalize() BENCHMARK_OVERRIDE; + bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; + void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; + void Finalize() BENCHMARK_OVERRIDE; private: void PrintRunData(const Run& report); @@ -1703,13 +1972,13 @@ class JSONReporter : public BenchmarkReporter { bool first_report_; }; -class BENCHMARK_DEPRECATED_MSG( +class BENCHMARK_EXPORT BENCHMARK_DEPRECATED_MSG( "The CSV Reporter will be removed in a future release") CSVReporter : public BenchmarkReporter { public: CSVReporter() : printed_header_(false) {} - virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; - virtual void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; + bool ReportContext(const Context& context) BENCHMARK_OVERRIDE; + void ReportRuns(const std::vector& reports) BENCHMARK_OVERRIDE; private: void PrintRunData(const Run& report); @@ -1748,18 +2017,24 @@ inline double GetTimeUnitMultiplier(TimeUnit unit) { // Creates a list of integer values for the given range and multiplier. // This can be used together with ArgsProduct() to allow multiple ranges -// with different multiplers. +// with different multipliers. // Example: // ArgsProduct({ // CreateRange(0, 1024, /*multi=*/32), // CreateRange(0, 100, /*multi=*/4), // CreateDenseRange(0, 4, /*step=*/1), // }); +BENCHMARK_EXPORT std::vector CreateRange(int64_t lo, int64_t hi, int multi); // Creates a list of integer values for the given range and step. +BENCHMARK_EXPORT std::vector CreateDenseRange(int64_t start, int64_t limit, int step); } // namespace benchmark +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + #endif // BENCHMARK_BENCHMARK_H_ diff --git a/third-party/benchmark/include/benchmark/export.h b/third-party/benchmark/include/benchmark/export.h new file mode 100644 index 000000000000..f96f8596cd9f --- /dev/null +++ b/third-party/benchmark/include/benchmark/export.h @@ -0,0 +1,47 @@ +#ifndef BENCHMARK_EXPORT_H +#define BENCHMARK_EXPORT_H + +#if defined(_WIN32) +#define EXPORT_ATTR __declspec(dllexport) +#define IMPORT_ATTR __declspec(dllimport) +#define NO_EXPORT_ATTR +#define DEPRECATED_ATTR __declspec(deprecated) +#else // _WIN32 +#define EXPORT_ATTR __attribute__((visibility("default"))) +#define IMPORT_ATTR __attribute__((visibility("default"))) +#define NO_EXPORT_ATTR __attribute__((visibility("hidden"))) +#define DEPRECATE_ATTR __attribute__((__deprecated__)) +#endif // _WIN32 + +#ifdef BENCHMARK_STATIC_DEFINE +#define BENCHMARK_EXPORT +#define BENCHMARK_NO_EXPORT +#else // BENCHMARK_STATIC_DEFINE +#ifndef BENCHMARK_EXPORT +#ifdef benchmark_EXPORTS +/* We are building this library */ +#define BENCHMARK_EXPORT EXPORT_ATTR +#else // benchmark_EXPORTS +/* We are using this library */ +#define BENCHMARK_EXPORT IMPORT_ATTR +#endif // benchmark_EXPORTS +#endif // !BENCHMARK_EXPORT + +#ifndef BENCHMARK_NO_EXPORT +#define BENCHMARK_NO_EXPORT NO_EXPORT_ATTR +#endif // !BENCHMARK_NO_EXPORT +#endif // BENCHMARK_STATIC_DEFINE + +#ifndef BENCHMARK_DEPRECATED +#define BENCHMARK_DEPRECATED DEPRECATE_ATTR +#endif // BENCHMARK_DEPRECATED + +#ifndef BENCHMARK_DEPRECATED_EXPORT +#define BENCHMARK_DEPRECATED_EXPORT BENCHMARK_EXPORT BENCHMARK_DEPRECATED +#endif // BENCHMARK_DEPRECATED_EXPORT + +#ifndef BENCHMARK_DEPRECATED_NO_EXPORT +#define BENCHMARK_DEPRECATED_NO_EXPORT BENCHMARK_NO_EXPORT BENCHMARK_DEPRECATED +#endif // BENCHMARK_DEPRECATED_EXPORT + +#endif /* BENCHMARK_EXPORT_H */ diff --git a/third-party/benchmark/pyproject.toml b/third-party/benchmark/pyproject.toml new file mode 100644 index 000000000000..aa24ae8c3f57 --- /dev/null +++ b/third-party/benchmark/pyproject.toml @@ -0,0 +1,85 @@ +[build-system] +requires = ["setuptools", "setuptools-scm[toml]", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "google_benchmark" +description = "A library to benchmark code snippets." +requires-python = ">=3.8" +license = {file = "LICENSE"} +keywords = ["benchmark"] + +authors = [ + {name = "Google", email = "benchmark-discuss@googlegroups.com"}, +] + +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Software Development :: Testing", + "Topic :: System :: Benchmark", +] + +dynamic = ["readme", "version"] + +dependencies = [ + "absl-py>=0.7.1", +] + +[project.optional-dependencies] +dev = [ + "pre-commit>=3.3.3", +] + +[project.urls] +Homepage = "https://github.com/google/benchmark" +Documentation = "https://github.com/google/benchmark/tree/main/docs" +Repository = "https://github.com/google/benchmark.git" +Discord = "https://discord.gg/cz7UX7wKC2" + +[tool.setuptools] +package-dir = {"" = "bindings/python"} +zip-safe = false + +[tool.setuptools.packages.find] +where = ["bindings/python"] + +[tool.setuptools.dynamic] +readme = { file = "README.md", content-type = "text/markdown" } + +[tool.setuptools_scm] + +[tool.mypy] +check_untyped_defs = true +disallow_incomplete_defs = true +pretty = true +python_version = "3.11" +strict_optional = false +warn_unreachable = true + +[[tool.mypy.overrides]] +module = ["yaml"] +ignore_missing_imports = true + +[tool.ruff] +# explicitly tell ruff the source directory to correctly identify first-party package. +src = ["bindings/python"] + +line-length = 80 +target-version = "py311" + +# Enable pycodestyle (`E`, `W`), Pyflakes (`F`), and isort (`I`) codes by default. +select = ["E", "F", "I", "W"] +ignore = [ + "E501", # line too long +] + +[tool.ruff.isort] +combine-as-imports = true diff --git a/third-party/benchmark/requirements.txt b/third-party/benchmark/requirements.txt deleted file mode 100644 index e451894e2356..000000000000 --- a/third-party/benchmark/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -numpy == 1.19.4 -scipy == 1.5.4 -pandas == 1.1.5 diff --git a/third-party/benchmark/setup.py b/third-party/benchmark/setup.py index 83069e566839..cb20042da512 100644 --- a/third-party/benchmark/setup.py +++ b/third-party/benchmark/setup.py @@ -1,56 +1,50 @@ +import contextlib import os -import posixpath import platform -import re import shutil -import sys +import sysconfig +from pathlib import Path +from typing import Generator -from distutils import sysconfig import setuptools from setuptools.command import build_ext +PYTHON_INCLUDE_PATH_PLACEHOLDER = "" -HERE = os.path.dirname(os.path.abspath(__file__)) +IS_WINDOWS = platform.system() == "Windows" +IS_MAC = platform.system() == "Darwin" -IS_WINDOWS = sys.platform.startswith("win") - - -def _get_version(): - """Parse the version string from __init__.py.""" - with open( - os.path.join(HERE, "bindings", "python", "google_benchmark", "__init__.py") - ) as init_file: +@contextlib.contextmanager +def temp_fill_include_path(fp: str) -> Generator[None, None, None]: + """Temporarily set the Python include path in a file.""" + with open(fp, "r+") as f: try: - version_line = next( - line for line in init_file if line.startswith("__version__") + content = f.read() + replaced = content.replace( + PYTHON_INCLUDE_PATH_PLACEHOLDER, + Path(sysconfig.get_paths()["include"]).as_posix(), ) - except StopIteration: - raise ValueError("__version__ not defined in __init__.py") - else: - namespace = {} - exec(version_line, namespace) # pylint: disable=exec-used - return namespace["__version__"] - - -def _parse_requirements(path): - with open(os.path.join(HERE, path)) as requirements: - return [ - line.rstrip() - for line in requirements - if not (line.isspace() or line.startswith("#")) - ] + f.seek(0) + f.write(replaced) + f.truncate() + yield + finally: + # revert to the original content after exit + f.seek(0) + f.write(content) + f.truncate() class BazelExtension(setuptools.Extension): """A C/C++ extension that is defined as a Bazel BUILD target.""" - def __init__(self, name, bazel_target): + def __init__(self, name: str, bazel_target: str): + super().__init__(name=name, sources=[]) + self.bazel_target = bazel_target - self.relpath, self.target_name = posixpath.relpath(bazel_target, "//").split( - ":" - ) - setuptools.Extension.__init__(self, name, sources=[]) + stripped_target = bazel_target.split("//")[-1] + self.relpath, self.target_name = stripped_target.split(":") class BuildBazelExtension(build_ext.build_ext): @@ -59,88 +53,71 @@ class BuildBazelExtension(build_ext.build_ext): def run(self): for ext in self.extensions: self.bazel_build(ext) - build_ext.build_ext.run(self) - - def bazel_build(self, ext): + super().run() + # explicitly call `bazel shutdown` for graceful exit + self.spawn(["bazel", "shutdown"]) + + def copy_extensions_to_source(self): + """ + Copy generated extensions into the source tree. + This is done in the ``bazel_build`` method, so it's not necessary to + do again in the `build_ext` base class. + """ + pass + + def bazel_build(self, ext: BazelExtension) -> None: """Runs the bazel build to create the package.""" - with open("WORKSPACE", "r") as workspace: - workspace_contents = workspace.read() - - with open("WORKSPACE", "w") as workspace: - workspace.write( - re.sub( - r'(?<=path = ").*(?=", # May be overwritten by setup\.py\.)', - sysconfig.get_python_inc().replace(os.path.sep, posixpath.sep), - workspace_contents, - ) + with temp_fill_include_path("WORKSPACE"): + temp_path = Path(self.build_temp) + + bazel_argv = [ + "bazel", + "build", + ext.bazel_target, + "--enable_bzlmod=false", + f"--symlink_prefix={temp_path / 'bazel-'}", + f"--compilation_mode={'dbg' if self.debug else 'opt'}", + # C++17 is required by nanobind + f"--cxxopt={'/std:c++17' if IS_WINDOWS else '-std=c++17'}", + ] + + if IS_WINDOWS: + # Link with python*.lib. + for library_dir in self.library_dirs: + bazel_argv.append("--linkopt=/LIBPATH:" + library_dir) + elif IS_MAC: + if platform.machine() == "x86_64": + # C++17 needs macOS 10.14 at minimum + bazel_argv.append("--macos_minimum_os=10.14") + + # cross-compilation for Mac ARM64 on GitHub Mac x86 runners. + # ARCHFLAGS is set by cibuildwheel before macOS wheel builds. + archflags = os.getenv("ARCHFLAGS", "") + if "arm64" in archflags: + bazel_argv.append("--cpu=darwin_arm64") + bazel_argv.append("--macos_cpus=arm64") + + elif platform.machine() == "arm64": + bazel_argv.append("--macos_minimum_os=11.0") + + self.spawn(bazel_argv) + + shared_lib_suffix = ".dll" if IS_WINDOWS else ".so" + ext_name = ext.target_name + shared_lib_suffix + ext_bazel_bin_path = ( + temp_path / "bazel-bin" / ext.relpath / ext_name ) - if not os.path.exists(self.build_temp): - os.makedirs(self.build_temp) - - bazel_argv = [ - "bazel", - "build", - ext.bazel_target, - "--symlink_prefix=" + os.path.join(self.build_temp, "bazel-"), - "--compilation_mode=" + ("dbg" if self.debug else "opt"), - ] - - if IS_WINDOWS: - # Link with python*.lib. - for library_dir in self.library_dirs: - bazel_argv.append("--linkopt=/LIBPATH:" + library_dir) - elif sys.platform == "darwin" and platform.machine() == "x86_64": - bazel_argv.append("--macos_minimum_os=10.9") - - self.spawn(bazel_argv) - - shared_lib_suffix = ".dll" if IS_WINDOWS else ".so" - ext_bazel_bin_path = os.path.join( - self.build_temp, - "bazel-bin", - ext.relpath, - ext.target_name + shared_lib_suffix, - ) - - ext_dest_path = self.get_ext_fullpath(ext.name) - ext_dest_dir = os.path.dirname(ext_dest_path) - if not os.path.exists(ext_dest_dir): - os.makedirs(ext_dest_dir) - shutil.copyfile(ext_bazel_bin_path, ext_dest_path) + ext_dest_path = Path(self.get_ext_fullpath(ext.name)) + shutil.copyfile(ext_bazel_bin_path, ext_dest_path) setuptools.setup( - name="google_benchmark", - version=_get_version(), - url="https://github.com/google/benchmark", - description="A library to benchmark code snippets.", - author="Google", - author_email="benchmark-py@google.com", - # Contained modules and scripts. - package_dir={"": "bindings/python"}, - packages=setuptools.find_packages("bindings/python"), - install_requires=_parse_requirements("bindings/python/requirements.txt"), cmdclass=dict(build_ext=BuildBazelExtension), ext_modules=[ BazelExtension( - "google_benchmark._benchmark", - "//bindings/python/google_benchmark:_benchmark", + name="google_benchmark._benchmark", + bazel_target="//bindings/python/google_benchmark:_benchmark", ) ], - zip_safe=False, - # PyPI package information. - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Topic :: Software Development :: Testing", - "Topic :: System :: Benchmark", - ], - license="Apache 2.0", - keywords="benchmark", ) diff --git a/third-party/benchmark/src/CMakeLists.txt b/third-party/benchmark/src/CMakeLists.txt index e814a4e00f7c..943594b70bcd 100644 --- a/third-party/benchmark/src/CMakeLists.txt +++ b/third-party/benchmark/src/CMakeLists.txt @@ -25,12 +25,25 @@ set_target_properties(benchmark PROPERTIES SOVERSION ${GENERIC_LIB_SOVERSION} ) target_include_directories(benchmark PUBLIC - $) + $ +) + +set_property( + SOURCE benchmark.cc + APPEND + PROPERTY COMPILE_DEFINITIONS + BENCHMARK_VERSION="${VERSION}" +) # libpfm, if available -if (HAVE_LIBPFM) - target_link_libraries(benchmark PRIVATE pfm) - add_definitions(-DHAVE_LIBPFM) +if (PFM_FOUND) + target_link_libraries(benchmark PRIVATE PFM::libpfm) + target_compile_definitions(benchmark PRIVATE -DHAVE_LIBPFM) +endif() + +# pthread affinity, if available +if(HAVE_PTHREAD_AFFINITY) + target_compile_definitions(benchmark PRIVATE -DBENCHMARK_HAS_PTHREAD_AFFINITY) endif() # Link threads. @@ -53,6 +66,10 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "SunOS") target_link_libraries(benchmark PRIVATE kstat) endif() +if (NOT BUILD_SHARED_LIBS) + target_compile_definitions(benchmark PUBLIC -DBENCHMARK_STATIC_DEFINE) +endif() + # Benchmark main library add_library(benchmark_main "benchmark_main.cc") add_library(benchmark::benchmark_main ALIAS benchmark_main) @@ -60,10 +77,10 @@ set_target_properties(benchmark_main PROPERTIES OUTPUT_NAME "benchmark_main" VERSION ${GENERIC_LIB_VERSION} SOVERSION ${GENERIC_LIB_SOVERSION} + DEFINE_SYMBOL benchmark_EXPORTS ) target_link_libraries(benchmark_main PUBLIC benchmark::benchmark) - set(generated_dir "${PROJECT_BINARY_DIR}") set(version_config "${generated_dir}/${PROJECT_NAME}ConfigVersion.cmake") @@ -107,6 +124,7 @@ if (BENCHMARK_ENABLE_INSTALL) install( DIRECTORY "${PROJECT_SOURCE_DIR}/include/benchmark" + "${PROJECT_BINARY_DIR}/include/benchmark" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} FILES_MATCHING PATTERN "*.*h") diff --git a/third-party/benchmark/src/benchmark.cc b/third-party/benchmark/src/benchmark.cc index 473151136202..495944db29ff 100644 --- a/third-party/benchmark/src/benchmark.cc +++ b/third-party/benchmark/src/benchmark.cc @@ -19,7 +19,7 @@ #include "internal_macros.h" #ifndef BENCHMARK_OS_WINDOWS -#ifndef BENCHMARK_OS_FUCHSIA +#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) #include #endif #include @@ -65,12 +65,28 @@ BM_DEFINE_bool(benchmark_list_tests, false); // linked into the binary are run. BM_DEFINE_string(benchmark_filter, ""); -// Minimum number of seconds we should run benchmark before results are -// considered significant. For cpu-time based tests, this is the lower bound +// Specification of how long to run the benchmark. +// +// It can be either an exact number of iterations (specified as `x`), +// or a minimum number of seconds (specified as `s`). If the latter +// format (ie., min seconds) is used, the system may run the benchmark longer +// until the results are considered significant. +// +// For backward compatibility, the `s` suffix may be omitted, in which case, +// the specified number is interpreted as the number of seconds. +// +// For cpu-time based tests, this is the lower bound // on the total cpu time used by all threads that make up the test. For // real-time based tests, this is the lower bound on the elapsed time of the // benchmark execution, regardless of number of threads. -BM_DEFINE_double(benchmark_min_time, 0.5); +BM_DEFINE_string(benchmark_min_time, kDefaultMinTimeStr); + +// Minimum number of seconds a benchmark should be run before results should be +// taken into account. This e.g can be necessary for benchmarks of code which +// needs to fill some form of cache before performance is of interest. +// Note: results gathered within this period are discarded and not used for +// reported result. +BM_DEFINE_double(benchmark_min_warmup_time, 0.0); // The number of runs of each benchmark. If greater than 1, the mean and // standard deviation of the runs will be reported. @@ -121,6 +137,10 @@ BM_DEFINE_string(benchmark_perf_counters, ""); // pairs. Kept internal as it's only used for parsing from env/command line. BM_DEFINE_kvpairs(benchmark_context, {}); +// Set the default time unit to use for reports +// Valid values are 'ns', 'us', 'ms' or 's' +BM_DEFINE_string(benchmark_time_unit, ""); + // The level of verbose logging to output BM_DEFINE_int32(v, 0); @@ -128,23 +148,28 @@ namespace internal { std::map* global_context = nullptr; +BENCHMARK_EXPORT std::map*& GetGlobalContext() { + return global_context; +} + // FIXME: wouldn't LTO mess this up? void UseCharPointer(char const volatile*) {} } // namespace internal -State::State(IterationCount max_iters, const std::vector& ranges, - int thread_i, int n_threads, internal::ThreadTimer* timer, - internal::ThreadManager* manager, +State::State(std::string name, IterationCount max_iters, + const std::vector& ranges, int thread_i, int n_threads, + internal::ThreadTimer* timer, internal::ThreadManager* manager, internal::PerfCountersMeasurement* perf_counters_measurement) : total_iterations_(0), batch_leftover_(0), max_iterations(max_iters), started_(false), finished_(false), - error_occurred_(false), + skipped_(internal::NotSkipped), range_(ranges), complexity_n_(0), + name_(std::move(name)), thread_index_(thread_i), threads_(n_threads), timer_(timer), @@ -154,6 +179,17 @@ State::State(IterationCount max_iters, const std::vector& ranges, BM_CHECK_LT(thread_index_, threads_) << "thread_index must be less than threads"; + // Add counters with correct flag now. If added with `counters[name]` in + // `PauseTiming`, a new `Counter` will be inserted the first time, which + // won't have the flag. Inserting them now also reduces the allocations + // during the benchmark. + if (perf_counters_measurement_) { + for (const std::string& counter_name : + perf_counters_measurement_->names()) { + counters[counter_name] = Counter(0.0, Counter::kAvgIterations); + } + } + // Note: The use of offsetof below is technically undefined until C++17 // because State is not a standard layout type. However, all compilers // currently provide well-defined behavior as an extension (which is @@ -169,12 +205,19 @@ State::State(IterationCount max_iters, const std::vector& ranges, #elif defined(__clang__) #pragma clang diagnostic push #pragma clang diagnostic ignored "-Winvalid-offsetof" +#endif +#if defined(__NVCC__) +#pragma nv_diagnostic push +#pragma nv_diag_suppress 1427 +#endif +#if defined(__NVCOMPILER) +#pragma diagnostic push +#pragma diag_suppress offset_in_non_POD_nonstandard #endif // Offset tests to ensure commonly accessed data is on the first cache line. const int cache_line_size = 64; - static_assert(offsetof(State, error_occurred_) <= - (cache_line_size - sizeof(error_occurred_)), - ""); + static_assert( + offsetof(State, skipped_) <= (cache_line_size - sizeof(skipped_)), ""); #if defined(__INTEL_COMPILER) #pragma warning pop #elif defined(__GNUC__) @@ -182,39 +225,61 @@ State::State(IterationCount max_iters, const std::vector& ranges, #elif defined(__clang__) #pragma clang diagnostic pop #endif +#if defined(__NVCC__) +#pragma nv_diagnostic pop +#endif +#if defined(__NVCOMPILER) +#pragma diagnostic pop +#endif } void State::PauseTiming() { // Add in time accumulated so far - BM_CHECK(started_ && !finished_ && !error_occurred_); + BM_CHECK(started_ && !finished_ && !skipped()); timer_->StopTimer(); if (perf_counters_measurement_) { - auto measurements = perf_counters_measurement_->StopAndGetMeasurements(); + std::vector> measurements; + if (!perf_counters_measurement_->Stop(measurements)) { + BM_CHECK(false) << "Perf counters read the value failed."; + } for (const auto& name_and_measurement : measurements) { - auto name = name_and_measurement.first; - auto measurement = name_and_measurement.second; - BM_CHECK_EQ(counters[name], 0.0); - counters[name] = Counter(measurement, Counter::kAvgIterations); + const std::string& name = name_and_measurement.first; + const double measurement = name_and_measurement.second; + // Counter was inserted with `kAvgIterations` flag by the constructor. + assert(counters.find(name) != counters.end()); + counters[name].value += measurement; } } } void State::ResumeTiming() { - BM_CHECK(started_ && !finished_ && !error_occurred_); + BM_CHECK(started_ && !finished_ && !skipped()); timer_->StartTimer(); if (perf_counters_measurement_) { perf_counters_measurement_->Start(); } } -void State::SkipWithError(const char* msg) { - BM_CHECK(msg); - error_occurred_ = true; +void State::SkipWithMessage(const std::string& msg) { + skipped_ = internal::SkippedWithMessage; + { + MutexLock l(manager_->GetBenchmarkMutex()); + if (internal::NotSkipped == manager_->results.skipped_) { + manager_->results.skip_message_ = msg; + manager_->results.skipped_ = skipped_; + } + } + total_iterations_ = 0; + if (timer_->running()) timer_->StopTimer(); +} + +void State::SkipWithError(const std::string& msg) { + skipped_ = internal::SkippedWithError; { MutexLock l(manager_->GetBenchmarkMutex()); - if (manager_->results.has_error_ == false) { - manager_->results.error_message_ = msg; - manager_->results.has_error_ = true; + if (internal::NotSkipped == manager_->results.skipped_) { + manager_->results.skip_message_ = msg; + manager_->results.skipped_ = skipped_; } } total_iterations_ = 0; @@ -225,7 +290,7 @@ void State::SetIterationTime(double seconds) { timer_->SetIterationTime(seconds); } -void State::SetLabel(const char* label) { +void State::SetLabel(const std::string& label) { MutexLock l(manager_->GetBenchmarkMutex()); manager_->results.report_label_ = label; } @@ -233,14 +298,14 @@ void State::SetLabel(const char* label) { void State::StartKeepRunning() { BM_CHECK(!started_ && !finished_); started_ = true; - total_iterations_ = error_occurred_ ? 0 : max_iterations; + total_iterations_ = skipped() ? 0 : max_iterations; manager_->StartStopBarrier(); - if (!error_occurred_) ResumeTiming(); + if (!skipped()) ResumeTiming(); } void State::FinishKeepRunning() { - BM_CHECK(started_ && (!finished_ || error_occurred_)); - if (!error_occurred_) { + BM_CHECK(started_ && (!finished_ || skipped())); + if (!skipped()) { PauseTiming(); } // Total iterations has now wrapped around past 0. Fix this. @@ -318,14 +383,26 @@ void RunBenchmarks(const std::vector& benchmarks, size_t num_repetitions_total = 0; + // This perfcounters object needs to be created before the runners vector + // below so it outlasts their lifetime. + PerfCountersMeasurement perfcounters( + StrSplit(FLAGS_benchmark_perf_counters, ',')); + + // Vector of benchmarks to run std::vector runners; runners.reserve(benchmarks.size()); + + // Count the number of benchmarks with threads to warn the user in case + // performance counters are used. + int benchmarks_with_threads = 0; + + // Loop through all benchmarks for (const BenchmarkInstance& benchmark : benchmarks) { BenchmarkReporter::PerFamilyRunReports* reports_for_family = nullptr; if (benchmark.complexity() != oNone) reports_for_family = &per_family_reports[benchmark.family_index()]; - - runners.emplace_back(benchmark, reports_for_family); + benchmarks_with_threads += (benchmark.threads() > 1); + runners.emplace_back(benchmark, &perfcounters, reports_for_family); int num_repeats_of_this_instance = runners.back().GetNumRepeats(); num_repetitions_total += num_repeats_of_this_instance; if (reports_for_family) @@ -333,6 +410,17 @@ void RunBenchmarks(const std::vector& benchmarks, } assert(runners.size() == benchmarks.size() && "Unexpected runner count."); + // The use of performance counters with threads would be unintuitive for + // the average user so we need to warn them about this case + if ((benchmarks_with_threads > 0) && (perfcounters.num_counters() > 0)) { + GetErrorLogInstance() + << "***WARNING*** There are " << benchmarks_with_threads + << " benchmarks with threads and " << perfcounters.num_counters() + << " performance counters were requested. Beware counters will " + "reflect the combined usage across all " + "threads.\n"; + } + std::vector repetition_indices; repetition_indices.reserve(num_repetitions_total); for (size_t runner_index = 0, num_runners = runners.size(); @@ -356,6 +444,12 @@ void RunBenchmarks(const std::vector& benchmarks, if (runner.HasRepeatsRemaining()) continue; // FIXME: report each repetition separately, not all of them in bulk. + display_reporter->ReportRunsConfig( + runner.GetMinTime(), runner.HasExplicitIters(), runner.GetIters()); + if (file_reporter) + file_reporter->ReportRunsConfig( + runner.GetMinTime(), runner.HasExplicitIters(), runner.GetIters()); + RunResults run_results = runner.GetResults(); // Maybe calculate complexity report @@ -389,14 +483,15 @@ std::unique_ptr CreateReporter( typedef std::unique_ptr PtrType; if (name == "console") { return PtrType(new ConsoleReporter(output_opts)); - } else if (name == "json") { - return PtrType(new JSONReporter); - } else if (name == "csv") { - return PtrType(new CSVReporter); - } else { - std::cerr << "Unexpected format: '" << name << "'\n"; - std::exit(1); } + if (name == "json") { + return PtrType(new JSONReporter()); + } + if (name == "csv") { + return PtrType(new CSVReporter()); + } + std::cerr << "Unexpected format: '" << name << "'\n"; + std::exit(1); } BENCHMARK_RESTORE_DEPRECATED_WARNING @@ -433,6 +528,14 @@ ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color) { } // end namespace internal +BenchmarkReporter* CreateDefaultDisplayReporter() { + static auto default_display_reporter = + internal::CreateReporter(FLAGS_benchmark_format, + internal::GetOutputOptions()) + .release(); + return default_display_reporter; +} + size_t RunSpecifiedBenchmarks() { return RunSpecifiedBenchmarks(nullptr, nullptr, FLAGS_benchmark_filter); } @@ -468,8 +571,7 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, std::unique_ptr default_display_reporter; std::unique_ptr default_file_reporter; if (!display_reporter) { - default_display_reporter = internal::CreateReporter( - FLAGS_benchmark_format, internal::GetOutputOptions()); + default_display_reporter.reset(CreateDefaultDisplayReporter()); display_reporter = default_display_reporter.get(); } auto& Out = display_reporter->GetOutputStream(); @@ -480,17 +582,23 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, Err << "A custom file reporter was provided but " "--benchmark_out= was not specified." << std::endl; + Out.flush(); + Err.flush(); std::exit(1); } if (!fname.empty()) { output_file.open(fname); if (!output_file.is_open()) { Err << "invalid file name: '" << fname << "'" << std::endl; + Out.flush(); + Err.flush(); std::exit(1); } if (!file_reporter) { default_file_reporter = internal::CreateReporter( - FLAGS_benchmark_out_format, ConsoleReporter::OO_None); + FLAGS_benchmark_out_format, FLAGS_benchmark_counters_tabular + ? ConsoleReporter::OO_Tabular + : ConsoleReporter::OO_None); file_reporter = default_file_reporter.get(); } file_reporter->SetOutputStream(&output_file); @@ -498,10 +606,16 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, } std::vector benchmarks; - if (!FindBenchmarksInternal(spec, &benchmarks, &Err)) return 0; + if (!FindBenchmarksInternal(spec, &benchmarks, &Err)) { + Out.flush(); + Err.flush(); + return 0; + } if (benchmarks.empty()) { Err << "Failed to match any benchmarks against regex: " << spec << "\n"; + Out.flush(); + Err.flush(); return 0; } @@ -512,11 +626,28 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, internal::RunBenchmarks(benchmarks, display_reporter, file_reporter); } + Out.flush(); + Err.flush(); return benchmarks.size(); } +namespace { +// stores the time unit benchmarks use by default +TimeUnit default_time_unit = kNanosecond; +} // namespace + +TimeUnit GetDefaultTimeUnit() { return default_time_unit; } + +void SetDefaultTimeUnit(TimeUnit unit) { default_time_unit = unit; } + std::string GetBenchmarkFilter() { return FLAGS_benchmark_filter; } +void SetBenchmarkFilter(std::string value) { + FLAGS_benchmark_filter = std::move(value); +} + +int32_t GetBenchmarkVerbosity() { return FLAGS_v; } + void RegisterMemoryManager(MemoryManager* manager) { internal::memory_manager = manager; } @@ -533,27 +664,31 @@ void AddCustomContext(const std::string& key, const std::string& value) { namespace internal { +void (*HelperPrintf)(); + void PrintUsageAndExit() { - fprintf(stdout, - "benchmark" - " [--benchmark_list_tests={true|false}]\n" - " [--benchmark_filter=]\n" - " [--benchmark_min_time=]\n" - " [--benchmark_repetitions=]\n" - " [--benchmark_enable_random_interleaving={true|false}]\n" - " [--benchmark_report_aggregates_only={true|false}]\n" - " [--benchmark_display_aggregates_only={true|false}]\n" - " [--benchmark_format=]\n" - " [--benchmark_out=]\n" - " [--benchmark_out_format=]\n" - " [--benchmark_color={auto|true|false}]\n" - " [--benchmark_counters_tabular={true|false}]\n" - " [--benchmark_perf_counters=,...]\n" - " [--benchmark_context==,...]\n" - " [--v=]\n"); + HelperPrintf(); exit(0); } +void SetDefaultTimeUnitFromFlag(const std::string& time_unit_flag) { + if (time_unit_flag == "s") { + return SetDefaultTimeUnit(kSecond); + } + if (time_unit_flag == "ms") { + return SetDefaultTimeUnit(kMillisecond); + } + if (time_unit_flag == "us") { + return SetDefaultTimeUnit(kMicrosecond); + } + if (time_unit_flag == "ns") { + return SetDefaultTimeUnit(kNanosecond); + } + if (!time_unit_flag.empty()) { + PrintUsageAndExit(); + } +} + void ParseCommandLineFlags(int* argc, char** argv) { using namespace benchmark; BenchmarkReporter::Context::executable_name = @@ -562,8 +697,10 @@ void ParseCommandLineFlags(int* argc, char** argv) { if (ParseBoolFlag(argv[i], "benchmark_list_tests", &FLAGS_benchmark_list_tests) || ParseStringFlag(argv[i], "benchmark_filter", &FLAGS_benchmark_filter) || - ParseDoubleFlag(argv[i], "benchmark_min_time", + ParseStringFlag(argv[i], "benchmark_min_time", &FLAGS_benchmark_min_time) || + ParseDoubleFlag(argv[i], "benchmark_min_warmup_time", + &FLAGS_benchmark_min_warmup_time) || ParseInt32Flag(argv[i], "benchmark_repetitions", &FLAGS_benchmark_repetitions) || ParseBoolFlag(argv[i], "benchmark_enable_random_interleaving", @@ -583,6 +720,8 @@ void ParseCommandLineFlags(int* argc, char** argv) { &FLAGS_benchmark_perf_counters) || ParseKeyValueFlag(argv[i], "benchmark_context", &FLAGS_benchmark_context) || + ParseStringFlag(argv[i], "benchmark_time_unit", + &FLAGS_benchmark_time_unit) || ParseInt32Flag(argv[i], "v", &FLAGS_v)) { for (int j = i; j != *argc - 1; ++j) argv[j] = argv[j + 1]; @@ -598,6 +737,7 @@ void ParseCommandLineFlags(int* argc, char** argv) { PrintUsageAndExit(); } } + SetDefaultTimeUnitFromFlag(FLAGS_benchmark_time_unit); if (FLAGS_benchmark_color.empty()) { PrintUsageAndExit(); } @@ -613,7 +753,34 @@ int InitializeStreams() { } // end namespace internal -void Initialize(int* argc, char** argv) { +std::string GetBenchmarkVersion() { return {BENCHMARK_VERSION}; } + +void PrintDefaultHelp() { + fprintf(stdout, + "benchmark" + " [--benchmark_list_tests={true|false}]\n" + " [--benchmark_filter=]\n" + " [--benchmark_min_time=`x` OR `s` ]\n" + " [--benchmark_min_warmup_time=]\n" + " [--benchmark_repetitions=]\n" + " [--benchmark_enable_random_interleaving={true|false}]\n" + " [--benchmark_report_aggregates_only={true|false}]\n" + " [--benchmark_display_aggregates_only={true|false}]\n" + " [--benchmark_format=]\n" + " [--benchmark_out=]\n" + " [--benchmark_out_format=]\n" + " [--benchmark_color={auto|true|false}]\n" + " [--benchmark_counters_tabular={true|false}]\n" +#if defined HAVE_LIBPFM + " [--benchmark_perf_counters=,...]\n" +#endif + " [--benchmark_context==,...]\n" + " [--benchmark_time_unit={ns|us|ms|s}]\n" + " [--v=]\n"); +} + +void Initialize(int* argc, char** argv, void (*HelperPrintf)()) { + internal::HelperPrintf = HelperPrintf; internal::ParseCommandLineFlags(argc, argv); internal::LogLevel() = FLAGS_v; } diff --git a/third-party/benchmark/src/benchmark_api_internal.cc b/third-party/benchmark/src/benchmark_api_internal.cc index 4de36e3c8ba3..286f98653085 100644 --- a/third-party/benchmark/src/benchmark_api_internal.cc +++ b/third-party/benchmark/src/benchmark_api_internal.cc @@ -16,7 +16,7 @@ BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, int family_idx, per_family_instance_index_(per_family_instance_idx), aggregation_report_mode_(benchmark_.aggregation_report_mode_), args_(args), - time_unit_(benchmark_.time_unit_), + time_unit_(benchmark_.GetTimeUnit()), measure_process_cpu_time_(benchmark_.measure_process_cpu_time_), use_real_time_(benchmark_.use_real_time_), use_manual_time_(benchmark_.use_manual_time_), @@ -25,6 +25,7 @@ BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, int family_idx, statistics_(benchmark_.statistics_), repetitions_(benchmark_.repetitions_), min_time_(benchmark_.min_time_), + min_warmup_time_(benchmark_.min_warmup_time_), iterations_(benchmark_.iterations_), threads_(thread_count) { name_.function_name = benchmark_.name_; @@ -50,6 +51,11 @@ BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, int family_idx, name_.min_time = StrFormat("min_time:%0.3f", benchmark_.min_time_); } + if (!IsZero(benchmark->min_warmup_time_)) { + name_.min_warmup_time = + StrFormat("min_warmup_time:%0.3f", benchmark_.min_warmup_time_); + } + if (benchmark_.iterations_ != 0) { name_.iterations = StrFormat( "iterations:%lu", static_cast(benchmark_.iterations_)); @@ -87,24 +93,24 @@ State BenchmarkInstance::Run( IterationCount iters, int thread_id, internal::ThreadTimer* timer, internal::ThreadManager* manager, internal::PerfCountersMeasurement* perf_counters_measurement) const { - State st(iters, args_, thread_id, threads_, timer, manager, - perf_counters_measurement); + State st(name_.function_name, iters, args_, thread_id, threads_, timer, + manager, perf_counters_measurement); benchmark_.Run(st); return st; } void BenchmarkInstance::Setup() const { if (setup_) { - State st(/*iters*/ 1, args_, /*thread_id*/ 0, threads_, nullptr, nullptr, - nullptr); + State st(name_.function_name, /*iters*/ 1, args_, /*thread_id*/ 0, threads_, + nullptr, nullptr, nullptr); setup_(st); } } void BenchmarkInstance::Teardown() const { if (teardown_) { - State st(/*iters*/ 1, args_, /*thread_id*/ 0, threads_, nullptr, nullptr, - nullptr); + State st(name_.function_name, /*iters*/ 1, args_, /*thread_id*/ 0, threads_, + nullptr, nullptr, nullptr); teardown_(st); } } diff --git a/third-party/benchmark/src/benchmark_api_internal.h b/third-party/benchmark/src/benchmark_api_internal.h index 94c2b2972bb1..94f516531bc4 100644 --- a/third-party/benchmark/src/benchmark_api_internal.h +++ b/third-party/benchmark/src/benchmark_api_internal.h @@ -36,6 +36,7 @@ class BenchmarkInstance { const std::vector& statistics() const { return statistics_; } int repetitions() const { return repetitions_; } double min_time() const { return min_time_; } + double min_warmup_time() const { return min_warmup_time_; } IterationCount iterations() const { return iterations_; } int threads() const { return threads_; } void Setup() const; @@ -62,6 +63,7 @@ class BenchmarkInstance { const std::vector& statistics_; int repetitions_; double min_time_; + double min_warmup_time_; IterationCount iterations_; int threads_; // Number of concurrent threads to us @@ -76,6 +78,7 @@ bool FindBenchmarksInternal(const std::string& re, bool IsZero(double n); +BENCHMARK_EXPORT ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color = false); } // end namespace internal diff --git a/third-party/benchmark/src/benchmark_main.cc b/third-party/benchmark/src/benchmark_main.cc index b3b247831496..cd61cd2ad506 100644 --- a/third-party/benchmark/src/benchmark_main.cc +++ b/third-party/benchmark/src/benchmark_main.cc @@ -14,4 +14,5 @@ #include "benchmark/benchmark.h" +BENCHMARK_EXPORT int main(int, char**); BENCHMARK_MAIN(); diff --git a/third-party/benchmark/src/benchmark_name.cc b/third-party/benchmark/src/benchmark_name.cc index 2a17ebce277f..01676bbc84df 100644 --- a/third-party/benchmark/src/benchmark_name.cc +++ b/third-party/benchmark/src/benchmark_name.cc @@ -51,8 +51,9 @@ std::string join(char delimiter, const Ts&... ts) { } } // namespace +BENCHMARK_EXPORT std::string BenchmarkName::str() const { - return join('/', function_name, args, min_time, iterations, repetitions, - time_type, threads); + return join('/', function_name, args, min_time, min_warmup_time, iterations, + repetitions, time_type, threads); } } // namespace benchmark diff --git a/third-party/benchmark/src/benchmark_register.cc b/third-party/benchmark/src/benchmark_register.cc index 61a0c26178e3..e447c9a2d39b 100644 --- a/third-party/benchmark/src/benchmark_register.cc +++ b/third-party/benchmark/src/benchmark_register.cc @@ -15,7 +15,7 @@ #include "benchmark_register.h" #ifndef BENCHMARK_OS_WINDOWS -#ifndef BENCHMARK_OS_FUCHSIA +#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) #include #endif #include @@ -53,10 +53,13 @@ namespace benchmark { namespace { // For non-dense Range, intermediate values are powers of kRangeMultiplier. -static const int kRangeMultiplier = 8; +static constexpr int kRangeMultiplier = 8; + // The size of a benchmark family determines is the number of inputs to repeat // the benchmark on. If this is "large" then warn the user during configuration. -static const size_t kMaxFamilySize = 100; +static constexpr size_t kMaxFamilySize = 100; + +static constexpr char kDisabledPrefix[] = "DISABLED_"; } // end namespace namespace internal { @@ -116,10 +119,10 @@ bool BenchmarkFamilies::FindBenchmarks( // Make regular expression out of command-line flag std::string error_msg; Regex re; - bool isNegativeFilter = false; + bool is_negative_filter = false; if (spec[0] == '-') { spec.replace(0, 1, ""); - isNegativeFilter = true; + is_negative_filter = true; } if (!re.Init(spec, &error_msg)) { Err << "Could not compile benchmark re: " << error_msg << std::endl; @@ -154,7 +157,8 @@ bool BenchmarkFamilies::FindBenchmarks( << " will be repeated at least " << family_size << " times.\n"; } // reserve in the special case the regex ".", since we know the final - // family size. + // family size. this doesn't take into account any disabled benchmarks + // so worst case we reserve more than we need. if (spec == ".") benchmarks->reserve(benchmarks->size() + family_size); for (auto const& args : family->args_) { @@ -164,8 +168,9 @@ bool BenchmarkFamilies::FindBenchmarks( num_threads); const auto full_name = instance.name().str(); - if ((re.Match(full_name) && !isNegativeFilter) || - (!re.Match(full_name) && isNegativeFilter)) { + if (full_name.rfind(kDisabledPrefix, 0) != 0 && + ((re.Match(full_name) && !is_negative_filter) || + (!re.Match(full_name) && is_negative_filter))) { benchmarks->push_back(std::move(instance)); ++per_family_instance_index; @@ -199,12 +204,14 @@ bool FindBenchmarksInternal(const std::string& re, // Benchmark //=============================================================================// -Benchmark::Benchmark(const char* name) +Benchmark::Benchmark(const std::string& name) : name_(name), aggregation_report_mode_(ARM_Unspecified), - time_unit_(kNanosecond), + time_unit_(GetDefaultTimeUnit()), + use_default_time_unit_(true), range_multiplier_(kRangeMultiplier), min_time_(0), + min_warmup_time_(0), iterations_(0), repetitions_(0), measure_process_cpu_time_(false), @@ -223,7 +230,7 @@ Benchmark::Benchmark(const char* name) Benchmark::~Benchmark() {} Benchmark* Benchmark::Name(const std::string& name) { - SetName(name.c_str()); + SetName(name); return this; } @@ -235,6 +242,7 @@ Benchmark* Benchmark::Arg(int64_t x) { Benchmark* Benchmark::Unit(TimeUnit unit) { time_unit_ = unit; + use_default_time_unit_ = false; return this; } @@ -348,9 +356,17 @@ Benchmark* Benchmark::MinTime(double t) { return this; } +Benchmark* Benchmark::MinWarmUpTime(double t) { + BM_CHECK(t >= 0.0); + BM_CHECK(iterations_ == 0); + min_warmup_time_ = t; + return this; +} + Benchmark* Benchmark::Iterations(IterationCount n) { BM_CHECK(n > 0); BM_CHECK(IsZero(min_time_)); + BM_CHECK(IsZero(min_warmup_time_)); iterations_ = n; return this; } @@ -452,7 +468,9 @@ Benchmark* Benchmark::ThreadPerCpu() { return this; } -void Benchmark::SetName(const char* name) { name_ = name; } +void Benchmark::SetName(const std::string& name) { name_ = name; } + +const char* Benchmark::GetName() const { return name_.c_str(); } int Benchmark::ArgsCnt() const { if (args_.empty()) { @@ -462,6 +480,16 @@ int Benchmark::ArgsCnt() const { return static_cast(args_.front().size()); } +const char* Benchmark::GetArgName(int arg) const { + BM_CHECK_GE(arg, 0); + BM_CHECK_LT(arg, static_cast(arg_names_.size())); + return arg_names_[arg].c_str(); +} + +TimeUnit Benchmark::GetTimeUnit() const { + return use_default_time_unit_ ? GetDefaultTimeUnit() : time_unit_; +} + //=============================================================================// // FunctionBenchmark //=============================================================================// diff --git a/third-party/benchmark/src/benchmark_register.h b/third-party/benchmark/src/benchmark_register.h index d3f4974e9074..53367c707cf4 100644 --- a/third-party/benchmark/src/benchmark_register.h +++ b/third-party/benchmark/src/benchmark_register.h @@ -1,6 +1,7 @@ #ifndef BENCHMARK_REGISTER_H #define BENCHMARK_REGISTER_H +#include #include #include @@ -23,7 +24,7 @@ typename std::vector::iterator AddPowers(std::vector* dst, T lo, T hi, static const T kmax = std::numeric_limits::max(); // Space out the values in multiples of "mult" - for (T i = static_cast(1); i <= hi; i *= mult) { + for (T i = static_cast(1); i <= hi; i *= static_cast(mult)) { if (i >= lo) { dst->push_back(i); } @@ -32,7 +33,7 @@ typename std::vector::iterator AddPowers(std::vector* dst, T lo, T hi, if (i > kmax / mult) break; } - return dst->begin() + start_offset; + return dst->begin() + static_cast(start_offset); } template diff --git a/third-party/benchmark/src/benchmark_runner.cc b/third-party/benchmark/src/benchmark_runner.cc index eac807b066f1..dcddb437e37d 100644 --- a/third-party/benchmark/src/benchmark_runner.cc +++ b/third-party/benchmark/src/benchmark_runner.cc @@ -19,7 +19,7 @@ #include "internal_macros.h" #ifndef BENCHMARK_OS_WINDOWS -#ifndef BENCHMARK_OS_FUCHSIA +#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) #include #endif #include @@ -28,11 +28,14 @@ #include #include +#include +#include #include #include #include #include #include +#include #include #include #include @@ -61,7 +64,9 @@ MemoryManager* memory_manager = nullptr; namespace { -static constexpr IterationCount kMaxIterations = 1000000000; +static constexpr IterationCount kMaxIterations = 1000000000000; +const double kDefaultMinTime = + std::strtod(::benchmark::kDefaultMinTimeStr, /*p_end*/ nullptr); BenchmarkReporter::Run CreateRunReport( const benchmark::internal::BenchmarkInstance& b, @@ -75,8 +80,8 @@ BenchmarkReporter::Run CreateRunReport( report.run_name = b.name(); report.family_index = b.family_index(); report.per_family_instance_index = b.per_family_instance_index(); - report.error_occurred = results.has_error_; - report.error_message = results.error_message_; + report.skipped = results.skipped_; + report.skip_message = results.skip_message_; report.report_label = results.report_label_; // This is the total iterations across all threads. report.iterations = results.iterations; @@ -85,12 +90,13 @@ BenchmarkReporter::Run CreateRunReport( report.repetition_index = repetition_index; report.repetitions = repeats; - if (!report.error_occurred) { + if (!report.skipped) { if (b.use_manual_time()) { report.real_accumulated_time = results.manual_time_used; } else { report.real_accumulated_time = results.real_time_used; } + report.use_real_time_for_initial_big_o = b.use_manual_time(); report.cpu_accumulated_time = results.cpu_time_used; report.complexity_n = results.complexity_n; report.complexity = b.complexity(); @@ -103,7 +109,7 @@ BenchmarkReporter::Run CreateRunReport( report.memory_result = memory_result; report.allocs_per_iter = memory_iterations ? static_cast(memory_result->num_allocs) / - memory_iterations + static_cast(memory_iterations) : 0; } @@ -122,9 +128,10 @@ void RunInThread(const BenchmarkInstance* b, IterationCount iters, b->measure_process_cpu_time() ? internal::ThreadTimer::CreateProcessCpuTime() : internal::ThreadTimer::Create()); + State st = b->Run(iters, thread_id, &timer, manager, perf_counters_measurement); - BM_CHECK(st.error_occurred() || st.iterations() >= st.max_iterations) + BM_CHECK(st.skipped() || st.iterations() >= st.max_iterations) << "Benchmark returned before State::KeepRunning() returned false!"; { MutexLock l(manager->GetBenchmarkMutex()); @@ -139,24 +146,100 @@ void RunInThread(const BenchmarkInstance* b, IterationCount iters, manager->NotifyThreadComplete(); } +double ComputeMinTime(const benchmark::internal::BenchmarkInstance& b, + const BenchTimeType& iters_or_time) { + if (!IsZero(b.min_time())) return b.min_time(); + // If the flag was used to specify number of iters, then return the default + // min_time. + if (iters_or_time.tag == BenchTimeType::ITERS) return kDefaultMinTime; + + return iters_or_time.time; +} + +IterationCount ComputeIters(const benchmark::internal::BenchmarkInstance& b, + const BenchTimeType& iters_or_time) { + if (b.iterations() != 0) return b.iterations(); + + // We've already concluded that this flag is currently used to pass + // iters but do a check here again anyway. + BM_CHECK(iters_or_time.tag == BenchTimeType::ITERS); + return iters_or_time.iters; +} + } // end namespace +BenchTimeType ParseBenchMinTime(const std::string& value) { + BenchTimeType ret; + + if (value.empty()) { + ret.tag = BenchTimeType::TIME; + ret.time = 0.0; + return ret; + } + + if (value.back() == 'x') { + char* p_end; + // Reset errno before it's changed by strtol. + errno = 0; + IterationCount num_iters = std::strtol(value.c_str(), &p_end, 10); + + // After a valid parse, p_end should have been set to + // point to the 'x' suffix. + BM_CHECK(errno == 0 && p_end != nullptr && *p_end == 'x') + << "Malformed iters value passed to --benchmark_min_time: `" << value + << "`. Expected --benchmark_min_time=x."; + + ret.tag = BenchTimeType::ITERS; + ret.iters = num_iters; + return ret; + } + + bool has_suffix = value.back() == 's'; + if (!has_suffix) { + BM_VLOG(0) << "Value passed to --benchmark_min_time should have a suffix. " + "Eg., `30s` for 30-seconds."; + } + + char* p_end; + // Reset errno before it's changed by strtod. + errno = 0; + double min_time = std::strtod(value.c_str(), &p_end); + + // After a successful parse, p_end should point to the suffix 's', + // or the end of the string if the suffix was omitted. + BM_CHECK(errno == 0 && p_end != nullptr && + ((has_suffix && *p_end == 's') || *p_end == '\0')) + << "Malformed seconds value passed to --benchmark_min_time: `" << value + << "`. Expected --benchmark_min_time=x."; + + ret.tag = BenchTimeType::TIME; + ret.time = min_time; + + return ret; +} + BenchmarkRunner::BenchmarkRunner( const benchmark::internal::BenchmarkInstance& b_, + PerfCountersMeasurement* pcm_, BenchmarkReporter::PerFamilyRunReports* reports_for_family_) : b(b_), reports_for_family(reports_for_family_), - min_time(!IsZero(b.min_time()) ? b.min_time() : FLAGS_benchmark_min_time), + parsed_benchtime_flag(ParseBenchMinTime(FLAGS_benchmark_min_time)), + min_time(ComputeMinTime(b_, parsed_benchtime_flag)), + min_warmup_time((!IsZero(b.min_time()) && b.min_warmup_time() > 0.0) + ? b.min_warmup_time() + : FLAGS_benchmark_min_warmup_time), + warmup_done(!(min_warmup_time > 0.0)), repeats(b.repetitions() != 0 ? b.repetitions() : FLAGS_benchmark_repetitions), - has_explicit_iteration_count(b.iterations() != 0), + has_explicit_iteration_count(b.iterations() != 0 || + parsed_benchtime_flag.tag == + BenchTimeType::ITERS), pool(b.threads() - 1), - iters(has_explicit_iteration_count ? b.iterations() : 1), - perf_counters_measurement( - PerfCounters::Create(StrSplit(FLAGS_benchmark_perf_counters, ','))), - perf_counters_measurement_ptr(perf_counters_measurement.IsValid() - ? &perf_counters_measurement - : nullptr) { + iters(has_explicit_iteration_count + ? ComputeIters(b_, parsed_benchtime_flag) + : 1), + perf_counters_measurement_ptr(pcm_) { run_results.display_report_aggregates_only = (FLAGS_benchmark_report_aggregates_only || FLAGS_benchmark_display_aggregates_only); @@ -169,7 +252,7 @@ BenchmarkRunner::BenchmarkRunner( run_results.file_report_aggregates_only = (b.aggregation_report_mode() & internal::ARM_FileReportAggregatesOnly); BM_CHECK(FLAGS_benchmark_perf_counters.empty() || - perf_counters_measurement.IsValid()) + (perf_counters_measurement_ptr->num_counters() == 0)) << "Perf counters were requested but could not be set up."; } } @@ -232,20 +315,20 @@ IterationCount BenchmarkRunner::PredictNumItersNeeded( const IterationResults& i) const { // See how much iterations should be increased by. // Note: Avoid division by zero with max(seconds, 1ns). - double multiplier = min_time * 1.4 / std::max(i.seconds, 1e-9); + double multiplier = GetMinTimeToApply() * 1.4 / std::max(i.seconds, 1e-9); // If our last run was at least 10% of FLAGS_benchmark_min_time then we // use the multiplier directly. // Otherwise we use at most 10 times expansion. // NOTE: When the last run was at least 10% of the min time the max // expansion should be 14x. - bool is_significant = (i.seconds / min_time) > 0.1; + const bool is_significant = (i.seconds / GetMinTimeToApply()) > 0.1; multiplier = is_significant ? multiplier : 10.0; // So what seems to be the sufficiently-large iteration count? Round up. const IterationCount max_next_iters = static_cast( - std::lround(std::max(multiplier * static_cast(i.iters), - static_cast(i.iters) + 1.0))); - // But we do have *some* sanity limits though.. + std::llround(std::max(multiplier * static_cast(i.iters), + static_cast(i.iters) + 1.0))); + // But we do have *some* limits though.. const IterationCount next_iters = std::min(max_next_iters, kMaxIterations); BM_VLOG(3) << "Next iters: " << next_iters << ", " << multiplier << "\n"; @@ -257,21 +340,80 @@ bool BenchmarkRunner::ShouldReportIterationResults( // Determine if this run should be reported; // Either it has run for a sufficient amount of time // or because an error was reported. - return i.results.has_error_ || + return i.results.skipped_ || i.iters >= kMaxIterations || // Too many iterations already. - i.seconds >= min_time || // The elapsed time is large enough. + i.seconds >= + GetMinTimeToApply() || // The elapsed time is large enough. // CPU time is specified but the elapsed real time greatly exceeds // the minimum time. - // Note that user provided timers are except from this sanity check. - ((i.results.real_time_used >= 5 * min_time) && !b.use_manual_time()); + // Note that user provided timers are except from this test. + ((i.results.real_time_used >= 5 * GetMinTimeToApply()) && + !b.use_manual_time()); +} + +double BenchmarkRunner::GetMinTimeToApply() const { + // In order to re-use functionality to run and measure benchmarks for running + // a warmup phase of the benchmark, we need a way of telling whether to apply + // min_time or min_warmup_time. This function will figure out if we are in the + // warmup phase and therefore need to apply min_warmup_time or if we already + // in the benchmarking phase and min_time needs to be applied. + return warmup_done ? min_time : min_warmup_time; +} + +void BenchmarkRunner::FinishWarmUp(const IterationCount& i) { + warmup_done = true; + iters = i; +} + +void BenchmarkRunner::RunWarmUp() { + // Use the same mechanisms for warming up the benchmark as used for actually + // running and measuring the benchmark. + IterationResults i_warmup; + // Dont use the iterations determined in the warmup phase for the actual + // measured benchmark phase. While this may be a good starting point for the + // benchmark and it would therefore get rid of the need to figure out how many + // iterations are needed if min_time is set again, this may also be a complete + // wrong guess since the warmup loops might be considerably slower (e.g + // because of caching effects). + const IterationCount i_backup = iters; + + for (;;) { + b.Setup(); + i_warmup = DoNIterations(); + b.Teardown(); + + const bool finish = ShouldReportIterationResults(i_warmup); + + if (finish) { + FinishWarmUp(i_backup); + break; + } + + // Although we are running "only" a warmup phase where running enough + // iterations at once without measuring time isn't as important as it is for + // the benchmarking phase, we still do it the same way as otherwise it is + // very confusing for the user to know how to choose a proper value for + // min_warmup_time if a different approach on running it is used. + iters = PredictNumItersNeeded(i_warmup); + assert(iters > i_warmup.iters && + "if we did more iterations than we want to do the next time, " + "then we should have accepted the current iteration run."); + } } void BenchmarkRunner::DoOneRepetition() { assert(HasRepeatsRemaining() && "Already done all repetitions?"); const bool is_the_first_repetition = num_repetitions_done == 0; - IterationResults i; + // In case a warmup phase is requested by the benchmark, run it now. + // After running the warmup phase the BenchmarkRunner should be in a state as + // this warmup never happened except the fact that warmup_done is set. Every + // other manipulation of the BenchmarkRunner instance would be a bug! Please + // fix it. + if (!warmup_done) RunWarmUp(); + + IterationResults i; // We *may* be gradually increasing the length (iteration count) // of the benchmark until we decide the results are significant. // And once we do, we report those last results and exit. @@ -324,10 +466,7 @@ void BenchmarkRunner::DoOneRepetition() { manager->WaitForAllThreads(); manager.reset(); b.Teardown(); - - BENCHMARK_DISABLE_DEPRECATED_WARNING - memory_manager->Stop(memory_result); - BENCHMARK_RESTORE_DEPRECATED_WARNING + memory_manager->Stop(*memory_result); } // Ok, now actually report. @@ -337,7 +476,7 @@ void BenchmarkRunner::DoOneRepetition() { if (reports_for_family) { ++reports_for_family->num_runs_done; - if (!report.error_occurred) reports_for_family->Runs.push_back(report); + if (!report.skipped) reports_for_family->Runs.push_back(report); } run_results.non_aggregates.push_back(report); diff --git a/third-party/benchmark/src/benchmark_runner.h b/third-party/benchmark/src/benchmark_runner.h index 752eefdc26fa..db2fa04396c5 100644 --- a/third-party/benchmark/src/benchmark_runner.h +++ b/third-party/benchmark/src/benchmark_runner.h @@ -25,7 +25,8 @@ namespace benchmark { -BM_DECLARE_double(benchmark_min_time); +BM_DECLARE_string(benchmark_min_time); +BM_DECLARE_double(benchmark_min_warmup_time); BM_DECLARE_int32(benchmark_repetitions); BM_DECLARE_bool(benchmark_report_aggregates_only); BM_DECLARE_bool(benchmark_display_aggregates_only); @@ -43,9 +44,21 @@ struct RunResults { bool file_report_aggregates_only = false; }; +struct BENCHMARK_EXPORT BenchTimeType { + enum { ITERS, TIME } tag; + union { + IterationCount iters; + double time; + }; +}; + +BENCHMARK_EXPORT +BenchTimeType ParseBenchMinTime(const std::string& value); + class BenchmarkRunner { public: BenchmarkRunner(const benchmark::internal::BenchmarkInstance& b_, + benchmark::internal::PerfCountersMeasurement* pmc_, BenchmarkReporter::PerFamilyRunReports* reports_for_family); int GetNumRepeats() const { return repeats; } @@ -62,13 +75,22 @@ class BenchmarkRunner { return reports_for_family; } + double GetMinTime() const { return min_time; } + + bool HasExplicitIters() const { return has_explicit_iteration_count; } + + IterationCount GetIters() const { return iters; } + private: RunResults run_results; const benchmark::internal::BenchmarkInstance& b; BenchmarkReporter::PerFamilyRunReports* reports_for_family; + BenchTimeType parsed_benchtime_flag; const double min_time; + const double min_warmup_time; + bool warmup_done; const int repeats; const bool has_explicit_iteration_count; @@ -82,8 +104,7 @@ class BenchmarkRunner { // So only the first repetition has to find/calculate it, // the other repetitions will just use that precomputed iteration count. - PerfCountersMeasurement perf_counters_measurement; - PerfCountersMeasurement* const perf_counters_measurement_ptr; + PerfCountersMeasurement* const perf_counters_measurement_ptr = nullptr; struct IterationResults { internal::ThreadManager::Result results; @@ -95,6 +116,12 @@ class BenchmarkRunner { IterationCount PredictNumItersNeeded(const IterationResults& i) const; bool ShouldReportIterationResults(const IterationResults& i) const; + + double GetMinTimeToApply() const; + + void FinishWarmUp(const IterationCount& i); + + void RunWarmUp(); }; } // namespace internal diff --git a/third-party/benchmark/src/check.cc b/third-party/benchmark/src/check.cc new file mode 100644 index 000000000000..5f7526e08d6e --- /dev/null +++ b/third-party/benchmark/src/check.cc @@ -0,0 +1,11 @@ +#include "check.h" + +namespace benchmark { +namespace internal { + +static AbortHandlerT* handler = &std::abort; + +BENCHMARK_EXPORT AbortHandlerT*& GetAbortHandler() { return handler; } + +} // namespace internal +} // namespace benchmark diff --git a/third-party/benchmark/src/check.h b/third-party/benchmark/src/check.h index 0efd13ff4db6..c1cd5e85e44c 100644 --- a/third-party/benchmark/src/check.h +++ b/third-party/benchmark/src/check.h @@ -5,18 +5,34 @@ #include #include +#include "benchmark/export.h" #include "internal_macros.h" #include "log.h" +#if defined(__GNUC__) || defined(__clang__) +#define BENCHMARK_NOEXCEPT noexcept +#define BENCHMARK_NOEXCEPT_OP(x) noexcept(x) +#elif defined(_MSC_VER) && !defined(__clang__) +#if _MSC_VER >= 1900 +#define BENCHMARK_NOEXCEPT noexcept +#define BENCHMARK_NOEXCEPT_OP(x) noexcept(x) +#else +#define BENCHMARK_NOEXCEPT +#define BENCHMARK_NOEXCEPT_OP(x) +#endif +#define __func__ __FUNCTION__ +#else +#define BENCHMARK_NOEXCEPT +#define BENCHMARK_NOEXCEPT_OP(x) +#endif + namespace benchmark { namespace internal { typedef void(AbortHandlerT)(); -inline AbortHandlerT*& GetAbortHandler() { - static AbortHandlerT* handler = &std::abort; - return handler; -} +BENCHMARK_EXPORT +AbortHandlerT*& GetAbortHandler(); BENCHMARK_NORETURN inline void CallAbortHandler() { GetAbortHandler()(); @@ -36,10 +52,17 @@ class CheckHandler { LogType& GetLog() { return log_; } +#if defined(COMPILER_MSVC) +#pragma warning(push) +#pragma warning(disable : 4722) +#endif BENCHMARK_NORETURN ~CheckHandler() BENCHMARK_NOEXCEPT_OP(false) { log_ << std::endl; CallAbortHandler(); } +#if defined(COMPILER_MSVC) +#pragma warning(pop) +#endif CheckHandler& operator=(const CheckHandler&) = delete; CheckHandler(const CheckHandler&) = delete; diff --git a/third-party/benchmark/src/colorprint.cc b/third-party/benchmark/src/colorprint.cc index 1a000a063736..abc71492f77a 100644 --- a/third-party/benchmark/src/colorprint.cc +++ b/third-party/benchmark/src/colorprint.cc @@ -96,18 +96,18 @@ std::string FormatString(const char* msg, va_list args) { // currently there is no error handling for failure, so this is hack. BM_CHECK(ret >= 0); - if (ret == 0) // handle empty expansion + if (ret == 0) { // handle empty expansion return {}; - else if (static_cast(ret) < size) + } + if (static_cast(ret) < size) { return local_buff; - else { - // we did not provide a long enough buffer on our first attempt. - size = static_cast(ret) + 1; // + 1 for the null byte - std::unique_ptr buff(new char[size]); - ret = vsnprintf(buff.get(), size, msg, args); - BM_CHECK(ret > 0 && (static_cast(ret)) < size); - return buff.get(); } + // we did not provide a long enough buffer on our first attempt. + size = static_cast(ret) + 1; // + 1 for the null byte + std::unique_ptr buff(new char[size]); + ret = vsnprintf(buff.get(), size, msg, args); + BM_CHECK(ret > 0 && (static_cast(ret)) < size); + return buff.get(); } std::string FormatString(const char* msg, ...) { @@ -140,12 +140,12 @@ void ColorPrintf(std::ostream& out, LogColor color, const char* fmt, // We need to flush the stream buffers into the console before each // SetConsoleTextAttribute call lest it affect the text that is already // printed but has not yet reached the console. - fflush(stdout); + out.flush(); SetConsoleTextAttribute(stdout_handle, GetPlatformColorCode(color) | FOREGROUND_INTENSITY); - vprintf(fmt, args); + out << FormatString(fmt, args); - fflush(stdout); + out.flush(); // Restores the text color. SetConsoleTextAttribute(stdout_handle, old_color_attrs); #else @@ -163,12 +163,24 @@ bool IsColorTerminal() { #else // On non-Windows platforms, we rely on the TERM variable. This list of // supported TERM values is copied from Google Test: - // . + // . const char* const SUPPORTED_TERM_VALUES[] = { - "xterm", "xterm-color", "xterm-256color", - "screen", "screen-256color", "tmux", - "tmux-256color", "rxvt-unicode", "rxvt-unicode-256color", - "linux", "cygwin", + "xterm", + "xterm-color", + "xterm-256color", + "screen", + "screen-256color", + "tmux", + "tmux-256color", + "rxvt-unicode", + "rxvt-unicode-256color", + "linux", + "cygwin", + "xterm-kitty", + "alacritty", + "foot", + "foot-extra", + "wezterm", }; const char* const term = getenv("TERM"); diff --git a/third-party/benchmark/src/commandlineflags.cc b/third-party/benchmark/src/commandlineflags.cc index 9615e351ffae..dcb414959df4 100644 --- a/third-party/benchmark/src/commandlineflags.cc +++ b/third-party/benchmark/src/commandlineflags.cc @@ -121,12 +121,14 @@ static std::string FlagToEnvVar(const char* flag) { } // namespace +BENCHMARK_EXPORT bool BoolFromEnv(const char* flag, bool default_val) { const std::string env_var = FlagToEnvVar(flag); const char* const value_str = getenv(env_var.c_str()); return value_str == nullptr ? default_val : IsTruthyFlagValue(value_str); } +BENCHMARK_EXPORT int32_t Int32FromEnv(const char* flag, int32_t default_val) { const std::string env_var = FlagToEnvVar(flag); const char* const value_str = getenv(env_var.c_str()); @@ -139,6 +141,7 @@ int32_t Int32FromEnv(const char* flag, int32_t default_val) { return value; } +BENCHMARK_EXPORT double DoubleFromEnv(const char* flag, double default_val) { const std::string env_var = FlagToEnvVar(flag); const char* const value_str = getenv(env_var.c_str()); @@ -151,12 +154,14 @@ double DoubleFromEnv(const char* flag, double default_val) { return value; } +BENCHMARK_EXPORT const char* StringFromEnv(const char* flag, const char* default_val) { const std::string env_var = FlagToEnvVar(flag); const char* const value = getenv(env_var.c_str()); return value == nullptr ? default_val : value; } +BENCHMARK_EXPORT std::map KvPairsFromEnv( const char* flag, std::map default_val) { const std::string env_var = FlagToEnvVar(flag); @@ -201,6 +206,7 @@ const char* ParseFlagValue(const char* str, const char* flag, return flag_end + 1; } +BENCHMARK_EXPORT bool ParseBoolFlag(const char* str, const char* flag, bool* value) { // Gets the value of the flag as a string. const char* const value_str = ParseFlagValue(str, flag, true); @@ -213,6 +219,7 @@ bool ParseBoolFlag(const char* str, const char* flag, bool* value) { return true; } +BENCHMARK_EXPORT bool ParseInt32Flag(const char* str, const char* flag, int32_t* value) { // Gets the value of the flag as a string. const char* const value_str = ParseFlagValue(str, flag, false); @@ -225,6 +232,7 @@ bool ParseInt32Flag(const char* str, const char* flag, int32_t* value) { value); } +BENCHMARK_EXPORT bool ParseDoubleFlag(const char* str, const char* flag, double* value) { // Gets the value of the flag as a string. const char* const value_str = ParseFlagValue(str, flag, false); @@ -237,6 +245,7 @@ bool ParseDoubleFlag(const char* str, const char* flag, double* value) { value); } +BENCHMARK_EXPORT bool ParseStringFlag(const char* str, const char* flag, std::string* value) { // Gets the value of the flag as a string. const char* const value_str = ParseFlagValue(str, flag, false); @@ -248,6 +257,7 @@ bool ParseStringFlag(const char* str, const char* flag, std::string* value) { return true; } +BENCHMARK_EXPORT bool ParseKeyValueFlag(const char* str, const char* flag, std::map* value) { const char* const value_str = ParseFlagValue(str, flag, false); @@ -263,23 +273,26 @@ bool ParseKeyValueFlag(const char* str, const char* flag, return true; } +BENCHMARK_EXPORT bool IsFlag(const char* str, const char* flag) { return (ParseFlagValue(str, flag, true) != nullptr); } +BENCHMARK_EXPORT bool IsTruthyFlagValue(const std::string& value) { if (value.size() == 1) { char v = value[0]; return isalnum(v) && !(v == '0' || v == 'f' || v == 'F' || v == 'n' || v == 'N'); - } else if (!value.empty()) { + } + if (!value.empty()) { std::string value_lower(value); std::transform(value_lower.begin(), value_lower.end(), value_lower.begin(), [](char c) { return static_cast(::tolower(c)); }); return !(value_lower == "false" || value_lower == "no" || value_lower == "off"); - } else - return true; + } + return true; } } // end namespace benchmark diff --git a/third-party/benchmark/src/commandlineflags.h b/third-party/benchmark/src/commandlineflags.h index 5baaf11784df..7882628975ea 100644 --- a/third-party/benchmark/src/commandlineflags.h +++ b/third-party/benchmark/src/commandlineflags.h @@ -5,28 +5,33 @@ #include #include +#include "benchmark/export.h" + // Macro for referencing flags. #define FLAG(name) FLAGS_##name // Macros for declaring flags. -#define BM_DECLARE_bool(name) extern bool FLAG(name) -#define BM_DECLARE_int32(name) extern int32_t FLAG(name) -#define BM_DECLARE_double(name) extern double FLAG(name) -#define BM_DECLARE_string(name) extern std::string FLAG(name) +#define BM_DECLARE_bool(name) BENCHMARK_EXPORT extern bool FLAG(name) +#define BM_DECLARE_int32(name) BENCHMARK_EXPORT extern int32_t FLAG(name) +#define BM_DECLARE_double(name) BENCHMARK_EXPORT extern double FLAG(name) +#define BM_DECLARE_string(name) BENCHMARK_EXPORT extern std::string FLAG(name) #define BM_DECLARE_kvpairs(name) \ - extern std::map FLAG(name) + BENCHMARK_EXPORT extern std::map FLAG(name) // Macros for defining flags. #define BM_DEFINE_bool(name, default_val) \ - bool FLAG(name) = benchmark::BoolFromEnv(#name, default_val) + BENCHMARK_EXPORT bool FLAG(name) = benchmark::BoolFromEnv(#name, default_val) #define BM_DEFINE_int32(name, default_val) \ - int32_t FLAG(name) = benchmark::Int32FromEnv(#name, default_val) + BENCHMARK_EXPORT int32_t FLAG(name) = \ + benchmark::Int32FromEnv(#name, default_val) #define BM_DEFINE_double(name, default_val) \ - double FLAG(name) = benchmark::DoubleFromEnv(#name, default_val) + BENCHMARK_EXPORT double FLAG(name) = \ + benchmark::DoubleFromEnv(#name, default_val) #define BM_DEFINE_string(name, default_val) \ - std::string FLAG(name) = benchmark::StringFromEnv(#name, default_val) -#define BM_DEFINE_kvpairs(name, default_val) \ - std::map FLAG(name) = \ + BENCHMARK_EXPORT std::string FLAG(name) = \ + benchmark::StringFromEnv(#name, default_val) +#define BM_DEFINE_kvpairs(name, default_val) \ + BENCHMARK_EXPORT std::map FLAG(name) = \ benchmark::KvPairsFromEnv(#name, default_val) namespace benchmark { @@ -35,6 +40,7 @@ namespace benchmark { // // If the variable exists, returns IsTruthyFlagValue() value; if not, // returns the given default value. +BENCHMARK_EXPORT bool BoolFromEnv(const char* flag, bool default_val); // Parses an Int32 from the environment variable corresponding to the given @@ -42,6 +48,7 @@ bool BoolFromEnv(const char* flag, bool default_val); // // If the variable exists, returns ParseInt32() value; if not, returns // the given default value. +BENCHMARK_EXPORT int32_t Int32FromEnv(const char* flag, int32_t default_val); // Parses an Double from the environment variable corresponding to the given @@ -49,6 +56,7 @@ int32_t Int32FromEnv(const char* flag, int32_t default_val); // // If the variable exists, returns ParseDouble(); if not, returns // the given default value. +BENCHMARK_EXPORT double DoubleFromEnv(const char* flag, double default_val); // Parses a string from the environment variable corresponding to the given @@ -56,6 +64,7 @@ double DoubleFromEnv(const char* flag, double default_val); // // If variable exists, returns its value; if not, returns // the given default value. +BENCHMARK_EXPORT const char* StringFromEnv(const char* flag, const char* default_val); // Parses a set of kvpairs from the environment variable corresponding to the @@ -63,6 +72,7 @@ const char* StringFromEnv(const char* flag, const char* default_val); // // If variable exists, returns its value; if not, returns // the given default value. +BENCHMARK_EXPORT std::map KvPairsFromEnv( const char* flag, std::map default_val); @@ -75,40 +85,47 @@ std::map KvPairsFromEnv( // // On success, stores the value of the flag in *value, and returns // true. On failure, returns false without changing *value. +BENCHMARK_EXPORT bool ParseBoolFlag(const char* str, const char* flag, bool* value); // Parses a string for an Int32 flag, in the form of "--flag=value". // // On success, stores the value of the flag in *value, and returns // true. On failure, returns false without changing *value. +BENCHMARK_EXPORT bool ParseInt32Flag(const char* str, const char* flag, int32_t* value); // Parses a string for a Double flag, in the form of "--flag=value". // // On success, stores the value of the flag in *value, and returns // true. On failure, returns false without changing *value. +BENCHMARK_EXPORT bool ParseDoubleFlag(const char* str, const char* flag, double* value); // Parses a string for a string flag, in the form of "--flag=value". // // On success, stores the value of the flag in *value, and returns // true. On failure, returns false without changing *value. +BENCHMARK_EXPORT bool ParseStringFlag(const char* str, const char* flag, std::string* value); // Parses a string for a kvpairs flag in the form "--flag=key=value,key=value" // // On success, stores the value of the flag in *value and returns true. On // failure returns false, though *value may have been mutated. +BENCHMARK_EXPORT bool ParseKeyValueFlag(const char* str, const char* flag, std::map* value); // Returns true if the string matches the flag. +BENCHMARK_EXPORT bool IsFlag(const char* str, const char* flag); // Returns true unless value starts with one of: '0', 'f', 'F', 'n' or 'N', or // some non-alphanumeric character. Also returns false if the value matches // one of 'no', 'false', 'off' (case-insensitive). As a special case, also // returns true if value is the empty string. +BENCHMARK_EXPORT bool IsTruthyFlagValue(const std::string& value); } // end namespace benchmark diff --git a/third-party/benchmark/src/complexity.cc b/third-party/benchmark/src/complexity.cc index 825c57394a8c..eee3122646f9 100644 --- a/third-party/benchmark/src/complexity.cc +++ b/third-party/benchmark/src/complexity.cc @@ -37,12 +37,14 @@ BigOFunc* FittingCurve(BigO complexity) { return [](IterationCount n) -> double { return std::pow(n, 3); }; case oLogN: /* Note: can't use log2 because Android's GNU STL lacks it */ - return - [](IterationCount n) { return kLog2E * log(static_cast(n)); }; + return [](IterationCount n) { + return kLog2E * std::log(static_cast(n)); + }; case oNLogN: /* Note: can't use log2 because Android's GNU STL lacks it */ return [](IterationCount n) { - return kLog2E * n * log(static_cast(n)); + return kLog2E * static_cast(n) * + std::log(static_cast(n)); }; case o1: default: @@ -75,12 +77,12 @@ std::string GetBigOString(BigO complexity) { // given by the lambda expression. // - n : Vector containing the size of the benchmark tests. // - time : Vector containing the times for the benchmark tests. -// - fitting_curve : lambda expression (e.g. [](int64_t n) {return n; };). +// - fitting_curve : lambda expression (e.g. [](ComplexityN n) {return n; };). // For a deeper explanation on the algorithm logic, please refer to // https://en.wikipedia.org/wiki/Least_squares#Least_squares,_regression_analysis_and_statistics -LeastSq MinimalLeastSq(const std::vector& n, +LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, BigOFunc* fitting_curve) { double sigma_gn_squared = 0.0; @@ -105,12 +107,12 @@ LeastSq MinimalLeastSq(const std::vector& n, double rms = 0.0; for (size_t i = 0; i < n.size(); ++i) { double fit = result.coef * fitting_curve(n[i]); - rms += pow((time[i] - fit), 2); + rms += std::pow((time[i] - fit), 2); } // Normalized RMS by the mean of the observed values - double mean = sigma_time / n.size(); - result.rms = sqrt(rms / n.size()) / mean; + double mean = sigma_time / static_cast(n.size()); + result.rms = std::sqrt(rms / static_cast(n.size())) / mean; return result; } @@ -122,7 +124,7 @@ LeastSq MinimalLeastSq(const std::vector& n, // - complexity : If different than oAuto, the fitting curve will stick to // this one. If it is oAuto, it will be calculated the best // fitting curve. -LeastSq MinimalLeastSq(const std::vector& n, +LeastSq MinimalLeastSq(const std::vector& n, const std::vector& time, const BigO complexity) { BM_CHECK_EQ(n.size(), time.size()); BM_CHECK_GE(n.size(), 2); // Do not compute fitting curve is less than two @@ -162,7 +164,7 @@ std::vector ComputeBigO( if (reports.size() < 2) return results; // Accumulators. - std::vector n; + std::vector n; std::vector real_time; std::vector cpu_time; @@ -171,8 +173,10 @@ std::vector ComputeBigO( BM_CHECK_GT(run.complexity_n, 0) << "Did you forget to call SetComplexityN?"; n.push_back(run.complexity_n); - real_time.push_back(run.real_accumulated_time / run.iterations); - cpu_time.push_back(run.cpu_accumulated_time / run.iterations); + real_time.push_back(run.real_accumulated_time / + static_cast(run.iterations)); + cpu_time.push_back(run.cpu_accumulated_time / + static_cast(run.iterations)); } LeastSq result_cpu; @@ -182,8 +186,19 @@ std::vector ComputeBigO( result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity_lambda); result_real = MinimalLeastSq(n, real_time, reports[0].complexity_lambda); } else { - result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity); - result_real = MinimalLeastSq(n, real_time, result_cpu.complexity); + const BigO* InitialBigO = &reports[0].complexity; + const bool use_real_time_for_initial_big_o = + reports[0].use_real_time_for_initial_big_o; + if (use_real_time_for_initial_big_o) { + result_real = MinimalLeastSq(n, real_time, *InitialBigO); + InitialBigO = &result_real.complexity; + // The Big-O complexity for CPU time must have the same Big-O function! + } + result_cpu = MinimalLeastSq(n, cpu_time, *InitialBigO); + InitialBigO = &result_cpu.complexity; + if (!use_real_time_for_initial_big_o) { + result_real = MinimalLeastSq(n, real_time, *InitialBigO); + } } // Drop the 'args' when reporting complexity. diff --git a/third-party/benchmark/src/complexity.h b/third-party/benchmark/src/complexity.h index df29b48d29b4..0a0679b48bc8 100644 --- a/third-party/benchmark/src/complexity.h +++ b/third-party/benchmark/src/complexity.h @@ -31,7 +31,7 @@ std::vector ComputeBigO( const std::vector& reports); // This data structure will contain the result returned by MinimalLeastSq -// - coef : Estimated coeficient for the high-order term as +// - coef : Estimated coefficient for the high-order term as // interpolated from data. // - rms : Normalized Root Mean Squared Error. // - complexity : Scalability form (e.g. oN, oNLogN). In case a scalability diff --git a/third-party/benchmark/src/console_reporter.cc b/third-party/benchmark/src/console_reporter.cc index 04cc0b74e58e..35c3de2a4dba 100644 --- a/third-party/benchmark/src/console_reporter.cc +++ b/third-party/benchmark/src/console_reporter.cc @@ -33,6 +33,7 @@ namespace benchmark { +BENCHMARK_EXPORT bool ConsoleReporter::ReportContext(const Context& context) { name_field_width_ = context.name_field_width; printed_header_ = false; @@ -41,17 +42,22 @@ bool ConsoleReporter::ReportContext(const Context& context) { PrintBasicContext(&GetErrorStream(), context); #ifdef BENCHMARK_OS_WINDOWS - if ((output_options_ & OO_Color) && &std::cout != &GetOutputStream()) { - GetErrorStream() - << "Color printing is only supported for stdout on windows." - " Disabling color printing\n"; - output_options_ = static_cast(output_options_ & ~OO_Color); + if ((output_options_ & OO_Color)) { + auto stdOutBuf = std::cout.rdbuf(); + auto outStreamBuf = GetOutputStream().rdbuf(); + if (stdOutBuf != outStreamBuf) { + GetErrorStream() + << "Color printing is only supported for stdout on windows." + " Disabling color printing\n"; + output_options_ = static_cast(output_options_ & ~OO_Color); + } } #endif return true; } +BENCHMARK_EXPORT void ConsoleReporter::PrintHeader(const Run& run) { std::string str = FormatString("%-*s %13s %15s %12s", static_cast(name_field_width_), @@ -69,6 +75,7 @@ void ConsoleReporter::PrintHeader(const Run& run) { GetOutputStream() << line << "\n" << str << "\n" << line << "\n"; } +BENCHMARK_EXPORT void ConsoleReporter::ReportRuns(const std::vector& reports) { for (const auto& run : reports) { // print the header: @@ -99,6 +106,9 @@ static void IgnoreColorPrint(std::ostream& out, LogColor, const char* fmt, } static std::string FormatTime(double time) { + // For the time columns of the console printer 13 digits are reserved. One of + // them is a space and max two of them are the time unit (e.g ns). That puts + // us at 10 digits usable for the number. // Align decimal places... if (time < 1.0) { return FormatString("%10.3f", time); @@ -109,9 +119,15 @@ static std::string FormatTime(double time) { if (time < 100.0) { return FormatString("%10.1f", time); } + // Assuming the time is at max 9.9999e+99 and we have 10 digits for the + // number, we get 10-1(.)-1(e)-1(sign)-2(exponent) = 5 digits to print. + if (time > 9999999999 /*max 10 digit number*/) { + return FormatString("%1.4e", time); + } return FormatString("%10.0f", time); } +BENCHMARK_EXPORT void ConsoleReporter::PrintRunData(const Run& result) { typedef void(PrinterFn)(std::ostream&, LogColor, const char*, ...); auto& Out = GetOutputStream(); @@ -123,9 +139,13 @@ void ConsoleReporter::PrintRunData(const Run& result) { printer(Out, name_color, "%-*s ", name_field_width_, result.benchmark_name().c_str()); - if (result.error_occurred) { + if (internal::SkippedWithError == result.skipped) { printer(Out, COLOR_RED, "ERROR OCCURRED: \'%s\'", - result.error_message.c_str()); + result.skip_message.c_str()); + printer(Out, COLOR_DEFAULT, "\n"); + return; + } else if (internal::SkippedWithMessage == result.skipped) { + printer(Out, COLOR_WHITE, "SKIPPED: \'%s\'", result.skip_message.c_str()); printer(Out, COLOR_DEFAULT, "\n"); return; } diff --git a/third-party/benchmark/src/counter.cc b/third-party/benchmark/src/counter.cc index cf5b78ee3ac6..aa14cd8092f9 100644 --- a/third-party/benchmark/src/counter.cc +++ b/third-party/benchmark/src/counter.cc @@ -27,10 +27,10 @@ double Finish(Counter const& c, IterationCount iterations, double cpu_time, v /= num_threads; } if (c.flags & Counter::kIsIterationInvariant) { - v *= iterations; + v *= static_cast(iterations); } if (c.flags & Counter::kAvgIterations) { - v /= iterations; + v /= static_cast(iterations); } if (c.flags & Counter::kInvert) { // Invert is *always* last. diff --git a/third-party/benchmark/src/csv_reporter.cc b/third-party/benchmark/src/csv_reporter.cc index 1c5e9fa6689c..4b39e2c52fb9 100644 --- a/third-party/benchmark/src/csv_reporter.cc +++ b/third-party/benchmark/src/csv_reporter.cc @@ -52,11 +52,13 @@ std::string CsvEscape(const std::string& s) { return '"' + tmp + '"'; } +BENCHMARK_EXPORT bool CSVReporter::ReportContext(const Context& context) { PrintBasicContext(&GetErrorStream(), context); return true; } +BENCHMARK_EXPORT void CSVReporter::ReportRuns(const std::vector& reports) { std::ostream& Out = GetOutputStream(); @@ -103,13 +105,14 @@ void CSVReporter::ReportRuns(const std::vector& reports) { } } +BENCHMARK_EXPORT void CSVReporter::PrintRunData(const Run& run) { std::ostream& Out = GetOutputStream(); Out << CsvEscape(run.benchmark_name()) << ","; - if (run.error_occurred) { + if (run.skipped) { Out << std::string(elements.size() - 3, ','); - Out << "true,"; - Out << CsvEscape(run.error_message) << "\n"; + Out << std::boolalpha << (internal::SkippedWithError == run.skipped) << ","; + Out << CsvEscape(run.skip_message) << "\n"; return; } @@ -119,13 +122,21 @@ void CSVReporter::PrintRunData(const Run& run) { } Out << ","; - Out << run.GetAdjustedRealTime() << ","; - Out << run.GetAdjustedCPUTime() << ","; + if (run.run_type != Run::RT_Aggregate || + run.aggregate_unit == StatisticUnit::kTime) { + Out << run.GetAdjustedRealTime() << ","; + Out << run.GetAdjustedCPUTime() << ","; + } else { + assert(run.aggregate_unit == StatisticUnit::kPercentage); + Out << run.real_accumulated_time << ","; + Out << run.cpu_accumulated_time << ","; + } // Do not print timeLabel on bigO and RMS report if (run.report_big_o) { Out << GetBigOString(run.complexity); - } else if (!run.report_rms) { + } else if (!run.report_rms && + run.aggregate_unit != StatisticUnit::kPercentage) { Out << GetTimeUnitString(run.time_unit); } Out << ","; diff --git a/third-party/benchmark/src/cycleclock.h b/third-party/benchmark/src/cycleclock.h index d65d32a39d3d..eff563e7fac2 100644 --- a/third-party/benchmark/src/cycleclock.h +++ b/third-party/benchmark/src/cycleclock.h @@ -36,7 +36,8 @@ // declarations of some other intrinsics, breaking compilation. // Therefore, we simply declare __rdtsc ourselves. See also // http://connect.microsoft.com/VisualStudio/feedback/details/262047 -#if defined(COMPILER_MSVC) && !defined(_M_IX86) && !defined(_M_ARM64) +#if defined(COMPILER_MSVC) && !defined(_M_IX86) && !defined(_M_ARM64) && \ + !defined(_M_ARM64EC) extern "C" uint64_t __rdtsc(); #pragma intrinsic(__rdtsc) #endif @@ -114,7 +115,7 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { // when I know it will work. Otherwise, I'll use __rdtsc and hope // the code is being compiled with a non-ancient compiler. _asm rdtsc -#elif defined(COMPILER_MSVC) && defined(_M_ARM64) +#elif defined(COMPILER_MSVC) && (defined(_M_ARM64) || defined(_M_ARM64EC)) // See // https://docs.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics // and https://reviews.llvm.org/D53115 int64_t virtual_timer_value; @@ -132,7 +133,7 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { // Native Client does not provide any API to access cycle counter. // Use clock_gettime(CLOCK_MONOTONIC, ...) instead of gettimeofday - // because is provides nanosecond resolution (which is noticable at + // because is provides nanosecond resolution (which is noticeable at // least for PNaCl modules running on x86 Mac & Linux). // Initialize to always return 0 if clock_gettime fails. struct timespec ts = {0, 0}; @@ -173,7 +174,7 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { struct timeval tv; gettimeofday(&tv, nullptr); return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; -#elif defined(__loongarch__) +#elif defined(__loongarch__) || defined(__csky__) struct timeval tv; gettimeofday(&tv, nullptr); return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; @@ -188,15 +189,16 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { #endif return tsc; #elif defined(__riscv) // RISC-V - // Use RDCYCLE (and RDCYCLEH on riscv32) + // Use RDTIME (and RDTIMEH on riscv32). + // RDCYCLE is a privileged instruction since Linux 6.6. #if __riscv_xlen == 32 uint32_t cycles_lo, cycles_hi0, cycles_hi1; // This asm also includes the PowerPC overflow handling strategy, as above. // Implemented in assembly because Clang insisted on branching. asm volatile( - "rdcycleh %0\n" - "rdcycle %1\n" - "rdcycleh %2\n" + "rdtimeh %0\n" + "rdtime %1\n" + "rdtimeh %2\n" "sub %0, %0, %2\n" "seqz %0, %0\n" "sub %0, zero, %0\n" @@ -205,17 +207,31 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { return (static_cast(cycles_hi1) << 32) | cycles_lo; #else uint64_t cycles; - asm volatile("rdcycle %0" : "=r"(cycles)); + asm volatile("rdtime %0" : "=r"(cycles)); return cycles; #endif #elif defined(__e2k__) || defined(__elbrus__) struct timeval tv; gettimeofday(&tv, nullptr); return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; +#elif defined(__hexagon__) + uint64_t pcycle; + asm volatile("%0 = C15:14" : "=r"(pcycle)); + return static_cast(pcycle); +#elif defined(__alpha__) + // Alpha has a cycle counter, the PCC register, but it is an unsigned 32-bit + // integer and thus wraps every ~4s, making using it for tick counts + // unreliable beyond this time range. The real-time clock is low-precision, + // roughtly ~1ms, but it is the only option that can reasonable count + // indefinitely. + struct timeval tv; + gettimeofday(&tv, nullptr); + return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; #else -// The soft failover to a generic implementation is automatic only for ARM. -// For other platforms the developer is expected to make an attempt to create -// a fast implementation and use generic version if nothing better is available. + // The soft failover to a generic implementation is automatic only for ARM. + // For other platforms the developer is expected to make an attempt to create + // a fast implementation and use generic version if nothing better is + // available. #error You need to define CycleTimer for your OS and CPU #endif } diff --git a/third-party/benchmark/src/internal_macros.h b/third-party/benchmark/src/internal_macros.h index 91f367b894bc..8dd7d0c6502e 100644 --- a/third-party/benchmark/src/internal_macros.h +++ b/third-party/benchmark/src/internal_macros.h @@ -1,8 +1,6 @@ #ifndef BENCHMARK_INTERNAL_MACROS_H_ #define BENCHMARK_INTERNAL_MACROS_H_ -#include "benchmark/benchmark.h" - /* Needed to detect STL */ #include @@ -44,6 +42,19 @@ #define BENCHMARK_OS_CYGWIN 1 #elif defined(_WIN32) #define BENCHMARK_OS_WINDOWS 1 + // WINAPI_FAMILY_PARTITION is defined in winapifamily.h. + // We include windows.h which implicitly includes winapifamily.h for compatibility. + #ifndef NOMINMAX + #define NOMINMAX + #endif + #include + #if defined(WINAPI_FAMILY_PARTITION) + #if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) + #define BENCHMARK_OS_WINDOWS_WIN32 1 + #elif WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) + #define BENCHMARK_OS_WINDOWS_RT 1 + #endif + #endif #if defined(__MINGW32__) #define BENCHMARK_OS_MINGW 1 #endif @@ -80,6 +91,8 @@ #define BENCHMARK_OS_QNX 1 #elif defined(__MVS__) #define BENCHMARK_OS_ZOS 1 +#elif defined(__hexagon__) +#define BENCHMARK_OS_QURT 1 #endif #if defined(__ANDROID__) && defined(__GLIBCXX__) diff --git a/third-party/benchmark/src/json_reporter.cc b/third-party/benchmark/src/json_reporter.cc index e84a4ed24f9d..b8c8c94c08a0 100644 --- a/third-party/benchmark/src/json_reporter.cc +++ b/third-party/benchmark/src/json_reporter.cc @@ -28,10 +28,6 @@ #include "timers.h" namespace benchmark { -namespace internal { -extern std::map* global_context; -} - namespace { std::string StrEscape(const std::string& s) { @@ -89,12 +85,6 @@ std::string FormatKV(std::string const& key, int64_t value) { return ss.str(); } -std::string FormatKV(std::string const& key, IterationCount value) { - std::stringstream ss; - ss << '"' << StrEscape(key) << "\": " << value; - return ss.str(); -} - std::string FormatKV(std::string const& key, double value) { std::stringstream ss; ss << '"' << StrEscape(key) << "\": "; @@ -177,15 +167,25 @@ bool JSONReporter::ReportContext(const Context& context) { } out << "],\n"; + out << indent << FormatKV("library_version", GetBenchmarkVersion()); + out << ",\n"; + #if defined(NDEBUG) const char build_type[] = "release"; #else const char build_type[] = "debug"; #endif out << indent << FormatKV("library_build_type", build_type); + out << ",\n"; + + // NOTE: our json schema is not strictly tied to the library version! + out << indent << FormatKV("json_schema_version", int64_t(1)); + + std::map* global_context = + internal::GetGlobalContext(); - if (internal::global_context != nullptr) { - for (const auto& kv : *internal::global_context) { + if (global_context != nullptr) { + for (const auto& kv : *global_context) { out << ",\n"; out << indent << FormatKV(kv.first, kv.second); } @@ -261,9 +261,12 @@ void JSONReporter::PrintRunData(Run const& run) { BENCHMARK_UNREACHABLE(); }()) << ",\n"; } - if (run.error_occurred) { - out << indent << FormatKV("error_occurred", run.error_occurred) << ",\n"; - out << indent << FormatKV("error_message", run.error_message) << ",\n"; + if (internal::SkippedWithError == run.skipped) { + out << indent << FormatKV("error_occurred", true) << ",\n"; + out << indent << FormatKV("error_message", run.skip_message) << ",\n"; + } else if (internal::SkippedWithMessage == run.skipped) { + out << indent << FormatKV("skipped", true) << ",\n"; + out << indent << FormatKV("skip_message", run.skip_message) << ",\n"; } if (!run.report_big_o && !run.report_rms) { out << indent << FormatKV("iterations", run.iterations) << ",\n"; @@ -301,7 +304,8 @@ void JSONReporter::PrintRunData(Run const& run) { out << ",\n" << indent << FormatKV("max_bytes_used", memory_result.max_bytes_used); - auto report_if_present = [&out, &indent](const char* label, int64_t val) { + auto report_if_present = [&out, &indent](const std::string& label, + int64_t val) { if (val != MemoryManager::TombstoneValue) out << ",\n" << indent << FormatKV(label, val); }; diff --git a/third-party/benchmark/src/log.h b/third-party/benchmark/src/log.h index 48c071aded8f..9a21400b096d 100644 --- a/third-party/benchmark/src/log.h +++ b/third-party/benchmark/src/log.h @@ -4,7 +4,12 @@ #include #include -#include "benchmark/benchmark.h" +// NOTE: this is also defined in benchmark.h but we're trying to avoid a +// dependency. +// The _MSVC_LANG check should detect Visual Studio 2015 Update 3 and newer. +#if __cplusplus >= 201103L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201103L) +#define BENCHMARK_HAS_CXX11 +#endif namespace benchmark { namespace internal { @@ -23,7 +28,16 @@ class LogType { private: LogType(std::ostream* out) : out_(out) {} std::ostream* out_; - BENCHMARK_DISALLOW_COPY_AND_ASSIGN(LogType); + + // NOTE: we could use BENCHMARK_DISALLOW_COPY_AND_ASSIGN but we shouldn't have + // a dependency on benchmark.h from here. +#ifndef BENCHMARK_HAS_CXX11 + LogType(const LogType&); + LogType& operator=(const LogType&); +#else + LogType(const LogType&) = delete; + LogType& operator=(const LogType&) = delete; +#endif }; template @@ -47,13 +61,13 @@ inline int& LogLevel() { } inline LogType& GetNullLogInstance() { - static LogType log(nullptr); - return log; + static LogType null_log(static_cast(nullptr)); + return null_log; } inline LogType& GetErrorLogInstance() { - static LogType log(&std::clog); - return log; + static LogType error_log(&std::clog); + return error_log; } inline LogType& GetLogInstanceForLevel(int level) { diff --git a/third-party/benchmark/src/perf_counters.cc b/third-party/benchmark/src/perf_counters.cc index b2ac7687efef..d466e27e86f9 100644 --- a/third-party/benchmark/src/perf_counters.cc +++ b/third-party/benchmark/src/perf_counters.cc @@ -15,6 +15,7 @@ #include "perf_counters.h" #include +#include #include #if defined HAVE_LIBPFM @@ -28,105 +29,254 @@ namespace internal { constexpr size_t PerfCounterValues::kMaxCounters; #if defined HAVE_LIBPFM + +size_t PerfCounterValues::Read(const std::vector& leaders) { + // Create a pointer for multiple reads + const size_t bufsize = values_.size() * sizeof(values_[0]); + char* ptr = reinterpret_cast(values_.data()); + size_t size = bufsize; + for (int lead : leaders) { + auto read_bytes = ::read(lead, ptr, size); + if (read_bytes >= ssize_t(sizeof(uint64_t))) { + // Actual data bytes are all bytes minus initial padding + std::size_t data_bytes = read_bytes - sizeof(uint64_t); + // This should be very cheap since it's in hot cache + std::memmove(ptr, ptr + sizeof(uint64_t), data_bytes); + // Increment our counters + ptr += data_bytes; + size -= data_bytes; + } else { + int err = errno; + GetErrorLogInstance() << "Error reading lead " << lead << " errno:" << err + << " " << ::strerror(err) << "\n"; + return 0; + } + } + return (bufsize - size) / sizeof(uint64_t); +} + const bool PerfCounters::kSupported = true; -bool PerfCounters::Initialize() { return pfm_initialize() == PFM_SUCCESS; } +// Initializes libpfm only on the first call. Returns whether that single +// initialization was successful. +bool PerfCounters::Initialize() { + // Function-scope static gets initialized only once on first call. + static const bool success = []() { + return pfm_initialize() == PFM_SUCCESS; + }(); + return success; +} + +bool PerfCounters::IsCounterSupported(const std::string& name) { + Initialize(); + perf_event_attr_t attr; + std::memset(&attr, 0, sizeof(attr)); + pfm_perf_encode_arg_t arg; + std::memset(&arg, 0, sizeof(arg)); + arg.attr = &attr; + const int mode = PFM_PLM3; // user mode only + int ret = pfm_get_os_event_encoding(name.c_str(), mode, PFM_OS_PERF_EVENT_EXT, + &arg); + return (ret == PFM_SUCCESS); +} PerfCounters PerfCounters::Create( const std::vector& counter_names) { - if (counter_names.empty()) { - return NoCounters(); - } - if (counter_names.size() > PerfCounterValues::kMaxCounters) { - GetErrorLogInstance() - << counter_names.size() - << " counters were requested. The minimum is 1, the maximum is " - << PerfCounterValues::kMaxCounters << "\n"; - return NoCounters(); + if (!counter_names.empty()) { + Initialize(); } - std::vector counter_ids(counter_names.size()); - const int mode = PFM_PLM3; // user mode only + // Valid counters will populate these arrays but we start empty + std::vector valid_names; + std::vector counter_ids; + std::vector leader_ids; + + // Resize to the maximum possible + valid_names.reserve(counter_names.size()); + counter_ids.reserve(counter_names.size()); + + const int kCounterMode = PFM_PLM3; // user mode only + + // Group leads will be assigned on demand. The idea is that once we cannot + // create a counter descriptor, the reason is that this group has maxed out + // so we set the group_id again to -1 and retry - giving the algorithm a + // chance to create a new group leader to hold the next set of counters. + int group_id = -1; + + // Loop through all performance counters for (size_t i = 0; i < counter_names.size(); ++i) { - const bool is_first = i == 0; - struct perf_event_attr attr {}; - attr.size = sizeof(attr); - const int group_id = !is_first ? counter_ids[0] : -1; + // we are about to push into the valid names vector + // check if we did not reach the maximum + if (valid_names.size() == PerfCounterValues::kMaxCounters) { + // Log a message if we maxed out and stop adding + GetErrorLogInstance() + << counter_names.size() << " counters were requested. The maximum is " + << PerfCounterValues::kMaxCounters << " and " << valid_names.size() + << " were already added. All remaining counters will be ignored\n"; + // stop the loop and return what we have already + break; + } + + // Check if this name is empty const auto& name = counter_names[i]; if (name.empty()) { - GetErrorLogInstance() << "A counter name was the empty string\n"; - return NoCounters(); + GetErrorLogInstance() + << "A performance counter name was the empty string\n"; + continue; } + + // Here first means first in group, ie the group leader + const bool is_first = (group_id < 0); + + // This struct will be populated by libpfm from the counter string + // and then fed into the syscall perf_event_open + struct perf_event_attr attr {}; + attr.size = sizeof(attr); + + // This is the input struct to libpfm. pfm_perf_encode_arg_t arg{}; arg.attr = &attr; - - const int pfm_get = - pfm_get_os_event_encoding(name.c_str(), mode, PFM_OS_PERF_EVENT, &arg); + const int pfm_get = pfm_get_os_event_encoding(name.c_str(), kCounterMode, + PFM_OS_PERF_EVENT, &arg); if (pfm_get != PFM_SUCCESS) { - GetErrorLogInstance() << "Unknown counter name: " << name << "\n"; - return NoCounters(); + GetErrorLogInstance() + << "Unknown performance counter name: " << name << "\n"; + continue; } - attr.disabled = is_first; - // Note: the man page for perf_event_create suggests inerit = true and + + // We then proceed to populate the remaining fields in our attribute struct + // Note: the man page for perf_event_create suggests inherit = true and // read_format = PERF_FORMAT_GROUP don't work together, but that's not the // case. + attr.disabled = is_first; attr.inherit = true; attr.pinned = is_first; attr.exclude_kernel = true; attr.exclude_user = false; attr.exclude_hv = true; - // Read all counters in one read. + + // Read all counters in a group in one read. attr.read_format = PERF_FORMAT_GROUP; int id = -1; - static constexpr size_t kNrOfSyscallRetries = 5; - // Retry syscall as it was interrupted often (b/64774091). - for (size_t num_retries = 0; num_retries < kNrOfSyscallRetries; - ++num_retries) { - id = perf_event_open(&attr, 0, -1, group_id, 0); - if (id >= 0 || errno != EINTR) { - break; + while (id < 0) { + static constexpr size_t kNrOfSyscallRetries = 5; + // Retry syscall as it was interrupted often (b/64774091). + for (size_t num_retries = 0; num_retries < kNrOfSyscallRetries; + ++num_retries) { + id = perf_event_open(&attr, 0, -1, group_id, 0); + if (id >= 0 || errno != EINTR) { + break; + } + } + if (id < 0) { + // If the file descriptor is negative we might have reached a limit + // in the current group. Set the group_id to -1 and retry + if (group_id >= 0) { + // Create a new group + group_id = -1; + } else { + // At this point we have already retried to set a new group id and + // failed. We then give up. + break; + } } } + + // We failed to get a new file descriptor. We might have reached a hard + // hardware limit that cannot be resolved even with group multiplexing if (id < 0) { - GetErrorLogInstance() - << "Failed to get a file descriptor for " << name << "\n"; - return NoCounters(); - } + GetErrorLogInstance() << "***WARNING** Failed to get a file descriptor " + "for performance counter " + << name << ". Ignoring\n"; - counter_ids[i] = id; + // We give up on this counter but try to keep going + // as the others would be fine + continue; + } + if (group_id < 0) { + // This is a leader, store and assign it to the current file descriptor + leader_ids.push_back(id); + group_id = id; + } + // This is a valid counter, add it to our descriptor's list + counter_ids.push_back(id); + valid_names.push_back(name); } - if (ioctl(counter_ids[0], PERF_EVENT_IOC_ENABLE) != 0) { - GetErrorLogInstance() << "Failed to start counters\n"; - return NoCounters(); + + // Loop through all group leaders activating them + // There is another option of starting ALL counters in a process but + // that would be far reaching an intrusion. If the user is using PMCs + // by themselves then this would have a side effect on them. It is + // friendlier to loop through all groups individually. + for (int lead : leader_ids) { + if (ioctl(lead, PERF_EVENT_IOC_ENABLE) != 0) { + // This should never happen but if it does, we give up on the + // entire batch as recovery would be a mess. + GetErrorLogInstance() << "***WARNING*** Failed to start counters. " + "Claring out all counters.\n"; + + // Close all peformance counters + for (int id : counter_ids) { + ::close(id); + } + + // Return an empty object so our internal state is still good and + // the process can continue normally without impact + return NoCounters(); + } } - return PerfCounters(counter_names, std::move(counter_ids)); + return PerfCounters(std::move(valid_names), std::move(counter_ids), + std::move(leader_ids)); } -PerfCounters::~PerfCounters() { +void PerfCounters::CloseCounters() const { if (counter_ids_.empty()) { return; } - ioctl(counter_ids_[0], PERF_EVENT_IOC_DISABLE); + for (int lead : leader_ids_) { + ioctl(lead, PERF_EVENT_IOC_DISABLE); + } for (int fd : counter_ids_) { close(fd); } } #else // defined HAVE_LIBPFM +size_t PerfCounterValues::Read(const std::vector&) { return 0; } + const bool PerfCounters::kSupported = false; bool PerfCounters::Initialize() { return false; } +bool PerfCounters::IsCounterSupported(const std::string&) { return false; } + PerfCounters PerfCounters::Create( const std::vector& counter_names) { if (!counter_names.empty()) { - GetErrorLogInstance() << "Performance counters not supported."; + GetErrorLogInstance() << "Performance counters not supported.\n"; } return NoCounters(); } -PerfCounters::~PerfCounters() = default; +void PerfCounters::CloseCounters() const {} #endif // defined HAVE_LIBPFM + +PerfCountersMeasurement::PerfCountersMeasurement( + const std::vector& counter_names) + : start_values_(counter_names.size()), end_values_(counter_names.size()) { + counters_ = PerfCounters::Create(counter_names); +} + +PerfCounters& PerfCounters::operator=(PerfCounters&& other) noexcept { + if (this != &other) { + CloseCounters(); + + counter_ids_ = std::move(other.counter_ids_); + leader_ids_ = std::move(other.leader_ids_); + counter_names_ = std::move(other.counter_names_); + } + return *this; +} } // namespace internal } // namespace benchmark diff --git a/third-party/benchmark/src/perf_counters.h b/third-party/benchmark/src/perf_counters.h index 47ca1385e24d..bf5eb6bc3aec 100644 --- a/third-party/benchmark/src/perf_counters.h +++ b/third-party/benchmark/src/perf_counters.h @@ -17,16 +17,25 @@ #include #include +#include +#include #include #include "benchmark/benchmark.h" #include "check.h" #include "log.h" +#include "mutex.h" #ifndef BENCHMARK_OS_WINDOWS #include #endif +#if defined(_MSC_VER) +#pragma warning(push) +// C4251: needs to have dll-interface to be used by clients of class +#pragma warning(disable : 4251) +#endif + namespace benchmark { namespace internal { @@ -36,18 +45,21 @@ namespace internal { // The implementation ensures the storage is inlined, and allows 0-based // indexing into the counter values. // The object is used in conjunction with a PerfCounters object, by passing it -// to Snapshot(). The values are populated such that -// perfCounters->names()[i]'s value is obtained at position i (as given by -// operator[]) of this object. -class PerfCounterValues { +// to Snapshot(). The Read() method relocates individual reads, discarding +// the initial padding from each group leader in the values buffer such that +// all user accesses through the [] operator are correct. +class BENCHMARK_EXPORT PerfCounterValues { public: explicit PerfCounterValues(size_t nr_counters) : nr_counters_(nr_counters) { BM_CHECK_LE(nr_counters_, kMaxCounters); } - uint64_t operator[](size_t pos) const { return values_[kPadding + pos]; } + // We are reading correctly now so the values don't need to skip padding + uint64_t operator[](size_t pos) const { return values_[pos]; } - static constexpr size_t kMaxCounters = 3; + // Increased the maximum to 32 only since the buffer + // is std::array<> backed + static constexpr size_t kMaxCounters = 32; private: friend class PerfCounters; @@ -58,7 +70,14 @@ class PerfCounterValues { sizeof(uint64_t) * (kPadding + nr_counters_)}; } - static constexpr size_t kPadding = 1; + // This reading is complex and as the goal of this class is to + // abstract away the intrincacies of the reading process, this is + // a better place for it + size_t Read(const std::vector& leaders); + + // Move the padding to 2 due to the reading algorithm (1st padding plus a + // current read padding) + static constexpr size_t kPadding = 2; std::array values_; const size_t nr_counters_; }; @@ -66,27 +85,34 @@ class PerfCounterValues { // Collect PMU counters. The object, once constructed, is ready to be used by // calling read(). PMU counter collection is enabled from the time create() is // called, to obtain the object, until the object's destructor is called. -class PerfCounters final { +class BENCHMARK_EXPORT PerfCounters final { public: // True iff this platform supports performance counters. static const bool kSupported; - bool IsValid() const { return is_valid_; } + // Returns an empty object static PerfCounters NoCounters() { return PerfCounters(); } - ~PerfCounters(); + ~PerfCounters() { CloseCounters(); } + PerfCounters() = default; PerfCounters(PerfCounters&&) = default; PerfCounters(const PerfCounters&) = delete; + PerfCounters& operator=(PerfCounters&&) noexcept; + PerfCounters& operator=(const PerfCounters&) = delete; // Platform-specific implementations may choose to do some library // initialization here. static bool Initialize(); + // Check if the given counter is supported, if the app wants to + // check before passing + static bool IsCounterSupported(const std::string& name); + // Return a PerfCounters object ready to read the counters with the names // specified. The values are user-mode only. The counter name format is // implementation and OS specific. - // TODO: once we move to C++-17, this should be a std::optional, and then the - // IsValid() boolean can be dropped. + // In case of failure, this method will in the worst case return an + // empty object whose state will still be valid. static PerfCounters Create(const std::vector& counter_names); // Take a snapshot of the current value of the counters into the provided @@ -95,10 +121,7 @@ class PerfCounters final { BENCHMARK_ALWAYS_INLINE bool Snapshot(PerfCounterValues* values) const { #ifndef BENCHMARK_OS_WINDOWS assert(values != nullptr); - assert(IsValid()); - auto buffer = values->get_data_buffer(); - auto read_bytes = ::read(counter_ids_[0], buffer.first, buffer.second); - return static_cast(read_bytes) == buffer.second; + return values->Read(leader_ids_) == counter_ids_.size(); #else (void)values; return false; @@ -110,63 +133,68 @@ class PerfCounters final { private: PerfCounters(const std::vector& counter_names, - std::vector&& counter_ids) + std::vector&& counter_ids, std::vector&& leader_ids) : counter_ids_(std::move(counter_ids)), - counter_names_(counter_names), - is_valid_(true) {} - PerfCounters() : is_valid_(false) {} + leader_ids_(std::move(leader_ids)), + counter_names_(counter_names) {} + + void CloseCounters() const; std::vector counter_ids_; - const std::vector counter_names_; - const bool is_valid_; + std::vector leader_ids_; + std::vector counter_names_; }; // Typical usage of the above primitives. -class PerfCountersMeasurement final { +class BENCHMARK_EXPORT PerfCountersMeasurement final { public: - PerfCountersMeasurement(PerfCounters&& c) - : counters_(std::move(c)), - start_values_(counters_.IsValid() ? counters_.names().size() : 0), - end_values_(counters_.IsValid() ? counters_.names().size() : 0) {} + PerfCountersMeasurement(const std::vector& counter_names); + + size_t num_counters() const { return counters_.num_counters(); } - bool IsValid() const { return counters_.IsValid(); } + std::vector names() const { return counters_.names(); } - BENCHMARK_ALWAYS_INLINE void Start() { - assert(IsValid()); + BENCHMARK_ALWAYS_INLINE bool Start() { + if (num_counters() == 0) return true; // Tell the compiler to not move instructions above/below where we take // the snapshot. ClobberMemory(); - counters_.Snapshot(&start_values_); + valid_read_ &= counters_.Snapshot(&start_values_); ClobberMemory(); + + return valid_read_; } - BENCHMARK_ALWAYS_INLINE std::vector> - StopAndGetMeasurements() { - assert(IsValid()); + BENCHMARK_ALWAYS_INLINE bool Stop( + std::vector>& measurements) { + if (num_counters() == 0) return true; // Tell the compiler to not move instructions above/below where we take // the snapshot. ClobberMemory(); - counters_.Snapshot(&end_values_); + valid_read_ &= counters_.Snapshot(&end_values_); ClobberMemory(); - std::vector> ret; for (size_t i = 0; i < counters_.names().size(); ++i) { double measurement = static_cast(end_values_[i]) - static_cast(start_values_[i]); - ret.push_back({counters_.names()[i], measurement}); + measurements.push_back({counters_.names()[i], measurement}); } - return ret; + + return valid_read_; } private: PerfCounters counters_; + bool valid_read_ = true; PerfCounterValues start_values_; PerfCounterValues end_values_; }; -BENCHMARK_UNUSED static bool perf_init_anchor = PerfCounters::Initialize(); - } // namespace internal } // namespace benchmark +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + #endif // BENCHMARK_PERF_COUNTERS_H diff --git a/third-party/benchmark/src/re.h b/third-party/benchmark/src/re.h index 630046782dc4..9afb869bea27 100644 --- a/third-party/benchmark/src/re.h +++ b/third-party/benchmark/src/re.h @@ -33,7 +33,7 @@ // Prefer C regex libraries when compiling w/o exceptions so that we can // correctly report errors. #if defined(BENCHMARK_HAS_NO_EXCEPTIONS) && \ - defined(BENCHMARK_HAVE_STD_REGEX) && \ + defined(HAVE_STD_REGEX) && \ (defined(HAVE_GNU_POSIX_REGEX) || defined(HAVE_POSIX_REGEX)) #undef HAVE_STD_REGEX #endif diff --git a/third-party/benchmark/src/reporter.cc b/third-party/benchmark/src/reporter.cc index 1d2df17b90f0..076bc31a2ecc 100644 --- a/third-party/benchmark/src/reporter.cc +++ b/third-party/benchmark/src/reporter.cc @@ -25,9 +25,6 @@ #include "timers.h" namespace benchmark { -namespace internal { -extern std::map *global_context; -} BenchmarkReporter::BenchmarkReporter() : output_stream_(&std::cout), error_stream_(&std::cerr) {} @@ -39,7 +36,11 @@ void BenchmarkReporter::PrintBasicContext(std::ostream *out, BM_CHECK(out) << "cannot be null"; auto &Out = *out; +#ifndef BENCHMARK_OS_QURT + // Date/time information is not available on QuRT. + // Attempting to get it via this call cause the binary to crash. Out << LocalDateTimeString() << "\n"; +#endif if (context.executable_name) Out << "Running " << context.executable_name << "\n"; @@ -67,8 +68,11 @@ void BenchmarkReporter::PrintBasicContext(std::ostream *out, Out << "\n"; } - if (internal::global_context != nullptr) { - for (const auto &kv : *internal::global_context) { + std::map *global_context = + internal::GetGlobalContext(); + + if (global_context != nullptr) { + for (const auto &kv : *global_context) { Out << kv.first << ": " << kv.second << "\n"; } } diff --git a/third-party/benchmark/src/sleep.cc b/third-party/benchmark/src/sleep.cc deleted file mode 100644 index ab59000f24ad..000000000000 --- a/third-party/benchmark/src/sleep.cc +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2015 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "sleep.h" - -#include -#include -#include - -#include "internal_macros.h" - -#ifdef BENCHMARK_OS_WINDOWS -#include -#endif - -#ifdef BENCHMARK_OS_ZOS -#include -#endif - -namespace benchmark { -#ifdef BENCHMARK_OS_WINDOWS -// Window's Sleep takes milliseconds argument. -void SleepForMilliseconds(int milliseconds) { Sleep(milliseconds); } -void SleepForSeconds(double seconds) { - SleepForMilliseconds(static_cast(kNumMillisPerSecond * seconds)); -} -#else // BENCHMARK_OS_WINDOWS -void SleepForMicroseconds(int microseconds) { -#ifdef BENCHMARK_OS_ZOS - // z/OS does not support nanosleep. Instead call sleep() and then usleep() to - // sleep for the remaining microseconds because usleep() will fail if its - // argument is greater than 1000000. - div_t sleepTime = div(microseconds, kNumMicrosPerSecond); - int seconds = sleepTime.quot; - while (seconds != 0) seconds = sleep(seconds); - while (usleep(sleepTime.rem) == -1 && errno == EINTR) - ; -#else - struct timespec sleep_time; - sleep_time.tv_sec = microseconds / kNumMicrosPerSecond; - sleep_time.tv_nsec = (microseconds % kNumMicrosPerSecond) * kNumNanosPerMicro; - while (nanosleep(&sleep_time, &sleep_time) != 0 && errno == EINTR) - ; // Ignore signals and wait for the full interval to elapse. -#endif -} - -void SleepForMilliseconds(int milliseconds) { - SleepForMicroseconds(milliseconds * kNumMicrosPerMilli); -} - -void SleepForSeconds(double seconds) { - SleepForMicroseconds(static_cast(seconds * kNumMicrosPerSecond)); -} -#endif // BENCHMARK_OS_WINDOWS -} // end namespace benchmark diff --git a/third-party/benchmark/src/sleep.h b/third-party/benchmark/src/sleep.h deleted file mode 100644 index f98551afe284..000000000000 --- a/third-party/benchmark/src/sleep.h +++ /dev/null @@ -1,15 +0,0 @@ -#ifndef BENCHMARK_SLEEP_H_ -#define BENCHMARK_SLEEP_H_ - -namespace benchmark { -const int kNumMillisPerSecond = 1000; -const int kNumMicrosPerMilli = 1000; -const int kNumMicrosPerSecond = kNumMillisPerSecond * 1000; -const int kNumNanosPerMicro = 1000; -const int kNumNanosPerSecond = kNumNanosPerMicro * kNumMicrosPerSecond; - -void SleepForMilliseconds(int milliseconds); -void SleepForSeconds(double seconds); -} // end namespace benchmark - -#endif // BENCHMARK_SLEEP_H_ diff --git a/third-party/benchmark/src/statistics.cc b/third-party/benchmark/src/statistics.cc index 3e5ef0993971..261dcb299a67 100644 --- a/third-party/benchmark/src/statistics.cc +++ b/third-party/benchmark/src/statistics.cc @@ -32,7 +32,7 @@ auto StatisticsSum = [](const std::vector& v) { double StatisticsMean(const std::vector& v) { if (v.empty()) return 0.0; - return StatisticsSum(v) * (1.0 / v.size()); + return StatisticsSum(v) * (1.0 / static_cast(v.size())); } double StatisticsMedian(const std::vector& v) { @@ -42,13 +42,13 @@ double StatisticsMedian(const std::vector& v) { auto center = copy.begin() + v.size() / 2; std::nth_element(copy.begin(), center, copy.end()); - // did we have an odd number of samples? - // if yes, then center is the median - // it no, then we are looking for the average between center and the value - // before + // Did we have an odd number of samples? If yes, then center is the median. + // If not, then we are looking for the average between center and the value + // before. Instead of resorting, we just look for the max value before it, + // which is not necessarily the element immediately preceding `center` Since + // `copy` is only partially sorted by `nth_element`. if (v.size() % 2 == 1) return *center; - auto center2 = copy.begin() + v.size() / 2 - 1; - std::nth_element(copy.begin(), center2, copy.end()); + auto center2 = std::max_element(copy.begin(), center); return (*center + *center2) / 2.0; } @@ -71,8 +71,11 @@ double StatisticsStdDev(const std::vector& v) { // Sample standard deviation is undefined for n = 1 if (v.size() == 1) return 0.0; - const double avg_squares = SumSquares(v) * (1.0 / v.size()); - return Sqrt(v.size() / (v.size() - 1.0) * (avg_squares - Sqr(mean))); + const double avg_squares = + SumSquares(v) * (1.0 / static_cast(v.size())); + return Sqrt(static_cast(v.size()) / + (static_cast(v.size()) - 1.0) * + (avg_squares - Sqr(mean))); } double StatisticsCV(const std::vector& v) { @@ -81,6 +84,8 @@ double StatisticsCV(const std::vector& v) { const auto stddev = StatisticsStdDev(v); const auto mean = StatisticsMean(v); + if (std::fpclassify(mean) == FP_ZERO) return 0.0; + return stddev / mean; } @@ -89,9 +94,8 @@ std::vector ComputeStats( typedef BenchmarkReporter::Run Run; std::vector results; - auto error_count = - std::count_if(reports.begin(), reports.end(), - [](Run const& run) { return run.error_occurred; }); + auto error_count = std::count_if(reports.begin(), reports.end(), + [](Run const& run) { return run.skipped; }); if (reports.size() - error_count < 2) { // We don't report aggregated data if there was a single run. @@ -118,11 +122,13 @@ std::vector ComputeStats( for (auto const& cnt : r.counters) { auto it = counter_stats.find(cnt.first); if (it == counter_stats.end()) { - counter_stats.insert({cnt.first, {cnt.second, std::vector{}}}); - it = counter_stats.find(cnt.first); + it = counter_stats + .emplace(cnt.first, + CounterStat{cnt.second, std::vector{}}) + .first; it->second.s.reserve(reports.size()); } else { - BM_CHECK_EQ(counter_stats[cnt.first].c.flags, cnt.second.flags); + BM_CHECK_EQ(it->second.c.flags, cnt.second.flags); } } } @@ -131,7 +137,7 @@ std::vector ComputeStats( for (Run const& run : reports) { BM_CHECK_EQ(reports[0].benchmark_name(), run.benchmark_name()); BM_CHECK_EQ(run_iterations, run.iterations); - if (run.error_occurred) continue; + if (run.skipped) continue; real_accumulated_time_stat.emplace_back(run.real_accumulated_time); cpu_accumulated_time_stat.emplace_back(run.cpu_accumulated_time); // user counters diff --git a/third-party/benchmark/src/statistics.h b/third-party/benchmark/src/statistics.h index a9545a58c648..6e5560e8f19f 100644 --- a/third-party/benchmark/src/statistics.h +++ b/third-party/benchmark/src/statistics.h @@ -22,15 +22,21 @@ namespace benchmark { -// Return a vector containing the mean, median and standard devation information -// (and any user-specified info) for the specified list of reports. If 'reports' -// contains less than two non-errored runs an empty vector is returned +// Return a vector containing the mean, median and standard deviation +// information (and any user-specified info) for the specified list of reports. +// If 'reports' contains less than two non-errored runs an empty vector is +// returned +BENCHMARK_EXPORT std::vector ComputeStats( const std::vector& reports); +BENCHMARK_EXPORT double StatisticsMean(const std::vector& v); +BENCHMARK_EXPORT double StatisticsMedian(const std::vector& v); +BENCHMARK_EXPORT double StatisticsStdDev(const std::vector& v); +BENCHMARK_EXPORT double StatisticsCV(const std::vector& v); } // end namespace benchmark diff --git a/third-party/benchmark/src/string_util.cc b/third-party/benchmark/src/string_util.cc index 401fa13df7af..c69e40a8133c 100644 --- a/third-party/benchmark/src/string_util.cc +++ b/third-party/benchmark/src/string_util.cc @@ -11,16 +11,17 @@ #include #include "arraysize.h" +#include "benchmark/benchmark.h" namespace benchmark { namespace { - // kilo, Mega, Giga, Tera, Peta, Exa, Zetta, Yotta. -const char kBigSIUnits[] = "kMGTPEZY"; +const char* const kBigSIUnits[] = {"k", "M", "G", "T", "P", "E", "Z", "Y"}; // Kibi, Mebi, Gibi, Tebi, Pebi, Exbi, Zebi, Yobi. -const char kBigIECUnits[] = "KMGTPEZY"; +const char* const kBigIECUnits[] = {"Ki", "Mi", "Gi", "Ti", + "Pi", "Ei", "Zi", "Yi"}; // milli, micro, nano, pico, femto, atto, zepto, yocto. -const char kSmallSIUnits[] = "munpfazy"; +const char* const kSmallSIUnits[] = {"m", "u", "n", "p", "f", "a", "z", "y"}; // We require that all three arrays have the same size. static_assert(arraysize(kBigSIUnits) == arraysize(kBigIECUnits), @@ -30,9 +31,8 @@ static_assert(arraysize(kSmallSIUnits) == arraysize(kBigSIUnits), static const int64_t kUnitsSize = arraysize(kBigSIUnits); -void ToExponentAndMantissa(double val, double thresh, int precision, - double one_k, std::string* mantissa, - int64_t* exponent) { +void ToExponentAndMantissa(double val, int precision, double one_k, + std::string* mantissa, int64_t* exponent) { std::stringstream mantissa_stream; if (val < 0) { @@ -43,8 +43,8 @@ void ToExponentAndMantissa(double val, double thresh, int precision, // Adjust threshold so that it never excludes things which can't be rendered // in 'precision' digits. const double adjusted_threshold = - std::max(thresh, 1.0 / std::pow(10.0, precision)); - const double big_threshold = adjusted_threshold * one_k; + std::max(1.0, 1.0 / std::pow(10.0, precision)); + const double big_threshold = (adjusted_threshold * one_k) - 1; const double small_threshold = adjusted_threshold; // Values in ]simple_threshold,small_threshold[ will be printed as-is const double simple_threshold = 0.01; @@ -92,37 +92,20 @@ std::string ExponentToPrefix(int64_t exponent, bool iec) { const int64_t index = (exponent > 0 ? exponent - 1 : -exponent - 1); if (index >= kUnitsSize) return ""; - const char* array = + const char* const* array = (exponent > 0 ? (iec ? kBigIECUnits : kBigSIUnits) : kSmallSIUnits); - if (iec) - return array[index] + std::string("i"); - else - return std::string(1, array[index]); + + return std::string(array[index]); } -std::string ToBinaryStringFullySpecified(double value, double threshold, - int precision, double one_k = 1024.0) { +std::string ToBinaryStringFullySpecified(double value, int precision, + Counter::OneK one_k) { std::string mantissa; int64_t exponent; - ToExponentAndMantissa(value, threshold, precision, one_k, &mantissa, + ToExponentAndMantissa(value, precision, + one_k == Counter::kIs1024 ? 1024.0 : 1000.0, &mantissa, &exponent); - return mantissa + ExponentToPrefix(exponent, false); -} - -} // end namespace - -void AppendHumanReadable(int n, std::string* str) { - std::stringstream ss; - // Round down to the nearest SI prefix. - ss << ToBinaryStringFullySpecified(n, 1.0, 0); - *str += ss.str(); -} - -std::string HumanReadableNumber(double n, double one_k) { - // 1.1 means that figures up to 1.1k should be shown with the next unit down; - // this softens edge effects. - // 1 means that we should show one decimal place of precision. - return ToBinaryStringFullySpecified(n, 1.1, 1, one_k); + return mantissa + ExponentToPrefix(exponent, one_k == Counter::kIs1024); } std::string StrFormatImp(const char* msg, va_list args) { @@ -133,21 +116,21 @@ std::string StrFormatImp(const char* msg, va_list args) { // TODO(ericwf): use std::array for first attempt to avoid one memory // allocation guess what the size might be std::array local_buff; - std::size_t size = local_buff.size(); + // 2015-10-08: vsnprintf is used instead of snd::vsnprintf due to a limitation // in the android-ndk - auto ret = vsnprintf(local_buff.data(), size, msg, args_cp); + auto ret = vsnprintf(local_buff.data(), local_buff.size(), msg, args_cp); va_end(args_cp); // handle empty expansion if (ret == 0) return std::string{}; - if (static_cast(ret) < size) + if (static_cast(ret) < local_buff.size()) return std::string(local_buff.data()); // we did not provide a long enough buffer on our first attempt. // add 1 to size to account for null-byte in size cast to prevent overflow - size = static_cast(ret) + 1; + std::size_t size = static_cast(ret) + 1; auto buff_ptr = std::unique_ptr(new char[size]); // 2015-10-08: vsnprintf is used instead of snd::vsnprintf due to a limitation // in the android-ndk @@ -155,6 +138,12 @@ std::string StrFormatImp(const char* msg, va_list args) { return std::string(buff_ptr.get()); } +} // end namespace + +std::string HumanReadableNumber(double n, Counter::OneK one_k) { + return ToBinaryStringFullySpecified(n, 1, one_k); +} + std::string StrFormat(const char* format, ...) { va_list args; va_start(args, format); diff --git a/third-party/benchmark/src/string_util.h b/third-party/benchmark/src/string_util.h index ff3b7da47d6c..731aa2c04c3e 100644 --- a/third-party/benchmark/src/string_util.h +++ b/third-party/benchmark/src/string_util.h @@ -4,15 +4,19 @@ #include #include #include +#include +#include "benchmark/benchmark.h" +#include "benchmark/export.h" +#include "check.h" #include "internal_macros.h" namespace benchmark { -void AppendHumanReadable(int n, std::string* str); - -std::string HumanReadableNumber(double n, double one_k = 1024.0); +BENCHMARK_EXPORT +std::string HumanReadableNumber(double n, Counter::OneK one_k); +BENCHMARK_EXPORT #if defined(__MINGW32__) __attribute__((format(__MINGW_PRINTF_FORMAT, 1, 2))) #elif defined(__GNUC__) @@ -38,6 +42,7 @@ inline std::string StrCat(Args&&... args) { return ss.str(); } +BENCHMARK_EXPORT std::vector StrSplit(const std::string& str, char delim); // Disable lint checking for this block since it re-implements C functions. diff --git a/third-party/benchmark/src/sysinfo.cc b/third-party/benchmark/src/sysinfo.cc index 3a56e8cace48..46df973b069a 100644 --- a/third-party/benchmark/src/sysinfo.cc +++ b/third-party/benchmark/src/sysinfo.cc @@ -22,6 +22,10 @@ #include "internal_macros.h" #ifdef BENCHMARK_OS_WINDOWS +#if !defined(WINVER) || WINVER < 0x0600 +#undef WINVER +#define WINVER 0x0600 +#endif // WINVER handling #include #undef StrCat // Don't let StrCat in string_util.h be renamed to lstrcatA #include @@ -30,7 +34,7 @@ #include #else #include -#ifndef BENCHMARK_OS_FUCHSIA +#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) #include #endif #include @@ -45,10 +49,17 @@ #endif #if defined(BENCHMARK_OS_SOLARIS) #include +#include #endif #if defined(BENCHMARK_OS_QNX) #include #endif +#if defined(BENCHMARK_OS_QURT) +#include +#endif +#if defined(BENCHMARK_HAS_PTHREAD_AFFINITY) +#include +#endif #include #include @@ -65,15 +76,17 @@ #include #include #include +#include #include #include +#include "benchmark/benchmark.h" #include "check.h" #include "cycleclock.h" #include "internal_macros.h" #include "log.h" -#include "sleep.h" #include "string_util.h" +#include "timers.h" namespace benchmark { namespace { @@ -98,67 +111,59 @@ BENCHMARK_NORETURN void PrintErrorAndDie(Args&&... args) { /// `sysctl` with the result type it's to be interpreted as. struct ValueUnion { union DataT { - uint32_t uint32_value; - uint64_t uint64_value; + int32_t int32_value; + int64_t int64_value; // For correct aliasing of union members from bytes. char bytes[8]; }; using DataPtr = std::unique_ptr; // The size of the data union member + its trailing array size. - size_t Size; - DataPtr Buff; + std::size_t size; + DataPtr buff; public: - ValueUnion() : Size(0), Buff(nullptr, &std::free) {} + ValueUnion() : size(0), buff(nullptr, &std::free) {} - explicit ValueUnion(size_t BuffSize) - : Size(sizeof(DataT) + BuffSize), - Buff(::new (std::malloc(Size)) DataT(), &std::free) {} + explicit ValueUnion(std::size_t buff_size) + : size(sizeof(DataT) + buff_size), + buff(::new (std::malloc(size)) DataT(), &std::free) {} ValueUnion(ValueUnion&& other) = default; - explicit operator bool() const { return bool(Buff); } + explicit operator bool() const { return bool(buff); } - char* data() const { return Buff->bytes; } + char* data() const { return buff->bytes; } std::string GetAsString() const { return std::string(data()); } int64_t GetAsInteger() const { - if (Size == sizeof(Buff->uint32_value)) - return static_cast(Buff->uint32_value); - else if (Size == sizeof(Buff->uint64_value)) - return static_cast(Buff->uint64_value); - BENCHMARK_UNREACHABLE(); - } - - uint64_t GetAsUnsigned() const { - if (Size == sizeof(Buff->uint32_value)) - return Buff->uint32_value; - else if (Size == sizeof(Buff->uint64_value)) - return Buff->uint64_value; + if (size == sizeof(buff->int32_value)) + return buff->int32_value; + else if (size == sizeof(buff->int64_value)) + return buff->int64_value; BENCHMARK_UNREACHABLE(); } template std::array GetAsArray() { - const int ArrSize = sizeof(T) * N; - BM_CHECK_LE(ArrSize, Size); - std::array Arr; - std::memcpy(Arr.data(), data(), ArrSize); - return Arr; + const int arr_size = sizeof(T) * N; + BM_CHECK_LE(arr_size, size); + std::array arr; + std::memcpy(arr.data(), data(), arr_size); + return arr; } }; -ValueUnion GetSysctlImp(std::string const& Name) { +ValueUnion GetSysctlImp(std::string const& name) { #if defined BENCHMARK_OS_OPENBSD int mib[2]; mib[0] = CTL_HW; - if ((Name == "hw.ncpu") || (Name == "hw.cpuspeed")) { + if ((name == "hw.ncpu") || (name == "hw.cpuspeed")) { ValueUnion buff(sizeof(int)); - if (Name == "hw.ncpu") { + if (name == "hw.ncpu") { mib[1] = HW_NCPU; } else { mib[1] = HW_CPUSPEED; @@ -171,41 +176,41 @@ ValueUnion GetSysctlImp(std::string const& Name) { } return ValueUnion(); #else - size_t CurBuffSize = 0; - if (sysctlbyname(Name.c_str(), nullptr, &CurBuffSize, nullptr, 0) == -1) + std::size_t cur_buff_size = 0; + if (sysctlbyname(name.c_str(), nullptr, &cur_buff_size, nullptr, 0) == -1) return ValueUnion(); - ValueUnion buff(CurBuffSize); - if (sysctlbyname(Name.c_str(), buff.data(), &buff.Size, nullptr, 0) == 0) + ValueUnion buff(cur_buff_size); + if (sysctlbyname(name.c_str(), buff.data(), &buff.size, nullptr, 0) == 0) return buff; return ValueUnion(); #endif } BENCHMARK_MAYBE_UNUSED -bool GetSysctl(std::string const& Name, std::string* Out) { - Out->clear(); - auto Buff = GetSysctlImp(Name); - if (!Buff) return false; - Out->assign(Buff.data()); +bool GetSysctl(std::string const& name, std::string* out) { + out->clear(); + auto buff = GetSysctlImp(name); + if (!buff) return false; + out->assign(buff.data()); return true; } template ::value>::type> -bool GetSysctl(std::string const& Name, Tp* Out) { - *Out = 0; - auto Buff = GetSysctlImp(Name); - if (!Buff) return false; - *Out = static_cast(Buff.GetAsUnsigned()); +bool GetSysctl(std::string const& name, Tp* out) { + *out = 0; + auto buff = GetSysctlImp(name); + if (!buff) return false; + *out = static_cast(buff.GetAsInteger()); return true; } template -bool GetSysctl(std::string const& Name, std::array* Out) { - auto Buff = GetSysctlImp(Name); - if (!Buff) return false; - *Out = Buff.GetAsArray(); +bool GetSysctl(std::string const& name, std::array* out) { + auto buff = GetSysctlImp(name); + if (!buff) return false; + *out = buff.GetAsArray(); return true; } #endif @@ -241,21 +246,21 @@ CPUInfo::Scaling CpuScaling(int num_cpus) { #endif } -int CountSetBitsInCPUMap(std::string Val) { - auto CountBits = [](std::string Part) { +int CountSetBitsInCPUMap(std::string val) { + auto CountBits = [](std::string part) { using CPUMask = std::bitset; - Part = "0x" + Part; - CPUMask Mask(benchmark::stoul(Part, nullptr, 16)); - return static_cast(Mask.count()); + part = "0x" + part; + CPUMask mask(benchmark::stoul(part, nullptr, 16)); + return static_cast(mask.count()); }; - size_t Pos; + std::size_t pos; int total = 0; - while ((Pos = Val.find(',')) != std::string::npos) { - total += CountBits(Val.substr(0, Pos)); - Val = Val.substr(Pos + 1); + while ((pos = val.find(',')) != std::string::npos) { + total += CountBits(val.substr(0, pos)); + val = val.substr(pos + 1); } - if (!Val.empty()) { - total += CountBits(Val); + if (!val.empty()) { + total += CountBits(val); } return total; } @@ -264,16 +269,16 @@ BENCHMARK_MAYBE_UNUSED std::vector GetCacheSizesFromKVFS() { std::vector res; std::string dir = "/sys/devices/system/cpu/cpu0/cache/"; - int Idx = 0; + int idx = 0; while (true) { CPUInfo::CacheInfo info; - std::string FPath = StrCat(dir, "index", Idx++, "/"); - std::ifstream f(StrCat(FPath, "size").c_str()); + std::string fpath = StrCat(dir, "index", idx++, "/"); + std::ifstream f(StrCat(fpath, "size").c_str()); if (!f.is_open()) break; std::string suffix; f >> info.size; if (f.fail()) - PrintErrorAndDie("Failed while reading file '", FPath, "size'"); + PrintErrorAndDie("Failed while reading file '", fpath, "size'"); if (f.good()) { f >> suffix; if (f.bad()) @@ -284,13 +289,13 @@ std::vector GetCacheSizesFromKVFS() { else if (suffix == "K") info.size *= 1024; } - if (!ReadFromFile(StrCat(FPath, "type"), &info.type)) - PrintErrorAndDie("Failed to read from file ", FPath, "type"); - if (!ReadFromFile(StrCat(FPath, "level"), &info.level)) - PrintErrorAndDie("Failed to read from file ", FPath, "level"); + if (!ReadFromFile(StrCat(fpath, "type"), &info.type)) + PrintErrorAndDie("Failed to read from file ", fpath, "type"); + if (!ReadFromFile(StrCat(fpath, "level"), &info.level)) + PrintErrorAndDie("Failed to read from file ", fpath, "level"); std::string map_str; - if (!ReadFromFile(StrCat(FPath, "shared_cpu_map"), &map_str)) - PrintErrorAndDie("Failed to read from file ", FPath, "shared_cpu_map"); + if (!ReadFromFile(StrCat(fpath, "shared_cpu_map"), &map_str)) + PrintErrorAndDie("Failed to read from file ", fpath, "shared_cpu_map"); info.num_sharing = CountSetBitsInCPUMap(map_str); res.push_back(info); } @@ -301,26 +306,26 @@ std::vector GetCacheSizesFromKVFS() { #ifdef BENCHMARK_OS_MACOSX std::vector GetCacheSizesMacOSX() { std::vector res; - std::array CacheCounts{{0, 0, 0, 0}}; - GetSysctl("hw.cacheconfig", &CacheCounts); + std::array cache_counts{{0, 0, 0, 0}}; + GetSysctl("hw.cacheconfig", &cache_counts); struct { std::string name; std::string type; int level; - uint64_t num_sharing; - } Cases[] = {{"hw.l1dcachesize", "Data", 1, CacheCounts[1]}, - {"hw.l1icachesize", "Instruction", 1, CacheCounts[1]}, - {"hw.l2cachesize", "Unified", 2, CacheCounts[2]}, - {"hw.l3cachesize", "Unified", 3, CacheCounts[3]}}; - for (auto& C : Cases) { + int num_sharing; + } cases[] = {{"hw.l1dcachesize", "Data", 1, cache_counts[1]}, + {"hw.l1icachesize", "Instruction", 1, cache_counts[1]}, + {"hw.l2cachesize", "Unified", 2, cache_counts[2]}, + {"hw.l3cachesize", "Unified", 3, cache_counts[3]}}; + for (auto& c : cases) { int val; - if (!GetSysctl(C.name, &val)) continue; + if (!GetSysctl(c.name, &val)) continue; CPUInfo::CacheInfo info; - info.type = C.type; - info.level = C.level; + info.type = c.type; + info.level = c.level; info.size = val; - info.num_sharing = static_cast(C.num_sharing); + info.num_sharing = c.num_sharing; res.push_back(std::move(info)); } return res; @@ -334,7 +339,7 @@ std::vector GetCacheSizesWindows() { using UPtr = std::unique_ptr; GetLogicalProcessorInformation(nullptr, &buffer_size); - UPtr buff((PInfo*)malloc(buffer_size), &std::free); + UPtr buff(static_cast(std::malloc(buffer_size)), &std::free); if (!GetLogicalProcessorInformation(buff.get(), &buffer_size)) PrintErrorAndDie("Failed during call to GetLogicalProcessorInformation: ", GetLastError()); @@ -345,16 +350,16 @@ std::vector GetCacheSizesWindows() { for (; it != end; ++it) { if (it->Relationship != RelationCache) continue; using BitSet = std::bitset; - BitSet B(it->ProcessorMask); + BitSet b(it->ProcessorMask); // To prevent duplicates, only consider caches where CPU 0 is specified - if (!B.test(0)) continue; - CInfo* Cache = &it->Cache; + if (!b.test(0)) continue; + const CInfo& cache = it->Cache; CPUInfo::CacheInfo C; - C.num_sharing = static_cast(B.count()); - C.level = Cache->Level; - C.size = Cache->Size; + C.num_sharing = static_cast(b.count()); + C.level = cache.Level; + C.size = cache.Size; C.type = "Unknown"; - switch (Cache->Type) { + switch (cache.Type) { case CacheUnified: C.type = "Unified"; break; @@ -417,6 +422,8 @@ std::vector GetCacheSizes() { return GetCacheSizesWindows(); #elif defined(BENCHMARK_OS_QNX) return GetCacheSizesQNX(); +#elif defined(BENCHMARK_OS_QURT) + return std::vector(); #else return GetCacheSizesFromKVFS(); #endif @@ -425,23 +432,32 @@ std::vector GetCacheSizes() { std::string GetSystemName() { #if defined(BENCHMARK_OS_WINDOWS) std::string str; - const unsigned COUNT = MAX_COMPUTERNAME_LENGTH + 1; + static constexpr int COUNT = MAX_COMPUTERNAME_LENGTH + 1; TCHAR hostname[COUNT] = {'\0'}; DWORD DWCOUNT = COUNT; if (!GetComputerName(hostname, &DWCOUNT)) return std::string(""); #ifndef UNICODE str = std::string(hostname, DWCOUNT); #else - // Using wstring_convert, Is deprecated in C++17 - using convert_type = std::codecvt_utf8; - std::wstring_convert converter; - std::wstring wStr(hostname, DWCOUNT); - str = converter.to_bytes(wStr); + // `WideCharToMultiByte` returns `0` when conversion fails. + int len = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, hostname, + DWCOUNT, NULL, 0, NULL, NULL); + str.resize(len); + WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, hostname, DWCOUNT, &str[0], + str.size(), NULL, NULL); #endif return str; -#else // defined(BENCHMARK_OS_WINDOWS) +#elif defined(BENCHMARK_OS_QURT) + std::string str = "Hexagon DSP"; + qurt_arch_version_t arch_version_struct; + if (qurt_sysenv_get_arch_version(&arch_version_struct) == QURT_EOK) { + str += " v"; + str += std::to_string(arch_version_struct.arch_version); + } + return str; +#else #ifndef HOST_NAME_MAX -#ifdef BENCHMARK_HAS_SYSCTL // BSD/Mac Doesnt have HOST_NAME_MAX defined +#ifdef BENCHMARK_HAS_SYSCTL // BSD/Mac doesn't have HOST_NAME_MAX defined #define HOST_NAME_MAX 64 #elif defined(BENCHMARK_OS_NACL) #define HOST_NAME_MAX 64 @@ -449,6 +465,8 @@ std::string GetSystemName() { #define HOST_NAME_MAX 154 #elif defined(BENCHMARK_OS_RTEMS) #define HOST_NAME_MAX 256 +#elif defined(BENCHMARK_OS_SOLARIS) +#define HOST_NAME_MAX MAXHOSTNAMELEN #elif defined(BENCHMARK_OS_ZOS) #define HOST_NAME_MAX _POSIX_HOST_NAME_MAX #else @@ -463,12 +481,11 @@ std::string GetSystemName() { #endif // Catch-all POSIX block. } -int GetNumCPUs() { +int GetNumCPUsImpl() { #ifdef BENCHMARK_HAS_SYSCTL - int NumCPU = -1; - if (GetSysctl("hw.ncpu", &NumCPU)) return NumCPU; - fprintf(stderr, "Err: %s\n", strerror(errno)); - std::exit(EXIT_FAILURE); + int num_cpu = -1; + if (GetSysctl("hw.ncpu", &num_cpu)) return num_cpu; + PrintErrorAndDie("Err: ", strerror(errno)); #elif defined(BENCHMARK_OS_WINDOWS) SYSTEM_INFO sysinfo; // Use memset as opposed to = {} to avoid GCC missing initializer false @@ -480,64 +497,155 @@ int GetNumCPUs() { // group #elif defined(BENCHMARK_OS_SOLARIS) // Returns -1 in case of a failure. - int NumCPU = sysconf(_SC_NPROCESSORS_ONLN); - if (NumCPU < 0) { - fprintf(stderr, "sysconf(_SC_NPROCESSORS_ONLN) failed with error: %s\n", - strerror(errno)); + long num_cpu = sysconf(_SC_NPROCESSORS_ONLN); + if (num_cpu < 0) { + PrintErrorAndDie("sysconf(_SC_NPROCESSORS_ONLN) failed with error: ", + strerror(errno)); } - return NumCPU; + return (int)num_cpu; #elif defined(BENCHMARK_OS_QNX) return static_cast(_syspage_ptr->num_cpu); +#elif defined(BENCHMARK_OS_QURT) + qurt_sysenv_max_hthreads_t hardware_threads; + if (qurt_sysenv_get_max_hw_threads(&hardware_threads) != QURT_EOK) { + hardware_threads.max_hthreads = 1; + } + return hardware_threads.max_hthreads; #else - int NumCPUs = 0; - int MaxID = -1; + int num_cpus = 0; + int max_id = -1; std::ifstream f("/proc/cpuinfo"); if (!f.is_open()) { - std::cerr << "failed to open /proc/cpuinfo\n"; - return -1; + PrintErrorAndDie("Failed to open /proc/cpuinfo"); } +#if defined(__alpha__) + const std::string Key = "cpus detected"; +#else const std::string Key = "processor"; +#endif std::string ln; while (std::getline(f, ln)) { if (ln.empty()) continue; - size_t SplitIdx = ln.find(':'); + std::size_t split_idx = ln.find(':'); std::string value; #if defined(__s390__) // s390 has another format in /proc/cpuinfo // it needs to be parsed differently - if (SplitIdx != std::string::npos) - value = ln.substr(Key.size() + 1, SplitIdx - Key.size() - 1); + if (split_idx != std::string::npos) + value = ln.substr(Key.size() + 1, split_idx - Key.size() - 1); #else - if (SplitIdx != std::string::npos) value = ln.substr(SplitIdx + 1); + if (split_idx != std::string::npos) value = ln.substr(split_idx + 1); #endif if (ln.size() >= Key.size() && ln.compare(0, Key.size(), Key) == 0) { - NumCPUs++; + num_cpus++; if (!value.empty()) { - int CurID = benchmark::stoi(value); - MaxID = std::max(CurID, MaxID); + const int cur_id = benchmark::stoi(value); + max_id = std::max(cur_id, max_id); } } } if (f.bad()) { - std::cerr << "Failure reading /proc/cpuinfo\n"; - return -1; + PrintErrorAndDie("Failure reading /proc/cpuinfo"); } if (!f.eof()) { - std::cerr << "Failed to read to end of /proc/cpuinfo\n"; - return -1; + PrintErrorAndDie("Failed to read to end of /proc/cpuinfo"); } f.close(); - if ((MaxID + 1) != NumCPUs) { + if ((max_id + 1) != num_cpus) { fprintf(stderr, "CPU ID assignments in /proc/cpuinfo seem messed up." " This is usually caused by a bad BIOS.\n"); } - return NumCPUs; + return num_cpus; #endif BENCHMARK_UNREACHABLE(); } +int GetNumCPUs() { + const int num_cpus = GetNumCPUsImpl(); + if (num_cpus < 1) { + PrintErrorAndDie( + "Unable to extract number of CPUs. If your platform uses " + "/proc/cpuinfo, custom support may need to be added."); + } + return num_cpus; +} + +class ThreadAffinityGuard final { + public: + ThreadAffinityGuard() : reset_affinity(SetAffinity()) { + if (!reset_affinity) + std::cerr << "***WARNING*** Failed to set thread affinity. Estimated CPU " + "frequency may be incorrect." + << std::endl; + } + + ~ThreadAffinityGuard() { + if (!reset_affinity) return; + +#if defined(BENCHMARK_HAS_PTHREAD_AFFINITY) + int ret = pthread_setaffinity_np(self, sizeof(previous_affinity), + &previous_affinity); + if (ret == 0) return; +#elif defined(BENCHMARK_OS_WINDOWS_WIN32) + DWORD_PTR ret = SetThreadAffinityMask(self, previous_affinity); + if (ret != 0) return; +#endif // def BENCHMARK_HAS_PTHREAD_AFFINITY + PrintErrorAndDie("Failed to reset thread affinity"); + } + + ThreadAffinityGuard(ThreadAffinityGuard&&) = delete; + ThreadAffinityGuard(const ThreadAffinityGuard&) = delete; + ThreadAffinityGuard& operator=(ThreadAffinityGuard&&) = delete; + ThreadAffinityGuard& operator=(const ThreadAffinityGuard&) = delete; + + private: + bool SetAffinity() { +#if defined(BENCHMARK_HAS_PTHREAD_AFFINITY) + int ret; + self = pthread_self(); + ret = pthread_getaffinity_np(self, sizeof(previous_affinity), + &previous_affinity); + if (ret != 0) return false; + + cpu_set_t affinity; + memcpy(&affinity, &previous_affinity, sizeof(affinity)); + + bool is_first_cpu = true; + + for (int i = 0; i < CPU_SETSIZE; ++i) + if (CPU_ISSET(i, &affinity)) { + if (is_first_cpu) + is_first_cpu = false; + else + CPU_CLR(i, &affinity); + } + + if (is_first_cpu) return false; + + ret = pthread_setaffinity_np(self, sizeof(affinity), &affinity); + return ret == 0; +#elif defined(BENCHMARK_OS_WINDOWS_WIN32) + self = GetCurrentThread(); + DWORD_PTR mask = static_cast(1) << GetCurrentProcessorNumber(); + previous_affinity = SetThreadAffinityMask(self, mask); + return previous_affinity != 0; +#else + return false; +#endif // def BENCHMARK_HAS_PTHREAD_AFFINITY + } + +#if defined(BENCHMARK_HAS_PTHREAD_AFFINITY) + pthread_t self; + cpu_set_t previous_affinity; +#elif defined(BENCHMARK_OS_WINDOWS_WIN32) + HANDLE self; + DWORD_PTR previous_affinity; +#endif // def BENCHMARK_HAS_PTHREAD_AFFINITY + bool reset_affinity; +}; + double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { // Currently, scaling is only used on linux path here, // suppress diagnostics about it being unused on other paths. @@ -566,7 +674,7 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { &freq)) { // The value is in kHz (as the file name suggests). For example, on a // 2GHz warpstation, the file contains the value "2000000". - return freq * 1000.0; + return static_cast(freq) * 1000.0; } const double error_value = -1; @@ -578,7 +686,7 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { return error_value; } - auto startsWithKey = [](std::string const& Value, std::string const& Key) { + auto StartsWithKey = [](std::string const& Value, std::string const& Key) { if (Key.size() > Value.size()) return false; auto Cmp = [&](char X, char Y) { return std::tolower(X) == std::tolower(Y); @@ -589,18 +697,18 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { std::string ln; while (std::getline(f, ln)) { if (ln.empty()) continue; - size_t SplitIdx = ln.find(':'); + std::size_t split_idx = ln.find(':'); std::string value; - if (SplitIdx != std::string::npos) value = ln.substr(SplitIdx + 1); + if (split_idx != std::string::npos) value = ln.substr(split_idx + 1); // When parsing the "cpu MHz" and "bogomips" (fallback) entries, we only // accept positive values. Some environments (virtual machines) report zero, // which would cause infinite looping in WallTime_Init. - if (startsWithKey(ln, "cpu MHz")) { + if (StartsWithKey(ln, "cpu MHz")) { if (!value.empty()) { double cycles_per_second = benchmark::stod(value) * 1000000.0; if (cycles_per_second > 0) return cycles_per_second; } - } else if (startsWithKey(ln, "bogomips")) { + } else if (StartsWithKey(ln, "bogomips")) { if (!value.empty()) { bogo_clock = benchmark::stod(value) * 1000000.0; if (bogo_clock < 0.0) bogo_clock = error_value; @@ -622,7 +730,7 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { if (bogo_clock >= 0.0) return bogo_clock; #elif defined BENCHMARK_HAS_SYSCTL - constexpr auto* FreqStr = + constexpr auto* freqStr = #if defined(BENCHMARK_OS_FREEBSD) || defined(BENCHMARK_OS_NETBSD) "machdep.tsc_freq"; #elif defined BENCHMARK_OS_OPENBSD @@ -634,14 +742,17 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { #endif unsigned long long hz = 0; #if defined BENCHMARK_OS_OPENBSD - if (GetSysctl(FreqStr, &hz)) return hz * 1000000; + if (GetSysctl(freqStr, &hz)) return hz * 1000000; #else - if (GetSysctl(FreqStr, &hz)) return hz; + if (GetSysctl(freqStr, &hz)) return hz; #endif fprintf(stderr, "Unable to determine clock rate from sysctl: %s: %s\n", - FreqStr, strerror(errno)); + freqStr, strerror(errno)); + fprintf(stderr, + "This does not affect benchmark measurements, only the " + "metadata output.\n"); -#elif defined BENCHMARK_OS_WINDOWS +#elif defined BENCHMARK_OS_WINDOWS_WIN32 // In NT, read MHz from the registry. If we fail to do so or we're in win9x // then make a crude estimate. DWORD data, data_size = sizeof(data); @@ -650,15 +761,16 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { SHGetValueA(HKEY_LOCAL_MACHINE, "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", "~MHz", nullptr, &data, &data_size))) - return static_cast((int64_t)data * - (int64_t)(1000 * 1000)); // was mhz + return static_cast(static_cast(data) * + static_cast(1000 * 1000)); // was mhz #elif defined(BENCHMARK_OS_SOLARIS) kstat_ctl_t* kc = kstat_open(); if (!kc) { std::cerr << "failed to open /dev/kstat\n"; return -1; } - kstat_t* ksp = kstat_lookup(kc, (char*)"cpu_info", -1, (char*)"cpu_info0"); + kstat_t* ksp = kstat_lookup(kc, const_cast("cpu_info"), -1, + const_cast("cpu_info0")); if (!ksp) { std::cerr << "failed to lookup in /dev/kstat\n"; return -1; @@ -667,8 +779,8 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { std::cerr << "failed to read from /dev/kstat\n"; return -1; } - kstat_named_t* knp = - (kstat_named_t*)kstat_data_lookup(ksp, (char*)"current_clock_Hz"); + kstat_named_t* knp = (kstat_named_t*)kstat_data_lookup( + ksp, const_cast("current_clock_Hz")); if (!knp) { std::cerr << "failed to lookup data in /dev/kstat\n"; return -1; @@ -682,22 +794,55 @@ double GetCPUCyclesPerSecond(CPUInfo::Scaling scaling) { kstat_close(kc); return clock_hz; #elif defined(BENCHMARK_OS_QNX) - return static_cast((int64_t)(SYSPAGE_ENTRY(cpuinfo)->speed) * - (int64_t)(1000 * 1000)); + return static_cast( + static_cast(SYSPAGE_ENTRY(cpuinfo)->speed) * + static_cast(1000 * 1000)); +#elif defined(BENCHMARK_OS_QURT) + // QuRT doesn't provide any API to query Hexagon frequency. + return 1000000000; #endif // If we've fallen through, attempt to roughly estimate the CPU clock rate. - const int estimate_time_ms = 1000; + + // Make sure to use the same cycle counter when starting and stopping the + // cycle timer. We just pin the current thread to a cpu in the previous + // affinity set. + ThreadAffinityGuard affinity_guard; + + static constexpr double estimate_time_s = 1.0; + const double start_time = ChronoClockNow(); const auto start_ticks = cycleclock::Now(); - SleepForMilliseconds(estimate_time_ms); - return static_cast(cycleclock::Now() - start_ticks); + + // Impose load instead of calling sleep() to make sure the cycle counter + // works. + using PRNG = std::minstd_rand; + using Result = PRNG::result_type; + PRNG rng(static_cast(start_ticks)); + + Result state = 0; + + do { + static constexpr size_t batch_size = 10000; + rng.discard(batch_size); + state += rng(); + + } while (ChronoClockNow() - start_time < estimate_time_s); + + DoNotOptimize(state); + + const auto end_ticks = cycleclock::Now(); + const double end_time = ChronoClockNow(); + + return static_cast(end_ticks - start_ticks) / (end_time - start_time); + // Reset the affinity of current thread when the lifetime of affinity_guard + // ends. } std::vector GetLoadAvg() { #if (defined BENCHMARK_OS_FREEBSD || defined(BENCHMARK_OS_LINUX) || \ defined BENCHMARK_OS_MACOSX || defined BENCHMARK_OS_NETBSD || \ defined BENCHMARK_OS_OPENBSD || defined BENCHMARK_OS_DRAGONFLY) && \ - !defined(__ANDROID__) - constexpr int kMaxSamples = 3; + !(defined(__ANDROID__) && __ANDROID_API__ < 29) + static constexpr int kMaxSamples = 3; std::vector res(kMaxSamples, 0.0); const int nelem = getloadavg(res.data(), kMaxSamples); if (nelem < 1) { diff --git a/third-party/benchmark/src/thread_manager.h b/third-party/benchmark/src/thread_manager.h index 468028508940..819b3c44db66 100644 --- a/third-party/benchmark/src/thread_manager.h +++ b/third-party/benchmark/src/thread_manager.h @@ -43,8 +43,8 @@ class ThreadManager { double manual_time_used = 0; int64_t complexity_n = 0; std::string report_label_; - std::string error_message_; - bool has_error_ = false; + std::string skip_message_; + internal::Skipped skipped_ = internal::NotSkipped; UserCounters counters; }; GUARDED_BY(GetBenchmarkMutex()) Result results; diff --git a/third-party/benchmark/src/timers.cc b/third-party/benchmark/src/timers.cc index ed35c01f5540..667e7b2eef3c 100644 --- a/third-party/benchmark/src/timers.cc +++ b/third-party/benchmark/src/timers.cc @@ -23,7 +23,7 @@ #include #else #include -#ifndef BENCHMARK_OS_FUCHSIA +#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) #include #endif #include @@ -38,6 +38,9 @@ #include #include #endif +#if defined(BENCHMARK_OS_QURT) +#include +#endif #endif #ifdef BENCHMARK_OS_EMSCRIPTEN @@ -56,7 +59,6 @@ #include "check.h" #include "log.h" -#include "sleep.h" #include "string_util.h" namespace benchmark { @@ -65,6 +67,9 @@ namespace benchmark { #if defined(__GNUC__) #pragma GCC diagnostic ignored "-Wunused-function" #endif +#if defined(__NVCOMPILER) +#pragma diag_suppress declared_but_not_referenced +#endif namespace { #if defined(BENCHMARK_OS_WINDOWS) @@ -79,7 +84,7 @@ double MakeTime(FILETIME const& kernel_time, FILETIME const& user_time) { static_cast(user.QuadPart)) * 1e-7; } -#elif !defined(BENCHMARK_OS_FUCHSIA) +#elif !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) double MakeTime(struct rusage const& ru) { return (static_cast(ru.ru_utime.tv_sec) + static_cast(ru.ru_utime.tv_usec) * 1e-6 + @@ -97,7 +102,8 @@ double MakeTime(thread_basic_info_data_t const& info) { #endif #if defined(CLOCK_PROCESS_CPUTIME_ID) || defined(CLOCK_THREAD_CPUTIME_ID) double MakeTime(struct timespec const& ts) { - return ts.tv_sec + (static_cast(ts.tv_nsec) * 1e-9); + return static_cast(ts.tv_sec) + + (static_cast(ts.tv_nsec) * 1e-9); } #endif @@ -119,11 +125,15 @@ double ProcessCPUUsage() { &user_time)) return MakeTime(kernel_time, user_time); DiagnoseAndExit("GetProccessTimes() failed"); +#elif defined(BENCHMARK_OS_QURT) + return static_cast( + qurt_timer_timetick_to_us(qurt_timer_get_ticks())) * + 1.0e-6; #elif defined(BENCHMARK_OS_EMSCRIPTEN) // clock_gettime(CLOCK_PROCESS_CPUTIME_ID, ...) returns 0 on Emscripten. // Use Emscripten-specific API. Reported CPU time would be exactly the // same as total time, but this is ok because there aren't long-latency - // syncronous system calls in Emscripten. + // synchronous system calls in Emscripten. return emscripten_get_now() * 1e-3; #elif defined(CLOCK_PROCESS_CPUTIME_ID) && !defined(BENCHMARK_OS_MACOSX) // FIXME We want to use clock_gettime, but its not available in MacOS 10.11. @@ -149,6 +159,10 @@ double ThreadCPUUsage() { GetThreadTimes(this_thread, &creation_time, &exit_time, &kernel_time, &user_time); return MakeTime(kernel_time, user_time); +#elif defined(BENCHMARK_OS_QURT) + return static_cast( + qurt_timer_timetick_to_us(qurt_timer_get_ticks())) * + 1.0e-6; #elif defined(BENCHMARK_OS_MACOSX) // FIXME We want to use clock_gettime, but its not available in MacOS 10.11. // See https://github.com/google/benchmark/pull/292 diff --git a/third-party/benchmark/test/AssemblyTests.cmake b/third-party/benchmark/test/AssemblyTests.cmake index 3d078586f1de..c43c711faf87 100644 --- a/third-party/benchmark/test/AssemblyTests.cmake +++ b/third-party/benchmark/test/AssemblyTests.cmake @@ -1,3 +1,23 @@ +set(CLANG_SUPPORTED_VERSION "5.0.0") +set(GCC_SUPPORTED_VERSION "5.5.0") + +if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") + if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL ${CLANG_SUPPORTED_VERSION}) + message (WARNING + "Unsupported Clang version " ${CMAKE_CXX_COMPILER_VERSION} + ". Expected is " ${CLANG_SUPPORTED_VERSION} + ". Assembly tests may be broken.") + endif() +elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU") + if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL ${GCC_SUPPORTED_VERSION}) + message (WARNING + "Unsupported GCC version " ${CMAKE_CXX_COMPILER_VERSION} + ". Expected is " ${GCC_SUPPORTED_VERSION} + ". Assembly tests may be broken.") + endif() +else() + message (WARNING "Unsupported compiler. Assembly tests may be broken.") +endif() include(split_list) @@ -23,6 +43,7 @@ string(TOUPPER "${CMAKE_CXX_COMPILER_ID}" ASM_TEST_COMPILER) macro(add_filecheck_test name) cmake_parse_arguments(ARG "" "" "CHECK_PREFIXES" ${ARGV}) add_library(${name} OBJECT ${name}.cc) + target_link_libraries(${name} PRIVATE benchmark::benchmark) set_target_properties(${name} PROPERTIES COMPILE_FLAGS "-S ${ASM_TEST_FLAGS}") set(ASM_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/${name}.s") add_custom_target(copy_${name} ALL diff --git a/third-party/benchmark/test/CMakeLists.txt b/third-party/benchmark/test/CMakeLists.txt index 162af53f80f7..1de175f98d34 100644 --- a/third-party/benchmark/test/CMakeLists.txt +++ b/third-party/benchmark/test/CMakeLists.txt @@ -1,8 +1,12 @@ # Enable the tests +set(THREADS_PREFER_PTHREAD_FLAG ON) + find_package(Threads REQUIRED) include(CheckCXXCompilerFlag) +add_cxx_compiler_flag(-Wno-unused-variable) + # NOTE: Some tests use `` to perform the test. Therefore we must # strip -DNDEBUG from the default CMake flags in DEBUG mode. string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) @@ -22,6 +26,10 @@ if( NOT uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG" ) endforeach() endif() +if (NOT BUILD_SHARED_LIBS) + add_definitions(-DBENCHMARK_STATIC_DEFINE) +endif() + check_cxx_compiler_flag(-O3 BENCHMARK_HAS_O3_FLAG) set(BENCHMARK_O3_FLAG "") if (BENCHMARK_HAS_O3_FLAG) @@ -35,10 +43,14 @@ if (DEFINED BENCHMARK_CXX_LINKER_FLAGS) endif() add_library(output_test_helper STATIC output_test_helper.cc output_test.h) +target_link_libraries(output_test_helper PRIVATE benchmark::benchmark) macro(compile_benchmark_test name) add_executable(${name} "${name}.cc") target_link_libraries(${name} benchmark::benchmark ${CMAKE_THREAD_LIBS_INIT}) + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "NVHPC") + target_compile_options( ${name} PRIVATE --diag_suppress partial_override ) + endif() endmacro(compile_benchmark_test) macro(compile_benchmark_test_with_main name) @@ -48,26 +60,43 @@ endmacro(compile_benchmark_test_with_main) macro(compile_output_test name) add_executable(${name} "${name}.cc" output_test.h) - target_link_libraries(${name} output_test_helper benchmark::benchmark + target_link_libraries(${name} output_test_helper benchmark::benchmark_main ${BENCHMARK_CXX_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) endmacro(compile_output_test) +macro(benchmark_add_test) + add_test(${ARGV}) + if(WIN32 AND BUILD_SHARED_LIBS) + cmake_parse_arguments(TEST "" "NAME" "" ${ARGN}) + set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + endif() +endmacro(benchmark_add_test) + # Demonstration executable compile_benchmark_test(benchmark_test) -add_test(NAME benchmark COMMAND benchmark_test --benchmark_min_time=0.01) +benchmark_add_test(NAME benchmark COMMAND benchmark_test --benchmark_min_time=0.01s) compile_benchmark_test(spec_arg_test) -add_test(NAME spec_arg COMMAND spec_arg_test --benchmark_filter=BM_NotChosen) +benchmark_add_test(NAME spec_arg COMMAND spec_arg_test --benchmark_filter=BM_NotChosen) + +compile_benchmark_test(spec_arg_verbosity_test) +benchmark_add_test(NAME spec_arg_verbosity COMMAND spec_arg_verbosity_test --v=42) compile_benchmark_test(benchmark_setup_teardown_test) -add_test(NAME benchmark_setup_teardown COMMAND benchmark_setup_teardown_test) +benchmark_add_test(NAME benchmark_setup_teardown COMMAND benchmark_setup_teardown_test) compile_benchmark_test(filter_test) macro(add_filter_test name filter expect) - add_test(NAME ${name} COMMAND filter_test --benchmark_min_time=0.01 --benchmark_filter=${filter} ${expect}) - add_test(NAME ${name}_list_only COMMAND filter_test --benchmark_list_tests --benchmark_filter=${filter} ${expect}) + benchmark_add_test(NAME ${name} COMMAND filter_test --benchmark_min_time=0.01s --benchmark_filter=${filter} ${expect}) + benchmark_add_test(NAME ${name}_list_only COMMAND filter_test --benchmark_list_tests --benchmark_filter=${filter} ${expect}) endmacro(add_filter_test) +compile_benchmark_test(benchmark_min_time_flag_time_test) +benchmark_add_test(NAME min_time_flag_time COMMAND benchmark_min_time_flag_time_test) + +compile_benchmark_test(benchmark_min_time_flag_iters_test) +benchmark_add_test(NAME min_time_flag_iters COMMAND benchmark_min_time_flag_iters_test) + add_filter_test(filter_simple "Foo" 3) add_filter_test(filter_simple_negative "-Foo" 2) add_filter_test(filter_suffix "BM_.*" 4) @@ -88,78 +117,83 @@ add_filter_test(filter_regex_end ".*Ba$" 1) add_filter_test(filter_regex_end_negative "-.*Ba$" 4) compile_benchmark_test(options_test) -add_test(NAME options_benchmarks COMMAND options_test --benchmark_min_time=0.01) +benchmark_add_test(NAME options_benchmarks COMMAND options_test --benchmark_min_time=0.01s) compile_benchmark_test(basic_test) -add_test(NAME basic_benchmark COMMAND basic_test --benchmark_min_time=0.01) +benchmark_add_test(NAME basic_benchmark COMMAND basic_test --benchmark_min_time=0.01s) compile_output_test(repetitions_test) -add_test(NAME repetitions_benchmark COMMAND repetitions_test --benchmark_min_time=0.01 --benchmark_repetitions=3) +benchmark_add_test(NAME repetitions_benchmark COMMAND repetitions_test --benchmark_min_time=0.01s --benchmark_repetitions=3) compile_benchmark_test(diagnostics_test) -add_test(NAME diagnostics_test COMMAND diagnostics_test --benchmark_min_time=0.01) +benchmark_add_test(NAME diagnostics_test COMMAND diagnostics_test --benchmark_min_time=0.01s) compile_benchmark_test(skip_with_error_test) -add_test(NAME skip_with_error_test COMMAND skip_with_error_test --benchmark_min_time=0.01) +benchmark_add_test(NAME skip_with_error_test COMMAND skip_with_error_test --benchmark_min_time=0.01s) compile_benchmark_test(donotoptimize_test) +# Enable errors for deprecated deprecations (DoNotOptimize(Tp const& value)). +check_cxx_compiler_flag(-Werror=deprecated-declarations BENCHMARK_HAS_DEPRECATED_DECLARATIONS_FLAG) +if (BENCHMARK_HAS_DEPRECATED_DECLARATIONS_FLAG) + target_compile_options (donotoptimize_test PRIVATE "-Werror=deprecated-declarations") +endif() # Some of the issues with DoNotOptimize only occur when optimization is enabled check_cxx_compiler_flag(-O3 BENCHMARK_HAS_O3_FLAG) if (BENCHMARK_HAS_O3_FLAG) set_target_properties(donotoptimize_test PROPERTIES COMPILE_FLAGS "-O3") endif() -add_test(NAME donotoptimize_test COMMAND donotoptimize_test --benchmark_min_time=0.01) +benchmark_add_test(NAME donotoptimize_test COMMAND donotoptimize_test --benchmark_min_time=0.01s) compile_benchmark_test(fixture_test) -add_test(NAME fixture_test COMMAND fixture_test --benchmark_min_time=0.01) +benchmark_add_test(NAME fixture_test COMMAND fixture_test --benchmark_min_time=0.01s) compile_benchmark_test(register_benchmark_test) -add_test(NAME register_benchmark_test COMMAND register_benchmark_test --benchmark_min_time=0.01) +benchmark_add_test(NAME register_benchmark_test COMMAND register_benchmark_test --benchmark_min_time=0.01s) compile_benchmark_test(map_test) -add_test(NAME map_test COMMAND map_test --benchmark_min_time=0.01) +benchmark_add_test(NAME map_test COMMAND map_test --benchmark_min_time=0.01s) compile_benchmark_test(multiple_ranges_test) -add_test(NAME multiple_ranges_test COMMAND multiple_ranges_test --benchmark_min_time=0.01) +benchmark_add_test(NAME multiple_ranges_test COMMAND multiple_ranges_test --benchmark_min_time=0.01s) compile_benchmark_test(args_product_test) -add_test(NAME args_product_test COMMAND args_product_test --benchmark_min_time=0.01) +benchmark_add_test(NAME args_product_test COMMAND args_product_test --benchmark_min_time=0.01s) compile_benchmark_test_with_main(link_main_test) -add_test(NAME link_main_test COMMAND link_main_test --benchmark_min_time=0.01) +benchmark_add_test(NAME link_main_test COMMAND link_main_test --benchmark_min_time=0.01s) compile_output_test(reporter_output_test) -add_test(NAME reporter_output_test COMMAND reporter_output_test --benchmark_min_time=0.01) +benchmark_add_test(NAME reporter_output_test COMMAND reporter_output_test --benchmark_min_time=0.01s) compile_output_test(templated_fixture_test) -add_test(NAME templated_fixture_test COMMAND templated_fixture_test --benchmark_min_time=0.01) +benchmark_add_test(NAME templated_fixture_test COMMAND templated_fixture_test --benchmark_min_time=0.01s) compile_output_test(user_counters_test) -add_test(NAME user_counters_test COMMAND user_counters_test --benchmark_min_time=0.01) +benchmark_add_test(NAME user_counters_test COMMAND user_counters_test --benchmark_min_time=0.01s) compile_output_test(perf_counters_test) -add_test(NAME perf_counters_test COMMAND perf_counters_test --benchmark_min_time=0.01 --benchmark_perf_counters=CYCLES,BRANCHES) +benchmark_add_test(NAME perf_counters_test COMMAND perf_counters_test --benchmark_min_time=0.01s --benchmark_perf_counters=CYCLES,INSTRUCTIONS) compile_output_test(internal_threading_test) -add_test(NAME internal_threading_test COMMAND internal_threading_test --benchmark_min_time=0.01) +benchmark_add_test(NAME internal_threading_test COMMAND internal_threading_test --benchmark_min_time=0.01s) compile_output_test(report_aggregates_only_test) -add_test(NAME report_aggregates_only_test COMMAND report_aggregates_only_test --benchmark_min_time=0.01) +benchmark_add_test(NAME report_aggregates_only_test COMMAND report_aggregates_only_test --benchmark_min_time=0.01s) compile_output_test(display_aggregates_only_test) -add_test(NAME display_aggregates_only_test COMMAND display_aggregates_only_test --benchmark_min_time=0.01) +benchmark_add_test(NAME display_aggregates_only_test COMMAND display_aggregates_only_test --benchmark_min_time=0.01s) compile_output_test(user_counters_tabular_test) -add_test(NAME user_counters_tabular_test COMMAND user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01) +benchmark_add_test(NAME user_counters_tabular_test COMMAND user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01s) compile_output_test(user_counters_thousands_test) -add_test(NAME user_counters_thousands_test COMMAND user_counters_thousands_test --benchmark_min_time=0.01) +benchmark_add_test(NAME user_counters_thousands_test COMMAND user_counters_thousands_test --benchmark_min_time=0.01s) compile_output_test(memory_manager_test) -add_test(NAME memory_manager_test COMMAND memory_manager_test --benchmark_min_time=0.01) +benchmark_add_test(NAME memory_manager_test COMMAND memory_manager_test --benchmark_min_time=0.01s) -check_cxx_compiler_flag(-std=c++03 BENCHMARK_HAS_CXX03_FLAG) -if (BENCHMARK_HAS_CXX03_FLAG) +# MSVC does not allow to set the language standard to C++98/03. +if(NOT (MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC")) compile_benchmark_test(cxx03_test) set_target_properties(cxx03_test PROPERTIES @@ -170,22 +204,22 @@ if (BENCHMARK_HAS_CXX03_FLAG) # causing the test to fail to compile. To prevent this we explicitly disable # the warning. check_cxx_compiler_flag(-Wno-odr BENCHMARK_HAS_WNO_ODR) - if (BENCHMARK_ENABLE_LTO AND BENCHMARK_HAS_WNO_ODR) - set_target_properties(cxx03_test - PROPERTIES - LINK_FLAGS "-Wno-odr") + check_cxx_compiler_flag(-Wno-lto-type-mismatch BENCHMARK_HAS_WNO_LTO_TYPE_MISMATCH) + # Cannot set_target_properties multiple times here because the warnings will + # be overwritten on each call + set (DISABLE_LTO_WARNINGS "") + if (BENCHMARK_HAS_WNO_ODR) + set(DISABLE_LTO_WARNINGS "${DISABLE_LTO_WARNINGS} -Wno-odr") endif() - add_test(NAME cxx03 COMMAND cxx03_test --benchmark_min_time=0.01) + if (BENCHMARK_HAS_WNO_LTO_TYPE_MISMATCH) + set(DISABLE_LTO_WARNINGS "${DISABLE_LTO_WARNINGS} -Wno-lto-type-mismatch") + endif() + set_target_properties(cxx03_test PROPERTIES LINK_FLAGS "${DISABLE_LTO_WARNINGS}") + benchmark_add_test(NAME cxx03 COMMAND cxx03_test --benchmark_min_time=0.01s) endif() -# Attempt to work around flaky test failures when running on Appveyor servers. -if (DEFINED ENV{APPVEYOR}) - set(COMPLEXITY_MIN_TIME "0.5") -else() - set(COMPLEXITY_MIN_TIME "0.01") -endif() compile_output_test(complexity_test) -add_test(NAME complexity_benchmark COMMAND complexity_test --benchmark_min_time=${COMPLEXITY_MIN_TIME}) +benchmark_add_test(NAME complexity_benchmark COMMAND complexity_test --benchmark_min_time=1000000x) ############################################################################### # GoogleTest Unit Tests @@ -200,7 +234,12 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) macro(add_gtest name) compile_gtest(${name}) - add_test(NAME ${name} COMMAND ${name}) + benchmark_add_test(NAME ${name} COMMAND ${name}) + if(WIN32 AND BUILD_SHARED_LIBS) + set_tests_properties(${name} PROPERTIES + ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$" + ) + endif() endmacro() add_gtest(benchmark_gtest) @@ -210,6 +249,8 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) add_gtest(statistics_gtest) add_gtest(string_util_gtest) add_gtest(perf_counters_gtest) + add_gtest(time_unit_gtest) + add_gtest(min_time_parse_gtest) endif(BENCHMARK_ENABLE_GTEST_TESTS) ############################################################################### diff --git a/third-party/benchmark/test/args_product_test.cc b/third-party/benchmark/test/args_product_test.cc index d44f391f7480..63b8b71e45a0 100644 --- a/third-party/benchmark/test/args_product_test.cc +++ b/third-party/benchmark/test/args_product_test.cc @@ -23,7 +23,7 @@ class ArgsProductFixture : public ::benchmark::Fixture { {2, 15, 10, 9}, {4, 5, 6, 11}}) {} - void SetUp(const ::benchmark::State& state) BENCHMARK_OVERRIDE { + void SetUp(const ::benchmark::State& state) override { std::vector ranges = {state.range(0), state.range(1), state.range(2), state.range(3)}; @@ -34,7 +34,7 @@ class ArgsProductFixture : public ::benchmark::Fixture { // NOTE: This is not TearDown as we want to check after _all_ runs are // complete. - virtual ~ArgsProductFixture() { + ~ArgsProductFixture() override { if (actualValues != expectedValues) { std::cout << "EXPECTED\n"; for (const auto& v : expectedValues) { diff --git a/third-party/benchmark/test/basic_test.cc b/third-party/benchmark/test/basic_test.cc index 3a8fd42a8cdd..c25bec7ddd58 100644 --- a/third-party/benchmark/test/basic_test.cc +++ b/third-party/benchmark/test/basic_test.cc @@ -5,7 +5,8 @@ void BM_empty(benchmark::State& state) { for (auto _ : state) { - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } } BENCHMARK(BM_empty); @@ -147,7 +148,7 @@ void BM_OneTemplateFunc(benchmark::State& state) { auto arg = state.range(0); T sum = 0; for (auto _ : state) { - sum += arg; + sum += static_cast(arg); } } BENCHMARK(BM_OneTemplateFunc)->Arg(1); @@ -159,8 +160,8 @@ void BM_TwoTemplateFunc(benchmark::State& state) { A sum = 0; B prod = 1; for (auto _ : state) { - sum += arg; - prod *= arg; + sum += static_cast(arg); + prod *= static_cast(arg); } } BENCHMARK(BM_TwoTemplateFunc)->Arg(1); diff --git a/third-party/benchmark/test/benchmark_gtest.cc b/third-party/benchmark/test/benchmark_gtest.cc index 14a885ba46da..2c9e555d92dc 100644 --- a/third-party/benchmark/test/benchmark_gtest.cc +++ b/third-party/benchmark/test/benchmark_gtest.cc @@ -3,12 +3,12 @@ #include #include "../src/benchmark_register.h" +#include "benchmark/benchmark.h" #include "gmock/gmock.h" #include "gtest/gtest.h" namespace benchmark { namespace internal { -extern std::map* global_context; namespace { @@ -38,8 +38,9 @@ TEST(AddRangeTest, Advanced64) { TEST(AddRangeTest, FullRange8) { std::vector dst; - AddRange(&dst, int8_t{1}, std::numeric_limits::max(), 8); - EXPECT_THAT(dst, testing::ElementsAre(1, 8, 64, 127)); + AddRange(&dst, int8_t{1}, std::numeric_limits::max(), int8_t{8}); + EXPECT_THAT( + dst, testing::ElementsAre(int8_t{1}, int8_t{8}, int8_t{64}, int8_t{127})); } TEST(AddRangeTest, FullRange64) { @@ -129,11 +130,13 @@ TEST(AddRangeTest, FullNegativeRange64) { TEST(AddRangeTest, Simple8) { std::vector dst; - AddRange(&dst, 1, 8, 2); - EXPECT_THAT(dst, testing::ElementsAre(1, 2, 4, 8)); + AddRange(&dst, int8_t{1}, int8_t{8}, int8_t{2}); + EXPECT_THAT(dst, + testing::ElementsAre(int8_t{1}, int8_t{2}, int8_t{4}, int8_t{8})); } TEST(AddCustomContext, Simple) { + std::map *&global_context = GetGlobalContext(); EXPECT_THAT(global_context, nullptr); AddCustomContext("foo", "bar"); @@ -148,6 +151,7 @@ TEST(AddCustomContext, Simple) { } TEST(AddCustomContext, DuplicateKey) { + std::map *&global_context = GetGlobalContext(); EXPECT_THAT(global_context, nullptr); AddCustomContext("foo", "bar"); diff --git a/third-party/benchmark/test/benchmark_min_time_flag_iters_test.cc b/third-party/benchmark/test/benchmark_min_time_flag_iters_test.cc new file mode 100644 index 000000000000..3de93a75057b --- /dev/null +++ b/third-party/benchmark/test/benchmark_min_time_flag_iters_test.cc @@ -0,0 +1,66 @@ +#include +#include +#include +#include +#include +#include + +#include "benchmark/benchmark.h" + +// Tests that we can specify the number of iterations with +// --benchmark_min_time=x. +namespace { + +class TestReporter : public benchmark::ConsoleReporter { + public: + virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { + return ConsoleReporter::ReportContext(context); + }; + + virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { + assert(report.size() == 1); + iter_nums_.push_back(report[0].iterations); + ConsoleReporter::ReportRuns(report); + }; + + TestReporter() {} + + virtual ~TestReporter() {} + + const std::vector& GetIters() const { + return iter_nums_; + } + + private: + std::vector iter_nums_; +}; + +} // end namespace + +static void BM_MyBench(benchmark::State& state) { + for (auto s : state) { + } +} +BENCHMARK(BM_MyBench); + +int main(int argc, char** argv) { + // Make a fake argv and append the new --benchmark_min_time= to it. + int fake_argc = argc + 1; + const char** fake_argv = new const char*[static_cast(fake_argc)]; + for (int i = 0; i < argc; ++i) fake_argv[i] = argv[i]; + fake_argv[argc] = "--benchmark_min_time=4x"; + + benchmark::Initialize(&fake_argc, const_cast(fake_argv)); + + TestReporter test_reporter; + const size_t returned_count = + benchmark::RunSpecifiedBenchmarks(&test_reporter, "BM_MyBench"); + assert(returned_count == 1); + + // Check the executed iters. + const std::vector iters = test_reporter.GetIters(); + assert(!iters.empty() && iters[0] == 4); + + delete[] fake_argv; + return 0; +} diff --git a/third-party/benchmark/test/benchmark_min_time_flag_time_test.cc b/third-party/benchmark/test/benchmark_min_time_flag_time_test.cc new file mode 100644 index 000000000000..04a82eb95bf9 --- /dev/null +++ b/third-party/benchmark/test/benchmark_min_time_flag_time_test.cc @@ -0,0 +1,90 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "benchmark/benchmark.h" + +// Tests that we can specify the min time with +// --benchmark_min_time= (no suffix needed) OR +// --benchmark_min_time=s +namespace { + +// This is from benchmark.h +typedef int64_t IterationCount; + +class TestReporter : public benchmark::ConsoleReporter { + public: + virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { + return ConsoleReporter::ReportContext(context); + }; + + virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { + assert(report.size() == 1); + ConsoleReporter::ReportRuns(report); + }; + + virtual void ReportRunsConfig(double min_time, bool /* has_explicit_iters */, + IterationCount /* iters */) BENCHMARK_OVERRIDE { + min_times_.push_back(min_time); + } + + TestReporter() {} + + virtual ~TestReporter() {} + + const std::vector& GetMinTimes() const { return min_times_; } + + private: + std::vector min_times_; +}; + +bool AlmostEqual(double a, double b) { + return std::fabs(a - b) < std::numeric_limits::epsilon(); +} + +void DoTestHelper(int* argc, const char** argv, double expected) { + benchmark::Initialize(argc, const_cast(argv)); + + TestReporter test_reporter; + const size_t returned_count = + benchmark::RunSpecifiedBenchmarks(&test_reporter, "BM_MyBench"); + assert(returned_count == 1); + + // Check the min_time + const std::vector& min_times = test_reporter.GetMinTimes(); + assert(!min_times.empty() && AlmostEqual(min_times[0], expected)); +} + +} // end namespace + +static void BM_MyBench(benchmark::State& state) { + for (auto s : state) { + } +} +BENCHMARK(BM_MyBench); + +int main(int argc, char** argv) { + // Make a fake argv and append the new --benchmark_min_time= to it. + int fake_argc = argc + 1; + const char** fake_argv = new const char*[static_cast(fake_argc)]; + + for (int i = 0; i < argc; ++i) fake_argv[i] = argv[i]; + + const char* no_suffix = "--benchmark_min_time=4"; + const char* with_suffix = "--benchmark_min_time=4.0s"; + double expected = 4.0; + + fake_argv[argc] = no_suffix; + DoTestHelper(&fake_argc, fake_argv, expected); + + fake_argv[argc] = with_suffix; + DoTestHelper(&fake_argc, fake_argv, expected); + + delete[] fake_argv; + return 0; +} diff --git a/third-party/benchmark/test/benchmark_name_gtest.cc b/third-party/benchmark/test/benchmark_name_gtest.cc index afb401c1f532..0a6746d04df7 100644 --- a/third-party/benchmark/test/benchmark_name_gtest.cc +++ b/third-party/benchmark/test/benchmark_name_gtest.cc @@ -32,6 +32,14 @@ TEST(BenchmarkNameTest, MinTime) { EXPECT_EQ(name.str(), "function_name/some_args:3/4/min_time:3.4s"); } +TEST(BenchmarkNameTest, MinWarmUpTime) { + auto name = BenchmarkName(); + name.function_name = "function_name"; + name.args = "some_args:3/4"; + name.min_warmup_time = "min_warmup_time:3.5s"; + EXPECT_EQ(name.str(), "function_name/some_args:3/4/min_warmup_time:3.5s"); +} + TEST(BenchmarkNameTest, Iterations) { auto name = BenchmarkName(); name.function_name = "function_name"; diff --git a/third-party/benchmark/test/benchmark_random_interleaving_gtest.cc b/third-party/benchmark/test/benchmark_random_interleaving_gtest.cc index d04befa8e381..7f2086750d53 100644 --- a/third-party/benchmark/test/benchmark_random_interleaving_gtest.cc +++ b/third-party/benchmark/test/benchmark_random_interleaving_gtest.cc @@ -51,10 +51,9 @@ class BenchmarkTest : public testing::Test { void Execute(const std::string& pattern) { queue->Clear(); - BenchmarkReporter* reporter = new NullReporter; + std::unique_ptr reporter(new NullReporter()); FLAGS_benchmark_filter = pattern; - RunSpecifiedBenchmarks(reporter); - delete reporter; + RunSpecifiedBenchmarks(reporter.get()); queue->Put("DONE"); // End marker } diff --git a/third-party/benchmark/test/benchmark_setup_teardown_test.cc b/third-party/benchmark/test/benchmark_setup_teardown_test.cc index efa34e15c129..6c3cc2e58fbd 100644 --- a/third-party/benchmark/test/benchmark_setup_teardown_test.cc +++ b/third-party/benchmark/test/benchmark_setup_teardown_test.cc @@ -10,19 +10,19 @@ // Test that Setup() and Teardown() are called exactly once // for each benchmark run (single-threaded). -namespace single { +namespace singlethreaded { static int setup_call = 0; static int teardown_call = 0; -} // namespace single +} // namespace singlethreaded static void DoSetup1(const benchmark::State& state) { - ++single::setup_call; + ++singlethreaded::setup_call; // Setup/Teardown should never be called with any thread_idx != 0. assert(state.thread_index() == 0); } static void DoTeardown1(const benchmark::State& state) { - ++single::teardown_call; + ++singlethreaded::teardown_call; assert(state.thread_index() == 0); } @@ -80,11 +80,11 @@ int fixture_setup = 0; class FIXTURE_BECHMARK_NAME : public ::benchmark::Fixture { public: - void SetUp(const ::benchmark::State&) BENCHMARK_OVERRIDE { + void SetUp(const ::benchmark::State&) override { fixture_interaction::fixture_setup++; } - ~FIXTURE_BECHMARK_NAME() {} + ~FIXTURE_BECHMARK_NAME() override {} }; BENCHMARK_F(FIXTURE_BECHMARK_NAME, BM_WithFixture)(benchmark::State& st) { @@ -134,8 +134,8 @@ int main(int argc, char** argv) { assert(ret > 0); // Setup/Teardown is called once for each arg group (1,3,5,7). - assert(single::setup_call == 4); - assert(single::teardown_call == 4); + assert(singlethreaded::setup_call == 4); + assert(singlethreaded::teardown_call == 4); // 3 group of threads calling this function (3,5,10). assert(concurrent::setup_call.load(std::memory_order_relaxed) == 3); @@ -145,7 +145,7 @@ int main(int argc, char** argv) { // Setup is called 4 times, once for each arg group (1,3,5,7) assert(fixture_interaction::setup == 4); - // Fixture::Setup is called everytime the bm routine is run. + // Fixture::Setup is called every time the bm routine is run. // The exact number is indeterministic, so we just assert that // it's more than setup. assert(fixture_interaction::fixture_setup > fixture_interaction::setup); diff --git a/third-party/benchmark/test/benchmark_test.cc b/third-party/benchmark/test/benchmark_test.cc index 2906cdcde997..8b14017d03a5 100644 --- a/third-party/benchmark/test/benchmark_test.cc +++ b/third-party/benchmark/test/benchmark_test.cc @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -15,6 +16,7 @@ #include #include #include +#include #include #include @@ -26,7 +28,7 @@ namespace { -int BENCHMARK_NOINLINE Factorial(uint32_t n) { +int BENCHMARK_NOINLINE Factorial(int n) { return (n == 1) ? 1 : n * Factorial(n - 1); } @@ -74,7 +76,8 @@ BENCHMARK_RANGE(BM_CalculatePiRange, 1, 1024 * 1024); static void BM_CalculatePi(benchmark::State& state) { static const int depth = 1024; for (auto _ : state) { - benchmark::DoNotOptimize(CalculatePi(static_cast(depth))); + double pi = CalculatePi(static_cast(depth)); + benchmark::DoNotOptimize(pi); } } BENCHMARK(BM_CalculatePi)->Threads(8); @@ -90,7 +93,8 @@ static void BM_SetInsert(benchmark::State& state) { for (int j = 0; j < state.range(1); ++j) data.insert(rand()); } state.SetItemsProcessed(state.iterations() * state.range(1)); - state.SetBytesProcessed(state.iterations() * state.range(1) * sizeof(int)); + state.SetBytesProcessed(state.iterations() * state.range(1) * + static_cast(sizeof(int))); } // Test many inserts at once to reduce the total iterations needed. Otherwise, @@ -108,7 +112,7 @@ static void BM_Sequential(benchmark::State& state) { } const int64_t items_processed = state.iterations() * state.range(0); state.SetItemsProcessed(items_processed); - state.SetBytesProcessed(items_processed * sizeof(v)); + state.SetBytesProcessed(items_processed * static_cast(sizeof(v))); } BENCHMARK_TEMPLATE2(BM_Sequential, std::vector, int) ->Range(1 << 0, 1 << 10); @@ -122,7 +126,10 @@ static void BM_StringCompare(benchmark::State& state) { size_t len = static_cast(state.range(0)); std::string s1(len, '-'); std::string s2(len, '-'); - for (auto _ : state) benchmark::DoNotOptimize(s1.compare(s2)); + for (auto _ : state) { + auto comp = s1.compare(s2); + benchmark::DoNotOptimize(comp); + } } BENCHMARK(BM_StringCompare)->Range(1, 1 << 20); @@ -169,7 +176,7 @@ static void BM_ParallelMemset(benchmark::State& state) { for (int i = from; i < to; i++) { // No need to lock test_vector_mu as ranges // do not overlap between threads. - benchmark::DoNotOptimize(test_vector->at(i) = 1); + benchmark::DoNotOptimize(test_vector->at(static_cast(i)) = 1); } } @@ -220,6 +227,31 @@ void BM_non_template_args(benchmark::State& state, int, double) { } BENCHMARK_CAPTURE(BM_non_template_args, basic_test, 0, 0); +template +void BM_template2_capture(benchmark::State& state, ExtraArgs&&... extra_args) { + static_assert(std::is_same::value, ""); + static_assert(std::is_same::value, ""); + static_assert(std::is_same::value, ""); + unsigned int dummy[sizeof...(ExtraArgs)] = {extra_args...}; + assert(dummy[0] == 42); + for (auto _ : state) { + } +} +BENCHMARK_TEMPLATE2_CAPTURE(BM_template2_capture, void, char*, foo, 42U); +BENCHMARK_CAPTURE((BM_template2_capture), foo, 42U); + +template +void BM_template1_capture(benchmark::State& state, ExtraArgs&&... extra_args) { + static_assert(std::is_same::value, ""); + static_assert(std::is_same::value, ""); + unsigned long dummy[sizeof...(ExtraArgs)] = {extra_args...}; + assert(dummy[0] == 24); + for (auto _ : state) { + } +} +BENCHMARK_TEMPLATE1_CAPTURE(BM_template1_capture, void, foo, 24UL); +BENCHMARK_CAPTURE(BM_template1_capture, foo, 24UL); + #endif // BENCHMARK_HAS_CXX11 static void BM_DenseThreadRanges(benchmark::State& st) { @@ -244,4 +276,25 @@ BENCHMARK(BM_DenseThreadRanges)->Arg(1)->DenseThreadRange(1, 3); BENCHMARK(BM_DenseThreadRanges)->Arg(2)->DenseThreadRange(1, 4, 2); BENCHMARK(BM_DenseThreadRanges)->Arg(3)->DenseThreadRange(5, 14, 3); +static void BM_BenchmarkName(benchmark::State& state) { + for (auto _ : state) { + } + + // Check that the benchmark name is passed correctly to `state`. + assert("BM_BenchmarkName" == state.name()); +} +BENCHMARK(BM_BenchmarkName); + +// regression test for #1446 +template +static void BM_templated_test(benchmark::State& state) { + for (auto _ : state) { + type created_string; + benchmark::DoNotOptimize(created_string); + } +} + +static auto BM_templated_test_double = BM_templated_test>; +BENCHMARK(BM_templated_test_double); + BENCHMARK_MAIN(); diff --git a/third-party/benchmark/test/clobber_memory_assembly_test.cc b/third-party/benchmark/test/clobber_memory_assembly_test.cc index ab269130cd5c..54e26ccdadf4 100644 --- a/third-party/benchmark/test/clobber_memory_assembly_test.cc +++ b/third-party/benchmark/test/clobber_memory_assembly_test.cc @@ -3,6 +3,7 @@ #ifdef __clang__ #pragma clang diagnostic ignored "-Wreturn-type" #endif +BENCHMARK_DISABLE_DEPRECATED_WARNING extern "C" { diff --git a/third-party/benchmark/test/complexity_test.cc b/third-party/benchmark/test/complexity_test.cc index ea268b545988..fb4ad1ad53a9 100644 --- a/third-party/benchmark/test/complexity_test.cc +++ b/third-party/benchmark/test/complexity_test.cc @@ -26,7 +26,7 @@ int AddComplexityTest(const std::string &test_name, AddCases( TC_ConsoleOut, {{"^%bigo_name %bigo_str %bigo_str[ ]*$"}, - {"^%bigo_name", MR_Not}, // Assert we didn't only matched a name. + {"^%bigo_name", MR_Not}, // Assert we we didn't only matched a name. {"^%rms_name %rms %rms[ ]*$", MR_Next}}); AddCases( TC_JSONOut, @@ -69,35 +69,44 @@ int AddComplexityTest(const std::string &test_name, void BM_Complexity_O1(benchmark::State &state) { for (auto _ : state) { - for (int i = 0; i < 1024; ++i) { - benchmark::DoNotOptimize(&i); + // This test requires a non-zero CPU time to avoid divide-by-zero + benchmark::DoNotOptimize(state.iterations()); + long tmp = state.iterations(); + benchmark::DoNotOptimize(tmp); + for (benchmark::IterationCount i = 0; i < state.iterations(); ++i) { + benchmark::DoNotOptimize(state.iterations()); + tmp *= state.iterations(); + benchmark::DoNotOptimize(tmp); } + + // always 1ns per iteration + state.SetIterationTime(42 * 1e-9); } state.SetComplexityN(state.range(0)); } -BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1); -BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(); BENCHMARK(BM_Complexity_O1) ->Range(1, 1 << 18) + ->UseManualTime() + ->Complexity(benchmark::o1); +BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->UseManualTime()->Complexity(); +BENCHMARK(BM_Complexity_O1) + ->Range(1, 1 << 18) + ->UseManualTime() ->Complexity([](benchmark::IterationCount) { return 1.0; }); -const char *one_test_name = "BM_Complexity_O1"; -const char *big_o_1_test_name = "BM_Complexity_O1_BigO"; -const char *rms_o_1_test_name = "BM_Complexity_O1_RMS"; -const char *enum_big_o_1 = "\\([0-9]+\\)"; -// FIXME: Tolerate both '(1)' and 'lgN' as output when the complexity is auto -// deduced. -// See https://github.com/google/benchmark/issues/272 -const char *auto_big_o_1 = "(\\([0-9]+\\))|(lgN)"; +const char *one_test_name = "BM_Complexity_O1/manual_time"; +const char *big_o_1_test_name = "BM_Complexity_O1/manual_time_BigO"; +const char *rms_o_1_test_name = "BM_Complexity_O1/manual_time_RMS"; +const char *enum_auto_big_o_1 = "\\([0-9]+\\)"; const char *lambda_big_o_1 = "f\\(N\\)"; // Add enum tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - enum_big_o_1, /*family_index=*/0); + enum_auto_big_o_1, /*family_index=*/0); -// Add auto enum tests +// Add auto tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, - auto_big_o_1, /*family_index=*/1); + enum_auto_big_o_1, /*family_index=*/1); // Add lambda tests ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, @@ -107,42 +116,44 @@ ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, // --------------------------- Testing BigO O(N) --------------------------- // // ========================================================================= // -std::vector ConstructRandomVector(int64_t size) { - std::vector v; - v.reserve(static_cast(size)); - for (int i = 0; i < size; ++i) { - v.push_back(static_cast(std::rand() % size)); - } - return v; -} - void BM_Complexity_O_N(benchmark::State &state) { - auto v = ConstructRandomVector(state.range(0)); - // Test worst case scenario (item not in vector) - const int64_t item_not_in_vector = state.range(0) * 2; for (auto _ : state) { - benchmark::DoNotOptimize(std::find(v.begin(), v.end(), item_not_in_vector)); + // This test requires a non-zero CPU time to avoid divide-by-zero + benchmark::DoNotOptimize(state.iterations()); + long tmp = state.iterations(); + benchmark::DoNotOptimize(tmp); + for (benchmark::IterationCount i = 0; i < state.iterations(); ++i) { + benchmark::DoNotOptimize(state.iterations()); + tmp *= state.iterations(); + benchmark::DoNotOptimize(tmp); + } + + // 1ns per iteration per entry + state.SetIterationTime(static_cast(state.range(0)) * 42.0 * 1e-9); } state.SetComplexityN(state.range(0)); } BENCHMARK(BM_Complexity_O_N) ->RangeMultiplier(2) - ->Range(1 << 10, 1 << 16) + ->Range(1 << 10, 1 << 20) + ->UseManualTime() ->Complexity(benchmark::oN); BENCHMARK(BM_Complexity_O_N) ->RangeMultiplier(2) - ->Range(1 << 10, 1 << 16) + ->Range(1 << 10, 1 << 20) + ->UseManualTime() + ->Complexity(); +BENCHMARK(BM_Complexity_O_N) + ->RangeMultiplier(2) + ->Range(1 << 10, 1 << 20) + ->UseManualTime() ->Complexity([](benchmark::IterationCount n) -> double { return static_cast(n); }); -BENCHMARK(BM_Complexity_O_N) - ->RangeMultiplier(2) - ->Range(1 << 10, 1 << 16) - ->Complexity(); -const char *n_test_name = "BM_Complexity_O_N"; -const char *big_o_n_test_name = "BM_Complexity_O_N_BigO"; -const char *rms_o_n_test_name = "BM_Complexity_O_N_RMS"; +const char *n_test_name = "BM_Complexity_O_N/manual_time"; +const char *big_o_n_test_name = "BM_Complexity_O_N/manual_time_BigO"; +const char *rms_o_n_test_name = "BM_Complexity_O_N/manual_time_RMS"; const char *enum_auto_big_o_n = "N"; const char *lambda_big_o_n = "f\\(N\\)"; @@ -150,40 +161,57 @@ const char *lambda_big_o_n = "f\\(N\\)"; ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, enum_auto_big_o_n, /*family_index=*/3); +// Add auto tests +ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, + enum_auto_big_o_n, /*family_index=*/4); + // Add lambda tests ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, - lambda_big_o_n, /*family_index=*/4); + lambda_big_o_n, /*family_index=*/5); // ========================================================================= // -// ------------------------- Testing BigO O(N*lgN) ------------------------- // +// ------------------------- Testing BigO O(NlgN) ------------------------- // // ========================================================================= // +static const double kLog2E = 1.44269504088896340736; static void BM_Complexity_O_N_log_N(benchmark::State &state) { - auto v = ConstructRandomVector(state.range(0)); for (auto _ : state) { - std::sort(v.begin(), v.end()); + // This test requires a non-zero CPU time to avoid divide-by-zero + benchmark::DoNotOptimize(state.iterations()); + long tmp = state.iterations(); + benchmark::DoNotOptimize(tmp); + for (benchmark::IterationCount i = 0; i < state.iterations(); ++i) { + benchmark::DoNotOptimize(state.iterations()); + tmp *= state.iterations(); + benchmark::DoNotOptimize(tmp); + } + + state.SetIterationTime(static_cast(state.range(0)) * kLog2E * + std::log(state.range(0)) * 42.0 * 1e-9); } state.SetComplexityN(state.range(0)); } -static const double kLog2E = 1.44269504088896340736; BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2) - ->Range(1 << 10, 1 << 16) + ->Range(1 << 10, 1U << 24) + ->UseManualTime() ->Complexity(benchmark::oNLogN); BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2) - ->Range(1 << 10, 1 << 16) - ->Complexity([](benchmark::IterationCount n) { - return kLog2E * n * log(static_cast(n)); - }); + ->Range(1 << 10, 1U << 24) + ->UseManualTime() + ->Complexity(); BENCHMARK(BM_Complexity_O_N_log_N) ->RangeMultiplier(2) - ->Range(1 << 10, 1 << 16) - ->Complexity(); + ->Range(1 << 10, 1U << 24) + ->UseManualTime() + ->Complexity([](benchmark::IterationCount n) { + return kLog2E * static_cast(n) * std::log(static_cast(n)); + }); -const char *n_lg_n_test_name = "BM_Complexity_O_N_log_N"; -const char *big_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_BigO"; -const char *rms_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_RMS"; +const char *n_lg_n_test_name = "BM_Complexity_O_N_log_N/manual_time"; +const char *big_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N/manual_time_BigO"; +const char *rms_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N/manual_time_RMS"; const char *enum_auto_big_o_n_lg_n = "NlgN"; const char *lambda_big_o_n_lg_n = "f\\(N\\)"; @@ -192,11 +220,16 @@ ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n, /*family_index=*/6); -// Add lambda tests +// NOTE: auto big-o is wron.g ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, - rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n, + rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n, /*family_index=*/7); +//// Add lambda tests +ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, + rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n, + /*family_index=*/8); + // ========================================================================= // // -------- Testing formatting of Complexity with captured args ------------ // // ========================================================================= // @@ -205,19 +238,30 @@ void BM_ComplexityCaptureArgs(benchmark::State &state, int n) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero benchmark::DoNotOptimize(state.iterations()); + long tmp = state.iterations(); + benchmark::DoNotOptimize(tmp); + for (benchmark::IterationCount i = 0; i < state.iterations(); ++i) { + benchmark::DoNotOptimize(state.iterations()); + tmp *= state.iterations(); + benchmark::DoNotOptimize(tmp); + } + + state.SetIterationTime(static_cast(state.range(0)) * 42.0 * 1e-9); } state.SetComplexityN(n); } BENCHMARK_CAPTURE(BM_ComplexityCaptureArgs, capture_test, 100) + ->UseManualTime() ->Complexity(benchmark::oN) ->Ranges({{1, 2}, {3, 4}}); const std::string complexity_capture_name = - "BM_ComplexityCaptureArgs/capture_test"; + "BM_ComplexityCaptureArgs/capture_test/manual_time"; ADD_COMPLEXITY_CASES(complexity_capture_name, complexity_capture_name + "_BigO", - complexity_capture_name + "_RMS", "N", /*family_index=*/9); + complexity_capture_name + "_RMS", "N", + /*family_index=*/9); // ========================================================================= // // --------------------------- TEST CASES END ------------------------------ // diff --git a/third-party/benchmark/test/diagnostics_test.cc b/third-party/benchmark/test/diagnostics_test.cc index c54d5b0d708a..7c68a98929d8 100644 --- a/third-party/benchmark/test/diagnostics_test.cc +++ b/third-party/benchmark/test/diagnostics_test.cc @@ -49,7 +49,8 @@ void BM_diagnostic_test(benchmark::State& state) { if (called_once == false) try_invalid_pause_resume(state); for (auto _ : state) { - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } if (called_once == false) try_invalid_pause_resume(state); @@ -64,7 +65,8 @@ void BM_diagnostic_test_keep_running(benchmark::State& state) { if (called_once == false) try_invalid_pause_resume(state); while (state.KeepRunning()) { - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } if (called_once == false) try_invalid_pause_resume(state); @@ -74,7 +76,16 @@ void BM_diagnostic_test_keep_running(benchmark::State& state) { BENCHMARK(BM_diagnostic_test_keep_running); int main(int argc, char* argv[]) { +#ifdef NDEBUG + // This test is exercising functionality for debug builds, which are not + // available in release builds. Skip the test if we are in that environment + // to avoid a test failure. + std::cout << "Diagnostic test disabled in release build" << std::endl; + (void)argc; + (void)argv; +#else benchmark::internal::GetAbortHandler() = &TestHandler; benchmark::Initialize(&argc, argv); benchmark::RunSpecifiedBenchmarks(); +#endif } diff --git a/third-party/benchmark/test/donotoptimize_assembly_test.cc b/third-party/benchmark/test/donotoptimize_assembly_test.cc index 2e86a51e2234..dc286f53e20f 100644 --- a/third-party/benchmark/test/donotoptimize_assembly_test.cc +++ b/third-party/benchmark/test/donotoptimize_assembly_test.cc @@ -3,12 +3,16 @@ #ifdef __clang__ #pragma clang diagnostic ignored "-Wreturn-type" #endif +BENCHMARK_DISABLE_DEPRECATED_WARNING extern "C" { extern int ExternInt; extern int ExternInt2; extern int ExternInt3; +extern int BigArray[2049]; + +const int ConstBigArray[2049]{}; inline int Add42(int x) { return x + 42; } @@ -23,7 +27,15 @@ struct Large { int value; int data[2]; }; + +struct ExtraLarge { + int arr[2049]; +}; } + +extern ExtraLarge ExtraLargeObj; +const ExtraLarge ConstExtraLargeObj{}; + // CHECK-LABEL: test_with_rvalue: extern "C" void test_with_rvalue() { benchmark::DoNotOptimize(Add42(0)); @@ -68,6 +80,22 @@ extern "C" void test_with_large_lvalue() { // CHECK: ret } +// CHECK-LABEL: test_with_extra_large_lvalue_with_op: +extern "C" void test_with_extra_large_lvalue_with_op() { + ExtraLargeObj.arr[16] = 42; + benchmark::DoNotOptimize(ExtraLargeObj); + // CHECK: movl $42, ExtraLargeObj+64(%rip) + // CHECK: ret +} + +// CHECK-LABEL: test_with_big_array_with_op +extern "C" void test_with_big_array_with_op() { + BigArray[16] = 42; + benchmark::DoNotOptimize(BigArray); + // CHECK: movl $42, BigArray+64(%rip) + // CHECK: ret +} + // CHECK-LABEL: test_with_non_trivial_lvalue: extern "C" void test_with_non_trivial_lvalue() { NotTriviallyCopyable NTC(ExternInt); @@ -96,6 +124,18 @@ extern "C" void test_with_large_const_lvalue() { // CHECK: ret } +// CHECK-LABEL: test_with_const_extra_large_obj: +extern "C" void test_with_const_extra_large_obj() { + benchmark::DoNotOptimize(ConstExtraLargeObj); + // CHECK: ret +} + +// CHECK-LABEL: test_with_const_big_array +extern "C" void test_with_const_big_array() { + benchmark::DoNotOptimize(ConstBigArray); + // CHECK: ret +} + // CHECK-LABEL: test_with_non_trivial_const_lvalue: extern "C" void test_with_non_trivial_const_lvalue() { const NotTriviallyCopyable Obj(ExternInt); diff --git a/third-party/benchmark/test/donotoptimize_test.cc b/third-party/benchmark/test/donotoptimize_test.cc index c321f156a1e0..04ec9386a3b4 100644 --- a/third-party/benchmark/test/donotoptimize_test.cc +++ b/third-party/benchmark/test/donotoptimize_test.cc @@ -4,9 +4,9 @@ namespace { #if defined(__GNUC__) -std::uint64_t double_up(const std::uint64_t x) __attribute__((const)); +std::int64_t double_up(const std::int64_t x) __attribute__((const)); #endif -std::uint64_t double_up(const std::uint64_t x) { return x * 2; } +std::int64_t double_up(const std::int64_t x) { return x * 2; } } // namespace // Using DoNotOptimize on types like BitRef seem to cause a lot of problems @@ -29,6 +29,15 @@ struct BitRef { int main(int, char*[]) { // this test verifies compilation of DoNotOptimize() for some types + char buffer1[1] = ""; + benchmark::DoNotOptimize(buffer1); + + char buffer2[2] = ""; + benchmark::DoNotOptimize(buffer2); + + char buffer3[3] = ""; + benchmark::DoNotOptimize(buffer3); + char buffer8[8] = ""; benchmark::DoNotOptimize(buffer8); @@ -37,17 +46,24 @@ int main(int, char*[]) { char buffer1024[1024] = ""; benchmark::DoNotOptimize(buffer1024); - benchmark::DoNotOptimize(&buffer1024[0]); + char* bptr = &buffer1024[0]; + benchmark::DoNotOptimize(bptr); int x = 123; benchmark::DoNotOptimize(x); - benchmark::DoNotOptimize(&x); + int* xp = &x; + benchmark::DoNotOptimize(xp); benchmark::DoNotOptimize(x += 42); - benchmark::DoNotOptimize(double_up(x)); + std::int64_t y = double_up(x); + benchmark::DoNotOptimize(y); // These tests are to e - benchmark::DoNotOptimize(BitRef::Make()); BitRef lval = BitRef::Make(); benchmark::DoNotOptimize(lval); + +#ifdef BENCHMARK_HAS_CXX11 + // Check that accept rvalue. + benchmark::DoNotOptimize(BitRef::Make()); +#endif } diff --git a/third-party/benchmark/test/filter_test.cc b/third-party/benchmark/test/filter_test.cc index a567de2dd58e..4c8b8ea488ad 100644 --- a/third-party/benchmark/test/filter_test.cc +++ b/third-party/benchmark/test/filter_test.cc @@ -14,28 +14,27 @@ namespace { class TestReporter : public benchmark::ConsoleReporter { public: - virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { + bool ReportContext(const Context& context) override { return ConsoleReporter::ReportContext(context); }; - virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { + void ReportRuns(const std::vector& report) override { ++count_; - max_family_index_ = - std::max(max_family_index_, report[0].family_index); + max_family_index_ = std::max(max_family_index_, report[0].family_index); ConsoleReporter::ReportRuns(report); }; TestReporter() : count_(0), max_family_index_(0) {} - virtual ~TestReporter() {} + ~TestReporter() override {} - size_t GetCount() const { return count_; } + int GetCount() const { return count_; } - size_t GetMaxFamilyIndex() const { return max_family_index_; } + int64_t GetMaxFamilyIndex() const { return max_family_index_; } private: - mutable size_t count_; - mutable size_t max_family_index_; + mutable int count_; + mutable int64_t max_family_index_; }; } // end namespace @@ -79,13 +78,13 @@ int main(int argc, char** argv) { benchmark::Initialize(&argc, argv); TestReporter test_reporter; - const size_t returned_count = - benchmark::RunSpecifiedBenchmarks(&test_reporter); + const int64_t returned_count = + static_cast(benchmark::RunSpecifiedBenchmarks(&test_reporter)); if (argc == 2) { // Make sure we ran all of the tests std::stringstream ss(argv[1]); - size_t expected_return; + int64_t expected_return; ss >> expected_return; if (returned_count != expected_return) { @@ -95,8 +94,8 @@ int main(int argc, char** argv) { return -1; } - const size_t expected_reports = list_only ? 0 : expected_return; - const size_t reports_count = test_reporter.GetCount(); + const int64_t expected_reports = list_only ? 0 : expected_return; + const int64_t reports_count = test_reporter.GetCount(); if (reports_count != expected_reports) { std::cerr << "ERROR: Expected " << expected_reports << " tests to be run but reported_count = " << reports_count @@ -104,8 +103,8 @@ int main(int argc, char** argv) { return -1; } - const size_t max_family_index = test_reporter.GetMaxFamilyIndex(); - const size_t num_families = reports_count == 0 ? 0 : 1 + max_family_index; + const int64_t max_family_index = test_reporter.GetMaxFamilyIndex(); + const int64_t num_families = reports_count == 0 ? 0 : 1 + max_family_index; if (num_families != expected_reports) { std::cerr << "ERROR: Expected " << expected_reports << " test families to be run but num_families = " diff --git a/third-party/benchmark/test/fixture_test.cc b/third-party/benchmark/test/fixture_test.cc index af650dbd0661..d1093ebf52fc 100644 --- a/third-party/benchmark/test/fixture_test.cc +++ b/third-party/benchmark/test/fixture_test.cc @@ -8,21 +8,21 @@ class FIXTURE_BECHMARK_NAME : public ::benchmark::Fixture { public: - void SetUp(const ::benchmark::State& state) BENCHMARK_OVERRIDE { + void SetUp(const ::benchmark::State& state) override { if (state.thread_index() == 0) { assert(data.get() == nullptr); data.reset(new int(42)); } } - void TearDown(const ::benchmark::State& state) BENCHMARK_OVERRIDE { + void TearDown(const ::benchmark::State& state) override { if (state.thread_index() == 0) { assert(data.get() != nullptr); data.reset(); } } - ~FIXTURE_BECHMARK_NAME() { assert(data == nullptr); } + ~FIXTURE_BECHMARK_NAME() override { assert(data == nullptr); } std::unique_ptr data; }; diff --git a/third-party/benchmark/test/link_main_test.cc b/third-party/benchmark/test/link_main_test.cc index 241ad5c3905e..131937eebca9 100644 --- a/third-party/benchmark/test/link_main_test.cc +++ b/third-party/benchmark/test/link_main_test.cc @@ -2,7 +2,8 @@ void BM_empty(benchmark::State& state) { for (auto _ : state) { - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } } BENCHMARK(BM_empty); diff --git a/third-party/benchmark/test/map_test.cc b/third-party/benchmark/test/map_test.cc index 509613457c1b..0fdba7c87c4f 100644 --- a/third-party/benchmark/test/map_test.cc +++ b/third-party/benchmark/test/map_test.cc @@ -24,7 +24,8 @@ static void BM_MapLookup(benchmark::State& state) { m = ConstructRandomMap(size); state.ResumeTiming(); for (int i = 0; i < size; ++i) { - benchmark::DoNotOptimize(m.find(std::rand() % size)); + auto it = m.find(std::rand() % size); + benchmark::DoNotOptimize(it); } } state.SetItemsProcessed(state.iterations() * size); @@ -34,11 +35,11 @@ BENCHMARK(BM_MapLookup)->Range(1 << 3, 1 << 12); // Using fixtures. class MapFixture : public ::benchmark::Fixture { public: - void SetUp(const ::benchmark::State& st) BENCHMARK_OVERRIDE { + void SetUp(const ::benchmark::State& st) override { m = ConstructRandomMap(static_cast(st.range(0))); } - void TearDown(const ::benchmark::State&) BENCHMARK_OVERRIDE { m.clear(); } + void TearDown(const ::benchmark::State&) override { m.clear(); } std::map m; }; @@ -47,7 +48,8 @@ BENCHMARK_DEFINE_F(MapFixture, Lookup)(benchmark::State& state) { const int size = static_cast(state.range(0)); for (auto _ : state) { for (int i = 0; i < size; ++i) { - benchmark::DoNotOptimize(m.find(std::rand() % size)); + auto it = m.find(std::rand() % size); + benchmark::DoNotOptimize(it); } } state.SetItemsProcessed(state.iterations() * size); diff --git a/third-party/benchmark/test/memory_manager_test.cc b/third-party/benchmark/test/memory_manager_test.cc index f0c192fcbd00..4df674d586ed 100644 --- a/third-party/benchmark/test/memory_manager_test.cc +++ b/third-party/benchmark/test/memory_manager_test.cc @@ -5,16 +5,17 @@ #include "output_test.h" class TestMemoryManager : public benchmark::MemoryManager { - void Start() BENCHMARK_OVERRIDE {} - void Stop(Result* result) BENCHMARK_OVERRIDE { - result->num_allocs = 42; - result->max_bytes_used = 42000; + void Start() override {} + void Stop(Result& result) override { + result.num_allocs = 42; + result.max_bytes_used = 42000; } }; void BM_empty(benchmark::State& state) { for (auto _ : state) { - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } } BENCHMARK(BM_empty); diff --git a/third-party/benchmark/test/min_time_parse_gtest.cc b/third-party/benchmark/test/min_time_parse_gtest.cc new file mode 100644 index 000000000000..e2bdf67850ab --- /dev/null +++ b/third-party/benchmark/test/min_time_parse_gtest.cc @@ -0,0 +1,30 @@ +#include "../src/benchmark_runner.h" +#include "gtest/gtest.h" + +namespace { + +TEST(ParseMinTimeTest, InvalidInput) { +#if GTEST_HAS_DEATH_TEST + // Tests only runnable in debug mode (when BM_CHECK is enabled). +#ifndef NDEBUG +#ifndef TEST_BENCHMARK_LIBRARY_HAS_NO_ASSERTIONS + ASSERT_DEATH_IF_SUPPORTED( + { benchmark::internal::ParseBenchMinTime("abc"); }, + "Malformed seconds value passed to --benchmark_min_time: `abc`"); + + ASSERT_DEATH_IF_SUPPORTED( + { benchmark::internal::ParseBenchMinTime("123ms"); }, + "Malformed seconds value passed to --benchmark_min_time: `123ms`"); + + ASSERT_DEATH_IF_SUPPORTED( + { benchmark::internal::ParseBenchMinTime("1z"); }, + "Malformed seconds value passed to --benchmark_min_time: `1z`"); + + ASSERT_DEATH_IF_SUPPORTED( + { benchmark::internal::ParseBenchMinTime("1hs"); }, + "Malformed seconds value passed to --benchmark_min_time: `1hs`"); +#endif +#endif +#endif +} +} // namespace diff --git a/third-party/benchmark/test/multiple_ranges_test.cc b/third-party/benchmark/test/multiple_ranges_test.cc index 7618c4da0892..5300a96036c1 100644 --- a/third-party/benchmark/test/multiple_ranges_test.cc +++ b/third-party/benchmark/test/multiple_ranges_test.cc @@ -28,7 +28,7 @@ class MultipleRangesFixture : public ::benchmark::Fixture { {2, 7, 15}, {7, 6, 3}}) {} - void SetUp(const ::benchmark::State& state) BENCHMARK_OVERRIDE { + void SetUp(const ::benchmark::State& state) override { std::vector ranges = {state.range(0), state.range(1), state.range(2)}; @@ -39,7 +39,7 @@ class MultipleRangesFixture : public ::benchmark::Fixture { // NOTE: This is not TearDown as we want to check after _all_ runs are // complete. - virtual ~MultipleRangesFixture() { + ~MultipleRangesFixture() override { if (actualValues != expectedValues) { std::cout << "EXPECTED\n"; for (const auto& v : expectedValues) { diff --git a/third-party/benchmark/test/options_test.cc b/third-party/benchmark/test/options_test.cc index d424d40b9518..a1b209f3eb33 100644 --- a/third-party/benchmark/test/options_test.cc +++ b/third-party/benchmark/test/options_test.cc @@ -33,6 +33,8 @@ BENCHMARK(BM_basic)->DenseRange(10, 15); BENCHMARK(BM_basic)->Args({42, 42}); BENCHMARK(BM_basic)->Ranges({{64, 512}, {64, 512}}); BENCHMARK(BM_basic)->MinTime(0.7); +BENCHMARK(BM_basic)->MinWarmUpTime(0.8); +BENCHMARK(BM_basic)->MinTime(0.1)->MinWarmUpTime(0.2); BENCHMARK(BM_basic)->UseRealTime(); BENCHMARK(BM_basic)->ThreadRange(2, 4); BENCHMARK(BM_basic)->ThreadPerCpu(); @@ -65,8 +67,8 @@ void BM_explicit_iteration_count(benchmark::State& state) { // Test that the requested iteration count is respected. assert(state.max_iterations == 42); - size_t actual_iterations = 0; - for (auto _ : state) ++actual_iterations; + for (auto _ : state) { + } assert(state.iterations() == state.max_iterations); assert(state.iterations() == 42); } diff --git a/third-party/benchmark/test/output_test.h b/third-party/benchmark/test/output_test.h index c6ff8ef2d303..c08fe1d87e6c 100644 --- a/third-party/benchmark/test/output_test.h +++ b/third-party/benchmark/test/output_test.h @@ -85,7 +85,7 @@ std::string GetFileReporterOutput(int argc, char* argv[]); struct Results; typedef std::function ResultsCheckFn; -size_t AddChecker(const char* bm_name_pattern, const ResultsCheckFn& fn); +size_t AddChecker(const std::string& bm_name_pattern, const ResultsCheckFn& fn); // Class holding the results of a benchmark. // It is passed in calls to checker functions. @@ -117,7 +117,7 @@ struct Results { // get the string for a result by name, or nullptr if the name // is not found - const std::string* Get(const char* entry_name) const { + const std::string* Get(const std::string& entry_name) const { auto it = values.find(entry_name); if (it == values.end()) return nullptr; return &it->second; @@ -126,12 +126,12 @@ struct Results { // get a result by name, parsed as a specific type. // NOTE: for counters, use GetCounterAs instead. template - T GetAs(const char* entry_name) const; + T GetAs(const std::string& entry_name) const; // counters are written as doubles, so they have to be read first // as a double, and only then converted to the asked type. template - T GetCounterAs(const char* entry_name) const { + T GetCounterAs(const std::string& entry_name) const { double dval = GetAs(entry_name); T tval = static_cast(dval); return tval; @@ -139,7 +139,7 @@ struct Results { }; template -T Results::GetAs(const char* entry_name) const { +T Results::GetAs(const std::string& entry_name) const { auto* sv = Get(entry_name); BM_CHECK(sv != nullptr && !sv->empty()); std::stringstream ss; diff --git a/third-party/benchmark/test/output_test_helper.cc b/third-party/benchmark/test/output_test_helper.cc index 81584cbf778b..265f28aae7c7 100644 --- a/third-party/benchmark/test/output_test_helper.cc +++ b/third-party/benchmark/test/output_test_helper.cc @@ -45,7 +45,7 @@ SubMap& GetSubstitutions() { static SubMap map = { {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"}, // human-readable float - {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"}, + {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kKMGTPEZYmunpfazy]?i?"}, {"%percentage", percentage_re}, {"%int", "[ ]*[0-9]+"}, {" %s ", "[ ]+"}, @@ -65,6 +65,7 @@ SubMap& GetSubstitutions() { {"%csv_us_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",us,,,,,"}, {"%csv_ms_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ms,,,,,"}, {"%csv_s_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",s,,,,,"}, + {"%csv_cv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",,,,,,"}, {"%csv_bytes_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + ",,,,"}, {"%csv_items_report", @@ -143,7 +144,7 @@ class TestReporter : public benchmark::BenchmarkReporter { TestReporter(std::vector reps) : reporters_(std::move(reps)) {} - virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { + bool ReportContext(const Context& context) override { bool last_ret = false; bool first = true; for (auto rep : reporters_) { @@ -157,10 +158,10 @@ class TestReporter : public benchmark::BenchmarkReporter { return last_ret; } - void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { + void ReportRuns(const std::vector& report) override { for (auto rep : reporters_) rep->ReportRuns(report); } - void Finalize() BENCHMARK_OVERRIDE { + void Finalize() override { for (auto rep : reporters_) rep->Finalize(); } @@ -248,9 +249,8 @@ void ResultsChecker::CheckResults(std::stringstream& output) { if (!p.regex->Match(r.name)) { BM_VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n"; continue; - } else { - BM_VLOG(2) << p.regex_str << " is matched by " << r.name << "\n"; } + BM_VLOG(2) << p.regex_str << " is matched by " << r.name << "\n"; BM_VLOG(1) << "Checking results of " << r.name << ": ... \n"; p.fn(r); BM_VLOG(1) << "Checking results of " << r.name << ": OK.\n"; @@ -300,7 +300,7 @@ std::vector ResultsChecker::SplitCsv_(const std::string& line) { } // end namespace internal -size_t AddChecker(const char* bm_name, const ResultsCheckFn& fn) { +size_t AddChecker(const std::string& bm_name, const ResultsCheckFn& fn) { auto& rc = internal::GetResultsChecker(); rc.Add(bm_name, fn); return rc.results.size(); @@ -328,16 +328,18 @@ double Results::GetTime(BenchmarkTime which) const { BM_CHECK(unit); if (*unit == "ns") { return val * 1.e-9; - } else if (*unit == "us") { + } + if (*unit == "us") { return val * 1.e-6; - } else if (*unit == "ms") { + } + if (*unit == "ms") { return val * 1.e-3; - } else if (*unit == "s") { + } + if (*unit == "s") { return val; - } else { - BM_CHECK(1 == 0) << "unknown time unit: " << *unit; - return 0; } + BM_CHECK(1 == 0) << "unknown time unit: " << *unit; + return 0; } // ========================================================================= // @@ -393,14 +395,14 @@ void RunOutputTests(int argc, char* argv[]) { benchmark::JSONReporter JR; benchmark::CSVReporter CSVR; struct ReporterTest { - const char* name; + std::string name; std::vector& output_cases; std::vector& error_cases; benchmark::BenchmarkReporter& reporter; std::stringstream out_stream; std::stringstream err_stream; - ReporterTest(const char* n, std::vector& out_tc, + ReporterTest(const std::string& n, std::vector& out_tc, std::vector& err_tc, benchmark::BenchmarkReporter& br) : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) { @@ -408,12 +410,12 @@ void RunOutputTests(int argc, char* argv[]) { reporter.SetErrorStream(&err_stream); } } TestCases[] = { - {"ConsoleReporter", GetTestCaseList(TC_ConsoleOut), + {std::string("ConsoleReporter"), GetTestCaseList(TC_ConsoleOut), GetTestCaseList(TC_ConsoleErr), CR}, - {"JSONReporter", GetTestCaseList(TC_JSONOut), GetTestCaseList(TC_JSONErr), - JR}, - {"CSVReporter", GetTestCaseList(TC_CSVOut), GetTestCaseList(TC_CSVErr), - CSVR}, + {std::string("JSONReporter"), GetTestCaseList(TC_JSONOut), + GetTestCaseList(TC_JSONErr), JR}, + {std::string("CSVReporter"), GetTestCaseList(TC_CSVOut), + GetTestCaseList(TC_CSVErr), CSVR}, }; // Create the test reporter and run the benchmarks. @@ -422,7 +424,8 @@ void RunOutputTests(int argc, char* argv[]) { benchmark::RunSpecifiedBenchmarks(&test_rep); for (auto& rep_test : TestCases) { - std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n"; + std::string msg = + std::string("\nTesting ") + rep_test.name + std::string(" Output\n"); std::string banner(msg.size() - 1, '-'); std::cout << banner << msg << banner << "\n"; @@ -439,7 +442,7 @@ void RunOutputTests(int argc, char* argv[]) { // the checks to subscribees. auto& csv = TestCases[2]; // would use == but gcc spits a warning - BM_CHECK(std::strcmp(csv.name, "CSVReporter") == 0); + BM_CHECK(csv.name == std::string("CSVReporter")); internal::GetResultsChecker().CheckResults(csv.out_stream); } diff --git a/third-party/benchmark/test/perf_counters_gtest.cc b/third-party/benchmark/test/perf_counters_gtest.cc index 3eac62463bc5..2e63049285d7 100644 --- a/third-party/benchmark/test/perf_counters_gtest.cc +++ b/third-party/benchmark/test/perf_counters_gtest.cc @@ -1,6 +1,8 @@ +#include #include #include "../src/perf_counters.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" #ifndef GTEST_SKIP @@ -11,12 +13,15 @@ struct MsgHandler { #endif using benchmark::internal::PerfCounters; +using benchmark::internal::PerfCountersMeasurement; using benchmark::internal::PerfCounterValues; +using ::testing::AllOf; +using ::testing::Gt; +using ::testing::Lt; namespace { const char kGenericPerfEvent1[] = "CYCLES"; -const char kGenericPerfEvent2[] = "BRANCHES"; -const char kGenericPerfEvent3[] = "INSTRUCTIONS"; +const char kGenericPerfEvent2[] = "INSTRUCTIONS"; TEST(PerfCountersTest, Init) { EXPECT_EQ(PerfCounters::Initialize(), PerfCounters::kSupported); @@ -27,7 +32,7 @@ TEST(PerfCountersTest, OneCounter) { GTEST_SKIP() << "Performance counters not supported.\n"; } EXPECT_TRUE(PerfCounters::Initialize()); - EXPECT_TRUE(PerfCounters::Create({kGenericPerfEvent1}).IsValid()); + EXPECT_EQ(PerfCounters::Create({kGenericPerfEvent1}).num_counters(), 1); } TEST(PerfCountersTest, NegativeTest) { @@ -36,29 +41,44 @@ TEST(PerfCountersTest, NegativeTest) { return; } EXPECT_TRUE(PerfCounters::Initialize()); - EXPECT_FALSE(PerfCounters::Create({}).IsValid()); - EXPECT_FALSE(PerfCounters::Create({""}).IsValid()); - EXPECT_FALSE(PerfCounters::Create({"not a counter name"}).IsValid()); + // Safety checks + // Create() will always create a valid object, even if passed no or + // wrong arguments as the new behavior is to warn and drop unsupported + // counters + EXPECT_EQ(PerfCounters::Create({}).num_counters(), 0); + EXPECT_EQ(PerfCounters::Create({""}).num_counters(), 0); + EXPECT_EQ(PerfCounters::Create({"not a counter name"}).num_counters(), 0); { - EXPECT_TRUE(PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2, - kGenericPerfEvent3}) - .IsValid()); - } - EXPECT_FALSE( - PerfCounters::Create({kGenericPerfEvent2, "", kGenericPerfEvent1}) - .IsValid()); - EXPECT_FALSE(PerfCounters::Create({kGenericPerfEvent3, "not a counter name", - kGenericPerfEvent1}) - .IsValid()); + // Try sneaking in a bad egg to see if it is filtered out. The + // number of counters has to be two, not zero + auto counter = + PerfCounters::Create({kGenericPerfEvent2, "", kGenericPerfEvent1}); + EXPECT_EQ(counter.num_counters(), 2); + EXPECT_EQ(counter.names(), std::vector( + {kGenericPerfEvent2, kGenericPerfEvent1})); + } + { + // Try sneaking in an outrageous counter, like a fat finger mistake + auto counter = PerfCounters::Create( + {kGenericPerfEvent2, "not a counter name", kGenericPerfEvent1}); + EXPECT_EQ(counter.num_counters(), 2); + EXPECT_EQ(counter.names(), std::vector( + {kGenericPerfEvent2, kGenericPerfEvent1})); + } { - EXPECT_TRUE(PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2, - kGenericPerfEvent3}) - .IsValid()); - } - EXPECT_FALSE( - PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2, - kGenericPerfEvent3, "MISPREDICTED_BRANCH_RETIRED"}) - .IsValid()); + // Finally try a golden input - it should like both of them + EXPECT_EQ(PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2}) + .num_counters(), + 2); + } + { + // Add a bad apple in the end of the chain to check the edges + auto counter = PerfCounters::Create( + {kGenericPerfEvent1, kGenericPerfEvent2, "bad event name"}); + EXPECT_EQ(counter.num_counters(), 2); + EXPECT_EQ(counter.names(), std::vector( + {kGenericPerfEvent1, kGenericPerfEvent2})); + } } TEST(PerfCountersTest, Read1Counter) { @@ -67,7 +87,7 @@ TEST(PerfCountersTest, Read1Counter) { } EXPECT_TRUE(PerfCounters::Initialize()); auto counters = PerfCounters::Create({kGenericPerfEvent1}); - EXPECT_TRUE(counters.IsValid()); + EXPECT_EQ(counters.num_counters(), 1); PerfCounterValues values1(1); EXPECT_TRUE(counters.Snapshot(&values1)); EXPECT_GT(values1[0], 0); @@ -84,7 +104,7 @@ TEST(PerfCountersTest, Read2Counters) { EXPECT_TRUE(PerfCounters::Initialize()); auto counters = PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2}); - EXPECT_TRUE(counters.IsValid()); + EXPECT_EQ(counters.num_counters(), 2); PerfCounterValues values1(2); EXPECT_TRUE(counters.Snapshot(&values1)); EXPECT_GT(values1[0], 0); @@ -95,30 +115,121 @@ TEST(PerfCountersTest, Read2Counters) { EXPECT_GT(values2[1], 0); } -size_t do_work() { - size_t res = 0; - for (size_t i = 0; i < 100000000; ++i) res += i * i; - return res; +TEST(PerfCountersTest, ReopenExistingCounters) { + // This test works in recent and old Intel hardware, Pixel 3, and Pixel 6. + // However we cannot make assumptions beyond 2 HW counters due to Pixel 6. + if (!PerfCounters::kSupported) { + GTEST_SKIP() << "Test skipped because libpfm is not supported.\n"; + } + EXPECT_TRUE(PerfCounters::Initialize()); + std::vector kMetrics({kGenericPerfEvent1}); + std::vector counters(2); + for (auto& counter : counters) { + counter = PerfCounters::Create(kMetrics); + } + PerfCounterValues values(1); + EXPECT_TRUE(counters[0].Snapshot(&values)); + EXPECT_TRUE(counters[1].Snapshot(&values)); } -void measure(size_t threadcount, PerfCounterValues* values1, - PerfCounterValues* values2) { - BM_CHECK_NE(values1, nullptr); - BM_CHECK_NE(values2, nullptr); +TEST(PerfCountersTest, CreateExistingMeasurements) { + // The test works (i.e. causes read to fail) for the assumptions + // about hardware capabilities (i.e. small number (2) hardware + // counters) at this date, + // the same as previous test ReopenExistingCounters. + if (!PerfCounters::kSupported) { + GTEST_SKIP() << "Test skipped because libpfm is not supported.\n"; + } + EXPECT_TRUE(PerfCounters::Initialize()); + + // This means we will try 10 counters but we can only guarantee + // for sure at this time that only 3 will work. Perhaps in the future + // we could use libpfm to query for the hardware limits on this + // particular platform. + const int kMaxCounters = 10; + const int kMinValidCounters = 2; + + // Let's use a ubiquitous counter that is guaranteed to work + // on all platforms + const std::vector kMetrics{"cycles"}; + + // Cannot create a vector of actual objects because the + // copy constructor of PerfCounters is deleted - and so is + // implicitly deleted on PerfCountersMeasurement too + std::vector> + perf_counter_measurements; + + perf_counter_measurements.reserve(kMaxCounters); + for (int j = 0; j < kMaxCounters; ++j) { + perf_counter_measurements.emplace_back( + new PerfCountersMeasurement(kMetrics)); + } + + std::vector> measurements; + + // Start all counters together to see if they hold + size_t max_counters = kMaxCounters; + for (size_t i = 0; i < kMaxCounters; ++i) { + auto& counter(*perf_counter_measurements[i]); + EXPECT_EQ(counter.num_counters(), 1); + if (!counter.Start()) { + max_counters = i; + break; + }; + } + + ASSERT_GE(max_counters, kMinValidCounters); + + // Start all together + for (size_t i = 0; i < max_counters; ++i) { + auto& counter(*perf_counter_measurements[i]); + EXPECT_TRUE(counter.Stop(measurements) || (i >= kMinValidCounters)); + } + + // Start/stop individually + for (size_t i = 0; i < max_counters; ++i) { + auto& counter(*perf_counter_measurements[i]); + measurements.clear(); + counter.Start(); + EXPECT_TRUE(counter.Stop(measurements) || (i >= kMinValidCounters)); + } +} + +// We try to do some meaningful work here but the compiler +// insists in optimizing away our loop so we had to add a +// no-optimize macro. In case it fails, we added some entropy +// to this pool as well. + +BENCHMARK_DONT_OPTIMIZE size_t do_work() { + static std::mt19937 rd{std::random_device{}()}; + static std::uniform_int_distribution mrand(0, 10); + const size_t kNumLoops = 1000000; + size_t sum = 0; + for (size_t j = 0; j < kNumLoops; ++j) { + sum += mrand(rd); + } + benchmark::DoNotOptimize(sum); + return sum; +} + +void measure(size_t threadcount, PerfCounterValues* before, + PerfCounterValues* after) { + BM_CHECK_NE(before, nullptr); + BM_CHECK_NE(after, nullptr); std::vector threads(threadcount); auto work = [&]() { BM_CHECK(do_work() > 1000); }; // We need to first set up the counters, then start the threads, so the - // threads would inherit the counters. But later, we need to first destroy the - // thread pool (so all the work finishes), then measure the counters. So the - // scopes overlap, and we need to explicitly control the scope of the + // threads would inherit the counters. But later, we need to first destroy + // the thread pool (so all the work finishes), then measure the counters. So + // the scopes overlap, and we need to explicitly control the scope of the // threadpool. auto counters = - PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent3}); + PerfCounters::Create({kGenericPerfEvent1, kGenericPerfEvent2}); for (auto& t : threads) t = std::thread(work); - counters.Snapshot(values1); + counters.Snapshot(before); for (auto& t : threads) t.join(); - counters.Snapshot(values2); + counters.Snapshot(after); } TEST(PerfCountersTest, MultiThreaded) { @@ -126,20 +237,71 @@ TEST(PerfCountersTest, MultiThreaded) { GTEST_SKIP() << "Test skipped because libpfm is not supported."; } EXPECT_TRUE(PerfCounters::Initialize()); - PerfCounterValues values1(2); - PerfCounterValues values2(2); + PerfCounterValues before(2); + PerfCounterValues after(2); - measure(2, &values1, &values2); - std::vector D1{static_cast(values2[0] - values1[0]), - static_cast(values2[1] - values1[1])}; + // Notice that this test will work even if we taskset it to a single CPU + // In this case the threads will run sequentially + // Start two threads and measure the number of combined cycles and + // instructions + measure(2, &before, &after); + std::vector Elapsed2Threads{ + static_cast(after[0] - before[0]), + static_cast(after[1] - before[1])}; - measure(4, &values1, &values2); - std::vector D2{static_cast(values2[0] - values1[0]), - static_cast(values2[1] - values1[1])}; + // Start four threads and measure the number of combined cycles and + // instructions + measure(4, &before, &after); + std::vector Elapsed4Threads{ + static_cast(after[0] - before[0]), + static_cast(after[1] - before[1])}; - // Some extra work will happen on the main thread - like joining the threads - // - so the ratio won't be quite 2.0, but very close. - EXPECT_GE(D2[0], 1.9 * D1[0]); - EXPECT_GE(D2[1], 1.9 * D1[1]); + // The following expectations fail (at least on a beefy workstation with lots + // of cpus) - it seems that in some circumstances the runtime of 4 threads + // can even be better than with 2. + // So instead of expecting 4 threads to be slower, let's just make sure they + // do not differ too much in general (one is not more than 10x than the + // other). + EXPECT_THAT(Elapsed4Threads[0] / Elapsed2Threads[0], AllOf(Gt(0.1), Lt(10))); + EXPECT_THAT(Elapsed4Threads[1] / Elapsed2Threads[1], AllOf(Gt(0.1), Lt(10))); } + +TEST(PerfCountersTest, HardwareLimits) { + // The test works (i.e. causes read to fail) for the assumptions + // about hardware capabilities (i.e. small number (3-4) hardware + // counters) at this date, + // the same as previous test ReopenExistingCounters. + if (!PerfCounters::kSupported) { + GTEST_SKIP() << "Test skipped because libpfm is not supported.\n"; + } + EXPECT_TRUE(PerfCounters::Initialize()); + + // Taken from `perf list`, but focusses only on those HW events that actually + // were reported when running `sudo perf stat -a sleep 10`, intersected over + // several platforms. All HW events listed in the first command not reported + // in the second seem to not work. This is sad as we don't really get to test + // the grouping here (groups can contain up to 6 members)... + std::vector counter_names{ + "cycles", // leader + "instructions", // + "branch-misses", // + }; + + // In the off-chance that some of these values are not supported, + // we filter them out so the test will complete without failure + // albeit it might not actually test the grouping on that platform + std::vector valid_names; + for (const std::string& name : counter_names) { + if (PerfCounters::IsCounterSupported(name)) { + valid_names.push_back(name); + } + } + PerfCountersMeasurement counter(valid_names); + + std::vector> measurements; + + counter.Start(); + EXPECT_TRUE(counter.Stop(measurements)); +} + } // namespace diff --git a/third-party/benchmark/test/perf_counters_test.cc b/third-party/benchmark/test/perf_counters_test.cc index 3017a452fe27..3cc593e629d8 100644 --- a/third-party/benchmark/test/perf_counters_test.cc +++ b/third-party/benchmark/test/perf_counters_test.cc @@ -1,27 +1,92 @@ +#include #undef NDEBUG +#include "../src/commandlineflags.h" #include "../src/perf_counters.h" - #include "benchmark/benchmark.h" #include "output_test.h" +namespace benchmark { + +BM_DECLARE_string(benchmark_perf_counters); + +} // namespace benchmark + static void BM_Simple(benchmark::State& state) { for (auto _ : state) { - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } } BENCHMARK(BM_Simple); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Simple\",$"}}); +const int kIters = 1000000; + +void BM_WithoutPauseResume(benchmark::State& state) { + int n = 0; + + for (auto _ : state) { + for (auto i = 0; i < kIters; ++i) { + n = 1 - n; + benchmark::DoNotOptimize(n); + } + } +} + +BENCHMARK(BM_WithoutPauseResume); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_WithoutPauseResume\",$"}}); + +void BM_WithPauseResume(benchmark::State& state) { + int m = 0, n = 0; + + for (auto _ : state) { + for (auto i = 0; i < kIters; ++i) { + n = 1 - n; + benchmark::DoNotOptimize(n); + } + + state.PauseTiming(); + for (auto j = 0; j < kIters; ++j) { + m = 1 - m; + benchmark::DoNotOptimize(m); + } + state.ResumeTiming(); + } +} + +BENCHMARK(BM_WithPauseResume); + +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_WithPauseResume\",$"}}); + static void CheckSimple(Results const& e) { CHECK_COUNTER_VALUE(e, double, "CYCLES", GT, 0); - CHECK_COUNTER_VALUE(e, double, "BRANCHES", GT, 0.0); } + +double withoutPauseResumeInstrCount = 0.0; +double withPauseResumeInstrCount = 0.0; + +static void SaveInstrCountWithoutResume(Results const& e) { + withoutPauseResumeInstrCount = e.GetAs("INSTRUCTIONS"); +} + +static void SaveInstrCountWithResume(Results const& e) { + withPauseResumeInstrCount = e.GetAs("INSTRUCTIONS"); +} + CHECK_BENCHMARK_RESULTS("BM_Simple", &CheckSimple); +CHECK_BENCHMARK_RESULTS("BM_WithoutPauseResume", &SaveInstrCountWithoutResume); +CHECK_BENCHMARK_RESULTS("BM_WithPauseResume", &SaveInstrCountWithResume); int main(int argc, char* argv[]) { if (!benchmark::internal::PerfCounters::kSupported) { return 0; } + benchmark::FLAGS_benchmark_perf_counters = "CYCLES,INSTRUCTIONS"; + benchmark::internal::PerfCounters::Initialize(); RunOutputTests(argc, argv); + + BM_CHECK_GT(withPauseResumeInstrCount, kIters); + BM_CHECK_GT(withoutPauseResumeInstrCount, kIters); + BM_CHECK_LT(withPauseResumeInstrCount, 1.5 * withoutPauseResumeInstrCount); } diff --git a/third-party/benchmark/test/register_benchmark_test.cc b/third-party/benchmark/test/register_benchmark_test.cc index 602405b67e8d..d69d144a4e17 100644 --- a/third-party/benchmark/test/register_benchmark_test.cc +++ b/third-party/benchmark/test/register_benchmark_test.cc @@ -10,7 +10,7 @@ namespace { class TestReporter : public benchmark::ConsoleReporter { public: - virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { + void ReportRuns(const std::vector& report) override { all_runs_.insert(all_runs_.end(), begin(report), end(report)); ConsoleReporter::ReportRuns(report); } @@ -19,11 +19,11 @@ class TestReporter : public benchmark::ConsoleReporter { }; struct TestCase { - std::string name; - const char* label; + const std::string name; + const std::string label; // Note: not explicit as we rely on it being converted through ADD_CASES. - TestCase(const char* xname) : TestCase(xname, nullptr) {} - TestCase(const char* xname, const char* xlabel) + TestCase(const std::string& xname) : TestCase(xname, "") {} + TestCase(const std::string& xname, const std::string& xlabel) : name(xname), label(xlabel) {} typedef benchmark::BenchmarkReporter::Run Run; @@ -32,7 +32,7 @@ struct TestCase { // clang-format off BM_CHECK(name == run.benchmark_name()) << "expected " << name << " got " << run.benchmark_name(); - if (label) { + if (!label.empty()) { BM_CHECK(run.report_label == label) << "expected " << label << " got " << run.report_label; } else { @@ -95,6 +95,18 @@ ADD_CASES({"test1", "One"}, {"test2", "Two"}, {"test3", "Three"}); #endif // BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK +//----------------------------------------------------------------------------// +// Test RegisterBenchmark with DISABLED_ benchmark +//----------------------------------------------------------------------------// +void DISABLED_BM_function(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(DISABLED_BM_function); +ReturnVal dummy3 = benchmark::RegisterBenchmark("DISABLED_BM_function_manual", + DISABLED_BM_function); +// No need to add cases because we don't expect them to run. + //----------------------------------------------------------------------------// // Test RegisterBenchmark with different callable types //----------------------------------------------------------------------------// @@ -111,7 +123,7 @@ void TestRegistrationAtRuntime() { { CustomFixture fx; benchmark::RegisterBenchmark("custom_fixture", fx); - AddCases({"custom_fixture"}); + AddCases({std::string("custom_fixture")}); } #endif #ifndef BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK diff --git a/third-party/benchmark/test/reporter_output_test.cc b/third-party/benchmark/test/reporter_output_test.cc index 2b6e6543dd2e..7867165d1f3d 100644 --- a/third-party/benchmark/test/reporter_output_test.cc +++ b/third-party/benchmark/test/reporter_output_test.cc @@ -17,7 +17,7 @@ static int AddContextCases() { AddCases(TC_ConsoleErr, { {"^%int-%int-%intT%int:%int:%int[-+]%int:%int$", MR_Default}, - {"Running .*/reporter_output_test(\\.exe)?$", MR_Next}, + {"Running .*(/|\\\\)reporter_output_test(\\.exe)?$", MR_Next}, {"Run on \\(%int X %float MHz CPU s?\\)", MR_Next}, }); AddCases(TC_JSONOut, @@ -55,6 +55,9 @@ static int AddContextCases() { {{"Load Average: (%float, ){0,2}%float$", MR_Next}}); } AddCases(TC_JSONOut, {{"\"load_avg\": \\[(%float,?){0,3}],$", MR_Next}}); + AddCases(TC_JSONOut, {{"\"library_version\": \".*\",$", MR_Next}}); + AddCases(TC_JSONOut, {{"\"library_build_type\": \".*\",$", MR_Next}}); + AddCases(TC_JSONOut, {{"\"json_schema_version\": 1$", MR_Next}}); return 0; } int dummy_register = AddContextCases(); @@ -93,7 +96,8 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}}); void BM_bytes_per_second(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } state.SetBytesProcessed(1); } @@ -124,7 +128,8 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}}); void BM_items_per_second(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } state.SetItemsProcessed(1); } @@ -318,7 +323,7 @@ ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}, ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}}); // ========================================================================= // -// ------------------------ Testing Arg Name Output ----------------------- // +// ------------------------ Testing Arg Name Output ------------------------ // // ========================================================================= // void BM_arg_name(benchmark::State& state) { @@ -404,7 +409,8 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_BigArgs/1073741824 %console_report$"}, void BM_Complexity_O1(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } state.SetComplexityN(state.range(0)); } @@ -1085,7 +1091,7 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_UserPercentStats/iterations:5/repeats:3/" {"^\"BM_UserPercentStats/iterations:5/repeats:3/" "manual_time_stddev\",%csv_report$"}, {"^\"BM_UserPercentStats/iterations:5/repeats:3/" - "manual_time_\",%csv_report$"}}); + "manual_time_\",%csv_cv_report$"}}); // ========================================================================= // // ------------------------- Testing StrEscape JSON ------------------------ // diff --git a/third-party/benchmark/test/skip_with_error_test.cc b/third-party/benchmark/test/skip_with_error_test.cc index 026d47913350..2139a19e2507 100644 --- a/third-party/benchmark/test/skip_with_error_test.cc +++ b/third-party/benchmark/test/skip_with_error_test.cc @@ -10,17 +10,17 @@ namespace { class TestReporter : public benchmark::ConsoleReporter { public: - virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { + bool ReportContext(const Context& context) override { return ConsoleReporter::ReportContext(context); }; - virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { + void ReportRuns(const std::vector& report) override { all_runs_.insert(all_runs_.end(), begin(report), end(report)); ConsoleReporter::ReportRuns(report); } TestReporter() {} - virtual ~TestReporter() {} + ~TestReporter() override {} mutable std::vector all_runs_; }; @@ -35,8 +35,9 @@ struct TestCase { void CheckRun(Run const& run) const { BM_CHECK(name == run.benchmark_name()) << "expected " << name << " got " << run.benchmark_name(); - BM_CHECK(error_occurred == run.error_occurred); - BM_CHECK(error_message == run.error_message); + BM_CHECK_EQ(error_occurred, + benchmark::internal::SkippedWithError == run.skipped); + BM_CHECK(error_message == run.skip_message); if (error_occurred) { // BM_CHECK(run.iterations == 0); } else { @@ -47,7 +48,8 @@ struct TestCase { std::vector ExpectedResults; -int AddCases(const char* base_name, std::initializer_list const& v) { +int AddCases(const std::string& base_name, + std::initializer_list const& v) { for (auto TC : v) { TC.name = base_name + TC.name; ExpectedResults.push_back(std::move(TC)); @@ -141,7 +143,8 @@ ADD_CASES("BM_error_during_running_ranged_for", void BM_error_after_running(benchmark::State& state) { for (auto _ : state) { - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } if (state.thread_index() <= (state.threads() / 2)) state.SkipWithError("error message"); diff --git a/third-party/benchmark/test/spec_arg_test.cc b/third-party/benchmark/test/spec_arg_test.cc index 043db1be47a2..06aafbeb9b5e 100644 --- a/third-party/benchmark/test/spec_arg_test.cc +++ b/third-party/benchmark/test/spec_arg_test.cc @@ -17,11 +17,11 @@ namespace { class TestReporter : public benchmark::ConsoleReporter { public: - virtual bool ReportContext(const Context& context) BENCHMARK_OVERRIDE { + bool ReportContext(const Context& context) override { return ConsoleReporter::ReportContext(context); }; - virtual void ReportRuns(const std::vector& report) BENCHMARK_OVERRIDE { + void ReportRuns(const std::vector& report) override { assert(report.size() == 1); matched_functions.push_back(report[0].run_name.function_name); ConsoleReporter::ReportRuns(report); @@ -29,7 +29,7 @@ class TestReporter : public benchmark::ConsoleReporter { TestReporter() {} - virtual ~TestReporter() {} + ~TestReporter() override {} const std::vector& GetMatchedFunctions() const { return matched_functions; @@ -91,5 +91,15 @@ int main(int argc, char** argv) { << matched_functions.front() << "]\n"; return 2; } + + // Test that SetBenchmarkFilter works. + const std::string golden_value = "golden_value"; + benchmark::SetBenchmarkFilter(golden_value); + std::string current_value = benchmark::GetBenchmarkFilter(); + if (golden_value != current_value) { + std::cerr << "Expected [" << golden_value + << "] for --benchmark_filter but got [" << current_value << "]\n"; + return 3; + } return 0; } diff --git a/third-party/benchmark/test/spec_arg_verbosity_test.cc b/third-party/benchmark/test/spec_arg_verbosity_test.cc new file mode 100644 index 000000000000..8f8eb6d37c59 --- /dev/null +++ b/third-party/benchmark/test/spec_arg_verbosity_test.cc @@ -0,0 +1,43 @@ +#include + +#include + +#include "benchmark/benchmark.h" + +// Tests that the user specified verbosity level can be get. +static void BM_Verbosity(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(BM_Verbosity); + +int main(int argc, char** argv) { + const int32_t flagv = 42; + + // Verify that argv specify --v=42. + bool found = false; + for (int i = 0; i < argc; ++i) { + if (strcmp("--v=42", argv[i]) == 0) { + found = true; + break; + } + } + if (!found) { + std::cerr << "This test requires '--v=42' to be passed as a command-line " + << "argument.\n"; + return 1; + } + + benchmark::Initialize(&argc, argv); + + // Check that the current flag value is reported accurately via the + // GetBenchmarkVerbosity() function. + if (flagv != benchmark::GetBenchmarkVerbosity()) { + std::cerr + << "Seeing different value for flags. GetBenchmarkVerbosity() returns [" + << benchmark::GetBenchmarkVerbosity() << "] expected flag=[" << flagv + << "]\n"; + return 1; + } + return 0; +} diff --git a/third-party/benchmark/test/statistics_gtest.cc b/third-party/benchmark/test/statistics_gtest.cc index 1de2d87d4ba5..48c77260fd53 100644 --- a/third-party/benchmark/test/statistics_gtest.cc +++ b/third-party/benchmark/test/statistics_gtest.cc @@ -28,8 +28,8 @@ TEST(StatisticsTest, StdDev) { TEST(StatisticsTest, CV) { EXPECT_DOUBLE_EQ(benchmark::StatisticsCV({101, 101, 101, 101}), 0.0); EXPECT_DOUBLE_EQ(benchmark::StatisticsCV({1, 2, 3}), 1. / 2.); - EXPECT_DOUBLE_EQ(benchmark::StatisticsCV({2.5, 2.4, 3.3, 4.2, 5.1}), - 0.32888184094918121); + ASSERT_NEAR(benchmark::StatisticsCV({2.5, 2.4, 3.3, 4.2, 5.1}), + 0.32888184094918121, 1e-15); } } // end namespace diff --git a/third-party/benchmark/test/string_util_gtest.cc b/third-party/benchmark/test/string_util_gtest.cc index 698f2d43eb88..67b4bc0c24f2 100644 --- a/third-party/benchmark/test/string_util_gtest.cc +++ b/third-party/benchmark/test/string_util_gtest.cc @@ -1,9 +1,12 @@ //===---------------------------------------------------------------------===// -// statistics_test - Unit tests for src/statistics.cc +// string_util_test - Unit tests for src/string_util.cc //===---------------------------------------------------------------------===// +#include + #include "../src/internal_macros.h" #include "../src/string_util.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" namespace { @@ -63,7 +66,10 @@ TEST(StringUtilTest, stoul) { EXPECT_EQ(4ul, pos); } #ifndef BENCHMARK_HAS_NO_EXCEPTIONS - { ASSERT_THROW(benchmark::stoul("this is a test"), std::invalid_argument); } + { + ASSERT_THROW(std::ignore = benchmark::stoul("this is a test"), + std::invalid_argument); + } #endif } @@ -107,7 +113,10 @@ EXPECT_EQ(1ul, pos); EXPECT_EQ(4ul, pos); } #ifndef BENCHMARK_HAS_NO_EXCEPTIONS -{ ASSERT_THROW(benchmark::stoi("this is a test"), std::invalid_argument); } +{ + ASSERT_THROW(std::ignore = benchmark::stoi("this is a test"), + std::invalid_argument); +} #endif } @@ -137,7 +146,10 @@ EXPECT_EQ(1ul, pos); EXPECT_EQ(8ul, pos); } #ifndef BENCHMARK_HAS_NO_EXCEPTIONS -{ ASSERT_THROW(benchmark::stod("this is a test"), std::invalid_argument); } +{ + ASSERT_THROW(std::ignore = benchmark::stod("this is a test"), + std::invalid_argument); +} #endif } @@ -149,4 +161,39 @@ TEST(StringUtilTest, StrSplit) { std::vector({"hello", "there", "is", "more"})); } +using HumanReadableFixture = ::testing::TestWithParam< + std::tuple>; + +INSTANTIATE_TEST_SUITE_P( + HumanReadableTests, HumanReadableFixture, + ::testing::Values( + std::make_tuple(0.0, benchmark::Counter::kIs1024, "0"), + std::make_tuple(999.0, benchmark::Counter::kIs1024, "999"), + std::make_tuple(1000.0, benchmark::Counter::kIs1024, "1000"), + std::make_tuple(1024.0, benchmark::Counter::kIs1024, "1Ki"), + std::make_tuple(1000 * 1000.0, benchmark::Counter::kIs1024, + "976\\.56.Ki"), + std::make_tuple(1024 * 1024.0, benchmark::Counter::kIs1024, "1Mi"), + std::make_tuple(1000 * 1000 * 1000.0, benchmark::Counter::kIs1024, + "953\\.674Mi"), + std::make_tuple(1024 * 1024 * 1024.0, benchmark::Counter::kIs1024, + "1Gi"), + std::make_tuple(0.0, benchmark::Counter::kIs1000, "0"), + std::make_tuple(999.0, benchmark::Counter::kIs1000, "999"), + std::make_tuple(1000.0, benchmark::Counter::kIs1000, "1k"), + std::make_tuple(1024.0, benchmark::Counter::kIs1000, "1.024k"), + std::make_tuple(1000 * 1000.0, benchmark::Counter::kIs1000, "1M"), + std::make_tuple(1024 * 1024.0, benchmark::Counter::kIs1000, + "1\\.04858M"), + std::make_tuple(1000 * 1000 * 1000.0, benchmark::Counter::kIs1000, + "1G"), + std::make_tuple(1024 * 1024 * 1024.0, benchmark::Counter::kIs1000, + "1\\.07374G"))); + +TEST_P(HumanReadableFixture, HumanReadableNumber) { + std::string str = benchmark::HumanReadableNumber(std::get<0>(GetParam()), + std::get<1>(GetParam())); + ASSERT_THAT(str, ::testing::MatchesRegex(std::get<2>(GetParam()))); +} + } // end namespace diff --git a/third-party/benchmark/test/time_unit_gtest.cc b/third-party/benchmark/test/time_unit_gtest.cc new file mode 100644 index 000000000000..484ecbcfb411 --- /dev/null +++ b/third-party/benchmark/test/time_unit_gtest.cc @@ -0,0 +1,37 @@ +#include "../include/benchmark/benchmark.h" +#include "gtest/gtest.h" + +namespace benchmark { +namespace internal { + +namespace { + +class DummyBenchmark : public Benchmark { + public: + DummyBenchmark() : Benchmark("dummy") {} + void Run(State&) override {} +}; + +TEST(DefaultTimeUnitTest, TimeUnitIsNotSet) { + DummyBenchmark benchmark; + EXPECT_EQ(benchmark.GetTimeUnit(), kNanosecond); +} + +TEST(DefaultTimeUnitTest, DefaultIsSet) { + DummyBenchmark benchmark; + EXPECT_EQ(benchmark.GetTimeUnit(), kNanosecond); + SetDefaultTimeUnit(kMillisecond); + EXPECT_EQ(benchmark.GetTimeUnit(), kMillisecond); +} + +TEST(DefaultTimeUnitTest, DefaultAndExplicitUnitIsSet) { + DummyBenchmark benchmark; + benchmark.Unit(kMillisecond); + SetDefaultTimeUnit(kMicrosecond); + + EXPECT_EQ(benchmark.GetTimeUnit(), kMillisecond); +} + +} // namespace +} // namespace internal +} // namespace benchmark diff --git a/third-party/benchmark/test/user_counters_tabular_test.cc b/third-party/benchmark/test/user_counters_tabular_test.cc index 45ac043d5193..cfc1ab069c78 100644 --- a/third-party/benchmark/test/user_counters_tabular_test.cc +++ b/third-party/benchmark/test/user_counters_tabular_test.cc @@ -63,6 +63,9 @@ ADD_CASES(TC_CSVOut, {{"%csv_header," void BM_Counters_Tabular(benchmark::State& state) { for (auto _ : state) { + // This test requires a non-zero CPU time to avoid divide-by-zero + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters.insert({ @@ -330,7 +333,7 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/repeats:2/threads:1_stddev\",%csv_report," "%float,%float,%float,%float,%float,%float$"}}); ADD_CASES(TC_CSVOut, - {{"^\"BM_Counters_Tabular/repeats:2/threads:1_cv\",%csv_report," + {{"^\"BM_Counters_Tabular/repeats:2/threads:1_cv\",%csv_cv_report," "%float,%float,%float,%float,%float,%float$"}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/repeats:2/threads:2\",%csv_report," @@ -348,7 +351,7 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/repeats:2/threads:2_stddev\",%csv_report," "%float,%float,%float,%float,%float,%float$"}}); ADD_CASES(TC_CSVOut, - {{"^\"BM_Counters_Tabular/repeats:2/threads:2_cv\",%csv_report," + {{"^\"BM_Counters_Tabular/repeats:2/threads:2_cv\",%csv_cv_report," "%float,%float,%float,%float,%float,%float$"}}); // VS2013 does not allow this function to be passed as a lambda argument // to CHECK_BENCHMARK_RESULTS() @@ -372,7 +375,8 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/repeats:2/threads:2$", void BM_CounterRates_Tabular(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters.insert({ diff --git a/third-party/benchmark/test/user_counters_test.cc b/third-party/benchmark/test/user_counters_test.cc index 1cc74552a1bd..22252acbf6a2 100644 --- a/third-party/benchmark/test/user_counters_test.cc +++ b/third-party/benchmark/test/user_counters_test.cc @@ -67,7 +67,8 @@ int num_calls1 = 0; void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } state.counters["foo"] = 1; state.counters["bar"] = ++num_calls1; @@ -118,7 +119,8 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_WithBytesAndItemsPSec", void BM_Counters_Rate(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters["foo"] = bm::Counter{1, bm::Counter::kIsRate}; @@ -161,7 +163,8 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_Rate", &CheckRate); void BM_Invert(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters["foo"] = bm::Counter{0.0001, bm::Counter::kInvert}; @@ -195,14 +198,14 @@ void CheckInvert(Results const& e) { CHECK_BENCHMARK_RESULTS("BM_Invert", &CheckInvert); // ========================================================================= // -// ------------------------- InvertedRate Counters Output -// -------------------------- // +// --------------------- InvertedRate Counters Output ---------------------- // // ========================================================================= // void BM_Counters_InvertedRate(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters["foo"] = @@ -330,7 +333,8 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_AvgThreads/threads:%int", void BM_Counters_AvgThreadsRate(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgThreadsRate}; @@ -417,7 +421,8 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_IterationInvariant", void BM_Counters_kIsIterationInvariantRate(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters["foo"] = @@ -460,7 +465,7 @@ CHECK_BENCHMARK_RESULTS("BM_Counters_kIsIterationInvariantRate", &CheckIsIterationInvariantRate); // ========================================================================= // -// ------------------- AvgIterations Counters Output ------------------ // +// --------------------- AvgIterations Counters Output --------------------- // // ========================================================================= // void BM_Counters_AvgIterations(benchmark::State& state) { @@ -502,13 +507,14 @@ void CheckAvgIterations(Results const& e) { CHECK_BENCHMARK_RESULTS("BM_Counters_AvgIterations", &CheckAvgIterations); // ========================================================================= // -// ----------------- AvgIterationsRate Counters Output ---------------- // +// ------------------- AvgIterationsRate Counters Output ------------------- // // ========================================================================= // void BM_Counters_kAvgIterationsRate(benchmark::State& state) { for (auto _ : state) { // This test requires a non-zero CPU time to avoid divide-by-zero - benchmark::DoNotOptimize(state.iterations()); + auto iterations = double(state.iterations()) * double(state.iterations()); + benchmark::DoNotOptimize(iterations); } namespace bm = benchmark; state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgIterationsRate}; diff --git a/third-party/benchmark/test/user_counters_thousands_test.cc b/third-party/benchmark/test/user_counters_thousands_test.cc index a42683b32fa7..fc153835f800 100644 --- a/third-party/benchmark/test/user_counters_thousands_test.cc +++ b/third-party/benchmark/test/user_counters_thousands_test.cc @@ -16,13 +16,13 @@ void BM_Counters_Thousands(benchmark::State& state) { {"t0_1000000DefaultBase", bm::Counter(1000 * 1000, bm::Counter::kDefaults)}, {"t1_1000000Base1000", bm::Counter(1000 * 1000, bm::Counter::kDefaults, - benchmark::Counter::OneK::kIs1000)}, + bm::Counter::OneK::kIs1000)}, {"t2_1000000Base1024", bm::Counter(1000 * 1000, bm::Counter::kDefaults, - benchmark::Counter::OneK::kIs1024)}, + bm::Counter::OneK::kIs1024)}, {"t3_1048576Base1000", bm::Counter(1024 * 1024, bm::Counter::kDefaults, - benchmark::Counter::OneK::kIs1000)}, + bm::Counter::OneK::kIs1000)}, {"t4_1048576Base1024", bm::Counter(1024 * 1024, bm::Counter::kDefaults, - benchmark::Counter::OneK::kIs1024)}, + bm::Counter::OneK::kIs1024)}, }); } BENCHMARK(BM_Counters_Thousands)->Repetitions(2); @@ -30,21 +30,21 @@ ADD_CASES( TC_ConsoleOut, { {"^BM_Counters_Thousands/repeats:2 %console_report " - "t0_1000000DefaultBase=1000k " - "t1_1000000Base1000=1000k t2_1000000Base1024=976.56[23]k " - "t3_1048576Base1000=1048.58k t4_1048576Base1024=1024k$"}, + "t0_1000000DefaultBase=1M " + "t1_1000000Base1000=1M t2_1000000Base1024=976.56[23]Ki " + "t3_1048576Base1000=1.04858M t4_1048576Base1024=1Mi$"}, {"^BM_Counters_Thousands/repeats:2 %console_report " - "t0_1000000DefaultBase=1000k " - "t1_1000000Base1000=1000k t2_1000000Base1024=976.56[23]k " - "t3_1048576Base1000=1048.58k t4_1048576Base1024=1024k$"}, + "t0_1000000DefaultBase=1M " + "t1_1000000Base1000=1M t2_1000000Base1024=976.56[23]Ki " + "t3_1048576Base1000=1.04858M t4_1048576Base1024=1Mi$"}, {"^BM_Counters_Thousands/repeats:2_mean %console_report " - "t0_1000000DefaultBase=1000k t1_1000000Base1000=1000k " - "t2_1000000Base1024=976.56[23]k t3_1048576Base1000=1048.58k " - "t4_1048576Base1024=1024k$"}, + "t0_1000000DefaultBase=1M t1_1000000Base1000=1M " + "t2_1000000Base1024=976.56[23]Ki t3_1048576Base1000=1.04858M " + "t4_1048576Base1024=1Mi$"}, {"^BM_Counters_Thousands/repeats:2_median %console_report " - "t0_1000000DefaultBase=1000k t1_1000000Base1000=1000k " - "t2_1000000Base1024=976.56[23]k t3_1048576Base1000=1048.58k " - "t4_1048576Base1024=1024k$"}, + "t0_1000000DefaultBase=1M t1_1000000Base1000=1M " + "t2_1000000Base1024=976.56[23]Ki t3_1048576Base1000=1.04858M " + "t4_1048576Base1024=1Mi$"}, {"^BM_Counters_Thousands/repeats:2_stddev %console_time_only_report [ " "]*2 t0_1000000DefaultBase=0 t1_1000000Base1000=0 " "t2_1000000Base1024=0 t3_1048576Base1000=0 t4_1048576Base1024=0$"}, diff --git a/third-party/benchmark/tools/compare.py b/third-party/benchmark/tools/compare.py index f1504c96fa2b..7572520cc0ca 100755 --- a/third-party/benchmark/tools/compare.py +++ b/third-party/benchmark/tools/compare.py @@ -1,29 +1,35 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -import unittest +# type: ignore """ compare.py - versatile benchmark output compare tool """ import argparse -from argparse import ArgumentParser import json +import os import sys +import unittest +from argparse import ArgumentParser + import gbench -from gbench import util, report -from gbench.util import * +from gbench import report, util def check_inputs(in1, in2, flags): """ Perform checking on the user provided inputs and diagnose any abnormalities """ - in1_kind, in1_err = classify_input_file(in1) - in2_kind, in2_err = classify_input_file(in2) - output_file = find_benchmark_flag("--benchmark_out=", flags) - output_type = find_benchmark_flag("--benchmark_out_format=", flags) - if in1_kind == IT_Executable and in2_kind == IT_Executable and output_file: + in1_kind, in1_err = util.classify_input_file(in1) + in2_kind, in2_err = util.classify_input_file(in2) + output_file = util.find_benchmark_flag("--benchmark_out=", flags) + output_type = util.find_benchmark_flag("--benchmark_out_format=", flags) + if ( + in1_kind == util.IT_Executable + and in2_kind == util.IT_Executable + and output_file + ): print( ( "WARNING: '--benchmark_out=%s' will be passed to both " @@ -31,11 +37,14 @@ def check_inputs(in1, in2, flags): ) % output_file ) - if in1_kind == IT_JSON and in2_kind == IT_JSON and len(flags) > 0: - print( - "WARNING: passing optional flags has no effect since both " - "inputs are JSON" - ) + if in1_kind == util.IT_JSON and in2_kind == util.IT_JSON: + # When both sides are JSON the only supported flag is + # --benchmark_filter= + for flag in util.remove_benchmark_flags("--benchmark_filter=", flags): + print( + "WARNING: passing %s has no effect since both " + "inputs are JSON" % flag + ) if output_type is not None and output_type != "json": print( ( @@ -48,7 +57,9 @@ def check_inputs(in1, in2, flags): def create_parser(): - parser = ArgumentParser(description="versatile benchmark output compare tool") + parser = ArgumentParser( + description="versatile benchmark output compare tool" + ) parser.add_argument( "-a", @@ -294,7 +305,9 @@ def main(): # Now, filter the benchmarks so that the difference report can work if filter_baseline and filter_contender: replacement = "[%s vs. %s]" % (filter_baseline, filter_contender) - json1 = gbench.report.filter_benchmark(json1_orig, filter_baseline, replacement) + json1 = gbench.report.filter_benchmark( + json1_orig, filter_baseline, replacement + ) json2 = gbench.report.filter_benchmark( json2_orig, filter_contender, replacement ) @@ -314,7 +327,7 @@ def main(): # Optionally, diff and output to JSON if args.dump_to_json is not None: with open(args.dump_to_json, "w") as f_json: - json.dump(diff_report, f_json) + json.dump(diff_report, f_json, indent=1) class TestParser(unittest.TestCase): @@ -423,7 +436,9 @@ class TestParser(unittest.TestCase): self.assertFalse(parsed.benchmark_options) def test_filters_with_remainder(self): - parsed = self.parser.parse_args(["filters", self.testInput0, "c", "d", "e"]) + parsed = self.parser.parse_args( + ["filters", self.testInput0, "c", "d", "e"] + ) self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) self.assertEqual(parsed.mode, "filters") @@ -459,7 +474,14 @@ class TestParser(unittest.TestCase): def test_benchmarksfiltered_with_remainder(self): parsed = self.parser.parse_args( - ["benchmarksfiltered", self.testInput0, "c", self.testInput1, "e", "f"] + [ + "benchmarksfiltered", + self.testInput0, + "c", + self.testInput1, + "e", + "f", + ] ) self.assertFalse(parsed.display_aggregates_only) self.assertTrue(parsed.utest) diff --git a/third-party/benchmark/tools/gbench/Inputs/test1_run1.json b/third-party/benchmark/tools/gbench/Inputs/test1_run1.json index 601e327aefb5..9daed0bcc6c4 100644 --- a/third-party/benchmark/tools/gbench/Inputs/test1_run1.json +++ b/third-party/benchmark/tools/gbench/Inputs/test1_run1.json @@ -114,6 +114,14 @@ "real_time": 1, "cpu_time": 1, "time_unit": "s" + }, + { + "name": "BM_hasLabel", + "label": "a label", + "iterations": 1, + "real_time": 1, + "cpu_time": 1, + "time_unit": "s" } ] } diff --git a/third-party/benchmark/tools/gbench/Inputs/test1_run2.json b/third-party/benchmark/tools/gbench/Inputs/test1_run2.json index 3cbcf39b0c93..dc52970abf8b 100644 --- a/third-party/benchmark/tools/gbench/Inputs/test1_run2.json +++ b/third-party/benchmark/tools/gbench/Inputs/test1_run2.json @@ -114,6 +114,14 @@ "real_time": 1, "cpu_time": 1, "time_unit": "ns" + }, + { + "name": "BM_hasLabel", + "label": "a label", + "iterations": 1, + "real_time": 1, + "cpu_time": 1, + "time_unit": "s" } ] } diff --git a/third-party/benchmark/tools/gbench/Inputs/test5_run0.json b/third-party/benchmark/tools/gbench/Inputs/test5_run0.json new file mode 100644 index 000000000000..074103b11dde --- /dev/null +++ b/third-party/benchmark/tools/gbench/Inputs/test5_run0.json @@ -0,0 +1,18 @@ +{ + "context": { + "date": "2016-08-02 17:44:46", + "num_cpus": 4, + "mhz_per_cpu": 4228, + "cpu_scaling_enabled": false, + "library_build_type": "release" + }, + "benchmarks": [ + { + "name": "BM_ManyRepetitions", + "iterations": 1000, + "real_time": 1, + "cpu_time": 1000, + "time_unit": "s" + } + ] +} diff --git a/third-party/benchmark/tools/gbench/Inputs/test5_run1.json b/third-party/benchmark/tools/gbench/Inputs/test5_run1.json new file mode 100644 index 000000000000..430df9f0da64 --- /dev/null +++ b/third-party/benchmark/tools/gbench/Inputs/test5_run1.json @@ -0,0 +1,18 @@ +{ + "context": { + "date": "2016-08-02 17:44:46", + "num_cpus": 4, + "mhz_per_cpu": 4228, + "cpu_scaling_enabled": false, + "library_build_type": "release" + }, + "benchmarks": [ + { + "name": "BM_ManyRepetitions", + "iterations": 1000, + "real_time": 1000, + "cpu_time": 1, + "time_unit": "s" + } + ] +} diff --git a/third-party/benchmark/tools/gbench/__init__.py b/third-party/benchmark/tools/gbench/__init__.py index ffca396b4c3f..921256881491 100644 --- a/third-party/benchmark/tools/gbench/__init__.py +++ b/third-party/benchmark/tools/gbench/__init__.py @@ -5,4 +5,4 @@ __email__ = "eric@efcs.ca" __versioninfo__ = (0, 5, 0) __version__ = ".".join(str(v) for v in __versioninfo__) + "dev" -__all__ = [] +__all__ = [] # type: ignore diff --git a/third-party/benchmark/tools/gbench/report.py b/third-party/benchmark/tools/gbench/report.py index 5092b0bf1469..7158fd1654cb 100644 --- a/third-party/benchmark/tools/gbench/report.py +++ b/third-party/benchmark/tools/gbench/report.py @@ -1,15 +1,17 @@ -"""report.py - Utilities for reporting statistics about benchmark results +# type: ignore + +""" +report.py - Utilities for reporting statistics about benchmark results """ -import unittest -import os -import re import copy +import os import random +import re +import unittest -from scipy.stats import mannwhitneyu, gmean from numpy import array -from pandas import Timedelta +from scipy.stats import gmean, mannwhitneyu class BenchmarkColor(object): @@ -42,6 +44,13 @@ UTEST_MIN_REPETITIONS = 2 UTEST_OPTIMAL_REPETITIONS = 9 # Lowest reasonable number, More is better. UTEST_COL_NAME = "_pvalue" +_TIME_UNIT_TO_SECONDS_MULTIPLIER = { + "s": 1.0, + "ms": 1e-3, + "us": 1e-6, + "ns": 1e-9, +} + def color_format(use_color, fmt_str, *args, **kwargs): """ @@ -52,7 +61,10 @@ def color_format(use_color, fmt_str, *args, **kwargs): """ assert use_color is True or use_color is False if not use_color: - args = [arg if not isinstance(arg, BenchmarkColor) else BC_NONE for arg in args] + args = [ + arg if not isinstance(arg, BenchmarkColor) else BC_NONE + for arg in args + ] kwargs = { key: arg if not isinstance(arg, BenchmarkColor) else BC_NONE for key, arg in kwargs.items() @@ -165,9 +177,9 @@ def get_timedelta_field_as_seconds(benchmark, field_name): Get value of field_name field of benchmark, which is time with time unit time_unit, as time in seconds. """ - time_unit = benchmark["time_unit"] if "time_unit" in benchmark else "s" - dt = Timedelta(benchmark[field_name], time_unit) - return dt / Timedelta(1, "s") + timedelta = benchmark[field_name] + time_unit = benchmark.get("time_unit", "s") + return timedelta * _TIME_UNIT_TO_SECONDS_MULTIPLIER.get(time_unit) def calculate_geomean(json): @@ -273,6 +285,7 @@ def get_difference_report(json1, json2, utest=False): partitions = partition_benchmarks(json1, json2) for partition in partitions: benchmark_name = partition[0][0]["name"] + label = partition[0][0]["label"] if "label" in partition[0][0] else "" time_unit = partition[0][0]["time_unit"] measurements = [] utest_results = {} @@ -286,8 +299,12 @@ def get_difference_report(json1, json2, utest=False): "cpu_time": bn["cpu_time"], "real_time_other": other_bench["real_time"], "cpu_time_other": other_bench["cpu_time"], - "time": calculate_change(bn["real_time"], other_bench["real_time"]), - "cpu": calculate_change(bn["cpu_time"], other_bench["cpu_time"]), + "time": calculate_change( + bn["real_time"], other_bench["real_time"] + ), + "cpu": calculate_change( + bn["cpu_time"], other_bench["cpu_time"] + ), } ) @@ -298,7 +315,7 @@ def get_difference_report(json1, json2, utest=False): have_optimal_repetitions, cpu_pvalue, time_pvalue = calc_utest( timings_cpu, timings_time ) - if cpu_pvalue and time_pvalue: + if cpu_pvalue is not None and time_pvalue is not None: utest_results = { "have_optimal_repetitions": have_optimal_repetitions, "cpu_pvalue": cpu_pvalue, @@ -313,16 +330,20 @@ def get_difference_report(json1, json2, utest=False): # benchmark suite. if measurements: run_type = ( - partition[0][0]["run_type"] if "run_type" in partition[0][0] else "" + partition[0][0]["run_type"] + if "run_type" in partition[0][0] + else "" ) aggregate_name = ( partition[0][0]["aggregate_name"] - if run_type == "aggregate" and "aggregate_name" in partition[0][0] + if run_type == "aggregate" + and "aggregate_name" in partition[0][0] else "" ) diff_report.append( { "name": benchmark_name, + "label": label, "measurements": measurements, "time_unit": time_unit, "run_type": run_type, @@ -337,6 +358,7 @@ def get_difference_report(json1, json2, utest=False): diff_report.append( { "name": "OVERALL_GEOMEAN", + "label": "", "measurements": [ { "real_time": lhs_gmean[0], @@ -392,7 +414,7 @@ def print_difference_report( # and if it is non-aggregate, then don't print it. if ( not include_aggregates_only - or not "run_type" in benchmark + or "run_type" not in benchmark or benchmark["run_type"] == "aggregate" ): for measurement in benchmark["measurements"]: @@ -438,7 +460,9 @@ class TestGetUniqueBenchmarkNames(unittest.TestCase): def load_results(self): import json - testInputs = os.path.join(os.path.dirname(os.path.realpath(__file__)), "Inputs") + testInputs = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "Inputs" + ) testOutput = os.path.join(testInputs, "test3_run0.json") with open(testOutput, "r") as f: json = json.load(f) @@ -485,16 +509,73 @@ class TestReportDifference(unittest.TestCase): ["BM_SameTimes", "+0.0000", "+0.0000", "10", "10", "10", "10"], ["BM_2xFaster", "-0.5000", "-0.5000", "50", "25", "50", "25"], ["BM_2xSlower", "+1.0000", "+1.0000", "50", "100", "50", "100"], - ["BM_1PercentFaster", "-0.0100", "-0.0100", "100", "99", "100", "99"], - ["BM_1PercentSlower", "+0.0100", "+0.0100", "100", "101", "100", "101"], - ["BM_10PercentFaster", "-0.1000", "-0.1000", "100", "90", "100", "90"], - ["BM_10PercentSlower", "+0.1000", "+0.1000", "100", "110", "100", "110"], - ["BM_100xSlower", "+99.0000", "+99.0000", "100", "10000", "100", "10000"], - ["BM_100xFaster", "-0.9900", "-0.9900", "10000", "100", "10000", "100"], - ["BM_10PercentCPUToTime", "+0.1000", "-0.1000", "100", "110", "100", "90"], + [ + "BM_1PercentFaster", + "-0.0100", + "-0.0100", + "100", + "99", + "100", + "99", + ], + [ + "BM_1PercentSlower", + "+0.0100", + "+0.0100", + "100", + "101", + "100", + "101", + ], + [ + "BM_10PercentFaster", + "-0.1000", + "-0.1000", + "100", + "90", + "100", + "90", + ], + [ + "BM_10PercentSlower", + "+0.1000", + "+0.1000", + "100", + "110", + "100", + "110", + ], + [ + "BM_100xSlower", + "+99.0000", + "+99.0000", + "100", + "10000", + "100", + "10000", + ], + [ + "BM_100xFaster", + "-0.9900", + "-0.9900", + "10000", + "100", + "10000", + "100", + ], + [ + "BM_10PercentCPUToTime", + "+0.1000", + "-0.1000", + "100", + "110", + "100", + "90", + ], ["BM_ThirdFaster", "-0.3333", "-0.3334", "100", "67", "100", "67"], ["BM_NotBadTimeUnit", "-0.9000", "+0.2000", "0", "0", "0", "1"], - ["OVERALL_GEOMEAN", "-0.8344", "-0.8026", "0", "0", "0", "0"], + ["BM_hasLabel", "+0.0000", "+0.0000", "1", "1", "1", "1"], + ["OVERALL_GEOMEAN", "-0.8113", "-0.7779", "0", "0", "0", "0"], ] output_lines_with_header = print_difference_report( self.json_diff_report, use_color=False @@ -512,6 +593,7 @@ class TestReportDifference(unittest.TestCase): expected_output = [ { "name": "BM_SameTimes", + "label": "", "measurements": [ { "time": 0.0000, @@ -527,6 +609,7 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_2xFaster", + "label": "", "measurements": [ { "time": -0.5000, @@ -542,6 +625,7 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_2xSlower", + "label": "", "measurements": [ { "time": 1.0000, @@ -557,6 +641,7 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_1PercentFaster", + "label": "", "measurements": [ { "time": -0.0100, @@ -572,6 +657,7 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_1PercentSlower", + "label": "", "measurements": [ { "time": 0.0100, @@ -587,6 +673,7 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_10PercentFaster", + "label": "", "measurements": [ { "time": -0.1000, @@ -602,6 +689,7 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_10PercentSlower", + "label": "", "measurements": [ { "time": 0.1000, @@ -617,6 +705,7 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_100xSlower", + "label": "", "measurements": [ { "time": 99.0000, @@ -632,6 +721,7 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_100xFaster", + "label": "", "measurements": [ { "time": -0.9900, @@ -647,6 +737,7 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_10PercentCPUToTime", + "label": "", "measurements": [ { "time": 0.1000, @@ -662,6 +753,7 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_ThirdFaster", + "label": "", "measurements": [ { "time": -0.3333, @@ -677,6 +769,7 @@ class TestReportDifference(unittest.TestCase): }, { "name": "BM_NotBadTimeUnit", + "label": "", "measurements": [ { "time": -0.9000, @@ -690,16 +783,33 @@ class TestReportDifference(unittest.TestCase): "time_unit": "s", "utest": {}, }, + { + "name": "BM_hasLabel", + "label": "a label", + "measurements": [ + { + "time": 0.0000, + "cpu": 0.0000, + "real_time": 1, + "real_time_other": 1, + "cpu_time": 1, + "cpu_time_other": 1, + } + ], + "time_unit": "s", + "utest": {}, + }, { "name": "OVERALL_GEOMEAN", + "label": "", "measurements": [ { - "real_time": 1.193776641714438e-06, - "cpu_time": 1.2144445585302297e-06, + "real_time": 3.1622776601683826e-06, + "cpu_time": 3.2130844755623912e-06, "real_time_other": 1.9768988699420897e-07, "cpu_time_other": 2.397447755209533e-07, - "time": -0.834399601997324, - "cpu": -0.8025889499549471, + "time": -0.8112976497120911, + "cpu": -0.7778551721181174, } ], "time_unit": "s", @@ -711,6 +821,7 @@ class TestReportDifference(unittest.TestCase): self.assertEqual(len(self.json_diff_report), len(expected_output)) for out, expected in zip(self.json_diff_report, expected_output): self.assertEqual(out["name"], expected["name"]) + self.assertEqual(out["label"], expected["label"]) self.assertEqual(out["time_unit"], expected["time_unit"]) assert_utest(self, out, expected) assert_measurements(self, out, expected) @@ -1086,7 +1197,9 @@ class TestReportDifferenceWithUTest(unittest.TestCase): assert_measurements(self, out, expected) -class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly(unittest.TestCase): +class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly( + unittest.TestCase +): @classmethod def setUpClass(cls): def load_results(): @@ -1369,12 +1482,108 @@ class TestReportSorting(unittest.TestCase): for n in range(len(self.json["benchmarks"]) ** 2): random.shuffle(self.json["benchmarks"]) - sorted_benchmarks = util.sort_benchmark_results(self.json)["benchmarks"] + sorted_benchmarks = util.sort_benchmark_results(self.json)[ + "benchmarks" + ] self.assertEqual(len(expected_names), len(sorted_benchmarks)) for out, expected in zip(sorted_benchmarks, expected_names): self.assertEqual(out["name"], expected) +class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly2( + unittest.TestCase +): + @classmethod + def setUpClass(cls): + def load_results(): + import json + + testInputs = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "Inputs" + ) + testOutput1 = os.path.join(testInputs, "test5_run0.json") + testOutput2 = os.path.join(testInputs, "test5_run1.json") + with open(testOutput1, "r") as f: + json1 = json.load(f) + json1["benchmarks"] = [ + json1["benchmarks"][0] for i in range(1000) + ] + with open(testOutput2, "r") as f: + json2 = json.load(f) + json2["benchmarks"] = [ + json2["benchmarks"][0] for i in range(1000) + ] + return json1, json2 + + json1, json2 = load_results() + cls.json_diff_report = get_difference_report(json1, json2, utest=True) + + def test_json_diff_report_pretty_printing(self): + expect_line = [ + "BM_ManyRepetitions_pvalue", + "0.0000", + "0.0000", + "U", + "Test,", + "Repetitions:", + "1000", + "vs", + "1000", + ] + output_lines_with_header = print_difference_report( + self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False + ) + output_lines = output_lines_with_header[2:] + found = False + for i in range(0, len(output_lines)): + parts = [x for x in output_lines[i].split(" ") if x] + found = expect_line == parts + if found: + break + self.assertTrue(found) + + def test_json_diff_report(self): + expected_output = [ + { + "name": "BM_ManyRepetitions", + "label": "", + "time_unit": "s", + "run_type": "", + "aggregate_name": "", + "utest": { + "have_optimal_repetitions": True, + "cpu_pvalue": 0.0, + "time_pvalue": 0.0, + "nr_of_repetitions": 1000, + "nr_of_repetitions_other": 1000, + }, + }, + { + "name": "OVERALL_GEOMEAN", + "label": "", + "measurements": [ + { + "real_time": 1.0, + "cpu_time": 1000.000000000069, + "real_time_other": 1000.000000000069, + "cpu_time_other": 1.0, + "time": 999.000000000069, + "cpu": -0.9990000000000001, + } + ], + "time_unit": "s", + "run_type": "aggregate", + "aggregate_name": "geomean", + "utest": {}, + }, + ] + self.assertEqual(len(self.json_diff_report), len(expected_output)) + for out, expected in zip(self.json_diff_report, expected_output): + self.assertEqual(out["name"], expected["name"]) + self.assertEqual(out["time_unit"], expected["time_unit"]) + assert_utest(self, out, expected) + + def assert_utest(unittest_instance, lhs, rhs): if lhs["utest"]: unittest_instance.assertAlmostEqual( diff --git a/third-party/benchmark/tools/gbench/util.py b/third-party/benchmark/tools/gbench/util.py index a46903a6d248..4d061a3a1e34 100644 --- a/third-party/benchmark/tools/gbench/util.py +++ b/third-party/benchmark/tools/gbench/util.py @@ -2,10 +2,10 @@ """ import json import os -import tempfile +import re import subprocess import sys -import functools +import tempfile # Input file type enumeration IT_Invalid = 0 @@ -58,7 +58,7 @@ def classify_input_file(filename): """ Return a tuple (type, msg) where 'type' specifies the classified type of 'filename'. If 'type' is 'IT_Invalid' then 'msg' is a human readable - string represeting the error. + string representing the error. """ ftype = IT_Invalid err_msg = None @@ -72,7 +72,8 @@ def classify_input_file(filename): ftype = IT_JSON else: err_msg = ( - "'%s' does not name a valid benchmark executable or JSON file" % filename + "'%s' does not name a valid benchmark executable or JSON file" + % filename ) return ftype, err_msg @@ -113,13 +114,41 @@ def remove_benchmark_flags(prefix, benchmark_flags): return [f for f in benchmark_flags if not f.startswith(prefix)] -def load_benchmark_results(fname): +def load_benchmark_results(fname, benchmark_filter): """ Read benchmark output from a file and return the JSON object. + + Apply benchmark_filter, a regular expression, with nearly the same + semantics of the --benchmark_filter argument. May be None. + Note: the Python regular expression engine is used instead of the + one used by the C++ code, which may produce different results + in complex cases. + REQUIRES: 'fname' names a file containing JSON benchmark output. """ + + def benchmark_wanted(benchmark): + if benchmark_filter is None: + return True + name = benchmark.get("run_name", None) or benchmark["name"] + return re.search(benchmark_filter, name) is not None + with open(fname, "r") as f: - return json.load(f) + results = json.load(f) + if "context" in results: + if "json_schema_version" in results["context"]: + json_schema_version = results["context"]["json_schema_version"] + if json_schema_version != 1: + print( + "In %s, got unnsupported JSON schema version: %i, expected 1" + % (fname, json_schema_version) + ) + sys.exit(1) + if "benchmarks" in results: + results["benchmarks"] = list( + filter(benchmark_wanted, results["benchmarks"]) + ) + return results def sort_benchmark_results(result): @@ -168,7 +197,9 @@ def run_benchmark(exe_name, benchmark_flags): is_temp_output = True thandle, output_name = tempfile.mkstemp() os.close(thandle) - benchmark_flags = list(benchmark_flags) + ["--benchmark_out=%s" % output_name] + benchmark_flags = list(benchmark_flags) + [ + "--benchmark_out=%s" % output_name + ] cmd = [exe_name] + benchmark_flags print("RUNNING: %s" % " ".join(cmd)) @@ -176,7 +207,7 @@ def run_benchmark(exe_name, benchmark_flags): if exitCode != 0: print("TEST FAILED...") sys.exit(exitCode) - json_res = load_benchmark_results(output_name) + json_res = load_benchmark_results(output_name, None) if is_temp_output: os.unlink(output_name) return json_res @@ -191,7 +222,10 @@ def run_or_load_benchmark(filename, benchmark_flags): """ ftype = check_input_file(filename) if ftype == IT_JSON: - return load_benchmark_results(filename) + benchmark_filter = find_benchmark_flag( + "--benchmark_filter=", benchmark_flags + ) + return load_benchmark_results(filename, benchmark_filter) if ftype == IT_Executable: return run_benchmark(filename, benchmark_flags) raise ValueError("Unknown file type %s" % ftype) diff --git a/third-party/benchmark/tools/libpfm.BUILD.bazel b/third-party/benchmark/tools/libpfm.BUILD.bazel new file mode 100644 index 000000000000..62695342aa81 --- /dev/null +++ b/third-party/benchmark/tools/libpfm.BUILD.bazel @@ -0,0 +1,22 @@ +# Build rule for libpfm, which is required to collect performance counters for +# BENCHMARK_ENABLE_LIBPFM builds. + +load("@rules_foreign_cc//foreign_cc:defs.bzl", "make") + +filegroup( + name = "pfm_srcs", + srcs = glob(["**"]), +) + +make( + name = "libpfm", + lib_source = ":pfm_srcs", + lib_name = "libpfm", + copts = [ + "-Wno-format-truncation", + "-Wno-use-after-free", + ], + visibility = [ + "//visibility:public", + ], +) diff --git a/third-party/benchmark/tools/requirements.txt b/third-party/benchmark/tools/requirements.txt index 3b3331b5af12..f32f35b8fbfd 100644 --- a/third-party/benchmark/tools/requirements.txt +++ b/third-party/benchmark/tools/requirements.txt @@ -1 +1,2 @@ -scipy>=1.5.0 \ No newline at end of file +numpy == 1.25 +scipy == 1.10.0 diff --git a/third-party/benchmark/tools/strip_asm.py b/third-party/benchmark/tools/strip_asm.py index 086255dc6577..bc3a774a7932 100755 --- a/third-party/benchmark/tools/strip_asm.py +++ b/third-party/benchmark/tools/strip_asm.py @@ -1,20 +1,20 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ strip_asm.py - Cleanup ASM output for the specified file """ -from argparse import ArgumentParser -import sys import os import re +import sys +from argparse import ArgumentParser def find_used_labels(asm): found = set() - label_re = re.compile("\s*j[a-z]+\s+\.L([a-zA-Z0-9][a-zA-Z0-9_]*)") - for l in asm.splitlines(): - m = label_re.match(l) + label_re = re.compile(r"\s*j[a-z]+\s+\.L([a-zA-Z0-9][a-zA-Z0-9_]*)") + for line in asm.splitlines(): + m = label_re.match(line) if m: found.add(".L%s" % m.group(1)) return found @@ -23,8 +23,8 @@ def find_used_labels(asm): def normalize_labels(asm): decls = set() label_decl = re.compile("^[.]{0,1}L([a-zA-Z0-9][a-zA-Z0-9_]*)(?=:)") - for l in asm.splitlines(): - m = label_decl.match(l) + for line in asm.splitlines(): + m = label_decl.match(line) if m: decls.add(m.group(0)) if len(decls) == 0: @@ -33,7 +33,7 @@ def normalize_labels(asm): if not needs_dot: return asm for ld in decls: - asm = re.sub("(^|\s+)" + ld + "(?=:|\s)", "\\1." + ld, asm) + asm = re.sub(r"(^|\s+)" + ld + r"(?=:|\s)", "\\1." + ld, asm) return asm @@ -41,11 +41,11 @@ def transform_labels(asm): asm = normalize_labels(asm) used_decls = find_used_labels(asm) new_asm = "" - label_decl = re.compile("^\.L([a-zA-Z0-9][a-zA-Z0-9_]*)(?=:)") - for l in asm.splitlines(): - m = label_decl.match(l) + label_decl = re.compile(r"^\.L([a-zA-Z0-9][a-zA-Z0-9_]*)(?=:)") + for line in asm.splitlines(): + m = label_decl.match(line) if not m or m.group(0) in used_decls: - new_asm += l + new_asm += line new_asm += "\n" return new_asm @@ -63,21 +63,24 @@ def is_identifier(tk): return True -def process_identifiers(l): +def process_identifiers(line): """ process_identifiers - process all identifiers and modify them to have consistent names across all platforms; specifically across ELF and MachO. For example, MachO inserts an additional understore at the beginning of names. This function removes that. """ - parts = re.split(r"([a-zA-Z0-9_]+)", l) + parts = re.split(r"([a-zA-Z0-9_]+)", line) new_line = "" for tk in parts: if is_identifier(tk): if tk.startswith("__Z"): tk = tk[1:] elif ( - tk.startswith("_") and len(tk) > 1 and tk[1].isalpha() and tk[1] != "Z" + tk.startswith("_") + and len(tk) > 1 + and tk[1].isalpha() + and tk[1] != "Z" ): tk = tk[1:] new_line += tk @@ -93,33 +96,35 @@ def process_asm(asm): # TODO: Add more things we want to remove discard_regexes = [ - re.compile("\s+\..*$"), # directive - re.compile("\s*#(NO_APP|APP)$"), # inline ASM - re.compile("\s*#.*$"), # comment line - re.compile("\s*\.globa?l\s*([.a-zA-Z_][a-zA-Z0-9$_.]*)"), # global directive + re.compile(r"\s+\..*$"), # directive + re.compile(r"\s*#(NO_APP|APP)$"), # inline ASM + re.compile(r"\s*#.*$"), # comment line + re.compile( + r"\s*\.globa?l\s*([.a-zA-Z_][a-zA-Z0-9$_.]*)" + ), # global directive re.compile( - "\s*\.(string|asciz|ascii|[1248]?byte|short|word|long|quad|value|zero)" + r"\s*\.(string|asciz|ascii|[1248]?byte|short|word|long|quad|value|zero)" ), ] - keep_regexes = [] + keep_regexes: list[re.Pattern] = [] fn_label_def = re.compile("^[a-zA-Z_][a-zA-Z0-9_.]*:") - for l in asm.splitlines(): + for line in asm.splitlines(): # Remove Mach-O attribute - l = l.replace("@GOTPCREL", "") + line = line.replace("@GOTPCREL", "") add_line = True for reg in discard_regexes: - if reg.match(l) is not None: + if reg.match(line) is not None: add_line = False break for reg in keep_regexes: - if reg.match(l) is not None: + if reg.match(line) is not None: add_line = True break if add_line: - if fn_label_def.match(l) and len(new_contents) != 0: + if fn_label_def.match(line) and len(new_contents) != 0: new_contents += "\n" - l = process_identifiers(l) - new_contents += l + line = process_identifiers(line) + new_contents += line new_contents += "\n" return new_contents @@ -127,7 +132,11 @@ def process_asm(asm): def main(): parser = ArgumentParser(description="generate a stripped assembly file") parser.add_argument( - "input", metavar="input", type=str, nargs=1, help="An input assembly file" + "input", + metavar="input", + type=str, + nargs=1, + help="An input assembly file", ) parser.add_argument( "out", metavar="output", type=str, nargs=1, help="The output file" @@ -136,9 +145,9 @@ def main(): input = args.input[0] output = args.out[0] if not os.path.isfile(input): - print(("ERROR: input file '%s' does not exist") % input) + print("ERROR: input file '%s' does not exist" % input) sys.exit(1) - contents = None + with open(input, "r") as f: contents = f.read() new_contents = process_asm(contents) -- GitLab From 8848258f7bf67fbaf49213a2fb7edb8a3c2f0493 Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Mon, 4 Mar 2024 14:21:26 -0800 Subject: [PATCH 054/929] [SROA] Unfold gep of index phi (round 2) (#83494) If a gep has only one phi as one of its operands and the remaining indexes are constant, we can unfold `gep ptr, (phi idx1, idx2)` to `phi ((gep ptr, idx1), (gep ptr, idx2))`. Take care not to unfold recursive phis. Followup to #80983. This was initially was #83087. Initial PR did not handle allocas in entry block that weren't at the beginning of the function, causing GEPs to be inserted after the first chunk of allocas but potentially before an alloca not at the beginning. Insert GEPs at the end of the entry block instead since constants/arguments/static allocas can all be used there. --- llvm/lib/Transforms/Scalar/SROA.cpp | 115 +++++++---- llvm/test/Transforms/SROA/phi-and-select.ll | 20 +- llvm/test/Transforms/SROA/phi-gep.ll | 210 ++++++++++++++++++-- 3 files changed, 278 insertions(+), 67 deletions(-) diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp index fad70e8bf286..b2de2209b3f1 100644 --- a/llvm/lib/Transforms/Scalar/SROA.cpp +++ b/llvm/lib/Transforms/Scalar/SROA.cpp @@ -3956,11 +3956,11 @@ private: return false; } - // Fold gep (select cond, ptr1, ptr2), idx + // Unfold gep (select cond, ptr1, ptr2), idx // => select cond, gep(ptr1, idx), gep(ptr2, idx) // and gep ptr, (select cond, idx1, idx2) // => select cond, gep(ptr, idx1), gep(ptr, idx2) - bool foldGEPSelect(GetElementPtrInst &GEPI) { + bool unfoldGEPSelect(GetElementPtrInst &GEPI) { // Check whether the GEP has exactly one select operand and all indices // will become constant after the transform. SelectInst *Sel = dyn_cast(GEPI.getPointerOperand()); @@ -4029,67 +4029,104 @@ private: return true; } - // Fold gep (phi ptr1, ptr2) => phi gep(ptr1), gep(ptr2) - bool foldGEPPhi(GetElementPtrInst &GEPI) { - if (!GEPI.hasAllConstantIndices()) - return false; + // Unfold gep (phi ptr1, ptr2), idx + // => phi ((gep ptr1, idx), (gep ptr2, idx)) + // and gep ptr, (phi idx1, idx2) + // => phi ((gep ptr, idx1), (gep ptr, idx2)) + bool unfoldGEPPhi(GetElementPtrInst &GEPI) { + // To prevent infinitely expanding recursive phis, bail if the GEP pointer + // operand (looking through the phi if it is the phi we want to unfold) is + // an instruction besides an alloca. + PHINode *Phi = dyn_cast(GEPI.getPointerOperand()); + auto IsInvalidPointerOperand = [](Value *V) { + return isa(V) && !isa(V); + }; + if (Phi) { + if (any_of(Phi->operands(), IsInvalidPointerOperand)) + return false; + } else { + if (IsInvalidPointerOperand(GEPI.getPointerOperand())) + return false; + } + // Check whether the GEP has exactly one phi operand (including the pointer + // operand) and all indices will become constant after the transform. + for (Value *Op : GEPI.indices()) { + if (auto *SI = dyn_cast(Op)) { + if (Phi) + return false; + + Phi = SI; + if (!all_of(Phi->incoming_values(), + [](Value *V) { return isa(V); })) + return false; + continue; + } - PHINode *PHI = cast(GEPI.getPointerOperand()); - if (GEPI.getParent() != PHI->getParent() || - llvm::any_of(PHI->incoming_values(), [](Value *In) { - Instruction *I = dyn_cast(In); - return !I || isa(I) || isa(I) || - succ_empty(I->getParent()) || - !I->getParent()->isLegalToHoistInto(); - })) + if (!isa(Op)) + return false; + } + + if (!Phi) return false; LLVM_DEBUG(dbgs() << " Rewriting gep(phi) -> phi(gep):\n"; - dbgs() << " original: " << *PHI << "\n"; + dbgs() << " original: " << *Phi << "\n"; dbgs() << " " << GEPI << "\n";); - SmallVector Index(GEPI.indices()); + auto GetNewOps = [&](Value *PhiOp) { + SmallVector NewOps; + for (Value *Op : GEPI.operands()) + if (Op == Phi) + NewOps.push_back(PhiOp); + else + NewOps.push_back(Op); + return NewOps; + }; + + IRB.SetInsertPoint(Phi); + PHINode *NewPhi = IRB.CreatePHI(GEPI.getType(), Phi->getNumIncomingValues(), + Phi->getName() + ".sroa.phi"); + bool IsInBounds = GEPI.isInBounds(); - IRB.SetInsertPoint(GEPI.getParent(), GEPI.getParent()->getFirstNonPHIIt()); - PHINode *NewPN = IRB.CreatePHI(GEPI.getType(), PHI->getNumIncomingValues(), - PHI->getName() + ".sroa.phi"); - for (unsigned I = 0, E = PHI->getNumIncomingValues(); I != E; ++I) { - BasicBlock *B = PHI->getIncomingBlock(I); - Value *NewVal = nullptr; - int Idx = NewPN->getBasicBlockIndex(B); - if (Idx >= 0) { - NewVal = NewPN->getIncomingValue(Idx); + Type *SourceTy = GEPI.getSourceElementType(); + // We only handle arguments, constants, and static allocas here, so we can + // insert GEPs at the end of the entry block. + IRB.SetInsertPoint(GEPI.getFunction()->getEntryBlock().getTerminator()); + for (unsigned I = 0, E = Phi->getNumIncomingValues(); I != E; ++I) { + Value *Op = Phi->getIncomingValue(I); + BasicBlock *BB = Phi->getIncomingBlock(I); + Value *NewGEP; + if (int NI = NewPhi->getBasicBlockIndex(BB); NI >= 0) { + NewGEP = NewPhi->getIncomingValue(NI); } else { - Instruction *In = cast(PHI->getIncomingValue(I)); - - IRB.SetInsertPoint(In->getParent(), std::next(In->getIterator())); - Type *Ty = GEPI.getSourceElementType(); - NewVal = IRB.CreateGEP(Ty, In, Index, In->getName() + ".sroa.gep", - IsInBounds); + SmallVector NewOps = GetNewOps(Op); + NewGEP = + IRB.CreateGEP(SourceTy, NewOps[0], ArrayRef(NewOps).drop_front(), + Phi->getName() + ".sroa.gep", IsInBounds); } - NewPN->addIncoming(NewVal, B); + NewPhi->addIncoming(NewGEP, BB); } Visited.erase(&GEPI); - GEPI.replaceAllUsesWith(NewPN); + GEPI.replaceAllUsesWith(NewPhi); GEPI.eraseFromParent(); - Visited.insert(NewPN); - enqueueUsers(*NewPN); + Visited.insert(NewPhi); + enqueueUsers(*NewPhi); LLVM_DEBUG(dbgs() << " to: "; for (Value *In - : NewPN->incoming_values()) dbgs() + : NewPhi->incoming_values()) dbgs() << "\n " << *In; - dbgs() << "\n " << *NewPN << '\n'); + dbgs() << "\n " << *NewPhi << '\n'); return true; } bool visitGetElementPtrInst(GetElementPtrInst &GEPI) { - if (foldGEPSelect(GEPI)) + if (unfoldGEPSelect(GEPI)) return true; - if (isa(GEPI.getPointerOperand()) && foldGEPPhi(GEPI)) + if (unfoldGEPPhi(GEPI)) return true; enqueueUsers(GEPI); diff --git a/llvm/test/Transforms/SROA/phi-and-select.ll b/llvm/test/Transforms/SROA/phi-and-select.ll index 54cfb10793a1..7c8b27c9de9c 100644 --- a/llvm/test/Transforms/SROA/phi-and-select.ll +++ b/llvm/test/Transforms/SROA/phi-and-select.ll @@ -114,13 +114,13 @@ define i32 @test3(i32 %x) { ; CHECK-LABEL: @test3( ; CHECK-NEXT: entry: ; CHECK-NEXT: switch i32 [[X:%.*]], label [[BB0:%.*]] [ -; CHECK-NEXT: i32 1, label [[BB1:%.*]] -; CHECK-NEXT: i32 2, label [[BB2:%.*]] -; CHECK-NEXT: i32 3, label [[BB3:%.*]] -; CHECK-NEXT: i32 4, label [[BB4:%.*]] -; CHECK-NEXT: i32 5, label [[BB5:%.*]] -; CHECK-NEXT: i32 6, label [[BB6:%.*]] -; CHECK-NEXT: i32 7, label [[BB7:%.*]] +; CHECK-NEXT: i32 1, label [[BB1:%.*]] +; CHECK-NEXT: i32 2, label [[BB2:%.*]] +; CHECK-NEXT: i32 3, label [[BB3:%.*]] +; CHECK-NEXT: i32 4, label [[BB4:%.*]] +; CHECK-NEXT: i32 5, label [[BB5:%.*]] +; CHECK-NEXT: i32 6, label [[BB6:%.*]] +; CHECK-NEXT: i32 7, label [[BB7:%.*]] ; CHECK-NEXT: ] ; CHECK: bb0: ; CHECK-NEXT: br label [[EXIT:%.*]] @@ -733,6 +733,7 @@ define void @PR20822(i1 %c1, i1 %c2, ptr %ptr) { ; CHECK-LABEL: @PR20822( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[F_SROA_0:%.*]] = alloca i32, align 4 +; CHECK-NEXT: [[F1_SROA_GEP:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], ptr [[PTR:%.*]], i32 0, i32 0 ; CHECK-NEXT: br i1 [[C1:%.*]], label [[IF_END:%.*]], label [[FOR_COND:%.*]] ; CHECK: for.cond: ; CHECK-NEXT: br label [[IF_END]] @@ -742,9 +743,8 @@ define void @PR20822(i1 %c1, i1 %c2, ptr %ptr) { ; CHECK: if.then2: ; CHECK-NEXT: br label [[IF_THEN5]] ; CHECK: if.then5: -; CHECK-NEXT: [[F1:%.*]] = phi ptr [ [[PTR:%.*]], [[IF_THEN2]] ], [ [[F_SROA_0]], [[IF_END]] ] -; CHECK-NEXT: [[DOTFCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], ptr [[F1]], i32 0, i32 0 -; CHECK-NEXT: store i32 0, ptr [[DOTFCA_0_GEP]], align 4 +; CHECK-NEXT: [[F1_SROA_PHI:%.*]] = phi ptr [ [[F1_SROA_GEP]], [[IF_THEN2]] ], [ [[F_SROA_0]], [[IF_END]] ] +; CHECK-NEXT: store i32 0, ptr [[F1_SROA_PHI]], align 4 ; CHECK-NEXT: ret void ; entry: diff --git a/llvm/test/Transforms/SROA/phi-gep.ll b/llvm/test/Transforms/SROA/phi-gep.ll index c5aa1cdd9cf6..33f42af69608 100644 --- a/llvm/test/Transforms/SROA/phi-gep.ll +++ b/llvm/test/Transforms/SROA/phi-gep.ll @@ -65,15 +65,13 @@ end: define i32 @test_sroa_phi_gep_poison(i1 %cond) { ; CHECK-LABEL: @test_sroa_phi_gep_poison( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[A:%.*]] = alloca [[PAIR:%.*]], align 4 ; CHECK-NEXT: br i1 [[COND:%.*]], label [[IF_THEN:%.*]], label [[END:%.*]] ; CHECK: if.then: +; CHECK-NEXT: [[PHI_SROA_PHI_SROA_SPECULATE_LOAD_IF_THEN:%.*]] = load i32, ptr poison, align 4 ; CHECK-NEXT: br label [[END]] ; CHECK: end: -; CHECK-NEXT: [[PHI:%.*]] = phi ptr [ [[A]], [[ENTRY:%.*]] ], [ poison, [[IF_THEN]] ] -; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds [[PAIR]], ptr [[PHI]], i32 0, i32 1 -; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP]], align 4 -; CHECK-NEXT: ret i32 [[LOAD]] +; CHECK-NEXT: [[PHI_SROA_PHI_SROA_SPECULATED:%.*]] = phi i32 [ undef, [[ENTRY:%.*]] ], [ [[PHI_SROA_PHI_SROA_SPECULATE_LOAD_IF_THEN]], [[IF_THEN]] ] +; CHECK-NEXT: ret i32 [[PHI_SROA_PHI_SROA_SPECULATED]] ; entry: %a = alloca %pair, align 4 @@ -94,17 +92,13 @@ end: define i32 @test_sroa_phi_gep_global(i1 %cond) { ; CHECK-LABEL: @test_sroa_phi_gep_global( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[A:%.*]] = alloca [[PAIR:%.*]], align 4 -; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds [[PAIR]], ptr [[A]], i32 0, i32 1 -; CHECK-NEXT: store i32 1, ptr [[GEP_A]], align 4 ; CHECK-NEXT: br i1 [[COND:%.*]], label [[IF_THEN:%.*]], label [[END:%.*]] ; CHECK: if.then: +; CHECK-NEXT: [[PHI_SROA_PHI_SROA_SPECULATE_LOAD_IF_THEN:%.*]] = load i32, ptr getelementptr inbounds ([[PAIR:%.*]], ptr @g, i32 0, i32 1), align 4 ; CHECK-NEXT: br label [[END]] ; CHECK: end: -; CHECK-NEXT: [[PHI:%.*]] = phi ptr [ [[A]], [[ENTRY:%.*]] ], [ @g, [[IF_THEN]] ] -; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds [[PAIR]], ptr [[PHI]], i32 0, i32 1 -; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP]], align 4 -; CHECK-NEXT: ret i32 [[LOAD]] +; CHECK-NEXT: [[PHI_SROA_PHI_SROA_SPECULATED:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[PHI_SROA_PHI_SROA_SPECULATE_LOAD_IF_THEN]], [[IF_THEN]] ] +; CHECK-NEXT: ret i32 [[PHI_SROA_PHI_SROA_SPECULATED]] ; entry: %a = alloca %pair, align 4 @@ -245,7 +239,7 @@ define i32 @test_sroa_invoke_phi_gep(i1 %cond) personality ptr @__gxx_personalit ; CHECK-NEXT: br i1 [[COND:%.*]], label [[CALL:%.*]], label [[END:%.*]] ; CHECK: call: ; CHECK-NEXT: [[B:%.*]] = invoke ptr @foo() -; CHECK-NEXT: to label [[END]] unwind label [[INVOKE_CATCH:%.*]] +; CHECK-NEXT: to label [[END]] unwind label [[INVOKE_CATCH:%.*]] ; CHECK: end: ; CHECK-NEXT: [[PHI:%.*]] = phi ptr [ [[A]], [[ENTRY:%.*]] ], [ [[B]], [[CALL]] ] ; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds [[PAIR]], ptr [[PHI]], i32 0, i32 1 @@ -253,7 +247,7 @@ define i32 @test_sroa_invoke_phi_gep(i1 %cond) personality ptr @__gxx_personalit ; CHECK-NEXT: ret i32 [[LOAD]] ; CHECK: invoke_catch: ; CHECK-NEXT: [[RES:%.*]] = landingpad { ptr, i32 } -; CHECK-NEXT: catch ptr null +; CHECK-NEXT: catch ptr null ; CHECK-NEXT: ret i32 0 ; entry: @@ -468,10 +462,10 @@ define i32 @test_sroa_phi_gep_multiple_values_from_same_block(i32 %arg) { ; CHECK-LABEL: @test_sroa_phi_gep_multiple_values_from_same_block( ; CHECK-NEXT: bb.1: ; CHECK-NEXT: switch i32 [[ARG:%.*]], label [[BB_3:%.*]] [ -; CHECK-NEXT: i32 1, label [[BB_2:%.*]] -; CHECK-NEXT: i32 2, label [[BB_2]] -; CHECK-NEXT: i32 3, label [[BB_4:%.*]] -; CHECK-NEXT: i32 4, label [[BB_4]] +; CHECK-NEXT: i32 1, label [[BB_2:%.*]] +; CHECK-NEXT: i32 2, label [[BB_2]] +; CHECK-NEXT: i32 3, label [[BB_4:%.*]] +; CHECK-NEXT: i32 4, label [[BB_4]] ; CHECK-NEXT: ] ; CHECK: bb.2: ; CHECK-NEXT: br label [[BB_4]] @@ -504,6 +498,186 @@ bb.4: ; preds = %bb.1, %bb.1, %bb ret i32 %load } +define i64 @test_phi_idx_mem2reg_const(i1 %arg) { +; CHECK-LABEL: @test_phi_idx_mem2reg_const( +; CHECK-NEXT: bb: +; CHECK-NEXT: br i1 [[ARG:%.*]], label [[BB1:%.*]], label [[BB2:%.*]] +; CHECK: bb1: +; CHECK-NEXT: br label [[END:%.*]] +; CHECK: bb2: +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[PHI_SROA_PHI_SROA_SPECULATED:%.*]] = phi i64 [ 2, [[BB1]] ], [ 3, [[BB2]] ] +; CHECK-NEXT: [[PHI:%.*]] = phi i64 [ 0, [[BB1]] ], [ 1, [[BB2]] ] +; CHECK-NEXT: ret i64 [[PHI_SROA_PHI_SROA_SPECULATED]] +; +bb: + %alloca = alloca [2 x i64], align 8 + %gep1 = getelementptr inbounds i64, ptr %alloca, i64 1 + store i64 2, ptr %alloca + store i64 3, ptr %gep1 + br i1 %arg, label %bb1, label %bb2 + +bb1: + br label %end + +bb2: + br label %end + +end: + %phi = phi i64 [ 0, %bb1 ], [ 1, %bb2 ] + %getelementptr = getelementptr inbounds i64, ptr %alloca, i64 %phi + %load = load i64, ptr %getelementptr + ret i64 %load +} + +define i64 @test_phi_idx_mem2reg_not_const(i1 %arg, i64 %idx) { +; CHECK-LABEL: @test_phi_idx_mem2reg_not_const( +; CHECK-NEXT: bb: +; CHECK-NEXT: [[ALLOCA:%.*]] = alloca [2 x i64], align 8 +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i64, ptr [[ALLOCA]], i64 1 +; CHECK-NEXT: store i64 2, ptr [[ALLOCA]], align 4 +; CHECK-NEXT: store i64 3, ptr [[GEP1]], align 4 +; CHECK-NEXT: br i1 [[ARG:%.*]], label [[BB1:%.*]], label [[BB2:%.*]] +; CHECK: bb1: +; CHECK-NEXT: br label [[END:%.*]] +; CHECK: bb2: +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[PHI:%.*]] = phi i64 [ 0, [[BB1]] ], [ [[IDX:%.*]], [[BB2]] ] +; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr inbounds i64, ptr [[ALLOCA]], i64 [[PHI]] +; CHECK-NEXT: [[LOAD:%.*]] = load i64, ptr [[GETELEMENTPTR]], align 4 +; CHECK-NEXT: ret i64 [[LOAD]] +; +bb: + %alloca = alloca [2 x i64], align 8 + %gep1 = getelementptr inbounds i64, ptr %alloca, i64 1 + store i64 2, ptr %alloca + store i64 3, ptr %gep1 + br i1 %arg, label %bb1, label %bb2 + +bb1: + br label %end + +bb2: + br label %end + +end: + %phi = phi i64 [ 0, %bb1 ], [ %idx, %bb2 ] + %getelementptr = getelementptr inbounds i64, ptr %alloca, i64 %phi + %load = load i64, ptr %getelementptr + ret i64 %load +} + +define i64 @test_phi_mem2reg_pointer_op_is_non_const_gep(i1 %arg, i64 %idx) { +; CHECK-LABEL: @test_phi_mem2reg_pointer_op_is_non_const_gep( +; CHECK-NEXT: bb: +; CHECK-NEXT: [[ALLOCA:%.*]] = alloca [2 x i64], align 8 +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i64, ptr [[ALLOCA]], i64 1 +; CHECK-NEXT: store i64 2, ptr [[ALLOCA]], align 4 +; CHECK-NEXT: store i64 3, ptr [[GEP1]], align 4 +; CHECK-NEXT: br i1 [[ARG:%.*]], label [[BB1:%.*]], label [[BB2:%.*]] +; CHECK: bb1: +; CHECK-NEXT: br label [[END:%.*]] +; CHECK: bb2: +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[PHI:%.*]] = phi i64 [ 0, [[BB1]] ], [ 1, [[BB2]] ] +; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr inbounds i64, ptr [[ALLOCA]], i64 [[IDX:%.*]] +; CHECK-NEXT: [[GETELEMENTPTR2:%.*]] = getelementptr inbounds i64, ptr [[GETELEMENTPTR]], i64 [[PHI]] +; CHECK-NEXT: [[LOAD:%.*]] = load i64, ptr [[GETELEMENTPTR]], align 4 +; CHECK-NEXT: ret i64 [[LOAD]] +; +bb: + %alloca = alloca [2 x i64], align 8 + %gep1 = getelementptr inbounds i64, ptr %alloca, i64 1 + store i64 2, ptr %alloca + store i64 3, ptr %gep1 + br i1 %arg, label %bb1, label %bb2 + +bb1: + br label %end + +bb2: + br label %end + +end: + %phi = phi i64 [ 0, %bb1 ], [ 1, %bb2 ] + %getelementptr = getelementptr inbounds i64, ptr %alloca, i64 %idx + %getelementptr2 = getelementptr inbounds i64, ptr %getelementptr, i64 %phi + %load = load i64, ptr %getelementptr + ret i64 %load +} + +define i1 @test_phi_mem2reg_entry_block_alloca_not_at_beginning(i1 %arg) { +; CHECK-LABEL: @test_phi_mem2reg_entry_block_alloca_not_at_beginning( +; CHECK-NEXT: bb: +; CHECK-NEXT: call void @f() +; CHECK-NEXT: [[ALLOCA:%.*]] = alloca i64, align 8 +; CHECK-NEXT: [[PHI_SROA_GEP:%.*]] = getelementptr i64, ptr [[ALLOCA]], i64 1 +; CHECK-NEXT: [[PHI_SROA_GEP1:%.*]] = getelementptr i64, ptr [[ALLOCA]], i64 2 +; CHECK-NEXT: br i1 [[ARG:%.*]], label [[BB2:%.*]], label [[BB3:%.*]] +; CHECK: bb2: +; CHECK-NEXT: br label [[BB3]] +; CHECK: bb3: +; CHECK-NEXT: [[PHI_SROA_PHI:%.*]] = phi ptr [ [[PHI_SROA_GEP]], [[BB:%.*]] ], [ [[PHI_SROA_GEP1]], [[BB2]] ] +; CHECK-NEXT: [[PHI:%.*]] = phi i64 [ 1, [[BB]] ], [ 2, [[BB2]] ] +; CHECK-NEXT: [[ICMP:%.*]] = icmp eq ptr [[PHI_SROA_PHI]], null +; CHECK-NEXT: ret i1 [[ICMP]] +; +bb: + call void @f() + %alloca = alloca i64 + br i1 %arg, label %bb2, label %bb3 +bb2: + br label %bb3 +bb3: + %phi = phi i64 [ 1, %bb ], [ 2, %bb2 ] + %gep = getelementptr i64, ptr %alloca, i64 %phi + %icmp = icmp eq ptr %gep, null + ret i1 %icmp +} + +define i64 @test_unfold_phi_duplicate_phi_entry(ptr %arg, i8 %arg1, i1 %arg2) { +; CHECK-LABEL: @test_unfold_phi_duplicate_phi_entry( +; CHECK-NEXT: bb: +; CHECK-NEXT: [[ALLOCA_SROA_0:%.*]] = alloca i64, align 8 +; CHECK-NEXT: [[PHI_SROA_GEP:%.*]] = getelementptr i64, ptr [[ARG:%.*]], i64 1 +; CHECK-NEXT: br i1 [[ARG2:%.*]], label [[BB5:%.*]], label [[BB3:%.*]] +; CHECK: bb3: +; CHECK-NEXT: switch i8 [[ARG1:%.*]], label [[BB4:%.*]] [ +; CHECK-NEXT: i8 0, label [[BB5]] +; CHECK-NEXT: i8 1, label [[BB5]] +; CHECK-NEXT: ] +; CHECK: bb4: +; CHECK-NEXT: ret i64 0 +; CHECK: bb5: +; CHECK-NEXT: [[PHI_SROA_PHI:%.*]] = phi ptr [ [[PHI_SROA_GEP]], [[BB3]] ], [ [[PHI_SROA_GEP]], [[BB3]] ], [ [[ALLOCA_SROA_0]], [[BB:%.*]] ] +; CHECK-NEXT: [[LOAD:%.*]] = load i64, ptr [[PHI_SROA_PHI]], align 4 +; CHECK-NEXT: ret i64 [[LOAD]] +; +bb: + %alloca = alloca [2 x i64], align 8 + br i1 %arg2, label %bb5, label %bb3 + +bb3: ; preds = %bb + switch i8 %arg1, label %bb4 [ + i8 0, label %bb5 + i8 1, label %bb5 + ] + +bb4: ; preds = %bb5, %bb3 + ret i64 0 + +bb5: ; preds = %bb3, %bb3, %bb + %phi = phi ptr [ %arg, %bb3 ], [ %arg, %bb3 ], [ %alloca, %bb ] + %getelementptr = getelementptr i64, ptr %phi, i64 1 + %load = load i64, ptr %getelementptr + ret i64 %load +} + +declare void @f() + declare ptr @foo() declare i32 @__gxx_personality_v0(...) -- GitLab From 691fc7cdcc3679c5fb7963147a9c358ff9019ab7 Mon Sep 17 00:00:00 2001 From: Aart Bik <39774503+aartbik@users.noreply.github.com> Date: Mon, 4 Mar 2024 14:32:49 -0800 Subject: [PATCH 055/929] [mlir][sparse] add dim/lvl information to sparse_tensor.print (#83913) More information is more testing! Also adjusts already migrated integration tests --- .../Transforms/SparseTensorRewriting.cpp | 28 +++++++++++++++++++ .../Dialect/SparseTensor/CPU/block.mlir | 6 ++++ .../SparseTensor/CPU/block_majors.mlir | 8 ++++++ .../SparseTensor/CPU/dense_output.mlir | 2 ++ .../SparseTensor/CPU/dense_output_bf16.mlir | 2 ++ .../SparseTensor/CPU/dense_output_f16.mlir | 2 ++ .../SparseTensor/CPU/sparse_insert_3d.mlir | 8 ++++++ .../SparseTensor/CPU/sparse_loose.mlir | 2 ++ .../SparseTensor/CPU/sparse_matmul.mlir | 28 +++++++++++++++++++ .../SparseTensor/CPU/sparse_matmul_slice.mlir | 10 +++++++ .../SparseTensor/CPU/sparse_matrix_ops.mlir | 12 ++++++++ .../SparseTensor/CPU/sparse_print.mlir | 24 ++++++++++++++++ .../SparseTensor/CPU/sparse_re_im.mlir | 4 +++ .../GPU/CUDA/sparse-gemm-lib.mlir | 2 ++ .../GPU/CUDA/sparse-sampled-matmul-lib.mlir | 4 +++ .../GPU/CUDA/sparse-sddmm-lib.mlir | 4 +++ 16 files changed, 146 insertions(+) diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp index 6ff21468e057..ddac602b9a43 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp @@ -616,6 +616,11 @@ public: rewriter.create( loc, rewriter.getStringAttr("---- Sparse Tensor ----\nnse = ")); rewriter.create(loc, nse); + // Print run-time contents for dim/lvl sizes. + rewriter.create(loc, rewriter.getStringAttr("dim = ")); + printSizes(rewriter, loc, tensor, stt.getDimRank(), /*isDim=*/true); + rewriter.create(loc, rewriter.getStringAttr("lvl = ")); + printSizes(rewriter, loc, tensor, stt.getLvlRank(), /*isDim=*/false); // Use the "codegen" foreach loop construct to iterate over // all typical sparse tensor components for printing. foreachFieldAndTypeInSparseTensor(stt, [&rewriter, &loc, &tensor, @@ -693,6 +698,29 @@ private: rewriter.create(loc, vector::PrintPunctuation::Close); rewriter.create(loc, vector::PrintPunctuation::NewLine); } + + // Helper method to print run-time lvl/dim sizes. + static void printSizes(PatternRewriter &rewriter, Location loc, Value tensor, + unsigned size, bool isDim) { + // Open bracket. + rewriter.create(loc, vector::PrintPunctuation::Open); + // Print unrolled contents (dimop requires constant value). + for (unsigned i = 0; i < size; i++) { + auto idx = constantIndex(rewriter, loc, i); + Value val; + if (isDim) + val = rewriter.create(loc, tensor, idx); + else + val = rewriter.create(loc, tensor, idx); + rewriter.create( + loc, val, + i != size - 1 ? vector::PrintPunctuation::Comma + : vector::PrintPunctuation::NoPunctuation); + } + // Close bracket and end of line. + rewriter.create(loc, vector::PrintPunctuation::Close); + rewriter.create(loc, vector::PrintPunctuation::NewLine); + } }; /// Sparse rewriting rule for sparse-to-sparse reshape operator. diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/block.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/block.mlir index 1184d407541b..f79e7e68f382 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/block.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/block.mlir @@ -91,6 +91,8 @@ module { // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 12 + // CHECK-NEXT: dim = ( 4, 6 ) + // CHECK-NEXT: lvl = ( 2, 3, 2, 2 ) // CHECK-NEXT: pos[1] : ( 0, 2, 3, // CHECK-NEXT: crd[1] : ( 0, 2, 1, // CHECK-NEXT: values : ( 1, 2, 0, 3, 4, 0, 0, 5, 6, 7, 8, 0, @@ -99,6 +101,8 @@ module { // CHECK-NEXT: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 12 + // CHECK-NEXT: dim = ( 2, 3, 2, 2 ) + // CHECK-NEXT: lvl = ( 2, 3, 2, 2 ) // CHECK-NEXT: pos[1] : ( 0, 2, 3, // CHECK-NEXT: crd[1] : ( 0, 2, 1 // CHECK-NEXT: values : ( 1, 2, 0, 3, 4, 0, 0, 5, 6, 7, 8, 0, @@ -109,6 +113,8 @@ module { // CHECK-NEXT: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 12 + // CHECK-NEXT: dim = ( 4, 6 ) + // CHECK-NEXT: lvl = ( 2, 3, 2, 2 ) // CHECK-NEXT: pos[1] : ( 0, 2, 3, // CHECK-NEXT: crd[1] : ( 0, 2, 1, // CHECK-NEXT: values : ( 3, 6, 0, 9, 12, 0, 0, 15, 18, 21, 24, 0, diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/block_majors.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/block_majors.mlir index f8e83b501967..3534e7d15207 100755 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/block_majors.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/block_majors.mlir @@ -106,6 +106,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 6, 16 ) + // CHECK-NEXT: lvl = ( 2, 4, 3, 4 ) // CHECK-NEXT: pos[1] : ( 0, 1, 2, // CHECK-NEXT: crd[1] : ( 0, 2, // CHECK-NEXT: values : ( 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, @@ -130,6 +132,8 @@ module { // // CHECK-NEXT: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 6, 16 ) + // CHECK-NEXT: lvl = ( 2, 4, 4, 3 ) // CHECK-NEXT: pos[1] : ( 0, 1, 2, // CHECK-NEXT: crd[1] : ( 0, 2, // CHECK-NEXT: values : ( 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 5, 0, 0, 0, 0, 6, 0, 0, 7, @@ -154,6 +158,8 @@ module { // // CHECK-NEXT: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 6, 16 ) + // CHECK-NEXT: lvl = ( 4, 2, 3, 4 ) // CHECK-NEXT: pos[1] : ( 0, 1, 1, 2, 2, // CHECK-NEXT: crd[1] : ( 0, 1, // CHECK-NEXT: values : ( 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, @@ -178,6 +184,8 @@ module { // // CHECK-NEXT: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 6, 16 ) + // CHECK-NEXT: lvl = ( 4, 2, 4, 3 ) // CHECK-NEXT: pos[1] : ( 0, 1, 1, 2, 2, // CHECK-NEXT: crd[1] : ( 0, 1, // CHECK-NEXT: values : ( 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 5, 0, 0, 0, 0, 6, 0, 0, 7, diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output.mlir index c6ee0ce07050..b2bbc64f1688 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output.mlir @@ -106,6 +106,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 25 + // CHECK-NEXT: dim = ( 5, 5 ) + // CHECK-NEXT: lvl = ( 5, 5 ) // CHECK-NEXT: values : ( 2, 0, 0, 2.8, 0, 0, 4, 0, 0, 5, 0, 0, 6, 0, 0, 8.2, 0, 0, 8, 0, 0, 10.4, 0, 0, 10, // CHECK-NEXT: ---- // diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_bf16.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_bf16.mlir index 0b34ff581016..ca9df03c69ee 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_bf16.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_bf16.mlir @@ -93,6 +93,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 32 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) // CHECK-NEXT: values : ( 1, 11, 0, 2, 13, 0, 0, 0, 0, 0, 14, 3, 0, 0, 0, 0, 15, 4, 16, 0, 5, 6, 0, 0, 0, 0, 0, 0, 7, 8, 0, 9, // CHECK-NEXT: ---- // diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_f16.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_f16.mlir index 495682169c29..4f5e6ddd48d8 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_f16.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_f16.mlir @@ -94,6 +94,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 32 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) // CHECK-NEXT: values : ( 1, 11, 0, 2, 13, 0, 0, 0, 0, 0, 14, 3, 0, 0, 0, 0, 15, 4, 16, 0, 5, 6, 0, 0, 0, 0, 0, 0, 7, 8, 0, 9, // CHECK-NEXT: ---- // diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir index 3a32ff285270..1917fd987c5d 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir @@ -62,6 +62,8 @@ module { // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 5 + // CHECK-NEXT: dim = ( 5, 4, 3 ) + // CHECK-NEXT: lvl = ( 5, 4, 3 ) // CHECK-NEXT: pos[0] : ( 0, 2 // CHECK-NEXT: crd[0] : ( 3, 4 // CHECK-NEXT: pos[2] : ( 0, 2, 2, 2, 3, 3, 3, 4, 5 @@ -79,6 +81,8 @@ module { // CHECK-NEXT: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 12 + // CHECK-NEXT: dim = ( 5, 4, 3 ) + // CHECK-NEXT: lvl = ( 5, 4, 3 ) // CHECK-NEXT: pos[0] : ( 0, 2 // CHECK-NEXT: crd[0] : ( 3, 4 // CHECK-NEXT: pos[1] : ( 0, 2, 4 @@ -96,6 +100,8 @@ module { // CHECK-NEXT: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 5 + // CHECK-NEXT: dim = ( 5, 4, 3 ) + // CHECK-NEXT: lvl = ( 5, 4, 3 ) // CHECK-NEXT: pos[0] : ( 0, 2 // CHECK-NEXT: crd[0] : ( 3, 4 // CHECK-NEXT: pos[1] : ( 0, 3, 5 @@ -113,6 +119,8 @@ module { // CHECK-NEXT: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 5 + // CHECK-NEXT: dim = ( 5, 4, 3 ) + // CHECK-NEXT: lvl = ( 5, 4, 3 ) // CHECK-NEXT: pos[1] : ( 0, 0, 0, 0, 3, 5 // CHECK-NEXT: crd[1] : ( 0, 1, 0, 2, 3, 1, 2, 2, 3, 2 // CHECK-NEXT: values : ( 1.1, 2.2, 3.3, 4.4, 5.5 diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_loose.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_loose.mlir index e1f062121b12..c05a9f574269 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_loose.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_loose.mlir @@ -41,6 +41,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 17 + // CHECK-NEXT: dim = ( 5, 4 ) + // CHECK-NEXT: lvl = ( 5, 4 ) // CHECK-NEXT: pos[1] : ( 0, 4, 4, 8, 8, 9, 9, 13 // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 2, 0, 1, 2, 3, 0, 1, 2, 3 // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 5.5, 9, 10, 11, 12, 13, 14, 15, 16 diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul.mlir index 863e1c62370e..e505559037a9 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul.mlir @@ -144,6 +144,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 32 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 4, 8 ) // CHECK-NEXT: pos[1] : ( 0, 8, 16, 24, 32 // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7 // CHECK-NEXT: values : ( 1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1, 8.1, 1.2, 2.2, 3.2, 4.2, 5.2, 6.2, 7.2, 8.2, 1.3, 2.3, 3.3, 4.3, 5.3, 6.3, 7.3, 8.3, 1.4, 2.4, 3.4, 4.4, 5.4, 6.4, 7.4, 8.4 @@ -154,6 +156,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 32 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 4, 8 ) // CHECK-NEXT: pos[0] : ( 0, 4 // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 // CHECK-NEXT: pos[1] : ( 0, 8, 16, 24, 32 @@ -166,6 +170,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 4 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 4, 8 ) // CHECK-NEXT: pos[1] : ( 0, 2, 2, 3, 4 // CHECK-NEXT: crd[1] : ( 1, 5, 1, 7 // CHECK-NEXT: values : ( 2.1, 6.1, 2.3, 1 @@ -176,6 +182,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 4 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 4, 8 ) // CHECK-NEXT: pos[0] : ( 0, 3 // CHECK-NEXT: crd[0] : ( 0, 2, 3 // CHECK-NEXT: pos[1] : ( 0, 2, 3, 4 @@ -188,6 +196,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 32 + // CHECK-NEXT: dim = ( 8, 4 ) + // CHECK-NEXT: lvl = ( 8, 4 ) // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12, 16, 20, 24, 28, 32 // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 // CHECK-NEXT: values : ( 10.1, 11.1, 12.1, 13.1, 10.2, 11.2, 12.2, 13.2, 10.3, 11.3, 12.3, 13.3, 10.4, 11.4, 12.4, 13.4, 10.5, 11.5, 12.5, 13.5, 10.6, 11.6, 12.6, 13.6, 10.7, 11.7, 12.7, 13.7, 10.8, 11.8, 12.8, 13.8 @@ -198,6 +208,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 32 + // CHECK-NEXT: dim = ( 8, 4 ) + // CHECK-NEXT: lvl = ( 8, 4 ) // CHECK-NEXT: pos[0] : ( 0, 8 // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7 // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12, 16, 20, 24, 28, 32 @@ -210,6 +222,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 8 + // CHECK-NEXT: dim = ( 8, 4 ) + // CHECK-NEXT: lvl = ( 8, 4 ) // CHECK-NEXT: pos[1] : ( 0, 1, 2, 3, 4, 4, 5, 6, 8 // CHECK-NEXT: crd[1] : ( 3, 2, 1, 0, 1, 2, 2, 3 // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8 @@ -220,6 +234,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 8 + // CHECK-NEXT: dim = ( 8, 4 ) + // CHECK-NEXT: lvl = ( 8, 4 ) // CHECK-NEXT: pos[0] : ( 0, 7 // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 5, 6, 7 // CHECK-NEXT: pos[1] : ( 0, 1, 2, 3, 4, 5, 6, 8 @@ -271,6 +287,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 16 + // CHECK-NEXT: dim = ( 4, 4 ) + // CHECK-NEXT: lvl = ( 4, 4 ) // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12, 16 // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 // CHECK-NEXT: values : ( 388.76, 425.56, 462.36, 499.16, 397.12, 434.72, 472.32, 509.92, 405.48, 443.88, 482.28, 520.68, 413.84, 453.04, 492.24, 531.44 @@ -281,6 +299,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 16 + // CHECK-NEXT: dim = ( 4, 4 ) + // CHECK-NEXT: lvl = ( 4, 4 ) // CHECK-NEXT: pos[0] : ( 0, 4 // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12, 16 @@ -302,6 +322,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 12 + // CHECK-NEXT: dim = ( 4, 4 ) + // CHECK-NEXT: lvl = ( 4, 4 ) // CHECK-NEXT: pos[1] : ( 0, 4, 4, 8, 12 // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 // CHECK-NEXT: values : ( 86.08, 94.28, 102.48, 110.68, 23.46, 25.76, 28.06, 30.36, 10.8, 11.8, 12.8, 13.8 @@ -312,6 +334,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 12 + // CHECK-NEXT: dim = ( 4, 4 ) + // CHECK-NEXT: lvl = ( 4, 4 ) // CHECK-NEXT: pos[0] : ( 0, 3 // CHECK-NEXT: crd[0] : ( 0, 2, 3 // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12 @@ -333,6 +357,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 5 + // CHECK-NEXT: dim = ( 4, 4 ) + // CHECK-NEXT: lvl = ( 4, 4 ) // CHECK-NEXT: pos[1] : ( 0, 2, 2, 3, 5 // CHECK-NEXT: crd[1] : ( 1, 2, 2, 2, 3 // CHECK-NEXT: values : ( 30.5, 4.2, 4.6, 7, 8 @@ -343,6 +369,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 5 + // CHECK-NEXT: dim = ( 4, 4 ) + // CHECK-NEXT: lvl = ( 4, 4 ) // CHECK-NEXT: pos[0] : ( 0, 3 // CHECK-NEXT: crd[0] : ( 0, 2, 3 // CHECK-NEXT: pos[1] : ( 0, 2, 3, 5 diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul_slice.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul_slice.mlir index b95f72e1a479..58e96d1fa51f 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul_slice.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul_slice.mlir @@ -172,6 +172,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 5 + // CHECK-NEXT: dim = ( 4, 4 ) + // CHECK-NEXT: lvl = ( 4, 4 ) // CHECK-NEXT: pos[0] : ( 0, 3 // CHECK-NEXT: crd[0] : ( 0, 2, 3 // CHECK-NEXT: pos[1] : ( 0, 2, 3, 5 @@ -192,6 +194,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 5 + // CHECK-NEXT: dim = ( 4, 4 ) + // CHECK-NEXT: lvl = ( 4, 4 ) // CHECK-NEXT: pos[1] : ( 0, 2, 2, 3, 5 // CHECK-NEXT: crd[1] : ( 1, 2, 2, 2, 3 // CHECK-NEXT: values : ( 30.5, 4.2, 4.6, 7, 8 @@ -204,6 +208,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 3 + // CHECK-NEXT: dim = ( 4, 4 ) + // CHECK-NEXT: lvl = ( 4, 4 ) // CHECK-NEXT: pos[1] : ( 0, 1, 2, 2, 3 // CHECK-NEXT: crd[1] : ( 0, 0, 0 // CHECK-NEXT: values : ( 2.3, 6.9, 12.6 @@ -220,6 +226,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 3 + // CHECK-NEXT: dim = ( 4, 4 ) + // CHECK-NEXT: lvl = ( 4, 4 ) // CHECK-NEXT: pos[0] : ( 0, 3 // CHECK-NEXT: crd[0] : ( 0, 0, 1, 0, 3, 0 // CHECK-NEXT: values : ( 2.3, 6.9, 12.6 @@ -236,6 +244,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 3 + // CHECK-NEXT: dim = ( 4, 4 ) + // CHECK-NEXT: lvl = ( 4, 4 ) // CHECK-NEXT: pos[1] : ( 0, 1, 2, 2, 3 // CHECK-NEXT: crd[1] : ( 0, 0, 0 // CHECK-NEXT: values : ( 2.3, 6.9, 12.6 diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matrix_ops.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matrix_ops.mlir index 2cef46f4cb15..8ea26fa3efdf 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matrix_ops.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matrix_ops.mlir @@ -161,6 +161,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 4, 8 ) // CHECK-NEXT: pos[0] : ( 0, 4 // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9 @@ -173,6 +175,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 6 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 4, 8 ) // CHECK-NEXT: pos[0] : ( 0, 3 // CHECK-NEXT: crd[0] : ( 0, 1, 2 // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6 @@ -185,6 +189,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 4, 8 ) // CHECK-NEXT: pos[0] : ( 0, 4 // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9 @@ -197,6 +203,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 4, 8 ) // CHECK-NEXT: pos[0] : ( 0, 4 // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9 @@ -209,6 +217,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 13 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 4, 8 ) // CHECK-NEXT: pos[0] : ( 0, 4 // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 // CHECK-NEXT: pos[1] : ( 0, 3, 6, 10, 13 @@ -221,6 +231,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 2 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 4, 8 ) // CHECK-NEXT: pos[0] : ( 0, 2 // CHECK-NEXT: crd[0] : ( 0, 2 // CHECK-NEXT: pos[1] : ( 0, 1, 2 diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir index 79728fdb0f8c..98d76ba350cb 100755 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir @@ -137,12 +137,16 @@ module { // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 32 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 4, 8 ) // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 5, 0, 0, // CHECK-NEXT: ---- sparse_tensor.print %XO : tensor<4x8xi32, #AllDense> // CHECK-NEXT: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 32 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 8, 4 ) // CHECK-NEXT: values : ( 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, // CHECK-NEXT: ---- sparse_tensor.print %XT : tensor<4x8xi32, #AllDenseT> @@ -160,6 +164,8 @@ module { // CHECK-NEXT: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 5 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 4, 8 ) // CHECK-NEXT: pos[1] : ( 0, 2, 2, 2, 5, // CHECK-NEXT: crd[1] : ( 0, 2, 2, 3, 5, // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, @@ -168,6 +174,8 @@ module { // CHECK-NEXT: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 5 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 4, 8 ) // CHECK-NEXT: pos[0] : ( 0, 2, // CHECK-NEXT: crd[0] : ( 0, 3, // CHECK-NEXT: pos[1] : ( 0, 2, 5, @@ -178,6 +186,8 @@ module { // CHECK-NEXT: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 5 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 8, 4 ) // CHECK-NEXT: pos[1] : ( 0, 1, 1, 3, 4, 4, 5, 5, 5, // CHECK-NEXT: crd[1] : ( 0, 0, 3, 3, 3, // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, @@ -186,6 +196,8 @@ module { // CHECK-NEXT: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 5 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 8, 4 ) // CHECK-NEXT: pos[0] : ( 0, 4, // CHECK-NEXT: crd[0] : ( 0, 2, 3, 5, // CHECK-NEXT: pos[1] : ( 0, 1, 3, 4, 5, @@ -196,6 +208,8 @@ module { // CHECK-NEXT: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 2, 2, 2, 4 ) // CHECK-NEXT: pos[0] : ( 0, 2, // CHECK-NEXT: crd[0] : ( 0, 1, // CHECK-NEXT: pos[1] : ( 0, 1, 3, @@ -206,6 +220,8 @@ module { // CHECK-NEXT: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 2, 2, 4, 2 ) // CHECK-NEXT: pos[0] : ( 0, 2, // CHECK-NEXT: crd[0] : ( 0, 1, // CHECK-NEXT: pos[1] : ( 0, 1, 3, @@ -216,6 +232,8 @@ module { // CHECK-NEXT: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 2, 2, 2, 4 ) // CHECK-NEXT: pos[0] : ( 0, 2, // CHECK-NEXT: crd[0] : ( 0, 1, // CHECK-NEXT: pos[1] : ( 0, 2, 3, @@ -226,6 +244,8 @@ module { // CHECK-NEXT: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 2, 2, 4, 2 ) // CHECK-NEXT: pos[0] : ( 0, 2, // CHECK-NEXT: crd[0] : ( 0, 1, // CHECK-NEXT: pos[1] : ( 0, 2, 3, @@ -236,6 +256,8 @@ module { // CHECK-NEXT: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 2, 2, 2, 4 ) // CHECK-NEXT: pos[1] : ( 0, 1, 3, // CHECK-NEXT: crd[1] : ( 0, 0, 1, // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 5, 0, 0, @@ -244,6 +266,8 @@ module { // CHECK-NEXT: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 2, 2, 2, 4 ) // CHECK-NEXT: pos[1] : ( 0, 2, 3, // CHECK-NEXT: crd[1] : ( 0, 1, 1, // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 5, 0, 0, diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_re_im.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_re_im.mlir index 1860fc1c7027..7bacbe3b87e4 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_re_im.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_re_im.mlir @@ -91,6 +91,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 3 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) // CHECK-NEXT: pos[0] : ( 0, 3, // CHECK-NEXT: crd[0] : ( 0, 20, 31, // CHECK-NEXT: values : ( 5.13, 3, 5, @@ -98,6 +100,8 @@ module { // // CHECK-NEXT: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 3 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) // CHECK-NEXT: pos[0] : ( 0, 3, // CHECK-NEXT: crd[0] : ( 0, 20, 31, // CHECK-NEXT: values : ( 2, 4, 6, diff --git a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-gemm-lib.mlir b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-gemm-lib.mlir index 984b3614eded..da78452d94fd 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-gemm-lib.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-gemm-lib.mlir @@ -66,6 +66,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 20 + // CHECK-NEXT: dim = ( 8, 8 ) + // CHECK-NEXT: lvl = ( 8, 8 ) // CHECK-NEXT: pos[1] : ( 0, 5, 5, 6, 7, 8, 12, 16, 20, // CHECK-NEXT: crd[1] : ( 0, 1, 2, 6, 7, 2, 3, 4, 1, 2, 6, 7, 1, 2, 6, 7, 1, 2, 6, 7, // CHECK-NEXT: values : ( 1, 39, 52, 45, 51, 16, 25, 36, 117, 158, 135, 144, 156, 318, 301, 324, 208, 430, 405, 436, diff --git a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sampled-matmul-lib.mlir b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sampled-matmul-lib.mlir index aad26556b00e..3d17b719732f 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sampled-matmul-lib.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sampled-matmul-lib.mlir @@ -115,6 +115,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 5, 5 ) + // CHECK-NEXT: lvl = ( 5, 5 ) // CHECK-NEXT: pos[1] : ( 0, 2, 4, 5, 7, 9, // CHECK-NEXT: crd[1] : ( 0, 3, 1, 4, 2, 0, 3, 1, 4, // CHECK-NEXT: values : ( 11, 41.4, 42, 102.5, 93, 44.1, 164, 105.2, 255, @@ -141,6 +143,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 5 + // CHECK-NEXT: dim = ( 8, 8 ) + // CHECK-NEXT: lvl = ( 8, 8 ) // CHECK-NEXT: pos[1] : ( 0, 2, 3, 3, 4, 4, 4, 4, 5, // CHECK-NEXT: crd[1] : ( 0, 1, 0, 4, 7, // CHECK-NEXT: values : ( 17, 18, 19, 20, 21, diff --git a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sddmm-lib.mlir b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sddmm-lib.mlir index bd2c72271c61..68bb32891f34 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sddmm-lib.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sddmm-lib.mlir @@ -168,6 +168,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 8 + // CHECK-NEXT: dim = ( 4, 6 ) + // CHECK-NEXT: lvl = ( 4, 6 ) // CHECK-NEXT: pos[1] : ( 0, 3, 5, 7, 8, // CHECK-NEXT: crd[1] : ( 0, 1, 4, 1, 5, 2, 3, 2, // CHECK-NEXT: values : ( 5, 10, 24, 19, 53, 42, 55, 56, @@ -175,6 +177,8 @@ module { // // CHECK: ---- Sparse Tensor ---- // CHECK-NEXT: nse = 12 + // CHECK-NEXT: dim = ( 4, 6 ) + // CHECK-NEXT: lvl = ( 2, 3, 2, 2 ) // CHECK-NEXT: pos[1] : ( 0, 2, 3, // CHECK-NEXT: crd[1] : ( 0, 2, 1, // CHECK-NEXT: values : ( 5, 10, 8, 19, 24, 24, 40, 53, 42, 55, 56, 64, -- GitLab From 8cc8fdaf5c2e799bc758919365bae601e59c03fc Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Mon, 4 Mar 2024 23:34:28 +0100 Subject: [PATCH 056/929] [AArch64] Also promote vector bf16 INT_TP_FP to f32 This mirrors the scalar version. --- .../Target/AArch64/AArch64ISelLowering.cpp | 15 + llvm/test/CodeGen/AArch64/itofp.ll | 1740 +++++++++++++++++ 2 files changed, 1755 insertions(+) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 63725f840b6f..8d6e92a00b5d 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -4421,6 +4421,21 @@ SDValue AArch64TargetLowering::LowerVectorINT_TO_FP(SDValue Op, useSVEForFixedLengthVectorVT(InVT, !Subtarget->isNeonAvailable())) return LowerFixedLengthIntToFPToSVE(Op, DAG); + // Promote bf16 conversions to f32. + if (VT.getVectorElementType() == MVT::bf16) { + EVT F32 = VT.changeElementType(MVT::f32); + if (IsStrict) { + SDValue Val = DAG.getNode(Op.getOpcode(), dl, {F32, MVT::Other}, + {Op.getOperand(0), In}); + return DAG.getNode( + ISD::STRICT_FP_ROUND, dl, {Op.getValueType(), MVT::Other}, + {Val.getValue(1), Val.getValue(0), DAG.getIntPtrConstant(0, dl)}); + } + return DAG.getNode(ISD::FP_ROUND, dl, Op.getValueType(), + DAG.getNode(Op.getOpcode(), dl, F32, In), + DAG.getIntPtrConstant(0, dl)); + } + uint64_t VTSize = VT.getFixedSizeInBits(); uint64_t InVTSize = InVT.getFixedSizeInBits(); if (VTSize < InVTSize) { diff --git a/llvm/test/CodeGen/AArch64/itofp.ll b/llvm/test/CodeGen/AArch64/itofp.ll index c40867ff7392..be57e1e26a92 100644 --- a/llvm/test/CodeGen/AArch64/itofp.ll +++ b/llvm/test/CodeGen/AArch64/itofp.ll @@ -6202,3 +6202,1743 @@ entry: %c = uitofp <32 x i8> %a to <32 x half> ret <32 x half> %c } + +define bfloat @stofp_i64_bf16(i64 %a) { +; CHECK-LABEL: stofp_i64_bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: scvtf s0, x0 +; CHECK-NEXT: mov w8, #32767 // =0x7fff +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w10, w9, #16, #1 +; CHECK-NEXT: add w8, w9, w8 +; CHECK-NEXT: add w8, w10, w8 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $s0 +; CHECK-NEXT: ret +entry: + %c = sitofp i64 %a to bfloat + ret bfloat %c +} + +define bfloat @utofp_i64_bf16(i64 %a) { +; CHECK-LABEL: utofp_i64_bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ucvtf s0, x0 +; CHECK-NEXT: mov w8, #32767 // =0x7fff +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w10, w9, #16, #1 +; CHECK-NEXT: add w8, w9, w8 +; CHECK-NEXT: add w8, w10, w8 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $s0 +; CHECK-NEXT: ret +entry: + %c = uitofp i64 %a to bfloat + ret bfloat %c +} + +define bfloat @stofp_i32_bf16(i32 %a) { +; CHECK-LABEL: stofp_i32_bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: scvtf s0, w0 +; CHECK-NEXT: mov w8, #32767 // =0x7fff +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w10, w9, #16, #1 +; CHECK-NEXT: add w8, w9, w8 +; CHECK-NEXT: add w8, w10, w8 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $s0 +; CHECK-NEXT: ret +entry: + %c = sitofp i32 %a to bfloat + ret bfloat %c +} + +define bfloat @utofp_i32_bf16(i32 %a) { +; CHECK-LABEL: utofp_i32_bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ucvtf s0, w0 +; CHECK-NEXT: mov w8, #32767 // =0x7fff +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w10, w9, #16, #1 +; CHECK-NEXT: add w8, w9, w8 +; CHECK-NEXT: add w8, w10, w8 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $s0 +; CHECK-NEXT: ret +entry: + %c = uitofp i32 %a to bfloat + ret bfloat %c +} + +define bfloat @stofp_i16_bf16(i16 %a) { +; CHECK-LABEL: stofp_i16_bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: sxth w9, w0 +; CHECK-NEXT: mov w8, #32767 // =0x7fff +; CHECK-NEXT: scvtf s0, w9 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w10, w9, #16, #1 +; CHECK-NEXT: add w8, w9, w8 +; CHECK-NEXT: add w8, w10, w8 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $s0 +; CHECK-NEXT: ret +entry: + %c = sitofp i16 %a to bfloat + ret bfloat %c +} + +define bfloat @utofp_i16_bf16(i16 %a) { +; CHECK-LABEL: utofp_i16_bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: and w9, w0, #0xffff +; CHECK-NEXT: mov w8, #32767 // =0x7fff +; CHECK-NEXT: ucvtf s0, w9 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w10, w9, #16, #1 +; CHECK-NEXT: add w8, w9, w8 +; CHECK-NEXT: add w8, w10, w8 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $s0 +; CHECK-NEXT: ret +entry: + %c = uitofp i16 %a to bfloat + ret bfloat %c +} + +define bfloat @stofp_i8_bf16(i8 %a) { +; CHECK-LABEL: stofp_i8_bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: sxtb w9, w0 +; CHECK-NEXT: mov w8, #32767 // =0x7fff +; CHECK-NEXT: scvtf s0, w9 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w10, w9, #16, #1 +; CHECK-NEXT: add w8, w9, w8 +; CHECK-NEXT: add w8, w10, w8 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $s0 +; CHECK-NEXT: ret +entry: + %c = sitofp i8 %a to bfloat + ret bfloat %c +} + +define bfloat @utofp_i8_bf16(i8 %a) { +; CHECK-LABEL: utofp_i8_bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: and w9, w0, #0xff +; CHECK-NEXT: mov w8, #32767 // =0x7fff +; CHECK-NEXT: ucvtf s0, w9 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w10, w9, #16, #1 +; CHECK-NEXT: add w8, w9, w8 +; CHECK-NEXT: add w8, w10, w8 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $s0 +; CHECK-NEXT: ret +entry: + %c = uitofp i8 %a to bfloat + ret bfloat %c +} + +define <2 x bfloat> @stofp_v2i64_v2bf16(<2 x i64> %a) { +; CHECK-LABEL: stofp_v2i64_v2bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: mov x9, v0.d[1] +; CHECK-NEXT: fmov x10, d0 +; CHECK-NEXT: mov w8, #32767 // =0x7fff +; CHECK-NEXT: scvtf s1, x10 +; CHECK-NEXT: scvtf s0, x9 +; CHECK-NEXT: fmov w10, s1 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w12, w10, #16, #1 +; CHECK-NEXT: ubfx w11, w9, #16, #1 +; CHECK-NEXT: add w9, w9, w8 +; CHECK-NEXT: add w8, w10, w8 +; CHECK-NEXT: add w8, w12, w8 +; CHECK-NEXT: add w9, w11, w9 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: lsr w9, w9, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: fmov s1, w9 +; CHECK-NEXT: mov v0.h[1], v1.h[0] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret +entry: + %c = sitofp <2 x i64> %a to <2 x bfloat> + ret <2 x bfloat> %c +} + +define <2 x bfloat> @utofp_v2i64_v2bf16(<2 x i64> %a) { +; CHECK-LABEL: utofp_v2i64_v2bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: mov x9, v0.d[1] +; CHECK-NEXT: fmov x10, d0 +; CHECK-NEXT: mov w8, #32767 // =0x7fff +; CHECK-NEXT: ucvtf s1, x10 +; CHECK-NEXT: ucvtf s0, x9 +; CHECK-NEXT: fmov w10, s1 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w12, w10, #16, #1 +; CHECK-NEXT: ubfx w11, w9, #16, #1 +; CHECK-NEXT: add w9, w9, w8 +; CHECK-NEXT: add w8, w10, w8 +; CHECK-NEXT: add w8, w12, w8 +; CHECK-NEXT: add w9, w11, w9 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: lsr w9, w9, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: fmov s1, w9 +; CHECK-NEXT: mov v0.h[1], v1.h[0] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret +entry: + %c = uitofp <2 x i64> %a to <2 x bfloat> + ret <2 x bfloat> %c +} + +define <3 x bfloat> @stofp_v3i64_v3bf16(<3 x i64> %a) { +; CHECK-LABEL: stofp_v3i64_v3bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1 +; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2 +; CHECK-NEXT: mov v0.d[1], v1.d[0] +; CHECK-NEXT: scvtf v1.2d, v2.2d +; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: scvtf v0.2d, v0.2d +; CHECK-NEXT: fcvtn v0.2s, v0.2d +; CHECK-NEXT: fcvtn2 v0.4s, v1.2d +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: ushr v3.4s, v0.4s, #16 +; CHECK-NEXT: add v1.4s, v0.4s, v1.4s +; CHECK-NEXT: and v2.16b, v3.16b, v2.16b +; CHECK-NEXT: add v1.4s, v2.4s, v1.4s +; CHECK-NEXT: fcmeq v2.4s, v0.4s, v0.4s +; CHECK-NEXT: orr v0.4s, #64, lsl #16 +; CHECK-NEXT: bit v0.16b, v1.16b, v2.16b +; CHECK-NEXT: shrn v0.4h, v0.4s, #16 +; CHECK-NEXT: ret +entry: + %c = sitofp <3 x i64> %a to <3 x bfloat> + ret <3 x bfloat> %c +} + +define <3 x bfloat> @utofp_v3i64_v3bf16(<3 x i64> %a) { +; CHECK-LABEL: utofp_v3i64_v3bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1 +; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2 +; CHECK-NEXT: mov v0.d[1], v1.d[0] +; CHECK-NEXT: ucvtf v1.2d, v2.2d +; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: ucvtf v0.2d, v0.2d +; CHECK-NEXT: fcvtn v0.2s, v0.2d +; CHECK-NEXT: fcvtn2 v0.4s, v1.2d +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: ushr v3.4s, v0.4s, #16 +; CHECK-NEXT: add v1.4s, v0.4s, v1.4s +; CHECK-NEXT: and v2.16b, v3.16b, v2.16b +; CHECK-NEXT: add v1.4s, v2.4s, v1.4s +; CHECK-NEXT: fcmeq v2.4s, v0.4s, v0.4s +; CHECK-NEXT: orr v0.4s, #64, lsl #16 +; CHECK-NEXT: bit v0.16b, v1.16b, v2.16b +; CHECK-NEXT: shrn v0.4h, v0.4s, #16 +; CHECK-NEXT: ret +entry: + %c = uitofp <3 x i64> %a to <3 x bfloat> + ret <3 x bfloat> %c +} + +define <4 x bfloat> @stofp_v4i64_v4bf16(<4 x i64> %a) { +; CHECK-LABEL: stofp_v4i64_v4bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: scvtf v0.2d, v0.2d +; CHECK-NEXT: scvtf v1.2d, v1.2d +; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: fcvtn v0.2s, v0.2d +; CHECK-NEXT: fcvtn2 v0.4s, v1.2d +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: ushr v3.4s, v0.4s, #16 +; CHECK-NEXT: add v1.4s, v0.4s, v1.4s +; CHECK-NEXT: and v2.16b, v3.16b, v2.16b +; CHECK-NEXT: add v1.4s, v2.4s, v1.4s +; CHECK-NEXT: fcmeq v2.4s, v0.4s, v0.4s +; CHECK-NEXT: orr v0.4s, #64, lsl #16 +; CHECK-NEXT: bit v0.16b, v1.16b, v2.16b +; CHECK-NEXT: shrn v0.4h, v0.4s, #16 +; CHECK-NEXT: ret +entry: + %c = sitofp <4 x i64> %a to <4 x bfloat> + ret <4 x bfloat> %c +} + +define <4 x bfloat> @utofp_v4i64_v4bf16(<4 x i64> %a) { +; CHECK-LABEL: utofp_v4i64_v4bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ucvtf v0.2d, v0.2d +; CHECK-NEXT: ucvtf v1.2d, v1.2d +; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: fcvtn v0.2s, v0.2d +; CHECK-NEXT: fcvtn2 v0.4s, v1.2d +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: ushr v3.4s, v0.4s, #16 +; CHECK-NEXT: add v1.4s, v0.4s, v1.4s +; CHECK-NEXT: and v2.16b, v3.16b, v2.16b +; CHECK-NEXT: add v1.4s, v2.4s, v1.4s +; CHECK-NEXT: fcmeq v2.4s, v0.4s, v0.4s +; CHECK-NEXT: orr v0.4s, #64, lsl #16 +; CHECK-NEXT: bit v0.16b, v1.16b, v2.16b +; CHECK-NEXT: shrn v0.4h, v0.4s, #16 +; CHECK-NEXT: ret +entry: + %c = uitofp <4 x i64> %a to <4 x bfloat> + ret <4 x bfloat> %c +} + +define <8 x bfloat> @stofp_v8i64_v8bf16(<8 x i64> %a) { +; CHECK-LABEL: stofp_v8i64_v8bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: scvtf v2.2d, v2.2d +; CHECK-NEXT: scvtf v0.2d, v0.2d +; CHECK-NEXT: scvtf v3.2d, v3.2d +; CHECK-NEXT: scvtf v1.2d, v1.2d +; CHECK-NEXT: fcvtn v2.2s, v2.2d +; CHECK-NEXT: fcvtn v0.2s, v0.2d +; CHECK-NEXT: fcvtn2 v2.4s, v3.2d +; CHECK-NEXT: fcvtn2 v0.4s, v1.2d +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: movi v3.4s, #1 +; CHECK-NEXT: ushr v4.4s, v2.4s, #16 +; CHECK-NEXT: ushr v5.4s, v0.4s, #16 +; CHECK-NEXT: add v6.4s, v2.4s, v1.4s +; CHECK-NEXT: add v1.4s, v0.4s, v1.4s +; CHECK-NEXT: and v4.16b, v4.16b, v3.16b +; CHECK-NEXT: and v3.16b, v5.16b, v3.16b +; CHECK-NEXT: fcmeq v5.4s, v2.4s, v2.4s +; CHECK-NEXT: orr v2.4s, #64, lsl #16 +; CHECK-NEXT: add v4.4s, v4.4s, v6.4s +; CHECK-NEXT: add v1.4s, v3.4s, v1.4s +; CHECK-NEXT: fcmeq v3.4s, v0.4s, v0.4s +; CHECK-NEXT: orr v0.4s, #64, lsl #16 +; CHECK-NEXT: bit v2.16b, v4.16b, v5.16b +; CHECK-NEXT: bit v0.16b, v1.16b, v3.16b +; CHECK-NEXT: uzp2 v0.8h, v0.8h, v2.8h +; CHECK-NEXT: ret +entry: + %c = sitofp <8 x i64> %a to <8 x bfloat> + ret <8 x bfloat> %c +} + +define <8 x bfloat> @utofp_v8i64_v8bf16(<8 x i64> %a) { +; CHECK-LABEL: utofp_v8i64_v8bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ucvtf v2.2d, v2.2d +; CHECK-NEXT: ucvtf v0.2d, v0.2d +; CHECK-NEXT: ucvtf v3.2d, v3.2d +; CHECK-NEXT: ucvtf v1.2d, v1.2d +; CHECK-NEXT: fcvtn v2.2s, v2.2d +; CHECK-NEXT: fcvtn v0.2s, v0.2d +; CHECK-NEXT: fcvtn2 v2.4s, v3.2d +; CHECK-NEXT: fcvtn2 v0.4s, v1.2d +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: movi v3.4s, #1 +; CHECK-NEXT: ushr v4.4s, v2.4s, #16 +; CHECK-NEXT: ushr v5.4s, v0.4s, #16 +; CHECK-NEXT: add v6.4s, v2.4s, v1.4s +; CHECK-NEXT: add v1.4s, v0.4s, v1.4s +; CHECK-NEXT: and v4.16b, v4.16b, v3.16b +; CHECK-NEXT: and v3.16b, v5.16b, v3.16b +; CHECK-NEXT: fcmeq v5.4s, v2.4s, v2.4s +; CHECK-NEXT: orr v2.4s, #64, lsl #16 +; CHECK-NEXT: add v4.4s, v4.4s, v6.4s +; CHECK-NEXT: add v1.4s, v3.4s, v1.4s +; CHECK-NEXT: fcmeq v3.4s, v0.4s, v0.4s +; CHECK-NEXT: orr v0.4s, #64, lsl #16 +; CHECK-NEXT: bit v2.16b, v4.16b, v5.16b +; CHECK-NEXT: bit v0.16b, v1.16b, v3.16b +; CHECK-NEXT: uzp2 v0.8h, v0.8h, v2.8h +; CHECK-NEXT: ret +entry: + %c = uitofp <8 x i64> %a to <8 x bfloat> + ret <8 x bfloat> %c +} + +define <16 x bfloat> @stofp_v16i64_v16bf16(<16 x i64> %a) { +; CHECK-LABEL: stofp_v16i64_v16bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: scvtf v2.2d, v2.2d +; CHECK-NEXT: scvtf v0.2d, v0.2d +; CHECK-NEXT: scvtf v6.2d, v6.2d +; CHECK-NEXT: scvtf v4.2d, v4.2d +; CHECK-NEXT: scvtf v3.2d, v3.2d +; CHECK-NEXT: scvtf v1.2d, v1.2d +; CHECK-NEXT: scvtf v7.2d, v7.2d +; CHECK-NEXT: scvtf v5.2d, v5.2d +; CHECK-NEXT: fcvtn v2.2s, v2.2d +; CHECK-NEXT: fcvtn v0.2s, v0.2d +; CHECK-NEXT: fcvtn v6.2s, v6.2d +; CHECK-NEXT: fcvtn v4.2s, v4.2d +; CHECK-NEXT: fcvtn2 v2.4s, v3.2d +; CHECK-NEXT: fcvtn2 v0.4s, v1.2d +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: fcvtn2 v6.4s, v7.2d +; CHECK-NEXT: fcvtn2 v4.4s, v5.2d +; CHECK-NEXT: movi v3.4s, #1 +; CHECK-NEXT: ushr v5.4s, v2.4s, #16 +; CHECK-NEXT: ushr v7.4s, v0.4s, #16 +; CHECK-NEXT: add v17.4s, v2.4s, v1.4s +; CHECK-NEXT: add v19.4s, v0.4s, v1.4s +; CHECK-NEXT: ushr v16.4s, v6.4s, #16 +; CHECK-NEXT: ushr v18.4s, v4.4s, #16 +; CHECK-NEXT: add v20.4s, v6.4s, v1.4s +; CHECK-NEXT: add v1.4s, v4.4s, v1.4s +; CHECK-NEXT: and v5.16b, v5.16b, v3.16b +; CHECK-NEXT: and v7.16b, v7.16b, v3.16b +; CHECK-NEXT: and v16.16b, v16.16b, v3.16b +; CHECK-NEXT: and v3.16b, v18.16b, v3.16b +; CHECK-NEXT: fcmeq v18.4s, v0.4s, v0.4s +; CHECK-NEXT: orr v0.4s, #64, lsl #16 +; CHECK-NEXT: add v5.4s, v5.4s, v17.4s +; CHECK-NEXT: fcmeq v17.4s, v2.4s, v2.4s +; CHECK-NEXT: add v7.4s, v7.4s, v19.4s +; CHECK-NEXT: fcmeq v19.4s, v6.4s, v6.4s +; CHECK-NEXT: orr v2.4s, #64, lsl #16 +; CHECK-NEXT: add v16.4s, v16.4s, v20.4s +; CHECK-NEXT: add v1.4s, v3.4s, v1.4s +; CHECK-NEXT: fcmeq v3.4s, v4.4s, v4.4s +; CHECK-NEXT: orr v6.4s, #64, lsl #16 +; CHECK-NEXT: orr v4.4s, #64, lsl #16 +; CHECK-NEXT: bit v2.16b, v5.16b, v17.16b +; CHECK-NEXT: bit v0.16b, v7.16b, v18.16b +; CHECK-NEXT: mov v5.16b, v19.16b +; CHECK-NEXT: bif v1.16b, v4.16b, v3.16b +; CHECK-NEXT: bsl v5.16b, v16.16b, v6.16b +; CHECK-NEXT: uzp2 v0.8h, v0.8h, v2.8h +; CHECK-NEXT: uzp2 v1.8h, v1.8h, v5.8h +; CHECK-NEXT: ret +entry: + %c = sitofp <16 x i64> %a to <16 x bfloat> + ret <16 x bfloat> %c +} + +define <16 x bfloat> @utofp_v16i64_v16bf16(<16 x i64> %a) { +; CHECK-LABEL: utofp_v16i64_v16bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ucvtf v2.2d, v2.2d +; CHECK-NEXT: ucvtf v0.2d, v0.2d +; CHECK-NEXT: ucvtf v6.2d, v6.2d +; CHECK-NEXT: ucvtf v4.2d, v4.2d +; CHECK-NEXT: ucvtf v3.2d, v3.2d +; CHECK-NEXT: ucvtf v1.2d, v1.2d +; CHECK-NEXT: ucvtf v7.2d, v7.2d +; CHECK-NEXT: ucvtf v5.2d, v5.2d +; CHECK-NEXT: fcvtn v2.2s, v2.2d +; CHECK-NEXT: fcvtn v0.2s, v0.2d +; CHECK-NEXT: fcvtn v6.2s, v6.2d +; CHECK-NEXT: fcvtn v4.2s, v4.2d +; CHECK-NEXT: fcvtn2 v2.4s, v3.2d +; CHECK-NEXT: fcvtn2 v0.4s, v1.2d +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: fcvtn2 v6.4s, v7.2d +; CHECK-NEXT: fcvtn2 v4.4s, v5.2d +; CHECK-NEXT: movi v3.4s, #1 +; CHECK-NEXT: ushr v5.4s, v2.4s, #16 +; CHECK-NEXT: ushr v7.4s, v0.4s, #16 +; CHECK-NEXT: add v17.4s, v2.4s, v1.4s +; CHECK-NEXT: add v19.4s, v0.4s, v1.4s +; CHECK-NEXT: ushr v16.4s, v6.4s, #16 +; CHECK-NEXT: ushr v18.4s, v4.4s, #16 +; CHECK-NEXT: add v20.4s, v6.4s, v1.4s +; CHECK-NEXT: add v1.4s, v4.4s, v1.4s +; CHECK-NEXT: and v5.16b, v5.16b, v3.16b +; CHECK-NEXT: and v7.16b, v7.16b, v3.16b +; CHECK-NEXT: and v16.16b, v16.16b, v3.16b +; CHECK-NEXT: and v3.16b, v18.16b, v3.16b +; CHECK-NEXT: fcmeq v18.4s, v0.4s, v0.4s +; CHECK-NEXT: orr v0.4s, #64, lsl #16 +; CHECK-NEXT: add v5.4s, v5.4s, v17.4s +; CHECK-NEXT: fcmeq v17.4s, v2.4s, v2.4s +; CHECK-NEXT: add v7.4s, v7.4s, v19.4s +; CHECK-NEXT: fcmeq v19.4s, v6.4s, v6.4s +; CHECK-NEXT: orr v2.4s, #64, lsl #16 +; CHECK-NEXT: add v16.4s, v16.4s, v20.4s +; CHECK-NEXT: add v1.4s, v3.4s, v1.4s +; CHECK-NEXT: fcmeq v3.4s, v4.4s, v4.4s +; CHECK-NEXT: orr v6.4s, #64, lsl #16 +; CHECK-NEXT: orr v4.4s, #64, lsl #16 +; CHECK-NEXT: bit v2.16b, v5.16b, v17.16b +; CHECK-NEXT: bit v0.16b, v7.16b, v18.16b +; CHECK-NEXT: mov v5.16b, v19.16b +; CHECK-NEXT: bif v1.16b, v4.16b, v3.16b +; CHECK-NEXT: bsl v5.16b, v16.16b, v6.16b +; CHECK-NEXT: uzp2 v0.8h, v0.8h, v2.8h +; CHECK-NEXT: uzp2 v1.8h, v1.8h, v5.8h +; CHECK-NEXT: ret +entry: + %c = uitofp <16 x i64> %a to <16 x bfloat> + ret <16 x bfloat> %c +} + +define <32 x bfloat> @stofp_v32i64_v32bf16(<32 x i64> %a) { +; CHECK-LABEL: stofp_v32i64_v32bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: scvtf v16.2d, v2.2d +; CHECK-NEXT: scvtf v17.2d, v0.2d +; CHECK-NEXT: scvtf v18.2d, v3.2d +; CHECK-NEXT: scvtf v19.2d, v6.2d +; CHECK-NEXT: ldp q24, q23, [sp, #96] +; CHECK-NEXT: scvtf v21.2d, v1.2d +; CHECK-NEXT: scvtf v22.2d, v4.2d +; CHECK-NEXT: scvtf v6.2d, v7.2d +; CHECK-NEXT: scvtf v7.2d, v5.2d +; CHECK-NEXT: movi v3.4s, #127, msl #8 +; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: fcvtn v0.2s, v16.2d +; CHECK-NEXT: ldp q20, q16, [sp, #32] +; CHECK-NEXT: fcvtn v1.2s, v17.2d +; CHECK-NEXT: ldp q5, q17, [sp] +; CHECK-NEXT: fcvtn v4.2s, v19.2d +; CHECK-NEXT: scvtf v23.2d, v23.2d +; CHECK-NEXT: scvtf v20.2d, v20.2d +; CHECK-NEXT: scvtf v16.2d, v16.2d +; CHECK-NEXT: fcvtn2 v0.4s, v18.2d +; CHECK-NEXT: ldp q19, q18, [sp, #64] +; CHECK-NEXT: scvtf v25.2d, v5.2d +; CHECK-NEXT: fcvtn v5.2s, v22.2d +; CHECK-NEXT: fcvtn2 v1.4s, v21.2d +; CHECK-NEXT: scvtf v21.2d, v24.2d +; CHECK-NEXT: scvtf v17.2d, v17.2d +; CHECK-NEXT: fcvtn2 v4.4s, v6.2d +; CHECK-NEXT: scvtf v19.2d, v19.2d +; CHECK-NEXT: scvtf v6.2d, v18.2d +; CHECK-NEXT: fcvtn v18.2s, v20.2d +; CHECK-NEXT: ushr v22.4s, v0.4s, #16 +; CHECK-NEXT: add v20.4s, v0.4s, v3.4s +; CHECK-NEXT: fcvtn2 v5.4s, v7.2d +; CHECK-NEXT: fcvtn v24.2s, v25.2d +; CHECK-NEXT: ushr v7.4s, v1.4s, #16 +; CHECK-NEXT: fcvtn v21.2s, v21.2d +; CHECK-NEXT: add v26.4s, v1.4s, v3.4s +; CHECK-NEXT: ushr v27.4s, v4.4s, #16 +; CHECK-NEXT: fcvtn v19.2s, v19.2d +; CHECK-NEXT: fcvtn2 v18.4s, v16.2d +; CHECK-NEXT: and v22.16b, v22.16b, v2.16b +; CHECK-NEXT: and v7.16b, v7.16b, v2.16b +; CHECK-NEXT: fcmeq v25.4s, v0.4s, v0.4s +; CHECK-NEXT: orr v0.4s, #64, lsl #16 +; CHECK-NEXT: ushr v28.4s, v5.4s, #16 +; CHECK-NEXT: fcvtn2 v24.4s, v17.2d +; CHECK-NEXT: add v17.4s, v5.4s, v3.4s +; CHECK-NEXT: fcvtn2 v21.4s, v23.2d +; CHECK-NEXT: and v16.16b, v27.16b, v2.16b +; CHECK-NEXT: add v20.4s, v22.4s, v20.4s +; CHECK-NEXT: fcvtn2 v19.4s, v6.2d +; CHECK-NEXT: add v7.4s, v7.4s, v26.4s +; CHECK-NEXT: ushr v26.4s, v18.4s, #16 +; CHECK-NEXT: and v23.16b, v28.16b, v2.16b +; CHECK-NEXT: add v22.4s, v4.4s, v3.4s +; CHECK-NEXT: fcmeq v6.4s, v1.4s, v1.4s +; CHECK-NEXT: ushr v27.4s, v24.4s, #16 +; CHECK-NEXT: add v30.4s, v24.4s, v3.4s +; CHECK-NEXT: orr v1.4s, #64, lsl #16 +; CHECK-NEXT: ushr v28.4s, v21.4s, #16 +; CHECK-NEXT: add v31.4s, v21.4s, v3.4s +; CHECK-NEXT: and v26.16b, v26.16b, v2.16b +; CHECK-NEXT: add v17.4s, v23.4s, v17.4s +; CHECK-NEXT: add v23.4s, v18.4s, v3.4s +; CHECK-NEXT: ushr v29.4s, v19.4s, #16 +; CHECK-NEXT: and v27.16b, v27.16b, v2.16b +; CHECK-NEXT: add v3.4s, v19.4s, v3.4s +; CHECK-NEXT: add v16.4s, v16.4s, v22.4s +; CHECK-NEXT: and v28.16b, v28.16b, v2.16b +; CHECK-NEXT: fcmeq v22.4s, v4.4s, v4.4s +; CHECK-NEXT: orr v4.4s, #64, lsl #16 +; CHECK-NEXT: and v2.16b, v29.16b, v2.16b +; CHECK-NEXT: fcmeq v29.4s, v5.4s, v5.4s +; CHECK-NEXT: orr v5.4s, #64, lsl #16 +; CHECK-NEXT: add v23.4s, v26.4s, v23.4s +; CHECK-NEXT: fcmeq v26.4s, v18.4s, v18.4s +; CHECK-NEXT: add v27.4s, v27.4s, v30.4s +; CHECK-NEXT: fcmeq v30.4s, v24.4s, v24.4s +; CHECK-NEXT: add v28.4s, v28.4s, v31.4s +; CHECK-NEXT: fcmeq v31.4s, v21.4s, v21.4s +; CHECK-NEXT: add v2.4s, v2.4s, v3.4s +; CHECK-NEXT: fcmeq v3.4s, v19.4s, v19.4s +; CHECK-NEXT: orr v18.4s, #64, lsl #16 +; CHECK-NEXT: orr v24.4s, #64, lsl #16 +; CHECK-NEXT: orr v21.4s, #64, lsl #16 +; CHECK-NEXT: orr v19.4s, #64, lsl #16 +; CHECK-NEXT: bit v1.16b, v7.16b, v6.16b +; CHECK-NEXT: bit v4.16b, v16.16b, v22.16b +; CHECK-NEXT: mov v6.16b, v26.16b +; CHECK-NEXT: mov v7.16b, v30.16b +; CHECK-NEXT: mov v16.16b, v31.16b +; CHECK-NEXT: bit v0.16b, v20.16b, v25.16b +; CHECK-NEXT: bit v5.16b, v17.16b, v29.16b +; CHECK-NEXT: bsl v3.16b, v2.16b, v19.16b +; CHECK-NEXT: bsl v6.16b, v23.16b, v18.16b +; CHECK-NEXT: bsl v7.16b, v27.16b, v24.16b +; CHECK-NEXT: bsl v16.16b, v28.16b, v21.16b +; CHECK-NEXT: uzp2 v0.8h, v1.8h, v0.8h +; CHECK-NEXT: uzp2 v1.8h, v5.8h, v4.8h +; CHECK-NEXT: uzp2 v2.8h, v7.8h, v6.8h +; CHECK-NEXT: uzp2 v3.8h, v3.8h, v16.8h +; CHECK-NEXT: ret +entry: + %c = sitofp <32 x i64> %a to <32 x bfloat> + ret <32 x bfloat> %c +} + +define <32 x bfloat> @utofp_v32i64_v32bf16(<32 x i64> %a) { +; CHECK-LABEL: utofp_v32i64_v32bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ucvtf v16.2d, v2.2d +; CHECK-NEXT: ucvtf v17.2d, v0.2d +; CHECK-NEXT: ucvtf v18.2d, v3.2d +; CHECK-NEXT: ucvtf v19.2d, v6.2d +; CHECK-NEXT: ldp q24, q23, [sp, #96] +; CHECK-NEXT: ucvtf v21.2d, v1.2d +; CHECK-NEXT: ucvtf v22.2d, v4.2d +; CHECK-NEXT: ucvtf v6.2d, v7.2d +; CHECK-NEXT: ucvtf v7.2d, v5.2d +; CHECK-NEXT: movi v3.4s, #127, msl #8 +; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: fcvtn v0.2s, v16.2d +; CHECK-NEXT: ldp q20, q16, [sp, #32] +; CHECK-NEXT: fcvtn v1.2s, v17.2d +; CHECK-NEXT: ldp q5, q17, [sp] +; CHECK-NEXT: fcvtn v4.2s, v19.2d +; CHECK-NEXT: ucvtf v23.2d, v23.2d +; CHECK-NEXT: ucvtf v20.2d, v20.2d +; CHECK-NEXT: ucvtf v16.2d, v16.2d +; CHECK-NEXT: fcvtn2 v0.4s, v18.2d +; CHECK-NEXT: ldp q19, q18, [sp, #64] +; CHECK-NEXT: ucvtf v25.2d, v5.2d +; CHECK-NEXT: fcvtn v5.2s, v22.2d +; CHECK-NEXT: fcvtn2 v1.4s, v21.2d +; CHECK-NEXT: ucvtf v21.2d, v24.2d +; CHECK-NEXT: ucvtf v17.2d, v17.2d +; CHECK-NEXT: fcvtn2 v4.4s, v6.2d +; CHECK-NEXT: ucvtf v19.2d, v19.2d +; CHECK-NEXT: ucvtf v6.2d, v18.2d +; CHECK-NEXT: fcvtn v18.2s, v20.2d +; CHECK-NEXT: ushr v22.4s, v0.4s, #16 +; CHECK-NEXT: add v20.4s, v0.4s, v3.4s +; CHECK-NEXT: fcvtn2 v5.4s, v7.2d +; CHECK-NEXT: fcvtn v24.2s, v25.2d +; CHECK-NEXT: ushr v7.4s, v1.4s, #16 +; CHECK-NEXT: fcvtn v21.2s, v21.2d +; CHECK-NEXT: add v26.4s, v1.4s, v3.4s +; CHECK-NEXT: ushr v27.4s, v4.4s, #16 +; CHECK-NEXT: fcvtn v19.2s, v19.2d +; CHECK-NEXT: fcvtn2 v18.4s, v16.2d +; CHECK-NEXT: and v22.16b, v22.16b, v2.16b +; CHECK-NEXT: and v7.16b, v7.16b, v2.16b +; CHECK-NEXT: fcmeq v25.4s, v0.4s, v0.4s +; CHECK-NEXT: orr v0.4s, #64, lsl #16 +; CHECK-NEXT: ushr v28.4s, v5.4s, #16 +; CHECK-NEXT: fcvtn2 v24.4s, v17.2d +; CHECK-NEXT: add v17.4s, v5.4s, v3.4s +; CHECK-NEXT: fcvtn2 v21.4s, v23.2d +; CHECK-NEXT: and v16.16b, v27.16b, v2.16b +; CHECK-NEXT: add v20.4s, v22.4s, v20.4s +; CHECK-NEXT: fcvtn2 v19.4s, v6.2d +; CHECK-NEXT: add v7.4s, v7.4s, v26.4s +; CHECK-NEXT: ushr v26.4s, v18.4s, #16 +; CHECK-NEXT: and v23.16b, v28.16b, v2.16b +; CHECK-NEXT: add v22.4s, v4.4s, v3.4s +; CHECK-NEXT: fcmeq v6.4s, v1.4s, v1.4s +; CHECK-NEXT: ushr v27.4s, v24.4s, #16 +; CHECK-NEXT: add v30.4s, v24.4s, v3.4s +; CHECK-NEXT: orr v1.4s, #64, lsl #16 +; CHECK-NEXT: ushr v28.4s, v21.4s, #16 +; CHECK-NEXT: add v31.4s, v21.4s, v3.4s +; CHECK-NEXT: and v26.16b, v26.16b, v2.16b +; CHECK-NEXT: add v17.4s, v23.4s, v17.4s +; CHECK-NEXT: add v23.4s, v18.4s, v3.4s +; CHECK-NEXT: ushr v29.4s, v19.4s, #16 +; CHECK-NEXT: and v27.16b, v27.16b, v2.16b +; CHECK-NEXT: add v3.4s, v19.4s, v3.4s +; CHECK-NEXT: add v16.4s, v16.4s, v22.4s +; CHECK-NEXT: and v28.16b, v28.16b, v2.16b +; CHECK-NEXT: fcmeq v22.4s, v4.4s, v4.4s +; CHECK-NEXT: orr v4.4s, #64, lsl #16 +; CHECK-NEXT: and v2.16b, v29.16b, v2.16b +; CHECK-NEXT: fcmeq v29.4s, v5.4s, v5.4s +; CHECK-NEXT: orr v5.4s, #64, lsl #16 +; CHECK-NEXT: add v23.4s, v26.4s, v23.4s +; CHECK-NEXT: fcmeq v26.4s, v18.4s, v18.4s +; CHECK-NEXT: add v27.4s, v27.4s, v30.4s +; CHECK-NEXT: fcmeq v30.4s, v24.4s, v24.4s +; CHECK-NEXT: add v28.4s, v28.4s, v31.4s +; CHECK-NEXT: fcmeq v31.4s, v21.4s, v21.4s +; CHECK-NEXT: add v2.4s, v2.4s, v3.4s +; CHECK-NEXT: fcmeq v3.4s, v19.4s, v19.4s +; CHECK-NEXT: orr v18.4s, #64, lsl #16 +; CHECK-NEXT: orr v24.4s, #64, lsl #16 +; CHECK-NEXT: orr v21.4s, #64, lsl #16 +; CHECK-NEXT: orr v19.4s, #64, lsl #16 +; CHECK-NEXT: bit v1.16b, v7.16b, v6.16b +; CHECK-NEXT: bit v4.16b, v16.16b, v22.16b +; CHECK-NEXT: mov v6.16b, v26.16b +; CHECK-NEXT: mov v7.16b, v30.16b +; CHECK-NEXT: mov v16.16b, v31.16b +; CHECK-NEXT: bit v0.16b, v20.16b, v25.16b +; CHECK-NEXT: bit v5.16b, v17.16b, v29.16b +; CHECK-NEXT: bsl v3.16b, v2.16b, v19.16b +; CHECK-NEXT: bsl v6.16b, v23.16b, v18.16b +; CHECK-NEXT: bsl v7.16b, v27.16b, v24.16b +; CHECK-NEXT: bsl v16.16b, v28.16b, v21.16b +; CHECK-NEXT: uzp2 v0.8h, v1.8h, v0.8h +; CHECK-NEXT: uzp2 v1.8h, v5.8h, v4.8h +; CHECK-NEXT: uzp2 v2.8h, v7.8h, v6.8h +; CHECK-NEXT: uzp2 v3.8h, v3.8h, v16.8h +; CHECK-NEXT: ret +entry: + %c = uitofp <32 x i64> %a to <32 x bfloat> + ret <32 x bfloat> %c +} + +define <2 x bfloat> @stofp_v2i32_v2bf16(<2 x i32> %a) { +; CHECK-LABEL: stofp_v2i32_v2bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: scvtf v0.4s, v0.4s +; CHECK-NEXT: ushr v2.4s, v0.4s, #16 +; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: ret +entry: + %c = sitofp <2 x i32> %a to <2 x bfloat> + ret <2 x bfloat> %c +} + +define <2 x bfloat> @utofp_v2i32_v2bf16(<2 x i32> %a) { +; CHECK-LABEL: utofp_v2i32_v2bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: ucvtf v0.4s, v0.4s +; CHECK-NEXT: ushr v2.4s, v0.4s, #16 +; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: ret +entry: + %c = uitofp <2 x i32> %a to <2 x bfloat> + ret <2 x bfloat> %c +} + +define <3 x bfloat> @stofp_v3i32_v3bf16(<3 x i32> %a) { +; CHECK-LABEL: stofp_v3i32_v3bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: scvtf v0.4s, v0.4s +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: ushr v2.4s, v0.4s, #16 +; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: ret +entry: + %c = sitofp <3 x i32> %a to <3 x bfloat> + ret <3 x bfloat> %c +} + +define <3 x bfloat> @utofp_v3i32_v3bf16(<3 x i32> %a) { +; CHECK-LABEL: utofp_v3i32_v3bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ucvtf v0.4s, v0.4s +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: ushr v2.4s, v0.4s, #16 +; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: ret +entry: + %c = uitofp <3 x i32> %a to <3 x bfloat> + ret <3 x bfloat> %c +} + +define <4 x bfloat> @stofp_v4i32_v4bf16(<4 x i32> %a) { +; CHECK-LABEL: stofp_v4i32_v4bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: scvtf v0.4s, v0.4s +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: ushr v2.4s, v0.4s, #16 +; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: ret +entry: + %c = sitofp <4 x i32> %a to <4 x bfloat> + ret <4 x bfloat> %c +} + +define <4 x bfloat> @utofp_v4i32_v4bf16(<4 x i32> %a) { +; CHECK-LABEL: utofp_v4i32_v4bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ucvtf v0.4s, v0.4s +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: ushr v2.4s, v0.4s, #16 +; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: ret +entry: + %c = uitofp <4 x i32> %a to <4 x bfloat> + ret <4 x bfloat> %c +} + +define <8 x bfloat> @stofp_v8i32_v8bf16(<8 x i32> %a) { +; CHECK-LABEL: stofp_v8i32_v8bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: scvtf v0.4s, v0.4s +; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: scvtf v1.4s, v1.4s +; CHECK-NEXT: ushr v3.4s, v0.4s, #16 +; CHECK-NEXT: ushr v4.4s, v1.4s, #16 +; CHECK-NEXT: and v3.16b, v3.16b, v2.16b +; CHECK-NEXT: and v2.16b, v4.16b, v2.16b +; CHECK-NEXT: add v0.4s, v3.4s, v0.4s +; CHECK-NEXT: movi v3.4s, #127, msl #8 +; CHECK-NEXT: add v1.4s, v2.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v3.4s +; CHECK-NEXT: addhn2 v0.8h, v1.4s, v3.4s +; CHECK-NEXT: ret +entry: + %c = sitofp <8 x i32> %a to <8 x bfloat> + ret <8 x bfloat> %c +} + +define <8 x bfloat> @utofp_v8i32_v8bf16(<8 x i32> %a) { +; CHECK-LABEL: utofp_v8i32_v8bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ucvtf v0.4s, v0.4s +; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: ucvtf v1.4s, v1.4s +; CHECK-NEXT: ushr v3.4s, v0.4s, #16 +; CHECK-NEXT: ushr v4.4s, v1.4s, #16 +; CHECK-NEXT: and v3.16b, v3.16b, v2.16b +; CHECK-NEXT: and v2.16b, v4.16b, v2.16b +; CHECK-NEXT: add v0.4s, v3.4s, v0.4s +; CHECK-NEXT: movi v3.4s, #127, msl #8 +; CHECK-NEXT: add v1.4s, v2.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v3.4s +; CHECK-NEXT: addhn2 v0.8h, v1.4s, v3.4s +; CHECK-NEXT: ret +entry: + %c = uitofp <8 x i32> %a to <8 x bfloat> + ret <8 x bfloat> %c +} + +define <16 x bfloat> @stofp_v16i32_v16bf16(<16 x i32> %a) { +; CHECK-LABEL: stofp_v16i32_v16bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: scvtf v0.4s, v0.4s +; CHECK-NEXT: scvtf v2.4s, v2.4s +; CHECK-NEXT: movi v4.4s, #1 +; CHECK-NEXT: scvtf v1.4s, v1.4s +; CHECK-NEXT: scvtf v3.4s, v3.4s +; CHECK-NEXT: ushr v5.4s, v0.4s, #16 +; CHECK-NEXT: ushr v6.4s, v2.4s, #16 +; CHECK-NEXT: ushr v7.4s, v1.4s, #16 +; CHECK-NEXT: ushr v16.4s, v3.4s, #16 +; CHECK-NEXT: and v5.16b, v5.16b, v4.16b +; CHECK-NEXT: and v6.16b, v6.16b, v4.16b +; CHECK-NEXT: add v0.4s, v5.4s, v0.4s +; CHECK-NEXT: add v2.4s, v6.4s, v2.4s +; CHECK-NEXT: movi v6.4s, #127, msl #8 +; CHECK-NEXT: and v5.16b, v7.16b, v4.16b +; CHECK-NEXT: and v4.16b, v16.16b, v4.16b +; CHECK-NEXT: add v5.4s, v5.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v6.4s +; CHECK-NEXT: add v3.4s, v4.4s, v3.4s +; CHECK-NEXT: addhn v1.4h, v2.4s, v6.4s +; CHECK-NEXT: addhn2 v0.8h, v5.4s, v6.4s +; CHECK-NEXT: addhn2 v1.8h, v3.4s, v6.4s +; CHECK-NEXT: ret +entry: + %c = sitofp <16 x i32> %a to <16 x bfloat> + ret <16 x bfloat> %c +} + +define <16 x bfloat> @utofp_v16i32_v16bf16(<16 x i32> %a) { +; CHECK-LABEL: utofp_v16i32_v16bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ucvtf v0.4s, v0.4s +; CHECK-NEXT: ucvtf v2.4s, v2.4s +; CHECK-NEXT: movi v4.4s, #1 +; CHECK-NEXT: ucvtf v1.4s, v1.4s +; CHECK-NEXT: ucvtf v3.4s, v3.4s +; CHECK-NEXT: ushr v5.4s, v0.4s, #16 +; CHECK-NEXT: ushr v6.4s, v2.4s, #16 +; CHECK-NEXT: ushr v7.4s, v1.4s, #16 +; CHECK-NEXT: ushr v16.4s, v3.4s, #16 +; CHECK-NEXT: and v5.16b, v5.16b, v4.16b +; CHECK-NEXT: and v6.16b, v6.16b, v4.16b +; CHECK-NEXT: add v0.4s, v5.4s, v0.4s +; CHECK-NEXT: add v2.4s, v6.4s, v2.4s +; CHECK-NEXT: movi v6.4s, #127, msl #8 +; CHECK-NEXT: and v5.16b, v7.16b, v4.16b +; CHECK-NEXT: and v4.16b, v16.16b, v4.16b +; CHECK-NEXT: add v5.4s, v5.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v6.4s +; CHECK-NEXT: add v3.4s, v4.4s, v3.4s +; CHECK-NEXT: addhn v1.4h, v2.4s, v6.4s +; CHECK-NEXT: addhn2 v0.8h, v5.4s, v6.4s +; CHECK-NEXT: addhn2 v1.8h, v3.4s, v6.4s +; CHECK-NEXT: ret +entry: + %c = uitofp <16 x i32> %a to <16 x bfloat> + ret <16 x bfloat> %c +} + +define <32 x bfloat> @stofp_v32i32_v32bf16(<32 x i32> %a) { +; CHECK-LABEL: stofp_v32i32_v32bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: scvtf v0.4s, v0.4s +; CHECK-NEXT: scvtf v2.4s, v2.4s +; CHECK-NEXT: scvtf v4.4s, v4.4s +; CHECK-NEXT: scvtf v6.4s, v6.4s +; CHECK-NEXT: movi v16.4s, #1 +; CHECK-NEXT: scvtf v1.4s, v1.4s +; CHECK-NEXT: scvtf v3.4s, v3.4s +; CHECK-NEXT: scvtf v5.4s, v5.4s +; CHECK-NEXT: scvtf v7.4s, v7.4s +; CHECK-NEXT: ushr v17.4s, v0.4s, #16 +; CHECK-NEXT: ushr v18.4s, v2.4s, #16 +; CHECK-NEXT: ushr v19.4s, v4.4s, #16 +; CHECK-NEXT: ushr v20.4s, v6.4s, #16 +; CHECK-NEXT: ushr v21.4s, v1.4s, #16 +; CHECK-NEXT: ushr v22.4s, v3.4s, #16 +; CHECK-NEXT: ushr v23.4s, v5.4s, #16 +; CHECK-NEXT: and v17.16b, v17.16b, v16.16b +; CHECK-NEXT: and v18.16b, v18.16b, v16.16b +; CHECK-NEXT: and v19.16b, v19.16b, v16.16b +; CHECK-NEXT: and v20.16b, v20.16b, v16.16b +; CHECK-NEXT: and v21.16b, v21.16b, v16.16b +; CHECK-NEXT: and v22.16b, v22.16b, v16.16b +; CHECK-NEXT: add v0.4s, v17.4s, v0.4s +; CHECK-NEXT: ushr v17.4s, v7.4s, #16 +; CHECK-NEXT: add v2.4s, v18.4s, v2.4s +; CHECK-NEXT: movi v18.4s, #127, msl #8 +; CHECK-NEXT: add v4.4s, v19.4s, v4.4s +; CHECK-NEXT: add v6.4s, v20.4s, v6.4s +; CHECK-NEXT: and v19.16b, v23.16b, v16.16b +; CHECK-NEXT: add v20.4s, v22.4s, v3.4s +; CHECK-NEXT: and v16.16b, v17.16b, v16.16b +; CHECK-NEXT: add v17.4s, v21.4s, v1.4s +; CHECK-NEXT: add v5.4s, v19.4s, v5.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v18.4s +; CHECK-NEXT: addhn v1.4h, v2.4s, v18.4s +; CHECK-NEXT: addhn v2.4h, v4.4s, v18.4s +; CHECK-NEXT: add v4.4s, v16.4s, v7.4s +; CHECK-NEXT: addhn v3.4h, v6.4s, v18.4s +; CHECK-NEXT: addhn2 v0.8h, v17.4s, v18.4s +; CHECK-NEXT: addhn2 v1.8h, v20.4s, v18.4s +; CHECK-NEXT: addhn2 v2.8h, v5.4s, v18.4s +; CHECK-NEXT: addhn2 v3.8h, v4.4s, v18.4s +; CHECK-NEXT: ret +entry: + %c = sitofp <32 x i32> %a to <32 x bfloat> + ret <32 x bfloat> %c +} + +define <32 x bfloat> @utofp_v32i32_v32bf16(<32 x i32> %a) { +; CHECK-LABEL: utofp_v32i32_v32bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ucvtf v0.4s, v0.4s +; CHECK-NEXT: ucvtf v2.4s, v2.4s +; CHECK-NEXT: ucvtf v4.4s, v4.4s +; CHECK-NEXT: ucvtf v6.4s, v6.4s +; CHECK-NEXT: movi v16.4s, #1 +; CHECK-NEXT: ucvtf v1.4s, v1.4s +; CHECK-NEXT: ucvtf v3.4s, v3.4s +; CHECK-NEXT: ucvtf v5.4s, v5.4s +; CHECK-NEXT: ucvtf v7.4s, v7.4s +; CHECK-NEXT: ushr v17.4s, v0.4s, #16 +; CHECK-NEXT: ushr v18.4s, v2.4s, #16 +; CHECK-NEXT: ushr v19.4s, v4.4s, #16 +; CHECK-NEXT: ushr v20.4s, v6.4s, #16 +; CHECK-NEXT: ushr v21.4s, v1.4s, #16 +; CHECK-NEXT: ushr v22.4s, v3.4s, #16 +; CHECK-NEXT: ushr v23.4s, v5.4s, #16 +; CHECK-NEXT: and v17.16b, v17.16b, v16.16b +; CHECK-NEXT: and v18.16b, v18.16b, v16.16b +; CHECK-NEXT: and v19.16b, v19.16b, v16.16b +; CHECK-NEXT: and v20.16b, v20.16b, v16.16b +; CHECK-NEXT: and v21.16b, v21.16b, v16.16b +; CHECK-NEXT: and v22.16b, v22.16b, v16.16b +; CHECK-NEXT: add v0.4s, v17.4s, v0.4s +; CHECK-NEXT: ushr v17.4s, v7.4s, #16 +; CHECK-NEXT: add v2.4s, v18.4s, v2.4s +; CHECK-NEXT: movi v18.4s, #127, msl #8 +; CHECK-NEXT: add v4.4s, v19.4s, v4.4s +; CHECK-NEXT: add v6.4s, v20.4s, v6.4s +; CHECK-NEXT: and v19.16b, v23.16b, v16.16b +; CHECK-NEXT: add v20.4s, v22.4s, v3.4s +; CHECK-NEXT: and v16.16b, v17.16b, v16.16b +; CHECK-NEXT: add v17.4s, v21.4s, v1.4s +; CHECK-NEXT: add v5.4s, v19.4s, v5.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v18.4s +; CHECK-NEXT: addhn v1.4h, v2.4s, v18.4s +; CHECK-NEXT: addhn v2.4h, v4.4s, v18.4s +; CHECK-NEXT: add v4.4s, v16.4s, v7.4s +; CHECK-NEXT: addhn v3.4h, v6.4s, v18.4s +; CHECK-NEXT: addhn2 v0.8h, v17.4s, v18.4s +; CHECK-NEXT: addhn2 v1.8h, v20.4s, v18.4s +; CHECK-NEXT: addhn2 v2.8h, v5.4s, v18.4s +; CHECK-NEXT: addhn2 v3.8h, v4.4s, v18.4s +; CHECK-NEXT: ret +entry: + %c = uitofp <32 x i32> %a to <32 x bfloat> + ret <32 x bfloat> %c +} + +define <2 x bfloat> @stofp_v2i16_v2bf16(<2 x i16> %a) { +; CHECK-LABEL: stofp_v2i16_v2bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: uzp1 v0.4h, v0.4h, v0.4h +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-NEXT: scvtf v0.4s, v0.4s +; CHECK-NEXT: ushr v2.4s, v0.4s, #16 +; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: ret +entry: + %c = sitofp <2 x i16> %a to <2 x bfloat> + ret <2 x bfloat> %c +} + +define <2 x bfloat> @utofp_v2i16_v2bf16(<2 x i16> %a) { +; CHECK-LABEL: utofp_v2i16_v2bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: uzp1 v0.4h, v0.4h, v0.4h +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-NEXT: ucvtf v0.4s, v0.4s +; CHECK-NEXT: ushr v2.4s, v0.4s, #16 +; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: ret +entry: + %c = uitofp <2 x i16> %a to <2 x bfloat> + ret <2 x bfloat> %c +} + +define <3 x bfloat> @stofp_v3i16_v3bf16(<3 x i16> %a) { +; CHECK-LABEL: stofp_v3i16_v3bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: scvtf v0.4s, v0.4s +; CHECK-NEXT: ushr v2.4s, v0.4s, #16 +; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: ret +entry: + %c = sitofp <3 x i16> %a to <3 x bfloat> + ret <3 x bfloat> %c +} + +define <3 x bfloat> @utofp_v3i16_v3bf16(<3 x i16> %a) { +; CHECK-LABEL: utofp_v3i16_v3bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: ucvtf v0.4s, v0.4s +; CHECK-NEXT: ushr v2.4s, v0.4s, #16 +; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: ret +entry: + %c = uitofp <3 x i16> %a to <3 x bfloat> + ret <3 x bfloat> %c +} + +define <4 x bfloat> @stofp_v4i16_v4bf16(<4 x i16> %a) { +; CHECK-LABEL: stofp_v4i16_v4bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: scvtf v0.4s, v0.4s +; CHECK-NEXT: ushr v2.4s, v0.4s, #16 +; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: ret +entry: + %c = sitofp <4 x i16> %a to <4 x bfloat> + ret <4 x bfloat> %c +} + +define <4 x bfloat> @utofp_v4i16_v4bf16(<4 x i16> %a) { +; CHECK-LABEL: utofp_v4i16_v4bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: ucvtf v0.4s, v0.4s +; CHECK-NEXT: ushr v2.4s, v0.4s, #16 +; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: ret +entry: + %c = uitofp <4 x i16> %a to <4 x bfloat> + ret <4 x bfloat> %c +} + +define <8 x bfloat> @stofp_v8i16_v8bf16(<8 x i16> %a) { +; CHECK-LABEL: stofp_v8i16_v8bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: sshll v2.4s, v0.4h, #0 +; CHECK-NEXT: sshll2 v0.4s, v0.8h, #0 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: movi v4.4s, #127, msl #8 +; CHECK-NEXT: scvtf v2.4s, v2.4s +; CHECK-NEXT: scvtf v3.4s, v0.4s +; CHECK-NEXT: ushr v0.4s, v2.4s, #16 +; CHECK-NEXT: ushr v5.4s, v3.4s, #16 +; CHECK-NEXT: and v0.16b, v0.16b, v1.16b +; CHECK-NEXT: and v1.16b, v5.16b, v1.16b +; CHECK-NEXT: add v0.4s, v0.4s, v4.4s +; CHECK-NEXT: add v1.4s, v1.4s, v4.4s +; CHECK-NEXT: addhn v0.4h, v2.4s, v0.4s +; CHECK-NEXT: addhn2 v0.8h, v3.4s, v1.4s +; CHECK-NEXT: ret +entry: + %c = sitofp <8 x i16> %a to <8 x bfloat> + ret <8 x bfloat> %c +} + +define <8 x bfloat> @utofp_v8i16_v8bf16(<8 x i16> %a) { +; CHECK-LABEL: utofp_v8i16_v8bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ushll v2.4s, v0.4h, #0 +; CHECK-NEXT: ushll2 v0.4s, v0.8h, #0 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: movi v4.4s, #127, msl #8 +; CHECK-NEXT: ucvtf v2.4s, v2.4s +; CHECK-NEXT: ucvtf v3.4s, v0.4s +; CHECK-NEXT: ushr v0.4s, v2.4s, #16 +; CHECK-NEXT: ushr v5.4s, v3.4s, #16 +; CHECK-NEXT: and v0.16b, v0.16b, v1.16b +; CHECK-NEXT: and v1.16b, v5.16b, v1.16b +; CHECK-NEXT: add v0.4s, v0.4s, v4.4s +; CHECK-NEXT: add v1.4s, v1.4s, v4.4s +; CHECK-NEXT: addhn v0.4h, v2.4s, v0.4s +; CHECK-NEXT: addhn2 v0.8h, v3.4s, v1.4s +; CHECK-NEXT: ret +entry: + %c = uitofp <8 x i16> %a to <8 x bfloat> + ret <8 x bfloat> %c +} + +define <16 x bfloat> @stofp_v16i16_v16bf16(<16 x i16> %a) { +; CHECK-LABEL: stofp_v16i16_v16bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: sshll v3.4s, v0.4h, #0 +; CHECK-NEXT: sshll v4.4s, v1.4h, #0 +; CHECK-NEXT: sshll2 v0.4s, v0.8h, #0 +; CHECK-NEXT: sshll2 v1.4s, v1.8h, #0 +; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: scvtf v3.4s, v3.4s +; CHECK-NEXT: scvtf v4.4s, v4.4s +; CHECK-NEXT: scvtf v6.4s, v0.4s +; CHECK-NEXT: scvtf v7.4s, v1.4s +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: ushr v5.4s, v3.4s, #16 +; CHECK-NEXT: ushr v0.4s, v4.4s, #16 +; CHECK-NEXT: ushr v16.4s, v6.4s, #16 +; CHECK-NEXT: ushr v17.4s, v7.4s, #16 +; CHECK-NEXT: and v5.16b, v5.16b, v2.16b +; CHECK-NEXT: and v0.16b, v0.16b, v2.16b +; CHECK-NEXT: and v16.16b, v16.16b, v2.16b +; CHECK-NEXT: and v2.16b, v17.16b, v2.16b +; CHECK-NEXT: add v5.4s, v5.4s, v1.4s +; CHECK-NEXT: add v18.4s, v0.4s, v1.4s +; CHECK-NEXT: add v2.4s, v2.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v3.4s, v5.4s +; CHECK-NEXT: add v3.4s, v16.4s, v1.4s +; CHECK-NEXT: addhn v1.4h, v4.4s, v18.4s +; CHECK-NEXT: addhn2 v0.8h, v6.4s, v3.4s +; CHECK-NEXT: addhn2 v1.8h, v7.4s, v2.4s +; CHECK-NEXT: ret +entry: + %c = sitofp <16 x i16> %a to <16 x bfloat> + ret <16 x bfloat> %c +} + +define <16 x bfloat> @utofp_v16i16_v16bf16(<16 x i16> %a) { +; CHECK-LABEL: utofp_v16i16_v16bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ushll v3.4s, v0.4h, #0 +; CHECK-NEXT: ushll v4.4s, v1.4h, #0 +; CHECK-NEXT: ushll2 v0.4s, v0.8h, #0 +; CHECK-NEXT: ushll2 v1.4s, v1.8h, #0 +; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: ucvtf v3.4s, v3.4s +; CHECK-NEXT: ucvtf v4.4s, v4.4s +; CHECK-NEXT: ucvtf v6.4s, v0.4s +; CHECK-NEXT: ucvtf v7.4s, v1.4s +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: ushr v5.4s, v3.4s, #16 +; CHECK-NEXT: ushr v0.4s, v4.4s, #16 +; CHECK-NEXT: ushr v16.4s, v6.4s, #16 +; CHECK-NEXT: ushr v17.4s, v7.4s, #16 +; CHECK-NEXT: and v5.16b, v5.16b, v2.16b +; CHECK-NEXT: and v0.16b, v0.16b, v2.16b +; CHECK-NEXT: and v16.16b, v16.16b, v2.16b +; CHECK-NEXT: and v2.16b, v17.16b, v2.16b +; CHECK-NEXT: add v5.4s, v5.4s, v1.4s +; CHECK-NEXT: add v18.4s, v0.4s, v1.4s +; CHECK-NEXT: add v2.4s, v2.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v3.4s, v5.4s +; CHECK-NEXT: add v3.4s, v16.4s, v1.4s +; CHECK-NEXT: addhn v1.4h, v4.4s, v18.4s +; CHECK-NEXT: addhn2 v0.8h, v6.4s, v3.4s +; CHECK-NEXT: addhn2 v1.8h, v7.4s, v2.4s +; CHECK-NEXT: ret +entry: + %c = uitofp <16 x i16> %a to <16 x bfloat> + ret <16 x bfloat> %c +} + +define <32 x bfloat> @stofp_v32i16_v32bf16(<32 x i16> %a) { +; CHECK-LABEL: stofp_v32i16_v32bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: sshll v5.4s, v1.4h, #0 +; CHECK-NEXT: sshll v4.4s, v0.4h, #0 +; CHECK-NEXT: sshll v6.4s, v2.4h, #0 +; CHECK-NEXT: sshll v7.4s, v3.4h, #0 +; CHECK-NEXT: sshll2 v0.4s, v0.8h, #0 +; CHECK-NEXT: movi v16.4s, #1 +; CHECK-NEXT: sshll2 v1.4s, v1.8h, #0 +; CHECK-NEXT: sshll2 v2.4s, v2.8h, #0 +; CHECK-NEXT: sshll2 v3.4s, v3.8h, #0 +; CHECK-NEXT: scvtf v5.4s, v5.4s +; CHECK-NEXT: scvtf v4.4s, v4.4s +; CHECK-NEXT: scvtf v6.4s, v6.4s +; CHECK-NEXT: scvtf v7.4s, v7.4s +; CHECK-NEXT: scvtf v19.4s, v0.4s +; CHECK-NEXT: movi v18.4s, #127, msl #8 +; CHECK-NEXT: scvtf v20.4s, v1.4s +; CHECK-NEXT: scvtf v21.4s, v2.4s +; CHECK-NEXT: scvtf v22.4s, v3.4s +; CHECK-NEXT: ushr v0.4s, v5.4s, #16 +; CHECK-NEXT: ushr v17.4s, v4.4s, #16 +; CHECK-NEXT: ushr v1.4s, v6.4s, #16 +; CHECK-NEXT: ushr v2.4s, v7.4s, #16 +; CHECK-NEXT: ushr v23.4s, v20.4s, #16 +; CHECK-NEXT: ushr v25.4s, v22.4s, #16 +; CHECK-NEXT: and v0.16b, v0.16b, v16.16b +; CHECK-NEXT: and v3.16b, v17.16b, v16.16b +; CHECK-NEXT: and v1.16b, v1.16b, v16.16b +; CHECK-NEXT: and v2.16b, v2.16b, v16.16b +; CHECK-NEXT: ushr v17.4s, v19.4s, #16 +; CHECK-NEXT: and v23.16b, v23.16b, v16.16b +; CHECK-NEXT: add v24.4s, v0.4s, v18.4s +; CHECK-NEXT: ushr v0.4s, v21.4s, #16 +; CHECK-NEXT: add v3.4s, v3.4s, v18.4s +; CHECK-NEXT: add v26.4s, v1.4s, v18.4s +; CHECK-NEXT: add v27.4s, v2.4s, v18.4s +; CHECK-NEXT: and v17.16b, v17.16b, v16.16b +; CHECK-NEXT: and v28.16b, v0.16b, v16.16b +; CHECK-NEXT: and v16.16b, v25.16b, v16.16b +; CHECK-NEXT: addhn v0.4h, v4.4s, v3.4s +; CHECK-NEXT: addhn v1.4h, v5.4s, v24.4s +; CHECK-NEXT: add v4.4s, v17.4s, v18.4s +; CHECK-NEXT: addhn v2.4h, v6.4s, v26.4s +; CHECK-NEXT: add v5.4s, v23.4s, v18.4s +; CHECK-NEXT: addhn v3.4h, v7.4s, v27.4s +; CHECK-NEXT: add v6.4s, v28.4s, v18.4s +; CHECK-NEXT: add v16.4s, v16.4s, v18.4s +; CHECK-NEXT: addhn2 v0.8h, v19.4s, v4.4s +; CHECK-NEXT: addhn2 v1.8h, v20.4s, v5.4s +; CHECK-NEXT: addhn2 v2.8h, v21.4s, v6.4s +; CHECK-NEXT: addhn2 v3.8h, v22.4s, v16.4s +; CHECK-NEXT: ret +entry: + %c = sitofp <32 x i16> %a to <32 x bfloat> + ret <32 x bfloat> %c +} + +define <32 x bfloat> @utofp_v32i16_v32bf16(<32 x i16> %a) { +; CHECK-LABEL: utofp_v32i16_v32bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ushll v5.4s, v1.4h, #0 +; CHECK-NEXT: ushll v4.4s, v0.4h, #0 +; CHECK-NEXT: ushll v6.4s, v2.4h, #0 +; CHECK-NEXT: ushll v7.4s, v3.4h, #0 +; CHECK-NEXT: ushll2 v0.4s, v0.8h, #0 +; CHECK-NEXT: movi v16.4s, #1 +; CHECK-NEXT: ushll2 v1.4s, v1.8h, #0 +; CHECK-NEXT: ushll2 v2.4s, v2.8h, #0 +; CHECK-NEXT: ushll2 v3.4s, v3.8h, #0 +; CHECK-NEXT: ucvtf v5.4s, v5.4s +; CHECK-NEXT: ucvtf v4.4s, v4.4s +; CHECK-NEXT: ucvtf v6.4s, v6.4s +; CHECK-NEXT: ucvtf v7.4s, v7.4s +; CHECK-NEXT: ucvtf v19.4s, v0.4s +; CHECK-NEXT: movi v18.4s, #127, msl #8 +; CHECK-NEXT: ucvtf v20.4s, v1.4s +; CHECK-NEXT: ucvtf v21.4s, v2.4s +; CHECK-NEXT: ucvtf v22.4s, v3.4s +; CHECK-NEXT: ushr v0.4s, v5.4s, #16 +; CHECK-NEXT: ushr v17.4s, v4.4s, #16 +; CHECK-NEXT: ushr v1.4s, v6.4s, #16 +; CHECK-NEXT: ushr v2.4s, v7.4s, #16 +; CHECK-NEXT: ushr v23.4s, v20.4s, #16 +; CHECK-NEXT: ushr v25.4s, v22.4s, #16 +; CHECK-NEXT: and v0.16b, v0.16b, v16.16b +; CHECK-NEXT: and v3.16b, v17.16b, v16.16b +; CHECK-NEXT: and v1.16b, v1.16b, v16.16b +; CHECK-NEXT: and v2.16b, v2.16b, v16.16b +; CHECK-NEXT: ushr v17.4s, v19.4s, #16 +; CHECK-NEXT: and v23.16b, v23.16b, v16.16b +; CHECK-NEXT: add v24.4s, v0.4s, v18.4s +; CHECK-NEXT: ushr v0.4s, v21.4s, #16 +; CHECK-NEXT: add v3.4s, v3.4s, v18.4s +; CHECK-NEXT: add v26.4s, v1.4s, v18.4s +; CHECK-NEXT: add v27.4s, v2.4s, v18.4s +; CHECK-NEXT: and v17.16b, v17.16b, v16.16b +; CHECK-NEXT: and v28.16b, v0.16b, v16.16b +; CHECK-NEXT: and v16.16b, v25.16b, v16.16b +; CHECK-NEXT: addhn v0.4h, v4.4s, v3.4s +; CHECK-NEXT: addhn v1.4h, v5.4s, v24.4s +; CHECK-NEXT: add v4.4s, v17.4s, v18.4s +; CHECK-NEXT: addhn v2.4h, v6.4s, v26.4s +; CHECK-NEXT: add v5.4s, v23.4s, v18.4s +; CHECK-NEXT: addhn v3.4h, v7.4s, v27.4s +; CHECK-NEXT: add v6.4s, v28.4s, v18.4s +; CHECK-NEXT: add v16.4s, v16.4s, v18.4s +; CHECK-NEXT: addhn2 v0.8h, v19.4s, v4.4s +; CHECK-NEXT: addhn2 v1.8h, v20.4s, v5.4s +; CHECK-NEXT: addhn2 v2.8h, v21.4s, v6.4s +; CHECK-NEXT: addhn2 v3.8h, v22.4s, v16.4s +; CHECK-NEXT: ret +entry: + %c = uitofp <32 x i16> %a to <32 x bfloat> + ret <32 x bfloat> %c +} + +define <2 x bfloat> @stofp_v2i8_v2bf16(<2 x i8> %a) { +; CHECK-LABEL: stofp_v2i8_v2bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NEXT: mov w9, v0.s[1] +; CHECK-NEXT: fmov w10, s0 +; CHECK-NEXT: mov w8, #32767 // =0x7fff +; CHECK-NEXT: sxtb w10, w10 +; CHECK-NEXT: sxtb w9, w9 +; CHECK-NEXT: scvtf s1, w10 +; CHECK-NEXT: scvtf s0, w9 +; CHECK-NEXT: fmov w10, s1 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w12, w10, #16, #1 +; CHECK-NEXT: ubfx w11, w9, #16, #1 +; CHECK-NEXT: add w9, w9, w8 +; CHECK-NEXT: add w8, w10, w8 +; CHECK-NEXT: add w8, w12, w8 +; CHECK-NEXT: add w9, w11, w9 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: lsr w9, w9, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: fmov s1, w9 +; CHECK-NEXT: mov v0.h[1], v1.h[0] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret +entry: + %c = sitofp <2 x i8> %a to <2 x bfloat> + ret <2 x bfloat> %c +} + +define <2 x bfloat> @utofp_v2i8_v2bf16(<2 x i8> %a) { +; CHECK-LABEL: utofp_v2i8_v2bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NEXT: mov w9, v0.s[1] +; CHECK-NEXT: fmov w10, s0 +; CHECK-NEXT: mov w8, #32767 // =0x7fff +; CHECK-NEXT: and w10, w10, #0xff +; CHECK-NEXT: and w9, w9, #0xff +; CHECK-NEXT: ucvtf s1, w10 +; CHECK-NEXT: ucvtf s0, w9 +; CHECK-NEXT: fmov w10, s1 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w12, w10, #16, #1 +; CHECK-NEXT: ubfx w11, w9, #16, #1 +; CHECK-NEXT: add w9, w9, w8 +; CHECK-NEXT: add w8, w10, w8 +; CHECK-NEXT: add w8, w12, w8 +; CHECK-NEXT: add w9, w11, w9 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: lsr w9, w9, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: fmov s1, w9 +; CHECK-NEXT: mov v0.h[1], v1.h[0] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret +entry: + %c = uitofp <2 x i8> %a to <2 x bfloat> + ret <2 x bfloat> %c +} + +define <3 x bfloat> @stofp_v3i8_v3bf16(<3 x i8> %a) { +; CHECK-LABEL: stofp_v3i8_v3bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: fmov s0, w0 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: mov v0.h[1], w1 +; CHECK-NEXT: mov v0.h[2], w2 +; CHECK-NEXT: shl v0.4h, v0.4h, #8 +; CHECK-NEXT: sshr v0.4h, v0.4h, #8 +; CHECK-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-NEXT: scvtf v0.4s, v0.4s +; CHECK-NEXT: ushr v2.4s, v0.4s, #16 +; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: ret +entry: + %c = sitofp <3 x i8> %a to <3 x bfloat> + ret <3 x bfloat> %c +} + +define <3 x bfloat> @utofp_v3i8_v3bf16(<3 x i8> %a) { +; CHECK-LABEL: utofp_v3i8_v3bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: fmov s0, w0 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: mov v0.h[1], w1 +; CHECK-NEXT: mov v0.h[2], w2 +; CHECK-NEXT: bic v0.4h, #255, lsl #8 +; CHECK-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-NEXT: ucvtf v0.4s, v0.4s +; CHECK-NEXT: ushr v2.4s, v0.4s, #16 +; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: ret +entry: + %c = uitofp <3 x i8> %a to <3 x bfloat> + ret <3 x bfloat> %c +} + +define <4 x bfloat> @stofp_v4i8_v4bf16(<4 x i8> %a) { +; CHECK-LABEL: stofp_v4i8_v4bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: shl v0.4h, v0.4h, #8 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: sshr v0.4h, v0.4h, #8 +; CHECK-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-NEXT: scvtf v0.4s, v0.4s +; CHECK-NEXT: ushr v2.4s, v0.4s, #16 +; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: ret +entry: + %c = sitofp <4 x i8> %a to <4 x bfloat> + ret <4 x bfloat> %c +} + +define <4 x bfloat> @utofp_v4i8_v4bf16(<4 x i8> %a) { +; CHECK-LABEL: utofp_v4i8_v4bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: bic v0.4h, #255, lsl #8 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-NEXT: ucvtf v0.4s, v0.4s +; CHECK-NEXT: ushr v2.4s, v0.4s, #16 +; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: ret +entry: + %c = uitofp <4 x i8> %a to <4 x bfloat> + ret <4 x bfloat> %c +} + +define <8 x bfloat> @stofp_v8i8_v8bf16(<8 x i8> %a) { +; CHECK-LABEL: stofp_v8i8_v8bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: movi v4.4s, #127, msl #8 +; CHECK-NEXT: sshll v2.4s, v0.4h, #0 +; CHECK-NEXT: sshll2 v0.4s, v0.8h, #0 +; CHECK-NEXT: scvtf v2.4s, v2.4s +; CHECK-NEXT: scvtf v3.4s, v0.4s +; CHECK-NEXT: ushr v0.4s, v2.4s, #16 +; CHECK-NEXT: ushr v5.4s, v3.4s, #16 +; CHECK-NEXT: and v0.16b, v0.16b, v1.16b +; CHECK-NEXT: and v1.16b, v5.16b, v1.16b +; CHECK-NEXT: add v0.4s, v0.4s, v4.4s +; CHECK-NEXT: add v1.4s, v1.4s, v4.4s +; CHECK-NEXT: addhn v0.4h, v2.4s, v0.4s +; CHECK-NEXT: addhn2 v0.8h, v3.4s, v1.4s +; CHECK-NEXT: ret +entry: + %c = sitofp <8 x i8> %a to <8 x bfloat> + ret <8 x bfloat> %c +} + +define <8 x bfloat> @utofp_v8i8_v8bf16(<8 x i8> %a) { +; CHECK-LABEL: utofp_v8i8_v8bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: movi v4.4s, #127, msl #8 +; CHECK-NEXT: ushll v2.4s, v0.4h, #0 +; CHECK-NEXT: ushll2 v0.4s, v0.8h, #0 +; CHECK-NEXT: ucvtf v2.4s, v2.4s +; CHECK-NEXT: ucvtf v3.4s, v0.4s +; CHECK-NEXT: ushr v0.4s, v2.4s, #16 +; CHECK-NEXT: ushr v5.4s, v3.4s, #16 +; CHECK-NEXT: and v0.16b, v0.16b, v1.16b +; CHECK-NEXT: and v1.16b, v5.16b, v1.16b +; CHECK-NEXT: add v0.4s, v0.4s, v4.4s +; CHECK-NEXT: add v1.4s, v1.4s, v4.4s +; CHECK-NEXT: addhn v0.4h, v2.4s, v0.4s +; CHECK-NEXT: addhn2 v0.8h, v3.4s, v1.4s +; CHECK-NEXT: ret +entry: + %c = uitofp <8 x i8> %a to <8 x bfloat> + ret <8 x bfloat> %c +} + +define <16 x bfloat> @stofp_v16i8_v16bf16(<16 x i8> %a) { +; CHECK-LABEL: stofp_v16i8_v16bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: sshll2 v2.8h, v0.16b, #0 +; CHECK-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: movi v7.4s, #127, msl #8 +; CHECK-NEXT: sshll v3.4s, v2.4h, #0 +; CHECK-NEXT: sshll v4.4s, v0.4h, #0 +; CHECK-NEXT: sshll2 v2.4s, v2.8h, #0 +; CHECK-NEXT: sshll2 v0.4s, v0.8h, #0 +; CHECK-NEXT: scvtf v3.4s, v3.4s +; CHECK-NEXT: scvtf v4.4s, v4.4s +; CHECK-NEXT: scvtf v2.4s, v2.4s +; CHECK-NEXT: scvtf v6.4s, v0.4s +; CHECK-NEXT: ushr v5.4s, v3.4s, #16 +; CHECK-NEXT: ushr v0.4s, v4.4s, #16 +; CHECK-NEXT: ushr v16.4s, v2.4s, #16 +; CHECK-NEXT: ushr v17.4s, v6.4s, #16 +; CHECK-NEXT: and v5.16b, v5.16b, v1.16b +; CHECK-NEXT: and v0.16b, v0.16b, v1.16b +; CHECK-NEXT: and v16.16b, v16.16b, v1.16b +; CHECK-NEXT: and v17.16b, v17.16b, v1.16b +; CHECK-NEXT: add v5.4s, v5.4s, v7.4s +; CHECK-NEXT: add v0.4s, v0.4s, v7.4s +; CHECK-NEXT: addhn v1.4h, v3.4s, v5.4s +; CHECK-NEXT: add v3.4s, v16.4s, v7.4s +; CHECK-NEXT: add v5.4s, v17.4s, v7.4s +; CHECK-NEXT: addhn v0.4h, v4.4s, v0.4s +; CHECK-NEXT: addhn2 v1.8h, v2.4s, v3.4s +; CHECK-NEXT: addhn2 v0.8h, v6.4s, v5.4s +; CHECK-NEXT: ret +entry: + %c = sitofp <16 x i8> %a to <16 x bfloat> + ret <16 x bfloat> %c +} + +define <16 x bfloat> @utofp_v16i8_v16bf16(<16 x i8> %a) { +; CHECK-LABEL: utofp_v16i8_v16bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ushll2 v2.8h, v0.16b, #0 +; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: movi v7.4s, #127, msl #8 +; CHECK-NEXT: ushll v3.4s, v2.4h, #0 +; CHECK-NEXT: ushll v4.4s, v0.4h, #0 +; CHECK-NEXT: ushll2 v2.4s, v2.8h, #0 +; CHECK-NEXT: ushll2 v0.4s, v0.8h, #0 +; CHECK-NEXT: ucvtf v3.4s, v3.4s +; CHECK-NEXT: ucvtf v4.4s, v4.4s +; CHECK-NEXT: ucvtf v2.4s, v2.4s +; CHECK-NEXT: ucvtf v6.4s, v0.4s +; CHECK-NEXT: ushr v5.4s, v3.4s, #16 +; CHECK-NEXT: ushr v0.4s, v4.4s, #16 +; CHECK-NEXT: ushr v16.4s, v2.4s, #16 +; CHECK-NEXT: ushr v17.4s, v6.4s, #16 +; CHECK-NEXT: and v5.16b, v5.16b, v1.16b +; CHECK-NEXT: and v0.16b, v0.16b, v1.16b +; CHECK-NEXT: and v16.16b, v16.16b, v1.16b +; CHECK-NEXT: and v17.16b, v17.16b, v1.16b +; CHECK-NEXT: add v5.4s, v5.4s, v7.4s +; CHECK-NEXT: add v0.4s, v0.4s, v7.4s +; CHECK-NEXT: addhn v1.4h, v3.4s, v5.4s +; CHECK-NEXT: add v3.4s, v16.4s, v7.4s +; CHECK-NEXT: add v5.4s, v17.4s, v7.4s +; CHECK-NEXT: addhn v0.4h, v4.4s, v0.4s +; CHECK-NEXT: addhn2 v1.8h, v2.4s, v3.4s +; CHECK-NEXT: addhn2 v0.8h, v6.4s, v5.4s +; CHECK-NEXT: ret +entry: + %c = uitofp <16 x i8> %a to <16 x bfloat> + ret <16 x bfloat> %c +} + +define <32 x bfloat> @stofp_v32i8_v32bf16(<32 x i8> %a) { +; CHECK-LABEL: stofp_v32i8_v32bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: sshll2 v3.8h, v0.16b, #0 +; CHECK-NEXT: sshll2 v4.8h, v1.16b, #0 +; CHECK-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: movi v20.4s, #127, msl #8 +; CHECK-NEXT: sshll v5.4s, v3.4h, #0 +; CHECK-NEXT: sshll v6.4s, v4.4h, #0 +; CHECK-NEXT: sshll v7.4s, v0.4h, #0 +; CHECK-NEXT: sshll v16.4s, v1.4h, #0 +; CHECK-NEXT: sshll2 v3.4s, v3.8h, #0 +; CHECK-NEXT: sshll2 v4.4s, v4.8h, #0 +; CHECK-NEXT: sshll2 v0.4s, v0.8h, #0 +; CHECK-NEXT: sshll2 v1.4s, v1.8h, #0 +; CHECK-NEXT: scvtf v5.4s, v5.4s +; CHECK-NEXT: scvtf v6.4s, v6.4s +; CHECK-NEXT: scvtf v7.4s, v7.4s +; CHECK-NEXT: scvtf v16.4s, v16.4s +; CHECK-NEXT: scvtf v17.4s, v3.4s +; CHECK-NEXT: scvtf v4.4s, v4.4s +; CHECK-NEXT: scvtf v19.4s, v0.4s +; CHECK-NEXT: scvtf v21.4s, v1.4s +; CHECK-NEXT: ushr v3.4s, v5.4s, #16 +; CHECK-NEXT: ushr v18.4s, v6.4s, #16 +; CHECK-NEXT: ushr v0.4s, v7.4s, #16 +; CHECK-NEXT: ushr v1.4s, v16.4s, #16 +; CHECK-NEXT: ushr v22.4s, v17.4s, #16 +; CHECK-NEXT: ushr v23.4s, v4.4s, #16 +; CHECK-NEXT: ushr v24.4s, v19.4s, #16 +; CHECK-NEXT: ushr v25.4s, v21.4s, #16 +; CHECK-NEXT: and v3.16b, v3.16b, v2.16b +; CHECK-NEXT: and v18.16b, v18.16b, v2.16b +; CHECK-NEXT: and v0.16b, v0.16b, v2.16b +; CHECK-NEXT: and v1.16b, v1.16b, v2.16b +; CHECK-NEXT: and v22.16b, v22.16b, v2.16b +; CHECK-NEXT: and v23.16b, v23.16b, v2.16b +; CHECK-NEXT: and v24.16b, v24.16b, v2.16b +; CHECK-NEXT: and v2.16b, v25.16b, v2.16b +; CHECK-NEXT: add v3.4s, v3.4s, v20.4s +; CHECK-NEXT: add v18.4s, v18.4s, v20.4s +; CHECK-NEXT: add v0.4s, v0.4s, v20.4s +; CHECK-NEXT: add v26.4s, v1.4s, v20.4s +; CHECK-NEXT: addhn v1.4h, v5.4s, v3.4s +; CHECK-NEXT: addhn v3.4h, v6.4s, v18.4s +; CHECK-NEXT: addhn v0.4h, v7.4s, v0.4s +; CHECK-NEXT: add v5.4s, v22.4s, v20.4s +; CHECK-NEXT: add v6.4s, v24.4s, v20.4s +; CHECK-NEXT: add v7.4s, v23.4s, v20.4s +; CHECK-NEXT: add v18.4s, v2.4s, v20.4s +; CHECK-NEXT: addhn v2.4h, v16.4s, v26.4s +; CHECK-NEXT: addhn2 v0.8h, v19.4s, v6.4s +; CHECK-NEXT: addhn2 v1.8h, v17.4s, v5.4s +; CHECK-NEXT: addhn2 v3.8h, v4.4s, v7.4s +; CHECK-NEXT: addhn2 v2.8h, v21.4s, v18.4s +; CHECK-NEXT: ret +entry: + %c = sitofp <32 x i8> %a to <32 x bfloat> + ret <32 x bfloat> %c +} + +define <32 x bfloat> @utofp_v32i8_v32bf16(<32 x i8> %a) { +; CHECK-LABEL: utofp_v32i8_v32bf16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ushll2 v3.8h, v0.16b, #0 +; CHECK-NEXT: ushll2 v4.8h, v1.16b, #0 +; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: movi v20.4s, #127, msl #8 +; CHECK-NEXT: ushll v5.4s, v3.4h, #0 +; CHECK-NEXT: ushll v6.4s, v4.4h, #0 +; CHECK-NEXT: ushll v7.4s, v0.4h, #0 +; CHECK-NEXT: ushll v16.4s, v1.4h, #0 +; CHECK-NEXT: ushll2 v3.4s, v3.8h, #0 +; CHECK-NEXT: ushll2 v4.4s, v4.8h, #0 +; CHECK-NEXT: ushll2 v0.4s, v0.8h, #0 +; CHECK-NEXT: ushll2 v1.4s, v1.8h, #0 +; CHECK-NEXT: ucvtf v5.4s, v5.4s +; CHECK-NEXT: ucvtf v6.4s, v6.4s +; CHECK-NEXT: ucvtf v7.4s, v7.4s +; CHECK-NEXT: ucvtf v16.4s, v16.4s +; CHECK-NEXT: ucvtf v17.4s, v3.4s +; CHECK-NEXT: ucvtf v4.4s, v4.4s +; CHECK-NEXT: ucvtf v19.4s, v0.4s +; CHECK-NEXT: ucvtf v21.4s, v1.4s +; CHECK-NEXT: ushr v3.4s, v5.4s, #16 +; CHECK-NEXT: ushr v18.4s, v6.4s, #16 +; CHECK-NEXT: ushr v0.4s, v7.4s, #16 +; CHECK-NEXT: ushr v1.4s, v16.4s, #16 +; CHECK-NEXT: ushr v22.4s, v17.4s, #16 +; CHECK-NEXT: ushr v23.4s, v4.4s, #16 +; CHECK-NEXT: ushr v24.4s, v19.4s, #16 +; CHECK-NEXT: ushr v25.4s, v21.4s, #16 +; CHECK-NEXT: and v3.16b, v3.16b, v2.16b +; CHECK-NEXT: and v18.16b, v18.16b, v2.16b +; CHECK-NEXT: and v0.16b, v0.16b, v2.16b +; CHECK-NEXT: and v1.16b, v1.16b, v2.16b +; CHECK-NEXT: and v22.16b, v22.16b, v2.16b +; CHECK-NEXT: and v23.16b, v23.16b, v2.16b +; CHECK-NEXT: and v24.16b, v24.16b, v2.16b +; CHECK-NEXT: and v2.16b, v25.16b, v2.16b +; CHECK-NEXT: add v3.4s, v3.4s, v20.4s +; CHECK-NEXT: add v18.4s, v18.4s, v20.4s +; CHECK-NEXT: add v0.4s, v0.4s, v20.4s +; CHECK-NEXT: add v26.4s, v1.4s, v20.4s +; CHECK-NEXT: addhn v1.4h, v5.4s, v3.4s +; CHECK-NEXT: addhn v3.4h, v6.4s, v18.4s +; CHECK-NEXT: addhn v0.4h, v7.4s, v0.4s +; CHECK-NEXT: add v5.4s, v22.4s, v20.4s +; CHECK-NEXT: add v6.4s, v24.4s, v20.4s +; CHECK-NEXT: add v7.4s, v23.4s, v20.4s +; CHECK-NEXT: add v18.4s, v2.4s, v20.4s +; CHECK-NEXT: addhn v2.4h, v16.4s, v26.4s +; CHECK-NEXT: addhn2 v0.8h, v19.4s, v6.4s +; CHECK-NEXT: addhn2 v1.8h, v17.4s, v5.4s +; CHECK-NEXT: addhn2 v3.8h, v4.4s, v7.4s +; CHECK-NEXT: addhn2 v2.8h, v21.4s, v18.4s +; CHECK-NEXT: ret +entry: + %c = uitofp <32 x i8> %a to <32 x bfloat> + ret <32 x bfloat> %c +} -- GitLab From 52b69aa32f5280ce600fbfea1c16a6f17a979c4d Mon Sep 17 00:00:00 2001 From: Peiming Liu <36770114+PeimingLiu@users.noreply.github.com> Date: Mon, 4 Mar 2024 14:39:06 -0800 Subject: [PATCH 057/929] [mlir][sparse] support sparsifying batch levels (#83898) --- .../IR/SparseTensorStorageLayout.h | 5 +- .../SparseTensor/IR/SparseTensorType.h | 7 +- .../SparseTensor/IR/SparseTensorDialect.cpp | 4 +- .../Transforms/SparseAssembler.cpp | 6 +- .../Transforms/SparseTensorCodegen.cpp | 44 ++++-- .../Transforms/SparseTensorRewriting.cpp | 2 +- .../Transforms/Sparsification.cpp | 23 +-- .../Transforms/Utils/CodegenUtils.cpp | 2 +- .../Transforms/Utils/CodegenUtils.h | 2 +- .../Transforms/Utils/LoopEmitter.h | 6 +- .../Transforms/Utils/SparseTensorLevel.cpp | 136 ++++++++++++------ .../Transforms/Utils/SparseTensorLevel.h | 18 ++- .../Dialect/SparseTensor/sparse_batch.mlir | 48 +++++++ .../sparse_conv_2d_slice_based.mlir | 4 +- 14 files changed, 224 insertions(+), 83 deletions(-) create mode 100644 mlir/test/Dialect/SparseTensor/sparse_batch.mlir diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorStorageLayout.h b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorStorageLayout.h index ce34ae43d1c1..7aa9cb611943 100644 --- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorStorageLayout.h +++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorStorageLayout.h @@ -42,7 +42,10 @@ namespace sparse_tensor { /// /// struct sparse_tensor.storage_specifier { /// array lvlSizes ; sizes/cardinalities for each level -/// array memSizes; ; sizes/lengths for each data memref +/// // TODO: memSizes need to be expanded to array<[batch] x n x int> to +/// // support different sizes for different batches. At the moment, we +/// // assume that every batch occupies the same memory size. +/// array memSizes ; sizes/lengths for each data memref /// } /// }; /// diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorType.h b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorType.h index bd2c3c1dd551..dc770c2e904c 100644 --- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorType.h +++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorType.h @@ -247,13 +247,16 @@ public: /// Returns the dimension-shape. ArrayRef getDimShape() const { return rtp.getShape(); } - /// Returns the Level-shape. + /// Returns the level-shape. SmallVector getLvlShape() const { return getEncoding().translateShape(getDimShape(), CrdTransDirectionKind::dim2lvl); } - /// Returns the Level-shape. + /// Returns the batched level-rank. + unsigned getBatchLvlRank() const { return getEncoding().getBatchLvlRank(); } + + /// Returns the batched level-shape. SmallVector getBatchLvlShape() const { auto lvlShape = getEncoding().translateShape( getDimShape(), CrdTransDirectionKind::dim2lvl); diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp index 244a082d0487..6ba8b46370b0 100644 --- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp +++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp @@ -374,7 +374,7 @@ Level SparseTensorEncodingAttr::getLvlRank() const { LevelType SparseTensorEncodingAttr::getLvlType(Level l) const { if (!getImpl()) - return LevelFormat::Dense; + return LevelFormat::Batch; assert(l < getLvlRank() && "Level is out of bounds"); return getLvlTypes()[l]; } @@ -1755,6 +1755,8 @@ LogicalResult ConcatenateOp::verify() { LogicalResult InsertOp::verify() { const auto stt = getSparseTensorType(getTensor()); + if (stt.getEncoding().getBatchLvlRank() > 0) + return emitOpError("batched sparse tensor insertion not implemented"); if (stt.getLvlRank() != static_cast(getLvlCoords().size())) return emitOpError("incorrect number of coordinates"); return success(); diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp index cd6b9b498937..b39a2d9c57d8 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp @@ -33,7 +33,8 @@ static void convTypes(TypeRange types, SmallVectorImpl &convTypes, } // Convert the external representation of the values array. const SparseTensorType stt(cast(type)); - auto shape = {ShapedType::kDynamic}; + auto shape = stt.getBatchLvlShape(); + shape.push_back(ShapedType::kDynamic); auto vtp = RankedTensorType::get(shape, stt.getElementType()); convTypes.push_back(vtp); if (extraTypes) @@ -72,7 +73,8 @@ static void convVals(OpBuilder &builder, Location loc, TypeRange types, // Convert the external representation of the values array. auto rtp = cast(type); const SparseTensorType stt(rtp); - auto shape = {ShapedType::kDynamic}; + auto shape = stt.getBatchLvlShape(); + shape.push_back(ShapedType::kDynamic); SmallVector inputs; SmallVector retTypes; SmallVector cntTypes; diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp index 4e3393195813..eb45a29fb389 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp @@ -429,11 +429,18 @@ static Value genSliceToSize(OpBuilder &builder, Location loc, Value mem, } /// Creates the reassociation array. -static ReassociationIndices getReassociationForFlattening(ShapedType srcTp) { - ReassociationIndices reassociation; - for (int i = 0, e = srcTp.getRank(); i < e; i++) - reassociation.push_back(i); - return reassociation; +static SmallVector +getReassociationForFlattening(ShapedType srcTp, unsigned batchLvls) { + SmallVector ret(batchLvls + 1, {}); + // Create reassociation in the form: + // {0}, {1}, ..., {batchLvl - 1}, {batchLvl, ..., rank} + for (unsigned i = 0; i < batchLvls; i++) + ret[i].push_back(i); + + for (int i = batchLvls, e = srcTp.getRank(); i < e; i++) + ret.back().push_back(i); + + return ret; } //===----------------------------------------------------------------------===// @@ -1287,9 +1294,10 @@ struct SparseAssembleOpConverter : public OpConversionPattern { : op.getLevels()[fIdx]; // TODO: handle batch. TypedValue mem = genToMemref(rewriter, loc, tensor); - if (mem.getType().getRank() > 1) { - // Flattens the buffer to rank 1. - auto reassoc = getReassociationForFlattening(mem.getType()); + if (mem.getType().getRank() > stt.getBatchLvlRank() + 1) { + // Flattens the buffer to batchLvlRank. + auto reassoc = getReassociationForFlattening( + mem.getType(), stt.getBatchLvlRank()); mem = rewriter.create( loc, fType, rewriter.create(loc, mem, reassoc)); @@ -1325,11 +1333,17 @@ struct SparseAssembleOpConverter : public OpConversionPattern { // Sets up the memory size by reading the last value in position array. LevelType lt = stt.getLvlType(lvl); // Simply forwards the position index when this is a dense level. - if (isDenseLT(lt)) { + if (lt.isa()) { memSize = rewriter.create(loc, lvlSize, memSize); posBack = rewriter.create(loc, memSize, c1); continue; } + if (lt.isa()) { + // Skips batch levels as it is not linearized. + // FIXME: this assumes that every batch has the same number of nse, need + // to be generalized to handle varied-size batches. + continue; + } if (isWithPosLT(lt)) { assert(isCompressedLT(lt) || isLooseCompressedLT(lt)); @@ -1343,7 +1357,12 @@ struct SparseAssembleOpConverter : public OpConversionPattern { } desc.setPosMemSize(rewriter, loc, lvl, memSize); // The last value in position array is the memory size for next level. - memSize = genIndexLoad(rewriter, loc, desc.getPosMemRef(lvl), posBack); + // FIXME: this assumes that every batch has the same number of nse, need + // to be generalized to handle varied-size batches. + SmallVector batched(stt.getBatchLvlRank(), + constantIndex(rewriter, loc, 0)); + batched.push_back(posBack); + memSize = genIndexLoad(rewriter, loc, desc.getPosMemRef(lvl), batched); posBack = rewriter.create(loc, posBack, c1); } assert(isWithCrdLT(lt) && lvl <= trailCOOStart); @@ -1413,8 +1432,9 @@ struct SparseDisassembleOpConverter retLen.push_back(genScalarToTensor(rewriter, loc, sz, lvlLenTp)); } Value flatOut = dst; - if (dst.getType().getRank() != 1) { - auto reassoc = getReassociationForFlattening(dst.getType()); + if (dst.getType().getRank() > stt.getBatchLvlRank() + 1) { + auto reassoc = + getReassociationForFlattening(dst.getType(), stt.getBatchLvlRank()); flatOut = rewriter.create(loc, dst, reassoc); } Value dstMem = genSliceToSize(rewriter, loc, flatOut, sz); diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp index ddac602b9a43..158845d88a44 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp @@ -1249,7 +1249,7 @@ public: } Value vals = loopEmitter.getValBuffer()[0]; - Value pos = loopEmitter.getValPosits(0); + SmallVector pos = loopEmitter.getValPosits(0); // Loads the value from sparse tensor using position-index; // loads the value from dense tensor using coords. Value val = enc ? rewriter.create(loc, vals, pos) diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp index 8f2ae60b311f..1fb70ed5035c 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp @@ -86,7 +86,7 @@ static bool findAffine(Merger &merger, TensorId tid, Level lvl, AffineExpr a, case AffineExprKind::Add: case AffineExprKind::Mul: case AffineExprKind::Constant: { - assert(isDenseLT(lt)); + assert(lt.hasDenseSemantic()); if (auto binOp = dyn_cast(a)) { // We do not set dim level format for affine expression like d0 + d1 on // either loop index at d0 or d1. We continue the recursion merely to @@ -211,7 +211,7 @@ static unsigned getNumNonTrivialIdxExpOnSparseLvls(AffineMap map, "AffineMap does not have dimension-rank many results"); unsigned num = 0; for (Level l = 0; l < lvlRank; l++) { - if (!isa(exprs[l]) && !stt.isDenseLvl(l)) + if (!isa(exprs[l]) && !stt.getLvlType(l).hasDenseSemantic()) num++; } return num; @@ -355,8 +355,8 @@ static Value genSubscript(CodegenEnv &env, OpBuilder &builder, OpOperand *t, if (stt.hasEncoding()) { // For sparse tensors we only push the last-level's position onto `args`. const auto pos = env.emitter().getValPosits(tid); - assert(pos); - args.push_back(pos); + assert(!pos.empty()); + args.append(pos); } else { // For dense tensors we push all level's coordinates onto `args`. const Level lvlRank = stt.getLvlRank(); @@ -801,7 +801,7 @@ static bool shouldTryParallize(CodegenEnv &env, LoopId curr, // `CodegenEnv::lt(TensorId, LoopId)`. The returned LT from CodegenEnv // should be consistent with the LT indexed by . const auto lt = env.lt(env.unpackTensorLevel(tidLvl).first, curr); - return isCompressedLT(lt) || isSingletonLT(lt); + return lt.hasSparseSemantic(); }); return isParallelFor(env, /*isOuter=*/curr == 0, isSparse); } @@ -890,15 +890,14 @@ static scf::IfOp genIf(CodegenEnv &env, OpBuilder &builder, LoopId curr, } assert(curr == env.merger().loop(b)); Value clause; - if (isCompressedLT(lt) || isSingletonLT(lt) || - isLooseCompressedLT(lt) || isNOutOfMLT(lt)) { + if (lt.hasSparseSemantic()) { assert(lvl.has_value()); const Value crd = env.emitter().getCoord(tid, *lvl); const Value lvar = env.getLoopVar(curr); clause = builder.create(loc, arith::CmpIPredicate::eq, crd, lvar); } else { - assert(isDenseLT(lt) || isUndefLT(lt)); + assert(lt.hasDenseSemantic() || isUndefLT(lt)); clause = constantI1(builder, loc, true); } cond = cond ? builder.create(loc, cond, clause) : clause; @@ -988,7 +987,7 @@ static bool getAllTidLvlsInLatPoints( hasNonUnique = !isUniqueLT(lt) || hasNonUnique; callback(env.makeTensorLevel(tid, *lvl), nullptr); numloopCond++; - } else if (isDenseLT(lt) || isIdxReduc) { + } else if (lt.hasDenseSemantic() || isIdxReduc) { callback(env.makeTensorLevel(tid, *lvl), nullptr); } else { assert(isUndefLT(lt)); @@ -1010,7 +1009,8 @@ static bool getAllTidLvlsInLatPoints( AffineExpr exp = affines[l]; // Skip simple affine expression and non-dense levels (which // have their own filter loop). - if (isa(exp) || !stt.isDenseLvl(l)) + LevelType lt = stt.getLvlType(l); + if (isa(exp) || !lt.hasDenseSemantic()) continue; // Constant affine expression are handled in genLoop. @@ -1103,7 +1103,8 @@ static void genConstantDenseAddressFromLevel(CodegenEnv &env, assert(lvlExprs.size() == static_cast(lvlRank)); for (Level l = startLvl; l < lvlRank; l++) { AffineExpr lvlExpr = lvlExprs[l]; - if (enc.isDenseLvl(l) && isa(lvlExpr)) + if (enc.getLvlType(l).hasDenseSemantic() && + isa(lvlExpr)) env.emitter().locateLvlAtAffineAddress( builder, loc, env.makeTensorLevel(tid, l), lvlExpr); else diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp index fa570159ba41..89af75dea2a0 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp @@ -175,7 +175,7 @@ Value sparse_tensor::genScalarToTensor(OpBuilder &builder, Location loc, } Value sparse_tensor::genIndexLoad(OpBuilder &builder, Location loc, Value mem, - Value s) { + ValueRange s) { Value load = builder.create(loc, mem, s); if (!isa(load.getType())) { if (load.getType().getIntOrFloatBitWidth() < 64) diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h index e8f6bd1c5eae..ce5831d999e9 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h +++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h @@ -149,7 +149,7 @@ Value genScalarToTensor(OpBuilder &builder, Location loc, Value elem, /// Generates a pointer/index load from the sparse storage scheme. Narrower /// data types need to be zero extended before casting the value into the /// index type used for looping and indexing. -Value genIndexLoad(OpBuilder &builder, Location loc, Value mem, Value s); +Value genIndexLoad(OpBuilder &builder, Location loc, Value mem, ValueRange s); /// Generates a 1-valued attribute of the given type. This supports /// all the same types as `getZeroAttr`; however, unlike `getZeroAttr`, diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.h index 7bfe713cdd9f..b5a0ac8484ab 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.h +++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.h @@ -220,9 +220,11 @@ public: /// /// Getters. /// - Value getValPosits(TensorId tid) const { + SmallVector getValPosits(TensorId tid) const { + SmallVector batchCrds = iters[tid].back().back()->getBatchCrds(); Value lastLvlPos = iters[tid].back().back()->getCurPosition().first; - return lastLvlPos; + batchCrds.push_back(lastLvlPos); + return batchCrds; }; Value getCoord(TensorId tid, Level lvl) const { return getCurIterator(tid, lvl).getCrd(); diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.cpp index 8edacaa9981e..bc27fae5d194 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.cpp @@ -52,8 +52,11 @@ public: Value crdBuffer) : SparseTensorLevel(tid, lvl, lt, lvlSize), crdBuffer(crdBuffer) {} - Value peekCrdAt(OpBuilder &b, Location l, Value iv) const override { - return genIndexLoad(b, l, crdBuffer, iv); + Value peekCrdAt(OpBuilder &b, Location l, ValueRange batchPrefix, + Value iv) const override { + SmallVector memCrd(batchPrefix); + memCrd.push_back(iv); + return genIndexLoad(b, l, crdBuffer, memCrd); } protected: @@ -62,26 +65,35 @@ protected: class DenseLevel : public SparseTensorLevel { public: - DenseLevel(unsigned tid, Level lvl, Value lvlSize, bool encoded) - : SparseTensorLevel(tid, lvl, LevelFormat::Dense, lvlSize), - encoded(encoded) {} + DenseLevel(unsigned tid, Level lvl, Value lvlSize) + : SparseTensorLevel(tid, lvl, LevelFormat::Dense, lvlSize) {} - Value peekCrdAt(OpBuilder &, Location, Value pos) const override { - return pos; + Value peekCrdAt(OpBuilder &, Location, ValueRange, Value) const override { + llvm_unreachable("locate random-accessible level instead"); } - ValuePair peekRangeAt(OpBuilder &b, Location l, Value p, + ValuePair peekRangeAt(OpBuilder &b, Location l, ValueRange, Value p, + Value max) const override { + Value posLo = MULI(p, lvlSize); + return {posLo, lvlSize}; + } +}; + +class BatchLevel : public SparseTensorLevel { +public: + BatchLevel(unsigned tid, Level lvl, Value lvlSize) + : SparseTensorLevel(tid, lvl, LevelFormat::Batch, lvlSize) {} + + Value peekCrdAt(OpBuilder &, Location, ValueRange, Value) const override { + llvm_unreachable("locate random-accessible level instead"); + } + + ValuePair peekRangeAt(OpBuilder &b, Location l, ValueRange, Value p, Value max) const override { assert(max == nullptr && "Dense level can not be non-unique."); - if (encoded) { - Value posLo = MULI(p, lvlSize); - return {posLo, lvlSize}; - } // No need to linearize the position for non-annotated tensors. return {C_IDX(0), lvlSize}; } - - const bool encoded; }; class CompressedLevel : public SparseLevel { @@ -90,14 +102,17 @@ public: Value posBuffer, Value crdBuffer) : SparseLevel(tid, lvl, lt, lvlSize, crdBuffer), posBuffer(posBuffer) {} - ValuePair peekRangeAt(OpBuilder &b, Location l, Value p, - Value max) const override { - if (max == nullptr) { - Value pLo = genIndexLoad(b, l, posBuffer, p); - Value pHi = genIndexLoad(b, l, posBuffer, ADDI(p, C_IDX(1))); - return {pLo, pHi}; - } - llvm_unreachable("compressed-nu should be the first non-unique level."); + ValuePair peekRangeAt(OpBuilder &b, Location l, ValueRange batchPrefix, + Value p, Value max) const override { + assert(max == nullptr && + "compressed level must be the first non-unique level."); + + SmallVector memCrd(batchPrefix); + memCrd.push_back(p); + Value pLo = genIndexLoad(b, l, posBuffer, memCrd); + memCrd.back() = ADDI(p, C_IDX(1)); + Value pHi = genIndexLoad(b, l, posBuffer, memCrd); + return {pLo, pHi}; } private: @@ -110,12 +125,17 @@ public: Value posBuffer, Value crdBuffer) : SparseLevel(tid, lvl, lt, lvlSize, crdBuffer), posBuffer(posBuffer) {} - ValuePair peekRangeAt(OpBuilder &b, Location l, Value p, - Value max) const override { - assert(max == nullptr && "loss compressed level can not be non-unique."); + ValuePair peekRangeAt(OpBuilder &b, Location l, ValueRange batchPrefix, + Value p, Value max) const override { + assert(max == nullptr && + "loose-compressed level must be the first non-unique level."); + SmallVector memCrd(batchPrefix); + p = MULI(p, C_IDX(2)); - Value pLo = genIndexLoad(b, l, posBuffer, p); - Value pHi = genIndexLoad(b, l, posBuffer, ADDI(p, C_IDX(1))); + memCrd.push_back(p); + Value pLo = genIndexLoad(b, l, posBuffer, memCrd); + memCrd.back() = ADDI(p, C_IDX(1)); + Value pHi = genIndexLoad(b, l, posBuffer, memCrd); return {pLo, pHi}; } @@ -129,8 +149,8 @@ public: Value crdBuffer) : SparseLevel(tid, lvl, lt, lvlSize, crdBuffer) {} - ValuePair peekRangeAt(OpBuilder &b, Location l, Value p, - Value segHi) const override { + ValuePair peekRangeAt(OpBuilder &b, Location l, ValueRange batchPrefix, + Value p, Value segHi) const override { if (segHi == nullptr) return {p, ADDI(p, C_IDX(1))}; @@ -145,8 +165,8 @@ public: Value crdBuffer) : SparseLevel(tid, lvl, lt, lvlSize, crdBuffer) {} - ValuePair peekRangeAt(OpBuilder &b, Location l, Value p, - Value max) const override { + ValuePair peekRangeAt(OpBuilder &b, Location l, ValueRange batchPrefix, + Value p, Value max) const override { assert(max == nullptr && isUnique() && "n:m level can not be non-unique."); // Each n:m blk has exactly n specified elements. auto n = getN(lt); @@ -225,7 +245,12 @@ public: return from->kind == IterKind::kTrivial; } - bool randomAccessible() const override { return isDenseLT(stl.getLT()); }; + bool isBatchIterator() const override { + return stl.getLT().isa(); + } + bool randomAccessible() const override { + return stl.getLT().hasDenseSemantic(); + }; bool iteratableByFor() const override { return kind != IterKind::kDedup; }; Value upperBound(OpBuilder &b, Location l) const override { return stl.getSize(); @@ -277,12 +302,19 @@ public: void genInitImpl(OpBuilder &b, Location l, const SparseIterator *parent) override { + + if (isBatchIterator() && batchCrds.size() <= stl.lvl) + batchCrds.resize(stl.lvl + 1, nullptr); + Value pos = C_IDX(0); Value hi = nullptr; - if (parent) + // If the parent iterator is a batch iterator, we also start from 0 (but + // on a different batch). + if (parent && !parent->isBatchIterator()) std::tie(pos, hi) = parent->getCurPosition(); - std::tie(posLo, posHi) = stl.peekRangeAt(b, l, pos, hi); + ValueRange batchPrefix = parent ? parent->getBatchCrds() : ValueRange{}; + std::tie(posLo, posHi) = stl.peekRangeAt(b, l, batchPrefix, pos, hi); // Seek to the lowest position. seek(posLo); } @@ -302,7 +334,7 @@ public: if (randomAccessible()) { updateCrd(SUBI(getItPos(), posLo)); } else { - updateCrd(stl.peekCrdAt(b, l, getItPos())); + updateCrd(stl.peekCrdAt(b, l, getBatchCrds(), getItPos())); } return getCrd(); }; @@ -324,6 +356,11 @@ public: // Seek to the linearized position. seek(ADDI(crd, posLo)); updateCrd(crd); + if (isBatchIterator()) { + // If this is a batch iterator, also update the batch coordinate. + assert(batchCrds.size() > lvl); + batchCrds[lvl] = crd; + } } Value getItPos() const { return getCursor().front(); } @@ -358,11 +395,14 @@ public: Value pos = C_IDX(0); Value hi = nullptr; - if (parent) + // If the parent iterator is a batch iterator, we also start from 0 (but + // on a different batch). + if (parent && !parent->isBatchIterator()) std::tie(pos, hi) = parent->getCurPosition(); Value posLo; - std::tie(posLo, posHi) = stl.peekRangeAt(b, l, pos, hi); + ValueRange batchPrefix = parent ? parent->getBatchCrds() : ValueRange{}; + std::tie(posLo, posHi) = stl.peekRangeAt(b, l, batchPrefix, pos, hi); seek({posLo, genSegmentHigh(b, l, posLo)}); } @@ -384,7 +424,7 @@ public: } Value derefImpl(OpBuilder &b, Location l) override { - updateCrd(stl.peekCrdAt(b, l, getPos())); + updateCrd(stl.peekCrdAt(b, l, getBatchCrds(), getPos())); return getCrd(); }; @@ -440,6 +480,7 @@ public: return wrap->getCursorValTypes(b); } + bool isBatchIterator() const override { return wrap->isBatchIterator(); } bool randomAccessible() const override { return wrap->randomAccessible(); }; bool iteratableByFor() const override { return randomAccessible(); }; Value upperBound(OpBuilder &b, Location l) const override { return size; }; @@ -506,7 +547,8 @@ public: assert(p->lvl + 1 == lvl); maxTupleCnt = MULI(p->maxTupleCnt, p->subSectSz); } - // We don't need an extra buffer to find subsections on dense levels. + // We don't need an extra buffer to find subsections on random-accessible + // levels. if (randomAccessible()) return; subSectPosBuf = allocSubSectPosBuf(b, l); @@ -576,6 +618,7 @@ public: ValueRange inflateSubSectTree(OpBuilder &b, Location l, ValueRange reduc, TraverseBuilder builder) const; + bool isBatchIterator() const override { return delegate->isBatchIterator(); } bool randomAccessible() const override { return delegate->randomAccessible(); }; @@ -689,6 +732,7 @@ public: return ret; } + bool isBatchIterator() const override { return wrap->isBatchIterator(); } bool randomAccessible() const override { return wrap->randomAccessible(); }; bool iteratableByFor() const override { return randomAccessible(); }; Value upperBound(OpBuilder &b, Location l) const override { @@ -783,6 +827,9 @@ void SparseIterator::genInit(OpBuilder &b, Location l, seek(begin->getResults()); return; } + // Inherent batch coordinates from parents. + if (p) + inherentBatch(*p); // TODO: support lowering to function call. return genInitImpl(b, l, p); } @@ -825,6 +872,7 @@ Value SparseIterator::deref(OpBuilder &b, Location l) { } ValueRange SparseIterator::forward(OpBuilder &b, Location l) { + assert(!randomAccessible()); if (emitStrategy == SparseEmitStrategy::kDebugInterface) { std::string prefix = getDebugInterfacePrefix(); Operation *next = b.create(l, b.getStringAttr(prefix + ".next"), @@ -861,8 +909,8 @@ Value DedupIterator::genSegmentHigh(OpBuilder &b, Location l, Value pos) { OpBuilder::InsertionGuard guard(b); // If in bound, load the next coordinates and check duplication. b.setInsertionPointToStart(ifInBound.thenBlock()); - Value headCrd = stl.peekCrdAt(b, l, pos); - Value tailCrd = stl.peekCrdAt(b, l, ivs.front()); + Value headCrd = stl.peekCrdAt(b, l, getBatchCrds(), pos); + Value tailCrd = stl.peekCrdAt(b, l, getBatchCrds(), ivs.front()); Value isDup = CMPI(eq, headCrd, tailCrd); YIELD(isDup); // Else, the position is out of bound, yield false. @@ -1277,9 +1325,9 @@ sparse_tensor::makeSparseTensorLevel(OpBuilder &b, Location l, Value t, switch (lt.getLvlFmt()) { case LevelFormat::Dense: - return std::make_unique(tid, lvl, sz, stt.hasEncoding()); + return std::make_unique(tid, lvl, sz); case LevelFormat::Batch: - llvm_unreachable("not implemented"); + return std::make_unique(tid, lvl, sz); case LevelFormat::Compressed: { Value pos = b.create(l, t, lvl); Value crd = b.create(l, t, lvl); @@ -1307,7 +1355,7 @@ sparse_tensor::makeSparseTensorLevel(OpBuilder &b, Location l, Value t, std::pair, std::unique_ptr> sparse_tensor::makeSynLevelAndIterator(Value sz, unsigned tid, unsigned lvl, SparseEmitStrategy strategy) { - auto stl = std::make_unique(tid, lvl, sz, /*encoded=*/false); + auto stl = std::make_unique(tid, lvl, sz); auto it = std::make_unique(*stl); it->setSparseEmitStrategy(strategy); return std::make_pair(std::move(stl), std::move(it)); diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.h index d1e94b790bea..9f92eecdf75c 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.h +++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorLevel.h @@ -32,7 +32,8 @@ public: std::to_string(lvl) + "]"; } - virtual Value peekCrdAt(OpBuilder &b, Location l, Value iv) const = 0; + virtual Value peekCrdAt(OpBuilder &b, Location l, ValueRange batchPrefix, + Value iv) const = 0; /// Peeks the lower and upper bound to *fully* traverse the level with /// the given position `p` that the immediate parent level is current at. @@ -47,7 +48,8 @@ public: /// /// `bound` is only used when the level is `non-unique` and deduplication is /// required. It specifies the max upper bound of the non-unique segment. - virtual std::pair peekRangeAt(OpBuilder &b, Location l, Value p, + virtual std::pair peekRangeAt(OpBuilder &b, Location l, + ValueRange batchPrefix, Value p, Value segHi = Value()) const = 0; Level getLevel() const { return lvl; } @@ -89,7 +91,7 @@ protected: SparseIterator(IterKind kind, unsigned tid, unsigned lvl, unsigned cursorValsCnt, SmallVectorImpl &cursorValStorage) - : kind(kind), tid(tid), lvl(lvl), crd(nullptr), + : batchCrds(0), kind(kind), tid(tid), lvl(lvl), crd(nullptr), cursorValsCnt(cursorValsCnt), cursorValsStorageRef(cursorValStorage){}; SparseIterator(IterKind kind, unsigned cursorValsCnt, @@ -119,6 +121,7 @@ public: virtual SmallVector getCursorValTypes(OpBuilder &b) const = 0; Value getCrd() const { return crd; } + ValueRange getBatchCrds() const { return batchCrds; } ValueRange getCursor() const { return ValueRange(cursorValsStorageRef).take_front(cursorValsCnt); }; @@ -135,6 +138,9 @@ public: // Iterator properties. // + // Whether the iterator is a iterator over a batch level. + virtual bool isBatchIterator() const = 0; + // Whether the iterator support random access (i.e., support look up by // *coordinate*). A random access iterator must also traverses a dense space. virtual bool randomAccessible() const = 0; @@ -243,12 +249,18 @@ public: protected: void updateCrd(Value crd) { this->crd = crd; } + MutableArrayRef getMutCursorVals() { MutableArrayRef ref = cursorValsStorageRef; return ref.take_front(cursorValsCnt); } + void inherentBatch(const SparseIterator &parent) { + batchCrds = parent.batchCrds; + } + SparseEmitStrategy emitStrategy; + SmallVector batchCrds; public: const IterKind kind; // For LLVM-style RTTI. diff --git a/mlir/test/Dialect/SparseTensor/sparse_batch.mlir b/mlir/test/Dialect/SparseTensor/sparse_batch.mlir new file mode 100644 index 000000000000..f6d2d0d4f766 --- /dev/null +++ b/mlir/test/Dialect/SparseTensor/sparse_batch.mlir @@ -0,0 +1,48 @@ +// RUN: mlir-opt %s --sparse-reinterpret-map -sparsification | FileCheck %s + +#map = affine_map<(d0, d1, d2) -> (d0, d1, d2)> +#BCSR = #sparse_tensor.encoding<{ map = (d0, d1, d2) -> (d0 : batch, d1 : dense, d2 : compressed)}> + +// CHECK-LABEL: func.func @main( +// CHECK-SAME: %[[VAL_0:.*]]: tensor<8x4x2xf32, #sparse{{[0-9]*}}>) -> tensor<8x4x2xf32> { +// CHECK: %[[VAL_1:.*]] = arith.constant 1 : index +// CHECK: %[[VAL_2:.*]] = arith.constant 0 : index +// CHECK: %[[VAL_3:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK: %[[VAL_4:.*]] = arith.constant 4 : index +// CHECK: %[[VAL_5:.*]] = arith.constant 8 : index +// CHECK: %[[VAL_6:.*]] = tensor.empty() : tensor<8x4x2xf32> +// CHECK: %[[VAL_7:.*]] = sparse_tensor.positions %[[VAL_0]] {level = 2 : index} : tensor<8x4x2xf32, #sparse{{[0-9]*}}> to memref<8x?xindex> +// CHECK: %[[VAL_8:.*]] = sparse_tensor.coordinates %[[VAL_0]] {level = 2 : index} : tensor<8x4x2xf32, #sparse{{[0-9]*}}> to memref<8x?xindex> +// CHECK: %[[VAL_9:.*]] = sparse_tensor.values %[[VAL_0]] : tensor<8x4x2xf32, #sparse{{[0-9]*}}> to memref<8x?xf32> +// CHECK: %[[VAL_10:.*]] = bufferization.to_memref %[[VAL_6]] : memref<8x4x2xf32> +// CHECK: linalg.fill ins(%[[VAL_3]] : f32) outs(%[[VAL_10]] : memref<8x4x2xf32>) +// CHECK: scf.for %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_5]] step %[[VAL_1]] { +// CHECK: scf.for %[[VAL_12:.*]] = %[[VAL_2]] to %[[VAL_4]] step %[[VAL_1]] { +// CHECK: %[[VAL_13:.*]] = memref.load %[[VAL_7]]{{\[}}%[[VAL_11]], %[[VAL_12]]] : memref<8x?xindex> +// CHECK: %[[VAL_14:.*]] = arith.addi %[[VAL_12]], %[[VAL_1]] : index +// CHECK: %[[VAL_15:.*]] = memref.load %[[VAL_7]]{{\[}}%[[VAL_11]], %[[VAL_14]]] : memref<8x?xindex> +// CHECK: scf.for %[[VAL_16:.*]] = %[[VAL_13]] to %[[VAL_15]] step %[[VAL_1]] { +// CHECK: %[[VAL_17:.*]] = memref.load %[[VAL_8]]{{\[}}%[[VAL_11]], %[[VAL_16]]] : memref<8x?xindex> +// CHECK: %[[VAL_18:.*]] = memref.load %[[VAL_9]]{{\[}}%[[VAL_11]], %[[VAL_16]]] : memref<8x?xf32> +// CHECK: %[[VAL_19:.*]] = arith.negf %[[VAL_18]] : f32 +// CHECK: memref.store %[[VAL_19]], %[[VAL_10]]{{\[}}%[[VAL_11]], %[[VAL_12]], %[[VAL_17]]] : memref<8x4x2xf32> +// CHECK: } +// CHECK: } +// CHECK: } +// CHECK: %[[VAL_20:.*]] = bufferization.to_tensor %[[VAL_10]] : memref<8x4x2xf32> +// CHECK: return %[[VAL_20]] : tensor<8x4x2xf32> +// CHECK: } +func.func @main(%arg0: tensor<8x4x2xf32, #BCSR>) -> tensor<8x4x2xf32> { + %0 = tensor.empty() : tensor<8x4x2xf32> + %1 = linalg.generic { + indexing_maps = [#map, #map], + iterator_types = ["parallel", "parallel", "parallel"] + } + ins(%arg0 : tensor<8x4x2xf32, #BCSR>) + outs(%0 : tensor<8x4x2xf32>) { + ^bb0(%in: f32, %out: f32): + %2 = arith.negf %in : f32 + linalg.yield %2 : f32 + } -> tensor<8x4x2xf32> + return %1 : tensor<8x4x2xf32> +} diff --git a/mlir/test/Dialect/SparseTensor/sparse_conv_2d_slice_based.mlir b/mlir/test/Dialect/SparseTensor/sparse_conv_2d_slice_based.mlir index 6aba0ada947e..6076c1fbe76f 100644 --- a/mlir/test/Dialect/SparseTensor/sparse_conv_2d_slice_based.mlir +++ b/mlir/test/Dialect/SparseTensor/sparse_conv_2d_slice_based.mlir @@ -24,13 +24,13 @@ // CHECK: "subsect>.not_end // CHECK: } do { // CHECK: %[[D2:.*]] = "subsect>.deref" -// CHECK: "trivial.locate"(%{{.*}}, %[[D2]]) +// CHECK: "trivial.locate"(%{{.*}}, %[[D2]]) // CHECK: "subsect>.begin" // CHECK: scf.while {{.*}} { // CHECK: "subsect>.not_end" // CHECK: } do { // CHECK: %[[D3:.*]] = "subsect>.deref" -// CHECK: "trivial.locate"(%{{.*}}, %[[D3]]) +// CHECK: "trivial.locate"(%{{.*}}, %[[D3]]) // CHECK: tensor.extract %{{.*}}{{\[}}%[[D2]], %[[D3]]] // CHECK: arith.muli // CHECK: arith.addi -- GitLab From 10ccde30e784622cfb16940f7dddd4bb3a94ce44 Mon Sep 17 00:00:00 2001 From: Cyndy Ishida Date: Mon, 4 Mar 2024 14:50:04 -0800 Subject: [PATCH 058/929] [InstallAPI] Collect symbols from ObjC Ivars (#83632) --- clang/include/clang/InstallAPI/Frontend.h | 35 ++++++++++++ clang/include/clang/InstallAPI/Visitor.h | 12 +++++ clang/lib/InstallAPI/Frontend.cpp | 25 +++++++++ clang/lib/InstallAPI/Visitor.cpp | 51 +++++++++++++++++- clang/test/InstallAPI/objcclasses.test | 65 +++++++++++++++++++++-- llvm/include/llvm/TextAPI/Record.h | 1 + llvm/lib/TextAPI/RecordsSlice.cpp | 1 + 7 files changed, 186 insertions(+), 4 deletions(-) diff --git a/clang/include/clang/InstallAPI/Frontend.h b/clang/include/clang/InstallAPI/Frontend.h index d72b4680fde4..8774321e990c 100644 --- a/clang/include/clang/InstallAPI/Frontend.h +++ b/clang/include/clang/InstallAPI/Frontend.h @@ -28,7 +28,10 @@ namespace installapi { using SymbolFlags = llvm::MachO::SymbolFlags; using RecordLinkage = llvm::MachO::RecordLinkage; using GlobalRecord = llvm::MachO::GlobalRecord; +using ObjCContainerRecord = llvm::MachO::ObjCContainerRecord; using ObjCInterfaceRecord = llvm::MachO::ObjCInterfaceRecord; +using ObjCCategoryRecord = llvm::MachO::ObjCCategoryRecord; +using ObjCIVarRecord = llvm::MachO::ObjCIVarRecord; // Represents a collection of frontend records for a library that are tied to a // darwin target triple. @@ -69,6 +72,38 @@ public: const Decl *D, HeaderType Access, bool IsEHType); + /// Add ObjC Category record with attributes from AST. + /// + /// \param ClassToExtend The name of class that is extended by category, not + /// symbol. + /// \param CategoryName The name of category, not symbol. + /// \param Avail The availability information tied + /// to the active target triple. + /// \param D The pointer to the declaration from traversing AST. + /// \param Access The intended access level of symbol. + /// \return The non-owning pointer to added record in slice. + ObjCCategoryRecord *addObjCCategory(StringRef ClassToExtend, + StringRef CategoryName, + const clang::AvailabilityInfo Avail, + const Decl *D, HeaderType Access); + + /// Add ObjC IVar record with attributes from AST. + /// + /// \param Container The owning pointer for instance variable. + /// \param Name The name of ivar, not symbol. + /// \param Linkage The linkage of symbol. + /// \param Avail The availability information tied to the active target + /// triple. + /// \param D The pointer to the declaration from traversing AST. + /// \param Access The intended access level of symbol. + /// \param AC The access control tied to the ivar declaration. + /// \return The non-owning pointer to added record in slice. + ObjCIVarRecord *addObjCIVar(ObjCContainerRecord *Container, + StringRef IvarName, RecordLinkage Linkage, + const clang::AvailabilityInfo Avail, + const Decl *D, HeaderType Access, + const clang::ObjCIvarDecl::AccessControl AC); + private: /// Frontend information captured about records. struct FrontendAttrs { diff --git a/clang/include/clang/InstallAPI/Visitor.h b/clang/include/clang/InstallAPI/Visitor.h index 60a05005df84..ff0a9957aa86 100644 --- a/clang/include/clang/InstallAPI/Visitor.h +++ b/clang/include/clang/InstallAPI/Visitor.h @@ -42,10 +42,22 @@ public: /// ivars, properties, and methods of the class. bool VisitObjCInterfaceDecl(const ObjCInterfaceDecl *D); + /// Collect Objective-C Category/Extension declarations. + /// + /// The class that is being extended might come from a different library and + /// is therefore itself not collected. + bool VisitObjCCategoryDecl(const ObjCCategoryDecl *D); + private: std::string getMangledName(const NamedDecl *D) const; std::string getBackendMangledName(llvm::Twine Name) const; std::optional getAccessForDecl(const NamedDecl *D) const; + void recordObjCInstanceVariables( + const ASTContext &ASTCtx, llvm::MachO::ObjCContainerRecord *Record, + StringRef SuperClass, + const llvm::iterator_range< + DeclContext::specific_decl_iterator> + Ivars); InstallAPIContext &Ctx; SourceManager &SrcMgr; diff --git a/clang/lib/InstallAPI/Frontend.cpp b/clang/lib/InstallAPI/Frontend.cpp index caa6e7e8a405..240a80e1d3d8 100644 --- a/clang/lib/InstallAPI/Frontend.cpp +++ b/clang/lib/InstallAPI/Frontend.cpp @@ -39,6 +39,31 @@ ObjCInterfaceRecord *FrontendRecordsSlice::addObjCInterface( return ObjCR; } +ObjCCategoryRecord *FrontendRecordsSlice::addObjCCategory( + StringRef ClassToExtend, StringRef CategoryName, + const clang::AvailabilityInfo Avail, const Decl *D, HeaderType Access) { + auto *ObjCR = + llvm::MachO::RecordsSlice::addObjCCategory(ClassToExtend, CategoryName); + FrontendRecords.insert({ObjCR, FrontendAttrs{Avail, D, Access}}); + return ObjCR; +} + +ObjCIVarRecord *FrontendRecordsSlice::addObjCIVar( + ObjCContainerRecord *Container, StringRef IvarName, RecordLinkage Linkage, + const clang::AvailabilityInfo Avail, const Decl *D, HeaderType Access, + const clang::ObjCIvarDecl::AccessControl AC) { + // If the decl otherwise would have been exported, check their access control. + // Ivar's linkage is also determined by this. + if ((Linkage == RecordLinkage::Exported) && + ((AC == ObjCIvarDecl::Private) || (AC == ObjCIvarDecl::Package))) + Linkage = RecordLinkage::Internal; + auto *ObjCR = + llvm::MachO::RecordsSlice::addObjCIVar(Container, IvarName, Linkage); + FrontendRecords.insert({ObjCR, FrontendAttrs{Avail, D, Access}}); + + return nullptr; +} + std::optional InstallAPIContext::findAndRecordFile(const FileEntry *FE, const Preprocessor &PP) { diff --git a/clang/lib/InstallAPI/Visitor.cpp b/clang/lib/InstallAPI/Visitor.cpp index 355a092520c3..fbe6f1dabe00 100644 --- a/clang/lib/InstallAPI/Visitor.cpp +++ b/clang/lib/InstallAPI/Visitor.cpp @@ -99,6 +99,29 @@ static bool hasObjCExceptionAttribute(const ObjCInterfaceDecl *D) { return false; } +void InstallAPIVisitor::recordObjCInstanceVariables( + const ASTContext &ASTCtx, ObjCContainerRecord *Record, StringRef SuperClass, + const llvm::iterator_range< + DeclContext::specific_decl_iterator> + Ivars) { + RecordLinkage Linkage = RecordLinkage::Exported; + const RecordLinkage ContainerLinkage = Record->getLinkage(); + // If fragile, set to unknown. + if (ASTCtx.getLangOpts().ObjCRuntime.isFragile()) + Linkage = RecordLinkage::Unknown; + // Linkage should be inherited from container. + else if (ContainerLinkage != RecordLinkage::Unknown) + Linkage = ContainerLinkage; + for (const auto *IV : Ivars) { + auto Access = getAccessForDecl(IV); + if (!Access) + continue; + StringRef Name = IV->getName(); + const AvailabilityInfo Avail = AvailabilityInfo::createFromDecl(IV); + auto AC = IV->getCanonicalAccessControl(); + Ctx.Slice->addObjCIVar(Record, Name, Linkage, Avail, IV, *Access, AC); + } +} bool InstallAPIVisitor::VisitObjCInterfaceDecl(const ObjCInterfaceDecl *D) { // Skip forward declaration for classes (@class) @@ -118,7 +141,33 @@ bool InstallAPIVisitor::VisitObjCInterfaceDecl(const ObjCInterfaceDecl *D) { (!D->getASTContext().getLangOpts().ObjCRuntime.isFragile() && hasObjCExceptionAttribute(D)); - Ctx.Slice->addObjCInterface(Name, Linkage, Avail, D, *Access, IsEHType); + ObjCInterfaceRecord *Class = + Ctx.Slice->addObjCInterface(Name, Linkage, Avail, D, *Access, IsEHType); + + // Get base class. + StringRef SuperClassName; + if (const auto *SuperClass = D->getSuperClass()) + SuperClassName = SuperClass->getObjCRuntimeNameAsString(); + + recordObjCInstanceVariables(D->getASTContext(), Class, SuperClassName, + D->ivars()); + return true; +} + +bool InstallAPIVisitor::VisitObjCCategoryDecl(const ObjCCategoryDecl *D) { + StringRef CategoryName = D->getName(); + // Skip over declarations that access could not be collected for. + auto Access = getAccessForDecl(D); + if (!Access) + return true; + const AvailabilityInfo Avail = AvailabilityInfo::createFromDecl(D); + const ObjCInterfaceDecl *InterfaceD = D->getClassInterface(); + const StringRef InterfaceName = InterfaceD->getName(); + + ObjCCategoryRecord *Category = Ctx.Slice->addObjCCategory( + InterfaceName, CategoryName, Avail, D, *Access); + recordObjCInstanceVariables(D->getASTContext(), Category, InterfaceName, + D->ivars()); return true; } diff --git a/clang/test/InstallAPI/objcclasses.test b/clang/test/InstallAPI/objcclasses.test index d32291c64c47..7b73dffbe920 100644 --- a/clang/test/InstallAPI/objcclasses.test +++ b/clang/test/InstallAPI/objcclasses.test @@ -20,19 +20,68 @@ @end __attribute__((visibility("hidden"))) -@interface Hidden +@interface Hidden +@end + +__attribute__((visibility("hidden"))) +@interface HiddenWithIvars { +@public +char _ivar; +} @end __attribute__((objc_exception)) @interface Exception @end +@interface PublicClass : Visible { +@package + int _internal; +@protected + int _external; +@private + int private; +@public +char _public; +} +@end + + +//--- Foo.framework/PrivateHeaders/Foo_Private.h +#import + +@interface ClassWithIvars : Visible { + char _ivar1; + char _ivar2; +@private + int _privateIVar; +@protected + int _externalIVar; +@package + int _internalIVar; +} +@end + +@interface Exception () { +@public + char _ivarFromExtension; +@private + int _privateIvarFromExtension; +} +@end + + //--- inputs.json.in { "headers": [ { "path" : "DSTROOT/Foo.framework/Headers/Foo.h", "type" : "public" - }], + }, + { + "path" : "DSTROOT/Foo.framework/PrivateHeaders/Foo_Private.h", + "type" : "private" + } + ], "version": "3" } @@ -53,11 +102,21 @@ __attribute__((objc_exception)) { "data": { "objc_class": [ + "PublicClass", "Exception", - "Visible" + "Visible", + "ClassWithIvars" ], "objc_eh_type": [ "Exception" + ], + "objc_ivar": [ + "Exception._ivarFromExtension", + "ClassWithIvars._ivar2", + "PublicClass._external", + "ClassWithIvars._ivar1", + "ClassWithIvars._externalIVar", + "PublicClass._public" ] } } diff --git a/llvm/include/llvm/TextAPI/Record.h b/llvm/include/llvm/TextAPI/Record.h index 3b30e6c8c267..867d6a235883 100644 --- a/llvm/include/llvm/TextAPI/Record.h +++ b/llvm/include/llvm/TextAPI/Record.h @@ -143,6 +143,7 @@ public: ObjCIVarRecord *addObjCIVar(StringRef IVar, RecordLinkage Linkage); ObjCIVarRecord *findObjCIVar(StringRef IVar) const; std::vector getObjCIVars() const; + RecordLinkage getLinkage() const { return Linkage; } private: RecordMap IVars; diff --git a/llvm/lib/TextAPI/RecordsSlice.cpp b/llvm/lib/TextAPI/RecordsSlice.cpp index fb961a91cc65..db52a2cdd85c 100644 --- a/llvm/lib/TextAPI/RecordsSlice.cpp +++ b/llvm/lib/TextAPI/RecordsSlice.cpp @@ -225,6 +225,7 @@ bool ObjCInterfaceRecord::addObjCCategory(ObjCCategoryRecord *Record) { ObjCCategoryRecord *RecordsSlice::addObjCCategory(StringRef ClassToExtend, StringRef Category) { Category = copyString(Category); + ClassToExtend = copyString(ClassToExtend); // Add owning record first into record slice. auto Result = -- GitLab From 3bc0ff28a4349f5b2836413fd2cc786997f388be Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Mon, 26 Feb 2024 15:54:04 -0600 Subject: [PATCH 059/929] [Analysis] Move `DomConditionCache::findAffectedValues` to a new file; NFC --- llvm/include/llvm/Analysis/ValueTracking.h | 7 +++ llvm/lib/Analysis/DomConditionCache.cpp | 68 +-------------------- llvm/lib/Analysis/ValueTracking.cpp | 69 ++++++++++++++++++++++ 3 files changed, 79 insertions(+), 65 deletions(-) diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h index f0d0ee554f12..9cfb7af9dba0 100644 --- a/llvm/include/llvm/Analysis/ValueTracking.h +++ b/llvm/include/llvm/Analysis/ValueTracking.h @@ -1195,6 +1195,13 @@ std::optional isImpliedByDomCondition(CmpInst::Predicate Pred, const Value *LHS, const Value *RHS, const Instruction *ContextI, const DataLayout &DL); + +/// Call \p InsertAffected on all Values whose known bits / value may be +/// affected by the condition \p Cond. Used by AssumptionCache and +/// DomConditionCache. +void findValuesAffectedByCondition(Value *Cond, bool IsAssume, + function_ref InsertAffected); + } // end namespace llvm #endif // LLVM_ANALYSIS_VALUETRACKING_H diff --git a/llvm/lib/Analysis/DomConditionCache.cpp b/llvm/lib/Analysis/DomConditionCache.cpp index da05e02b4b57..66bd15b47901 100644 --- a/llvm/lib/Analysis/DomConditionCache.cpp +++ b/llvm/lib/Analysis/DomConditionCache.cpp @@ -7,75 +7,13 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/DomConditionCache.h" -#include "llvm/IR/PatternMatch.h" - +#include "llvm/Analysis/ValueTracking.h" using namespace llvm; -using namespace llvm::PatternMatch; -// TODO: This code is very similar to findAffectedValues() in -// AssumptionCache, but currently specialized to just the patterns that -// computeKnownBits() supports, and without the notion of result elem indices -// that are AC specific. Deduplicate this code once we have a clearer picture -// of how much they can be shared. static void findAffectedValues(Value *Cond, SmallVectorImpl &Affected) { - auto AddAffected = [&Affected](Value *V) { - if (isa(V) || isa(V)) { - Affected.push_back(V); - } else if (auto *I = dyn_cast(V)) { - Affected.push_back(I); - - // Peek through unary operators to find the source of the condition. - Value *Op; - if (match(I, m_PtrToInt(m_Value(Op)))) { - if (isa(Op) || isa(Op)) - Affected.push_back(Op); - } - } - }; - - SmallVector Worklist; - SmallPtrSet Visited; - Worklist.push_back(Cond); - while (!Worklist.empty()) { - Value *V = Worklist.pop_back_val(); - if (!Visited.insert(V).second) - continue; - - CmpInst::Predicate Pred; - Value *A, *B; - if (match(V, m_LogicalOp(m_Value(A), m_Value(B)))) { - Worklist.push_back(A); - Worklist.push_back(B); - } else if (match(V, m_ICmp(Pred, m_Value(A), m_Constant()))) { - AddAffected(A); - - if (ICmpInst::isEquality(Pred)) { - Value *X; - // (X & C) or (X | C) or (X ^ C). - // (X << C) or (X >>_s C) or (X >>_u C). - if (match(A, m_BitwiseLogic(m_Value(X), m_ConstantInt())) || - match(A, m_Shift(m_Value(X), m_ConstantInt()))) - AddAffected(X); - } else { - Value *X; - // Handle (A + C1) u< C2, which is the canonical form of - // A > C3 && A < C4. - if (match(A, m_Add(m_Value(X), m_ConstantInt()))) - AddAffected(X); - // Handle icmp slt/sgt (bitcast X to int), 0/-1, which is supported by - // computeKnownFPClass(). - if ((Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGT) && - match(A, m_ElementWiseBitCast(m_Value(X)))) - Affected.push_back(X); - } - } else if (match(Cond, m_CombineOr(m_FCmp(Pred, m_Value(A), m_Constant()), - m_Intrinsic( - m_Value(A), m_Constant())))) { - // Handle patterns that computeKnownFPClass() support. - AddAffected(A); - } - } + auto InsertAffected = [&Affected](Value *V) { Affected.push_back(V); }; + findValuesAffectedByCondition(Cond, /*IsAssume=*/false, InsertAffected); } void DomConditionCache::registerBranch(BranchInst *BI) { diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index e591ac504e9f..4b230863b167 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -9099,3 +9099,72 @@ ConstantRange llvm::computeConstantRange(const Value *V, bool ForSigned, return CR; } + +static void +addValueAffectedByCondition(Value *V, + function_ref InsertAffected) { + assert(V != nullptr); + if (isa(V) || isa(V)) { + InsertAffected(V); + } else if (auto *I = dyn_cast(V)) { + InsertAffected(V); + + // Peek through unary operators to find the source of the condition. + Value *Op; + if (match(I, m_PtrToInt(m_Value(Op)))) { + if (isa(Op) || isa(Op)) + InsertAffected(Op); + } + } +} + +void llvm::findValuesAffectedByCondition( + Value *Cond, bool IsAssume, function_ref InsertAffected) { + auto AddAffected = [&InsertAffected](Value *V) { + addValueAffectedByCondition(V, InsertAffected); + }; + + assert(!IsAssume); + SmallVector Worklist; + SmallPtrSet Visited; + Worklist.push_back(Cond); + while (!Worklist.empty()) { + Value *V = Worklist.pop_back_val(); + if (!Visited.insert(V).second) + continue; + + CmpInst::Predicate Pred; + Value *A, *B; + if (match(V, m_LogicalOp(m_Value(A), m_Value(B)))) { + Worklist.push_back(A); + Worklist.push_back(B); + } else if (match(V, m_ICmp(Pred, m_Value(A), m_Constant()))) { + AddAffected(A); + + if (ICmpInst::isEquality(Pred)) { + Value *X; + // (X & C) or (X | C) or (X ^ C). + // (X << C) or (X >>_s C) or (X >>_u C). + if (match(A, m_BitwiseLogic(m_Value(X), m_ConstantInt())) || + match(A, m_Shift(m_Value(X), m_ConstantInt()))) + AddAffected(X); + } else { + Value *X; + // Handle (A + C1) u< C2, which is the canonical form of + // A > C3 && A < C4. + if (match(A, m_Add(m_Value(X), m_ConstantInt()))) + AddAffected(X); + // Handle icmp slt/sgt (bitcast X to int), 0/-1, which is supported by + // computeKnownFPClass(). + if ((Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGT) && + match(A, m_ElementWiseBitCast(m_Value(X)))) + InsertAffected(X); + } + } else if (match(Cond, m_CombineOr(m_FCmp(Pred, m_Value(A), m_Constant()), + m_Intrinsic( + m_Value(A), m_Constant())))) { + // Handle patterns that computeKnownFPClass() support. + AddAffected(A); + } + } +} -- GitLab From 6ee46aba0695bd004e5b229b73dabe8fd5a70513 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Mon, 26 Feb 2024 16:19:48 -0600 Subject: [PATCH 060/929] [Analysis] Share `findAffectedValues` between DomConditionCache and AssumptionCache; NFC --- llvm/lib/Analysis/AssumptionCache.cpp | 73 +++++------------------ llvm/lib/Analysis/ValueTracking.cpp | 83 ++++++++++++++++++--------- 2 files changed, 70 insertions(+), 86 deletions(-) diff --git a/llvm/lib/Analysis/AssumptionCache.cpp b/llvm/lib/Analysis/AssumptionCache.cpp index 1b7277df0e0c..9c86ef13f6df 100644 --- a/llvm/lib/Analysis/AssumptionCache.cpp +++ b/llvm/lib/Analysis/AssumptionCache.cpp @@ -61,19 +61,13 @@ findAffectedValues(CallBase *CI, TargetTransformInfo *TTI, // Note: This code must be kept in-sync with the code in // computeKnownBitsFromAssume in ValueTracking. - auto AddAffected = [&Affected](Value *V, unsigned Idx = - AssumptionCache::ExprResultIdx) { - if (isa(V) || isa(V)) { + auto InsertAffected = [&Affected](Value *V) { + Affected.push_back({V, AssumptionCache::ExprResultIdx}); + }; + + auto AddAffectedVal = [&Affected](Value *V, unsigned Idx) { + if (isa(V) || isa(V) || isa(V)) { Affected.push_back({V, Idx}); - } else if (auto *I = dyn_cast(V)) { - Affected.push_back({I, Idx}); - - // Peek through unary operators to find the source of the condition. - Value *Op; - if (match(I, m_PtrToInt(m_Value(Op)))) { - if (isa(Op) || isa(Op)) - Affected.push_back({Op, Idx}); - } } }; @@ -82,64 +76,23 @@ findAffectedValues(CallBase *CI, TargetTransformInfo *TTI, if (Bundle.getTagName() == "separate_storage") { assert(Bundle.Inputs.size() == 2 && "separate_storage must have two args"); - AddAffected(getUnderlyingObject(Bundle.Inputs[0]), Idx); - AddAffected(getUnderlyingObject(Bundle.Inputs[1]), Idx); + AddAffectedVal(getUnderlyingObject(Bundle.Inputs[0]), Idx); + AddAffectedVal(getUnderlyingObject(Bundle.Inputs[1]), Idx); } else if (Bundle.Inputs.size() > ABA_WasOn && Bundle.getTagName() != IgnoreBundleTag) - AddAffected(Bundle.Inputs[ABA_WasOn], Idx); + AddAffectedVal(Bundle.Inputs[ABA_WasOn], Idx); } - Value *Cond = CI->getArgOperand(0), *A, *B; - AddAffected(Cond); - if (match(Cond, m_Not(m_Value(A)))) - AddAffected(A); - - CmpInst::Predicate Pred; - if (match(Cond, m_Cmp(Pred, m_Value(A), m_Value(B)))) { - AddAffected(A); - AddAffected(B); - - if (Pred == ICmpInst::ICMP_EQ) { - if (match(B, m_ConstantInt())) { - Value *X; - // (X & C) or (X | C) or (X ^ C). - // (X << C) or (X >>_s C) or (X >>_u C). - if (match(A, m_BitwiseLogic(m_Value(X), m_ConstantInt())) || - match(A, m_Shift(m_Value(X), m_ConstantInt()))) - AddAffected(X); - } - } else if (Pred == ICmpInst::ICMP_NE) { - Value *X; - // Handle (X & pow2 != 0). - if (match(A, m_And(m_Value(X), m_Power2())) && match(B, m_Zero())) - AddAffected(X); - } else if (Pred == ICmpInst::ICMP_ULT) { - Value *X; - // Handle (A + C1) u< C2, which is the canonical form of A > C3 && A < C4, - // and recognized by LVI at least. - if (match(A, m_Add(m_Value(X), m_ConstantInt())) && - match(B, m_ConstantInt())) - AddAffected(X); - } else if (CmpInst::isFPPredicate(Pred)) { - // fcmp fneg(x), y - // fcmp fabs(x), y - // fcmp fneg(fabs(x)), y - if (match(A, m_FNeg(m_Value(A)))) - AddAffected(A); - if (match(A, m_FAbs(m_Value(A)))) - AddAffected(A); - } - } else if (match(Cond, m_Intrinsic(m_Value(A), - m_Value(B)))) { - AddAffected(A); - } + Value *Cond = CI->getArgOperand(0); + findValuesAffectedByCondition(Cond, /*IsAssume=*/true, InsertAffected); if (TTI) { const Value *Ptr; unsigned AS; std::tie(Ptr, AS) = TTI->getPredicatedAddrSpace(Cond); if (Ptr) - AddAffected(const_cast(Ptr->stripInBoundsOffsets())); + AddAffectedVal(const_cast(Ptr->stripInBoundsOffsets()), + AssumptionCache::ExprResultIdx); } } diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index 4b230863b167..dd78c165c477 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -9124,7 +9124,6 @@ void llvm::findValuesAffectedByCondition( addValueAffectedByCondition(V, InsertAffected); }; - assert(!IsAssume); SmallVector Worklist; SmallPtrSet Visited; Worklist.push_back(Cond); @@ -9134,35 +9133,67 @@ void llvm::findValuesAffectedByCondition( continue; CmpInst::Predicate Pred; - Value *A, *B; - if (match(V, m_LogicalOp(m_Value(A), m_Value(B)))) { + Value *A, *B, *X; + + if (IsAssume) + AddAffected(V); + + if (IsAssume && match(V, m_Not(m_Value(X)))) + AddAffected(X); + if (!IsAssume && match(V, m_LogicalOp(m_Value(A), m_Value(B)))) { Worklist.push_back(A); Worklist.push_back(B); - } else if (match(V, m_ICmp(Pred, m_Value(A), m_Constant()))) { - AddAffected(A); + } else if (match(V, m_Cmp(Pred, m_Value(A), m_Value(B))) && + (IsAssume || isa(V))) { + if (IsAssume || match(B, m_Constant())) { + AddAffected(A); + if (IsAssume) + AddAffected(B); + + if (IsAssume ? (Pred == ICmpInst::ICMP_EQ) + : ICmpInst::isEquality(Pred)) { + if (match(B, m_ConstantInt())) { + // (X & C) or (X | C) or (X ^ C). + // (X << C) or (X >>_s C) or (X >>_u C). + if (match(A, m_BitwiseLogic(m_Value(X), m_ConstantInt())) || + match(A, m_Shift(m_Value(X), m_ConstantInt()))) + AddAffected(X); + } + } else { + if (Pred == ICmpInst::ICMP_NE) + if (match(A, m_And(m_Value(X), m_Power2())) && match(B, m_Zero())) + AddAffected(X); + + if (!IsAssume || Pred == ICmpInst::ICMP_ULT) { + // Handle (A + C1) u< C2, which is the canonical form of + // A > C3 && A < C4. + if (match(A, m_Add(m_Value(X), m_ConstantInt())) && + match(B, m_ConstantInt())) + AddAffected(X); + } + if (!IsAssume) { + // Handle icmp slt/sgt (bitcast X to int), 0/-1, which is supported + // by computeKnownFPClass(). + if ((Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGT) && + match(A, m_ElementWiseBitCast(m_Value(X)))) + InsertAffected(X); + } - if (ICmpInst::isEquality(Pred)) { - Value *X; - // (X & C) or (X | C) or (X ^ C). - // (X << C) or (X >>_s C) or (X >>_u C). - if (match(A, m_BitwiseLogic(m_Value(X), m_ConstantInt())) || - match(A, m_Shift(m_Value(X), m_ConstantInt()))) - AddAffected(X); - } else { - Value *X; - // Handle (A + C1) u< C2, which is the canonical form of - // A > C3 && A < C4. - if (match(A, m_Add(m_Value(X), m_ConstantInt()))) - AddAffected(X); - // Handle icmp slt/sgt (bitcast X to int), 0/-1, which is supported by - // computeKnownFPClass(). - if ((Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGT) && - match(A, m_ElementWiseBitCast(m_Value(X)))) - InsertAffected(X); + if (IsAssume && CmpInst::isFPPredicate(Pred)) { + // fcmp fneg(x), y + // fcmp fabs(x), y + // fcmp fneg(fabs(x)), y + if (match(A, m_FNeg(m_Value(A)))) + AddAffected(A); + if (match(A, m_FAbs(m_Value(A)))) + AddAffected(A); + } + } } - } else if (match(Cond, m_CombineOr(m_FCmp(Pred, m_Value(A), m_Constant()), - m_Intrinsic( - m_Value(A), m_Constant())))) { + } else if ((!IsAssume && + match(Cond, m_FCmp(Pred, m_Value(A), m_Constant()))) || + match(Cond, m_Intrinsic(m_Value(A), + m_Value(B)))) { // Handle patterns that computeKnownFPClass() support. AddAffected(A); } -- GitLab From db3bbe03f1c93100434a1394d293d13d4e5c1a2e Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Mon, 26 Feb 2024 16:30:39 -0600 Subject: [PATCH 061/929] [Analysis] Unify most of the tracking between AssumptionCache and DomConditionCache This helps cover some missing cases in both and hopefully serves as creating an easier framework for extending general condition based analysis. Closes #83161 --- llvm/lib/Analysis/ValueTracking.cpp | 115 +++++++++--------- .../ValueTracking/numsignbits-from-assume.ll | 2 +- .../InstCombine/fpclass-from-dom-cond.ll | 5 +- 3 files changed, 63 insertions(+), 59 deletions(-) diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index dd78c165c477..69b271042039 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -9124,6 +9124,14 @@ void llvm::findValuesAffectedByCondition( addValueAffectedByCondition(V, InsertAffected); }; + auto AddCmpOperands = [&AddAffected, IsAssume](Value *LHS, Value *RHS) { + if (IsAssume) { + AddAffected(LHS); + AddAffected(RHS); + } else if (match(RHS, m_Constant())) + AddAffected(LHS); + }; + SmallVector Worklist; SmallPtrSet Visited; Worklist.push_back(Cond); @@ -9135,65 +9143,62 @@ void llvm::findValuesAffectedByCondition( CmpInst::Predicate Pred; Value *A, *B, *X; - if (IsAssume) + if (IsAssume) { AddAffected(V); + if (match(V, m_Not(m_Value(X)))) + AddAffected(X); + } + + if (match(V, m_LogicalOp(m_Value(A), m_Value(B)))) { + // assume(A && B) is split to -> assume(A); assume(B); + // assume(!(A || B)) is split to -> assume(!A); assume(!B); + // Finally, assume(A || B) / assume(!(A && B)) generally don't provide + // enough information to be worth handling (intersection of information as + // opposed to union). + if (!IsAssume) { + Worklist.push_back(A); + Worklist.push_back(B); + } + } else if (match(V, m_ICmp(Pred, m_Value(A), m_Value(B)))) { + AddCmpOperands(A, B); + + if (ICmpInst::isEquality(Pred)) { + if (match(B, m_ConstantInt())) { + // (X & C) or (X | C) or (X ^ C). + // (X << C) or (X >>_s C) or (X >>_u C). + if (match(A, m_BitwiseLogic(m_Value(X), m_ConstantInt())) || + match(A, m_Shift(m_Value(X), m_ConstantInt()))) + AddAffected(X); + } + } else { + // Handle (A + C1) u< C2, which is the canonical form of + // A > C3 && A < C4. + if (match(A, m_Add(m_Value(X), m_ConstantInt())) && + match(B, m_ConstantInt())) + AddAffected(X); + + // Handle icmp slt/sgt (bitcast X to int), 0/-1, which is supported + // by computeKnownFPClass(). + if (match(A, m_ElementWiseBitCast(m_Value(X)))) { + if (Pred == ICmpInst::ICMP_SLT && match(B, m_Zero())) + InsertAffected(X); + else if (Pred == ICmpInst::ICMP_SGT && match(B, m_AllOnes())) + InsertAffected(X); + } + } + } else if (match(Cond, m_FCmp(Pred, m_Value(A), m_Value(B)))) { + AddCmpOperands(A, B); - if (IsAssume && match(V, m_Not(m_Value(X)))) - AddAffected(X); - if (!IsAssume && match(V, m_LogicalOp(m_Value(A), m_Value(B)))) { - Worklist.push_back(A); - Worklist.push_back(B); - } else if (match(V, m_Cmp(Pred, m_Value(A), m_Value(B))) && - (IsAssume || isa(V))) { - if (IsAssume || match(B, m_Constant())) { + // fcmp fneg(x), y + // fcmp fabs(x), y + // fcmp fneg(fabs(x)), y + if (match(A, m_FNeg(m_Value(A)))) + AddAffected(A); + if (match(A, m_FAbs(m_Value(A)))) AddAffected(A); - if (IsAssume) - AddAffected(B); - - if (IsAssume ? (Pred == ICmpInst::ICMP_EQ) - : ICmpInst::isEquality(Pred)) { - if (match(B, m_ConstantInt())) { - // (X & C) or (X | C) or (X ^ C). - // (X << C) or (X >>_s C) or (X >>_u C). - if (match(A, m_BitwiseLogic(m_Value(X), m_ConstantInt())) || - match(A, m_Shift(m_Value(X), m_ConstantInt()))) - AddAffected(X); - } - } else { - if (Pred == ICmpInst::ICMP_NE) - if (match(A, m_And(m_Value(X), m_Power2())) && match(B, m_Zero())) - AddAffected(X); - - if (!IsAssume || Pred == ICmpInst::ICMP_ULT) { - // Handle (A + C1) u< C2, which is the canonical form of - // A > C3 && A < C4. - if (match(A, m_Add(m_Value(X), m_ConstantInt())) && - match(B, m_ConstantInt())) - AddAffected(X); - } - if (!IsAssume) { - // Handle icmp slt/sgt (bitcast X to int), 0/-1, which is supported - // by computeKnownFPClass(). - if ((Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGT) && - match(A, m_ElementWiseBitCast(m_Value(X)))) - InsertAffected(X); - } - if (IsAssume && CmpInst::isFPPredicate(Pred)) { - // fcmp fneg(x), y - // fcmp fabs(x), y - // fcmp fneg(fabs(x)), y - if (match(A, m_FNeg(m_Value(A)))) - AddAffected(A); - if (match(A, m_FAbs(m_Value(A)))) - AddAffected(A); - } - } - } - } else if ((!IsAssume && - match(Cond, m_FCmp(Pred, m_Value(A), m_Constant()))) || - match(Cond, m_Intrinsic(m_Value(A), - m_Value(B)))) { + } else if (match(V, m_Intrinsic(m_Value(A), + m_Value()))) { // Handle patterns that computeKnownFPClass() support. AddAffected(A); } diff --git a/llvm/test/Analysis/ValueTracking/numsignbits-from-assume.ll b/llvm/test/Analysis/ValueTracking/numsignbits-from-assume.ll index 00c66eeb5995..95ac98532da6 100644 --- a/llvm/test/Analysis/ValueTracking/numsignbits-from-assume.ll +++ b/llvm/test/Analysis/ValueTracking/numsignbits-from-assume.ll @@ -51,7 +51,7 @@ define i32 @computeNumSignBits_sub1(i32 %in) { define i32 @computeNumSignBits_sub2(i32 %in) { ; CHECK-LABEL: @computeNumSignBits_sub2( -; CHECK-NEXT: [[SUB:%.*]] = add i32 [[IN:%.*]], -1 +; CHECK-NEXT: [[SUB:%.*]] = add nsw i32 [[IN:%.*]], -1 ; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[SUB]], 43 ; CHECK-NEXT: call void @llvm.assume(i1 [[COND]]) ; CHECK-NEXT: [[SH:%.*]] = shl nuw nsw i32 [[SUB]], 3 diff --git a/llvm/test/Transforms/InstCombine/fpclass-from-dom-cond.ll b/llvm/test/Transforms/InstCombine/fpclass-from-dom-cond.ll index d40cd7fd503e..d6706d76056e 100644 --- a/llvm/test/Transforms/InstCombine/fpclass-from-dom-cond.ll +++ b/llvm/test/Transforms/InstCombine/fpclass-from-dom-cond.ll @@ -185,10 +185,9 @@ define i1 @test8(float %x) { ; CHECK-NEXT: [[COND:%.*]] = fcmp oeq float [[ABS]], 0x7FF0000000000000 ; CHECK-NEXT: br i1 [[COND]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] ; CHECK: if.then: -; CHECK-NEXT: [[RET1:%.*]] = call i1 @llvm.is.fpclass.f32(float [[X]], i32 575) -; CHECK-NEXT: ret i1 [[RET1]] +; CHECK-NEXT: ret i1 true ; CHECK: if.else: -; CHECK-NEXT: [[RET2:%.*]] = call i1 @llvm.is.fpclass.f32(float [[X]], i32 575) +; CHECK-NEXT: [[RET2:%.*]] = call i1 @llvm.is.fpclass.f32(float [[X]], i32 59) ; CHECK-NEXT: ret i1 [[RET2]] ; %abs = call float @llvm.fabs.f32(float %x) -- GitLab From 82cc2a67014deb1ad7509eeb287ffd8e7fa1ea1b Mon Sep 17 00:00:00 2001 From: Petr Hosek Date: Mon, 4 Mar 2024 15:07:42 -0800 Subject: [PATCH 062/929] [libc] Include stdfix.h in baremetal targets (#83900) Fixed-point arithmetic support is targeted towards baremetal targets. --- libc/config/baremetal/api.td | 1 + libc/config/baremetal/arm/entrypoints.txt | 31 ++++++++++++++++++++- libc/config/baremetal/arm/headers.txt | 1 + libc/config/baremetal/riscv/entrypoints.txt | 30 ++++++++++++++++++++ libc/config/baremetal/riscv/headers.txt | 1 + 5 files changed, 63 insertions(+), 1 deletion(-) diff --git a/libc/config/baremetal/api.td b/libc/config/baremetal/api.td index 3da83d9eb30c..a132d8308122 100644 --- a/libc/config/baremetal/api.td +++ b/libc/config/baremetal/api.td @@ -1,6 +1,7 @@ include "config/public_api.td" include "spec/stdc.td" +include "spec/stdc_ext.td" def AssertMacro : MacroDef<"assert"> { let Defn = [{ diff --git a/libc/config/baremetal/arm/entrypoints.txt b/libc/config/baremetal/arm/entrypoints.txt index a61d9feac293..c9887b6e855a 100644 --- a/libc/config/baremetal/arm/entrypoints.txt +++ b/libc/config/baremetal/arm/entrypoints.txt @@ -76,7 +76,6 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdio.vsprintf libc.src.stdio.vsnprintf - # stdbit.h entrypoints libc.src.stdbit.stdc_leading_zeros_uc libc.src.stdbit.stdc_leading_zeros_us @@ -280,6 +279,36 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.truncl ) +if(LIBC_COMPILER_HAS_FIXED_POINT) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # stdfix.h _Fract and _Accum entrypoints + libc.src.stdfix.abshk + libc.src.stdfix.abshr + libc.src.stdfix.absk + libc.src.stdfix.absr + libc.src.stdfix.abslk + libc.src.stdfix.abslr + libc.src.stdfix.roundhk + libc.src.stdfix.roundhr + libc.src.stdfix.roundk + libc.src.stdfix.roundr + libc.src.stdfix.roundlk + libc.src.stdfix.roundlr + libc.src.stdfix.rounduhk + libc.src.stdfix.rounduhr + libc.src.stdfix.rounduk + libc.src.stdfix.roundur + libc.src.stdfix.roundulk + libc.src.stdfix.roundulr + libc.src.stdfix.sqrtuhk + libc.src.stdfix.sqrtuhr + libc.src.stdfix.sqrtuk + libc.src.stdfix.sqrtur + # libc.src.stdfix.sqrtulk + libc.src.stdfix.sqrtulr + ) +endif() + set(TARGET_LLVMLIBC_ENTRYPOINTS ${TARGET_LIBC_ENTRYPOINTS} ${TARGET_LIBM_ENTRYPOINTS} diff --git a/libc/config/baremetal/arm/headers.txt b/libc/config/baremetal/arm/headers.txt index bc43abf47bbd..68d7017fda80 100644 --- a/libc/config/baremetal/arm/headers.txt +++ b/libc/config/baremetal/arm/headers.txt @@ -7,6 +7,7 @@ set(TARGET_PUBLIC_HEADERS libc.include.stdint libc.include.inttypes libc.include.math + libc.include.stdfix libc.include.stdio libc.include.stdlib libc.include.string diff --git a/libc/config/baremetal/riscv/entrypoints.txt b/libc/config/baremetal/riscv/entrypoints.txt index 533f9f9f3685..c9887b6e855a 100644 --- a/libc/config/baremetal/riscv/entrypoints.txt +++ b/libc/config/baremetal/riscv/entrypoints.txt @@ -279,6 +279,36 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.truncl ) +if(LIBC_COMPILER_HAS_FIXED_POINT) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # stdfix.h _Fract and _Accum entrypoints + libc.src.stdfix.abshk + libc.src.stdfix.abshr + libc.src.stdfix.absk + libc.src.stdfix.absr + libc.src.stdfix.abslk + libc.src.stdfix.abslr + libc.src.stdfix.roundhk + libc.src.stdfix.roundhr + libc.src.stdfix.roundk + libc.src.stdfix.roundr + libc.src.stdfix.roundlk + libc.src.stdfix.roundlr + libc.src.stdfix.rounduhk + libc.src.stdfix.rounduhr + libc.src.stdfix.rounduk + libc.src.stdfix.roundur + libc.src.stdfix.roundulk + libc.src.stdfix.roundulr + libc.src.stdfix.sqrtuhk + libc.src.stdfix.sqrtuhr + libc.src.stdfix.sqrtuk + libc.src.stdfix.sqrtur + # libc.src.stdfix.sqrtulk + libc.src.stdfix.sqrtulr + ) +endif() + set(TARGET_LLVMLIBC_ENTRYPOINTS ${TARGET_LIBC_ENTRYPOINTS} ${TARGET_LIBM_ENTRYPOINTS} diff --git a/libc/config/baremetal/riscv/headers.txt b/libc/config/baremetal/riscv/headers.txt index bc43abf47bbd..68d7017fda80 100644 --- a/libc/config/baremetal/riscv/headers.txt +++ b/libc/config/baremetal/riscv/headers.txt @@ -7,6 +7,7 @@ set(TARGET_PUBLIC_HEADERS libc.include.stdint libc.include.inttypes libc.include.math + libc.include.stdfix libc.include.stdio libc.include.stdlib libc.include.string -- GitLab From 6fd27d5b0321f65c8a09624e456e33874ae3730b Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Mon, 4 Mar 2024 18:16:16 -0500 Subject: [PATCH 063/929] [libc++] Don't generate the modulemap file (#80352) We actually didn't generate anything in that file, so generating it via CMake is useless. --- libcxx/docs/Contributing.rst | 2 +- libcxx/include/CMakeLists.txt | 9 +-------- libcxx/include/{module.modulemap.in => module.modulemap} | 0 libcxx/test/libcxx/lint/lint_headers.sh.py | 2 +- libcxx/utils/libcxx/header_information.py | 1 - 5 files changed, 3 insertions(+), 11 deletions(-) rename libcxx/include/{module.modulemap.in => module.modulemap} (100%) diff --git a/libcxx/docs/Contributing.rst b/libcxx/docs/Contributing.rst index 596d86ef2244..90aabc9c4ff6 100644 --- a/libcxx/docs/Contributing.rst +++ b/libcxx/docs/Contributing.rst @@ -156,7 +156,7 @@ sure you don't forget anything: - Did you add all new named declarations to the ``std`` module? - If you added a header: - - Did you add it to ``include/module.modulemap.in``? + - Did you add it to ``include/module.modulemap``? - Did you add it to ``include/CMakeLists.txt``? - If it's a public header, did you update ``utils/libcxx/header_information.py``? diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt index 459b077087e2..3ea3360186dc 100644 --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -968,6 +968,7 @@ set(files mdspan memory memory_resource + module.modulemap mutex new numbers @@ -1021,18 +1022,10 @@ set(files wctype.h ) -foreach(feature LIBCXX_ENABLE_FILESYSTEM LIBCXX_ENABLE_LOCALIZATION LIBCXX_ENABLE_THREADS LIBCXX_ENABLE_WIDE_CHARACTERS) - if (NOT ${${feature}}) - set(requires_${feature} "requires LIBCXX_CONFIGURED_WITHOUT_SUPPORT_FOR_THIS_HEADER") - endif() -endforeach() - configure_file("__config_site.in" "${LIBCXX_GENERATED_INCLUDE_TARGET_DIR}/__config_site" @ONLY) -configure_file("module.modulemap.in" "${LIBCXX_GENERATED_INCLUDE_DIR}/module.modulemap" @ONLY) configure_file("${LIBCXX_ASSERTION_HANDLER_FILE}" "${LIBCXX_GENERATED_INCLUDE_DIR}/__assertion_handler" COPYONLY) set(_all_includes "${LIBCXX_GENERATED_INCLUDE_TARGET_DIR}/__config_site" - "${LIBCXX_GENERATED_INCLUDE_DIR}/module.modulemap" "${LIBCXX_GENERATED_INCLUDE_DIR}/__assertion_handler") foreach(f ${files}) set(src "${CMAKE_CURRENT_SOURCE_DIR}/${f}") diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap similarity index 100% rename from libcxx/include/module.modulemap.in rename to libcxx/include/module.modulemap diff --git a/libcxx/test/libcxx/lint/lint_headers.sh.py b/libcxx/test/libcxx/lint/lint_headers.sh.py index ab237c968da7..c5e582cb0f7c 100644 --- a/libcxx/test/libcxx/lint/lint_headers.sh.py +++ b/libcxx/test/libcxx/lint/lint_headers.sh.py @@ -11,7 +11,7 @@ import re def exclude_from_consideration(path): return ( path.endswith(".txt") - or path.endswith(".modulemap.in") + or path.endswith(".modulemap") or os.path.basename(path) == "__config" or os.path.basename(path) == "__config_site.in" or os.path.basename(path) == "libcxx.imp" diff --git a/libcxx/utils/libcxx/header_information.py b/libcxx/utils/libcxx/header_information.py index b2aa3739ea69..bccae353b0c6 100644 --- a/libcxx/utils/libcxx/header_information.py +++ b/libcxx/utils/libcxx/header_information.py @@ -155,7 +155,6 @@ headers_not_available = [ def is_header(file): """Returns whether the given file is a header (i.e. not a directory or the modulemap file).""" return not file.is_dir() and not file.name in [ - "module.modulemap.in", "module.modulemap", "CMakeLists.txt", "libcxx.imp", -- GitLab From 4d80df0922ef7e48d53d4ae382977d8a2ff34ce0 Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Mon, 4 Mar 2024 18:16:52 -0500 Subject: [PATCH 064/929] [libc++] Do not forward-declare syncstream outside experimental (#82511) We only define the classes in `` when experimental library features are enabled, but we would forward-declare them in `` even when they are disabled. This led to confusing error messages about being unable to instantiate an undefined template. --- libcxx/include/iosfwd | 4 ++-- libcxx/modules/std/iosfwd.inc | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/libcxx/include/iosfwd b/libcxx/include/iosfwd index 1579fa12754d..f1c2cbd96696 100644 --- a/libcxx/include/iosfwd +++ b/libcxx/include/iosfwd @@ -142,7 +142,7 @@ typedef fpos u8streampos; typedef fpos u16streampos; typedef fpos u32streampos; -#if _LIBCPP_STD_VER >= 20 +#if _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_SYNCSTREAM) template , class _Allocator = allocator<_CharT>> class basic_syncbuf; @@ -160,7 +160,7 @@ using osyncstream = basic_osyncstream; using wosyncstream = basic_osyncstream; # endif -#endif // _LIBCPP_STD_VER >=20 +#endif // _LIBCPP_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_SYNCSTREAM) // Include other forward declarations here template > diff --git a/libcxx/modules/std/iosfwd.inc b/libcxx/modules/std/iosfwd.inc index 410fb6aefed8..090d990d061e 100644 --- a/libcxx/modules/std/iosfwd.inc +++ b/libcxx/modules/std/iosfwd.inc @@ -18,19 +18,23 @@ export namespace std { using std::u8streampos; #endif +#ifndef _LIBCPP_HAS_NO_EXPERIMENTAL_SYNCSTREAM using std::basic_osyncstream; using std::basic_syncbuf; +#endif using std::istreambuf_iterator; using std::ostreambuf_iterator; +#ifndef _LIBCPP_HAS_NO_EXPERIMENTAL_SYNCSTREAM using std::osyncstream; using std::syncbuf; -#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS +# ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS using std::wosyncstream; using std::wsyncbuf; -#endif +# endif +#endif // _LIBCPP_HAS_NO_EXPERIMENTAL_SYNCSTREAM using std::fpos; } // namespace std -- GitLab From 5174b3802575425dcbc58680ccce10961fdb8b67 Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Mon, 4 Mar 2024 18:23:53 -0500 Subject: [PATCH 065/929] [libc++] Use __wrap_iter in string_view and array in the unstable ABI (#74482) std::string_view and std::array iterators don't have to be raw pointers, and in fact other implementations don't represent them as raw pointers. Them being raw pointers in libc++ makes it easier for users to write non-portable code. This is bad in itself, but this is even worse when considering efforts like hardening where we want an easy ability to swap for a different iterator type. If users depend on iterators being raw pointers, this becomes a build break. Hence, this patch enables the use of __wrap_iter in the unstable ABI, creating a long term path towards making this the default. This patch may break code that assumes these iterators are raw pointers for people compiling with the unstable ABI. This patch also removes several assumptions that array iterators are raw pointers in the code base and in the test suite. --- libcxx/include/__config | 6 ++++++ libcxx/include/__iterator/wrap_iter.h | 4 ++++ libcxx/include/array | 22 ++++++++++++++-------- libcxx/include/string_view | 11 +++++++---- 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/libcxx/include/__config b/libcxx/include/__config index 8d4d17378b29..3a438e85a7b8 100644 --- a/libcxx/include/__config +++ b/libcxx/include/__config @@ -174,6 +174,12 @@ // The implementation moved to the header, but we still export the symbols from // the dylib for backwards compatibility. # define _LIBCPP_ABI_DO_NOT_EXPORT_TO_CHARS_BASE_10 +// Define std::array/std::string_view iterators to be __wrap_iters instead of raw +// pointers, which prevents people from relying on a non-portable implementation +// detail. This is especially useful because enabling bounded iterators hardening +// requires code not to make these assumptions. +# define _LIBCPP_ABI_USE_WRAP_ITER_IN_STD_ARRAY +# define _LIBCPP_ABI_USE_WRAP_ITER_IN_STD_STRING_VIEW # elif _LIBCPP_ABI_VERSION == 1 # if !(defined(_LIBCPP_OBJECT_FORMAT_COFF) || defined(_LIBCPP_OBJECT_FORMAT_XCOFF)) // Enable compiling copies of now inline methods into the dylib to support diff --git a/libcxx/include/__iterator/wrap_iter.h b/libcxx/include/__iterator/wrap_iter.h index 3827241e5fe4..3124826189ad 100644 --- a/libcxx/include/__iterator/wrap_iter.h +++ b/libcxx/include/__iterator/wrap_iter.h @@ -97,10 +97,14 @@ private: friend class __wrap_iter; template friend class basic_string; + template + friend class basic_string_view; template friend class _LIBCPP_TEMPLATE_VIS vector; template friend class _LIBCPP_TEMPLATE_VIS span; + template + friend struct array; }; template diff --git a/libcxx/include/array b/libcxx/include/array index 961b620efb93..7fa5dc147934 100644 --- a/libcxx/include/array +++ b/libcxx/include/array @@ -120,6 +120,7 @@ template const T&& get(const array&&) noexce #include <__config> #include <__fwd/array.h> #include <__iterator/reverse_iterator.h> +#include <__iterator/wrap_iter.h> #include <__tuple/sfinae_helpers.h> #include <__type_traits/conditional.h> #include <__type_traits/is_array.h> @@ -167,14 +168,19 @@ _LIBCPP_BEGIN_NAMESPACE_STD template struct _LIBCPP_TEMPLATE_VIS array { // types: - using __self = array; - using value_type = _Tp; - using reference = value_type&; - using const_reference = const value_type&; - using iterator = value_type*; - using const_iterator = const value_type*; - using pointer = value_type*; - using const_pointer = const value_type*; + using __self = array; + using value_type = _Tp; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = value_type*; + using const_pointer = const value_type*; +#if defined(_LIBCPP_ABI_USE_WRAP_ITER_IN_STD_ARRAY) + using iterator = __wrap_iter; + using const_iterator = __wrap_iter; +#else + using iterator = pointer; + using const_iterator = const_pointer; +#endif using size_type = size_t; using difference_type = ptrdiff_t; using reverse_iterator = std::reverse_iterator; diff --git a/libcxx/include/string_view b/libcxx/include/string_view index 48bbcd800216..e0dd5c5b19ac 100644 --- a/libcxx/include/string_view +++ b/libcxx/include/string_view @@ -215,6 +215,7 @@ namespace std { #include <__iterator/concepts.h> #include <__iterator/iterator_traits.h> #include <__iterator/reverse_iterator.h> +#include <__iterator/wrap_iter.h> #include <__memory/pointer_traits.h> #include <__ranges/concepts.h> #include <__ranges/data.h> @@ -278,10 +279,12 @@ public: using const_pointer = const _CharT*; using reference = _CharT&; using const_reference = const _CharT&; -#ifdef _LIBCPP_ABI_BOUNDED_ITERATORS +#if defined(_LIBCPP_ABI_BOUNDED_ITERATORS) using const_iterator = __bounded_iter; +#elif defined(_LIBCPP_ABI_USE_WRAP_ITER_IN_STD_STRING_VIEW) + using const_iterator = __wrap_iter; #else - using const_iterator = const_pointer; // See [string.view.iterators] + using const_iterator = const_pointer; #endif using iterator = const_iterator; using const_reverse_iterator = std::reverse_iterator; @@ -353,7 +356,7 @@ public: #ifdef _LIBCPP_ABI_BOUNDED_ITERATORS return std::__make_bounded_iter(data(), data(), data() + size()); #else - return __data_; + return const_iterator(__data_); #endif } @@ -361,7 +364,7 @@ public: #ifdef _LIBCPP_ABI_BOUNDED_ITERATORS return std::__make_bounded_iter(data() + size(), data(), data() + size()); #else - return __data_ + __size_; + return const_iterator(__data_ + __size_); #endif } -- GitLab From 40081a45a14f7aa6249fa034d961549c0b1762a0 Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Mon, 4 Mar 2024 18:24:51 -0500 Subject: [PATCH 066/929] [libc++] Fix diagnostic for before C++23 (#83351) We normally try to issue a reasonable diagnostic when mixing and before C++23. However, after granularizing the header, the check and the #error message was moved to *after* the point where mixing both causes problems. When mixing both headers, we would hence get the diagnostic burried under a pile of previous diagnostics in e.g. __atomic/kill_dependency.h. This patch moves the check earlier to restore the intended behavior. It also switches from `#ifdef kill_dependency` to an explicit check of the inclusion of the header and the Standard version, which seems to be more reliable than checking whether a macro is defined. --- libcxx/include/atomic | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/libcxx/include/atomic b/libcxx/include/atomic index 61ff61d415dd..cb142b09bff3 100644 --- a/libcxx/include/atomic +++ b/libcxx/include/atomic @@ -587,6 +587,12 @@ template */ +#include <__config> + +#if _LIBCPP_STD_VER < 23 && defined(_LIBCPP_STDATOMIC_H) +# error is incompatible with before C++23. Please compile with -std=c++23. +#endif + #include <__atomic/aliases.h> #include <__atomic/atomic.h> #include <__atomic/atomic_base.h> @@ -601,7 +607,6 @@ template #include <__atomic/is_always_lock_free.h> #include <__atomic/kill_dependency.h> #include <__atomic/memory_order.h> -#include <__config> #include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) @@ -612,10 +617,6 @@ template # error is not implemented #endif -#ifdef kill_dependency -# error is incompatible with before C++23. Please compile with -std=c++23. -#endif - #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20 # include # include -- GitLab From 07b1aebced8015c62d4ce0afe07358049afae5b1 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Mon, 4 Mar 2024 17:38:42 -0600 Subject: [PATCH 067/929] [LLVM][NFC] Ignore including the GNUInstallDirs on the GPU (#83910) Summary: I've begun treating GPU runtimes builds as cross-compiling with the LLVM infrastructure. However, we include a lot of random stuff that the GPU build isn't prepared to handle. This currently emits a warning, and while it's not striclty necessary, is annoying. This patch suppresses it by not including the standard GNU install directory resources when used from the GPU. --- llvm/cmake/modules/AddLLVM.cmake | 5 ++++- llvm/cmake/modules/HandleLLVMOptions.cmake | 3 +-- runtimes/CMakeLists.txt | 6 ++++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/llvm/cmake/modules/AddLLVM.cmake b/llvm/cmake/modules/AddLLVM.cmake index 828de4bd9940..e18ec23fc76f 100644 --- a/llvm/cmake/modules/AddLLVM.cmake +++ b/llvm/cmake/modules/AddLLVM.cmake @@ -1,4 +1,7 @@ -include(GNUInstallDirs) +if(NOT LLVM_RUNTIMES_GPU_BUILD) + include(GNUInstallDirs) +endif() + include(LLVMDistributionSupport) include(LLVMProcessSources) include(LLVM-Config) diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake index 08ff49ded57a..eca2962cf820 100644 --- a/llvm/cmake/modules/HandleLLVMOptions.cmake +++ b/llvm/cmake/modules/HandleLLVMOptions.cmake @@ -122,8 +122,7 @@ endif() # If we are targeting a GPU architecture in a runtimes build we want to ignore # all the standard flag handling. -if("${LLVM_RUNTIMES_TARGET}" MATCHES "^amdgcn" OR - "${LLVM_RUNTIMES_TARGET}" MATCHES "^nvptx64") +if(LLVM_RUNTIMES_GPU_BUILD) return() endif() diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt index 634ffe710b06..29b47b862c21 100644 --- a/runtimes/CMakeLists.txt +++ b/runtimes/CMakeLists.txt @@ -151,6 +151,12 @@ endif() # Avoid checking whether the compiler is working. set(LLVM_COMPILER_CHECKED ON) +# This can be used to detect whether we're targeting a GPU architecture. +if("${LLVM_RUNTIMES_TARGET}" MATCHES "^amdgcn" OR + "${LLVM_RUNTIMES_TARGET}" MATCHES "^nvptx64") + set(LLVM_RUNTIMES_GPU_BUILD ON) +endif() + # Handle common options used by all runtimes. include(AddLLVM) include(HandleLLVMOptions) -- GitLab From 5000e4c2527ae53bf7c1a609f739a97cdc522bbe Mon Sep 17 00:00:00 2001 From: Med Ismail Bennani Date: Mon, 4 Mar 2024 15:44:44 -0800 Subject: [PATCH 068/929] [lldb/crashlog] Fix breaking changes in textual report format (#83861) This patch should address some register parsing issue in the legacy report format. rdar://107210149 Signed-off-by: Med Ismail Bennani --- lldb/examples/python/crashlog.py | 11 ++-- .../Crashlog/Inputs/altered_threadState.crash | 50 +++++++++++++++++++ .../Python/Crashlog/altered_threadState.test | 13 +++++ 3 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 lldb/test/Shell/ScriptInterpreter/Python/Crashlog/Inputs/altered_threadState.crash create mode 100644 lldb/test/Shell/ScriptInterpreter/Python/Crashlog/altered_threadState.test diff --git a/lldb/examples/python/crashlog.py b/lldb/examples/python/crashlog.py index 9e4f94264037..c992348b24be 100755 --- a/lldb/examples/python/crashlog.py +++ b/lldb/examples/python/crashlog.py @@ -849,10 +849,10 @@ class JSONCrashLogParser(CrashLogParser): class TextCrashLogParser(CrashLogParser): parent_process_regex = re.compile(r"^Parent Process:\s*(.*)\[(\d+)\]") - thread_state_regex = re.compile(r"^Thread \d+ crashed with") + thread_state_regex = re.compile(r"^Thread (\d+ crashed with|State)") thread_instrs_regex = re.compile(r"^Thread \d+ instruction stream") - thread_regex = re.compile(r"^Thread (\d+).*:") - app_backtrace_regex = re.compile(r"^Application Specific Backtrace (\d+).*:") + thread_regex = re.compile(r"^Thread (\d+).*") + app_backtrace_regex = re.compile(r"^Application Specific Backtrace (\d+).*") class VersionRegex: version = r"\(.+\)|(?:arm|x86_)[0-9a-z]+" @@ -1081,7 +1081,10 @@ class TextCrashLogParser(CrashLogParser): if thread_state_match: self.app_specific_backtrace = False thread_state_match = self.thread_regex.search(line) - thread_idx = int(thread_state_match.group(1)) + if thread_state_match: + thread_idx = int(thread_state_match.group(1)) + else: + thread_idx = self.crashlog.crashed_thread_idx self.parse_mode = self.CrashLogParseMode.THREGS self.thread = self.crashlog.threads[thread_idx] return diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/Inputs/altered_threadState.crash b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/Inputs/altered_threadState.crash new file mode 100644 index 000000000000..8fcbdd24b3cf --- /dev/null +++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/Inputs/altered_threadState.crash @@ -0,0 +1,50 @@ +Process: a.out [21606] +Path: /private/tmp/a.out +Identifier: a.out +Version: 0 +Code Type: X86-64 (Native) +Parent Process: fish [88883] +User ID: 501 + +Date/Time: 2020-11-11 14:47:34.600 -0800 +OS Version: macOS 11.0.1 +Report Version: 12 +Bridge OS Version: redacted +Anonymous UUID: DCEF35CB-68D5-F524-FF13-060901F52EA8 + + +Time Awake Since Boot: 400000 seconds + +System Integrity Protection: enabled + +Crashed Thread: 0 Dispatch queue: com.apple.main-thread + +Exception Type: EXC_BAD_ACCESS (SIGSEGV) +Exception Codes: KERN_INVALID_ADDRESS at 0x0000000000000000 +Exception Note: EXC_CORPSE_NOTIFY + +Termination Signal: Segmentation fault: 11 +Termination Reason: Namespace SIGNAL, Code 0xb +Terminating Process: exc handler [21606] + +Thread 0 Crashed:: Dispatch queue: com.apple.main-thread +0 a.out @foo@ foo + 16 (test.c:3) +1 a.out @bar@ +2 a.out @main@ main + 20 (test.c:8) +3 libdyld.dylib 0x1000000 start + 1 + +Thread State + rax: 0x0000000000000000 rbx: 0x0000000000000000 rcx: 0x00007ffee42d81d0 rdx: 0x00007ffee42d8080 + rdi: 0x0000000000000001 rsi: 0x00007ffee42d8070 rbp: 0x00007ffee42d8020 rsp: 0x00007ffee42d8020 + r8: 0x0000000000000000 r9: 0x0000000000000000 r10: 0x0000000000000000 r11: 0x0000000000000000 + r12: 0x0000000000000000 r13: 0x0000000000000000 r14: 0x0000000000000000 r15: 0x0000000000000000 + rip: 0x000000010b92af70 rfl: 0x0000000000010202 cr2: 0x0000000000000000 + +Logical CPU: 2 +Error Code: 0x00000006 (no mapping for user data write) +Trap Number: 14 + + +Binary Images: + 0x100000000 - 0x200000000 +a.out (0) <@UUID@> @EXEC@ + 0x0 - 0xffffffffffffffff ??? (*) <00000000-0000-0000-0000-000000000000> ??? diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/altered_threadState.test b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/altered_threadState.test new file mode 100644 index 000000000000..5a946a38b195 --- /dev/null +++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/altered_threadState.test @@ -0,0 +1,13 @@ +# RUN: %clang_host -g %S/Inputs/test.c -o %t.out +# RUN: cp %S/Inputs/altered_threadState.crash %t.crash +# RUN: %python %S/patch-crashlog.py --binary %t.out --crashlog %t.crash --offsets '{"main":20, "bar":9, "foo":16}' +# RUN: %lldb %t.out -o 'command script import lldb.macosx.crashlog' -o 'crashlog %t.crash' 2>&1 | FileCheck %s + +# CHECK: "crashlog" {{.*}} commands have been installed, use the "--help" options on these commands + +# CHECK: Thread[0] EXC_BAD_ACCESS (SIGSEGV) (KERN_INVALID_ADDRESS at 0x0000000000000000) +# CHECK: [ 0] {{.*}}out`foo + 16 at test.c +# CHECK: [ 1] {{.*}}out`bar + 8 at test.c +# CHECK: [ 2] {{.*}}out`main + 19 at test.c +# CHECK: [ 3] 0x{{[0]+}}1000000 start + 1 +# CHECK: rbp = 0x00007ffee42d8020 -- GitLab From 488ac3d5ef31237e38de6da627a619459e0ca19a Mon Sep 17 00:00:00 2001 From: Felipe de Azevedo Piovezan Date: Mon, 4 Mar 2024 15:45:43 -0800 Subject: [PATCH 069/929] [lldb] Enable a test that was never enabled (#83925) According to the git log (d9442afba1bd6), this test has never been enabled/disabled, it was checked in without being called anywhere. But it passes and it is useful, so this commit enables it. --- .../breakpoint/breakpoint_options/TestBreakpointOptions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_options/TestBreakpointOptions.py b/lldb/test/API/functionalities/breakpoint/breakpoint_options/TestBreakpointOptions.py index b2c9551eda6a..129290909029 100644 --- a/lldb/test/API/functionalities/breakpoint/breakpoint_options/TestBreakpointOptions.py +++ b/lldb/test/API/functionalities/breakpoint/breakpoint_options/TestBreakpointOptions.py @@ -13,6 +13,7 @@ class BreakpointOptionsTestCase(TestBase): """Test breakpoint command for different options.""" self.build() self.breakpoint_options_test() + self.breakpoint_options_language_test() def setUp(self): # Call super's setUp(). -- GitLab From 05390df497535b26879a7a96e03a76af26c8bcd3 Mon Sep 17 00:00:00 2001 From: Aart Bik Date: Mon, 4 Mar 2024 15:49:09 -0800 Subject: [PATCH 070/929] [mlir][sparse] migration to sparse_tensor.print (#83926) Continuing the efforts started in #83357 --- .../Dialect/SparseTensor/CPU/sparse_abs.mlir | 37 +-- .../SparseTensor/CPU/sparse_binary.mlir | 260 ++++++++++-------- 2 files changed, 165 insertions(+), 132 deletions(-) diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_abs.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_abs.mlir index 3c10e2662f86..4228bcdb1c0d 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_abs.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_abs.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -73,7 +73,7 @@ module { } // Driver method to call and verify sign kernel. - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %df = arith.constant 99.99 : f64 %di = arith.constant 9999 : i32 @@ -116,21 +116,26 @@ module { // // Verify the results. // - // CHECK: 12 - // CHECK-NEXT: ( 1.5, 1.5, 10.2, 11.3, 1, 1, nan, nan, inf, inf, 0, 0 ) - // CHECK-NEXT: 9 - // CHECK-NEXT: ( -2147483648, 2147483647, 1000, 1, 0, 1, 1000, 2147483646, 2147483647 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 12 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 12, + // CHECK-NEXT: crd[0] : ( 0, 3, 5, 11, 13, 17, 18, 20, 21, 28, 29, 31, + // CHECK-NEXT: values : ( 1.5, 1.5, 10.2, 11.3, 1, 1, nan, nan, inf, inf, 0, 0, + // CHECK-NEXT: ---- // - %x = sparse_tensor.values %0 : tensor to memref - %y = sparse_tensor.values %1 : tensor to memref - %a = vector.transfer_read %x[%c0], %df: memref, vector<12xf64> - %b = vector.transfer_read %y[%c0], %di: memref, vector<9xi32> - %na = sparse_tensor.number_of_entries %0 : tensor - %nb = sparse_tensor.number_of_entries %1 : tensor - vector.print %na : index - vector.print %a : vector<12xf64> - vector.print %nb : index - vector.print %b : vector<9xi32> + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 9, + // CHECK-NEXT: crd[0] : ( 0, 3, 5, 11, 13, 17, 18, 21, 31, + // CHECK-NEXT: values : ( -2147483648, 2147483647, 1000, 1, 0, 1, 1000, 2147483646, 2147483647, + // CHECK-NEXT: ---- + // + sparse_tensor.print %0 : tensor + sparse_tensor.print %1 : tensor // Release the resources. bufferization.dealloc_tensor %sv1 : tensor diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_binary.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_binary.mlir index 59ecbfdef850..36701b4385a2 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_binary.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_binary.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -365,84 +365,8 @@ module { return %0 : tensor<4x4xf64, #DCSR> } - // - // Utility functions to dump the value of a tensor. - // - - func.func @dump_vec(%arg0: tensor) { - // Dump the values array to verify only sparse contents are stored. - %c0 = arith.constant 0 : index - %d0 = arith.constant 0.0 : f64 - %0 = sparse_tensor.values %arg0 : tensor to memref - %1 = vector.transfer_read %0[%c0], %d0: memref, vector<16xf64> - vector.print %1 : vector<16xf64> - // Dump the dense vector to verify structure is correct. - %dv = sparse_tensor.convert %arg0 : tensor to tensor - %3 = vector.transfer_read %dv[%c0], %d0: tensor, vector<32xf64> - vector.print %3 : vector<32xf64> - bufferization.dealloc_tensor %dv : tensor - return - } - - func.func @dump_vec_i32(%arg0: tensor) { - // Dump the values array to verify only sparse contents are stored. - %c0 = arith.constant 0 : index - %d0 = arith.constant 0 : i32 - %0 = sparse_tensor.values %arg0 : tensor to memref - %1 = vector.transfer_read %0[%c0], %d0: memref, vector<24xi32> - vector.print %1 : vector<24xi32> - // Dump the dense vector to verify structure is correct. - %dv = sparse_tensor.convert %arg0 : tensor to tensor - %3 = vector.transfer_read %dv[%c0], %d0: tensor, vector<32xi32> - vector.print %3 : vector<32xi32> - bufferization.dealloc_tensor %dv : tensor - return - } - - func.func @dump_mat(%arg0: tensor) { - %d0 = arith.constant 0.0 : f64 - %c0 = arith.constant 0 : index - %dm = sparse_tensor.convert %arg0 : tensor to tensor - %1 = vector.transfer_read %dm[%c0, %c0], %d0: tensor, vector<4x8xf64> - vector.print %1 : vector<4x8xf64> - bufferization.dealloc_tensor %dm : tensor - return - } - - func.func @dump_mat_4x4(%A: tensor<4x4xf64, #DCSR>) { - %c0 = arith.constant 0 : index - %du = arith.constant 0.0 : f64 - - %c = sparse_tensor.convert %A : tensor<4x4xf64, #DCSR> to tensor<4x4xf64> - %v = vector.transfer_read %c[%c0, %c0], %du: tensor<4x4xf64>, vector<4x4xf64> - vector.print %v : vector<4x4xf64> - - %1 = sparse_tensor.values %A : tensor<4x4xf64, #DCSR> to memref - %2 = vector.transfer_read %1[%c0], %du: memref, vector<16xf64> - vector.print %2 : vector<16xf64> - - bufferization.dealloc_tensor %c : tensor<4x4xf64> - return - } - - func.func @dump_mat_4x4_i8(%A: tensor<4x4xi8, #DCSR>) { - %c0 = arith.constant 0 : index - %du = arith.constant 0 : i8 - - %c = sparse_tensor.convert %A : tensor<4x4xi8, #DCSR> to tensor<4x4xi8> - %v = vector.transfer_read %c[%c0, %c0], %du: tensor<4x4xi8>, vector<4x4xi8> - vector.print %v : vector<4x4xi8> - - %1 = sparse_tensor.values %A : tensor<4x4xi8, #DCSR> to memref - %2 = vector.transfer_read %1[%c0], %du: memref, vector<16xi8> - vector.print %2 : vector<16xi8> - - bufferization.dealloc_tensor %c : tensor<4x4xi8> - return - } - // Driver method to call and verify kernels. - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index // Setup sparse vectors. @@ -525,45 +449,149 @@ module { // // Verify the results. // - // CHECK: ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0, 7, 8, 0, 9 ) - // CHECK-NEXT: ( 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 11, 0, 12, 13, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 15, 0, 16, 0, 0, 17, 0, 0, 0, 0, 0, 0, 18, 19, 0, 20 ) - // CHECK-NEXT: ( 1, 11, 2, 13, 14, 3, 15, 4, 16, 5, 6, 7, 8, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 1, 11, 0, 2, 13, 0, 0, 0, 0, 0, 14, 3, 0, 0, 0, 0, 15, 4, 16, 0, 5, 6, 0, 0, 0, 0, 0, 0, 7, 8, 0, 9 ) - // CHECK-NEXT: ( 0, 6, 3, 28, 0, 6, 56, 72, 9, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 28, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 56, 72, 0, 9 ) - // CHECK-NEXT: ( 1, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 3, 11, 17, 20, 21, 28, 29, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 17, 0, 0, 20, 21, 0, 0, 0, 0, 0, 0, 28, 29, 0, 31 ) - // CHECK-NEXT: ( ( 7, 0, 0, 0, 0, 0, 0, -5 ), ( -4, 0, 0, 0, 0, 0, -3, 0 ), ( 0, -2, 0, 0, 0, 0, 0, 7 ), ( 0, 0, 0, 0, 0, 0, 0, 0 ) ) - // CHECK-NEXT: ( ( 2, 0, 4, 1 ), ( 0, 2.5, 0, 0 ), ( 1, 5, 2, 4 ), ( 5, 4, 0, 0 ) ) - // CHECK-NEXT: ( 2, 4, 1, 2.5, 1, 5, 2, 4, 5, 4, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( ( 2, 0, 4, 1 ), ( 0, 2.5, 0, 0 ), ( 1, 5, 2, 4 ), ( 5, 4, 0, 0 ) ) - // CHECK-NEXT: ( 2, 4, 1, 2.5, 1, 5, 2, 4, 5, 4, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( ( 2, 0, 4, 1 ), ( 0, 2.5, 0, 0 ), ( -1, -5, 2, 4 ), ( 1, 4, 0, 0 ) ) - // CHECK-NEXT: ( 2, 4, 1, 2.5, -1, -5, 2, 4, 1, 4, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( ( 0, 0, 1, -1 ), ( 0, 1, 0, 0 ), ( -1, -2, -2, 2 ), ( 1, 2, 0, 0 ) ) - // CHECK-NEXT: ( 0, 1, -1, 1, -1, -2, -2, 2, 1, 2, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( ( 1, 0, 0, 0 ), ( 0, 0, 0, 0 ), ( 0, 0, 0, 0 ), ( 0, 0, 0, 0 ) ) - // CHECK-NEXT: ( 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( ( 0, 0, 0, -1 ), ( 0, 0, 0, 0 ), ( -1, -5, -2, 4 ), ( 0, 4, 0, 0 ) ) - // CHECK-NEXT: ( -1, -1, -5, -2, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 9, + // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31, + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, + // CHECK-NEXT: ---- + // + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 10 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 10, + // CHECK-NEXT: crd[0] : ( 1, 3, 4, 10, 16, 18, 21, 28, 29, 31, + // CHECK-NEXT: values : ( 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + // CHECK-NEXT: ---- + // + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 14 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 14, + // CHECK-NEXT: crd[0] : ( 0, 1, 3, 4, 10, 11, 16, 17, 18, 20, 21, 28, 29, 31, + // CHECK-NEXT: values : ( 1, 11, 2, 13, 14, 3, 15, 4, 16, 5, 6, 7, 8, 9, + // CHECK-NEXT: ---- + // + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 9, + // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31, + // CHECK-NEXT: values : ( 0, 6, 3, 28, 0, 6, 56, 72, 9, + // CHECK-NEXT: ---- + // + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 4 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 4, + // CHECK-NEXT: crd[0] : ( 0, 11, 17, 20, + // CHECK-NEXT: values : ( 1, 3, 4, 5, + // CHECK-NEXT: ---- + // + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 9, + // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31, + // CHECK-NEXT: values : ( 0, 3, 11, 17, 20, 21, 28, 29, 31, + // CHECK-NEXT: ---- + // + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 6 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 4, 8 ) + // CHECK-NEXT: pos[0] : ( 0, 3, + // CHECK-NEXT: crd[0] : ( 0, 1, 2, + // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6, + // CHECK-NEXT: crd[1] : ( 0, 7, 0, 6, 1, 7, + // CHECK-NEXT: values : ( 7, -5, -4, -3, -2, 7, + // CHECK-NEXT: ---- + // + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 10 + // CHECK-NEXT: dim = ( 4, 4 ) + // CHECK-NEXT: lvl = ( 4, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 4, + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, + // CHECK-NEXT: pos[1] : ( 0, 3, 4, 8, 10, + // CHECK-NEXT: crd[1] : ( 0, 2, 3, 1, 0, 1, 2, 3, 0, 1, + // CHECK-NEXT: values : ( 2, 4, 1, 2.5, 1, 5, 2, 4, 5, 4, + // CHECK-NEXT: ---- + // + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 10 + // CHECK-NEXT: dim = ( 4, 4 ) + // CHECK-NEXT: lvl = ( 4, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 4, + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, + // CHECK-NEXT: pos[1] : ( 0, 3, 4, 8, 10, + // CHECK-NEXT: crd[1] : ( 0, 2, 3, 1, 0, 1, 2, 3, 0, 1, + // CHECK-NEXT: values : ( 2, 4, 1, 2.5, 1, 5, 2, 4, 5, 4, + // CHECK-NEXT: ---- + // + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 10 + // CHECK-NEXT: dim = ( 4, 4 ) + // CHECK-NEXT: lvl = ( 4, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 4, + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, + // CHECK-NEXT: pos[1] : ( 0, 3, 4, 8, 10, + // CHECK-NEXT: crd[1] : ( 0, 2, 3, 1, 0, 1, 2, 3, 0, 1, + // CHECK-NEXT: values : ( 2, 4, 1, 2.5, -1, -5, 2, 4, 1, 4, + // CHECK-NEXT: ---- + // + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 10 + // CHECK-NEXT: dim = ( 4, 4 ) + // CHECK-NEXT: lvl = ( 4, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 4, + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, + // CHECK-NEXT: pos[1] : ( 0, 3, 4, 8, 10, + // CHECK-NEXT: crd[1] : ( 0, 2, 3, 1, 0, 1, 2, 3, 0, 1, + // CHECK-NEXT: values : ( 0, 1, -1, 1, -1, -2, -2, 2, 1, 2, + // CHECK-NEXT: ---- + // + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 4 + // CHECK-NEXT: dim = ( 4, 4 ) + // CHECK-NEXT: lvl = ( 4, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 3, + // CHECK-NEXT: crd[0] : ( 0, 1, 3, + // CHECK-NEXT: pos[1] : ( 0, 2, 3, 4, + // CHECK-NEXT: crd[1] : ( 0, 2, 1, 0, + // CHECK-NEXT: values : ( 1, 0, 0, 0, + // CHECK-NEXT: ---- + // + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 6 + // CHECK-NEXT: dim = ( 4, 4 ) + // CHECK-NEXT: lvl = ( 4, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 3, + // CHECK-NEXT: crd[0] : ( 0, 2, 3, + // CHECK-NEXT: pos[1] : ( 0, 1, 5, 6, + // CHECK-NEXT: crd[1] : ( 3, 0, 1, 2, 3, 1, + // CHECK-NEXT: values : ( -1, -1, -5, -2, 4, 4, // - call @dump_vec(%sv1) : (tensor) -> () - call @dump_vec(%sv2) : (tensor) -> () - call @dump_vec_i32(%0) : (tensor) -> () - call @dump_vec(%1) : (tensor) -> () - call @dump_vec(%2) : (tensor) -> () - call @dump_vec_i32(%3) : (tensor) -> () - call @dump_mat(%5) : (tensor) -> () - call @dump_mat_4x4(%6) : (tensor<4x4xf64, #DCSR>) -> () - call @dump_mat_4x4(%7) : (tensor<4x4xf64, #DCSR>) -> () - call @dump_mat_4x4(%8) : (tensor<4x4xf64, #DCSR>) -> () - call @dump_mat_4x4(%9) : (tensor<4x4xf64, #DCSR>) -> () - call @dump_mat_4x4_i8(%10) : (tensor<4x4xi8, #DCSR>) -> () - call @dump_mat_4x4(%11) : (tensor<4x4xf64, #DCSR>) -> () + sparse_tensor.print %sv1 : tensor + sparse_tensor.print %sv2 : tensor + sparse_tensor.print %0 : tensor + sparse_tensor.print %1 : tensor + sparse_tensor.print %2 : tensor + sparse_tensor.print %3 : tensor + sparse_tensor.print %5 : tensor + sparse_tensor.print %6 : tensor<4x4xf64, #DCSR> + sparse_tensor.print %7 : tensor<4x4xf64, #DCSR> + sparse_tensor.print %8 : tensor<4x4xf64, #DCSR> + sparse_tensor.print %9 : tensor<4x4xf64, #DCSR> + sparse_tensor.print %10 : tensor<4x4xi8, #DCSR> + sparse_tensor.print %11 : tensor<4x4xf64, #DCSR> // Release the resources. bufferization.dealloc_tensor %sv1 : tensor -- GitLab From 8e56fb824a43d54208d44a403366faa5d633ee8b Mon Sep 17 00:00:00 2001 From: isuckatcs <65320245+isuckatcs@users.noreply.github.com> Date: Tue, 5 Mar 2024 01:09:39 +0100 Subject: [PATCH 071/929] [clang-tidy] CRTP Constructor Accessibility Check (#82403) Detects error-prone Curiously Recurring Template Pattern usage, when the CRTP can be constructed outside itself and the derived class. --- .../bugprone/BugproneTidyModule.cpp | 3 + .../clang-tidy/bugprone/CMakeLists.txt | 1 + .../CrtpConstructorAccessibilityCheck.cpp | 181 +++++++++++++ .../CrtpConstructorAccessibilityCheck.h | 32 +++ clang-tools-extra/docs/ReleaseNotes.rst | 6 + .../crtp-constructor-accessibility.rst | 98 +++++++ .../docs/clang-tidy/checks/list.rst | 1 + .../crtp-constructor-accessibility.cpp | 255 ++++++++++++++++++ 8 files changed, 577 insertions(+) create mode 100644 clang-tools-extra/clang-tidy/bugprone/CrtpConstructorAccessibilityCheck.cpp create mode 100644 clang-tools-extra/clang-tidy/bugprone/CrtpConstructorAccessibilityCheck.h create mode 100644 clang-tools-extra/docs/clang-tidy/checks/bugprone/crtp-constructor-accessibility.rst create mode 100644 clang-tools-extra/test/clang-tidy/checkers/bugprone/crtp-constructor-accessibility.cpp diff --git a/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp b/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp index a8a23b045f80..e518a64abc52 100644 --- a/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp @@ -20,6 +20,7 @@ #include "ChainedComparisonCheck.h" #include "ComparePointerToMemberVirtualFunctionCheck.h" #include "CopyConstructorInitCheck.h" +#include "CrtpConstructorAccessibilityCheck.h" #include "DanglingHandleCheck.h" #include "DynamicStaticInitializersCheck.h" #include "EasilySwappableParametersCheck.h" @@ -237,6 +238,8 @@ public: "bugprone-unhandled-exception-at-new"); CheckFactories.registerCheck( "bugprone-unique-ptr-array-mismatch"); + CheckFactories.registerCheck( + "bugprone-crtp-constructor-accessibility"); CheckFactories.registerCheck( "bugprone-unsafe-functions"); CheckFactories.registerCheck( diff --git a/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt b/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt index 1cd6fb207d76..638fba03a435 100644 --- a/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt +++ b/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt @@ -79,6 +79,7 @@ add_clang_library(clangTidyBugproneModule UnhandledExceptionAtNewCheck.cpp UnhandledSelfAssignmentCheck.cpp UniquePtrArrayMismatchCheck.cpp + CrtpConstructorAccessibilityCheck.cpp UnsafeFunctionsCheck.cpp UnusedLocalNonTrivialVariableCheck.cpp UnusedRaiiCheck.cpp diff --git a/clang-tools-extra/clang-tidy/bugprone/CrtpConstructorAccessibilityCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/CrtpConstructorAccessibilityCheck.cpp new file mode 100644 index 000000000000..6175fcdfd229 --- /dev/null +++ b/clang-tools-extra/clang-tidy/bugprone/CrtpConstructorAccessibilityCheck.cpp @@ -0,0 +1,181 @@ +//===--- CrtpConstructorAccessibilityCheck.cpp - clang-tidy ---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "CrtpConstructorAccessibilityCheck.h" +#include "../utils/LexerUtils.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" + +using namespace clang::ast_matchers; + +namespace clang::tidy::bugprone { + +static bool hasPrivateConstructor(const CXXRecordDecl *RD) { + return llvm::any_of(RD->ctors(), [](const CXXConstructorDecl *Ctor) { + return Ctor->getAccess() == AS_private; + }); +} + +static bool isDerivedParameterBefriended(const CXXRecordDecl *CRTP, + const NamedDecl *Param) { + return llvm::any_of(CRTP->friends(), [&](const FriendDecl *Friend) { + const TypeSourceInfo *const FriendType = Friend->getFriendType(); + if (!FriendType) { + return false; + } + + const auto *const TTPT = + dyn_cast(FriendType->getType()); + + return TTPT && TTPT->getDecl() == Param; + }); +} + +static bool isDerivedClassBefriended(const CXXRecordDecl *CRTP, + const CXXRecordDecl *Derived) { + return llvm::any_of(CRTP->friends(), [&](const FriendDecl *Friend) { + const TypeSourceInfo *const FriendType = Friend->getFriendType(); + if (!FriendType) { + return false; + } + + return FriendType->getType()->getAsCXXRecordDecl() == Derived; + }); +} + +static const NamedDecl * +getDerivedParameter(const ClassTemplateSpecializationDecl *CRTP, + const CXXRecordDecl *Derived) { + size_t Idx = 0; + const bool AnyOf = llvm::any_of( + CRTP->getTemplateArgs().asArray(), [&](const TemplateArgument &Arg) { + ++Idx; + return Arg.getKind() == TemplateArgument::Type && + Arg.getAsType()->getAsCXXRecordDecl() == Derived; + }); + + return AnyOf ? CRTP->getSpecializedTemplate() + ->getTemplateParameters() + ->getParam(Idx - 1) + : nullptr; +} + +static std::vector +hintMakeCtorPrivate(const CXXConstructorDecl *Ctor, + const std::string &OriginalAccess) { + std::vector Hints; + + Hints.emplace_back(FixItHint::CreateInsertion( + Ctor->getBeginLoc().getLocWithOffset(-1), "private:\n")); + + const ASTContext &ASTCtx = Ctor->getASTContext(); + const SourceLocation CtorEndLoc = + Ctor->isExplicitlyDefaulted() + ? utils::lexer::findNextTerminator(Ctor->getEndLoc(), + ASTCtx.getSourceManager(), + ASTCtx.getLangOpts()) + : Ctor->getEndLoc(); + Hints.emplace_back(FixItHint::CreateInsertion( + CtorEndLoc.getLocWithOffset(1), '\n' + OriginalAccess + ':' + '\n')); + + return Hints; +} + +void CrtpConstructorAccessibilityCheck::registerMatchers(MatchFinder *Finder) { + Finder->addMatcher( + classTemplateSpecializationDecl( + decl().bind("crtp"), + hasAnyTemplateArgument(refersToType(recordType(hasDeclaration( + cxxRecordDecl( + isDerivedFrom(cxxRecordDecl(equalsBoundNode("crtp")))) + .bind("derived")))))), + this); +} + +void CrtpConstructorAccessibilityCheck::check( + const MatchFinder::MatchResult &Result) { + const auto *CRTPInstantiation = + Result.Nodes.getNodeAs("crtp"); + const auto *DerivedRecord = Result.Nodes.getNodeAs("derived"); + const CXXRecordDecl *CRTPDeclaration = + CRTPInstantiation->getSpecializedTemplate()->getTemplatedDecl(); + + if (!CRTPDeclaration->hasDefinition()) { + return; + } + + const auto *DerivedTemplateParameter = + getDerivedParameter(CRTPInstantiation, DerivedRecord); + + assert(DerivedTemplateParameter && + "No template parameter corresponds to the derived class of the CRTP."); + + bool NeedsFriend = !isDerivedParameterBefriended(CRTPDeclaration, + DerivedTemplateParameter) && + !isDerivedClassBefriended(CRTPDeclaration, DerivedRecord); + + const FixItHint HintFriend = FixItHint::CreateInsertion( + CRTPDeclaration->getBraceRange().getEnd(), + "friend " + DerivedTemplateParameter->getNameAsString() + ';' + '\n'); + + if (hasPrivateConstructor(CRTPDeclaration) && NeedsFriend) { + diag(CRTPDeclaration->getLocation(), + "the CRTP cannot be constructed from the derived class; consider " + "declaring the derived class as friend") + << HintFriend; + } + + auto WithFriendHintIfNeeded = + [&](const DiagnosticBuilder &Diag, + bool NeedsFriend) -> const DiagnosticBuilder & { + if (NeedsFriend) + Diag << HintFriend; + + return Diag; + }; + + if (!CRTPDeclaration->hasUserDeclaredConstructor()) { + const bool IsStruct = CRTPDeclaration->isStruct(); + + WithFriendHintIfNeeded( + diag(CRTPDeclaration->getLocation(), + "the implicit default constructor of the CRTP is publicly " + "accessible; consider making it private%select{| and declaring " + "the derived class as friend}0") + << NeedsFriend + << FixItHint::CreateInsertion( + CRTPDeclaration->getBraceRange().getBegin().getLocWithOffset( + 1), + (IsStruct ? "\nprivate:\n" : "\n") + + CRTPDeclaration->getNameAsString() + "() = default;\n" + + (IsStruct ? "public:\n" : "")), + NeedsFriend); + } + + for (auto &&Ctor : CRTPDeclaration->ctors()) { + if (Ctor->getAccess() == AS_private) + continue; + + const bool IsPublic = Ctor->getAccess() == AS_public; + const std::string Access = IsPublic ? "public" : "protected"; + + WithFriendHintIfNeeded( + diag(Ctor->getLocation(), + "%0 contructor allows the CRTP to be %select{inherited " + "from|constructed}1 as a regular template class; consider making " + "it private%select{| and declaring the derived class as friend}2") + << Access << IsPublic << NeedsFriend + << hintMakeCtorPrivate(Ctor, Access), + NeedsFriend); + } +} + +bool CrtpConstructorAccessibilityCheck::isLanguageVersionSupported( + const LangOptions &LangOpts) const { + return LangOpts.CPlusPlus11; +} +} // namespace clang::tidy::bugprone diff --git a/clang-tools-extra/clang-tidy/bugprone/CrtpConstructorAccessibilityCheck.h b/clang-tools-extra/clang-tidy/bugprone/CrtpConstructorAccessibilityCheck.h new file mode 100644 index 000000000000..785116218f46 --- /dev/null +++ b/clang-tools-extra/clang-tidy/bugprone/CrtpConstructorAccessibilityCheck.h @@ -0,0 +1,32 @@ +//===--- CrtpConstructorAccessibilityCheck.h - clang-tidy -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_CRTPCONSTRUCTORACCESSIBILITYCHECK_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_CRTPCONSTRUCTORACCESSIBILITYCHECK_H + +#include "../ClangTidyCheck.h" + +namespace clang::tidy::bugprone { + +/// Detects error-prone Curiously Recurring Template Pattern usage, when the +/// CRTP can be constructed outside itself and the derived class. +/// +/// For the user-facing documentation see: +/// http://clang.llvm.org/extra/clang-tidy/checks/bugprone/crtp-constructor-accessibility.html +class CrtpConstructorAccessibilityCheck : public ClangTidyCheck { +public: + CrtpConstructorAccessibilityCheck(StringRef Name, ClangTidyContext *Context) + : ClangTidyCheck(Name, Context) {} + void registerMatchers(ast_matchers::MatchFinder *Finder) override; + void check(const ast_matchers::MatchFinder::MatchResult &Result) override; + bool isLanguageVersionSupported(const LangOptions &LangOpts) const override; +}; + +} // namespace clang::tidy::bugprone + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_CRTPCONSTRUCTORACCESSIBILITYCHECK_H diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst index 1e1949756e0e..143ae230fc44 100644 --- a/clang-tools-extra/docs/ReleaseNotes.rst +++ b/clang-tools-extra/docs/ReleaseNotes.rst @@ -104,6 +104,12 @@ Improvements to clang-tidy New checks ^^^^^^^^^^ +- New :doc:`bugprone-crtp-constructor-accessibility + ` check. + + Detects error-prone Curiously Recurring Template Pattern usage, when the CRTP + can be constructed outside itself and the derived class. + - New :doc:`modernize-use-designated-initializers ` check. diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone/crtp-constructor-accessibility.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone/crtp-constructor-accessibility.rst new file mode 100644 index 000000000000..afd88764b596 --- /dev/null +++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone/crtp-constructor-accessibility.rst @@ -0,0 +1,98 @@ +.. title:: clang-tidy - bugprone-crtp-constructor-accessibility + +bugprone-crtp-constructor-accessibility +======================================= + +Detects error-prone Curiously Recurring Template Pattern usage, when the CRTP +can be constructed outside itself and the derived class. + +The CRTP is an idiom, in which a class derives from a template class, where +itself is the template argument. It should be ensured that if a class is +intended to be a base class in this idiom, it can only be instantiated if +the derived class is it's template argument. + +Example: + +.. code-block:: c++ + + template class CRTP { + private: + CRTP() = default; + friend T; + }; + + class Derived : CRTP {}; + +Below can be seen some common mistakes that will allow the breaking of the +idiom. + +If the constructor of a class intended to be used in a CRTP is public, then +it allows users to construct that class on its own. + +Example: + +.. code-block:: c++ + + template class CRTP { + public: + CRTP() = default; + }; + + class Good : CRTP {}; + Good GoodInstance; + + CRTP BadInstance; + +If the constructor is protected, the possibility of an accidental instantiation +is prevented, however it can fade an error, when a different class is used as +the template parameter instead of the derived one. + +Example: + +.. code-block:: c++ + + template class CRTP { + protected: + CRTP() = default; + }; + + class Good : CRTP {}; + Good GoodInstance; + + class Bad : CRTP {}; + Bad BadInstance; + +To ensure that no accidental instantiation happens, the best practice is to +make the constructor private and declare the derived class as friend. Note +that as a tradeoff, this also gives the derived class access to every other +private members of the CRTP. + +Example: + +.. code-block:: c++ + + template class CRTP { + CRTP() = default; + friend T; + }; + + class Good : CRTP {}; + Good GoodInstance; + + class Bad : CRTP {}; + Bad CompileTimeError; + + CRTP AlsoCompileTimeError; + +Limitations: + +* The check is not supported below C++11 + +* The check does not handle when the derived class is passed as a variadic + template argument + +* Accessible functions that can construct the CRTP, like factory functions + are not checked + +The check also suggests a fix-its in some cases. + diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst index 5e57bc0ee483..d03e7af688f0 100644 --- a/clang-tools-extra/docs/clang-tidy/checks/list.rst +++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst @@ -86,6 +86,7 @@ Clang-Tidy Checks :doc:`bugprone-chained-comparison `, :doc:`bugprone-compare-pointer-to-member-virtual-function `, :doc:`bugprone-copy-constructor-init `, "Yes" + :doc:`bugprone-crtp-constructor-accessibility `, "Yes" :doc:`bugprone-dangling-handle `, :doc:`bugprone-dynamic-static-initializers `, :doc:`bugprone-easily-swappable-parameters `, diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/crtp-constructor-accessibility.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/crtp-constructor-accessibility.cpp new file mode 100644 index 000000000000..cb41923df157 --- /dev/null +++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/crtp-constructor-accessibility.cpp @@ -0,0 +1,255 @@ +// RUN: %check_clang_tidy -std=c++11-or-later %s bugprone-crtp-constructor-accessibility %t -- -- -fno-delayed-template-parsing + +namespace class_implicit_ctor { +template +class CRTP {}; +// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: the implicit default constructor of the CRTP is publicly accessible; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility] +// CHECK-FIXES: CRTP() = default; +// CHECK-FIXES: friend T; + +class A : CRTP {}; +} // namespace class_implicit_ctor + +namespace class_unconstructible { +template +class CRTP { +// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: the CRTP cannot be constructed from the derived class; consider declaring the derived class as friend [bugprone-crtp-constructor-accessibility] +// CHECK-FIXES: friend T; + CRTP() = default; +}; + +class A : CRTP {}; +} // namespace class_unconstructible + +namespace class_public_default_ctor { +template +class CRTP { +public: + CRTP() = default; + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: public contructor allows the CRTP to be constructed as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility] + // CHECK-FIXES: private:{{[[:space:]]*}}CRTP() = default;{{[[:space:]]*}}public: + // CHECK-FIXES: friend T; +}; + +class A : CRTP {}; +} // namespace class_public_default_ctor + +namespace class_public_user_provided_ctor { +template +class CRTP { +public: + CRTP(int) {} + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: public contructor allows the CRTP to be constructed as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility] + // CHECK-FIXES: private:{{[[:space:]]*}}CRTP(int) {}{{[[:space:]]*}}public: + // CHECK-FIXES: friend T; +}; + +class A : CRTP {}; +} // namespace class_public_user_provided_ctor + +namespace class_public_multiple_user_provided_ctors { +template +class CRTP { +public: + CRTP(int) {} + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: public contructor allows the CRTP to be constructed as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility] + // CHECK-FIXES: private:{{[[:space:]]*}}CRTP(int) {}{{[[:space:]]*}}public: + CRTP(float) {} + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: public contructor allows the CRTP to be constructed as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility] + // CHECK-FIXES: private:{{[[:space:]]*}}CRTP(float) {}{{[[:space:]]*}}public: + + // CHECK-FIXES: friend T; + // CHECK-FIXES: friend T; +}; + +class A : CRTP {}; +} // namespace class_public_multiple_user_provided_ctors + +namespace class_protected_ctors { +template +class CRTP { +protected: + CRTP(int) {} + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: protected contructor allows the CRTP to be inherited from as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility] + // CHECK-FIXES: private:{{[[:space:]]*}}CRTP(int) {}{{[[:space:]]*}}protected: + CRTP() = default; + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: protected contructor allows the CRTP to be inherited from as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility] + // CHECK-FIXES: private:{{[[:space:]]*}}CRTP() = default;{{[[:space:]]*}}protected: + CRTP(float) {} + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: protected contructor allows the CRTP to be inherited from as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility] + // CHECK-FIXES: private:{{[[:space:]]*}}CRTP(float) {}{{[[:space:]]*}}protected: + + // CHECK-FIXES: friend T; + // CHECK-FIXES: friend T; + // CHECK-FIXES: friend T; +}; + +class A : CRTP {}; +} // namespace class_protected_ctors + +namespace struct_implicit_ctor { +template +struct CRTP {}; +// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: the implicit default constructor of the CRTP is publicly accessible; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility] +// CHECK-FIXES: private:{{[[:space:]]*}}CRTP() = default;{{[[:space:]]*}}public: +// CHECK-FIXES: friend T; + +class A : CRTP {}; +} // namespace struct_implicit_ctor + +namespace struct_default_ctor { +template +struct CRTP { + CRTP() = default; + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: public contructor allows the CRTP to be constructed as a regular template class; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility] + // CHECK-FIXES: private:{{[[:space:]]*}}CRTP() = default;{{[[:space:]]*}}public: + // CHECK-FIXES: friend T; +}; + +class A : CRTP {}; +} // namespace struct_default_ctor + +namespace same_class_multiple_crtps { +template +struct CRTP {}; +// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: the implicit default constructor of the CRTP is publicly accessible; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility] +// CHECK-FIXES: private:{{[[:space:]]*}}CRTP() = default;{{[[:space:]]*}}public: +// CHECK-FIXES: friend T; + +template +struct CRTP2 {}; +// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: the implicit default constructor of the CRTP is publicly accessible; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility] +// CHECK-FIXES: private:{{[[:space:]]*}}CRTP2() = default;{{[[:space:]]*}}public: +// CHECK-FIXES: friend T; + +class A : CRTP, CRTP2 {}; +} // namespace same_class_multiple_crtps + +namespace same_crtp_multiple_classes { +template +class CRTP { +// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: the CRTP cannot be constructed from the derived class; consider declaring the derived class as friend [bugprone-crtp-constructor-accessibility] +// CHECK-FIXES: friend T; + CRTP() = default; +}; + +class A : CRTP {}; +class B : CRTP {}; +} // namespace same_crtp_multiple_classes + +namespace crtp_template { +template +class CRTP { +// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: the CRTP cannot be constructed from the derived class; consider declaring the derived class as friend [bugprone-crtp-constructor-accessibility] +// CHECK-FIXES: friend U; + CRTP() = default; +}; + +class A : CRTP {}; +} // namespace crtp_template + +namespace crtp_template2 { +template +class CRTP { +// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: the CRTP cannot be constructed from the derived class; consider declaring the derived class as friend [bugprone-crtp-constructor-accessibility] +// CHECK-FIXES: friend T; + CRTP() = default; +}; + +class A : CRTP {}; +} // namespace crtp_template2 + +namespace template_derived { +template +class CRTP {}; +// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: the implicit default constructor of the CRTP is publicly accessible; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility] +// CHECK-FIXES: CRTP() = default; +// CHECK-FIXES: friend T; + +template +class A : CRTP> {}; + +// FIXME: Ideally the warning should be triggered without instantiation. +void foo() { + A A; + (void) A; +} +} // namespace template_derived + +namespace template_derived_explicit_specialization { +template +class CRTP {}; +// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: the implicit default constructor of the CRTP is publicly accessible; consider making it private and declaring the derived class as friend [bugprone-crtp-constructor-accessibility] +// CHECK-FIXES: CRTP() = default; +// CHECK-FIXES: friend T; + +template +class A : CRTP> {}; + +template<> +class A : CRTP> {}; +} // namespace template_derived_explicit_specialization + +namespace explicit_derived_friend { +class A; + +template +class CRTP { + CRTP() = default; + friend A; +}; + +class A : CRTP {}; +} // namespace explicit_derived_friend + +namespace explicit_derived_friend_multiple { +class A; + +template +class CRTP { +// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: the CRTP cannot be constructed from the derived class; consider declaring the derived class as friend [bugprone-crtp-constructor-accessibility] +// CHECK-FIXES: friend T; + CRTP() = default; + friend A; +}; + +class A : CRTP {}; +class B : CRTP {}; +} // namespace explicit_derived_friend_multiple + +namespace no_need_for_friend { +class A; + +template +class CRTP { +// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: the implicit default constructor of the CRTP is publicly accessible; consider making it private [bugprone-crtp-constructor-accessibility] +// CHECK-FIXES: CRTP() = default; + friend A; +}; + +class A : CRTP {}; +} // namespace no_need_for_friend + +namespace no_warning { +template +class CRTP +{ + CRTP() = default; + friend T; +}; + +class A : CRTP {}; +} // namespace no_warning + +namespace no_warning_unsupported { +template +class CRTP +{}; + +class A : CRTP {}; + +void foo() { + A A; + (void) A; +} +} // namespace no_warning_unsupported -- GitLab From 1ebbf97316729e38b3189c6c55516266a7909121 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Mon, 4 Mar 2024 18:12:43 -0600 Subject: [PATCH 072/929] [LLVM] Partially revert GPU change to include dirs Summary: Turns out that we need this for the header install directory. I didn't notice because I had old values cached. Revert for now until I think of a better way to suppress the error. --- llvm/cmake/modules/AddLLVM.cmake | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/llvm/cmake/modules/AddLLVM.cmake b/llvm/cmake/modules/AddLLVM.cmake index e18ec23fc76f..828de4bd9940 100644 --- a/llvm/cmake/modules/AddLLVM.cmake +++ b/llvm/cmake/modules/AddLLVM.cmake @@ -1,7 +1,4 @@ -if(NOT LLVM_RUNTIMES_GPU_BUILD) - include(GNUInstallDirs) -endif() - +include(GNUInstallDirs) include(LLVMDistributionSupport) include(LLVMProcessSources) include(LLVM-Config) -- GitLab From bdfebc310e67c8e96909e155669277ab75d784ba Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Mon, 4 Mar 2024 16:17:02 -0800 Subject: [PATCH 073/929] [X86] Use update_mir_test_checks.py to generate CHECK lines in masked_compressstore_isel.ll. NFC --- .../CodeGen/X86/masked_compressstore_isel.ll | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/llvm/test/CodeGen/X86/masked_compressstore_isel.ll b/llvm/test/CodeGen/X86/masked_compressstore_isel.ll index 1851a21c8c06..0587434bff25 100644 --- a/llvm/test/CodeGen/X86/masked_compressstore_isel.ll +++ b/llvm/test/CodeGen/X86/masked_compressstore_isel.ll @@ -1,21 +1,24 @@ +; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -start-after=codegenprepare -stop-before finalize-isel | FileCheck %s define void @_Z3fooiPiPs(<8 x i32> %gepload, <8 x i1> %0) #0 { + ; CHECK-LABEL: name: _Z3fooiPiPs + ; CHECK: bb.0.entry: + ; CHECK-NEXT: liveins: $ymm0, $xmm1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:vr128x = COPY $xmm1 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vr256x = COPY $ymm0 + ; CHECK-NEXT: [[VPSLLWZ128ri:%[0-9]+]]:vr128x = VPSLLWZ128ri [[COPY]], 15 + ; CHECK-NEXT: [[VPMOVW2MZ128rr:%[0-9]+]]:vk16wm = VPMOVW2MZ128rr killed [[VPSLLWZ128ri]] + ; CHECK-NEXT: [[VPMOVDWZ256rr:%[0-9]+]]:vr128x = VPMOVDWZ256rr [[COPY1]] + ; CHECK-NEXT: VPCOMPRESSWZ128mrk $noreg, 1, $noreg, 0, $noreg, killed [[VPMOVW2MZ128rr]], killed [[VPMOVDWZ256rr]] :: (store unknown-size into `ptr null`, align 16) + ; CHECK-NEXT: RET 0 entry: %1 = trunc <8 x i32> %gepload to <8 x i16> tail call void @llvm.masked.compressstore.v8i16(<8 x i16> %1, ptr null, <8 x i1> %0) ret void } -; CHECK-LABEL: bb.0.entry: -; CHECK: %1:vr128x = COPY $xmm1 -; CHECK-NEXT: %0:vr256x = COPY $ymm0 -; CHECK-NEXT: %2:vr128x = VPSLLWZ128ri %1, 15 -; CHECK-NEXT: %3:vk16wm = VPMOVW2MZ128rr killed %2 -; CHECK-NEXT: %4:vr128x = VPMOVDWZ256rr %0 -; CHECK-NEXT: VPCOMPRESSWZ128mrk $noreg, 1, $noreg, 0, $noreg, killed %3, killed %4 :: (store unknown-size into `ptr null`, align 16) -; CHECK-NEXT: RET 0 - ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: write) declare void @llvm.masked.compressstore.v8i16(<8 x i16>, ptr nocapture, <8 x i1>) #1 -- GitLab From 8d6e867eb2de32ce28ece972c91405db976192c3 Mon Sep 17 00:00:00 2001 From: Patrick O'Neill <102189596+patrick-rivos@users.noreply.github.com> Date: Mon, 4 Mar 2024 16:40:40 -0800 Subject: [PATCH 074/929] [LSR][term-fold] Ensure the simple recurrence is from the current loop (#83085) If the phi node found by matchSimpleRecurrence is not from the current loop, then isAlmostDeadIV panics. With this patch we bail out early. Signed-off-by: Patrick O'Neill --------- Signed-off-by: Patrick O'Neill --- .../Transforms/Scalar/LoopStrengthReduce.cpp | 4 ++ llvm/lib/Transforms/Utils/LoopUtils.cpp | 1 + .../lsr-unreachable-bb-phi-node.ll | 40 +++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 llvm/test/Transforms/LoopStrengthReduce/lsr-unreachable-bb-phi-node.ll diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp index 08021f3ba853..4f5501611484 100644 --- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp +++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp @@ -6808,6 +6808,10 @@ canFoldTermCondOfLoop(Loop *L, ScalarEvolution &SE, DominatorTree &DT, if (!matchSimpleRecurrence(LHS, ToFold, ToFoldStart, ToFoldStep)) return std::nullopt; + // Ensure the simple recurrence is a part of the current loop. + if (ToFold->getParent() != L->getHeader()) + return std::nullopt; + // If that IV isn't dead after we rewrite the exit condition in terms of // another IV, there's no point in doing the transform. if (!isAlmostDeadIV(ToFold, LoopLatch, TermCond)) diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp index a4fdc1f8c12e..7491a99b03f6 100644 --- a/llvm/lib/Transforms/Utils/LoopUtils.cpp +++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp @@ -468,6 +468,7 @@ llvm::collectChildrenInLoop(DomTreeNode *N, const Loop *CurLoop) { bool llvm::isAlmostDeadIV(PHINode *PN, BasicBlock *LatchBlock, Value *Cond) { int LatchIdx = PN->getBasicBlockIndex(LatchBlock); + assert(LatchIdx != -1 && "LatchBlock is not a case in this PHINode"); Value *IncV = PN->getIncomingValue(LatchIdx); for (User *U : PN->users()) diff --git a/llvm/test/Transforms/LoopStrengthReduce/lsr-unreachable-bb-phi-node.ll b/llvm/test/Transforms/LoopStrengthReduce/lsr-unreachable-bb-phi-node.ll new file mode 100644 index 000000000000..1454535b52bc --- /dev/null +++ b/llvm/test/Transforms/LoopStrengthReduce/lsr-unreachable-bb-phi-node.ll @@ -0,0 +1,40 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -loop-reduce -S -lsr-term-fold | FileCheck %s + +; This test used to crash due to matchSimpleRecurrence matching the simple +; recurrence in pn-loop when evaluating unrelated-loop. Since unrelated-loop +; cannot jump to pn-node isAlmostDeadIV panics. +define void @phi_node_different_bb() { +; CHECK-LABEL: @phi_node_different_bb( +; CHECK-NEXT: br label [[PN_LOOP:%.*]] +; CHECK: pn-loop: +; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ 1, [[TMP0:%.*]] ], [ [[TMP2:%.*]], [[PN_LOOP]] ] +; CHECK-NEXT: [[TMP2]] = add i32 [[TMP1]], 1 +; CHECK-NEXT: [[TMP3:%.*]] = icmp ugt i32 [[TMP2]], 1 +; CHECK-NEXT: br i1 [[TMP3]], label [[PN_LOOP]], label [[UNRELATED_LOOP_PREHEADER:%.*]] +; CHECK: unrelated-loop.preheader: +; CHECK-NEXT: br label [[UNRELATED_LOOP:%.*]] +; CHECK: unrelated-loop: +; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[TMP2]], 0 +; CHECK-NEXT: br i1 [[TMP4]], label [[END:%.*]], label [[UNRELATED_LOOP]] +; CHECK: end: +; CHECK-NEXT: ret void +; + br label %pn-loop + +pn-loop: ; preds = %pn-loop, %0 + %1 = phi i32 [ 1, %0 ], [ %2, %pn-loop ] + %2 = add i32 %1, 1 + %3 = icmp ugt i32 %2, 1 + br i1 %3, label %pn-loop, label %unrelated-loop.preheader + +unrelated-loop.preheader: ; preds = %pn-loop + br label %unrelated-loop + +unrelated-loop: ; preds = %unrelated-loop, %unrelated-loop.preheader + %4 = icmp eq i32 %2, 0 + br i1 %4, label %end, label %unrelated-loop + +end: ; preds = %unrelated-loop + ret void +} -- GitLab From 57592e9ae78e16ffe4f2a2cd3190c422bcdba5a0 Mon Sep 17 00:00:00 2001 From: michele-scandale Date: Mon, 4 Mar 2024 16:43:33 -0800 Subject: [PATCH 075/929] [Clang] Fixes of builtin definitions after PR #68324. (#81022) This commit addresses few differences between the `Builtins.def` file before the refactoring done in PR #68324 and the currently generated `Builtins.inc` file. --- clang/include/clang/Basic/Builtins.td | 80 +++++++++++++++-------- clang/include/clang/Basic/BuiltinsBase.td | 2 + 2 files changed, 54 insertions(+), 28 deletions(-) diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td index 2c83dca248fb..384442e6a935 100644 --- a/clang/include/clang/Basic/Builtins.td +++ b/clang/include/clang/Basic/Builtins.td @@ -12,13 +12,17 @@ class FPMathTemplate : Template<["float", "double", "long double"], ["f", "", "l"]>; class FPMathWithF16Template : - Template<["float", "double", "long double", "__fp16", "__float128"], - ["f", "", "l", "f16", "f128"]>; + Template<["float", "double", "long double", "__fp16"], + ["f", "", "l", "f16"]>; class FPMathWithF16F128Template : Template<["float", "double", "long double", "__fp16", "__float128"], ["f", "", "l", "f16", "f128"]>; +class FPMathWithF128Template : + Template<["float", "double", "long double", "__float128"], + ["f", "", "l", "f128"]>; + class F16F128MathTemplate : Template<["__fp16", "__float128"], ["f16", "f128"]>; @@ -253,18 +257,30 @@ def FrexpF16F128 : F16F128MathTemplate, Builtin { let Prototype = "T(T, int*)"; } -def HugeVal : Builtin, FPMathWithF16F128Template { +def HugeVal : Builtin, FPMathWithF128Template { let Spellings = ["__builtin_huge_val"]; let Attributes = [NoThrow, Const, Constexpr]; let Prototype = "T()"; } -def Inf : Builtin, FPMathWithF16F128Template { +def HugeValF16 : Builtin { + let Spellings = ["__builtin_huge_valf16"]; + let Attributes = [NoThrow, Const, Constexpr]; + let Prototype = "_Float16()"; +} + +def Inf : Builtin, FPMathWithF128Template { let Spellings = ["__builtin_inf"]; let Attributes = [NoThrow, Const, Constexpr]; let Prototype = "T()"; } +def InfF16 : Builtin { + let Spellings = ["__builtin_inff16"]; + let Attributes = [NoThrow, Const, Constexpr]; + let Prototype = "_Float16()"; +} + def LdexpF16F128 : F16F128MathTemplate, Builtin { let Spellings = ["__builtin_ldexp"]; let Attributes = [FunctionWithBuiltinPrefix, NoThrow, ConstIgnoringErrnoAndExceptions]; @@ -1550,7 +1566,7 @@ def SyncBoolCompareAndSwap : Builtin { def SyncBoolCompareAndSwapN : Builtin, SyncBuiltinsTemplate { let Spellings = ["__sync_bool_compare_and_swap_"]; let Attributes = [CustomTypeChecking, NoThrow]; - let Prototype = "T(T volatile*, T, ...)"; + let Prototype = "bool(T volatile*, T, T, ...)"; } def SyncValCompareAndSwap : Builtin { @@ -1562,7 +1578,7 @@ def SyncValCompareAndSwap : Builtin { def SynLockValCompareAndSwapN : Builtin, SyncBuiltinsTemplate { let Spellings = ["__sync_val_compare_and_swap_"]; let Attributes = [CustomTypeChecking, NoThrow]; - let Prototype = "T(T volatile*, T, ...)"; + let Prototype = "T(T volatile*, T, T, ...)"; } def SyncLockTestAndSet : Builtin { @@ -1577,16 +1593,16 @@ def SynLockLockTestAndSetN : Builtin, SyncBuiltinsTemplate { let Prototype = "T(T volatile*, T, ...)"; } -def SyncLockReleaseN : Builtin { +def SyncLockRelease : Builtin { let Spellings = ["__sync_lock_release"]; let Attributes = [CustomTypeChecking]; let Prototype = "void(...)"; } -def SynLockReleaseN : Builtin, SyncBuiltinsTemplate { +def SyncLockReleaseN : Builtin, SyncBuiltinsTemplate { let Spellings = ["__sync_lock_release_"]; let Attributes = [CustomTypeChecking, NoThrow]; - let Prototype = "T(T volatile*, T, ...)"; + let Prototype = "void(T volatile*, ...)"; } def SyncSwap : Builtin { @@ -2569,6 +2585,13 @@ def Abort : LibBuiltin<"stdlib.h"> { let AddBuiltinPrefixedAlias = 1; } +def Abs : IntMathTemplate, LibBuiltin<"stdlib.h"> { + let Spellings = ["abs"]; + let Attributes = [NoThrow, Const]; + let Prototype = "T(T)"; + let AddBuiltinPrefixedAlias = 1; +} + def Calloc : LibBuiltin<"stdlib.h"> { let Spellings = ["calloc"]; let Prototype = "void*(size_t, size_t)"; @@ -3085,38 +3108,38 @@ def MemAlign : GNULibBuiltin<"malloc.h"> { // POSIX string.h -def MemcCpy : GNULibBuiltin<"stdlib.h"> { +def MemcCpy : GNULibBuiltin<"string.h"> { let Spellings = ["memccpy"]; let Prototype = "void*(void*, void const*, int, size_t)"; } -def MempCpy : GNULibBuiltin<"stdlib.h"> { +def MempCpy : GNULibBuiltin<"string.h"> { let Spellings = ["mempcpy"]; let Prototype = "void*(void*, void const*, size_t)"; } -def StpCpy : GNULibBuiltin<"stdlib.h"> { +def StpCpy : GNULibBuiltin<"string.h"> { let Spellings = ["stpcpy"]; let Attributes = [NoThrow]; let Prototype = "char*(char*, char const*)"; let AddBuiltinPrefixedAlias = 1; } -def StpnCpy : GNULibBuiltin<"stdlib.h"> { +def StpnCpy : GNULibBuiltin<"string.h"> { let Spellings = ["stpncpy"]; let Attributes = [NoThrow]; let Prototype = "char*(char*, char const*, size_t)"; let AddBuiltinPrefixedAlias = 1; } -def StrDup : GNULibBuiltin<"stdlib.h"> { +def StrDup : GNULibBuiltin<"string.h"> { let Spellings = ["strdup"]; let Attributes = [NoThrow]; let Prototype = "char*(char const*)"; let AddBuiltinPrefixedAlias = 1; } -def StrnDup : GNULibBuiltin<"stdlib.h"> { +def StrnDup : GNULibBuiltin<"string.h"> { let Spellings = ["strndup"]; let Attributes = [NoThrow]; let Prototype = "char*(char const*, size_t)"; @@ -3286,22 +3309,22 @@ def ObjcEnumerationMutation : ObjCLibBuiltin<"objc/runtime.h"> { let Prototype = "void(id)"; } -def ObjcReadWeak : ObjCLibBuiltin<"objc/message.h"> { +def ObjcReadWeak : ObjCLibBuiltin<"objc/objc-auto.h"> { let Spellings = ["objc_read_weak"]; let Prototype = "id(id*)"; } -def ObjcAssignWeak : ObjCLibBuiltin<"objc/message.h"> { +def ObjcAssignWeak : ObjCLibBuiltin<"objc/objc-auto.h"> { let Spellings = ["objc_assign_weak"]; let Prototype = "id(id, id*)"; } -def ObjcAssignIvar : ObjCLibBuiltin<"objc/message.h"> { +def ObjcAssignIvar : ObjCLibBuiltin<"objc/objc-auto.h"> { let Spellings = ["objc_assign_ivar"]; let Prototype = "id(id, id, ptrdiff_t)"; } -def ObjcAssignGlobal : ObjCLibBuiltin<"objc/message.h"> { +def ObjcAssignGlobal : ObjCLibBuiltin<"objc/objc-auto.h"> { let Spellings = ["objc_assign_global"]; let Prototype = "id(id, id*)"; } @@ -3371,13 +3394,6 @@ def Atan2 : FPMathTemplate, LibBuiltin<"math.h"> { let AddBuiltinPrefixedAlias = 1; } -def Abs : IntMathTemplate, LibBuiltin<"math.h"> { - let Spellings = ["abs"]; - let Attributes = [NoThrow, Const]; - let Prototype = "T(T)"; - let AddBuiltinPrefixedAlias = 1; -} - def Copysign : FPMathTemplate, LibBuiltin<"math.h"> { let Spellings = ["copysign"]; let Attributes = [NoThrow, Const]; @@ -3996,8 +4012,16 @@ def BlockObjectDispose : LibBuiltin<"blocks.h"> { } // FIXME: Also declare NSConcreteGlobalBlock and NSConcreteStackBlock. +def __Addressof : LangBuiltin<"CXX_LANG"> { + let Spellings = ["__addressof"]; + let Attributes = [FunctionWithoutBuiltinPrefix, NoThrow, Const, + IgnoreSignature, Constexpr]; + let Prototype = "void*(void&)"; + let Namespace = "std"; +} + def Addressof : CxxLibBuiltin<"memory"> { - let Spellings = ["addressof", "__addressof"]; + let Spellings = ["addressof"]; let Attributes = [NoThrow, Const, IgnoreSignature, RequireDeclaration, Constexpr]; let Prototype = "void*(void&)"; @@ -4036,7 +4060,7 @@ def Move : CxxLibBuiltin<"utility"> { let Namespace = "std"; } -def MoveIfNsoexcept : CxxLibBuiltin<"memory"> { +def MoveIfNsoexcept : CxxLibBuiltin<"utility"> { let Spellings = ["move_if_noexcept"]; let Attributes = [NoThrow, Const, IgnoreSignature, RequireDeclaration, Constexpr]; diff --git a/clang/include/clang/Basic/BuiltinsBase.td b/clang/include/clang/Basic/BuiltinsBase.td index bfccff5600dd..724747ec76d7 100644 --- a/clang/include/clang/Basic/BuiltinsBase.td +++ b/clang/include/clang/Basic/BuiltinsBase.td @@ -49,6 +49,8 @@ def UnevaluatedArguments : Attribute<"u">; // is required for a builtin. def FunctionWithBuiltinPrefix : Attribute<"F">; +def FunctionWithoutBuiltinPrefix : Attribute<"f">; + // const, but only when -fno-math-errno and FP exceptions are ignored. def ConstIgnoringErrnoAndExceptions : Attribute<"e">; -- GitLab From 81617f85009b4bf5f7eb45f9ff0db53dc1bdf310 Mon Sep 17 00:00:00 2001 From: Brad Smith Date: Mon, 4 Mar 2024 19:44:15 -0500 Subject: [PATCH 076/929] =?UTF-8?q?[Driver][RISCV]=20Forward=20--no-relax?= =?UTF-8?q?=20option=20to=20linker=20for=20RISC-V=20on=20*BS=E2=80=A6=20(#?= =?UTF-8?q?83216)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …D, Fuchsia and Haiku Based on https://github.com/llvm/llvm-project/pull/76432 --- clang/lib/Driver/ToolChains/FreeBSD.cpp | 11 ++++++++--- clang/lib/Driver/ToolChains/Fuchsia.cpp | 5 ++++- clang/lib/Driver/ToolChains/Haiku.cpp | 7 +++++-- clang/lib/Driver/ToolChains/NetBSD.cpp | 5 ++++- clang/lib/Driver/ToolChains/OpenBSD.cpp | 6 +++++- clang/test/Driver/freebsd.c | 4 ++++ clang/test/Driver/fuchsia.c | 5 +++++ clang/test/Driver/haiku.c | 5 +++++ clang/test/Driver/netbsd.c | 7 +++++++ clang/test/Driver/openbsd.c | 8 ++++---- 10 files changed, 51 insertions(+), 12 deletions(-) diff --git a/clang/lib/Driver/ToolChains/FreeBSD.cpp b/clang/lib/Driver/ToolChains/FreeBSD.cpp index 9d698f775839..c5757ddebb0f 100644 --- a/clang/lib/Driver/ToolChains/FreeBSD.cpp +++ b/clang/lib/Driver/ToolChains/FreeBSD.cpp @@ -133,6 +133,7 @@ void freebsd::Linker::ConstructJob(Compilation &C, const JobAction &JA, const char *LinkingOutput) const { const auto &ToolChain = static_cast(getToolChain()); const Driver &D = ToolChain.getDriver(); + const llvm::Triple &Triple = ToolChain.getTriple(); const llvm::Triple::ArchType Arch = ToolChain.getArch(); const bool IsPIE = !Args.hasArg(options::OPT_shared) && @@ -165,8 +166,7 @@ void freebsd::Linker::ConstructJob(Compilation &C, const JobAction &JA, CmdArgs.push_back("-dynamic-linker"); CmdArgs.push_back("/libexec/ld-elf.so.1"); } - const llvm::Triple &T = ToolChain.getTriple(); - if (Arch == llvm::Triple::arm || T.isX86()) + if (Arch == llvm::Triple::arm || Triple.isX86()) CmdArgs.push_back("--hash-style=both"); CmdArgs.push_back("--enable-new-dtags"); } @@ -212,12 +212,17 @@ void freebsd::Linker::ConstructJob(Compilation &C, const JobAction &JA, case llvm::Triple::riscv64: CmdArgs.push_back("-m"); CmdArgs.push_back("elf64lriscv"); - CmdArgs.push_back("-X"); break; default: break; } + if (Triple.isRISCV64()) { + CmdArgs.push_back("-X"); + if (Args.hasArg(options::OPT_mno_relax)) + CmdArgs.push_back("--no-relax"); + } + if (Arg *A = Args.getLastArg(options::OPT_G)) { if (ToolChain.getTriple().isMIPS()) { StringRef v = A->getValue(); diff --git a/clang/lib/Driver/ToolChains/Fuchsia.cpp b/clang/lib/Driver/ToolChains/Fuchsia.cpp index 9123ddf2fe99..598289f48ff4 100644 --- a/clang/lib/Driver/ToolChains/Fuchsia.cpp +++ b/clang/lib/Driver/ToolChains/Fuchsia.cpp @@ -119,8 +119,11 @@ void fuchsia::Linker::ConstructJob(Compilation &C, const JobAction &JA, CmdArgs.push_back(Args.MakeArgString(Dyld)); } - if (ToolChain.getArch() == llvm::Triple::riscv64) + if (Triple.isRISCV64()) { CmdArgs.push_back("-X"); + if (Args.hasArg(options::OPT_mno_relax)) + CmdArgs.push_back("--no-relax"); + } CmdArgs.push_back("-o"); CmdArgs.push_back(Output.getFilename()); diff --git a/clang/lib/Driver/ToolChains/Haiku.cpp b/clang/lib/Driver/ToolChains/Haiku.cpp index ca7faa68765a..30464e2229e6 100644 --- a/clang/lib/Driver/ToolChains/Haiku.cpp +++ b/clang/lib/Driver/ToolChains/Haiku.cpp @@ -25,7 +25,7 @@ void haiku::Linker::ConstructJob(Compilation &C, const JobAction &JA, const char *LinkingOutput) const { const auto &ToolChain = static_cast(getToolChain()); const Driver &D = ToolChain.getDriver(); - const llvm::Triple::ArchType Arch = ToolChain.getArch(); + const llvm::Triple &Triple = ToolChain.getTriple(); const bool Static = Args.hasArg(options::OPT_static); const bool Shared = Args.hasArg(options::OPT_shared); ArgStringList CmdArgs; @@ -61,8 +61,11 @@ void haiku::Linker::ConstructJob(Compilation &C, const JobAction &JA, if (!Shared) CmdArgs.push_back("--no-undefined"); - if (Arch == llvm::Triple::riscv64) + if (Triple.isRISCV64()) { CmdArgs.push_back("-X"); + if (Args.hasArg(options::OPT_mno_relax)) + CmdArgs.push_back("--no-relax"); + } assert((Output.isFilename() || Output.isNothing()) && "Invalid output."); if (Output.isFilename()) { diff --git a/clang/lib/Driver/ToolChains/NetBSD.cpp b/clang/lib/Driver/ToolChains/NetBSD.cpp index 645d0311641f..0eec8fddabd5 100644 --- a/clang/lib/Driver/ToolChains/NetBSD.cpp +++ b/clang/lib/Driver/ToolChains/NetBSD.cpp @@ -240,8 +240,11 @@ void netbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA, break; } - if (Triple.isRISCV()) + if (Triple.isRISCV()) { CmdArgs.push_back("-X"); + if (Args.hasArg(options::OPT_mno_relax)) + CmdArgs.push_back("--no-relax"); + } assert((Output.isFilename() || Output.isNothing()) && "Invalid output."); if (Output.isFilename()) { diff --git a/clang/lib/Driver/ToolChains/OpenBSD.cpp b/clang/lib/Driver/ToolChains/OpenBSD.cpp index 97f88b7b79df..6da6728585df 100644 --- a/clang/lib/Driver/ToolChains/OpenBSD.cpp +++ b/clang/lib/Driver/ToolChains/OpenBSD.cpp @@ -111,6 +111,7 @@ void openbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA, const char *LinkingOutput) const { const auto &ToolChain = static_cast(getToolChain()); const Driver &D = ToolChain.getDriver(); + const llvm::Triple &Triple = ToolChain.getTriple(); const llvm::Triple::ArchType Arch = ToolChain.getArch(); const bool Static = Args.hasArg(options::OPT_static); const bool Shared = Args.hasArg(options::OPT_shared); @@ -160,8 +161,11 @@ void openbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA, if (Nopie || Profiling) CmdArgs.push_back("-nopie"); - if (Arch == llvm::Triple::riscv64) + if (Triple.isRISCV64()) { CmdArgs.push_back("-X"); + if (Args.hasArg(options::OPT_mno_relax)) + CmdArgs.push_back("--no-relax"); + } assert((Output.isFilename() || Output.isNothing()) && "Invalid output."); if (Output.isFilename()) { diff --git a/clang/test/Driver/freebsd.c b/clang/test/Driver/freebsd.c index e1ce8889459f..10fe155fee87 100644 --- a/clang/test/Driver/freebsd.c +++ b/clang/test/Driver/freebsd.c @@ -203,3 +203,7 @@ // RELOCATABLE-NOT: "-l // RELOCATABLE-NOT: crt{{[^./\\]+}}.o +// Check that the -X and --no-relax flags are passed to the linker on riscv64 +// RUN: %clang --target=riscv64-unknown-freebsd -mno-relax -### %s 2>&1 \ +// RUN: | FileCheck -check-prefix=RISCV64-FLAGS %s +// RISCV64-FLAGS: "-X" "--no-relax" diff --git a/clang/test/Driver/fuchsia.c b/clang/test/Driver/fuchsia.c index ca53f0d107a0..c67f7f8c005b 100644 --- a/clang/test/Driver/fuchsia.c +++ b/clang/test/Driver/fuchsia.c @@ -292,3 +292,8 @@ // RUN: | FileCheck %s -check-prefix=CHECK-PROFRT-X86_64 // CHECK-PROFRT-X86_64: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]" // CHECK-PROFRT-X86_64: "[[RESOURCE_DIR]]{{/|\\\\}}lib{{/|\\\\}}x86_64-unknown-fuchsia{{/|\\\\}}libclang_rt.profile.a" + +// Check that the -X and --no-relax flags are passed to the linker on riscv64 +// RUN: %clang --target=riscv64-unknown-fuchsia -mno-relax -### %s 2>&1 \ +// RUN: | FileCheck -check-prefix=RISCV64-FLAGS %s +// RISCV64-FLAGS: "-X" "--no-relax" diff --git a/clang/test/Driver/haiku.c b/clang/test/Driver/haiku.c index e907c34b29b9..060a56b3c70e 100644 --- a/clang/test/Driver/haiku.c +++ b/clang/test/Driver/haiku.c @@ -76,6 +76,11 @@ // RUN: | FileCheck --check-prefix=CHECK-ARM-CPU %s // CHECK-ARM-CPU: "-target-cpu" "arm1176jzf-s" +// Check that the -X and --no-relax flags are passed to the linker on riscv64 +// RUN: %clang --target=riscv64-unknown-haiku -mno-relax -### %s 2>&1 \ +// RUN: | FileCheck -check-prefix=RISCV64-FLAGS %s +// RISCV64-FLAGS: "-X" "--no-relax" + // Check passing LTO flags to the linker // RUN: %clang --target=x86_64-unknown-haiku -flto -### %s 2>&1 \ // RUN: | FileCheck -check-prefix=CHECK-LTO-FLAGS %s diff --git a/clang/test/Driver/netbsd.c b/clang/test/Driver/netbsd.c index 73777e7a3e38..1b7c674e18af 100644 --- a/clang/test/Driver/netbsd.c +++ b/clang/test/Driver/netbsd.c @@ -342,3 +342,10 @@ // DRIVER-PASS-INCLUDES: "-cc1" {{.*}}"-resource-dir" "[[RESOURCE:[^"]+]]" // DRIVER-PASS-INCLUDES-SAME: "-internal-isystem" "[[RESOURCE]]{{/|\\\\}}include" // DRIVER-PASS-INCLUDES-SAME: {{^}} "-internal-externc-isystem" "{{.*}}/usr/include" + +// Check that the -X and --no-relax flags are passed to the linker on riscv +// RUN: %clang --target=riscv32-unknown-netbsd -mno-relax -### %s 2>&1 \ +// RUN: | FileCheck -check-prefix=RISCV-FLAGS %s +// RUN: %clang --target=riscv64-unknown-netbsd -mno-relax -### %s 2>&1 \ +// RUN: | FileCheck -check-prefix=RISCV-FLAGS %s +// RISCV-FLAGS: "-X" "--no-relax" diff --git a/clang/test/Driver/openbsd.c b/clang/test/Driver/openbsd.c index 68d33c5dd01e..672cd3adf44a 100644 --- a/clang/test/Driver/openbsd.c +++ b/clang/test/Driver/openbsd.c @@ -127,10 +127,10 @@ // UNWIND-TABLES: "-funwind-tables=2" // NO-UNWIND-TABLES-NOT: "-funwind-tables=2" -// Check that the -X flag is passed to the linker on riscv64 -// RUN: %clang --target=riscv64-unknown-openbsd -### %s 2>&1 \ -// RUN: | FileCheck -check-prefix=CHECK-RISCV64-FLAGS %s -// CHECK-RISCV64-FLAGS: "-X" +// Check that the -X and --no-relax flags are passed to the linker on riscv64 +// RUN: %clang --target=riscv64-unknown-openbsd -mno-relax -### %s 2>&1 \ +// RUN: | FileCheck -check-prefix=RISCV64-FLAGS %s +// RISCV64-FLAGS: "-X" "--no-relax" // Check passing LTO flags to the linker // RUN: %clang --target=amd64-unknown-openbsd -flto -### %s 2>&1 \ -- GitLab From 3e40c96d8970f8a52a1f711b4f28aec5cb13e89e Mon Sep 17 00:00:00 2001 From: AtariDreams <83477269+AtariDreams@users.noreply.github.com> Date: Mon, 4 Mar 2024 19:47:05 -0500 Subject: [PATCH 077/929] [X86] Resolve FIXME: Add FPCW as a rounding control register (#82452) To prevent tests from breaking, another fix had to be made: Now, we check if the instruction after a waiting instruction is a call, and if so, we insert the wait. --- llvm/lib/Target/X86/X86ISelLoweringCall.cpp | 4 +- llvm/lib/Target/X86/X86InstrInfo.cpp | 5 ++ llvm/test/CodeGen/X86/pr59305.ll | 89 +++++++++++++++------ 3 files changed, 69 insertions(+), 29 deletions(-) diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp index be8275c92e11..c7ef11aede88 100644 --- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp +++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp @@ -670,9 +670,7 @@ const MCPhysReg *X86TargetLowering::getScratchRegisters(CallingConv::ID) const { } ArrayRef X86TargetLowering::getRoundingControlRegisters() const { - // FIXME: We should def X86::FPCW for x87 as well. But it affects a lot of lit - // tests at the moment, which is not what we expected. - static const MCPhysReg RCRegs[] = {X86::MXCSR}; + static const MCPhysReg RCRegs[] = {X86::FPCW, X86::MXCSR}; return RCRegs; } diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp index 0f21880f6df9..25be1e2f6833 100644 --- a/llvm/lib/Target/X86/X86InstrInfo.cpp +++ b/llvm/lib/Target/X86/X86InstrInfo.cpp @@ -3443,6 +3443,11 @@ static bool isX87Reg(unsigned Reg) { /// check if the instruction is X87 instruction bool X86::isX87Instruction(MachineInstr &MI) { + // Call defs X87 register, so we special case it here because + // otherwise calls are incorrectly flagged as x87 instructions + // as a result. + if (MI.isCall()) + return false; for (const MachineOperand &MO : MI.operands()) { if (!MO.isReg()) continue; diff --git a/llvm/test/CodeGen/X86/pr59305.ll b/llvm/test/CodeGen/X86/pr59305.ll index c2f6d21a41d4..4172aa6204de 100644 --- a/llvm/test/CodeGen/X86/pr59305.ll +++ b/llvm/test/CodeGen/X86/pr59305.ll @@ -1,32 +1,69 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=x86_64-pc-linux < %s | FileCheck %s +; RUN: llc -mtriple=x86_64-pc-linux < %s | FileCheck %s --check-prefix=X86-64 +; RUN: llc -mtriple=i686-pc-linux < %s | FileCheck %s --check-prefix=X86 define double @foo(double %0) #0 { -; CHECK-LABEL: foo: -; CHECK: # %bb.0: -; CHECK-NEXT: subq $24, %rsp -; CHECK-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill -; CHECK-NEXT: movl $1024, %edi # imm = 0x400 -; CHECK-NEXT: callq fesetround@PLT -; CHECK-NEXT: movsd {{.*#+}} xmm1 = [1.0E+0,0.0E+0] -; CHECK-NEXT: divsd (%rsp), %xmm1 # 8-byte Folded Reload -; CHECK-NEXT: movsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; CHECK-NEXT: movl $1024, %edi # imm = 0x400 -; CHECK-NEXT: callq fesetround@PLT -; CHECK-NEXT: movsd {{.*#+}} xmm0 = [1.0E+0,0.0E+0] -; CHECK-NEXT: divsd (%rsp), %xmm0 # 8-byte Folded Reload -; CHECK-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; CHECK-NEXT: movl $1024, %edi # imm = 0x400 -; CHECK-NEXT: callq fesetround@PLT -; CHECK-NEXT: movsd {{.*#+}} xmm2 = [1.0E+0,0.0E+0] -; CHECK-NEXT: divsd (%rsp), %xmm2 # 8-byte Folded Reload -; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload -; CHECK-NEXT: # xmm0 = mem[0],zero -; CHECK-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload -; CHECK-NEXT: # xmm1 = mem[0],zero -; CHECK-NEXT: callq fma@PLT -; CHECK-NEXT: addq $24, %rsp -; CHECK-NEXT: retq +; X86-64-LABEL: foo: +; X86-64: # %bb.0: +; X86-64-NEXT: subq $24, %rsp +; X86-64-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill +; X86-64-NEXT: movl $1024, %edi # imm = 0x400 +; X86-64-NEXT: callq fesetround@PLT +; X86-64-NEXT: movsd {{.*#+}} xmm1 = [1.0E+0,0.0E+0] +; X86-64-NEXT: divsd (%rsp), %xmm1 # 8-byte Folded Reload +; X86-64-NEXT: movsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; X86-64-NEXT: movl $1024, %edi # imm = 0x400 +; X86-64-NEXT: callq fesetround@PLT +; X86-64-NEXT: movsd {{.*#+}} xmm0 = [1.0E+0,0.0E+0] +; X86-64-NEXT: divsd (%rsp), %xmm0 # 8-byte Folded Reload +; X86-64-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; X86-64-NEXT: movl $1024, %edi # imm = 0x400 +; X86-64-NEXT: callq fesetround@PLT +; X86-64-NEXT: movsd {{.*#+}} xmm2 = [1.0E+0,0.0E+0] +; X86-64-NEXT: divsd (%rsp), %xmm2 # 8-byte Folded Reload +; X86-64-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload +; X86-64-NEXT: # xmm0 = mem[0],zero +; X86-64-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload +; X86-64-NEXT: # xmm1 = mem[0],zero +; X86-64-NEXT: callq fma@PLT +; X86-64-NEXT: addq $24, %rsp +; X86-64-NEXT: retq +; +; X86-LABEL: foo: +; X86: # %bb.0: +; X86-NEXT: subl $60, %esp +; X86-NEXT: fldl {{[0-9]+}}(%esp) +; X86-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill +; X86-NEXT: wait +; X86-NEXT: movl $1024, (%esp) # imm = 0x400 +; X86-NEXT: calll fesetround@PLT +; X86-NEXT: fld1 +; X86-NEXT: fstl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill +; X86-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload +; X86-NEXT: fdivrp %st, %st(1) +; X86-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill +; X86-NEXT: wait +; X86-NEXT: movl $1024, (%esp) # imm = 0x400 +; X86-NEXT: calll fesetround@PLT +; X86-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload +; X86-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload +; X86-NEXT: fdivp %st, %st(1) +; X86-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill +; X86-NEXT: wait +; X86-NEXT: movl $1024, (%esp) # imm = 0x400 +; X86-NEXT: calll fesetround@PLT +; X86-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload +; X86-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload +; X86-NEXT: fdivp %st, %st(1) +; X86-NEXT: fstpl {{[0-9]+}}(%esp) +; X86-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload +; X86-NEXT: fstpl {{[0-9]+}}(%esp) +; X86-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload +; X86-NEXT: fstpl (%esp) +; X86-NEXT: wait +; X86-NEXT: calll fma +; X86-NEXT: addl $60, %esp +; X86-NEXT: retl %2 = call i32 @fesetround(i32 noundef 1024) %3 = call double @llvm.experimental.constrained.fdiv.f64(double 1.000000e+00, double %0, metadata !"round.dynamic", metadata !"fpexcept.ignore") #0 %4 = call i32 @fesetround(i32 noundef 1024) -- GitLab From 275fe3ae2dced8275a1dd85a4f892fee99d322e2 Mon Sep 17 00:00:00 2001 From: Aart Bik Date: Mon, 4 Mar 2024 17:14:31 -0800 Subject: [PATCH 078/929] [mlir][sparse] support complex type for sparse_tensor.print (#83934) With an integration test example --- .../Transforms/SparseTensorRewriting.cpp | 16 +- .../SparseTensor/CPU/sparse_complex_ops.mlir | 145 +++++++++--------- 2 files changed, 87 insertions(+), 74 deletions(-) diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp index 158845d88a44..a65bce78d095 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp @@ -692,7 +692,21 @@ private: rewriter.setInsertionPointToStart(forOp.getBody()); auto idx = forOp.getInductionVar(); auto val = rewriter.create(loc, vec, idx); - rewriter.create(loc, val, vector::PrintPunctuation::Comma); + if (llvm::isa(val.getType())) { + // Since the vector dialect does not support complex types in any op, + // we split those into (real, imag) pairs here. + Value real = rewriter.create(loc, val); + Value imag = rewriter.create(loc, val); + rewriter.create(loc, vector::PrintPunctuation::Open); + rewriter.create(loc, real, + vector::PrintPunctuation::Comma); + rewriter.create(loc, imag, + vector::PrintPunctuation::Close); + rewriter.create(loc, vector::PrintPunctuation::Comma); + } else { + rewriter.create(loc, val, + vector::PrintPunctuation::Comma); + } rewriter.setInsertionPointAfter(forOp); // Close bracket and end of line. rewriter.create(loc, vector::PrintPunctuation::Close); diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex_ops.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex_ops.mlir index f233a92fa14a..c4fc8b080787 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex_ops.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex_ops.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -162,31 +162,8 @@ module { return %0 : tensor } - func.func @dumpc(%arg0: tensor, #SparseVector>, %d: index) { - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %mem = sparse_tensor.values %arg0 : tensor, #SparseVector> to memref> - scf.for %i = %c0 to %d step %c1 { - %v = memref.load %mem[%i] : memref> - %real = complex.re %v : complex - %imag = complex.im %v : complex - vector.print %real : f64 - vector.print %imag : f64 - } - return - } - - func.func @dumpf(%arg0: tensor) { - %c0 = arith.constant 0 : index - %d0 = arith.constant 0.0 : f64 - %values = sparse_tensor.values %arg0 : tensor to memref - %0 = vector.transfer_read %values[%c0], %d0: memref, vector<3xf64> - vector.print %0 : vector<3xf64> - return - } - // Driver method to call and verify complex kernels. - func.func @entry() { + func.func @main() { // Setup sparse vectors. %v1 = arith.constant sparse< [ [0], [28], [31] ], @@ -217,54 +194,76 @@ module { // // Verify the results. // - %d3 = arith.constant 3 : index - %d4 = arith.constant 4 : index - // CHECK: -5.13 - // CHECK-NEXT: 2 - // CHECK-NEXT: 1 - // CHECK-NEXT: 0 - // CHECK-NEXT: 1 - // CHECK-NEXT: 4 - // CHECK-NEXT: 8 - // CHECK-NEXT: 6 - call @dumpc(%0, %d4) : (tensor, #SparseVector>, index) -> () - // CHECK-NEXT: 3.43887 - // CHECK-NEXT: 1.47097 - // CHECK-NEXT: 3.85374 - // CHECK-NEXT: -27.0168 - // CHECK-NEXT: -193.43 - // CHECK-NEXT: 57.2184 - call @dumpc(%1, %d3) : (tensor, #SparseVector>, index) -> () - // CHECK-NEXT: 0.433635 - // CHECK-NEXT: 2.30609 - // CHECK-NEXT: 2 - // CHECK-NEXT: 1 - // CHECK-NEXT: 2.53083 - // CHECK-NEXT: 1.18538 - call @dumpc(%2, %d3) : (tensor, #SparseVector>, index) -> () - // CHECK-NEXT: 0.761594 - // CHECK-NEXT: 0 - // CHECK-NEXT: -0.964028 - // CHECK-NEXT: 0 - // CHECK-NEXT: 0.995055 - // CHECK-NEXT: 0 - call @dumpc(%3, %d3) : (tensor, #SparseVector>, index) -> () - // CHECK-NEXT: -5.13 - // CHECK-NEXT: 2 - // CHECK-NEXT: 3 - // CHECK-NEXT: 4 - // CHECK-NEXT: 5 - // CHECK-NEXT: 6 - call @dumpc(%4, %d3) : (tensor, #SparseVector>, index) -> () - // CHECK-NEXT: -2.565 - // CHECK-NEXT: 1 - // CHECK-NEXT: 1.5 - // CHECK-NEXT: 2 - // CHECK-NEXT: 2.5 - // CHECK-NEXT: 3 - call @dumpc(%5, %d3) : (tensor, #SparseVector>, index) -> () - // CHECK-NEXT: ( 5.50608, 5, 7.81025 ) - call @dumpf(%6) : (tensor) -> () + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 4 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 4, + // CHECK-NEXT: crd[0] : ( 0, 1, 28, 31, + // CHECK-NEXT: values : ( ( -5.13, 2 ), ( 1, 0 ), ( 1, 4 ), ( 8, 6 ), + // CHECK-NEXT: ---- + // + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 3 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 3, + // CHECK-NEXT: crd[0] : ( 0, 28, 31, + // CHECK-NEXT: values : ( ( 3.43887, 1.47097 ), ( 3.85374, -27.0168 ), ( -193.43, 57.2184 ), + // CHECK-NEXT: ---- + // + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 3 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 3, + // CHECK-NEXT: crd[0] : ( 0, 28, 31, + // CHECK-NEXT: values : ( ( 0.433635, 2.30609 ), ( 2, 1 ), ( 2.53083, 1.18538 ), + // CHECK-NEXT: ---- + // + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 3 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 3, + // CHECK-NEXT: crd[0] : ( 1, 28, 31, + // CHECK-NEXT: values : ( ( 0.761594, 0 ), ( -0.964028, 0 ), ( 0.995055, 0 ), + // CHECK-NEXT: ---- + // + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 3 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 3, + // CHECK-NEXT: crd[0] : ( 0, 28, 31, + // CHECK-NEXT: values : ( ( -5.13, 2 ), ( 3, 4 ), ( 5, 6 ), + // CHECK-NEXT: ---- + // + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 3 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 3, + // CHECK-NEXT: crd[0] : ( 0, 28, 31, + // CHECK-NEXT: values : ( ( -2.565, 1 ), ( 1.5, 2 ), ( 2.5, 3 ), + // CHECK-NEXT: ---- + // + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 3 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 3, + // CHECK-NEXT: crd[0] : ( 0, 28, 31, + // CHECK-NEXT: values : ( 5.50608, 5, 7.81025, + // CHECK-NEXT: ---- + // + sparse_tensor.print %0 : tensor, #SparseVector> + sparse_tensor.print %1 : tensor, #SparseVector> + sparse_tensor.print %2 : tensor, #SparseVector> + sparse_tensor.print %3 : tensor, #SparseVector> + sparse_tensor.print %4 : tensor, #SparseVector> + sparse_tensor.print %5 : tensor, #SparseVector> + sparse_tensor.print %6 : tensor // Release the resources. bufferization.dealloc_tensor %sv1 : tensor, #SparseVector> -- GitLab From a5c90e48b6f11bc6db7344503589648f76b16d80 Mon Sep 17 00:00:00 2001 From: wanglei Date: Tue, 5 Mar 2024 09:15:44 +0800 Subject: [PATCH 079/929] [LoongArch] Switch to the Machine Scheduler (#83759) The SelectionDAG scheduling preference now becomes source order scheduling (machine scheduler generates better code -- even without there being a machine model defined for LoongArch yet). Most of the test changes are trivial instruction reorderings and differing register allocations, without any obvious performance impact. This is similar to commit: 3d0fbafd0bce43bb9106230a45d1130f7a40e5ec --- .../lib/Target/LoongArch/LoongArchSubtarget.h | 1 + llvm/test/CodeGen/LoongArch/alloca.ll | 8 +- llvm/test/CodeGen/LoongArch/alsl.ll | 16 +- .../LoongArch/atomicrmw-uinc-udec-wrap.ll | 216 +-- llvm/test/CodeGen/LoongArch/bitreverse.ll | 46 +- .../CodeGen/LoongArch/branch-relaxation.ll | 12 +- .../CodeGen/LoongArch/bswap-bitreverse.ll | 12 +- llvm/test/CodeGen/LoongArch/bswap.ll | 40 +- llvm/test/CodeGen/LoongArch/bytepick.ll | 15 +- .../CodeGen/LoongArch/calling-conv-common.ll | 136 +- .../CodeGen/LoongArch/calling-conv-lp64d.ll | 12 +- .../CodeGen/LoongArch/calling-conv-lp64s.ll | 4 +- .../LoongArch/can-not-realign-stack.ll | 24 +- .../CodeGen/LoongArch/cfr-pseudo-copy.mir | 4 +- .../test/CodeGen/LoongArch/ctlz-cttz-ctpop.ll | 98 +- llvm/test/CodeGen/LoongArch/fcopysign.ll | 8 +- .../LoongArch/get-setcc-result-type.ll | 24 +- llvm/test/CodeGen/LoongArch/ghc-cc.ll | 96 +- .../CodeGen/LoongArch/intrinsic-memcpy.ll | 16 +- .../CodeGen/LoongArch/ir-instruction/and.ll | 40 +- .../CodeGen/LoongArch/ir-instruction/ashr.ll | 4 +- .../ir-instruction/atomic-cmpxchg.ll | 210 +-- .../LoongArch/ir-instruction/atomicrmw-fp.ll | 736 +++++---- .../ir-instruction/atomicrmw-minmax.ll | 520 +++--- .../LoongArch/ir-instruction/atomicrmw.ll | 1470 ++++++++--------- .../ir-instruction/double-convert.ll | 32 +- .../LoongArch/ir-instruction/float-convert.ll | 46 +- .../LoongArch/ir-instruction/load-store.ll | 1124 ++++++++----- .../CodeGen/LoongArch/ir-instruction/lshr.ll | 4 +- .../CodeGen/LoongArch/ir-instruction/mul.ll | 103 +- .../CodeGen/LoongArch/ir-instruction/shl.ll | 4 +- .../CodeGen/LoongArch/ir-instruction/sub.ll | 4 +- .../CodeGen/LoongArch/lasx/build-vector.ll | 88 +- llvm/test/CodeGen/LoongArch/lasx/fma-v4f64.ll | 448 ++--- llvm/test/CodeGen/LoongArch/lasx/fma-v8f32.ll | 448 ++--- .../LoongArch/lasx/ir-instruction/add.ll | 24 +- .../LoongArch/lasx/ir-instruction/and.ll | 24 +- .../LoongArch/lasx/ir-instruction/ashr.ll | 24 +- .../LoongArch/lasx/ir-instruction/fadd.ll | 12 +- .../LoongArch/lasx/ir-instruction/fcmp.ll | 168 +- .../LoongArch/lasx/ir-instruction/fdiv.ll | 12 +- .../LoongArch/lasx/ir-instruction/fmul.ll | 12 +- .../LoongArch/lasx/ir-instruction/fsub.ll | 12 +- .../LoongArch/lasx/ir-instruction/icmp.ll | 144 +- .../lasx/ir-instruction/insertelement.ll | 16 +- .../LoongArch/lasx/ir-instruction/lshr.ll | 24 +- .../LoongArch/lasx/ir-instruction/mul.ll | 24 +- .../LoongArch/lasx/ir-instruction/or.ll | 24 +- .../LoongArch/lasx/ir-instruction/sdiv.ll | 24 +- .../LoongArch/lasx/ir-instruction/shl.ll | 24 +- .../LoongArch/lasx/ir-instruction/sub.ll | 24 +- .../LoongArch/lasx/ir-instruction/udiv.ll | 24 +- .../LoongArch/lasx/ir-instruction/xor.ll | 24 +- llvm/test/CodeGen/LoongArch/lasx/mulh.ll | 48 +- llvm/test/CodeGen/LoongArch/lasx/vselect.ll | 22 +- .../CodeGen/LoongArch/lsx/build-vector.ll | 40 +- llvm/test/CodeGen/LoongArch/lsx/fma-v2f64.ll | 448 ++--- llvm/test/CodeGen/LoongArch/lsx/fma-v4f32.ll | 448 ++--- .../LoongArch/lsx/ir-instruction/add.ll | 24 +- .../LoongArch/lsx/ir-instruction/and.ll | 24 +- .../LoongArch/lsx/ir-instruction/ashr.ll | 24 +- .../lsx/ir-instruction/extractelement.ll | 24 +- .../LoongArch/lsx/ir-instruction/fadd.ll | 12 +- .../LoongArch/lsx/ir-instruction/fcmp.ll | 168 +- .../LoongArch/lsx/ir-instruction/fdiv.ll | 12 +- .../LoongArch/lsx/ir-instruction/fmul.ll | 12 +- .../LoongArch/lsx/ir-instruction/fsub.ll | 12 +- .../LoongArch/lsx/ir-instruction/icmp.ll | 144 +- .../lsx/ir-instruction/insertelement.ll | 16 +- .../LoongArch/lsx/ir-instruction/lshr.ll | 24 +- .../LoongArch/lsx/ir-instruction/mul.ll | 24 +- .../LoongArch/lsx/ir-instruction/or.ll | 24 +- .../LoongArch/lsx/ir-instruction/sdiv.ll | 24 +- .../LoongArch/lsx/ir-instruction/shl.ll | 24 +- .../LoongArch/lsx/ir-instruction/sub.ll | 24 +- .../LoongArch/lsx/ir-instruction/udiv.ll | 24 +- .../LoongArch/lsx/ir-instruction/xor.ll | 24 +- llvm/test/CodeGen/LoongArch/lsx/mulh.ll | 48 +- llvm/test/CodeGen/LoongArch/lsx/vselect.ll | 22 +- .../CodeGen/LoongArch/preferred-alignments.ll | 16 +- llvm/test/CodeGen/LoongArch/rotl-rotr.ll | 388 +++-- .../CodeGen/LoongArch/select-to-shiftand.ll | 4 +- .../CodeGen/LoongArch/shift-masked-shamt.ll | 36 +- llvm/test/CodeGen/LoongArch/shrinkwrap.ll | 4 +- .../CodeGen/LoongArch/smul-with-overflow.ll | 706 ++++---- llvm/test/CodeGen/LoongArch/soft-fp-to-int.ll | 32 +- .../LoongArch/spill-ra-without-kill.ll | 12 +- .../CodeGen/LoongArch/spill-reload-cfr.ll | 62 +- llvm/test/CodeGen/LoongArch/tail-calls.ll | 18 +- .../CodeGen/LoongArch/unaligned-access.ll | 24 +- llvm/test/CodeGen/LoongArch/vararg.ll | 20 +- llvm/test/CodeGen/LoongArch/vector-fp-imm.ll | 1325 ++++++++------- .../LoongArch/zext-with-load-is-free.ll | 8 +- ...arch_generated_funcs.ll.generated.expected | 33 +- ...ch_generated_funcs.ll.nogenerated.expected | 33 +- 95 files changed, 5866 insertions(+), 5304 deletions(-) diff --git a/llvm/lib/Target/LoongArch/LoongArchSubtarget.h b/llvm/lib/Target/LoongArch/LoongArchSubtarget.h index 11c0b39e176e..cecb4a50aa76 100644 --- a/llvm/lib/Target/LoongArch/LoongArchSubtarget.h +++ b/llvm/lib/Target/LoongArch/LoongArchSubtarget.h @@ -113,6 +113,7 @@ public: Align getPrefFunctionAlignment() const { return PrefFunctionAlignment; } Align getPrefLoopAlignment() const { return PrefLoopAlignment; } unsigned getMaxBytesForAlignment() const { return MaxBytesForAlignment; } + bool enableMachineScheduler() const override { return true; } }; } // end namespace llvm diff --git a/llvm/test/CodeGen/LoongArch/alloca.ll b/llvm/test/CodeGen/LoongArch/alloca.ll index d766be6aac95..75a05689e417 100644 --- a/llvm/test/CodeGen/LoongArch/alloca.ll +++ b/llvm/test/CodeGen/LoongArch/alloca.ll @@ -126,8 +126,7 @@ define void @alloca_callframe(i32 %n) nounwind { ; LA32-NEXT: st.w $a1, $sp, 8 ; LA32-NEXT: ori $a1, $zero, 10 ; LA32-NEXT: st.w $a1, $sp, 4 -; LA32-NEXT: ori $a1, $zero, 9 -; LA32-NEXT: st.w $a1, $sp, 0 +; LA32-NEXT: ori $t0, $zero, 9 ; LA32-NEXT: ori $a1, $zero, 2 ; LA32-NEXT: ori $a2, $zero, 3 ; LA32-NEXT: ori $a3, $zero, 4 @@ -135,6 +134,7 @@ define void @alloca_callframe(i32 %n) nounwind { ; LA32-NEXT: ori $a5, $zero, 6 ; LA32-NEXT: ori $a6, $zero, 7 ; LA32-NEXT: ori $a7, $zero, 8 +; LA32-NEXT: st.w $t0, $sp, 0 ; LA32-NEXT: bl %plt(func) ; LA32-NEXT: addi.w $sp, $sp, 16 ; LA32-NEXT: addi.w $sp, $fp, -16 @@ -162,8 +162,7 @@ define void @alloca_callframe(i32 %n) nounwind { ; LA64-NEXT: st.d $a1, $sp, 16 ; LA64-NEXT: ori $a1, $zero, 10 ; LA64-NEXT: st.d $a1, $sp, 8 -; LA64-NEXT: ori $a1, $zero, 9 -; LA64-NEXT: st.d $a1, $sp, 0 +; LA64-NEXT: ori $t0, $zero, 9 ; LA64-NEXT: ori $a1, $zero, 2 ; LA64-NEXT: ori $a2, $zero, 3 ; LA64-NEXT: ori $a3, $zero, 4 @@ -171,6 +170,7 @@ define void @alloca_callframe(i32 %n) nounwind { ; LA64-NEXT: ori $a5, $zero, 6 ; LA64-NEXT: ori $a6, $zero, 7 ; LA64-NEXT: ori $a7, $zero, 8 +; LA64-NEXT: st.d $t0, $sp, 0 ; LA64-NEXT: bl %plt(func) ; LA64-NEXT: addi.d $sp, $sp, 32 ; LA64-NEXT: addi.d $sp, $fp, -16 diff --git a/llvm/test/CodeGen/LoongArch/alsl.ll b/llvm/test/CodeGen/LoongArch/alsl.ll index 650f504dcaf8..177e37de0952 100644 --- a/llvm/test/CodeGen/LoongArch/alsl.ll +++ b/llvm/test/CodeGen/LoongArch/alsl.ll @@ -53,12 +53,12 @@ entry: define i64 @alsl_i64(i64 signext %a, i64 signext %b) nounwind { ; LA32-LABEL: alsl_i64: ; LA32: # %bb.0: # %entry -; LA32-NEXT: slli.w $a1, $a1, 4 ; LA32-NEXT: srli.w $a4, $a0, 28 +; LA32-NEXT: slli.w $a1, $a1, 4 ; LA32-NEXT: or $a1, $a1, $a4 -; LA32-NEXT: add.w $a1, $a3, $a1 ; LA32-NEXT: alsl.w $a0, $a0, $a2, 4 ; LA32-NEXT: sltu $a2, $a0, $a2 +; LA32-NEXT: add.w $a1, $a3, $a1 ; LA32-NEXT: add.w $a1, $a1, $a2 ; LA32-NEXT: ret ; @@ -189,14 +189,14 @@ entry: define i64 @mul_add_i64(i64 signext %a, i64 signext %b) nounwind { ; LA32-LABEL: mul_add_i64: ; LA32: # %bb.0: # %entry -; LA32-NEXT: slli.w $a4, $a1, 4 -; LA32-NEXT: sub.w $a1, $a4, $a1 ; LA32-NEXT: ori $a4, $zero, 15 ; LA32-NEXT: mulh.wu $a4, $a0, $a4 +; LA32-NEXT: slli.w $a5, $a1, 4 +; LA32-NEXT: sub.w $a1, $a5, $a1 ; LA32-NEXT: add.w $a1, $a4, $a1 +; LA32-NEXT: slli.w $a4, $a0, 4 +; LA32-NEXT: sub.w $a0, $a4, $a0 ; LA32-NEXT: add.w $a1, $a3, $a1 -; LA32-NEXT: slli.w $a3, $a0, 4 -; LA32-NEXT: sub.w $a0, $a3, $a0 ; LA32-NEXT: add.w $a0, $a2, $a0 ; LA32-NEXT: sltu $a2, $a0, $a2 ; LA32-NEXT: add.w $a1, $a1, $a2 @@ -342,9 +342,9 @@ define i64 @mul_add_neg_i64(i64 signext %a, i64 signext %b) nounwind { ; LA32-NEXT: mulh.wu $a4, $a0, $a4 ; LA32-NEXT: sub.w $a4, $a4, $a0 ; LA32-NEXT: add.w $a1, $a4, $a1 +; LA32-NEXT: slli.w $a4, $a0, 4 +; LA32-NEXT: sub.w $a0, $a0, $a4 ; LA32-NEXT: add.w $a1, $a3, $a1 -; LA32-NEXT: slli.w $a3, $a0, 4 -; LA32-NEXT: sub.w $a0, $a0, $a3 ; LA32-NEXT: add.w $a0, $a2, $a0 ; LA32-NEXT: sltu $a2, $a0, $a2 ; LA32-NEXT: add.w $a1, $a1, $a2 diff --git a/llvm/test/CodeGen/LoongArch/atomicrmw-uinc-udec-wrap.ll b/llvm/test/CodeGen/LoongArch/atomicrmw-uinc-udec-wrap.ll index b84c1093eb75..bf48c0df3e49 100644 --- a/llvm/test/CodeGen/LoongArch/atomicrmw-uinc-udec-wrap.ll +++ b/llvm/test/CodeGen/LoongArch/atomicrmw-uinc-udec-wrap.ll @@ -4,34 +4,34 @@ define i8 @atomicrmw_uinc_wrap_i8(ptr %ptr, i8 %val) { ; LA64-LABEL: atomicrmw_uinc_wrap_i8: ; LA64: # %bb.0: -; LA64-NEXT: slli.d $a3, $a0, 3 +; LA64-NEXT: slli.d $a2, $a0, 3 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: ld.w $a2, $a0, 0 -; LA64-NEXT: ori $a4, $zero, 255 -; LA64-NEXT: sll.w $a4, $a4, $a3 -; LA64-NEXT: andi $a3, $a3, 24 +; LA64-NEXT: ori $a3, $zero, 255 +; LA64-NEXT: sll.w $a4, $a3, $a2 +; LA64-NEXT: ld.w $a3, $a0, 0 +; LA64-NEXT: andi $a2, $a2, 24 ; LA64-NEXT: nor $a4, $a4, $zero ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: .p2align 4, , 16 ; LA64-NEXT: .LBB0_1: # %atomicrmw.start ; LA64-NEXT: # =>This Loop Header: Depth=1 ; LA64-NEXT: # Child Loop BB0_3 Depth 2 -; LA64-NEXT: srl.w $a5, $a2, $a3 -; LA64-NEXT: andi $a6, $a5, 255 -; LA64-NEXT: sltu $a6, $a6, $a1 +; LA64-NEXT: srl.w $a5, $a3, $a2 +; LA64-NEXT: addi.w $a6, $a3, 0 +; LA64-NEXT: andi $a7, $a5, 255 ; LA64-NEXT: addi.d $a5, $a5, 1 -; LA64-NEXT: xori $a6, $a6, 1 -; LA64-NEXT: masknez $a5, $a5, $a6 +; LA64-NEXT: sltu $a7, $a7, $a1 +; LA64-NEXT: xori $a7, $a7, 1 +; LA64-NEXT: masknez $a5, $a5, $a7 ; LA64-NEXT: andi $a5, $a5, 255 -; LA64-NEXT: sll.w $a5, $a5, $a3 -; LA64-NEXT: and $a6, $a2, $a4 -; LA64-NEXT: or $a5, $a6, $a5 -; LA64-NEXT: addi.w $a6, $a2, 0 +; LA64-NEXT: sll.w $a5, $a5, $a2 +; LA64-NEXT: and $a3, $a3, $a4 +; LA64-NEXT: or $a5, $a3, $a5 ; LA64-NEXT: .LBB0_3: # %atomicrmw.start ; LA64-NEXT: # Parent Loop BB0_1 Depth=1 ; LA64-NEXT: # => This Inner Loop Header: Depth=2 -; LA64-NEXT: ll.w $a2, $a0, 0 -; LA64-NEXT: bne $a2, $a6, .LBB0_5 +; LA64-NEXT: ll.w $a3, $a0, 0 +; LA64-NEXT: bne $a3, $a6, .LBB0_5 ; LA64-NEXT: # %bb.4: # %atomicrmw.start ; LA64-NEXT: # in Loop: Header=BB0_3 Depth=2 ; LA64-NEXT: move $a7, $a5 @@ -43,9 +43,9 @@ define i8 @atomicrmw_uinc_wrap_i8(ptr %ptr, i8 %val) { ; LA64-NEXT: dbar 20 ; LA64-NEXT: .LBB0_6: # %atomicrmw.start ; LA64-NEXT: # in Loop: Header=BB0_1 Depth=1 -; LA64-NEXT: bne $a2, $a6, .LBB0_1 +; LA64-NEXT: bne $a3, $a6, .LBB0_1 ; LA64-NEXT: # %bb.2: # %atomicrmw.end -; LA64-NEXT: srl.w $a0, $a2, $a3 +; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret %result = atomicrmw uinc_wrap ptr %ptr, i8 %val seq_cst ret i8 %result @@ -54,35 +54,35 @@ define i8 @atomicrmw_uinc_wrap_i8(ptr %ptr, i8 %val) { define i16 @atomicrmw_uinc_wrap_i16(ptr %ptr, i16 %val) { ; LA64-LABEL: atomicrmw_uinc_wrap_i16: ; LA64: # %bb.0: -; LA64-NEXT: slli.d $a3, $a0, 3 +; LA64-NEXT: slli.d $a2, $a0, 3 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: ld.w $a2, $a0, 0 -; LA64-NEXT: lu12i.w $a4, 15 -; LA64-NEXT: ori $a4, $a4, 4095 -; LA64-NEXT: sll.w $a4, $a4, $a3 -; LA64-NEXT: andi $a3, $a3, 24 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a4, $a3, $a2 +; LA64-NEXT: ld.w $a3, $a0, 0 +; LA64-NEXT: andi $a2, $a2, 24 ; LA64-NEXT: nor $a4, $a4, $zero ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 ; LA64-NEXT: .p2align 4, , 16 ; LA64-NEXT: .LBB1_1: # %atomicrmw.start ; LA64-NEXT: # =>This Loop Header: Depth=1 ; LA64-NEXT: # Child Loop BB1_3 Depth 2 -; LA64-NEXT: srl.w $a5, $a2, $a3 -; LA64-NEXT: bstrpick.d $a6, $a5, 15, 0 -; LA64-NEXT: sltu $a6, $a6, $a1 +; LA64-NEXT: srl.w $a5, $a3, $a2 +; LA64-NEXT: addi.w $a6, $a3, 0 +; LA64-NEXT: bstrpick.d $a7, $a5, 15, 0 ; LA64-NEXT: addi.d $a5, $a5, 1 -; LA64-NEXT: xori $a6, $a6, 1 -; LA64-NEXT: masknez $a5, $a5, $a6 +; LA64-NEXT: sltu $a7, $a7, $a1 +; LA64-NEXT: xori $a7, $a7, 1 +; LA64-NEXT: masknez $a5, $a5, $a7 ; LA64-NEXT: bstrpick.d $a5, $a5, 15, 0 -; LA64-NEXT: sll.w $a5, $a5, $a3 -; LA64-NEXT: and $a6, $a2, $a4 -; LA64-NEXT: or $a5, $a6, $a5 -; LA64-NEXT: addi.w $a6, $a2, 0 +; LA64-NEXT: sll.w $a5, $a5, $a2 +; LA64-NEXT: and $a3, $a3, $a4 +; LA64-NEXT: or $a5, $a3, $a5 ; LA64-NEXT: .LBB1_3: # %atomicrmw.start ; LA64-NEXT: # Parent Loop BB1_1 Depth=1 ; LA64-NEXT: # => This Inner Loop Header: Depth=2 -; LA64-NEXT: ll.w $a2, $a0, 0 -; LA64-NEXT: bne $a2, $a6, .LBB1_5 +; LA64-NEXT: ll.w $a3, $a0, 0 +; LA64-NEXT: bne $a3, $a6, .LBB1_5 ; LA64-NEXT: # %bb.4: # %atomicrmw.start ; LA64-NEXT: # in Loop: Header=BB1_3 Depth=2 ; LA64-NEXT: move $a7, $a5 @@ -94,9 +94,9 @@ define i16 @atomicrmw_uinc_wrap_i16(ptr %ptr, i16 %val) { ; LA64-NEXT: dbar 20 ; LA64-NEXT: .LBB1_6: # %atomicrmw.start ; LA64-NEXT: # in Loop: Header=BB1_1 Depth=1 -; LA64-NEXT: bne $a2, $a6, .LBB1_1 +; LA64-NEXT: bne $a3, $a6, .LBB1_1 ; LA64-NEXT: # %bb.2: # %atomicrmw.end -; LA64-NEXT: srl.w $a0, $a2, $a3 +; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret %result = atomicrmw uinc_wrap ptr %ptr, i16 %val seq_cst ret i16 %result @@ -111,19 +111,19 @@ define i32 @atomicrmw_uinc_wrap_i32(ptr %ptr, i32 %val) { ; LA64-NEXT: .LBB2_1: # %atomicrmw.start ; LA64-NEXT: # =>This Loop Header: Depth=1 ; LA64-NEXT: # Child Loop BB2_3 Depth 2 -; LA64-NEXT: addi.w $a3, $a2, 0 -; LA64-NEXT: sltu $a4, $a3, $a1 -; LA64-NEXT: xori $a4, $a4, 1 -; LA64-NEXT: addi.d $a2, $a2, 1 -; LA64-NEXT: masknez $a4, $a2, $a4 +; LA64-NEXT: addi.d $a3, $a2, 1 +; LA64-NEXT: addi.w $a4, $a2, 0 +; LA64-NEXT: sltu $a2, $a4, $a1 +; LA64-NEXT: xori $a2, $a2, 1 +; LA64-NEXT: masknez $a3, $a3, $a2 ; LA64-NEXT: .LBB2_3: # %atomicrmw.start ; LA64-NEXT: # Parent Loop BB2_1 Depth=1 ; LA64-NEXT: # => This Inner Loop Header: Depth=2 ; LA64-NEXT: ll.w $a2, $a0, 0 -; LA64-NEXT: bne $a2, $a3, .LBB2_5 +; LA64-NEXT: bne $a2, $a4, .LBB2_5 ; LA64-NEXT: # %bb.4: # %atomicrmw.start ; LA64-NEXT: # in Loop: Header=BB2_3 Depth=2 -; LA64-NEXT: move $a5, $a4 +; LA64-NEXT: move $a5, $a3 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB2_3 ; LA64-NEXT: b .LBB2_6 @@ -132,7 +132,7 @@ define i32 @atomicrmw_uinc_wrap_i32(ptr %ptr, i32 %val) { ; LA64-NEXT: dbar 20 ; LA64-NEXT: .LBB2_6: # %atomicrmw.start ; LA64-NEXT: # in Loop: Header=BB2_1 Depth=1 -; LA64-NEXT: bne $a2, $a3, .LBB2_1 +; LA64-NEXT: bne $a2, $a4, .LBB2_1 ; LA64-NEXT: # %bb.2: # %atomicrmw.end ; LA64-NEXT: move $a0, $a2 ; LA64-NEXT: ret @@ -149,10 +149,10 @@ define i64 @atomicrmw_uinc_wrap_i64(ptr %ptr, i64 %val) { ; LA64-NEXT: # =>This Loop Header: Depth=1 ; LA64-NEXT: # Child Loop BB3_3 Depth 2 ; LA64-NEXT: move $a3, $a2 -; LA64-NEXT: sltu $a2, $a2, $a1 -; LA64-NEXT: xori $a2, $a2, 1 -; LA64-NEXT: addi.d $a4, $a3, 1 -; LA64-NEXT: masknez $a4, $a4, $a2 +; LA64-NEXT: addi.d $a2, $a2, 1 +; LA64-NEXT: sltu $a4, $a3, $a1 +; LA64-NEXT: xori $a4, $a4, 1 +; LA64-NEXT: masknez $a4, $a2, $a4 ; LA64-NEXT: .LBB3_3: # %atomicrmw.start ; LA64-NEXT: # Parent Loop BB3_1 Depth=1 ; LA64-NEXT: # => This Inner Loop Header: Depth=2 @@ -180,39 +180,39 @@ define i64 @atomicrmw_uinc_wrap_i64(ptr %ptr, i64 %val) { define i8 @atomicrmw_udec_wrap_i8(ptr %ptr, i8 %val) { ; LA64-LABEL: atomicrmw_udec_wrap_i8: ; LA64: # %bb.0: -; LA64-NEXT: slli.d $a3, $a0, 3 +; LA64-NEXT: slli.d $a4, $a0, 3 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: ld.w $a2, $a0, 0 -; LA64-NEXT: ori $a4, $zero, 255 -; LA64-NEXT: sll.w $a4, $a4, $a3 -; LA64-NEXT: andi $a3, $a3, 24 +; LA64-NEXT: andi $a2, $a4, 24 +; LA64-NEXT: ori $a5, $zero, 255 +; LA64-NEXT: ld.w $a3, $a0, 0 +; LA64-NEXT: sll.w $a4, $a5, $a4 ; LA64-NEXT: nor $a4, $a4, $zero ; LA64-NEXT: andi $a5, $a1, 255 ; LA64-NEXT: .p2align 4, , 16 ; LA64-NEXT: .LBB4_1: # %atomicrmw.start ; LA64-NEXT: # =>This Loop Header: Depth=1 ; LA64-NEXT: # Child Loop BB4_3 Depth 2 -; LA64-NEXT: srl.w $a6, $a2, $a3 -; LA64-NEXT: andi $a7, $a6, 255 -; LA64-NEXT: sltu $t0, $a5, $a7 +; LA64-NEXT: srl.w $a6, $a3, $a2 +; LA64-NEXT: addi.w $a7, $a3, 0 +; LA64-NEXT: andi $t0, $a6, 255 ; LA64-NEXT: addi.d $a6, $a6, -1 +; LA64-NEXT: sltui $t1, $t0, 1 +; LA64-NEXT: sltu $t0, $a5, $t0 ; LA64-NEXT: masknez $a6, $a6, $t0 ; LA64-NEXT: maskeqz $t0, $a1, $t0 ; LA64-NEXT: or $a6, $t0, $a6 -; LA64-NEXT: sltui $a7, $a7, 1 -; LA64-NEXT: masknez $a6, $a6, $a7 -; LA64-NEXT: maskeqz $a7, $a1, $a7 -; LA64-NEXT: or $a6, $a7, $a6 +; LA64-NEXT: masknez $a6, $a6, $t1 +; LA64-NEXT: maskeqz $t0, $a1, $t1 +; LA64-NEXT: or $a6, $t0, $a6 ; LA64-NEXT: andi $a6, $a6, 255 -; LA64-NEXT: sll.w $a6, $a6, $a3 -; LA64-NEXT: and $a7, $a2, $a4 -; LA64-NEXT: or $a6, $a7, $a6 -; LA64-NEXT: addi.w $a7, $a2, 0 +; LA64-NEXT: sll.w $a6, $a6, $a2 +; LA64-NEXT: and $a3, $a3, $a4 +; LA64-NEXT: or $a6, $a3, $a6 ; LA64-NEXT: .LBB4_3: # %atomicrmw.start ; LA64-NEXT: # Parent Loop BB4_1 Depth=1 ; LA64-NEXT: # => This Inner Loop Header: Depth=2 -; LA64-NEXT: ll.w $a2, $a0, 0 -; LA64-NEXT: bne $a2, $a7, .LBB4_5 +; LA64-NEXT: ll.w $a3, $a0, 0 +; LA64-NEXT: bne $a3, $a7, .LBB4_5 ; LA64-NEXT: # %bb.4: # %atomicrmw.start ; LA64-NEXT: # in Loop: Header=BB4_3 Depth=2 ; LA64-NEXT: move $t0, $a6 @@ -224,9 +224,9 @@ define i8 @atomicrmw_udec_wrap_i8(ptr %ptr, i8 %val) { ; LA64-NEXT: dbar 20 ; LA64-NEXT: .LBB4_6: # %atomicrmw.start ; LA64-NEXT: # in Loop: Header=BB4_1 Depth=1 -; LA64-NEXT: bne $a2, $a7, .LBB4_1 +; LA64-NEXT: bne $a3, $a7, .LBB4_1 ; LA64-NEXT: # %bb.2: # %atomicrmw.end -; LA64-NEXT: srl.w $a0, $a2, $a3 +; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret %result = atomicrmw udec_wrap ptr %ptr, i8 %val seq_cst ret i8 %result @@ -235,40 +235,40 @@ define i8 @atomicrmw_udec_wrap_i8(ptr %ptr, i8 %val) { define i16 @atomicrmw_udec_wrap_i16(ptr %ptr, i16 %val) { ; LA64-LABEL: atomicrmw_udec_wrap_i16: ; LA64: # %bb.0: -; LA64-NEXT: slli.d $a3, $a0, 3 +; LA64-NEXT: slli.d $a4, $a0, 3 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: ld.w $a2, $a0, 0 -; LA64-NEXT: lu12i.w $a4, 15 -; LA64-NEXT: ori $a4, $a4, 4095 -; LA64-NEXT: sll.w $a4, $a4, $a3 -; LA64-NEXT: andi $a3, $a3, 24 +; LA64-NEXT: andi $a2, $a4, 24 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a5, $a3, 4095 +; LA64-NEXT: ld.w $a3, $a0, 0 +; LA64-NEXT: sll.w $a4, $a5, $a4 ; LA64-NEXT: nor $a4, $a4, $zero ; LA64-NEXT: bstrpick.d $a5, $a1, 15, 0 ; LA64-NEXT: .p2align 4, , 16 ; LA64-NEXT: .LBB5_1: # %atomicrmw.start ; LA64-NEXT: # =>This Loop Header: Depth=1 ; LA64-NEXT: # Child Loop BB5_3 Depth 2 -; LA64-NEXT: srl.w $a6, $a2, $a3 -; LA64-NEXT: bstrpick.d $a7, $a6, 15, 0 -; LA64-NEXT: sltu $t0, $a5, $a7 +; LA64-NEXT: srl.w $a6, $a3, $a2 +; LA64-NEXT: addi.w $a7, $a3, 0 +; LA64-NEXT: bstrpick.d $t0, $a6, 15, 0 ; LA64-NEXT: addi.d $a6, $a6, -1 +; LA64-NEXT: sltui $t1, $t0, 1 +; LA64-NEXT: sltu $t0, $a5, $t0 ; LA64-NEXT: masknez $a6, $a6, $t0 ; LA64-NEXT: maskeqz $t0, $a1, $t0 ; LA64-NEXT: or $a6, $t0, $a6 -; LA64-NEXT: sltui $a7, $a7, 1 -; LA64-NEXT: masknez $a6, $a6, $a7 -; LA64-NEXT: maskeqz $a7, $a1, $a7 -; LA64-NEXT: or $a6, $a7, $a6 +; LA64-NEXT: masknez $a6, $a6, $t1 +; LA64-NEXT: maskeqz $t0, $a1, $t1 +; LA64-NEXT: or $a6, $t0, $a6 ; LA64-NEXT: bstrpick.d $a6, $a6, 15, 0 -; LA64-NEXT: sll.w $a6, $a6, $a3 -; LA64-NEXT: and $a7, $a2, $a4 -; LA64-NEXT: or $a6, $a7, $a6 -; LA64-NEXT: addi.w $a7, $a2, 0 +; LA64-NEXT: sll.w $a6, $a6, $a2 +; LA64-NEXT: and $a3, $a3, $a4 +; LA64-NEXT: or $a6, $a3, $a6 ; LA64-NEXT: .LBB5_3: # %atomicrmw.start ; LA64-NEXT: # Parent Loop BB5_1 Depth=1 ; LA64-NEXT: # => This Inner Loop Header: Depth=2 -; LA64-NEXT: ll.w $a2, $a0, 0 -; LA64-NEXT: bne $a2, $a7, .LBB5_5 +; LA64-NEXT: ll.w $a3, $a0, 0 +; LA64-NEXT: bne $a3, $a7, .LBB5_5 ; LA64-NEXT: # %bb.4: # %atomicrmw.start ; LA64-NEXT: # in Loop: Header=BB5_3 Depth=2 ; LA64-NEXT: move $t0, $a6 @@ -280,9 +280,9 @@ define i16 @atomicrmw_udec_wrap_i16(ptr %ptr, i16 %val) { ; LA64-NEXT: dbar 20 ; LA64-NEXT: .LBB5_6: # %atomicrmw.start ; LA64-NEXT: # in Loop: Header=BB5_1 Depth=1 -; LA64-NEXT: bne $a2, $a7, .LBB5_1 +; LA64-NEXT: bne $a3, $a7, .LBB5_1 ; LA64-NEXT: # %bb.2: # %atomicrmw.end -; LA64-NEXT: srl.w $a0, $a2, $a3 +; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret %result = atomicrmw udec_wrap ptr %ptr, i16 %val seq_cst ret i16 %result @@ -297,24 +297,24 @@ define i32 @atomicrmw_udec_wrap_i32(ptr %ptr, i32 %val) { ; LA64-NEXT: .LBB6_1: # %atomicrmw.start ; LA64-NEXT: # =>This Loop Header: Depth=1 ; LA64-NEXT: # Child Loop BB6_3 Depth 2 -; LA64-NEXT: addi.w $a4, $a2, 0 -; LA64-NEXT: sltu $a5, $a3, $a4 -; LA64-NEXT: addi.d $a2, $a2, -1 -; LA64-NEXT: masknez $a2, $a2, $a5 -; LA64-NEXT: maskeqz $a5, $a1, $a5 -; LA64-NEXT: or $a2, $a5, $a2 -; LA64-NEXT: sltui $a5, $a4, 1 -; LA64-NEXT: masknez $a2, $a2, $a5 -; LA64-NEXT: maskeqz $a5, $a1, $a5 -; LA64-NEXT: or $a5, $a5, $a2 +; LA64-NEXT: addi.d $a4, $a2, -1 +; LA64-NEXT: addi.w $a5, $a2, 0 +; LA64-NEXT: sltui $a2, $a5, 1 +; LA64-NEXT: sltu $a6, $a3, $a5 +; LA64-NEXT: masknez $a4, $a4, $a6 +; LA64-NEXT: maskeqz $a6, $a1, $a6 +; LA64-NEXT: or $a4, $a6, $a4 +; LA64-NEXT: masknez $a4, $a4, $a2 +; LA64-NEXT: maskeqz $a2, $a1, $a2 +; LA64-NEXT: or $a4, $a2, $a4 ; LA64-NEXT: .LBB6_3: # %atomicrmw.start ; LA64-NEXT: # Parent Loop BB6_1 Depth=1 ; LA64-NEXT: # => This Inner Loop Header: Depth=2 ; LA64-NEXT: ll.w $a2, $a0, 0 -; LA64-NEXT: bne $a2, $a4, .LBB6_5 +; LA64-NEXT: bne $a2, $a5, .LBB6_5 ; LA64-NEXT: # %bb.4: # %atomicrmw.start ; LA64-NEXT: # in Loop: Header=BB6_3 Depth=2 -; LA64-NEXT: move $a6, $a5 +; LA64-NEXT: move $a6, $a4 ; LA64-NEXT: sc.w $a6, $a0, 0 ; LA64-NEXT: beqz $a6, .LBB6_3 ; LA64-NEXT: b .LBB6_6 @@ -323,7 +323,7 @@ define i32 @atomicrmw_udec_wrap_i32(ptr %ptr, i32 %val) { ; LA64-NEXT: dbar 20 ; LA64-NEXT: .LBB6_6: # %atomicrmw.start ; LA64-NEXT: # in Loop: Header=BB6_1 Depth=1 -; LA64-NEXT: bne $a2, $a4, .LBB6_1 +; LA64-NEXT: bne $a2, $a5, .LBB6_1 ; LA64-NEXT: # %bb.2: # %atomicrmw.end ; LA64-NEXT: move $a0, $a2 ; LA64-NEXT: ret @@ -340,12 +340,12 @@ define i64 @atomicrmw_udec_wrap_i64(ptr %ptr, i64 %val) { ; LA64-NEXT: # =>This Loop Header: Depth=1 ; LA64-NEXT: # Child Loop BB7_3 Depth 2 ; LA64-NEXT: move $a3, $a2 -; LA64-NEXT: sltu $a2, $a1, $a2 -; LA64-NEXT: addi.d $a4, $a3, -1 -; LA64-NEXT: masknez $a4, $a4, $a2 -; LA64-NEXT: maskeqz $a2, $a1, $a2 -; LA64-NEXT: or $a2, $a2, $a4 +; LA64-NEXT: addi.d $a2, $a2, -1 ; LA64-NEXT: sltui $a4, $a3, 1 +; LA64-NEXT: sltu $a5, $a1, $a3 +; LA64-NEXT: masknez $a2, $a2, $a5 +; LA64-NEXT: maskeqz $a5, $a1, $a5 +; LA64-NEXT: or $a2, $a5, $a2 ; LA64-NEXT: masknez $a2, $a2, $a4 ; LA64-NEXT: maskeqz $a4, $a1, $a4 ; LA64-NEXT: or $a4, $a4, $a2 diff --git a/llvm/test/CodeGen/LoongArch/bitreverse.ll b/llvm/test/CodeGen/LoongArch/bitreverse.ll index 259d8565c684..fcf523aa3c88 100644 --- a/llvm/test/CodeGen/LoongArch/bitreverse.ll +++ b/llvm/test/CodeGen/LoongArch/bitreverse.ll @@ -129,30 +129,30 @@ define i48 @test_bitreverse_i48(i48 %a) nounwind { define i77 @test_bitreverse_i77(i77 %a) nounwind { ; LA32-LABEL: test_bitreverse_i77: ; LA32: # %bb.0: -; LA32-NEXT: ld.w $a2, $a1, 0 +; LA32-NEXT: ld.w $a2, $a1, 4 +; LA32-NEXT: ld.w $a3, $a1, 8 +; LA32-NEXT: ld.w $a1, $a1, 0 ; LA32-NEXT: bitrev.w $a2, $a2 -; LA32-NEXT: ld.w $a3, $a1, 4 +; LA32-NEXT: slli.w $a4, $a2, 13 ; LA32-NEXT: bitrev.w $a3, $a3 -; LA32-NEXT: srli.w $a4, $a3, 19 -; LA32-NEXT: slli.w $a5, $a2, 13 -; LA32-NEXT: or $a4, $a5, $a4 +; LA32-NEXT: srli.w $a3, $a3, 19 +; LA32-NEXT: or $a3, $a3, $a4 ; LA32-NEXT: srli.w $a2, $a2, 19 -; LA32-NEXT: st.h $a2, $a0, 8 -; LA32-NEXT: st.w $a4, $a0, 4 -; LA32-NEXT: slli.w $a2, $a3, 13 -; LA32-NEXT: ld.w $a1, $a1, 8 ; LA32-NEXT: bitrev.w $a1, $a1 +; LA32-NEXT: slli.w $a4, $a1, 13 +; LA32-NEXT: or $a2, $a4, $a2 ; LA32-NEXT: srli.w $a1, $a1, 19 -; LA32-NEXT: or $a1, $a1, $a2 -; LA32-NEXT: st.w $a1, $a0, 0 +; LA32-NEXT: st.h $a1, $a0, 8 +; LA32-NEXT: st.w $a2, $a0, 4 +; LA32-NEXT: st.w $a3, $a0, 0 ; LA32-NEXT: ret ; ; LA64-LABEL: test_bitreverse_i77: ; LA64: # %bb.0: -; LA64-NEXT: bitrev.d $a1, $a1 -; LA64-NEXT: srli.d $a1, $a1, 51 ; LA64-NEXT: bitrev.d $a2, $a0 ; LA64-NEXT: slli.d $a0, $a2, 13 +; LA64-NEXT: bitrev.d $a1, $a1 +; LA64-NEXT: srli.d $a1, $a1, 51 ; LA64-NEXT: or $a0, $a1, $a0 ; LA64-NEXT: srli.d $a1, $a2, 51 ; LA64-NEXT: ret @@ -163,18 +163,18 @@ define i77 @test_bitreverse_i77(i77 %a) nounwind { define i128 @test_bitreverse_i128(i128 %a) nounwind { ; LA32-LABEL: test_bitreverse_i128: ; LA32: # %bb.0: -; LA32-NEXT: ld.w $a2, $a1, 0 +; LA32-NEXT: ld.w $a2, $a1, 12 +; LA32-NEXT: ld.w $a3, $a1, 8 +; LA32-NEXT: ld.w $a4, $a1, 4 +; LA32-NEXT: ld.w $a1, $a1, 0 ; LA32-NEXT: bitrev.w $a2, $a2 -; LA32-NEXT: st.w $a2, $a0, 12 -; LA32-NEXT: ld.w $a2, $a1, 4 -; LA32-NEXT: bitrev.w $a2, $a2 -; LA32-NEXT: st.w $a2, $a0, 8 -; LA32-NEXT: ld.w $a2, $a1, 8 -; LA32-NEXT: bitrev.w $a2, $a2 -; LA32-NEXT: st.w $a2, $a0, 4 -; LA32-NEXT: ld.w $a1, $a1, 12 +; LA32-NEXT: bitrev.w $a3, $a3 +; LA32-NEXT: bitrev.w $a4, $a4 ; LA32-NEXT: bitrev.w $a1, $a1 -; LA32-NEXT: st.w $a1, $a0, 0 +; LA32-NEXT: st.w $a1, $a0, 12 +; LA32-NEXT: st.w $a4, $a0, 8 +; LA32-NEXT: st.w $a3, $a0, 4 +; LA32-NEXT: st.w $a2, $a0, 0 ; LA32-NEXT: ret ; ; LA64-LABEL: test_bitreverse_i128: diff --git a/llvm/test/CodeGen/LoongArch/branch-relaxation.ll b/llvm/test/CodeGen/LoongArch/branch-relaxation.ll index 7d064ddcf310..296f543e18d9 100644 --- a/llvm/test/CodeGen/LoongArch/branch-relaxation.ll +++ b/llvm/test/CodeGen/LoongArch/branch-relaxation.ll @@ -12,10 +12,10 @@ define i32 @relax_b18(i32 signext %a, i32 signext %b) { ; LA32-NEXT: beq $a0, $a1, .LBB0_1 ; LA32-NEXT: b .LBB0_2 ; LA32-NEXT: .LBB0_1: # %iftrue +; LA32-NEXT: ori $a0, $zero, 1 ; LA32-NEXT: #APP ; LA32-NEXT: .space 1048576 ; LA32-NEXT: #NO_APP -; LA32-NEXT: ori $a0, $zero, 1 ; LA32-NEXT: ret ; LA32-NEXT: .LBB0_2: # %iffalse ; LA32-NEXT: move $a0, $zero @@ -26,10 +26,10 @@ define i32 @relax_b18(i32 signext %a, i32 signext %b) { ; LA64-NEXT: beq $a0, $a1, .LBB0_1 ; LA64-NEXT: b .LBB0_2 ; LA64-NEXT: .LBB0_1: # %iftrue +; LA64-NEXT: ori $a0, $zero, 1 ; LA64-NEXT: #APP ; LA64-NEXT: .space 1048576 ; LA64-NEXT: #NO_APP -; LA64-NEXT: ori $a0, $zero, 1 ; LA64-NEXT: ret ; LA64-NEXT: .LBB0_2: # %iffalse ; LA64-NEXT: move $a0, $zero @@ -52,10 +52,10 @@ define i32 @relax_b23(i1 %a) { ; LA32-NEXT: bnez $a0, .LBB1_1 ; LA32-NEXT: b .LBB1_2 ; LA32-NEXT: .LBB1_1: # %iftrue +; LA32-NEXT: ori $a0, $zero, 1 ; LA32-NEXT: #APP ; LA32-NEXT: .space 16777216 ; LA32-NEXT: #NO_APP -; LA32-NEXT: ori $a0, $zero, 1 ; LA32-NEXT: ret ; LA32-NEXT: .LBB1_2: # %iffalse ; LA32-NEXT: move $a0, $zero @@ -67,10 +67,10 @@ define i32 @relax_b23(i1 %a) { ; LA64-NEXT: bnez $a0, .LBB1_1 ; LA64-NEXT: b .LBB1_2 ; LA64-NEXT: .LBB1_1: # %iftrue +; LA64-NEXT: ori $a0, $zero, 1 ; LA64-NEXT: #APP ; LA64-NEXT: .space 16777216 ; LA64-NEXT: #NO_APP -; LA64-NEXT: ori $a0, $zero, 1 ; LA64-NEXT: ret ; LA64-NEXT: .LBB1_2: # %iffalse ; LA64-NEXT: move $a0, $zero @@ -97,10 +97,10 @@ define i32 @relax_b28(i1 %a) { ; LA32-NEXT: addi.w $a0, $a0, %pc_lo12(.LBB2_2) ; LA32-NEXT: jr $a0 ; LA32-NEXT: .LBB2_1: # %iftrue +; LA32-NEXT: ori $a0, $zero, 1 ; LA32-NEXT: #APP ; LA32-NEXT: .space 536870912 ; LA32-NEXT: #NO_APP -; LA32-NEXT: ori $a0, $zero, 1 ; LA32-NEXT: addi.w $sp, $sp, 16 ; LA32-NEXT: ret ; LA32-NEXT: .LBB2_2: # %iffalse @@ -119,10 +119,10 @@ define i32 @relax_b28(i1 %a) { ; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(.LBB2_2) ; LA64-NEXT: jr $a0 ; LA64-NEXT: .LBB2_1: # %iftrue +; LA64-NEXT: ori $a0, $zero, 1 ; LA64-NEXT: #APP ; LA64-NEXT: .space 536870912 ; LA64-NEXT: #NO_APP -; LA64-NEXT: ori $a0, $zero, 1 ; LA64-NEXT: addi.d $sp, $sp, 16 ; LA64-NEXT: ret ; LA64-NEXT: .LBB2_2: # %iffalse diff --git a/llvm/test/CodeGen/LoongArch/bswap-bitreverse.ll b/llvm/test/CodeGen/LoongArch/bswap-bitreverse.ll index c99adfbb0574..828fb933bf3c 100644 --- a/llvm/test/CodeGen/LoongArch/bswap-bitreverse.ll +++ b/llvm/test/CodeGen/LoongArch/bswap-bitreverse.ll @@ -114,17 +114,17 @@ define i64 @test_bitreverse_bswap_i64(i64 %a) nounwind { define i32 @pr55484(i32 %0) { ; LA32-LABEL: pr55484: ; LA32: # %bb.0: -; LA32-NEXT: slli.w $a1, $a0, 8 -; LA32-NEXT: srli.w $a0, $a0, 8 -; LA32-NEXT: or $a0, $a0, $a1 +; LA32-NEXT: srli.w $a1, $a0, 8 +; LA32-NEXT: slli.w $a0, $a0, 8 +; LA32-NEXT: or $a0, $a1, $a0 ; LA32-NEXT: ext.w.h $a0, $a0 ; LA32-NEXT: ret ; ; LA64-LABEL: pr55484: ; LA64: # %bb.0: -; LA64-NEXT: slli.d $a1, $a0, 8 -; LA64-NEXT: srli.d $a0, $a0, 8 -; LA64-NEXT: or $a0, $a0, $a1 +; LA64-NEXT: srli.d $a1, $a0, 8 +; LA64-NEXT: slli.d $a0, $a0, 8 +; LA64-NEXT: or $a0, $a1, $a0 ; LA64-NEXT: ext.w.h $a0, $a0 ; LA64-NEXT: ret %2 = lshr i32 %0, 8 diff --git a/llvm/test/CodeGen/LoongArch/bswap.ll b/llvm/test/CodeGen/LoongArch/bswap.ll index eb9107302ef6..71095ab972e3 100644 --- a/llvm/test/CodeGen/LoongArch/bswap.ll +++ b/llvm/test/CodeGen/LoongArch/bswap.ll @@ -83,20 +83,20 @@ define i48 @test_bswap_i48(i48 %a) nounwind { define i80 @test_bswap_i80(i80 %a) nounwind { ; LA32-LABEL: test_bswap_i80: ; LA32: # %bb.0: -; LA32-NEXT: ld.w $a2, $a1, 0 +; LA32-NEXT: ld.w $a2, $a1, 4 +; LA32-NEXT: ld.w $a3, $a1, 8 +; LA32-NEXT: ld.w $a1, $a1, 0 ; LA32-NEXT: revb.2h $a2, $a2 ; LA32-NEXT: rotri.w $a2, $a2, 16 -; LA32-NEXT: ld.w $a3, $a1, 4 ; LA32-NEXT: revb.2h $a3, $a3 ; LA32-NEXT: rotri.w $a3, $a3, 16 -; LA32-NEXT: bytepick.w $a4, $a3, $a2, 2 -; LA32-NEXT: st.w $a4, $a0, 4 -; LA32-NEXT: ld.w $a1, $a1, 8 +; LA32-NEXT: bytepick.w $a3, $a3, $a2, 2 ; LA32-NEXT: revb.2h $a1, $a1 ; LA32-NEXT: rotri.w $a1, $a1, 16 -; LA32-NEXT: bytepick.w $a1, $a1, $a3, 2 -; LA32-NEXT: st.w $a1, $a0, 0 -; LA32-NEXT: srli.w $a1, $a2, 16 +; LA32-NEXT: bytepick.w $a2, $a2, $a1, 2 +; LA32-NEXT: srli.w $a1, $a1, 16 +; LA32-NEXT: st.w $a2, $a0, 4 +; LA32-NEXT: st.w $a3, $a0, 0 ; LA32-NEXT: st.h $a1, $a0, 8 ; LA32-NEXT: ret ; @@ -114,22 +114,22 @@ define i80 @test_bswap_i80(i80 %a) nounwind { define i128 @test_bswap_i128(i128 %a) nounwind { ; LA32-LABEL: test_bswap_i128: ; LA32: # %bb.0: -; LA32-NEXT: ld.w $a2, $a1, 0 -; LA32-NEXT: revb.2h $a2, $a2 -; LA32-NEXT: rotri.w $a2, $a2, 16 -; LA32-NEXT: st.w $a2, $a0, 12 -; LA32-NEXT: ld.w $a2, $a1, 4 +; LA32-NEXT: ld.w $a2, $a1, 12 +; LA32-NEXT: ld.w $a3, $a1, 0 +; LA32-NEXT: ld.w $a4, $a1, 8 +; LA32-NEXT: ld.w $a1, $a1, 4 ; LA32-NEXT: revb.2h $a2, $a2 ; LA32-NEXT: rotri.w $a2, $a2, 16 -; LA32-NEXT: st.w $a2, $a0, 8 -; LA32-NEXT: ld.w $a2, $a1, 8 -; LA32-NEXT: revb.2h $a2, $a2 -; LA32-NEXT: rotri.w $a2, $a2, 16 -; LA32-NEXT: st.w $a2, $a0, 4 -; LA32-NEXT: ld.w $a1, $a1, 12 +; LA32-NEXT: revb.2h $a4, $a4 +; LA32-NEXT: rotri.w $a4, $a4, 16 ; LA32-NEXT: revb.2h $a1, $a1 ; LA32-NEXT: rotri.w $a1, $a1, 16 -; LA32-NEXT: st.w $a1, $a0, 0 +; LA32-NEXT: revb.2h $a3, $a3 +; LA32-NEXT: rotri.w $a3, $a3, 16 +; LA32-NEXT: st.w $a3, $a0, 12 +; LA32-NEXT: st.w $a1, $a0, 8 +; LA32-NEXT: st.w $a4, $a0, 4 +; LA32-NEXT: st.w $a2, $a0, 0 ; LA32-NEXT: ret ; ; LA64-LABEL: test_bswap_i128: diff --git a/llvm/test/CodeGen/LoongArch/bytepick.ll b/llvm/test/CodeGen/LoongArch/bytepick.ll index 86148b374e70..1a2cd48448ba 100644 --- a/llvm/test/CodeGen/LoongArch/bytepick.ll +++ b/llvm/test/CodeGen/LoongArch/bytepick.ll @@ -14,8 +14,8 @@ define i32 @pick_i32_1(i32 %a, i32 %b) { ; ; LA64-LABEL: pick_i32_1: ; LA64: # %bb.0: -; LA64-NEXT: slli.d $a0, $a0, 8 ; LA64-NEXT: bstrpick.d $a1, $a1, 31, 24 +; LA64-NEXT: slli.d $a0, $a0, 8 ; LA64-NEXT: or $a0, $a1, $a0 ; LA64-NEXT: ret %1 = lshr i32 %b, 24 @@ -52,8 +52,8 @@ define i32 @pick_i32_2(i32 %a, i32 %b) { ; ; LA64-LABEL: pick_i32_2: ; LA64: # %bb.0: -; LA64-NEXT: slli.d $a0, $a0, 16 ; LA64-NEXT: bstrpick.d $a1, $a1, 31, 16 +; LA64-NEXT: slli.d $a0, $a0, 16 ; LA64-NEXT: or $a0, $a1, $a0 ; LA64-NEXT: ret %1 = lshr i32 %b, 16 @@ -90,8 +90,8 @@ define i32 @pick_i32_3(i32 %a, i32 %b) { ; ; LA64-LABEL: pick_i32_3: ; LA64: # %bb.0: -; LA64-NEXT: slli.d $a0, $a0, 24 ; LA64-NEXT: bstrpick.d $a1, $a1, 31, 8 +; LA64-NEXT: slli.d $a0, $a0, 24 ; LA64-NEXT: or $a0, $a1, $a0 ; LA64-NEXT: ret %1 = lshr i32 %b, 8 @@ -123,9 +123,8 @@ define signext i32 @pick_i32_3_sext(i32 %a, i32 %b) { define i64 @pick_i64_1(i64 %a, i64 %b) { ; LA32-LABEL: pick_i64_1: ; LA32: # %bb.0: -; LA32-NEXT: bytepick.w $a2, $a3, $a0, 1 ; LA32-NEXT: bytepick.w $a1, $a0, $a1, 1 -; LA32-NEXT: move $a0, $a2 +; LA32-NEXT: bytepick.w $a0, $a3, $a0, 1 ; LA32-NEXT: ret ; ; LA64-LABEL: pick_i64_1: @@ -143,9 +142,8 @@ define i64 @pick_i64_1(i64 %a, i64 %b) { define i64 @pick_i64_2(i64 %a, i64 %b) { ; LA32-LABEL: pick_i64_2: ; LA32: # %bb.0: -; LA32-NEXT: bytepick.w $a2, $a3, $a0, 2 ; LA32-NEXT: bytepick.w $a1, $a0, $a1, 2 -; LA32-NEXT: move $a0, $a2 +; LA32-NEXT: bytepick.w $a0, $a3, $a0, 2 ; LA32-NEXT: ret ; ; LA64-LABEL: pick_i64_2: @@ -163,9 +161,8 @@ define i64 @pick_i64_2(i64 %a, i64 %b) { define i64 @pick_i64_3(i64 %a, i64 %b) { ; LA32-LABEL: pick_i64_3: ; LA32: # %bb.0: -; LA32-NEXT: bytepick.w $a2, $a3, $a0, 3 ; LA32-NEXT: bytepick.w $a1, $a0, $a1, 3 -; LA32-NEXT: move $a0, $a2 +; LA32-NEXT: bytepick.w $a0, $a3, $a0, 3 ; LA32-NEXT: ret ; ; LA64-LABEL: pick_i64_3: diff --git a/llvm/test/CodeGen/LoongArch/calling-conv-common.ll b/llvm/test/CodeGen/LoongArch/calling-conv-common.ll index 08fff9f8ceed..06dfe00d9084 100644 --- a/llvm/test/CodeGen/LoongArch/calling-conv-common.ll +++ b/llvm/test/CodeGen/LoongArch/calling-conv-common.ll @@ -36,21 +36,21 @@ define i64 @caller_i128_in_regs() nounwind { define i64 @callee_many_scalars(i8 %a, i16 %b, i32 %c, i64 %d, i128 %e, i64 %f, i128 %g, i64 %h) nounwind { ; CHECK-LABEL: callee_many_scalars: ; CHECK: # %bb.0: -; CHECK-NEXT: ld.d $t0, $sp, 0 -; CHECK-NEXT: xor $a5, $a5, $t0 -; CHECK-NEXT: xor $a4, $a4, $a7 -; CHECK-NEXT: or $a4, $a4, $a5 -; CHECK-NEXT: bstrpick.d $a1, $a1, 15, 0 +; CHECK-NEXT: ld.d $t0, $sp, 8 +; CHECK-NEXT: ld.d $t1, $sp, 0 ; CHECK-NEXT: andi $a0, $a0, 255 +; CHECK-NEXT: bstrpick.d $a1, $a1, 15, 0 +; CHECK-NEXT: bstrpick.d $a2, $a2, 31, 0 ; CHECK-NEXT: add.d $a0, $a0, $a1 -; CHECK-NEXT: bstrpick.d $a1, $a2, 31, 0 -; CHECK-NEXT: add.d $a0, $a0, $a1 +; CHECK-NEXT: add.d $a0, $a0, $a2 ; CHECK-NEXT: add.d $a0, $a0, $a3 -; CHECK-NEXT: sltui $a1, $a4, 1 +; CHECK-NEXT: xor $a1, $a5, $t1 +; CHECK-NEXT: xor $a2, $a4, $a7 +; CHECK-NEXT: or $a1, $a2, $a1 +; CHECK-NEXT: sltui $a1, $a1, 1 ; CHECK-NEXT: add.d $a0, $a1, $a0 ; CHECK-NEXT: add.d $a0, $a0, $a6 -; CHECK-NEXT: ld.d $a1, $sp, 8 -; CHECK-NEXT: add.d $a0, $a0, $a1 +; CHECK-NEXT: add.d $a0, $a0, $t0 ; CHECK-NEXT: ret %a_ext = zext i8 %a to i64 %b_ext = zext i16 %b to i64 @@ -73,7 +73,6 @@ define i64 @caller_many_scalars() nounwind { ; CHECK-NEXT: st.d $ra, $sp, 24 # 8-byte Folded Spill ; CHECK-NEXT: ori $a0, $zero, 8 ; CHECK-NEXT: st.d $a0, $sp, 8 -; CHECK-NEXT: st.d $zero, $sp, 0 ; CHECK-NEXT: ori $a0, $zero, 1 ; CHECK-NEXT: ori $a1, $zero, 2 ; CHECK-NEXT: ori $a2, $zero, 3 @@ -81,6 +80,7 @@ define i64 @caller_many_scalars() nounwind { ; CHECK-NEXT: ori $a4, $zero, 5 ; CHECK-NEXT: ori $a6, $zero, 6 ; CHECK-NEXT: ori $a7, $zero, 7 +; CHECK-NEXT: st.d $zero, $sp, 0 ; CHECK-NEXT: move $a5, $zero ; CHECK-NEXT: bl %plt(callee_many_scalars) ; CHECK-NEXT: ld.d $ra, $sp, 24 # 8-byte Folded Reload @@ -95,21 +95,21 @@ define i64 @caller_many_scalars() nounwind { define i64 @callee_large_scalars(i256 %a, i256 %b) nounwind { ; CHECK-LABEL: callee_large_scalars: ; CHECK: # %bb.0: -; CHECK-NEXT: ld.d $a2, $a1, 24 -; CHECK-NEXT: ld.d $a3, $a0, 24 -; CHECK-NEXT: xor $a2, $a3, $a2 -; CHECK-NEXT: ld.d $a3, $a1, 8 -; CHECK-NEXT: ld.d $a4, $a0, 8 -; CHECK-NEXT: xor $a3, $a4, $a3 -; CHECK-NEXT: or $a2, $a3, $a2 -; CHECK-NEXT: ld.d $a3, $a1, 16 -; CHECK-NEXT: ld.d $a4, $a0, 16 -; CHECK-NEXT: xor $a3, $a4, $a3 -; CHECK-NEXT: ld.d $a1, $a1, 0 -; CHECK-NEXT: ld.d $a0, $a0, 0 +; CHECK-NEXT: ld.d $a2, $a1, 0 +; CHECK-NEXT: ld.d $a3, $a0, 0 +; CHECK-NEXT: ld.d $a4, $a1, 8 +; CHECK-NEXT: ld.d $a5, $a1, 24 +; CHECK-NEXT: ld.d $a6, $a0, 24 +; CHECK-NEXT: ld.d $a7, $a0, 8 +; CHECK-NEXT: ld.d $a1, $a1, 16 +; CHECK-NEXT: ld.d $a0, $a0, 16 +; CHECK-NEXT: xor $a5, $a6, $a5 +; CHECK-NEXT: xor $a4, $a7, $a4 +; CHECK-NEXT: or $a4, $a4, $a5 ; CHECK-NEXT: xor $a0, $a0, $a1 -; CHECK-NEXT: or $a0, $a0, $a3 -; CHECK-NEXT: or $a0, $a0, $a2 +; CHECK-NEXT: xor $a1, $a3, $a2 +; CHECK-NEXT: or $a0, $a1, $a0 +; CHECK-NEXT: or $a0, $a0, $a4 ; CHECK-NEXT: sltui $a0, $a0, 1 ; CHECK-NEXT: ret %1 = icmp eq i256 %a, %b @@ -122,18 +122,18 @@ define i64 @caller_large_scalars() nounwind { ; CHECK: # %bb.0: ; CHECK-NEXT: addi.d $sp, $sp, -80 ; CHECK-NEXT: st.d $ra, $sp, 72 # 8-byte Folded Spill -; CHECK-NEXT: ori $a0, $zero, 2 -; CHECK-NEXT: st.d $a0, $sp, 0 ; CHECK-NEXT: st.d $zero, $sp, 24 ; CHECK-NEXT: st.d $zero, $sp, 16 ; CHECK-NEXT: st.d $zero, $sp, 8 +; CHECK-NEXT: ori $a0, $zero, 2 +; CHECK-NEXT: st.d $a0, $sp, 0 ; CHECK-NEXT: st.d $zero, $sp, 56 ; CHECK-NEXT: st.d $zero, $sp, 48 ; CHECK-NEXT: st.d $zero, $sp, 40 -; CHECK-NEXT: ori $a0, $zero, 1 -; CHECK-NEXT: st.d $a0, $sp, 32 +; CHECK-NEXT: ori $a2, $zero, 1 ; CHECK-NEXT: addi.d $a0, $sp, 32 ; CHECK-NEXT: addi.d $a1, $sp, 0 +; CHECK-NEXT: st.d $a2, $sp, 32 ; CHECK-NEXT: bl %plt(callee_large_scalars) ; CHECK-NEXT: ld.d $ra, $sp, 72 # 8-byte Folded Reload ; CHECK-NEXT: addi.d $sp, $sp, 80 @@ -150,21 +150,21 @@ define i64 @callee_large_scalars_exhausted_regs(i64 %a, i64 %b, i64 %c, i64 %d, ; CHECK-LABEL: callee_large_scalars_exhausted_regs: ; CHECK: # %bb.0: ; CHECK-NEXT: ld.d $a0, $sp, 8 -; CHECK-NEXT: ld.d $a1, $a0, 24 -; CHECK-NEXT: ld.d $a2, $a7, 24 +; CHECK-NEXT: ld.d $a1, $a0, 0 +; CHECK-NEXT: ld.d $a2, $a7, 0 +; CHECK-NEXT: ld.d $a3, $a0, 8 +; CHECK-NEXT: ld.d $a4, $a0, 24 +; CHECK-NEXT: ld.d $a5, $a7, 24 +; CHECK-NEXT: ld.d $a6, $a7, 8 +; CHECK-NEXT: ld.d $a0, $a0, 16 +; CHECK-NEXT: ld.d $a7, $a7, 16 +; CHECK-NEXT: xor $a4, $a5, $a4 +; CHECK-NEXT: xor $a3, $a6, $a3 +; CHECK-NEXT: or $a3, $a3, $a4 +; CHECK-NEXT: xor $a0, $a7, $a0 ; CHECK-NEXT: xor $a1, $a2, $a1 -; CHECK-NEXT: ld.d $a2, $a0, 8 -; CHECK-NEXT: ld.d $a3, $a7, 8 -; CHECK-NEXT: xor $a2, $a3, $a2 -; CHECK-NEXT: or $a1, $a2, $a1 -; CHECK-NEXT: ld.d $a2, $a0, 16 -; CHECK-NEXT: ld.d $a3, $a7, 16 -; CHECK-NEXT: xor $a2, $a3, $a2 -; CHECK-NEXT: ld.d $a0, $a0, 0 -; CHECK-NEXT: ld.d $a3, $a7, 0 -; CHECK-NEXT: xor $a0, $a3, $a0 -; CHECK-NEXT: or $a0, $a0, $a2 -; CHECK-NEXT: or $a0, $a0, $a1 +; CHECK-NEXT: or $a0, $a1, $a0 +; CHECK-NEXT: or $a0, $a0, $a3 ; CHECK-NEXT: sltui $a0, $a0, 1 ; CHECK-NEXT: ret %1 = icmp eq i256 %h, %j @@ -181,16 +181,15 @@ define i64 @caller_large_scalars_exhausted_regs() nounwind { ; CHECK-NEXT: st.d $a0, $sp, 8 ; CHECK-NEXT: ori $a0, $zero, 9 ; CHECK-NEXT: st.d $a0, $sp, 0 -; CHECK-NEXT: ori $a0, $zero, 10 -; CHECK-NEXT: st.d $a0, $sp, 16 ; CHECK-NEXT: st.d $zero, $sp, 40 ; CHECK-NEXT: st.d $zero, $sp, 32 ; CHECK-NEXT: st.d $zero, $sp, 24 +; CHECK-NEXT: ori $a0, $zero, 10 +; CHECK-NEXT: st.d $a0, $sp, 16 ; CHECK-NEXT: st.d $zero, $sp, 72 ; CHECK-NEXT: st.d $zero, $sp, 64 ; CHECK-NEXT: st.d $zero, $sp, 56 -; CHECK-NEXT: ori $a0, $zero, 8 -; CHECK-NEXT: st.d $a0, $sp, 48 +; CHECK-NEXT: ori $t0, $zero, 8 ; CHECK-NEXT: ori $a0, $zero, 1 ; CHECK-NEXT: ori $a1, $zero, 2 ; CHECK-NEXT: ori $a2, $zero, 3 @@ -199,6 +198,7 @@ define i64 @caller_large_scalars_exhausted_regs() nounwind { ; CHECK-NEXT: ori $a5, $zero, 6 ; CHECK-NEXT: ori $a6, $zero, 7 ; CHECK-NEXT: addi.d $a7, $sp, 48 +; CHECK-NEXT: st.d $t0, $sp, 48 ; CHECK-NEXT: bl %plt(callee_large_scalars_exhausted_regs) ; CHECK-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload ; CHECK-NEXT: addi.d $sp, $sp, 96 @@ -216,9 +216,9 @@ define i64 @caller_large_scalars_exhausted_regs() nounwind { define i64 @callee_large_struct(ptr byval(%struct.large) align 8 %a) nounwind { ; CHECK-LABEL: callee_large_struct: ; CHECK: # %bb.0: -; CHECK-NEXT: ld.d $a1, $a0, 24 -; CHECK-NEXT: ld.d $a0, $a0, 0 -; CHECK-NEXT: add.d $a0, $a0, $a1 +; CHECK-NEXT: ld.d $a1, $a0, 0 +; CHECK-NEXT: ld.d $a0, $a0, 24 +; CHECK-NEXT: add.d $a0, $a1, $a0 ; CHECK-NEXT: ret %1 = getelementptr inbounds %struct.large, ptr %a, i64 0, i32 0 %2 = getelementptr inbounds %struct.large, ptr %a, i64 0, i32 3 @@ -235,16 +235,16 @@ define i64 @caller_large_struct() nounwind { ; CHECK-NEXT: st.d $ra, $sp, 72 # 8-byte Folded Spill ; CHECK-NEXT: ori $a0, $zero, 1 ; CHECK-NEXT: st.d $a0, $sp, 40 +; CHECK-NEXT: ori $a1, $zero, 2 +; CHECK-NEXT: st.d $a1, $sp, 48 +; CHECK-NEXT: ori $a2, $zero, 3 +; CHECK-NEXT: st.d $a2, $sp, 56 +; CHECK-NEXT: ori $a3, $zero, 4 +; CHECK-NEXT: st.d $a3, $sp, 64 ; CHECK-NEXT: st.d $a0, $sp, 8 -; CHECK-NEXT: ori $a0, $zero, 2 -; CHECK-NEXT: st.d $a0, $sp, 48 -; CHECK-NEXT: st.d $a0, $sp, 16 -; CHECK-NEXT: ori $a0, $zero, 3 -; CHECK-NEXT: st.d $a0, $sp, 56 -; CHECK-NEXT: st.d $a0, $sp, 24 -; CHECK-NEXT: ori $a0, $zero, 4 -; CHECK-NEXT: st.d $a0, $sp, 64 -; CHECK-NEXT: st.d $a0, $sp, 32 +; CHECK-NEXT: st.d $a1, $sp, 16 +; CHECK-NEXT: st.d $a2, $sp, 24 +; CHECK-NEXT: st.d $a3, $sp, 32 ; CHECK-NEXT: addi.d $a0, $sp, 8 ; CHECK-NEXT: bl %plt(callee_large_struct) ; CHECK-NEXT: ld.d $ra, $sp, 72 # 8-byte Folded Reload @@ -359,14 +359,14 @@ define void @caller_large_scalar_ret() nounwind { define void @callee_large_struct_ret(ptr noalias sret(%struct.large) %agg.result) nounwind { ; CHECK-LABEL: callee_large_struct_ret: ; CHECK: # %bb.0: -; CHECK-NEXT: ori $a1, $zero, 4 -; CHECK-NEXT: st.d $a1, $a0, 24 -; CHECK-NEXT: ori $a1, $zero, 3 -; CHECK-NEXT: st.d $a1, $a0, 16 -; CHECK-NEXT: ori $a1, $zero, 2 -; CHECK-NEXT: st.d $a1, $a0, 8 ; CHECK-NEXT: ori $a1, $zero, 1 ; CHECK-NEXT: st.d $a1, $a0, 0 +; CHECK-NEXT: ori $a1, $zero, 2 +; CHECK-NEXT: st.d $a1, $a0, 8 +; CHECK-NEXT: ori $a1, $zero, 3 +; CHECK-NEXT: st.d $a1, $a0, 16 +; CHECK-NEXT: ori $a1, $zero, 4 +; CHECK-NEXT: st.d $a1, $a0, 24 ; CHECK-NEXT: ret %a = getelementptr inbounds %struct.large, ptr %agg.result, i64 0, i32 0 store i64 1, ptr %a, align 4 @@ -386,9 +386,9 @@ define i64 @caller_large_struct_ret() nounwind { ; CHECK-NEXT: st.d $ra, $sp, 40 # 8-byte Folded Spill ; CHECK-NEXT: addi.d $a0, $sp, 8 ; CHECK-NEXT: bl %plt(callee_large_struct_ret) -; CHECK-NEXT: ld.d $a0, $sp, 32 -; CHECK-NEXT: ld.d $a1, $sp, 8 -; CHECK-NEXT: add.d $a0, $a1, $a0 +; CHECK-NEXT: ld.d $a0, $sp, 8 +; CHECK-NEXT: ld.d $a1, $sp, 32 +; CHECK-NEXT: add.d $a0, $a0, $a1 ; CHECK-NEXT: ld.d $ra, $sp, 40 # 8-byte Folded Reload ; CHECK-NEXT: addi.d $sp, $sp, 48 ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/LoongArch/calling-conv-lp64d.ll b/llvm/test/CodeGen/LoongArch/calling-conv-lp64d.ll index ceb38876c384..5e89fd41d6a7 100644 --- a/llvm/test/CodeGen/LoongArch/calling-conv-lp64d.ll +++ b/llvm/test/CodeGen/LoongArch/calling-conv-lp64d.ll @@ -11,10 +11,10 @@ define i64 @callee_float_in_fpr(i64 %a, float %b, double %c) nounwind { ; CHECK: # %bb.0: ; CHECK-NEXT: ftintrz.l.s $fa0, $fa0 ; CHECK-NEXT: movfr2gr.d $a1, $fa0 -; CHECK-NEXT: add.d $a0, $a0, $a1 ; CHECK-NEXT: ftintrz.l.d $fa0, $fa1 -; CHECK-NEXT: movfr2gr.d $a1, $fa0 +; CHECK-NEXT: movfr2gr.d $a2, $fa0 ; CHECK-NEXT: add.d $a0, $a0, $a1 +; CHECK-NEXT: add.d $a0, $a0, $a2 ; CHECK-NEXT: ret %b_fptosi = fptosi float %b to i64 %c_fptosi = fptosi double %c to i64 @@ -45,12 +45,12 @@ define i64 @caller_float_in_fpr() nounwind { define i64 @callee_double_in_gpr_exhausted_fprs(double %a, double %b, double %c, double %d, double %e, double %f, double %g, double %h, double %i) nounwind { ; CHECK-LABEL: callee_double_in_gpr_exhausted_fprs: ; CHECK: # %bb.0: -; CHECK-NEXT: ftintrz.l.d $fa0, $fa7 -; CHECK-NEXT: movfr2gr.d $a1, $fa0 ; CHECK-NEXT: movgr2fr.d $fa0, $a0 +; CHECK-NEXT: ftintrz.l.d $fa1, $fa7 +; CHECK-NEXT: movfr2gr.d $a0, $fa1 ; CHECK-NEXT: ftintrz.l.d $fa0, $fa0 -; CHECK-NEXT: movfr2gr.d $a0, $fa0 -; CHECK-NEXT: add.d $a0, $a1, $a0 +; CHECK-NEXT: movfr2gr.d $a1, $fa0 +; CHECK-NEXT: add.d $a0, $a0, $a1 ; CHECK-NEXT: ret %h_fptosi = fptosi double %h to i64 %i_fptosi = fptosi double %i to i64 diff --git a/llvm/test/CodeGen/LoongArch/calling-conv-lp64s.ll b/llvm/test/CodeGen/LoongArch/calling-conv-lp64s.ll index d738c066e1ad..f57743511496 100644 --- a/llvm/test/CodeGen/LoongArch/calling-conv-lp64s.ll +++ b/llvm/test/CodeGen/LoongArch/calling-conv-lp64s.ll @@ -54,12 +54,12 @@ define i64 @caller_float_on_stack() nounwind { ; CHECK: # %bb.0: ; CHECK-NEXT: addi.d $sp, $sp, -16 ; CHECK-NEXT: st.d $ra, $sp, 8 # 8-byte Folded Spill -; CHECK-NEXT: lu12i.w $a0, 264704 -; CHECK-NEXT: st.d $a0, $sp, 0 +; CHECK-NEXT: lu12i.w $a1, 264704 ; CHECK-NEXT: ori $a0, $zero, 1 ; CHECK-NEXT: ori $a2, $zero, 2 ; CHECK-NEXT: ori $a4, $zero, 3 ; CHECK-NEXT: ori $a6, $zero, 4 +; CHECK-NEXT: st.d $a1, $sp, 0 ; CHECK-NEXT: move $a1, $zero ; CHECK-NEXT: move $a3, $zero ; CHECK-NEXT: move $a5, $zero diff --git a/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll b/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll index af24ae64b7c7..a3d85f6bd06d 100644 --- a/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll +++ b/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll @@ -25,29 +25,29 @@ define dso_local noundef signext i32 @main() nounwind { ; CHECK-NEXT: addi.d $a0, $a0, %pc_lo12(.LCPI0_1) ; CHECK-NEXT: xvld $xr1, $a0, 0 ; CHECK-NEXT: xvst $xr1, $sp, 64 # 32-byte Folded Spill -; CHECK-NEXT: xvst $xr1, $sp, 224 -; CHECK-NEXT: xvst $xr0, $sp, 192 ; CHECK-NEXT: pcalau12i $a0, %pc_hi20(.LCPI0_2) ; CHECK-NEXT: addi.d $a0, $a0, %pc_lo12(.LCPI0_2) -; CHECK-NEXT: xvld $xr0, $a0, 0 -; CHECK-NEXT: xvst $xr0, $sp, 32 # 32-byte Folded Spill -; CHECK-NEXT: xvst $xr0, $sp, 160 +; CHECK-NEXT: xvld $xr2, $a0, 0 +; CHECK-NEXT: xvst $xr2, $sp, 32 # 32-byte Folded Spill ; CHECK-NEXT: pcalau12i $a0, %pc_hi20(.LCPI0_3) ; CHECK-NEXT: addi.d $a0, $a0, %pc_lo12(.LCPI0_3) -; CHECK-NEXT: xvld $xr0, $a0, 0 -; CHECK-NEXT: xvst $xr0, $sp, 0 # 32-byte Folded Spill +; CHECK-NEXT: xvld $xr3, $a0, 0 +; CHECK-NEXT: xvst $xr3, $sp, 0 # 32-byte Folded Spill ; CHECK-NEXT: xvst $xr0, $sp, 128 +; CHECK-NEXT: xvst $xr1, $sp, 160 +; CHECK-NEXT: xvst $xr2, $sp, 192 +; CHECK-NEXT: xvst $xr3, $sp, 224 ; CHECK-NEXT: addi.d $fp, $sp, 128 ; CHECK-NEXT: move $a0, $fp ; CHECK-NEXT: bl %plt(foo) -; CHECK-NEXT: xvld $xr0, $sp, 64 # 32-byte Folded Reload -; CHECK-NEXT: xvst $xr0, $sp, 224 ; CHECK-NEXT: xvld $xr0, $sp, 96 # 32-byte Folded Reload -; CHECK-NEXT: xvst $xr0, $sp, 192 -; CHECK-NEXT: xvld $xr0, $sp, 32 # 32-byte Folded Reload +; CHECK-NEXT: xvst $xr0, $sp, 128 +; CHECK-NEXT: xvld $xr0, $sp, 64 # 32-byte Folded Reload ; CHECK-NEXT: xvst $xr0, $sp, 160 +; CHECK-NEXT: xvld $xr0, $sp, 32 # 32-byte Folded Reload +; CHECK-NEXT: xvst $xr0, $sp, 192 ; CHECK-NEXT: xvld $xr0, $sp, 0 # 32-byte Folded Reload -; CHECK-NEXT: xvst $xr0, $sp, 128 +; CHECK-NEXT: xvst $xr0, $sp, 224 ; CHECK-NEXT: move $a0, $fp ; CHECK-NEXT: bl %plt(bar) ; CHECK-NEXT: move $a0, $zero diff --git a/llvm/test/CodeGen/LoongArch/cfr-pseudo-copy.mir b/llvm/test/CodeGen/LoongArch/cfr-pseudo-copy.mir index c5a6da72389f..b8b358eeadc1 100644 --- a/llvm/test/CodeGen/LoongArch/cfr-pseudo-copy.mir +++ b/llvm/test/CodeGen/LoongArch/cfr-pseudo-copy.mir @@ -17,8 +17,8 @@ body: | ; CHECK: liveins: $fcc0 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: $fcc1 = PseudoCopyCFR $fcc0 - ; CHECK-NEXT: $r4 = MOVCF2GR killed $fcc1 - ; CHECK-NEXT: PseudoRET implicit killed $r4 + ; CHECK-NEXT: $r4 = MOVCF2GR $fcc1 + ; CHECK-NEXT: PseudoRET implicit $r4 $fcc1 = COPY $fcc0 $r4 = COPY $fcc1 PseudoRET implicit killed $r4 diff --git a/llvm/test/CodeGen/LoongArch/ctlz-cttz-ctpop.ll b/llvm/test/CodeGen/LoongArch/ctlz-cttz-ctpop.ll index fa4fda9b8972..9fa3f5076bb2 100644 --- a/llvm/test/CodeGen/LoongArch/ctlz-cttz-ctpop.ll +++ b/llvm/test/CodeGen/LoongArch/ctlz-cttz-ctpop.ll @@ -201,10 +201,10 @@ define i8 @test_ctpop_i8(i8 %a) nounwind { define i16 @test_ctpop_i16(i16 %a) nounwind { ; LA32-LABEL: test_ctpop_i16: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a1, 5 -; LA32-NEXT: ori $a1, $a1, 1365 -; LA32-NEXT: srli.w $a2, $a0, 1 -; LA32-NEXT: and $a1, $a2, $a1 +; LA32-NEXT: srli.w $a1, $a0, 1 +; LA32-NEXT: lu12i.w $a2, 5 +; LA32-NEXT: ori $a2, $a2, 1365 +; LA32-NEXT: and $a1, $a1, $a2 ; LA32-NEXT: sub.w $a0, $a0, $a1 ; LA32-NEXT: lu12i.w $a1, 3 ; LA32-NEXT: ori $a1, $a1, 819 @@ -221,10 +221,10 @@ define i16 @test_ctpop_i16(i16 %a) nounwind { ; ; LA64-LABEL: test_ctpop_i16: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a1, 5 -; LA64-NEXT: ori $a1, $a1, 1365 -; LA64-NEXT: srli.d $a2, $a0, 1 -; LA64-NEXT: and $a1, $a2, $a1 +; LA64-NEXT: srli.d $a1, $a0, 1 +; LA64-NEXT: lu12i.w $a2, 5 +; LA64-NEXT: ori $a2, $a2, 1365 +; LA64-NEXT: and $a1, $a1, $a2 ; LA64-NEXT: sub.d $a0, $a0, $a1 ; LA64-NEXT: lu12i.w $a1, 3 ; LA64-NEXT: ori $a1, $a1, 819 @@ -245,10 +245,10 @@ define i16 @test_ctpop_i16(i16 %a) nounwind { define i32 @test_ctpop_i32(i32 %a) nounwind { ; LA32-LABEL: test_ctpop_i32: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a1, 349525 -; LA32-NEXT: ori $a1, $a1, 1365 -; LA32-NEXT: srli.w $a2, $a0, 1 -; LA32-NEXT: and $a1, $a2, $a1 +; LA32-NEXT: srli.w $a1, $a0, 1 +; LA32-NEXT: lu12i.w $a2, 349525 +; LA32-NEXT: ori $a2, $a2, 1365 +; LA32-NEXT: and $a1, $a1, $a2 ; LA32-NEXT: sub.w $a0, $a0, $a1 ; LA32-NEXT: lu12i.w $a1, 209715 ; LA32-NEXT: ori $a1, $a1, 819 @@ -269,10 +269,10 @@ define i32 @test_ctpop_i32(i32 %a) nounwind { ; ; LA64-LABEL: test_ctpop_i32: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a1, 349525 -; LA64-NEXT: ori $a1, $a1, 1365 -; LA64-NEXT: srli.d $a2, $a0, 1 -; LA64-NEXT: and $a1, $a2, $a1 +; LA64-NEXT: srli.d $a1, $a0, 1 +; LA64-NEXT: lu12i.w $a2, 349525 +; LA64-NEXT: ori $a2, $a2, 1365 +; LA64-NEXT: and $a1, $a1, $a2 ; LA64-NEXT: sub.d $a0, $a0, $a1 ; LA64-NEXT: lu12i.w $a1, 209715 ; LA64-NEXT: ori $a1, $a1, 819 @@ -297,37 +297,37 @@ define i32 @test_ctpop_i32(i32 %a) nounwind { define i64 @test_ctpop_i64(i64 %a) nounwind { ; LA32-LABEL: test_ctpop_i64: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 349525 -; LA32-NEXT: ori $a2, $a2, 1365 -; LA32-NEXT: srli.w $a3, $a0, 1 -; LA32-NEXT: and $a3, $a3, $a2 -; LA32-NEXT: sub.w $a0, $a0, $a3 -; LA32-NEXT: lu12i.w $a3, 209715 -; LA32-NEXT: ori $a3, $a3, 819 -; LA32-NEXT: and $a4, $a0, $a3 -; LA32-NEXT: srli.w $a0, $a0, 2 -; LA32-NEXT: and $a0, $a0, $a3 -; LA32-NEXT: add.w $a0, $a4, $a0 -; LA32-NEXT: srli.w $a4, $a1, 1 -; LA32-NEXT: and $a2, $a4, $a2 +; LA32-NEXT: srli.w $a2, $a1, 1 +; LA32-NEXT: lu12i.w $a3, 349525 +; LA32-NEXT: ori $a3, $a3, 1365 +; LA32-NEXT: and $a2, $a2, $a3 ; LA32-NEXT: sub.w $a1, $a1, $a2 -; LA32-NEXT: srli.w $a2, $a0, 4 -; LA32-NEXT: add.w $a0, $a0, $a2 -; LA32-NEXT: and $a2, $a1, $a3 +; LA32-NEXT: lu12i.w $a2, 209715 +; LA32-NEXT: ori $a2, $a2, 819 +; LA32-NEXT: and $a4, $a1, $a2 ; LA32-NEXT: srli.w $a1, $a1, 2 -; LA32-NEXT: and $a1, $a1, $a3 -; LA32-NEXT: add.w $a1, $a2, $a1 -; LA32-NEXT: srli.w $a2, $a1, 4 -; LA32-NEXT: add.w $a1, $a1, $a2 -; LA32-NEXT: lu12i.w $a2, 61680 -; LA32-NEXT: ori $a2, $a2, 3855 ; LA32-NEXT: and $a1, $a1, $a2 -; LA32-NEXT: and $a0, $a0, $a2 -; LA32-NEXT: lu12i.w $a2, 4112 -; LA32-NEXT: ori $a2, $a2, 257 -; LA32-NEXT: mul.w $a0, $a0, $a2 -; LA32-NEXT: mul.w $a1, $a1, $a2 +; LA32-NEXT: add.w $a1, $a4, $a1 +; LA32-NEXT: srli.w $a4, $a1, 4 +; LA32-NEXT: add.w $a1, $a1, $a4 +; LA32-NEXT: lu12i.w $a4, 61680 +; LA32-NEXT: ori $a4, $a4, 3855 +; LA32-NEXT: and $a1, $a1, $a4 +; LA32-NEXT: lu12i.w $a5, 4112 +; LA32-NEXT: ori $a5, $a5, 257 +; LA32-NEXT: mul.w $a1, $a1, $a5 ; LA32-NEXT: srli.w $a1, $a1, 24 +; LA32-NEXT: srli.w $a6, $a0, 1 +; LA32-NEXT: and $a3, $a6, $a3 +; LA32-NEXT: sub.w $a0, $a0, $a3 +; LA32-NEXT: and $a3, $a0, $a2 +; LA32-NEXT: srli.w $a0, $a0, 2 +; LA32-NEXT: and $a0, $a0, $a2 +; LA32-NEXT: add.w $a0, $a3, $a0 +; LA32-NEXT: srli.w $a2, $a0, 4 +; LA32-NEXT: add.w $a0, $a0, $a2 +; LA32-NEXT: and $a0, $a0, $a4 +; LA32-NEXT: mul.w $a0, $a0, $a5 ; LA32-NEXT: srli.w $a0, $a0, 24 ; LA32-NEXT: add.w $a0, $a0, $a1 ; LA32-NEXT: move $a1, $zero @@ -335,12 +335,12 @@ define i64 @test_ctpop_i64(i64 %a) nounwind { ; ; LA64-LABEL: test_ctpop_i64: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a1, 349525 -; LA64-NEXT: ori $a1, $a1, 1365 -; LA64-NEXT: lu32i.d $a1, 349525 -; LA64-NEXT: lu52i.d $a1, $a1, 1365 -; LA64-NEXT: srli.d $a2, $a0, 1 -; LA64-NEXT: and $a1, $a2, $a1 +; LA64-NEXT: srli.d $a1, $a0, 1 +; LA64-NEXT: lu12i.w $a2, 349525 +; LA64-NEXT: ori $a2, $a2, 1365 +; LA64-NEXT: lu32i.d $a2, 349525 +; LA64-NEXT: lu52i.d $a2, $a2, 1365 +; LA64-NEXT: and $a1, $a1, $a2 ; LA64-NEXT: sub.d $a0, $a0, $a1 ; LA64-NEXT: lu12i.w $a1, 209715 ; LA64-NEXT: ori $a1, $a1, 819 diff --git a/llvm/test/CodeGen/LoongArch/fcopysign.ll b/llvm/test/CodeGen/LoongArch/fcopysign.ll index 181130d2c6a5..49e8fbca3e12 100644 --- a/llvm/test/CodeGen/LoongArch/fcopysign.ll +++ b/llvm/test/CodeGen/LoongArch/fcopysign.ll @@ -73,10 +73,10 @@ define double @fold_promote_d_s(double %a, float %b) nounwind { ; ; LA64F-LABEL: fold_promote_d_s: ; LA64F: # %bb.0: -; LA64F-NEXT: lu12i.w $a1, -524288 -; LA64F-NEXT: lu32i.d $a1, 0 -; LA64F-NEXT: movfr2gr.s $a2, $fa0 -; LA64F-NEXT: and $a1, $a2, $a1 +; LA64F-NEXT: movfr2gr.s $a1, $fa0 +; LA64F-NEXT: lu12i.w $a2, -524288 +; LA64F-NEXT: lu32i.d $a2, 0 +; LA64F-NEXT: and $a1, $a1, $a2 ; LA64F-NEXT: slli.d $a1, $a1, 32 ; LA64F-NEXT: bstrins.d $a1, $a0, 62, 0 ; LA64F-NEXT: move $a0, $a1 diff --git a/llvm/test/CodeGen/LoongArch/get-setcc-result-type.ll b/llvm/test/CodeGen/LoongArch/get-setcc-result-type.ll index 432cedff6d83..5e4c8418b222 100644 --- a/llvm/test/CodeGen/LoongArch/get-setcc-result-type.ll +++ b/llvm/test/CodeGen/LoongArch/get-setcc-result-type.ll @@ -5,21 +5,21 @@ define void @getSetCCResultType(ptr %p) { ; CHECK-LABEL: getSetCCResultType: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: ld.w $a1, $a0, 12 -; CHECK-NEXT: sltui $a1, $a1, 1 -; CHECK-NEXT: sub.d $a1, $zero, $a1 -; CHECK-NEXT: st.w $a1, $a0, 12 -; CHECK-NEXT: ld.w $a1, $a0, 8 -; CHECK-NEXT: sltui $a1, $a1, 1 -; CHECK-NEXT: sub.d $a1, $zero, $a1 -; CHECK-NEXT: st.w $a1, $a0, 8 -; CHECK-NEXT: ld.w $a1, $a0, 4 -; CHECK-NEXT: sltui $a1, $a1, 1 -; CHECK-NEXT: sub.d $a1, $zero, $a1 -; CHECK-NEXT: st.w $a1, $a0, 4 ; CHECK-NEXT: ld.w $a1, $a0, 0 +; CHECK-NEXT: ld.w $a2, $a0, 12 +; CHECK-NEXT: ld.w $a3, $a0, 4 +; CHECK-NEXT: ld.w $a4, $a0, 8 ; CHECK-NEXT: sltui $a1, $a1, 1 ; CHECK-NEXT: sub.d $a1, $zero, $a1 +; CHECK-NEXT: sltui $a3, $a3, 1 +; CHECK-NEXT: sub.d $a3, $zero, $a3 +; CHECK-NEXT: sltui $a4, $a4, 1 +; CHECK-NEXT: sub.d $a4, $zero, $a4 +; CHECK-NEXT: sltui $a2, $a2, 1 +; CHECK-NEXT: sub.d $a2, $zero, $a2 +; CHECK-NEXT: st.w $a2, $a0, 12 +; CHECK-NEXT: st.w $a4, $a0, 8 +; CHECK-NEXT: st.w $a3, $a0, 4 ; CHECK-NEXT: st.w $a1, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/ghc-cc.ll b/llvm/test/CodeGen/LoongArch/ghc-cc.ll index 0ab125e875b9..735315d323a3 100644 --- a/llvm/test/CodeGen/LoongArch/ghc-cc.ll +++ b/llvm/test/CodeGen/LoongArch/ghc-cc.ll @@ -26,57 +26,57 @@ define ghccc void @foo() nounwind { ; LA64-LABEL: foo: ; LA64: # %bb.0: # %entry -; LA64-NEXT: pcalau12i $a0, %pc_hi20(base) -; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(base) -; LA64-NEXT: ld.d $s0, $a0, 0 -; LA64-NEXT: pcalau12i $a0, %pc_hi20(sp) -; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(sp) -; LA64-NEXT: ld.d $s1, $a0, 0 -; LA64-NEXT: pcalau12i $a0, %pc_hi20(hp) -; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(hp) -; LA64-NEXT: ld.d $s2, $a0, 0 -; LA64-NEXT: pcalau12i $a0, %pc_hi20(r1) -; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(r1) -; LA64-NEXT: ld.d $s3, $a0, 0 -; LA64-NEXT: pcalau12i $a0, %pc_hi20(r2) -; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(r2) -; LA64-NEXT: ld.d $s4, $a0, 0 -; LA64-NEXT: pcalau12i $a0, %pc_hi20(r3) -; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(r3) -; LA64-NEXT: ld.d $s5, $a0, 0 -; LA64-NEXT: pcalau12i $a0, %pc_hi20(r4) -; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(r4) -; LA64-NEXT: ld.d $s6, $a0, 0 -; LA64-NEXT: pcalau12i $a0, %pc_hi20(r5) -; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(r5) -; LA64-NEXT: ld.d $s7, $a0, 0 -; LA64-NEXT: pcalau12i $a0, %pc_hi20(splim) -; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(splim) -; LA64-NEXT: ld.d $s8, $a0, 0 -; LA64-NEXT: pcalau12i $a0, %pc_hi20(f1) -; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(f1) -; LA64-NEXT: fld.s $fs0, $a0, 0 -; LA64-NEXT: pcalau12i $a0, %pc_hi20(f2) -; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(f2) -; LA64-NEXT: fld.s $fs1, $a0, 0 -; LA64-NEXT: pcalau12i $a0, %pc_hi20(f3) -; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(f3) -; LA64-NEXT: fld.s $fs2, $a0, 0 -; LA64-NEXT: pcalau12i $a0, %pc_hi20(f4) -; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(f4) -; LA64-NEXT: fld.s $fs3, $a0, 0 -; LA64-NEXT: pcalau12i $a0, %pc_hi20(d1) -; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(d1) -; LA64-NEXT: fld.d $fs4, $a0, 0 -; LA64-NEXT: pcalau12i $a0, %pc_hi20(d2) -; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(d2) -; LA64-NEXT: fld.d $fs5, $a0, 0 -; LA64-NEXT: pcalau12i $a0, %pc_hi20(d3) -; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(d3) -; LA64-NEXT: fld.d $fs6, $a0, 0 ; LA64-NEXT: pcalau12i $a0, %pc_hi20(d4) ; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(d4) ; LA64-NEXT: fld.d $fs7, $a0, 0 +; LA64-NEXT: pcalau12i $a0, %pc_hi20(d3) +; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(d3) +; LA64-NEXT: fld.d $fs6, $a0, 0 +; LA64-NEXT: pcalau12i $a0, %pc_hi20(d2) +; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(d2) +; LA64-NEXT: fld.d $fs5, $a0, 0 +; LA64-NEXT: pcalau12i $a0, %pc_hi20(d1) +; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(d1) +; LA64-NEXT: fld.d $fs4, $a0, 0 +; LA64-NEXT: pcalau12i $a0, %pc_hi20(f4) +; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(f4) +; LA64-NEXT: fld.s $fs3, $a0, 0 +; LA64-NEXT: pcalau12i $a0, %pc_hi20(f3) +; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(f3) +; LA64-NEXT: fld.s $fs2, $a0, 0 +; LA64-NEXT: pcalau12i $a0, %pc_hi20(f2) +; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(f2) +; LA64-NEXT: fld.s $fs1, $a0, 0 +; LA64-NEXT: pcalau12i $a0, %pc_hi20(f1) +; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(f1) +; LA64-NEXT: fld.s $fs0, $a0, 0 +; LA64-NEXT: pcalau12i $a0, %pc_hi20(splim) +; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(splim) +; LA64-NEXT: ld.d $s8, $a0, 0 +; LA64-NEXT: pcalau12i $a0, %pc_hi20(r5) +; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(r5) +; LA64-NEXT: ld.d $s7, $a0, 0 +; LA64-NEXT: pcalau12i $a0, %pc_hi20(r4) +; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(r4) +; LA64-NEXT: ld.d $s6, $a0, 0 +; LA64-NEXT: pcalau12i $a0, %pc_hi20(r3) +; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(r3) +; LA64-NEXT: ld.d $s5, $a0, 0 +; LA64-NEXT: pcalau12i $a0, %pc_hi20(r2) +; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(r2) +; LA64-NEXT: ld.d $s4, $a0, 0 +; LA64-NEXT: pcalau12i $a0, %pc_hi20(r1) +; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(r1) +; LA64-NEXT: ld.d $s3, $a0, 0 +; LA64-NEXT: pcalau12i $a0, %pc_hi20(hp) +; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(hp) +; LA64-NEXT: ld.d $s2, $a0, 0 +; LA64-NEXT: pcalau12i $a0, %pc_hi20(sp) +; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(sp) +; LA64-NEXT: ld.d $s1, $a0, 0 +; LA64-NEXT: pcalau12i $a0, %pc_hi20(base) +; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(base) +; LA64-NEXT: ld.d $s0, $a0, 0 ; LA64-NEXT: b %plt(bar) entry: diff --git a/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll b/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll index ff845cb1f3db..06ef4d2f6c15 100644 --- a/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll +++ b/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll @@ -13,17 +13,17 @@ define void @box(ptr noalias nocapture noundef writeonly sret(%Box) align 16 der ; CHECK-NEXT: addi.d $a2, $sp, 0 ; CHECK-NEXT: add.d $a3, $a2, $a1 ; CHECK-NEXT: ldx.d $a1, $a1, $a2 +; CHECK-NEXT: ld.d $a2, $a3, 40 ; CHECK-NEXT: st.d $a1, $a0, 0 -; CHECK-NEXT: ld.d $a1, $a3, 40 -; CHECK-NEXT: st.d $a1, $a0, 40 +; CHECK-NEXT: st.d $a2, $a0, 40 ; CHECK-NEXT: ld.d $a1, $a3, 32 +; CHECK-NEXT: ld.d $a2, $a3, 24 +; CHECK-NEXT: ld.d $a4, $a3, 16 +; CHECK-NEXT: ld.d $a3, $a3, 8 ; CHECK-NEXT: st.d $a1, $a0, 32 -; CHECK-NEXT: ld.d $a1, $a3, 24 -; CHECK-NEXT: st.d $a1, $a0, 24 -; CHECK-NEXT: ld.d $a1, $a3, 16 -; CHECK-NEXT: st.d $a1, $a0, 16 -; CHECK-NEXT: ld.d $a1, $a3, 8 -; CHECK-NEXT: st.d $a1, $a0, 8 +; CHECK-NEXT: st.d $a2, $a0, 24 +; CHECK-NEXT: st.d $a4, $a0, 16 +; CHECK-NEXT: st.d $a3, $a0, 8 ; CHECK-NEXT: addi.d $sp, $sp, 96 ; CHECK-NEXT: ret %1 = alloca [2 x %Box], align 16 diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/and.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/and.ll index b43ed7859cde..b3e32cc5c00c 100644 --- a/llvm/test/CodeGen/LoongArch/ir-instruction/and.ll +++ b/llvm/test/CodeGen/LoongArch/ir-instruction/and.ll @@ -285,19 +285,19 @@ define signext i32 @and_i32_0xfff0(i32 %a) { define signext i32 @and_i32_0xfff0_twice(i32 %a, i32 %b) { ; LA32-LABEL: and_i32_0xfff0_twice: ; LA32: # %bb.0: -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 4 -; LA32-NEXT: slli.w $a1, $a1, 4 ; LA32-NEXT: bstrpick.w $a0, $a0, 15, 4 ; LA32-NEXT: slli.w $a0, $a0, 4 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 4 +; LA32-NEXT: slli.w $a1, $a1, 4 ; LA32-NEXT: sub.w $a0, $a0, $a1 ; LA32-NEXT: ret ; ; LA64-LABEL: and_i32_0xfff0_twice: ; LA64: # %bb.0: -; LA64-NEXT: bstrpick.d $a1, $a1, 15, 4 -; LA64-NEXT: slli.d $a1, $a1, 4 ; LA64-NEXT: bstrpick.d $a0, $a0, 15, 4 ; LA64-NEXT: slli.d $a0, $a0, 4 +; LA64-NEXT: bstrpick.d $a1, $a1, 15, 4 +; LA64-NEXT: slli.d $a1, $a1, 4 ; LA64-NEXT: sub.d $a0, $a0, $a1 ; LA64-NEXT: ret %c = and i32 %a, 65520 @@ -326,21 +326,21 @@ define i64 @and_i64_0xfff0(i64 %a) { define i64 @and_i64_0xfff0_twice(i64 %a, i64 %b) { ; LA32-LABEL: and_i64_0xfff0_twice: ; LA32: # %bb.0: -; LA32-NEXT: bstrpick.w $a1, $a2, 15, 4 -; LA32-NEXT: slli.w $a1, $a1, 4 ; LA32-NEXT: bstrpick.w $a0, $a0, 15, 4 +; LA32-NEXT: slli.w $a1, $a0, 4 +; LA32-NEXT: bstrpick.w $a0, $a2, 15, 4 ; LA32-NEXT: slli.w $a2, $a0, 4 -; LA32-NEXT: sub.w $a0, $a2, $a1 -; LA32-NEXT: sltu $a1, $a2, $a1 +; LA32-NEXT: sub.w $a0, $a1, $a2 +; LA32-NEXT: sltu $a1, $a1, $a2 ; LA32-NEXT: sub.w $a1, $zero, $a1 ; LA32-NEXT: ret ; ; LA64-LABEL: and_i64_0xfff0_twice: ; LA64: # %bb.0: -; LA64-NEXT: bstrpick.d $a1, $a1, 15, 4 -; LA64-NEXT: slli.d $a1, $a1, 4 ; LA64-NEXT: bstrpick.d $a0, $a0, 15, 4 ; LA64-NEXT: slli.d $a0, $a0, 4 +; LA64-NEXT: bstrpick.d $a1, $a1, 15, 4 +; LA64-NEXT: slli.d $a1, $a1, 4 ; LA64-NEXT: sub.d $a0, $a0, $a1 ; LA64-NEXT: ret %c = and i64 %a, 65520 @@ -390,14 +390,14 @@ define i64 @and_i64_0xfff0_multiple_times(i64 %a, i64 %b, i64 %c) { ; LA32: # %bb.0: ; LA32-NEXT: lu12i.w $a1, 15 ; LA32-NEXT: ori $a1, $a1, 4080 -; LA32-NEXT: and $a3, $a0, $a1 -; LA32-NEXT: and $a0, $a4, $a1 -; LA32-NEXT: and $a1, $a2, $a1 -; LA32-NEXT: mul.w $a0, $a1, $a0 -; LA32-NEXT: sub.w $a2, $a3, $a1 -; LA32-NEXT: xor $a0, $a2, $a0 -; LA32-NEXT: sltu $a1, $a3, $a1 +; LA32-NEXT: and $a0, $a0, $a1 +; LA32-NEXT: and $a2, $a2, $a1 +; LA32-NEXT: and $a3, $a4, $a1 +; LA32-NEXT: sltu $a1, $a0, $a2 ; LA32-NEXT: sub.w $a1, $zero, $a1 +; LA32-NEXT: sub.w $a0, $a0, $a2 +; LA32-NEXT: mul.w $a2, $a2, $a3 +; LA32-NEXT: xor $a0, $a0, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: and_i64_0xfff0_multiple_times: @@ -405,11 +405,11 @@ define i64 @and_i64_0xfff0_multiple_times(i64 %a, i64 %b, i64 %c) { ; LA64-NEXT: lu12i.w $a3, 15 ; LA64-NEXT: ori $a3, $a3, 4080 ; LA64-NEXT: and $a0, $a0, $a3 -; LA64-NEXT: and $a2, $a2, $a3 ; LA64-NEXT: and $a1, $a1, $a3 -; LA64-NEXT: mul.d $a2, $a1, $a2 +; LA64-NEXT: and $a2, $a2, $a3 ; LA64-NEXT: sub.d $a0, $a0, $a1 -; LA64-NEXT: xor $a0, $a0, $a2 +; LA64-NEXT: mul.d $a1, $a1, $a2 +; LA64-NEXT: xor $a0, $a0, $a1 ; LA64-NEXT: ret %d = and i64 %a, 65520 %e = and i64 %b, 65520 diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/ashr.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/ashr.ll index 0d8e7127d0df..d4f766d460d1 100644 --- a/llvm/test/CodeGen/LoongArch/ir-instruction/ashr.ll +++ b/llvm/test/CodeGen/LoongArch/ir-instruction/ashr.ll @@ -77,8 +77,8 @@ define i64 @ashr_i64(i64 %x, i64 %y) { ; LA32-NEXT: slli.w $a6, $a1, 1 ; LA32-NEXT: sll.w $a2, $a6, $a2 ; LA32-NEXT: or $a0, $a0, $a2 -; LA32-NEXT: sra.w $a1, $a1, $a4 ; LA32-NEXT: maskeqz $a0, $a0, $a5 +; LA32-NEXT: sra.w $a1, $a1, $a4 ; LA32-NEXT: masknez $a1, $a1, $a5 ; LA32-NEXT: or $a0, $a0, $a1 ; LA32-NEXT: move $a1, $a3 @@ -154,8 +154,8 @@ define i32 @ashr_i32_3(i32 %x) { define i64 @ashr_i64_3(i64 %x) { ; LA32-LABEL: ashr_i64_3: ; LA32: # %bb.0: -; LA32-NEXT: srli.w $a0, $a0, 3 ; LA32-NEXT: slli.w $a2, $a1, 29 +; LA32-NEXT: srli.w $a0, $a0, 3 ; LA32-NEXT: or $a0, $a0, $a2 ; LA32-NEXT: srai.w $a1, $a1, 3 ; LA32-NEXT: ret diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/atomic-cmpxchg.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/atomic-cmpxchg.ll index 31ecec6ea805..06ad89972b84 100644 --- a/llvm/test/CodeGen/LoongArch/ir-instruction/atomic-cmpxchg.ll +++ b/llvm/test/CodeGen/LoongArch/ir-instruction/atomic-cmpxchg.ll @@ -4,17 +4,17 @@ define void @cmpxchg_i8_acquire_acquire(ptr %ptr, i8 %cmp, i8 %val) nounwind { ; LA64-LABEL: cmpxchg_i8_acquire_acquire: ; LA64: # %bb.0: -; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: slli.d $a3, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: ori $a4, $zero, 255 +; LA64-NEXT: sll.w $a4, $a4, $a3 +; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a3 ; LA64-NEXT: andi $a2, $a2, 255 ; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: ori $a4, $zero, 255 -; LA64-NEXT: sll.w $a3, $a4, $a3 -; LA64-NEXT: addi.w $a3, $a3, 0 -; LA64-NEXT: addi.w $a2, $a2, 0 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: addi.w $a3, $a4, 0 ; LA64-NEXT: .LBB0_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a5, $a4, $a3 @@ -36,18 +36,18 @@ define void @cmpxchg_i8_acquire_acquire(ptr %ptr, i8 %cmp, i8 %val) nounwind { define void @cmpxchg_i16_acquire_acquire(ptr %ptr, i16 %cmp, i16 %val) nounwind { ; LA64-LABEL: cmpxchg_i16_acquire_acquire: ; LA64: # %bb.0: -; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 ; LA64-NEXT: slli.d $a3, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a4, 15 +; LA64-NEXT: ori $a4, $a4, 4095 +; LA64-NEXT: sll.w $a4, $a4, $a3 +; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 ; LA64-NEXT: sll.w $a1, $a1, $a3 ; LA64-NEXT: bstrpick.d $a2, $a2, 15, 0 ; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: lu12i.w $a4, 15 -; LA64-NEXT: ori $a4, $a4, 4095 -; LA64-NEXT: sll.w $a3, $a4, $a3 -; LA64-NEXT: addi.w $a3, $a3, 0 -; LA64-NEXT: addi.w $a2, $a2, 0 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: addi.w $a3, $a4, 0 ; LA64-NEXT: .LBB1_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a5, $a4, $a3 @@ -108,17 +108,17 @@ define void @cmpxchg_i64_acquire_acquire(ptr %ptr, i64 %cmp, i64 %val) nounwind define void @cmpxchg_i8_acquire_monotonic(ptr %ptr, i8 %cmp, i8 %val) nounwind { ; LA64-LABEL: cmpxchg_i8_acquire_monotonic: ; LA64: # %bb.0: -; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: slli.d $a3, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: ori $a4, $zero, 255 +; LA64-NEXT: sll.w $a4, $a4, $a3 +; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a3 ; LA64-NEXT: andi $a2, $a2, 255 ; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: ori $a4, $zero, 255 -; LA64-NEXT: sll.w $a3, $a4, $a3 -; LA64-NEXT: addi.w $a3, $a3, 0 -; LA64-NEXT: addi.w $a2, $a2, 0 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: addi.w $a3, $a4, 0 ; LA64-NEXT: .LBB4_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a5, $a4, $a3 @@ -140,18 +140,18 @@ define void @cmpxchg_i8_acquire_monotonic(ptr %ptr, i8 %cmp, i8 %val) nounwind { define void @cmpxchg_i16_acquire_monotonic(ptr %ptr, i16 %cmp, i16 %val) nounwind { ; LA64-LABEL: cmpxchg_i16_acquire_monotonic: ; LA64: # %bb.0: -; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 ; LA64-NEXT: slli.d $a3, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a4, 15 +; LA64-NEXT: ori $a4, $a4, 4095 +; LA64-NEXT: sll.w $a4, $a4, $a3 +; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 ; LA64-NEXT: sll.w $a1, $a1, $a3 ; LA64-NEXT: bstrpick.d $a2, $a2, 15, 0 ; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: lu12i.w $a4, 15 -; LA64-NEXT: ori $a4, $a4, 4095 -; LA64-NEXT: sll.w $a3, $a4, $a3 -; LA64-NEXT: addi.w $a3, $a3, 0 -; LA64-NEXT: addi.w $a2, $a2, 0 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: addi.w $a3, $a4, 0 ; LA64-NEXT: .LBB5_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a5, $a4, $a3 @@ -213,16 +213,16 @@ define i8 @cmpxchg_i8_acquire_acquire_reti8(ptr %ptr, i8 %cmp, i8 %val) nounwind ; LA64-LABEL: cmpxchg_i8_acquire_acquire_reti8: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a3, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a4, $zero, 255 ; LA64-NEXT: sll.w $a4, $a4, $a3 -; LA64-NEXT: andi $a2, $a2, 255 -; LA64-NEXT: addi.w $a4, $a4, 0 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: andi $a2, $a2, 255 +; LA64-NEXT: sll.w $a2, $a2, $a3 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: addi.w $a4, $a4, 0 ; LA64-NEXT: .LBB8_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 ; LA64-NEXT: and $a6, $a5, $a4 @@ -246,24 +246,24 @@ define i8 @cmpxchg_i8_acquire_acquire_reti8(ptr %ptr, i8 %cmp, i8 %val) nounwind define i16 @cmpxchg_i16_acquire_acquire_reti16(ptr %ptr, i16 %cmp, i16 %val) nounwind { ; LA64-LABEL: cmpxchg_i16_acquire_acquire_reti16: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a3, 15 -; LA64-NEXT: ori $a3, $a3, 4095 -; LA64-NEXT: slli.d $a4, $a0, 3 -; LA64-NEXT: sll.w $a3, $a3, $a4 -; LA64-NEXT: bstrpick.d $a2, $a2, 15, 0 -; LA64-NEXT: addi.w $a3, $a3, 0 -; LA64-NEXT: sll.w $a2, $a2, $a4 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a3, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a4, 15 +; LA64-NEXT: ori $a4, $a4, 4095 +; LA64-NEXT: sll.w $a4, $a4, $a3 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a4 +; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: bstrpick.d $a2, $a2, 15, 0 +; LA64-NEXT: sll.w $a2, $a2, $a3 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: addi.w $a4, $a4, 0 ; LA64-NEXT: .LBB9_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 -; LA64-NEXT: and $a6, $a5, $a3 +; LA64-NEXT: and $a6, $a5, $a4 ; LA64-NEXT: bne $a6, $a1, .LBB9_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB9_1 Depth=1 -; LA64-NEXT: andn $a6, $a5, $a3 +; LA64-NEXT: andn $a6, $a5, $a4 ; LA64-NEXT: or $a6, $a6, $a2 ; LA64-NEXT: sc.w $a6, $a0, 0 ; LA64-NEXT: beqz $a6, .LBB9_1 @@ -271,7 +271,7 @@ define i16 @cmpxchg_i16_acquire_acquire_reti16(ptr %ptr, i16 %cmp, i16 %val) nou ; LA64-NEXT: .LBB9_3: ; LA64-NEXT: dbar 20 ; LA64-NEXT: .LBB9_4: -; LA64-NEXT: srl.w $a0, $a5, $a4 +; LA64-NEXT: srl.w $a0, $a5, $a3 ; LA64-NEXT: ret %tmp = cmpxchg ptr %ptr, i16 %cmp, i16 %val acquire acquire %res = extractvalue { i16, i1 } %tmp, 0 @@ -324,16 +324,16 @@ define i64 @cmpxchg_i64_acquire_acquire_reti64(ptr %ptr, i64 %cmp, i64 %val) nou define i1 @cmpxchg_i8_acquire_acquire_reti1(ptr %ptr, i8 %cmp, i8 %val) nounwind { ; LA64-LABEL: cmpxchg_i8_acquire_acquire_reti1: ; LA64: # %bb.0: -; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: slli.d $a3, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a4, $zero, 255 ; LA64-NEXT: sll.w $a4, $a4, $a3 +; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a3 ; LA64-NEXT: andi $a2, $a2, 255 ; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a2, $a2, 0 ; LA64-NEXT: addi.w $a3, $a4, 0 ; LA64-NEXT: .LBB12_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 @@ -361,24 +361,24 @@ define i1 @cmpxchg_i8_acquire_acquire_reti1(ptr %ptr, i8 %cmp, i8 %val) nounwind define i1 @cmpxchg_i16_acquire_acquire_reti1(ptr %ptr, i16 %cmp, i16 %val) nounwind { ; LA64-LABEL: cmpxchg_i16_acquire_acquire_reti1: ; LA64: # %bb.0: +; LA64-NEXT: slli.d $a3, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a4, 15 +; LA64-NEXT: ori $a4, $a4, 4095 +; LA64-NEXT: sll.w $a4, $a4, $a3 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: lu12i.w $a3, 15 -; LA64-NEXT: ori $a3, $a3, 4095 -; LA64-NEXT: slli.d $a4, $a0, 3 -; LA64-NEXT: sll.w $a3, $a3, $a4 -; LA64-NEXT: sll.w $a1, $a1, $a4 +; LA64-NEXT: sll.w $a1, $a1, $a3 ; LA64-NEXT: bstrpick.d $a2, $a2, 15, 0 -; LA64-NEXT: sll.w $a2, $a2, $a4 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: sll.w $a2, $a2, $a3 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: addi.w $a4, $a3, 0 +; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: addi.w $a3, $a4, 0 ; LA64-NEXT: .LBB13_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 -; LA64-NEXT: and $a6, $a5, $a4 +; LA64-NEXT: and $a6, $a5, $a3 ; LA64-NEXT: bne $a6, $a1, .LBB13_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB13_1 Depth=1 -; LA64-NEXT: andn $a6, $a5, $a4 +; LA64-NEXT: andn $a6, $a5, $a3 ; LA64-NEXT: or $a6, $a6, $a2 ; LA64-NEXT: sc.w $a6, $a0, 0 ; LA64-NEXT: beqz $a6, .LBB13_1 @@ -386,7 +386,7 @@ define i1 @cmpxchg_i16_acquire_acquire_reti1(ptr %ptr, i16 %cmp, i16 %val) nounw ; LA64-NEXT: .LBB13_3: ; LA64-NEXT: dbar 20 ; LA64-NEXT: .LBB13_4: -; LA64-NEXT: and $a0, $a5, $a3 +; LA64-NEXT: and $a0, $a5, $a4 ; LA64-NEXT: addi.w $a0, $a0, 0 ; LA64-NEXT: xor $a0, $a1, $a0 ; LA64-NEXT: sltui $a0, $a0, 1 @@ -444,17 +444,17 @@ define i1 @cmpxchg_i64_acquire_acquire_reti1(ptr %ptr, i64 %cmp, i64 %val) nounw define void @cmpxchg_i8_monotonic_monotonic(ptr %ptr, i8 %cmp, i8 %val) nounwind { ; LA64-LABEL: cmpxchg_i8_monotonic_monotonic: ; LA64: # %bb.0: -; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: slli.d $a3, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: ori $a4, $zero, 255 +; LA64-NEXT: sll.w $a4, $a4, $a3 +; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a3 ; LA64-NEXT: andi $a2, $a2, 255 ; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: ori $a4, $zero, 255 -; LA64-NEXT: sll.w $a3, $a4, $a3 -; LA64-NEXT: addi.w $a3, $a3, 0 -; LA64-NEXT: addi.w $a2, $a2, 0 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: addi.w $a3, $a4, 0 ; LA64-NEXT: .LBB16_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a5, $a4, $a3 @@ -476,18 +476,18 @@ define void @cmpxchg_i8_monotonic_monotonic(ptr %ptr, i8 %cmp, i8 %val) nounwind define void @cmpxchg_i16_monotonic_monotonic(ptr %ptr, i16 %cmp, i16 %val) nounwind { ; LA64-LABEL: cmpxchg_i16_monotonic_monotonic: ; LA64: # %bb.0: -; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 ; LA64-NEXT: slli.d $a3, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a4, 15 +; LA64-NEXT: ori $a4, $a4, 4095 +; LA64-NEXT: sll.w $a4, $a4, $a3 +; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 ; LA64-NEXT: sll.w $a1, $a1, $a3 ; LA64-NEXT: bstrpick.d $a2, $a2, 15, 0 ; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: lu12i.w $a4, 15 -; LA64-NEXT: ori $a4, $a4, 4095 -; LA64-NEXT: sll.w $a3, $a4, $a3 -; LA64-NEXT: addi.w $a3, $a3, 0 -; LA64-NEXT: addi.w $a2, $a2, 0 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: addi.w $a3, $a4, 0 ; LA64-NEXT: .LBB17_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a5, $a4, $a3 @@ -549,16 +549,16 @@ define i8 @cmpxchg_i8_monotonic_monotonic_reti8(ptr %ptr, i8 %cmp, i8 %val) noun ; LA64-LABEL: cmpxchg_i8_monotonic_monotonic_reti8: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a3, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a4, $zero, 255 ; LA64-NEXT: sll.w $a4, $a4, $a3 -; LA64-NEXT: andi $a2, $a2, 255 -; LA64-NEXT: addi.w $a4, $a4, 0 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: andi $a2, $a2, 255 +; LA64-NEXT: sll.w $a2, $a2, $a3 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: addi.w $a4, $a4, 0 ; LA64-NEXT: .LBB20_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 ; LA64-NEXT: and $a6, $a5, $a4 @@ -582,24 +582,24 @@ define i8 @cmpxchg_i8_monotonic_monotonic_reti8(ptr %ptr, i8 %cmp, i8 %val) noun define i16 @cmpxchg_i16_monotonic_monotonic_reti16(ptr %ptr, i16 %cmp, i16 %val) nounwind { ; LA64-LABEL: cmpxchg_i16_monotonic_monotonic_reti16: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a3, 15 -; LA64-NEXT: ori $a3, $a3, 4095 -; LA64-NEXT: slli.d $a4, $a0, 3 -; LA64-NEXT: sll.w $a3, $a3, $a4 -; LA64-NEXT: bstrpick.d $a2, $a2, 15, 0 -; LA64-NEXT: addi.w $a3, $a3, 0 -; LA64-NEXT: sll.w $a2, $a2, $a4 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a3, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a4, 15 +; LA64-NEXT: ori $a4, $a4, 4095 +; LA64-NEXT: sll.w $a4, $a4, $a3 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a4 +; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: bstrpick.d $a2, $a2, 15, 0 +; LA64-NEXT: sll.w $a2, $a2, $a3 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: addi.w $a4, $a4, 0 ; LA64-NEXT: .LBB21_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 -; LA64-NEXT: and $a6, $a5, $a3 +; LA64-NEXT: and $a6, $a5, $a4 ; LA64-NEXT: bne $a6, $a1, .LBB21_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB21_1 Depth=1 -; LA64-NEXT: andn $a6, $a5, $a3 +; LA64-NEXT: andn $a6, $a5, $a4 ; LA64-NEXT: or $a6, $a6, $a2 ; LA64-NEXT: sc.w $a6, $a0, 0 ; LA64-NEXT: beqz $a6, .LBB21_1 @@ -607,7 +607,7 @@ define i16 @cmpxchg_i16_monotonic_monotonic_reti16(ptr %ptr, i16 %cmp, i16 %val) ; LA64-NEXT: .LBB21_3: ; LA64-NEXT: dbar 1792 ; LA64-NEXT: .LBB21_4: -; LA64-NEXT: srl.w $a0, $a5, $a4 +; LA64-NEXT: srl.w $a0, $a5, $a3 ; LA64-NEXT: ret %tmp = cmpxchg ptr %ptr, i16 %cmp, i16 %val monotonic monotonic %res = extractvalue { i16, i1 } %tmp, 0 @@ -660,16 +660,16 @@ define i64 @cmpxchg_i64_monotonic_monotonic_reti64(ptr %ptr, i64 %cmp, i64 %val) define i1 @cmpxchg_i8_monotonic_monotonic_reti1(ptr %ptr, i8 %cmp, i8 %val) nounwind { ; LA64-LABEL: cmpxchg_i8_monotonic_monotonic_reti1: ; LA64: # %bb.0: -; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: slli.d $a3, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a4, $zero, 255 ; LA64-NEXT: sll.w $a4, $a4, $a3 +; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a3 ; LA64-NEXT: andi $a2, $a2, 255 ; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a2, $a2, 0 ; LA64-NEXT: addi.w $a3, $a4, 0 ; LA64-NEXT: .LBB24_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 @@ -697,24 +697,24 @@ define i1 @cmpxchg_i8_monotonic_monotonic_reti1(ptr %ptr, i8 %cmp, i8 %val) noun define i1 @cmpxchg_i16_monotonic_monotonic_reti1(ptr %ptr, i16 %cmp, i16 %val) nounwind { ; LA64-LABEL: cmpxchg_i16_monotonic_monotonic_reti1: ; LA64: # %bb.0: +; LA64-NEXT: slli.d $a3, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a4, 15 +; LA64-NEXT: ori $a4, $a4, 4095 +; LA64-NEXT: sll.w $a4, $a4, $a3 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: lu12i.w $a3, 15 -; LA64-NEXT: ori $a3, $a3, 4095 -; LA64-NEXT: slli.d $a4, $a0, 3 -; LA64-NEXT: sll.w $a3, $a3, $a4 -; LA64-NEXT: sll.w $a1, $a1, $a4 +; LA64-NEXT: sll.w $a1, $a1, $a3 ; LA64-NEXT: bstrpick.d $a2, $a2, 15, 0 -; LA64-NEXT: sll.w $a2, $a2, $a4 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: sll.w $a2, $a2, $a3 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: addi.w $a4, $a3, 0 +; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: addi.w $a3, $a4, 0 ; LA64-NEXT: .LBB25_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 -; LA64-NEXT: and $a6, $a5, $a4 +; LA64-NEXT: and $a6, $a5, $a3 ; LA64-NEXT: bne $a6, $a1, .LBB25_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB25_1 Depth=1 -; LA64-NEXT: andn $a6, $a5, $a4 +; LA64-NEXT: andn $a6, $a5, $a3 ; LA64-NEXT: or $a6, $a6, $a2 ; LA64-NEXT: sc.w $a6, $a0, 0 ; LA64-NEXT: beqz $a6, .LBB25_1 @@ -722,7 +722,7 @@ define i1 @cmpxchg_i16_monotonic_monotonic_reti1(ptr %ptr, i16 %cmp, i16 %val) n ; LA64-NEXT: .LBB25_3: ; LA64-NEXT: dbar 1792 ; LA64-NEXT: .LBB25_4: -; LA64-NEXT: and $a0, $a5, $a3 +; LA64-NEXT: and $a0, $a5, $a4 ; LA64-NEXT: addi.w $a0, $a0, 0 ; LA64-NEXT: xor $a0, $a1, $a0 ; LA64-NEXT: sltui $a0, $a0, 1 diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-fp.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-fp.ll index 4d8160d70803..ba08790fb7cb 100644 --- a/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-fp.ll +++ b/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-fp.ll @@ -313,20 +313,22 @@ define double @double_fadd_acquire(ptr %p) nounwind { ; LA64F-NEXT: st.d $s2, $sp, 40 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s3, $sp, 32 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s4, $sp, 24 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s5, $sp, 16 # 8-byte Folded Spill ; LA64F-NEXT: move $fp, $a0 -; LA64F-NEXT: ld.d $a0, $a0, 0 +; LA64F-NEXT: ld.d $s5, $a0, 0 ; LA64F-NEXT: lu52i.d $s0, $zero, 1023 ; LA64F-NEXT: ori $s1, $zero, 8 -; LA64F-NEXT: addi.d $s2, $sp, 16 -; LA64F-NEXT: addi.d $s3, $sp, 8 +; LA64F-NEXT: addi.d $s2, $sp, 8 +; LA64F-NEXT: addi.d $s3, $sp, 0 ; LA64F-NEXT: ori $s4, $zero, 2 ; LA64F-NEXT: .p2align 4, , 16 ; LA64F-NEXT: .LBB4_1: # %atomicrmw.start ; LA64F-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64F-NEXT: st.d $a0, $sp, 16 +; LA64F-NEXT: move $a0, $s5 ; LA64F-NEXT: move $a1, $s0 ; LA64F-NEXT: bl %plt(__adddf3) -; LA64F-NEXT: st.d $a0, $sp, 8 +; LA64F-NEXT: st.d $s5, $sp, 8 +; LA64F-NEXT: st.d $a0, $sp, 0 ; LA64F-NEXT: move $a0, $s1 ; LA64F-NEXT: move $a1, $fp ; LA64F-NEXT: move $a2, $s2 @@ -334,10 +336,11 @@ define double @double_fadd_acquire(ptr %p) nounwind { ; LA64F-NEXT: move $a4, $s4 ; LA64F-NEXT: move $a5, $s4 ; LA64F-NEXT: bl %plt(__atomic_compare_exchange) -; LA64F-NEXT: move $a1, $a0 -; LA64F-NEXT: ld.d $a0, $sp, 16 -; LA64F-NEXT: beqz $a1, .LBB4_1 +; LA64F-NEXT: ld.d $s5, $sp, 8 +; LA64F-NEXT: beqz $a0, .LBB4_1 ; LA64F-NEXT: # %bb.2: # %atomicrmw.end +; LA64F-NEXT: move $a0, $s5 +; LA64F-NEXT: ld.d $s5, $sp, 16 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s4, $sp, 24 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s3, $sp, 32 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s2, $sp, 40 # 8-byte Folded Reload @@ -370,9 +373,9 @@ define double @double_fadd_acquire(ptr %p) nounwind { ; LA64D-NEXT: .p2align 4, , 16 ; LA64D-NEXT: .LBB4_1: # %atomicrmw.start ; LA64D-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64D-NEXT: fadd.d $fa1, $fa0, $fs0 ; LA64D-NEXT: fst.d $fa0, $sp, 16 -; LA64D-NEXT: fadd.d $fa0, $fa0, $fs0 -; LA64D-NEXT: fst.d $fa0, $sp, 8 +; LA64D-NEXT: fst.d $fa1, $sp, 8 ; LA64D-NEXT: move $a0, $s0 ; LA64D-NEXT: move $a1, $fp ; LA64D-NEXT: move $a2, $s1 @@ -407,20 +410,22 @@ define double @double_fsub_acquire(ptr %p) nounwind { ; LA64F-NEXT: st.d $s2, $sp, 40 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s3, $sp, 32 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s4, $sp, 24 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s5, $sp, 16 # 8-byte Folded Spill ; LA64F-NEXT: move $fp, $a0 -; LA64F-NEXT: ld.d $a0, $a0, 0 +; LA64F-NEXT: ld.d $s5, $a0, 0 ; LA64F-NEXT: lu52i.d $s0, $zero, -1025 ; LA64F-NEXT: ori $s1, $zero, 8 -; LA64F-NEXT: addi.d $s2, $sp, 16 -; LA64F-NEXT: addi.d $s3, $sp, 8 +; LA64F-NEXT: addi.d $s2, $sp, 8 +; LA64F-NEXT: addi.d $s3, $sp, 0 ; LA64F-NEXT: ori $s4, $zero, 2 ; LA64F-NEXT: .p2align 4, , 16 ; LA64F-NEXT: .LBB5_1: # %atomicrmw.start ; LA64F-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64F-NEXT: st.d $a0, $sp, 16 +; LA64F-NEXT: move $a0, $s5 ; LA64F-NEXT: move $a1, $s0 ; LA64F-NEXT: bl %plt(__adddf3) -; LA64F-NEXT: st.d $a0, $sp, 8 +; LA64F-NEXT: st.d $s5, $sp, 8 +; LA64F-NEXT: st.d $a0, $sp, 0 ; LA64F-NEXT: move $a0, $s1 ; LA64F-NEXT: move $a1, $fp ; LA64F-NEXT: move $a2, $s2 @@ -428,10 +433,11 @@ define double @double_fsub_acquire(ptr %p) nounwind { ; LA64F-NEXT: move $a4, $s4 ; LA64F-NEXT: move $a5, $s4 ; LA64F-NEXT: bl %plt(__atomic_compare_exchange) -; LA64F-NEXT: move $a1, $a0 -; LA64F-NEXT: ld.d $a0, $sp, 16 -; LA64F-NEXT: beqz $a1, .LBB5_1 +; LA64F-NEXT: ld.d $s5, $sp, 8 +; LA64F-NEXT: beqz $a0, .LBB5_1 ; LA64F-NEXT: # %bb.2: # %atomicrmw.end +; LA64F-NEXT: move $a0, $s5 +; LA64F-NEXT: ld.d $s5, $sp, 16 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s4, $sp, 24 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s3, $sp, 32 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s2, $sp, 40 # 8-byte Folded Reload @@ -464,9 +470,9 @@ define double @double_fsub_acquire(ptr %p) nounwind { ; LA64D-NEXT: .p2align 4, , 16 ; LA64D-NEXT: .LBB5_1: # %atomicrmw.start ; LA64D-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64D-NEXT: fadd.d $fa1, $fa0, $fs0 ; LA64D-NEXT: fst.d $fa0, $sp, 16 -; LA64D-NEXT: fadd.d $fa0, $fa0, $fs0 -; LA64D-NEXT: fst.d $fa0, $sp, 8 +; LA64D-NEXT: fst.d $fa1, $sp, 8 ; LA64D-NEXT: move $a0, $s0 ; LA64D-NEXT: move $a1, $fp ; LA64D-NEXT: move $a2, $s1 @@ -501,20 +507,22 @@ define double @double_fmin_acquire(ptr %p) nounwind { ; LA64F-NEXT: st.d $s2, $sp, 40 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s3, $sp, 32 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s4, $sp, 24 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s5, $sp, 16 # 8-byte Folded Spill ; LA64F-NEXT: move $fp, $a0 -; LA64F-NEXT: ld.d $a0, $a0, 0 +; LA64F-NEXT: ld.d $s5, $a0, 0 ; LA64F-NEXT: lu52i.d $s0, $zero, 1023 ; LA64F-NEXT: ori $s1, $zero, 8 -; LA64F-NEXT: addi.d $s2, $sp, 16 -; LA64F-NEXT: addi.d $s3, $sp, 8 +; LA64F-NEXT: addi.d $s2, $sp, 8 +; LA64F-NEXT: addi.d $s3, $sp, 0 ; LA64F-NEXT: ori $s4, $zero, 2 ; LA64F-NEXT: .p2align 4, , 16 ; LA64F-NEXT: .LBB6_1: # %atomicrmw.start ; LA64F-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64F-NEXT: st.d $a0, $sp, 16 +; LA64F-NEXT: move $a0, $s5 ; LA64F-NEXT: move $a1, $s0 ; LA64F-NEXT: bl %plt(fmin) -; LA64F-NEXT: st.d $a0, $sp, 8 +; LA64F-NEXT: st.d $s5, $sp, 8 +; LA64F-NEXT: st.d $a0, $sp, 0 ; LA64F-NEXT: move $a0, $s1 ; LA64F-NEXT: move $a1, $fp ; LA64F-NEXT: move $a2, $s2 @@ -522,10 +530,11 @@ define double @double_fmin_acquire(ptr %p) nounwind { ; LA64F-NEXT: move $a4, $s4 ; LA64F-NEXT: move $a5, $s4 ; LA64F-NEXT: bl %plt(__atomic_compare_exchange) -; LA64F-NEXT: move $a1, $a0 -; LA64F-NEXT: ld.d $a0, $sp, 16 -; LA64F-NEXT: beqz $a1, .LBB6_1 +; LA64F-NEXT: ld.d $s5, $sp, 8 +; LA64F-NEXT: beqz $a0, .LBB6_1 ; LA64F-NEXT: # %bb.2: # %atomicrmw.end +; LA64F-NEXT: move $a0, $s5 +; LA64F-NEXT: ld.d $s5, $sp, 16 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s4, $sp, 24 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s3, $sp, 32 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s2, $sp, 40 # 8-byte Folded Reload @@ -558,10 +567,10 @@ define double @double_fmin_acquire(ptr %p) nounwind { ; LA64D-NEXT: .p2align 4, , 16 ; LA64D-NEXT: .LBB6_1: # %atomicrmw.start ; LA64D-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64D-NEXT: fmax.d $fa1, $fa0, $fa0 +; LA64D-NEXT: fmin.d $fa1, $fa1, $fs0 ; LA64D-NEXT: fst.d $fa0, $sp, 16 -; LA64D-NEXT: fmax.d $fa0, $fa0, $fa0 -; LA64D-NEXT: fmin.d $fa0, $fa0, $fs0 -; LA64D-NEXT: fst.d $fa0, $sp, 8 +; LA64D-NEXT: fst.d $fa1, $sp, 8 ; LA64D-NEXT: move $a0, $s0 ; LA64D-NEXT: move $a1, $fp ; LA64D-NEXT: move $a2, $s1 @@ -596,20 +605,22 @@ define double @double_fmax_acquire(ptr %p) nounwind { ; LA64F-NEXT: st.d $s2, $sp, 40 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s3, $sp, 32 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s4, $sp, 24 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s5, $sp, 16 # 8-byte Folded Spill ; LA64F-NEXT: move $fp, $a0 -; LA64F-NEXT: ld.d $a0, $a0, 0 +; LA64F-NEXT: ld.d $s5, $a0, 0 ; LA64F-NEXT: lu52i.d $s0, $zero, 1023 ; LA64F-NEXT: ori $s1, $zero, 8 -; LA64F-NEXT: addi.d $s2, $sp, 16 -; LA64F-NEXT: addi.d $s3, $sp, 8 +; LA64F-NEXT: addi.d $s2, $sp, 8 +; LA64F-NEXT: addi.d $s3, $sp, 0 ; LA64F-NEXT: ori $s4, $zero, 2 ; LA64F-NEXT: .p2align 4, , 16 ; LA64F-NEXT: .LBB7_1: # %atomicrmw.start ; LA64F-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64F-NEXT: st.d $a0, $sp, 16 +; LA64F-NEXT: move $a0, $s5 ; LA64F-NEXT: move $a1, $s0 ; LA64F-NEXT: bl %plt(fmax) -; LA64F-NEXT: st.d $a0, $sp, 8 +; LA64F-NEXT: st.d $s5, $sp, 8 +; LA64F-NEXT: st.d $a0, $sp, 0 ; LA64F-NEXT: move $a0, $s1 ; LA64F-NEXT: move $a1, $fp ; LA64F-NEXT: move $a2, $s2 @@ -617,10 +628,11 @@ define double @double_fmax_acquire(ptr %p) nounwind { ; LA64F-NEXT: move $a4, $s4 ; LA64F-NEXT: move $a5, $s4 ; LA64F-NEXT: bl %plt(__atomic_compare_exchange) -; LA64F-NEXT: move $a1, $a0 -; LA64F-NEXT: ld.d $a0, $sp, 16 -; LA64F-NEXT: beqz $a1, .LBB7_1 +; LA64F-NEXT: ld.d $s5, $sp, 8 +; LA64F-NEXT: beqz $a0, .LBB7_1 ; LA64F-NEXT: # %bb.2: # %atomicrmw.end +; LA64F-NEXT: move $a0, $s5 +; LA64F-NEXT: ld.d $s5, $sp, 16 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s4, $sp, 24 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s3, $sp, 32 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s2, $sp, 40 # 8-byte Folded Reload @@ -653,10 +665,10 @@ define double @double_fmax_acquire(ptr %p) nounwind { ; LA64D-NEXT: .p2align 4, , 16 ; LA64D-NEXT: .LBB7_1: # %atomicrmw.start ; LA64D-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64D-NEXT: fmax.d $fa1, $fa0, $fa0 +; LA64D-NEXT: fmax.d $fa1, $fa1, $fs0 ; LA64D-NEXT: fst.d $fa0, $sp, 16 -; LA64D-NEXT: fmax.d $fa0, $fa0, $fa0 -; LA64D-NEXT: fmax.d $fa0, $fa0, $fs0 -; LA64D-NEXT: fst.d $fa0, $sp, 8 +; LA64D-NEXT: fst.d $fa1, $sp, 8 ; LA64D-NEXT: move $a0, $s0 ; LA64D-NEXT: move $a1, $fp ; LA64D-NEXT: move $a2, $s1 @@ -991,20 +1003,22 @@ define double @double_fadd_release(ptr %p) nounwind { ; LA64F-NEXT: st.d $s2, $sp, 40 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s3, $sp, 32 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s4, $sp, 24 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s5, $sp, 16 # 8-byte Folded Spill ; LA64F-NEXT: move $fp, $a0 -; LA64F-NEXT: ld.d $a0, $a0, 0 +; LA64F-NEXT: ld.d $s5, $a0, 0 ; LA64F-NEXT: lu52i.d $s0, $zero, 1023 ; LA64F-NEXT: ori $s1, $zero, 8 -; LA64F-NEXT: addi.d $s2, $sp, 16 -; LA64F-NEXT: addi.d $s3, $sp, 8 +; LA64F-NEXT: addi.d $s2, $sp, 8 +; LA64F-NEXT: addi.d $s3, $sp, 0 ; LA64F-NEXT: ori $s4, $zero, 3 ; LA64F-NEXT: .p2align 4, , 16 ; LA64F-NEXT: .LBB12_1: # %atomicrmw.start ; LA64F-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64F-NEXT: st.d $a0, $sp, 16 +; LA64F-NEXT: move $a0, $s5 ; LA64F-NEXT: move $a1, $s0 ; LA64F-NEXT: bl %plt(__adddf3) -; LA64F-NEXT: st.d $a0, $sp, 8 +; LA64F-NEXT: st.d $s5, $sp, 8 +; LA64F-NEXT: st.d $a0, $sp, 0 ; LA64F-NEXT: move $a0, $s1 ; LA64F-NEXT: move $a1, $fp ; LA64F-NEXT: move $a2, $s2 @@ -1012,10 +1026,11 @@ define double @double_fadd_release(ptr %p) nounwind { ; LA64F-NEXT: move $a4, $s4 ; LA64F-NEXT: move $a5, $zero ; LA64F-NEXT: bl %plt(__atomic_compare_exchange) -; LA64F-NEXT: move $a1, $a0 -; LA64F-NEXT: ld.d $a0, $sp, 16 -; LA64F-NEXT: beqz $a1, .LBB12_1 +; LA64F-NEXT: ld.d $s5, $sp, 8 +; LA64F-NEXT: beqz $a0, .LBB12_1 ; LA64F-NEXT: # %bb.2: # %atomicrmw.end +; LA64F-NEXT: move $a0, $s5 +; LA64F-NEXT: ld.d $s5, $sp, 16 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s4, $sp, 24 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s3, $sp, 32 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s2, $sp, 40 # 8-byte Folded Reload @@ -1048,9 +1063,9 @@ define double @double_fadd_release(ptr %p) nounwind { ; LA64D-NEXT: .p2align 4, , 16 ; LA64D-NEXT: .LBB12_1: # %atomicrmw.start ; LA64D-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64D-NEXT: fadd.d $fa1, $fa0, $fs0 ; LA64D-NEXT: fst.d $fa0, $sp, 16 -; LA64D-NEXT: fadd.d $fa0, $fa0, $fs0 -; LA64D-NEXT: fst.d $fa0, $sp, 8 +; LA64D-NEXT: fst.d $fa1, $sp, 8 ; LA64D-NEXT: move $a0, $s0 ; LA64D-NEXT: move $a1, $fp ; LA64D-NEXT: move $a2, $s1 @@ -1085,20 +1100,22 @@ define double @double_fsub_release(ptr %p) nounwind { ; LA64F-NEXT: st.d $s2, $sp, 40 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s3, $sp, 32 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s4, $sp, 24 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s5, $sp, 16 # 8-byte Folded Spill ; LA64F-NEXT: move $fp, $a0 -; LA64F-NEXT: ld.d $a0, $a0, 0 +; LA64F-NEXT: ld.d $s5, $a0, 0 ; LA64F-NEXT: lu52i.d $s0, $zero, -1025 ; LA64F-NEXT: ori $s1, $zero, 8 -; LA64F-NEXT: addi.d $s2, $sp, 16 -; LA64F-NEXT: addi.d $s3, $sp, 8 +; LA64F-NEXT: addi.d $s2, $sp, 8 +; LA64F-NEXT: addi.d $s3, $sp, 0 ; LA64F-NEXT: ori $s4, $zero, 3 ; LA64F-NEXT: .p2align 4, , 16 ; LA64F-NEXT: .LBB13_1: # %atomicrmw.start ; LA64F-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64F-NEXT: st.d $a0, $sp, 16 +; LA64F-NEXT: move $a0, $s5 ; LA64F-NEXT: move $a1, $s0 ; LA64F-NEXT: bl %plt(__adddf3) -; LA64F-NEXT: st.d $a0, $sp, 8 +; LA64F-NEXT: st.d $s5, $sp, 8 +; LA64F-NEXT: st.d $a0, $sp, 0 ; LA64F-NEXT: move $a0, $s1 ; LA64F-NEXT: move $a1, $fp ; LA64F-NEXT: move $a2, $s2 @@ -1106,10 +1123,11 @@ define double @double_fsub_release(ptr %p) nounwind { ; LA64F-NEXT: move $a4, $s4 ; LA64F-NEXT: move $a5, $zero ; LA64F-NEXT: bl %plt(__atomic_compare_exchange) -; LA64F-NEXT: move $a1, $a0 -; LA64F-NEXT: ld.d $a0, $sp, 16 -; LA64F-NEXT: beqz $a1, .LBB13_1 +; LA64F-NEXT: ld.d $s5, $sp, 8 +; LA64F-NEXT: beqz $a0, .LBB13_1 ; LA64F-NEXT: # %bb.2: # %atomicrmw.end +; LA64F-NEXT: move $a0, $s5 +; LA64F-NEXT: ld.d $s5, $sp, 16 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s4, $sp, 24 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s3, $sp, 32 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s2, $sp, 40 # 8-byte Folded Reload @@ -1142,9 +1160,9 @@ define double @double_fsub_release(ptr %p) nounwind { ; LA64D-NEXT: .p2align 4, , 16 ; LA64D-NEXT: .LBB13_1: # %atomicrmw.start ; LA64D-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64D-NEXT: fadd.d $fa1, $fa0, $fs0 ; LA64D-NEXT: fst.d $fa0, $sp, 16 -; LA64D-NEXT: fadd.d $fa0, $fa0, $fs0 -; LA64D-NEXT: fst.d $fa0, $sp, 8 +; LA64D-NEXT: fst.d $fa1, $sp, 8 ; LA64D-NEXT: move $a0, $s0 ; LA64D-NEXT: move $a1, $fp ; LA64D-NEXT: move $a2, $s1 @@ -1179,20 +1197,22 @@ define double @double_fmin_release(ptr %p) nounwind { ; LA64F-NEXT: st.d $s2, $sp, 40 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s3, $sp, 32 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s4, $sp, 24 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s5, $sp, 16 # 8-byte Folded Spill ; LA64F-NEXT: move $fp, $a0 -; LA64F-NEXT: ld.d $a0, $a0, 0 +; LA64F-NEXT: ld.d $s5, $a0, 0 ; LA64F-NEXT: lu52i.d $s0, $zero, 1023 ; LA64F-NEXT: ori $s1, $zero, 8 -; LA64F-NEXT: addi.d $s2, $sp, 16 -; LA64F-NEXT: addi.d $s3, $sp, 8 +; LA64F-NEXT: addi.d $s2, $sp, 8 +; LA64F-NEXT: addi.d $s3, $sp, 0 ; LA64F-NEXT: ori $s4, $zero, 3 ; LA64F-NEXT: .p2align 4, , 16 ; LA64F-NEXT: .LBB14_1: # %atomicrmw.start ; LA64F-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64F-NEXT: st.d $a0, $sp, 16 +; LA64F-NEXT: move $a0, $s5 ; LA64F-NEXT: move $a1, $s0 ; LA64F-NEXT: bl %plt(fmin) -; LA64F-NEXT: st.d $a0, $sp, 8 +; LA64F-NEXT: st.d $s5, $sp, 8 +; LA64F-NEXT: st.d $a0, $sp, 0 ; LA64F-NEXT: move $a0, $s1 ; LA64F-NEXT: move $a1, $fp ; LA64F-NEXT: move $a2, $s2 @@ -1200,10 +1220,11 @@ define double @double_fmin_release(ptr %p) nounwind { ; LA64F-NEXT: move $a4, $s4 ; LA64F-NEXT: move $a5, $zero ; LA64F-NEXT: bl %plt(__atomic_compare_exchange) -; LA64F-NEXT: move $a1, $a0 -; LA64F-NEXT: ld.d $a0, $sp, 16 -; LA64F-NEXT: beqz $a1, .LBB14_1 +; LA64F-NEXT: ld.d $s5, $sp, 8 +; LA64F-NEXT: beqz $a0, .LBB14_1 ; LA64F-NEXT: # %bb.2: # %atomicrmw.end +; LA64F-NEXT: move $a0, $s5 +; LA64F-NEXT: ld.d $s5, $sp, 16 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s4, $sp, 24 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s3, $sp, 32 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s2, $sp, 40 # 8-byte Folded Reload @@ -1236,10 +1257,10 @@ define double @double_fmin_release(ptr %p) nounwind { ; LA64D-NEXT: .p2align 4, , 16 ; LA64D-NEXT: .LBB14_1: # %atomicrmw.start ; LA64D-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64D-NEXT: fmax.d $fa1, $fa0, $fa0 +; LA64D-NEXT: fmin.d $fa1, $fa1, $fs0 ; LA64D-NEXT: fst.d $fa0, $sp, 16 -; LA64D-NEXT: fmax.d $fa0, $fa0, $fa0 -; LA64D-NEXT: fmin.d $fa0, $fa0, $fs0 -; LA64D-NEXT: fst.d $fa0, $sp, 8 +; LA64D-NEXT: fst.d $fa1, $sp, 8 ; LA64D-NEXT: move $a0, $s0 ; LA64D-NEXT: move $a1, $fp ; LA64D-NEXT: move $a2, $s1 @@ -1274,20 +1295,22 @@ define double @double_fmax_release(ptr %p) nounwind { ; LA64F-NEXT: st.d $s2, $sp, 40 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s3, $sp, 32 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s4, $sp, 24 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s5, $sp, 16 # 8-byte Folded Spill ; LA64F-NEXT: move $fp, $a0 -; LA64F-NEXT: ld.d $a0, $a0, 0 +; LA64F-NEXT: ld.d $s5, $a0, 0 ; LA64F-NEXT: lu52i.d $s0, $zero, 1023 ; LA64F-NEXT: ori $s1, $zero, 8 -; LA64F-NEXT: addi.d $s2, $sp, 16 -; LA64F-NEXT: addi.d $s3, $sp, 8 +; LA64F-NEXT: addi.d $s2, $sp, 8 +; LA64F-NEXT: addi.d $s3, $sp, 0 ; LA64F-NEXT: ori $s4, $zero, 3 ; LA64F-NEXT: .p2align 4, , 16 ; LA64F-NEXT: .LBB15_1: # %atomicrmw.start ; LA64F-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64F-NEXT: st.d $a0, $sp, 16 +; LA64F-NEXT: move $a0, $s5 ; LA64F-NEXT: move $a1, $s0 ; LA64F-NEXT: bl %plt(fmax) -; LA64F-NEXT: st.d $a0, $sp, 8 +; LA64F-NEXT: st.d $s5, $sp, 8 +; LA64F-NEXT: st.d $a0, $sp, 0 ; LA64F-NEXT: move $a0, $s1 ; LA64F-NEXT: move $a1, $fp ; LA64F-NEXT: move $a2, $s2 @@ -1295,10 +1318,11 @@ define double @double_fmax_release(ptr %p) nounwind { ; LA64F-NEXT: move $a4, $s4 ; LA64F-NEXT: move $a5, $zero ; LA64F-NEXT: bl %plt(__atomic_compare_exchange) -; LA64F-NEXT: move $a1, $a0 -; LA64F-NEXT: ld.d $a0, $sp, 16 -; LA64F-NEXT: beqz $a1, .LBB15_1 +; LA64F-NEXT: ld.d $s5, $sp, 8 +; LA64F-NEXT: beqz $a0, .LBB15_1 ; LA64F-NEXT: # %bb.2: # %atomicrmw.end +; LA64F-NEXT: move $a0, $s5 +; LA64F-NEXT: ld.d $s5, $sp, 16 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s4, $sp, 24 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s3, $sp, 32 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s2, $sp, 40 # 8-byte Folded Reload @@ -1331,10 +1355,10 @@ define double @double_fmax_release(ptr %p) nounwind { ; LA64D-NEXT: .p2align 4, , 16 ; LA64D-NEXT: .LBB15_1: # %atomicrmw.start ; LA64D-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64D-NEXT: fmax.d $fa1, $fa0, $fa0 +; LA64D-NEXT: fmax.d $fa1, $fa1, $fs0 ; LA64D-NEXT: fst.d $fa0, $sp, 16 -; LA64D-NEXT: fmax.d $fa0, $fa0, $fa0 -; LA64D-NEXT: fmax.d $fa0, $fa0, $fs0 -; LA64D-NEXT: fst.d $fa0, $sp, 8 +; LA64D-NEXT: fst.d $fa1, $sp, 8 ; LA64D-NEXT: move $a0, $s0 ; LA64D-NEXT: move $a1, $fp ; LA64D-NEXT: move $a2, $s1 @@ -1661,30 +1685,32 @@ define float @float_fmax_acq_rel(ptr %p) nounwind { define double @double_fadd_acq_rel(ptr %p) nounwind { ; LA64F-LABEL: double_fadd_acq_rel: ; LA64F: # %bb.0: -; LA64F-NEXT: addi.d $sp, $sp, -80 -; LA64F-NEXT: st.d $ra, $sp, 72 # 8-byte Folded Spill -; LA64F-NEXT: st.d $fp, $sp, 64 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s0, $sp, 56 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s1, $sp, 48 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s2, $sp, 40 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s3, $sp, 32 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s4, $sp, 24 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s5, $sp, 16 # 8-byte Folded Spill +; LA64F-NEXT: addi.d $sp, $sp, -96 +; LA64F-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill +; LA64F-NEXT: st.d $fp, $sp, 80 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s0, $sp, 72 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s1, $sp, 64 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s2, $sp, 56 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s3, $sp, 48 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s4, $sp, 40 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s5, $sp, 32 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s6, $sp, 24 # 8-byte Folded Spill ; LA64F-NEXT: move $fp, $a0 -; LA64F-NEXT: ld.d $a0, $a0, 0 +; LA64F-NEXT: ld.d $s6, $a0, 0 ; LA64F-NEXT: lu52i.d $s0, $zero, 1023 ; LA64F-NEXT: ori $s1, $zero, 8 -; LA64F-NEXT: addi.d $s2, $sp, 8 -; LA64F-NEXT: addi.d $s3, $sp, 0 +; LA64F-NEXT: addi.d $s2, $sp, 16 +; LA64F-NEXT: addi.d $s3, $sp, 8 ; LA64F-NEXT: ori $s4, $zero, 4 ; LA64F-NEXT: ori $s5, $zero, 2 ; LA64F-NEXT: .p2align 4, , 16 ; LA64F-NEXT: .LBB20_1: # %atomicrmw.start ; LA64F-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64F-NEXT: st.d $a0, $sp, 8 +; LA64F-NEXT: move $a0, $s6 ; LA64F-NEXT: move $a1, $s0 ; LA64F-NEXT: bl %plt(__adddf3) -; LA64F-NEXT: st.d $a0, $sp, 0 +; LA64F-NEXT: st.d $s6, $sp, 16 +; LA64F-NEXT: st.d $a0, $sp, 8 ; LA64F-NEXT: move $a0, $s1 ; LA64F-NEXT: move $a1, $fp ; LA64F-NEXT: move $a2, $s2 @@ -1692,19 +1718,20 @@ define double @double_fadd_acq_rel(ptr %p) nounwind { ; LA64F-NEXT: move $a4, $s4 ; LA64F-NEXT: move $a5, $s5 ; LA64F-NEXT: bl %plt(__atomic_compare_exchange) -; LA64F-NEXT: move $a1, $a0 -; LA64F-NEXT: ld.d $a0, $sp, 8 -; LA64F-NEXT: beqz $a1, .LBB20_1 +; LA64F-NEXT: ld.d $s6, $sp, 16 +; LA64F-NEXT: beqz $a0, .LBB20_1 ; LA64F-NEXT: # %bb.2: # %atomicrmw.end -; LA64F-NEXT: ld.d $s5, $sp, 16 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s4, $sp, 24 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s3, $sp, 32 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s2, $sp, 40 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s1, $sp, 48 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s0, $sp, 56 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $fp, $sp, 64 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $ra, $sp, 72 # 8-byte Folded Reload -; LA64F-NEXT: addi.d $sp, $sp, 80 +; LA64F-NEXT: move $a0, $s6 +; LA64F-NEXT: ld.d $s6, $sp, 24 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s5, $sp, 32 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s4, $sp, 40 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s3, $sp, 48 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s2, $sp, 56 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s1, $sp, 64 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s0, $sp, 72 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $fp, $sp, 80 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload +; LA64F-NEXT: addi.d $sp, $sp, 96 ; LA64F-NEXT: ret ; ; LA64D-LABEL: double_fadd_acq_rel: @@ -1731,9 +1758,9 @@ define double @double_fadd_acq_rel(ptr %p) nounwind { ; LA64D-NEXT: .p2align 4, , 16 ; LA64D-NEXT: .LBB20_1: # %atomicrmw.start ; LA64D-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64D-NEXT: fadd.d $fa1, $fa0, $fs0 ; LA64D-NEXT: fst.d $fa0, $sp, 8 -; LA64D-NEXT: fadd.d $fa0, $fa0, $fs0 -; LA64D-NEXT: fst.d $fa0, $sp, 0 +; LA64D-NEXT: fst.d $fa1, $sp, 0 ; LA64D-NEXT: move $a0, $s0 ; LA64D-NEXT: move $a1, $fp ; LA64D-NEXT: move $a2, $s1 @@ -1761,30 +1788,32 @@ define double @double_fadd_acq_rel(ptr %p) nounwind { define double @double_fsub_acq_rel(ptr %p) nounwind { ; LA64F-LABEL: double_fsub_acq_rel: ; LA64F: # %bb.0: -; LA64F-NEXT: addi.d $sp, $sp, -80 -; LA64F-NEXT: st.d $ra, $sp, 72 # 8-byte Folded Spill -; LA64F-NEXT: st.d $fp, $sp, 64 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s0, $sp, 56 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s1, $sp, 48 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s2, $sp, 40 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s3, $sp, 32 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s4, $sp, 24 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s5, $sp, 16 # 8-byte Folded Spill +; LA64F-NEXT: addi.d $sp, $sp, -96 +; LA64F-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill +; LA64F-NEXT: st.d $fp, $sp, 80 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s0, $sp, 72 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s1, $sp, 64 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s2, $sp, 56 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s3, $sp, 48 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s4, $sp, 40 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s5, $sp, 32 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s6, $sp, 24 # 8-byte Folded Spill ; LA64F-NEXT: move $fp, $a0 -; LA64F-NEXT: ld.d $a0, $a0, 0 +; LA64F-NEXT: ld.d $s6, $a0, 0 ; LA64F-NEXT: lu52i.d $s0, $zero, -1025 ; LA64F-NEXT: ori $s1, $zero, 8 -; LA64F-NEXT: addi.d $s2, $sp, 8 -; LA64F-NEXT: addi.d $s3, $sp, 0 +; LA64F-NEXT: addi.d $s2, $sp, 16 +; LA64F-NEXT: addi.d $s3, $sp, 8 ; LA64F-NEXT: ori $s4, $zero, 4 ; LA64F-NEXT: ori $s5, $zero, 2 ; LA64F-NEXT: .p2align 4, , 16 ; LA64F-NEXT: .LBB21_1: # %atomicrmw.start ; LA64F-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64F-NEXT: st.d $a0, $sp, 8 +; LA64F-NEXT: move $a0, $s6 ; LA64F-NEXT: move $a1, $s0 ; LA64F-NEXT: bl %plt(__adddf3) -; LA64F-NEXT: st.d $a0, $sp, 0 +; LA64F-NEXT: st.d $s6, $sp, 16 +; LA64F-NEXT: st.d $a0, $sp, 8 ; LA64F-NEXT: move $a0, $s1 ; LA64F-NEXT: move $a1, $fp ; LA64F-NEXT: move $a2, $s2 @@ -1792,19 +1821,20 @@ define double @double_fsub_acq_rel(ptr %p) nounwind { ; LA64F-NEXT: move $a4, $s4 ; LA64F-NEXT: move $a5, $s5 ; LA64F-NEXT: bl %plt(__atomic_compare_exchange) -; LA64F-NEXT: move $a1, $a0 -; LA64F-NEXT: ld.d $a0, $sp, 8 -; LA64F-NEXT: beqz $a1, .LBB21_1 +; LA64F-NEXT: ld.d $s6, $sp, 16 +; LA64F-NEXT: beqz $a0, .LBB21_1 ; LA64F-NEXT: # %bb.2: # %atomicrmw.end -; LA64F-NEXT: ld.d $s5, $sp, 16 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s4, $sp, 24 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s3, $sp, 32 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s2, $sp, 40 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s1, $sp, 48 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s0, $sp, 56 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $fp, $sp, 64 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $ra, $sp, 72 # 8-byte Folded Reload -; LA64F-NEXT: addi.d $sp, $sp, 80 +; LA64F-NEXT: move $a0, $s6 +; LA64F-NEXT: ld.d $s6, $sp, 24 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s5, $sp, 32 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s4, $sp, 40 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s3, $sp, 48 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s2, $sp, 56 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s1, $sp, 64 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s0, $sp, 72 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $fp, $sp, 80 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload +; LA64F-NEXT: addi.d $sp, $sp, 96 ; LA64F-NEXT: ret ; ; LA64D-LABEL: double_fsub_acq_rel: @@ -1831,9 +1861,9 @@ define double @double_fsub_acq_rel(ptr %p) nounwind { ; LA64D-NEXT: .p2align 4, , 16 ; LA64D-NEXT: .LBB21_1: # %atomicrmw.start ; LA64D-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64D-NEXT: fadd.d $fa1, $fa0, $fs0 ; LA64D-NEXT: fst.d $fa0, $sp, 8 -; LA64D-NEXT: fadd.d $fa0, $fa0, $fs0 -; LA64D-NEXT: fst.d $fa0, $sp, 0 +; LA64D-NEXT: fst.d $fa1, $sp, 0 ; LA64D-NEXT: move $a0, $s0 ; LA64D-NEXT: move $a1, $fp ; LA64D-NEXT: move $a2, $s1 @@ -1861,30 +1891,32 @@ define double @double_fsub_acq_rel(ptr %p) nounwind { define double @double_fmin_acq_rel(ptr %p) nounwind { ; LA64F-LABEL: double_fmin_acq_rel: ; LA64F: # %bb.0: -; LA64F-NEXT: addi.d $sp, $sp, -80 -; LA64F-NEXT: st.d $ra, $sp, 72 # 8-byte Folded Spill -; LA64F-NEXT: st.d $fp, $sp, 64 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s0, $sp, 56 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s1, $sp, 48 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s2, $sp, 40 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s3, $sp, 32 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s4, $sp, 24 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s5, $sp, 16 # 8-byte Folded Spill +; LA64F-NEXT: addi.d $sp, $sp, -96 +; LA64F-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill +; LA64F-NEXT: st.d $fp, $sp, 80 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s0, $sp, 72 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s1, $sp, 64 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s2, $sp, 56 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s3, $sp, 48 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s4, $sp, 40 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s5, $sp, 32 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s6, $sp, 24 # 8-byte Folded Spill ; LA64F-NEXT: move $fp, $a0 -; LA64F-NEXT: ld.d $a0, $a0, 0 +; LA64F-NEXT: ld.d $s6, $a0, 0 ; LA64F-NEXT: lu52i.d $s0, $zero, 1023 ; LA64F-NEXT: ori $s1, $zero, 8 -; LA64F-NEXT: addi.d $s2, $sp, 8 -; LA64F-NEXT: addi.d $s3, $sp, 0 +; LA64F-NEXT: addi.d $s2, $sp, 16 +; LA64F-NEXT: addi.d $s3, $sp, 8 ; LA64F-NEXT: ori $s4, $zero, 4 ; LA64F-NEXT: ori $s5, $zero, 2 ; LA64F-NEXT: .p2align 4, , 16 ; LA64F-NEXT: .LBB22_1: # %atomicrmw.start ; LA64F-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64F-NEXT: st.d $a0, $sp, 8 +; LA64F-NEXT: move $a0, $s6 ; LA64F-NEXT: move $a1, $s0 ; LA64F-NEXT: bl %plt(fmin) -; LA64F-NEXT: st.d $a0, $sp, 0 +; LA64F-NEXT: st.d $s6, $sp, 16 +; LA64F-NEXT: st.d $a0, $sp, 8 ; LA64F-NEXT: move $a0, $s1 ; LA64F-NEXT: move $a1, $fp ; LA64F-NEXT: move $a2, $s2 @@ -1892,19 +1924,20 @@ define double @double_fmin_acq_rel(ptr %p) nounwind { ; LA64F-NEXT: move $a4, $s4 ; LA64F-NEXT: move $a5, $s5 ; LA64F-NEXT: bl %plt(__atomic_compare_exchange) -; LA64F-NEXT: move $a1, $a0 -; LA64F-NEXT: ld.d $a0, $sp, 8 -; LA64F-NEXT: beqz $a1, .LBB22_1 +; LA64F-NEXT: ld.d $s6, $sp, 16 +; LA64F-NEXT: beqz $a0, .LBB22_1 ; LA64F-NEXT: # %bb.2: # %atomicrmw.end -; LA64F-NEXT: ld.d $s5, $sp, 16 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s4, $sp, 24 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s3, $sp, 32 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s2, $sp, 40 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s1, $sp, 48 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s0, $sp, 56 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $fp, $sp, 64 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $ra, $sp, 72 # 8-byte Folded Reload -; LA64F-NEXT: addi.d $sp, $sp, 80 +; LA64F-NEXT: move $a0, $s6 +; LA64F-NEXT: ld.d $s6, $sp, 24 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s5, $sp, 32 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s4, $sp, 40 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s3, $sp, 48 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s2, $sp, 56 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s1, $sp, 64 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s0, $sp, 72 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $fp, $sp, 80 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload +; LA64F-NEXT: addi.d $sp, $sp, 96 ; LA64F-NEXT: ret ; ; LA64D-LABEL: double_fmin_acq_rel: @@ -1931,10 +1964,10 @@ define double @double_fmin_acq_rel(ptr %p) nounwind { ; LA64D-NEXT: .p2align 4, , 16 ; LA64D-NEXT: .LBB22_1: # %atomicrmw.start ; LA64D-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64D-NEXT: fmax.d $fa1, $fa0, $fa0 +; LA64D-NEXT: fmin.d $fa1, $fa1, $fs0 ; LA64D-NEXT: fst.d $fa0, $sp, 8 -; LA64D-NEXT: fmax.d $fa0, $fa0, $fa0 -; LA64D-NEXT: fmin.d $fa0, $fa0, $fs0 -; LA64D-NEXT: fst.d $fa0, $sp, 0 +; LA64D-NEXT: fst.d $fa1, $sp, 0 ; LA64D-NEXT: move $a0, $s0 ; LA64D-NEXT: move $a1, $fp ; LA64D-NEXT: move $a2, $s1 @@ -1962,30 +1995,32 @@ define double @double_fmin_acq_rel(ptr %p) nounwind { define double @double_fmax_acq_rel(ptr %p) nounwind { ; LA64F-LABEL: double_fmax_acq_rel: ; LA64F: # %bb.0: -; LA64F-NEXT: addi.d $sp, $sp, -80 -; LA64F-NEXT: st.d $ra, $sp, 72 # 8-byte Folded Spill -; LA64F-NEXT: st.d $fp, $sp, 64 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s0, $sp, 56 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s1, $sp, 48 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s2, $sp, 40 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s3, $sp, 32 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s4, $sp, 24 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s5, $sp, 16 # 8-byte Folded Spill +; LA64F-NEXT: addi.d $sp, $sp, -96 +; LA64F-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill +; LA64F-NEXT: st.d $fp, $sp, 80 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s0, $sp, 72 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s1, $sp, 64 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s2, $sp, 56 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s3, $sp, 48 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s4, $sp, 40 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s5, $sp, 32 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s6, $sp, 24 # 8-byte Folded Spill ; LA64F-NEXT: move $fp, $a0 -; LA64F-NEXT: ld.d $a0, $a0, 0 +; LA64F-NEXT: ld.d $s6, $a0, 0 ; LA64F-NEXT: lu52i.d $s0, $zero, 1023 ; LA64F-NEXT: ori $s1, $zero, 8 -; LA64F-NEXT: addi.d $s2, $sp, 8 -; LA64F-NEXT: addi.d $s3, $sp, 0 +; LA64F-NEXT: addi.d $s2, $sp, 16 +; LA64F-NEXT: addi.d $s3, $sp, 8 ; LA64F-NEXT: ori $s4, $zero, 4 ; LA64F-NEXT: ori $s5, $zero, 2 ; LA64F-NEXT: .p2align 4, , 16 ; LA64F-NEXT: .LBB23_1: # %atomicrmw.start ; LA64F-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64F-NEXT: st.d $a0, $sp, 8 +; LA64F-NEXT: move $a0, $s6 ; LA64F-NEXT: move $a1, $s0 ; LA64F-NEXT: bl %plt(fmax) -; LA64F-NEXT: st.d $a0, $sp, 0 +; LA64F-NEXT: st.d $s6, $sp, 16 +; LA64F-NEXT: st.d $a0, $sp, 8 ; LA64F-NEXT: move $a0, $s1 ; LA64F-NEXT: move $a1, $fp ; LA64F-NEXT: move $a2, $s2 @@ -1993,19 +2028,20 @@ define double @double_fmax_acq_rel(ptr %p) nounwind { ; LA64F-NEXT: move $a4, $s4 ; LA64F-NEXT: move $a5, $s5 ; LA64F-NEXT: bl %plt(__atomic_compare_exchange) -; LA64F-NEXT: move $a1, $a0 -; LA64F-NEXT: ld.d $a0, $sp, 8 -; LA64F-NEXT: beqz $a1, .LBB23_1 +; LA64F-NEXT: ld.d $s6, $sp, 16 +; LA64F-NEXT: beqz $a0, .LBB23_1 ; LA64F-NEXT: # %bb.2: # %atomicrmw.end -; LA64F-NEXT: ld.d $s5, $sp, 16 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s4, $sp, 24 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s3, $sp, 32 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s2, $sp, 40 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s1, $sp, 48 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s0, $sp, 56 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $fp, $sp, 64 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $ra, $sp, 72 # 8-byte Folded Reload -; LA64F-NEXT: addi.d $sp, $sp, 80 +; LA64F-NEXT: move $a0, $s6 +; LA64F-NEXT: ld.d $s6, $sp, 24 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s5, $sp, 32 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s4, $sp, 40 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s3, $sp, 48 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s2, $sp, 56 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s1, $sp, 64 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s0, $sp, 72 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $fp, $sp, 80 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload +; LA64F-NEXT: addi.d $sp, $sp, 96 ; LA64F-NEXT: ret ; ; LA64D-LABEL: double_fmax_acq_rel: @@ -2032,10 +2068,10 @@ define double @double_fmax_acq_rel(ptr %p) nounwind { ; LA64D-NEXT: .p2align 4, , 16 ; LA64D-NEXT: .LBB23_1: # %atomicrmw.start ; LA64D-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64D-NEXT: fmax.d $fa1, $fa0, $fa0 +; LA64D-NEXT: fmax.d $fa1, $fa1, $fs0 ; LA64D-NEXT: fst.d $fa0, $sp, 8 -; LA64D-NEXT: fmax.d $fa0, $fa0, $fa0 -; LA64D-NEXT: fmax.d $fa0, $fa0, $fs0 -; LA64D-NEXT: fst.d $fa0, $sp, 0 +; LA64D-NEXT: fst.d $fa1, $sp, 0 ; LA64D-NEXT: move $a0, $s0 ; LA64D-NEXT: move $a1, $fp ; LA64D-NEXT: move $a2, $s1 @@ -2371,20 +2407,22 @@ define double @double_fadd_seq_cst(ptr %p) nounwind { ; LA64F-NEXT: st.d $s2, $sp, 40 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s3, $sp, 32 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s4, $sp, 24 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s5, $sp, 16 # 8-byte Folded Spill ; LA64F-NEXT: move $fp, $a0 -; LA64F-NEXT: ld.d $a0, $a0, 0 +; LA64F-NEXT: ld.d $s5, $a0, 0 ; LA64F-NEXT: lu52i.d $s0, $zero, 1023 ; LA64F-NEXT: ori $s1, $zero, 8 -; LA64F-NEXT: addi.d $s2, $sp, 16 -; LA64F-NEXT: addi.d $s3, $sp, 8 +; LA64F-NEXT: addi.d $s2, $sp, 8 +; LA64F-NEXT: addi.d $s3, $sp, 0 ; LA64F-NEXT: ori $s4, $zero, 5 ; LA64F-NEXT: .p2align 4, , 16 ; LA64F-NEXT: .LBB28_1: # %atomicrmw.start ; LA64F-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64F-NEXT: st.d $a0, $sp, 16 +; LA64F-NEXT: move $a0, $s5 ; LA64F-NEXT: move $a1, $s0 ; LA64F-NEXT: bl %plt(__adddf3) -; LA64F-NEXT: st.d $a0, $sp, 8 +; LA64F-NEXT: st.d $s5, $sp, 8 +; LA64F-NEXT: st.d $a0, $sp, 0 ; LA64F-NEXT: move $a0, $s1 ; LA64F-NEXT: move $a1, $fp ; LA64F-NEXT: move $a2, $s2 @@ -2392,10 +2430,11 @@ define double @double_fadd_seq_cst(ptr %p) nounwind { ; LA64F-NEXT: move $a4, $s4 ; LA64F-NEXT: move $a5, $s4 ; LA64F-NEXT: bl %plt(__atomic_compare_exchange) -; LA64F-NEXT: move $a1, $a0 -; LA64F-NEXT: ld.d $a0, $sp, 16 -; LA64F-NEXT: beqz $a1, .LBB28_1 +; LA64F-NEXT: ld.d $s5, $sp, 8 +; LA64F-NEXT: beqz $a0, .LBB28_1 ; LA64F-NEXT: # %bb.2: # %atomicrmw.end +; LA64F-NEXT: move $a0, $s5 +; LA64F-NEXT: ld.d $s5, $sp, 16 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s4, $sp, 24 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s3, $sp, 32 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s2, $sp, 40 # 8-byte Folded Reload @@ -2428,9 +2467,9 @@ define double @double_fadd_seq_cst(ptr %p) nounwind { ; LA64D-NEXT: .p2align 4, , 16 ; LA64D-NEXT: .LBB28_1: # %atomicrmw.start ; LA64D-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64D-NEXT: fadd.d $fa1, $fa0, $fs0 ; LA64D-NEXT: fst.d $fa0, $sp, 16 -; LA64D-NEXT: fadd.d $fa0, $fa0, $fs0 -; LA64D-NEXT: fst.d $fa0, $sp, 8 +; LA64D-NEXT: fst.d $fa1, $sp, 8 ; LA64D-NEXT: move $a0, $s0 ; LA64D-NEXT: move $a1, $fp ; LA64D-NEXT: move $a2, $s1 @@ -2465,20 +2504,22 @@ define double @double_fsub_seq_cst(ptr %p) nounwind { ; LA64F-NEXT: st.d $s2, $sp, 40 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s3, $sp, 32 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s4, $sp, 24 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s5, $sp, 16 # 8-byte Folded Spill ; LA64F-NEXT: move $fp, $a0 -; LA64F-NEXT: ld.d $a0, $a0, 0 +; LA64F-NEXT: ld.d $s5, $a0, 0 ; LA64F-NEXT: lu52i.d $s0, $zero, -1025 ; LA64F-NEXT: ori $s1, $zero, 8 -; LA64F-NEXT: addi.d $s2, $sp, 16 -; LA64F-NEXT: addi.d $s3, $sp, 8 +; LA64F-NEXT: addi.d $s2, $sp, 8 +; LA64F-NEXT: addi.d $s3, $sp, 0 ; LA64F-NEXT: ori $s4, $zero, 5 ; LA64F-NEXT: .p2align 4, , 16 ; LA64F-NEXT: .LBB29_1: # %atomicrmw.start ; LA64F-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64F-NEXT: st.d $a0, $sp, 16 +; LA64F-NEXT: move $a0, $s5 ; LA64F-NEXT: move $a1, $s0 ; LA64F-NEXT: bl %plt(__adddf3) -; LA64F-NEXT: st.d $a0, $sp, 8 +; LA64F-NEXT: st.d $s5, $sp, 8 +; LA64F-NEXT: st.d $a0, $sp, 0 ; LA64F-NEXT: move $a0, $s1 ; LA64F-NEXT: move $a1, $fp ; LA64F-NEXT: move $a2, $s2 @@ -2486,10 +2527,11 @@ define double @double_fsub_seq_cst(ptr %p) nounwind { ; LA64F-NEXT: move $a4, $s4 ; LA64F-NEXT: move $a5, $s4 ; LA64F-NEXT: bl %plt(__atomic_compare_exchange) -; LA64F-NEXT: move $a1, $a0 -; LA64F-NEXT: ld.d $a0, $sp, 16 -; LA64F-NEXT: beqz $a1, .LBB29_1 +; LA64F-NEXT: ld.d $s5, $sp, 8 +; LA64F-NEXT: beqz $a0, .LBB29_1 ; LA64F-NEXT: # %bb.2: # %atomicrmw.end +; LA64F-NEXT: move $a0, $s5 +; LA64F-NEXT: ld.d $s5, $sp, 16 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s4, $sp, 24 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s3, $sp, 32 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s2, $sp, 40 # 8-byte Folded Reload @@ -2522,9 +2564,9 @@ define double @double_fsub_seq_cst(ptr %p) nounwind { ; LA64D-NEXT: .p2align 4, , 16 ; LA64D-NEXT: .LBB29_1: # %atomicrmw.start ; LA64D-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64D-NEXT: fadd.d $fa1, $fa0, $fs0 ; LA64D-NEXT: fst.d $fa0, $sp, 16 -; LA64D-NEXT: fadd.d $fa0, $fa0, $fs0 -; LA64D-NEXT: fst.d $fa0, $sp, 8 +; LA64D-NEXT: fst.d $fa1, $sp, 8 ; LA64D-NEXT: move $a0, $s0 ; LA64D-NEXT: move $a1, $fp ; LA64D-NEXT: move $a2, $s1 @@ -2559,20 +2601,22 @@ define double @double_fmin_seq_cst(ptr %p) nounwind { ; LA64F-NEXT: st.d $s2, $sp, 40 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s3, $sp, 32 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s4, $sp, 24 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s5, $sp, 16 # 8-byte Folded Spill ; LA64F-NEXT: move $fp, $a0 -; LA64F-NEXT: ld.d $a0, $a0, 0 +; LA64F-NEXT: ld.d $s5, $a0, 0 ; LA64F-NEXT: lu52i.d $s0, $zero, 1023 ; LA64F-NEXT: ori $s1, $zero, 8 -; LA64F-NEXT: addi.d $s2, $sp, 16 -; LA64F-NEXT: addi.d $s3, $sp, 8 +; LA64F-NEXT: addi.d $s2, $sp, 8 +; LA64F-NEXT: addi.d $s3, $sp, 0 ; LA64F-NEXT: ori $s4, $zero, 5 ; LA64F-NEXT: .p2align 4, , 16 ; LA64F-NEXT: .LBB30_1: # %atomicrmw.start ; LA64F-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64F-NEXT: st.d $a0, $sp, 16 +; LA64F-NEXT: move $a0, $s5 ; LA64F-NEXT: move $a1, $s0 ; LA64F-NEXT: bl %plt(fmin) -; LA64F-NEXT: st.d $a0, $sp, 8 +; LA64F-NEXT: st.d $s5, $sp, 8 +; LA64F-NEXT: st.d $a0, $sp, 0 ; LA64F-NEXT: move $a0, $s1 ; LA64F-NEXT: move $a1, $fp ; LA64F-NEXT: move $a2, $s2 @@ -2580,10 +2624,11 @@ define double @double_fmin_seq_cst(ptr %p) nounwind { ; LA64F-NEXT: move $a4, $s4 ; LA64F-NEXT: move $a5, $s4 ; LA64F-NEXT: bl %plt(__atomic_compare_exchange) -; LA64F-NEXT: move $a1, $a0 -; LA64F-NEXT: ld.d $a0, $sp, 16 -; LA64F-NEXT: beqz $a1, .LBB30_1 +; LA64F-NEXT: ld.d $s5, $sp, 8 +; LA64F-NEXT: beqz $a0, .LBB30_1 ; LA64F-NEXT: # %bb.2: # %atomicrmw.end +; LA64F-NEXT: move $a0, $s5 +; LA64F-NEXT: ld.d $s5, $sp, 16 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s4, $sp, 24 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s3, $sp, 32 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s2, $sp, 40 # 8-byte Folded Reload @@ -2616,10 +2661,10 @@ define double @double_fmin_seq_cst(ptr %p) nounwind { ; LA64D-NEXT: .p2align 4, , 16 ; LA64D-NEXT: .LBB30_1: # %atomicrmw.start ; LA64D-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64D-NEXT: fmax.d $fa1, $fa0, $fa0 +; LA64D-NEXT: fmin.d $fa1, $fa1, $fs0 ; LA64D-NEXT: fst.d $fa0, $sp, 16 -; LA64D-NEXT: fmax.d $fa0, $fa0, $fa0 -; LA64D-NEXT: fmin.d $fa0, $fa0, $fs0 -; LA64D-NEXT: fst.d $fa0, $sp, 8 +; LA64D-NEXT: fst.d $fa1, $sp, 8 ; LA64D-NEXT: move $a0, $s0 ; LA64D-NEXT: move $a1, $fp ; LA64D-NEXT: move $a2, $s1 @@ -2654,20 +2699,22 @@ define double @double_fmax_seq_cst(ptr %p) nounwind { ; LA64F-NEXT: st.d $s2, $sp, 40 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s3, $sp, 32 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s4, $sp, 24 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s5, $sp, 16 # 8-byte Folded Spill ; LA64F-NEXT: move $fp, $a0 -; LA64F-NEXT: ld.d $a0, $a0, 0 +; LA64F-NEXT: ld.d $s5, $a0, 0 ; LA64F-NEXT: lu52i.d $s0, $zero, 1023 ; LA64F-NEXT: ori $s1, $zero, 8 -; LA64F-NEXT: addi.d $s2, $sp, 16 -; LA64F-NEXT: addi.d $s3, $sp, 8 +; LA64F-NEXT: addi.d $s2, $sp, 8 +; LA64F-NEXT: addi.d $s3, $sp, 0 ; LA64F-NEXT: ori $s4, $zero, 5 ; LA64F-NEXT: .p2align 4, , 16 ; LA64F-NEXT: .LBB31_1: # %atomicrmw.start ; LA64F-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64F-NEXT: st.d $a0, $sp, 16 +; LA64F-NEXT: move $a0, $s5 ; LA64F-NEXT: move $a1, $s0 ; LA64F-NEXT: bl %plt(fmax) -; LA64F-NEXT: st.d $a0, $sp, 8 +; LA64F-NEXT: st.d $s5, $sp, 8 +; LA64F-NEXT: st.d $a0, $sp, 0 ; LA64F-NEXT: move $a0, $s1 ; LA64F-NEXT: move $a1, $fp ; LA64F-NEXT: move $a2, $s2 @@ -2675,10 +2722,11 @@ define double @double_fmax_seq_cst(ptr %p) nounwind { ; LA64F-NEXT: move $a4, $s4 ; LA64F-NEXT: move $a5, $s4 ; LA64F-NEXT: bl %plt(__atomic_compare_exchange) -; LA64F-NEXT: move $a1, $a0 -; LA64F-NEXT: ld.d $a0, $sp, 16 -; LA64F-NEXT: beqz $a1, .LBB31_1 +; LA64F-NEXT: ld.d $s5, $sp, 8 +; LA64F-NEXT: beqz $a0, .LBB31_1 ; LA64F-NEXT: # %bb.2: # %atomicrmw.end +; LA64F-NEXT: move $a0, $s5 +; LA64F-NEXT: ld.d $s5, $sp, 16 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s4, $sp, 24 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s3, $sp, 32 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s2, $sp, 40 # 8-byte Folded Reload @@ -2711,10 +2759,10 @@ define double @double_fmax_seq_cst(ptr %p) nounwind { ; LA64D-NEXT: .p2align 4, , 16 ; LA64D-NEXT: .LBB31_1: # %atomicrmw.start ; LA64D-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64D-NEXT: fmax.d $fa1, $fa0, $fa0 +; LA64D-NEXT: fmax.d $fa1, $fa1, $fs0 ; LA64D-NEXT: fst.d $fa0, $sp, 16 -; LA64D-NEXT: fmax.d $fa0, $fa0, $fa0 -; LA64D-NEXT: fmax.d $fa0, $fa0, $fs0 -; LA64D-NEXT: fst.d $fa0, $sp, 8 +; LA64D-NEXT: fst.d $fa1, $sp, 8 ; LA64D-NEXT: move $a0, $s0 ; LA64D-NEXT: move $a1, $fp ; LA64D-NEXT: move $a2, $s1 @@ -3041,26 +3089,28 @@ define float @float_fmax_monotonic(ptr %p) nounwind { define double @double_fadd_monotonic(ptr %p) nounwind { ; LA64F-LABEL: double_fadd_monotonic: ; LA64F: # %bb.0: -; LA64F-NEXT: addi.d $sp, $sp, -64 -; LA64F-NEXT: st.d $ra, $sp, 56 # 8-byte Folded Spill -; LA64F-NEXT: st.d $fp, $sp, 48 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s0, $sp, 40 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s1, $sp, 32 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s2, $sp, 24 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s3, $sp, 16 # 8-byte Folded Spill +; LA64F-NEXT: addi.d $sp, $sp, -80 +; LA64F-NEXT: st.d $ra, $sp, 72 # 8-byte Folded Spill +; LA64F-NEXT: st.d $fp, $sp, 64 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s0, $sp, 56 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s1, $sp, 48 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s2, $sp, 40 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s3, $sp, 32 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s4, $sp, 24 # 8-byte Folded Spill ; LA64F-NEXT: move $fp, $a0 -; LA64F-NEXT: ld.d $a0, $a0, 0 +; LA64F-NEXT: ld.d $s4, $a0, 0 ; LA64F-NEXT: lu52i.d $s0, $zero, 1023 ; LA64F-NEXT: ori $s1, $zero, 8 -; LA64F-NEXT: addi.d $s2, $sp, 8 -; LA64F-NEXT: addi.d $s3, $sp, 0 +; LA64F-NEXT: addi.d $s2, $sp, 16 +; LA64F-NEXT: addi.d $s3, $sp, 8 ; LA64F-NEXT: .p2align 4, , 16 ; LA64F-NEXT: .LBB36_1: # %atomicrmw.start ; LA64F-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64F-NEXT: st.d $a0, $sp, 8 +; LA64F-NEXT: move $a0, $s4 ; LA64F-NEXT: move $a1, $s0 ; LA64F-NEXT: bl %plt(__adddf3) -; LA64F-NEXT: st.d $a0, $sp, 0 +; LA64F-NEXT: st.d $s4, $sp, 16 +; LA64F-NEXT: st.d $a0, $sp, 8 ; LA64F-NEXT: move $a0, $s1 ; LA64F-NEXT: move $a1, $fp ; LA64F-NEXT: move $a2, $s2 @@ -3068,17 +3118,18 @@ define double @double_fadd_monotonic(ptr %p) nounwind { ; LA64F-NEXT: move $a4, $zero ; LA64F-NEXT: move $a5, $zero ; LA64F-NEXT: bl %plt(__atomic_compare_exchange) -; LA64F-NEXT: move $a1, $a0 -; LA64F-NEXT: ld.d $a0, $sp, 8 -; LA64F-NEXT: beqz $a1, .LBB36_1 +; LA64F-NEXT: ld.d $s4, $sp, 16 +; LA64F-NEXT: beqz $a0, .LBB36_1 ; LA64F-NEXT: # %bb.2: # %atomicrmw.end -; LA64F-NEXT: ld.d $s3, $sp, 16 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s2, $sp, 24 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s1, $sp, 32 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s0, $sp, 40 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $fp, $sp, 48 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $ra, $sp, 56 # 8-byte Folded Reload -; LA64F-NEXT: addi.d $sp, $sp, 64 +; LA64F-NEXT: move $a0, $s4 +; LA64F-NEXT: ld.d $s4, $sp, 24 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s3, $sp, 32 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s2, $sp, 40 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s1, $sp, 48 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s0, $sp, 56 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $fp, $sp, 64 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $ra, $sp, 72 # 8-byte Folded Reload +; LA64F-NEXT: addi.d $sp, $sp, 80 ; LA64F-NEXT: ret ; ; LA64D-LABEL: double_fadd_monotonic: @@ -3101,9 +3152,9 @@ define double @double_fadd_monotonic(ptr %p) nounwind { ; LA64D-NEXT: .p2align 4, , 16 ; LA64D-NEXT: .LBB36_1: # %atomicrmw.start ; LA64D-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64D-NEXT: fadd.d $fa1, $fa0, $fs0 ; LA64D-NEXT: fst.d $fa0, $sp, 8 -; LA64D-NEXT: fadd.d $fa0, $fa0, $fs0 -; LA64D-NEXT: fst.d $fa0, $sp, 0 +; LA64D-NEXT: fst.d $fa1, $sp, 0 ; LA64D-NEXT: move $a0, $s0 ; LA64D-NEXT: move $a1, $fp ; LA64D-NEXT: move $a2, $s1 @@ -3129,26 +3180,28 @@ define double @double_fadd_monotonic(ptr %p) nounwind { define double @double_fsub_monotonic(ptr %p) nounwind { ; LA64F-LABEL: double_fsub_monotonic: ; LA64F: # %bb.0: -; LA64F-NEXT: addi.d $sp, $sp, -64 -; LA64F-NEXT: st.d $ra, $sp, 56 # 8-byte Folded Spill -; LA64F-NEXT: st.d $fp, $sp, 48 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s0, $sp, 40 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s1, $sp, 32 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s2, $sp, 24 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s3, $sp, 16 # 8-byte Folded Spill +; LA64F-NEXT: addi.d $sp, $sp, -80 +; LA64F-NEXT: st.d $ra, $sp, 72 # 8-byte Folded Spill +; LA64F-NEXT: st.d $fp, $sp, 64 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s0, $sp, 56 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s1, $sp, 48 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s2, $sp, 40 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s3, $sp, 32 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s4, $sp, 24 # 8-byte Folded Spill ; LA64F-NEXT: move $fp, $a0 -; LA64F-NEXT: ld.d $a0, $a0, 0 +; LA64F-NEXT: ld.d $s4, $a0, 0 ; LA64F-NEXT: lu52i.d $s0, $zero, -1025 ; LA64F-NEXT: ori $s1, $zero, 8 -; LA64F-NEXT: addi.d $s2, $sp, 8 -; LA64F-NEXT: addi.d $s3, $sp, 0 +; LA64F-NEXT: addi.d $s2, $sp, 16 +; LA64F-NEXT: addi.d $s3, $sp, 8 ; LA64F-NEXT: .p2align 4, , 16 ; LA64F-NEXT: .LBB37_1: # %atomicrmw.start ; LA64F-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64F-NEXT: st.d $a0, $sp, 8 +; LA64F-NEXT: move $a0, $s4 ; LA64F-NEXT: move $a1, $s0 ; LA64F-NEXT: bl %plt(__adddf3) -; LA64F-NEXT: st.d $a0, $sp, 0 +; LA64F-NEXT: st.d $s4, $sp, 16 +; LA64F-NEXT: st.d $a0, $sp, 8 ; LA64F-NEXT: move $a0, $s1 ; LA64F-NEXT: move $a1, $fp ; LA64F-NEXT: move $a2, $s2 @@ -3156,17 +3209,18 @@ define double @double_fsub_monotonic(ptr %p) nounwind { ; LA64F-NEXT: move $a4, $zero ; LA64F-NEXT: move $a5, $zero ; LA64F-NEXT: bl %plt(__atomic_compare_exchange) -; LA64F-NEXT: move $a1, $a0 -; LA64F-NEXT: ld.d $a0, $sp, 8 -; LA64F-NEXT: beqz $a1, .LBB37_1 +; LA64F-NEXT: ld.d $s4, $sp, 16 +; LA64F-NEXT: beqz $a0, .LBB37_1 ; LA64F-NEXT: # %bb.2: # %atomicrmw.end -; LA64F-NEXT: ld.d $s3, $sp, 16 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s2, $sp, 24 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s1, $sp, 32 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s0, $sp, 40 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $fp, $sp, 48 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $ra, $sp, 56 # 8-byte Folded Reload -; LA64F-NEXT: addi.d $sp, $sp, 64 +; LA64F-NEXT: move $a0, $s4 +; LA64F-NEXT: ld.d $s4, $sp, 24 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s3, $sp, 32 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s2, $sp, 40 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s1, $sp, 48 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s0, $sp, 56 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $fp, $sp, 64 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $ra, $sp, 72 # 8-byte Folded Reload +; LA64F-NEXT: addi.d $sp, $sp, 80 ; LA64F-NEXT: ret ; ; LA64D-LABEL: double_fsub_monotonic: @@ -3189,9 +3243,9 @@ define double @double_fsub_monotonic(ptr %p) nounwind { ; LA64D-NEXT: .p2align 4, , 16 ; LA64D-NEXT: .LBB37_1: # %atomicrmw.start ; LA64D-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64D-NEXT: fadd.d $fa1, $fa0, $fs0 ; LA64D-NEXT: fst.d $fa0, $sp, 8 -; LA64D-NEXT: fadd.d $fa0, $fa0, $fs0 -; LA64D-NEXT: fst.d $fa0, $sp, 0 +; LA64D-NEXT: fst.d $fa1, $sp, 0 ; LA64D-NEXT: move $a0, $s0 ; LA64D-NEXT: move $a1, $fp ; LA64D-NEXT: move $a2, $s1 @@ -3217,26 +3271,28 @@ define double @double_fsub_monotonic(ptr %p) nounwind { define double @double_fmin_monotonic(ptr %p) nounwind { ; LA64F-LABEL: double_fmin_monotonic: ; LA64F: # %bb.0: -; LA64F-NEXT: addi.d $sp, $sp, -64 -; LA64F-NEXT: st.d $ra, $sp, 56 # 8-byte Folded Spill -; LA64F-NEXT: st.d $fp, $sp, 48 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s0, $sp, 40 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s1, $sp, 32 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s2, $sp, 24 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s3, $sp, 16 # 8-byte Folded Spill +; LA64F-NEXT: addi.d $sp, $sp, -80 +; LA64F-NEXT: st.d $ra, $sp, 72 # 8-byte Folded Spill +; LA64F-NEXT: st.d $fp, $sp, 64 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s0, $sp, 56 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s1, $sp, 48 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s2, $sp, 40 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s3, $sp, 32 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s4, $sp, 24 # 8-byte Folded Spill ; LA64F-NEXT: move $fp, $a0 -; LA64F-NEXT: ld.d $a0, $a0, 0 +; LA64F-NEXT: ld.d $s4, $a0, 0 ; LA64F-NEXT: lu52i.d $s0, $zero, 1023 ; LA64F-NEXT: ori $s1, $zero, 8 -; LA64F-NEXT: addi.d $s2, $sp, 8 -; LA64F-NEXT: addi.d $s3, $sp, 0 +; LA64F-NEXT: addi.d $s2, $sp, 16 +; LA64F-NEXT: addi.d $s3, $sp, 8 ; LA64F-NEXT: .p2align 4, , 16 ; LA64F-NEXT: .LBB38_1: # %atomicrmw.start ; LA64F-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64F-NEXT: st.d $a0, $sp, 8 +; LA64F-NEXT: move $a0, $s4 ; LA64F-NEXT: move $a1, $s0 ; LA64F-NEXT: bl %plt(fmin) -; LA64F-NEXT: st.d $a0, $sp, 0 +; LA64F-NEXT: st.d $s4, $sp, 16 +; LA64F-NEXT: st.d $a0, $sp, 8 ; LA64F-NEXT: move $a0, $s1 ; LA64F-NEXT: move $a1, $fp ; LA64F-NEXT: move $a2, $s2 @@ -3244,17 +3300,18 @@ define double @double_fmin_monotonic(ptr %p) nounwind { ; LA64F-NEXT: move $a4, $zero ; LA64F-NEXT: move $a5, $zero ; LA64F-NEXT: bl %plt(__atomic_compare_exchange) -; LA64F-NEXT: move $a1, $a0 -; LA64F-NEXT: ld.d $a0, $sp, 8 -; LA64F-NEXT: beqz $a1, .LBB38_1 +; LA64F-NEXT: ld.d $s4, $sp, 16 +; LA64F-NEXT: beqz $a0, .LBB38_1 ; LA64F-NEXT: # %bb.2: # %atomicrmw.end -; LA64F-NEXT: ld.d $s3, $sp, 16 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s2, $sp, 24 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s1, $sp, 32 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s0, $sp, 40 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $fp, $sp, 48 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $ra, $sp, 56 # 8-byte Folded Reload -; LA64F-NEXT: addi.d $sp, $sp, 64 +; LA64F-NEXT: move $a0, $s4 +; LA64F-NEXT: ld.d $s4, $sp, 24 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s3, $sp, 32 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s2, $sp, 40 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s1, $sp, 48 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s0, $sp, 56 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $fp, $sp, 64 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $ra, $sp, 72 # 8-byte Folded Reload +; LA64F-NEXT: addi.d $sp, $sp, 80 ; LA64F-NEXT: ret ; ; LA64D-LABEL: double_fmin_monotonic: @@ -3277,10 +3334,10 @@ define double @double_fmin_monotonic(ptr %p) nounwind { ; LA64D-NEXT: .p2align 4, , 16 ; LA64D-NEXT: .LBB38_1: # %atomicrmw.start ; LA64D-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64D-NEXT: fmax.d $fa1, $fa0, $fa0 +; LA64D-NEXT: fmin.d $fa1, $fa1, $fs0 ; LA64D-NEXT: fst.d $fa0, $sp, 8 -; LA64D-NEXT: fmax.d $fa0, $fa0, $fa0 -; LA64D-NEXT: fmin.d $fa0, $fa0, $fs0 -; LA64D-NEXT: fst.d $fa0, $sp, 0 +; LA64D-NEXT: fst.d $fa1, $sp, 0 ; LA64D-NEXT: move $a0, $s0 ; LA64D-NEXT: move $a1, $fp ; LA64D-NEXT: move $a2, $s1 @@ -3306,26 +3363,28 @@ define double @double_fmin_monotonic(ptr %p) nounwind { define double @double_fmax_monotonic(ptr %p) nounwind { ; LA64F-LABEL: double_fmax_monotonic: ; LA64F: # %bb.0: -; LA64F-NEXT: addi.d $sp, $sp, -64 -; LA64F-NEXT: st.d $ra, $sp, 56 # 8-byte Folded Spill -; LA64F-NEXT: st.d $fp, $sp, 48 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s0, $sp, 40 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s1, $sp, 32 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s2, $sp, 24 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s3, $sp, 16 # 8-byte Folded Spill +; LA64F-NEXT: addi.d $sp, $sp, -80 +; LA64F-NEXT: st.d $ra, $sp, 72 # 8-byte Folded Spill +; LA64F-NEXT: st.d $fp, $sp, 64 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s0, $sp, 56 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s1, $sp, 48 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s2, $sp, 40 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s3, $sp, 32 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s4, $sp, 24 # 8-byte Folded Spill ; LA64F-NEXT: move $fp, $a0 -; LA64F-NEXT: ld.d $a0, $a0, 0 +; LA64F-NEXT: ld.d $s4, $a0, 0 ; LA64F-NEXT: lu52i.d $s0, $zero, 1023 ; LA64F-NEXT: ori $s1, $zero, 8 -; LA64F-NEXT: addi.d $s2, $sp, 8 -; LA64F-NEXT: addi.d $s3, $sp, 0 +; LA64F-NEXT: addi.d $s2, $sp, 16 +; LA64F-NEXT: addi.d $s3, $sp, 8 ; LA64F-NEXT: .p2align 4, , 16 ; LA64F-NEXT: .LBB39_1: # %atomicrmw.start ; LA64F-NEXT: # =>This Inner Loop Header: Depth=1 -; LA64F-NEXT: st.d $a0, $sp, 8 +; LA64F-NEXT: move $a0, $s4 ; LA64F-NEXT: move $a1, $s0 ; LA64F-NEXT: bl %plt(fmax) -; LA64F-NEXT: st.d $a0, $sp, 0 +; LA64F-NEXT: st.d $s4, $sp, 16 +; LA64F-NEXT: st.d $a0, $sp, 8 ; LA64F-NEXT: move $a0, $s1 ; LA64F-NEXT: move $a1, $fp ; LA64F-NEXT: move $a2, $s2 @@ -3333,17 +3392,18 @@ define double @double_fmax_monotonic(ptr %p) nounwind { ; LA64F-NEXT: move $a4, $zero ; LA64F-NEXT: move $a5, $zero ; LA64F-NEXT: bl %plt(__atomic_compare_exchange) -; LA64F-NEXT: move $a1, $a0 -; LA64F-NEXT: ld.d $a0, $sp, 8 -; LA64F-NEXT: beqz $a1, .LBB39_1 +; LA64F-NEXT: ld.d $s4, $sp, 16 +; LA64F-NEXT: beqz $a0, .LBB39_1 ; LA64F-NEXT: # %bb.2: # %atomicrmw.end -; LA64F-NEXT: ld.d $s3, $sp, 16 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s2, $sp, 24 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s1, $sp, 32 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s0, $sp, 40 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $fp, $sp, 48 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $ra, $sp, 56 # 8-byte Folded Reload -; LA64F-NEXT: addi.d $sp, $sp, 64 +; LA64F-NEXT: move $a0, $s4 +; LA64F-NEXT: ld.d $s4, $sp, 24 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s3, $sp, 32 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s2, $sp, 40 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s1, $sp, 48 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s0, $sp, 56 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $fp, $sp, 64 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $ra, $sp, 72 # 8-byte Folded Reload +; LA64F-NEXT: addi.d $sp, $sp, 80 ; LA64F-NEXT: ret ; ; LA64D-LABEL: double_fmax_monotonic: @@ -3366,10 +3426,10 @@ define double @double_fmax_monotonic(ptr %p) nounwind { ; LA64D-NEXT: .p2align 4, , 16 ; LA64D-NEXT: .LBB39_1: # %atomicrmw.start ; LA64D-NEXT: # =>This Inner Loop Header: Depth=1 +; LA64D-NEXT: fmax.d $fa1, $fa0, $fa0 +; LA64D-NEXT: fmax.d $fa1, $fa1, $fs0 ; LA64D-NEXT: fst.d $fa0, $sp, 8 -; LA64D-NEXT: fmax.d $fa0, $fa0, $fa0 -; LA64D-NEXT: fmax.d $fa0, $fa0, $fs0 -; LA64D-NEXT: fst.d $fa0, $sp, 0 +; LA64D-NEXT: fst.d $fa1, $sp, 0 ; LA64D-NEXT: move $a0, $s0 ; LA64D-NEXT: move $a1, $fp ; LA64D-NEXT: move $a2, $s1 diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-minmax.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-minmax.ll index 464c9ce97c5a..c36734e11f01 100644 --- a/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-minmax.ll +++ b/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-minmax.ll @@ -8,13 +8,13 @@ define i8 @atomicrmw_umax_i8_acquire(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_umax_i8_acquire: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB0_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a6, $a4, $a3 @@ -37,29 +37,29 @@ define i8 @atomicrmw_umax_i8_acquire(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_umax_i16_acquire(ptr %a, i16 %b) nounwind { ; LA64-LABEL: atomicrmw_umax_i16_acquire: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB1_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 -; LA64-NEXT: and $a6, $a4, $a2 +; LA64-NEXT: and $a6, $a4, $a3 ; LA64-NEXT: move $a5, $a4 ; LA64-NEXT: bgeu $a6, $a1, .LBB1_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB1_1 Depth=1 ; LA64-NEXT: xor $a5, $a4, $a1 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: .LBB1_3: # in Loop: Header=BB1_1 Depth=1 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB1_1 ; LA64-NEXT: # %bb.4: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw umax ptr %a, i16 %b acquire ret i16 %1 @@ -89,13 +89,13 @@ define i8 @atomicrmw_umin_i8_acquire(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_umin_i8_acquire: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB4_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a6, $a4, $a3 @@ -118,29 +118,29 @@ define i8 @atomicrmw_umin_i8_acquire(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_umin_i16_acquire(ptr %a, i16 %b) nounwind { ; LA64-LABEL: atomicrmw_umin_i16_acquire: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB5_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 -; LA64-NEXT: and $a6, $a4, $a2 +; LA64-NEXT: and $a6, $a4, $a3 ; LA64-NEXT: move $a5, $a4 ; LA64-NEXT: bgeu $a1, $a6, .LBB5_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB5_1 Depth=1 ; LA64-NEXT: xor $a5, $a4, $a1 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: .LBB5_3: # in Loop: Header=BB5_1 Depth=1 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB5_1 ; LA64-NEXT: # %bb.4: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw umin ptr %a, i16 %b acquire ret i16 %1 @@ -170,25 +170,25 @@ define i8 @atomicrmw_max_i8_acquire(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_max_i8_acquire: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: ori $a3, $zero, 255 -; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: andi $a3, $a2, 24 +; LA64-NEXT: ori $a4, $zero, 255 +; LA64-NEXT: sll.w $a4, $a4, $a2 ; LA64-NEXT: ext.w.b $a1, $a1 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: andi $a4, $a2, 24 -; LA64-NEXT: xori $a4, $a4, 56 +; LA64-NEXT: addi.w $a4, $a4, 0 +; LA64-NEXT: xori $a3, $a3, 56 ; LA64-NEXT: .LBB8_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 -; LA64-NEXT: and $a7, $a5, $a3 +; LA64-NEXT: and $a7, $a5, $a4 ; LA64-NEXT: move $a6, $a5 -; LA64-NEXT: sll.w $a7, $a7, $a4 -; LA64-NEXT: sra.w $a7, $a7, $a4 +; LA64-NEXT: sll.w $a7, $a7, $a3 +; LA64-NEXT: sra.w $a7, $a7, $a3 ; LA64-NEXT: bge $a7, $a1, .LBB8_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB8_1 Depth=1 ; LA64-NEXT: xor $a6, $a5, $a1 -; LA64-NEXT: and $a6, $a6, $a3 +; LA64-NEXT: and $a6, $a6, $a4 ; LA64-NEXT: xor $a6, $a5, $a6 ; LA64-NEXT: .LBB8_3: # in Loop: Header=BB8_1 Depth=1 ; LA64-NEXT: sc.w $a6, $a0, 0 @@ -204,17 +204,17 @@ define i16 @atomicrmw_max_i16_acquire(ptr %a, i16 %b) nounwind { ; LA64-LABEL: atomicrmw_max_i16_acquire: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: andi $a3, $a2, 24 -; LA64-NEXT: ori $a4, $zero, 48 -; LA64-NEXT: sub.d $a3, $a4, $a3 ; LA64-NEXT: lu12i.w $a4, 15 ; LA64-NEXT: ori $a4, $a4, 4095 ; LA64-NEXT: sll.w $a4, $a4, $a2 -; LA64-NEXT: addi.w $a4, $a4, 0 ; LA64-NEXT: ext.w.h $a1, $a1 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a4, $a4, 0 +; LA64-NEXT: ori $a5, $zero, 48 +; LA64-NEXT: sub.d $a3, $a5, $a3 ; LA64-NEXT: .LBB9_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 ; LA64-NEXT: and $a7, $a5, $a4 @@ -260,25 +260,25 @@ define i8 @atomicrmw_min_i8_acquire(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_min_i8_acquire: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: ori $a3, $zero, 255 -; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: andi $a3, $a2, 24 +; LA64-NEXT: ori $a4, $zero, 255 +; LA64-NEXT: sll.w $a4, $a4, $a2 ; LA64-NEXT: ext.w.b $a1, $a1 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: andi $a4, $a2, 24 -; LA64-NEXT: xori $a4, $a4, 56 +; LA64-NEXT: addi.w $a4, $a4, 0 +; LA64-NEXT: xori $a3, $a3, 56 ; LA64-NEXT: .LBB12_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 -; LA64-NEXT: and $a7, $a5, $a3 +; LA64-NEXT: and $a7, $a5, $a4 ; LA64-NEXT: move $a6, $a5 -; LA64-NEXT: sll.w $a7, $a7, $a4 -; LA64-NEXT: sra.w $a7, $a7, $a4 +; LA64-NEXT: sll.w $a7, $a7, $a3 +; LA64-NEXT: sra.w $a7, $a7, $a3 ; LA64-NEXT: bge $a1, $a7, .LBB12_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB12_1 Depth=1 ; LA64-NEXT: xor $a6, $a5, $a1 -; LA64-NEXT: and $a6, $a6, $a3 +; LA64-NEXT: and $a6, $a6, $a4 ; LA64-NEXT: xor $a6, $a5, $a6 ; LA64-NEXT: .LBB12_3: # in Loop: Header=BB12_1 Depth=1 ; LA64-NEXT: sc.w $a6, $a0, 0 @@ -294,17 +294,17 @@ define i16 @atomicrmw_min_i16_acquire(ptr %a, i16 %b) nounwind { ; LA64-LABEL: atomicrmw_min_i16_acquire: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: andi $a3, $a2, 24 -; LA64-NEXT: ori $a4, $zero, 48 -; LA64-NEXT: sub.d $a3, $a4, $a3 ; LA64-NEXT: lu12i.w $a4, 15 ; LA64-NEXT: ori $a4, $a4, 4095 ; LA64-NEXT: sll.w $a4, $a4, $a2 -; LA64-NEXT: addi.w $a4, $a4, 0 ; LA64-NEXT: ext.w.h $a1, $a1 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a4, $a4, 0 +; LA64-NEXT: ori $a5, $zero, 48 +; LA64-NEXT: sub.d $a3, $a5, $a3 ; LA64-NEXT: .LBB13_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 ; LA64-NEXT: and $a7, $a5, $a4 @@ -350,13 +350,13 @@ define i8 @atomicrmw_umax_i8_release(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_umax_i8_release: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB16_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a6, $a4, $a3 @@ -379,29 +379,29 @@ define i8 @atomicrmw_umax_i8_release(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_umax_i16_release(ptr %a, i16 %b) nounwind { ; LA64-LABEL: atomicrmw_umax_i16_release: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB17_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 -; LA64-NEXT: and $a6, $a4, $a2 +; LA64-NEXT: and $a6, $a4, $a3 ; LA64-NEXT: move $a5, $a4 ; LA64-NEXT: bgeu $a6, $a1, .LBB17_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB17_1 Depth=1 ; LA64-NEXT: xor $a5, $a4, $a1 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: .LBB17_3: # in Loop: Header=BB17_1 Depth=1 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB17_1 ; LA64-NEXT: # %bb.4: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw umax ptr %a, i16 %b release ret i16 %1 @@ -431,13 +431,13 @@ define i8 @atomicrmw_umin_i8_release(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_umin_i8_release: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB20_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a6, $a4, $a3 @@ -460,29 +460,29 @@ define i8 @atomicrmw_umin_i8_release(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_umin_i16_release(ptr %a, i16 %b) nounwind { ; LA64-LABEL: atomicrmw_umin_i16_release: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB21_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 -; LA64-NEXT: and $a6, $a4, $a2 +; LA64-NEXT: and $a6, $a4, $a3 ; LA64-NEXT: move $a5, $a4 ; LA64-NEXT: bgeu $a1, $a6, .LBB21_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB21_1 Depth=1 ; LA64-NEXT: xor $a5, $a4, $a1 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: .LBB21_3: # in Loop: Header=BB21_1 Depth=1 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB21_1 ; LA64-NEXT: # %bb.4: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw umin ptr %a, i16 %b release ret i16 %1 @@ -512,25 +512,25 @@ define i8 @atomicrmw_max_i8_release(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_max_i8_release: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: ori $a3, $zero, 255 -; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: andi $a3, $a2, 24 +; LA64-NEXT: ori $a4, $zero, 255 +; LA64-NEXT: sll.w $a4, $a4, $a2 ; LA64-NEXT: ext.w.b $a1, $a1 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: andi $a4, $a2, 24 -; LA64-NEXT: xori $a4, $a4, 56 +; LA64-NEXT: addi.w $a4, $a4, 0 +; LA64-NEXT: xori $a3, $a3, 56 ; LA64-NEXT: .LBB24_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 -; LA64-NEXT: and $a7, $a5, $a3 +; LA64-NEXT: and $a7, $a5, $a4 ; LA64-NEXT: move $a6, $a5 -; LA64-NEXT: sll.w $a7, $a7, $a4 -; LA64-NEXT: sra.w $a7, $a7, $a4 +; LA64-NEXT: sll.w $a7, $a7, $a3 +; LA64-NEXT: sra.w $a7, $a7, $a3 ; LA64-NEXT: bge $a7, $a1, .LBB24_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB24_1 Depth=1 ; LA64-NEXT: xor $a6, $a5, $a1 -; LA64-NEXT: and $a6, $a6, $a3 +; LA64-NEXT: and $a6, $a6, $a4 ; LA64-NEXT: xor $a6, $a5, $a6 ; LA64-NEXT: .LBB24_3: # in Loop: Header=BB24_1 Depth=1 ; LA64-NEXT: sc.w $a6, $a0, 0 @@ -546,17 +546,17 @@ define i16 @atomicrmw_max_i16_release(ptr %a, i16 %b) nounwind { ; LA64-LABEL: atomicrmw_max_i16_release: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: andi $a3, $a2, 24 -; LA64-NEXT: ori $a4, $zero, 48 -; LA64-NEXT: sub.d $a3, $a4, $a3 ; LA64-NEXT: lu12i.w $a4, 15 ; LA64-NEXT: ori $a4, $a4, 4095 ; LA64-NEXT: sll.w $a4, $a4, $a2 -; LA64-NEXT: addi.w $a4, $a4, 0 ; LA64-NEXT: ext.w.h $a1, $a1 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a4, $a4, 0 +; LA64-NEXT: ori $a5, $zero, 48 +; LA64-NEXT: sub.d $a3, $a5, $a3 ; LA64-NEXT: .LBB25_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 ; LA64-NEXT: and $a7, $a5, $a4 @@ -602,25 +602,25 @@ define i8 @atomicrmw_min_i8_release(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_min_i8_release: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: ori $a3, $zero, 255 -; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: andi $a3, $a2, 24 +; LA64-NEXT: ori $a4, $zero, 255 +; LA64-NEXT: sll.w $a4, $a4, $a2 ; LA64-NEXT: ext.w.b $a1, $a1 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: andi $a4, $a2, 24 -; LA64-NEXT: xori $a4, $a4, 56 +; LA64-NEXT: addi.w $a4, $a4, 0 +; LA64-NEXT: xori $a3, $a3, 56 ; LA64-NEXT: .LBB28_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 -; LA64-NEXT: and $a7, $a5, $a3 +; LA64-NEXT: and $a7, $a5, $a4 ; LA64-NEXT: move $a6, $a5 -; LA64-NEXT: sll.w $a7, $a7, $a4 -; LA64-NEXT: sra.w $a7, $a7, $a4 +; LA64-NEXT: sll.w $a7, $a7, $a3 +; LA64-NEXT: sra.w $a7, $a7, $a3 ; LA64-NEXT: bge $a1, $a7, .LBB28_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB28_1 Depth=1 ; LA64-NEXT: xor $a6, $a5, $a1 -; LA64-NEXT: and $a6, $a6, $a3 +; LA64-NEXT: and $a6, $a6, $a4 ; LA64-NEXT: xor $a6, $a5, $a6 ; LA64-NEXT: .LBB28_3: # in Loop: Header=BB28_1 Depth=1 ; LA64-NEXT: sc.w $a6, $a0, 0 @@ -636,17 +636,17 @@ define i16 @atomicrmw_min_i16_release(ptr %a, i16 %b) nounwind { ; LA64-LABEL: atomicrmw_min_i16_release: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: andi $a3, $a2, 24 -; LA64-NEXT: ori $a4, $zero, 48 -; LA64-NEXT: sub.d $a3, $a4, $a3 ; LA64-NEXT: lu12i.w $a4, 15 ; LA64-NEXT: ori $a4, $a4, 4095 ; LA64-NEXT: sll.w $a4, $a4, $a2 -; LA64-NEXT: addi.w $a4, $a4, 0 ; LA64-NEXT: ext.w.h $a1, $a1 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a4, $a4, 0 +; LA64-NEXT: ori $a5, $zero, 48 +; LA64-NEXT: sub.d $a3, $a5, $a3 ; LA64-NEXT: .LBB29_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 ; LA64-NEXT: and $a7, $a5, $a4 @@ -692,13 +692,13 @@ define i8 @atomicrmw_umax_i8_acq_rel(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_umax_i8_acq_rel: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB32_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a6, $a4, $a3 @@ -721,29 +721,29 @@ define i8 @atomicrmw_umax_i8_acq_rel(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_umax_i16_acq_rel(ptr %a, i16 %b) nounwind { ; LA64-LABEL: atomicrmw_umax_i16_acq_rel: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB33_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 -; LA64-NEXT: and $a6, $a4, $a2 +; LA64-NEXT: and $a6, $a4, $a3 ; LA64-NEXT: move $a5, $a4 ; LA64-NEXT: bgeu $a6, $a1, .LBB33_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB33_1 Depth=1 ; LA64-NEXT: xor $a5, $a4, $a1 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: .LBB33_3: # in Loop: Header=BB33_1 Depth=1 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB33_1 ; LA64-NEXT: # %bb.4: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw umax ptr %a, i16 %b acq_rel ret i16 %1 @@ -773,13 +773,13 @@ define i8 @atomicrmw_umin_i8_acq_rel(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_umin_i8_acq_rel: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB36_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a6, $a4, $a3 @@ -802,29 +802,29 @@ define i8 @atomicrmw_umin_i8_acq_rel(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_umin_i16_acq_rel(ptr %a, i16 %b) nounwind { ; LA64-LABEL: atomicrmw_umin_i16_acq_rel: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB37_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 -; LA64-NEXT: and $a6, $a4, $a2 +; LA64-NEXT: and $a6, $a4, $a3 ; LA64-NEXT: move $a5, $a4 ; LA64-NEXT: bgeu $a1, $a6, .LBB37_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB37_1 Depth=1 ; LA64-NEXT: xor $a5, $a4, $a1 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: .LBB37_3: # in Loop: Header=BB37_1 Depth=1 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB37_1 ; LA64-NEXT: # %bb.4: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw umin ptr %a, i16 %b acq_rel ret i16 %1 @@ -854,25 +854,25 @@ define i8 @atomicrmw_max_i8_acq_rel(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_max_i8_acq_rel: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: ori $a3, $zero, 255 -; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: andi $a3, $a2, 24 +; LA64-NEXT: ori $a4, $zero, 255 +; LA64-NEXT: sll.w $a4, $a4, $a2 ; LA64-NEXT: ext.w.b $a1, $a1 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: andi $a4, $a2, 24 -; LA64-NEXT: xori $a4, $a4, 56 +; LA64-NEXT: addi.w $a4, $a4, 0 +; LA64-NEXT: xori $a3, $a3, 56 ; LA64-NEXT: .LBB40_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 -; LA64-NEXT: and $a7, $a5, $a3 +; LA64-NEXT: and $a7, $a5, $a4 ; LA64-NEXT: move $a6, $a5 -; LA64-NEXT: sll.w $a7, $a7, $a4 -; LA64-NEXT: sra.w $a7, $a7, $a4 +; LA64-NEXT: sll.w $a7, $a7, $a3 +; LA64-NEXT: sra.w $a7, $a7, $a3 ; LA64-NEXT: bge $a7, $a1, .LBB40_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB40_1 Depth=1 ; LA64-NEXT: xor $a6, $a5, $a1 -; LA64-NEXT: and $a6, $a6, $a3 +; LA64-NEXT: and $a6, $a6, $a4 ; LA64-NEXT: xor $a6, $a5, $a6 ; LA64-NEXT: .LBB40_3: # in Loop: Header=BB40_1 Depth=1 ; LA64-NEXT: sc.w $a6, $a0, 0 @@ -888,17 +888,17 @@ define i16 @atomicrmw_max_i16_acq_rel(ptr %a, i16 %b) nounwind { ; LA64-LABEL: atomicrmw_max_i16_acq_rel: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: andi $a3, $a2, 24 -; LA64-NEXT: ori $a4, $zero, 48 -; LA64-NEXT: sub.d $a3, $a4, $a3 ; LA64-NEXT: lu12i.w $a4, 15 ; LA64-NEXT: ori $a4, $a4, 4095 ; LA64-NEXT: sll.w $a4, $a4, $a2 -; LA64-NEXT: addi.w $a4, $a4, 0 ; LA64-NEXT: ext.w.h $a1, $a1 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a4, $a4, 0 +; LA64-NEXT: ori $a5, $zero, 48 +; LA64-NEXT: sub.d $a3, $a5, $a3 ; LA64-NEXT: .LBB41_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 ; LA64-NEXT: and $a7, $a5, $a4 @@ -944,25 +944,25 @@ define i8 @atomicrmw_min_i8_acq_rel(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_min_i8_acq_rel: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: ori $a3, $zero, 255 -; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: andi $a3, $a2, 24 +; LA64-NEXT: ori $a4, $zero, 255 +; LA64-NEXT: sll.w $a4, $a4, $a2 ; LA64-NEXT: ext.w.b $a1, $a1 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: andi $a4, $a2, 24 -; LA64-NEXT: xori $a4, $a4, 56 +; LA64-NEXT: addi.w $a4, $a4, 0 +; LA64-NEXT: xori $a3, $a3, 56 ; LA64-NEXT: .LBB44_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 -; LA64-NEXT: and $a7, $a5, $a3 +; LA64-NEXT: and $a7, $a5, $a4 ; LA64-NEXT: move $a6, $a5 -; LA64-NEXT: sll.w $a7, $a7, $a4 -; LA64-NEXT: sra.w $a7, $a7, $a4 +; LA64-NEXT: sll.w $a7, $a7, $a3 +; LA64-NEXT: sra.w $a7, $a7, $a3 ; LA64-NEXT: bge $a1, $a7, .LBB44_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB44_1 Depth=1 ; LA64-NEXT: xor $a6, $a5, $a1 -; LA64-NEXT: and $a6, $a6, $a3 +; LA64-NEXT: and $a6, $a6, $a4 ; LA64-NEXT: xor $a6, $a5, $a6 ; LA64-NEXT: .LBB44_3: # in Loop: Header=BB44_1 Depth=1 ; LA64-NEXT: sc.w $a6, $a0, 0 @@ -978,17 +978,17 @@ define i16 @atomicrmw_min_i16_acq_rel(ptr %a, i16 %b) nounwind { ; LA64-LABEL: atomicrmw_min_i16_acq_rel: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: andi $a3, $a2, 24 -; LA64-NEXT: ori $a4, $zero, 48 -; LA64-NEXT: sub.d $a3, $a4, $a3 ; LA64-NEXT: lu12i.w $a4, 15 ; LA64-NEXT: ori $a4, $a4, 4095 ; LA64-NEXT: sll.w $a4, $a4, $a2 -; LA64-NEXT: addi.w $a4, $a4, 0 ; LA64-NEXT: ext.w.h $a1, $a1 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a4, $a4, 0 +; LA64-NEXT: ori $a5, $zero, 48 +; LA64-NEXT: sub.d $a3, $a5, $a3 ; LA64-NEXT: .LBB45_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 ; LA64-NEXT: and $a7, $a5, $a4 @@ -1034,13 +1034,13 @@ define i8 @atomicrmw_umax_i8_seq_cst(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_umax_i8_seq_cst: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB48_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a6, $a4, $a3 @@ -1063,29 +1063,29 @@ define i8 @atomicrmw_umax_i8_seq_cst(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_umax_i16_seq_cst(ptr %a, i16 %b) nounwind { ; LA64-LABEL: atomicrmw_umax_i16_seq_cst: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB49_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 -; LA64-NEXT: and $a6, $a4, $a2 +; LA64-NEXT: and $a6, $a4, $a3 ; LA64-NEXT: move $a5, $a4 ; LA64-NEXT: bgeu $a6, $a1, .LBB49_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB49_1 Depth=1 ; LA64-NEXT: xor $a5, $a4, $a1 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: .LBB49_3: # in Loop: Header=BB49_1 Depth=1 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB49_1 ; LA64-NEXT: # %bb.4: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw umax ptr %a, i16 %b seq_cst ret i16 %1 @@ -1115,13 +1115,13 @@ define i8 @atomicrmw_umin_i8_seq_cst(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_umin_i8_seq_cst: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB52_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a6, $a4, $a3 @@ -1144,29 +1144,29 @@ define i8 @atomicrmw_umin_i8_seq_cst(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_umin_i16_seq_cst(ptr %a, i16 %b) nounwind { ; LA64-LABEL: atomicrmw_umin_i16_seq_cst: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB53_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 -; LA64-NEXT: and $a6, $a4, $a2 +; LA64-NEXT: and $a6, $a4, $a3 ; LA64-NEXT: move $a5, $a4 ; LA64-NEXT: bgeu $a1, $a6, .LBB53_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB53_1 Depth=1 ; LA64-NEXT: xor $a5, $a4, $a1 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: .LBB53_3: # in Loop: Header=BB53_1 Depth=1 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB53_1 ; LA64-NEXT: # %bb.4: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw umin ptr %a, i16 %b seq_cst ret i16 %1 @@ -1196,25 +1196,25 @@ define i8 @atomicrmw_max_i8_seq_cst(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_max_i8_seq_cst: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: ori $a3, $zero, 255 -; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: andi $a3, $a2, 24 +; LA64-NEXT: ori $a4, $zero, 255 +; LA64-NEXT: sll.w $a4, $a4, $a2 ; LA64-NEXT: ext.w.b $a1, $a1 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: andi $a4, $a2, 24 -; LA64-NEXT: xori $a4, $a4, 56 +; LA64-NEXT: addi.w $a4, $a4, 0 +; LA64-NEXT: xori $a3, $a3, 56 ; LA64-NEXT: .LBB56_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 -; LA64-NEXT: and $a7, $a5, $a3 +; LA64-NEXT: and $a7, $a5, $a4 ; LA64-NEXT: move $a6, $a5 -; LA64-NEXT: sll.w $a7, $a7, $a4 -; LA64-NEXT: sra.w $a7, $a7, $a4 +; LA64-NEXT: sll.w $a7, $a7, $a3 +; LA64-NEXT: sra.w $a7, $a7, $a3 ; LA64-NEXT: bge $a7, $a1, .LBB56_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB56_1 Depth=1 ; LA64-NEXT: xor $a6, $a5, $a1 -; LA64-NEXT: and $a6, $a6, $a3 +; LA64-NEXT: and $a6, $a6, $a4 ; LA64-NEXT: xor $a6, $a5, $a6 ; LA64-NEXT: .LBB56_3: # in Loop: Header=BB56_1 Depth=1 ; LA64-NEXT: sc.w $a6, $a0, 0 @@ -1230,17 +1230,17 @@ define i16 @atomicrmw_max_i16_seq_cst(ptr %a, i16 %b) nounwind { ; LA64-LABEL: atomicrmw_max_i16_seq_cst: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: andi $a3, $a2, 24 -; LA64-NEXT: ori $a4, $zero, 48 -; LA64-NEXT: sub.d $a3, $a4, $a3 ; LA64-NEXT: lu12i.w $a4, 15 ; LA64-NEXT: ori $a4, $a4, 4095 ; LA64-NEXT: sll.w $a4, $a4, $a2 -; LA64-NEXT: addi.w $a4, $a4, 0 ; LA64-NEXT: ext.w.h $a1, $a1 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a4, $a4, 0 +; LA64-NEXT: ori $a5, $zero, 48 +; LA64-NEXT: sub.d $a3, $a5, $a3 ; LA64-NEXT: .LBB57_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 ; LA64-NEXT: and $a7, $a5, $a4 @@ -1286,25 +1286,25 @@ define i8 @atomicrmw_min_i8_seq_cst(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_min_i8_seq_cst: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: ori $a3, $zero, 255 -; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: andi $a3, $a2, 24 +; LA64-NEXT: ori $a4, $zero, 255 +; LA64-NEXT: sll.w $a4, $a4, $a2 ; LA64-NEXT: ext.w.b $a1, $a1 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: andi $a4, $a2, 24 -; LA64-NEXT: xori $a4, $a4, 56 +; LA64-NEXT: addi.w $a4, $a4, 0 +; LA64-NEXT: xori $a3, $a3, 56 ; LA64-NEXT: .LBB60_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 -; LA64-NEXT: and $a7, $a5, $a3 +; LA64-NEXT: and $a7, $a5, $a4 ; LA64-NEXT: move $a6, $a5 -; LA64-NEXT: sll.w $a7, $a7, $a4 -; LA64-NEXT: sra.w $a7, $a7, $a4 +; LA64-NEXT: sll.w $a7, $a7, $a3 +; LA64-NEXT: sra.w $a7, $a7, $a3 ; LA64-NEXT: bge $a1, $a7, .LBB60_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB60_1 Depth=1 ; LA64-NEXT: xor $a6, $a5, $a1 -; LA64-NEXT: and $a6, $a6, $a3 +; LA64-NEXT: and $a6, $a6, $a4 ; LA64-NEXT: xor $a6, $a5, $a6 ; LA64-NEXT: .LBB60_3: # in Loop: Header=BB60_1 Depth=1 ; LA64-NEXT: sc.w $a6, $a0, 0 @@ -1320,17 +1320,17 @@ define i16 @atomicrmw_min_i16_seq_cst(ptr %a, i16 %b) nounwind { ; LA64-LABEL: atomicrmw_min_i16_seq_cst: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: andi $a3, $a2, 24 -; LA64-NEXT: ori $a4, $zero, 48 -; LA64-NEXT: sub.d $a3, $a4, $a3 ; LA64-NEXT: lu12i.w $a4, 15 ; LA64-NEXT: ori $a4, $a4, 4095 ; LA64-NEXT: sll.w $a4, $a4, $a2 -; LA64-NEXT: addi.w $a4, $a4, 0 ; LA64-NEXT: ext.w.h $a1, $a1 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a4, $a4, 0 +; LA64-NEXT: ori $a5, $zero, 48 +; LA64-NEXT: sub.d $a3, $a5, $a3 ; LA64-NEXT: .LBB61_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 ; LA64-NEXT: and $a7, $a5, $a4 @@ -1376,13 +1376,13 @@ define i8 @atomicrmw_umax_i8_monotonic(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_umax_i8_monotonic: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB64_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a6, $a4, $a3 @@ -1405,29 +1405,29 @@ define i8 @atomicrmw_umax_i8_monotonic(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_umax_i16_monotonic(ptr %a, i16 %b) nounwind { ; LA64-LABEL: atomicrmw_umax_i16_monotonic: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB65_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 -; LA64-NEXT: and $a6, $a4, $a2 +; LA64-NEXT: and $a6, $a4, $a3 ; LA64-NEXT: move $a5, $a4 ; LA64-NEXT: bgeu $a6, $a1, .LBB65_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB65_1 Depth=1 ; LA64-NEXT: xor $a5, $a4, $a1 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: .LBB65_3: # in Loop: Header=BB65_1 Depth=1 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB65_1 ; LA64-NEXT: # %bb.4: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw umax ptr %a, i16 %b monotonic ret i16 %1 @@ -1457,13 +1457,13 @@ define i8 @atomicrmw_umin_i8_monotonic(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_umin_i8_monotonic: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB68_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a6, $a4, $a3 @@ -1486,29 +1486,29 @@ define i8 @atomicrmw_umin_i8_monotonic(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_umin_i16_monotonic(ptr %a, i16 %b) nounwind { ; LA64-LABEL: atomicrmw_umin_i16_monotonic: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB69_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 -; LA64-NEXT: and $a6, $a4, $a2 +; LA64-NEXT: and $a6, $a4, $a3 ; LA64-NEXT: move $a5, $a4 ; LA64-NEXT: bgeu $a1, $a6, .LBB69_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB69_1 Depth=1 ; LA64-NEXT: xor $a5, $a4, $a1 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: .LBB69_3: # in Loop: Header=BB69_1 Depth=1 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB69_1 ; LA64-NEXT: # %bb.4: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw umin ptr %a, i16 %b monotonic ret i16 %1 @@ -1538,25 +1538,25 @@ define i8 @atomicrmw_max_i8_monotonic(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_max_i8_monotonic: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: ori $a3, $zero, 255 -; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: andi $a3, $a2, 24 +; LA64-NEXT: ori $a4, $zero, 255 +; LA64-NEXT: sll.w $a4, $a4, $a2 ; LA64-NEXT: ext.w.b $a1, $a1 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: andi $a4, $a2, 24 -; LA64-NEXT: xori $a4, $a4, 56 +; LA64-NEXT: addi.w $a4, $a4, 0 +; LA64-NEXT: xori $a3, $a3, 56 ; LA64-NEXT: .LBB72_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 -; LA64-NEXT: and $a7, $a5, $a3 +; LA64-NEXT: and $a7, $a5, $a4 ; LA64-NEXT: move $a6, $a5 -; LA64-NEXT: sll.w $a7, $a7, $a4 -; LA64-NEXT: sra.w $a7, $a7, $a4 +; LA64-NEXT: sll.w $a7, $a7, $a3 +; LA64-NEXT: sra.w $a7, $a7, $a3 ; LA64-NEXT: bge $a7, $a1, .LBB72_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB72_1 Depth=1 ; LA64-NEXT: xor $a6, $a5, $a1 -; LA64-NEXT: and $a6, $a6, $a3 +; LA64-NEXT: and $a6, $a6, $a4 ; LA64-NEXT: xor $a6, $a5, $a6 ; LA64-NEXT: .LBB72_3: # in Loop: Header=BB72_1 Depth=1 ; LA64-NEXT: sc.w $a6, $a0, 0 @@ -1572,17 +1572,17 @@ define i16 @atomicrmw_max_i16_monotonic(ptr %a, i16 %b) nounwind { ; LA64-LABEL: atomicrmw_max_i16_monotonic: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: andi $a3, $a2, 24 -; LA64-NEXT: ori $a4, $zero, 48 -; LA64-NEXT: sub.d $a3, $a4, $a3 ; LA64-NEXT: lu12i.w $a4, 15 ; LA64-NEXT: ori $a4, $a4, 4095 ; LA64-NEXT: sll.w $a4, $a4, $a2 -; LA64-NEXT: addi.w $a4, $a4, 0 ; LA64-NEXT: ext.w.h $a1, $a1 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a4, $a4, 0 +; LA64-NEXT: ori $a5, $zero, 48 +; LA64-NEXT: sub.d $a3, $a5, $a3 ; LA64-NEXT: .LBB73_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 ; LA64-NEXT: and $a7, $a5, $a4 @@ -1628,25 +1628,25 @@ define i8 @atomicrmw_min_i8_monotonic(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_min_i8_monotonic: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: ori $a3, $zero, 255 -; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: andi $a3, $a2, 24 +; LA64-NEXT: ori $a4, $zero, 255 +; LA64-NEXT: sll.w $a4, $a4, $a2 ; LA64-NEXT: ext.w.b $a1, $a1 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: andi $a4, $a2, 24 -; LA64-NEXT: xori $a4, $a4, 56 +; LA64-NEXT: addi.w $a4, $a4, 0 +; LA64-NEXT: xori $a3, $a3, 56 ; LA64-NEXT: .LBB76_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 -; LA64-NEXT: and $a7, $a5, $a3 +; LA64-NEXT: and $a7, $a5, $a4 ; LA64-NEXT: move $a6, $a5 -; LA64-NEXT: sll.w $a7, $a7, $a4 -; LA64-NEXT: sra.w $a7, $a7, $a4 +; LA64-NEXT: sll.w $a7, $a7, $a3 +; LA64-NEXT: sra.w $a7, $a7, $a3 ; LA64-NEXT: bge $a1, $a7, .LBB76_3 ; LA64-NEXT: # %bb.2: # in Loop: Header=BB76_1 Depth=1 ; LA64-NEXT: xor $a6, $a5, $a1 -; LA64-NEXT: and $a6, $a6, $a3 +; LA64-NEXT: and $a6, $a6, $a4 ; LA64-NEXT: xor $a6, $a5, $a6 ; LA64-NEXT: .LBB76_3: # in Loop: Header=BB76_1 Depth=1 ; LA64-NEXT: sc.w $a6, $a0, 0 @@ -1662,17 +1662,17 @@ define i16 @atomicrmw_min_i16_monotonic(ptr %a, i16 %b) nounwind { ; LA64-LABEL: atomicrmw_min_i16_monotonic: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: andi $a3, $a2, 24 -; LA64-NEXT: ori $a4, $zero, 48 -; LA64-NEXT: sub.d $a3, $a4, $a3 ; LA64-NEXT: lu12i.w $a4, 15 ; LA64-NEXT: ori $a4, $a4, 4095 ; LA64-NEXT: sll.w $a4, $a4, $a2 -; LA64-NEXT: addi.w $a4, $a4, 0 ; LA64-NEXT: ext.w.h $a1, $a1 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a4, $a4, 0 +; LA64-NEXT: ori $a5, $zero, 48 +; LA64-NEXT: sub.d $a3, $a5, $a3 ; LA64-NEXT: .LBB77_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a5, $a0, 0 ; LA64-NEXT: and $a7, $a5, $a4 diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw.ll index d4f7ed017121..4669065114f0 100644 --- a/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw.ll +++ b/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw.ll @@ -6,11 +6,11 @@ define i8 @atomicrmw_xchg_i8_acquire(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_xchg_i8_acquire: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB0_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: addi.w $a5, $a1, 0 @@ -26,13 +26,13 @@ define i8 @atomicrmw_xchg_i8_acquire(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_xchg_i8_acquire: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB0_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: addi.w $a5, $a1, 0 @@ -52,10 +52,10 @@ define i8 @atomicrmw_xchg_0_i8_acquire(ptr %a) nounwind { ; LA32-LABEL: atomicrmw_xchg_0_i8_acquire: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a1, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a2, $zero, 255 ; LA32-NEXT: sll.w $a2, $a2, $a1 ; LA32-NEXT: nor $a2, $a2, $zero -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB1_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: and $a4, $a3, $a2 @@ -68,10 +68,10 @@ define i8 @atomicrmw_xchg_0_i8_acquire(ptr %a) nounwind { ; LA64-LABEL: atomicrmw_xchg_0_i8_acquire: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a1, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a2, $zero, 255 ; LA64-NEXT: sll.w $a2, $a2, $a1 ; LA64-NEXT: nor $a2, $a2, $zero -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: amand_db.w $a3, $a2, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a1 ; LA64-NEXT: ret @@ -83,9 +83,9 @@ define i8 @atomicrmw_xchg_minus_1_i8_acquire(ptr %a) nounwind { ; LA32-LABEL: atomicrmw_xchg_minus_1_i8_acquire: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a1, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a2, $zero, 255 ; LA32-NEXT: sll.w $a2, $a2, $a1 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB2_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: or $a4, $a3, $a2 @@ -98,9 +98,9 @@ define i8 @atomicrmw_xchg_minus_1_i8_acquire(ptr %a) nounwind { ; LA64-LABEL: atomicrmw_xchg_minus_1_i8_acquire: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a1, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a2, $zero, 255 ; LA64-NEXT: sll.w $a2, $a2, $a1 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: amor_db.w $a3, $a2, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a1 ; LA64-NEXT: ret @@ -111,46 +111,46 @@ define i8 @atomicrmw_xchg_minus_1_i8_acquire(ptr %a) nounwind { define i16 @atomicrmw_xchg_i16_acquire(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_xchg_i16_acquire: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB3_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: addi.w $a5, $a1, 0 ; LA32-NEXT: xor $a5, $a4, $a5 -; LA32-NEXT: and $a5, $a5, $a2 +; LA32-NEXT: and $a5, $a5, $a3 ; LA32-NEXT: xor $a5, $a4, $a5 ; LA32-NEXT: sc.w $a5, $a0, 0 ; LA32-NEXT: beqz $a5, .LBB3_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a4, $a3 +; LA32-NEXT: srl.w $a0, $a4, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_xchg_i16_acquire: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB3_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: addi.w $a5, $a1, 0 ; LA64-NEXT: xor $a5, $a4, $a5 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB3_1 ; LA64-NEXT: # %bb.2: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw xchg ptr %a, i16 %b acquire ret i16 %1 @@ -159,31 +159,31 @@ define i16 @atomicrmw_xchg_i16_acquire(ptr %a, i16 %b) nounwind { define i16 @atomicrmw_xchg_0_i16_acquire(ptr %a) nounwind { ; LA32-LABEL: atomicrmw_xchg_0_i16_acquire: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a1, 15 -; LA32-NEXT: ori $a1, $a1, 4095 -; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: nor $a1, $a1, $zero +; LA32-NEXT: slli.w $a1, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a2, 15 +; LA32-NEXT: ori $a2, $a2, 4095 +; LA32-NEXT: sll.w $a2, $a2, $a1 +; LA32-NEXT: nor $a2, $a2, $zero ; LA32-NEXT: .LBB4_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 -; LA32-NEXT: and $a4, $a3, $a1 +; LA32-NEXT: and $a4, $a3, $a2 ; LA32-NEXT: sc.w $a4, $a0, 0 ; LA32-NEXT: beqz $a4, .LBB4_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a3, $a2 +; LA32-NEXT: srl.w $a0, $a3, $a1 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_xchg_0_i16_acquire: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a1, 15 -; LA64-NEXT: ori $a1, $a1, 4095 -; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 -; LA64-NEXT: nor $a1, $a1, $zero +; LA64-NEXT: slli.d $a1, $a0, 3 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: amand_db.w $a3, $a1, $a0 -; LA64-NEXT: srl.w $a0, $a3, $a2 +; LA64-NEXT: lu12i.w $a2, 15 +; LA64-NEXT: ori $a2, $a2, 4095 +; LA64-NEXT: sll.w $a2, $a2, $a1 +; LA64-NEXT: nor $a2, $a2, $zero +; LA64-NEXT: amand_db.w $a3, $a2, $a0 +; LA64-NEXT: srl.w $a0, $a3, $a1 ; LA64-NEXT: ret %1 = atomicrmw xchg ptr %a, i16 0 acquire ret i16 %1 @@ -192,29 +192,29 @@ define i16 @atomicrmw_xchg_0_i16_acquire(ptr %a) nounwind { define i16 @atomicrmw_xchg_minus_1_i16_acquire(ptr %a) nounwind { ; LA32-LABEL: atomicrmw_xchg_minus_1_i16_acquire: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a1, 15 -; LA32-NEXT: ori $a1, $a1, 4095 -; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 +; LA32-NEXT: slli.w $a1, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a2, 15 +; LA32-NEXT: ori $a2, $a2, 4095 +; LA32-NEXT: sll.w $a2, $a2, $a1 ; LA32-NEXT: .LBB5_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 -; LA32-NEXT: or $a4, $a3, $a1 +; LA32-NEXT: or $a4, $a3, $a2 ; LA32-NEXT: sc.w $a4, $a0, 0 ; LA32-NEXT: beqz $a4, .LBB5_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a3, $a2 +; LA32-NEXT: srl.w $a0, $a3, $a1 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_xchg_minus_1_i16_acquire: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a1, 15 -; LA64-NEXT: ori $a1, $a1, 4095 -; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 +; LA64-NEXT: slli.d $a1, $a0, 3 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: amor_db.w $a3, $a1, $a0 -; LA64-NEXT: srl.w $a0, $a3, $a2 +; LA64-NEXT: lu12i.w $a2, 15 +; LA64-NEXT: ori $a2, $a2, 4095 +; LA64-NEXT: sll.w $a2, $a2, $a1 +; LA64-NEXT: amor_db.w $a3, $a2, $a0 +; LA64-NEXT: srl.w $a0, $a3, $a1 ; LA64-NEXT: ret %1 = atomicrmw xchg ptr %a, i16 -1 acquire ret i16 %1 @@ -265,11 +265,11 @@ define i8 @atomicrmw_add_i8_acquire(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_add_i8_acquire: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB8_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: add.w $a5, $a4, $a1 @@ -285,13 +285,13 @@ define i8 @atomicrmw_add_i8_acquire(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_add_i8_acquire: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB8_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: add.w $a5, $a4, $a1 @@ -310,46 +310,46 @@ define i8 @atomicrmw_add_i8_acquire(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_add_i16_acquire(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_add_i16_acquire: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB9_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: add.w $a5, $a4, $a1 ; LA32-NEXT: xor $a5, $a4, $a5 -; LA32-NEXT: and $a5, $a5, $a2 +; LA32-NEXT: and $a5, $a5, $a3 ; LA32-NEXT: xor $a5, $a4, $a5 ; LA32-NEXT: sc.w $a5, $a0, 0 ; LA32-NEXT: beqz $a5, .LBB9_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a4, $a3 +; LA32-NEXT: srl.w $a0, $a4, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_add_i16_acquire: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB9_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: add.w $a5, $a4, $a1 ; LA64-NEXT: xor $a5, $a4, $a5 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB9_1 ; LA64-NEXT: # %bb.2: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw add ptr %a, i16 %b acquire ret i16 %1 @@ -400,11 +400,11 @@ define i8 @atomicrmw_sub_i8_acquire(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_sub_i8_acquire: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB12_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: sub.w $a5, $a4, $a1 @@ -420,13 +420,13 @@ define i8 @atomicrmw_sub_i8_acquire(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_sub_i8_acquire: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB12_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: sub.w $a5, $a4, $a1 @@ -445,46 +445,46 @@ define i8 @atomicrmw_sub_i8_acquire(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_sub_i16_acquire(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_sub_i16_acquire: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB13_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: sub.w $a5, $a4, $a1 ; LA32-NEXT: xor $a5, $a4, $a5 -; LA32-NEXT: and $a5, $a5, $a2 +; LA32-NEXT: and $a5, $a5, $a3 ; LA32-NEXT: xor $a5, $a4, $a5 ; LA32-NEXT: sc.w $a5, $a0, 0 ; LA32-NEXT: beqz $a5, .LBB13_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a4, $a3 +; LA32-NEXT: srl.w $a0, $a4, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_sub_i16_acquire: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB13_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: sub.w $a5, $a4, $a1 ; LA64-NEXT: xor $a5, $a4, $a5 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB13_1 ; LA64-NEXT: # %bb.2: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw sub ptr %a, i16 %b acquire ret i16 %1 @@ -537,11 +537,11 @@ define i8 @atomicrmw_nand_i8_acquire(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_nand_i8_acquire: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB16_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: and $a5, $a4, $a1 @@ -558,13 +558,13 @@ define i8 @atomicrmw_nand_i8_acquire(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_nand_i8_acquire: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB16_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a5, $a4, $a1 @@ -584,48 +584,48 @@ define i8 @atomicrmw_nand_i8_acquire(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_nand_i16_acquire(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_nand_i16_acquire: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB17_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: and $a5, $a4, $a1 ; LA32-NEXT: nor $a5, $a5, $zero ; LA32-NEXT: xor $a5, $a4, $a5 -; LA32-NEXT: and $a5, $a5, $a2 +; LA32-NEXT: and $a5, $a5, $a3 ; LA32-NEXT: xor $a5, $a4, $a5 ; LA32-NEXT: sc.w $a5, $a0, 0 ; LA32-NEXT: beqz $a5, .LBB17_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a4, $a3 +; LA32-NEXT: srl.w $a0, $a4, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_nand_i16_acquire: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB17_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a5, $a4, $a1 ; LA64-NEXT: nor $a5, $a5, $zero ; LA64-NEXT: xor $a5, $a4, $a5 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB17_1 ; LA64-NEXT: # %bb.2: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw nand ptr %a, i16 %b acquire ret i16 %1 @@ -689,12 +689,12 @@ define i8 @atomicrmw_and_i8_acquire(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_and_i8_acquire: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: orn $a1, $a1, $a3 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB20_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: and $a4, $a3, $a1 @@ -707,12 +707,12 @@ define i8 @atomicrmw_and_i8_acquire(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_and_i8_acquire: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: orn $a1, $a1, $a3 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: amand_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -723,35 +723,35 @@ define i8 @atomicrmw_and_i8_acquire(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_and_i16_acquire(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_and_i16_acquire: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 -; LA32-NEXT: orn $a1, $a1, $a2 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 +; LA32-NEXT: orn $a1, $a1, $a3 ; LA32-NEXT: .LBB21_1: # =>This Inner Loop Header: Depth=1 -; LA32-NEXT: ll.w $a2, $a0, 0 -; LA32-NEXT: and $a4, $a2, $a1 +; LA32-NEXT: ll.w $a3, $a0, 0 +; LA32-NEXT: and $a4, $a3, $a1 ; LA32-NEXT: sc.w $a4, $a0, 0 ; LA32-NEXT: beqz $a4, .LBB21_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a2, $a3 +; LA32-NEXT: srl.w $a0, $a3, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_and_i16_acquire: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 -; LA64-NEXT: orn $a1, $a1, $a2 +; LA64-NEXT: slli.d $a2, $a0, 3 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: amand_db.w $a2, $a1, $a0 -; LA64-NEXT: srl.w $a0, $a2, $a3 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 +; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 +; LA64-NEXT: sll.w $a1, $a1, $a2 +; LA64-NEXT: orn $a1, $a1, $a3 +; LA64-NEXT: amand_db.w $a3, $a1, $a0 +; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret %1 = atomicrmw and ptr %a, i16 %b acquire ret i16 %1 @@ -801,10 +801,10 @@ define i64 @atomicrmw_and_i64_acquire(ptr %a, i64 %b) nounwind { define i8 @atomicrmw_or_i8_acquire(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_or_i8_acquire: ; LA32: # %bb.0: -; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: andi $a1, $a1, 255 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB24_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: or $a4, $a3, $a1 @@ -816,10 +816,10 @@ define i8 @atomicrmw_or_i8_acquire(ptr %a, i8 %b) nounwind { ; ; LA64-LABEL: atomicrmw_or_i8_acquire: ; LA64: # %bb.0: -; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: andi $a1, $a1, 255 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: amor_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -830,10 +830,10 @@ define i8 @atomicrmw_or_i8_acquire(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_or_i16_acquire(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_or_i16_acquire: ; LA32: # %bb.0: -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 ; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB25_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: or $a4, $a3, $a1 @@ -845,10 +845,10 @@ define i16 @atomicrmw_or_i16_acquire(ptr %a, i16 %b) nounwind { ; ; LA64-LABEL: atomicrmw_or_i16_acquire: ; LA64: # %bb.0: -; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: amor_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -900,10 +900,10 @@ define i64 @atomicrmw_or_i64_acquire(ptr %a, i64 %b) nounwind { define i8 @atomicrmw_xor_i8_acquire(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_xor_i8_acquire: ; LA32: # %bb.0: -; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: andi $a1, $a1, 255 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB28_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: xor $a4, $a3, $a1 @@ -915,10 +915,10 @@ define i8 @atomicrmw_xor_i8_acquire(ptr %a, i8 %b) nounwind { ; ; LA64-LABEL: atomicrmw_xor_i8_acquire: ; LA64: # %bb.0: -; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: andi $a1, $a1, 255 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: amxor_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -929,10 +929,10 @@ define i8 @atomicrmw_xor_i8_acquire(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_xor_i16_acquire(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_xor_i16_acquire: ; LA32: # %bb.0: -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 ; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB29_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: xor $a4, $a3, $a1 @@ -944,10 +944,10 @@ define i16 @atomicrmw_xor_i16_acquire(ptr %a, i16 %b) nounwind { ; ; LA64-LABEL: atomicrmw_xor_i16_acquire: ; LA64: # %bb.0: -; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: amxor_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -1000,11 +1000,11 @@ define i8 @atomicrmw_xchg_i8_release(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_xchg_i8_release: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB32_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: addi.w $a5, $a1, 0 @@ -1020,13 +1020,13 @@ define i8 @atomicrmw_xchg_i8_release(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_xchg_i8_release: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB32_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: addi.w $a5, $a1, 0 @@ -1046,10 +1046,10 @@ define i8 @atomicrmw_xchg_0_i8_release(ptr %a) nounwind { ; LA32-LABEL: atomicrmw_xchg_0_i8_release: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a1, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a2, $zero, 255 ; LA32-NEXT: sll.w $a2, $a2, $a1 ; LA32-NEXT: nor $a2, $a2, $zero -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB33_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: and $a4, $a3, $a2 @@ -1062,10 +1062,10 @@ define i8 @atomicrmw_xchg_0_i8_release(ptr %a) nounwind { ; LA64-LABEL: atomicrmw_xchg_0_i8_release: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a1, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a2, $zero, 255 ; LA64-NEXT: sll.w $a2, $a2, $a1 ; LA64-NEXT: nor $a2, $a2, $zero -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: amand_db.w $a3, $a2, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a1 ; LA64-NEXT: ret @@ -1077,9 +1077,9 @@ define i8 @atomicrmw_xchg_minus_1_i8_release(ptr %a) nounwind { ; LA32-LABEL: atomicrmw_xchg_minus_1_i8_release: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a1, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a2, $zero, 255 ; LA32-NEXT: sll.w $a2, $a2, $a1 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB34_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: or $a4, $a3, $a2 @@ -1092,9 +1092,9 @@ define i8 @atomicrmw_xchg_minus_1_i8_release(ptr %a) nounwind { ; LA64-LABEL: atomicrmw_xchg_minus_1_i8_release: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a1, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a2, $zero, 255 ; LA64-NEXT: sll.w $a2, $a2, $a1 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: amor_db.w $a3, $a2, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a1 ; LA64-NEXT: ret @@ -1105,46 +1105,46 @@ define i8 @atomicrmw_xchg_minus_1_i8_release(ptr %a) nounwind { define i16 @atomicrmw_xchg_i16_release(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_xchg_i16_release: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB35_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: addi.w $a5, $a1, 0 ; LA32-NEXT: xor $a5, $a4, $a5 -; LA32-NEXT: and $a5, $a5, $a2 +; LA32-NEXT: and $a5, $a5, $a3 ; LA32-NEXT: xor $a5, $a4, $a5 ; LA32-NEXT: sc.w $a5, $a0, 0 ; LA32-NEXT: beqz $a5, .LBB35_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a4, $a3 +; LA32-NEXT: srl.w $a0, $a4, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_xchg_i16_release: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB35_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: addi.w $a5, $a1, 0 ; LA64-NEXT: xor $a5, $a4, $a5 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB35_1 ; LA64-NEXT: # %bb.2: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw xchg ptr %a, i16 %b release ret i16 %1 @@ -1153,31 +1153,31 @@ define i16 @atomicrmw_xchg_i16_release(ptr %a, i16 %b) nounwind { define i16 @atomicrmw_xchg_0_i16_release(ptr %a) nounwind { ; LA32-LABEL: atomicrmw_xchg_0_i16_release: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a1, 15 -; LA32-NEXT: ori $a1, $a1, 4095 -; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: nor $a1, $a1, $zero +; LA32-NEXT: slli.w $a1, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a2, 15 +; LA32-NEXT: ori $a2, $a2, 4095 +; LA32-NEXT: sll.w $a2, $a2, $a1 +; LA32-NEXT: nor $a2, $a2, $zero ; LA32-NEXT: .LBB36_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 -; LA32-NEXT: and $a4, $a3, $a1 +; LA32-NEXT: and $a4, $a3, $a2 ; LA32-NEXT: sc.w $a4, $a0, 0 ; LA32-NEXT: beqz $a4, .LBB36_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a3, $a2 +; LA32-NEXT: srl.w $a0, $a3, $a1 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_xchg_0_i16_release: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a1, 15 -; LA64-NEXT: ori $a1, $a1, 4095 -; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 -; LA64-NEXT: nor $a1, $a1, $zero +; LA64-NEXT: slli.d $a1, $a0, 3 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: amand_db.w $a3, $a1, $a0 -; LA64-NEXT: srl.w $a0, $a3, $a2 +; LA64-NEXT: lu12i.w $a2, 15 +; LA64-NEXT: ori $a2, $a2, 4095 +; LA64-NEXT: sll.w $a2, $a2, $a1 +; LA64-NEXT: nor $a2, $a2, $zero +; LA64-NEXT: amand_db.w $a3, $a2, $a0 +; LA64-NEXT: srl.w $a0, $a3, $a1 ; LA64-NEXT: ret %1 = atomicrmw xchg ptr %a, i16 0 release ret i16 %1 @@ -1186,29 +1186,29 @@ define i16 @atomicrmw_xchg_0_i16_release(ptr %a) nounwind { define i16 @atomicrmw_xchg_minus_1_i16_release(ptr %a) nounwind { ; LA32-LABEL: atomicrmw_xchg_minus_1_i16_release: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a1, 15 -; LA32-NEXT: ori $a1, $a1, 4095 -; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 +; LA32-NEXT: slli.w $a1, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a2, 15 +; LA32-NEXT: ori $a2, $a2, 4095 +; LA32-NEXT: sll.w $a2, $a2, $a1 ; LA32-NEXT: .LBB37_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 -; LA32-NEXT: or $a4, $a3, $a1 +; LA32-NEXT: or $a4, $a3, $a2 ; LA32-NEXT: sc.w $a4, $a0, 0 ; LA32-NEXT: beqz $a4, .LBB37_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a3, $a2 +; LA32-NEXT: srl.w $a0, $a3, $a1 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_xchg_minus_1_i16_release: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a1, 15 -; LA64-NEXT: ori $a1, $a1, 4095 -; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 +; LA64-NEXT: slli.d $a1, $a0, 3 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: amor_db.w $a3, $a1, $a0 -; LA64-NEXT: srl.w $a0, $a3, $a2 +; LA64-NEXT: lu12i.w $a2, 15 +; LA64-NEXT: ori $a2, $a2, 4095 +; LA64-NEXT: sll.w $a2, $a2, $a1 +; LA64-NEXT: amor_db.w $a3, $a2, $a0 +; LA64-NEXT: srl.w $a0, $a3, $a1 ; LA64-NEXT: ret %1 = atomicrmw xchg ptr %a, i16 -1 release ret i16 %1 @@ -1259,11 +1259,11 @@ define i8 @atomicrmw_add_i8_release(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_add_i8_release: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB40_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: add.w $a5, $a4, $a1 @@ -1279,13 +1279,13 @@ define i8 @atomicrmw_add_i8_release(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_add_i8_release: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB40_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: add.w $a5, $a4, $a1 @@ -1304,46 +1304,46 @@ define i8 @atomicrmw_add_i8_release(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_add_i16_release(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_add_i16_release: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB41_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: add.w $a5, $a4, $a1 ; LA32-NEXT: xor $a5, $a4, $a5 -; LA32-NEXT: and $a5, $a5, $a2 +; LA32-NEXT: and $a5, $a5, $a3 ; LA32-NEXT: xor $a5, $a4, $a5 ; LA32-NEXT: sc.w $a5, $a0, 0 ; LA32-NEXT: beqz $a5, .LBB41_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a4, $a3 +; LA32-NEXT: srl.w $a0, $a4, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_add_i16_release: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB41_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: add.w $a5, $a4, $a1 ; LA64-NEXT: xor $a5, $a4, $a5 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB41_1 ; LA64-NEXT: # %bb.2: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw add ptr %a, i16 %b release ret i16 %1 @@ -1394,11 +1394,11 @@ define i8 @atomicrmw_sub_i8_release(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_sub_i8_release: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB44_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: sub.w $a5, $a4, $a1 @@ -1414,13 +1414,13 @@ define i8 @atomicrmw_sub_i8_release(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_sub_i8_release: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB44_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: sub.w $a5, $a4, $a1 @@ -1439,46 +1439,46 @@ define i8 @atomicrmw_sub_i8_release(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_sub_i16_release(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_sub_i16_release: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB45_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: sub.w $a5, $a4, $a1 ; LA32-NEXT: xor $a5, $a4, $a5 -; LA32-NEXT: and $a5, $a5, $a2 +; LA32-NEXT: and $a5, $a5, $a3 ; LA32-NEXT: xor $a5, $a4, $a5 ; LA32-NEXT: sc.w $a5, $a0, 0 ; LA32-NEXT: beqz $a5, .LBB45_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a4, $a3 +; LA32-NEXT: srl.w $a0, $a4, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_sub_i16_release: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB45_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: sub.w $a5, $a4, $a1 ; LA64-NEXT: xor $a5, $a4, $a5 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB45_1 ; LA64-NEXT: # %bb.2: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw sub ptr %a, i16 %b release ret i16 %1 @@ -1531,11 +1531,11 @@ define i8 @atomicrmw_nand_i8_release(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_nand_i8_release: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB48_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: and $a5, $a4, $a1 @@ -1552,13 +1552,13 @@ define i8 @atomicrmw_nand_i8_release(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_nand_i8_release: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB48_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a5, $a4, $a1 @@ -1578,48 +1578,48 @@ define i8 @atomicrmw_nand_i8_release(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_nand_i16_release(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_nand_i16_release: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB49_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: and $a5, $a4, $a1 ; LA32-NEXT: nor $a5, $a5, $zero ; LA32-NEXT: xor $a5, $a4, $a5 -; LA32-NEXT: and $a5, $a5, $a2 +; LA32-NEXT: and $a5, $a5, $a3 ; LA32-NEXT: xor $a5, $a4, $a5 ; LA32-NEXT: sc.w $a5, $a0, 0 ; LA32-NEXT: beqz $a5, .LBB49_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a4, $a3 +; LA32-NEXT: srl.w $a0, $a4, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_nand_i16_release: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB49_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a5, $a4, $a1 ; LA64-NEXT: nor $a5, $a5, $zero ; LA64-NEXT: xor $a5, $a4, $a5 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB49_1 ; LA64-NEXT: # %bb.2: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw nand ptr %a, i16 %b release ret i16 %1 @@ -1683,12 +1683,12 @@ define i8 @atomicrmw_and_i8_release(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_and_i8_release: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: orn $a1, $a1, $a3 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB52_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: and $a4, $a3, $a1 @@ -1701,12 +1701,12 @@ define i8 @atomicrmw_and_i8_release(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_and_i8_release: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: orn $a1, $a1, $a3 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: amand_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -1717,35 +1717,35 @@ define i8 @atomicrmw_and_i8_release(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_and_i16_release(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_and_i16_release: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 -; LA32-NEXT: orn $a1, $a1, $a2 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 +; LA32-NEXT: orn $a1, $a1, $a3 ; LA32-NEXT: .LBB53_1: # =>This Inner Loop Header: Depth=1 -; LA32-NEXT: ll.w $a2, $a0, 0 -; LA32-NEXT: and $a4, $a2, $a1 +; LA32-NEXT: ll.w $a3, $a0, 0 +; LA32-NEXT: and $a4, $a3, $a1 ; LA32-NEXT: sc.w $a4, $a0, 0 ; LA32-NEXT: beqz $a4, .LBB53_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a2, $a3 +; LA32-NEXT: srl.w $a0, $a3, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_and_i16_release: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 -; LA64-NEXT: orn $a1, $a1, $a2 +; LA64-NEXT: slli.d $a2, $a0, 3 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: amand_db.w $a2, $a1, $a0 -; LA64-NEXT: srl.w $a0, $a2, $a3 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 +; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 +; LA64-NEXT: sll.w $a1, $a1, $a2 +; LA64-NEXT: orn $a1, $a1, $a3 +; LA64-NEXT: amand_db.w $a3, $a1, $a0 +; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret %1 = atomicrmw and ptr %a, i16 %b release ret i16 %1 @@ -1795,10 +1795,10 @@ define i64 @atomicrmw_and_i64_release(ptr %a, i64 %b) nounwind { define i8 @atomicrmw_or_i8_release(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_or_i8_release: ; LA32: # %bb.0: -; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: andi $a1, $a1, 255 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB56_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: or $a4, $a3, $a1 @@ -1810,10 +1810,10 @@ define i8 @atomicrmw_or_i8_release(ptr %a, i8 %b) nounwind { ; ; LA64-LABEL: atomicrmw_or_i8_release: ; LA64: # %bb.0: -; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: andi $a1, $a1, 255 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: amor_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -1824,10 +1824,10 @@ define i8 @atomicrmw_or_i8_release(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_or_i16_release(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_or_i16_release: ; LA32: # %bb.0: -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 ; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB57_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: or $a4, $a3, $a1 @@ -1839,10 +1839,10 @@ define i16 @atomicrmw_or_i16_release(ptr %a, i16 %b) nounwind { ; ; LA64-LABEL: atomicrmw_or_i16_release: ; LA64: # %bb.0: -; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: amor_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -1894,10 +1894,10 @@ define i64 @atomicrmw_or_i64_release(ptr %a, i64 %b) nounwind { define i8 @atomicrmw_xor_i8_release(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_xor_i8_release: ; LA32: # %bb.0: -; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: andi $a1, $a1, 255 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB60_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: xor $a4, $a3, $a1 @@ -1909,10 +1909,10 @@ define i8 @atomicrmw_xor_i8_release(ptr %a, i8 %b) nounwind { ; ; LA64-LABEL: atomicrmw_xor_i8_release: ; LA64: # %bb.0: -; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: andi $a1, $a1, 255 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: amxor_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -1923,10 +1923,10 @@ define i8 @atomicrmw_xor_i8_release(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_xor_i16_release(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_xor_i16_release: ; LA32: # %bb.0: -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 ; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB61_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: xor $a4, $a3, $a1 @@ -1938,10 +1938,10 @@ define i16 @atomicrmw_xor_i16_release(ptr %a, i16 %b) nounwind { ; ; LA64-LABEL: atomicrmw_xor_i16_release: ; LA64: # %bb.0: -; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: amxor_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -1994,11 +1994,11 @@ define i8 @atomicrmw_xchg_i8_acq_rel(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_xchg_i8_acq_rel: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB64_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: addi.w $a5, $a1, 0 @@ -2014,13 +2014,13 @@ define i8 @atomicrmw_xchg_i8_acq_rel(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_xchg_i8_acq_rel: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB64_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: addi.w $a5, $a1, 0 @@ -2040,10 +2040,10 @@ define i8 @atomicrmw_xchg_0_i8_acq_rel(ptr %a) nounwind { ; LA32-LABEL: atomicrmw_xchg_0_i8_acq_rel: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a1, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a2, $zero, 255 ; LA32-NEXT: sll.w $a2, $a2, $a1 ; LA32-NEXT: nor $a2, $a2, $zero -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB65_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: and $a4, $a3, $a2 @@ -2056,10 +2056,10 @@ define i8 @atomicrmw_xchg_0_i8_acq_rel(ptr %a) nounwind { ; LA64-LABEL: atomicrmw_xchg_0_i8_acq_rel: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a1, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a2, $zero, 255 ; LA64-NEXT: sll.w $a2, $a2, $a1 ; LA64-NEXT: nor $a2, $a2, $zero -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: amand_db.w $a3, $a2, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a1 ; LA64-NEXT: ret @@ -2071,9 +2071,9 @@ define i8 @atomicrmw_xchg_minus_1_i8_acq_rel(ptr %a) nounwind { ; LA32-LABEL: atomicrmw_xchg_minus_1_i8_acq_rel: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a1, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a2, $zero, 255 ; LA32-NEXT: sll.w $a2, $a2, $a1 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB66_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: or $a4, $a3, $a2 @@ -2086,9 +2086,9 @@ define i8 @atomicrmw_xchg_minus_1_i8_acq_rel(ptr %a) nounwind { ; LA64-LABEL: atomicrmw_xchg_minus_1_i8_acq_rel: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a1, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a2, $zero, 255 ; LA64-NEXT: sll.w $a2, $a2, $a1 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: amor_db.w $a3, $a2, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a1 ; LA64-NEXT: ret @@ -2099,46 +2099,46 @@ define i8 @atomicrmw_xchg_minus_1_i8_acq_rel(ptr %a) nounwind { define i16 @atomicrmw_xchg_i16_acq_rel(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_xchg_i16_acq_rel: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB67_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: addi.w $a5, $a1, 0 ; LA32-NEXT: xor $a5, $a4, $a5 -; LA32-NEXT: and $a5, $a5, $a2 +; LA32-NEXT: and $a5, $a5, $a3 ; LA32-NEXT: xor $a5, $a4, $a5 ; LA32-NEXT: sc.w $a5, $a0, 0 ; LA32-NEXT: beqz $a5, .LBB67_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a4, $a3 +; LA32-NEXT: srl.w $a0, $a4, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_xchg_i16_acq_rel: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB67_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: addi.w $a5, $a1, 0 ; LA64-NEXT: xor $a5, $a4, $a5 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB67_1 ; LA64-NEXT: # %bb.2: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw xchg ptr %a, i16 %b acq_rel ret i16 %1 @@ -2147,31 +2147,31 @@ define i16 @atomicrmw_xchg_i16_acq_rel(ptr %a, i16 %b) nounwind { define i16 @atomicrmw_xchg_0_i16_acq_rel(ptr %a) nounwind { ; LA32-LABEL: atomicrmw_xchg_0_i16_acq_rel: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a1, 15 -; LA32-NEXT: ori $a1, $a1, 4095 -; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: nor $a1, $a1, $zero +; LA32-NEXT: slli.w $a1, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a2, 15 +; LA32-NEXT: ori $a2, $a2, 4095 +; LA32-NEXT: sll.w $a2, $a2, $a1 +; LA32-NEXT: nor $a2, $a2, $zero ; LA32-NEXT: .LBB68_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 -; LA32-NEXT: and $a4, $a3, $a1 +; LA32-NEXT: and $a4, $a3, $a2 ; LA32-NEXT: sc.w $a4, $a0, 0 ; LA32-NEXT: beqz $a4, .LBB68_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a3, $a2 +; LA32-NEXT: srl.w $a0, $a3, $a1 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_xchg_0_i16_acq_rel: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a1, 15 -; LA64-NEXT: ori $a1, $a1, 4095 -; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 -; LA64-NEXT: nor $a1, $a1, $zero +; LA64-NEXT: slli.d $a1, $a0, 3 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: amand_db.w $a3, $a1, $a0 -; LA64-NEXT: srl.w $a0, $a3, $a2 +; LA64-NEXT: lu12i.w $a2, 15 +; LA64-NEXT: ori $a2, $a2, 4095 +; LA64-NEXT: sll.w $a2, $a2, $a1 +; LA64-NEXT: nor $a2, $a2, $zero +; LA64-NEXT: amand_db.w $a3, $a2, $a0 +; LA64-NEXT: srl.w $a0, $a3, $a1 ; LA64-NEXT: ret %1 = atomicrmw xchg ptr %a, i16 0 acq_rel ret i16 %1 @@ -2180,29 +2180,29 @@ define i16 @atomicrmw_xchg_0_i16_acq_rel(ptr %a) nounwind { define i16 @atomicrmw_xchg_minus_1_i16_acq_rel(ptr %a) nounwind { ; LA32-LABEL: atomicrmw_xchg_minus_1_i16_acq_rel: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a1, 15 -; LA32-NEXT: ori $a1, $a1, 4095 -; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 +; LA32-NEXT: slli.w $a1, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a2, 15 +; LA32-NEXT: ori $a2, $a2, 4095 +; LA32-NEXT: sll.w $a2, $a2, $a1 ; LA32-NEXT: .LBB69_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 -; LA32-NEXT: or $a4, $a3, $a1 +; LA32-NEXT: or $a4, $a3, $a2 ; LA32-NEXT: sc.w $a4, $a0, 0 ; LA32-NEXT: beqz $a4, .LBB69_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a3, $a2 +; LA32-NEXT: srl.w $a0, $a3, $a1 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_xchg_minus_1_i16_acq_rel: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a1, 15 -; LA64-NEXT: ori $a1, $a1, 4095 -; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 +; LA64-NEXT: slli.d $a1, $a0, 3 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: amor_db.w $a3, $a1, $a0 -; LA64-NEXT: srl.w $a0, $a3, $a2 +; LA64-NEXT: lu12i.w $a2, 15 +; LA64-NEXT: ori $a2, $a2, 4095 +; LA64-NEXT: sll.w $a2, $a2, $a1 +; LA64-NEXT: amor_db.w $a3, $a2, $a0 +; LA64-NEXT: srl.w $a0, $a3, $a1 ; LA64-NEXT: ret %1 = atomicrmw xchg ptr %a, i16 -1 acq_rel ret i16 %1 @@ -2253,11 +2253,11 @@ define i8 @atomicrmw_add_i8_acq_rel(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_add_i8_acq_rel: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB72_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: add.w $a5, $a4, $a1 @@ -2273,13 +2273,13 @@ define i8 @atomicrmw_add_i8_acq_rel(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_add_i8_acq_rel: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB72_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: add.w $a5, $a4, $a1 @@ -2298,46 +2298,46 @@ define i8 @atomicrmw_add_i8_acq_rel(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_add_i16_acq_rel(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_add_i16_acq_rel: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB73_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: add.w $a5, $a4, $a1 ; LA32-NEXT: xor $a5, $a4, $a5 -; LA32-NEXT: and $a5, $a5, $a2 +; LA32-NEXT: and $a5, $a5, $a3 ; LA32-NEXT: xor $a5, $a4, $a5 ; LA32-NEXT: sc.w $a5, $a0, 0 ; LA32-NEXT: beqz $a5, .LBB73_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a4, $a3 +; LA32-NEXT: srl.w $a0, $a4, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_add_i16_acq_rel: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB73_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: add.w $a5, $a4, $a1 ; LA64-NEXT: xor $a5, $a4, $a5 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB73_1 ; LA64-NEXT: # %bb.2: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw add ptr %a, i16 %b acq_rel ret i16 %1 @@ -2388,11 +2388,11 @@ define i8 @atomicrmw_sub_i8_acq_rel(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_sub_i8_acq_rel: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB76_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: sub.w $a5, $a4, $a1 @@ -2408,13 +2408,13 @@ define i8 @atomicrmw_sub_i8_acq_rel(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_sub_i8_acq_rel: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB76_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: sub.w $a5, $a4, $a1 @@ -2433,46 +2433,46 @@ define i8 @atomicrmw_sub_i8_acq_rel(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_sub_i16_acq_rel(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_sub_i16_acq_rel: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB77_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: sub.w $a5, $a4, $a1 ; LA32-NEXT: xor $a5, $a4, $a5 -; LA32-NEXT: and $a5, $a5, $a2 +; LA32-NEXT: and $a5, $a5, $a3 ; LA32-NEXT: xor $a5, $a4, $a5 ; LA32-NEXT: sc.w $a5, $a0, 0 ; LA32-NEXT: beqz $a5, .LBB77_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a4, $a3 +; LA32-NEXT: srl.w $a0, $a4, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_sub_i16_acq_rel: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB77_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: sub.w $a5, $a4, $a1 ; LA64-NEXT: xor $a5, $a4, $a5 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB77_1 ; LA64-NEXT: # %bb.2: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw sub ptr %a, i16 %b acq_rel ret i16 %1 @@ -2525,11 +2525,11 @@ define i8 @atomicrmw_nand_i8_acq_rel(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_nand_i8_acq_rel: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB80_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: and $a5, $a4, $a1 @@ -2546,13 +2546,13 @@ define i8 @atomicrmw_nand_i8_acq_rel(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_nand_i8_acq_rel: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB80_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a5, $a4, $a1 @@ -2572,48 +2572,48 @@ define i8 @atomicrmw_nand_i8_acq_rel(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_nand_i16_acq_rel(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_nand_i16_acq_rel: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB81_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: and $a5, $a4, $a1 ; LA32-NEXT: nor $a5, $a5, $zero ; LA32-NEXT: xor $a5, $a4, $a5 -; LA32-NEXT: and $a5, $a5, $a2 +; LA32-NEXT: and $a5, $a5, $a3 ; LA32-NEXT: xor $a5, $a4, $a5 ; LA32-NEXT: sc.w $a5, $a0, 0 ; LA32-NEXT: beqz $a5, .LBB81_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a4, $a3 +; LA32-NEXT: srl.w $a0, $a4, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_nand_i16_acq_rel: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB81_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a5, $a4, $a1 ; LA64-NEXT: nor $a5, $a5, $zero ; LA64-NEXT: xor $a5, $a4, $a5 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB81_1 ; LA64-NEXT: # %bb.2: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw nand ptr %a, i16 %b acq_rel ret i16 %1 @@ -2677,12 +2677,12 @@ define i8 @atomicrmw_and_i8_acq_rel(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_and_i8_acq_rel: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: orn $a1, $a1, $a3 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB84_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: and $a4, $a3, $a1 @@ -2695,12 +2695,12 @@ define i8 @atomicrmw_and_i8_acq_rel(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_and_i8_acq_rel: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: orn $a1, $a1, $a3 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: amand_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -2711,35 +2711,35 @@ define i8 @atomicrmw_and_i8_acq_rel(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_and_i16_acq_rel(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_and_i16_acq_rel: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 -; LA32-NEXT: orn $a1, $a1, $a2 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 +; LA32-NEXT: orn $a1, $a1, $a3 ; LA32-NEXT: .LBB85_1: # =>This Inner Loop Header: Depth=1 -; LA32-NEXT: ll.w $a2, $a0, 0 -; LA32-NEXT: and $a4, $a2, $a1 +; LA32-NEXT: ll.w $a3, $a0, 0 +; LA32-NEXT: and $a4, $a3, $a1 ; LA32-NEXT: sc.w $a4, $a0, 0 ; LA32-NEXT: beqz $a4, .LBB85_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a2, $a3 +; LA32-NEXT: srl.w $a0, $a3, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_and_i16_acq_rel: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 -; LA64-NEXT: orn $a1, $a1, $a2 +; LA64-NEXT: slli.d $a2, $a0, 3 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: amand_db.w $a2, $a1, $a0 -; LA64-NEXT: srl.w $a0, $a2, $a3 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 +; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 +; LA64-NEXT: sll.w $a1, $a1, $a2 +; LA64-NEXT: orn $a1, $a1, $a3 +; LA64-NEXT: amand_db.w $a3, $a1, $a0 +; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret %1 = atomicrmw and ptr %a, i16 %b acq_rel ret i16 %1 @@ -2789,10 +2789,10 @@ define i64 @atomicrmw_and_i64_acq_rel(ptr %a, i64 %b) nounwind { define i8 @atomicrmw_or_i8_acq_rel(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_or_i8_acq_rel: ; LA32: # %bb.0: -; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: andi $a1, $a1, 255 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB88_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: or $a4, $a3, $a1 @@ -2804,10 +2804,10 @@ define i8 @atomicrmw_or_i8_acq_rel(ptr %a, i8 %b) nounwind { ; ; LA64-LABEL: atomicrmw_or_i8_acq_rel: ; LA64: # %bb.0: -; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: andi $a1, $a1, 255 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: amor_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -2818,10 +2818,10 @@ define i8 @atomicrmw_or_i8_acq_rel(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_or_i16_acq_rel(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_or_i16_acq_rel: ; LA32: # %bb.0: -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 ; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB89_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: or $a4, $a3, $a1 @@ -2833,10 +2833,10 @@ define i16 @atomicrmw_or_i16_acq_rel(ptr %a, i16 %b) nounwind { ; ; LA64-LABEL: atomicrmw_or_i16_acq_rel: ; LA64: # %bb.0: -; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: amor_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -2888,10 +2888,10 @@ define i64 @atomicrmw_or_i64_acq_rel(ptr %a, i64 %b) nounwind { define i8 @atomicrmw_xor_i8_acq_rel(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_xor_i8_acq_rel: ; LA32: # %bb.0: -; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: andi $a1, $a1, 255 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB92_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: xor $a4, $a3, $a1 @@ -2903,10 +2903,10 @@ define i8 @atomicrmw_xor_i8_acq_rel(ptr %a, i8 %b) nounwind { ; ; LA64-LABEL: atomicrmw_xor_i8_acq_rel: ; LA64: # %bb.0: -; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: andi $a1, $a1, 255 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: amxor_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -2917,10 +2917,10 @@ define i8 @atomicrmw_xor_i8_acq_rel(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_xor_i16_acq_rel(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_xor_i16_acq_rel: ; LA32: # %bb.0: -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 ; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB93_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: xor $a4, $a3, $a1 @@ -2932,10 +2932,10 @@ define i16 @atomicrmw_xor_i16_acq_rel(ptr %a, i16 %b) nounwind { ; ; LA64-LABEL: atomicrmw_xor_i16_acq_rel: ; LA64: # %bb.0: -; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: amxor_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -2988,11 +2988,11 @@ define i8 @atomicrmw_xchg_i8_seq_cst(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_xchg_i8_seq_cst: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB96_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: addi.w $a5, $a1, 0 @@ -3008,13 +3008,13 @@ define i8 @atomicrmw_xchg_i8_seq_cst(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_xchg_i8_seq_cst: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB96_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: addi.w $a5, $a1, 0 @@ -3034,10 +3034,10 @@ define i8 @atomicrmw_xchg_0_i8_seq_cst(ptr %a) nounwind { ; LA32-LABEL: atomicrmw_xchg_0_i8_seq_cst: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a1, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a2, $zero, 255 ; LA32-NEXT: sll.w $a2, $a2, $a1 ; LA32-NEXT: nor $a2, $a2, $zero -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB97_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: and $a4, $a3, $a2 @@ -3050,10 +3050,10 @@ define i8 @atomicrmw_xchg_0_i8_seq_cst(ptr %a) nounwind { ; LA64-LABEL: atomicrmw_xchg_0_i8_seq_cst: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a1, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a2, $zero, 255 ; LA64-NEXT: sll.w $a2, $a2, $a1 ; LA64-NEXT: nor $a2, $a2, $zero -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: amand_db.w $a3, $a2, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a1 ; LA64-NEXT: ret @@ -3065,9 +3065,9 @@ define i8 @atomicrmw_xchg_minus_1_i8_seq_cst(ptr %a) nounwind { ; LA32-LABEL: atomicrmw_xchg_minus_1_i8_seq_cst: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a1, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a2, $zero, 255 ; LA32-NEXT: sll.w $a2, $a2, $a1 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB98_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: or $a4, $a3, $a2 @@ -3080,9 +3080,9 @@ define i8 @atomicrmw_xchg_minus_1_i8_seq_cst(ptr %a) nounwind { ; LA64-LABEL: atomicrmw_xchg_minus_1_i8_seq_cst: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a1, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a2, $zero, 255 ; LA64-NEXT: sll.w $a2, $a2, $a1 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: amor_db.w $a3, $a2, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a1 ; LA64-NEXT: ret @@ -3093,46 +3093,46 @@ define i8 @atomicrmw_xchg_minus_1_i8_seq_cst(ptr %a) nounwind { define i16 @atomicrmw_xchg_i16_seq_cst(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_xchg_i16_seq_cst: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB99_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: addi.w $a5, $a1, 0 ; LA32-NEXT: xor $a5, $a4, $a5 -; LA32-NEXT: and $a5, $a5, $a2 +; LA32-NEXT: and $a5, $a5, $a3 ; LA32-NEXT: xor $a5, $a4, $a5 ; LA32-NEXT: sc.w $a5, $a0, 0 ; LA32-NEXT: beqz $a5, .LBB99_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a4, $a3 +; LA32-NEXT: srl.w $a0, $a4, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_xchg_i16_seq_cst: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB99_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: addi.w $a5, $a1, 0 ; LA64-NEXT: xor $a5, $a4, $a5 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB99_1 ; LA64-NEXT: # %bb.2: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw xchg ptr %a, i16 %b seq_cst ret i16 %1 @@ -3141,31 +3141,31 @@ define i16 @atomicrmw_xchg_i16_seq_cst(ptr %a, i16 %b) nounwind { define i16 @atomicrmw_xchg_0_i16_seq_cst(ptr %a) nounwind { ; LA32-LABEL: atomicrmw_xchg_0_i16_seq_cst: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a1, 15 -; LA32-NEXT: ori $a1, $a1, 4095 -; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: nor $a1, $a1, $zero +; LA32-NEXT: slli.w $a1, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a2, 15 +; LA32-NEXT: ori $a2, $a2, 4095 +; LA32-NEXT: sll.w $a2, $a2, $a1 +; LA32-NEXT: nor $a2, $a2, $zero ; LA32-NEXT: .LBB100_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 -; LA32-NEXT: and $a4, $a3, $a1 +; LA32-NEXT: and $a4, $a3, $a2 ; LA32-NEXT: sc.w $a4, $a0, 0 ; LA32-NEXT: beqz $a4, .LBB100_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a3, $a2 +; LA32-NEXT: srl.w $a0, $a3, $a1 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_xchg_0_i16_seq_cst: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a1, 15 -; LA64-NEXT: ori $a1, $a1, 4095 -; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 -; LA64-NEXT: nor $a1, $a1, $zero +; LA64-NEXT: slli.d $a1, $a0, 3 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: amand_db.w $a3, $a1, $a0 -; LA64-NEXT: srl.w $a0, $a3, $a2 +; LA64-NEXT: lu12i.w $a2, 15 +; LA64-NEXT: ori $a2, $a2, 4095 +; LA64-NEXT: sll.w $a2, $a2, $a1 +; LA64-NEXT: nor $a2, $a2, $zero +; LA64-NEXT: amand_db.w $a3, $a2, $a0 +; LA64-NEXT: srl.w $a0, $a3, $a1 ; LA64-NEXT: ret %1 = atomicrmw xchg ptr %a, i16 0 seq_cst ret i16 %1 @@ -3174,29 +3174,29 @@ define i16 @atomicrmw_xchg_0_i16_seq_cst(ptr %a) nounwind { define i16 @atomicrmw_xchg_minus_1_i16_seq_cst(ptr %a) nounwind { ; LA32-LABEL: atomicrmw_xchg_minus_1_i16_seq_cst: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a1, 15 -; LA32-NEXT: ori $a1, $a1, 4095 -; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 +; LA32-NEXT: slli.w $a1, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a2, 15 +; LA32-NEXT: ori $a2, $a2, 4095 +; LA32-NEXT: sll.w $a2, $a2, $a1 ; LA32-NEXT: .LBB101_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 -; LA32-NEXT: or $a4, $a3, $a1 +; LA32-NEXT: or $a4, $a3, $a2 ; LA32-NEXT: sc.w $a4, $a0, 0 ; LA32-NEXT: beqz $a4, .LBB101_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a3, $a2 +; LA32-NEXT: srl.w $a0, $a3, $a1 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_xchg_minus_1_i16_seq_cst: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a1, 15 -; LA64-NEXT: ori $a1, $a1, 4095 -; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 +; LA64-NEXT: slli.d $a1, $a0, 3 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: amor_db.w $a3, $a1, $a0 -; LA64-NEXT: srl.w $a0, $a3, $a2 +; LA64-NEXT: lu12i.w $a2, 15 +; LA64-NEXT: ori $a2, $a2, 4095 +; LA64-NEXT: sll.w $a2, $a2, $a1 +; LA64-NEXT: amor_db.w $a3, $a2, $a0 +; LA64-NEXT: srl.w $a0, $a3, $a1 ; LA64-NEXT: ret %1 = atomicrmw xchg ptr %a, i16 -1 seq_cst ret i16 %1 @@ -3247,11 +3247,11 @@ define i8 @atomicrmw_add_i8_seq_cst(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_add_i8_seq_cst: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB104_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: add.w $a5, $a4, $a1 @@ -3267,13 +3267,13 @@ define i8 @atomicrmw_add_i8_seq_cst(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_add_i8_seq_cst: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB104_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: add.w $a5, $a4, $a1 @@ -3292,46 +3292,46 @@ define i8 @atomicrmw_add_i8_seq_cst(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_add_i16_seq_cst(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_add_i16_seq_cst: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB105_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: add.w $a5, $a4, $a1 ; LA32-NEXT: xor $a5, $a4, $a5 -; LA32-NEXT: and $a5, $a5, $a2 +; LA32-NEXT: and $a5, $a5, $a3 ; LA32-NEXT: xor $a5, $a4, $a5 ; LA32-NEXT: sc.w $a5, $a0, 0 ; LA32-NEXT: beqz $a5, .LBB105_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a4, $a3 +; LA32-NEXT: srl.w $a0, $a4, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_add_i16_seq_cst: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB105_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: add.w $a5, $a4, $a1 ; LA64-NEXT: xor $a5, $a4, $a5 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB105_1 ; LA64-NEXT: # %bb.2: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw add ptr %a, i16 %b seq_cst ret i16 %1 @@ -3382,11 +3382,11 @@ define i8 @atomicrmw_sub_i8_seq_cst(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_sub_i8_seq_cst: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB108_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: sub.w $a5, $a4, $a1 @@ -3402,13 +3402,13 @@ define i8 @atomicrmw_sub_i8_seq_cst(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_sub_i8_seq_cst: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB108_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: sub.w $a5, $a4, $a1 @@ -3427,46 +3427,46 @@ define i8 @atomicrmw_sub_i8_seq_cst(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_sub_i16_seq_cst(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_sub_i16_seq_cst: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB109_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: sub.w $a5, $a4, $a1 ; LA32-NEXT: xor $a5, $a4, $a5 -; LA32-NEXT: and $a5, $a5, $a2 +; LA32-NEXT: and $a5, $a5, $a3 ; LA32-NEXT: xor $a5, $a4, $a5 ; LA32-NEXT: sc.w $a5, $a0, 0 ; LA32-NEXT: beqz $a5, .LBB109_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a4, $a3 +; LA32-NEXT: srl.w $a0, $a4, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_sub_i16_seq_cst: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB109_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: sub.w $a5, $a4, $a1 ; LA64-NEXT: xor $a5, $a4, $a5 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB109_1 ; LA64-NEXT: # %bb.2: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw sub ptr %a, i16 %b seq_cst ret i16 %1 @@ -3519,11 +3519,11 @@ define i8 @atomicrmw_nand_i8_seq_cst(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_nand_i8_seq_cst: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB112_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: and $a5, $a4, $a1 @@ -3540,13 +3540,13 @@ define i8 @atomicrmw_nand_i8_seq_cst(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_nand_i8_seq_cst: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB112_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a5, $a4, $a1 @@ -3566,48 +3566,48 @@ define i8 @atomicrmw_nand_i8_seq_cst(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_nand_i16_seq_cst(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_nand_i16_seq_cst: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB113_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: and $a5, $a4, $a1 ; LA32-NEXT: nor $a5, $a5, $zero ; LA32-NEXT: xor $a5, $a4, $a5 -; LA32-NEXT: and $a5, $a5, $a2 +; LA32-NEXT: and $a5, $a5, $a3 ; LA32-NEXT: xor $a5, $a4, $a5 ; LA32-NEXT: sc.w $a5, $a0, 0 ; LA32-NEXT: beqz $a5, .LBB113_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a4, $a3 +; LA32-NEXT: srl.w $a0, $a4, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_nand_i16_seq_cst: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB113_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a5, $a4, $a1 ; LA64-NEXT: nor $a5, $a5, $zero ; LA64-NEXT: xor $a5, $a4, $a5 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB113_1 ; LA64-NEXT: # %bb.2: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw nand ptr %a, i16 %b seq_cst ret i16 %1 @@ -3671,12 +3671,12 @@ define i8 @atomicrmw_and_i8_seq_cst(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_and_i8_seq_cst: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: orn $a1, $a1, $a3 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB116_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: and $a4, $a3, $a1 @@ -3689,12 +3689,12 @@ define i8 @atomicrmw_and_i8_seq_cst(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_and_i8_seq_cst: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: orn $a1, $a1, $a3 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: amand_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -3705,35 +3705,35 @@ define i8 @atomicrmw_and_i8_seq_cst(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_and_i16_seq_cst(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_and_i16_seq_cst: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 -; LA32-NEXT: orn $a1, $a1, $a2 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 +; LA32-NEXT: orn $a1, $a1, $a3 ; LA32-NEXT: .LBB117_1: # =>This Inner Loop Header: Depth=1 -; LA32-NEXT: ll.w $a2, $a0, 0 -; LA32-NEXT: and $a4, $a2, $a1 +; LA32-NEXT: ll.w $a3, $a0, 0 +; LA32-NEXT: and $a4, $a3, $a1 ; LA32-NEXT: sc.w $a4, $a0, 0 ; LA32-NEXT: beqz $a4, .LBB117_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a2, $a3 +; LA32-NEXT: srl.w $a0, $a3, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_and_i16_seq_cst: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 -; LA64-NEXT: orn $a1, $a1, $a2 +; LA64-NEXT: slli.d $a2, $a0, 3 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: amand_db.w $a2, $a1, $a0 -; LA64-NEXT: srl.w $a0, $a2, $a3 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 +; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 +; LA64-NEXT: sll.w $a1, $a1, $a2 +; LA64-NEXT: orn $a1, $a1, $a3 +; LA64-NEXT: amand_db.w $a3, $a1, $a0 +; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret %1 = atomicrmw and ptr %a, i16 %b seq_cst ret i16 %1 @@ -3783,10 +3783,10 @@ define i64 @atomicrmw_and_i64_seq_cst(ptr %a, i64 %b) nounwind { define i8 @atomicrmw_or_i8_seq_cst(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_or_i8_seq_cst: ; LA32: # %bb.0: -; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: andi $a1, $a1, 255 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB120_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: or $a4, $a3, $a1 @@ -3798,10 +3798,10 @@ define i8 @atomicrmw_or_i8_seq_cst(ptr %a, i8 %b) nounwind { ; ; LA64-LABEL: atomicrmw_or_i8_seq_cst: ; LA64: # %bb.0: -; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: andi $a1, $a1, 255 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: amor_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -3812,10 +3812,10 @@ define i8 @atomicrmw_or_i8_seq_cst(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_or_i16_seq_cst(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_or_i16_seq_cst: ; LA32: # %bb.0: -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 ; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB121_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: or $a4, $a3, $a1 @@ -3827,10 +3827,10 @@ define i16 @atomicrmw_or_i16_seq_cst(ptr %a, i16 %b) nounwind { ; ; LA64-LABEL: atomicrmw_or_i16_seq_cst: ; LA64: # %bb.0: -; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: amor_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -3882,10 +3882,10 @@ define i64 @atomicrmw_or_i64_seq_cst(ptr %a, i64 %b) nounwind { define i8 @atomicrmw_xor_i8_seq_cst(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_xor_i8_seq_cst: ; LA32: # %bb.0: -; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: andi $a1, $a1, 255 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB124_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: xor $a4, $a3, $a1 @@ -3897,10 +3897,10 @@ define i8 @atomicrmw_xor_i8_seq_cst(ptr %a, i8 %b) nounwind { ; ; LA64-LABEL: atomicrmw_xor_i8_seq_cst: ; LA64: # %bb.0: -; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: andi $a1, $a1, 255 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: amxor_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -3911,10 +3911,10 @@ define i8 @atomicrmw_xor_i8_seq_cst(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_xor_i16_seq_cst(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_xor_i16_seq_cst: ; LA32: # %bb.0: -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 ; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB125_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: xor $a4, $a3, $a1 @@ -3926,10 +3926,10 @@ define i16 @atomicrmw_xor_i16_seq_cst(ptr %a, i16 %b) nounwind { ; ; LA64-LABEL: atomicrmw_xor_i16_seq_cst: ; LA64: # %bb.0: -; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: amxor_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -3982,11 +3982,11 @@ define i8 @atomicrmw_xchg_i8_monotonic(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_xchg_i8_monotonic: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB128_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: addi.w $a5, $a1, 0 @@ -4002,13 +4002,13 @@ define i8 @atomicrmw_xchg_i8_monotonic(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_xchg_i8_monotonic: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB128_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: addi.w $a5, $a1, 0 @@ -4028,10 +4028,10 @@ define i8 @atomicrmw_xchg_0_i8_monotonic(ptr %a) nounwind { ; LA32-LABEL: atomicrmw_xchg_0_i8_monotonic: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a1, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a2, $zero, 255 ; LA32-NEXT: sll.w $a2, $a2, $a1 ; LA32-NEXT: nor $a2, $a2, $zero -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB129_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: and $a4, $a3, $a2 @@ -4044,10 +4044,10 @@ define i8 @atomicrmw_xchg_0_i8_monotonic(ptr %a) nounwind { ; LA64-LABEL: atomicrmw_xchg_0_i8_monotonic: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a1, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a2, $zero, 255 ; LA64-NEXT: sll.w $a2, $a2, $a1 ; LA64-NEXT: nor $a2, $a2, $zero -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: amand_db.w $a3, $a2, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a1 ; LA64-NEXT: ret @@ -4059,9 +4059,9 @@ define i8 @atomicrmw_xchg_minus_1_i8_monotonic(ptr %a) nounwind { ; LA32-LABEL: atomicrmw_xchg_minus_1_i8_monotonic: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a1, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a2, $zero, 255 ; LA32-NEXT: sll.w $a2, $a2, $a1 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB130_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: or $a4, $a3, $a2 @@ -4074,9 +4074,9 @@ define i8 @atomicrmw_xchg_minus_1_i8_monotonic(ptr %a) nounwind { ; LA64-LABEL: atomicrmw_xchg_minus_1_i8_monotonic: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a1, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a2, $zero, 255 ; LA64-NEXT: sll.w $a2, $a2, $a1 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: amor_db.w $a3, $a2, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a1 ; LA64-NEXT: ret @@ -4087,46 +4087,46 @@ define i8 @atomicrmw_xchg_minus_1_i8_monotonic(ptr %a) nounwind { define i16 @atomicrmw_xchg_i16_monotonic(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_xchg_i16_monotonic: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB131_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: addi.w $a5, $a1, 0 ; LA32-NEXT: xor $a5, $a4, $a5 -; LA32-NEXT: and $a5, $a5, $a2 +; LA32-NEXT: and $a5, $a5, $a3 ; LA32-NEXT: xor $a5, $a4, $a5 ; LA32-NEXT: sc.w $a5, $a0, 0 ; LA32-NEXT: beqz $a5, .LBB131_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a4, $a3 +; LA32-NEXT: srl.w $a0, $a4, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_xchg_i16_monotonic: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB131_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: addi.w $a5, $a1, 0 ; LA64-NEXT: xor $a5, $a4, $a5 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB131_1 ; LA64-NEXT: # %bb.2: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw xchg ptr %a, i16 %b monotonic ret i16 %1 @@ -4135,31 +4135,31 @@ define i16 @atomicrmw_xchg_i16_monotonic(ptr %a, i16 %b) nounwind { define i16 @atomicrmw_xchg_0_i16_monotonic(ptr %a) nounwind { ; LA32-LABEL: atomicrmw_xchg_0_i16_monotonic: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a1, 15 -; LA32-NEXT: ori $a1, $a1, 4095 -; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: nor $a1, $a1, $zero +; LA32-NEXT: slli.w $a1, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a2, 15 +; LA32-NEXT: ori $a2, $a2, 4095 +; LA32-NEXT: sll.w $a2, $a2, $a1 +; LA32-NEXT: nor $a2, $a2, $zero ; LA32-NEXT: .LBB132_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 -; LA32-NEXT: and $a4, $a3, $a1 +; LA32-NEXT: and $a4, $a3, $a2 ; LA32-NEXT: sc.w $a4, $a0, 0 ; LA32-NEXT: beqz $a4, .LBB132_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a3, $a2 +; LA32-NEXT: srl.w $a0, $a3, $a1 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_xchg_0_i16_monotonic: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a1, 15 -; LA64-NEXT: ori $a1, $a1, 4095 -; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 -; LA64-NEXT: nor $a1, $a1, $zero +; LA64-NEXT: slli.d $a1, $a0, 3 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: amand_db.w $a3, $a1, $a0 -; LA64-NEXT: srl.w $a0, $a3, $a2 +; LA64-NEXT: lu12i.w $a2, 15 +; LA64-NEXT: ori $a2, $a2, 4095 +; LA64-NEXT: sll.w $a2, $a2, $a1 +; LA64-NEXT: nor $a2, $a2, $zero +; LA64-NEXT: amand_db.w $a3, $a2, $a0 +; LA64-NEXT: srl.w $a0, $a3, $a1 ; LA64-NEXT: ret %1 = atomicrmw xchg ptr %a, i16 0 monotonic ret i16 %1 @@ -4168,29 +4168,29 @@ define i16 @atomicrmw_xchg_0_i16_monotonic(ptr %a) nounwind { define i16 @atomicrmw_xchg_minus_1_i16_monotonic(ptr %a) nounwind { ; LA32-LABEL: atomicrmw_xchg_minus_1_i16_monotonic: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a1, 15 -; LA32-NEXT: ori $a1, $a1, 4095 -; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 +; LA32-NEXT: slli.w $a1, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a2, 15 +; LA32-NEXT: ori $a2, $a2, 4095 +; LA32-NEXT: sll.w $a2, $a2, $a1 ; LA32-NEXT: .LBB133_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 -; LA32-NEXT: or $a4, $a3, $a1 +; LA32-NEXT: or $a4, $a3, $a2 ; LA32-NEXT: sc.w $a4, $a0, 0 ; LA32-NEXT: beqz $a4, .LBB133_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a3, $a2 +; LA32-NEXT: srl.w $a0, $a3, $a1 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_xchg_minus_1_i16_monotonic: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a1, 15 -; LA64-NEXT: ori $a1, $a1, 4095 -; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 +; LA64-NEXT: slli.d $a1, $a0, 3 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: amor_db.w $a3, $a1, $a0 -; LA64-NEXT: srl.w $a0, $a3, $a2 +; LA64-NEXT: lu12i.w $a2, 15 +; LA64-NEXT: ori $a2, $a2, 4095 +; LA64-NEXT: sll.w $a2, $a2, $a1 +; LA64-NEXT: amor_db.w $a3, $a2, $a0 +; LA64-NEXT: srl.w $a0, $a3, $a1 ; LA64-NEXT: ret %1 = atomicrmw xchg ptr %a, i16 -1 monotonic ret i16 %1 @@ -4241,11 +4241,11 @@ define i8 @atomicrmw_add_i8_monotonic(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_add_i8_monotonic: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB136_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: add.w $a5, $a4, $a1 @@ -4261,13 +4261,13 @@ define i8 @atomicrmw_add_i8_monotonic(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_add_i8_monotonic: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB136_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: add.w $a5, $a4, $a1 @@ -4286,46 +4286,46 @@ define i8 @atomicrmw_add_i8_monotonic(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_add_i16_monotonic(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_add_i16_monotonic: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB137_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: add.w $a5, $a4, $a1 ; LA32-NEXT: xor $a5, $a4, $a5 -; LA32-NEXT: and $a5, $a5, $a2 +; LA32-NEXT: and $a5, $a5, $a3 ; LA32-NEXT: xor $a5, $a4, $a5 ; LA32-NEXT: sc.w $a5, $a0, 0 ; LA32-NEXT: beqz $a5, .LBB137_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a4, $a3 +; LA32-NEXT: srl.w $a0, $a4, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_add_i16_monotonic: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB137_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: add.w $a5, $a4, $a1 ; LA64-NEXT: xor $a5, $a4, $a5 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB137_1 ; LA64-NEXT: # %bb.2: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw add ptr %a, i16 %b monotonic ret i16 %1 @@ -4376,11 +4376,11 @@ define i8 @atomicrmw_sub_i8_monotonic(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_sub_i8_monotonic: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB140_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: sub.w $a5, $a4, $a1 @@ -4396,13 +4396,13 @@ define i8 @atomicrmw_sub_i8_monotonic(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_sub_i8_monotonic: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB140_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: sub.w $a5, $a4, $a1 @@ -4421,46 +4421,46 @@ define i8 @atomicrmw_sub_i8_monotonic(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_sub_i16_monotonic(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_sub_i16_monotonic: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB141_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: sub.w $a5, $a4, $a1 ; LA32-NEXT: xor $a5, $a4, $a5 -; LA32-NEXT: and $a5, $a5, $a2 +; LA32-NEXT: and $a5, $a5, $a3 ; LA32-NEXT: xor $a5, $a4, $a5 ; LA32-NEXT: sc.w $a5, $a0, 0 ; LA32-NEXT: beqz $a5, .LBB141_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a4, $a3 +; LA32-NEXT: srl.w $a0, $a4, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_sub_i16_monotonic: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB141_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: sub.w $a5, $a4, $a1 ; LA64-NEXT: xor $a5, $a4, $a5 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB141_1 ; LA64-NEXT: # %bb.2: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw sub ptr %a, i16 %b monotonic ret i16 %1 @@ -4513,11 +4513,11 @@ define i8 @atomicrmw_nand_i8_monotonic(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_nand_i8_monotonic: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB144_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: and $a5, $a4, $a1 @@ -4534,13 +4534,13 @@ define i8 @atomicrmw_nand_i8_monotonic(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_nand_i8_monotonic: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 -; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB144_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a5, $a4, $a1 @@ -4560,48 +4560,48 @@ define i8 @atomicrmw_nand_i8_monotonic(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_nand_i16_monotonic(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_nand_i16_monotonic: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB145_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a4, $a0, 0 ; LA32-NEXT: and $a5, $a4, $a1 ; LA32-NEXT: nor $a5, $a5, $zero ; LA32-NEXT: xor $a5, $a4, $a5 -; LA32-NEXT: and $a5, $a5, $a2 +; LA32-NEXT: and $a5, $a5, $a3 ; LA32-NEXT: xor $a5, $a4, $a5 ; LA32-NEXT: sc.w $a5, $a0, 0 ; LA32-NEXT: beqz $a5, .LBB145_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a4, $a3 +; LA32-NEXT: srl.w $a0, $a4, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_nand_i16_monotonic: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: addi.w $a2, $a2, 0 +; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: addi.w $a1, $a1, 0 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: addi.w $a3, $a3, 0 ; LA64-NEXT: .LBB145_1: # =>This Inner Loop Header: Depth=1 ; LA64-NEXT: ll.w $a4, $a0, 0 ; LA64-NEXT: and $a5, $a4, $a1 ; LA64-NEXT: nor $a5, $a5, $zero ; LA64-NEXT: xor $a5, $a4, $a5 -; LA64-NEXT: and $a5, $a5, $a2 +; LA64-NEXT: and $a5, $a5, $a3 ; LA64-NEXT: xor $a5, $a4, $a5 ; LA64-NEXT: sc.w $a5, $a0, 0 ; LA64-NEXT: beqz $a5, .LBB145_1 ; LA64-NEXT: # %bb.2: -; LA64-NEXT: srl.w $a0, $a4, $a3 +; LA64-NEXT: srl.w $a0, $a4, $a2 ; LA64-NEXT: ret %1 = atomicrmw nand ptr %a, i16 %b monotonic ret i16 %1 @@ -4665,12 +4665,12 @@ define i8 @atomicrmw_and_i8_monotonic(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_and_i8_monotonic: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a0, 3 +; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: ori $a3, $zero, 255 ; LA32-NEXT: sll.w $a3, $a3, $a2 ; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: orn $a1, $a1, $a3 -; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 ; LA32-NEXT: .LBB148_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: and $a4, $a3, $a1 @@ -4683,12 +4683,12 @@ define i8 @atomicrmw_and_i8_monotonic(ptr %a, i8 %b) nounwind { ; LA64-LABEL: atomicrmw_and_i8_monotonic: ; LA64: # %bb.0: ; LA64-NEXT: slli.d $a2, $a0, 3 +; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: ori $a3, $zero, 255 ; LA64-NEXT: sll.w $a3, $a3, $a2 ; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: orn $a1, $a1, $a3 -; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 ; LA64-NEXT: amand_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -4699,35 +4699,35 @@ define i8 @atomicrmw_and_i8_monotonic(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_and_i16_monotonic(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_and_i16_monotonic: ; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a2, 15 -; LA32-NEXT: ori $a2, $a2, 4095 -; LA32-NEXT: slli.w $a3, $a0, 3 -; LA32-NEXT: sll.w $a2, $a2, $a3 -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 -; LA32-NEXT: sll.w $a1, $a1, $a3 -; LA32-NEXT: orn $a1, $a1, $a2 +; LA32-NEXT: slli.w $a2, $a0, 3 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: lu12i.w $a3, 15 +; LA32-NEXT: ori $a3, $a3, 4095 +; LA32-NEXT: sll.w $a3, $a3, $a2 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 +; LA32-NEXT: orn $a1, $a1, $a3 ; LA32-NEXT: .LBB149_1: # =>This Inner Loop Header: Depth=1 -; LA32-NEXT: ll.w $a2, $a0, 0 -; LA32-NEXT: and $a4, $a2, $a1 +; LA32-NEXT: ll.w $a3, $a0, 0 +; LA32-NEXT: and $a4, $a3, $a1 ; LA32-NEXT: sc.w $a4, $a0, 0 ; LA32-NEXT: beqz $a4, .LBB149_1 ; LA32-NEXT: # %bb.2: -; LA32-NEXT: srl.w $a0, $a2, $a3 +; LA32-NEXT: srl.w $a0, $a3, $a2 ; LA32-NEXT: ret ; ; LA64-LABEL: atomicrmw_and_i16_monotonic: ; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a2, 15 -; LA64-NEXT: ori $a2, $a2, 4095 -; LA64-NEXT: slli.d $a3, $a0, 3 -; LA64-NEXT: sll.w $a2, $a2, $a3 -; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 -; LA64-NEXT: sll.w $a1, $a1, $a3 -; LA64-NEXT: orn $a1, $a1, $a2 +; LA64-NEXT: slli.d $a2, $a0, 3 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 -; LA64-NEXT: amand_db.w $a2, $a1, $a0 -; LA64-NEXT: srl.w $a0, $a2, $a3 +; LA64-NEXT: lu12i.w $a3, 15 +; LA64-NEXT: ori $a3, $a3, 4095 +; LA64-NEXT: sll.w $a3, $a3, $a2 +; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 +; LA64-NEXT: sll.w $a1, $a1, $a2 +; LA64-NEXT: orn $a1, $a1, $a3 +; LA64-NEXT: amand_db.w $a3, $a1, $a0 +; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret %1 = atomicrmw and ptr %a, i16 %b monotonic ret i16 %1 @@ -4777,10 +4777,10 @@ define i64 @atomicrmw_and_i64_monotonic(ptr %a, i64 %b) nounwind { define i8 @atomicrmw_or_i8_monotonic(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_or_i8_monotonic: ; LA32: # %bb.0: -; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: andi $a1, $a1, 255 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB152_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: or $a4, $a3, $a1 @@ -4792,10 +4792,10 @@ define i8 @atomicrmw_or_i8_monotonic(ptr %a, i8 %b) nounwind { ; ; LA64-LABEL: atomicrmw_or_i8_monotonic: ; LA64: # %bb.0: -; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: andi $a1, $a1, 255 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: amor_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -4806,10 +4806,10 @@ define i8 @atomicrmw_or_i8_monotonic(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_or_i16_monotonic(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_or_i16_monotonic: ; LA32: # %bb.0: -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 ; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB153_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: or $a4, $a3, $a1 @@ -4821,10 +4821,10 @@ define i16 @atomicrmw_or_i16_monotonic(ptr %a, i16 %b) nounwind { ; ; LA64-LABEL: atomicrmw_or_i16_monotonic: ; LA64: # %bb.0: -; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: amor_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -4876,10 +4876,10 @@ define i64 @atomicrmw_or_i64_monotonic(ptr %a, i64 %b) nounwind { define i8 @atomicrmw_xor_i8_monotonic(ptr %a, i8 %b) nounwind { ; LA32-LABEL: atomicrmw_xor_i8_monotonic: ; LA32: # %bb.0: -; LA32-NEXT: andi $a1, $a1, 255 ; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: andi $a1, $a1, 255 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB156_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: xor $a4, $a3, $a1 @@ -4891,10 +4891,10 @@ define i8 @atomicrmw_xor_i8_monotonic(ptr %a, i8 %b) nounwind { ; ; LA64-LABEL: atomicrmw_xor_i8_monotonic: ; LA64: # %bb.0: -; LA64-NEXT: andi $a1, $a1, 255 ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: andi $a1, $a1, 255 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: amxor_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret @@ -4905,10 +4905,10 @@ define i8 @atomicrmw_xor_i8_monotonic(ptr %a, i8 %b) nounwind { define i16 @atomicrmw_xor_i16_monotonic(ptr %a, i16 %b) nounwind { ; LA32-LABEL: atomicrmw_xor_i16_monotonic: ; LA32: # %bb.0: -; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 ; LA32-NEXT: slli.w $a2, $a0, 3 -; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: bstrins.w $a0, $zero, 1, 0 +; LA32-NEXT: bstrpick.w $a1, $a1, 15, 0 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: .LBB157_1: # =>This Inner Loop Header: Depth=1 ; LA32-NEXT: ll.w $a3, $a0, 0 ; LA32-NEXT: xor $a4, $a3, $a1 @@ -4920,10 +4920,10 @@ define i16 @atomicrmw_xor_i16_monotonic(ptr %a, i16 %b) nounwind { ; ; LA64-LABEL: atomicrmw_xor_i16_monotonic: ; LA64: # %bb.0: -; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 ; LA64-NEXT: slli.d $a2, $a0, 3 -; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: bstrins.d $a0, $zero, 1, 0 +; LA64-NEXT: bstrpick.d $a1, $a1, 15, 0 +; LA64-NEXT: sll.w $a1, $a1, $a2 ; LA64-NEXT: amxor_db.w $a3, $a1, $a0 ; LA64-NEXT: srl.w $a0, $a3, $a2 ; LA64-NEXT: ret diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/double-convert.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/double-convert.ll index 03a126a736ef..ef117f974887 100644 --- a/llvm/test/CodeGen/LoongArch/ir-instruction/double-convert.ll +++ b/llvm/test/CodeGen/LoongArch/ir-instruction/double-convert.ll @@ -119,12 +119,12 @@ define i32 @convert_double_to_u32(double %a) nounwind { ; LA32-NEXT: pcalau12i $a0, %pc_hi20(.LCPI7_0) ; LA32-NEXT: addi.w $a0, $a0, %pc_lo12(.LCPI7_0) ; LA32-NEXT: fld.d $fa1, $a0, 0 -; LA32-NEXT: fsub.d $fa2, $fa0, $fa1 -; LA32-NEXT: ftintrz.w.d $fa2, $fa2 -; LA32-NEXT: movfr2gr.s $a0, $fa2 +; LA32-NEXT: fcmp.clt.d $fcc0, $fa0, $fa1 +; LA32-NEXT: fsub.d $fa1, $fa0, $fa1 +; LA32-NEXT: ftintrz.w.d $fa1, $fa1 +; LA32-NEXT: movfr2gr.s $a0, $fa1 ; LA32-NEXT: lu12i.w $a1, -524288 ; LA32-NEXT: xor $a0, $a0, $a1 -; LA32-NEXT: fcmp.clt.d $fcc0, $fa0, $fa1 ; LA32-NEXT: movcf2gr $a1, $fcc0 ; LA32-NEXT: masknez $a0, $a0, $a1 ; LA32-NEXT: ftintrz.w.d $fa0, $fa0 @@ -176,12 +176,12 @@ define i64 @convert_double_to_u64(double %a) nounwind { ; LA64-NEXT: pcalau12i $a0, %pc_hi20(.LCPI9_0) ; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(.LCPI9_0) ; LA64-NEXT: fld.d $fa1, $a0, 0 -; LA64-NEXT: fsub.d $fa2, $fa0, $fa1 -; LA64-NEXT: ftintrz.l.d $fa2, $fa2 -; LA64-NEXT: movfr2gr.d $a0, $fa2 +; LA64-NEXT: fcmp.clt.d $fcc0, $fa0, $fa1 +; LA64-NEXT: fsub.d $fa1, $fa0, $fa1 +; LA64-NEXT: ftintrz.l.d $fa1, $fa1 +; LA64-NEXT: movfr2gr.d $a0, $fa1 ; LA64-NEXT: lu52i.d $a1, $zero, -2048 ; LA64-NEXT: xor $a0, $a0, $a1 -; LA64-NEXT: fcmp.clt.d $fcc0, $fa0, $fa1 ; LA64-NEXT: movcf2gr $a1, $fcc0 ; LA64-NEXT: masknez $a0, $a0, $a1 ; LA64-NEXT: ftintrz.l.d $fa0, $fa0 @@ -232,11 +232,11 @@ define double @convert_u32_to_double(i32 %a) nounwind { ; LA32-NEXT: lu12i.w $a1, 275200 ; LA32-NEXT: st.w $a1, $sp, 12 ; LA32-NEXT: st.w $a0, $sp, 8 +; LA32-NEXT: fld.d $fa0, $sp, 8 ; LA32-NEXT: pcalau12i $a0, %pc_hi20(.LCPI12_0) ; LA32-NEXT: addi.w $a0, $a0, %pc_lo12(.LCPI12_0) -; LA32-NEXT: fld.d $fa0, $a0, 0 -; LA32-NEXT: fld.d $fa1, $sp, 8 -; LA32-NEXT: fsub.d $fa0, $fa1, $fa0 +; LA32-NEXT: fld.d $fa1, $a0, 0 +; LA32-NEXT: fsub.d $fa0, $fa0, $fa1 ; LA32-NEXT: addi.w $sp, $sp, 16 ; LA32-NEXT: ret ; @@ -263,13 +263,13 @@ define double @convert_u64_to_double(i64 %a) nounwind { ; LA64-LABEL: convert_u64_to_double: ; LA64: # %bb.0: ; LA64-NEXT: srli.d $a1, $a0, 32 +; LA64-NEXT: pcalau12i $a2, %pc_hi20(.LCPI13_0) +; LA64-NEXT: addi.d $a2, $a2, %pc_lo12(.LCPI13_0) +; LA64-NEXT: fld.d $fa0, $a2, 0 ; LA64-NEXT: lu52i.d $a2, $zero, 1107 ; LA64-NEXT: or $a1, $a1, $a2 -; LA64-NEXT: movgr2fr.d $fa0, $a1 -; LA64-NEXT: pcalau12i $a1, %pc_hi20(.LCPI13_0) -; LA64-NEXT: addi.d $a1, $a1, %pc_lo12(.LCPI13_0) -; LA64-NEXT: fld.d $fa1, $a1, 0 -; LA64-NEXT: fsub.d $fa0, $fa0, $fa1 +; LA64-NEXT: movgr2fr.d $fa1, $a1 +; LA64-NEXT: fsub.d $fa0, $fa1, $fa0 ; LA64-NEXT: lu12i.w $a1, 275200 ; LA64-NEXT: bstrins.d $a0, $a1, 63, 32 ; LA64-NEXT: movgr2fr.d $fa1, $a0 diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/float-convert.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/float-convert.ll index b57d96aee32f..b01b84ba385e 100644 --- a/llvm/test/CodeGen/LoongArch/ir-instruction/float-convert.ll +++ b/llvm/test/CodeGen/LoongArch/ir-instruction/float-convert.ll @@ -184,12 +184,12 @@ define i32 @convert_float_to_u32(float %a) nounwind { ; LA32F-NEXT: pcalau12i $a0, %pc_hi20(.LCPI6_0) ; LA32F-NEXT: addi.w $a0, $a0, %pc_lo12(.LCPI6_0) ; LA32F-NEXT: fld.s $fa1, $a0, 0 -; LA32F-NEXT: fsub.s $fa2, $fa0, $fa1 -; LA32F-NEXT: ftintrz.w.s $fa2, $fa2 -; LA32F-NEXT: movfr2gr.s $a0, $fa2 +; LA32F-NEXT: fcmp.clt.s $fcc0, $fa0, $fa1 +; LA32F-NEXT: fsub.s $fa1, $fa0, $fa1 +; LA32F-NEXT: ftintrz.w.s $fa1, $fa1 +; LA32F-NEXT: movfr2gr.s $a0, $fa1 ; LA32F-NEXT: lu12i.w $a1, -524288 ; LA32F-NEXT: xor $a0, $a0, $a1 -; LA32F-NEXT: fcmp.clt.s $fcc0, $fa0, $fa1 ; LA32F-NEXT: movcf2gr $a1, $fcc0 ; LA32F-NEXT: masknez $a0, $a0, $a1 ; LA32F-NEXT: ftintrz.w.s $fa0, $fa0 @@ -203,12 +203,12 @@ define i32 @convert_float_to_u32(float %a) nounwind { ; LA32D-NEXT: pcalau12i $a0, %pc_hi20(.LCPI6_0) ; LA32D-NEXT: addi.w $a0, $a0, %pc_lo12(.LCPI6_0) ; LA32D-NEXT: fld.s $fa1, $a0, 0 -; LA32D-NEXT: fsub.s $fa2, $fa0, $fa1 -; LA32D-NEXT: ftintrz.w.s $fa2, $fa2 -; LA32D-NEXT: movfr2gr.s $a0, $fa2 +; LA32D-NEXT: fcmp.clt.s $fcc0, $fa0, $fa1 +; LA32D-NEXT: fsub.s $fa1, $fa0, $fa1 +; LA32D-NEXT: ftintrz.w.s $fa1, $fa1 +; LA32D-NEXT: movfr2gr.s $a0, $fa1 ; LA32D-NEXT: lu12i.w $a1, -524288 ; LA32D-NEXT: xor $a0, $a0, $a1 -; LA32D-NEXT: fcmp.clt.s $fcc0, $fa0, $fa1 ; LA32D-NEXT: movcf2gr $a1, $fcc0 ; LA32D-NEXT: masknez $a0, $a0, $a1 ; LA32D-NEXT: ftintrz.w.s $fa0, $fa0 @@ -222,12 +222,12 @@ define i32 @convert_float_to_u32(float %a) nounwind { ; LA64F-NEXT: pcalau12i $a0, %pc_hi20(.LCPI6_0) ; LA64F-NEXT: addi.d $a0, $a0, %pc_lo12(.LCPI6_0) ; LA64F-NEXT: fld.s $fa1, $a0, 0 -; LA64F-NEXT: fsub.s $fa2, $fa0, $fa1 -; LA64F-NEXT: ftintrz.w.s $fa2, $fa2 -; LA64F-NEXT: movfr2gr.s $a0, $fa2 +; LA64F-NEXT: fcmp.clt.s $fcc0, $fa0, $fa1 +; LA64F-NEXT: fsub.s $fa1, $fa0, $fa1 +; LA64F-NEXT: ftintrz.w.s $fa1, $fa1 +; LA64F-NEXT: movfr2gr.s $a0, $fa1 ; LA64F-NEXT: lu12i.w $a1, -524288 ; LA64F-NEXT: xor $a0, $a0, $a1 -; LA64F-NEXT: fcmp.clt.s $fcc0, $fa0, $fa1 ; LA64F-NEXT: movcf2gr $a1, $fcc0 ; LA64F-NEXT: masknez $a0, $a0, $a1 ; LA64F-NEXT: ftintrz.w.s $fa0, $fa0 @@ -269,12 +269,12 @@ define i64 @convert_float_to_u64(float %a) nounwind { ; LA64F-NEXT: pcalau12i $a0, %pc_hi20(.LCPI7_0) ; LA64F-NEXT: addi.d $a0, $a0, %pc_lo12(.LCPI7_0) ; LA64F-NEXT: fld.s $fa1, $a0, 0 -; LA64F-NEXT: fsub.s $fa2, $fa0, $fa1 -; LA64F-NEXT: ftintrz.w.s $fa2, $fa2 -; LA64F-NEXT: movfr2gr.s $a0, $fa2 +; LA64F-NEXT: fcmp.clt.s $fcc0, $fa0, $fa1 +; LA64F-NEXT: fsub.s $fa1, $fa0, $fa1 +; LA64F-NEXT: ftintrz.w.s $fa1, $fa1 +; LA64F-NEXT: movfr2gr.s $a0, $fa1 ; LA64F-NEXT: lu52i.d $a1, $zero, -2048 ; LA64F-NEXT: xor $a0, $a0, $a1 -; LA64F-NEXT: fcmp.clt.s $fcc0, $fa0, $fa1 ; LA64F-NEXT: movcf2gr $a1, $fcc0 ; LA64F-NEXT: masknez $a0, $a0, $a1 ; LA64F-NEXT: ftintrz.w.s $fa0, $fa0 @@ -288,12 +288,12 @@ define i64 @convert_float_to_u64(float %a) nounwind { ; LA64D-NEXT: pcalau12i $a0, %pc_hi20(.LCPI7_0) ; LA64D-NEXT: addi.d $a0, $a0, %pc_lo12(.LCPI7_0) ; LA64D-NEXT: fld.s $fa1, $a0, 0 -; LA64D-NEXT: fsub.s $fa2, $fa0, $fa1 -; LA64D-NEXT: ftintrz.l.s $fa2, $fa2 -; LA64D-NEXT: movfr2gr.d $a0, $fa2 +; LA64D-NEXT: fcmp.clt.s $fcc0, $fa0, $fa1 +; LA64D-NEXT: fsub.s $fa1, $fa0, $fa1 +; LA64D-NEXT: ftintrz.l.s $fa1, $fa1 +; LA64D-NEXT: movfr2gr.d $a0, $fa1 ; LA64D-NEXT: lu52i.d $a1, $zero, -2048 ; LA64D-NEXT: xor $a0, $a0, $a1 -; LA64D-NEXT: fcmp.clt.s $fcc0, $fa0, $fa1 ; LA64D-NEXT: movcf2gr $a1, $fcc0 ; LA64D-NEXT: masknez $a0, $a0, $a1 ; LA64D-NEXT: ftintrz.l.s $fa0, $fa0 @@ -504,11 +504,11 @@ define float @convert_u32_to_float(i32 %a) nounwind { ; LA32D-NEXT: lu12i.w $a1, 275200 ; LA32D-NEXT: st.w $a1, $sp, 12 ; LA32D-NEXT: st.w $a0, $sp, 8 +; LA32D-NEXT: fld.d $fa0, $sp, 8 ; LA32D-NEXT: pcalau12i $a0, %pc_hi20(.LCPI14_0) ; LA32D-NEXT: addi.w $a0, $a0, %pc_lo12(.LCPI14_0) -; LA32D-NEXT: fld.d $fa0, $a0, 0 -; LA32D-NEXT: fld.d $fa1, $sp, 8 -; LA32D-NEXT: fsub.d $fa0, $fa1, $fa0 +; LA32D-NEXT: fld.d $fa1, $a0, 0 +; LA32D-NEXT: fsub.d $fa0, $fa0, $fa1 ; LA32D-NEXT: fcvt.s.d $fa0, $fa0 ; LA32D-NEXT: addi.w $sp, $sp, 16 ; LA32D-NEXT: ret diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/load-store.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/load-store.ll index 387a62bd6c00..1b3cda5547da 100644 --- a/llvm/test/CodeGen/LoongArch/ir-instruction/load-store.ll +++ b/llvm/test/CodeGen/LoongArch/ir-instruction/load-store.ll @@ -1,34 +1,49 @@ -; RUN: llc --mtriple=loongarch32 --mattr=+d --relocation-model=static < %s | FileCheck %s --check-prefixes=ALL,LA32NOPIC,LA32 -; RUN: llc --mtriple=loongarch32 --mattr=+d --relocation-model=pic < %s | FileCheck %s --check-prefixes=ALL,LA32PIC,LA32 -; RUN: llc --mtriple=loongarch64 --mattr=+d --relocation-model=static < %s | FileCheck %s --check-prefixes=ALL,LA64NOPIC,LA64 -; RUN: llc --mtriple=loongarch64 --mattr=+d --relocation-model=pic < %s | FileCheck %s --check-prefixes=ALL,LA64PIC,LA64 +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc --mtriple=loongarch32 --mattr=+d --relocation-model=static < %s | FileCheck %s --check-prefixes=ALL,LA32NOPIC +; RUN: llc --mtriple=loongarch32 --mattr=+d --relocation-model=pic < %s | FileCheck %s --check-prefixes=ALL,LA32PIC +; RUN: llc --mtriple=loongarch64 --mattr=+d --relocation-model=static < %s | FileCheck %s --check-prefixes=ALL,LA64NOPIC +; RUN: llc --mtriple=loongarch64 --mattr=+d --relocation-model=pic < %s | FileCheck %s --check-prefixes=ALL,LA64PIC ;; Check load from and store to global variables. @G = dso_local global i32 zeroinitializer, align 4 @arr = dso_local global [10 x i32] zeroinitializer, align 4 define i32 @load_store_global() nounwind { -; ALL-LABEL: load_store_global: -; ALL: # %bb.0: - -; LA32NOPIC-NEXT: pcalau12i $a0, %pc_hi20(G) -; LA32NOPIC-NEXT: addi.w $a1, $a0, %pc_lo12(G) -; LA32PIC-NEXT: pcalau12i $a0, %pc_hi20(.LG$local) -; LA32PIC-NEXT: addi.w $a1, $a0, %pc_lo12(.LG$local) -; LA32-NEXT: ld.w $a0, $a1, 0 -; LA32-NEXT: addi.w $a0, $a0, 1 -; LA32-NEXT: st.w $a0, $a1, 0 - -; LA64NOPIC-NEXT: pcalau12i $a0, %pc_hi20(G) -; LA64NOPIC-NEXT: addi.d $a1, $a0, %pc_lo12(G) -; LA64PIC-NEXT: pcalau12i $a0, %pc_hi20(.LG$local) -; LA64PIC-NEXT: addi.d $a1, $a0, %pc_lo12(.LG$local) -; LA64-NEXT: ld.w $a0, $a1, 0 -; LA64-NEXT: addi.d $a0, $a0, 1 -; LA64-NEXT: st.w $a0, $a1, 0 - -; ALL-NEXT: ret - +; LA32NOPIC-LABEL: load_store_global: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: pcalau12i $a0, %pc_hi20(G) +; LA32NOPIC-NEXT: addi.w $a1, $a0, %pc_lo12(G) +; LA32NOPIC-NEXT: ld.w $a0, $a1, 0 +; LA32NOPIC-NEXT: addi.w $a0, $a0, 1 +; LA32NOPIC-NEXT: st.w $a0, $a1, 0 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: load_store_global: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: pcalau12i $a0, %pc_hi20(.LG$local) +; LA32PIC-NEXT: addi.w $a1, $a0, %pc_lo12(.LG$local) +; LA32PIC-NEXT: ld.w $a0, $a1, 0 +; LA32PIC-NEXT: addi.w $a0, $a0, 1 +; LA32PIC-NEXT: st.w $a0, $a1, 0 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: load_store_global: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: pcalau12i $a0, %pc_hi20(G) +; LA64NOPIC-NEXT: addi.d $a1, $a0, %pc_lo12(G) +; LA64NOPIC-NEXT: ld.w $a0, $a1, 0 +; LA64NOPIC-NEXT: addi.d $a0, $a0, 1 +; LA64NOPIC-NEXT: st.w $a0, $a1, 0 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: load_store_global: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: pcalau12i $a0, %pc_hi20(.LG$local) +; LA64PIC-NEXT: addi.d $a1, $a0, %pc_lo12(.LG$local) +; LA64PIC-NEXT: ld.w $a0, $a1, 0 +; LA64PIC-NEXT: addi.d $a0, $a0, 1 +; LA64PIC-NEXT: st.w $a0, $a1, 0 +; LA64PIC-NEXT: ret %v = load i32, ptr @G %sum = add i32 %v, 1 store i32 %sum, ptr @G @@ -36,34 +51,49 @@ define i32 @load_store_global() nounwind { } define i32 @load_store_global_array(i32 %a) nounwind { -; ALL-LABEL: load_store_global_array: -; ALL: # %bb.0: - -; LA32NOPIC-NEXT: pcalau12i $a1, %pc_hi20(arr) -; LA32NOPIC-NEXT: addi.w $a2, $a1, %pc_lo12(arr) -; LA32PIC-NEXT: pcalau12i $a1, %pc_hi20(.Larr$local) -; LA32PIC-NEXT: addi.w $a2, $a1, %pc_lo12(.Larr$local) -; LA32-NEXT: ld.w $a1, $a2, 0 -; LA32-NEXT: st.w $a0, $a2, 0 -; LA32NOPIC-NEXT: ld.w $a3, $a2, 36 -; LA32NOPIC-NEXT: st.w $a0, $a2, 36 -; LA32PIC-NEXT: ld.w $a3, $a2, 36 -; LA32PIC-NEXT: st.w $a0, $a2, 36 - -; LA64NOPIC-NEXT: pcalau12i $a1, %pc_hi20(arr) -; LA64NOPIC-NEXT: addi.d $a2, $a1, %pc_lo12(arr) -; LA64PIC-NEXT: pcalau12i $a1, %pc_hi20(.Larr$local) -; LA64PIC-NEXT: addi.d $a2, $a1, %pc_lo12(.Larr$local) -; LA64-NEXT: ld.w $a1, $a2, 0 -; LA64-NEXT: st.w $a0, $a2, 0 -; LA64NOPIC-NEXT: ld.w $a3, $a2, 36 -; LA64NOPIC-NEXT: st.w $a0, $a2, 36 -; LA64PIC-NEXT: ld.w $a3, $a2, 36 -; LA64PIC-NEXT: st.w $a0, $a2, 36 - -; ALL-NEXT: move $a0, $a1 -; ALL-NEXT: ret - +; LA32NOPIC-LABEL: load_store_global_array: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: pcalau12i $a1, %pc_hi20(arr) +; LA32NOPIC-NEXT: addi.w $a2, $a1, %pc_lo12(arr) +; LA32NOPIC-NEXT: ld.w $a1, $a2, 0 +; LA32NOPIC-NEXT: st.w $a0, $a2, 0 +; LA32NOPIC-NEXT: ld.w $a3, $a2, 36 +; LA32NOPIC-NEXT: st.w $a0, $a2, 36 +; LA32NOPIC-NEXT: move $a0, $a1 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: load_store_global_array: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: pcalau12i $a1, %pc_hi20(.Larr$local) +; LA32PIC-NEXT: addi.w $a2, $a1, %pc_lo12(.Larr$local) +; LA32PIC-NEXT: ld.w $a1, $a2, 0 +; LA32PIC-NEXT: st.w $a0, $a2, 0 +; LA32PIC-NEXT: ld.w $a3, $a2, 36 +; LA32PIC-NEXT: st.w $a0, $a2, 36 +; LA32PIC-NEXT: move $a0, $a1 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: load_store_global_array: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: pcalau12i $a1, %pc_hi20(arr) +; LA64NOPIC-NEXT: addi.d $a2, $a1, %pc_lo12(arr) +; LA64NOPIC-NEXT: ld.w $a1, $a2, 0 +; LA64NOPIC-NEXT: st.w $a0, $a2, 0 +; LA64NOPIC-NEXT: ld.w $a3, $a2, 36 +; LA64NOPIC-NEXT: st.w $a0, $a2, 36 +; LA64NOPIC-NEXT: move $a0, $a1 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: load_store_global_array: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: pcalau12i $a1, %pc_hi20(.Larr$local) +; LA64PIC-NEXT: addi.d $a2, $a1, %pc_lo12(.Larr$local) +; LA64PIC-NEXT: ld.w $a1, $a2, 0 +; LA64PIC-NEXT: st.w $a0, $a2, 0 +; LA64PIC-NEXT: ld.w $a3, $a2, 36 +; LA64PIC-NEXT: st.w $a0, $a2, 36 +; LA64PIC-NEXT: move $a0, $a1 +; LA64PIC-NEXT: ret %1 = load volatile i32, ptr @arr, align 4 store i32 %a, ptr @arr, align 4 %2 = getelementptr [10 x i32], ptr @arr, i32 0, i32 9 @@ -75,18 +105,35 @@ define i32 @load_store_global_array(i32 %a) nounwind { ;; Check indexed and unindexed, sext, zext and anyext loads. define i64 @ld_b(ptr %a) nounwind { -; LA32-LABEL: ld_b: -; LA32: # %bb.0: -; LA32-NEXT: ld.b $a1, $a0, 0 -; LA32-NEXT: ld.b $a0, $a0, 1 -; LA32-NEXT: srai.w $a1, $a0, 31 -; LA32-NEXT: ret -; -; LA64-LABEL: ld_b: -; LA64: # %bb.0: -; LA64-NEXT: ld.b $a1, $a0, 0 -; LA64-NEXT: ld.b $a0, $a0, 1 -; LA64-NEXT: ret +; LA32NOPIC-LABEL: ld_b: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: ld.b $a2, $a0, 1 +; LA32NOPIC-NEXT: ld.b $a0, $a0, 0 +; LA32NOPIC-NEXT: srai.w $a1, $a2, 31 +; LA32NOPIC-NEXT: move $a0, $a2 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: ld_b: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: ld.b $a2, $a0, 1 +; LA32PIC-NEXT: ld.b $a0, $a0, 0 +; LA32PIC-NEXT: srai.w $a1, $a2, 31 +; LA32PIC-NEXT: move $a0, $a2 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: ld_b: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: ld.b $a1, $a0, 1 +; LA64NOPIC-NEXT: ld.b $a0, $a0, 0 +; LA64NOPIC-NEXT: move $a0, $a1 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: ld_b: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: ld.b $a1, $a0, 1 +; LA64PIC-NEXT: ld.b $a0, $a0, 0 +; LA64PIC-NEXT: move $a0, $a1 +; LA64PIC-NEXT: ret %1 = getelementptr i8, ptr %a, i64 1 %2 = load i8, ptr %1 %3 = sext i8 %2 to i64 @@ -95,18 +142,35 @@ define i64 @ld_b(ptr %a) nounwind { } define i64 @ld_h(ptr %a) nounwind { -; LA32-LABEL: ld_h: -; LA32: # %bb.0: -; LA32-NEXT: ld.h $a1, $a0, 0 -; LA32-NEXT: ld.h $a0, $a0, 4 -; LA32-NEXT: srai.w $a1, $a0, 31 -; LA32-NEXT: ret -; -; LA64-LABEL: ld_h: -; LA64: # %bb.0: -; LA64-NEXT: ld.h $a1, $a0, 0 -; LA64-NEXT: ld.h $a0, $a0, 4 -; LA64-NEXT: ret +; LA32NOPIC-LABEL: ld_h: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: ld.h $a2, $a0, 4 +; LA32NOPIC-NEXT: ld.h $a0, $a0, 0 +; LA32NOPIC-NEXT: srai.w $a1, $a2, 31 +; LA32NOPIC-NEXT: move $a0, $a2 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: ld_h: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: ld.h $a2, $a0, 4 +; LA32PIC-NEXT: ld.h $a0, $a0, 0 +; LA32PIC-NEXT: srai.w $a1, $a2, 31 +; LA32PIC-NEXT: move $a0, $a2 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: ld_h: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: ld.h $a1, $a0, 4 +; LA64NOPIC-NEXT: ld.h $a0, $a0, 0 +; LA64NOPIC-NEXT: move $a0, $a1 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: ld_h: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: ld.h $a1, $a0, 4 +; LA64PIC-NEXT: ld.h $a0, $a0, 0 +; LA64PIC-NEXT: move $a0, $a1 +; LA64PIC-NEXT: ret %1 = getelementptr i16, ptr %a, i64 2 %2 = load i16, ptr %1 %3 = sext i16 %2 to i64 @@ -115,18 +179,35 @@ define i64 @ld_h(ptr %a) nounwind { } define i64 @ld_w(ptr %a) nounwind { -; LA32-LABEL: ld_w: -; LA32: # %bb.0: -; LA32-NEXT: ld.w $a1, $a0, 0 -; LA32-NEXT: ld.w $a0, $a0, 12 -; LA32-NEXT: srai.w $a1, $a0, 31 -; LA32-NEXT: ret -; -; LA64-LABEL: ld_w: -; LA64: # %bb.0: -; LA64-NEXT: ld.w $a1, $a0, 0 -; LA64-NEXT: ld.w $a0, $a0, 12 -; LA64-NEXT: ret +; LA32NOPIC-LABEL: ld_w: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: ld.w $a2, $a0, 12 +; LA32NOPIC-NEXT: ld.w $a0, $a0, 0 +; LA32NOPIC-NEXT: srai.w $a1, $a2, 31 +; LA32NOPIC-NEXT: move $a0, $a2 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: ld_w: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: ld.w $a2, $a0, 12 +; LA32PIC-NEXT: ld.w $a0, $a0, 0 +; LA32PIC-NEXT: srai.w $a1, $a2, 31 +; LA32PIC-NEXT: move $a0, $a2 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: ld_w: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: ld.w $a1, $a0, 12 +; LA64NOPIC-NEXT: ld.w $a0, $a0, 0 +; LA64NOPIC-NEXT: move $a0, $a1 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: ld_w: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: ld.w $a1, $a0, 12 +; LA64PIC-NEXT: ld.w $a0, $a0, 0 +; LA64PIC-NEXT: move $a0, $a1 +; LA64PIC-NEXT: ret %1 = getelementptr i32, ptr %a, i64 3 %2 = load i32, ptr %1 %3 = sext i32 %2 to i64 @@ -135,19 +216,37 @@ define i64 @ld_w(ptr %a) nounwind { } define i64 @ld_d(ptr %a) nounwind { -; LA32-LABEL: ld_d: -; LA32: # %bb.0: -; LA32-NEXT: ld.w $a1, $a0, 4 -; LA32-NEXT: ld.w $a1, $a0, 0 -; LA32-NEXT: ld.w $a1, $a0, 28 -; LA32-NEXT: ld.w $a0, $a0, 24 -; LA32-NEXT: ret -; -; LA64-LABEL: ld_d: -; LA64: # %bb.0: -; LA64-NEXT: ld.d $a1, $a0, 0 -; LA64-NEXT: ld.d $a0, $a0, 24 -; LA64-NEXT: ret +; LA32NOPIC-LABEL: ld_d: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: ld.w $a1, $a0, 28 +; LA32NOPIC-NEXT: ld.w $a2, $a0, 24 +; LA32NOPIC-NEXT: ld.w $a3, $a0, 4 +; LA32NOPIC-NEXT: ld.w $a0, $a0, 0 +; LA32NOPIC-NEXT: move $a0, $a2 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: ld_d: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: ld.w $a1, $a0, 28 +; LA32PIC-NEXT: ld.w $a2, $a0, 24 +; LA32PIC-NEXT: ld.w $a3, $a0, 4 +; LA32PIC-NEXT: ld.w $a0, $a0, 0 +; LA32PIC-NEXT: move $a0, $a2 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: ld_d: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: ld.d $a1, $a0, 24 +; LA64NOPIC-NEXT: ld.d $a0, $a0, 0 +; LA64NOPIC-NEXT: move $a0, $a1 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: ld_d: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: ld.d $a1, $a0, 24 +; LA64PIC-NEXT: ld.d $a0, $a0, 0 +; LA64PIC-NEXT: move $a0, $a1 +; LA64PIC-NEXT: ret %1 = getelementptr i64, ptr %a, i64 3 %2 = load i64, ptr %1 %3 = load volatile i64, ptr %a @@ -155,20 +254,35 @@ define i64 @ld_d(ptr %a) nounwind { } define i64 @ld_bu(ptr %a) nounwind { -; LA32-LABEL: ld_bu: -; LA32: # %bb.0: -; LA32-NEXT: ld.bu $a1, $a0, 0 -; LA32-NEXT: ld.bu $a2, $a0, 4 -; LA32-NEXT: add.w $a0, $a2, $a1 -; LA32-NEXT: sltu $a1, $a0, $a2 -; LA32-NEXT: ret -; -; LA64-LABEL: ld_bu: -; LA64: # %bb.0: -; LA64-NEXT: ld.bu $a1, $a0, 0 -; LA64-NEXT: ld.bu $a0, $a0, 4 -; LA64-NEXT: add.d $a0, $a0, $a1 -; LA64-NEXT: ret +; LA32NOPIC-LABEL: ld_bu: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: ld.bu $a1, $a0, 4 +; LA32NOPIC-NEXT: ld.bu $a0, $a0, 0 +; LA32NOPIC-NEXT: add.w $a0, $a1, $a0 +; LA32NOPIC-NEXT: sltu $a1, $a0, $a1 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: ld_bu: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: ld.bu $a1, $a0, 4 +; LA32PIC-NEXT: ld.bu $a0, $a0, 0 +; LA32PIC-NEXT: add.w $a0, $a1, $a0 +; LA32PIC-NEXT: sltu $a1, $a0, $a1 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: ld_bu: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: ld.bu $a1, $a0, 4 +; LA64NOPIC-NEXT: ld.bu $a0, $a0, 0 +; LA64NOPIC-NEXT: add.d $a0, $a1, $a0 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: ld_bu: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: ld.bu $a1, $a0, 4 +; LA64PIC-NEXT: ld.bu $a0, $a0, 0 +; LA64PIC-NEXT: add.d $a0, $a1, $a0 +; LA64PIC-NEXT: ret %1 = getelementptr i8, ptr %a, i64 4 %2 = load i8, ptr %1 %3 = zext i8 %2 to i64 @@ -179,20 +293,35 @@ define i64 @ld_bu(ptr %a) nounwind { } define i64 @ld_hu(ptr %a) nounwind { -; LA32-LABEL: ld_hu: -; LA32: # %bb.0: -; LA32-NEXT: ld.hu $a1, $a0, 0 -; LA32-NEXT: ld.hu $a2, $a0, 10 -; LA32-NEXT: add.w $a0, $a2, $a1 -; LA32-NEXT: sltu $a1, $a0, $a2 -; LA32-NEXT: ret -; -; LA64-LABEL: ld_hu: -; LA64: # %bb.0: -; LA64-NEXT: ld.hu $a1, $a0, 0 -; LA64-NEXT: ld.hu $a0, $a0, 10 -; LA64-NEXT: add.d $a0, $a0, $a1 -; LA64-NEXT: ret +; LA32NOPIC-LABEL: ld_hu: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: ld.hu $a1, $a0, 10 +; LA32NOPIC-NEXT: ld.hu $a0, $a0, 0 +; LA32NOPIC-NEXT: add.w $a0, $a1, $a0 +; LA32NOPIC-NEXT: sltu $a1, $a0, $a1 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: ld_hu: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: ld.hu $a1, $a0, 10 +; LA32PIC-NEXT: ld.hu $a0, $a0, 0 +; LA32PIC-NEXT: add.w $a0, $a1, $a0 +; LA32PIC-NEXT: sltu $a1, $a0, $a1 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: ld_hu: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: ld.hu $a1, $a0, 10 +; LA64NOPIC-NEXT: ld.hu $a0, $a0, 0 +; LA64NOPIC-NEXT: add.d $a0, $a1, $a0 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: ld_hu: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: ld.hu $a1, $a0, 10 +; LA64PIC-NEXT: ld.hu $a0, $a0, 0 +; LA64PIC-NEXT: add.d $a0, $a1, $a0 +; LA64PIC-NEXT: ret %1 = getelementptr i16, ptr %a, i64 5 %2 = load i16, ptr %1 %3 = zext i16 %2 to i64 @@ -203,20 +332,35 @@ define i64 @ld_hu(ptr %a) nounwind { } define i64 @ld_wu(ptr %a) nounwind { -; LA32-LABEL: ld_wu: -; LA32: # %bb.0: -; LA32-NEXT: ld.w $a1, $a0, 0 -; LA32-NEXT: ld.w $a2, $a0, 20 -; LA32-NEXT: add.w $a0, $a2, $a1 -; LA32-NEXT: sltu $a1, $a0, $a2 -; LA32-NEXT: ret -; -; LA64-LABEL: ld_wu: -; LA64: # %bb.0: -; LA64-NEXT: ld.wu $a1, $a0, 0 -; LA64-NEXT: ld.wu $a0, $a0, 20 -; LA64-NEXT: add.d $a0, $a0, $a1 -; LA64-NEXT: ret +; LA32NOPIC-LABEL: ld_wu: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: ld.w $a1, $a0, 20 +; LA32NOPIC-NEXT: ld.w $a0, $a0, 0 +; LA32NOPIC-NEXT: add.w $a0, $a1, $a0 +; LA32NOPIC-NEXT: sltu $a1, $a0, $a1 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: ld_wu: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: ld.w $a1, $a0, 20 +; LA32PIC-NEXT: ld.w $a0, $a0, 0 +; LA32PIC-NEXT: add.w $a0, $a1, $a0 +; LA32PIC-NEXT: sltu $a1, $a0, $a1 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: ld_wu: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: ld.wu $a1, $a0, 20 +; LA64NOPIC-NEXT: ld.wu $a0, $a0, 0 +; LA64NOPIC-NEXT: add.d $a0, $a1, $a0 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: ld_wu: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: ld.wu $a1, $a0, 20 +; LA64PIC-NEXT: ld.wu $a0, $a0, 0 +; LA64PIC-NEXT: add.d $a0, $a1, $a0 +; LA64PIC-NEXT: ret %1 = getelementptr i32, ptr %a, i64 5 %2 = load i32, ptr %1 %3 = zext i32 %2 to i64 @@ -227,21 +371,37 @@ define i64 @ld_wu(ptr %a) nounwind { } define i64 @ldx_b(ptr %a, i64 %idx) nounwind { -; LA32-LABEL: ldx_b: -; LA32: # %bb.0: -; LA32-NEXT: add.w $a1, $a0, $a1 -; LA32-NEXT: ld.b $a2, $a1, 0 -; LA32-NEXT: ld.b $a0, $a0, 0 -; LA32-NEXT: srai.w $a1, $a2, 31 -; LA32-NEXT: move $a0, $a2 -; LA32-NEXT: ret -; -; LA64-LABEL: ldx_b: -; LA64: # %bb.0: -; LA64-NEXT: ldx.b $a1, $a0, $a1 -; LA64-NEXT: ld.b $a0, $a0, 0 -; LA64-NEXT: move $a0, $a1 -; LA64-NEXT: ret +; LA32NOPIC-LABEL: ldx_b: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: add.w $a1, $a0, $a1 +; LA32NOPIC-NEXT: ld.b $a2, $a1, 0 +; LA32NOPIC-NEXT: ld.b $a0, $a0, 0 +; LA32NOPIC-NEXT: srai.w $a1, $a2, 31 +; LA32NOPIC-NEXT: move $a0, $a2 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: ldx_b: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: add.w $a1, $a0, $a1 +; LA32PIC-NEXT: ld.b $a2, $a1, 0 +; LA32PIC-NEXT: ld.b $a0, $a0, 0 +; LA32PIC-NEXT: srai.w $a1, $a2, 31 +; LA32PIC-NEXT: move $a0, $a2 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: ldx_b: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: ldx.b $a1, $a0, $a1 +; LA64NOPIC-NEXT: ld.b $a0, $a0, 0 +; LA64NOPIC-NEXT: move $a0, $a1 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: ldx_b: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: ldx.b $a1, $a0, $a1 +; LA64PIC-NEXT: ld.b $a0, $a0, 0 +; LA64PIC-NEXT: move $a0, $a1 +; LA64PIC-NEXT: ret %1 = getelementptr i8, ptr %a, i64 %idx %2 = load i8, ptr %1 %3 = sext i8 %2 to i64 @@ -250,22 +410,39 @@ define i64 @ldx_b(ptr %a, i64 %idx) nounwind { } define i64 @ldx_h(ptr %a, i64 %idx) nounwind { -; LA32-LABEL: ldx_h: -; LA32: # %bb.0: -; LA32-NEXT: alsl.w $a1, $a1, $a0, 1 -; LA32-NEXT: ld.h $a2, $a1, 0 -; LA32-NEXT: ld.h $a0, $a0, 0 -; LA32-NEXT: srai.w $a1, $a2, 31 -; LA32-NEXT: move $a0, $a2 -; LA32-NEXT: ret -; -; LA64-LABEL: ldx_h: -; LA64: # %bb.0: -; LA64-NEXT: slli.d $a1, $a1, 1 -; LA64-NEXT: ldx.h $a1, $a0, $a1 -; LA64-NEXT: ld.h $a0, $a0, 0 -; LA64-NEXT: move $a0, $a1 -; LA64-NEXT: ret +; LA32NOPIC-LABEL: ldx_h: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: alsl.w $a1, $a1, $a0, 1 +; LA32NOPIC-NEXT: ld.h $a2, $a1, 0 +; LA32NOPIC-NEXT: ld.h $a0, $a0, 0 +; LA32NOPIC-NEXT: srai.w $a1, $a2, 31 +; LA32NOPIC-NEXT: move $a0, $a2 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: ldx_h: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: alsl.w $a1, $a1, $a0, 1 +; LA32PIC-NEXT: ld.h $a2, $a1, 0 +; LA32PIC-NEXT: ld.h $a0, $a0, 0 +; LA32PIC-NEXT: srai.w $a1, $a2, 31 +; LA32PIC-NEXT: move $a0, $a2 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: ldx_h: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: slli.d $a1, $a1, 1 +; LA64NOPIC-NEXT: ldx.h $a1, $a0, $a1 +; LA64NOPIC-NEXT: ld.h $a0, $a0, 0 +; LA64NOPIC-NEXT: move $a0, $a1 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: ldx_h: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: slli.d $a1, $a1, 1 +; LA64PIC-NEXT: ldx.h $a1, $a0, $a1 +; LA64PIC-NEXT: ld.h $a0, $a0, 0 +; LA64PIC-NEXT: move $a0, $a1 +; LA64PIC-NEXT: ret %1 = getelementptr i16, ptr %a, i64 %idx %2 = load i16, ptr %1 %3 = sext i16 %2 to i64 @@ -274,22 +451,39 @@ define i64 @ldx_h(ptr %a, i64 %idx) nounwind { } define i64 @ldx_w(ptr %a, i64 %idx) nounwind { -; LA32-LABEL: ldx_w: -; LA32: # %bb.0: -; LA32-NEXT: alsl.w $a1, $a1, $a0, 2 -; LA32-NEXT: ld.w $a2, $a1, 0 -; LA32-NEXT: ld.w $a0, $a0, 0 -; LA32-NEXT: srai.w $a1, $a2, 31 -; LA32-NEXT: move $a0, $a2 -; LA32-NEXT: ret -; -; LA64-LABEL: ldx_w: -; LA64: # %bb.0: -; LA64-NEXT: slli.d $a1, $a1, 2 -; LA64-NEXT: ldx.w $a1, $a0, $a1 -; LA64-NEXT: ld.w $a0, $a0, 0 -; LA64-NEXT: move $a0, $a1 -; LA64-NEXT: ret +; LA32NOPIC-LABEL: ldx_w: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: alsl.w $a1, $a1, $a0, 2 +; LA32NOPIC-NEXT: ld.w $a2, $a1, 0 +; LA32NOPIC-NEXT: ld.w $a0, $a0, 0 +; LA32NOPIC-NEXT: srai.w $a1, $a2, 31 +; LA32NOPIC-NEXT: move $a0, $a2 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: ldx_w: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: alsl.w $a1, $a1, $a0, 2 +; LA32PIC-NEXT: ld.w $a2, $a1, 0 +; LA32PIC-NEXT: ld.w $a0, $a0, 0 +; LA32PIC-NEXT: srai.w $a1, $a2, 31 +; LA32PIC-NEXT: move $a0, $a2 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: ldx_w: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: slli.d $a1, $a1, 2 +; LA64NOPIC-NEXT: ldx.w $a1, $a0, $a1 +; LA64NOPIC-NEXT: ld.w $a0, $a0, 0 +; LA64NOPIC-NEXT: move $a0, $a1 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: ldx_w: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: slli.d $a1, $a1, 2 +; LA64PIC-NEXT: ldx.w $a1, $a0, $a1 +; LA64PIC-NEXT: ld.w $a0, $a0, 0 +; LA64PIC-NEXT: move $a0, $a1 +; LA64PIC-NEXT: ret %1 = getelementptr i32, ptr %a, i64 %idx %2 = load i32, ptr %1 %3 = sext i32 %2 to i64 @@ -298,23 +492,41 @@ define i64 @ldx_w(ptr %a, i64 %idx) nounwind { } define i64 @ldx_d(ptr %a, i64 %idx) nounwind { -; LA32-LABEL: ldx_d: -; LA32: # %bb.0: -; LA32-NEXT: alsl.w $a1, $a1, $a0, 3 -; LA32-NEXT: ld.w $a2, $a1, 0 -; LA32-NEXT: ld.w $a3, $a0, 0 -; LA32-NEXT: ld.w $a1, $a1, 4 -; LA32-NEXT: ld.w $a0, $a0, 4 -; LA32-NEXT: move $a0, $a2 -; LA32-NEXT: ret -; -; LA64-LABEL: ldx_d: -; LA64: # %bb.0: -; LA64-NEXT: slli.d $a1, $a1, 3 -; LA64-NEXT: ldx.d $a1, $a0, $a1 -; LA64-NEXT: ld.d $a0, $a0, 0 -; LA64-NEXT: move $a0, $a1 -; LA64-NEXT: ret +; LA32NOPIC-LABEL: ldx_d: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: alsl.w $a1, $a1, $a0, 3 +; LA32NOPIC-NEXT: ld.w $a2, $a1, 0 +; LA32NOPIC-NEXT: ld.w $a1, $a1, 4 +; LA32NOPIC-NEXT: ld.w $a3, $a0, 0 +; LA32NOPIC-NEXT: ld.w $a0, $a0, 4 +; LA32NOPIC-NEXT: move $a0, $a2 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: ldx_d: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: alsl.w $a1, $a1, $a0, 3 +; LA32PIC-NEXT: ld.w $a2, $a1, 0 +; LA32PIC-NEXT: ld.w $a1, $a1, 4 +; LA32PIC-NEXT: ld.w $a3, $a0, 0 +; LA32PIC-NEXT: ld.w $a0, $a0, 4 +; LA32PIC-NEXT: move $a0, $a2 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: ldx_d: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: slli.d $a1, $a1, 3 +; LA64NOPIC-NEXT: ldx.d $a1, $a0, $a1 +; LA64NOPIC-NEXT: ld.d $a0, $a0, 0 +; LA64NOPIC-NEXT: move $a0, $a1 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: ldx_d: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: slli.d $a1, $a1, 3 +; LA64PIC-NEXT: ldx.d $a1, $a0, $a1 +; LA64PIC-NEXT: ld.d $a0, $a0, 0 +; LA64PIC-NEXT: move $a0, $a1 +; LA64PIC-NEXT: ret %1 = getelementptr i64, ptr %a, i64 %idx %2 = load i64, ptr %1 %3 = load volatile i64, ptr %a @@ -322,21 +534,37 @@ define i64 @ldx_d(ptr %a, i64 %idx) nounwind { } define i64 @ldx_bu(ptr %a, i64 %idx) nounwind { -; LA32-LABEL: ldx_bu: -; LA32: # %bb.0: -; LA32-NEXT: add.w $a1, $a0, $a1 -; LA32-NEXT: ld.bu $a1, $a1, 0 -; LA32-NEXT: ld.bu $a0, $a0, 0 -; LA32-NEXT: add.w $a0, $a1, $a0 -; LA32-NEXT: sltu $a1, $a0, $a1 -; LA32-NEXT: ret -; -; LA64-LABEL: ldx_bu: -; LA64: # %bb.0: -; LA64-NEXT: ldx.bu $a1, $a0, $a1 -; LA64-NEXT: ld.bu $a0, $a0, 0 -; LA64-NEXT: add.d $a0, $a1, $a0 -; LA64-NEXT: ret +; LA32NOPIC-LABEL: ldx_bu: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: add.w $a1, $a0, $a1 +; LA32NOPIC-NEXT: ld.bu $a1, $a1, 0 +; LA32NOPIC-NEXT: ld.bu $a0, $a0, 0 +; LA32NOPIC-NEXT: add.w $a0, $a1, $a0 +; LA32NOPIC-NEXT: sltu $a1, $a0, $a1 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: ldx_bu: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: add.w $a1, $a0, $a1 +; LA32PIC-NEXT: ld.bu $a1, $a1, 0 +; LA32PIC-NEXT: ld.bu $a0, $a0, 0 +; LA32PIC-NEXT: add.w $a0, $a1, $a0 +; LA32PIC-NEXT: sltu $a1, $a0, $a1 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: ldx_bu: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: ldx.bu $a1, $a0, $a1 +; LA64NOPIC-NEXT: ld.bu $a0, $a0, 0 +; LA64NOPIC-NEXT: add.d $a0, $a1, $a0 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: ldx_bu: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: ldx.bu $a1, $a0, $a1 +; LA64PIC-NEXT: ld.bu $a0, $a0, 0 +; LA64PIC-NEXT: add.d $a0, $a1, $a0 +; LA64PIC-NEXT: ret %1 = getelementptr i8, ptr %a, i64 %idx %2 = load i8, ptr %1 %3 = zext i8 %2 to i64 @@ -347,22 +575,39 @@ define i64 @ldx_bu(ptr %a, i64 %idx) nounwind { } define i64 @ldx_hu(ptr %a, i64 %idx) nounwind { -; LA32-LABEL: ldx_hu: -; LA32: # %bb.0: -; LA32-NEXT: alsl.w $a1, $a1, $a0, 1 -; LA32-NEXT: ld.hu $a1, $a1, 0 -; LA32-NEXT: ld.hu $a0, $a0, 0 -; LA32-NEXT: add.w $a0, $a1, $a0 -; LA32-NEXT: sltu $a1, $a0, $a1 -; LA32-NEXT: ret -; -; LA64-LABEL: ldx_hu: -; LA64: # %bb.0: -; LA64-NEXT: slli.d $a1, $a1, 1 -; LA64-NEXT: ldx.hu $a1, $a0, $a1 -; LA64-NEXT: ld.hu $a0, $a0, 0 -; LA64-NEXT: add.d $a0, $a1, $a0 -; LA64-NEXT: ret +; LA32NOPIC-LABEL: ldx_hu: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: alsl.w $a1, $a1, $a0, 1 +; LA32NOPIC-NEXT: ld.hu $a1, $a1, 0 +; LA32NOPIC-NEXT: ld.hu $a0, $a0, 0 +; LA32NOPIC-NEXT: add.w $a0, $a1, $a0 +; LA32NOPIC-NEXT: sltu $a1, $a0, $a1 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: ldx_hu: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: alsl.w $a1, $a1, $a0, 1 +; LA32PIC-NEXT: ld.hu $a1, $a1, 0 +; LA32PIC-NEXT: ld.hu $a0, $a0, 0 +; LA32PIC-NEXT: add.w $a0, $a1, $a0 +; LA32PIC-NEXT: sltu $a1, $a0, $a1 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: ldx_hu: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: slli.d $a1, $a1, 1 +; LA64NOPIC-NEXT: ldx.hu $a1, $a0, $a1 +; LA64NOPIC-NEXT: ld.hu $a0, $a0, 0 +; LA64NOPIC-NEXT: add.d $a0, $a1, $a0 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: ldx_hu: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: slli.d $a1, $a1, 1 +; LA64PIC-NEXT: ldx.hu $a1, $a0, $a1 +; LA64PIC-NEXT: ld.hu $a0, $a0, 0 +; LA64PIC-NEXT: add.d $a0, $a1, $a0 +; LA64PIC-NEXT: ret %1 = getelementptr i16, ptr %a, i64 %idx %2 = load i16, ptr %1 %3 = zext i16 %2 to i64 @@ -373,22 +618,39 @@ define i64 @ldx_hu(ptr %a, i64 %idx) nounwind { } define i64 @ldx_wu(ptr %a, i64 %idx) nounwind { -; LA32-LABEL: ldx_wu: -; LA32: # %bb.0: -; LA32-NEXT: alsl.w $a1, $a1, $a0, 2 -; LA32-NEXT: ld.w $a1, $a1, 0 -; LA32-NEXT: ld.w $a0, $a0, 0 -; LA32-NEXT: add.w $a0, $a1, $a0 -; LA32-NEXT: sltu $a1, $a0, $a1 -; LA32-NEXT: ret -; -; LA64-LABEL: ldx_wu: -; LA64: # %bb.0: -; LA64-NEXT: slli.d $a1, $a1, 2 -; LA64-NEXT: ldx.wu $a1, $a0, $a1 -; LA64-NEXT: ld.wu $a0, $a0, 0 -; LA64-NEXT: add.d $a0, $a1, $a0 -; LA64-NEXT: ret +; LA32NOPIC-LABEL: ldx_wu: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: alsl.w $a1, $a1, $a0, 2 +; LA32NOPIC-NEXT: ld.w $a1, $a1, 0 +; LA32NOPIC-NEXT: ld.w $a0, $a0, 0 +; LA32NOPIC-NEXT: add.w $a0, $a1, $a0 +; LA32NOPIC-NEXT: sltu $a1, $a0, $a1 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: ldx_wu: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: alsl.w $a1, $a1, $a0, 2 +; LA32PIC-NEXT: ld.w $a1, $a1, 0 +; LA32PIC-NEXT: ld.w $a0, $a0, 0 +; LA32PIC-NEXT: add.w $a0, $a1, $a0 +; LA32PIC-NEXT: sltu $a1, $a0, $a1 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: ldx_wu: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: slli.d $a1, $a1, 2 +; LA64NOPIC-NEXT: ldx.wu $a1, $a0, $a1 +; LA64NOPIC-NEXT: ld.wu $a0, $a0, 0 +; LA64NOPIC-NEXT: add.d $a0, $a1, $a0 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: ldx_wu: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: slli.d $a1, $a1, 2 +; LA64PIC-NEXT: ldx.wu $a1, $a0, $a1 +; LA64PIC-NEXT: ld.wu $a0, $a0, 0 +; LA64PIC-NEXT: add.d $a0, $a1, $a0 +; LA64PIC-NEXT: ret %1 = getelementptr i32, ptr %a, i64 %idx %2 = load i32, ptr %1 %3 = zext i32 %2 to i64 @@ -403,8 +665,8 @@ define i64 @ldx_wu(ptr %a, i64 %idx) nounwind { define void @st_b(ptr %a, i8 %b) nounwind { ; ALL-LABEL: st_b: ; ALL: # %bb.0: -; ALL-NEXT: st.b $a1, $a0, 6 ; ALL-NEXT: st.b $a1, $a0, 0 +; ALL-NEXT: st.b $a1, $a0, 6 ; ALL-NEXT: ret store i8 %b, ptr %a %1 = getelementptr i8, ptr %a, i64 6 @@ -415,8 +677,8 @@ define void @st_b(ptr %a, i8 %b) nounwind { define void @st_h(ptr %a, i16 %b) nounwind { ; ALL-LABEL: st_h: ; ALL: # %bb.0: -; ALL-NEXT: st.h $a1, $a0, 14 ; ALL-NEXT: st.h $a1, $a0, 0 +; ALL-NEXT: st.h $a1, $a0, 14 ; ALL-NEXT: ret store i16 %b, ptr %a %1 = getelementptr i16, ptr %a, i64 7 @@ -427,8 +689,8 @@ define void @st_h(ptr %a, i16 %b) nounwind { define void @st_w(ptr %a, i32 %b) nounwind { ; ALL-LABEL: st_w: ; ALL: # %bb.0: -; ALL-NEXT: st.w $a1, $a0, 28 ; ALL-NEXT: st.w $a1, $a0, 0 +; ALL-NEXT: st.w $a1, $a0, 28 ; ALL-NEXT: ret store i32 %b, ptr %a %1 = getelementptr i32, ptr %a, i64 7 @@ -437,19 +699,33 @@ define void @st_w(ptr %a, i32 %b) nounwind { } define void @st_d(ptr %a, i64 %b) nounwind { -; LA32-LABEL: st_d: -; LA32: # %bb.0: -; LA32-NEXT: st.w $a2, $a0, 68 -; LA32-NEXT: st.w $a2, $a0, 4 -; LA32-NEXT: st.w $a1, $a0, 64 -; LA32-NEXT: st.w $a1, $a0, 0 -; LA32-NEXT: ret -; -; LA64-LABEL: st_d: -; LA64: # %bb.0: -; LA64-NEXT: st.d $a1, $a0, 64 -; LA64-NEXT: st.d $a1, $a0, 0 -; LA64-NEXT: ret +; LA32NOPIC-LABEL: st_d: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: st.w $a2, $a0, 4 +; LA32NOPIC-NEXT: st.w $a1, $a0, 0 +; LA32NOPIC-NEXT: st.w $a2, $a0, 68 +; LA32NOPIC-NEXT: st.w $a1, $a0, 64 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: st_d: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: st.w $a2, $a0, 4 +; LA32PIC-NEXT: st.w $a1, $a0, 0 +; LA32PIC-NEXT: st.w $a2, $a0, 68 +; LA32PIC-NEXT: st.w $a1, $a0, 64 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: st_d: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: st.d $a1, $a0, 0 +; LA64NOPIC-NEXT: st.d $a1, $a0, 64 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: st_d: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: st.d $a1, $a0, 0 +; LA64PIC-NEXT: st.d $a1, $a0, 64 +; LA64PIC-NEXT: ret store i64 %b, ptr %a %1 = getelementptr i64, ptr %a, i64 8 store i64 %b, ptr %1 @@ -457,68 +733,116 @@ define void @st_d(ptr %a, i64 %b) nounwind { } define void @stx_b(ptr %dst, i64 %idx, i8 %val) nounwind { -; LA32-LABEL: stx_b: -; LA32: # %bb.0: -; LA32-NEXT: add.w $a0, $a0, $a1 -; LA32-NEXT: st.b $a3, $a0, 0 -; LA32-NEXT: ret -; -; LA64-LABEL: stx_b: -; LA64: # %bb.0: -; LA64-NEXT: stx.b $a2, $a0, $a1 -; LA64-NEXT: ret +; LA32NOPIC-LABEL: stx_b: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: add.w $a0, $a0, $a1 +; LA32NOPIC-NEXT: st.b $a3, $a0, 0 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: stx_b: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: add.w $a0, $a0, $a1 +; LA32PIC-NEXT: st.b $a3, $a0, 0 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: stx_b: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: stx.b $a2, $a0, $a1 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: stx_b: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: stx.b $a2, $a0, $a1 +; LA64PIC-NEXT: ret %1 = getelementptr i8, ptr %dst, i64 %idx store i8 %val, ptr %1 ret void } define void @stx_h(ptr %dst, i64 %idx, i16 %val) nounwind { -; LA32-LABEL: stx_h: -; LA32: # %bb.0: -; LA32-NEXT: alsl.w $a0, $a1, $a0, 1 -; LA32-NEXT: st.h $a3, $a0, 0 -; LA32-NEXT: ret -; -; LA64-LABEL: stx_h: -; LA64: # %bb.0: -; LA64-NEXT: slli.d $a1, $a1, 1 -; LA64-NEXT: stx.h $a2, $a0, $a1 -; LA64-NEXT: ret +; LA32NOPIC-LABEL: stx_h: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: alsl.w $a0, $a1, $a0, 1 +; LA32NOPIC-NEXT: st.h $a3, $a0, 0 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: stx_h: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: alsl.w $a0, $a1, $a0, 1 +; LA32PIC-NEXT: st.h $a3, $a0, 0 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: stx_h: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: slli.d $a1, $a1, 1 +; LA64NOPIC-NEXT: stx.h $a2, $a0, $a1 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: stx_h: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: slli.d $a1, $a1, 1 +; LA64PIC-NEXT: stx.h $a2, $a0, $a1 +; LA64PIC-NEXT: ret %1 = getelementptr i16, ptr %dst, i64 %idx store i16 %val, ptr %1 ret void } define void @stx_w(ptr %dst, i64 %idx, i32 %val) nounwind { -; LA32-LABEL: stx_w: -; LA32: # %bb.0: -; LA32-NEXT: alsl.w $a0, $a1, $a0, 2 -; LA32-NEXT: st.w $a3, $a0, 0 -; LA32-NEXT: ret -; -; LA64-LABEL: stx_w: -; LA64: # %bb.0: -; LA64-NEXT: slli.d $a1, $a1, 2 -; LA64-NEXT: stx.w $a2, $a0, $a1 -; LA64-NEXT: ret +; LA32NOPIC-LABEL: stx_w: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: alsl.w $a0, $a1, $a0, 2 +; LA32NOPIC-NEXT: st.w $a3, $a0, 0 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: stx_w: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: alsl.w $a0, $a1, $a0, 2 +; LA32PIC-NEXT: st.w $a3, $a0, 0 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: stx_w: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: slli.d $a1, $a1, 2 +; LA64NOPIC-NEXT: stx.w $a2, $a0, $a1 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: stx_w: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: slli.d $a1, $a1, 2 +; LA64PIC-NEXT: stx.w $a2, $a0, $a1 +; LA64PIC-NEXT: ret %1 = getelementptr i32, ptr %dst, i64 %idx store i32 %val, ptr %1 ret void } define void @stx_d(ptr %dst, i64 %idx, i64 %val) nounwind { -; LA32-LABEL: stx_d: -; LA32: # %bb.0: -; LA32-NEXT: alsl.w $a0, $a1, $a0, 3 -; LA32-NEXT: st.w $a4, $a0, 4 -; LA32-NEXT: st.w $a3, $a0, 0 -; LA32-NEXT: ret -; -; LA64-LABEL: stx_d: -; LA64: # %bb.0: -; LA64-NEXT: slli.d $a1, $a1, 3 -; LA64-NEXT: stx.d $a2, $a0, $a1 -; LA64-NEXT: ret +; LA32NOPIC-LABEL: stx_d: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: alsl.w $a0, $a1, $a0, 3 +; LA32NOPIC-NEXT: st.w $a4, $a0, 4 +; LA32NOPIC-NEXT: st.w $a3, $a0, 0 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: stx_d: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: alsl.w $a0, $a1, $a0, 3 +; LA32PIC-NEXT: st.w $a4, $a0, 4 +; LA32PIC-NEXT: st.w $a3, $a0, 0 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: stx_d: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: slli.d $a1, $a1, 3 +; LA64NOPIC-NEXT: stx.d $a2, $a0, $a1 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: stx_d: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: slli.d $a1, $a1, 3 +; LA64PIC-NEXT: stx.d $a2, $a0, $a1 +; LA64PIC-NEXT: ret %1 = getelementptr i64, ptr %dst, i64 %idx store i64 %val, ptr %1 ret void @@ -526,24 +850,44 @@ define void @stx_d(ptr %dst, i64 %idx, i64 %val) nounwind { ;; Check load from and store to an i1 location. define i64 @load_sext_zext_anyext_i1(ptr %a) nounwind { +; LA32NOPIC-LABEL: load_sext_zext_anyext_i1: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: ld.bu $a1, $a0, 1 +; LA32NOPIC-NEXT: ld.bu $a3, $a0, 2 +; LA32NOPIC-NEXT: sub.w $a2, $a3, $a1 +; LA32NOPIC-NEXT: ld.b $a0, $a0, 0 +; LA32NOPIC-NEXT: sltu $a0, $a3, $a1 +; LA32NOPIC-NEXT: sub.w $a1, $zero, $a0 +; LA32NOPIC-NEXT: move $a0, $a2 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: load_sext_zext_anyext_i1: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: ld.bu $a1, $a0, 1 +; LA32PIC-NEXT: ld.bu $a3, $a0, 2 +; LA32PIC-NEXT: sub.w $a2, $a3, $a1 +; LA32PIC-NEXT: ld.b $a0, $a0, 0 +; LA32PIC-NEXT: sltu $a0, $a3, $a1 +; LA32PIC-NEXT: sub.w $a1, $zero, $a0 +; LA32PIC-NEXT: move $a0, $a2 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: load_sext_zext_anyext_i1: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: ld.bu $a1, $a0, 1 +; LA64NOPIC-NEXT: ld.bu $a2, $a0, 2 +; LA64NOPIC-NEXT: ld.b $a0, $a0, 0 +; LA64NOPIC-NEXT: sub.d $a0, $a2, $a1 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: load_sext_zext_anyext_i1: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: ld.bu $a1, $a0, 1 +; LA64PIC-NEXT: ld.bu $a2, $a0, 2 +; LA64PIC-NEXT: ld.b $a0, $a0, 0 +; LA64PIC-NEXT: sub.d $a0, $a2, $a1 +; LA64PIC-NEXT: ret ;; sextload i1 -; LA32-LABEL: load_sext_zext_anyext_i1: -; LA32: # %bb.0: -; LA32-NEXT: ld.b $a1, $a0, 0 -; LA32-NEXT: ld.bu $a1, $a0, 1 -; LA32-NEXT: ld.bu $a2, $a0, 2 -; LA32-NEXT: sub.w $a0, $a2, $a1 -; LA32-NEXT: sltu $a1, $a2, $a1 -; LA32-NEXT: sub.w $a1, $zero, $a1 -; LA32-NEXT: ret -; -; LA64-LABEL: load_sext_zext_anyext_i1: -; LA64: # %bb.0: -; LA64-NEXT: ld.b $a1, $a0, 0 -; LA64-NEXT: ld.bu $a1, $a0, 1 -; LA64-NEXT: ld.bu $a0, $a0, 2 -; LA64-NEXT: sub.d $a0, $a0, $a1 -; LA64-NEXT: ret %1 = getelementptr i1, ptr %a, i64 1 %2 = load i1, ptr %1 %3 = sext i1 %2 to i64 @@ -558,22 +902,38 @@ define i64 @load_sext_zext_anyext_i1(ptr %a) nounwind { } define i16 @load_sext_zext_anyext_i1_i16(ptr %a) nounwind { +; LA32NOPIC-LABEL: load_sext_zext_anyext_i1_i16: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: ld.bu $a1, $a0, 1 +; LA32NOPIC-NEXT: ld.bu $a2, $a0, 2 +; LA32NOPIC-NEXT: ld.b $a0, $a0, 0 +; LA32NOPIC-NEXT: sub.w $a0, $a2, $a1 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: load_sext_zext_anyext_i1_i16: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: ld.bu $a1, $a0, 1 +; LA32PIC-NEXT: ld.bu $a2, $a0, 2 +; LA32PIC-NEXT: ld.b $a0, $a0, 0 +; LA32PIC-NEXT: sub.w $a0, $a2, $a1 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: load_sext_zext_anyext_i1_i16: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: ld.bu $a1, $a0, 1 +; LA64NOPIC-NEXT: ld.bu $a2, $a0, 2 +; LA64NOPIC-NEXT: ld.b $a0, $a0, 0 +; LA64NOPIC-NEXT: sub.d $a0, $a2, $a1 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: load_sext_zext_anyext_i1_i16: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: ld.bu $a1, $a0, 1 +; LA64PIC-NEXT: ld.bu $a2, $a0, 2 +; LA64PIC-NEXT: ld.b $a0, $a0, 0 +; LA64PIC-NEXT: sub.d $a0, $a2, $a1 +; LA64PIC-NEXT: ret ;; sextload i1 -; LA32-LABEL: load_sext_zext_anyext_i1_i16: -; LA32: # %bb.0: -; LA32-NEXT: ld.b $a1, $a0, 0 -; LA32-NEXT: ld.bu $a1, $a0, 1 -; LA32-NEXT: ld.bu $a0, $a0, 2 -; LA32-NEXT: sub.w $a0, $a0, $a1 -; LA32-NEXT: ret -; -; LA64-LABEL: load_sext_zext_anyext_i1_i16: -; LA64: # %bb.0: -; LA64-NEXT: ld.b $a1, $a0, 0 -; LA64-NEXT: ld.bu $a1, $a0, 1 -; LA64-NEXT: ld.bu $a0, $a0, 2 -; LA64-NEXT: sub.d $a0, $a0, $a1 -; LA64-NEXT: ret %1 = getelementptr i1, ptr %a, i64 1 %2 = load i1, ptr %1 %3 = sext i1 %2 to i16 @@ -588,29 +948,53 @@ define i16 @load_sext_zext_anyext_i1_i16(ptr %a) nounwind { } define i64 @ld_sd_constant(i64 %a) nounwind { -; LA32-LABEL: ld_sd_constant: -; LA32: # %bb.0: -; LA32-NEXT: lu12i.w $a3, -136485 -; LA32-NEXT: ori $a4, $a3, 3823 -; LA32-NEXT: ld.w $a2, $a4, 0 -; LA32-NEXT: st.w $a0, $a4, 0 -; LA32-NEXT: ori $a0, $a3, 3827 -; LA32-NEXT: ld.w $a3, $a0, 0 -; LA32-NEXT: st.w $a1, $a0, 0 -; LA32-NEXT: move $a0, $a2 -; LA32-NEXT: move $a1, $a3 -; LA32-NEXT: ret -; -; LA64-LABEL: ld_sd_constant: -; LA64: # %bb.0: -; LA64-NEXT: lu12i.w $a1, -136485 -; LA64-NEXT: ori $a1, $a1, 3823 -; LA64-NEXT: lu32i.d $a1, -147729 -; LA64-NEXT: lu52i.d $a2, $a1, -534 -; LA64-NEXT: ld.d $a1, $a2, 0 -; LA64-NEXT: st.d $a0, $a2, 0 -; LA64-NEXT: move $a0, $a1 -; LA64-NEXT: ret +; LA32NOPIC-LABEL: ld_sd_constant: +; LA32NOPIC: # %bb.0: +; LA32NOPIC-NEXT: lu12i.w $a3, -136485 +; LA32NOPIC-NEXT: ori $a4, $a3, 3823 +; LA32NOPIC-NEXT: ld.w $a2, $a4, 0 +; LA32NOPIC-NEXT: ori $a5, $a3, 3827 +; LA32NOPIC-NEXT: ld.w $a3, $a5, 0 +; LA32NOPIC-NEXT: st.w $a0, $a4, 0 +; LA32NOPIC-NEXT: st.w $a1, $a5, 0 +; LA32NOPIC-NEXT: move $a0, $a2 +; LA32NOPIC-NEXT: move $a1, $a3 +; LA32NOPIC-NEXT: ret +; +; LA32PIC-LABEL: ld_sd_constant: +; LA32PIC: # %bb.0: +; LA32PIC-NEXT: lu12i.w $a3, -136485 +; LA32PIC-NEXT: ori $a4, $a3, 3823 +; LA32PIC-NEXT: ld.w $a2, $a4, 0 +; LA32PIC-NEXT: ori $a5, $a3, 3827 +; LA32PIC-NEXT: ld.w $a3, $a5, 0 +; LA32PIC-NEXT: st.w $a0, $a4, 0 +; LA32PIC-NEXT: st.w $a1, $a5, 0 +; LA32PIC-NEXT: move $a0, $a2 +; LA32PIC-NEXT: move $a1, $a3 +; LA32PIC-NEXT: ret +; +; LA64NOPIC-LABEL: ld_sd_constant: +; LA64NOPIC: # %bb.0: +; LA64NOPIC-NEXT: lu12i.w $a1, -136485 +; LA64NOPIC-NEXT: ori $a1, $a1, 3823 +; LA64NOPIC-NEXT: lu32i.d $a1, -147729 +; LA64NOPIC-NEXT: lu52i.d $a2, $a1, -534 +; LA64NOPIC-NEXT: ld.d $a1, $a2, 0 +; LA64NOPIC-NEXT: st.d $a0, $a2, 0 +; LA64NOPIC-NEXT: move $a0, $a1 +; LA64NOPIC-NEXT: ret +; +; LA64PIC-LABEL: ld_sd_constant: +; LA64PIC: # %bb.0: +; LA64PIC-NEXT: lu12i.w $a1, -136485 +; LA64PIC-NEXT: ori $a1, $a1, 3823 +; LA64PIC-NEXT: lu32i.d $a1, -147729 +; LA64PIC-NEXT: lu52i.d $a2, $a1, -534 +; LA64PIC-NEXT: ld.d $a1, $a2, 0 +; LA64PIC-NEXT: st.d $a0, $a2, 0 +; LA64PIC-NEXT: move $a0, $a1 +; LA64PIC-NEXT: ret %1 = inttoptr i64 16045690984833335023 to ptr %2 = load volatile i64, ptr %1 store i64 %a, ptr %1 diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/lshr.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/lshr.ll index 3916298e298f..7b28872780e8 100644 --- a/llvm/test/CodeGen/LoongArch/ir-instruction/lshr.ll +++ b/llvm/test/CodeGen/LoongArch/ir-instruction/lshr.ll @@ -65,10 +65,10 @@ define i32 @lshr_i32(i32 %x, i32 %y) { define i64 @lshr_i64(i64 %x, i64 %y) { ; LA32-LABEL: lshr_i64: ; LA32: # %bb.0: +; LA32-NEXT: srl.w $a0, $a0, $a2 ; LA32-NEXT: xori $a3, $a2, 31 ; LA32-NEXT: slli.w $a4, $a1, 1 ; LA32-NEXT: sll.w $a3, $a4, $a3 -; LA32-NEXT: srl.w $a0, $a0, $a2 ; LA32-NEXT: or $a0, $a0, $a3 ; LA32-NEXT: addi.w $a3, $a2, -32 ; LA32-NEXT: slti $a4, $a3, 0 @@ -146,8 +146,8 @@ define i32 @lshr_i32_3(i32 %x) { define i64 @lshr_i64_3(i64 %x) { ; LA32-LABEL: lshr_i64_3: ; LA32: # %bb.0: -; LA32-NEXT: srli.w $a0, $a0, 3 ; LA32-NEXT: slli.w $a2, $a1, 29 +; LA32-NEXT: srli.w $a0, $a0, 3 ; LA32-NEXT: or $a0, $a0, $a2 ; LA32-NEXT: srli.w $a1, $a1, 3 ; LA32-NEXT: ret diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/mul.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/mul.ll index 53a3529219fe..cfa6ceae78f9 100644 --- a/llvm/test/CodeGen/LoongArch/ir-instruction/mul.ll +++ b/llvm/test/CodeGen/LoongArch/ir-instruction/mul.ll @@ -87,8 +87,8 @@ entry: define i64 @mul_pow2(i64 %a) { ; LA32-LABEL: mul_pow2: ; LA32: # %bb.0: -; LA32-NEXT: slli.w $a1, $a1, 3 ; LA32-NEXT: srli.w $a2, $a0, 29 +; LA32-NEXT: slli.w $a1, $a1, 3 ; LA32-NEXT: or $a1, $a1, $a2 ; LA32-NEXT: slli.w $a0, $a0, 3 ; LA32-NEXT: ret @@ -160,45 +160,46 @@ define i32 @mulh_wu(i32 %a, i32 %b) { define i64 @mulh_d(i64 %a, i64 %b) { ; LA32-LABEL: mulh_d: ; LA32: # %bb.0: +; LA32-NEXT: srai.w $a5, $a1, 31 +; LA32-NEXT: srai.w $a6, $a3, 31 ; LA32-NEXT: mulh.wu $a4, $a0, $a2 -; LA32-NEXT: mul.w $a5, $a1, $a2 -; LA32-NEXT: add.w $a4, $a5, $a4 -; LA32-NEXT: sltu $a5, $a4, $a5 -; LA32-NEXT: mulh.wu $a6, $a1, $a2 -; LA32-NEXT: add.w $a5, $a6, $a5 -; LA32-NEXT: mul.w $a6, $a0, $a3 -; LA32-NEXT: add.w $a4, $a6, $a4 -; LA32-NEXT: sltu $a4, $a4, $a6 -; LA32-NEXT: mulh.wu $a6, $a0, $a3 -; LA32-NEXT: add.w $a4, $a6, $a4 -; LA32-NEXT: add.w $a4, $a5, $a4 -; LA32-NEXT: sltu $a5, $a4, $a5 -; LA32-NEXT: mulh.wu $a6, $a1, $a3 -; LA32-NEXT: add.w $a5, $a6, $a5 -; LA32-NEXT: mul.w $a6, $a1, $a3 -; LA32-NEXT: add.w $a4, $a6, $a4 -; LA32-NEXT: sltu $a6, $a4, $a6 -; LA32-NEXT: add.w $a5, $a5, $a6 -; LA32-NEXT: srai.w $a6, $a1, 31 -; LA32-NEXT: mul.w $a7, $a2, $a6 -; LA32-NEXT: mulh.wu $a2, $a2, $a6 -; LA32-NEXT: add.w $a2, $a2, $a7 -; LA32-NEXT: mul.w $a6, $a3, $a6 -; LA32-NEXT: add.w $a2, $a2, $a6 -; LA32-NEXT: srai.w $a3, $a3, 31 -; LA32-NEXT: mul.w $a1, $a3, $a1 -; LA32-NEXT: mulh.wu $a6, $a3, $a0 -; LA32-NEXT: add.w $a1, $a6, $a1 -; LA32-NEXT: mul.w $a0, $a3, $a0 -; LA32-NEXT: add.w $a1, $a1, $a0 -; LA32-NEXT: add.w $a1, $a1, $a2 -; LA32-NEXT: add.w $a2, $a0, $a7 -; LA32-NEXT: sltu $a0, $a2, $a0 -; LA32-NEXT: add.w $a0, $a1, $a0 -; LA32-NEXT: add.w $a1, $a5, $a0 -; LA32-NEXT: add.w $a0, $a4, $a2 -; LA32-NEXT: sltu $a2, $a0, $a4 -; LA32-NEXT: add.w $a1, $a1, $a2 +; LA32-NEXT: mul.w $a7, $a1, $a2 +; LA32-NEXT: add.w $a4, $a7, $a4 +; LA32-NEXT: sltu $a7, $a4, $a7 +; LA32-NEXT: mulh.wu $t0, $a1, $a2 +; LA32-NEXT: add.w $a7, $t0, $a7 +; LA32-NEXT: mul.w $t0, $a0, $a3 +; LA32-NEXT: add.w $a4, $t0, $a4 +; LA32-NEXT: sltu $a4, $a4, $t0 +; LA32-NEXT: mulh.wu $t0, $a0, $a3 +; LA32-NEXT: add.w $a4, $t0, $a4 +; LA32-NEXT: add.w $t0, $a7, $a4 +; LA32-NEXT: mul.w $t1, $a1, $a3 +; LA32-NEXT: add.w $t2, $t1, $t0 +; LA32-NEXT: mul.w $t3, $a2, $a5 +; LA32-NEXT: mul.w $t4, $a6, $a0 +; LA32-NEXT: add.w $t5, $t4, $t3 +; LA32-NEXT: add.w $a4, $t2, $t5 +; LA32-NEXT: sltu $t6, $a4, $t2 +; LA32-NEXT: sltu $t1, $t2, $t1 +; LA32-NEXT: sltu $a7, $t0, $a7 +; LA32-NEXT: mulh.wu $t0, $a1, $a3 +; LA32-NEXT: add.w $a7, $t0, $a7 +; LA32-NEXT: add.w $a7, $a7, $t1 +; LA32-NEXT: mulh.wu $a2, $a2, $a5 +; LA32-NEXT: add.w $a2, $a2, $t3 +; LA32-NEXT: mul.w $a3, $a3, $a5 +; LA32-NEXT: add.w $a2, $a2, $a3 +; LA32-NEXT: mul.w $a1, $a6, $a1 +; LA32-NEXT: mulh.wu $a0, $a6, $a0 +; LA32-NEXT: add.w $a0, $a0, $a1 +; LA32-NEXT: add.w $a0, $a0, $t4 +; LA32-NEXT: add.w $a0, $a0, $a2 +; LA32-NEXT: sltu $a1, $t5, $t4 +; LA32-NEXT: add.w $a0, $a0, $a1 +; LA32-NEXT: add.w $a0, $a7, $a0 +; LA32-NEXT: add.w $a1, $a0, $t6 +; LA32-NEXT: move $a0, $a4 ; LA32-NEXT: ret ; ; LA64-LABEL: mulh_d: @@ -227,14 +228,14 @@ define i64 @mulh_du(i64 %a, i64 %b) { ; LA32-NEXT: sltu $a4, $a4, $a5 ; LA32-NEXT: mulh.wu $a0, $a0, $a3 ; LA32-NEXT: add.w $a0, $a0, $a4 -; LA32-NEXT: mul.w $a4, $a1, $a3 +; LA32-NEXT: add.w $a4, $a2, $a0 +; LA32-NEXT: mul.w $a5, $a1, $a3 +; LA32-NEXT: add.w $a0, $a5, $a4 +; LA32-NEXT: sltu $a5, $a0, $a5 +; LA32-NEXT: sltu $a2, $a4, $a2 ; LA32-NEXT: mulh.wu $a1, $a1, $a3 -; LA32-NEXT: add.w $a0, $a2, $a0 -; LA32-NEXT: sltu $a2, $a0, $a2 -; LA32-NEXT: add.w $a1, $a1, $a2 -; LA32-NEXT: add.w $a0, $a4, $a0 -; LA32-NEXT: sltu $a2, $a0, $a4 ; LA32-NEXT: add.w $a1, $a1, $a2 +; LA32-NEXT: add.w $a1, $a1, $a5 ; LA32-NEXT: ret ; ; LA64-LABEL: mulh_du: @@ -1494,15 +1495,15 @@ define i64 @mul_i64_65280_twice(i64 %a, i64 %b) { ; LA32: # %bb.0: ; LA32-NEXT: lu12i.w $a4, 15 ; LA32-NEXT: ori $a4, $a4, 3840 -; LA32-NEXT: mul.w $a3, $a3, $a4 -; LA32-NEXT: mulh.wu $a5, $a2, $a4 -; LA32-NEXT: add.w $a3, $a5, $a3 ; LA32-NEXT: mul.w $a1, $a1, $a4 ; LA32-NEXT: mulh.wu $a5, $a0, $a4 ; LA32-NEXT: add.w $a1, $a5, $a1 -; LA32-NEXT: xor $a1, $a1, $a3 -; LA32-NEXT: mul.w $a2, $a2, $a4 ; LA32-NEXT: mul.w $a0, $a0, $a4 +; LA32-NEXT: mul.w $a3, $a3, $a4 +; LA32-NEXT: mulh.wu $a5, $a2, $a4 +; LA32-NEXT: add.w $a3, $a5, $a3 +; LA32-NEXT: mul.w $a2, $a2, $a4 +; LA32-NEXT: xor $a1, $a1, $a3 ; LA32-NEXT: xor $a0, $a0, $a2 ; LA32-NEXT: ret ; @@ -1510,8 +1511,8 @@ define i64 @mul_i64_65280_twice(i64 %a, i64 %b) { ; LA64: # %bb.0: ; LA64-NEXT: lu12i.w $a2, 15 ; LA64-NEXT: ori $a2, $a2, 3840 -; LA64-NEXT: mul.d $a1, $a1, $a2 ; LA64-NEXT: mul.d $a0, $a0, $a2 +; LA64-NEXT: mul.d $a1, $a1, $a2 ; LA64-NEXT: xor $a0, $a0, $a1 ; LA64-NEXT: ret %c = mul i64 %a, 65280 diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/shl.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/shl.ll index 4baf18931dc5..3f35b76b1603 100644 --- a/llvm/test/CodeGen/LoongArch/ir-instruction/shl.ll +++ b/llvm/test/CodeGen/LoongArch/ir-instruction/shl.ll @@ -61,10 +61,10 @@ define i32 @shl_i32(i32 %x, i32 %y) { define i64 @shl_i64(i64 %x, i64 %y) { ; LA32-LABEL: shl_i64: ; LA32: # %bb.0: +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: xori $a3, $a2, 31 ; LA32-NEXT: srli.w $a4, $a0, 1 ; LA32-NEXT: srl.w $a3, $a4, $a3 -; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: or $a1, $a1, $a3 ; LA32-NEXT: addi.w $a3, $a2, -32 ; LA32-NEXT: slti $a4, $a3, 0 @@ -142,8 +142,8 @@ define i32 @shl_i32_3(i32 %x) { define i64 @shl_i64_3(i64 %x) { ; LA32-LABEL: shl_i64_3: ; LA32: # %bb.0: -; LA32-NEXT: slli.w $a1, $a1, 3 ; LA32-NEXT: srli.w $a2, $a0, 29 +; LA32-NEXT: slli.w $a1, $a1, 3 ; LA32-NEXT: or $a1, $a1, $a2 ; LA32-NEXT: slli.w $a0, $a0, 3 ; LA32-NEXT: ret diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/sub.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/sub.ll index 98357744f521..bb236c11bb81 100644 --- a/llvm/test/CodeGen/LoongArch/ir-instruction/sub.ll +++ b/llvm/test/CodeGen/LoongArch/ir-instruction/sub.ll @@ -79,9 +79,9 @@ define signext i32 @sub_i32_sext(i32 %x, i32 %y) { define i64 @sub_i64(i64 %x, i64 %y) { ; LA32-LABEL: sub_i64: ; LA32: # %bb.0: +; LA32-NEXT: sltu $a4, $a0, $a2 ; LA32-NEXT: sub.w $a1, $a1, $a3 -; LA32-NEXT: sltu $a3, $a0, $a2 -; LA32-NEXT: sub.w $a1, $a1, $a3 +; LA32-NEXT: sub.w $a1, $a1, $a4 ; LA32-NEXT: sub.w $a0, $a0, $a2 ; LA32-NEXT: ret ; diff --git a/llvm/test/CodeGen/LoongArch/lasx/build-vector.ll b/llvm/test/CodeGen/LoongArch/lasx/build-vector.ll index ae6f31aaec64..3a0fd0758cb3 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/build-vector.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/build-vector.ll @@ -235,102 +235,102 @@ define void @buildvector_v32i8(ptr %dst, i8 %a0, i8 %a1, i8 %a2, i8 %a3, i8 %a4, ; CHECK-NEXT: vinsgr2vr.b $vr0, $a3, 2 ; CHECK-NEXT: vinsgr2vr.b $vr0, $a4, 3 ; CHECK-NEXT: vinsgr2vr.b $vr0, $a5, 4 +; CHECK-NEXT: ld.b $a1, $sp, 0 ; CHECK-NEXT: vinsgr2vr.b $vr0, $a6, 5 +; CHECK-NEXT: ld.b $a2, $sp, 8 ; CHECK-NEXT: vinsgr2vr.b $vr0, $a7, 6 -; CHECK-NEXT: ld.b $a1, $sp, 0 +; CHECK-NEXT: ld.b $a3, $sp, 16 ; CHECK-NEXT: vinsgr2vr.b $vr0, $a1, 7 -; CHECK-NEXT: ld.b $a1, $sp, 8 -; CHECK-NEXT: vinsgr2vr.b $vr0, $a1, 8 -; CHECK-NEXT: ld.b $a1, $sp, 16 -; CHECK-NEXT: vinsgr2vr.b $vr0, $a1, 9 ; CHECK-NEXT: ld.b $a1, $sp, 24 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a2, 8 +; CHECK-NEXT: ld.b $a2, $sp, 32 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a3, 9 +; CHECK-NEXT: ld.b $a3, $sp, 40 ; CHECK-NEXT: vinsgr2vr.b $vr0, $a1, 10 -; CHECK-NEXT: ld.b $a1, $sp, 32 -; CHECK-NEXT: vinsgr2vr.b $vr0, $a1, 11 -; CHECK-NEXT: ld.b $a1, $sp, 40 -; CHECK-NEXT: vinsgr2vr.b $vr0, $a1, 12 ; CHECK-NEXT: ld.b $a1, $sp, 48 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a2, 11 +; CHECK-NEXT: ld.b $a2, $sp, 56 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a3, 12 +; CHECK-NEXT: ld.b $a3, $sp, 64 ; CHECK-NEXT: vinsgr2vr.b $vr0, $a1, 13 -; CHECK-NEXT: ld.b $a1, $sp, 56 -; CHECK-NEXT: vinsgr2vr.b $vr0, $a1, 14 -; CHECK-NEXT: ld.b $a1, $sp, 64 -; CHECK-NEXT: vinsgr2vr.b $vr0, $a1, 15 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a2, 14 ; CHECK-NEXT: ld.b $a1, $sp, 72 +; CHECK-NEXT: vinsgr2vr.b $vr0, $a3, 15 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 ; CHECK-NEXT: vinsgr2vr.b $vr1, $a1, 0 -; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: ld.b $a1, $sp, 80 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 ; CHECK-NEXT: vinsgr2vr.b $vr1, $a1, 1 -; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: ld.b $a1, $sp, 88 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 ; CHECK-NEXT: vinsgr2vr.b $vr1, $a1, 2 -; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: ld.b $a1, $sp, 96 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 ; CHECK-NEXT: vinsgr2vr.b $vr1, $a1, 3 -; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: ld.b $a1, $sp, 104 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 ; CHECK-NEXT: vinsgr2vr.b $vr1, $a1, 4 -; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: ld.b $a1, $sp, 112 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 ; CHECK-NEXT: vinsgr2vr.b $vr1, $a1, 5 -; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: ld.b $a1, $sp, 120 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 ; CHECK-NEXT: vinsgr2vr.b $vr1, $a1, 6 -; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: ld.b $a1, $sp, 128 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 ; CHECK-NEXT: vinsgr2vr.b $vr1, $a1, 7 -; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: ld.b $a1, $sp, 136 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 ; CHECK-NEXT: vinsgr2vr.b $vr1, $a1, 8 -; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: ld.b $a1, $sp, 144 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 ; CHECK-NEXT: vinsgr2vr.b $vr1, $a1, 9 -; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: ld.b $a1, $sp, 152 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 ; CHECK-NEXT: vinsgr2vr.b $vr1, $a1, 10 -; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: ld.b $a1, $sp, 160 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 ; CHECK-NEXT: vinsgr2vr.b $vr1, $a1, 11 -; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: ld.b $a1, $sp, 168 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 ; CHECK-NEXT: vinsgr2vr.b $vr1, $a1, 12 -; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: ld.b $a1, $sp, 176 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 ; CHECK-NEXT: vinsgr2vr.b $vr1, $a1, 13 -; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: ld.b $a1, $sp, 184 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 ; CHECK-NEXT: vinsgr2vr.b $vr1, $a1, 14 -; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: ld.b $a1, $sp, 192 +; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 ; CHECK-NEXT: vinsgr2vr.b $vr1, $a1, 15 @@ -377,6 +377,8 @@ entry: define void @buildvector_v16i16(ptr %dst, i16 %a0, i16 %a1, i16 %a2, i16 %a3, i16 %a4, i16 %a5, i16 %a6, i16 %a7, i16 %a8, i16 %a9, i16 %a10, i16 %a11, i16 %a12, i16 %a13, i16 %a14, i16 %a15) nounwind { ; CHECK-LABEL: buildvector_v16i16: ; CHECK: # %bb.0: # %entry +; CHECK-NEXT: ld.h $t0, $sp, 8 +; CHECK-NEXT: ld.h $t1, $sp, 0 ; CHECK-NEXT: vinsgr2vr.h $vr0, $a1, 0 ; CHECK-NEXT: vinsgr2vr.h $vr0, $a2, 1 ; CHECK-NEXT: vinsgr2vr.h $vr0, $a3, 2 @@ -384,44 +386,42 @@ define void @buildvector_v16i16(ptr %dst, i16 %a0, i16 %a1, i16 %a2, i16 %a3, i1 ; CHECK-NEXT: vinsgr2vr.h $vr0, $a5, 4 ; CHECK-NEXT: vinsgr2vr.h $vr0, $a6, 5 ; CHECK-NEXT: vinsgr2vr.h $vr0, $a7, 6 -; CHECK-NEXT: ld.h $a1, $sp, 0 -; CHECK-NEXT: vinsgr2vr.h $vr0, $a1, 7 -; CHECK-NEXT: ld.h $a1, $sp, 8 +; CHECK-NEXT: vinsgr2vr.h $vr0, $t1, 7 +; CHECK-NEXT: ld.h $a1, $sp, 16 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 -; CHECK-NEXT: vinsgr2vr.h $vr1, $a1, 0 +; CHECK-NEXT: vinsgr2vr.h $vr1, $t0, 0 ; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 -; CHECK-NEXT: ld.h $a1, $sp, 16 +; CHECK-NEXT: ld.h $a2, $sp, 24 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 ; CHECK-NEXT: vinsgr2vr.h $vr1, $a1, 1 ; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 -; CHECK-NEXT: ld.h $a1, $sp, 24 +; CHECK-NEXT: ld.h $a1, $sp, 32 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 -; CHECK-NEXT: vinsgr2vr.h $vr1, $a1, 2 +; CHECK-NEXT: vinsgr2vr.h $vr1, $a2, 2 ; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 -; CHECK-NEXT: ld.h $a1, $sp, 32 +; CHECK-NEXT: ld.h $a2, $sp, 40 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 ; CHECK-NEXT: vinsgr2vr.h $vr1, $a1, 3 ; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 -; CHECK-NEXT: ld.h $a1, $sp, 40 +; CHECK-NEXT: ld.h $a1, $sp, 48 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 -; CHECK-NEXT: vinsgr2vr.h $vr1, $a1, 4 +; CHECK-NEXT: vinsgr2vr.h $vr1, $a2, 4 ; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 -; CHECK-NEXT: ld.h $a1, $sp, 48 +; CHECK-NEXT: ld.h $a2, $sp, 56 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 ; CHECK-NEXT: vinsgr2vr.h $vr1, $a1, 5 ; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 -; CHECK-NEXT: ld.h $a1, $sp, 56 +; CHECK-NEXT: ld.h $a1, $sp, 64 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 -; CHECK-NEXT: vinsgr2vr.h $vr1, $a1, 6 +; CHECK-NEXT: vinsgr2vr.h $vr1, $a2, 6 ; CHECK-NEXT: xvpermi.q $xr0, $xr1, 2 -; CHECK-NEXT: ld.h $a1, $sp, 64 ; CHECK-NEXT: xvori.b $xr1, $xr0, 0 ; CHECK-NEXT: xvpermi.q $xr1, $xr0, 1 ; CHECK-NEXT: vinsgr2vr.h $vr1, $a1, 7 @@ -452,6 +452,7 @@ entry: define void @buildvector_v8i32(ptr %dst, i32 %a0, i32 %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, i32 %a6, i32 %a7) nounwind { ; CHECK-LABEL: buildvector_v8i32: ; CHECK: # %bb.0: # %entry +; CHECK-NEXT: ld.w $t0, $sp, 0 ; CHECK-NEXT: xvinsgr2vr.w $xr0, $a1, 0 ; CHECK-NEXT: xvinsgr2vr.w $xr0, $a2, 1 ; CHECK-NEXT: xvinsgr2vr.w $xr0, $a3, 2 @@ -459,8 +460,7 @@ define void @buildvector_v8i32(ptr %dst, i32 %a0, i32 %a1, i32 %a2, i32 %a3, i32 ; CHECK-NEXT: xvinsgr2vr.w $xr0, $a5, 4 ; CHECK-NEXT: xvinsgr2vr.w $xr0, $a6, 5 ; CHECK-NEXT: xvinsgr2vr.w $xr0, $a7, 6 -; CHECK-NEXT: ld.w $a1, $sp, 0 -; CHECK-NEXT: xvinsgr2vr.w $xr0, $a1, 7 +; CHECK-NEXT: xvinsgr2vr.w $xr0, $t0, 7 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lasx/fma-v4f64.ll b/llvm/test/CodeGen/LoongArch/lasx/fma-v4f64.ll index af18c52b096c..0f3df3d573b6 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/fma-v4f64.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/fma-v4f64.ll @@ -9,30 +9,30 @@ define void @xvfmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: xvfmadd_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfmadd.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: xvfmadd_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 -; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-ON-NEXT: xvfadd.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvfadd.d $xr0, $xr0, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: xvfmadd_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-OFF-NEXT: xvfadd.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvfadd.d $xr0, $xr0, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -48,30 +48,30 @@ entry: define void @xvfmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: xvfmsub_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfmsub.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: xvfmsub_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 -; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr0, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: xvfmsub_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr0, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -87,31 +87,31 @@ entry: define void @xvfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: xvfnmadd_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: xvfnmadd_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 -; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-ON-NEXT: xvfadd.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvfadd.d $xr0, $xr0, $xr2 ; CONTRACT-ON-NEXT: xvbitrevi.d $xr0, $xr0, 63 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: xvfnmadd_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-OFF-NEXT: xvfadd.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvfadd.d $xr0, $xr0, $xr2 ; CONTRACT-OFF-NEXT: xvbitrevi.d $xr0, $xr0, 63 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret @@ -129,32 +129,32 @@ entry: define void @xvfnmadd_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: xvfnmadd_d_nsz: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: xvfnmadd_d_nsz: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-ON-NEXT: xvbitrevi.d $xr1, $xr1, 63 -; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 -; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvbitrevi.d $xr0, $xr0, 63 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr0, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: xvfnmadd_d_nsz: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-OFF-NEXT: xvbitrevi.d $xr1, $xr1, 63 -; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.d $xr0, $xr0, 63 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr0, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -173,33 +173,33 @@ entry: define void @not_xvfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: not_xvfnmadd_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvbitrevi.d $xr2, $xr2, 63 -; CONTRACT-FAST-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.d $xr0, $xr0, 63 +; CONTRACT-FAST-NEXT: xvfmsub.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: not_xvfnmadd_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-ON-NEXT: xvbitrevi.d $xr1, $xr1, 63 -; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 -; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvbitrevi.d $xr0, $xr0, 63 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr0, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: not_xvfnmadd_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-OFF-NEXT: xvbitrevi.d $xr1, $xr1, 63 -; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.d $xr0, $xr0, 63 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr0, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -217,31 +217,31 @@ entry: define void @xvfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: xvfnmsub_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: xvfnmsub_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 -; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr0, $xr2 ; CONTRACT-ON-NEXT: xvbitrevi.d $xr0, $xr0, 63 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: xvfnmsub_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr0, $xr2 ; CONTRACT-OFF-NEXT: xvbitrevi.d $xr0, $xr0, 63 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret @@ -260,30 +260,30 @@ entry: define void @xvfnmsub_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: xvfnmsub_d_nsz: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: xvfnmsub_d_nsz: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 -; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr2, $xr0 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: xvfnmsub_d_nsz: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr2, $xr0 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -301,31 +301,31 @@ entry: define void @not_xvfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: not_xvfnmsub_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvbitrevi.d $xr2, $xr2, 63 -; CONTRACT-FAST-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.d $xr0, $xr0, 63 +; CONTRACT-FAST-NEXT: xvfmadd.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: not_xvfnmsub_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 -; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr2, $xr0 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: not_xvfnmsub_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr2, $xr0 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -342,28 +342,28 @@ entry: define void @contract_xvfmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_xvfmadd_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfmadd.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_xvfmadd_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfmadd.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_xvfmadd_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfmadd.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -379,28 +379,28 @@ entry: define void @contract_xvfmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_xvfmsub_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfmsub.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_xvfmsub_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfmsub.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_xvfmsub_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfmsub.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -416,28 +416,28 @@ entry: define void @contract_xvfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_xvfnmadd_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_xvfnmadd_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfnmadd.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_xvfnmadd_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfnmadd.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -454,28 +454,28 @@ entry: define void @contract_xvfnmadd_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_xvfnmadd_d_nsz: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_xvfnmadd_d_nsz: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfnmadd.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_xvfnmadd_d_nsz: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfnmadd.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -494,31 +494,31 @@ entry: define void @not_contract_xvfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: not_contract_xvfnmadd_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvbitrevi.d $xr2, $xr2, 63 -; CONTRACT-FAST-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.d $xr0, $xr0, 63 +; CONTRACT-FAST-NEXT: xvfmsub.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: not_contract_xvfnmadd_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvbitrevi.d $xr2, $xr2, 63 -; CONTRACT-ON-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvbitrevi.d $xr0, $xr0, 63 +; CONTRACT-ON-NEXT: xvfmsub.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: not_contract_xvfnmadd_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvbitrevi.d $xr2, $xr2, 63 -; CONTRACT-OFF-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.d $xr0, $xr0, 63 +; CONTRACT-OFF-NEXT: xvfmsub.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -536,28 +536,28 @@ entry: define void @contract_xvfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_xvfnmsub_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_xvfnmsub_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfnmsub.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_xvfnmsub_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfnmsub.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -575,28 +575,28 @@ entry: define void @contract_xvfnmsub_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_xvfnmsub_d_nsz: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_xvfnmsub_d_nsz: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfnmsub.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_xvfnmsub_d_nsz: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfnmsub.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -614,31 +614,31 @@ entry: define void @not_contract_xvfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: not_contract_xvfnmsub_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvbitrevi.d $xr2, $xr2, 63 -; CONTRACT-FAST-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.d $xr0, $xr0, 63 +; CONTRACT-FAST-NEXT: xvfmadd.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: not_contract_xvfnmsub_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvbitrevi.d $xr2, $xr2, 63 -; CONTRACT-ON-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvbitrevi.d $xr0, $xr0, 63 +; CONTRACT-ON-NEXT: xvfmadd.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: not_contract_xvfnmsub_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvbitrevi.d $xr2, $xr2, 63 -; CONTRACT-OFF-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.d $xr0, $xr0, 63 +; CONTRACT-OFF-NEXT: xvfmadd.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -655,28 +655,28 @@ entry: define void @xvfmadd_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: xvfmadd_d_contract: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfmadd.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: xvfmadd_d_contract: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfmadd.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: xvfmadd_d_contract: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfmadd.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -692,28 +692,28 @@ entry: define void @xvfmsub_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: xvfmsub_d_contract: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfmsub.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: xvfmsub_d_contract: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfmsub.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: xvfmsub_d_contract: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfmsub.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -729,28 +729,28 @@ entry: define void @xvfnmadd_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: xvfnmadd_d_contract: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: xvfnmadd_d_contract: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfnmadd.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: xvfnmadd_d_contract: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfnmadd.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -767,28 +767,28 @@ entry: define void @xvfnmsub_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: xvfnmsub_d_contract: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: xvfnmsub_d_contract: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfnmsub.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: xvfnmsub_d_contract: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfnmsub.d $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lasx/fma-v8f32.ll b/llvm/test/CodeGen/LoongArch/lasx/fma-v8f32.ll index b7b3cb3a2e66..6fd14d93a751 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/fma-v8f32.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/fma-v8f32.ll @@ -9,30 +9,30 @@ define void @xvfmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: xvfmadd_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfmadd.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: xvfmadd_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 -; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-ON-NEXT: xvfadd.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvfadd.s $xr0, $xr0, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: xvfmadd_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-OFF-NEXT: xvfadd.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvfadd.s $xr0, $xr0, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -48,30 +48,30 @@ entry: define void @xvfmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: xvfmsub_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfmsub.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: xvfmsub_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 -; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr0, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: xvfmsub_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr0, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -87,31 +87,31 @@ entry: define void @xvfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: xvfnmadd_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: xvfnmadd_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 -; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-ON-NEXT: xvfadd.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvfadd.s $xr0, $xr0, $xr2 ; CONTRACT-ON-NEXT: xvbitrevi.w $xr0, $xr0, 31 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: xvfnmadd_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-OFF-NEXT: xvfadd.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvfadd.s $xr0, $xr0, $xr2 ; CONTRACT-OFF-NEXT: xvbitrevi.w $xr0, $xr0, 31 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret @@ -129,32 +129,32 @@ entry: define void @xvfnmadd_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: xvfnmadd_s_nsz: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: xvfnmadd_s_nsz: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-ON-NEXT: xvbitrevi.w $xr1, $xr1, 31 -; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 -; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvbitrevi.w $xr0, $xr0, 31 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr0, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: xvfnmadd_s_nsz: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-OFF-NEXT: xvbitrevi.w $xr1, $xr1, 31 -; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.w $xr0, $xr0, 31 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr0, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -173,33 +173,33 @@ entry: define void @not_xvfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: not_xvfnmadd_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvbitrevi.w $xr2, $xr2, 31 -; CONTRACT-FAST-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.w $xr0, $xr0, 31 +; CONTRACT-FAST-NEXT: xvfmsub.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: not_xvfnmadd_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-ON-NEXT: xvbitrevi.w $xr1, $xr1, 31 -; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 -; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvbitrevi.w $xr0, $xr0, 31 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr0, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: not_xvfnmadd_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-OFF-NEXT: xvbitrevi.w $xr1, $xr1, 31 -; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.w $xr0, $xr0, 31 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr0, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -217,31 +217,31 @@ entry: define void @xvfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: xvfnmsub_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: xvfnmsub_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 -; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr0, $xr2 ; CONTRACT-ON-NEXT: xvbitrevi.w $xr0, $xr0, 31 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: xvfnmsub_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr0, $xr2 ; CONTRACT-OFF-NEXT: xvbitrevi.w $xr0, $xr0, 31 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret @@ -260,30 +260,30 @@ entry: define void @xvfnmsub_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: xvfnmsub_s_nsz: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: xvfnmsub_s_nsz: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 -; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr2, $xr0 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: xvfnmsub_s_nsz: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr2, $xr0 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -301,31 +301,31 @@ entry: define void @not_xvfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: not_xvfnmsub_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvbitrevi.w $xr2, $xr2, 31 -; CONTRACT-FAST-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.w $xr0, $xr0, 31 +; CONTRACT-FAST-NEXT: xvfmadd.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: not_xvfnmsub_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 -; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr2, $xr0 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: not_xvfnmsub_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 -; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 -; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 -; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr2, $xr0 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -342,28 +342,28 @@ entry: define void @contract_xvfmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_xvfmadd_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfmadd.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_xvfmadd_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfmadd.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_xvfmadd_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfmadd.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -379,28 +379,28 @@ entry: define void @contract_xvfmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_xvfmsub_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfmsub.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_xvfmsub_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfmsub.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_xvfmsub_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfmsub.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -416,28 +416,28 @@ entry: define void @contract_xvfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_xvfnmadd_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_xvfnmadd_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfnmadd.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_xvfnmadd_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfnmadd.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -454,28 +454,28 @@ entry: define void @contract_xvfnmadd_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_xvfnmadd_s_nsz: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_xvfnmadd_s_nsz: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfnmadd.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_xvfnmadd_s_nsz: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfnmadd.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -494,31 +494,31 @@ entry: define void @not_contract_xvfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: not_contract_xvfnmadd_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvbitrevi.w $xr2, $xr2, 31 -; CONTRACT-FAST-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.w $xr0, $xr0, 31 +; CONTRACT-FAST-NEXT: xvfmsub.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: not_contract_xvfnmadd_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvbitrevi.w $xr2, $xr2, 31 -; CONTRACT-ON-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvbitrevi.w $xr0, $xr0, 31 +; CONTRACT-ON-NEXT: xvfmsub.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: not_contract_xvfnmadd_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvbitrevi.w $xr2, $xr2, 31 -; CONTRACT-OFF-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.w $xr0, $xr0, 31 +; CONTRACT-OFF-NEXT: xvfmsub.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -536,28 +536,28 @@ entry: define void @contract_xvfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_xvfnmsub_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_xvfnmsub_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfnmsub.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_xvfnmsub_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfnmsub.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -575,28 +575,28 @@ entry: define void @contract_xvfnmsub_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_xvfnmsub_s_nsz: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_xvfnmsub_s_nsz: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfnmsub.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_xvfnmsub_s_nsz: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfnmsub.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -614,31 +614,31 @@ entry: define void @not_contract_xvfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: not_contract_xvfnmsub_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvbitrevi.w $xr2, $xr2, 31 -; CONTRACT-FAST-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.w $xr0, $xr0, 31 +; CONTRACT-FAST-NEXT: xvfmadd.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: not_contract_xvfnmsub_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvbitrevi.w $xr2, $xr2, 31 -; CONTRACT-ON-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvbitrevi.w $xr0, $xr0, 31 +; CONTRACT-ON-NEXT: xvfmadd.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: not_contract_xvfnmsub_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvbitrevi.w $xr2, $xr2, 31 -; CONTRACT-OFF-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.w $xr0, $xr0, 31 +; CONTRACT-OFF-NEXT: xvfmadd.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -655,28 +655,28 @@ entry: define void @xvfmadd_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: xvfmadd_s_contract: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfmadd.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: xvfmadd_s_contract: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfmadd.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: xvfmadd_s_contract: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfmadd.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -692,28 +692,28 @@ entry: define void @xvfmsub_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: xvfmsub_s_contract: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfmsub.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: xvfmsub_s_contract: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfmsub.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: xvfmsub_s_contract: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfmsub.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -729,28 +729,28 @@ entry: define void @xvfnmadd_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: xvfnmadd_s_contract: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: xvfnmadd_s_contract: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfnmadd.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: xvfnmadd_s_contract: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfnmadd.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -767,28 +767,28 @@ entry: define void @xvfnmsub_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: xvfnmsub_s_contract: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-FAST-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: xvfnmsub_s_contract: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-ON-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-ON-NEXT: xvfnmsub.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: xvfnmsub_s_contract: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr0, $a1, 0 ; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 -; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 -; CONTRACT-OFF-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a3, 0 +; CONTRACT-OFF-NEXT: xvfnmsub.s $xr0, $xr0, $xr1, $xr2 ; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/add.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/add.ll index 8e4d0dc6f1c3..136f34bafb32 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/add.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/add.ll @@ -4,9 +4,9 @@ define void @add_v32i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: add_v32i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvadd.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvadd.b $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @add_v16i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: add_v16i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvadd.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvadd.h $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -36,9 +36,9 @@ entry: define void @add_v8i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: add_v8i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvadd.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvadd.w $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -52,9 +52,9 @@ entry: define void @add_v4i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: add_v4i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvadd.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvadd.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/and.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/and.ll index 98c87cadeeb5..b06d1bea4ef6 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/and.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/and.ll @@ -4,9 +4,9 @@ define void @and_v32i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: and_v32i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvand.v $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvand.v $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @and_v16i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: and_v16i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvand.v $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvand.v $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -36,9 +36,9 @@ entry: define void @and_v8i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: and_v8i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvand.v $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvand.v $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -52,9 +52,9 @@ entry: define void @and_v4i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: and_v4i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvand.v $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvand.v $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/ashr.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/ashr.ll index fcbf0f1400fe..4dd2cee7a2ed 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/ashr.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/ashr.ll @@ -4,9 +4,9 @@ define void @ashr_v32i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: ashr_v32i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsra.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsra.b $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @ashr_v16i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: ashr_v16i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsra.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsra.h $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -36,9 +36,9 @@ entry: define void @ashr_v8i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: ashr_v8i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsra.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsra.w $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -52,9 +52,9 @@ entry: define void @ashr_v4i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: ashr_v4i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsra.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsra.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fadd.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fadd.ll index 365bb305fc5a..b3eb328e8d44 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fadd.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fadd.ll @@ -4,9 +4,9 @@ define void @fadd_v8f32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: fadd_v8f32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfadd.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfadd.s $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @fadd_v4f64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: fadd_v4f64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfadd.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfadd.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fcmp.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fcmp.ll index ef67dbc100c0..4f56dd29c1b2 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fcmp.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fcmp.ll @@ -35,9 +35,9 @@ define void @v4f64_fcmp_false(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v8f32_fcmp_oeq(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8f32_fcmp_oeq: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.ceq.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.ceq.s $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x float>, ptr %a0 @@ -51,9 +51,9 @@ define void @v8f32_fcmp_oeq(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f64_fcmp_oeq(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f64_fcmp_oeq: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.ceq.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.ceq.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x double>, ptr %a0 @@ -68,9 +68,9 @@ define void @v4f64_fcmp_oeq(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v8f32_fcmp_ueq(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8f32_fcmp_ueq: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.cueq.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cueq.s $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x float>, ptr %a0 @@ -84,9 +84,9 @@ define void @v8f32_fcmp_ueq(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f64_fcmp_ueq(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f64_fcmp_ueq: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.cueq.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cueq.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x double>, ptr %a0 @@ -101,9 +101,9 @@ define void @v4f64_fcmp_ueq(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v8f32_fcmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8f32_fcmp_eq: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.ceq.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.ceq.s $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x float>, ptr %a0 @@ -117,9 +117,9 @@ define void @v8f32_fcmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f64_fcmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f64_fcmp_eq: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.ceq.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.ceq.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x double>, ptr %a0 @@ -134,9 +134,9 @@ define void @v4f64_fcmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v8f32_fcmp_ole(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8f32_fcmp_ole: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.cle.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cle.s $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x float>, ptr %a0 @@ -150,9 +150,9 @@ define void @v8f32_fcmp_ole(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f64_fcmp_ole(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f64_fcmp_ole: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.cle.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cle.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x double>, ptr %a0 @@ -167,9 +167,9 @@ define void @v4f64_fcmp_ole(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v8f32_fcmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8f32_fcmp_ule: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.cule.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cule.s $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x float>, ptr %a0 @@ -183,9 +183,9 @@ define void @v8f32_fcmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f64_fcmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f64_fcmp_ule: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.cule.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cule.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x double>, ptr %a0 @@ -200,9 +200,9 @@ define void @v4f64_fcmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v8f32_fcmp_le(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8f32_fcmp_le: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.cle.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cle.s $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x float>, ptr %a0 @@ -216,9 +216,9 @@ define void @v8f32_fcmp_le(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f64_fcmp_le(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f64_fcmp_le: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.cle.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cle.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x double>, ptr %a0 @@ -233,9 +233,9 @@ define void @v4f64_fcmp_le(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v8f32_fcmp_olt(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8f32_fcmp_olt: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.clt.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.clt.s $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x float>, ptr %a0 @@ -249,9 +249,9 @@ define void @v8f32_fcmp_olt(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f64_fcmp_olt(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f64_fcmp_olt: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.clt.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.clt.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x double>, ptr %a0 @@ -266,9 +266,9 @@ define void @v4f64_fcmp_olt(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v8f32_fcmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8f32_fcmp_ult: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.cult.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cult.s $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x float>, ptr %a0 @@ -282,9 +282,9 @@ define void @v8f32_fcmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f64_fcmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f64_fcmp_ult: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.cult.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cult.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x double>, ptr %a0 @@ -299,9 +299,9 @@ define void @v4f64_fcmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v8f32_fcmp_lt(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8f32_fcmp_lt: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.clt.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.clt.s $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x float>, ptr %a0 @@ -315,9 +315,9 @@ define void @v8f32_fcmp_lt(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f64_fcmp_lt(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f64_fcmp_lt: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.clt.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.clt.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x double>, ptr %a0 @@ -332,9 +332,9 @@ define void @v4f64_fcmp_lt(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v8f32_fcmp_one(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8f32_fcmp_one: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.cne.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cne.s $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x float>, ptr %a0 @@ -348,9 +348,9 @@ define void @v8f32_fcmp_one(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f64_fcmp_one(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f64_fcmp_one: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.cne.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cne.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x double>, ptr %a0 @@ -365,9 +365,9 @@ define void @v4f64_fcmp_one(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v8f32_fcmp_une(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8f32_fcmp_une: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.cune.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cune.s $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x float>, ptr %a0 @@ -381,9 +381,9 @@ define void @v8f32_fcmp_une(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f64_fcmp_une(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f64_fcmp_une: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.cune.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cune.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x double>, ptr %a0 @@ -398,9 +398,9 @@ define void @v4f64_fcmp_une(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v8f32_fcmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8f32_fcmp_ne: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.cne.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cne.s $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x float>, ptr %a0 @@ -414,9 +414,9 @@ define void @v8f32_fcmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f64_fcmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f64_fcmp_ne: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.cne.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cne.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x double>, ptr %a0 @@ -431,9 +431,9 @@ define void @v4f64_fcmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v8f32_fcmp_ord(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8f32_fcmp_ord: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.cor.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cor.s $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x float>, ptr %a0 @@ -447,9 +447,9 @@ define void @v8f32_fcmp_ord(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f64_fcmp_ord(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f64_fcmp_ord: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.cor.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cor.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x double>, ptr %a0 @@ -464,9 +464,9 @@ define void @v4f64_fcmp_ord(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v8f32_fcmp_uno(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8f32_fcmp_uno: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.cun.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cun.s $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x float>, ptr %a0 @@ -480,9 +480,9 @@ define void @v8f32_fcmp_uno(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f64_fcmp_uno(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f64_fcmp_uno: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfcmp.cun.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cun.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x double>, ptr %a0 diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fdiv.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fdiv.ll index 6004565b0b78..63d8c222ae54 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fdiv.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fdiv.ll @@ -4,9 +4,9 @@ define void @fdiv_v8f32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: fdiv_v8f32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfdiv.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfdiv.s $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @fdiv_v4f64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: fdiv_v4f64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfdiv.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfdiv.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fmul.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fmul.ll index a48dca8d2847..f777151cdb0a 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fmul.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fmul.ll @@ -4,9 +4,9 @@ define void @fmul_v8f32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: fmul_v8f32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfmul.s $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @fmul_v4f64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: fmul_v4f64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfmul.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fsub.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fsub.ll index 6164aa5a55c7..201ba5f5df66 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fsub.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fsub.ll @@ -4,9 +4,9 @@ define void @fsub_v8f32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: fsub_v8f32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfsub.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfsub.s $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @fsub_v4f64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: fsub_v4f64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvfsub.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfsub.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/icmp.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/icmp.ll index 6693fe0f6ec7..d15c4133855f 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/icmp.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/icmp.ll @@ -19,9 +19,9 @@ define void @v32i8_icmp_eq_imm(ptr %res, ptr %a0) nounwind { define void @v32i8_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v32i8_icmp_eq: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvseq.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvseq.b $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <32 x i8>, ptr %a0 @@ -49,9 +49,9 @@ define void @v16i16_icmp_eq_imm(ptr %res, ptr %a0) nounwind { define void @v16i16_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v16i16_icmp_eq: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvseq.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvseq.h $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <16 x i16>, ptr %a0 @@ -79,9 +79,9 @@ define void @v8i32_icmp_eq_imm(ptr %res, ptr %a0) nounwind { define void @v8i32_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8i32_icmp_eq: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvseq.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvseq.w $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x i32>, ptr %a0 @@ -109,9 +109,9 @@ define void @v4i64_icmp_eq_imm(ptr %res, ptr %a0) nounwind { define void @v4i64_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4i64_icmp_eq: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvseq.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvseq.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x i64>, ptr %a0 @@ -140,9 +140,9 @@ define void @v32i8_icmp_sle_imm(ptr %res, ptr %a0) nounwind { define void @v32i8_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v32i8_icmp_sle: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsle.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.b $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <32 x i8>, ptr %a0 @@ -170,9 +170,9 @@ define void @v16i16_icmp_sle_imm(ptr %res, ptr %a0) nounwind { define void @v16i16_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v16i16_icmp_sle: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsle.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.h $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <16 x i16>, ptr %a0 @@ -200,9 +200,9 @@ define void @v8i32_icmp_sle_imm(ptr %res, ptr %a0) nounwind { define void @v8i32_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8i32_icmp_sle: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsle.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.w $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x i32>, ptr %a0 @@ -230,9 +230,9 @@ define void @v4i64_icmp_sle_imm(ptr %res, ptr %a0) nounwind { define void @v4i64_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4i64_icmp_sle: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsle.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x i64>, ptr %a0 @@ -261,9 +261,9 @@ define void @v32i8_icmp_ule_imm(ptr %res, ptr %a0) nounwind { define void @v32i8_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v32i8_icmp_ule: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsle.bu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.bu $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <32 x i8>, ptr %a0 @@ -291,9 +291,9 @@ define void @v16i16_icmp_ule_imm(ptr %res, ptr %a0) nounwind { define void @v16i16_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v16i16_icmp_ule: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsle.hu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.hu $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <16 x i16>, ptr %a0 @@ -321,9 +321,9 @@ define void @v8i32_icmp_ule_imm(ptr %res, ptr %a0) nounwind { define void @v8i32_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8i32_icmp_ule: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsle.wu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.wu $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x i32>, ptr %a0 @@ -351,9 +351,9 @@ define void @v4i64_icmp_ule_imm(ptr %res, ptr %a0) nounwind { define void @v4i64_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4i64_icmp_ule: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsle.du $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.du $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x i64>, ptr %a0 @@ -382,9 +382,9 @@ define void @v32i8_icmp_slt_imm(ptr %res, ptr %a0) nounwind { define void @v32i8_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v32i8_icmp_slt: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvslt.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.b $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <32 x i8>, ptr %a0 @@ -412,9 +412,9 @@ define void @v16i16_icmp_slt_imm(ptr %res, ptr %a0) nounwind { define void @v16i16_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v16i16_icmp_slt: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvslt.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.h $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <16 x i16>, ptr %a0 @@ -442,9 +442,9 @@ define void @v8i32_icmp_slt_imm(ptr %res, ptr %a0) nounwind { define void @v8i32_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8i32_icmp_slt: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvslt.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.w $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x i32>, ptr %a0 @@ -472,9 +472,9 @@ define void @v4i64_icmp_slt_imm(ptr %res, ptr %a0) nounwind { define void @v4i64_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4i64_icmp_slt: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvslt.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x i64>, ptr %a0 @@ -503,9 +503,9 @@ define void @v32i8_icmp_ult_imm(ptr %res, ptr %a0) nounwind { define void @v32i8_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v32i8_icmp_ult: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvslt.bu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.bu $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <32 x i8>, ptr %a0 @@ -533,9 +533,9 @@ define void @v16i16_icmp_ult_imm(ptr %res, ptr %a0) nounwind { define void @v16i16_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v16i16_icmp_ult: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvslt.hu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.hu $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <16 x i16>, ptr %a0 @@ -563,9 +563,9 @@ define void @v8i32_icmp_ult_imm(ptr %res, ptr %a0) nounwind { define void @v8i32_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8i32_icmp_ult: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvslt.wu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.wu $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x i32>, ptr %a0 @@ -593,9 +593,9 @@ define void @v4i64_icmp_ult_imm(ptr %res, ptr %a0) nounwind { define void @v4i64_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4i64_icmp_ult: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvslt.du $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.du $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x i64>, ptr %a0 @@ -610,9 +610,9 @@ define void @v4i64_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v32i8_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v32i8_icmp_ne: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvseq.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvseq.b $xr0, $xr0, $xr1 ; CHECK-NEXT: xvxori.b $xr0, $xr0, 255 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret @@ -627,9 +627,9 @@ define void @v32i8_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v16i16_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v16i16_icmp_ne: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvseq.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvseq.h $xr0, $xr0, $xr1 ; CHECK-NEXT: xvrepli.b $xr1, -1 ; CHECK-NEXT: xvxor.v $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 @@ -645,9 +645,9 @@ define void @v16i16_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v8i32_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8i32_icmp_ne: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvseq.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvseq.w $xr0, $xr0, $xr1 ; CHECK-NEXT: xvrepli.b $xr1, -1 ; CHECK-NEXT: xvxor.v $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 @@ -663,9 +663,9 @@ define void @v8i32_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4i64_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4i64_icmp_ne: ; CHECK: # %bb.0: -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvseq.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvseq.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvrepli.b $xr1, -1 ; CHECK-NEXT: xvxor.v $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll index ceaf40027ffc..25106b456d2f 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll @@ -88,10 +88,10 @@ define void @insert_4xi64(ptr %src, ptr %dst, i64 %in) nounwind { define void @insert_8xfloat(ptr %src, ptr %dst, float %in) nounwind { ; CHECK-LABEL: insert_8xfloat: ; CHECK: # %bb.0: -; CHECK-NEXT: movfr2gr.s $a2, $fa0 -; CHECK-NEXT: xvld $xr0, $a0, 0 -; CHECK-NEXT: xvinsgr2vr.w $xr0, $a2, 1 -; CHECK-NEXT: xvst $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a0, 0 +; CHECK-NEXT: movfr2gr.s $a0, $fa0 +; CHECK-NEXT: xvinsgr2vr.w $xr1, $a0, 1 +; CHECK-NEXT: xvst $xr1, $a1, 0 ; CHECK-NEXT: ret %v = load volatile <8 x float>, ptr %src %v_new = insertelement <8 x float> %v, float %in, i32 1 @@ -102,10 +102,10 @@ define void @insert_8xfloat(ptr %src, ptr %dst, float %in) nounwind { define void @insert_4xdouble(ptr %src, ptr %dst, double %in) nounwind { ; CHECK-LABEL: insert_4xdouble: ; CHECK: # %bb.0: -; CHECK-NEXT: movfr2gr.d $a2, $fa0 -; CHECK-NEXT: xvld $xr0, $a0, 0 -; CHECK-NEXT: xvinsgr2vr.d $xr0, $a2, 1 -; CHECK-NEXT: xvst $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a0, 0 +; CHECK-NEXT: movfr2gr.d $a0, $fa0 +; CHECK-NEXT: xvinsgr2vr.d $xr1, $a0, 1 +; CHECK-NEXT: xvst $xr1, $a1, 0 ; CHECK-NEXT: ret %v = load volatile <4 x double>, ptr %src %v_new = insertelement <4 x double> %v, double %in, i32 1 diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/lshr.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/lshr.ll index 24be69d8032a..5b992b5e38de 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/lshr.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/lshr.ll @@ -4,9 +4,9 @@ define void @lshr_v32i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: lshr_v32i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsrl.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsrl.b $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @lshr_v16i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: lshr_v16i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsrl.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsrl.h $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -36,9 +36,9 @@ entry: define void @lshr_v8i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: lshr_v8i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsrl.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsrl.w $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -52,9 +52,9 @@ entry: define void @lshr_v4i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: lshr_v4i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsrl.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsrl.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/mul.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/mul.ll index dcb893caa255..4745e7003cb1 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/mul.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/mul.ll @@ -4,9 +4,9 @@ define void @mul_v32i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mul_v32i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvmul.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvmul.b $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @mul_v16i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mul_v16i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvmul.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvmul.h $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -36,9 +36,9 @@ entry: define void @mul_v8i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mul_v8i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvmul.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvmul.w $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -52,9 +52,9 @@ entry: define void @mul_v4i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mul_v4i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvmul.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvmul.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/or.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/or.ll index f37cbf1cefed..f32b8897bebc 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/or.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/or.ll @@ -4,9 +4,9 @@ define void @or_v32i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: or_v32i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvor.v $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvor.v $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @or_v16i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: or_v16i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvor.v $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvor.v $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -36,9 +36,9 @@ entry: define void @or_v8i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: or_v8i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvor.v $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvor.v $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -52,9 +52,9 @@ entry: define void @or_v4i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: or_v4i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvor.v $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvor.v $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/sdiv.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/sdiv.ll index e3635a5f14a2..879caa5a6700 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/sdiv.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/sdiv.ll @@ -4,9 +4,9 @@ define void @sdiv_v32i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: sdiv_v32i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvdiv.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvdiv.b $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @sdiv_v16i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: sdiv_v16i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvdiv.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvdiv.h $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -36,9 +36,9 @@ entry: define void @sdiv_v8i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: sdiv_v8i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvdiv.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvdiv.w $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -52,9 +52,9 @@ entry: define void @sdiv_v4i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: sdiv_v4i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvdiv.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvdiv.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/shl.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/shl.ll index 8a02c7e3ac97..56c69171c9d4 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/shl.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/shl.ll @@ -4,9 +4,9 @@ define void @shl_v32i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: shl_v32i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsll.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsll.b $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @shl_v16i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: shl_v16i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsll.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsll.h $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -36,9 +36,9 @@ entry: define void @shl_v8i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: shl_v8i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsll.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsll.w $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -52,9 +52,9 @@ entry: define void @shl_v4i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: shl_v4i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsll.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsll.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/sub.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/sub.ll index bcfff1651477..5102abac83d8 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/sub.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/sub.ll @@ -4,9 +4,9 @@ define void @sub_v32i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: sub_v32i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsub.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsub.b $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @sub_v16i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: sub_v16i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsub.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsub.h $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -36,9 +36,9 @@ entry: define void @sub_v8i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: sub_v8i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsub.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsub.w $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -52,9 +52,9 @@ entry: define void @sub_v4i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: sub_v4i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvsub.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsub.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/udiv.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/udiv.ll index e78084c7186d..43f558f3cdf3 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/udiv.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/udiv.ll @@ -4,9 +4,9 @@ define void @udiv_v32i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: udiv_v32i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvdiv.bu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvdiv.bu $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @udiv_v16i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: udiv_v16i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvdiv.hu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvdiv.hu $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -36,9 +36,9 @@ entry: define void @udiv_v8i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: udiv_v8i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvdiv.wu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvdiv.wu $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -52,9 +52,9 @@ entry: define void @udiv_v4i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: udiv_v4i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvdiv.du $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvdiv.du $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/xor.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/xor.ll index c2fb1462b7a2..e062e10b21d9 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/xor.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/xor.ll @@ -4,9 +4,9 @@ define void @xor_v32i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: xor_v32i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvxor.v $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvxor.v $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @xor_v16i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: xor_v16i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvxor.v $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvxor.v $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -36,9 +36,9 @@ entry: define void @xor_v8i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: xor_v8i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvxor.v $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvxor.v $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -52,9 +52,9 @@ entry: define void @xor_v4i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: xor_v4i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvxor.v $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvxor.v $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lasx/mulh.ll b/llvm/test/CodeGen/LoongArch/lasx/mulh.ll index aac711a4a371..db3cc7f38774 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/mulh.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/mulh.ll @@ -4,9 +4,9 @@ define void @mulhs_v32i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mulhs_v32i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvmuh.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvmuh.b $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -24,9 +24,9 @@ entry: define void @mulhu_v32i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mulhu_v32i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvmuh.bu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvmuh.bu $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -44,9 +44,9 @@ entry: define void @mulhs_v16i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mulhs_v16i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvmuh.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvmuh.h $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -64,9 +64,9 @@ entry: define void @mulhu_v16i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mulhu_v16i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvmuh.hu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvmuh.hu $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -84,9 +84,9 @@ entry: define void @mulhs_v8i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mulhs_v8i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvmuh.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvmuh.w $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -104,9 +104,9 @@ entry: define void @mulhu_v8i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mulhu_v8i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvmuh.wu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvmuh.wu $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -124,9 +124,9 @@ entry: define void @mulhs_v4i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mulhs_v4i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvmuh.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvmuh.d $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -144,9 +144,9 @@ entry: define void @mulhu_v4i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mulhu_v4i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: xvld $xr0, $a2, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvmuh.du $xr0, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvmuh.du $xr0, $xr0, $xr1 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lasx/vselect.ll b/llvm/test/CodeGen/LoongArch/lasx/vselect.ll index a9a54257917a..7b4c7ced4b5f 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/vselect.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/vselect.ll @@ -34,11 +34,11 @@ define void @select_v32i8(ptr %res, ptr %a0, ptr %a1) nounwind { define void @select_v16i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: select_v16i16: ; CHECK: # %bb.0: -; CHECK-NEXT: lu12i.w $a3, -16 -; CHECK-NEXT: xvreplgr2vr.w $xr0, $a3 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvld $xr2, $a2, 0 -; CHECK-NEXT: xvbitsel.v $xr0, $xr2, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: lu12i.w $a1, -16 +; CHECK-NEXT: xvreplgr2vr.w $xr2, $a1 +; CHECK-NEXT: xvbitsel.v $xr0, $xr1, $xr0, $xr2 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <16 x i16>, ptr %a0 @@ -69,12 +69,12 @@ define void @select_v8i32(ptr %res, ptr %a0, ptr %a1) nounwind { define void @select_v4i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: select_v4i64: ; CHECK: # %bb.0: -; CHECK-NEXT: pcalau12i $a3, %pc_hi20(.LCPI4_0) -; CHECK-NEXT: addi.d $a3, $a3, %pc_lo12(.LCPI4_0) -; CHECK-NEXT: xvld $xr0, $a3, 0 -; CHECK-NEXT: xvld $xr1, $a1, 0 -; CHECK-NEXT: xvld $xr2, $a2, 0 -; CHECK-NEXT: xvbitsel.v $xr0, $xr2, $xr1, $xr0 +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: pcalau12i $a1, %pc_hi20(.LCPI4_0) +; CHECK-NEXT: addi.d $a1, $a1, %pc_lo12(.LCPI4_0) +; CHECK-NEXT: xvld $xr2, $a1, 0 +; CHECK-NEXT: xvbitsel.v $xr0, $xr1, $xr0, $xr2 ; CHECK-NEXT: xvst $xr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x i64>, ptr %a0 diff --git a/llvm/test/CodeGen/LoongArch/lsx/build-vector.ll b/llvm/test/CodeGen/LoongArch/lsx/build-vector.ll index ed1f610a5fa6..c04d7ca889f7 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/build-vector.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/build-vector.ll @@ -230,6 +230,15 @@ entry: define void @buildvector_v16i8(ptr %dst, i8 %a0, i8 %a1, i8 %a2, i8 %a3, i8 %a4, i8 %a5, i8 %a6, i8 %a7, i8 %a8, i8 %a9, i8 %a10, i8 %a11, i8 %a12, i8 %a13, i8 %a14, i8 %a15) nounwind { ; CHECK-LABEL: buildvector_v16i8: ; CHECK: # %bb.0: # %entry +; CHECK-NEXT: ld.b $t0, $sp, 64 +; CHECK-NEXT: ld.b $t1, $sp, 56 +; CHECK-NEXT: ld.b $t2, $sp, 48 +; CHECK-NEXT: ld.b $t3, $sp, 40 +; CHECK-NEXT: ld.b $t4, $sp, 32 +; CHECK-NEXT: ld.b $t5, $sp, 24 +; CHECK-NEXT: ld.b $t6, $sp, 16 +; CHECK-NEXT: ld.b $t7, $sp, 8 +; CHECK-NEXT: ld.b $t8, $sp, 0 ; CHECK-NEXT: vinsgr2vr.b $vr0, $a1, 0 ; CHECK-NEXT: vinsgr2vr.b $vr0, $a2, 1 ; CHECK-NEXT: vinsgr2vr.b $vr0, $a3, 2 @@ -237,24 +246,15 @@ define void @buildvector_v16i8(ptr %dst, i8 %a0, i8 %a1, i8 %a2, i8 %a3, i8 %a4, ; CHECK-NEXT: vinsgr2vr.b $vr0, $a5, 4 ; CHECK-NEXT: vinsgr2vr.b $vr0, $a6, 5 ; CHECK-NEXT: vinsgr2vr.b $vr0, $a7, 6 -; CHECK-NEXT: ld.b $a1, $sp, 0 -; CHECK-NEXT: vinsgr2vr.b $vr0, $a1, 7 -; CHECK-NEXT: ld.b $a1, $sp, 8 -; CHECK-NEXT: vinsgr2vr.b $vr0, $a1, 8 -; CHECK-NEXT: ld.b $a1, $sp, 16 -; CHECK-NEXT: vinsgr2vr.b $vr0, $a1, 9 -; CHECK-NEXT: ld.b $a1, $sp, 24 -; CHECK-NEXT: vinsgr2vr.b $vr0, $a1, 10 -; CHECK-NEXT: ld.b $a1, $sp, 32 -; CHECK-NEXT: vinsgr2vr.b $vr0, $a1, 11 -; CHECK-NEXT: ld.b $a1, $sp, 40 -; CHECK-NEXT: vinsgr2vr.b $vr0, $a1, 12 -; CHECK-NEXT: ld.b $a1, $sp, 48 -; CHECK-NEXT: vinsgr2vr.b $vr0, $a1, 13 -; CHECK-NEXT: ld.b $a1, $sp, 56 -; CHECK-NEXT: vinsgr2vr.b $vr0, $a1, 14 -; CHECK-NEXT: ld.b $a1, $sp, 64 -; CHECK-NEXT: vinsgr2vr.b $vr0, $a1, 15 +; CHECK-NEXT: vinsgr2vr.b $vr0, $t8, 7 +; CHECK-NEXT: vinsgr2vr.b $vr0, $t7, 8 +; CHECK-NEXT: vinsgr2vr.b $vr0, $t6, 9 +; CHECK-NEXT: vinsgr2vr.b $vr0, $t5, 10 +; CHECK-NEXT: vinsgr2vr.b $vr0, $t4, 11 +; CHECK-NEXT: vinsgr2vr.b $vr0, $t3, 12 +; CHECK-NEXT: vinsgr2vr.b $vr0, $t2, 13 +; CHECK-NEXT: vinsgr2vr.b $vr0, $t1, 14 +; CHECK-NEXT: vinsgr2vr.b $vr0, $t0, 15 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -281,6 +281,7 @@ entry: define void @buildvector_v8i16(ptr %dst, i16 %a0, i16 %a1, i16 %a2, i16 %a3, i16 %a4, i16 %a5, i16 %a6, i16 %a7) nounwind { ; CHECK-LABEL: buildvector_v8i16: ; CHECK: # %bb.0: # %entry +; CHECK-NEXT: ld.h $t0, $sp, 0 ; CHECK-NEXT: vinsgr2vr.h $vr0, $a1, 0 ; CHECK-NEXT: vinsgr2vr.h $vr0, $a2, 1 ; CHECK-NEXT: vinsgr2vr.h $vr0, $a3, 2 @@ -288,8 +289,7 @@ define void @buildvector_v8i16(ptr %dst, i16 %a0, i16 %a1, i16 %a2, i16 %a3, i16 ; CHECK-NEXT: vinsgr2vr.h $vr0, $a5, 4 ; CHECK-NEXT: vinsgr2vr.h $vr0, $a6, 5 ; CHECK-NEXT: vinsgr2vr.h $vr0, $a7, 6 -; CHECK-NEXT: ld.h $a1, $sp, 0 -; CHECK-NEXT: vinsgr2vr.h $vr0, $a1, 7 +; CHECK-NEXT: vinsgr2vr.h $vr0, $t0, 7 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lsx/fma-v2f64.ll b/llvm/test/CodeGen/LoongArch/lsx/fma-v2f64.ll index 8e0459b4afab..c83c563952d4 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/fma-v2f64.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/fma-v2f64.ll @@ -9,30 +9,30 @@ define void @vfmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: vfmadd_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfmadd.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: vfmadd_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 -; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 -; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 -; CONTRACT-ON-NEXT: vfadd.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vfadd.d $vr0, $vr0, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: vfmadd_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 -; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 -; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 -; CONTRACT-OFF-NEXT: vfadd.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vfadd.d $vr0, $vr0, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -48,30 +48,30 @@ entry: define void @vfmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: vfmsub_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfmsub.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: vfmsub_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 -; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 -; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 -; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr0, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: vfmsub_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 -; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 -; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 -; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr0, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -87,31 +87,31 @@ entry: define void @vfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: vfnmadd_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfnmadd.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: vfnmadd_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 -; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 -; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 -; CONTRACT-ON-NEXT: vfadd.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vfadd.d $vr0, $vr0, $vr2 ; CONTRACT-ON-NEXT: vbitrevi.d $vr0, $vr0, 63 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: vfnmadd_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 -; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 -; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 -; CONTRACT-OFF-NEXT: vfadd.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vfadd.d $vr0, $vr0, $vr2 ; CONTRACT-OFF-NEXT: vbitrevi.d $vr0, $vr0, 63 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret @@ -129,32 +129,32 @@ entry: define void @vfnmadd_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: vfnmadd_d_nsz: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfnmadd.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: vfnmadd_d_nsz: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 -; CONTRACT-ON-NEXT: vbitrevi.d $vr1, $vr1, 63 -; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 -; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 -; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vbitrevi.d $vr0, $vr0, 63 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr0, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: vfnmadd_d_nsz: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 -; CONTRACT-OFF-NEXT: vbitrevi.d $vr1, $vr1, 63 -; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 -; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 -; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vbitrevi.d $vr0, $vr0, 63 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr0, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -173,33 +173,33 @@ entry: define void @not_vfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: not_vfnmadd_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vbitrevi.d $vr2, $vr2, 63 -; CONTRACT-FAST-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vbitrevi.d $vr0, $vr0, 63 +; CONTRACT-FAST-NEXT: vfmsub.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: not_vfnmadd_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 -; CONTRACT-ON-NEXT: vbitrevi.d $vr1, $vr1, 63 -; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 -; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 -; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vbitrevi.d $vr0, $vr0, 63 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr0, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: not_vfnmadd_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 -; CONTRACT-OFF-NEXT: vbitrevi.d $vr1, $vr1, 63 -; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 -; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 -; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vbitrevi.d $vr0, $vr0, 63 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr0, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -217,31 +217,31 @@ entry: define void @vfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: vfnmsub_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfnmsub.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: vfnmsub_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 -; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 -; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 -; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr0, $vr2 ; CONTRACT-ON-NEXT: vbitrevi.d $vr0, $vr0, 63 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: vfnmsub_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 -; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 -; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 -; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr0, $vr2 ; CONTRACT-OFF-NEXT: vbitrevi.d $vr0, $vr0, 63 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret @@ -260,30 +260,30 @@ entry: define void @vfnmsub_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: vfnmsub_d_nsz: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfnmsub.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: vfnmsub_d_nsz: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 -; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 -; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 -; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr2, $vr0 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: vfnmsub_d_nsz: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 -; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 -; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 -; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr2, $vr0 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -301,31 +301,31 @@ entry: define void @not_vfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: not_vfnmsub_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vbitrevi.d $vr2, $vr2, 63 -; CONTRACT-FAST-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vbitrevi.d $vr0, $vr0, 63 +; CONTRACT-FAST-NEXT: vfmadd.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: not_vfnmsub_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 -; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 -; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 -; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr2, $vr0 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: not_vfnmsub_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 -; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 -; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 -; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr2, $vr0 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -342,28 +342,28 @@ entry: define void @contract_vfmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_vfmadd_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfmadd.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_vfmadd_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfmadd.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_vfmadd_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfmadd.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -379,28 +379,28 @@ entry: define void @contract_vfmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_vfmsub_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfmsub.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_vfmsub_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfmsub.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_vfmsub_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfmsub.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -416,28 +416,28 @@ entry: define void @contract_vfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_vfnmadd_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfnmadd.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_vfnmadd_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfnmadd.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_vfnmadd_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfnmadd.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -454,28 +454,28 @@ entry: define void @contract_vfnmadd_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_vfnmadd_d_nsz: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfnmadd.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_vfnmadd_d_nsz: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfnmadd.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_vfnmadd_d_nsz: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfnmadd.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -494,31 +494,31 @@ entry: define void @not_contract_vfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: not_contract_vfnmadd_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vbitrevi.d $vr2, $vr2, 63 -; CONTRACT-FAST-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vbitrevi.d $vr0, $vr0, 63 +; CONTRACT-FAST-NEXT: vfmsub.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: not_contract_vfnmadd_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vbitrevi.d $vr2, $vr2, 63 -; CONTRACT-ON-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vbitrevi.d $vr0, $vr0, 63 +; CONTRACT-ON-NEXT: vfmsub.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: not_contract_vfnmadd_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vbitrevi.d $vr2, $vr2, 63 -; CONTRACT-OFF-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vbitrevi.d $vr0, $vr0, 63 +; CONTRACT-OFF-NEXT: vfmsub.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -536,28 +536,28 @@ entry: define void @contract_vfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_vfnmsub_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfnmsub.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_vfnmsub_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfnmsub.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_vfnmsub_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfnmsub.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -575,28 +575,28 @@ entry: define void @contract_vfnmsub_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_vfnmsub_d_nsz: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfnmsub.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_vfnmsub_d_nsz: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfnmsub.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_vfnmsub_d_nsz: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfnmsub.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -614,31 +614,31 @@ entry: define void @not_contract_vfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: not_contract_vfnmsub_d: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vbitrevi.d $vr2, $vr2, 63 -; CONTRACT-FAST-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vbitrevi.d $vr0, $vr0, 63 +; CONTRACT-FAST-NEXT: vfmadd.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: not_contract_vfnmsub_d: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vbitrevi.d $vr2, $vr2, 63 -; CONTRACT-ON-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vbitrevi.d $vr0, $vr0, 63 +; CONTRACT-ON-NEXT: vfmadd.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: not_contract_vfnmsub_d: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vbitrevi.d $vr2, $vr2, 63 -; CONTRACT-OFF-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vbitrevi.d $vr0, $vr0, 63 +; CONTRACT-OFF-NEXT: vfmadd.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -655,28 +655,28 @@ entry: define void @vfmadd_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: vfmadd_d_contract: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfmadd.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: vfmadd_d_contract: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfmadd.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: vfmadd_d_contract: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfmadd.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -692,28 +692,28 @@ entry: define void @vfmsub_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: vfmsub_d_contract: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfmsub.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: vfmsub_d_contract: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfmsub.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: vfmsub_d_contract: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfmsub.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -729,28 +729,28 @@ entry: define void @vfnmadd_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: vfnmadd_d_contract: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfnmadd.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: vfnmadd_d_contract: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfnmadd.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: vfnmadd_d_contract: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfnmadd.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -767,28 +767,28 @@ entry: define void @vfnmsub_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: vfnmsub_d_contract: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfnmsub.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: vfnmsub_d_contract: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfnmsub.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: vfnmsub_d_contract: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfnmsub.d $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lsx/fma-v4f32.ll b/llvm/test/CodeGen/LoongArch/lsx/fma-v4f32.ll index 7efbd61c0c4f..1f316d5b1c8a 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/fma-v4f32.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/fma-v4f32.ll @@ -9,30 +9,30 @@ define void @vfmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: vfmadd_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfmadd.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: vfmadd_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 -; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 -; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 -; CONTRACT-ON-NEXT: vfadd.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vfadd.s $vr0, $vr0, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: vfmadd_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 -; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 -; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 -; CONTRACT-OFF-NEXT: vfadd.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vfadd.s $vr0, $vr0, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -48,30 +48,30 @@ entry: define void @vfmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: vfmsub_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfmsub.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: vfmsub_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 -; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 -; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 -; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr0, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: vfmsub_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 -; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 -; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 -; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr0, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -87,31 +87,31 @@ entry: define void @vfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: vfnmadd_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfnmadd.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: vfnmadd_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 -; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 -; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 -; CONTRACT-ON-NEXT: vfadd.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vfadd.s $vr0, $vr0, $vr2 ; CONTRACT-ON-NEXT: vbitrevi.w $vr0, $vr0, 31 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: vfnmadd_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 -; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 -; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 -; CONTRACT-OFF-NEXT: vfadd.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vfadd.s $vr0, $vr0, $vr2 ; CONTRACT-OFF-NEXT: vbitrevi.w $vr0, $vr0, 31 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret @@ -129,32 +129,32 @@ entry: define void @vfnmadd_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: vfnmadd_s_nsz: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfnmadd.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: vfnmadd_s_nsz: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 -; CONTRACT-ON-NEXT: vbitrevi.w $vr1, $vr1, 31 -; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 -; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 -; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vbitrevi.w $vr0, $vr0, 31 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr0, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: vfnmadd_s_nsz: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 -; CONTRACT-OFF-NEXT: vbitrevi.w $vr1, $vr1, 31 -; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 -; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 -; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vbitrevi.w $vr0, $vr0, 31 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr0, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -173,33 +173,33 @@ entry: define void @not_vfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: not_vfnmadd_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vbitrevi.w $vr2, $vr2, 31 -; CONTRACT-FAST-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vbitrevi.w $vr0, $vr0, 31 +; CONTRACT-FAST-NEXT: vfmsub.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: not_vfnmadd_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 -; CONTRACT-ON-NEXT: vbitrevi.w $vr1, $vr1, 31 -; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 -; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 -; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vbitrevi.w $vr0, $vr0, 31 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr0, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: not_vfnmadd_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 -; CONTRACT-OFF-NEXT: vbitrevi.w $vr1, $vr1, 31 -; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 -; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 -; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vbitrevi.w $vr0, $vr0, 31 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr0, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -217,31 +217,31 @@ entry: define void @vfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: vfnmsub_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfnmsub.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: vfnmsub_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 -; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 -; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 -; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr0, $vr2 ; CONTRACT-ON-NEXT: vbitrevi.w $vr0, $vr0, 31 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: vfnmsub_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 -; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 -; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 -; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr0, $vr2 ; CONTRACT-OFF-NEXT: vbitrevi.w $vr0, $vr0, 31 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret @@ -260,30 +260,30 @@ entry: define void @vfnmsub_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: vfnmsub_s_nsz: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfnmsub.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: vfnmsub_s_nsz: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 -; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 -; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 -; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr2, $vr0 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: vfnmsub_s_nsz: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 -; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 -; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 -; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr2, $vr0 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -301,31 +301,31 @@ entry: define void @not_vfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: not_vfnmsub_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vbitrevi.w $vr2, $vr2, 31 -; CONTRACT-FAST-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vbitrevi.w $vr0, $vr0, 31 +; CONTRACT-FAST-NEXT: vfmadd.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: not_vfnmsub_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 -; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 -; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 -; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr2, $vr0 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: not_vfnmsub_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 -; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 -; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 -; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr2, $vr0 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -342,28 +342,28 @@ entry: define void @contract_vfmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_vfmadd_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfmadd.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_vfmadd_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfmadd.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_vfmadd_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfmadd.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -379,28 +379,28 @@ entry: define void @contract_vfmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_vfmsub_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfmsub.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_vfmsub_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfmsub.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_vfmsub_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfmsub.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -416,28 +416,28 @@ entry: define void @contract_vfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_vfnmadd_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfnmadd.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_vfnmadd_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfnmadd.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_vfnmadd_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfnmadd.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -454,28 +454,28 @@ entry: define void @contract_vfnmadd_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_vfnmadd_s_nsz: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfnmadd.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_vfnmadd_s_nsz: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfnmadd.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_vfnmadd_s_nsz: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfnmadd.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -494,31 +494,31 @@ entry: define void @not_contract_vfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: not_contract_vfnmadd_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vbitrevi.w $vr2, $vr2, 31 -; CONTRACT-FAST-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vbitrevi.w $vr0, $vr0, 31 +; CONTRACT-FAST-NEXT: vfmsub.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: not_contract_vfnmadd_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vbitrevi.w $vr2, $vr2, 31 -; CONTRACT-ON-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vbitrevi.w $vr0, $vr0, 31 +; CONTRACT-ON-NEXT: vfmsub.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: not_contract_vfnmadd_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vbitrevi.w $vr2, $vr2, 31 -; CONTRACT-OFF-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vbitrevi.w $vr0, $vr0, 31 +; CONTRACT-OFF-NEXT: vfmsub.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -536,28 +536,28 @@ entry: define void @contract_vfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_vfnmsub_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfnmsub.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_vfnmsub_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfnmsub.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_vfnmsub_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfnmsub.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -575,28 +575,28 @@ entry: define void @contract_vfnmsub_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: contract_vfnmsub_s_nsz: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfnmsub.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: contract_vfnmsub_s_nsz: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfnmsub.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: contract_vfnmsub_s_nsz: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfnmsub.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -614,31 +614,31 @@ entry: define void @not_contract_vfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: not_contract_vfnmsub_s: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vbitrevi.w $vr2, $vr2, 31 -; CONTRACT-FAST-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vbitrevi.w $vr0, $vr0, 31 +; CONTRACT-FAST-NEXT: vfmadd.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: not_contract_vfnmsub_s: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vbitrevi.w $vr2, $vr2, 31 -; CONTRACT-ON-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vbitrevi.w $vr0, $vr0, 31 +; CONTRACT-ON-NEXT: vfmadd.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: not_contract_vfnmsub_s: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vbitrevi.w $vr2, $vr2, 31 -; CONTRACT-OFF-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vbitrevi.w $vr0, $vr0, 31 +; CONTRACT-OFF-NEXT: vfmadd.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -655,28 +655,28 @@ entry: define void @vfmadd_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: vfmadd_s_contract: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfmadd.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: vfmadd_s_contract: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfmadd.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: vfmadd_s_contract: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfmadd.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -692,28 +692,28 @@ entry: define void @vfmsub_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: vfmsub_s_contract: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfmsub.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: vfmsub_s_contract: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfmsub.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: vfmsub_s_contract: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfmsub.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -729,28 +729,28 @@ entry: define void @vfnmadd_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: vfnmadd_s_contract: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfnmadd.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: vfnmadd_s_contract: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfnmadd.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: vfnmadd_s_contract: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfnmadd.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: @@ -767,28 +767,28 @@ entry: define void @vfnmsub_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { ; CONTRACT-FAST-LABEL: vfnmsub_s_contract: ; CONTRACT-FAST: # %bb.0: # %entry -; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr0, $a1, 0 ; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 -; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 -; CONTRACT-FAST-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vld $vr2, $a3, 0 +; CONTRACT-FAST-NEXT: vfnmsub.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 ; CONTRACT-FAST-NEXT: ret ; ; CONTRACT-ON-LABEL: vfnmsub_s_contract: ; CONTRACT-ON: # %bb.0: # %entry -; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr0, $a1, 0 ; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 -; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 -; CONTRACT-ON-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr2, $a3, 0 +; CONTRACT-ON-NEXT: vfnmsub.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 ; CONTRACT-ON-NEXT: ret ; ; CONTRACT-OFF-LABEL: vfnmsub_s_contract: ; CONTRACT-OFF: # %bb.0: # %entry -; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr0, $a1, 0 ; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 -; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 -; CONTRACT-OFF-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr2, $a3, 0 +; CONTRACT-OFF-NEXT: vfnmsub.s $vr0, $vr0, $vr1, $vr2 ; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 ; CONTRACT-OFF-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/add.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/add.ll index 2a7c37c2ae34..485bd1df8d66 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/add.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/add.ll @@ -4,9 +4,9 @@ define void @add_v16i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: add_v16i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vadd.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vadd.b $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @add_v8i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: add_v8i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vadd.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vadd.h $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -36,9 +36,9 @@ entry: define void @add_v4i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: add_v4i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vadd.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vadd.w $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -52,9 +52,9 @@ entry: define void @add_v2i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: add_v2i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vadd.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vadd.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/and.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/and.ll index 523255159a81..d3e4efb1b1c2 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/and.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/and.ll @@ -4,9 +4,9 @@ define void @and_v16i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: and_v16i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vand.v $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vand.v $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @and_v8i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: and_v8i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vand.v $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vand.v $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -36,9 +36,9 @@ entry: define void @and_v4i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: and_v4i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vand.v $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vand.v $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -52,9 +52,9 @@ entry: define void @and_v2i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: and_v2i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vand.v $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vand.v $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/ashr.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/ashr.ll index fbc570d77ba8..2a3107447098 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/ashr.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/ashr.ll @@ -4,9 +4,9 @@ define void @ashr_v16i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: ashr_v16i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsra.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsra.b $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @ashr_v8i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: ashr_v8i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsra.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsra.h $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -36,9 +36,9 @@ entry: define void @ashr_v4i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: ashr_v4i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsra.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsra.w $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -52,9 +52,9 @@ entry: define void @ashr_v2i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: ashr_v2i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsra.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsra.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/extractelement.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/extractelement.ll index b8798c97861e..f3b8e02ac28f 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/extractelement.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/extractelement.ll @@ -82,9 +82,9 @@ define void @extract_2xdouble(ptr %src, ptr %dst) nounwind { define void @extract_16xi8_idx(ptr %src, ptr %dst, i32 %idx) nounwind { ; CHECK-LABEL: extract_16xi8_idx: ; CHECK: # %bb.0: -; CHECK-NEXT: bstrpick.d $a2, $a2, 31, 0 ; CHECK-NEXT: vld $vr0, $a0, 0 -; CHECK-NEXT: vreplve.b $vr0, $vr0, $a2 +; CHECK-NEXT: bstrpick.d $a0, $a2, 31, 0 +; CHECK-NEXT: vreplve.b $vr0, $vr0, $a0 ; CHECK-NEXT: movfr2gr.s $a0, $fa0 ; CHECK-NEXT: srai.w $a0, $a0, 24 ; CHECK-NEXT: st.b $a0, $a1, 0 @@ -98,9 +98,9 @@ define void @extract_16xi8_idx(ptr %src, ptr %dst, i32 %idx) nounwind { define void @extract_8xi16_idx(ptr %src, ptr %dst, i32 %idx) nounwind { ; CHECK-LABEL: extract_8xi16_idx: ; CHECK: # %bb.0: -; CHECK-NEXT: bstrpick.d $a2, $a2, 31, 0 ; CHECK-NEXT: vld $vr0, $a0, 0 -; CHECK-NEXT: vreplve.h $vr0, $vr0, $a2 +; CHECK-NEXT: bstrpick.d $a0, $a2, 31, 0 +; CHECK-NEXT: vreplve.h $vr0, $vr0, $a0 ; CHECK-NEXT: movfr2gr.s $a0, $fa0 ; CHECK-NEXT: srai.w $a0, $a0, 16 ; CHECK-NEXT: st.h $a0, $a1, 0 @@ -114,9 +114,9 @@ define void @extract_8xi16_idx(ptr %src, ptr %dst, i32 %idx) nounwind { define void @extract_4xi32_idx(ptr %src, ptr %dst, i32 %idx) nounwind { ; CHECK-LABEL: extract_4xi32_idx: ; CHECK: # %bb.0: -; CHECK-NEXT: bstrpick.d $a2, $a2, 31, 0 ; CHECK-NEXT: vld $vr0, $a0, 0 -; CHECK-NEXT: vreplve.w $vr0, $vr0, $a2 +; CHECK-NEXT: bstrpick.d $a0, $a2, 31, 0 +; CHECK-NEXT: vreplve.w $vr0, $vr0, $a0 ; CHECK-NEXT: movfr2gr.s $a0, $fa0 ; CHECK-NEXT: st.w $a0, $a1, 0 ; CHECK-NEXT: ret @@ -129,9 +129,9 @@ define void @extract_4xi32_idx(ptr %src, ptr %dst, i32 %idx) nounwind { define void @extract_2xi64_idx(ptr %src, ptr %dst, i32 %idx) nounwind { ; CHECK-LABEL: extract_2xi64_idx: ; CHECK: # %bb.0: -; CHECK-NEXT: bstrpick.d $a2, $a2, 31, 0 ; CHECK-NEXT: vld $vr0, $a0, 0 -; CHECK-NEXT: vreplve.d $vr0, $vr0, $a2 +; CHECK-NEXT: bstrpick.d $a0, $a2, 31, 0 +; CHECK-NEXT: vreplve.d $vr0, $vr0, $a0 ; CHECK-NEXT: movfr2gr.d $a0, $fa0 ; CHECK-NEXT: st.d $a0, $a1, 0 ; CHECK-NEXT: ret @@ -144,9 +144,9 @@ define void @extract_2xi64_idx(ptr %src, ptr %dst, i32 %idx) nounwind { define void @extract_4xfloat_idx(ptr %src, ptr %dst, i32 %idx) nounwind { ; CHECK-LABEL: extract_4xfloat_idx: ; CHECK: # %bb.0: -; CHECK-NEXT: bstrpick.d $a2, $a2, 31, 0 ; CHECK-NEXT: vld $vr0, $a0, 0 -; CHECK-NEXT: vreplve.w $vr0, $vr0, $a2 +; CHECK-NEXT: bstrpick.d $a0, $a2, 31, 0 +; CHECK-NEXT: vreplve.w $vr0, $vr0, $a0 ; CHECK-NEXT: fst.s $fa0, $a1, 0 ; CHECK-NEXT: ret %v = load volatile <4 x float>, ptr %src @@ -158,9 +158,9 @@ define void @extract_4xfloat_idx(ptr %src, ptr %dst, i32 %idx) nounwind { define void @extract_2xdouble_idx(ptr %src, ptr %dst, i32 %idx) nounwind { ; CHECK-LABEL: extract_2xdouble_idx: ; CHECK: # %bb.0: -; CHECK-NEXT: bstrpick.d $a2, $a2, 31, 0 ; CHECK-NEXT: vld $vr0, $a0, 0 -; CHECK-NEXT: vreplve.d $vr0, $vr0, $a2 +; CHECK-NEXT: bstrpick.d $a0, $a2, 31, 0 +; CHECK-NEXT: vreplve.d $vr0, $vr0, $a0 ; CHECK-NEXT: fst.d $fa0, $a1, 0 ; CHECK-NEXT: ret %v = load volatile <2 x double>, ptr %src diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fadd.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fadd.ll index 1fa1f611c4a3..989ad10a44ff 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fadd.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fadd.ll @@ -4,9 +4,9 @@ define void @fadd_v4f32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: fadd_v4f32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfadd.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfadd.s $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @fadd_v2f64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: fadd_v2f64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfadd.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfadd.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fcmp.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fcmp.ll index 53fbf0b2f86f..95e46a4e71da 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fcmp.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fcmp.ll @@ -35,9 +35,9 @@ define void @v2f64_fcmp_false(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f32_fcmp_oeq(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f32_fcmp_oeq: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.ceq.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.ceq.s $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x float>, ptr %a0 @@ -51,9 +51,9 @@ define void @v4f32_fcmp_oeq(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v2f64_fcmp_oeq(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v2f64_fcmp_oeq: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.ceq.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.ceq.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <2 x double>, ptr %a0 @@ -68,9 +68,9 @@ define void @v2f64_fcmp_oeq(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f32_fcmp_ueq(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f32_fcmp_ueq: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.cueq.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cueq.s $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x float>, ptr %a0 @@ -84,9 +84,9 @@ define void @v4f32_fcmp_ueq(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v2f64_fcmp_ueq(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v2f64_fcmp_ueq: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.cueq.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cueq.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <2 x double>, ptr %a0 @@ -101,9 +101,9 @@ define void @v2f64_fcmp_ueq(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f32_fcmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f32_fcmp_eq: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.ceq.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.ceq.s $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x float>, ptr %a0 @@ -117,9 +117,9 @@ define void @v4f32_fcmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v2f64_fcmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v2f64_fcmp_eq: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.ceq.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.ceq.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <2 x double>, ptr %a0 @@ -134,9 +134,9 @@ define void @v2f64_fcmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f32_fcmp_ole(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f32_fcmp_ole: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.cle.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cle.s $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x float>, ptr %a0 @@ -150,9 +150,9 @@ define void @v4f32_fcmp_ole(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v2f64_fcmp_ole(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v2f64_fcmp_ole: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.cle.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cle.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <2 x double>, ptr %a0 @@ -167,9 +167,9 @@ define void @v2f64_fcmp_ole(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f32_fcmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f32_fcmp_ule: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.cule.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cule.s $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x float>, ptr %a0 @@ -183,9 +183,9 @@ define void @v4f32_fcmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v2f64_fcmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v2f64_fcmp_ule: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.cule.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cule.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <2 x double>, ptr %a0 @@ -200,9 +200,9 @@ define void @v2f64_fcmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f32_fcmp_le(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f32_fcmp_le: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.cle.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cle.s $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x float>, ptr %a0 @@ -216,9 +216,9 @@ define void @v4f32_fcmp_le(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v2f64_fcmp_le(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v2f64_fcmp_le: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.cle.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cle.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <2 x double>, ptr %a0 @@ -233,9 +233,9 @@ define void @v2f64_fcmp_le(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f32_fcmp_olt(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f32_fcmp_olt: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.clt.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.clt.s $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x float>, ptr %a0 @@ -249,9 +249,9 @@ define void @v4f32_fcmp_olt(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v2f64_fcmp_olt(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v2f64_fcmp_olt: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.clt.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.clt.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <2 x double>, ptr %a0 @@ -266,9 +266,9 @@ define void @v2f64_fcmp_olt(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f32_fcmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f32_fcmp_ult: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.cult.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cult.s $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x float>, ptr %a0 @@ -282,9 +282,9 @@ define void @v4f32_fcmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v2f64_fcmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v2f64_fcmp_ult: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.cult.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cult.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <2 x double>, ptr %a0 @@ -299,9 +299,9 @@ define void @v2f64_fcmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f32_fcmp_lt(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f32_fcmp_lt: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.clt.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.clt.s $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x float>, ptr %a0 @@ -315,9 +315,9 @@ define void @v4f32_fcmp_lt(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v2f64_fcmp_lt(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v2f64_fcmp_lt: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.clt.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.clt.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <2 x double>, ptr %a0 @@ -332,9 +332,9 @@ define void @v2f64_fcmp_lt(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f32_fcmp_one(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f32_fcmp_one: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.cne.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cne.s $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x float>, ptr %a0 @@ -348,9 +348,9 @@ define void @v4f32_fcmp_one(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v2f64_fcmp_one(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v2f64_fcmp_one: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.cne.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cne.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <2 x double>, ptr %a0 @@ -365,9 +365,9 @@ define void @v2f64_fcmp_one(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f32_fcmp_une(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f32_fcmp_une: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.cune.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cune.s $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x float>, ptr %a0 @@ -381,9 +381,9 @@ define void @v4f32_fcmp_une(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v2f64_fcmp_une(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v2f64_fcmp_une: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.cune.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cune.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <2 x double>, ptr %a0 @@ -398,9 +398,9 @@ define void @v2f64_fcmp_une(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f32_fcmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f32_fcmp_ne: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.cne.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cne.s $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x float>, ptr %a0 @@ -414,9 +414,9 @@ define void @v4f32_fcmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v2f64_fcmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v2f64_fcmp_ne: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.cne.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cne.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <2 x double>, ptr %a0 @@ -431,9 +431,9 @@ define void @v2f64_fcmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f32_fcmp_ord(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f32_fcmp_ord: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.cor.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cor.s $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x float>, ptr %a0 @@ -447,9 +447,9 @@ define void @v4f32_fcmp_ord(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v2f64_fcmp_ord(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v2f64_fcmp_ord: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.cor.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cor.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <2 x double>, ptr %a0 @@ -464,9 +464,9 @@ define void @v2f64_fcmp_ord(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4f32_fcmp_uno(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4f32_fcmp_uno: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.cun.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cun.s $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x float>, ptr %a0 @@ -480,9 +480,9 @@ define void @v4f32_fcmp_uno(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v2f64_fcmp_uno(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v2f64_fcmp_uno: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfcmp.cun.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cun.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <2 x double>, ptr %a0 diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fdiv.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fdiv.ll index 5f1ee9e4d212..3b9642e31b02 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fdiv.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fdiv.ll @@ -4,9 +4,9 @@ define void @fdiv_v4f32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: fdiv_v4f32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfdiv.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfdiv.s $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @fdiv_v2f64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: fdiv_v2f64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfdiv.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfdiv.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fmul.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fmul.ll index e7fb527f7805..f604a8962958 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fmul.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fmul.ll @@ -4,9 +4,9 @@ define void @fmul_v4f32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: fmul_v4f32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfmul.s $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @fmul_v2f64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: fmul_v2f64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfmul.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fsub.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fsub.ll index df98182321da..02350c0763ba 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fsub.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fsub.ll @@ -4,9 +4,9 @@ define void @fsub_v4f32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: fsub_v4f32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfsub.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfsub.s $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @fsub_v2f64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: fsub_v2f64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vfsub.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfsub.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/icmp.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/icmp.ll index 448f3fa6c6e0..04b4831f1188 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/icmp.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/icmp.ll @@ -19,9 +19,9 @@ define void @v16i8_icmp_eq_imm(ptr %res, ptr %a0) nounwind { define void @v16i8_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v16i8_icmp_eq: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vseq.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vseq.b $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <16 x i8>, ptr %a0 @@ -49,9 +49,9 @@ define void @v8i16_icmp_eq_imm(ptr %res, ptr %a0) nounwind { define void @v8i16_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8i16_icmp_eq: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vseq.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vseq.h $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x i16>, ptr %a0 @@ -79,9 +79,9 @@ define void @v4i32_icmp_eq_imm(ptr %res, ptr %a0) nounwind { define void @v4i32_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4i32_icmp_eq: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vseq.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vseq.w $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x i32>, ptr %a0 @@ -109,9 +109,9 @@ define void @v2i64_icmp_eq_imm(ptr %res, ptr %a0) nounwind { define void @v2i64_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v2i64_icmp_eq: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vseq.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vseq.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <2 x i64>, ptr %a0 @@ -140,9 +140,9 @@ define void @v16i8_icmp_sle_imm(ptr %res, ptr %a0) nounwind { define void @v16i8_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v16i8_icmp_sle: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsle.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.b $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <16 x i8>, ptr %a0 @@ -170,9 +170,9 @@ define void @v8i16_icmp_sle_imm(ptr %res, ptr %a0) nounwind { define void @v8i16_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8i16_icmp_sle: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsle.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.h $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x i16>, ptr %a0 @@ -200,9 +200,9 @@ define void @v4i32_icmp_sle_imm(ptr %res, ptr %a0) nounwind { define void @v4i32_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4i32_icmp_sle: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsle.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.w $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x i32>, ptr %a0 @@ -230,9 +230,9 @@ define void @v2i64_icmp_sle_imm(ptr %res, ptr %a0) nounwind { define void @v2i64_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v2i64_icmp_sle: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsle.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <2 x i64>, ptr %a0 @@ -261,9 +261,9 @@ define void @v16i8_icmp_ule_imm(ptr %res, ptr %a0) nounwind { define void @v16i8_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v16i8_icmp_ule: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsle.bu $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.bu $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <16 x i8>, ptr %a0 @@ -291,9 +291,9 @@ define void @v8i16_icmp_ule_imm(ptr %res, ptr %a0) nounwind { define void @v8i16_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8i16_icmp_ule: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsle.hu $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.hu $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x i16>, ptr %a0 @@ -321,9 +321,9 @@ define void @v4i32_icmp_ule_imm(ptr %res, ptr %a0) nounwind { define void @v4i32_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4i32_icmp_ule: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsle.wu $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.wu $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x i32>, ptr %a0 @@ -351,9 +351,9 @@ define void @v2i64_icmp_ule_imm(ptr %res, ptr %a0) nounwind { define void @v2i64_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v2i64_icmp_ule: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsle.du $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.du $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <2 x i64>, ptr %a0 @@ -382,9 +382,9 @@ define void @v16i8_icmp_slt_imm(ptr %res, ptr %a0) nounwind { define void @v16i8_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v16i8_icmp_slt: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vslt.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.b $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <16 x i8>, ptr %a0 @@ -412,9 +412,9 @@ define void @v8i16_icmp_slt_imm(ptr %res, ptr %a0) nounwind { define void @v8i16_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8i16_icmp_slt: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vslt.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.h $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x i16>, ptr %a0 @@ -442,9 +442,9 @@ define void @v4i32_icmp_slt_imm(ptr %res, ptr %a0) nounwind { define void @v4i32_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4i32_icmp_slt: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vslt.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.w $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x i32>, ptr %a0 @@ -472,9 +472,9 @@ define void @v2i64_icmp_slt_imm(ptr %res, ptr %a0) nounwind { define void @v2i64_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v2i64_icmp_slt: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vslt.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <2 x i64>, ptr %a0 @@ -503,9 +503,9 @@ define void @v16i8_icmp_ult_imm(ptr %res, ptr %a0) nounwind { define void @v16i8_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v16i8_icmp_ult: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vslt.bu $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.bu $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <16 x i8>, ptr %a0 @@ -533,9 +533,9 @@ define void @v8i16_icmp_ult_imm(ptr %res, ptr %a0) nounwind { define void @v8i16_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8i16_icmp_ult: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vslt.hu $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.hu $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x i16>, ptr %a0 @@ -563,9 +563,9 @@ define void @v4i32_icmp_ult_imm(ptr %res, ptr %a0) nounwind { define void @v4i32_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4i32_icmp_ult: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vslt.wu $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.wu $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <4 x i32>, ptr %a0 @@ -593,9 +593,9 @@ define void @v2i64_icmp_ult_imm(ptr %res, ptr %a0) nounwind { define void @v2i64_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v2i64_icmp_ult: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vslt.du $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.du $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <2 x i64>, ptr %a0 @@ -610,9 +610,9 @@ define void @v2i64_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v16i8_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v16i8_icmp_ne: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vseq.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vseq.b $vr0, $vr0, $vr1 ; CHECK-NEXT: vxori.b $vr0, $vr0, 255 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret @@ -627,9 +627,9 @@ define void @v16i8_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v8i16_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v8i16_icmp_ne: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vseq.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vseq.h $vr0, $vr0, $vr1 ; CHECK-NEXT: vrepli.b $vr1, -1 ; CHECK-NEXT: vxor.v $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 @@ -645,9 +645,9 @@ define void @v8i16_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v4i32_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v4i32_icmp_ne: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vseq.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vseq.w $vr0, $vr0, $vr1 ; CHECK-NEXT: vrepli.b $vr1, -1 ; CHECK-NEXT: vxor.v $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 @@ -663,9 +663,9 @@ define void @v4i32_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { define void @v2i64_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: v2i64_icmp_ne: ; CHECK: # %bb.0: -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vseq.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vseq.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vrepli.b $vr1, -1 ; CHECK-NEXT: vxor.v $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insertelement.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insertelement.ll index a9834591aa0e..7f232073ae12 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insertelement.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insertelement.ll @@ -56,10 +56,10 @@ define void @insert_2xi64(ptr %src, ptr %dst, i64 %ins) nounwind { define void @insert_4xfloat(ptr %src, ptr %dst, float %ins) nounwind { ; CHECK-LABEL: insert_4xfloat: ; CHECK: # %bb.0: -; CHECK-NEXT: movfr2gr.s $a2, $fa0 -; CHECK-NEXT: vld $vr0, $a0, 0 -; CHECK-NEXT: vinsgr2vr.w $vr0, $a2, 1 -; CHECK-NEXT: vst $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a0, 0 +; CHECK-NEXT: movfr2gr.s $a0, $fa0 +; CHECK-NEXT: vinsgr2vr.w $vr1, $a0, 1 +; CHECK-NEXT: vst $vr1, $a1, 0 ; CHECK-NEXT: ret %v = load volatile <4 x float>, ptr %src %v_new = insertelement <4 x float> %v, float %ins, i32 1 @@ -70,10 +70,10 @@ define void @insert_4xfloat(ptr %src, ptr %dst, float %ins) nounwind { define void @insert_2xdouble(ptr %src, ptr %dst, double %ins) nounwind { ; CHECK-LABEL: insert_2xdouble: ; CHECK: # %bb.0: -; CHECK-NEXT: movfr2gr.d $a2, $fa0 -; CHECK-NEXT: vld $vr0, $a0, 0 -; CHECK-NEXT: vinsgr2vr.d $vr0, $a2, 1 -; CHECK-NEXT: vst $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a0, 0 +; CHECK-NEXT: movfr2gr.d $a0, $fa0 +; CHECK-NEXT: vinsgr2vr.d $vr1, $a0, 1 +; CHECK-NEXT: vst $vr1, $a1, 0 ; CHECK-NEXT: ret %v = load volatile <2 x double>, ptr %src %v_new = insertelement <2 x double> %v, double %ins, i32 1 diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/lshr.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/lshr.ll index dada52f93060..2693310b4f50 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/lshr.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/lshr.ll @@ -4,9 +4,9 @@ define void @lshr_v16i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: lshr_v16i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsrl.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsrl.b $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @lshr_v8i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: lshr_v8i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsrl.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsrl.h $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -36,9 +36,9 @@ entry: define void @lshr_v4i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: lshr_v4i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsrl.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsrl.w $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -52,9 +52,9 @@ entry: define void @lshr_v2i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: lshr_v2i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsrl.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsrl.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/mul.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/mul.ll index d0be9cb7e3c8..f66cae6a1802 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/mul.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/mul.ll @@ -4,9 +4,9 @@ define void @mul_v16i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mul_v16i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vmul.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vmul.b $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @mul_v8i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mul_v8i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vmul.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vmul.h $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -36,9 +36,9 @@ entry: define void @mul_v4i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mul_v4i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vmul.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vmul.w $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -52,9 +52,9 @@ entry: define void @mul_v2i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mul_v2i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vmul.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vmul.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/or.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/or.ll index f124512acce7..89702e60c01f 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/or.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/or.ll @@ -4,9 +4,9 @@ define void @or_v16i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: or_v16i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vor.v $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vor.v $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @or_v8i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: or_v8i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vor.v $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vor.v $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -36,9 +36,9 @@ entry: define void @or_v4i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: or_v4i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vor.v $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vor.v $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -52,9 +52,9 @@ entry: define void @or_v2i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: or_v2i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vor.v $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vor.v $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/sdiv.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/sdiv.ll index b68f73a74913..cdff58defdae 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/sdiv.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/sdiv.ll @@ -4,9 +4,9 @@ define void @sdiv_v16i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: sdiv_v16i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vdiv.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vdiv.b $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @sdiv_v8i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: sdiv_v8i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vdiv.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vdiv.h $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -36,9 +36,9 @@ entry: define void @sdiv_v4i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: sdiv_v4i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vdiv.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vdiv.w $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -52,9 +52,9 @@ entry: define void @sdiv_v2i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: sdiv_v2i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vdiv.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vdiv.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/shl.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/shl.ll index fa0aebaf28b3..4b34c04f3374 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/shl.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/shl.ll @@ -4,9 +4,9 @@ define void @shl_v16i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: shl_v16i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsll.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsll.b $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @shl_v8i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: shl_v8i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsll.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsll.h $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -36,9 +36,9 @@ entry: define void @shl_v4i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: shl_v4i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsll.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsll.w $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -52,9 +52,9 @@ entry: define void @shl_v2i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: shl_v2i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsll.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsll.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/sub.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/sub.ll index 25b4623a47d1..2813d9c97e68 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/sub.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/sub.ll @@ -4,9 +4,9 @@ define void @sub_v16i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: sub_v16i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsub.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsub.b $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @sub_v8i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: sub_v8i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsub.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsub.h $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -36,9 +36,9 @@ entry: define void @sub_v4i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: sub_v4i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsub.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsub.w $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -52,9 +52,9 @@ entry: define void @sub_v2i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: sub_v2i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vsub.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsub.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/udiv.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/udiv.ll index abb60b91dd48..32dac67d36a8 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/udiv.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/udiv.ll @@ -4,9 +4,9 @@ define void @udiv_v16i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: udiv_v16i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vdiv.bu $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vdiv.bu $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @udiv_v8i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: udiv_v8i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vdiv.hu $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vdiv.hu $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -36,9 +36,9 @@ entry: define void @udiv_v4i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: udiv_v4i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vdiv.wu $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vdiv.wu $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -52,9 +52,9 @@ entry: define void @udiv_v2i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: udiv_v2i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vdiv.du $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vdiv.du $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/xor.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/xor.ll index ce3e49c990ff..482cecb1d752 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/xor.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/xor.ll @@ -4,9 +4,9 @@ define void @xor_v16i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: xor_v16i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vxor.v $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vxor.v $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -20,9 +20,9 @@ entry: define void @xor_v8i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: xor_v8i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vxor.v $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vxor.v $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -36,9 +36,9 @@ entry: define void @xor_v4i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: xor_v4i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vxor.v $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vxor.v $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -52,9 +52,9 @@ entry: define void @xor_v2i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: xor_v2i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vxor.v $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vxor.v $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lsx/mulh.ll b/llvm/test/CodeGen/LoongArch/lsx/mulh.ll index e1388f00e355..b0ca556eeff3 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/mulh.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/mulh.ll @@ -4,9 +4,9 @@ define void @mulhs_v16i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mulhs_v16i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vmuh.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vmuh.b $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -24,9 +24,9 @@ entry: define void @mulhu_v16i8(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mulhu_v16i8: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vmuh.bu $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vmuh.bu $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -44,9 +44,9 @@ entry: define void @mulhs_v8i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mulhs_v8i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vmuh.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vmuh.h $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -64,9 +64,9 @@ entry: define void @mulhu_v8i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mulhu_v8i16: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vmuh.hu $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vmuh.hu $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -84,9 +84,9 @@ entry: define void @mulhs_v4i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mulhs_v4i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vmuh.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vmuh.w $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -104,9 +104,9 @@ entry: define void @mulhu_v4i32(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mulhu_v4i32: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vmuh.wu $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vmuh.wu $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -124,9 +124,9 @@ entry: define void @mulhs_v2i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mulhs_v2i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vmuh.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vmuh.d $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: @@ -144,9 +144,9 @@ entry: define void @mulhu_v2i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: mulhu_v2i64: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vld $vr0, $a2, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vmuh.du $vr0, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vmuh.du $vr0, $vr0, $vr1 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/LoongArch/lsx/vselect.ll b/llvm/test/CodeGen/LoongArch/lsx/vselect.ll index 97a55532907b..48ef3c14a4bf 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/vselect.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/vselect.ll @@ -34,11 +34,11 @@ define void @select_v16i8(ptr %res, ptr %a0, ptr %a1) nounwind { define void @select_v8i16(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: select_v8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: lu12i.w $a3, -16 -; CHECK-NEXT: vreplgr2vr.w $vr0, $a3 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vld $vr2, $a2, 0 -; CHECK-NEXT: vbitsel.v $vr0, $vr2, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: lu12i.w $a1, -16 +; CHECK-NEXT: vreplgr2vr.w $vr2, $a1 +; CHECK-NEXT: vbitsel.v $vr0, $vr1, $vr0, $vr2 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <8 x i16>, ptr %a0 @@ -69,12 +69,12 @@ define void @select_v4i32(ptr %res, ptr %a0, ptr %a1) nounwind { define void @select_v2i64(ptr %res, ptr %a0, ptr %a1) nounwind { ; CHECK-LABEL: select_v2i64: ; CHECK: # %bb.0: -; CHECK-NEXT: pcalau12i $a3, %pc_hi20(.LCPI4_0) -; CHECK-NEXT: addi.d $a3, $a3, %pc_lo12(.LCPI4_0) -; CHECK-NEXT: vld $vr0, $a3, 0 -; CHECK-NEXT: vld $vr1, $a1, 0 -; CHECK-NEXT: vld $vr2, $a2, 0 -; CHECK-NEXT: vbitsel.v $vr0, $vr2, $vr1, $vr0 +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: pcalau12i $a1, %pc_hi20(.LCPI4_0) +; CHECK-NEXT: addi.d $a1, $a1, %pc_lo12(.LCPI4_0) +; CHECK-NEXT: vld $vr2, $a1, 0 +; CHECK-NEXT: vbitsel.v $vr0, $vr1, $vr0, $vr2 ; CHECK-NEXT: vst $vr0, $a0, 0 ; CHECK-NEXT: ret %v0 = load <2 x i64>, ptr %a0 diff --git a/llvm/test/CodeGen/LoongArch/preferred-alignments.ll b/llvm/test/CodeGen/LoongArch/preferred-alignments.ll index 6dac525874ec..2b6a109228a6 100644 --- a/llvm/test/CodeGen/LoongArch/preferred-alignments.ll +++ b/llvm/test/CodeGen/LoongArch/preferred-alignments.ll @@ -8,21 +8,21 @@ define signext i32 @sum(ptr noalias nocapture noundef readonly %0, i32 noundef s ; LA464-NEXT: ori $a2, $zero, 1 ; LA464-NEXT: blt $a1, $a2, .LBB0_4 ; LA464-NEXT: # %bb.1: -; LA464-NEXT: bstrpick.d $a2, $a1, 31, 0 -; LA464-NEXT: move $a1, $zero +; LA464-NEXT: move $a2, $zero +; LA464-NEXT: bstrpick.d $a1, $a1, 31, 0 ; LA464-NEXT: .p2align 4, , 16 ; LA464-NEXT: .LBB0_2: # =>This Inner Loop Header: Depth=1 ; LA464-NEXT: ld.w $a3, $a0, 0 -; LA464-NEXT: add.d $a1, $a3, $a1 +; LA464-NEXT: add.d $a2, $a3, $a2 +; LA464-NEXT: addi.d $a1, $a1, -1 ; LA464-NEXT: addi.d $a0, $a0, 4 -; LA464-NEXT: addi.d $a2, $a2, -1 -; LA464-NEXT: bnez $a2, .LBB0_2 +; LA464-NEXT: bnez $a1, .LBB0_2 ; LA464-NEXT: # %bb.3: -; LA464-NEXT: addi.w $a0, $a1, 0 +; LA464-NEXT: addi.w $a0, $a2, 0 ; LA464-NEXT: ret ; LA464-NEXT: .LBB0_4: -; LA464-NEXT: move $a1, $zero -; LA464-NEXT: addi.w $a0, $a1, 0 +; LA464-NEXT: move $a2, $zero +; LA464-NEXT: addi.w $a0, $a2, 0 ; LA464-NEXT: ret %3 = icmp sgt i32 %1, 0 br i1 %3, label %4, label %6 diff --git a/llvm/test/CodeGen/LoongArch/rotl-rotr.ll b/llvm/test/CodeGen/LoongArch/rotl-rotr.ll index b9a6ebdcdd22..b067eb9cfa92 100644 --- a/llvm/test/CodeGen/LoongArch/rotl-rotr.ll +++ b/llvm/test/CodeGen/LoongArch/rotl-rotr.ll @@ -14,10 +14,10 @@ define i32 @rotl_32(i32 %x, i32 %y) nounwind { ; ; LA64-LABEL: rotl_32: ; LA64: # %bb.0: -; LA64-NEXT: sub.d $a2, $zero, $a1 -; LA64-NEXT: sll.w $a1, $a0, $a1 -; LA64-NEXT: srl.w $a0, $a0, $a2 -; LA64-NEXT: or $a0, $a1, $a0 +; LA64-NEXT: sll.w $a2, $a0, $a1 +; LA64-NEXT: sub.d $a1, $zero, $a1 +; LA64-NEXT: srl.w $a0, $a0, $a1 +; LA64-NEXT: or $a0, $a2, $a0 ; LA64-NEXT: ret %z = sub i32 32, %y %b = shl i32 %x, %y @@ -46,39 +46,39 @@ define i32 @rotr_32(i32 %x, i32 %y) nounwind { define i64 @rotl_64(i64 %x, i64 %y) nounwind { ; LA32-LABEL: rotl_64: ; LA32: # %bb.0: -; LA32-NEXT: xori $a3, $a2, 31 -; LA32-NEXT: srli.w $a4, $a0, 1 -; LA32-NEXT: srl.w $a3, $a4, $a3 -; LA32-NEXT: sll.w $a4, $a1, $a2 -; LA32-NEXT: or $a3, $a4, $a3 +; LA32-NEXT: sll.w $a3, $a1, $a2 +; LA32-NEXT: xori $a4, $a2, 31 +; LA32-NEXT: srli.w $a5, $a0, 1 +; LA32-NEXT: srl.w $a4, $a5, $a4 +; LA32-NEXT: or $a3, $a3, $a4 ; LA32-NEXT: addi.w $a4, $a2, -32 ; LA32-NEXT: slti $a5, $a4, 0 ; LA32-NEXT: maskeqz $a3, $a3, $a5 ; LA32-NEXT: sll.w $a6, $a0, $a4 ; LA32-NEXT: masknez $a5, $a6, $a5 ; LA32-NEXT: or $a3, $a3, $a5 +; LA32-NEXT: sll.w $a5, $a0, $a2 +; LA32-NEXT: srai.w $a4, $a4, 31 +; LA32-NEXT: and $a4, $a4, $a5 +; LA32-NEXT: sub.w $a5, $zero, $a2 +; LA32-NEXT: srl.w $a6, $a1, $a5 +; LA32-NEXT: ori $a7, $zero, 32 +; LA32-NEXT: sub.w $a7, $a7, $a2 +; LA32-NEXT: slti $t0, $a7, 0 +; LA32-NEXT: masknez $t1, $a6, $t0 +; LA32-NEXT: srl.w $a0, $a0, $a5 ; LA32-NEXT: ori $a5, $zero, 64 -; LA32-NEXT: sub.w $a5, $a5, $a2 -; LA32-NEXT: xori $a5, $a5, 31 -; LA32-NEXT: slli.w $a6, $a1, 1 -; LA32-NEXT: sll.w $a5, $a6, $a5 -; LA32-NEXT: sub.w $a6, $zero, $a2 -; LA32-NEXT: srl.w $a7, $a1, $a6 -; LA32-NEXT: ori $a1, $zero, 32 -; LA32-NEXT: sub.w $t0, $a1, $a2 -; LA32-NEXT: srai.w $a1, $t0, 31 -; LA32-NEXT: and $a1, $a1, $a7 +; LA32-NEXT: sub.w $a2, $a5, $a2 +; LA32-NEXT: xori $a2, $a2, 31 +; LA32-NEXT: slli.w $a1, $a1, 1 +; LA32-NEXT: sll.w $a1, $a1, $a2 +; LA32-NEXT: or $a0, $a0, $a1 +; LA32-NEXT: maskeqz $a0, $a0, $t0 +; LA32-NEXT: or $a0, $a0, $t1 +; LA32-NEXT: srai.w $a1, $a7, 31 +; LA32-NEXT: and $a1, $a1, $a6 ; LA32-NEXT: or $a1, $a3, $a1 -; LA32-NEXT: srl.w $a3, $a0, $a6 -; LA32-NEXT: or $a3, $a3, $a5 -; LA32-NEXT: slti $a5, $t0, 0 -; LA32-NEXT: masknez $a6, $a7, $a5 -; LA32-NEXT: maskeqz $a3, $a3, $a5 -; LA32-NEXT: or $a3, $a3, $a6 -; LA32-NEXT: sll.w $a0, $a0, $a2 -; LA32-NEXT: srai.w $a2, $a4, 31 -; LA32-NEXT: and $a0, $a2, $a0 -; LA32-NEXT: or $a0, $a0, $a3 +; LA32-NEXT: or $a0, $a4, $a0 ; LA32-NEXT: ret ; ; LA64-LABEL: rotl_64: @@ -97,39 +97,39 @@ define i64 @rotl_64(i64 %x, i64 %y) nounwind { define i64 @rotr_64(i64 %x, i64 %y) nounwind { ; LA32-LABEL: rotr_64: ; LA32: # %bb.0: -; LA32-NEXT: xori $a3, $a2, 31 -; LA32-NEXT: slli.w $a4, $a1, 1 -; LA32-NEXT: sll.w $a3, $a4, $a3 -; LA32-NEXT: srl.w $a4, $a0, $a2 -; LA32-NEXT: or $a3, $a4, $a3 +; LA32-NEXT: srl.w $a3, $a0, $a2 +; LA32-NEXT: xori $a4, $a2, 31 +; LA32-NEXT: slli.w $a5, $a1, 1 +; LA32-NEXT: sll.w $a4, $a5, $a4 +; LA32-NEXT: or $a3, $a3, $a4 ; LA32-NEXT: addi.w $a4, $a2, -32 ; LA32-NEXT: slti $a5, $a4, 0 ; LA32-NEXT: maskeqz $a3, $a3, $a5 ; LA32-NEXT: srl.w $a6, $a1, $a4 ; LA32-NEXT: masknez $a5, $a6, $a5 ; LA32-NEXT: or $a3, $a3, $a5 +; LA32-NEXT: srl.w $a5, $a1, $a2 +; LA32-NEXT: srai.w $a4, $a4, 31 +; LA32-NEXT: and $a4, $a4, $a5 +; LA32-NEXT: sub.w $a5, $zero, $a2 +; LA32-NEXT: sll.w $a6, $a0, $a5 +; LA32-NEXT: ori $a7, $zero, 32 +; LA32-NEXT: sub.w $a7, $a7, $a2 +; LA32-NEXT: slti $t0, $a7, 0 +; LA32-NEXT: masknez $t1, $a6, $t0 +; LA32-NEXT: sll.w $a1, $a1, $a5 ; LA32-NEXT: ori $a5, $zero, 64 -; LA32-NEXT: sub.w $a5, $a5, $a2 -; LA32-NEXT: xori $a5, $a5, 31 -; LA32-NEXT: srli.w $a6, $a0, 1 -; LA32-NEXT: srl.w $a5, $a6, $a5 -; LA32-NEXT: sub.w $a6, $zero, $a2 -; LA32-NEXT: sll.w $a7, $a0, $a6 -; LA32-NEXT: ori $a0, $zero, 32 -; LA32-NEXT: sub.w $t0, $a0, $a2 -; LA32-NEXT: srai.w $a0, $t0, 31 -; LA32-NEXT: and $a0, $a0, $a7 +; LA32-NEXT: sub.w $a2, $a5, $a2 +; LA32-NEXT: xori $a2, $a2, 31 +; LA32-NEXT: srli.w $a0, $a0, 1 +; LA32-NEXT: srl.w $a0, $a0, $a2 +; LA32-NEXT: or $a0, $a1, $a0 +; LA32-NEXT: maskeqz $a0, $a0, $t0 +; LA32-NEXT: or $a1, $a0, $t1 +; LA32-NEXT: srai.w $a0, $a7, 31 +; LA32-NEXT: and $a0, $a0, $a6 ; LA32-NEXT: or $a0, $a3, $a0 -; LA32-NEXT: sll.w $a3, $a1, $a6 -; LA32-NEXT: or $a3, $a3, $a5 -; LA32-NEXT: slti $a5, $t0, 0 -; LA32-NEXT: masknez $a6, $a7, $a5 -; LA32-NEXT: maskeqz $a3, $a3, $a5 -; LA32-NEXT: or $a3, $a3, $a6 -; LA32-NEXT: srl.w $a1, $a1, $a2 -; LA32-NEXT: srai.w $a2, $a4, 31 -; LA32-NEXT: and $a1, $a2, $a1 -; LA32-NEXT: or $a1, $a1, $a3 +; LA32-NEXT: or $a1, $a4, $a1 ; LA32-NEXT: ret ; ; LA64-LABEL: rotr_64: @@ -152,10 +152,10 @@ define i32 @rotl_32_mask(i32 %x, i32 %y) nounwind { ; ; LA64-LABEL: rotl_32_mask: ; LA64: # %bb.0: -; LA64-NEXT: sub.d $a2, $zero, $a1 -; LA64-NEXT: sll.w $a1, $a0, $a1 -; LA64-NEXT: srl.w $a0, $a0, $a2 -; LA64-NEXT: or $a0, $a1, $a0 +; LA64-NEXT: sll.w $a2, $a0, $a1 +; LA64-NEXT: sub.d $a1, $zero, $a1 +; LA64-NEXT: srl.w $a0, $a0, $a1 +; LA64-NEXT: or $a0, $a2, $a0 ; LA64-NEXT: ret %z = sub i32 0, %y %and = and i32 %z, 31 @@ -174,10 +174,10 @@ define i32 @rotl_32_mask_and_63_and_31(i32 %x, i32 %y) nounwind { ; ; LA64-LABEL: rotl_32_mask_and_63_and_31: ; LA64: # %bb.0: -; LA64-NEXT: sub.d $a2, $zero, $a1 -; LA64-NEXT: sll.w $a1, $a0, $a1 -; LA64-NEXT: srl.w $a0, $a0, $a2 -; LA64-NEXT: or $a0, $a1, $a0 +; LA64-NEXT: sll.w $a2, $a0, $a1 +; LA64-NEXT: sub.d $a1, $zero, $a1 +; LA64-NEXT: srl.w $a0, $a0, $a1 +; LA64-NEXT: or $a0, $a2, $a0 ; LA64-NEXT: ret %a = and i32 %y, 63 %b = shl i32 %x, %a @@ -197,10 +197,10 @@ define i32 @rotl_32_mask_or_64_or_32(i32 %x, i32 %y) nounwind { ; ; LA64-LABEL: rotl_32_mask_or_64_or_32: ; LA64: # %bb.0: -; LA64-NEXT: sub.d $a2, $zero, $a1 -; LA64-NEXT: sll.w $a1, $a0, $a1 -; LA64-NEXT: srl.w $a0, $a0, $a2 -; LA64-NEXT: or $a0, $a1, $a0 +; LA64-NEXT: sll.w $a2, $a0, $a1 +; LA64-NEXT: sub.d $a1, $zero, $a1 +; LA64-NEXT: srl.w $a0, $a0, $a1 +; LA64-NEXT: or $a0, $a2, $a0 ; LA64-NEXT: ret %a = or i32 %y, 64 %b = shl i32 %x, %a @@ -270,39 +270,38 @@ define i32 @rotr_32_mask_or_64_or_32(i32 %x, i32 %y) nounwind { define i64 @rotl_64_mask(i64 %x, i64 %y) nounwind { ; LA32-LABEL: rotl_64_mask: ; LA32: # %bb.0: -; LA32-NEXT: xori $a3, $a2, 31 -; LA32-NEXT: srli.w $a4, $a0, 1 -; LA32-NEXT: srl.w $a3, $a4, $a3 -; LA32-NEXT: sll.w $a4, $a1, $a2 -; LA32-NEXT: or $a3, $a4, $a3 -; LA32-NEXT: sub.w $a4, $zero, $a2 -; LA32-NEXT: srl.w $a5, $a1, $a4 -; LA32-NEXT: andi $a6, $a4, 63 -; LA32-NEXT: addi.w $a7, $a6, -32 -; LA32-NEXT: srai.w $t0, $a7, 31 -; LA32-NEXT: and $a5, $t0, $a5 -; LA32-NEXT: addi.w $t0, $a2, -32 -; LA32-NEXT: slti $t1, $t0, 0 -; LA32-NEXT: maskeqz $a3, $a3, $t1 -; LA32-NEXT: sll.w $t2, $a0, $t0 -; LA32-NEXT: masknez $t1, $t2, $t1 -; LA32-NEXT: or $a3, $a3, $t1 -; LA32-NEXT: xori $a6, $a6, 31 -; LA32-NEXT: slli.w $t1, $a1, 1 -; LA32-NEXT: sll.w $a6, $t1, $a6 +; LA32-NEXT: sll.w $a3, $a1, $a2 +; LA32-NEXT: xori $a4, $a2, 31 +; LA32-NEXT: srli.w $a5, $a0, 1 +; LA32-NEXT: srl.w $a4, $a5, $a4 +; LA32-NEXT: or $a3, $a3, $a4 +; LA32-NEXT: addi.w $a4, $a2, -32 +; LA32-NEXT: slti $a5, $a4, 0 +; LA32-NEXT: maskeqz $a3, $a3, $a5 +; LA32-NEXT: sll.w $a6, $a0, $a4 +; LA32-NEXT: masknez $a5, $a6, $a5 ; LA32-NEXT: or $a3, $a3, $a5 -; LA32-NEXT: srl.w $a4, $a0, $a4 -; LA32-NEXT: or $a4, $a4, $a6 -; LA32-NEXT: srl.w $a1, $a1, $a7 -; LA32-NEXT: slti $a5, $a7, 0 -; LA32-NEXT: masknez $a1, $a1, $a5 -; LA32-NEXT: maskeqz $a4, $a4, $a5 -; LA32-NEXT: or $a1, $a4, $a1 -; LA32-NEXT: sll.w $a0, $a0, $a2 -; LA32-NEXT: srai.w $a2, $t0, 31 -; LA32-NEXT: and $a0, $a2, $a0 -; LA32-NEXT: or $a0, $a0, $a1 -; LA32-NEXT: move $a1, $a3 +; LA32-NEXT: sll.w $a5, $a0, $a2 +; LA32-NEXT: srai.w $a4, $a4, 31 +; LA32-NEXT: and $a4, $a4, $a5 +; LA32-NEXT: sub.w $a2, $zero, $a2 +; LA32-NEXT: andi $a5, $a2, 63 +; LA32-NEXT: addi.w $a6, $a5, -32 +; LA32-NEXT: srl.w $a7, $a1, $a6 +; LA32-NEXT: slti $t0, $a6, 0 +; LA32-NEXT: masknez $a7, $a7, $t0 +; LA32-NEXT: srl.w $a0, $a0, $a2 +; LA32-NEXT: xori $a5, $a5, 31 +; LA32-NEXT: slli.w $t1, $a1, 1 +; LA32-NEXT: sll.w $a5, $t1, $a5 +; LA32-NEXT: or $a0, $a0, $a5 +; LA32-NEXT: maskeqz $a0, $a0, $t0 +; LA32-NEXT: or $a0, $a0, $a7 +; LA32-NEXT: srl.w $a1, $a1, $a2 +; LA32-NEXT: srai.w $a2, $a6, 31 +; LA32-NEXT: and $a1, $a2, $a1 +; LA32-NEXT: or $a1, $a3, $a1 +; LA32-NEXT: or $a0, $a4, $a0 ; LA32-NEXT: ret ; ; LA64-LABEL: rotl_64_mask: @@ -321,40 +320,39 @@ define i64 @rotl_64_mask(i64 %x, i64 %y) nounwind { define i64 @rotl_64_mask_and_127_and_63(i64 %x, i64 %y) nounwind { ; LA32-LABEL: rotl_64_mask_and_127_and_63: ; LA32: # %bb.0: -; LA32-NEXT: srli.w $a3, $a0, 1 -; LA32-NEXT: andi $a4, $a2, 127 -; LA32-NEXT: xori $a5, $a4, 31 -; LA32-NEXT: srl.w $a3, $a3, $a5 -; LA32-NEXT: sll.w $a5, $a1, $a2 -; LA32-NEXT: or $a3, $a5, $a3 -; LA32-NEXT: sub.w $a5, $zero, $a2 -; LA32-NEXT: srl.w $a6, $a1, $a5 -; LA32-NEXT: andi $a7, $a5, 63 -; LA32-NEXT: addi.w $t0, $a7, -32 -; LA32-NEXT: srai.w $t1, $t0, 31 -; LA32-NEXT: and $a6, $t1, $a6 -; LA32-NEXT: addi.w $a4, $a4, -32 -; LA32-NEXT: slti $t1, $a4, 0 -; LA32-NEXT: maskeqz $a3, $a3, $t1 -; LA32-NEXT: sll.w $t2, $a0, $a4 -; LA32-NEXT: masknez $t1, $t2, $t1 -; LA32-NEXT: or $a3, $a3, $t1 -; LA32-NEXT: xori $a7, $a7, 31 +; LA32-NEXT: sll.w $a3, $a1, $a2 +; LA32-NEXT: srli.w $a4, $a0, 1 +; LA32-NEXT: andi $a5, $a2, 127 +; LA32-NEXT: xori $a6, $a5, 31 +; LA32-NEXT: srl.w $a4, $a4, $a6 +; LA32-NEXT: or $a3, $a3, $a4 +; LA32-NEXT: addi.w $a4, $a5, -32 +; LA32-NEXT: slti $a5, $a4, 0 +; LA32-NEXT: maskeqz $a3, $a3, $a5 +; LA32-NEXT: sll.w $a6, $a0, $a4 +; LA32-NEXT: masknez $a5, $a6, $a5 +; LA32-NEXT: or $a3, $a3, $a5 +; LA32-NEXT: sll.w $a5, $a0, $a2 +; LA32-NEXT: srai.w $a4, $a4, 31 +; LA32-NEXT: and $a4, $a4, $a5 +; LA32-NEXT: sub.w $a2, $zero, $a2 +; LA32-NEXT: andi $a5, $a2, 63 +; LA32-NEXT: addi.w $a6, $a5, -32 +; LA32-NEXT: srl.w $a7, $a1, $a6 +; LA32-NEXT: slti $t0, $a6, 0 +; LA32-NEXT: masknez $a7, $a7, $t0 +; LA32-NEXT: srl.w $a0, $a0, $a2 +; LA32-NEXT: xori $a5, $a5, 31 ; LA32-NEXT: slli.w $t1, $a1, 1 -; LA32-NEXT: sll.w $a7, $t1, $a7 -; LA32-NEXT: or $a3, $a3, $a6 -; LA32-NEXT: srl.w $a5, $a0, $a5 -; LA32-NEXT: or $a5, $a5, $a7 -; LA32-NEXT: srl.w $a1, $a1, $t0 -; LA32-NEXT: slti $a6, $t0, 0 -; LA32-NEXT: masknez $a1, $a1, $a6 -; LA32-NEXT: maskeqz $a5, $a5, $a6 -; LA32-NEXT: or $a1, $a5, $a1 -; LA32-NEXT: sll.w $a0, $a0, $a2 -; LA32-NEXT: srai.w $a2, $a4, 31 -; LA32-NEXT: and $a0, $a2, $a0 -; LA32-NEXT: or $a0, $a0, $a1 -; LA32-NEXT: move $a1, $a3 +; LA32-NEXT: sll.w $a5, $t1, $a5 +; LA32-NEXT: or $a0, $a0, $a5 +; LA32-NEXT: maskeqz $a0, $a0, $t0 +; LA32-NEXT: or $a0, $a0, $a7 +; LA32-NEXT: srl.w $a1, $a1, $a2 +; LA32-NEXT: srai.w $a2, $a6, 31 +; LA32-NEXT: and $a1, $a2, $a1 +; LA32-NEXT: or $a1, $a3, $a1 +; LA32-NEXT: or $a0, $a4, $a0 ; LA32-NEXT: ret ; ; LA64-LABEL: rotl_64_mask_and_127_and_63: @@ -395,39 +393,38 @@ define i64 @rotl_64_mask_or_128_or_64(i64 %x, i64 %y) nounwind { define i64 @rotr_64_mask(i64 %x, i64 %y) nounwind { ; LA32-LABEL: rotr_64_mask: ; LA32: # %bb.0: -; LA32-NEXT: xori $a3, $a2, 31 -; LA32-NEXT: slli.w $a4, $a1, 1 -; LA32-NEXT: sll.w $a3, $a4, $a3 -; LA32-NEXT: srl.w $a4, $a0, $a2 -; LA32-NEXT: or $a3, $a4, $a3 -; LA32-NEXT: sub.w $a4, $zero, $a2 -; LA32-NEXT: sll.w $a5, $a0, $a4 -; LA32-NEXT: andi $a6, $a4, 63 -; LA32-NEXT: addi.w $a7, $a6, -32 -; LA32-NEXT: srai.w $t0, $a7, 31 -; LA32-NEXT: and $a5, $t0, $a5 -; LA32-NEXT: addi.w $t0, $a2, -32 -; LA32-NEXT: slti $t1, $t0, 0 -; LA32-NEXT: maskeqz $a3, $a3, $t1 -; LA32-NEXT: srl.w $t2, $a1, $t0 -; LA32-NEXT: masknez $t1, $t2, $t1 -; LA32-NEXT: or $a3, $a3, $t1 -; LA32-NEXT: xori $a6, $a6, 31 -; LA32-NEXT: srli.w $t1, $a0, 1 -; LA32-NEXT: srl.w $a6, $t1, $a6 +; LA32-NEXT: srl.w $a3, $a0, $a2 +; LA32-NEXT: xori $a4, $a2, 31 +; LA32-NEXT: slli.w $a5, $a1, 1 +; LA32-NEXT: sll.w $a4, $a5, $a4 +; LA32-NEXT: or $a3, $a3, $a4 +; LA32-NEXT: addi.w $a4, $a2, -32 +; LA32-NEXT: slti $a5, $a4, 0 +; LA32-NEXT: maskeqz $a3, $a3, $a5 +; LA32-NEXT: srl.w $a6, $a1, $a4 +; LA32-NEXT: masknez $a5, $a6, $a5 ; LA32-NEXT: or $a3, $a3, $a5 -; LA32-NEXT: sll.w $a4, $a1, $a4 -; LA32-NEXT: or $a4, $a4, $a6 -; LA32-NEXT: sll.w $a0, $a0, $a7 -; LA32-NEXT: slti $a5, $a7, 0 -; LA32-NEXT: masknez $a0, $a0, $a5 -; LA32-NEXT: maskeqz $a4, $a4, $a5 -; LA32-NEXT: or $a0, $a4, $a0 -; LA32-NEXT: srl.w $a1, $a1, $a2 -; LA32-NEXT: srai.w $a2, $t0, 31 -; LA32-NEXT: and $a1, $a2, $a1 -; LA32-NEXT: or $a1, $a1, $a0 -; LA32-NEXT: move $a0, $a3 +; LA32-NEXT: srl.w $a5, $a1, $a2 +; LA32-NEXT: srai.w $a4, $a4, 31 +; LA32-NEXT: and $a4, $a4, $a5 +; LA32-NEXT: sub.w $a2, $zero, $a2 +; LA32-NEXT: andi $a5, $a2, 63 +; LA32-NEXT: addi.w $a6, $a5, -32 +; LA32-NEXT: sll.w $a7, $a0, $a6 +; LA32-NEXT: slti $t0, $a6, 0 +; LA32-NEXT: masknez $a7, $a7, $t0 +; LA32-NEXT: sll.w $a1, $a1, $a2 +; LA32-NEXT: xori $a5, $a5, 31 +; LA32-NEXT: srli.w $t1, $a0, 1 +; LA32-NEXT: srl.w $a5, $t1, $a5 +; LA32-NEXT: or $a1, $a1, $a5 +; LA32-NEXT: maskeqz $a1, $a1, $t0 +; LA32-NEXT: or $a1, $a1, $a7 +; LA32-NEXT: sll.w $a0, $a0, $a2 +; LA32-NEXT: srai.w $a2, $a6, 31 +; LA32-NEXT: and $a0, $a2, $a0 +; LA32-NEXT: or $a0, $a3, $a0 +; LA32-NEXT: or $a1, $a4, $a1 ; LA32-NEXT: ret ; ; LA64-LABEL: rotr_64_mask: @@ -445,40 +442,39 @@ define i64 @rotr_64_mask(i64 %x, i64 %y) nounwind { define i64 @rotr_64_mask_and_127_and_63(i64 %x, i64 %y) nounwind { ; LA32-LABEL: rotr_64_mask_and_127_and_63: ; LA32: # %bb.0: -; LA32-NEXT: slli.w $a3, $a1, 1 -; LA32-NEXT: andi $a4, $a2, 127 -; LA32-NEXT: xori $a5, $a4, 31 -; LA32-NEXT: sll.w $a3, $a3, $a5 -; LA32-NEXT: srl.w $a5, $a0, $a2 -; LA32-NEXT: or $a3, $a5, $a3 -; LA32-NEXT: sub.w $a5, $zero, $a2 -; LA32-NEXT: sll.w $a6, $a0, $a5 -; LA32-NEXT: andi $a7, $a5, 63 -; LA32-NEXT: addi.w $t0, $a7, -32 -; LA32-NEXT: srai.w $t1, $t0, 31 -; LA32-NEXT: and $a6, $t1, $a6 -; LA32-NEXT: addi.w $a4, $a4, -32 -; LA32-NEXT: slti $t1, $a4, 0 -; LA32-NEXT: maskeqz $a3, $a3, $t1 -; LA32-NEXT: srl.w $t2, $a1, $a4 -; LA32-NEXT: masknez $t1, $t2, $t1 -; LA32-NEXT: or $a3, $a3, $t1 -; LA32-NEXT: xori $a7, $a7, 31 +; LA32-NEXT: srl.w $a3, $a0, $a2 +; LA32-NEXT: slli.w $a4, $a1, 1 +; LA32-NEXT: andi $a5, $a2, 127 +; LA32-NEXT: xori $a6, $a5, 31 +; LA32-NEXT: sll.w $a4, $a4, $a6 +; LA32-NEXT: or $a3, $a3, $a4 +; LA32-NEXT: addi.w $a4, $a5, -32 +; LA32-NEXT: slti $a5, $a4, 0 +; LA32-NEXT: maskeqz $a3, $a3, $a5 +; LA32-NEXT: srl.w $a6, $a1, $a4 +; LA32-NEXT: masknez $a5, $a6, $a5 +; LA32-NEXT: or $a3, $a3, $a5 +; LA32-NEXT: srl.w $a5, $a1, $a2 +; LA32-NEXT: srai.w $a4, $a4, 31 +; LA32-NEXT: and $a4, $a4, $a5 +; LA32-NEXT: sub.w $a2, $zero, $a2 +; LA32-NEXT: andi $a5, $a2, 63 +; LA32-NEXT: addi.w $a6, $a5, -32 +; LA32-NEXT: sll.w $a7, $a0, $a6 +; LA32-NEXT: slti $t0, $a6, 0 +; LA32-NEXT: masknez $a7, $a7, $t0 +; LA32-NEXT: sll.w $a1, $a1, $a2 +; LA32-NEXT: xori $a5, $a5, 31 ; LA32-NEXT: srli.w $t1, $a0, 1 -; LA32-NEXT: srl.w $a7, $t1, $a7 -; LA32-NEXT: or $a3, $a3, $a6 -; LA32-NEXT: sll.w $a5, $a1, $a5 -; LA32-NEXT: or $a5, $a5, $a7 -; LA32-NEXT: sll.w $a0, $a0, $t0 -; LA32-NEXT: slti $a6, $t0, 0 -; LA32-NEXT: masknez $a0, $a0, $a6 -; LA32-NEXT: maskeqz $a5, $a5, $a6 -; LA32-NEXT: or $a0, $a5, $a0 -; LA32-NEXT: srl.w $a1, $a1, $a2 -; LA32-NEXT: srai.w $a2, $a4, 31 -; LA32-NEXT: and $a1, $a2, $a1 -; LA32-NEXT: or $a1, $a1, $a0 -; LA32-NEXT: move $a0, $a3 +; LA32-NEXT: srl.w $a5, $t1, $a5 +; LA32-NEXT: or $a1, $a1, $a5 +; LA32-NEXT: maskeqz $a1, $a1, $t0 +; LA32-NEXT: or $a1, $a1, $a7 +; LA32-NEXT: sll.w $a0, $a0, $a2 +; LA32-NEXT: srai.w $a2, $a6, 31 +; LA32-NEXT: and $a0, $a2, $a0 +; LA32-NEXT: or $a0, $a3, $a0 +; LA32-NEXT: or $a1, $a4, $a1 ; LA32-NEXT: ret ; ; LA64-LABEL: rotr_64_mask_and_127_and_63: diff --git a/llvm/test/CodeGen/LoongArch/select-to-shiftand.ll b/llvm/test/CodeGen/LoongArch/select-to-shiftand.ll index 61fe123ee6aa..fa8879ea69dd 100644 --- a/llvm/test/CodeGen/LoongArch/select-to-shiftand.ll +++ b/llvm/test/CodeGen/LoongArch/select-to-shiftand.ll @@ -212,9 +212,9 @@ define i32 @sub_clamp_zero_i32(i32 signext %x, i32 signext %y) { define i64 @sub_clamp_zero_i64(i64 signext %x, i64 signext %y) { ; LA32-LABEL: sub_clamp_zero_i64: ; LA32: # %bb.0: +; LA32-NEXT: sltu $a4, $a0, $a2 ; LA32-NEXT: sub.w $a1, $a1, $a3 -; LA32-NEXT: sltu $a3, $a0, $a2 -; LA32-NEXT: sub.w $a1, $a1, $a3 +; LA32-NEXT: sub.w $a1, $a1, $a4 ; LA32-NEXT: sub.w $a0, $a0, $a2 ; LA32-NEXT: srai.w $a2, $a1, 31 ; LA32-NEXT: andn $a1, $a1, $a2 diff --git a/llvm/test/CodeGen/LoongArch/shift-masked-shamt.ll b/llvm/test/CodeGen/LoongArch/shift-masked-shamt.ll index e151624d908c..3494329e3e7c 100644 --- a/llvm/test/CodeGen/LoongArch/shift-masked-shamt.ll +++ b/llvm/test/CodeGen/LoongArch/shift-masked-shamt.ll @@ -160,11 +160,11 @@ define i64 @sll_redundant_mask_zeros_i64(i64 %a, i64 %b) { ; LA32-LABEL: sll_redundant_mask_zeros_i64: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a2, 2 +; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: srli.w $a3, $a0, 1 ; LA32-NEXT: andi $a4, $a2, 60 ; LA32-NEXT: xori $a5, $a4, 31 ; LA32-NEXT: srl.w $a3, $a3, $a5 -; LA32-NEXT: sll.w $a1, $a1, $a2 ; LA32-NEXT: or $a1, $a1, $a3 ; LA32-NEXT: addi.w $a3, $a4, -32 ; LA32-NEXT: slti $a4, $a3, 0 @@ -192,11 +192,11 @@ define i64 @srl_redundant_mask_zeros_i64(i64 %a, i64 %b) { ; LA32-LABEL: srl_redundant_mask_zeros_i64: ; LA32: # %bb.0: ; LA32-NEXT: slli.w $a2, $a2, 3 +; LA32-NEXT: srl.w $a0, $a0, $a2 ; LA32-NEXT: slli.w $a3, $a1, 1 ; LA32-NEXT: andi $a4, $a2, 56 ; LA32-NEXT: xori $a5, $a4, 31 ; LA32-NEXT: sll.w $a3, $a3, $a5 -; LA32-NEXT: srl.w $a0, $a0, $a2 ; LA32-NEXT: or $a0, $a0, $a3 ; LA32-NEXT: addi.w $a3, $a4, -32 ; LA32-NEXT: slti $a4, $a3, 0 @@ -223,23 +223,23 @@ define i64 @srl_redundant_mask_zeros_i64(i64 %a, i64 %b) { define i64 @sra_redundant_mask_zeros_i64(i64 %a, i64 %b) { ; LA32-LABEL: sra_redundant_mask_zeros_i64: ; LA32: # %bb.0: -; LA32-NEXT: slli.w $a3, $a2, 4 -; LA32-NEXT: srai.w $a2, $a1, 31 -; LA32-NEXT: andi $a4, $a3, 48 -; LA32-NEXT: addi.w $a5, $a4, -32 -; LA32-NEXT: slti $a6, $a5, 0 -; LA32-NEXT: masknez $a2, $a2, $a6 -; LA32-NEXT: sra.w $a7, $a1, $a3 -; LA32-NEXT: maskeqz $a7, $a7, $a6 -; LA32-NEXT: or $a2, $a7, $a2 -; LA32-NEXT: srl.w $a0, $a0, $a3 -; LA32-NEXT: xori $a3, $a4, 31 -; LA32-NEXT: slli.w $a4, $a1, 1 -; LA32-NEXT: sll.w $a3, $a4, $a3 +; LA32-NEXT: srai.w $a3, $a1, 31 +; LA32-NEXT: slli.w $a4, $a2, 4 +; LA32-NEXT: andi $a5, $a4, 48 +; LA32-NEXT: addi.w $a6, $a5, -32 +; LA32-NEXT: slti $a7, $a6, 0 +; LA32-NEXT: masknez $a2, $a3, $a7 +; LA32-NEXT: sra.w $a3, $a1, $a4 +; LA32-NEXT: maskeqz $a3, $a3, $a7 +; LA32-NEXT: or $a2, $a3, $a2 +; LA32-NEXT: srl.w $a0, $a0, $a4 +; LA32-NEXT: slli.w $a3, $a1, 1 +; LA32-NEXT: xori $a4, $a5, 31 +; LA32-NEXT: sll.w $a3, $a3, $a4 ; LA32-NEXT: or $a0, $a0, $a3 -; LA32-NEXT: sra.w $a1, $a1, $a5 -; LA32-NEXT: maskeqz $a0, $a0, $a6 -; LA32-NEXT: masknez $a1, $a1, $a6 +; LA32-NEXT: maskeqz $a0, $a0, $a7 +; LA32-NEXT: sra.w $a1, $a1, $a6 +; LA32-NEXT: masknez $a1, $a1, $a7 ; LA32-NEXT: or $a0, $a0, $a1 ; LA32-NEXT: move $a1, $a2 ; LA32-NEXT: ret diff --git a/llvm/test/CodeGen/LoongArch/shrinkwrap.ll b/llvm/test/CodeGen/LoongArch/shrinkwrap.ll index 5f15dd2e7eaf..0323b56080f8 100644 --- a/llvm/test/CodeGen/LoongArch/shrinkwrap.ll +++ b/llvm/test/CodeGen/LoongArch/shrinkwrap.ll @@ -9,7 +9,6 @@ define void @eliminate_restore(i32 %n) nounwind { ; NOSHRINKW: # %bb.0: ; NOSHRINKW-NEXT: addi.d $sp, $sp, -16 ; NOSHRINKW-NEXT: st.d $ra, $sp, 8 # 8-byte Folded Spill -; NOSHRINKW-NEXT: # kill: def $r5 killed $r4 ; NOSHRINKW-NEXT: addi.w $a1, $a0, 0 ; NOSHRINKW-NEXT: ori $a0, $zero, 32 ; NOSHRINKW-NEXT: bltu $a0, $a1, .LBB0_2 @@ -52,9 +51,8 @@ define void @conditional_alloca(i32 %n) nounwind { ; NOSHRINKW-NEXT: st.d $ra, $sp, 24 # 8-byte Folded Spill ; NOSHRINKW-NEXT: st.d $fp, $sp, 16 # 8-byte Folded Spill ; NOSHRINKW-NEXT: addi.d $fp, $sp, 32 -; NOSHRINKW-NEXT: move $a1, $a0 -; NOSHRINKW-NEXT: st.d $a1, $fp, -24 # 8-byte Folded Spill ; NOSHRINKW-NEXT: addi.w $a1, $a0, 0 +; NOSHRINKW-NEXT: st.d $a0, $fp, -24 # 8-byte Folded Spill ; NOSHRINKW-NEXT: ori $a0, $zero, 32 ; NOSHRINKW-NEXT: bltu $a0, $a1, .LBB1_2 ; NOSHRINKW-NEXT: b .LBB1_1 diff --git a/llvm/test/CodeGen/LoongArch/smul-with-overflow.ll b/llvm/test/CodeGen/LoongArch/smul-with-overflow.ll index 6cba4108d63c..0efb5fd4e640 100644 --- a/llvm/test/CodeGen/LoongArch/smul-with-overflow.ll +++ b/llvm/test/CodeGen/LoongArch/smul-with-overflow.ll @@ -5,63 +5,64 @@ define zeroext i1 @smuloi64(i64 %v1, i64 %v2, ptr %res) { ; LA32-LABEL: smuloi64: ; LA32: # %bb.0: -; LA32-NEXT: srai.w $a5, $a1, 31 -; LA32-NEXT: mul.w $a6, $a2, $a5 -; LA32-NEXT: mulh.wu $a7, $a2, $a5 -; LA32-NEXT: add.w $a7, $a7, $a6 -; LA32-NEXT: mul.w $a5, $a3, $a5 +; LA32-NEXT: mulh.wu $a5, $a0, $a2 +; LA32-NEXT: mul.w $a6, $a1, $a2 +; LA32-NEXT: add.w $a5, $a6, $a5 +; LA32-NEXT: sltu $a6, $a5, $a6 +; LA32-NEXT: mulh.wu $a7, $a1, $a2 +; LA32-NEXT: add.w $a6, $a7, $a6 +; LA32-NEXT: mul.w $a7, $a0, $a3 ; LA32-NEXT: add.w $a5, $a7, $a5 -; LA32-NEXT: srai.w $a7, $a3, 31 -; LA32-NEXT: mul.w $t0, $a7, $a1 -; LA32-NEXT: mulh.wu $t1, $a7, $a0 -; LA32-NEXT: add.w $t0, $t1, $t0 -; LA32-NEXT: mul.w $a7, $a7, $a0 -; LA32-NEXT: add.w $t0, $t0, $a7 -; LA32-NEXT: add.w $a5, $t0, $a5 -; LA32-NEXT: mulh.wu $t0, $a0, $a2 -; LA32-NEXT: mul.w $t1, $a1, $a2 -; LA32-NEXT: add.w $t0, $t1, $t0 -; LA32-NEXT: sltu $t1, $t0, $t1 -; LA32-NEXT: mulh.wu $t2, $a1, $a2 -; LA32-NEXT: add.w $t1, $t2, $t1 -; LA32-NEXT: mul.w $t2, $a0, $a3 -; LA32-NEXT: add.w $t0, $t2, $t0 -; LA32-NEXT: sltu $t2, $t0, $t2 -; LA32-NEXT: mulh.wu $t3, $a0, $a3 -; LA32-NEXT: add.w $t2, $t3, $t2 +; LA32-NEXT: sltu $a7, $a5, $a7 +; LA32-NEXT: mulh.wu $t0, $a0, $a3 +; LA32-NEXT: add.w $a7, $t0, $a7 +; LA32-NEXT: add.w $a7, $a6, $a7 +; LA32-NEXT: mul.w $t0, $a1, $a3 +; LA32-NEXT: add.w $t1, $t0, $a7 +; LA32-NEXT: srai.w $t2, $a1, 31 +; LA32-NEXT: mul.w $t3, $a2, $t2 +; LA32-NEXT: srai.w $t4, $a3, 31 +; LA32-NEXT: mul.w $t5, $t4, $a0 +; LA32-NEXT: add.w $t6, $t5, $t3 +; LA32-NEXT: add.w $t7, $t1, $t6 +; LA32-NEXT: sltu $t8, $t7, $t1 +; LA32-NEXT: sltu $t0, $t1, $t0 +; LA32-NEXT: sltu $a6, $a7, $a6 +; LA32-NEXT: mulh.wu $a7, $a1, $a3 ; LA32-NEXT: add.w $a6, $a7, $a6 -; LA32-NEXT: sltu $a7, $a6, $a7 -; LA32-NEXT: add.w $a5, $a5, $a7 +; LA32-NEXT: add.w $a6, $a6, $t0 +; LA32-NEXT: mulh.wu $a7, $a2, $t2 +; LA32-NEXT: add.w $a7, $a7, $t3 +; LA32-NEXT: mul.w $a3, $a3, $t2 +; LA32-NEXT: add.w $a3, $a7, $a3 +; LA32-NEXT: mul.w $a1, $t4, $a1 +; LA32-NEXT: mulh.wu $a7, $t4, $a0 +; LA32-NEXT: add.w $a1, $a7, $a1 +; LA32-NEXT: add.w $a1, $a1, $t5 +; LA32-NEXT: add.w $a1, $a1, $a3 +; LA32-NEXT: sltu $a3, $t6, $t5 +; LA32-NEXT: add.w $a1, $a1, $a3 +; LA32-NEXT: add.w $a1, $a6, $a1 +; LA32-NEXT: add.w $a1, $a1, $t8 +; LA32-NEXT: srai.w $a3, $a5, 31 +; LA32-NEXT: xor $a1, $a1, $a3 +; LA32-NEXT: xor $a3, $t7, $a3 +; LA32-NEXT: or $a1, $a3, $a1 +; LA32-NEXT: sltu $a1, $zero, $a1 ; LA32-NEXT: mul.w $a0, $a0, $a2 -; LA32-NEXT: mul.w $a2, $a1, $a3 -; LA32-NEXT: mulh.wu $a1, $a1, $a3 -; LA32-NEXT: add.w $a3, $t1, $t2 -; LA32-NEXT: sltu $a7, $a3, $t1 -; LA32-NEXT: add.w $a1, $a1, $a7 ; LA32-NEXT: st.w $a0, $a4, 0 -; LA32-NEXT: add.w $a0, $a2, $a3 -; LA32-NEXT: sltu $a2, $a0, $a2 -; LA32-NEXT: add.w $a1, $a1, $a2 -; LA32-NEXT: st.w $t0, $a4, 4 -; LA32-NEXT: add.w $a1, $a1, $a5 -; LA32-NEXT: add.w $a2, $a0, $a6 -; LA32-NEXT: sltu $a0, $a2, $a0 -; LA32-NEXT: add.w $a0, $a1, $a0 -; LA32-NEXT: srai.w $a1, $t0, 31 -; LA32-NEXT: xor $a0, $a0, $a1 -; LA32-NEXT: xor $a1, $a2, $a1 -; LA32-NEXT: or $a0, $a1, $a0 -; LA32-NEXT: sltu $a0, $zero, $a0 +; LA32-NEXT: st.w $a5, $a4, 4 +; LA32-NEXT: move $a0, $a1 ; LA32-NEXT: ret ; ; LA64-LABEL: smuloi64: ; LA64: # %bb.0: -; LA64-NEXT: mul.d $a3, $a0, $a1 -; LA64-NEXT: st.d $a3, $a2, 0 -; LA64-NEXT: mulh.d $a0, $a0, $a1 -; LA64-NEXT: srai.d $a1, $a3, 63 -; LA64-NEXT: xor $a0, $a0, $a1 +; LA64-NEXT: mulh.d $a3, $a0, $a1 +; LA64-NEXT: mul.d $a1, $a0, $a1 +; LA64-NEXT: srai.d $a0, $a1, 63 +; LA64-NEXT: xor $a0, $a3, $a0 ; LA64-NEXT: sltu $a0, $zero, $a0 +; LA64-NEXT: st.d $a1, $a2, 0 ; LA64-NEXT: ret %t = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %v1, i64 %v2) %val = extractvalue {i64, i1} %t, 0 @@ -97,273 +98,283 @@ define zeroext i1 @smuloi128(i128 %v1, i128 %v2, ptr %res) { ; LA32-NEXT: .cfi_offset 29, -36 ; LA32-NEXT: .cfi_offset 30, -40 ; LA32-NEXT: .cfi_offset 31, -44 -; LA32-NEXT: st.w $a2, $sp, 12 # 4-byte Folded Spill -; LA32-NEXT: ld.w $a6, $a1, 0 -; LA32-NEXT: ld.w $a7, $a0, 0 -; LA32-NEXT: mulh.wu $a3, $a7, $a6 -; LA32-NEXT: ld.w $a5, $a0, 4 -; LA32-NEXT: mul.w $a4, $a5, $a6 -; LA32-NEXT: add.w $a3, $a4, $a3 -; LA32-NEXT: sltu $a4, $a3, $a4 -; LA32-NEXT: mulh.wu $t0, $a5, $a6 -; LA32-NEXT: add.w $a4, $t0, $a4 -; LA32-NEXT: ld.w $t0, $a1, 4 -; LA32-NEXT: mul.w $t1, $a7, $t0 -; LA32-NEXT: add.w $a3, $t1, $a3 -; LA32-NEXT: st.w $a3, $sp, 44 # 4-byte Folded Spill -; LA32-NEXT: sltu $t1, $a3, $t1 -; LA32-NEXT: mulh.wu $t2, $a7, $t0 -; LA32-NEXT: add.w $t1, $t2, $t1 -; LA32-NEXT: ld.w $t4, $a0, 12 -; LA32-NEXT: ld.w $t2, $a0, 8 -; LA32-NEXT: ld.w $t3, $a1, 8 -; LA32-NEXT: mulh.wu $a0, $t2, $t3 -; LA32-NEXT: mul.w $t5, $t4, $t3 -; LA32-NEXT: add.w $a0, $t5, $a0 -; LA32-NEXT: sltu $t5, $a0, $t5 -; LA32-NEXT: mulh.wu $t6, $t4, $t3 -; LA32-NEXT: add.w $t5, $t6, $t5 -; LA32-NEXT: ld.w $t7, $a1, 12 -; LA32-NEXT: mul.w $a1, $t2, $t7 +; LA32-NEXT: st.w $a2, $sp, 48 # 4-byte Folded Spill +; LA32-NEXT: ld.w $t0, $a1, 12 +; LA32-NEXT: ld.w $t1, $a1, 8 +; LA32-NEXT: ld.w $a5, $a0, 12 +; LA32-NEXT: ld.w $a7, $a1, 0 +; LA32-NEXT: ld.w $a3, $a0, 0 +; LA32-NEXT: ld.w $a6, $a0, 4 +; LA32-NEXT: ld.w $a4, $a0, 8 +; LA32-NEXT: ld.w $t3, $a1, 4 +; LA32-NEXT: mulh.wu $a0, $a3, $a7 +; LA32-NEXT: mul.w $a1, $a6, $a7 ; LA32-NEXT: add.w $a0, $a1, $a0 -; LA32-NEXT: st.w $a0, $sp, 48 # 4-byte Folded Spill ; LA32-NEXT: sltu $a1, $a0, $a1 -; LA32-NEXT: mulh.wu $t6, $t2, $t7 -; LA32-NEXT: add.w $t6, $t6, $a1 -; LA32-NEXT: srai.w $s7, $t4, 31 -; LA32-NEXT: mul.w $a1, $s7, $t7 -; LA32-NEXT: mulh.wu $t8, $s7, $t3 -; LA32-NEXT: add.w $t8, $t8, $a1 -; LA32-NEXT: mulh.wu $fp, $a6, $s7 -; LA32-NEXT: mul.w $s6, $t0, $s7 -; LA32-NEXT: add.w $s8, $s6, $fp -; LA32-NEXT: mul.w $a1, $a6, $s7 -; LA32-NEXT: add.w $ra, $a1, $s8 -; LA32-NEXT: sltu $s0, $ra, $a1 -; LA32-NEXT: add.w $a0, $fp, $s0 -; LA32-NEXT: add.w $a3, $a4, $t1 +; LA32-NEXT: mulh.wu $t2, $a6, $a7 +; LA32-NEXT: add.w $a1, $t2, $a1 +; LA32-NEXT: mul.w $t2, $a3, $t3 +; LA32-NEXT: add.w $a0, $t2, $a0 +; LA32-NEXT: st.w $a0, $sp, 44 # 4-byte Folded Spill +; LA32-NEXT: sltu $t2, $a0, $t2 +; LA32-NEXT: mulh.wu $t4, $a3, $t3 +; LA32-NEXT: add.w $t2, $t4, $t2 +; LA32-NEXT: add.w $t2, $a1, $t2 +; LA32-NEXT: mul.w $t4, $a6, $t3 +; LA32-NEXT: add.w $t5, $t4, $t2 +; LA32-NEXT: sltu $t4, $t5, $t4 +; LA32-NEXT: sltu $a1, $t2, $a1 +; LA32-NEXT: mulh.wu $t2, $a6, $t3 +; LA32-NEXT: add.w $a1, $t2, $a1 +; LA32-NEXT: add.w $a1, $a1, $t4 +; LA32-NEXT: mulh.wu $t2, $a4, $a7 +; LA32-NEXT: mul.w $t4, $a5, $a7 +; LA32-NEXT: add.w $t2, $t4, $t2 +; LA32-NEXT: mul.w $t6, $a4, $t3 +; LA32-NEXT: add.w $t7, $t6, $t2 +; LA32-NEXT: add.w $a1, $t7, $a1 +; LA32-NEXT: mul.w $t8, $a4, $a7 +; LA32-NEXT: add.w $t5, $t8, $t5 +; LA32-NEXT: sltu $t8, $t5, $t8 +; LA32-NEXT: add.w $a1, $a1, $t8 +; LA32-NEXT: sltu $fp, $a1, $t7 +; LA32-NEXT: xor $s0, $a1, $t7 +; LA32-NEXT: sltui $s0, $s0, 1 +; LA32-NEXT: masknez $fp, $fp, $s0 +; LA32-NEXT: maskeqz $t8, $t8, $s0 +; LA32-NEXT: or $t8, $t8, $fp +; LA32-NEXT: sltu $t2, $t2, $t4 +; LA32-NEXT: mulh.wu $t4, $a5, $a7 +; LA32-NEXT: add.w $t4, $t4, $t2 +; LA32-NEXT: sltu $t2, $t7, $t6 +; LA32-NEXT: mulh.wu $t6, $a4, $t3 +; LA32-NEXT: add.w $t2, $t6, $t2 +; LA32-NEXT: add.w $fp, $t4, $t2 +; LA32-NEXT: mul.w $t6, $a5, $t3 +; LA32-NEXT: add.w $s0, $t6, $fp +; LA32-NEXT: add.w $s1, $s0, $t8 +; LA32-NEXT: mulh.wu $t2, $a3, $t1 +; LA32-NEXT: mul.w $t7, $a6, $t1 +; LA32-NEXT: add.w $t8, $t7, $t2 +; LA32-NEXT: mul.w $s2, $a3, $t0 +; LA32-NEXT: add.w $s3, $s2, $t8 +; LA32-NEXT: add.w $t2, $s3, $a1 +; LA32-NEXT: mul.w $s4, $a3, $t1 +; LA32-NEXT: add.w $a0, $s4, $t5 +; LA32-NEXT: st.w $a0, $sp, 40 # 4-byte Folded Spill +; LA32-NEXT: sltu $t5, $a0, $s4 +; LA32-NEXT: add.w $a0, $t2, $t5 +; LA32-NEXT: st.w $a0, $sp, 36 # 4-byte Folded Spill +; LA32-NEXT: sltu $s4, $a0, $s3 +; LA32-NEXT: xor $s5, $a0, $s3 +; LA32-NEXT: sltui $s5, $s5, 1 +; LA32-NEXT: masknez $s4, $s4, $s5 +; LA32-NEXT: maskeqz $t5, $t5, $s5 +; LA32-NEXT: or $t5, $t5, $s4 +; LA32-NEXT: sltu $t7, $t8, $t7 +; LA32-NEXT: mulh.wu $t8, $a6, $t1 +; LA32-NEXT: add.w $s4, $t8, $t7 +; LA32-NEXT: sltu $t7, $s3, $s2 +; LA32-NEXT: mulh.wu $t8, $a3, $t0 +; LA32-NEXT: add.w $t7, $t8, $t7 +; LA32-NEXT: add.w $s2, $s4, $t7 +; LA32-NEXT: mul.w $s3, $a6, $t0 +; LA32-NEXT: add.w $s6, $s3, $s2 +; LA32-NEXT: add.w $s7, $s6, $t5 +; LA32-NEXT: add.w $s5, $s1, $s7 +; LA32-NEXT: mul.w $s8, $a4, $t1 +; LA32-NEXT: add.w $ra, $s8, $s5 +; LA32-NEXT: srai.w $t8, $a5, 31 +; LA32-NEXT: mul.w $t7, $a7, $t8 +; LA32-NEXT: st.w $a7, $sp, 28 # 4-byte Folded Spill +; LA32-NEXT: srai.w $t5, $t0, 31 +; LA32-NEXT: sltu $s5, $s5, $s1 +; LA32-NEXT: sltu $s1, $s1, $s0 +; LA32-NEXT: sltu $s0, $s0, $t6 +; LA32-NEXT: mul.w $t2, $a3, $t5 +; LA32-NEXT: st.w $a3, $sp, 24 # 4-byte Folded Spill +; LA32-NEXT: sltu $t4, $fp, $t4 +; LA32-NEXT: mulh.wu $fp, $a5, $t3 +; LA32-NEXT: st.w $a5, $sp, 0 # 4-byte Folded Spill +; LA32-NEXT: add.w $t4, $fp, $t4 +; LA32-NEXT: add.w $fp, $t2, $t7 +; LA32-NEXT: add.w $s0, $t4, $s0 +; LA32-NEXT: add.w $a0, $ra, $fp +; LA32-NEXT: st.w $a0, $sp, 32 # 4-byte Folded Spill +; LA32-NEXT: add.w $a2, $s0, $s1 +; LA32-NEXT: sltu $s0, $a0, $ra +; LA32-NEXT: sltu $s1, $s7, $s6 +; LA32-NEXT: sltu $s3, $s6, $s3 +; LA32-NEXT: sltu $s2, $s2, $s4 +; LA32-NEXT: move $s6, $a6 +; LA32-NEXT: st.w $a6, $sp, 16 # 4-byte Folded Spill +; LA32-NEXT: mulh.wu $s4, $a6, $t0 +; LA32-NEXT: add.w $s2, $s4, $s2 +; LA32-NEXT: add.w $s2, $s2, $s3 +; LA32-NEXT: add.w $s1, $s2, $s1 +; LA32-NEXT: add.w $s1, $a2, $s1 +; LA32-NEXT: add.w $s7, $s1, $s5 +; LA32-NEXT: move $a0, $a4 +; LA32-NEXT: st.w $a4, $sp, 4 # 4-byte Folded Spill +; LA32-NEXT: mulh.wu $s1, $a4, $t1 +; LA32-NEXT: mul.w $a5, $a5, $t1 +; LA32-NEXT: add.w $a4, $a5, $s1 +; LA32-NEXT: mul.w $a6, $a0, $t0 +; LA32-NEXT: add.w $a1, $a6, $a4 +; LA32-NEXT: sltu $ra, $ra, $s8 +; LA32-NEXT: add.w $s1, $a1, $s7 +; LA32-NEXT: add.w $s8, $s1, $ra +; LA32-NEXT: move $a0, $t2 +; LA32-NEXT: st.w $t2, $sp, 8 # 4-byte Folded Spill +; LA32-NEXT: sltu $t6, $fp, $t2 +; LA32-NEXT: mulh.wu $t2, $a7, $t8 +; LA32-NEXT: mul.w $s4, $t3, $t8 +; LA32-NEXT: add.w $a7, $s4, $t2 +; LA32-NEXT: st.w $a7, $sp, 12 # 4-byte Folded Spill +; LA32-NEXT: add.w $s3, $t7, $a7 +; LA32-NEXT: mulh.wu $a7, $a3, $t5 +; LA32-NEXT: add.w $t4, $a7, $a0 +; LA32-NEXT: mul.w $s2, $s6, $t5 +; LA32-NEXT: add.w $s1, $t4, $s2 +; LA32-NEXT: add.w $fp, $s1, $s3 +; LA32-NEXT: add.w $a0, $fp, $t6 +; LA32-NEXT: add.w $fp, $s8, $a0 +; LA32-NEXT: add.w $a3, $fp, $s0 ; LA32-NEXT: st.w $a3, $sp, 20 # 4-byte Folded Spill +; LA32-NEXT: xor $fp, $a3, $s8 +; LA32-NEXT: sltui $fp, $fp, 1 +; LA32-NEXT: sltu $s6, $a3, $s8 +; LA32-NEXT: masknez $s6, $s6, $fp +; LA32-NEXT: maskeqz $fp, $s0, $fp +; LA32-NEXT: or $s6, $fp, $s6 +; LA32-NEXT: sltu $fp, $s7, $a2 +; LA32-NEXT: xor $a2, $s7, $a2 +; LA32-NEXT: sltui $a2, $a2, 1 +; LA32-NEXT: masknez $fp, $fp, $a2 +; LA32-NEXT: maskeqz $a2, $s5, $a2 +; LA32-NEXT: or $s0, $a2, $fp +; LA32-NEXT: sltu $a2, $a4, $a5 +; LA32-NEXT: ld.w $a5, $sp, 0 # 4-byte Folded Reload +; LA32-NEXT: mulh.wu $a3, $a5, $t1 +; LA32-NEXT: add.w $a2, $a3, $a2 +; LA32-NEXT: sltu $a3, $a1, $a6 +; LA32-NEXT: ld.w $fp, $sp, 4 # 4-byte Folded Reload +; LA32-NEXT: mulh.wu $a4, $fp, $t0 +; LA32-NEXT: add.w $a3, $a4, $a3 +; LA32-NEXT: sltu $a4, $s8, $a1 +; LA32-NEXT: xor $a1, $s8, $a1 +; LA32-NEXT: sltui $a1, $a1, 1 +; LA32-NEXT: masknez $a4, $a4, $a1 +; LA32-NEXT: maskeqz $a1, $ra, $a1 +; LA32-NEXT: or $a1, $a1, $a4 +; LA32-NEXT: sltu $a4, $a0, $s1 +; LA32-NEXT: xor $a0, $a0, $s1 +; LA32-NEXT: sltui $a0, $a0, 1 +; LA32-NEXT: masknez $a4, $a4, $a0 +; LA32-NEXT: maskeqz $a0, $t6, $a0 +; LA32-NEXT: or $s5, $a0, $a4 +; LA32-NEXT: sltu $a0, $s3, $t7 +; LA32-NEXT: add.w $a0, $t2, $a0 +; LA32-NEXT: ld.w $t2, $sp, 8 # 4-byte Folded Reload +; LA32-NEXT: sltu $a4, $t4, $t2 +; LA32-NEXT: add.w $s7, $a7, $a4 +; LA32-NEXT: add.w $a3, $a2, $a3 +; LA32-NEXT: sltu $a2, $a3, $a2 +; LA32-NEXT: mulh.wu $a4, $a5, $t0 +; LA32-NEXT: add.w $a2, $a4, $a2 +; LA32-NEXT: mul.w $a4, $a5, $t0 +; LA32-NEXT: move $a6, $a5 +; LA32-NEXT: add.w $a3, $a4, $a3 ; LA32-NEXT: sltu $a4, $a3, $a4 -; LA32-NEXT: mulh.wu $t1, $a5, $t0 -; LA32-NEXT: add.w $a3, $t1, $a4 -; LA32-NEXT: st.w $a3, $sp, 28 # 4-byte Folded Spill -; LA32-NEXT: srai.w $s4, $t7, 31 -; LA32-NEXT: mul.w $fp, $a7, $s4 -; LA32-NEXT: mulh.wu $a4, $a7, $s4 -; LA32-NEXT: add.w $s1, $a4, $fp -; LA32-NEXT: sltu $s0, $s1, $fp -; LA32-NEXT: add.w $s5, $a4, $s0 -; LA32-NEXT: mul.w $a4, $s7, $t3 -; LA32-NEXT: add.w $t8, $t8, $a4 -; LA32-NEXT: add.w $s0, $ra, $t8 -; LA32-NEXT: add.w $a3, $a1, $a4 -; LA32-NEXT: st.w $a3, $sp, 32 # 4-byte Folded Spill -; LA32-NEXT: sltu $a4, $a3, $a1 -; LA32-NEXT: add.w $a3, $s0, $a4 -; LA32-NEXT: st.w $a3, $sp, 24 # 4-byte Folded Spill -; LA32-NEXT: add.w $s3, $t5, $t6 -; LA32-NEXT: sltu $a4, $s3, $t5 -; LA32-NEXT: mulh.wu $t5, $t4, $t7 -; LA32-NEXT: add.w $a3, $t5, $a4 -; LA32-NEXT: st.w $a3, $sp, 16 # 4-byte Folded Spill -; LA32-NEXT: mul.w $a4, $a7, $a6 -; LA32-NEXT: st.w $a4, $a2, 0 -; LA32-NEXT: sltu $a4, $s8, $s6 -; LA32-NEXT: mulh.wu $t5, $t0, $s7 -; LA32-NEXT: add.w $a4, $t5, $a4 -; LA32-NEXT: add.w $t1, $a4, $a0 -; LA32-NEXT: sltu $a4, $t1, $a4 -; LA32-NEXT: add.w $s2, $t5, $a4 -; LA32-NEXT: mulh.wu $a4, $a7, $t3 -; LA32-NEXT: mul.w $t5, $a5, $t3 -; LA32-NEXT: add.w $a4, $t5, $a4 -; LA32-NEXT: sltu $t5, $a4, $t5 -; LA32-NEXT: mulh.wu $t6, $a5, $t3 -; LA32-NEXT: add.w $a3, $t6, $t5 -; LA32-NEXT: mul.w $t6, $a7, $t7 -; LA32-NEXT: add.w $t5, $t6, $a4 -; LA32-NEXT: sltu $a4, $t5, $t6 -; LA32-NEXT: mulh.wu $t6, $a7, $t7 -; LA32-NEXT: add.w $a4, $t6, $a4 -; LA32-NEXT: mulh.wu $t6, $t2, $a6 -; LA32-NEXT: mul.w $s7, $t4, $a6 -; LA32-NEXT: add.w $t6, $s7, $t6 -; LA32-NEXT: sltu $s7, $t6, $s7 -; LA32-NEXT: mulh.wu $s8, $t4, $a6 -; LA32-NEXT: add.w $a0, $s8, $s7 -; LA32-NEXT: mul.w $s7, $t2, $t0 -; LA32-NEXT: add.w $t6, $s7, $t6 -; LA32-NEXT: sltu $s7, $t6, $s7 -; LA32-NEXT: mulh.wu $s8, $t2, $t0 -; LA32-NEXT: add.w $a2, $s8, $s7 -; LA32-NEXT: mul.w $s8, $a5, $s4 -; LA32-NEXT: add.w $s7, $s1, $s8 -; LA32-NEXT: add.w $s1, $s7, $ra -; LA32-NEXT: add.w $a1, $fp, $a1 -; LA32-NEXT: st.w $a1, $sp, 40 # 4-byte Folded Spill -; LA32-NEXT: sltu $ra, $a1, $fp -; LA32-NEXT: add.w $a1, $s1, $ra -; LA32-NEXT: st.w $a1, $sp, 36 # 4-byte Folded Spill -; LA32-NEXT: xor $s0, $a1, $s7 -; LA32-NEXT: sltui $s0, $s0, 1 -; LA32-NEXT: sltu $a1, $a1, $s7 -; LA32-NEXT: masknez $s1, $a1, $s0 -; LA32-NEXT: maskeqz $s0, $ra, $s0 -; LA32-NEXT: add.w $t1, $s6, $t1 -; LA32-NEXT: sltu $s6, $t1, $s6 -; LA32-NEXT: add.w $s2, $s2, $s6 +; LA32-NEXT: add.w $a2, $a2, $a4 +; LA32-NEXT: add.w $a4, $a3, $s0 +; LA32-NEXT: sltu $a3, $a4, $a3 +; LA32-NEXT: add.w $a2, $a2, $a3 +; LA32-NEXT: add.w $s8, $a4, $a1 +; LA32-NEXT: sltu $a1, $s8, $a4 +; LA32-NEXT: add.w $ra, $a2, $a1 +; LA32-NEXT: ld.w $a1, $sp, 12 # 4-byte Folded Reload +; LA32-NEXT: sltu $a1, $a1, $s4 +; LA32-NEXT: mulh.wu $a2, $t3, $t8 +; LA32-NEXT: add.w $a1, $a2, $a1 +; LA32-NEXT: add.w $a0, $a1, $a0 +; LA32-NEXT: sltu $a1, $a0, $a1 +; LA32-NEXT: add.w $a1, $a2, $a1 +; LA32-NEXT: add.w $a0, $s4, $a0 +; LA32-NEXT: sltu $a2, $a0, $s4 +; LA32-NEXT: add.w $a1, $a1, $a2 +; LA32-NEXT: mul.w $a2, $t8, $t1 +; LA32-NEXT: mul.w $a3, $t8, $t0 +; LA32-NEXT: mulh.wu $a4, $t8, $t1 +; LA32-NEXT: add.w $a3, $a4, $a3 +; LA32-NEXT: add.w $a3, $a3, $a2 +; LA32-NEXT: add.w $a3, $s3, $a3 +; LA32-NEXT: add.w $a2, $t7, $a2 +; LA32-NEXT: sltu $a4, $a2, $t7 +; LA32-NEXT: add.w $a3, $a3, $a4 +; LA32-NEXT: add.w $a1, $a1, $a3 ; LA32-NEXT: add.w $a2, $a0, $a2 ; LA32-NEXT: sltu $a0, $a2, $a0 -; LA32-NEXT: mulh.wu $s6, $t4, $t0 -; LA32-NEXT: add.w $t8, $s6, $a0 -; LA32-NEXT: add.w $a4, $a3, $a4 -; LA32-NEXT: sltu $a3, $a4, $a3 -; LA32-NEXT: mulh.wu $s6, $a5, $t7 -; LA32-NEXT: add.w $a3, $s6, $a3 -; LA32-NEXT: mul.w $s6, $t4, $t7 -; LA32-NEXT: mul.w $t7, $a5, $t7 -; LA32-NEXT: mul.w $ra, $t4, $t0 -; LA32-NEXT: mul.w $t0, $a5, $t0 -; LA32-NEXT: mul.w $t4, $t4, $s4 -; LA32-NEXT: mul.w $a7, $a7, $t3 -; LA32-NEXT: mul.w $a6, $t2, $a6 -; LA32-NEXT: mul.w $t3, $t2, $t3 -; LA32-NEXT: mul.w $a0, $t2, $s4 -; LA32-NEXT: mulh.wu $t2, $t2, $s4 -; LA32-NEXT: mulh.wu $a5, $s4, $a5 -; LA32-NEXT: sltu $s4, $s7, $s8 -; LA32-NEXT: add.w $s4, $a5, $s4 -; LA32-NEXT: add.w $s4, $s5, $s4 -; LA32-NEXT: sltu $s5, $s4, $s5 -; LA32-NEXT: add.w $s5, $a5, $s5 -; LA32-NEXT: ld.w $a1, $sp, 20 # 4-byte Folded Reload -; LA32-NEXT: add.w $a1, $t0, $a1 -; LA32-NEXT: sltu $a5, $a1, $t0 -; LA32-NEXT: ld.w $t0, $sp, 28 # 4-byte Folded Reload -; LA32-NEXT: add.w $t0, $t0, $a5 -; LA32-NEXT: or $s0, $s0, $s1 -; LA32-NEXT: add.w $a4, $t7, $a4 -; LA32-NEXT: sltu $a5, $a4, $t7 -; LA32-NEXT: add.w $t7, $a3, $a5 -; LA32-NEXT: add.w $s1, $ra, $a2 -; LA32-NEXT: sltu $a2, $s1, $ra -; LA32-NEXT: add.w $t8, $t8, $a2 -; LA32-NEXT: add.w $a5, $s6, $s3 -; LA32-NEXT: sltu $a2, $a5, $s6 +; LA32-NEXT: add.w $a0, $a1, $a0 +; LA32-NEXT: sltu $a1, $s1, $s2 ; LA32-NEXT: ld.w $a3, $sp, 16 # 4-byte Folded Reload -; LA32-NEXT: add.w $a2, $a3, $a2 -; LA32-NEXT: ld.w $s6, $sp, 12 # 4-byte Folded Reload -; LA32-NEXT: ld.w $a3, $sp, 44 # 4-byte Folded Reload -; LA32-NEXT: st.w $a3, $s6, 4 -; LA32-NEXT: ld.w $a3, $sp, 24 # 4-byte Folded Reload -; LA32-NEXT: add.w $a3, $s2, $a3 -; LA32-NEXT: ld.w $s2, $sp, 32 # 4-byte Folded Reload -; LA32-NEXT: add.w $s2, $t1, $s2 -; LA32-NEXT: sltu $t1, $s2, $t1 -; LA32-NEXT: add.w $a3, $a3, $t1 -; LA32-NEXT: add.w $t1, $s8, $s4 -; LA32-NEXT: sltu $s3, $t1, $s8 -; LA32-NEXT: add.w $s3, $s5, $s3 -; LA32-NEXT: add.w $t2, $t2, $a0 -; LA32-NEXT: add.w $t2, $t2, $t4 -; LA32-NEXT: add.w $t2, $t2, $s7 -; LA32-NEXT: add.w $t4, $a0, $fp -; LA32-NEXT: sltu $a0, $t4, $a0 -; LA32-NEXT: add.w $a0, $t2, $a0 -; LA32-NEXT: add.w $a0, $s3, $a0 -; LA32-NEXT: add.w $t2, $t1, $t4 -; LA32-NEXT: sltu $t1, $t2, $t1 -; LA32-NEXT: add.w $a0, $a0, $t1 -; LA32-NEXT: add.w $a0, $a0, $a3 -; LA32-NEXT: add.w $t1, $t2, $s2 -; LA32-NEXT: sltu $a3, $t1, $t2 -; LA32-NEXT: add.w $a0, $a0, $a3 -; LA32-NEXT: add.w $a3, $t6, $t0 -; LA32-NEXT: add.w $a1, $a6, $a1 -; LA32-NEXT: sltu $a6, $a1, $a6 -; LA32-NEXT: add.w $t0, $a3, $a6 -; LA32-NEXT: add.w $a1, $a7, $a1 -; LA32-NEXT: sltu $a7, $a1, $a7 -; LA32-NEXT: add.w $a3, $t5, $t0 -; LA32-NEXT: add.w $a3, $a3, $a7 -; LA32-NEXT: sltu $t2, $a3, $t5 -; LA32-NEXT: xor $t4, $a3, $t5 -; LA32-NEXT: sltui $t4, $t4, 1 -; LA32-NEXT: masknez $t2, $t2, $t4 -; LA32-NEXT: maskeqz $a7, $a7, $t4 -; LA32-NEXT: st.w $a1, $s6, 8 -; LA32-NEXT: or $a1, $a7, $t2 -; LA32-NEXT: sltu $a7, $t0, $t6 -; LA32-NEXT: xor $t0, $t0, $t6 -; LA32-NEXT: sltui $t0, $t0, 1 -; LA32-NEXT: masknez $a7, $a7, $t0 -; LA32-NEXT: maskeqz $a6, $a6, $t0 -; LA32-NEXT: or $a6, $a6, $a7 -; LA32-NEXT: add.w $a6, $s1, $a6 -; LA32-NEXT: sltu $a7, $a6, $s1 -; LA32-NEXT: add.w $a7, $t8, $a7 -; LA32-NEXT: add.w $a1, $a4, $a1 -; LA32-NEXT: sltu $a4, $a1, $a4 -; LA32-NEXT: add.w $a4, $t7, $a4 -; LA32-NEXT: add.w $t0, $t1, $s0 -; LA32-NEXT: sltu $t1, $t0, $t1 -; LA32-NEXT: add.w $a0, $a0, $t1 -; LA32-NEXT: st.w $a3, $s6, 12 -; LA32-NEXT: add.w $a1, $a6, $a1 -; LA32-NEXT: sltu $a6, $a1, $a6 -; LA32-NEXT: add.w $a4, $a7, $a4 -; LA32-NEXT: add.w $a4, $a4, $a6 -; LA32-NEXT: sltu $t1, $a4, $a7 -; LA32-NEXT: xor $a7, $a4, $a7 -; LA32-NEXT: sltui $a7, $a7, 1 -; LA32-NEXT: masknez $t1, $t1, $a7 -; LA32-NEXT: maskeqz $a6, $a6, $a7 -; LA32-NEXT: or $a6, $a6, $t1 -; LA32-NEXT: add.w $a6, $a5, $a6 -; LA32-NEXT: sltu $a5, $a6, $a5 -; LA32-NEXT: add.w $a2, $a2, $a5 -; LA32-NEXT: ld.w $t1, $sp, 48 # 4-byte Folded Reload -; LA32-NEXT: add.w $a4, $t1, $a4 -; LA32-NEXT: add.w $a1, $t3, $a1 -; LA32-NEXT: sltu $a5, $a1, $t3 -; LA32-NEXT: add.w $a4, $a4, $a5 -; LA32-NEXT: sltu $a7, $a4, $t1 -; LA32-NEXT: xor $t1, $a4, $t1 -; LA32-NEXT: sltui $t1, $t1, 1 -; LA32-NEXT: masknez $a7, $a7, $t1 -; LA32-NEXT: maskeqz $a5, $a5, $t1 -; LA32-NEXT: or $a5, $a5, $a7 -; LA32-NEXT: add.w $a5, $a6, $a5 -; LA32-NEXT: sltu $a6, $a5, $a6 -; LA32-NEXT: add.w $a2, $a2, $a6 -; LA32-NEXT: add.w $a0, $a2, $a0 -; LA32-NEXT: add.w $a2, $a5, $t0 -; LA32-NEXT: sltu $a5, $a2, $a5 -; LA32-NEXT: add.w $a0, $a0, $a5 -; LA32-NEXT: ld.w $a5, $sp, 40 # 4-byte Folded Reload -; LA32-NEXT: add.w $a5, $a1, $a5 -; LA32-NEXT: sltu $a1, $a5, $a1 -; LA32-NEXT: ld.w $a6, $sp, 36 # 4-byte Folded Reload -; LA32-NEXT: add.w $a6, $a4, $a6 -; LA32-NEXT: add.w $a6, $a6, $a1 -; LA32-NEXT: sltu $a7, $a6, $a4 -; LA32-NEXT: xor $a4, $a6, $a4 -; LA32-NEXT: sltui $a4, $a4, 1 -; LA32-NEXT: masknez $a7, $a7, $a4 -; LA32-NEXT: maskeqz $a1, $a1, $a4 -; LA32-NEXT: or $a1, $a1, $a7 -; LA32-NEXT: add.w $a1, $a2, $a1 -; LA32-NEXT: sltu $a2, $a1, $a2 +; LA32-NEXT: mulh.wu $a3, $t5, $a3 +; LA32-NEXT: add.w $a1, $a3, $a1 +; LA32-NEXT: add.w $a1, $s7, $a1 +; LA32-NEXT: sltu $a4, $a1, $s7 +; LA32-NEXT: add.w $a3, $a3, $a4 +; LA32-NEXT: add.w $a1, $s2, $a1 +; LA32-NEXT: sltu $a4, $a1, $s2 +; LA32-NEXT: add.w $a3, $a3, $a4 +; LA32-NEXT: mul.w $a4, $fp, $t5 +; LA32-NEXT: mulh.wu $a5, $fp, $t5 +; LA32-NEXT: mul.w $a6, $a6, $t5 +; LA32-NEXT: add.w $a5, $a5, $a4 +; LA32-NEXT: add.w $a5, $a5, $a6 +; LA32-NEXT: add.w $a5, $a5, $s1 +; LA32-NEXT: add.w $a6, $a4, $t2 +; LA32-NEXT: sltu $a4, $a6, $a4 +; LA32-NEXT: add.w $a4, $a5, $a4 +; LA32-NEXT: add.w $a3, $a3, $a4 +; LA32-NEXT: add.w $a4, $a1, $a6 +; LA32-NEXT: sltu $a1, $a4, $a1 +; LA32-NEXT: add.w $a1, $a3, $a1 +; LA32-NEXT: add.w $a0, $a1, $a0 +; LA32-NEXT: add.w $a1, $a4, $a2 +; LA32-NEXT: sltu $a2, $a1, $a4 ; LA32-NEXT: add.w $a0, $a0, $a2 -; LA32-NEXT: srai.w $a2, $a3, 31 -; LA32-NEXT: xor $a3, $a6, $a2 -; LA32-NEXT: xor $a0, $a0, $a2 +; LA32-NEXT: add.w $a2, $a1, $s5 +; LA32-NEXT: sltu $a1, $a2, $a1 +; LA32-NEXT: add.w $a0, $a0, $a1 +; LA32-NEXT: add.w $a0, $ra, $a0 +; LA32-NEXT: add.w $a1, $s8, $a2 +; LA32-NEXT: sltu $a2, $a1, $s8 +; LA32-NEXT: add.w $a0, $a0, $a2 +; LA32-NEXT: add.w $a2, $a1, $s6 +; LA32-NEXT: sltu $a1, $a2, $a1 +; LA32-NEXT: add.w $a0, $a0, $a1 +; LA32-NEXT: ld.w $a4, $sp, 36 # 4-byte Folded Reload +; LA32-NEXT: srai.w $a1, $a4, 31 +; LA32-NEXT: xor $a0, $a0, $a1 +; LA32-NEXT: ld.w $a3, $sp, 20 # 4-byte Folded Reload +; LA32-NEXT: xor $a3, $a3, $a1 ; LA32-NEXT: or $a0, $a3, $a0 -; LA32-NEXT: xor $a3, $a5, $a2 -; LA32-NEXT: xor $a1, $a1, $a2 -; LA32-NEXT: or $a1, $a3, $a1 +; LA32-NEXT: xor $a2, $a2, $a1 +; LA32-NEXT: ld.w $a3, $sp, 32 # 4-byte Folded Reload +; LA32-NEXT: xor $a1, $a3, $a1 +; LA32-NEXT: or $a1, $a1, $a2 ; LA32-NEXT: or $a0, $a1, $a0 +; LA32-NEXT: ld.w $a1, $sp, 28 # 4-byte Folded Reload +; LA32-NEXT: ld.w $a2, $sp, 24 # 4-byte Folded Reload +; LA32-NEXT: mul.w $a1, $a2, $a1 +; LA32-NEXT: ld.w $a2, $sp, 48 # 4-byte Folded Reload +; LA32-NEXT: st.w $a1, $a2, 0 +; LA32-NEXT: ld.w $a1, $sp, 44 # 4-byte Folded Reload +; LA32-NEXT: st.w $a1, $a2, 4 +; LA32-NEXT: ld.w $a1, $sp, 40 # 4-byte Folded Reload +; LA32-NEXT: st.w $a1, $a2, 8 ; LA32-NEXT: sltu $a0, $zero, $a0 +; LA32-NEXT: st.w $a4, $a2, 12 ; LA32-NEXT: ld.w $s8, $sp, 52 # 4-byte Folded Reload ; LA32-NEXT: ld.w $s7, $sp, 56 # 4-byte Folded Reload ; LA32-NEXT: ld.w $s6, $sp, 60 # 4-byte Folded Reload @@ -380,53 +391,54 @@ define zeroext i1 @smuloi128(i128 %v1, i128 %v2, ptr %res) { ; ; LA64-LABEL: smuloi128: ; LA64: # %bb.0: -; LA64-NEXT: srai.d $a5, $a1, 63 -; LA64-NEXT: mul.d $a6, $a2, $a5 -; LA64-NEXT: mulh.du $a7, $a2, $a5 -; LA64-NEXT: add.d $a7, $a7, $a6 -; LA64-NEXT: mul.d $a5, $a3, $a5 +; LA64-NEXT: mulh.du $a5, $a0, $a2 +; LA64-NEXT: mul.d $a6, $a1, $a2 +; LA64-NEXT: add.d $a5, $a6, $a5 +; LA64-NEXT: sltu $a6, $a5, $a6 +; LA64-NEXT: mulh.du $a7, $a1, $a2 +; LA64-NEXT: add.d $a6, $a7, $a6 +; LA64-NEXT: mul.d $a7, $a0, $a3 ; LA64-NEXT: add.d $a5, $a7, $a5 -; LA64-NEXT: srai.d $a7, $a3, 63 -; LA64-NEXT: mul.d $t0, $a7, $a1 -; LA64-NEXT: mulh.du $t1, $a7, $a0 -; LA64-NEXT: add.d $t0, $t1, $t0 -; LA64-NEXT: mul.d $a7, $a7, $a0 -; LA64-NEXT: add.d $t0, $t0, $a7 -; LA64-NEXT: add.d $a5, $t0, $a5 -; LA64-NEXT: mulh.du $t0, $a0, $a2 -; LA64-NEXT: mul.d $t1, $a1, $a2 -; LA64-NEXT: add.d $t0, $t1, $t0 -; LA64-NEXT: sltu $t1, $t0, $t1 -; LA64-NEXT: mulh.du $t2, $a1, $a2 -; LA64-NEXT: add.d $t1, $t2, $t1 -; LA64-NEXT: mul.d $t2, $a0, $a3 -; LA64-NEXT: add.d $t0, $t2, $t0 -; LA64-NEXT: sltu $t2, $t0, $t2 -; LA64-NEXT: mulh.du $t3, $a0, $a3 -; LA64-NEXT: add.d $t2, $t3, $t2 +; LA64-NEXT: sltu $a7, $a5, $a7 +; LA64-NEXT: mulh.du $t0, $a0, $a3 +; LA64-NEXT: add.d $a7, $t0, $a7 +; LA64-NEXT: add.d $a7, $a6, $a7 +; LA64-NEXT: mul.d $t0, $a1, $a3 +; LA64-NEXT: add.d $t1, $t0, $a7 +; LA64-NEXT: srai.d $t2, $a1, 63 +; LA64-NEXT: mul.d $t3, $a2, $t2 +; LA64-NEXT: srai.d $t4, $a3, 63 +; LA64-NEXT: mul.d $t5, $t4, $a0 +; LA64-NEXT: add.d $t6, $t5, $t3 +; LA64-NEXT: add.d $t7, $t1, $t6 +; LA64-NEXT: sltu $t8, $t7, $t1 +; LA64-NEXT: sltu $t0, $t1, $t0 +; LA64-NEXT: sltu $a6, $a7, $a6 +; LA64-NEXT: mulh.du $a7, $a1, $a3 ; LA64-NEXT: add.d $a6, $a7, $a6 -; LA64-NEXT: sltu $a7, $a6, $a7 -; LA64-NEXT: add.d $a5, $a5, $a7 +; LA64-NEXT: add.d $a6, $a6, $t0 +; LA64-NEXT: mulh.du $a7, $a2, $t2 +; LA64-NEXT: add.d $a7, $a7, $t3 +; LA64-NEXT: mul.d $a3, $a3, $t2 +; LA64-NEXT: add.d $a3, $a7, $a3 +; LA64-NEXT: mul.d $a1, $t4, $a1 +; LA64-NEXT: mulh.du $a7, $t4, $a0 +; LA64-NEXT: add.d $a1, $a7, $a1 +; LA64-NEXT: add.d $a1, $a1, $t5 +; LA64-NEXT: add.d $a1, $a1, $a3 +; LA64-NEXT: sltu $a3, $t6, $t5 +; LA64-NEXT: add.d $a1, $a1, $a3 +; LA64-NEXT: add.d $a1, $a6, $a1 +; LA64-NEXT: add.d $a1, $a1, $t8 +; LA64-NEXT: srai.d $a3, $a5, 63 +; LA64-NEXT: xor $a1, $a1, $a3 +; LA64-NEXT: xor $a3, $t7, $a3 +; LA64-NEXT: or $a1, $a3, $a1 +; LA64-NEXT: sltu $a1, $zero, $a1 ; LA64-NEXT: mul.d $a0, $a0, $a2 -; LA64-NEXT: mul.d $a2, $a1, $a3 -; LA64-NEXT: mulh.du $a1, $a1, $a3 -; LA64-NEXT: add.d $a3, $t1, $t2 -; LA64-NEXT: sltu $a7, $a3, $t1 -; LA64-NEXT: add.d $a1, $a1, $a7 ; LA64-NEXT: st.d $a0, $a4, 0 -; LA64-NEXT: add.d $a0, $a2, $a3 -; LA64-NEXT: sltu $a2, $a0, $a2 -; LA64-NEXT: add.d $a1, $a1, $a2 -; LA64-NEXT: st.d $t0, $a4, 8 -; LA64-NEXT: add.d $a1, $a1, $a5 -; LA64-NEXT: add.d $a2, $a0, $a6 -; LA64-NEXT: sltu $a0, $a2, $a0 -; LA64-NEXT: add.d $a0, $a1, $a0 -; LA64-NEXT: srai.d $a1, $t0, 63 -; LA64-NEXT: xor $a0, $a0, $a1 -; LA64-NEXT: xor $a1, $a2, $a1 -; LA64-NEXT: or $a0, $a1, $a0 -; LA64-NEXT: sltu $a0, $zero, $a0 +; LA64-NEXT: st.d $a5, $a4, 8 +; LA64-NEXT: move $a0, $a1 ; LA64-NEXT: ret %t = call {i128, i1} @llvm.smul.with.overflow.i128(i128 %v1, i128 %v2) %val = extractvalue {i128, i1} %t, 0 diff --git a/llvm/test/CodeGen/LoongArch/soft-fp-to-int.ll b/llvm/test/CodeGen/LoongArch/soft-fp-to-int.ll index d12cbaaabb98..e6df401f0fef 100644 --- a/llvm/test/CodeGen/LoongArch/soft-fp-to-int.ll +++ b/llvm/test/CodeGen/LoongArch/soft-fp-to-int.ll @@ -7,15 +7,15 @@ define i32 @fptosi_i32_fp128(fp128 %X) nounwind { ; LA32: # %bb.0: ; LA32-NEXT: addi.w $sp, $sp, -32 ; LA32-NEXT: st.w $ra, $sp, 28 # 4-byte Folded Spill -; LA32-NEXT: ld.w $a1, $a0, 12 -; LA32-NEXT: st.w $a1, $sp, 20 -; LA32-NEXT: ld.w $a1, $a0, 8 -; LA32-NEXT: st.w $a1, $sp, 16 -; LA32-NEXT: ld.w $a1, $a0, 4 -; LA32-NEXT: st.w $a1, $sp, 12 -; LA32-NEXT: ld.w $a0, $a0, 0 -; LA32-NEXT: st.w $a0, $sp, 8 +; LA32-NEXT: ld.w $a1, $a0, 0 +; LA32-NEXT: ld.w $a2, $a0, 4 +; LA32-NEXT: ld.w $a3, $a0, 8 +; LA32-NEXT: ld.w $a0, $a0, 12 +; LA32-NEXT: st.w $a0, $sp, 20 +; LA32-NEXT: st.w $a3, $sp, 16 +; LA32-NEXT: st.w $a2, $sp, 12 ; LA32-NEXT: addi.w $a0, $sp, 8 +; LA32-NEXT: st.w $a1, $sp, 8 ; LA32-NEXT: bl %plt(__fixtfsi) ; LA32-NEXT: ld.w $ra, $sp, 28 # 4-byte Folded Reload ; LA32-NEXT: addi.w $sp, $sp, 32 @@ -83,15 +83,15 @@ define i64 @fptosi_i64_fp128(fp128 %X) nounwind { ; LA32: # %bb.0: ; LA32-NEXT: addi.w $sp, $sp, -32 ; LA32-NEXT: st.w $ra, $sp, 28 # 4-byte Folded Spill -; LA32-NEXT: ld.w $a1, $a0, 12 -; LA32-NEXT: st.w $a1, $sp, 12 -; LA32-NEXT: ld.w $a1, $a0, 8 -; LA32-NEXT: st.w $a1, $sp, 8 -; LA32-NEXT: ld.w $a1, $a0, 4 -; LA32-NEXT: st.w $a1, $sp, 4 -; LA32-NEXT: ld.w $a0, $a0, 0 -; LA32-NEXT: st.w $a0, $sp, 0 +; LA32-NEXT: ld.w $a1, $a0, 0 +; LA32-NEXT: ld.w $a2, $a0, 4 +; LA32-NEXT: ld.w $a3, $a0, 8 +; LA32-NEXT: ld.w $a0, $a0, 12 +; LA32-NEXT: st.w $a0, $sp, 12 +; LA32-NEXT: st.w $a3, $sp, 8 +; LA32-NEXT: st.w $a2, $sp, 4 ; LA32-NEXT: addi.w $a0, $sp, 0 +; LA32-NEXT: st.w $a1, $sp, 0 ; LA32-NEXT: bl %plt(__fixtfdi) ; LA32-NEXT: ld.w $ra, $sp, 28 # 4-byte Folded Reload ; LA32-NEXT: addi.w $sp, $sp, 32 diff --git a/llvm/test/CodeGen/LoongArch/spill-ra-without-kill.ll b/llvm/test/CodeGen/LoongArch/spill-ra-without-kill.ll index 092da5aba2d9..7a52697d1529 100644 --- a/llvm/test/CodeGen/LoongArch/spill-ra-without-kill.ll +++ b/llvm/test/CodeGen/LoongArch/spill-ra-without-kill.ll @@ -19,13 +19,13 @@ define dso_local ptr @f(i32 noundef signext %i) "frame-pointer"="all" { ; CHECK-NEXT: addi.d $fp, $sp, 48 ; CHECK-NEXT: .cfi_def_cfa 22, 0 ; CHECK-NEXT: st.d $ra, $fp, -40 # 8-byte Folded Spill -; CHECK-NEXT: move $a1, $a0 +; CHECK-NEXT: # kill: def $r5 killed $r4 +; CHECK-NEXT: st.w $a0, $fp, -28 ; CHECK-NEXT: pcalau12i $a0, %pc_hi20(calls) -; CHECK-NEXT: addi.d $a3, $a0, %pc_lo12(calls) -; CHECK-NEXT: ld.w $a0, $a3, 0 -; CHECK-NEXT: addi.d $a2, $a0, 1 -; CHECK-NEXT: st.w $a2, $a3, 0 -; CHECK-NEXT: st.w $a1, $fp, -28 +; CHECK-NEXT: addi.d $a2, $a0, %pc_lo12(calls) +; CHECK-NEXT: ld.w $a0, $a2, 0 +; CHECK-NEXT: addi.d $a1, $a0, 1 +; CHECK-NEXT: st.w $a1, $a2, 0 ; CHECK-NEXT: bnez $a0, .LBB0_2 ; CHECK-NEXT: b .LBB0_1 ; CHECK-NEXT: .LBB0_1: # %if.then diff --git a/llvm/test/CodeGen/LoongArch/spill-reload-cfr.ll b/llvm/test/CodeGen/LoongArch/spill-reload-cfr.ll index 63407ad003f6..8dd95befb827 100644 --- a/llvm/test/CodeGen/LoongArch/spill-reload-cfr.ll +++ b/llvm/test/CodeGen/LoongArch/spill-reload-cfr.ll @@ -8,74 +8,60 @@ declare void @foo() define i1 @load_store_fcc_reg(float %a, i1 %c) { ; LA32-LABEL: load_store_fcc_reg: ; LA32: # %bb.0: -; LA32-NEXT: addi.w $sp, $sp, -32 -; LA32-NEXT: .cfi_def_cfa_offset 32 -; LA32-NEXT: st.w $ra, $sp, 28 # 4-byte Folded Spill -; LA32-NEXT: st.w $fp, $sp, 24 # 4-byte Folded Spill -; LA32-NEXT: fst.d $fs0, $sp, 16 # 8-byte Folded Spill -; LA32-NEXT: fst.d $fs1, $sp, 8 # 8-byte Folded Spill +; LA32-NEXT: addi.w $sp, $sp, -16 +; LA32-NEXT: .cfi_def_cfa_offset 16 +; LA32-NEXT: st.w $ra, $sp, 12 # 4-byte Folded Spill +; LA32-NEXT: st.w $fp, $sp, 8 # 4-byte Folded Spill +; LA32-NEXT: fst.d $fs0, $sp, 0 # 8-byte Folded Spill ; LA32-NEXT: .cfi_offset 1, -4 ; LA32-NEXT: .cfi_offset 22, -8 ; LA32-NEXT: .cfi_offset 56, -16 -; LA32-NEXT: .cfi_offset 57, -24 ; LA32-NEXT: move $fp, $a0 ; LA32-NEXT: fmov.s $fs0, $fa0 -; LA32-NEXT: movgr2fr.w $fs1, $zero -; LA32-NEXT: fcmp.cult.s $fcc0, $fs1, $fa0 -; LA32-NEXT: movcf2gr $a0, $fcc0 -; LA32-NEXT: st.w $a0, $sp, 4 ; LA32-NEXT: bl %plt(foo) -; LA32-NEXT: ld.w $a0, $sp, 4 -; LA32-NEXT: movgr2cf $fcc0, $a0 +; LA32-NEXT: movgr2fr.w $fa0, $zero +; LA32-NEXT: fcmp.cult.s $fcc0, $fa0, $fs0 ; LA32-NEXT: bcnez $fcc0, .LBB0_2 ; LA32-NEXT: # %bb.1: # %if.then ; LA32-NEXT: move $a0, $fp ; LA32-NEXT: b .LBB0_3 ; LA32-NEXT: .LBB0_2: # %if.else -; LA32-NEXT: fcmp.cle.s $fcc0, $fs0, $fs1 +; LA32-NEXT: fcmp.cle.s $fcc0, $fs0, $fa0 ; LA32-NEXT: movcf2gr $a0, $fcc0 ; LA32-NEXT: .LBB0_3: # %if.then -; LA32-NEXT: fld.d $fs1, $sp, 8 # 8-byte Folded Reload -; LA32-NEXT: fld.d $fs0, $sp, 16 # 8-byte Folded Reload -; LA32-NEXT: ld.w $fp, $sp, 24 # 4-byte Folded Reload -; LA32-NEXT: ld.w $ra, $sp, 28 # 4-byte Folded Reload -; LA32-NEXT: addi.w $sp, $sp, 32 +; LA32-NEXT: fld.d $fs0, $sp, 0 # 8-byte Folded Reload +; LA32-NEXT: ld.w $fp, $sp, 8 # 4-byte Folded Reload +; LA32-NEXT: ld.w $ra, $sp, 12 # 4-byte Folded Reload +; LA32-NEXT: addi.w $sp, $sp, 16 ; LA32-NEXT: ret ; ; LA64-LABEL: load_store_fcc_reg: ; LA64: # %bb.0: -; LA64-NEXT: addi.d $sp, $sp, -48 -; LA64-NEXT: .cfi_def_cfa_offset 48 -; LA64-NEXT: st.d $ra, $sp, 40 # 8-byte Folded Spill -; LA64-NEXT: st.d $fp, $sp, 32 # 8-byte Folded Spill -; LA64-NEXT: fst.d $fs0, $sp, 24 # 8-byte Folded Spill -; LA64-NEXT: fst.d $fs1, $sp, 16 # 8-byte Folded Spill +; LA64-NEXT: addi.d $sp, $sp, -32 +; LA64-NEXT: .cfi_def_cfa_offset 32 +; LA64-NEXT: st.d $ra, $sp, 24 # 8-byte Folded Spill +; LA64-NEXT: st.d $fp, $sp, 16 # 8-byte Folded Spill +; LA64-NEXT: fst.d $fs0, $sp, 8 # 8-byte Folded Spill ; LA64-NEXT: .cfi_offset 1, -8 ; LA64-NEXT: .cfi_offset 22, -16 ; LA64-NEXT: .cfi_offset 56, -24 -; LA64-NEXT: .cfi_offset 57, -32 ; LA64-NEXT: move $fp, $a0 ; LA64-NEXT: fmov.s $fs0, $fa0 -; LA64-NEXT: movgr2fr.w $fs1, $zero -; LA64-NEXT: fcmp.cult.s $fcc0, $fs1, $fa0 -; LA64-NEXT: movcf2gr $a0, $fcc0 -; LA64-NEXT: st.d $a0, $sp, 8 ; LA64-NEXT: bl %plt(foo) -; LA64-NEXT: ld.d $a0, $sp, 8 -; LA64-NEXT: movgr2cf $fcc0, $a0 +; LA64-NEXT: movgr2fr.w $fa0, $zero +; LA64-NEXT: fcmp.cult.s $fcc0, $fa0, $fs0 ; LA64-NEXT: bcnez $fcc0, .LBB0_2 ; LA64-NEXT: # %bb.1: # %if.then ; LA64-NEXT: move $a0, $fp ; LA64-NEXT: b .LBB0_3 ; LA64-NEXT: .LBB0_2: # %if.else -; LA64-NEXT: fcmp.cle.s $fcc0, $fs0, $fs1 +; LA64-NEXT: fcmp.cle.s $fcc0, $fs0, $fa0 ; LA64-NEXT: movcf2gr $a0, $fcc0 ; LA64-NEXT: .LBB0_3: # %if.then -; LA64-NEXT: fld.d $fs1, $sp, 16 # 8-byte Folded Reload -; LA64-NEXT: fld.d $fs0, $sp, 24 # 8-byte Folded Reload -; LA64-NEXT: ld.d $fp, $sp, 32 # 8-byte Folded Reload -; LA64-NEXT: ld.d $ra, $sp, 40 # 8-byte Folded Reload -; LA64-NEXT: addi.d $sp, $sp, 48 +; LA64-NEXT: fld.d $fs0, $sp, 8 # 8-byte Folded Reload +; LA64-NEXT: ld.d $fp, $sp, 16 # 8-byte Folded Reload +; LA64-NEXT: ld.d $ra, $sp, 24 # 8-byte Folded Reload +; LA64-NEXT: addi.d $sp, $sp, 32 ; LA64-NEXT: ret %cmp = fcmp ole float %a, 0.000000e+00 call void @foo() diff --git a/llvm/test/CodeGen/LoongArch/tail-calls.ll b/llvm/test/CodeGen/LoongArch/tail-calls.ll index 52bdd230816f..c22a65c77e70 100644 --- a/llvm/test/CodeGen/LoongArch/tail-calls.ll +++ b/llvm/test/CodeGen/LoongArch/tail-calls.ll @@ -19,10 +19,12 @@ declare void @llvm.memcpy.p0.p0.i32(ptr, ptr, i32, i1) define void @caller_extern(ptr %src) optsize { ; CHECK-LABEL: caller_extern: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: move $a1, $a0 -; CHECK-NEXT: pcalau12i $a0, %got_pc_hi20(dest) -; CHECK-NEXT: ld.d $a0, $a0, %got_pc_lo12(dest) +; CHECK-NEXT: pcalau12i $a1, %got_pc_hi20(dest) +; CHECK-NEXT: ld.d $a1, $a1, %got_pc_lo12(dest) ; CHECK-NEXT: ori $a2, $zero, 33 +; CHECK-NEXT: move $a3, $a0 +; CHECK-NEXT: move $a0, $a1 +; CHECK-NEXT: move $a1, $a3 ; CHECK-NEXT: b %plt(memcpy) entry: tail call void @llvm.memcpy.p0.p0.i32(ptr @dest, ptr %src, i32 33, i1 false) @@ -35,13 +37,13 @@ declare void @callee_indirect2() define void @caller_indirect_tail(i32 %a) nounwind { ; CHECK-LABEL: caller_indirect_tail: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: addi.w $a0, $a0, 0 -; CHECK-NEXT: sltui $a0, $a0, 1 ; CHECK-NEXT: pcalau12i $a1, %got_pc_hi20(callee_indirect2) ; CHECK-NEXT: ld.d $a1, $a1, %got_pc_lo12(callee_indirect2) -; CHECK-NEXT: masknez $a1, $a1, $a0 ; CHECK-NEXT: pcalau12i $a2, %got_pc_hi20(callee_indirect1) ; CHECK-NEXT: ld.d $a2, $a2, %got_pc_lo12(callee_indirect1) +; CHECK-NEXT: addi.w $a0, $a0, 0 +; CHECK-NEXT: sltui $a0, $a0, 1 +; CHECK-NEXT: masknez $a1, $a1, $a0 ; CHECK-NEXT: maskeqz $a0, $a2, $a0 ; CHECK-NEXT: or $a0, $a0, $a1 ; CHECK-NEXT: jr $a0 @@ -103,9 +105,9 @@ define void @caller_indirect_args() nounwind { ; CHECK-NEXT: st.d $zero, $sp, 24 ; CHECK-NEXT: st.d $zero, $sp, 16 ; CHECK-NEXT: st.d $zero, $sp, 8 -; CHECK-NEXT: ori $a0, $zero, 1 -; CHECK-NEXT: st.d $a0, $sp, 0 +; CHECK-NEXT: ori $a1, $zero, 1 ; CHECK-NEXT: addi.d $a0, $sp, 0 +; CHECK-NEXT: st.d $a1, $sp, 0 ; CHECK-NEXT: bl %plt(callee_indirect_args) ; CHECK-NEXT: ld.d $ra, $sp, 40 # 8-byte Folded Reload ; CHECK-NEXT: addi.d $sp, $sp, 48 diff --git a/llvm/test/CodeGen/LoongArch/unaligned-access.ll b/llvm/test/CodeGen/LoongArch/unaligned-access.ll index 871c17f06e96..dd5b585fcca5 100644 --- a/llvm/test/CodeGen/LoongArch/unaligned-access.ll +++ b/llvm/test/CodeGen/LoongArch/unaligned-access.ll @@ -13,10 +13,10 @@ define i32 @f0(ptr %p) nounwind { ; LA32-ALIGNED-LABEL: f0: ; LA32-ALIGNED: # %bb.0: -; LA32-ALIGNED-NEXT: ld.hu $a1, $a0, 0 -; LA32-ALIGNED-NEXT: ld.hu $a0, $a0, 2 -; LA32-ALIGNED-NEXT: slli.w $a0, $a0, 16 -; LA32-ALIGNED-NEXT: or $a0, $a0, $a1 +; LA32-ALIGNED-NEXT: ld.hu $a1, $a0, 2 +; LA32-ALIGNED-NEXT: ld.hu $a0, $a0, 0 +; LA32-ALIGNED-NEXT: slli.w $a1, $a1, 16 +; LA32-ALIGNED-NEXT: or $a0, $a1, $a0 ; LA32-ALIGNED-NEXT: ret ; ; LA32-UNALIGNED-LABEL: f0: @@ -31,10 +31,10 @@ define i32 @f0(ptr %p) nounwind { ; ; LA64-ALIGNED-LABEL: f0: ; LA64-ALIGNED: # %bb.0: -; LA64-ALIGNED-NEXT: ld.hu $a1, $a0, 0 -; LA64-ALIGNED-NEXT: ld.h $a0, $a0, 2 -; LA64-ALIGNED-NEXT: slli.d $a0, $a0, 16 -; LA64-ALIGNED-NEXT: or $a0, $a0, $a1 +; LA64-ALIGNED-NEXT: ld.h $a1, $a0, 2 +; LA64-ALIGNED-NEXT: ld.hu $a0, $a0, 0 +; LA64-ALIGNED-NEXT: slli.d $a1, $a1, 16 +; LA64-ALIGNED-NEXT: or $a0, $a1, $a0 ; LA64-ALIGNED-NEXT: ret %tmp = load i32, ptr %p, align 2 ret i32 %tmp @@ -62,10 +62,10 @@ define i64 @f1(ptr %p) nounwind { ; ; LA64-ALIGNED-LABEL: f1: ; LA64-ALIGNED: # %bb.0: -; LA64-ALIGNED-NEXT: ld.wu $a1, $a0, 0 -; LA64-ALIGNED-NEXT: ld.wu $a0, $a0, 4 -; LA64-ALIGNED-NEXT: slli.d $a0, $a0, 32 -; LA64-ALIGNED-NEXT: or $a0, $a0, $a1 +; LA64-ALIGNED-NEXT: ld.wu $a1, $a0, 4 +; LA64-ALIGNED-NEXT: ld.wu $a0, $a0, 0 +; LA64-ALIGNED-NEXT: slli.d $a1, $a1, 32 +; LA64-ALIGNED-NEXT: or $a0, $a1, $a0 ; LA64-ALIGNED-NEXT: ret %tmp = load i64, ptr %p, align 4 ret i64 %tmp diff --git a/llvm/test/CodeGen/LoongArch/vararg.ll b/llvm/test/CodeGen/LoongArch/vararg.ll index a377628c3d78..112459ab1f4d 100644 --- a/llvm/test/CodeGen/LoongArch/vararg.ll +++ b/llvm/test/CodeGen/LoongArch/vararg.ll @@ -22,9 +22,9 @@ define i64 @va1(ptr %fmt, ...) { ; LA64-FPELIM-NEXT: st.d $a4, $sp, 48 ; LA64-FPELIM-NEXT: st.d $a3, $sp, 40 ; LA64-FPELIM-NEXT: st.d $a2, $sp, 32 +; LA64-FPELIM-NEXT: st.d $a1, $sp, 24 ; LA64-FPELIM-NEXT: addi.d $a1, $sp, 32 ; LA64-FPELIM-NEXT: st.d $a1, $sp, 8 -; LA64-FPELIM-NEXT: st.d $a0, $sp, 24 ; LA64-FPELIM-NEXT: addi.d $sp, $sp, 80 ; LA64-FPELIM-NEXT: ret ; @@ -45,9 +45,9 @@ define i64 @va1(ptr %fmt, ...) { ; LA64-WITHFP-NEXT: st.d $a4, $fp, 32 ; LA64-WITHFP-NEXT: st.d $a3, $fp, 24 ; LA64-WITHFP-NEXT: st.d $a2, $fp, 16 +; LA64-WITHFP-NEXT: st.d $a1, $fp, 8 ; LA64-WITHFP-NEXT: addi.d $a1, $fp, 16 ; LA64-WITHFP-NEXT: st.d $a1, $fp, -24 -; LA64-WITHFP-NEXT: st.d $a0, $fp, 8 ; LA64-WITHFP-NEXT: ld.d $fp, $sp, 16 # 8-byte Folded Reload ; LA64-WITHFP-NEXT: ld.d $ra, $sp, 24 # 8-byte Folded Reload ; LA64-WITHFP-NEXT: addi.d $sp, $sp, 96 @@ -73,9 +73,9 @@ define i64 @va1_va_arg(ptr %fmt, ...) nounwind { ; LA64-FPELIM-NEXT: st.d $a4, $sp, 48 ; LA64-FPELIM-NEXT: st.d $a3, $sp, 40 ; LA64-FPELIM-NEXT: st.d $a2, $sp, 32 +; LA64-FPELIM-NEXT: st.d $a1, $sp, 24 ; LA64-FPELIM-NEXT: addi.d $a1, $sp, 32 ; LA64-FPELIM-NEXT: st.d $a1, $sp, 8 -; LA64-FPELIM-NEXT: st.d $a0, $sp, 24 ; LA64-FPELIM-NEXT: addi.d $sp, $sp, 80 ; LA64-FPELIM-NEXT: ret ; @@ -92,9 +92,9 @@ define i64 @va1_va_arg(ptr %fmt, ...) nounwind { ; LA64-WITHFP-NEXT: st.d $a4, $fp, 32 ; LA64-WITHFP-NEXT: st.d $a3, $fp, 24 ; LA64-WITHFP-NEXT: st.d $a2, $fp, 16 +; LA64-WITHFP-NEXT: st.d $a1, $fp, 8 ; LA64-WITHFP-NEXT: addi.d $a1, $fp, 16 ; LA64-WITHFP-NEXT: st.d $a1, $fp, -24 -; LA64-WITHFP-NEXT: st.d $a0, $fp, 8 ; LA64-WITHFP-NEXT: ld.d $fp, $sp, 16 # 8-byte Folded Reload ; LA64-WITHFP-NEXT: ld.d $ra, $sp, 24 # 8-byte Folded Reload ; LA64-WITHFP-NEXT: addi.d $sp, $sp, 96 @@ -124,9 +124,9 @@ define i64 @va1_va_arg_alloca(ptr %fmt, ...) nounwind { ; LA64-FPELIM-NEXT: st.d $a4, $fp, 32 ; LA64-FPELIM-NEXT: st.d $a3, $fp, 24 ; LA64-FPELIM-NEXT: st.d $a2, $fp, 16 +; LA64-FPELIM-NEXT: st.d $a1, $fp, 8 ; LA64-FPELIM-NEXT: addi.d $a0, $fp, 16 ; LA64-FPELIM-NEXT: st.d $a0, $fp, -32 -; LA64-FPELIM-NEXT: st.d $a1, $fp, 8 ; LA64-FPELIM-NEXT: addi.d $a0, $a1, 15 ; LA64-FPELIM-NEXT: bstrins.d $a0, $zero, 3, 0 ; LA64-FPELIM-NEXT: sub.d $a0, $sp, $a0 @@ -154,9 +154,9 @@ define i64 @va1_va_arg_alloca(ptr %fmt, ...) nounwind { ; LA64-WITHFP-NEXT: st.d $a4, $fp, 32 ; LA64-WITHFP-NEXT: st.d $a3, $fp, 24 ; LA64-WITHFP-NEXT: st.d $a2, $fp, 16 +; LA64-WITHFP-NEXT: st.d $a1, $fp, 8 ; LA64-WITHFP-NEXT: addi.d $a0, $fp, 16 ; LA64-WITHFP-NEXT: st.d $a0, $fp, -32 -; LA64-WITHFP-NEXT: st.d $a1, $fp, 8 ; LA64-WITHFP-NEXT: addi.d $a0, $a1, 15 ; LA64-WITHFP-NEXT: bstrins.d $a0, $zero, 3, 0 ; LA64-WITHFP-NEXT: sub.d $a0, $sp, $a0 @@ -288,16 +288,16 @@ define void @va_aligned_stack_caller() nounwind { ; LA64-FPELIM-NEXT: lu32i.d $a0, 335544 ; LA64-FPELIM-NEXT: lu52i.d $a0, $a0, -328 ; LA64-FPELIM-NEXT: st.d $a0, $sp, 16 -; LA64-FPELIM-NEXT: ori $a0, $zero, 1000 -; LA64-FPELIM-NEXT: st.d $a0, $sp, 64 ; LA64-FPELIM-NEXT: st.d $zero, $sp, 88 ; LA64-FPELIM-NEXT: st.d $zero, $sp, 80 ; LA64-FPELIM-NEXT: st.d $zero, $sp, 72 +; LA64-FPELIM-NEXT: ori $a5, $zero, 1000 ; LA64-FPELIM-NEXT: ori $a1, $zero, 11 ; LA64-FPELIM-NEXT: addi.d $a2, $sp, 64 ; LA64-FPELIM-NEXT: ori $a3, $zero, 12 ; LA64-FPELIM-NEXT: ori $a4, $zero, 13 ; LA64-FPELIM-NEXT: ori $a0, $zero, 1 +; LA64-FPELIM-NEXT: st.d $a5, $sp, 64 ; LA64-FPELIM-NEXT: move $a6, $zero ; LA64-FPELIM-NEXT: move $a7, $a0 ; LA64-FPELIM-NEXT: bl %plt(va_aligned_stack_callee) @@ -329,16 +329,16 @@ define void @va_aligned_stack_caller() nounwind { ; LA64-WITHFP-NEXT: lu32i.d $a0, 335544 ; LA64-WITHFP-NEXT: lu52i.d $a0, $a0, -328 ; LA64-WITHFP-NEXT: st.d $a0, $sp, 16 -; LA64-WITHFP-NEXT: ori $a0, $zero, 1000 -; LA64-WITHFP-NEXT: st.d $a0, $fp, -48 ; LA64-WITHFP-NEXT: st.d $zero, $fp, -24 ; LA64-WITHFP-NEXT: st.d $zero, $fp, -32 ; LA64-WITHFP-NEXT: st.d $zero, $fp, -40 +; LA64-WITHFP-NEXT: ori $a5, $zero, 1000 ; LA64-WITHFP-NEXT: ori $a1, $zero, 11 ; LA64-WITHFP-NEXT: addi.d $a2, $fp, -48 ; LA64-WITHFP-NEXT: ori $a3, $zero, 12 ; LA64-WITHFP-NEXT: ori $a4, $zero, 13 ; LA64-WITHFP-NEXT: ori $a0, $zero, 1 +; LA64-WITHFP-NEXT: st.d $a5, $fp, -48 ; LA64-WITHFP-NEXT: move $a6, $zero ; LA64-WITHFP-NEXT: move $a7, $a0 ; LA64-WITHFP-NEXT: bl %plt(va_aligned_stack_callee) diff --git a/llvm/test/CodeGen/LoongArch/vector-fp-imm.ll b/llvm/test/CodeGen/LoongArch/vector-fp-imm.ll index 8009866d3953..d03af114bcee 100644 --- a/llvm/test/CodeGen/LoongArch/vector-fp-imm.ll +++ b/llvm/test/CodeGen/LoongArch/vector-fp-imm.ll @@ -17,69 +17,69 @@ define void @test_zero(ptr %P, ptr %S) nounwind { ; LA32F-LABEL: test_zero: ; LA32F: # %bb.0: ; LA32F-NEXT: fld.s $fa0, $a0, 12 -; LA32F-NEXT: movgr2fr.w $fa1, $zero -; LA32F-NEXT: fadd.s $fa0, $fa0, $fa1 +; LA32F-NEXT: fld.s $fa1, $a0, 0 +; LA32F-NEXT: fld.s $fa2, $a0, 4 +; LA32F-NEXT: fld.s $fa3, $a0, 8 +; LA32F-NEXT: movgr2fr.w $fa4, $zero +; LA32F-NEXT: fadd.s $fa1, $fa1, $fa4 +; LA32F-NEXT: fadd.s $fa2, $fa2, $fa4 +; LA32F-NEXT: fadd.s $fa3, $fa3, $fa4 +; LA32F-NEXT: fadd.s $fa0, $fa0, $fa4 ; LA32F-NEXT: fst.s $fa0, $a1, 12 -; LA32F-NEXT: fld.s $fa0, $a0, 8 -; LA32F-NEXT: fadd.s $fa0, $fa0, $fa1 -; LA32F-NEXT: fst.s $fa0, $a1, 8 -; LA32F-NEXT: fld.s $fa0, $a0, 4 -; LA32F-NEXT: fadd.s $fa0, $fa0, $fa1 -; LA32F-NEXT: fst.s $fa0, $a1, 4 -; LA32F-NEXT: fld.s $fa0, $a0, 0 -; LA32F-NEXT: fadd.s $fa0, $fa0, $fa1 -; LA32F-NEXT: fst.s $fa0, $a1, 0 +; LA32F-NEXT: fst.s $fa3, $a1, 8 +; LA32F-NEXT: fst.s $fa2, $a1, 4 +; LA32F-NEXT: fst.s $fa1, $a1, 0 ; LA32F-NEXT: ret ; ; LA32D-LABEL: test_zero: ; LA32D: # %bb.0: ; LA32D-NEXT: fld.s $fa0, $a0, 12 -; LA32D-NEXT: movgr2fr.w $fa1, $zero -; LA32D-NEXT: fadd.s $fa0, $fa0, $fa1 +; LA32D-NEXT: fld.s $fa1, $a0, 0 +; LA32D-NEXT: fld.s $fa2, $a0, 4 +; LA32D-NEXT: fld.s $fa3, $a0, 8 +; LA32D-NEXT: movgr2fr.w $fa4, $zero +; LA32D-NEXT: fadd.s $fa1, $fa1, $fa4 +; LA32D-NEXT: fadd.s $fa2, $fa2, $fa4 +; LA32D-NEXT: fadd.s $fa3, $fa3, $fa4 +; LA32D-NEXT: fadd.s $fa0, $fa0, $fa4 ; LA32D-NEXT: fst.s $fa0, $a1, 12 -; LA32D-NEXT: fld.s $fa0, $a0, 8 -; LA32D-NEXT: fadd.s $fa0, $fa0, $fa1 -; LA32D-NEXT: fst.s $fa0, $a1, 8 -; LA32D-NEXT: fld.s $fa0, $a0, 4 -; LA32D-NEXT: fadd.s $fa0, $fa0, $fa1 -; LA32D-NEXT: fst.s $fa0, $a1, 4 -; LA32D-NEXT: fld.s $fa0, $a0, 0 -; LA32D-NEXT: fadd.s $fa0, $fa0, $fa1 -; LA32D-NEXT: fst.s $fa0, $a1, 0 +; LA32D-NEXT: fst.s $fa3, $a1, 8 +; LA32D-NEXT: fst.s $fa2, $a1, 4 +; LA32D-NEXT: fst.s $fa1, $a1, 0 ; LA32D-NEXT: ret ; ; LA64F-LABEL: test_zero: ; LA64F: # %bb.0: ; LA64F-NEXT: fld.s $fa0, $a0, 12 -; LA64F-NEXT: movgr2fr.w $fa1, $zero -; LA64F-NEXT: fadd.s $fa0, $fa0, $fa1 +; LA64F-NEXT: fld.s $fa1, $a0, 0 +; LA64F-NEXT: fld.s $fa2, $a0, 4 +; LA64F-NEXT: fld.s $fa3, $a0, 8 +; LA64F-NEXT: movgr2fr.w $fa4, $zero +; LA64F-NEXT: fadd.s $fa1, $fa1, $fa4 +; LA64F-NEXT: fadd.s $fa2, $fa2, $fa4 +; LA64F-NEXT: fadd.s $fa3, $fa3, $fa4 +; LA64F-NEXT: fadd.s $fa0, $fa0, $fa4 ; LA64F-NEXT: fst.s $fa0, $a1, 12 -; LA64F-NEXT: fld.s $fa0, $a0, 8 -; LA64F-NEXT: fadd.s $fa0, $fa0, $fa1 -; LA64F-NEXT: fst.s $fa0, $a1, 8 -; LA64F-NEXT: fld.s $fa0, $a0, 4 -; LA64F-NEXT: fadd.s $fa0, $fa0, $fa1 -; LA64F-NEXT: fst.s $fa0, $a1, 4 -; LA64F-NEXT: fld.s $fa0, $a0, 0 -; LA64F-NEXT: fadd.s $fa0, $fa0, $fa1 -; LA64F-NEXT: fst.s $fa0, $a1, 0 +; LA64F-NEXT: fst.s $fa3, $a1, 8 +; LA64F-NEXT: fst.s $fa2, $a1, 4 +; LA64F-NEXT: fst.s $fa1, $a1, 0 ; LA64F-NEXT: ret ; ; LA64D-LABEL: test_zero: ; LA64D: # %bb.0: ; LA64D-NEXT: fld.s $fa0, $a0, 12 -; LA64D-NEXT: movgr2fr.w $fa1, $zero -; LA64D-NEXT: fadd.s $fa0, $fa0, $fa1 +; LA64D-NEXT: fld.s $fa1, $a0, 0 +; LA64D-NEXT: fld.s $fa2, $a0, 4 +; LA64D-NEXT: fld.s $fa3, $a0, 8 +; LA64D-NEXT: movgr2fr.w $fa4, $zero +; LA64D-NEXT: fadd.s $fa1, $fa1, $fa4 +; LA64D-NEXT: fadd.s $fa2, $fa2, $fa4 +; LA64D-NEXT: fadd.s $fa3, $fa3, $fa4 +; LA64D-NEXT: fadd.s $fa0, $fa0, $fa4 ; LA64D-NEXT: fst.s $fa0, $a1, 12 -; LA64D-NEXT: fld.s $fa0, $a0, 8 -; LA64D-NEXT: fadd.s $fa0, $fa0, $fa1 -; LA64D-NEXT: fst.s $fa0, $a1, 8 -; LA64D-NEXT: fld.s $fa0, $a0, 4 -; LA64D-NEXT: fadd.s $fa0, $fa0, $fa1 -; LA64D-NEXT: fst.s $fa0, $a1, 4 -; LA64D-NEXT: fld.s $fa0, $a0, 0 -; LA64D-NEXT: fadd.s $fa0, $fa0, $fa1 -; LA64D-NEXT: fst.s $fa0, $a1, 0 +; LA64D-NEXT: fst.s $fa3, $a1, 8 +; LA64D-NEXT: fst.s $fa2, $a1, 4 +; LA64D-NEXT: fst.s $fa1, $a1, 0 ; LA64D-NEXT: ret %p = load %f4, ptr %P %R = fadd %f4 %p, zeroinitializer @@ -90,66 +90,66 @@ define void @test_zero(ptr %P, ptr %S) nounwind { define void @test_f2(ptr %P, ptr %S) nounwind { ; LA32F-LABEL: test_f2: ; LA32F: # %bb.0: +; LA32F-NEXT: fld.s $fa0, $a0, 4 +; LA32F-NEXT: fld.s $fa1, $a0, 0 +; LA32F-NEXT: addi.w $a0, $zero, 1 ; LA32F-NEXT: pcalau12i $a2, %pc_hi20(.LCPI1_0) ; LA32F-NEXT: addi.w $a2, $a2, %pc_lo12(.LCPI1_0) -; LA32F-NEXT: fld.s $fa0, $a2, 0 -; LA32F-NEXT: fld.s $fa1, $a0, 4 -; LA32F-NEXT: fadd.s $fa0, $fa1, $fa0 +; LA32F-NEXT: fld.s $fa2, $a2, 0 +; LA32F-NEXT: movgr2fr.w $fa3, $a0 +; LA32F-NEXT: ffint.s.w $fa3, $fa3 +; LA32F-NEXT: fadd.s $fa1, $fa1, $fa3 +; LA32F-NEXT: fadd.s $fa0, $fa0, $fa2 ; LA32F-NEXT: fst.s $fa0, $a1, 4 -; LA32F-NEXT: fld.s $fa0, $a0, 0 -; LA32F-NEXT: addi.w $a0, $zero, 1 -; LA32F-NEXT: movgr2fr.w $fa1, $a0 -; LA32F-NEXT: ffint.s.w $fa1, $fa1 -; LA32F-NEXT: fadd.s $fa0, $fa0, $fa1 -; LA32F-NEXT: fst.s $fa0, $a1, 0 +; LA32F-NEXT: fst.s $fa1, $a1, 0 ; LA32F-NEXT: ret ; ; LA32D-LABEL: test_f2: ; LA32D: # %bb.0: +; LA32D-NEXT: fld.s $fa0, $a0, 4 +; LA32D-NEXT: fld.s $fa1, $a0, 0 +; LA32D-NEXT: addi.w $a0, $zero, 1 ; LA32D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI1_0) ; LA32D-NEXT: addi.w $a2, $a2, %pc_lo12(.LCPI1_0) -; LA32D-NEXT: fld.s $fa0, $a2, 0 -; LA32D-NEXT: fld.s $fa1, $a0, 4 -; LA32D-NEXT: fadd.s $fa0, $fa1, $fa0 +; LA32D-NEXT: fld.s $fa2, $a2, 0 +; LA32D-NEXT: movgr2fr.w $fa3, $a0 +; LA32D-NEXT: ffint.s.w $fa3, $fa3 +; LA32D-NEXT: fadd.s $fa1, $fa1, $fa3 +; LA32D-NEXT: fadd.s $fa0, $fa0, $fa2 ; LA32D-NEXT: fst.s $fa0, $a1, 4 -; LA32D-NEXT: fld.s $fa0, $a0, 0 -; LA32D-NEXT: addi.w $a0, $zero, 1 -; LA32D-NEXT: movgr2fr.w $fa1, $a0 -; LA32D-NEXT: ffint.s.w $fa1, $fa1 -; LA32D-NEXT: fadd.s $fa0, $fa0, $fa1 -; LA32D-NEXT: fst.s $fa0, $a1, 0 +; LA32D-NEXT: fst.s $fa1, $a1, 0 ; LA32D-NEXT: ret ; ; LA64F-LABEL: test_f2: ; LA64F: # %bb.0: -; LA64F-NEXT: pcalau12i $a2, %pc_hi20(.LCPI1_0) -; LA64F-NEXT: addi.d $a2, $a2, %pc_lo12(.LCPI1_0) -; LA64F-NEXT: fld.s $fa0, $a2, 0 -; LA64F-NEXT: fld.s $fa1, $a0, 4 -; LA64F-NEXT: fadd.s $fa0, $fa1, $fa0 -; LA64F-NEXT: fst.s $fa0, $a1, 4 -; LA64F-NEXT: fld.s $fa0, $a0, 0 +; LA64F-NEXT: fld.s $fa0, $a0, 4 +; LA64F-NEXT: fld.s $fa1, $a0, 0 +; LA64F-NEXT: pcalau12i $a0, %pc_hi20(.LCPI1_0) +; LA64F-NEXT: addi.d $a0, $a0, %pc_lo12(.LCPI1_0) +; LA64F-NEXT: fld.s $fa2, $a0, 0 ; LA64F-NEXT: addi.w $a0, $zero, 1 -; LA64F-NEXT: movgr2fr.w $fa1, $a0 -; LA64F-NEXT: ffint.s.w $fa1, $fa1 -; LA64F-NEXT: fadd.s $fa0, $fa0, $fa1 -; LA64F-NEXT: fst.s $fa0, $a1, 0 +; LA64F-NEXT: movgr2fr.w $fa3, $a0 +; LA64F-NEXT: ffint.s.w $fa3, $fa3 +; LA64F-NEXT: fadd.s $fa1, $fa1, $fa3 +; LA64F-NEXT: fadd.s $fa0, $fa0, $fa2 +; LA64F-NEXT: fst.s $fa0, $a1, 4 +; LA64F-NEXT: fst.s $fa1, $a1, 0 ; LA64F-NEXT: ret ; ; LA64D-LABEL: test_f2: ; LA64D: # %bb.0: -; LA64D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI1_0) -; LA64D-NEXT: addi.d $a2, $a2, %pc_lo12(.LCPI1_0) -; LA64D-NEXT: fld.s $fa0, $a2, 0 -; LA64D-NEXT: fld.s $fa1, $a0, 4 -; LA64D-NEXT: fadd.s $fa0, $fa1, $fa0 -; LA64D-NEXT: fst.s $fa0, $a1, 4 -; LA64D-NEXT: fld.s $fa0, $a0, 0 +; LA64D-NEXT: fld.s $fa0, $a0, 4 +; LA64D-NEXT: fld.s $fa1, $a0, 0 +; LA64D-NEXT: pcalau12i $a0, %pc_hi20(.LCPI1_0) +; LA64D-NEXT: addi.d $a0, $a0, %pc_lo12(.LCPI1_0) +; LA64D-NEXT: fld.s $fa2, $a0, 0 ; LA64D-NEXT: addi.w $a0, $zero, 1 -; LA64D-NEXT: movgr2fr.w $fa1, $a0 -; LA64D-NEXT: ffint.s.w $fa1, $fa1 -; LA64D-NEXT: fadd.s $fa0, $fa0, $fa1 -; LA64D-NEXT: fst.s $fa0, $a1, 0 +; LA64D-NEXT: movgr2fr.w $fa3, $a0 +; LA64D-NEXT: ffint.s.w $fa3, $fa3 +; LA64D-NEXT: fadd.s $fa1, $fa1, $fa3 +; LA64D-NEXT: fadd.s $fa0, $fa0, $fa2 +; LA64D-NEXT: fst.s $fa0, $a1, 4 +; LA64D-NEXT: fst.s $fa1, $a1, 0 ; LA64D-NEXT: ret %p = load %f2, ptr %P %R = fadd %f2 %p, < float 1.000000e+00, float 2.000000e+00 > @@ -160,114 +160,114 @@ define void @test_f2(ptr %P, ptr %S) nounwind { define void @test_f4(ptr %P, ptr %S) nounwind { ; LA32F-LABEL: test_f4: ; LA32F: # %bb.0: -; LA32F-NEXT: pcalau12i $a2, %pc_hi20(.LCPI2_0) -; LA32F-NEXT: addi.w $a2, $a2, %pc_lo12(.LCPI2_0) -; LA32F-NEXT: fld.s $fa0, $a2, 0 -; LA32F-NEXT: fld.s $fa1, $a0, 4 -; LA32F-NEXT: fadd.s $fa0, $fa1, $fa0 -; LA32F-NEXT: pcalau12i $a2, %pc_hi20(.LCPI2_1) -; LA32F-NEXT: addi.w $a2, $a2, %pc_lo12(.LCPI2_1) -; LA32F-NEXT: fld.s $fa1, $a2, 0 -; LA32F-NEXT: fld.s $fa2, $a0, 8 -; LA32F-NEXT: fadd.s $fa1, $fa2, $fa1 -; LA32F-NEXT: pcalau12i $a2, %pc_hi20(.LCPI2_2) -; LA32F-NEXT: addi.w $a2, $a2, %pc_lo12(.LCPI2_2) -; LA32F-NEXT: fld.s $fa2, $a2, 0 -; LA32F-NEXT: fld.s $fa3, $a0, 12 -; LA32F-NEXT: fadd.s $fa2, $fa3, $fa2 -; LA32F-NEXT: fst.s $fa2, $a1, 12 -; LA32F-NEXT: fst.s $fa1, $a1, 8 -; LA32F-NEXT: fst.s $fa0, $a1, 4 -; LA32F-NEXT: fld.s $fa0, $a0, 0 +; LA32F-NEXT: fld.s $fa0, $a0, 12 +; LA32F-NEXT: fld.s $fa1, $a0, 8 +; LA32F-NEXT: fld.s $fa2, $a0, 4 +; LA32F-NEXT: fld.s $fa3, $a0, 0 ; LA32F-NEXT: addi.w $a0, $zero, 1 -; LA32F-NEXT: movgr2fr.w $fa1, $a0 -; LA32F-NEXT: ffint.s.w $fa1, $fa1 -; LA32F-NEXT: fadd.s $fa0, $fa0, $fa1 -; LA32F-NEXT: fst.s $fa0, $a1, 0 +; LA32F-NEXT: movgr2fr.w $fa4, $a0 +; LA32F-NEXT: ffint.s.w $fa4, $fa4 +; LA32F-NEXT: pcalau12i $a0, %pc_hi20(.LCPI2_0) +; LA32F-NEXT: addi.w $a0, $a0, %pc_lo12(.LCPI2_0) +; LA32F-NEXT: fld.s $fa5, $a0, 0 +; LA32F-NEXT: pcalau12i $a0, %pc_hi20(.LCPI2_1) +; LA32F-NEXT: addi.w $a0, $a0, %pc_lo12(.LCPI2_1) +; LA32F-NEXT: fld.s $fa6, $a0, 0 +; LA32F-NEXT: pcalau12i $a0, %pc_hi20(.LCPI2_2) +; LA32F-NEXT: addi.w $a0, $a0, %pc_lo12(.LCPI2_2) +; LA32F-NEXT: fld.s $fa7, $a0, 0 +; LA32F-NEXT: fadd.s $fa3, $fa3, $fa4 +; LA32F-NEXT: fadd.s $fa2, $fa2, $fa5 +; LA32F-NEXT: fadd.s $fa1, $fa1, $fa6 +; LA32F-NEXT: fadd.s $fa0, $fa0, $fa7 +; LA32F-NEXT: fst.s $fa0, $a1, 12 +; LA32F-NEXT: fst.s $fa1, $a1, 8 +; LA32F-NEXT: fst.s $fa2, $a1, 4 +; LA32F-NEXT: fst.s $fa3, $a1, 0 ; LA32F-NEXT: ret ; ; LA32D-LABEL: test_f4: ; LA32D: # %bb.0: -; LA32D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI2_0) -; LA32D-NEXT: addi.w $a2, $a2, %pc_lo12(.LCPI2_0) -; LA32D-NEXT: fld.s $fa0, $a2, 0 -; LA32D-NEXT: fld.s $fa1, $a0, 4 -; LA32D-NEXT: fadd.s $fa0, $fa1, $fa0 -; LA32D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI2_1) -; LA32D-NEXT: addi.w $a2, $a2, %pc_lo12(.LCPI2_1) -; LA32D-NEXT: fld.s $fa1, $a2, 0 -; LA32D-NEXT: fld.s $fa2, $a0, 8 -; LA32D-NEXT: fadd.s $fa1, $fa2, $fa1 -; LA32D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI2_2) -; LA32D-NEXT: addi.w $a2, $a2, %pc_lo12(.LCPI2_2) -; LA32D-NEXT: fld.s $fa2, $a2, 0 -; LA32D-NEXT: fld.s $fa3, $a0, 12 -; LA32D-NEXT: fadd.s $fa2, $fa3, $fa2 -; LA32D-NEXT: fst.s $fa2, $a1, 12 -; LA32D-NEXT: fst.s $fa1, $a1, 8 -; LA32D-NEXT: fst.s $fa0, $a1, 4 -; LA32D-NEXT: fld.s $fa0, $a0, 0 +; LA32D-NEXT: fld.s $fa0, $a0, 12 +; LA32D-NEXT: fld.s $fa1, $a0, 8 +; LA32D-NEXT: fld.s $fa2, $a0, 4 +; LA32D-NEXT: fld.s $fa3, $a0, 0 ; LA32D-NEXT: addi.w $a0, $zero, 1 -; LA32D-NEXT: movgr2fr.w $fa1, $a0 -; LA32D-NEXT: ffint.s.w $fa1, $fa1 -; LA32D-NEXT: fadd.s $fa0, $fa0, $fa1 -; LA32D-NEXT: fst.s $fa0, $a1, 0 +; LA32D-NEXT: movgr2fr.w $fa4, $a0 +; LA32D-NEXT: ffint.s.w $fa4, $fa4 +; LA32D-NEXT: pcalau12i $a0, %pc_hi20(.LCPI2_0) +; LA32D-NEXT: addi.w $a0, $a0, %pc_lo12(.LCPI2_0) +; LA32D-NEXT: fld.s $fa5, $a0, 0 +; LA32D-NEXT: pcalau12i $a0, %pc_hi20(.LCPI2_1) +; LA32D-NEXT: addi.w $a0, $a0, %pc_lo12(.LCPI2_1) +; LA32D-NEXT: fld.s $fa6, $a0, 0 +; LA32D-NEXT: pcalau12i $a0, %pc_hi20(.LCPI2_2) +; LA32D-NEXT: addi.w $a0, $a0, %pc_lo12(.LCPI2_2) +; LA32D-NEXT: fld.s $fa7, $a0, 0 +; LA32D-NEXT: fadd.s $fa3, $fa3, $fa4 +; LA32D-NEXT: fadd.s $fa2, $fa2, $fa5 +; LA32D-NEXT: fadd.s $fa1, $fa1, $fa6 +; LA32D-NEXT: fadd.s $fa0, $fa0, $fa7 +; LA32D-NEXT: fst.s $fa0, $a1, 12 +; LA32D-NEXT: fst.s $fa1, $a1, 8 +; LA32D-NEXT: fst.s $fa2, $a1, 4 +; LA32D-NEXT: fst.s $fa3, $a1, 0 ; LA32D-NEXT: ret ; ; LA64F-LABEL: test_f4: ; LA64F: # %bb.0: -; LA64F-NEXT: pcalau12i $a2, %pc_hi20(.LCPI2_0) -; LA64F-NEXT: addi.d $a2, $a2, %pc_lo12(.LCPI2_0) -; LA64F-NEXT: fld.s $fa0, $a2, 0 -; LA64F-NEXT: fld.s $fa1, $a0, 4 -; LA64F-NEXT: fadd.s $fa0, $fa1, $fa0 -; LA64F-NEXT: pcalau12i $a2, %pc_hi20(.LCPI2_1) -; LA64F-NEXT: addi.d $a2, $a2, %pc_lo12(.LCPI2_1) -; LA64F-NEXT: fld.s $fa1, $a2, 0 -; LA64F-NEXT: fld.s $fa2, $a0, 8 -; LA64F-NEXT: fadd.s $fa1, $fa2, $fa1 -; LA64F-NEXT: pcalau12i $a2, %pc_hi20(.LCPI2_2) -; LA64F-NEXT: addi.d $a2, $a2, %pc_lo12(.LCPI2_2) -; LA64F-NEXT: fld.s $fa2, $a2, 0 -; LA64F-NEXT: fld.s $fa3, $a0, 12 -; LA64F-NEXT: fadd.s $fa2, $fa3, $fa2 -; LA64F-NEXT: fst.s $fa2, $a1, 12 -; LA64F-NEXT: fst.s $fa1, $a1, 8 -; LA64F-NEXT: fst.s $fa0, $a1, 4 -; LA64F-NEXT: fld.s $fa0, $a0, 0 +; LA64F-NEXT: fld.s $fa0, $a0, 12 +; LA64F-NEXT: fld.s $fa1, $a0, 8 +; LA64F-NEXT: fld.s $fa2, $a0, 4 +; LA64F-NEXT: fld.s $fa3, $a0, 0 ; LA64F-NEXT: addi.w $a0, $zero, 1 -; LA64F-NEXT: movgr2fr.w $fa1, $a0 -; LA64F-NEXT: ffint.s.w $fa1, $fa1 -; LA64F-NEXT: fadd.s $fa0, $fa0, $fa1 -; LA64F-NEXT: fst.s $fa0, $a1, 0 +; LA64F-NEXT: movgr2fr.w $fa4, $a0 +; LA64F-NEXT: ffint.s.w $fa4, $fa4 +; LA64F-NEXT: pcalau12i $a0, %pc_hi20(.LCPI2_0) +; LA64F-NEXT: addi.d $a0, $a0, %pc_lo12(.LCPI2_0) +; LA64F-NEXT: fld.s $fa5, $a0, 0 +; LA64F-NEXT: pcalau12i $a0, %pc_hi20(.LCPI2_1) +; LA64F-NEXT: addi.d $a0, $a0, %pc_lo12(.LCPI2_1) +; LA64F-NEXT: fld.s $fa6, $a0, 0 +; LA64F-NEXT: pcalau12i $a0, %pc_hi20(.LCPI2_2) +; LA64F-NEXT: addi.d $a0, $a0, %pc_lo12(.LCPI2_2) +; LA64F-NEXT: fld.s $fa7, $a0, 0 +; LA64F-NEXT: fadd.s $fa3, $fa3, $fa4 +; LA64F-NEXT: fadd.s $fa2, $fa2, $fa5 +; LA64F-NEXT: fadd.s $fa1, $fa1, $fa6 +; LA64F-NEXT: fadd.s $fa0, $fa0, $fa7 +; LA64F-NEXT: fst.s $fa0, $a1, 12 +; LA64F-NEXT: fst.s $fa1, $a1, 8 +; LA64F-NEXT: fst.s $fa2, $a1, 4 +; LA64F-NEXT: fst.s $fa3, $a1, 0 ; LA64F-NEXT: ret ; ; LA64D-LABEL: test_f4: ; LA64D: # %bb.0: -; LA64D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI2_0) -; LA64D-NEXT: addi.d $a2, $a2, %pc_lo12(.LCPI2_0) -; LA64D-NEXT: fld.s $fa0, $a2, 0 -; LA64D-NEXT: fld.s $fa1, $a0, 4 -; LA64D-NEXT: fadd.s $fa0, $fa1, $fa0 -; LA64D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI2_1) -; LA64D-NEXT: addi.d $a2, $a2, %pc_lo12(.LCPI2_1) -; LA64D-NEXT: fld.s $fa1, $a2, 0 -; LA64D-NEXT: fld.s $fa2, $a0, 8 -; LA64D-NEXT: fadd.s $fa1, $fa2, $fa1 -; LA64D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI2_2) -; LA64D-NEXT: addi.d $a2, $a2, %pc_lo12(.LCPI2_2) -; LA64D-NEXT: fld.s $fa2, $a2, 0 -; LA64D-NEXT: fld.s $fa3, $a0, 12 -; LA64D-NEXT: fadd.s $fa2, $fa3, $fa2 -; LA64D-NEXT: fst.s $fa2, $a1, 12 -; LA64D-NEXT: fst.s $fa1, $a1, 8 -; LA64D-NEXT: fst.s $fa0, $a1, 4 -; LA64D-NEXT: fld.s $fa0, $a0, 0 +; LA64D-NEXT: fld.s $fa0, $a0, 12 +; LA64D-NEXT: fld.s $fa1, $a0, 8 +; LA64D-NEXT: fld.s $fa2, $a0, 4 +; LA64D-NEXT: fld.s $fa3, $a0, 0 ; LA64D-NEXT: addi.w $a0, $zero, 1 -; LA64D-NEXT: movgr2fr.w $fa1, $a0 -; LA64D-NEXT: ffint.s.w $fa1, $fa1 -; LA64D-NEXT: fadd.s $fa0, $fa0, $fa1 -; LA64D-NEXT: fst.s $fa0, $a1, 0 +; LA64D-NEXT: movgr2fr.w $fa4, $a0 +; LA64D-NEXT: ffint.s.w $fa4, $fa4 +; LA64D-NEXT: pcalau12i $a0, %pc_hi20(.LCPI2_0) +; LA64D-NEXT: addi.d $a0, $a0, %pc_lo12(.LCPI2_0) +; LA64D-NEXT: fld.s $fa5, $a0, 0 +; LA64D-NEXT: pcalau12i $a0, %pc_hi20(.LCPI2_1) +; LA64D-NEXT: addi.d $a0, $a0, %pc_lo12(.LCPI2_1) +; LA64D-NEXT: fld.s $fa6, $a0, 0 +; LA64D-NEXT: pcalau12i $a0, %pc_hi20(.LCPI2_2) +; LA64D-NEXT: addi.d $a0, $a0, %pc_lo12(.LCPI2_2) +; LA64D-NEXT: fld.s $fa7, $a0, 0 +; LA64D-NEXT: fadd.s $fa3, $fa3, $fa4 +; LA64D-NEXT: fadd.s $fa2, $fa2, $fa5 +; LA64D-NEXT: fadd.s $fa1, $fa1, $fa6 +; LA64D-NEXT: fadd.s $fa0, $fa0, $fa7 +; LA64D-NEXT: fst.s $fa0, $a1, 12 +; LA64D-NEXT: fst.s $fa1, $a1, 8 +; LA64D-NEXT: fst.s $fa2, $a1, 4 +; LA64D-NEXT: fst.s $fa3, $a1, 0 ; LA64D-NEXT: ret %p = load %f4, ptr %P %R = fadd %f4 %p, < float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00 > @@ -278,162 +278,162 @@ define void @test_f4(ptr %P, ptr %S) nounwind { define void @test_f8(ptr %P, ptr %S) nounwind { ; LA32F-LABEL: test_f8: ; LA32F: # %bb.0: +; LA32F-NEXT: addi.w $a2, $zero, 1 +; LA32F-NEXT: movgr2fr.w $fa0, $a2 ; LA32F-NEXT: pcalau12i $a2, %pc_hi20(.LCPI3_0) ; LA32F-NEXT: addi.w $a2, $a2, %pc_lo12(.LCPI3_0) -; LA32F-NEXT: fld.s $fa0, $a2, 0 -; LA32F-NEXT: fld.s $fa1, $a0, 4 -; LA32F-NEXT: fadd.s $fa1, $fa1, $fa0 -; LA32F-NEXT: fld.s $fa2, $a0, 20 -; LA32F-NEXT: fadd.s $fa0, $fa2, $fa0 +; LA32F-NEXT: fld.s $fa1, $a2, 0 ; LA32F-NEXT: pcalau12i $a2, %pc_hi20(.LCPI3_1) ; LA32F-NEXT: addi.w $a2, $a2, %pc_lo12(.LCPI3_1) ; LA32F-NEXT: fld.s $fa2, $a2, 0 -; LA32F-NEXT: fld.s $fa3, $a0, 8 -; LA32F-NEXT: fadd.s $fa3, $fa3, $fa2 -; LA32F-NEXT: fld.s $fa4, $a0, 24 -; LA32F-NEXT: fadd.s $fa2, $fa4, $fa2 ; LA32F-NEXT: pcalau12i $a2, %pc_hi20(.LCPI3_2) ; LA32F-NEXT: addi.w $a2, $a2, %pc_lo12(.LCPI3_2) -; LA32F-NEXT: fld.s $fa4, $a2, 0 -; LA32F-NEXT: fld.s $fa5, $a0, 12 -; LA32F-NEXT: fadd.s $fa5, $fa5, $fa4 -; LA32F-NEXT: fld.s $fa6, $a0, 28 -; LA32F-NEXT: fadd.s $fa4, $fa6, $fa4 -; LA32F-NEXT: fst.s $fa4, $a1, 28 -; LA32F-NEXT: fst.s $fa2, $a1, 24 -; LA32F-NEXT: fst.s $fa0, $a1, 20 -; LA32F-NEXT: fst.s $fa5, $a1, 12 -; LA32F-NEXT: fst.s $fa3, $a1, 8 -; LA32F-NEXT: fst.s $fa1, $a1, 4 -; LA32F-NEXT: addi.w $a2, $zero, 1 -; LA32F-NEXT: movgr2fr.w $fa0, $a2 +; LA32F-NEXT: fld.s $fa3, $a2, 0 +; LA32F-NEXT: fld.s $fa4, $a0, 28 +; LA32F-NEXT: fld.s $fa5, $a0, 24 +; LA32F-NEXT: fld.s $fa6, $a0, 12 +; LA32F-NEXT: fld.s $fa7, $a0, 8 +; LA32F-NEXT: fld.s $ft0, $a0, 0 +; LA32F-NEXT: fld.s $ft1, $a0, 16 +; LA32F-NEXT: fld.s $ft2, $a0, 4 ; LA32F-NEXT: ffint.s.w $fa0, $fa0 -; LA32F-NEXT: fld.s $fa1, $a0, 16 -; LA32F-NEXT: fadd.s $fa1, $fa1, $fa0 -; LA32F-NEXT: fst.s $fa1, $a1, 16 -; LA32F-NEXT: fld.s $fa1, $a0, 0 -; LA32F-NEXT: fadd.s $fa0, $fa1, $fa0 -; LA32F-NEXT: fst.s $fa0, $a1, 0 +; LA32F-NEXT: fadd.s $ft0, $ft0, $fa0 +; LA32F-NEXT: fadd.s $fa0, $ft1, $fa0 +; LA32F-NEXT: fld.s $ft1, $a0, 20 +; LA32F-NEXT: fadd.s $ft2, $ft2, $fa1 +; LA32F-NEXT: fadd.s $fa7, $fa7, $fa2 +; LA32F-NEXT: fadd.s $fa6, $fa6, $fa3 +; LA32F-NEXT: fadd.s $fa1, $ft1, $fa1 +; LA32F-NEXT: fadd.s $fa2, $fa5, $fa2 +; LA32F-NEXT: fadd.s $fa3, $fa4, $fa3 +; LA32F-NEXT: fst.s $fa3, $a1, 28 +; LA32F-NEXT: fst.s $fa2, $a1, 24 +; LA32F-NEXT: fst.s $fa1, $a1, 20 +; LA32F-NEXT: fst.s $fa6, $a1, 12 +; LA32F-NEXT: fst.s $fa7, $a1, 8 +; LA32F-NEXT: fst.s $ft2, $a1, 4 +; LA32F-NEXT: fst.s $fa0, $a1, 16 +; LA32F-NEXT: fst.s $ft0, $a1, 0 ; LA32F-NEXT: ret ; ; LA32D-LABEL: test_f8: ; LA32D: # %bb.0: +; LA32D-NEXT: addi.w $a2, $zero, 1 +; LA32D-NEXT: movgr2fr.w $fa0, $a2 ; LA32D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI3_0) ; LA32D-NEXT: addi.w $a2, $a2, %pc_lo12(.LCPI3_0) -; LA32D-NEXT: fld.s $fa0, $a2, 0 -; LA32D-NEXT: fld.s $fa1, $a0, 4 -; LA32D-NEXT: fadd.s $fa1, $fa1, $fa0 -; LA32D-NEXT: fld.s $fa2, $a0, 20 -; LA32D-NEXT: fadd.s $fa0, $fa2, $fa0 +; LA32D-NEXT: fld.s $fa1, $a2, 0 ; LA32D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI3_1) ; LA32D-NEXT: addi.w $a2, $a2, %pc_lo12(.LCPI3_1) ; LA32D-NEXT: fld.s $fa2, $a2, 0 -; LA32D-NEXT: fld.s $fa3, $a0, 8 -; LA32D-NEXT: fadd.s $fa3, $fa3, $fa2 -; LA32D-NEXT: fld.s $fa4, $a0, 24 -; LA32D-NEXT: fadd.s $fa2, $fa4, $fa2 ; LA32D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI3_2) ; LA32D-NEXT: addi.w $a2, $a2, %pc_lo12(.LCPI3_2) -; LA32D-NEXT: fld.s $fa4, $a2, 0 -; LA32D-NEXT: fld.s $fa5, $a0, 12 -; LA32D-NEXT: fadd.s $fa5, $fa5, $fa4 -; LA32D-NEXT: fld.s $fa6, $a0, 28 -; LA32D-NEXT: fadd.s $fa4, $fa6, $fa4 -; LA32D-NEXT: fst.s $fa4, $a1, 28 -; LA32D-NEXT: fst.s $fa2, $a1, 24 -; LA32D-NEXT: fst.s $fa0, $a1, 20 -; LA32D-NEXT: fst.s $fa5, $a1, 12 -; LA32D-NEXT: fst.s $fa3, $a1, 8 -; LA32D-NEXT: fst.s $fa1, $a1, 4 -; LA32D-NEXT: addi.w $a2, $zero, 1 -; LA32D-NEXT: movgr2fr.w $fa0, $a2 +; LA32D-NEXT: fld.s $fa3, $a2, 0 +; LA32D-NEXT: fld.s $fa4, $a0, 28 +; LA32D-NEXT: fld.s $fa5, $a0, 24 +; LA32D-NEXT: fld.s $fa6, $a0, 12 +; LA32D-NEXT: fld.s $fa7, $a0, 8 +; LA32D-NEXT: fld.s $ft0, $a0, 0 +; LA32D-NEXT: fld.s $ft1, $a0, 16 +; LA32D-NEXT: fld.s $ft2, $a0, 4 ; LA32D-NEXT: ffint.s.w $fa0, $fa0 -; LA32D-NEXT: fld.s $fa1, $a0, 16 -; LA32D-NEXT: fadd.s $fa1, $fa1, $fa0 -; LA32D-NEXT: fst.s $fa1, $a1, 16 -; LA32D-NEXT: fld.s $fa1, $a0, 0 -; LA32D-NEXT: fadd.s $fa0, $fa1, $fa0 -; LA32D-NEXT: fst.s $fa0, $a1, 0 +; LA32D-NEXT: fadd.s $ft0, $ft0, $fa0 +; LA32D-NEXT: fadd.s $fa0, $ft1, $fa0 +; LA32D-NEXT: fld.s $ft1, $a0, 20 +; LA32D-NEXT: fadd.s $ft2, $ft2, $fa1 +; LA32D-NEXT: fadd.s $fa7, $fa7, $fa2 +; LA32D-NEXT: fadd.s $fa6, $fa6, $fa3 +; LA32D-NEXT: fadd.s $fa1, $ft1, $fa1 +; LA32D-NEXT: fadd.s $fa2, $fa5, $fa2 +; LA32D-NEXT: fadd.s $fa3, $fa4, $fa3 +; LA32D-NEXT: fst.s $fa3, $a1, 28 +; LA32D-NEXT: fst.s $fa2, $a1, 24 +; LA32D-NEXT: fst.s $fa1, $a1, 20 +; LA32D-NEXT: fst.s $fa6, $a1, 12 +; LA32D-NEXT: fst.s $fa7, $a1, 8 +; LA32D-NEXT: fst.s $ft2, $a1, 4 +; LA32D-NEXT: fst.s $fa0, $a1, 16 +; LA32D-NEXT: fst.s $ft0, $a1, 0 ; LA32D-NEXT: ret ; ; LA64F-LABEL: test_f8: ; LA64F: # %bb.0: +; LA64F-NEXT: addi.w $a2, $zero, 1 +; LA64F-NEXT: movgr2fr.w $fa0, $a2 ; LA64F-NEXT: pcalau12i $a2, %pc_hi20(.LCPI3_0) ; LA64F-NEXT: addi.d $a2, $a2, %pc_lo12(.LCPI3_0) -; LA64F-NEXT: fld.s $fa0, $a2, 0 -; LA64F-NEXT: fld.s $fa1, $a0, 4 -; LA64F-NEXT: fadd.s $fa1, $fa1, $fa0 -; LA64F-NEXT: fld.s $fa2, $a0, 20 -; LA64F-NEXT: fadd.s $fa0, $fa2, $fa0 +; LA64F-NEXT: fld.s $fa1, $a2, 0 ; LA64F-NEXT: pcalau12i $a2, %pc_hi20(.LCPI3_1) ; LA64F-NEXT: addi.d $a2, $a2, %pc_lo12(.LCPI3_1) ; LA64F-NEXT: fld.s $fa2, $a2, 0 -; LA64F-NEXT: fld.s $fa3, $a0, 8 -; LA64F-NEXT: fadd.s $fa3, $fa3, $fa2 -; LA64F-NEXT: fld.s $fa4, $a0, 24 -; LA64F-NEXT: fadd.s $fa2, $fa4, $fa2 ; LA64F-NEXT: pcalau12i $a2, %pc_hi20(.LCPI3_2) ; LA64F-NEXT: addi.d $a2, $a2, %pc_lo12(.LCPI3_2) -; LA64F-NEXT: fld.s $fa4, $a2, 0 -; LA64F-NEXT: fld.s $fa5, $a0, 12 -; LA64F-NEXT: fadd.s $fa5, $fa5, $fa4 -; LA64F-NEXT: fld.s $fa6, $a0, 28 -; LA64F-NEXT: fadd.s $fa4, $fa6, $fa4 -; LA64F-NEXT: fst.s $fa4, $a1, 28 -; LA64F-NEXT: fst.s $fa2, $a1, 24 -; LA64F-NEXT: fst.s $fa0, $a1, 20 -; LA64F-NEXT: fst.s $fa5, $a1, 12 -; LA64F-NEXT: fst.s $fa3, $a1, 8 -; LA64F-NEXT: fst.s $fa1, $a1, 4 -; LA64F-NEXT: addi.w $a2, $zero, 1 -; LA64F-NEXT: movgr2fr.w $fa0, $a2 +; LA64F-NEXT: fld.s $fa3, $a2, 0 +; LA64F-NEXT: fld.s $fa4, $a0, 28 +; LA64F-NEXT: fld.s $fa5, $a0, 24 +; LA64F-NEXT: fld.s $fa6, $a0, 12 +; LA64F-NEXT: fld.s $fa7, $a0, 8 +; LA64F-NEXT: fld.s $ft0, $a0, 0 +; LA64F-NEXT: fld.s $ft1, $a0, 16 +; LA64F-NEXT: fld.s $ft2, $a0, 4 ; LA64F-NEXT: ffint.s.w $fa0, $fa0 -; LA64F-NEXT: fld.s $fa1, $a0, 16 -; LA64F-NEXT: fadd.s $fa1, $fa1, $fa0 -; LA64F-NEXT: fst.s $fa1, $a1, 16 -; LA64F-NEXT: fld.s $fa1, $a0, 0 -; LA64F-NEXT: fadd.s $fa0, $fa1, $fa0 -; LA64F-NEXT: fst.s $fa0, $a1, 0 +; LA64F-NEXT: fadd.s $ft0, $ft0, $fa0 +; LA64F-NEXT: fadd.s $fa0, $ft1, $fa0 +; LA64F-NEXT: fld.s $ft1, $a0, 20 +; LA64F-NEXT: fadd.s $ft2, $ft2, $fa1 +; LA64F-NEXT: fadd.s $fa7, $fa7, $fa2 +; LA64F-NEXT: fadd.s $fa6, $fa6, $fa3 +; LA64F-NEXT: fadd.s $fa1, $ft1, $fa1 +; LA64F-NEXT: fadd.s $fa2, $fa5, $fa2 +; LA64F-NEXT: fadd.s $fa3, $fa4, $fa3 +; LA64F-NEXT: fst.s $fa3, $a1, 28 +; LA64F-NEXT: fst.s $fa2, $a1, 24 +; LA64F-NEXT: fst.s $fa1, $a1, 20 +; LA64F-NEXT: fst.s $fa6, $a1, 12 +; LA64F-NEXT: fst.s $fa7, $a1, 8 +; LA64F-NEXT: fst.s $ft2, $a1, 4 +; LA64F-NEXT: fst.s $fa0, $a1, 16 +; LA64F-NEXT: fst.s $ft0, $a1, 0 ; LA64F-NEXT: ret ; ; LA64D-LABEL: test_f8: ; LA64D: # %bb.0: +; LA64D-NEXT: addi.w $a2, $zero, 1 +; LA64D-NEXT: movgr2fr.w $fa0, $a2 ; LA64D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI3_0) ; LA64D-NEXT: addi.d $a2, $a2, %pc_lo12(.LCPI3_0) -; LA64D-NEXT: fld.s $fa0, $a2, 0 -; LA64D-NEXT: fld.s $fa1, $a0, 4 -; LA64D-NEXT: fadd.s $fa1, $fa1, $fa0 -; LA64D-NEXT: fld.s $fa2, $a0, 20 -; LA64D-NEXT: fadd.s $fa0, $fa2, $fa0 +; LA64D-NEXT: fld.s $fa1, $a2, 0 ; LA64D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI3_1) ; LA64D-NEXT: addi.d $a2, $a2, %pc_lo12(.LCPI3_1) ; LA64D-NEXT: fld.s $fa2, $a2, 0 -; LA64D-NEXT: fld.s $fa3, $a0, 8 -; LA64D-NEXT: fadd.s $fa3, $fa3, $fa2 -; LA64D-NEXT: fld.s $fa4, $a0, 24 -; LA64D-NEXT: fadd.s $fa2, $fa4, $fa2 ; LA64D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI3_2) ; LA64D-NEXT: addi.d $a2, $a2, %pc_lo12(.LCPI3_2) -; LA64D-NEXT: fld.s $fa4, $a2, 0 -; LA64D-NEXT: fld.s $fa5, $a0, 12 -; LA64D-NEXT: fadd.s $fa5, $fa5, $fa4 -; LA64D-NEXT: fld.s $fa6, $a0, 28 -; LA64D-NEXT: fadd.s $fa4, $fa6, $fa4 -; LA64D-NEXT: fst.s $fa4, $a1, 28 -; LA64D-NEXT: fst.s $fa2, $a1, 24 -; LA64D-NEXT: fst.s $fa0, $a1, 20 -; LA64D-NEXT: fst.s $fa5, $a1, 12 -; LA64D-NEXT: fst.s $fa3, $a1, 8 -; LA64D-NEXT: fst.s $fa1, $a1, 4 -; LA64D-NEXT: addi.w $a2, $zero, 1 -; LA64D-NEXT: movgr2fr.w $fa0, $a2 +; LA64D-NEXT: fld.s $fa3, $a2, 0 +; LA64D-NEXT: fld.s $fa4, $a0, 28 +; LA64D-NEXT: fld.s $fa5, $a0, 24 +; LA64D-NEXT: fld.s $fa6, $a0, 12 +; LA64D-NEXT: fld.s $fa7, $a0, 8 +; LA64D-NEXT: fld.s $ft0, $a0, 0 +; LA64D-NEXT: fld.s $ft1, $a0, 16 +; LA64D-NEXT: fld.s $ft2, $a0, 4 ; LA64D-NEXT: ffint.s.w $fa0, $fa0 -; LA64D-NEXT: fld.s $fa1, $a0, 16 -; LA64D-NEXT: fadd.s $fa1, $fa1, $fa0 -; LA64D-NEXT: fst.s $fa1, $a1, 16 -; LA64D-NEXT: fld.s $fa1, $a0, 0 -; LA64D-NEXT: fadd.s $fa0, $fa1, $fa0 -; LA64D-NEXT: fst.s $fa0, $a1, 0 +; LA64D-NEXT: fadd.s $ft0, $ft0, $fa0 +; LA64D-NEXT: fadd.s $fa0, $ft1, $fa0 +; LA64D-NEXT: fld.s $ft1, $a0, 20 +; LA64D-NEXT: fadd.s $ft2, $ft2, $fa1 +; LA64D-NEXT: fadd.s $fa7, $fa7, $fa2 +; LA64D-NEXT: fadd.s $fa6, $fa6, $fa3 +; LA64D-NEXT: fadd.s $fa1, $ft1, $fa1 +; LA64D-NEXT: fadd.s $fa2, $fa5, $fa2 +; LA64D-NEXT: fadd.s $fa3, $fa4, $fa3 +; LA64D-NEXT: fst.s $fa3, $a1, 28 +; LA64D-NEXT: fst.s $fa2, $a1, 24 +; LA64D-NEXT: fst.s $fa1, $a1, 20 +; LA64D-NEXT: fst.s $fa6, $a1, 12 +; LA64D-NEXT: fst.s $fa7, $a1, 8 +; LA64D-NEXT: fst.s $ft2, $a1, 4 +; LA64D-NEXT: fst.s $fa0, $a1, 16 +; LA64D-NEXT: fst.s $ft0, $a1, 0 ; LA64D-NEXT: ret %p = load %f8, ptr %P %R = fadd %f8 %p, < float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00 > @@ -444,47 +444,58 @@ define void @test_f8(ptr %P, ptr %S) nounwind { define void @test_d2(ptr %P, ptr %S) nounwind { ; LA32F-LABEL: test_d2: ; LA32F: # %bb.0: -; LA32F-NEXT: addi.w $sp, $sp, -16 -; LA32F-NEXT: st.w $ra, $sp, 12 # 4-byte Folded Spill -; LA32F-NEXT: st.w $fp, $sp, 8 # 4-byte Folded Spill -; LA32F-NEXT: st.w $s0, $sp, 4 # 4-byte Folded Spill -; LA32F-NEXT: move $fp, $a1 -; LA32F-NEXT: move $s0, $a0 -; LA32F-NEXT: ld.w $a0, $a0, 8 -; LA32F-NEXT: ld.w $a1, $s0, 12 -; LA32F-NEXT: lu12i.w $a3, 262144 +; LA32F-NEXT: addi.w $sp, $sp, -32 +; LA32F-NEXT: st.w $ra, $sp, 28 # 4-byte Folded Spill +; LA32F-NEXT: st.w $fp, $sp, 24 # 4-byte Folded Spill +; LA32F-NEXT: st.w $s0, $sp, 20 # 4-byte Folded Spill +; LA32F-NEXT: st.w $s1, $sp, 16 # 4-byte Folded Spill +; LA32F-NEXT: st.w $s2, $sp, 12 # 4-byte Folded Spill +; LA32F-NEXT: st.w $s3, $sp, 8 # 4-byte Folded Spill +; LA32F-NEXT: ld.w $fp, $a0, 8 +; LA32F-NEXT: ld.w $s0, $a0, 12 +; LA32F-NEXT: ld.w $a2, $a0, 0 +; LA32F-NEXT: ld.w $a4, $a0, 4 +; LA32F-NEXT: move $s1, $a1 +; LA32F-NEXT: lu12i.w $a3, 261888 +; LA32F-NEXT: move $a0, $a2 +; LA32F-NEXT: move $a1, $a4 ; LA32F-NEXT: move $a2, $zero ; LA32F-NEXT: bl %plt(__adddf3) -; LA32F-NEXT: st.w $a0, $fp, 8 -; LA32F-NEXT: st.w $a1, $fp, 12 -; LA32F-NEXT: ld.w $a0, $s0, 0 -; LA32F-NEXT: ld.w $a1, $s0, 4 -; LA32F-NEXT: lu12i.w $a3, 261888 +; LA32F-NEXT: move $s2, $a0 +; LA32F-NEXT: move $s3, $a1 +; LA32F-NEXT: lu12i.w $a3, 262144 +; LA32F-NEXT: move $a0, $fp +; LA32F-NEXT: move $a1, $s0 ; LA32F-NEXT: move $a2, $zero ; LA32F-NEXT: bl %plt(__adddf3) -; LA32F-NEXT: st.w $a0, $fp, 0 -; LA32F-NEXT: st.w $a1, $fp, 4 -; LA32F-NEXT: ld.w $s0, $sp, 4 # 4-byte Folded Reload -; LA32F-NEXT: ld.w $fp, $sp, 8 # 4-byte Folded Reload -; LA32F-NEXT: ld.w $ra, $sp, 12 # 4-byte Folded Reload -; LA32F-NEXT: addi.w $sp, $sp, 16 +; LA32F-NEXT: st.w $a0, $s1, 8 +; LA32F-NEXT: st.w $a1, $s1, 12 +; LA32F-NEXT: st.w $s2, $s1, 0 +; LA32F-NEXT: st.w $s3, $s1, 4 +; LA32F-NEXT: ld.w $s3, $sp, 8 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $s2, $sp, 12 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $s1, $sp, 16 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $s0, $sp, 20 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $fp, $sp, 24 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $ra, $sp, 28 # 4-byte Folded Reload +; LA32F-NEXT: addi.w $sp, $sp, 32 ; LA32F-NEXT: ret ; ; LA32D-LABEL: test_d2: ; LA32D: # %bb.0: -; LA32D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI4_0) -; LA32D-NEXT: addi.w $a2, $a2, %pc_lo12(.LCPI4_0) -; LA32D-NEXT: fld.d $fa0, $a2, 0 -; LA32D-NEXT: fld.d $fa1, $a0, 8 -; LA32D-NEXT: fadd.d $fa0, $fa1, $fa0 -; LA32D-NEXT: fst.d $fa0, $a1, 8 -; LA32D-NEXT: fld.d $fa0, $a0, 0 +; LA32D-NEXT: fld.d $fa0, $a0, 8 +; LA32D-NEXT: fld.d $fa1, $a0, 0 ; LA32D-NEXT: addi.w $a0, $zero, 1 -; LA32D-NEXT: movgr2fr.w $fa1, $a0 -; LA32D-NEXT: ffint.s.w $fa1, $fa1 -; LA32D-NEXT: fcvt.d.s $fa1, $fa1 -; LA32D-NEXT: fadd.d $fa0, $fa0, $fa1 -; LA32D-NEXT: fst.d $fa0, $a1, 0 +; LA32D-NEXT: movgr2fr.w $fa2, $a0 +; LA32D-NEXT: pcalau12i $a0, %pc_hi20(.LCPI4_0) +; LA32D-NEXT: addi.w $a0, $a0, %pc_lo12(.LCPI4_0) +; LA32D-NEXT: fld.d $fa3, $a0, 0 +; LA32D-NEXT: ffint.s.w $fa2, $fa2 +; LA32D-NEXT: fcvt.d.s $fa2, $fa2 +; LA32D-NEXT: fadd.d $fa1, $fa1, $fa2 +; LA32D-NEXT: fadd.d $fa0, $fa0, $fa3 +; LA32D-NEXT: fst.d $fa0, $a1, 8 +; LA32D-NEXT: fst.d $fa1, $a1, 0 ; LA32D-NEXT: ret ; ; LA64F-LABEL: test_d2: @@ -493,16 +504,19 @@ define void @test_d2(ptr %P, ptr %S) nounwind { ; LA64F-NEXT: st.d $ra, $sp, 24 # 8-byte Folded Spill ; LA64F-NEXT: st.d $fp, $sp, 16 # 8-byte Folded Spill ; LA64F-NEXT: st.d $s0, $sp, 8 # 8-byte Folded Spill -; LA64F-NEXT: move $fp, $a1 -; LA64F-NEXT: move $s0, $a0 -; LA64F-NEXT: ld.d $a0, $a0, 8 -; LA64F-NEXT: lu52i.d $a1, $zero, 1024 -; LA64F-NEXT: bl %plt(__adddf3) -; LA64F-NEXT: st.d $a0, $fp, 8 -; LA64F-NEXT: ld.d $a0, $s0, 0 +; LA64F-NEXT: st.d $s1, $sp, 0 # 8-byte Folded Spill +; LA64F-NEXT: ld.d $fp, $a0, 8 +; LA64F-NEXT: ld.d $a0, $a0, 0 +; LA64F-NEXT: move $s0, $a1 ; LA64F-NEXT: lu52i.d $a1, $zero, 1023 ; LA64F-NEXT: bl %plt(__adddf3) -; LA64F-NEXT: st.d $a0, $fp, 0 +; LA64F-NEXT: move $s1, $a0 +; LA64F-NEXT: lu52i.d $a1, $zero, 1024 +; LA64F-NEXT: move $a0, $fp +; LA64F-NEXT: bl %plt(__adddf3) +; LA64F-NEXT: st.d $a0, $s0, 8 +; LA64F-NEXT: st.d $s1, $s0, 0 +; LA64F-NEXT: ld.d $s1, $sp, 0 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $s0, $sp, 8 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $fp, $sp, 16 # 8-byte Folded Reload ; LA64F-NEXT: ld.d $ra, $sp, 24 # 8-byte Folded Reload @@ -511,18 +525,18 @@ define void @test_d2(ptr %P, ptr %S) nounwind { ; ; LA64D-LABEL: test_d2: ; LA64D: # %bb.0: -; LA64D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI4_0) -; LA64D-NEXT: addi.d $a2, $a2, %pc_lo12(.LCPI4_0) -; LA64D-NEXT: fld.d $fa0, $a2, 0 -; LA64D-NEXT: fld.d $fa1, $a0, 8 -; LA64D-NEXT: fadd.d $fa0, $fa1, $fa0 -; LA64D-NEXT: fst.d $fa0, $a1, 8 -; LA64D-NEXT: fld.d $fa0, $a0, 0 +; LA64D-NEXT: fld.d $fa0, $a0, 8 +; LA64D-NEXT: fld.d $fa1, $a0, 0 +; LA64D-NEXT: pcalau12i $a0, %pc_hi20(.LCPI4_0) +; LA64D-NEXT: addi.d $a0, $a0, %pc_lo12(.LCPI4_0) +; LA64D-NEXT: fld.d $fa2, $a0, 0 ; LA64D-NEXT: addi.d $a0, $zero, 1 -; LA64D-NEXT: movgr2fr.d $fa1, $a0 -; LA64D-NEXT: ffint.d.l $fa1, $fa1 -; LA64D-NEXT: fadd.d $fa0, $fa0, $fa1 -; LA64D-NEXT: fst.d $fa0, $a1, 0 +; LA64D-NEXT: movgr2fr.d $fa3, $a0 +; LA64D-NEXT: ffint.d.l $fa3, $fa3 +; LA64D-NEXT: fadd.d $fa1, $fa1, $fa3 +; LA64D-NEXT: fadd.d $fa0, $fa0, $fa2 +; LA64D-NEXT: fst.d $fa0, $a1, 8 +; LA64D-NEXT: fst.d $fa1, $a1, 0 ; LA64D-NEXT: ret %p = load %d2, ptr %P %R = fadd %d2 %p, < double 1.000000e+00, double 2.000000e+00 > @@ -533,133 +547,171 @@ define void @test_d2(ptr %P, ptr %S) nounwind { define void @test_d4(ptr %P, ptr %S) nounwind { ; LA32F-LABEL: test_d4: ; LA32F: # %bb.0: -; LA32F-NEXT: addi.w $sp, $sp, -16 -; LA32F-NEXT: st.w $ra, $sp, 12 # 4-byte Folded Spill -; LA32F-NEXT: st.w $fp, $sp, 8 # 4-byte Folded Spill -; LA32F-NEXT: st.w $s0, $sp, 4 # 4-byte Folded Spill -; LA32F-NEXT: move $fp, $a1 -; LA32F-NEXT: move $s0, $a0 -; LA32F-NEXT: ld.w $a0, $a0, 24 -; LA32F-NEXT: ld.w $a1, $s0, 28 -; LA32F-NEXT: lu12i.w $a3, 262400 +; LA32F-NEXT: addi.w $sp, $sp, -48 +; LA32F-NEXT: st.w $ra, $sp, 44 # 4-byte Folded Spill +; LA32F-NEXT: st.w $fp, $sp, 40 # 4-byte Folded Spill +; LA32F-NEXT: st.w $s0, $sp, 36 # 4-byte Folded Spill +; LA32F-NEXT: st.w $s1, $sp, 32 # 4-byte Folded Spill +; LA32F-NEXT: st.w $s2, $sp, 28 # 4-byte Folded Spill +; LA32F-NEXT: st.w $s3, $sp, 24 # 4-byte Folded Spill +; LA32F-NEXT: st.w $s4, $sp, 20 # 4-byte Folded Spill +; LA32F-NEXT: st.w $s5, $sp, 16 # 4-byte Folded Spill +; LA32F-NEXT: st.w $s6, $sp, 12 # 4-byte Folded Spill +; LA32F-NEXT: st.w $s7, $sp, 8 # 4-byte Folded Spill +; LA32F-NEXT: ld.w $fp, $a0, 24 +; LA32F-NEXT: ld.w $s0, $a0, 28 +; LA32F-NEXT: ld.w $s1, $a0, 16 +; LA32F-NEXT: ld.w $s2, $a0, 20 +; LA32F-NEXT: ld.w $s3, $a0, 8 +; LA32F-NEXT: ld.w $s4, $a0, 12 +; LA32F-NEXT: ld.w $a2, $a0, 0 +; LA32F-NEXT: ld.w $a4, $a0, 4 +; LA32F-NEXT: move $s5, $a1 +; LA32F-NEXT: lu12i.w $a3, 261888 +; LA32F-NEXT: move $a0, $a2 +; LA32F-NEXT: move $a1, $a4 ; LA32F-NEXT: move $a2, $zero ; LA32F-NEXT: bl %plt(__adddf3) -; LA32F-NEXT: st.w $a0, $fp, 24 -; LA32F-NEXT: st.w $a1, $fp, 28 -; LA32F-NEXT: ld.w $a0, $s0, 16 -; LA32F-NEXT: ld.w $a1, $s0, 20 -; LA32F-NEXT: lu12i.w $a3, 262272 +; LA32F-NEXT: move $s6, $a0 +; LA32F-NEXT: move $s7, $a1 +; LA32F-NEXT: lu12i.w $a3, 262144 +; LA32F-NEXT: move $a0, $s3 +; LA32F-NEXT: move $a1, $s4 ; LA32F-NEXT: move $a2, $zero ; LA32F-NEXT: bl %plt(__adddf3) -; LA32F-NEXT: st.w $a0, $fp, 16 -; LA32F-NEXT: st.w $a1, $fp, 20 -; LA32F-NEXT: ld.w $a0, $s0, 8 -; LA32F-NEXT: ld.w $a1, $s0, 12 -; LA32F-NEXT: lu12i.w $a3, 262144 +; LA32F-NEXT: move $s3, $a0 +; LA32F-NEXT: move $s4, $a1 +; LA32F-NEXT: lu12i.w $a3, 262272 +; LA32F-NEXT: move $a0, $s1 +; LA32F-NEXT: move $a1, $s2 ; LA32F-NEXT: move $a2, $zero ; LA32F-NEXT: bl %plt(__adddf3) -; LA32F-NEXT: st.w $a0, $fp, 8 -; LA32F-NEXT: st.w $a1, $fp, 12 -; LA32F-NEXT: ld.w $a0, $s0, 0 -; LA32F-NEXT: ld.w $a1, $s0, 4 -; LA32F-NEXT: lu12i.w $a3, 261888 +; LA32F-NEXT: move $s1, $a0 +; LA32F-NEXT: move $s2, $a1 +; LA32F-NEXT: lu12i.w $a3, 262400 +; LA32F-NEXT: move $a0, $fp +; LA32F-NEXT: move $a1, $s0 ; LA32F-NEXT: move $a2, $zero ; LA32F-NEXT: bl %plt(__adddf3) -; LA32F-NEXT: st.w $a0, $fp, 0 -; LA32F-NEXT: st.w $a1, $fp, 4 -; LA32F-NEXT: ld.w $s0, $sp, 4 # 4-byte Folded Reload -; LA32F-NEXT: ld.w $fp, $sp, 8 # 4-byte Folded Reload -; LA32F-NEXT: ld.w $ra, $sp, 12 # 4-byte Folded Reload -; LA32F-NEXT: addi.w $sp, $sp, 16 +; LA32F-NEXT: st.w $a0, $s5, 24 +; LA32F-NEXT: st.w $a1, $s5, 28 +; LA32F-NEXT: st.w $s1, $s5, 16 +; LA32F-NEXT: st.w $s2, $s5, 20 +; LA32F-NEXT: st.w $s3, $s5, 8 +; LA32F-NEXT: st.w $s4, $s5, 12 +; LA32F-NEXT: st.w $s6, $s5, 0 +; LA32F-NEXT: st.w $s7, $s5, 4 +; LA32F-NEXT: ld.w $s7, $sp, 8 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $s6, $sp, 12 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $s5, $sp, 16 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $s4, $sp, 20 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $s3, $sp, 24 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $s2, $sp, 28 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $s1, $sp, 32 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $s0, $sp, 36 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $fp, $sp, 40 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $ra, $sp, 44 # 4-byte Folded Reload +; LA32F-NEXT: addi.w $sp, $sp, 48 ; LA32F-NEXT: ret ; ; LA32D-LABEL: test_d4: ; LA32D: # %bb.0: -; LA32D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI5_0) -; LA32D-NEXT: addi.w $a2, $a2, %pc_lo12(.LCPI5_0) -; LA32D-NEXT: fld.d $fa0, $a2, 0 -; LA32D-NEXT: fld.d $fa1, $a0, 8 -; LA32D-NEXT: fadd.d $fa0, $fa1, $fa0 -; LA32D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI5_1) -; LA32D-NEXT: addi.w $a2, $a2, %pc_lo12(.LCPI5_1) -; LA32D-NEXT: fld.d $fa1, $a2, 0 -; LA32D-NEXT: fld.d $fa2, $a0, 16 -; LA32D-NEXT: fadd.d $fa1, $fa2, $fa1 -; LA32D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI5_2) -; LA32D-NEXT: addi.w $a2, $a2, %pc_lo12(.LCPI5_2) -; LA32D-NEXT: fld.d $fa2, $a2, 0 -; LA32D-NEXT: fld.d $fa3, $a0, 24 -; LA32D-NEXT: fadd.d $fa2, $fa3, $fa2 -; LA32D-NEXT: fst.d $fa2, $a1, 24 -; LA32D-NEXT: fst.d $fa1, $a1, 16 -; LA32D-NEXT: fst.d $fa0, $a1, 8 -; LA32D-NEXT: fld.d $fa0, $a0, 0 +; LA32D-NEXT: fld.d $fa0, $a0, 24 +; LA32D-NEXT: fld.d $fa1, $a0, 16 +; LA32D-NEXT: fld.d $fa2, $a0, 8 +; LA32D-NEXT: fld.d $fa3, $a0, 0 ; LA32D-NEXT: addi.w $a0, $zero, 1 -; LA32D-NEXT: movgr2fr.w $fa1, $a0 -; LA32D-NEXT: ffint.s.w $fa1, $fa1 -; LA32D-NEXT: fcvt.d.s $fa1, $fa1 -; LA32D-NEXT: fadd.d $fa0, $fa0, $fa1 -; LA32D-NEXT: fst.d $fa0, $a1, 0 +; LA32D-NEXT: movgr2fr.w $fa4, $a0 +; LA32D-NEXT: ffint.s.w $fa4, $fa4 +; LA32D-NEXT: fcvt.d.s $fa4, $fa4 +; LA32D-NEXT: pcalau12i $a0, %pc_hi20(.LCPI5_0) +; LA32D-NEXT: addi.w $a0, $a0, %pc_lo12(.LCPI5_0) +; LA32D-NEXT: fld.d $fa5, $a0, 0 +; LA32D-NEXT: pcalau12i $a0, %pc_hi20(.LCPI5_1) +; LA32D-NEXT: addi.w $a0, $a0, %pc_lo12(.LCPI5_1) +; LA32D-NEXT: fld.d $fa6, $a0, 0 +; LA32D-NEXT: pcalau12i $a0, %pc_hi20(.LCPI5_2) +; LA32D-NEXT: addi.w $a0, $a0, %pc_lo12(.LCPI5_2) +; LA32D-NEXT: fld.d $fa7, $a0, 0 +; LA32D-NEXT: fadd.d $fa3, $fa3, $fa4 +; LA32D-NEXT: fadd.d $fa2, $fa2, $fa5 +; LA32D-NEXT: fadd.d $fa1, $fa1, $fa6 +; LA32D-NEXT: fadd.d $fa0, $fa0, $fa7 +; LA32D-NEXT: fst.d $fa0, $a1, 24 +; LA32D-NEXT: fst.d $fa1, $a1, 16 +; LA32D-NEXT: fst.d $fa2, $a1, 8 +; LA32D-NEXT: fst.d $fa3, $a1, 0 ; LA32D-NEXT: ret ; ; LA64F-LABEL: test_d4: ; LA64F: # %bb.0: -; LA64F-NEXT: addi.d $sp, $sp, -32 -; LA64F-NEXT: st.d $ra, $sp, 24 # 8-byte Folded Spill -; LA64F-NEXT: st.d $fp, $sp, 16 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s0, $sp, 8 # 8-byte Folded Spill -; LA64F-NEXT: move $fp, $a1 -; LA64F-NEXT: move $s0, $a0 -; LA64F-NEXT: ld.d $a0, $a0, 24 -; LA64F-NEXT: lu52i.d $a1, $zero, 1025 -; LA64F-NEXT: bl %plt(__adddf3) -; LA64F-NEXT: st.d $a0, $fp, 24 -; LA64F-NEXT: ld.d $a0, $s0, 8 -; LA64F-NEXT: lu52i.d $a1, $zero, 1024 -; LA64F-NEXT: bl %plt(__adddf3) -; LA64F-NEXT: st.d $a0, $fp, 8 -; LA64F-NEXT: ld.d $a0, $s0, 0 -; LA64F-NEXT: lu52i.d $a1, $zero, 1023 -; LA64F-NEXT: bl %plt(__adddf3) -; LA64F-NEXT: st.d $a0, $fp, 0 -; LA64F-NEXT: ld.d $a0, $s0, 16 +; LA64F-NEXT: addi.d $sp, $sp, -48 +; LA64F-NEXT: st.d $ra, $sp, 40 # 8-byte Folded Spill +; LA64F-NEXT: st.d $fp, $sp, 32 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s0, $sp, 24 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s1, $sp, 16 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s2, $sp, 8 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s3, $sp, 0 # 8-byte Folded Spill +; LA64F-NEXT: ld.d $fp, $a0, 24 +; LA64F-NEXT: ld.d $s0, $a0, 8 +; LA64F-NEXT: ld.d $s1, $a0, 0 +; LA64F-NEXT: ld.d $a0, $a0, 16 +; LA64F-NEXT: move $s2, $a1 ; LA64F-NEXT: ori $a1, $zero, 0 ; LA64F-NEXT: lu32i.d $a1, -524288 ; LA64F-NEXT: lu52i.d $a1, $a1, 1024 ; LA64F-NEXT: bl %plt(__adddf3) -; LA64F-NEXT: st.d $a0, $fp, 16 -; LA64F-NEXT: ld.d $s0, $sp, 8 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $fp, $sp, 16 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $ra, $sp, 24 # 8-byte Folded Reload -; LA64F-NEXT: addi.d $sp, $sp, 32 +; LA64F-NEXT: move $s3, $a0 +; LA64F-NEXT: lu52i.d $a1, $zero, 1023 +; LA64F-NEXT: move $a0, $s1 +; LA64F-NEXT: bl %plt(__adddf3) +; LA64F-NEXT: move $s1, $a0 +; LA64F-NEXT: lu52i.d $a1, $zero, 1024 +; LA64F-NEXT: move $a0, $s0 +; LA64F-NEXT: bl %plt(__adddf3) +; LA64F-NEXT: move $s0, $a0 +; LA64F-NEXT: lu52i.d $a1, $zero, 1025 +; LA64F-NEXT: move $a0, $fp +; LA64F-NEXT: bl %plt(__adddf3) +; LA64F-NEXT: st.d $a0, $s2, 24 +; LA64F-NEXT: st.d $s0, $s2, 8 +; LA64F-NEXT: st.d $s1, $s2, 0 +; LA64F-NEXT: st.d $s3, $s2, 16 +; LA64F-NEXT: ld.d $s3, $sp, 0 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s2, $sp, 8 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s1, $sp, 16 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s0, $sp, 24 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $fp, $sp, 32 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $ra, $sp, 40 # 8-byte Folded Reload +; LA64F-NEXT: addi.d $sp, $sp, 48 ; LA64F-NEXT: ret ; ; LA64D-LABEL: test_d4: ; LA64D: # %bb.0: -; LA64D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI5_0) -; LA64D-NEXT: addi.d $a2, $a2, %pc_lo12(.LCPI5_0) -; LA64D-NEXT: fld.d $fa0, $a2, 0 -; LA64D-NEXT: fld.d $fa1, $a0, 8 -; LA64D-NEXT: fadd.d $fa0, $fa1, $fa0 -; LA64D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI5_1) -; LA64D-NEXT: addi.d $a2, $a2, %pc_lo12(.LCPI5_1) -; LA64D-NEXT: fld.d $fa1, $a2, 0 -; LA64D-NEXT: fld.d $fa2, $a0, 16 -; LA64D-NEXT: fadd.d $fa1, $fa2, $fa1 -; LA64D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI5_2) -; LA64D-NEXT: addi.d $a2, $a2, %pc_lo12(.LCPI5_2) -; LA64D-NEXT: fld.d $fa2, $a2, 0 -; LA64D-NEXT: fld.d $fa3, $a0, 24 -; LA64D-NEXT: fadd.d $fa2, $fa3, $fa2 -; LA64D-NEXT: fst.d $fa2, $a1, 24 -; LA64D-NEXT: fst.d $fa1, $a1, 16 -; LA64D-NEXT: fst.d $fa0, $a1, 8 -; LA64D-NEXT: fld.d $fa0, $a0, 0 +; LA64D-NEXT: fld.d $fa0, $a0, 24 +; LA64D-NEXT: fld.d $fa1, $a0, 16 +; LA64D-NEXT: fld.d $fa2, $a0, 8 +; LA64D-NEXT: fld.d $fa3, $a0, 0 ; LA64D-NEXT: addi.d $a0, $zero, 1 -; LA64D-NEXT: movgr2fr.d $fa1, $a0 -; LA64D-NEXT: ffint.d.l $fa1, $fa1 -; LA64D-NEXT: fadd.d $fa0, $fa0, $fa1 -; LA64D-NEXT: fst.d $fa0, $a1, 0 +; LA64D-NEXT: movgr2fr.d $fa4, $a0 +; LA64D-NEXT: ffint.d.l $fa4, $fa4 +; LA64D-NEXT: pcalau12i $a0, %pc_hi20(.LCPI5_0) +; LA64D-NEXT: addi.d $a0, $a0, %pc_lo12(.LCPI5_0) +; LA64D-NEXT: fld.d $fa5, $a0, 0 +; LA64D-NEXT: pcalau12i $a0, %pc_hi20(.LCPI5_1) +; LA64D-NEXT: addi.d $a0, $a0, %pc_lo12(.LCPI5_1) +; LA64D-NEXT: fld.d $fa6, $a0, 0 +; LA64D-NEXT: pcalau12i $a0, %pc_hi20(.LCPI5_2) +; LA64D-NEXT: addi.d $a0, $a0, %pc_lo12(.LCPI5_2) +; LA64D-NEXT: fld.d $fa7, $a0, 0 +; LA64D-NEXT: fadd.d $fa3, $fa3, $fa4 +; LA64D-NEXT: fadd.d $fa2, $fa2, $fa5 +; LA64D-NEXT: fadd.d $fa1, $fa1, $fa6 +; LA64D-NEXT: fadd.d $fa0, $fa0, $fa7 +; LA64D-NEXT: fst.d $fa0, $a1, 24 +; LA64D-NEXT: fst.d $fa1, $a1, 16 +; LA64D-NEXT: fst.d $fa2, $a1, 8 +; LA64D-NEXT: fst.d $fa3, $a1, 0 ; LA64D-NEXT: ret %p = load %d4, ptr %P %R = fadd %d4 %p, < double 1.000000e+00, double 2.000000e+00, double 3.000000e+00, double 4.000000e+00 > @@ -670,223 +722,298 @@ define void @test_d4(ptr %P, ptr %S) nounwind { define void @test_d8(ptr %P, ptr %S) nounwind { ; LA32F-LABEL: test_d8: ; LA32F: # %bb.0: -; LA32F-NEXT: addi.w $sp, $sp, -32 -; LA32F-NEXT: st.w $ra, $sp, 28 # 4-byte Folded Spill -; LA32F-NEXT: st.w $fp, $sp, 24 # 4-byte Folded Spill -; LA32F-NEXT: st.w $s0, $sp, 20 # 4-byte Folded Spill -; LA32F-NEXT: st.w $s1, $sp, 16 # 4-byte Folded Spill -; LA32F-NEXT: st.w $s2, $sp, 12 # 4-byte Folded Spill -; LA32F-NEXT: st.w $s3, $sp, 8 # 4-byte Folded Spill -; LA32F-NEXT: st.w $s4, $sp, 4 # 4-byte Folded Spill +; LA32F-NEXT: addi.w $sp, $sp, -96 +; LA32F-NEXT: st.w $ra, $sp, 92 # 4-byte Folded Spill +; LA32F-NEXT: st.w $fp, $sp, 88 # 4-byte Folded Spill +; LA32F-NEXT: st.w $s0, $sp, 84 # 4-byte Folded Spill +; LA32F-NEXT: st.w $s1, $sp, 80 # 4-byte Folded Spill +; LA32F-NEXT: st.w $s2, $sp, 76 # 4-byte Folded Spill +; LA32F-NEXT: st.w $s3, $sp, 72 # 4-byte Folded Spill +; LA32F-NEXT: st.w $s4, $sp, 68 # 4-byte Folded Spill +; LA32F-NEXT: st.w $s5, $sp, 64 # 4-byte Folded Spill +; LA32F-NEXT: st.w $s6, $sp, 60 # 4-byte Folded Spill +; LA32F-NEXT: st.w $s7, $sp, 56 # 4-byte Folded Spill +; LA32F-NEXT: st.w $s8, $sp, 52 # 4-byte Folded Spill +; LA32F-NEXT: ld.w $a2, $a0, 56 +; LA32F-NEXT: st.w $a2, $sp, 48 # 4-byte Folded Spill +; LA32F-NEXT: ld.w $a2, $a0, 60 +; LA32F-NEXT: st.w $a2, $sp, 44 # 4-byte Folded Spill +; LA32F-NEXT: ld.w $a2, $a0, 48 +; LA32F-NEXT: st.w $a2, $sp, 32 # 4-byte Folded Spill +; LA32F-NEXT: ld.w $a2, $a0, 52 +; LA32F-NEXT: st.w $a2, $sp, 28 # 4-byte Folded Spill +; LA32F-NEXT: ld.w $a2, $a0, 40 +; LA32F-NEXT: st.w $a2, $sp, 16 # 4-byte Folded Spill +; LA32F-NEXT: ld.w $a2, $a0, 44 +; LA32F-NEXT: st.w $a2, $sp, 12 # 4-byte Folded Spill +; LA32F-NEXT: ld.w $a2, $a0, 32 +; LA32F-NEXT: st.w $a2, $sp, 0 # 4-byte Folded Spill +; LA32F-NEXT: ld.w $s4, $a0, 36 +; LA32F-NEXT: ld.w $s5, $a0, 24 +; LA32F-NEXT: ld.w $s6, $a0, 28 +; LA32F-NEXT: ld.w $s1, $a0, 16 +; LA32F-NEXT: ld.w $s2, $a0, 20 +; LA32F-NEXT: ld.w $s7, $a0, 8 +; LA32F-NEXT: ld.w $s0, $a0, 12 +; LA32F-NEXT: ld.w $a2, $a0, 0 +; LA32F-NEXT: ld.w $a3, $a0, 4 ; LA32F-NEXT: move $fp, $a1 -; LA32F-NEXT: move $s0, $a0 -; LA32F-NEXT: ld.w $a0, $a0, 56 -; LA32F-NEXT: ld.w $a1, $s0, 60 -; LA32F-NEXT: lu12i.w $s1, 262400 -; LA32F-NEXT: move $a2, $zero -; LA32F-NEXT: move $a3, $s1 -; LA32F-NEXT: bl %plt(__adddf3) -; LA32F-NEXT: st.w $a0, $fp, 56 -; LA32F-NEXT: st.w $a1, $fp, 60 -; LA32F-NEXT: ld.w $a0, $s0, 48 -; LA32F-NEXT: ld.w $a1, $s0, 52 -; LA32F-NEXT: lu12i.w $s2, 262272 +; LA32F-NEXT: lu12i.w $s8, 261888 +; LA32F-NEXT: move $a0, $a2 +; LA32F-NEXT: move $a1, $a3 ; LA32F-NEXT: move $a2, $zero -; LA32F-NEXT: move $a3, $s2 +; LA32F-NEXT: move $a3, $s8 ; LA32F-NEXT: bl %plt(__adddf3) -; LA32F-NEXT: st.w $a0, $fp, 48 -; LA32F-NEXT: st.w $a1, $fp, 52 -; LA32F-NEXT: ld.w $a0, $s0, 40 -; LA32F-NEXT: ld.w $a1, $s0, 44 +; LA32F-NEXT: st.w $a0, $sp, 40 # 4-byte Folded Spill +; LA32F-NEXT: st.w $a1, $sp, 36 # 4-byte Folded Spill ; LA32F-NEXT: lu12i.w $s3, 262144 +; LA32F-NEXT: move $a0, $s7 +; LA32F-NEXT: move $a1, $s0 ; LA32F-NEXT: move $a2, $zero ; LA32F-NEXT: move $a3, $s3 ; LA32F-NEXT: bl %plt(__adddf3) -; LA32F-NEXT: st.w $a0, $fp, 40 -; LA32F-NEXT: st.w $a1, $fp, 44 -; LA32F-NEXT: ld.w $a0, $s0, 32 -; LA32F-NEXT: ld.w $a1, $s0, 36 -; LA32F-NEXT: lu12i.w $s4, 261888 +; LA32F-NEXT: st.w $a0, $sp, 24 # 4-byte Folded Spill +; LA32F-NEXT: st.w $a1, $sp, 20 # 4-byte Folded Spill +; LA32F-NEXT: lu12i.w $s0, 262272 +; LA32F-NEXT: move $a0, $s1 +; LA32F-NEXT: move $a1, $s2 ; LA32F-NEXT: move $a2, $zero -; LA32F-NEXT: move $a3, $s4 +; LA32F-NEXT: move $a3, $s0 ; LA32F-NEXT: bl %plt(__adddf3) -; LA32F-NEXT: st.w $a0, $fp, 32 -; LA32F-NEXT: st.w $a1, $fp, 36 -; LA32F-NEXT: ld.w $a0, $s0, 24 -; LA32F-NEXT: ld.w $a1, $s0, 28 +; LA32F-NEXT: st.w $a0, $sp, 8 # 4-byte Folded Spill +; LA32F-NEXT: st.w $a1, $sp, 4 # 4-byte Folded Spill +; LA32F-NEXT: lu12i.w $s7, 262400 +; LA32F-NEXT: move $a0, $s5 +; LA32F-NEXT: move $a1, $s6 ; LA32F-NEXT: move $a2, $zero -; LA32F-NEXT: move $a3, $s1 +; LA32F-NEXT: move $a3, $s7 ; LA32F-NEXT: bl %plt(__adddf3) -; LA32F-NEXT: st.w $a0, $fp, 24 -; LA32F-NEXT: st.w $a1, $fp, 28 -; LA32F-NEXT: ld.w $a0, $s0, 16 -; LA32F-NEXT: ld.w $a1, $s0, 20 +; LA32F-NEXT: move $s5, $a0 +; LA32F-NEXT: move $s6, $a1 +; LA32F-NEXT: ld.w $a0, $sp, 0 # 4-byte Folded Reload +; LA32F-NEXT: move $a1, $s4 ; LA32F-NEXT: move $a2, $zero -; LA32F-NEXT: move $a3, $s2 +; LA32F-NEXT: move $a3, $s8 ; LA32F-NEXT: bl %plt(__adddf3) -; LA32F-NEXT: st.w $a0, $fp, 16 -; LA32F-NEXT: st.w $a1, $fp, 20 -; LA32F-NEXT: ld.w $a0, $s0, 8 -; LA32F-NEXT: ld.w $a1, $s0, 12 +; LA32F-NEXT: move $s4, $a0 +; LA32F-NEXT: move $s8, $a1 +; LA32F-NEXT: ld.w $a0, $sp, 16 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $a1, $sp, 12 # 4-byte Folded Reload ; LA32F-NEXT: move $a2, $zero ; LA32F-NEXT: move $a3, $s3 ; LA32F-NEXT: bl %plt(__adddf3) -; LA32F-NEXT: st.w $a0, $fp, 8 -; LA32F-NEXT: st.w $a1, $fp, 12 -; LA32F-NEXT: ld.w $a0, $s0, 0 -; LA32F-NEXT: ld.w $a1, $s0, 4 +; LA32F-NEXT: move $s3, $a0 +; LA32F-NEXT: move $s1, $a1 +; LA32F-NEXT: ld.w $a0, $sp, 32 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $a1, $sp, 28 # 4-byte Folded Reload ; LA32F-NEXT: move $a2, $zero -; LA32F-NEXT: move $a3, $s4 +; LA32F-NEXT: move $a3, $s0 ; LA32F-NEXT: bl %plt(__adddf3) +; LA32F-NEXT: move $s0, $a0 +; LA32F-NEXT: move $s2, $a1 +; LA32F-NEXT: ld.w $a0, $sp, 48 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $a1, $sp, 44 # 4-byte Folded Reload +; LA32F-NEXT: move $a2, $zero +; LA32F-NEXT: move $a3, $s7 +; LA32F-NEXT: bl %plt(__adddf3) +; LA32F-NEXT: st.w $a0, $fp, 56 +; LA32F-NEXT: st.w $a1, $fp, 60 +; LA32F-NEXT: st.w $s0, $fp, 48 +; LA32F-NEXT: st.w $s2, $fp, 52 +; LA32F-NEXT: st.w $s3, $fp, 40 +; LA32F-NEXT: st.w $s1, $fp, 44 +; LA32F-NEXT: st.w $s4, $fp, 32 +; LA32F-NEXT: st.w $s8, $fp, 36 +; LA32F-NEXT: st.w $s5, $fp, 24 +; LA32F-NEXT: st.w $s6, $fp, 28 +; LA32F-NEXT: ld.w $a0, $sp, 8 # 4-byte Folded Reload +; LA32F-NEXT: st.w $a0, $fp, 16 +; LA32F-NEXT: ld.w $a0, $sp, 4 # 4-byte Folded Reload +; LA32F-NEXT: st.w $a0, $fp, 20 +; LA32F-NEXT: ld.w $a0, $sp, 24 # 4-byte Folded Reload +; LA32F-NEXT: st.w $a0, $fp, 8 +; LA32F-NEXT: ld.w $a0, $sp, 20 # 4-byte Folded Reload +; LA32F-NEXT: st.w $a0, $fp, 12 +; LA32F-NEXT: ld.w $a0, $sp, 40 # 4-byte Folded Reload ; LA32F-NEXT: st.w $a0, $fp, 0 -; LA32F-NEXT: st.w $a1, $fp, 4 -; LA32F-NEXT: ld.w $s4, $sp, 4 # 4-byte Folded Reload -; LA32F-NEXT: ld.w $s3, $sp, 8 # 4-byte Folded Reload -; LA32F-NEXT: ld.w $s2, $sp, 12 # 4-byte Folded Reload -; LA32F-NEXT: ld.w $s1, $sp, 16 # 4-byte Folded Reload -; LA32F-NEXT: ld.w $s0, $sp, 20 # 4-byte Folded Reload -; LA32F-NEXT: ld.w $fp, $sp, 24 # 4-byte Folded Reload -; LA32F-NEXT: ld.w $ra, $sp, 28 # 4-byte Folded Reload -; LA32F-NEXT: addi.w $sp, $sp, 32 +; LA32F-NEXT: ld.w $a0, $sp, 36 # 4-byte Folded Reload +; LA32F-NEXT: st.w $a0, $fp, 4 +; LA32F-NEXT: ld.w $s8, $sp, 52 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $s7, $sp, 56 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $s6, $sp, 60 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $s5, $sp, 64 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $s4, $sp, 68 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $s3, $sp, 72 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $s2, $sp, 76 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $s1, $sp, 80 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $s0, $sp, 84 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $fp, $sp, 88 # 4-byte Folded Reload +; LA32F-NEXT: ld.w $ra, $sp, 92 # 4-byte Folded Reload +; LA32F-NEXT: addi.w $sp, $sp, 96 ; LA32F-NEXT: ret ; ; LA32D-LABEL: test_d8: ; LA32D: # %bb.0: +; LA32D-NEXT: addi.w $a2, $zero, 1 +; LA32D-NEXT: movgr2fr.w $fa0, $a2 ; LA32D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI6_0) ; LA32D-NEXT: addi.w $a2, $a2, %pc_lo12(.LCPI6_0) -; LA32D-NEXT: fld.d $fa0, $a2, 0 -; LA32D-NEXT: fld.d $fa1, $a0, 8 -; LA32D-NEXT: fadd.d $fa1, $fa1, $fa0 -; LA32D-NEXT: fld.d $fa2, $a0, 40 -; LA32D-NEXT: fadd.d $fa0, $fa2, $fa0 +; LA32D-NEXT: fld.d $fa1, $a2, 0 ; LA32D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI6_1) ; LA32D-NEXT: addi.w $a2, $a2, %pc_lo12(.LCPI6_1) ; LA32D-NEXT: fld.d $fa2, $a2, 0 -; LA32D-NEXT: fld.d $fa3, $a0, 16 -; LA32D-NEXT: fadd.d $fa3, $fa3, $fa2 -; LA32D-NEXT: fld.d $fa4, $a0, 48 -; LA32D-NEXT: fadd.d $fa2, $fa4, $fa2 ; LA32D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI6_2) ; LA32D-NEXT: addi.w $a2, $a2, %pc_lo12(.LCPI6_2) -; LA32D-NEXT: fld.d $fa4, $a2, 0 -; LA32D-NEXT: fld.d $fa5, $a0, 24 -; LA32D-NEXT: fadd.d $fa5, $fa5, $fa4 -; LA32D-NEXT: fld.d $fa6, $a0, 56 -; LA32D-NEXT: fadd.d $fa4, $fa6, $fa4 -; LA32D-NEXT: fst.d $fa4, $a1, 56 -; LA32D-NEXT: fst.d $fa2, $a1, 48 -; LA32D-NEXT: fst.d $fa0, $a1, 40 -; LA32D-NEXT: fst.d $fa5, $a1, 24 -; LA32D-NEXT: fst.d $fa3, $a1, 16 -; LA32D-NEXT: fst.d $fa1, $a1, 8 -; LA32D-NEXT: addi.w $a2, $zero, 1 -; LA32D-NEXT: movgr2fr.w $fa0, $a2 +; LA32D-NEXT: fld.d $fa3, $a2, 0 +; LA32D-NEXT: fld.d $fa4, $a0, 56 +; LA32D-NEXT: fld.d $fa5, $a0, 48 +; LA32D-NEXT: fld.d $fa6, $a0, 24 +; LA32D-NEXT: fld.d $fa7, $a0, 16 +; LA32D-NEXT: fld.d $ft0, $a0, 8 +; LA32D-NEXT: fld.d $ft1, $a0, 0 +; LA32D-NEXT: fld.d $ft2, $a0, 32 ; LA32D-NEXT: ffint.s.w $fa0, $fa0 ; LA32D-NEXT: fcvt.d.s $fa0, $fa0 -; LA32D-NEXT: fld.d $fa1, $a0, 32 -; LA32D-NEXT: fadd.d $fa1, $fa1, $fa0 -; LA32D-NEXT: fst.d $fa1, $a1, 32 -; LA32D-NEXT: fld.d $fa1, $a0, 0 -; LA32D-NEXT: fadd.d $fa0, $fa1, $fa0 -; LA32D-NEXT: fst.d $fa0, $a1, 0 +; LA32D-NEXT: fadd.d $ft1, $ft1, $fa0 +; LA32D-NEXT: fadd.d $fa0, $ft2, $fa0 +; LA32D-NEXT: fld.d $ft2, $a0, 40 +; LA32D-NEXT: fadd.d $ft0, $ft0, $fa1 +; LA32D-NEXT: fadd.d $fa7, $fa7, $fa2 +; LA32D-NEXT: fadd.d $fa6, $fa6, $fa3 +; LA32D-NEXT: fadd.d $fa1, $ft2, $fa1 +; LA32D-NEXT: fadd.d $fa2, $fa5, $fa2 +; LA32D-NEXT: fadd.d $fa3, $fa4, $fa3 +; LA32D-NEXT: fst.d $fa3, $a1, 56 +; LA32D-NEXT: fst.d $fa2, $a1, 48 +; LA32D-NEXT: fst.d $fa1, $a1, 40 +; LA32D-NEXT: fst.d $fa6, $a1, 24 +; LA32D-NEXT: fst.d $fa7, $a1, 16 +; LA32D-NEXT: fst.d $ft0, $a1, 8 +; LA32D-NEXT: fst.d $fa0, $a1, 32 +; LA32D-NEXT: fst.d $ft1, $a1, 0 ; LA32D-NEXT: ret ; ; LA64F-LABEL: test_d8: ; LA64F: # %bb.0: -; LA64F-NEXT: addi.d $sp, $sp, -48 -; LA64F-NEXT: st.d $ra, $sp, 40 # 8-byte Folded Spill -; LA64F-NEXT: st.d $fp, $sp, 32 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s0, $sp, 24 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s1, $sp, 16 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s2, $sp, 8 # 8-byte Folded Spill -; LA64F-NEXT: st.d $s3, $sp, 0 # 8-byte Folded Spill +; LA64F-NEXT: addi.d $sp, $sp, -112 +; LA64F-NEXT: st.d $ra, $sp, 104 # 8-byte Folded Spill +; LA64F-NEXT: st.d $fp, $sp, 96 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s0, $sp, 88 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s1, $sp, 80 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s2, $sp, 72 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s3, $sp, 64 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s4, $sp, 56 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s5, $sp, 48 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s6, $sp, 40 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s7, $sp, 32 # 8-byte Folded Spill +; LA64F-NEXT: st.d $s8, $sp, 24 # 8-byte Folded Spill +; LA64F-NEXT: ld.d $a2, $a0, 56 +; LA64F-NEXT: st.d $a2, $sp, 16 # 8-byte Folded Spill +; LA64F-NEXT: ld.d $s1, $a0, 40 +; LA64F-NEXT: ld.d $s2, $a0, 32 +; LA64F-NEXT: ld.d $s3, $a0, 24 +; LA64F-NEXT: ld.d $s4, $a0, 8 +; LA64F-NEXT: ld.d $s5, $a0, 0 +; LA64F-NEXT: ld.d $s6, $a0, 48 +; LA64F-NEXT: ld.d $a0, $a0, 16 ; LA64F-NEXT: move $fp, $a1 -; LA64F-NEXT: move $s0, $a0 -; LA64F-NEXT: ld.d $a0, $a0, 56 -; LA64F-NEXT: lu52i.d $s1, $zero, 1025 -; LA64F-NEXT: move $a1, $s1 +; LA64F-NEXT: ori $a1, $zero, 0 +; LA64F-NEXT: lu32i.d $a1, -524288 +; LA64F-NEXT: lu52i.d $s7, $a1, 1024 +; LA64F-NEXT: move $a1, $s7 ; LA64F-NEXT: bl %plt(__adddf3) -; LA64F-NEXT: st.d $a0, $fp, 56 -; LA64F-NEXT: ld.d $a0, $s0, 40 -; LA64F-NEXT: lu52i.d $s2, $zero, 1024 -; LA64F-NEXT: move $a1, $s2 +; LA64F-NEXT: st.d $a0, $sp, 8 # 8-byte Folded Spill +; LA64F-NEXT: move $a0, $s6 +; LA64F-NEXT: move $a1, $s7 ; LA64F-NEXT: bl %plt(__adddf3) -; LA64F-NEXT: st.d $a0, $fp, 40 -; LA64F-NEXT: ld.d $a0, $s0, 32 -; LA64F-NEXT: lu52i.d $s3, $zero, 1023 -; LA64F-NEXT: move $a1, $s3 +; LA64F-NEXT: move $s6, $a0 +; LA64F-NEXT: lu52i.d $s7, $zero, 1023 +; LA64F-NEXT: move $a0, $s5 +; LA64F-NEXT: move $a1, $s7 ; LA64F-NEXT: bl %plt(__adddf3) -; LA64F-NEXT: st.d $a0, $fp, 32 -; LA64F-NEXT: ld.d $a0, $s0, 24 -; LA64F-NEXT: move $a1, $s1 +; LA64F-NEXT: move $s5, $a0 +; LA64F-NEXT: lu52i.d $s0, $zero, 1024 +; LA64F-NEXT: move $a0, $s4 +; LA64F-NEXT: move $a1, $s0 ; LA64F-NEXT: bl %plt(__adddf3) -; LA64F-NEXT: st.d $a0, $fp, 24 -; LA64F-NEXT: ld.d $a0, $s0, 8 -; LA64F-NEXT: move $a1, $s2 +; LA64F-NEXT: move $s4, $a0 +; LA64F-NEXT: lu52i.d $s8, $zero, 1025 +; LA64F-NEXT: move $a0, $s3 +; LA64F-NEXT: move $a1, $s8 ; LA64F-NEXT: bl %plt(__adddf3) -; LA64F-NEXT: st.d $a0, $fp, 8 -; LA64F-NEXT: ld.d $a0, $s0, 0 -; LA64F-NEXT: move $a1, $s3 +; LA64F-NEXT: move $s3, $a0 +; LA64F-NEXT: move $a0, $s2 +; LA64F-NEXT: move $a1, $s7 ; LA64F-NEXT: bl %plt(__adddf3) -; LA64F-NEXT: st.d $a0, $fp, 0 -; LA64F-NEXT: ori $a0, $zero, 0 -; LA64F-NEXT: lu32i.d $a0, -524288 -; LA64F-NEXT: lu52i.d $s1, $a0, 1024 -; LA64F-NEXT: ld.d $a0, $s0, 48 -; LA64F-NEXT: move $a1, $s1 +; LA64F-NEXT: move $s2, $a0 +; LA64F-NEXT: move $a0, $s1 +; LA64F-NEXT: move $a1, $s0 ; LA64F-NEXT: bl %plt(__adddf3) -; LA64F-NEXT: st.d $a0, $fp, 48 -; LA64F-NEXT: ld.d $a0, $s0, 16 -; LA64F-NEXT: move $a1, $s1 +; LA64F-NEXT: move $s0, $a0 +; LA64F-NEXT: ld.d $a0, $sp, 16 # 8-byte Folded Reload +; LA64F-NEXT: move $a1, $s8 ; LA64F-NEXT: bl %plt(__adddf3) +; LA64F-NEXT: st.d $a0, $fp, 56 +; LA64F-NEXT: st.d $s0, $fp, 40 +; LA64F-NEXT: st.d $s2, $fp, 32 +; LA64F-NEXT: st.d $s3, $fp, 24 +; LA64F-NEXT: st.d $s4, $fp, 8 +; LA64F-NEXT: st.d $s5, $fp, 0 +; LA64F-NEXT: st.d $s6, $fp, 48 +; LA64F-NEXT: ld.d $a0, $sp, 8 # 8-byte Folded Reload ; LA64F-NEXT: st.d $a0, $fp, 16 -; LA64F-NEXT: ld.d $s3, $sp, 0 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s2, $sp, 8 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s1, $sp, 16 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $s0, $sp, 24 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $fp, $sp, 32 # 8-byte Folded Reload -; LA64F-NEXT: ld.d $ra, $sp, 40 # 8-byte Folded Reload -; LA64F-NEXT: addi.d $sp, $sp, 48 +; LA64F-NEXT: ld.d $s8, $sp, 24 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s7, $sp, 32 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s6, $sp, 40 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s5, $sp, 48 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s4, $sp, 56 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s3, $sp, 64 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s2, $sp, 72 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s1, $sp, 80 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $s0, $sp, 88 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $fp, $sp, 96 # 8-byte Folded Reload +; LA64F-NEXT: ld.d $ra, $sp, 104 # 8-byte Folded Reload +; LA64F-NEXT: addi.d $sp, $sp, 112 ; LA64F-NEXT: ret ; ; LA64D-LABEL: test_d8: ; LA64D: # %bb.0: +; LA64D-NEXT: addi.d $a2, $zero, 1 +; LA64D-NEXT: movgr2fr.d $fa0, $a2 ; LA64D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI6_0) ; LA64D-NEXT: addi.d $a2, $a2, %pc_lo12(.LCPI6_0) -; LA64D-NEXT: fld.d $fa0, $a2, 0 -; LA64D-NEXT: fld.d $fa1, $a0, 8 -; LA64D-NEXT: fadd.d $fa1, $fa1, $fa0 -; LA64D-NEXT: fld.d $fa2, $a0, 40 -; LA64D-NEXT: fadd.d $fa0, $fa2, $fa0 +; LA64D-NEXT: fld.d $fa1, $a2, 0 ; LA64D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI6_1) ; LA64D-NEXT: addi.d $a2, $a2, %pc_lo12(.LCPI6_1) ; LA64D-NEXT: fld.d $fa2, $a2, 0 -; LA64D-NEXT: fld.d $fa3, $a0, 16 -; LA64D-NEXT: fadd.d $fa3, $fa3, $fa2 -; LA64D-NEXT: fld.d $fa4, $a0, 48 -; LA64D-NEXT: fadd.d $fa2, $fa4, $fa2 ; LA64D-NEXT: pcalau12i $a2, %pc_hi20(.LCPI6_2) ; LA64D-NEXT: addi.d $a2, $a2, %pc_lo12(.LCPI6_2) -; LA64D-NEXT: fld.d $fa4, $a2, 0 -; LA64D-NEXT: fld.d $fa5, $a0, 24 -; LA64D-NEXT: fadd.d $fa5, $fa5, $fa4 -; LA64D-NEXT: fld.d $fa6, $a0, 56 -; LA64D-NEXT: fadd.d $fa4, $fa6, $fa4 -; LA64D-NEXT: fst.d $fa4, $a1, 56 -; LA64D-NEXT: fst.d $fa2, $a1, 48 -; LA64D-NEXT: fst.d $fa0, $a1, 40 -; LA64D-NEXT: fst.d $fa5, $a1, 24 -; LA64D-NEXT: fst.d $fa3, $a1, 16 -; LA64D-NEXT: fst.d $fa1, $a1, 8 -; LA64D-NEXT: addi.d $a2, $zero, 1 -; LA64D-NEXT: movgr2fr.d $fa0, $a2 +; LA64D-NEXT: fld.d $fa3, $a2, 0 +; LA64D-NEXT: fld.d $fa4, $a0, 56 +; LA64D-NEXT: fld.d $fa5, $a0, 48 +; LA64D-NEXT: fld.d $fa6, $a0, 24 +; LA64D-NEXT: fld.d $fa7, $a0, 16 +; LA64D-NEXT: fld.d $ft0, $a0, 0 +; LA64D-NEXT: fld.d $ft1, $a0, 32 +; LA64D-NEXT: fld.d $ft2, $a0, 8 ; LA64D-NEXT: ffint.d.l $fa0, $fa0 -; LA64D-NEXT: fld.d $fa1, $a0, 32 -; LA64D-NEXT: fadd.d $fa1, $fa1, $fa0 -; LA64D-NEXT: fst.d $fa1, $a1, 32 -; LA64D-NEXT: fld.d $fa1, $a0, 0 -; LA64D-NEXT: fadd.d $fa0, $fa1, $fa0 -; LA64D-NEXT: fst.d $fa0, $a1, 0 +; LA64D-NEXT: fadd.d $ft0, $ft0, $fa0 +; LA64D-NEXT: fadd.d $fa0, $ft1, $fa0 +; LA64D-NEXT: fld.d $ft1, $a0, 40 +; LA64D-NEXT: fadd.d $ft2, $ft2, $fa1 +; LA64D-NEXT: fadd.d $fa7, $fa7, $fa2 +; LA64D-NEXT: fadd.d $fa6, $fa6, $fa3 +; LA64D-NEXT: fadd.d $fa1, $ft1, $fa1 +; LA64D-NEXT: fadd.d $fa2, $fa5, $fa2 +; LA64D-NEXT: fadd.d $fa3, $fa4, $fa3 +; LA64D-NEXT: fst.d $fa3, $a1, 56 +; LA64D-NEXT: fst.d $fa2, $a1, 48 +; LA64D-NEXT: fst.d $fa1, $a1, 40 +; LA64D-NEXT: fst.d $fa6, $a1, 24 +; LA64D-NEXT: fst.d $fa7, $a1, 16 +; LA64D-NEXT: fst.d $ft2, $a1, 8 +; LA64D-NEXT: fst.d $fa0, $a1, 32 +; LA64D-NEXT: fst.d $ft0, $a1, 0 ; LA64D-NEXT: ret %p = load %d8, ptr %P %R = fadd %d8 %p, < double 1.000000e+00, double 2.000000e+00, double 3.000000e+00, double 4.000000e+00, double 1.000000e+00, double 2.000000e+00, double 3.000000e+00, double 4.000000e+00 > diff --git a/llvm/test/CodeGen/LoongArch/zext-with-load-is-free.ll b/llvm/test/CodeGen/LoongArch/zext-with-load-is-free.ll index d05a0c745348..d5c505f7160e 100644 --- a/llvm/test/CodeGen/LoongArch/zext-with-load-is-free.ll +++ b/llvm/test/CodeGen/LoongArch/zext-with-load-is-free.ll @@ -23,10 +23,10 @@ exit: define zeroext i16 @test_zext_i16(ptr %p) nounwind { ; LA32-LABEL: test_zext_i16: ; LA32: # %bb.0: -; LA32-NEXT: ld.bu $a1, $a0, 0 -; LA32-NEXT: ld.bu $a0, $a0, 1 -; LA32-NEXT: slli.w $a0, $a0, 8 -; LA32-NEXT: or $a0, $a0, $a1 +; LA32-NEXT: ld.bu $a1, $a0, 1 +; LA32-NEXT: ld.bu $a0, $a0, 0 +; LA32-NEXT: slli.w $a1, $a1, 8 +; LA32-NEXT: or $a0, $a1, $a0 ; LA32-NEXT: ret ; ; LA64-LABEL: test_zext_i16: diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/loongarch_generated_funcs.ll.generated.expected b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/loongarch_generated_funcs.ll.generated.expected index e5bdc8b010e4..56b6c90a2f6f 100644 --- a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/loongarch_generated_funcs.ll.generated.expected +++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/loongarch_generated_funcs.ll.generated.expected @@ -72,11 +72,11 @@ attributes #0 = { noredzone nounwind ssp uwtable "frame-pointer"="all" } ; CHECK-NEXT: .cfi_offset 22, -8 ; CHECK-NEXT: addi.w $fp, $sp, 32 ; CHECK-NEXT: .cfi_def_cfa 22, 0 -; CHECK-NEXT: st.w $zero, $fp, -16 ; CHECK-NEXT: st.w $zero, $fp, -12 +; CHECK-NEXT: st.w $zero, $fp, -16 +; CHECK-NEXT: ori $a0, $zero, 1 ; CHECK-NEXT: beqz $zero, .LBB0_3 ; CHECK-NEXT: # %bb.1: -; CHECK-NEXT: ori $a0, $zero, 1 ; CHECK-NEXT: st.w $a0, $fp, -24 ; CHECK-NEXT: ld.w $a0, $fp, -16 ; CHECK-NEXT: beqz $a0, .LBB0_4 @@ -85,10 +85,9 @@ attributes #0 = { noredzone nounwind ssp uwtable "frame-pointer"="all" } ; CHECK-NEXT: st.w $a0, $fp, -24 ; CHECK-NEXT: b .LBB0_5 ; CHECK-NEXT: .LBB0_3: +; CHECK-NEXT: st.w $a0, $fp, -16 ; CHECK-NEXT: ori $a0, $zero, 2 ; CHECK-NEXT: st.w $a0, $fp, -20 -; CHECK-NEXT: ori $a0, $zero, 1 -; CHECK-NEXT: st.w $a0, $fp, -16 ; CHECK-NEXT: ori $a0, $zero, 3 ; CHECK-NEXT: st.w $a0, $fp, -24 ; CHECK-NEXT: ori $a0, $zero, 4 @@ -96,10 +95,10 @@ attributes #0 = { noredzone nounwind ssp uwtable "frame-pointer"="all" } ; CHECK-NEXT: ld.w $a0, $fp, -16 ; CHECK-NEXT: bnez $a0, .LBB0_2 ; CHECK-NEXT: .LBB0_4: -; CHECK-NEXT: ori $a0, $zero, 2 -; CHECK-NEXT: st.w $a0, $fp, -20 ; CHECK-NEXT: ori $a0, $zero, 1 ; CHECK-NEXT: st.w $a0, $fp, -16 +; CHECK-NEXT: ori $a0, $zero, 2 +; CHECK-NEXT: st.w $a0, $fp, -20 ; CHECK-NEXT: ori $a0, $zero, 3 ; CHECK-NEXT: st.w $a0, $fp, -24 ; CHECK-NEXT: ori $a0, $zero, 4 @@ -121,24 +120,24 @@ attributes #0 = { noredzone nounwind ssp uwtable "frame-pointer"="all" } ; CHECK-NEXT: .cfi_offset 22, -8 ; CHECK-NEXT: addi.w $fp, $sp, 32 ; CHECK-NEXT: .cfi_def_cfa 22, 0 +; CHECK-NEXT: st.w $zero, $fp, -12 ; CHECK-NEXT: pcalau12i $a0, %pc_hi20(x) ; CHECK-NEXT: addi.w $a0, $a0, %pc_lo12(x) ; CHECK-NEXT: ori $a1, $zero, 1 -; CHECK-NEXT: st.w $a1, $a0, 0 -; CHECK-NEXT: st.w $zero, $fp, -12 ; CHECK-NEXT: st.w $a1, $fp, -16 -; CHECK-NEXT: ori $a0, $zero, 2 -; CHECK-NEXT: st.w $a0, $fp, -20 -; CHECK-NEXT: ori $a2, $zero, 3 -; CHECK-NEXT: st.w $a2, $fp, -24 -; CHECK-NEXT: ori $a3, $zero, 4 -; CHECK-NEXT: st.w $a3, $fp, -28 +; CHECK-NEXT: ori $a2, $zero, 2 +; CHECK-NEXT: st.w $a2, $fp, -20 +; CHECK-NEXT: ori $a3, $zero, 3 +; CHECK-NEXT: st.w $a3, $fp, -24 +; CHECK-NEXT: ori $a4, $zero, 4 +; CHECK-NEXT: st.w $a4, $fp, -28 +; CHECK-NEXT: st.w $a1, $a0, 0 ; CHECK-NEXT: #APP ; CHECK-NEXT: #NO_APP -; CHECK-NEXT: st.w $a0, $fp, -20 ; CHECK-NEXT: st.w $a1, $fp, -16 -; CHECK-NEXT: st.w $a2, $fp, -24 -; CHECK-NEXT: st.w $a3, $fp, -28 +; CHECK-NEXT: st.w $a2, $fp, -20 +; CHECK-NEXT: st.w $a3, $fp, -24 +; CHECK-NEXT: st.w $a4, $fp, -28 ; CHECK-NEXT: move $a0, $zero ; CHECK-NEXT: ld.w $fp, $sp, 24 # 4-byte Folded Reload ; CHECK-NEXT: ld.w $ra, $sp, 28 # 4-byte Folded Reload diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/loongarch_generated_funcs.ll.nogenerated.expected b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/loongarch_generated_funcs.ll.nogenerated.expected index 20e34cdf3c64..2e063202fcf7 100644 --- a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/loongarch_generated_funcs.ll.nogenerated.expected +++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/loongarch_generated_funcs.ll.nogenerated.expected @@ -13,11 +13,11 @@ define dso_local i32 @check_boundaries() #0 { ; CHECK-NEXT: .cfi_offset 22, -8 ; CHECK-NEXT: addi.w $fp, $sp, 32 ; CHECK-NEXT: .cfi_def_cfa 22, 0 -; CHECK-NEXT: st.w $zero, $fp, -16 ; CHECK-NEXT: st.w $zero, $fp, -12 +; CHECK-NEXT: st.w $zero, $fp, -16 +; CHECK-NEXT: ori $a0, $zero, 1 ; CHECK-NEXT: beqz $zero, .LBB0_3 ; CHECK-NEXT: # %bb.1: -; CHECK-NEXT: ori $a0, $zero, 1 ; CHECK-NEXT: st.w $a0, $fp, -24 ; CHECK-NEXT: ld.w $a0, $fp, -16 ; CHECK-NEXT: beqz $a0, .LBB0_4 @@ -26,10 +26,9 @@ define dso_local i32 @check_boundaries() #0 { ; CHECK-NEXT: st.w $a0, $fp, -24 ; CHECK-NEXT: b .LBB0_5 ; CHECK-NEXT: .LBB0_3: +; CHECK-NEXT: st.w $a0, $fp, -16 ; CHECK-NEXT: ori $a0, $zero, 2 ; CHECK-NEXT: st.w $a0, $fp, -20 -; CHECK-NEXT: ori $a0, $zero, 1 -; CHECK-NEXT: st.w $a0, $fp, -16 ; CHECK-NEXT: ori $a0, $zero, 3 ; CHECK-NEXT: st.w $a0, $fp, -24 ; CHECK-NEXT: ori $a0, $zero, 4 @@ -37,10 +36,10 @@ define dso_local i32 @check_boundaries() #0 { ; CHECK-NEXT: ld.w $a0, $fp, -16 ; CHECK-NEXT: bnez $a0, .LBB0_2 ; CHECK-NEXT: .LBB0_4: -; CHECK-NEXT: ori $a0, $zero, 2 -; CHECK-NEXT: st.w $a0, $fp, -20 ; CHECK-NEXT: ori $a0, $zero, 1 ; CHECK-NEXT: st.w $a0, $fp, -16 +; CHECK-NEXT: ori $a0, $zero, 2 +; CHECK-NEXT: st.w $a0, $fp, -20 ; CHECK-NEXT: ori $a0, $zero, 3 ; CHECK-NEXT: st.w $a0, $fp, -24 ; CHECK-NEXT: ori $a0, $zero, 4 @@ -98,24 +97,24 @@ define dso_local i32 @main() #0 { ; CHECK-NEXT: .cfi_offset 22, -8 ; CHECK-NEXT: addi.w $fp, $sp, 32 ; CHECK-NEXT: .cfi_def_cfa 22, 0 +; CHECK-NEXT: st.w $zero, $fp, -12 ; CHECK-NEXT: pcalau12i $a0, %pc_hi20(x) ; CHECK-NEXT: addi.w $a0, $a0, %pc_lo12(x) ; CHECK-NEXT: ori $a1, $zero, 1 -; CHECK-NEXT: st.w $a1, $a0, 0 -; CHECK-NEXT: st.w $zero, $fp, -12 ; CHECK-NEXT: st.w $a1, $fp, -16 -; CHECK-NEXT: ori $a0, $zero, 2 -; CHECK-NEXT: st.w $a0, $fp, -20 -; CHECK-NEXT: ori $a2, $zero, 3 -; CHECK-NEXT: st.w $a2, $fp, -24 -; CHECK-NEXT: ori $a3, $zero, 4 -; CHECK-NEXT: st.w $a3, $fp, -28 +; CHECK-NEXT: ori $a2, $zero, 2 +; CHECK-NEXT: st.w $a2, $fp, -20 +; CHECK-NEXT: ori $a3, $zero, 3 +; CHECK-NEXT: st.w $a3, $fp, -24 +; CHECK-NEXT: ori $a4, $zero, 4 +; CHECK-NEXT: st.w $a4, $fp, -28 +; CHECK-NEXT: st.w $a1, $a0, 0 ; CHECK-NEXT: #APP ; CHECK-NEXT: #NO_APP -; CHECK-NEXT: st.w $a0, $fp, -20 ; CHECK-NEXT: st.w $a1, $fp, -16 -; CHECK-NEXT: st.w $a2, $fp, -24 -; CHECK-NEXT: st.w $a3, $fp, -28 +; CHECK-NEXT: st.w $a2, $fp, -20 +; CHECK-NEXT: st.w $a3, $fp, -24 +; CHECK-NEXT: st.w $a4, $fp, -28 ; CHECK-NEXT: move $a0, $zero ; CHECK-NEXT: ld.w $fp, $sp, 24 # 4-byte Folded Reload ; CHECK-NEXT: ld.w $ra, $sp, 28 # 4-byte Folded Reload -- GitLab From 6f5c4f2eacf24cecfc0faf0a204137ce65eecc2d Mon Sep 17 00:00:00 2001 From: "Balaji V. Iyer" <43187390+bviyer@users.noreply.github.com> Date: Mon, 4 Mar 2024 19:17:51 -0600 Subject: [PATCH 080/929] [mlir][vector]Add Vector bitwidth target to Linearize Vectorizable and Constant Ops (#83314) Added a new flag `targetVectorBitwidth` to capture bit-width input. --- .../Vector/Transforms/VectorRewritePatterns.h | 2 +- .../Vector/Transforms/VectorLinearize.cpp | 61 ++++++++++++--- mlir/test/Dialect/Vector/linearize.mlir | 75 ++++++++++++++++++- .../Dialect/Vector/TestVectorTransforms.cpp | 12 ++- 4 files changed, 136 insertions(+), 14 deletions(-) diff --git a/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h b/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h index 46bb3ddec0ba..453fa73429dd 100644 --- a/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h +++ b/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h @@ -387,7 +387,7 @@ void populateVectorTransposeNarrowTypeRewritePatterns( /// the ops to get converted properly. void populateVectorLinearizeTypeConversionsAndLegality( TypeConverter &typeConverter, RewritePatternSet &patterns, - ConversionTarget &target); + ConversionTarget &target, unsigned targetBitWidth); } // namespace vector } // namespace mlir diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp index c53520439555..7ca035370498 100644 --- a/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp +++ b/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp @@ -19,10 +19,30 @@ using namespace mlir; +static bool isLessThanTargetBitWidth(Operation *op, unsigned targetBitWidth) { + auto resultTypes = op->getResultTypes(); + for (auto resType : resultTypes) { + VectorType vecType = cast(resType); + // Reject index since getElementTypeBitWidth will abort for Index types. + if (vecType.getElementType().isIndex()) + return false; + unsigned trailingVecDimBitWidth = + vecType.getShape().back() * vecType.getElementTypeBitWidth(); + if (trailingVecDimBitWidth >= targetBitWidth) + return false; + } + return true; +} + namespace { struct LinearizeConstant final : OpConversionPattern { using OpConversionPattern::OpConversionPattern; - + LinearizeConstant( + const TypeConverter &typeConverter, MLIRContext *context, + unsigned targetVectBitWidth = std::numeric_limits::max(), + PatternBenefit benefit = 1) + : OpConversionPattern(typeConverter, context, benefit), + targetVectorBitWidth(targetVectBitWidth) {} LogicalResult matchAndRewrite(arith::ConstantOp constOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override { @@ -31,7 +51,9 @@ struct LinearizeConstant final : OpConversionPattern { getTypeConverter()->convertType(constOp.getType()); if (!resType) return rewriter.notifyMatchFailure(loc, "can't convert return type"); - + if (!isLessThanTargetBitWidth(constOp, targetVectorBitWidth)) + return rewriter.notifyMatchFailure( + loc, "Can't flatten since targetBitWidth <= OpSize"); auto dstElementsAttr = dyn_cast(constOp.getValue()); if (!dstElementsAttr) return rewriter.notifyMatchFailure(loc, "unsupported attr type"); @@ -41,15 +63,28 @@ struct LinearizeConstant final : OpConversionPattern { dstElementsAttr); return success(); } + +private: + unsigned targetVectorBitWidth; }; struct LinearizeVectorizable final : OpTraitConversionPattern { using OpTraitConversionPattern::OpTraitConversionPattern; +public: + LinearizeVectorizable( + const TypeConverter &typeConverter, MLIRContext *context, + unsigned targetVectBitWidth = std::numeric_limits::max(), + PatternBenefit benefit = 1) + : OpTraitConversionPattern(typeConverter, context, benefit), + targetVectorBitWidth(targetVectBitWidth) {} LogicalResult matchAndRewrite(Operation *op, ArrayRef operands, ConversionPatternRewriter &rewriter) const override { + if (!isLessThanTargetBitWidth(op, targetVectorBitWidth)) + return rewriter.notifyMatchFailure( + op->getLoc(), "Can't flatten since targetBitWidth <= OpSize"); FailureOr newOp = convertOpResultTypes(op, operands, *getTypeConverter(), rewriter); if (failed(newOp)) @@ -58,12 +93,16 @@ struct LinearizeVectorizable final rewriter.replaceOp(op, (*newOp)->getResults()); return success(); } + +private: + unsigned targetVectorBitWidth; }; } // namespace void mlir::vector::populateVectorLinearizeTypeConversionsAndLegality( TypeConverter &typeConverter, RewritePatternSet &patterns, - ConversionTarget &target) { + ConversionTarget &target, unsigned targetBitWidth) { + typeConverter.addConversion([](VectorType type) -> std::optional { // Ignore scalable vectors for now. if (type.getRank() <= 1 || type.isScalable()) @@ -83,15 +122,17 @@ void mlir::vector::populateVectorLinearizeTypeConversionsAndLegality( typeConverter.addArgumentMaterialization(materializeCast); typeConverter.addSourceMaterialization(materializeCast); typeConverter.addTargetMaterialization(materializeCast); - target.markUnknownOpDynamicallyLegal( - [&](Operation *op) -> std::optional { - if (isa(op) || op->hasTrait()) - return typeConverter.isLegal(op); - + [=](Operation *op) -> std::optional { + if ((isa(op) || + op->hasTrait())) { + return (isLessThanTargetBitWidth(op, targetBitWidth) + ? typeConverter.isLegal(op) + : true); + } return std::nullopt; }); - patterns.add(typeConverter, - patterns.getContext()); + patterns.add( + typeConverter, patterns.getContext(), targetBitWidth); } diff --git a/mlir/test/Dialect/Vector/linearize.mlir b/mlir/test/Dialect/Vector/linearize.mlir index 85e23103eaed..2cbf9bec7a41 100644 --- a/mlir/test/Dialect/Vector/linearize.mlir +++ b/mlir/test/Dialect/Vector/linearize.mlir @@ -1,19 +1,92 @@ // RUN: mlir-opt %s -split-input-file -test-vector-linearize | FileCheck %s +// RUN: mlir-opt %s -split-input-file -test-vector-linearize=target-vector-bitwidth=128 | FileCheck %s --check-prefix=CHECK128 +// RUN: mlir-opt %s -split-input-file -test-vector-linearize=target-vector-bitwidth=0 | FileCheck %s --check-prefix=CHECK0 // CHECK-LABEL: test_linearize +// CHECK128-LABEL: test_linearize +// CHECK0-LABEL: test_linearize // CHECK-SAME: (%[[ORIG_ARG:.*]]: vector<2x2xf32>) +// CHECK128-SAME: (%[[ORIG_ARG:.*]]: vector<2x2xf32>) // CHECK: %[[ARG:.*]] = vector.shape_cast %[[ORIG_ARG]] : vector<2x2xf32> to vector<4xf32> +// CHECK128: %[[ARG:.*]] = vector.shape_cast %[[ORIG_ARG]] : vector<2x2xf32> to vector<4xf32> func.func @test_linearize(%arg0: vector<2x2xf32>) -> vector<2x2xf32> { // CHECK: %[[C1:.*]] = arith.constant dense<[1.000000e+00, 2.000000e+00, 3.000000e+00, 4.000000e+00]> : vector<4xf32> +// CHECK128: %[[C1:.*]] = arith.constant dense<[1.000000e+00, 2.000000e+00, 3.000000e+00, 4.000000e+00]> : vector<4xf32> +// CHECK0: %[[C1:.*]] = arith.constant dense<{{.*}}> : vector<2x2xf32> + %0 = arith.constant dense<[[1.0, 2.0], [3.0, 4.0]]> : vector<2x2xf32> // CHECK: %[[RES:.*]] = vector.shape_cast %[[C1]] : vector<4xf32> to vector<2x2xf32> - +// CHECK128: %[[RES:.*]] = vector.shape_cast %[[C1]] : vector<4xf32> to vector<2x2xf32> // Arith and math ops are handled in generic way, check some of them // CHECK: %{{.*}} = math.sin %[[ARG]] : vector<4xf32> +// CHECK128: %{{.*}} = math.sin %[[ARG]] : vector<4xf32> +// CHECK0: %{{.*}} = math.sin %{{.*}} : vector<2x2xf32> + %1 = math.sin %arg0 : vector<2x2xf32> +// CHECK: %{{.*}} = arith.addf %[[ARG]], %[[C1]] : vector<4xf32> +// CHECK128: %{{.*}} = arith.addf %[[ARG]], %[[C1]] : vector<4xf32> +// CHECK0: %{{.*}} = arith.addf %{{.*}} : vector<2x2xf32> + + %2 = arith.addf %arg0, %0 : vector<2x2xf32> + +// CHECK: return %[[RES]] : vector<2x2xf32> +// CHECK128: return %[[RES]] : vector<2x2xf32> + return %0 : vector<2x2xf32> +} + +// CHECK-LABEL: test_partial_linearize +// CHECK128-LABEL: test_partial_linearize +// CHECK0-LABEL: test_partial_linearize +// CHECK-SAME: (%[[ORIG_ARG:.*]]: vector<2x2xf32>, %[[ORIG_ARG2:.*]]: vector<4x4xf32>) +// CHECK128-SAME: (%[[ORIG_ARG:.*]]: vector<2x2xf32>, %[[ORIG_ARG2:.*]]: vector<4x4xf32>) +// CHECK0-SAME: (%[[ORIG_ARG:.*]]: vector<2x2xf32>, %[[ORIG_ARG2:.*]]: vector<4x4xf32>) +// CHECK: %[[ARG:.*]] = vector.shape_cast %[[ORIG_ARG]] : vector<2x2xf32> to vector<4xf32> +// CHECK128: %[[ARG:.*]] = vector.shape_cast %[[ORIG_ARG]] : vector<2x2xf32> to vector<4xf32> +// CHECK: %[[ARG2:.*]] = vector.shape_cast %[[ORIG_ARG2]] : vector<4x4xf32> to vector<16xf32> +func.func @test_partial_linearize(%arg0: vector<2x2xf32>, %arg1: vector<4x4xf32>) -> vector<2x2xf32> { +// CHECK: %[[C1:.*]] = arith.constant dense<[1.000000e+00, 2.000000e+00, 3.000000e+00, 4.000000e+00]> : vector<4xf32> +// CHECK128: %[[C1:.*]] = arith.constant dense<[1.000000e+00, 2.000000e+00, 3.000000e+00, 4.000000e+00]> : vector<4xf32> +// CHECK0: %[[C1:.*]] = arith.constant dense<{{.*}}> : vector<2x2xf32> + + %0 = arith.constant dense<[[1.0, 2.0], [3.0, 4.0]]> : vector<2x2xf32> +// CHECK: %[[RES:.*]] = vector.shape_cast %[[C1]] : vector<4xf32> to vector<2x2xf32> +// CHECK128: %[[RES:.*]] = vector.shape_cast %[[C1]] : vector<4xf32> to vector<2x2xf32> + + // CHECK: %[[C2:.*]] = arith.constant dense<[1.000000e+00, 2.000000e+00, 3.000000e+00, 4.000000e+00, 1.000000e+00, 2.000000e+00, 3.000000e+00, 4.000000e+00, 1.000000e+00, 2.000000e+00, 3.000000e+00, 4.000000e+00, 1.000000e+00, 2.000000e+00, 5.000000e+00, 6.000000e+00]> : vector<16xf32> + // CHECK128: %[[C2:.*]] = arith.constant dense<{{.*}}> : vector<4x4xf32> + // CHECK0: %[[C2:.*]] = arith.constant dense<{{.*}}> : vector<4x4xf32> + %5 = arith.constant dense<[[1.0, 2.0, 3.0, 4.0], [1.0, 2.0,3.0, 4.0], [1.0, 2.0, 3.0, 4.0], [1.0, 2.0, 5.0, 6.0]]> : vector<4x4xf32> +// Arith and math ops are handled in generic way, check some of them +// CHECK: %[[SIN:.*]] = math.sin %[[ARG]] : vector<4xf32> +// CHECK128: %[[SIN:.*]] = math.sin %[[ARG]] : vector<4xf32> +// CHECK0: %[[SIN:.*]] = math.sin %[[ORIG_ARG]] : vector<2x2xf32> %1 = math.sin %arg0 : vector<2x2xf32> + + // CHECK: %[[SIN1:.*]] = math.sin %[[ARG2]] : vector<16xf32> +// CHECK128: %[[SIN1:.*]] = math.sin %[[ORIG_ARG2]] : vector<4x4xf32> +// CHECK0: %[[SIN1:.*]] = math.sin %[[ORIG_ARG2]] : vector<4x4xf32> + %6 = math.sin %arg1 : vector<4x4xf32> // CHECK: %{{.*}} = arith.addf %[[ARG]], %[[C1]] : vector<4xf32> +// CHECK128: %{{.*}} = arith.addf %[[ARG]], %[[C1]] : vector<4xf32> +// CHECK0: %{{.*}} = arith.addf %{{.*}} : vector<2x2xf32> + %2 = arith.addf %arg0, %0 : vector<2x2xf32> + // CHECK: %[[ADD2:.*]] = arith.addf %[[ARG2]], %[[C2]] : vector<16xf32> + // CHECK128: %[[ADD2:.*]] = arith.addf %[[ORIG_ARG2]], %[[C2]] : vector<4x4xf32> + // CHECK0: %[[ADD2:.*]] = arith.addf %[[ORIG_ARG2]], %[[C2]] : vector<4x4xf32> + %7 = arith.addf %arg1, %5 : vector<4x4xf32> // CHECK: return %[[RES]] : vector<2x2xf32> +// CHECK128: return %[[RES]] : vector<2x2xf32> return %0 : vector<2x2xf32> } + +// CHECK-LABEL: test_index_no_linearize +// CHECK128-LABEL: test_index_no_linearize +// CHECK0-LABEL: test_index_no_linearize +func.func @test_index_no_linearize(%arg0: vector<2x2xindex>, %arg1: vector<2x2xindex>) -> vector<2x2xindex> { + // CHECK: %[[ADD:.*]] = arith.addi {{.*}} : vector<2x2xindex> + // CHECK128: %[[ADD:.*]] = arith.addi {{.*}} : vector<2x2xindex> + // CHECK0: %[[ADD:.*]] = arith.addi {{.*}} : vector<2x2xindex> + %0 = arith.addi %arg0, %arg1 : vector<2x2xindex> + return %0 : vector<2x2xindex> +} diff --git a/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp b/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp index 915f713f7047..f14fb18706d1 100644 --- a/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp +++ b/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp @@ -840,6 +840,9 @@ struct TestVectorLinearize final : public PassWrapper> { MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestVectorLinearize) + TestVectorLinearize() = default; + TestVectorLinearize(const TestVectorLinearize &pass) : PassWrapper(pass) {} + StringRef getArgument() const override { return "test-vector-linearize"; } StringRef getDescription() const override { return "Linearizes ND vectors for N >= 2 into 1D vectors"; @@ -848,6 +851,11 @@ struct TestVectorLinearize final registry.insert(); } + Option targetVectorBitwidth{ + *this, "target-vector-bitwidth", + llvm::cl::desc( + "Minimum vector bitwidth to enable the flattening transformation"), + llvm::cl::init(std::numeric_limits::max())}; void runOnOperation() override { auto *context = &getContext(); @@ -855,8 +863,8 @@ struct TestVectorLinearize final RewritePatternSet patterns(context); ConversionTarget target(*context); - vector::populateVectorLinearizeTypeConversionsAndLegality(typeConverter, - patterns, target); + vector::populateVectorLinearizeTypeConversionsAndLegality( + typeConverter, patterns, target, targetVectorBitwidth); if (failed(applyPartialConversion(getOperation(), target, std::move(patterns)))) return signalPassFailure(); -- GitLab From ccf0c8da1a0e6eea5e31fd5872ac864bf7005147 Mon Sep 17 00:00:00 2001 From: Maksim Panchenko Date: Mon, 4 Mar 2024 17:24:16 -0800 Subject: [PATCH 081/929] [BOLT] Add reading support for Linux kernel exception table (#83100) Read Linux exception table and ignore functions with exceptions for now. Proper support requires an introduction of new control flow since some instructions with memory access can cause a control flow change. Hence looking at disassembly or CFG with exceptions annotations is valuable for code analysis, delay marking functions with exceptions as non-simple until immediately before emitting the code. --- bolt/lib/Rewrite/LinuxKernelRewriter.cpp | 210 +++++++++++++++-------- bolt/test/X86/linux-exceptions.s | 64 +++++++ 2 files changed, 204 insertions(+), 70 deletions(-) create mode 100644 bolt/test/X86/linux-exceptions.s diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp index 0d7dc1070ce7..145acd32356c 100644 --- a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp +++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp @@ -14,6 +14,7 @@ #include "bolt/Rewrite/MetadataRewriter.h" #include "bolt/Rewrite/MetadataRewriters.h" #include "bolt/Utils/CommandLineOpts.h" +#include "llvm/ADT/DenseSet.h" #include "llvm/Support/BinaryStreamWriter.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" @@ -27,9 +28,9 @@ using namespace bolt; namespace opts { static cl::opt - PrintORC("print-orc", - cl::desc("print ORC unwind information for instructions"), - cl::init(true), cl::Hidden, cl::cat(BoltCategory)); + DumpExceptions("dump-linux-exceptions", + cl::desc("dump Linux kernel exception table"), + cl::init(false), cl::Hidden, cl::cat(BoltCategory)); static cl::opt DumpORC("dump-orc", cl::desc("dump raw ORC unwind information (sorted)"), @@ -40,6 +41,11 @@ static cl::opt DumpStaticCalls("dump-static-calls", cl::init(false), cl::Hidden, cl::cat(BoltCategory)); +static cl::opt + PrintORC("print-orc", + cl::desc("print ORC unwind information for instructions"), + cl::init(true), cl::Hidden, cl::cat(BoltCategory)); + } // namespace opts /// Linux Kernel supports stack unwinding using ORC (oops rewind capability). @@ -134,6 +140,13 @@ class LinuxKernelRewriter final : public MetadataRewriter { using StaticCallListType = std::vector; StaticCallListType StaticCallEntries; + /// Section containing the Linux exception table. + ErrorOr ExceptionsSection = std::errc::bad_address; + static constexpr size_t EXCEPTION_TABLE_ENTRY_SIZE = 12; + + /// Functions with exception handling code. + DenseSet FunctionsWithExceptions; + /// Insert an LKMarker for a given code pointer \p PC from a non-code section /// \p SectionName. void insertLKMarker(uint64_t PC, uint64_t SectionOffset, @@ -143,9 +156,6 @@ class LinuxKernelRewriter final : public MetadataRewriter { /// Process linux kernel special sections and their relocations. void processLKSections(); - /// Process special linux kernel section, __ex_table. - void processLKExTable(); - /// Process special linux kernel section, .pci_fixup. void processLKPCIFixup(); @@ -174,6 +184,9 @@ class LinuxKernelRewriter final : public MetadataRewriter { Error readStaticCalls(); Error rewriteStaticCalls(); + Error readExceptionTable(); + Error rewriteExceptionTable(); + /// Mark instructions referenced by kernel metadata. Error markInstructions(); @@ -192,6 +205,9 @@ public: if (Error E = readStaticCalls()) return E; + if (Error E = readExceptionTable()) + return E; + return Error::success(); } @@ -203,6 +219,11 @@ public: } Error preEmitFinalizer() override { + // Since rewriteExceptionTable() can mark functions as non-simple, run it + // before other rewriters that depend on simple/emit status. + if (Error E = rewriteExceptionTable()) + return E; + if (Error E = rewriteORCTables()) return E; @@ -249,7 +270,6 @@ void LinuxKernelRewriter::insertLKMarker(uint64_t PC, uint64_t SectionOffset, } void LinuxKernelRewriter::processLKSections() { - processLKExTable(); processLKPCIFixup(); processLKKSymtab(); processLKKSymtab(true); @@ -257,69 +277,6 @@ void LinuxKernelRewriter::processLKSections() { processLKSMPLocks(); } -/// Process __ex_table section of Linux Kernel. -/// This section contains information regarding kernel level exception -/// handling (https://www.kernel.org/doc/html/latest/x86/exception-tables.html). -/// More documentation is in arch/x86/include/asm/extable.h. -/// -/// The section is the list of the following structures: -/// -/// struct exception_table_entry { -/// int insn; -/// int fixup; -/// int handler; -/// }; -/// -void LinuxKernelRewriter::processLKExTable() { - ErrorOr SectionOrError = - BC.getUniqueSectionByName("__ex_table"); - if (!SectionOrError) - return; - - const uint64_t SectionSize = SectionOrError->getSize(); - const uint64_t SectionAddress = SectionOrError->getAddress(); - assert((SectionSize % 12) == 0 && - "The size of the __ex_table section should be a multiple of 12"); - for (uint64_t I = 0; I < SectionSize; I += 4) { - const uint64_t EntryAddress = SectionAddress + I; - ErrorOr Offset = BC.getSignedValueAtAddress(EntryAddress, 4); - assert(Offset && "failed reading PC-relative offset for __ex_table"); - int32_t SignedOffset = *Offset; - const uint64_t RefAddress = EntryAddress + SignedOffset; - - BinaryFunction *ContainingBF = - BC.getBinaryFunctionContainingAddress(RefAddress); - if (!ContainingBF) - continue; - - MCSymbol *ReferencedSymbol = ContainingBF->getSymbol(); - const uint64_t FunctionOffset = RefAddress - ContainingBF->getAddress(); - switch (I % 12) { - default: - llvm_unreachable("bad alignment of __ex_table"); - break; - case 0: - // insn - insertLKMarker(RefAddress, I, SignedOffset, true, "__ex_table"); - break; - case 4: - // fixup - if (FunctionOffset) - ReferencedSymbol = ContainingBF->addEntryPointAtOffset(FunctionOffset); - BC.addRelocation(EntryAddress, ReferencedSymbol, Relocation::getPC32(), 0, - *Offset); - break; - case 8: - // handler - assert(!FunctionOffset && - "__ex_table handler entry should point to function start"); - BC.addRelocation(EntryAddress, ReferencedSymbol, Relocation::getPC32(), 0, - *Offset); - break; - } - } -} - /// Process .pci_fixup section of Linux Kernel. /// This section contains a list of entries for different PCI devices and their /// corresponding hook handler (code pointer where the fixup @@ -943,6 +900,119 @@ Error LinuxKernelRewriter::rewriteStaticCalls() { return Error::success(); } +/// Instructions that access user-space memory can cause page faults. These +/// faults will be handled by the kernel and execution will resume at the fixup +/// code location if the address was invalid. The kernel uses the exception +/// table to match the faulting instruction to its fixup. The table consists of +/// the following entries: +/// +/// struct exception_table_entry { +/// int insn; +/// int fixup; +/// int data; +/// }; +/// +/// More info at: +/// https://www.kernel.org/doc/Documentation/x86/exception-tables.txt +Error LinuxKernelRewriter::readExceptionTable() { + ExceptionsSection = BC.getUniqueSectionByName("__ex_table"); + if (!ExceptionsSection) + return Error::success(); + + if (ExceptionsSection->getSize() % EXCEPTION_TABLE_ENTRY_SIZE) + return createStringError(errc::executable_format_error, + "exception table size error"); + + const uint64_t SectionAddress = ExceptionsSection->getAddress(); + DataExtractor DE(ExceptionsSection->getContents(), + BC.AsmInfo->isLittleEndian(), + BC.AsmInfo->getCodePointerSize()); + DataExtractor::Cursor Cursor(0); + uint32_t EntryID = 0; + while (Cursor && Cursor.tell() < ExceptionsSection->getSize()) { + const uint64_t InstAddress = + SectionAddress + Cursor.tell() + (int32_t)DE.getU32(Cursor); + const uint64_t FixupAddress = + SectionAddress + Cursor.tell() + (int32_t)DE.getU32(Cursor); + const uint64_t Data = DE.getU32(Cursor); + + // Consume the status of the cursor. + if (!Cursor) + return createStringError(errc::executable_format_error, + "out of bounds while reading exception table"); + + ++EntryID; + + if (opts::DumpExceptions) { + BC.outs() << "Exception Entry: " << EntryID << '\n'; + BC.outs() << "\tInsn: 0x" << Twine::utohexstr(InstAddress) << '\n' + << "\tFixup: 0x" << Twine::utohexstr(FixupAddress) << '\n' + << "\tData: 0x" << Twine::utohexstr(Data) << '\n'; + } + + MCInst *Inst = nullptr; + MCSymbol *FixupLabel = nullptr; + + BinaryFunction *InstBF = BC.getBinaryFunctionContainingAddress(InstAddress); + if (InstBF && BC.shouldEmit(*InstBF)) { + Inst = InstBF->getInstructionAtOffset(InstAddress - InstBF->getAddress()); + if (!Inst) + return createStringError(errc::executable_format_error, + "no instruction at address 0x%" PRIx64 + " in exception table", + InstAddress); + BC.MIB->addAnnotation(*Inst, "ExceptionEntry", EntryID); + FunctionsWithExceptions.insert(InstBF); + } + + if (!InstBF && opts::Verbosity) { + BC.outs() << "BOLT-INFO: no function matches instruction at 0x" + << Twine::utohexstr(InstAddress) + << " referenced by Linux exception table\n"; + } + + BinaryFunction *FixupBF = + BC.getBinaryFunctionContainingAddress(FixupAddress); + if (FixupBF && BC.shouldEmit(*FixupBF)) { + const uint64_t Offset = FixupAddress - FixupBF->getAddress(); + if (!FixupBF->getInstructionAtOffset(Offset)) + return createStringError(errc::executable_format_error, + "no instruction at fixup address 0x%" PRIx64 + " in exception table", + FixupAddress); + FixupLabel = Offset ? FixupBF->addEntryPointAtOffset(Offset) + : FixupBF->getSymbol(); + if (Inst) + BC.MIB->addAnnotation(*Inst, "Fixup", FixupLabel->getName()); + FunctionsWithExceptions.insert(FixupBF); + } + + if (!FixupBF && opts::Verbosity) { + BC.outs() << "BOLT-INFO: no function matches fixup code at 0x" + << Twine::utohexstr(FixupAddress) + << " referenced by Linux exception table\n"; + } + } + + BC.outs() << "BOLT-INFO: parsed " + << ExceptionsSection->getSize() / EXCEPTION_TABLE_ENTRY_SIZE + << " exception table entries\n"; + + return Error::success(); +} + +/// Depending on the value of CONFIG_BUILDTIME_TABLE_SORT, the kernel expects +/// the exception table to be sorted. Hence we have to sort it after code +/// reordering. +Error LinuxKernelRewriter::rewriteExceptionTable() { + // Disable output of functions with exceptions before rewrite support is + // added. + for (BinaryFunction *BF : FunctionsWithExceptions) + BF->setSimple(false); + + return Error::success(); +} + } // namespace std::unique_ptr diff --git a/bolt/test/X86/linux-exceptions.s b/bolt/test/X86/linux-exceptions.s new file mode 100644 index 000000000000..20b8c965f853 --- /dev/null +++ b/bolt/test/X86/linux-exceptions.s @@ -0,0 +1,64 @@ +# REQUIRES: system-linux + +## Check that BOLT correctly parses the Linux kernel exception table. + +# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o +# RUN: %clang %cflags -nostdlib %t.o -o %t.exe \ +# RUN: -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr + +## Verify exception bindings to instructions. + +# RUN: llvm-bolt %t.exe --print-normalized -o %t.out --keep-nops=0 \ +# RUN: --bolt-info=0 | FileCheck %s + +## Verify the bindings again on the rewritten binary with nops removed. + +# RUN: llvm-bolt %t.out -o %t.out.1 --print-normalized | FileCheck %s + +# CHECK: BOLT-INFO: Linux kernel binary detected +# CHECK: BOLT-INFO: parsed 2 exception table entries + + .text + .globl _start + .type _start, %function +_start: +# CHECK: Binary Function "_start" + nop +.L0: + mov (%rdi), %rax +# CHECK: mov +# CHECK-SAME: ExceptionEntry: 1 # Fixup: [[FIXUP:[a-zA-Z0-9_]+]] + nop +.L1: + mov (%rsi), %rax +# CHECK: mov +# CHECK-SAME: ExceptionEntry: 2 # Fixup: [[FIXUP]] + nop + ret +.LF0: +# CHECK: Secondary Entry Point: [[FIXUP]] + jmp foo + .size _start, .-_start + + .globl foo + .type foo, %function +foo: + ret + .size foo, .-foo + + +## Exception table. + .section __ex_table,"a",@progbits + .align 4 + + .long .L0 - . # instruction + .long .LF0 - . # fixup + .long 0 # data + + .long .L1 - . # instruction + .long .LF0 - . # fixup + .long 0 # data + +## Fake Linux Kernel sections. + .section __ksymtab,"a",@progbits + .section __ksymtab_gpl,"a",@progbits -- GitLab From 1a67dee089130f06c9c60dccc1463a37d7b4fce8 Mon Sep 17 00:00:00 2001 From: Jordan Rupprecht Date: Mon, 4 Mar 2024 17:50:44 -0800 Subject: [PATCH 082/929] Revert "[ORC][JITLink] Add Intel VTune support to JITLink (#81826)" This reverts commit 17efdad2296a2757813e4f11d0575ee6fb826e39. It introduces a layering violation: https://github.com/llvm/llvm-project/pull/81826#issuecomment-1977455140 --- .../Orc/Debugging/VTuneSupportPlugin.h | 145 ------------ .../Orc/TargetProcess/JITLoaderVTune.h | 31 --- .../Orc/Debugging/CMakeLists.txt | 1 - .../Orc/Debugging/VTuneSupportPlugin.cpp | 188 --------------- .../Orc/TargetProcess/CMakeLists.txt | 9 - .../Orc/TargetProcess/JITLoaderVTune.cpp | 217 ------------------ .../JITLink/x86-64/ELF_vtune.s | 52 ----- .../JITLink/x86-64/lit.local.cfg | 3 - llvm/tools/llvm-jitlink/llvm-jitlink.cpp | 18 +- 9 files changed, 1 insertion(+), 663 deletions(-) delete mode 100644 llvm/include/llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h delete mode 100644 llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h delete mode 100644 llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp delete mode 100644 llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.cpp delete mode 100644 llvm/test/ExecutionEngine/JITLink/x86-64/ELF_vtune.s diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h b/llvm/include/llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h deleted file mode 100644 index 5ded714c5f88..000000000000 --- a/llvm/include/llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h +++ /dev/null @@ -1,145 +0,0 @@ -//===--- VTuneSupportPlugin.h -- Support for VTune profiler ---*- C++ -*---===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// Handles support for registering code with VIntel Tune's Amplifier JIT API. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_EXECUTIONENGINE_ORC_AMPLIFIERSUPPORTPLUGIN_H -#define LLVM_EXECUTIONENGINE_ORC_AMPLIFIERSUPPORTPLUGIN_H - -#include "llvm/ExecutionEngine/Orc/Core.h" -#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" - -#include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h" - -namespace llvm { - -namespace orc { - -class VTuneSupportPlugin : public ObjectLinkingLayer::Plugin { -public: - VTuneSupportPlugin(ExecutorProcessControl &EPC, ExecutorAddr RegisterImplAddr, - ExecutorAddr UnregisterImplAddr, bool EmitDebugInfo) - : EPC(EPC), RegisterVTuneImplAddr(RegisterImplAddr), - UnregisterVTuneImplAddr(UnregisterImplAddr), - EmitDebugInfo(EmitDebugInfo) {} - - void modifyPassConfig(MaterializationResponsibility &MR, - jitlink::LinkGraph &G, - jitlink::PassConfiguration &Config) override; - - Error notifyEmitted(MaterializationResponsibility &MR) override; - Error notifyFailed(MaterializationResponsibility &MR) override; - Error notifyRemovingResources(JITDylib &JD, ResourceKey K) override; - void notifyTransferringResources(JITDylib &JD, ResourceKey DstKey, - ResourceKey SrcKey) override; - - static Expected> - Create(ExecutorProcessControl &EPC, JITDylib &JD, bool EmitDebugInfo, - bool TestMode = false); - -private: - ExecutorProcessControl &EPC; - ExecutorAddr RegisterVTuneImplAddr; - ExecutorAddr UnregisterVTuneImplAddr; - std::mutex PluginMutex; - uint64_t NextMethodID{0}; - DenseMap> - PendingMethodIDs; - DenseMap>> - LoadedMethodIDs; - bool EmitDebugInfo; -}; - -typedef std::vector> VTuneLineTable; - -// SI = String Index, 1-indexed into the VTuneMethodBatch::Strings table. -// SI == 0 means replace with nullptr. - -// MI = Method Index, 1-indexed into the VTuneMethodBatch::Methods table. -// MI == 0 means this is a parent method and was not inlined. - -struct VTuneMethodInfo { - VTuneLineTable LineTable; - ExecutorAddr LoadAddr; - uint64_t LoadSize; - uint64_t MethodID; - uint32_t NameSI; - uint32_t ClassFileSI; - uint32_t SourceFileSI; - uint32_t ParentMI; -}; - -typedef std::vector VTuneMethodTable; -typedef std::vector VTuneStringTable; - -struct VTuneMethodBatch { - VTuneMethodTable Methods; - VTuneStringTable Strings; -}; - -typedef std::vector> VTuneUnloadedMethodIDs; - -namespace shared { - -using SPSVTuneLineTable = SPSSequence>; -using SPSVTuneMethodInfo = - SPSTuple; -using SPSVTuneMethodTable = SPSSequence; -using SPSVTuneStringTable = SPSSequence; -using SPSVTuneMethodBatch = SPSTuple; -using SPSVTuneUnloadedMethodIDs = SPSSequence>; - -template <> class SPSSerializationTraits { -public: - static size_t size(const VTuneMethodInfo &MI) { - return SPSVTuneMethodInfo::AsArgList::size( - MI.LineTable, MI.LoadAddr, MI.LoadSize, MI.MethodID, MI.NameSI, - MI.ClassFileSI, MI.SourceFileSI, MI.ParentMI); - } - - static bool deserialize(SPSInputBuffer &IB, VTuneMethodInfo &MI) { - return SPSVTuneMethodInfo::AsArgList::deserialize( - IB, MI.LineTable, MI.LoadAddr, MI.LoadSize, MI.MethodID, MI.NameSI, - MI.ClassFileSI, MI.SourceFileSI, MI.ParentMI); - } - - static bool serialize(SPSOutputBuffer &OB, const VTuneMethodInfo &MI) { - return SPSVTuneMethodInfo::AsArgList::serialize( - OB, MI.LineTable, MI.LoadAddr, MI.LoadSize, MI.MethodID, MI.NameSI, - MI.ClassFileSI, MI.SourceFileSI, MI.ParentMI); - } -}; - -template <> -class SPSSerializationTraits { -public: - static size_t size(const VTuneMethodBatch &MB) { - return SPSVTuneMethodBatch::AsArgList::size(MB.Methods, MB.Strings); - } - - static bool deserialize(SPSInputBuffer &IB, VTuneMethodBatch &MB) { - return SPSVTuneMethodBatch::AsArgList::deserialize(IB, MB.Methods, - MB.Strings); - } - - static bool serialize(SPSOutputBuffer &OB, const VTuneMethodBatch &MB) { - return SPSVTuneMethodBatch::AsArgList::serialize(OB, MB.Methods, - MB.Strings); - } -}; - -} // end namespace shared - -} // end namespace orc - -} // end namespace llvm - -#endif diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h deleted file mode 100644 index a9fa9967ace9..000000000000 --- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h +++ /dev/null @@ -1,31 +0,0 @@ - -//===------- JITLoaderVTune.h --- Register profiler objects ------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// Register objects for access by profilers via the perf JIT interface. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_JITLOADERVTUNE_H -#define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_JITLOADERVTUNE_H - -#include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" -#include - -extern "C" llvm::orc::shared::CWrapperFunctionResult -llvm_orc_registerVTuneImpl(const char *Data, uint64_t Size); - -extern "C" llvm::orc::shared::CWrapperFunctionResult -llvm_orc_unregisterVTuneImpl(const char *Data, uint64_t Size); - -extern "C" llvm::orc::shared::CWrapperFunctionResult -llvm_orc_test_registerVTuneImpl(const char *Data, uint64_t Size); - -#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_JITLOADERVTUNE_H - - diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/Debugging/CMakeLists.txt index ed52692662a8..5bf23a7ec0bc 100644 --- a/llvm/lib/ExecutionEngine/Orc/Debugging/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/Orc/Debugging/CMakeLists.txt @@ -8,7 +8,6 @@ add_llvm_component_library(LLVMOrcDebugging DebuggerSupportPlugin.cpp LLJITUtilsCBindings.cpp PerfSupportPlugin.cpp - VTuneSupportPlugin.cpp ADDITIONAL_HEADER_DIRS ${LLVM_MAIN_INCLUDE_DIR}/llvm/ExecutionEngine/Orc/Debugging/ diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp deleted file mode 100644 index e21ed4bf84d0..000000000000 --- a/llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp +++ /dev/null @@ -1,188 +0,0 @@ -//===--- VTuneSupportPlugin.cpp -- Support for VTune profiler --*- C++ -*--===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// Handles support for registering code with VIntel Tune's Amplfiier JIT API. -// -//===----------------------------------------------------------------------===// -#include "llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h" -#include "llvm/DebugInfo/DWARF/DWARFContext.h" -#include "llvm/ExecutionEngine/Orc/Debugging/DebugInfoSupport.h" - -using namespace llvm; -using namespace llvm::orc; -using namespace llvm::jitlink; - -namespace { - -constexpr StringRef RegisterVTuneImplName = "llvm_orc_registerVTuneImpl"; -constexpr StringRef UnregisterVTuneImplName = "llvm_orc_unregisterVTuneImpl"; -constexpr StringRef RegisterTestVTuneImplName = - "llvm_orc_test_registerVTuneImpl"; - -static VTuneMethodBatch getMethodBatch(LinkGraph &G, bool EmitDebugInfo) { - VTuneMethodBatch Batch; - std::unique_ptr DC; - StringMap> DCBacking; - if (EmitDebugInfo) { - auto EDC = createDWARFContext(G); - if (!EDC) { - EmitDebugInfo = false; - } else { - DC = std::move(EDC->first); - DCBacking = std::move(EDC->second); - } - } - - auto GetStringIdx = [Deduplicator = StringMap(), - &Batch](StringRef S) mutable { - auto I = Deduplicator.find(S); - if (I != Deduplicator.end()) - return I->second; - - Batch.Strings.push_back(S.str()); - return Deduplicator[S] = Batch.Strings.size(); - }; - for (auto Sym : G.defined_symbols()) { - if (!Sym->isCallable()) - continue; - - Batch.Methods.push_back(VTuneMethodInfo()); - auto &Method = Batch.Methods.back(); - Method.MethodID = 0; - Method.ParentMI = 0; - Method.LoadAddr = Sym->getAddress(); - Method.LoadSize = Sym->getSize(); - Method.NameSI = GetStringIdx(Sym->getName()); - Method.ClassFileSI = 0; - Method.SourceFileSI = 0; - - if (!EmitDebugInfo) - continue; - - auto &Section = Sym->getBlock().getSection(); - auto Addr = Sym->getAddress(); - auto SAddr = - object::SectionedAddress{Addr.getValue(), Section.getOrdinal()}; - DILineInfoTable LinesInfo = DC->getLineInfoForAddressRange( - SAddr, Sym->getSize(), - DILineInfoSpecifier::FileLineInfoKind::AbsoluteFilePath); - Method.SourceFileSI = Batch.Strings.size(); - Batch.Strings.push_back(DC->getLineInfoForAddress(SAddr).FileName); - for (auto &LInfo : LinesInfo) { - Method.LineTable.push_back( - std::pair{/*unsigned*/ Sym->getOffset(), - /*DILineInfo*/ LInfo.second.Line}); - } - } - return Batch; -} - -} // namespace - -void VTuneSupportPlugin::modifyPassConfig(MaterializationResponsibility &MR, - LinkGraph &G, - PassConfiguration &Config) { - Config.PostFixupPasses.push_back([this, MR = &MR](LinkGraph &G) { - // the object file is generated but not linked yet - auto Batch = getMethodBatch(G, EmitDebugInfo); - if (Batch.Methods.empty()) { - return Error::success(); - } - { - std::lock_guard Lock(PluginMutex); - uint64_t Allocated = Batch.Methods.size(); - uint64_t Start = NextMethodID; - NextMethodID += Allocated; - for (size_t i = Start; i < NextMethodID; ++i) { - Batch.Methods[i - Start].MethodID = i; - } - this->PendingMethodIDs[MR] = {Start, Allocated}; - } - G.allocActions().push_back( - {cantFail(shared::WrapperFunctionCall::Create< - shared::SPSArgList>( - RegisterVTuneImplAddr, Batch)), - {}}); - return Error::success(); - }); -} - -Error VTuneSupportPlugin::notifyEmitted(MaterializationResponsibility &MR) { - if (auto Err = MR.withResourceKeyDo([this, MR = &MR](ResourceKey K) { - std::lock_guard Lock(PluginMutex); - auto I = PendingMethodIDs.find(MR); - if (I == PendingMethodIDs.end()) - return; - - LoadedMethodIDs[K].push_back(I->second); - PendingMethodIDs.erase(I); - })) { - return Err; - } - return Error::success(); -} - -Error VTuneSupportPlugin::notifyFailed(MaterializationResponsibility &MR) { - std::lock_guard Lock(PluginMutex); - PendingMethodIDs.erase(&MR); - return Error::success(); -} - -Error VTuneSupportPlugin::notifyRemovingResources(JITDylib &JD, ResourceKey K) { - // Unregistration not required if not provided - if (!UnregisterVTuneImplAddr) { - return Error::success(); - } - VTuneUnloadedMethodIDs UnloadedIDs; - { - std::lock_guard Lock(PluginMutex); - auto I = LoadedMethodIDs.find(K); - if (I == LoadedMethodIDs.end()) - return Error::success(); - - UnloadedIDs = std::move(I->second); - LoadedMethodIDs.erase(I); - } - if (auto Err = EPC.callSPSWrapper( - UnregisterVTuneImplAddr, UnloadedIDs)) - return Err; - - return Error::success(); -} - -void VTuneSupportPlugin::notifyTransferringResources(JITDylib &JD, - ResourceKey DstKey, - ResourceKey SrcKey) { - std::lock_guard Lock(PluginMutex); - auto I = LoadedMethodIDs.find(SrcKey); - if (I == LoadedMethodIDs.end()) - return; - - auto &Dest = LoadedMethodIDs[DstKey]; - Dest.insert(Dest.end(), I->second.begin(), I->second.end()); - LoadedMethodIDs.erase(SrcKey); -} - -Expected> -VTuneSupportPlugin::Create(ExecutorProcessControl &EPC, JITDylib &JD, - bool EmitDebugInfo, bool TestMode) { - auto &ES = EPC.getExecutionSession(); - auto RegisterImplName = - ES.intern(TestMode ? RegisterTestVTuneImplName : RegisterVTuneImplName); - auto UnregisterImplName = ES.intern(UnregisterVTuneImplName); - SymbolLookupSet SLS{RegisterImplName, UnregisterImplName}; - auto Res = ES.lookup(makeJITDylibSearchOrder({&JD}), std::move(SLS)); - if (!Res) - return Res.takeError(); - ExecutorAddr RegisterImplAddr( - Res->find(RegisterImplName)->second.getAddress()); - ExecutorAddr UnregisterImplAddr( - Res->find(UnregisterImplName)->second.getAddress()); - return std::make_unique( - EPC, RegisterImplAddr, UnregisterImplAddr, EmitDebugInfo); -} diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt index 4ab8dae4a9ba..f2005dc1775e 100644 --- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt @@ -2,18 +2,10 @@ if( CMAKE_HOST_UNIX AND HAVE_LIBRT ) set(rt_lib rt) endif() -set(intel_jit_profiling ) -if( LLVM_USE_INTEL_JITEVENTS ) -set(intel_jit_profiling IntelJITProfiling) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../IntelJITProfiling) -include_directories(${PROJECT_BINARY_DIR}/ittapi/include/ ) -endif() - add_llvm_component_library(LLVMOrcTargetProcess ExecutorSharedMemoryMapperService.cpp JITLoaderGDB.cpp JITLoaderPerf.cpp - JITLoaderVTune.cpp OrcRTBootstrap.cpp RegisterEHFrames.cpp SimpleExecutorDylibManager.cpp @@ -29,7 +21,6 @@ add_llvm_component_library(LLVMOrcTargetProcess ${rt_lib} LINK_COMPONENTS - ${intel_jit_profiling} OrcShared Support TargetParser diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.cpp deleted file mode 100644 index e109d230bc62..000000000000 --- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.cpp +++ /dev/null @@ -1,217 +0,0 @@ -//===------- JITLoaderVTune.cpp - Register profiler objects -----*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// Register objects for access by profilers via the VTune JIT interface. -//===----------------------------------------------------------------------===// - -#include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h" -#include "llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h" - -#if LLVM_USE_INTEL_JITEVENTS -#include "IntelJITEventsWrapper.h" -#include "ittnotify.h" - -using namespace llvm; -using namespace llvm::orc; - -static std::unique_ptr Wrapper; - -static Error registerJITLoaderVTuneRegisterImpl(const VTuneMethodBatch &MB) { - const size_t StringsSize = MB.Strings.size(); - - for (size_t i = 0; i < MB.Methods.size(); ++i) { - VTuneMethodInfo MethodInfo = MB.Methods.at(i); - iJIT_Method_Load MethodMessage; - memset(&MethodMessage, 0, sizeof(iJIT_Method_Load)); - - MethodMessage.method_id = MethodInfo.MethodID; - if (MethodInfo.NameSI != 0 && MethodInfo.NameSI < StringsSize) { - MethodMessage.method_name = - const_cast(MB.Strings.at(MethodInfo.NameSI).data()); - } else { - MethodMessage.method_name = NULL; - } - if (MethodInfo.ClassFileSI != 0 && MethodInfo.ClassFileSI < StringsSize) { - MethodMessage.class_file_name = - const_cast(MB.Strings.at(MethodInfo.ClassFileSI).data()); - } else { - MethodMessage.class_file_name = NULL; - } - if (MethodInfo.SourceFileSI != 0 && MethodInfo.SourceFileSI < StringsSize) { - MethodMessage.source_file_name = - const_cast(MB.Strings.at(MethodInfo.SourceFileSI).data()); - } else { - MethodMessage.source_file_name = NULL; - } - - MethodMessage.method_load_address = MethodInfo.LoadAddr.toPtr(); - MethodMessage.method_size = MethodInfo.LoadSize; - MethodMessage.class_id = 0; - - MethodMessage.user_data = NULL; - MethodMessage.user_data_size = 0; - MethodMessage.env = iJDE_JittingAPI; - - std::vector LineInfo; - for (const auto &LInfo : MethodInfo.LineTable) { - LineInfo.push_back(LineNumberInfo{LInfo.first, LInfo.second}); - } - - if (LineInfo.size() == 0) { - MethodMessage.line_number_size = 0; - MethodMessage.line_number_table = 0; - } else { - MethodMessage.line_number_size = LineInfo.size(); - MethodMessage.line_number_table = &*LineInfo.begin(); - } - Wrapper->iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, - &MethodMessage); - } - - return Error::success(); -} - -static void registerJITLoaderVTuneUnregisterImpl( - const std::vector> &UM) { - for (auto &Method : UM) { - Wrapper->iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_UNLOAD_START, - const_cast(&Method.first)); - } -} - -extern "C" llvm::orc::shared::CWrapperFunctionResult -llvm_orc_registerVTuneImpl(const char *Data, uint64_t Size) { - using namespace orc::shared; - if (!Wrapper) - Wrapper.reset(new IntelJITEventsWrapper); - - return WrapperFunction::handle( - Data, Size, registerJITLoaderVTuneRegisterImpl) - .release(); -} - -extern "C" llvm::orc::shared::CWrapperFunctionResult -llvm_orc_unregisterVTuneImpl(const char *Data, uint64_t Size) { - using namespace orc::shared; - return WrapperFunction::handle( - Data, Size, registerJITLoaderVTuneUnregisterImpl) - .release(); -} - -// For Testing: following code comes from llvm-jitlistener.cpp in llvm tools -namespace { -typedef std::vector> SourceLocations; -typedef std::map NativeCodeMap; -NativeCodeMap ReportedDebugFuncs; -} // namespace - -static int NotifyEvent(iJIT_JVM_EVENT EventType, void *EventSpecificData) { - switch (EventType) { - case iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED: { - if (!EventSpecificData) { - errs() << "Error: The JIT event listener did not provide a event data."; - return -1; - } - iJIT_Method_Load *msg = static_cast(EventSpecificData); - - ReportedDebugFuncs[msg->method_id]; - - outs() << "Method load [" << msg->method_id << "]: " << msg->method_name - << ", Size = " << msg->method_size << "\n"; - - for (unsigned int i = 0; i < msg->line_number_size; ++i) { - if (!msg->line_number_table) { - errs() << "A function with a non-zero line count had no line table."; - return -1; - } - std::pair loc( - std::string(msg->source_file_name), - msg->line_number_table[i].LineNumber); - ReportedDebugFuncs[msg->method_id].push_back(loc); - outs() << " Line info @ " << msg->line_number_table[i].Offset << ": " - << msg->source_file_name << ", line " - << msg->line_number_table[i].LineNumber << "\n"; - } - outs() << "\n"; - } break; - case iJVM_EVENT_TYPE_METHOD_UNLOAD_START: { - if (!EventSpecificData) { - errs() << "Error: The JIT event listener did not provide a event data."; - return -1; - } - unsigned int UnloadId = - *reinterpret_cast(EventSpecificData); - assert(1 == ReportedDebugFuncs.erase(UnloadId)); - outs() << "Method unload [" << UnloadId << "]\n"; - } break; - default: - break; - } - return 0; -} - -static iJIT_IsProfilingActiveFlags IsProfilingActive(void) { - // for testing, pretend we have an Intel Parallel Amplifier XE 2011 - // instance attached - return iJIT_SAMPLING_ON; -} - -static unsigned int GetNewMethodID(void) { - static unsigned int id = 0; - return ++id; -} - -extern "C" llvm::orc::shared::CWrapperFunctionResult -llvm_orc_test_registerVTuneImpl(const char *Data, uint64_t Size) { - using namespace orc::shared; - Wrapper.reset(new IntelJITEventsWrapper( - NotifyEvent, NULL, NULL, IsProfilingActive, 0, 0, GetNewMethodID)); - return WrapperFunction::handle( - Data, Size, registerJITLoaderVTuneRegisterImpl) - .release(); -} - -#else - -using namespace llvm; -using namespace llvm::orc; - -static Error unsupportedBatch(const VTuneMethodBatch &MB) { - return llvm::make_error("unsupported for Intel VTune", - inconvertibleErrorCode()); -} - -static void unsuppported(const std::vector> &UM) { - -} - -extern "C" llvm::orc::shared::CWrapperFunctionResult -llvm_orc_registerVTuneImpl(const char *Data, uint64_t Size) { - using namespace orc::shared; - return WrapperFunction::handle( - Data, Size, unsupportedBatch) - .release(); -} - -extern "C" llvm::orc::shared::CWrapperFunctionResult -llvm_orc_unregisterVTuneImpl(const char *Data, uint64_t Size) { - using namespace orc::shared; - return WrapperFunction::handle(Data, Size, - unsuppported) - .release(); -} - -extern "C" llvm::orc::shared::CWrapperFunctionResult -llvm_orc_test_registerVTuneImpl(const char *Data, uint64_t Size) { - using namespace orc::shared; - return WrapperFunction::handle( - Data, Size, unsupportedBatch) - .release(); -} - -#endif diff --git a/llvm/test/ExecutionEngine/JITLink/x86-64/ELF_vtune.s b/llvm/test/ExecutionEngine/JITLink/x86-64/ELF_vtune.s deleted file mode 100644 index 0126b9ff3a61..000000000000 --- a/llvm/test/ExecutionEngine/JITLink/x86-64/ELF_vtune.s +++ /dev/null @@ -1,52 +0,0 @@ -# REQUIRES: native && x86_64-linux && intel-jitevents - -# RUN: rm -rf %t && mkdir -p %t -# RUN: llvm-mc -triple=x86_64-unknown-linux -position-independent \ -# RUN: -filetype=obj -o %t/ELF_x86-64_vtune.o %s -# RUN: llvm-jitlink -vtune-support %t/ELF_x86-64_vtune.o | \ -# RUN: FileCheck %s - -# CHECK: Method load [0]: {{.*}}, Size = {{[0-9]+}} -# CHECK: Method unload [0] - .file "test.c" - .text - .globl main - .type main, @function -main: -.LFB0: - .cfi_startproc - endbr64 - pushq %rbp - .cfi_def_cfa_offset 16 - .cfi_offset 6, -16 - movq %rsp, %rbp - .cfi_def_cfa_register 6 - movl %edi, -4(%rbp) - movq %rsi, -16(%rbp) - movl -4(%rbp), %ebx - addl $1, %ebx - movl $0, %eax - popq %rbp - .cfi_def_cfa 7, 8 - ret - .cfi_endproc -.LFE0: - .size main, .-main - .ident "GCC: (Ubuntu 9.4.0-1ubuntu1~20.04.2) 9.4.0" - .section .note.GNU-stack,"",@progbits - .section .note.gnu.property,"a" - .align 8 - .long 1f - 0f - .long 4f - 1f - .long 5 -0: - .string "GNU" -1: - .align 8 - .long 0xc0000002 - .long 3f - 2f -2: - .long 0x3 -3: - .align 8 -4: diff --git a/llvm/test/ExecutionEngine/JITLink/x86-64/lit.local.cfg b/llvm/test/ExecutionEngine/JITLink/x86-64/lit.local.cfg index d5a1ad626b65..42bf50dcc13c 100644 --- a/llvm/test/ExecutionEngine/JITLink/x86-64/lit.local.cfg +++ b/llvm/test/ExecutionEngine/JITLink/x86-64/lit.local.cfg @@ -1,5 +1,2 @@ if not "X86" in config.root.targets: config.unsupported = True - -if config.llvm_use_intel_jitevents: - config.available_features.add("intel-jitevents") diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp index 0217f645a0f6..f0b8310a32ef 100644 --- a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp +++ b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp @@ -21,7 +21,6 @@ #include "llvm/ExecutionEngine/Orc/Debugging/DebugInfoSupport.h" #include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.h" #include "llvm/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.h" -#include "llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h" #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h" #include "llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h" #include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h" @@ -35,7 +34,6 @@ #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderPerf.h" -#include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" @@ -150,10 +148,6 @@ static cl::opt PerfSupport("perf-support", cl::init(false), cl::Hidden, cl::cat(JITLinkCategory)); -static cl::opt VTuneSupport("vtune-support", - cl::desc("Enable vtune profiling support"), - cl::init(false), cl::Hidden, - cl::cat(JITLinkCategory)); static cl::opt NoProcessSymbols("no-process-syms", cl::desc("Do not resolve to llvm-jitlink process symbols"), @@ -270,10 +264,7 @@ static LLVM_ATTRIBUTE_USED void linkComponents() { << (void *)&llvm_orc_registerJITLoaderGDBAllocAction << '\n' << (void *)&llvm_orc_registerJITLoaderPerfStart << '\n' << (void *)&llvm_orc_registerJITLoaderPerfEnd << '\n' - << (void *)&llvm_orc_registerJITLoaderPerfImpl << '\n' - << (void *)&llvm_orc_registerVTuneImpl << '\n' - << (void *)&llvm_orc_unregisterVTuneImpl << '\n' - << (void *)&llvm_orc_test_registerVTuneImpl << '\n'; + << (void *)&llvm_orc_registerJITLoaderPerfImpl << '\n'; } static bool UseTestResultOverride = false; @@ -1013,13 +1004,6 @@ Session::Session(std::unique_ptr EPC, Error &Err) this->ES.getExecutorProcessControl(), *ProcessSymsJD, true, true))); } - if (VTuneSupport && TT.isOSBinFormatELF()) { - ObjLayer.addPlugin(ExitOnErr(DebugInfoPreservationPlugin::Create())); - ObjLayer.addPlugin(ExitOnErr(VTuneSupportPlugin::Create( - this->ES.getExecutorProcessControl(), *ProcessSymsJD, /*EmitDebugInfo=*/true, - /*TestMode=*/true))); - } - // Set up the platform. if (!OrcRuntime.empty()) { assert(ProcessSymsJD && "ProcessSymsJD should have been set"); -- GitLab From 564b81db8541468ce296d3bdcacab05b9581297c Mon Sep 17 00:00:00 2001 From: MalaySanghiIntel <148750629+MalaySanghiIntel@users.noreply.github.com> Date: Tue, 5 Mar 2024 07:30:14 +0530 Subject: [PATCH 083/929] Add support for x87 registers on GISel register selection (#83528) We handle 3 register classes for x87 - rfp32, rfp64 and rfp80. 1. X87 registers are assigned a pseudo register class. We need a new register bank for these classes. 2. We add bank information and enums for these. 3. Legalizer is updated to allow 32b and 64b even in absence of SSE. This is required because with SSE enabled, x86 doesn't use fp stack and instead uses SSE registers. 4. Functions in X86RegisterBankInfo need to decide whether to use the pseudo classes or SSE-enabled classes. I add MachineInstr as an argument to static helper function getPartialMappingIdx to this end. Add/Update tests. --- .../lib/Target/X86/GISel/X86LegalizerInfo.cpp | 13 +- .../Target/X86/GISel/X86RegisterBankInfo.cpp | 48 ++-- .../Target/X86/GISel/X86RegisterBankInfo.h | 3 +- .../lib/Target/X86/X86GenRegisterBankInfo.def | 17 +- llvm/lib/Target/X86/X86RegisterBanks.td | 3 + .../X86/GlobalISel/regbankselect-x87.ll | 237 ++++++++++++++++++ .../CodeGen/X86/GlobalISel/x86_64-fallback.ll | 2 +- 7 files changed, 298 insertions(+), 25 deletions(-) create mode 100644 llvm/test/CodeGen/X86/GlobalISel/regbankselect-x87.ll diff --git a/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp b/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp index 27381dff338e..2e33adaed7a8 100644 --- a/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp +++ b/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp @@ -39,6 +39,7 @@ X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI, bool HasVLX = Subtarget.hasVLX(); bool HasDQI = Subtarget.hasAVX512() && Subtarget.hasDQI(); bool HasBWI = Subtarget.hasAVX512() && Subtarget.hasBWI(); + bool UseX87 = !Subtarget.useSoftFloat() && Subtarget.hasX87(); const LLT p0 = LLT::pointer(0, TM.getPointerSizeInBits(0)); const LLT s1 = LLT::scalar(1); @@ -415,17 +416,19 @@ X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI, // fp constants getActionDefinitionsBuilder(G_FCONSTANT) .legalIf([=](const LegalityQuery &Query) -> bool { - return (HasSSE1 && typeInSet(0, {s32})(Query)) || - (HasSSE2 && typeInSet(0, {s64})(Query)); + return (typeInSet(0, {s32, s64})(Query)) || + (UseX87 && typeInSet(0, {s80})(Query)); }); // fp arithmetic getActionDefinitionsBuilder({G_FADD, G_FSUB, G_FMUL, G_FDIV}) .legalIf([=](const LegalityQuery &Query) { - return (HasSSE1 && typeInSet(0, {s32, v4s32})(Query)) || - (HasSSE2 && typeInSet(0, {s64, v2s64})(Query)) || + return (typeInSet(0, {s32, s64})(Query)) || + (HasSSE1 && typeInSet(0, {v4s32})(Query)) || + (HasSSE2 && typeInSet(0, {v2s64})(Query)) || (HasAVX && typeInSet(0, {v8s32, v4s64})(Query)) || - (HasAVX512 && typeInSet(0, {v16s32, v8s64})(Query)); + (HasAVX512 && typeInSet(0, {v16s32, v8s64})(Query)) || + (UseX87 && typeInSet(0, {s80})(Query)); }); // fp comparison diff --git a/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp b/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp index 72828f961f93..e7c9e60ba95f 100644 --- a/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp +++ b/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp @@ -12,6 +12,7 @@ #include "X86RegisterBankInfo.h" #include "X86InstrInfo.h" +#include "X86Subtarget.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/RegisterBank.h" #include "llvm/CodeGen/RegisterBankInfo.h" @@ -59,11 +60,24 @@ X86RegisterBankInfo::getRegBankFromRegClass(const TargetRegisterClass &RC, X86::VR512RegClass.hasSubClassEq(&RC)) return getRegBank(X86::VECRRegBankID); + if (X86::RFP80RegClass.hasSubClassEq(&RC) || + X86::RFP32RegClass.hasSubClassEq(&RC) || + X86::RFP64RegClass.hasSubClassEq(&RC)) + return getRegBank(X86::PSRRegBankID); + llvm_unreachable("Unsupported register kind yet."); } X86GenRegisterBankInfo::PartialMappingIdx -X86GenRegisterBankInfo::getPartialMappingIdx(const LLT &Ty, bool isFP) { +X86GenRegisterBankInfo::getPartialMappingIdx(const MachineInstr &MI, + const LLT &Ty, bool isFP) { + const MachineFunction *MF = MI.getMF(); + const X86Subtarget *ST = &MF->getSubtarget(); + bool HasSSE1 = ST->hasSSE1(); + bool HasSSE2 = ST->hasSSE2(); + // 80 bits is only generated for X87 floating points. + if (Ty.getSizeInBits() == 80) + isFP = true; if ((Ty.isScalar() && !isFP) || Ty.isPointer()) { switch (Ty.getSizeInBits()) { case 1: @@ -84,11 +98,13 @@ X86GenRegisterBankInfo::getPartialMappingIdx(const LLT &Ty, bool isFP) { } else if (Ty.isScalar()) { switch (Ty.getSizeInBits()) { case 32: - return PMI_FP32; + return HasSSE1 ? PMI_FP32 : PMI_PSR32; case 64: - return PMI_FP64; + return HasSSE2 ? PMI_FP64 : PMI_PSR64; case 128: return PMI_VEC128; + case 80: + return PMI_PSR80; default: llvm_unreachable("Unsupported register size."); } @@ -118,7 +134,8 @@ void X86RegisterBankInfo::getInstrPartialMappingIdxs( if (!MO.isReg() || !MO.getReg()) OpRegBankIdx[Idx] = PMI_None; else - OpRegBankIdx[Idx] = getPartialMappingIdx(MRI.getType(MO.getReg()), isFP); + OpRegBankIdx[Idx] = + getPartialMappingIdx(MI, MRI.getType(MO.getReg()), isFP); } } @@ -156,7 +173,7 @@ X86RegisterBankInfo::getSameOperandsMapping(const MachineInstr &MI, (Ty != MRI.getType(MI.getOperand(2).getReg()))) llvm_unreachable("Unsupported operand mapping yet."); - auto Mapping = getValueMapping(getPartialMappingIdx(Ty, isFP), 3); + auto Mapping = getValueMapping(getPartialMappingIdx(MI, Ty, isFP), 3); return getInstructionMapping(DefaultMappingID, 1, Mapping, NumOperands); } @@ -190,9 +207,8 @@ X86RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { unsigned NumOperands = MI.getNumOperands(); LLT Ty = MRI.getType(MI.getOperand(0).getReg()); - auto Mapping = getValueMapping(getPartialMappingIdx(Ty, false), 3); + auto Mapping = getValueMapping(getPartialMappingIdx(MI, Ty, false), 3); return getInstructionMapping(DefaultMappingID, 1, Mapping, NumOperands); - } default: break; @@ -206,7 +222,7 @@ X86RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { case TargetOpcode::G_FPTRUNC: case TargetOpcode::G_FCONSTANT: // Instruction having only floating-point operands (all scalars in VECRReg) - getInstrPartialMappingIdxs(MI, MRI, /* isFP */ true, OpRegBankIdx); + getInstrPartialMappingIdxs(MI, MRI, /* isFP= */ true, OpRegBankIdx); break; case TargetOpcode::G_SITOFP: case TargetOpcode::G_FPTOSI: { @@ -219,8 +235,8 @@ X86RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { bool FirstArgIsFP = Opc == TargetOpcode::G_SITOFP; bool SecondArgIsFP = Opc == TargetOpcode::G_FPTOSI; - OpRegBankIdx[0] = getPartialMappingIdx(Ty0, /* isFP */ FirstArgIsFP); - OpRegBankIdx[1] = getPartialMappingIdx(Ty1, /* isFP */ SecondArgIsFP); + OpRegBankIdx[0] = getPartialMappingIdx(MI, Ty0, /* isFP= */ FirstArgIsFP); + OpRegBankIdx[1] = getPartialMappingIdx(MI, Ty1, /* isFP= */ SecondArgIsFP); break; } case TargetOpcode::G_FCMP: { @@ -234,7 +250,7 @@ X86RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { (void)Size; assert((Size == 32 || Size == 64) && "Unsupported size for G_FCMP"); - auto FpRegBank = getPartialMappingIdx(Ty1, /* isFP */ true); + auto FpRegBank = getPartialMappingIdx(MI, Ty1, /* isFP= */ true); OpRegBankIdx = {PMI_GPR8, /* Predicate */ PMI_None, FpRegBank, FpRegBank}; break; @@ -253,12 +269,12 @@ X86RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { (Ty1.getSizeInBits() == 32 || Ty1.getSizeInBits() == 64) && Opc == TargetOpcode::G_ANYEXT; - getInstrPartialMappingIdxs(MI, MRI, /* isFP */ isFPTrunc || isFPAnyExt, + getInstrPartialMappingIdxs(MI, MRI, /* isFP= */ isFPTrunc || isFPAnyExt, OpRegBankIdx); } break; default: // Track the bank of each register, use NotFP mapping (all scalars in GPRs) - getInstrPartialMappingIdxs(MI, MRI, /* isFP */ false, OpRegBankIdx); + getInstrPartialMappingIdxs(MI, MRI, /* isFP= */ false, OpRegBankIdx); break; } @@ -288,16 +304,16 @@ X86RegisterBankInfo::getInstrAlternativeMappings(const MachineInstr &MI) const { case TargetOpcode::G_LOAD: case TargetOpcode::G_STORE: case TargetOpcode::G_IMPLICIT_DEF: { - // we going to try to map 32/64 bit to PMI_FP32/PMI_FP64 + // we going to try to map 32/64/80 bit to PMI_FP32/PMI_FP64/PMI_FP80 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI); - if (Size != 32 && Size != 64) + if (Size != 32 && Size != 64 && Size != 80) break; unsigned NumOperands = MI.getNumOperands(); // Track the bank of each register, use FP mapping (all scalars in VEC) SmallVector OpRegBankIdx(NumOperands); - getInstrPartialMappingIdxs(MI, MRI, /* isFP */ true, OpRegBankIdx); + getInstrPartialMappingIdxs(MI, MRI, /* isFP= */ true, OpRegBankIdx); // Finally construct the computed mapping. SmallVector OpdsMapping(NumOperands); diff --git a/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.h b/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.h index 9a4e23d8b34d..989c5956ad59 100644 --- a/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.h +++ b/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.h @@ -32,7 +32,8 @@ protected: static RegisterBankInfo::PartialMapping PartMappings[]; static RegisterBankInfo::ValueMapping ValMappings[]; - static PartialMappingIdx getPartialMappingIdx(const LLT &Ty, bool isFP); + static PartialMappingIdx getPartialMappingIdx(const MachineInstr &MI, + const LLT &Ty, bool isFP); static const RegisterBankInfo::ValueMapping * getValueMapping(PartialMappingIdx Idx, unsigned NumOperands); }; diff --git a/llvm/lib/Target/X86/X86GenRegisterBankInfo.def b/llvm/lib/Target/X86/X86GenRegisterBankInfo.def index 0fdea9071c29..11a28806710d 100644 --- a/llvm/lib/Target/X86/X86GenRegisterBankInfo.def +++ b/llvm/lib/Target/X86/X86GenRegisterBankInfo.def @@ -25,6 +25,10 @@ RegisterBankInfo::PartialMapping X86GenRegisterBankInfo::PartMappings[]{ {0, 128, X86::VECRRegBank}, // :6 {0, 256, X86::VECRRegBank}, // :7 {0, 512, X86::VECRRegBank}, // :8 + // RFP32/64/80 + {0, 32, X86::PSRRegBank}, // :9 + {0, 64, X86::PSRRegBank}, // :10 + {0, 80, X86::PSRRegBank}, // :11 }; #endif // GET_TARGET_REGBANK_INFO_IMPL @@ -39,7 +43,10 @@ enum PartialMappingIdx { PMI_FP64, PMI_VEC128, PMI_VEC256, - PMI_VEC512 + PMI_VEC512, + PMI_PSR32, + PMI_PSR64, + PMI_PSR80 }; #endif // GET_TARGET_REGBANK_INFO_CLASS @@ -61,6 +68,9 @@ RegisterBankInfo::ValueMapping X86GenRegisterBankInfo::ValMappings[]{ INSTR_3OP(BREAKDOWN(PMI_VEC128, 1)) // 18: Vec128 INSTR_3OP(BREAKDOWN(PMI_VEC256, 1)) // 21: Vec256 INSTR_3OP(BREAKDOWN(PMI_VEC512, 1)) // 24: Vec512 + INSTR_3OP(BREAKDOWN(PMI_PSR32, 1)) // 25: Rfp32 + INSTR_3OP(BREAKDOWN(PMI_PSR64, 1)) // 26: Rfp64 + INSTR_3OP(BREAKDOWN(PMI_PSR80, 1)) // 27: Rfp80 }; #undef INSTR_3OP #undef BREAKDOWN @@ -78,6 +88,9 @@ enum ValueMappingIdx { VMI_3OpsVec128Idx = PMI_VEC128 * 3, VMI_3OpsVec256Idx = PMI_VEC256 * 3, VMI_3OpsVec512Idx = PMI_VEC512 * 3, + VMI_3OpsPs32Idx = PMI_PSR32 * 3, + VMI_3OpsPs64Idx = PMI_PSR64 * 3, + VMI_3OpsPs80Idx = PMI_PSR80 * 3, }; #undef GET_TARGET_REGBANK_INFO_CLASS #endif // GET_TARGET_REGBANK_INFO_CLASS @@ -89,7 +102,7 @@ X86GenRegisterBankInfo::getValueMapping(PartialMappingIdx Idx, unsigned NumOperands) { // We can use VMI_3Ops Mapping for all the cases. - if (NumOperands <= 3 && (Idx >= PMI_GPR8 && Idx <= PMI_VEC512)) + if (NumOperands <= 3 && (Idx >= PMI_GPR8 && Idx <= PMI_PSR80)) return &ValMappings[(unsigned)Idx * 3]; llvm_unreachable("Unsupported PartialMappingIdx."); diff --git a/llvm/lib/Target/X86/X86RegisterBanks.td b/llvm/lib/Target/X86/X86RegisterBanks.td index 91a497252595..05aa880bc6cb 100644 --- a/llvm/lib/Target/X86/X86RegisterBanks.td +++ b/llvm/lib/Target/X86/X86RegisterBanks.td @@ -14,3 +14,6 @@ def GPRRegBank : RegisterBank<"GPR", [GR64]>; /// Floating Point/Vector Registers def VECRRegBank : RegisterBank<"VECR", [VR512]>; + +/// Pseudo Registers: RFP80 +def PSRRegBank : RegisterBank<"PSR", [RFP32, RFP64, RFP80]>; diff --git a/llvm/test/CodeGen/X86/GlobalISel/regbankselect-x87.ll b/llvm/test/CodeGen/X86/GlobalISel/regbankselect-x87.ll new file mode 100644 index 000000000000..d09db0f2474c --- /dev/null +++ b/llvm/test/CodeGen/X86/GlobalISel/regbankselect-x87.ll @@ -0,0 +1,237 @@ +; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4 +; RUN: llc < %s -mtriple=i686-- -mattr=+x87,-sse,-sse2 -global-isel -stop-after=regbankselect | FileCheck %s --check-prefix=X86 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+x87,-sse,-sse2 -global-isel -stop-after=regbankselect | FileCheck %s --check-prefix=X64 + +define x86_fp80 @f0(x86_fp80 noundef %a) { + ; X86-LABEL: name: f0 + ; X86: bb.1.entry: + ; X86-NEXT: [[FRAME_INDEX:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.0 + ; X86-NEXT: [[LOAD:%[0-9]+]]:psr(s80) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (s80) from %fixed-stack.0, align 4) + ; X86-NEXT: [[C:%[0-9]+]]:psr(s80) = G_FCONSTANT x86_fp80 0xK400A8000000000000000 + ; X86-NEXT: [[FRAME_INDEX1:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %stack.0.a.addr + ; X86-NEXT: [[FRAME_INDEX2:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %stack.1.x + ; X86-NEXT: G_STORE [[LOAD]](s80), [[FRAME_INDEX1]](p0) :: (store (s80) into %ir.a.addr, align 16) + ; X86-NEXT: G_STORE [[C]](s80), [[FRAME_INDEX2]](p0) :: (store (s80) into %ir.x, align 16) + ; X86-NEXT: [[LOAD1:%[0-9]+]]:psr(s80) = G_LOAD [[FRAME_INDEX1]](p0) :: (dereferenceable load (s80) from %ir.a.addr, align 16) + ; X86-NEXT: [[LOAD2:%[0-9]+]]:psr(s80) = G_LOAD [[FRAME_INDEX2]](p0) :: (dereferenceable load (s80) from %ir.x, align 16) + ; X86-NEXT: [[FADD:%[0-9]+]]:psr(s80) = G_FADD [[LOAD1]], [[LOAD2]] + ; X86-NEXT: $fp0 = COPY [[FADD]](s80) + ; X86-NEXT: RET 0, implicit $fp0 + ; + ; X64-LABEL: name: f0 + ; X64: bb.1.entry: + ; X64-NEXT: [[FRAME_INDEX:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.0 + ; X64-NEXT: [[LOAD:%[0-9]+]]:psr(s80) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (s80) from %fixed-stack.0, align 16) + ; X64-NEXT: [[C:%[0-9]+]]:psr(s80) = G_FCONSTANT x86_fp80 0xK400A8000000000000000 + ; X64-NEXT: [[FRAME_INDEX1:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %stack.0.a.addr + ; X64-NEXT: [[FRAME_INDEX2:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %stack.1.x + ; X64-NEXT: G_STORE [[LOAD]](s80), [[FRAME_INDEX1]](p0) :: (store (s80) into %ir.a.addr, align 16) + ; X64-NEXT: G_STORE [[C]](s80), [[FRAME_INDEX2]](p0) :: (store (s80) into %ir.x, align 16) + ; X64-NEXT: [[LOAD1:%[0-9]+]]:psr(s80) = G_LOAD [[FRAME_INDEX1]](p0) :: (dereferenceable load (s80) from %ir.a.addr, align 16) + ; X64-NEXT: [[LOAD2:%[0-9]+]]:psr(s80) = G_LOAD [[FRAME_INDEX2]](p0) :: (dereferenceable load (s80) from %ir.x, align 16) + ; X64-NEXT: [[FADD:%[0-9]+]]:psr(s80) = G_FADD [[LOAD1]], [[LOAD2]] + ; X64-NEXT: $fp0 = COPY [[FADD]](s80) + ; X64-NEXT: RET 0, implicit $fp0 +entry: + %a.addr = alloca x86_fp80, align 16 + %x = alloca x86_fp80, align 16 + store x86_fp80 %a, ptr %a.addr, align 16 + store x86_fp80 0xK400A8000000000000000, ptr %x, align 16 + %load1 = load x86_fp80, ptr %a.addr, align 16 + %load2 = load x86_fp80, ptr %x, align 16 + %add = fadd x86_fp80 %load1, %load2 + ret x86_fp80 %add +} + +declare x86_fp80 @llvm.sqrt.f32(x86_fp80) + +define void @f1(ptr %a, ptr %b) { + ; X86-LABEL: name: f1 + ; X86: bb.1 (%ir-block.0): + ; X86-NEXT: [[FRAME_INDEX:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.1 + ; X86-NEXT: [[LOAD:%[0-9]+]]:gpr(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (p0) from %fixed-stack.1) + ; X86-NEXT: [[FRAME_INDEX1:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.0 + ; X86-NEXT: [[LOAD1:%[0-9]+]]:gpr(p0) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load (p0) from %fixed-stack.0) + ; X86-NEXT: [[LOAD2:%[0-9]+]]:psr(s80) = G_LOAD [[LOAD]](p0) :: (load (s80) from %ir.a, align 4) + ; X86-NEXT: [[LOAD3:%[0-9]+]]:psr(s80) = G_LOAD [[LOAD1]](p0) :: (load (s80) from %ir.b, align 4) + ; X86-NEXT: [[FSUB:%[0-9]+]]:psr(s80) = G_FSUB [[LOAD2]], [[LOAD3]] + ; X86-NEXT: G_STORE [[FSUB]](s80), [[LOAD]](p0) :: (store (s80) into %ir.a, align 4) + ; X86-NEXT: RET 0 + ; + ; X64-LABEL: name: f1 + ; X64: bb.1 (%ir-block.0): + ; X64-NEXT: liveins: $rdi, $rsi + ; X64-NEXT: {{ $}} + ; X64-NEXT: [[COPY:%[0-9]+]]:gpr(p0) = COPY $rdi + ; X64-NEXT: [[COPY1:%[0-9]+]]:gpr(p0) = COPY $rsi + ; X64-NEXT: [[LOAD:%[0-9]+]]:psr(s80) = G_LOAD [[COPY]](p0) :: (load (s80) from %ir.a, align 4) + ; X64-NEXT: [[LOAD1:%[0-9]+]]:psr(s80) = G_LOAD [[COPY1]](p0) :: (load (s80) from %ir.b, align 4) + ; X64-NEXT: [[FSUB:%[0-9]+]]:psr(s80) = G_FSUB [[LOAD]], [[LOAD1]] + ; X64-NEXT: G_STORE [[FSUB]](s80), [[COPY]](p0) :: (store (s80) into %ir.a, align 4) + ; X64-NEXT: RET 0 + %load1 = load x86_fp80, ptr %a, align 4 + %load2 = load x86_fp80, ptr %b, align 4 + %sub = fsub x86_fp80 %load1, %load2 + store x86_fp80 %sub, ptr %a, align 4 + ret void +} + +define void @f2(ptr %a, ptr %b) { + ; X86-LABEL: name: f2 + ; X86: bb.1 (%ir-block.0): + ; X86-NEXT: [[FRAME_INDEX:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.1 + ; X86-NEXT: [[LOAD:%[0-9]+]]:gpr(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (p0) from %fixed-stack.1) + ; X86-NEXT: [[FRAME_INDEX1:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.0 + ; X86-NEXT: [[LOAD1:%[0-9]+]]:gpr(p0) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load (p0) from %fixed-stack.0) + ; X86-NEXT: [[LOAD2:%[0-9]+]]:psr(s80) = G_LOAD [[LOAD]](p0) :: (load (s80) from %ir.a, align 16) + ; X86-NEXT: [[LOAD3:%[0-9]+]]:psr(s80) = G_LOAD [[LOAD1]](p0) :: (load (s80) from %ir.b, align 16) + ; X86-NEXT: [[FMUL:%[0-9]+]]:psr(s80) = G_FMUL [[LOAD2]], [[LOAD3]] + ; X86-NEXT: G_STORE [[FMUL]](s80), [[LOAD]](p0) :: (store (s80) into %ir.a, align 16) + ; X86-NEXT: RET 0 + ; + ; X64-LABEL: name: f2 + ; X64: bb.1 (%ir-block.0): + ; X64-NEXT: liveins: $rdi, $rsi + ; X64-NEXT: {{ $}} + ; X64-NEXT: [[COPY:%[0-9]+]]:gpr(p0) = COPY $rdi + ; X64-NEXT: [[COPY1:%[0-9]+]]:gpr(p0) = COPY $rsi + ; X64-NEXT: [[LOAD:%[0-9]+]]:psr(s80) = G_LOAD [[COPY]](p0) :: (load (s80) from %ir.a, align 16) + ; X64-NEXT: [[LOAD1:%[0-9]+]]:psr(s80) = G_LOAD [[COPY1]](p0) :: (load (s80) from %ir.b, align 16) + ; X64-NEXT: [[FMUL:%[0-9]+]]:psr(s80) = G_FMUL [[LOAD]], [[LOAD1]] + ; X64-NEXT: G_STORE [[FMUL]](s80), [[COPY]](p0) :: (store (s80) into %ir.a, align 16) + ; X64-NEXT: RET 0 + %load1 = load x86_fp80, ptr %a, align 16 + %load2 = load x86_fp80, ptr %b, align 16 + %mul = fmul x86_fp80 %load1, %load2 + store x86_fp80 %mul, ptr %a, align 16 + ret void +} + +define void @f3(ptr %a, ptr %b) { + ; X86-LABEL: name: f3 + ; X86: bb.1 (%ir-block.0): + ; X86-NEXT: [[FRAME_INDEX:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.1 + ; X86-NEXT: [[LOAD:%[0-9]+]]:gpr(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (p0) from %fixed-stack.1) + ; X86-NEXT: [[FRAME_INDEX1:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.0 + ; X86-NEXT: [[LOAD1:%[0-9]+]]:gpr(p0) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load (p0) from %fixed-stack.0) + ; X86-NEXT: [[LOAD2:%[0-9]+]]:psr(s80) = G_LOAD [[LOAD]](p0) :: (load (s80) from %ir.a, align 4) + ; X86-NEXT: [[LOAD3:%[0-9]+]]:psr(s80) = G_LOAD [[LOAD1]](p0) :: (load (s80) from %ir.b, align 4) + ; X86-NEXT: [[FDIV:%[0-9]+]]:psr(s80) = G_FDIV [[LOAD2]], [[LOAD3]] + ; X86-NEXT: G_STORE [[FDIV]](s80), [[LOAD]](p0) :: (store (s80) into %ir.a, align 4) + ; X86-NEXT: RET 0 + ; + ; X64-LABEL: name: f3 + ; X64: bb.1 (%ir-block.0): + ; X64-NEXT: liveins: $rdi, $rsi + ; X64-NEXT: {{ $}} + ; X64-NEXT: [[COPY:%[0-9]+]]:gpr(p0) = COPY $rdi + ; X64-NEXT: [[COPY1:%[0-9]+]]:gpr(p0) = COPY $rsi + ; X64-NEXT: [[LOAD:%[0-9]+]]:psr(s80) = G_LOAD [[COPY]](p0) :: (load (s80) from %ir.a, align 4) + ; X64-NEXT: [[LOAD1:%[0-9]+]]:psr(s80) = G_LOAD [[COPY1]](p0) :: (load (s80) from %ir.b, align 4) + ; X64-NEXT: [[FDIV:%[0-9]+]]:psr(s80) = G_FDIV [[LOAD]], [[LOAD1]] + ; X64-NEXT: G_STORE [[FDIV]](s80), [[COPY]](p0) :: (store (s80) into %ir.a, align 4) + ; X64-NEXT: RET 0 + %load1 = load x86_fp80, ptr %a, align 4 + %load2 = load x86_fp80, ptr %b, align 4 + %div = fdiv x86_fp80 %load1, %load2 + store x86_fp80 %div, ptr %a, align 4 + ret void +} + +define float @f4(float %val) { + ; X86-LABEL: name: f4 + ; X86: bb.1 (%ir-block.0): + ; X86-NEXT: [[FRAME_INDEX:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.0 + ; X86-NEXT: [[LOAD:%[0-9]+]]:gpr(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (s32) from %fixed-stack.0) + ; X86-NEXT: $fp0 = COPY [[LOAD]](s32) + ; X86-NEXT: RET 0, implicit $fp0 + ; + ; X64-LABEL: name: f4 + ; X64: bb.1 (%ir-block.0): + ; X64-NEXT: [[FRAME_INDEX:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.0 + ; X64-NEXT: [[LOAD:%[0-9]+]]:gpr(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (s32) from %fixed-stack.0, align 16) + ; X64-NEXT: $xmm0 = COPY [[LOAD]](s32) + ; X64-NEXT: RET 0, implicit $xmm0 + ret float %val +} + +define void @f5(ptr %a, ptr %b) { + ; X86-LABEL: name: f5 + ; X86: bb.1 (%ir-block.0): + ; X86-NEXT: [[FRAME_INDEX:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.1 + ; X86-NEXT: [[LOAD:%[0-9]+]]:gpr(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (p0) from %fixed-stack.1) + ; X86-NEXT: [[FRAME_INDEX1:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.0 + ; X86-NEXT: [[LOAD1:%[0-9]+]]:gpr(p0) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load (p0) from %fixed-stack.0) + ; X86-NEXT: [[LOAD2:%[0-9]+]]:gpr(s32) = G_LOAD [[LOAD]](p0) :: (load (s32) from %ir.a, align 8) + ; X86-NEXT: [[C:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 4 + ; X86-NEXT: [[PTR_ADD:%[0-9]+]]:gpr(p0) = G_PTR_ADD [[LOAD]], [[C]](s32) + ; X86-NEXT: [[COPY:%[0-9]+]]:gpr(p0) = COPY [[PTR_ADD]](p0) + ; X86-NEXT: [[LOAD3:%[0-9]+]]:gpr(s32) = G_LOAD [[COPY]](p0) :: (load (s32) from %ir.a + 4, basealign 8) + ; X86-NEXT: [[MV:%[0-9]+]]:gpr(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32) + ; X86-NEXT: [[LOAD4:%[0-9]+]]:gpr(s32) = G_LOAD [[LOAD1]](p0) :: (load (s32) from %ir.b, align 8) + ; X86-NEXT: [[PTR_ADD1:%[0-9]+]]:gpr(p0) = G_PTR_ADD [[LOAD1]], [[C]](s32) + ; X86-NEXT: [[LOAD5:%[0-9]+]]:gpr(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s32) from %ir.b + 4, basealign 8) + ; X86-NEXT: [[MV1:%[0-9]+]]:gpr(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32) + ; X86-NEXT: [[COPY1:%[0-9]+]]:psr(s64) = COPY [[MV]](s64) + ; X86-NEXT: [[COPY2:%[0-9]+]]:psr(s64) = COPY [[MV1]](s64) + ; X86-NEXT: [[FADD:%[0-9]+]]:psr(s64) = G_FADD [[COPY1]], [[COPY2]] + ; X86-NEXT: [[COPY3:%[0-9]+]]:gpr(s64) = COPY [[FADD]](s64) + ; X86-NEXT: [[UV:%[0-9]+]]:gpr(s32), [[UV1:%[0-9]+]]:gpr(s32) = G_UNMERGE_VALUES [[COPY3]](s64) + ; X86-NEXT: G_STORE [[UV]](s32), [[LOAD]](p0) :: (store (s32) into %ir.a, align 8) + ; X86-NEXT: G_STORE [[UV1]](s32), [[PTR_ADD]](p0) :: (store (s32) into %ir.a + 4, basealign 8) + ; X86-NEXT: RET 0 + ; + ; X64-LABEL: name: f5 + ; X64: bb.1 (%ir-block.0): + ; X64-NEXT: liveins: $rdi, $rsi + ; X64-NEXT: {{ $}} + ; X64-NEXT: [[COPY:%[0-9]+]]:gpr(p0) = COPY $rdi + ; X64-NEXT: [[COPY1:%[0-9]+]]:gpr(p0) = COPY $rsi + ; X64-NEXT: [[LOAD:%[0-9]+]]:gpr(s64) = G_LOAD [[COPY]](p0) :: (load (s64) from %ir.a) + ; X64-NEXT: [[LOAD1:%[0-9]+]]:gpr(s64) = G_LOAD [[COPY1]](p0) :: (load (s64) from %ir.b) + ; X64-NEXT: [[COPY2:%[0-9]+]]:psr(s64) = COPY [[LOAD]](s64) + ; X64-NEXT: [[COPY3:%[0-9]+]]:psr(s64) = COPY [[LOAD1]](s64) + ; X64-NEXT: [[FADD:%[0-9]+]]:psr(s64) = G_FADD [[COPY2]], [[COPY3]] + ; X64-NEXT: [[COPY4:%[0-9]+]]:gpr(s64) = COPY [[FADD]](s64) + ; X64-NEXT: G_STORE [[COPY4]](s64), [[COPY]](p0) :: (store (s64) into %ir.a) + ; X64-NEXT: RET 0 + %load1 = load double, ptr %a, align 8 + %load2 = load double, ptr %b, align 8 + %add = fadd double %load1, %load2 + store double %add, ptr %a, align 8 + ret void +} + +define void @f6(ptr %0, ptr %1) { + ; X86-LABEL: name: f6 + ; X86: bb.1 (%ir-block.2): + ; X86-NEXT: [[FRAME_INDEX:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.1 + ; X86-NEXT: [[LOAD:%[0-9]+]]:gpr(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (p0) from %fixed-stack.1) + ; X86-NEXT: [[FRAME_INDEX1:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.0 + ; X86-NEXT: [[LOAD1:%[0-9]+]]:gpr(p0) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load (p0) from %fixed-stack.0) + ; X86-NEXT: [[C:%[0-9]+]]:psr(s32) = G_FCONSTANT float 2.000000e+01 + ; X86-NEXT: [[LOAD2:%[0-9]+]]:gpr(s32) = G_LOAD [[LOAD]](p0) :: (load (s32) from %ir.0) + ; X86-NEXT: [[COPY:%[0-9]+]]:psr(s32) = COPY [[LOAD2]](s32) + ; X86-NEXT: [[FADD:%[0-9]+]]:psr(s32) = G_FADD [[COPY]], [[C]] + ; X86-NEXT: [[COPY1:%[0-9]+]]:gpr(s32) = COPY [[FADD]](s32) + ; X86-NEXT: G_STORE [[COPY1]](s32), [[LOAD1]](p0) :: (store (s32) into %ir.1) + ; X86-NEXT: RET 0 + ; + ; X64-LABEL: name: f6 + ; X64: bb.1 (%ir-block.2): + ; X64-NEXT: liveins: $rdi, $rsi + ; X64-NEXT: {{ $}} + ; X64-NEXT: [[COPY:%[0-9]+]]:gpr(p0) = COPY $rdi + ; X64-NEXT: [[COPY1:%[0-9]+]]:gpr(p0) = COPY $rsi + ; X64-NEXT: [[C:%[0-9]+]]:psr(s32) = G_FCONSTANT float 2.000000e+01 + ; X64-NEXT: [[LOAD:%[0-9]+]]:gpr(s32) = G_LOAD [[COPY]](p0) :: (load (s32) from %ir.0) + ; X64-NEXT: [[COPY2:%[0-9]+]]:psr(s32) = COPY [[LOAD]](s32) + ; X64-NEXT: [[FADD:%[0-9]+]]:psr(s32) = G_FADD [[COPY2]], [[C]] + ; X64-NEXT: [[COPY3:%[0-9]+]]:gpr(s32) = COPY [[FADD]](s32) + ; X64-NEXT: G_STORE [[COPY3]](s32), [[COPY1]](p0) :: (store (s32) into %ir.1) + ; X64-NEXT: RET 0 + %load1 = load float, ptr %0 + %add = fadd float %load1, 20.0 + store float %add, ptr %1 + ret void +} diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86_64-fallback.ll b/llvm/test/CodeGen/X86/GlobalISel/x86_64-fallback.ll index 28b009fa6db7..39302734dde7 100644 --- a/llvm/test/CodeGen/X86/GlobalISel/x86_64-fallback.ll +++ b/llvm/test/CodeGen/X86/GlobalISel/x86_64-fallback.ll @@ -8,7 +8,7 @@ ; the fallback path. ; Check that we fallback on invoke translation failures. -; FALLBACK-WITH-REPORT-ERR: remark: :0:0: unable to legalize instruction: %1:_(s80) = G_FCONSTANT x86_fp80 0xK4002A000000000000000 +; FALLBACK-WITH-REPORT-ERR: remark: :0:0: cannot select: G_STORE %1:psr(s80), %0:gpr(p0) :: (store (s80) into %ir.ptr, align 16) (in function: test_x86_fp80_dump) ; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for test_x86_fp80_dump ; FALLBACK-WITH-REPORT-OUT-LABEL: test_x86_fp80_dump: define void @test_x86_fp80_dump(ptr %ptr){ -- GitLab From 377feaea8c6b2b55adfc56210159986eb5e37c46 Mon Sep 17 00:00:00 2001 From: Shengchen Kan Date: Tue, 5 Mar 2024 10:26:48 +0800 Subject: [PATCH 084/929] [X86][NFC] Clang-format X86DisassemblerDecoder.h This is to extract NFC in #83863 into a separate commit. --- .../X86/Disassembler/X86DisassemblerDecoder.h | 431 +++++++++--------- 1 file changed, 217 insertions(+), 214 deletions(-) diff --git a/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h b/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h index 4c7b1c094522..9cae0f02926f 100644 --- a/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h +++ b/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h @@ -348,213 +348,213 @@ namespace X86Disassembler { ENTRY(R30) \ ENTRY(R31) -#define REGS_MMX \ - ENTRY(MM0) \ - ENTRY(MM1) \ - ENTRY(MM2) \ - ENTRY(MM3) \ - ENTRY(MM4) \ - ENTRY(MM5) \ - ENTRY(MM6) \ +#define REGS_MMX \ + ENTRY(MM0) \ + ENTRY(MM1) \ + ENTRY(MM2) \ + ENTRY(MM3) \ + ENTRY(MM4) \ + ENTRY(MM5) \ + ENTRY(MM6) \ ENTRY(MM7) -#define REGS_XMM \ - ENTRY(XMM0) \ - ENTRY(XMM1) \ - ENTRY(XMM2) \ - ENTRY(XMM3) \ - ENTRY(XMM4) \ - ENTRY(XMM5) \ - ENTRY(XMM6) \ - ENTRY(XMM7) \ - ENTRY(XMM8) \ - ENTRY(XMM9) \ - ENTRY(XMM10) \ - ENTRY(XMM11) \ - ENTRY(XMM12) \ - ENTRY(XMM13) \ - ENTRY(XMM14) \ - ENTRY(XMM15) \ - ENTRY(XMM16) \ - ENTRY(XMM17) \ - ENTRY(XMM18) \ - ENTRY(XMM19) \ - ENTRY(XMM20) \ - ENTRY(XMM21) \ - ENTRY(XMM22) \ - ENTRY(XMM23) \ - ENTRY(XMM24) \ - ENTRY(XMM25) \ - ENTRY(XMM26) \ - ENTRY(XMM27) \ - ENTRY(XMM28) \ - ENTRY(XMM29) \ - ENTRY(XMM30) \ +#define REGS_XMM \ + ENTRY(XMM0) \ + ENTRY(XMM1) \ + ENTRY(XMM2) \ + ENTRY(XMM3) \ + ENTRY(XMM4) \ + ENTRY(XMM5) \ + ENTRY(XMM6) \ + ENTRY(XMM7) \ + ENTRY(XMM8) \ + ENTRY(XMM9) \ + ENTRY(XMM10) \ + ENTRY(XMM11) \ + ENTRY(XMM12) \ + ENTRY(XMM13) \ + ENTRY(XMM14) \ + ENTRY(XMM15) \ + ENTRY(XMM16) \ + ENTRY(XMM17) \ + ENTRY(XMM18) \ + ENTRY(XMM19) \ + ENTRY(XMM20) \ + ENTRY(XMM21) \ + ENTRY(XMM22) \ + ENTRY(XMM23) \ + ENTRY(XMM24) \ + ENTRY(XMM25) \ + ENTRY(XMM26) \ + ENTRY(XMM27) \ + ENTRY(XMM28) \ + ENTRY(XMM29) \ + ENTRY(XMM30) \ ENTRY(XMM31) -#define REGS_YMM \ - ENTRY(YMM0) \ - ENTRY(YMM1) \ - ENTRY(YMM2) \ - ENTRY(YMM3) \ - ENTRY(YMM4) \ - ENTRY(YMM5) \ - ENTRY(YMM6) \ - ENTRY(YMM7) \ - ENTRY(YMM8) \ - ENTRY(YMM9) \ - ENTRY(YMM10) \ - ENTRY(YMM11) \ - ENTRY(YMM12) \ - ENTRY(YMM13) \ - ENTRY(YMM14) \ - ENTRY(YMM15) \ - ENTRY(YMM16) \ - ENTRY(YMM17) \ - ENTRY(YMM18) \ - ENTRY(YMM19) \ - ENTRY(YMM20) \ - ENTRY(YMM21) \ - ENTRY(YMM22) \ - ENTRY(YMM23) \ - ENTRY(YMM24) \ - ENTRY(YMM25) \ - ENTRY(YMM26) \ - ENTRY(YMM27) \ - ENTRY(YMM28) \ - ENTRY(YMM29) \ - ENTRY(YMM30) \ +#define REGS_YMM \ + ENTRY(YMM0) \ + ENTRY(YMM1) \ + ENTRY(YMM2) \ + ENTRY(YMM3) \ + ENTRY(YMM4) \ + ENTRY(YMM5) \ + ENTRY(YMM6) \ + ENTRY(YMM7) \ + ENTRY(YMM8) \ + ENTRY(YMM9) \ + ENTRY(YMM10) \ + ENTRY(YMM11) \ + ENTRY(YMM12) \ + ENTRY(YMM13) \ + ENTRY(YMM14) \ + ENTRY(YMM15) \ + ENTRY(YMM16) \ + ENTRY(YMM17) \ + ENTRY(YMM18) \ + ENTRY(YMM19) \ + ENTRY(YMM20) \ + ENTRY(YMM21) \ + ENTRY(YMM22) \ + ENTRY(YMM23) \ + ENTRY(YMM24) \ + ENTRY(YMM25) \ + ENTRY(YMM26) \ + ENTRY(YMM27) \ + ENTRY(YMM28) \ + ENTRY(YMM29) \ + ENTRY(YMM30) \ ENTRY(YMM31) -#define REGS_ZMM \ - ENTRY(ZMM0) \ - ENTRY(ZMM1) \ - ENTRY(ZMM2) \ - ENTRY(ZMM3) \ - ENTRY(ZMM4) \ - ENTRY(ZMM5) \ - ENTRY(ZMM6) \ - ENTRY(ZMM7) \ - ENTRY(ZMM8) \ - ENTRY(ZMM9) \ - ENTRY(ZMM10) \ - ENTRY(ZMM11) \ - ENTRY(ZMM12) \ - ENTRY(ZMM13) \ - ENTRY(ZMM14) \ - ENTRY(ZMM15) \ - ENTRY(ZMM16) \ - ENTRY(ZMM17) \ - ENTRY(ZMM18) \ - ENTRY(ZMM19) \ - ENTRY(ZMM20) \ - ENTRY(ZMM21) \ - ENTRY(ZMM22) \ - ENTRY(ZMM23) \ - ENTRY(ZMM24) \ - ENTRY(ZMM25) \ - ENTRY(ZMM26) \ - ENTRY(ZMM27) \ - ENTRY(ZMM28) \ - ENTRY(ZMM29) \ - ENTRY(ZMM30) \ +#define REGS_ZMM \ + ENTRY(ZMM0) \ + ENTRY(ZMM1) \ + ENTRY(ZMM2) \ + ENTRY(ZMM3) \ + ENTRY(ZMM4) \ + ENTRY(ZMM5) \ + ENTRY(ZMM6) \ + ENTRY(ZMM7) \ + ENTRY(ZMM8) \ + ENTRY(ZMM9) \ + ENTRY(ZMM10) \ + ENTRY(ZMM11) \ + ENTRY(ZMM12) \ + ENTRY(ZMM13) \ + ENTRY(ZMM14) \ + ENTRY(ZMM15) \ + ENTRY(ZMM16) \ + ENTRY(ZMM17) \ + ENTRY(ZMM18) \ + ENTRY(ZMM19) \ + ENTRY(ZMM20) \ + ENTRY(ZMM21) \ + ENTRY(ZMM22) \ + ENTRY(ZMM23) \ + ENTRY(ZMM24) \ + ENTRY(ZMM25) \ + ENTRY(ZMM26) \ + ENTRY(ZMM27) \ + ENTRY(ZMM28) \ + ENTRY(ZMM29) \ + ENTRY(ZMM30) \ ENTRY(ZMM31) -#define REGS_MASKS \ - ENTRY(K0) \ - ENTRY(K1) \ - ENTRY(K2) \ - ENTRY(K3) \ - ENTRY(K4) \ - ENTRY(K5) \ - ENTRY(K6) \ +#define REGS_MASKS \ + ENTRY(K0) \ + ENTRY(K1) \ + ENTRY(K2) \ + ENTRY(K3) \ + ENTRY(K4) \ + ENTRY(K5) \ + ENTRY(K6) \ ENTRY(K7) -#define REGS_MASK_PAIRS \ - ENTRY(K0_K1) \ - ENTRY(K2_K3) \ - ENTRY(K4_K5) \ +#define REGS_MASK_PAIRS \ + ENTRY(K0_K1) \ + ENTRY(K2_K3) \ + ENTRY(K4_K5) \ ENTRY(K6_K7) -#define REGS_SEGMENT \ - ENTRY(ES) \ - ENTRY(CS) \ - ENTRY(SS) \ - ENTRY(DS) \ - ENTRY(FS) \ +#define REGS_SEGMENT \ + ENTRY(ES) \ + ENTRY(CS) \ + ENTRY(SS) \ + ENTRY(DS) \ + ENTRY(FS) \ ENTRY(GS) -#define REGS_DEBUG \ - ENTRY(DR0) \ - ENTRY(DR1) \ - ENTRY(DR2) \ - ENTRY(DR3) \ - ENTRY(DR4) \ - ENTRY(DR5) \ - ENTRY(DR6) \ - ENTRY(DR7) \ - ENTRY(DR8) \ - ENTRY(DR9) \ - ENTRY(DR10) \ - ENTRY(DR11) \ - ENTRY(DR12) \ - ENTRY(DR13) \ - ENTRY(DR14) \ +#define REGS_DEBUG \ + ENTRY(DR0) \ + ENTRY(DR1) \ + ENTRY(DR2) \ + ENTRY(DR3) \ + ENTRY(DR4) \ + ENTRY(DR5) \ + ENTRY(DR6) \ + ENTRY(DR7) \ + ENTRY(DR8) \ + ENTRY(DR9) \ + ENTRY(DR10) \ + ENTRY(DR11) \ + ENTRY(DR12) \ + ENTRY(DR13) \ + ENTRY(DR14) \ ENTRY(DR15) -#define REGS_CONTROL \ - ENTRY(CR0) \ - ENTRY(CR1) \ - ENTRY(CR2) \ - ENTRY(CR3) \ - ENTRY(CR4) \ - ENTRY(CR5) \ - ENTRY(CR6) \ - ENTRY(CR7) \ - ENTRY(CR8) \ - ENTRY(CR9) \ - ENTRY(CR10) \ - ENTRY(CR11) \ - ENTRY(CR12) \ - ENTRY(CR13) \ - ENTRY(CR14) \ +#define REGS_CONTROL \ + ENTRY(CR0) \ + ENTRY(CR1) \ + ENTRY(CR2) \ + ENTRY(CR3) \ + ENTRY(CR4) \ + ENTRY(CR5) \ + ENTRY(CR6) \ + ENTRY(CR7) \ + ENTRY(CR8) \ + ENTRY(CR9) \ + ENTRY(CR10) \ + ENTRY(CR11) \ + ENTRY(CR12) \ + ENTRY(CR13) \ + ENTRY(CR14) \ ENTRY(CR15) -#undef REGS_TMM -#define REGS_TMM \ - ENTRY(TMM0) \ - ENTRY(TMM1) \ - ENTRY(TMM2) \ - ENTRY(TMM3) \ - ENTRY(TMM4) \ - ENTRY(TMM5) \ - ENTRY(TMM6) \ +#undef REGS_TMM +#define REGS_TMM \ + ENTRY(TMM0) \ + ENTRY(TMM1) \ + ENTRY(TMM2) \ + ENTRY(TMM3) \ + ENTRY(TMM4) \ + ENTRY(TMM5) \ + ENTRY(TMM6) \ ENTRY(TMM7) -#define ALL_EA_BASES \ - EA_BASES_16BIT \ - EA_BASES_32BIT \ +#define ALL_EA_BASES \ + EA_BASES_16BIT \ + EA_BASES_32BIT \ EA_BASES_64BIT -#define ALL_SIB_BASES \ - REGS_32BIT \ +#define ALL_SIB_BASES \ + REGS_32BIT \ REGS_64BIT -#define ALL_REGS \ - REGS_8BIT \ - REGS_16BIT \ - REGS_32BIT \ - REGS_64BIT \ - REGS_MMX \ - REGS_XMM \ - REGS_YMM \ - REGS_ZMM \ - REGS_MASKS \ - REGS_MASK_PAIRS \ - REGS_SEGMENT \ - REGS_DEBUG \ - REGS_CONTROL \ - REGS_TMM \ +#define ALL_REGS \ + REGS_8BIT \ + REGS_16BIT \ + REGS_32BIT \ + REGS_64BIT \ + REGS_MMX \ + REGS_XMM \ + REGS_YMM \ + REGS_ZMM \ + REGS_MASKS \ + REGS_MASK_PAIRS \ + REGS_SEGMENT \ + REGS_DEBUG \ + REGS_CONTROL \ + REGS_TMM \ ENTRY(RIP) /// All possible values of the base field for effective-address @@ -562,6 +562,7 @@ namespace X86Disassembler { /// We distinguish between bases (EA_BASE_*) and registers that just happen /// to be referred to when Mod == 0b11 (EA_REG_*). enum EABase { + // clang-format off EA_BASE_NONE, #define ENTRY(x) EA_BASE_##x, ALL_EA_BASES @@ -570,6 +571,7 @@ enum EABase { ALL_REGS #undef ENTRY EA_max + // clang-format on }; /// All possible values of the SIB index field. @@ -577,6 +579,7 @@ enum EABase { /// sib is synonymous with NONE. /// Vector SIB: index can be XMM or YMM. enum SIBIndex { + // clang-format off SIB_INDEX_NONE, #define ENTRY(x) SIB_INDEX_##x, ALL_EA_BASES @@ -585,32 +588,32 @@ enum SIBIndex { REGS_ZMM #undef ENTRY SIB_INDEX_max + // clang-format on }; /// All possible values of the SIB base field. enum SIBBase { + // clang-format off SIB_BASE_NONE, #define ENTRY(x) SIB_BASE_##x, ALL_SIB_BASES #undef ENTRY SIB_BASE_max + // clang-format on }; /// Possible displacement types for effective-address computations. -enum EADisplacement { - EA_DISP_NONE, - EA_DISP_8, - EA_DISP_16, - EA_DISP_32 -}; +enum EADisplacement { EA_DISP_NONE, EA_DISP_8, EA_DISP_16, EA_DISP_32 }; /// All possible values of the reg field in the ModR/M byte. +// clang-format off enum Reg { #define ENTRY(x) MODRM_REG_##x, ALL_REGS #undef ENTRY MODRM_REG_max }; +// clang-format on /// All possible segment overrides. enum SegmentOverride { @@ -650,11 +653,11 @@ enum VEXPrefixCode { }; enum VectorExtensionType { - TYPE_NO_VEX_XOP = 0x0, - TYPE_VEX_2B = 0x1, - TYPE_VEX_3B = 0x2, - TYPE_EVEX = 0x3, - TYPE_XOP = 0x4 + TYPE_NO_VEX_XOP = 0x0, + TYPE_VEX_2B = 0x1, + TYPE_VEX_3B = 0x2, + TYPE_EVEX = 0x3, + TYPE_XOP = 0x4 }; /// The specification for how to extract and interpret a full instruction and @@ -736,52 +739,52 @@ struct InternalInstruction { // The VEX.vvvv field, which contains a third register operand for some AVX // instructions. - Reg vvvv; + Reg vvvv; // The writemask for AVX-512 instructions which is contained in EVEX.aaa - Reg writemask; + Reg writemask; // The ModR/M byte, which contains most register operands and some portion of // all memory operands. - bool consumedModRM; - uint8_t modRM; + bool consumedModRM; + uint8_t modRM; // The SIB byte, used for more complex 32- or 64-bit memory operands - uint8_t sib; + uint8_t sib; // The displacement, used for memory operands - int32_t displacement; + int32_t displacement; // Immediates. There can be two in some cases - uint8_t numImmediatesConsumed; - uint8_t numImmediatesTranslated; - uint64_t immediates[2]; + uint8_t numImmediatesConsumed; + uint8_t numImmediatesTranslated; + uint64_t immediates[2]; // A register or immediate operand encoded into the opcode - Reg opcodeRegister; + Reg opcodeRegister; // Portions of the ModR/M byte // These fields determine the allowable values for the ModR/M fields, which // depend on operand and address widths. - EABase eaRegBase; - Reg regBase; + EABase eaRegBase; + Reg regBase; // The Mod and R/M fields can encode a base for an effective address, or a // register. These are separated into two fields here. - EABase eaBase; - EADisplacement eaDisplacement; + EABase eaBase; + EADisplacement eaDisplacement; // The reg field always encodes a register - Reg reg; + Reg reg; // SIB state - SIBIndex sibIndexBase; - SIBIndex sibIndex; - uint8_t sibScale; - SIBBase sibBase; + SIBIndex sibIndexBase; + SIBIndex sibIndex; + uint8_t sibScale; + SIBBase sibBase; // Embedded rounding control. - uint8_t RC; + uint8_t RC; ArrayRef operands; }; -- GitLab From 65a8e3a400f23eaa1270a5a000e262df954a4ef6 Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Mon, 4 Mar 2024 18:36:28 -0800 Subject: [PATCH 085/929] [MLIR] Fix crash in notifyBlockInserted() debug output (NFC) notifyBlockInserted can be called when inserting a block in a region before the op is built (like when building a scf::ForOp). This make us defensive by checking the parent op before printing it. --- mlir/lib/Transforms/Utils/DialectConversion.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp index 9dc806730d01..4741110bc606 100644 --- a/mlir/lib/Transforms/Utils/DialectConversion.cpp +++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp @@ -1478,11 +1478,17 @@ void ConversionPatternRewriterImpl::notifyBlockInserted( Block *block, Region *previous, Region::iterator previousIt) { assert(!wasOpReplaced(block->getParentOp()) && "attempting to insert into a region within a replaced/erased op"); - LLVM_DEBUG({ - logger.startLine() << "** Insert Block into : '" - << block->getParentOp()->getName() << "'(" - << block->getParentOp() << ")\n"; - }); + LLVM_DEBUG( + { + Operation *parent = block->getParentOp(); + if (parent) { + logger.startLine() << "** Insert Block into : '" << parent->getName() + << "'(" << parent << ")\n"; + } else { + logger.startLine() + << "** Insert Block into detached Region (nullptr parent op)'"; + } + }); if (!previous) { // This is a newly created block. -- GitLab From 0fbe45bdb909ee4c1a928cc1da030f8b09860431 Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Tue, 5 Mar 2024 10:45:13 +0800 Subject: [PATCH 086/929] [RISCV] Add support of Sscofpmf (#83831) This is used in profile, but somehow we missed it. --- clang/test/Preprocessor/riscv-target-features.c | 9 +++++++++ llvm/docs/RISCVUsage.rst | 3 ++- llvm/lib/Support/RISCVISAInfo.cpp | 1 + llvm/lib/Target/RISCV/RISCVFeatures.td | 3 +++ llvm/test/CodeGen/RISCV/attributes.ll | 4 ++++ llvm/test/MC/RISCV/attribute-arch.s | 3 +++ llvm/unittests/Support/RISCVISAInfoTest.cpp | 1 + 7 files changed, 23 insertions(+), 1 deletion(-) diff --git a/clang/test/Preprocessor/riscv-target-features.c b/clang/test/Preprocessor/riscv-target-features.c index 664279cb1239..1a15be1c6e4d 100644 --- a/clang/test/Preprocessor/riscv-target-features.c +++ b/clang/test/Preprocessor/riscv-target-features.c @@ -29,6 +29,7 @@ // CHECK-NOT: __riscv_smepmp {{.*$}} // CHECK-NOT: __riscv_ssaia {{.*$}} // CHECK-NOT: __riscv_ssccptr {{.*$}} +// CHECK-NOT: __riscv_sscofpmf {{.*$}} // CHECK-NOT: __riscv_sscounterenw {{.*$}} // CHECK-NOT: __riscv_ssstateen {{.*$}} // CHECK-NOT: __riscv_ssstrict {{.*$}} @@ -351,6 +352,14 @@ // RUN: -o - | FileCheck --check-prefix=CHECK-SSCCPTR-EXT %s // CHECK-SSCCPTR-EXT: __riscv_ssccptr 1000000{{$}} +// RUN: %clang --target=riscv32-unknown-linux-gnu \ +// RUN: -march=rv32isscofpmf -E -dM %s \ +// RUN: -o - | FileCheck --check-prefix=CHECK-SSCOFPMF-EXT %s +// RUN: %clang --target=riscv64-unknown-linux-gnu \ +// RUN: -march=rv64isscofpmf -E -dM %s \ +// RUN: -o - | FileCheck --check-prefix=CHECK-SSCOFPMF-EXT %s +// CHECK-SSCOFPMF-EXT: __riscv_sscofpmf 1000000{{$}} + // RUN: %clang --target=riscv32-unknown-linux-gnu \ // RUN: -march=rv32isscounterenw -E -dM %s \ // RUN: -o - | FileCheck --check-prefix=CHECK-SSCOUNTERENW-EXT %s diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst index 8d293b021443..2595af598f1e 100644 --- a/llvm/docs/RISCVUsage.rst +++ b/llvm/docs/RISCVUsage.rst @@ -101,6 +101,7 @@ on support follow. ``Smepmp`` Supported ``Ssaia`` Supported ``Ssccptr`` Assembly Support (`See note <#riscv-profiles-extensions-note>`__) + ``Sscofpmf`` Assembly Support (`See note <#riscv-profiles-extensions-note>`__) ``Sscounterenw`` Assembly Support (`See note <#riscv-profiles-extensions-note>`__) ``Ssstateen`` Assembly Support (`See note <#riscv-profiles-extensions-note>`__) ``Ssstrict`` Assembly Support (`See note <#riscv-profiles-extensions-note>`__) @@ -234,7 +235,7 @@ Supported .. _riscv-profiles-extensions-note: -``Za128rs``, ``Za64rs``, ``Zic64b``, ``Ziccamoa``, ``Ziccif``, ``Zicclsm``, ``Ziccrse``, ``Shcounterenvw``, ``Shgatpa``, ``Shtvala``, ``Shvsatpa``, ``Shvstvala``, ``Shvstvecd``, ``Ssccptr``, ``Sscounterenw``, ``Ssstateen``, ``Ssstrict``, ``Sstvala``, ``Sstvecd``, ``Ssu64xl``, ``Svade``, ``Svbare`` +``Za128rs``, ``Za64rs``, ``Zic64b``, ``Ziccamoa``, ``Ziccif``, ``Zicclsm``, ``Ziccrse``, ``Shcounterenvw``, ``Shgatpa``, ``Shtvala``, ``Shvsatpa``, ``Shvstvala``, ``Shvstvecd``, ``Ssccptr``, ``Sscofpmf``, ``Sscounterenw``, ``Ssstateen``, ``Ssstrict``, ``Sstvala``, ``Sstvecd``, ``Ssu64xl``, ``Svade``, ``Svbare`` These extensions are defined as part of the `RISC-V Profiles specification `__. They do not introduce any new features themselves, but instead describe existing hardware features. .. _riscv-zacas-note: diff --git a/llvm/lib/Support/RISCVISAInfo.cpp b/llvm/lib/Support/RISCVISAInfo.cpp index 68f5c36e8faf..54b4dcb22de8 100644 --- a/llvm/lib/Support/RISCVISAInfo.cpp +++ b/llvm/lib/Support/RISCVISAInfo.cpp @@ -65,6 +65,7 @@ static const RISCVSupportedExtension SupportedExtensions[] = { {"smepmp", {1, 0}}, {"ssaia", {1, 0}}, {"ssccptr", {1, 0}}, + {"sscofpmf", {1, 0}}, {"sscounterenw", {1, 0}}, {"ssstateen", {1, 0}}, {"ssstrict", {1, 0}}, diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td index 9773b2998c7d..78f3a82d66df 100644 --- a/llvm/lib/Target/RISCV/RISCVFeatures.td +++ b/llvm/lib/Target/RISCV/RISCVFeatures.td @@ -807,6 +807,9 @@ def FeatureStdExtSsccptr : SubtargetFeature<"ssccptr", "HasStdExtSsccptr", "true", "'Ssccptr' (Main memory supports page table reads)", []>; +def FeatureStdExtSscofpmf + : SubtargetFeature<"sscofpmf", "HasStdExtSscofpmf", "true", + "'Sscofpmf' (Count Overflow and Mode-Based Filtering)", []>; def FeatureStdExtShcounterenw : SubtargetFeature<"shcounterenw", "HasStdExtShcounterenw", "true", "'Shcounterenw' (Support writeable hcounteren enable " diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll index 561b0f21dc37..cc332df27104 100644 --- a/llvm/test/CodeGen/RISCV/attributes.ll +++ b/llvm/test/CodeGen/RISCV/attributes.ll @@ -42,6 +42,7 @@ ; RUN: llc -mtriple=riscv32 -mattr=+shgatpa %s -o - | FileCheck --check-prefixes=CHECK,RV32SHGATPA %s ; RUN: llc -mtriple=riscv32 -mattr=+shvsatpa %s -o - | FileCheck --check-prefixes=CHECK,RV32SHVSATPA %s ; RUN: llc -mtriple=riscv32 -mattr=+ssccptr %s -o - | FileCheck --check-prefixes=CHECK,RV32SSCCPTR %s +; RUN: llc -mtriple=riscv32 -mattr=+sscofpmf %s -o - | FileCheck --check-prefixes=CHECK,RV32SSCOFPMF %s ; RUN: llc -mtriple=riscv32 -mattr=+sscounterenw %s -o - | FileCheck --check-prefixes=CHECK,RV32SSCOUNTERENW %s ; RUN: llc -mtriple=riscv32 -mattr=+ssstateen %s -o - | FileCheck --check-prefixes=CHECK,RV32SSSTATEEN %s ; RUN: llc -mtriple=riscv32 -mattr=+ssstrict %s -o - | FileCheck --check-prefixes=CHECK,RV32SSSTRICT %s @@ -166,6 +167,7 @@ ; RUN: llc -mtriple=riscv64 -mattr=+shgatpa %s -o - | FileCheck --check-prefixes=CHECK,RV64SHGATPA %s ; RUN: llc -mtriple=riscv64 -mattr=+shvsatpa %s -o - | FileCheck --check-prefixes=CHECK,RV64SHVSATPA %s ; RUN: llc -mtriple=riscv64 -mattr=+ssccptr %s -o - | FileCheck --check-prefixes=CHECK,RV64SSCCPTR %s +; RUN: llc -mtriple=riscv64 -mattr=+sscofpmf %s -o - | FileCheck --check-prefixes=CHECK,RV64SSCOFPMF %s ; RUN: llc -mtriple=riscv64 -mattr=+sscounterenw %s -o - | FileCheck --check-prefixes=CHECK,RV64SSCOUNTERENW %s ; RUN: llc -mtriple=riscv64 -mattr=+ssstateen %s -o - | FileCheck --check-prefixes=CHECK,RV64SSSTATEEN %s ; RUN: llc -mtriple=riscv64 -mattr=+ssstrict %s -o - | FileCheck --check-prefixes=CHECK,RV64SSSTRICT %s @@ -295,6 +297,7 @@ ; RV32SHGATPA: .attribute 5, "rv32i2p1_shgatpa1p0" ; RV32SHVSATPA: .attribute 5, "rv32i2p1_shvsatpa1p0" ; RV32SSCCPTR: .attribute 5, "rv32i2p1_ssccptr1p0" +; RV32SSCOFPMF: .attribute 5, "rv32i2p1_sscofpmf1p0" ; RV32SSCOUNTERENW: .attribute 5, "rv32i2p1_sscounterenw1p0" ; RV32SSSTATEEN: .attribute 5, "rv32i2p1_ssstateen1p0" ; RV32SSSTRICT: .attribute 5, "rv32i2p1_ssstrict1p0" @@ -421,6 +424,7 @@ ; RV64SHGATPA: .attribute 5, "rv64i2p1_shgatpa1p0" ; RV64SHVSATPA: .attribute 5, "rv64i2p1_shvsatpa1p0" ; RV64SSCCPTR: .attribute 5, "rv64i2p1_ssccptr1p0" +; RV64SSCOFPMF: .attribute 5, "rv64i2p1_sscofpmf1p0" ; RV64SSCOUNTERENW: .attribute 5, "rv64i2p1_sscounterenw1p0" ; RV64SSSTATEEN: .attribute 5, "rv64i2p1_ssstateen1p0" ; RV64SSSTRICT: .attribute 5, "rv64i2p1_ssstrict1p0" diff --git a/llvm/test/MC/RISCV/attribute-arch.s b/llvm/test/MC/RISCV/attribute-arch.s index 368d8daf72b3..09daeee2c1b3 100644 --- a/llvm/test/MC/RISCV/attribute-arch.s +++ b/llvm/test/MC/RISCV/attribute-arch.s @@ -318,6 +318,9 @@ .attribute arch, "rv32i_ssccptr1p0" # CHECK: attribute 5, "rv32i2p1_ssccptr1p0" +.attribute arch, "rv32i_sscofpmf1p0" +# CHECK: attribute 5, "rv32i2p1_sscofpmf1p0" + .attribute arch, "rv32i_sscounterenw1p0" # CHECK: attribute 5, "rv32i2p1_sscounterenw1p0" diff --git a/llvm/unittests/Support/RISCVISAInfoTest.cpp b/llvm/unittests/Support/RISCVISAInfoTest.cpp index 82cce23638d5..8a627eca05cb 100644 --- a/llvm/unittests/Support/RISCVISAInfoTest.cpp +++ b/llvm/unittests/Support/RISCVISAInfoTest.cpp @@ -830,6 +830,7 @@ R"(All available -march extensions for RISC-V smepmp 1.0 ssaia 1.0 ssccptr 1.0 + sscofpmf 1.0 sscounterenw 1.0 ssstateen 1.0 ssstrict 1.0 -- GitLab From 9dab2e3064a2ad4d5ce65832d605787dc394cb72 Mon Sep 17 00:00:00 2001 From: Youngsuk Kim Date: Mon, 4 Mar 2024 22:10:20 -0500 Subject: [PATCH 087/929] [clang][Sema] Warn on return of pointer/reference to compound literal (#83741) Emit a warning if pointer/reference to compound literal is returned from a function. In C, compound literals in block scope are lvalues that have automatic storage duration. In C++, compound literals in block scope are temporaries. In either case, returning a pointer/reference to a compound literal can cause a use-after-free bug. Fixes #8678 --- clang/include/clang/Basic/DiagnosticSemaKinds.td | 2 +- clang/lib/Sema/SemaInit.cpp | 12 ++++++++++++ clang/test/Analysis/stack-addr-ps.c | 4 ++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index 8ea194ceadb5..b50a4895e171 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -9901,7 +9901,7 @@ def err_lifetimebound_ctor_dtor : Error< // CHECK: returning address/reference of stack memory def warn_ret_stack_addr_ref : Warning< "%select{address of|reference to}0 stack memory associated with " - "%select{local variable|parameter}2 %1 returned">, + "%select{local variable|parameter|compound literal}2 %1 returned">, InGroup; def warn_ret_local_temp_addr_ref : Warning< "returning %select{address of|reference to}0 local temporary object">, diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp index 0fd458837163..93b125382b16 100644 --- a/clang/lib/Sema/SemaInit.cpp +++ b/clang/lib/Sema/SemaInit.cpp @@ -7734,6 +7734,14 @@ static void visitLocalsRetainedByReferenceBinding(IndirectLocalPath &Path, break; } + case Stmt::CompoundLiteralExprClass: { + if (auto *CLE = dyn_cast(Init)) { + if (!CLE->isFileScope()) + Visit(Path, Local(CLE), RK); + } + break; + } + // FIXME: Visit the left-hand side of an -> or ->*. default: @@ -8289,6 +8297,10 @@ void Sema::checkInitializerLifetime(const InitializedEntity &Entity, if (LK == LK_StmtExprResult) return false; Diag(DiagLoc, diag::warn_ret_addr_label) << DiagRange; + } else if (auto *CLE = dyn_cast(L)) { + Diag(DiagLoc, diag::warn_ret_stack_addr_ref) + << Entity.getType()->isReferenceType() << CLE->getInitializer() << 2 + << DiagRange; } else { Diag(DiagLoc, diag::warn_ret_local_temp_addr_ref) << Entity.getType()->isReferenceType() << DiagRange; diff --git a/clang/test/Analysis/stack-addr-ps.c b/clang/test/Analysis/stack-addr-ps.c index 26e1cc58350c..e469396e1bb2 100644 --- a/clang/test/Analysis/stack-addr-ps.c +++ b/clang/test/Analysis/stack-addr-ps.c @@ -20,13 +20,13 @@ int* f3(int x, int *y) { void* compound_literal(int x, int y) { if (x) - return &(unsigned short){((unsigned short)0x22EF)}; // expected-warning{{Address of stack memory}} + return &(unsigned short){((unsigned short)0x22EF)}; // expected-warning{{Address of stack memory}} expected-warning{{address of stack memory}} int* array[] = {}; struct s { int z; double y; int w; }; if (y) - return &((struct s){ 2, 0.4, 5 * 8 }); // expected-warning{{Address of stack memory}} + return &((struct s){ 2, 0.4, 5 * 8 }); // expected-warning{{Address of stack memory}} expected-warning{{address of stack memory}} void* p = &((struct s){ 42, 0.4, x ? 42 : 0 }); -- GitLab From 7cd32688042988d2d02f791dd106286a67041b63 Mon Sep 17 00:00:00 2001 From: Shengchen Kan Date: Tue, 5 Mar 2024 11:23:13 +0800 Subject: [PATCH 088/929] [X86][TableGen] Fix the mnemonic table for CMPCCXADD The mnemonic of CMPCCXADD is `cmp${cond}xadd` and the condition code is in the middle of mnemonic. When generating the function name for CMPCCXADD, the substring `xadd` should be kept. Before this patch, the name is `isCMPCC`. After this patch, the name is `isCMPCCXADD`. --- .../X86/MCTargetDesc/X86InstPrinterCommon.cpp | 20 +++++++++---------- llvm/utils/TableGen/X86RecognizableInstr.cpp | 17 ++++++++-------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp index e519c00a2110..7422a9897346 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp @@ -29,29 +29,29 @@ using namespace llvm; void X86InstPrinterCommon::printCondCode(const MCInst *MI, unsigned Op, raw_ostream &O) { int64_t Imm = MI->getOperand(Op).getImm(); - bool Flavor = MI->getOpcode() == X86::CMPCCXADDmr32 || - MI->getOpcode() == X86::CMPCCXADDmr64 || - MI->getOpcode() == X86::CMPCCXADDmr32_EVEX || - MI->getOpcode() == X86::CMPCCXADDmr64_EVEX; + bool IsCMPCCXADD = X86::isCMPCCXADD(MI->getOpcode()); + + // clang-format off switch (Imm) { default: llvm_unreachable("Invalid condcode argument!"); case 0: O << "o"; break; case 1: O << "no"; break; case 2: O << "b"; break; - case 3: O << (Flavor ? "nb" : "ae"); break; - case 4: O << (Flavor ? "z" : "e"); break; - case 5: O << (Flavor ? "nz" : "ne"); break; + case 3: O << (IsCMPCCXADD ? "nb" : "ae"); break; + case 4: O << (IsCMPCCXADD ? "z" : "e"); break; + case 5: O << (IsCMPCCXADD ? "nz" : "ne"); break; case 6: O << "be"; break; - case 7: O << (Flavor ? "nbe" : "a"); break; + case 7: O << (IsCMPCCXADD ? "nbe" : "a"); break; case 8: O << "s"; break; case 9: O << "ns"; break; case 0xa: O << "p"; break; case 0xb: O << "np"; break; case 0xc: O << "l"; break; - case 0xd: O << (Flavor ? "nl" : "ge"); break; + case 0xd: O << (IsCMPCCXADD ? "nl" : "ge"); break; case 0xe: O << "le"; break; - case 0xf: O << (Flavor ? "nle" : "g"); break; + case 0xf: O << (IsCMPCCXADD ? "nle" : "g"); break; } + // clang-format on } void X86InstPrinterCommon::printSSEAVXCC(const MCInst *MI, unsigned Op, diff --git a/llvm/utils/TableGen/X86RecognizableInstr.cpp b/llvm/utils/TableGen/X86RecognizableInstr.cpp index 873f3aea053c..5aa2e51c78ba 100644 --- a/llvm/utils/TableGen/X86RecognizableInstr.cpp +++ b/llvm/utils/TableGen/X86RecognizableInstr.cpp @@ -26,17 +26,18 @@ using namespace X86Disassembler; std::string X86Disassembler::getMnemonic(const CodeGenInstruction *I, unsigned Variant) { - std::string AsmString = I->FlattenAsmStringVariants(I->AsmString, Variant); - StringRef Mnemonic(AsmString); // Extract a mnemonic assuming it's separated by \t - Mnemonic = Mnemonic.take_until([](char C) { return C == '\t'; }); + std::string Mnemonic = + StringRef(I->FlattenAsmStringVariants(I->AsmString, Variant)) + .take_until([](char C) { return C == '\t'; }) + .str(); - // Special case: CMOVCC, JCC, SETCC have "${cond}" in mnemonic. + // Special case: CMOVCC, JCC, SETCC, CMPCCXADD have "${cond}" in mnemonic. // Replace it with "CC" in-place. - size_t CondPos = Mnemonic.find("${cond}"); - if (CondPos != StringRef::npos) - Mnemonic = AsmString.replace(CondPos, StringRef::npos, "CC"); - return Mnemonic.upper(); + auto CondPos = Mnemonic.find("${cond}"); + if (CondPos != std::string::npos) + Mnemonic = Mnemonic.replace(CondPos, 7, "CC"); + return StringRef(Mnemonic).upper(); } bool X86Disassembler::isRegisterOperand(const Record *Rec) { -- GitLab From cec2073f8e82c2d72a7246300aaa7b2a85ca4012 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Mon, 4 Mar 2024 17:15:09 +0100 Subject: [PATCH 089/929] [clang][Interp] Diagnose comparisions against weak function pointers --- clang/lib/AST/Interp/Interp.h | 11 +++++++++++ clang/test/AST/Interp/functions.cpp | 10 ++++++++++ 2 files changed, 21 insertions(+) diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h index 3d49f73a5676..548405a53dfa 100644 --- a/clang/lib/AST/Interp/Interp.h +++ b/clang/lib/AST/Interp/Interp.h @@ -746,6 +746,17 @@ inline bool CmpHelperEQ(InterpState &S, CodePtr OpPC, CompareFn Fn) { const auto &RHS = S.Stk.pop(); const auto &LHS = S.Stk.pop(); + + // We cannot compare against weak declarations at compile time. + for (const auto &FP : {LHS, RHS}) { + if (!FP.isZero() && FP.getFunction()->getDecl()->isWeak()) { + const SourceInfo &Loc = S.Current->getSource(OpPC); + S.FFDiag(Loc, diag::note_constexpr_pointer_weak_comparison) + << FP.toDiagnosticString(S.getCtx()); + return false; + } + } + S.Stk.push(Boolean::from(Fn(LHS.compare(RHS)))); return true; } diff --git a/clang/test/AST/Interp/functions.cpp b/clang/test/AST/Interp/functions.cpp index 38f761f563be..67fd9036d81e 100644 --- a/clang/test/AST/Interp/functions.cpp +++ b/clang/test/AST/Interp/functions.cpp @@ -565,3 +565,13 @@ namespace VariadicOperator { float &fr = c(10); } } + +namespace WeakCompare { + [[gnu::weak]]void weak_method(); + static_assert(weak_method != nullptr, ""); // both-error {{not an integral constant expression}} \ + // both-note {{comparison against address of weak declaration '&weak_method' can only be performed at runtim}} + + constexpr auto A = &weak_method; + static_assert(A != nullptr, ""); // both-error {{not an integral constant expression}} \ + // both-note {{comparison against address of weak declaration '&weak_method' can only be performed at runtim}} +} -- GitLab From d016712b2cbeecf13c0d558c7c8932fd66e14847 Mon Sep 17 00:00:00 2001 From: Jordan Rupprecht Date: Mon, 4 Mar 2024 23:00:04 -0600 Subject: [PATCH 090/929] [bzl] Remove obsolete `output_to_genfiles = True` (#83944) The [bazel docs](https://bazel.build/rules/lib/globals/bzl#rule) discourage setting this. The comments about being necessary for headers or genrules seem to be obsolete, at least for the LLVM tree itself. The effect of this is that generated files will go to `bazel-bin` instead of `bazel-genfiles`. One external use was fixed here: https://github.com/google/jax/commit/32bb3b06132b4256cd8674fb98bce057dc968610. --- utils/bazel/llvm-project-overlay/llvm/enum_targets_gen.bzl | 2 -- utils/bazel/llvm-project-overlay/mlir/tblgen.bzl | 2 -- 2 files changed, 4 deletions(-) diff --git a/utils/bazel/llvm-project-overlay/llvm/enum_targets_gen.bzl b/utils/bazel/llvm-project-overlay/llvm/enum_targets_gen.bzl index 998b706dbbbb..79617490bf7a 100644 --- a/utils/bazel/llvm-project-overlay/llvm/enum_targets_gen.bzl +++ b/utils/bazel/llvm-project-overlay/llvm/enum_targets_gen.bzl @@ -64,7 +64,5 @@ enum_targets_gen = rule( " `@LLVM_ENUM_{macro_name}S@`", ), }, - # output_to_genfiles is required for header files. - output_to_genfiles = True, implementation = enum_targets_gen_impl, ) diff --git a/utils/bazel/llvm-project-overlay/mlir/tblgen.bzl b/utils/bazel/llvm-project-overlay/mlir/tblgen.bzl index 9d26822ac163..fdf6a57107ac 100644 --- a/utils/bazel/llvm-project-overlay/mlir/tblgen.bzl +++ b/utils/bazel/llvm-project-overlay/mlir/tblgen.bzl @@ -181,8 +181,6 @@ def _gentbl_rule_impl(ctx): gentbl_rule = rule( _gentbl_rule_impl, doc = "Generates tabular code from a table definition file.", - # Match genrule behavior - output_to_genfiles = True, attrs = { "tblgen": attr.label( doc = "The TableGen executable with which to generate `out`.", -- GitLab From 662d821d44420566cd0bb649f30d7689562bdb88 Mon Sep 17 00:00:00 2001 From: Aart Bik Date: Mon, 4 Mar 2024 21:14:32 -0800 Subject: [PATCH 091/929] [mlir][sparse] migrate datastructure tests to sparse_tensor.print (#83956) Continuing the efforts started in llvm#83357 --- .../SparseTensor/CPU/sparse_insert_1d.mlir | 62 +++--- .../SparseTensor/CPU/sparse_insert_2d.mlir | 176 +++++------------- 2 files changed, 72 insertions(+), 166 deletions(-) diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir index bc7ecb08ab2f..61c68507ea51 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -30,8 +30,6 @@ // Do the same run, but now with direct IR generation and VLA vectorization. // RUN: %if mlir_arm_sve_tests %{ %{compile_sve} | %{run_sve} | FileCheck %s %} -// Insertion example using pure codegen (no sparse runtime support lib). - #SparseVector = #sparse_tensor.encoding<{ map = (d0) -> (d0 : compressed) }> #trait_mul_s = { @@ -43,27 +41,7 @@ } module { - - // Dumps positions, indices, values for verification. - func.func @dump(%argx: tensor<1024xf32, #SparseVector>) { - %c0 = arith.constant 0 : index - %f0 = arith.constant 0.0 : f32 - %p = sparse_tensor.positions %argx { level = 0 : index } - : tensor<1024xf32, #SparseVector> to memref - %i = sparse_tensor.coordinates %argx { level = 0 : index } - : tensor<1024xf32, #SparseVector> to memref - %v = sparse_tensor.values %argx - : tensor<1024xf32, #SparseVector> to memref - %vp = vector.transfer_read %p[%c0], %c0: memref, vector<2xindex> - %vi = vector.transfer_read %i[%c0], %c0: memref, vector<8xindex> - %vv = vector.transfer_read %v[%c0], %f0: memref, vector<8xf32> - vector.print %vp : vector<2xindex> - vector.print %vi : vector<8xindex> - vector.print %vv : vector<8xf32> - return - } - - func.func @entry() { + func.func @main() { %f1 = arith.constant 1.0 : f32 %f2 = arith.constant 2.0 : f32 %f3 = arith.constant 3.0 : f32 @@ -82,10 +60,17 @@ module { %4 = sparse_tensor.insert %f4 into %3[%c1023] : tensor<1024xf32, #SparseVector> %5 = sparse_tensor.load %4 hasInserts : tensor<1024xf32, #SparseVector> - // CHECK: ( 0, 4 ) - // CHECK-NEXT: ( 0, 1, 3, 1023 - // CHECK-NEXT: ( 1, 2, 3, 4 - call @dump(%5) : (tensor<1024xf32, #SparseVector>) -> () + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 4 + // CHECK-NEXT: dim = ( 1024 ) + // CHECK-NEXT: lvl = ( 1024 ) + // CHECK-NEXT: pos[0] : ( 0, 4, + // CHECK-NEXT: crd[0] : ( 0, 1, 3, 1023, + // CHECK-NEXT: values : ( 1, 2, 3, 4, + // CHECK-NEXT: ---- + // + sparse_tensor.print %5 : tensor<1024xf32, #SparseVector> // Build another sparse vector in a loop. %6 = tensor.empty() : tensor<1024xf32, #SparseVector> @@ -96,18 +81,17 @@ module { } %8 = sparse_tensor.load %7 hasInserts : tensor<1024xf32, #SparseVector> - // CHECK-NEXT: ( 0, 8 ) - // CHECK-NEXT: ( 0, 3, 6, 9, 12, 15, 18, 21 ) - // CHECK-NEXT: ( 1, 1, 1, 1, 1, 1, 1, 1 ) // - call @dump(%8) : (tensor<1024xf32, #SparseVector>) -> () - - // CHECK-NEXT: 4 - // CHECK-NEXT: 8 - %noe1 = sparse_tensor.number_of_entries %5 : tensor<1024xf32, #SparseVector> - %noe2 = sparse_tensor.number_of_entries %8 : tensor<1024xf32, #SparseVector> - vector.print %noe1 : index - vector.print %noe2 : index + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 8 + // CHECK-NEXT: dim = ( 1024 ) + // CHECK-NEXT: lvl = ( 1024 ) + // CHECK-NEXT: pos[0] : ( 0, 8, + // CHECK-NEXT: crd[0] : ( 0, 3, 6, 9, 12, 15, 18, 21, + // CHECK-NEXT: values : ( 1, 1, 1, 1, 1, 1, 1, 1, + // CHECK-NEXT: ---- + // + sparse_tensor.print %8 : tensor<1024xf32, #SparseVector> // Free resources. bufferization.dealloc_tensor %5 : tensor<1024xf32, #SparseVector> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir index b8cc1997783a..d51b67792337 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -32,7 +32,7 @@ }> #SortedCOO = #sparse_tensor.encoding<{ - map = (d0, d1) -> (d0 : compressed(nonunique), d1 : singleton) + map = (d0, d1) -> (d0 : compressed(nonunique), d1 : singleton(soa)) }> #CSR = #sparse_tensor.encoding<{ @@ -48,94 +48,11 @@ }> module { - - func.func @dump_dense(%arg0: tensor<4x3xf64, #Dense>) { - %c0 = arith.constant 0 : index - %fu = arith.constant 99.0 : f64 - %v = sparse_tensor.values %arg0 : tensor<4x3xf64, #Dense> to memref - %vv = vector.transfer_read %v[%c0], %fu: memref, vector<12xf64> - vector.print %vv : vector<12xf64> - return - } - - func.func @dump_coo(%arg0: tensor<4x3xf64, #SortedCOO>) { - %c0 = arith.constant 0 : index - %cu = arith.constant -1 : index - %fu = arith.constant 99.0 : f64 - %p0 = sparse_tensor.positions %arg0 { level = 0 : index } : tensor<4x3xf64, #SortedCOO> to memref - %i0 = sparse_tensor.coordinates %arg0 { level = 0 : index } : tensor<4x3xf64, #SortedCOO> to memref> - %i1 = sparse_tensor.coordinates %arg0 { level = 1 : index } : tensor<4x3xf64, #SortedCOO> to memref> - %v = sparse_tensor.values %arg0 : tensor<4x3xf64, #SortedCOO> to memref - %vp0 = vector.transfer_read %p0[%c0], %cu: memref, vector<2xindex> - vector.print %vp0 : vector<2xindex> - %vi0 = vector.transfer_read %i0[%c0], %cu: memref>, vector<4xindex> - vector.print %vi0 : vector<4xindex> - %vi1 = vector.transfer_read %i1[%c0], %cu: memref>, vector<4xindex> - vector.print %vi1 : vector<4xindex> - %vv = vector.transfer_read %v[%c0], %fu: memref, vector<4xf64> - vector.print %vv : vector<4xf64> - return - } - - func.func @dump_csr(%arg0: tensor<4x3xf64, #CSR>) { - %c0 = arith.constant 0 : index - %cu = arith.constant -1 : index - %fu = arith.constant 99.0 : f64 - %p1 = sparse_tensor.positions %arg0 { level = 1 : index } : tensor<4x3xf64, #CSR> to memref - %i1 = sparse_tensor.coordinates %arg0 { level = 1 : index } : tensor<4x3xf64, #CSR> to memref - %v = sparse_tensor.values %arg0 : tensor<4x3xf64, #CSR> to memref - %vp1 = vector.transfer_read %p1[%c0], %cu: memref, vector<5xindex> - vector.print %vp1 : vector<5xindex> - %vi1 = vector.transfer_read %i1[%c0], %cu: memref, vector<4xindex> - vector.print %vi1 : vector<4xindex> - %vv = vector.transfer_read %v[%c0], %fu: memref, vector<4xf64> - vector.print %vv : vector<4xf64> - return - } - - func.func @dump_dcsr(%arg0: tensor<4x3xf64, #DCSR>) { - %c0 = arith.constant 0 : index - %cu = arith.constant -1 : index - %fu = arith.constant 99.0 : f64 - %p0 = sparse_tensor.positions %arg0 { level = 0 : index } : tensor<4x3xf64, #DCSR> to memref - %i0 = sparse_tensor.coordinates %arg0 { level = 0 : index } : tensor<4x3xf64, #DCSR> to memref - %p1 = sparse_tensor.positions %arg0 { level = 1 : index } : tensor<4x3xf64, #DCSR> to memref - %i1 = sparse_tensor.coordinates %arg0 { level = 1 : index } : tensor<4x3xf64, #DCSR> to memref - %v = sparse_tensor.values %arg0 : tensor<4x3xf64, #DCSR> to memref - %vp0 = vector.transfer_read %p0[%c0], %cu: memref, vector<2xindex> - vector.print %vp0 : vector<2xindex> - %vi0 = vector.transfer_read %i0[%c0], %cu: memref, vector<3xindex> - vector.print %vi0 : vector<3xindex> - %vp1 = vector.transfer_read %p1[%c0], %cu: memref, vector<4xindex> - vector.print %vp1 : vector<4xindex> - %vi1 = vector.transfer_read %i1[%c0], %cu: memref, vector<4xindex> - vector.print %vi1 : vector<4xindex> - %vv = vector.transfer_read %v[%c0], %fu: memref, vector<4xf64> - vector.print %vv : vector<4xf64> - return - } - - func.func @dump_row(%arg0: tensor<4x3xf64, #Row>) { - %c0 = arith.constant 0 : index - %cu = arith.constant -1 : index - %fu = arith.constant 99.0 : f64 - %p0 = sparse_tensor.positions %arg0 { level = 0 : index } : tensor<4x3xf64, #Row> to memref - %i0 = sparse_tensor.coordinates %arg0 { level = 0 : index } : tensor<4x3xf64, #Row> to memref - %v = sparse_tensor.values %arg0 : tensor<4x3xf64, #Row> to memref - %vp0 = vector.transfer_read %p0[%c0], %cu: memref, vector<2xindex> - vector.print %vp0 : vector<2xindex> - %vi0 = vector.transfer_read %i0[%c0], %cu: memref, vector<3xindex> - vector.print %vi0 : vector<3xindex> - %vv = vector.transfer_read %v[%c0], %fu: memref, vector<9xf64> - vector.print %vv : vector<9xf64> - return - } - // // Main driver. We test the contents of various sparse tensor // schemes when they are still empty and after a few insertions. // - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %c2 = arith.constant 2 : index %c3 = arith.constant 3 : index @@ -147,7 +64,12 @@ module { // // Dense case. // - // CHECK: ( 1, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 4 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 12 + // CHECK-NEXT: dim = ( 4, 3 ) + // CHECK-NEXT: lvl = ( 4, 3 ) + // CHECK-NEXT: values : ( 1, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 4, + // CHECK-NEXT: ---- // %densea = tensor.empty() : tensor<4x3xf64, #Dense> %dense1 = sparse_tensor.insert %f1 into %densea[%c0, %c0] : tensor<4x3xf64, #Dense> @@ -155,15 +77,20 @@ module { %dense3 = sparse_tensor.insert %f3 into %dense2[%c3, %c0] : tensor<4x3xf64, #Dense> %dense4 = sparse_tensor.insert %f4 into %dense3[%c3, %c2] : tensor<4x3xf64, #Dense> %densem = sparse_tensor.load %dense4 hasInserts : tensor<4x3xf64, #Dense> - call @dump_dense(%densem) : (tensor<4x3xf64, #Dense>) -> () + sparse_tensor.print %densem : tensor<4x3xf64, #Dense> // // COO case. // - // CHECK-NEXT: ( 0, 4 ) - // CHECK-NEXT: ( 0, 2, 3, 3 ) - // CHECK-NEXT: ( 0, 2, 0, 2 ) - // CHECK-NEXT: ( 1, 2, 3, 4 ) + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 4 + // CHECK-NEXT: dim = ( 4, 3 ) + // CHECK-NEXT: lvl = ( 4, 3 ) + // CHECK-NEXT: pos[0] : ( 0, 4, + // CHECK-NEXT: crd[0] : ( 0, 2, 3, 3, + // CHECK-NEXT: crd[1] : ( 0, 2, 0, 2, + // CHECK-NEXT: values : ( 1, 2, 3, 4, + // CHECK-NEXT: ---- // %cooa = tensor.empty() : tensor<4x3xf64, #SortedCOO> %coo1 = sparse_tensor.insert %f1 into %cooa[%c0, %c0] : tensor<4x3xf64, #SortedCOO> @@ -171,14 +98,19 @@ module { %coo3 = sparse_tensor.insert %f3 into %coo2[%c3, %c0] : tensor<4x3xf64, #SortedCOO> %coo4 = sparse_tensor.insert %f4 into %coo3[%c3, %c2] : tensor<4x3xf64, #SortedCOO> %coom = sparse_tensor.load %coo4 hasInserts : tensor<4x3xf64, #SortedCOO> - call @dump_coo(%coom) : (tensor<4x3xf64, #SortedCOO>) -> () + sparse_tensor.print %coom : tensor<4x3xf64, #SortedCOO> // // CSR case. // - // CHECK-NEXT: ( 0, 1, 1, 2, 4 ) - // CHECK-NEXT: ( 0, 2, 0, 2 ) - // CHECK-NEXT: ( 1, 2, 3, 4 ) + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 4 + // CHECK-NEXT: dim = ( 4, 3 ) + // CHECK-NEXT: lvl = ( 4, 3 ) + // CHECK-NEXT: pos[1] : ( 0, 1, 1, 2, 4, + // CHECK-NEXT: crd[1] : ( 0, 2, 0, 2, + // CHECK-NEXT: values : ( 1, 2, 3, 4, + // CHECK-NEXT: ---- // %csra = tensor.empty() : tensor<4x3xf64, #CSR> %csr1 = sparse_tensor.insert %f1 into %csra[%c0, %c0] : tensor<4x3xf64, #CSR> @@ -186,16 +118,21 @@ module { %csr3 = sparse_tensor.insert %f3 into %csr2[%c3, %c0] : tensor<4x3xf64, #CSR> %csr4 = sparse_tensor.insert %f4 into %csr3[%c3, %c2] : tensor<4x3xf64, #CSR> %csrm = sparse_tensor.load %csr4 hasInserts : tensor<4x3xf64, #CSR> - call @dump_csr(%csrm) : (tensor<4x3xf64, #CSR>) -> () + sparse_tensor.print %csrm : tensor<4x3xf64, #CSR> // // DCSR case. // - // CHECK-NEXT: ( 0, 3 ) - // CHECK-NEXT: ( 0, 2, 3 ) - // CHECK-NEXT: ( 0, 1, 2, 4 ) - // CHECK-NEXT: ( 0, 2, 0, 2 ) - // CHECK-NEXT: ( 1, 2, 3, 4 ) + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 4 + // CHECK-NEXT: dim = ( 4, 3 ) + // CHECK-NEXT: lvl = ( 4, 3 ) + // CHECK-NEXT: pos[0] : ( 0, 3, + // CHECK-NEXT: crd[0] : ( 0, 2, 3, + // CHECK-NEXT: pos[1] : ( 0, 1, 2, 4, + // CHECK-NEXT: crd[1] : ( 0, 2, 0, 2, + // CHECK-NEXT: values : ( 1, 2, 3, 4, + // CHECK-NEXT: ---- // %dcsra = tensor.empty() : tensor<4x3xf64, #DCSR> %dcsr1 = sparse_tensor.insert %f1 into %dcsra[%c0, %c0] : tensor<4x3xf64, #DCSR> @@ -203,14 +140,19 @@ module { %dcsr3 = sparse_tensor.insert %f3 into %dcsr2[%c3, %c0] : tensor<4x3xf64, #DCSR> %dcsr4 = sparse_tensor.insert %f4 into %dcsr3[%c3, %c2] : tensor<4x3xf64, #DCSR> %dcsrm = sparse_tensor.load %dcsr4 hasInserts : tensor<4x3xf64, #DCSR> - call @dump_dcsr(%dcsrm) : (tensor<4x3xf64, #DCSR>) -> () + sparse_tensor.print %dcsrm : tensor<4x3xf64, #DCSR> // // Row case. // - // CHECK-NEXT: ( 0, 3 ) - // CHECK-NEXT: ( 0, 2, 3 ) - // CHECK-NEXT: ( 1, 0, 0, 0, 0, 2, 3, 0, 4 ) + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 4, 3 ) + // CHECK-NEXT: lvl = ( 4, 3 ) + // CHECK-NEXT: pos[0] : ( 0, 3, + // CHECK-NEXT: crd[0] : ( 0, 2, 3, + // CHECK-NEXT: values : ( 1, 0, 0, 0, 0, 2, 3, 0, 4, + // CHECK-NEXT: ---- // %rowa = tensor.empty() : tensor<4x3xf64, #Row> %row1 = sparse_tensor.insert %f1 into %rowa[%c0, %c0] : tensor<4x3xf64, #Row> @@ -218,27 +160,7 @@ module { %row3 = sparse_tensor.insert %f3 into %row2[%c3, %c0] : tensor<4x3xf64, #Row> %row4 = sparse_tensor.insert %f4 into %row3[%c3, %c2] : tensor<4x3xf64, #Row> %rowm = sparse_tensor.load %row4 hasInserts : tensor<4x3xf64, #Row> - call @dump_row(%rowm) : (tensor<4x3xf64, #Row>) -> () - - // - // NOE sanity check. - // - // CHECK-NEXT: 12 - // CHECK-NEXT: 4 - // CHECK-NEXT: 4 - // CHECK-NEXT: 4 - // CHECK-NEXT: 9 - // - %noe1 = sparse_tensor.number_of_entries %densem : tensor<4x3xf64, #Dense> - %noe2 = sparse_tensor.number_of_entries %coom : tensor<4x3xf64, #SortedCOO> - %noe3 = sparse_tensor.number_of_entries %csrm : tensor<4x3xf64, #CSR> - %noe4 = sparse_tensor.number_of_entries %dcsrm : tensor<4x3xf64, #DCSR> - %noe5 = sparse_tensor.number_of_entries %rowm : tensor<4x3xf64, #Row> - vector.print %noe1 : index - vector.print %noe2 : index - vector.print %noe3 : index - vector.print %noe4 : index - vector.print %noe5 : index + sparse_tensor.print %rowm : tensor<4x3xf64, #Row> // Release resources. bufferization.dealloc_tensor %densem : tensor<4x3xf64, #Dense> -- GitLab From ed6275868bd38ef87dac24ecec7a6f55129bd96d Mon Sep 17 00:00:00 2001 From: "Felix (Ting Wang)" Date: Tue, 5 Mar 2024 14:07:47 +0800 Subject: [PATCH 092/929] [PowerPC][NFC] Update aix-tls-xcoff-reloc.ll (#83764) Update test case changed by #66316 --- llvm/test/CodeGen/PowerPC/aix-tls-xcoff-reloc.ll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/test/CodeGen/PowerPC/aix-tls-xcoff-reloc.ll b/llvm/test/CodeGen/PowerPC/aix-tls-xcoff-reloc.ll index 0a3e7637b2e7..c17b038a6960 100644 --- a/llvm/test/CodeGen/PowerPC/aix-tls-xcoff-reloc.ll +++ b/llvm/test/CodeGen/PowerPC/aix-tls-xcoff-reloc.ll @@ -208,7 +208,7 @@ entry: ; SYM-NEXT: StorageClass: C_EXT (0x2) ; SYM-NEXT: NumberOfAuxEntries: 1 ; SYM-NEXT: CSECT Auxiliary Entry { -; SYM-NEXT: Index: 6 +; SYM-NEXT: Index: [[#NFA+4]] ; SYM-NEXT: SectionLen: 0 ; SYM-NEXT: ParameterHashIndex: 0x0 ; SYM-NEXT: TypeChkSectNum: 0x0 -- GitLab From 0c4736338596d6e527e286b7b551af4bb8b63a55 Mon Sep 17 00:00:00 2001 From: Yingwei Zheng Date: Tue, 5 Mar 2024 14:11:37 +0800 Subject: [PATCH 093/929] [InstCombine] Simplify nested selects with implied condition (#83739) This patch does the following simplification: ``` sel1 = select cond1, X, Y sel2 = select cond2, sel1, Z --> sel2 = select cond2, X, Z if cond2 implies cond1 sel2 = select cond2, Y, Z if cond2 implies !cond1 ``` Alive2: https://alive2.llvm.org/ce/z/9A_arU It cannot be done in CVP/SCCP since we should guarantee that `cond2` is not an undef. --- .../InstCombine/InstCombineSelect.cpp | 84 +++++++---------- ...etween-negative-and-positive-thresholds.ll | 12 +-- .../Transforms/InstCombine/nested-select.ll | 91 +++++++++++++++++++ 3 files changed, 128 insertions(+), 59 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp index 71fa9b9ba41e..c47bc33df070 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp @@ -2643,46 +2643,33 @@ static Value *foldSelectWithFrozenICmp(SelectInst &Sel, InstCombiner::BuilderTy return nullptr; } +/// Given that \p CondVal is known to be \p CondIsTrue, try to simplify \p SI. +static Value *simplifyNestedSelectsUsingImpliedCond(SelectInst &SI, + Value *CondVal, + bool CondIsTrue, + const DataLayout &DL) { + Value *InnerCondVal = SI.getCondition(); + Value *InnerTrueVal = SI.getTrueValue(); + Value *InnerFalseVal = SI.getFalseValue(); + assert(CondVal->getType() == InnerCondVal->getType() && + "The type of inner condition must match with the outer."); + if (auto Implied = isImpliedCondition(CondVal, InnerCondVal, DL, CondIsTrue)) + return *Implied ? InnerTrueVal : InnerFalseVal; + return nullptr; +} + Instruction *InstCombinerImpl::foldAndOrOfSelectUsingImpliedCond(Value *Op, SelectInst &SI, bool IsAnd) { - Value *CondVal = SI.getCondition(); - Value *A = SI.getTrueValue(); - Value *B = SI.getFalseValue(); - assert(Op->getType()->isIntOrIntVectorTy(1) && "Op must be either i1 or vector of i1."); - - std::optional Res = isImpliedCondition(Op, CondVal, DL, IsAnd); - if (!Res) + if (SI.getCondition()->getType() != Op->getType()) return nullptr; - - Value *Zero = Constant::getNullValue(A->getType()); - Value *One = Constant::getAllOnesValue(A->getType()); - - if (*Res == true) { - if (IsAnd) - // select op, (select cond, A, B), false => select op, A, false - // and op, (select cond, A, B) => select op, A, false - // if op = true implies condval = true. - return SelectInst::Create(Op, A, Zero); - else - // select op, true, (select cond, A, B) => select op, true, A - // or op, (select cond, A, B) => select op, true, A - // if op = false implies condval = true. - return SelectInst::Create(Op, One, A); - } else { - if (IsAnd) - // select op, (select cond, A, B), false => select op, B, false - // and op, (select cond, A, B) => select op, B, false - // if op = true implies condval = false. - return SelectInst::Create(Op, B, Zero); - else - // select op, true, (select cond, A, B) => select op, true, B - // or op, (select cond, A, B) => select op, true, B - // if op = false implies condval = false. - return SelectInst::Create(Op, One, B); - } + if (Value *V = simplifyNestedSelectsUsingImpliedCond(SI, Op, IsAnd, DL)) + return SelectInst::Create(Op, + IsAnd ? V : ConstantInt::getTrue(Op->getType()), + IsAnd ? ConstantInt::getFalse(Op->getType()) : V); + return nullptr; } // Canonicalize select with fcmp to fabs(). -0.0 makes this tricky. We need @@ -3138,11 +3125,6 @@ Instruction *InstCombinerImpl::foldSelectOfBools(SelectInst &SI) { return replaceInstUsesWith(SI, Op1); } - if (auto *Op1SI = dyn_cast(Op1)) - if (auto *I = foldAndOrOfSelectUsingImpliedCond(CondVal, *Op1SI, - /* IsAnd */ IsAnd)) - return I; - if (auto *ICmp0 = dyn_cast(CondVal)) if (auto *ICmp1 = dyn_cast(Op1)) if (auto *V = foldAndOrOfICmps(ICmp0, ICmp1, SI, IsAnd, @@ -3643,12 +3625,12 @@ Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) { if (SelectInst *TrueSI = dyn_cast(TrueVal)) { if (TrueSI->getCondition()->getType() == CondVal->getType()) { - // select(C, select(C, a, b), c) -> select(C, a, c) - if (TrueSI->getCondition() == CondVal) { - if (SI.getTrueValue() == TrueSI->getTrueValue()) - return nullptr; - return replaceOperand(SI, 1, TrueSI->getTrueValue()); - } + // Fold nested selects if the inner condition can be implied by the outer + // condition. + if (Value *V = simplifyNestedSelectsUsingImpliedCond( + *TrueSI, CondVal, /*CondIsTrue=*/true, DL)) + return replaceOperand(SI, 1, V); + // select(C0, select(C1, a, b), b) -> select(C0&C1, a, b) // We choose this as normal form to enable folding on the And and // shortening paths for the values (this helps getUnderlyingObjects() for @@ -3663,12 +3645,12 @@ Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) { } if (SelectInst *FalseSI = dyn_cast(FalseVal)) { if (FalseSI->getCondition()->getType() == CondVal->getType()) { - // select(C, a, select(C, b, c)) -> select(C, a, c) - if (FalseSI->getCondition() == CondVal) { - if (SI.getFalseValue() == FalseSI->getFalseValue()) - return nullptr; - return replaceOperand(SI, 2, FalseSI->getFalseValue()); - } + // Fold nested selects if the inner condition can be implied by the outer + // condition. + if (Value *V = simplifyNestedSelectsUsingImpliedCond( + *FalseSI, CondVal, /*CondIsTrue=*/false, DL)) + return replaceOperand(SI, 2, V); + // select(C0, a, select(C1, a, b)) -> select(C0|C1, a, b) if (FalseSI->getTrueValue() == TrueVal && FalseSI->hasOneUse()) { Value *Or = Builder.CreateLogicalOr(CondVal, FalseSI->getCondition()); diff --git a/llvm/test/Transforms/InstCombine/canonicalize-clamp-like-pattern-between-negative-and-positive-thresholds.ll b/llvm/test/Transforms/InstCombine/canonicalize-clamp-like-pattern-between-negative-and-positive-thresholds.ll index d03e22bc4c9f..b5ef1f466958 100644 --- a/llvm/test/Transforms/InstCombine/canonicalize-clamp-like-pattern-between-negative-and-positive-thresholds.ll +++ b/llvm/test/Transforms/InstCombine/canonicalize-clamp-like-pattern-between-negative-and-positive-thresholds.ll @@ -189,10 +189,8 @@ define i32 @n9_ult_slt_neg17(i32 %x, i32 %replacement_low, i32 %replacement_high ; Regression test for PR53252. define i32 @n10_ugt_slt(i32 %x, i32 %replacement_low, i32 %replacement_high) { ; CHECK-LABEL: @n10_ugt_slt( -; CHECK-NEXT: [[T0:%.*]] = icmp slt i32 [[X:%.*]], 0 -; CHECK-NEXT: [[T1:%.*]] = select i1 [[T0]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[REPLACEMENT_HIGH:%.*]] -; CHECK-NEXT: [[T2:%.*]] = icmp ugt i32 [[X]], 128 -; CHECK-NEXT: [[R:%.*]] = select i1 [[T2]], i32 [[X]], i32 [[T1]] +; CHECK-NEXT: [[T2:%.*]] = icmp ugt i32 [[X:%.*]], 128 +; CHECK-NEXT: [[R:%.*]] = select i1 [[T2]], i32 [[X]], i32 [[REPLACEMENT_HIGH:%.*]] ; CHECK-NEXT: ret i32 [[R]] ; %t0 = icmp slt i32 %x, 0 @@ -204,10 +202,8 @@ define i32 @n10_ugt_slt(i32 %x, i32 %replacement_low, i32 %replacement_high) { define i32 @n11_uge_slt(i32 %x, i32 %replacement_low, i32 %replacement_high) { ; CHECK-LABEL: @n11_uge_slt( -; CHECK-NEXT: [[T0:%.*]] = icmp slt i32 [[X:%.*]], 0 -; CHECK-NEXT: [[T1:%.*]] = select i1 [[T0]], i32 [[REPLACEMENT_LOW:%.*]], i32 [[REPLACEMENT_HIGH:%.*]] -; CHECK-NEXT: [[T2:%.*]] = icmp ult i32 [[X]], 129 -; CHECK-NEXT: [[R:%.*]] = select i1 [[T2]], i32 [[T1]], i32 [[X]] +; CHECK-NEXT: [[T2:%.*]] = icmp ult i32 [[X:%.*]], 129 +; CHECK-NEXT: [[R:%.*]] = select i1 [[T2]], i32 [[REPLACEMENT_HIGH:%.*]], i32 [[X]] ; CHECK-NEXT: ret i32 [[R]] ; %t0 = icmp slt i32 %x, 0 diff --git a/llvm/test/Transforms/InstCombine/nested-select.ll b/llvm/test/Transforms/InstCombine/nested-select.ll index 42a0f81e7b85..d01dcf0793ad 100644 --- a/llvm/test/Transforms/InstCombine/nested-select.ll +++ b/llvm/test/Transforms/InstCombine/nested-select.ll @@ -498,3 +498,94 @@ define i1 @orcond.111.inv.all.conds(i1 %inner.cond, i1 %alt.cond, i1 %inner.sel. %outer.sel = select i1 %not.outer.cond, i1 true, i1 %inner.sel ret i1 %outer.sel } + +define i8 @test_implied_true(i8 %x) { +; CHECK-LABEL: @test_implied_true( +; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i8 [[X:%.*]], 0 +; CHECK-NEXT: [[SEL2:%.*]] = select i1 [[CMP2]], i8 0, i8 20 +; CHECK-NEXT: ret i8 [[SEL2]] +; + %cmp1 = icmp slt i8 %x, 10 + %cmp2 = icmp slt i8 %x, 0 + %sel1 = select i1 %cmp1, i8 0, i8 5 + %sel2 = select i1 %cmp2, i8 %sel1, i8 20 + ret i8 %sel2 +} + +define <2 x i8> @test_implied_true_vec(<2 x i8> %x) { +; CHECK-LABEL: @test_implied_true_vec( +; CHECK-NEXT: [[CMP2:%.*]] = icmp slt <2 x i8> [[X:%.*]], zeroinitializer +; CHECK-NEXT: [[SEL2:%.*]] = select <2 x i1> [[CMP2]], <2 x i8> zeroinitializer, <2 x i8> +; CHECK-NEXT: ret <2 x i8> [[SEL2]] +; + %cmp1 = icmp slt <2 x i8> %x, + %cmp2 = icmp slt <2 x i8> %x, zeroinitializer + %sel1 = select <2 x i1> %cmp1, <2 x i8> zeroinitializer, <2 x i8> + %sel2 = select <2 x i1> %cmp2, <2 x i8> %sel1, <2 x i8> + ret <2 x i8> %sel2 +} + +define i8 @test_implied_true_falseval(i8 %x) { +; CHECK-LABEL: @test_implied_true_falseval( +; CHECK-NEXT: [[CMP2:%.*]] = icmp sgt i8 [[X:%.*]], 0 +; CHECK-NEXT: [[SEL2:%.*]] = select i1 [[CMP2]], i8 20, i8 0 +; CHECK-NEXT: ret i8 [[SEL2]] +; + %cmp1 = icmp slt i8 %x, 10 + %cmp2 = icmp sgt i8 %x, 0 + %sel1 = select i1 %cmp1, i8 0, i8 5 + %sel2 = select i1 %cmp2, i8 20, i8 %sel1 + ret i8 %sel2 +} + +define i8 @test_implied_false(i8 %x) { +; CHECK-LABEL: @test_implied_false( +; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i8 [[X:%.*]], 0 +; CHECK-NEXT: [[SEL2:%.*]] = select i1 [[CMP2]], i8 5, i8 20 +; CHECK-NEXT: ret i8 [[SEL2]] +; + %cmp1 = icmp sgt i8 %x, 10 + %cmp2 = icmp slt i8 %x, 0 + %sel1 = select i1 %cmp1, i8 0, i8 5 + %sel2 = select i1 %cmp2, i8 %sel1, i8 20 + ret i8 %sel2 +} + +; Negative tests + +define i8 @test_imply_fail(i8 %x) { +; CHECK-LABEL: @test_imply_fail( +; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i8 [[X:%.*]], -10 +; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i8 [[X]], 0 +; CHECK-NEXT: [[SEL1:%.*]] = select i1 [[CMP1]], i8 0, i8 5 +; CHECK-NEXT: [[SEL2:%.*]] = select i1 [[CMP2]], i8 [[SEL1]], i8 20 +; CHECK-NEXT: ret i8 [[SEL2]] +; + %cmp1 = icmp slt i8 %x, -10 + %cmp2 = icmp slt i8 %x, 0 + %sel1 = select i1 %cmp1, i8 0, i8 5 + %sel2 = select i1 %cmp2, i8 %sel1, i8 20 + ret i8 %sel2 +} + +define <2 x i8> @test_imply_type_mismatch(<2 x i8> %x, i8 %y) { +; CHECK-LABEL: @test_imply_type_mismatch( +; CHECK-NEXT: [[CMP1:%.*]] = icmp slt <2 x i8> [[X:%.*]], +; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i8 [[Y:%.*]], 0 +; CHECK-NEXT: [[SEL1:%.*]] = select <2 x i1> [[CMP1]], <2 x i8> zeroinitializer, <2 x i8> +; CHECK-NEXT: [[SEL2:%.*]] = select i1 [[CMP2]], <2 x i8> [[SEL1]], <2 x i8> +; CHECK-NEXT: ret <2 x i8> [[SEL2]] +; + %cmp1 = icmp slt <2 x i8> %x, + %cmp2 = icmp slt i8 %y, 0 + %sel1 = select <2 x i1> %cmp1, <2 x i8> zeroinitializer, <2 x i8> + %sel2 = select i1 %cmp2, <2 x i8> %sel1, <2 x i8> + ret <2 x i8> %sel2 +} + +define <4 x i1> @test_dont_crash(i1 %cond, <4 x i1> %a, <4 x i1> %b) { +entry: + %sel = select i1 %cond, <4 x i1> %a, <4 x i1> zeroinitializer + %and = and <4 x i1> %sel, %b + ret <4 x i1> %and +} -- GitLab From 3cb999c413b7e934828936ae2314b50a84205e89 Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Tue, 5 Mar 2024 14:26:54 +0800 Subject: [PATCH 094/929] [RISCV] Remove note of Sscofpmf and add a newline after FeatureStdExtSscofpmf (#83958) To address comments after committing #83831. --- llvm/docs/RISCVUsage.rst | 4 ++-- llvm/lib/Target/RISCV/RISCVFeatures.td | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst index 2595af598f1e..582b4a5b830c 100644 --- a/llvm/docs/RISCVUsage.rst +++ b/llvm/docs/RISCVUsage.rst @@ -101,7 +101,7 @@ on support follow. ``Smepmp`` Supported ``Ssaia`` Supported ``Ssccptr`` Assembly Support (`See note <#riscv-profiles-extensions-note>`__) - ``Sscofpmf`` Assembly Support (`See note <#riscv-profiles-extensions-note>`__) + ``Sscofpmf`` Assembly Support ``Sscounterenw`` Assembly Support (`See note <#riscv-profiles-extensions-note>`__) ``Ssstateen`` Assembly Support (`See note <#riscv-profiles-extensions-note>`__) ``Ssstrict`` Assembly Support (`See note <#riscv-profiles-extensions-note>`__) @@ -235,7 +235,7 @@ Supported .. _riscv-profiles-extensions-note: -``Za128rs``, ``Za64rs``, ``Zic64b``, ``Ziccamoa``, ``Ziccif``, ``Zicclsm``, ``Ziccrse``, ``Shcounterenvw``, ``Shgatpa``, ``Shtvala``, ``Shvsatpa``, ``Shvstvala``, ``Shvstvecd``, ``Ssccptr``, ``Sscofpmf``, ``Sscounterenw``, ``Ssstateen``, ``Ssstrict``, ``Sstvala``, ``Sstvecd``, ``Ssu64xl``, ``Svade``, ``Svbare`` +``Za128rs``, ``Za64rs``, ``Zic64b``, ``Ziccamoa``, ``Ziccif``, ``Zicclsm``, ``Ziccrse``, ``Shcounterenvw``, ``Shgatpa``, ``Shtvala``, ``Shvsatpa``, ``Shvstvala``, ``Shvstvecd``, ``Ssccptr``, ``Sscounterenw``, ``Ssstateen``, ``Ssstrict``, ``Sstvala``, ``Sstvecd``, ``Ssu64xl``, ``Svade``, ``Svbare`` These extensions are defined as part of the `RISC-V Profiles specification `__. They do not introduce any new features themselves, but instead describe existing hardware features. .. _riscv-zacas-note: diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td index 78f3a82d66df..83619ccb24ba 100644 --- a/llvm/lib/Target/RISCV/RISCVFeatures.td +++ b/llvm/lib/Target/RISCV/RISCVFeatures.td @@ -810,6 +810,7 @@ def FeatureStdExtSsccptr def FeatureStdExtSscofpmf : SubtargetFeature<"sscofpmf", "HasStdExtSscofpmf", "true", "'Sscofpmf' (Count Overflow and Mode-Based Filtering)", []>; + def FeatureStdExtShcounterenw : SubtargetFeature<"shcounterenw", "HasStdExtShcounterenw", "true", "'Shcounterenw' (Support writeable hcounteren enable " -- GitLab From 80f9458cf30d13eef21b09042ea590945c5e64db Mon Sep 17 00:00:00 2001 From: Hristo Hristov Date: Tue, 5 Mar 2024 08:49:49 +0200 Subject: [PATCH 095/929] [libc++][sstream] P2495R3: Interfacing `stringstream`s with `string_view` (#80552) Implements P2495R3 - https://eel.is/c++draft/version.syn#headerref:%3csstream%3e - https://eel.is/c++draft/stringbuf - https://eel.is/c++draft/stringbuf.general - https://eel.is/c++draft/stringbuf.cons - https://eel.is/c++draft/stringbuf.members - https://eel.is/c++draft/istringstream - https://eel.is/c++draft/istringstream.general - https://eel.is/c++draft/istringstream.cons - https://eel.is/c++draft/istringstream.members - https://eel.is/c++draft/ostringstream - https://eel.is/c++draft/ostringstream.general - https://eel.is/c++draft/ostringstream.cons - https://eel.is/c++draft/ostringstream.members - https://eel.is/c++draft/stringstream - https://eel.is/c++draft/stringstream.general - https://eel.is/c++draft/stringstream.cons - https://eel.is/c++draft/stringstream.members References: - https://eel.is/c++draft/string.streams --- libcxx/docs/FeatureTestMacroTable.rst | 2 +- libcxx/docs/ReleaseNotes/19.rst | 1 + libcxx/docs/Status/Cxx2cPapers.csv | 2 +- libcxx/include/sstream | 154 ++++++++++++++ libcxx/include/version | 2 +- .../test/libcxx/transitive_includes/cxx03.csv | 1 + .../test/libcxx/transitive_includes/cxx11.csv | 1 + .../test/libcxx/transitive_includes/cxx14.csv | 1 + .../test/libcxx/transitive_includes/cxx17.csv | 1 + .../test/libcxx/transitive_includes/cxx20.csv | 1 + .../test/libcxx/transitive_includes/cxx23.csv | 1 + .../test/libcxx/transitive_includes/cxx26.csv | 1 + .../string.streams/helper_concepts.h | 17 ++ .../string.streams/helper_string_macros.h | 18 ++ .../string.streams/helper_types.h | 75 +++++++ .../string_view.alloc.pass.cpp | 145 ++++++++++++++ .../string_view.mode.alloc.pass.cpp | 189 ++++++++++++++++++ .../string_view.mode.pass.cpp | 150 ++++++++++++++ .../str.string_view.pass.cpp | 110 ++++++++++ .../string_view.alloc.pass.cpp | 145 ++++++++++++++ .../string_view.mode.alloc.pass.cpp | 189 ++++++++++++++++++ .../string_view.mode.pass.cpp | 150 ++++++++++++++ .../str.string_view.pass.cpp | 110 ++++++++++ .../stringbuf.cons/string_view.alloc.pass.cpp | 145 ++++++++++++++ .../string_view.mode.alloc.pass.cpp | 184 +++++++++++++++++ .../stringbuf.cons/string_view.mode.pass.cpp | 150 ++++++++++++++ .../str.string_view.pass.cpp | 110 ++++++++++ .../string_view.alloc.pass.cpp | 145 ++++++++++++++ .../string_view.mode.alloc.pass.cpp | 189 ++++++++++++++++++ .../string_view.mode.pass.cpp | 150 ++++++++++++++ .../str.string_view.pass.cpp | 110 ++++++++++ .../sstream.version.compile.pass.cpp | 16 +- .../version.version.compile.pass.cpp | 16 +- .../generate_feature_test_macro_components.py | 1 - 34 files changed, 2656 insertions(+), 26 deletions(-) create mode 100644 libcxx/test/std/input.output/string.streams/helper_concepts.h create mode 100644 libcxx/test/std/input.output/string.streams/helper_string_macros.h create mode 100644 libcxx/test/std/input.output/string.streams/helper_types.h create mode 100644 libcxx/test/std/input.output/string.streams/istringstream/istringstream.cons/string_view.alloc.pass.cpp create mode 100644 libcxx/test/std/input.output/string.streams/istringstream/istringstream.cons/string_view.mode.alloc.pass.cpp create mode 100644 libcxx/test/std/input.output/string.streams/istringstream/istringstream.cons/string_view.mode.pass.cpp create mode 100644 libcxx/test/std/input.output/string.streams/istringstream/istringstream.members/str.string_view.pass.cpp create mode 100644 libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.cons/string_view.alloc.pass.cpp create mode 100644 libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.cons/string_view.mode.alloc.pass.cpp create mode 100644 libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.cons/string_view.mode.pass.cpp create mode 100644 libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.members/str.string_view.pass.cpp create mode 100644 libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.cons/string_view.alloc.pass.cpp create mode 100644 libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.cons/string_view.mode.alloc.pass.cpp create mode 100644 libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.cons/string_view.mode.pass.cpp create mode 100644 libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.members/str.string_view.pass.cpp create mode 100644 libcxx/test/std/input.output/string.streams/stringstream/stringstream.cons/string_view.alloc.pass.cpp create mode 100644 libcxx/test/std/input.output/string.streams/stringstream/stringstream.cons/string_view.mode.alloc.pass.cpp create mode 100644 libcxx/test/std/input.output/string.streams/stringstream/stringstream.cons/string_view.mode.pass.cpp create mode 100644 libcxx/test/std/input.output/string.streams/stringstream/stringstream.members/str.string_view.pass.cpp diff --git a/libcxx/docs/FeatureTestMacroTable.rst b/libcxx/docs/FeatureTestMacroTable.rst index 60e0aea9768b..b213f430aa59 100644 --- a/libcxx/docs/FeatureTestMacroTable.rst +++ b/libcxx/docs/FeatureTestMacroTable.rst @@ -442,7 +442,7 @@ Status --------------------------------------------------- ----------------- ``__cpp_lib_span_initializer_list`` ``202311L`` --------------------------------------------------- ----------------- - ``__cpp_lib_sstream_from_string_view`` *unimplemented* + ``__cpp_lib_sstream_from_string_view`` ``202306L`` --------------------------------------------------- ----------------- ``__cpp_lib_submdspan`` *unimplemented* --------------------------------------------------- ----------------- diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst index 0d381df5f044..b068325b35c9 100644 --- a/libcxx/docs/ReleaseNotes/19.rst +++ b/libcxx/docs/ReleaseNotes/19.rst @@ -41,6 +41,7 @@ Implemented Papers - P2637R3 - Member ``visit`` - P2652R2 - Disallow User Specialization of ``allocator_traits`` - P2819R2 - Add ``tuple`` protocol to ``complex`` +- P2495R3 - Interfacing ``stringstream``s with ``string_view`` - P2302R4 - ``std::ranges::contains`` - P1659R3 - ``std::ranges::starts_with`` and ``std::ranges::ends_with`` diff --git a/libcxx/docs/Status/Cxx2cPapers.csv b/libcxx/docs/Status/Cxx2cPapers.csv index 1c895f79a4c0..febb0977255b 100644 --- a/libcxx/docs/Status/Cxx2cPapers.csv +++ b/libcxx/docs/Status/Cxx2cPapers.csv @@ -6,7 +6,7 @@ "`P2545R4 `__","LWG","Read-Copy Update (RCU)","Varna June 2023","","","" "`P2530R3 `__","LWG","Hazard Pointers for C++26","Varna June 2023","","","" "`P2538R1 `__","LWG","ADL-proof ``std::projected``","Varna June 2023","|Complete|","18.0","|ranges|" -"`P2495R3 `__","LWG","Interfacing ``stringstreams`` with ``string_view``","Varna June 2023","","","" +"`P2495R3 `__","LWG","Interfacing ``stringstream``s with ``string_view``","Varna June 2023","|Complete|","19.0","" "`P2510R3 `__","LWG","Formatting pointers","Varna June 2023","|Complete| [#note-P2510R3]_","17.0","|format|" "`P2198R7 `__","LWG","Freestanding Feature-Test Macros and Implementation-Defined Extensions","Varna June 2023","","","" "`P2338R4 `__","LWG","Freestanding Library: Character primitives and the C library","Varna June 2023","","","" diff --git a/libcxx/include/sstream b/libcxx/include/sstream index 60bec52209d7..5873deb8318e 100644 --- a/libcxx/include/sstream +++ b/libcxx/include/sstream @@ -48,6 +48,13 @@ public: template explicit basic_stringbuf(const basic_string& s, ios_base::openmode which = ios_base::in | ios_base::out); // C++20 + template + explicit basic_stringbuf(const T& t, + ios_base::openmode which = ios_base::in | ios_base::out); // Since C++26 + template + basic_stringbuf(const T& t, const Allocator& a); // Since C++26 + template + basic_stringbuf(const T& t, ios_base::openmode which, const Allocator& a); // Since C++26 basic_stringbuf(const basic_stringbuf&) = delete; basic_stringbuf(basic_stringbuf&& rhs); basic_stringbuf(basic_stringbuf&& rhs, const allocator_type& a); // C++20 @@ -69,6 +76,8 @@ public: template void str(const basic_string& s); // C++20 void str(basic_string&& s); // C++20 + template + void str(const T& t); // Since C++26 protected: // [stringbuf.virtuals] Overridden virtual functions: @@ -121,6 +130,12 @@ public: template explicit basic_istringstream(const basic_string& s, ios_base::openmode which = ios_base::in); // C++20 + template + explicit basic_istringstream(const T& t, ios_base::openmode which = ios_base::in); // Since C++26 + template + basic_istringstream(const T& t, const Allocator& a); // Since C++26 + template + basic_istringstream(const T& t, ios_base::openmode which, const Allocator& a); // Since C++26 basic_istringstream(const basic_istringstream&) = delete; basic_istringstream(basic_istringstream&& rhs); @@ -141,6 +156,8 @@ public: template void str(const basic_string& s); // C++20 void str(basic_string&& s); // C++20 + template + void str(const T& t); // Since C++26 }; template @@ -182,6 +199,12 @@ public: template explicit basic_ostringstream(const basic_string& s, ios_base::openmode which = ios_base::out); // C++20 + template + explicit basic_ostringstream(const T& t, ios_base::openmode which = ios_base::out); // Since C++26 + template + basic_ostringstream(const T& t, const Allocator& a); // Since C++26 + template + basic_ostringstream(const T& t, ios_base::openmode which, const Allocator& a); // Since C++26 basic_ostringstream(const basic_ostringstream&) = delete; basic_ostringstream(basic_ostringstream&& rhs); @@ -202,6 +225,8 @@ public: template void str(const basic_string& s); // C++20 void str(basic_string&& s); // C++20 + template + void str(const T& t); // Since C++26 }; template @@ -243,6 +268,13 @@ public: template explicit basic_stringstream(const basic_string& s, ios_base::openmode which = ios_base::out | ios_base::in); // C++20 + template + explicit basic_stringstream(const T& t, + ios_base::openmode which = ios_base::out | ios_base::in); // Since C++26 + template + basic_stringstream(const T& t, const Allocator& a); // Since C++26 + template + basic_stringstream(const T& t, ios_base::openmode which, const Allocator& a); // Since C++26 basic_stringstream(const basic_stringstream&) = delete; basic_stringstream(basic_stringstream&& rhs); @@ -263,6 +295,8 @@ public: template void str(const basic_string& s); // C++20 void str(basic_string&& s); // C++20 + template + void str(const T& t); // Since C++26 }; template @@ -281,10 +315,12 @@ typedef basic_stringstream wstringstream; #include <__availability> #include <__config> #include <__fwd/sstream.h> +#include <__type_traits/is_convertible.h> #include <__utility/swap.h> #include #include #include +#include #include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) @@ -371,6 +407,30 @@ public: } #endif // _LIBCPP_STD_VER >= 20 +#if _LIBCPP_STD_VER >= 26 + + template + requires is_convertible_v> + _LIBCPP_HIDE_FROM_ABI explicit basic_stringbuf(const _Tp& __t, + ios_base::openmode __which = ios_base::in | ios_base::out) + : basic_stringbuf(__t, __which, _Allocator()) {} + + template + requires is_convertible_v> + _LIBCPP_HIDE_FROM_ABI basic_stringbuf(const _Tp& __t, const _Allocator& __a) + : basic_stringbuf(__t, ios_base::in | ios_base::out, __a) {} + + template + requires is_convertible_v> + _LIBCPP_HIDE_FROM_ABI basic_stringbuf(const _Tp& __t, ios_base::openmode __which, const _Allocator& __a) + : __hm_(nullptr), __mode_(__which) { + basic_string_view<_CharT, _Traits> __sv = __t; + __str_ = string_type(__sv, __a); + __init_buf_ptrs(); + } + +#endif // _LIBCPP_STD_VER >= 26 + basic_stringbuf(const basic_stringbuf&) = delete; basic_stringbuf(basic_stringbuf&& __rhs) : __mode_(__rhs.__mode_) { __move_init(std::move(__rhs)); } @@ -444,6 +504,18 @@ public: } #endif // _LIBCPP_STD_VER >= 20 +#if _LIBCPP_STD_VER >= 26 + + template + requires is_convertible_v> + _LIBCPP_HIDE_FROM_ABI void str(const _Tp& __t) { + basic_string_view<_CharT, _Traits> __sv = __t; + __str_ = __sv; + __init_buf_ptrs(); + } + +#endif // _LIBCPP_STD_VER >= 26 + protected: // [stringbuf.virtuals] Overridden virtual functions: int_type underflow() override; @@ -831,6 +903,25 @@ public: : basic_istream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__s, __wch | ios_base::in) {} #endif // _LIBCPP_STD_VER >= 20 +#if _LIBCPP_STD_VER >= 26 + + template + requires is_convertible_v> + _LIBCPP_HIDE_FROM_ABI explicit basic_istringstream(const _Tp& __t, ios_base::openmode __which = ios_base::in) + : basic_istringstream(__t, __which, _Allocator()) {} + + template + requires is_convertible_v> + _LIBCPP_HIDE_FROM_ABI basic_istringstream(const _Tp& __t, const _Allocator& __a) + : basic_istringstream(__t, ios_base::in, __a) {} + + template + requires is_convertible_v> + _LIBCPP_HIDE_FROM_ABI basic_istringstream(const _Tp& __t, ios_base::openmode __which, const _Allocator& __a) + : basic_istream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__t, __which | ios_base::in, __a) {} + +#endif // _LIBCPP_STD_VER >= 26 + basic_istringstream(const basic_istringstream&) = delete; _LIBCPP_HIDE_FROM_ABI basic_istringstream(basic_istringstream&& __rhs) : basic_istream<_CharT, _Traits>(std::move(__rhs)), __sb_(std::move(__rhs.__sb_)) { @@ -882,6 +973,14 @@ public: _LIBCPP_HIDE_FROM_ABI void str(string_type&& __s) { __sb_.str(std::move(__s)); } #endif // _LIBCPP_STD_VER >= 20 + +#if _LIBCPP_STD_VER >= 26 + template + requires is_convertible_v> + _LIBCPP_HIDE_FROM_ABI void str(const _Tp& __t) { + rdbuf()->str(__t); + } +#endif // _LIBCPP_STD_VER >= 26 }; template @@ -940,6 +1039,25 @@ public: : basic_ostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__s, __wch | ios_base::out) {} #endif // _LIBCPP_STD_VER >= 20 +#if _LIBCPP_STD_VER >= 26 + + template + requires is_convertible_v> + _LIBCPP_HIDE_FROM_ABI explicit basic_ostringstream(const _Tp& __t, ios_base::openmode __which = ios_base::out) + : basic_ostringstream(__t, __which | ios_base::out, _Allocator()) {} + + template + requires is_convertible_v> + _LIBCPP_HIDE_FROM_ABI basic_ostringstream(const _Tp& __t, const _Allocator& __a) + : basic_ostringstream(__t, ios_base::out, __a) {} + + template + requires is_convertible_v> + _LIBCPP_HIDE_FROM_ABI basic_ostringstream(const _Tp& __t, ios_base::openmode __which, const _Allocator& __a) + : basic_ostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__t, __which | ios_base::out, __a) {} + +#endif // _LIBCPP_STD_VER >= 26 + basic_ostringstream(const basic_ostringstream&) = delete; _LIBCPP_HIDE_FROM_ABI basic_ostringstream(basic_ostringstream&& __rhs) : basic_ostream<_CharT, _Traits>(std::move(__rhs)), __sb_(std::move(__rhs.__sb_)) { @@ -992,6 +1110,14 @@ public: _LIBCPP_HIDE_FROM_ABI void str(string_type&& __s) { __sb_.str(std::move(__s)); } #endif // _LIBCPP_STD_VER >= 20 + +#if _LIBCPP_STD_VER >= 26 + template + requires is_convertible_v> + _LIBCPP_HIDE_FROM_ABI void str(const _Tp& __t) { + rdbuf()->str(__t); + } +#endif // _LIBCPP_STD_VER >= 26 }; template @@ -1053,6 +1179,26 @@ public: : basic_iostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__s, __wch) {} #endif // _LIBCPP_STD_VER >= 20 +#if _LIBCPP_STD_VER >= 26 + + template + requires is_convertible_v> + _LIBCPP_HIDE_FROM_ABI explicit basic_stringstream(const _Tp& __t, + ios_base::openmode __which = ios_base::out | ios_base::in) + : basic_stringstream(__t, __which, _Allocator()) {} + + template + requires is_convertible_v> + _LIBCPP_HIDE_FROM_ABI basic_stringstream(const _Tp& __t, const _Allocator& __a) + : basic_stringstream(__t, ios_base::out | ios_base::in, __a) {} + + template + requires is_convertible_v> + _LIBCPP_HIDE_FROM_ABI basic_stringstream(const _Tp& __t, ios_base::openmode __which, const _Allocator& __a) + : basic_iostream<_CharT, _Traits>(std::addressof(__sb_)), __sb_(__t, __which, __a) {} + +#endif // _LIBCPP_STD_VER >= 26 + basic_stringstream(const basic_stringstream&) = delete; _LIBCPP_HIDE_FROM_ABI basic_stringstream(basic_stringstream&& __rhs) : basic_iostream<_CharT, _Traits>(std::move(__rhs)), __sb_(std::move(__rhs.__sb_)) { @@ -1104,6 +1250,14 @@ public: _LIBCPP_HIDE_FROM_ABI void str(string_type&& __s) { __sb_.str(std::move(__s)); } #endif // _LIBCPP_STD_VER >= 20 + +#if _LIBCPP_STD_VER >= 26 + template + requires is_convertible_v> + _LIBCPP_HIDE_FROM_ABI void str(const _Tp& __t) { + rdbuf()->str(__t); + } +#endif // _LIBCPP_STD_VER >= 26 }; template diff --git a/libcxx/include/version b/libcxx/include/version index 055d0f30f9c4..3bd296e34aa4 100644 --- a/libcxx/include/version +++ b/libcxx/include/version @@ -515,7 +515,7 @@ __cpp_lib_within_lifetime 202306L // # define __cpp_lib_smart_ptr_owner_equality 202306L # define __cpp_lib_span_at 202311L # define __cpp_lib_span_initializer_list 202311L -// # define __cpp_lib_sstream_from_string_view 202306L +# define __cpp_lib_sstream_from_string_view 202306L // # define __cpp_lib_submdspan 202306L // # define __cpp_lib_text_encoding 202306L # undef __cpp_lib_to_chars diff --git a/libcxx/test/libcxx/transitive_includes/cxx03.csv b/libcxx/test/libcxx/transitive_includes/cxx03.csv index 0acc48c80eb7..678a986e522a 100644 --- a/libcxx/test/libcxx/transitive_includes/cxx03.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx03.csv @@ -745,6 +745,7 @@ sstream cstddef sstream istream sstream ostream sstream string +sstream string_view sstream type_traits sstream version stack compare diff --git a/libcxx/test/libcxx/transitive_includes/cxx11.csv b/libcxx/test/libcxx/transitive_includes/cxx11.csv index d2acb1356d13..c3875fa2cfc0 100644 --- a/libcxx/test/libcxx/transitive_includes/cxx11.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx11.csv @@ -751,6 +751,7 @@ sstream cstddef sstream istream sstream ostream sstream string +sstream string_view sstream type_traits sstream version stack compare diff --git a/libcxx/test/libcxx/transitive_includes/cxx14.csv b/libcxx/test/libcxx/transitive_includes/cxx14.csv index ccba63f44cb2..e28e0cd44fed 100644 --- a/libcxx/test/libcxx/transitive_includes/cxx14.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx14.csv @@ -753,6 +753,7 @@ sstream cstddef sstream istream sstream ostream sstream string +sstream string_view sstream type_traits sstream version stack compare diff --git a/libcxx/test/libcxx/transitive_includes/cxx17.csv b/libcxx/test/libcxx/transitive_includes/cxx17.csv index ccba63f44cb2..e28e0cd44fed 100644 --- a/libcxx/test/libcxx/transitive_includes/cxx17.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx17.csv @@ -753,6 +753,7 @@ sstream cstddef sstream istream sstream ostream sstream string +sstream string_view sstream type_traits sstream version stack compare diff --git a/libcxx/test/libcxx/transitive_includes/cxx20.csv b/libcxx/test/libcxx/transitive_includes/cxx20.csv index aa9239bda21f..eec71f4fc628 100644 --- a/libcxx/test/libcxx/transitive_includes/cxx20.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx20.csv @@ -758,6 +758,7 @@ sstream cstddef sstream istream sstream ostream sstream string +sstream string_view sstream type_traits sstream version stack compare diff --git a/libcxx/test/libcxx/transitive_includes/cxx23.csv b/libcxx/test/libcxx/transitive_includes/cxx23.csv index ce11a358fc3e..64ff9261820a 100644 --- a/libcxx/test/libcxx/transitive_includes/cxx23.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx23.csv @@ -519,6 +519,7 @@ sstream cstddef sstream istream sstream ostream sstream string +sstream string_view sstream version stack compare stack cstddef diff --git a/libcxx/test/libcxx/transitive_includes/cxx26.csv b/libcxx/test/libcxx/transitive_includes/cxx26.csv index ce11a358fc3e..64ff9261820a 100644 --- a/libcxx/test/libcxx/transitive_includes/cxx26.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx26.csv @@ -519,6 +519,7 @@ sstream cstddef sstream istream sstream ostream sstream string +sstream string_view sstream version stack compare stack cstddef diff --git a/libcxx/test/std/input.output/string.streams/helper_concepts.h b/libcxx/test/std/input.output/string.streams/helper_concepts.h new file mode 100644 index 000000000000..18f6e5474164 --- /dev/null +++ b/libcxx/test/std/input.output/string.streams/helper_concepts.h @@ -0,0 +1,17 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef TEST_STD_INPUTOUTPUT_STRINGSTREAMS_HELPER_CONCEPTS_H +#define TEST_STD_INPUTOUTPUT_STRINGSTREAMS_HELPER_CONCEPTS_H + +template +concept is_valid_argument_for_str_member = requires(S s, const T& sv) { + { s.str(sv) }; +}; + +#endif // TEST_STD_INPUTOUTPUT_STRINGSTREAMS_HELPER_CONCEPTS_H diff --git a/libcxx/test/std/input.output/string.streams/helper_string_macros.h b/libcxx/test/std/input.output/string.streams/helper_string_macros.h new file mode 100644 index 000000000000..4a7d22daa788 --- /dev/null +++ b/libcxx/test/std/input.output/string.streams/helper_string_macros.h @@ -0,0 +1,18 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef TEST_STD_INPUTOUTPUT_STRINGSTREAMS_HELPER_MACROS_H +#define TEST_STD_INPUTOUTPUT_STRINGSTREAMS_HELPER_MACROS_H + +#include "make_string.h" + +#define CS(S) MAKE_CSTRING(CharT, S) +#define ST(S, a) std::basic_string(MAKE_CSTRING(CharT, S), MKSTR_LEN(CharT, S), a) +#define SV(S) std::basic_string_view(MAKE_CSTRING(CharT, S), MKSTR_LEN(CharT, S)) + +#endif // TEST_STD_INPUTOUTPUT_STRINGSTREAMS_HELPER_MACROS_H diff --git a/libcxx/test/std/input.output/string.streams/helper_types.h b/libcxx/test/std/input.output/string.streams/helper_types.h new file mode 100644 index 000000000000..8bd830d0fbda --- /dev/null +++ b/libcxx/test/std/input.output/string.streams/helper_types.h @@ -0,0 +1,75 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef TEST_STD_INPUTOUTPUT_STRINGSTREAMS_HELPER_TYPES_H +#define TEST_STD_INPUTOUTPUT_STRINGSTREAMS_HELPER_TYPES_H + +#include +#include + +#include "test_macros.h" + +template > +class ConstConvertibleStringView { +public: + explicit ConstConvertibleStringView(const CharT* cs) : cs_{cs} {} + + operator std::basic_string_view() = delete; + operator std::basic_string_view() const { return std::basic_string_view(cs_); } + +private: + const CharT* cs_; +}; + +static_assert(!std::constructible_from, ConstConvertibleStringView>); +static_assert(!std::convertible_to, std::basic_string_view>); + +static_assert(std::constructible_from, const ConstConvertibleStringView>); +static_assert(std::convertible_to, std::basic_string_view>); + +#ifndef TEST_HAS_NO_WIDE_CHARACTERS +static_assert(!std::constructible_from, ConstConvertibleStringView>); +static_assert(!std::convertible_to, std::basic_string_view>); + +static_assert(std::constructible_from, const ConstConvertibleStringView>); +static_assert(std::convertible_to, std::basic_string_view>); +#endif + +template > +class NonConstConvertibleStringView { +public: + explicit NonConstConvertibleStringView(const CharT* cs) : cs_{cs} {} + + operator std::basic_string_view() { return std::basic_string_view(cs_); } + operator std::basic_string_view() const = delete; + +private: + const CharT* cs_; +}; + +static_assert(std::constructible_from, NonConstConvertibleStringView>); +static_assert(std::convertible_to, std::basic_string_view>); + +static_assert(!std::constructible_from, const NonConstConvertibleStringView>); +static_assert(!std::convertible_to, std::basic_string_view>); + +#ifndef TEST_HAS_NO_WIDE_CHARACTERS +static_assert(std::constructible_from, NonConstConvertibleStringView>); +static_assert(std::convertible_to, std::basic_string_view>); + +static_assert(!std::constructible_from, const NonConstConvertibleStringView>); +static_assert(!std::convertible_to, std::basic_string_view>); +#endif + +struct SomeObject {}; + +struct NonMode {}; + +struct NonAllocator {}; + +#endif // TEST_STD_INPUTOUTPUT_STRINGSTREAMS_HELPER_TYPES_H diff --git a/libcxx/test/std/input.output/string.streams/istringstream/istringstream.cons/string_view.alloc.pass.cpp b/libcxx/test/std/input.output/string.streams/istringstream/istringstream.cons/string_view.alloc.pass.cpp new file mode 100644 index 000000000000..f087aa3d9119 --- /dev/null +++ b/libcxx/test/std/input.output/string.streams/istringstream/istringstream.cons/string_view.alloc.pass.cpp @@ -0,0 +1,145 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// + +// template , class Allocator = allocator> +// class basic_istringstream + +// template +// basic_istringstream(const T& t, const Allocator& a); + +#include +#include +#include +#include +#include +#include + +#include "constexpr_char_traits.h" +#include "nasty_string.h" +#include "test_allocator.h" +#include "test_convertible.h" +#include "test_macros.h" + +#include "../../helper_string_macros.h" +#include "../../helper_types.h" + +template > +void test_sfinae_with_nasty_char() { + // nasty_char* + using NStrStream = std::basic_istringstream; + + static_assert(std::constructible_from); + static_assert(test_convertible()); + + // const nasty_char* + static_assert(std::constructible_from); + static_assert(test_convertible()); +} + +template , typename AllocT = std::allocator> +void test_sfinae() { + using StrStream = std::basic_istringstream; + + // `CharT*` + static_assert(std::constructible_from); + static_assert(test_convertible()); + + // `const CharT*` + static_assert(std::constructible_from); + static_assert(test_convertible()); + + // `std::basic_string_view` + static_assert(std::constructible_from, const AllocT>); + static_assert(test_convertible, const AllocT>()); + + // `std::basic_string` + static_assert(std::constructible_from, const AllocT>); + static_assert(test_convertible, const AllocT>()); + + // ConstConvertibleStringView + static_assert(std::constructible_from, const AllocT>); + static_assert(test_convertible, const AllocT>()); + + // NonConstConvertibleStringView + static_assert(!std::constructible_from, const AllocT>); + static_assert(!test_convertible, const AllocT>()); + + static_assert(!std::constructible_from, const AllocT>); + static_assert(!test_convertible, const AllocT>()); + + // Non-`string-view-like` + static_assert(!std::constructible_from); + static_assert(!test_convertible()); + + // Non-allocator + static_assert(!std::constructible_from, const NonAllocator>); + static_assert(!test_convertible, const NonAllocator>()); +} + +template , typename AllocT = std::allocator> +void test() { + using StrStream = std::basic_istringstream; + + const AllocT allocator; + + // const CharT* + { + StrStream ss(CS("zmt"), allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } + // std::basic_string_view + { + const std::basic_string_view csv = SV("zmt"); + StrStream ss(csv, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } + // std::basic_string + { + const std::basic_string cs = ST("zmt", allocator); + StrStream ss(cs, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } + // ConstConvertibleStringView + { + const ConstConvertibleStringView sv{CS("zmt")}; + StrStream ss(sv, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } +} + +int main(int, char**) { + test_sfinae_with_nasty_char(); + test_sfinae_with_nasty_char>(); + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#endif + return 0; +} diff --git a/libcxx/test/std/input.output/string.streams/istringstream/istringstream.cons/string_view.mode.alloc.pass.cpp b/libcxx/test/std/input.output/string.streams/istringstream/istringstream.cons/string_view.mode.alloc.pass.cpp new file mode 100644 index 000000000000..35309bdb0389 --- /dev/null +++ b/libcxx/test/std/input.output/string.streams/istringstream/istringstream.cons/string_view.mode.alloc.pass.cpp @@ -0,0 +1,189 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// + +// template , class Allocator = allocator> +// class basic_istringstream + +// template +// basic_istringstream(const T& t, ios_base::openmode which, const Allocator& a); + +#include +#include +#include +#include +#include +#include + +#include "constexpr_char_traits.h" +#include "nasty_string.h" +#include "test_allocator.h" +#include "test_convertible.h" +#include "test_macros.h" + +#include "../../helper_string_macros.h" +#include "../../helper_types.h" + +template > +void test_sfinae_with_nasty_char() { + // nasty_char* + using NStrStream = std::basic_istringstream>; + + static_assert(std::constructible_from>); + static_assert(test_convertible>()); + + // const nasty_char* + static_assert(std::constructible_from>); + static_assert( + test_convertible>()); +} + +template , typename AllocT = std::allocator> +void test_sfinae() { + using StrStream = std::basic_istringstream; + + // `CharT*` + static_assert(std::constructible_from); + static_assert(test_convertible()); + + // `const CharT*` + static_assert(std::constructible_from); + static_assert(test_convertible()); + + // `std::basic_string_view` + static_assert(std::constructible_from, + std::ios_base::openmode, + const AllocT>); + static_assert(test_convertible, + std::ios_base::openmode, + const AllocT>()); + + // `std::basic_string` + static_assert(std::constructible_from, + std::ios_base::openmode, + const AllocT>); + static_assert( + test_convertible, std::ios_base::openmode, const AllocT>()); + + // ConstConvertibleStringView + static_assert(std::constructible_from, + std::ios_base::openmode, + const AllocT>); + static_assert(test_convertible, + std::ios_base::openmode, + const AllocT>()); + + // NonConstConvertibleStringView + static_assert(!std::constructible_from, + std::ios_base::openmode, + const AllocT>); + static_assert(!test_convertible, + std::ios_base::openmode, + const AllocT>()); + + static_assert(!std::constructible_from, + std::ios_base::openmode, + const AllocT>); + static_assert(!test_convertible, + std::ios_base::openmode, + const AllocT>()); + + // Non-`string-view-like` + static_assert(!std::constructible_from); + static_assert(!test_convertible()); + + static_assert(!std::constructible_from); + static_assert(!test_convertible()); + + // Non-mode + static_assert( + !std::constructible_from, NonMode, const NonAllocator>); + static_assert( + !test_convertible, NonMode, const NonAllocator>()); + + // Non-allocator + static_assert(!std::constructible_from, + std::ios_base::openmode, + const NonAllocator>); + static_assert(!test_convertible, + std::ios_base::openmode, + const NonAllocator>()); +} + +template , typename AllocT = std::allocator> +void test() { + using StrStream = std::basic_istringstream; + + const AllocT allocator; + + // const CharT* + { + StrStream ss(CS("zmt"), std::ios_base::binary, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } + // std::basic_string_view + { + const std::basic_string_view csv = SV("zmt"); + StrStream ss(csv, std::ios_base::binary, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } + // std::basic_string + { + const std::basic_string cs = ST("zmt", allocator); + StrStream ss(cs, std::ios_base::binary, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } + // ConstConvertibleStringView + { + const ConstConvertibleStringView sv{CS("zmt")}; + StrStream ss(sv, std::ios_base::binary, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } +} + +int main(int, char**) { + test_sfinae_with_nasty_char(); + test_sfinae_with_nasty_char>(); + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#endif + return 0; +} diff --git a/libcxx/test/std/input.output/string.streams/istringstream/istringstream.cons/string_view.mode.pass.cpp b/libcxx/test/std/input.output/string.streams/istringstream/istringstream.cons/string_view.mode.pass.cpp new file mode 100644 index 000000000000..b54972f423f2 --- /dev/null +++ b/libcxx/test/std/input.output/string.streams/istringstream/istringstream.cons/string_view.mode.pass.cpp @@ -0,0 +1,150 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// + +// template , class Allocator = allocator> +// class basic_istringstream + +// template +// explicit basic_istringstream(const T& t, ios_base::openmode which); + +#include +#include +#include +#include +#include +#include + +#include "constexpr_char_traits.h" +#include "nasty_string.h" +#include "test_allocator.h" +#include "test_convertible.h" +#include "test_macros.h" + +#include "../../helper_string_macros.h" +#include "../../helper_types.h" + +template > +void test_sfinae_with_nasty_char() { + // nasty_char* + using NStrStream = std::basic_istringstream>; + + static_assert(std::constructible_from>); + static_assert(!test_convertible()); + + // const nasty_char* + using NStrStream = std::basic_istringstream>; + + static_assert(std::constructible_from>); + static_assert(!test_convertible()); +} + +template , typename AllocT = std::allocator> +void test_sfinae() { + using StrStream = std::basic_istringstream; + + // `CharT*` + static_assert(std::constructible_from); + static_assert(!test_convertible()); + + // `const CharT*` + static_assert(std::constructible_from); + static_assert(!test_convertible()); + + // `std::basic_string_view` + static_assert( + std::constructible_from, std::ios_base::openmode>); + static_assert(!test_convertible, std::ios_base::openmode>()); + + // `std::basic_string` + static_assert(std::constructible_from, std::ios_base::openmode>); + static_assert(!test_convertible, std::ios_base::openmode>()); + + // ConstConvertibleStringView + static_assert( + std::constructible_from, std::ios_base::openmode>); + static_assert( + !test_convertible, std::ios_base::openmode>()); + + // NonConstConvertibleStringView + static_assert( + !std::constructible_from, std::ios_base::openmode>); + static_assert(!test_convertible, std::ios_base::openmode>()); + + static_assert( + !std:: + constructible_from, std::ios_base::openmode>); + static_assert( + !test_convertible, std::ios_base::openmode>()); + + // Non-`string-view-like` + static_assert(!std::constructible_from); + static_assert(!test_convertible()); + + // Non-mode + static_assert(!std::constructible_from, const SomeObject>); + static_assert(!test_convertible, const SomeObject>()); +} + +template , typename AllocT = std::allocator> +void test() { + using StrStream = std::basic_istringstream; + + const AllocT allocator; + + // const CharT* + { + StrStream ss(CS("zmt"), std::ios_base::binary); + assert(ss.str() == CS("zmt")); + } + // std::basic_string_view + { + const std::basic_string_view csv = SV("zmt"); + StrStream ss(csv, std::ios_base::binary); + assert(ss.str() == CS("zmt")); + } + // std::basic_string + { + const std::basic_string cs = ST("zmt", allocator); + StrStream ss(cs, std::ios_base::binary); + assert(ss.str() == CS("zmt")); + } + // ConstConvertibleStringView + { + const ConstConvertibleStringView sv{CS("zmt")}; + StrStream ss(sv, std::ios_base::binary); + assert(ss.str() == CS("zmt")); + } +} + +int main(int, char**) { + test_sfinae_with_nasty_char(); + test_sfinae_with_nasty_char>(); + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#endif + return 0; +} diff --git a/libcxx/test/std/input.output/string.streams/istringstream/istringstream.members/str.string_view.pass.cpp b/libcxx/test/std/input.output/string.streams/istringstream/istringstream.members/str.string_view.pass.cpp new file mode 100644 index 000000000000..0b8b5fdd93c2 --- /dev/null +++ b/libcxx/test/std/input.output/string.streams/istringstream/istringstream.members/str.string_view.pass.cpp @@ -0,0 +1,110 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// + +// template , class Allocator = allocator> +// class basic_istringstream + +// template +// void str(const T& t); + +#include +#include +#include +#include +#include + +#include "constexpr_char_traits.h" +#include "nasty_string.h" +#include "test_allocator.h" +#include "test_macros.h" + +#include "../../helper_concepts.h" +#include "../../helper_string_macros.h" +#include "../../helper_types.h" + +template > +void test_sfinae_with_nasty_char() { + using NStrStream = std::basic_istringstream; + + static_assert(is_valid_argument_for_str_member); + static_assert(is_valid_argument_for_str_member); +} + +template , typename AllocT = std::allocator> +void test_sfinae() { + using StrStream = std::basic_istringstream; + + static_assert(is_valid_argument_for_str_member); + static_assert(is_valid_argument_for_str_member); + static_assert(is_valid_argument_for_str_member>); + static_assert(is_valid_argument_for_str_member>); + static_assert(is_valid_argument_for_str_member>); + + static_assert(!is_valid_argument_for_str_member); + static_assert(!is_valid_argument_for_str_member); + static_assert(!is_valid_argument_for_str_member); + static_assert(!is_valid_argument_for_str_member); + static_assert(!is_valid_argument_for_str_member>); +} + +template , typename AllocT = std::allocator> +void test() { + AllocT allocator; + + std::basic_istringstream ss(std::ios_base::binary, allocator); + assert(ss.str().empty()); + + // const CharT* + ss.str(CS("ba")); + assert(ss.str() == CS("ba")); + + // std::basic_string_view + ss.str(SV("ma")); + assert(ss.str() == CS("ma")); + + // std::basic_string + ss.str(ST("zmt", allocator)); + assert(ss.str() == CS("zmt")); + + // ConstConvertibleStringView + ss.str(ConstConvertibleStringView{CS("da")}); + assert(ss.str() == CS("da")); + + const std::basic_string s(allocator); + ss.str(s); + assert(ss.str().empty()); +} + +int main(int, char**) { + test_sfinae_with_nasty_char(); + test_sfinae_with_nasty_char>(); + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#endif + + return 0; +} diff --git a/libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.cons/string_view.alloc.pass.cpp b/libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.cons/string_view.alloc.pass.cpp new file mode 100644 index 000000000000..c4e68916022f --- /dev/null +++ b/libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.cons/string_view.alloc.pass.cpp @@ -0,0 +1,145 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// + +// template , class Allocator = allocator> +// class basic_ostringstream + +// template +// basic_ostringstream(const T& t, const Allocator& a); + +#include +#include +#include +#include +#include +#include + +#include "constexpr_char_traits.h" +#include "nasty_string.h" +#include "test_allocator.h" +#include "test_convertible.h" +#include "test_macros.h" + +#include "../../helper_string_macros.h" +#include "../../helper_types.h" + +template > +void test_sfinae_with_nasty_char() { + // nasty_char* + using NStrStream = std::basic_ostringstream; + + static_assert(std::constructible_from); + static_assert(test_convertible()); + + // const nasty_char* + static_assert(std::constructible_from); + static_assert(test_convertible()); +} + +template , typename AllocT = std::allocator> +void test_sfinae() { + using StrStream = std::basic_ostringstream; + + // `CharT*` + static_assert(std::constructible_from); + static_assert(test_convertible()); + + // `const CharT*` + static_assert(std::constructible_from); + static_assert(test_convertible()); + + // `std::basic_string_view` + static_assert(std::constructible_from, const AllocT>); + static_assert(test_convertible, const AllocT>()); + + // `std::basic_string` + static_assert(std::constructible_from, const AllocT>); + static_assert(test_convertible, const AllocT>()); + + // ConstConvertibleStringView + static_assert(std::constructible_from, const AllocT>); + static_assert(test_convertible, const AllocT>()); + + // NonConstConvertibleStringView + static_assert(!std::constructible_from, const AllocT>); + static_assert(!test_convertible, const AllocT>()); + + static_assert(!std::constructible_from, const AllocT>); + static_assert(!test_convertible, const AllocT>()); + + // Non-`string-view-like` + static_assert(!std::constructible_from); + static_assert(!test_convertible()); + + // Non-allocator + static_assert(!std::constructible_from, const NonAllocator>); + static_assert(!test_convertible, const NonAllocator>()); +} + +template , typename AllocT = std::allocator> +void test() { + using StrStream = std::basic_ostringstream; + + const AllocT allocator; + + // const CharT* + { + StrStream ss(CS("zmt"), allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } + // std::basic_string_view + { + const std::basic_string_view csv = SV("zmt"); + StrStream ss(csv, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } + // std::basic_string + { + const std::basic_string cs = ST("zmt", allocator); + StrStream ss(cs, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } + // ConstConvertibleStringView + { + const ConstConvertibleStringView sv{CS("zmt")}; + StrStream ss(sv, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } +} + +int main(int, char**) { + test_sfinae_with_nasty_char(); + test_sfinae_with_nasty_char>(); + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#endif + return 0; +} diff --git a/libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.cons/string_view.mode.alloc.pass.cpp b/libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.cons/string_view.mode.alloc.pass.cpp new file mode 100644 index 000000000000..f41311b66250 --- /dev/null +++ b/libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.cons/string_view.mode.alloc.pass.cpp @@ -0,0 +1,189 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// + +// template , class Allocator = allocator> +// class basic_ostringstream + +// template +// basic_ostringstream(const T& t, ios_base::openmode which, const Allocator& a); + +#include +#include +#include +#include +#include +#include + +#include "constexpr_char_traits.h" +#include "nasty_string.h" +#include "test_allocator.h" +#include "test_convertible.h" +#include "test_macros.h" + +#include "../../helper_string_macros.h" +#include "../../helper_types.h" + +template > +void test_sfinae_with_nasty_char() { + // nasty_char* + using NStrStream = std::basic_ostringstream>; + + static_assert(std::constructible_from>); + static_assert(test_convertible>()); + + // const nasty_char* + static_assert(std::constructible_from>); + static_assert( + test_convertible>()); +} + +template , typename AllocT = std::allocator> +void test_sfinae() { + using StrStream = std::basic_ostringstream; + + // `CharT*` + static_assert(std::constructible_from); + static_assert(test_convertible()); + + // `const CharT*` + static_assert(std::constructible_from); + static_assert(test_convertible()); + + // `std::basic_string_view` + static_assert(std::constructible_from, + std::ios_base::openmode, + const AllocT>); + static_assert(test_convertible, + std::ios_base::openmode, + const AllocT>()); + + // `std::basic_string` + static_assert(std::constructible_from, + std::ios_base::openmode, + const AllocT>); + static_assert( + test_convertible, std::ios_base::openmode, const AllocT>()); + + // ConstConvertibleStringView + static_assert(std::constructible_from, + std::ios_base::openmode, + const AllocT>); + static_assert(test_convertible, + std::ios_base::openmode, + const AllocT>()); + + // NonConstConvertibleStringView + static_assert(!std::constructible_from, + std::ios_base::openmode, + const AllocT>); + static_assert(!test_convertible, + std::ios_base::openmode, + const AllocT>()); + + static_assert(!std::constructible_from, + std::ios_base::openmode, + const AllocT>); + static_assert(!test_convertible, + std::ios_base::openmode, + const AllocT>()); + + // Non-`string-view-like` + static_assert(!std::constructible_from); + static_assert(!test_convertible()); + + static_assert(!std::constructible_from); + static_assert(!test_convertible()); + + // Non-mode + static_assert( + !std::constructible_from, NonMode, const NonAllocator>); + static_assert( + !test_convertible, NonMode, const NonAllocator>()); + + // Non-allocator + static_assert(!std::constructible_from, + std::ios_base::openmode, + const NonAllocator>); + static_assert(!test_convertible, + std::ios_base::openmode, + const NonAllocator>()); +} + +template , typename AllocT = std::allocator> +void test() { + using StrStream = std::basic_ostringstream; + + const AllocT allocator; + + // const CharT* + { + StrStream ss(CS("zmt"), std::ios_base::binary, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } + // std::basic_string_view + { + const std::basic_string_view csv = SV("zmt"); + StrStream ss(csv, std::ios_base::binary, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } + // std::basic_string + { + const std::basic_string cs = ST("zmt", allocator); + StrStream ss(cs, std::ios_base::binary, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } + // ConstConvertibleStringView + { + const ConstConvertibleStringView sv{CS("zmt")}; + StrStream ss(sv, std::ios_base::binary, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } +} + +int main(int, char**) { + test_sfinae_with_nasty_char(); + test_sfinae_with_nasty_char>(); + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#endif + return 0; +} diff --git a/libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.cons/string_view.mode.pass.cpp b/libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.cons/string_view.mode.pass.cpp new file mode 100644 index 000000000000..9eb3724da3d2 --- /dev/null +++ b/libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.cons/string_view.mode.pass.cpp @@ -0,0 +1,150 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// + +// template , class Allocator = allocator> +// class basic_ostringstream + +// template +// explicit basic_ostringstream(const T& t, ios_base::openmode which); + +#include +#include +#include +#include +#include +#include + +#include "constexpr_char_traits.h" +#include "nasty_string.h" +#include "test_allocator.h" +#include "test_convertible.h" +#include "test_macros.h" + +#include "../../helper_string_macros.h" +#include "../../helper_types.h" + +template > +void test_sfinae_with_nasty_char() { + // nasty_char* + using NStrStream = std::basic_ostringstream>; + + static_assert(std::constructible_from>); + static_assert(!test_convertible()); + + // const nasty_char* + using NStrStream = std::basic_ostringstream>; + + static_assert(std::constructible_from>); + static_assert(!test_convertible()); +} + +template , typename AllocT = std::allocator> +void test_sfinae() { + using StrStream = std::basic_ostringstream; + + // `CharT*` + static_assert(std::constructible_from); + static_assert(!test_convertible()); + + // `const CharT*` + static_assert(std::constructible_from); + static_assert(!test_convertible()); + + // `std::basic_string_view` + static_assert( + std::constructible_from, std::ios_base::openmode>); + static_assert(!test_convertible, std::ios_base::openmode>()); + + // `std::basic_string` + static_assert(std::constructible_from, std::ios_base::openmode>); + static_assert(!test_convertible, std::ios_base::openmode>()); + + // ConstConvertibleStringView + static_assert( + std::constructible_from, std::ios_base::openmode>); + static_assert( + !test_convertible, std::ios_base::openmode>()); + + // NonConstConvertibleStringView + static_assert( + !std::constructible_from, std::ios_base::openmode>); + static_assert(!test_convertible, std::ios_base::openmode>()); + + static_assert( + !std:: + constructible_from, std::ios_base::openmode>); + static_assert( + !test_convertible, std::ios_base::openmode>()); + + // Non-`string-view-like` + static_assert(!std::constructible_from); + static_assert(!test_convertible()); + + // Non-mode + static_assert(!std::constructible_from, const SomeObject>); + static_assert(!test_convertible, const SomeObject>()); +} + +template , typename AllocT = std::allocator> +void test() { + using StrStream = std::basic_ostringstream; + + const AllocT allocator; + + // const CharT* + { + StrStream ss(CS("zmt"), std::ios_base::binary); + assert(ss.str() == CS("zmt")); + } + // std::basic_string_view + { + const std::basic_string_view csv = SV("zmt"); + StrStream ss(csv, std::ios_base::binary); + assert(ss.str() == CS("zmt")); + } + // std::basic_string + { + const std::basic_string cs = ST("zmt", allocator); + StrStream ss(cs, std::ios_base::binary); + assert(ss.str() == CS("zmt")); + } + // ConstConvertibleStringView + { + const ConstConvertibleStringView sv{CS("zmt")}; + StrStream ss(sv, std::ios_base::binary); + assert(ss.str() == CS("zmt")); + } +} + +int main(int, char**) { + test_sfinae_with_nasty_char(); + test_sfinae_with_nasty_char>(); + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#endif + return 0; +} diff --git a/libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.members/str.string_view.pass.cpp b/libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.members/str.string_view.pass.cpp new file mode 100644 index 000000000000..ec7106726300 --- /dev/null +++ b/libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.members/str.string_view.pass.cpp @@ -0,0 +1,110 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// + +// template , class Allocator = allocator> +// class basic_ostringstream + +// template +// void str(const T& t); + +#include +#include +#include +#include +#include + +#include "constexpr_char_traits.h" +#include "nasty_string.h" +#include "test_allocator.h" +#include "test_macros.h" + +#include "../../helper_concepts.h" +#include "../../helper_string_macros.h" +#include "../../helper_types.h" + +template > +void test_sfinae_with_nasty_char() { + using NStrStream = std::basic_ostringstream; + + static_assert(is_valid_argument_for_str_member); + static_assert(is_valid_argument_for_str_member); +} + +template , typename AllocT = std::allocator> +void test_sfinae() { + using StrStream = std::basic_ostringstream; + + static_assert(is_valid_argument_for_str_member); + static_assert(is_valid_argument_for_str_member); + static_assert(is_valid_argument_for_str_member>); + static_assert(is_valid_argument_for_str_member>); + static_assert(is_valid_argument_for_str_member>); + + static_assert(!is_valid_argument_for_str_member); + static_assert(!is_valid_argument_for_str_member); + static_assert(!is_valid_argument_for_str_member); + static_assert(!is_valid_argument_for_str_member); + static_assert(!is_valid_argument_for_str_member>); +} + +template , typename AllocT = std::allocator> +void test() { + AllocT allocator; + + std::basic_ostringstream ss(std::ios_base::binary, allocator); + assert(ss.str().empty()); + + // const CharT* + ss.str(CS("ba")); + assert(ss.str() == CS("ba")); + + // std::basic_string_view + ss.str(SV("ma")); + assert(ss.str() == CS("ma")); + + // std::basic_string + ss.str(ST("zmt", allocator)); + assert(ss.str() == CS("zmt")); + + // ConstConvertibleStringView + ss.str(ConstConvertibleStringView{CS("da")}); + assert(ss.str() == CS("da")); + + const std::basic_string s(allocator); + ss.str(s); + assert(ss.str().empty()); +} + +int main(int, char**) { + test_sfinae_with_nasty_char(); + test_sfinae_with_nasty_char>(); + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#endif + + return 0; +} diff --git a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.cons/string_view.alloc.pass.cpp b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.cons/string_view.alloc.pass.cpp new file mode 100644 index 000000000000..de07e3a6d23a --- /dev/null +++ b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.cons/string_view.alloc.pass.cpp @@ -0,0 +1,145 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// + +// template , class Allocator = allocator> +// class basic_stringbuf + +// template +// basic_stringbuf(const T& t, const Allocator& a); + +#include +#include +#include +#include +#include +#include + +#include "constexpr_char_traits.h" +#include "nasty_string.h" +#include "test_allocator.h" +#include "test_convertible.h" +#include "test_macros.h" + +#include "../../helper_string_macros.h" +#include "../../helper_types.h" + +template > +void test_sfinae_with_nasty_char() { + // nasty_char* + using NStrBuf = std::basic_istringstream; + + static_assert(std::constructible_from); + static_assert(test_convertible()); + + // const nasty_char* + static_assert(std::constructible_from); + static_assert(test_convertible()); +} + +template , typename AllocT = std::allocator> +void test_sfinae() { + using StrBuf = std::basic_stringbuf; + + // `CharT*` + static_assert(std::constructible_from); + static_assert(test_convertible()); + + // `const CharT*` + static_assert(std::constructible_from); + static_assert(test_convertible()); + + // `std::basic_string_view` + static_assert(std::constructible_from, const AllocT>); + static_assert(test_convertible, const AllocT>()); + + // `std::basic_string` + static_assert(std::constructible_from, const AllocT>); + static_assert(test_convertible, const AllocT>()); + + // ConstConvertibleStringView + static_assert(std::constructible_from, const AllocT>); + static_assert(test_convertible, const AllocT>()); + + // NonConstConvertibleStringView + static_assert(!std::constructible_from, const AllocT>); + static_assert(!test_convertible, const AllocT>()); + + static_assert(!std::constructible_from, const AllocT>); + static_assert(!test_convertible, const AllocT>()); + + // Non-`string-view-like` + static_assert(!std::constructible_from); + static_assert(!test_convertible()); + + // Non-allocator + static_assert(!std::constructible_from, const NonAllocator>); + static_assert(!test_convertible, const NonAllocator>()); +} + +template , typename AllocT = std::allocator> +void test() { + using StrBuf = std::basic_stringbuf; + + const AllocT allocator; + + // const CharT* + { + StrBuf ss(CS("zmt"), allocator); + assert(ss.str() == CS("zmt")); + assert(ss.get_allocator() == allocator); + } + // std::basic_string_view + { + const std::basic_string_view csv = SV("zmt"); + StrBuf ss(csv, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.get_allocator() == allocator); + } + // std::basic_string + { + const std::basic_string cs = ST("zmt", allocator); + StrBuf ss(cs, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.get_allocator() == allocator); + } + // ConstConvertibleStringView + { + const ConstConvertibleStringView sv{CS("zmt")}; + StrBuf ss(sv, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.get_allocator() == allocator); + } +} + +int main(int, char**) { + test_sfinae_with_nasty_char(); + test_sfinae_with_nasty_char>(); + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#endif + return 0; +} diff --git a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.cons/string_view.mode.alloc.pass.cpp b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.cons/string_view.mode.alloc.pass.cpp new file mode 100644 index 000000000000..fa06e82318cd --- /dev/null +++ b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.cons/string_view.mode.alloc.pass.cpp @@ -0,0 +1,184 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// + +// template , class Allocator = allocator> +// class basic_stringbuf + +// template +// basic_stringbuf(const T& t, ios_base::openmode which, const Allocator& a); + +#include +#include +#include +#include +#include +#include + +#include "constexpr_char_traits.h" +#include "nasty_string.h" +#include "test_allocator.h" +#include "test_convertible.h" +#include "test_macros.h" + +#include "../../helper_string_macros.h" +#include "../../helper_types.h" + +template > +void test_sfinae_with_nasty_char() { + // nasty_char* + using NStrBuf = std::basic_stringbuf>; + + static_assert(std::constructible_from>); + static_assert(test_convertible>()); + + // const nasty_char* + static_assert(std::constructible_from>); + static_assert( + test_convertible>()); +} + +template , typename AllocT = std::allocator> +void test_sfinae() { + using StrBuf = std::basic_stringbuf; + + // `CharT*` + static_assert(std::constructible_from); + static_assert(test_convertible()); + + // `const CharT*` + static_assert(std::constructible_from); + static_assert(test_convertible()); + + // `std::basic_string_view` + static_assert(std::constructible_from, + std::ios_base::openmode, + const AllocT>); + static_assert( + test_convertible, std::ios_base::openmode, const AllocT>()); + + // `std::basic_string` + static_assert( + std::constructible_from, std::ios_base::openmode, const AllocT>); + static_assert( + test_convertible, std::ios_base::openmode, const AllocT>()); + + // ConstConvertibleStringView + static_assert(std::constructible_from, + std::ios_base::openmode, + const AllocT>); + static_assert(test_convertible, + std::ios_base::openmode, + const AllocT>()); + + // NonConstConvertibleStringView + static_assert(!std::constructible_from, + std::ios_base::openmode, + const AllocT>); + static_assert(!test_convertible, + std::ios_base::openmode, + const AllocT>()); + + static_assert(!std::constructible_from, + std::ios_base::openmode, + const AllocT>); + static_assert(!test_convertible, + std::ios_base::openmode, + const AllocT>()); + + // Non-`string-view-like` + static_assert(!std::constructible_from); + static_assert(!test_convertible()); + + static_assert(!std::constructible_from); + static_assert(!test_convertible()); + + // Non-mode + static_assert( + !std::constructible_from, NonMode, const NonAllocator>); + static_assert(!test_convertible, NonMode, const NonAllocator>()); + + // Non-allocator + static_assert(!std::constructible_from, + std::ios_base::openmode, + const NonAllocator>); + static_assert(!test_convertible, + std::ios_base::openmode, + const NonAllocator>()); +} + +template , typename AllocT = std::allocator> +void test() { + using StrBuf = std::basic_stringbuf; + + const AllocT allocator; + + // const CharT* + { + StrBuf ss(CS("zmt"), std::ios_base::out | std::ios_base::in, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.get_allocator() == allocator); + } + // std::basic_string_view + { + const std::basic_string_view csv = SV("zmt"); + StrBuf ss(csv, std::ios_base::out | std::ios_base::in, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.get_allocator() == allocator); + } + // std::basic_string + { + const std::basic_string cs = ST("zmt", allocator); + StrBuf ss(cs, std::ios_base::out | std::ios_base::in, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.get_allocator() == allocator); + } + // ConstConvertibleStringView + { + const ConstConvertibleStringView sv{CS("zmt")}; + StrBuf ss(sv, std::ios_base::out | std::ios_base::in, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.get_allocator() == allocator); + } +} + +int main(int, char**) { + test_sfinae_with_nasty_char(); + test_sfinae_with_nasty_char>(); + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#endif + return 0; +} diff --git a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.cons/string_view.mode.pass.cpp b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.cons/string_view.mode.pass.cpp new file mode 100644 index 000000000000..28bc6af737ec --- /dev/null +++ b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.cons/string_view.mode.pass.cpp @@ -0,0 +1,150 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// + +// template , class Allocator = allocator> +// class basic_stringbuf + +// template +// explicit basic_stringbuf(const T& t, ios_base::openmode which); + +#include +#include +#include +#include +#include +#include + +#include "constexpr_char_traits.h" +#include "nasty_string.h" +#include "test_allocator.h" +#include "test_convertible.h" +#include "test_macros.h" + +#include "../../helper_string_macros.h" +#include "../../helper_types.h" + +template > +void test_sfinae_with_nasty_char() { + // nasty_char* + using NStrStream = std::basic_stringbuf>; + + static_assert(std::constructible_from>); + static_assert(!test_convertible()); + + // const nasty_char* + using NStrStream = std::basic_stringbuf>; + + static_assert(std::constructible_from>); + static_assert(!test_convertible()); +} + +template , typename AllocT = std::allocator> +void test_sfinae() { + using StrStream = std::basic_stringbuf; + + // `CharT*` + static_assert(std::constructible_from); + static_assert(!test_convertible()); + + // `const CharT*` + static_assert(std::constructible_from); + static_assert(!test_convertible()); + + // `std::basic_string_view` + static_assert( + std::constructible_from, std::ios_base::openmode>); + static_assert(!test_convertible, std::ios_base::openmode>()); + + // `std::basic_string` + static_assert(std::constructible_from, std::ios_base::openmode>); + static_assert(!test_convertible, std::ios_base::openmode>()); + + // ConstConvertibleStringView + static_assert( + std::constructible_from, std::ios_base::openmode>); + static_assert( + !test_convertible, std::ios_base::openmode>()); + + // NonConstConvertibleStringView + static_assert( + !std::constructible_from, std::ios_base::openmode>); + static_assert(!test_convertible, std::ios_base::openmode>()); + + static_assert( + !std:: + constructible_from, std::ios_base::openmode>); + static_assert( + !test_convertible, std::ios_base::openmode>()); + + // Non-`string-view-like` + static_assert(!std::constructible_from); + static_assert(!test_convertible()); + + // Non-mode + static_assert(!std::constructible_from, const SomeObject>); + static_assert(!test_convertible, const SomeObject>()); +} + +template , typename AllocT = std::allocator> +void test() { + using StrStream = std::basic_stringbuf; + + const AllocT allocator; + + // const CharT* + { + StrStream ss(CS("zmt"), std::ios_base::out | std::ios_base::in); + assert(ss.str() == CS("zmt")); + } + // std::basic_string_view + { + const std::basic_string_view csv = SV("zmt"); + StrStream ss(csv, std::ios_base::out | std::ios_base::in); + assert(ss.str() == CS("zmt")); + } + // std::basic_string + { + const std::basic_string cs = ST("zmt", allocator); + StrStream ss(cs, std::ios_base::out | std::ios_base::in); + assert(ss.str() == CS("zmt")); + } + // ConstConvertibleStringView + { + const ConstConvertibleStringView sv{CS("zmt")}; + StrStream ss(sv, std::ios_base::out | std::ios_base::in); + assert(ss.str() == CS("zmt")); + } +} + +int main(int, char**) { + test_sfinae_with_nasty_char(); + test_sfinae_with_nasty_char>(); + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#endif + return 0; +} diff --git a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.members/str.string_view.pass.cpp b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.members/str.string_view.pass.cpp new file mode 100644 index 000000000000..2a880b93f5ec --- /dev/null +++ b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.members/str.string_view.pass.cpp @@ -0,0 +1,110 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// + +// template , class Allocator = allocator> +// class basic_stringbuf + +// template +// void str(const T& t); + +#include +#include +#include +#include +#include + +#include "constexpr_char_traits.h" +#include "nasty_string.h" +#include "test_allocator.h" +#include "test_macros.h" + +#include "../../helper_concepts.h" +#include "../../helper_string_macros.h" +#include "../../helper_types.h" + +template > +void test_sfinae_with_nasty_char() { + using NStrBuf = std::basic_stringbuf; + + static_assert(is_valid_argument_for_str_member); + static_assert(is_valid_argument_for_str_member); +} + +template , typename AllocT = std::allocator> +void test_sfinae() { + using StrBuff = std::basic_stringbuf; + + static_assert(is_valid_argument_for_str_member); + static_assert(is_valid_argument_for_str_member); + static_assert(is_valid_argument_for_str_member>); + static_assert(is_valid_argument_for_str_member>); + static_assert(is_valid_argument_for_str_member>); + + static_assert(!is_valid_argument_for_str_member); + static_assert(!is_valid_argument_for_str_member); + static_assert(!is_valid_argument_for_str_member); + static_assert(!is_valid_argument_for_str_member); + static_assert(!is_valid_argument_for_str_member>); +} + +template , typename AllocT = std::allocator> +void test() { + AllocT allocator; + + std::basic_stringbuf ss(std::ios_base::in | std::ios_base::out, allocator); + assert(ss.str().empty()); + + // const CharT* + ss.str(CS("ba")); + assert(ss.str() == CS("ba")); + + // std::basic_string_view + ss.str(SV("ma")); + assert(ss.str() == CS("ma")); + + // std::basic_string + ss.str(ST("zmt", allocator)); + assert(ss.str() == CS("zmt")); + + // ConstConvertibleStringView + ss.str(ConstConvertibleStringView{CS("da")}); + assert(ss.str() == CS("da")); + + const std::basic_string s(allocator); + ss.str(s); + assert(ss.str().empty()); +} + +int main(int, char**) { + test_sfinae_with_nasty_char(); + test_sfinae_with_nasty_char>(); + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#endif + + return 0; +} diff --git a/libcxx/test/std/input.output/string.streams/stringstream/stringstream.cons/string_view.alloc.pass.cpp b/libcxx/test/std/input.output/string.streams/stringstream/stringstream.cons/string_view.alloc.pass.cpp new file mode 100644 index 000000000000..0aa05b17be6f --- /dev/null +++ b/libcxx/test/std/input.output/string.streams/stringstream/stringstream.cons/string_view.alloc.pass.cpp @@ -0,0 +1,145 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// + +// template , class Allocator = allocator> +// class basic_stringstream + +// template +// basic_stringstream(const T& t, const Allocator& a); + +#include +#include +#include +#include +#include +#include + +#include "constexpr_char_traits.h" +#include "nasty_string.h" +#include "test_allocator.h" +#include "test_convertible.h" +#include "test_macros.h" + +#include "../../helper_string_macros.h" +#include "../../helper_types.h" + +template > +void test_sfinae_with_nasty_char() { + // nasty_char* + using NStrStream = std::basic_stringstream; + + static_assert(std::constructible_from); + static_assert(test_convertible()); + + // const nasty_char* + static_assert(std::constructible_from); + static_assert(test_convertible()); +} + +template , typename AllocT = std::allocator> +void test_sfinae() { + using StrStream = std::basic_stringstream; + + // `CharT*` + static_assert(std::constructible_from); + static_assert(test_convertible()); + + // `const CharT*` + static_assert(std::constructible_from); + static_assert(test_convertible()); + + // `std::basic_string_view` + static_assert(std::constructible_from, const AllocT>); + static_assert(test_convertible, const AllocT>()); + + // `std::basic_string` + static_assert(std::constructible_from, const AllocT>); + static_assert(test_convertible, const AllocT>()); + + // ConstConvertibleStringView + static_assert(std::constructible_from, const AllocT>); + static_assert(test_convertible, const AllocT>()); + + // NonConstConvertibleStringView + static_assert(!std::constructible_from, const AllocT>); + static_assert(!test_convertible, const AllocT>()); + + static_assert(!std::constructible_from, const AllocT>); + static_assert(!test_convertible, const AllocT>()); + + // Non-`string-view-like` + static_assert(!std::constructible_from); + static_assert(!test_convertible()); + + // Non-allocator + static_assert(!std::constructible_from, const NonAllocator>); + static_assert(!test_convertible, const NonAllocator>()); +} + +template , typename AllocT = std::allocator> +void test() { + using StrStream = std::basic_stringstream; + + const AllocT allocator; + + // const CharT* + { + StrStream ss(CS("zmt"), allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } + // std::basic_string_view + { + const std::basic_string_view csv = SV("zmt"); + StrStream ss(csv, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } + // std::basic_string + { + const std::basic_string cs = ST("zmt", allocator); + StrStream ss(cs, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } + // ConstConvertibleStringView + { + const ConstConvertibleStringView sv{CS("zmt")}; + StrStream ss(sv, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } +} + +int main(int, char**) { + test_sfinae_with_nasty_char(); + test_sfinae_with_nasty_char>(); + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#endif + return 0; +} diff --git a/libcxx/test/std/input.output/string.streams/stringstream/stringstream.cons/string_view.mode.alloc.pass.cpp b/libcxx/test/std/input.output/string.streams/stringstream/stringstream.cons/string_view.mode.alloc.pass.cpp new file mode 100644 index 000000000000..18164b072cf5 --- /dev/null +++ b/libcxx/test/std/input.output/string.streams/stringstream/stringstream.cons/string_view.mode.alloc.pass.cpp @@ -0,0 +1,189 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// + +// template , class Allocator = allocator> +// class basic_stringstream + +// template +// basic_stringstream(const T& t, ios_base::openmode which, const Allocator& a); + +#include +#include +#include +#include +#include +#include + +#include "constexpr_char_traits.h" +#include "nasty_string.h" +#include "test_allocator.h" +#include "test_convertible.h" +#include "test_macros.h" + +#include "../../helper_string_macros.h" +#include "../../helper_types.h" + +template > +void test_sfinae_with_nasty_char() { + // nasty_char* + using NStrStream = std::basic_stringstream>; + + static_assert(std::constructible_from>); + static_assert(test_convertible>()); + + // const nasty_char* + static_assert(std::constructible_from>); + static_assert( + test_convertible>()); +} + +template , typename AllocT = std::allocator> +void test_sfinae() { + using StrStream = std::basic_stringstream; + + // `CharT*` + static_assert(std::constructible_from); + static_assert(test_convertible()); + + // `const CharT*` + static_assert(std::constructible_from); + static_assert(test_convertible()); + + // `std::basic_string_view` + static_assert(std::constructible_from, + std::ios_base::openmode, + const AllocT>); + static_assert(test_convertible, + std::ios_base::openmode, + const AllocT>()); + + // `std::basic_string` + static_assert(std::constructible_from, + std::ios_base::openmode, + const AllocT>); + static_assert( + test_convertible, std::ios_base::openmode, const AllocT>()); + + // ConstConvertibleStringView + static_assert(std::constructible_from, + std::ios_base::openmode, + const AllocT>); + static_assert(test_convertible, + std::ios_base::openmode, + const AllocT>()); + + // NonConstConvertibleStringView + static_assert(!std::constructible_from, + std::ios_base::openmode, + const AllocT>); + static_assert(!test_convertible, + std::ios_base::openmode, + const AllocT>()); + + static_assert(!std::constructible_from, + std::ios_base::openmode, + const AllocT>); + static_assert(!test_convertible, + std::ios_base::openmode, + const AllocT>()); + + // Non-`string-view-like` + static_assert(!std::constructible_from); + static_assert(!test_convertible()); + + static_assert(!std::constructible_from); + static_assert(!test_convertible()); + + // Non-mode + static_assert( + !std::constructible_from, NonMode, const NonAllocator>); + static_assert( + !test_convertible, NonMode, const NonAllocator>()); + + // Non-allocator + static_assert(!std::constructible_from, + std::ios_base::openmode, + const NonAllocator>); + static_assert(!test_convertible, + std::ios_base::openmode, + const NonAllocator>()); +} + +template , typename AllocT = std::allocator> +void test() { + using StrStream = std::basic_stringstream; + + const AllocT allocator; + + // const CharT* + { + StrStream ss(CS("zmt"), std::ios_base::out | std::ios_base::in, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } + // std::basic_string_view + { + const std::basic_string_view csv = SV("zmt"); + StrStream ss(csv, std::ios_base::out | std::ios_base::in, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } + // std::basic_string + { + const std::basic_string cs = ST("zmt", allocator); + StrStream ss(cs, std::ios_base::out | std::ios_base::in, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } + // ConstConvertibleStringView + { + const ConstConvertibleStringView sv{CS("zmt")}; + StrStream ss(sv, std::ios_base::out | std::ios_base::in, allocator); + assert(ss.str() == CS("zmt")); + assert(ss.rdbuf()->get_allocator() == allocator); + } +} + +int main(int, char**) { + test_sfinae_with_nasty_char(); + test_sfinae_with_nasty_char>(); + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#endif + return 0; +} diff --git a/libcxx/test/std/input.output/string.streams/stringstream/stringstream.cons/string_view.mode.pass.cpp b/libcxx/test/std/input.output/string.streams/stringstream/stringstream.cons/string_view.mode.pass.cpp new file mode 100644 index 000000000000..b7b56f9123b9 --- /dev/null +++ b/libcxx/test/std/input.output/string.streams/stringstream/stringstream.cons/string_view.mode.pass.cpp @@ -0,0 +1,150 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// + +// template , class Allocator = allocator> +// class basic_stringstream + +// template +// explicit basic_stringstream(const T& t, ios_base::openmode which); + +#include +#include +#include +#include +#include +#include + +#include "constexpr_char_traits.h" +#include "nasty_string.h" +#include "test_allocator.h" +#include "test_convertible.h" +#include "test_macros.h" + +#include "../../helper_string_macros.h" +#include "../../helper_types.h" + +template > +void test_sfinae_with_nasty_char() { + // nasty_char* + using NStrStream = std::basic_stringstream>; + + static_assert(std::constructible_from>); + static_assert(!test_convertible()); + + // const nasty_char* + using NStrStream = std::basic_stringstream>; + + static_assert(std::constructible_from>); + static_assert(!test_convertible()); +} + +template , typename AllocT = std::allocator> +void test_sfinae() { + using StrStream = std::basic_stringstream; + + // `CharT*` + static_assert(std::constructible_from); + static_assert(!test_convertible()); + + // `const CharT*` + static_assert(std::constructible_from); + static_assert(!test_convertible()); + + // `std::basic_string_view` + static_assert( + std::constructible_from, std::ios_base::openmode>); + static_assert(!test_convertible, std::ios_base::openmode>()); + + // `std::basic_string` + static_assert(std::constructible_from, std::ios_base::openmode>); + static_assert(!test_convertible, std::ios_base::openmode>()); + + // ConstConvertibleStringView + static_assert( + std::constructible_from, std::ios_base::openmode>); + static_assert( + !test_convertible, std::ios_base::openmode>()); + + // NonConstConvertibleStringView + static_assert( + !std::constructible_from, std::ios_base::openmode>); + static_assert(!test_convertible, std::ios_base::openmode>()); + + static_assert( + !std:: + constructible_from, std::ios_base::openmode>); + static_assert( + !test_convertible, std::ios_base::openmode>()); + + // Non-`string-view-like` + static_assert(!std::constructible_from); + static_assert(!test_convertible()); + + // Non-mode + static_assert(!std::constructible_from, const SomeObject>); + static_assert(!test_convertible, const SomeObject>()); +} + +template , typename AllocT = std::allocator> +void test() { + using StrStream = std::basic_stringstream; + + const AllocT allocator; + + // const CharT* + { + StrStream ss(CS("zmt"), std::ios_base::out | std::ios_base::in); + assert(ss.str() == CS("zmt")); + } + // std::basic_string_view + { + const std::basic_string_view csv = SV("zmt"); + StrStream ss(csv, std::ios_base::out | std::ios_base::in); + assert(ss.str() == CS("zmt")); + } + // std::basic_string + { + const std::basic_string cs = ST("zmt", allocator); + StrStream ss(cs, std::ios_base::out | std::ios_base::in); + assert(ss.str() == CS("zmt")); + } + // ConstConvertibleStringView + { + const ConstConvertibleStringView sv{CS("zmt")}; + StrStream ss(sv, std::ios_base::out | std::ios_base::in); + assert(ss.str() == CS("zmt")); + } +} + +int main(int, char**) { + test_sfinae_with_nasty_char(); + test_sfinae_with_nasty_char>(); + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#endif + return 0; +} diff --git a/libcxx/test/std/input.output/string.streams/stringstream/stringstream.members/str.string_view.pass.cpp b/libcxx/test/std/input.output/string.streams/stringstream/stringstream.members/str.string_view.pass.cpp new file mode 100644 index 000000000000..22ea3c6d56e0 --- /dev/null +++ b/libcxx/test/std/input.output/string.streams/stringstream/stringstream.members/str.string_view.pass.cpp @@ -0,0 +1,110 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// + +// template , class Allocator = allocator> +// class basic_stringstream + +// template +// void str(const T& t); + +#include +#include +#include +#include +#include + +#include "constexpr_char_traits.h" +#include "nasty_string.h" +#include "test_allocator.h" +#include "test_macros.h" + +#include "../../helper_concepts.h" +#include "../../helper_string_macros.h" +#include "../../helper_types.h" + +template > +void test_sfinae_with_nasty_char() { + using NStrStream = std::basic_stringstream; + + static_assert(is_valid_argument_for_str_member); + static_assert(is_valid_argument_for_str_member); +} + +template , typename AllocT = std::allocator> +void test_sfinae() { + using StrStream = std::basic_stringstream; + + static_assert(is_valid_argument_for_str_member); + static_assert(is_valid_argument_for_str_member); + static_assert(is_valid_argument_for_str_member>); + static_assert(is_valid_argument_for_str_member>); + static_assert(is_valid_argument_for_str_member>); + + static_assert(!is_valid_argument_for_str_member); + static_assert(!is_valid_argument_for_str_member); + static_assert(!is_valid_argument_for_str_member); + static_assert(!is_valid_argument_for_str_member); + static_assert(!is_valid_argument_for_str_member>); +} + +template , typename AllocT = std::allocator> +void test() { + AllocT allocator; + + std::basic_stringstream ss(std::ios_base::out | std::ios_base::in, allocator); + assert(ss.str().empty()); + + // const CharT* + ss.str(CS("ba")); + assert(ss.str() == CS("ba")); + + // std::basic_string_view + ss.str(SV("ma")); + assert(ss.str() == CS("ma")); + + // std::basic_string + ss.str(ST("zmt", allocator)); + assert(ss.str() == CS("zmt")); + + // ConstConvertibleStringView + ss.str(ConstConvertibleStringView{CS("da")}); + assert(ss.str() == CS("da")); + + const std::basic_string s(allocator); + ss.str(s); + assert(ss.str().empty()); +} + +int main(int, char**) { + test_sfinae_with_nasty_char(); + test_sfinae_with_nasty_char>(); + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_sfinae(); + test_sfinae, std::allocator>(); + test_sfinae, test_allocator>(); + test_sfinae, test_allocator>(); + test(); + test, std::allocator>(); + test, test_allocator>(); + test, test_allocator>(); +#endif + + return 0; +} diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/sstream.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/sstream.version.compile.pass.cpp index 78e17e5f1ddb..ad3c8e4012ec 100644 --- a/libcxx/test/std/language.support/support.limits/support.limits.general/sstream.version.compile.pass.cpp +++ b/libcxx/test/std/language.support/support.limits/support.limits.general/sstream.version.compile.pass.cpp @@ -56,17 +56,11 @@ #elif TEST_STD_VER > 23 -# if !defined(_LIBCPP_VERSION) -# ifndef __cpp_lib_sstream_from_string_view -# error "__cpp_lib_sstream_from_string_view should be defined in c++26" -# endif -# if __cpp_lib_sstream_from_string_view != 202306L -# error "__cpp_lib_sstream_from_string_view should have the value 202306L in c++26" -# endif -# else // _LIBCPP_VERSION -# ifdef __cpp_lib_sstream_from_string_view -# error "__cpp_lib_sstream_from_string_view should not be defined because it is unimplemented in libc++!" -# endif +# ifndef __cpp_lib_sstream_from_string_view +# error "__cpp_lib_sstream_from_string_view should be defined in c++26" +# endif +# if __cpp_lib_sstream_from_string_view != 202306L +# error "__cpp_lib_sstream_from_string_view should have the value 202306L in c++26" # endif #endif // TEST_STD_VER > 23 diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp index 20804d835015..5501587915ff 100644 --- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp +++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp @@ -7394,17 +7394,11 @@ # error "__cpp_lib_ssize should have the value 201902L in c++26" # endif -# if !defined(_LIBCPP_VERSION) -# ifndef __cpp_lib_sstream_from_string_view -# error "__cpp_lib_sstream_from_string_view should be defined in c++26" -# endif -# if __cpp_lib_sstream_from_string_view != 202306L -# error "__cpp_lib_sstream_from_string_view should have the value 202306L in c++26" -# endif -# else // _LIBCPP_VERSION -# ifdef __cpp_lib_sstream_from_string_view -# error "__cpp_lib_sstream_from_string_view should not be defined because it is unimplemented in libc++!" -# endif +# ifndef __cpp_lib_sstream_from_string_view +# error "__cpp_lib_sstream_from_string_view should be defined in c++26" +# endif +# if __cpp_lib_sstream_from_string_view != 202306L +# error "__cpp_lib_sstream_from_string_view should have the value 202306L in c++26" # endif # if !defined(_LIBCPP_VERSION) diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py index 16d2fc6a532d..c55f5c7d1900 100755 --- a/libcxx/utils/generate_feature_test_macro_components.py +++ b/libcxx/utils/generate_feature_test_macro_components.py @@ -1118,7 +1118,6 @@ feature_test_macros = [ "c++26": 202306 # P2495R3 Interfacing stringstreams with string_view }, "headers": ["sstream"], - "unimplemented": True, }, { "name": "__cpp_lib_stacktrace", -- GitLab From 9b672de9976084872fbca764898ed722dea1dd9b Mon Sep 17 00:00:00 2001 From: Francesco Petrogalli Date: Tue, 5 Mar 2024 07:51:35 +0100 Subject: [PATCH 096/929] [clang][Builtins] Parse clang extended vectors types. (#83584) Clang extended vector types are mangled as follows: '_ExtVector<' ',' '>' This is used to defetmine the builtins signature for builtins that use parameters defined as typedef ext_vector_type__ __attribute__((ext_vector_type())) or template using _ExtVector __attribute__((ext_vector_type(N))) = T; For example: typedef double ext_vector_type_4_double __attribute__((ext_vector_type(4))) --- .../target-builtins-prototype-parser.td | 115 ++++++++++++++++++ clang/utils/TableGen/ClangBuiltinsEmitter.cpp | 66 +++++++++- 2 files changed, 178 insertions(+), 3 deletions(-) create mode 100644 clang/test/TableGen/target-builtins-prototype-parser.td diff --git a/clang/test/TableGen/target-builtins-prototype-parser.td b/clang/test/TableGen/target-builtins-prototype-parser.td new file mode 100644 index 000000000000..3d6c92341ac4 --- /dev/null +++ b/clang/test/TableGen/target-builtins-prototype-parser.td @@ -0,0 +1,115 @@ +// RUN: clang-tblgen -I %p/../../../clang/include/ %s --gen-clang-builtins | FileCheck %s +// RUN: not clang-tblgen -I %p/../../../clang/include/ %s --gen-clang-builtins -DERROR_EXPECTED_LANES 2>&1 | FileCheck %s --check-prefix=ERROR_EXPECTED_LANES +// RUN: not clang-tblgen -I %p/../../../clang/include/ %s --gen-clang-builtins -DERROR_EXPECTED_COMMA 2>&1 | FileCheck %s --check-prefix=ERROR_EXPECTED_COMMA +// RUN: not clang-tblgen -I %p/../../../clang/include/ %s --gen-clang-builtins -DERROR_EXPECTED_TYPE 2>&1 | FileCheck %s --check-prefix=ERROR_EXPECTED_TYPE +// RUN: not clang-tblgen -I %p/../../../clang/include/ %s --gen-clang-builtins -DERROR_EXPECTED_A 2>&1 | FileCheck %s --check-prefix=ERROR_EXPECTED_A +// RUN: not clang-tblgen -I %p/../../../clang/include/ %s --gen-clang-builtins -DERROR_EXPECTED_B 2>&1 | FileCheck %s --check-prefix=ERROR_EXPECTED_B +// RUN: not clang-tblgen -I %p/../../../clang/include/ %s --gen-clang-builtins -DERROR_EXPECTED_C 2>&1 | FileCheck %s --check-prefix=ERROR_EXPECTED_C +// RUN: not clang-tblgen -I %p/../../../clang/include/ %s --gen-clang-builtins -DERROR_EXPECTED_D 2>&1 | FileCheck %s --check-prefix=ERROR_EXPECTED_D + +include "clang/Basic/BuiltinsBase.td" + +def : Builtin { +// CHECK: BUILTIN(__builtin_01, "E8idE4b", "") + let Prototype = "_ExtVector<8,int>(double, _ExtVector<4, bool>)"; + let Spellings = ["__builtin_01"]; +} + +def : Builtin { +// CHECK: BUILTIN(__builtin_02, "E8UiE4s", "") + let Prototype = "_ExtVector<8,unsigned int>(_ExtVector<4, short>)"; + let Spellings = ["__builtin_02"]; +} + +def : Builtin { +// CHECK: BUILTIN(__builtin_03, "di", "") + let Prototype = "double(int)"; + let Spellings = ["__builtin_03"]; +} + +def : Builtin { +// CHECK: BUILTIN(__builtin_04, "diIUi", "") + let Prototype = "double(int, _Constant unsigned int)"; + let Spellings = ["__builtin_04"]; +} + +def : Builtin { +// CHECK: BUILTIN(__builtin_05, "v&v&", "") + let Prototype = "void&(void&)"; + let Spellings = ["__builtin_05"]; +} + +def : Builtin { +// CHECK: BUILTIN(__builtin_06, "v*v*cC*.", "") + let Prototype = "void*(void*, char const*, ...)"; + let Spellings = ["__builtin_06"]; +} + +def : Builtin { +// CHECK: BUILTIN(__builtin_07, "E8iE4dE4b.", "") + let Prototype = "_ExtVector<8, int>(_ExtVector<4,double>, _ExtVector<4, bool>, ...)"; + let Spellings = ["__builtin_07"]; +} + +def : Builtin { +// CHECK: BUILTIN(__builtin_08, "di*R", "") + let Prototype = "double(int * restrict)"; + let Spellings = ["__builtin_08"]; +} + +#ifdef ERROR_EXPECTED_LANES +def : Builtin { +// ERROR_EXPECTED_LANES: :[[# @LINE + 1]]:7: error: Expected number of lanes after '_ExtVector<' + let Prototype = "_ExtVector(double)"; + let Spellings = ["__builtin_test_use_clang_extended_vectors"]; +} +#endif + +#ifdef ERROR_EXPECTED_COMMA +def : Builtin { +// ERROR_EXPECTED_COMMA: :[[# @LINE + 1]]:7: error: Expected ',' after number of lanes in '_ExtVector<' + let Prototype = "_ExtVector<8 int>(double)"; + let Spellings = ["__builtin_test_use_clang_extended_vectors"]; +} +#endif + +#ifdef ERROR_EXPECTED_TYPE +def : Builtin { +// ERROR_EXPECTED_TYPE: :[[# @LINE + 1]]:7: error: Expected '>' after scalar type in '_ExtVector' + let Prototype = "_ExtVector<8, int (double)"; + let Spellings = ["__builtin_test_use_clang_extended_vectors"]; +} +#endif + +#ifdef ERROR_EXPECTED_A +def : Builtin { +// ERROR_EXPECTED_A: :[[# @LINE + 1]]:7: error: Expected '<' after '_ExtVector' + let Prototype = "_ExtVector(8, int) (double)"; + let Spellings = ["__builtin_test_use_clang_extended_vectors"]; +} +#endif + +#ifdef ERROR_EXPECTED_B +def : Builtin { +// ERROR_EXPECTED_B: :[[# @LINE + 1]]:7: error: Expected '<' after '_ExtVector' + let Prototype = "double(_ExtVector(8, int))"; + let Spellings = ["__builtin_test_use_clang_extended_vectors"]; +} +#endif + +#ifdef ERROR_EXPECTED_C +def : Builtin { +// ERROR_EXPECTED_C: :[[# @LINE + 1]]:7: error: Unknown Type: _EtxVector<8, int> + let Prototype = "_EtxVector<8, int>(void)"; + let Spellings = ["__builtin_test_use_clang_extended_vectors"]; +} +#endif + +#ifdef ERROR_EXPECTED_D +def : Builtin { +// ERROR_EXPECTED_D: :[[# @LINE + 1]]:7: error: Expected number of lanes after '_ExtVector<' + let Prototype = "_ExtVector<>(void)"; + let Spellings = ["__builtin_test_use_clang_extended_vectors"]; +} +#endif + diff --git a/clang/utils/TableGen/ClangBuiltinsEmitter.cpp b/clang/utils/TableGen/ClangBuiltinsEmitter.cpp index 48f55b8af97e..94f12a08164f 100644 --- a/clang/utils/TableGen/ClangBuiltinsEmitter.cpp +++ b/clang/utils/TableGen/ClangBuiltinsEmitter.cpp @@ -47,9 +47,47 @@ private: if (!Prototype.ends_with(")")) PrintFatalError(Loc, "Expected closing brace at end of prototype"); Prototype = Prototype.drop_back(); - for (auto T = Prototype.split(','); !T.first.empty(); - Prototype = T.second, T = Prototype.split(',')) - ParseType(T.first); + + // Look through the input parameters. + const size_t end = Prototype.size(); + for (size_t I = 0; I != end;) { + const StringRef Current = Prototype.substr(I, end); + // Skip any leading space or commas + if (Current.starts_with(" ") || Current.starts_with(",")) { + ++I; + continue; + } + + // Check if we are in _ExtVector. We do this first because + // extended vectors are written in template form with the syntax + // _ExtVector< ..., ...>, so we need to make sure we are not + // detecting the comma of the template class as a separator for + // the parameters of the prototype. Note: the assumption is that + // we cannot have nested _ExtVector. + if (Current.starts_with("_ExtVector<")) { + const size_t EndTemplate = Current.find('>', 0); + ParseType(Current.substr(0, EndTemplate + 1)); + // Move the prototype beyond _ExtVector<...> + I += EndTemplate + 1; + continue; + } + + // We know that we are past _ExtVector, therefore the first seen + // comma is the boundary of a parameter in the prototype. + if (size_t CommaPos = Current.find(',', 0)) { + if (CommaPos != StringRef::npos) { + StringRef T = Current.substr(0, CommaPos); + ParseType(T); + // Move the prototype beyond the comma. + I += CommaPos + 1; + continue; + } + } + + // No more commas, parse final parameter. + ParseType(Current); + I = end; + } } void ParseType(StringRef T) { @@ -85,6 +123,28 @@ private: if (Substitution.empty()) PrintFatalError(Loc, "Not a template"); ParseType(Substitution); + } else if (T.consume_front("_ExtVector")) { + // Clang extended vector types are mangled as follows: + // + // '_ExtVector<' ',' '>' + + // Before parsing T(=), make sure the syntax of + // `_ExtVector` is correct... + if (!T.consume_front("<")) + PrintFatalError(Loc, "Expected '<' after '_ExtVector'"); + unsigned long long Lanes; + if (llvm::consumeUnsignedInteger(T, 10, Lanes)) + PrintFatalError(Loc, "Expected number of lanes after '_ExtVector<'"); + Type += "E" + std::to_string(Lanes); + if (!T.consume_front(",")) + PrintFatalError(Loc, + "Expected ',' after number of lanes in '_ExtVector<'"); + if (!T.consume_back(">")) + PrintFatalError( + Loc, "Expected '>' after scalar type in '_ExtVector'"); + + // ...all good, we can check if we have a valid ``. + ParseType(T); } else { auto ReturnTypeVal = StringSwitch(T) .Case("__builtin_va_list_ref", "A") -- GitLab From 90e9e962e18fc4304c6aba81de2bb53069bcd358 Mon Sep 17 00:00:00 2001 From: Andrei Golubev Date: Tue, 5 Mar 2024 09:07:43 +0200 Subject: [PATCH 097/929] [mlir][NFC] Apply rule of five to *Pass classes (#80998) Define all special member functions for mlir::Pass, mlir::OperationPass, mlir::PassWrapper and PassGen types since these classes explicitly specify copy-ctor. This, subsequently, should silence static analysis checkers that report rule-of-3 / rule-of-5 violations. Given the nature of the types, however, mark other special member functions deleted: the semantics of a Pass type object seems to be that it is only ever created by being wrapped in a smart pointer, so the special member functions are never to be used externally (except for the copy-ctor - it is "special" since it is a "delegating" ctor for derived pass types to use during cloning - see https://reviews.llvm.org/D104302 for details). Deleting other member functions means that `Pass x(std::move(y))` - that used to silently work (via copy-ctor) - would fail to compile now. Yet, as the copy ctors through the hierarchy are under 'protected' access, the issue is unlikely to appear in practice. Co-authored-by: Asya Pronina Co-authored-by: Harald Rotuna --- mlir/include/mlir/Pass/Pass.h | 19 +++++++++++++++++++ mlir/tools/mlir-tblgen/PassGen.cpp | 8 ++++++++ 2 files changed, 27 insertions(+) diff --git a/mlir/include/mlir/Pass/Pass.h b/mlir/include/mlir/Pass/Pass.h index 121b253eb83f..070e0cad3878 100644 --- a/mlir/include/mlir/Pass/Pass.h +++ b/mlir/include/mlir/Pass/Pass.h @@ -161,6 +161,9 @@ protected: explicit Pass(TypeID passID, std::optional opName = std::nullopt) : passID(passID), opName(opName) {} Pass(const Pass &other) : Pass(other.passID, other.opName) {} + Pass &operator=(const Pass &) = delete; + Pass(Pass &&) = delete; + Pass &operator=(Pass &&) = delete; /// Returns the current pass state. detail::PassExecutionState &getPassState() { @@ -349,9 +352,15 @@ private: /// - A 'std::unique_ptr clonePass() const' method. template class OperationPass : public Pass { +public: + ~OperationPass() = default; + protected: OperationPass(TypeID passID) : Pass(passID, OpT::getOperationName()) {} OperationPass(const OperationPass &) = default; + OperationPass &operator=(const OperationPass &) = delete; + OperationPass(OperationPass &&) = delete; + OperationPass &operator=(OperationPass &&) = delete; /// Support isa/dyn_cast functionality. static bool classof(const Pass *pass) { @@ -388,9 +397,15 @@ protected: /// - A 'std::unique_ptr clonePass() const' method. template <> class OperationPass : public Pass { +public: + ~OperationPass() = default; + protected: OperationPass(TypeID passID) : Pass(passID) {} OperationPass(const OperationPass &) = default; + OperationPass &operator=(const OperationPass &) = delete; + OperationPass(OperationPass &&) = delete; + OperationPass &operator=(OperationPass &&) = delete; /// Indicate if the current pass can be scheduled on the given operation type. /// By default, generic operation passes can be scheduled on any operation. @@ -444,10 +459,14 @@ public: static bool classof(const Pass *pass) { return pass->getTypeID() == TypeID::get(); } + ~PassWrapper() = default; protected: PassWrapper() : BaseT(TypeID::get()) {} PassWrapper(const PassWrapper &) = default; + PassWrapper &operator=(const PassWrapper &) = delete; + PassWrapper(PassWrapper &&) = delete; + PassWrapper &operator=(PassWrapper &&) = delete; /// Returns the derived pass name. StringRef getName() const override { return llvm::getTypeName(); } diff --git a/mlir/tools/mlir-tblgen/PassGen.cpp b/mlir/tools/mlir-tblgen/PassGen.cpp index 11af6497cecf..90aa67115a40 100644 --- a/mlir/tools/mlir-tblgen/PassGen.cpp +++ b/mlir/tools/mlir-tblgen/PassGen.cpp @@ -183,6 +183,10 @@ public: {0}Base() : {1}(::mlir::TypeID::get()) {{} {0}Base(const {0}Base &other) : {1}(other) {{} + {0}Base& operator=(const {0}Base &) = delete; + {0}Base({0}Base &&) = delete; + {0}Base& operator=({0}Base &&) = delete; + ~{0}Base() = default; /// Returns the command-line argument attached to this pass. static constexpr ::llvm::StringLiteral getArgumentName() { @@ -380,6 +384,10 @@ public: {0}Base() : {1}(::mlir::TypeID::get()) {{} {0}Base(const {0}Base &other) : {1}(other) {{} + {0}Base& operator=(const {0}Base &) = delete; + {0}Base({0}Base &&) = delete; + {0}Base& operator=({0}Base &&) = delete; + ~{0}Base() = default; /// Returns the command-line argument attached to this pass. static constexpr ::llvm::StringLiteral getArgumentName() { -- GitLab From a36b73e5a7f5b2f4b65f67544922be35bf81a67e Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Fri, 1 Mar 2024 00:52:55 -0800 Subject: [PATCH 098/929] Apply clang-tidy fixes for modernize-loop-convert in Barvinok.cpp (NFC) --- mlir/lib/Analysis/Presburger/Barvinok.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mlir/lib/Analysis/Presburger/Barvinok.cpp b/mlir/lib/Analysis/Presburger/Barvinok.cpp index 4be81c25951b..da45bdc6a978 100644 --- a/mlir/lib/Analysis/Presburger/Barvinok.cpp +++ b/mlir/lib/Analysis/Presburger/Barvinok.cpp @@ -613,10 +613,10 @@ void normalizeDenominatorExponents(int &sign, QuasiPolynomial &num, // denominator, and convert them to their absolute values. unsigned numNegExps = 0; Fraction sumNegExps(0, 1); - for (unsigned j = 0, e = dens.size(); j < e; ++j) { - if (dens[j] < 0) { + for (const auto &den : dens) { + if (den < 0) { numNegExps += 1; - sumNegExps += dens[j]; + sumNegExps += den; } } @@ -722,8 +722,8 @@ mlir::presburger::detail::computeNumTerms(const GeneratingFunction &gf) { // Then, using the formula for geometric series, we replace each (1 - // (s+1)^(dens[j])) with // (-s)(\sum_{0 ≤ k < dens[j]} (s+1)^k). - for (unsigned j = 0, e = dens.size(); j < e; ++j) - dens[j] = abs(dens[j]) - 1; + for (auto &j : dens) + j = abs(j) - 1; // Note that at this point, the semantics of `dens[j]` changes to mean // a term (\sum_{0 ≤ k ≤ dens[j]} (s+1)^k). The denominator is, as before, // a product of these terms. -- GitLab From 991541814459bbee94db178a1328d344b2869d59 Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Fri, 1 Mar 2024 00:53:25 -0800 Subject: [PATCH 099/929] Apply clang-tidy fixes for modernize-use-emplace in Barvinok.cpp (NFC) --- mlir/lib/Analysis/Presburger/Barvinok.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mlir/lib/Analysis/Presburger/Barvinok.cpp b/mlir/lib/Analysis/Presburger/Barvinok.cpp index da45bdc6a978..26039811f0b0 100644 --- a/mlir/lib/Analysis/Presburger/Barvinok.cpp +++ b/mlir/lib/Analysis/Presburger/Barvinok.cpp @@ -640,7 +640,7 @@ std::vector getBinomialCoefficients(QuasiPolynomial n, unsigned numParams = n.getNumInputs(); std::vector coefficients; coefficients.reserve(r + 1); - coefficients.push_back(QuasiPolynomial(numParams, 1)); + coefficients.emplace_back(numParams, 1); for (unsigned j = 1; j <= r; ++j) // We use the recursive formula for binomial coefficients here and below. coefficients.push_back( @@ -655,7 +655,7 @@ std::vector getBinomialCoefficients(QuasiPolynomial n, std::vector getBinomialCoefficients(Fraction n, Fraction r) { std::vector coefficients; coefficients.reserve((int64_t)floor(r)); - coefficients.push_back(1); + coefficients.emplace_back(1); for (unsigned j = 1; j <= r; ++j) coefficients.push_back(coefficients[j - 1] * (n - (j - 1)) / (j)); return coefficients; -- GitLab From 2db8b9413862a159cd77ba47eda56abcda1b6dfd Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Fri, 1 Mar 2024 00:53:56 -0800 Subject: [PATCH 100/929] Apply clang-tidy fixes for performance-unnecessary-value-param in Barvinok.cpp (NFC) --- mlir/include/mlir/Analysis/Presburger/Barvinok.h | 4 ++-- mlir/lib/Analysis/Presburger/Barvinok.cpp | 13 +++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/mlir/include/mlir/Analysis/Presburger/Barvinok.h b/mlir/include/mlir/Analysis/Presburger/Barvinok.h index f730a0739333..cd1ea3a9571b 100644 --- a/mlir/include/mlir/Analysis/Presburger/Barvinok.h +++ b/mlir/include/mlir/Analysis/Presburger/Barvinok.h @@ -74,7 +74,7 @@ inline PolyhedronH defineHRep(int numVars, int numSymbols = 0) { /// Barvinok, A., and J. E. Pommersheim. "An algorithmic theory of lattice /// points in polyhedra." p. 107 If it has more rays than the dimension, return /// 0. -MPInt getIndex(ConeV cone); +MPInt getIndex(const ConeV &cone); /// Given a cone in H-representation, return its dual. The dual cone is in /// V-representation. @@ -91,7 +91,7 @@ ConeH getDual(ConeV cone); /// The input cone must be unimodular; it assert-fails otherwise. GeneratingFunction computeUnimodularConeGeneratingFunction(ParamPoint vertex, int sign, - ConeH cone); + const ConeH &cone); /// Find the solution of a set of equations that express affine constraints /// between a set of variables and a set of parameters. The solution expresses diff --git a/mlir/lib/Analysis/Presburger/Barvinok.cpp b/mlir/lib/Analysis/Presburger/Barvinok.cpp index 26039811f0b0..8952f867c4d8 100644 --- a/mlir/lib/Analysis/Presburger/Barvinok.cpp +++ b/mlir/lib/Analysis/Presburger/Barvinok.cpp @@ -61,7 +61,7 @@ ConeH mlir::presburger::detail::getDual(ConeV cone) { } /// Find the index of a cone in V-representation. -MPInt mlir::presburger::detail::getIndex(ConeV cone) { +MPInt mlir::presburger::detail::getIndex(const ConeV &cone) { if (cone.getNumRows() > cone.getNumColumns()) return MPInt(0); @@ -79,7 +79,7 @@ MPInt mlir::presburger::detail::getIndex(ConeV cone) { /// coefficients. GeneratingFunction mlir::presburger::detail::computeUnimodularConeGeneratingFunction( - ParamPoint vertex, int sign, ConeH cone) { + ParamPoint vertex, int sign, const ConeH &cone) { // Consider a cone with H-representation [0 -1]. // [-1 -2] // Let the vertex be given by the matrix [ 2 2 0], with 2 params. @@ -556,8 +556,8 @@ QuasiPolynomial mlir::presburger::detail::getCoefficientInRationalFunction( /// v represents the affine functions whose floors are multiplied by the /// generators, and ds represents the list of generators. std::pair> -substituteMuInTerm(unsigned numParams, ParamPoint v, std::vector ds, - Point mu) { +substituteMuInTerm(unsigned numParams, const ParamPoint &v, + const std::vector &ds, const Point &mu) { unsigned numDims = mu.size(); #ifndef NDEBUG for (const Point &d : ds) @@ -635,7 +635,7 @@ void normalizeDenominatorExponents(int &sign, QuasiPolynomial &num, /// Compute the binomial coefficients nCi for 0 ≤ i ≤ r, /// where n is a QuasiPolynomial. -std::vector getBinomialCoefficients(QuasiPolynomial n, +std::vector getBinomialCoefficients(const QuasiPolynomial &n, unsigned r) { unsigned numParams = n.getNumInputs(); std::vector coefficients; @@ -652,7 +652,8 @@ std::vector getBinomialCoefficients(QuasiPolynomial n, /// Compute the binomial coefficients nCi for 0 ≤ i ≤ r, /// where n is a QuasiPolynomial. -std::vector getBinomialCoefficients(Fraction n, Fraction r) { +std::vector getBinomialCoefficients(const Fraction &n, + const Fraction &r) { std::vector coefficients; coefficients.reserve((int64_t)floor(r)); coefficients.emplace_back(1); -- GitLab From 1934fc6a0cad62cb80d66834e5481cb17f517802 Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Fri, 1 Mar 2024 00:54:25 -0800 Subject: [PATCH 101/929] Apply clang-tidy fixes for readability-container-size-empty in Barvinok.cpp (NFC) --- mlir/lib/Analysis/Presburger/Barvinok.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mlir/lib/Analysis/Presburger/Barvinok.cpp b/mlir/lib/Analysis/Presburger/Barvinok.cpp index 8952f867c4d8..0b55185c4378 100644 --- a/mlir/lib/Analysis/Presburger/Barvinok.cpp +++ b/mlir/lib/Analysis/Presburger/Barvinok.cpp @@ -515,8 +515,7 @@ Point mlir::presburger::detail::getNonOrthogonalVector( /// barvinokalgorithm-latte1.pdf, p. 1285 QuasiPolynomial mlir::presburger::detail::getCoefficientInRationalFunction( unsigned power, ArrayRef num, ArrayRef den) { - assert(den.size() != 0 && - "division by empty denominator in rational function!"); + assert(!den.empty() && "division by empty denominator in rational function!"); unsigned numParam = num[0].getNumInputs(); // We use the `isEqual` method of PresburgerSpace, which QuasiPolynomial -- GitLab From 1837579bbc9c5e0b49a54e60d99eed3d38b47d3b Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Fri, 1 Mar 2024 00:55:30 -0800 Subject: [PATCH 102/929] Apply clang-tidy fixes for readability-simplify-boolean-expr in IntegerRelation.cpp (NFC) --- mlir/lib/Analysis/Presburger/IntegerRelation.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mlir/lib/Analysis/Presburger/IntegerRelation.cpp b/mlir/lib/Analysis/Presburger/IntegerRelation.cpp index 2ac271e2e055..a3f971db4bd4 100644 --- a/mlir/lib/Analysis/Presburger/IntegerRelation.cpp +++ b/mlir/lib/Analysis/Presburger/IntegerRelation.cpp @@ -730,9 +730,7 @@ bool IntegerRelation::isEmpty() const { } bool IntegerRelation::isObviouslyEmpty() const { - if (isEmptyByGCDTest() || hasInvalidConstraint()) - return true; - return false; + return isEmptyByGCDTest() || hasInvalidConstraint(); } // Runs the GCD test on all equality constraints. Returns 'true' if this test -- GitLab From 46f65e45e0f5ce4cc0edabceebee681231d24687 Mon Sep 17 00:00:00 2001 From: Congcong Cai Date: Tue, 5 Mar 2024 15:33:49 +0800 Subject: [PATCH 103/929] [mlir]use correct iterator when eraseOp (#83444) #66771 introduce `llvm::post_order(&r.front())` which is equal to `r.front().getSuccessor(...)`. It will visit the succ block of current block. But actually here need to visit all block of region in reverse order. Fixes: #77420. --- mlir/lib/IR/PatternMatch.cpp | 5 ++++- mlir/test/Transforms/gh-77420.mlir | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 mlir/test/Transforms/gh-77420.mlir diff --git a/mlir/lib/IR/PatternMatch.cpp b/mlir/lib/IR/PatternMatch.cpp index 5ba5328f14b8..8796289d7257 100644 --- a/mlir/lib/IR/PatternMatch.cpp +++ b/mlir/lib/IR/PatternMatch.cpp @@ -229,7 +229,10 @@ void RewriterBase::eraseOp(Operation *op) { // until the region is empty. (The block graph could be disconnected.) while (!r.empty()) { SmallVector erasedBlocks; - for (Block *b : llvm::post_order(&r.front())) { + // Some blocks may have invalid successor, use a set including nullptr + // to avoid null pointer. + llvm::SmallPtrSet visited{nullptr}; + for (Block *b : llvm::post_order_ext(&r.front(), visited)) { // Visit ops in reverse order. for (Operation &op : llvm::make_early_inc_range(ReverseIterator::makeIterable(*b))) diff --git a/mlir/test/Transforms/gh-77420.mlir b/mlir/test/Transforms/gh-77420.mlir new file mode 100644 index 000000000000..0037cec0a96a --- /dev/null +++ b/mlir/test/Transforms/gh-77420.mlir @@ -0,0 +1,21 @@ +// RUN: mlir-opt --canonicalize %s | FileCheck %s + + +module { + +// CHECK: func.func @f() { +// CHECK-NEXT: return +// CHECK-NEXT: } + func.func @f() { + return + ^bb1: // no predecessors + omp.parallel { + %0 = llvm.intr.stacksave : !llvm.ptr + llvm.br ^bb1 + ^bb1: // pred: ^bb0 + omp.terminator + } + return + } + +} -- GitLab From eaa9ef678c63bf392ec2d5b736605db7ea7e7338 Mon Sep 17 00:00:00 2001 From: Jinyang He Date: Tue, 5 Mar 2024 15:50:14 +0800 Subject: [PATCH 104/929] [lld][LoongArch] Support the R_LARCH_{ADD,SUB}_ULEB128 relocation types (#81133) For a label difference like `.uleb128 A-B`, MC generates a pair of R_LARCH_{ADD,SUB}_ULEB128 if A-B cannot be folded as a constant. GNU assembler generates a pair of relocations in more cases (when A or B is in a code section with linker relaxation). It is similar to RISCV. R_LARCH_{ADD,SUB}_ULEB128 relocations are created by Clang and GCC in `.gcc_except_table` and other debug sections with linker relaxation enabled. On LoongArch, first read the buf and count the available space. Then add or sub the value. Finally truncate the expected value and fill it into the available space. --- lld/ELF/Arch/LoongArch.cpp | 19 +++++ lld/test/ELF/loongarch-reloc-leb128.s | 102 ++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 lld/test/ELF/loongarch-reloc-leb128.s diff --git a/lld/ELF/Arch/LoongArch.cpp b/lld/ELF/Arch/LoongArch.cpp index e0f9abfb382a..5f0b73df48c9 100644 --- a/lld/ELF/Arch/LoongArch.cpp +++ b/lld/ELF/Arch/LoongArch.cpp @@ -11,6 +11,7 @@ #include "Symbols.h" #include "SyntheticSections.h" #include "Target.h" +#include "llvm/Support/LEB128.h" using namespace llvm; using namespace llvm::object; @@ -155,6 +156,16 @@ static bool isJirl(uint32_t insn) { return (insn & 0xfc000000) == JIRL; } +static void handleUleb128(uint8_t *loc, uint64_t val) { + const uint32_t maxcount = 1 + 64 / 7; + uint32_t count; + uint64_t orig = decodeULEB128(loc, &count); + if (count > maxcount) + errorOrWarn(getErrorLocation(loc) + "extra space for uleb128"); + uint64_t mask = count < maxcount ? (1ULL << 7 * count) - 1 : -1ULL; + encodeULEB128((orig + val) & mask, loc, count); +} + LoongArch::LoongArch() { // The LoongArch ISA itself does not have a limit on page sizes. According to // the ISA manual, the PS (page size) field in MTLB entries and CSR.STLBPS is @@ -396,11 +407,13 @@ RelExpr LoongArch::getRelExpr(const RelType type, const Symbol &s, case R_LARCH_ADD16: case R_LARCH_ADD32: case R_LARCH_ADD64: + case R_LARCH_ADD_ULEB128: case R_LARCH_SUB6: case R_LARCH_SUB8: case R_LARCH_SUB16: case R_LARCH_SUB32: case R_LARCH_SUB64: + case R_LARCH_SUB_ULEB128: // The LoongArch add/sub relocs behave like the RISCV counterparts; reuse // the RelExpr to avoid code duplication. return R_RISCV_ADD; @@ -635,6 +648,9 @@ void LoongArch::relocate(uint8_t *loc, const Relocation &rel, case R_LARCH_ADD64: write64le(loc, read64le(loc) + val); return; + case R_LARCH_ADD_ULEB128: + handleUleb128(loc, val); + return; case R_LARCH_SUB6: *loc = (*loc & 0xc0) | ((*loc - val) & 0x3f); return; @@ -650,6 +666,9 @@ void LoongArch::relocate(uint8_t *loc, const Relocation &rel, case R_LARCH_SUB64: write64le(loc, read64le(loc) - val); return; + case R_LARCH_SUB_ULEB128: + handleUleb128(loc, -val); + return; case R_LARCH_MARK_LA: case R_LARCH_MARK_PCREL: diff --git a/lld/test/ELF/loongarch-reloc-leb128.s b/lld/test/ELF/loongarch-reloc-leb128.s new file mode 100644 index 000000000000..9e6f221e62b6 --- /dev/null +++ b/lld/test/ELF/loongarch-reloc-leb128.s @@ -0,0 +1,102 @@ +# REQUIRES: loongarch +# RUN: rm -rf %t && split-file %s %t && cd %t + +# RUN: llvm-mc --filetype=obj --triple=loongarch64 --mattr=+relax a.s -o a.o +# RUN: llvm-readobj -r -x .gcc_except_table -x .debug_rnglists -x .debug_loclists a.o | FileCheck %s --check-prefix=REL +# RUN: ld.lld -shared --gc-sections a.o -o a.so +# RUN: llvm-readelf -x .gcc_except_table -x .debug_rnglists -x .debug_loclists a.so | FileCheck %s + +# RUN: llvm-mc --filetype=obj --triple=loongarch32 --mattr=+relax a.s -o a32.o +# RUN: llvm-readobj -r -x .gcc_except_table -x .debug_rnglists -x .debug_loclists a32.o | FileCheck %s --check-prefix=REL +# RUN: ld.lld -shared --gc-sections a32.o -o a32.so +# RUN: llvm-readelf -x .gcc_except_table -x .debug_rnglists -x .debug_loclists a32.so | FileCheck %s + +# RUN: llvm-mc --filetype=obj --triple=loongarch32 --mattr=+relax extraspace.s -o extraspace32.o +# RUN: llvm-mc --filetype=obj --triple=loongarch64 --mattr=+relax extraspace.s -o extraspace64.o +# RUN: not ld.lld -shared extraspace32.o 2>&1 | FileCheck %s --check-prefix=ERROR +# RUN: not ld.lld -shared extraspace64.o 2>&1 | FileCheck %s --check-prefix=ERROR +# ERROR: error: extraspace{{.*}}.o:(.rodata+0x0): extra space for uleb128 + +#--- a.s +.cfi_startproc +.cfi_lsda 0x1b,.LLSDA0 +.cfi_endproc + +.section .text.w,"axR" +break 0; break 0; break 0; w1: + .p2align 4 # 4 bytes after relaxation +w2: break 0 + +.section .text.x,"ax" +break 0; break 0; break 0; x1: + .p2align 4 # 4 bytes after relaxation +x2: break 0 + +.section .gcc_except_table,"a" +.LLSDA0: +.uleb128 w2-w1+116 # initial value: 0x0080 +.uleb128 w1-w2+141 # initial value: 0x0080 +.uleb128 w2-w1+16372 # initial value: 0x008080 +.uleb128 w1-w2+16397 # initial value: 0x008080 +.uleb128 w2-w1+2097140 # initial value: 0x00808080 +.uleb128 w1-w2+2097165 # initial value: 0x00808080 + +.section .debug_rnglists +.uleb128 w2-w1+116 # initial value: 0x0080 +.uleb128 w1-w2+141 # initial value: 0x0080 +.uleb128 w2-w1+16372 # initial value: 0x008080 +.uleb128 w1-w2+16397 # initial value: 0x008080 +.uleb128 w2-w1+2097140 # initial value: 0x00808080 +.uleb128 w1-w2+2097165 # initial value: 0x00808080 + +.section .debug_loclists +.uleb128 x2-x1 # references discarded symbols + +# REL: Section ({{.*}}) .rela.debug_rnglists { +# REL-NEXT: 0x0 R_LARCH_ADD_ULEB128 w2 0x74 +# REL-NEXT: 0x0 R_LARCH_SUB_ULEB128 w1 0x0 +# REL-NEXT: 0x2 R_LARCH_ADD_ULEB128 w1 0x8D +# REL-NEXT: 0x2 R_LARCH_SUB_ULEB128 w2 0x0 +# REL-NEXT: 0x4 R_LARCH_ADD_ULEB128 w2 0x3FF4 +# REL-NEXT: 0x4 R_LARCH_SUB_ULEB128 w1 0x0 +# REL-NEXT: 0x7 R_LARCH_ADD_ULEB128 w1 0x400D +# REL-NEXT: 0x7 R_LARCH_SUB_ULEB128 w2 0x0 +# REL-NEXT: 0xA R_LARCH_ADD_ULEB128 w2 0x1FFFF4 +# REL-NEXT: 0xA R_LARCH_SUB_ULEB128 w1 0x0 +# REL-NEXT: 0xE R_LARCH_ADD_ULEB128 w1 0x20000D +# REL-NEXT: 0xE R_LARCH_SUB_ULEB128 w2 0x0 +# REL-NEXT: } +# REL: Section ({{.*}}) .rela.debug_loclists { +# REL-NEXT: 0x0 R_LARCH_ADD_ULEB128 x2 0x0 +# REL-NEXT: 0x0 R_LARCH_SUB_ULEB128 x1 0x0 +# REL-NEXT: } + +# REL: Hex dump of section '.gcc_except_table': +# REL-NEXT: 0x00000000 80008000 80800080 80008080 80008080 . +# REL-NEXT: 0x00000010 8000 . +# REL: Hex dump of section '.debug_rnglists': +# REL-NEXT: 0x00000000 80008000 80800080 80008080 80008080 . +# REL-NEXT: 0x00000010 8000 . +# REL: Hex dump of section '.debug_loclists': +# REL-NEXT: 0x00000000 00 . + +# CHECK: Hex dump of section '.gcc_except_table': +# CHECK-NEXT: 0x[[#%x,]] f8008901 f8ff0089 8001f8ff ff008980 . +# CHECK-NEXT: 0x[[#%x,]] 8001 . +# CHECK: Hex dump of section '.debug_rnglists': +# CHECK-NEXT: 0x00000000 f8008901 f8ff0089 8001f8ff ff008980 . +# CHECK-NEXT: 0x00000010 8001 . +# CHECK: Hex dump of section '.debug_loclists': +# CHECK-NEXT: 0x00000000 00 . + +#--- extraspace.s +.text +w1: + la.pcrel $t0, w1 +w2: + +.rodata +.reloc ., R_LARCH_ADD_ULEB128, w2 +.reloc ., R_LARCH_SUB_ULEB128, w1 +.fill 10, 1, 0x80 +.byte 0 -- GitLab From 0e337c67c8b9b0e04fb47712faba204c8d999af7 Mon Sep 17 00:00:00 2001 From: MalaySanghiIntel <148750629+MalaySanghiIntel@users.noreply.github.com> Date: Tue, 5 Mar 2024 13:21:43 +0530 Subject: [PATCH 105/929] Replace copy with a reference. (#82485) These are relatively larger structures and we don't update them so ref should be fine --- llvm/lib/CodeGen/MachinePipeliner.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp index 1bda19b2e6e9..d8cb68168833 100644 --- a/llvm/lib/CodeGen/MachinePipeliner.cpp +++ b/llvm/lib/CodeGen/MachinePipeliner.cpp @@ -1334,7 +1334,7 @@ private: Register Reg = getLoopPhiReg(*MI, OrigMBB); UpdateTargetRegs(Reg); } else { - for (auto Use : ROMap.find(MI)->getSecond().Uses) + for (auto &Use : ROMap.find(MI)->getSecond().Uses) UpdateTargetRegs(Use.RegUnit); } } @@ -1438,7 +1438,7 @@ private: const unsigned Iter = I - Stage; - for (auto Def : ROMap.find(MI)->getSecond().Defs) + for (auto &Def : ROMap.find(MI)->getSecond().Defs) InsertReg(LiveRegSets[Iter], Def.RegUnit); for (auto LastUse : LastUses[MI]) { -- GitLab From 3105cfe783d861e63c647469cc3a6c9b91f8bb4a Mon Sep 17 00:00:00 2001 From: Daniil Kovalev Date: Tue, 5 Mar 2024 11:20:16 +0300 Subject: [PATCH 106/929] [FileCheck] Fix parsing empty global and pseudo variable names (#83667) Reland #82595 with fixes of build failures related to colored output. See https://lab.llvm.org/buildbot/#/builders/139/builds/60549 Use `%ProtectFileCheckOutput` to avoid colored output. Original commit message below. In `Pattern::parseVariable`, for global variables (those starting with '$') and for pseudo variables (those starting with '@') the first character is consumed before actual variable name parsing. If the name is empty, it leads to out-of-bound access to the corresponding `StringRef`. This patch adds an if statement against the case described. --- llvm/lib/FileCheck/FileCheck.cpp | 6 ++++ llvm/test/FileCheck/empty-variable-name.txt | 32 +++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 llvm/test/FileCheck/empty-variable-name.txt diff --git a/llvm/lib/FileCheck/FileCheck.cpp b/llvm/lib/FileCheck/FileCheck.cpp index 6d3a2b9cf46f..8f80a69c4abd 100644 --- a/llvm/lib/FileCheck/FileCheck.cpp +++ b/llvm/lib/FileCheck/FileCheck.cpp @@ -297,6 +297,12 @@ Pattern::parseVariable(StringRef &Str, const SourceMgr &SM) { if (Str[0] == '$' || IsPseudo) ++I; + if (I == Str.size()) + return ErrorDiagnostic::get(SM, Str.slice(I, StringRef::npos), + StringRef("empty ") + + (IsPseudo ? "pseudo " : "global ") + + "variable name"); + if (!isValidVarNameStart(Str[I++])) return ErrorDiagnostic::get(SM, Str, "invalid variable name"); diff --git a/llvm/test/FileCheck/empty-variable-name.txt b/llvm/test/FileCheck/empty-variable-name.txt new file mode 100644 index 000000000000..c23542bbdf27 --- /dev/null +++ b/llvm/test/FileCheck/empty-variable-name.txt @@ -0,0 +1,32 @@ +a + +; RUN: %ProtectFileCheckOutput not FileCheck -input-file %s %s 2>&1 | \ +; RUN: FileCheck -check-prefix CHECK-ERROR -DDIR=%S \ +; RUN: --match-full-lines --strict-whitespace %s + +; CHECK: a[[]] +; CHECK-ERROR:[[DIR]]{{/|\\}}empty-variable-name.txt:7:13: error: empty variable name +; CHECK-ERROR-NEXT:; CHECK: a{{\[\[\]\]}} +; CHECK-ERROR-NEXT: ^ + +b + +; RUN: %ProtectFileCheckOutput not FileCheck -input-file %s -check-prefix CHECK-PSEUDO %s 2>&1 | \ +; RUN: FileCheck -check-prefix CHECK-ERROR-PSEUDO -DDIR=%S \ +; RUN: --match-full-lines --strict-whitespace %s + +; CHECK-PSEUDO: b[[@]] +; CHECK-ERROR-PSEUDO:[[DIR]]{{/|\\}}empty-variable-name.txt:18:21: error: empty pseudo variable name +; CHECK-ERROR-PSEUDO-NEXT:; CHECK-PSEUDO: b{{\[\[@\]\]}} +; CHECK-ERROR-PSEUDO-NEXT: ^ + +c + +; RUN: %ProtectFileCheckOutput not FileCheck -input-file %s -check-prefix CHECK-GLOBAL %s 2>&1 | \ +; RUN: FileCheck -check-prefix CHECK-ERROR-GLOBAL -DDIR=%S \ +; RUN: --match-full-lines --strict-whitespace %s + +; CHECK-GLOBAL: c[[$]] +; CHECK-ERROR-GLOBAL:[[DIR]]{{/|\\}}empty-variable-name.txt:29:21: error: empty global variable name +; CHECK-ERROR-GLOBAL-NEXT:; CHECK-GLOBAL: c{{\[\[\$\]\]}} +; CHECK-ERROR-GLOBAL-NEXT: ^ -- GitLab From ad5aea3712860f8173ea64773014f5b48fd08542 Mon Sep 17 00:00:00 2001 From: jeanPerier Date: Tue, 5 Mar 2024 09:23:13 +0100 Subject: [PATCH 107/929] Revert "[flang] disable memory-allocation-opt.fir test on windows" (#83822) Reverts llvm/llvm-project#83535 Bug fixed by https://github.com/llvm/llvm-project/pull/83768 --- flang/test/Fir/memory-allocation-opt.fir | 2 -- 1 file changed, 2 deletions(-) diff --git a/flang/test/Fir/memory-allocation-opt.fir b/flang/test/Fir/memory-allocation-opt.fir index 44fdaad4a5c2..cfbca2f83ef8 100644 --- a/flang/test/Fir/memory-allocation-opt.fir +++ b/flang/test/Fir/memory-allocation-opt.fir @@ -1,6 +1,4 @@ // RUN: fir-opt --memory-allocation-opt="dynamic-array-on-heap=true maximum-array-alloc-size=1024" %s | FileCheck %s -// FIXME: started crashing on windows https://github.com/llvm/llvm-project/issues/83534 -// UNSUPPORTED: system-windows // Test for size of array being too big. -- GitLab From 74dfded444ed97c1fe93cc541de1fcdd3a7f62ab Mon Sep 17 00:00:00 2001 From: jeanPerier Date: Tue, 5 Mar 2024 09:25:17 +0100 Subject: [PATCH 108/929] [flang] Deallocate structure constructor allocatable components (#83824) Allocatable components of structure constructors were not deallocated. Deallocate them without calling final subroutines. This was already properly done for array constructors. --- flang/docs/Extensions.md | 15 +++++++++++++++ flang/lib/Lower/ConvertExprToHLFIR.cpp | 10 ++++++++++ flang/test/Lower/HLFIR/structure-constructor.f90 | 12 ++++++++++++ 3 files changed, 37 insertions(+) diff --git a/flang/docs/Extensions.md b/flang/docs/Extensions.md index baecfd7c48fd..697bd131c04c 100644 --- a/flang/docs/Extensions.md +++ b/flang/docs/Extensions.md @@ -706,6 +706,21 @@ end `INDEX` include an optional `BACK=` argument, but it doesn't actually work. +* Allocatable components of array and structure constructors are deallocated + after use without calling final subroutines. + The standard does not specify when and how deallocation of array and structure + constructors allocatable components should happen. All compilers free the + memory after use, but the behavior when the allocatable component is a derived + type with finalization differ, especially when dealing with nested array and + structure constructors expressions. Some compilers call final routine for the + allocatable components of each constructor sub-expressions, some call it only + for the allocatable component of the top level constructor, and some only + deallocate the memory. Deallocating only the memory offers the most + flexibility when lowering such expressions, and it is not clear finalization + is desirable in such context (Fortran interop 1.6.2 in F2018 standards require + array and structure constructors not to be finalized, so it also makes sense + not to finalize their allocatable components when releasing their storage). + ## De Facto Standard Features * `EXTENDS_TYPE_OF()` returns `.TRUE.` if both of its arguments have the diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp index 68ecaa19d2d5..731c5072c45c 100644 --- a/flang/lib/Lower/ConvertExprToHLFIR.cpp +++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp @@ -1844,6 +1844,16 @@ private: builder.genIfThen(loc, isAlloc).genThen(genAssign).end(); } + if (fir::isRecordWithAllocatableMember(recTy)) { + // Deallocate allocatable components without calling final subroutines. + // The Fortran 2018 section 9.7.3.2 about deallocation is not ruling + // about the fate of allocatable components of structure constructors, + // and there is no behavior consensus in other compilers. + fir::FirOpBuilder *bldr = &builder; + getStmtCtx().attachCleanup([=]() { + fir::runtime::genDerivedTypeDestroyWithoutFinalization(*bldr, loc, box); + }); + } return varOp; } diff --git a/flang/test/Lower/HLFIR/structure-constructor.f90 b/flang/test/Lower/HLFIR/structure-constructor.f90 index ff0c27f274f3..d02427d2ff67 100644 --- a/flang/test/Lower/HLFIR/structure-constructor.f90 +++ b/flang/test/Lower/HLFIR/structure-constructor.f90 @@ -161,6 +161,8 @@ end subroutine test4 ! CHECK: hlfir.assign %[[VAL_26]] to %[[VAL_20]] realloc keep_lhs_len temporary_lhs : !fir.box>>>, !fir.ref>>>> ! CHECK: } ! CHECK: hlfir.assign %[[VAL_12]]#0 to %[[VAL_3]]#0 : !fir.ref>>>}>>, !fir.ref>>>}>> +! CHECK: %[[VAL_27:.*]] = fir.convert %[[VAL_13]] : (!fir.box>>>}>>) -> !fir.box +! CHECK: fir.call @_FortranADestroyWithoutFinalization(%[[VAL_27]] ! CHECK: return ! CHECK: } @@ -201,6 +203,8 @@ end subroutine test5 ! CHECK: hlfir.assign %[[VAL_24]] to %[[VAL_18]] realloc temporary_lhs : !fir.box>>>}>>>>, !fir.ref>>>}>>>>> ! CHECK: } ! CHECK: hlfir.assign %[[VAL_11]]#0 to %[[VAL_3]]#0 : !fir.ref>>>}>>>>}>>, !fir.ref>>>}>>>>}>> +! CHECK: %[[VAL_24:.*]] = fir.convert %[[VAL_12]] : (!fir.box>>>}>>>>}>>) -> !fir.box +! CHECK: fir.call @_FortranADestroyWithoutFinalization(%[[VAL_24]] ! CHECK: return ! CHECK: } @@ -291,6 +295,10 @@ end subroutine test6 ! CHECK: %[[VAL_70:.*]] = hlfir.as_expr %[[VAL_48]]#0 move %[[VAL_69]] : (!fir.heap}>>>, i1) -> !hlfir.expr<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>> ! CHECK: hlfir.assign %[[VAL_70]] to %[[VAL_44]] temporary_lhs : !hlfir.expr<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>, !fir.ref}>>> ! CHECK: hlfir.assign %[[VAL_20]]#0 to %[[VAL_12]]#0 : !fir.ref>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>, !fir.ref>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>> +! CHECK: %[[VAL_71:.*]] = fir.convert %[[VAL_21]] : (!fir.box>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>) -> !fir.box +! CHECK: fir.call @_FortranADestroyWithoutFinalization(%[[VAL_71]] +! CHECK: %[[VAL_72:.*]] = fir.convert %[[VAL_29]] : (!fir.box>>>}>>>>}>>) -> !fir.box +! CHECK: fir.call @_FortranADestroyWithoutFinalization(%[[VAL_72]] ! CHECK: return ! CHECK: } @@ -375,6 +383,8 @@ end subroutine test8 ! CHECK: hlfir.assign %[[VAL_29]] to %[[VAL_22]] realloc keep_lhs_len temporary_lhs : !fir.heap>, !fir.ref>>> ! CHECK: } ! CHECK: hlfir.assign %[[VAL_14]]#0 to %[[VAL_2]]#0 : !fir.ref>>}>>, !fir.ref>>}>> +! CHECK: %[[VAL_30:.*]] = fir.convert %[[VAL_15]] : (!fir.box>>}>>) -> !fir.box +! CHECK: fir.call @_FortranADestroyWithoutFinalization(%[[VAL_30]] ! CHECK: return ! CHECK: } @@ -410,6 +420,8 @@ end subroutine test9 ! CHECK: %[[VAL_20:.*]] = hlfir.designate %[[VAL_12]]#0{"c"} typeparams %[[VAL_19]] {fortran_attrs = #fir.var_attrs} : (!fir.ref>>}>>, index) -> !fir.ref>>> ! CHECK: hlfir.assign %[[VAL_11]]#0 to %[[VAL_20]] realloc keep_lhs_len temporary_lhs : !fir.ref>, !fir.ref>>> ! CHECK: hlfir.assign %[[VAL_12]]#0 to %[[VAL_2]]#0 : !fir.ref>>}>>, !fir.ref>>}>> +! CHECK: %[[VAL_21:.*]] = fir.convert %[[VAL_13]] : (!fir.box>>}>>) -> !fir.box +! CHECK: fir.call @_FortranADestroyWithoutFinalization(%[[VAL_21]] ! CHECK: return ! CHECK: } -- GitLab From 2984699a3d9266926302d65dc57993f875be67e5 Mon Sep 17 00:00:00 2001 From: jeanPerier Date: Tue, 5 Mar 2024 09:27:29 +0100 Subject: [PATCH 109/929] [flang] Implement passing of assumed-type actual arguments. (#83851) Passing `TYPE(*)`actual to `TYPE(*)` dummy was left TODO. Implement it. The difference with other actual arguments is that `TYPE(*)` are not represented as Fortran::evaluate::Expr, so inquiries on evaluate::Expr must be updated to use evaluate::ActualArgument or also handle semantics::Symbol case (except in portion of the code where `TYPE(*)` is impossible, where asserts are added). --- flang/lib/Lower/ConvertCall.cpp | 75 ++++-- flang/test/HLFIR/assumed-type-actual-args.f90 | 226 ++++++++++++++++++ 2 files changed, 284 insertions(+), 17 deletions(-) create mode 100644 flang/test/HLFIR/assumed-type-actual-args.f90 diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp index baf08b58a91b..6e3ce101ef1a 100644 --- a/flang/lib/Lower/ConvertCall.cpp +++ b/flang/lib/Lower/ConvertCall.cpp @@ -970,6 +970,18 @@ mlir::Value static getZeroLowerBounds(mlir::Location loc, return builder.genShift(loc, lowerBounds); } +static bool +isSimplyContiguous(const Fortran::evaluate::ActualArgument &arg, + Fortran::evaluate::FoldingContext &foldingContext) { + if (const auto *expr = arg.UnwrapExpr()) + return Fortran::evaluate::IsSimplyContiguous(*expr, foldingContext); + const Fortran::semantics::Symbol *sym = arg.GetAssumedTypeDummy(); + assert(sym && + "expect ActualArguments to be expression or assumed-type symbols"); + return sym->Rank() == 0 || + Fortran::evaluate::IsSimplyContiguous(*sym, foldingContext); +} + /// When dummy is not ALLOCATABLE, POINTER and is not passed in register, /// prepare the actual argument according to the interface. Do as needed: /// - address element if this is an array argument in an elemental call. @@ -985,7 +997,7 @@ static PreparedDummyArgument preparePresentUserCallActualArgument( const Fortran::lower::PreparedActualArgument &preparedActual, mlir::Type dummyType, const Fortran::lower::CallerInterface::PassedEntity &arg, - const Fortran::lower::SomeExpr &expr, CallContext &callContext) { + CallContext &callContext) { Fortran::evaluate::FoldingContext &foldingContext = callContext.converter.getFoldingContext(); @@ -1036,7 +1048,7 @@ static PreparedDummyArgument preparePresentUserCallActualArgument( const bool mustDoCopyInOut = actual.isArray() && arg.mustBeMadeContiguous() && (passingPolymorphicToNonPolymorphic || - !Fortran::evaluate::IsSimplyContiguous(expr, foldingContext)); + !isSimplyContiguous(*arg.entity, foldingContext)); const bool actualIsAssumedRank = actual.isAssumedRank(); // Create dummy type with actual argument rank when the dummy is an assumed @@ -1114,9 +1126,11 @@ static PreparedDummyArgument preparePresentUserCallActualArgument( arg.mayBeModifiedByCall() ? copyIn.getVar() : mlir::Value{}); } } else { + const Fortran::lower::SomeExpr *expr = arg.entity->UnwrapExpr(); + assert(expr && "expression actual argument cannot be an assumed type"); // The actual is an expression value, place it into a temporary // and register the temporary destruction after the call. - mlir::Type storageType = callContext.converter.genType(expr); + mlir::Type storageType = callContext.converter.genType(*expr); mlir::NamedAttribute byRefAttr = fir::getAdaptToByRefAttr(builder); hlfir::AssociateOp associate = hlfir::genAssociateExpr( loc, builder, entity, storageType, "", byRefAttr); @@ -1202,7 +1216,7 @@ static PreparedDummyArgument preparePresentUserCallActualArgument( if (auto baseBoxDummy = mlir::dyn_cast(dummyType)) if (baseBoxDummy.isAssumedRank()) if (const Fortran::semantics::Symbol *sym = - Fortran::evaluate::UnwrapWholeSymbolDataRef(expr)) + Fortran::evaluate::UnwrapWholeSymbolDataRef(*arg.entity)) if (Fortran::semantics::IsAssumedSizeArray(sym->GetUltimate())) TODO(loc, "passing assumed-size to assumed-rank array"); @@ -1224,10 +1238,10 @@ static PreparedDummyArgument prepareUserCallActualArgument( const Fortran::lower::PreparedActualArgument &preparedActual, mlir::Type dummyType, const Fortran::lower::CallerInterface::PassedEntity &arg, - const Fortran::lower::SomeExpr &expr, CallContext &callContext) { + CallContext &callContext) { if (!preparedActual.handleDynamicOptional()) - return preparePresentUserCallActualArgument( - loc, builder, preparedActual, dummyType, arg, expr, callContext); + return preparePresentUserCallActualArgument(loc, builder, preparedActual, + dummyType, arg, callContext); // Conditional dummy argument preparation. The actual may be absent // at runtime, causing any addressing, copy, and packaging to have @@ -1249,7 +1263,7 @@ static PreparedDummyArgument prepareUserCallActualArgument( builder.setInsertionPointToStart(preparationBlock); PreparedDummyArgument unconditionalDummy = preparePresentUserCallActualArgument(loc, builder, preparedActual, - dummyType, arg, expr, callContext); + dummyType, arg, callContext); builder.restoreInsertionPoint(insertPt); // TODO: when forwarding an optional to an optional of the same kind @@ -1291,10 +1305,11 @@ static PreparedDummyArgument prepareProcedurePointerActualArgument( const Fortran::lower::PreparedActualArgument &preparedActual, mlir::Type dummyType, const Fortran::lower::CallerInterface::PassedEntity &arg, - const Fortran::lower::SomeExpr &expr, CallContext &callContext) { + CallContext &callContext) { // NULL() actual to procedure pointer dummy - if (Fortran::evaluate::UnwrapExpr(expr) && + if (Fortran::evaluate::UnwrapExpr( + *arg.entity) && fir::isBoxProcAddressType(dummyType)) { auto boxTy{Fortran::lower::getUntypedBoxProcType(builder.getContext())}; auto tempBoxProc{builder.createTemporary(loc, boxTy)}; @@ -1335,9 +1350,6 @@ genUserCall(Fortran::lower::PreparedActualArguments &loweredActuals, caller.placeInput(arg, builder.genAbsentOp(loc, argTy)); continue; } - const auto *expr = arg.entity->UnwrapExpr(); - if (!expr) - TODO(loc, "assumed type actual argument"); switch (arg.passBy) { case PassBy::Value: { @@ -1380,7 +1392,7 @@ genUserCall(Fortran::lower::PreparedActualArguments &loweredActuals, case PassBy::BaseAddress: case PassBy::BoxChar: { PreparedDummyArgument preparedDummy = prepareUserCallActualArgument( - loc, builder, *preparedActual, argTy, arg, *expr, callContext); + loc, builder, *preparedActual, argTy, arg, callContext); callCleanUps.append(preparedDummy.cleanups.rbegin(), preparedDummy.cleanups.rend()); caller.placeInput(arg, preparedDummy.dummy); @@ -1388,7 +1400,7 @@ genUserCall(Fortran::lower::PreparedActualArguments &loweredActuals, case PassBy::BoxProcRef: { PreparedDummyArgument preparedDummy = prepareProcedurePointerActualArgument(loc, builder, *preparedActual, - argTy, arg, *expr, callContext); + argTy, arg, callContext); callCleanUps.append(preparedDummy.cleanups.rbegin(), preparedDummy.cleanups.rend()); caller.placeInput(arg, preparedDummy.dummy); @@ -1408,6 +1420,9 @@ genUserCall(Fortran::lower::PreparedActualArguments &loweredActuals, caller.placeInput(arg, actual); } break; case PassBy::MutableBox: { + const Fortran::lower::SomeExpr *expr = arg.entity->UnwrapExpr(); + // C709 and C710. + assert(expr && "cannot pass TYPE(*) to POINTER or ALLOCATABLE"); hlfir::Entity actual = preparedActual->getActual(loc, builder); if (Fortran::evaluate::UnwrapExpr( *expr)) { @@ -2405,8 +2420,34 @@ genProcedureRef(CallContext &callContext) { caller.getPassedArguments()) if (const auto *actual = arg.entity) { const auto *expr = actual->UnwrapExpr(); - if (!expr) - TODO(loc, "assumed type actual argument"); + if (!expr) { + // TYPE(*) actual argument. + const Fortran::evaluate::Symbol *assumedTypeSym = + actual->GetAssumedTypeDummy(); + if (!assumedTypeSym) + fir::emitFatalError( + loc, "expected assumed-type symbol as actual argument"); + std::optional var = + callContext.symMap.lookupVariableDefinition(*assumedTypeSym); + if (!var) + fir::emitFatalError(loc, "assumed-type symbol was not lowered"); + hlfir::Entity actual{*var}; + std::optional isPresent; + if (arg.isOptional()) { + // Passing an optional TYPE(*) to an optional TYPE(*). Note that + // TYPE(*) cannot be ALLOCATABLE/POINTER (C709) so there is no + // need to cover the case of passing an ALLOCATABLE/POINTER to an + // OPTIONAL. + fir::FirOpBuilder &builder = callContext.getBuilder(); + isPresent = + builder.create(loc, builder.getI1Type(), actual) + .getResult(); + } + loweredActuals.push_back(Fortran::lower::PreparedActualArgument{ + hlfir::Entity{*var}, isPresent}); + continue; + } + if (Fortran::evaluate::UnwrapExpr( *expr)) { if ((arg.passBy != diff --git a/flang/test/HLFIR/assumed-type-actual-args.f90 b/flang/test/HLFIR/assumed-type-actual-args.f90 new file mode 100644 index 000000000000..58c282b6ab18 --- /dev/null +++ b/flang/test/HLFIR/assumed-type-actual-args.f90 @@ -0,0 +1,226 @@ +! Test lowering to FIR of actual arguments that are assumed type +! variables (Fortran 2018 7.3.2.2 point 3). +! RUN: bbc --polymorphic-type -emit-hlfir -o - %s | FileCheck %s + +subroutine test1(x) + interface + subroutine s1(x) + type(*) :: x + end subroutine + end interface + type(*) :: x + call s1(x) +end subroutine + +subroutine test2(x) + interface + subroutine s2(x) + type(*) :: x(*) + end subroutine + end interface + type(*) :: x(*) + call s2(x) +end subroutine + +subroutine test3(x) + interface + subroutine s3(x) + type(*) :: x(:) + end subroutine + end interface + type(*) :: x(:) + call s3(x) +end subroutine + +subroutine test4(x) + interface + subroutine s4(x) + type(*) :: x(*) + end subroutine + end interface + type(*) :: x(:) + call s4(x) +end subroutine + +subroutine test3b(x) + interface + subroutine s3b(x) + type(*), optional, contiguous :: x(:) + end subroutine + end interface + type(*), optional :: x(:) + call s3b(x) +end subroutine + +subroutine test4b(x) + interface + subroutine s4b(x) + type(*), optional :: x(*) + end subroutine + end interface + type(*), optional :: x(:) + call s4b(x) +end subroutine + +subroutine test4c(x) + interface + subroutine s4c(x) + type(*), optional :: x(*) + end subroutine + end interface + type(*), contiguous, optional :: x(:) + call s4c(x) +end subroutine + +subroutine test4d(x) + interface + subroutine s4d(x) + type(*) :: x(*) + end subroutine + end interface + type(*), contiguous :: x(:) + call s4d(x) +end subroutine + +subroutine test5(x) + interface + subroutine s5(x) + type(*) :: x(..) + end subroutine + end interface + type(*) :: x(:) + call s5(x) +end subroutine + +subroutine test5b(x) + interface + subroutine s5b(x) + type(*), optional, contiguous :: x(..) + end subroutine + end interface + type(*), optional :: x(:) + call s5b(x) +end subroutine + +! CHECK-LABEL: func.func @_QPtest1( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref {fir.bindc_name = "x"}) { +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest1Ex"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: fir.call @_QPs1(%[[VAL_1]]#1) fastmath : (!fir.ref) -> () +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest2( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref> {fir.bindc_name = "x"}) { +! CHECK: %[[VAL_1:.*]] = arith.constant -1 : index +! CHECK: %[[VAL_2:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1> +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]](%[[VAL_2]]) {uniq_name = "_QFtest2Ex"} : (!fir.ref>, !fir.shape<1>) -> (!fir.box>, !fir.ref>) +! CHECK: fir.call @_QPs2(%[[VAL_3]]#1) fastmath : (!fir.ref>) -> () +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest3( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.box> {fir.bindc_name = "x"}) { +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest3Ex"} : (!fir.box>) -> (!fir.box>, !fir.box>) +! CHECK: fir.call @_QPs3(%[[VAL_1]]#0) fastmath : (!fir.box>) -> () +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest4( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.box> {fir.bindc_name = "x"}) { +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest4Ex"} : (!fir.box>) -> (!fir.box>, !fir.box>) +! CHECK: %[[VAL_2:.*]]:2 = hlfir.copy_in %[[VAL_1]]#0 : (!fir.box>) -> (!fir.box>, i1) +! CHECK: %[[VAL_3:.*]] = fir.box_addr %[[VAL_2]]#0 : (!fir.box>) -> !fir.ref> +! CHECK: fir.call @_QPs4(%[[VAL_3]]) fastmath : (!fir.ref>) -> () +! CHECK: hlfir.copy_out %[[VAL_2]]#0, %[[VAL_2]]#1 to %[[VAL_1]]#0 : (!fir.box>, i1, !fir.box>) -> () +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest3b( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.box> {fir.bindc_name = "x", fir.optional}) { +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest3bEx"} : (!fir.box>) -> (!fir.box>, !fir.box>) +! CHECK: %[[VAL_2:.*]] = fir.is_present %[[VAL_1]]#0 : (!fir.box>) -> i1 +! CHECK: %[[VAL_3:.*]]:4 = fir.if %[[VAL_2]] -> (!fir.box>, !fir.box>, i1, !fir.box>) { +! CHECK: %[[VAL_4:.*]]:2 = hlfir.copy_in %[[VAL_1]]#0 : (!fir.box>) -> (!fir.box>, i1) +! CHECK: fir.result %[[VAL_4]]#0, %[[VAL_4]]#0, %[[VAL_4]]#1, %[[VAL_1]]#0 : !fir.box>, !fir.box>, i1, !fir.box> +! CHECK: } else { +! CHECK: %[[VAL_5:.*]] = fir.absent !fir.box> +! CHECK: %[[VAL_6:.*]] = fir.absent !fir.box> +! CHECK: %[[VAL_7:.*]] = arith.constant false +! CHECK: %[[VAL_8:.*]] = fir.absent !fir.box> +! CHECK: fir.result %[[VAL_5]], %[[VAL_6]], %[[VAL_7]], %[[VAL_8]] : !fir.box>, !fir.box>, i1, !fir.box> +! CHECK: } +! CHECK: fir.call @_QPs3b(%[[VAL_9:.*]]#0) fastmath : (!fir.box>) -> () +! CHECK: hlfir.copy_out %[[VAL_9]]#1, %[[VAL_9]]#2 to %[[VAL_9]]#3 : (!fir.box>, i1, !fir.box>) -> () +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest4b( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.box> {fir.bindc_name = "x", fir.optional}) { +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest4bEx"} : (!fir.box>) -> (!fir.box>, !fir.box>) +! CHECK: %[[VAL_2:.*]] = fir.is_present %[[VAL_1]]#0 : (!fir.box>) -> i1 +! CHECK: %[[VAL_3:.*]]:4 = fir.if %[[VAL_2]] -> (!fir.ref>, !fir.box>, i1, !fir.box>) { +! CHECK: %[[VAL_4:.*]]:2 = hlfir.copy_in %[[VAL_1]]#0 : (!fir.box>) -> (!fir.box>, i1) +! CHECK: %[[VAL_5:.*]] = fir.box_addr %[[VAL_4]]#0 : (!fir.box>) -> !fir.ref> +! CHECK: fir.result %[[VAL_5]], %[[VAL_4]]#0, %[[VAL_4]]#1, %[[VAL_1]]#0 : !fir.ref>, !fir.box>, i1, !fir.box> +! CHECK: } else { +! CHECK: %[[VAL_6:.*]] = fir.absent !fir.ref> +! CHECK: %[[VAL_7:.*]] = fir.absent !fir.box> +! CHECK: %[[VAL_8:.*]] = arith.constant false +! CHECK: %[[VAL_9:.*]] = fir.absent !fir.box> +! CHECK: fir.result %[[VAL_6]], %[[VAL_7]], %[[VAL_8]], %[[VAL_9]] : !fir.ref>, !fir.box>, i1, !fir.box> +! CHECK: } +! CHECK: fir.call @_QPs4b(%[[VAL_10:.*]]#0) fastmath : (!fir.ref>) -> () +! CHECK: hlfir.copy_out %[[VAL_10]]#1, %[[VAL_10]]#2 to %[[VAL_10]]#3 : (!fir.box>, i1, !fir.box>) -> () +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest4c( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.box> {fir.bindc_name = "x", fir.contiguous, fir.optional}) { +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest4cEx"} : (!fir.box>) -> (!fir.box>, !fir.box>) +! CHECK: %[[VAL_2:.*]] = fir.is_present %[[VAL_1]]#0 : (!fir.box>) -> i1 +! CHECK: %[[VAL_3:.*]] = fir.if %[[VAL_2]] -> (!fir.ref>) { +! CHECK: %[[VAL_4:.*]] = fir.box_addr %[[VAL_1]]#1 : (!fir.box>) -> !fir.ref> +! CHECK: fir.result %[[VAL_4]] : !fir.ref> +! CHECK: } else { +! CHECK: %[[VAL_5:.*]] = fir.absent !fir.ref> +! CHECK: fir.result %[[VAL_5]] : !fir.ref> +! CHECK: } +! CHECK: fir.call @_QPs4c(%[[VAL_3]]) fastmath : (!fir.ref>) -> () +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest4d( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.box> {fir.bindc_name = "x", fir.contiguous}) { +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest4dEx"} : (!fir.box>) -> (!fir.box>, !fir.box>) +! CHECK: %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]]#1 : (!fir.box>) -> !fir.ref> +! CHECK: fir.call @_QPs4d(%[[VAL_2]]) fastmath : (!fir.ref>) -> () +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest5( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.box> {fir.bindc_name = "x"}) { +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest5Ex"} : (!fir.box>) -> (!fir.box>, !fir.box>) +! CHECK: %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#0 : (!fir.box>) -> !fir.box> +! CHECK: fir.call @_QPs5(%[[VAL_2]]) fastmath : (!fir.box>) -> () +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func @_QPtest5b( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.box> {fir.bindc_name = "x", fir.optional}) { +! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest5bEx"} : (!fir.box>) -> (!fir.box>, !fir.box>) +! CHECK: %[[VAL_2:.*]] = fir.is_present %[[VAL_1]]#0 : (!fir.box>) -> i1 +! CHECK: %[[VAL_3:.*]]:4 = fir.if %[[VAL_2]] -> (!fir.box>, !fir.box>, i1, !fir.box>) { +! CHECK: %[[VAL_4:.*]]:2 = hlfir.copy_in %[[VAL_1]]#0 : (!fir.box>) -> (!fir.box>, i1) +! CHECK: fir.result %[[VAL_4]]#0, %[[VAL_4]]#0, %[[VAL_4]]#1, %[[VAL_1]]#0 : !fir.box>, !fir.box>, i1, !fir.box> +! CHECK: } else { +! CHECK: %[[VAL_5:.*]] = fir.absent !fir.box> +! CHECK: %[[VAL_6:.*]] = fir.absent !fir.box> +! CHECK: %[[VAL_7:.*]] = arith.constant false +! CHECK: %[[VAL_8:.*]] = fir.absent !fir.box> +! CHECK: fir.result %[[VAL_5]], %[[VAL_6]], %[[VAL_7]], %[[VAL_8]] : !fir.box>, !fir.box>, i1, !fir.box> +! CHECK: } +! CHECK: %[[VAL_9:.*]] = fir.convert %[[VAL_10:.*]]#0 : (!fir.box>) -> !fir.box> +! CHECK: fir.call @_QPs5b(%[[VAL_9]]) fastmath : (!fir.box>) -> () +! CHECK: hlfir.copy_out %[[VAL_10]]#1, %[[VAL_10]]#2 to %[[VAL_10]]#3 : (!fir.box>, i1, !fir.box>) -> () +! CHECK: return +! CHECK: } -- GitLab From 9a894e7d84892489826375f94c08ab16ccacc4bb Mon Sep 17 00:00:00 2001 From: Shourya Goel Date: Tue, 5 Mar 2024 13:58:44 +0530 Subject: [PATCH 110/929] [Github Automation] Allow colon after cherry-pick command (#81002) Fixes: https://github.com/llvm/llvm-project/issues/64803 Removed unsupported branch command and changed Regex to accept /cherry-pick: command. --- llvm/utils/git/github-automation.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/llvm/utils/git/github-automation.py b/llvm/utils/git/github-automation.py index b475eff06fc3..b2e6843eb9af 100755 --- a/llvm/utils/git/github-automation.py +++ b/llvm/utils/git/github-automation.py @@ -616,22 +616,20 @@ class ReleaseWorkflow: def execute_command(self) -> bool: """ This function reads lines from STDIN and executes the first command - that it finds. The 2 supported commands are: - /cherry-pick commit0 <...> - /branch // + that it finds. The supported command is: + /cherry-pick< ><:> commit0 <...> """ for line in sys.stdin: line.rstrip() - m = re.search(r"/([a-z-]+)\s(.+)", line) + m = re.search(r"/cherry-pick\s*:? *(.*)", line) if not m: continue - command = m.group(1) - args = m.group(2) - if command == "cherry-pick": - arg_list = args.split() - commits = list(map(lambda a: extract_commit_hash(a), arg_list)) - return self.create_branch(commits) + args = m.group(1) + + arg_list = args.split() + commits = list(map(lambda a: extract_commit_hash(a), arg_list)) + return self.create_branch(commits) print("Do not understand input:") print(sys.stdin.readlines()) -- GitLab From ff66e9b7e2fad71d1c65e884d6f94fb6ea5bdc21 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Tue, 5 Mar 2024 09:34:06 +0100 Subject: [PATCH 111/929] [Docs] Update frontend perfomance tips about loads/stores (#83833) This contains two updates: * Generalize the "no loads stores of aggregates" to "no values of aggregate type" in general, and be clearer about the exceptions where it is okay to use them. * Mention that you should not load/store non-byte-size types. --- llvm/docs/Frontend/PerformanceTips.rst | 36 ++++++++++++++++++++------ 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/llvm/docs/Frontend/PerformanceTips.rst b/llvm/docs/Frontend/PerformanceTips.rst index dfa3ccdd1ddd..289106cd1e28 100644 --- a/llvm/docs/Frontend/PerformanceTips.rst +++ b/llvm/docs/Frontend/PerformanceTips.rst @@ -64,16 +64,36 @@ SSA is the canonical form expected by much of the optimizer; if allocas can not be eliminated by Mem2Reg or SROA, the optimizer is likely to be less effective than it could be. -Avoid loads and stores of large aggregate type -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Avoid creating values of aggregate type +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Avoid creating values of :ref:`aggregate types ` (i.e. structs and +arrays). In particular, avoid loading and storing them, or manipulating them +with insertvalue and extractvalue instructions. Instead, only load and store +individual fields of the aggregate. + +There are some exceptions to this rule: + +* It is fine to use values of aggregate type in global variable initializers. +* It is fine to return structs, if this is done to represent the return of + multiple values in registers. +* It is fine to work with structs returned by LLVM intrinsics, such as the + ``with.overflow`` family of intrinsics. +* It is fine to use aggregate *types* without creating values. For example, + they are commonly used in ``getelementptr`` instructions or attributes like + ``sret``. + +Avoid loads and stores of non-byte-sized types +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Avoid loading or storing non-byte-sized types like ``i1``. Instead, +appropriately extend them to the next byte-sized type. -LLVM currently does not optimize well loads and stores of large :ref:`aggregate -types ` (i.e. structs and arrays). As an alternative, consider -loading individual fields from memory. +For example, when working with boolean values, store them by zero-extending +``i1`` to ``i8`` and load them by loading ``i8`` and truncating to ``i1``. -Aggregates that are smaller than the largest (performant) load or store -instruction supported by the targeted hardware are well supported. These can -be an effective way to represent collections of small packed fields. +If you do use loads/stores on non-byte-sized types, make sure that you *always* +use those types. For example, do not first store ``i8`` and then load ``i1``. Prefer zext over sext when legal ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- GitLab From d773c00e52f1acd68267c6f2f5bfa269b73810a0 Mon Sep 17 00:00:00 2001 From: cor3ntin Date: Tue, 5 Mar 2024 10:04:45 +0100 Subject: [PATCH 112/929] [Clang] Fix looking for immediate calls in default arguments. (#80690) Due to improper use of RecursiveASTVisitor. Fixes #80630 --- clang/docs/ReleaseNotes.rst | 2 ++ clang/lib/Sema/SemaExpr.cpp | 6 ------ .../SemaCXX/cxx2a-consteval-default-params.cpp | 15 +++++++++++++++ clang/test/SemaCXX/source_location.cpp | 18 ++++++++++++++++++ 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 612b43297274..b94e491de5e4 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -298,6 +298,8 @@ Bug Fixes to C++ Support - Fixed an issue where an attribute on a declarator would cause the attribute to be destructed prematurely. This fixes a pair of Chromium that were brought to our attention by an attempt to fix in (#GH77703). Fixes (#GH83385). +- Fix evaluation of some immediate calls in default arguments. + Fixes (#GH80630) Bug Fixes to AST Handling ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp index 0a449fc1082b..47bb263f56aa 100644 --- a/clang/lib/Sema/SemaExpr.cpp +++ b/clang/lib/Sema/SemaExpr.cpp @@ -6227,12 +6227,6 @@ struct ImmediateCallVisitor : public RecursiveASTVisitor { return VisitCXXMethodDecl(E->getCallOperator()); } - // Blocks don't support default parameters, and, as for lambdas, - // we don't consider their body a subexpression. - bool VisitBlockDecl(BlockDecl *B) { return false; } - - bool VisitCompoundStmt(CompoundStmt *B) { return false; } - bool VisitCXXDefaultArgExpr(CXXDefaultArgExpr *E) { return TraverseStmt(E->getExpr()); } diff --git a/clang/test/SemaCXX/cxx2a-consteval-default-params.cpp b/clang/test/SemaCXX/cxx2a-consteval-default-params.cpp index be8f7cc78858..e4b13725b2da 100644 --- a/clang/test/SemaCXX/cxx2a-consteval-default-params.cpp +++ b/clang/test/SemaCXX/cxx2a-consteval-default-params.cpp @@ -82,3 +82,18 @@ namespace GH62224 { C<> Val; // No error since fwd is defined already. static_assert(Val.get() == 42); } + +namespace GH80630 { + +consteval const char* ce() { return "Hello"; } + +auto f2(const char* loc = []( char const* fn ) + { return fn; } ( ce() ) ) { + return loc; +} + +auto g() { + return f2(); +} + +} diff --git a/clang/test/SemaCXX/source_location.cpp b/clang/test/SemaCXX/source_location.cpp index 7414fbce7828..b151fc45fdad 100644 --- a/clang/test/SemaCXX/source_location.cpp +++ b/clang/test/SemaCXX/source_location.cpp @@ -832,3 +832,21 @@ void test() { } } + +namespace GH80630 { + +#define GH80630_LAMBDA \ + []( char const* fn ) { \ + static constexpr std::source_location loc = std::source_location::current(); \ + return &loc; \ + }( std::source_location::current().function() ) + +auto f( std::source_location const* loc = GH80630_LAMBDA ) { + return loc; +} + +auto g() { + return f(); +} + +} -- GitLab From 690bf64f077a281d434537b0907c9fc170123dcc Mon Sep 17 00:00:00 2001 From: Amirreza Ashouri Date: Tue, 5 Mar 2024 12:37:00 +0330 Subject: [PATCH 113/929] [clang] Support `__is_trivially_copyable(int()&)==false` (#81298) IMHO it would be productive to make a similar change for `typeid`, in `ParseCXXTypeid`, in order to improve Clang's error message for https://godbolt.org/z/oKKWxeYra But that might be better done by adding a new DeclaratorContext specifically for TypeidArg, instead of pretending that the argument to `typeid` is a template argument, because I don't know what else that change might affect. Fixes #77585 --- clang/docs/ReleaseNotes.rst | 2 ++ clang/lib/Parse/ParseExprCXX.cpp | 8 ++++++-- clang/test/Sema/static-assert.c | 9 +++++---- clang/test/SemaCXX/type-traits-nonobject.cpp | 3 +++ 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index b94e491de5e4..d4e6bcf661da 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -265,6 +265,8 @@ Bug Fixes to C++ Support - Fix a crash when trying to call a varargs function that also has an explicit object parameter. (#GH80971) - Fixed a bug where abbreviated function templates would append their invented template parameters to an empty template parameter lists. +- Fix parsing of abominable function types inside type traits. + Fixes (`#77585 `_) - Clang now classifies aggregate initialization in C++17 and newer as constant or non-constant more accurately. Previously, only a subset of the initializer elements were considered, misclassifying some initializers as constant. Partially fixes diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp index 22ee60af4616..9471f6f725ef 100644 --- a/clang/lib/Parse/ParseExprCXX.cpp +++ b/clang/lib/Parse/ParseExprCXX.cpp @@ -3908,7 +3908,10 @@ ExprResult Parser::ParseTypeTrait() { SmallVector Args; do { // Parse the next type. - TypeResult Ty = ParseTypeName(); + TypeResult Ty = + ParseTypeName(/*SourceRange=*/nullptr, + getLangOpts().CPlusPlus ? DeclaratorContext::TemplateArg + : DeclaratorContext::TypeName); if (Ty.isInvalid()) { Parens.skipToEnd(); return ExprError(); @@ -3950,7 +3953,8 @@ ExprResult Parser::ParseArrayTypeTrait() { if (T.expectAndConsume()) return ExprError(); - TypeResult Ty = ParseTypeName(); + TypeResult Ty = + ParseTypeName(/*SourceRange=*/nullptr, DeclaratorContext::TemplateArg); if (Ty.isInvalid()) { SkipUntil(tok::comma, StopAtSemi); SkipUntil(tok::r_paren, StopAtSemi); diff --git a/clang/test/Sema/static-assert.c b/clang/test/Sema/static-assert.c index 4e9e6b7ee558..ae5e8076e0be 100644 --- a/clang/test/Sema/static-assert.c +++ b/clang/test/Sema/static-assert.c @@ -1,6 +1,6 @@ -// RUN: %clang_cc1 -std=c11 -Wgnu-folding-constant -fsyntax-only -verify %s -// RUN: %clang_cc1 -fms-compatibility -Wgnu-folding-constant -DMS -fsyntax-only -verify=expected,ms %s -// RUN: %clang_cc1 -std=c99 -pedantic -Wgnu-folding-constant -fsyntax-only -verify=expected,ext %s +// RUN: %clang_cc1 -std=c11 -Wgnu-folding-constant -fsyntax-only -verify=expected,c %s +// RUN: %clang_cc1 -fms-compatibility -Wgnu-folding-constant -DMS -fsyntax-only -verify=expected,ms,c %s +// RUN: %clang_cc1 -std=c99 -pedantic -Wgnu-folding-constant -fsyntax-only -verify=expected,ext,c %s // RUN: %clang_cc1 -xc++ -std=c++11 -pedantic -fsyntax-only -verify=expected,ext,cxx %s _Static_assert("foo", "string is nonzero"); // ext-warning {{'_Static_assert' is a C11 extension}} @@ -57,7 +57,8 @@ UNION(char[2], short) u2 = { .one = { 'a', 'b' } }; // ext-warning 3 {{'_Static_ typedef UNION(char, short) U3; // expected-error {{static assertion failed due to requirement 'sizeof(char) == sizeof(short)': type size mismatch}} \ // expected-note{{evaluates to '1 == 2'}} \ // ext-warning 3 {{'_Static_assert' is a C11 extension}} -typedef UNION(float, 0.5f) U4; // expected-error {{expected a type}} \ +typedef UNION(float, 0.5f) U4; // c-error {{expected a type}} \ + // cxx-error {{type name requires a specifier or qualifier}} \ // ext-warning 3 {{'_Static_assert' is a C11 extension}} // After defining the assert macro in MS-compatibility mode, we should diff --git a/clang/test/SemaCXX/type-traits-nonobject.cpp b/clang/test/SemaCXX/type-traits-nonobject.cpp index c9e3c30e5533..5f7c20cc2e11 100644 --- a/clang/test/SemaCXX/type-traits-nonobject.cpp +++ b/clang/test/SemaCXX/type-traits-nonobject.cpp @@ -6,11 +6,14 @@ static_assert(!__is_pod(void), ""); static_assert(!__is_pod(int&), ""); static_assert(!__is_pod(int()), ""); +static_assert(!__is_pod(int()&), ""); static_assert(!__is_trivially_copyable(void), ""); static_assert(!__is_trivially_copyable(int&), ""); static_assert(!__is_trivially_copyable(int()), ""); +static_assert(!__is_trivially_copyable(int()&), ""); static_assert(!__is_trivially_relocatable(void), ""); static_assert(!__is_trivially_relocatable(int&), ""); static_assert(!__is_trivially_relocatable(int()), ""); +static_assert(!__is_trivially_relocatable(int()&), ""); -- GitLab From 686223376b43bf8489aed18262c60f8dbffdddbd Mon Sep 17 00:00:00 2001 From: Yeting Kuo <46629943+yetingk@users.noreply.github.com> Date: Tue, 5 Mar 2024 17:14:29 +0800 Subject: [PATCH 114/929] [LangRef] Add aligned attribute description into masked.compress/expandload sections. (#83808) Align attribute has already been used for masked.compress/expandload in commit #83519, #83763 and #83516. --- llvm/docs/LangRef.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index f56d4ed28f28..0e3f6f8daaba 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -24276,6 +24276,9 @@ Arguments: The first operand is the base pointer for the load. It has the same underlying type as the element of the returned vector. The second operand, mask, is a vector of boolean values with the same number of elements as the return type. The third is a pass-through value that is used to fill the masked-off lanes of the result. The return type and the type of the '``passthru``' operand have the same vector type. +The :ref:`align ` parameter attribute can be provided for the first +operand. The pointer alignment defaults to 1. + Semantics: """""""""" @@ -24333,6 +24336,8 @@ Arguments: The first operand is the input vector, from which elements are collected and written to memory. The second operand is the base pointer for the store, it has the same underlying type as the element of the input vector operand. The third operand is the mask, a vector of boolean values. The mask and the input vector must have the same number of vector elements. +The :ref:`align ` parameter attribute can be provided for the second +operand. The pointer alignment defaults to 1. Semantics: """""""""" -- GitLab From 88414c8862c58fa4e708a092acb87bd0687121ce Mon Sep 17 00:00:00 2001 From: Balazs Benics Date: Tue, 5 Mar 2024 10:30:28 +0100 Subject: [PATCH 115/929] [analyzer][NFC] Remove dead code (#83968) Remove the unused method `CoreEngine::ExecuteWorkListWithInitialState`. --- .../StaticAnalyzer/Core/PathSensitive/CoreEngine.h | 6 ------ .../StaticAnalyzer/Core/PathSensitive/ExprEngine.h | 10 ---------- clang/lib/StaticAnalyzer/Core/CoreEngine.cpp | 12 ------------ 3 files changed, 28 deletions(-) diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CoreEngine.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CoreEngine.h index 8dbe767cef9d..8e392421fef9 100644 --- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CoreEngine.h +++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CoreEngine.h @@ -149,12 +149,6 @@ public: bool ExecuteWorkList(const LocationContext *L, unsigned Steps, ProgramStateRef InitState); - /// Returns true if there is still simulation state on the worklist. - bool ExecuteWorkListWithInitialState(const LocationContext *L, - unsigned Steps, - ProgramStateRef InitState, - ExplodedNodeSet &Dst); - /// Dispatch the work list item based on the given location information. /// Use Pred parameter as the predecessor state. void dispatchWorkItem(ExplodedNode* Pred, ProgramPoint Loc, diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h index ed5c4adb5e3d..f7894fb83ce6 100644 --- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h +++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h @@ -190,16 +190,6 @@ public: return Engine.ExecuteWorkList(L, Steps, nullptr); } - /// Execute the work list with an initial state. Nodes that reaches the exit - /// of the function are added into the Dst set, which represent the exit - /// state of the function call. Returns true if there is still simulation - /// state on the worklist. - bool ExecuteWorkListWithInitialState(const LocationContext *L, unsigned Steps, - ProgramStateRef InitState, - ExplodedNodeSet &Dst) { - return Engine.ExecuteWorkListWithInitialState(L, Steps, InitState, Dst); - } - /// getContext - Return the ASTContext associated with this analysis. ASTContext &getContext() const { return AMgr.getASTContext(); } diff --git a/clang/lib/StaticAnalyzer/Core/CoreEngine.cpp b/clang/lib/StaticAnalyzer/Core/CoreEngine.cpp index d3499e7a917d..141d0cb320bf 100644 --- a/clang/lib/StaticAnalyzer/Core/CoreEngine.cpp +++ b/clang/lib/StaticAnalyzer/Core/CoreEngine.cpp @@ -222,18 +222,6 @@ void CoreEngine::dispatchWorkItem(ExplodedNode* Pred, ProgramPoint Loc, } } -bool CoreEngine::ExecuteWorkListWithInitialState(const LocationContext *L, - unsigned Steps, - ProgramStateRef InitState, - ExplodedNodeSet &Dst) { - bool DidNotFinish = ExecuteWorkList(L, Steps, InitState); - for (ExplodedGraph::eop_iterator I = G.eop_begin(), E = G.eop_end(); I != E; - ++I) { - Dst.Add(*I); - } - return DidNotFinish; -} - void CoreEngine::HandleBlockEdge(const BlockEdge &L, ExplodedNode *Pred) { const CFGBlock *Blk = L.getDst(); NodeBuilderContext BuilderCtx(*this, Blk, Pred); -- GitLab From c5d16e76892dc9dc733a844e46bf03c9c6e1d759 Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Tue, 5 Mar 2024 09:36:20 +0000 Subject: [PATCH 116/929] [IR] Update comment in Verifier::visitPHINode Since svn r133708 visitPHINode no longer has to check basic block argument types. Better 13 years late than never! --- llvm/lib/IR/Verifier.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 027c6ccd1ab3..74c7354e7bf1 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -3381,7 +3381,7 @@ void Verifier::visitPHINode(PHINode &PN) { Check(!PN.getType()->isTokenTy(), "PHI nodes cannot have token type!"); // Check that all of the values of the PHI node have the same type as the - // result, and that the incoming blocks are really basic blocks. + // result. for (Value *IncValue : PN.incoming_values()) { Check(PN.getType() == IncValue->getType(), "PHI node operands are not the same type as the result!", &PN); -- GitLab From 16f2a1fb97de0b0f57b025b1d9e79b1798a2c38f Mon Sep 17 00:00:00 2001 From: Adrian Kuegel Date: Tue, 5 Mar 2024 09:43:31 +0000 Subject: [PATCH 117/929] [mlir] Apply ClangTidy findings move constructors should be marked noexcept --- mlir/include/mlir/IR/BuiltinAttributeInterfaces.h | 2 +- mlir/lib/Bindings/Python/TransformInterpreter.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mlir/include/mlir/IR/BuiltinAttributeInterfaces.h b/mlir/include/mlir/IR/BuiltinAttributeInterfaces.h index b0b9493c1519..2104ffc9fd32 100644 --- a/mlir/include/mlir/IR/BuiltinAttributeInterfaces.h +++ b/mlir/include/mlir/IR/BuiltinAttributeInterfaces.h @@ -36,7 +36,7 @@ public: ElementsAttrIndexer() : ElementsAttrIndexer(/*isContiguous=*/true, /*isSplat=*/true) {} ElementsAttrIndexer(ElementsAttrIndexer &&rhs) - : isContiguous(rhs.isContiguous), isSplat(rhs.isSplat) { + : isContiguous(rhs.isContiguous), isSplat(rhs.isSplat) noexcept { if (isContiguous) conState = rhs.conState; else diff --git a/mlir/lib/Bindings/Python/TransformInterpreter.cpp b/mlir/lib/Bindings/Python/TransformInterpreter.cpp index 6517f8c39dfa..3530f295ed82 100644 --- a/mlir/lib/Bindings/Python/TransformInterpreter.cpp +++ b/mlir/lib/Bindings/Python/TransformInterpreter.cpp @@ -23,7 +23,7 @@ namespace py = pybind11; namespace { struct PyMlirTransformOptions { PyMlirTransformOptions() { options = mlirTransformOptionsCreate(); }; - PyMlirTransformOptions(PyMlirTransformOptions &&other) { + PyMlirTransformOptions(PyMlirTransformOptions &&other) noexcept { options = other.options; other.options.ptr = nullptr; } -- GitLab From 56abb8d5355420cf7f66183f2d526009471b4fcc Mon Sep 17 00:00:00 2001 From: Graham Hunter Date: Tue, 5 Mar 2024 09:35:16 +0000 Subject: [PATCH 118/929] [AArch64] Be stricter about insert/extract index Post-commit fixup patch for a request on https://github.com/llvm/llvm-project/pull/81135 --- llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index 48f053c39b1b..755b034764ed 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -600,9 +600,9 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, TargetLoweringBase::LegalizeKind VecLK = getTLI()->getTypeConversion(C, VecVT); const Value *Idx = IsExtract ? ICA.getArgs()[1] : ICA.getArgs()[2]; - const ConstantInt *CIdx = dyn_cast(Idx); + const ConstantInt *CIdx = cast(Idx); if (SubVecLK.first == TargetLoweringBase::TypeLegal && - VecLK.first == TargetLoweringBase::TypeLegal && CIdx && CIdx->isZero()) + VecLK.first == TargetLoweringBase::TypeLegal && CIdx->isZero()) return TTI::TCC_Free; break; } -- GitLab From 0709eeb583bd6cc029019c6ec2b3e210ff6d402d Mon Sep 17 00:00:00 2001 From: Adrian Kuegel Date: Tue, 5 Mar 2024 09:51:12 +0000 Subject: [PATCH 119/929] [mlir] Followup fix, added noexcept in wrong place --- mlir/include/mlir/IR/BuiltinAttributeInterfaces.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mlir/include/mlir/IR/BuiltinAttributeInterfaces.h b/mlir/include/mlir/IR/BuiltinAttributeInterfaces.h index 2104ffc9fd32..1507c62e3fe0 100644 --- a/mlir/include/mlir/IR/BuiltinAttributeInterfaces.h +++ b/mlir/include/mlir/IR/BuiltinAttributeInterfaces.h @@ -35,8 +35,8 @@ struct ElementsAttrIndexer { public: ElementsAttrIndexer() : ElementsAttrIndexer(/*isContiguous=*/true, /*isSplat=*/true) {} - ElementsAttrIndexer(ElementsAttrIndexer &&rhs) - : isContiguous(rhs.isContiguous), isSplat(rhs.isSplat) noexcept { + ElementsAttrIndexer(ElementsAttrIndexer &&rhs) noexcept + : isContiguous(rhs.isContiguous), isSplat(rhs.isSplat) { if (isContiguous) conState = rhs.conState; else -- GitLab From b585c43dccb2c608f698419a9c8d7645d3120fdb Mon Sep 17 00:00:00 2001 From: Kiran Chandramohan Date: Tue, 5 Mar 2024 10:28:36 +0000 Subject: [PATCH 120/929] [Flang][OpenMP] : Add a temporary lowering for workshare directive (#78268) As a temporary solution, lower workshare to the single directive --- flang/lib/Lower/OpenMP/OpenMP.cpp | 9 ++++-- flang/test/Lower/OpenMP/workshare.f90 | 42 +++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 flang/test/Lower/OpenMP/workshare.f90 diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp index 90fc1f80f57a..83aebdaecd35 100644 --- a/flang/lib/Lower/OpenMP/OpenMP.cpp +++ b/flang/lib/Lower/OpenMP/OpenMP.cpp @@ -1810,7 +1810,11 @@ genOMP(Fortran::lower::AbstractConverter &converter, /*outerCombined=*/false); break; case llvm::omp::Directive::OMPD_workshare: - TODO(currentLocation, "Workshare construct"); + // FIXME: Workshare is not a commonly used OpenMP construct, an + // implementation for this feature will come later. For the codes + // that use this construct, add a single construct for now. + genSingleOp(converter, semaCtx, eval, /*genNested=*/true, currentLocation, + beginClauseList, endClauseList); break; default: singleDirective = false; @@ -1845,7 +1849,8 @@ genOMP(Fortran::lower::AbstractConverter &converter, } if ((llvm::omp::workShareSet & llvm::omp::blockConstructSet) .test(directive.v)) { - TODO(currentLocation, "Workshare construct"); + genSingleOp(converter, semaCtx, eval, /*genNested=*/false, currentLocation, + beginClauseList, endClauseList); combinedDirective = true; } if (!combinedDirective) diff --git a/flang/test/Lower/OpenMP/workshare.f90 b/flang/test/Lower/OpenMP/workshare.f90 new file mode 100644 index 000000000000..1e11677a15e1 --- /dev/null +++ b/flang/test/Lower/OpenMP/workshare.f90 @@ -0,0 +1,42 @@ + +!RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s + +!CHECK-LABEL: func @_QPsb1 +subroutine sb1(arr) + integer :: arr(:) +!CHECK: omp.parallel { + !$omp parallel +!CHECK: omp.single { + !$omp workshare + arr = 0 + !$omp end workshare +!CHECK: } + !$omp end parallel +!CHECK: } +end subroutine + +!CHECK-LABEL: func @_QPsb2 +subroutine sb2(arr) + integer :: arr(:) +!CHECK: omp.parallel { + !$omp parallel +!CHECK: omp.single nowait { + !$omp workshare + arr = 0 + !$omp end workshare nowait +!CHECK: } + !$omp end parallel +!CHECK: } +end subroutine + +!CHECK-LABEL: func @_QPsb3 +subroutine sb3(arr) + integer :: arr(:) +!CHECK: omp.parallel { +!CHECK: omp.single { + !$omp parallel workshare + arr = 0 + !$omp end parallel workshare +!CHECK: } +!CHECK: } +end subroutine -- GitLab From a668846202bcc34b1742f300ba1eb1a0e64fa36d Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Tue, 5 Mar 2024 18:31:33 +0800 Subject: [PATCH 121/929] [DAGCombiner] Handle extending EXTRACT_VECTOR_ELTs in calculateByteProvider (#83963) An EXTRACT_VECTOR_ELT can extend the element to the width of its result type, leaving the high bits undefined. Previously if we attempted to query the bytes in these high bits we would recurse and hit an assertion. This fixes it by bailing if the index is outside of the vector element size. I think the assertion Index < ByteWidth may still be incorrect, since ByteWidth is calculated from Op.getValueSizeInBits(). I believe this should be Op.getScalarValueSizeInBits() whenever VectorIndex is set since we're querying the element now, not the vector. But I couldn't think of a test case to trigger it. It can be addressed in a follow-up patch. Fixes #83920 --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 4 ++++ llvm/test/CodeGen/RISCV/rvv/pr83920.ll | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 llvm/test/CodeGen/RISCV/rvv/pr83920.ll diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 33ada3655dc7..f160dde25d26 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -8627,6 +8627,10 @@ calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth, if (NarrowBitWidth % 8 != 0) return std::nullopt; uint64_t NarrowByteWidth = NarrowBitWidth / 8; + // EXTRACT_VECTOR_ELT can extend the element type to the width of the return + // type, leaving the high bits undefined. + if (Index >= NarrowByteWidth) + return std::nullopt; // Check to see if the position of the element in the vector corresponds // with the byte we are trying to provide for. In the case of a vector of diff --git a/llvm/test/CodeGen/RISCV/rvv/pr83920.ll b/llvm/test/CodeGen/RISCV/rvv/pr83920.ll new file mode 100644 index 000000000000..319f0bd9ef1c --- /dev/null +++ b/llvm/test/CodeGen/RISCV/rvv/pr83920.ll @@ -0,0 +1,16 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc < %s -mtriple=riscv64 -mattr=+v | FileCheck %s + +define i8 @or_load_combine(ptr %p) { +; CHECK-LABEL: or_load_combine: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma +; CHECK-NEXT: vle8.v v8, (a0) +; CHECK-NEXT: vmv.x.s a0, v8 +; CHECK-NEXT: ori a0, a0, 1 +; CHECK-NEXT: ret + %load = load <2 x i8>, ptr %p + %extract = extractelement <2 x i8> %load, i64 0 + %or = or i8 %extract, 1 + ret i8 %or +} -- GitLab From 762f762504967efbe159db5c737154b989afc9bb Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Tue, 5 Mar 2024 11:21:38 +0000 Subject: [PATCH 122/929] [AMDGPU] Rename get_MUBUF_ps and use it for MTBUF too. NFC. (#83991) This allows removing a couple of MTBUF helper (multi)classes. --- llvm/lib/Target/AMDGPU/BUFInstructions.td | 37 +++++++++++------------ 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index 7bb92256fbdd..a2ff56d68218 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -2431,10 +2431,10 @@ class VBUFFER_MTBUF_Real_gfx12 op, MTBUF_Pseudo ps, // MUBUF - GFX11, GFX12. //===----------------------------------------------------------------------===// -// Shortcut to default Mnemonic from MUBUF_Pseudo. Hides the cast to the +// Shortcut to default Mnemonic from BUF_Pseudo. Hides the cast to the // specific pseudo (bothen in this case) since any of them will work. -class get_MUBUF_ps { - string Mnemonic = !cast(name # "_BOTHEN").Mnemonic; +class get_BUF_ps { + string Mnemonic = !cast(name # "_BOTHEN").Mnemonic; } // gfx11 instruction that accept both old and new assembler name. @@ -2487,14 +2487,14 @@ multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl op, string real_n // Non-renamed, non-atomic gfx11/gfx12 mubuf instructions. multiclass MUBUF_Real_AllAddr_gfx11 op, bit hasTFE = 1> : - MUBUF_Real_AllAddr_gfx11_Renamed_Impl.Mnemonic, hasTFE>; + MUBUF_Real_AllAddr_gfx11_Renamed_Impl.Mnemonic, hasTFE>; multiclass MUBUF_Real_AllAddr_gfx11_gfx12 op, bit hasTFE = 1> : - MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl.Mnemonic, hasTFE>; + MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl.Mnemonic, hasTFE>; multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Renamed op, string real_name> : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl { - def : Mnem_gfx11_gfx12.Mnemonic, real_name>; + def : Mnem_gfx11_gfx12.Mnemonic, real_name>; } class MUBUF_Real_Atomic_gfx11_impl op, string ps_name, @@ -2538,23 +2538,23 @@ multiclass MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl op, bit is_return, // Non-renamed gfx11/gfx12 mubuf atomic. multiclass MUBUF_Real_Atomic_gfx11_gfx12 op> : - MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl.Mnemonic>, - MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl.Mnemonic>; + MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl.Mnemonic>, + MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl.Mnemonic>; multiclass MUBUF_Real_Atomic_gfx12 op> : - MUBUF_Real_Atomic_gfx12_Renamed_impl.Mnemonic>, - MUBUF_Real_Atomic_gfx12_Renamed_impl.Mnemonic>; + MUBUF_Real_Atomic_gfx12_Renamed_impl.Mnemonic>, + MUBUF_Real_Atomic_gfx12_Renamed_impl.Mnemonic>; multiclass MUBUF_Real_Atomic_gfx11_Renamed op, string real_name> : MUBUF_Real_Atomic_gfx11_Renamed_impl, MUBUF_Real_Atomic_gfx11_Renamed_impl { - def : Mnem_gfx11_gfx12.Mnemonic, real_name>; + def : Mnem_gfx11_gfx12.Mnemonic, real_name>; } multiclass MUBUF_Real_Atomic_gfx11_gfx12_Renamed op, string real_name> : MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl, MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl { - def : Mnem_gfx11_gfx12.Mnemonic, real_name>; + def : Mnem_gfx11_gfx12.Mnemonic, real_name>; } multiclass MUBUF_Real_Atomic_gfx11_gfx12_Renamed_gfx12_Renamed op, string gfx12_name, string gfx11_name> : @@ -2562,8 +2562,8 @@ multiclass MUBUF_Real_Atomic_gfx11_gfx12_Renamed_gfx12_Renamed op, strin MUBUF_Real_Atomic_gfx11_Renamed_impl, MUBUF_Real_Atomic_gfx12_Renamed_impl, MUBUF_Real_Atomic_gfx12_Renamed_impl { - def : Mnem_gfx11.Mnemonic, gfx11_name>; - def : Mnem_gfx12.Mnemonic, gfx12_name>; + def : Mnem_gfx11.Mnemonic, gfx11_name>; + def : Mnem_gfx12.Mnemonic, gfx12_name>; def : Mnem_gfx12; } @@ -2944,17 +2944,14 @@ multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl op, string real_n VBUFFER_MTBUF_Real_gfx12(NAME#"_VBUFFER_OFFSET"), real_name>; } -multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Impl op, MTBUF_Pseudo ps> - : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl; multiclass MTBUF_Real_AllAddr_gfx11_gfx12 op> - : MTBUF_Real_AllAddr_gfx11_gfx12_Impl(NAME#"_BOTHEN")>; + : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl.Mnemonic>; -class Pre_gfx11_MTBUF_Name - : MnemonicAlias, Requires<[isGFX11Plus]>; multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Renamed op, string real_name> : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl { - def : Pre_gfx11_MTBUF_Name(NAME#"_BOTHEN"), real_name>; + defvar ps = get_BUF_ps; + def : MnemonicAlias, Requires<[isGFX11Plus]>; } defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x008, "tbuffer_load_d16_format_x">; -- GitLab From 923ddf65f4e21ec67018cf56e823895de18d83bc Mon Sep 17 00:00:00 2001 From: AtariDreams <83477269+AtariDreams@users.noreply.github.com> Date: Tue, 5 Mar 2024 06:32:45 -0500 Subject: [PATCH 123/929] [ObjC] Check entire chain of superclasses to see if class layout is statically known (#81335) As of now, we only check if a class directly inherits from NSObject to determine if said class has fixed offsets and can therefore "opt-out" from the non-fragile ABI for ivars. However, if an NSObject subclass has fixed offsets, then so must the subclasses of that subclass, so this allows us to optimize instances of subclasses of subclasses that inherit from NSObject and so on. To determine this, we need to find that the compiler can see the implementation of each intermediate class, as that means it is statically linked. Fixes: #81369 --- clang/lib/CodeGen/CGObjCMac.cpp | 20 ++-- .../constant-non-fragile-ivar-offset.m | 102 ++++++++++++++++++ 2 files changed, 116 insertions(+), 6 deletions(-) diff --git a/clang/lib/CodeGen/CGObjCMac.cpp b/clang/lib/CodeGen/CGObjCMac.cpp index 27d77e9a8a55..e815e097e1fb 100644 --- a/clang/lib/CodeGen/CGObjCMac.cpp +++ b/clang/lib/CodeGen/CGObjCMac.cpp @@ -1593,12 +1593,20 @@ private: } bool isClassLayoutKnownStatically(const ObjCInterfaceDecl *ID) { - // NSObject is a fixed size. If we can see the @implementation of a class - // which inherits from NSObject then we know that all it's offsets also must - // be fixed. FIXME: Can we do this if see a chain of super classes with - // implementations leading to NSObject? - return ID->getImplementation() && ID->getSuperClass() && - ID->getSuperClass()->getName() == "NSObject"; + // Test a class by checking its superclasses up to + // its base class if it has one. + for (; ID; ID = ID->getSuperClass()) { + // The layout of base class NSObject + // is guaranteed to be statically known + if (ID->getIdentifier()->getName() == "NSObject") + return true; + + // If we cannot see the @implementation of a class, + // we cannot statically know the class layout. + if (!ID->getImplementation()) + return false; + } + return false; } public: diff --git a/clang/test/CodeGenObjC/constant-non-fragile-ivar-offset.m b/clang/test/CodeGenObjC/constant-non-fragile-ivar-offset.m index 788b3220af30..8d55e6c7d230 100644 --- a/clang/test/CodeGenObjC/constant-non-fragile-ivar-offset.m +++ b/clang/test/CodeGenObjC/constant-non-fragile-ivar-offset.m @@ -1,6 +1,13 @@ // RUN: %clang_cc1 -triple x86_64-apple-macosx10.14.0 -emit-llvm %s -o - | FileCheck %s // CHECK: @"OBJC_IVAR_$_StaticLayout.static_layout_ivar" = hidden constant i64 20 +// CHECK: @"OBJC_IVAR_$_SuperClass.superClassIvar" = hidden constant i64 20 +// CHECK: @"OBJC_IVAR_$_SuperClass._superClassProperty" = hidden constant i64 24 +// CHECK: @"OBJC_IVAR_$_IntermediateClass.intermediateClassIvar" = constant i64 32 +// CHECK: @"OBJC_IVAR_$_IntermediateClass.intermediateClassIvar2" = constant i64 40 +// CHECK: @"OBJC_IVAR_$_IntermediateClass._intermediateProperty" = hidden constant i64 48 +// CHECK: @"OBJC_IVAR_$_SubClass.subClassIvar" = constant i64 56 +// CHECK: @"OBJC_IVAR_$_SubClass._subClassProperty" = hidden constant i64 64 // CHECK: @"OBJC_IVAR_$_NotStaticLayout.not_static_layout_ivar" = hidden global i64 12 @interface NSObject { @@ -14,12 +21,105 @@ @implementation StaticLayout { int static_layout_ivar; } + +// CHECK-LABEL: define internal void @"\01-[StaticLayout meth]" -(void)meth { static_layout_ivar = 0; // CHECK-NOT: load i64, ptr @"OBJC_IVAR_$_StaticLayout + // CHECK: getelementptr inbounds i8, ptr %0, i64 20 +} +@end + +@interface SuperClass : NSObject +@property (nonatomic, assign) int superClassProperty; +@end + +@implementation SuperClass { + int superClassIvar; // Declare an ivar +} + +// CHECK-LABEL: define internal void @"\01-[SuperClass superClassMethod]" +- (void)superClassMethod { + _superClassProperty = 42; + superClassIvar = 10; + // CHECK-NOT: load i64, ptr @"OBJC_IVAR_$_SuperClass + // CHECK: getelementptr inbounds i8, ptr %1, i64 20 +} + +// Implicitly synthesized method here +// CHECK-LABEL: define internal i32 @"\01-[SuperClass superClassProperty]" +// CHECK: getelementptr inbounds i8, ptr %0, i64 24 + +// CHECK-LABEL: define internal void @"\01-[SuperClass setSuperClassProperty:]" +// CHECK: getelementptr inbounds i8, ptr %1, i64 24 +@end + +@interface IntermediateClass : SuperClass { + double intermediateClassIvar; + + @protected + int intermediateClassIvar2; +} +@property (nonatomic, strong) SuperClass *intermediateProperty; +@end + +@implementation IntermediateClass +@synthesize intermediateProperty = _intermediateProperty; + +// CHECK-LABEL: define internal void @"\01-[IntermediateClass intermediateClassMethod]" +- (void)intermediateClassMethod { + intermediateClassIvar = 3.14; + // CHECK-NOT: load i64, ptr @"OBJC_IVAR_$_IntermediateClass + // CHECK: getelementptr inbounds i8, ptr %0, i64 32 +} + +// CHECK-LABEL: define internal void @"\01-[IntermediateClass intermediateClassPropertyMethod]" +- (void)intermediateClassPropertyMethod { + self.intermediateProperty = 0; + // CHECK: load ptr, ptr @OBJC_SELECTOR_REFERENCES_ + // CHECK: call void @objc_msgSend(ptr noundef %0, ptr noundef %1, ptr noundef null) +} + +// CHECK-LABEL: define internal void @"\01-[IntermediateClass intermediateClassPropertyMethodDirect]" +- (void)intermediateClassPropertyMethodDirect { + _intermediateProperty = 0; + // CHECK-NOT: load i64, ptr @"OBJC_IVAR_$_IntermediateClass._intermediateProperty" + // CHECK: getelementptr inbounds i8, ptr %0, i64 48 } @end +@interface SubClass : IntermediateClass { + double subClassIvar; +} +@property (nonatomic, assign) SubClass *subClassProperty; +@end + +@implementation SubClass + +// CHECK-LABEL: define internal void @"\01-[SubClass subclassVar]" +- (void)subclassVar { + subClassIvar = 6.28; + // CHECK-NOT: load i64, ptr @"OBJC_IVAR_$_SubClass + // CHECK: getelementptr inbounds i8, ptr %0, i64 56 +} + +// CHECK-LABEL: define internal void @"\01-[SubClass intermediateSubclassVar]" +-(void)intermediateSubclassVar { + intermediateClassIvar = 3.14; + // CHECK-NOT: load i64, ptr @"OBJC_IVAR_$_IntermediateClass + // CHECK: getelementptr inbounds i8, ptr %0, i64 32 +} + +// Implicit synthesized method here: +// CHECK-LABEL: define internal ptr @"\01-[SubClass subClassProperty]" +// CHECK-NOT: load i64, ptr @"OBJC_IVAR_$_SubClass._subClassProperty" +// CHECK: getelementptr inbounds i8, ptr %0, i64 64 + +// CHECK-LABEL: define internal void @"\01-[SubClass setSubClassProperty:]" +// CHECK-NOT: load i64, ptr @"OBJC_IVAR_$_SubClass._subClassProperty" +// CHECK: getelementptr inbounds i8, ptr %1, i64 64 +@end + @interface NotNSObject { int these, might, change; } @@ -31,6 +131,8 @@ @implementation NotStaticLayout { int not_static_layout_ivar; } + +// CHECK-LABEL: define internal void @"\01-[NotStaticLayout meth]" -(void)meth { not_static_layout_ivar = 0; // CHECK: load i64, ptr @"OBJC_IVAR_$_NotStaticLayout.not_static_layout_ivar -- GitLab From 191f7678f7a76436b2b368d2c15f5ed61b9c5e2d Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Tue, 5 Mar 2024 11:26:19 +0000 Subject: [PATCH 124/929] [X86] 2007-03-15-GEP-Idx-Sink.ll - regenerate test checks --- .../CodeGen/X86/2007-03-15-GEP-Idx-Sink.ll | 51 ++++++++++++++++--- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/llvm/test/CodeGen/X86/2007-03-15-GEP-Idx-Sink.ll b/llvm/test/CodeGen/X86/2007-03-15-GEP-Idx-Sink.ll index 5eeb9ed1c868..f21aaca7ca5f 100644 --- a/llvm/test/CodeGen/X86/2007-03-15-GEP-Idx-Sink.ll +++ b/llvm/test/CodeGen/X86/2007-03-15-GEP-Idx-Sink.ll @@ -1,13 +1,52 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc < %s -mtriple=i686-darwin | FileCheck %s define void @foo(ptr %buf, i32 %size, i32 %col, ptr %p) nounwind { +; CHECK-LABEL: foo: +; CHECK: ## %bb.0: ## %entry +; CHECK-NEXT: pushl %ebx +; CHECK-NEXT: pushl %edi +; CHECK-NEXT: pushl %esi +; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax +; CHECK-NEXT: testl %eax, %eax +; CHECK-NEXT: jle LBB0_3 +; CHECK-NEXT: ## %bb.1: ## %bb.preheader +; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx +; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edx +; CHECK-NEXT: movl {{[0-9]+}}(%esp), %esi +; CHECK-NEXT: addl $8, %ecx +; CHECK-NEXT: .p2align 4, 0x90 +; CHECK-NEXT: LBB0_2: ## %bb +; CHECK-NEXT: ## =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: movl (%esi), %edi +; CHECK-NEXT: movzbl -8(%ecx), %ebx +; CHECK-NEXT: movb %bl, (%edi,%edx) +; CHECK-NEXT: movzbl -7(%ecx), %ebx +; CHECK-NEXT: movb %bl, 7(%edi,%edx) +; CHECK-NEXT: movzbl -6(%ecx), %ebx +; CHECK-NEXT: movb %bl, 5(%edi,%edx) +; CHECK-NEXT: movzbl -5(%ecx), %ebx +; CHECK-NEXT: movb %bl, 3(%edi,%edx) +; CHECK-NEXT: movzbl -4(%ecx), %ebx +; CHECK-NEXT: movb %bl, 2(%edi,%edx) +; CHECK-NEXT: movzbl -3(%ecx), %ebx +; CHECK-NEXT: movb %bl, 1(%edi,%edx) +; CHECK-NEXT: movzbl -2(%ecx), %ebx +; CHECK-NEXT: movb %bl, 2(%edi,%edx) +; CHECK-NEXT: movzbl -1(%ecx), %ebx +; CHECK-NEXT: movb %bl, 4(%edi,%edx) +; CHECK-NEXT: movzbl (%ecx), %ebx +; CHECK-NEXT: movb %bl, 6(%edi,%edx) +; CHECK-NEXT: addl $4, %esi +; CHECK-NEXT: addl $9, %ecx +; CHECK-NEXT: decl %eax +; CHECK-NEXT: jne LBB0_2 +; CHECK-NEXT: LBB0_3: ## %return +; CHECK-NEXT: popl %esi +; CHECK-NEXT: popl %edi +; CHECK-NEXT: popl %ebx +; CHECK-NEXT: retl entry: -; CHECK-LABEL: @foo -; CHECK: push -; CHECK: push -; CHECK: push -; CHECK-NOT: push - icmp sgt i32 %size, 0 ; :0 [#uses=1] br i1 %0, label %bb.preheader, label %return -- GitLab From 49f95052c8cd6e611bd46e5d2730b432f1df6815 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Tue, 5 Mar 2024 11:28:56 +0000 Subject: [PATCH 125/929] [X86] pr59305.ll - replace "X86-64" check prefix with "X64" --- llvm/test/CodeGen/X86/pr59305.ll | 52 ++++++++++++++++---------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/llvm/test/CodeGen/X86/pr59305.ll b/llvm/test/CodeGen/X86/pr59305.ll index 4172aa6204de..4d59192fdc4d 100644 --- a/llvm/test/CodeGen/X86/pr59305.ll +++ b/llvm/test/CodeGen/X86/pr59305.ll @@ -1,33 +1,33 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=x86_64-pc-linux < %s | FileCheck %s --check-prefix=X86-64 +; RUN: llc -mtriple=x86_64-pc-linux < %s | FileCheck %s --check-prefix=X64 ; RUN: llc -mtriple=i686-pc-linux < %s | FileCheck %s --check-prefix=X86 define double @foo(double %0) #0 { -; X86-64-LABEL: foo: -; X86-64: # %bb.0: -; X86-64-NEXT: subq $24, %rsp -; X86-64-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill -; X86-64-NEXT: movl $1024, %edi # imm = 0x400 -; X86-64-NEXT: callq fesetround@PLT -; X86-64-NEXT: movsd {{.*#+}} xmm1 = [1.0E+0,0.0E+0] -; X86-64-NEXT: divsd (%rsp), %xmm1 # 8-byte Folded Reload -; X86-64-NEXT: movsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; X86-64-NEXT: movl $1024, %edi # imm = 0x400 -; X86-64-NEXT: callq fesetround@PLT -; X86-64-NEXT: movsd {{.*#+}} xmm0 = [1.0E+0,0.0E+0] -; X86-64-NEXT: divsd (%rsp), %xmm0 # 8-byte Folded Reload -; X86-64-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; X86-64-NEXT: movl $1024, %edi # imm = 0x400 -; X86-64-NEXT: callq fesetround@PLT -; X86-64-NEXT: movsd {{.*#+}} xmm2 = [1.0E+0,0.0E+0] -; X86-64-NEXT: divsd (%rsp), %xmm2 # 8-byte Folded Reload -; X86-64-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload -; X86-64-NEXT: # xmm0 = mem[0],zero -; X86-64-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload -; X86-64-NEXT: # xmm1 = mem[0],zero -; X86-64-NEXT: callq fma@PLT -; X86-64-NEXT: addq $24, %rsp -; X86-64-NEXT: retq +; X64-LABEL: foo: +; X64: # %bb.0: +; X64-NEXT: subq $24, %rsp +; X64-NEXT: movsd %xmm0, (%rsp) # 8-byte Spill +; X64-NEXT: movl $1024, %edi # imm = 0x400 +; X64-NEXT: callq fesetround@PLT +; X64-NEXT: movsd {{.*#+}} xmm1 = [1.0E+0,0.0E+0] +; X64-NEXT: divsd (%rsp), %xmm1 # 8-byte Folded Reload +; X64-NEXT: movsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; X64-NEXT: movl $1024, %edi # imm = 0x400 +; X64-NEXT: callq fesetround@PLT +; X64-NEXT: movsd {{.*#+}} xmm0 = [1.0E+0,0.0E+0] +; X64-NEXT: divsd (%rsp), %xmm0 # 8-byte Folded Reload +; X64-NEXT: movsd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; X64-NEXT: movl $1024, %edi # imm = 0x400 +; X64-NEXT: callq fesetround@PLT +; X64-NEXT: movsd {{.*#+}} xmm2 = [1.0E+0,0.0E+0] +; X64-NEXT: divsd (%rsp), %xmm2 # 8-byte Folded Reload +; X64-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload +; X64-NEXT: # xmm0 = mem[0],zero +; X64-NEXT: movsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Reload +; X64-NEXT: # xmm1 = mem[0],zero +; X64-NEXT: callq fma@PLT +; X64-NEXT: addq $24, %rsp +; X64-NEXT: retq ; ; X86-LABEL: foo: ; X86: # %bb.0: -- GitLab From d1d2932cb1dc24e0c8149f07b75599981ac405a7 Mon Sep 17 00:00:00 2001 From: martinboehme Date: Tue, 5 Mar 2024 12:34:17 +0100 Subject: [PATCH 126/929] [Support][NFC] Add test documenting that empty `Regex` pattern matches nothing. (#83849) I was wondering about this when I recently used `Regex`, and I thought it would be nice to have a test documenting this behavior. --- llvm/unittests/Support/RegexTest.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/llvm/unittests/Support/RegexTest.cpp b/llvm/unittests/Support/RegexTest.cpp index 09f674bb209c..c6ac42591d1f 100644 --- a/llvm/unittests/Support/RegexTest.cpp +++ b/llvm/unittests/Support/RegexTest.cpp @@ -60,6 +60,14 @@ TEST_F(RegexTest, Basics) { EXPECT_TRUE(r5.match(String)); } +TEST_F(RegexTest, EmptyPattern) { + // The empty pattern doesn't match anything -- not even the empty string. + // (This is different from some other regex implementations.) + Regex r(""); + EXPECT_FALSE(r.match("123")); + EXPECT_FALSE(r.match("")); +} + TEST_F(RegexTest, Backreferences) { Regex r1("([a-z]+)_\\1"); SmallVector Matches; -- GitLab From 20895965b2ed1bd037c64430dba98245ffa1232b Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Tue, 5 Mar 2024 12:39:50 +0100 Subject: [PATCH 127/929] [NVPTX] Remove sub.s16x2 instruction According to the PTX ISA this doesn't exist (and ptxas rejects it) See https://github.com/pytorch/pytorch/issues/118589 --- llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 1 - llvm/lib/Target/NVPTX/NVPTXInstrInfo.td | 1 - llvm/test/CodeGen/NVPTX/i16x2-instructions.ll | 11 +++++------ 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp index 66a101036f91..c979c03dc1b8 100644 --- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp @@ -465,7 +465,6 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM, case ISD::SMIN: case ISD::UMIN: case ISD::UMAX: - case ISD::SUB: IsOpSupported = STI.getSmVersion() >= 90 && STI.getPTXVersion() >= 80; break; } diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td index b3517ce066b8..3dc5b450cbf5 100644 --- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td +++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td @@ -838,7 +838,6 @@ defm ADD : I3<"add.s", add>; defm SUB : I3<"sub.s", sub>; def ADD16x2 : I16x2<"add.s", add>; -def SUB16x2 : I16x2<"sub.s", sub>; // in32 and int64 addition and subtraction with carry-out. defm ADDCC : ADD_SUB_INT_CARRY<"add.cc", addc>; diff --git a/llvm/test/CodeGen/NVPTX/i16x2-instructions.ll b/llvm/test/CodeGen/NVPTX/i16x2-instructions.ll index 684e4bc38d83..ce9adfc7aa4f 100644 --- a/llvm/test/CodeGen/NVPTX/i16x2-instructions.ll +++ b/llvm/test/CodeGen/NVPTX/i16x2-instructions.ll @@ -123,13 +123,12 @@ define <2 x i16> @test_add_imm_1(<2 x i16> %a) #0 { ; COMMON-DAG: ld.param.u32 [[A:%r[0-9]+]], [test_sub_param_0]; ; ; COMMON-DAG: ld.param.u32 [[B:%r[0-9]+]], [test_sub_param_1]; -; I16x2: sub.s16x2 [[R:%r[0-9]+]], [[A]], [[B]]; ; -; NO-I16x2-DAG: mov.b32 {[[RS0:%rs[0-9]+]], [[RS1:%rs[0-9]+]]}, [[A]]; -; NO-I16x2-DAG: mov.b32 {[[RS2:%rs[0-9]+]], [[RS3:%rs[0-9]+]]}, [[B]]; -; NO-I16x2-DAG: sub.s16 [[RS4:%rs[0-9]+]], [[RS0]], [[RS2]]; -; NO-I16x2-DAG: sub.s16 [[RS5:%rs[0-9]+]], [[RS1]], [[RS3]]; -; NO-I16x2-DAG: mov.b32 [[R:%r[0-9]+]], {[[RS4]], [[RS5]]}; +; COMMON-DAG: mov.b32 {[[RS0:%rs[0-9]+]], [[RS1:%rs[0-9]+]]}, [[A]]; +; COMMON-DAG: mov.b32 {[[RS2:%rs[0-9]+]], [[RS3:%rs[0-9]+]]}, [[B]]; +; COMMON-DAG: sub.s16 [[RS4:%rs[0-9]+]], [[RS0]], [[RS2]]; +; COMMON-DAG: sub.s16 [[RS5:%rs[0-9]+]], [[RS1]], [[RS3]]; +; COMMON-DAG: mov.b32 [[R:%r[0-9]+]], {[[RS4]], [[RS5]]}; ; ; COMMON-NEXT: st.param.b32 [func_retval0+0], [[R]]; ; COMMON-NEXT: ret; -- GitLab From 94a0dd5a1960e1ace0a161228f91c2e18865d061 Mon Sep 17 00:00:00 2001 From: SahilPatidar <89641424+SahilPatidar@users.noreply.github.com> Date: Tue, 5 Mar 2024 17:19:01 +0530 Subject: [PATCH 128/929] [InstCombine] Fix Failure to convert vector fp comparisons that can be represented as integers #82241 (#83274) Resolve #82241 --------- Co-authored-by: SahilPatidar --- .../InstCombine/InstCombineCompares.cpp | 92 ++++++++++--------- .../InstCombine/cast-int-fcmp-eq-0.ll | 58 ++++++++++++ .../Transforms/InstCombine/clamp-to-minmax.ll | 17 ++++ llvm/test/Transforms/InstCombine/sitofp.ll | 37 ++++++++ 4 files changed, 159 insertions(+), 45 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp index 49e597171b1c..55207a8343d0 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp @@ -7286,8 +7286,9 @@ Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) { Instruction *InstCombinerImpl::foldFCmpIntToFPConst(FCmpInst &I, Instruction *LHSI, Constant *RHSC) { - if (!isa(RHSC)) return nullptr; - const APFloat &RHS = cast(RHSC)->getValueAPF(); + const APFloat *RHS; + if (!match(RHSC, m_APFloat(RHS))) + return nullptr; // Get the width of the mantissa. We don't want to hack on conversions that // might lose information from the integer, e.g. "i64 -> float" @@ -7302,20 +7303,20 @@ Instruction *InstCombinerImpl::foldFCmpIntToFPConst(FCmpInst &I, FCmpInst::Predicate P = I.getPredicate(); bool IsExact = false; APSInt RHSCvt(IntWidth, LHSUnsigned); - RHS.convertToInteger(RHSCvt, APFloat::rmNearestTiesToEven, &IsExact); + RHS->convertToInteger(RHSCvt, APFloat::rmNearestTiesToEven, &IsExact); // If the floating point constant isn't an integer value, we know if we will // ever compare equal / not equal to it. if (!IsExact) { // TODO: Can never be -0.0 and other non-representable values - APFloat RHSRoundInt(RHS); + APFloat RHSRoundInt(*RHS); RHSRoundInt.roundToIntegral(APFloat::rmNearestTiesToEven); - if (RHS != RHSRoundInt) { + if (*RHS != RHSRoundInt) { if (P == FCmpInst::FCMP_OEQ || P == FCmpInst::FCMP_UEQ) - return replaceInstUsesWith(I, Builder.getFalse()); + return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); assert(P == FCmpInst::FCMP_ONE || P == FCmpInst::FCMP_UNE); - return replaceInstUsesWith(I, Builder.getTrue()); + return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); } } @@ -7332,9 +7333,9 @@ Instruction *InstCombinerImpl::foldFCmpIntToFPConst(FCmpInst &I, // to distinguish it from one less than that value. if ((int)IntWidth > MantissaWidth) { // Conversion would lose accuracy. Check if loss can impact comparison. - int Exp = ilogb(RHS); + int Exp = ilogb(*RHS); if (Exp == APFloat::IEK_Inf) { - int MaxExponent = ilogb(APFloat::getLargest(RHS.getSemantics())); + int MaxExponent = ilogb(APFloat::getLargest(RHS->getSemantics())); if (MaxExponent < (int)IntWidth - !LHSUnsigned) // Conversion could create infinity. return nullptr; @@ -7350,7 +7351,7 @@ Instruction *InstCombinerImpl::foldFCmpIntToFPConst(FCmpInst &I, // Otherwise, we can potentially simplify the comparison. We know that it // will always come through as an integer value and we know the constant is // not a NAN (it would have been previously simplified). - assert(!RHS.isNaN() && "NaN comparison not already folded!"); + assert(!RHS->isNaN() && "NaN comparison not already folded!"); ICmpInst::Predicate Pred; switch (I.getPredicate()) { @@ -7380,9 +7381,9 @@ Instruction *InstCombinerImpl::foldFCmpIntToFPConst(FCmpInst &I, Pred = ICmpInst::ICMP_NE; break; case FCmpInst::FCMP_ORD: - return replaceInstUsesWith(I, Builder.getTrue()); + return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); case FCmpInst::FCMP_UNO: - return replaceInstUsesWith(I, Builder.getFalse()); + return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); } // Now we know that the APFloat is a normal number, zero or inf. @@ -7392,50 +7393,50 @@ Instruction *InstCombinerImpl::foldFCmpIntToFPConst(FCmpInst &I, if (!LHSUnsigned) { // If the RHS value is > SignedMax, fold the comparison. This handles +INF // and large values. - APFloat SMax(RHS.getSemantics()); + APFloat SMax(RHS->getSemantics()); SMax.convertFromAPInt(APInt::getSignedMaxValue(IntWidth), true, APFloat::rmNearestTiesToEven); - if (SMax < RHS) { // smax < 13123.0 + if (SMax < *RHS) { // smax < 13123.0 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) - return replaceInstUsesWith(I, Builder.getTrue()); - return replaceInstUsesWith(I, Builder.getFalse()); + return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); + return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); } } else { // If the RHS value is > UnsignedMax, fold the comparison. This handles // +INF and large values. - APFloat UMax(RHS.getSemantics()); + APFloat UMax(RHS->getSemantics()); UMax.convertFromAPInt(APInt::getMaxValue(IntWidth), false, APFloat::rmNearestTiesToEven); - if (UMax < RHS) { // umax < 13123.0 + if (UMax < *RHS) { // umax < 13123.0 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) - return replaceInstUsesWith(I, Builder.getTrue()); - return replaceInstUsesWith(I, Builder.getFalse()); + return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); + return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); } } if (!LHSUnsigned) { // See if the RHS value is < SignedMin. - APFloat SMin(RHS.getSemantics()); + APFloat SMin(RHS->getSemantics()); SMin.convertFromAPInt(APInt::getSignedMinValue(IntWidth), true, APFloat::rmNearestTiesToEven); - if (SMin > RHS) { // smin > 12312.0 + if (SMin > *RHS) { // smin > 12312.0 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) - return replaceInstUsesWith(I, Builder.getTrue()); - return replaceInstUsesWith(I, Builder.getFalse()); + return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); + return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); } } else { // See if the RHS value is < UnsignedMin. - APFloat UMin(RHS.getSemantics()); + APFloat UMin(RHS->getSemantics()); UMin.convertFromAPInt(APInt::getMinValue(IntWidth), false, APFloat::rmNearestTiesToEven); - if (UMin > RHS) { // umin > 12312.0 + if (UMin > *RHS) { // umin > 12312.0 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) - return replaceInstUsesWith(I, Builder.getTrue()); - return replaceInstUsesWith(I, Builder.getFalse()); + return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); + return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); } } @@ -7445,8 +7446,8 @@ Instruction *InstCombinerImpl::foldFCmpIntToFPConst(FCmpInst &I, // Don't do this for zero, because -0.0 is not fractional. APSInt RHSInt(IntWidth, LHSUnsigned); bool IsExact; - RHS.convertToInteger(RHSInt, APFloat::rmTowardZero, &IsExact); - if (!RHS.isZero()) { + RHS->convertToInteger(RHSInt, APFloat::rmTowardZero, &IsExact); + if (!RHS->isZero()) { if (!IsExact) { // If we had a comparison against a fractional value, we have to adjust // the compare predicate and sometimes the value. RHSC is rounded towards @@ -7454,57 +7455,57 @@ Instruction *InstCombinerImpl::foldFCmpIntToFPConst(FCmpInst &I, switch (Pred) { default: llvm_unreachable("Unexpected integer comparison!"); case ICmpInst::ICMP_NE: // (float)int != 4.4 --> true - return replaceInstUsesWith(I, Builder.getTrue()); + return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); case ICmpInst::ICMP_EQ: // (float)int == 4.4 --> false - return replaceInstUsesWith(I, Builder.getFalse()); + return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); case ICmpInst::ICMP_ULE: // (float)int <= 4.4 --> int <= 4 // (float)int <= -4.4 --> false - if (RHS.isNegative()) - return replaceInstUsesWith(I, Builder.getFalse()); + if (RHS->isNegative()) + return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); break; case ICmpInst::ICMP_SLE: // (float)int <= 4.4 --> int <= 4 // (float)int <= -4.4 --> int < -4 - if (RHS.isNegative()) + if (RHS->isNegative()) Pred = ICmpInst::ICMP_SLT; break; case ICmpInst::ICMP_ULT: // (float)int < -4.4 --> false // (float)int < 4.4 --> int <= 4 - if (RHS.isNegative()) - return replaceInstUsesWith(I, Builder.getFalse()); + if (RHS->isNegative()) + return replaceInstUsesWith(I, ConstantInt::getFalse(I.getType())); Pred = ICmpInst::ICMP_ULE; break; case ICmpInst::ICMP_SLT: // (float)int < -4.4 --> int < -4 // (float)int < 4.4 --> int <= 4 - if (!RHS.isNegative()) + if (!RHS->isNegative()) Pred = ICmpInst::ICMP_SLE; break; case ICmpInst::ICMP_UGT: // (float)int > 4.4 --> int > 4 // (float)int > -4.4 --> true - if (RHS.isNegative()) - return replaceInstUsesWith(I, Builder.getTrue()); + if (RHS->isNegative()) + return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); break; case ICmpInst::ICMP_SGT: // (float)int > 4.4 --> int > 4 // (float)int > -4.4 --> int >= -4 - if (RHS.isNegative()) + if (RHS->isNegative()) Pred = ICmpInst::ICMP_SGE; break; case ICmpInst::ICMP_UGE: // (float)int >= -4.4 --> true // (float)int >= 4.4 --> int > 4 - if (RHS.isNegative()) - return replaceInstUsesWith(I, Builder.getTrue()); + if (RHS->isNegative()) + return replaceInstUsesWith(I, ConstantInt::getTrue(I.getType())); Pred = ICmpInst::ICMP_UGT; break; case ICmpInst::ICMP_SGE: // (float)int >= -4.4 --> int >= -4 // (float)int >= 4.4 --> int > 4 - if (!RHS.isNegative()) + if (!RHS->isNegative()) Pred = ICmpInst::ICMP_SGT; break; } @@ -7513,7 +7514,8 @@ Instruction *InstCombinerImpl::foldFCmpIntToFPConst(FCmpInst &I, // Lower this FP comparison into an appropriate integer version of the // comparison. - return new ICmpInst(Pred, LHSI->getOperand(0), Builder.getInt(RHSInt)); + return new ICmpInst(Pred, LHSI->getOperand(0), + ConstantInt::get(LHSI->getOperand(0)->getType(), RHSInt)); } /// Fold (C / X) < 0.0 --> X < 0.0 if possible. Swap predicate if necessary. diff --git a/llvm/test/Transforms/InstCombine/cast-int-fcmp-eq-0.ll b/llvm/test/Transforms/InstCombine/cast-int-fcmp-eq-0.ll index b04033d94a26..68a386ec0042 100644 --- a/llvm/test/Transforms/InstCombine/cast-int-fcmp-eq-0.ll +++ b/llvm/test/Transforms/InstCombine/cast-int-fcmp-eq-0.ll @@ -509,3 +509,61 @@ define i1 @i128_cast_cmp_oeq_int_inf_uitofp(i128 %i) { %cmp = fcmp oeq float %f, 0x7FF0000000000000 ret i1 %cmp } + +define <2 x i1> @i32_vec_cast_cmp_oeq_vec_int_0_sitofp(<2 x i32> %i) { +; CHECK-LABEL: @i32_vec_cast_cmp_oeq_vec_int_0_sitofp( +; CHECK-NEXT: [[CMP:%.*]] = icmp eq <2 x i32> [[I:%.*]], zeroinitializer +; CHECK-NEXT: ret <2 x i1> [[CMP]] +; + %f = sitofp <2 x i32> %i to <2 x float> + %cmp = fcmp oeq <2 x float> %f, + ret <2 x i1> %cmp +} + +define <2 x i1> @i32_vec_cast_cmp_oeq_vec_int_n0_sitofp(<2 x i32> %i) { +; CHECK-LABEL: @i32_vec_cast_cmp_oeq_vec_int_n0_sitofp( +; CHECK-NEXT: [[CMP:%.*]] = icmp eq <2 x i32> [[I:%.*]], zeroinitializer +; CHECK-NEXT: ret <2 x i1> [[CMP]] +; + %f = sitofp <2 x i32> %i to <2 x float> + %cmp = fcmp oeq <2 x float> %f, + ret <2 x i1> %cmp +} + +define <2 x i1> @i32_vec_cast_cmp_oeq_vec_int_i32imax_sitofp(<2 x i32> %i) { +; CHECK-LABEL: @i32_vec_cast_cmp_oeq_vec_int_i32imax_sitofp( +; CHECK-NEXT: [[F:%.*]] = sitofp <2 x i32> [[I:%.*]] to <2 x float> +; CHECK-NEXT: [[CMP:%.*]] = fcmp oeq <2 x float> [[F]], +; CHECK-NEXT: ret <2 x i1> [[CMP]] +; + %f = sitofp <2 x i32> %i to <2 x float> + %cmp = fcmp oeq <2 x float> %f, + ret <2 x i1> %cmp +} + +define <2 x i1> @i32_vec_cast_cmp_oeq_vec_int_negi32umax_sitofp(<2 x i32> %i) { +; CHECK-LABEL: @i32_vec_cast_cmp_oeq_vec_int_negi32umax_sitofp( +; CHECK-NEXT: ret <2 x i1> zeroinitializer +; + %f = sitofp <2 x i32> %i to <2 x float> + %cmp = fcmp oeq <2 x float> %f, + ret <2 x i1> %cmp +} + +define <2 x i1> @i32_vec_cast_cmp_oeq_vec_half_sitofp(<2 x i32> %i) { +; CHECK-LABEL: @i32_vec_cast_cmp_oeq_vec_half_sitofp( +; CHECK-NEXT: ret <2 x i1> zeroinitializer +; + %f = sitofp <2 x i32> %i to <2 x float> + %cmp = fcmp oeq <2 x float> %f, + ret <2 x i1> %cmp +} + +define <2 x i1> @i32_vec_cast_cmp_oeq_vec_int_inf_sitofp(<2 x i32> %i) { +; CHECK-LABEL: @i32_vec_cast_cmp_oeq_vec_int_inf_sitofp( +; CHECK-NEXT: ret <2 x i1> zeroinitializer +; + %f = sitofp <2 x i32> %i to <2 x float> + %cmp = fcmp oeq <2 x float> %f, + ret <2 x i1> %cmp +} diff --git a/llvm/test/Transforms/InstCombine/clamp-to-minmax.ll b/llvm/test/Transforms/InstCombine/clamp-to-minmax.ll index fad1176cc18f..9da9eb36d381 100644 --- a/llvm/test/Transforms/InstCombine/clamp-to-minmax.ll +++ b/llvm/test/Transforms/InstCombine/clamp-to-minmax.ll @@ -566,3 +566,20 @@ define i32 @mixed_clamp_to_i32_2(float %x) { %r = select i1 %lo_cmp, i32 1, i32 %i32_min ret i32 %r } + + +define <2 x float> @mixed_clamp_to_float_vec(<2 x i32> %x) { +; CHECK-LABEL: @mixed_clamp_to_float_vec( +; CHECK-NEXT: [[SI_MIN:%.*]] = call <2 x i32> @llvm.smin.v2i32(<2 x i32> [[X:%.*]], <2 x i32> ) +; CHECK-NEXT: [[R1:%.*]] = call <2 x i32> @llvm.smax.v2i32(<2 x i32> [[SI_MIN]], <2 x i32> ) +; CHECK-NEXT: [[R:%.*]] = sitofp <2 x i32> [[R1]] to <2 x float> +; CHECK-NEXT: ret <2 x float> [[R]] +; + %si_min_cmp = icmp sgt <2 x i32> %x, + %si_min = select <2 x i1> %si_min_cmp, <2 x i32> , <2 x i32> %x + %f_min = sitofp <2 x i32> %si_min to <2 x float> + %f_x = sitofp <2 x i32> %x to <2 x float> + %lo_cmp = fcmp ult <2 x float> %f_x, + %r = select <2 x i1> %lo_cmp, <2 x float> , <2 x float> %f_min + ret <2 x float> %r +} diff --git a/llvm/test/Transforms/InstCombine/sitofp.ll b/llvm/test/Transforms/InstCombine/sitofp.ll index 5e0cf9448800..cc6b6425eb03 100644 --- a/llvm/test/Transforms/InstCombine/sitofp.ll +++ b/llvm/test/Transforms/InstCombine/sitofp.ll @@ -378,3 +378,40 @@ define i12 @u32_half_u12(i32 %x) { %r = fptoui half %h to i12 ret i12 %r } + +define <2 x i1> @i8_vec_sitofp_test1(<2 x i8> %A) { +; CHECK-LABEL: @i8_vec_sitofp_test1( +; CHECK-NEXT: ret <2 x i1> +; + %B = sitofp <2 x i8> %A to <2 x double> + %C = fcmp ult <2 x double> %B, + ret <2 x i1> %C +} + +define <2 x i1> @i8_vec_sitofp_test2(<2 x i8> %A) { +; CHECK-LABEL: @i8_vec_sitofp_test2( +; CHECK-NEXT: ret <2 x i1> +; + %B = sitofp <2 x i8> %A to <2 x double> + %C = fcmp ugt <2 x double> %B, + ret <2 x i1> %C +} + +define <2 x i1> @i8_vec_sitofp_test3(<2 x i8> %A) { +; CHECK-LABEL: @i8_vec_sitofp_test3( +; CHECK-NEXT: ret <2 x i1> +; + %B = sitofp <2 x i8> %A to <2 x double> + %C = fcmp ule <2 x double> %B, + ret <2 x i1> %C +} + +define <2 x i1> @i8_vec_sitofp_test4(<2 x i8> %A) { +; CHECK-LABEL: @i8_vec_sitofp_test4( +; CHECK-NEXT: [[C:%.*]] = icmp ne <2 x i8> [[A:%.*]], +; CHECK-NEXT: ret <2 x i1> [[C]] +; + %B = sitofp <2 x i8> %A to <2 x double> + %C = fcmp ult <2 x double> %B, + ret <2 x i1> %C +} -- GitLab From de1f33873beff93063577195e1214a9509e229e0 Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Tue, 5 Mar 2024 19:54:02 +0800 Subject: [PATCH 129/929] [TableGen] Fix wrong codegen of BothFusionPredicateWithMCInstPredicate (#83990) We should generate the `MCInstPredicate` twice, one with `FirstMI` and another with `SecondMI`. --- llvm/include/llvm/Target/TargetSchedule.td | 2 +- llvm/test/TableGen/MacroFusion.td | 25 +++++++++++++++++++ .../TableGen/MacroFusionPredicatorEmitter.cpp | 12 ++++----- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/llvm/include/llvm/Target/TargetSchedule.td b/llvm/include/llvm/Target/TargetSchedule.td index 48c9387977c0..069eb2900bfe 100644 --- a/llvm/include/llvm/Target/TargetSchedule.td +++ b/llvm/include/llvm/Target/TargetSchedule.td @@ -617,7 +617,7 @@ class SecondFusionPredicateWithMCInstPredicate : FusionPredicateWithMCInstPredicate; // The pred will be applied on both firstMI and secondMI. class BothFusionPredicateWithMCInstPredicate - : FusionPredicateWithMCInstPredicate; + : FusionPredicateWithMCInstPredicate; // Tie firstOpIdx and secondOpIdx. The operand of `FirstMI` at position // `firstOpIdx` should be the same as the operand of `SecondMI` at position diff --git a/llvm/test/TableGen/MacroFusion.td b/llvm/test/TableGen/MacroFusion.td index 4aa6c8d9acb2..ce76e7f0f7fa 100644 --- a/llvm/test/TableGen/MacroFusion.td +++ b/llvm/test/TableGen/MacroFusion.td @@ -34,6 +34,11 @@ let Namespace = "Test" in { def Inst0 : TestInst<0>; def Inst1 : TestInst<1>; +def BothFusionPredicate: BothFusionPredicateWithMCInstPredicate>; +def TestBothFusionPredicate: Fusion<"test-both-fusion-predicate", "HasBothFusionPredicate", + "Test BothFusionPredicate", + [BothFusionPredicate]>; + def TestFusion: SimpleFusion<"test-fusion", "HasTestFusion", "Test Fusion", CheckOpcode<[Inst0]>, CheckAll<[ @@ -45,6 +50,7 @@ def TestFusion: SimpleFusion<"test-fusion", "HasTestFusion", "Test Fusion", // CHECK-PREDICATOR-NEXT: #undef GET_Test_MACRO_FUSION_PRED_DECL // CHECK-PREDICATOR-EMPTY: // CHECK-PREDICATOR-NEXT: namespace llvm { +// CHECK-PREDICATOR-NEXT: bool isTestBothFusionPredicate(const TargetInstrInfo &, const TargetSubtargetInfo &, const MachineInstr *, const MachineInstr &); // CHECK-PREDICATOR-NEXT: bool isTestFusion(const TargetInstrInfo &, const TargetSubtargetInfo &, const MachineInstr *, const MachineInstr &); // CHECK-PREDICATOR-NEXT: } // end namespace llvm // CHECK-PREDICATOR-EMPTY: @@ -54,6 +60,24 @@ def TestFusion: SimpleFusion<"test-fusion", "HasTestFusion", "Test Fusion", // CHECK-PREDICATOR-NEXT: #undef GET_Test_MACRO_FUSION_PRED_IMPL // CHECK-PREDICATOR-EMPTY: // CHECK-PREDICATOR-NEXT: namespace llvm { +// CHECK-PREDICATOR-NEXT: bool isTestBothFusionPredicate( +// CHECK-PREDICATOR-NEXT: const TargetInstrInfo &TII, +// CHECK-PREDICATOR-NEXT: const TargetSubtargetInfo &STI, +// CHECK-PREDICATOR-NEXT: const MachineInstr *FirstMI, +// CHECK-PREDICATOR-NEXT: const MachineInstr &SecondMI) { +// CHECK-PREDICATOR-NEXT: auto &MRI = SecondMI.getMF()->getRegInfo(); +// CHECK-PREDICATOR-NEXT: { +// CHECK-PREDICATOR-NEXT: const MachineInstr *MI = FirstMI; +// CHECK-PREDICATOR-NEXT: if (MI->getOperand(0).getReg() != Test::X0) +// CHECK-PREDICATOR-NEXT: return false; +// CHECK-PREDICATOR-NEXT: } +// CHECK-PREDICATOR-NEXT: { +// CHECK-PREDICATOR-NEXT: const MachineInstr *MI = &SecondMI; +// CHECK-PREDICATOR-NEXT: if (MI->getOperand(0).getReg() != Test::X0) +// CHECK-PREDICATOR-NEXT: return false; +// CHECK-PREDICATOR-NEXT: } +// CHECK-PREDICATOR-NEXT: return true; +// CHECK-PREDICATOR-NEXT: } // CHECK-PREDICATOR-NEXT: bool isTestFusion( // CHECK-PREDICATOR-NEXT: const TargetInstrInfo &TII, // CHECK-PREDICATOR-NEXT: const TargetSubtargetInfo &STI, @@ -106,6 +130,7 @@ def TestFusion: SimpleFusion<"test-fusion", "HasTestFusion", "Test Fusion", // CHECK-SUBTARGET: std::vector TestGenSubtargetInfo::getMacroFusions() const { // CHECK-SUBTARGET-NEXT: std::vector Fusions; +// CHECK-SUBTARGET-NEXT: if (hasFeature(Test::TestBothFusionPredicate)) Fusions.push_back(llvm::isTestBothFusionPredicate); // CHECK-SUBTARGET-NEXT: if (hasFeature(Test::TestFusion)) Fusions.push_back(llvm::isTestFusion); // CHECK-SUBTARGET-NEXT: return Fusions; // CHECK-SUBTARGET-NEXT: } diff --git a/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp b/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp index 78dcd4471ae7..7f494e532b1f 100644 --- a/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp +++ b/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp @@ -152,8 +152,7 @@ void MacroFusionPredicatorEmitter::emitFirstPredicate(Record *Predicate, << "if (FirstDest.isVirtual() && !MRI.hasOneNonDBGUse(FirstDest))\n"; OS.indent(4) << " return false;\n"; OS.indent(2) << "}\n"; - } else if (Predicate->isSubClassOf( - "FirstFusionPredicateWithMCInstPredicate")) { + } else if (Predicate->isSubClassOf("FusionPredicateWithMCInstPredicate")) { OS.indent(2) << "{\n"; OS.indent(4) << "const MachineInstr *MI = FirstMI;\n"; OS.indent(4) << "if ("; @@ -173,7 +172,7 @@ void MacroFusionPredicatorEmitter::emitFirstPredicate(Record *Predicate, void MacroFusionPredicatorEmitter::emitSecondPredicate(Record *Predicate, PredicateExpander &PE, raw_ostream &OS) { - if (Predicate->isSubClassOf("SecondFusionPredicateWithMCInstPredicate")) { + if (Predicate->isSubClassOf("FusionPredicateWithMCInstPredicate")) { OS.indent(2) << "{\n"; OS.indent(4) << "const MachineInstr *MI = &SecondMI;\n"; OS.indent(4) << "if ("; @@ -185,7 +184,7 @@ void MacroFusionPredicatorEmitter::emitSecondPredicate(Record *Predicate, OS.indent(2) << "}\n"; } else { PrintFatalError(Predicate->getLoc(), - "Unsupported predicate for first instruction: " + + "Unsupported predicate for second instruction: " + Predicate->getType()->getAsString()); } } @@ -196,9 +195,8 @@ void MacroFusionPredicatorEmitter::emitBothPredicate(Record *Predicate, if (Predicate->isSubClassOf("FusionPredicateWithCode")) OS << Predicate->getValueAsString("Predicate"); else if (Predicate->isSubClassOf("BothFusionPredicateWithMCInstPredicate")) { - Record *MCPred = Predicate->getValueAsDef("Predicate"); - emitFirstPredicate(MCPred, PE, OS); - emitSecondPredicate(MCPred, PE, OS); + emitFirstPredicate(Predicate, PE, OS); + emitSecondPredicate(Predicate, PE, OS); } else if (Predicate->isSubClassOf("TieReg")) { int FirstOpIdx = Predicate->getValueAsInt("FirstOpIdx"); int SecondOpIdx = Predicate->getValueAsInt("SecondOpIdx"); -- GitLab From 4693efe19c1cbb19ecac61e46802ffe74bd1a7ce Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Tue, 5 Mar 2024 11:58:17 +0000 Subject: [PATCH 130/929] [AMDGPU] Remove Base_MUBUF_Real_Atomic_gfx11. NFC. (#83994) This class only existed to set the dlc bit for GFX11 atomics. It is simpler to set dlc for all loads/stores/atomics in the base class. --- llvm/lib/Target/AMDGPU/BUFInstructions.td | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index a2ff56d68218..20e91da4995b 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -2301,7 +2301,8 @@ class MUBUF_Real_gfx11 op, MUBUF_Pseudo ps, string real_name = ps.Mnemonic> : Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11 { let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); - let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); + // In GFX11 dlc is applicable to all loads/stores/atomics. + let Inst{13} = !if(!or(ps.mayLoad, ps.mayStore), cpol{CPolBit.DLC}, ps.dlc_value); let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); let Inst{25-18} = op; let Inst{53} = ps.tfe; @@ -2311,12 +2312,6 @@ class MUBUF_Real_gfx11 op, MUBUF_Pseudo ps, let DecoderNamespace = "GFX11"; } -class Base_MUBUF_Real_Atomic_gfx11 op, MUBUF_Pseudo ps, - string real_name> : - MUBUF_Real_gfx11 { - let Inst{13} = cpol{CPolBit.DLC}; -} - class Base_MUBUF_Real_gfx6_gfx7_gfx10 op, MUBUF_Pseudo ps, int ef> : Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11 { let Inst{12} = ps.offen; @@ -2499,7 +2494,7 @@ multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Renamed op, string real_name> class MUBUF_Real_Atomic_gfx11_impl op, string ps_name, string real_name> : - Base_MUBUF_Real_Atomic_gfx11(ps_name), real_name>; + MUBUF_Real_gfx11(ps_name), real_name>; class MUBUF_Real_Atomic_gfx12_impl op, string ps_name, string real_name> : -- GitLab From 67a7a5e89d31cc332f2d782c30623dd3ddbd0862 Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Tue, 5 Mar 2024 12:01:45 +0000 Subject: [PATCH 131/929] [AMDGPU] Only use the BUF Base_ prefix for multiple architectures. NFC. The Base_ prefix seems redundant on a class that is only used for GFX11. --- llvm/lib/Target/AMDGPU/BUFInstructions.td | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index 20e91da4995b..a8af95422085 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -2892,8 +2892,8 @@ class Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11 op, MTBUF_Pseudo ps, - string real_name = ps.Mnemonic> : +class MTBUF_Real_gfx11 op, MTBUF_Pseudo ps, + string real_name = ps.Mnemonic> : Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11 { let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); @@ -2921,13 +2921,13 @@ class Base_MTBUF_Real_gfx6_gfx7_gfx10 op, MTBUF_Pseudo ps, int ef> : multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl op, string real_name> { def _BOTHEN_gfx11 : - Base_MTBUF_Real_gfx11(NAME#"_BOTHEN"), real_name>; + MTBUF_Real_gfx11(NAME#"_BOTHEN"), real_name>; def _IDXEN_gfx11 : - Base_MTBUF_Real_gfx11(NAME#"_IDXEN"), real_name>; + MTBUF_Real_gfx11(NAME#"_IDXEN"), real_name>; def _OFFEN_gfx11 : - Base_MTBUF_Real_gfx11(NAME#"_OFFEN"), real_name>; + MTBUF_Real_gfx11(NAME#"_OFFEN"), real_name>; def _OFFSET_gfx11 : - Base_MTBUF_Real_gfx11(NAME#"_OFFSET"), real_name>; + MTBUF_Real_gfx11(NAME#"_OFFSET"), real_name>; def _BOTHEN_gfx12 : VBUFFER_MTBUF_Real_gfx12(NAME#"_VBUFFER_BOTHEN"), real_name>; -- GitLab From 90e97e71060cea8cb8bc4099d724d1fe9912cfc4 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Tue, 5 Mar 2024 06:29:22 -0600 Subject: [PATCH 132/929] [libc] Disable verbose logging messages on hermetic tests (#83954) Summary: The other test locations only give these messages when we are in verbose logging mode. The average user does not care about which tests are not being built, and most platforms will have missing tests. --- libc/cmake/modules/LLVMLibCTestRules.cmake | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libc/cmake/modules/LLVMLibCTestRules.cmake b/libc/cmake/modules/LLVMLibCTestRules.cmake index 836e15d34741..0bdd72091fe8 100644 --- a/libc/cmake/modules/LLVMLibCTestRules.cmake +++ b/libc/cmake/modules/LLVMLibCTestRules.cmake @@ -595,9 +595,11 @@ function(add_libc_hermetic_test test_name) get_object_files_for_test( link_object_files skipped_entrypoints_list ${fq_deps_list}) if(skipped_entrypoints_list) - set(msg "Skipping hermetic test ${fq_target_name} as it has missing deps: " - "${skipped_entrypoints_list}.") - message(STATUS ${msg}) + if(LIBC_CMAKE_VERBOSE_LOGGING) + set(msg "Skipping hermetic test ${fq_target_name} as it has missing deps: " + "${skipped_entrypoints_list}.") + message(STATUS ${msg}) + endif() return() endif() list(REMOVE_DUPLICATES link_object_files) -- GitLab From 341d674b6f1863d027ed30c44a14cd32599eb42d Mon Sep 17 00:00:00 2001 From: Paul Walker Date: Tue, 5 Mar 2024 12:34:15 +0000 Subject: [PATCH 133/929] [LLVM][AArch64][CodeGen] Mark FFR as a reserved register. (#83437) This allows the removal of FFR related psuedo nodes that only existed to work round machine verifier failures. --- .../Target/AArch64/AArch64RegisterInfo.cpp | 4 + .../lib/Target/AArch64/AArch64SVEInstrInfo.td | 10 +- llvm/lib/Target/AArch64/AArch64SchedA510.td | 52 ++-- .../Target/AArch64/AArch64SchedNeoverseN2.td | 52 ++-- .../Target/AArch64/AArch64SchedNeoverseV1.td | 42 +-- .../Target/AArch64/AArch64SchedNeoverseV2.td | 52 ++-- llvm/lib/Target/AArch64/SVEInstrFormats.td | 243 +++++------------ .../AArch64/sve-fixed-length-mask-opt.ll | 4 +- .../AArch64/sve-fixed-length-masked-gather.ll | 120 ++++----- .../CodeGen/AArch64/sve-fp-int-min-max.ll | 2 +- .../AArch64/sve-gather-scatter-dag-combine.ll | 28 +- ...cs-ff-gather-loads-32bit-scaled-offsets.ll | 2 +- ...-ff-gather-loads-32bit-unscaled-offsets.ll | 2 +- ...ics-ff-gather-loads-64bit-scaled-offset.ll | 2 +- ...s-ff-gather-loads-64bit-unscaled-offset.ll | 2 +- ...-ff-gather-loads-vector-base-imm-offset.ll | 2 +- ...-gather-loads-vector-base-scalar-offset.ll | 2 +- .../sve-intrinsics-ffr-manipulation.ll | 2 +- .../AArch64/sve-intrinsics-loads-ff.ll | 2 +- .../AArch64/sve-intrinsics-loads-nf.ll | 2 +- llvm/test/CodeGen/AArch64/sve-ldnf1.mir | 248 +++++++++--------- .../AArch64/sve-masked-gather-legalize.ll | 2 +- .../AArch64/sve-ptest-removal-rdffr.mir | 8 +- .../AArch64/A64FX/A64FX-sve-instructions.s | 8 +- .../AArch64/Cortex/A510-sve-instructions.s | 8 +- .../AArch64/Neoverse/N2-sve-instructions.s | 8 +- .../AArch64/Neoverse/V1-sve-instructions.s | 8 +- .../AArch64/Neoverse/V2-sve-instructions.s | 8 +- 28 files changed, 411 insertions(+), 514 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp index 531f21f9c043..ec59778cbb0e 100644 --- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp @@ -437,6 +437,10 @@ AArch64RegisterInfo::getStrictlyReservedRegs(const MachineFunction &MF) const { if (MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening)) markSuperRegs(Reserved, AArch64::W16); + // FFR is modelled as global state that cannot be allocated. + if (MF.getSubtarget().hasSVE()) + Reserved.set(AArch64::FFR); + // SME tiles are not allocatable. if (MF.getSubtarget().hasSME()) { for (MCPhysReg SubReg : subregs_inclusive(AArch64::ZA)) diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index 2b0524754b2e..7c98f934a131 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -488,11 +488,11 @@ def AArch64fmin_m1 : VSelectCommPredOrPassthruPatFrags; let Predicates = [HasSVE] in { - defm RDFFR_PPz : sve_int_rdffr_pred<0b0, "rdffr", int_aarch64_sve_rdffr_z>; - def RDFFRS_PPz : sve_int_rdffr_pred<0b1, "rdffrs">; - defm RDFFR_P : sve_int_rdffr_unpred<"rdffr", int_aarch64_sve_rdffr>; - def SETFFR : sve_int_setffr<"setffr", int_aarch64_sve_setffr>; - def WRFFR : sve_int_wrffr<"wrffr", int_aarch64_sve_wrffr>; + def RDFFR_PPz : sve_int_rdffr_pred<0b0, "rdffr", int_aarch64_sve_rdffr_z>; + def RDFFRS_PPz : sve_int_rdffr_pred<0b1, "rdffrs">; + def RDFFR_P : sve_int_rdffr_unpred<"rdffr", int_aarch64_sve_rdffr>; + def SETFFR : sve_int_setffr<"setffr", int_aarch64_sve_setffr>; + def WRFFR : sve_int_wrffr<"wrffr", int_aarch64_sve_wrffr>; } // End HasSVE let Predicates = [HasSVEorSME] in { diff --git a/llvm/lib/Target/AArch64/AArch64SchedA510.td b/llvm/lib/Target/AArch64/AArch64SchedA510.td index 5e36b6f4d34a..68343674bc81 100644 --- a/llvm/lib/Target/AArch64/AArch64SchedA510.td +++ b/llvm/lib/Target/AArch64/AArch64SchedA510.td @@ -1198,24 +1198,24 @@ def : InstRW<[CortexA510Write<3, CortexA510UnitLdSt>], (instregex "^LDNT1[BHWD]_ def : InstRW<[CortexA510Write<3, CortexA510UnitLdSt>], (instregex "^LDNT1[BHWD]_ZRR$")>; // Non temporal gather load, vector + scalar 32-bit element size -def : InstRW<[CortexA510MCWrite<9, 9, CortexA510UnitLdSt>], (instregex "^LDNT1[BHW]_ZZR_S_REAL$", - "^LDNT1S[BH]_ZZR_S_REAL$")>; +def : InstRW<[CortexA510MCWrite<9, 9, CortexA510UnitLdSt>], (instregex "^LDNT1[BHW]_ZZR_S$", + "^LDNT1S[BH]_ZZR_S$")>; // Non temporal gather load, vector + scalar 64-bit element size -def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLdSt>], (instregex "^LDNT1S?[BHW]_ZZR_D_REAL$")>; -def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLdSt>], (instrs LDNT1D_ZZR_D_REAL)>; +def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLdSt>], (instregex "^LDNT1S?[BHW]_ZZR_D$")>; +def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLdSt>], (instrs LDNT1D_ZZR_D)>; // Contiguous first faulting load, scalar + scalar -def : InstRW<[CortexA510Write<3, CortexA510UnitLd>], (instregex "^LDFF1[BHWD]_REAL$", - "^LDFF1S?B_[HSD]_REAL$", - "^LDFF1S?H_[SD]_REAL$", - "^LDFF1S?W_D_REAL$")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitLd>], (instregex "^LDFF1[BHWD]$", + "^LDFF1S?B_[HSD]$", + "^LDFF1S?H_[SD]$", + "^LDFF1S?W_D$")>; // Contiguous non faulting load, scalar + imm -def : InstRW<[CortexA510Write<3, CortexA510UnitLd>], (instregex "^LDNF1[BHWD]_IMM_REAL$", - "^LDNF1S?B_[HSD]_IMM_REAL$", - "^LDNF1S?H_[SD]_IMM_REAL$", - "^LDNF1S?W_D_IMM_REAL$")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitLd>], (instregex "^LDNF1[BHWD]_IMM$", + "^LDNF1S?B_[HSD]_IMM$", + "^LDNF1S?H_[SD]_IMM$", + "^LDNF1S?W_D_IMM$")>; // Contiguous Load two structures to two vectors, scalar + imm def : InstRW<[CortexA510MCWrite<3, 1, CortexA510UnitLdSt>], (instregex "^LD2[BHWD]_IMM$")>; @@ -1236,28 +1236,28 @@ def : InstRW<[CortexA510MCWrite<5, 3, CortexA510UnitLdSt>], (instregex "^LD4[BHW def : InstRW<[CortexA510MCWrite<5, 3, CortexA510UnitLdSt>], (instregex "^LD4[BHWD]$")>; // Gather load, vector + imm, 32-bit element size -def : InstRW<[CortexA510MCWrite<9, 9, CortexA510UnitLdSt>], (instregex "^GLD(FF)?1S?[BH]_S_IMM_REAL$", - "^GLD(FF)?1W_IMM_REAL$")>; +def : InstRW<[CortexA510MCWrite<9, 9, CortexA510UnitLdSt>], (instregex "^GLD(FF)?1S?[BH]_S_IMM$", + "^GLD(FF)?1W_IMM$")>; // Gather load, vector + imm, 64-bit element size -def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLdSt>], (instregex "^GLD(FF)?1S?[BHW]_D_IMM_REAL$", - "^GLD(FF)?1D_IMM_REAL$")>; +def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLdSt>], (instregex "^GLD(FF)?1S?[BHW]_D_IMM$", + "^GLD(FF)?1D_IMM$")>; // Gather load, 64-bit element size def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLdSt>], - (instregex "^GLD(FF)?1S?[BHW]_D_[SU]XTW_(SCALED_)?REAL$", - "^GLD(FF)?1S?[BHW]_D_(SCALED_)?REAL$", - "^GLD(FF)?1D_[SU]XTW_(SCALED_)?REAL$", - "^GLD(FF)?1D_(SCALED_)?REAL$")>; + (instregex "^GLD(FF)?1S?[BHW]_D_[SU]XTW(_SCALED)?$", + "^GLD(FF)?1S?[BHW]_D(_SCALED)?$", + "^GLD(FF)?1D_[SU]XTW(_SCALED)?$", + "^GLD(FF)?1D(_SCALED)?$")>; // Gather load, 32-bit scaled offset def : InstRW<[CortexA510MCWrite<9, 9, CortexA510UnitLd>], - (instregex "^GLD(FF)?1S?[HW]_S_[SU]XTW_SCALED_REAL$", - "^GLD(FF)?1W_[SU]XTW_SCALED_REAL")>; + (instregex "^GLD(FF)?1S?[HW]_S_[SU]XTW_SCALED$", + "^GLD(FF)?1W_[SU]XTW_SCALED")>; // Gather load, 32-bit unpacked unscaled offset -def : InstRW<[CortexA510MCWrite<9, 9, CortexA510UnitLd>], (instregex "^GLD(FF)?1S?[BH]_S_[SU]XTW_REAL$", - "^GLD(FF)?1W_[SU]XTW_REAL$")>; +def : InstRW<[CortexA510MCWrite<9, 9, CortexA510UnitLd>], (instregex "^GLD(FF)?1S?[BH]_S_[SU]XTW$", + "^GLD(FF)?1W_[SU]XTW$")>; def : InstRW<[CortexA510Write<0, CortexA510UnitVALU>], (instregex "^PRF(B|H|W|D).*")>; // SVE Store instructions @@ -1357,10 +1357,10 @@ def : InstRW<[CortexA510VSt<8>], (instregex "^SST1[BHW]_D$", // ----------------------------------------------------------------------------- // Read first fault register, unpredicated -def : InstRW<[CortexA510Write<1, CortexA510UnitALU>], (instrs RDFFR_P_REAL)>; +def : InstRW<[CortexA510Write<1, CortexA510UnitALU>], (instrs RDFFR_P)>; // Read first fault register, predicated -def : InstRW<[CortexA510Write<3, CortexA510UnitALU0>], (instrs RDFFR_PPz_REAL)>; +def : InstRW<[CortexA510Write<3, CortexA510UnitALU0>], (instrs RDFFR_PPz)>; // Read first fault register and set flags def : InstRW<[CortexA510Write<3, CortexA510UnitALU0>], (instrs RDFFRS_PPz)>; diff --git a/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td b/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td index 8ec124954362..c18045e7c8f9 100644 --- a/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td +++ b/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td @@ -2110,24 +2110,24 @@ def : InstRW<[N2Write_6cyc_1L], (instregex "^LDNT1[BHWD]_ZRI$")>; def : InstRW<[N2Write_6cyc_1L_1S], (instregex "^LDNT1[BHWD]_ZRR$")>; // Non temporal gather load, vector + scalar 32-bit element size -def : InstRW<[N2Write_9cyc_1L_1V], (instregex "^LDNT1[BHW]_ZZR_S_REAL$", - "^LDNT1S[BH]_ZZR_S_REAL$")>; +def : InstRW<[N2Write_9cyc_1L_1V], (instregex "^LDNT1[BHW]_ZZR_S$", + "^LDNT1S[BH]_ZZR_S$")>; // Non temporal gather load, vector + scalar 64-bit element size -def : InstRW<[N2Write_10cyc_2L_2V1], (instregex "^LDNT1S?[BHW]_ZZR_D_REAL$")>; -def : InstRW<[N2Write_10cyc_2L_2V1], (instrs LDNT1D_ZZR_D_REAL)>; +def : InstRW<[N2Write_10cyc_2L_2V1], (instregex "^LDNT1S?[BHW]_ZZR_D$")>; +def : InstRW<[N2Write_10cyc_2L_2V1], (instrs LDNT1D_ZZR_D)>; // Contiguous first faulting load, scalar + scalar -def : InstRW<[N2Write_6cyc_1L_1S], (instregex "^LDFF1[BHWD]_REAL$", - "^LDFF1S?B_[HSD]_REAL$", - "^LDFF1S?H_[SD]_REAL$", - "^LDFF1S?W_D_REAL$")>; +def : InstRW<[N2Write_6cyc_1L_1S], (instregex "^LDFF1[BHWD]$", + "^LDFF1S?B_[HSD]$", + "^LDFF1S?H_[SD]$", + "^LDFF1S?W_D$")>; // Contiguous non faulting load, scalar + imm -def : InstRW<[N2Write_6cyc_1L], (instregex "^LDNF1[BHWD]_IMM_REAL$", - "^LDNF1S?B_[HSD]_IMM_REAL$", - "^LDNF1S?H_[SD]_IMM_REAL$", - "^LDNF1S?W_D_IMM_REAL$")>; +def : InstRW<[N2Write_6cyc_1L], (instregex "^LDNF1[BHWD]_IMM$", + "^LDNF1S?B_[HSD]_IMM$", + "^LDNF1S?H_[SD]_IMM$", + "^LDNF1S?W_D_IMM$")>; // Contiguous Load two structures to two vectors, scalar + imm def : InstRW<[N2Write_8cyc_1L_1V], (instregex "^LD2[BHWD]_IMM$")>; @@ -2148,28 +2148,28 @@ def : InstRW<[N2Write_9cyc_2L_2V], (instregex "^LD4[BHWD]_IMM$")>; def : InstRW<[N2Write_10cyc_2L_2V_2S], (instregex "^LD4[BHWD]$")>; // Gather load, vector + imm, 32-bit element size -def : InstRW<[N2Write_9cyc_1L_1V], (instregex "^GLD(FF)?1S?[BH]_S_IMM_REAL$", - "^GLD(FF)?1W_IMM_REAL$")>; +def : InstRW<[N2Write_9cyc_1L_1V], (instregex "^GLD(FF)?1S?[BH]_S_IMM$", + "^GLD(FF)?1W_IMM$")>; // Gather load, vector + imm, 64-bit element size -def : InstRW<[N2Write_9cyc_2L_2V], (instregex "^GLD(FF)?1S?[BHW]_D_IMM_REAL$", - "^GLD(FF)?1D_IMM_REAL$")>; +def : InstRW<[N2Write_9cyc_2L_2V], (instregex "^GLD(FF)?1S?[BHW]_D_IMM$", + "^GLD(FF)?1D_IMM$")>; // Gather load, 64-bit element size def : InstRW<[N2Write_9cyc_2L_2V], - (instregex "^GLD(FF)?1S?[BHW]_D_[SU]XTW_(SCALED_)?REAL$", - "^GLD(FF)?1S?[BHW]_D_(SCALED_)?REAL$", - "^GLD(FF)?1D_[SU]XTW_(SCALED_)?REAL$", - "^GLD(FF)?1D_(SCALED_)?REAL$")>; + (instregex "^GLD(FF)?1S?[BHW]_D_[SU]XTW(_SCALED)?$", + "^GLD(FF)?1S?[BHW]_D(_SCALED)?$", + "^GLD(FF)?1D_[SU]XTW(_SCALED)?$", + "^GLD(FF)?1D(_SCALED)?$")>; // Gather load, 32-bit scaled offset def : InstRW<[N2Write_10cyc_2L_2V], - (instregex "^GLD(FF)?1S?[HW]_S_[SU]XTW_SCALED_REAL$", - "^GLD(FF)?1W_[SU]XTW_SCALED_REAL")>; + (instregex "^GLD(FF)?1S?[HW]_S_[SU]XTW_SCALED$", + "^GLD(FF)?1W_[SU]XTW_SCALED")>; // Gather load, 32-bit unpacked unscaled offset -def : InstRW<[N2Write_9cyc_1L_1V], (instregex "^GLD(FF)?1S?[BH]_S_[SU]XTW_REAL$", - "^GLD(FF)?1W_[SU]XTW_REAL$")>; +def : InstRW<[N2Write_9cyc_1L_1V], (instregex "^GLD(FF)?1S?[BH]_S_[SU]XTW$", + "^GLD(FF)?1W_[SU]XTW$")>; // SVE Store instructions // ----------------------------------------------------------------------------- @@ -2268,10 +2268,10 @@ def : InstRW<[N2Write_2cyc_1L01_1V], (instregex "^SST1[BHW]_D$", // ----------------------------------------------------------------------------- // Read first fault register, unpredicated -def : InstRW<[N2Write_2cyc_1M0], (instrs RDFFR_P_REAL)>; +def : InstRW<[N2Write_2cyc_1M0], (instrs RDFFR_P)>; // Read first fault register, predicated -def : InstRW<[N2Write_3cyc_1M0_1M], (instrs RDFFR_PPz_REAL)>; +def : InstRW<[N2Write_3cyc_1M0_1M], (instrs RDFFR_PPz)>; // Read first fault register and set flags def : InstRW<[N2Write_4cyc_2M0_2M], (instrs RDFFRS_PPz)>; diff --git a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td index 613db353cb0a..e50a401f8b2a 100644 --- a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td +++ b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td @@ -1714,17 +1714,17 @@ def : InstRW<[V1Write_7c_1L01_1S], (instrs LDNT1H_ZRR)>; def : InstRW<[V1Write_6c_1L01_1S], (instregex "^LDNT1[BWD]_ZRR$")>; // Contiguous first faulting load, scalar + scalar -def : InstRW<[V1Write_7c_1L01_1S], (instregex "^LDFF1H_REAL$", - "^LDFF1S?H_[SD]_REAL$")>; -def : InstRW<[V1Write_6c_1L01_1S], (instregex "^LDFF1[BWD]_REAL$", - "^LDFF1S?B_[HSD]_REAL$", - "^LDFF1S?W_D_REAL$")>; +def : InstRW<[V1Write_7c_1L01_1S], (instregex "^LDFF1H$", + "^LDFF1S?H_[SD]$")>; +def : InstRW<[V1Write_6c_1L01_1S], (instregex "^LDFF1[BWD]$", + "^LDFF1S?B_[HSD]$", + "^LDFF1S?W_D$")>; // Contiguous non faulting load, scalar + imm -def : InstRW<[V1Write_6c_1L01], (instregex "^LDNF1[BHWD]_IMM_REAL$", - "^LDNF1S?B_[HSD]_IMM_REAL$", - "^LDNF1S?H_[SD]_IMM_REAL$", - "^LDNF1S?W_D_IMM_REAL$")>; +def : InstRW<[V1Write_6c_1L01], (instregex "^LDNF1[BHWD]_IMM$", + "^LDNF1S?B_[HSD]_IMM$", + "^LDNF1S?H_[SD]_IMM$", + "^LDNF1S?W_D_IMM$")>; // Contiguous Load two structures to two vectors, scalar + imm def : InstRW<[V1Write_8c_2L01_2V01], (instregex "^LD2[BHWD]_IMM$")>; @@ -1746,25 +1746,25 @@ def : InstRW<[V1Write_12c_4L01_4V01], (instregex "^LD4[BHWD]_IMM$")>; def : InstRW<[V1Write_13c_4L01_2S_4V01], (instregex "^LD4[BHWD]$")>; // Gather load, vector + imm, 32-bit element size -def : InstRW<[V1Write_11c_1L_1V], (instregex "^GLD(FF)?1S?[BH]_S_IMM_REAL$", - "^GLD(FF)?1W_IMM_REAL$")>; +def : InstRW<[V1Write_11c_1L_1V], (instregex "^GLD(FF)?1S?[BH]_S_IMM$", + "^GLD(FF)?1W_IMM$")>; // Gather load, vector + imm, 64-bit element size def : InstRW<[V1Write_9c_2L_2V], - (instregex "^GLD(FF)?1S?[BHW]_D_IMM_REAL$", - "^GLD(FF)?1S?[BHW]_D_([SU]XTW_)?(SCALED_)?REAL$", - "^GLD(FF)?1D_IMM_REAL$", - "^GLD(FF)?1D_([SU]XTW_)?(SCALED_)?REAL$")>; + (instregex "^GLD(FF)?1S?[BHW]_D_IMM$", + "^GLD(FF)?1S?[BHW]_D(_[SU]XTW)?(_SCALED)?$", + "^GLD(FF)?1D_IMM$", + "^GLD(FF)?1D(_[SU]XTW)?(_SCALED)?$")>; // Gather load, 32-bit scaled offset def : InstRW<[V1Write_11c_2L_2V], - (instregex "^GLD(FF)?1S?[HW]_S_[SU]XTW_SCALED_REAL$", - "^GLD(FF)?1W_[SU]XTW_SCALED_REAL")>; + (instregex "^GLD(FF)?1S?[HW]_S_[SU]XTW_SCALED$", + "^GLD(FF)?1W_[SU]XTW_SCALED")>; // Gather load, 32-bit unpacked unscaled offset def : InstRW<[V1Write_9c_1L_1V], - (instregex "^GLD(FF)?1S?[BH]_S_[SU]XTW_REAL$", - "^GLD(FF)?1W_[SU]XTW_REAL$")>; + (instregex "^GLD(FF)?1S?[BH]_S_[SU]XTW$", + "^GLD(FF)?1W_[SU]XTW$")>; // Prefetch // NOTE: This is not specified in the SOG. @@ -1848,12 +1848,12 @@ def : InstRW<[V1Write_6c_1L01_1V], (instregex "^SST1[BHW]_D_IMM$", // Read first fault register, unpredicated // Set first fault register // Write to first fault register -def : InstRW<[V1Write_2c_1M0], (instrs RDFFR_P_REAL, +def : InstRW<[V1Write_2c_1M0], (instrs RDFFR_P, SETFFR, WRFFR)>; // Read first fault register, predicated -def : InstRW<[V1Write_3c_2M0], (instrs RDFFR_PPz_REAL)>; +def : InstRW<[V1Write_3c_2M0], (instrs RDFFR_PPz)>; // Read first fault register and set flags def : InstRW<[V1Write_4c_1M], (instrs RDFFRS_PPz)>; diff --git a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td index e7de40fdf1de..807ce40bc5ea 100644 --- a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td +++ b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td @@ -2635,24 +2635,24 @@ def : InstRW<[V2Write_6cyc_1L], (instregex "^LD1RQ_[BHWD]$")>; def : InstRW<[V2Write_6cyc_1L], (instregex "^LDNT1[BHWD]_ZR[IR]$")>; // Non temporal gather load, vector + scalar 32-bit element size -def : InstRW<[V2Write_9cyc_2L_4V], (instregex "^LDNT1[BHW]_ZZR_S_REAL$", - "^LDNT1S[BH]_ZZR_S_REAL$")>; +def : InstRW<[V2Write_9cyc_2L_4V], (instregex "^LDNT1[BHW]_ZZR_S$", + "^LDNT1S[BH]_ZZR_S$")>; // Non temporal gather load, vector + scalar 64-bit element size -def : InstRW<[V2Write_9cyc_2L_2V1], (instregex "^LDNT1S?[BHW]_ZZR_D_REAL$")>; -def : InstRW<[V2Write_9cyc_2L_2V1], (instrs LDNT1D_ZZR_D_REAL)>; +def : InstRW<[V2Write_9cyc_2L_2V1], (instregex "^LDNT1S?[BHW]_ZZR_D$")>; +def : InstRW<[V2Write_9cyc_2L_2V1], (instrs LDNT1D_ZZR_D)>; // Contiguous first faulting load, scalar + scalar -def : InstRW<[V2Write_6cyc_1L_1S], (instregex "^LDFF1[BHWD]_REAL$", - "^LDFF1S?B_[HSD]_REAL$", - "^LDFF1S?H_[SD]_REAL$", - "^LDFF1S?W_D_REAL$")>; +def : InstRW<[V2Write_6cyc_1L_1S], (instregex "^LDFF1[BHWD]$", + "^LDFF1S?B_[HSD]$", + "^LDFF1S?H_[SD]$", + "^LDFF1S?W_D$")>; // Contiguous non faulting load, scalar + imm -def : InstRW<[V2Write_6cyc_1L], (instregex "^LDNF1[BHWD]_IMM_REAL$", - "^LDNF1S?B_[HSD]_IMM_REAL$", - "^LDNF1S?H_[SD]_IMM_REAL$", - "^LDNF1S?W_D_IMM_REAL$")>; +def : InstRW<[V2Write_6cyc_1L], (instregex "^LDNF1[BHWD]_IMM$", + "^LDNF1S?B_[HSD]_IMM$", + "^LDNF1S?H_[SD]_IMM$", + "^LDNF1S?W_D_IMM$")>; // Contiguous Load two structures to two vectors, scalar + imm def : InstRW<[V2Write_8cyc_2L_2V], (instregex "^LD2[BHWD]_IMM$")>; @@ -2673,33 +2673,33 @@ def : InstRW<[V2Write_9cyc_4L_8V], (instregex "^LD4[BHWD]_IMM$")>; def : InstRW<[V2Write_10cyc_4L_8V_4S], (instregex "^LD4[BHWD]$")>; // Gather load, vector + imm, 32-bit element size -def : InstRW<[V2Write_9cyc_1L_4V], (instregex "^GLD(FF)?1S?[BH]_S_IMM_REAL$", - "^GLD(FF)?1W_IMM_REAL$")>; +def : InstRW<[V2Write_9cyc_1L_4V], (instregex "^GLD(FF)?1S?[BH]_S_IMM$", + "^GLD(FF)?1W_IMM$")>; // Gather load, vector + imm, 64-bit element size -def : InstRW<[V2Write_9cyc_1L_4V], (instregex "^GLD(FF)?1S?[BHW]_D_IMM_REAL$", - "^GLD(FF)?1D_IMM_REAL$")>; +def : InstRW<[V2Write_9cyc_1L_4V], (instregex "^GLD(FF)?1S?[BHW]_D_IMM$", + "^GLD(FF)?1D_IMM$")>; // Gather load, 32-bit scaled offset def : InstRW<[V2Write_10cyc_1L_8V], - (instregex "^GLD(FF)?1S?H_S_[SU]XTW_SCALED_REAL$", - "^GLD(FF)?1W_[SU]XTW_SCALED_REAL")>; + (instregex "^GLD(FF)?1S?H_S_[SU]XTW_SCALED$", + "^GLD(FF)?1W_[SU]XTW_SCALED")>; // Gather load, 64-bit scaled offset // NOTE: These instructions are not specified in the SOG. def : InstRW<[V2Write_10cyc_1L_4V], - (instregex "^GLD(FF)?1S?[HW]_D_([SU]XTW_)?SCALED_REAL$", - "^GLD(FF)?1D_([SU]XTW_)?SCALED_REAL$")>; + (instregex "^GLD(FF)?1S?[HW]_D_([SU]XTW_)?SCALED$", + "^GLD(FF)?1D_([SU]XTW_)?SCALED$")>; // Gather load, 32-bit unpacked unscaled offset -def : InstRW<[V2Write_9cyc_1L_4V], (instregex "^GLD(FF)?1S?[BH]_S_[SU]XTW_REAL$", - "^GLD(FF)?1W_[SU]XTW_REAL$")>; +def : InstRW<[V2Write_9cyc_1L_4V], (instregex "^GLD(FF)?1S?[BH]_S_[SU]XTW$", + "^GLD(FF)?1W_[SU]XTW$")>; // Gather load, 64-bit unpacked unscaled offset // NOTE: These instructions are not specified in the SOG. def : InstRW<[V2Write_9cyc_1L_2V], - (instregex "^GLD(FF)?1S?[BHW]_D_([SU]XTW_)?REAL$", - "^GLD(FF)?1D_([SU]XTW_)?REAL$")>; + (instregex "^GLD(FF)?1S?[BHW]_D(_[SU]XTW)?$", + "^GLD(FF)?1D(_[SU]XTW)?$")>; // SVE Store instructions // ----------------------------------------------------------------------------- @@ -2790,10 +2790,10 @@ def : InstRW<[V2Write_2cyc_2L01_2V01], (instregex "^SST1[BHW]_D$", // ----------------------------------------------------------------------------- // Read first fault register, unpredicated -def : InstRW<[V2Write_2cyc_1M0], (instrs RDFFR_P_REAL)>; +def : InstRW<[V2Write_2cyc_1M0], (instrs RDFFR_P)>; // Read first fault register, predicated -def : InstRW<[V2Write_3or4cyc_1M0_1M], (instrs RDFFR_PPz_REAL)>; +def : InstRW<[V2Write_3or4cyc_1M0_1M], (instrs RDFFR_PPz)>; // Read first fault register and set flags def : InstRW<[V2Write_4or5cyc_2M0_2M], (instrs RDFFRS_PPz)>; diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td index c8ca1832ec18..8baf3a6d3d81 100644 --- a/llvm/lib/Target/AArch64/SVEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -6421,33 +6421,33 @@ class sve2_mem_sstnt_vs_base opc, string asm, multiclass sve2_mem_sstnt_vs_32_ptrs opc, string asm, SDPatternOperator op, ValueType vt> { - def _REAL : sve2_mem_sstnt_vs_base; + def NAME : sve2_mem_sstnt_vs_base; def : InstAlias(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>; + (!cast(NAME) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>; def : InstAlias(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>; + (!cast(NAME) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>; def : InstAlias(NAME # _REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>; + (!cast(NAME) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>; def : Pat <(op (nxv4i32 ZPR32:$Zt), (nxv4i1 PPR3bAny:$Pg), (nxv4i32 ZPR32:$Zn), (i64 GPR64:$Rm), vt), - (!cast(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm)>; + (!cast(NAME) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm)>; } multiclass sve2_mem_sstnt_vs_64_ptrs opc, string asm, SDPatternOperator op, ValueType vt> { - def _REAL : sve2_mem_sstnt_vs_base; + def NAME : sve2_mem_sstnt_vs_base; def : InstAlias(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>; + (!cast(NAME) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>; def : InstAlias(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>; + (!cast(NAME) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>; def : InstAlias(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>; + (!cast(NAME) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>; def : Pat <(op (nxv2i64 ZPR64:$Zt), (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zn), (i64 GPR64:$Rm), vt), - (!cast(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm)>; + (!cast(NAME) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm)>; } class sve_mem_sst_sv opc, bit xs, bit scaled, string asm, @@ -6782,11 +6782,11 @@ multiclass sve_int_perm_punpk { def : SVE_1_Op_Pat(NAME)>; } -class sve_int_rdffr_pred +class sve_int_rdffr_pred : I<(outs PPR8:$Pd), (ins PPRAny:$Pg), asm, "\t$Pd, $Pg/z", "", - []>, Sched<[]> { + [(set (nxv16i1 PPR8:$Pd), (op (nxv16i1 PPRAny:$Pg)))]>, Sched<[]> { bits<4> Pd; bits<4> Pg; let Inst{31-23} = 0b001001010; @@ -6801,22 +6801,11 @@ class sve_int_rdffr_pred let hasSideEffects = 1; } -multiclass sve_int_rdffr_pred { - def _REAL : sve_int_rdffr_pred; - - // We need a layer of indirection because early machine code passes balk at - // physical register (i.e. FFR) uses that have no previous definition. - let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { - def "" : Pseudo<(outs PPR8:$Pd), (ins PPRAny:$Pg), [(set (nxv16i1 PPR8:$Pd), (op (nxv16i1 PPRAny:$Pg)))]>, - PseudoInstExpansion<(!cast(NAME # _REAL) PPR8:$Pd, PPRAny:$Pg)>; - } -} - -class sve_int_rdffr_unpred : I< +class sve_int_rdffr_unpred : I< (outs PPR8:$Pd), (ins), asm, "\t$Pd", "", - []>, Sched<[]> { + [(set (nxv16i1 PPR8:$Pd), (op))]>, Sched<[]> { bits<4> Pd; let Inst{31-4} = 0b0010010100011001111100000000; let Inst{3-0} = Pd; @@ -6825,17 +6814,6 @@ class sve_int_rdffr_unpred : I< let hasSideEffects = 1; } -multiclass sve_int_rdffr_unpred { - def _REAL : sve_int_rdffr_unpred; - - // We need a layer of indirection because early machine code passes balk at - // physical register (i.e. FFR) uses that have no previous definition. - let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { - def "" : Pseudo<(outs PPR8:$Pd), (ins), [(set (nxv16i1 PPR8:$Pd), (op))]>, - PseudoInstExpansion<(!cast(NAME # _REAL) PPR8:$Pd)>; - } -} - class sve_int_wrffr : I<(outs), (ins PPR8:$Pn), asm, "\t$Pn", @@ -7327,9 +7305,9 @@ class sve_mem_cld_si_base dtype, bit nf, string asm, let mayLoad = 1; } -multiclass sve_mem_cld_si dtype, string asm, RegisterOperand listty, - ZPRRegOp zprty> { - def "" : sve_mem_cld_si_base; +multiclass sve_mem_cld_si_base dtype, bit nf, string asm, + RegisterOperand listty, ZPRRegOp zprty> { + def NAME : sve_mem_cld_si_base; def : InstAlias(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; @@ -7339,6 +7317,14 @@ multiclass sve_mem_cld_si dtype, string asm, RegisterOperand listty, (!cast(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; } +multiclass sve_mem_cld_si dtype, string asm, RegisterOperand listty, + ZPRRegOp zprty> +: sve_mem_cld_si_base; + +multiclass sve_mem_cldnf_si dtype, string asm, RegisterOperand listty, + ZPRRegOp zprty> +: sve_mem_cld_si_base; + class sve_mem_cldnt_si_base msz, string asm, RegisterOperand VecList> : I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", @@ -7527,7 +7513,7 @@ class sve_mem_cld_ss_base dtype, bit ff, dag iops, string asm, multiclass sve_mem_cld_ss dtype, string asm, RegisterOperand listty, ZPRRegOp zprty, RegisterOperand gprty> { - def "" : sve_mem_cld_ss_base; def : InstAlias dtype, string asm, RegisterOperand listty, multiclass sve_mem_cldff_ss dtype, string asm, RegisterOperand listty, ZPRRegOp zprty, RegisterOperand gprty> { - def _REAL : sve_mem_cld_ss_base; + def NAME : sve_mem_cld_ss_base; def : InstAlias(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; - - def : InstAlias(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 1>; + (!cast(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; def : InstAlias(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 0>; - - // We need a layer of indirection because early machine code passes balk at - // physical register (i.e. FFR) uses that have no previous definition. - let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { - def "" : Pseudo<(outs listty:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), []>, - PseudoInstExpansion<(!cast(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm)>; - } -} - -multiclass sve_mem_cldnf_si dtype, string asm, RegisterOperand listty, - ZPRRegOp zprty> { - def _REAL : sve_mem_cld_si_base; + (!cast(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 1>; def : InstAlias(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; - def : InstAlias(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; - def : InstAlias(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; - - // We need a layer of indirection because early machine code passes balk at - // physical register (i.e. FFR) uses that have no previous definition. - let hasSideEffects = 1, hasNoSchedulingInfo = 1, mayLoad = 1 in { - def "" : Pseudo<(outs listty:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), []>, - PseudoInstExpansion<(!cast(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4)>; - } + (!cast(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 0>; } class sve_mem_eld_si sz, bits<3> nregs, RegisterOperand VecList, @@ -7673,22 +7632,13 @@ multiclass sve_mem_32b_gld_sv_32_scaled opc, string asm, RegisterOperand sxtw_opnd, RegisterOperand uxtw_opnd, ValueType vt> { - def _UXTW_SCALED_REAL : sve_mem_32b_gld_sv; - def _SXTW_SCALED_REAL : sve_mem_32b_gld_sv; + def _UXTW_SCALED : sve_mem_32b_gld_sv; + def _SXTW_SCALED : sve_mem_32b_gld_sv; def : InstAlias(NAME # _UXTW_SCALED_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; + (!cast(NAME # _UXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; def : InstAlias(NAME # _SXTW_SCALED_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; - - // We need a layer of indirection because early machine code passes balk at - // physical register (i.e. FFR) uses that have no previous definition. - let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { - def _UXTW_SCALED : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, - PseudoInstExpansion<(!cast(NAME # _UXTW_SCALED_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; - def _SXTW_SCALED : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, - PseudoInstExpansion<(!cast(NAME # _SXTW_SCALED_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; - } + (!cast(NAME # _SXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; def : Pat<(nxv4i32 (uxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$indices), vt)), (!cast(NAME # _UXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; @@ -7702,22 +7652,13 @@ multiclass sve_mem_32b_gld_vs_32_unscaled opc, string asm, RegisterOperand sxtw_opnd, RegisterOperand uxtw_opnd, ValueType vt> { - def _UXTW_REAL : sve_mem_32b_gld_sv; - def _SXTW_REAL : sve_mem_32b_gld_sv; + def _UXTW : sve_mem_32b_gld_sv; + def _SXTW : sve_mem_32b_gld_sv; def : InstAlias(NAME # _UXTW_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; + (!cast(NAME # _UXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; def : InstAlias(NAME # _SXTW_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; - - // We need a layer of indirection because early machine code passes balk at - // physical register (i.e. FFR) uses that have no previous definition. - let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { - def _UXTW : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, - PseudoInstExpansion<(!cast(NAME # _UXTW_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; - def _SXTW : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, - PseudoInstExpansion<(!cast(NAME # _SXTW_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; - } + (!cast(NAME # _SXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; def : Pat<(nxv4i32 (uxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt)), (!cast(NAME # _UXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; @@ -7754,21 +7695,14 @@ class sve_mem_32b_gld_vi opc, string asm, Operand imm_ty> multiclass sve_mem_32b_gld_vi_32_ptrs opc, string asm, Operand imm_ty, SDPatternOperator op, ValueType vt> { - def _IMM_REAL : sve_mem_32b_gld_vi; + def _IMM : sve_mem_32b_gld_vi; def : InstAlias(NAME # _IMM_REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 0>; + (!cast(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 0>; def : InstAlias(NAME # _IMM_REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>; + (!cast(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>; def : InstAlias(NAME # _IMM_REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; - - // We need a layer of indirection because early machine code passes balk at - // physical register (i.e. FFR) uses that have no previous definition. - let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { - def _IMM : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), []>, - PseudoInstExpansion<(!cast(NAME # _IMM_REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5)>; - } + (!cast(NAME # _IMM) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; def : Pat<(nxv4i32 (op (nxv4i1 PPR:$gp), (nxv4i32 ZPR:$ptrs), imm_ty:$index, vt)), (!cast(NAME # _IMM) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; @@ -7979,35 +7913,33 @@ class sve2_mem_gldnt_vs_base opc, dag iops, string asm, multiclass sve2_mem_gldnt_vs_32_ptrs opc, string asm, SDPatternOperator op, ValueType vt> { - def _REAL : sve2_mem_gldnt_vs_base; + def NAME : sve2_mem_gldnt_vs_base; def : InstAlias(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>; + (!cast(NAME) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>; def : InstAlias(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>; + (!cast(NAME) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>; def : InstAlias(NAME # _REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>; + (!cast(NAME) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>; def : Pat <(nxv4i32 (op (nxv4i1 PPR3bAny:$Pg), (nxv4i32 ZPR32:$Zd), (i64 GPR64:$Rm), vt)), - (!cast(NAME # _REAL) PPR3bAny:$Pg, ZPR32:$Zd, GPR64:$Rm)>; + (!cast(NAME) PPR3bAny:$Pg, ZPR32:$Zd, GPR64:$Rm)>; } multiclass sve2_mem_gldnt_vs_64_ptrs opc, string asm, SDPatternOperator op, ValueType vt> { - def _REAL : sve2_mem_gldnt_vs_base; + def NAME : sve2_mem_gldnt_vs_base; def : InstAlias(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>; + (!cast(NAME) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>; def : InstAlias(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>; + (!cast(NAME) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>; def : InstAlias(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>; + (!cast(NAME) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>; def : Pat <(nxv2i64 (op (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zd), (i64 GPR64:$Rm), vt)), - (!cast(NAME # _REAL) PPR3bAny:$Pg, ZPR64:$Zd, GPR64:$Rm)>; + (!cast(NAME) PPR3bAny:$Pg, ZPR64:$Zd, GPR64:$Rm)>; } //===----------------------------------------------------------------------===// @@ -8051,22 +7983,13 @@ multiclass sve_mem_64b_gld_sv_32_scaled opc, string asm, RegisterOperand sxtw_opnd, RegisterOperand uxtw_opnd, ValueType vt> { - def _UXTW_SCALED_REAL : sve_mem_64b_gld_sv; - def _SXTW_SCALED_REAL : sve_mem_64b_gld_sv; + def _UXTW_SCALED : sve_mem_64b_gld_sv; + def _SXTW_SCALED : sve_mem_64b_gld_sv; def : InstAlias(NAME # _UXTW_SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; + (!cast(NAME # _UXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; def : InstAlias(NAME # _SXTW_SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; - - // We need a layer of indirection because early machine code passes balk at - // physical register (i.e. FFR) uses that have no previous definition. - let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { - def _UXTW_SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, - PseudoInstExpansion<(!cast(NAME # _UXTW_SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; - def _SXTW_SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, - PseudoInstExpansion<(!cast(NAME # _SXTW_SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; - } + (!cast(NAME # _SXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; def : Pat<(nxv2i64 (uxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), (!cast(NAME # _UXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; @@ -8080,22 +8003,13 @@ multiclass sve_mem_64b_gld_vs_32_unscaled opc, string asm, RegisterOperand sxtw_opnd, RegisterOperand uxtw_opnd, ValueType vt> { - def _UXTW_REAL : sve_mem_64b_gld_sv; - def _SXTW_REAL : sve_mem_64b_gld_sv; + def _UXTW : sve_mem_64b_gld_sv; + def _SXTW : sve_mem_64b_gld_sv; def : InstAlias(NAME # _UXTW_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; + (!cast(NAME # _UXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; def : InstAlias(NAME # _SXTW_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; - - // We need a layer of indirection because early machine code passes balk at - // physical register (i.e. FFR) uses that have no previous definition. - let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { - def _UXTW : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, - PseudoInstExpansion<(!cast(NAME # _UXTW_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; - def _SXTW : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, - PseudoInstExpansion<(!cast(NAME # _SXTW_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; - } + (!cast(NAME # _SXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; def : Pat<(nxv2i64 (uxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), (!cast(NAME # _UXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; @@ -8106,17 +8020,10 @@ multiclass sve_mem_64b_gld_vs_32_unscaled opc, string asm, multiclass sve_mem_64b_gld_sv2_64_scaled opc, string asm, SDPatternOperator op, RegisterOperand zprext, ValueType vt> { - def _SCALED_REAL : sve_mem_64b_gld_sv; + def _SCALED : sve_mem_64b_gld_sv; def : InstAlias(NAME # _SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 0>; - - // We need a layer of indirection because early machine code passes balk at - // physical register (i.e. FFR) uses that have no previous definition. - let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { - def _SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), []>, - PseudoInstExpansion<(!cast(NAME # _SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm)>; - } + (!cast(NAME # _SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 0>; def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), (!cast(NAME # _SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; @@ -8124,17 +8031,10 @@ multiclass sve_mem_64b_gld_sv2_64_scaled opc, string asm, multiclass sve_mem_64b_gld_vs2_64_unscaled opc, string asm, SDPatternOperator op, ValueType vt> { - def _REAL : sve_mem_64b_gld_sv; + def NAME : sve_mem_64b_gld_sv; def : InstAlias(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), 0>; - - // We need a layer of indirection because early machine code passes balk at - // physical register (i.e. FFR) uses that have no previous definition. - let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { - def "" : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), []>, - PseudoInstExpansion<(!cast(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm)>; - } + (!cast(NAME) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), 0>; def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), (!cast(NAME) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; @@ -8167,21 +8067,14 @@ class sve_mem_64b_gld_vi opc, string asm, Operand imm_ty> multiclass sve_mem_64b_gld_vi_64_ptrs opc, string asm, Operand imm_ty, SDPatternOperator op, ValueType vt> { - def _IMM_REAL : sve_mem_64b_gld_vi; + def _IMM : sve_mem_64b_gld_vi; def : InstAlias(NAME # _IMM_REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 0>; + (!cast(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 0>; def : InstAlias(NAME # _IMM_REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>; + (!cast(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>; def : InstAlias(NAME # _IMM_REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; - - // We need a layer of indirection because early machine code passes balk at - // physical register (i.e. FFR) uses that have no previous definition. - let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { - def _IMM : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), []>, - PseudoInstExpansion<(!cast(NAME # _IMM_REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5)>; - } + (!cast(NAME # _IMM) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), (nxv2i64 ZPR:$ptrs), imm_ty:$index, vt)), (!cast(NAME # _IMM) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-mask-opt.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-mask-opt.ll index 29ad550c40d9..e23151475014 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-mask-opt.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-mask-opt.ll @@ -49,8 +49,8 @@ define void @masked_gather_v8i8(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: ld1b { z0.d }, p0/z, [z0.d] ; VBITS_GE_256-NEXT: ld1b { z1.d }, p0/z, [z1.d] ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s -; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h +; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: uzp1 v0.8b, v1.8b, v0.8b ; VBITS_GE_256-NEXT: str d0, [x0] @@ -153,8 +153,8 @@ define void @masked_gather_v8i16(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: ld1h { z0.d }, p0/z, [z0.d] ; VBITS_GE_256-NEXT: ld1h { z1.d }, p0/z, [z1.d] ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s -; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h +; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: mov v1.d[1], v0.d[0] ; VBITS_GE_256-NEXT: str q1, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll index 4ac0abcb851d..92fce4584f6a 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll @@ -66,7 +66,6 @@ define void @masked_gather_v8i8(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: zip2 v1.8b, v0.8b, v0.8b ; VBITS_GE_256-NEXT: zip1 v0.8b, v0.8b, v0.8b ; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] -; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1] ; VBITS_GE_256-NEXT: shl v1.4h, v1.4h, #8 ; VBITS_GE_256-NEXT: shl v0.4h, v0.4h, #8 ; VBITS_GE_256-NEXT: sshr v1.4h, v1.4h, #8 @@ -76,14 +75,15 @@ define void @masked_gather_v8i8(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s ; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s ; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z1.d, #0 +; VBITS_GE_256-NEXT: ld1b { z1.d }, p1/z, [z2.d] +; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1] ; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; VBITS_GE_256-NEXT: ld1b { z0.d }, p1/z, [z2.d] -; VBITS_GE_256-NEXT: ld1b { z1.d }, p0/z, [z3.d] -; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s +; VBITS_GE_256-NEXT: ld1b { z0.d }, p0/z, [z2.d] ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s -; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h -; VBITS_GE_256-NEXT: uzp1 v0.8b, v1.8b, v0.8b +; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s +; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h +; VBITS_GE_256-NEXT: uzp1 v0.8b, v0.8b, v1.8b ; VBITS_GE_256-NEXT: str d0, [x0] ; VBITS_GE_256-NEXT: ret ; @@ -217,23 +217,23 @@ define void @masked_gather_v8i16(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 ; VBITS_GE_256-NEXT: cmeq v0.8h, v0.8h, #0 -; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8 +; VBITS_GE_256-NEXT: sunpklo z1.s, z0.h +; VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v0.16b, #8 +; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1] ; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] -; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1] -; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h -; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s ; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s -; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z0.d, #0 -; VBITS_GE_256-NEXT: ld1h { z0.d }, p1/z, [z3.d] -; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s -; VBITS_GE_256-NEXT: ld1h { z1.d }, p0/z, [z2.d] +; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s +; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z1.d, #0 +; VBITS_GE_256-NEXT: ld1h { z1.d }, p1/z, [z2.d] +; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; VBITS_GE_256-NEXT: ld1h { z0.d }, p0/z, [z2.d] ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s -; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h -; VBITS_GE_256-NEXT: mov v0.d[1], v1.d[0] -; VBITS_GE_256-NEXT: str q0, [x0] +; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s +; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h +; VBITS_GE_256-NEXT: mov v1.d[1], v0.d[0] +; VBITS_GE_256-NEXT: str q1, [x0] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: masked_gather_v8i16: @@ -353,22 +353,22 @@ define void @masked_gather_v8i32(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 ; VBITS_GE_256-NEXT: ptrue p2.d, vl4 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0] -; VBITS_GE_256-NEXT: ld1d { z1.d }, p2/z, [x1, x8, lsl #3] -; VBITS_GE_256-NEXT: ld1d { z2.d }, p2/z, [x1] +; VBITS_GE_256-NEXT: ld1d { z1.d }, p2/z, [x1] +; VBITS_GE_256-NEXT: ld1d { z2.d }, p2/z, [x1, x8, lsl #3] ; VBITS_GE_256-NEXT: cmpeq p1.s, p0/z, z0.s, #0 +; VBITS_GE_256-NEXT: punpklo p3.h, p1.b ; VBITS_GE_256-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff -; VBITS_GE_256-NEXT: punpklo p1.h, p1.b ; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16 +; VBITS_GE_256-NEXT: and p1.b, p3/z, p3.b, p2.b ; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s -; VBITS_GE_256-NEXT: and p1.b, p1/z, p1.b, p2.b -; VBITS_GE_256-NEXT: cmpne p2.d, p2/z, z0.d, #0 +; VBITS_GE_256-NEXT: ld1w { z1.d }, p1/z, [z1.d] +; VBITS_GE_256-NEXT: cmpne p1.d, p2/z, z0.d, #0 ; VBITS_GE_256-NEXT: ld1w { z0.d }, p1/z, [z2.d] ; VBITS_GE_256-NEXT: ptrue p1.s, vl4 -; VBITS_GE_256-NEXT: ld1w { z1.d }, p2/z, [z1.d] -; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s -; VBITS_GE_256-NEXT: splice z0.s, p1, z0.s, z1.s -; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0] +; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s +; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s +; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: masked_gather_v8i32: @@ -500,15 +500,15 @@ define void @masked_gather_v8i64(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 ; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0] -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1] ; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z0.d, #0 -; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1] -; VBITS_GE_256-NEXT: cmpeq p2.d, p0/z, z1.d, #0 +; VBITS_GE_256-NEXT: ld1d { z0.d }, p1/z, [z1.d] +; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0] +; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z1.d, #0 ; VBITS_GE_256-NEXT: ld1d { z1.d }, p1/z, [z2.d] -; VBITS_GE_256-NEXT: ld1d { z0.d }, p2/z, [z0.d] -; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0, x8, lsl #3] -; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0] +; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3] +; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: masked_gather_v8i64: @@ -626,23 +626,23 @@ define void @masked_gather_v8f16(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 ; VBITS_GE_256-NEXT: fcmeq v0.8h, v0.8h, #0.0 -; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8 +; VBITS_GE_256-NEXT: sunpklo z2.s, z0.h +; VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v0.16b, #8 +; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] -; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1] -; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h +; VBITS_GE_256-NEXT: sunpklo z2.d, z2.s ; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s -; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s -; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z0.d, #0 -; VBITS_GE_256-NEXT: ld1h { z0.d }, p1/z, [z3.d] -; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s -; VBITS_GE_256-NEXT: ld1h { z1.d }, p0/z, [z2.d] +; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z2.d, #0 +; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: ld1h { z1.d }, p1/z, [z1.d] +; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; VBITS_GE_256-NEXT: ld1h { z0.d }, p0/z, [z2.d] ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s -; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h -; VBITS_GE_256-NEXT: mov v0.d[1], v1.d[0] -; VBITS_GE_256-NEXT: str q0, [x0] +; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s +; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h +; VBITS_GE_256-NEXT: mov v1.d[1], v0.d[0] +; VBITS_GE_256-NEXT: str q1, [x0] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: masked_gather_v8f16: @@ -762,22 +762,22 @@ define void @masked_gather_v8f32(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 ; VBITS_GE_256-NEXT: ptrue p2.d, vl4 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0] -; VBITS_GE_256-NEXT: ld1d { z1.d }, p2/z, [x1, x8, lsl #3] -; VBITS_GE_256-NEXT: ld1d { z2.d }, p2/z, [x1] +; VBITS_GE_256-NEXT: ld1d { z1.d }, p2/z, [x1] +; VBITS_GE_256-NEXT: ld1d { z2.d }, p2/z, [x1, x8, lsl #3] ; VBITS_GE_256-NEXT: fcmeq p1.s, p0/z, z0.s, #0.0 +; VBITS_GE_256-NEXT: punpklo p3.h, p1.b ; VBITS_GE_256-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff -; VBITS_GE_256-NEXT: punpklo p1.h, p1.b ; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16 +; VBITS_GE_256-NEXT: and p1.b, p3/z, p3.b, p2.b ; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s -; VBITS_GE_256-NEXT: and p1.b, p1/z, p1.b, p2.b -; VBITS_GE_256-NEXT: cmpne p2.d, p2/z, z0.d, #0 +; VBITS_GE_256-NEXT: ld1w { z1.d }, p1/z, [z1.d] +; VBITS_GE_256-NEXT: cmpne p1.d, p2/z, z0.d, #0 ; VBITS_GE_256-NEXT: ld1w { z0.d }, p1/z, [z2.d] ; VBITS_GE_256-NEXT: ptrue p1.s, vl4 -; VBITS_GE_256-NEXT: ld1w { z1.d }, p2/z, [z1.d] -; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s -; VBITS_GE_256-NEXT: splice z0.s, p1, z0.s, z1.s -; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0] +; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s +; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s +; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: masked_gather_v8f32: @@ -911,11 +911,11 @@ define void @masked_gather_v8f64(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] ; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0] ; VBITS_GE_256-NEXT: fcmeq p1.d, p0/z, z0.d, #0.0 -; VBITS_GE_256-NEXT: fcmeq p2.d, p0/z, z1.d, #0.0 ; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3] -; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_256-NEXT: ld1d { z0.d }, p1/z, [z0.d] -; VBITS_GE_256-NEXT: ld1d { z1.d }, p2/z, [z1.d] +; VBITS_GE_256-NEXT: fcmeq p1.d, p0/z, z1.d, #0.0 +; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1] +; VBITS_GE_256-NEXT: ld1d { z1.d }, p1/z, [z1.d] ; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3] ; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0] ; VBITS_GE_256-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-fp-int-min-max.ll b/llvm/test/CodeGen/AArch64/sve-fp-int-min-max.ll index bdaea0ecf144..5ff9f0f0df62 100644 --- a/llvm/test/CodeGen/AArch64/sve-fp-int-min-max.ll +++ b/llvm/test/CodeGen/AArch64/sve-fp-int-min-max.ll @@ -19,8 +19,8 @@ define i64 @scalable_int_min_max(ptr %arg, ptr %arg1, %i37, < ; CHECK-NEXT: smax z4.d, z4.d, #0 ; CHECK-NEXT: smin z4.d, p0/m, z4.d, z5.d ; CHECK-NEXT: cmpne p1.d, p0/z, z4.d, #0 -; CHECK-NEXT: ld1w { z4.d }, p1/z, [x1] ; CHECK-NEXT: ld1w { z0.d }, p1/z, [z0.d] +; CHECK-NEXT: ld1w { z4.d }, p1/z, [x1] ; CHECK-NEXT: fadd z0.s, p0/m, z0.s, z4.s ; CHECK-NEXT: fcmge p2.s, p0/z, z0.s, z3.s ; CHECK-NEXT: add z0.d, z2.d, z1.d diff --git a/llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll b/llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll index a40d55085279..4d2bc4bde13f 100644 --- a/llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll +++ b/llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll @@ -31,9 +31,9 @@ define @no_dag_combine_sext( %pg, ; CHECK-LABEL: no_dag_combine_sext: ; CHECK: // %bb.0: ; CHECK-NEXT: ld1b { z1.d }, p0/z, [z0.d, #16] -; CHECK-NEXT: ptrue p2.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: movprfx z0, z1 -; CHECK-NEXT: sxtb z0.d, p2/m, z1.d +; CHECK-NEXT: sxtb z0.d, p0/m, z1.d ; CHECK-NEXT: st1b { z1.d }, p1, [x0] ; CHECK-NEXT: ret %base, @@ -79,15 +79,15 @@ define @narrow_i64_gather_index_i8_zext(ptr %out, ptr %in, @narrow_i64_gather_index_i8_sext(ptr %out, ptr %in, @masked_gather_nxv8f16( %ptrs, Date: Tue, 5 Mar 2024 04:38:42 -0800 Subject: [PATCH 134/929] [clang] Use separator for large numeric values in overflow diagnostic (#80939) Add functionality to APInt::toString() that allows it to insert separators between groups of digits, using the C++ literal separator ' between groups. Fixes issue #58228 Reviewers: @AaronBallman, @cjdb, @tbaederr --- clang/docs/ReleaseNotes.rst | 3 + clang/lib/AST/ExprConstant.cpp | 8 +- clang/lib/AST/Interp/Interp.h | 12 ++- clang/test/AST/Interp/c.c | 2 +- clang/test/C/drs/dr0xx.c | 2 +- clang/test/C/drs/dr2xx.c | 2 +- clang/test/Sema/integer-overflow.c | 100 ++++++++--------- clang/test/Sema/switch-1.c | 6 +- clang/test/SemaCXX/enum.cpp | 4 +- clang/test/SemaCXX/integer-overflow.cpp | 112 ++++++++++---------- clang/test/SemaObjC/integer-overflow.m | 4 +- clang/test/SemaObjC/objc-literal-nsnumber.m | 2 +- llvm/include/llvm/ADT/APInt.h | 3 +- llvm/include/llvm/ADT/StringExtras.h | 6 +- llvm/lib/Support/APInt.cpp | 20 +++- llvm/unittests/ADT/APIntTest.cpp | 35 ++++++ 16 files changed, 195 insertions(+), 126 deletions(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index d4e6bcf661da..c328ae18c024 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -177,6 +177,9 @@ Improvements to Clang's diagnostics - The ``-Wshorten-64-to-32`` diagnostic is now grouped under ``-Wimplicit-int-conversion`` instead of ``-Wconversion``. Fixes #GH69444. +- Clang now uses thousand separators when printing large numbers in integer overflow diagnostics. + Fixes #GH80939. + - Clang now diagnoses friend declarations with an ``enum`` elaborated-type-specifier in language modes after C++98. - Added diagnostics for C11 keywords being incompatible with language standards diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp index fcf8f6591a79..4b6ccafbb7f0 100644 --- a/clang/lib/AST/ExprConstant.cpp +++ b/clang/lib/AST/ExprConstant.cpp @@ -2778,7 +2778,9 @@ static bool CheckedIntArithmetic(EvalInfo &Info, const Expr *E, if (Info.checkingForUndefinedBehavior()) Info.Ctx.getDiagnostics().Report(E->getExprLoc(), diag::warn_integer_constant_overflow) - << toString(Result, 10) << E->getType() << E->getSourceRange(); + << toString(Result, 10, Result.isSigned(), /*formatAsCLiteral=*/false, + /*UpperCase=*/true, /*InsertSeparators=*/true) + << E->getType() << E->getSourceRange(); return HandleOverflow(Info, E, Value, E->getType()); } return true; @@ -13910,7 +13912,9 @@ bool IntExprEvaluator::VisitUnaryOperator(const UnaryOperator *E) { if (Info.checkingForUndefinedBehavior()) Info.Ctx.getDiagnostics().Report(E->getExprLoc(), diag::warn_integer_constant_overflow) - << toString(Value, 10) << E->getType() << E->getSourceRange(); + << toString(Value, 10, Value.isSigned(), /*formatAsCLiteral=*/false, + /*UpperCase=*/true, /*InsertSeparators=*/true) + << E->getType() << E->getSourceRange(); if (!HandleOverflow(Info, E, -Value.extend(Value.getBitWidth() + 1), E->getType())) diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h index 548405a53dfa..baa1b8e9071a 100644 --- a/clang/lib/AST/Interp/Interp.h +++ b/clang/lib/AST/Interp/Interp.h @@ -287,7 +287,9 @@ bool AddSubMulHelper(InterpState &S, CodePtr OpPC, unsigned Bits, const T &LHS, QualType Type = E->getType(); if (S.checkingForUndefinedBehavior()) { SmallString<32> Trunc; - Value.trunc(Result.bitWidth()).toString(Trunc, 10); + Value.trunc(Result.bitWidth()) + .toString(Trunc, 10, Result.isSigned(), /*formatAsCLiteral=*/false, + /*UpperCase=*/true, /*InsertSeparators=*/true); auto Loc = E->getExprLoc(); S.report(Loc, diag::warn_integer_constant_overflow) << Trunc << Type << E->getSourceRange(); @@ -499,7 +501,9 @@ bool Neg(InterpState &S, CodePtr OpPC) { if (S.checkingForUndefinedBehavior()) { SmallString<32> Trunc; - NegatedValue.trunc(Result.bitWidth()).toString(Trunc, 10); + NegatedValue.trunc(Result.bitWidth()) + .toString(Trunc, 10, Result.isSigned(), /*formatAsCLiteral=*/false, + /*UpperCase=*/true, /*InsertSeparators=*/true); auto Loc = E->getExprLoc(); S.report(Loc, diag::warn_integer_constant_overflow) << Trunc << Type << E->getSourceRange(); @@ -561,7 +565,9 @@ bool IncDecHelper(InterpState &S, CodePtr OpPC, const Pointer &Ptr) { QualType Type = E->getType(); if (S.checkingForUndefinedBehavior()) { SmallString<32> Trunc; - APResult.trunc(Result.bitWidth()).toString(Trunc, 10); + APResult.trunc(Result.bitWidth()) + .toString(Trunc, 10, Result.isSigned(), /*formatAsCLiteral=*/false, + /*UpperCase=*/true, /*InsertSeparators=*/true); auto Loc = E->getExprLoc(); S.report(Loc, diag::warn_integer_constant_overflow) << Trunc << Type << E->getSourceRange(); diff --git a/clang/test/AST/Interp/c.c b/clang/test/AST/Interp/c.c index 260e5bdfeefb..a5270172cfc7 100644 --- a/clang/test/AST/Interp/c.c +++ b/clang/test/AST/Interp/c.c @@ -95,7 +95,7 @@ int chooseexpr[__builtin_choose_expr(1, 1, expr)]; int somefunc(int i) { return (i, 65537) * 65537; // all-warning {{left operand of comma operator has no effect}} \ - // all-warning {{overflow in expression; result is 131073}} + // all-warning {{overflow in expression; result is 131'073 with type 'int'}} } /// FIXME: The following test is incorrect in the new interpreter. diff --git a/clang/test/C/drs/dr0xx.c b/clang/test/C/drs/dr0xx.c index d9c1fbe4ee40..c93cfb63d604 100644 --- a/clang/test/C/drs/dr0xx.c +++ b/clang/test/C/drs/dr0xx.c @@ -214,7 +214,7 @@ _Static_assert(__builtin_types_compatible_p(struct S { int a; }, union U { int a */ void dr031(int i) { switch (i) { - case __INT_MAX__ + 1: break; /* expected-warning {{overflow in expression; result is -2147483648 with type 'int'}} */ + case __INT_MAX__ + 1: break; /* expected-warning {{overflow in expression; result is -2'147'483'648 with type 'int'}} */ #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wswitch" /* Silence the targets which issue: diff --git a/clang/test/C/drs/dr2xx.c b/clang/test/C/drs/dr2xx.c index 9c8d77518ab5..1b68b65acca6 100644 --- a/clang/test/C/drs/dr2xx.c +++ b/clang/test/C/drs/dr2xx.c @@ -277,7 +277,7 @@ void dr258(void) { void dr261(void) { /* This is still an integer constant expression despite the overflow. */ enum e1 { - ex1 = __INT_MAX__ + 1 /* expected-warning {{overflow in expression; result is -2147483648 with type 'int'}} */ + ex1 = __INT_MAX__ + 1 /* expected-warning {{overflow in expression; result is -2'147'483'648 with type 'int'}} */ }; /* This is not an integer constant expression, because of the comma operator, diff --git a/clang/test/Sema/integer-overflow.c b/clang/test/Sema/integer-overflow.c index cf822f346e8b..220fc1bed515 100644 --- a/clang/test/Sema/integer-overflow.c +++ b/clang/test/Sema/integer-overflow.c @@ -11,169 +11,169 @@ uint64_t f0(uint64_t); uint64_t f1(uint64_t, uint32_t); uint64_t f2(uint64_t, ...); -static const uint64_t overflow = 1 * 4608 * 1024 * 1024; // expected-warning {{overflow in expression; result is 536870912 with type 'int'}} +static const uint64_t overflow = 1 * 4608 * 1024 * 1024; // expected-warning {{overflow in expression; result is 536'870'912 with type 'int'}} uint64_t check_integer_overflows(int i) { -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} uint64_t overflow = 4608 * 1024 * 1024, -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} overflow2 = (uint64_t)(4608 * 1024 * 1024), -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} overflow3 = (uint64_t)(4608 * 1024 * 1024 * i), -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} overflow4 = (1ULL * ((4608) * ((1024) * (1024))) + 2ULL), -// expected-warning@+1 2{{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 2{{overflow in expression; result is 536'870'912 with type 'int'}} multi_overflow = (uint64_t)((uint64_t)(4608 * 1024 * 1024) * (uint64_t)(4608 * 1024 * 1024)); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} overflow += overflow2 = overflow3 = (uint64_t)(4608 * 1024 * 1024); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} overflow += overflow2 = overflow3 = 4608 * 1024 * 1024; uint64_t not_overflow = 4608 * 1024 * 1024ULL; uint64_t not_overflow2 = (1ULL * ((uint64_t)(4608) * (1024 * 1024)) + 2ULL); -// expected-warning@+1 2{{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 2{{overflow in expression; result is 536'870'912 with type 'int'}} overflow = 4608 * 1024 * 1024 ? 4608 * 1024 * 1024 : 0; -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} overflow = 0 ? 0 : 4608 * 1024 * 1024; -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} if (4608 * 1024 * 1024) return 0; -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} if ((uint64_t)(4608 * 1024 * 1024)) return 1; -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} if ((uint64_t)(4608 * 1024 * 1024)) return 2; -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} if ((uint64_t)(4608 * 1024 * 1024 * i)) return 3; -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} if ((1ULL * ((4608) * ((1024) * (1024))) + 2ULL)) return 4; -// expected-warning@+1 2{{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 2{{overflow in expression; result is 536'870'912 with type 'int'}} if ((uint64_t)((uint64_t)(4608 * 1024 * 1024) * (uint64_t)(4608 * 1024 * 1024))) return 5; switch (i) { -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} case 4608 * 1024 * 1024: return 6; -// expected-warning@+1 {{overflow in expression; result is 537919488 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 537'919'488 with type 'int'}} case (uint64_t)(4609 * 1024 * 1024): return 7; // expected-error@+1 {{expression is not an integer constant expression}} case ((uint64_t)(4608 * 1024 * 1024 * i)): return 8; -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} case ((1ULL * ((4608) * ((1024) * (1024))) + 2ULL)): return 9; -// expected-warning@+2 2{{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+2 2{{overflow in expression; result is 536'870'912 with type 'int'}} // expected-warning@+1 {{overflow converting case value to switch condition type (288230376151711744 to 0)}} case ((uint64_t)((uint64_t)(4608 * 1024 * 1024) * (uint64_t)(4608 * 1024 * 1024))): return 10; } -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} while (4608 * 1024 * 1024); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} while ((uint64_t)(4608 * 1024 * 1024)); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} while ((uint64_t)(4608 * 1024 * 1024)); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} while ((uint64_t)(4608 * 1024 * 1024 * i)); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} while ((1ULL * ((4608) * ((1024) * (1024))) + 2ULL)); -// expected-warning@+1 2{{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 2{{overflow in expression; result is 536'870'912 with type 'int'}} while ((uint64_t)((uint64_t)(4608 * 1024 * 1024) * (uint64_t)(4608 * 1024 * 1024))); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} do { } while (4608 * 1024 * 1024); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} do { } while ((uint64_t)(4608 * 1024 * 1024)); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} do { } while ((uint64_t)(4608 * 1024 * 1024)); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} do { } while ((uint64_t)(4608 * 1024 * 1024 * i)); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} do { } while ((1ULL * ((4608) * ((1024) * (1024))) + 2ULL)); -// expected-warning@+1 2{{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 2{{overflow in expression; result is 536'870'912 with type 'int'}} do { } while ((uint64_t)((uint64_t)(4608 * 1024 * 1024) * (uint64_t)(4608 * 1024 * 1024))); -// expected-warning@+3 {{overflow in expression; result is 536870912 with type 'int'}} -// expected-warning@+3 {{overflow in expression; result is 536870912 with type 'int'}} -// expected-warning@+3 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+3 {{overflow in expression; result is 536'870'912 with type 'int'}} +// expected-warning@+3 {{overflow in expression; result is 536'870'912 with type 'int'}} +// expected-warning@+3 {{overflow in expression; result is 536'870'912 with type 'int'}} for (uint64_t i = 4608 * 1024 * 1024; (uint64_t)(4608 * 1024 * 1024); i += (uint64_t)(4608 * 1024 * 1024 * i)); -// expected-warning@+3 {{overflow in expression; result is 536870912 with type 'int'}} -// expected-warning@+3 2{{overflow in expression; result is 536870912 with type 'int'}} -// expected-warning@+3 2{{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+3 {{overflow in expression; result is 536'870'912 with type 'int'}} +// expected-warning@+3 2{{overflow in expression; result is 536'870'912 with type 'int'}} +// expected-warning@+3 2{{overflow in expression; result is 536'870'912 with type 'int'}} for (uint64_t i = (1ULL * ((4608) * ((1024) * (1024))) + 2ULL); ((uint64_t)((uint64_t)(4608 * 1024 * 1024) * (uint64_t)(4608 * 1024 * 1024))); i = ((4608 * 1024 * 1024) + ((uint64_t)(4608 * 1024 * 1024)))); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} _Complex long long x = 4608 * 1024 * 1024; -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} (__real__ x) = 4608 * 1024 * 1024; -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} (__imag__ x) = 4608 * 1024 * 1024; -// expected-warning@+4 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+4 {{overflow in expression; result is 536'870'912 with type 'int'}} // expected-warning@+3 {{array index 536870912 is past the end of the array (that has type 'uint64_t[10]' (aka 'unsigned long long[10]'))}} // expected-note@+1 {{array 'a' declared here}} uint64_t a[10]; a[4608 * 1024 * 1024] = 1i; -// expected-warning@+2 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+2 {{overflow in expression; result is 536'870'912 with type 'int'}} uint64_t *b; uint64_t b2 = b[4608 * 1024 * 1024] + 1; -// expected-warning@+1 2{{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 2{{overflow in expression; result is 536'870'912 with type 'int'}} (void)((i ? (4608 * 1024 * 1024) : (4608 * 1024 * 1024)) + 1); -// expected-warning@+1 2{{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 2{{overflow in expression; result is 536'870'912 with type 'int'}} return ((4608 * 1024 * 1024) + ((uint64_t)(4608 * 1024 * 1024))); } void check_integer_overflows_in_function_calls(void) { -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} (void)f0(4608 * 1024 * 1024); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} uint64_t x = f0(4608 * 1024 * 1024); -// expected-warning@+2 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+2 {{overflow in expression; result is 536'870'912 with type 'int'}} uint64_t (*f0_ptr)(uint64_t) = &f0; (void)(*f0_ptr)(4608 * 1024 * 1024); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} (void)f2(0, f0(4608 * 1024 * 1024)); } void check_integer_overflows_in_array_size(void) { - int arr[4608 * 1024 * 1024]; // expected-warning {{overflow in expression; result is 536870912 with type 'int'}} + int arr[4608 * 1024 * 1024]; // expected-warning {{overflow in expression; result is 536'870'912 with type 'int'}} } struct s { diff --git a/clang/test/Sema/switch-1.c b/clang/test/Sema/switch-1.c index 95e64748fb1f..09221990ac11 100644 --- a/clang/test/Sema/switch-1.c +++ b/clang/test/Sema/switch-1.c @@ -7,7 +7,7 @@ int f(int i) { switch (i) { case 2147483647 + 2: #if (__cplusplus <= 199711L) // C or C++03 or earlier modes - // expected-warning@-2 {{overflow in expression; result is -2147483647 with type 'int'}} + // expected-warning@-2 {{overflow in expression; result is -2'147'483'647 with type 'int'}} #else // expected-error@-4 {{case value is not a constant expression}} \ // expected-note@-4 {{value 2147483649 is outside the range of representable values of type 'int'}} @@ -23,7 +23,7 @@ int f(int i) { return 2; case (123456 *789012) + 1: #if (__cplusplus <= 199711L) - // expected-warning@-2 {{overflow in expression; result is -1375982336 with type 'int'}} + // expected-warning@-2 {{overflow in expression; result is -1'375'982'336 with type 'int'}} #else // expected-error@-4 {{case value is not a constant expression}} \ // expected-note@-4 {{value 97408265472 is outside the range of representable values of type 'int'}} @@ -47,7 +47,7 @@ int f(int i) { case 2147483647: return 0; } - return (i, 65537) * 65537; // expected-warning {{overflow in expression; result is 131073 with type 'int'}} \ + return (i, 65537) * 65537; // expected-warning {{overflow in expression; result is 131'073 with type 'int'}} \ // expected-warning {{left operand of comma operator has no effect}} } diff --git a/clang/test/SemaCXX/enum.cpp b/clang/test/SemaCXX/enum.cpp index fc65fd16f8c3..c482b3c571ab 100644 --- a/clang/test/SemaCXX/enum.cpp +++ b/clang/test/SemaCXX/enum.cpp @@ -103,7 +103,7 @@ enum { overflow = 123456 * 234567 }; // expected-warning@-2 {{not an integral constant expression}} // expected-note@-3 {{value 28958703552 is outside the range of representable values}} #else -// expected-warning@-5 {{overflow in expression; result is -1106067520 with type 'int'}} +// expected-warning@-5 {{overflow in expression; result is -1'106'067'520 with type 'int'}} #endif // FIXME: This is not consistent with the above case. @@ -112,7 +112,7 @@ enum NoFold : int { overflow2 = 123456 * 234567 }; // expected-error@-2 {{enumerator value is not a constant expression}} // expected-note@-3 {{value 28958703552 is outside the range of representable values}} #else -// expected-warning@-5 {{overflow in expression; result is -1106067520 with type 'int'}} +// expected-warning@-5 {{overflow in expression; result is -1'106'067'520 with type 'int'}} // expected-warning@-6 {{extension}} #endif diff --git a/clang/test/SemaCXX/integer-overflow.cpp b/clang/test/SemaCXX/integer-overflow.cpp index 0e8ad050aa14..6049458b93bb 100644 --- a/clang/test/SemaCXX/integer-overflow.cpp +++ b/clang/test/SemaCXX/integer-overflow.cpp @@ -13,157 +13,157 @@ uint64_t f0(uint64_t); uint64_t f1(uint64_t, uint32_t); uint64_t f2(uint64_t, ...); -static const uint64_t overflow = 1 * 4608 * 1024 * 1024; // expected-warning {{overflow in expression; result is 536870912 with type 'int'}} +static const uint64_t overflow = 1 * 4608 * 1024 * 1024; // expected-warning {{overflow in expression; result is 536'870'912 with type 'int'}} uint64_t check_integer_overflows(int i) { //expected-note 0+{{declared here}} -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} uint64_t overflow = 4608 * 1024 * 1024, -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} overflow2 = (uint64_t)(4608 * 1024 * 1024), -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} overflow3 = (uint64_t)(4608 * 1024 * 1024 * i), -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} overflow4 = (1ULL * ((4608) * ((1024) * (1024))) + 2ULL), -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} overflow5 = static_cast(4608 * 1024 * 1024), -// expected-warning@+1 2{{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 2{{overflow in expression; result is 536'870'912 with type 'int'}} multi_overflow = (uint64_t)((uint64_t)(4608 * 1024 * 1024) * (uint64_t)(4608 * 1024 * 1024)); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} overflow += overflow2 = overflow3 = (uint64_t)(4608 * 1024 * 1024); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} overflow += overflow2 = overflow3 = 4608 * 1024 * 1024; -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} overflow += overflow2 = overflow3 = static_cast(4608 * 1024 * 1024); uint64_t not_overflow = 4608 * 1024 * 1024ULL; uint64_t not_overflow2 = (1ULL * ((uint64_t)(4608) * (1024 * 1024)) + 2ULL); -// expected-warning@+1 2{{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 2{{overflow in expression; result is 536'870'912 with type 'int'}} overflow = 4608 * 1024 * 1024 ? 4608 * 1024 * 1024 : 0; -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} overflow = 0 ? 0 : 4608 * 1024 * 1024; -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} if (4608 * 1024 * 1024) return 0; -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} if ((uint64_t)(4608 * 1024 * 1024)) return 1; -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} if (static_cast(4608 * 1024 * 1024)) return 1; -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} if ((uint64_t)(4608 * 1024 * 1024)) return 2; -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} if ((uint64_t)(4608 * 1024 * 1024 * i)) return 3; -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} if ((1ULL * ((4608) * ((1024) * (1024))) + 2ULL)) return 4; -// expected-warning@+1 2{{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 2{{overflow in expression; result is 536'870'912 with type 'int'}} if ((uint64_t)((uint64_t)(4608 * 1024 * 1024) * (uint64_t)(4608 * 1024 * 1024))) return 5; #if __cplusplus < 201103L switch (i) { -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} case 4608 * 1024 * 1024: return 6; -// expected-warning@+1 {{overflow in expression; result is 537919488 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 537'919'488 with type 'int'}} case (uint64_t)(4609 * 1024 * 1024): return 7; -// expected-warning@+1 {{overflow in expression; result is 537919488 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 537'919'488 with type 'int'}} case 1 + static_cast(4609 * 1024 * 1024): return 7; // expected-error@+1 {{expression is not an integral constant expression}} case ((uint64_t)(4608 * 1024 * 1024 * i)): return 8; -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} case ((1ULL * ((4608) * ((1024) * (1024))) + 2ULL)): return 9; -// expected-warning@+2 2{{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+2 2{{overflow in expression; result is 536'870'912 with type 'int'}} // expected-warning@+1 {{overflow converting case value to switch condition type (288230376151711744 to 0)}} case ((uint64_t)((uint64_t)(4608 * 1024 * 1024) * (uint64_t)(4608 * 1024 * 1024))): return 10; } #endif -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} while (4608 * 1024 * 1024); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} while ((uint64_t)(4608 * 1024 * 1024)); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} while (static_cast(4608 * 1024 * 1024)); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} while ((uint64_t)(4608 * 1024 * 1024)); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} while ((uint64_t)(4608 * 1024 * 1024 * i)); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} while ((1ULL * ((4608) * ((1024) * (1024))) + 2ULL)); -// expected-warning@+1 2{{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 2{{overflow in expression; result is 536'870'912 with type 'int'}} while ((uint64_t)((uint64_t)(4608 * 1024 * 1024) * (uint64_t)(4608 * 1024 * 1024))); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} do { } while (4608 * 1024 * 1024); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} do { } while ((uint64_t)(4608 * 1024 * 1024)); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} do { } while (static_cast(4608 * 1024 * 1024)); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} do { } while ((uint64_t)(4608 * 1024 * 1024)); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} do { } while ((uint64_t)(4608 * 1024 * 1024 * i)); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} do { } while ((1ULL * ((4608) * ((1024) * (1024))) + 2ULL)); -// expected-warning@+1 2{{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 2{{overflow in expression; result is 536'870'912 with type 'int'}} do { } while ((uint64_t)((uint64_t)(4608 * 1024 * 1024) * (uint64_t)(4608 * 1024 * 1024))); -// expected-warning@+3 {{overflow in expression; result is 536870912 with type 'int'}} -// expected-warning@+3 {{overflow in expression; result is 536870912 with type 'int'}} -// expected-warning@+3 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+3 {{overflow in expression; result is 536'870'912 with type 'int'}} +// expected-warning@+3 {{overflow in expression; result is 536'870'912 with type 'int'}} +// expected-warning@+3 {{overflow in expression; result is 536'870'912 with type 'int'}} for (uint64_t i = 4608 * 1024 * 1024; (uint64_t)(4608 * 1024 * 1024); i += (uint64_t)(4608 * 1024 * 1024 * i)); -// expected-warning@+3 {{overflow in expression; result is 536870912 with type 'int'}} -// expected-warning@+3 2{{overflow in expression; result is 536870912 with type 'int'}} -// expected-warning@+3 2{{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+3 {{overflow in expression; result is 536'870'912 with type 'int'}} +// expected-warning@+3 2{{overflow in expression; result is 536'870'912 with type 'int'}} +// expected-warning@+3 2{{overflow in expression; result is 536'870'912 with type 'int'}} for (uint64_t i = (1ULL * ((4608) * ((1024) * (1024))) + 2ULL); ((uint64_t)((uint64_t)(4608 * 1024 * 1024) * (uint64_t)(4608 * 1024 * 1024))); i = ((4608 * 1024 * 1024) + ((uint64_t)(4608 * 1024 * 1024)))); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} _Complex long long x = 4608 * 1024 * 1024; -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} (__real__ x) = 4608 * 1024 * 1024; -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} (__imag__ x) = 4608 * 1024 * 1024; -// expected-warning@+2 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+2 {{overflow in expression; result is 536'870'912 with type 'int'}} uint64_t a[10]; a[4608 * 1024 * 1024] = 1; #if __cplusplus < 201103L @@ -171,22 +171,22 @@ uint64_t check_integer_overflows(int i) { //expected-note 0+{{declared here}} // expected-note@-4 {{array 'a' declared here}} #endif -// expected-warning@+1 2{{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 2{{overflow in expression; result is 536'870'912 with type 'int'}} return ((4608 * 1024 * 1024) + ((uint64_t)(4608 * 1024 * 1024))); } void check_integer_overflows_in_function_calls() { -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} (void)f0(4608 * 1024 * 1024); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} uint64_t x = f0(4608 * 1024 * 1024); -// expected-warning@+2 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+2 {{overflow in expression; result is 536'870'912 with type 'int'}} uint64_t (*f0_ptr)(uint64_t) = &f0; (void)(*f0_ptr)(4608 * 1024 * 1024); -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} (void)f2(0, f0(4608 * 1024 * 1024)); } @@ -211,7 +211,7 @@ namespace EvaluationCrashes { namespace GH31643 { void f() { - int a = -(1<<31); // expected-warning {{overflow in expression; result is -2147483648 with type 'int'}} + int a = -(1<<31); // expected-warning {{overflow in expression; result is -2'147'483'648 with type 'int'}} } } @@ -237,8 +237,8 @@ u_ptr Wrap(int64_t x) { int64_t Pass(int64_t x) { return x; } int m() { - int64_t x = Pass(30 * 24 * 60 * 59 * 1000); // expected-warning {{overflow in expression; result is -1746167296 with type 'int'}} - auto r = Wrap(Pass(30 * 24 * 60 * 59 * 1000)); // expected-warning {{overflow in expression; result is -1746167296 with type 'int'}} + int64_t x = Pass(30 * 24 * 60 * 59 * 1000); // expected-warning {{overflow in expression; result is -1'746'167'296 with type 'int'}} + auto r = Wrap(Pass(30 * 24 * 60 * 59 * 1000)); // expected-warning {{overflow in expression; result is -1'746'167'296 with type 'int'}} return 0; } } diff --git a/clang/test/SemaObjC/integer-overflow.m b/clang/test/SemaObjC/integer-overflow.m index 6d82e2951c1e..255142729f8d 100644 --- a/clang/test/SemaObjC/integer-overflow.m +++ b/clang/test/SemaObjC/integer-overflow.m @@ -9,10 +9,10 @@ } - (void)testIntegerOverflows { -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} (void)[self add:0 with:4608 * 1024 * 1024]; -// expected-warning@+1 {{overflow in expression; result is 536870912 with type 'int'}} +// expected-warning@+1 {{overflow in expression; result is 536'870'912 with type 'int'}} (void)[self add:0 with:[self add:4608 * 1024 * 1024 with:0]]; } @end diff --git a/clang/test/SemaObjC/objc-literal-nsnumber.m b/clang/test/SemaObjC/objc-literal-nsnumber.m index aa7dc955fb5f..a6c098947b0b 100644 --- a/clang/test/SemaObjC/objc-literal-nsnumber.m +++ b/clang/test/SemaObjC/objc-literal-nsnumber.m @@ -64,7 +64,7 @@ int main(void) { @-five; // expected-error{{@- must be followed by a number to form an NSNumber object}} @+five; // expected-error{{@+ must be followed by a number to form an NSNumber object}} NSNumber *av = @(1391126400000); - NSNumber *bv = @(1391126400 * 1000); // expected-warning {{overflow in expression; result is -443003904 with type 'int'}} + NSNumber *bv = @(1391126400 * 1000); // expected-warning {{overflow in expression; result is -443'003'904 with type 'int'}} NSNumber *cv = @(big * thousand); } diff --git a/llvm/include/llvm/ADT/APInt.h b/llvm/include/llvm/ADT/APInt.h index 6c05367cecb1..1fc3c7b2236a 100644 --- a/llvm/include/llvm/ADT/APInt.h +++ b/llvm/include/llvm/ADT/APInt.h @@ -1626,7 +1626,8 @@ public: /// SmallString. If Radix > 10, UpperCase determine the case of letter /// digits. void toString(SmallVectorImpl &Str, unsigned Radix, bool Signed, - bool formatAsCLiteral = false, bool UpperCase = true) const; + bool formatAsCLiteral = false, bool UpperCase = true, + bool InsertSeparators = false) const; /// Considers the APInt to be unsigned and converts it into a string in the /// radix given. The radix can be 2, 8, 10 16, or 36. diff --git a/llvm/include/llvm/ADT/StringExtras.h b/llvm/include/llvm/ADT/StringExtras.h index a24368924bc9..20e6ad1f68f9 100644 --- a/llvm/include/llvm/ADT/StringExtras.h +++ b/llvm/include/llvm/ADT/StringExtras.h @@ -329,9 +329,11 @@ inline std::string itostr(int64_t X) { } inline std::string toString(const APInt &I, unsigned Radix, bool Signed, - bool formatAsCLiteral = false) { + bool formatAsCLiteral = false, + bool UpperCase = true, + bool InsertSeparators = false) { SmallString<40> S; - I.toString(S, Radix, Signed, formatAsCLiteral); + I.toString(S, Radix, Signed, formatAsCLiteral, UpperCase, InsertSeparators); return std::string(S); } diff --git a/llvm/lib/Support/APInt.cpp b/llvm/lib/Support/APInt.cpp index 05b1526da95f..e686b9765233 100644 --- a/llvm/lib/Support/APInt.cpp +++ b/llvm/lib/Support/APInt.cpp @@ -2161,7 +2161,8 @@ void APInt::fromString(unsigned numbits, StringRef str, uint8_t radix) { } void APInt::toString(SmallVectorImpl &Str, unsigned Radix, bool Signed, - bool formatAsCLiteral, bool UpperCase) const { + bool formatAsCLiteral, bool UpperCase, + bool InsertSeparators) const { assert((Radix == 10 || Radix == 8 || Radix == 16 || Radix == 2 || Radix == 36) && "Radix should be 2, 8, 10, 16, or 36!"); @@ -2187,6 +2188,9 @@ void APInt::toString(SmallVectorImpl &Str, unsigned Radix, bool Signed, } } + // Number of digits in a group between separators. + unsigned Grouping = (Radix == 8 || Radix == 10) ? 3 : 4; + // First, check for a zero value and just short circuit the logic below. if (isZero()) { while (*Prefix) { @@ -2223,9 +2227,13 @@ void APInt::toString(SmallVectorImpl &Str, unsigned Radix, bool Signed, ++Prefix; }; + int Pos = 0; while (N) { + if (InsertSeparators && Pos % Grouping == 0 && Pos > 0) + *--BufPtr = '\''; *--BufPtr = Digits[N % Radix]; N /= Radix; + Pos++; } Str.append(BufPtr, std::end(Buffer)); return; @@ -2257,17 +2265,27 @@ void APInt::toString(SmallVectorImpl &Str, unsigned Radix, bool Signed, unsigned ShiftAmt = (Radix == 16 ? 4 : (Radix == 8 ? 3 : 1)); unsigned MaskAmt = Radix - 1; + int Pos = 0; while (Tmp.getBoolValue()) { unsigned Digit = unsigned(Tmp.getRawData()[0]) & MaskAmt; + if (InsertSeparators && Pos % Grouping == 0 && Pos > 0) + Str.push_back('\''); + Str.push_back(Digits[Digit]); Tmp.lshrInPlace(ShiftAmt); + Pos++; } } else { + int Pos = 0; while (Tmp.getBoolValue()) { uint64_t Digit; udivrem(Tmp, Radix, Tmp, Digit); assert(Digit < Radix && "divide failed"); + if (InsertSeparators && Pos % Grouping == 0 && Pos > 0) + Str.push_back('\''); + Str.push_back(Digits[Digit]); + Pos++; } } diff --git a/llvm/unittests/ADT/APIntTest.cpp b/llvm/unittests/ADT/APIntTest.cpp index 2fe59f05ca75..24324822356b 100644 --- a/llvm/unittests/ADT/APIntTest.cpp +++ b/llvm/unittests/ADT/APIntTest.cpp @@ -1379,6 +1379,23 @@ TEST(APIntTest, toString) { EXPECT_EQ(std::string(S), "0"); S.clear(); + // with separators + APInt(64, 140).toString(S, 2, false, true, false, true); + EXPECT_EQ(std::string(S), "0b1000'1100"); + S.clear(); + APInt(64, 1024).toString(S, 8, false, true, false, true); + EXPECT_EQ(std::string(S), "02'000"); + S.clear(); + APInt(64, 1000000).toString(S, 10, false, true, false, true); + EXPECT_EQ(std::string(S), "1'000'000"); + S.clear(); + APInt(64, 1000000).toString(S, 16, false, true, true, true); + EXPECT_EQ(std::string(S), "0xF'4240"); + S.clear(); + APInt(64, 1'000'000'000).toString(S, 36, false, false, false, true); + EXPECT_EQ(std::string(S), "gj'dgxs"); + S.clear(); + isSigned = false; APInt(8, 255, isSigned).toString(S, 2, isSigned, true); EXPECT_EQ(std::string(S), "0b11111111"); @@ -1415,6 +1432,24 @@ TEST(APIntTest, toString) { APInt(8, 255, isSigned).toString(S, 36, isSigned, false); EXPECT_EQ(std::string(S), "-1"); S.clear(); + + // negative with separators + APInt(64, -140, isSigned).toString(S, 2, isSigned, true, false, true); + EXPECT_EQ(std::string(S), "-0b1000'1100"); + S.clear(); + APInt(64, -1024, isSigned).toString(S, 8, isSigned, true, false, true); + EXPECT_EQ(std::string(S), "-02'000"); + S.clear(); + APInt(64, -1000000, isSigned).toString(S, 10, isSigned, true, false, true); + EXPECT_EQ(std::string(S), "-1'000'000"); + S.clear(); + APInt(64, -1000000, isSigned).toString(S, 16, isSigned, true, true, true); + EXPECT_EQ(std::string(S), "-0xF'4240"); + S.clear(); + APInt(64, -1'000'000'000, isSigned) + .toString(S, 36, isSigned, false, false, true); + EXPECT_EQ(std::string(S), "-gj'dgxs"); + S.clear(); } TEST(APIntTest, Log2) { -- GitLab From d95a0d7c0ff324c1e84606d26591416e6bf02984 Mon Sep 17 00:00:00 2001 From: Yeting Kuo <46629943+yetingk@users.noreply.github.com> Date: Tue, 5 Mar 2024 20:48:37 +0800 Subject: [PATCH 135/929] [DAG] Teach SelectionDAGBuilder to read parameter alignment of compressstore/expandload. (#83763) Previously SelectionDAGBuilder used ABI alignment for compressstore/expandload. This patch allows SelectionDAGBuilder to use parameter alignment like vp intrinsics. This does not follow the original code to default use vector type alignment, since it is possible implemented to unaligned vector alignment. --- .../SelectionDAG/SelectionDAGBuilder.cpp | 29 ++++++------- .../CodeGen/X86/masked_compressstore_isel.ll | 21 +++++++++- .../CodeGen/X86/masked_expandload_isel.ll | 42 +++++++++++++++++++ 3 files changed, 74 insertions(+), 18 deletions(-) create mode 100644 llvm/test/CodeGen/X86/masked_expandload_isel.ll diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 48476b0ef970..d6ab0cf6df87 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -4721,24 +4721,24 @@ void SelectionDAGBuilder::visitMaskedStore(const CallInst &I, SDLoc sdl = getCurSDLoc(); auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0, - MaybeAlign &Alignment) { + Align &Alignment) { // llvm.masked.store.*(Src0, Ptr, alignment, Mask) Src0 = I.getArgOperand(0); Ptr = I.getArgOperand(1); - Alignment = cast(I.getArgOperand(2))->getMaybeAlignValue(); + Alignment = cast(I.getArgOperand(2))->getAlignValue(); Mask = I.getArgOperand(3); }; auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0, - MaybeAlign &Alignment) { + Align &Alignment) { // llvm.masked.compressstore.*(Src0, Ptr, Mask) Src0 = I.getArgOperand(0); Ptr = I.getArgOperand(1); Mask = I.getArgOperand(2); - Alignment = std::nullopt; + Alignment = I.getParamAlign(1).valueOrOne(); }; Value *PtrOperand, *MaskOperand, *Src0Operand; - MaybeAlign Alignment; + Align Alignment; if (IsCompressing) getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment); else @@ -4750,12 +4750,10 @@ void SelectionDAGBuilder::visitMaskedStore(const CallInst &I, SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); EVT VT = Src0.getValueType(); - if (!Alignment) - Alignment = DAG.getEVTAlign(VT); MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore, - MemoryLocation::UnknownSize, *Alignment, I.getAAMetadata()); + MemoryLocation::UnknownSize, Alignment, I.getAAMetadata()); SDValue StoreNode = DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO, ISD::UNINDEXED, false /* Truncating */, IsCompressing); @@ -4887,24 +4885,24 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) { SDLoc sdl = getCurSDLoc(); auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0, - MaybeAlign &Alignment) { + Align &Alignment) { // @llvm.masked.load.*(Ptr, alignment, Mask, Src0) Ptr = I.getArgOperand(0); - Alignment = cast(I.getArgOperand(1))->getMaybeAlignValue(); + Alignment = cast(I.getArgOperand(1))->getAlignValue(); Mask = I.getArgOperand(2); Src0 = I.getArgOperand(3); }; auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0, - MaybeAlign &Alignment) { + Align &Alignment) { // @llvm.masked.expandload.*(Ptr, Mask, Src0) Ptr = I.getArgOperand(0); - Alignment = std::nullopt; + Alignment = I.getParamAlign(0).valueOrOne(); Mask = I.getArgOperand(1); Src0 = I.getArgOperand(2); }; Value *PtrOperand, *MaskOperand, *Src0Operand; - MaybeAlign Alignment; + Align Alignment; if (IsExpanding) getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment); else @@ -4916,9 +4914,6 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) { SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); EVT VT = Src0.getValueType(); - if (!Alignment) - Alignment = DAG.getEVTAlign(VT); - AAMDNodes AAInfo = I.getAAMetadata(); const MDNode *Ranges = getRangeMetadata(I); @@ -4930,7 +4925,7 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) { MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad, - MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges); + MemoryLocation::UnknownSize, Alignment, AAInfo, Ranges); SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO, diff --git a/llvm/test/CodeGen/X86/masked_compressstore_isel.ll b/llvm/test/CodeGen/X86/masked_compressstore_isel.ll index 0587434bff25..2a557ac9b97b 100644 --- a/llvm/test/CodeGen/X86/masked_compressstore_isel.ll +++ b/llvm/test/CodeGen/X86/masked_compressstore_isel.ll @@ -11,7 +11,7 @@ define void @_Z3fooiPiPs(<8 x i32> %gepload, <8 x i1> %0) #0 { ; CHECK-NEXT: [[VPSLLWZ128ri:%[0-9]+]]:vr128x = VPSLLWZ128ri [[COPY]], 15 ; CHECK-NEXT: [[VPMOVW2MZ128rr:%[0-9]+]]:vk16wm = VPMOVW2MZ128rr killed [[VPSLLWZ128ri]] ; CHECK-NEXT: [[VPMOVDWZ256rr:%[0-9]+]]:vr128x = VPMOVDWZ256rr [[COPY1]] - ; CHECK-NEXT: VPCOMPRESSWZ128mrk $noreg, 1, $noreg, 0, $noreg, killed [[VPMOVW2MZ128rr]], killed [[VPMOVDWZ256rr]] :: (store unknown-size into `ptr null`, align 16) + ; CHECK-NEXT: VPCOMPRESSWZ128mrk $noreg, 1, $noreg, 0, $noreg, killed [[VPMOVW2MZ128rr]], killed [[VPMOVDWZ256rr]] :: (store unknown-size into `ptr null`, align 1) ; CHECK-NEXT: RET 0 entry: %1 = trunc <8 x i32> %gepload to <8 x i16> @@ -19,6 +19,25 @@ entry: ret void } + +define void @_Z3foo2iPiPs(<8 x i32> %gepload, <8 x i1> %0) #0 { + ; CHECK-LABEL: name: _Z3foo2iPiPs + ; CHECK: bb.0.entry: + ; CHECK-NEXT: liveins: $ymm0, $xmm1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:vr128x = COPY $xmm1 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vr256x = COPY $ymm0 + ; CHECK-NEXT: [[VPSLLWZ128ri:%[0-9]+]]:vr128x = VPSLLWZ128ri [[COPY]], 15 + ; CHECK-NEXT: [[VPMOVW2MZ128rr:%[0-9]+]]:vk16wm = VPMOVW2MZ128rr killed [[VPSLLWZ128ri]] + ; CHECK-NEXT: [[VPMOVDWZ256rr:%[0-9]+]]:vr128x = VPMOVDWZ256rr [[COPY1]] + ; CHECK-NEXT: VPCOMPRESSWZ128mrk $noreg, 1, $noreg, 0, $noreg, killed [[VPMOVW2MZ128rr]], killed [[VPMOVDWZ256rr]] :: (store unknown-size into `ptr null`, align 16) + ; CHECK-NEXT: RET 0 +entry: + %1 = trunc <8 x i32> %gepload to <8 x i16> + tail call void @llvm.masked.compressstore.v8i16(<8 x i16> %1, ptr align 16 null, <8 x i1> %0) + ret void +} + ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: write) declare void @llvm.masked.compressstore.v8i16(<8 x i16>, ptr nocapture, <8 x i1>) #1 diff --git a/llvm/test/CodeGen/X86/masked_expandload_isel.ll b/llvm/test/CodeGen/X86/masked_expandload_isel.ll new file mode 100644 index 000000000000..b364625a1e6f --- /dev/null +++ b/llvm/test/CodeGen/X86/masked_expandload_isel.ll @@ -0,0 +1,42 @@ +; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4 +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -start-after=codegenprepare -stop-before finalize-isel | FileCheck %s + +define <8 x i16> @_Z3fooiPiPs(<8 x i16> %src, <8 x i1> %mask) #0 { + ; CHECK-LABEL: name: _Z3fooiPiPs + ; CHECK: bb.0.entry: + ; CHECK-NEXT: liveins: $xmm0, $xmm1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:vr128x = COPY $xmm1 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vr128x = COPY $xmm0 + ; CHECK-NEXT: [[VPSLLWZ128ri:%[0-9]+]]:vr128x = VPSLLWZ128ri [[COPY]], 15 + ; CHECK-NEXT: [[VPMOVW2MZ128rr:%[0-9]+]]:vk16wm = VPMOVW2MZ128rr killed [[VPSLLWZ128ri]] + ; CHECK-NEXT: [[VPEXPANDWZ128rmk:%[0-9]+]]:vr128x = VPEXPANDWZ128rmk [[COPY1]], killed [[VPMOVW2MZ128rr]], $noreg, 1, $noreg, 0, $noreg :: (load unknown-size from `ptr null`, align 1) + ; CHECK-NEXT: $xmm0 = COPY [[VPEXPANDWZ128rmk]] + ; CHECK-NEXT: RET 0, $xmm0 +entry: + %res = call <8 x i16> @llvm.masked.expandload.v8i16(ptr null, <8 x i1> %mask, <8 x i16> %src) + ret <8 x i16> %res +} + +define <8 x i16> @_Z3foo2iPiPs(<8 x i16> %src, <8 x i1> %mask) #0 { + ; CHECK-LABEL: name: _Z3foo2iPiPs + ; CHECK: bb.0.entry: + ; CHECK-NEXT: liveins: $xmm0, $xmm1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:vr128x = COPY $xmm1 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vr128x = COPY $xmm0 + ; CHECK-NEXT: [[VPSLLWZ128ri:%[0-9]+]]:vr128x = VPSLLWZ128ri [[COPY]], 15 + ; CHECK-NEXT: [[VPMOVW2MZ128rr:%[0-9]+]]:vk16wm = VPMOVW2MZ128rr killed [[VPSLLWZ128ri]] + ; CHECK-NEXT: [[VPEXPANDWZ128rmk:%[0-9]+]]:vr128x = VPEXPANDWZ128rmk [[COPY1]], killed [[VPMOVW2MZ128rr]], $noreg, 1, $noreg, 0, $noreg :: (load unknown-size from `ptr null`, align 16) + ; CHECK-NEXT: $xmm0 = COPY [[VPEXPANDWZ128rmk]] + ; CHECK-NEXT: RET 0, $xmm0 +entry: + %res = call <8 x i16> @llvm.masked.expandload.v8i16(ptr align 16 null, <8 x i1> %mask, <8 x i16> %src) + ret <8 x i16> %res +} + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: write) +declare <8 x i16> @llvm.masked.expandload.v8i16(ptr, <8 x i1>, <8 x i16>) + +attributes #0 = { "target-cpu"="icelake-server" } +attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: write) } -- GitLab From d51fcd4ed86ac6075c8a25b053c2b66051feaf62 Mon Sep 17 00:00:00 2001 From: Yingwei Zheng Date: Tue, 5 Mar 2024 21:08:15 +0800 Subject: [PATCH 136/929] [InstCombine] Handle scalable splat in `getFlippedStrictnessPredicateAndConstant` (#83980) This patch adds support for canonicalization of icmp with a scalable splat. Some optimizations assume that `icmp pred X, APInt C` is in canonical form. Fixes https://github.com/llvm/llvm-project/issues/83931. --- .../InstCombine/InstCombineCompares.cpp | 7 +++++++ llvm/test/Transforms/InstCombine/pr83931.ll | 15 +++++++++++++++ llvm/test/Transforms/InstCombine/select.ll | 2 +- llvm/test/Transforms/InstCombine/vscale_cmp.ll | 2 +- 4 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 llvm/test/Transforms/InstCombine/pr83931.ll diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp index 55207a8343d0..1cebab8203ea 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp @@ -6544,6 +6544,13 @@ InstCombiner::getFlippedStrictnessPredicateAndConstant(CmpInst::Predicate Pred, if (!SafeReplacementConstant) SafeReplacementConstant = CI; } + } else if (isa(C->getType())) { + // Handle scalable splat + Value *SplatC = C->getSplatValue(); + auto *CI = dyn_cast_or_null(SplatC); + // Bail out if the constant can't be safely incremented/decremented. + if (!CI || !ConstantIsOk(CI)) + return std::nullopt; } else { // ConstantExpr? return std::nullopt; diff --git a/llvm/test/Transforms/InstCombine/pr83931.ll b/llvm/test/Transforms/InstCombine/pr83931.ll new file mode 100644 index 000000000000..d36ac8d91abd --- /dev/null +++ b/llvm/test/Transforms/InstCombine/pr83931.ll @@ -0,0 +1,15 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -passes=instcombine < %s | FileCheck %s + +define @dont_crash( %x) { +; CHECK-LABEL: define @dont_crash( +; CHECK-SAME: [[X:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[RET:%.*]] = icmp sgt [[X]], shufflevector ( insertelement ( poison, i64 -309383, i64 0), poison, zeroinitializer) +; CHECK-NEXT: ret [[RET]] +; +entry: + %div = sdiv %x, splat (i64 309383) + %ret = icmp sge %div, zeroinitializer + ret %ret +} diff --git a/llvm/test/Transforms/InstCombine/select.ll b/llvm/test/Transforms/InstCombine/select.ll index 82baf05977db..4676129e3a1c 100644 --- a/llvm/test/Transforms/InstCombine/select.ll +++ b/llvm/test/Transforms/InstCombine/select.ll @@ -3423,7 +3423,7 @@ define @scalable_sign_bits( %x) { define @scalable_non_zero( %x) { ; CHECK-LABEL: @scalable_non_zero( ; CHECK-NEXT: [[A:%.*]] = or [[X:%.*]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[CMP:%.*]] = icmp ule [[A]], shufflevector ( insertelement ( poison, i32 56, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[CMP:%.*]] = icmp ult [[A]], shufflevector ( insertelement ( poison, i32 57, i64 0), poison, zeroinitializer) ; CHECK-NEXT: ret [[CMP]] ; %a = or %x, splat (i32 1) diff --git a/llvm/test/Transforms/InstCombine/vscale_cmp.ll b/llvm/test/Transforms/InstCombine/vscale_cmp.ll index a7f8368c5d62..b2bfc93da089 100644 --- a/llvm/test/Transforms/InstCombine/vscale_cmp.ll +++ b/llvm/test/Transforms/InstCombine/vscale_cmp.ll @@ -3,7 +3,7 @@ define @sge( %x) { ; CHECK-LABEL: @sge( -; CHECK-NEXT: [[CMP:%.*]] = icmp sge [[X:%.*]], zeroinitializer +; CHECK-NEXT: [[CMP:%.*]] = icmp sgt [[X:%.*]], shufflevector ( insertelement ( poison, i8 -1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: ret [[CMP]] ; %cmp = icmp sge %x, zeroinitializer -- GitLab From e84182af919d136d74b75ded4d599b38fb47dfb0 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Tue, 5 Mar 2024 14:21:33 +0100 Subject: [PATCH 137/929] [X86][Inline] Skip inline asm in inlining target feature check (#83820) When inlining across functions with different target features, we perform roughly two checks: 1. The caller features must be a superset of the callee features. 2. Calls in the callee cannot use types where the target features would change the call ABI (e.g. by changing whether something is passed in a zmm or two ymm registers). The latter check is very crude right now. The latter check currently also catches inline asm "calls". I believe that inline asm should be excluded from this check, as it is independent from the usual call ABI, and instead governed by the inline asm constraint string. Fixes https://github.com/llvm/llvm-project/issues/67054. --- llvm/lib/Target/X86/X86TargetTransformInfo.cpp | 4 ++++ .../Inline/X86/call-abi-compatibility.ll | 17 ++++++----------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp index 18bf32fe1aca..4cca291a2456 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -6087,6 +6087,10 @@ bool X86TTIImpl::areInlineCompatible(const Function *Caller, for (const Instruction &I : instructions(Callee)) { if (const auto *CB = dyn_cast(&I)) { + // Having more target features is fine for inline ASM. + if (CB->isInlineAsm()) + continue; + SmallVector Types; for (Value *Arg : CB->args()) Types.push_back(Arg->getType()); diff --git a/llvm/test/Transforms/Inline/X86/call-abi-compatibility.ll b/llvm/test/Transforms/Inline/X86/call-abi-compatibility.ll index f03270bafea9..6f582cab2f14 100644 --- a/llvm/test/Transforms/Inline/X86/call-abi-compatibility.ll +++ b/llvm/test/Transforms/Inline/X86/call-abi-compatibility.ll @@ -94,27 +94,22 @@ define internal void @caller_not_avx4() { declare i64 @caller_unknown_simple(i64) -; FIXME: This call should get inlined, because the callee only contains +; This call should get inlined, because the callee only contains ; inline ASM, not real calls. define <8 x i64> @caller_inline_asm(ptr %p0, i64 %k, ptr %p1, ptr %p2) #0 { ; CHECK-LABEL: define {{[^@]+}}@caller_inline_asm ; CHECK-SAME: (ptr [[P0:%.*]], i64 [[K:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]]) #[[ATTR2:[0-9]+]] { -; CHECK-NEXT: [[CALL:%.*]] = call <8 x i64> @callee_inline_asm(ptr [[P0]], i64 [[K]], ptr [[P1]], ptr [[P2]]) -; CHECK-NEXT: ret <8 x i64> [[CALL]] +; CHECK-NEXT: [[SRC_I:%.*]] = load <8 x i64>, ptr [[P0]], align 64 +; CHECK-NEXT: [[A_I:%.*]] = load <8 x i64>, ptr [[P1]], align 64 +; CHECK-NEXT: [[B_I:%.*]] = load <8 x i64>, ptr [[P2]], align 64 +; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i64> asm "vpaddb\09$($3, $2, $0 {$1}", "=v,^Yk,v,v,0,~{dirflag},~{fpsr},~{flags}"(i64 [[K]], <8 x i64> [[A_I]], <8 x i64> [[B_I]], <8 x i64> [[SRC_I]]) +; CHECK-NEXT: ret <8 x i64> [[TMP1]] ; %call = call <8 x i64> @callee_inline_asm(ptr %p0, i64 %k, ptr %p1, ptr %p2) ret <8 x i64> %call } define internal <8 x i64> @callee_inline_asm(ptr %p0, i64 %k, ptr %p1, ptr %p2) #1 { -; CHECK-LABEL: define {{[^@]+}}@callee_inline_asm -; CHECK-SAME: (ptr [[P0:%.*]], i64 [[K:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]]) #[[ATTR3:[0-9]+]] { -; CHECK-NEXT: [[SRC:%.*]] = load <8 x i64>, ptr [[P0]], align 64 -; CHECK-NEXT: [[A:%.*]] = load <8 x i64>, ptr [[P1]], align 64 -; CHECK-NEXT: [[B:%.*]] = load <8 x i64>, ptr [[P2]], align 64 -; CHECK-NEXT: [[TMP1:%.*]] = tail call <8 x i64> asm "vpaddb\09$($3, $2, $0 {$1}", "=v,^Yk,v,v,0,~{dirflag},~{fpsr},~{flags}"(i64 [[K]], <8 x i64> [[A]], <8 x i64> [[B]], <8 x i64> [[SRC]]) -; CHECK-NEXT: ret <8 x i64> [[TMP1]] -; %src = load <8 x i64>, ptr %p0, align 64 %a = load <8 x i64>, ptr %p1, align 64 %b = load <8 x i64>, ptr %p2, align 64 -- GitLab From 894f52fc0d4adbe8782e97ce7dd100da02abf020 Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Tue, 5 Mar 2024 13:27:51 +0000 Subject: [PATCH 138/929] [AMDGPU] Use BUF multiclasses to reduce repetition. NFC. (#84003) Define BUF Real instructions with this general pattern for all architectures (not just GFX11): multiclass Something_Real_gfx11<...> { defvar ps = !cast(NAME); def _gfx11 : ...; } This allows removing a huge amount of repetition in the definitions of individual Real instructions, where they would have to !cast their own name to a Pseudo and pass that in as a class argument. --- llvm/lib/Target/AMDGPU/BUFInstructions.td | 605 +++++++++++----------- 1 file changed, 289 insertions(+), 316 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index a8af95422085..cb68e45d69e8 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -2323,19 +2323,41 @@ class Base_MUBUF_Real_gfx6_gfx7_gfx10 op, MUBUF_Pseudo ps, int ef> : let Inst{55} = ps.tfe; } -class MUBUF_Real_gfx10 op, MUBUF_Pseudo ps> : - Base_MUBUF_Real_gfx6_gfx7_gfx10 { - let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); - let Inst{25} = op{7}; - let AssemblerPredicate = isGFX10Only; - let DecoderNamespace = "GFX10"; +multiclass MUBUF_Real_gfx10 op> { + defvar ps = !cast(NAME); + def _gfx10 : Base_MUBUF_Real_gfx6_gfx7_gfx10 { + let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); + let Inst{25} = op{7}; + let AssemblerPredicate = isGFX10Only; + let DecoderNamespace = "GFX10"; + } +} + +multiclass MUBUF_Real_gfx6_gfx7 op> { + defvar ps = !cast(NAME); + def _gfx6_gfx7 : Base_MUBUF_Real_gfx6_gfx7_gfx10 { + let Inst{15} = ps.addr64; + let AssemblerPredicate = isGFX6GFX7; + let DecoderNamespace = "GFX6GFX7"; + } +} + +multiclass MUBUF_Real_gfx6 op> { + defvar ps = !cast(NAME); + def _gfx6 : Base_MUBUF_Real_gfx6_gfx7_gfx10 { + let Inst{15} = ps.addr64; + let AssemblerPredicate = isGFX6; + let DecoderNamespace = "GFX6"; + } } -class MUBUF_Real_gfx6_gfx7 op, MUBUF_Pseudo ps> : - Base_MUBUF_Real_gfx6_gfx7_gfx10 { - let Inst{15} = ps.addr64; - let AssemblerPredicate = isGFX6GFX7; - let DecoderNamespace = "GFX6GFX7"; +multiclass MUBUF_Real_gfx7 op> { + defvar ps = !cast(NAME); + def _gfx7 : Base_MUBUF_Real_gfx6_gfx7_gfx10 { + let Inst{15} = ps.addr64; + let AssemblerPredicate = isGFX7Only; + let DecoderNamespace = "GFX7"; + } } //===----------------------------------------------------------------------===// @@ -2407,19 +2429,19 @@ class VBUFFER_MUBUF_Real_gfx12 op, MUBUF_Pseudo ps, let DecoderNamespace = "GFX12"; } -class VBUFFER_MTBUF_Real_gfx12 op, MTBUF_Pseudo ps, - string real_name = ps.Mnemonic> : - VBUFFER_Real, SIMCInstr { +multiclass VBUFFER_MTBUF_Real_gfx12 op, string real_name> { + defvar ps = !cast(NAME); + def _gfx12 : VBUFFER_Real, SIMCInstr { + let MTBUF = 1; - let MTBUF = 1; - - bits<7> format; + bits<7> format; - let Inst{17-14} = op; - let Inst{21-18} = 0b1000; - let Inst{61-55} = format; - let AssemblerPredicate = isGFX12Only; - let DecoderNamespace = "GFX12"; + let Inst{17-14} = op; + let Inst{21-18} = 0b1000; + let Inst{61-55} = format; + let AssemblerPredicate = isGFX12Only; + let DecoderNamespace = "GFX12"; + } } //===----------------------------------------------------------------------===// @@ -2442,24 +2464,26 @@ class Mnem_gfx11 : class Mnem_gfx12 : MnemonicAlias, Requires<[isGFX12Plus]>; -class MUBUF_Real_gfx11_impl op, string ps_name, string real_name> : - MUBUF_Real_gfx11(ps_name), real_name>; +multiclass MUBUF_Real_gfx11_impl op, string real_name> { + def _gfx11 : MUBUF_Real_gfx11(NAME), real_name>; +} -class VBUFFER_MUBUF_Real_gfx12_impl op, string ps_name, string real_name> : - VBUFFER_MUBUF_Real_gfx12(ps_name), real_name>; +multiclass VBUFFER_MUBUF_Real_gfx12_impl op, string real_name> { + def _gfx12 : VBUFFER_MUBUF_Real_gfx12(NAME), real_name>; +} multiclass MUBUF_Real_AllAddr_gfx11_Renamed_Impl2 op, string real_name> { - def _BOTHEN_gfx11 : MUBUF_Real_gfx11_impl; - def _IDXEN_gfx11 : MUBUF_Real_gfx11_impl; - def _OFFEN_gfx11 : MUBUF_Real_gfx11_impl; - def _OFFSET_gfx11 : MUBUF_Real_gfx11_impl; + defm _BOTHEN : MUBUF_Real_gfx11_impl; + defm _IDXEN : MUBUF_Real_gfx11_impl; + defm _OFFEN : MUBUF_Real_gfx11_impl; + defm _OFFSET : MUBUF_Real_gfx11_impl; } multiclass MUBUF_Real_AllAddr_gfx12_Renamed_Impl2 op, string real_name> { - def _BOTHEN_gfx12 : VBUFFER_MUBUF_Real_gfx12_impl; - def _IDXEN_gfx12 : VBUFFER_MUBUF_Real_gfx12_impl; - def _OFFEN_gfx12 : VBUFFER_MUBUF_Real_gfx12_impl; - def _OFFSET_gfx12 : VBUFFER_MUBUF_Real_gfx12_impl; + defm _VBUFFER_BOTHEN : VBUFFER_MUBUF_Real_gfx12_impl; + defm _VBUFFER_IDXEN : VBUFFER_MUBUF_Real_gfx12_impl; + defm _VBUFFER_OFFEN : VBUFFER_MUBUF_Real_gfx12_impl; + defm _VBUFFER_OFFSET : VBUFFER_MUBUF_Real_gfx12_impl; } multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl2 op, string real_name> : @@ -2492,38 +2516,30 @@ multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Renamed op, string real_name> def : Mnem_gfx11_gfx12.Mnemonic, real_name>; } -class MUBUF_Real_Atomic_gfx11_impl op, string ps_name, - string real_name> : - MUBUF_Real_gfx11(ps_name), real_name>; +multiclass MUBUF_Real_Atomic_gfx11_impl op, string real_name> { + def _gfx11 : MUBUF_Real_gfx11(NAME), real_name>; +} -class MUBUF_Real_Atomic_gfx12_impl op, string ps_name, - string real_name> : - VBUFFER_MUBUF_Real_gfx12(ps_name), real_name>; +multiclass MUBUF_Real_Atomic_gfx12_impl op, string real_name> { + def _gfx12 : VBUFFER_MUBUF_Real_gfx12(NAME), real_name>; +} multiclass MUBUF_Real_Atomic_gfx11_Renamed_impl op, bit is_return, string real_name> { defvar Rtn = !if(is_return, "_RTN", ""); - def _BOTHEN#Rtn#_gfx11 : - MUBUF_Real_Atomic_gfx11_impl; - def _IDXEN#Rtn#_gfx11 : - MUBUF_Real_Atomic_gfx11_impl; - def _OFFEN#Rtn#_gfx11 : - MUBUF_Real_Atomic_gfx11_impl; - def _OFFSET#Rtn#_gfx11 : - MUBUF_Real_Atomic_gfx11_impl; + defm _BOTHEN#Rtn : MUBUF_Real_Atomic_gfx11_impl; + defm _IDXEN#Rtn : MUBUF_Real_Atomic_gfx11_impl; + defm _OFFEN#Rtn : MUBUF_Real_Atomic_gfx11_impl; + defm _OFFSET#Rtn : MUBUF_Real_Atomic_gfx11_impl; } multiclass MUBUF_Real_Atomic_gfx12_Renamed_impl op, bit is_return, string real_name> { defvar Rtn = !if(is_return, "_RTN", ""); - def _BOTHEN#Rtn#_gfx12 : - MUBUF_Real_Atomic_gfx12_impl; - def _IDXEN#Rtn#_gfx12 : - MUBUF_Real_Atomic_gfx12_impl; - def _OFFEN#Rtn#_gfx12 : - MUBUF_Real_Atomic_gfx12_impl; - def _OFFSET#Rtn#_gfx12 : - MUBUF_Real_Atomic_gfx12_impl; + defm _VBUFFER_BOTHEN#Rtn : MUBUF_Real_Atomic_gfx12_impl; + defm _VBUFFER_IDXEN#Rtn : MUBUF_Real_Atomic_gfx12_impl; + defm _VBUFFER_OFFEN#Rtn : MUBUF_Real_Atomic_gfx12_impl; + defm _VBUFFER_OFFSET#Rtn : MUBUF_Real_Atomic_gfx12_impl; } multiclass MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl op, bit is_return, @@ -2652,52 +2668,40 @@ defm BUFFER_ATOMIC_PK_ADD_BF16 : MUBUF_Real_Atomic_gfx12<0x05a>; //===----------------------------------------------------------------------===// multiclass MUBUF_Real_AllAddr_Helper_gfx10 op> { - def _BOTHEN_gfx10 : - MUBUF_Real_gfx10(NAME#"_BOTHEN")>; - def _IDXEN_gfx10 : - MUBUF_Real_gfx10(NAME#"_IDXEN")>; - def _OFFEN_gfx10 : - MUBUF_Real_gfx10(NAME#"_OFFEN")>; - def _OFFSET_gfx10 : - MUBUF_Real_gfx10(NAME#"_OFFSET")>; + defm _BOTHEN : MUBUF_Real_gfx10; + defm _IDXEN : MUBUF_Real_gfx10; + defm _OFFEN : MUBUF_Real_gfx10; + defm _OFFSET : MUBUF_Real_gfx10; } multiclass MUBUF_Real_AllAddr_gfx10 op> { defm NAME : MUBUF_Real_AllAddr_Helper_gfx10; defm _TFE : MUBUF_Real_AllAddr_Helper_gfx10; } multiclass MUBUF_Real_AllAddr_Lds_gfx10 op, bit isTFE = 0> { - def _OFFSET_gfx10 : MUBUF_Real_gfx10(NAME#"_OFFSET")>; - def _OFFEN_gfx10 : MUBUF_Real_gfx10(NAME#"_OFFEN")>; - def _IDXEN_gfx10 : MUBUF_Real_gfx10(NAME#"_IDXEN")>; - def _BOTHEN_gfx10 : MUBUF_Real_gfx10(NAME#"_BOTHEN")>; + defm _OFFSET : MUBUF_Real_gfx10; + defm _OFFEN : MUBUF_Real_gfx10; + defm _IDXEN : MUBUF_Real_gfx10; + defm _BOTHEN : MUBUF_Real_gfx10; if !not(isTFE) then { - def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10(NAME#"_LDS_OFFSET")>; - def _LDS_OFFEN_gfx10 : MUBUF_Real_gfx10(NAME#"_LDS_OFFEN")>; - def _LDS_IDXEN_gfx10 : MUBUF_Real_gfx10(NAME#"_LDS_IDXEN")>; - def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10(NAME#"_LDS_BOTHEN")>; + defm _LDS_OFFSET : MUBUF_Real_gfx10; + defm _LDS_OFFEN : MUBUF_Real_gfx10; + defm _LDS_IDXEN : MUBUF_Real_gfx10; + defm _LDS_BOTHEN : MUBUF_Real_gfx10; } } multiclass MUBUF_Real_Atomics_RTN_gfx10 op> { - def _BOTHEN_RTN_gfx10 : - MUBUF_Real_gfx10(NAME#"_BOTHEN_RTN")>; - def _IDXEN_RTN_gfx10 : - MUBUF_Real_gfx10(NAME#"_IDXEN_RTN")>; - def _OFFEN_RTN_gfx10 : - MUBUF_Real_gfx10(NAME#"_OFFEN_RTN")>; - def _OFFSET_RTN_gfx10 : - MUBUF_Real_gfx10(NAME#"_OFFSET_RTN")>; + defm _BOTHEN_RTN : MUBUF_Real_gfx10; + defm _IDXEN_RTN : MUBUF_Real_gfx10; + defm _OFFEN_RTN : MUBUF_Real_gfx10; + defm _OFFSET_RTN : MUBUF_Real_gfx10; } multiclass MUBUF_Real_Atomics_gfx10 op> : MUBUF_Real_Atomics_RTN_gfx10 { - def _BOTHEN_gfx10 : - MUBUF_Real_gfx10(NAME#"_BOTHEN")>; - def _IDXEN_gfx10 : - MUBUF_Real_gfx10(NAME#"_IDXEN")>; - def _OFFEN_gfx10 : - MUBUF_Real_gfx10(NAME#"_OFFEN")>; - def _OFFSET_gfx10 : - MUBUF_Real_gfx10(NAME#"_OFFSET")>; + defm _BOTHEN : MUBUF_Real_gfx10; + defm _IDXEN : MUBUF_Real_gfx10; + defm _OFFEN : MUBUF_Real_gfx10; + defm _OFFSET : MUBUF_Real_gfx10; } defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>; @@ -2720,78 +2724,51 @@ defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>; defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>; defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>; -def BUFFER_GL0_INV_gfx10 : - MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>; -def BUFFER_GL1_INV_gfx10 : - MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>; +defm BUFFER_GL0_INV : MUBUF_Real_gfx10<0x071>; +defm BUFFER_GL1_INV : MUBUF_Real_gfx10<0x072>; //===----------------------------------------------------------------------===// // MUBUF - GFX6, GFX7, GFX10. //===----------------------------------------------------------------------===// -multiclass MUBUF_Real_gfx6 op> { - let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in - def _gfx6 : MUBUF_Real_gfx6_gfx7(NAME)>; -} - -multiclass MUBUF_Real_gfx7 op> { - let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in - def _gfx7 : MUBUF_Real_gfx6_gfx7(NAME)>; -} - multiclass MUBUF_Real_AllAddr_Helper_gfx6_gfx7 op> { - def _ADDR64_gfx6_gfx7 : - MUBUF_Real_gfx6_gfx7(NAME#"_ADDR64")>; - def _BOTHEN_gfx6_gfx7 : - MUBUF_Real_gfx6_gfx7(NAME#"_BOTHEN")>; - def _IDXEN_gfx6_gfx7 : - MUBUF_Real_gfx6_gfx7(NAME#"_IDXEN")>; - def _OFFEN_gfx6_gfx7 : - MUBUF_Real_gfx6_gfx7(NAME#"_OFFEN")>; - def _OFFSET_gfx6_gfx7 : - MUBUF_Real_gfx6_gfx7(NAME#"_OFFSET")>; + defm _ADDR64 : MUBUF_Real_gfx6_gfx7; + defm _BOTHEN : MUBUF_Real_gfx6_gfx7; + defm _IDXEN : MUBUF_Real_gfx6_gfx7; + defm _OFFEN : MUBUF_Real_gfx6_gfx7; + defm _OFFSET : MUBUF_Real_gfx6_gfx7; } multiclass MUBUF_Real_AllAddr_gfx6_gfx7 op> { defm NAME : MUBUF_Real_AllAddr_Helper_gfx6_gfx7; defm _TFE : MUBUF_Real_AllAddr_Helper_gfx6_gfx7; } multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7 op, bit isTFE = 0> { - def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7(NAME#"_OFFSET")>; - def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7(NAME#"_ADDR64")>; - def _OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7(NAME#"_OFFEN")>; - def _IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7(NAME#"_IDXEN")>; - def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7(NAME#"_BOTHEN")>; + defm _OFFSET : MUBUF_Real_gfx6_gfx7; + defm _ADDR64 : MUBUF_Real_gfx6_gfx7; + defm _OFFEN : MUBUF_Real_gfx6_gfx7; + defm _IDXEN : MUBUF_Real_gfx6_gfx7; + defm _BOTHEN : MUBUF_Real_gfx6_gfx7; if !not(isTFE) then { - def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7(NAME#"_LDS_OFFSET")>; - def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7(NAME#"_LDS_ADDR64")>; - def _LDS_OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7(NAME#"_LDS_OFFEN")>; - def _LDS_IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7(NAME#"_LDS_IDXEN")>; - def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7(NAME#"_LDS_BOTHEN")>; + defm _LDS_OFFSET : MUBUF_Real_gfx6_gfx7; + defm _LDS_ADDR64 : MUBUF_Real_gfx6_gfx7; + defm _LDS_OFFEN : MUBUF_Real_gfx6_gfx7; + defm _LDS_IDXEN : MUBUF_Real_gfx6_gfx7; + defm _LDS_BOTHEN : MUBUF_Real_gfx6_gfx7; } } multiclass MUBUF_Real_Atomics_gfx6_gfx7 op> { - def _ADDR64_gfx6_gfx7 : - MUBUF_Real_gfx6_gfx7(NAME#"_ADDR64")>; - def _BOTHEN_gfx6_gfx7 : - MUBUF_Real_gfx6_gfx7(NAME#"_BOTHEN")>; - def _IDXEN_gfx6_gfx7 : - MUBUF_Real_gfx6_gfx7(NAME#"_IDXEN")>; - def _OFFEN_gfx6_gfx7 : - MUBUF_Real_gfx6_gfx7(NAME#"_OFFEN")>; - def _OFFSET_gfx6_gfx7 : - MUBUF_Real_gfx6_gfx7(NAME#"_OFFSET")>; - - def _ADDR64_RTN_gfx6_gfx7 : - MUBUF_Real_gfx6_gfx7(NAME#"_ADDR64_RTN")>; - def _BOTHEN_RTN_gfx6_gfx7 : - MUBUF_Real_gfx6_gfx7(NAME#"_BOTHEN_RTN")>; - def _IDXEN_RTN_gfx6_gfx7 : - MUBUF_Real_gfx6_gfx7(NAME#"_IDXEN_RTN")>; - def _OFFEN_RTN_gfx6_gfx7 : - MUBUF_Real_gfx6_gfx7(NAME#"_OFFEN_RTN")>; - def _OFFSET_RTN_gfx6_gfx7 : - MUBUF_Real_gfx6_gfx7(NAME#"_OFFSET_RTN")>; + defm _ADDR64 : MUBUF_Real_gfx6_gfx7; + defm _BOTHEN : MUBUF_Real_gfx6_gfx7; + defm _IDXEN : MUBUF_Real_gfx6_gfx7; + defm _OFFEN : MUBUF_Real_gfx6_gfx7; + defm _OFFSET : MUBUF_Real_gfx6_gfx7; + + defm _ADDR64_RTN : MUBUF_Real_gfx6_gfx7; + defm _BOTHEN_RTN : MUBUF_Real_gfx6_gfx7; + defm _IDXEN_RTN : MUBUF_Real_gfx6_gfx7; + defm _OFFEN_RTN : MUBUF_Real_gfx6_gfx7; + defm _OFFSET_RTN : MUBUF_Real_gfx6_gfx7; } multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10 op> : @@ -2874,7 +2851,7 @@ defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_gfx10<0x034>; defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>; defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>; -def BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>; +defm BUFFER_WBINVL1 : MUBUF_Real_gfx6_gfx7<0x071>; //===----------------------------------------------------------------------===// // Base ENC_MTBUF for GFX6, GFX7, GFX10, GFX11. @@ -2892,18 +2869,19 @@ class Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11 op, MTBUF_Pseudo ps, - string real_name = ps.Mnemonic> : - Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11 { - let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); - let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); - let Inst{18-15} = op; - let Inst{25-19} = format; - let Inst{53} = ps.tfe; - let Inst{54} = ps.offen; - let Inst{55} = ps.idxen; - let AssemblerPredicate = isGFX11Only; - let DecoderNamespace = "GFX11"; +multiclass MTBUF_Real_gfx11 op, string real_name> { + defvar ps = !cast(NAME); + def _gfx11 : Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11 { + let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); + let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); + let Inst{18-15} = op; + let Inst{25-19} = format; + let Inst{53} = ps.tfe; + let Inst{54} = ps.offen; + let Inst{55} = ps.idxen; + let AssemblerPredicate = isGFX11Only; + let DecoderNamespace = "GFX11"; + } } class Base_MTBUF_Real_gfx6_gfx7_gfx10 op, MTBUF_Pseudo ps, int ef> : @@ -2920,23 +2898,15 @@ class Base_MTBUF_Real_gfx6_gfx7_gfx10 op, MTBUF_Pseudo ps, int ef> : //===----------------------------------------------------------------------===// multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl op, string real_name> { - def _BOTHEN_gfx11 : - MTBUF_Real_gfx11(NAME#"_BOTHEN"), real_name>; - def _IDXEN_gfx11 : - MTBUF_Real_gfx11(NAME#"_IDXEN"), real_name>; - def _OFFEN_gfx11 : - MTBUF_Real_gfx11(NAME#"_OFFEN"), real_name>; - def _OFFSET_gfx11 : - MTBUF_Real_gfx11(NAME#"_OFFSET"), real_name>; - - def _BOTHEN_gfx12 : - VBUFFER_MTBUF_Real_gfx12(NAME#"_VBUFFER_BOTHEN"), real_name>; - def _IDXEN_gfx12 : - VBUFFER_MTBUF_Real_gfx12(NAME#"_VBUFFER_IDXEN"), real_name>; - def _OFFEN_gfx12 : - VBUFFER_MTBUF_Real_gfx12(NAME#"_VBUFFER_OFFEN"), real_name>; - def _OFFSET_gfx12 : - VBUFFER_MTBUF_Real_gfx12(NAME#"_VBUFFER_OFFSET"), real_name>; + defm _BOTHEN : MTBUF_Real_gfx11; + defm _IDXEN : MTBUF_Real_gfx11; + defm _OFFEN : MTBUF_Real_gfx11; + defm _OFFSET : MTBUF_Real_gfx11; + + defm _VBUFFER_BOTHEN : VBUFFER_MTBUF_Real_gfx12; + defm _VBUFFER_IDXEN : VBUFFER_MTBUF_Real_gfx12; + defm _VBUFFER_OFFEN : VBUFFER_MTBUF_Real_gfx12; + defm _VBUFFER_OFFSET : VBUFFER_MTBUF_Real_gfx12; } multiclass MTBUF_Real_AllAddr_gfx11_gfx12 op> @@ -2970,24 +2940,22 @@ defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12<0x007>; // MTBUF - GFX10. //===----------------------------------------------------------------------===// -class MTBUF_Real_gfx10 op, MTBUF_Pseudo ps> : - Base_MTBUF_Real_gfx6_gfx7_gfx10 { - let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); - let Inst{25-19} = format; - let Inst{53} = op{3}; - let AssemblerPredicate = isGFX10Only; - let DecoderNamespace = "GFX10"; +multiclass MTBUF_Real_gfx10 op> { + defvar ps = !cast(NAME); + def _gfx10 : Base_MTBUF_Real_gfx6_gfx7_gfx10 { + let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); + let Inst{25-19} = format; + let Inst{53} = op{3}; + let AssemblerPredicate = isGFX10Only; + let DecoderNamespace = "GFX10"; + } } multiclass MTBUF_Real_AllAddr_gfx10 op> { - def _BOTHEN_gfx10 : - MTBUF_Real_gfx10(NAME#"_BOTHEN")>; - def _IDXEN_gfx10 : - MTBUF_Real_gfx10(NAME#"_IDXEN")>; - def _OFFEN_gfx10 : - MTBUF_Real_gfx10(NAME#"_OFFEN")>; - def _OFFSET_gfx10 : - MTBUF_Real_gfx10(NAME#"_OFFSET")>; + defm _BOTHEN : MTBUF_Real_gfx10; + defm _IDXEN : MTBUF_Real_gfx10; + defm _OFFEN : MTBUF_Real_gfx10; + defm _OFFSET : MTBUF_Real_gfx10; } defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>; @@ -3003,26 +2971,23 @@ defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>; // MTBUF - GFX6, GFX7, GFX10. //===----------------------------------------------------------------------===// -class MTBUF_Real_gfx6_gfx7 op, MTBUF_Pseudo ps> : - Base_MTBUF_Real_gfx6_gfx7_gfx10 { - let Inst{15} = ps.addr64; - let Inst{22-19} = dfmt; - let Inst{25-23} = nfmt; - let AssemblerPredicate = isGFX6GFX7; - let DecoderNamespace = "GFX6GFX7"; +multiclass MTBUF_Real_gfx6_gfx7 op> { + defvar ps = !cast(NAME); + def _gfx6_gfx7 : Base_MTBUF_Real_gfx6_gfx7_gfx10 { + let Inst{15} = ps.addr64; + let Inst{22-19} = dfmt; + let Inst{25-23} = nfmt; + let AssemblerPredicate = isGFX6GFX7; + let DecoderNamespace = "GFX6GFX7"; + } } multiclass MTBUF_Real_AllAddr_gfx6_gfx7 op> { - def _ADDR64_gfx6_gfx7 : - MTBUF_Real_gfx6_gfx7(NAME#"_ADDR64")>; - def _BOTHEN_gfx6_gfx7 : - MTBUF_Real_gfx6_gfx7(NAME#"_BOTHEN")>; - def _IDXEN_gfx6_gfx7 : - MTBUF_Real_gfx6_gfx7(NAME#"_IDXEN")>; - def _OFFEN_gfx6_gfx7 : - MTBUF_Real_gfx6_gfx7(NAME#"_OFFEN")>; - def _OFFSET_gfx6_gfx7 : - MTBUF_Real_gfx6_gfx7(NAME#"_OFFSET")>; + defm _ADDR64 : MTBUF_Real_gfx6_gfx7; + defm _BOTHEN : MTBUF_Real_gfx6_gfx7; + defm _IDXEN : MTBUF_Real_gfx6_gfx7; + defm _OFFEN : MTBUF_Real_gfx6_gfx7; + defm _OFFSET : MTBUF_Real_gfx6_gfx7; } multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10 op> : @@ -3062,23 +3027,28 @@ class MUBUF_Real_Base_vi op, MUBUF_Pseudo ps, int Enc, let Inst{63-56} = !if(ps.has_soffset, soffset, ?); } -class MUBUF_Real_vi op, MUBUF_Pseudo ps, bit has_sccb = ps.has_sccb> : - MUBUF_Real_Base_vi { - let AssemblerPredicate = isGFX8GFX9NotGFX90A; - let DecoderNamespace = "GFX8"; +multiclass MUBUF_Real_vi op, + bit has_sccb = !cast(NAME).has_sccb> { + defvar ps = !cast(NAME); + def _vi : MUBUF_Real_Base_vi { + let AssemblerPredicate = isGFX8GFX9NotGFX90A; + let DecoderNamespace = "GFX8"; - let Inst{55} = ps.tfe; + let Inst{55} = ps.tfe; + } } -class MUBUF_Real_gfx90a op, MUBUF_Pseudo ps, - bit has_sccb = ps.has_sccb> : - MUBUF_Real_Base_vi { - let AssemblerPredicate = isGFX90APlus; - let DecoderNamespace = "GFX90A"; - let AsmString = ps.Mnemonic # !subst("$sccb", !if(has_sccb, "$sccb",""), - ps.AsmOperands); +multiclass MUBUF_Real_gfx90a op, + bit has_sccb = !cast(NAME).has_sccb> { + defvar ps = !cast(NAME); + def _gfx90a : MUBUF_Real_Base_vi { + let AssemblerPredicate = isGFX90APlus; + let DecoderNamespace = "GFX90A"; + let AsmString = ps.Mnemonic # !subst("$sccb", !if(has_sccb, "$sccb",""), + ps.AsmOperands); - let Inst{55} = acc; + let Inst{55} = acc; + } } class MUBUF_Real_gfx940 op, MUBUF_Pseudo ps> : @@ -3090,28 +3060,27 @@ class MUBUF_Real_gfx940 op, MUBUF_Pseudo ps> : let Inst{55} = acc; } -multiclass MUBUF_Real_vi_gfx90a op, MUBUF_Pseudo ps, bit isTFE = 0> { - def _vi : MUBUF_Real_vi; +multiclass MUBUF_Real_vi_gfx90a op, bit isTFE = 0> : MUBUF_Real_vi { + defvar ps = !cast(NAME); if !not(isTFE) then { if !not(ps.FPAtomic) then - def _gfx90a : MUBUF_Real_gfx90a; + defm NAME : MUBUF_Real_gfx90a; } if ps.FPAtomic then { - def _gfx90a : MUBUF_Real_gfx90a { - let SubtargetPredicate = isGFX90AOnly; - let AssemblerPredicate = isGFX90AOnly; - } + let SubtargetPredicate = isGFX90AOnly, + AssemblerPredicate = isGFX90AOnly in + defm NAME : MUBUF_Real_gfx90a; def _gfx940 : MUBUF_Real_gfx940; } } multiclass MUBUF_Real_AllAddr_Helper_vi op, bit isTFE = 0> { - defm _OFFSET : MUBUF_Real_vi_gfx90a (NAME#"_OFFSET"), isTFE>; - defm _OFFEN : MUBUF_Real_vi_gfx90a (NAME#"_OFFEN"), isTFE>; - defm _IDXEN : MUBUF_Real_vi_gfx90a (NAME#"_IDXEN"), isTFE>; - defm _BOTHEN : MUBUF_Real_vi_gfx90a (NAME#"_BOTHEN"), isTFE>; + defm _OFFSET : MUBUF_Real_vi_gfx90a ; + defm _OFFEN : MUBUF_Real_vi_gfx90a ; + defm _IDXEN : MUBUF_Real_vi_gfx90a ; + defm _BOTHEN : MUBUF_Real_vi_gfx90a ; } multiclass MUBUF_Real_AllAddr_vi op, bit hasTFE = 1> { @@ -3121,26 +3090,26 @@ multiclass MUBUF_Real_AllAddr_vi op, bit hasTFE = 1> { } multiclass MUBUF_Real_AllAddr_Lds_Helper_vi op, bit isTFE = 0> { - def _OFFSET_vi : MUBUF_Real_vi (NAME#"_OFFSET")>; - def _OFFEN_vi : MUBUF_Real_vi (NAME#"_OFFEN")>; - def _IDXEN_vi : MUBUF_Real_vi (NAME#"_IDXEN")>; - def _BOTHEN_vi : MUBUF_Real_vi (NAME#"_BOTHEN")>; + defm _OFFSET : MUBUF_Real_vi ; + defm _OFFEN : MUBUF_Real_vi ; + defm _IDXEN : MUBUF_Real_vi ; + defm _BOTHEN : MUBUF_Real_vi ; if !not(isTFE) then { - def _LDS_OFFSET_vi : MUBUF_Real_vi (NAME#"_LDS_OFFSET")>; - def _LDS_OFFEN_vi : MUBUF_Real_vi (NAME#"_LDS_OFFEN")>; - def _LDS_IDXEN_vi : MUBUF_Real_vi (NAME#"_LDS_IDXEN")>; - def _LDS_BOTHEN_vi : MUBUF_Real_vi (NAME#"_LDS_BOTHEN")>; - - def _OFFSET_gfx90a : MUBUF_Real_gfx90a (NAME#"_OFFSET")>; - def _OFFEN_gfx90a : MUBUF_Real_gfx90a (NAME#"_OFFEN")>; - def _IDXEN_gfx90a : MUBUF_Real_gfx90a (NAME#"_IDXEN")>; - def _BOTHEN_gfx90a : MUBUF_Real_gfx90a (NAME#"_BOTHEN")>; - - def _LDS_OFFSET_gfx90a : MUBUF_Real_gfx90a (NAME#"_LDS_OFFSET")>; - def _LDS_OFFEN_gfx90a : MUBUF_Real_gfx90a (NAME#"_LDS_OFFEN")>; - def _LDS_IDXEN_gfx90a : MUBUF_Real_gfx90a (NAME#"_LDS_IDXEN")>; - def _LDS_BOTHEN_gfx90a : MUBUF_Real_gfx90a (NAME#"_LDS_BOTHEN")>; + defm _LDS_OFFSET : MUBUF_Real_vi ; + defm _LDS_OFFEN : MUBUF_Real_vi ; + defm _LDS_IDXEN : MUBUF_Real_vi ; + defm _LDS_BOTHEN : MUBUF_Real_vi ; + + defm _OFFSET : MUBUF_Real_gfx90a ; + defm _OFFEN : MUBUF_Real_gfx90a ; + defm _IDXEN : MUBUF_Real_gfx90a ; + defm _BOTHEN : MUBUF_Real_gfx90a ; + + defm _LDS_OFFSET : MUBUF_Real_gfx90a ; + defm _LDS_OFFEN : MUBUF_Real_gfx90a ; + defm _LDS_IDXEN : MUBUF_Real_gfx90a ; + defm _LDS_BOTHEN : MUBUF_Real_gfx90a ; } } @@ -3149,33 +3118,35 @@ multiclass MUBUF_Real_AllAddr_Lds_vi op> { defm _TFE : MUBUF_Real_AllAddr_Lds_Helper_vi; } -class MUBUF_Real_gfx80 op, MUBUF_Pseudo ps> : - MUBUF_Real, - Enc64, - SIMCInstr { - let AssemblerPredicate=HasUnpackedD16VMem; - let DecoderNamespace="GFX80_UNPACKED"; - - let Inst{11-0} = !if(ps.has_offset, offset, ?); - let Inst{12} = ps.offen; - let Inst{13} = ps.idxen; - let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); - let Inst{16} = ps.lds; - let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); - let Inst{24-18} = op; - let Inst{31-26} = 0x38; //encoding - let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); - let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); - let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); - let Inst{55} = ps.tfe; - let Inst{63-56} = !if(ps.has_soffset, soffset, ?); +multiclass MUBUF_Real_gfx80 op> { + defvar ps = !cast(NAME); + def _gfx80 : MUBUF_Real, + Enc64, + SIMCInstr { + let AssemblerPredicate=HasUnpackedD16VMem; + let DecoderNamespace="GFX80_UNPACKED"; + + let Inst{11-0} = !if(ps.has_offset, offset, ?); + let Inst{12} = ps.offen; + let Inst{13} = ps.idxen; + let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); + let Inst{16} = ps.lds; + let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); + let Inst{24-18} = op; + let Inst{31-26} = 0x38; //encoding + let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); + let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); + let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); + let Inst{55} = ps.tfe; + let Inst{63-56} = !if(ps.has_soffset, soffset, ?); + } } multiclass MUBUF_Real_AllAddr_Helper_gfx80 op> { - def _OFFSET_gfx80 : MUBUF_Real_gfx80 (NAME#"_OFFSET")>; - def _OFFEN_gfx80 : MUBUF_Real_gfx80 (NAME#"_OFFEN")>; - def _IDXEN_gfx80 : MUBUF_Real_gfx80 (NAME#"_IDXEN")>; - def _BOTHEN_gfx80 : MUBUF_Real_gfx80 (NAME#"_BOTHEN")>; + defm _OFFSET : MUBUF_Real_gfx80 ; + defm _OFFEN : MUBUF_Real_gfx80 ; + defm _IDXEN : MUBUF_Real_gfx80 ; + defm _BOTHEN : MUBUF_Real_gfx80 ; } multiclass MUBUF_Real_AllAddr_gfx80 op> { @@ -3185,10 +3156,10 @@ multiclass MUBUF_Real_AllAddr_gfx80 op> { multiclass MUBUF_Real_Atomic_vi op> : MUBUF_Real_AllAddr_vi { - defm _OFFSET_RTN : MUBUF_Real_vi_gfx90a (NAME#"_OFFSET_RTN")>; - defm _OFFEN_RTN : MUBUF_Real_vi_gfx90a (NAME#"_OFFEN_RTN")>; - defm _IDXEN_RTN : MUBUF_Real_vi_gfx90a (NAME#"_IDXEN_RTN")>; - defm _BOTHEN_RTN : MUBUF_Real_vi_gfx90a (NAME#"_BOTHEN_RTN")>; + defm _OFFSET_RTN : MUBUF_Real_vi_gfx90a ; + defm _OFFEN_RTN : MUBUF_Real_vi_gfx90a ; + defm _IDXEN_RTN : MUBUF_Real_vi_gfx90a ; + defm _BOTHEN_RTN : MUBUF_Real_vi_gfx90a ; } defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; @@ -3274,11 +3245,11 @@ defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; -defm BUFFER_STORE_LDS_DWORD : MUBUF_Real_vi_gfx90a <0x3d, BUFFER_STORE_LDS_DWORD>; +defm BUFFER_STORE_LDS_DWORD : MUBUF_Real_vi_gfx90a <0x3d>; let AssemblerPredicate = isGFX8GFX9 in { -def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; -def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; +defm BUFFER_WBINVL1 : MUBUF_Real_vi <0x3e>; +defm BUFFER_WBINVL1_VOL : MUBUF_Real_vi <0x3f>; } // End AssemblerPredicate = isGFX8GFX9 @@ -3294,12 +3265,11 @@ let SubtargetPredicate = isGFX90APlus in { defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Real_Atomic_vi<0x51>; } // End SubtargetPredicate = isGFX90APlus -def BUFFER_WBL2_gfx90a : MUBUF_Real_gfx90a<0x28, BUFFER_WBL2> { - let AsmString = BUFFER_WBL2.Mnemonic; // drop flags - let AssemblerPredicate = isGFX90AOnly; - let SubtargetPredicate = isGFX90AOnly; -} -def BUFFER_INVL2_gfx90a : MUBUF_Real_gfx90a<0x29, BUFFER_INVL2>; +let AsmString = BUFFER_WBL2.Mnemonic, // drop flags + AssemblerPredicate = isGFX90AOnly, + SubtargetPredicate = isGFX90AOnly in +defm BUFFER_WBL2 : MUBUF_Real_gfx90a<0x28>; +defm BUFFER_INVL2 : MUBUF_Real_gfx90a<0x29>; let SubtargetPredicate = isGFX940Plus in { def BUFFER_WBL2_gfx940 : MUBUF_Real_gfx940<0x28, BUFFER_WBL2>; @@ -3345,46 +3315,49 @@ class MTBUF_Real_gfx90a op, MTBUF_Pseudo ps> : let Inst{55} = acc; } -multiclass MTBUF_Real_vi_gfx90a op, MTBUF_Pseudo ps> { +multiclass MTBUF_Real_vi_gfx90a op> { + defvar ps = !cast(NAME); def _vi : MTBUF_Real_vi; def _gfx90a : MTBUF_Real_gfx90a; } multiclass MTBUF_Real_AllAddr_vi op> { - defm _OFFSET : MTBUF_Real_vi_gfx90a (NAME#"_OFFSET")>; - defm _OFFEN : MTBUF_Real_vi_gfx90a (NAME#"_OFFEN")>; - defm _IDXEN : MTBUF_Real_vi_gfx90a (NAME#"_IDXEN")>; - defm _BOTHEN : MTBUF_Real_vi_gfx90a (NAME#"_BOTHEN")>; -} - -class MTBUF_Real_gfx80 op, MTBUF_Pseudo ps> : - MTBUF_Real, - Enc64, - SIMCInstr { - let AssemblerPredicate=HasUnpackedD16VMem; - let DecoderNamespace="GFX80_UNPACKED"; - - let Inst{11-0} = !if(ps.has_offset, offset, ?); - let Inst{12} = ps.offen; - let Inst{13} = ps.idxen; - let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); - let Inst{18-15} = op; - let Inst{22-19} = dfmt; - let Inst{25-23} = nfmt; - let Inst{31-26} = 0x3a; //encoding - let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); - let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); - let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); - let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); - let Inst{55} = ps.tfe; - let Inst{63-56} = !if(ps.has_soffset, soffset, ?); + defm _OFFSET : MTBUF_Real_vi_gfx90a ; + defm _OFFEN : MTBUF_Real_vi_gfx90a ; + defm _IDXEN : MTBUF_Real_vi_gfx90a ; + defm _BOTHEN : MTBUF_Real_vi_gfx90a ; +} + +multiclass MTBUF_Real_gfx80 op> { + defvar ps = !cast(NAME); + def _gfx80 : MTBUF_Real, + Enc64, + SIMCInstr { + let AssemblerPredicate=HasUnpackedD16VMem; + let DecoderNamespace="GFX80_UNPACKED"; + + let Inst{11-0} = !if(ps.has_offset, offset, ?); + let Inst{12} = ps.offen; + let Inst{13} = ps.idxen; + let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); + let Inst{18-15} = op; + let Inst{22-19} = dfmt; + let Inst{25-23} = nfmt; + let Inst{31-26} = 0x3a; //encoding + let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); + let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); + let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); + let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); + let Inst{55} = ps.tfe; + let Inst{63-56} = !if(ps.has_soffset, soffset, ?); + } } multiclass MTBUF_Real_AllAddr_gfx80 op> { - def _OFFSET_gfx80 : MTBUF_Real_gfx80 (NAME#"_OFFSET")>; - def _OFFEN_gfx80 : MTBUF_Real_gfx80 (NAME#"_OFFEN")>; - def _IDXEN_gfx80 : MTBUF_Real_gfx80 (NAME#"_IDXEN")>; - def _BOTHEN_gfx80 : MTBUF_Real_gfx80 (NAME#"_BOTHEN")>; + defm _OFFSET : MTBUF_Real_gfx80 ; + defm _OFFEN : MTBUF_Real_gfx80 ; + defm _IDXEN : MTBUF_Real_gfx80 ; + defm _BOTHEN : MTBUF_Real_gfx80 ; } defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; -- GitLab From a642eb89bdaf10c6b4994fc1187de27b441236ed Mon Sep 17 00:00:00 2001 From: Krystian Stasiowski Date: Tue, 5 Mar 2024 08:52:20 -0500 Subject: [PATCH 139/929] [Clang][Sema] Fix crash when using name of UnresolvedUsingValueDecl with template arguments (#83842) The following snippet causes a crash: ``` template struct A : T { using T::f; void f(); void g() { f(); // crash here } }; ``` This happens because we cast the result of `getAsTemplateNameDecl` as a `TemplateDecl` in `Sema::ClassifyName`, which we cannot do for an `UnresolvedUsingValueDecl`. This patch fixes the crash by considering a name to be that of a template if _any_ function declaration is found per [temp.names] p3.3. --- clang/docs/ReleaseNotes.rst | 2 ++ clang/lib/Sema/SemaDecl.cpp | 5 +++- clang/lib/Sema/SemaTemplate.cpp | 10 ++++--- .../unqual-unresolved-using-value.cpp | 30 +++++++++++++++++++ 4 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 clang/test/SemaTemplate/unqual-unresolved-using-value.cpp diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index c328ae18c024..de5f339cc7f0 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -305,6 +305,8 @@ Bug Fixes to C++ Support our attention by an attempt to fix in (#GH77703). Fixes (#GH83385). - Fix evaluation of some immediate calls in default arguments. Fixes (#GH80630) +- Fix a crash when an explicit template argument list is used with a name for which lookup + finds a non-template function and a dependent using declarator. Bug Fixes to AST Handling ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 3ae78748a4e4..8fcaf6ab312b 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -1110,7 +1110,9 @@ Corrected: // unqualified-id followed by a < and name lookup finds either one // or more functions or finds nothing. if (!IsFilteredTemplateName) - FilterAcceptableTemplateNames(Result); + FilterAcceptableTemplateNames(Result, + /*AllowFunctionTemplates=*/true, + /*AllowDependent=*/true); bool IsFunctionTemplate; bool IsVarTemplate; @@ -1120,6 +1122,7 @@ Corrected: Template = Context.getOverloadedTemplateName(Result.begin(), Result.end()); } else if (!Result.empty()) { + assert(!Result.isUnresolvableResult()); auto *TD = cast(getAsTemplateNameDecl( *Result.begin(), /*AllowFunctionTemplates=*/true, /*AllowDependent=*/false)); diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp index 873ea10ebe06..df20f83ac8af 100644 --- a/clang/lib/Sema/SemaTemplate.cpp +++ b/clang/lib/Sema/SemaTemplate.cpp @@ -491,18 +491,20 @@ bool Sema::LookupTemplateName(LookupResult &Found, // To keep our behavior consistent, we apply the "finds nothing" part in // all language modes, and diagnose the empty lookup in ActOnCallExpr if we // successfully form a call to an undeclared template-id. - bool AllFunctions = - getLangOpts().CPlusPlus20 && llvm::all_of(Found, [](NamedDecl *ND) { + bool AnyFunctions = + getLangOpts().CPlusPlus20 && llvm::any_of(Found, [](NamedDecl *ND) { return isa(ND->getUnderlyingDecl()); }); - if (AllFunctions || (Found.empty() && !IsDependent)) { + if (AnyFunctions || (Found.empty() && !IsDependent)) { // If lookup found any functions, or if this is a name that can only be // used for a function, then strongly assume this is a function // template-id. *ATK = (Found.empty() && Found.getLookupName().isIdentifier()) ? AssumedTemplateKind::FoundNothing : AssumedTemplateKind::FoundFunctions; - Found.clear(); + FilterAcceptableTemplateNames(Found, + /*AllowFunctionTemplates*/ true, + /*AllowDependent*/ true); return false; } } diff --git a/clang/test/SemaTemplate/unqual-unresolved-using-value.cpp b/clang/test/SemaTemplate/unqual-unresolved-using-value.cpp new file mode 100644 index 000000000000..688e7a0a10b7 --- /dev/null +++ b/clang/test/SemaTemplate/unqual-unresolved-using-value.cpp @@ -0,0 +1,30 @@ +// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify %s + +template +struct A : T { + using T::f; + using T::g; + using T::h; + + void f(); + void g(); + + void i() { + f(); + g(); // expected-error{{no member named 'g' in 'A'}} + h(); // expected-error{{expected '(' for function-style cast or type construction}} + // expected-error@-1{{expected expression}} + } +}; + +struct B { + template + void f(); + + void g(); + + template + void h(); +}; + +template struct A; // expected-note{{in instantiation of member function 'A::i' requested here}} -- GitLab From 0d8e16a0967cfd2dbcd00c7bb7a56f476c0e64fb Mon Sep 17 00:00:00 2001 From: Balazs Benics Date: Tue, 5 Mar 2024 15:02:41 +0100 Subject: [PATCH 140/929] [analyzer][NFC] Make CheckerDocumentation checker in-sync with actual checker callbacks (#83973) In PR #83677 I was surprised to see that outdated checker callback signatures are a problem. It turns out, we need the `registerChecker...` function to invoke the `Mgr.registerChecker<>()` which would instantiate the `_register` calls, that would take the address of the defined checker callbacks. Consequently, if the expected signatures mismatch, it won't compile from now on, so we have static guarantee that this issue never pops up again. Given we need the `register` call, at this point we could just hook this checker into the `debug` package and make it never registered. It shouldn't hurt anyone :) --- .../clang/StaticAnalyzer/Checkers/Checkers.td | 4 ++++ .../Checkers/CheckerDocumentation.cpp | 13 +++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td index a224b81c33a6..686e5e99f4a6 100644 --- a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td +++ b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td @@ -1654,6 +1654,10 @@ def StdCLibraryFunctionsTesterChecker : Checker<"StdCLibraryFunctionsTester">, WeakDependencies<[StdCLibraryFunctionsChecker]>, Documentation; +def CheckerDocumentationChecker : Checker<"CheckerDocumentation">, + HelpText<"Defines an empty checker callback for all possible handlers.">, + Documentation; + } // end "debug" diff --git a/clang/lib/StaticAnalyzer/Checkers/CheckerDocumentation.cpp b/clang/lib/StaticAnalyzer/Checkers/CheckerDocumentation.cpp index 0ca0c487b645..01e0bed54cc6 100644 --- a/clang/lib/StaticAnalyzer/Checkers/CheckerDocumentation.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/CheckerDocumentation.cpp @@ -137,10 +137,7 @@ public: /// (2) and (3). Post-call for the allocator is called after step (1). /// Pre-statement for the new-expression is called on step (4) when the value /// of the expression is evaluated. - /// \param NE The C++ new-expression that triggered the allocation. - /// \param Target The allocated region, casted to the class type. - void checkNewAllocator(const CXXNewExpr *NE, SVal Target, - CheckerContext &) const {} + void checkNewAllocator(const CXXAllocatorCall &, CheckerContext &) const {} /// Called on a load from and a store to a location. /// @@ -330,5 +327,13 @@ void CheckerDocumentation::checkPostStmt(const DeclStmt *DS, CheckerContext &C) const { } +void registerCheckerDocumentationChecker(CheckerManager &Mgr) { + Mgr.registerChecker(); +} + +bool shouldRegisterCheckerDocumentationChecker(const CheckerManager &) { + return false; +} + } // end namespace ento } // end namespace clang -- GitLab From 4ce737bfd6fd0aafb436eb220c3e724bfc831db4 Mon Sep 17 00:00:00 2001 From: Douglas Deslauriers <48334845+vapdrs@users.noreply.github.com> Date: Tue, 5 Mar 2024 14:07:54 +0000 Subject: [PATCH 141/929] [clang] Sequence C++20 Parenthesized List Init (#83476) Parenthesized list intializers are sequenced operations, see C++20 [decl.init]p16.5 and [decl.init]p16.6.2.2 for more details. Fixes #83474 --- clang/lib/Sema/SemaChecking.cpp | 30 +++++++++---------- .../warn-unsequenced-paren-list-init.cpp | 15 ++++++++++ 2 files changed, 29 insertions(+), 16 deletions(-) create mode 100644 clang/test/SemaCXX/warn-unsequenced-paren-list-init.cpp diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index 0d4d57db01c9..2cda1d08784e 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -17624,20 +17624,8 @@ public: return VisitExpr(CCE); // In C++11, list initializations are sequenced. - SmallVector Elts; - SequenceTree::Seq Parent = Region; - for (CXXConstructExpr::const_arg_iterator I = CCE->arg_begin(), - E = CCE->arg_end(); - I != E; ++I) { - Region = Tree.allocate(Parent); - Elts.push_back(Region); - Visit(*I); - } - - // Forget that the initializers are sequenced. - Region = Parent; - for (unsigned I = 0; I < Elts.size(); ++I) - Tree.merge(Elts[I]); + SequenceExpressionsInOrder( + llvm::ArrayRef(CCE->getArgs(), CCE->getNumArgs())); } void VisitInitListExpr(const InitListExpr *ILE) { @@ -17645,10 +17633,20 @@ public: return VisitExpr(ILE); // In C++11, list initializations are sequenced. + SequenceExpressionsInOrder(ILE->inits()); + } + + void VisitCXXParenListInitExpr(const CXXParenListInitExpr *PLIE) { + // C++20 parenthesized list initializations are sequenced. See C++20 + // [decl.init.general]p16.5 and [decl.init.general]p16.6.2.2. + SequenceExpressionsInOrder(PLIE->getInitExprs()); + } + +private: + void SequenceExpressionsInOrder(ArrayRef ExpressionList) { SmallVector Elts; SequenceTree::Seq Parent = Region; - for (unsigned I = 0; I < ILE->getNumInits(); ++I) { - const Expr *E = ILE->getInit(I); + for (const Expr *E : ExpressionList) { if (!E) continue; Region = Tree.allocate(Parent); diff --git a/clang/test/SemaCXX/warn-unsequenced-paren-list-init.cpp b/clang/test/SemaCXX/warn-unsequenced-paren-list-init.cpp new file mode 100644 index 000000000000..5aeeb45f81e2 --- /dev/null +++ b/clang/test/SemaCXX/warn-unsequenced-paren-list-init.cpp @@ -0,0 +1,15 @@ +// RUN: %clang_cc1 -fsyntax-only -std=c++20 -Wno-unused -Wunsequenced -verify %s + +struct A { + int x, y; +}; + +void test() { + int a = 0; + + A agg1( a++, a++ ); // no warning + A agg2( a++ + a, a++ ); // expected-warning {{unsequenced modification and access to 'a'}} + + int arr1[]( a++, a++ ); // no warning + int arr2[]( a++ + a, a++ ); // expected-warning {{unsequenced modification and access to 'a'}} +} -- GitLab From 4cf8b298cf1837e75243a299ddefd59e6ed80e1b Mon Sep 17 00:00:00 2001 From: bcahoon <59846893+bcahoon@users.noreply.github.com> Date: Tue, 5 Mar 2024 08:18:17 -0600 Subject: [PATCH 142/929] [AMDGPU][PromoteAlloca] Correctly handle a variable vector index (#83597) The promote alloca to vector transformation assumes that the vector index is a constant value. If it is not a constant, then either an assert occurs or the tranformation generates an incorrect index. --- .../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 32 +++++++++------- .../promote-alloca-non-constant-index.ll | 37 +++++++++++++++++++ 2 files changed, 55 insertions(+), 14 deletions(-) create mode 100644 llvm/test/CodeGen/AMDGPU/promote-alloca-non-constant-index.ll diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp index c1b244f50d93..b1b15e9915ae 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp @@ -401,14 +401,16 @@ static Value *promoteAllocaUserToVector( // We're loading the full vector. Type *AccessTy = Inst->getType(); TypeSize AccessSize = DL.getTypeStoreSize(AccessTy); - if (AccessSize == VecStoreSize && cast(Index)->isZeroValue()) { - if (AccessTy->isPtrOrPtrVectorTy()) - CurVal = CreateTempPtrIntCast(CurVal, AccessTy); - else if (CurVal->getType()->isPtrOrPtrVectorTy()) - CurVal = CreateTempPtrIntCast(CurVal, CurVal->getType()); - Value *NewVal = Builder.CreateBitOrPointerCast(CurVal, AccessTy); - Inst->replaceAllUsesWith(NewVal); - return nullptr; + if (Constant *CI = dyn_cast(Index)) { + if (CI->isZeroValue() && AccessSize == VecStoreSize) { + if (AccessTy->isPtrOrPtrVectorTy()) + CurVal = CreateTempPtrIntCast(CurVal, AccessTy); + else if (CurVal->getType()->isPtrOrPtrVectorTy()) + CurVal = CreateTempPtrIntCast(CurVal, CurVal->getType()); + Value *NewVal = Builder.CreateBitOrPointerCast(CurVal, AccessTy); + Inst->replaceAllUsesWith(NewVal); + return nullptr; + } } // Loading a subvector. @@ -456,12 +458,14 @@ static Value *promoteAllocaUserToVector( // We're storing the full vector, we can handle this without knowing CurVal. Type *AccessTy = Val->getType(); TypeSize AccessSize = DL.getTypeStoreSize(AccessTy); - if (AccessSize == VecStoreSize && cast(Index)->isZeroValue()) { - if (AccessTy->isPtrOrPtrVectorTy()) - Val = CreateTempPtrIntCast(Val, AccessTy); - else if (VectorTy->isPtrOrPtrVectorTy()) - Val = CreateTempPtrIntCast(Val, VectorTy); - return Builder.CreateBitOrPointerCast(Val, VectorTy); + if (Constant *CI = dyn_cast(Index)) { + if (CI->isZeroValue() && AccessSize == VecStoreSize) { + if (AccessTy->isPtrOrPtrVectorTy()) + Val = CreateTempPtrIntCast(Val, AccessTy); + else if (VectorTy->isPtrOrPtrVectorTy()) + Val = CreateTempPtrIntCast(Val, VectorTy); + return Builder.CreateBitOrPointerCast(Val, VectorTy); + } } // Storing a subvector. diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-non-constant-index.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-non-constant-index.ll new file mode 100644 index 000000000000..272a9ebe536c --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-non-constant-index.ll @@ -0,0 +1,37 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -mtriple=amdgcn-unknown-amdhsa -passes=amdgpu-promote-alloca < %s | FileCheck %s + +; Check that promoting an alloca to a vector form works correctly when a variable +; vector index is used. + +define amdgpu_kernel void @non_constant_index(i32 %arg) { +; CHECK-LABEL: define amdgpu_kernel void @non_constant_index( +; CHECK-SAME: i32 [[ARG:%.*]]) { +; CHECK-NEXT: bb: +; CHECK-NEXT: br label [[BB1:%.*]] +; CHECK: bb1: +; CHECK-NEXT: br label [[BB1]] +; CHECK: bb2: +; CHECK-NEXT: br label [[BB3:%.*]] +; CHECK: bb3: +; CHECK-NEXT: [[PROMOTEALLOCA:%.*]] = phi <2 x float> [ [[TMP2:%.*]], [[BB3]] ], [ undef, [[BB2:%.*]] ] +; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x float> [[PROMOTEALLOCA]], float 0.000000e+00, i32 [[ARG]] +; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[ARG]], 1 +; CHECK-NEXT: [[TMP2]] = insertelement <2 x float> [[TMP0]], float 0.000000e+00, i32 [[TMP1]] +; CHECK-NEXT: br label [[BB3]] +; +bb: + %i = alloca [2 x float], align 4, addrspace(5) + br label %bb1 + +bb1: + br label %bb1 + +bb2: + br label %bb3 + +bb3: + %i4 = getelementptr float, ptr addrspace(5) %i, i32 %arg + store <2 x float> zeroinitializer, ptr addrspace(5) %i4, align 8 + br label %bb3 +} -- GitLab From d9b435c24ddddcc8148fd97b42f6bb1124e52307 Mon Sep 17 00:00:00 2001 From: Weining Lu Date: Tue, 5 Mar 2024 22:01:07 +0800 Subject: [PATCH 143/929] [lld][test] Fix sanitizer buildbot failure Buildbot failure: https://lab.llvm.org/buildbot/#/builders/5/builds/41530/steps/9/logs/stdio --- lld/test/ELF/loongarch-reloc-leb128.s | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lld/test/ELF/loongarch-reloc-leb128.s b/lld/test/ELF/loongarch-reloc-leb128.s index 9e6f221e62b6..2dd327d1564e 100644 --- a/lld/test/ELF/loongarch-reloc-leb128.s +++ b/lld/test/ELF/loongarch-reloc-leb128.s @@ -99,4 +99,4 @@ w2: .reloc ., R_LARCH_ADD_ULEB128, w2 .reloc ., R_LARCH_SUB_ULEB128, w1 .fill 10, 1, 0x80 -.byte 0 +.byte 1 -- GitLab From a1a590ef120c273b5c278f2412c4c4eac5280f23 Mon Sep 17 00:00:00 2001 From: Yingwei Zheng Date: Tue, 5 Mar 2024 22:34:04 +0800 Subject: [PATCH 144/929] [InstCombine] Fix miscompilation in PR83947 (#83993) https://github.com/llvm/llvm-project/blob/762f762504967efbe159db5c737154b989afc9bb/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp#L394-L407 Comment from @topperc: > This transforms assumes the mask is a non-zero splat. We only know its a splat and not provably all 0s. The mask is a constexpr that includes the address of the global variable. We can't resolve the constant expression to an exact value. Fixes #83947. --- llvm/include/llvm/Analysis/VectorUtils.h | 5 ++ llvm/lib/Analysis/VectorUtils.cpp | 25 +++++++ .../InstCombine/InstCombineCalls.cpp | 13 ++-- llvm/test/Transforms/InstCombine/pr83947.ll | 67 +++++++++++++++++++ 4 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 llvm/test/Transforms/InstCombine/pr83947.ll diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h index 7a92e62b53c5..c6eb66cc9660 100644 --- a/llvm/include/llvm/Analysis/VectorUtils.h +++ b/llvm/include/llvm/Analysis/VectorUtils.h @@ -406,6 +406,11 @@ bool maskIsAllZeroOrUndef(Value *Mask); /// lanes can be assumed active. bool maskIsAllOneOrUndef(Value *Mask); +/// Given a mask vector of i1, Return true if any of the elements of this +/// predicate mask are known to be true or undef. That is, return true if at +/// least one lane can be assumed active. +bool maskContainsAllOneOrUndef(Value *Mask); + /// Given a mask vector of the form , return an APInt (of bitwidth Y) /// for each lane which may be active. APInt possiblyDemandedEltsInMask(Value *Mask); diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp index 73facc76a92b..bf7bc0ba84a0 100644 --- a/llvm/lib/Analysis/VectorUtils.cpp +++ b/llvm/lib/Analysis/VectorUtils.cpp @@ -1012,6 +1012,31 @@ bool llvm::maskIsAllOneOrUndef(Value *Mask) { return true; } +bool llvm::maskContainsAllOneOrUndef(Value *Mask) { + assert(isa(Mask->getType()) && + isa(Mask->getType()->getScalarType()) && + cast(Mask->getType()->getScalarType())->getBitWidth() == + 1 && + "Mask must be a vector of i1"); + + auto *ConstMask = dyn_cast(Mask); + if (!ConstMask) + return false; + if (ConstMask->isAllOnesValue() || isa(ConstMask)) + return true; + if (isa(ConstMask->getType())) + return false; + for (unsigned + I = 0, + E = cast(ConstMask->getType())->getNumElements(); + I != E; ++I) { + if (auto *MaskElt = ConstMask->getAggregateElement(I)) + if (MaskElt->isAllOnesValue() || isa(MaskElt)) + return true; + } + return false; +} + /// TODO: This is a lot like known bits, but for /// vectors. Is there something we can common this with? APInt llvm::possiblyDemandedEltsInMask(Value *Mask) { diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index 50c0f9a913f3..89ca40f1c20d 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -399,11 +399,14 @@ Instruction *InstCombinerImpl::simplifyMaskedScatter(IntrinsicInst &II) { if (auto *SplatPtr = getSplatValue(II.getArgOperand(1))) { // scatter(splat(value), splat(ptr), non-zero-mask) -> store value, ptr if (auto *SplatValue = getSplatValue(II.getArgOperand(0))) { - Align Alignment = cast(II.getArgOperand(2))->getAlignValue(); - StoreInst *S = - new StoreInst(SplatValue, SplatPtr, /*IsVolatile=*/false, Alignment); - S->copyMetadata(II); - return S; + if (maskContainsAllOneOrUndef(ConstMask)) { + Align Alignment = + cast(II.getArgOperand(2))->getAlignValue(); + StoreInst *S = new StoreInst(SplatValue, SplatPtr, /*IsVolatile=*/false, + Alignment); + S->copyMetadata(II); + return S; + } } // scatter(vector, splat(ptr), splat(true)) -> store extract(vector, // lastlane), ptr diff --git a/llvm/test/Transforms/InstCombine/pr83947.ll b/llvm/test/Transforms/InstCombine/pr83947.ll new file mode 100644 index 000000000000..c1d601ff6371 --- /dev/null +++ b/llvm/test/Transforms/InstCombine/pr83947.ll @@ -0,0 +1,67 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -passes=instcombine < %s | FileCheck %s + +@c = global i32 0, align 4 +@b = global i32 0, align 4 + +define void @masked_scatter1() { +; CHECK-LABEL: define void @masked_scatter1() { +; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( zeroinitializer, shufflevector ( insertelement ( poison, ptr @c, i64 0), poison, zeroinitializer), i32 4, shufflevector ( insertelement ( poison, i1 icmp eq (ptr getelementptr inbounds (i32, ptr @b, i64 1), ptr @c), i64 0), poison, zeroinitializer)) +; CHECK-NEXT: ret void +; + call void @llvm.masked.scatter.nxv4i32.nxv4p0( zeroinitializer, splat (ptr @c), i32 4, splat (i1 icmp eq (ptr getelementptr (i32, ptr @b, i64 1), ptr @c))) + ret void +} + +define void @masked_scatter2() { +; CHECK-LABEL: define void @masked_scatter2() { +; CHECK-NEXT: store i32 0, ptr @c, align 4 +; CHECK-NEXT: ret void +; + call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> zeroinitializer, <2 x ptr> splat (ptr @c), i32 4, <2 x i1> splat (i1 true)) + ret void +} + +define void @masked_scatter3() { +; CHECK-LABEL: define void @masked_scatter3() { +; CHECK-NEXT: store i32 0, ptr @c, align 4 +; CHECK-NEXT: ret void +; + call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> zeroinitializer, <2 x ptr> splat (ptr @c), i32 4, <2 x i1> undef) + ret void +} + +define void @masked_scatter4() { +; CHECK-LABEL: define void @masked_scatter4() { +; CHECK-NEXT: ret void +; + call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> zeroinitializer, <2 x ptr> splat (ptr @c), i32 4, <2 x i1> splat (i1 false)) + ret void +} + +define void @masked_scatter5() { +; CHECK-LABEL: define void @masked_scatter5() { +; CHECK-NEXT: store i32 0, ptr @c, align 4 +; CHECK-NEXT: ret void +; + call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> zeroinitializer, <2 x ptr> splat (ptr @c), i32 4, <2 x i1> ) + ret void +} + +define void @masked_scatter6() { +; CHECK-LABEL: define void @masked_scatter6() { +; CHECK-NEXT: store i32 0, ptr @c, align 4 +; CHECK-NEXT: ret void +; + call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> zeroinitializer, <2 x ptr> splat (ptr @c), i32 4, <2 x i1> ) + ret void +} + +define void @masked_scatter7() { +; CHECK-LABEL: define void @masked_scatter7() { +; CHECK-NEXT: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> zeroinitializer, <2 x ptr> , i32 4, <2 x i1> ) +; CHECK-NEXT: ret void +; + call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> zeroinitializer, <2 x ptr> splat (ptr @c), i32 4, <2 x i1> splat (i1 icmp eq (ptr getelementptr (i32, ptr @b, i64 1), ptr @c))) + ret void +} -- GitLab From b2c16e7ff48a41eead168e1b95ed0c7813dd3b9c Mon Sep 17 00:00:00 2001 From: James Westwood Date: Tue, 5 Mar 2024 14:34:43 +0000 Subject: [PATCH 145/929] =?UTF-8?q?Revert=20"[ARM]=20R11=20not=20pushed=20?= =?UTF-8?q?adjacent=20to=20link=20register=20with=20PAC-M=20and=E2=80=A6?= =?UTF-8?q?=20(#84019)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … AAPCS frame chain fix (#82801)" This reverts commit 00e4a4197137410129d4725ffb82bae9ce44bdde. This patch was found to cause miscompilations and compilation failures. --- llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp | 6 +- llvm/lib/Target/ARM/ARMFrameLowering.cpp | 215 +++++------------- llvm/lib/Target/ARM/ARMSubtarget.cpp | 37 +-- llvm/lib/Target/ARM/ARMSubtarget.h | 35 ++- llvm/lib/Target/ARM/Thumb1FrameLowering.cpp | 15 +- .../CodeGen/Thumb2/pacbti-m-frame-chain.ll | 82 ------- 6 files changed, 93 insertions(+), 297 deletions(-) delete mode 100644 llvm/test/CodeGen/Thumb2/pacbti-m-frame-chain.ll diff --git a/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp b/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp index 4971281ec238..9adf758b46c4 100644 --- a/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp +++ b/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp @@ -62,16 +62,14 @@ ARMBaseRegisterInfo::ARMBaseRegisterInfo() const MCPhysReg* ARMBaseRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const { const ARMSubtarget &STI = MF->getSubtarget(); - bool UseSplitPush = (STI.getPushPopSplitVariation(*MF) == - ARMSubtarget::PushPopSplitVariation::R7Split); + bool UseSplitPush = STI.splitFramePushPop(*MF); const Function &F = MF->getFunction(); if (F.getCallingConv() == CallingConv::GHC) { // GHC set of callee saved regs is empty as all those regs are // used for passing STG regs around return CSR_NoRegs_SaveList; - } else if (STI.getPushPopSplitVariation(*MF) == - ARMSubtarget::PushPopSplitVariation::R11SplitWindowsSEHUnwind) { + } else if (STI.splitFramePointerPush(*MF)) { return CSR_Win_SplitFP_SaveList; } else if (F.getCallingConv() == CallingConv::CFGuard_Check) { return CSR_Win_AAPCS_CFGuard_Check_SaveList; diff --git a/llvm/lib/Target/ARM/ARMFrameLowering.cpp b/llvm/lib/Target/ARM/ARMFrameLowering.cpp index d251ad5543ba..9b54dd4e4e61 100644 --- a/llvm/lib/Target/ARM/ARMFrameLowering.cpp +++ b/llvm/lib/Target/ARM/ARMFrameLowering.cpp @@ -718,14 +718,9 @@ static int getMaxFPOffset(const ARMSubtarget &STI, const ARMFunctionInfo &AFI, // This is a conservative estimation: Assume the frame pointer being r7 and // pc("r15") up to r8 getting spilled before (= 8 registers). int MaxRegBytes = 8 * 4; - if (STI.getPushPopSplitVariation(MF) == - ARMSubtarget::PushPopSplitVariation::R11SplitAAPCSBranchSigning) + if (STI.splitFramePointerPush(MF)) { // Here, r11 can be stored below all of r4-r15 (3 registers more than - // above). - MaxRegBytes = 11 * 4; - if (STI.getPushPopSplitVariation(MF) == - ARMSubtarget::PushPopSplitVariation::R11SplitWindowsSEHUnwind) { - // Here, r11 can be stored below all of r4-r15 plus d8-d15. + // above), plus d8-d15. MaxRegBytes = 11 * 4 + 8 * 8; } int FPCXTSaveSize = @@ -793,10 +788,7 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF, } // Determine spill area sizes. - if (STI.getPushPopSplitVariation(MF) == - ARMSubtarget::PushPopSplitVariation::R11SplitAAPCSBranchSigning || - STI.getPushPopSplitVariation(MF) == - ARMSubtarget::PushPopSplitVariation::R11SplitWindowsSEHUnwind) { + if (STI.splitFramePointerPush(MF)) { for (const CalleeSavedInfo &I : CSI) { Register Reg = I.getReg(); int FI = I.getFrameIdx(); @@ -842,8 +834,7 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF, case ARM::R10: case ARM::R11: case ARM::R12: - if (STI.getPushPopSplitVariation(MF) == - ARMSubtarget::PushPopSplitVariation::R7Split) { + if (STI.splitFramePushPop(MF)) { GPRCS2Size += 4; break; } @@ -906,15 +897,13 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF, unsigned GPRCS2Offset = GPRCS1Offset - GPRCS2Size; Align DPRAlign = DPRCSSize ? std::min(Align(8), Alignment) : Align(4); unsigned DPRGapSize = GPRCS1Size + FPCXTSaveSize + ArgRegsSaveSize; - if (STI.getPushPopSplitVariation(MF) != - ARMSubtarget::PushPopSplitVariation::R11SplitWindowsSEHUnwind) { + if (!STI.splitFramePointerPush(MF)) { DPRGapSize += GPRCS2Size; } DPRGapSize %= DPRAlign.value(); unsigned DPRCSOffset; - if (STI.getPushPopSplitVariation(MF) == - ARMSubtarget::PushPopSplitVariation::R11SplitWindowsSEHUnwind) { + if (STI.splitFramePointerPush(MF)) { DPRCSOffset = GPRCS1Offset - DPRGapSize - DPRCSSize; GPRCS2Offset = DPRCSOffset - GPRCS2Size; } else { @@ -933,10 +922,8 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF, AFI->setGPRCalleeSavedArea2Offset(GPRCS2Offset); AFI->setDPRCalleeSavedAreaOffset(DPRCSOffset); - // Move past area 2, unless following the CSR_Win_SplitFP calling convention. - if (GPRCS2Size > 0 && - STI.getPushPopSplitVariation(MF) != - ARMSubtarget::PushPopSplitVariation::R11SplitWindowsSEHUnwind) { + // Move past area 2. + if (GPRCS2Size > 0 && !STI.splitFramePointerPush(MF)) { GPRCS2Push = LastPush = MBBI++; DefCFAOffsetCandidates.addInst(LastPush, GPRCS2Size); } @@ -976,18 +963,13 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF, } else NumBytes = DPRCSOffset; - // Move past area 2 if following the CSR_Win_SplitFP calling convention. - if (GPRCS2Size > 0 && - STI.getPushPopSplitVariation(MF) == - ARMSubtarget::PushPopSplitVariation::R11SplitWindowsSEHUnwind) { + if (GPRCS2Size > 0 && STI.splitFramePointerPush(MF)) { GPRCS2Push = LastPush = MBBI++; DefCFAOffsetCandidates.addInst(LastPush, GPRCS2Size); } bool NeedsWinCFIStackAlloc = NeedsWinCFI; - if (STI.getPushPopSplitVariation(MF) == - ARMSubtarget::PushPopSplitVariation::R11SplitWindowsSEHUnwind && - HasFP) + if (STI.splitFramePointerPush(MF) && HasFP) NeedsWinCFIStackAlloc = false; if (STI.isTargetWindows() && WindowsRequiresStackProbe(MF, NumBytes)) { @@ -1092,10 +1074,7 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF, AfterPush = std::next(GPRCS1Push); unsigned PushSize = sizeOfSPAdjustment(*GPRCS1Push); int FPOffset = PushSize + FramePtrOffsetInPush; - if (STI.getPushPopSplitVariation(MF) == - ARMSubtarget::PushPopSplitVariation::R11SplitAAPCSBranchSigning || - STI.getPushPopSplitVariation(MF) == - ARMSubtarget::PushPopSplitVariation::R11SplitWindowsSEHUnwind) { + if (STI.splitFramePointerPush(MF)) { AfterPush = std::next(GPRCS2Push); emitRegPlusImmediate(!AFI->isThumbFunction(), MBB, AfterPush, dl, TII, FramePtr, ARM::SP, 0, MachineInstr::FrameSetup); @@ -1127,9 +1106,7 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF, // instructions below don't need to be replayed to unwind the stack. if (NeedsWinCFI && MBBI != MBB.begin()) { MachineBasicBlock::iterator End = MBBI; - if (HasFP && - STI.getPushPopSplitVariation(MF) == - ARMSubtarget::PushPopSplitVariation::R11SplitWindowsSEHUnwind) + if (HasFP && STI.splitFramePointerPush(MF)) End = AfterPush; insertSEHRange(MBB, {}, End, TII, MachineInstr::FrameSetup); BuildMI(MBB, End, dl, TII.get(ARM::SEH_PrologEnd)) @@ -1141,103 +1118,51 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF, // the necessary DWARF cf instructions to describe the situation. Start by // recording where each register ended up: if (GPRCS1Size > 0 && !NeedsWinCFI) { - if (STI.getPushPopSplitVariation(MF) == - ARMSubtarget::PushPopSplitVariation::R11SplitAAPCSBranchSigning) { - MachineBasicBlock::iterator Pos = std::next(GPRCS1Push); - int CFIIndex; - for (const auto &Entry : CSI) { - Register Reg = Entry.getReg(); - int FI = Entry.getFrameIdx(); - switch (Reg) { - case ARM::R0: - case ARM::R1: - case ARM::R2: - case ARM::R3: - case ARM::R4: - case ARM::R5: - case ARM::R6: - case ARM::R7: - case ARM::R8: - case ARM::R9: - case ARM::R10: - case ARM::R12: - CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset( - nullptr, MRI->getDwarfRegNum(Reg, true), - MFI.getObjectOffset(FI))); - BuildMI(MBB, Pos, dl, TII.get(TargetOpcode::CFI_INSTRUCTION)) - .addCFIIndex(CFIIndex) - .setMIFlags(MachineInstr::FrameSetup); - break; - } - } - } else { - MachineBasicBlock::iterator Pos = std::next(GPRCS1Push); - int CFIIndex; - for (const auto &Entry : CSI) { - Register Reg = Entry.getReg(); - int FI = Entry.getFrameIdx(); - switch (Reg) { - case ARM::R8: - case ARM::R9: - case ARM::R10: - case ARM::R11: - case ARM::R12: - if (STI.getPushPopSplitVariation(MF) == - ARMSubtarget::PushPopSplitVariation::R7Split) - break; - [[fallthrough]]; - case ARM::R0: - case ARM::R1: - case ARM::R2: - case ARM::R3: - case ARM::R4: - case ARM::R5: - case ARM::R6: - case ARM::R7: - case ARM::LR: - CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset( - nullptr, MRI->getDwarfRegNum(Reg, true), - MFI.getObjectOffset(FI))); - BuildMI(MBB, Pos, dl, TII.get(TargetOpcode::CFI_INSTRUCTION)) - .addCFIIndex(CFIIndex) - .setMIFlags(MachineInstr::FrameSetup); + MachineBasicBlock::iterator Pos = std::next(GPRCS1Push); + int CFIIndex; + for (const auto &Entry : CSI) { + Register Reg = Entry.getReg(); + int FI = Entry.getFrameIdx(); + switch (Reg) { + case ARM::R8: + case ARM::R9: + case ARM::R10: + case ARM::R11: + case ARM::R12: + if (STI.splitFramePushPop(MF)) break; - } + [[fallthrough]]; + case ARM::R0: + case ARM::R1: + case ARM::R2: + case ARM::R3: + case ARM::R4: + case ARM::R5: + case ARM::R6: + case ARM::R7: + case ARM::LR: + CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset( + nullptr, MRI->getDwarfRegNum(Reg, true), MFI.getObjectOffset(FI))); + BuildMI(MBB, Pos, dl, TII.get(TargetOpcode::CFI_INSTRUCTION)) + .addCFIIndex(CFIIndex) + .setMIFlags(MachineInstr::FrameSetup); + break; } } } if (GPRCS2Size > 0 && !NeedsWinCFI) { MachineBasicBlock::iterator Pos = std::next(GPRCS2Push); - if (STI.getPushPopSplitVariation(MF) == - ARMSubtarget::PushPopSplitVariation::R11SplitAAPCSBranchSigning) { - for (const auto &Entry : CSI) { - Register Reg = Entry.getReg(); - int FI = Entry.getFrameIdx(); - switch (Reg) { - case ARM::R11: - case ARM::LR: - unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true); - unsigned Offset = MFI.getObjectOffset(FI); - unsigned CFIIndex = MF.addFrameInst( - MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset)); - BuildMI(MBB, Pos, dl, TII.get(TargetOpcode::CFI_INSTRUCTION)) - .addCFIIndex(CFIIndex) - .setMIFlags(MachineInstr::FrameSetup); - break; - } - } - } else { - MachineBasicBlock::iterator Pos = std::next(GPRCS2Push); - for (const auto &Entry : CSI) { - Register Reg = Entry.getReg(); - int FI = Entry.getFrameIdx(); - switch (Reg) { - case ARM::R8: - case ARM::R9: - case ARM::R10: - case ARM::R11: - case ARM::R12: + for (const auto &Entry : CSI) { + Register Reg = Entry.getReg(); + int FI = Entry.getFrameIdx(); + switch (Reg) { + case ARM::R8: + case ARM::R9: + case ARM::R10: + case ARM::R11: + case ARM::R12: + if (STI.splitFramePushPop(MF)) { unsigned DwarfReg = MRI->getDwarfRegNum( Reg == ARM::R12 ? ARM::RA_AUTH_CODE : Reg, true); unsigned Offset = MFI.getObjectOffset(FI); @@ -1246,8 +1171,8 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF, BuildMI(MBB, Pos, dl, TII.get(TargetOpcode::CFI_INSTRUCTION)) .addCFIIndex(CFIIndex) .setMIFlags(MachineInstr::FrameSetup); - break; } + break; } } } @@ -1457,9 +1382,7 @@ void ARMFrameLowering::emitEpilogue(MachineFunction &MF, MachineInstr::FrameDestroy); // Increment past our save areas. - if (AFI->getGPRCalleeSavedArea2Size() && - STI.getPushPopSplitVariation(MF) == - ARMSubtarget::PushPopSplitVariation::R11SplitWindowsSEHUnwind) + if (AFI->getGPRCalleeSavedArea2Size() && STI.splitFramePointerPush(MF)) MBBI++; if (MBBI != MBB.end() && AFI->getDPRCalleeSavedAreaSize()) { @@ -1476,9 +1399,7 @@ void ARMFrameLowering::emitEpilogue(MachineFunction &MF, MachineInstr::FrameDestroy); } - if (AFI->getGPRCalleeSavedArea2Size() && - STI.getPushPopSplitVariation(MF) != - ARMSubtarget::PushPopSplitVariation::R11SplitWindowsSEHUnwind) + if (AFI->getGPRCalleeSavedArea2Size() && !STI.splitFramePointerPush(MF)) MBBI++; if (AFI->getGPRCalleeSavedArea1Size()) MBBI++; @@ -1618,9 +1539,7 @@ void ARMFrameLowering::emitPushInst(MachineBasicBlock &MBB, unsigned LastReg = 0; for (; i != 0; --i) { Register Reg = CSI[i-1].getReg(); - if (!(Func)(Reg, STI.getPushPopSplitVariation(MF) == - ARMSubtarget::PushPopSplitVariation::R7Split)) - continue; + if (!(Func)(Reg, STI.splitFramePushPop(MF))) continue; // D-registers in the aligned area DPRCS2 are NOT spilled here. if (Reg >= ARM::D8 && Reg < ARM::D8 + NumAlignedDPRCS2Regs) @@ -1713,9 +1632,7 @@ void ARMFrameLowering::emitPopInst(MachineBasicBlock &MBB, for (; i != 0; --i) { CalleeSavedInfo &Info = CSI[i-1]; Register Reg = Info.getReg(); - if (!(Func)(Reg, STI.getPushPopSplitVariation(MF) == - ARMSubtarget::PushPopSplitVariation::R7Split)) - continue; + if (!(Func)(Reg, STI.splitFramePushPop(MF))) continue; // The aligned reloads from area DPRCS2 are not inserted here. if (Reg >= ARM::D8 && Reg < ARM::D8 + NumAlignedDPRCS2Regs) @@ -1723,11 +1640,7 @@ void ARMFrameLowering::emitPopInst(MachineBasicBlock &MBB, if (Reg == ARM::LR && !isTailCall && !isVarArg && !isInterrupt && !isCmseEntry && !isTrap && AFI->getArgumentStackToRestore() == 0 && STI.hasV5TOps() && MBB.succ_empty() && !hasPAC && - (STI.getPushPopSplitVariation(MF) != - ARMSubtarget::PushPopSplitVariation:: - R11SplitAAPCSBranchSigning && - STI.getPushPopSplitVariation(MF) != - ARMSubtarget::PushPopSplitVariation::R11SplitWindowsSEHUnwind)) { + !STI.splitFramePointerPush(MF)) { Reg = ARM::PC; // Fold the return instruction into the LDM. DeleteRet = true; @@ -2088,10 +2001,7 @@ bool ARMFrameLowering::spillCalleeSavedRegisters( .addImm(-4) .add(predOps(ARMCC::AL)); } - if (STI.getPushPopSplitVariation(MF) == - ARMSubtarget::PushPopSplitVariation::R11SplitAAPCSBranchSigning || - STI.getPushPopSplitVariation(MF) == - ARMSubtarget::PushPopSplitVariation::R11SplitWindowsSEHUnwind) { + if (STI.splitFramePointerPush(MF)) { emitPushInst(MBB, MI, CSI, PushOpc, PushOneOpc, false, &isSplitFPArea1Register, 0, MachineInstr::FrameSetup); emitPushInst(MBB, MI, CSI, FltOpc, 0, true, &isARMArea3Register, @@ -2136,10 +2046,7 @@ bool ARMFrameLowering::restoreCalleeSavedRegisters( unsigned LdrOpc = AFI->isThumbFunction() ? ARM::t2LDR_POST : ARM::LDR_POST_IMM; unsigned FltOpc = ARM::VLDMDIA_UPD; - if (STI.getPushPopSplitVariation(MF) == - ARMSubtarget::PushPopSplitVariation::R11SplitAAPCSBranchSigning || - STI.getPushPopSplitVariation(MF) == - ARMSubtarget::PushPopSplitVariation::R11SplitWindowsSEHUnwind) { + if (STI.splitFramePointerPush(MF)) { emitPopInst(MBB, MI, CSI, PopOpc, LdrOpc, isVarArg, false, &isSplitFPArea2Register, 0); emitPopInst(MBB, MI, CSI, FltOpc, 0, isVarArg, true, &isARMArea3Register, @@ -2455,8 +2362,7 @@ void ARMFrameLowering::determineCalleeSaves(MachineFunction &MF, if (Spilled) { NumGPRSpills++; - if (STI.getPushPopSplitVariation(MF) != - ARMSubtarget::PushPopSplitVariation::R7Split) { + if (!STI.splitFramePushPop(MF)) { if (Reg == ARM::LR) LRSpilled = true; CS1Spilled = true; @@ -2478,8 +2384,7 @@ void ARMFrameLowering::determineCalleeSaves(MachineFunction &MF, break; } } else { - if (STI.getPushPopSplitVariation(MF) != - ARMSubtarget::PushPopSplitVariation::R7Split) { + if (!STI.splitFramePushPop(MF)) { UnspilledCS1GPRs.push_back(Reg); continue; } diff --git a/llvm/lib/Target/ARM/ARMSubtarget.cpp b/llvm/lib/Target/ARM/ARMSubtarget.cpp index b9e63ab3afc5..691715dc2963 100644 --- a/llvm/lib/Target/ARM/ARMSubtarget.cpp +++ b/llvm/lib/Target/ARM/ARMSubtarget.cpp @@ -493,38 +493,11 @@ bool ARMSubtarget::ignoreCSRForAllocationOrder(const MachineFunction &MF, ARM::GPRRegClass.contains(PhysReg); } -ARMSubtarget::PushPopSplitVariation -ARMSubtarget::getPushPopSplitVariation(const MachineFunction &MF) const { +bool ARMSubtarget::splitFramePointerPush(const MachineFunction &MF) const { const Function &F = MF.getFunction(); + if (!MF.getTarget().getMCAsmInfo()->usesWindowsCFI() || + !F.needsUnwindTableEntry()) + return false; const MachineFrameInfo &MFI = MF.getFrameInfo(); - const std::vector CSI = - MF.getFrameInfo().getCalleeSavedInfo(); - // Returns R7Split if the frame setup must be split into two separate pushes - // of r0-r7,lr and another containing r8-r11 (+r12 if necessary). This is - // always required on Thumb1-only targets, as the push and pop instructions - // can't access the high registers. This is also required when R7 is the frame - // pointer and frame pointer elimiination is disabled, or branch signing is - // enabled and AAPCS is disabled. - if ((MF.getInfo()->shouldSignReturnAddress() && - !createAAPCSFrameChain()) || - (getFramePointerReg() == ARM::R7 && - MF.getTarget().Options.DisableFramePointerElim(MF)) || - isThumb1Only()) - return R7Split; - // Returns R11SplitWindowsSEHUnwind when the stack pointer needs to be - // restored from the frame pointer r11 + an offset and Windows CFI is enabled. - // This stack unwinding cannot be expressed with SEH unwind opcodes when done - // with a single push, making it necessary to split the push into r4-r10, and - // another containing r11+lr. - if (MF.getTarget().getMCAsmInfo()->usesWindowsCFI() && - F.needsUnwindTableEntry() && - (MFI.hasVarSizedObjects() || getRegisterInfo()->hasStackRealignment(MF))) - return R11SplitWindowsSEHUnwind; - // Returns R11SplitAAPCSBranchSigning if R11 and lr are not adjacent to each - // other in the list of callee saved registers in a frame, and branch - // signing is enabled. - if (MF.getInfo()->shouldSignReturnAddress() && - getFramePointerReg() == ARM::R11) - return R11SplitAAPCSBranchSigning; - return NoSplit; + return MFI.hasVarSizedObjects() || getRegisterInfo()->hasStackRealignment(MF); } diff --git a/llvm/lib/Target/ARM/ARMSubtarget.h b/llvm/lib/Target/ARM/ARMSubtarget.h index 435a2d2c5f1e..044b1c4c54e0 100644 --- a/llvm/lib/Target/ARM/ARMSubtarget.h +++ b/llvm/lib/Target/ARM/ARMSubtarget.h @@ -150,26 +150,6 @@ public: SingleIssuePlusExtras, }; - /// How the pushing and popping of callee saved registers to and from the - /// stack should be split. - enum PushPopSplitVariation { - /// r4-r11+lr (+r12 if necessary) can be pushed in a single instruction. - NoSplit, - /// The registers need to be split into a push of r4-r7+lr and another - /// containing r8-r11 (+r12 if necessary). Due to Thumb1FrameLowering - /// having separate handling of AAPCS being enabled, and therefore r11+lr - /// needing to be in a separate push, this value is also used as the push - /// variation in that case. - R7Split, - /// The registers need to be split into a push containing r4-r10+r12 and - /// another containing r11 + lr. - R11SplitAAPCSBranchSigning, - /// The registers need to be split into a push containing r4-r10 and another - /// containing r11 + lr. In this case, the floating point registers are - /// pushed between these two pushes. - R11SplitWindowsSEHUnwind - }; - protected: // Bool members corresponding to the SubtargetFeatures defined in tablegen #define GET_SUBTARGETINFO_MACRO(ATTRIBUTE, DEFAULT, GETTER) \ @@ -467,8 +447,19 @@ public: return ARM::R11; } - enum PushPopSplitVariation - getPushPopSplitVariation(const MachineFunction &MF) const; + /// Returns true if the frame setup is split into two separate pushes (first + /// r0-r7,lr then r8-r11), principally so that the frame pointer is adjacent + /// to lr. This is always required on Thumb1-only targets, as the push and + /// pop instructions can't access the high registers. + bool splitFramePushPop(const MachineFunction &MF) const { + if (MF.getInfo()->shouldSignReturnAddress()) + return true; + return (getFramePointerReg() == ARM::R7 && + MF.getTarget().Options.DisableFramePointerElim(MF)) || + isThumb1Only(); + } + + bool splitFramePointerPush(const MachineFunction &MF) const; bool useStride4VFPs() const; diff --git a/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp b/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp index f1558e64ed3e..0f4ece64bff5 100644 --- a/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp +++ b/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp @@ -222,8 +222,11 @@ void Thumb1FrameLowering::emitPrologue(MachineFunction &MF, case ARM::R8: case ARM::R9: case ARM::R10: - GPRCS2Size += 4; - break; + if (STI.splitFramePushPop(MF)) { + GPRCS2Size += 4; + break; + } + [[fallthrough]]; case ARM::LR: if (HasFrameRecordArea) { FRSize += 4; @@ -358,6 +361,14 @@ void Thumb1FrameLowering::emitPrologue(MachineFunction &MF, Register Reg = I.getReg(); int FI = I.getFrameIdx(); switch (Reg) { + case ARM::R8: + case ARM::R9: + case ARM::R10: + case ARM::R11: + case ARM::R12: + if (STI.splitFramePushPop(MF)) + break; + [[fallthrough]]; case ARM::R0: case ARM::R1: case ARM::R2: diff --git a/llvm/test/CodeGen/Thumb2/pacbti-m-frame-chain.ll b/llvm/test/CodeGen/Thumb2/pacbti-m-frame-chain.ll deleted file mode 100644 index 77759355e576..000000000000 --- a/llvm/test/CodeGen/Thumb2/pacbti-m-frame-chain.ll +++ /dev/null @@ -1,82 +0,0 @@ -; RUN: llc -filetype asm -o - %s --frame-pointer=all -mattr=+aapcs-frame-chain -mattr=+aapcs-frame-chain-leaf -force-dwarf-frame-section | FileCheck %s -target triple = "thumbv8m.main-none-none-eabi" - -; int f() { -; return 0; -; } -; -; int x(int, char *); -; int y(int n) { -; char a[n]; -; return 1 + x(n, a); -; } - -define hidden i32 @f() local_unnamed_addr { -entry: - ret i32 0; -} - -define hidden i32 @x(i32 noundef %n) local_unnamed_addr { -entry: - %vla = alloca i8, i32 %n, align 1 - %call = call i32 @y(i32 noundef %n, ptr noundef nonnull %vla) - %add = add nsw i32 %call, 1 - ret i32 %add -} - -declare dso_local i32 @y(i32 noundef, ptr noundef) local_unnamed_addr - -; CHECK-LABEL: f: -; CHECK: pac r12, lr, sp -; CHECK-NEXT: .save {ra_auth_code} -; CHECK-NEXT: str r12, [sp, #-4]! -; CHECK-NEXT: .cfi_def_cfa_offset 4 -; CHECK-NEXT: .cfi_offset r12, -8 -; CHECK-NEXT: .save {r11, lr} -; CHECK-NEXT: push.w {r11, lr} -; CHECK-NEXT: .cfi_offset lr, -4 -; CHECK-NEXT: .cfi_offset r11, -12 -; CHECK-NEXT: .setfp r11, sp -; CHECK-NEXT: mov r11, sp -; CHECK-NEXT: .cfi_def_cfa r11, 12 -; CHECK-NEXT: movs r0, #0 -; CHECK-NEXT: pop.w {r11, lr} -; CHECK-NEXT: ldr r12, [sp], #4 -; CHECK-NEXT: aut r12, lr, sp -; CHECK-NEXT: bx lr - -; CHECK-LABEL: x: -; CHECK: pac r12, lr, sp -; CHECK-NEXT: .save {r4, r7, ra_auth_code} -; CHECK-NEXT: push.w {r4, r7, r12} -; CHECK-NEXT: .cfi_def_cfa_offset 12 -; CHECK-NEXT: .cfi_offset r12, -8 -; CHECK-NEXT: .cfi_offset r7, -16 -; CHECK-NEXT: .cfi_offset r4, -20 -; CHECK-NEXT: .save {r11, lr} -; CHECK-NEXT: push.w {r11, lr} -; CHECK-NEXT: .cfi_offset lr, -4 -; CHECK-NEXT: .cfi_offset r11, -12 -; CHECK-NEXT: .setfp r11, sp -; CHECK-NEXT: mov r11, sp -; CHECK-NEXT: .cfi_def_cfa_register r11 -; CHECK-NEXT: .pad #4 -; CHECK-NEXT: sub sp, #4 -; CHECK-NEXT: adds r1, r0, #7 -; CHECK-NEXT: bic r1, r1, #7 -; CHECK-NEXT: sub.w r1, sp, r1 -; CHECK-NEXT: mov sp, r1 -; CHECK-NEXT: bl y -; CHECK-NEXT: sub.w r4, r11, #8 -; CHECK-NEXT: adds r0, #1 -; CHECK-NEXT: mov sp, r4 -; CHECK-NEXT: pop.w {r11, lr} -; CHECK-NEXT: pop.w {r4, r7, r12} -; CHECK-NEXT: aut r12, lr, sp -; CHECK-NEXT: bx lr - -!llvm.module.flags = !{!0, !1, !2} - -!0 = !{i32 8, !"sign-return-address", i32 1} -!1 = !{i32 8, !"sign-return-address-all", i32 0} -!2 = !{i32 8, !"branch-target-enforcement", i32 0} -- GitLab From f3be8427288a8888a0343e5f9ea7bf6573897ae5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingo=20M=C3=BCller?= Date: Tue, 5 Mar 2024 15:37:14 +0100 Subject: [PATCH 146/929] [mlir] Expose MLIR_ROCM_CONVERSIONS_ENABLED in mlir-config.h. (#83977) This is a follow up of #83004, which made the same change for `MLIR_CUDA_CONVERSIONS_ENABLED`. As the previous PR, this PR commit exposes mentioned CMake variable through `mlir-config.h` and uses the macro that is introduced with the same name. This replaces the macro `MLIR_ROCM_CONVERSIONS_ENABLED`, which the CMake files previously defined manually. --- mlir/CMakeLists.txt | 1 - mlir/include/mlir/Config/mlir-config.h.cmake | 4 ++++ mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp | 2 +- mlir/lib/Target/LLVM/ROCDL/Target.cpp | 10 +++++----- mlir/unittests/Target/LLVM/SerializeROCDLTarget.cpp | 6 +++--- utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 1 + 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/mlir/CMakeLists.txt b/mlir/CMakeLists.txt index 070609c94a3b..e37cb465b919 100644 --- a/mlir/CMakeLists.txt +++ b/mlir/CMakeLists.txt @@ -119,7 +119,6 @@ if ("AMDGPU" IN_LIST LLVM_TARGETS_TO_BUILD) else() set(MLIR_ENABLE_ROCM_CONVERSIONS 0) endif() -add_definitions(-DMLIR_ROCM_CONVERSIONS_ENABLED=${MLIR_ENABLE_ROCM_CONVERSIONS}) set(MLIR_ENABLE_CUDA_RUNNER 0 CACHE BOOL "Enable building the mlir CUDA runner") set(MLIR_ENABLE_ROCM_RUNNER 0 CACHE BOOL "Enable building the mlir ROCm runner") diff --git a/mlir/include/mlir/Config/mlir-config.h.cmake b/mlir/include/mlir/Config/mlir-config.h.cmake index 4a7d75e22668..a0176063b474 100644 --- a/mlir/include/mlir/Config/mlir-config.h.cmake +++ b/mlir/include/mlir/Config/mlir-config.h.cmake @@ -33,4 +33,8 @@ and targets. */ #cmakedefine01 MLIR_ENABLE_CUDA_CONVERSIONS +/* If set, enables ROCm-related features in ROCM-related transforms, pipelines, + and targets. */ +#cmakedefine01 MLIR_ENABLE_ROCM_CONVERSIONS + #endif diff --git a/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp b/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp index f379ea819392..01613ab5268b 100644 --- a/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp +++ b/mlir/lib/Dialect/GPU/Transforms/ModuleToBinary.cpp @@ -52,7 +52,7 @@ void GpuModuleToBinaryPass::getDependentDialects( #if MLIR_ENABLE_CUDA_CONVERSIONS registry.insert(); #endif -#if MLIR_ROCM_CONVERSIONS_ENABLED == 1 +#if MLIR_ENABLE_ROCM_CONVERSIONS registry.insert(); #endif registry.insert(); diff --git a/mlir/lib/Target/LLVM/ROCDL/Target.cpp b/mlir/lib/Target/LLVM/ROCDL/Target.cpp index cdcef1d6b459..4f01c6a66781 100644 --- a/mlir/lib/Target/LLVM/ROCDL/Target.cpp +++ b/mlir/lib/Target/LLVM/ROCDL/Target.cpp @@ -120,7 +120,7 @@ void SerializeGPUModuleBase::init() { static llvm::once_flag initializeBackendOnce; llvm::call_once(initializeBackendOnce, []() { // If the `AMDGPU` LLVM target was built, initialize it. -#if MLIR_ROCM_CONVERSIONS_ENABLED == 1 +#if MLIR_ENABLE_ROCM_CONVERSIONS LLVMInitializeAMDGPUTarget(); LLVMInitializeAMDGPUTargetInfo(); LLVMInitializeAMDGPUTargetMC(); @@ -318,7 +318,7 @@ SerializeGPUModuleBase::assembleIsa(StringRef isa) { return result; } -#if MLIR_ROCM_CONVERSIONS_ENABLED == 1 +#if MLIR_ENABLE_ROCM_CONVERSIONS namespace { class AMDGPUSerializer : public SerializeGPUModuleBase { public: @@ -450,7 +450,7 @@ AMDGPUSerializer::moduleToObject(llvm::Module &llvmModule) { // Compile to binary. return compileToBinary(*serializedISA); } -#endif // MLIR_ROCM_CONVERSIONS_ENABLED +#endif // MLIR_ENABLE_ROCM_CONVERSIONS std::optional> ROCDLTargetAttrImpl::serializeToObject( Attribute attribute, Operation *module, @@ -462,7 +462,7 @@ std::optional> ROCDLTargetAttrImpl::serializeToObject( module->emitError("Module must be a GPU module."); return std::nullopt; } -#if MLIR_ROCM_CONVERSIONS_ENABLED == 1 +#if MLIR_ENABLE_ROCM_CONVERSIONS AMDGPUSerializer serializer(*module, cast(attribute), options); serializer.init(); @@ -471,7 +471,7 @@ std::optional> ROCDLTargetAttrImpl::serializeToObject( module->emitError("The `AMDGPU` target was not built. Please enable it when " "building LLVM."); return std::nullopt; -#endif // MLIR_ROCM_CONVERSIONS_ENABLED == 1 +#endif // MLIR_ENABLE_ROCM_CONVERSIONS } Attribute diff --git a/mlir/unittests/Target/LLVM/SerializeROCDLTarget.cpp b/mlir/unittests/Target/LLVM/SerializeROCDLTarget.cpp index e91e69b10204..33291bc4bcae 100644 --- a/mlir/unittests/Target/LLVM/SerializeROCDLTarget.cpp +++ b/mlir/unittests/Target/LLVM/SerializeROCDLTarget.cpp @@ -31,10 +31,10 @@ using namespace mlir; // Skip the test if the AMDGPU target was not built. -#if MLIR_ROCM_CONVERSIONS_ENABLED == 0 -#define SKIP_WITHOUT_AMDGPU(x) DISABLED_##x -#else +#if MLIR_ENABLE_ROCM_CONVERSIONS #define SKIP_WITHOUT_AMDGPU(x) x +#else +#define SKIP_WITHOUT_AMDGPU(x) DISABLED_##x #endif class MLIRTargetLLVMROCDL : public ::testing::Test { diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 8a8dd6e10c48..7f33f1659922 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -35,6 +35,7 @@ expand_template( "#cmakedefine01 MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS": "#define MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS 0", "#cmakedefine MLIR_GREEDY_REWRITE_RANDOMIZER_SEED ${MLIR_GREEDY_REWRITE_RANDOMIZER_SEED}": "/* #undef MLIR_GREEDY_REWRITE_RANDOMIZER_SEED */", "#cmakedefine01 MLIR_ENABLE_PDL_IN_PATTERNMATCH": "#define MLIR_ENABLE_PDL_IN_PATTERNMATCH 1", + "#cmakedefine01 MLIR_ENABLE_ROCM_CONVERSIONS": "#define MLIR_ENABLE_ROCM_CONVERSIONS 0", } | if_cuda_available( {"#cmakedefine01 MLIR_ENABLE_CUDA_CONVERSIONS": "#define MLIR_ENABLE_CUDA_CONVERSIONS 1"}, {"#cmakedefine01 MLIR_ENABLE_CUDA_CONVERSIONS": "#define MLIR_ENABLE_CUDA_CONVERSIONS 0"}, -- GitLab From 1c2b79add6ec913441a3420ba13c03de0e20fd4f Mon Sep 17 00:00:00 2001 From: Alexey Bataev <5361294+alexey-bataev@users.noreply.github.com> Date: Tue, 5 Mar 2024 09:38:25 -0500 Subject: [PATCH 147/929] [SLP]Add runtime stride support for strided loads. Added support for runtime strides. Reviewers: preames, RKSimon Reviewed By: preames Pull Request: https://github.com/llvm/llvm-project/pull/81517 --- .../Transforms/Vectorize/SLPVectorizer.cpp | 166 +++++++++++++++++- .../RISCV/strided-loads-vectorized.ll | 98 +---------- 2 files changed, 170 insertions(+), 94 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index e0e3648f718e..36dc9094538a 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -88,6 +88,7 @@ #include "llvm/Transforms/Utils/InjectTLIMappings.h" #include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/LoopUtils.h" +#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h" #include #include #include @@ -4014,6 +4015,126 @@ static bool isReverseOrder(ArrayRef Order) { }); } +/// Checks if the provided list of pointers \p Pointers represents the strided +/// pointers for type ElemTy. If they are not, std::nullopt is returned. +/// Otherwise, if \p Inst is not specified, just initialized optional value is +/// returned to show that the pointers represent strided pointers. If \p Inst +/// specified, the runtime stride is materialized before the given \p Inst. +/// \returns std::nullopt if the pointers are not pointers with the runtime +/// stride, nullptr or actual stride value, otherwise. +static std::optional +calculateRtStride(ArrayRef PointerOps, Type *ElemTy, + const DataLayout &DL, ScalarEvolution &SE, + SmallVectorImpl &SortedIndices, + Instruction *Inst = nullptr) { + SmallVector SCEVs; + const SCEV *PtrSCEVLowest = nullptr; + const SCEV *PtrSCEVHighest = nullptr; + // Find lower/upper pointers from the PointerOps (i.e. with lowest and highest + // addresses). + for (Value *Ptr : PointerOps) { + const SCEV *PtrSCEV = SE.getSCEV(Ptr); + if (!PtrSCEV) + return std::nullopt; + SCEVs.push_back(PtrSCEV); + if (!PtrSCEVLowest && !PtrSCEVHighest) { + PtrSCEVLowest = PtrSCEVHighest = PtrSCEV; + continue; + } + const SCEV *Diff = SE.getMinusSCEV(PtrSCEV, PtrSCEVLowest); + if (isa(Diff)) + return std::nullopt; + if (Diff->isNonConstantNegative()) { + PtrSCEVLowest = PtrSCEV; + continue; + } + const SCEV *Diff1 = SE.getMinusSCEV(PtrSCEVHighest, PtrSCEV); + if (isa(Diff1)) + return std::nullopt; + if (Diff1->isNonConstantNegative()) { + PtrSCEVHighest = PtrSCEV; + continue; + } + } + // Dist = PtrSCEVHighest - PtrSCEVLowest; + const SCEV *Dist = SE.getMinusSCEV(PtrSCEVHighest, PtrSCEVLowest); + if (isa(Dist)) + return std::nullopt; + int Size = DL.getTypeStoreSize(ElemTy); + auto TryGetStride = [&](const SCEV *Dist, + const SCEV *Multiplier) -> const SCEV * { + if (const auto *M = dyn_cast(Dist)) { + if (M->getOperand(0) == Multiplier) + return M->getOperand(1); + if (M->getOperand(1) == Multiplier) + return M->getOperand(0); + return nullptr; + } + if (Multiplier == Dist) + return SE.getConstant(Dist->getType(), 1); + return SE.getUDivExactExpr(Dist, Multiplier); + }; + // Stride_in_elements = Dist / element_size * (num_elems - 1). + const SCEV *Stride = nullptr; + if (Size != 1 || SCEVs.size() > 2) { + const SCEV *Sz = SE.getConstant(Dist->getType(), Size * (SCEVs.size() - 1)); + Stride = TryGetStride(Dist, Sz); + if (!Stride) + return std::nullopt; + } + if (!Stride || isa(Stride)) + return std::nullopt; + // Iterate through all pointers and check if all distances are + // unique multiple of Stride. + using DistOrdPair = std::pair; + auto Compare = llvm::less_first(); + std::set Offsets(Compare); + int Cnt = 0; + bool IsConsecutive = true; + for (const SCEV *PtrSCEV : SCEVs) { + unsigned Dist = 0; + if (PtrSCEV != PtrSCEVLowest) { + const SCEV *Diff = SE.getMinusSCEV(PtrSCEV, PtrSCEVLowest); + const SCEV *Coeff = TryGetStride(Diff, Stride); + if (!Coeff) + return std::nullopt; + const auto *SC = dyn_cast(Coeff); + if (!SC || isa(SC)) + return std::nullopt; + if (!SE.getMinusSCEV(PtrSCEV, SE.getAddExpr(PtrSCEVLowest, + SE.getMulExpr(Stride, SC))) + ->isZero()) + return std::nullopt; + Dist = SC->getAPInt().getZExtValue(); + } + // If the strides are not the same or repeated, we can't vectorize. + if ((Dist / Size) * Size != Dist || (Dist / Size) >= SCEVs.size()) + return std::nullopt; + auto Res = Offsets.emplace(Dist, Cnt); + if (!Res.second) + return std::nullopt; + // Consecutive order if the inserted element is the last one. + IsConsecutive = IsConsecutive && std::next(Res.first) == Offsets.end(); + ++Cnt; + } + if (Offsets.size() != SCEVs.size()) + return std::nullopt; + SortedIndices.clear(); + if (!IsConsecutive) { + // Fill SortedIndices array only if it is non-consecutive. + SortedIndices.resize(PointerOps.size()); + Cnt = 0; + for (const std::pair &Pair : Offsets) { + SortedIndices[Cnt] = Pair.second; + ++Cnt; + } + } + if (!Inst) + return nullptr; + SCEVExpander Expander(SE, DL, "strided-load-vec"); + return Expander.expandCodeFor(Stride, Stride->getType(), Inst); +} + BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads( ArrayRef VL, const Value *VL0, SmallVectorImpl &Order, SmallVectorImpl &PointerOps, bool TryRecursiveCheck) const { @@ -4046,6 +4167,11 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads( auto *VecTy = FixedVectorType::get(ScalarTy, Sz); // Check the order of pointer operands or that all pointers are the same. bool IsSorted = sortPtrAccesses(PointerOps, ScalarTy, *DL, *SE, Order); + Align CommonAlignment = computeCommonAlignment(VL); + if (!IsSorted && Sz > MinProfitableStridedLoads && TTI->isTypeLegal(VecTy) && + TTI->isLegalStridedLoadStore(VecTy, CommonAlignment) && + calculateRtStride(PointerOps, ScalarTy, *DL, *SE, Order)) + return LoadsState::StridedVectorize; if (IsSorted || all_of(PointerOps, [&](Value *P) { return arePointersCompatible(P, PointerOps.front(), *TLI); })) { @@ -4465,6 +4591,13 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) { return std::nullopt; // No need to reorder. return std::move(ResOrder); } + if (TE.State == TreeEntry::StridedVectorize && !TopToBottom && + any_of(TE.UserTreeIndices, + [](const EdgeInfo &EI) { + return !Instruction::isBinaryOp(EI.UserTE->getOpcode()); + }) && + (TE.ReorderIndices.empty() || isReverseOrder(TE.ReorderIndices))) + return std::nullopt; if ((TE.State == TreeEntry::Vectorize || TE.State == TreeEntry::StridedVectorize) && (isa(TE.getMainOp()) || @@ -4930,7 +5063,8 @@ bool BoUpSLP::canReorderOperands( for (unsigned I = 0, E = UserTE->getNumOperands(); I < E; ++I) { if (any_of(Edges, [I](const std::pair &OpData) { return OpData.first == I && - OpData.second->State == TreeEntry::Vectorize; + (OpData.second->State == TreeEntry::Vectorize || + OpData.second->State == TreeEntry::StridedVectorize); })) continue; if (TreeEntry *TE = getVectorizedOperand(UserTE, I)) { @@ -4947,6 +5081,7 @@ bool BoUpSLP::canReorderOperands( // If there are reused scalars, process this node as a regular vectorize // node, just reorder reuses mask. if (TE->State != TreeEntry::Vectorize && + TE->State != TreeEntry::StridedVectorize && TE->ReuseShuffleIndices.empty() && TE->ReorderIndices.empty()) GatherOps.push_back(TE); continue; @@ -4955,6 +5090,7 @@ bool BoUpSLP::canReorderOperands( if (count_if(ReorderableGathers, [&Gather, UserTE, I](TreeEntry *TE) { assert(TE->State != TreeEntry::Vectorize && + TE->State != TreeEntry::StridedVectorize && "Only non-vectorized nodes are expected."); if (any_of(TE->UserTreeIndices, [UserTE, I](const EdgeInfo &EI) { @@ -12032,10 +12168,30 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) { std::optional Diff = getPointersDiff( VL0->getType(), Ptr0, VL0->getType(), PtrN, *DL, *SE); Type *StrideTy = DL->getIndexType(PO->getType()); - int Stride = *Diff / (static_cast(E->Scalars.size()) - 1); - Value *StrideVal = - ConstantInt::get(StrideTy, (IsReverseOrder ? -1 : 1) * Stride * - DL->getTypeAllocSize(ScalarTy)); + Value *StrideVal; + if (Diff) { + int Stride = *Diff / (static_cast(E->Scalars.size()) - 1); + StrideVal = + ConstantInt::get(StrideTy, (IsReverseOrder ? -1 : 1) * Stride * + DL->getTypeAllocSize(ScalarTy)); + } else { + SmallVector PointerOps(E->Scalars.size(), nullptr); + transform(E->Scalars, PointerOps.begin(), [](Value *V) { + return cast(V)->getPointerOperand(); + }); + OrdersType Order; + std::optional Stride = + calculateRtStride(PointerOps, ScalarTy, *DL, *SE, Order, + &*Builder.GetInsertPoint()); + Value *NewStride = + Builder.CreateIntCast(*Stride, StrideTy, /*isSigned=*/true); + StrideVal = Builder.CreateMul( + NewStride, + ConstantInt::get( + StrideTy, + (IsReverseOrder ? -1 : 1) * + static_cast(DL->getTypeAllocSize(ScalarTy)))); + } Align CommonAlignment = computeCommonAlignment(E->Scalars); auto *Inst = Builder.CreateIntrinsic( Intrinsic::experimental_vp_strided_load, diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/strided-loads-vectorized.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/strided-loads-vectorized.ll index 4b0b41970bbb..03acc0009fb0 100644 --- a/llvm/test/Transforms/SLPVectorizer/RISCV/strided-loads-vectorized.ll +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/strided-loads-vectorized.ll @@ -78,67 +78,13 @@ define void @test1(ptr %p, ptr noalias %s, i32 %stride) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[STR:%.*]] = zext i32 [[STRIDE:%.*]] to i64 ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [48 x float], ptr [[P:%.*]], i64 0, i64 0 -; CHECK-NEXT: [[I:%.*]] = load float, ptr [[ARRAYIDX]], align 4 ; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [48 x float], ptr [[P]], i64 0, i64 30 -; CHECK-NEXT: [[I1:%.*]] = load float, ptr [[ARRAYIDX1]], align 4 -; CHECK-NEXT: [[ADD:%.*]] = fsub fast float [[I1]], [[I]] ; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[S:%.*]], i64 0 -; CHECK-NEXT: store float [[ADD]], ptr [[ARRAYIDX2]], align 4 -; CHECK-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds [48 x float], ptr [[P]], i64 0, i64 [[STR]] -; CHECK-NEXT: [[I2:%.*]] = load float, ptr [[ARRAYIDX4]], align 4 -; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds [48 x float], ptr [[P]], i64 0, i64 26 -; CHECK-NEXT: [[I3:%.*]] = load float, ptr [[ARRAYIDX6]], align 4 -; CHECK-NEXT: [[ADD7:%.*]] = fsub fast float [[I3]], [[I2]] -; CHECK-NEXT: [[ARRAYIDX9:%.*]] = getelementptr inbounds float, ptr [[S]], i64 1 -; CHECK-NEXT: store float [[ADD7]], ptr [[ARRAYIDX9]], align 4 -; CHECK-NEXT: [[ST1:%.*]] = mul i64 [[STR]], 2 -; CHECK-NEXT: [[ARRAYIDX11:%.*]] = getelementptr inbounds [48 x float], ptr [[P]], i64 0, i64 [[ST1]] -; CHECK-NEXT: [[I4:%.*]] = load float, ptr [[ARRAYIDX11]], align 4 -; CHECK-NEXT: [[ARRAYIDX13:%.*]] = getelementptr inbounds [48 x float], ptr [[P]], i64 0, i64 22 -; CHECK-NEXT: [[I5:%.*]] = load float, ptr [[ARRAYIDX13]], align 4 -; CHECK-NEXT: [[ADD14:%.*]] = fsub fast float [[I5]], [[I4]] -; CHECK-NEXT: [[ARRAYIDX16:%.*]] = getelementptr inbounds float, ptr [[S]], i64 2 -; CHECK-NEXT: store float [[ADD14]], ptr [[ARRAYIDX16]], align 4 -; CHECK-NEXT: [[ST2:%.*]] = mul i64 [[STR]], 3 -; CHECK-NEXT: [[ARRAYIDX18:%.*]] = getelementptr inbounds [48 x float], ptr [[P]], i64 0, i64 [[ST2]] -; CHECK-NEXT: [[I6:%.*]] = load float, ptr [[ARRAYIDX18]], align 4 -; CHECK-NEXT: [[ARRAYIDX20:%.*]] = getelementptr inbounds [48 x float], ptr [[P]], i64 0, i64 18 -; CHECK-NEXT: [[I7:%.*]] = load float, ptr [[ARRAYIDX20]], align 4 -; CHECK-NEXT: [[ADD21:%.*]] = fsub fast float [[I7]], [[I6]] -; CHECK-NEXT: [[ARRAYIDX23:%.*]] = getelementptr inbounds float, ptr [[S]], i64 3 -; CHECK-NEXT: store float [[ADD21]], ptr [[ARRAYIDX23]], align 4 -; CHECK-NEXT: [[ST3:%.*]] = mul i64 [[STR]], 4 -; CHECK-NEXT: [[ARRAYIDX25:%.*]] = getelementptr inbounds [48 x float], ptr [[P]], i64 0, i64 [[ST3]] -; CHECK-NEXT: [[I8:%.*]] = load float, ptr [[ARRAYIDX25]], align 4 -; CHECK-NEXT: [[ARRAYIDX27:%.*]] = getelementptr inbounds [48 x float], ptr [[P]], i64 0, i64 14 -; CHECK-NEXT: [[I9:%.*]] = load float, ptr [[ARRAYIDX27]], align 4 -; CHECK-NEXT: [[ADD28:%.*]] = fsub fast float [[I9]], [[I8]] -; CHECK-NEXT: [[ARRAYIDX30:%.*]] = getelementptr inbounds float, ptr [[S]], i64 4 -; CHECK-NEXT: store float [[ADD28]], ptr [[ARRAYIDX30]], align 4 -; CHECK-NEXT: [[ST4:%.*]] = mul i64 [[STR]], 5 -; CHECK-NEXT: [[ARRAYIDX32:%.*]] = getelementptr inbounds [48 x float], ptr [[P]], i64 0, i64 [[ST4]] -; CHECK-NEXT: [[I10:%.*]] = load float, ptr [[ARRAYIDX32]], align 4 -; CHECK-NEXT: [[ARRAYIDX34:%.*]] = getelementptr inbounds [48 x float], ptr [[P]], i64 0, i64 10 -; CHECK-NEXT: [[I11:%.*]] = load float, ptr [[ARRAYIDX34]], align 4 -; CHECK-NEXT: [[ADD35:%.*]] = fsub fast float [[I11]], [[I10]] -; CHECK-NEXT: [[ARRAYIDX37:%.*]] = getelementptr inbounds float, ptr [[S]], i64 5 -; CHECK-NEXT: store float [[ADD35]], ptr [[ARRAYIDX37]], align 4 -; CHECK-NEXT: [[ST5:%.*]] = mul i64 [[STR]], 6 -; CHECK-NEXT: [[ARRAYIDX39:%.*]] = getelementptr inbounds [48 x float], ptr [[P]], i64 0, i64 [[ST5]] -; CHECK-NEXT: [[I12:%.*]] = load float, ptr [[ARRAYIDX39]], align 4 -; CHECK-NEXT: [[ARRAYIDX41:%.*]] = getelementptr inbounds [48 x float], ptr [[P]], i64 0, i64 6 -; CHECK-NEXT: [[I13:%.*]] = load float, ptr [[ARRAYIDX41]], align 4 -; CHECK-NEXT: [[ADD42:%.*]] = fsub fast float [[I13]], [[I12]] -; CHECK-NEXT: [[ARRAYIDX44:%.*]] = getelementptr inbounds float, ptr [[S]], i64 6 -; CHECK-NEXT: store float [[ADD42]], ptr [[ARRAYIDX44]], align 4 -; CHECK-NEXT: [[ST6:%.*]] = mul i64 [[STR]], 7 -; CHECK-NEXT: [[ARRAYIDX46:%.*]] = getelementptr inbounds [48 x float], ptr [[P]], i64 0, i64 [[ST6]] -; CHECK-NEXT: [[I14:%.*]] = load float, ptr [[ARRAYIDX46]], align 4 -; CHECK-NEXT: [[ARRAYIDX48:%.*]] = getelementptr inbounds [48 x float], ptr [[P]], i64 0, i64 2 -; CHECK-NEXT: [[I15:%.*]] = load float, ptr [[ARRAYIDX48]], align 4 -; CHECK-NEXT: [[ADD49:%.*]] = fsub fast float [[I15]], [[I14]] -; CHECK-NEXT: [[ARRAYIDX51:%.*]] = getelementptr inbounds float, ptr [[S]], i64 7 -; CHECK-NEXT: store float [[ADD49]], ptr [[ARRAYIDX51]], align 4 +; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[STR]], 4 +; CHECK-NEXT: [[TMP1:%.*]] = call <8 x float> @llvm.experimental.vp.strided.load.v8f32.p0.i64(ptr align 4 [[ARRAYIDX]], i64 [[TMP0]], <8 x i1> , i32 8) +; CHECK-NEXT: [[TMP2:%.*]] = call <8 x float> @llvm.experimental.vp.strided.load.v8f32.p0.i64(ptr align 4 [[ARRAYIDX1]], i64 -16, <8 x i1> , i32 8) +; CHECK-NEXT: [[TMP3:%.*]] = fsub fast <8 x float> [[TMP2]], [[TMP1]] +; CHECK-NEXT: store <8 x float> [[TMP3]], ptr [[ARRAYIDX2]], align 4 ; CHECK-NEXT: ret void ; entry: @@ -215,38 +161,12 @@ define void @test2(ptr %p, ptr noalias %s, i32 %stride) { ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [48 x float], ptr [[P:%.*]], i64 0, i64 2 ; CHECK-NEXT: [[ST6:%.*]] = mul i64 [[STR]], 7 ; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [48 x float], ptr [[P]], i64 0, i64 [[ST6]] -; CHECK-NEXT: [[I1:%.*]] = load float, ptr [[ARRAYIDX1]], align 4 ; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[S:%.*]], i64 0 -; CHECK-NEXT: [[ST5:%.*]] = mul i64 [[STR]], 6 -; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds [48 x float], ptr [[P]], i64 0, i64 [[ST5]] -; CHECK-NEXT: [[I3:%.*]] = load float, ptr [[ARRAYIDX6]], align 4 -; CHECK-NEXT: [[ST4:%.*]] = mul i64 [[STR]], 5 -; CHECK-NEXT: [[ARRAYIDX13:%.*]] = getelementptr inbounds [48 x float], ptr [[P]], i64 0, i64 [[ST4]] -; CHECK-NEXT: [[I5:%.*]] = load float, ptr [[ARRAYIDX13]], align 4 -; CHECK-NEXT: [[ST3:%.*]] = mul i64 [[STR]], 4 -; CHECK-NEXT: [[ARRAYIDX20:%.*]] = getelementptr inbounds [48 x float], ptr [[P]], i64 0, i64 [[ST3]] -; CHECK-NEXT: [[I7:%.*]] = load float, ptr [[ARRAYIDX20]], align 4 -; CHECK-NEXT: [[ST2:%.*]] = mul i64 [[STR]], 3 -; CHECK-NEXT: [[ARRAYIDX27:%.*]] = getelementptr inbounds [48 x float], ptr [[P]], i64 0, i64 [[ST2]] -; CHECK-NEXT: [[I9:%.*]] = load float, ptr [[ARRAYIDX27]], align 4 -; CHECK-NEXT: [[ST1:%.*]] = mul i64 [[STR]], 2 -; CHECK-NEXT: [[ARRAYIDX34:%.*]] = getelementptr inbounds [48 x float], ptr [[P]], i64 0, i64 [[ST1]] -; CHECK-NEXT: [[I11:%.*]] = load float, ptr [[ARRAYIDX34]], align 4 -; CHECK-NEXT: [[ARRAYIDX41:%.*]] = getelementptr inbounds [48 x float], ptr [[P]], i64 0, i64 [[STR]] -; CHECK-NEXT: [[I13:%.*]] = load float, ptr [[ARRAYIDX41]], align 4 -; CHECK-NEXT: [[ARRAYIDX48:%.*]] = getelementptr inbounds [48 x float], ptr [[P]], i64 0, i64 0 -; CHECK-NEXT: [[I15:%.*]] = load float, ptr [[ARRAYIDX48]], align 4 ; CHECK-NEXT: [[TMP0:%.*]] = call <8 x float> @llvm.experimental.vp.strided.load.v8f32.p0.i64(ptr align 4 [[ARRAYIDX]], i64 16, <8 x i1> , i32 8) -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <8 x float> poison, float [[I1]], i32 0 -; CHECK-NEXT: [[TMP2:%.*]] = insertelement <8 x float> [[TMP1]], float [[I3]], i32 1 -; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x float> [[TMP2]], float [[I5]], i32 2 -; CHECK-NEXT: [[TMP4:%.*]] = insertelement <8 x float> [[TMP3]], float [[I7]], i32 3 -; CHECK-NEXT: [[TMP5:%.*]] = insertelement <8 x float> [[TMP4]], float [[I9]], i32 4 -; CHECK-NEXT: [[TMP6:%.*]] = insertelement <8 x float> [[TMP5]], float [[I11]], i32 5 -; CHECK-NEXT: [[TMP7:%.*]] = insertelement <8 x float> [[TMP6]], float [[I13]], i32 6 -; CHECK-NEXT: [[TMP8:%.*]] = insertelement <8 x float> [[TMP7]], float [[I15]], i32 7 -; CHECK-NEXT: [[TMP9:%.*]] = fsub fast <8 x float> [[TMP8]], [[TMP0]] -; CHECK-NEXT: store <8 x float> [[TMP9]], ptr [[ARRAYIDX2]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[STR]], -4 +; CHECK-NEXT: [[TMP2:%.*]] = call <8 x float> @llvm.experimental.vp.strided.load.v8f32.p0.i64(ptr align 4 [[ARRAYIDX1]], i64 [[TMP1]], <8 x i1> , i32 8) +; CHECK-NEXT: [[TMP3:%.*]] = fsub fast <8 x float> [[TMP2]], [[TMP0]] +; CHECK-NEXT: store <8 x float> [[TMP3]], ptr [[ARRAYIDX2]], align 4 ; CHECK-NEXT: ret void ; entry: -- GitLab From a64975f9660e02f5f6688f8bcc55daf9eaa99fda Mon Sep 17 00:00:00 2001 From: Fehr Mathieu Date: Tue, 5 Mar 2024 14:53:59 +0000 Subject: [PATCH 148/929] [mlir][irdl] Add support for basic structural constraints in tblgen-to-irdl (#82862) --- mlir/include/mlir/IR/CommonTypeConstraints.td | 22 +++--- mlir/test/tblgen-to-irdl/CMathDialect.td | 12 +-- mlir/test/tblgen-to-irdl/TestDialect.td | 74 +++++++++++++++++++ .../tools/tblgen-to-irdl/OpDefinitionsGen.cpp | 48 ++++++++++-- 4 files changed, 134 insertions(+), 22 deletions(-) create mode 100644 mlir/test/tblgen-to-irdl/TestDialect.td diff --git a/mlir/include/mlir/IR/CommonTypeConstraints.td b/mlir/include/mlir/IR/CommonTypeConstraints.td index 03180a687523..af4f13dc0936 100644 --- a/mlir/include/mlir/IR/CommonTypeConstraints.td +++ b/mlir/include/mlir/IR/CommonTypeConstraints.td @@ -168,24 +168,28 @@ def NoneType : Type($_self)">, "none type", BuildableType<"$_builder.getType<::mlir::NoneType>()">; // Any type from the given list -class AnyTypeOf allowedTypes, string summary = "", +class AnyTypeOf allowedTypeList, string summary = "", string cppClassName = "::mlir::Type"> : Type< // Satisfy any of the allowed types' conditions. - Or, + Or, !if(!eq(summary, ""), - !interleave(!foreach(t, allowedTypes, t.summary), " or "), + !interleave(!foreach(t, allowedTypeList, t.summary), " or "), summary), - cppClassName>; + cppClassName> { + list allowedTypes = allowedTypeList; +} // A type that satisfies the constraints of all given types. -class AllOfType allowedTypes, string summary = "", +class AllOfType allowedTypeList, string summary = "", string cppClassName = "::mlir::Type"> : Type< - // Satisfy all of the allowedf types' conditions. - And, + // Satisfy all of the allowed types' conditions. + And, !if(!eq(summary, ""), - !interleave(!foreach(t, allowedTypes, t.summary), " and "), + !interleave(!foreach(t, allowedTypeList, t.summary), " and "), summary), - cppClassName>; + cppClassName> { + list allowedTypes = allowedTypeList; +} // A type that satisfies additional predicates. class ConfinedType predicates, string summary = "", diff --git a/mlir/test/tblgen-to-irdl/CMathDialect.td b/mlir/test/tblgen-to-irdl/CMathDialect.td index 57ae8afbba5e..5b9e756727cb 100644 --- a/mlir/test/tblgen-to-irdl/CMathDialect.td +++ b/mlir/test/tblgen-to-irdl/CMathDialect.td @@ -24,7 +24,7 @@ def CMath_ComplexType : CMath_Type<"ComplexType", "complex"> { } // CHECK: irdl.operation @identity { -// CHECK-NEXT: %0 = irdl.c_pred "(::llvm::isa($_self))" +// CHECK-NEXT: %0 = irdl.base "!cmath.complex" // CHECK-NEXT: irdl.operands() // CHECK-NEXT: irdl.results(%0) // CHECK-NEXT: } @@ -33,9 +33,9 @@ def CMath_IdentityOp : CMath_Op<"identity"> { } // CHECK: irdl.operation @mul { -// CHECK-NEXT: %0 = irdl.c_pred "(::llvm::isa($_self))" -// CHECK-NEXT: %1 = irdl.c_pred "(::llvm::isa($_self))" -// CHECK-NEXT: %2 = irdl.c_pred "(::llvm::isa($_self))" +// CHECK-NEXT: %0 = irdl.base "!cmath.complex" +// CHECK-NEXT: %1 = irdl.base "!cmath.complex" +// CHECK-NEXT: %2 = irdl.base "!cmath.complex" // CHECK-NEXT: irdl.operands(%0, %1) // CHECK-NEXT: irdl.results(%2) // CHECK-NEXT: } @@ -45,8 +45,8 @@ def CMath_MulOp : CMath_Op<"mul"> { } // CHECK: irdl.operation @norm { -// CHECK-NEXT: %0 = irdl.c_pred "(true)" -// CHECK-NEXT: %1 = irdl.c_pred "(::llvm::isa($_self))" +// CHECK-NEXT: %0 = irdl.any +// CHECK-NEXT: %1 = irdl.base "!cmath.complex" // CHECK-NEXT: irdl.operands(%0) // CHECK-NEXT: irdl.results(%1) // CHECK-NEXT: } diff --git a/mlir/test/tblgen-to-irdl/TestDialect.td b/mlir/test/tblgen-to-irdl/TestDialect.td new file mode 100644 index 000000000000..fc40da527db0 --- /dev/null +++ b/mlir/test/tblgen-to-irdl/TestDialect.td @@ -0,0 +1,74 @@ +// RUN: tblgen-to-irdl %s -I=%S/../../include --gen-dialect-irdl-defs --dialect=test | FileCheck %s + +include "mlir/IR/OpBase.td" +include "mlir/IR/AttrTypeBase.td" + +// CHECK-LABEL: irdl.dialect @test { +def Test_Dialect : Dialect { + let name = "test"; +} + +class Test_Type traits = []> +: TypeDef { + let mnemonic = typeMnemonic; +} + +class Test_Op traits = []> + : Op; + +def Test_SingletonAType : Test_Type<"SingletonAType", "singleton_a"> {} +def Test_SingletonBType : Test_Type<"SingletonBType", "singleton_b"> {} +def Test_SingletonCType : Test_Type<"SingletonCType", "singleton_c"> {} + + +// Check that AllOfType is converted correctly. +def Test_AndOp : Test_Op<"and"> { + let arguments = (ins AllOfType<[Test_SingletonAType, AnyType]>:$in); +} +// CHECK-LABEL: irdl.operation @and { +// CHECK-NEXT: %[[v0:[^ ]*]] = irdl.base "!test.singleton_a" +// CHECK-NEXT: %[[v1:[^ ]*]] = irdl.any +// CHECK-NEXT: %[[v2:[^ ]*]] = irdl.all_of(%[[v0]], %[[v1]]) +// CHECK-NEXT: irdl.operands(%[[v2]]) +// CHECK-NEXT: irdl.results() +// CHECK-NEXT: } + + +// Check that AnyType is converted correctly. +def Test_AnyOp : Test_Op<"any"> { + let arguments = (ins AnyType:$in); +} +// CHECK-LABEL: irdl.operation @any { +// CHECK-NEXT: %[[v0:[^ ]*]] = irdl.any +// CHECK-NEXT: irdl.operands(%[[v0]]) +// CHECK-NEXT: irdl.results() +// CHECK-NEXT: } + + +// Check that AnyTypeOf is converted correctly. +def Test_OrOp : Test_Op<"or"> { + let arguments = (ins AnyTypeOf<[Test_SingletonAType, Test_SingletonBType, Test_SingletonCType]>:$in); +} +// CHECK-LABEL: irdl.operation @or { +// CHECK-NEXT: %[[v0:[^ ]*]] = irdl.base "!test.singleton_a" +// CHECK-NEXT: %[[v1:[^ ]*]] = irdl.base "!test.singleton_b" +// CHECK-NEXT: %[[v2:[^ ]*]] = irdl.base "!test.singleton_c" +// CHECK-NEXT: %[[v3:[^ ]*]] = irdl.any_of(%[[v0]], %[[v1]], %[[v2]]) +// CHECK-NEXT: irdl.operands(%[[v3]]) +// CHECK-NEXT: irdl.results() +// CHECK-NEXT: } + + +// Check that variadics and optionals are converted correctly. +def Test_VariadicityOp : Test_Op<"variadicity"> { + let arguments = (ins Variadic:$variadic, + Optional:$optional, + Test_SingletonCType:$required); +} +// CHECK-LABEL: irdl.operation @variadicity { +// CHECK-NEXT: %[[v0:[^ ]*]] = irdl.base "!test.singleton_a" +// CHECK-NEXT: %[[v1:[^ ]*]] = irdl.base "!test.singleton_b" +// CHECK-NEXT: %[[v2:[^ ]*]] = irdl.base "!test.singleton_c" +// CHECK-NEXT: irdl.operands(variadic %[[v0]], optional %[[v1]], %[[v2]]) +// CHECK-NEXT: irdl.results() +// CHECK-NEXT: } diff --git a/mlir/tools/tblgen-to-irdl/OpDefinitionsGen.cpp b/mlir/tools/tblgen-to-irdl/OpDefinitionsGen.cpp index ba5bf4d9d4ab..a55f3539f31d 100644 --- a/mlir/tools/tblgen-to-irdl/OpDefinitionsGen.cpp +++ b/mlir/tools/tblgen-to-irdl/OpDefinitionsGen.cpp @@ -39,15 +39,49 @@ llvm::cl::opt selectedDialect("dialect", llvm::cl::desc("The dialect to gen for"), llvm::cl::cat(dialectGenCat), llvm::cl::Required); -irdl::CPredOp createConstraint(OpBuilder &builder, - NamedTypeConstraint namedConstraint) { +Value createConstraint(OpBuilder &builder, tblgen::Constraint constraint) { MLIRContext *ctx = builder.getContext(); - // Build the constraint as a string. - std::string constraint = - namedConstraint.constraint.getPredicate().getCondition(); + const Record &predRec = constraint.getDef(); + + if (predRec.isSubClassOf("Variadic") || predRec.isSubClassOf("Optional")) + return createConstraint(builder, predRec.getValueAsDef("baseType")); + + if (predRec.getName() == "AnyType") { + auto op = builder.create(UnknownLoc::get(ctx)); + return op.getOutput(); + } + + if (predRec.isSubClassOf("TypeDef")) { + std::string typeName = ("!" + predRec.getValueAsString("typeName")).str(); + auto op = builder.create(UnknownLoc::get(ctx), + StringAttr::get(ctx, typeName)); + return op.getOutput(); + } + + if (predRec.isSubClassOf("AnyTypeOf")) { + std::vector constraints; + for (Record *child : predRec.getValueAsListOfDefs("allowedTypes")) { + constraints.push_back( + createConstraint(builder, tblgen::Constraint(child))); + } + auto op = builder.create(UnknownLoc::get(ctx), constraints); + return op.getOutput(); + } + + if (predRec.isSubClassOf("AllOfType")) { + std::vector constraints; + for (Record *child : predRec.getValueAsListOfDefs("allowedTypes")) { + constraints.push_back( + createConstraint(builder, tblgen::Constraint(child))); + } + auto op = builder.create(UnknownLoc::get(ctx), constraints); + return op.getOutput(); + } + + std::string condition = constraint.getPredicate().getCondition(); // Build a CPredOp to match the C constraint built. irdl::CPredOp op = builder.create( - UnknownLoc::get(ctx), StringAttr::get(ctx, constraint)); + UnknownLoc::get(ctx), StringAttr::get(ctx, condition)); return op; } @@ -74,7 +108,7 @@ irdl::OperationOp createIRDLOperation(OpBuilder &builder, SmallVector operands; SmallVector variadicity; for (const NamedTypeConstraint &namedCons : namedCons) { - auto operand = createConstraint(consBuilder, namedCons); + auto operand = createConstraint(consBuilder, namedCons.constraint); operands.push_back(operand); irdl::VariadicityAttr var; -- GitLab From fac791d4e1d879c19bfbbbfa89dc4186f56af34c Mon Sep 17 00:00:00 2001 From: Benoit Jacob Date: Tue, 5 Mar 2024 09:59:28 -0500 Subject: [PATCH 149/929] Export `LLVM_VERSION_MAJOR` CMake variable as a directory property (#83346) This allows users who include `llvm-project` as a subrepository to access the `LLVM_VERSION_MAJOR` variable on par with if they were relying on an installed LLVM and `FindLLVM.cmake`. They just need to do something like: ``` get_directory_property(LLVM_VERSION_MAJOR DIRECTORY "third_party/llvm-project/llvm" LLVM_VERSION_MAJOR) ``` Context: https://github.com/openxla/iree/pull/16606 -- like other projects with similar needs that I found by some googling, our work-around had been to rely on the CMake cached variable `CLANG_EXECUTABLE_VERSION`. Being cached, it over time inevitably ended up having a wrong value. --- llvm/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt index 651f17879fad..111c8cfa15d8 100644 --- a/llvm/CMakeLists.txt +++ b/llvm/CMakeLists.txt @@ -28,6 +28,8 @@ if(NOT DEFINED LLVM_VERSION_SUFFIX) set(LLVM_VERSION_SUFFIX git) endif() +set_directory_properties(PROPERTIES LLVM_VERSION_MAJOR "${LLVM_VERSION_MAJOR}") + if (NOT PACKAGE_VERSION) set(PACKAGE_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}${LLVM_VERSION_SUFFIX}") -- GitLab From a691f65a845d6d5e639a83e8240a2543663c103a Mon Sep 17 00:00:00 2001 From: "Oleksandr \"Alex\" Zinenko" Date: Tue, 5 Mar 2024 16:09:59 +0100 Subject: [PATCH 150/929] [mlir][py] better support for arith.constant construction (#83259) Arithmetic constants for vector types can be constructed from objects implementing Python buffer protocol such as `array.array`. Note that until Python 3.12, there is no typing support for buffer protocol implementers, so the annotations use array explicitly. --- mlir/python/mlir/dialects/arith.py | 23 +++++++++++-- mlir/test/python/dialects/arith_dialect.py | 38 ++++++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/mlir/python/mlir/dialects/arith.py b/mlir/python/mlir/dialects/arith.py index 61c6917393f1..83a50c7ef244 100644 --- a/mlir/python/mlir/dialects/arith.py +++ b/mlir/python/mlir/dialects/arith.py @@ -5,6 +5,8 @@ from ._arith_ops_gen import * from ._arith_ops_gen import _Dialect from ._arith_enum_gen import * +from array import array as _array +from typing import overload try: from ..ir import * @@ -43,13 +45,30 @@ def _is_float_type(type: Type): class ConstantOp(ConstantOp): """Specialization for the constant op class.""" + @overload + def __init__(self, value: Attribute, *, loc=None, ip=None): + ... + + @overload def __init__( - self, result: Type, value: Union[int, float, Attribute], *, loc=None, ip=None + self, result: Type, value: Union[int, float, _array], *, loc=None, ip=None ): + ... + + def __init__(self, result, value, *, loc=None, ip=None): + if value is None: + assert isinstance(result, Attribute) + super().__init__(result, loc=loc, ip=ip) + return + if isinstance(value, int): super().__init__(IntegerAttr.get(result, value), loc=loc, ip=ip) elif isinstance(value, float): super().__init__(FloatAttr.get(result, value), loc=loc, ip=ip) + elif isinstance(value, _array) and value.typecode in ["i", "l"]: + super().__init__(DenseIntElementsAttr.get(value, type=result)) + elif isinstance(value, _array) and value.typecode in ["f", "d"]: + super().__init__(DenseFPElementsAttr.get(value, type=result)) else: super().__init__(value, loc=loc, ip=ip) @@ -79,6 +98,6 @@ class ConstantOp(ConstantOp): def constant( - result: Type, value: Union[int, float, Attribute], *, loc=None, ip=None + result: Type, value: Union[int, float, Attribute, _array], *, loc=None, ip=None ) -> Value: return _get_op_result_or_op_results(ConstantOp(result, value, loc=loc, ip=ip)) diff --git a/mlir/test/python/dialects/arith_dialect.py b/mlir/test/python/dialects/arith_dialect.py index 8bb80eed2b81..ef0e1620bba9 100644 --- a/mlir/test/python/dialects/arith_dialect.py +++ b/mlir/test/python/dialects/arith_dialect.py @@ -4,6 +4,7 @@ from functools import partialmethod from mlir.ir import * import mlir.dialects.arith as arith import mlir.dialects.func as func +from array import array def run(f): @@ -92,3 +93,40 @@ def testArithValue(): b = a * a # CHECK: ArithValue(%2 = arith.mulf %cst_1, %cst_1 : f64) print(b) + + +# CHECK-LABEL: TEST: testArrayConstantConstruction +@run +def testArrayConstantConstruction(): + with Context(), Location.unknown(): + module = Module.create() + with InsertionPoint(module.body): + i32_array = array("i", [1, 2, 3, 4]) + i32 = IntegerType.get_signless(32) + vec_i32 = VectorType.get([2, 2], i32) + arith.constant(vec_i32, i32_array) + arith.ConstantOp(vec_i32, DenseIntElementsAttr.get(i32_array, type=vec_i32)) + + i64_array = array("l", [5, 6, 7, 8]) + i64 = IntegerType.get_signless(64) + vec_i64 = VectorType.get([1, 4], i64) + arith.constant(vec_i64, i64_array) + arith.ConstantOp(vec_i64, DenseIntElementsAttr.get(i64_array, type=vec_i64)) + + f32_array = array("f", [1.0, 2.0, 3.0, 4.0]) + f32 = F32Type.get() + vec_f32 = VectorType.get([4, 1], f32) + arith.constant(vec_f32, f32_array) + arith.ConstantOp(vec_f32, DenseFPElementsAttr.get(f32_array, type=vec_f32)) + + f64_array = array("d", [1.0, 2.0, 3.0, 4.0]) + f64 = F64Type.get() + vec_f64 = VectorType.get([2, 1, 2], f64) + arith.constant(vec_f64, f64_array) + arith.ConstantOp(vec_f64, DenseFPElementsAttr.get(f64_array, type=vec_f64)) + + # CHECK-COUNT-2: arith.constant dense<[{{\[}}1, 2], [3, 4]]> : vector<2x2xi32> + # CHECK-COUNT-2: arith.constant dense<[{{\[}}5, 6, 7, 8]]> : vector<1x4xi64> + # CHECK-COUNT-2: arith.constant dense<[{{\[}}1.000000e+00], [2.000000e+00], [3.000000e+00], [4.000000e+00]]> : vector<4x1xf32> + # CHECK-COUNT-2: arith.constant dense<[{{\[}}[1.000000e+00, 2.000000e+00]], [{{\[}}3.000000e+00, 4.000000e+00]]]> : vector<2x1x2xf64> + print(module) -- GitLab From 2fe81edef6f0b35abffbbc59b649b30ea9c15a62 Mon Sep 17 00:00:00 2001 From: Jeremy Morse Date: Mon, 4 Mar 2024 14:51:56 +0000 Subject: [PATCH 151/929] [NFC][RemoveDIs] Insert instruction using iterators in Transforms/ As part of the RemoveDIs project we need LLVM to insert instructions using iterators wherever possible, so that the iterators can carry a bit of debug-info. This commit implements some of that by updating the contents of llvm/lib/Transforms/Utils to always use iterator-versions of instruction constructors. There are two general flavours of update: * Almost all call-sites just call getIterator on an instruction * Several make use of an existing iterator (scenarios where the code is actually significant for debug-info) The underlying logic is that any call to getFirstInsertionPt or similar APIs that identify the start of a block need to have that iterator passed directly to the insertion function, without being converted to a bare Instruction pointer along the way. Noteworthy changes: * FindInsertedValue now takes an optional iterator rather than an instruction pointer, as we need to always insert with iterators, * I've added a few iterator-taking versions of some value-tracking and DomTree methods -- they just unwrap the iterator. These are purely convenience methods to avoid extra syntax in some passes. * A few calls to getNextNode become std::next instead (to keep in the theme of using iterators for positions), * SeparateConstOffsetFromGEP has it's insertion-position field changed. Noteworthy because it's not a purely localised spelling change. All this should be NFC. --- llvm/include/llvm/Analysis/ValueTracking.h | 25 ++- llvm/include/llvm/IR/Dominators.h | 4 + .../llvm/Transforms/Scalar/Reassociate.h | 4 +- llvm/lib/Analysis/ValueTracking.cpp | 16 +- llvm/lib/Transforms/Coroutines/CoroElide.cpp | 5 +- llvm/lib/Transforms/Coroutines/CoroSplit.cpp | 2 +- llvm/lib/Transforms/Coroutines/Coroutines.cpp | 8 +- llvm/lib/Transforms/IPO/ArgumentPromotion.cpp | 5 +- llvm/lib/Transforms/IPO/Attributor.cpp | 8 +- .../Transforms/IPO/AttributorAttributes.cpp | 54 ++--- .../IPO/DeadArgumentElimination.cpp | 9 +- llvm/lib/Transforms/IPO/GlobalOpt.cpp | 38 ++-- llvm/lib/Transforms/IPO/IROutliner.cpp | 2 +- llvm/lib/Transforms/IPO/OpenMPOpt.cpp | 34 +-- .../lib/Transforms/IPO/WholeProgramDevirt.cpp | 8 +- .../InstCombine/InstCombineCalls.cpp | 26 ++- .../InstCombine/InstCombineCompares.cpp | 5 +- .../InstCombineLoadStoreAlloca.cpp | 2 +- .../Transforms/InstCombine/InstCombinePHI.cpp | 3 +- .../InstCombine/InstCombineVectorOps.cpp | 3 +- .../Instrumentation/AddressSanitizer.cpp | 2 +- .../Instrumentation/DataFlowSanitizer.cpp | 210 ++++++++++-------- llvm/lib/Transforms/Instrumentation/KCFI.cpp | 4 +- llvm/lib/Transforms/ObjCARC/ObjCARC.cpp | 10 +- llvm/lib/Transforms/ObjCARC/ObjCARC.h | 10 +- .../Transforms/ObjCARC/ObjCARCContract.cpp | 25 ++- llvm/lib/Transforms/ObjCARC/ObjCARCOpts.cpp | 42 ++-- .../Scalar/CorrelatedValuePropagation.cpp | 15 +- .../Scalar/DeadStoreElimination.cpp | 3 +- llvm/lib/Transforms/Scalar/DivRemPairs.cpp | 6 +- llvm/lib/Transforms/Scalar/GVN.cpp | 16 +- llvm/lib/Transforms/Scalar/GuardWidening.cpp | 76 ++++--- llvm/lib/Transforms/Scalar/IndVarSimplify.cpp | 21 +- .../Transforms/Scalar/InferAddressSpaces.cpp | 2 +- llvm/lib/Transforms/Scalar/JumpThreading.cpp | 18 +- llvm/lib/Transforms/Scalar/LICM.cpp | 2 +- llvm/lib/Transforms/Scalar/LoopFlatten.cpp | 2 +- .../Transforms/Scalar/LoopIdiomRecognize.cpp | 8 +- .../Transforms/Scalar/LoopLoadElimination.cpp | 12 +- .../Transforms/Scalar/LoopStrengthReduce.cpp | 37 ++- .../lib/Transforms/Scalar/NaryReassociate.cpp | 4 +- llvm/lib/Transforms/Scalar/NewGVN.cpp | 2 +- .../lib/Transforms/Scalar/PlaceSafepoints.cpp | 6 +- llvm/lib/Transforms/Scalar/Reassociate.cpp | 55 ++--- .../Scalar/RewriteStatepointsForGC.cpp | 40 ++-- llvm/lib/Transforms/Scalar/SROA.cpp | 5 +- .../Scalar/SeparateConstOffsetFromGEP.cpp | 19 +- .../Transforms/Scalar/SimpleLoopUnswitch.cpp | 13 +- .../Scalar/TailRecursionElimination.cpp | 14 +- .../Transforms/Vectorize/LoopVectorize.cpp | 16 +- 50 files changed, 526 insertions(+), 430 deletions(-) diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h index 9cfb7af9dba0..3970efba18cc 100644 --- a/llvm/include/llvm/Analysis/ValueTracking.h +++ b/llvm/include/llvm/Analysis/ValueTracking.h @@ -596,10 +596,11 @@ Value *isBytewiseValue(Value *V, const DataLayout &DL); /// indexed is already around as a register, for example if it were inserted /// directly into the aggregate. /// -/// If InsertBefore is not null, this function will duplicate (modified) +/// If InsertBefore is not empty, this function will duplicate (modified) /// insertvalues when a part of a nested struct is extracted. -Value *FindInsertedValue(Value *V, ArrayRef idx_range, - Instruction *InsertBefore = nullptr); +Value *FindInsertedValue( + Value *V, ArrayRef idx_range, + std::optional InsertBefore = std::nullopt); /// Analyze the specified pointer to see if it can be expressed as a base /// pointer plus a constant offset. Return the base and offset to the caller. @@ -793,6 +794,15 @@ bool isSafeToSpeculativelyExecute(const Instruction *I, const DominatorTree *DT = nullptr, const TargetLibraryInfo *TLI = nullptr); +inline bool +isSafeToSpeculativelyExecute(const Instruction *I, BasicBlock::iterator CtxI, + AssumptionCache *AC = nullptr, + const DominatorTree *DT = nullptr, + const TargetLibraryInfo *TLI = nullptr) { + // Take an iterator, and unwrap it into an Instruction *. + return isSafeToSpeculativelyExecute(I, &*CtxI, AC, DT, TLI); +} + /// This returns the same result as isSafeToSpeculativelyExecute if Opcode is /// the actual opcode of Inst. If the provided and actual opcode differ, the /// function (virtually) overrides the opcode of Inst with the provided @@ -1019,6 +1029,15 @@ bool isGuaranteedNotToBePoison(const Value *V, AssumptionCache *AC = nullptr, const DominatorTree *DT = nullptr, unsigned Depth = 0); +inline bool isGuaranteedNotToBePoison(const Value *V, AssumptionCache *AC, + BasicBlock::iterator CtxI, + const DominatorTree *DT = nullptr, + unsigned Depth = 0) { + // Takes an iterator as a position, passes down to Instruction * + // implementation. + return isGuaranteedNotToBePoison(V, AC, &*CtxI, DT, Depth); +} + /// Returns true if V cannot be undef, but may be poison. bool isGuaranteedNotToBeUndef(const Value *V, AssumptionCache *AC = nullptr, const Instruction *CtxI = nullptr, diff --git a/llvm/include/llvm/IR/Dominators.h b/llvm/include/llvm/IR/Dominators.h index 42db4c4ea3a5..287f419f893d 100644 --- a/llvm/include/llvm/IR/Dominators.h +++ b/llvm/include/llvm/IR/Dominators.h @@ -189,9 +189,13 @@ class DominatorTree : public DominatorTreeBase { /// like unreachable code or trivial phi cycles). /// * Invoke Defs only dominate uses in their default destination. bool dominates(const Value *Def, const Use &U) const; + /// Return true if value Def dominates all possible uses inside instruction /// User. Same comments as for the Use-based API apply. bool dominates(const Value *Def, const Instruction *User) const; + bool dominates(const Value *Def, BasicBlock::iterator User) const { + return dominates(Def, &*User); + } /// Returns true if Def would dominate a use in any instruction in BB. /// If Def is an instruction in BB, then Def does not dominate BB. diff --git a/llvm/include/llvm/Transforms/Scalar/Reassociate.h b/llvm/include/llvm/Transforms/Scalar/Reassociate.h index 7e47f8ae5d81..f3a2e0f4380e 100644 --- a/llvm/include/llvm/Transforms/Scalar/Reassociate.h +++ b/llvm/include/llvm/Transforms/Scalar/Reassociate.h @@ -110,9 +110,9 @@ private: SmallVectorImpl &Ops); Value *OptimizeXor(Instruction *I, SmallVectorImpl &Ops); - bool CombineXorOpnd(Instruction *I, reassociate::XorOpnd *Opnd1, + bool CombineXorOpnd(BasicBlock::iterator It, reassociate::XorOpnd *Opnd1, APInt &ConstOpnd, Value *&Res); - bool CombineXorOpnd(Instruction *I, reassociate::XorOpnd *Opnd1, + bool CombineXorOpnd(BasicBlock::iterator It, reassociate::XorOpnd *Opnd1, reassociate::XorOpnd *Opnd2, APInt &ConstOpnd, Value *&Res); Value *buildMinimalMultiplyDAG(IRBuilderBase &Builder, diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index 69b271042039..8d33b1c37b19 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -5536,10 +5536,10 @@ Value *llvm::isBytewiseValue(Value *V, const DataLayout &DL) { // indices from Idxs that should be left out when inserting into the resulting // struct. To is the result struct built so far, new insertvalue instructions // build on that. -static Value *BuildSubAggregate(Value *From, Value* To, Type *IndexedType, +static Value *BuildSubAggregate(Value *From, Value *To, Type *IndexedType, SmallVectorImpl &Idxs, unsigned IdxSkip, - Instruction *InsertBefore) { + BasicBlock::iterator InsertBefore) { StructType *STy = dyn_cast(IndexedType); if (STy) { // Save the original To argument so we can modify it @@ -5596,8 +5596,7 @@ static Value *BuildSubAggregate(Value *From, Value* To, Type *IndexedType, // // All inserted insertvalue instructions are inserted before InsertBefore static Value *BuildSubAggregate(Value *From, ArrayRef idx_range, - Instruction *InsertBefore) { - assert(InsertBefore && "Must have someplace to insert!"); + BasicBlock::iterator InsertBefore) { Type *IndexedType = ExtractValueInst::getIndexedType(From->getType(), idx_range); Value *To = PoisonValue::get(IndexedType); @@ -5613,8 +5612,9 @@ static Value *BuildSubAggregate(Value *From, ArrayRef idx_range, /// /// If InsertBefore is not null, this function will duplicate (modified) /// insertvalues when a part of a nested struct is extracted. -Value *llvm::FindInsertedValue(Value *V, ArrayRef idx_range, - Instruction *InsertBefore) { +Value * +llvm::FindInsertedValue(Value *V, ArrayRef idx_range, + std::optional InsertBefore) { // Nothing to index? Just return V then (this is useful at the end of our // recursion). if (idx_range.empty()) @@ -5653,7 +5653,7 @@ Value *llvm::FindInsertedValue(Value *V, ArrayRef idx_range, // which allows the unused 0,0 element from the nested struct to be // removed. return BuildSubAggregate(V, ArrayRef(idx_range.begin(), req_idx), - InsertBefore); + *InsertBefore); } // This insert value inserts something else than what we are looking for. @@ -5661,7 +5661,7 @@ Value *llvm::FindInsertedValue(Value *V, ArrayRef idx_range, // looking for, then. if (*req_idx != *i) return FindInsertedValue(I->getAggregateOperand(), idx_range, - InsertBefore); + *InsertBefore); } // If we end up here, the indices of the insertvalue match with those // requested (though possibly only partially). Now we recursively look at diff --git a/llvm/lib/Transforms/Coroutines/CoroElide.cpp b/llvm/lib/Transforms/Coroutines/CoroElide.cpp index 2f4083028ae0..d356a6d2e575 100644 --- a/llvm/lib/Transforms/Coroutines/CoroElide.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroElide.cpp @@ -141,8 +141,9 @@ static std::unique_ptr getOrCreateLogFile() { void Lowerer::elideHeapAllocations(Function *F, uint64_t FrameSize, Align FrameAlign, AAResults &AA) { LLVMContext &C = F->getContext(); - auto *InsertPt = - getFirstNonAllocaInTheEntryBlock(CoroIds.front()->getFunction()); + BasicBlock::iterator InsertPt = + getFirstNonAllocaInTheEntryBlock(CoroIds.front()->getFunction()) + ->getIterator(); // Replacing llvm.coro.alloc with false will suppress dynamic // allocation as it is expected for the frontend to generate the code that diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp index 90d40242ff2e..99675aa495f5 100644 --- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp @@ -1423,7 +1423,7 @@ static bool simplifySuspendPoint(CoroSuspendInst *Suspend, // No longer need a call to coro.resume or coro.destroy. if (auto *Invoke = dyn_cast(CB)) { - BranchInst::Create(Invoke->getNormalDest(), Invoke); + BranchInst::Create(Invoke->getNormalDest(), Invoke->getIterator()); } // Grab the CalledValue from CB before erasing the CallInstr. diff --git a/llvm/lib/Transforms/Coroutines/Coroutines.cpp b/llvm/lib/Transforms/Coroutines/Coroutines.cpp index eef5543bae24..7bd151ed4dc1 100644 --- a/llvm/lib/Transforms/Coroutines/Coroutines.cpp +++ b/llvm/lib/Transforms/Coroutines/Coroutines.cpp @@ -55,7 +55,7 @@ Value *coro::LowererBase::makeSubFnCall(Value *Arg, int Index, assert(Index >= CoroSubFnInst::IndexFirst && Index < CoroSubFnInst::IndexLast && "makeSubFnCall: Index value out of range"); - return CallInst::Create(Fn, {Arg, IndexVal}, "", InsertPt); + return CallInst::Create(Fn, {Arg, IndexVal}, "", InsertPt->getIterator()); } // NOTE: Must be sorted! @@ -157,8 +157,8 @@ static CoroSaveInst *createCoroSave(CoroBeginInst *CoroBegin, CoroSuspendInst *SuspendInst) { Module *M = SuspendInst->getModule(); auto *Fn = Intrinsic::getDeclaration(M, Intrinsic::coro_save); - auto *SaveInst = - cast(CallInst::Create(Fn, CoroBegin, "", SuspendInst)); + auto *SaveInst = cast( + CallInst::Create(Fn, CoroBegin, "", SuspendInst->getIterator())); assert(!SuspendInst->getCoroSave()); SuspendInst->setArgOperand(0, SaveInst); return SaveInst; @@ -362,7 +362,7 @@ void coro::Shape::buildFrom(Function &F) { // calls, but that messes with our invariants. Re-insert the // bitcast and ignore this type mismatch. if (CastInst::isBitCastable(SrcTy, *RI)) { - auto BCI = new BitCastInst(*SI, *RI, "", Suspend); + auto BCI = new BitCastInst(*SI, *RI, "", Suspend->getIterator()); SI->set(BCI); continue; } diff --git a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp index 8058282c4225..e89ec353487e 100644 --- a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp +++ b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp @@ -266,9 +266,10 @@ doPromotion(Function *F, FunctionAnalysisManager &FAM, CallBase *NewCS = nullptr; if (InvokeInst *II = dyn_cast(&CB)) { NewCS = InvokeInst::Create(NF, II->getNormalDest(), II->getUnwindDest(), - Args, OpBundles, "", &CB); + Args, OpBundles, "", CB.getIterator()); } else { - auto *NewCall = CallInst::Create(NF, Args, OpBundles, "", &CB); + auto *NewCall = + CallInst::Create(NF, Args, OpBundles, "", CB.getIterator()); NewCall->setTailCallKind(cast(&CB)->getTailCallKind()); NewCS = NewCall; } diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp index 72a2aadc204b..e3920b9e1d2b 100644 --- a/llvm/lib/Transforms/IPO/Attributor.cpp +++ b/llvm/lib/Transforms/IPO/Attributor.cpp @@ -3123,12 +3123,12 @@ ChangeStatus Attributor::rewriteFunctionSignatures( // Create a new call or invoke instruction to replace the old one. CallBase *NewCB; if (InvokeInst *II = dyn_cast(OldCB)) { - NewCB = - InvokeInst::Create(NewFn, II->getNormalDest(), II->getUnwindDest(), - NewArgOperands, OperandBundleDefs, "", OldCB); + NewCB = InvokeInst::Create(NewFn, II->getNormalDest(), + II->getUnwindDest(), NewArgOperands, + OperandBundleDefs, "", OldCB->getIterator()); } else { auto *NewCI = CallInst::Create(NewFn, NewArgOperands, OperandBundleDefs, - "", OldCB); + "", OldCB->getIterator()); NewCI->setTailCallKind(cast(OldCB)->getTailCallKind()); NewCB = NewCI; } diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp index 585364dd7aa2..488a6f0bb153 100644 --- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp +++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp @@ -6128,8 +6128,8 @@ struct AAValueSimplifyImpl : AAValueSimplify { return TypedV; if (CtxI && V.getType()->canLosslesslyBitCastTo(&Ty)) return Check ? &V - : BitCastInst::CreatePointerBitCastOrAddrSpaceCast(&V, &Ty, - "", CtxI); + : BitCastInst::CreatePointerBitCastOrAddrSpaceCast( + &V, &Ty, "", CtxI->getIterator()); return nullptr; } @@ -6731,8 +6731,9 @@ struct AAHeapToStackFunction final : public AAHeapToStack { Size = SizeOffsetPair.Size; } - Instruction *IP = - AI.MoveAllocaIntoEntry ? &F->getEntryBlock().front() : AI.CB; + BasicBlock::iterator IP = AI.MoveAllocaIntoEntry + ? F->getEntryBlock().begin() + : AI.CB->getIterator(); Align Alignment(1); if (MaybeAlign RetAlign = AI.CB->getRetAlign()) @@ -6753,7 +6754,7 @@ struct AAHeapToStackFunction final : public AAHeapToStack { if (Alloca->getType() != AI.CB->getType()) Alloca = BitCastInst::CreatePointerBitCastOrAddrSpaceCast( - Alloca, AI.CB->getType(), "malloc_cast", AI.CB); + Alloca, AI.CB->getType(), "malloc_cast", AI.CB->getIterator()); auto *I8Ty = Type::getInt8Ty(F->getContext()); auto *InitVal = getInitialValueOfAllocation(AI.CB, TLI, I8Ty); @@ -7450,10 +7451,10 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { /// The values needed are taken from the arguments of \p F starting at /// position \p ArgNo. static void createInitialization(Type *PrivType, Value &Base, Function &F, - unsigned ArgNo, Instruction &IP) { + unsigned ArgNo, BasicBlock::iterator IP) { assert(PrivType && "Expected privatizable type!"); - IRBuilder IRB(&IP); + IRBuilder IRB(IP->getParent(), IP); const DataLayout &DL = F.getParent()->getDataLayout(); // Traverse the type, build GEPs and stores. @@ -7462,17 +7463,17 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) { Value *Ptr = constructPointer(&Base, PrivStructLayout->getElementOffset(u), IRB); - new StoreInst(F.getArg(ArgNo + u), Ptr, &IP); + new StoreInst(F.getArg(ArgNo + u), Ptr, IP); } } else if (auto *PrivArrayType = dyn_cast(PrivType)) { Type *PointeeTy = PrivArrayType->getElementType(); uint64_t PointeeTySize = DL.getTypeStoreSize(PointeeTy); for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) { Value *Ptr = constructPointer(&Base, u * PointeeTySize, IRB); - new StoreInst(F.getArg(ArgNo + u), Ptr, &IP); + new StoreInst(F.getArg(ArgNo + u), Ptr, IP); } } else { - new StoreInst(F.getArg(ArgNo), &Base, &IP); + new StoreInst(F.getArg(ArgNo), &Base, IP); } } @@ -7495,7 +7496,7 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { Type *PointeeTy = PrivStructType->getElementType(u); Value *Ptr = constructPointer(Base, PrivStructLayout->getElementOffset(u), IRB); - LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP); + LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator()); L->setAlignment(Alignment); ReplacementValues.push_back(L); } @@ -7504,12 +7505,12 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { uint64_t PointeeTySize = DL.getTypeStoreSize(PointeeTy); for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) { Value *Ptr = constructPointer(Base, u * PointeeTySize, IRB); - LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP); + LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator()); L->setAlignment(Alignment); ReplacementValues.push_back(L); } } else { - LoadInst *L = new LoadInst(PrivType, Base, "", IP); + LoadInst *L = new LoadInst(PrivType, Base, "", IP->getIterator()); L->setAlignment(Alignment); ReplacementValues.push_back(L); } @@ -7549,13 +7550,13 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { [=](const Attributor::ArgumentReplacementInfo &ARI, Function &ReplacementFn, Function::arg_iterator ArgIt) { BasicBlock &EntryBB = ReplacementFn.getEntryBlock(); - Instruction *IP = &*EntryBB.getFirstInsertionPt(); + BasicBlock::iterator IP = EntryBB.getFirstInsertionPt(); const DataLayout &DL = IP->getModule()->getDataLayout(); unsigned AS = DL.getAllocaAddrSpace(); Instruction *AI = new AllocaInst(*PrivatizableType, AS, Arg->getName() + ".priv", IP); createInitialization(*PrivatizableType, *AI, ReplacementFn, - ArgIt->getArgNo(), *IP); + ArgIt->getArgNo(), IP); if (AI->getType() != Arg->getType()) AI = BitCastInst::CreatePointerBitCastOrAddrSpaceCast( @@ -12313,10 +12314,10 @@ struct AAIndirectCallInfoCallSite : public AAIndirectCallInfo { Value *FP = CB->getCalledOperand(); if (FP->getType()->getPointerAddressSpace()) FP = new AddrSpaceCastInst(FP, PointerType::get(FP->getType(), 0), - FP->getName() + ".as0", CB); + FP->getName() + ".as0", CB->getIterator()); bool CBIsVoid = CB->getType()->isVoidTy(); - Instruction *IP = CB; + BasicBlock::iterator IP = CB->getIterator(); FunctionType *CSFT = CB->getFunctionType(); SmallVector CSArgs(CB->arg_begin(), CB->arg_end()); @@ -12336,8 +12337,9 @@ struct AAIndirectCallInfoCallSite : public AAIndirectCallInfo { promoteCall(*CB, NewCallee, nullptr); return ChangeStatus::CHANGED; } - Instruction *NewCall = CallInst::Create(FunctionCallee(CSFT, NewCallee), - CSArgs, CB->getName(), CB); + Instruction *NewCall = + CallInst::Create(FunctionCallee(CSFT, NewCallee), CSArgs, + CB->getName(), CB->getIterator()); if (!CBIsVoid) A.changeAfterManifest(IRPosition::callsite_returned(*CB), *NewCall); A.deleteAfterManifest(*CB); @@ -12372,11 +12374,11 @@ struct AAIndirectCallInfoCallSite : public AAIndirectCallInfo { A.registerManifestAddedBasicBlock(*CBBB); auto *SplitTI = cast(LastCmp->getNextNode()); BasicBlock *ElseBB; - if (IP == CB) { + if (&*IP == CB) { ElseBB = BasicBlock::Create(ThenTI->getContext(), "", ThenTI->getFunction(), CBBB); A.registerManifestAddedBasicBlock(*ElseBB); - IP = BranchInst::Create(CBBB, ElseBB); + IP = BranchInst::Create(CBBB, ElseBB)->getIterator(); SplitTI->replaceUsesOfWith(CBBB, ElseBB); } else { ElseBB = IP->getParent(); @@ -12390,7 +12392,7 @@ struct AAIndirectCallInfoCallSite : public AAIndirectCallInfo { NewCall = &cast(promoteCall(*CBClone, NewCallee, &RetBC)); } else { NewCall = CallInst::Create(FunctionCallee(CSFT, NewCallee), CSArgs, - CB->getName(), ThenTI); + CB->getName(), ThenTI->getIterator()); } NewCalls.push_back({NewCall, RetBC}); } @@ -12416,7 +12418,7 @@ struct AAIndirectCallInfoCallSite : public AAIndirectCallInfo { } else { auto *CBClone = cast(CB->clone()); CBClone->setName(CB->getName()); - CBClone->insertBefore(IP); + CBClone->insertBefore(*IP->getParent(), IP); NewCalls.push_back({CBClone, nullptr}); AttachCalleeMetadata(*CBClone); } @@ -12425,7 +12427,7 @@ struct AAIndirectCallInfoCallSite : public AAIndirectCallInfo { if (!CBIsVoid) { auto *PHI = PHINode::Create(CB->getType(), NewCalls.size(), CB->getName() + ".phi", - &*CB->getParent()->getFirstInsertionPt()); + CB->getParent()->getFirstInsertionPt()); for (auto &It : NewCalls) { CallBase *NewCall = It.first; Instruction *CallRet = It.second ? It.second : It.first; @@ -12783,9 +12785,11 @@ struct AAAllocationInfoImpl : public AAAllocationInfo { auto *NumBytesToValue = ConstantInt::get(I->getContext(), APInt(32, NumBytesToAllocate)); + BasicBlock::iterator insertPt = AI->getIterator(); + insertPt = std::next(insertPt); AllocaInst *NewAllocaInst = new AllocaInst(CharType, AI->getAddressSpace(), NumBytesToValue, - AI->getAlign(), AI->getName(), AI->getNextNode()); + AI->getAlign(), AI->getName(), insertPt); if (A.changeAfterManifest(IRPosition::inst(*AI), *NewAllocaInst)) return ChangeStatus::CHANGED; diff --git a/llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp b/llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp index 4f65748c19e6..f19031383f5c 100644 --- a/llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp +++ b/llvm/lib/Transforms/IPO/DeadArgumentElimination.cpp @@ -204,9 +204,9 @@ bool DeadArgumentEliminationPass::deleteDeadVarargs(Function &F) { CallBase *NewCB = nullptr; if (InvokeInst *II = dyn_cast(CB)) { NewCB = InvokeInst::Create(NF, II->getNormalDest(), II->getUnwindDest(), - Args, OpBundles, "", CB); + Args, OpBundles, "", CB->getIterator()); } else { - NewCB = CallInst::Create(NF, Args, OpBundles, "", CB); + NewCB = CallInst::Create(NF, Args, OpBundles, "", CB->getIterator()); cast(NewCB)->setTailCallKind( cast(CB)->getTailCallKind()); } @@ -946,7 +946,7 @@ bool DeadArgumentEliminationPass::removeDeadStuffFromFunction(Function *F) { NewCB = InvokeInst::Create(NF, II->getNormalDest(), II->getUnwindDest(), Args, OpBundles, "", CB.getParent()); } else { - NewCB = CallInst::Create(NFTy, NF, Args, OpBundles, "", &CB); + NewCB = CallInst::Create(NFTy, NF, Args, OpBundles, "", CB.getIterator()); cast(NewCB)->setTailCallKind( cast(&CB)->getTailCallKind()); } @@ -1070,7 +1070,8 @@ bool DeadArgumentEliminationPass::removeDeadStuffFromFunction(Function *F) { } // Replace the return instruction with one returning the new return // value (possibly 0 if we became void). - auto *NewRet = ReturnInst::Create(F->getContext(), RetVal, RI); + auto *NewRet = + ReturnInst::Create(F->getContext(), RetVal, RI->getIterator()); NewRet->setDebugLoc(RI->getDebugLoc()); RI->eraseFromParent(); } diff --git a/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/llvm/lib/Transforms/IPO/GlobalOpt.cpp index c92b5d82fc85..da714c9a7570 100644 --- a/llvm/lib/Transforms/IPO/GlobalOpt.cpp +++ b/llvm/lib/Transforms/IPO/GlobalOpt.cpp @@ -953,7 +953,7 @@ OptimizeGlobalAddressOfAllocation(GlobalVariable *GV, CallInst *CI, GV->getContext(), !isa(SI->getValueOperand())), InitBool, false, Align(1), SI->getOrdering(), - SI->getSyncScopeID(), SI); + SI->getSyncScopeID(), SI->getIterator()); SI->eraseFromParent(); continue; } @@ -970,7 +970,8 @@ OptimizeGlobalAddressOfAllocation(GlobalVariable *GV, CallInst *CI, // Replace the cmp X, 0 with a use of the bool value. Value *LV = new LoadInst(InitBool->getValueType(), InitBool, InitBool->getName() + ".val", false, Align(1), - LI->getOrdering(), LI->getSyncScopeID(), LI); + LI->getOrdering(), LI->getSyncScopeID(), + LI->getIterator()); InitBoolUsed = true; switch (ICI->getPredicate()) { default: llvm_unreachable("Unknown ICmp Predicate!"); @@ -982,7 +983,7 @@ OptimizeGlobalAddressOfAllocation(GlobalVariable *GV, CallInst *CI, break; case ICmpInst::ICMP_ULE: case ICmpInst::ICMP_EQ: - LV = BinaryOperator::CreateNot(LV, "notinit", ICI); + LV = BinaryOperator::CreateNot(LV, "notinit", ICI->getIterator()); break; case ICmpInst::ICMP_NE: case ICmpInst::ICMP_UGT: @@ -1260,9 +1261,10 @@ static bool TryToShrinkGlobalToBoolean(GlobalVariable *GV, Constant *OtherVal) { if (LoadInst *LI = dyn_cast(StoredVal)) { assert(LI->getOperand(0) == GV && "Not a copy!"); // Insert a new load, to preserve the saved value. - StoreVal = new LoadInst(NewGV->getValueType(), NewGV, - LI->getName() + ".b", false, Align(1), - LI->getOrdering(), LI->getSyncScopeID(), LI); + StoreVal = + new LoadInst(NewGV->getValueType(), NewGV, LI->getName() + ".b", + false, Align(1), LI->getOrdering(), + LI->getSyncScopeID(), LI->getIterator()); } else { assert((isa(StoredVal) || isa(StoredVal)) && "This is not a form that we understand!"); @@ -1272,19 +1274,19 @@ static bool TryToShrinkGlobalToBoolean(GlobalVariable *GV, Constant *OtherVal) { } StoreInst *NSI = new StoreInst(StoreVal, NewGV, false, Align(1), SI->getOrdering(), - SI->getSyncScopeID(), SI); + SI->getSyncScopeID(), SI->getIterator()); NSI->setDebugLoc(SI->getDebugLoc()); } else { // Change the load into a load of bool then a select. LoadInst *LI = cast(UI); - LoadInst *NLI = new LoadInst(NewGV->getValueType(), NewGV, - LI->getName() + ".b", false, Align(1), - LI->getOrdering(), LI->getSyncScopeID(), LI); + LoadInst *NLI = new LoadInst( + NewGV->getValueType(), NewGV, LI->getName() + ".b", false, Align(1), + LI->getOrdering(), LI->getSyncScopeID(), LI->getIterator()); Instruction *NSI; if (IsOneZero) - NSI = new ZExtInst(NLI, LI->getType(), "", LI); + NSI = new ZExtInst(NLI, LI->getType(), "", LI->getIterator()); else - NSI = SelectInst::Create(NLI, OtherVal, InitVal, "", LI); + NSI = SelectInst::Create(NLI, OtherVal, InitVal, "", LI->getIterator()); NSI->takeName(LI); // Since LI is split into two instructions, NLI and NSI both inherit the // same DebugLoc @@ -1462,14 +1464,14 @@ processInternalGlobal(GlobalVariable *GV, const GlobalStatus &GS, const DataLayout &DL = GV->getParent()->getDataLayout(); LLVM_DEBUG(dbgs() << "LOCALIZING GLOBAL: " << *GV << "\n"); - Instruction &FirstI = const_cast(*GS.AccessingFunction - ->getEntryBlock().begin()); + BasicBlock::iterator FirstI = + GS.AccessingFunction->getEntryBlock().begin().getNonConst(); Type *ElemTy = GV->getValueType(); // FIXME: Pass Global's alignment when globals have alignment - AllocaInst *Alloca = new AllocaInst(ElemTy, DL.getAllocaAddrSpace(), nullptr, - GV->getName(), &FirstI); + AllocaInst *Alloca = new AllocaInst(ElemTy, DL.getAllocaAddrSpace(), + nullptr, GV->getName(), FirstI); if (!isa(GV->getInitializer())) - new StoreInst(GV->getInitializer(), Alloca, &FirstI); + new StoreInst(GV->getInitializer(), Alloca, FirstI); GV->replaceAllUsesWith(Alloca); GV->eraseFromParent(); @@ -1859,7 +1861,7 @@ static void RemovePreallocated(Function *F) { assert((isa(CB) || isa(CB)) && "Unknown indirect call type"); - CallBase *NewCB = CallBase::Create(CB, OpBundles, CB); + CallBase *NewCB = CallBase::Create(CB, OpBundles, CB->getIterator()); CB->replaceAllUsesWith(NewCB); NewCB->takeName(CB); CB->eraseFromParent(); diff --git a/llvm/lib/Transforms/IPO/IROutliner.cpp b/llvm/lib/Transforms/IPO/IROutliner.cpp index 48470bc71ff3..03d4d503b80a 100644 --- a/llvm/lib/Transforms/IPO/IROutliner.cpp +++ b/llvm/lib/Transforms/IPO/IROutliner.cpp @@ -1501,7 +1501,7 @@ CallInst *replaceCalledFunction(Module &M, OutlinableRegion &Region) { << *AggFunc << " with new set of arguments\n"); // Create the new call instruction and erase the old one. Call = CallInst::Create(AggFunc->getFunctionType(), AggFunc, NewCallArgs, "", - Call); + Call->getIterator()); // It is possible that the call to the outlined function is either the first // instruction is in the new block, the last instruction, or both. If either diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp index 77ca36d64029..eea9399127e8 100644 --- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp +++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp @@ -1146,17 +1146,18 @@ private: const DataLayout &DL = M.getDataLayout(); AllocaInst *AllocaI = new AllocaInst( I.getType(), DL.getAllocaAddrSpace(), nullptr, - I.getName() + ".seq.output.alloc", &OuterFn->front().front()); + I.getName() + ".seq.output.alloc", OuterFn->front().begin()); // Emit a store instruction in the sequential BB to update the // value. - new StoreInst(&I, AllocaI, SeqStartBB->getTerminator()); + new StoreInst(&I, AllocaI, SeqStartBB->getTerminator()->getIterator()); // Emit a load instruction and replace the use of the output value // with it. for (Instruction *UsrI : OutsideUsers) { - LoadInst *LoadI = new LoadInst( - I.getType(), AllocaI, I.getName() + ".seq.output.load", UsrI); + LoadInst *LoadI = new LoadInst(I.getType(), AllocaI, + I.getName() + ".seq.output.load", + UsrI->getIterator()); UsrI->replaceUsesOfWith(&I, LoadI); } } @@ -1261,7 +1262,8 @@ private: ++U) Args.push_back(CI->getArgOperand(U)); - CallInst *NewCI = CallInst::Create(FT, Callee, Args, "", CI); + CallInst *NewCI = + CallInst::Create(FT, Callee, Args, "", CI->getIterator()); if (CI->getDebugLoc()) NewCI->setDebugLoc(CI->getDebugLoc()); @@ -1739,8 +1741,8 @@ private: Args.push_back(Arg.get()); Args.push_back(Handle); - CallInst *IssueCallsite = - CallInst::Create(IssueDecl, Args, /*NameStr=*/"", &RuntimeCall); + CallInst *IssueCallsite = CallInst::Create(IssueDecl, Args, /*NameStr=*/"", + RuntimeCall.getIterator()); OMPInfoCache.setCallingConvention(IssueDecl, IssueCallsite); RuntimeCall.eraseFromParent(); @@ -1755,7 +1757,7 @@ private: Handle // handle to wait on. }; CallInst *WaitCallsite = CallInst::Create( - WaitDecl, WaitParams, /*NameStr=*/"", &WaitMovementPoint); + WaitDecl, WaitParams, /*NameStr=*/"", WaitMovementPoint.getIterator()); OMPInfoCache.setCallingConvention(WaitDecl, WaitCallsite); return true; @@ -4025,11 +4027,12 @@ struct AAKernelInfoFunction : AAKernelInfo { static_cast(AddressSpace::Shared)); // Emit a store instruction to update the value. - new StoreInst(&I, SharedMem, RegionEndBB->getTerminator()); + new StoreInst(&I, SharedMem, + RegionEndBB->getTerminator()->getIterator()); - LoadInst *LoadI = new LoadInst(I.getType(), SharedMem, - I.getName() + ".guarded.output.load", - RegionBarrierBB->getTerminator()); + LoadInst *LoadI = new LoadInst( + I.getType(), SharedMem, I.getName() + ".guarded.output.load", + RegionBarrierBB->getTerminator()->getIterator()); // Emit a load instruction and replace uses of the output value. for (Use *U : OutsideUses) @@ -4082,8 +4085,9 @@ struct AAKernelInfoFunction : AAKernelInfo { // Second barrier ensures workers have read broadcast values. if (HasBroadcastValues) { - CallInst *Barrier = CallInst::Create(BarrierFn, {Ident, Tid}, "", - RegionBarrierBB->getTerminator()); + CallInst *Barrier = + CallInst::Create(BarrierFn, {Ident, Tid}, "", + RegionBarrierBB->getTerminator()->getIterator()); Barrier->setDebugLoc(DL); OMPInfoCache.setCallingConvention(BarrierFn, Barrier); } @@ -4488,7 +4492,7 @@ struct AAKernelInfoFunction : AAKernelInfo { Type *VoidPtrTy = PointerType::getUnqual(Ctx); Instruction *WorkFnAI = new AllocaInst(VoidPtrTy, DL.getAllocaAddrSpace(), nullptr, - "worker.work_fn.addr", &Kernel->getEntryBlock().front()); + "worker.work_fn.addr", Kernel->getEntryBlock().begin()); WorkFnAI->setDebugLoc(DLoc); OMPInfoCache.OMPBuilder.updateToLocation( diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp index 75f7de4290a7..cf13c91a8677 100644 --- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp +++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp @@ -434,7 +434,7 @@ struct VirtualCallSite { emitRemark(OptName, TargetName, OREGetter); CB.replaceAllUsesWith(New); if (auto *II = dyn_cast(&CB)) { - BranchInst::Create(II->getNormalDest(), &CB); + BranchInst::Create(II->getNormalDest(), CB.getIterator()); II->getUnwindDest()->removePredecessor(II->getParent()); } CB.eraseFromParent(); @@ -861,7 +861,7 @@ void llvm::updatePublicTypeTestCalls(Module &M, auto *CI = cast(U.getUser()); auto *NewCI = CallInst::Create( TypeTestFunc, {CI->getArgOperand(0), CI->getArgOperand(1)}, - std::nullopt, "", CI); + std::nullopt, "", CI->getIterator()); CI->replaceAllUsesWith(NewCI); CI->eraseFromParent(); } @@ -1225,8 +1225,8 @@ void DevirtModule::applySingleImplDevirt(VTableSlotInfo &SlotInfo, CB.setMetadata(LLVMContext::MD_callees, nullptr); if (CB.getCalledOperand() && CB.getOperandBundle(LLVMContext::OB_ptrauth)) { - auto *NewCS = - CallBase::removeOperandBundle(&CB, LLVMContext::OB_ptrauth, &CB); + auto *NewCS = CallBase::removeOperandBundle( + &CB, LLVMContext::OB_ptrauth, CB.getIterator()); CB.replaceAllUsesWith(NewCS); // Schedule for deletion at the end of pass run. CallsWithPtrAuthBundleRemoved.push_back(&CB); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index 89ca40f1c20d..d2756b0d4d54 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -3140,28 +3140,30 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { if (match(BO0, m_VecReverse(m_Value(X)))) { // rev(binop rev(X), rev(Y)) --> binop X, Y if (match(BO1, m_VecReverse(m_Value(Y)))) - return replaceInstUsesWith(CI, - BinaryOperator::CreateWithCopiedFlags( - OldBinOp->getOpcode(), X, Y, OldBinOp, - OldBinOp->getName(), II)); + return replaceInstUsesWith(CI, BinaryOperator::CreateWithCopiedFlags( + OldBinOp->getOpcode(), X, Y, + OldBinOp, OldBinOp->getName(), + II->getIterator())); // rev(binop rev(X), BO1Splat) --> binop X, BO1Splat if (isSplatValue(BO1)) - return replaceInstUsesWith(CI, - BinaryOperator::CreateWithCopiedFlags( - OldBinOp->getOpcode(), X, BO1, - OldBinOp, OldBinOp->getName(), II)); + return replaceInstUsesWith(CI, BinaryOperator::CreateWithCopiedFlags( + OldBinOp->getOpcode(), X, BO1, + OldBinOp, OldBinOp->getName(), + II->getIterator())); } // rev(binop BO0Splat, rev(Y)) --> binop BO0Splat, Y if (match(BO1, m_VecReverse(m_Value(Y))) && isSplatValue(BO0)) - return replaceInstUsesWith(CI, BinaryOperator::CreateWithCopiedFlags( - OldBinOp->getOpcode(), BO0, Y, - OldBinOp, OldBinOp->getName(), II)); + return replaceInstUsesWith(CI, + BinaryOperator::CreateWithCopiedFlags( + OldBinOp->getOpcode(), BO0, Y, OldBinOp, + OldBinOp->getName(), II->getIterator())); } // rev(unop rev(X)) --> unop X if (match(Vec, m_OneUse(m_UnOp(m_VecReverse(m_Value(X)))))) { auto *OldUnOp = cast(Vec); auto *NewUnOp = UnaryOperator::CreateWithCopiedFlags( - OldUnOp->getOpcode(), X, OldUnOp, OldUnOp->getName(), II); + OldUnOp->getOpcode(), X, OldUnOp, OldUnOp->getName(), + II->getIterator()); return replaceInstUsesWith(CI, NewUnOp); } break; diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp index 1cebab8203ea..fc2688f425bb 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp @@ -556,8 +556,9 @@ static Value *rewriteGEPAsOffset(Value *Start, Value *Base, // Create empty phi nodes. This avoids cyclic dependencies when creating // the remaining instructions. if (auto *PHI = dyn_cast(Val)) - NewInsts[PHI] = PHINode::Create(IndexType, PHI->getNumIncomingValues(), - PHI->getName() + ".idx", PHI); + NewInsts[PHI] = + PHINode::Create(IndexType, PHI->getNumIncomingValues(), + PHI->getName() + ".idx", PHI->getIterator()); } IRBuilder<> Builder(Base->getContext()); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp index a222889842f5..c70872c12917 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp @@ -374,7 +374,7 @@ void PointerReplacer::replace(Instruction *I) { } else if (auto *PHI = dyn_cast(I)) { Type *NewTy = getReplacement(PHI->getIncomingValue(0))->getType(); auto *NewPHI = PHINode::Create(NewTy, PHI->getNumIncomingValues(), - PHI->getName(), PHI); + PHI->getName(), PHI->getIterator()); for (unsigned int I = 0; I < PHI->getNumIncomingValues(); ++I) NewPHI->addIncoming(getReplacement(PHI->getIncomingValue(I)), PHI->getIncomingBlock(I)); diff --git a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp index 192ccbbcb7c7..46bca4b722a0 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp @@ -1205,7 +1205,8 @@ Instruction *InstCombinerImpl::SliceUpIllegalIntegerPHI(PHINode &FirstPhi) { // Otherwise, Create the new PHI node for this user. EltPHI = PHINode::Create(Ty, PN->getNumIncomingValues(), - PN->getName()+".off"+Twine(Offset), PN); + PN->getName() + ".off" + Twine(Offset), + PN->getIterator()); assert(EltPHI->getType() != PN->getType() && "Truncate didn't shrink phi?"); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp index 18ab510aae7f..3c4c0f35eb6d 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp @@ -1263,7 +1263,8 @@ static Instruction *foldInsSequenceIntoSplat(InsertElementInst &InsElt) { PoisonValue *PoisonVec = PoisonValue::get(VecTy); Constant *Zero = ConstantInt::get(Int64Ty, 0); if (!cast(FirstIE->getOperand(2))->isZero()) - FirstIE = InsertElementInst::Create(PoisonVec, SplatVal, Zero, "", &InsElt); + FirstIE = InsertElementInst::Create(PoisonVec, SplatVal, Zero, "", + InsElt.getIterator()); // Splat from element 0, but replace absent elements with poison in the mask. SmallVector Mask(NumElements, 0); diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp index 5d5c4ea57ed5..f22f53b8cd8f 100644 --- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -1879,7 +1879,7 @@ void ModuleAddressSanitizer::poisonOneInitializer(Function &GlobalInit, // Add calls to unpoison all globals before each return instruction. for (auto &BB : GlobalInit) if (ReturnInst *RI = dyn_cast(BB.getTerminator())) - CallInst::Create(AsanUnpoisonGlobals, "", RI); + CallInst::Create(AsanUnpoisonGlobals, "", RI->getIterator()); } void ModuleAddressSanitizer::createInitializerPoisonCalls( diff --git a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp index 2ba127bba6f6..9a2ec7618c1a 100644 --- a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp @@ -516,10 +516,12 @@ class DataFlowSanitizer { const MemoryMapParams *MapParams; Value *getShadowOffset(Value *Addr, IRBuilder<> &IRB); - Value *getShadowAddress(Value *Addr, Instruction *Pos); - Value *getShadowAddress(Value *Addr, Instruction *Pos, Value *ShadowOffset); - std::pair - getShadowOriginAddress(Value *Addr, Align InstAlignment, Instruction *Pos); + Value *getShadowAddress(Value *Addr, BasicBlock::iterator Pos); + Value *getShadowAddress(Value *Addr, BasicBlock::iterator Pos, + Value *ShadowOffset); + std::pair getShadowOriginAddress(Value *Addr, + Align InstAlignment, + BasicBlock::iterator Pos); bool isInstrumented(const Function *F); bool isInstrumented(const GlobalAlias *GA); bool isForceZeroLabels(const Function *F); @@ -536,7 +538,7 @@ class DataFlowSanitizer { /// Advances \p OriginAddr to point to the next 32-bit origin and then loads /// from it. Returns the origin's loaded value. - Value *loadNextOrigin(Instruction *Pos, Align OriginAlign, + Value *loadNextOrigin(BasicBlock::iterator Pos, Align OriginAlign, Value **OriginAddr); /// Returns whether the given load byte size is amenable to inlined @@ -647,18 +649,18 @@ struct DFSanFunction { /// When Zero is nullptr, it uses ZeroPrimitiveShadow. Otherwise it can be /// zeros with other bitwidths. Value *combineOrigins(const std::vector &Shadows, - const std::vector &Origins, Instruction *Pos, - ConstantInt *Zero = nullptr); + const std::vector &Origins, + BasicBlock::iterator Pos, ConstantInt *Zero = nullptr); Value *getShadow(Value *V); void setShadow(Instruction *I, Value *Shadow); /// Generates IR to compute the union of the two given shadows, inserting it /// before Pos. The combined value is with primitive type. - Value *combineShadows(Value *V1, Value *V2, Instruction *Pos); + Value *combineShadows(Value *V1, Value *V2, BasicBlock::iterator Pos); /// Combines the shadow values of V1 and V2, then converts the combined value /// with primitive type into a shadow value with the original type T. Value *combineShadowsThenConvert(Type *T, Value *V1, Value *V2, - Instruction *Pos); + BasicBlock::iterator Pos); Value *combineOperandShadows(Instruction *Inst); /// Generates IR to load shadow and origin corresponding to bytes [\p @@ -670,11 +672,11 @@ struct DFSanFunction { /// current stack if the returned shadow is tainted. std::pair loadShadowOrigin(Value *Addr, uint64_t Size, Align InstAlignment, - Instruction *Pos); + BasicBlock::iterator Pos); void storePrimitiveShadowOrigin(Value *Addr, uint64_t Size, Align InstAlignment, Value *PrimitiveShadow, - Value *Origin, Instruction *Pos); + Value *Origin, BasicBlock::iterator Pos); /// Applies PrimitiveShadow to all primitive subtypes of T, returning /// the expanded shadow value. /// @@ -682,7 +684,7 @@ struct DFSanFunction { /// EFP([n x T], PS) = [n x EFP(T,PS)] /// EFP(other types, PS) = PS Value *expandFromPrimitiveShadow(Type *T, Value *PrimitiveShadow, - Instruction *Pos); + BasicBlock::iterator Pos); /// Collapses Shadow into a single primitive shadow value, unioning all /// primitive shadow values in the process. Returns the final primitive /// shadow value. @@ -690,10 +692,10 @@ struct DFSanFunction { /// CTP({V1,V2, ...}) = UNION(CFP(V1,PS),CFP(V2,PS),...) /// CTP([V1,V2,...]) = UNION(CFP(V1,PS),CFP(V2,PS),...) /// CTP(other types, PS) = PS - Value *collapseToPrimitiveShadow(Value *Shadow, Instruction *Pos); + Value *collapseToPrimitiveShadow(Value *Shadow, BasicBlock::iterator Pos); void storeZeroPrimitiveShadow(Value *Addr, uint64_t Size, Align ShadowAlign, - Instruction *Pos); + BasicBlock::iterator Pos); Align getShadowAlign(Align InstAlignment); @@ -724,7 +726,7 @@ private: std::pair loadShadowFast(Value *ShadowAddr, Value *OriginAddr, uint64_t Size, Align ShadowAlign, Align OriginAlign, Value *FirstOrigin, - Instruction *Pos); + BasicBlock::iterator Pos); Align getOriginAlign(Align InstAlignment); @@ -760,8 +762,9 @@ private: /// for untainted sinks. /// * Use __dfsan_maybe_store_origin if there are too many origin store /// instrumentations. - void storeOrigin(Instruction *Pos, Value *Addr, uint64_t Size, Value *Shadow, - Value *Origin, Value *StoreOriginAddr, Align InstAlignment); + void storeOrigin(BasicBlock::iterator Pos, Value *Addr, uint64_t Size, + Value *Shadow, Value *Origin, Value *StoreOriginAddr, + Align InstAlignment); /// Convert a scalar value to an i1 by comparing with 0. Value *convertToBool(Value *V, IRBuilder<> &IRB, const Twine &Name = ""); @@ -774,7 +777,8 @@ private: /// shadow always has primitive type. std::pair loadShadowOriginSansLoadTracking(Value *Addr, uint64_t Size, - Align InstAlignment, Instruction *Pos); + Align InstAlignment, + BasicBlock::iterator Pos); int NumOriginStores = 0; }; @@ -975,7 +979,7 @@ bool DFSanFunction::shouldInstrumentWithCall() { } Value *DFSanFunction::expandFromPrimitiveShadow(Type *T, Value *PrimitiveShadow, - Instruction *Pos) { + BasicBlock::iterator Pos) { Type *ShadowTy = DFS.getShadowTy(T); if (!isa(ShadowTy) && !isa(ShadowTy)) @@ -984,7 +988,7 @@ Value *DFSanFunction::expandFromPrimitiveShadow(Type *T, Value *PrimitiveShadow, if (DFS.isZeroShadow(PrimitiveShadow)) return DFS.getZeroShadow(ShadowTy); - IRBuilder<> IRB(Pos); + IRBuilder<> IRB(Pos->getParent(), Pos); SmallVector Indices; Value *Shadow = UndefValue::get(ShadowTy); Shadow = expandFromPrimitiveShadowRecursive(Shadow, Indices, ShadowTy, @@ -1025,7 +1029,7 @@ Value *DFSanFunction::collapseToPrimitiveShadow(Value *Shadow, } Value *DFSanFunction::collapseToPrimitiveShadow(Value *Shadow, - Instruction *Pos) { + BasicBlock::iterator Pos) { Type *ShadowTy = Shadow->getType(); if (!isa(ShadowTy) && !isa(ShadowTy)) return Shadow; @@ -1035,7 +1039,7 @@ Value *DFSanFunction::collapseToPrimitiveShadow(Value *Shadow, if (CS && DT.dominates(CS, Pos)) return CS; - IRBuilder<> IRB(Pos); + IRBuilder<> IRB(Pos->getParent(), Pos); Value *PrimitiveShadow = collapseToPrimitiveShadow(Shadow, IRB); // Caches the converted primitive shadow value. CS = PrimitiveShadow; @@ -1760,14 +1764,14 @@ bool DataFlowSanitizer::runImpl( // instrumentation. if (ClDebugNonzeroLabels) { for (Value *V : DFSF.NonZeroChecks) { - Instruction *Pos; + BasicBlock::iterator Pos; if (Instruction *I = dyn_cast(V)) - Pos = I->getNextNode(); + Pos = std::next(I->getIterator()); else - Pos = &DFSF.F->getEntryBlock().front(); + Pos = DFSF.F->getEntryBlock().begin(); while (isa(Pos) || isa(Pos)) - Pos = Pos->getNextNode(); - IRBuilder<> IRB(Pos); + Pos = std::next(Pos->getIterator()); + IRBuilder<> IRB(Pos->getParent(), Pos); Value *PrimitiveShadow = DFSF.collapseToPrimitiveShadow(V, Pos); Value *Ne = IRB.CreateICmpNE(PrimitiveShadow, DFSF.DFS.ZeroPrimitiveShadow); @@ -1912,9 +1916,9 @@ Value *DataFlowSanitizer::getShadowOffset(Value *Addr, IRBuilder<> &IRB) { std::pair DataFlowSanitizer::getShadowOriginAddress(Value *Addr, Align InstAlignment, - Instruction *Pos) { + BasicBlock::iterator Pos) { // Returns ((Addr & shadow_mask) + origin_base - shadow_base) & ~4UL - IRBuilder<> IRB(Pos); + IRBuilder<> IRB(Pos->getParent(), Pos); Value *ShadowOffset = getShadowOffset(Addr, IRB); Value *ShadowLong = ShadowOffset; uint64_t ShadowBase = MapParams->ShadowBase; @@ -1944,27 +1948,30 @@ DataFlowSanitizer::getShadowOriginAddress(Value *Addr, Align InstAlignment, return std::make_pair(ShadowPtr, OriginPtr); } -Value *DataFlowSanitizer::getShadowAddress(Value *Addr, Instruction *Pos, +Value *DataFlowSanitizer::getShadowAddress(Value *Addr, + BasicBlock::iterator Pos, Value *ShadowOffset) { - IRBuilder<> IRB(Pos); + IRBuilder<> IRB(Pos->getParent(), Pos); return IRB.CreateIntToPtr(ShadowOffset, PrimitiveShadowPtrTy); } -Value *DataFlowSanitizer::getShadowAddress(Value *Addr, Instruction *Pos) { - IRBuilder<> IRB(Pos); +Value *DataFlowSanitizer::getShadowAddress(Value *Addr, + BasicBlock::iterator Pos) { + IRBuilder<> IRB(Pos->getParent(), Pos); Value *ShadowOffset = getShadowOffset(Addr, IRB); return getShadowAddress(Addr, Pos, ShadowOffset); } Value *DFSanFunction::combineShadowsThenConvert(Type *T, Value *V1, Value *V2, - Instruction *Pos) { + BasicBlock::iterator Pos) { Value *PrimitiveValue = combineShadows(V1, V2, Pos); return expandFromPrimitiveShadow(T, PrimitiveValue, Pos); } // Generates IR to compute the union of the two given shadows, inserting it // before Pos. The combined value is with primitive type. -Value *DFSanFunction::combineShadows(Value *V1, Value *V2, Instruction *Pos) { +Value *DFSanFunction::combineShadows(Value *V1, Value *V2, + BasicBlock::iterator Pos) { if (DFS.isZeroShadow(V1)) return collapseToPrimitiveShadow(V2, Pos); if (DFS.isZeroShadow(V2)) @@ -2002,7 +2009,7 @@ Value *DFSanFunction::combineShadows(Value *V1, Value *V2, Instruction *Pos) { Value *PV1 = collapseToPrimitiveShadow(V1, Pos); Value *PV2 = collapseToPrimitiveShadow(V2, Pos); - IRBuilder<> IRB(Pos); + IRBuilder<> IRB(Pos->getParent(), Pos); CCS.Block = Pos->getParent(); CCS.Shadow = IRB.CreateOr(PV1, PV2); @@ -2031,9 +2038,11 @@ Value *DFSanFunction::combineOperandShadows(Instruction *Inst) { Value *Shadow = getShadow(Inst->getOperand(0)); for (unsigned I = 1, N = Inst->getNumOperands(); I < N; ++I) - Shadow = combineShadows(Shadow, getShadow(Inst->getOperand(I)), Inst); + Shadow = combineShadows(Shadow, getShadow(Inst->getOperand(I)), + Inst->getIterator()); - return expandFromPrimitiveShadow(Inst->getType(), Shadow, Inst); + return expandFromPrimitiveShadow(Inst->getType(), Shadow, + Inst->getIterator()); } void DFSanVisitor::visitInstOperands(Instruction &I) { @@ -2044,7 +2053,8 @@ void DFSanVisitor::visitInstOperands(Instruction &I) { Value *DFSanFunction::combineOrigins(const std::vector &Shadows, const std::vector &Origins, - Instruction *Pos, ConstantInt *Zero) { + BasicBlock::iterator Pos, + ConstantInt *Zero) { assert(Shadows.size() == Origins.size()); size_t Size = Origins.size(); if (Size == 0) @@ -2063,7 +2073,7 @@ Value *DFSanFunction::combineOrigins(const std::vector &Shadows, } Value *OpShadow = Shadows[I]; Value *PrimitiveShadow = collapseToPrimitiveShadow(OpShadow, Pos); - IRBuilder<> IRB(Pos); + IRBuilder<> IRB(Pos->getParent(), Pos); Value *Cond = IRB.CreateICmpNE(PrimitiveShadow, Zero); Origin = IRB.CreateSelect(Cond, OpOrigin, Origin); } @@ -2078,7 +2088,7 @@ Value *DFSanFunction::combineOperandOrigins(Instruction *Inst) { Shadows[I] = getShadow(Inst->getOperand(I)); Origins[I] = getOrigin(Inst->getOperand(I)); } - return combineOrigins(Shadows, Origins, Inst); + return combineOrigins(Shadows, Origins, Inst->getIterator()); } void DFSanVisitor::visitInstOperandOrigins(Instruction &I) { @@ -2129,9 +2139,10 @@ bool DFSanFunction::useCallbackLoadLabelAndOrigin(uint64_t Size, return Alignment < MinOriginAlignment || !DFS.hasLoadSizeForFastPath(Size); } -Value *DataFlowSanitizer::loadNextOrigin(Instruction *Pos, Align OriginAlign, +Value *DataFlowSanitizer::loadNextOrigin(BasicBlock::iterator Pos, + Align OriginAlign, Value **OriginAddr) { - IRBuilder<> IRB(Pos); + IRBuilder<> IRB(Pos->getParent(), Pos); *OriginAddr = IRB.CreateGEP(OriginTy, *OriginAddr, ConstantInt::get(IntptrTy, 1)); return IRB.CreateAlignedLoad(OriginTy, *OriginAddr, OriginAlign); @@ -2139,7 +2150,7 @@ Value *DataFlowSanitizer::loadNextOrigin(Instruction *Pos, Align OriginAlign, std::pair DFSanFunction::loadShadowFast( Value *ShadowAddr, Value *OriginAddr, uint64_t Size, Align ShadowAlign, - Align OriginAlign, Value *FirstOrigin, Instruction *Pos) { + Align OriginAlign, Value *FirstOrigin, BasicBlock::iterator Pos) { const bool ShouldTrackOrigins = DFS.shouldTrackOrigins(); const uint64_t ShadowSize = Size * DFS.ShadowWidthBytes; @@ -2163,7 +2174,7 @@ std::pair DFSanFunction::loadShadowFast( Type *WideShadowTy = ShadowSize == 4 ? Type::getInt32Ty(*DFS.Ctx) : Type::getInt64Ty(*DFS.Ctx); - IRBuilder<> IRB(Pos); + IRBuilder<> IRB(Pos->getParent(), Pos); Value *CombinedWideShadow = IRB.CreateAlignedLoad(WideShadowTy, ShadowAddr, ShadowAlign); @@ -2225,14 +2236,14 @@ std::pair DFSanFunction::loadShadowFast( } std::pair DFSanFunction::loadShadowOriginSansLoadTracking( - Value *Addr, uint64_t Size, Align InstAlignment, Instruction *Pos) { + Value *Addr, uint64_t Size, Align InstAlignment, BasicBlock::iterator Pos) { const bool ShouldTrackOrigins = DFS.shouldTrackOrigins(); // Non-escaped loads. if (AllocaInst *AI = dyn_cast(Addr)) { const auto SI = AllocaShadowMap.find(AI); if (SI != AllocaShadowMap.end()) { - IRBuilder<> IRB(Pos); + IRBuilder<> IRB(Pos->getParent(), Pos); Value *ShadowLI = IRB.CreateLoad(DFS.PrimitiveShadowTy, SI->second); const auto OI = AllocaOriginMap.find(AI); assert(!ShouldTrackOrigins || OI != AllocaOriginMap.end()); @@ -2267,7 +2278,7 @@ std::pair DFSanFunction::loadShadowOriginSansLoadTracking( // tracking. if (ShouldTrackOrigins && useCallbackLoadLabelAndOrigin(Size, InstAlignment)) { - IRBuilder<> IRB(Pos); + IRBuilder<> IRB(Pos->getParent(), Pos); CallInst *Call = IRB.CreateCall(DFS.DFSanLoadLabelAndOriginFn, {Addr, ConstantInt::get(DFS.IntptrTy, Size)}); @@ -2286,7 +2297,7 @@ std::pair DFSanFunction::loadShadowOriginSansLoadTracking( const Align OriginAlign = getOriginAlign(InstAlignment); Value *Origin = nullptr; if (ShouldTrackOrigins) { - IRBuilder<> IRB(Pos); + IRBuilder<> IRB(Pos->getParent(), Pos); Origin = IRB.CreateAlignedLoad(DFS.OriginTy, OriginAddr, OriginAlign); } @@ -2299,7 +2310,7 @@ std::pair DFSanFunction::loadShadowOriginSansLoadTracking( return {LI, Origin}; } case 2: { - IRBuilder<> IRB(Pos); + IRBuilder<> IRB(Pos->getParent(), Pos); Value *ShadowAddr1 = IRB.CreateGEP(DFS.PrimitiveShadowTy, ShadowAddr, ConstantInt::get(DFS.IntptrTy, 1)); Value *Load = @@ -2315,23 +2326,22 @@ std::pair DFSanFunction::loadShadowOriginSansLoadTracking( return loadShadowFast(ShadowAddr, OriginAddr, Size, ShadowAlign, OriginAlign, Origin, Pos); - IRBuilder<> IRB(Pos); + IRBuilder<> IRB(Pos->getParent(), Pos); CallInst *FallbackCall = IRB.CreateCall( DFS.DFSanUnionLoadFn, {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)}); FallbackCall->addRetAttr(Attribute::ZExt); return {FallbackCall, Origin}; } -std::pair DFSanFunction::loadShadowOrigin(Value *Addr, - uint64_t Size, - Align InstAlignment, - Instruction *Pos) { +std::pair +DFSanFunction::loadShadowOrigin(Value *Addr, uint64_t Size, Align InstAlignment, + BasicBlock::iterator Pos) { Value *PrimitiveShadow, *Origin; std::tie(PrimitiveShadow, Origin) = loadShadowOriginSansLoadTracking(Addr, Size, InstAlignment, Pos); if (DFS.shouldTrackOrigins()) { if (ClTrackOrigins == 2) { - IRBuilder<> IRB(Pos); + IRBuilder<> IRB(Pos->getParent(), Pos); auto *ConstantShadow = dyn_cast(PrimitiveShadow); if (!ConstantShadow || !ConstantShadow->isZeroValue()) Origin = updateOriginIfTainted(PrimitiveShadow, Origin, IRB); @@ -2397,8 +2407,11 @@ void DFSanVisitor::visitLoadInst(LoadInst &LI) { if (LI.isAtomic()) LI.setOrdering(addAcquireOrdering(LI.getOrdering())); - Instruction *AfterLi = LI.getNextNode(); - Instruction *Pos = LI.isAtomic() ? LI.getNextNode() : &LI; + BasicBlock::iterator AfterLi = std::next(LI.getIterator()); + BasicBlock::iterator Pos = LI.getIterator(); + if (LI.isAtomic()) + Pos = std::next(Pos); + std::vector Shadows; std::vector Origins; Value *PrimitiveShadow, *Origin; @@ -2431,14 +2444,14 @@ void DFSanVisitor::visitLoadInst(LoadInst &LI) { } if (ClEventCallbacks) { - IRBuilder<> IRB(Pos); + IRBuilder<> IRB(Pos->getParent(), Pos); Value *Addr = LI.getPointerOperand(); CallInst *CI = IRB.CreateCall(DFSF.DFS.DFSanLoadCallbackFn, {PrimitiveShadow, Addr}); CI->addParamAttr(0, Attribute::ZExt); } - IRBuilder<> IRB(AfterLi); + IRBuilder<> IRB(AfterLi->getParent(), AfterLi); DFSF.addReachesFunctionCallbacksIfEnabled(IRB, LI, &LI); } @@ -2510,14 +2523,14 @@ Value *DFSanFunction::convertToBool(Value *V, IRBuilder<> &IRB, return IRB.CreateICmpNE(V, ConstantInt::get(VTy, 0), Name); } -void DFSanFunction::storeOrigin(Instruction *Pos, Value *Addr, uint64_t Size, - Value *Shadow, Value *Origin, +void DFSanFunction::storeOrigin(BasicBlock::iterator Pos, Value *Addr, + uint64_t Size, Value *Shadow, Value *Origin, Value *StoreOriginAddr, Align InstAlignment) { // Do not write origins for zero shadows because we do not trace origins for // untainted sinks. const Align OriginAlignment = getOriginAlign(InstAlignment); Value *CollapsedShadow = collapseToPrimitiveShadow(Shadow, Pos); - IRBuilder<> IRB(Pos); + IRBuilder<> IRB(Pos->getParent(), Pos); if (auto *ConstantShadow = dyn_cast(CollapsedShadow)) { if (!ConstantShadow->isZeroValue()) paintOrigin(IRB, updateOrigin(Origin, IRB), StoreOriginAddr, Size, @@ -2543,8 +2556,8 @@ void DFSanFunction::storeOrigin(Instruction *Pos, Value *Addr, uint64_t Size, void DFSanFunction::storeZeroPrimitiveShadow(Value *Addr, uint64_t Size, Align ShadowAlign, - Instruction *Pos) { - IRBuilder<> IRB(Pos); + BasicBlock::iterator Pos) { + IRBuilder<> IRB(Pos->getParent(), Pos); IntegerType *ShadowTy = IntegerType::get(*DFS.Ctx, Size * DFS.ShadowWidthBits); Value *ExtZeroShadow = ConstantInt::get(ShadowTy, 0); @@ -2558,13 +2571,13 @@ void DFSanFunction::storePrimitiveShadowOrigin(Value *Addr, uint64_t Size, Align InstAlignment, Value *PrimitiveShadow, Value *Origin, - Instruction *Pos) { + BasicBlock::iterator Pos) { const bool ShouldTrackOrigins = DFS.shouldTrackOrigins() && Origin; if (AllocaInst *AI = dyn_cast(Addr)) { const auto SI = AllocaShadowMap.find(AI); if (SI != AllocaShadowMap.end()) { - IRBuilder<> IRB(Pos); + IRBuilder<> IRB(Pos->getParent(), Pos); IRB.CreateStore(PrimitiveShadow, SI->second); // Do not write origins for 0 shadows because we do not trace origins for @@ -2584,7 +2597,7 @@ void DFSanFunction::storePrimitiveShadowOrigin(Value *Addr, uint64_t Size, return; } - IRBuilder<> IRB(Pos); + IRBuilder<> IRB(Pos->getParent(), Pos); Value *ShadowAddr, *OriginAddr; std::tie(ShadowAddr, OriginAddr) = DFS.getShadowOriginAddress(Addr, InstAlignment, Pos); @@ -2679,15 +2692,15 @@ void DFSanVisitor::visitStoreInst(StoreInst &SI) { Shadows.push_back(PtrShadow); Origins.push_back(DFSF.getOrigin(SI.getPointerOperand())); } - PrimitiveShadow = DFSF.combineShadows(Shadow, PtrShadow, &SI); + PrimitiveShadow = DFSF.combineShadows(Shadow, PtrShadow, SI.getIterator()); } else { - PrimitiveShadow = DFSF.collapseToPrimitiveShadow(Shadow, &SI); + PrimitiveShadow = DFSF.collapseToPrimitiveShadow(Shadow, SI.getIterator()); } Value *Origin = nullptr; if (ShouldTrackOrigins) - Origin = DFSF.combineOrigins(Shadows, Origins, &SI); + Origin = DFSF.combineOrigins(Shadows, Origins, SI.getIterator()); DFSF.storePrimitiveShadowOrigin(SI.getPointerOperand(), Size, SI.getAlign(), - PrimitiveShadow, Origin, &SI); + PrimitiveShadow, Origin, SI.getIterator()); if (ClEventCallbacks) { IRBuilder<> IRB(&SI); Value *Addr = SI.getPointerOperand(); @@ -2711,7 +2724,7 @@ void DFSanVisitor::visitCASOrRMW(Align InstAlignment, Instruction &I) { IRBuilder<> IRB(&I); Value *Addr = I.getOperand(0); const Align ShadowAlign = DFSF.getShadowAlign(InstAlignment); - DFSF.storeZeroPrimitiveShadow(Addr, Size, ShadowAlign, &I); + DFSF.storeZeroPrimitiveShadow(Addr, Size, ShadowAlign, I.getIterator()); DFSF.setShadow(&I, DFSF.DFS.getZeroShadow(&I)); DFSF.setOrigin(&I, DFSF.DFS.ZeroOrigin); } @@ -2866,7 +2879,7 @@ void DFSanVisitor::visitSelectInst(SelectInst &I) { if (isa(I.getCondition()->getType())) { ShadowSel = DFSF.combineShadowsThenConvert(I.getType(), TrueShadow, - FalseShadow, &I); + FalseShadow, I.getIterator()); if (ShouldTrackOrigins) { Shadows.push_back(TrueShadow); Shadows.push_back(FalseShadow); @@ -2881,25 +2894,25 @@ void DFSanVisitor::visitSelectInst(SelectInst &I) { Origins.push_back(TrueOrigin); } } else { - ShadowSel = - SelectInst::Create(I.getCondition(), TrueShadow, FalseShadow, "", &I); + ShadowSel = SelectInst::Create(I.getCondition(), TrueShadow, FalseShadow, + "", I.getIterator()); if (ShouldTrackOrigins) { Shadows.push_back(ShadowSel); Origins.push_back(SelectInst::Create(I.getCondition(), TrueOrigin, - FalseOrigin, "", &I)); + FalseOrigin, "", I.getIterator())); } } } - DFSF.setShadow(&I, ClTrackSelectControlFlow - ? DFSF.combineShadowsThenConvert( - I.getType(), CondShadow, ShadowSel, &I) - : ShadowSel); + DFSF.setShadow(&I, ClTrackSelectControlFlow ? DFSF.combineShadowsThenConvert( + I.getType(), CondShadow, + ShadowSel, I.getIterator()) + : ShadowSel); if (ShouldTrackOrigins) { if (ClTrackSelectControlFlow) { Shadows.push_back(CondShadow); Origins.push_back(DFSF.getOrigin(I.getCondition())); } - DFSF.setOrigin(&I, DFSF.combineOrigins(Shadows, Origins, &I)); + DFSF.setOrigin(&I, DFSF.combineOrigins(Shadows, Origins, I.getIterator())); } } @@ -2926,8 +2939,8 @@ void DFSanVisitor::visitMemTransferInst(MemTransferInst &I) { IRB.CreateIntCast(I.getArgOperand(2), DFSF.DFS.IntptrTy, false)}); } - Value *DestShadow = DFSF.DFS.getShadowAddress(I.getDest(), &I); - Value *SrcShadow = DFSF.DFS.getShadowAddress(I.getSource(), &I); + Value *DestShadow = DFSF.DFS.getShadowAddress(I.getDest(), I.getIterator()); + Value *SrcShadow = DFSF.DFS.getShadowAddress(I.getSource(), I.getIterator()); Value *LenShadow = IRB.CreateMul(I.getLength(), ConstantInt::get(I.getLength()->getType(), DFSF.DFS.ShadowWidthBytes)); @@ -2996,7 +3009,8 @@ void DFSanVisitor::addShadowArguments(Function &F, CallBase &CB, // Adds non-variable argument shadows. for (unsigned N = FT->getNumParams(); N != 0; ++I, --N) - Args.push_back(DFSF.collapseToPrimitiveShadow(DFSF.getShadow(*I), &CB)); + Args.push_back( + DFSF.collapseToPrimitiveShadow(DFSF.getShadow(*I), CB.getIterator())); // Adds variable argument shadows. if (FT->isVarArg()) { @@ -3004,12 +3018,13 @@ void DFSanVisitor::addShadowArguments(Function &F, CallBase &CB, CB.arg_size() - FT->getNumParams()); auto *LabelVAAlloca = new AllocaInst(LabelVATy, getDataLayout().getAllocaAddrSpace(), - "labelva", &DFSF.F->getEntryBlock().front()); + "labelva", DFSF.F->getEntryBlock().begin()); for (unsigned N = 0; I != CB.arg_end(); ++I, ++N) { auto *LabelVAPtr = IRB.CreateStructGEP(LabelVATy, LabelVAAlloca, N); - IRB.CreateStore(DFSF.collapseToPrimitiveShadow(DFSF.getShadow(*I), &CB), - LabelVAPtr); + IRB.CreateStore( + DFSF.collapseToPrimitiveShadow(DFSF.getShadow(*I), CB.getIterator()), + LabelVAPtr); } Args.push_back(IRB.CreateStructGEP(LabelVATy, LabelVAAlloca, 0)); @@ -3020,7 +3035,7 @@ void DFSanVisitor::addShadowArguments(Function &F, CallBase &CB, if (!DFSF.LabelReturnAlloca) { DFSF.LabelReturnAlloca = new AllocaInst( DFSF.DFS.PrimitiveShadowTy, getDataLayout().getAllocaAddrSpace(), - "labelreturn", &DFSF.F->getEntryBlock().front()); + "labelreturn", DFSF.F->getEntryBlock().begin()); } Args.push_back(DFSF.LabelReturnAlloca); } @@ -3043,7 +3058,7 @@ void DFSanVisitor::addOriginArguments(Function &F, CallBase &CB, ArrayType::get(DFSF.DFS.OriginTy, CB.arg_size() - FT->getNumParams()); auto *OriginVAAlloca = new AllocaInst(OriginVATy, getDataLayout().getAllocaAddrSpace(), - "originva", &DFSF.F->getEntryBlock().front()); + "originva", DFSF.F->getEntryBlock().begin()); for (unsigned N = 0; I != CB.arg_end(); ++I, ++N) { auto *OriginVAPtr = IRB.CreateStructGEP(OriginVATy, OriginVAAlloca, N); @@ -3058,7 +3073,7 @@ void DFSanVisitor::addOriginArguments(Function &F, CallBase &CB, if (!DFSF.OriginReturnAlloca) { DFSF.OriginReturnAlloca = new AllocaInst( DFSF.DFS.OriginTy, getDataLayout().getAllocaAddrSpace(), - "originreturn", &DFSF.F->getEntryBlock().front()); + "originreturn", DFSF.F->getEntryBlock().begin()); } Args.push_back(DFSF.OriginReturnAlloca); } @@ -3155,8 +3170,9 @@ bool DFSanVisitor::visitWrappedCallBase(Function &F, CallBase &CB) { if (!FT->getReturnType()->isVoidTy()) { LoadInst *LabelLoad = IRB.CreateLoad(DFSF.DFS.PrimitiveShadowTy, DFSF.LabelReturnAlloca); - DFSF.setShadow(CustomCI, DFSF.expandFromPrimitiveShadow( - FT->getReturnType(), LabelLoad, &CB)); + DFSF.setShadow(CustomCI, + DFSF.expandFromPrimitiveShadow( + FT->getReturnType(), LabelLoad, CB.getIterator())); if (ShouldTrackOrigins) { LoadInst *OriginLoad = IRB.CreateLoad(DFSF.DFS.OriginTy, DFSF.OriginReturnAlloca); @@ -3433,8 +3449,8 @@ void DFSanVisitor::visitCallBase(CallBase &CB) { void DFSanVisitor::visitPHINode(PHINode &PN) { Type *ShadowTy = DFSF.DFS.getShadowTy(&PN); - PHINode *ShadowPN = - PHINode::Create(ShadowTy, PN.getNumIncomingValues(), "", &PN); + PHINode *ShadowPN = PHINode::Create(ShadowTy, PN.getNumIncomingValues(), "", + PN.getIterator()); // Give the shadow phi node valid predecessors to fool SplitEdge into working. Value *UndefShadow = UndefValue::get(ShadowTy); @@ -3445,8 +3461,8 @@ void DFSanVisitor::visitPHINode(PHINode &PN) { PHINode *OriginPN = nullptr; if (DFSF.DFS.shouldTrackOrigins()) { - OriginPN = - PHINode::Create(DFSF.DFS.OriginTy, PN.getNumIncomingValues(), "", &PN); + OriginPN = PHINode::Create(DFSF.DFS.OriginTy, PN.getNumIncomingValues(), "", + PN.getIterator()); Value *UndefOrigin = UndefValue::get(DFSF.DFS.OriginTy); for (BasicBlock *BB : PN.blocks()) OriginPN->addIncoming(UndefOrigin, BB); diff --git a/llvm/lib/Transforms/Instrumentation/KCFI.cpp b/llvm/lib/Transforms/Instrumentation/KCFI.cpp index b1a26880c701..b22e7f7fc0be 100644 --- a/llvm/lib/Transforms/Instrumentation/KCFI.cpp +++ b/llvm/lib/Transforms/Instrumentation/KCFI.cpp @@ -82,8 +82,8 @@ PreservedAnalyses KCFIPass::run(Function &F, FunctionAnalysisManager &AM) { ->getZExtValue(); // Drop the KCFI operand bundle. - CallBase *Call = - CallBase::removeOperandBundle(CI, LLVMContext::OB_kcfi, CI); + CallBase *Call = CallBase::removeOperandBundle(CI, LLVMContext::OB_kcfi, + CI->getIterator()); assert(Call != CI); Call->copyMetadata(*CI); CI->replaceAllUsesWith(Call); diff --git a/llvm/lib/Transforms/ObjCARC/ObjCARC.cpp b/llvm/lib/Transforms/ObjCARC/ObjCARC.cpp index 02f9db719e26..33870d7ea192 100644 --- a/llvm/lib/Transforms/ObjCARC/ObjCARC.cpp +++ b/llvm/lib/Transforms/ObjCARC/ObjCARC.cpp @@ -23,7 +23,7 @@ using namespace llvm::objcarc; CallInst *objcarc::createCallInstWithColors( FunctionCallee Func, ArrayRef Args, const Twine &NameStr, - Instruction *InsertBefore, + BasicBlock::iterator InsertBefore, const DenseMap &BlockColors) { FunctionType *FTy = Func.getFunctionType(); Value *Callee = Func.getCallee(); @@ -64,23 +64,23 @@ BundledRetainClaimRVs::insertAfterInvokes(Function &F, DominatorTree *DT) { // We don't have to call insertRVCallWithColors since DestBB is the normal // destination of the invoke. - insertRVCall(&*DestBB->getFirstInsertionPt(), I); + insertRVCall(DestBB->getFirstInsertionPt(), I); Changed = true; } return std::make_pair(Changed, CFGChanged); } -CallInst *BundledRetainClaimRVs::insertRVCall(Instruction *InsertPt, +CallInst *BundledRetainClaimRVs::insertRVCall(BasicBlock::iterator InsertPt, CallBase *AnnotatedCall) { DenseMap BlockColors; return insertRVCallWithColors(InsertPt, AnnotatedCall, BlockColors); } CallInst *BundledRetainClaimRVs::insertRVCallWithColors( - Instruction *InsertPt, CallBase *AnnotatedCall, + BasicBlock::iterator InsertPt, CallBase *AnnotatedCall, const DenseMap &BlockColors) { - IRBuilder<> Builder(InsertPt); + IRBuilder<> Builder(InsertPt->getParent(), InsertPt); Function *Func = *objcarc::getAttachedARCFunction(AnnotatedCall); assert(Func && "operand isn't a Function"); Type *ParamTy = Func->getArg(0)->getType(); diff --git a/llvm/lib/Transforms/ObjCARC/ObjCARC.h b/llvm/lib/Transforms/ObjCARC/ObjCARC.h index 9e68bd574851..f4d7c92d499c 100644 --- a/llvm/lib/Transforms/ObjCARC/ObjCARC.h +++ b/llvm/lib/Transforms/ObjCARC/ObjCARC.h @@ -99,7 +99,7 @@ static inline MDString *getRVInstMarker(Module &M) { /// going to be removed from the IR before WinEHPrepare. CallInst *createCallInstWithColors( FunctionCallee Func, ArrayRef Args, const Twine &NameStr, - Instruction *InsertBefore, + BasicBlock::iterator InsertBefore, const DenseMap &BlockColors); class BundledRetainClaimRVs { @@ -113,11 +113,12 @@ public: std::pair insertAfterInvokes(Function &F, DominatorTree *DT); /// Insert a retainRV/claimRV call. - CallInst *insertRVCall(Instruction *InsertPt, CallBase *AnnotatedCall); + CallInst *insertRVCall(BasicBlock::iterator InsertPt, + CallBase *AnnotatedCall); /// Insert a retainRV/claimRV call with colors. CallInst *insertRVCallWithColors( - Instruction *InsertPt, CallBase *AnnotatedCall, + BasicBlock::iterator InsertPt, CallBase *AnnotatedCall, const DenseMap &BlockColors); /// See if an instruction is a bundled retainRV/claimRV call. @@ -140,7 +141,8 @@ public: } auto *NewCall = CallBase::removeOperandBundle( - It->second, LLVMContext::OB_clang_arc_attachedcall, It->second); + It->second, LLVMContext::OB_clang_arc_attachedcall, + It->second->getIterator()); NewCall->copyMetadata(*It->second); It->second->replaceAllUsesWith(NewCall); It->second->eraseFromParent(); diff --git a/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp b/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp index c397ab63f388..0d0f5c72928a 100644 --- a/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp +++ b/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp @@ -382,12 +382,12 @@ void ObjCARCContract::tryToContractReleaseIntoStoreStrong( Value *Args[] = { Load->getPointerOperand(), New }; if (Args[0]->getType() != I8XX) - Args[0] = new BitCastInst(Args[0], I8XX, "", Store); + Args[0] = new BitCastInst(Args[0], I8XX, "", Store->getIterator()); if (Args[1]->getType() != I8X) - Args[1] = new BitCastInst(Args[1], I8X, "", Store); + Args[1] = new BitCastInst(Args[1], I8X, "", Store->getIterator()); Function *Decl = EP.get(ARCRuntimeEntryPointKind::StoreStrong); - CallInst *StoreStrong = - objcarc::createCallInstWithColors(Decl, Args, "", Store, BlockColors); + CallInst *StoreStrong = objcarc::createCallInstWithColors( + Decl, Args, "", Store->getIterator(), BlockColors); StoreStrong->setDoesNotThrow(); StoreStrong->setDebugLoc(Store->getDebugLoc()); @@ -472,8 +472,8 @@ bool ObjCARCContract::tryToPeepholeInstruction( RVInstMarker->getString(), /*Constraints=*/"", /*hasSideEffects=*/true); - objcarc::createCallInstWithColors(IA, std::nullopt, "", Inst, - BlockColors); + objcarc::createCallInstWithColors(IA, std::nullopt, "", + Inst->getIterator(), BlockColors); } decline_rv_optimization: return false; @@ -484,7 +484,7 @@ bool ObjCARCContract::tryToPeepholeInstruction( if (IsNullOrUndef(CI->getArgOperand(1))) { Value *Null = ConstantPointerNull::get(cast(CI->getType())); Changed = true; - new StoreInst(Null, CI->getArgOperand(0), CI); + new StoreInst(Null, CI->getArgOperand(0), CI->getIterator()); LLVM_DEBUG(dbgs() << "OBJCARCContract: Old = " << *CI << "\n" << " New = " << *Null << "\n"); @@ -575,7 +575,7 @@ bool ObjCARCContract::run(Function &F, AAResults *A, DominatorTree *D) { if (auto *CI = dyn_cast(Inst)) if (objcarc::hasAttachedCallOpBundle(CI)) { - BundledInsts->insertRVCallWithColors(&*I, CI, BlockColors); + BundledInsts->insertRVCallWithColors(I->getIterator(), CI, BlockColors); --I; Changed = true; } @@ -631,8 +631,8 @@ bool ObjCARCContract::run(Function &F, AAResults *A, DominatorTree *D) { assert(DT->dominates(Inst, &InsertBB->back()) && "Invalid insertion point for bitcast"); - Replacement = - new BitCastInst(Replacement, UseTy, "", &InsertBB->back()); + Replacement = new BitCastInst(Replacement, UseTy, "", + InsertBB->back().getIterator()); } // While we're here, rewrite all edges for this PHI, rather @@ -649,8 +649,9 @@ bool ObjCARCContract::run(Function &F, AAResults *A, DominatorTree *D) { } } else { if (Replacement->getType() != UseTy) - Replacement = new BitCastInst(Replacement, UseTy, "", - cast(U.getUser())); + Replacement = + new BitCastInst(Replacement, UseTy, "", + cast(U.getUser())->getIterator()); U.set(Replacement); } } diff --git a/llvm/lib/Transforms/ObjCARC/ObjCARCOpts.cpp b/llvm/lib/Transforms/ObjCARC/ObjCARCOpts.cpp index b51e4d46bffe..72e860d7dcfa 100644 --- a/llvm/lib/Transforms/ObjCARC/ObjCARCOpts.cpp +++ b/llvm/lib/Transforms/ObjCARC/ObjCARCOpts.cpp @@ -693,8 +693,9 @@ bool ObjCARCOpt::OptimizeInlinedAutoreleaseRVCall( // AutoreleaseRV and RetainRV cancel out, replace UnsafeClaimRV with Release. assert(Class == ARCInstKind::UnsafeClaimRV); Value *CallArg = cast(Inst)->getArgOperand(0); - CallInst *Release = CallInst::Create( - EP.get(ARCRuntimeEntryPointKind::Release), CallArg, "", Inst); + CallInst *Release = + CallInst::Create(EP.get(ARCRuntimeEntryPointKind::Release), CallArg, "", + Inst->getIterator()); assert(IsAlwaysTail(ARCInstKind::UnsafeClaimRV) && "Expected UnsafeClaimRV to be safe to tail call"); Release->setTailCall(); @@ -808,7 +809,7 @@ void ObjCARCOpt::OptimizeIndividualCalls(Function &F) { if (auto *CI = dyn_cast(Inst)) if (objcarc::hasAttachedCallOpBundle(CI)) { - BundledInsts->insertRVCall(&*I, CI); + BundledInsts->insertRVCall(I->getIterator(), CI); Changed = true; } @@ -934,7 +935,7 @@ void ObjCARCOpt::OptimizeIndividualCallImpl(Function &F, Instruction *Inst, Changed = true; new StoreInst(ConstantInt::getTrue(CI->getContext()), PoisonValue::get(PointerType::getUnqual(CI->getContext())), - CI); + CI->getIterator()); Value *NewValue = PoisonValue::get(CI->getType()); LLVM_DEBUG( dbgs() << "A null pointer-to-weak-pointer is undefined behavior." @@ -954,7 +955,7 @@ void ObjCARCOpt::OptimizeIndividualCallImpl(Function &F, Instruction *Inst, Changed = true; new StoreInst(ConstantInt::getTrue(CI->getContext()), PoisonValue::get(PointerType::getUnqual(CI->getContext())), - CI); + CI->getIterator()); Value *NewValue = PoisonValue::get(CI->getType()); LLVM_DEBUG( @@ -990,8 +991,8 @@ void ObjCARCOpt::OptimizeIndividualCallImpl(Function &F, Instruction *Inst, LLVMContext &C = Inst->getContext(); Function *Decl = EP.get(ARCRuntimeEntryPointKind::Release); - CallInst *NewCall = - CallInst::Create(Decl, Call->getArgOperand(0), "", Call); + CallInst *NewCall = CallInst::Create(Decl, Call->getArgOperand(0), "", + Call->getIterator()); NewCall->setMetadata(MDKindCache.get(ARCMDKindID::ImpreciseRelease), MDNode::get(C, std::nullopt)); @@ -1143,7 +1144,8 @@ void ObjCARCOpt::OptimizeIndividualCallImpl(Function &F, Instruction *Inst, if (IsNullOrUndef(Incoming)) continue; Value *Op = PN->getIncomingValue(i); - Instruction *InsertPos = &PN->getIncomingBlock(i)->back(); + BasicBlock::iterator InsertPos = + PN->getIncomingBlock(i)->back().getIterator(); SmallVector OpBundles; cloneOpBundlesIf(CInst, OpBundles, [](const OperandBundleUse &B) { return B.getTagID() != LLVMContext::OB_funclet; @@ -1153,7 +1155,7 @@ void ObjCARCOpt::OptimizeIndividualCallImpl(Function &F, Instruction *Inst, if (Op->getType() != ParamTy) Op = new BitCastInst(Op, ParamTy, "", InsertPos); Clone->setArgOperand(0, Op); - Clone->insertBefore(InsertPos); + Clone->insertBefore(*InsertPos->getParent(), InsertPos); LLVM_DEBUG(dbgs() << "Cloning " << *CInst << "\n" "And inserting clone at " @@ -1768,12 +1770,14 @@ void ObjCARCOpt::MoveCalls(Value *Arg, RRInfo &RetainsToMove, // Insert the new retain and release calls. for (Instruction *InsertPt : ReleasesToMove.ReverseInsertPts) { - Value *MyArg = ArgTy == ParamTy ? Arg : - new BitCastInst(Arg, ParamTy, "", InsertPt); + Value *MyArg = ArgTy == ParamTy ? Arg + : new BitCastInst(Arg, ParamTy, "", + InsertPt->getIterator()); Function *Decl = EP.get(ARCRuntimeEntryPointKind::Retain); SmallVector BundleList; addOpBundleForFunclet(InsertPt->getParent(), BundleList); - CallInst *Call = CallInst::Create(Decl, MyArg, BundleList, "", InsertPt); + CallInst *Call = + CallInst::Create(Decl, MyArg, BundleList, "", InsertPt->getIterator()); Call->setDoesNotThrow(); Call->setTailCall(); @@ -1783,12 +1787,14 @@ void ObjCARCOpt::MoveCalls(Value *Arg, RRInfo &RetainsToMove, << *InsertPt << "\n"); } for (Instruction *InsertPt : RetainsToMove.ReverseInsertPts) { - Value *MyArg = ArgTy == ParamTy ? Arg : - new BitCastInst(Arg, ParamTy, "", InsertPt); + Value *MyArg = ArgTy == ParamTy ? Arg + : new BitCastInst(Arg, ParamTy, "", + InsertPt->getIterator()); Function *Decl = EP.get(ARCRuntimeEntryPointKind::Release); SmallVector BundleList; addOpBundleForFunclet(InsertPt->getParent(), BundleList); - CallInst *Call = CallInst::Create(Decl, MyArg, BundleList, "", InsertPt); + CallInst *Call = + CallInst::Create(Decl, MyArg, BundleList, "", InsertPt->getIterator()); // Attach a clang.imprecise_release metadata tag, if appropriate. if (MDNode *M = ReleasesToMove.ReleaseMetadata) Call->setMetadata(MDKindCache.get(ARCMDKindID::ImpreciseRelease), M); @@ -2125,7 +2131,8 @@ void ObjCARCOpt::OptimizeWeakCalls(Function &F) { // If the load has a builtin retain, insert a plain retain for it. if (Class == ARCInstKind::LoadWeakRetained) { Function *Decl = EP.get(ARCRuntimeEntryPointKind::Retain); - CallInst *CI = CallInst::Create(Decl, EarlierCall, "", Call); + CallInst *CI = + CallInst::Create(Decl, EarlierCall, "", Call->getIterator()); CI->setTailCall(); } // Zap the fully redundant load. @@ -2154,7 +2161,8 @@ void ObjCARCOpt::OptimizeWeakCalls(Function &F) { // If the load has a builtin retain, insert a plain retain for it. if (Class == ARCInstKind::LoadWeakRetained) { Function *Decl = EP.get(ARCRuntimeEntryPointKind::Retain); - CallInst *CI = CallInst::Create(Decl, EarlierCall, "", Call); + CallInst *CI = + CallInst::Create(Decl, EarlierCall, "", Call->getIterator()); CI->setTailCall(); } // Zap the fully redundant load. diff --git a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp index 490cb7e528eb..7a2011888ab0 100644 --- a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp +++ b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp @@ -590,7 +590,7 @@ static bool processSaturatingInst(SaturatingInst *SI, LazyValueInfo *LVI) { bool NSW = SI->isSigned(); bool NUW = !SI->isSigned(); BinaryOperator *BinOp = BinaryOperator::Create( - Opcode, SI->getLHS(), SI->getRHS(), SI->getName(), SI); + Opcode, SI->getLHS(), SI->getRHS(), SI->getName(), SI->getIterator()); BinOp->setDebugLoc(SI->getDebugLoc()); setDeducedOverflowingFlags(BinOp, Opcode, NSW, NUW); @@ -911,8 +911,8 @@ static bool processSRem(BinaryOperator *SDI, const ConstantRange &LCR, Op.V = BO; } - auto *URem = - BinaryOperator::CreateURem(Ops[0].V, Ops[1].V, SDI->getName(), SDI); + auto *URem = BinaryOperator::CreateURem(Ops[0].V, Ops[1].V, SDI->getName(), + SDI->getIterator()); URem->setDebugLoc(SDI->getDebugLoc()); auto *Res = URem; @@ -973,8 +973,8 @@ static bool processSDiv(BinaryOperator *SDI, const ConstantRange &LCR, Op.V = BO; } - auto *UDiv = - BinaryOperator::CreateUDiv(Ops[0].V, Ops[1].V, SDI->getName(), SDI); + auto *UDiv = BinaryOperator::CreateUDiv(Ops[0].V, Ops[1].V, SDI->getName(), + SDI->getIterator()); UDiv->setDebugLoc(SDI->getDebugLoc()); UDiv->setIsExact(SDI->isExact()); @@ -1041,7 +1041,7 @@ static bool processAShr(BinaryOperator *SDI, LazyValueInfo *LVI) { ++NumAShrsConverted; auto *BO = BinaryOperator::CreateLShr(SDI->getOperand(0), SDI->getOperand(1), - "", SDI); + "", SDI->getIterator()); BO->takeName(SDI); BO->setDebugLoc(SDI->getDebugLoc()); BO->setIsExact(SDI->isExact()); @@ -1061,7 +1061,8 @@ static bool processSExt(SExtInst *SDI, LazyValueInfo *LVI) { return false; ++NumSExt; - auto *ZExt = CastInst::CreateZExtOrBitCast(Base, SDI->getType(), "", SDI); + auto *ZExt = CastInst::CreateZExtOrBitCast(Base, SDI->getType(), "", + SDI->getIterator()); ZExt->takeName(SDI); ZExt->setDebugLoc(SDI->getDebugLoc()); ZExt->setNonNeg(); diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp index c2c63d100014..3d1dac5ea17e 100644 --- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp +++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp @@ -634,7 +634,8 @@ static bool tryToShorten(Instruction *DeadI, int64_t &DeadStart, Value *Indices[1] = { ConstantInt::get(DeadWriteLength->getType(), ToRemoveSize)}; Instruction *NewDestGEP = GetElementPtrInst::CreateInBounds( - Type::getInt8Ty(DeadIntrinsic->getContext()), OrigDest, Indices, "", DeadI); + Type::getInt8Ty(DeadIntrinsic->getContext()), OrigDest, Indices, "", + DeadI->getIterator()); NewDestGEP->setDebugLoc(DeadIntrinsic->getDebugLoc()); DeadIntrinsic->setDest(NewDestGEP); } diff --git a/llvm/lib/Transforms/Scalar/DivRemPairs.cpp b/llvm/lib/Transforms/Scalar/DivRemPairs.cpp index 57d3f312186e..45f36a36b5dd 100644 --- a/llvm/lib/Transforms/Scalar/DivRemPairs.cpp +++ b/llvm/lib/Transforms/Scalar/DivRemPairs.cpp @@ -383,14 +383,16 @@ static bool optimizeDivRem(Function &F, const TargetTransformInfo &TTI, // If X is not frozen, %rem becomes undef after transformation. // TODO: We need a undef-specific checking function in ValueTracking if (!isGuaranteedNotToBeUndefOrPoison(X, nullptr, DivInst, &DT)) { - auto *FrX = new FreezeInst(X, X->getName() + ".frozen", DivInst); + auto *FrX = + new FreezeInst(X, X->getName() + ".frozen", DivInst->getIterator()); DivInst->setOperand(0, FrX); Sub->setOperand(0, FrX); } // Same for Y. If X = 1 and Y = (undef | 1), %rem in src is either 1 or 0, // but %rem in tgt can be one of many integer values. if (!isGuaranteedNotToBeUndefOrPoison(Y, nullptr, DivInst, &DT)) { - auto *FrY = new FreezeInst(Y, Y->getName() + ".frozen", DivInst); + auto *FrY = + new FreezeInst(Y, Y->getName() + ".frozen", DivInst->getIterator()); DivInst->setOperand(1, FrY); Mul->setOperand(1, FrY); } diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp index dcb1ed334b61..67fb2a5da3bb 100644 --- a/llvm/lib/Transforms/Scalar/GVN.cpp +++ b/llvm/lib/Transforms/Scalar/GVN.cpp @@ -1056,7 +1056,8 @@ Value *AvailableValue::MaterializeAdjustedValue(LoadInst *Load, // Introduce a new value select for a load from an eligible pointer select. SelectInst *Sel = getSelectValue(); assert(V1 && V2 && "both value operands of the select must be present"); - Res = SelectInst::Create(Sel->getCondition(), V1, V2, "", Sel); + Res = + SelectInst::Create(Sel->getCondition(), V1, V2, "", Sel->getIterator()); } else { llvm_unreachable("Should not materialize value from dead block"); } @@ -1412,10 +1413,10 @@ void GVNPass::eliminatePartiallyRedundantLoad( BasicBlock *UnavailableBlock = AvailableLoad.first; Value *LoadPtr = AvailableLoad.second; - auto *NewLoad = - new LoadInst(Load->getType(), LoadPtr, Load->getName() + ".pre", - Load->isVolatile(), Load->getAlign(), Load->getOrdering(), - Load->getSyncScopeID(), UnavailableBlock->getTerminator()); + auto *NewLoad = new LoadInst( + Load->getType(), LoadPtr, Load->getName() + ".pre", Load->isVolatile(), + Load->getAlign(), Load->getOrdering(), Load->getSyncScopeID(), + UnavailableBlock->getTerminator()->getIterator()); NewLoad->setDebugLoc(Load->getDebugLoc()); if (MSSAU) { auto *NewAccess = MSSAU->createMemoryAccessInBB( @@ -1994,8 +1995,9 @@ bool GVNPass::processAssumeIntrinsic(AssumeInst *IntrinsicI) { // Insert a new store to null instruction before the load to indicate that // this code is not reachable. FIXME: We could insert unreachable // instruction directly because we can modify the CFG. - auto *NewS = new StoreInst(PoisonValue::get(Int8Ty), - Constant::getNullValue(PtrTy), IntrinsicI); + auto *NewS = + new StoreInst(PoisonValue::get(Int8Ty), Constant::getNullValue(PtrTy), + IntrinsicI->getIterator()); if (MSSAU) { const MemoryUseOrDef *FirstNonDom = nullptr; const auto *AL = diff --git a/llvm/lib/Transforms/Scalar/GuardWidening.cpp b/llvm/lib/Transforms/Scalar/GuardWidening.cpp index 3bbf6642a90c..d3787b28347c 100644 --- a/llvm/lib/Transforms/Scalar/GuardWidening.cpp +++ b/llvm/lib/Transforms/Scalar/GuardWidening.cpp @@ -121,12 +121,13 @@ static void eliminateGuard(Instruction *GuardInst, MemorySSAUpdater *MSSAU) { /// condition should stay invariant. Otherwise there can be a miscompile, like /// the one described at https://github.com/llvm/llvm-project/issues/60234. The /// safest way to do it is to expand the new condition at WC's block. -static Instruction *findInsertionPointForWideCondition(Instruction *WCOrGuard) { +static std::optional +findInsertionPointForWideCondition(Instruction *WCOrGuard) { if (isGuard(WCOrGuard)) - return WCOrGuard; + return WCOrGuard->getIterator(); if (auto WC = extractWidenableCondition(WCOrGuard)) - return cast(WC); - return nullptr; + return cast(WC)->getIterator(); + return std::nullopt; } class GuardWideningImpl { @@ -182,30 +183,30 @@ class GuardWideningImpl { /// into \p WideningPoint. WideningScore computeWideningScore(Instruction *DominatedInstr, Instruction *ToWiden, - Instruction *WideningPoint, + BasicBlock::iterator WideningPoint, SmallVectorImpl &ChecksToHoist, SmallVectorImpl &ChecksToWiden); /// Helper to check if \p V can be hoisted to \p InsertPos. - bool canBeHoistedTo(const Value *V, const Instruction *InsertPos) const { + bool canBeHoistedTo(const Value *V, BasicBlock::iterator InsertPos) const { SmallPtrSet Visited; return canBeHoistedTo(V, InsertPos, Visited); } - bool canBeHoistedTo(const Value *V, const Instruction *InsertPos, + bool canBeHoistedTo(const Value *V, BasicBlock::iterator InsertPos, SmallPtrSetImpl &Visited) const; bool canBeHoistedTo(const SmallVectorImpl &Checks, - const Instruction *InsertPos) const { + BasicBlock::iterator InsertPos) const { return all_of(Checks, [&](const Value *V) { return canBeHoistedTo(V, InsertPos); }); } /// Helper to hoist \p V to \p InsertPos. Guaranteed to succeed if \c /// canBeHoistedTo returned true. - void makeAvailableAt(Value *V, Instruction *InsertPos) const; + void makeAvailableAt(Value *V, BasicBlock::iterator InsertPos) const; void makeAvailableAt(const SmallVectorImpl &Checks, - Instruction *InsertPos) const { + BasicBlock::iterator InsertPos) const { for (Value *V : Checks) makeAvailableAt(V, InsertPos); } @@ -217,18 +218,19 @@ class GuardWideningImpl { /// InsertPt is true then actually generate the resulting expression, make it /// available at \p InsertPt and return it in \p Result (else no change to the /// IR is made). - std::optional mergeChecks(SmallVectorImpl &ChecksToHoist, - SmallVectorImpl &ChecksToWiden, - Instruction *InsertPt); + std::optional + mergeChecks(SmallVectorImpl &ChecksToHoist, + SmallVectorImpl &ChecksToWiden, + std::optional InsertPt); /// Generate the logical AND of \p ChecksToHoist and \p OldCondition and make /// it available at InsertPt Value *hoistChecks(SmallVectorImpl &ChecksToHoist, - Value *OldCondition, Instruction *InsertPt); + Value *OldCondition, BasicBlock::iterator InsertPt); /// Adds freeze to Orig and push it as far as possible very aggressively. /// Also replaces all uses of frozen instruction with frozen version. - Value *freezeAndPush(Value *Orig, Instruction *InsertPt); + Value *freezeAndPush(Value *Orig, BasicBlock::iterator InsertPt); /// Represents a range check of the form \c Base + \c Offset u< \c Length, /// with the constraint that \c Length is not negative. \c CheckInst is the @@ -294,7 +296,7 @@ class GuardWideningImpl { /// for the price of computing only one of the set of expressions? bool isWideningCondProfitable(SmallVectorImpl &ChecksToHoist, SmallVectorImpl &ChecksToWiden) { - return mergeChecks(ChecksToHoist, ChecksToWiden, /*InsertPt=*/nullptr) + return mergeChecks(ChecksToHoist, ChecksToWiden, /*InsertPt=*/std::nullopt) .has_value(); } @@ -302,11 +304,11 @@ class GuardWideningImpl { void widenGuard(SmallVectorImpl &ChecksToHoist, SmallVectorImpl &ChecksToWiden, Instruction *ToWiden) { - Instruction *InsertPt = findInsertionPointForWideCondition(ToWiden); + auto InsertPt = findInsertionPointForWideCondition(ToWiden); auto MergedCheck = mergeChecks(ChecksToHoist, ChecksToWiden, InsertPt); Value *Result = MergedCheck ? *MergedCheck : hoistChecks(ChecksToHoist, - getCondition(ToWiden), InsertPt); + getCondition(ToWiden), *InsertPt); if (isGuardAsWidenableBranch(ToWiden)) { setWidenableBranchCond(cast(ToWiden), Result); @@ -417,12 +419,12 @@ bool GuardWideningImpl::eliminateInstrViaWidening( assert((i == (e - 1)) == (Instr->getParent() == CurBB) && "Bad DFS?"); for (auto *Candidate : make_range(I, E)) { - auto *WideningPoint = findInsertionPointForWideCondition(Candidate); + auto WideningPoint = findInsertionPointForWideCondition(Candidate); if (!WideningPoint) continue; SmallVector CandidateChecks; parseWidenableGuard(Candidate, CandidateChecks); - auto Score = computeWideningScore(Instr, Candidate, WideningPoint, + auto Score = computeWideningScore(Instr, Candidate, *WideningPoint, ChecksToHoist, CandidateChecks); LLVM_DEBUG(dbgs() << "Score between " << *Instr << " and " << *Candidate << " is " << scoreTypeToString(Score) << "\n"); @@ -456,7 +458,7 @@ bool GuardWideningImpl::eliminateInstrViaWidening( GuardWideningImpl::WideningScore GuardWideningImpl::computeWideningScore( Instruction *DominatedInstr, Instruction *ToWiden, - Instruction *WideningPoint, SmallVectorImpl &ChecksToHoist, + BasicBlock::iterator WideningPoint, SmallVectorImpl &ChecksToHoist, SmallVectorImpl &ChecksToWiden) { Loop *DominatedInstrLoop = LI.getLoopFor(DominatedInstr->getParent()); Loop *DominatingGuardLoop = LI.getLoopFor(WideningPoint->getParent()); @@ -559,7 +561,7 @@ GuardWideningImpl::WideningScore GuardWideningImpl::computeWideningScore( } bool GuardWideningImpl::canBeHoistedTo( - const Value *V, const Instruction *Loc, + const Value *V, BasicBlock::iterator Loc, SmallPtrSetImpl &Visited) const { auto *Inst = dyn_cast(V); if (!Inst || DT.dominates(Inst, Loc) || Visited.count(Inst)) @@ -580,7 +582,8 @@ bool GuardWideningImpl::canBeHoistedTo( [&](Value *Op) { return canBeHoistedTo(Op, Loc, Visited); }); } -void GuardWideningImpl::makeAvailableAt(Value *V, Instruction *Loc) const { +void GuardWideningImpl::makeAvailableAt(Value *V, + BasicBlock::iterator Loc) const { auto *Inst = dyn_cast(V); if (!Inst || DT.dominates(Inst, Loc)) return; @@ -592,7 +595,7 @@ void GuardWideningImpl::makeAvailableAt(Value *V, Instruction *Loc) const { for (Value *Op : Inst->operands()) makeAvailableAt(Op, Loc); - Inst->moveBefore(Loc); + Inst->moveBefore(*Loc->getParent(), Loc); } // Return Instruction before which we can insert freeze for the value V as close @@ -621,14 +624,15 @@ getFreezeInsertPt(Value *V, const DominatorTree &DT) { return Res; } -Value *GuardWideningImpl::freezeAndPush(Value *Orig, Instruction *InsertPt) { +Value *GuardWideningImpl::freezeAndPush(Value *Orig, + BasicBlock::iterator InsertPt) { if (isGuaranteedNotToBePoison(Orig, nullptr, InsertPt, &DT)) return Orig; std::optional InsertPtAtDef = getFreezeInsertPt(Orig, DT); if (!InsertPtAtDef) { FreezeInst *FI = new FreezeInst(Orig, "gw.freeze"); - FI->insertBefore(InsertPt); + FI->insertBefore(*InsertPt->getParent(), InsertPt); return FI; } if (isa(Orig) || isa(Orig)) { @@ -715,7 +719,7 @@ Value *GuardWideningImpl::freezeAndPush(Value *Orig, Instruction *InsertPt) { std::optional GuardWideningImpl::mergeChecks(SmallVectorImpl &ChecksToHoist, SmallVectorImpl &ChecksToWiden, - Instruction *InsertPt) { + std::optional InsertPt) { using namespace llvm::PatternMatch; Value *Result = nullptr; @@ -747,10 +751,10 @@ GuardWideningImpl::mergeChecks(SmallVectorImpl &ChecksToHoist, if (Intersect->getEquivalentICmp(Pred, NewRHSAP)) { if (InsertPt) { ConstantInt *NewRHS = - ConstantInt::get(InsertPt->getContext(), NewRHSAP); - assert(canBeHoistedTo(LHS, InsertPt) && "must be"); - makeAvailableAt(LHS, InsertPt); - Result = new ICmpInst(InsertPt, Pred, LHS, NewRHS, "wide.chk"); + ConstantInt::get((*InsertPt)->getContext(), NewRHSAP); + assert(canBeHoistedTo(LHS, *InsertPt) && "must be"); + makeAvailableAt(LHS, *InsertPt); + Result = new ICmpInst(*InsertPt, Pred, LHS, NewRHS, "wide.chk"); } return Result; } @@ -765,16 +769,16 @@ GuardWideningImpl::mergeChecks(SmallVectorImpl &ChecksToHoist, combineRangeChecks(Checks, CombinedChecks)) { if (InsertPt) { for (auto &RC : CombinedChecks) { - makeAvailableAt(RC.getCheckInst(), InsertPt); + makeAvailableAt(RC.getCheckInst(), *InsertPt); if (Result) Result = BinaryOperator::CreateAnd(RC.getCheckInst(), Result, "", - InsertPt); + *InsertPt); else Result = RC.getCheckInst(); } assert(Result && "Failed to find result value"); Result->setName("wide.chk"); - Result = freezeAndPush(Result, InsertPt); + Result = freezeAndPush(Result, *InsertPt); } return Result; } @@ -786,9 +790,9 @@ GuardWideningImpl::mergeChecks(SmallVectorImpl &ChecksToHoist, Value *GuardWideningImpl::hoistChecks(SmallVectorImpl &ChecksToHoist, Value *OldCondition, - Instruction *InsertPt) { + BasicBlock::iterator InsertPt) { assert(!ChecksToHoist.empty()); - IRBuilder<> Builder(InsertPt); + IRBuilder<> Builder(InsertPt->getParent(), InsertPt); makeAvailableAt(ChecksToHoist, InsertPt); makeAvailableAt(OldCondition, InsertPt); Value *Result = Builder.CreateAnd(ChecksToHoist); diff --git a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp index 41c4d6236173..38104afa7f78 100644 --- a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp +++ b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp @@ -350,18 +350,19 @@ bool IndVarSimplify::handleFloatingPointIV(Loop *L, PHINode *PN) { IntegerType *Int32Ty = Type::getInt32Ty(PN->getContext()); // Insert new integer induction variable. - PHINode *NewPHI = PHINode::Create(Int32Ty, 2, PN->getName()+".int", PN); + PHINode *NewPHI = + PHINode::Create(Int32Ty, 2, PN->getName() + ".int", PN->getIterator()); NewPHI->addIncoming(ConstantInt::get(Int32Ty, InitValue), PN->getIncomingBlock(IncomingEdge)); Value *NewAdd = - BinaryOperator::CreateAdd(NewPHI, ConstantInt::get(Int32Ty, IncValue), - Incr->getName()+".int", Incr); + BinaryOperator::CreateAdd(NewPHI, ConstantInt::get(Int32Ty, IncValue), + Incr->getName() + ".int", Incr->getIterator()); NewPHI->addIncoming(NewAdd, PN->getIncomingBlock(BackEdge)); - ICmpInst *NewCompare = new ICmpInst(TheBr, NewPred, NewAdd, - ConstantInt::get(Int32Ty, ExitValue), - Compare->getName()); + ICmpInst *NewCompare = + new ICmpInst(TheBr->getIterator(), NewPred, NewAdd, + ConstantInt::get(Int32Ty, ExitValue), Compare->getName()); // In the following deletions, PN may become dead and may be deleted. // Use a WeakTrackingVH to observe whether this happens. @@ -386,7 +387,7 @@ bool IndVarSimplify::handleFloatingPointIV(Loop *L, PHINode *PN) { // platforms. if (WeakPH) { Value *Conv = new SIToFPInst(NewPHI, PN->getType(), "indvar.conv", - &*PN->getParent()->getFirstInsertionPt()); + PN->getParent()->getFirstInsertionPt()); PN->replaceAllUsesWith(Conv); RecursivelyDeleteTriviallyDeadInstructions(PN, TLI, MSSAU.get()); } @@ -1516,9 +1517,9 @@ bool IndVarSimplify::canonicalizeExitCondition(Loop *L) { // loop varying work to loop-invariant work. auto doRotateTransform = [&]() { assert(ICmp->isUnsigned() && "must have proven unsigned already"); - auto *NewRHS = - CastInst::Create(Instruction::Trunc, RHS, LHSOp->getType(), "", - L->getLoopPreheader()->getTerminator()); + auto *NewRHS = CastInst::Create( + Instruction::Trunc, RHS, LHSOp->getType(), "", + L->getLoopPreheader()->getTerminator()->getIterator()); ICmp->setOperand(Swapped ? 1 : 0, LHSOp); ICmp->setOperand(Swapped ? 0 : 1, NewRHS); if (LHS->use_empty()) diff --git a/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp b/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp index 851eab04c8db..fbefd0e9368b 100644 --- a/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp +++ b/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp @@ -1313,7 +1313,7 @@ bool InferAddressSpacesImpl::rewriteWithNewAddressSpaces( ++InsertPos; // This instruction may contain multiple uses of V, update them all. CurUser->replaceUsesOfWith( - V, new AddrSpaceCastInst(NewV, V->getType(), "", &*InsertPos)); + V, new AddrSpaceCastInst(NewV, V->getType(), "", InsertPos)); } else { CurUser->replaceUsesOfWith( V, ConstantExpr::getAddrSpaceCast(cast(NewV), diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp index 5816bf2a2259..a04987ce6624 100644 --- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp +++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp @@ -1037,7 +1037,7 @@ bool JumpThreadingPass::processBlock(BasicBlock *BB) { LLVM_DEBUG(dbgs() << " In block '" << BB->getName() << "' folding undef terminator: " << *BBTerm << '\n'); - BranchInst::Create(BBTerm->getSuccessor(BestSucc), BBTerm); + BranchInst::Create(BBTerm->getSuccessor(BestSucc), BBTerm->getIterator()); ++NumFolds; BBTerm->eraseFromParent(); DTU->applyUpdatesPermissive(Updates); @@ -1202,7 +1202,7 @@ bool JumpThreadingPass::processImpliedCondition(BasicBlock *BB) { BasicBlock *KeepSucc = BI->getSuccessor(*Implication ? 0 : 1); BasicBlock *RemoveSucc = BI->getSuccessor(*Implication ? 1 : 0); RemoveSucc->removePredecessor(BB); - BranchInst *UncondBI = BranchInst::Create(KeepSucc, BI); + BranchInst *UncondBI = BranchInst::Create(KeepSucc, BI->getIterator()); UncondBI->setDebugLoc(BI->getDebugLoc()); ++NumFolds; BI->eraseFromParent(); @@ -1280,7 +1280,7 @@ bool JumpThreadingPass::simplifyPartiallyRedundantLoad(LoadInst *LoadI) { AvailableVal = PoisonValue::get(LoadI->getType()); if (AvailableVal->getType() != LoadI->getType()) AvailableVal = CastInst::CreateBitOrPointerCast( - AvailableVal, LoadI->getType(), "", LoadI); + AvailableVal, LoadI->getType(), "", LoadI->getIterator()); LoadI->replaceAllUsesWith(AvailableVal); LoadI->eraseFromParent(); return true; @@ -1421,7 +1421,7 @@ bool JumpThreadingPass::simplifyPartiallyRedundantLoad(LoadInst *LoadI) { LoadI->getType(), LoadedPtr->DoPHITranslation(LoadBB, UnavailablePred), LoadI->getName() + ".pr", false, LoadI->getAlign(), LoadI->getOrdering(), LoadI->getSyncScopeID(), - UnavailablePred->getTerminator()); + UnavailablePred->getTerminator()->getIterator()); NewVal->setDebugLoc(LoadI->getDebugLoc()); if (AATags) NewVal->setAAMetadata(AATags); @@ -1454,8 +1454,8 @@ bool JumpThreadingPass::simplifyPartiallyRedundantLoad(LoadInst *LoadI) { // predecessor use the same bitcast. Value *&PredV = I->second; if (PredV->getType() != LoadI->getType()) - PredV = CastInst::CreateBitOrPointerCast(PredV, LoadI->getType(), "", - P->getTerminator()); + PredV = CastInst::CreateBitOrPointerCast( + PredV, LoadI->getType(), "", P->getTerminator()->getIterator()); PN->addIncoming(PredV, I->first); } @@ -1653,7 +1653,7 @@ bool JumpThreadingPass::processThreadableEdges(Value *Cond, BasicBlock *BB, // Finally update the terminator. Instruction *Term = BB->getTerminator(); - BranchInst::Create(OnlyDest, Term); + BranchInst::Create(OnlyDest, Term->getIterator()); ++NumFolds; Term->eraseFromParent(); DTU->applyUpdatesPermissive(Updates); @@ -2971,13 +2971,13 @@ bool JumpThreadingPass::tryToUnfoldSelectInCurrBB(BasicBlock *BB) { // Expand the select. Value *Cond = SI->getCondition(); if (!isGuaranteedNotToBeUndefOrPoison(Cond, nullptr, SI)) - Cond = new FreezeInst(Cond, "cond.fr", SI); + Cond = new FreezeInst(Cond, "cond.fr", SI->getIterator()); MDNode *BranchWeights = getBranchWeightMDNode(*SI); Instruction *Term = SplitBlockAndInsertIfThen(Cond, SI, false, BranchWeights); BasicBlock *SplitBB = SI->getParent(); BasicBlock *NewBB = Term->getParent(); - PHINode *NewPN = PHINode::Create(SI->getType(), 2, "", SI); + PHINode *NewPN = PHINode::Create(SI->getType(), 2, "", SI->getIterator()); NewPN->addIncoming(SI->getTrueValue(), Term->getParent()); NewPN->addIncoming(SI->getFalseValue(), BB); SI->replaceAllUsesWith(NewPN); diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp index 546e718cb508..40bc16f9b575 100644 --- a/llvm/lib/Transforms/Scalar/LICM.cpp +++ b/llvm/lib/Transforms/Scalar/LICM.cpp @@ -2234,7 +2234,7 @@ bool llvm::promoteLoopAccessesToScalars( if (FoundLoadToPromote || !StoreIsGuanteedToExecute) { PreheaderLoad = new LoadInst(AccessTy, SomePtr, SomePtr->getName() + ".promoted", - Preheader->getTerminator()); + Preheader->getTerminator()->getIterator()); if (SawUnorderedAtomic) PreheaderLoad->setOrdering(AtomicOrdering::Unordered); PreheaderLoad->setAlignment(Alignment); diff --git a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp index 3eca1520a6bd..0e9cf328f149 100644 --- a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp +++ b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp @@ -762,7 +762,7 @@ static bool DoFlattenLoopPair(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI, if (!FI.NewTripCount) { FI.NewTripCount = BinaryOperator::CreateMul( FI.InnerTripCount, FI.OuterTripCount, "flatten.tripcount", - FI.OuterLoop->getLoopPreheader()->getTerminator()); + FI.OuterLoop->getLoopPreheader()->getTerminator()->getIterator()); LLVM_DEBUG(dbgs() << "Created new trip count in preheader: "; FI.NewTripCount->dump()); } diff --git a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp index 3721564890dd..c7e25c9f3d2c 100644 --- a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp +++ b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp @@ -2409,15 +2409,15 @@ bool LoopIdiomRecognize::recognizeShiftUntilBitTest() { if (!isGuaranteedNotToBeUndefOrPoison(BitPos)) { // BitMask may be computed from BitPos, Freeze BitPos so we can increase // it's use count. - Instruction *InsertPt = nullptr; + std::optional InsertPt = std::nullopt; if (auto *BitPosI = dyn_cast(BitPos)) - InsertPt = &**BitPosI->getInsertionPointAfterDef(); + InsertPt = BitPosI->getInsertionPointAfterDef(); else - InsertPt = &*DT->getRoot()->getFirstNonPHIOrDbgOrAlloca(); + InsertPt = DT->getRoot()->getFirstNonPHIOrDbgOrAlloca(); if (!InsertPt) return false; FreezeInst *BitPosFrozen = - new FreezeInst(BitPos, BitPos->getName() + ".fr", InsertPt); + new FreezeInst(BitPos, BitPos->getName() + ".fr", *InsertPt); BitPos->replaceUsesWithIf(BitPosFrozen, [BitPosFrozen](Use &U) { return U.getUser() != BitPosFrozen; }); diff --git a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp index 5ec387300aac..914cf6e21028 100644 --- a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp +++ b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp @@ -440,9 +440,10 @@ public: assert(PH && "Preheader should exist!"); Value *InitialPtr = SEE.expandCodeFor(PtrSCEV->getStart(), Ptr->getType(), PH->getTerminator()); - Value *Initial = new LoadInst( - Cand.Load->getType(), InitialPtr, "load_initial", - /* isVolatile */ false, Cand.Load->getAlign(), PH->getTerminator()); + Value *Initial = + new LoadInst(Cand.Load->getType(), InitialPtr, "load_initial", + /* isVolatile */ false, Cand.Load->getAlign(), + PH->getTerminator()->getIterator()); PHINode *PHI = PHINode::Create(Initial->getType(), 2, "store_forwarded"); PHI->insertBefore(L->getHeader()->begin()); @@ -458,8 +459,9 @@ public: Value *StoreValue = Cand.Store->getValueOperand(); if (LoadType != StoreType) - StoreValue = CastInst::CreateBitOrPointerCast( - StoreValue, LoadType, "store_forward_cast", Cand.Store); + StoreValue = CastInst::CreateBitOrPointerCast(StoreValue, LoadType, + "store_forward_cast", + Cand.Store->getIterator()); PHI->addIncoming(StoreValue, L->getLoopLatch()); diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp index 4f5501611484..4238098181af 100644 --- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp +++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp @@ -2215,17 +2215,18 @@ void LSRInstance::OptimizeShadowIV() { // Ignore negative constants, as the code below doesn't handle them // correctly. TODO: Remove this restriction. - if (!C->getValue().isStrictlyPositive()) continue; + if (!C->getValue().isStrictlyPositive()) + continue; /* Add new PHINode. */ - PHINode *NewPH = PHINode::Create(DestTy, 2, "IV.S.", PH); + PHINode *NewPH = PHINode::Create(DestTy, 2, "IV.S.", PH->getIterator()); /* create new increment. '++d' in above example. */ Constant *CFP = ConstantFP::get(DestTy, C->getZExtValue()); - BinaryOperator *NewIncr = - BinaryOperator::Create(Incr->getOpcode() == Instruction::Add ? - Instruction::FAdd : Instruction::FSub, - NewPH, CFP, "IV.S.next.", Incr); + BinaryOperator *NewIncr = BinaryOperator::Create( + Incr->getOpcode() == Instruction::Add ? Instruction::FAdd + : Instruction::FSub, + NewPH, CFP, "IV.S.next.", Incr->getIterator()); NewPH->addIncoming(NewInit, PH->getIncomingBlock(Entry)); NewPH->addIncoming(NewIncr, PH->getIncomingBlock(Latch)); @@ -2395,8 +2396,8 @@ ICmpInst *LSRInstance::OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse) { // Ok, everything looks ok to change the condition into an SLT or SGE and // delete the max calculation. - ICmpInst *NewCond = - new ICmpInst(Cond, Pred, Cond->getOperand(0), NewRHS, "scmp"); + ICmpInst *NewCond = new ICmpInst(Cond->getIterator(), Pred, + Cond->getOperand(0), NewRHS, "scmp"); // Delete the max calculation instructions. NewCond->setDebugLoc(Cond->getDebugLoc()); @@ -5532,10 +5533,9 @@ Value *LSRInstance::Expand(const LSRUse &LU, const LSRFixup &LF, "a scale at the same time!"); if (F.Scale == -1) { if (ICmpScaledV->getType() != OpTy) { - Instruction *Cast = - CastInst::Create(CastInst::getCastOpcode(ICmpScaledV, false, - OpTy, false), - ICmpScaledV, OpTy, "tmp", CI); + Instruction *Cast = CastInst::Create( + CastInst::getCastOpcode(ICmpScaledV, false, OpTy, false), + ICmpScaledV, OpTy, "tmp", CI->getIterator()); ICmpScaledV = Cast; } CI->setOperand(1, ICmpScaledV); @@ -5635,11 +5635,10 @@ void LSRInstance::RewriteForPHI( // If this is reuse-by-noop-cast, insert the noop cast. Type *OpTy = LF.OperandValToReplace->getType(); if (FullV->getType() != OpTy) - FullV = - CastInst::Create(CastInst::getCastOpcode(FullV, false, - OpTy, false), - FullV, LF.OperandValToReplace->getType(), - "tmp", BB->getTerminator()); + FullV = CastInst::Create( + CastInst::getCastOpcode(FullV, false, OpTy, false), FullV, + LF.OperandValToReplace->getType(), "tmp", + BB->getTerminator()->getIterator()); // If the incoming block for this value is not in the loop, it means the // current PHI is not in a loop exit, so we must create a LCSSA PHI for @@ -5711,8 +5710,8 @@ void LSRInstance::Rewrite(const LSRUse &LU, const LSRFixup &LF, Type *OpTy = LF.OperandValToReplace->getType(); if (FullV->getType() != OpTy) { Instruction *Cast = - CastInst::Create(CastInst::getCastOpcode(FullV, false, OpTy, false), - FullV, OpTy, "tmp", LF.UserInst); + CastInst::Create(CastInst::getCastOpcode(FullV, false, OpTy, false), + FullV, OpTy, "tmp", LF.UserInst->getIterator()); FullV = Cast; } diff --git a/llvm/lib/Transforms/Scalar/NaryReassociate.cpp b/llvm/lib/Transforms/Scalar/NaryReassociate.cpp index 7fe1a222021e..308622615332 100644 --- a/llvm/lib/Transforms/Scalar/NaryReassociate.cpp +++ b/llvm/lib/Transforms/Scalar/NaryReassociate.cpp @@ -511,10 +511,10 @@ Instruction *NaryReassociatePass::tryReassociatedBinaryOp(const SCEV *LHSExpr, Instruction *NewI = nullptr; switch (I->getOpcode()) { case Instruction::Add: - NewI = BinaryOperator::CreateAdd(LHS, RHS, "", I); + NewI = BinaryOperator::CreateAdd(LHS, RHS, "", I->getIterator()); break; case Instruction::Mul: - NewI = BinaryOperator::CreateMul(LHS, RHS, "", I); + NewI = BinaryOperator::CreateMul(LHS, RHS, "", I->getIterator()); break; default: llvm_unreachable("Unexpected instruction."); diff --git a/llvm/lib/Transforms/Scalar/NewGVN.cpp b/llvm/lib/Transforms/Scalar/NewGVN.cpp index 19ac9526b5f8..9caaf720ec91 100644 --- a/llvm/lib/Transforms/Scalar/NewGVN.cpp +++ b/llvm/lib/Transforms/Scalar/NewGVN.cpp @@ -3721,7 +3721,7 @@ void NewGVN::deleteInstructionsInBlock(BasicBlock *BB) { new StoreInst( PoisonValue::get(Int8Ty), Constant::getNullValue(PointerType::getUnqual(BB->getContext())), - BB->getTerminator()); + BB->getTerminator()->getIterator()); } void NewGVN::markInstructionForDeletion(Instruction *I) { diff --git a/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp b/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp index 0266eb1a9f50..436a85f62df6 100644 --- a/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp +++ b/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp @@ -190,7 +190,7 @@ static bool enableBackedgeSafepoints(Function &F); static bool enableCallSafepoints(Function &F); static void -InsertSafepointPoll(Instruction *InsertBefore, +InsertSafepointPoll(BasicBlock::iterator InsertBefore, std::vector &ParsePointsNeeded /*rval*/, const TargetLibraryInfo &TLI); @@ -368,7 +368,7 @@ bool PlaceSafepointsPass::runImpl(Function &F, const TargetLibraryInfo &TLI) { // safepoint polls themselves. for (Instruction *PollLocation : PollsNeeded) { std::vector RuntimeCalls; - InsertSafepointPoll(PollLocation, RuntimeCalls, TLI); + InsertSafepointPoll(PollLocation->getIterator(), RuntimeCalls, TLI); llvm::append_range(ParsePointNeeded, RuntimeCalls); } @@ -619,7 +619,7 @@ static bool enableCallSafepoints(Function &F) { return !NoCall; } // not handle the parsability of state at the runtime call, that's the // callers job. static void -InsertSafepointPoll(Instruction *InsertBefore, +InsertSafepointPoll(BasicBlock::iterator InsertBefore, std::vector &ParsePointsNeeded /*rval*/, const TargetLibraryInfo &TLI) { BasicBlock *OrigBB = InsertBefore->getParent(); diff --git a/llvm/lib/Transforms/Scalar/Reassociate.cpp b/llvm/lib/Transforms/Scalar/Reassociate.cpp index 61109ed37659..d91320863e24 100644 --- a/llvm/lib/Transforms/Scalar/Reassociate.cpp +++ b/llvm/lib/Transforms/Scalar/Reassociate.cpp @@ -246,7 +246,8 @@ void ReassociatePass::canonicalizeOperands(Instruction *I) { } static BinaryOperator *CreateAdd(Value *S1, Value *S2, const Twine &Name, - Instruction *InsertBefore, Value *FlagsOp) { + BasicBlock::iterator InsertBefore, + Value *FlagsOp) { if (S1->getType()->isIntOrIntVectorTy()) return BinaryOperator::CreateAdd(S1, S2, Name, InsertBefore); else { @@ -258,7 +259,8 @@ static BinaryOperator *CreateAdd(Value *S1, Value *S2, const Twine &Name, } static BinaryOperator *CreateMul(Value *S1, Value *S2, const Twine &Name, - Instruction *InsertBefore, Value *FlagsOp) { + BasicBlock::iterator InsertBefore, + Value *FlagsOp) { if (S1->getType()->isIntOrIntVectorTy()) return BinaryOperator::CreateMul(S1, S2, Name, InsertBefore); else { @@ -291,7 +293,8 @@ static BinaryOperator *LowerNegateToMultiply(Instruction *Neg) { Constant *NegOne = Ty->isIntOrIntVectorTy() ? ConstantInt::getAllOnesValue(Ty) : ConstantFP::get(Ty, -1.0); - BinaryOperator *Res = CreateMul(Neg->getOperand(OpNo), NegOne, "", Neg, Neg); + BinaryOperator *Res = + CreateMul(Neg->getOperand(OpNo), NegOne, "", Neg->getIterator(), Neg); Neg->setOperand(OpNo, Constant::getNullValue(Ty)); // Drop use of op. Res->takeName(Neg); Neg->replaceAllUsesWith(Res); @@ -794,8 +797,8 @@ void ReassociatePass::RewriteExprTree(BinaryOperator *I, BinaryOperator *NewOp; if (NodesToRewrite.empty()) { Constant *Undef = UndefValue::get(I->getType()); - NewOp = BinaryOperator::Create(Instruction::BinaryOps(Opcode), - Undef, Undef, "", I); + NewOp = BinaryOperator::Create(Instruction::BinaryOps(Opcode), Undef, + Undef, "", I->getIterator()); if (isa(NewOp)) NewOp->setFastMathFlags(I->getFastMathFlags()); } else { @@ -1046,8 +1049,8 @@ static bool shouldConvertOrWithNoCommonBitsToAdd(Instruction *Or) { /// transform this into (X+Y) to allow arithmetics reassociation. static BinaryOperator *convertOrWithNoCommonBitsToAdd(Instruction *Or) { // Convert an or into an add. - BinaryOperator *New = - CreateAdd(Or->getOperand(0), Or->getOperand(1), "", Or, Or); + BinaryOperator *New = CreateAdd(Or->getOperand(0), Or->getOperand(1), "", + Or->getIterator(), Or); New->setHasNoSignedWrap(); New->setHasNoUnsignedWrap(); New->takeName(Or); @@ -1099,7 +1102,8 @@ static BinaryOperator *BreakUpSubtract(Instruction *Sub, // Calculate the negative value of Operand 1 of the sub instruction, // and set it as the RHS of the add instruction we just made. Value *NegVal = NegateValue(Sub->getOperand(1), Sub, ToRedo); - BinaryOperator *New = CreateAdd(Sub->getOperand(0), NegVal, "", Sub, Sub); + BinaryOperator *New = + CreateAdd(Sub->getOperand(0), NegVal, "", Sub->getIterator(), Sub); Sub->setOperand(0, Constant::getNullValue(Sub->getType())); // Drop use of op. Sub->setOperand(1, Constant::getNullValue(Sub->getType())); // Drop use of op. New->takeName(Sub); @@ -1119,8 +1123,8 @@ static BinaryOperator *ConvertShiftToMul(Instruction *Shl) { auto *SA = cast(Shl->getOperand(1)); MulCst = ConstantExpr::getShl(MulCst, SA); - BinaryOperator *Mul = - BinaryOperator::CreateMul(Shl->getOperand(0), MulCst, "", Shl); + BinaryOperator *Mul = BinaryOperator::CreateMul(Shl->getOperand(0), MulCst, + "", Shl->getIterator()); Shl->setOperand(0, PoisonValue::get(Shl->getType())); // Drop use of op. Mul->takeName(Shl); @@ -1170,13 +1174,13 @@ static unsigned FindInOperandList(const SmallVectorImpl &Ops, /// Emit a tree of add instructions, summing Ops together /// and returning the result. Insert the tree before I. -static Value *EmitAddTreeOfValues(Instruction *I, +static Value *EmitAddTreeOfValues(BasicBlock::iterator It, SmallVectorImpl &Ops) { if (Ops.size() == 1) return Ops.back(); Value *V1 = Ops.pop_back_val(); - Value *V2 = EmitAddTreeOfValues(I, Ops); - return CreateAdd(V2, V1, "reass.add", I, I); + Value *V2 = EmitAddTreeOfValues(It, Ops); + return CreateAdd(V2, V1, "reass.add", It, &*It); } /// If V is an expression tree that is a multiplication sequence, @@ -1323,7 +1327,7 @@ static Value *OptimizeAndOrXor(unsigned Opcode, /// instruction. There are two special cases: 1) if the constant operand is 0, /// it will return NULL. 2) if the constant is ~0, the symbolic operand will /// be returned. -static Value *createAndInstr(Instruction *InsertBefore, Value *Opnd, +static Value *createAndInstr(BasicBlock::iterator InsertBefore, Value *Opnd, const APInt &ConstOpnd) { if (ConstOpnd.isZero()) return nullptr; @@ -1344,7 +1348,7 @@ static Value *createAndInstr(Instruction *InsertBefore, Value *Opnd, // If it was successful, true is returned, and the "R" and "C" is returned // via "Res" and "ConstOpnd", respectively; otherwise, false is returned, // and both "Res" and "ConstOpnd" remain unchanged. -bool ReassociatePass::CombineXorOpnd(Instruction *I, XorOpnd *Opnd1, +bool ReassociatePass::CombineXorOpnd(BasicBlock::iterator It, XorOpnd *Opnd1, APInt &ConstOpnd, Value *&Res) { // Xor-Rule 1: (x | c1) ^ c2 = (x | c1) ^ (c1 ^ c1) ^ c2 // = ((x | c1) ^ c1) ^ (c1 ^ c2) @@ -1361,7 +1365,7 @@ bool ReassociatePass::CombineXorOpnd(Instruction *I, XorOpnd *Opnd1, return false; Value *X = Opnd1->getSymbolicPart(); - Res = createAndInstr(I, X, ~C1); + Res = createAndInstr(It, X, ~C1); // ConstOpnd was C2, now C1 ^ C2. ConstOpnd ^= C1; @@ -1378,7 +1382,7 @@ bool ReassociatePass::CombineXorOpnd(Instruction *I, XorOpnd *Opnd1, // via "Res" and "ConstOpnd", respectively (If the entire expression is // evaluated to a constant, the Res is set to NULL); otherwise, false is // returned, and both "Res" and "ConstOpnd" remain unchanged. -bool ReassociatePass::CombineXorOpnd(Instruction *I, XorOpnd *Opnd1, +bool ReassociatePass::CombineXorOpnd(BasicBlock::iterator It, XorOpnd *Opnd1, XorOpnd *Opnd2, APInt &ConstOpnd, Value *&Res) { Value *X = Opnd1->getSymbolicPart(); @@ -1413,7 +1417,7 @@ bool ReassociatePass::CombineXorOpnd(Instruction *I, XorOpnd *Opnd1, return false; } - Res = createAndInstr(I, X, C3); + Res = createAndInstr(It, X, C3); ConstOpnd ^= C1; } else if (Opnd1->isOrExpr()) { // Xor-Rule 3: (x | c1) ^ (x | c2) = (x & c3) ^ c3 where c3 = c1 ^ c2 @@ -1429,7 +1433,7 @@ bool ReassociatePass::CombineXorOpnd(Instruction *I, XorOpnd *Opnd1, return false; } - Res = createAndInstr(I, X, C3); + Res = createAndInstr(It, X, C3); ConstOpnd ^= C3; } else { // Xor-Rule 4: (x & c1) ^ (x & c2) = (x & (c1^c2)) @@ -1437,7 +1441,7 @@ bool ReassociatePass::CombineXorOpnd(Instruction *I, XorOpnd *Opnd1, const APInt &C1 = Opnd1->getConstPart(); const APInt &C2 = Opnd2->getConstPart(); APInt C3 = C1 ^ C2; - Res = createAndInstr(I, X, C3); + Res = createAndInstr(It, X, C3); } // Put the original operands in the Redo list; hope they will be deleted @@ -1514,7 +1518,8 @@ Value *ReassociatePass::OptimizeXor(Instruction *I, Value *CV; // Step 3.1: Try simplifying "CurrOpnd ^ ConstOpnd" - if (!ConstOpnd.isZero() && CombineXorOpnd(I, CurrOpnd, ConstOpnd, CV)) { + if (!ConstOpnd.isZero() && + CombineXorOpnd(I->getIterator(), CurrOpnd, ConstOpnd, CV)) { Changed = true; if (CV) *CurrOpnd = XorOpnd(CV); @@ -1531,7 +1536,7 @@ Value *ReassociatePass::OptimizeXor(Instruction *I, // step 3.2: When previous and current operands share the same symbolic // value, try to simplify "PrevOpnd ^ CurrOpnd ^ ConstOpnd" - if (CombineXorOpnd(I, CurrOpnd, PrevOpnd, ConstOpnd, CV)) { + if (CombineXorOpnd(I->getIterator(), CurrOpnd, PrevOpnd, ConstOpnd, CV)) { // Remove previous operand PrevOpnd->Invalidate(); if (CV) { @@ -1602,7 +1607,7 @@ Value *ReassociatePass::OptimizeAdd(Instruction *I, Type *Ty = TheOp->getType(); Constant *C = Ty->isIntOrIntVectorTy() ? ConstantInt::get(Ty, NumFound) : ConstantFP::get(Ty, NumFound); - Instruction *Mul = CreateMul(TheOp, C, "factor", I, I); + Instruction *Mul = CreateMul(TheOp, C, "factor", I->getIterator(), I); // Now that we have inserted a multiply, optimize it. This allows us to // handle cases that require multiple factoring steps, such as this: @@ -1766,7 +1771,7 @@ Value *ReassociatePass::OptimizeAdd(Instruction *I, DummyInst->deleteValue(); unsigned NumAddedValues = NewMulOps.size(); - Value *V = EmitAddTreeOfValues(I, NewMulOps); + Value *V = EmitAddTreeOfValues(I->getIterator(), NewMulOps); // Now that we have inserted the add tree, optimize it. This allows us to // handle cases that require multiple factoring steps, such as this: @@ -1777,7 +1782,7 @@ Value *ReassociatePass::OptimizeAdd(Instruction *I, RedoInsts.insert(VI); // Create the multiply. - Instruction *V2 = CreateMul(V, MaxOccVal, "reass.mul", I, I); + Instruction *V2 = CreateMul(V, MaxOccVal, "reass.mul", I->getIterator(), I); // Rerun associate on the multiply in case the inner expression turned into // a multiply. We want to make sure that we keep things in canonical form. diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp index 45ce3bf3ceae..330b464667ee 100644 --- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp +++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp @@ -1143,7 +1143,8 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache, assert(Base && "Can't be null"); // The cast is needed since base traversal may strip away bitcasts if (Base->getType() != Input->getType() && InsertPt) - Base = new BitCastInst(Base, Input->getType(), "cast", InsertPt); + Base = new BitCastInst(Base, Input->getType(), "cast", + InsertPt->getIterator()); return Base; }; @@ -1612,7 +1613,7 @@ public: // Note: we've inserted instructions, so the call to llvm.deoptimize may // not necessarily be followed by the matching return. auto *RI = cast(OldI->getParent()->getTerminator()); - new UnreachableInst(RI->getContext(), RI); + new UnreachableInst(RI->getContext(), RI->getIterator()); RI->eraseFromParent(); } @@ -1976,7 +1977,7 @@ insertRelocationStores(iterator_range GCRelocs, // Emit store into the related alloca. assert(Relocate->getNextNode() && "Should always have one since it's not a terminator"); - new StoreInst(Relocate, Alloca, Relocate->getNextNode()); + new StoreInst(Relocate, Alloca, std::next(Relocate->getIterator())); #ifndef NDEBUG VisitedLiveValues.insert(OriginalValue); @@ -1999,7 +2000,7 @@ static void insertRematerializationStores( Value *Alloca = AllocaMap[OriginalValue]; new StoreInst(RematerializedValue, Alloca, - RematerializedValue->getNextNode()); + std::next(RematerializedValue->getIterator())); #ifndef NDEBUG VisitedLiveValues.insert(OriginalValue); @@ -2031,9 +2032,9 @@ static void relocationViaAlloca( // "PromotableAllocas" const DataLayout &DL = F.getParent()->getDataLayout(); auto emitAllocaFor = [&](Value *LiveValue) { - AllocaInst *Alloca = new AllocaInst(LiveValue->getType(), - DL.getAllocaAddrSpace(), "", - F.getEntryBlock().getFirstNonPHI()); + AllocaInst *Alloca = + new AllocaInst(LiveValue->getType(), DL.getAllocaAddrSpace(), "", + F.getEntryBlock().getFirstNonPHIIt()); AllocaMap[LiveValue] = Alloca; PromotableAllocas.push_back(Alloca); }; @@ -2100,7 +2101,7 @@ static void relocationViaAlloca( ToClobber.push_back(Alloca); } - auto InsertClobbersAt = [&](Instruction *IP) { + auto InsertClobbersAt = [&](BasicBlock::iterator IP) { for (auto *AI : ToClobber) { auto AT = AI->getAllocatedType(); Constant *CPN; @@ -2115,10 +2116,11 @@ static void relocationViaAlloca( // Insert the clobbering stores. These may get intermixed with the // gc.results and gc.relocates, but that's fine. if (auto II = dyn_cast(Statepoint)) { - InsertClobbersAt(&*II->getNormalDest()->getFirstInsertionPt()); - InsertClobbersAt(&*II->getUnwindDest()->getFirstInsertionPt()); + InsertClobbersAt(II->getNormalDest()->getFirstInsertionPt()); + InsertClobbersAt(II->getUnwindDest()->getFirstInsertionPt()); } else { - InsertClobbersAt(cast(Statepoint)->getNextNode()); + InsertClobbersAt( + std::next(cast(Statepoint)->getIterator())); } } } @@ -2154,15 +2156,15 @@ static void relocationViaAlloca( PHINode *Phi = cast(Use); for (unsigned i = 0; i < Phi->getNumIncomingValues(); i++) { if (Def == Phi->getIncomingValue(i)) { - LoadInst *Load = - new LoadInst(Alloca->getAllocatedType(), Alloca, "", - Phi->getIncomingBlock(i)->getTerminator()); + LoadInst *Load = new LoadInst( + Alloca->getAllocatedType(), Alloca, "", + Phi->getIncomingBlock(i)->getTerminator()->getIterator()); Phi->setIncomingValue(i, Load); } } } else { - LoadInst *Load = - new LoadInst(Alloca->getAllocatedType(), Alloca, "", Use); + LoadInst *Load = new LoadInst(Alloca->getAllocatedType(), Alloca, "", + Use->getIterator()); Use->replaceUsesOfWith(Def, Load); } } @@ -2229,16 +2231,16 @@ static void insertUseHolderAfter(CallBase *Call, const ArrayRef Values, if (isa(Call)) { // For call safepoints insert dummy calls right after safepoint Holders.push_back( - CallInst::Create(Func, Values, "", &*++Call->getIterator())); + CallInst::Create(Func, Values, "", std::next(Call->getIterator()))); return; } // For invoke safepooints insert dummy calls both in normal and // exceptional destination blocks auto *II = cast(Call); Holders.push_back(CallInst::Create( - Func, Values, "", &*II->getNormalDest()->getFirstInsertionPt())); + Func, Values, "", II->getNormalDest()->getFirstInsertionPt())); Holders.push_back(CallInst::Create( - Func, Values, "", &*II->getUnwindDest()->getFirstInsertionPt())); + Func, Values, "", II->getUnwindDest()->getFirstInsertionPt())); } static void findLiveReferences( diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp index b2de2209b3f1..cd81fb702f4d 100644 --- a/llvm/lib/Transforms/Scalar/SROA.cpp +++ b/llvm/lib/Transforms/Scalar/SROA.cpp @@ -1853,7 +1853,7 @@ static void rewriteMemOpOfSelect(SelectInst &SI, T &I, Tail->setName(Head->getName() + ".cont"); PHINode *PN; if (isa(I)) - PN = PHINode::Create(I.getType(), 2, "", &I); + PN = PHINode::Create(I.getType(), 2, "", I.getIterator()); for (BasicBlock *SuccBB : successors(Head)) { bool IsThen = SuccBB == HeadBI->getSuccessor(0); int SuccIdx = IsThen ? 0 : 1; @@ -4883,7 +4883,8 @@ AllocaInst *SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS, NewAI = new AllocaInst( SliceTy, AI.getAddressSpace(), nullptr, IsUnconstrained ? DL.getPrefTypeAlign(SliceTy) : Alignment, - AI.getName() + ".sroa." + Twine(P.begin() - AS.begin()), &AI); + AI.getName() + ".sroa." + Twine(P.begin() - AS.begin()), + AI.getIterator()); // Copy the old AI debug location over to the new one. NewAI->setDebugLoc(AI.getDebugLoc()); ++NumNewAllocas; diff --git a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp index 5124909696aa..c54a956fc7e2 100644 --- a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp +++ b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp @@ -244,7 +244,7 @@ public: static int64_t Find(Value *Idx, GetElementPtrInst *GEP); private: - ConstantOffsetExtractor(Instruction *InsertionPt) + ConstantOffsetExtractor(BasicBlock::iterator InsertionPt) : IP(InsertionPt), DL(InsertionPt->getModule()->getDataLayout()) {} /// Searches the expression that computes V for a non-zero constant C s.t. @@ -332,7 +332,7 @@ private: SmallVector ExtInsts; /// Insertion position of cloned instructions. - Instruction *IP; + BasicBlock::iterator IP; const DataLayout &DL; }; @@ -670,7 +670,7 @@ Value *ConstantOffsetExtractor::applyExts(Value *V) { Instruction *Ext = I->clone(); Ext->setOperand(0, Current); - Ext->insertBefore(IP); + Ext->insertBefore(*IP->getParent(), IP); Current = Ext; } return Current; @@ -780,7 +780,7 @@ Value *ConstantOffsetExtractor::removeConstOffset(unsigned ChainIndex) { Value *ConstantOffsetExtractor::Extract(Value *Idx, GetElementPtrInst *GEP, User *&UserChainTail) { - ConstantOffsetExtractor Extractor(GEP); + ConstantOffsetExtractor Extractor(GEP->getIterator()); // Find a non-zero constant offset first. APInt ConstantOffset = Extractor.find(Idx, /* SignExtended */ false, /* ZeroExtended */ false, @@ -797,7 +797,7 @@ Value *ConstantOffsetExtractor::Extract(Value *Idx, GetElementPtrInst *GEP, int64_t ConstantOffsetExtractor::Find(Value *Idx, GetElementPtrInst *GEP) { // If Idx is an index of an inbound GEP, Idx is guaranteed to be non-negative. - return ConstantOffsetExtractor(GEP) + return ConstantOffsetExtractor(GEP->getIterator()) .find(Idx, /* SignExtended */ false, /* ZeroExtended */ false, GEP->isInBounds()) .getSExtValue(); @@ -813,7 +813,8 @@ bool SeparateConstOffsetFromGEP::canonicalizeArrayIndicesToIndexSize( // Skip struct member indices which must be i32. if (GTI.isSequential()) { if ((*I)->getType() != PtrIdxTy) { - *I = CastInst::CreateIntegerCast(*I, PtrIdxTy, true, "idxprom", GEP); + *I = CastInst::CreateIntegerCast(*I, PtrIdxTy, true, "idxprom", + GEP->getIterator()); Changed = true; } } @@ -1249,7 +1250,8 @@ bool SeparateConstOffsetFromGEP::reuniteExts(Instruction *I) { if (LHS->getType() == RHS->getType()) { ExprKey Key = createNormalizedCommutablePair(LHS, RHS); if (auto *Dom = findClosestMatchingDominator(Key, I, DominatingAdds)) { - Instruction *NewSExt = new SExtInst(Dom, I->getType(), "", I); + Instruction *NewSExt = + new SExtInst(Dom, I->getType(), "", I->getIterator()); NewSExt->takeName(I); I->replaceAllUsesWith(NewSExt); RecursivelyDeleteTriviallyDeadInstructions(I); @@ -1260,7 +1262,8 @@ bool SeparateConstOffsetFromGEP::reuniteExts(Instruction *I) { if (LHS->getType() == RHS->getType()) { if (auto *Dom = findClosestMatchingDominator({LHS, RHS}, I, DominatingSubs)) { - Instruction *NewSExt = new SExtInst(Dom, I->getType(), "", I); + Instruction *NewSExt = + new SExtInst(Dom, I->getType(), "", I->getIterator()); NewSExt->takeName(I); I->replaceAllUsesWith(NewSExt); RecursivelyDeleteTriviallyDeadInstructions(I); diff --git a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp index 7eb0ba1c2c17..3d146086d31a 100644 --- a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp +++ b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp @@ -2347,8 +2347,7 @@ static void unswitchNontrivialInvariants( BI->setSuccessor(1 - ClonedSucc, LoopPH); Value *Cond = skipTrivialSelect(BI->getCondition()); if (InsertFreeze) - Cond = new FreezeInst( - Cond, Cond->getName() + ".fr", BI); + Cond = new FreezeInst(Cond, Cond->getName() + ".fr", BI->getIterator()); BI->setCondition(Cond); DTUpdates.push_back({DominatorTree::Insert, SplitBB, ClonedPH}); } else { @@ -2365,8 +2364,9 @@ static void unswitchNontrivialInvariants( Case.setSuccessor(ClonedPHs.find(Case.getCaseSuccessor())->second); if (InsertFreeze) - SI->setCondition(new FreezeInst( - SI->getCondition(), SI->getCondition()->getName() + ".fr", SI)); + SI->setCondition(new FreezeInst(SI->getCondition(), + SI->getCondition()->getName() + ".fr", + SI->getIterator())); // We need to use the set to populate domtree updates as even when there // are multiple cases pointing at the same successor we only want to @@ -2704,7 +2704,8 @@ static BranchInst *turnSelectIntoBranch(SelectInst *SI, DominatorTree &DT, if (MSSAU) MSSAU->moveAllAfterSpliceBlocks(HeadBB, TailBB, SI); - PHINode *Phi = PHINode::Create(SI->getType(), 2, "unswitched.select", SI); + PHINode *Phi = + PHINode::Create(SI->getType(), 2, "unswitched.select", SI->getIterator()); Phi->addIncoming(SI->getTrueValue(), ThenBB); Phi->addIncoming(SI->getFalseValue(), HeadBB); SI->replaceAllUsesWith(Phi); @@ -3092,7 +3093,7 @@ injectPendingInvariantConditions(NonTrivialUnswitchCandidate Candidate, Loop &L, // unswitching will break. Better optimize it away later. auto *InjectedCond = ICmpInst::Create(Instruction::ICmp, Pred, LHS, RHS, "injected.cond", - Preheader->getTerminator()); + Preheader->getTerminator()->getIterator()); BasicBlock *CheckBlock = BasicBlock::Create(Ctx, BB->getName() + ".check", BB->getParent(), InLoopSucc); diff --git a/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp b/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp index c6e8505d5ab4..519ff3221a3b 100644 --- a/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp +++ b/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp @@ -601,7 +601,7 @@ void TailRecursionEliminator::copyByValueOperandIntoLocalTemp(CallInst *CI, // Put alloca into the entry block. Value *NewAlloca = new AllocaInst( AggTy, DL.getAllocaAddrSpace(), nullptr, Alignment, - CI->getArgOperand(OpndIdx)->getName(), &*F.getEntryBlock().begin()); + CI->getArgOperand(OpndIdx)->getName(), F.getEntryBlock().begin()); IRBuilder<> Builder(CI); Value *Size = Builder.getInt64(DL.getTypeAllocSize(AggTy)); @@ -714,8 +714,9 @@ bool TailRecursionEliminator::eliminateCall(CallInst *CI) { // We found a return value we want to use, insert a select instruction to // select it if we don't already know what our return value will be and // store the result in our return value PHI node. - SelectInst *SI = SelectInst::Create( - RetKnownPN, RetPN, Ret->getReturnValue(), "current.ret.tr", Ret); + SelectInst *SI = + SelectInst::Create(RetKnownPN, RetPN, Ret->getReturnValue(), + "current.ret.tr", Ret->getIterator()); RetSelects.push_back(SI); RetPN->addIncoming(SI, BB); @@ -728,7 +729,7 @@ bool TailRecursionEliminator::eliminateCall(CallInst *CI) { // Now that all of the PHI nodes are in place, remove the call and // ret instructions, replacing them with an unconditional branch. - BranchInst *NewBI = BranchInst::Create(HeaderBB, Ret); + BranchInst *NewBI = BranchInst::Create(HeaderBB, Ret->getIterator()); NewBI->setDebugLoc(CI->getDebugLoc()); Ret->eraseFromParent(); // Remove return. @@ -787,8 +788,9 @@ void TailRecursionEliminator::cleanupAndFinalize() { if (!RI) continue; - SelectInst *SI = SelectInst::Create( - RetKnownPN, RetPN, RI->getOperand(0), "current.ret.tr", RI); + SelectInst *SI = + SelectInst::Create(RetKnownPN, RetPN, RI->getOperand(0), + "current.ret.tr", RI->getIterator()); RetSelects.push_back(SI); RI->setOperand(0, SI); } diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 50a073e89062..edaad4d033bd 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -3007,8 +3007,9 @@ PHINode *InnerLoopVectorizer::createInductionResumeValue( } // Create phi nodes to merge from the backedge-taken check block. - PHINode *BCResumeVal = PHINode::Create(OrigPhi->getType(), 3, "bc.resume.val", - LoopScalarPreHeader->getTerminator()); + PHINode *BCResumeVal = + PHINode::Create(OrigPhi->getType(), 3, "bc.resume.val", + LoopScalarPreHeader->getTerminator()->getIterator()); // Copy original phi DL over to the new one. BCResumeVal->setDebugLoc(OrigPhi->getDebugLoc()); @@ -7415,8 +7416,9 @@ static void createAndCollectMergePhiForReduction( BasicBlock *LoopScalarPreHeader = OrigLoop->getLoopPreheader(); // Create a phi node that merges control-flow from the backedge-taken check // block and the middle block. - auto *BCBlockPhi = PHINode::Create(FinalValue->getType(), 2, "bc.merge.rdx", - LoopScalarPreHeader->getTerminator()); + auto *BCBlockPhi = + PHINode::Create(FinalValue->getType(), 2, "bc.merge.rdx", + LoopScalarPreHeader->getTerminator()->getIterator()); // If we are fixing reductions in the epilogue loop then we should already // have created a bc.merge.rdx Phi after the main vector body. Ensure that @@ -9176,14 +9178,14 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) { // Build a pointer phi Value *ScalarStartValue = getStartValue()->getLiveInIRValue(); Type *ScStValueType = ScalarStartValue->getType(); - PHINode *NewPointerPhi = - PHINode::Create(ScStValueType, 2, "pointer.phi", CanonicalIV); + PHINode *NewPointerPhi = PHINode::Create(ScStValueType, 2, "pointer.phi", + CanonicalIV->getIterator()); BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this); NewPointerPhi->addIncoming(ScalarStartValue, VectorPH); // A pointer induction, performed by using a gep - Instruction *InductionLoc = &*State.Builder.GetInsertPoint(); + BasicBlock::iterator InductionLoc = State.Builder.GetInsertPoint(); Value *ScalarStepValue = State.get(getOperand(1), VPIteration(0, 0)); Value *RuntimeVF = getRuntimeVF(State.Builder, PhiType, State.VF); -- GitLab From a41bcb3930534ef1525b4fc30e53e818b39e2b60 Mon Sep 17 00:00:00 2001 From: Weining Lu Date: Tue, 5 Mar 2024 23:19:16 +0800 Subject: [PATCH 152/929] [lld][LoongArch] Fix handleUleb128 --- lld/ELF/Arch/LoongArch.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lld/ELF/Arch/LoongArch.cpp b/lld/ELF/Arch/LoongArch.cpp index 5f0b73df48c9..2c5d5df922c0 100644 --- a/lld/ELF/Arch/LoongArch.cpp +++ b/lld/ELF/Arch/LoongArch.cpp @@ -159,8 +159,9 @@ static bool isJirl(uint32_t insn) { static void handleUleb128(uint8_t *loc, uint64_t val) { const uint32_t maxcount = 1 + 64 / 7; uint32_t count; - uint64_t orig = decodeULEB128(loc, &count); - if (count > maxcount) + const char *error = nullptr; + uint64_t orig = decodeULEB128(loc, &count, nullptr, &error); + if (count > maxcount || (count == maxcount && error)) errorOrWarn(getErrorLocation(loc) + "extra space for uleb128"); uint64_t mask = count < maxcount ? (1ULL << 7 * count) - 1 : -1ULL; encodeULEB128((orig + val) & mask, loc, count); -- GitLab From 6409c21857373bbb5563b37c3bacc066ad633658 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Tue, 5 Mar 2024 16:38:09 +0100 Subject: [PATCH 153/929] [SCEVExpander] Use PoisoningVH for OrigFlags It's common to delete some instructions after using SCEVExpander, while it is still live (but will not be used afterwards). In that case, the AssertingVH may trigger. Replace it with a PoisoningVH so that we only detect the case where the SCEVExpander actually is used in a problematic fashion after the instruction removal. The alternative would be to add clear() calls to more code paths. Fixes https://github.com/llvm/llvm-project/issues/83404. --- .../Utils/ScalarEvolutionExpander.h | 2 +- .../LoopStrengthReduce/X86/pr83404.ll | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/pr83404.ll diff --git a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h index 9de0996fb1e3..62c1e15a9a60 100644 --- a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h +++ b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h @@ -85,7 +85,7 @@ class SCEVExpander : public SCEVVisitor { /// Original flags of instructions for which they were modified. Used /// by SCEVExpanderCleaner to undo changes. - DenseMap, PoisonFlags> OrigFlags; + DenseMap, PoisonFlags> OrigFlags; // The induction variables generated. SmallVector InsertedIVs; diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/pr83404.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/pr83404.ll new file mode 100644 index 000000000000..1a53d321f26e --- /dev/null +++ b/llvm/test/Transforms/LoopStrengthReduce/X86/pr83404.ll @@ -0,0 +1,23 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -passes=loop-reduce -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s + +; Make sure this does not assert. +define void @test() { +; CHECK-LABEL: define void @test() { +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.body: +; CHECK-NEXT: br label [[FOR_BODY]] +; +entry: + br label %for.body + +for.body: + %0 = phi i64 [ %dec, %for.body ], [ 0, %entry ] + %h.18 = phi i32 [ 1, %for.body ], [ 0, %entry ] + %dec = add i64 %0, 1 + %rem = urem i64 %0, 5 + %1 = trunc i64 %rem to i32 + %tobool3.not = icmp eq i32 %h.18, %1 + br label %for.body +} -- GitLab From 08e036e734fe842cde9af13fceb8bb3d951b56a6 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Tue, 5 Mar 2024 15:15:36 +0000 Subject: [PATCH 154/929] [PhaseOrdering][X86] Add test coverage for #67803 --- .../Transforms/PhaseOrdering/X86/pr67803.ll | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll new file mode 100644 index 000000000000..211c90b5604e --- /dev/null +++ b/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll @@ -0,0 +1,83 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s +; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s +; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s + +define <4 x i64> @PR67803(<4 x i64> %x, <4 x i64> %y, <4 x i64> %a, <4 x i64> %b) { +; CHECK-LABEL: @PR67803( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i64> [[X:%.*]] to <8 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i64> [[Y:%.*]] to <8 x i32> +; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <8 x i32> [[TMP0]], [[TMP1]] +; CHECK-NEXT: [[CMP_I21:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <4 x i32> +; CHECK-NEXT: [[SEXT_I22:%.*]] = sext <4 x i1> [[CMP_I21]] to <4 x i32> +; CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[SEXT_I22]] to <2 x i64> +; CHECK-NEXT: [[CMP_I:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <4 x i32> +; CHECK-NEXT: [[SEXT_I:%.*]] = sext <4 x i1> [[CMP_I]] to <4 x i32> +; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x i32> [[SEXT_I]] to <2 x i64> +; CHECK-NEXT: [[SHUFFLE_I:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <4 x i32> +; CHECK-NEXT: [[TMP5:%.*]] = bitcast <4 x i64> [[A:%.*]] to <32 x i8> +; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <32 x i8> [[TMP5]], <32 x i8> poison, <16 x i32> +; CHECK-NEXT: [[TMP7:%.*]] = bitcast <4 x i64> [[B:%.*]] to <32 x i8> +; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <32 x i8> [[TMP7]], <32 x i8> poison, <16 x i32> +; CHECK-NEXT: [[TMP9:%.*]] = bitcast <4 x i64> [[SHUFFLE_I]] to <32 x i8> +; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <32 x i8> [[TMP9]], <32 x i8> poison, <16 x i32> +; CHECK-NEXT: [[TMP11:%.*]] = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> [[TMP6]], <16 x i8> [[TMP8]], <16 x i8> [[TMP10]]) +; CHECK-NEXT: [[TMP12:%.*]] = bitcast <16 x i8> [[TMP11]] to <2 x i64> +; CHECK-NEXT: [[TMP13:%.*]] = bitcast <4 x i64> [[A]] to <32 x i8> +; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <32 x i8> [[TMP13]], <32 x i8> poison, <16 x i32> +; CHECK-NEXT: [[TMP15:%.*]] = bitcast <4 x i64> [[B]] to <32 x i8> +; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <32 x i8> [[TMP15]], <32 x i8> poison, <16 x i32> +; CHECK-NEXT: [[TMP17:%.*]] = bitcast <4 x i64> [[SHUFFLE_I]] to <32 x i8> +; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <32 x i8> [[TMP17]], <32 x i8> poison, <16 x i32> +; CHECK-NEXT: [[TMP19:%.*]] = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> [[TMP14]], <16 x i8> [[TMP16]], <16 x i8> [[TMP18]]) +; CHECK-NEXT: [[TMP20:%.*]] = bitcast <16 x i8> [[TMP19]] to <2 x i64> +; CHECK-NEXT: [[SHUFFLE_I23:%.*]] = shufflevector <2 x i64> [[TMP12]], <2 x i64> [[TMP20]], <4 x i32> +; CHECK-NEXT: ret <4 x i64> [[SHUFFLE_I23]] +; +entry: + %0 = bitcast <4 x i64> %x to <8 x i32> + %extract = shufflevector <8 x i32> %0, <8 x i32> poison, <4 x i32> + %1 = bitcast <4 x i32> %extract to <2 x i64> + %2 = bitcast <4 x i64> %y to <8 x i32> + %extract1 = shufflevector <8 x i32> %2, <8 x i32> poison, <4 x i32> + %3 = bitcast <4 x i32> %extract1 to <2 x i64> + %cmp.i21 = icmp sgt <4 x i32> %extract, %extract1 + %sext.i22 = sext <4 x i1> %cmp.i21 to <4 x i32> + %4 = bitcast <4 x i32> %sext.i22 to <2 x i64> + %extract2 = shufflevector <8 x i32> %0, <8 x i32> poison, <4 x i32> + %5 = bitcast <4 x i32> %extract2 to <2 x i64> + %extract3 = shufflevector <8 x i32> %2, <8 x i32> poison, <4 x i32> + %6 = bitcast <4 x i32> %extract3 to <2 x i64> + %cmp.i = icmp sgt <4 x i32> %extract2, %extract3 + %sext.i = sext <4 x i1> %cmp.i to <4 x i32> + %7 = bitcast <4 x i32> %sext.i to <2 x i64> + %shuffle.i = shufflevector <2 x i64> %4, <2 x i64> %7, <4 x i32> + %8 = bitcast <4 x i64> %a to <8 x i32> + %extract6 = shufflevector <8 x i32> %8, <8 x i32> poison, <4 x i32> + %9 = bitcast <4 x i32> %extract6 to <2 x i64> + %10 = bitcast <4 x i64> %b to <8 x i32> + %extract7 = shufflevector <8 x i32> %10, <8 x i32> poison, <4 x i32> + %11 = bitcast <4 x i32> %extract7 to <2 x i64> + %12 = bitcast <4 x i64> %shuffle.i to <8 x i32> + %extract8 = shufflevector <8 x i32> %12, <8 x i32> poison, <4 x i32> + %13 = bitcast <4 x i32> %extract8 to <2 x i64> + %14 = bitcast <2 x i64> %9 to <16 x i8> + %15 = bitcast <2 x i64> %11 to <16 x i8> + %16 = bitcast <2 x i64> %13 to <16 x i8> + %17 = call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> %14, <16 x i8> %15, <16 x i8> %16) + %18 = bitcast <16 x i8> %17 to <2 x i64> + %extract10 = shufflevector <8 x i32> %8, <8 x i32> poison, <4 x i32> + %19 = bitcast <4 x i32> %extract10 to <2 x i64> + %extract11 = shufflevector <8 x i32> %10, <8 x i32> poison, <4 x i32> + %20 = bitcast <4 x i32> %extract11 to <2 x i64> + %extract12 = shufflevector <8 x i32> %12, <8 x i32> poison, <4 x i32> + %21 = bitcast <4 x i32> %extract12 to <2 x i64> + %22 = bitcast <2 x i64> %19 to <16 x i8> + %23 = bitcast <2 x i64> %20 to <16 x i8> + %24 = bitcast <2 x i64> %21 to <16 x i8> + %25 = call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> %22, <16 x i8> %23, <16 x i8> %24) + %26 = bitcast <16 x i8> %25 to <2 x i64> + %shuffle.i23 = shufflevector <2 x i64> %18, <2 x i64> %26, <4 x i32> + ret <4 x i64> %shuffle.i23 +} -- GitLab From da63746bdfaf8a077131d932d04c1e349e715536 Mon Sep 17 00:00:00 2001 From: Natalie Chouinard Date: Tue, 5 Mar 2024 11:09:10 -0500 Subject: [PATCH 155/929] [infra] Add git to Linux container (#82687) Fixes #82646 --- .github/workflows/containers/github-action-ci/stage2.Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/containers/github-action-ci/stage2.Dockerfile b/.github/workflows/containers/github-action-ci/stage2.Dockerfile index e1a06cb68a58..0ca0da87734c 100644 --- a/.github/workflows/containers/github-action-ci/stage2.Dockerfile +++ b/.github/workflows/containers/github-action-ci/stage2.Dockerfile @@ -12,11 +12,13 @@ COPY --from=stage2-toolchain $LLVM_SYSROOT $LLVM_SYSROOT # Need to install curl for hendrikmuhs/ccache-action # Need nodejs for some of the GitHub actions. # Need perl-modules for clang analyzer tests. +# Need git for SPIRV-Tools tests. RUN apt-get update && \ apt-get install -y \ binutils \ cmake \ curl \ + git \ libstdc++-11-dev \ ninja-build \ nodejs \ -- GitLab From ec7062d9d822df42dee6dccada13f42798976f3c Mon Sep 17 00:00:00 2001 From: David Goldman Date: Tue, 5 Mar 2024 11:10:57 -0500 Subject: [PATCH 156/929] [clangd] Add metric for rename decl kind (#83867) This will give us insight into what users are renaming in practice - for instance, try to gauge the impact of the ObjC rename support. --- clang-tools-extra/clangd/refactor/Rename.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/clang-tools-extra/clangd/refactor/Rename.cpp b/clang-tools-extra/clangd/refactor/Rename.cpp index 4e135801f685..75b30e66d637 100644 --- a/clang-tools-extra/clangd/refactor/Rename.cpp +++ b/clang-tools-extra/clangd/refactor/Rename.cpp @@ -1062,6 +1062,10 @@ llvm::Expected rename(const RenameInputs &RInputs) { return makeError(ReasonToReject::AmbiguousSymbol); const auto &RenameDecl = **DeclsUnderCursor.begin(); + static constexpr trace::Metric RenameTriggerCounter( + "rename_trigger_count", trace::Metric::Counter, "decl_kind"); + RenameTriggerCounter.record(1, RenameDecl.getDeclKindName()); + std::string Placeholder = getName(RenameDecl); auto Invalid = checkName(RenameDecl, RInputs.NewName, Placeholder); if (Invalid) -- GitLab From 88d82b747cdc521686e6bbc108b021753842a902 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Tue, 5 Mar 2024 08:15:56 -0800 Subject: [PATCH 157/929] [libc] fix more readability-identifier-naming lints (#83914) Found via: $ ninja -k2000 libc-lint 2>&1 | grep readability-identifier-naming Auto fixed via: $ clang-tidy -p build/compile_commands.json \ -checks="-*,readability-identifier-naming" \ --fix This doesn't fix all instances, just the obvious simple cases where it makes sense to change the identifier names. Subsequent PRs will fix up the stragglers. --- libc/src/__support/blockstore.h | 30 +++---- libc/src/__support/integer_to_string.h | 14 ++-- libc/src/__support/math_extras.h | 14 ++-- libc/src/math/generic/hypotf.cpp | 4 +- libc/src/string/memory_utils/op_generic.h | 22 ++--- libc/src/string/memory_utils/op_x86.h | 12 +-- libc/src/string/memory_utils/utils.h | 18 ++--- .../memory_utils/x86_64/inline_memcpy.h | 80 +++++++++---------- .../memory_utils/x86_64/inline_memset.h | 22 ++--- 9 files changed, 109 insertions(+), 107 deletions(-) diff --git a/libc/src/__support/blockstore.h b/libc/src/__support/blockstore.h index d78e4be5fa9c..dc5fdd1b92fc 100644 --- a/libc/src/__support/blockstore.h +++ b/libc/src/__support/blockstore.h @@ -45,7 +45,7 @@ protected: struct Pair { Block *first, *second; }; - Pair getLastBlocks() { + Pair get_last_blocks() { if (REVERSE_ORDER) return {current, current->next}; Block *prev = nullptr; @@ -56,20 +56,20 @@ protected: return {curr, prev}; } - Block *getLastBlock() { return getLastBlocks().first; } + Block *get_last_block() { return get_last_blocks().first; } public: constexpr BlockStore() = default; ~BlockStore() = default; - class iterator { + class Iterator { Block *block; size_t index; public: - constexpr iterator(Block *b, size_t i) : block(b), index(i) {} + constexpr Iterator(Block *b, size_t i) : block(b), index(i) {} - iterator &operator++() { + Iterator &operator++() { if (REVERSE_ORDER) { if (index == 0) return *this; @@ -98,11 +98,11 @@ public: return *reinterpret_cast(block->data + sizeof(T) * true_index); } - bool operator==(const iterator &rhs) const { + bool operator==(const Iterator &rhs) const { return block == rhs.block && index == rhs.index; } - bool operator!=(const iterator &rhs) const { + bool operator!=(const Iterator &rhs) const { return block != rhs.block || index != rhs.index; } }; @@ -138,7 +138,7 @@ public: } T &back() { - return *reinterpret_cast(getLastBlock()->data + + return *reinterpret_cast(get_last_block()->data + sizeof(T) * (fill_count - 1)); } @@ -146,7 +146,7 @@ public: fill_count--; if (fill_count || current == &first) return; - auto [last, prev] = getLastBlocks(); + auto [last, prev] = get_last_blocks(); if (REVERSE_ORDER) { LIBC_ASSERT(last == current); current = current->next; @@ -162,18 +162,18 @@ public: bool empty() const { return current == &first && !fill_count; } - iterator begin() { + Iterator begin() { if (REVERSE_ORDER) - return iterator(current, fill_count); + return Iterator(current, fill_count); else - return iterator(&first, 0); + return Iterator(&first, 0); } - iterator end() { + Iterator end() { if (REVERSE_ORDER) - return iterator(&first, 0); + return Iterator(&first, 0); else - return iterator(current, fill_count); + return Iterator(current, fill_count); } }; diff --git a/libc/src/__support/integer_to_string.h b/libc/src/__support/integer_to_string.h index 8d3859c8eb0c..81ed21ccfca1 100644 --- a/libc/src/__support/integer_to_string.h +++ b/libc/src/__support/integer_to_string.h @@ -166,7 +166,7 @@ template class IntegerToString { static_assert(cpp::is_integral_v); LIBC_INLINE static constexpr size_t compute_buffer_size() { - constexpr auto max_digits = []() -> size_t { + constexpr auto MAX_DIGITS = []() -> size_t { // We size the string buffer for base 10 using an approximation algorithm: // // size = ceil(sizeof(T) * 5 / 2) @@ -188,19 +188,19 @@ template class IntegerToString { // For other bases, we approximate by rounding down to the nearest power // of two base, since the space needed is easy to calculate and it won't // overestimate by too much. - constexpr auto floor_log_2 = [](size_t num) -> size_t { + constexpr auto FLOOR_LOG_2 = [](size_t num) -> size_t { size_t i = 0; for (; num > 1; num /= 2) ++i; return i; }; - constexpr size_t BITS_PER_DIGIT = floor_log_2(Fmt::BASE); + constexpr size_t BITS_PER_DIGIT = FLOOR_LOG_2(Fmt::BASE); return ((sizeof(T) * 8 + (BITS_PER_DIGIT - 1)) / BITS_PER_DIGIT); }; - constexpr size_t digit_size = cpp::max(max_digits(), Fmt::MIN_DIGITS); - constexpr size_t sign_size = Fmt::BASE == 10 ? 1 : 0; - constexpr size_t prefix_size = Fmt::PREFIX ? 2 : 0; - return digit_size + sign_size + prefix_size; + constexpr size_t DIGIT_SIZE = cpp::max(MAX_DIGITS(), Fmt::MIN_DIGITS); + constexpr size_t SIGN_SIZE = Fmt::BASE == 10 ? 1 : 0; + constexpr size_t PREFIX_SIZE = Fmt::PREFIX ? 2 : 0; + return DIGIT_SIZE + SIGN_SIZE + PREFIX_SIZE; } static constexpr size_t BUFFER_SIZE = compute_buffer_size(); diff --git a/libc/src/__support/math_extras.h b/libc/src/__support/math_extras.h index ae367994706c..7a89fbb11b2a 100644 --- a/libc/src/__support/math_extras.h +++ b/libc/src/__support/math_extras.h @@ -22,21 +22,21 @@ namespace LIBC_NAMESPACE { template LIBC_INLINE constexpr T mask_trailing_ones() { static_assert(cpp::is_unsigned_v); - constexpr unsigned t_bits = CHAR_BIT * sizeof(T); - static_assert(count <= t_bits && "Invalid bit index"); + constexpr unsigned T_BITS = CHAR_BIT * sizeof(T); + static_assert(count <= T_BITS && "Invalid bit index"); // It's important not to initialize T with -1, since T may be BigInt which // will take -1 as a uint64_t and only initialize the low 64 bits. - constexpr T all_zeroes(0); - constexpr T all_ones(~all_zeroes); // bitwise NOT performs integer promotion. - return count == 0 ? 0 : (all_ones >> (t_bits - count)); + constexpr T ALL_ZEROES(0); + constexpr T ALL_ONES(~ALL_ZEROES); // bitwise NOT performs integer promotion. + return count == 0 ? 0 : (ALL_ONES >> (T_BITS - count)); } // Create a bitmask with the count left-most bits set to 1, and all other bits // set to 0. Only unsigned types are allowed. template LIBC_INLINE constexpr T mask_leading_ones() { - constexpr T mask(mask_trailing_ones()); - return T(~mask); // bitwise NOT performs integer promotion. + constexpr T MASK(mask_trailing_ones()); + return T(~MASK); // bitwise NOT performs integer promotion. } // Add with carry diff --git a/libc/src/math/generic/hypotf.cpp b/libc/src/math/generic/hypotf.cpp index 614aa399fcc2..ffbf706aefaf 100644 --- a/libc/src/math/generic/hypotf.cpp +++ b/libc/src/math/generic/hypotf.cpp @@ -48,8 +48,8 @@ LLVM_LIBC_FUNCTION(float, hypotf, (float x, float y)) { // Correct rounding. double r_sq = result.get_val() * result.get_val(); double diff = sum_sq - r_sq; - constexpr uint64_t mask = 0x0000'0000'3FFF'FFFFULL; - uint64_t lrs = result.uintval() & mask; + constexpr uint64_t MASK = 0x0000'0000'3FFF'FFFFULL; + uint64_t lrs = result.uintval() & MASK; if (lrs == 0x0000'0000'1000'0000ULL && err < diff) { result.set_uintval(result.uintval() | 1ULL); diff --git a/libc/src/string/memory_utils/op_generic.h b/libc/src/string/memory_utils/op_generic.h index c7dbd5dd1d6c..28243c7a1816 100644 --- a/libc/src/string/memory_utils/op_generic.h +++ b/libc/src/string/memory_utils/op_generic.h @@ -95,10 +95,10 @@ template T load(CPtr src) { return ::LIBC_NAMESPACE::load(src); } else if constexpr (is_array_v) { using value_type = typename T::value_type; - T Value; - for (size_t I = 0; I < array_size_v; ++I) - Value[I] = load(src + (I * sizeof(value_type))); - return Value; + T value; + for (size_t i = 0; i < array_size_v; ++i) + value[i] = load(src + (i * sizeof(value_type))); + return value; } } @@ -108,8 +108,8 @@ template void store(Ptr dst, T value) { ::LIBC_NAMESPACE::store(dst, value); } else if constexpr (is_array_v) { using value_type = typename T::value_type; - for (size_t I = 0; I < array_size_v; ++I) - store(dst + (I * sizeof(value_type)), value[I]); + for (size_t i = 0; i < array_size_v; ++i) + store(dst + (i * sizeof(value_type)), value[i]); } } @@ -118,11 +118,11 @@ template T splat(uint8_t value) { if constexpr (is_scalar_v) return T(~0) / T(0xFF) * T(value); else if constexpr (is_vector_v) { - T Out; + T out; // This for loop is optimized out for vector types. for (size_t i = 0; i < sizeof(T); ++i) - Out[i] = value; - return Out; + out[i] = value; + return out; } } @@ -140,8 +140,8 @@ template struct Memset { } else if constexpr (is_array_v) { using value_type = typename T::value_type; const auto Splat = splat(value); - for (size_t I = 0; I < array_size_v; ++I) - store(dst + (I * sizeof(value_type)), Splat); + for (size_t i = 0; i < array_size_v; ++i) + store(dst + (i * sizeof(value_type)), Splat); } } diff --git a/libc/src/string/memory_utils/op_x86.h b/libc/src/string/memory_utils/op_x86.h index 2852636c48a7..1afa91f20e65 100644 --- a/libc/src/string/memory_utils/op_x86.h +++ b/libc/src/string/memory_utils/op_x86.h @@ -40,12 +40,12 @@ namespace LIBC_NAMESPACE::x86 { // A set of constants to check compile time features. -LIBC_INLINE_VAR constexpr bool kSse2 = LLVM_LIBC_IS_DEFINED(__SSE2__); -LIBC_INLINE_VAR constexpr bool kSse41 = LLVM_LIBC_IS_DEFINED(__SSE4_1__); -LIBC_INLINE_VAR constexpr bool kAvx = LLVM_LIBC_IS_DEFINED(__AVX__); -LIBC_INLINE_VAR constexpr bool kAvx2 = LLVM_LIBC_IS_DEFINED(__AVX2__); -LIBC_INLINE_VAR constexpr bool kAvx512F = LLVM_LIBC_IS_DEFINED(__AVX512F__); -LIBC_INLINE_VAR constexpr bool kAvx512BW = LLVM_LIBC_IS_DEFINED(__AVX512BW__); +LIBC_INLINE_VAR constexpr bool K_SSE2 = LLVM_LIBC_IS_DEFINED(__SSE2__); +LIBC_INLINE_VAR constexpr bool K_SSE41 = LLVM_LIBC_IS_DEFINED(__SSE4_1__); +LIBC_INLINE_VAR constexpr bool K_AVX = LLVM_LIBC_IS_DEFINED(__AVX__); +LIBC_INLINE_VAR constexpr bool K_AVX2 = LLVM_LIBC_IS_DEFINED(__AVX2__); +LIBC_INLINE_VAR constexpr bool K_AVX512_F = LLVM_LIBC_IS_DEFINED(__AVX512F__); +LIBC_INLINE_VAR constexpr bool K_AVX512_BW = LLVM_LIBC_IS_DEFINED(__AVX512BW__); /////////////////////////////////////////////////////////////////////////////// // Memcpy repmovsb implementation diff --git a/libc/src/string/memory_utils/utils.h b/libc/src/string/memory_utils/utils.h index 701a84375ea8..6e26e0c6a54d 100644 --- a/libc/src/string/memory_utils/utils.h +++ b/libc/src/string/memory_utils/utils.h @@ -205,9 +205,9 @@ LIBC_INLINE MemcmpReturnType cmp_neq_uint64_t(uint64_t a, uint64_t b) { // Loads bytes from memory (possibly unaligned) and materializes them as // type. template LIBC_INLINE T load(CPtr ptr) { - T Out; - memcpy_inline(&Out, ptr); - return Out; + T out; + memcpy_inline(&out, ptr); + return out; } // Stores a value of type T in memory (possibly unaligned). @@ -228,12 +228,12 @@ LIBC_INLINE ValueType load_aligned(CPtr src) { static_assert(sizeof(ValueType) >= (sizeof(T) + ... + sizeof(TS))); const ValueType value = load(assume_aligned(src)); if constexpr (sizeof...(TS) > 0) { - constexpr size_t shift = sizeof(T) * 8; + constexpr size_t SHIFT = sizeof(T) * 8; const ValueType next = load_aligned(src + sizeof(T)); if constexpr (Endian::IS_LITTLE) - return value | (next << shift); + return value | (next << SHIFT); else if constexpr (Endian::IS_BIG) - return (value << shift) | next; + return (value << SHIFT) | next; else static_assert(cpp::always_false, "Invalid endianness"); } else { @@ -261,16 +261,16 @@ LIBC_INLINE auto load64_aligned(CPtr src, size_t offset) { template LIBC_INLINE void store_aligned(ValueType value, Ptr dst) { static_assert(sizeof(ValueType) >= (sizeof(T) + ... + sizeof(TS))); - constexpr size_t shift = sizeof(T) * 8; + constexpr size_t SHIFT = sizeof(T) * 8; if constexpr (Endian::IS_LITTLE) { store(assume_aligned(dst), value & ~T(0)); if constexpr (sizeof...(TS) > 0) - store_aligned(value >> shift, dst + sizeof(T)); + store_aligned(value >> SHIFT, dst + sizeof(T)); } else if constexpr (Endian::IS_BIG) { constexpr size_t OFFSET = (0 + ... + sizeof(TS)); store(assume_aligned(dst + OFFSET), value & ~T(0)); if constexpr (sizeof...(TS) > 0) - store_aligned(value >> shift, dst); + store_aligned(value >> SHIFT, dst); } else { static_assert(cpp::always_false, "Invalid endianness"); } diff --git a/libc/src/string/memory_utils/x86_64/inline_memcpy.h b/libc/src/string/memory_utils/x86_64/inline_memcpy.h index dd09d4f3e812..ae61b1235bd0 100644 --- a/libc/src/string/memory_utils/x86_64/inline_memcpy.h +++ b/libc/src/string/memory_utils/x86_64/inline_memcpy.h @@ -30,11 +30,11 @@ namespace LIBC_NAMESPACE { namespace x86 { -LIBC_INLINE_VAR constexpr size_t kOneCacheline = 64; -LIBC_INLINE_VAR constexpr size_t kTwoCachelines = 2 * kOneCacheline; -LIBC_INLINE_VAR constexpr size_t kThreeCachelines = 3 * kOneCacheline; +LIBC_INLINE_VAR constexpr size_t K_ONE_CACHELINE = 64; +LIBC_INLINE_VAR constexpr size_t K_TWO_CACHELINES = 2 * K_ONE_CACHELINE; +LIBC_INLINE_VAR constexpr size_t K_THREE_CACHELINES = 3 * K_ONE_CACHELINE; -LIBC_INLINE_VAR constexpr bool kUseSoftwarePrefetching = +LIBC_INLINE_VAR constexpr bool K_USE_SOFTWARE_PREFETCHING = LLVM_LIBC_IS_DEFINED(LIBC_COPT_MEMCPY_X86_USE_SOFTWARE_PREFETCHING); // Whether to use rep;movsb exclusively (0), not at all (SIZE_MAX), or only @@ -42,7 +42,7 @@ LIBC_INLINE_VAR constexpr bool kUseSoftwarePrefetching = #ifndef LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE #define LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE SIZE_MAX #endif -LIBC_INLINE_VAR constexpr size_t kRepMovsbThreshold = +LIBC_INLINE_VAR constexpr size_t K_REP_MOVSB_THRESHOLD = LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE; } // namespace x86 @@ -73,10 +73,10 @@ inline_memcpy_x86_avx_ge64(Ptr __restrict dst, CPtr __restrict src, inline_memcpy_x86_sse2_ge64_sw_prefetching(Ptr __restrict dst, CPtr __restrict src, size_t count) { using namespace LIBC_NAMESPACE::x86; - prefetch_to_local_cache(src + kOneCacheline); + prefetch_to_local_cache(src + K_ONE_CACHELINE); if (count <= 128) return builtin::Memcpy<64>::head_tail(dst, src, count); - prefetch_to_local_cache(src + kTwoCachelines); + prefetch_to_local_cache(src + K_TWO_CACHELINES); // Aligning 'dst' on a 32B boundary. builtin::Memcpy<32>::block(dst, src); align_to_next_boundary<32, Arg::Dst>(dst, src, count); @@ -89,22 +89,22 @@ inline_memcpy_x86_sse2_ge64_sw_prefetching(Ptr __restrict dst, // - count >= 128. if (count < 352) { // Two cache lines at a time. - while (offset + kTwoCachelines + 32 <= count) { - prefetch_to_local_cache(src + offset + kOneCacheline); - prefetch_to_local_cache(src + offset + kTwoCachelines); - builtin::Memcpy::block_offset(dst, src, offset); - offset += kTwoCachelines; + while (offset + K_TWO_CACHELINES + 32 <= count) { + prefetch_to_local_cache(src + offset + K_ONE_CACHELINE); + prefetch_to_local_cache(src + offset + K_TWO_CACHELINES); + builtin::Memcpy::block_offset(dst, src, offset); + offset += K_TWO_CACHELINES; } } else { // Three cache lines at a time. - while (offset + kThreeCachelines + 32 <= count) { - prefetch_to_local_cache(src + offset + kOneCacheline); - prefetch_to_local_cache(src + offset + kTwoCachelines); - prefetch_to_local_cache(src + offset + kThreeCachelines); + while (offset + K_THREE_CACHELINES + 32 <= count) { + prefetch_to_local_cache(src + offset + K_ONE_CACHELINE); + prefetch_to_local_cache(src + offset + K_TWO_CACHELINES); + prefetch_to_local_cache(src + offset + K_THREE_CACHELINES); // It is likely that this copy will be turned into a 'rep;movsb' on // non-AVX machines. - builtin::Memcpy::block_offset(dst, src, offset); - offset += kThreeCachelines; + builtin::Memcpy::block_offset(dst, src, offset); + offset += K_THREE_CACHELINES; } } return builtin::Memcpy<32>::loop_and_tail_offset(dst, src, count, offset); @@ -114,11 +114,11 @@ inline_memcpy_x86_sse2_ge64_sw_prefetching(Ptr __restrict dst, inline_memcpy_x86_avx_ge64_sw_prefetching(Ptr __restrict dst, CPtr __restrict src, size_t count) { using namespace LIBC_NAMESPACE::x86; - prefetch_to_local_cache(src + kOneCacheline); + prefetch_to_local_cache(src + K_ONE_CACHELINE); if (count <= 128) return builtin::Memcpy<64>::head_tail(dst, src, count); - prefetch_to_local_cache(src + kTwoCachelines); - prefetch_to_local_cache(src + kThreeCachelines); + prefetch_to_local_cache(src + K_TWO_CACHELINES); + prefetch_to_local_cache(src + K_THREE_CACHELINES); if (count < 256) return builtin::Memcpy<128>::head_tail(dst, src, count); // Aligning 'dst' on a 32B boundary. @@ -131,13 +131,13 @@ inline_memcpy_x86_avx_ge64_sw_prefetching(Ptr __restrict dst, // - we prefetched cachelines at 'src + 64', 'src + 128', and 'src + 196' // - 'dst' is 32B aligned, // - count >= 128. - while (offset + kThreeCachelines + 64 <= count) { + while (offset + K_THREE_CACHELINES + 64 <= count) { // Three cache lines at a time. - prefetch_to_local_cache(src + offset + kOneCacheline); - prefetch_to_local_cache(src + offset + kTwoCachelines); - prefetch_to_local_cache(src + offset + kThreeCachelines); - builtin::Memcpy::block_offset(dst, src, offset); - offset += kThreeCachelines; + prefetch_to_local_cache(src + offset + K_ONE_CACHELINE); + prefetch_to_local_cache(src + offset + K_TWO_CACHELINES); + prefetch_to_local_cache(src + offset + K_THREE_CACHELINES); + builtin::Memcpy::block_offset(dst, src, offset); + offset += K_THREE_CACHELINES; } return builtin::Memcpy<64>::loop_and_tail_offset(dst, src, count, offset); } @@ -145,13 +145,13 @@ inline_memcpy_x86_avx_ge64_sw_prefetching(Ptr __restrict dst, [[maybe_unused]] LIBC_INLINE void inline_memcpy_x86(Ptr __restrict dst, CPtr __restrict src, size_t count) { #if defined(__AVX512F__) - constexpr size_t vector_size = 64; + constexpr size_t VECTOR_SIZE = 64; #elif defined(__AVX__) - constexpr size_t vector_size = 32; + constexpr size_t VECTOR_SIZE = 32; #elif defined(__SSE2__) - constexpr size_t vector_size = 16; + constexpr size_t VECTOR_SIZE = 16; #else - constexpr size_t vector_size = 8; + constexpr size_t VECTOR_SIZE = 8; #endif if (count == 0) return; @@ -174,20 +174,20 @@ inline_memcpy_x86(Ptr __restrict dst, CPtr __restrict src, size_t count) { // But it's not profitable to use larger size if it's not natively // supported: we will both use more instructions and handle fewer // sizes in earlier branches. - if (vector_size >= 16 ? count < 16 : count <= 16) + if (VECTOR_SIZE >= 16 ? count < 16 : count <= 16) return builtin::Memcpy<8>::head_tail(dst, src, count); - if (vector_size >= 32 ? count < 32 : count <= 32) + if (VECTOR_SIZE >= 32 ? count < 32 : count <= 32) return builtin::Memcpy<16>::head_tail(dst, src, count); - if (vector_size >= 64 ? count < 64 : count <= 64) + if (VECTOR_SIZE >= 64 ? count < 64 : count <= 64) return builtin::Memcpy<32>::head_tail(dst, src, count); - if constexpr (x86::kAvx) { - if constexpr (x86::kUseSoftwarePrefetching) { + if constexpr (x86::K_AVX) { + if constexpr (x86::K_USE_SOFTWARE_PREFETCHING) { return inline_memcpy_x86_avx_ge64_sw_prefetching(dst, src, count); } else { return inline_memcpy_x86_avx_ge64(dst, src, count); } } else { - if constexpr (x86::kUseSoftwarePrefetching) { + if constexpr (x86::K_USE_SOFTWARE_PREFETCHING) { return inline_memcpy_x86_sse2_ge64_sw_prefetching(dst, src, count); } else { return inline_memcpy_x86_sse2_ge64(dst, src, count); @@ -198,12 +198,12 @@ inline_memcpy_x86(Ptr __restrict dst, CPtr __restrict src, size_t count) { [[maybe_unused]] LIBC_INLINE void inline_memcpy_x86_maybe_interpose_repmovsb(Ptr __restrict dst, CPtr __restrict src, size_t count) { - if constexpr (x86::kRepMovsbThreshold == 0) { + if constexpr (x86::K_REP_MOVSB_THRESHOLD == 0) { return x86::Memcpy::repmovsb(dst, src, count); - } else if constexpr (x86::kRepMovsbThreshold == SIZE_MAX) { + } else if constexpr (x86::K_REP_MOVSB_THRESHOLD == SIZE_MAX) { return inline_memcpy_x86(dst, src, count); } else { - if (LIBC_UNLIKELY(count >= x86::kRepMovsbThreshold)) + if (LIBC_UNLIKELY(count >= x86::K_REP_MOVSB_THRESHOLD)) return x86::Memcpy::repmovsb(dst, src, count); else return inline_memcpy_x86(dst, src, count); diff --git a/libc/src/string/memory_utils/x86_64/inline_memset.h b/libc/src/string/memory_utils/x86_64/inline_memset.h index 41eadf2dcc00..584efcbea4be 100644 --- a/libc/src/string/memory_utils/x86_64/inline_memset.h +++ b/libc/src/string/memory_utils/x86_64/inline_memset.h @@ -18,11 +18,13 @@ namespace LIBC_NAMESPACE { namespace x86 { // Size of one cache line for software prefetching -LIBC_INLINE_VAR constexpr size_t kOneCachelineSize = 64; -LIBC_INLINE_VAR constexpr size_t kTwoCachelinesSize = kOneCachelineSize * 2; -LIBC_INLINE_VAR constexpr size_t kFiveCachelinesSize = kOneCachelineSize * 5; +LIBC_INLINE_VAR constexpr size_t K_ONE_CACHELINE_SIZE = 64; +LIBC_INLINE_VAR constexpr size_t K_TWO_CACHELINES_SIZE = + K_ONE_CACHELINE_SIZE * 2; +LIBC_INLINE_VAR constexpr size_t K_FIVE_CACHELINES_SIZE = + K_ONE_CACHELINE_SIZE * 5; -LIBC_INLINE_VAR constexpr bool kUseSoftwarePrefetchingMemset = +LIBC_INLINE_VAR constexpr bool K_USE_SOFTWARE_PREFETCHING_MEMSET = LLVM_LIBC_IS_DEFINED(LIBC_COPT_MEMSET_X86_USE_SOFTWARE_PREFETCHING); } // namespace x86 @@ -47,15 +49,15 @@ using uint512_t = cpp::array; [[maybe_unused]] LIBC_INLINE static void inline_memset_x86_gt64_sw_prefetching(Ptr dst, uint8_t value, size_t count) { - constexpr size_t PREFETCH_DISTANCE = x86::kFiveCachelinesSize; - constexpr size_t PREFETCH_DEGREE = x86::kTwoCachelinesSize; + constexpr size_t PREFETCH_DISTANCE = x86::K_FIVE_CACHELINES_SIZE; + constexpr size_t PREFETCH_DEGREE = x86::K_TWO_CACHELINES_SIZE; constexpr size_t SIZE = sizeof(uint256_t); // Prefetch one cache line - prefetch_for_write(dst + x86::kOneCachelineSize); + prefetch_for_write(dst + x86::K_ONE_CACHELINE_SIZE); if (count <= 128) return generic::Memset::head_tail(dst, value, count); // Prefetch the second cache line - prefetch_for_write(dst + x86::kTwoCachelinesSize); + prefetch_for_write(dst + x86::K_TWO_CACHELINES_SIZE); // Aligned loop generic::Memset::block(dst, value); align_to_next_boundary<32>(dst, count); @@ -67,7 +69,7 @@ inline_memset_x86_gt64_sw_prefetching(Ptr dst, uint8_t value, size_t count) { while (offset + PREFETCH_DEGREE + SIZE <= count) { prefetch_for_write(dst + offset + PREFETCH_DISTANCE); prefetch_for_write(dst + offset + PREFETCH_DISTANCE + - x86::kOneCachelineSize); + x86::K_ONE_CACHELINE_SIZE); for (size_t i = 0; i < PREFETCH_DEGREE; i += SIZE, offset += SIZE) generic::Memset::block(dst + offset, value); } @@ -93,7 +95,7 @@ inline_memset_x86(Ptr dst, uint8_t value, size_t count) { return generic::Memset::head_tail(dst, value, count); if (count <= 64) return generic::Memset::head_tail(dst, value, count); - if constexpr (x86::kUseSoftwarePrefetchingMemset) + if constexpr (x86::K_USE_SOFTWARE_PREFETCHING_MEMSET) return inline_memset_x86_gt64_sw_prefetching(dst, value, count); if (count <= 128) return generic::Memset::head_tail(dst, value, count); -- GitLab From 640c85748ec823e91a3cd412829f644cf4f10ffc Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Tue, 5 Mar 2024 08:16:50 -0800 Subject: [PATCH 158/929] [libc] fix readability-identifier-naming in memory_utils/utils.h (#83919) Fixes: libc/src/string/memory_utils/utils.h:345:13: warning: invalid case style for member 'offset_' [readability-identifier-naming] Having a trailing underscore for members is a google3 style, not LLVM style. Removing the underscore is insufficient, as we would then have 2 members with the same identifier which is not allowed (it is a compile time error). Remove the getter, and just access the renamed member that's now made public. --- libc/src/string/memory_utils/op_generic.h | 4 ++-- libc/src/string/memory_utils/utils.h | 9 +++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/libc/src/string/memory_utils/op_generic.h b/libc/src/string/memory_utils/op_generic.h index 28243c7a1816..90269c0fa803 100644 --- a/libc/src/string/memory_utils/op_generic.h +++ b/libc/src/string/memory_utils/op_generic.h @@ -453,7 +453,7 @@ public: if (LIBC_UNLIKELY(count >= threshold) && helper.not_aligned()) { if (auto value = block(p1, p2)) return value; - adjust(helper.offset(), p1, p2, count); + adjust(helper.offset, p1, p2, count); } return loop_and_tail(p1, p2, count); } @@ -533,7 +533,7 @@ template struct Bcmp { if (LIBC_UNLIKELY(count >= threshold) && helper.not_aligned()) { if (auto value = block(p1, p2)) return value; - adjust(helper.offset(), p1, p2, count); + adjust(helper.offset, p1, p2, count); } return loop_and_tail(p1, p2, count); } diff --git a/libc/src/string/memory_utils/utils.h b/libc/src/string/memory_utils/utils.h index 6e26e0c6a54d..79526d19c6b3 100644 --- a/libc/src/string/memory_utils/utils.h +++ b/libc/src/string/memory_utils/utils.h @@ -336,13 +336,10 @@ LIBC_INLINE void align_to_next_boundary(T1 *__restrict &p1, T2 *__restrict &p2, template struct AlignHelper { LIBC_INLINE AlignHelper(CPtr ptr) - : offset_(distance_to_next_aligned(ptr)) {} + : offset(distance_to_next_aligned(ptr)) {} - LIBC_INLINE bool not_aligned() const { return offset_ != SIZE; } - LIBC_INLINE uintptr_t offset() const { return offset_; } - -private: - uintptr_t offset_; + LIBC_INLINE bool not_aligned() const { return offset != SIZE; } + uintptr_t offset; }; LIBC_INLINE void prefetch_for_write(CPtr dst) { -- GitLab From 2aa22ca2ca4c7cd709665624ccc51be4676c6fd3 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Tue, 5 Mar 2024 08:17:27 -0800 Subject: [PATCH 159/929] [libc] suppress readability-identifier-naming for std::numeric_limits interfaces (#83921) These templates are made to match the ergonomics of std::numeric_limits. Because our style for constexpr variables is ALL_CAPS, we must silence the linter for these manually. Link: https://clang.llvm.org/extra/clang-tidy/#suppressing-undesired-diagnostics --- libc/src/__support/UInt.h | 4 ++++ libc/src/string/memory_utils/op_generic.h | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/libc/src/__support/UInt.h b/libc/src/__support/UInt.h index ae1fe7aaa182..5973e6fab1d7 100644 --- a/libc/src/__support/UInt.h +++ b/libc/src/__support/UInt.h @@ -898,6 +898,8 @@ public: return UInt<128>({0xffff'ffff'ffff'ffff, 0xffff'ffff'ffff'ffff}); } LIBC_INLINE static constexpr UInt<128> min() { return UInt<128>(0); } + // Meant to match std::numeric_limits interface. + // NOLINTNEXTLINE(readability-identifier-naming) LIBC_INLINE_VAR static constexpr int digits = 128; }; @@ -909,6 +911,8 @@ public: LIBC_INLINE static constexpr Int<128> min() { return Int<128>({0, 0x8000'0000'0000'0000}); } + // Meant to match std::numeric_limits interface. + // NOLINTNEXTLINE(readability-identifier-naming) LIBC_INLINE_VAR static constexpr int digits = 128; }; diff --git a/libc/src/string/memory_utils/op_generic.h b/libc/src/string/memory_utils/op_generic.h index 90269c0fa803..41fc1fa0f1ff 100644 --- a/libc/src/string/memory_utils/op_generic.h +++ b/libc/src/string/memory_utils/op_generic.h @@ -63,28 +63,41 @@ template <> struct is_scalar : cpp::true_type {}; #ifdef LLVM_LIBC_HAS_UINT64 template <> struct is_scalar : cpp::true_type {}; #endif // LLVM_LIBC_HAS_UINT64 +// Meant to match std::numeric_limits interface. +// NOLINTNEXTLINE(readability-identifier-naming) template constexpr bool is_scalar_v = is_scalar::value; template struct is_vector : cpp::false_type {}; template <> struct is_vector : cpp::true_type {}; template <> struct is_vector : cpp::true_type {}; template <> struct is_vector : cpp::true_type {}; +// Meant to match std::numeric_limits interface. +// NOLINTNEXTLINE(readability-identifier-naming) template constexpr bool is_vector_v = is_vector::value; template struct is_array : cpp::false_type {}; template struct is_array> { + // Meant to match std::numeric_limits interface. + // NOLINTNEXTLINE(readability-identifier-naming) static constexpr bool value = is_scalar_v || is_vector_v; }; +// Meant to match std::numeric_limits interface. +// NOLINTNEXTLINE(readability-identifier-naming) template constexpr bool is_array_v = is_array::value; +// Meant to match std::numeric_limits interface. +// NOLINTBEGIN(readability-identifier-naming) template constexpr bool is_element_type_v = is_scalar_v || is_vector_v || is_array_v; +// NOLINTEND(readability-identifier-naming) // Helper struct to retrieve the number of elements of an array. template struct array_size {}; template struct array_size> : cpp::integral_constant {}; +// Meant to match std::numeric_limits interface. +// NOLINTNEXTLINE(readability-identifier-naming) template constexpr size_t array_size_v = array_size::value; // Generic operations for the above type categories. -- GitLab From 169824ba401d87707d72634cb7c2db48667b8fbe Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Tue, 5 Mar 2024 08:09:54 -0800 Subject: [PATCH 160/929] [SLP][NFC]SPlit test/Transforms/SLPVectorizer/AArch64/getelementptr.ll, NFC. --- .../SLPVectorizer/AArch64/getelementptr.ll | 163 +---------------- .../SLPVectorizer/AArch64/getelementptr2.ll | 173 ++++++++++++++++++ 2 files changed, 175 insertions(+), 161 deletions(-) create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll index d756d57fa08a..821abca63c29 100644 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll @@ -1,13 +1,9 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -passes=slp-vectorizer,dce,instcombine -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s +; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer,dce,instcombine -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s ; RUN: cat %t | FileCheck -check-prefix=YAML %s -; RUN: opt -S -passes='slp-vectorizer,dce,instcombine' -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s +; RUN: opt -S -mtriple=aarch64--linux-gnu -passes='slp-vectorizer,dce,instcombine' -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s ; RUN: cat %t | FileCheck -check-prefix=YAML %s - -target datalayout = "e-m:e-i32:64-i128:128-n32:64-S128" -target triple = "aarch64--linux-gnu" - ; These tests check that we remove from consideration pairs of seed ; getelementptrs when they are known to have a constant difference. Such pairs ; are likely not good candidates for vectorization since one can be computed @@ -223,158 +219,3 @@ for.body: %exitcond = icmp eq i32 %indvars.iv.next , %n br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body } - -@global = internal global { ptr } zeroinitializer, align 8 - -; Make sure we vectorize to maximize the load with when loading i16 and -; extending it for compute operations. -define void @test_i16_extend(ptr %p.1, ptr %p.2, i32 %idx.i32) { -; CHECK-LABEL: @test_i16_extend( -; CHECK-NEXT: [[P_0:%.*]] = load ptr, ptr @global, align 8 -; CHECK-NEXT: [[IDX_0:%.*]] = zext i32 [[IDX_I32:%.*]] to i64 -; CHECK-NEXT: [[T53:%.*]] = getelementptr inbounds i16, ptr [[P_1:%.*]], i64 [[IDX_0]] -; CHECK-NEXT: [[T56:%.*]] = getelementptr inbounds i16, ptr [[P_2:%.*]], i64 [[IDX_0]] -; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i16>, ptr [[T53]], align 2 -; CHECK-NEXT: [[TMP2:%.*]] = zext <8 x i16> [[TMP1]] to <8 x i32> -; CHECK-NEXT: [[TMP3:%.*]] = load <8 x i16>, ptr [[T56]], align 2 -; CHECK-NEXT: [[TMP4:%.*]] = zext <8 x i16> [[TMP3]] to <8 x i32> -; CHECK-NEXT: [[TMP5:%.*]] = sub nsw <8 x i32> [[TMP2]], [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i32> [[TMP5]], i64 0 -; CHECK-NEXT: [[TMP7:%.*]] = sext i32 [[TMP6]] to i64 -; CHECK-NEXT: [[T60:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP7]] -; CHECK-NEXT: [[L_1:%.*]] = load i32, ptr [[T60]], align 4 -; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i32> [[TMP5]], i64 1 -; CHECK-NEXT: [[TMP9:%.*]] = sext i32 [[TMP8]] to i64 -; CHECK-NEXT: [[T71:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP9]] -; CHECK-NEXT: [[L_2:%.*]] = load i32, ptr [[T71]], align 4 -; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[TMP5]], i64 2 -; CHECK-NEXT: [[TMP11:%.*]] = sext i32 [[TMP10]] to i64 -; CHECK-NEXT: [[T82:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP11]] -; CHECK-NEXT: [[L_3:%.*]] = load i32, ptr [[T82]], align 4 -; CHECK-NEXT: [[TMP12:%.*]] = extractelement <8 x i32> [[TMP5]], i64 3 -; CHECK-NEXT: [[TMP13:%.*]] = sext i32 [[TMP12]] to i64 -; CHECK-NEXT: [[T93:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP13]] -; CHECK-NEXT: [[L_4:%.*]] = load i32, ptr [[T93]], align 4 -; CHECK-NEXT: [[TMP14:%.*]] = extractelement <8 x i32> [[TMP5]], i64 4 -; CHECK-NEXT: [[TMP15:%.*]] = sext i32 [[TMP14]] to i64 -; CHECK-NEXT: [[T104:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP15]] -; CHECK-NEXT: [[L_5:%.*]] = load i32, ptr [[T104]], align 4 -; CHECK-NEXT: [[TMP16:%.*]] = extractelement <8 x i32> [[TMP5]], i64 5 -; CHECK-NEXT: [[TMP17:%.*]] = sext i32 [[TMP16]] to i64 -; CHECK-NEXT: [[T115:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP17]] -; CHECK-NEXT: [[L_6:%.*]] = load i32, ptr [[T115]], align 4 -; CHECK-NEXT: [[TMP18:%.*]] = extractelement <8 x i32> [[TMP5]], i64 6 -; CHECK-NEXT: [[TMP19:%.*]] = sext i32 [[TMP18]] to i64 -; CHECK-NEXT: [[T126:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP19]] -; CHECK-NEXT: [[L_7:%.*]] = load i32, ptr [[T126]], align 4 -; CHECK-NEXT: [[TMP20:%.*]] = extractelement <8 x i32> [[TMP5]], i64 7 -; CHECK-NEXT: [[TMP21:%.*]] = sext i32 [[TMP20]] to i64 -; CHECK-NEXT: [[T137:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP21]] -; CHECK-NEXT: [[L_8:%.*]] = load i32, ptr [[T137]], align 4 -; CHECK-NEXT: call void @use(i32 [[L_1]], i32 [[L_2]], i32 [[L_3]], i32 [[L_4]], i32 [[L_5]], i32 [[L_6]], i32 [[L_7]], i32 [[L_8]]) -; CHECK-NEXT: ret void -; - %p.0 = load ptr, ptr @global, align 8 - - %idx.0 = zext i32 %idx.i32 to i64 - %idx.1 = add nsw i64 %idx.0, 1 - %idx.2 = add nsw i64 %idx.0, 2 - %idx.3 = add nsw i64 %idx.0, 3 - %idx.4 = add nsw i64 %idx.0, 4 - %idx.5 = add nsw i64 %idx.0, 5 - %idx.6 = add nsw i64 %idx.0, 6 - %idx.7 = add nsw i64 %idx.0, 7 - - %t53 = getelementptr inbounds i16, ptr %p.1, i64 %idx.0 - %op1.l = load i16, ptr %t53, align 2 - %op1.ext = zext i16 %op1.l to i64 - %t56 = getelementptr inbounds i16, ptr %p.2, i64 %idx.0 - %op2.l = load i16, ptr %t56, align 2 - %op2.ext = zext i16 %op2.l to i64 - %sub.1 = sub nsw i64 %op1.ext, %op2.ext - - %t60 = getelementptr inbounds i32, ptr %p.0, i64 %sub.1 - %l.1 = load i32, ptr %t60, align 4 - - %t64 = getelementptr inbounds i16, ptr %p.1, i64 %idx.1 - %t65 = load i16, ptr %t64, align 2 - %t66 = zext i16 %t65 to i64 - %t67 = getelementptr inbounds i16, ptr %p.2, i64 %idx.1 - %t68 = load i16, ptr %t67, align 2 - %t69 = zext i16 %t68 to i64 - %sub.2 = sub nsw i64 %t66, %t69 - - %t71 = getelementptr inbounds i32, ptr %p.0, i64 %sub.2 - %l.2 = load i32, ptr %t71, align 4 - - %t75 = getelementptr inbounds i16, ptr %p.1, i64 %idx.2 - %t76 = load i16, ptr %t75, align 2 - %t77 = zext i16 %t76 to i64 - %t78 = getelementptr inbounds i16, ptr %p.2, i64 %idx.2 - %t79 = load i16, ptr %t78, align 2 - %t80 = zext i16 %t79 to i64 - %sub.3 = sub nsw i64 %t77, %t80 - - %t82 = getelementptr inbounds i32, ptr %p.0, i64 %sub.3 - %l.3 = load i32, ptr %t82, align 4 - - %t86 = getelementptr inbounds i16, ptr %p.1, i64 %idx.3 - %t87 = load i16, ptr %t86, align 2 - %t88 = zext i16 %t87 to i64 - %t89 = getelementptr inbounds i16, ptr %p.2, i64 %idx.3 - %t90 = load i16, ptr %t89, align 2 - %t91 = zext i16 %t90 to i64 - %sub.4 = sub nsw i64 %t88, %t91 - - %t93 = getelementptr inbounds i32, ptr %p.0, i64 %sub.4 - %l.4 = load i32, ptr %t93, align 4 - - %t97 = getelementptr inbounds i16, ptr %p.1, i64 %idx.4 - %t98 = load i16, ptr %t97, align 2 - %t99 = zext i16 %t98 to i64 - %t100 = getelementptr inbounds i16, ptr %p.2, i64 %idx.4 - %t101 = load i16, ptr %t100, align 2 - %t102 = zext i16 %t101 to i64 - %sub.5 = sub nsw i64 %t99, %t102 - - %t104 = getelementptr inbounds i32, ptr %p.0, i64 %sub.5 - %l.5 = load i32, ptr %t104, align 4 - - %t108 = getelementptr inbounds i16, ptr %p.1, i64 %idx.5 - %t109 = load i16, ptr %t108, align 2 - %t110 = zext i16 %t109 to i64 - %t111 = getelementptr inbounds i16, ptr %p.2, i64 %idx.5 - %t112 = load i16, ptr %t111, align 2 - %t113 = zext i16 %t112 to i64 - %sub.6 = sub nsw i64 %t110, %t113 - - %t115 = getelementptr inbounds i32, ptr %p.0, i64 %sub.6 - %l.6 = load i32, ptr %t115, align 4 - - %t119 = getelementptr inbounds i16, ptr %p.1, i64 %idx.6 - %t120 = load i16, ptr %t119, align 2 - %t121 = zext i16 %t120 to i64 - %t122 = getelementptr inbounds i16, ptr %p.2, i64 %idx.6 - %t123 = load i16, ptr %t122, align 2 - %t124 = zext i16 %t123 to i64 - %sub.7 = sub nsw i64 %t121, %t124 - - %t126 = getelementptr inbounds i32, ptr %p.0, i64 %sub.7 - %l.7 = load i32, ptr %t126, align 4 - - %t130 = getelementptr inbounds i16, ptr %p.1, i64 %idx.7 - %t131 = load i16, ptr %t130, align 2 - %t132 = zext i16 %t131 to i64 - %t133 = getelementptr inbounds i16, ptr %p.2, i64 %idx.7 - %t134 = load i16, ptr %t133, align 2 - %t135 = zext i16 %t134 to i64 - %sub.8 = sub nsw i64 %t132, %t135 - - %t137 = getelementptr inbounds i32, ptr %p.0, i64 %sub.8 - %l.8 = load i32, ptr %t137, align 4 - - call void @use(i32 %l.1, i32 %l.2, i32 %l.3, i32 %l.4, i32 %l.5, i32 %l.6, i32 %l.7, i32 %l.8) - ret void -} - -declare void @use(i32, i32, i32, i32, i32, i32, i32, i32) diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll new file mode 100644 index 000000000000..47485e514ec2 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll @@ -0,0 +1,173 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +;test_i16_extend NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer,dce,instcombine -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s +; RUN: cat %t | FileCheck -check-prefix=YAML %s +; RUN: opt -S -mtriple=aarch64--linux-gnu -passes='slp-vectorizer,dce,instcombine' -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s +; RUN: cat %t | FileCheck -check-prefix=YAML %s + + +@global = internal global { ptr } zeroinitializer, align 8 + +; YAML-LABEL: --- !Passed +; YAML-NEXT: Pass: slp-vectorizer +; YAML-NEXT: Name: VectorizedList +; YAML-NEXT: Function: test_i16_extend +; YAML-NEXT: Args: +; YAML-NEXT: - String: 'SLP vectorized with cost ' +; YAML-NEXT: - Cost: '-20' +; YAML-NEXT: - String: ' and with tree size ' +; YAML-NEXT: - TreeSize: '5' +; YAML-NEXT: ... + +; Make sure we vectorize to maximize the load with when loading i16 and +; extending it for compute operations. +define void @test_i16_extend(ptr %p.1, ptr %p.2, i32 %idx.i32) { +; CHECK-LABEL: @test_i16_extend( +; CHECK-NEXT: [[P_0:%.*]] = load ptr, ptr @global, align 8 +; CHECK-NEXT: [[IDX_0:%.*]] = zext i32 [[IDX_I32:%.*]] to i64 +; CHECK-NEXT: [[T53:%.*]] = getelementptr inbounds i16, ptr [[P_1:%.*]], i64 [[IDX_0]] +; CHECK-NEXT: [[T56:%.*]] = getelementptr inbounds i16, ptr [[P_2:%.*]], i64 [[IDX_0]] +; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i16>, ptr [[T53]], align 2 +; CHECK-NEXT: [[TMP2:%.*]] = zext <8 x i16> [[TMP1]] to <8 x i32> +; CHECK-NEXT: [[TMP3:%.*]] = load <8 x i16>, ptr [[T56]], align 2 +; CHECK-NEXT: [[TMP4:%.*]] = zext <8 x i16> [[TMP3]] to <8 x i32> +; CHECK-NEXT: [[TMP5:%.*]] = sub nsw <8 x i32> [[TMP2]], [[TMP4]] +; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i32> [[TMP5]], i64 0 +; CHECK-NEXT: [[TMP7:%.*]] = sext i32 [[TMP6]] to i64 +; CHECK-NEXT: [[T60:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP7]] +; CHECK-NEXT: [[L_1:%.*]] = load i32, ptr [[T60]], align 4 +; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i32> [[TMP5]], i64 1 +; CHECK-NEXT: [[TMP9:%.*]] = sext i32 [[TMP8]] to i64 +; CHECK-NEXT: [[T71:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP9]] +; CHECK-NEXT: [[L_2:%.*]] = load i32, ptr [[T71]], align 4 +; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[TMP5]], i64 2 +; CHECK-NEXT: [[TMP11:%.*]] = sext i32 [[TMP10]] to i64 +; CHECK-NEXT: [[T82:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP11]] +; CHECK-NEXT: [[L_3:%.*]] = load i32, ptr [[T82]], align 4 +; CHECK-NEXT: [[TMP12:%.*]] = extractelement <8 x i32> [[TMP5]], i64 3 +; CHECK-NEXT: [[TMP13:%.*]] = sext i32 [[TMP12]] to i64 +; CHECK-NEXT: [[T93:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP13]] +; CHECK-NEXT: [[L_4:%.*]] = load i32, ptr [[T93]], align 4 +; CHECK-NEXT: [[TMP14:%.*]] = extractelement <8 x i32> [[TMP5]], i64 4 +; CHECK-NEXT: [[TMP15:%.*]] = sext i32 [[TMP14]] to i64 +; CHECK-NEXT: [[T104:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP15]] +; CHECK-NEXT: [[L_5:%.*]] = load i32, ptr [[T104]], align 4 +; CHECK-NEXT: [[TMP16:%.*]] = extractelement <8 x i32> [[TMP5]], i64 5 +; CHECK-NEXT: [[TMP17:%.*]] = sext i32 [[TMP16]] to i64 +; CHECK-NEXT: [[T115:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP17]] +; CHECK-NEXT: [[L_6:%.*]] = load i32, ptr [[T115]], align 4 +; CHECK-NEXT: [[TMP18:%.*]] = extractelement <8 x i32> [[TMP5]], i64 6 +; CHECK-NEXT: [[TMP19:%.*]] = sext i32 [[TMP18]] to i64 +; CHECK-NEXT: [[T126:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP19]] +; CHECK-NEXT: [[L_7:%.*]] = load i32, ptr [[T126]], align 4 +; CHECK-NEXT: [[TMP20:%.*]] = extractelement <8 x i32> [[TMP5]], i64 7 +; CHECK-NEXT: [[TMP21:%.*]] = sext i32 [[TMP20]] to i64 +; CHECK-NEXT: [[T137:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP21]] +; CHECK-NEXT: [[L_8:%.*]] = load i32, ptr [[T137]], align 4 +; CHECK-NEXT: call void @use(i32 [[L_1]], i32 [[L_2]], i32 [[L_3]], i32 [[L_4]], i32 [[L_5]], i32 [[L_6]], i32 [[L_7]], i32 [[L_8]]) +; CHECK-NEXT: ret void +; + %p.0 = load ptr, ptr @global, align 8 + + %idx.0 = zext i32 %idx.i32 to i64 + %idx.1 = add nsw i64 %idx.0, 1 + %idx.2 = add nsw i64 %idx.0, 2 + %idx.3 = add nsw i64 %idx.0, 3 + %idx.4 = add nsw i64 %idx.0, 4 + %idx.5 = add nsw i64 %idx.0, 5 + %idx.6 = add nsw i64 %idx.0, 6 + %idx.7 = add nsw i64 %idx.0, 7 + + %t53 = getelementptr inbounds i16, ptr %p.1, i64 %idx.0 + %op1.l = load i16, ptr %t53, align 2 + %op1.ext = zext i16 %op1.l to i64 + %t56 = getelementptr inbounds i16, ptr %p.2, i64 %idx.0 + %op2.l = load i16, ptr %t56, align 2 + %op2.ext = zext i16 %op2.l to i64 + %sub.1 = sub nsw i64 %op1.ext, %op2.ext + + %t60 = getelementptr inbounds i32, ptr %p.0, i64 %sub.1 + %l.1 = load i32, ptr %t60, align 4 + + %t64 = getelementptr inbounds i16, ptr %p.1, i64 %idx.1 + %t65 = load i16, ptr %t64, align 2 + %t66 = zext i16 %t65 to i64 + %t67 = getelementptr inbounds i16, ptr %p.2, i64 %idx.1 + %t68 = load i16, ptr %t67, align 2 + %t69 = zext i16 %t68 to i64 + %sub.2 = sub nsw i64 %t66, %t69 + + %t71 = getelementptr inbounds i32, ptr %p.0, i64 %sub.2 + %l.2 = load i32, ptr %t71, align 4 + + %t75 = getelementptr inbounds i16, ptr %p.1, i64 %idx.2 + %t76 = load i16, ptr %t75, align 2 + %t77 = zext i16 %t76 to i64 + %t78 = getelementptr inbounds i16, ptr %p.2, i64 %idx.2 + %t79 = load i16, ptr %t78, align 2 + %t80 = zext i16 %t79 to i64 + %sub.3 = sub nsw i64 %t77, %t80 + + %t82 = getelementptr inbounds i32, ptr %p.0, i64 %sub.3 + %l.3 = load i32, ptr %t82, align 4 + + %t86 = getelementptr inbounds i16, ptr %p.1, i64 %idx.3 + %t87 = load i16, ptr %t86, align 2 + %t88 = zext i16 %t87 to i64 + %t89 = getelementptr inbounds i16, ptr %p.2, i64 %idx.3 + %t90 = load i16, ptr %t89, align 2 + %t91 = zext i16 %t90 to i64 + %sub.4 = sub nsw i64 %t88, %t91 + + %t93 = getelementptr inbounds i32, ptr %p.0, i64 %sub.4 + %l.4 = load i32, ptr %t93, align 4 + + %t97 = getelementptr inbounds i16, ptr %p.1, i64 %idx.4 + %t98 = load i16, ptr %t97, align 2 + %t99 = zext i16 %t98 to i64 + %t100 = getelementptr inbounds i16, ptr %p.2, i64 %idx.4 + %t101 = load i16, ptr %t100, align 2 + %t102 = zext i16 %t101 to i64 + %sub.5 = sub nsw i64 %t99, %t102 + + %t104 = getelementptr inbounds i32, ptr %p.0, i64 %sub.5 + %l.5 = load i32, ptr %t104, align 4 + + %t108 = getelementptr inbounds i16, ptr %p.1, i64 %idx.5 + %t109 = load i16, ptr %t108, align 2 + %t110 = zext i16 %t109 to i64 + %t111 = getelementptr inbounds i16, ptr %p.2, i64 %idx.5 + %t112 = load i16, ptr %t111, align 2 + %t113 = zext i16 %t112 to i64 + %sub.6 = sub nsw i64 %t110, %t113 + + %t115 = getelementptr inbounds i32, ptr %p.0, i64 %sub.6 + %l.6 = load i32, ptr %t115, align 4 + + %t119 = getelementptr inbounds i16, ptr %p.1, i64 %idx.6 + %t120 = load i16, ptr %t119, align 2 + %t121 = zext i16 %t120 to i64 + %t122 = getelementptr inbounds i16, ptr %p.2, i64 %idx.6 + %t123 = load i16, ptr %t122, align 2 + %t124 = zext i16 %t123 to i64 + %sub.7 = sub nsw i64 %t121, %t124 + + %t126 = getelementptr inbounds i32, ptr %p.0, i64 %sub.7 + %l.7 = load i32, ptr %t126, align 4 + + %t130 = getelementptr inbounds i16, ptr %p.1, i64 %idx.7 + %t131 = load i16, ptr %t130, align 2 + %t132 = zext i16 %t131 to i64 + %t133 = getelementptr inbounds i16, ptr %p.2, i64 %idx.7 + %t134 = load i16, ptr %t133, align 2 + %t135 = zext i16 %t134 to i64 + %sub.8 = sub nsw i64 %t132, %t135 + + %t137 = getelementptr inbounds i32, ptr %p.0, i64 %sub.8 + %l.8 = load i32, ptr %t137, align 4 + + call void @use(i32 %l.1, i32 %l.2, i32 %l.3, i32 %l.4, i32 %l.5, i32 %l.6, i32 %l.7, i32 %l.8) + ret void +} + +declare void @use(i32, i32, i32, i32, i32, i32, i32, i32) -- GitLab From 3b84b6f1763159dbcdc487b201aa46a5c1c7b074 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Tue, 5 Mar 2024 16:21:03 +0000 Subject: [PATCH 161/929] [CostModel][X86] Add test coverage for 'concat subvector' style shuffles Shows 2 major issues: - SSE should be free as it splits everything to 128-bit - Negative costs for 128 -> 512 concat shuffles --- .../X86/shuffle-concat_subvector-codesize.ll | 191 ++++++++++++++++++ .../X86/shuffle-concat_subvector-latency.ll | 191 ++++++++++++++++++ .../shuffle-concat_subvector-sizelatency.ll | 191 ++++++++++++++++++ .../CostModel/X86/shuffle-concat_subvector.ll | 191 ++++++++++++++++++ 4 files changed, 764 insertions(+) create mode 100644 llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-codesize.ll create mode 100644 llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-latency.ll create mode 100644 llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-sizelatency.ll create mode 100644 llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector.ll diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-codesize.ll b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-codesize.ll new file mode 100644 index 000000000000..0c1c085f5afc --- /dev/null +++ b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-codesize.ll @@ -0,0 +1,191 @@ +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mattr=+sse2 | FileCheck %s -check-prefixes=SSE +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mattr=+ssse3 | FileCheck %s -check-prefixes=SSE +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mattr=+sse4.2 | FileCheck %s -check-prefixes=SSE +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mattr=+avx | FileCheck %s -check-prefixes=AVX +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mattr=+avx2 | FileCheck %s -check-prefixes=AVX +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512 +; +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=slm | FileCheck %s --check-prefixes=SSE +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=goldmont | FileCheck %s --check-prefixes=SSE +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=btver2 | FileCheck %s --check-prefixes=AVX + +; +; Verify the cost model for concat_subvector style shuffles. +; + +define void @test_vXf64(<2 x double> %a128, <4 x double> %a256, <8 x double> %a512, <2 x double> %b128, <4 x double> %b256, <8 x double> %b512) { +; SSE-LABEL: 'test_vXf64' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX-LABEL: 'test_vXf64' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512-LABEL: 'test_vXf64' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> + %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> + %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> + %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> + ret void +} + +define void @test_vXi64(<2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, <2 x i64> %b128, <4 x i64> %b256, <8 x i64> %b512) { +; SSE-LABEL: 'test_vXi64' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX-LABEL: 'test_vXi64' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512-LABEL: 'test_vXi64' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> + %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> + %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> + %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> + ret void +} + +define void @test_vXf32(<4 x float> %a128, <8 x float> %a256, <16 x float> %a512, <4 x float> %b128, <8 x float> %b256, <16 x float> %b512) { +; SSE-LABEL: 'test_vXf32' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX-LABEL: 'test_vXf32' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512-LABEL: 'test_vXf32' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> + %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> + %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> + %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> + ret void +} + +define void @test_vXi32(<4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, <4 x i32> %b128, <8 x i32> %b256, <16 x i32> %b512) { +; SSE-LABEL: 'test_vXi32' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX-LABEL: 'test_vXi32' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512-LABEL: 'test_vXi32' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> + %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> + %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> + %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> + ret void +} + +define void @test_vXi16(<8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, <8 x i16> %b128, <16 x i16> %b256, <32 x i16> %b512) { +; SSE-LABEL: 'test_vXi16' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX-LABEL: 'test_vXi16' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512-LABEL: 'test_vXi16' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> + %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> + %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> + %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> + ret void +} + +define void @test_vXi8(<16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512, <16 x i8> %b128, <32 x i8> %b256, <64 x i8> %b512) { +; SSE-LABEL: 'test_vXi8' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX-LABEL: 'test_vXi8' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512-LABEL: 'test_vXi8' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> + %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> + %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> + %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> + ret void +} diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-latency.ll b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-latency.ll new file mode 100644 index 000000000000..7244b7b4f00f --- /dev/null +++ b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-latency.ll @@ -0,0 +1,191 @@ +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mattr=+sse2 | FileCheck %s -check-prefixes=SSE +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mattr=+ssse3 | FileCheck %s -check-prefixes=SSE +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mattr=+sse4.2 | FileCheck %s -check-prefixes=SSE +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mattr=+avx | FileCheck %s -check-prefixes=AVX +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mattr=+avx2 | FileCheck %s -check-prefixes=AVX +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512 +; +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=slm | FileCheck %s --check-prefixes=SSE +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=goldmont | FileCheck %s --check-prefixes=SSE +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=btver2 | FileCheck %s --check-prefixes=AVX + +; +; Verify the cost model for concat_subvector style shuffles. +; + +define void @test_vXf64(<2 x double> %a128, <4 x double> %a256, <8 x double> %a512, <2 x double> %b128, <4 x double> %b256, <8 x double> %b512) { +; SSE-LABEL: 'test_vXf64' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX-LABEL: 'test_vXf64' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512-LABEL: 'test_vXf64' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> + %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> + %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> + %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> + ret void +} + +define void @test_vXi64(<2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, <2 x i64> %b128, <4 x i64> %b256, <8 x i64> %b512) { +; SSE-LABEL: 'test_vXi64' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX-LABEL: 'test_vXi64' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512-LABEL: 'test_vXi64' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> + %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> + %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> + %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> + ret void +} + +define void @test_vXf32(<4 x float> %a128, <8 x float> %a256, <16 x float> %a512, <4 x float> %b128, <8 x float> %b256, <16 x float> %b512) { +; SSE-LABEL: 'test_vXf32' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX-LABEL: 'test_vXf32' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512-LABEL: 'test_vXf32' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> + %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> + %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> + %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> + ret void +} + +define void @test_vXi32(<4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, <4 x i32> %b128, <8 x i32> %b256, <16 x i32> %b512) { +; SSE-LABEL: 'test_vXi32' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX-LABEL: 'test_vXi32' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512-LABEL: 'test_vXi32' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> + %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> + %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> + %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> + ret void +} + +define void @test_vXi16(<8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, <8 x i16> %b128, <16 x i16> %b256, <32 x i16> %b512) { +; SSE-LABEL: 'test_vXi16' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX-LABEL: 'test_vXi16' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512-LABEL: 'test_vXi16' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> + %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> + %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> + %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> + ret void +} + +define void @test_vXi8(<16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512, <16 x i8> %b128, <32 x i8> %b256, <64 x i8> %b512) { +; SSE-LABEL: 'test_vXi8' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX-LABEL: 'test_vXi8' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512-LABEL: 'test_vXi8' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> + %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> + %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> + %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> + ret void +} diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-sizelatency.ll new file mode 100644 index 000000000000..b1fa00b5a713 --- /dev/null +++ b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-sizelatency.ll @@ -0,0 +1,191 @@ +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mattr=+sse2 | FileCheck %s -check-prefixes=SSE +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mattr=+ssse3 | FileCheck %s -check-prefixes=SSE +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mattr=+sse4.2 | FileCheck %s -check-prefixes=SSE +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mattr=+avx | FileCheck %s -check-prefixes=AVX +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mattr=+avx2 | FileCheck %s -check-prefixes=AVX +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512 +; +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=slm | FileCheck %s --check-prefixes=SSE +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=goldmont | FileCheck %s --check-prefixes=SSE +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=btver2 | FileCheck %s --check-prefixes=AVX + +; +; Verify the cost model for concat_subvector style shuffles. +; + +define void @test_vXf64(<2 x double> %a128, <4 x double> %a256, <8 x double> %a512, <2 x double> %b128, <4 x double> %b256, <8 x double> %b512) { +; SSE-LABEL: 'test_vXf64' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX-LABEL: 'test_vXf64' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512-LABEL: 'test_vXf64' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> + %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> + %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> + %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> + ret void +} + +define void @test_vXi64(<2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, <2 x i64> %b128, <4 x i64> %b256, <8 x i64> %b512) { +; SSE-LABEL: 'test_vXi64' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX-LABEL: 'test_vXi64' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512-LABEL: 'test_vXi64' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> + %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> + %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> + %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> + ret void +} + +define void @test_vXf32(<4 x float> %a128, <8 x float> %a256, <16 x float> %a512, <4 x float> %b128, <8 x float> %b256, <16 x float> %b512) { +; SSE-LABEL: 'test_vXf32' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX-LABEL: 'test_vXf32' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512-LABEL: 'test_vXf32' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> + %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> + %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> + %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> + ret void +} + +define void @test_vXi32(<4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, <4 x i32> %b128, <8 x i32> %b256, <16 x i32> %b512) { +; SSE-LABEL: 'test_vXi32' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX-LABEL: 'test_vXi32' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512-LABEL: 'test_vXi32' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> + %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> + %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> + %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> + ret void +} + +define void @test_vXi16(<8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, <8 x i16> %b128, <16 x i16> %b256, <32 x i16> %b512) { +; SSE-LABEL: 'test_vXi16' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX-LABEL: 'test_vXi16' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512-LABEL: 'test_vXi16' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> + %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> + %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> + %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> + ret void +} + +define void @test_vXi8(<16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512, <16 x i8> %b128, <32 x i8> %b256, <64 x i8> %b512) { +; SSE-LABEL: 'test_vXi8' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX-LABEL: 'test_vXi8' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512-LABEL: 'test_vXi8' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> + %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> + %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> + %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> + ret void +} diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector.ll b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector.ll new file mode 100644 index 000000000000..60cb8cffd1a5 --- /dev/null +++ b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector.ll @@ -0,0 +1,191 @@ +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mattr=+sse2 | FileCheck %s -check-prefixes=SSE +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mattr=+ssse3 | FileCheck %s -check-prefixes=SSE +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mattr=+sse4.2 | FileCheck %s -check-prefixes=SSE +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mattr=+avx | FileCheck %s -check-prefixes=AVX +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mattr=+avx2 | FileCheck %s -check-prefixes=AVX +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512 +; +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=slm | FileCheck %s --check-prefixes=SSE +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=goldmont | FileCheck %s --check-prefixes=SSE +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=btver2 | FileCheck %s --check-prefixes=AVX + +; +; Verify the cost model for concat_subvector style shuffles. +; + +define void @test_vXf64(<2 x double> %a128, <4 x double> %a256, <8 x double> %a512, <2 x double> %b128, <4 x double> %b256, <8 x double> %b512) { +; SSE-LABEL: 'test_vXf64' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX-LABEL: 'test_vXf64' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512-LABEL: 'test_vXf64' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; + %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> + %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> + %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> + %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> + ret void +} + +define void @test_vXi64(<2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, <2 x i64> %b128, <4 x i64> %b256, <8 x i64> %b512) { +; SSE-LABEL: 'test_vXi64' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX-LABEL: 'test_vXi64' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512-LABEL: 'test_vXi64' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; + %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> + %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> + %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> + %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> + ret void +} + +define void @test_vXf32(<4 x float> %a128, <8 x float> %a256, <16 x float> %a512, <4 x float> %b128, <8 x float> %b256, <16 x float> %b512) { +; SSE-LABEL: 'test_vXf32' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX-LABEL: 'test_vXf32' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512-LABEL: 'test_vXf32' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; + %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> + %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> + %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> + %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> + ret void +} + +define void @test_vXi32(<4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, <4 x i32> %b128, <8 x i32> %b256, <16 x i32> %b512) { +; SSE-LABEL: 'test_vXi32' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX-LABEL: 'test_vXi32' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512-LABEL: 'test_vXi32' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; + %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> + %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> + %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> + %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> + ret void +} + +define void @test_vXi16(<8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, <8 x i16> %b128, <16 x i16> %b256, <32 x i16> %b512) { +; SSE-LABEL: 'test_vXi16' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX-LABEL: 'test_vXi16' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512-LABEL: 'test_vXi16' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; + %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> + %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> + %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> + %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> + ret void +} + +define void @test_vXi8(<16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512, <16 x i8> %b128, <32 x i8> %b256, <64 x i8> %b512) { +; SSE-LABEL: 'test_vXi8' +; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX-LABEL: 'test_vXi8' +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512-LABEL: 'test_vXi8' +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; + %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> + %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> + %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> + %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> + ret void +} -- GitLab From afb05cd6469215232bd83e7cfbe59d2e1852567f Mon Sep 17 00:00:00 2001 From: agozillon Date: Tue, 5 Mar 2024 17:27:16 +0100 Subject: [PATCH 162/929] [Flang][MLIR][OpenMP] Create a deferred declare target marking process for Bridge.cpp (#78502) This patch seeks to create a process that happens on module finalization for OpenMP, in which a list of operations that had declare target directives applied to them and were not generated at the time of processing the original declare target directive are re-checked to apply the appropriate declare target semantics. This works by maintaining a vector of declare target related data inside of the FIR converter, in this case the symbol and the two relevant unsigned integers representing the enumerators. This vector is added to via a new function called from Bridge.cpp, insertDeferredDeclareTargets, which happens prior to the processing of the directive (similarly to getDeclareTargetFunctionDevice currently for requires), it effectively checks if the Operation the declare target directive is applied to currently exists, if it doesn't it appends to the vector. This is a seperate function to the processing of the declare target via the overloaded genOMP as we unfortunately do not have access to the list without passing it through every call, as the AbstractConverter we pass will not allow access to it (I've seen no other cases of casting it to a FirConverter, so I opted to not do that). The list is then processed at the end of the module in the finalizeOpenMPLowering function in Bridge by calling a new function markDelayedDeclareTargetFunctions which marks the latently generated operations. In certain cases, some still will not be generated, e.g. if an interface is defined, marked as declare target, but has no definition or usage in the module then it will not be emitted to the module, so due to these cases we must silently ignore when an operation has not been found via it's symbol. The main use-case for this (although, I imagine there is others) is for processing interfaces that have been declared in a module with a declare target directive but do not have their implementation defined in the same module. For example, inside of a seperate C++ module that will be linked in. In cases where the interface is called inside of a target region it'll be marked as used on device appropriately (although, realistically a user should explicitly mark it to match the corresponding definition), however, in cases where it's used in a non-clear manner through something like a function pointer passed to an external call we require this explicit marking, which this patch adds support for (currently will cause the compiler to crash). This patch also adds documentation on the declare target process and mechanisms within the compiler currently. --- flang/docs/OpenMP-declare-target.md | 257 ++++++++++++++++++ flang/docs/index.md | 1 + flang/include/flang/Lower/OpenMP.h | 21 ++ flang/lib/Lower/Bridge.cpp | 17 ++ flang/lib/Lower/OpenMP/OpenMP.cpp | 159 ++++++++--- flang/lib/Lower/OpenMP/Utils.h | 2 +- .../declare-target-deferred-marking.f90 | 60 ++++ 7 files changed, 476 insertions(+), 41 deletions(-) create mode 100644 flang/docs/OpenMP-declare-target.md create mode 100644 flang/test/Lower/OpenMP/declare-target-deferred-marking.f90 diff --git a/flang/docs/OpenMP-declare-target.md b/flang/docs/OpenMP-declare-target.md new file mode 100644 index 000000000000..d29a46807e1e --- /dev/null +++ b/flang/docs/OpenMP-declare-target.md @@ -0,0 +1,257 @@ + + +# Introduction to Declare Target + +In OpenMP `declare target` is a directive that can be applied to a function or +variable (primarily global) to notate to the compiler that it should be +generated in a particular device's environment. In essence whether something +should be emitted for host or device, or both. An example of its usage for +both data and functions can be seen below. + +```Fortran +module test_0 + integer :: sp = 0 +!$omp declare target link(sp) +end module test_0 + +program main + use test_0 +!$omp target map(tofrom:sp) + sp = 1 +!$omp end target +end program +``` + +In the above example, we create a variable in a separate module, mark it +as `declare target` and then map it, embedding it into the device IR and +assigning to it. + + +```Fortran +function func_t_device() result(i) + !$omp declare target to(func_t_device) device_type(nohost) + INTEGER :: I + I = 1 +end function func_t_device + +program main +!$omp target + call func_t_device() +!$omp end target +end program +``` + +In the above example, we are stating that a function is required on device +utilising `declare target`, and that we will not be utilising it on host, +so we are in theory free to remove or ignore it there. A user could also +in this case, leave off the `declare target` from the function and it +would be implicitly marked `declare target any` (for both host and device), +as it's been utilised within a target region. + +# Declare Target as represented in the OpenMP Dialect + +In the OpenMP Dialect `declare target` is not represented by a specific +`operation`. Instead, it's an OpenMP dialect specific `attribute` that can be +applied to any operation in any dialect, which helps to simplify the +utilisation of it. Rather than replacing or modifying existing global or +function `operations` in a dialect, it applies to it as extra metadata that +the lowering can use in different ways as is necessary. + +The `attribute` is composed of multiple fields representing the clauses you +would find on the `declare target` directive i.e. device type (`nohost`, +`any`, `host`) or the capture clause (`link` or `to`). A small example of +`declare target` applied to a Fortran `real` can be found below: + +``` +fir.global internal @_QFEi {omp.declare_target = +#omp.declaretarget} : f32 { + %0 = fir.undefined f32 + fir.has_value %0 : f32 +} +``` + +This would look similar for function style `operations`. + +The application and access of this attribute is aided by an OpenMP Dialect +MLIR Interface named `DeclareTargetInterface`, which can be utilised on +operations to access the appropriate interface functions, e.g.: + +```C++ +auto declareTargetGlobal = +llvm::dyn_cast(Op.getOperation()); +declareTargetGlobal.isDeclareTarget(); +``` + +# Declare Target Fortran OpenMP Lowering + +The initial lowering of `declare target` to MLIR for both use-cases is done +inside of the usual OpenMP lowering in flang/lib/Lower/OpenMP.cpp. However, +some direct calls to `declare target` related functions from Flang's +lowering bridge in flang/lib/Lower/Bridge.cpp are made. + +The marking of operations with the declare target attribute happens in two +phases, the second one optional and contingent on the first failing. The +initial phase happens when the declare target directive and its clauses +are initially processed, with the primary data gathering for the directive and +clause happening in a function called `getDeclareTargetInfo`. This is then used +to feed the `markDeclareTarget` function, which does the actual marking +utilising the `DeclareTargetInterface`. If it encounters a variable or function +that has been marked twice over multiple directives with two differing device +types (e.g. `host`, `nohost`), then it will swap the device type to `any`. + +Whenever we invoke `genFIR` on an `OpenMPDeclarativeConstruct` from the +lowering bridge, we are also invoking another function called +`gatherOpenMPDeferredDeclareTargets`, which gathers information relevant to the +application of the `declare target` attribute. This information +includes the symbol that it should be applied to, device type clause, +and capture clause, and it is stored in a vector that is part of the lowering +bridge's instantiation of the `AbstractConverter`. It is only stored if we +encounter a function or variable symbol that does not have an operation +instantiated for it yet. This cannot happen as part of the +initial marking as we must store this data in the lowering bridge and we +only have access to the abstract version of the converter via the OpenMP +lowering. + +The information produced by the first phase is used in the second phase, +which is a form of deferred processing of the `declare target` marked +operations that have delayed generation and cannot be proccessed in the +first phase. The main notable case this occurs currently is when a +Fortran function interface has been marked. This is +done via the function +`markOpenMPDeferredDeclareTargetFunctions`, which is called from the lowering +bridge at the end of the lowering process allowing us to mark those where +possible. It iterates over the data previously gathered by +`gatherOpenMPDeferredDeclareTargets` +checking if any of the recorded symbols have now had their corresponding +operations instantiated and applying the declare target attribute where +possible utilising `markDeclareTarget`. However, it must be noted that it +is still possible for operations not to be generated for certain symbols, +in particular the case of function interfaces that are not directly used +or defined within the current module. This means we cannot emit errors in +the case of left-over unmarked symbols. These must (and should) be caught +by the initial semantic analysis. + +NOTE: `declare target` can be applied to implicit `SAVE` attributed variables. +However, by default Flang does not represent these as `GlobalOp`'s, which means +we cannot tag and lower them as `declare target` normally. Instead, similarly +to the way `threadprivate` handles these cases, we raise and initialize the +variable as an internal `GlobalOp` and apply the attribute. This occurs in the +flang/lib/Lower/OpenMP.cpp function `genDeclareTargetIntGlobal`. + +# Declare Target Transformation Passes for Flang + +There are currently two passes within Flang that are related to the processing +of `declare target`: +* `OMPMarkDeclareTarget` - This pass is in charge of marking functions captured +(called from) in `target` regions or other `declare target` marked functions as +`declare target`. It does so recursively, i.e. nested calls will also be +implicitly marked. It currently will try to mark things as conservatively as +possible, e.g. if captured in a `target` region it will apply `nohost`, unless +it encounters a `host` `declare target` in which case it will apply the `any` +device type. Functions are handled similarly, except we utilise the parent's +device type where possible. +* `OMPFunctionFiltering` - This is executed after the `OMPMarkDeclareTarget` +pass, and its job is to conservatively remove host functions from +the module where possible when compiling for the device. This helps make +sure that most incompatible code for the host is not lowered for the +device. Host functions with `target` regions in them need to be preserved +(e.g. for lowering the `target region`(s) inside). Otherwise, it removes +any function marked as a `declare target host` function and any uses will be +replaced with `undef`'s so that the remaining host code doesn't become broken. +Host functions with `target` regions are marked with a `declare target host` +attribute so they will be removed after outlining the target regions contained +inside. + +While this infrastructure could be generally applicable to more than just Flang, +it is only utilised in the Flang frontend, so it resides there rather than in +the OpenMP dialect codebase. + +# Declare Target OpenMP Dialect To LLVM-IR Lowering + +The OpenMP dialect lowering of `declare target` is done through the +`amendOperation` flow, as it's not an `operation` but rather an +`attribute`. This is triggered immediately after the corresponding +operation has been lowered to LLVM-IR. As it is applicable to +different types of operations, we must specialise this function for +each operation type that we may encounter. Currently, this is +`GlobalOp`'s and `FuncOp`'s. + +`FuncOp` processing is fairly simple. When compiling for the device, +`host` marked functions are removed, including those that could not +be removed earlier due to having `target` directives within. This +leaves `any`, `device` or indeterminable functions left in the +module to lower further. When compiling for the host, no filtering is +done because `nohost` functions must be available as a fallback +implementation. + +For `GlobalOp`'s, the processing is a little more complex. We +currently leverage the `registerTargetGlobalVariable` and +`getAddrOfDeclareTargetVar` `OMPIRBuilder` functions shared with Clang. +These two functions invoke each other depending on the clauses and options +provided to the `OMPIRBuilder` (in particular, unified shared memory). Their +main purposes are the generation of a new global device pointer with a +"ref_" prefix on the device and enqueuing metadata generation by the +`OMPIRBuilder` to be produced at module finalization time. This is done +for both host and device and it links the newly generated device global +pointer and the host pointer together across the two modules. + +Similarly to other metadata (e.g. for `TargetOp`) that is shared across +both host and device modules, processing of `GlobalOp`'s in the device +needs access to the previously generated host IR file, which is done +through another `attribute` applied to the `ModuleOp` by the compiler +frontend. The file is loaded in and consumed by the `OMPIRBuilder` to +populate it's `OffloadInfoManager` data structures, keeping host and +device appropriately synchronised. + +The second (and more important to remember) is that as we effectively replace +the original LLVM-IR generated for the `declare target` marked `GlobalOp` we +have some corrections we need to do for `TargetOp`'s (or other region +operations that use them directly) which still refer to the original lowered +global operation. This is done via `handleDeclareTargetMapVar` which is invoked +as the final function and alteration to the lowered `target` region, it's only +invoked for device as it's only required in the case where we have emitted the +"ref" pointer , and it effectively replaces all uses of the originally lowered +global symbol, with our new global ref pointer's symbol. Currently we do not +remove or delete the old symbol, this is due to the fact that the same symbol +can be utilised across multiple target regions, if we remove it, we risk +breaking lowerings of target regions that will be processed at a later time. +To appropriately delete these no longer necessary symbols we would need a +deferred removal process at the end of the module, which is currently not in +place. It may be possible to store this information in the OMPIRBuilder and +then perform this cleanup process on finalization, but this is open for +discussion and implementation still. + +# Current Support + +For the moment, `declare target` should work for: +* Marking functions/subroutines and function/subroutine interfaces for +generation on host, device or both. +* Implicit function/subroutine capture for calls emitted in a `target` region +or explicitly marked `declare target` function/subroutine. Note: Calls made +via arguments passed to other functions must still be themselves marked +`declare target`, e.g. passing a `C` function pointer and invoking it, then +the interface and the `C` function in the other module must be marked +`declare target`, with the same type of marking as indicated by the +specification. +* Marking global variables with `declare target`'s `link` clause and mapping +the data to the device data environment utilising `declare target`. This may +not work for all types yet, but for scalars and arrays of scalars, it +should. + +Doesn't work for, or needs further testing for: +* Marking the following types with `declare target link` (needs further +testing): + * Descriptor based types, e.g. pointers/allocatables. + * Derived types. + * Members of derived types (use-case needs legality checking with OpenMP +specification). +* Marking global variables with `declare target`'s `to` clause. A lot of the +lowering should exist, but it needs further testing and likely some further +changes to fully function. diff --git a/flang/docs/index.md b/flang/docs/index.md index d974a3628b01..b4dbdc87fdf6 100644 --- a/flang/docs/index.md +++ b/flang/docs/index.md @@ -68,6 +68,7 @@ on how to get in touch with us and to learn more about the current status. OpenACC OpenACC-descriptor-management.md OpenMP-4.5-grammar.md + OpenMP-declare-target OpenMP-descriptor-management OpenMP-semantics OptionComparison diff --git a/flang/include/flang/Lower/OpenMP.h b/flang/include/flang/Lower/OpenMP.h index e6fe7fb2276d..3b22a652d1fc 100644 --- a/flang/include/flang/Lower/OpenMP.h +++ b/flang/include/flang/Lower/OpenMP.h @@ -13,12 +13,19 @@ #ifndef FORTRAN_LOWER_OPENMP_H #define FORTRAN_LOWER_OPENMP_H +#include "llvm/ADT/SmallVector.h" + #include +#include namespace mlir { class Value; class Operation; class Location; +namespace omp { +enum class DeclareTargetDeviceType : uint32_t; +enum class DeclareTargetCaptureClause : uint32_t; +} // namespace omp } // namespace mlir namespace fir { @@ -49,6 +56,12 @@ struct Evaluation; struct Variable; } // namespace pft +struct OMPDeferredDeclareTargetInfo { + mlir::omp::DeclareTargetCaptureClause declareTargetCaptureClause; + mlir::omp::DeclareTargetDeviceType declareTargetDeviceType; + const Fortran::semantics::Symbol &sym; +}; + // Generate the OpenMP terminator for Operation at Location. mlir::Operation *genOpenMPTerminator(fir::FirOpBuilder &, mlir::Operation *, mlir::Location); @@ -86,6 +99,14 @@ bool isOpenMPDeviceDeclareTarget(Fortran::lower::AbstractConverter &, Fortran::semantics::SemanticsContext &, Fortran::lower::pft::Evaluation &, const parser::OpenMPDeclarativeConstruct &); +void gatherOpenMPDeferredDeclareTargets( + Fortran::lower::AbstractConverter &, Fortran::semantics::SemanticsContext &, + Fortran::lower::pft::Evaluation &, + const parser::OpenMPDeclarativeConstruct &, + llvm::SmallVectorImpl &); +bool markOpenMPDeferredDeclareTargetFunctions( + mlir::Operation *, llvm::SmallVectorImpl &, + AbstractConverter &); void genOpenMPRequires(mlir::Operation *, const Fortran::semantics::Symbol *); } // namespace lower diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp index 153ce0623ab3..8048693119b4 100644 --- a/flang/lib/Lower/Bridge.cpp +++ b/flang/lib/Lower/Bridge.cpp @@ -2633,6 +2633,9 @@ private: ompDeviceCodeFound || Fortran::lower::isOpenMPDeviceDeclareTarget( *this, bridge.getSemanticsContext(), getEval(), ompDecl); + Fortran::lower::gatherOpenMPDeferredDeclareTargets( + *this, bridge.getSemanticsContext(), getEval(), ompDecl, + ompDeferredDeclareTarget); genOpenMPDeclarativeConstruct( *this, localSymbols, bridge.getSemanticsContext(), getEval(), ompDecl); builder->restoreInsertionPoint(insertPt); @@ -5171,6 +5174,13 @@ private: /// lowering. void finalizeOpenMPLowering( const Fortran::semantics::Symbol *globalOmpRequiresSymbol) { + if (!ompDeferredDeclareTarget.empty()) { + bool deferredDeviceFuncFound = + Fortran::lower::markOpenMPDeferredDeclareTargetFunctions( + getModuleOp().getOperation(), ompDeferredDeclareTarget, *this); + ompDeviceCodeFound = ompDeviceCodeFound || deferredDeviceFuncFound; + } + // Set the module attribute related to OpenMP requires directives if (ompDeviceCodeFound) Fortran::lower::genOpenMPRequires(getModuleOp().getOperation(), @@ -5227,6 +5237,13 @@ private: /// intended for device offloading has been detected bool ompDeviceCodeFound = false; + /// Keeps track of symbols defined as declare target that could not be + /// processed at the time of lowering the declare target construct, such + /// as certain cases where interfaces are declared but not defined within + /// a module. + llvm::SmallVector + ompDeferredDeclareTarget; + const Fortran::lower::ExprToValueMap *exprValueOverrides{nullptr}; /// Stack of derived type under construction to avoid infinite loops when diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp index 83aebdaecd35..185e0316870e 100644 --- a/flang/lib/Lower/OpenMP/OpenMP.cpp +++ b/flang/lib/Lower/OpenMP/OpenMP.cpp @@ -1238,6 +1238,31 @@ static mlir::omp::DeclareTargetDeviceType getDeclareTargetInfo( return deviceType; } +static void collectDeferredDeclareTargets( + Fortran::lower::AbstractConverter &converter, + Fortran::semantics::SemanticsContext &semaCtx, + Fortran::lower::pft::Evaluation &eval, + const Fortran::parser::OpenMPDeclareTargetConstruct &declareTargetConstruct, + llvm::SmallVectorImpl + &deferredDeclareTarget) { + llvm::SmallVector symbolAndClause; + mlir::omp::DeclareTargetDeviceType devType = getDeclareTargetInfo( + converter, semaCtx, eval, declareTargetConstruct, symbolAndClause); + // Return the device type only if at least one of the targets for the + // directive is a function or subroutine + mlir::ModuleOp mod = converter.getFirOpBuilder().getModule(); + + for (const DeclareTargetCapturePair &symClause : symbolAndClause) { + mlir::Operation *op = mod.lookupSymbol(converter.mangleName( + std::get(symClause))); + + if (!op) { + deferredDeclareTarget.push_back( + {std::get<0>(symClause), devType, std::get<1>(symClause)}); + } + } +} + static std::optional getDeclareTargetFunctionDevice( Fortran::lower::AbstractConverter &converter, @@ -1245,7 +1270,7 @@ getDeclareTargetFunctionDevice( Fortran::lower::pft::Evaluation &eval, const Fortran::parser::OpenMPDeclareTargetConstruct &declareTargetConstruct) { - llvm::SmallVector symbolAndClause; + llvm::SmallVector symbolAndClause; mlir::omp::DeclareTargetDeviceType deviceType = getDeclareTargetInfo( converter, semaCtx, eval, declareTargetConstruct, symbolAndClause); @@ -1253,10 +1278,10 @@ getDeclareTargetFunctionDevice( // directive is a function or subroutine mlir::ModuleOp mod = converter.getFirOpBuilder().getModule(); for (const DeclareTargetCapturePair &symClause : symbolAndClause) { - mlir::Operation *op = mod.lookupSymbol( - converter.mangleName(std::get(symClause))); + mlir::Operation *op = mod.lookupSymbol(converter.mangleName( + std::get(symClause))); - if (mlir::isa(op)) + if (mlir::isa_and_nonnull(op)) return deviceType; } @@ -2007,56 +2032,56 @@ genOMP(Fortran::lower::AbstractConverter &converter, atomicConstruct.u); } +static void +markDeclareTarget(mlir::Operation *op, + Fortran::lower::AbstractConverter &converter, + mlir::omp::DeclareTargetCaptureClause captureClause, + mlir::omp::DeclareTargetDeviceType deviceType) { + // TODO: Add support for program local variables with declare target applied + auto declareTargetOp = llvm::dyn_cast(op); + if (!declareTargetOp) + fir::emitFatalError( + converter.getCurrentLocation(), + "Attempt to apply declare target on unsupported operation"); + + // The function or global already has a declare target applied to it, very + // likely through implicit capture (usage in another declare target + // function/subroutine). It should be marked as any if it has been assigned + // both host and nohost, else we skip, as there is no change + if (declareTargetOp.isDeclareTarget()) { + if (declareTargetOp.getDeclareTargetDeviceType() != deviceType) + declareTargetOp.setDeclareTarget(mlir::omp::DeclareTargetDeviceType::any, + captureClause); + return; + } + + declareTargetOp.setDeclareTarget(deviceType, captureClause); +} + static void genOMP(Fortran::lower::AbstractConverter &converter, Fortran::lower::SymMap &symTable, Fortran::semantics::SemanticsContext &semaCtx, Fortran::lower::pft::Evaluation &eval, const Fortran::parser::OpenMPDeclareTargetConstruct &declareTargetConstruct) { - llvm::SmallVector symbolAndClause; + llvm::SmallVector symbolAndClause; mlir::ModuleOp mod = converter.getFirOpBuilder().getModule(); mlir::omp::DeclareTargetDeviceType deviceType = getDeclareTargetInfo( converter, semaCtx, eval, declareTargetConstruct, symbolAndClause); for (const DeclareTargetCapturePair &symClause : symbolAndClause) { - mlir::Operation *op = mod.lookupSymbol( - converter.mangleName(std::get(symClause))); - // There's several cases this can currently be triggered and it could be - // one of the following: - // 1) Invalid argument passed to a declare target that currently isn't - // captured by a frontend semantic check - // 2) The symbol of a valid argument is not correctly updated by one of - // the prior passes, resulting in missing symbol information - // 3) It's a variable internal to a module or program, that is legal by - // Fortran OpenMP standards, but is currently unhandled as they do not - // appear in the symbol table as they are represented as allocas + mlir::Operation *op = mod.lookupSymbol(converter.mangleName( + std::get(symClause))); + + // Some symbols are deferred until later in the module, these are handled + // upon finalization of the module for OpenMP inside of Bridge, so we simply + // skip for now. if (!op) - TODO(converter.getCurrentLocation(), - "Missing symbol, possible case of currently unsupported use of " - "a program local variable in declare target or erroneous symbol " - "information "); - - auto declareTargetOp = - llvm::dyn_cast(op); - if (!declareTargetOp) - fir::emitFatalError( - converter.getCurrentLocation(), - "Attempt to apply declare target on unsupported operation"); - - // The function or global already has a declare target applied to it, very - // likely through implicit capture (usage in another declare target - // function/subroutine). It should be marked as any if it has been assigned - // both host and nohost, else we skip, as there is no change - if (declareTargetOp.isDeclareTarget()) { - if (declareTargetOp.getDeclareTargetDeviceType() != deviceType) - declareTargetOp.setDeclareTarget( - mlir::omp::DeclareTargetDeviceType::any, - std::get(symClause)); continue; - } - declareTargetOp.setDeclareTarget( - deviceType, std::get(symClause)); + markDeclareTarget( + op, converter, + std::get(symClause), deviceType); } } @@ -2515,6 +2540,24 @@ bool Fortran::lower::isOpenMPTargetConstruct( return llvm::omp::allTargetSet.test(dir); } +void Fortran::lower::gatherOpenMPDeferredDeclareTargets( + Fortran::lower::AbstractConverter &converter, + Fortran::semantics::SemanticsContext &semaCtx, + Fortran::lower::pft::Evaluation &eval, + const Fortran::parser::OpenMPDeclarativeConstruct &ompDecl, + llvm::SmallVectorImpl + &deferredDeclareTarget) { + std::visit( + Fortran::common::visitors{ + [&](const Fortran::parser::OpenMPDeclareTargetConstruct &ompReq) { + collectDeferredDeclareTargets(converter, semaCtx, eval, ompReq, + deferredDeclareTarget); + }, + [&](const auto &) {}, + }, + ompDecl.u); +} + bool Fortran::lower::isOpenMPDeviceDeclareTarget( Fortran::lower::AbstractConverter &converter, Fortran::semantics::SemanticsContext &semaCtx, @@ -2533,6 +2576,42 @@ bool Fortran::lower::isOpenMPDeviceDeclareTarget( ompDecl.u); } +// In certain cases such as subroutine or function interfaces which declare +// but do not define or directly call the subroutine or function in the same +// module, their lowering is delayed until after the declare target construct +// itself is processed, so there symbol is not within the table. +// +// This function will also return true if we encounter any device declare +// target cases, to satisfy checking if we require the requires attributes +// on the module. +bool Fortran::lower::markOpenMPDeferredDeclareTargetFunctions( + mlir::Operation *mod, + llvm::SmallVectorImpl &deferredDeclareTargets, + AbstractConverter &converter) { + bool deviceCodeFound = false; + auto modOp = llvm::cast(mod); + for (auto declTar : deferredDeclareTargets) { + mlir::Operation *op = modOp.lookupSymbol(converter.mangleName(declTar.sym)); + + // Due to interfaces being optionally emitted on usage in a module, + // not finding an operation at this point cannot be a hard error, we + // simply ignore it for now. + // TODO: Add semantic checks for detecting cases where an erronous + // (undefined) symbol has been supplied to a declare target clause + if (!op) + continue; + + auto devType = declTar.declareTargetDeviceType; + if (!deviceCodeFound && devType != mlir::omp::DeclareTargetDeviceType::host) + deviceCodeFound = true; + + markDeclareTarget(op, converter, declTar.declareTargetCaptureClause, + devType); + } + + return deviceCodeFound; +} + void Fortran::lower::genOpenMPRequires( mlir::Operation *mod, const Fortran::semantics::Symbol *symbol) { using MlirRequires = mlir::omp::ClauseRequires; diff --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h index f57cd7420ce4..76a15e8bcaab 100644 --- a/flang/lib/Lower/OpenMP/Utils.h +++ b/flang/lib/Lower/OpenMP/Utils.h @@ -40,7 +40,7 @@ namespace omp { using DeclareTargetCapturePair = std::pair; + const Fortran::semantics::Symbol &>; mlir::omp::MapInfoOp createMapInfoOp(fir::FirOpBuilder &builder, mlir::Location loc, diff --git a/flang/test/Lower/OpenMP/declare-target-deferred-marking.f90 b/flang/test/Lower/OpenMP/declare-target-deferred-marking.f90 new file mode 100644 index 000000000000..1998c3da23af --- /dev/null +++ b/flang/test/Lower/OpenMP/declare-target-deferred-marking.f90 @@ -0,0 +1,60 @@ +!RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s --check-prefixes ALL,HOST +!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-is-device %s -o - | FileCheck %s --check-prefixes ALL + +program main + use, intrinsic :: iso_c_binding + implicit none + interface + subroutine any_interface() bind(c,name="any_interface") + use, intrinsic :: iso_c_binding + implicit none + !$omp declare target enter(any_interface) device_type(any) + end subroutine any_interface + + subroutine host_interface() bind(c,name="host_interface") + use, intrinsic :: iso_c_binding + implicit none + !$omp declare target enter(host_interface) device_type(host) + end subroutine host_interface + + subroutine device_interface() bind(c,name="device_interface") + use, intrinsic :: iso_c_binding + implicit none + !$omp declare target enter(device_interface) device_type(nohost) + end subroutine device_interface + + subroutine called_from_target_interface(f1, f2) bind(c,name="called_from_target_interface") + use, intrinsic :: iso_c_binding + implicit none + type(c_funptr),value :: f1 + type(c_funptr),value :: f2 + end subroutine called_from_target_interface + + subroutine called_from_host_interface(f1) bind(c,name="called_from_host_interface") + use, intrinsic :: iso_c_binding + implicit none + type(c_funptr),value :: f1 + end subroutine called_from_host_interface + + subroutine unused_unemitted_interface() bind(c,name="unused_unemitted_interface") + use, intrinsic :: iso_c_binding + implicit none + !$omp declare target enter(unused_unemitted_interface) device_type(nohost) + end subroutine unused_unemitted_interface + + end interface + + CALL called_from_host_interface(c_funloc(host_interface)) +!$omp target + CALL called_from_target_interface(c_funloc(any_interface), c_funloc(device_interface)) +!$omp end target + end program main + +!HOST-LABEL: func.func {{.*}} @host_interface() +!HOST-SAME: {{.*}}, omp.declare_target = #omp.declaretarget{{.*}} +!ALL-LABEL: func.func {{.*}} @called_from_target_interface(!fir.ref, !fir.ref) +!ALL-SAME: {{.*}}, omp.declare_target = #omp.declaretarget{{.*}} +!ALL-LABEL: func.func {{.*}} @any_interface() +!ALL-SAME: {{.*}}, omp.declare_target = #omp.declaretarget{{.*}} +!ALL-LABEL: func.func {{.*}} @device_interface() +!ALL-SAME: {{.*}}, omp.declare_target = #omp.declaretarget{{.*}} -- GitLab From 7df43ccf08e40468354d0564f9bcd7dfe681590b Mon Sep 17 00:00:00 2001 From: Vadim D <36827317+vvd170501@users.noreply.github.com> Date: Tue, 5 Mar 2024 19:27:58 +0300 Subject: [PATCH 163/929] [clang] Add -Wmissing-designated-field-initializers (#81364) #56628 changed the behavior of `-Wmissing-field-initializers`, which introduces many new warnings in C++ code that uses partial designated initializers. If such code is being built with `-Wextra -Werror`, this change will break the build. This PR adds a new flag that allows to disable these new warnings and keep the old ones, as was suggested by @AaronBallman in the original issue: https://github.com/llvm/llvm-project/issues/56628#issuecomment-1761510850 Fixes #68933 --- clang/docs/ReleaseNotes.rst | 4 +++ clang/include/clang/Basic/DiagnosticGroups.td | 10 +++++++- .../clang/Basic/DiagnosticSemaKinds.td | 4 +++ clang/lib/Sema/SemaInit.cpp | 25 +++++++++---------- .../SemaCXX/cxx2a-initializer-aggregates.cpp | 11 ++++---- 5 files changed, 35 insertions(+), 19 deletions(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index de5f339cc7f0..5e0352a7eaf6 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -150,6 +150,10 @@ Non-comprehensive list of changes in this release New Compiler Flags ------------------ +- ``-Wmissing-designated-field-initializers``, grouped under ``-Wmissing-field-initializers``. + This diagnostic can be disabled to make ``-Wmissing-field-initializers`` behave + like it did before Clang 18.x. Fixes (`#56628 `_) + Deprecated Compiler Flags ------------------------- diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td index e8b4139d7893..0791a0002319 100644 --- a/clang/include/clang/Basic/DiagnosticGroups.td +++ b/clang/include/clang/Basic/DiagnosticGroups.td @@ -517,7 +517,15 @@ def MethodSignatures : DiagGroup<"method-signatures">; def MismatchedParameterTypes : DiagGroup<"mismatched-parameter-types">; def MismatchedReturnTypes : DiagGroup<"mismatched-return-types">; def MismatchedTags : DiagGroup<"mismatched-tags">; -def MissingFieldInitializers : DiagGroup<"missing-field-initializers">; +def MissingDesignatedFieldInitializers : DiagGroup<"missing-designated-field-initializers">{ + code Documentation = [{ +Warn about designated initializers with some fields missing (only in C++). + }]; +} +// Default -Wmissing-field-initializers matches gcc behavior, +// but missing-designated-field-initializers can be turned off to match old clang behavior. +def MissingFieldInitializers : DiagGroup<"missing-field-initializers", + [MissingDesignatedFieldInitializers]>; def ModuleLock : DiagGroup<"module-lock">; def ModuleBuild : DiagGroup<"module-build">; def ModuleImport : DiagGroup<"module-import">; diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index b50a4895e171..708b3c7f1ede 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -6170,6 +6170,10 @@ def ext_initializer_string_for_char_array_too_long : ExtWarn< def warn_missing_field_initializers : Warning< "missing field %0 initializer">, InGroup, DefaultIgnore; +// The same warning, but another group is needed to disable it separately. +def warn_missing_designated_field_initializers : Warning< + warn_missing_field_initializers.Summary>, + InGroup, DefaultIgnore; def warn_braces_around_init : Warning< "braces around %select{scalar |}0initializer">, InGroup>; diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp index 93b125382b16..060fe35ad968 100644 --- a/clang/lib/Sema/SemaInit.cpp +++ b/clang/lib/Sema/SemaInit.cpp @@ -2227,8 +2227,6 @@ void InitListChecker::CheckStructUnionTypes( size_t NumRecordDecls = llvm::count_if(RD->decls(), [&](const Decl *D) { return isa(D) || isa(D); }); - bool CheckForMissingFields = - !IList->isIdiomaticZeroInitializer(SemaRef.getLangOpts()); bool HasDesignatedInit = false; llvm::SmallPtrSet InitializedFields; @@ -2269,11 +2267,6 @@ void InitListChecker::CheckStructUnionTypes( } InitializedSomething = true; - - // Disable check for missing fields when designators are used. - // This matches gcc behaviour. - if (!SemaRef.getLangOpts().CPlusPlus) - CheckForMissingFields = false; continue; } @@ -2285,7 +2278,7 @@ void InitListChecker::CheckStructUnionTypes( // These are okay for randomized structures. [C99 6.7.8p19] // // Also, if there is only one element in the structure, we allow something - // like this, because it's really not randomized in the tranditional sense. + // like this, because it's really not randomized in the traditional sense. // // struct foo h = {bar}; auto IsZeroInitializer = [&](const Expr *I) { @@ -2363,8 +2356,13 @@ void InitListChecker::CheckStructUnionTypes( } // Emit warnings for missing struct field initializers. - if (!VerifyOnly && InitializedSomething && CheckForMissingFields && - !RD->isUnion()) { + // This check is disabled for designated initializers in C. + // This matches gcc behaviour. + bool IsCDesignatedInitializer = + HasDesignatedInit && !SemaRef.getLangOpts().CPlusPlus; + if (!VerifyOnly && InitializedSomething && !RD->isUnion() && + !IList->isIdiomaticZeroInitializer(SemaRef.getLangOpts()) && + !IsCDesignatedInitializer) { // It is possible we have one or more unnamed bitfields remaining. // Find first (if any) named field and emit warning. for (RecordDecl::field_iterator it = HasDesignatedInit ? RD->field_begin() @@ -2376,9 +2374,10 @@ void InitListChecker::CheckStructUnionTypes( if (!it->isUnnamedBitfield() && !it->hasInClassInitializer() && !it->getType()->isIncompleteArrayType()) { - SemaRef.Diag(IList->getSourceRange().getEnd(), - diag::warn_missing_field_initializers) - << *it; + auto Diag = HasDesignatedInit + ? diag::warn_missing_designated_field_initializers + : diag::warn_missing_field_initializers; + SemaRef.Diag(IList->getSourceRange().getEnd(), Diag) << *it; break; } } diff --git a/clang/test/SemaCXX/cxx2a-initializer-aggregates.cpp b/clang/test/SemaCXX/cxx2a-initializer-aggregates.cpp index 510ace58c35a..1e9c5fa082d0 100644 --- a/clang/test/SemaCXX/cxx2a-initializer-aggregates.cpp +++ b/clang/test/SemaCXX/cxx2a-initializer-aggregates.cpp @@ -4,7 +4,8 @@ // RUN: %clang_cc1 -std=c++20 %s -verify=cxx20,expected,reorder -Wno-c99-designator -Werror=reorder-init-list -Wno-initializer-overrides // RUN: %clang_cc1 -std=c++20 %s -verify=cxx20,expected,override -Wno-c99-designator -Wno-reorder-init-list -Werror=initializer-overrides // RUN: %clang_cc1 -std=c++20 %s -verify=cxx20,expected -Wno-c99-designator -Wno-reorder-init-list -Wno-initializer-overrides -// RUN: %clang_cc1 -std=c++20 %s -verify=cxx20,expected,wmissing -Wmissing-field-initializers -Wno-c99-designator -Wno-reorder-init-list -Wno-initializer-overrides +// RUN: %clang_cc1 -std=c++20 %s -verify=cxx20,expected,wmissing,wmissing-designated -Wmissing-field-initializers -Wno-c99-designator -Wno-reorder-init-list -Wno-initializer-overrides +// RUN: %clang_cc1 -std=c++20 %s -verify=cxx20,expected,wmissing -Wmissing-field-initializers -Wno-missing-designated-field-initializers -Wno-c99-designator -Wno-reorder-init-list -Wno-initializer-overrides namespace class_with_ctor { @@ -50,11 +51,11 @@ A a3 = { A a4 = { .x = 1, // override-note {{previous}} .x = 1 // override-error {{overrides prior initialization}} -}; // wmissing-warning {{missing field 'y' initializer}} +}; // wmissing-designated-warning {{missing field 'y' initializer}} A a5 = { .y = 1, // override-note {{previous}} .y = 1 // override-error {{overrides prior initialization}} -}; // wmissing-warning {{missing field 'x' initializer}} +}; // wmissing-designated-warning {{missing field 'x' initializer}} B b2 = {.a = 1}; // pedantic-error {{brace elision for designated initializer is a C99 extension}} // wmissing-warning@-1 {{missing field 'y' initializer}} B b3 = {.a = 1, 2}; // pedantic-error {{mixture of designated and non-designated}} pedantic-note {{first non-designated}} pedantic-error {{brace elision}} @@ -74,8 +75,8 @@ C c = { struct Foo { int a, b; }; struct Foo foo0 = { 1 }; // wmissing-warning {{missing field 'b' initializer}} -struct Foo foo1 = { .a = 1 }; // wmissing-warning {{missing field 'b' initializer}} -struct Foo foo2 = { .b = 1 }; // wmissing-warning {{missing field 'a' initializer}} +struct Foo foo1 = { .a = 1 }; // wmissing-designated-warning {{missing field 'b' initializer}} +struct Foo foo2 = { .b = 1 }; // wmissing-designated-warning {{missing field 'a' initializer}} } -- GitLab From e49479b881f856e0473b2b3f0e6489feeddf2418 Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Tue, 5 Mar 2024 16:35:29 +0000 Subject: [PATCH 164/929] [AMDGPU] Remove unneeded BUF _impl multiclasses. NFC. (#84034) Remove MUBUF_Real_gfx11_impl and others. By converting the underlying class MUBUF_Real_gfx11 into a multiclass, the _impl wrapper is no longer needed. --- llvm/lib/Target/AMDGPU/BUFInstructions.td | 104 ++++++++++------------ 1 file changed, 45 insertions(+), 59 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index cb68e45d69e8..b984126d8447 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -2297,19 +2297,20 @@ class Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11 op, MUBUF_Pseudo ps, - string real_name = ps.Mnemonic> : - Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11 { - let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); - // In GFX11 dlc is applicable to all loads/stores/atomics. - let Inst{13} = !if(!or(ps.mayLoad, ps.mayStore), cpol{CPolBit.DLC}, ps.dlc_value); - let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); - let Inst{25-18} = op; - let Inst{53} = ps.tfe; - let Inst{54} = ps.offen; - let Inst{55} = ps.idxen; - let AssemblerPredicate = isGFX11Only; - let DecoderNamespace = "GFX11"; +multiclass MUBUF_Real_gfx11 op, string real_name = !cast(NAME).Mnemonic> { + defvar ps = !cast(NAME); + def _gfx11 : Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11 { + let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); + // In GFX11 dlc is applicable to all loads/stores/atomics. + let Inst{13} = !if(!or(ps.mayLoad, ps.mayStore), cpol{CPolBit.DLC}, ps.dlc_value); + let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); + let Inst{25-18} = op; + let Inst{53} = ps.tfe; + let Inst{54} = ps.offen; + let Inst{55} = ps.idxen; + let AssemblerPredicate = isGFX11Only; + let DecoderNamespace = "GFX11"; + } } class Base_MUBUF_Real_gfx6_gfx7_gfx10 op, MUBUF_Pseudo ps, int ef> : @@ -2415,18 +2416,19 @@ class VBUFFER_Real : let Inst{31-26} = 0b110001; } -class VBUFFER_MUBUF_Real_gfx12 op, MUBUF_Pseudo ps, - string real_name = ps.Mnemonic> : - VBUFFER_Real, SIMCInstr { - - let MUBUF = 1; - - // Set the last bit of format to 1 to avoid round-trip issues, as some tools - // print BUF_FMT_INVALID for format 0. - let Inst{55} = 0b1; - let Inst{21-14} = op; - let AssemblerPredicate = isGFX12Only; - let DecoderNamespace = "GFX12"; +multiclass VBUFFER_MUBUF_Real_gfx12 op, string real_name> { + defvar ps = !cast(NAME); + def _gfx12 : VBUFFER_Real, + SIMCInstr { + let MUBUF = 1; + + // Set the last bit of format to 1 to avoid round-trip issues, as some tools + // print BUF_FMT_INVALID for format 0. + let Inst{55} = 0b1; + let Inst{21-14} = op; + let AssemblerPredicate = isGFX12Only; + let DecoderNamespace = "GFX12"; + } } multiclass VBUFFER_MTBUF_Real_gfx12 op, string real_name> { @@ -2464,26 +2466,18 @@ class Mnem_gfx11 : class Mnem_gfx12 : MnemonicAlias, Requires<[isGFX12Plus]>; -multiclass MUBUF_Real_gfx11_impl op, string real_name> { - def _gfx11 : MUBUF_Real_gfx11(NAME), real_name>; -} - -multiclass VBUFFER_MUBUF_Real_gfx12_impl op, string real_name> { - def _gfx12 : VBUFFER_MUBUF_Real_gfx12(NAME), real_name>; -} - multiclass MUBUF_Real_AllAddr_gfx11_Renamed_Impl2 op, string real_name> { - defm _BOTHEN : MUBUF_Real_gfx11_impl; - defm _IDXEN : MUBUF_Real_gfx11_impl; - defm _OFFEN : MUBUF_Real_gfx11_impl; - defm _OFFSET : MUBUF_Real_gfx11_impl; + defm _BOTHEN : MUBUF_Real_gfx11; + defm _IDXEN : MUBUF_Real_gfx11; + defm _OFFEN : MUBUF_Real_gfx11; + defm _OFFSET : MUBUF_Real_gfx11; } multiclass MUBUF_Real_AllAddr_gfx12_Renamed_Impl2 op, string real_name> { - defm _VBUFFER_BOTHEN : VBUFFER_MUBUF_Real_gfx12_impl; - defm _VBUFFER_IDXEN : VBUFFER_MUBUF_Real_gfx12_impl; - defm _VBUFFER_OFFEN : VBUFFER_MUBUF_Real_gfx12_impl; - defm _VBUFFER_OFFSET : VBUFFER_MUBUF_Real_gfx12_impl; + defm _VBUFFER_BOTHEN : VBUFFER_MUBUF_Real_gfx12; + defm _VBUFFER_IDXEN : VBUFFER_MUBUF_Real_gfx12; + defm _VBUFFER_OFFEN : VBUFFER_MUBUF_Real_gfx12; + defm _VBUFFER_OFFSET : VBUFFER_MUBUF_Real_gfx12; } multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl2 op, string real_name> : @@ -2516,30 +2510,22 @@ multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Renamed op, string real_name> def : Mnem_gfx11_gfx12.Mnemonic, real_name>; } -multiclass MUBUF_Real_Atomic_gfx11_impl op, string real_name> { - def _gfx11 : MUBUF_Real_gfx11(NAME), real_name>; -} - -multiclass MUBUF_Real_Atomic_gfx12_impl op, string real_name> { - def _gfx12 : VBUFFER_MUBUF_Real_gfx12(NAME), real_name>; -} - multiclass MUBUF_Real_Atomic_gfx11_Renamed_impl op, bit is_return, string real_name> { defvar Rtn = !if(is_return, "_RTN", ""); - defm _BOTHEN#Rtn : MUBUF_Real_Atomic_gfx11_impl; - defm _IDXEN#Rtn : MUBUF_Real_Atomic_gfx11_impl; - defm _OFFEN#Rtn : MUBUF_Real_Atomic_gfx11_impl; - defm _OFFSET#Rtn : MUBUF_Real_Atomic_gfx11_impl; + defm _BOTHEN#Rtn : MUBUF_Real_gfx11; + defm _IDXEN#Rtn : MUBUF_Real_gfx11; + defm _OFFEN#Rtn : MUBUF_Real_gfx11; + defm _OFFSET#Rtn : MUBUF_Real_gfx11; } multiclass MUBUF_Real_Atomic_gfx12_Renamed_impl op, bit is_return, string real_name> { defvar Rtn = !if(is_return, "_RTN", ""); - defm _VBUFFER_BOTHEN#Rtn : MUBUF_Real_Atomic_gfx12_impl; - defm _VBUFFER_IDXEN#Rtn : MUBUF_Real_Atomic_gfx12_impl; - defm _VBUFFER_OFFEN#Rtn : MUBUF_Real_Atomic_gfx12_impl; - defm _VBUFFER_OFFSET#Rtn : MUBUF_Real_Atomic_gfx12_impl; + defm _VBUFFER_BOTHEN#Rtn : VBUFFER_MUBUF_Real_gfx12; + defm _VBUFFER_IDXEN#Rtn : VBUFFER_MUBUF_Real_gfx12; + defm _VBUFFER_OFFEN#Rtn : VBUFFER_MUBUF_Real_gfx12; + defm _VBUFFER_OFFSET#Rtn : VBUFFER_MUBUF_Real_gfx12; } multiclass MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl op, bit is_return, @@ -2578,8 +2564,8 @@ multiclass MUBUF_Real_Atomic_gfx11_gfx12_Renamed_gfx12_Renamed op, strin def : Mnem_gfx12; } -def BUFFER_GL0_INV_gfx11 : MUBUF_Real_gfx11<0x02B, BUFFER_GL0_INV>; -def BUFFER_GL1_INV_gfx11 : MUBUF_Real_gfx11<0x02C, BUFFER_GL1_INV>; +defm BUFFER_GL0_INV : MUBUF_Real_gfx11<0x02B>; +defm BUFFER_GL1_INV : MUBUF_Real_gfx11<0x02C>; defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x014, "buffer_load_b32">; defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x015, "buffer_load_b64">; -- GitLab From 26058e68ea24602e0b7f4e7baa0e17c7db75a623 Mon Sep 17 00:00:00 2001 From: elhewaty Date: Tue, 5 Mar 2024 18:41:41 +0200 Subject: [PATCH 165/929] [DAG] select (sext m), (add X, C), X --> (add X, (and C, (sext m)))) (#83640) - [DAG][X86] Add tests for Folding select m, add(X, C), X --> add (X, and(C, m))(NFC) - [DAG][X86] Fold select (sext m), (add X, C), X --> (add X, (and C, (sext m)))) - Fixes: https://github.com/llvm/llvm-project/issues/66101 --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 11 +++++++ llvm/test/CodeGen/X86/vselect.ll | 29 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index f160dde25d26..932944d4b6f3 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -12074,6 +12074,17 @@ SDValue DAGCombiner::visitVSELECT(SDNode *N) { if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false)) return DAG.getSelect(DL, VT, F, N2, N1); + // select (sext m), (add X, C), X --> (add X, (and C, (sext m)))) + if (N1.getOpcode() == ISD::ADD && N1.getOperand(0) == N2 && N1->hasOneUse() && + DAG.isConstantIntBuildVectorOrConstantInt(N1.getOperand(1)) && + N0.getScalarValueSizeInBits() == N1.getScalarValueSizeInBits() && + TLI.getBooleanContents(N0.getValueType()) == + TargetLowering::ZeroOrNegativeOneBooleanContent) { + return DAG.getNode( + ISD::ADD, DL, N1.getValueType(), N2, + DAG.getNode(ISD::AND, DL, N0.getValueType(), N1.getOperand(1), N0)); + } + // Canonicalize integer abs. // vselect (setg[te] X, 0), X, -X -> // vselect (setgt X, -1), X, -X -> diff --git a/llvm/test/CodeGen/X86/vselect.ll b/llvm/test/CodeGen/X86/vselect.ll index cc4eb0c8f734..9acd995d612c 100644 --- a/llvm/test/CodeGen/X86/vselect.ll +++ b/llvm/test/CodeGen/X86/vselect.ll @@ -4,6 +4,35 @@ ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2 +; PR66101 - Fold select (sext m), (add X, C), X --> (add X, (and C, (sext m)))) +define <4 x i32> @masked_select_const(<4 x i32> %a, <4 x i32> %x, <4 x i32> %y) { +; SSE-LABEL: masked_select_const: +; SSE: # %bb.0: +; SSE-NEXT: pcmpgtd %xmm2, %xmm1 +; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; SSE-NEXT: paddd %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX1-LABEL: masked_select_const: +; AVX1: # %bb.0: +; AVX1-NEXT: vpcmpgtd %xmm2, %xmm1, %xmm1 +; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 +; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: masked_select_const: +; AVX2: # %bb.0: +; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm3 = [4294967272,4294967272,4294967272,4294967272] +; AVX2-NEXT: vpcmpgtd %xmm2, %xmm1, %xmm1 +; AVX2-NEXT: vpand %xmm3, %xmm1, %xmm1 +; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq + %sub.i = add <4 x i32> %a, + %cmp.i = icmp sgt <4 x i32> %x, %y + %sel = select <4 x i1> %cmp.i, <4 x i32> %sub.i, <4 x i32> %a + ret <4 x i32> %sel +} + ; Verify that we don't emit packed vector shifts instructions if the ; condition used by the vector select is a vector of constants. -- GitLab From 6b5888c27f2d7cfc5fce582500a12a7eda16a7e2 Mon Sep 17 00:00:00 2001 From: Yuta Mukai Date: Wed, 6 Mar 2024 01:47:28 +0900 Subject: [PATCH 166/929] [AArch64] Verify ldp/stp alignment stricter (#83948) When ldp-aligned-only/stp-aligned-only is specified, modified to cancel ldp/stp transformation if MachineMemOperand is not present or the access size is unknown. In the previous implementation, the test passed when there was no MachineMemOperand. Also, if the size was unknown, an incorrect value was used or an assertion failed. (But actually, if there is no MachineMemOperand, it will be excluded from the target by isCandidateToMergeOrPair() before reaching the part.) A statistic NumFailedAlignmentCheck is added. NumPairCreated is modified so that it only counts if it is not cancelled. --- .../AArch64/AArch64LoadStoreOptimizer.cpp | 44 ++++++++------- .../CodeGen/AArch64/ldp-stp-unknown-size.mir | 56 +++++++++++++++++++ 2 files changed, 81 insertions(+), 19 deletions(-) create mode 100644 llvm/test/CodeGen/AArch64/ldp-stp-unknown-size.mir diff --git a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp index 926a89466255..abfb1f43a9c8 100644 --- a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp +++ b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp @@ -62,6 +62,8 @@ STATISTIC(NumUnscaledPairCreated, "Number of load/store from unscaled generated"); STATISTIC(NumZeroStoresPromoted, "Number of narrow zero stores promoted"); STATISTIC(NumLoadsFromStoresPromoted, "Number of loads from stores promoted"); +STATISTIC(NumFailedAlignmentCheck, "Number of load/store pair transformation " + "not passed the alignment check"); DEBUG_COUNTER(RegRenamingCounter, DEBUG_TYPE "-reg-renaming", "Controls which pairs are considered for renaming"); @@ -2337,9 +2339,6 @@ bool AArch64LoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) { MachineBasicBlock::iterator Paired = findMatchingInsn(MBBI, Flags, LdStLimit, /* FindNarrowMerge = */ false); if (Paired != E) { - ++NumPairCreated; - if (TII->hasUnscaledLdStOffset(MI)) - ++NumUnscaledPairCreated; // Keeping the iterator straight is a pain, so we let the merge routine tell // us what the next instruction is after it's done mucking about. auto Prev = std::prev(MBBI); @@ -2349,24 +2348,27 @@ bool AArch64LoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) { MachineMemOperand *MemOp = MI.memoperands_empty() ? nullptr : MI.memoperands().front(); - // Get the needed alignments to check them if - // ldp-aligned-only/stp-aligned-only features are opted. - uint64_t MemAlignment = MemOp ? MemOp->getAlign().value() : -1; - uint64_t TypeAlignment = MemOp ? Align(MemOp->getSize()).value() : -1; + // If a load/store arrives and ldp/stp-aligned-only feature is opted, check + // that the alignment of the source pointer is at least double the alignment + // of the type. + if ((MI.mayLoad() && Subtarget->hasLdpAlignedOnly()) || + (MI.mayStore() && Subtarget->hasStpAlignedOnly())) { + // If there is no size/align information, cancel the transformation. + if (!MemOp || !MemOp->getMemoryType().isValid()) { + NumFailedAlignmentCheck++; + return false; + } - // If a load arrives and ldp-aligned-only feature is opted, check that the - // alignment of the source pointer is at least double the alignment of the - // type. - if (MI.mayLoad() && Subtarget->hasLdpAlignedOnly() && MemOp && - MemAlignment < 2 * TypeAlignment) - return false; + // Get the needed alignments to check them if + // ldp-aligned-only/stp-aligned-only features are opted. + uint64_t MemAlignment = MemOp->getAlign().value(); + uint64_t TypeAlignment = Align(MemOp->getSize()).value(); - // If a store arrives and stp-aligned-only feature is opted, check that the - // alignment of the source pointer is at least double the alignment of the - // type. - if (MI.mayStore() && Subtarget->hasStpAlignedOnly() && MemOp && - MemAlignment < 2 * TypeAlignment) - return false; + if (MemAlignment < 2 * TypeAlignment) { + NumFailedAlignmentCheck++; + return false; + } + } MBBI = mergePairedInsns(MBBI, Paired, Flags); // Collect liveness info for instructions between Prev and the new position @@ -2374,6 +2376,10 @@ bool AArch64LoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) { for (auto I = std::next(Prev); I != MBBI; I++) updateDefinedRegisters(*I, DefinedInBB, TRI); + ++NumPairCreated; + if (TII->hasUnscaledLdStOffset(MI)) + ++NumUnscaledPairCreated; + return true; } return false; diff --git a/llvm/test/CodeGen/AArch64/ldp-stp-unknown-size.mir b/llvm/test/CodeGen/AArch64/ldp-stp-unknown-size.mir new file mode 100644 index 000000000000..3234a7dc11f0 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/ldp-stp-unknown-size.mir @@ -0,0 +1,56 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4 +# RUN: llc -O2 -mtriple=aarch64 -mcpu=ampere1 -simplify-mir -o - %s -run-pass=aarch64-ldst-opt | FileCheck %s --check-prefixes=CHECK +# RUN: llc -O2 -mtriple=aarch64 -simplify-mir -o - %s -run-pass=aarch64-ldst-opt | FileCheck %s --check-prefixes=CHECK-DEFAULT + +--- | + define i32 @ldp_no_size_info(ptr %0) #0 { + %2 = ptrtoint ptr %0 to i64 + %3 = and i64 %2, -64 + %4 = inttoptr i64 %3 to ptr + %5 = load i32, ptr %4, align 4 + %6 = getelementptr inbounds i32, ptr %4, i64 1 + %7 = load i32, ptr %6, align 4 + %8 = add nsw i32 %7, %5 + ret i32 %8 + } + +... +--- +name: ldp_no_size_info +alignment: 64 +tracksRegLiveness: true +tracksDebugUserValues: true +liveins: + - { reg: '$x0' } +frameInfo: + maxAlignment: 1 + maxCallFrameSize: 0 +machineFunctionInfo: + hasRedZone: false +body: | + bb.0 (%ir-block.1): + liveins: $x0 + + ; CHECK-LABEL: name: ldp_no_size_info + ; CHECK: liveins: $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $x8 = ANDXri killed renamable $x0, 7865 + ; CHECK-NEXT: renamable $w9 = LDRWui renamable $x8, 0 :: (load unknown-size from %ir.4, align 1) + ; CHECK-NEXT: renamable $w8 = LDRWui killed renamable $x8, 1 :: (load unknown-size from %ir.6, align 1) + ; CHECK-NEXT: $w0 = ADDWrs killed renamable $w8, killed renamable $w9, 0 + ; CHECK-NEXT: RET undef $lr, implicit $w0 + ; + ; CHECK-DEFAULT-LABEL: name: ldp_no_size_info + ; CHECK-DEFAULT: liveins: $x0 + ; CHECK-DEFAULT-NEXT: {{ $}} + ; CHECK-DEFAULT-NEXT: renamable $x8 = ANDXri killed renamable $x0, 7865 + ; CHECK-DEFAULT-NEXT: renamable $w9, renamable $w8 = LDPWi renamable $x8, 0 :: (load unknown-size from %ir.4, align 1), (load unknown-size from %ir.6, align 1) + ; CHECK-DEFAULT-NEXT: $w0 = ADDWrs killed renamable $w8, killed renamable $w9, 0 + ; CHECK-DEFAULT-NEXT: RET undef $lr, implicit $w0 + renamable $x8 = ANDXri killed renamable $x0, 7865 + renamable $w9 = LDRWui renamable $x8, 0 :: (load unknown-size from %ir.4) + renamable $w8 = LDRWui killed renamable $x8, 1 :: (load unknown-size from %ir.6) + $w0 = ADDWrs killed renamable $w8, killed renamable $w9, 0 + RET undef $lr, implicit $w0 + +... -- GitLab From a8cb9db5f59dc97c9b3d0370a5879539e7b233d8 Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Tue, 5 Mar 2024 17:48:06 +0100 Subject: [PATCH 167/929] [SystemZ] Use proper relocation for TLS variable debug info (#83975) Debug info refering to a TLS variable via DW_OP_GNU_push_tls_address needs to use a R_390_TLS_LDO64 relocation instead of R_390_64. Fixed by adding a SystemZELFTargetObjectFile override class and proving a getDebugThreadLocalSymbol implementation. --- llvm/lib/Target/SystemZ/CMakeLists.txt | 1 + .../Target/SystemZ/SystemZTargetMachine.cpp | 3 ++- .../SystemZ/SystemZTargetObjectFile.cpp | 19 +++++++++++++ .../Target/SystemZ/SystemZTargetObjectFile.h | 27 +++++++++++++++++++ 4 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 llvm/lib/Target/SystemZ/SystemZTargetObjectFile.cpp create mode 100644 llvm/lib/Target/SystemZ/SystemZTargetObjectFile.h diff --git a/llvm/lib/Target/SystemZ/CMakeLists.txt b/llvm/lib/Target/SystemZ/CMakeLists.txt index 0614e07bde8a..063e5bcd4417 100644 --- a/llvm/lib/Target/SystemZ/CMakeLists.txt +++ b/llvm/lib/Target/SystemZ/CMakeLists.txt @@ -36,6 +36,7 @@ add_llvm_target(SystemZCodeGen SystemZShortenInst.cpp SystemZSubtarget.cpp SystemZTargetMachine.cpp + SystemZTargetObjectFile.cpp SystemZTargetTransformInfo.cpp SystemZTDC.cpp diff --git a/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp b/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp index 121512d5a7e5..2491bd2ee2c1 100644 --- a/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp +++ b/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp @@ -11,6 +11,7 @@ #include "SystemZ.h" #include "SystemZMachineFunctionInfo.h" #include "SystemZMachineScheduler.h" +#include "SystemZTargetObjectFile.h" #include "SystemZTargetTransformInfo.h" #include "TargetInfo/SystemZTargetInfo.h" #include "llvm/ADT/StringRef.h" @@ -83,7 +84,7 @@ static std::unique_ptr createTLOF(const Triple &TT) { // Note: Some times run with -triple s390x-unknown. // In this case, default to ELF unless z/OS specifically provided. - return std::make_unique(); + return std::make_unique(); } static Reloc::Model getEffectiveRelocModel(std::optional RM) { diff --git a/llvm/lib/Target/SystemZ/SystemZTargetObjectFile.cpp b/llvm/lib/Target/SystemZ/SystemZTargetObjectFile.cpp new file mode 100644 index 000000000000..4e7e4c8b8c56 --- /dev/null +++ b/llvm/lib/Target/SystemZ/SystemZTargetObjectFile.cpp @@ -0,0 +1,19 @@ +//===-- SystemZTargetObjectFile.cpp - SystemZ Object Info -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "SystemZTargetObjectFile.h" +#include "llvm/MC/MCExpr.h" +#include "llvm/MC/MCValue.h" +#include "llvm/Target/TargetMachine.h" + +using namespace llvm; + +const MCExpr *SystemZELFTargetObjectFile::getDebugThreadLocalSymbol( + const MCSymbol *Sym) const { + return MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_DTPOFF, getContext()); +} diff --git a/llvm/lib/Target/SystemZ/SystemZTargetObjectFile.h b/llvm/lib/Target/SystemZ/SystemZTargetObjectFile.h new file mode 100644 index 000000000000..9d0adbb81d86 --- /dev/null +++ b/llvm/lib/Target/SystemZ/SystemZTargetObjectFile.h @@ -0,0 +1,27 @@ +//===-- SystemZTargetObjectFile.h - SystemZ Object Info ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_SYSTEMZ_SYSTEMZTARGETOBJECTFILE_H +#define LLVM_LIB_TARGET_SYSTEMZ_SYSTEMZTARGETOBJECTFILE_H + +#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" + +namespace llvm { + +/// This implementation is used for SystemZ ELF targets. +class SystemZELFTargetObjectFile : public TargetLoweringObjectFileELF { +public: + SystemZELFTargetObjectFile() {} + + /// Describe a TLS variable address within debug info. + const MCExpr *getDebugThreadLocalSymbol(const MCSymbol *Sym) const override; +}; + +} // end namespace llvm + +#endif -- GitLab From 1e828f838cc0f15074f3dbbb04929c06ef0c9729 Mon Sep 17 00:00:00 2001 From: Jeffrey Byrnes Date: Mon, 19 Feb 2024 12:59:13 -0800 Subject: [PATCH 168/929] [SROA]: Only defer trying partial sized ptr or ptr vector types Change-Id: Ic77f87290905addadd5819dff2d0c62f031022ab --- llvm/lib/Transforms/Scalar/SROA.cpp | 80 ++++++++++++------- llvm/test/Transforms/SROA/vector-promotion.ll | 62 ++++++++++++++ 2 files changed, 115 insertions(+), 27 deletions(-) diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp index cd81fb702f4d..a575dc74d8db 100644 --- a/llvm/lib/Transforms/Scalar/SROA.cpp +++ b/llvm/lib/Transforms/Scalar/SROA.cpp @@ -2257,6 +2257,41 @@ checkVectorTypesForPromotion(Partition &P, const DataLayout &DL, return nullptr; } +static VectorType *createAndCheckVectorTypesForPromotion( + SetVector &OtherTys, ArrayRef CandidateTysCopy, + function_ref CheckCandidateType, Partition &P, + const DataLayout &DL, SmallVectorImpl &CandidateTys, + bool &HaveCommonEltTy, Type *&CommonEltTy, bool &HaveVecPtrTy, + bool &HaveCommonVecPtrTy, VectorType *&CommonVecPtrTy) { + [[maybe_unused]] VectorType *OriginalElt = + CandidateTysCopy.size() ? CandidateTysCopy[0] : nullptr; + // Consider additional vector types where the element type size is a + // multiple of load/store element size. + for (Type *Ty : OtherTys) { + if (!VectorType::isValidElementType(Ty)) + continue; + unsigned TypeSize = DL.getTypeSizeInBits(Ty).getFixedValue(); + // Make a copy of CandidateTys and iterate through it, because we + // might append to CandidateTys in the loop. + for (VectorType *const VTy : CandidateTysCopy) { + // The elements in the copy should remain invariant throughout the loop + assert(CandidateTysCopy[0] == OriginalElt && "Different Element"); + unsigned VectorSize = DL.getTypeSizeInBits(VTy).getFixedValue(); + unsigned ElementSize = + DL.getTypeSizeInBits(VTy->getElementType()).getFixedValue(); + if (TypeSize != VectorSize && TypeSize != ElementSize && + VectorSize % TypeSize == 0) { + VectorType *NewVTy = VectorType::get(Ty, VectorSize / TypeSize, false); + CheckCandidateType(NewVTy); + } + } + } + + return checkVectorTypesForPromotion(P, DL, CandidateTys, HaveCommonEltTy, + CommonEltTy, HaveVecPtrTy, + HaveCommonVecPtrTy, CommonVecPtrTy); +} + /// Test whether the given alloca partitioning and range of slices can be /// promoted to a vector. /// @@ -2271,6 +2306,7 @@ static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL) { // we have different element types. SmallVector CandidateTys; SetVector LoadStoreTys; + SetVector DeferredTys; Type *CommonEltTy = nullptr; VectorType *CommonVecPtrTy = nullptr; bool HaveVecPtrTy = false; @@ -2314,42 +2350,32 @@ static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL) { Ty = SI->getValueOperand()->getType(); else continue; + + auto CandTy = Ty->getScalarType(); + if (CandTy->isPointerTy() && (S.beginOffset() != P.beginOffset() || + S.endOffset() != P.endOffset())) { + DeferredTys.insert(Ty); + continue; + } + LoadStoreTys.insert(Ty); // Consider any loads or stores that are the exact size of the slice. if (S.beginOffset() == P.beginOffset() && S.endOffset() == P.endOffset()) CheckCandidateType(Ty); } - if (auto *VTy = checkVectorTypesForPromotion( - P, DL, CandidateTys, HaveCommonEltTy, CommonEltTy, HaveVecPtrTy, + SmallVector CandidateTysCopy = CandidateTys; + if (auto *VTy = createAndCheckVectorTypesForPromotion( + LoadStoreTys, CandidateTysCopy, CheckCandidateType, P, DL, + CandidateTys, HaveCommonEltTy, CommonEltTy, HaveVecPtrTy, HaveCommonVecPtrTy, CommonVecPtrTy)) return VTy; - // Consider additional vector types where the element type size is a - // multiple of load/store element size. - for (Type *Ty : LoadStoreTys) { - if (!VectorType::isValidElementType(Ty)) - continue; - unsigned TypeSize = DL.getTypeSizeInBits(Ty).getFixedValue(); - // Make a copy of CandidateTys and iterate through it, because we might - // append to CandidateTys in the loop. - SmallVector CandidateTysCopy = CandidateTys; - CandidateTys.clear(); - for (VectorType *&VTy : CandidateTysCopy) { - unsigned VectorSize = DL.getTypeSizeInBits(VTy).getFixedValue(); - unsigned ElementSize = - DL.getTypeSizeInBits(VTy->getElementType()).getFixedValue(); - if (TypeSize != VectorSize && TypeSize != ElementSize && - VectorSize % TypeSize == 0) { - VectorType *NewVTy = VectorType::get(Ty, VectorSize / TypeSize, false); - CheckCandidateType(NewVTy); - } - } - } - - return checkVectorTypesForPromotion(P, DL, CandidateTys, HaveCommonEltTy, - CommonEltTy, HaveVecPtrTy, - HaveCommonVecPtrTy, CommonVecPtrTy); + CandidateTys.clear(); + return createAndCheckVectorTypesForPromotion( + DeferredTys, CandidateTysCopy, CheckCandidateType, P, DL, CandidateTys, + HaveCommonEltTy, CommonEltTy, HaveVecPtrTy, HaveCommonVecPtrTy, + CommonVecPtrTy); } /// Test whether a slice of an alloca is valid for integer widening. diff --git a/llvm/test/Transforms/SROA/vector-promotion.ll b/llvm/test/Transforms/SROA/vector-promotion.ll index 1691f7733ace..917006712258 100644 --- a/llvm/test/Transforms/SROA/vector-promotion.ll +++ b/llvm/test/Transforms/SROA/vector-promotion.ll @@ -1392,6 +1392,68 @@ define <4 x ptr> @ptrLoadStoreTysPtr(ptr %init, i64 %val2) { ret <4 x ptr> %sroaval } +define <4 x i32> @validLoadStoreTy([2 x i64] %cond.coerce) { +; CHECK-LABEL: @validLoadStoreTy( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[COND_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x i64] [[COND_COERCE:%.*]], 0 +; CHECK-NEXT: [[COND_SROA_0_0_VEC_INSERT:%.*]] = insertelement <2 x i64> undef, i64 [[COND_COERCE_FCA_0_EXTRACT]], i32 0 +; CHECK-NEXT: [[COND_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x i64] [[COND_COERCE]], 1 +; CHECK-NEXT: [[COND_SROA_0_8_VEC_INSERT:%.*]] = insertelement <2 x i64> [[COND_SROA_0_0_VEC_INSERT]], i64 [[COND_COERCE_FCA_1_EXTRACT]], i32 1 +; CHECK-NEXT: [[TMP0:%.*]] = bitcast <2 x i64> [[COND_SROA_0_8_VEC_INSERT]] to <4 x i32> +; CHECK-NEXT: ret <4 x i32> [[TMP0]] +; +; DEBUG-LABEL: @validLoadStoreTy( +; DEBUG-NEXT: entry: +; DEBUG-NEXT: call void @llvm.dbg.value(metadata ptr undef, metadata [[META553:![0-9]+]], metadata !DIExpression()), !dbg [[DBG557:![0-9]+]] +; DEBUG-NEXT: call void @llvm.dbg.value(metadata ptr undef, metadata [[META554:![0-9]+]], metadata !DIExpression()), !dbg [[DBG558:![0-9]+]] +; DEBUG-NEXT: [[COND_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x i64] [[COND_COERCE:%.*]], 0, !dbg [[DBG559:![0-9]+]] +; DEBUG-NEXT: [[COND_SROA_0_0_VEC_INSERT:%.*]] = insertelement <2 x i64> undef, i64 [[COND_COERCE_FCA_0_EXTRACT]], i32 0, !dbg [[DBG559]] +; DEBUG-NEXT: [[COND_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x i64] [[COND_COERCE]], 1, !dbg [[DBG559]] +; DEBUG-NEXT: [[COND_SROA_0_8_VEC_INSERT:%.*]] = insertelement <2 x i64> [[COND_SROA_0_0_VEC_INSERT]], i64 [[COND_COERCE_FCA_1_EXTRACT]], i32 1, !dbg [[DBG559]] +; DEBUG-NEXT: call void @llvm.dbg.value(metadata ptr undef, metadata [[META555:![0-9]+]], metadata !DIExpression()), !dbg [[DBG560:![0-9]+]] +; DEBUG-NEXT: [[TMP0:%.*]] = bitcast <2 x i64> [[COND_SROA_0_8_VEC_INSERT]] to <4 x i32>, !dbg [[DBG561:![0-9]+]] +; DEBUG-NEXT: call void @llvm.dbg.value(metadata <4 x i32> [[TMP0]], metadata [[META556:![0-9]+]], metadata !DIExpression()), !dbg [[DBG561]] +; DEBUG-NEXT: ret <4 x i32> [[TMP0]], !dbg [[DBG562:![0-9]+]] +; +entry: + %cond = alloca <4 x i32>, align 8 + %coerce.dive2 = getelementptr inbounds <4 x i32>, ptr %cond, i32 0, i32 0 + store [2 x i64] %cond.coerce, ptr %coerce.dive2, align 8 + %m5 = getelementptr inbounds <4 x i32>, ptr %cond, i32 0, i32 0 + %0 = load <4 x i32>, ptr %m5, align 8 + ret <4 x i32> %0 +} + +; The following test should not crash the compiler +; (calls to CheckCandidateType from createAndCheckVectorTypesForPromotion may change the memory to hold CandidateTys.data()) +define noundef zeroext i1 @CandidateTysRealloc() personality ptr null { +entry: + %alloca = alloca <4x i32>, align 16 + store <4 x i32> , ptr %alloca, align 16 + br label %bb.1 + +bb.1: + br label %bb.1 + +bb.2: + %Load0 = load <4 x i32>, ptr %alloca, align 16 + store <4 x i32> zeroinitializer, ptr %alloca, align 16 + %Load1 = load <4 x i32>, ptr %alloca, align 16 + br label %bb.3 + +bb.3: + br label %bb.3 + +bb.4: + %Load2 = load i64, ptr %alloca, align 16 + %Load3 = load <4 x i32>, ptr %alloca, align 16 + store <4 x i32> zeroinitializer, ptr %alloca, align 16 + br label %bb.5 + +bb.5: + br label %bb.5 +} + declare void @llvm.memcpy.p0.p0.i64(ptr, ptr, i64, i1) declare void @llvm.lifetime.end.p0(i64, ptr) ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: -- GitLab From f33f66be7dc586a597437d7ce7619d87e8637209 Mon Sep 17 00:00:00 2001 From: Jeremy Morse Date: Tue, 5 Mar 2024 15:48:15 +0000 Subject: [PATCH 169/929] [NFC][RemoveDIs] Always use iterators for inserting PHIs It's becoming potentially unsafe to insert a PHI instruction using a plain Instruction pointer. Switch all the remaining sites that create and insert PHIs to use iterators instead. For example, the code in ComplexDeinterleavingPass.cpp is definitely at-risk of mixing PHIs and debug-info. --- llvm/lib/CodeGen/CodeGenPrepare.cpp | 4 ++-- llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp | 2 +- llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp | 2 +- llvm/lib/Target/PowerPC/PPCBoolRetToInt.cpp | 2 +- llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp | 2 +- llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp | 2 +- polly/lib/CodeGen/BlockGenerators.cpp | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp index feefe87f4063..36f6cc83be2c 100644 --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -4130,7 +4130,7 @@ private: PHINode *CurrentPhi = cast(Current); unsigned PredCount = CurrentPhi->getNumIncomingValues(); PHINode *PHI = - PHINode::Create(CommonType, PredCount, "sunk_phi", CurrentPhi); + PHINode::Create(CommonType, PredCount, "sunk_phi", CurrentPhi->getIterator()); Map[Current] = PHI; ST.insertNewPhi(PHI); append_range(Worklist, CurrentPhi->incoming_values()); @@ -6436,7 +6436,7 @@ bool CodeGenPrepare::optimizePhiType( } for (PHINode *Phi : PhiNodes) ValMap[Phi] = PHINode::Create(ConvertTy, Phi->getNumIncomingValues(), - Phi->getName() + ".tc", Phi); + Phi->getName() + ".tc", Phi->getIterator()); // Pipe together all the PhiNodes. for (PHINode *Phi : PhiNodes) { PHINode *NewPhi = cast(ValMap[Phi]); diff --git a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp index a6cacf874bdc..031a271de5bd 100644 --- a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp +++ b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp @@ -1975,7 +1975,7 @@ Value *ComplexDeinterleavingGraph::replaceNode(IRBuilderBase &Builder, // It is filled later when the ReductionOperation is processed. auto *VTy = cast(Node->Real->getType()); auto *NewVTy = VectorType::getDoubleElementsVectorType(VTy); - auto *NewPHI = PHINode::Create(NewVTy, 0, "", BackEdge->getFirstNonPHI()); + auto *NewPHI = PHINode::Create(NewVTy, 0, "", BackEdge->getFirstNonPHIIt()); OldToNewPHI[dyn_cast(Node->Real)] = NewPHI; ReplacementNode = NewPHI; break; diff --git a/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp b/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp index bcfedd341425..404c280ab979 100644 --- a/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp +++ b/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp @@ -1060,7 +1060,7 @@ bool MVEGatherScatterLowering::optimiseOffsets(Value *Offsets, BasicBlock *BB, NewPhi = Phi; } else { // There are other users -> create a new phi - NewPhi = PHINode::Create(Phi->getType(), 2, "NewPhi", Phi); + NewPhi = PHINode::Create(Phi->getType(), 2, "NewPhi", Phi->getIterator()); // Copy the incoming values of the old phi NewPhi->addIncoming(Phi->getIncomingValue(IncrementingBlock == 1 ? 0 : 1), Phi->getIncomingBlock(IncrementingBlock == 1 ? 0 : 1)); diff --git a/llvm/lib/Target/PowerPC/PPCBoolRetToInt.cpp b/llvm/lib/Target/PowerPC/PPCBoolRetToInt.cpp index 20f53bd4badf..f1abb78d4dd3 100644 --- a/llvm/lib/Target/PowerPC/PPCBoolRetToInt.cpp +++ b/llvm/lib/Target/PowerPC/PPCBoolRetToInt.cpp @@ -101,7 +101,7 @@ class PPCBoolRetToInt : public FunctionPass { // runOnUse. Value *Zero = Constant::getNullValue(IntTy); PHINode *Q = - PHINode::Create(IntTy, P->getNumIncomingValues(), P->getName(), P); + PHINode::Create(IntTy, P->getNumIncomingValues(), P->getName(), P->getIterator()); for (unsigned i = 0; i < P->getNumOperands(); ++i) Q->addIncoming(Zero, P->getIncomingBlock(i)); return Q; diff --git a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp index 1dcb83a6078e..0530e3d08be8 100644 --- a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp @@ -229,7 +229,7 @@ bool RISCVGatherScatterLowering::matchStridedRecurrence(Value *Index, Loop *L, // Build scalar phi and increment. BasePtr = - PHINode::Create(Start->getType(), 2, Phi->getName() + ".scalar", Phi); + PHINode::Create(Start->getType(), 2, Phi->getName() + ".scalar", Phi->getIterator()); Inc = BinaryOperator::CreateAdd(BasePtr, Step, Inc->getName() + ".scalar", Inc); BasePtr->addIncoming(Start, Phi->getIncomingBlock(1 - IncrementingBlock)); diff --git a/llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp b/llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp index 591a76e6fd6c..e355a4b9d35b 100644 --- a/llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp +++ b/llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp @@ -116,7 +116,7 @@ BasicBlock *X86LowerAMXIntrinsics::createLoop(BasicBlock *Preheader, BranchInst::Create(Body, Header); BranchInst::Create(Latch, Body); PHINode *IV = - PHINode::Create(I16Ty, 2, Name + ".iv", Header->getTerminator()); + PHINode::Create(I16Ty, 2, Name + ".iv", Header->getTerminator()->getIterator()); IV->addIncoming(ConstantInt::get(I16Ty, 0), Preheader); B.SetInsertPoint(Latch); diff --git a/polly/lib/CodeGen/BlockGenerators.cpp b/polly/lib/CodeGen/BlockGenerators.cpp index 3de1781a27a8..da0e503b23ef 100644 --- a/polly/lib/CodeGen/BlockGenerators.cpp +++ b/polly/lib/CodeGen/BlockGenerators.cpp @@ -1234,7 +1234,7 @@ PHINode *RegionGenerator::buildExitPHI(MemoryAccess *MA, LoopToScevMapT <S, PHINode *NewPHI = PHINode::Create(OrigPHI->getType(), Incoming.size(), "polly." + OrigPHI->getName(), - NewSubregionExit->getFirstNonPHI()); + NewSubregionExit->getFirstNonPHIIt()); // Add the incoming values to the PHI. for (auto &Pair : Incoming) { -- GitLab From f836048a2b452f5f2a8440c9f5945ee1a7bcdac2 Mon Sep 17 00:00:00 2001 From: Nico Weber Date: Tue, 5 Mar 2024 12:02:30 -0500 Subject: [PATCH 170/929] [gn] port 6fd27d5b0321f (no more module.modulemap.in) --- llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn index 5ba0d04b4745..72aa8b4e8e38 100644 --- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn +++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn @@ -58,14 +58,6 @@ if (current_toolchain == default_toolchain) { } } - write_cmake_config("write_modulemap") { - input = "module.modulemap.in" - output = "$libcxx_generated_include_dir/module.modulemap" - - #no = "requires LIBCXX_CONFIGURED_WITHOUT_SUPPORT_FOR_THIS_HEADER" - values = [] - } - # This uses write_cmake_config() instead of a normal copy() rule because # copy() uses `sources`, and if there's more than one `sources` in a file, # sync_source_lists_from_cmake.py can't auto-sync changes. @@ -1042,6 +1034,7 @@ if (current_toolchain == default_toolchain) { "mdspan", "memory", "memory_resource", + "module.modulemap", "mutex", "new", "numbers", @@ -1097,7 +1090,6 @@ if (current_toolchain == default_toolchain) { deps = [ ":write_assertion_handler", ":write_config_site", - ":write_modulemap", ] if (target_os != "mac" && target_os != "win") { # libcxx/cmake/Modules/HandleLibCXXABI.cmake sets -- GitLab From 1b1aea79194117d8f1729ef9c8f80454aea381fe Mon Sep 17 00:00:00 2001 From: Changpeng Fang Date: Tue, 5 Mar 2024 09:16:42 -0800 Subject: [PATCH 171/929] AMDGPU: Make s_wait_samplecnt(_bvhcnt) dependent on hasImageInsts, NFC (#83932) --- llvm/lib/Target/AMDGPU/SOPInstructions.td | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td index b5de311f8c58..ff79538a86ae 100644 --- a/llvm/lib/Target/AMDGPU/SOPInstructions.td +++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td @@ -1598,8 +1598,10 @@ def S_WAITCNT_VSCNT_soft : SOPK_WAITCNT<"s_soft_waitcnt_vscnt">; let SubtargetPredicate = isGFX12Plus in { def S_WAIT_LOADCNT_soft : SOPP_Pseudo <"s_soft_wait_loadcnt", (ins s16imm:$simm16), "$simm16">; def S_WAIT_STORECNT_soft : SOPP_Pseudo <"s_soft_wait_storecnt", (ins s16imm:$simm16), "$simm16">; +let OtherPredicates = [HasImageInsts] in { def S_WAIT_SAMPLECNT_soft : SOPP_Pseudo <"s_soft_wait_samplecnt", (ins s16imm:$simm16), "$simm16">; def S_WAIT_BVHCNT_soft : SOPP_Pseudo <"s_soft_wait_bvhcnt", (ins s16imm:$simm16), "$simm16">; +} // End OtherPredicates = [HasImageInsts]. def S_WAIT_DSCNT_soft : SOPP_Pseudo <"s_soft_wait_dscnt", (ins s16imm:$simm16), "$simm16">; def S_WAIT_KMCNT_soft : SOPP_Pseudo <"s_soft_wait_kmcnt", (ins s16imm:$simm16), "$simm16">; } @@ -1727,12 +1729,14 @@ let SubtargetPredicate = isGFX12Plus, hasSideEffects = 1 in { [(int_amdgcn_s_wait_storecnt timm:$simm16)]>; def S_WAIT_STORECNT_DSCNT : SOPP_Pseudo<"s_wait_storecnt_dscnt", (ins s16imm:$simm16), "$simm16">; +let OtherPredicates = [HasImageInsts] in { def S_WAIT_SAMPLECNT : SOPP_Pseudo<"s_wait_samplecnt", (ins s16imm:$simm16), "$simm16", [(int_amdgcn_s_wait_samplecnt timm:$simm16)]>; def S_WAIT_BVHCNT : SOPP_Pseudo<"s_wait_bvhcnt", (ins s16imm:$simm16), "$simm16", [(int_amdgcn_s_wait_bvhcnt timm:$simm16)]>; +} // End OtherPredicates = [HasImageInsts]. def S_WAIT_EXPCNT : SOPP_Pseudo<"s_wait_expcnt", (ins s16imm:$simm16), "$simm16", [(int_amdgcn_s_wait_expcnt timm:$simm16)]>; -- GitLab From a730ed7c1a4a35f5219df720ffb0ba6122d64fe4 Mon Sep 17 00:00:00 2001 From: Alexey Bataev <5361294+alexey-bataev@users.noreply.github.com> Date: Tue, 5 Mar 2024 12:20:28 -0500 Subject: [PATCH 172/929] [SLP]Improve minbitwidth analysis. This improves overall analysis for minbitwidth in SLP. It allows to analyze the trees with store/insertelement root nodes. Also, instead of using single minbitwidth, detected from the very first analysis stage, it tries to detect the best one for each trunc/ext subtree in the graph and use it for the subtree. Results in better code and less vector register pressure. Metric: size..text Program size..text results results0 diff test-suite :: SingleSource/Benchmarks/Adobe-C++/simple_types_loop_invariant.test 92549.00 92609.00 0.1% test-suite :: External/SPEC/CINT2017speed/625.x264_s/625.x264_s.test 663381.00 663493.00 0.0% test-suite :: External/SPEC/CINT2017rate/525.x264_r/525.x264_r.test 663381.00 663493.00 0.0% test-suite :: MultiSource/Benchmarks/Bullet/bullet.test 307182.00 307214.00 0.0% test-suite :: External/SPEC/CFP2017speed/638.imagick_s/638.imagick_s.test 1394420.00 1394484.00 0.0% test-suite :: External/SPEC/CFP2017rate/538.imagick_r/538.imagick_r.test 1394420.00 1394484.00 0.0% test-suite :: External/SPEC/CFP2017rate/510.parest_r/510.parest_r.test 2040257.00 2040273.00 0.0% test-suite :: External/SPEC/CFP2017rate/526.blender_r/526.blender_r.test 12396098.00 12395858.00 -0.0% test-suite :: External/SPEC/CINT2006/445.gobmk/445.gobmk.test 909944.00 909768.00 -0.0% SingleSource/Benchmarks/Adobe-C++/simple_types_loop_invariant - 4 scalar instructions remain scalar (good). Spec2017/x264 - the whole function idct4x4dc is vectorized using <16 x i16> instead of <16 x i32>, also zext/trunc are removed. In other places last vector zext/sext removed and replaced by extractelement + scalar zext/sext pair. MultiSource/Benchmarks/Bullet/bullet - reduce or <4 x i32> replaced by reduce or <4 x i8> Spec2017/imagick - Removed extra zext from 2 packs of the operations. Spec2017/parest - Removed extra zext, replaced by extractelement+scalar zext Spec2017/blender - the whole bunch of vector zext/sext replaced by extractelement+scalar zext/sext, some extra code vectorized in smaller types. Spec2006/gobmk - fixed cost estimation, some small code remains scalar. Reviewers: RKSimon Reviewed By: RKSimon Pull Request: https://github.com/llvm/llvm-project/pull/78976 --- .../Transforms/Vectorize/SLPVectorizer.cpp | 522 ++++++++++++------ .../SLPVectorizer/AArch64/ext-trunc.ll | 9 +- .../SLPVectorizer/AArch64/getelementptr2.ll | 4 +- .../SLPVectorizer/AArch64/reduce-add-i64.ll | 20 +- .../SLPVectorizer/RISCV/reductions.ll | 7 +- .../Transforms/SLPVectorizer/X86/PR35777.ll | 9 +- .../X86/int-bitcast-minbitwidth.ll | 2 +- ...minbitwidth-multiuse-with-insertelement.ll | 17 +- .../X86/minbitwidth-transformed-operand.ll | 21 +- .../SLPVectorizer/X86/minimum-sizes.ll | 43 +- .../SLPVectorizer/X86/phi-undef-input.ll | 24 +- .../Transforms/SLPVectorizer/X86/resched.ll | 32 +- .../X86/reused-reductions-with-minbitwidth.ll | 10 +- .../X86/store-insertelement-minbitwidth.ll | 22 +- .../SLPVectorizer/alt-cmp-vectorize.ll | 4 +- 15 files changed, 451 insertions(+), 295 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 36dc9094538a..8cf59a18381a 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -1085,6 +1085,7 @@ public: BS->clear(); } MinBWs.clear(); + ReductionBitWidth = 0; InstrElementSize.clear(); UserIgnoreList = nullptr; PostponedGathers.clear(); @@ -2307,9 +2308,11 @@ private: /// constant and to be demoted. Required to correctly identify constant nodes /// to be demoted. bool collectValuesToDemote( - Value *V, SmallVectorImpl &ToDemote, + Value *V, bool IsProfitableToDemoteRoot, unsigned &BitWidth, + SmallVectorImpl &ToDemote, DenseMap> &DemotedConsts, - SmallVectorImpl &Roots, DenseSet &Visited) const; + DenseSet &Visited, unsigned &MaxDepthLevel, + bool &IsProfitableToDemote) const; /// Check if the operands on the edges \p Edges of the \p UserTE allows /// reordering (i.e. the operands can be reordered because they have only one @@ -2375,6 +2378,9 @@ private: /// \ returns the graph entry for the \p Idx operand of the \p E entry. const TreeEntry *getOperandEntry(const TreeEntry *E, unsigned Idx) const; + /// \returns Cast context for the given graph node. + TTI::CastContextHint getCastContextHint(const TreeEntry &TE) const; + /// \returns the cost of the vectorizable entry. InstructionCost getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, @@ -3629,6 +3635,11 @@ private: /// value must be signed-extended, rather than zero-extended, back to its /// original width. DenseMap> MinBWs; + + /// Final size of the reduced vector, if the current graph represents the + /// input for the reduction and it was possible to narrow the size of the + /// reduction. + unsigned ReductionBitWidth = 0; }; } // end namespace slpvectorizer @@ -8362,6 +8373,22 @@ const BoUpSLP::TreeEntry *BoUpSLP::getOperandEntry(const TreeEntry *E, return It->get(); } +TTI::CastContextHint BoUpSLP::getCastContextHint(const TreeEntry &TE) const { + if (TE.State == TreeEntry::ScatterVectorize || + TE.State == TreeEntry::StridedVectorize) + return TTI::CastContextHint::GatherScatter; + if (TE.State == TreeEntry::Vectorize && TE.getOpcode() == Instruction::Load && + !TE.isAltShuffle()) { + if (TE.ReorderIndices.empty()) + return TTI::CastContextHint::Normal; + SmallVector Mask; + inversePermutation(TE.ReorderIndices, Mask); + if (ShuffleVectorInst::isReverseMask(Mask, Mask.size())) + return TTI::CastContextHint::Reversed; + } + return TTI::CastContextHint::None; +} + InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, SmallPtrSetImpl &CheckedExtracts) { @@ -8384,6 +8411,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, // If we have computed a smaller type for the expression, update VecTy so // that the costs will be accurate. auto It = MinBWs.find(E); + Type *OrigScalarTy = ScalarTy; if (It != MinBWs.end()) { ScalarTy = IntegerType::get(F->getContext(), It->second.first); VecTy = FixedVectorType::get(ScalarTy, VL.size()); @@ -8441,24 +8469,11 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, UsedScalars.set(I); } auto GetCastContextHint = [&](Value *V) { - if (const TreeEntry *OpTE = getTreeEntry(V)) { - if (OpTE->State == TreeEntry::ScatterVectorize || - OpTE->State == TreeEntry::StridedVectorize) - return TTI::CastContextHint::GatherScatter; - if (OpTE->State == TreeEntry::Vectorize && - OpTE->getOpcode() == Instruction::Load && !OpTE->isAltShuffle()) { - if (OpTE->ReorderIndices.empty()) - return TTI::CastContextHint::Normal; - SmallVector Mask; - inversePermutation(OpTE->ReorderIndices, Mask); - if (ShuffleVectorInst::isReverseMask(Mask, Mask.size())) - return TTI::CastContextHint::Reversed; - } - } else { - InstructionsState SrcState = getSameOpcode(E->getOperand(0), *TLI); - if (SrcState.getOpcode() == Instruction::Load && !SrcState.isAltShuffle()) - return TTI::CastContextHint::GatherScatter; - } + if (const TreeEntry *OpTE = getTreeEntry(V)) + return getCastContextHint(*OpTE); + InstructionsState SrcState = getSameOpcode(E->getOperand(0), *TLI); + if (SrcState.getOpcode() == Instruction::Load && !SrcState.isAltShuffle()) + return TTI::CastContextHint::GatherScatter; return TTI::CastContextHint::None; }; auto GetCostDiff = @@ -8507,8 +8522,6 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, TTI::CastContextHint CCH = GetCastContextHint(VL0); VecCost += TTI->getCastInstrCost(VecOpcode, UserVecTy, VecTy, CCH, CostKind); - ScalarCost += Sz * TTI->getCastInstrCost(VecOpcode, UserScalarTy, - ScalarTy, CCH, CostKind); } } } @@ -8525,7 +8538,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, InstructionCost ScalarCost = 0; InstructionCost VecCost = 0; std::tie(ScalarCost, VecCost) = getGEPCosts( - *TTI, Ptrs, BasePtr, E->getOpcode(), CostKind, ScalarTy, VecTy); + *TTI, Ptrs, BasePtr, E->getOpcode(), CostKind, OrigScalarTy, VecTy); LLVM_DEBUG(dumpTreeCosts(E, 0, VecCost, ScalarCost, "Calculated GEPs cost for Tree")); @@ -8572,7 +8585,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, NumElts = ATy->getNumElements(); else NumElts = AggregateTy->getStructNumElements(); - SrcVecTy = FixedVectorType::get(ScalarTy, NumElts); + SrcVecTy = FixedVectorType::get(OrigScalarTy, NumElts); } if (I->hasOneUse()) { Instruction *Ext = I->user_back(); @@ -8740,13 +8753,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, } } auto GetScalarCost = [&](unsigned Idx) -> InstructionCost { - // Do not count cost here if minimum bitwidth is in effect and it is just - // a bitcast (here it is just a noop). - if (VecOpcode != Opcode && VecOpcode == Instruction::BitCast) - return TTI::TCC_Free; - auto *VI = VL0->getOpcode() == Opcode - ? cast(UniqueValues[Idx]) - : nullptr; + auto *VI = cast(UniqueValues[Idx]); return TTI->getCastInstrCost(Opcode, VL0->getType(), VL0->getOperand(0)->getType(), TTI::getCastContextHint(VI), CostKind, VI); @@ -8789,7 +8796,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, ? CmpInst::BAD_FCMP_PREDICATE : CmpInst::BAD_ICMP_PREDICATE; - return TTI->getCmpSelInstrCost(E->getOpcode(), ScalarTy, + return TTI->getCmpSelInstrCost(E->getOpcode(), OrigScalarTy, Builder.getInt1Ty(), CurrentPred, CostKind, VI); }; @@ -8844,7 +8851,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(VI->getOperand(OpIdx)); SmallVector Operands(VI->operand_values()); - return TTI->getArithmeticInstrCost(ShuffleOrOp, ScalarTy, CostKind, + return TTI->getArithmeticInstrCost(ShuffleOrOp, OrigScalarTy, CostKind, Op1Info, Op2Info, Operands, VI); }; auto GetVectorCost = [=](InstructionCost CommonCost) { @@ -8863,9 +8870,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, case Instruction::Load: { auto GetScalarCost = [&](unsigned Idx) { auto *VI = cast(UniqueValues[Idx]); - return TTI->getMemoryOpCost(Instruction::Load, ScalarTy, VI->getAlign(), - VI->getPointerAddressSpace(), CostKind, - TTI::OperandValueInfo(), VI); + return TTI->getMemoryOpCost(Instruction::Load, OrigScalarTy, + VI->getAlign(), VI->getPointerAddressSpace(), + CostKind, TTI::OperandValueInfo(), VI); }; auto *LI0 = cast(VL0); auto GetVectorCost = [&](InstructionCost CommonCost) { @@ -8908,9 +8915,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, auto GetScalarCost = [=](unsigned Idx) { auto *VI = cast(VL[Idx]); TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(VI->getValueOperand()); - return TTI->getMemoryOpCost(Instruction::Store, ScalarTy, VI->getAlign(), - VI->getPointerAddressSpace(), CostKind, - OpInfo, VI); + return TTI->getMemoryOpCost(Instruction::Store, OrigScalarTy, + VI->getAlign(), VI->getPointerAddressSpace(), + CostKind, OpInfo, VI); }; auto *BaseSI = cast(IsReorder ? VL[E->ReorderIndices.front()] : VL0); @@ -9772,6 +9779,44 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef VectorizedVals) { Cost -= InsertCost; } + // Add the cost for reduced value resize (if required). + if (ReductionBitWidth != 0) { + assert(UserIgnoreList && "Expected reduction tree."); + const TreeEntry &E = *VectorizableTree.front().get(); + auto It = MinBWs.find(&E); + if (It != MinBWs.end() && It->second.first != ReductionBitWidth) { + unsigned SrcSize = It->second.first; + unsigned DstSize = ReductionBitWidth; + unsigned Opcode = Instruction::Trunc; + if (SrcSize < DstSize) + Opcode = It->second.second ? Instruction::SExt : Instruction::ZExt; + auto *SrcVecTy = + FixedVectorType::get(Builder.getIntNTy(SrcSize), E.getVectorFactor()); + auto *DstVecTy = + FixedVectorType::get(Builder.getIntNTy(DstSize), E.getVectorFactor()); + TTI::CastContextHint CCH = getCastContextHint(E); + InstructionCost CastCost; + switch (E.getOpcode()) { + case Instruction::SExt: + case Instruction::ZExt: + case Instruction::Trunc: { + const TreeEntry *OpTE = getOperandEntry(&E, 0); + CCH = getCastContextHint(*OpTE); + break; + } + default: + break; + } + CastCost += TTI->getCastInstrCost(Opcode, DstVecTy, SrcVecTy, CCH, + TTI::TCK_RecipThroughput); + Cost += CastCost; + LLVM_DEBUG(dbgs() << "SLP: Adding cost " << CastCost + << " for final resize for reduction from " << SrcVecTy + << " to " << DstVecTy << "\n"; + dbgs() << "SLP: Current total cost = " << Cost << "\n"); + } + } + #ifndef NDEBUG SmallString<256> Str; { @@ -12929,7 +12974,21 @@ Value *BoUpSLP::vectorizeTree( Builder.ClearInsertionPoint(); InstrElementSize.clear(); - return VectorizableTree[0]->VectorizedValue; + const TreeEntry &RootTE = *VectorizableTree.front().get(); + Value *Vec = RootTE.VectorizedValue; + if (auto It = MinBWs.find(&RootTE); ReductionBitWidth != 0 && + It != MinBWs.end() && + ReductionBitWidth != It->second.first) { + IRBuilder<>::InsertPointGuard Guard(Builder); + Builder.SetInsertPoint(ReductionRoot->getParent(), + ReductionRoot->getIterator()); + Vec = Builder.CreateIntCast( + Vec, + VectorType::get(Builder.getIntNTy(ReductionBitWidth), + cast(Vec->getType())->getElementCount()), + It->second.second); + } + return Vec; } void BoUpSLP::optimizeGatherSequence() { @@ -13749,16 +13808,21 @@ unsigned BoUpSLP::getVectorElementSize(Value *V) { // smaller type with a truncation. We collect the values that will be demoted // in ToDemote and additional roots that require investigating in Roots. bool BoUpSLP::collectValuesToDemote( - Value *V, SmallVectorImpl &ToDemote, + Value *V, bool IsProfitableToDemoteRoot, unsigned &BitWidth, + SmallVectorImpl &ToDemote, DenseMap> &DemotedConsts, - SmallVectorImpl &Roots, DenseSet &Visited) const { + DenseSet &Visited, unsigned &MaxDepthLevel, + bool &IsProfitableToDemote) const { // We can always demote constants. - if (isa(V)) + if (isa(V)) { + MaxDepthLevel = 1; return true; + } // If the value is not a vectorized instruction in the expression and not used // by the insertelement instruction and not used in multiple vector nodes, it // cannot be demoted. + // TODO: improve handling of gathered values and others. auto *I = dyn_cast(V); if (!I || !getTreeEntry(I) || MultiNodeScalars.contains(I) || !Visited.insert(I).second || all_of(I->users(), [&](User *U) { @@ -13766,6 +13830,20 @@ bool BoUpSLP::collectValuesToDemote( })) return false; + auto IsPotentiallyTruncated = [&](Value *V, unsigned &BitWidth) -> bool { + if (MultiNodeScalars.contains(V)) + return false; + uint32_t OrigBitWidth = DL->getTypeSizeInBits(V->getType()); + APInt Mask = APInt::getBitsSetFrom(OrigBitWidth, BitWidth); + if (MaskedValueIsZero(V, Mask, SimplifyQuery(*DL))) + return true; + auto NumSignBits = ComputeNumSignBits(V, *DL, 0, AC, nullptr, DT); + unsigned BitWidth1 = OrigBitWidth - NumSignBits; + if (!isKnownNonNegative(V, SimplifyQuery(*DL))) + ++BitWidth1; + BitWidth = std::max(BitWidth, BitWidth1); + return BitWidth > 0 && OrigBitWidth >= (BitWidth * 2); + }; unsigned Start = 0; unsigned End = I->getNumOperands(); switch (I->getOpcode()) { @@ -13773,12 +13851,14 @@ bool BoUpSLP::collectValuesToDemote( // We can always demote truncations and extensions. Since truncations can // seed additional demotion, we save the truncated value. case Instruction::Trunc: - Roots.push_back(I->getOperand(0)); + MaxDepthLevel = 1; + if (IsProfitableToDemoteRoot) + IsProfitableToDemote = true; break; case Instruction::ZExt: case Instruction::SExt: - if (isa(I->getOperand(0))) - return false; + MaxDepthLevel = 1; + IsProfitableToDemote = true; break; // We can demote certain binary operations if we can demote both of their @@ -13788,23 +13868,32 @@ bool BoUpSLP::collectValuesToDemote( case Instruction::Mul: case Instruction::And: case Instruction::Or: - case Instruction::Xor: - if (!collectValuesToDemote(I->getOperand(0), ToDemote, DemotedConsts, Roots, - Visited) || - !collectValuesToDemote(I->getOperand(1), ToDemote, DemotedConsts, Roots, - Visited)) + case Instruction::Xor: { + unsigned Level1, Level2; + if (!collectValuesToDemote(I->getOperand(0), IsProfitableToDemoteRoot, + BitWidth, ToDemote, DemotedConsts, Visited, + Level1, IsProfitableToDemote) || + !collectValuesToDemote(I->getOperand(1), IsProfitableToDemoteRoot, + BitWidth, ToDemote, DemotedConsts, Visited, + Level2, IsProfitableToDemote)) return false; + MaxDepthLevel = std::max(Level1, Level2); break; + } // We can demote selects if we can demote their true and false values. case Instruction::Select: { Start = 1; + unsigned Level1, Level2; SelectInst *SI = cast(I); - if (!collectValuesToDemote(SI->getTrueValue(), ToDemote, DemotedConsts, - Roots, Visited) || - !collectValuesToDemote(SI->getFalseValue(), ToDemote, DemotedConsts, - Roots, Visited)) + if (!collectValuesToDemote(SI->getTrueValue(), IsProfitableToDemoteRoot, + BitWidth, ToDemote, DemotedConsts, Visited, + Level1, IsProfitableToDemote) || + !collectValuesToDemote(SI->getFalseValue(), IsProfitableToDemoteRoot, + BitWidth, ToDemote, DemotedConsts, Visited, + Level2, IsProfitableToDemote)) return false; + MaxDepthLevel = std::max(Level1, Level2); break; } @@ -13813,171 +13902,236 @@ bool BoUpSLP::collectValuesToDemote( case Instruction::PHI: { PHINode *PN = cast(I); for (Value *IncValue : PN->incoming_values()) - if (!collectValuesToDemote(IncValue, ToDemote, DemotedConsts, Roots, - Visited)) + if (!collectValuesToDemote(IncValue, IsProfitableToDemoteRoot, BitWidth, + ToDemote, DemotedConsts, Visited, + MaxDepthLevel, IsProfitableToDemote)) return false; break; } // Otherwise, conservatively give up. default: - return false; + if (!IsPotentiallyTruncated(I, BitWidth)) + return false; + MaxDepthLevel = 0; + Start = End = 0; + break; } + ++MaxDepthLevel; // Gather demoted constant operands. for (unsigned Idx : seq(Start, End)) if (isa(I->getOperand(Idx))) DemotedConsts.try_emplace(I).first->getSecond().push_back(Idx); // Record the value that we can demote. ToDemote.push_back(V); - return true; + return IsProfitableToDemote; } void BoUpSLP::computeMinimumValueSizes() { // We only attempt to truncate integer expressions. - auto &TreeRoot = VectorizableTree[0]->Scalars; - auto *TreeRootIT = dyn_cast(TreeRoot[0]->getType()); - if (!TreeRootIT || VectorizableTree.front()->State == TreeEntry::NeedToGather) - return; + bool IsStoreOrInsertElt = + VectorizableTree.front()->getOpcode() == Instruction::Store || + VectorizableTree.front()->getOpcode() == Instruction::InsertElement; + unsigned NodeIdx = 0; + if (IsStoreOrInsertElt && + VectorizableTree.front()->State != TreeEntry::NeedToGather) + NodeIdx = 1; // Ensure the roots of the vectorizable tree don't form a cycle. - if (!VectorizableTree.front()->UserTreeIndices.empty()) + if (VectorizableTree[NodeIdx]->State == TreeEntry::NeedToGather || + (NodeIdx == 0 && !VectorizableTree[NodeIdx]->UserTreeIndices.empty()) || + (NodeIdx != 0 && any_of(VectorizableTree[NodeIdx]->UserTreeIndices, + [NodeIdx](const EdgeInfo &EI) { + return EI.UserTE->Idx > + static_cast(NodeIdx); + }))) return; - // Conservatively determine if we can actually truncate the roots of the - // expression. Collect the values that can be demoted in ToDemote and - // additional roots that require investigating in Roots. - SmallVector ToDemote; - DenseMap> DemotedConsts; - SmallVector Roots; - for (auto *Root : TreeRoot) { - DenseSet Visited; - if (!collectValuesToDemote(Root, ToDemote, DemotedConsts, Roots, Visited)) - return; + // The first value node for store/insertelement is sext/zext/trunc? Skip it, + // resize to the final type. + bool IsProfitableToDemoteRoot = !IsStoreOrInsertElt; + if (NodeIdx != 0 && + VectorizableTree[NodeIdx]->State == TreeEntry::Vectorize && + (VectorizableTree[NodeIdx]->getOpcode() == Instruction::ZExt || + VectorizableTree[NodeIdx]->getOpcode() == Instruction::SExt || + VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc)) { + assert(IsStoreOrInsertElt && "Expected store/insertelement seeded graph."); + ++NodeIdx; + IsProfitableToDemoteRoot = true; } - // The maximum bit width required to represent all the values that can be - // demoted without loss of precision. It would be safe to truncate the roots - // of the expression to this width. - auto MaxBitWidth = 1u; - - // We first check if all the bits of the roots are demanded. If they're not, - // we can truncate the roots to this narrower type. - for (auto *Root : TreeRoot) { - auto Mask = DB->getDemandedBits(cast(Root)); - MaxBitWidth = std::max(Mask.getBitWidth() - Mask.countl_zero(), - MaxBitWidth); - } - - // True if the roots can be zero-extended back to their original type, rather - // than sign-extended. We know that if the leading bits are not demanded, we - // can safely zero-extend. So we initialize IsKnownPositive to True. - bool IsKnownPositive = true; - - // If all the bits of the roots are demanded, we can try a little harder to - // compute a narrower type. This can happen, for example, if the roots are - // getelementptr indices. InstCombine promotes these indices to the pointer - // width. Thus, all their bits are technically demanded even though the - // address computation might be vectorized in a smaller type. - // - // We start by looking at each entry that can be demoted. We compute the - // maximum bit width required to store the scalar by using ValueTracking to - // compute the number of high-order bits we can truncate. - if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType()) && - all_of(TreeRoot, [](Value *V) { - return all_of(V->users(), - [](User *U) { return isa(U); }); - })) { - MaxBitWidth = 8u; - + SmallVector ToDemote; + DenseMap> DemotedConsts; + auto ComputeMaxBitWidth = [&](ArrayRef TreeRoot, unsigned VF, + bool IsTopRoot, bool IsProfitableToDemoteRoot, + unsigned Opcode, unsigned Limit) { + ToDemote.clear(); + auto *TreeRootIT = dyn_cast(TreeRoot[0]->getType()); + if (!TreeRootIT || !Opcode) + return 0u; + + unsigned NumParts = TTI->getNumberOfParts( + FixedVectorType::get(TreeRoot.front()->getType(), VF)); + + // The maximum bit width required to represent all the values that can be + // demoted without loss of precision. It would be safe to truncate the roots + // of the expression to this width. + unsigned MaxBitWidth = 1u; + + // True if the roots can be zero-extended back to their original type, + // rather than sign-extended. We know that if the leading bits are not + // demanded, we can safely zero-extend. So we initialize IsKnownPositive to + // True. // Determine if the sign bit of all the roots is known to be zero. If not, // IsKnownPositive is set to False. - IsKnownPositive = llvm::all_of(TreeRoot, [&](Value *R) { + bool IsKnownPositive = all_of(TreeRoot, [&](Value *R) { KnownBits Known = computeKnownBits(R, *DL); return Known.isNonNegative(); }); - // Determine the maximum number of bits required to store the scalar - // values. - for (auto *Scalar : ToDemote) { - auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, nullptr, DT); - auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType()); - MaxBitWidth = std::max(NumTypeBits - NumSignBits, MaxBitWidth); - } - - // If we can't prove that the sign bit is zero, we must add one to the - // maximum bit width to account for the unknown sign bit. This preserves - // the existing sign bit so we can safely sign-extend the root back to the - // original type. Otherwise, if we know the sign bit is zero, we will - // zero-extend the root instead. - // - // FIXME: This is somewhat suboptimal, as there will be cases where adding - // one to the maximum bit width will yield a larger-than-necessary - // type. In general, we need to add an extra bit only if we can't - // prove that the upper bit of the original type is equal to the - // upper bit of the proposed smaller type. If these two bits are the - // same (either zero or one) we know that sign-extending from the - // smaller type will result in the same value. Here, since we can't - // yet prove this, we are just making the proposed smaller type - // larger to ensure correctness. - if (!IsKnownPositive) - ++MaxBitWidth; - } - - // Round MaxBitWidth up to the next power-of-two. - MaxBitWidth = llvm::bit_ceil(MaxBitWidth); - - // If the maximum bit width we compute is less than the with of the roots' - // type, we can proceed with the narrowing. Otherwise, do nothing. - if (MaxBitWidth >= TreeRootIT->getBitWidth()) - return; + // We first check if all the bits of the roots are demanded. If they're not, + // we can truncate the roots to this narrower type. + for (auto *Root : TreeRoot) { + unsigned NumSignBits = ComputeNumSignBits(Root, *DL, 0, AC, nullptr, DT); + TypeSize NumTypeBits = DL->getTypeSizeInBits(Root->getType()); + unsigned BitWidth1 = NumTypeBits - NumSignBits; + // If we can't prove that the sign bit is zero, we must add one to the + // maximum bit width to account for the unknown sign bit. This preserves + // the existing sign bit so we can safely sign-extend the root back to the + // original type. Otherwise, if we know the sign bit is zero, we will + // zero-extend the root instead. + // + // FIXME: This is somewhat suboptimal, as there will be cases where adding + // one to the maximum bit width will yield a larger-than-necessary + // type. In general, we need to add an extra bit only if we can't + // prove that the upper bit of the original type is equal to the + // upper bit of the proposed smaller type. If these two bits are + // the same (either zero or one) we know that sign-extending from + // the smaller type will result in the same value. Here, since we + // can't yet prove this, we are just making the proposed smaller + // type larger to ensure correctness. + if (!IsKnownPositive) + ++BitWidth1; + + APInt Mask = DB->getDemandedBits(cast(Root)); + unsigned BitWidth2 = Mask.getBitWidth() - Mask.countl_zero(); + MaxBitWidth = + std::max(std::min(BitWidth1, BitWidth2), MaxBitWidth); + } + + if (MaxBitWidth < 8 && MaxBitWidth > 1) + MaxBitWidth = 8; + + // If the original type is large, but reduced type does not improve the reg + // use - ignore it. + if (NumParts > 1 && + NumParts == + TTI->getNumberOfParts(FixedVectorType::get( + IntegerType::get(F->getContext(), bit_ceil(MaxBitWidth)), VF))) + return 0u; + + bool IsProfitableToDemote = Opcode == Instruction::Trunc || + Opcode == Instruction::SExt || + Opcode == Instruction::ZExt || NumParts > 1; + // Conservatively determine if we can actually truncate the roots of the + // expression. Collect the values that can be demoted in ToDemote and + // additional roots that require investigating in Roots. + for (auto *Root : TreeRoot) { + DenseSet Visited; + unsigned MaxDepthLevel; + bool NeedToDemote = IsProfitableToDemote; + if (!collectValuesToDemote(Root, IsProfitableToDemoteRoot, MaxBitWidth, + ToDemote, DemotedConsts, Visited, + MaxDepthLevel, NeedToDemote) || + (MaxDepthLevel <= Limit && Opcode != Instruction::Trunc && + Opcode != Instruction::SExt && Opcode != Instruction::ZExt)) + return 0u; + } + // Round MaxBitWidth up to the next power-of-two. + MaxBitWidth = bit_ceil(MaxBitWidth); + + return MaxBitWidth; + }; // If we can truncate the root, we must collect additional values that might // be demoted as a result. That is, those seeded by truncations we will // modify. - while (!Roots.empty()) { - DenseSet Visited; - collectValuesToDemote(Roots.pop_back_val(), ToDemote, DemotedConsts, Roots, - Visited); - } - - // Check that all users are marked for demotion. - DenseSet Demoted(ToDemote.begin(), ToDemote.end()); - DenseSet Visited; - for (Value *V: ToDemote) { - const TreeEntry *TE = getTreeEntry(V); - assert(TE && "Expected vectorized scalar."); - if (!Visited.insert(TE).second) - continue; - if (!all_of(TE->UserTreeIndices, [&](const EdgeInfo &EI) { - return all_of(EI.UserTE->Scalars, - [&](Value *V) { return Demoted.contains(V); }); - })) - return; - } - // Finally, map the values we can demote to the maximum bit with we computed. - for (auto *Scalar : ToDemote) { - auto *TE = getTreeEntry(Scalar); - assert(TE && "Expected vectorized scalar."); - if (MinBWs.contains(TE)) + // Add reduction ops sizes, if any. + if (UserIgnoreList && + isa(VectorizableTree.front()->Scalars.front()->getType())) { + for (Value *V : *UserIgnoreList) { + auto NumSignBits = ComputeNumSignBits(V, *DL, 0, AC, nullptr, DT); + auto NumTypeBits = DL->getTypeSizeInBits(V->getType()); + unsigned BitWidth1 = NumTypeBits - NumSignBits; + if (!isKnownNonNegative(V, SimplifyQuery(*DL))) + ++BitWidth1; + auto Mask = DB->getDemandedBits(cast(V)); + unsigned BitWidth2 = Mask.getBitWidth() - Mask.countl_zero(); + ReductionBitWidth = + std::max(std::min(BitWidth1, BitWidth2), ReductionBitWidth); + } + if (ReductionBitWidth < 8 && ReductionBitWidth > 1) + ReductionBitWidth = 8; + + ReductionBitWidth = bit_ceil(ReductionBitWidth); + } + bool IsTopRoot = NodeIdx == 0; + while (NodeIdx < VectorizableTree.size()) { + ArrayRef TreeRoot = VectorizableTree[NodeIdx]->Scalars; + unsigned Limit = 2; + if (NodeIdx == 0 && + ReductionBitWidth == DL->getTypeSizeInBits(TreeRoot.front()->getType())) + Limit = 3; + unsigned MaxBitWidth = ComputeMaxBitWidth( + TreeRoot, VectorizableTree[NodeIdx]->getVectorFactor(), IsTopRoot, + IsProfitableToDemoteRoot, VectorizableTree[NodeIdx]->getOpcode(), + Limit); + IsTopRoot = false; + IsProfitableToDemoteRoot = true; + + ++NodeIdx; + for (unsigned E = VectorizableTree.size(); NodeIdx < E; ++NodeIdx) { + if (VectorizableTree[NodeIdx]->State != TreeEntry::NeedToGather && + !VectorizableTree[NodeIdx]->isAltShuffle() && + VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc) { + ++NodeIdx; + break; + } + } + + // If the maximum bit width we compute is less than the with of the roots' + // type, we can proceed with the narrowing. Otherwise, do nothing. + if (MaxBitWidth == 0 || + MaxBitWidth >= + cast(TreeRoot.front()->getType())->getBitWidth()) continue; - bool IsSigned = any_of(TE->Scalars, [&](Value *R) { - KnownBits Known = computeKnownBits(R, *DL); - return !Known.isNonNegative(); - }); - MinBWs.try_emplace(TE, MaxBitWidth, IsSigned); - const auto *I = cast(Scalar); - auto DCIt = DemotedConsts.find(I); - if (DCIt != DemotedConsts.end()) { - for (unsigned Idx : DCIt->getSecond()) { - // Check that all instructions operands are demoted. - if (all_of(TE->Scalars, [&](Value *V) { - auto SIt = DemotedConsts.find(cast(V)); - return SIt != DemotedConsts.end() && - is_contained(SIt->getSecond(), Idx); - })) { + + // Finally, map the values we can demote to the maximum bit with we + // computed. + for (Value *Scalar : ToDemote) { + TreeEntry *TE = getTreeEntry(Scalar); + assert(TE && "Expected vectorized scalar."); + if (MinBWs.contains(TE)) + continue; + bool IsSigned = any_of(TE->Scalars, [&](Value *R) { + return !isKnownNonNegative(R, SimplifyQuery(*DL)); + }); + MinBWs.try_emplace(TE, MaxBitWidth, IsSigned); + const auto *I = cast(Scalar); + auto DCIt = DemotedConsts.find(I); + if (DCIt != DemotedConsts.end()) { + for (unsigned Idx : DCIt->getSecond()) { + // Check that all instructions operands are demoted. const TreeEntry *CTE = getOperandEntry(TE, Idx); - MinBWs.try_emplace(CTE, MaxBitWidth, IsSigned); + if (all_of(TE->Scalars, + [&](Value *V) { + auto SIt = DemotedConsts.find(cast(V)); + return SIt != DemotedConsts.end() && + is_contained(SIt->getSecond(), Idx); + }) || + all_of(CTE->Scalars, Constant::classof)) + MinBWs.try_emplace(CTE, MaxBitWidth, IsSigned); } } } diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll index cef791633655..5e3fd156666f 100644 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll @@ -17,12 +17,13 @@ define void @test1(<4 x i16> %a, <4 x i16> %b, ptr %p) { ; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i64 [[S0]] ; CHECK-NEXT: [[LOAD0:%.*]] = load i64, ptr [[GEP0]], align 4 ; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <4 x i32> [[SUB0]], <4 x i32> poison, <2 x i32> -; CHECK-NEXT: [[TMP1:%.*]] = sext <2 x i32> [[TMP0]] to <2 x i64> -; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0 +; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i32> [[TMP0]], i32 0 +; CHECK-NEXT: [[TMP2:%.*]] = sext i32 [[TMP1]] to i64 ; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP2]] ; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[GEP1]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1 -; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP3]] +; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP0]], i32 1 +; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[TMP3]] to i64 +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP4]] ; CHECK-NEXT: [[LOAD2:%.*]] = load i64, ptr [[GEP2]], align 4 ; CHECK-NEXT: [[E3:%.*]] = extractelement <4 x i32> [[SUB0]], i32 3 ; CHECK-NEXT: [[S3:%.*]] = sext i32 [[E3]] to i64 diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll index 47485e514ec2..1cce52060c47 100644 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ;test_i16_extend NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer,dce,instcombine -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s +; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer,dce,instcombine -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s ; RUN: cat %t | FileCheck -check-prefix=YAML %s -; RUN: opt -S -mtriple=aarch64--linux-gnu -passes='slp-vectorizer,dce,instcombine' -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s +; RUN: opt -S -mtriple=aarch64--linux-gnu -passes='slp-vectorizer,dce,instcombine' -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s ; RUN: cat %t | FileCheck -check-prefix=YAML %s diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll index d67fdc1cd6aa..a7a7f642ced5 100644 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll @@ -28,21 +28,11 @@ entry: define i64 @red_zext_ld_4xi64(ptr %ptr) { ; CHECK-LABEL: @red_zext_ld_4xi64( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[LD0:%.*]] = load i8, ptr [[PTR:%.*]], align 1 -; CHECK-NEXT: [[ZEXT:%.*]] = zext i8 [[LD0]] to i64 -; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 1 -; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[GEP]], align 1 -; CHECK-NEXT: [[ZEXT_1:%.*]] = zext i8 [[LD1]] to i64 -; CHECK-NEXT: [[ADD_1:%.*]] = add nuw nsw i64 [[ZEXT]], [[ZEXT_1]] -; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 2 -; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[GEP_1]], align 1 -; CHECK-NEXT: [[ZEXT_2:%.*]] = zext i8 [[LD2]] to i64 -; CHECK-NEXT: [[ADD_2:%.*]] = add nuw nsw i64 [[ADD_1]], [[ZEXT_2]] -; CHECK-NEXT: [[GEP_2:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 3 -; CHECK-NEXT: [[LD3:%.*]] = load i8, ptr [[GEP_2]], align 1 -; CHECK-NEXT: [[ZEXT_3:%.*]] = zext i8 [[LD3]] to i64 -; CHECK-NEXT: [[ADD_3:%.*]] = add nuw nsw i64 [[ADD_2]], [[ZEXT_3]] -; CHECK-NEXT: ret i64 [[ADD_3]] +; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[PTR:%.*]], align 1 +; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i16> +; CHECK-NEXT: [[TMP2:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]]) +; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[TMP2]] to i64 +; CHECK-NEXT: ret i64 [[TMP3]] ; entry: %ld0 = load i8, ptr %ptr diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll index 000e7a56df37..500f10659f04 100644 --- a/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll @@ -802,9 +802,10 @@ define i64 @red_zext_ld_4xi64(ptr %ptr) { ; CHECK-LABEL: @red_zext_ld_4xi64( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[PTR:%.*]], align 1 -; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i64> -; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP1]]) -; CHECK-NEXT: ret i64 [[TMP2]] +; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i16> +; CHECK-NEXT: [[TMP2:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]]) +; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[TMP2]] to i64 +; CHECK-NEXT: ret i64 [[TMP3]] ; entry: %ld0 = load i8, ptr %ptr diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll index 4565d4928ba4..05511f843a68 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll @@ -15,11 +15,12 @@ define { i64, i64 } @patatino(double %arg) { ; CHECK-NEXT: [[TMP6:%.*]] = load <2 x double>, ptr getelementptr inbounds ([6 x double], ptr @global, i64 0, i64 4), align 16 ; CHECK-NEXT: [[TMP7:%.*]] = fadd <2 x double> [[TMP6]], [[TMP5]] ; CHECK-NEXT: [[TMP8:%.*]] = fptosi <2 x double> [[TMP7]] to <2 x i32> -; CHECK-NEXT: [[TMP9:%.*]] = sext <2 x i32> [[TMP8]] to <2 x i64> -; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i64> [[TMP9]], i32 0 +; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[TMP8]], i32 0 +; CHECK-NEXT: [[TMP10:%.*]] = sext i32 [[TMP9]] to i64 ; CHECK-NEXT: [[T16:%.*]] = insertvalue { i64, i64 } undef, i64 [[TMP10]], 0 -; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i64> [[TMP9]], i32 1 -; CHECK-NEXT: [[T17:%.*]] = insertvalue { i64, i64 } [[T16]], i64 [[TMP11]], 1 +; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[TMP8]], i32 1 +; CHECK-NEXT: [[TMP12:%.*]] = sext i32 [[TMP11]] to i64 +; CHECK-NEXT: [[T17:%.*]] = insertvalue { i64, i64 } [[T16]], i64 [[TMP12]], 1 ; CHECK-NEXT: ret { i64, i64 } [[T17]] ; bb: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll index a0af8e36b36c..5ee801607653 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 -; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-3 < %s | FileCheck %s +; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-6 < %s | FileCheck %s define void @t(i64 %v) { ; CHECK-LABEL: define void @t( diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll index 6e512fcbb739..6051638562b5 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll @@ -6,18 +6,17 @@ define void @test(i8 %0) { ; CHECK-SAME: i8 [[TMP0:%.*]]) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> , i8 [[TMP0]], i32 1 -; CHECK-NEXT: [[TMP2:%.*]] = sext <2 x i8> [[TMP1]] to <2 x i16> -; CHECK-NEXT: [[TMP3:%.*]] = sext <2 x i16> [[TMP2]] to <2 x i32> -; CHECK-NEXT: [[TMP4:%.*]] = mul <2 x i16> [[TMP2]], zeroinitializer -; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i16> [[TMP4]], i32 0 -; CHECK-NEXT: [[TMP6:%.*]] = zext i16 [[TMP5]] to i32 -; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i16> [[TMP4]], i32 1 -; CHECK-NEXT: [[TMP8:%.*]] = zext i16 [[TMP7]] to i32 -; CHECK-NEXT: [[ADD:%.*]] = or i32 [[TMP6]], [[TMP8]] +; CHECK-NEXT: [[TMP2:%.*]] = sext <2 x i8> [[TMP1]] to <2 x i32> +; CHECK-NEXT: [[TMP3:%.*]] = mul <2 x i8> [[TMP1]], zeroinitializer +; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[TMP3]], i32 0 +; CHECK-NEXT: [[TMP5:%.*]] = zext i8 [[TMP4]] to i32 +; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i8> [[TMP3]], i32 1 +; CHECK-NEXT: [[TMP7:%.*]] = zext i8 [[TMP6]] to i32 +; CHECK-NEXT: [[ADD:%.*]] = or i32 [[TMP5]], [[TMP7]] ; CHECK-NEXT: [[SHR:%.*]] = lshr i32 [[ADD]], 1 ; CHECK-NEXT: [[CONV9:%.*]] = trunc i32 [[SHR]] to i8 ; CHECK-NEXT: store i8 [[CONV9]], ptr null, align 1 -; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <8 x i32> +; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> ; CHECK-NEXT: ret void ; entry: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll index 2c834616becc..4acd63078b82 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll @@ -6,15 +6,20 @@ define void @test(i64 %d.promoted.i) { ; CHECK-SAME: i64 [[D_PROMOTED_I:%.*]]) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[AND_1_I:%.*]] = and i64 0, [[D_PROMOTED_I]] +; CHECK-NEXT: [[TMP0:%.*]] = insertelement <8 x i64> , i64 [[AND_1_I]], i32 1 +; CHECK-NEXT: [[TMP1:%.*]] = trunc <8 x i64> [[TMP0]] to <8 x i1> +; CHECK-NEXT: [[TMP2:%.*]] = mul <8 x i1> [[TMP1]], zeroinitializer ; CHECK-NEXT: [[AND_1_I_1:%.*]] = and i64 0, 0 -; CHECK-NEXT: [[TMP0:%.*]] = insertelement <16 x i64> , i64 [[AND_1_I_1]], i32 1 -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <16 x i64> [[TMP0]], i64 [[AND_1_I]], i32 9 -; CHECK-NEXT: [[TMP2:%.*]] = trunc <16 x i64> [[TMP1]] to <16 x i1> -; CHECK-NEXT: [[TMP3:%.*]] = mul <16 x i1> [[TMP2]], zeroinitializer -; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP3]]) -; CHECK-NEXT: [[TMP5:%.*]] = zext i1 [[TMP4]] to i32 -; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[TMP5]], 0 -; CHECK-NEXT: store i32 [[TMP6]], ptr null, align 4 +; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i64> , i64 [[AND_1_I_1]], i32 1 +; CHECK-NEXT: [[TMP4:%.*]] = trunc <8 x i64> [[TMP3]] to <8 x i1> +; CHECK-NEXT: [[TMP5:%.*]] = mul <8 x i1> [[TMP4]], zeroinitializer +; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP5]]) +; CHECK-NEXT: [[TMP7:%.*]] = zext i1 [[TMP6]] to i32 +; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP2]]) +; CHECK-NEXT: [[TMP9:%.*]] = zext i1 [[TMP8]] to i32 +; CHECK-NEXT: [[OP_RDX:%.*]] = or i32 [[TMP7]], [[TMP9]] +; CHECK-NEXT: [[TMP10:%.*]] = and i32 [[OP_RDX]], 0 +; CHECK-NEXT: store i32 [[TMP10]], ptr null, align 4 ; CHECK-NEXT: ret void ; entry: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll index 651631de2c35..a316415dcc6b 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll @@ -17,12 +17,15 @@ target triple = "x86_64-unknown-linux-gnu" define i8 @PR31243_zext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) { ; SSE-LABEL: @PR31243_zext( ; SSE-NEXT: entry: -; SSE-NEXT: [[TMP0:%.*]] = or i8 [[V0:%.*]], 1 -; SSE-NEXT: [[TMP1:%.*]] = or i8 [[V1:%.*]], 1 -; SSE-NEXT: [[TMP2:%.*]] = zext i8 [[TMP0]] to i64 -; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP2]] -; SSE-NEXT: [[TMP3:%.*]] = zext i8 [[TMP1]] to i64 -; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP3]] +; SSE-NEXT: [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0 +; SSE-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1 +; SSE-NEXT: [[TMP2:%.*]] = or <2 x i8> [[TMP1]], +; SSE-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0 +; SSE-NEXT: [[TMP4:%.*]] = zext i8 [[TMP3]] to i64 +; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]] +; SSE-NEXT: [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1 +; SSE-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i64 +; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]] ; SSE-NEXT: [[T6:%.*]] = load i8, ptr [[T4]], align 1 ; SSE-NEXT: [[T7:%.*]] = load i8, ptr [[T5]], align 1 ; SSE-NEXT: [[T8:%.*]] = add i8 [[T6]], [[T7]] @@ -73,12 +76,15 @@ entry: define i8 @PR31243_sext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) { ; SSE-LABEL: @PR31243_sext( ; SSE-NEXT: entry: -; SSE-NEXT: [[TMP0:%.*]] = or i8 [[V0:%.*]], 1 -; SSE-NEXT: [[TMP1:%.*]] = or i8 [[V1:%.*]], 1 -; SSE-NEXT: [[TMP2:%.*]] = sext i8 [[TMP0]] to i64 -; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP2]] -; SSE-NEXT: [[TMP3:%.*]] = sext i8 [[TMP1]] to i64 -; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP3]] +; SSE-NEXT: [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0 +; SSE-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1 +; SSE-NEXT: [[TMP2:%.*]] = or <2 x i8> [[TMP1]], +; SSE-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0 +; SSE-NEXT: [[TMP4:%.*]] = sext i8 [[TMP3]] to i64 +; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]] +; SSE-NEXT: [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1 +; SSE-NEXT: [[TMP6:%.*]] = sext i8 [[TMP5]] to i64 +; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]] ; SSE-NEXT: [[T6:%.*]] = load i8, ptr [[T4]], align 1 ; SSE-NEXT: [[T7:%.*]] = load i8, ptr [[T5]], align 1 ; SSE-NEXT: [[T8:%.*]] = add i8 [[T6]], [[T7]] @@ -89,13 +95,12 @@ define i8 @PR31243_sext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) { ; AVX-NEXT: [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0 ; AVX-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1 ; AVX-NEXT: [[TMP2:%.*]] = or <2 x i8> [[TMP1]], -; AVX-NEXT: [[TMP3:%.*]] = sext <2 x i8> [[TMP2]] to <2 x i16> -; AVX-NEXT: [[TMP4:%.*]] = extractelement <2 x i16> [[TMP3]], i64 0 -; AVX-NEXT: [[TMP5:%.*]] = sext i16 [[TMP4]] to i64 -; AVX-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP5]] -; AVX-NEXT: [[TMP6:%.*]] = extractelement <2 x i16> [[TMP3]], i64 1 -; AVX-NEXT: [[TMP7:%.*]] = sext i16 [[TMP6]] to i64 -; AVX-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP7]] +; AVX-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0 +; AVX-NEXT: [[TMP4:%.*]] = sext i8 [[TMP3]] to i64 +; AVX-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]] +; AVX-NEXT: [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1 +; AVX-NEXT: [[TMP6:%.*]] = sext i8 [[TMP5]] to i64 +; AVX-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]] ; AVX-NEXT: [[T6:%.*]] = load i8, ptr [[T4]], align 1 ; AVX-NEXT: [[T7:%.*]] = load i8, ptr [[T5]], align 1 ; AVX-NEXT: [[T8:%.*]] = add i8 [[T6]], [[T7]] diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll index 88f75c37846e..3cc32c1fc7b2 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll @@ -15,8 +15,8 @@ define i32 @phi3UndefInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %arg3) { ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> -; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) +; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) +; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: @@ -52,8 +52,8 @@ define i32 @phi2UndefInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %arg3) { ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> -; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) +; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) +; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: @@ -89,8 +89,8 @@ define i32 @phi1UndefInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %arg3) { ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> -; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) +; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) +; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: @@ -127,8 +127,8 @@ define i32 @phi1Undef1PoisonInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %ar ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> -; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) +; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) +; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: @@ -165,8 +165,8 @@ define i32 @phi1Undef2PoisonInputs(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %a ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> -; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) +; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) +; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: @@ -202,8 +202,8 @@ define i32 @phi1Undef1PoisonGapInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> -; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) +; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) +; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/resched.ll b/llvm/test/Transforms/SLPVectorizer/X86/resched.ll index 78c6d9516a3d..b7237cbb02bb 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/resched.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/resched.ll @@ -11,26 +11,26 @@ define fastcc void @_ZN12_GLOBAL__N_127PolynomialMultiplyRecognize9recognizeEv() ; CHECK: if.then22.i: ; CHECK-NEXT: [[SUB_I:%.*]] = add nsw i32 undef, -1 ; CHECK-NEXT: [[CONV31_I:%.*]] = and i32 undef, [[SUB_I]] -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[CONV31_I]], i32 0 -; CHECK-NEXT: [[SHUFFLE1:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP2:%.*]] = lshr <4 x i32> [[SHUFFLE1]], +; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[CONV31_I]], i32 0 +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = lshr <4 x i32> [[TMP1]], ; CHECK-NEXT: [[SHR_4_I_I:%.*]] = lshr i32 [[CONV31_I]], 5 ; CHECK-NEXT: [[SHR_5_I_I:%.*]] = lshr i32 [[CONV31_I]], 6 ; CHECK-NEXT: [[SHR_6_I_I:%.*]] = lshr i32 [[CONV31_I]], 7 ; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i32> poison, i32 [[CONV31_I]], i32 0 -; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> poison, <8 x i32> zeroinitializer -; CHECK-NEXT: [[TMP4:%.*]] = lshr <8 x i32> [[SHUFFLE]], -; CHECK-NEXT: [[TMP5:%.*]] = insertelement <16 x i32> poison, i32 [[SUB_I]], i32 0 -; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <16 x i32> -; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> [[TMP6]], <16 x i32> -; CHECK-NEXT: [[TMP8:%.*]] = insertelement <16 x i32> [[TMP7]], i32 [[SHR_4_I_I]], i32 5 -; CHECK-NEXT: [[TMP9:%.*]] = insertelement <16 x i32> [[TMP8]], i32 [[SHR_5_I_I]], i32 6 -; CHECK-NEXT: [[TMP10:%.*]] = insertelement <16 x i32> [[TMP9]], i32 [[SHR_6_I_I]], i32 7 -; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <16 x i32> -; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <16 x i32> [[TMP10]], <16 x i32> [[TMP11]], <16 x i32> -; CHECK-NEXT: [[TMP13:%.*]] = trunc <16 x i32> [[TMP12]] to <16 x i8> -; CHECK-NEXT: [[TMP14:%.*]] = and <16 x i8> [[TMP13]], -; CHECK-NEXT: store <16 x i8> [[TMP14]], ptr undef, align 1 +; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> poison, <8 x i32> zeroinitializer +; CHECK-NEXT: [[TMP5:%.*]] = lshr <8 x i32> [[TMP4]], +; CHECK-NEXT: [[TMP6:%.*]] = insertelement <16 x i32> poison, i32 [[SUB_I]], i32 0 +; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <16 x i32> +; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <16 x i32> [[TMP6]], <16 x i32> [[TMP7]], <16 x i32> +; CHECK-NEXT: [[TMP9:%.*]] = insertelement <16 x i32> [[TMP8]], i32 [[SHR_4_I_I]], i32 5 +; CHECK-NEXT: [[TMP10:%.*]] = insertelement <16 x i32> [[TMP9]], i32 [[SHR_5_I_I]], i32 6 +; CHECK-NEXT: [[TMP11:%.*]] = insertelement <16 x i32> [[TMP10]], i32 [[SHR_6_I_I]], i32 7 +; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> poison, <16 x i32> +; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <16 x i32> [[TMP11]], <16 x i32> [[TMP12]], <16 x i32> +; CHECK-NEXT: [[TMP14:%.*]] = trunc <16 x i32> [[TMP13]] to <16 x i8> +; CHECK-NEXT: [[TMP15:%.*]] = and <16 x i8> [[TMP14]], +; CHECK-NEXT: store <16 x i8> [[TMP15]], ptr undef, align 1 ; CHECK-NEXT: unreachable ; CHECK: if.end50.i: ; CHECK-NEXT: ret void diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll index 5d22b5a4873b..1d1fcec2a7ae 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll @@ -7,12 +7,10 @@ define i1 @test(i1 %cmp5.not.31) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i1> , i1 [[CMP5_NOT_31]], i32 0 ; CHECK-NEXT: [[TMP1:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> zeroinitializer, <4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP2:%.*]] = trunc <4 x i32> [[TMP1]] to <4 x i1> -; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32> -; CHECK-NEXT: [[TMP4:%.*]] = mul <4 x i32> [[TMP3]], -; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]]) -; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[TMP5]], 0 -; CHECK-NEXT: [[CMP_NOT_I_I:%.*]] = icmp eq i32 [[TMP6]], 0 +; CHECK-NEXT: [[TMP2:%.*]] = mul <4 x i32> [[TMP1]], +; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]]) +; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[TMP3]], 0 +; CHECK-NEXT: [[CMP_NOT_I_I:%.*]] = icmp eq i32 [[TMP4]], 0 ; CHECK-NEXT: ret i1 [[CMP_NOT_I_I]] ; entry: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll index c1dd90d0e9a7..2f6868d8dfd6 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll @@ -8,17 +8,18 @@ ; YAML-NEXT: Function: stores ; YAML-NEXT: Args: ; YAML-NEXT: - String: 'Stores SLP vectorized with cost ' -; YAML-NEXT: - Cost: '-3' +; YAML-NEXT: - Cost: '-7' ; YAML-NEXT: - String: ' and with tree size ' ; YAML-NEXT: - TreeSize: '6' define void @stores(ptr noalias %in, ptr noalias %inn, ptr noalias %out) { ; CHECK-LABEL: @stores( ; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[IN:%.*]], align 1 ; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[INN:%.*]], align 1 -; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i64> -; CHECK-NEXT: [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i64> -; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[TMP3]], [[TMP4]] -; CHECK-NEXT: store <4 x i64> [[TMP5]], ptr [[OUT:%.*]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i16> +; CHECK-NEXT: [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i16> +; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i16> [[TMP3]], [[TMP4]] +; CHECK-NEXT: [[TMP6:%.*]] = zext <4 x i16> [[TMP5]] to <4 x i64> +; CHECK-NEXT: store <4 x i64> [[TMP6]], ptr [[OUT:%.*]], align 4 ; CHECK-NEXT: ret void ; %load.1 = load i8, ptr %in, align 1 @@ -63,17 +64,18 @@ define void @stores(ptr noalias %in, ptr noalias %inn, ptr noalias %out) { ; YAML-NEXT: Function: insertelems ; YAML-NEXT: Args: ; YAML-NEXT: - String: 'SLP vectorized with cost ' -; YAML-NEXT: - Cost: '-5' +; YAML-NEXT: - Cost: '-9' ; YAML-NEXT: - String: ' and with tree size ' ; YAML-NEXT: - TreeSize: '6' define <4 x i64> @insertelems(ptr noalias %in, ptr noalias %inn) { ; CHECK-LABEL: @insertelems( ; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[IN:%.*]], align 1 ; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[INN:%.*]], align 1 -; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i64> -; CHECK-NEXT: [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i64> -; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[TMP3]], [[TMP4]] -; CHECK-NEXT: ret <4 x i64> [[TMP5]] +; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i16> +; CHECK-NEXT: [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i16> +; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i16> [[TMP3]], [[TMP4]] +; CHECK-NEXT: [[TMP6:%.*]] = zext <4 x i16> [[TMP5]] to <4 x i64> +; CHECK-NEXT: ret <4 x i64> [[TMP6]] ; %load.1 = load i8, ptr %in, align 1 %gep.1 = getelementptr inbounds i8, ptr %in, i64 1 diff --git a/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll b/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll index 061fbdb45a13..ff6f0bdd3db8 100644 --- a/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll +++ b/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll @@ -10,8 +10,8 @@ define i32 @alt_cmp(i16 %call46) { ; CHECK-NEXT: [[TMP2:%.*]] = icmp ult <4 x i16> [[TMP0]], [[TMP1]] ; CHECK-NEXT: [[TMP3:%.*]] = icmp ugt <4 x i16> [[TMP0]], [[TMP1]] ; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i1> [[TMP2]], <4 x i1> [[TMP3]], <4 x i32> -; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i1> [[TMP4]] to <4 x i16> -; CHECK-NEXT: [[TMP6:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP5]]) +; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) +; CHECK-NEXT: [[TMP6:%.*]] = zext i1 [[TMP5]] to i16 ; CHECK-NEXT: [[OP_RDX:%.*]] = or i16 [[TMP6]], 0 ; CHECK-NEXT: [[EXT:%.*]] = zext i16 [[OP_RDX]] to i32 ; CHECK-NEXT: ret i32 [[EXT]] -- GitLab From 643b31dbe8a515e007a0f1b2e1072c34e461b778 Mon Sep 17 00:00:00 2001 From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com> Date: Tue, 5 Mar 2024 12:23:26 -0500 Subject: [PATCH 173/929] [HLSL] implement `mad` intrinsic (#83826) This change implements #83736 The dot product lowering needs a tertiary multipy add operation. DXIL has three mad opcodes for `fmad`(46), `imad`(48), and `umad`(49). Dot product in DXIL only uses `imad`\ `umad`, but for completeness and because the hlsl `mad` intrinsic requires it `fmad` was also included. Two new intrinsics were needed to be created to complete this change. the `fmad` case already supported by llvm via `fmuladd` intrinsic. - `hlsl_intrinsics.h` - exposed mad api call. - `Builtins.td` - exposed a `mad` builtin. - `Sema.h` - make `tertiary` calls check for float types optional. - `CGBuiltin.cpp` - pick the intrinsic for singed\unsigned & float also reuse `int_fmuladd`. - `SemaChecking.cpp` - type checks for `__builtin_hlsl_mad`. - `IntrinsicsDirectX.td` create the two new intrinsics for `imad`\`umad`/ - `DXIL.td` - create the llvm intrinsic to `DXIL` opcode mapping. --------- Co-authored-by: Farzon Lotfi --- clang/include/clang/Basic/Builtins.td | 6 + clang/include/clang/Sema/Sema.h | 3 +- clang/lib/CodeGen/CGBuiltin.cpp | 19 ++ clang/lib/Headers/hlsl/hlsl_intrinsics.h | 105 ++++++++++ clang/lib/Sema/SemaChecking.cpp | 28 ++- clang/test/CodeGenHLSL/builtins/mad.hlsl | 191 +++++++++++++++++++ clang/test/SemaHLSL/BuiltIns/mad-errors.hlsl | 86 +++++++++ llvm/include/llvm/IR/IntrinsicsDirectX.td | 3 + llvm/lib/Target/DirectX/DXIL.td | 6 + llvm/test/CodeGen/DirectX/fmad.ll | 67 +++++++ llvm/test/CodeGen/DirectX/imad.ll | 65 +++++++ llvm/test/CodeGen/DirectX/umad.ll | 65 +++++++ 12 files changed, 638 insertions(+), 6 deletions(-) create mode 100644 clang/test/CodeGenHLSL/builtins/mad.hlsl create mode 100644 clang/test/SemaHLSL/BuiltIns/mad-errors.hlsl create mode 100644 llvm/test/CodeGen/DirectX/fmad.ll create mode 100644 llvm/test/CodeGen/DirectX/imad.ll create mode 100644 llvm/test/CodeGen/DirectX/umad.ll diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td index 384442e6a935..26c7be478914 100644 --- a/clang/include/clang/Basic/Builtins.td +++ b/clang/include/clang/Basic/Builtins.td @@ -4572,6 +4572,12 @@ def HLSLLerp : LangBuiltin<"HLSL_LANG"> { let Prototype = "void(...)"; } +def HLSLMad : LangBuiltin<"HLSL_LANG"> { + let Spellings = ["__builtin_hlsl_mad"]; + let Attributes = [NoThrow, Const]; + let Prototype = "void(...)"; +} + // Builtins for XRay. def XRayCustomEvent : Builtin { let Spellings = ["__xray_customevent"]; diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index 29baf542f6cb..25c4c58ad4ae 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -14146,7 +14146,8 @@ private: bool SemaBuiltinVectorMath(CallExpr *TheCall, QualType &Res); bool SemaBuiltinVectorToScalarMath(CallExpr *TheCall); bool SemaBuiltinElementwiseMath(CallExpr *TheCall); - bool SemaBuiltinElementwiseTernaryMath(CallExpr *TheCall); + bool SemaBuiltinElementwiseTernaryMath(CallExpr *TheCall, + bool CheckForFloatArgs = true); bool PrepareBuiltinElementwiseMathOneArgCall(CallExpr *TheCall); bool PrepareBuiltinReduceMathOneArgCall(CallExpr *TheCall); diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index 9ee51ca7142c..191f6d5a9fab 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -18044,6 +18044,25 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID, /*ReturnType*/ Op0->getType(), Intrinsic::dx_frac, ArrayRef{Op0}, nullptr, "dx.frac"); } + case Builtin::BI__builtin_hlsl_mad: { + Value *M = EmitScalarExpr(E->getArg(0)); + Value *A = EmitScalarExpr(E->getArg(1)); + Value *B = EmitScalarExpr(E->getArg(2)); + if (E->getArg(0)->getType()->hasFloatingRepresentation()) { + return Builder.CreateIntrinsic( + /*ReturnType*/ M->getType(), Intrinsic::fmuladd, + ArrayRef{M, A, B}, nullptr, "dx.fmad"); + } + if (E->getArg(0)->getType()->hasSignedIntegerRepresentation()) { + return Builder.CreateIntrinsic( + /*ReturnType*/ M->getType(), Intrinsic::dx_imad, + ArrayRef{M, A, B}, nullptr, "dx.imad"); + } + assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation()); + return Builder.CreateIntrinsic( + /*ReturnType*/ M->getType(), Intrinsic::dx_umad, + ArrayRef{M, A, B}, nullptr, "dx.umad"); + } } return nullptr; } diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_intrinsics.h index 518053036388..b5bef78fae72 100644 --- a/clang/lib/Headers/hlsl/hlsl_intrinsics.h +++ b/clang/lib/Headers/hlsl/hlsl_intrinsics.h @@ -511,6 +511,111 @@ double3 log2(double3); _HLSL_BUILTIN_ALIAS(__builtin_elementwise_log2) double4 log2(double4); +//===----------------------------------------------------------------------===// +// mad builtins +//===----------------------------------------------------------------------===// + +/// \fn T mad(T M, T A, T B) +/// \brief The result of \a M * \a A + \a B. +/// \param M The multiplication value. +/// \param A The first addition value. +/// \param B The second addition value. + +_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +half mad(half, half, half); +_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +half2 mad(half2, half2, half2); +_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +half3 mad(half3, half3, half3); +_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +half4 mad(half4, half4, half4); + +#ifdef __HLSL_ENABLE_16_BIT +_HLSL_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +int16_t mad(int16_t, int16_t, int16_t); +_HLSL_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +int16_t2 mad(int16_t2, int16_t2, int16_t2); +_HLSL_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +int16_t3 mad(int16_t3, int16_t3, int16_t3); +_HLSL_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +int16_t4 mad(int16_t4, int16_t4, int16_t4); + +_HLSL_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +uint16_t mad(uint16_t, uint16_t, uint16_t); +_HLSL_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +uint16_t2 mad(uint16_t2, uint16_t2, uint16_t2); +_HLSL_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +uint16_t3 mad(uint16_t3, uint16_t3, uint16_t3); +_HLSL_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +uint16_t4 mad(uint16_t4, uint16_t4, uint16_t4); +#endif + +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +int mad(int, int, int); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +int2 mad(int2, int2, int2); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +int3 mad(int3, int3, int3); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +int4 mad(int4, int4, int4); + +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +uint mad(uint, uint, uint); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +uint2 mad(uint2, uint2, uint2); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +uint3 mad(uint3, uint3, uint3); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +uint4 mad(uint4, uint4, uint4); + +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +int64_t mad(int64_t, int64_t, int64_t); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +int64_t2 mad(int64_t2, int64_t2, int64_t2); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +int64_t3 mad(int64_t3, int64_t3, int64_t3); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +int64_t4 mad(int64_t4, int64_t4, int64_t4); + +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +uint64_t mad(uint64_t, uint64_t, uint64_t); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +uint64_t2 mad(uint64_t2, uint64_t2, uint64_t2); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +uint64_t3 mad(uint64_t3, uint64_t3, uint64_t3); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +uint64_t4 mad(uint64_t4, uint64_t4, uint64_t4); + +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +float mad(float, float, float); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +float2 mad(float2, float2, float2); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +float3 mad(float3, float3, float3); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +float4 mad(float4, float4, float4); + +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +double mad(double, double, double); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +double2 mad(double2, double2, double2); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +double3 mad(double3, double3, double3); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_mad) +double4 mad(double4, double4, double4); + //===----------------------------------------------------------------------===// // max builtins //===----------------------------------------------------------------------===// diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index 2cda1d08784e..85477061d190 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -5298,6 +5298,14 @@ bool Sema::CheckHLSLBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { return true; break; } + case Builtin::BI__builtin_hlsl_mad: { + if (checkArgCount(*this, TheCall, 3)) + return true; + if (CheckVectorElementCallArgs(this, TheCall)) + return true; + if (SemaBuiltinElementwiseTernaryMath(TheCall, /*CheckForFloatArgs*/ false)) + return true; + } } return false; } @@ -19798,7 +19806,8 @@ bool Sema::SemaBuiltinVectorMath(CallExpr *TheCall, QualType &Res) { return false; } -bool Sema::SemaBuiltinElementwiseTernaryMath(CallExpr *TheCall) { +bool Sema::SemaBuiltinElementwiseTernaryMath(CallExpr *TheCall, + bool CheckForFloatArgs) { if (checkArgCount(*this, TheCall, 3)) return true; @@ -19810,11 +19819,20 @@ bool Sema::SemaBuiltinElementwiseTernaryMath(CallExpr *TheCall) { Args[I] = Converted.get(); } - int ArgOrdinal = 1; - for (Expr *Arg : Args) { - if (checkFPMathBuiltinElementType(*this, Arg->getBeginLoc(), Arg->getType(), + if (CheckForFloatArgs) { + int ArgOrdinal = 1; + for (Expr *Arg : Args) { + if (checkFPMathBuiltinElementType(*this, Arg->getBeginLoc(), + Arg->getType(), ArgOrdinal++)) + return true; + } + } else { + int ArgOrdinal = 1; + for (Expr *Arg : Args) { + if (checkMathBuiltinElementType(*this, Arg->getBeginLoc(), Arg->getType(), ArgOrdinal++)) - return true; + return true; + } } for (int I = 1; I < 3; ++I) { diff --git a/clang/test/CodeGenHLSL/builtins/mad.hlsl b/clang/test/CodeGenHLSL/builtins/mad.hlsl new file mode 100644 index 000000000000..749eac6d6473 --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/mad.hlsl @@ -0,0 +1,191 @@ +// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \ +// RUN: dxil-pc-shadermodel6.3-library %s -fnative-half-type \ +// RUN: -emit-llvm -disable-llvm-passes -o - | FileCheck %s \ +// RUN: --check-prefixes=CHECK,NATIVE_HALF +// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \ +// RUN: dxil-pc-shadermodel6.3-library %s -emit-llvm -disable-llvm-passes \ +// RUN: -o - | FileCheck %s --check-prefixes=CHECK,NO_HALF + +#ifdef __HLSL_ENABLE_16_BIT +// NATIVE_HALF: %dx.umad = call i16 @llvm.dx.umad.i16(i16 %0, i16 %1, i16 %2) +// NATIVE_HALF: ret i16 %dx.umad +uint16_t test_mad_uint16_t(uint16_t p0, uint16_t p1, uint16_t p2) { return mad(p0, p1, p2); } + +// NATIVE_HALF: %dx.umad = call <2 x i16> @llvm.dx.umad.v2i16(<2 x i16> %0, <2 x i16> %1, <2 x i16> %2) +// NATIVE_HALF: ret <2 x i16> %dx.umad +uint16_t2 test_mad_uint16_t2(uint16_t2 p0, uint16_t2 p1, uint16_t2 p2) { return mad(p0, p1, p2); } + +// NATIVE_HALF: %dx.umad = call <3 x i16> @llvm.dx.umad.v3i16(<3 x i16> %0, <3 x i16> %1, <3 x i16> %2) +// NATIVE_HALF: ret <3 x i16> %dx.umad +uint16_t3 test_mad_uint16_t3(uint16_t3 p0, uint16_t3 p1, uint16_t3 p2) { return mad(p0, p1, p2); } + +// NATIVE_HALF: %dx.umad = call <4 x i16> @llvm.dx.umad.v4i16(<4 x i16> %0, <4 x i16> %1, <4 x i16> %2) +// NATIVE_HALF: ret <4 x i16> %dx.umad +uint16_t4 test_mad_uint16_t4(uint16_t4 p0, uint16_t4 p1, uint16_t4 p2) { return mad(p0, p1, p2); } + +// NATIVE_HALF: %dx.imad = call i16 @llvm.dx.imad.i16(i16 %0, i16 %1, i16 %2) +// NATIVE_HALF: ret i16 %dx.imad +int16_t test_mad_int16_t(int16_t p0, int16_t p1, int16_t p2) { return mad(p0, p1, p2); } + +// NATIVE_HALF: %dx.imad = call <2 x i16> @llvm.dx.imad.v2i16(<2 x i16> %0, <2 x i16> %1, <2 x i16> %2) +// NATIVE_HALF: ret <2 x i16> %dx.imad +int16_t2 test_mad_int16_t2(int16_t2 p0, int16_t2 p1, int16_t2 p2) { return mad(p0, p1, p2); } + +// NATIVE_HALF: %dx.imad = call <3 x i16> @llvm.dx.imad.v3i16(<3 x i16> %0, <3 x i16> %1, <3 x i16> %2) +// NATIVE_HALF: ret <3 x i16> %dx.imad +int16_t3 test_mad_int16_t3(int16_t3 p0, int16_t3 p1, int16_t3 p2) { return mad(p0, p1, p2); } + +// NATIVE_HALF: %dx.imad = call <4 x i16> @llvm.dx.imad.v4i16(<4 x i16> %0, <4 x i16> %1, <4 x i16> %2) +// NATIVE_HALF: ret <4 x i16> %dx.imad +int16_t4 test_mad_int16_t4(int16_t4 p0, int16_t4 p1, int16_t4 p2) { return mad(p0, p1, p2); } +#endif // __HLSL_ENABLE_16_BIT + +// NATIVE_HALF: %dx.fmad = call half @llvm.fmuladd.f16(half %0, half %1, half %2) +// NATIVE_HALF: ret half %dx.fmad +// NO_HALF: %dx.fmad = call float @llvm.fmuladd.f32(float %0, float %1, float %2) +// NO_HALF: ret float %dx.fmad +half test_mad_half(half p0, half p1, half p2) { return mad(p0, p1, p2); } + +// NATIVE_HALF: %dx.fmad = call <2 x half> @llvm.fmuladd.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2) +// NATIVE_HALF: ret <2 x half> %dx.fmad +// NO_HALF: %dx.fmad = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %0, <2 x float> %1, <2 x float> %2) +// NO_HALF: ret <2 x float> %dx.fmad +half2 test_mad_half2(half2 p0, half2 p1, half2 p2) { return mad(p0, p1, p2); } + +// NATIVE_HALF: %dx.fmad = call <3 x half> @llvm.fmuladd.v3f16(<3 x half> %0, <3 x half> %1, <3 x half> %2) +// NATIVE_HALF: ret <3 x half> %dx.fmad +// NO_HALF: %dx.fmad = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> %0, <3 x float> %1, <3 x float> %2) +// NO_HALF: ret <3 x float> %dx.fmad +half3 test_mad_half3(half3 p0, half3 p1, half3 p2) { return mad(p0, p1, p2); } + +// NATIVE_HALF: %dx.fmad = call <4 x half> @llvm.fmuladd.v4f16(<4 x half> %0, <4 x half> %1, <4 x half> %2) +// NATIVE_HALF: ret <4 x half> %dx.fmad +// NO_HALF: %dx.fmad = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2) +// NO_HALF: ret <4 x float> %dx.fmad +half4 test_mad_half4(half4 p0, half4 p1, half4 p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.fmad = call float @llvm.fmuladd.f32(float %0, float %1, float %2) +// CHECK: ret float %dx.fmad +float test_mad_float(float p0, float p1, float p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.fmad = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %0, <2 x float> %1, <2 x float> %2) +// CHECK: ret <2 x float> %dx.fmad +float2 test_mad_float2(float2 p0, float2 p1, float2 p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.fmad = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> %0, <3 x float> %1, <3 x float> %2) +// CHECK: ret <3 x float> %dx.fmad +float3 test_mad_float3(float3 p0, float3 p1, float3 p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.fmad = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2) +// CHECK: ret <4 x float> %dx.fmad +float4 test_mad_float4(float4 p0, float4 p1, float4 p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.fmad = call double @llvm.fmuladd.f64(double %0, double %1, double %2) +// CHECK: ret double %dx.fmad +double test_mad_double(double p0, double p1, double p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.fmad = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> %0, <2 x double> %1, <2 x double> %2) +// CHECK: ret <2 x double> %dx.fmad +double2 test_mad_double2(double2 p0, double2 p1, double2 p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.fmad = call <3 x double> @llvm.fmuladd.v3f64(<3 x double> %0, <3 x double> %1, <3 x double> %2) +// CHECK: ret <3 x double> %dx.fmad +double3 test_mad_double3(double3 p0, double3 p1, double3 p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.fmad = call <4 x double> @llvm.fmuladd.v4f64(<4 x double> %0, <4 x double> %1, <4 x double> %2) +// CHECK: ret <4 x double> %dx.fmad +double4 test_mad_double4(double4 p0, double4 p1, double4 p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.imad = call i32 @llvm.dx.imad.i32(i32 %0, i32 %1, i32 %2) +// CHECK: ret i32 %dx.imad +int test_mad_int(int p0, int p1, int p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.imad = call <2 x i32> @llvm.dx.imad.v2i32(<2 x i32> %0, <2 x i32> %1, <2 x i32> %2) +// CHECK: ret <2 x i32> %dx.imad +int2 test_mad_int2(int2 p0, int2 p1, int2 p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.imad = call <3 x i32> @llvm.dx.imad.v3i32(<3 x i32> %0, <3 x i32> %1, <3 x i32> %2) +// CHECK: ret <3 x i32> %dx.imad +int3 test_mad_int3(int3 p0, int3 p1, int3 p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.imad = call <4 x i32> @llvm.dx.imad.v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2) +// CHECK: ret <4 x i32> %dx.imad +int4 test_mad_int4(int4 p0, int4 p1, int4 p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.imad = call i64 @llvm.dx.imad.i64(i64 %0, i64 %1, i64 %2) +// CHECK: ret i64 %dx.imad +int64_t test_mad_int64_t(int64_t p0, int64_t p1, int64_t p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.imad = call <2 x i64> @llvm.dx.imad.v2i64(<2 x i64> %0, <2 x i64> %1, <2 x i64> %2) +// CHECK: ret <2 x i64> %dx.imad +int64_t2 test_mad_int64_t2(int64_t2 p0, int64_t2 p1, int64_t2 p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.imad = call <3 x i64> @llvm.dx.imad.v3i64(<3 x i64> %0, <3 x i64> %1, <3 x i64> %2) +// CHECK: ret <3 x i64> %dx.imad +int64_t3 test_mad_int64_t3(int64_t3 p0, int64_t3 p1, int64_t3 p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.imad = call <4 x i64> @llvm.dx.imad.v4i64(<4 x i64> %0, <4 x i64> %1, <4 x i64> %2) +// CHECK: ret <4 x i64> %dx.imad +int64_t4 test_mad_int64_t4(int64_t4 p0, int64_t4 p1, int64_t4 p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.umad = call i32 @llvm.dx.umad.i32(i32 %0, i32 %1, i32 %2) +// CHECK: ret i32 %dx.umad +uint test_mad_uint(uint p0, uint p1, uint p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.umad = call <2 x i32> @llvm.dx.umad.v2i32(<2 x i32> %0, <2 x i32> %1, <2 x i32> %2) +// CHECK: ret <2 x i32> %dx.umad +uint2 test_mad_uint2(uint2 p0, uint2 p1, uint2 p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.umad = call <3 x i32> @llvm.dx.umad.v3i32(<3 x i32> %0, <3 x i32> %1, <3 x i32> %2) +// CHECK: ret <3 x i32> %dx.umad +uint3 test_mad_uint3(uint3 p0, uint3 p1, uint3 p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.umad = call <4 x i32> @llvm.dx.umad.v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2) +// CHECK: ret <4 x i32> %dx.umad +uint4 test_mad_uint4(uint4 p0, uint4 p1, uint4 p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.umad = call i64 @llvm.dx.umad.i64(i64 %0, i64 %1, i64 %2) +// CHECK: ret i64 %dx.umad +uint64_t test_mad_uint64_t(uint64_t p0, uint64_t p1, uint64_t p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.umad = call <2 x i64> @llvm.dx.umad.v2i64(<2 x i64> %0, <2 x i64> %1, <2 x i64> %2) +// CHECK: ret <2 x i64> %dx.umad +uint64_t2 test_mad_uint64_t2(uint64_t2 p0, uint64_t2 p1, uint64_t2 p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.umad = call <3 x i64> @llvm.dx.umad.v3i64(<3 x i64> %0, <3 x i64> %1, <3 x i64> %2) +// CHECK: ret <3 x i64> %dx.umad +uint64_t3 test_mad_uint64_t3(uint64_t3 p0, uint64_t3 p1, uint64_t3 p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.umad = call <4 x i64> @llvm.dx.umad.v4i64(<4 x i64> %0, <4 x i64> %1, <4 x i64> %2) +// CHECK: ret <4 x i64> %dx.umad +uint64_t4 test_mad_uint64_t4(uint64_t4 p0, uint64_t4 p1, uint64_t4 p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.fmad = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %splat.splat, <2 x float> %1, <2 x float> %2) +// CHECK: ret <2 x float> %dx.fmad +float2 test_mad_float2_splat(float p0, float2 p1, float2 p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.fmad = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> %splat.splat, <3 x float> %1, <3 x float> %2) +// CHECK: ret <3 x float> %dx.fmad +float3 test_mad_float3_splat(float p0, float3 p1, float3 p2) { return mad(p0, p1, p2); } + +// CHECK: %dx.fmad = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> %splat.splat, <4 x float> %1, <4 x float> %2) +// CHECK: ret <4 x float> %dx.fmad +float4 test_mad_float4_splat(float p0, float4 p1, float4 p2) { return mad(p0, p1, p2); } + +// CHECK: %conv = sitofp i32 %2 to float +// CHECK: %splat.splatinsert = insertelement <2 x float> poison, float %conv, i64 0 +// CHECK: %splat.splat = shufflevector <2 x float> %splat.splatinsert, <2 x float> poison, <2 x i32> zeroinitializer +// CHECK: %dx.fmad = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %0, <2 x float> %1, <2 x float> %splat.splat) +// CHECK: ret <2 x float> %dx.fmad +float2 test_mad_float2_int_splat(float2 p0, float2 p1, int p2) { + return mad(p0, p1, p2); +} + +// CHECK: %conv = sitofp i32 %2 to float +// CHECK: %splat.splatinsert = insertelement <3 x float> poison, float %conv, i64 0 +// CHECK: %splat.splat = shufflevector <3 x float> %splat.splatinsert, <3 x float> poison, <3 x i32> zeroinitializer +// CHECK: %dx.fmad = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> %0, <3 x float> %1, <3 x float> %splat.splat) +// CHECK: ret <3 x float> %dx.fmad +float3 test_mad_float3_int_splat(float3 p0, float3 p1, int p2) { + return mad(p0, p1, p2); +} diff --git a/clang/test/SemaHLSL/BuiltIns/mad-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/mad-errors.hlsl new file mode 100644 index 000000000000..0b6843591455 --- /dev/null +++ b/clang/test/SemaHLSL/BuiltIns/mad-errors.hlsl @@ -0,0 +1,86 @@ +// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm -disable-llvm-passes -verify -verify-ignore-unexpected + +float2 test_no_second_arg(float2 p0) { + return __builtin_hlsl_mad(p0); + // expected-error@-1 {{too few arguments to function call, expected 3, have 1}} +} + +float2 test_no_third_arg(float2 p0) { + return __builtin_hlsl_mad(p0, p0); + // expected-error@-1 {{too few arguments to function call, expected 3, have 2}} +} + +float2 test_too_many_arg(float2 p0) { + return __builtin_hlsl_mad(p0, p0, p0, p0); + // expected-error@-1 {{too many arguments to function call, expected 3, have 4}} +} + +float2 test_mad_no_second_arg(float2 p0) { + return mad(p0); + // expected-error@-1 {{no matching function for call to 'mad'}} +} + +float2 test_mad_vector_size_mismatch(float3 p0, float2 p1) { + return mad(p0, p0, p1); + // expected-warning@-1 {{implicit conversion truncates vector: 'float3' (aka 'vector') to 'float __attribute__((ext_vector_type(2)))' (vector of 2 'float' values)}} +} + +float2 test_mad_builtin_vector_size_mismatch(float3 p0, float2 p1) { + return __builtin_hlsl_mad(p0, p1, p1); + // expected-error@-1 {{all arguments to '__builtin_hlsl_mad' must have the same type}} +} + +float test_mad_scalar_mismatch(float p0, half p1) { + return mad(p1, p0, p1); + // expected-error@-1 {{call to 'mad' is ambiguous}} +} + +float2 test_mad_element_type_mismatch(half2 p0, float2 p1) { + return mad(p1, p0, p1); + // expected-error@-1 {{call to 'mad' is ambiguous}} +} + +float2 test_builtin_mad_float2_splat(float p0, float2 p1) { + return __builtin_hlsl_mad(p0, p1, p1); + // expected-error@-1 {{all arguments to '__builtin_hlsl_mad' must be vectors}} +} + +float3 test_builtin_mad_float3_splat(float p0, float3 p1) { + return __builtin_hlsl_mad(p0, p1, p1); + // expected-error@-1 {{all arguments to '__builtin_hlsl_mad' must be vectors}} +} + +float4 test_builtin_mad_float4_splat(float p0, float4 p1) { + return __builtin_hlsl_mad(p0, p1, p1); + // expected-error@-1 {{all arguments to '__builtin_hlsl_mad' must be vectors}} +} + +float2 test_mad_float2_int_splat(float2 p0, int p1) { + return __builtin_hlsl_mad(p0, p1, p1); + // expected-error@-1 {{all arguments to '__builtin_hlsl_mad' must be vectors}} +} + +float3 test_mad_float3_int_splat(float3 p0, int p1) { + return __builtin_hlsl_mad(p0, p1, p1); + // expected-error@-1 {{all arguments to '__builtin_hlsl_mad' must be vectors}} +} + +float2 test_builtin_mad_int_vect_to_float_vec_promotion(int2 p0, float p1) { + return __builtin_hlsl_mad(p0, p1, p1); + // expected-error@-1 {{all arguments to '__builtin_hlsl_mad' must be vectors}} +} + +float builtin_bool_to_float_type_promotion(float p0, bool p1) { + return __builtin_hlsl_mad(p0, p0, p1); + // expected-error@-1 {{3rd argument must be a vector, integer or floating point type (was 'bool')}} +} + +float builtin_bool_to_float_type_promotion2(bool p0, float p1) { + return __builtin_hlsl_mad(p1, p0, p1); + // expected-error@-1 {{2nd argument must be a vector, integer or floating point type (was 'bool')}} +} + +float builtin_mad_int_to_float_promotion(float p0, int p1) { + return __builtin_hlsl_mad(p0, p0, p1); + // expected-error@-1 {{arguments are of different types ('double' vs 'int')}} +} diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td index b44d1c6d3d2f..3096442335ce 100644 --- a/llvm/include/llvm/IR/IntrinsicsDirectX.td +++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td @@ -31,4 +31,7 @@ def int_dx_lerp : Intrinsic<[LLVMScalarOrSameVectorWidth<0, LLVMVectorElementType<0>>], [llvm_anyvector_ty, LLVMScalarOrSameVectorWidth<0, LLVMVectorElementType<0>>,LLVMScalarOrSameVectorWidth<0, LLVMVectorElementType<0>>], [IntrNoMem, IntrWillReturn] >; + +def int_dx_imad : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>; +def int_dx_umad : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>; } diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td index 33b08ed93e3d..8f73c08a6580 100644 --- a/llvm/lib/Target/DirectX/DXIL.td +++ b/llvm/lib/Target/DirectX/DXIL.td @@ -226,6 +226,12 @@ def Round : DXILOpMapping<26, unary, int_round, "within a floating-point type.">; def UMax : DXILOpMapping<39, binary, int_umax, "Unsigned integer maximum. UMax(a,b) = a > b ? a : b">; +def FMad : DXILOpMapping<46, tertiary, int_fmuladd, + "Floating point arithmetic multiply/add operation. fmad(m,a,b) = m * a + b.">; +def IMad : DXILOpMapping<48, tertiary, int_dx_imad, + "Signed integer arithmetic multiply/add operation. imad(m,a,b) = m * a + b.">; +def UMad : DXILOpMapping<49, tertiary, int_dx_umad, + "Unsigned integer arithmetic multiply/add operation. umad(m,a,b) = m * a + b.">; def ThreadId : DXILOpMapping<93, threadId, int_dx_thread_id, "Reads the thread ID">; def GroupId : DXILOpMapping<94, groupId, int_dx_group_id, diff --git a/llvm/test/CodeGen/DirectX/fmad.ll b/llvm/test/CodeGen/DirectX/fmad.ll new file mode 100644 index 000000000000..693e237e70dc --- /dev/null +++ b/llvm/test/CodeGen/DirectX/fmad.ll @@ -0,0 +1,67 @@ +; RUN: opt -S -dxil-op-lower < %s | FileCheck %s + +; Make sure dxil operation function calls for round are generated for float and half. +; CHECK:call half @dx.op.tertiary.f16(i32 46, half %{{.*}}, half %{{.*}}, half %{{.*}}) +; CHECK:call float @dx.op.tertiary.f32(i32 46, float %{{.*}}, float %{{.*}}, float %{{.*}}) +; CHECK:call double @dx.op.tertiary.f64(i32 46, double %{{.*}}, double %{{.*}}, double %{{.*}}) + + +target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64" +target triple = "dxil-pc-shadermodel6.7-library" + +; Function Attrs: noinline nounwind optnone +define noundef half @fmad_half(half noundef %p0, half noundef %p1, half noundef %p2) #0 { +entry: + %p2.addr = alloca half, align 2 + %p1.addr = alloca half, align 2 + %p0.addr = alloca half, align 2 + store half %p2, ptr %p2.addr, align 2 + store half %p1, ptr %p1.addr, align 2 + store half %p0, ptr %p0.addr, align 2 + %0 = load half, ptr %p0.addr, align 2 + %1 = load half, ptr %p1.addr, align 2 + %2 = load half, ptr %p2.addr, align 2 + %dx.fmad = call half @llvm.fmuladd.f16(half %0, half %1, half %2) + ret half %dx.fmad +} + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare half @llvm.fmuladd.f16(half, half, half) #2 + +; Function Attrs: noinline nounwind optnone +define noundef float @fmad_float(float noundef %p0, float noundef %p1, float noundef %p2) #0 { +entry: + %p2.addr = alloca float, align 4 + %p1.addr = alloca float, align 4 + %p0.addr = alloca float, align 4 + store float %p2, ptr %p2.addr, align 4 + store float %p1, ptr %p1.addr, align 4 + store float %p0, ptr %p0.addr, align 4 + %0 = load float, ptr %p0.addr, align 4 + %1 = load float, ptr %p1.addr, align 4 + %2 = load float, ptr %p2.addr, align 4 + %dx.fmad = call float @llvm.fmuladd.f32(float %0, float %1, float %2) + ret float %dx.fmad +} + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare float @llvm.fmuladd.f32(float, float, float) #2 + +; Function Attrs: noinline nounwind optnone +define noundef double @fmad_double(double noundef %p0, double noundef %p1, double noundef %p2) #0 { +entry: + %p2.addr = alloca double, align 8 + %p1.addr = alloca double, align 8 + %p0.addr = alloca double, align 8 + store double %p2, ptr %p2.addr, align 8 + store double %p1, ptr %p1.addr, align 8 + store double %p0, ptr %p0.addr, align 8 + %0 = load double, ptr %p0.addr, align 8 + %1 = load double, ptr %p1.addr, align 8 + %2 = load double, ptr %p2.addr, align 8 + %dx.fmad = call double @llvm.fmuladd.f64(double %0, double %1, double %2) + ret double %dx.fmad +} + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare double @llvm.fmuladd.f64(double, double, double) #2 diff --git a/llvm/test/CodeGen/DirectX/imad.ll b/llvm/test/CodeGen/DirectX/imad.ll new file mode 100644 index 000000000000..5b818f86bc7f --- /dev/null +++ b/llvm/test/CodeGen/DirectX/imad.ll @@ -0,0 +1,65 @@ +; RUN: opt -S -dxil-op-lower < %s | FileCheck %s + +; Make sure dxil operation function calls for round are generated for float and half. +; CHECK:call i16 @dx.op.tertiary.i16(i32 48, i16 %{{.*}}, i16 %{{.*}}, i16 %{{.*}}) +; CHECK:call i32 @dx.op.tertiary.i32(i32 48, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}) +; CHECK:call i64 @dx.op.tertiary.i64(i32 48, i64 %{{.*}}, i64 %{{.*}}, i64 %{{.*}}) + +target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64" +target triple = "dxil-pc-shadermodel6.7-library" +; Function Attrs: noinline nounwind optnone +define noundef i16 @imad_short(i16 noundef %p0, i16 noundef %p1, i16 noundef %p2) #0 { +entry: + %p2.addr = alloca i16, align 2 + %p1.addr = alloca i16, align 2 + %p0.addr = alloca i16, align 2 + store i16 %p2, ptr %p2.addr, align 2 + store i16 %p1, ptr %p1.addr, align 2 + store i16 %p0, ptr %p0.addr, align 2 + %0 = load i16, ptr %p0.addr, align 2 + %1 = load i16, ptr %p1.addr, align 2 + %2 = load i16, ptr %p2.addr, align 2 + %dx.imad = call i16 @llvm.dx.imad.i16(i16 %0, i16 %1, i16 %2) + ret i16 %dx.imad +} + +; Function Attrs: nocallback nofree nosync nounwind willreturn +declare i16 @llvm.dx.imad.i16(i16, i16, i16) #1 + +; Function Attrs: noinline nounwind optnone +define noundef i32 @imad_int(i32 noundef %p0, i32 noundef %p1, i32 noundef %p2) #0 { +entry: + %p2.addr = alloca i32, align 4 + %p1.addr = alloca i32, align 4 + %p0.addr = alloca i32, align 4 + store i32 %p2, ptr %p2.addr, align 4 + store i32 %p1, ptr %p1.addr, align 4 + store i32 %p0, ptr %p0.addr, align 4 + %0 = load i32, ptr %p0.addr, align 4 + %1 = load i32, ptr %p1.addr, align 4 + %2 = load i32, ptr %p2.addr, align 4 + %dx.imad = call i32 @llvm.dx.imad.i32(i32 %0, i32 %1, i32 %2) + ret i32 %dx.imad +} + +; Function Attrs: nocallback nofree nosync nounwind willreturn +declare i32 @llvm.dx.imad.i32(i32, i32, i32) #1 + +; Function Attrs: noinline nounwind optnone +define noundef i64 @imad_int64(i64 noundef %p0, i64 noundef %p1, i64 noundef %p2) #0 { +entry: + %p2.addr = alloca i64, align 8 + %p1.addr = alloca i64, align 8 + %p0.addr = alloca i64, align 8 + store i64 %p2, ptr %p2.addr, align 8 + store i64 %p1, ptr %p1.addr, align 8 + store i64 %p0, ptr %p0.addr, align 8 + %0 = load i64, ptr %p0.addr, align 8 + %1 = load i64, ptr %p1.addr, align 8 + %2 = load i64, ptr %p2.addr, align 8 + %dx.imad = call i64 @llvm.dx.imad.i64(i64 %0, i64 %1, i64 %2) + ret i64 %dx.imad +} + +; Function Attrs: nocallback nofree nosync nounwind willreturn +declare i64 @llvm.dx.imad.i64(i64, i64, i64) #1 diff --git a/llvm/test/CodeGen/DirectX/umad.ll b/llvm/test/CodeGen/DirectX/umad.ll new file mode 100644 index 000000000000..583fdddfe03f --- /dev/null +++ b/llvm/test/CodeGen/DirectX/umad.ll @@ -0,0 +1,65 @@ +; RUN: opt -S -dxil-op-lower < %s | FileCheck %s + +; Make sure dxil operation function calls for round are generated for float and half. +; CHECK:call i16 @dx.op.tertiary.i16(i32 49, i16 %{{.*}}, i16 %{{.*}}, i16 %{{.*}}) +; CHECK:call i32 @dx.op.tertiary.i32(i32 49, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}) +; CHECK:call i64 @dx.op.tertiary.i64(i32 49, i64 %{{.*}}, i64 %{{.*}}, i64 %{{.*}}) + +target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64" +target triple = "dxil-pc-shadermodel6.7-library" +; Function Attrs: noinline nounwind optnone +define noundef i16 @umad_ushort(i16 noundef %p0, i16 noundef %p1, i16 noundef %p2) #0 { +entry: + %p2.addr = alloca i16, align 2 + %p1.addr = alloca i16, align 2 + %p0.addr = alloca i16, align 2 + store i16 %p2, ptr %p2.addr, align 2 + store i16 %p1, ptr %p1.addr, align 2 + store i16 %p0, ptr %p0.addr, align 2 + %0 = load i16, ptr %p0.addr, align 2 + %1 = load i16, ptr %p1.addr, align 2 + %2 = load i16, ptr %p2.addr, align 2 + %dx.umad = call i16 @llvm.dx.umad.i16(i16 %0, i16 %1, i16 %2) + ret i16 %dx.umad +} + +; Function Attrs: nocallback nofree nosync nounwind willreturn +declare i16 @llvm.dx.umad.i16(i16, i16, i16) #1 + +; Function Attrs: noinline nounwind optnone +define noundef i32 @umad_uint(i32 noundef %p0, i32 noundef %p1, i32 noundef %p2) #0 { +entry: + %p2.addr = alloca i32, align 4 + %p1.addr = alloca i32, align 4 + %p0.addr = alloca i32, align 4 + store i32 %p2, ptr %p2.addr, align 4 + store i32 %p1, ptr %p1.addr, align 4 + store i32 %p0, ptr %p0.addr, align 4 + %0 = load i32, ptr %p0.addr, align 4 + %1 = load i32, ptr %p1.addr, align 4 + %2 = load i32, ptr %p2.addr, align 4 + %dx.umad = call i32 @llvm.dx.umad.i32(i32 %0, i32 %1, i32 %2) + ret i32 %dx.umad +} + +; Function Attrs: nocallback nofree nosync nounwind willreturn +declare i32 @llvm.dx.umad.i32(i32, i32, i32) #1 + +; Function Attrs: noinline nounwind optnone +define noundef i64 @umad_uint64(i64 noundef %p0, i64 noundef %p1, i64 noundef %p2) #0 { +entry: + %p2.addr = alloca i64, align 8 + %p1.addr = alloca i64, align 8 + %p0.addr = alloca i64, align 8 + store i64 %p2, ptr %p2.addr, align 8 + store i64 %p1, ptr %p1.addr, align 8 + store i64 %p0, ptr %p0.addr, align 8 + %0 = load i64, ptr %p0.addr, align 8 + %1 = load i64, ptr %p1.addr, align 8 + %2 = load i64, ptr %p2.addr, align 8 + %dx.umad = call i64 @llvm.dx.umad.i64(i64 %0, i64 %1, i64 %2) + ret i64 %dx.umad +} + +; Function Attrs: nocallback nofree nosync nounwind willreturn +declare i64 @llvm.dx.umad.i64(i64, i64, i64) #1 -- GitLab From 06fea93341ae7d0d0faa82c4c8704591963c2d8c Mon Sep 17 00:00:00 2001 From: Cyndy Ishida Date: Tue, 5 Mar 2024 09:33:59 -0800 Subject: [PATCH 174/929] [TextAPI] add missing platforms for translating triples to tapi targets --- llvm/lib/TextAPI/Platform.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/llvm/lib/TextAPI/Platform.cpp b/llvm/lib/TextAPI/Platform.cpp index e891594300c2..d7d391102ce6 100644 --- a/llvm/lib/TextAPI/Platform.cpp +++ b/llvm/lib/TextAPI/Platform.cpp @@ -49,7 +49,13 @@ PlatformType mapToPlatformType(const Triple &Target) { case Triple::WatchOS: return Target.isSimulatorEnvironment() ? PLATFORM_WATCHOSSIMULATOR : PLATFORM_WATCHOS; - // TODO: add bridgeOS & driverKit once in llvm::Triple + case Triple::BridgeOS: + return PLATFORM_BRIDGEOS; + case Triple::DriverKit: + return PLATFORM_DRIVERKIT; + case Triple::XROS: + return Target.isSimulatorEnvironment() ? PLATFORM_XROS_SIMULATOR + : PLATFORM_XROS; } } -- GitLab From b2ca23aed802abc43ed216ce9bf4c80c056a04c0 Mon Sep 17 00:00:00 2001 From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com> Date: Tue, 5 Mar 2024 12:42:33 -0500 Subject: [PATCH 175/929] [HLSL] implement exp intrinsic (#83832) This change implements: #70072 - `hlsl_intrinsics.h` - add the `exp` api - `DXIL.td` - add the llvm intrinsic to DXIL opcode lowering mapping. - This change reuses llvm's existing intrinsic `__builtin_elementwise_exp` \ `int_exp` & `__builtin_elementwise_exp2` \ `int_exp2` - This PR is part 1 of 2. - Part 2 requires an intrinsic to instructions lowering. Part2 will expand `int_exp` to ``` A = Builder.CreateFMul(log2eConst, val); int_exp2(A) ``` just like we do in [TranslateExp](https://github.com/microsoft/DirectXShaderCompiler/blob/main/lib/HLSL/HLOperationLower.cpp#L2220C1-L2236C2) --- clang/lib/Headers/hlsl/hlsl_intrinsics.h | 64 ++++++++++++++++++++ clang/test/CodeGenHLSL/builtins/exp.hlsl | 53 ++++++++++++++++ clang/test/CodeGenHLSL/builtins/exp2.hlsl | 53 ++++++++++++++++ clang/test/SemaHLSL/BuiltIns/exp-errors.hlsl | 27 +++++++++ llvm/lib/Target/DirectX/DXIL.td | 3 + llvm/test/CodeGen/DirectX/exp2.ll | 31 ++++++++++ 6 files changed, 231 insertions(+) create mode 100644 clang/test/CodeGenHLSL/builtins/exp.hlsl create mode 100644 clang/test/CodeGenHLSL/builtins/exp2.hlsl create mode 100644 clang/test/SemaHLSL/BuiltIns/exp-errors.hlsl create mode 100644 llvm/test/CodeGen/DirectX/exp2.ll diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_intrinsics.h index b5bef78fae72..ed288fa1b692 100644 --- a/clang/lib/Headers/hlsl/hlsl_intrinsics.h +++ b/clang/lib/Headers/hlsl/hlsl_intrinsics.h @@ -277,6 +277,70 @@ uint64_t dot(uint64_t3, uint64_t3); _HLSL_BUILTIN_ALIAS(__builtin_hlsl_dot) uint64_t dot(uint64_t4, uint64_t4); +//===----------------------------------------------------------------------===// +// exp builtins +//===----------------------------------------------------------------------===// + +/// \fn T exp(T x) +/// \brief Returns the base-e exponential, or \a e**x, of the specified value. +/// \param x The specified input value. +/// +/// The return value is the base-e exponential of the \a x parameter. + +_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_exp) +half exp(half); +_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_exp) +half2 exp(half2); +_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_exp) +half3 exp(half3); +_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_exp) +half4 exp(half4); + +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_exp) +float exp(float); +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_exp) +float2 exp(float2); +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_exp) +float3 exp(float3); +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_exp) +float4 exp(float4); + +//===----------------------------------------------------------------------===// +// exp2 builtins +//===----------------------------------------------------------------------===// + +/// \fn T exp2(T x) +/// \brief Returns the base 2 exponential, or \a 2**x, of the specified value. +/// \param x The specified input value. +/// +/// The base 2 exponential of the \a x parameter. + +_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_exp2) +half exp2(half); +_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_exp2) +half2 exp2(half2); +_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_exp2) +half3 exp2(half3); +_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_exp2) +half4 exp2(half4); + +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_exp2) +float exp2(float); +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_exp2) +float2 exp2(float2); +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_exp2) +float3 exp2(float3); +_HLSL_BUILTIN_ALIAS(__builtin_elementwise_exp2) +float4 exp2(float4); + //===----------------------------------------------------------------------===// // floor builtins //===----------------------------------------------------------------------===// diff --git a/clang/test/CodeGenHLSL/builtins/exp.hlsl b/clang/test/CodeGenHLSL/builtins/exp.hlsl new file mode 100644 index 000000000000..773edbe3364f --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/exp.hlsl @@ -0,0 +1,53 @@ +// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \ +// RUN: dxil-pc-shadermodel6.3-library %s -fnative-half-type \ +// RUN: -emit-llvm -disable-llvm-passes -o - | FileCheck %s \ +// RUN: --check-prefixes=CHECK,NATIVE_HALF +// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \ +// RUN: dxil-pc-shadermodel6.3-library %s -emit-llvm -disable-llvm-passes \ +// RUN: -o - | FileCheck %s --check-prefixes=CHECK,NO_HALF + +// NATIVE_HALF: define noundef half @ +// NATIVE_HALF: %elt.exp = call half @llvm.exp.f16( +// NATIVE_HALF: ret half %elt.exp +// NO_HALF: define noundef float @"?test_exp_half@@YA$halff@$halff@@Z"( +// NO_HALF: %elt.exp = call float @llvm.exp.f32( +// NO_HALF: ret float %elt.exp +half test_exp_half(half p0) { return exp(p0); } +// NATIVE_HALF: define noundef <2 x half> @ +// NATIVE_HALF: %elt.exp = call <2 x half> @llvm.exp.v2f16 +// NATIVE_HALF: ret <2 x half> %elt.exp +// NO_HALF: define noundef <2 x float> @ +// NO_HALF: %elt.exp = call <2 x float> @llvm.exp.v2f32( +// NO_HALF: ret <2 x float> %elt.exp +half2 test_exp_half2(half2 p0) { return exp(p0); } +// NATIVE_HALF: define noundef <3 x half> @ +// NATIVE_HALF: %elt.exp = call <3 x half> @llvm.exp.v3f16 +// NATIVE_HALF: ret <3 x half> %elt.exp +// NO_HALF: define noundef <3 x float> @ +// NO_HALF: %elt.exp = call <3 x float> @llvm.exp.v3f32( +// NO_HALF: ret <3 x float> %elt.exp +half3 test_exp_half3(half3 p0) { return exp(p0); } +// NATIVE_HALF: define noundef <4 x half> @ +// NATIVE_HALF: %elt.exp = call <4 x half> @llvm.exp.v4f16 +// NATIVE_HALF: ret <4 x half> %elt.exp +// NO_HALF: define noundef <4 x float> @ +// NO_HALF: %elt.exp = call <4 x float> @llvm.exp.v4f32( +// NO_HALF: ret <4 x float> %elt.exp +half4 test_exp_half4(half4 p0) { return exp(p0); } + +// CHECK: define noundef float @ +// CHECK: %elt.exp = call float @llvm.exp.f32( +// CHECK: ret float %elt.exp +float test_exp_float(float p0) { return exp(p0); } +// CHECK: define noundef <2 x float> @ +// CHECK: %elt.exp = call <2 x float> @llvm.exp.v2f32 +// CHECK: ret <2 x float> %elt.exp +float2 test_exp_float2(float2 p0) { return exp(p0); } +// CHECK: define noundef <3 x float> @ +// CHECK: %elt.exp = call <3 x float> @llvm.exp.v3f32 +// CHECK: ret <3 x float> %elt.exp +float3 test_exp_float3(float3 p0) { return exp(p0); } +// CHECK: define noundef <4 x float> @ +// CHECK: %elt.exp = call <4 x float> @llvm.exp.v4f32 +// CHECK: ret <4 x float> %elt.exp +float4 test_exp_float4(float4 p0) { return exp(p0); } diff --git a/clang/test/CodeGenHLSL/builtins/exp2.hlsl b/clang/test/CodeGenHLSL/builtins/exp2.hlsl new file mode 100644 index 000000000000..f21cdd95774a --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/exp2.hlsl @@ -0,0 +1,53 @@ +// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \ +// RUN: dxil-pc-shadermodel6.3-library %s -fnative-half-type \ +// RUN: -emit-llvm -disable-llvm-passes -o - | FileCheck %s \ +// RUN: --check-prefixes=CHECK,NATIVE_HALF +// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \ +// RUN: dxil-pc-shadermodel6.3-library %s -emit-llvm -disable-llvm-passes \ +// RUN: -o - | FileCheck %s --check-prefixes=CHECK,NO_HALF + +// NATIVE_HALF: define noundef half @ +// NATIVE_HALF: %elt.exp2 = call half @llvm.exp2.f16( +// NATIVE_HALF: ret half %elt.exp2 +// NO_HALF: define noundef float @"?test_exp2_half@@YA$halff@$halff@@Z"( +// NO_HALF: %elt.exp2 = call float @llvm.exp2.f32( +// NO_HALF: ret float %elt.exp2 +half test_exp2_half(half p0) { return exp2(p0); } +// NATIVE_HALF: define noundef <2 x half> @ +// NATIVE_HALF: %elt.exp2 = call <2 x half> @llvm.exp2.v2f16 +// NATIVE_HALF: ret <2 x half> %elt.exp2 +// NO_HALF: define noundef <2 x float> @ +// NO_HALF: %elt.exp2 = call <2 x float> @llvm.exp2.v2f32( +// NO_HALF: ret <2 x float> %elt.exp2 +half2 test_exp2_half2(half2 p0) { return exp2(p0); } +// NATIVE_HALF: define noundef <3 x half> @ +// NATIVE_HALF: %elt.exp2 = call <3 x half> @llvm.exp2.v3f16 +// NATIVE_HALF: ret <3 x half> %elt.exp2 +// NO_HALF: define noundef <3 x float> @ +// NO_HALF: %elt.exp2 = call <3 x float> @llvm.exp2.v3f32( +// NO_HALF: ret <3 x float> %elt.exp2 +half3 test_exp2_half3(half3 p0) { return exp2(p0); } +// NATIVE_HALF: define noundef <4 x half> @ +// NATIVE_HALF: %elt.exp2 = call <4 x half> @llvm.exp2.v4f16 +// NATIVE_HALF: ret <4 x half> %elt.exp2 +// NO_HALF: define noundef <4 x float> @ +// NO_HALF: %elt.exp2 = call <4 x float> @llvm.exp2.v4f32( +// NO_HALF: ret <4 x float> %elt.exp2 +half4 test_exp2_half4(half4 p0) { return exp2(p0); } + +// CHECK: define noundef float @ +// CHECK: %elt.exp2 = call float @llvm.exp2.f32( +// CHECK: ret float %elt.exp2 +float test_exp2_float(float p0) { return exp2(p0); } +// CHECK: define noundef <2 x float> @ +// CHECK: %elt.exp2 = call <2 x float> @llvm.exp2.v2f32 +// CHECK: ret <2 x float> %elt.exp2 +float2 test_exp2_float2(float2 p0) { return exp2(p0); } +// CHECK: define noundef <3 x float> @ +// CHECK: %elt.exp2 = call <3 x float> @llvm.exp2.v3f32 +// CHECK: ret <3 x float> %elt.exp2 +float3 test_exp2_float3(float3 p0) { return exp2(p0); } +// CHECK: define noundef <4 x float> @ +// CHECK: %elt.exp2 = call <4 x float> @llvm.exp2.v4f32 +// CHECK: ret <4 x float> %elt.exp2 +float4 test_exp2_float4(float4 p0) { return exp2(p0); } diff --git a/clang/test/SemaHLSL/BuiltIns/exp-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/exp-errors.hlsl new file mode 100644 index 000000000000..e2e79abb74a3 --- /dev/null +++ b/clang/test/SemaHLSL/BuiltIns/exp-errors.hlsl @@ -0,0 +1,27 @@ + +// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm -disable-llvm-passes -verify -verify-ignore-unexpected -DTEST_FUNC=__builtin_elementwise_exp +// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm -disable-llvm-passes -verify -verify-ignore-unexpected -DTEST_FUNC=__builtin_elementwise_exp2 +float test_too_few_arg() { + return TEST_FUNC(); + // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} +} + +float2 test_too_many_arg(float2 p0) { + return TEST_FUNC(p0, p0); + // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} +} + +float builtin_bool_to_float_type_promotion(bool p1) { + return TEST_FUNC(p1); + // expected-error@-1 {{1st argument must be a vector, integer or floating point type (was 'bool')}} +} + +float builtin_exp_int_to_float_promotion(int p1) { + return TEST_FUNC(p1); + // expected-error@-1 {{1st argument must be a floating point type (was 'int')}} +} + +float2 builtin_exp_int2_to_float2_promotion(int2 p1) { + return TEST_FUNC(p1); + // expected-error@-1 {{1st argument must be a floating point type (was 'int2' (aka 'vector'))}} +} diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td index 8f73c08a6580..9536a01e125b 100644 --- a/llvm/lib/Target/DirectX/DXIL.td +++ b/llvm/lib/Target/DirectX/DXIL.td @@ -218,6 +218,9 @@ class DXILOpMapping; +def Exp2 : DXILOpMapping<21, unary, int_exp2, + "Returns the base 2 exponential, or 2**x, of the specified value." + "exp2(x) = 2**x.">; def Frac : DXILOpMapping<22, unary, int_dx_frac, "Returns a fraction from 0 to 1 that represents the " "decimal part of the input.">; diff --git a/llvm/test/CodeGen/DirectX/exp2.ll b/llvm/test/CodeGen/DirectX/exp2.ll new file mode 100644 index 000000000000..b70b87dedc4d --- /dev/null +++ b/llvm/test/CodeGen/DirectX/exp2.ll @@ -0,0 +1,31 @@ +; RUN: opt -S -dxil-op-lower < %s | FileCheck %s + +; Make sure dxil operation function calls for exp2 are generated for float and half. +; CHECK:call float @dx.op.unary.f32(i32 21, float %{{.*}}) +; CHECK:call half @dx.op.unary.f16(i32 21, half %{{.*}}) + +target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64" +target triple = "dxil-pc-shadermodel6.7-library" + +; Function Attrs: noinline nounwind optnone +define noundef float @exp2_float(float noundef %a) #0 { +entry: + %a.addr = alloca float, align 4 + store float %a, ptr %a.addr, align 4 + %0 = load float, ptr %a.addr, align 4 + %elt.exp2 = call float @llvm.exp2.f32(float %0) + ret float %elt.exp2 +} + +; Function Attrs: nocallback nofree nosync nounwind readnone speculatable willreturn +declare float @llvm.exp2.f32(float) #1 + +; Function Attrs: noinline nounwind optnone +define noundef half @exp2_half(half noundef %a) #0 { +entry: + %a.addr = alloca half, align 2 + store half %a, ptr %a.addr, align 2 + %0 = load half, ptr %a.addr, align 2 + %elt.exp2 = call half @llvm.exp2.f16(half %0) + ret half %elt.exp2 +} -- GitLab From 2807ea6b8047780b5e66a122faf09fae786c917b Mon Sep 17 00:00:00 2001 From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com> Date: Tue, 5 Mar 2024 12:46:01 -0500 Subject: [PATCH 176/929] [HLSL] implement the any intrinsic (#83903) This PR implements the frontend for #70076 This PR is part 1 of 2. Part 2 requires an intrinsic to instructions lowering. - `Builtins.td` - add an `any` builtin - `CGBuiltin.cpp` add the builtin to intrinsic lowering - `hlsl_basic_types.h` -add the `bool` vectors since that is an input for any - `hlsl_intrinsics.h` - add the `any` api - `SemaChecking.cpp` - addy `any` builtin checking - `IntrinsicsDirectX.td` - add the llvm intrinsic --- clang/include/clang/Basic/Builtins.td | 6 + clang/lib/CodeGen/CGBuiltin.cpp | 6 + clang/lib/Headers/hlsl/hlsl_basic_types.h | 4 +- clang/lib/Headers/hlsl/hlsl_intrinsics.h | 112 +++++++++++ clang/lib/Sema/SemaChecking.cpp | 5 + clang/test/CodeGenHLSL/builtins/any.hlsl | 186 +++++++++++++++++++ clang/test/SemaHLSL/BuiltIns/any-errors.hlsl | 12 ++ llvm/include/llvm/IR/IntrinsicsDirectX.td | 2 + 8 files changed, 332 insertions(+), 1 deletion(-) create mode 100644 clang/test/CodeGenHLSL/builtins/any.hlsl create mode 100644 clang/test/SemaHLSL/BuiltIns/any-errors.hlsl diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td index 26c7be478914..dd9c1bb9e5f8 100644 --- a/clang/include/clang/Basic/Builtins.td +++ b/clang/include/clang/Basic/Builtins.td @@ -4542,6 +4542,12 @@ def GetDeviceSideMangledName : LangBuiltin<"CUDA_LANG"> { } // HLSL +def HLSLAny : LangBuiltin<"HLSL_LANG"> { + let Spellings = ["__builtin_hlsl_elementwise_any"]; + let Attributes = [NoThrow, Const]; + let Prototype = "bool(...)"; +} + def HLSLWaveActiveCountBits : LangBuiltin<"HLSL_LANG"> { let Spellings = ["__builtin_hlsl_wave_active_count_bits"]; let Attributes = [NoThrow, Const]; diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index 191f6d5a9fab..31d13d590bc7 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -17963,6 +17963,12 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID, return nullptr; switch (BuiltinID) { + case Builtin::BI__builtin_hlsl_elementwise_any: { + Value *Op0 = EmitScalarExpr(E->getArg(0)); + return Builder.CreateIntrinsic( + /*ReturnType*/ llvm::Type::getInt1Ty(getLLVMContext()), + Intrinsic::dx_any, ArrayRef{Op0}, nullptr, "dx.any"); + } case Builtin::BI__builtin_hlsl_dot: { Value *Op0 = EmitScalarExpr(E->getArg(0)); Value *Op1 = EmitScalarExpr(E->getArg(1)); diff --git a/clang/lib/Headers/hlsl/hlsl_basic_types.h b/clang/lib/Headers/hlsl/hlsl_basic_types.h index 3d0d296aadca..da6903df65ff 100644 --- a/clang/lib/Headers/hlsl/hlsl_basic_types.h +++ b/clang/lib/Headers/hlsl/hlsl_basic_types.h @@ -42,7 +42,9 @@ typedef vector uint16_t2; typedef vector uint16_t3; typedef vector uint16_t4; #endif - +typedef vector bool2; +typedef vector bool3; +typedef vector bool4; typedef vector int2; typedef vector int3; typedef vector int4; diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_intrinsics.h index ed288fa1b692..b794a96f0bf3 100644 --- a/clang/lib/Headers/hlsl/hlsl_intrinsics.h +++ b/clang/lib/Headers/hlsl/hlsl_intrinsics.h @@ -100,6 +100,118 @@ double3 abs(double3); _HLSL_BUILTIN_ALIAS(__builtin_elementwise_abs) double4 abs(double4); +//===----------------------------------------------------------------------===// +// any builtins +//===----------------------------------------------------------------------===// + +/// \fn bool any(T x) +/// \brief Returns True if any components of the \a x parameter are non-zero; +/// otherwise, false. \param x The input value. + +#ifdef __HLSL_ENABLE_16_BIT +_HLSL_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(int16_t); +_HLSL_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(int16_t2); +_HLSL_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(int16_t3); +_HLSL_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(int16_t4); +_HLSL_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(uint16_t); +_HLSL_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(uint16_t2); +_HLSL_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(uint16_t3); +_HLSL_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(uint16_t4); +#endif + +_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(half); +_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(half2); +_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(half3); +_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(half4); + +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(bool); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(bool2); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(bool3); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(bool4); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) + +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(int); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(int2); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(int3); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(int4); + +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(uint); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(uint2); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(uint3); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(uint4); + +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(float); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(float2); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(float3); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(float4); + +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(int64_t); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(int64_t2); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(int64_t3); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(int64_t4); + +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(uint64_t); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(uint64_t2); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(uint64_t3); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(uint64_t4); + +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(double); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(double2); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(double3); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_any) +bool any(double4); + //===----------------------------------------------------------------------===// // ceil builtins //===----------------------------------------------------------------------===// diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index 85477061d190..c4dee874e65c 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -5271,6 +5271,11 @@ bool CheckAllArgsHaveFloatRepresentation(Sema *S, CallExpr *TheCall) { // returning an ExprError bool Sema::CheckHLSLBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { switch (BuiltinID) { + case Builtin::BI__builtin_hlsl_elementwise_any: { + if (checkArgCount(*this, TheCall, 1)) + return true; + break; + } case Builtin::BI__builtin_hlsl_dot: { if (checkArgCount(*this, TheCall, 2)) return true; diff --git a/clang/test/CodeGenHLSL/builtins/any.hlsl b/clang/test/CodeGenHLSL/builtins/any.hlsl new file mode 100644 index 000000000000..ae348fec756b --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/any.hlsl @@ -0,0 +1,186 @@ +// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \ +// RUN: dxil-pc-shadermodel6.3-library %s -fnative-half-type \ +// RUN: -emit-llvm -disable-llvm-passes -o - | FileCheck %s \ +// RUN: --check-prefixes=CHECK,NATIVE_HALF +// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \ +// RUN: dxil-pc-shadermodel6.3-library %s -emit-llvm -disable-llvm-passes \ +// RUN: -o - | FileCheck %s --check-prefixes=CHECK,NO_HALF + +#ifdef __HLSL_ENABLE_16_BIT +// NATIVE_HALF: define noundef i1 @ +// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.i16 +// NATIVE_HALF: ret i1 %dx.any +bool test_any_int16_t(int16_t p0) { return any(p0); } +// NATIVE_HALF: define noundef i1 @ +// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.v2i16 +// NATIVE_HALF: ret i1 %dx.any +bool test_any_int16_t2(int16_t2 p0) { return any(p0); } +// NATIVE_HALF: define noundef i1 @ +// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.v3i16 +// NATIVE_HALF: ret i1 %dx.any +bool test_any_int16_t3(int16_t3 p0) { return any(p0); } +// NATIVE_HALF: define noundef i1 @ +// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.v4i16 +// NATIVE_HALF: ret i1 %dx.any +bool test_any_int16_t4(int16_t4 p0) { return any(p0); } + +// NATIVE_HALF: define noundef i1 @ +// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.i16 +// NATIVE_HALF: ret i1 %dx.any +bool test_any_uint16_t(uint16_t p0) { return any(p0); } +// NATIVE_HALF: define noundef i1 @ +// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.v2i16 +// NATIVE_HALF: ret i1 %dx.any +bool test_any_uint16_t2(uint16_t2 p0) { return any(p0); } +// NATIVE_HALF: define noundef i1 @ +// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.v3i16 +// NATIVE_HALF: ret i1 %dx.any +bool test_any_uint16_t3(uint16_t3 p0) { return any(p0); } +// NATIVE_HALF: define noundef i1 @ +// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.v4i16 +// NATIVE_HALF: ret i1 %dx.any +bool test_any_uint16_t4(uint16_t4 p0) { return any(p0); } +#endif // __HLSL_ENABLE_16_BIT + +// CHECK: define noundef i1 @ +// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.f16 +// NO_HALF: %dx.any = call i1 @llvm.dx.any.f32 +// CHECK: ret i1 %dx.any +bool test_any_half(half p0) { return any(p0); } + +// CHECK: define noundef i1 @ +// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.v2f16 +// NO_HALF: %dx.any = call i1 @llvm.dx.any.v2f32 +// CHECK: ret i1 %dx.any +bool test_any_half2(half2 p0) { return any(p0); } + +// CHECK: define noundef i1 @ +// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.v3f16 +// NO_HALF: %dx.any = call i1 @llvm.dx.any.v3f32 +// CHECK: ret i1 %dx.any +bool test_any_half3(half3 p0) { return any(p0); } + +// CHECK: define noundef i1 @ +// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.v4f16 +// NO_HALF: %dx.any = call i1 @llvm.dx.any.v4f32 +// CHECK: ret i1 %dx.any +bool test_any_half4(half4 p0) { return any(p0); } + +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.f32 +// CHECK: ret i1 %dx.any +bool test_any_float(float p0) { return any(p0); } +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.v2f32 +// CHECK: ret i1 %dx.any +bool test_any_float2(float2 p0) { return any(p0); } +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.v3f32 +// CHECK: ret i1 %dx.any +bool test_any_float3(float3 p0) { return any(p0); } +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.v4f32 +// CHECK: ret i1 %dx.any +bool test_any_float4(float4 p0) { return any(p0); } + +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.f64 +// CHECK: ret i1 %dx.any +bool test_any_double(double p0) { return any(p0); } +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.v2f64 +// CHECK: ret i1 %dx.any +bool test_any_double2(double2 p0) { return any(p0); } +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.v3f64 +// CHECK: ret i1 %dx.any +bool test_any_double3(double3 p0) { return any(p0); } +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.v4f64 +// CHECK: ret i1 %dx.any +bool test_any_double4(double4 p0) { return any(p0); } + +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.i32 +// CHECK: ret i1 %dx.any +bool test_any_int(int p0) { return any(p0); } +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.v2i32 +// CHECK: ret i1 %dx.any +bool test_any_int2(int2 p0) { return any(p0); } +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.v3i32 +// CHECK: ret i1 %dx.any +bool test_any_int3(int3 p0) { return any(p0); } +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.v4i32 +// CHECK: ret i1 %dx.any +bool test_any_int4(int4 p0) { return any(p0); } + +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.i32 +// CHECK: ret i1 %dx.any +bool test_any_uint(uint p0) { return any(p0); } +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.v2i32 +// CHECK: ret i1 %dx.any +bool test_any_uint2(uint2 p0) { return any(p0); } +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.v3i32 +// CHECK: ret i1 %dx.any +bool test_any_uint3(uint3 p0) { return any(p0); } +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.v4i32 +// CHECK: ret i1 %dx.any +bool test_any_uint4(uint4 p0) { return any(p0); } + +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.i64 +// CHECK: ret i1 %dx.any +bool test_any_int64_t(int64_t p0) { return any(p0); } +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.v2i64 +// CHECK: ret i1 %dx.any +bool test_any_int64_t2(int64_t2 p0) { return any(p0); } +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.v3i64 +// CHECK: ret i1 %dx.any +bool test_any_int64_t3(int64_t3 p0) { return any(p0); } +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.v4i64 +// CHECK: ret i1 %dx.any +bool test_any_int64_t4(int64_t4 p0) { return any(p0); } + +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.i64 +// CHECK: ret i1 %dx.any +bool test_any_uint64_t(uint64_t p0) { return any(p0); } +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.v2i64 +// CHECK: ret i1 %dx.any +bool test_any_uint64_t2(uint64_t2 p0) { return any(p0); } +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.v3i64 +// CHECK: ret i1 %dx.any +bool test_any_uint64_t3(uint64_t3 p0) { return any(p0); } +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.v4i64 +// CHECK: ret i1 %dx.any +bool test_any_uint64_t4(uint64_t4 p0) { return any(p0); } + +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.i1 +// CHECK: ret i1 %dx.any +bool test_any_bool(bool p0) { return any(p0); } +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.v2i1 +// CHECK: ret i1 %dx.any +bool test_any_bool2(bool2 p0) { return any(p0); } +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.v3i1 +// CHECK: ret i1 %dx.any +bool test_any_bool3(bool3 p0) { return any(p0); } +// CHECK: define noundef i1 @ +// CHECK: %dx.any = call i1 @llvm.dx.any.v4i1 +// CHECK: ret i1 %dx.any +bool test_any_bool4(bool4 p0) { return any(p0); } diff --git a/clang/test/SemaHLSL/BuiltIns/any-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/any-errors.hlsl new file mode 100644 index 000000000000..862b94652073 --- /dev/null +++ b/clang/test/SemaHLSL/BuiltIns/any-errors.hlsl @@ -0,0 +1,12 @@ + +// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm -disable-llvm-passes -verify -verify-ignore-unexpected + +bool test_too_few_arg() { + return __builtin_hlsl_elementwise_any(); + // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} +} + +bool test_too_many_arg(float2 p0) { + return __builtin_hlsl_elementwise_any(p0, p0); + // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} +} diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td index 3096442335ce..acf45b6b3172 100644 --- a/llvm/include/llvm/IR/IntrinsicsDirectX.td +++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td @@ -20,6 +20,8 @@ def int_dx_flattened_thread_id_in_group : Intrinsic<[llvm_i32_ty], [], [IntrNoMe def int_dx_create_handle : ClangBuiltin<"__builtin_hlsl_create_handle">, Intrinsic<[ llvm_ptr_ty ], [llvm_i8_ty], [IntrWillReturn]>; +def int_dx_any : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_any_ty]>; + def int_dx_dot : Intrinsic<[LLVMVectorElementType<0>], [llvm_anyvector_ty, LLVMScalarOrSameVectorWidth<0, LLVMVectorElementType<0>>], -- GitLab From 9f67f19614e952ede385a59bb62f7b57771ca4c3 Mon Sep 17 00:00:00 2001 From: Nico Weber Date: Tue, 5 Mar 2024 12:41:43 -0500 Subject: [PATCH 177/929] [gn] Remove ScudoBenchmarks It's the only thing depending on third-party/benchmark. The recent third-party/benchmark uprev made it not build in the GN build, so remove ScudoBenchmarks until someone feels motivated to update the third-party/benchmark BUILD.gn file. --- .../secondary/compiler-rt/lib/scudo/BUILD.gn | 1 - .../lib/scudo/standalone/benchmarks/BUILD.gn | 8 ----- .../secondary/third-party/benchmark/BUILD.gn | 29 ------------------- 3 files changed, 38 deletions(-) delete mode 100644 llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/benchmarks/BUILD.gn delete mode 100644 llvm/utils/gn/secondary/third-party/benchmark/BUILD.gn diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/scudo/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/scudo/BUILD.gn index 2fe08699bca3..ee244782651b 100644 --- a/llvm/utils/gn/secondary/compiler-rt/lib/scudo/BUILD.gn +++ b/llvm/utils/gn/secondary/compiler-rt/lib/scudo/BUILD.gn @@ -11,7 +11,6 @@ group("scudo") { deps = [] foreach(toolchain, supported_toolchains) { deps += [ - "standalone/benchmarks:ScudoBenchmarks($toolchain)", "standalone/tests:ScudoCUnitTest($toolchain)", "standalone/tests:ScudoCxxUnitTest($toolchain)", "standalone/tests:ScudoHooksUnitTest($toolchain)", diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/benchmarks/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/benchmarks/BUILD.gn deleted file mode 100644 index 9489f68af363..000000000000 --- a/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/benchmarks/BUILD.gn +++ /dev/null @@ -1,8 +0,0 @@ -executable("ScudoBenchmarks") { - configs += [ "//llvm/utils/gn/build:crt_code" ] - sources = [ "malloc_benchmark.cpp" ] - deps = [ - "//compiler-rt/lib/scudo/standalone:sources", - "//third-party/benchmark", - ] -} diff --git a/llvm/utils/gn/secondary/third-party/benchmark/BUILD.gn b/llvm/utils/gn/secondary/third-party/benchmark/BUILD.gn deleted file mode 100644 index 51627ecd0604..000000000000 --- a/llvm/utils/gn/secondary/third-party/benchmark/BUILD.gn +++ /dev/null @@ -1,29 +0,0 @@ -static_library("benchmark") { - sources = [ - "src/benchmark.cc", - "src/benchmark_api_internal.cc", - "src/benchmark_name.cc", - "src/benchmark_register.cc", - "src/benchmark_runner.cc", - "src/colorprint.cc", - "src/commandlineflags.cc", - "src/complexity.cc", - "src/console_reporter.cc", - "src/counter.cc", - "src/csv_reporter.cc", - "src/json_reporter.cc", - "src/perf_counters.cc", - "src/reporter.cc", - "src/sleep.cc", - "src/statistics.cc", - "src/string_util.cc", - "src/sysinfo.cc", - "src/timers.cc", - ] - deps = [ "//llvm/utils/gn/build/libs/pthread" ] - public_configs = [ ":benchmark_config" ] -} - -config("benchmark_config") { - include_dirs = [ "include" ] -} -- GitLab From 233f750c3dad14e330f5358e8dbcc6c30e805edb Mon Sep 17 00:00:00 2001 From: Peter Klausler <35819229+klausler@users.noreply.github.com> Date: Tue, 5 Mar 2024 09:48:45 -0800 Subject: [PATCH 178/929] [flang] Catch more bad pointer initialization targets (#83731) A pointer variable initialization or pointer component default initialization cannot reference another pointer. Fixes https://github.com/llvm/llvm-project/issues/82944. --- flang/lib/Evaluate/check-expression.cpp | 37 ++++++++-------- flang/test/Semantics/init01.f90 | 57 +++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 18 deletions(-) diff --git a/flang/lib/Evaluate/check-expression.cpp b/flang/lib/Evaluate/check-expression.cpp index 14abac5ff9ba..0e7d97900328 100644 --- a/flang/lib/Evaluate/check-expression.cpp +++ b/flang/lib/Evaluate/check-expression.cpp @@ -250,6 +250,8 @@ public: } } return false; + } else if (!CheckVarOrComponent(ultimate)) { + return false; } else if (!ultimate.attrs().test(semantics::Attr::TARGET)) { if (messages_) { messages_->Say( @@ -267,7 +269,7 @@ public: } return false; } else { - return CheckVarOrComponent(ultimate); + return true; } } bool operator()(const StaticDataObject &) const { return false; } @@ -318,24 +320,23 @@ public: private: bool CheckVarOrComponent(const semantics::Symbol &symbol) { const Symbol &ultimate{symbol.GetUltimate()}; - if (IsAllocatable(ultimate)) { - if (messages_) { - messages_->Say( - "An initial data target may not be a reference to an ALLOCATABLE '%s'"_err_en_US, - ultimate.name()); - emittedMessage_ = true; - } - return false; - } else if (ultimate.Corank() > 0) { - if (messages_) { - messages_->Say( - "An initial data target may not be a reference to a coarray '%s'"_err_en_US, - ultimate.name()); - emittedMessage_ = true; - } - return false; + const char *unacceptable{nullptr}; + if (ultimate.Corank() > 0) { + unacceptable = "a coarray"; + } else if (IsAllocatable(ultimate)) { + unacceptable = "an ALLOCATABLE"; + } else if (IsPointer(ultimate)) { + unacceptable = "a POINTER"; + } else { + return true; } - return true; + if (messages_) { + messages_->Say( + "An initial data target may not be a reference to %s '%s'"_err_en_US, + unacceptable, ultimate.name()); + emittedMessage_ = true; + } + return false; } parser::ContextualMessages *messages_; diff --git a/flang/test/Semantics/init01.f90 b/flang/test/Semantics/init01.f90 index f58c034d5dea..f85feef097cd 100644 --- a/flang/test/Semantics/init01.f90 +++ b/flang/test/Semantics/init01.f90 @@ -8,6 +8,17 @@ subroutine objectpointers(j) real, save :: x3 real, target :: x4 real, target, save :: x5(10) + real, pointer :: x6 + type t1 + real, allocatable :: c1 + real, allocatable, codimension[:] :: c2 + real :: c3 + real :: c4(10) + real, pointer :: c5 + end type + type(t1), target, save :: o1 + type(t1), save :: o2 + type(t1), target :: o3 !ERROR: An initial data target may not be a reference to an ALLOCATABLE 'x1' real, pointer :: p1 => x1 !ERROR: An initial data target may not be a reference to a coarray 'x2' @@ -20,6 +31,52 @@ subroutine objectpointers(j) real, pointer :: p5 => x5(j) !ERROR: Pointer has rank 0 but target has rank 1 real, pointer :: p6 => x5 +!ERROR: An initial data target may not be a reference to a POINTER 'x6' + real, pointer :: p7 => x6 +!ERROR: An initial data target may not be a reference to an ALLOCATABLE 'c1' + real, pointer :: p1o => o1%c1 +!ERROR: An initial data target may not be a reference to a coarray 'c2' + real, pointer :: p2o => o1%c2 +!ERROR: An initial data target may not be a reference to an object 'o2' that lacks the TARGET attribute + real, pointer :: p3o => o2%c3 +!ERROR: An initial data target may not be a reference to an object 'o3' that lacks the SAVE attribute + real, pointer :: p4o => o3%c3 +!ERROR: An initial data target must be a designator with constant subscripts + real, pointer :: p5o => o1%c4(j) +!ERROR: Pointer has rank 0 but target has rank 1 + real, pointer :: p6o => o1%c4 +!ERROR: An initial data target may not be a reference to a POINTER 'c5' + real, pointer :: p7o => o1%c5 + type t2 + !ERROR: An initial data target may not be a reference to an ALLOCATABLE 'x1' + real, pointer :: p1 => x1 + !ERROR: An initial data target may not be a reference to a coarray 'x2' + real, pointer :: p2 => x2 + !ERROR: An initial data target may not be a reference to an object 'x3' that lacks the TARGET attribute + real, pointer :: p3 => x3 + !ERROR: An initial data target may not be a reference to an object 'x4' that lacks the SAVE attribute + real, pointer :: p4 => x4 + !ERROR: An initial data target must be a designator with constant subscripts + real, pointer :: p5 => x5(j) + !ERROR: Pointer has rank 0 but target has rank 1 + real, pointer :: p6 => x5 + !ERROR: An initial data target may not be a reference to a POINTER 'x6' + real, pointer :: p7 => x6 + !ERROR: An initial data target may not be a reference to an ALLOCATABLE 'c1' + real, pointer :: p1o => o1%c1 + !ERROR: An initial data target may not be a reference to a coarray 'c2' + real, pointer :: p2o => o1%c2 + !ERROR: An initial data target may not be a reference to an object 'o2' that lacks the TARGET attribute + real, pointer :: p3o => o2%c3 + !ERROR: An initial data target may not be a reference to an object 'o3' that lacks the SAVE attribute + real, pointer :: p4o => o3%c3 + !ERROR: An initial data target must be a designator with constant subscripts + real, pointer :: p5o => o1%c4(j) + !ERROR: Pointer has rank 0 but target has rank 1 + real, pointer :: p6o => o1%c4 + !ERROR: An initial data target may not be a reference to a POINTER 'c5' + real, pointer :: p7o => o1%c5 + end type !TODO: type incompatibility, non-deferred type parameter values, contiguity -- GitLab From 041638c4294a9a8375851e0add1ab2c99412c032 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Tue, 5 Mar 2024 09:49:41 -0800 Subject: [PATCH 179/929] [libc][stdbit] implement stdc_bit_width (C23) (#83892) --- libc/config/linux/x86_64/entrypoints.txt | 5 +++++ libc/docs/stdbit.rst | 12 +++++------ libc/include/llvm-libc-macros/stdbit-macros.h | 16 ++++++++++++++ libc/spec/stdc.td | 10 +++++++-- libc/src/stdbit/CMakeLists.txt | 1 + libc/src/stdbit/stdc_bit_width_uc.cpp | 20 ++++++++++++++++++ libc/src/stdbit/stdc_bit_width_uc.h | 18 ++++++++++++++++ libc/src/stdbit/stdc_bit_width_ui.cpp | 20 ++++++++++++++++++ libc/src/stdbit/stdc_bit_width_ui.h | 18 ++++++++++++++++ libc/src/stdbit/stdc_bit_width_ul.cpp | 20 ++++++++++++++++++ libc/src/stdbit/stdc_bit_width_ul.h | 18 ++++++++++++++++ libc/src/stdbit/stdc_bit_width_ull.cpp | 20 ++++++++++++++++++ libc/src/stdbit/stdc_bit_width_ull.h | 18 ++++++++++++++++ libc/src/stdbit/stdc_bit_width_us.cpp | 20 ++++++++++++++++++ libc/src/stdbit/stdc_bit_width_us.h | 18 ++++++++++++++++ libc/test/include/stdbit_test.cpp | 13 ++++++++++++ libc/test/src/stdbit/CMakeLists.txt | 1 + .../src/stdbit/stdc_bit_width_uc_test.cpp | 21 +++++++++++++++++++ .../src/stdbit/stdc_bit_width_ui_test.cpp | 20 ++++++++++++++++++ .../src/stdbit/stdc_bit_width_ul_test.cpp | 21 +++++++++++++++++++ .../src/stdbit/stdc_bit_width_ull_test.cpp | 21 +++++++++++++++++++ .../src/stdbit/stdc_bit_width_us_test.cpp | 21 +++++++++++++++++++ 22 files changed, 344 insertions(+), 8 deletions(-) create mode 100644 libc/src/stdbit/stdc_bit_width_uc.cpp create mode 100644 libc/src/stdbit/stdc_bit_width_uc.h create mode 100644 libc/src/stdbit/stdc_bit_width_ui.cpp create mode 100644 libc/src/stdbit/stdc_bit_width_ui.h create mode 100644 libc/src/stdbit/stdc_bit_width_ul.cpp create mode 100644 libc/src/stdbit/stdc_bit_width_ul.h create mode 100644 libc/src/stdbit/stdc_bit_width_ull.cpp create mode 100644 libc/src/stdbit/stdc_bit_width_ull.h create mode 100644 libc/src/stdbit/stdc_bit_width_us.cpp create mode 100644 libc/src/stdbit/stdc_bit_width_us.h create mode 100644 libc/test/src/stdbit/stdc_bit_width_uc_test.cpp create mode 100644 libc/test/src/stdbit/stdc_bit_width_ui_test.cpp create mode 100644 libc/test/src/stdbit/stdc_bit_width_ul_test.cpp create mode 100644 libc/test/src/stdbit/stdc_bit_width_ull_test.cpp create mode 100644 libc/test/src/stdbit/stdc_bit_width_us_test.cpp diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index bc10512d942f..a6c3041773df 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -147,6 +147,11 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdbit.stdc_has_single_bit_ui libc.src.stdbit.stdc_has_single_bit_ul libc.src.stdbit.stdc_has_single_bit_ull + libc.src.stdbit.stdc_bit_width_uc + libc.src.stdbit.stdc_bit_width_us + libc.src.stdbit.stdc_bit_width_ui + libc.src.stdbit.stdc_bit_width_ul + libc.src.stdbit.stdc_bit_width_ull # stdlib.h entrypoints libc.src.stdlib.abs diff --git a/libc/docs/stdbit.rst b/libc/docs/stdbit.rst index b579e9dbbc2f..ccd1393ef5d9 100644 --- a/libc/docs/stdbit.rst +++ b/libc/docs/stdbit.rst @@ -86,11 +86,11 @@ stdc_has_single_bit_us |check| stdc_has_single_bit_ui |check| stdc_has_single_bit_ul |check| stdc_has_single_bit_ull |check| -stdc_bit_width_uc -stdc_bit_width_us -stdc_bit_width_ui -stdc_bit_width_ul -stdc_bit_width_ull +stdc_bit_width_uc |check| +stdc_bit_width_us |check| +stdc_bit_width_ui |check| +stdc_bit_width_ul |check| +stdc_bit_width_ull |check| stdc_bit_floor_uc stdc_bit_floor_us stdc_bit_floor_ui @@ -125,7 +125,7 @@ stdc_first_trailing_one |check| stdc_count_zeros |check| stdc_count_ones |check| stdc_has_single_bit |check| -stdc_bit_width +stdc_bit_width |check| stdc_bit_floor stdc_bit_ceil ========================= ========= diff --git a/libc/include/llvm-libc-macros/stdbit-macros.h b/libc/include/llvm-libc-macros/stdbit-macros.h index e3a36d10ed92..104418ca4856 100644 --- a/libc/include/llvm-libc-macros/stdbit-macros.h +++ b/libc/include/llvm-libc-macros/stdbit-macros.h @@ -172,6 +172,15 @@ inline bool stdc_has_single_bit(unsigned long x) { inline bool stdc_has_single_bit(unsigned long long x) { return stdc_has_single_bit_ull(x); } +inline unsigned stdc_bit_width(unsigned char x) { return stdc_bit_width_uc(x); } +inline unsigned stdc_bit_width(unsigned short x) { + return stdc_bit_width_us(x); +} +inline unsigned stdc_bit_width(unsigned x) { return stdc_bit_width_ui(x); } +inline unsigned stdc_bit_width(unsigned long x) { return stdc_bit_width_ul(x); } +inline unsigned stdc_bit_width(unsigned long long x) { + return stdc_bit_width_ull(x); +} #else #define stdc_leading_zeros(x) \ _Generic((x), \ @@ -250,6 +259,13 @@ inline bool stdc_has_single_bit(unsigned long long x) { unsigned: stdc_has_single_bit_ui, \ unsigned long: stdc_has_single_bit_ul, \ unsigned long long: stdc_has_single_bit_ull)(x) +#define stdc_bit_width(x) \ + _Generic((x), \ + unsigned char: stdc_bit_width_ui, \ + unsigned short: stdc_bit_width_us, \ + unsigned: stdc_bit_width_ui, \ + unsigned long: stdc_bit_width_ul, \ + unsigned long long: stdc_bit_width_ull)(x) #endif // __cplusplus #endif // __LLVM_LIBC_MACROS_STDBIT_MACROS_H diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index fc5a2f78f7bd..cfebc60a0a9a 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -800,7 +800,8 @@ def StdC : StandardSpec<"stdc"> { Macro<"stdc_first_trailing_one">, Macro<"stdc_count_zeros">, Macro<"stdc_count_ones">, - Macro<"stdc_has_single_bit"> + Macro<"stdc_has_single_bit">, + Macro<"std_bit_width"> ], // Macros [], // Types [], // Enumerations @@ -854,7 +855,12 @@ def StdC : StandardSpec<"stdc"> { FunctionSpec<"stdc_has_single_bit_us", RetValSpec, [ArgSpec]>, FunctionSpec<"stdc_has_single_bit_ui", RetValSpec, [ArgSpec]>, FunctionSpec<"stdc_has_single_bit_ul", RetValSpec, [ArgSpec]>, - FunctionSpec<"stdc_has_single_bit_ull", RetValSpec, [ArgSpec]> + FunctionSpec<"stdc_has_single_bit_ull", RetValSpec, [ArgSpec]>, + FunctionSpec<"stdc_bit_width_uc", RetValSpec, [ArgSpec]>, + FunctionSpec<"stdc_bit_width_us", RetValSpec, [ArgSpec]>, + FunctionSpec<"stdc_bit_width_ui", RetValSpec, [ArgSpec]>, + FunctionSpec<"stdc_bit_width_ul", RetValSpec, [ArgSpec]>, + FunctionSpec<"stdc_bit_width_ull", RetValSpec, [ArgSpec]> ] // Functions >; diff --git a/libc/src/stdbit/CMakeLists.txt b/libc/src/stdbit/CMakeLists.txt index 8bc7dd7852bb..f077baeee6d2 100644 --- a/libc/src/stdbit/CMakeLists.txt +++ b/libc/src/stdbit/CMakeLists.txt @@ -10,6 +10,7 @@ set(prefixes count_zeros count_ones has_single_bit + bit_width ) set(suffixes c s i l ll) foreach(prefix IN LISTS prefixes) diff --git a/libc/src/stdbit/stdc_bit_width_uc.cpp b/libc/src/stdbit/stdc_bit_width_uc.cpp new file mode 100644 index 000000000000..2c361c1bbb1c --- /dev/null +++ b/libc/src/stdbit/stdc_bit_width_uc.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of stdc_bit_width_uc -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdbit/stdc_bit_width_uc.h" + +#include "src/__support/CPP/bit.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(unsigned, stdc_bit_width_uc, (unsigned char value)) { + return static_cast(cpp::bit_width(value)); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_bit_width_uc.h b/libc/src/stdbit/stdc_bit_width_uc.h new file mode 100644 index 000000000000..70c038aaf1df --- /dev/null +++ b/libc/src/stdbit/stdc_bit_width_uc.h @@ -0,0 +1,18 @@ +//===-- Implementation header for stdc_bit_width_uc -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDBIT_STDC_BIT_WIDTH_UC_H +#define LLVM_LIBC_SRC_STDBIT_STDC_BIT_WIDTH_UC_H + +namespace LIBC_NAMESPACE { + +unsigned stdc_bit_width_uc(unsigned char value); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDBIT_STDC_BIT_WIDTH_UC_H diff --git a/libc/src/stdbit/stdc_bit_width_ui.cpp b/libc/src/stdbit/stdc_bit_width_ui.cpp new file mode 100644 index 000000000000..b94452b09bd5 --- /dev/null +++ b/libc/src/stdbit/stdc_bit_width_ui.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of stdc_bit_width_ui -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdbit/stdc_bit_width_ui.h" + +#include "src/__support/CPP/bit.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(unsigned, stdc_bit_width_ui, (unsigned value)) { + return static_cast(cpp::bit_width(value)); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_bit_width_ui.h b/libc/src/stdbit/stdc_bit_width_ui.h new file mode 100644 index 000000000000..9e8de3d6ef46 --- /dev/null +++ b/libc/src/stdbit/stdc_bit_width_ui.h @@ -0,0 +1,18 @@ +//===-- Implementation header for stdc_bit_width_ui -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDBIT_STDC_BIT_WIDTH_UI_H +#define LLVM_LIBC_SRC_STDBIT_STDC_BIT_WIDTH_UI_H + +namespace LIBC_NAMESPACE { + +unsigned stdc_bit_width_ui(unsigned value); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDBIT_STDC_BIT_WIDTH_UI_H diff --git a/libc/src/stdbit/stdc_bit_width_ul.cpp b/libc/src/stdbit/stdc_bit_width_ul.cpp new file mode 100644 index 000000000000..80044314e4b2 --- /dev/null +++ b/libc/src/stdbit/stdc_bit_width_ul.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of stdc_bit_width_ul -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdbit/stdc_bit_width_ul.h" + +#include "src/__support/CPP/bit.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(unsigned, stdc_bit_width_ul, (unsigned long value)) { + return static_cast(cpp::bit_width(value)); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_bit_width_ul.h b/libc/src/stdbit/stdc_bit_width_ul.h new file mode 100644 index 000000000000..447a2918e2f2 --- /dev/null +++ b/libc/src/stdbit/stdc_bit_width_ul.h @@ -0,0 +1,18 @@ +//===-- Implementation header for stdc_bit_width_ul -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDBIT_STDC_BIT_WIDTH_UL_H +#define LLVM_LIBC_SRC_STDBIT_STDC_BIT_WIDTH_UL_H + +namespace LIBC_NAMESPACE { + +unsigned stdc_bit_width_ul(unsigned long value); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDBIT_STDC_BIT_WIDTH_UL_H diff --git a/libc/src/stdbit/stdc_bit_width_ull.cpp b/libc/src/stdbit/stdc_bit_width_ull.cpp new file mode 100644 index 000000000000..006fa20b2de5 --- /dev/null +++ b/libc/src/stdbit/stdc_bit_width_ull.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of stdc_bit_width_ull ------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdbit/stdc_bit_width_ull.h" + +#include "src/__support/CPP/bit.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(unsigned, stdc_bit_width_ull, (unsigned long long value)) { + return static_cast(cpp::bit_width(value)); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_bit_width_ull.h b/libc/src/stdbit/stdc_bit_width_ull.h new file mode 100644 index 000000000000..bc51897f448f --- /dev/null +++ b/libc/src/stdbit/stdc_bit_width_ull.h @@ -0,0 +1,18 @@ +//===-- Implementation header for stdc_bit_width_ull ------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDBIT_STDC_BIT_WIDTH_ULL_H +#define LLVM_LIBC_SRC_STDBIT_STDC_BIT_WIDTH_ULL_H + +namespace LIBC_NAMESPACE { + +unsigned stdc_bit_width_ull(unsigned long long value); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDBIT_STDC_BIT_WIDTH_ULL_H diff --git a/libc/src/stdbit/stdc_bit_width_us.cpp b/libc/src/stdbit/stdc_bit_width_us.cpp new file mode 100644 index 000000000000..3d9f72bf5d06 --- /dev/null +++ b/libc/src/stdbit/stdc_bit_width_us.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of stdc_bit_width_us -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdbit/stdc_bit_width_us.h" + +#include "src/__support/CPP/bit.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(unsigned, stdc_bit_width_us, (unsigned short value)) { + return static_cast(cpp::bit_width(value)); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_bit_width_us.h b/libc/src/stdbit/stdc_bit_width_us.h new file mode 100644 index 000000000000..02cd37426eb4 --- /dev/null +++ b/libc/src/stdbit/stdc_bit_width_us.h @@ -0,0 +1,18 @@ +//===-- Implementation header for stdc_bit_width_us -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDBIT_STDC_BIT_WIDTH_US_H +#define LLVM_LIBC_SRC_STDBIT_STDC_BIT_WIDTH_US_H + +namespace LIBC_NAMESPACE { + +unsigned stdc_bit_width_us(unsigned short value); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDBIT_STDC_BIT_WIDTH_US_H diff --git a/libc/test/include/stdbit_test.cpp b/libc/test/include/stdbit_test.cpp index 16dcd8e085f4..dfb7c97e3d9e 100644 --- a/libc/test/include/stdbit_test.cpp +++ b/libc/test/include/stdbit_test.cpp @@ -86,6 +86,11 @@ bool stdc_has_single_bit_us(unsigned short) noexcept { return false; } bool stdc_has_single_bit_ui(unsigned) noexcept { return false; } bool stdc_has_single_bit_ul(unsigned long) noexcept { return false; } bool stdc_has_single_bit_ull(unsigned long long) noexcept { return false; } +unsigned stdc_bit_width_uc(unsigned char) noexcept { return 0x4AU; } +unsigned stdc_bit_width_us(unsigned short) noexcept { return 0x4BU; } +unsigned stdc_bit_width_ui(unsigned) noexcept { return 0x4CU; } +unsigned stdc_bit_width_ul(unsigned long) noexcept { return 0x4DU; } +unsigned stdc_bit_width_ull(unsigned long long) noexcept { return 0x4EU; } } #include "include/llvm-libc-macros/stdbit-macros.h" @@ -177,3 +182,11 @@ TEST(LlvmLibcStdbitTest, TypeGenericMacroHasSingleBit) { EXPECT_EQ(stdc_has_single_bit(1UL), false); EXPECT_EQ(stdc_has_single_bit(1ULL), false); } + +TEST(LlvmLibcStdbitTest, TypeGenericMacroBitWidth) { + EXPECT_EQ(stdc_bit_width(static_cast(1U)), 0x4AU); + EXPECT_EQ(stdc_bit_width(static_cast(1U)), 0x4BU); + EXPECT_EQ(stdc_bit_width(1U), 0x4CU); + EXPECT_EQ(stdc_bit_width(1UL), 0x4DU); + EXPECT_EQ(stdc_bit_width(1ULL), 0x4EU); +} diff --git a/libc/test/src/stdbit/CMakeLists.txt b/libc/test/src/stdbit/CMakeLists.txt index a886ee4a3532..f7e17d732299 100644 --- a/libc/test/src/stdbit/CMakeLists.txt +++ b/libc/test/src/stdbit/CMakeLists.txt @@ -12,6 +12,7 @@ set(prefixes count_zeros count_ones has_single_bit + bit_width ) set(suffixes c s i l ll) foreach(prefix IN LISTS prefixes) diff --git a/libc/test/src/stdbit/stdc_bit_width_uc_test.cpp b/libc/test/src/stdbit/stdc_bit_width_uc_test.cpp new file mode 100644 index 000000000000..63c6503542b1 --- /dev/null +++ b/libc/test/src/stdbit/stdc_bit_width_uc_test.cpp @@ -0,0 +1,21 @@ +//===-- Unittests for stdc_bit_width_uc -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/CPP/limits.h" +#include "src/stdbit/stdc_bit_width_uc.h" +#include "test/UnitTest/Test.h" + +TEST(LlvmLibcStdcBitWidthUcTest, Zero) { + EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_width_uc(0U), 0U); +} + +TEST(LlvmLibcStdcBitWidthUcTest, Ones) { + for (unsigned i = 0U; i != UCHAR_WIDTH; ++i) + EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_width_uc(UCHAR_MAX >> i), + UCHAR_WIDTH - i); +} diff --git a/libc/test/src/stdbit/stdc_bit_width_ui_test.cpp b/libc/test/src/stdbit/stdc_bit_width_ui_test.cpp new file mode 100644 index 000000000000..43acdde5dd20 --- /dev/null +++ b/libc/test/src/stdbit/stdc_bit_width_ui_test.cpp @@ -0,0 +1,20 @@ +//===-- Unittests for stdc_bit_width_ui -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/CPP/limits.h" +#include "src/stdbit/stdc_bit_width_ui.h" +#include "test/UnitTest/Test.h" + +TEST(LlvmLibcStdcBitWidthUiTest, Zero) { + EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_width_ui(0U), 0U); +} + +TEST(LlvmLibcStdcBitWidthUiTest, Ones) { + for (unsigned i = 0U; i != UINT_WIDTH; ++i) + EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_width_ui(UINT_MAX >> i), UINT_WIDTH - i); +} diff --git a/libc/test/src/stdbit/stdc_bit_width_ul_test.cpp b/libc/test/src/stdbit/stdc_bit_width_ul_test.cpp new file mode 100644 index 000000000000..0a286942655f --- /dev/null +++ b/libc/test/src/stdbit/stdc_bit_width_ul_test.cpp @@ -0,0 +1,21 @@ +//===-- Unittests for stdc_bit_width_ul -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/CPP/limits.h" +#include "src/stdbit/stdc_bit_width_ul.h" +#include "test/UnitTest/Test.h" + +TEST(LlvmLibcStdcBitWidthUlTest, Zero) { + EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_width_ul(0U), 0U); +} + +TEST(LlvmLibcStdcBitWidthUlTest, Ones) { + for (unsigned i = 0U; i != ULONG_WIDTH; ++i) + EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_width_ul(ULONG_MAX >> i), + ULONG_WIDTH - i); +} diff --git a/libc/test/src/stdbit/stdc_bit_width_ull_test.cpp b/libc/test/src/stdbit/stdc_bit_width_ull_test.cpp new file mode 100644 index 000000000000..31475f6e6541 --- /dev/null +++ b/libc/test/src/stdbit/stdc_bit_width_ull_test.cpp @@ -0,0 +1,21 @@ +//===-- Unittests for stdc_bit_width_ull ----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/CPP/limits.h" +#include "src/stdbit/stdc_bit_width_ull.h" +#include "test/UnitTest/Test.h" + +TEST(LlvmLibcStdcBitWidthUllTest, Zero) { + EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_width_ull(0U), 0U); +} + +TEST(LlvmLibcStdcBitWidthUllTest, Ones) { + for (unsigned i = 0U; i != ULLONG_WIDTH; ++i) + EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_width_ull(ULLONG_MAX >> i), + ULLONG_WIDTH - i); +} diff --git a/libc/test/src/stdbit/stdc_bit_width_us_test.cpp b/libc/test/src/stdbit/stdc_bit_width_us_test.cpp new file mode 100644 index 000000000000..031c502f19c5 --- /dev/null +++ b/libc/test/src/stdbit/stdc_bit_width_us_test.cpp @@ -0,0 +1,21 @@ +//===-- Unittests for stdc_bit_width_us -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/CPP/limits.h" +#include "src/stdbit/stdc_bit_width_us.h" +#include "test/UnitTest/Test.h" + +TEST(LlvmLibcStdcBitWidthUsTest, Zero) { + EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_width_us(0U), 0U); +} + +TEST(LlvmLibcStdcBitWidthUsTest, Ones) { + for (unsigned i = 0U; i != USHRT_WIDTH; ++i) + EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_width_us(USHRT_MAX >> i), + USHRT_WIDTH - i); +} -- GitLab From aa4e3d594a0cfc01f1260378552dccc0943a9268 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Tue, 5 Mar 2024 17:55:54 +0000 Subject: [PATCH 180/929] [gn build] Port 1a67dee08913 --- .../gn/secondary/llvm/lib/ExecutionEngine/Orc/Debugging/BUILD.gn | 1 - .../llvm/lib/ExecutionEngine/Orc/TargetProcess/BUILD.gn | 1 - 2 files changed, 2 deletions(-) diff --git a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Debugging/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Debugging/BUILD.gn index 5610679ff333..1d3fc6cfdfaa 100644 --- a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Debugging/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Debugging/BUILD.gn @@ -13,7 +13,6 @@ static_library("Debugging") { "DebuggerSupportPlugin.cpp", "LLJITUtilsCBindings.cpp", "PerfSupportPlugin.cpp", - "VTuneSupportPlugin.cpp", ] if (current_os == "linux") { libs = [ "rt" ] diff --git a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/TargetProcess/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/TargetProcess/BUILD.gn index d62f5042c946..f34855e8e1cc 100644 --- a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/TargetProcess/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/TargetProcess/BUILD.gn @@ -9,7 +9,6 @@ static_library("TargetProcess") { "ExecutorSharedMemoryMapperService.cpp", "JITLoaderGDB.cpp", "JITLoaderPerf.cpp", - "JITLoaderVTune.cpp", "OrcRTBootstrap.cpp", "RegisterEHFrames.cpp", "SimpleExecutorDylibManager.cpp", -- GitLab From 97e9780e4cca0dd2e47ac6ba8462fa965b32da76 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Tue, 5 Mar 2024 17:55:55 +0000 Subject: [PATCH 181/929] [gn build] Port 8e56fb824a43 --- .../gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn index 1863aca2bd3a..3b41f9b4007f 100644 --- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn +++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn @@ -27,6 +27,7 @@ static_library("bugprone") { "ChainedComparisonCheck.cpp", "ComparePointerToMemberVirtualFunctionCheck.cpp", "CopyConstructorInitCheck.cpp", + "CrtpConstructorAccessibilityCheck.cpp", "DanglingHandleCheck.cpp", "DynamicStaticInitializersCheck.cpp", "EasilySwappableParametersCheck.cpp", -- GitLab From caad3794f081321e7c9f370ebe69f297fc13097e Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Tue, 5 Mar 2024 17:55:56 +0000 Subject: [PATCH 182/929] [gn build] Port a8cb9db5f59d --- llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/BUILD.gn index a6b0997f54d5..a5718e0c48fe 100644 --- a/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/BUILD.gn @@ -54,6 +54,7 @@ static_library("LLVMSystemZCodeGen") { "SystemZSubtarget.cpp", "SystemZTDC.cpp", "SystemZTargetMachine.cpp", + "SystemZTargetObjectFile.cpp", "SystemZTargetTransformInfo.cpp", ] } -- GitLab From ee044d5e651787c5d73b37b2cbb7ca6444bb0502 Mon Sep 17 00:00:00 2001 From: Michael Spencer Date: Tue, 5 Mar 2024 10:15:21 -0800 Subject: [PATCH 183/929] [clang] Diagnose config_macros before building modules (#83641) Before this patch, if a module fails to build because of a missing config_macro, the user will never see the config macro warning. This patch diagnoses this before building, and each subsequent time a module is imported. rdar://123921931 --- clang/lib/Frontend/CompilerInstance.cpp | 35 ++++++++++---- clang/test/Modules/Inputs/config.h | 7 --- clang/test/Modules/Inputs/module.modulemap | 5 -- clang/test/Modules/config_macros.m | 54 ++++++++++++++++++++-- 4 files changed, 78 insertions(+), 23 deletions(-) delete mode 100644 clang/test/Modules/Inputs/config.h diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index 444ffff30737..ec4e68209d65 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -1591,6 +1591,14 @@ static void checkConfigMacro(Preprocessor &PP, StringRef ConfigMacro, } } +static void checkConfigMacros(Preprocessor &PP, Module *M, + SourceLocation ImportLoc) { + clang::Module *TopModule = M->getTopLevelModule(); + for (const StringRef ConMacro : TopModule->ConfigMacros) { + checkConfigMacro(PP, ConMacro, M, ImportLoc); + } +} + /// Write a new timestamp file with the given path. static void writeTimestampFile(StringRef TimestampFile) { std::error_code EC; @@ -1829,6 +1837,13 @@ ModuleLoadResult CompilerInstance::findOrCompileModuleAndReadAST( Module *M = HS.lookupModule(ModuleName, ImportLoc, true, !IsInclusionDirective); + // Check for any configuration macros that have changed. This is done + // immediately before potentially building a module in case this module + // depends on having one of its configuration macros defined to successfully + // build. If this is not done the user will never see the warning. + if (M) + checkConfigMacros(getPreprocessor(), M, ImportLoc); + // Select the source and filename for loading the named module. std::string ModuleFilename; ModuleSource Source = @@ -2006,12 +2021,23 @@ CompilerInstance::loadModule(SourceLocation ImportLoc, if (auto MaybeModule = MM.getCachedModuleLoad(*Path[0].first)) { // Use the cached result, which may be nullptr. Module = *MaybeModule; + // Config macros are already checked before building a module, but they need + // to be checked at each import location in case any of the config macros + // have a new value at the current `ImportLoc`. + if (Module) + checkConfigMacros(getPreprocessor(), Module, ImportLoc); } else if (ModuleName == getLangOpts().CurrentModule) { // This is the module we're building. Module = PP->getHeaderSearchInfo().lookupModule( ModuleName, ImportLoc, /*AllowSearch*/ true, /*AllowExtraModuleMapSearch*/ !IsInclusionDirective); + // Config macros do not need to be checked here for two reasons. + // * This will always be textual inclusion, and thus the config macros + // actually do impact the content of the header. + // * `Preprocessor::HandleHeaderIncludeOrImport` will never call this + // function as the `#include` or `#import` is textual. + MM.cacheModuleLoad(*Path[0].first, Module); } else { ModuleLoadResult Result = findOrCompileModuleAndReadAST( @@ -2146,18 +2172,11 @@ CompilerInstance::loadModule(SourceLocation ImportLoc, TheASTReader->makeModuleVisible(Module, Visibility, ImportLoc); } - // Check for any configuration macros that have changed. - clang::Module *TopModule = Module->getTopLevelModule(); - for (unsigned I = 0, N = TopModule->ConfigMacros.size(); I != N; ++I) { - checkConfigMacro(getPreprocessor(), TopModule->ConfigMacros[I], - Module, ImportLoc); - } - // Resolve any remaining module using export_as for this one. getPreprocessor() .getHeaderSearchInfo() .getModuleMap() - .resolveLinkAsDependencies(TopModule); + .resolveLinkAsDependencies(Module->getTopLevelModule()); LastModuleImportLoc = ImportLoc; LastModuleImportResult = ModuleLoadResult(Module); diff --git a/clang/test/Modules/Inputs/config.h b/clang/test/Modules/Inputs/config.h deleted file mode 100644 index 4c124b0bf82b..000000000000 --- a/clang/test/Modules/Inputs/config.h +++ /dev/null @@ -1,7 +0,0 @@ -#ifdef WANT_FOO -int* foo(void); -#endif - -#ifdef WANT_BAR -char *bar(void); -#endif diff --git a/clang/test/Modules/Inputs/module.modulemap b/clang/test/Modules/Inputs/module.modulemap index e7cb4b27bc08..47f6c5c1010d 100644 --- a/clang/test/Modules/Inputs/module.modulemap +++ b/clang/test/Modules/Inputs/module.modulemap @@ -260,11 +260,6 @@ module cxx_decls_merged { header "cxx-decls-merged.h" } -module config { - header "config.h" - config_macros [exhaustive] WANT_FOO, WANT_BAR -} - module diag_flags { header "diag_flags.h" } diff --git a/clang/test/Modules/config_macros.m b/clang/test/Modules/config_macros.m index 15e2c16606ba..adb2a8f6e5ce 100644 --- a/clang/test/Modules/config_macros.m +++ b/clang/test/Modules/config_macros.m @@ -1,3 +1,50 @@ +// This test verifies that config macro warnings are emitted when it looks like +// the user expected a `#define` to impact the import of a module. + +// RUN: rm -rf %t +// RUN: split-file %s %t + +// Prebuild the `config` module so it's in the module cache. +// RUN: %clang_cc1 -std=c99 -fmodules -fimplicit-module-maps -x objective-c -fmodules-cache-path=%t -DWANT_FOO=1 -emit-module -fmodule-name=config %t/module.modulemap + +// Verify that each time the `config` module is imported the current macro state +// is checked. +// RUN: %clang_cc1 -std=c99 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -I %t -DWANT_FOO=1 %t/config.m -verify + +// Verify that warnings are emitted before building a module in case the command +// line macro state causes the module to fail to build. +// RUN: %clang_cc1 -std=c99 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -I %t %t/config_error.m -verify + +//--- module.modulemap + +module config { + header "config.h" + config_macros [exhaustive] WANT_FOO, WANT_BAR +} + +module config_error { + header "config_error.h" + config_macros SOME_VALUE +} + +//--- config.h + +#ifdef WANT_FOO +int* foo(void); +#endif + +#ifdef WANT_BAR +char *bar(void); +#endif + +//--- config_error.h + +struct my_thing { + char buf[SOME_VALUE]; +}; + +//--- config.m + @import config; int *test_foo(void) { @@ -22,7 +69,8 @@ char *test_bar(void) { #define WANT_BAR 1 // expected-note{{macro was defined here}} @import config; // expected-warning{{definition of configuration macro 'WANT_BAR' has no effect on the import of 'config'; pass '-DWANT_BAR=...' on the command line to configure the module}} -// RUN: rm -rf %t -// RUN: %clang_cc1 -std=c99 -fmodules -fimplicit-module-maps -x objective-c -fmodules-cache-path=%t -DWANT_FOO=1 -emit-module -fmodule-name=config %S/Inputs/module.modulemap -// RUN: %clang_cc1 -std=c99 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -I %S/Inputs -DWANT_FOO=1 %s -verify +//--- config_error.m +#define SOME_VALUE 5 // expected-note{{macro was defined here}} +@import config_error; // expected-error{{could not build module}} \ + // expected-warning{{definition of configuration macro 'SOME_VALUE' has no effect on the import of 'config_error';}} -- GitLab From 083d8aa03aca55b88098a91e41e41a8e321a5721 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Tue, 5 Mar 2024 10:23:33 -0800 Subject: [PATCH 184/929] [SLP][NFC]Use TargetTransformInfo:: instead of TTI:: in BoUpSLP to avoid some compilers confusion. --- llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 8cf59a18381a..69bbaae0e724 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -2379,7 +2379,8 @@ private: const TreeEntry *getOperandEntry(const TreeEntry *E, unsigned Idx) const; /// \returns Cast context for the given graph node. - TTI::CastContextHint getCastContextHint(const TreeEntry &TE) const; + TargetTransformInfo::CastContextHint + getCastContextHint(const TreeEntry &TE) const; /// \returns the cost of the vectorizable entry. InstructionCost getEntryCost(const TreeEntry *E, @@ -7037,7 +7038,8 @@ static bool isAlternateInstruction(const Instruction *I, return I->getOpcode() == AltOp->getOpcode(); } -TTI::OperandValueInfo BoUpSLP::getOperandInfo(ArrayRef Ops) { +TargetTransformInfo::OperandValueInfo +BoUpSLP::getOperandInfo(ArrayRef Ops) { assert(!Ops.empty()); const auto *Op0 = Ops.front(); -- GitLab From 1b812f9cd64c14ab7600626c147da88f21e0217c Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Tue, 5 Mar 2024 10:56:01 -0800 Subject: [PATCH 185/929] [lldb] Log to system log instead of stderr from Host::SystemLog (#83366) Currently, calls to Host::SystemLog print to stderr on all host platforms except Darwin. This severely limits its value on the command line, where we don't want to overload the user with log messages. Switch to using the syslog function on POSIX systems to send messages to the system logger instead of stdout. On Darwin systems this sends the log message to os_log, which matches what we do today. Nevertheless I kept the current implementation that uses os_log directly as it gives us more freedom. I'm not sure if there's an equivalent on Windows, so I kept the existing behavior of logging to stderr. --- lldb/source/Host/common/Host.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lldb/source/Host/common/Host.cpp b/lldb/source/Host/common/Host.cpp index f4cec97f5af6..565138ba1703 100644 --- a/lldb/source/Host/common/Host.cpp +++ b/lldb/source/Host/common/Host.cpp @@ -89,8 +89,19 @@ using namespace lldb; using namespace lldb_private; #if !defined(__APPLE__) +#if !defined(_WIN32) +#include +void Host::SystemLog(llvm::StringRef message) { + static llvm::once_flag g_openlog_once; + llvm::call_once(g_openlog_once, [] { + openlog("lldb", LOG_CONS | LOG_PID | LOG_NDELAY, LOG_USER); + }); + syslog(LOG_INFO, "%s", message.data()); +} +#else void Host::SystemLog(llvm::StringRef message) { llvm::errs() << message; } #endif +#endif #if !defined(__APPLE__) && !defined(_WIN32) static thread_result_t -- GitLab From 3cef82d60796b1f18deebf0d844f38d6e85cd4e7 Mon Sep 17 00:00:00 2001 From: Peter Klausler <35819229+klausler@users.noreply.github.com> Date: Tue, 5 Mar 2024 10:57:38 -0800 Subject: [PATCH 186/929] [flang] Fix bogus error message about invalid polymorphic entity (#83733) The check for declarations of polymorphic entities was emitting a bogus error for one (or more) layers of pointers to procedures returning pointers to polymorphic types. Fixes https://github.com/llvm/llvm-project/issues/83292. --- flang/lib/Semantics/check-declarations.cpp | 2 ++ flang/test/Semantics/declarations06.f90 | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp index 719bea34406a..729321d3bf17 100644 --- a/flang/lib/Semantics/check-declarations.cpp +++ b/flang/lib/Semantics/check-declarations.cpp @@ -3236,6 +3236,8 @@ void CheckHelper::CheckSymbolType(const Symbol &symbol) { const Symbol *result{FindFunctionResult(symbol)}; const Symbol &relevant{result ? *result : symbol}; if (IsAllocatable(relevant)) { // always ok + } else if (IsProcedurePointer(symbol) && result && IsPointer(*result)) { + // procedure pointer returning allocatable or pointer: ok } else if (IsPointer(relevant) && !IsProcedure(relevant)) { // object pointers are always ok } else if (auto dyType{evaluate::DynamicType::From(relevant)}) { diff --git a/flang/test/Semantics/declarations06.f90 b/flang/test/Semantics/declarations06.f90 index 532b0461d391..ae9ef6acd754 100644 --- a/flang/test/Semantics/declarations06.f90 +++ b/flang/test/Semantics/declarations06.f90 @@ -16,6 +16,7 @@ module m procedure(cf1), pointer :: pp1 procedure(cf2), pointer :: pp2 procedure(cf3), pointer :: pp3 + procedure(cf5), pointer :: pp4 ! ok contains !ERROR: CLASS entity 'cf1' must be a dummy argument, allocatable, or object pointer class(t) function cf1() @@ -33,4 +34,12 @@ module m !ERROR: CLASS entity 'd3' must be a dummy argument, allocatable, or object pointer class(t), external, pointer :: d3 end + function cf4() + class(t), pointer :: cf4 + cf4 => v3 + end + function cf5 + procedure(cf4), pointer :: cf5 + cf5 => cf4 + end end -- GitLab From 61c06775c96a93bf2b6ac0145b78b4ecb1a858b6 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Thu, 29 Feb 2024 11:22:30 -0600 Subject: [PATCH 187/929] [KnownBits] Add API for `nuw` flag in `computeForAddSub`; NFC --- llvm/include/llvm/Support/KnownBits.h | 4 +- llvm/lib/Analysis/ValueTracking.cpp | 46 ++++++++++--------- .../lib/CodeGen/GlobalISel/GISelKnownBits.cpp | 10 ++-- .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 5 +- .../CodeGen/SelectionDAG/TargetLowering.cpp | 6 +-- llvm/lib/Support/KnownBits.cpp | 16 ++++--- llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 3 +- .../AMDGPU/AMDGPUInstructionSelector.cpp | 2 +- llvm/lib/Target/ARM/ARMISelLowering.cpp | 3 +- .../InstCombineSimplifyDemanded.cpp | 14 ++++-- llvm/unittests/Support/KnownBitsTest.cpp | 6 +-- 11 files changed, 66 insertions(+), 49 deletions(-) diff --git a/llvm/include/llvm/Support/KnownBits.h b/llvm/include/llvm/Support/KnownBits.h index 69c569b97cca..f5fce296fefe 100644 --- a/llvm/include/llvm/Support/KnownBits.h +++ b/llvm/include/llvm/Support/KnownBits.h @@ -329,8 +329,8 @@ public: const KnownBits &LHS, const KnownBits &RHS, const KnownBits &Carry); /// Compute known bits resulting from adding LHS and RHS. - static KnownBits computeForAddSub(bool Add, bool NSW, const KnownBits &LHS, - KnownBits RHS); + static KnownBits computeForAddSub(bool Add, bool NSW, bool NUW, + const KnownBits &LHS, KnownBits RHS); /// Compute known bits results from subtracting RHS from LHS with 1-bit /// Borrow. diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index 8d33b1c37b19..9d78c5d323cc 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -350,18 +350,19 @@ unsigned llvm::ComputeMaxSignificantBits(const Value *V, const DataLayout &DL, } static void computeKnownBitsAddSub(bool Add, const Value *Op0, const Value *Op1, - bool NSW, const APInt &DemandedElts, + bool NSW, bool NUW, + const APInt &DemandedElts, KnownBits &KnownOut, KnownBits &Known2, unsigned Depth, const SimplifyQuery &Q) { computeKnownBits(Op1, DemandedElts, KnownOut, Depth + 1, Q); // If one operand is unknown and we have no nowrap information, // the result will be unknown independently of the second operand. - if (KnownOut.isUnknown() && !NSW) + if (KnownOut.isUnknown() && !NSW && !NUW) return; computeKnownBits(Op0, DemandedElts, Known2, Depth + 1, Q); - KnownOut = KnownBits::computeForAddSub(Add, NSW, Known2, KnownOut); + KnownOut = KnownBits::computeForAddSub(Add, NSW, NUW, Known2, KnownOut); } static void computeKnownBitsMul(const Value *Op0, const Value *Op1, bool NSW, @@ -1145,13 +1146,15 @@ static void computeKnownBitsFromOperator(const Operator *I, } case Instruction::Sub: { bool NSW = Q.IIQ.hasNoSignedWrap(cast(I)); - computeKnownBitsAddSub(false, I->getOperand(0), I->getOperand(1), NSW, + bool NUW = Q.IIQ.hasNoUnsignedWrap(cast(I)); + computeKnownBitsAddSub(false, I->getOperand(0), I->getOperand(1), NSW, NUW, DemandedElts, Known, Known2, Depth, Q); break; } case Instruction::Add: { bool NSW = Q.IIQ.hasNoSignedWrap(cast(I)); - computeKnownBitsAddSub(true, I->getOperand(0), I->getOperand(1), NSW, + bool NUW = Q.IIQ.hasNoUnsignedWrap(cast(I)); + computeKnownBitsAddSub(true, I->getOperand(0), I->getOperand(1), NSW, NUW, DemandedElts, Known, Known2, Depth, Q); break; } @@ -1245,12 +1248,12 @@ static void computeKnownBitsFromOperator(const Operator *I, // Note that inbounds does *not* guarantee nsw for the addition, as only // the offset is signed, while the base address is unsigned. Known = KnownBits::computeForAddSub( - /*Add=*/true, /*NSW=*/false, Known, IndexBits); + /*Add=*/true, /*NSW=*/false, /* NUW=*/false, Known, IndexBits); } if (!Known.isUnknown() && !AccConstIndices.isZero()) { KnownBits Index = KnownBits::makeConstant(AccConstIndices); Known = KnownBits::computeForAddSub( - /*Add=*/true, /*NSW=*/false, Known, Index); + /*Add=*/true, /*NSW=*/false, /* NUW=*/false, Known, Index); } break; } @@ -1689,15 +1692,15 @@ static void computeKnownBitsFromOperator(const Operator *I, default: break; case Intrinsic::uadd_with_overflow: case Intrinsic::sadd_with_overflow: - computeKnownBitsAddSub(true, II->getArgOperand(0), - II->getArgOperand(1), false, DemandedElts, - Known, Known2, Depth, Q); + computeKnownBitsAddSub( + true, II->getArgOperand(0), II->getArgOperand(1), /*NSW=*/false, + /* NUW=*/false, DemandedElts, Known, Known2, Depth, Q); break; case Intrinsic::usub_with_overflow: case Intrinsic::ssub_with_overflow: - computeKnownBitsAddSub(false, II->getArgOperand(0), - II->getArgOperand(1), false, DemandedElts, - Known, Known2, Depth, Q); + computeKnownBitsAddSub( + false, II->getArgOperand(0), II->getArgOperand(1), /*NSW=*/false, + /* NUW=*/false, DemandedElts, Known, Known2, Depth, Q); break; case Intrinsic::umul_with_overflow: case Intrinsic::smul_with_overflow: @@ -2318,7 +2321,11 @@ static bool isNonZeroRecurrence(const PHINode *PN) { static bool isNonZeroAdd(const APInt &DemandedElts, unsigned Depth, const SimplifyQuery &Q, unsigned BitWidth, Value *X, - Value *Y, bool NSW) { + Value *Y, bool NSW, bool NUW) { + if (NUW) + return isKnownNonZero(Y, DemandedElts, Depth, Q) || + isKnownNonZero(X, DemandedElts, Depth, Q); + KnownBits XKnown = computeKnownBits(X, DemandedElts, Depth, Q); KnownBits YKnown = computeKnownBits(Y, DemandedElts, Depth, Q); @@ -2351,7 +2358,7 @@ static bool isNonZeroAdd(const APInt &DemandedElts, unsigned Depth, isKnownToBeAPowerOfTwo(X, /*OrZero*/ false, Depth, Q)) return true; - return KnownBits::computeForAddSub(/*Add*/ true, NSW, XKnown, YKnown) + return KnownBits::computeForAddSub(/*Add=*/true, NSW, NUW, XKnown, YKnown) .isNonZero(); } @@ -2556,12 +2563,9 @@ static bool isKnownNonZeroFromOperator(const Operator *I, // If Add has nuw wrap flag, then if either X or Y is non-zero the result is // non-zero. auto *BO = cast(I); - if (Q.IIQ.hasNoUnsignedWrap(BO)) - return isKnownNonZero(I->getOperand(1), DemandedElts, Depth, Q) || - isKnownNonZero(I->getOperand(0), DemandedElts, Depth, Q); - return isNonZeroAdd(DemandedElts, Depth, Q, BitWidth, I->getOperand(0), - I->getOperand(1), Q.IIQ.hasNoSignedWrap(BO)); + I->getOperand(1), Q.IIQ.hasNoSignedWrap(BO), + Q.IIQ.hasNoUnsignedWrap(BO)); } case Instruction::Mul: { // If X and Y are non-zero then so is X * Y as long as the multiplication @@ -2716,7 +2720,7 @@ static bool isKnownNonZeroFromOperator(const Operator *I, case Intrinsic::sadd_sat: return isNonZeroAdd(DemandedElts, Depth, Q, BitWidth, II->getArgOperand(0), II->getArgOperand(1), - /*NSW*/ true); + /*NSW=*/true, /* NUW=*/false); case Intrinsic::umax: case Intrinsic::uadd_sat: return isKnownNonZero(II->getArgOperand(1), DemandedElts, Depth, Q) || diff --git a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp index ea8c20cdcd45..099bf45b2734 100644 --- a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp +++ b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp @@ -269,8 +269,8 @@ void GISelKnownBits::computeKnownBitsImpl(Register R, KnownBits &Known, Depth + 1); computeKnownBitsImpl(MI.getOperand(2).getReg(), Known2, DemandedElts, Depth + 1); - Known = KnownBits::computeForAddSub(/*Add*/ false, /*NSW*/ false, Known, - Known2); + Known = KnownBits::computeForAddSub(/*Add=*/false, /*NSW=*/false, + /* NUW=*/false, Known, Known2); break; } case TargetOpcode::G_XOR: { @@ -296,8 +296,8 @@ void GISelKnownBits::computeKnownBitsImpl(Register R, KnownBits &Known, Depth + 1); computeKnownBitsImpl(MI.getOperand(2).getReg(), Known2, DemandedElts, Depth + 1); - Known = - KnownBits::computeForAddSub(/*Add*/ true, /*NSW*/ false, Known, Known2); + Known = KnownBits::computeForAddSub(/*Add=*/true, /*NSW=*/false, + /* NUW=*/false, Known, Known2); break; } case TargetOpcode::G_AND: { @@ -564,7 +564,7 @@ void GISelKnownBits::computeKnownBitsImpl(Register R, KnownBits &Known, // right. KnownBits ExtKnown = KnownBits::makeConstant(APInt(BitWidth, BitWidth)); KnownBits ShiftKnown = KnownBits::computeForAddSub( - /*Add*/ false, /*NSW*/ false, ExtKnown, WidthKnown); + /*Add=*/false, /*NSW=*/false, /* NUW=*/false, ExtKnown, WidthKnown); Known = KnownBits::ashr(KnownBits::shl(Known, ShiftKnown), ShiftKnown); break; } diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 5b1b7c7c6277..f7ace79e8c51 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -3763,8 +3763,9 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, SDNodeFlags Flags = Op.getNode()->getFlags(); Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); - Known = KnownBits::computeForAddSub(Op.getOpcode() == ISD::ADD, - Flags.hasNoSignedWrap(), Known, Known2); + Known = KnownBits::computeForAddSub( + Op.getOpcode() == ISD::ADD, Flags.hasNoSignedWrap(), + Flags.hasNoUnsignedWrap(), Known, Known2); break; } case ISD::USUBO: diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 6970b230837f..a639cba5e35a 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -2876,9 +2876,9 @@ bool TargetLowering::SimplifyDemandedBits( if (Op.getOpcode() == ISD::MUL) { Known = KnownBits::mul(KnownOp0, KnownOp1); } else { // Op.getOpcode() is either ISD::ADD or ISD::SUB. - Known = KnownBits::computeForAddSub(Op.getOpcode() == ISD::ADD, - Flags.hasNoSignedWrap(), KnownOp0, - KnownOp1); + Known = KnownBits::computeForAddSub( + Op.getOpcode() == ISD::ADD, Flags.hasNoSignedWrap(), + Flags.hasNoUnsignedWrap(), KnownOp0, KnownOp1); } break; } diff --git a/llvm/lib/Support/KnownBits.cpp b/llvm/lib/Support/KnownBits.cpp index c44a08cc1c2e..f999abe7dd14 100644 --- a/llvm/lib/Support/KnownBits.cpp +++ b/llvm/lib/Support/KnownBits.cpp @@ -54,7 +54,7 @@ KnownBits KnownBits::computeForAddCarry( LHS, RHS, Carry.Zero.getBoolValue(), Carry.One.getBoolValue()); } -KnownBits KnownBits::computeForAddSub(bool Add, bool NSW, +KnownBits KnownBits::computeForAddSub(bool Add, bool NSW, bool /*NUW*/, const KnownBits &LHS, KnownBits RHS) { KnownBits KnownOut; if (Add) { @@ -180,11 +180,14 @@ KnownBits KnownBits::absdiff(const KnownBits &LHS, const KnownBits &RHS) { // absdiff(LHS,RHS) = sub(umax(LHS,RHS), umin(LHS,RHS)). KnownBits UMaxValue = umax(LHS, RHS); KnownBits UMinValue = umin(LHS, RHS); - KnownBits MinMaxDiff = computeForAddSub(false, false, UMaxValue, UMinValue); + KnownBits MinMaxDiff = computeForAddSub(/*Add=*/false, /*NSW=*/false, + /*NUW=*/true, UMaxValue, UMinValue); // find the common bits between sub(LHS,RHS) and sub(RHS,LHS). - KnownBits Diff0 = computeForAddSub(false, false, LHS, RHS); - KnownBits Diff1 = computeForAddSub(false, false, RHS, LHS); + KnownBits Diff0 = + computeForAddSub(/*Add=*/false, /*NSW=*/false, /*NUW=*/false, LHS, RHS); + KnownBits Diff1 = + computeForAddSub(/*Add=*/false, /*NSW=*/false, /*NUW=*/false, RHS, LHS); KnownBits SubDiff = Diff0.intersectWith(Diff1); KnownBits KnownAbsDiff = MinMaxDiff.unionWith(SubDiff); @@ -459,7 +462,7 @@ KnownBits KnownBits::abs(bool IntMinIsPoison) const { Tmp.One.setBit(countMinTrailingZeros()); KnownAbs = computeForAddSub( - /*Add*/ false, IntMinIsPoison, + /*Add*/ false, IntMinIsPoison, /*NUW=*/false, KnownBits::makeConstant(APInt(getBitWidth(), 0)), Tmp); // One more special case for IntMinIsPoison. If we don't know any ones other @@ -505,7 +508,8 @@ static KnownBits computeForSatAddSub(bool Add, bool Signed, assert(!LHS.hasConflict() && !RHS.hasConflict() && "Bad inputs"); // We don't see NSW even for sadd/ssub as we want to check if the result has // signed overflow. - KnownBits Res = KnownBits::computeForAddSub(Add, /*NSW*/ false, LHS, RHS); + KnownBits Res = + KnownBits::computeForAddSub(Add, /*NSW=*/false, /*NUW=*/false, LHS, RHS); unsigned BitWidth = Res.getBitWidth(); auto SignBitKnown = [&](const KnownBits &K) { return K.Zero[BitWidth - 1] || K.One[BitWidth - 1]; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp index 4896ae8bad9e..2e5b02fbe856 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp @@ -1903,7 +1903,8 @@ bool AMDGPUDAGToDAGISel::checkFlatScratchSVSSwizzleBug( // voffset to (soffset + inst_offset). KnownBits VKnown = CurDAG->computeKnownBits(VAddr); KnownBits SKnown = KnownBits::computeForAddSub( - true, false, CurDAG->computeKnownBits(SAddr), + /*Add=*/true, /*NSW=*/false, /*NUW=*/false, + CurDAG->computeKnownBits(SAddr), KnownBits::makeConstant(APInt(32, ImmOffset))); uint64_t VMax = VKnown.getMaxValue().getZExtValue(); uint64_t SMax = SKnown.getMaxValue().getZExtValue(); diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp index b2c65e61b009..94cc1d90e0ca 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp @@ -4573,7 +4573,7 @@ bool AMDGPUInstructionSelector::checkFlatScratchSVSSwizzleBug( // voffset to (soffset + inst_offset). auto VKnown = KB->getKnownBits(VAddr); auto SKnown = KnownBits::computeForAddSub( - true, false, KB->getKnownBits(SAddr), + /*Add=*/true, /*NSW=*/false, /*NUW=*/false, KB->getKnownBits(SAddr), KnownBits::makeConstant(APInt(32, ImmOffset))); uint64_t VMax = VKnown.getMaxValue().getZExtValue(); uint64_t SMax = SKnown.getMaxValue().getZExtValue(); diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp index 06d4a39cde77..dc81178311b6 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -20154,7 +20154,8 @@ void ARMTargetLowering::computeKnownBitsForTargetNode(const SDValue Op, // CSNEG: KnownOp0 or KnownOp1 * -1 if (Op.getOpcode() == ARMISD::CSINC) KnownOp1 = KnownBits::computeForAddSub( - true, false, KnownOp1, KnownBits::makeConstant(APInt(32, 1))); + /*Add=*/true, /*NSW=*/false, /*NUW=*/false, KnownOp1, + KnownBits::makeConstant(APInt(32, 1))); else if (Op.getOpcode() == ARMISD::CSINV) std::swap(KnownOp1.Zero, KnownOp1.One); else if (Op.getOpcode() == ARMISD::CSNEG) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp index 97ae980a7cba..1b963a7de4a8 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp @@ -565,7 +565,8 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask, // Otherwise just compute the known bits of the result. bool NSW = cast(I)->hasNoSignedWrap(); - Known = KnownBits::computeForAddSub(true, NSW, LHSKnown, RHSKnown); + bool NUW = cast(I)->hasNoUnsignedWrap(); + Known = KnownBits::computeForAddSub(true, NSW, NUW, LHSKnown, RHSKnown); break; } case Instruction::Sub: { @@ -598,7 +599,8 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask, // Otherwise just compute the known bits of the result. bool NSW = cast(I)->hasNoSignedWrap(); - Known = KnownBits::computeForAddSub(false, NSW, LHSKnown, RHSKnown); + bool NUW = cast(I)->hasNoUnsignedWrap(); + Known = KnownBits::computeForAddSub(false, NSW, NUW, LHSKnown, RHSKnown); break; } case Instruction::Mul: { @@ -1206,7 +1208,9 @@ Value *InstCombinerImpl::SimplifyMultipleUseDemandedBits( return I->getOperand(1); bool NSW = cast(I)->hasNoSignedWrap(); - Known = KnownBits::computeForAddSub(/*Add*/ true, NSW, LHSKnown, RHSKnown); + bool NUW = cast(I)->hasNoUnsignedWrap(); + Known = + KnownBits::computeForAddSub(/*Add=*/true, NSW, NUW, LHSKnown, RHSKnown); computeKnownBitsFromContext(I, Known, Depth, SQ.getWithInstruction(CxtI)); break; } @@ -1221,8 +1225,10 @@ Value *InstCombinerImpl::SimplifyMultipleUseDemandedBits( return I->getOperand(0); bool NSW = cast(I)->hasNoSignedWrap(); + bool NUW = cast(I)->hasNoUnsignedWrap(); computeKnownBits(I->getOperand(0), LHSKnown, Depth + 1, CxtI); - Known = KnownBits::computeForAddSub(/*Add*/ false, NSW, LHSKnown, RHSKnown); + Known = KnownBits::computeForAddSub(/*Add=*/false, NSW, NUW, LHSKnown, + RHSKnown); computeKnownBitsFromContext(I, Known, Depth, SQ.getWithInstruction(CxtI)); break; } diff --git a/llvm/unittests/Support/KnownBitsTest.cpp b/llvm/unittests/Support/KnownBitsTest.cpp index fb9210bffcb4..d0ea10950566 100644 --- a/llvm/unittests/Support/KnownBitsTest.cpp +++ b/llvm/unittests/Support/KnownBitsTest.cpp @@ -194,14 +194,14 @@ static void TestAddSubExhaustive(bool IsAdd) { }); }); - KnownBits KnownComputed = - KnownBits::computeForAddSub(IsAdd, /*NSW*/ false, Known1, Known2); + KnownBits KnownComputed = KnownBits::computeForAddSub( + IsAdd, /*NSW=*/false, /*NUW=*/false, Known1, Known2); EXPECT_EQ(Known, KnownComputed); // The NSW calculation is not precise, only check that it's // conservatively correct. KnownBits KnownNSWComputed = KnownBits::computeForAddSub( - IsAdd, /*NSW*/true, Known1, Known2); + IsAdd, /*NSW=*/true, /*NUW=*/false, Known1, Known2); EXPECT_TRUE(KnownNSWComputed.Zero.isSubsetOf(KnownNSW.Zero)); EXPECT_TRUE(KnownNSWComputed.One.isSubsetOf(KnownNSW.One)); }); -- GitLab From 17162b61c2e6968482fab928f89bdca8b4ac06d9 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Wed, 28 Feb 2024 21:59:38 -0600 Subject: [PATCH 188/929] [KnownBits] Make `nuw` and `nsw` support in `computeForAddSub` optimal Just some improvements that should hopefully strengthen analysis. Closes #83580 --- llvm/include/llvm/Support/KnownBits.h | 7 +- llvm/lib/Support/KnownBits.cpp | 101 ++++++++++++++---- llvm/test/CodeGen/AArch64/sve-cmp-folds.ll | 9 +- .../CodeGen/AArch64/sve-extract-element.ll | 8 +- llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll | 39 +++---- .../InstCombine/fold-log2-ceil-idiom.ll | 2 +- llvm/test/Transforms/InstCombine/icmp-sub.ll | 5 +- llvm/test/Transforms/InstCombine/sub.ll | 2 +- llvm/unittests/Support/KnownBitsTest.cpp | 74 ++++++++++--- 9 files changed, 178 insertions(+), 69 deletions(-) diff --git a/llvm/include/llvm/Support/KnownBits.h b/llvm/include/llvm/Support/KnownBits.h index f5fce296fefe..46dbf0c2baa5 100644 --- a/llvm/include/llvm/Support/KnownBits.h +++ b/llvm/include/llvm/Support/KnownBits.h @@ -62,6 +62,11 @@ public: /// Returns true if we don't know any bits. bool isUnknown() const { return Zero.isZero() && One.isZero(); } + /// Returns true if we don't know the sign bit. + bool isSignUnknown() const { + return !Zero.isSignBitSet() && !One.isSignBitSet(); + } + /// Resets the known state of all bits. void resetAll() { Zero.clearAllBits(); @@ -330,7 +335,7 @@ public: /// Compute known bits resulting from adding LHS and RHS. static KnownBits computeForAddSub(bool Add, bool NSW, bool NUW, - const KnownBits &LHS, KnownBits RHS); + const KnownBits &LHS, const KnownBits &RHS); /// Compute known bits results from subtracting RHS from LHS with 1-bit /// Borrow. diff --git a/llvm/lib/Support/KnownBits.cpp b/llvm/lib/Support/KnownBits.cpp index f999abe7dd14..74d857457aec 100644 --- a/llvm/lib/Support/KnownBits.cpp +++ b/llvm/lib/Support/KnownBits.cpp @@ -54,34 +54,89 @@ KnownBits KnownBits::computeForAddCarry( LHS, RHS, Carry.Zero.getBoolValue(), Carry.One.getBoolValue()); } -KnownBits KnownBits::computeForAddSub(bool Add, bool NSW, bool /*NUW*/, - const KnownBits &LHS, KnownBits RHS) { - KnownBits KnownOut; - if (Add) { - // Sum = LHS + RHS + 0 - KnownOut = ::computeForAddCarry( - LHS, RHS, /*CarryZero*/true, /*CarryOne*/false); - } else { - // Sum = LHS + ~RHS + 1 - std::swap(RHS.Zero, RHS.One); - KnownOut = ::computeForAddCarry( - LHS, RHS, /*CarryZero*/false, /*CarryOne*/true); +KnownBits KnownBits::computeForAddSub(bool Add, bool NSW, bool NUW, + const KnownBits &LHS, + const KnownBits &RHS) { + unsigned BitWidth = LHS.getBitWidth(); + KnownBits KnownOut(BitWidth); + // This can be a relatively expensive helper, so optimistically save some + // work. + if (LHS.isUnknown() && RHS.isUnknown()) + return KnownOut; + + if (!LHS.isUnknown() && !RHS.isUnknown()) { + if (Add) { + // Sum = LHS + RHS + 0 + KnownOut = ::computeForAddCarry(LHS, RHS, /*CarryZero=*/true, + /*CarryOne=*/false); + } else { + // Sum = LHS + ~RHS + 1 + KnownBits NotRHS = RHS; + std::swap(NotRHS.Zero, NotRHS.One); + KnownOut = ::computeForAddCarry(LHS, NotRHS, /*CarryZero=*/false, + /*CarryOne=*/true); + } } - // Are we still trying to solve for the sign bit? - if (!KnownOut.isNegative() && !KnownOut.isNonNegative()) { - if (NSW) { - // Adding two non-negative numbers, or subtracting a negative number from - // a non-negative one, can't wrap into negative. - if (LHS.isNonNegative() && RHS.isNonNegative()) - KnownOut.makeNonNegative(); - // Adding two negative numbers, or subtracting a non-negative number from - // a negative one, can't wrap into non-negative. - else if (LHS.isNegative() && RHS.isNegative()) - KnownOut.makeNegative(); + // Handle add/sub given nsw and/or nuw. + if (NUW) { + if (Add) { + // (add nuw X, Y) + APInt MinVal = LHS.getMinValue().uadd_sat(RHS.getMinValue()); + // None of the adds can end up overflowing, so min consecutive highbits + // in minimum possible of X + Y must all remain set. + if (NSW) { + unsigned NumBits = MinVal.trunc(BitWidth - 1).countl_one(); + // If we have NSW as well, we also know we can't overflow the signbit so + // can start counting from 1 bit back. + KnownOut.One.setBits(BitWidth - 1 - NumBits, BitWidth - 1); + } + KnownOut.One.setHighBits(MinVal.countl_one()); + } else { + // (sub nuw X, Y) + APInt MaxVal = LHS.getMaxValue().usub_sat(RHS.getMinValue()); + // None of the subs can overflow at any point, so any common high bits + // will subtract away and result in zeros. + if (NSW) { + // If we have NSW as well, we also know we can't overflow the signbit so + // can start counting from 1 bit back. + unsigned NumBits = MaxVal.trunc(BitWidth - 1).countl_zero(); + KnownOut.Zero.setBits(BitWidth - 1 - NumBits, BitWidth - 1); + } + KnownOut.Zero.setHighBits(MaxVal.countl_zero()); + } + } + + if (NSW) { + APInt MinVal; + APInt MaxVal; + if (Add) { + // (add nsw X, Y) + MinVal = LHS.getSignedMinValue().sadd_sat(RHS.getSignedMinValue()); + MaxVal = LHS.getSignedMaxValue().sadd_sat(RHS.getSignedMaxValue()); + } else { + // (sub nsw X, Y) + MinVal = LHS.getSignedMinValue().ssub_sat(RHS.getSignedMaxValue()); + MaxVal = LHS.getSignedMaxValue().ssub_sat(RHS.getSignedMinValue()); + } + if (MinVal.isNonNegative()) { + // If min is non-negative, result will always be non-neg (can't overflow + // around). + unsigned NumBits = MinVal.trunc(BitWidth - 1).countl_one(); + KnownOut.One.setBits(BitWidth - 1 - NumBits, BitWidth - 1); + KnownOut.Zero.setSignBit(); + } + if (MaxVal.isNegative()) { + // If max is negative, result will always be neg (can't overflow around). + unsigned NumBits = MaxVal.trunc(BitWidth - 1).countl_zero(); + KnownOut.Zero.setBits(BitWidth - 1 - NumBits, BitWidth - 1); + KnownOut.One.setSignBit(); } } + // Just return 0 if the nsw/nuw is violated and we have poison. + if (KnownOut.hasConflict()) + KnownOut.setAllZero(); return KnownOut; } diff --git a/llvm/test/CodeGen/AArch64/sve-cmp-folds.ll b/llvm/test/CodeGen/AArch64/sve-cmp-folds.ll index beded623272c..c8a36e47efca 100644 --- a/llvm/test/CodeGen/AArch64/sve-cmp-folds.ll +++ b/llvm/test/CodeGen/AArch64/sve-cmp-folds.ll @@ -114,9 +114,12 @@ define i1 @foo_last( %a, %b) { ; CHECK-LABEL: foo_last: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: fcmeq p1.s, p0/z, z0.s, z1.s -; CHECK-NEXT: ptest p0, p1.b -; CHECK-NEXT: cset w0, lo +; CHECK-NEXT: mov x8, #-1 // =0xffffffffffffffff +; CHECK-NEXT: whilels p1.s, xzr, x8 +; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s +; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1 +; CHECK-NEXT: lastb w8, p1, z0.s +; CHECK-NEXT: and w0, w8, #0x1 ; CHECK-NEXT: ret %vcond = fcmp oeq %a, %b %vscale = call i64 @llvm.vscale.i64() diff --git a/llvm/test/CodeGen/AArch64/sve-extract-element.ll b/llvm/test/CodeGen/AArch64/sve-extract-element.ll index 273785f24364..a3c34b53baa0 100644 --- a/llvm/test/CodeGen/AArch64/sve-extract-element.ll +++ b/llvm/test/CodeGen/AArch64/sve-extract-element.ll @@ -614,9 +614,11 @@ define i1 @test_lane9_8xi1( %a) #0 { define i1 @test_last_8xi1( %a) #0 { ; CHECK-LABEL: test_last_8xi1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.h -; CHECK-NEXT: ptest p1, p0.b -; CHECK-NEXT: cset w0, lo +; CHECK-NEXT: mov x8, #-1 // =0xffffffffffffffff +; CHECK-NEXT: mov z0.h, p0/z, #1 // =0x1 +; CHECK-NEXT: whilels p1.h, xzr, x8 +; CHECK-NEXT: lastb w8, p1, z0.h +; CHECK-NEXT: and w0, w8, #0x1 ; CHECK-NEXT: ret %vscale = call i64 @llvm.vscale.i64() %shl = shl nuw nsw i64 %vscale, 3 diff --git a/llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll b/llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll index 6e6b204031c0..7b9b130e1cf7 100644 --- a/llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll +++ b/llvm/test/CodeGen/AMDGPU/ds-sub-offset.ll @@ -137,19 +137,18 @@ define amdgpu_kernel void @write_ds_sub_max_offset_global_clamp_bit(float %dummy ; CI: ; %bb.0: ; CI-NEXT: s_load_dword s0, s[0:1], 0x0 ; CI-NEXT: s_mov_b64 vcc, 0 -; CI-NEXT: v_not_b32_e32 v0, v0 -; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0 -; CI-NEXT: v_mov_b32_e32 v2, 0x7b +; CI-NEXT: v_mov_b32_e32 v1, 0x7b +; CI-NEXT: v_mov_b32_e32 v2, 0 +; CI-NEXT: s_mov_b32 m0, -1 ; CI-NEXT: s_waitcnt lgkmcnt(0) -; CI-NEXT: v_mov_b32_e32 v1, s0 -; CI-NEXT: v_div_fmas_f32 v1, v1, v1, v1 +; CI-NEXT: v_mov_b32_e32 v0, s0 +; CI-NEXT: v_div_fmas_f32 v0, v0, v0, v0 ; CI-NEXT: s_mov_b32 s0, 0 -; CI-NEXT: s_mov_b32 m0, -1 ; CI-NEXT: s_mov_b32 s3, 0xf000 ; CI-NEXT: s_mov_b32 s2, -1 ; CI-NEXT: s_mov_b32 s1, s0 -; CI-NEXT: ds_write_b32 v0, v2 offset:65532 -; CI-NEXT: buffer_store_dword v1, off, s[0:3], 0 +; CI-NEXT: ds_write_b32 v2, v1 +; CI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; CI-NEXT: s_waitcnt vmcnt(0) ; CI-NEXT: s_endpgm ; @@ -157,15 +156,14 @@ define amdgpu_kernel void @write_ds_sub_max_offset_global_clamp_bit(float %dummy ; GFX9: ; %bb.0: ; GFX9-NEXT: s_load_dword s0, s[0:1], 0x0 ; GFX9-NEXT: s_mov_b64 vcc, 0 -; GFX9-NEXT: v_not_b32_e32 v0, v0 -; GFX9-NEXT: v_lshlrev_b32_e32 v3, 2, v0 -; GFX9-NEXT: v_mov_b32_e32 v4, 0x7b +; GFX9-NEXT: v_mov_b32_e32 v3, 0x7b +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: ds_write_b32 v4, v3 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v1, s0 -; GFX9-NEXT: v_div_fmas_f32 v2, v1, v1, v1 +; GFX9-NEXT: v_mov_b32_e32 v0, s0 +; GFX9-NEXT: v_div_fmas_f32 v2, v0, v0, v0 ; GFX9-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NEXT: v_mov_b32_e32 v1, 0 -; GFX9-NEXT: ds_write_b32 v3, v4 offset:65532 ; GFX9-NEXT: global_store_dword v[0:1], v2, off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_endpgm @@ -173,13 +171,12 @@ define amdgpu_kernel void @write_ds_sub_max_offset_global_clamp_bit(float %dummy ; GFX10-LABEL: write_ds_sub_max_offset_global_clamp_bit: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_load_dword s0, s[0:1], 0x0 -; GFX10-NEXT: v_not_b32_e32 v0, v0 ; GFX10-NEXT: s_mov_b32 vcc_lo, 0 -; GFX10-NEXT: v_mov_b32_e32 v3, 0x7b -; GFX10-NEXT: v_lshlrev_b32_e32 v2, 2, v0 ; GFX10-NEXT: v_mov_b32_e32 v0, 0 +; GFX10-NEXT: v_mov_b32_e32 v2, 0x7b +; GFX10-NEXT: v_mov_b32_e32 v3, 0 ; GFX10-NEXT: v_mov_b32_e32 v1, 0 -; GFX10-NEXT: ds_write_b32 v2, v3 offset:65532 +; GFX10-NEXT: ds_write_b32 v3, v2 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: v_div_fmas_f32 v4, s0, s0, s0 ; GFX10-NEXT: global_store_dword v[0:1], v4, off @@ -189,13 +186,11 @@ define amdgpu_kernel void @write_ds_sub_max_offset_global_clamp_bit(float %dummy ; GFX11-LABEL: write_ds_sub_max_offset_global_clamp_bit: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_load_b32 s0, s[0:1], 0x0 -; GFX11-NEXT: v_not_b32_e32 v0, v0 ; GFX11-NEXT: s_mov_b32 vcc_lo, 0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_dual_mov_b32 v3, 0x7b :: v_dual_lshlrev_b32 v2, 2, v0 ; GFX11-NEXT: v_mov_b32_e32 v0, 0 +; GFX11-NEXT: v_dual_mov_b32 v2, 0x7b :: v_dual_mov_b32 v3, 0 ; GFX11-NEXT: v_mov_b32_e32 v1, 0 -; GFX11-NEXT: ds_store_b32 v2, v3 offset:65532 +; GFX11-NEXT: ds_store_b32 v3, v2 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: v_div_fmas_f32 v4, s0, s0, s0 ; GFX11-NEXT: global_store_b32 v[0:1], v4, off dlc diff --git a/llvm/test/Transforms/InstCombine/fold-log2-ceil-idiom.ll b/llvm/test/Transforms/InstCombine/fold-log2-ceil-idiom.ll index 2594c3fce814..434d98449f99 100644 --- a/llvm/test/Transforms/InstCombine/fold-log2-ceil-idiom.ll +++ b/llvm/test/Transforms/InstCombine/fold-log2-ceil-idiom.ll @@ -43,7 +43,7 @@ define i64 @log2_ceil_idiom_zext(i32 %x) { ; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[X]], -1 ; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[TMP1]], i1 false), !range [[RNG0]] ; CHECK-NEXT: [[TMP3:%.*]] = sub nuw nsw i32 32, [[TMP2]] -; CHECK-NEXT: [[RET:%.*]] = zext i32 [[TMP3]] to i64 +; CHECK-NEXT: [[RET:%.*]] = zext nneg i32 [[TMP3]] to i64 ; CHECK-NEXT: ret i64 [[RET]] ; %ctlz = tail call i32 @llvm.ctlz.i32(i32 %x, i1 true) diff --git a/llvm/test/Transforms/InstCombine/icmp-sub.ll b/llvm/test/Transforms/InstCombine/icmp-sub.ll index 2dad575fede8..5645dededf2e 100644 --- a/llvm/test/Transforms/InstCombine/icmp-sub.ll +++ b/llvm/test/Transforms/InstCombine/icmp-sub.ll @@ -36,7 +36,7 @@ define i1 @test_nuw_nsw_and_unsigned_pred(i64 %x) { define i1 @test_nuw_nsw_and_signed_pred(i64 %x) { ; CHECK-LABEL: @test_nuw_nsw_and_signed_pred( -; CHECK-NEXT: [[Z:%.*]] = icmp sgt i64 [[X:%.*]], 7 +; CHECK-NEXT: [[Z:%.*]] = icmp ugt i64 [[X:%.*]], 7 ; CHECK-NEXT: ret i1 [[Z]] ; %y = sub nuw nsw i64 10, %x @@ -46,8 +46,7 @@ define i1 @test_nuw_nsw_and_signed_pred(i64 %x) { define i1 @test_negative_nuw_and_signed_pred(i64 %x) { ; CHECK-LABEL: @test_negative_nuw_and_signed_pred( -; CHECK-NEXT: [[NOTSUB:%.*]] = add nuw i64 [[X:%.*]], -11 -; CHECK-NEXT: [[Z:%.*]] = icmp sgt i64 [[NOTSUB]], -4 +; CHECK-NEXT: [[Z:%.*]] = icmp ugt i64 [[X:%.*]], 7 ; CHECK-NEXT: ret i1 [[Z]] ; %y = sub nuw i64 10, %x diff --git a/llvm/test/Transforms/InstCombine/sub.ll b/llvm/test/Transforms/InstCombine/sub.ll index 76cd7ab5c10c..249b5673c8ac 100644 --- a/llvm/test/Transforms/InstCombine/sub.ll +++ b/llvm/test/Transforms/InstCombine/sub.ll @@ -2367,7 +2367,7 @@ define <2 x i8> @sub_to_and_vector3(<2 x i8> %x) { ; CHECK-LABEL: @sub_to_and_vector3( ; CHECK-NEXT: [[SUB:%.*]] = sub nuw <2 x i8> , [[X:%.*]] ; CHECK-NEXT: [[AND:%.*]] = and <2 x i8> [[SUB]], -; CHECK-NEXT: [[R:%.*]] = sub <2 x i8> , [[AND]] +; CHECK-NEXT: [[R:%.*]] = sub nsw <2 x i8> , [[AND]] ; CHECK-NEXT: ret <2 x i8> [[R]] ; %sub = sub nuw <2 x i8> , %x diff --git a/llvm/unittests/Support/KnownBitsTest.cpp b/llvm/unittests/Support/KnownBitsTest.cpp index d0ea10950566..658f3796721c 100644 --- a/llvm/unittests/Support/KnownBitsTest.cpp +++ b/llvm/unittests/Support/KnownBitsTest.cpp @@ -169,41 +169,69 @@ static void TestAddSubExhaustive(bool IsAdd) { unsigned Bits = 4; ForeachKnownBits(Bits, [&](const KnownBits &Known1) { ForeachKnownBits(Bits, [&](const KnownBits &Known2) { - KnownBits Known(Bits), KnownNSW(Bits); + KnownBits Known(Bits), KnownNSW(Bits), KnownNUW(Bits), + KnownNSWAndNUW(Bits); Known.Zero.setAllBits(); Known.One.setAllBits(); KnownNSW.Zero.setAllBits(); KnownNSW.One.setAllBits(); + KnownNUW.Zero.setAllBits(); + KnownNUW.One.setAllBits(); + KnownNSWAndNUW.Zero.setAllBits(); + KnownNSWAndNUW.One.setAllBits(); ForeachNumInKnownBits(Known1, [&](const APInt &N1) { ForeachNumInKnownBits(Known2, [&](const APInt &N2) { - bool Overflow; + bool SignedOverflow; + bool UnsignedOverflow; APInt Res; - if (IsAdd) - Res = N1.sadd_ov(N2, Overflow); - else - Res = N1.ssub_ov(N2, Overflow); + if (IsAdd) { + Res = N1.uadd_ov(N2, UnsignedOverflow); + Res = N1.sadd_ov(N2, SignedOverflow); + } else { + Res = N1.usub_ov(N2, UnsignedOverflow); + Res = N1.ssub_ov(N2, SignedOverflow); + } Known.One &= Res; Known.Zero &= ~Res; - if (!Overflow) { + if (!SignedOverflow) { KnownNSW.One &= Res; KnownNSW.Zero &= ~Res; } + + if (!UnsignedOverflow) { + KnownNUW.One &= Res; + KnownNUW.Zero &= ~Res; + } + + if (!UnsignedOverflow && !SignedOverflow) { + KnownNSWAndNUW.One &= Res; + KnownNSWAndNUW.Zero &= ~Res; + } }); }); KnownBits KnownComputed = KnownBits::computeForAddSub( IsAdd, /*NSW=*/false, /*NUW=*/false, Known1, Known2); - EXPECT_EQ(Known, KnownComputed); + EXPECT_TRUE(isOptimal(Known, KnownComputed, {Known1, Known2})); - // The NSW calculation is not precise, only check that it's - // conservatively correct. KnownBits KnownNSWComputed = KnownBits::computeForAddSub( IsAdd, /*NSW=*/true, /*NUW=*/false, Known1, Known2); - EXPECT_TRUE(KnownNSWComputed.Zero.isSubsetOf(KnownNSW.Zero)); - EXPECT_TRUE(KnownNSWComputed.One.isSubsetOf(KnownNSW.One)); + if (!KnownNSW.hasConflict()) + EXPECT_TRUE(isOptimal(KnownNSW, KnownNSWComputed, {Known1, Known2})); + + KnownBits KnownNUWComputed = KnownBits::computeForAddSub( + IsAdd, /*NSW=*/false, /*NUW=*/true, Known1, Known2); + if (!KnownNUW.hasConflict()) + EXPECT_TRUE(isOptimal(KnownNUW, KnownNUWComputed, {Known1, Known2})); + + KnownBits KnownNSWAndNUWComputed = KnownBits::computeForAddSub( + IsAdd, /*NSW=*/true, /*NUW=*/true, Known1, Known2); + if (!KnownNSWAndNUW.hasConflict()) + EXPECT_TRUE(isOptimal(KnownNSWAndNUW, KnownNSWAndNUWComputed, + {Known1, Known2})); }); }); } @@ -244,6 +272,28 @@ TEST(KnownBitsTest, SubBorrowExhaustive) { }); } +TEST(KnownBitsTest, SignBitUnknown) { + KnownBits Known(2); + EXPECT_TRUE(Known.isSignUnknown()); + Known.Zero.setBit(0); + EXPECT_TRUE(Known.isSignUnknown()); + Known.Zero.setBit(1); + EXPECT_FALSE(Known.isSignUnknown()); + Known.Zero.clearBit(0); + EXPECT_FALSE(Known.isSignUnknown()); + Known.Zero.clearBit(1); + EXPECT_TRUE(Known.isSignUnknown()); + + Known.One.setBit(0); + EXPECT_TRUE(Known.isSignUnknown()); + Known.One.setBit(1); + EXPECT_FALSE(Known.isSignUnknown()); + Known.One.clearBit(0); + EXPECT_FALSE(Known.isSignUnknown()); + Known.One.clearBit(1); + EXPECT_TRUE(Known.isSignUnknown()); +} + TEST(KnownBitsTest, AbsDiffSpecialCase) { // There are 2 implementation of absdiff - both are currently needed to cover // extra cases. -- GitLab From 9a20612d960bbcbf6bd59d2d94400574a3cccde0 Mon Sep 17 00:00:00 2001 From: Peter Klausler <35819229+klausler@users.noreply.github.com> Date: Tue, 5 Mar 2024 11:09:48 -0800 Subject: [PATCH 189/929] [flang] NULL(NULL(NULL(...(NULL()...))) means NULL() (#83738) When the actual MOLD= argument of a reference to the intrinsic function NULL is itself just NULL() (possibly nested), treat the MOLD= as if it had not been present. Fixes https://github.com/llvm/llvm-project/issues/83572. --- flang/lib/Evaluate/intrinsics.cpp | 33 +++++++++++++++++++++---------- flang/test/Semantics/null01.f90 | 10 ++++++++++ 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp index a8f2e5b445ed..9b98d22cc58e 100644 --- a/flang/lib/Evaluate/intrinsics.cpp +++ b/flang/lib/Evaluate/intrinsics.cpp @@ -2635,19 +2635,30 @@ SpecificCall IntrinsicProcTable::Implementation::HandleNull( static const char *const keywords[]{"mold", nullptr}; if (CheckAndRearrangeArguments(arguments, context.messages(), keywords, 1) && arguments[0]) { - if (Expr * mold{arguments[0]->UnwrapExpr()}) { - bool isProcPtrTarget{IsProcedurePointerTarget(*mold)}; + Expr *mold{arguments[0]->UnwrapExpr()}; + bool isBareNull{IsBareNullPointer(mold)}; + if (isBareNull) { + // NULL(NULL()), NULL(NULL(NULL())), &c. are all just NULL() + mold = nullptr; + } + if (mold) { + bool isProcPtrTarget{ + IsProcedurePointerTarget(*mold) && !IsNullObjectPointer(*mold)}; if (isProcPtrTarget || IsAllocatableOrPointerObject(*mold)) { characteristics::DummyArguments args; std::optional fResult; if (isProcPtrTarget) { // MOLD= procedure pointer - const Symbol *last{GetLastSymbol(*mold)}; - CHECK(last); - auto procPointer{IsProcedure(*last) - ? characteristics::Procedure::Characterize(*last, context) - : std::nullopt}; - // procPointer is null if there was an error with the analysis + std::optional procPointer; + if (IsNullProcedurePointer(*mold)) { + procPointer = + characteristics::Procedure::Characterize(*mold, context); + } else { + const Symbol *last{GetLastSymbol(*mold)}; + procPointer = + characteristics::Procedure::Characterize(DEREF(last), context); + } + // procPointer is vacant if there was an error with the analysis // associated with the procedure pointer if (procPointer) { args.emplace_back("mold"s, @@ -2676,8 +2687,10 @@ SpecificCall IntrinsicProcTable::Implementation::HandleNull( } } } - context.messages().Say(arguments[0]->sourceLocation(), - "MOLD= argument to NULL() must be a pointer or allocatable"_err_en_US); + if (!isBareNull) { + context.messages().Say(arguments[0]->sourceLocation(), + "MOLD= argument to NULL() must be a pointer or allocatable"_err_en_US); + } } characteristics::Procedure::Attrs attrs; attrs.set(characteristics::Procedure::Attr::NullPointer); diff --git a/flang/test/Semantics/null01.f90 b/flang/test/Semantics/null01.f90 index 71567fb0a673..b61d464d0e7c 100644 --- a/flang/test/Semantics/null01.f90 +++ b/flang/test/Semantics/null01.f90 @@ -65,12 +65,22 @@ subroutine test real(kind=eight) :: r8check logical, pointer :: lp ip0 => null() ! ok + ip0 => null(null()) ! ok + ip0 => null(null(null())) ! ok ip1 => null() ! ok + ip1 => null(null()) ! ok + ip1 => null(null(null())) ! ok ip2 => null() ! ok + ip2 => null(null()) ! ok + ip2 => null(null(null())) ! ok !ERROR: MOLD= argument to NULL() must be a pointer or allocatable ip0 => null(mold=1) !ERROR: MOLD= argument to NULL() must be a pointer or allocatable + ip0 => null(null(mold=1)) + !ERROR: MOLD= argument to NULL() must be a pointer or allocatable ip0 => null(mold=j) + !ERROR: MOLD= argument to NULL() must be a pointer or allocatable + ip0 => null(mold=null(mold=j)) dt0x = dt0(null()) dt0x = dt0(ip0=null()) dt0x = dt0(ip0=null(ip0)) -- GitLab From d35f2c439a12b8e97e1bad02b6cc4349c9914345 Mon Sep 17 00:00:00 2001 From: Martin Wehking Date: Tue, 5 Mar 2024 19:23:09 +0000 Subject: [PATCH 190/929] Remove constant local variable (#83850) Remove isThisReturn, which always has the value false. Replace its uses with false directly. --- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 30a65bb33265..f30dc06ed408 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -3598,7 +3598,6 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI, bool &IsTailCall = CLI.IsTailCall; bool IsVarArg = CLI.IsVarArg; bool IsSibCall = false; - bool IsThisReturn = false; MachineFunction &MF = DAG.getMachineFunction(); if (Callee.isUndef() || isNullConstant(Callee)) { @@ -3888,8 +3887,7 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI, // Handle result values, copying them out of physregs into vregs that we // return. return LowerCallResult(Chain, InGlue, CallConv, IsVarArg, Ins, DL, DAG, - InVals, IsThisReturn, - IsThisReturn ? OutVals[0] : SDValue()); + InVals, /*IsThisReturn=*/false, SDValue()); } // This is identical to the default implementation in ExpandDYNAMIC_STACKALLOC, -- GitLab From 069aee0793064b800f130e740e37dd7d264b7802 Mon Sep 17 00:00:00 2001 From: Peter Klausler <35819229+klausler@users.noreply.github.com> Date: Tue, 5 Mar 2024 11:28:11 -0800 Subject: [PATCH 191/929] [flang] Rework F'2023 constraint C1167 checking (#83888) The code that verifies that the type in a TYPE IS or CLASS IS clause is a match or an extension of the type of the SELECT TYPE selector needs rework to avoid emitting a bogus error for a test. Fixes https://github.com/llvm/llvm-project/issues/83612. --- flang/include/flang/Semantics/type.h | 2 +- flang/lib/Semantics/check-select-type.cpp | 30 +++++++--------- flang/lib/Semantics/type.cpp | 43 ++++++++++++++--------- flang/test/Semantics/selecttype04.f90 | 31 ++++++++++++++++ 4 files changed, 70 insertions(+), 36 deletions(-) create mode 100644 flang/test/Semantics/selecttype04.f90 diff --git a/flang/include/flang/Semantics/type.h b/flang/include/flang/Semantics/type.h index 8965d29d8889..5520b02e6790 100644 --- a/flang/include/flang/Semantics/type.h +++ b/flang/include/flang/Semantics/type.h @@ -306,7 +306,7 @@ public: } // For TYPE IS & CLASS IS: kind type parameters must be // explicit and equal, len type parameters are ignored. - bool Match(const DerivedTypeSpec &) const; + bool MatchesOrExtends(const DerivedTypeSpec &) const; std::string AsFortran() const; std::string VectorTypeAsFortran() const; diff --git a/flang/lib/Semantics/check-select-type.cpp b/flang/lib/Semantics/check-select-type.cpp index 6515cf25e0d7..94d16a719277 100644 --- a/flang/lib/Semantics/check-select-type.cpp +++ b/flang/lib/Semantics/check-select-type.cpp @@ -120,31 +120,25 @@ private: bool PassesDerivedTypeChecks(const semantics::DerivedTypeSpec &derived, parser::CharBlock sourceLoc) const { for (const auto &pair : derived.parameters()) { - if (pair.second.isLen() && !pair.second.isAssumed()) { // C1160 + if (pair.second.isLen() && !pair.second.isAssumed()) { // F'2023 C1165 context_.Say(sourceLoc, - "The type specification statement must have " - "LEN type parameter as assumed"_err_en_US); + "The type specification statement must have LEN type parameter as assumed"_err_en_US); return false; } } - if (!IsExtensibleType(&derived)) { // C1161 + if (!IsExtensibleType(&derived)) { // F'2023 C1166 context_.Say(sourceLoc, - "The type specification statement must not specify " - "a type with a SEQUENCE attribute or a BIND attribute"_err_en_US); + "The type specification statement must not specify a type with a SEQUENCE attribute or a BIND attribute"_err_en_US); return false; } - if (!selectorType_.IsUnlimitedPolymorphic()) { // C1162 - if (const semantics::Scope * guardScope{derived.typeSymbol().scope()}) { - if (const auto *selDerivedTypeSpec{ - evaluate::GetDerivedTypeSpec(selectorType_)}) { - if (!derived.Match(*selDerivedTypeSpec) && - !guardScope->FindComponent(selDerivedTypeSpec->name())) { - context_.Say(sourceLoc, - "Type specification '%s' must be an extension" - " of TYPE '%s'"_err_en_US, - derived.AsFortran(), selDerivedTypeSpec->AsFortran()); - return false; - } + if (!selectorType_.IsUnlimitedPolymorphic()) { // F'2023 C1167 + if (const auto *selDerivedTypeSpec{ + evaluate::GetDerivedTypeSpec(selectorType_)}) { + if (!derived.MatchesOrExtends(*selDerivedTypeSpec)) { + context_.Say(sourceLoc, + "Type specification '%s' must be an extension of TYPE '%s'"_err_en_US, + derived.AsFortran(), selDerivedTypeSpec->AsFortran()); + return false; } } } diff --git a/flang/lib/Semantics/type.cpp b/flang/lib/Semantics/type.cpp index e812283fc6f1..44e49673300b 100644 --- a/flang/lib/Semantics/type.cpp +++ b/flang/lib/Semantics/type.cpp @@ -231,27 +231,36 @@ ParamValue *DerivedTypeSpec::FindParameter(SourceName target) { const_cast(this)->FindParameter(target)); } -bool DerivedTypeSpec::Match(const DerivedTypeSpec &that) const { - if (&typeSymbol_ != &that.typeSymbol_) { - return false; - } - for (const auto &pair : parameters_) { - const Symbol *tpSym{scope_ ? scope_->FindSymbol(pair.first) : nullptr}; - const auto *tpDetails{ - tpSym ? tpSym->detailsIf() : nullptr}; - if (!tpDetails) { - return false; - } - if (tpDetails->attr() != common::TypeParamAttr::Kind) { - continue; +static bool MatchKindParams(const Symbol &typeSymbol, + const DerivedTypeSpec &thisSpec, const DerivedTypeSpec &thatSpec) { + for (auto ref : typeSymbol.get().paramDecls()) { + if (ref->get().attr() == common::TypeParamAttr::Kind) { + const auto *thisValue{thisSpec.FindParameter(ref->name())}; + const auto *thatValue{thatSpec.FindParameter(ref->name())}; + if (!thisValue || !thatValue || *thisValue != *thatValue) { + return false; + } } - const ParamValue &value{pair.second}; - auto iter{that.parameters_.find(pair.first)}; - if (iter == that.parameters_.end() || iter->second != value) { + } + if (const DerivedTypeSpec * + parent{typeSymbol.GetParentTypeSpec(typeSymbol.scope())}) { + return MatchKindParams(parent->typeSymbol(), thisSpec, thatSpec); + } else { + return true; + } +} + +bool DerivedTypeSpec::MatchesOrExtends(const DerivedTypeSpec &that) const { + const Symbol *typeSymbol{&typeSymbol_}; + while (typeSymbol != &that.typeSymbol_) { + if (const DerivedTypeSpec * + parent{typeSymbol->GetParentTypeSpec(typeSymbol->scope())}) { + typeSymbol = &parent->typeSymbol_; + } else { return false; } } - return true; + return MatchKindParams(*typeSymbol, *this, that); } class InstantiateHelper { diff --git a/flang/test/Semantics/selecttype04.f90 b/flang/test/Semantics/selecttype04.f90 new file mode 100644 index 000000000000..535576b0ac9a --- /dev/null +++ b/flang/test/Semantics/selecttype04.f90 @@ -0,0 +1,31 @@ +! RUN: %python %S/test_errors.py %s %flang_fc1 +! Check F'2023 C1167 +module m + type :: base(kindparam, lenparam) + integer, kind :: kindparam + integer, len :: lenparam + end type + type, extends(base) :: ext1 + contains + procedure :: tbp + end type + type, extends(ext1) :: ext2 + end type + contains + function tbp(x) + class(ext1(123,*)), target :: x + class(ext1(123,:)), pointer :: tbp + tbp => x + end + subroutine test + type(ext1(123,456)), target :: var + select type (sel => var%tbp()) + type is (ext1(123,*)) ! ok + type is (ext2(123,*)) ! ok + !ERROR: Type specification 'ext1(kindparam=234_4,lenparam=*)' must be an extension of TYPE 'ext1(kindparam=123_4,lenparam=:)' + type is (ext1(234,*)) + !ERROR: Type specification 'ext2(kindparam=234_4,lenparam=*)' must be an extension of TYPE 'ext1(kindparam=123_4,lenparam=:)' + type is (ext2(234,*)) + end select + end +end -- GitLab From cc7544e230f8943e52b779d7f1e4392fd30a0c1a Mon Sep 17 00:00:00 2001 From: Jordan Rupprecht Date: Tue, 5 Mar 2024 13:31:57 -0600 Subject: [PATCH 192/929] [benchmark] Delete WORKSPACE bzl files (#84013) As like 48d868493fa74025e7768afacdbbbd3ea9c82468, `WORKSPACE` is another bazel-specific file that is unused. LLVM's bazel configuration is entirely in `utils/bazel`. --- third-party/benchmark/WORKSPACE | 30 ------------------------------ third-party/update_benchmark.sh | 1 + 2 files changed, 1 insertion(+), 30 deletions(-) delete mode 100644 third-party/benchmark/WORKSPACE diff --git a/third-party/benchmark/WORKSPACE b/third-party/benchmark/WORKSPACE deleted file mode 100644 index 256207022597..000000000000 --- a/third-party/benchmark/WORKSPACE +++ /dev/null @@ -1,30 +0,0 @@ -workspace(name = "com_github_google_benchmark") - -load("//:bazel/benchmark_deps.bzl", "benchmark_deps") - -benchmark_deps() - -load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies") - -rules_foreign_cc_dependencies() - -load("@rules_python//python:repositories.bzl", "py_repositories") - -py_repositories() - -load("@rules_python//python:pip.bzl", "pip_parse") - -pip_parse( - name = "tools_pip_deps", - requirements_lock = "//tools:requirements.txt", -) - -load("@tools_pip_deps//:requirements.bzl", "install_deps") - -install_deps() - -new_local_repository( - name = "python_headers", - build_file = "@//bindings/python:python_headers.BUILD", - path = "", # May be overwritten by setup.py. -) diff --git a/third-party/update_benchmark.sh b/third-party/update_benchmark.sh index cda0bcfb1a7a..047efd945a86 100755 --- a/third-party/update_benchmark.sh +++ b/third-party/update_benchmark.sh @@ -13,4 +13,5 @@ git clone https://github.com/google/benchmark.git rm -rf benchmark/.git* find benchmark/ -name BUILD -delete find benchmark/ -name BUILD.bazel -delete +find benchmark/ -name WORKSPACE -delete -- GitLab From a9304edf20756dd63f896a98bad89e9eac54aebd Mon Sep 17 00:00:00 2001 From: Thomas Preud'homme Date: Tue, 5 Mar 2024 19:32:27 +0000 Subject: [PATCH 193/929] Fix remaining build failures with GCC 8.3 (#83266) When compiling for GCC 8.x (< 8.4), SFINAE is disabled for iterator_range constructor causing ambiguous resolution to construct an OperandRange from a MutableOperatorRange, even in the presence of a static_cast. This adds an explicit conversion method to lift the ambiguity. Tested with a full MLIR build with GCC 8.3. --- mlir/include/mlir/IR/ValueRange.h | 3 +++ .../Transforms/OwnershipBasedBufferDeallocation.cpp | 4 ++-- .../Transforms/BufferDeallocationOpInterfaceImpl.cpp | 2 +- mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp | 2 +- mlir/lib/IR/OperationSupport.cpp | 7 ++++++- mlir/lib/Transforms/Utils/CFGToSCF.cpp | 3 +-- 6 files changed, 14 insertions(+), 7 deletions(-) diff --git a/mlir/include/mlir/IR/ValueRange.h b/mlir/include/mlir/IR/ValueRange.h index 51262e2d7871..4b421c08d841 100644 --- a/mlir/include/mlir/IR/ValueRange.h +++ b/mlir/include/mlir/IR/ValueRange.h @@ -155,6 +155,9 @@ public: /// Returns if the current range is empty. bool empty() const { return size() == 0; } + /// Explicit conversion to an OperandRange. + OperandRange getAsOperandRange() const; + /// Allow implicit conversion to an OperandRange. operator OperandRange() const; diff --git a/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp b/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp index 0a7494c86d86..c9fd110d48d9 100644 --- a/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp +++ b/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp @@ -956,13 +956,13 @@ BufferDeallocation::handleInterface(RegionBranchTerminatorOpInterface op) { SmallVector updatedOwnerships; auto result = deallocation_impl::insertDeallocOpForReturnLike( - state, op, OperandRange(operands), updatedOwnerships); + state, op, operands.getAsOperandRange(), updatedOwnerships); if (failed(result) || !*result) return result; // Add an additional operand for every MemRef for the ownership indicator. if (!funcWithoutDynamicOwnership) { - SmallVector newOperands{OperandRange(operands)}; + SmallVector newOperands{operands.getAsOperandRange()}; newOperands.append(updatedOwnerships.begin(), updatedOwnerships.end()); operands.assign(newOperands); } diff --git a/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp b/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp index 9423af254269..0dc357c2298f 100644 --- a/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp +++ b/mlir/lib/Dialect/ControlFlow/Transforms/BufferDeallocationOpInterfaceImpl.cpp @@ -84,7 +84,7 @@ struct CondBranchOpInterface DenseMap &mapping) -> DeallocOp { SmallVector toRetain; state.getMemrefsToRetain(condBr->getBlock(), target, - OperandRange(destOperands), toRetain); + destOperands.getAsOperandRange(), toRetain); SmallVector adaptedConditions( llvm::map_range(conditions, conditionModifier)); auto deallocOp = builder.create( diff --git a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp index a85532b2f755..223d728b0b27 100644 --- a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp +++ b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp @@ -149,7 +149,7 @@ static LinalgOp fuse(OpBuilder &b, LinalgOp producer, SmallVector resultTypes; resultTypes.reserve(producer->getNumResults()); int64_t firstInitOperandIdx = - static_cast(producerDpsInits).getBeginOperandIndex(); + producerDpsInits.getAsOperandRange().getBeginOperandIndex(); for (int64_t i = 0, e = producer->getNumResults(); i < e; ++i) { resultTypes.push_back(clonedShapes[firstInitOperandIdx + i].getType()); } diff --git a/mlir/lib/IR/OperationSupport.cpp b/mlir/lib/IR/OperationSupport.cpp index a168fe30ba8a..a72ccb9ca490 100644 --- a/mlir/lib/IR/OperationSupport.cpp +++ b/mlir/lib/IR/OperationSupport.cpp @@ -497,9 +497,14 @@ void MutableOperandRange::clear() { } } +/// Explicit conversion to an OperandRange. +OperandRange MutableOperandRange::getAsOperandRange() const { + return owner->getOperands().slice(start, length); +} + /// Allow implicit conversion to an OperandRange. MutableOperandRange::operator OperandRange() const { - return owner->getOperands().slice(start, length); + return getAsOperandRange(); } MutableOperandRange::operator MutableArrayRef() const { diff --git a/mlir/lib/Transforms/Utils/CFGToSCF.cpp b/mlir/lib/Transforms/Utils/CFGToSCF.cpp index f2998b4047e2..eefdf1d4e393 100644 --- a/mlir/lib/Transforms/Utils/CFGToSCF.cpp +++ b/mlir/lib/Transforms/Utils/CFGToSCF.cpp @@ -1183,8 +1183,7 @@ static FailureOr> transformToStructuredCFBranches( auto builder = OpBuilder::atBlockTerminator(user->getBlock()); LogicalResult result = interface.createStructuredBranchRegionTerminatorOp( user->getLoc(), builder, structuredCondOp, user, - static_cast( - getMutableSuccessorOperands(user->getBlock(), 0))); + getMutableSuccessorOperands(user->getBlock(), 0).getAsOperandRange()); if (failed(result)) return failure(); user->erase(); -- GitLab From 13cd0a905beab77fc31e08282c466246aae7eda6 Mon Sep 17 00:00:00 2001 From: Peter Klausler <35819229+klausler@users.noreply.github.com> Date: Tue, 5 Mar 2024 11:41:12 -0800 Subject: [PATCH 194/929] =?UTF-8?q?[flang]=20Skim=20usage=20before=20marki?= =?UTF-8?q?ng=20unknown=20module=20externals=20as=20subrout=E2=80=A6=20(#8?= =?UTF-8?q?3897)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …ines Name resolution needs to delay its default determination of module external procedures as subroutines until after it has skimmed the execution parts of module procedures. Fixes https://github.com/llvm/llvm-project/issues/83622. --- flang/lib/Semantics/resolve-names.cpp | 20 ++++++------ flang/test/Semantics/resolve09.f90 | 44 +++++++++++++++++++++------ 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp index 5a95d3a98992..26cae833edfc 100644 --- a/flang/lib/Semantics/resolve-names.cpp +++ b/flang/lib/Semantics/resolve-names.cpp @@ -8447,7 +8447,6 @@ void ResolveNamesVisitor::FinishSpecificationPart( misparsedStmtFuncFound_ = false; funcResultStack().CompleteFunctionResultType(); CheckImports(); - bool inModule{currScope().kind() == Scope::Kind::Module}; for (auto &pair : currScope()) { auto &symbol{*pair.second}; if (NeedsExplicitType(symbol)) { @@ -8462,13 +8461,6 @@ void ResolveNamesVisitor::FinishSpecificationPart( if (symbol.has()) { CheckGenericProcedures(symbol); } - if (inModule && symbol.attrs().test(Attr::EXTERNAL) && - !symbol.test(Symbol::Flag::Function) && - !symbol.test(Symbol::Flag::Subroutine)) { - // in a module, external proc without return type is subroutine - symbol.set( - symbol.GetType() ? Symbol::Flag::Function : Symbol::Flag::Subroutine); - } if (!symbol.has()) { CheckPossibleBadForwardRef(symbol); } @@ -8990,8 +8982,18 @@ void ResolveNamesVisitor::ResolveSpecificationParts(ProgramTree &node) { } EndScopeForNode(node); // Ensure that every object entity has a type. + bool inModule{node.GetKind() == ProgramTree::Kind::Module || + node.GetKind() == ProgramTree::Kind::Submodule}; for (auto &pair : *node.scope()) { - ApplyImplicitRules(*pair.second); + Symbol &symbol{*pair.second}; + if (inModule && symbol.attrs().test(Attr::EXTERNAL) && + !symbol.test(Symbol::Flag::Function) && + !symbol.test(Symbol::Flag::Subroutine)) { + // in a module, external proc without return type is subroutine + symbol.set( + symbol.GetType() ? Symbol::Flag::Function : Symbol::Flag::Subroutine); + } + ApplyImplicitRules(symbol); } } diff --git a/flang/test/Semantics/resolve09.f90 b/flang/test/Semantics/resolve09.f90 index c5e4277b3b61..634b9861f3b6 100644 --- a/flang/test/Semantics/resolve09.f90 +++ b/flang/test/Semantics/resolve09.f90 @@ -52,25 +52,49 @@ contains end end -module m - ! subroutine vs. function is determined at end of specification part - external :: a - procedure() :: b +module m1 + !Function vs subroutine in a module is resolved to a subroutine if + !no other information. + external :: exts, extf, extunk + procedure() :: procs, procf, procunk contains - subroutine s() - call a() - !ERROR: Cannot call subroutine 'b' like a function - x = b() + subroutine s + call exts() + call procs() + x = extf() + x = procf() end end +module m2 + use m1 + contains + subroutine test + call exts() ! ok + call procs() ! ok + call extunk() ! ok + call procunk() ! ok + x = extf() ! ok + x = procf() ! ok + !ERROR: Cannot call subroutine 'extunk' like a function + !ERROR: Function result characteristics are not known + x = extunk() + !ERROR: Cannot call subroutine 'procunk' like a function + !ERROR: Function result characteristics are not known + x = procunk() + end +end + +module modulename +end + ! Call to entity in global scope, even with IMPORT, NONE subroutine s4 block import, none integer :: i - !ERROR: 'm' is not a callable procedure - call m() + !ERROR: 'modulename' is not a callable procedure + call modulename() end block end -- GitLab From b3b408bbb379a1cfdac66a71534d86c23a9cf3f1 Mon Sep 17 00:00:00 2001 From: Petr Hosek Date: Tue, 5 Mar 2024 11:42:51 -0800 Subject: [PATCH 195/929] [Fuchsia] Include baremetal ARM builtins and libc (#83949) For now we only include the armv6m-unknown-eabi target but we plan to include more targets in the future (including multilibs). --- clang/cmake/caches/Fuchsia-stage2.cmake | 33 +++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/clang/cmake/caches/Fuchsia-stage2.cmake b/clang/cmake/caches/Fuchsia-stage2.cmake index eee37c5e7901..db7430b3344c 100644 --- a/clang/cmake/caches/Fuchsia-stage2.cmake +++ b/clang/cmake/caches/Fuchsia-stage2.cmake @@ -300,6 +300,39 @@ if(FUCHSIA_SDK) set(LLVM_RUNTIME_MULTILIB_hwasan+noexcept_TARGETS "aarch64-unknown-fuchsia;riscv64-unknown-fuchsia" CACHE STRING "") endif() +foreach(target armv6m-unknown-eabi) + list(APPEND BUILTIN_TARGETS "${target}") + set(BUILTINS_${target}_CMAKE_SYSTEM_NAME Generic CACHE STRING "") + set(BUILTINS_${target}_CMAKE_SYSTEM_PROCESSOR arm CACHE STRING "") + set(BUILTINS_${target}_CMAKE_SYSROOT "" CACHE STRING "") + set(BUILTINS_${target}_CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "") + foreach(lang C;CXX;ASM) + set(BUILTINS_${target}_CMAKE_${lang}_FLAGS "--target=${target} -mcpu=cortex-m0plus -mthumb" CACHE STRING "") + endforeach() + foreach(type SHARED;MODULE;EXE) + set(BUILTINS_${target}_CMAKE_${type}_LINKER_FLAGS "-fuse-ld=lld" CACHE STRING "") + endforeach() + set(BUILTINS_${target}_COMPILER_RT_BAREMETAL_BUILD ON CACHE BOOL "") + + list(APPEND RUNTIME_TARGETS "${target}") + set(RUNTIMES_${target}_CMAKE_SYSTEM_NAME Generic CACHE STRING "") + set(RUNTIMES_${target}_CMAKE_SYSTEM_PROCESSOR arm CACHE STRING "") + set(RUNTIMES_${target}_CMAKE_SYSROOT "" CACHE STRING "") + set(RUNTIMES_${target}_CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "") + set(RUNTIMES_${target}_CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY CACHE STRING "") + foreach(lang C;CXX;ASM) + set(RUNTIMES_${target}_CMAKE_${lang}_FLAGS "--target=${target} -mcpu=cortex-m0plus -mthumb" CACHE STRING "") + endforeach() + foreach(type SHARED;MODULE;EXE) + set(RUNTIMES_${target}_CMAKE_${type}_LINKER_FLAGS "-fuse-ld=lld" CACHE STRING "") + endforeach() + set(RUNTIMES_${target}_LLVM_LIBC_FULL_BUILD ON CACHE BOOL "") + set(RUNTIMES_${target}_LIBC_ENABLE_USE_BY_CLANG ON CACHE BOOL "") + set(RUNTIMES_${target}_LLVM_INCLUDE_TESTS OFF CACHE BOOL "") + set(RUNTIMES_${target}_LLVM_ENABLE_ASSERTIONS OFF CACHE BOOL "") + set(RUNTIMES_${target}_LLVM_ENABLE_RUNTIMES "libc" CACHE STRING "") +endforeach() + foreach(target riscv32-unknown-elf) list(APPEND BUILTIN_TARGETS "${target}") set(BUILTINS_${target}_CMAKE_SYSTEM_NAME Generic CACHE STRING "") -- GitLab From 83ca78deb9075e764f89fad83af926357eb78967 Mon Sep 17 00:00:00 2001 From: Peter Klausler <35819229+klausler@users.noreply.github.com> Date: Tue, 5 Mar 2024 12:00:46 -0800 Subject: [PATCH 196/929] [flang] Emit "raw" name for procedure interface in module file (#83915) Save both the raw procedure interface symbol as well as the result of passing it through GetUltimate() and BypassGeneric() in symbol table entries with ProcEntityDetails. The raw symbol of the interface needs to be the one used for emitting procedure symbols to module files. Fixes https://github.com/llvm/llvm-project/issues/83836. --- flang/include/flang/Semantics/symbol.h | 7 ++++- flang/lib/Semantics/check-declarations.cpp | 4 +-- flang/lib/Semantics/mod-file.cpp | 8 +++--- flang/lib/Semantics/resolve-names-utils.cpp | 5 ++-- flang/lib/Semantics/resolve-names.cpp | 16 ++++++------ flang/lib/Semantics/symbol.cpp | 3 +++ flang/lib/Semantics/tools.cpp | 4 +-- flang/test/Semantics/modfile64.f90 | 29 +++++++++++++++++++++ 8 files changed, 55 insertions(+), 21 deletions(-) create mode 100644 flang/test/Semantics/modfile64.f90 diff --git a/flang/include/flang/Semantics/symbol.h b/flang/include/flang/Semantics/symbol.h index 125025dab5f4..c3175a5d1a11 100644 --- a/flang/include/flang/Semantics/symbol.h +++ b/flang/include/flang/Semantics/symbol.h @@ -417,8 +417,12 @@ public: ProcEntityDetails(ProcEntityDetails &&) = default; ProcEntityDetails &operator=(const ProcEntityDetails &) = default; + const Symbol *rawProcInterface() const { return rawProcInterface_; } const Symbol *procInterface() const { return procInterface_; } - void set_procInterface(const Symbol &sym) { procInterface_ = &sym; } + void set_procInterfaces(const Symbol &raw, const Symbol &resolved) { + rawProcInterface_ = &raw; + procInterface_ = &resolved; + } inline bool HasExplicitInterface() const; // Be advised: !init().has_value() => uninitialized pointer, @@ -430,6 +434,7 @@ public: void set_isCUDAKernel(bool yes = true) { isCUDAKernel_ = yes; } private: + const Symbol *rawProcInterface_{nullptr}; const Symbol *procInterface_{nullptr}; std::optional init_; bool isCUDAKernel_{false}; diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp index 729321d3bf17..431cef207935 100644 --- a/flang/lib/Semantics/check-declarations.cpp +++ b/flang/lib/Semantics/check-declarations.cpp @@ -1195,9 +1195,7 @@ void CheckHelper::CheckArraySpec( void CheckHelper::CheckProcEntity( const Symbol &symbol, const ProcEntityDetails &details) { CheckSymbolType(symbol); - const Symbol *interface { - details.procInterface() ? &details.procInterface()->GetUltimate() : nullptr - }; + const Symbol *interface{details.procInterface()}; if (details.isDummy()) { if (!symbol.attrs().test(Attr::POINTER) && // C843 (symbol.attrs().test(Attr::INTENT_IN) || diff --git a/flang/lib/Semantics/mod-file.cpp b/flang/lib/Semantics/mod-file.cpp index 37fe0240537b..b4df7216a33e 100644 --- a/flang/lib/Semantics/mod-file.cpp +++ b/flang/lib/Semantics/mod-file.cpp @@ -924,8 +924,8 @@ void ModFileWriter::PutProcEntity(llvm::raw_ostream &os, const Symbol &symbol) { os, symbol, [&]() { os << "procedure("; - if (details.procInterface()) { - os << details.procInterface()->name(); + if (details.rawProcInterface()) { + os << details.rawProcInterface()->name(); } else if (details.type()) { PutType(os, *details.type()); } @@ -1622,8 +1622,8 @@ void SubprogramSymbolCollector::DoSymbol( } }, [this](const ProcEntityDetails &details) { - if (details.procInterface()) { - DoSymbol(*details.procInterface()); + if (details.rawProcInterface()) { + DoSymbol(*details.rawProcInterface()); } else { DoType(details.type()); } diff --git a/flang/lib/Semantics/resolve-names-utils.cpp b/flang/lib/Semantics/resolve-names-utils.cpp index b901080e2860..801473876e7e 100644 --- a/flang/lib/Semantics/resolve-names-utils.cpp +++ b/flang/lib/Semantics/resolve-names-utils.cpp @@ -845,8 +845,9 @@ void SymbolMapper::MapSymbolExprs(Symbol &symbol) { }, [&](ProcEntityDetails &proc) { if (const Symbol * - mappedSymbol{MapInterface(proc.procInterface())}) { - proc.set_procInterface(*mappedSymbol); + mappedSymbol{MapInterface(proc.rawProcInterface())}) { + proc.set_procInterfaces( + *mappedSymbol, BypassGeneric(mappedSymbol->GetUltimate())); } else if (const DeclTypeSpec * mappedType{MapType(proc.type())}) { proc.set_type(*mappedType); } diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp index 26cae833edfc..67392a02cf18 100644 --- a/flang/lib/Semantics/resolve-names.cpp +++ b/flang/lib/Semantics/resolve-names.cpp @@ -4983,7 +4983,8 @@ Symbol &DeclarationVisitor::DeclareProcEntity( "The interface for procedure '%s' has already been declared"_err_en_US); context().SetError(symbol); } else if (interface) { - details->set_procInterface(*interface); + details->set_procInterfaces( + *interface, BypassGeneric(interface->GetUltimate())); if (interface->test(Symbol::Flag::Function)) { symbol.set(Symbol::Flag::Function); } else if (interface->test(Symbol::Flag::Subroutine)) { @@ -5658,10 +5659,10 @@ void DeclarationVisitor::Post(const parser::ProcInterface &x) { } void DeclarationVisitor::Post(const parser::ProcDecl &x) { const auto &name{std::get(x.t)}; - const Symbol *procInterface{nullptr}; - if (interfaceName_ && interfaceName_->symbol) { - procInterface = &BypassGeneric(*interfaceName_->symbol); - } + // Don't use BypassGeneric or GetUltimate on this symbol, they can + // lead to unusable names in module files. + const Symbol *procInterface{ + interfaceName_ ? interfaceName_->symbol : nullptr}; auto attrs{HandleSaveName(name.source, GetAttrs())}; DerivedTypeDetails *dtDetails{nullptr}; if (Symbol * symbol{currScope().symbol()}) { @@ -6624,10 +6625,9 @@ void DeclarationVisitor::CheckExplicitInterface(const parser::Name &name) { if (const Symbol * symbol{name.symbol}) { const Symbol &ultimate{symbol->GetUltimate()}; if (!context().HasError(*symbol) && !context().HasError(ultimate) && - !ultimate.HasExplicitInterface()) { + !BypassGeneric(ultimate).HasExplicitInterface()) { Say(name, - "'%s' must be an abstract interface or a procedure with " - "an explicit interface"_err_en_US, + "'%s' must be an abstract interface or a procedure with an explicit interface"_err_en_US, symbol->name()); } } diff --git a/flang/lib/Semantics/symbol.cpp b/flang/lib/Semantics/symbol.cpp index 2ab3189cf406..381905b89fb2 100644 --- a/flang/lib/Semantics/symbol.cpp +++ b/flang/lib/Semantics/symbol.cpp @@ -452,6 +452,9 @@ llvm::raw_ostream &operator<<( llvm::raw_ostream &operator<<( llvm::raw_ostream &os, const ProcEntityDetails &x) { if (x.procInterface_) { + if (x.rawProcInterface_ != x.procInterface_) { + os << ' ' << x.rawProcInterface_->name() << " ->"; + } os << ' ' << x.procInterface_->name(); } else { DumpType(os, x.type()); diff --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp index f931ae070720..bf999b090419 100644 --- a/flang/lib/Semantics/tools.cpp +++ b/flang/lib/Semantics/tools.cpp @@ -465,9 +465,7 @@ const Symbol *FindInterface(const Symbol &symbol) { return common::visit( common::visitors{ [](const ProcEntityDetails &details) { - const Symbol *interface { - details.procInterface() - }; + const Symbol *interface{details.procInterface()}; return interface ? FindInterface(*interface) : nullptr; }, [](const ProcBindingDetails &details) { diff --git a/flang/test/Semantics/modfile64.f90 b/flang/test/Semantics/modfile64.f90 new file mode 100644 index 000000000000..e9df005c9ffd --- /dev/null +++ b/flang/test/Semantics/modfile64.f90 @@ -0,0 +1,29 @@ +! RUN: %python %S/test_modfile.py %s %flang_fc1 +module mod0 + interface proc + module procedure proc + end interface + contains + subroutine proc + end +end +module mod1 + use mod0,renamed_proc=>proc + procedure(renamed_proc),pointer :: p +end module + +!Expect: mod0.mod +!module mod0 +!interface proc +!procedure::proc +!end interface +!contains +!subroutine proc() +!end +!end + +!Expect: mod1.mod +!module mod1 +!use mod0,only:renamed_proc=>proc +!procedure(renamed_proc),pointer::p +!end -- GitLab From 2f343fc1574f36b3b5ff1acf63407c53dcdac331 Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Tue, 5 Mar 2024 12:04:32 -0800 Subject: [PATCH 197/929] [lldb] Print a message when background tasks take a while to complete (#82799) When terminating the debugger, we wait for all background tasks to complete. Given that there's no way to interrupt those treads, this can take a while. When that happens, the debugger appears to hang at exit. The above situation is unfortunately not uncommon when background downloading of dSYMs is enabled (`symbols.auto-download background`). Even when calling dsymForUUID with a reasonable timeout, it can take a while to complete. This patch improves the user experience by printing a message from the driver when it takes more than one (1) second to terminate the debugger. --- lldb/tools/driver/Driver.cpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/lldb/tools/driver/Driver.cpp b/lldb/tools/driver/Driver.cpp index 9286abb27e13..a821699c5e2e 100644 --- a/lldb/tools/driver/Driver.cpp +++ b/lldb/tools/driver/Driver.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -801,6 +802,18 @@ int main(int argc, char const *argv[]) { } } - SBDebugger::Terminate(); + // When terminating the debugger we have to wait on all the background tasks + // to complete, which can take a while. Print a message when this takes longer + // than 1 second. + { + std::future future = + std::async(std::launch::async, []() { SBDebugger::Terminate(); }); + + if (future.wait_for(std::chrono::seconds(1)) == std::future_status::timeout) + fprintf(stderr, "Waiting for background tasks to complete...\n"); + + future.wait(); + } + return exit_code; } -- GitLab From c8b3edcddd221a5007565507fb16a07a027d32bc Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Tue, 5 Mar 2024 12:07:46 -0800 Subject: [PATCH 198/929] Revert "[SLP][NFC]Use TargetTransformInfo:: instead of TTI:: in BoUpSLP to avoid" This reverts commit 083d8aa03aca55b88098a91e41e41a8e321a5721. --- llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 69bbaae0e724..8cf59a18381a 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -2379,8 +2379,7 @@ private: const TreeEntry *getOperandEntry(const TreeEntry *E, unsigned Idx) const; /// \returns Cast context for the given graph node. - TargetTransformInfo::CastContextHint - getCastContextHint(const TreeEntry &TE) const; + TTI::CastContextHint getCastContextHint(const TreeEntry &TE) const; /// \returns the cost of the vectorizable entry. InstructionCost getEntryCost(const TreeEntry *E, @@ -7038,8 +7037,7 @@ static bool isAlternateInstruction(const Instruction *I, return I->getOpcode() == AltOp->getOpcode(); } -TargetTransformInfo::OperandValueInfo -BoUpSLP::getOperandInfo(ArrayRef Ops) { +TTI::OperandValueInfo BoUpSLP::getOperandInfo(ArrayRef Ops) { assert(!Ops.empty()); const auto *Op0 = Ops.front(); -- GitLab From aae152f1be1db1b4760c7464a3accd3d72b5f4bd Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Tue, 5 Mar 2024 12:08:02 -0800 Subject: [PATCH 199/929] Revert "[SLP]Improve minbitwidth analysis." This reverts commit a730ed7c1a4a35f5219df720ffb0ba6122d64fe4 to fix compile time issue. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 522 ++++++------------ .../SLPVectorizer/AArch64/ext-trunc.ll | 9 +- .../SLPVectorizer/AArch64/getelementptr2.ll | 4 +- .../SLPVectorizer/AArch64/reduce-add-i64.ll | 20 +- .../SLPVectorizer/RISCV/reductions.ll | 7 +- .../Transforms/SLPVectorizer/X86/PR35777.ll | 9 +- .../X86/int-bitcast-minbitwidth.ll | 2 +- ...minbitwidth-multiuse-with-insertelement.ll | 17 +- .../X86/minbitwidth-transformed-operand.ll | 21 +- .../SLPVectorizer/X86/minimum-sizes.ll | 43 +- .../SLPVectorizer/X86/phi-undef-input.ll | 24 +- .../Transforms/SLPVectorizer/X86/resched.ll | 32 +- .../X86/reused-reductions-with-minbitwidth.ll | 10 +- .../X86/store-insertelement-minbitwidth.ll | 22 +- .../SLPVectorizer/alt-cmp-vectorize.ll | 4 +- 15 files changed, 295 insertions(+), 451 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 8cf59a18381a..36dc9094538a 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -1085,7 +1085,6 @@ public: BS->clear(); } MinBWs.clear(); - ReductionBitWidth = 0; InstrElementSize.clear(); UserIgnoreList = nullptr; PostponedGathers.clear(); @@ -2308,11 +2307,9 @@ private: /// constant and to be demoted. Required to correctly identify constant nodes /// to be demoted. bool collectValuesToDemote( - Value *V, bool IsProfitableToDemoteRoot, unsigned &BitWidth, - SmallVectorImpl &ToDemote, + Value *V, SmallVectorImpl &ToDemote, DenseMap> &DemotedConsts, - DenseSet &Visited, unsigned &MaxDepthLevel, - bool &IsProfitableToDemote) const; + SmallVectorImpl &Roots, DenseSet &Visited) const; /// Check if the operands on the edges \p Edges of the \p UserTE allows /// reordering (i.e. the operands can be reordered because they have only one @@ -2378,9 +2375,6 @@ private: /// \ returns the graph entry for the \p Idx operand of the \p E entry. const TreeEntry *getOperandEntry(const TreeEntry *E, unsigned Idx) const; - /// \returns Cast context for the given graph node. - TTI::CastContextHint getCastContextHint(const TreeEntry &TE) const; - /// \returns the cost of the vectorizable entry. InstructionCost getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, @@ -3635,11 +3629,6 @@ private: /// value must be signed-extended, rather than zero-extended, back to its /// original width. DenseMap> MinBWs; - - /// Final size of the reduced vector, if the current graph represents the - /// input for the reduction and it was possible to narrow the size of the - /// reduction. - unsigned ReductionBitWidth = 0; }; } // end namespace slpvectorizer @@ -8373,22 +8362,6 @@ const BoUpSLP::TreeEntry *BoUpSLP::getOperandEntry(const TreeEntry *E, return It->get(); } -TTI::CastContextHint BoUpSLP::getCastContextHint(const TreeEntry &TE) const { - if (TE.State == TreeEntry::ScatterVectorize || - TE.State == TreeEntry::StridedVectorize) - return TTI::CastContextHint::GatherScatter; - if (TE.State == TreeEntry::Vectorize && TE.getOpcode() == Instruction::Load && - !TE.isAltShuffle()) { - if (TE.ReorderIndices.empty()) - return TTI::CastContextHint::Normal; - SmallVector Mask; - inversePermutation(TE.ReorderIndices, Mask); - if (ShuffleVectorInst::isReverseMask(Mask, Mask.size())) - return TTI::CastContextHint::Reversed; - } - return TTI::CastContextHint::None; -} - InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, SmallPtrSetImpl &CheckedExtracts) { @@ -8411,7 +8384,6 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, // If we have computed a smaller type for the expression, update VecTy so // that the costs will be accurate. auto It = MinBWs.find(E); - Type *OrigScalarTy = ScalarTy; if (It != MinBWs.end()) { ScalarTy = IntegerType::get(F->getContext(), It->second.first); VecTy = FixedVectorType::get(ScalarTy, VL.size()); @@ -8469,11 +8441,24 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, UsedScalars.set(I); } auto GetCastContextHint = [&](Value *V) { - if (const TreeEntry *OpTE = getTreeEntry(V)) - return getCastContextHint(*OpTE); - InstructionsState SrcState = getSameOpcode(E->getOperand(0), *TLI); - if (SrcState.getOpcode() == Instruction::Load && !SrcState.isAltShuffle()) - return TTI::CastContextHint::GatherScatter; + if (const TreeEntry *OpTE = getTreeEntry(V)) { + if (OpTE->State == TreeEntry::ScatterVectorize || + OpTE->State == TreeEntry::StridedVectorize) + return TTI::CastContextHint::GatherScatter; + if (OpTE->State == TreeEntry::Vectorize && + OpTE->getOpcode() == Instruction::Load && !OpTE->isAltShuffle()) { + if (OpTE->ReorderIndices.empty()) + return TTI::CastContextHint::Normal; + SmallVector Mask; + inversePermutation(OpTE->ReorderIndices, Mask); + if (ShuffleVectorInst::isReverseMask(Mask, Mask.size())) + return TTI::CastContextHint::Reversed; + } + } else { + InstructionsState SrcState = getSameOpcode(E->getOperand(0), *TLI); + if (SrcState.getOpcode() == Instruction::Load && !SrcState.isAltShuffle()) + return TTI::CastContextHint::GatherScatter; + } return TTI::CastContextHint::None; }; auto GetCostDiff = @@ -8522,6 +8507,8 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, TTI::CastContextHint CCH = GetCastContextHint(VL0); VecCost += TTI->getCastInstrCost(VecOpcode, UserVecTy, VecTy, CCH, CostKind); + ScalarCost += Sz * TTI->getCastInstrCost(VecOpcode, UserScalarTy, + ScalarTy, CCH, CostKind); } } } @@ -8538,7 +8525,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, InstructionCost ScalarCost = 0; InstructionCost VecCost = 0; std::tie(ScalarCost, VecCost) = getGEPCosts( - *TTI, Ptrs, BasePtr, E->getOpcode(), CostKind, OrigScalarTy, VecTy); + *TTI, Ptrs, BasePtr, E->getOpcode(), CostKind, ScalarTy, VecTy); LLVM_DEBUG(dumpTreeCosts(E, 0, VecCost, ScalarCost, "Calculated GEPs cost for Tree")); @@ -8585,7 +8572,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, NumElts = ATy->getNumElements(); else NumElts = AggregateTy->getStructNumElements(); - SrcVecTy = FixedVectorType::get(OrigScalarTy, NumElts); + SrcVecTy = FixedVectorType::get(ScalarTy, NumElts); } if (I->hasOneUse()) { Instruction *Ext = I->user_back(); @@ -8753,7 +8740,13 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, } } auto GetScalarCost = [&](unsigned Idx) -> InstructionCost { - auto *VI = cast(UniqueValues[Idx]); + // Do not count cost here if minimum bitwidth is in effect and it is just + // a bitcast (here it is just a noop). + if (VecOpcode != Opcode && VecOpcode == Instruction::BitCast) + return TTI::TCC_Free; + auto *VI = VL0->getOpcode() == Opcode + ? cast(UniqueValues[Idx]) + : nullptr; return TTI->getCastInstrCost(Opcode, VL0->getType(), VL0->getOperand(0)->getType(), TTI::getCastContextHint(VI), CostKind, VI); @@ -8796,7 +8789,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, ? CmpInst::BAD_FCMP_PREDICATE : CmpInst::BAD_ICMP_PREDICATE; - return TTI->getCmpSelInstrCost(E->getOpcode(), OrigScalarTy, + return TTI->getCmpSelInstrCost(E->getOpcode(), ScalarTy, Builder.getInt1Ty(), CurrentPred, CostKind, VI); }; @@ -8851,7 +8844,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(VI->getOperand(OpIdx)); SmallVector Operands(VI->operand_values()); - return TTI->getArithmeticInstrCost(ShuffleOrOp, OrigScalarTy, CostKind, + return TTI->getArithmeticInstrCost(ShuffleOrOp, ScalarTy, CostKind, Op1Info, Op2Info, Operands, VI); }; auto GetVectorCost = [=](InstructionCost CommonCost) { @@ -8870,9 +8863,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, case Instruction::Load: { auto GetScalarCost = [&](unsigned Idx) { auto *VI = cast(UniqueValues[Idx]); - return TTI->getMemoryOpCost(Instruction::Load, OrigScalarTy, - VI->getAlign(), VI->getPointerAddressSpace(), - CostKind, TTI::OperandValueInfo(), VI); + return TTI->getMemoryOpCost(Instruction::Load, ScalarTy, VI->getAlign(), + VI->getPointerAddressSpace(), CostKind, + TTI::OperandValueInfo(), VI); }; auto *LI0 = cast(VL0); auto GetVectorCost = [&](InstructionCost CommonCost) { @@ -8915,9 +8908,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, auto GetScalarCost = [=](unsigned Idx) { auto *VI = cast(VL[Idx]); TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(VI->getValueOperand()); - return TTI->getMemoryOpCost(Instruction::Store, OrigScalarTy, - VI->getAlign(), VI->getPointerAddressSpace(), - CostKind, OpInfo, VI); + return TTI->getMemoryOpCost(Instruction::Store, ScalarTy, VI->getAlign(), + VI->getPointerAddressSpace(), CostKind, + OpInfo, VI); }; auto *BaseSI = cast(IsReorder ? VL[E->ReorderIndices.front()] : VL0); @@ -9779,44 +9772,6 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef VectorizedVals) { Cost -= InsertCost; } - // Add the cost for reduced value resize (if required). - if (ReductionBitWidth != 0) { - assert(UserIgnoreList && "Expected reduction tree."); - const TreeEntry &E = *VectorizableTree.front().get(); - auto It = MinBWs.find(&E); - if (It != MinBWs.end() && It->second.first != ReductionBitWidth) { - unsigned SrcSize = It->second.first; - unsigned DstSize = ReductionBitWidth; - unsigned Opcode = Instruction::Trunc; - if (SrcSize < DstSize) - Opcode = It->second.second ? Instruction::SExt : Instruction::ZExt; - auto *SrcVecTy = - FixedVectorType::get(Builder.getIntNTy(SrcSize), E.getVectorFactor()); - auto *DstVecTy = - FixedVectorType::get(Builder.getIntNTy(DstSize), E.getVectorFactor()); - TTI::CastContextHint CCH = getCastContextHint(E); - InstructionCost CastCost; - switch (E.getOpcode()) { - case Instruction::SExt: - case Instruction::ZExt: - case Instruction::Trunc: { - const TreeEntry *OpTE = getOperandEntry(&E, 0); - CCH = getCastContextHint(*OpTE); - break; - } - default: - break; - } - CastCost += TTI->getCastInstrCost(Opcode, DstVecTy, SrcVecTy, CCH, - TTI::TCK_RecipThroughput); - Cost += CastCost; - LLVM_DEBUG(dbgs() << "SLP: Adding cost " << CastCost - << " for final resize for reduction from " << SrcVecTy - << " to " << DstVecTy << "\n"; - dbgs() << "SLP: Current total cost = " << Cost << "\n"); - } - } - #ifndef NDEBUG SmallString<256> Str; { @@ -12974,21 +12929,7 @@ Value *BoUpSLP::vectorizeTree( Builder.ClearInsertionPoint(); InstrElementSize.clear(); - const TreeEntry &RootTE = *VectorizableTree.front().get(); - Value *Vec = RootTE.VectorizedValue; - if (auto It = MinBWs.find(&RootTE); ReductionBitWidth != 0 && - It != MinBWs.end() && - ReductionBitWidth != It->second.first) { - IRBuilder<>::InsertPointGuard Guard(Builder); - Builder.SetInsertPoint(ReductionRoot->getParent(), - ReductionRoot->getIterator()); - Vec = Builder.CreateIntCast( - Vec, - VectorType::get(Builder.getIntNTy(ReductionBitWidth), - cast(Vec->getType())->getElementCount()), - It->second.second); - } - return Vec; + return VectorizableTree[0]->VectorizedValue; } void BoUpSLP::optimizeGatherSequence() { @@ -13808,21 +13749,16 @@ unsigned BoUpSLP::getVectorElementSize(Value *V) { // smaller type with a truncation. We collect the values that will be demoted // in ToDemote and additional roots that require investigating in Roots. bool BoUpSLP::collectValuesToDemote( - Value *V, bool IsProfitableToDemoteRoot, unsigned &BitWidth, - SmallVectorImpl &ToDemote, + Value *V, SmallVectorImpl &ToDemote, DenseMap> &DemotedConsts, - DenseSet &Visited, unsigned &MaxDepthLevel, - bool &IsProfitableToDemote) const { + SmallVectorImpl &Roots, DenseSet &Visited) const { // We can always demote constants. - if (isa(V)) { - MaxDepthLevel = 1; + if (isa(V)) return true; - } // If the value is not a vectorized instruction in the expression and not used // by the insertelement instruction and not used in multiple vector nodes, it // cannot be demoted. - // TODO: improve handling of gathered values and others. auto *I = dyn_cast(V); if (!I || !getTreeEntry(I) || MultiNodeScalars.contains(I) || !Visited.insert(I).second || all_of(I->users(), [&](User *U) { @@ -13830,20 +13766,6 @@ bool BoUpSLP::collectValuesToDemote( })) return false; - auto IsPotentiallyTruncated = [&](Value *V, unsigned &BitWidth) -> bool { - if (MultiNodeScalars.contains(V)) - return false; - uint32_t OrigBitWidth = DL->getTypeSizeInBits(V->getType()); - APInt Mask = APInt::getBitsSetFrom(OrigBitWidth, BitWidth); - if (MaskedValueIsZero(V, Mask, SimplifyQuery(*DL))) - return true; - auto NumSignBits = ComputeNumSignBits(V, *DL, 0, AC, nullptr, DT); - unsigned BitWidth1 = OrigBitWidth - NumSignBits; - if (!isKnownNonNegative(V, SimplifyQuery(*DL))) - ++BitWidth1; - BitWidth = std::max(BitWidth, BitWidth1); - return BitWidth > 0 && OrigBitWidth >= (BitWidth * 2); - }; unsigned Start = 0; unsigned End = I->getNumOperands(); switch (I->getOpcode()) { @@ -13851,14 +13773,12 @@ bool BoUpSLP::collectValuesToDemote( // We can always demote truncations and extensions. Since truncations can // seed additional demotion, we save the truncated value. case Instruction::Trunc: - MaxDepthLevel = 1; - if (IsProfitableToDemoteRoot) - IsProfitableToDemote = true; + Roots.push_back(I->getOperand(0)); break; case Instruction::ZExt: case Instruction::SExt: - MaxDepthLevel = 1; - IsProfitableToDemote = true; + if (isa(I->getOperand(0))) + return false; break; // We can demote certain binary operations if we can demote both of their @@ -13868,32 +13788,23 @@ bool BoUpSLP::collectValuesToDemote( case Instruction::Mul: case Instruction::And: case Instruction::Or: - case Instruction::Xor: { - unsigned Level1, Level2; - if (!collectValuesToDemote(I->getOperand(0), IsProfitableToDemoteRoot, - BitWidth, ToDemote, DemotedConsts, Visited, - Level1, IsProfitableToDemote) || - !collectValuesToDemote(I->getOperand(1), IsProfitableToDemoteRoot, - BitWidth, ToDemote, DemotedConsts, Visited, - Level2, IsProfitableToDemote)) + case Instruction::Xor: + if (!collectValuesToDemote(I->getOperand(0), ToDemote, DemotedConsts, Roots, + Visited) || + !collectValuesToDemote(I->getOperand(1), ToDemote, DemotedConsts, Roots, + Visited)) return false; - MaxDepthLevel = std::max(Level1, Level2); break; - } // We can demote selects if we can demote their true and false values. case Instruction::Select: { Start = 1; - unsigned Level1, Level2; SelectInst *SI = cast(I); - if (!collectValuesToDemote(SI->getTrueValue(), IsProfitableToDemoteRoot, - BitWidth, ToDemote, DemotedConsts, Visited, - Level1, IsProfitableToDemote) || - !collectValuesToDemote(SI->getFalseValue(), IsProfitableToDemoteRoot, - BitWidth, ToDemote, DemotedConsts, Visited, - Level2, IsProfitableToDemote)) + if (!collectValuesToDemote(SI->getTrueValue(), ToDemote, DemotedConsts, + Roots, Visited) || + !collectValuesToDemote(SI->getFalseValue(), ToDemote, DemotedConsts, + Roots, Visited)) return false; - MaxDepthLevel = std::max(Level1, Level2); break; } @@ -13902,236 +13813,171 @@ bool BoUpSLP::collectValuesToDemote( case Instruction::PHI: { PHINode *PN = cast(I); for (Value *IncValue : PN->incoming_values()) - if (!collectValuesToDemote(IncValue, IsProfitableToDemoteRoot, BitWidth, - ToDemote, DemotedConsts, Visited, - MaxDepthLevel, IsProfitableToDemote)) + if (!collectValuesToDemote(IncValue, ToDemote, DemotedConsts, Roots, + Visited)) return false; break; } // Otherwise, conservatively give up. default: - if (!IsPotentiallyTruncated(I, BitWidth)) - return false; - MaxDepthLevel = 0; - Start = End = 0; - break; + return false; } - ++MaxDepthLevel; // Gather demoted constant operands. for (unsigned Idx : seq(Start, End)) if (isa(I->getOperand(Idx))) DemotedConsts.try_emplace(I).first->getSecond().push_back(Idx); // Record the value that we can demote. ToDemote.push_back(V); - return IsProfitableToDemote; + return true; } void BoUpSLP::computeMinimumValueSizes() { // We only attempt to truncate integer expressions. - bool IsStoreOrInsertElt = - VectorizableTree.front()->getOpcode() == Instruction::Store || - VectorizableTree.front()->getOpcode() == Instruction::InsertElement; - unsigned NodeIdx = 0; - if (IsStoreOrInsertElt && - VectorizableTree.front()->State != TreeEntry::NeedToGather) - NodeIdx = 1; + auto &TreeRoot = VectorizableTree[0]->Scalars; + auto *TreeRootIT = dyn_cast(TreeRoot[0]->getType()); + if (!TreeRootIT || VectorizableTree.front()->State == TreeEntry::NeedToGather) + return; // Ensure the roots of the vectorizable tree don't form a cycle. - if (VectorizableTree[NodeIdx]->State == TreeEntry::NeedToGather || - (NodeIdx == 0 && !VectorizableTree[NodeIdx]->UserTreeIndices.empty()) || - (NodeIdx != 0 && any_of(VectorizableTree[NodeIdx]->UserTreeIndices, - [NodeIdx](const EdgeInfo &EI) { - return EI.UserTE->Idx > - static_cast(NodeIdx); - }))) + if (!VectorizableTree.front()->UserTreeIndices.empty()) return; - // The first value node for store/insertelement is sext/zext/trunc? Skip it, - // resize to the final type. - bool IsProfitableToDemoteRoot = !IsStoreOrInsertElt; - if (NodeIdx != 0 && - VectorizableTree[NodeIdx]->State == TreeEntry::Vectorize && - (VectorizableTree[NodeIdx]->getOpcode() == Instruction::ZExt || - VectorizableTree[NodeIdx]->getOpcode() == Instruction::SExt || - VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc)) { - assert(IsStoreOrInsertElt && "Expected store/insertelement seeded graph."); - ++NodeIdx; - IsProfitableToDemoteRoot = true; + // Conservatively determine if we can actually truncate the roots of the + // expression. Collect the values that can be demoted in ToDemote and + // additional roots that require investigating in Roots. + SmallVector ToDemote; + DenseMap> DemotedConsts; + SmallVector Roots; + for (auto *Root : TreeRoot) { + DenseSet Visited; + if (!collectValuesToDemote(Root, ToDemote, DemotedConsts, Roots, Visited)) + return; } - SmallVector ToDemote; - DenseMap> DemotedConsts; - auto ComputeMaxBitWidth = [&](ArrayRef TreeRoot, unsigned VF, - bool IsTopRoot, bool IsProfitableToDemoteRoot, - unsigned Opcode, unsigned Limit) { - ToDemote.clear(); - auto *TreeRootIT = dyn_cast(TreeRoot[0]->getType()); - if (!TreeRootIT || !Opcode) - return 0u; - - unsigned NumParts = TTI->getNumberOfParts( - FixedVectorType::get(TreeRoot.front()->getType(), VF)); - - // The maximum bit width required to represent all the values that can be - // demoted without loss of precision. It would be safe to truncate the roots - // of the expression to this width. - unsigned MaxBitWidth = 1u; - - // True if the roots can be zero-extended back to their original type, - // rather than sign-extended. We know that if the leading bits are not - // demanded, we can safely zero-extend. So we initialize IsKnownPositive to - // True. + // The maximum bit width required to represent all the values that can be + // demoted without loss of precision. It would be safe to truncate the roots + // of the expression to this width. + auto MaxBitWidth = 1u; + + // We first check if all the bits of the roots are demanded. If they're not, + // we can truncate the roots to this narrower type. + for (auto *Root : TreeRoot) { + auto Mask = DB->getDemandedBits(cast(Root)); + MaxBitWidth = std::max(Mask.getBitWidth() - Mask.countl_zero(), + MaxBitWidth); + } + + // True if the roots can be zero-extended back to their original type, rather + // than sign-extended. We know that if the leading bits are not demanded, we + // can safely zero-extend. So we initialize IsKnownPositive to True. + bool IsKnownPositive = true; + + // If all the bits of the roots are demanded, we can try a little harder to + // compute a narrower type. This can happen, for example, if the roots are + // getelementptr indices. InstCombine promotes these indices to the pointer + // width. Thus, all their bits are technically demanded even though the + // address computation might be vectorized in a smaller type. + // + // We start by looking at each entry that can be demoted. We compute the + // maximum bit width required to store the scalar by using ValueTracking to + // compute the number of high-order bits we can truncate. + if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType()) && + all_of(TreeRoot, [](Value *V) { + return all_of(V->users(), + [](User *U) { return isa(U); }); + })) { + MaxBitWidth = 8u; + // Determine if the sign bit of all the roots is known to be zero. If not, // IsKnownPositive is set to False. - bool IsKnownPositive = all_of(TreeRoot, [&](Value *R) { + IsKnownPositive = llvm::all_of(TreeRoot, [&](Value *R) { KnownBits Known = computeKnownBits(R, *DL); return Known.isNonNegative(); }); - // We first check if all the bits of the roots are demanded. If they're not, - // we can truncate the roots to this narrower type. - for (auto *Root : TreeRoot) { - unsigned NumSignBits = ComputeNumSignBits(Root, *DL, 0, AC, nullptr, DT); - TypeSize NumTypeBits = DL->getTypeSizeInBits(Root->getType()); - unsigned BitWidth1 = NumTypeBits - NumSignBits; - // If we can't prove that the sign bit is zero, we must add one to the - // maximum bit width to account for the unknown sign bit. This preserves - // the existing sign bit so we can safely sign-extend the root back to the - // original type. Otherwise, if we know the sign bit is zero, we will - // zero-extend the root instead. - // - // FIXME: This is somewhat suboptimal, as there will be cases where adding - // one to the maximum bit width will yield a larger-than-necessary - // type. In general, we need to add an extra bit only if we can't - // prove that the upper bit of the original type is equal to the - // upper bit of the proposed smaller type. If these two bits are - // the same (either zero or one) we know that sign-extending from - // the smaller type will result in the same value. Here, since we - // can't yet prove this, we are just making the proposed smaller - // type larger to ensure correctness. - if (!IsKnownPositive) - ++BitWidth1; - - APInt Mask = DB->getDemandedBits(cast(Root)); - unsigned BitWidth2 = Mask.getBitWidth() - Mask.countl_zero(); - MaxBitWidth = - std::max(std::min(BitWidth1, BitWidth2), MaxBitWidth); - } - - if (MaxBitWidth < 8 && MaxBitWidth > 1) - MaxBitWidth = 8; - - // If the original type is large, but reduced type does not improve the reg - // use - ignore it. - if (NumParts > 1 && - NumParts == - TTI->getNumberOfParts(FixedVectorType::get( - IntegerType::get(F->getContext(), bit_ceil(MaxBitWidth)), VF))) - return 0u; - - bool IsProfitableToDemote = Opcode == Instruction::Trunc || - Opcode == Instruction::SExt || - Opcode == Instruction::ZExt || NumParts > 1; - // Conservatively determine if we can actually truncate the roots of the - // expression. Collect the values that can be demoted in ToDemote and - // additional roots that require investigating in Roots. - for (auto *Root : TreeRoot) { - DenseSet Visited; - unsigned MaxDepthLevel; - bool NeedToDemote = IsProfitableToDemote; - if (!collectValuesToDemote(Root, IsProfitableToDemoteRoot, MaxBitWidth, - ToDemote, DemotedConsts, Visited, - MaxDepthLevel, NeedToDemote) || - (MaxDepthLevel <= Limit && Opcode != Instruction::Trunc && - Opcode != Instruction::SExt && Opcode != Instruction::ZExt)) - return 0u; - } - // Round MaxBitWidth up to the next power-of-two. - MaxBitWidth = bit_ceil(MaxBitWidth); - - return MaxBitWidth; - }; + // Determine the maximum number of bits required to store the scalar + // values. + for (auto *Scalar : ToDemote) { + auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, nullptr, DT); + auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType()); + MaxBitWidth = std::max(NumTypeBits - NumSignBits, MaxBitWidth); + } + + // If we can't prove that the sign bit is zero, we must add one to the + // maximum bit width to account for the unknown sign bit. This preserves + // the existing sign bit so we can safely sign-extend the root back to the + // original type. Otherwise, if we know the sign bit is zero, we will + // zero-extend the root instead. + // + // FIXME: This is somewhat suboptimal, as there will be cases where adding + // one to the maximum bit width will yield a larger-than-necessary + // type. In general, we need to add an extra bit only if we can't + // prove that the upper bit of the original type is equal to the + // upper bit of the proposed smaller type. If these two bits are the + // same (either zero or one) we know that sign-extending from the + // smaller type will result in the same value. Here, since we can't + // yet prove this, we are just making the proposed smaller type + // larger to ensure correctness. + if (!IsKnownPositive) + ++MaxBitWidth; + } + + // Round MaxBitWidth up to the next power-of-two. + MaxBitWidth = llvm::bit_ceil(MaxBitWidth); + + // If the maximum bit width we compute is less than the with of the roots' + // type, we can proceed with the narrowing. Otherwise, do nothing. + if (MaxBitWidth >= TreeRootIT->getBitWidth()) + return; // If we can truncate the root, we must collect additional values that might // be demoted as a result. That is, those seeded by truncations we will // modify. - // Add reduction ops sizes, if any. - if (UserIgnoreList && - isa(VectorizableTree.front()->Scalars.front()->getType())) { - for (Value *V : *UserIgnoreList) { - auto NumSignBits = ComputeNumSignBits(V, *DL, 0, AC, nullptr, DT); - auto NumTypeBits = DL->getTypeSizeInBits(V->getType()); - unsigned BitWidth1 = NumTypeBits - NumSignBits; - if (!isKnownNonNegative(V, SimplifyQuery(*DL))) - ++BitWidth1; - auto Mask = DB->getDemandedBits(cast(V)); - unsigned BitWidth2 = Mask.getBitWidth() - Mask.countl_zero(); - ReductionBitWidth = - std::max(std::min(BitWidth1, BitWidth2), ReductionBitWidth); - } - if (ReductionBitWidth < 8 && ReductionBitWidth > 1) - ReductionBitWidth = 8; - - ReductionBitWidth = bit_ceil(ReductionBitWidth); - } - bool IsTopRoot = NodeIdx == 0; - while (NodeIdx < VectorizableTree.size()) { - ArrayRef TreeRoot = VectorizableTree[NodeIdx]->Scalars; - unsigned Limit = 2; - if (NodeIdx == 0 && - ReductionBitWidth == DL->getTypeSizeInBits(TreeRoot.front()->getType())) - Limit = 3; - unsigned MaxBitWidth = ComputeMaxBitWidth( - TreeRoot, VectorizableTree[NodeIdx]->getVectorFactor(), IsTopRoot, - IsProfitableToDemoteRoot, VectorizableTree[NodeIdx]->getOpcode(), - Limit); - IsTopRoot = false; - IsProfitableToDemoteRoot = true; - - ++NodeIdx; - for (unsigned E = VectorizableTree.size(); NodeIdx < E; ++NodeIdx) { - if (VectorizableTree[NodeIdx]->State != TreeEntry::NeedToGather && - !VectorizableTree[NodeIdx]->isAltShuffle() && - VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc) { - ++NodeIdx; - break; - } - } - - // If the maximum bit width we compute is less than the with of the roots' - // type, we can proceed with the narrowing. Otherwise, do nothing. - if (MaxBitWidth == 0 || - MaxBitWidth >= - cast(TreeRoot.front()->getType())->getBitWidth()) + while (!Roots.empty()) { + DenseSet Visited; + collectValuesToDemote(Roots.pop_back_val(), ToDemote, DemotedConsts, Roots, + Visited); + } + + // Check that all users are marked for demotion. + DenseSet Demoted(ToDemote.begin(), ToDemote.end()); + DenseSet Visited; + for (Value *V: ToDemote) { + const TreeEntry *TE = getTreeEntry(V); + assert(TE && "Expected vectorized scalar."); + if (!Visited.insert(TE).second) continue; - - // Finally, map the values we can demote to the maximum bit with we - // computed. - for (Value *Scalar : ToDemote) { - TreeEntry *TE = getTreeEntry(Scalar); - assert(TE && "Expected vectorized scalar."); - if (MinBWs.contains(TE)) - continue; - bool IsSigned = any_of(TE->Scalars, [&](Value *R) { - return !isKnownNonNegative(R, SimplifyQuery(*DL)); - }); - MinBWs.try_emplace(TE, MaxBitWidth, IsSigned); - const auto *I = cast(Scalar); - auto DCIt = DemotedConsts.find(I); - if (DCIt != DemotedConsts.end()) { - for (unsigned Idx : DCIt->getSecond()) { - // Check that all instructions operands are demoted. + if (!all_of(TE->UserTreeIndices, [&](const EdgeInfo &EI) { + return all_of(EI.UserTE->Scalars, + [&](Value *V) { return Demoted.contains(V); }); + })) + return; + } + // Finally, map the values we can demote to the maximum bit with we computed. + for (auto *Scalar : ToDemote) { + auto *TE = getTreeEntry(Scalar); + assert(TE && "Expected vectorized scalar."); + if (MinBWs.contains(TE)) + continue; + bool IsSigned = any_of(TE->Scalars, [&](Value *R) { + KnownBits Known = computeKnownBits(R, *DL); + return !Known.isNonNegative(); + }); + MinBWs.try_emplace(TE, MaxBitWidth, IsSigned); + const auto *I = cast(Scalar); + auto DCIt = DemotedConsts.find(I); + if (DCIt != DemotedConsts.end()) { + for (unsigned Idx : DCIt->getSecond()) { + // Check that all instructions operands are demoted. + if (all_of(TE->Scalars, [&](Value *V) { + auto SIt = DemotedConsts.find(cast(V)); + return SIt != DemotedConsts.end() && + is_contained(SIt->getSecond(), Idx); + })) { const TreeEntry *CTE = getOperandEntry(TE, Idx); - if (all_of(TE->Scalars, - [&](Value *V) { - auto SIt = DemotedConsts.find(cast(V)); - return SIt != DemotedConsts.end() && - is_contained(SIt->getSecond(), Idx); - }) || - all_of(CTE->Scalars, Constant::classof)) - MinBWs.try_emplace(CTE, MaxBitWidth, IsSigned); + MinBWs.try_emplace(CTE, MaxBitWidth, IsSigned); } } } diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll index 5e3fd156666f..cef791633655 100644 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll @@ -17,13 +17,12 @@ define void @test1(<4 x i16> %a, <4 x i16> %b, ptr %p) { ; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i64 [[S0]] ; CHECK-NEXT: [[LOAD0:%.*]] = load i64, ptr [[GEP0]], align 4 ; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <4 x i32> [[SUB0]], <4 x i32> poison, <2 x i32> -; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i32> [[TMP0]], i32 0 -; CHECK-NEXT: [[TMP2:%.*]] = sext i32 [[TMP1]] to i64 +; CHECK-NEXT: [[TMP1:%.*]] = sext <2 x i32> [[TMP0]] to <2 x i64> +; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0 ; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP2]] ; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[GEP1]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP0]], i32 1 -; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[TMP3]] to i64 -; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP4]] +; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1 +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP3]] ; CHECK-NEXT: [[LOAD2:%.*]] = load i64, ptr [[GEP2]], align 4 ; CHECK-NEXT: [[E3:%.*]] = extractelement <4 x i32> [[SUB0]], i32 3 ; CHECK-NEXT: [[S3:%.*]] = sext i32 [[E3]] to i64 diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll index 1cce52060c47..47485e514ec2 100644 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ;test_i16_extend NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer,dce,instcombine -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s +; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer,dce,instcombine -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s ; RUN: cat %t | FileCheck -check-prefix=YAML %s -; RUN: opt -S -mtriple=aarch64--linux-gnu -passes='slp-vectorizer,dce,instcombine' -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s +; RUN: opt -S -mtriple=aarch64--linux-gnu -passes='slp-vectorizer,dce,instcombine' -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s ; RUN: cat %t | FileCheck -check-prefix=YAML %s diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll index a7a7f642ced5..d67fdc1cd6aa 100644 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll @@ -28,11 +28,21 @@ entry: define i64 @red_zext_ld_4xi64(ptr %ptr) { ; CHECK-LABEL: @red_zext_ld_4xi64( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[PTR:%.*]], align 1 -; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i16> -; CHECK-NEXT: [[TMP2:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]]) -; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[TMP2]] to i64 -; CHECK-NEXT: ret i64 [[TMP3]] +; CHECK-NEXT: [[LD0:%.*]] = load i8, ptr [[PTR:%.*]], align 1 +; CHECK-NEXT: [[ZEXT:%.*]] = zext i8 [[LD0]] to i64 +; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 1 +; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[GEP]], align 1 +; CHECK-NEXT: [[ZEXT_1:%.*]] = zext i8 [[LD1]] to i64 +; CHECK-NEXT: [[ADD_1:%.*]] = add nuw nsw i64 [[ZEXT]], [[ZEXT_1]] +; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 2 +; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[GEP_1]], align 1 +; CHECK-NEXT: [[ZEXT_2:%.*]] = zext i8 [[LD2]] to i64 +; CHECK-NEXT: [[ADD_2:%.*]] = add nuw nsw i64 [[ADD_1]], [[ZEXT_2]] +; CHECK-NEXT: [[GEP_2:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 3 +; CHECK-NEXT: [[LD3:%.*]] = load i8, ptr [[GEP_2]], align 1 +; CHECK-NEXT: [[ZEXT_3:%.*]] = zext i8 [[LD3]] to i64 +; CHECK-NEXT: [[ADD_3:%.*]] = add nuw nsw i64 [[ADD_2]], [[ZEXT_3]] +; CHECK-NEXT: ret i64 [[ADD_3]] ; entry: %ld0 = load i8, ptr %ptr diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll index 500f10659f04..000e7a56df37 100644 --- a/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll @@ -802,10 +802,9 @@ define i64 @red_zext_ld_4xi64(ptr %ptr) { ; CHECK-LABEL: @red_zext_ld_4xi64( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[PTR:%.*]], align 1 -; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i16> -; CHECK-NEXT: [[TMP2:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]]) -; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[TMP2]] to i64 -; CHECK-NEXT: ret i64 [[TMP3]] +; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i64> +; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP1]]) +; CHECK-NEXT: ret i64 [[TMP2]] ; entry: %ld0 = load i8, ptr %ptr diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll index 05511f843a68..4565d4928ba4 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll @@ -15,12 +15,11 @@ define { i64, i64 } @patatino(double %arg) { ; CHECK-NEXT: [[TMP6:%.*]] = load <2 x double>, ptr getelementptr inbounds ([6 x double], ptr @global, i64 0, i64 4), align 16 ; CHECK-NEXT: [[TMP7:%.*]] = fadd <2 x double> [[TMP6]], [[TMP5]] ; CHECK-NEXT: [[TMP8:%.*]] = fptosi <2 x double> [[TMP7]] to <2 x i32> -; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[TMP8]], i32 0 -; CHECK-NEXT: [[TMP10:%.*]] = sext i32 [[TMP9]] to i64 +; CHECK-NEXT: [[TMP9:%.*]] = sext <2 x i32> [[TMP8]] to <2 x i64> +; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i64> [[TMP9]], i32 0 ; CHECK-NEXT: [[T16:%.*]] = insertvalue { i64, i64 } undef, i64 [[TMP10]], 0 -; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[TMP8]], i32 1 -; CHECK-NEXT: [[TMP12:%.*]] = sext i32 [[TMP11]] to i64 -; CHECK-NEXT: [[T17:%.*]] = insertvalue { i64, i64 } [[T16]], i64 [[TMP12]], 1 +; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i64> [[TMP9]], i32 1 +; CHECK-NEXT: [[T17:%.*]] = insertvalue { i64, i64 } [[T16]], i64 [[TMP11]], 1 ; CHECK-NEXT: ret { i64, i64 } [[T17]] ; bb: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll index 5ee801607653..a0af8e36b36c 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 -; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-6 < %s | FileCheck %s +; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-3 < %s | FileCheck %s define void @t(i64 %v) { ; CHECK-LABEL: define void @t( diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll index 6051638562b5..6e512fcbb739 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll @@ -6,17 +6,18 @@ define void @test(i8 %0) { ; CHECK-SAME: i8 [[TMP0:%.*]]) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> , i8 [[TMP0]], i32 1 -; CHECK-NEXT: [[TMP2:%.*]] = sext <2 x i8> [[TMP1]] to <2 x i32> -; CHECK-NEXT: [[TMP3:%.*]] = mul <2 x i8> [[TMP1]], zeroinitializer -; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[TMP3]], i32 0 -; CHECK-NEXT: [[TMP5:%.*]] = zext i8 [[TMP4]] to i32 -; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i8> [[TMP3]], i32 1 -; CHECK-NEXT: [[TMP7:%.*]] = zext i8 [[TMP6]] to i32 -; CHECK-NEXT: [[ADD:%.*]] = or i32 [[TMP5]], [[TMP7]] +; CHECK-NEXT: [[TMP2:%.*]] = sext <2 x i8> [[TMP1]] to <2 x i16> +; CHECK-NEXT: [[TMP3:%.*]] = sext <2 x i16> [[TMP2]] to <2 x i32> +; CHECK-NEXT: [[TMP4:%.*]] = mul <2 x i16> [[TMP2]], zeroinitializer +; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i16> [[TMP4]], i32 0 +; CHECK-NEXT: [[TMP6:%.*]] = zext i16 [[TMP5]] to i32 +; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i16> [[TMP4]], i32 1 +; CHECK-NEXT: [[TMP8:%.*]] = zext i16 [[TMP7]] to i32 +; CHECK-NEXT: [[ADD:%.*]] = or i32 [[TMP6]], [[TMP8]] ; CHECK-NEXT: [[SHR:%.*]] = lshr i32 [[ADD]], 1 ; CHECK-NEXT: [[CONV9:%.*]] = trunc i32 [[SHR]] to i8 ; CHECK-NEXT: store i8 [[CONV9]], ptr null, align 1 -; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> +; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <8 x i32> ; CHECK-NEXT: ret void ; entry: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll index 4acd63078b82..2c834616becc 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll @@ -6,20 +6,15 @@ define void @test(i64 %d.promoted.i) { ; CHECK-SAME: i64 [[D_PROMOTED_I:%.*]]) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[AND_1_I:%.*]] = and i64 0, [[D_PROMOTED_I]] -; CHECK-NEXT: [[TMP0:%.*]] = insertelement <8 x i64> , i64 [[AND_1_I]], i32 1 -; CHECK-NEXT: [[TMP1:%.*]] = trunc <8 x i64> [[TMP0]] to <8 x i1> -; CHECK-NEXT: [[TMP2:%.*]] = mul <8 x i1> [[TMP1]], zeroinitializer ; CHECK-NEXT: [[AND_1_I_1:%.*]] = and i64 0, 0 -; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i64> , i64 [[AND_1_I_1]], i32 1 -; CHECK-NEXT: [[TMP4:%.*]] = trunc <8 x i64> [[TMP3]] to <8 x i1> -; CHECK-NEXT: [[TMP5:%.*]] = mul <8 x i1> [[TMP4]], zeroinitializer -; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP5]]) -; CHECK-NEXT: [[TMP7:%.*]] = zext i1 [[TMP6]] to i32 -; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP2]]) -; CHECK-NEXT: [[TMP9:%.*]] = zext i1 [[TMP8]] to i32 -; CHECK-NEXT: [[OP_RDX:%.*]] = or i32 [[TMP7]], [[TMP9]] -; CHECK-NEXT: [[TMP10:%.*]] = and i32 [[OP_RDX]], 0 -; CHECK-NEXT: store i32 [[TMP10]], ptr null, align 4 +; CHECK-NEXT: [[TMP0:%.*]] = insertelement <16 x i64> , i64 [[AND_1_I_1]], i32 1 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <16 x i64> [[TMP0]], i64 [[AND_1_I]], i32 9 +; CHECK-NEXT: [[TMP2:%.*]] = trunc <16 x i64> [[TMP1]] to <16 x i1> +; CHECK-NEXT: [[TMP3:%.*]] = mul <16 x i1> [[TMP2]], zeroinitializer +; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP3]]) +; CHECK-NEXT: [[TMP5:%.*]] = zext i1 [[TMP4]] to i32 +; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[TMP5]], 0 +; CHECK-NEXT: store i32 [[TMP6]], ptr null, align 4 ; CHECK-NEXT: ret void ; entry: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll index a316415dcc6b..651631de2c35 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll @@ -17,15 +17,12 @@ target triple = "x86_64-unknown-linux-gnu" define i8 @PR31243_zext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) { ; SSE-LABEL: @PR31243_zext( ; SSE-NEXT: entry: -; SSE-NEXT: [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0 -; SSE-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1 -; SSE-NEXT: [[TMP2:%.*]] = or <2 x i8> [[TMP1]], -; SSE-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0 -; SSE-NEXT: [[TMP4:%.*]] = zext i8 [[TMP3]] to i64 -; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]] -; SSE-NEXT: [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1 -; SSE-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i64 -; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]] +; SSE-NEXT: [[TMP0:%.*]] = or i8 [[V0:%.*]], 1 +; SSE-NEXT: [[TMP1:%.*]] = or i8 [[V1:%.*]], 1 +; SSE-NEXT: [[TMP2:%.*]] = zext i8 [[TMP0]] to i64 +; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP2]] +; SSE-NEXT: [[TMP3:%.*]] = zext i8 [[TMP1]] to i64 +; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP3]] ; SSE-NEXT: [[T6:%.*]] = load i8, ptr [[T4]], align 1 ; SSE-NEXT: [[T7:%.*]] = load i8, ptr [[T5]], align 1 ; SSE-NEXT: [[T8:%.*]] = add i8 [[T6]], [[T7]] @@ -76,15 +73,12 @@ entry: define i8 @PR31243_sext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) { ; SSE-LABEL: @PR31243_sext( ; SSE-NEXT: entry: -; SSE-NEXT: [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0 -; SSE-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1 -; SSE-NEXT: [[TMP2:%.*]] = or <2 x i8> [[TMP1]], -; SSE-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0 -; SSE-NEXT: [[TMP4:%.*]] = sext i8 [[TMP3]] to i64 -; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]] -; SSE-NEXT: [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1 -; SSE-NEXT: [[TMP6:%.*]] = sext i8 [[TMP5]] to i64 -; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]] +; SSE-NEXT: [[TMP0:%.*]] = or i8 [[V0:%.*]], 1 +; SSE-NEXT: [[TMP1:%.*]] = or i8 [[V1:%.*]], 1 +; SSE-NEXT: [[TMP2:%.*]] = sext i8 [[TMP0]] to i64 +; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP2]] +; SSE-NEXT: [[TMP3:%.*]] = sext i8 [[TMP1]] to i64 +; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP3]] ; SSE-NEXT: [[T6:%.*]] = load i8, ptr [[T4]], align 1 ; SSE-NEXT: [[T7:%.*]] = load i8, ptr [[T5]], align 1 ; SSE-NEXT: [[T8:%.*]] = add i8 [[T6]], [[T7]] @@ -95,12 +89,13 @@ define i8 @PR31243_sext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) { ; AVX-NEXT: [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0 ; AVX-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1 ; AVX-NEXT: [[TMP2:%.*]] = or <2 x i8> [[TMP1]], -; AVX-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0 -; AVX-NEXT: [[TMP4:%.*]] = sext i8 [[TMP3]] to i64 -; AVX-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]] -; AVX-NEXT: [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1 -; AVX-NEXT: [[TMP6:%.*]] = sext i8 [[TMP5]] to i64 -; AVX-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]] +; AVX-NEXT: [[TMP3:%.*]] = sext <2 x i8> [[TMP2]] to <2 x i16> +; AVX-NEXT: [[TMP4:%.*]] = extractelement <2 x i16> [[TMP3]], i64 0 +; AVX-NEXT: [[TMP5:%.*]] = sext i16 [[TMP4]] to i64 +; AVX-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP5]] +; AVX-NEXT: [[TMP6:%.*]] = extractelement <2 x i16> [[TMP3]], i64 1 +; AVX-NEXT: [[TMP7:%.*]] = sext i16 [[TMP6]] to i64 +; AVX-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP7]] ; AVX-NEXT: [[T6:%.*]] = load i8, ptr [[T4]], align 1 ; AVX-NEXT: [[T7:%.*]] = load i8, ptr [[T5]], align 1 ; AVX-NEXT: [[T8:%.*]] = add i8 [[T6]], [[T7]] diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll index 3cc32c1fc7b2..88f75c37846e 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll @@ -15,8 +15,8 @@ define i32 @phi3UndefInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %arg3) { ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) -; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 +; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> +; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: @@ -52,8 +52,8 @@ define i32 @phi2UndefInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %arg3) { ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) -; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 +; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> +; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: @@ -89,8 +89,8 @@ define i32 @phi1UndefInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %arg3) { ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) -; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 +; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> +; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: @@ -127,8 +127,8 @@ define i32 @phi1Undef1PoisonInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %ar ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) -; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 +; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> +; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: @@ -165,8 +165,8 @@ define i32 @phi1Undef2PoisonInputs(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %a ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) -; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 +; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> +; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: @@ -202,8 +202,8 @@ define i32 @phi1Undef1PoisonGapInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) -; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 +; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> +; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/resched.ll b/llvm/test/Transforms/SLPVectorizer/X86/resched.ll index b7237cbb02bb..78c6d9516a3d 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/resched.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/resched.ll @@ -11,26 +11,26 @@ define fastcc void @_ZN12_GLOBAL__N_127PolynomialMultiplyRecognize9recognizeEv() ; CHECK: if.then22.i: ; CHECK-NEXT: [[SUB_I:%.*]] = add nsw i32 undef, -1 ; CHECK-NEXT: [[CONV31_I:%.*]] = and i32 undef, [[SUB_I]] -; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[CONV31_I]], i32 0 -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP2:%.*]] = lshr <4 x i32> [[TMP1]], +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[CONV31_I]], i32 0 +; CHECK-NEXT: [[SHUFFLE1:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = lshr <4 x i32> [[SHUFFLE1]], ; CHECK-NEXT: [[SHR_4_I_I:%.*]] = lshr i32 [[CONV31_I]], 5 ; CHECK-NEXT: [[SHR_5_I_I:%.*]] = lshr i32 [[CONV31_I]], 6 ; CHECK-NEXT: [[SHR_6_I_I:%.*]] = lshr i32 [[CONV31_I]], 7 ; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i32> poison, i32 [[CONV31_I]], i32 0 -; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> poison, <8 x i32> zeroinitializer -; CHECK-NEXT: [[TMP5:%.*]] = lshr <8 x i32> [[TMP4]], -; CHECK-NEXT: [[TMP6:%.*]] = insertelement <16 x i32> poison, i32 [[SUB_I]], i32 0 -; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <16 x i32> -; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <16 x i32> [[TMP6]], <16 x i32> [[TMP7]], <16 x i32> -; CHECK-NEXT: [[TMP9:%.*]] = insertelement <16 x i32> [[TMP8]], i32 [[SHR_4_I_I]], i32 5 -; CHECK-NEXT: [[TMP10:%.*]] = insertelement <16 x i32> [[TMP9]], i32 [[SHR_5_I_I]], i32 6 -; CHECK-NEXT: [[TMP11:%.*]] = insertelement <16 x i32> [[TMP10]], i32 [[SHR_6_I_I]], i32 7 -; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> poison, <16 x i32> -; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <16 x i32> [[TMP11]], <16 x i32> [[TMP12]], <16 x i32> -; CHECK-NEXT: [[TMP14:%.*]] = trunc <16 x i32> [[TMP13]] to <16 x i8> -; CHECK-NEXT: [[TMP15:%.*]] = and <16 x i8> [[TMP14]], -; CHECK-NEXT: store <16 x i8> [[TMP15]], ptr undef, align 1 +; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> poison, <8 x i32> zeroinitializer +; CHECK-NEXT: [[TMP4:%.*]] = lshr <8 x i32> [[SHUFFLE]], +; CHECK-NEXT: [[TMP5:%.*]] = insertelement <16 x i32> poison, i32 [[SUB_I]], i32 0 +; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <16 x i32> +; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> [[TMP6]], <16 x i32> +; CHECK-NEXT: [[TMP8:%.*]] = insertelement <16 x i32> [[TMP7]], i32 [[SHR_4_I_I]], i32 5 +; CHECK-NEXT: [[TMP9:%.*]] = insertelement <16 x i32> [[TMP8]], i32 [[SHR_5_I_I]], i32 6 +; CHECK-NEXT: [[TMP10:%.*]] = insertelement <16 x i32> [[TMP9]], i32 [[SHR_6_I_I]], i32 7 +; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <16 x i32> +; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <16 x i32> [[TMP10]], <16 x i32> [[TMP11]], <16 x i32> +; CHECK-NEXT: [[TMP13:%.*]] = trunc <16 x i32> [[TMP12]] to <16 x i8> +; CHECK-NEXT: [[TMP14:%.*]] = and <16 x i8> [[TMP13]], +; CHECK-NEXT: store <16 x i8> [[TMP14]], ptr undef, align 1 ; CHECK-NEXT: unreachable ; CHECK: if.end50.i: ; CHECK-NEXT: ret void diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll index 1d1fcec2a7ae..5d22b5a4873b 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll @@ -7,10 +7,12 @@ define i1 @test(i1 %cmp5.not.31) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i1> , i1 [[CMP5_NOT_31]], i32 0 ; CHECK-NEXT: [[TMP1:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> zeroinitializer, <4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP2:%.*]] = mul <4 x i32> [[TMP1]], -; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]]) -; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[TMP3]], 0 -; CHECK-NEXT: [[CMP_NOT_I_I:%.*]] = icmp eq i32 [[TMP4]], 0 +; CHECK-NEXT: [[TMP2:%.*]] = trunc <4 x i32> [[TMP1]] to <4 x i1> +; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32> +; CHECK-NEXT: [[TMP4:%.*]] = mul <4 x i32> [[TMP3]], +; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]]) +; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[TMP5]], 0 +; CHECK-NEXT: [[CMP_NOT_I_I:%.*]] = icmp eq i32 [[TMP6]], 0 ; CHECK-NEXT: ret i1 [[CMP_NOT_I_I]] ; entry: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll index 2f6868d8dfd6..c1dd90d0e9a7 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll @@ -8,18 +8,17 @@ ; YAML-NEXT: Function: stores ; YAML-NEXT: Args: ; YAML-NEXT: - String: 'Stores SLP vectorized with cost ' -; YAML-NEXT: - Cost: '-7' +; YAML-NEXT: - Cost: '-3' ; YAML-NEXT: - String: ' and with tree size ' ; YAML-NEXT: - TreeSize: '6' define void @stores(ptr noalias %in, ptr noalias %inn, ptr noalias %out) { ; CHECK-LABEL: @stores( ; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[IN:%.*]], align 1 ; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[INN:%.*]], align 1 -; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i16> -; CHECK-NEXT: [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i16> -; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i16> [[TMP3]], [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = zext <4 x i16> [[TMP5]] to <4 x i64> -; CHECK-NEXT: store <4 x i64> [[TMP6]], ptr [[OUT:%.*]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i64> +; CHECK-NEXT: [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i64> +; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[TMP3]], [[TMP4]] +; CHECK-NEXT: store <4 x i64> [[TMP5]], ptr [[OUT:%.*]], align 4 ; CHECK-NEXT: ret void ; %load.1 = load i8, ptr %in, align 1 @@ -64,18 +63,17 @@ define void @stores(ptr noalias %in, ptr noalias %inn, ptr noalias %out) { ; YAML-NEXT: Function: insertelems ; YAML-NEXT: Args: ; YAML-NEXT: - String: 'SLP vectorized with cost ' -; YAML-NEXT: - Cost: '-9' +; YAML-NEXT: - Cost: '-5' ; YAML-NEXT: - String: ' and with tree size ' ; YAML-NEXT: - TreeSize: '6' define <4 x i64> @insertelems(ptr noalias %in, ptr noalias %inn) { ; CHECK-LABEL: @insertelems( ; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[IN:%.*]], align 1 ; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[INN:%.*]], align 1 -; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i16> -; CHECK-NEXT: [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i16> -; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i16> [[TMP3]], [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = zext <4 x i16> [[TMP5]] to <4 x i64> -; CHECK-NEXT: ret <4 x i64> [[TMP6]] +; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i64> +; CHECK-NEXT: [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i64> +; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[TMP3]], [[TMP4]] +; CHECK-NEXT: ret <4 x i64> [[TMP5]] ; %load.1 = load i8, ptr %in, align 1 %gep.1 = getelementptr inbounds i8, ptr %in, i64 1 diff --git a/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll b/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll index ff6f0bdd3db8..061fbdb45a13 100644 --- a/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll +++ b/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll @@ -10,8 +10,8 @@ define i32 @alt_cmp(i16 %call46) { ; CHECK-NEXT: [[TMP2:%.*]] = icmp ult <4 x i16> [[TMP0]], [[TMP1]] ; CHECK-NEXT: [[TMP3:%.*]] = icmp ugt <4 x i16> [[TMP0]], [[TMP1]] ; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i1> [[TMP2]], <4 x i1> [[TMP3]], <4 x i32> -; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) -; CHECK-NEXT: [[TMP6:%.*]] = zext i1 [[TMP5]] to i16 +; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i1> [[TMP4]] to <4 x i16> +; CHECK-NEXT: [[TMP6:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP5]]) ; CHECK-NEXT: [[OP_RDX:%.*]] = or i16 [[TMP6]], 0 ; CHECK-NEXT: [[EXT:%.*]] = zext i16 [[OP_RDX]] to i32 ; CHECK-NEXT: ret i32 [[EXT]] -- GitLab From 64faa52b1ef7778d1cde9a7191fbd0a167e9dfd8 Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Tue, 5 Mar 2024 12:14:09 -0800 Subject: [PATCH 200/929] Disable clang-tidy misc-include-cleaner (#83945) This does not apply well to LLVM which intentionally rely on forward declarations. Also depending on the config flags passed to CMake the result can be different. --- .clang-tidy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.clang-tidy b/.clang-tidy index 4e1cb114f43b..9cece0de812b 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,4 +1,4 @@ -Checks: '-*,clang-diagnostic-*,llvm-*,misc-*,-misc-const-correctness,-misc-unused-parameters,-misc-non-private-member-variables-in-classes,-misc-no-recursion,-misc-use-anonymous-namespace,readability-identifier-naming' +Checks: '-*,clang-diagnostic-*,llvm-*,misc-*,-misc-const-correctness,-misc-unused-parameters,-misc-non-private-member-variables-in-classes,-misc-no-recursion,-misc-use-anonymous-namespace,readability-identifier-naming,-misc-include-cleaner' CheckOptions: - key: readability-identifier-naming.ClassCase value: CamelCase -- GitLab From b6ca602658c101b783540418130ac5fadc0b2360 Mon Sep 17 00:00:00 2001 From: Aart Bik Date: Tue, 5 Mar 2024 12:17:45 -0800 Subject: [PATCH 201/929] [mlir][sparse] migrate tests to sparse_tensor.print (#84055) Continuing the efforts started in #83357 --- .../SparseTensor/CPU/concatenate_dim_0.mlir | 85 +++++------ .../SparseTensor/CPU/sparse_block_matmul.mlir | 138 +++++++++--------- .../SparseTensor/CPU/sparse_complex32.mlir | 53 +++---- .../SparseTensor/CPU/sparse_complex64.mlir | 53 +++---- .../Dialect/SparseTensor/CPU/sparse_ds.mlir | 88 +++++------ 5 files changed, 183 insertions(+), 234 deletions(-) diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0.mlir index 515f5b88b480..6a4902057362 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -51,11 +51,6 @@ module { func.func private @printMemrefF64(%ptr : tensor<*xf64>) - func.func private @printMemref1dF64(%ptr : memref) attributes { llvm.emit_c_interface } - - // - // Tests without permutation. - // // Concats all sparse matrices (with different encodings) to a sparse matrix. func.func @concat_sparse_sparse(%arg0: tensor<2x4xf64, #MAT_C_C>, %arg1: tensor<3x4xf64, #MAT_C_D>, %arg2: tensor<4x4xf64, #MAT_D_C>) -> tensor<9x4xf64, #MAT_C_C> { @@ -85,30 +80,15 @@ module { return %0 : tensor<9x4xf64> } - func.func @dump_mat_9x4(%A: tensor<9x4xf64, #MAT_C_C>) { - %c = sparse_tensor.convert %A : tensor<9x4xf64, #MAT_C_C> to tensor<9x4xf64> - %cu = tensor.cast %c : tensor<9x4xf64> to tensor<*xf64> - call @printMemrefF64(%cu) : (tensor<*xf64>) -> () - - %n = sparse_tensor.number_of_entries %A : tensor<9x4xf64, #MAT_C_C> - vector.print %n : index - - %1 = sparse_tensor.values %A : tensor<9x4xf64, #MAT_C_C> to memref - call @printMemref1dF64(%1) : (memref) -> () - - bufferization.dealloc_tensor %c : tensor<9x4xf64> - return - } - + // Outputs dense matrix. func.func @dump_mat_dense_9x4(%A: tensor<9x4xf64>) { %u = tensor.cast %A : tensor<9x4xf64> to tensor<*xf64> call @printMemrefF64(%u) : (tensor<*xf64>) -> () - return } // Driver method to call and verify kernels. - func.func @entry() { + func.func @main() { %m24 = arith.constant dense< [ [ 1.0, 0.0, 3.0, 0.0], [ 0.0, 2.0, 0.0, 0.0] ]> : tensor<2x4xf64> @@ -126,22 +106,24 @@ module { %sm34cd = sparse_tensor.convert %m34 : tensor<3x4xf64> to tensor<3x4xf64, #MAT_C_D> %sm44dc = sparse_tensor.convert %m44 : tensor<4x4xf64> to tensor<4x4xf64, #MAT_D_C> - // CHECK: {{\[}}[1, 0, 3, 0], - // CHECK-NEXT: [0, 2, 0, 0], - // CHECK-NEXT: [1, 0, 1, 1], - // CHECK-NEXT: [0, 0.5, 0, 0], - // CHECK-NEXT: [1, 5, 2, 0], - // CHECK-NEXT: [0, 0, 1.5, 1], - // CHECK-NEXT: [0, 3.5, 0, 0], - // CHECK-NEXT: [1, 5, 2, 0], - // CHECK-NEXT: [1, 0.5, 0, 0]] - // CHECK-NEXT: 18 - // CHECK: [1, 3, 2, 1, 1, 1, 0.5, 1, 5, 2, 1.5, 1, 3.5, 1, 5, 2, 1, 0.5 + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 18 + // CHECK-NEXT: dim = ( 9, 4 ) + // CHECK-NEXT: lvl = ( 9, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 9, + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, + // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 7, 10, 12, 13, 16, 18, + // CHECK-NEXT: crd[1] : ( 0, 2, 1, 0, 2, 3, 1, 0, 1, 2, 2, 3, 1, 0, 1, 2, 0, 1, + // CHECK-NEXT: values : ( 1, 3, 2, 1, 1, 1, 0.5, 1, 5, 2, 1.5, 1, 3.5, 1, 5, 2, 1, 0.5, + // CHECK-NEXT: ---- + // %0 = call @concat_sparse_sparse(%sm24cc, %sm34cd, %sm44dc) : (tensor<2x4xf64, #MAT_C_C>, tensor<3x4xf64, #MAT_C_D>, tensor<4x4xf64, #MAT_D_C>) -> tensor<9x4xf64, #MAT_C_C> - call @dump_mat_9x4(%0) : (tensor<9x4xf64, #MAT_C_C>) -> () + sparse_tensor.print %0 : tensor<9x4xf64, #MAT_C_C> - // CHECK: {{\[}}[1, 0, 3, 0], + // + // CHECK: {{\[}}[1, 0, 3, 0], // CHECK-NEXT: [0, 2, 0, 0], // CHECK-NEXT: [1, 0, 1, 1], // CHECK-NEXT: [0, 0.5, 0, 0], @@ -150,26 +132,29 @@ module { // CHECK-NEXT: [0, 3.5, 0, 0], // CHECK-NEXT: [1, 5, 2, 0], // CHECK-NEXT: [1, 0.5, 0, 0]] + // %1 = call @concat_sparse_dense(%sm24cc, %sm34cd, %sm44dc) : (tensor<2x4xf64, #MAT_C_C>, tensor<3x4xf64, #MAT_C_D>, tensor<4x4xf64, #MAT_D_C>) -> tensor<9x4xf64> call @dump_mat_dense_9x4(%1) : (tensor<9x4xf64>) -> () - // CHECK: {{\[}}[1, 0, 3, 0], - // CHECK-NEXT: [0, 2, 0, 0], - // CHECK-NEXT: [1, 0, 1, 1], - // CHECK-NEXT: [0, 0.5, 0, 0], - // CHECK-NEXT: [1, 5, 2, 0], - // CHECK-NEXT: [0, 0, 1.5, 1], - // CHECK-NEXT: [0, 3.5, 0, 0], - // CHECK-NEXT: [1, 5, 2, 0], - // CHECK-NEXT: [1, 0.5, 0, 0]] - // CHECK-NEXT: 18 - // CHECK: [1, 3, 2, 1, 1, 1, 0.5, 1, 5, 2, 1.5, 1, 3.5, 1, 5, 2, 1, 0.5 + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 18 + // CHECK-NEXT: dim = ( 9, 4 ) + // CHECK-NEXT: lvl = ( 9, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 9, + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, + // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 7, 10, 12, 13, 16, 18, + // CHECK-NEXT: crd[1] : ( 0, 2, 1, 0, 2, 3, 1, 0, 1, 2, 2, 3, 1, 0, 1, 2, 0, 1, + // CHECK-NEXT: values : ( 1, 3, 2, 1, 1, 1, 0.5, 1, 5, 2, 1.5, 1, 3.5, 1, 5, 2, 1, 0.5, + // CHECK-NEXT: ---- + // %2 = call @concat_mix_sparse(%m24, %sm34cd, %sm44dc) : (tensor<2x4xf64>, tensor<3x4xf64, #MAT_C_D>, tensor<4x4xf64, #MAT_D_C>) -> tensor<9x4xf64, #MAT_C_C> - call @dump_mat_9x4(%2) : (tensor<9x4xf64, #MAT_C_C>) -> () + sparse_tensor.print %2 : tensor<9x4xf64, #MAT_C_C> - // CHECK: {{\[}}[1, 0, 3, 0], + // + // CHECK: {{\[}}[1, 0, 3, 0], // CHECK-NEXT: [0, 2, 0, 0], // CHECK-NEXT: [1, 0, 1, 1], // CHECK-NEXT: [0, 0.5, 0, 0], @@ -178,11 +163,11 @@ module { // CHECK-NEXT: [0, 3.5, 0, 0], // CHECK-NEXT: [1, 5, 2, 0], // CHECK-NEXT: [1, 0.5, 0, 0]] + // %3 = call @concat_mix_dense(%m24, %sm34cd, %sm44dc) : (tensor<2x4xf64>, tensor<3x4xf64, #MAT_C_D>, tensor<4x4xf64, #MAT_D_C>) -> tensor<9x4xf64> call @dump_mat_dense_9x4(%3) : (tensor<9x4xf64>) -> () - // Release resources. bufferization.dealloc_tensor %sm24cc : tensor<2x4xf64, #MAT_C_C> bufferization.dealloc_tensor %sm34cd : tensor<3x4xf64, #MAT_C_D> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block_matmul.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block_matmul.mlir index e47ac46597b7..464de9c8a2c3 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block_matmul.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block_matmul.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -45,7 +45,6 @@ map = ( i, j ) -> (i : dense, j : compressed) }> - #BSR = #sparse_tensor.encoding<{ map = ( i, j ) -> ( i floordiv 2 : dense, @@ -65,67 +64,66 @@ module { -func.func @mul(%arg0: tensor<4x8xf64>, - %arg1: tensor<4x8xf64, #BSR>) -> tensor<4x4xf64> { - %out = arith.constant dense<0.0> : tensor<4x4xf64> - %0 = linalg.generic #trait_mul - ins(%arg0, %arg1: tensor<4x8xf64>, tensor<4x8xf64, #BSR>) - outs(%out: tensor<4x4xf64>) { - ^bb(%x: f64, %y : f64, %z : f64): - %1 = arith.mulf %x, %y : f64 - %2 = arith.addf %1, %z : f64 - linalg.yield %2 : f64 - } -> tensor<4x4xf64> - return %0 : tensor<4x4xf64> -} - -func.func @mul_24(%arg0: tensor<4x8xf64>, - %arg1: tensor<4x8xf64, #NV_24>) -> tensor<4x4xf64> { - %out = arith.constant dense<0.0> : tensor<4x4xf64> - %0 = linalg.generic #trait_mul - ins(%arg0, %arg1: tensor<4x8xf64>, tensor<4x8xf64, #NV_24>) - outs(%out: tensor<4x4xf64>) { - ^bb(%x: f64, %y : f64, %z : f64): - %1 = arith.mulf %x, %y : f64 - %2 = arith.addf %1, %z : f64 - linalg.yield %2 : f64 - } -> tensor<4x4xf64> - return %0 : tensor<4x4xf64> -} + func.func @mul(%arg0: tensor<4x8xf64>, + %arg1: tensor<4x8xf64, #BSR>) -> tensor<4x4xf64> { + %out = arith.constant dense<0.0> : tensor<4x4xf64> + %0 = linalg.generic #trait_mul + ins(%arg0, %arg1: tensor<4x8xf64>, tensor<4x8xf64, #BSR>) + outs(%out: tensor<4x4xf64>) { + ^bb(%x: f64, %y : f64, %z : f64): + %1 = arith.mulf %x, %y : f64 + %2 = arith.addf %1, %z : f64 + linalg.yield %2 : f64 + } -> tensor<4x4xf64> + return %0 : tensor<4x4xf64> + } -func.func @mul_csr_bsr(%arg0: tensor<4x8xf64, #CSR>, - %arg1: tensor<4x8xf64, #BSR>) -> tensor<4x4xf64> { - %out = arith.constant dense<0.0> : tensor<4x4xf64> - %0 = linalg.generic #trait_mul - ins(%arg0, %arg1: tensor<4x8xf64, #CSR>, tensor<4x8xf64, #BSR>) - outs(%out: tensor<4x4xf64>) { - ^bb(%x: f64, %y : f64, %z : f64): - %1 = arith.mulf %x, %y : f64 - %2 = arith.addf %1, %z : f64 - linalg.yield %2 : f64 - } -> tensor<4x4xf64> - return %0 : tensor<4x4xf64> -} + func.func @mul_24(%arg0: tensor<4x8xf64>, + %arg1: tensor<4x8xf64, #NV_24>) -> tensor<4x4xf64> { + %out = arith.constant dense<0.0> : tensor<4x4xf64> + %0 = linalg.generic #trait_mul + ins(%arg0, %arg1: tensor<4x8xf64>, tensor<4x8xf64, #NV_24>) + outs(%out: tensor<4x4xf64>) { + ^bb(%x: f64, %y : f64, %z : f64): + %1 = arith.mulf %x, %y : f64 + %2 = arith.addf %1, %z : f64 + linalg.yield %2 : f64 + } -> tensor<4x4xf64> + return %0 : tensor<4x4xf64> + } -func.func @mul_dense(%arg0: tensor<4x8xf64>, - %arg1: tensor<4x8xf64>) -> tensor<4x4xf64> { - %out = arith.constant dense<0.0> : tensor<4x4xf64> - %0 = linalg.generic #trait_mul - ins(%arg0, %arg1: tensor<4x8xf64>, tensor<4x8xf64>) - outs(%out: tensor<4x4xf64>) { - ^bb(%x: f64, %y : f64, %z : f64): - %1 = arith.mulf %x, %y : f64 - %2 = arith.addf %1, %z : f64 - linalg.yield %2 : f64 - } -> tensor<4x4xf64> - return %0 : tensor<4x4xf64> -} + func.func @mul_csr_bsr(%arg0: tensor<4x8xf64, #CSR>, + %arg1: tensor<4x8xf64, #BSR>) -> tensor<4x4xf64> { + %out = arith.constant dense<0.0> : tensor<4x4xf64> + %0 = linalg.generic #trait_mul + ins(%arg0, %arg1: tensor<4x8xf64, #CSR>, tensor<4x8xf64, #BSR>) + outs(%out: tensor<4x4xf64>) { + ^bb(%x: f64, %y : f64, %z : f64): + %1 = arith.mulf %x, %y : f64 + %2 = arith.addf %1, %z : f64 + linalg.yield %2 : f64 + } -> tensor<4x4xf64> + return %0 : tensor<4x4xf64> + } + func.func @mul_dense(%arg0: tensor<4x8xf64>, + %arg1: tensor<4x8xf64>) -> tensor<4x4xf64> { + %out = arith.constant dense<0.0> : tensor<4x4xf64> + %0 = linalg.generic #trait_mul + ins(%arg0, %arg1: tensor<4x8xf64>, tensor<4x8xf64>) + outs(%out: tensor<4x4xf64>) { + ^bb(%x: f64, %y : f64, %z : f64): + %1 = arith.mulf %x, %y : f64 + %2 = arith.addf %1, %z : f64 + linalg.yield %2 : f64 + } -> tensor<4x4xf64> + return %0 : tensor<4x4xf64> + } // - // Output utilities. + // Output utility. // - func.func @dumpf64(%arg0: tensor<4x4xf64>) { + func.func @dump_dense_f64(%arg0: tensor<4x4xf64>) { %c0 = arith.constant 0 : index %d0 = arith.constant -1.0 : f64 %0 = vector.transfer_read %arg0[%c0, %c0], %d0: tensor<4x4xf64>, vector<4x4xf64> @@ -136,36 +134,32 @@ func.func @mul_dense(%arg0: tensor<4x8xf64>, // // Main driver. // - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %td = arith.constant dense<[[ 1.0, 2.0, 0.0, 0.0, 0.0, 0.0, 4.0, 5.0], [ 6.0, 7.0, 0.0, 0.0, 0.0, 0.0, 10.0, 11.0], [ 0.0, 0.0, 12.0, 13.0, 16.0, 17.0, 0.0, 0.0], [ 0.0, 0.0, 18.0, 19.0, 22.0, 23.0, 0.0, 0.0]]> : tensor<4x8xf64> - - %2 = sparse_tensor.convert %td : tensor<4x8xf64> to tensor<4x8xf64, #BSR> - %3 = sparse_tensor.convert %td : tensor<4x8xf64> to tensor<4x8xf64, #NV_24> - %4 = sparse_tensor.convert %td : tensor<4x8xf64> to tensor<4x8xf64, #CSR> + %a = sparse_tensor.convert %td : tensor<4x8xf64> to tensor<4x8xf64, #BSR> + %b = sparse_tensor.convert %td : tensor<4x8xf64> to tensor<4x8xf64, #NV_24> + %c = sparse_tensor.convert %td : tensor<4x8xf64> to tensor<4x8xf64, #CSR> %d = call @mul_dense(%td, %td) : (tensor<4x8xf64>, tensor<4x8xf64>) -> tensor<4x4xf64> - %s = call @mul(%td, %2) + %s = call @mul(%td, %a) : (tensor<4x8xf64>, tensor<4x8xf64, #BSR>) -> tensor<4x4xf64> - %s24 = call @mul_24(%td, %3) + %s24 = call @mul_24(%td, %b) : (tensor<4x8xf64>, tensor<4x8xf64, #NV_24>) -> tensor<4x4xf64> - %scsr = call @mul_csr_bsr(%4, %2) + %scsr = call @mul_csr_bsr(%c, %a) : (tensor<4x8xf64, #CSR>, tensor<4x8xf64, #BSR>) -> tensor<4x4xf64> // CHECK-COUNT-4: ( ( 46, 115, 0, 0 ), ( 115, 306, 0, 0 ), ( 0, 0, 858, 1206 ), ( 0, 0, 1206, 1698 ) ) - call @dumpf64(%d) : (tensor<4x4xf64>) -> () - call @dumpf64(%s) : (tensor<4x4xf64>) -> () - call @dumpf64(%s24) : (tensor<4x4xf64>) -> () - call @dumpf64(%scsr) : (tensor<4x4xf64>) -> () + call @dump_dense_f64(%d) : (tensor<4x4xf64>) -> () + call @dump_dense_f64(%s) : (tensor<4x4xf64>) -> () + call @dump_dense_f64(%s24) : (tensor<4x4xf64>) -> () + call @dump_dense_f64(%scsr) : (tensor<4x4xf64>) -> () return } diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex32.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex32.mlir index d97b1a933595..9747da27f9e9 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex32.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex32.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -77,22 +77,8 @@ module { return %0 : tensor, #SparseVector> } - func.func @dump(%arg0: tensor, #SparseVector>, %d: index) { - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %mem = sparse_tensor.values %arg0 : tensor, #SparseVector> to memref> - scf.for %i = %c0 to %d step %c1 { - %v = memref.load %mem[%i] : memref> - %real = complex.re %v : complex - %imag = complex.im %v : complex - vector.print %real : f32 - vector.print %imag : f32 - } - return - } - // Driver method to call and verify complex kernels. - func.func @entry() { + func.func @main() { // Setup sparse vectors. %v1 = arith.constant sparse< [ [0], [28], [31] ], @@ -114,23 +100,26 @@ module { // // Verify the results. // - // CHECK: 511.13 - // CHECK-NEXT: 2 - // CHECK-NEXT: 1 - // CHECK-NEXT: 0 - // CHECK-NEXT: 5 - // CHECK-NEXT: 4 - // CHECK-NEXT: 8 - // CHECK-NEXT: 6 - // CHECK-NEXT: 6 - // CHECK-NEXT: 8 - // CHECK-NEXT: 15 - // CHECK-NEXT: 18 + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 4 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 4, + // CHECK-NEXT: crd[0] : ( 0, 1, 28, 31, + // CHECK-NEXT: values : ( ( 511.13, 2 ), ( 1, 0 ), ( 5, 4 ), ( 8, 6 ), + // CHECK-NEXT: ---- + // + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 2 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 2, + // CHECK-NEXT: crd[0] : ( 28, 31, + // CHECK-NEXT: values : ( ( 6, 8 ), ( 15, 18 ), + // CHECK-NEXT: ---- // - %d1 = arith.constant 4 : index - %d2 = arith.constant 2 : index - call @dump(%0, %d1) : (tensor, #SparseVector>, index) -> () - call @dump(%1, %d2) : (tensor, #SparseVector>, index) -> () + sparse_tensor.print %0 : tensor, #SparseVector> + sparse_tensor.print %1 : tensor, #SparseVector> // Release the resources. bufferization.dealloc_tensor %sv1 : tensor, #SparseVector> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex64.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex64.mlir index 29008473d481..d4b43eb57676 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex64.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex64.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -74,22 +74,8 @@ module { return %0 : tensor, #SparseVector> } - func.func @dump(%arg0: tensor, #SparseVector>, %d: index) { - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %mem = sparse_tensor.values %arg0 : tensor, #SparseVector> to memref> - scf.for %i = %c0 to %d step %c1 { - %v = memref.load %mem[%i] : memref> - %real = complex.re %v : complex - %imag = complex.im %v : complex - vector.print %real : f64 - vector.print %imag : f64 - } - return - } - // Driver method to call and verify complex kernels. - func.func @entry() { + func.func @main() { // Setup sparse vectors. %v1 = arith.constant sparse< [ [0], [28], [31] ], @@ -111,23 +97,26 @@ module { // // Verify the results. // - // CHECK: 511.13 - // CHECK-NEXT: 2 - // CHECK-NEXT: 1 - // CHECK-NEXT: 0 - // CHECK-NEXT: 5 - // CHECK-NEXT: 4 - // CHECK-NEXT: 8 - // CHECK-NEXT: 6 - // CHECK-NEXT: 6 - // CHECK-NEXT: 8 - // CHECK-NEXT: 15 - // CHECK-NEXT: 18 + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 4 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 4, + // CHECK-NEXT: crd[0] : ( 0, 1, 28, 31, + // CHECK-NEXT: values : ( ( 511.13, 2 ), ( 1, 0 ), ( 5, 4 ), ( 8, 6 ), + // CHECK-NEXT: ---- + // + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 2 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 2, + // CHECK-NEXT: crd[0] : ( 28, 31, + // CHECK-NEXT: values : ( ( 6, 8 ), ( 15, 18 ), + // CHECK-NEXT: ---- // - %d1 = arith.constant 4 : index - %d2 = arith.constant 2 : index - call @dump(%0, %d1) : (tensor, #SparseVector>, index) -> () - call @dump(%1, %d2) : (tensor, #SparseVector>, index) -> () + sparse_tensor.print %0 : tensor, #SparseVector> + sparse_tensor.print %1 : tensor, #SparseVector> // Release the resources. bufferization.dealloc_tensor %sv1 : tensor, #SparseVector> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_ds.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_ds.mlir index 251944c657cb..37d8a42a2990 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_ds.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_ds.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -63,12 +63,10 @@ module { // [0.0, 5.0, 6.0, 0.0, 7.0, 0.0, 0.0, 8.0], // [9.0, 0.0, 10.0, 0.0, 11.0, 12.0, 0.0, 0.0]] // - func.func @entry() { - %u0 = arith.constant 0 : i8 - %c0 = arith.constant 0 : index - %f0 = arith.constant 0.0 : f64 - + func.func @main() { + %c0 = arith.constant 0 : index %fileName = call @getTensorFilename(%c0) : (index) -> (!Filename) + %A1 = sparse_tensor.new %fileName : !Filename to tensor %A2 = sparse_tensor.new %fileName : !Filename to tensor %A3 = sparse_tensor.new %fileName : !Filename to tensor @@ -77,62 +75,56 @@ module { // // CSR: // - // CHECK: ( 0, 4, 8, 12 ) - // CHECK-NEXT: ( 2, 3, 5, 7, 1, 2, 4, 7, 0, 2, 4, 5 ) - // CHECK-NEXT: ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 12 + // CHECK-NEXT: dim = ( 3, 8 ) + // CHECK-NEXT: lvl = ( 3, 8 ) + // CHECK-NEXT: pos[1] : ( 0, 4, 8, 12, + // CHECK-NEXT: crd[1] : ( 2, 3, 5, 7, 1, 2, 4, 7, 0, 2, 4, 5, + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + // CHECK-NEXT: ---- // - %pos1 = sparse_tensor.positions %A1 {level = 1 : index } : tensor to memref - %vecp1 = vector.transfer_read %pos1[%c0], %c0 : memref, vector<4xindex> - vector.print %vecp1 : vector<4xindex> - %crd1 = sparse_tensor.coordinates %A1 {level = 1 : index } : tensor to memref - %vecc1 = vector.transfer_read %crd1[%c0], %c0 : memref, vector<12xindex> - vector.print %vecc1 : vector<12xindex> - %val1 = sparse_tensor.values %A1 : tensor to memref - %vecv1 = vector.transfer_read %val1[%c0], %f0 : memref, vector<12xf64> - vector.print %vecv1 : vector<12xf64> + sparse_tensor.print %A1 : tensor // // CSR_hi: // - // CHECK-NEXT: ( 0, 4, 4, 8, 8, 12 ) - // CHECK-NEXT: ( 2, 3, 5, 7, 1, 2, 4, 7, 0, 2, 4, 5 ) - // CHECK-NEXT: ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 ) + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 12 + // CHECK-NEXT: dim = ( 3, 8 ) + // CHECK-NEXT: lvl = ( 3, 8 ) + // CHECK-NEXT: pos[1] : ( 0, 4, 4, 8, 8, 12, + // CHECK-NEXT: crd[1] : ( 2, 3, 5, 7, 1, 2, 4, 7, 0, 2, 4, 5, + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + // CHECK-NEXT: ---- // - %pos2 = sparse_tensor.positions %A2 {level = 1 : index } : tensor to memref - %vecp2 = vector.transfer_read %pos2[%c0], %c0 : memref, vector<6xindex> - vector.print %vecp2 : vector<6xindex> - %crd2 = sparse_tensor.coordinates %A2 {level = 1 : index } : tensor to memref - %vecc2 = vector.transfer_read %crd2[%c0], %c0 : memref, vector<12xindex> - vector.print %vecc2 : vector<12xindex> - %val2 = sparse_tensor.values %A2 : tensor to memref - %vecv2 = vector.transfer_read %val2[%c0], %f0 : memref, vector<12xf64> - vector.print %vecv2 : vector<12xf64> + sparse_tensor.print %A2 : tensor // - // NV_24 + // NV_24: // - // CHECK-NEXT: ( 2, 3, 1, 3, 1, 2, 0, 3, 0, 2, 0, 1 ) - // CHECK-NEXT: ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 ) + // CHECK-NEXT: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 12 + // CHECK-NEXT: dim = ( 3, 8 ) + // CHECK-NEXT: lvl = ( 3, 2, 4 ) + // CHECK-NEXT: crd[2] : ( 2, 3, 1, 3, 1, 2, 0, 3, 0, 2, 0, 1, + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + // CHECK-NEXT: ---- + // CHECK-NEXT: ---- Sparse Tensor ---- // - %crd3 = sparse_tensor.coordinates %A3 {level = 2 : index } : tensor to memref - %vecc3 = vector.transfer_read %crd3[%c0], %u0 : memref, vector<12xi8> - vector.print %vecc3 : vector<12xi8> - %val3 = sparse_tensor.values %A3 : tensor to memref - %vecv3 = vector.transfer_read %val3[%c0], %f0 : memref, vector<12xf64> - vector.print %vecv3 : vector<12xf64> + sparse_tensor.print %A3 : tensor // - // NV_58 + // NV_58: // - // CHECK-NEXT: ( 2, 3, 5, 7, 1, 2, 4, 7, 0, 2, 4, 5 ) - // CHECK-NEXT: ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 ) + // CHECK-NEXT: nse = 12 + // CHECK-NEXT: dim = ( 3, 8 ) + // CHECK-NEXT: lvl = ( 3, 1, 8 ) + // CHECK-NEXT: crd[2] : ( 2, 3, 5, 7, 1, 2, 4, 7, 0, 2, 4, 5, + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + // CHECK-NEXT: ---- // - %crd4 = sparse_tensor.coordinates %A4 {level = 2 : index } : tensor to memref - %vecc4 = vector.transfer_read %crd4[%c0], %u0 : memref, vector<12xi8> - vector.print %vecc4 : vector<12xi8> - %val4 = sparse_tensor.values %A4 : tensor to memref - %vecv4 = vector.transfer_read %val4[%c0], %f0 : memref, vector<12xf64> - vector.print %vecv4 : vector<12xf64> + sparse_tensor.print %A4 : tensor // Release the resources. bufferization.dealloc_tensor %A1: tensor -- GitLab From b8c6252bc19e05fc968c6504b69f5fed208f32d4 Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Tue, 5 Mar 2024 15:25:58 -0500 Subject: [PATCH 202/929] [libc++abi] Always re-export std:: exception types from libc++abi (#84031) We always provide the std:: exception types, even when exceptions are disabled. This is a bit counter-intuitive, but these exception types are just normal types at the end of the day so we made the decision to always provide their definition. Failure to re-export these types would cause libc++ to fail to link on Apple platforms when exceptions are disabled. --- libcxxabi/src/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libcxxabi/src/CMakeLists.txt b/libcxxabi/src/CMakeLists.txt index 0f17ea9184c8..0af4dc1448e9 100644 --- a/libcxxabi/src/CMakeLists.txt +++ b/libcxxabi/src/CMakeLists.txt @@ -230,9 +230,12 @@ if (LIBCXXABI_ENABLE_SHARED) reexport_symbols("${CMAKE_CURRENT_SOURCE_DIR}/../lib/new-delete.exp") endif() + # Note that std:: exception types are always defined by the library regardless of + # whether the exception runtime machinery is provided. + reexport_symbols("${CMAKE_CURRENT_SOURCE_DIR}/../lib/std-exceptions.exp") + if (LIBCXXABI_ENABLE_EXCEPTIONS) reexport_symbols("${CMAKE_CURRENT_SOURCE_DIR}/../lib/itanium-exceptions.exp") - reexport_symbols("${CMAKE_CURRENT_SOURCE_DIR}/../lib/std-exceptions.exp") if ("${CMAKE_OSX_ARCHITECTURES}" MATCHES "^(armv6|armv7|armv7s)$") reexport_symbols("${CMAKE_CURRENT_SOURCE_DIR}/../lib/personality-sjlj.exp") -- GitLab From eccc71783c4a7682e4cc876f62feca74889fb192 Mon Sep 17 00:00:00 2001 From: Neumann Hon Date: Tue, 5 Mar 2024 15:29:07 -0500 Subject: [PATCH 203/929] [SystemZ] [z/OS] Emit offset to PPA2 in separate MCSection (#84043) The ppa2list section isn't really part of the ppa2 section. The ppa2list section contains the offset to the ppa2, and must be created with a special section name (specifically, C_@@QPPA2). The binder searches for a section with this name, then uses this value to locate the ppa2. In GOFF terms, these are entirely separate sections; the PPA2 section isn't even really a section but rather belongs to the code section. On the other hand, the ppa2list section is a section in its own right and resides in a separate TXT record. --- llvm/include/llvm/MC/MCObjectFileInfo.h | 2 ++ llvm/lib/MC/MCObjectFileInfo.cpp | 5 +++++ llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp | 3 +++ 3 files changed, 10 insertions(+) diff --git a/llvm/include/llvm/MC/MCObjectFileInfo.h b/llvm/include/llvm/MC/MCObjectFileInfo.h index 2b2adf5012de..dda3e8a020f3 100644 --- a/llvm/include/llvm/MC/MCObjectFileInfo.h +++ b/llvm/include/llvm/MC/MCObjectFileInfo.h @@ -228,6 +228,7 @@ protected: // GOFF specific sections. MCSection *PPA1Section = nullptr; MCSection *PPA2Section = nullptr; + MCSection *PPA2ListSection = nullptr; MCSection *ADASection = nullptr; MCSection *IDRLSection = nullptr; @@ -434,6 +435,7 @@ public: // GOFF specific sections. MCSection *getPPA1Section() const { return PPA1Section; } MCSection *getPPA2Section() const { return PPA2Section; } + MCSection *getPPA2ListSection() const { return PPA2ListSection; } MCSection *getADASection() const { return ADASection; } MCSection *getIDRLSection() const { return IDRLSection; } diff --git a/llvm/lib/MC/MCObjectFileInfo.cpp b/llvm/lib/MC/MCObjectFileInfo.cpp index 7b382c131ef9..1f8f8ec55727 100644 --- a/llvm/lib/MC/MCObjectFileInfo.cpp +++ b/llvm/lib/MC/MCObjectFileInfo.cpp @@ -554,6 +554,11 @@ void MCObjectFileInfo::initGOFFMCObjectFileInfo(const Triple &T) { PPA2Section = Ctx->getGOFFSection(".ppa2", SectionKind::getMetadata(), TextSection, MCConstantExpr::create(GOFF::SK_PPA2, *Ctx)); + + PPA2ListSection = + Ctx->getGOFFSection(".ppa2list", SectionKind::getData(), + nullptr, nullptr); + ADASection = Ctx->getGOFFSection(".ada", SectionKind::getData(), nullptr, nullptr); IDRLSection = diff --git a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp index 819177068726..5696ae117d69 100644 --- a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp +++ b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp @@ -1531,6 +1531,9 @@ void SystemZAsmPrinter::emitPPA2(Module &M) { OutStreamer->emitInt16(0x0000); // Service level string length. + // The binder requires that the offset to the PPA2 be emitted in a different, + // specially-named section. + OutStreamer->switchSection(getObjFileLowering().getPPA2ListSection()); // Emit 8 byte alignment. // Emit pointer to PPA2 label. OutStreamer->AddComment("A(PPA2-CELQSTRT)"); -- GitLab From 4a4fb930a539c91eb4e9d8b1ea427a7cef72d054 Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Tue, 5 Mar 2024 12:37:11 -0800 Subject: [PATCH 204/929] Use the new ThreadPoolInterface base class instead of the concrete implementation (NFC) (#84056) --- bolt/include/bolt/Core/ParallelUtilities.h | 4 ++-- bolt/lib/Core/ParallelUtilities.cpp | 6 +++--- bolt/lib/Passes/IdenticalCodeFolding.cpp | 2 +- bolt/lib/Rewrite/DWARFRewriter.cpp | 2 +- llvm/tools/llvm-reduce/deltas/Delta.cpp | 4 ++-- mlir/lib/ExecutionEngine/AsyncRuntime.cpp | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/bolt/include/bolt/Core/ParallelUtilities.h b/bolt/include/bolt/Core/ParallelUtilities.h index e510525bc51d..e7b35a79acc7 100644 --- a/bolt/include/bolt/Core/ParallelUtilities.h +++ b/bolt/include/bolt/Core/ParallelUtilities.h @@ -49,8 +49,8 @@ enum SchedulingPolicy { SP_BB_QUADRATIC, /// cost is estimated by the square of the BB count }; -/// Return the managed thread pool and initialize it if not initiliazed. -ThreadPool &getThreadPool(); +/// Return the managed thread pool and initialize it if not initialized. +ThreadPoolInterface &getThreadPool(); /// Perform the work on each BinaryFunction except those that are accepted /// by SkipPredicate, scheduling heuristic is based on SchedPolicy. diff --git a/bolt/lib/Core/ParallelUtilities.cpp b/bolt/lib/Core/ParallelUtilities.cpp index 1a28bc4346ec..88d9444a6a2b 100644 --- a/bolt/lib/Core/ParallelUtilities.cpp +++ b/bolt/lib/Core/ParallelUtilities.cpp @@ -102,7 +102,7 @@ inline unsigned estimateTotalCost(const BinaryContext &BC, } // namespace -ThreadPool &getThreadPool() { +ThreadPoolInterface &getThreadPool() { if (ThreadPoolPtr.get()) return *ThreadPoolPtr; @@ -145,7 +145,7 @@ void runOnEachFunction(BinaryContext &BC, SchedulingPolicy SchedPolicy, TotalCost > BlocksCount ? TotalCost / BlocksCount : 1; // Divide work into blocks of equal cost - ThreadPool &Pool = getThreadPool(); + ThreadPoolInterface &Pool = getThreadPool(); auto BlockBegin = BC.getBinaryFunctions().begin(); unsigned CurrentCost = 0; @@ -202,7 +202,7 @@ void runOnEachFunctionWithUniqueAllocId( TotalCost > BlocksCount ? TotalCost / BlocksCount : 1; // Divide work into blocks of equal cost - ThreadPool &Pool = getThreadPool(); + ThreadPoolInterface &Pool = getThreadPool(); auto BlockBegin = BC.getBinaryFunctions().begin(); unsigned CurrentCost = 0; unsigned AllocId = 1; diff --git a/bolt/lib/Passes/IdenticalCodeFolding.cpp b/bolt/lib/Passes/IdenticalCodeFolding.cpp index 9f8d82b05ccf..87eba10354a3 100644 --- a/bolt/lib/Passes/IdenticalCodeFolding.cpp +++ b/bolt/lib/Passes/IdenticalCodeFolding.cpp @@ -397,7 +397,7 @@ Error IdenticalCodeFolding::runOnFunctions(BinaryContext &BC) { Timer SinglePass("single fold pass", "single fold pass"); LLVM_DEBUG(SinglePass.startTimer()); - ThreadPool *ThPool; + ThreadPoolInterface *ThPool; if (!opts::NoThreads) ThPool = &ParallelUtilities::getThreadPool(); diff --git a/bolt/lib/Rewrite/DWARFRewriter.cpp b/bolt/lib/Rewrite/DWARFRewriter.cpp index ca9d24245ceb..85c2397dcc5b 100644 --- a/bolt/lib/Rewrite/DWARFRewriter.cpp +++ b/bolt/lib/Rewrite/DWARFRewriter.cpp @@ -784,7 +784,7 @@ void DWARFRewriter::updateDebugInfo() { } } else { // Update unit debug info in parallel - ThreadPool &ThreadPool = ParallelUtilities::getThreadPool(); + ThreadPoolInterface &ThreadPool = ParallelUtilities::getThreadPool(); for (std::unique_ptr &CU : BC.DwCtx->compile_units()) ThreadPool.async(processUnitDIE, CU.get(), &DIEBlder); ThreadPool.wait(); diff --git a/llvm/tools/llvm-reduce/deltas/Delta.cpp b/llvm/tools/llvm-reduce/deltas/Delta.cpp index 62dfd62a4d5a..569117e70d6b 100644 --- a/llvm/tools/llvm-reduce/deltas/Delta.cpp +++ b/llvm/tools/llvm-reduce/deltas/Delta.cpp @@ -219,7 +219,7 @@ void llvm::runDeltaPass(TestRunner &Test, ReductionFunc ExtractChunksFromModule, } std::atomic AnyReduced; - std::unique_ptr ChunkThreadPoolPtr; + std::unique_ptr ChunkThreadPoolPtr; if (NumJobs > 1) ChunkThreadPoolPtr = std::make_unique(hardware_concurrency(NumJobs)); @@ -251,7 +251,7 @@ void llvm::runDeltaPass(TestRunner &Test, ReductionFunc ExtractChunksFromModule, unsigned NumInitialTasks = std::min(WorkLeft, unsigned(NumJobs)); unsigned NumChunksProcessed = 0; - ThreadPool &ChunkThreadPool = *ChunkThreadPoolPtr; + ThreadPoolInterface &ChunkThreadPool = *ChunkThreadPoolPtr; assert(TaskQueue.empty()); AnyReduced = false; diff --git a/mlir/lib/ExecutionEngine/AsyncRuntime.cpp b/mlir/lib/ExecutionEngine/AsyncRuntime.cpp index 189902969f8d..ec4a81c042c2 100644 --- a/mlir/lib/ExecutionEngine/AsyncRuntime.cpp +++ b/mlir/lib/ExecutionEngine/AsyncRuntime.cpp @@ -57,7 +57,7 @@ public: return numRefCountedObjects.load(std::memory_order_relaxed); } - llvm::ThreadPool &getThreadPool() { return threadPool; } + llvm::ThreadPoolInterface &getThreadPool() { return threadPool; } private: friend class RefCounted; -- GitLab From 13bb726be31c1e48cea946217badceb5b3d9db95 Mon Sep 17 00:00:00 2001 From: erichkeane Date: Tue, 5 Mar 2024 12:41:33 -0800 Subject: [PATCH 205/929] [OpenACC] Fix typo in StmtOpenACC.cpp header. --- clang/lib/AST/StmtOpenACC.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/lib/AST/StmtOpenACC.cpp b/clang/lib/AST/StmtOpenACC.cpp index f74a777cd695..e6191bc6db70 100644 --- a/clang/lib/AST/StmtOpenACC.cpp +++ b/clang/lib/AST/StmtOpenACC.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // -// This file implements the subclesses of Stmt class declared in StmtOpenACC.h +// This file implements the subclasses of Stmt class declared in StmtOpenACC.h // //===----------------------------------------------------------------------===// -- GitLab From 9faca1e4015582534028f9a5cd14eed7063dbedd Mon Sep 17 00:00:00 2001 From: rohit-rao Date: Tue, 5 Mar 2024 15:55:36 -0500 Subject: [PATCH 206/929] [compiler-rt/darwin] Disable building sanitizers on platforms without fork(). (#83485) The watchOS and tvOS sanitizers do not compile with publicly-available SDKs, failing on calls such as fork() and posix_spawn(). Updating the darwin_test_archs test program to include a call to fork() allows cmake to exclude those platforms when compiling runtimes. This allows public builds to enable watchOS/tvOS and compile builtins but not sanitizers. --- compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake b/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake index 6a62d3bf7adc..1882893ad42c 100644 --- a/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake +++ b/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake @@ -116,7 +116,7 @@ function(darwin_test_archs os valid_archs) if(NOT TEST_COMPILE_ONLY) message(STATUS "Finding valid architectures for ${os}...") set(SIMPLE_C ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/src.c) - file(WRITE ${SIMPLE_C} "#include \nint main(void) { printf(__FILE__); return 0; }\n") + file(WRITE ${SIMPLE_C} "#include \n#include \nint main(void) { printf(__FILE__); fork(); return 0; }\n") set(os_linker_flags) foreach(flag ${DARWIN_${os}_LINK_FLAGS}) -- GitLab From 7de6f61a9da3b73a729413a41477d400d9f08b84 Mon Sep 17 00:00:00 2001 From: bwlodarcz Date: Tue, 5 Mar 2024 21:57:11 +0100 Subject: [PATCH 207/929] [SPIR-V] Memory leak fix in SPIRVEmitIntrinsics (#83015) The architecture of SPIRVEmitIntrinsics is build in such way that every private method is called by one main function runOnFunction which then calls private methods. Private member IRB is allocated in runOnFunction method but it's not freed. Due to that every time when IR function contains intrinsics to emit, runOnFunction is entered and memory is leaked on exit. It's especially true when there are two or more IR functions to emit. IRB is set to nullptr during construction of object and it's left without pointing resource until runOnFunction is entered. This also create possibility of simple mistake when private method is called but there is no resource pointed. Change requires passing IRBuilder by reference to private methods. The visit* functions create it's own IRBuilder thus IRB is eliminated from class scope. In addition there is a small performance improvement because IRBuilder is not allocated by heap. --- llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 273 ++++++++++-------- 1 file changed, 152 insertions(+), 121 deletions(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index c83537bc7ae8..5c432d682732 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -52,16 +52,15 @@ class SPIRVEmitIntrinsics : public FunctionPass, public InstVisitor { SPIRVTargetMachine *TM = nullptr; - IRBuilder<> *IRB = nullptr; Function *F = nullptr; bool TrackConstants = true; DenseMap AggrConsts; DenseSet AggrStores; - void preprocessCompositeConstants(); - void preprocessUndefs(); + void preprocessCompositeConstants(IRBuilder<> &B); + void preprocessUndefs(IRBuilder<> &B); CallInst *buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef Types, - Value *Arg, Value *Arg2, - ArrayRef Imms) { + Value *Arg, Value *Arg2, ArrayRef Imms, + IRBuilder<> &B) { ConstantAsMetadata *CM = ValueAsMetadata::getConstant(Arg); MDTuple *TyMD = MDNode::get(F->getContext(), CM); MetadataAsValue *VMD = MetadataAsValue::get(F->getContext(), TyMD); @@ -70,19 +69,20 @@ class SPIRVEmitIntrinsics Args.push_back(VMD); for (auto *Imm : Imms) Args.push_back(Imm); - return IRB->CreateIntrinsic(IntrID, {Types}, Args); + return B.CreateIntrinsic(IntrID, {Types}, Args); } - void replaceMemInstrUses(Instruction *Old, Instruction *New); - void processInstrAfterVisit(Instruction *I); - void insertAssignPtrTypeIntrs(Instruction *I); - void insertAssignTypeIntrs(Instruction *I); + void replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B); + void processInstrAfterVisit(Instruction *I, IRBuilder<> &B); + void insertAssignPtrTypeIntrs(Instruction *I, IRBuilder<> &B); + void insertAssignTypeIntrs(Instruction *I, IRBuilder<> &B); void insertAssignTypeInstrForTargetExtTypes(TargetExtType *AssignedType, - Value *V); + Value *V, IRBuilder<> &B); void replacePointerOperandWithPtrCast(Instruction *I, Value *Pointer, Type *ExpectedElementType, - unsigned OperandToReplace); - void insertPtrCastOrAssignTypeInstr(Instruction *I); - void processGlobalValue(GlobalVariable &GV); + unsigned OperandToReplace, + IRBuilder<> &B); + void insertPtrCastOrAssignTypeInstr(Instruction *I, IRBuilder<> &B); + void processGlobalValue(GlobalVariable &GV, IRBuilder<> &B); public: static char ID; @@ -156,13 +156,14 @@ static inline void reportFatalOnTokenType(const Instruction *I) { } void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old, - Instruction *New) { + Instruction *New, + IRBuilder<> &B) { while (!Old->user_empty()) { auto *U = Old->user_back(); if (isAssignTypeInstr(U)) { - IRB->SetInsertPoint(U); + B.SetInsertPoint(U); SmallVector Args = {New, U->getOperand(1)}; - IRB->CreateIntrinsic(Intrinsic::spv_assign_type, {New->getType()}, Args); + B.CreateIntrinsic(Intrinsic::spv_assign_type, {New->getType()}, Args); U->eraseFromParent(); } else if (isMemInstrToReplace(U) || isa(U) || isa(U)) { @@ -174,7 +175,7 @@ void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old, Old->eraseFromParent(); } -void SPIRVEmitIntrinsics::preprocessUndefs() { +void SPIRVEmitIntrinsics::preprocessUndefs(IRBuilder<> &B) { std::queue Worklist; for (auto &I : instructions(F)) Worklist.push(&I); @@ -188,8 +189,8 @@ void SPIRVEmitIntrinsics::preprocessUndefs() { if (!AggrUndef || !Op->getType()->isAggregateType()) continue; - IRB->SetInsertPoint(I); - auto *IntrUndef = IRB->CreateIntrinsic(Intrinsic::spv_undef, {}, {}); + B.SetInsertPoint(I); + auto *IntrUndef = B.CreateIntrinsic(Intrinsic::spv_undef, {}, {}); Worklist.push(IntrUndef); I->replaceUsesOfWith(Op, IntrUndef); AggrConsts[IntrUndef] = AggrUndef; @@ -197,7 +198,7 @@ void SPIRVEmitIntrinsics::preprocessUndefs() { } } -void SPIRVEmitIntrinsics::preprocessCompositeConstants() { +void SPIRVEmitIntrinsics::preprocessCompositeConstants(IRBuilder<> &B) { std::queue Worklist; for (auto &I : instructions(F)) Worklist.push(&I); @@ -207,31 +208,35 @@ void SPIRVEmitIntrinsics::preprocessCompositeConstants() { assert(I); bool KeepInst = false; for (const auto &Op : I->operands()) { - auto BuildCompositeIntrinsic = [&KeepInst, &Worklist, &I, &Op, - this](Constant *AggrC, - ArrayRef Args) { - IRB->SetInsertPoint(I); - auto *CCI = - IRB->CreateIntrinsic(Intrinsic::spv_const_composite, {}, {Args}); - Worklist.push(CCI); - I->replaceUsesOfWith(Op, CCI); - KeepInst = true; - AggrConsts[CCI] = AggrC; - }; + auto BuildCompositeIntrinsic = + [](Constant *AggrC, ArrayRef Args, Value *Op, Instruction *I, + IRBuilder<> &B, std::queue &Worklist, + bool &KeepInst, SPIRVEmitIntrinsics &SEI) { + B.SetInsertPoint(I); + auto *CCI = + B.CreateIntrinsic(Intrinsic::spv_const_composite, {}, {Args}); + Worklist.push(CCI); + I->replaceUsesOfWith(Op, CCI); + KeepInst = true; + SEI.AggrConsts[CCI] = AggrC; + }; if (auto *AggrC = dyn_cast(Op)) { SmallVector Args(AggrC->op_begin(), AggrC->op_end()); - BuildCompositeIntrinsic(AggrC, Args); + BuildCompositeIntrinsic(AggrC, Args, Op, I, B, Worklist, KeepInst, + *this); } else if (auto *AggrC = dyn_cast(Op)) { SmallVector Args; for (unsigned i = 0; i < AggrC->getNumElements(); ++i) Args.push_back(AggrC->getElementAsConstant(i)); - BuildCompositeIntrinsic(AggrC, Args); + BuildCompositeIntrinsic(AggrC, Args, Op, I, B, Worklist, KeepInst, + *this); } else if (isa(Op) && !Op->getType()->isVectorTy()) { auto *AggrC = cast(Op); SmallVector Args(AggrC->op_begin(), AggrC->op_end()); - BuildCompositeIntrinsic(AggrC, Args); + BuildCompositeIntrinsic(AggrC, Args, Op, I, B, Worklist, KeepInst, + *this); } } if (!KeepInst) @@ -240,29 +245,34 @@ void SPIRVEmitIntrinsics::preprocessCompositeConstants() { } Instruction *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) { + IRBuilder<> B(I.getParent()); SmallVector Args; for (auto &Op : I.operands()) if (Op.get()->getType()->isSized()) Args.push_back(Op); - IRB->SetInsertPoint(&I); - IRB->CreateIntrinsic(Intrinsic::spv_switch, {I.getOperand(0)->getType()}, - {Args}); + B.SetInsertPoint(&I); + B.CreateIntrinsic(Intrinsic::spv_switch, {I.getOperand(0)->getType()}, + {Args}); return &I; } Instruction *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) { + IRBuilder<> B(I.getParent()); + B.SetInsertPoint(&I); SmallVector Types = {I.getType(), I.getOperand(0)->getType()}; SmallVector Args; - Args.push_back(IRB->getInt1(I.isInBounds())); + Args.push_back(B.getInt1(I.isInBounds())); for (auto &Op : I.operands()) Args.push_back(Op); - auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args}); + auto *NewI = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args}); I.replaceAllUsesWith(NewI); I.eraseFromParent(); return NewI; } Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) { + IRBuilder<> B(I.getParent()); + B.SetInsertPoint(&I); Value *Source = I.getOperand(0); // SPIR-V, contrary to LLVM 17+ IR, supports bitcasts between pointers of @@ -277,7 +287,7 @@ Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) { SmallVector Types = {I.getType(), Source->getType()}; SmallVector Args(I.op_begin(), I.op_end()); - auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_bitcast, {Types}, {Args}); + auto *NewI = B.CreateIntrinsic(Intrinsic::spv_bitcast, {Types}, {Args}); std::string InstName = I.hasName() ? I.getName().str() : ""; I.replaceAllUsesWith(NewI); I.eraseFromParent(); @@ -286,7 +296,7 @@ Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) { } void SPIRVEmitIntrinsics::insertAssignTypeInstrForTargetExtTypes( - TargetExtType *AssignedType, Value *V) { + TargetExtType *AssignedType, Value *V, IRBuilder<> &B) { // Do not emit spv_assign_type if the V is of the AssignedType already. if (V->getType() == AssignedType) return; @@ -311,12 +321,12 @@ void SPIRVEmitIntrinsics::insertAssignTypeInstrForTargetExtTypes( } Constant *Const = UndefValue::get(AssignedType); - buildIntrWithMD(Intrinsic::spv_assign_type, {V->getType()}, Const, V, {}); + buildIntrWithMD(Intrinsic::spv_assign_type, {V->getType()}, Const, V, {}, B); } void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast( Instruction *I, Value *Pointer, Type *ExpectedElementType, - unsigned OperandToReplace) { + unsigned OperandToReplace, IRBuilder<> &B) { // If Pointer is the result of nop BitCastInst (ptr -> ptr), use the source // pointer instead. The BitCastInst should be later removed when visited. while (BitCastInst *BC = dyn_cast(Pointer)) @@ -338,7 +348,7 @@ void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast( if (GEPI && GEPI->getResultElementType() == ExpectedElementType) return; - setInsertPointSkippingPhis(*IRB, I); + setInsertPointSkippingPhis(B, I); Constant *ExpectedElementTypeConst = Constant::getNullValue(ExpectedElementType); ConstantAsMetadata *CM = @@ -392,34 +402,36 @@ void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast( (isa(Pointer) || isa(Pointer))) { buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {Pointer->getType()}, ExpectedElementTypeConst, Pointer, - {IRB->getInt32(AddressSpace)}); + {B.getInt32(AddressSpace)}, B); return; } // Emit spv_ptrcast SmallVector Types = {Pointer->getType(), Pointer->getType()}; - SmallVector Args = {Pointer, VMD, IRB->getInt32(AddressSpace)}; - auto *PtrCastI = IRB->CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args); + SmallVector Args = {Pointer, VMD, B.getInt32(AddressSpace)}; + auto *PtrCastI = B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args); I->setOperand(OperandToReplace, PtrCastI); } -void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I) { +void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I, + IRBuilder<> &B) { // Handle basic instructions: StoreInst *SI = dyn_cast(I); if (SI && F->getCallingConv() == CallingConv::SPIR_KERNEL && SI->getValueOperand()->getType()->isPointerTy() && isa(SI->getValueOperand())) { return replacePointerOperandWithPtrCast( - I, SI->getValueOperand(), IntegerType::getInt8Ty(F->getContext()), 0); + I, SI->getValueOperand(), IntegerType::getInt8Ty(F->getContext()), 0, + B); } else if (SI) { return replacePointerOperandWithPtrCast( - I, SI->getPointerOperand(), SI->getValueOperand()->getType(), 1); + I, SI->getPointerOperand(), SI->getValueOperand()->getType(), 1, B); } else if (LoadInst *LI = dyn_cast(I)) { return replacePointerOperandWithPtrCast(I, LI->getPointerOperand(), - LI->getType(), 0); + LI->getType(), 0, B); } else if (GetElementPtrInst *GEPI = dyn_cast(I)) { return replacePointerOperandWithPtrCast(I, GEPI->getPointerOperand(), - GEPI->getSourceElementType(), 0); + GEPI->getSourceElementType(), 0, B); } // Handle calls to builtins (non-intrinsics): @@ -448,9 +460,9 @@ void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I) { if (ExpectedType->isTargetExtTy()) insertAssignTypeInstrForTargetExtTypes(cast(ExpectedType), - ArgOperand); + ArgOperand, B); else - replacePointerOperandWithPtrCast(CI, ArgOperand, ExpectedType, OpIdx); + replacePointerOperandWithPtrCast(CI, ArgOperand, ExpectedType, OpIdx, B); } } @@ -458,8 +470,10 @@ Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) { SmallVector Types = {I.getType(), I.getOperand(0)->getType(), I.getOperand(1)->getType(), I.getOperand(2)->getType()}; + IRBuilder<> B(I.getParent()); + B.SetInsertPoint(&I); SmallVector Args(I.op_begin(), I.op_end()); - auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args}); + auto *NewI = B.CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args}); std::string InstName = I.hasName() ? I.getName().str() : ""; I.replaceAllUsesWith(NewI); I.eraseFromParent(); @@ -469,10 +483,12 @@ Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) { Instruction * SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) { + IRBuilder<> B(I.getParent()); + B.SetInsertPoint(&I); SmallVector Types = {I.getType(), I.getVectorOperandType(), I.getIndexOperand()->getType()}; SmallVector Args = {I.getVectorOperand(), I.getIndexOperand()}; - auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args}); + auto *NewI = B.CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args}); std::string InstName = I.hasName() ? I.getName().str() : ""; I.replaceAllUsesWith(NewI); I.eraseFromParent(); @@ -481,29 +497,33 @@ SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) { } Instruction *SPIRVEmitIntrinsics::visitInsertValueInst(InsertValueInst &I) { + IRBuilder<> B(I.getParent()); + B.SetInsertPoint(&I); SmallVector Types = {I.getInsertedValueOperand()->getType()}; SmallVector Args; for (auto &Op : I.operands()) if (isa(Op)) - Args.push_back(UndefValue::get(IRB->getInt32Ty())); + Args.push_back(UndefValue::get(B.getInt32Ty())); else Args.push_back(Op); for (auto &Op : I.indices()) - Args.push_back(IRB->getInt32(Op)); + Args.push_back(B.getInt32(Op)); Instruction *NewI = - IRB->CreateIntrinsic(Intrinsic::spv_insertv, {Types}, {Args}); - replaceMemInstrUses(&I, NewI); + B.CreateIntrinsic(Intrinsic::spv_insertv, {Types}, {Args}); + replaceMemInstrUses(&I, NewI, B); return NewI; } Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) { + IRBuilder<> B(I.getParent()); + B.SetInsertPoint(&I); SmallVector Args; for (auto &Op : I.operands()) Args.push_back(Op); for (auto &Op : I.indices()) - Args.push_back(IRB->getInt32(Op)); + Args.push_back(B.getInt32(Op)); auto *NewI = - IRB->CreateIntrinsic(Intrinsic::spv_extractv, {I.getType()}, {Args}); + B.CreateIntrinsic(Intrinsic::spv_extractv, {I.getType()}, {Args}); I.replaceAllUsesWith(NewI); I.eraseFromParent(); return NewI; @@ -512,30 +532,34 @@ Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) { Instruction *SPIRVEmitIntrinsics::visitLoadInst(LoadInst &I) { if (!I.getType()->isAggregateType()) return &I; + IRBuilder<> B(I.getParent()); + B.SetInsertPoint(&I); TrackConstants = false; const auto *TLI = TM->getSubtargetImpl()->getTargetLowering(); MachineMemOperand::Flags Flags = TLI->getLoadMemOperandFlags(I, F->getParent()->getDataLayout()); auto *NewI = - IRB->CreateIntrinsic(Intrinsic::spv_load, {I.getOperand(0)->getType()}, - {I.getPointerOperand(), IRB->getInt16(Flags), - IRB->getInt8(I.getAlign().value())}); - replaceMemInstrUses(&I, NewI); + B.CreateIntrinsic(Intrinsic::spv_load, {I.getOperand(0)->getType()}, + {I.getPointerOperand(), B.getInt16(Flags), + B.getInt8(I.getAlign().value())}); + replaceMemInstrUses(&I, NewI, B); return NewI; } Instruction *SPIRVEmitIntrinsics::visitStoreInst(StoreInst &I) { if (!AggrStores.contains(&I)) return &I; + IRBuilder<> B(I.getParent()); + B.SetInsertPoint(&I); TrackConstants = false; const auto *TLI = TM->getSubtargetImpl()->getTargetLowering(); MachineMemOperand::Flags Flags = TLI->getStoreMemOperandFlags(I, F->getParent()->getDataLayout()); auto *PtrOp = I.getPointerOperand(); - auto *NewI = IRB->CreateIntrinsic( + auto *NewI = B.CreateIntrinsic( Intrinsic::spv_store, {I.getValueOperand()->getType(), PtrOp->getType()}, - {I.getValueOperand(), PtrOp, IRB->getInt16(Flags), - IRB->getInt8(I.getAlign().value())}); + {I.getValueOperand(), PtrOp, B.getInt16(Flags), + B.getInt8(I.getAlign().value())}); I.eraseFromParent(); return NewI; } @@ -552,14 +576,14 @@ Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) { false); ArraySize = I.getArraySize(); } - + IRBuilder<> B(I.getParent()); + B.SetInsertPoint(&I); TrackConstants = false; Type *PtrTy = I.getType(); auto *NewI = - ArraySize - ? IRB->CreateIntrinsic(Intrinsic::spv_alloca_array, - {PtrTy, ArraySize->getType()}, {ArraySize}) - : IRB->CreateIntrinsic(Intrinsic::spv_alloca, {PtrTy}, {}); + ArraySize ? B.CreateIntrinsic(Intrinsic::spv_alloca_array, + {PtrTy, ArraySize->getType()}, {ArraySize}) + : B.CreateIntrinsic(Intrinsic::spv_alloca, {PtrTy}, {}); std::string InstName = I.hasName() ? I.getName().str() : ""; I.replaceAllUsesWith(NewI); I.eraseFromParent(); @@ -569,50 +593,55 @@ Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) { Instruction *SPIRVEmitIntrinsics::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) { assert(I.getType()->isAggregateType() && "Aggregate result is expected"); + IRBuilder<> B(I.getParent()); + B.SetInsertPoint(&I); SmallVector Args; for (auto &Op : I.operands()) Args.push_back(Op); - Args.push_back(IRB->getInt32(I.getSyncScopeID())); - Args.push_back(IRB->getInt32( + Args.push_back(B.getInt32(I.getSyncScopeID())); + Args.push_back(B.getInt32( static_cast(getMemSemantics(I.getSuccessOrdering())))); - Args.push_back(IRB->getInt32( + Args.push_back(B.getInt32( static_cast(getMemSemantics(I.getFailureOrdering())))); - auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_cmpxchg, - {I.getPointerOperand()->getType()}, {Args}); - replaceMemInstrUses(&I, NewI); + auto *NewI = B.CreateIntrinsic(Intrinsic::spv_cmpxchg, + {I.getPointerOperand()->getType()}, {Args}); + replaceMemInstrUses(&I, NewI, B); return NewI; } Instruction *SPIRVEmitIntrinsics::visitUnreachableInst(UnreachableInst &I) { - IRB->SetInsertPoint(&I); - IRB->CreateIntrinsic(Intrinsic::spv_unreachable, {}, {}); + IRBuilder<> B(I.getParent()); + B.SetInsertPoint(&I); + B.CreateIntrinsic(Intrinsic::spv_unreachable, {}, {}); return &I; } -void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV) { +void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV, + IRBuilder<> &B) { // Skip special artifical variable llvm.global.annotations. if (GV.getName() == "llvm.global.annotations") return; if (GV.hasInitializer() && !isa(GV.getInitializer())) { Constant *Init = GV.getInitializer(); - Type *Ty = isAggrToReplace(Init) ? IRB->getInt32Ty() : Init->getType(); - Constant *Const = isAggrToReplace(Init) ? IRB->getInt32(1) : Init; - auto *InitInst = IRB->CreateIntrinsic(Intrinsic::spv_init_global, - {GV.getType(), Ty}, {&GV, Const}); + Type *Ty = isAggrToReplace(Init) ? B.getInt32Ty() : Init->getType(); + Constant *Const = isAggrToReplace(Init) ? B.getInt32(1) : Init; + auto *InitInst = B.CreateIntrinsic(Intrinsic::spv_init_global, + {GV.getType(), Ty}, {&GV, Const}); InitInst->setArgOperand(1, Init); } if ((!GV.hasInitializer() || isa(GV.getInitializer())) && GV.getNumUses() == 0) - IRB->CreateIntrinsic(Intrinsic::spv_unref_global, GV.getType(), &GV); + B.CreateIntrinsic(Intrinsic::spv_unref_global, GV.getType(), &GV); } -void SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(Instruction *I) { +void SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(Instruction *I, + IRBuilder<> &B) { reportFatalOnTokenType(I); if (!I->getType()->isPointerTy() || !requireAssignType(I) || isa(I)) return; - setInsertPointSkippingPhis(*IRB, I->getNextNode()); + setInsertPointSkippingPhis(B, I->getNextNode()); Constant *EltTyConst; unsigned AddressSpace = I->getType()->getPointerAddressSpace(); @@ -624,14 +653,15 @@ void SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(Instruction *I) { EltTyConst = UndefValue::get(IntegerType::getInt8Ty(I->getContext())); buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {I->getType()}, EltTyConst, I, - {IRB->getInt32(AddressSpace)}); + {B.getInt32(AddressSpace)}, B); } -void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I) { +void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I, + IRBuilder<> &B) { reportFatalOnTokenType(I); Type *Ty = I->getType(); if (!Ty->isVoidTy() && !Ty->isPointerTy() && requireAssignType(I)) { - setInsertPointSkippingPhis(*IRB, I->getNextNode()); + setInsertPointSkippingPhis(B, I->getNextNode()); Type *TypeToAssign = Ty; if (auto *II = dyn_cast(I)) { if (II->getIntrinsicID() == Intrinsic::spv_const_composite || @@ -642,33 +672,34 @@ void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I) { } } Constant *Const = UndefValue::get(TypeToAssign); - buildIntrWithMD(Intrinsic::spv_assign_type, {Ty}, Const, I, {}); + buildIntrWithMD(Intrinsic::spv_assign_type, {Ty}, Const, I, {}, B); } for (const auto &Op : I->operands()) { if (isa(Op) || isa(Op) || // Check GetElementPtrConstantExpr case. (isa(Op) && isa(Op))) { - setInsertPointSkippingPhis(*IRB, I); + setInsertPointSkippingPhis(B, I); if (isa(Op) && Op->getType()->isAggregateType()) - buildIntrWithMD(Intrinsic::spv_assign_type, {IRB->getInt32Ty()}, Op, - UndefValue::get(IRB->getInt32Ty()), {}); + buildIntrWithMD(Intrinsic::spv_assign_type, {B.getInt32Ty()}, Op, + UndefValue::get(B.getInt32Ty()), {}, B); else if (!isa(Op)) // TODO: This case could be removed - buildIntrWithMD(Intrinsic::spv_assign_type, {Op->getType()}, Op, Op, - {}); + buildIntrWithMD(Intrinsic::spv_assign_type, {Op->getType()}, Op, Op, {}, + B); } } } -void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I) { +void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I, + IRBuilder<> &B) { auto *II = dyn_cast(I); if (II && II->getIntrinsicID() == Intrinsic::spv_const_composite && TrackConstants) { - IRB->SetInsertPoint(I->getNextNode()); - Type *Ty = IRB->getInt32Ty(); + B.SetInsertPoint(I->getNextNode()); + Type *Ty = B.getInt32Ty(); auto t = AggrConsts.find(I); assert(t != AggrConsts.end()); auto *NewOp = buildIntrWithMD(Intrinsic::spv_track_constant, {Ty, Ty}, - t->second, I, {}); + t->second, I, {}, B); I->replaceAllUsesWith(NewOp); NewOp->setArgOperand(0, I); } @@ -681,18 +712,19 @@ void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I) { if (II && ((II->getIntrinsicID() == Intrinsic::spv_gep && OpNo == 0) || (II->paramHasAttr(OpNo, Attribute::ImmArg)))) continue; - IRB->SetInsertPoint(I); - auto *NewOp = buildIntrWithMD(Intrinsic::spv_track_constant, - {Op->getType(), Op->getType()}, Op, Op, {}); + B.SetInsertPoint(I); + auto *NewOp = + buildIntrWithMD(Intrinsic::spv_track_constant, + {Op->getType(), Op->getType()}, Op, Op, {}, B); I->setOperand(OpNo, NewOp); } } if (I->hasName()) { reportFatalOnTokenType(I); - setInsertPointSkippingPhis(*IRB, I->getNextNode()); + setInsertPointSkippingPhis(B, I->getNextNode()); std::vector Args = {I}; - addStringImm(I->getName(), *IRB, Args); - IRB->CreateIntrinsic(Intrinsic::spv_assign_name, {I->getType()}, Args); + addStringImm(I->getName(), B, Args); + B.CreateIntrinsic(Intrinsic::spv_assign_name, {I->getType()}, Args); } } @@ -700,7 +732,7 @@ bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) { if (Func.isDeclaration()) return false; F = &Func; - IRB = new IRBuilder<>(Func.getContext()); + IRBuilder<> B(Func.getContext()); AggrConsts.clear(); AggrStores.clear(); @@ -715,32 +747,31 @@ bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) { AggrStores.insert(&I); } - IRB->SetInsertPoint(&Func.getEntryBlock(), Func.getEntryBlock().begin()); + B.SetInsertPoint(&Func.getEntryBlock(), Func.getEntryBlock().begin()); for (auto &GV : Func.getParent()->globals()) - processGlobalValue(GV); + processGlobalValue(GV, B); - preprocessUndefs(); - preprocessCompositeConstants(); + preprocessUndefs(B); + preprocessCompositeConstants(B); SmallVector Worklist; for (auto &I : instructions(Func)) Worklist.push_back(&I); for (auto &I : Worklist) { - insertAssignPtrTypeIntrs(I); - insertAssignTypeIntrs(I); - insertPtrCastOrAssignTypeInstr(I); + insertAssignPtrTypeIntrs(I, B); + insertAssignTypeIntrs(I, B); + insertPtrCastOrAssignTypeInstr(I, B); } - for (auto *I : Worklist) { TrackConstants = true; if (!I->getType()->isVoidTy() || isa(I)) - IRB->SetInsertPoint(I->getNextNode()); + B.SetInsertPoint(I->getNextNode()); // Visitors return either the original/newly created instruction for further // processing, nullptr otherwise. I = visit(*I); if (!I) continue; - processInstrAfterVisit(I); + processInstrAfterVisit(I, B); } return true; } -- GitLab From a5095b9892123fb41fd894a1b08fa45c6b3c7f03 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Tue, 5 Mar 2024 12:29:47 -0800 Subject: [PATCH 208/929] [RISCV] Add test for incorrect FP build vector lowering. NFC The lowering is not distinquishing -2147483648.0 and 2147483648.0. --- llvm/test/CodeGen/RISCV/rvv/vle_vid-vfcvt.ll | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/llvm/test/CodeGen/RISCV/rvv/vle_vid-vfcvt.ll b/llvm/test/CodeGen/RISCV/rvv/vle_vid-vfcvt.ll index e764aa2a9aca..5588dd5db598 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vle_vid-vfcvt.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vle_vid-vfcvt.ll @@ -112,3 +112,20 @@ entry: store <4 x float> , ptr %t, align 16 ret void } + +; FIXME: This is miscompiled. This will create -2147483648.0 instead of +; 2147483648.0 for the 4th element. +define void @foo_9(ptr nocapture noundef writeonly %t) { +; CHECK-LABEL: foo_9: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vid.v v8 +; CHECK-NEXT: vsll.vi v8, v8, 31 +; CHECK-NEXT: vrsub.vi v8, v8, 0 +; CHECK-NEXT: vfcvt.f.x.v v8, v8 +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: ret +entry: + store <4 x float> , ptr %t, align 16 + ret void +} -- GitLab From 5a5266248d4f7af101ad5a54960cbd814599a8d6 Mon Sep 17 00:00:00 2001 From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com> Date: Tue, 5 Mar 2024 16:11:13 -0500 Subject: [PATCH 209/929] [HLSL] implement the rcp intrinsic (#83857) This PR implements the frontend for llvm#70100 This PR is part 1 of 2. Part 2 requires an intrinsic to instructions lowering. - `Builtins.td` - add an `rcp` builtin - `CGBuiltin.cpp` - add the builtin to intrinsic lowering - `hlsl_intrinsics.h` - add the `rcp` api - `SemaChecking.cpp` - reuse frac's sema checks - `IntrinsicsDirectX.td` - add the llvm intrinsic --- clang/include/clang/Basic/Builtins.td | 6 +++ clang/lib/CodeGen/CGBuiltin.cpp | 18 +++++-- clang/lib/Headers/hlsl/hlsl_intrinsics.h | 41 +++++++++++++++ clang/lib/Sema/SemaChecking.cpp | 1 + clang/test/CodeGenHLSL/builtins/rcp.hlsl | 53 ++++++++++++++++++++ clang/test/SemaHLSL/BuiltIns/rcp-errors.hlsl | 27 ++++++++++ llvm/include/llvm/IR/IntrinsicsDirectX.td | 5 +- 7 files changed, 144 insertions(+), 7 deletions(-) create mode 100644 clang/test/CodeGenHLSL/builtins/rcp.hlsl create mode 100644 clang/test/SemaHLSL/BuiltIns/rcp-errors.hlsl diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td index dd9c1bb9e5f8..a81131d82c4c 100644 --- a/clang/include/clang/Basic/Builtins.td +++ b/clang/include/clang/Basic/Builtins.td @@ -4584,6 +4584,12 @@ def HLSLMad : LangBuiltin<"HLSL_LANG"> { let Prototype = "void(...)"; } +def HLSLRcp : LangBuiltin<"HLSL_LANG"> { + let Spellings = ["__builtin_hlsl_elementwise_rcp"]; + let Attributes = [NoThrow, Const]; + let Prototype = "void(...)"; +} + // Builtins for XRay. def XRayCustomEvent : Builtin { let Spellings = ["__xray_customevent"]; diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index 31d13d590bc7..cc30665a4eee 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -17966,7 +17966,7 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID, case Builtin::BI__builtin_hlsl_elementwise_any: { Value *Op0 = EmitScalarExpr(E->getArg(0)); return Builder.CreateIntrinsic( - /*ReturnType*/ llvm::Type::getInt1Ty(getLLVMContext()), + /*ReturnType=*/llvm::Type::getInt1Ty(getLLVMContext()), Intrinsic::dx_any, ArrayRef{Op0}, nullptr, "dx.any"); } case Builtin::BI__builtin_hlsl_dot: { @@ -18002,7 +18002,7 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID, "Dot product requires vectors to be of the same size."); return Builder.CreateIntrinsic( - /*ReturnType*/ T0->getScalarType(), Intrinsic::dx_dot, + /*ReturnType=*/T0->getScalarType(), Intrinsic::dx_dot, ArrayRef{Op0, Op1}, nullptr, "dx.dot"); } break; case Builtin::BI__builtin_hlsl_lerp: { @@ -18039,7 +18039,7 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID, XVecTy->getElementType() == SVecTy->getElementType() && "Lerp requires float vectors to be of the same type."); return Builder.CreateIntrinsic( - /*ReturnType*/ Xty, Intrinsic::dx_lerp, ArrayRef{X, Y, S}, + /*ReturnType=*/Xty, Intrinsic::dx_lerp, ArrayRef{X, Y, S}, nullptr, "dx.lerp"); } case Builtin::BI__builtin_hlsl_elementwise_frac: { @@ -18047,7 +18047,7 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID, if (!E->getArg(0)->getType()->hasFloatingRepresentation()) llvm_unreachable("frac operand must have a float representation"); return Builder.CreateIntrinsic( - /*ReturnType*/ Op0->getType(), Intrinsic::dx_frac, + /*ReturnType=*/Op0->getType(), Intrinsic::dx_frac, ArrayRef{Op0}, nullptr, "dx.frac"); } case Builtin::BI__builtin_hlsl_mad: { @@ -18066,9 +18066,17 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID, } assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation()); return Builder.CreateIntrinsic( - /*ReturnType*/ M->getType(), Intrinsic::dx_umad, + /*ReturnType=*/M->getType(), Intrinsic::dx_umad, ArrayRef{M, A, B}, nullptr, "dx.umad"); } + case Builtin::BI__builtin_hlsl_elementwise_rcp: { + Value *Op0 = EmitScalarExpr(E->getArg(0)); + if (!E->getArg(0)->getType()->hasFloatingRepresentation()) + llvm_unreachable("rcp operand must have a float representation"); + return Builder.CreateIntrinsic( + /*ReturnType=*/Op0->getType(), Intrinsic::dx_rcp, + ArrayRef{Op0}, nullptr, "dx.rcp"); + } } return nullptr; } diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_intrinsics.h index b794a96f0bf3..45f854439258 100644 --- a/clang/lib/Headers/hlsl/hlsl_intrinsics.h +++ b/clang/lib/Headers/hlsl/hlsl_intrinsics.h @@ -1112,6 +1112,47 @@ uint64_t3 reversebits(uint64_t3); _HLSL_BUILTIN_ALIAS(__builtin_elementwise_bitreverse) uint64_t4 reversebits(uint64_t4); +//===----------------------------------------------------------------------===// +// rcp builtins +//===----------------------------------------------------------------------===// + +/// \fn T rcp(T x) +/// \brief Calculates a fast, approximate, per-component reciprocal ie 1 / \a x. +/// \param x The specified input value. +/// +/// The return value is the reciprocal of the \a x parameter. + +_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_rcp) +half rcp(half); +_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_rcp) +half2 rcp(half2); +_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_rcp) +half3 rcp(half3); +_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2) +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_rcp) +half4 rcp(half4); + +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_rcp) +float rcp(float); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_rcp) +float2 rcp(float2); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_rcp) +float3 rcp(float3); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_rcp) +float4 rcp(float4); + +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_rcp) +double rcp(double); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_rcp) +double2 rcp(double2); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_rcp) +double3 rcp(double3); +_HLSL_BUILTIN_ALIAS(__builtin_hlsl_elementwise_rcp) +double4 rcp(double4); + //===----------------------------------------------------------------------===// // round builtins //===----------------------------------------------------------------------===// diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index c4dee874e65c..9a2aa2120ca2 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -5285,6 +5285,7 @@ bool Sema::CheckHLSLBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) { return true; break; } + case Builtin::BI__builtin_hlsl_elementwise_rcp: case Builtin::BI__builtin_hlsl_elementwise_frac: { if (PrepareBuiltinElementwiseMathOneArgCall(TheCall)) return true; diff --git a/clang/test/CodeGenHLSL/builtins/rcp.hlsl b/clang/test/CodeGenHLSL/builtins/rcp.hlsl new file mode 100644 index 000000000000..9b8406e1f0b6 --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/rcp.hlsl @@ -0,0 +1,53 @@ +// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \ +// RUN: dxil-pc-shadermodel6.3-library %s -fnative-half-type \ +// RUN: -emit-llvm -disable-llvm-passes -o - | FileCheck %s \ +// RUN: --check-prefixes=CHECK,NATIVE_HALF +// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \ +// RUN: dxil-pc-shadermodel6.3-library %s -emit-llvm -disable-llvm-passes \ +// RUN: -o - | FileCheck %s --check-prefixes=CHECK,NO_HALF + +// NATIVE_HALF: define noundef half @ +// NATIVE_HALF: %dx.rcp = call half @llvm.dx.rcp.f16( +// NATIVE_HALF: ret half %dx.rcp +// NO_HALF: define noundef float @"?test_rcp_half@@YA$halff@$halff@@Z"( +// NO_HALF: %dx.rcp = call float @llvm.dx.rcp.f32( +// NO_HALF: ret float %dx.rcp +half test_rcp_half(half p0) { return rcp(p0); } +// NATIVE_HALF: define noundef <2 x half> @ +// NATIVE_HALF: %dx.rcp = call <2 x half> @llvm.dx.rcp.v2f16 +// NATIVE_HALF: ret <2 x half> %dx.rcp +// NO_HALF: define noundef <2 x float> @ +// NO_HALF: %dx.rcp = call <2 x float> @llvm.dx.rcp.v2f32( +// NO_HALF: ret <2 x float> %dx.rcp +half2 test_rcp_half2(half2 p0) { return rcp(p0); } +// NATIVE_HALF: define noundef <3 x half> @ +// NATIVE_HALF: %dx.rcp = call <3 x half> @llvm.dx.rcp.v3f16 +// NATIVE_HALF: ret <3 x half> %dx.rcp +// NO_HALF: define noundef <3 x float> @ +// NO_HALF: %dx.rcp = call <3 x float> @llvm.dx.rcp.v3f32( +// NO_HALF: ret <3 x float> %dx.rcp +half3 test_rcp_half3(half3 p0) { return rcp(p0); } +// NATIVE_HALF: define noundef <4 x half> @ +// NATIVE_HALF: %dx.rcp = call <4 x half> @llvm.dx.rcp.v4f16 +// NATIVE_HALF: ret <4 x half> %dx.rcp +// NO_HALF: define noundef <4 x float> @ +// NO_HALF: %dx.rcp = call <4 x float> @llvm.dx.rcp.v4f32( +// NO_HALF: ret <4 x float> %dx.rcp +half4 test_rcp_half4(half4 p0) { return rcp(p0); } + +// CHECK: define noundef float @ +// CHECK: %dx.rcp = call float @llvm.dx.rcp.f32( +// CHECK: ret float %dx.rcp +float test_rcp_float(float p0) { return rcp(p0); } +// CHECK: define noundef <2 x float> @ +// CHECK: %dx.rcp = call <2 x float> @llvm.dx.rcp.v2f32 +// CHECK: ret <2 x float> %dx.rcp +float2 test_rcp_float2(float2 p0) { return rcp(p0); } +// CHECK: define noundef <3 x float> @ +// CHECK: %dx.rcp = call <3 x float> @llvm.dx.rcp.v3f32 +// CHECK: ret <3 x float> %dx.rcp +float3 test_rcp_float3(float3 p0) { return rcp(p0); } +// CHECK: define noundef <4 x float> @ +// CHECK: %dx.rcp = call <4 x float> @llvm.dx.rcp.v4f32 +// CHECK: ret <4 x float> %dx.rcp +float4 test_rcp_float4(float4 p0) { return rcp(p0); } diff --git a/clang/test/SemaHLSL/BuiltIns/rcp-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/rcp-errors.hlsl new file mode 100644 index 000000000000..dc4501dbd6d1 --- /dev/null +++ b/clang/test/SemaHLSL/BuiltIns/rcp-errors.hlsl @@ -0,0 +1,27 @@ + +// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm -disable-llvm-passes -verify -verify-ignore-unexpected + +float test_too_few_arg() { + return __builtin_hlsl_elementwise_rcp(); + // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} +} + +float2 test_too_many_arg(float2 p0) { + return __builtin_hlsl_elementwise_rcp(p0, p0); + // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} +} + +float builtin_bool_to_float_type_promotion(bool p1) { + return __builtin_hlsl_elementwise_rcp(p1); + // expected-error@-1 {{1st argument must be a vector, integer or floating point type (was 'bool')}} +} + +float builtin_rcp_int_to_float_promotion(int p1) { + return __builtin_hlsl_elementwise_rcp(p1); + // expected-error@-1 {{passing 'int' to parameter of incompatible type 'float'}} +} + +float2 builtin_rcp_int2_to_float2_promotion(int2 p1) { + return __builtin_hlsl_elementwise_rcp(p1); + // expected-error@-1 {{passing 'int2' (aka 'vector') to parameter of incompatible type '__attribute__((__vector_size__(2 * sizeof(float)))) float' (vector of 2 'float' values)}} +} diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td index acf45b6b3172..7229292e377a 100644 --- a/llvm/include/llvm/IR/IntrinsicsDirectX.td +++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td @@ -34,6 +34,7 @@ def int_dx_lerp : [llvm_anyvector_ty, LLVMScalarOrSameVectorWidth<0, LLVMVectorElementType<0>>,LLVMScalarOrSameVectorWidth<0, LLVMVectorElementType<0>>], [IntrNoMem, IntrWillReturn] >; -def int_dx_imad : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>; -def int_dx_umad : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>; +def int_dx_imad : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>; +def int_dx_umad : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>; +def int_dx_rcp : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>; } -- GitLab From 50d848d0761e052e203136f3de9a332bd619595a Mon Sep 17 00:00:00 2001 From: Slava Zakharin Date: Tue, 5 Mar 2024 13:36:48 -0800 Subject: [PATCH 210/929] [flang] Added lowering and runtime for COMPLEX(16) intrinsics. (#83874) For `LDBL_MANT_DIG == 113` targets the FortranFloat128Math library is just an interface library that provides sources and compilation options to be used for building FortranRuntime - there are not extra dependencies on other libraries, so it can be a part of FortranRuntime, which helps to avoid extra linking steps in the compiler driver. Targets with __float128 support in libc will also use this path. Other targets, where the math support comes from FLANG_RUNTIME_F128_MATH_LIB, FortranFloat128Math is built as a standalone static library, and the compiler driver needs to conduct the linking. Flang APIs for COMPLEX(16) are just thin C wrappers around the C math functions. Flang uses C _Complex ABI for passing/returning COMPLEX values, so the runtime is aligned to this. --- flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 56 ++++++-- flang/runtime/CMakeLists.txt | 43 +++++-- flang/runtime/Float128Math/CMakeLists.txt | 99 +++++++++------ flang/runtime/Float128Math/cabs.cpp | 24 ---- flang/runtime/Float128Math/complex-math.c | 55 ++++++++ flang/runtime/Float128Math/complex-math.h | 62 +++++++++ flang/runtime/Float128Math/exponent.cpp | 2 +- flang/runtime/Float128Math/fraction.cpp | 2 +- flang/runtime/Float128Math/math-entries.h | 120 ++++++++---------- flang/runtime/Float128Math/mod-real.cpp | 2 +- flang/runtime/Float128Math/modulo-real.cpp | 2 +- flang/runtime/Float128Math/nearest.cpp | 2 +- flang/runtime/Float128Math/rrspacing.cpp | 2 +- flang/runtime/Float128Math/scale.cpp | 2 +- flang/runtime/Float128Math/set-exponent.cpp | 2 +- flang/runtime/Float128Math/spacing.cpp | 2 +- flang/runtime/numeric.cpp | 60 --------- .../test/Lower/Intrinsics/acos_complex16.f90 | 8 ++ .../test/Lower/Intrinsics/acosh_complex16.f90 | 8 ++ .../test/Lower/Intrinsics/asin_complex16.f90 | 8 ++ .../test/Lower/Intrinsics/asinh_complex16.f90 | 8 ++ .../test/Lower/Intrinsics/atan_complex16.f90 | 8 ++ .../test/Lower/Intrinsics/atanh_complex16.f90 | 8 ++ flang/test/Lower/Intrinsics/cos_complex16.f90 | 8 ++ .../test/Lower/Intrinsics/cosh_complex16.f90 | 8 ++ flang/test/Lower/Intrinsics/exp_complex16.f90 | 8 ++ flang/test/Lower/Intrinsics/log_complex16.f90 | 8 ++ .../Lower/Intrinsics/missing-math-runtime.f90 | 12 -- flang/test/Lower/Intrinsics/pow_complex16.f90 | 8 ++ flang/test/Lower/Intrinsics/sin_complex16.f90 | 8 ++ .../test/Lower/Intrinsics/sinh_complex16.f90 | 8 ++ .../test/Lower/Intrinsics/sqrt_complex16.f90 | 8 ++ flang/test/Lower/Intrinsics/tan_complex16.f90 | 8 ++ .../test/Lower/Intrinsics/tanh_complex16.f90 | 8 ++ 34 files changed, 444 insertions(+), 233 deletions(-) delete mode 100644 flang/runtime/Float128Math/cabs.cpp create mode 100644 flang/runtime/Float128Math/complex-math.c create mode 100644 flang/runtime/Float128Math/complex-math.h create mode 100644 flang/test/Lower/Intrinsics/acos_complex16.f90 create mode 100644 flang/test/Lower/Intrinsics/acosh_complex16.f90 create mode 100644 flang/test/Lower/Intrinsics/asin_complex16.f90 create mode 100644 flang/test/Lower/Intrinsics/asinh_complex16.f90 create mode 100644 flang/test/Lower/Intrinsics/atan_complex16.f90 create mode 100644 flang/test/Lower/Intrinsics/atanh_complex16.f90 create mode 100644 flang/test/Lower/Intrinsics/cos_complex16.f90 create mode 100644 flang/test/Lower/Intrinsics/cosh_complex16.f90 create mode 100644 flang/test/Lower/Intrinsics/exp_complex16.f90 create mode 100644 flang/test/Lower/Intrinsics/log_complex16.f90 delete mode 100644 flang/test/Lower/Intrinsics/missing-math-runtime.f90 create mode 100644 flang/test/Lower/Intrinsics/pow_complex16.f90 create mode 100644 flang/test/Lower/Intrinsics/sin_complex16.f90 create mode 100644 flang/test/Lower/Intrinsics/sinh_complex16.f90 create mode 100644 flang/test/Lower/Intrinsics/sqrt_complex16.f90 create mode 100644 flang/test/Lower/Intrinsics/tan_complex16.f90 create mode 100644 flang/test/Lower/Intrinsics/tanh_complex16.f90 diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp index fb9b58ef69c6..25598ed16831 100644 --- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp +++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp @@ -689,22 +689,22 @@ prettyPrintIntrinsicName(fir::FirOpBuilder &builder, mlir::Location loc, } // Generate a call to the Fortran runtime library providing -// support for 128-bit float math via a third-party library. -// If the compiler is built without FLANG_RUNTIME_F128_MATH_LIB, -// this function will report an error. +// support for 128-bit float math. +// On 'LDBL_MANT_DIG == 113' targets the implementation +// is provided by FortranRuntime, otherwise, it is done via +// FortranFloat128Math library. In the latter case the compiler +// has to be built with FLANG_RUNTIME_F128_MATH_LIB to guarantee +// proper linking actions in the driver. static mlir::Value genLibF128Call(fir::FirOpBuilder &builder, mlir::Location loc, const MathOperation &mathOp, mlir::FunctionType libFuncType, llvm::ArrayRef args) { -#ifndef FLANG_RUNTIME_F128_MATH_LIB - std::string message = prettyPrintIntrinsicName( - builder, loc, "compiler is built without support for '", mathOp.key, "'", - libFuncType); - fir::emitFatalError(loc, message, /*genCrashDiag=*/false); -#else // FLANG_RUNTIME_F128_MATH_LIB + // TODO: if we knew that the C 'long double' does not have 113-bit mantissa + // on the target, we could have asserted that FLANG_RUNTIME_F128_MATH_LIB + // must be specified. For now just always generate the call even + // if it will be unresolved. return genLibCall(builder, loc, mathOp, libFuncType, args); -#endif // FLANG_RUNTIME_F128_MATH_LIB } mlir::Value genLibCall(fir::FirOpBuilder &builder, mlir::Location loc, @@ -926,6 +926,10 @@ constexpr auto FuncTypeInteger8Real16 = genFuncType, Ty::Real<16>>; constexpr auto FuncTypeReal16Complex16 = genFuncType, Ty::Complex<16>>; +constexpr auto FuncTypeComplex16Complex16 = + genFuncType, Ty::Complex<16>>; +constexpr auto FuncTypeComplex16Complex16Complex16 = + genFuncType, Ty::Complex<16>, Ty::Complex<16>>; static constexpr MathOperation mathOperations[] = { {"abs", "fabsf", genFuncType, Ty::Real<4>>, @@ -944,6 +948,8 @@ static constexpr MathOperation mathOperations[] = { {"acos", RTNAME_STRING(AcosF128), FuncTypeReal16Real16, genLibF128Call}, {"acos", "cacosf", genFuncType, Ty::Complex<4>>, genLibCall}, {"acos", "cacos", genFuncType, Ty::Complex<8>>, genLibCall}, + {"acos", RTNAME_STRING(CAcosF128), FuncTypeComplex16Complex16, + genLibF128Call}, {"acosh", "acoshf", genFuncType, Ty::Real<4>>, genLibCall}, {"acosh", "acosh", genFuncType, Ty::Real<8>>, genLibCall}, {"acosh", RTNAME_STRING(AcoshF128), FuncTypeReal16Real16, genLibF128Call}, @@ -951,6 +957,8 @@ static constexpr MathOperation mathOperations[] = { genLibCall}, {"acosh", "cacosh", genFuncType, Ty::Complex<8>>, genLibCall}, + {"acosh", RTNAME_STRING(CAcoshF128), FuncTypeComplex16Complex16, + genLibF128Call}, // llvm.trunc behaves the same way as libm's trunc. {"aint", "llvm.trunc.f32", genFuncType, Ty::Real<4>>, genLibCall}, @@ -972,6 +980,8 @@ static constexpr MathOperation mathOperations[] = { {"asin", RTNAME_STRING(AsinF128), FuncTypeReal16Real16, genLibF128Call}, {"asin", "casinf", genFuncType, Ty::Complex<4>>, genLibCall}, {"asin", "casin", genFuncType, Ty::Complex<8>>, genLibCall}, + {"asin", RTNAME_STRING(CAsinF128), FuncTypeComplex16Complex16, + genLibF128Call}, {"asinh", "asinhf", genFuncType, Ty::Real<4>>, genLibCall}, {"asinh", "asinh", genFuncType, Ty::Real<8>>, genLibCall}, {"asinh", RTNAME_STRING(AsinhF128), FuncTypeReal16Real16, genLibF128Call}, @@ -979,6 +989,8 @@ static constexpr MathOperation mathOperations[] = { genLibCall}, {"asinh", "casinh", genFuncType, Ty::Complex<8>>, genLibCall}, + {"asinh", RTNAME_STRING(CAsinhF128), FuncTypeComplex16Complex16, + genLibF128Call}, {"atan", "atanf", genFuncType, Ty::Real<4>>, genMathOp}, {"atan", "atan", genFuncType, Ty::Real<8>>, @@ -986,6 +998,8 @@ static constexpr MathOperation mathOperations[] = { {"atan", RTNAME_STRING(AtanF128), FuncTypeReal16Real16, genLibF128Call}, {"atan", "catanf", genFuncType, Ty::Complex<4>>, genLibCall}, {"atan", "catan", genFuncType, Ty::Complex<8>>, genLibCall}, + {"atan", RTNAME_STRING(CAtanF128), FuncTypeComplex16Complex16, + genLibF128Call}, {"atan2", "atan2f", genFuncType, Ty::Real<4>, Ty::Real<4>>, genMathOp}, {"atan2", "atan2", genFuncType, Ty::Real<8>, Ty::Real<8>>, @@ -999,6 +1013,8 @@ static constexpr MathOperation mathOperations[] = { genLibCall}, {"atanh", "catanh", genFuncType, Ty::Complex<8>>, genLibCall}, + {"atanh", RTNAME_STRING(CAtanhF128), FuncTypeComplex16Complex16, + genLibF128Call}, {"bessel_j0", "j0f", genFuncType, Ty::Real<4>>, genLibCall}, {"bessel_j0", "j0", genFuncType, Ty::Real<8>>, genLibCall}, {"bessel_j0", RTNAME_STRING(J0F128), FuncTypeReal16Real16, genLibF128Call}, @@ -1038,11 +1054,15 @@ static constexpr MathOperation mathOperations[] = { genComplexMathOp}, {"cos", "ccos", genFuncType, Ty::Complex<8>>, genComplexMathOp}, + {"cos", RTNAME_STRING(CCosF128), FuncTypeComplex16Complex16, + genLibF128Call}, {"cosh", "coshf", genFuncType, Ty::Real<4>>, genLibCall}, {"cosh", "cosh", genFuncType, Ty::Real<8>>, genLibCall}, {"cosh", RTNAME_STRING(CoshF128), FuncTypeReal16Real16, genLibF128Call}, {"cosh", "ccoshf", genFuncType, Ty::Complex<4>>, genLibCall}, {"cosh", "ccosh", genFuncType, Ty::Complex<8>>, genLibCall}, + {"cosh", RTNAME_STRING(CCoshF128), FuncTypeComplex16Complex16, + genLibF128Call}, {"divc", {}, genFuncType, Ty::Complex<2>, Ty::Complex<2>>, @@ -1080,6 +1100,8 @@ static constexpr MathOperation mathOperations[] = { genComplexMathOp}, {"exp", "cexp", genFuncType, Ty::Complex<8>>, genComplexMathOp}, + {"exp", RTNAME_STRING(CExpF128), FuncTypeComplex16Complex16, + genLibF128Call}, {"feclearexcept", "feclearexcept", genFuncType, Ty::Integer<4>>, genLibCall}, {"fedisableexcept", "fedisableexcept", @@ -1131,6 +1153,8 @@ static constexpr MathOperation mathOperations[] = { genComplexMathOp}, {"log", "clog", genFuncType, Ty::Complex<8>>, genComplexMathOp}, + {"log", RTNAME_STRING(CLogF128), FuncTypeComplex16Complex16, + genLibF128Call}, {"log10", "log10f", genFuncType, Ty::Real<4>>, genMathOp}, {"log10", "log10", genFuncType, Ty::Real<8>>, @@ -1178,6 +1202,8 @@ static constexpr MathOperation mathOperations[] = { genComplexMathOp}, {"pow", "cpow", genFuncType, Ty::Complex<8>, Ty::Complex<8>>, genComplexMathOp}, + {"pow", RTNAME_STRING(CPowF128), FuncTypeComplex16Complex16Complex16, + genLibF128Call}, {"pow", RTNAME_STRING(FPow4i), genFuncType, Ty::Real<4>, Ty::Integer<4>>, genMathOp}, @@ -1222,11 +1248,15 @@ static constexpr MathOperation mathOperations[] = { genComplexMathOp}, {"sin", "csin", genFuncType, Ty::Complex<8>>, genComplexMathOp}, + {"sin", RTNAME_STRING(CSinF128), FuncTypeComplex16Complex16, + genLibF128Call}, {"sinh", "sinhf", genFuncType, Ty::Real<4>>, genLibCall}, {"sinh", "sinh", genFuncType, Ty::Real<8>>, genLibCall}, {"sinh", RTNAME_STRING(SinhF128), FuncTypeReal16Real16, genLibF128Call}, {"sinh", "csinhf", genFuncType, Ty::Complex<4>>, genLibCall}, {"sinh", "csinh", genFuncType, Ty::Complex<8>>, genLibCall}, + {"sinh", RTNAME_STRING(CSinhF128), FuncTypeComplex16Complex16, + genLibF128Call}, {"sqrt", "sqrtf", genFuncType, Ty::Real<4>>, genMathOp}, {"sqrt", "sqrt", genFuncType, Ty::Real<8>>, @@ -1236,6 +1266,8 @@ static constexpr MathOperation mathOperations[] = { genComplexMathOp}, {"sqrt", "csqrt", genFuncType, Ty::Complex<8>>, genComplexMathOp}, + {"sqrt", RTNAME_STRING(CSqrtF128), FuncTypeComplex16Complex16, + genLibF128Call}, {"tan", "tanf", genFuncType, Ty::Real<4>>, genMathOp}, {"tan", "tan", genFuncType, Ty::Real<8>>, @@ -1245,6 +1277,8 @@ static constexpr MathOperation mathOperations[] = { genComplexMathOp}, {"tan", "ctan", genFuncType, Ty::Complex<8>>, genComplexMathOp}, + {"tan", RTNAME_STRING(CTanF128), FuncTypeComplex16Complex16, + genLibF128Call}, {"tanh", "tanhf", genFuncType, Ty::Real<4>>, genMathOp}, {"tanh", "tanh", genFuncType, Ty::Real<8>>, @@ -1254,6 +1288,8 @@ static constexpr MathOperation mathOperations[] = { genComplexMathOp}, {"tanh", "ctanh", genFuncType, Ty::Complex<8>>, genComplexMathOp}, + {"tanh", RTNAME_STRING(CTanhF128), FuncTypeComplex16Complex16, + genLibF128Call}, }; // This helper class computes a "distance" between two function types. diff --git a/flang/runtime/CMakeLists.txt b/flang/runtime/CMakeLists.txt index ac89184a7cbf..7dd60b5edcd5 100644 --- a/flang/runtime/CMakeLists.txt +++ b/flang/runtime/CMakeLists.txt @@ -57,12 +57,6 @@ if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) REAL(16) is mapped to __float128, or libm for targets where REAL(16) \ is mapped to long double, etc." ) - - if (NOT FLANG_RUNTIME_F128_MATH_LIB STREQUAL "") - add_compile_definitions( - -DFLANG_RUNTIME_F128_MATH_LIB="${FLANG_RUNTIME_F128_MATH_LIB}" - ) - endif() endif() include(CheckCXXSymbolExists) @@ -78,6 +72,16 @@ check_cxx_source_compiles( " HAVE_DECL_STRERROR_S) +# Check if 128-bit float computations can be done via long double. +check_cxx_source_compiles( + "#include + #if LDBL_MANT_DIG != 113 + #error LDBL_MANT_DIG != 113 + #endif + int main() { return 0; } + " + HAVE_LDBL_MANT_DIG_113) + check_cxx_compiler_flag(-fno-lto FLANG_RUNTIME_HAS_FNO_LTO_FLAG) if (FLANG_RUNTIME_HAS_FNO_LTO_FLAG) set(NO_LTO_FLAGS "-fno-lto") @@ -100,9 +104,7 @@ add_definitions(-U_GLIBCXX_ASSERTIONS) add_definitions(-U_LIBCPP_ENABLE_ASSERTIONS) add_subdirectory(FortranMain) -if (NOT ${FLANG_RUNTIME_F128_MATH_LIB} STREQUAL "") - add_subdirectory(Float128Math) -endif() +add_subdirectory(Float128Math) set(sources ISO_Fortran_binding.cpp @@ -319,6 +321,29 @@ if (NOT FLANG_EXPERIMENTAL_OMP_OFFLOAD_BUILD STREQUAL "off") endif() endif() +if (NOT TARGET FortranFloat128Math) + # If FortranFloat128Math is not defined, then we are not building + # standalone FortranFloat128Math library. Instead, include + # the relevant sources into FortranRuntime itself. + # The information is provided via FortranFloat128MathILib + # interface library. + get_target_property(f128_sources + FortranFloat128MathILib INTERFACE_SOURCES + ) + if (f128_sources) + # The interface may define special macros for Float128Math files, + # so we need to propagate them. + get_target_property(f128_defs + FortranFloat128MathILib INTERFACE_COMPILE_DEFINITIONS + ) + set_property(SOURCE ${f128_sources} + APPEND PROPERTY COMPILE_DEFINITIONS + ${f128_defs} + ) + list(APPEND sources ${f128_sources}) + endif() +endif() + if (NOT DEFINED MSVC) add_flang_library(FortranRuntime ${sources} diff --git a/flang/runtime/Float128Math/CMakeLists.txt b/flang/runtime/Float128Math/CMakeLists.txt index 60d44c78be0f..980356131b68 100644 --- a/flang/runtime/Float128Math/CMakeLists.txt +++ b/flang/runtime/Float128Math/CMakeLists.txt @@ -16,34 +16,6 @@ include(CheckLibraryExists) -if (${FLANG_RUNTIME_F128_MATH_LIB} STREQUAL "libquadmath") - check_include_file(quadmath.h FOUND_QUADMATH_HEADER) - if(FOUND_QUADMATH_HEADER) - add_compile_definitions(HAS_QUADMATHLIB) - else() - message(FATAL_ERROR - "FLANG_RUNTIME_F128_MATH_LIB setting requires quadmath.h " - "to be available: ${FLANG_RUNTIME_F128_MATH_LIB}" - ) - endif() -elseif (${FLANG_RUNTIME_F128_MATH_LIB} STREQUAL "libm") - check_library_exists(m sinl "" FOUND_LIBM) - check_library_exists(m sinf128 "" FOUND_LIBMF128) - if (FOUND_LIBM) - add_compile_definitions(HAS_LIBM) - endif() - if (FOUND_LIBMF128) - add_compile_definitions(HAS_LIBMF128) - endif() -endif() - -if (NOT FOUND_QUADMATH_HEADER AND NOT FOUND_LIBM) - message(FATAL_ERROR - "Unsupported third-party library for Fortran F128 math runtime: " - "${FLANG_RUNTIME_F128_MATH_LIB}" - ) -endif() - set(sources acos.cpp acosh.cpp @@ -52,8 +24,8 @@ set(sources atan.cpp atan2.cpp atanh.cpp - cabs.cpp ceil.cpp + complex-math.c cos.cpp cosh.cpp erf.cpp @@ -94,18 +66,61 @@ set(sources ) include_directories(AFTER "${CMAKE_CURRENT_SOURCE_DIR}/..") -add_flang_library(FortranFloat128Math STATIC INSTALL_WITH_TOOLCHAIN ${sources}) +add_library(FortranFloat128MathILib INTERFACE) -if (DEFINED MSVC) - set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreaded) - add_flang_library(FortranFloat128Math.static STATIC INSTALL_WITH_TOOLCHAIN - ${sources} - ) - set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreadedDebug) - add_flang_library(FortranFloat128Math.static_dbg STATIC INSTALL_WITH_TOOLCHAIN - ${sources} - ) - add_dependencies(FortranFloat128Math FortranFloat128Math.static - FortranFloat128Math.static_dbg - ) +if (FLANG_RUNTIME_F128_MATH_LIB) + if (${FLANG_RUNTIME_F128_MATH_LIB} STREQUAL "libquadmath") + check_include_file(quadmath.h FOUND_QUADMATH_HEADER) + if(FOUND_QUADMATH_HEADER) + add_compile_definitions(HAS_QUADMATHLIB) + else() + message(FATAL_ERROR + "FLANG_RUNTIME_F128_MATH_LIB setting requires quadmath.h " + "to be available: ${FLANG_RUNTIME_F128_MATH_LIB}" + ) + endif() + else() + message(FATAL_ERROR + "Unsupported third-party library for Fortran F128 math runtime: " + "${FLANG_RUNTIME_F128_MATH_LIB}" + ) + endif() + + add_flang_library(FortranFloat128Math STATIC INSTALL_WITH_TOOLCHAIN + ${sources}) + + if (DEFINED MSVC) + set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreaded) + add_flang_library(FortranFloat128Math.static STATIC INSTALL_WITH_TOOLCHAIN + ${sources} + ) + set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreadedDebug) + add_flang_library(FortranFloat128Math.static_dbg STATIC INSTALL_WITH_TOOLCHAIN + ${sources} + ) + add_dependencies(FortranFloat128Math FortranFloat128Math.static + FortranFloat128Math.static_dbg + ) + endif() +elseif (HAVE_LDBL_MANT_DIG_113) + # We can use 'long double' versions from libc. + check_library_exists(m sinl "" FOUND_LIBM) + if (FOUND_LIBM) + target_compile_definitions(FortranFloat128MathILib INTERFACE + HAS_LIBM + ) + target_sources(FortranFloat128MathILib INTERFACE ${sources}) + else() + message(FATAL_ERROR "FortranRuntime cannot build without libm") + endif() +else() + # We can use '__float128' version from libc, if it has them. + check_library_exists(m sinf128 "" FOUND_LIBMF128) + if (FOUND_LIBMF128) + target_compile_definitions(FortranFloat128MathILib INTERFACE + HAS_LIBMF128 + ) + # Enable this, when math-entries.h and complex-math.h is ready. + # target_sources(FortranFloat128MathILib INTERFACE ${sources}) + endif() endif() diff --git a/flang/runtime/Float128Math/cabs.cpp b/flang/runtime/Float128Math/cabs.cpp deleted file mode 100644 index 3b8c9d17003c..000000000000 --- a/flang/runtime/Float128Math/cabs.cpp +++ /dev/null @@ -1,24 +0,0 @@ -//===-- runtime/Float128Math/cabs.cpp -------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "math-entries.h" - -namespace Fortran::runtime { -extern "C" { -#if 0 -// FIXME: temporarily disabled. Need to add pure C entry point -// using C _Complex ABI. -#if LDBL_MANT_DIG == 113 || HAS_FLOAT128 -// NOTE: Flang calls the runtime APIs using C _Complex ABI -CppTypeFor RTDEF(CAbsF128)(CFloat128ComplexType x) { - return CAbs::invoke(x); -} -#endif -#endif -} // extern "C" -} // namespace Fortran::runtime diff --git a/flang/runtime/Float128Math/complex-math.c b/flang/runtime/Float128Math/complex-math.c new file mode 100644 index 000000000000..d0180c63a0d7 --- /dev/null +++ b/flang/runtime/Float128Math/complex-math.c @@ -0,0 +1,55 @@ +/*===-- runtime/Float128Math/complex-math.c -------------------------*- C -*-=== + * + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + * ===-----------------------------------------------------------------------=== + */ + +#include "complex-math.h" + +#if LDBL_MANT_DIG == 113 || HAS_FLOAT128 + +CFloat128Type RTDEF(CAbsF128)(CFloat128ComplexType x) { return CAbs(x); } +CFloat128ComplexType RTDEF(CAcosF128)(CFloat128ComplexType x) { + return CAcos(x); +} +CFloat128ComplexType RTDEF(CAcoshF128)(CFloat128ComplexType x) { + return CAcosh(x); +} +CFloat128ComplexType RTDEF(CAsinF128)(CFloat128ComplexType x) { + return CAsin(x); +} +CFloat128ComplexType RTDEF(CAsinhF128)(CFloat128ComplexType x) { + return CAsinh(x); +} +CFloat128ComplexType RTDEF(CAtanF128)(CFloat128ComplexType x) { + return CAtan(x); +} +CFloat128ComplexType RTDEF(CAtanhF128)(CFloat128ComplexType x) { + return CAtanh(x); +} +CFloat128ComplexType RTDEF(CCosF128)(CFloat128ComplexType x) { return CCos(x); } +CFloat128ComplexType RTDEF(CCoshF128)(CFloat128ComplexType x) { + return CCosh(x); +} +CFloat128ComplexType RTDEF(CExpF128)(CFloat128ComplexType x) { return CExp(x); } +CFloat128ComplexType RTDEF(CLogF128)(CFloat128ComplexType x) { return CLog(x); } +CFloat128ComplexType RTDEF(CPowF128)( + CFloat128ComplexType x, CFloat128ComplexType p) { + return CPow(x, p); +} +CFloat128ComplexType RTDEF(CSinF128)(CFloat128ComplexType x) { return CSin(x); } +CFloat128ComplexType RTDEF(CSinhF128)(CFloat128ComplexType x) { + return CSinh(x); +} +CFloat128ComplexType RTDEF(CSqrtF128)(CFloat128ComplexType x) { + return CSqrt(x); +} +CFloat128ComplexType RTDEF(CTanF128)(CFloat128ComplexType x) { return CTan(x); } +CFloat128ComplexType RTDEF(CTanhF128)(CFloat128ComplexType x) { + return CTanh(x); +} + +#endif // LDBL_MANT_DIG == 113 || HAS_FLOAT128 diff --git a/flang/runtime/Float128Math/complex-math.h b/flang/runtime/Float128Math/complex-math.h new file mode 100644 index 000000000000..81dd53a175d1 --- /dev/null +++ b/flang/runtime/Float128Math/complex-math.h @@ -0,0 +1,62 @@ +/*===-- runtime/Float128Math/complex-math.h -------------------------*- C -*-=== + * + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + *===----------------------------------------------------------------------===*/ + +#ifndef FORTRAN_RUNTIME_FLOAT128MATH_COMPLEX_MATH_H_ +#define FORTRAN_RUNTIME_FLOAT128MATH_COMPLEX_MATH_H_ + +#include "flang/Common/float128.h" +#include "flang/Runtime/entry-names.h" + +#if HAS_QUADMATHLIB +#include "quadmath.h" +#define CAbs(x) cabsq(x) +#define CAcos(x) cacosq(x) +#define CAcosh(x) cacoshq(x) +#define CAsin(x) casinq(x) +#define CAsinh(x) casinhq(x) +#define CAtan(x) catanq(x) +#define CAtanh(x) catanhq(x) +#define CCos(x) ccosq(x) +#define CCosh(x) ccoshq(x) +#define CExp(x) cexpq(x) +#define CLog(x) clogq(x) +#define CPow(x, p) cpowq(x, p) +#define CSin(x) csinq(x) +#define CSinh(x) csinhq(x) +#define CSqrt(x) csqrtq(x) +#define CTan(x) ctanq(x) +#define CTanh(x) ctanhq(x) +#elif LDBL_MANT_DIG == 113 +/* Use 'long double' versions of libm functions. */ +#include + +#define CAbs(x) cabsl(x) +#define CAcos(x) cacosl(x) +#define CAcosh(x) cacoshl(x) +#define CAsin(x) casinl(x) +#define CAsinh(x) casinhl(x) +#define CAtan(x) catanl(x) +#define CAtanh(x) catanhl(x) +#define CCos(x) ccosl(x) +#define CCosh(x) ccoshl(x) +#define CExp(x) cexpl(x) +#define CLog(x) clogl(x) +#define CPow(x, p) cpowl(x, p) +#define CSin(x) csinl(x) +#define CSinh(x) csinhl(x) +#define CSqrt(x) csqrtl(x) +#define CTan(x) ctanl(x) +#define CTanh(x) ctanhl(x) +#elif HAS_LIBMF128 +/* We can use __float128 versions of libm functions. + * __STDC_WANT_IEC_60559_TYPES_EXT__ needs to be defined + * before including math.h to enable the *f128 prototypes. */ +#error "Float128Math build with glibc>=2.26 is unsupported yet" +#endif + +#endif /* FORTRAN_RUNTIME_FLOAT128MATH_COMPLEX_MATH_H_ */ diff --git a/flang/runtime/Float128Math/exponent.cpp b/flang/runtime/Float128Math/exponent.cpp index c0e43c0ee8d3..1be1dd0d0ac8 100644 --- a/flang/runtime/Float128Math/exponent.cpp +++ b/flang/runtime/Float128Math/exponent.cpp @@ -12,7 +12,7 @@ namespace Fortran::runtime { extern "C" { -#if LDBL_MANT_DIG != 113 && HAS_FLOAT128 +#if LDBL_MANT_DIG == 113 || HAS_FLOAT128 // EXPONENT (16.9.75) CppTypeFor RTDEF(Exponent16_4)(F128Type x) { return Exponent>(x); diff --git a/flang/runtime/Float128Math/fraction.cpp b/flang/runtime/Float128Math/fraction.cpp index 8de6d3c7ff6c..8c9889b7f687 100644 --- a/flang/runtime/Float128Math/fraction.cpp +++ b/flang/runtime/Float128Math/fraction.cpp @@ -12,7 +12,7 @@ namespace Fortran::runtime { extern "C" { -#if LDBL_MANT_DIG != 113 && HAS_FLOAT128 +#if LDBL_MANT_DIG == 113 || HAS_FLOAT128 // FRACTION (16.9.80) F128Type RTDEF(Fraction16)(F128Type x) { return Fraction(x); } #endif diff --git a/flang/runtime/Float128Math/math-entries.h b/flang/runtime/Float128Math/math-entries.h index ad3f6aa18aa9..1eab7c86f2ed 100644 --- a/flang/runtime/Float128Math/math-entries.h +++ b/flang/runtime/Float128Math/math-entries.h @@ -106,11 +106,59 @@ DEFINE_FALLBACK_F128(Y0) DEFINE_FALLBACK_F128(Y1) DEFINE_FALLBACK_F128(Yn) -#if HAS_LIBM -#include +#if HAS_QUADMATHLIB +// Define wrapper callers for libquadmath. +#include "quadmath.h" +DEFINE_SIMPLE_ALIAS(Abs, fabsq) +DEFINE_SIMPLE_ALIAS(Acos, acosq) +DEFINE_SIMPLE_ALIAS(Acosh, acoshq) +DEFINE_SIMPLE_ALIAS(Asin, asinq) +DEFINE_SIMPLE_ALIAS(Asinh, asinhq) +DEFINE_SIMPLE_ALIAS(Atan, atanq) +DEFINE_SIMPLE_ALIAS(Atan2, atan2q) +DEFINE_SIMPLE_ALIAS(Atanh, atanhq) +DEFINE_SIMPLE_ALIAS(Ceil, ceilq) +DEFINE_SIMPLE_ALIAS(Cos, cosq) +DEFINE_SIMPLE_ALIAS(Cosh, coshq) +DEFINE_SIMPLE_ALIAS(Erf, erfq) +DEFINE_SIMPLE_ALIAS(Erfc, erfcq) +DEFINE_SIMPLE_ALIAS(Exp, expq) +DEFINE_SIMPLE_ALIAS(Floor, floorq) +DEFINE_SIMPLE_ALIAS(Frexp, frexpq) +DEFINE_SIMPLE_ALIAS(Hypot, hypotq) +DEFINE_SIMPLE_ALIAS(Ilogb, ilogbq) +DEFINE_SIMPLE_ALIAS(Isinf, isinfq) +DEFINE_SIMPLE_ALIAS(Isnan, isnanq) +DEFINE_SIMPLE_ALIAS(J0, j0q) +DEFINE_SIMPLE_ALIAS(J1, j1q) +DEFINE_SIMPLE_ALIAS(Jn, jnq) +DEFINE_SIMPLE_ALIAS(Ldexp, ldexpq) +DEFINE_SIMPLE_ALIAS(Lgamma, lgammaq) +DEFINE_SIMPLE_ALIAS(Llround, llroundq) +DEFINE_SIMPLE_ALIAS(Log, logq) +DEFINE_SIMPLE_ALIAS(Log10, log10q) +DEFINE_SIMPLE_ALIAS(Lround, lroundq) +DEFINE_SIMPLE_ALIAS(Nextafter, nextafterq) +DEFINE_SIMPLE_ALIAS(Pow, powq) +DEFINE_SIMPLE_ALIAS(Round, roundq) +DEFINE_SIMPLE_ALIAS(Sin, sinq) +DEFINE_SIMPLE_ALIAS(Sinh, sinhq) +DEFINE_SIMPLE_ALIAS(Sqrt, sqrtq) +DEFINE_SIMPLE_ALIAS(Tan, tanq) +DEFINE_SIMPLE_ALIAS(Tanh, tanhq) +DEFINE_SIMPLE_ALIAS(Tgamma, tgammaq) +DEFINE_SIMPLE_ALIAS(Trunc, truncq) +DEFINE_SIMPLE_ALIAS(Y0, y0q) +DEFINE_SIMPLE_ALIAS(Y1, y1q) +DEFINE_SIMPLE_ALIAS(Yn, ynq) +// Use cmath INFINITY/NAN definition. Rely on C implicit conversions. +#define F128_RT_INFINITY (INFINITY) +#define F128_RT_QNAN (NAN) +#elif LDBL_MANT_DIG == 113 // Define wrapper callers for libm. -#if LDBL_MANT_DIG == 113 +#include + // Use STD math functions. They provide IEEE-754 128-bit float // support either via 'long double' or __float128. // The Bessel's functions are not present in STD namespace. @@ -122,9 +170,6 @@ DEFINE_SIMPLE_ALIAS(Asinh, std::asinh) DEFINE_SIMPLE_ALIAS(Atan, std::atan) DEFINE_SIMPLE_ALIAS(Atan2, std::atan2) DEFINE_SIMPLE_ALIAS(Atanh, std::atanh) -// TODO: enable complex abs, when ABI adjustment for complex -// data type is resolved. -// DEFINE_SIMPLE_ALIAS(CAbs, std::abs) DEFINE_SIMPLE_ALIAS(Ceil, std::ceil) DEFINE_SIMPLE_ALIAS(Cos, std::cos) DEFINE_SIMPLE_ALIAS(Cosh, std::cosh) @@ -165,70 +210,11 @@ DEFINE_SIMPLE_ALIAS(Yn, ynl) (std::numeric_limits>::infinity()) #define F128_RT_QNAN \ (std::numeric_limits>::quiet_NaN()) -#else // LDBL_MANT_DIG != 113 -#if !HAS_LIBMF128 -// glibc >=2.26 seems to have complete support for __float128 -// versions of the math functions. -#error "FLANG_RUNTIME_F128_MATH_LIB=libm build requires libm >=2.26" -#endif - +#elif HAS_LIBMF128 // We can use __float128 versions of libm functions. // __STDC_WANT_IEC_60559_TYPES_EXT__ needs to be defined // before including cmath to enable the *f128 prototypes. -// TODO: this needs to be enabled separately, especially -// for complex data types that require C++ complex to C complex -// adjustment to match the ABIs. -#error "Unsupported FLANG_RUNTIME_F128_MATH_LIB=libm build" -#endif // LDBL_MANT_DIG != 113 -#elif HAS_QUADMATHLIB -// Define wrapper callers for libquadmath. -#include "quadmath.h" -DEFINE_SIMPLE_ALIAS(Abs, fabsq) -DEFINE_SIMPLE_ALIAS(Acos, acosq) -DEFINE_SIMPLE_ALIAS(Acosh, acoshq) -DEFINE_SIMPLE_ALIAS(Asin, asinq) -DEFINE_SIMPLE_ALIAS(Asinh, asinhq) -DEFINE_SIMPLE_ALIAS(Atan, atanq) -DEFINE_SIMPLE_ALIAS(Atan2, atan2q) -DEFINE_SIMPLE_ALIAS(Atanh, atanhq) -DEFINE_SIMPLE_ALIAS(Ceil, ceilq) -DEFINE_SIMPLE_ALIAS(Cos, cosq) -DEFINE_SIMPLE_ALIAS(Cosh, coshq) -DEFINE_SIMPLE_ALIAS(Erf, erfq) -DEFINE_SIMPLE_ALIAS(Erfc, erfcq) -DEFINE_SIMPLE_ALIAS(Exp, expq) -DEFINE_SIMPLE_ALIAS(Floor, floorq) -DEFINE_SIMPLE_ALIAS(Frexp, frexpq) -DEFINE_SIMPLE_ALIAS(Hypot, hypotq) -DEFINE_SIMPLE_ALIAS(Ilogb, ilogbq) -DEFINE_SIMPLE_ALIAS(Isinf, isinfq) -DEFINE_SIMPLE_ALIAS(Isnan, isnanq) -DEFINE_SIMPLE_ALIAS(J0, j0q) -DEFINE_SIMPLE_ALIAS(J1, j1q) -DEFINE_SIMPLE_ALIAS(Jn, jnq) -DEFINE_SIMPLE_ALIAS(Ldexp, ldexpq) -DEFINE_SIMPLE_ALIAS(Lgamma, lgammaq) -DEFINE_SIMPLE_ALIAS(Llround, llroundq) -DEFINE_SIMPLE_ALIAS(Log, logq) -DEFINE_SIMPLE_ALIAS(Log10, log10q) -DEFINE_SIMPLE_ALIAS(Lround, lroundq) -DEFINE_SIMPLE_ALIAS(Nextafter, nextafterq) -DEFINE_SIMPLE_ALIAS(Pow, powq) -DEFINE_SIMPLE_ALIAS(Round, roundq) -DEFINE_SIMPLE_ALIAS(Sin, sinq) -DEFINE_SIMPLE_ALIAS(Sinh, sinhq) -DEFINE_SIMPLE_ALIAS(Sqrt, sqrtq) -DEFINE_SIMPLE_ALIAS(Tan, tanq) -DEFINE_SIMPLE_ALIAS(Tanh, tanhq) -DEFINE_SIMPLE_ALIAS(Tgamma, tgammaq) -DEFINE_SIMPLE_ALIAS(Trunc, truncq) -DEFINE_SIMPLE_ALIAS(Y0, y0q) -DEFINE_SIMPLE_ALIAS(Y1, y1q) -DEFINE_SIMPLE_ALIAS(Yn, ynq) - -// Use cmath INFINITY/NAN definition. Rely on C implicit conversions. -#define F128_RT_INFINITY (INFINITY) -#define F128_RT_QNAN (NAN) +#error "Float128Math build with glibc>=2.26 is unsupported yet" #endif } // namespace Fortran::runtime diff --git a/flang/runtime/Float128Math/mod-real.cpp b/flang/runtime/Float128Math/mod-real.cpp index 9cc2926e45d5..42e6ce76e2fa 100644 --- a/flang/runtime/Float128Math/mod-real.cpp +++ b/flang/runtime/Float128Math/mod-real.cpp @@ -12,7 +12,7 @@ namespace Fortran::runtime { extern "C" { -#if LDBL_MANT_DIG != 113 && HAS_FLOAT128 +#if LDBL_MANT_DIG == 113 || HAS_FLOAT128 // MOD (16.9.135) F128Type RTDEF(ModReal16)( F128Type x, F128Type p, const char *sourceFile, int sourceLine) { diff --git a/flang/runtime/Float128Math/modulo-real.cpp b/flang/runtime/Float128Math/modulo-real.cpp index b25797fd8f41..13000aba8c83 100644 --- a/flang/runtime/Float128Math/modulo-real.cpp +++ b/flang/runtime/Float128Math/modulo-real.cpp @@ -12,7 +12,7 @@ namespace Fortran::runtime { extern "C" { -#if LDBL_MANT_DIG != 113 && HAS_FLOAT128 +#if LDBL_MANT_DIG == 113 || HAS_FLOAT128 // MODULO (16.9.136) F128Type RTDEF(ModuloReal16)( F128Type x, F128Type p, const char *sourceFile, int sourceLine) { diff --git a/flang/runtime/Float128Math/nearest.cpp b/flang/runtime/Float128Math/nearest.cpp index fd990532e522..148ac4ef8391 100644 --- a/flang/runtime/Float128Math/nearest.cpp +++ b/flang/runtime/Float128Math/nearest.cpp @@ -11,7 +11,7 @@ namespace Fortran::runtime { extern "C" { -#if LDBL_MANT_DIG != 113 && HAS_FLOAT128 +#if LDBL_MANT_DIG == 113 || HAS_FLOAT128 CppTypeFor RTDEF(Nearest16)( CppTypeFor x, bool positive) { return Nextafter::invoke( diff --git a/flang/runtime/Float128Math/rrspacing.cpp b/flang/runtime/Float128Math/rrspacing.cpp index f2187f42313a..feddac418eec 100644 --- a/flang/runtime/Float128Math/rrspacing.cpp +++ b/flang/runtime/Float128Math/rrspacing.cpp @@ -12,7 +12,7 @@ namespace Fortran::runtime { extern "C" { -#if LDBL_MANT_DIG != 113 && HAS_FLOAT128 +#if LDBL_MANT_DIG == 113 || HAS_FLOAT128 // FRACTION (16.9.80) F128Type RTDEF(RRSpacing16)(F128Type x) { return RRSpacing<113>(x); } #endif diff --git a/flang/runtime/Float128Math/scale.cpp b/flang/runtime/Float128Math/scale.cpp index d6b843150e72..0be958bd9f2a 100644 --- a/flang/runtime/Float128Math/scale.cpp +++ b/flang/runtime/Float128Math/scale.cpp @@ -13,7 +13,7 @@ namespace Fortran::runtime { extern "C" { -#if LDBL_MANT_DIG != 113 && HAS_FLOAT128 +#if LDBL_MANT_DIG == 113 || HAS_FLOAT128 F128Type RTDEF(Scale16)(F128Type x, std::int64_t p) { auto ip{static_cast(p)}; if (ip != p) { diff --git a/flang/runtime/Float128Math/set-exponent.cpp b/flang/runtime/Float128Math/set-exponent.cpp index 0f942d238b8f..99c34af7962b 100644 --- a/flang/runtime/Float128Math/set-exponent.cpp +++ b/flang/runtime/Float128Math/set-exponent.cpp @@ -12,7 +12,7 @@ namespace Fortran::runtime { extern "C" { -#if LDBL_MANT_DIG != 113 && HAS_FLOAT128 +#if LDBL_MANT_DIG == 113 || HAS_FLOAT128 // SET_EXPONENT (16.9.171) F128Type RTDEF(SetExponent16)(F128Type x, std::int64_t p) { return SetExponent(x, p); diff --git a/flang/runtime/Float128Math/spacing.cpp b/flang/runtime/Float128Math/spacing.cpp index d00e74644f8a..a86c0b30e567 100644 --- a/flang/runtime/Float128Math/spacing.cpp +++ b/flang/runtime/Float128Math/spacing.cpp @@ -12,7 +12,7 @@ namespace Fortran::runtime { extern "C" { -#if LDBL_MANT_DIG != 113 && HAS_FLOAT128 +#if LDBL_MANT_DIG == 113 || HAS_FLOAT128 // SPACING (16.9.180) F128Type RTDEF(Spacing16)(F128Type x) { return Spacing<113>(x); } #endif diff --git a/flang/runtime/numeric.cpp b/flang/runtime/numeric.cpp index d61f32e1d5b8..abd3e500029f 100644 --- a/flang/runtime/numeric.cpp +++ b/flang/runtime/numeric.cpp @@ -324,16 +324,6 @@ CppTypeFor RTDEF(Exponent10_8)( CppTypeFor x) { return Exponent>(x); } -#elif LDBL_MANT_DIG == 113 -// The __float128 implementation resides in FortranFloat128Math library. -CppTypeFor RTDEF(Exponent16_4)( - CppTypeFor x) { - return Exponent>(x); -} -CppTypeFor RTDEF(Exponent16_8)( - CppTypeFor x) { - return Exponent>(x); -} #endif CppTypeFor RTDEF(Floor4_1)( @@ -441,12 +431,6 @@ CppTypeFor RTDEF(Fraction10)( CppTypeFor x) { return Fraction(x); } -#elif LDBL_MANT_DIG == 113 -// The __float128 implementation resides in FortranFloat128Math library. -CppTypeFor RTDEF(Fraction16)( - CppTypeFor x) { - return Fraction(x); -} #endif bool RTDEF(IsFinite4)(CppTypeFor x) { @@ -529,13 +513,6 @@ CppTypeFor RTDEF(ModReal10)( const char *sourceFile, int sourceLine) { return RealMod(x, p, sourceFile, sourceLine); } -#elif LDBL_MANT_DIG == 113 -// The __float128 implementation resides in FortranFloat128Math library. -CppTypeFor RTDEF(ModReal16)( - CppTypeFor x, CppTypeFor p, - const char *sourceFile, int sourceLine) { - return RealMod(x, p, sourceFile, sourceLine); -} #endif CppTypeFor RTDEF(ModuloInteger1)( @@ -586,13 +563,6 @@ CppTypeFor RTDEF(ModuloReal10)( const char *sourceFile, int sourceLine) { return RealMod(x, p, sourceFile, sourceLine); } -#elif LDBL_MANT_DIG == 113 -// The __float128 implementation resides in FortranFloat128Math library. -CppTypeFor RTDEF(ModuloReal16)( - CppTypeFor x, CppTypeFor p, - const char *sourceFile, int sourceLine) { - return RealMod(x, p, sourceFile, sourceLine); -} #endif CppTypeFor RTDEF(Nearest4)( @@ -608,12 +578,6 @@ CppTypeFor RTDEF(Nearest10)( CppTypeFor x, bool positive) { return Nearest<64>(x, positive); } -#elif LDBL_MANT_DIG == 113 -// The __float128 implementation resides in FortranFloat128Math library. -CppTypeFor RTDEF(Nearest16)( - CppTypeFor x, bool positive) { - return Nearest<113>(x, positive); -} #endif CppTypeFor RTDEF(Nint4_1)( @@ -721,12 +685,6 @@ CppTypeFor RTDEF(RRSpacing10)( CppTypeFor x) { return RRSpacing<64>(x); } -#elif LDBL_MANT_DIG == 113 -// The __float128 implementation resides in FortranFloat128Math library. -CppTypeFor RTDEF(RRSpacing16)( - CppTypeFor x) { - return RRSpacing<113>(x); -} #endif CppTypeFor RTDEF(SetExponent4)( @@ -742,12 +700,6 @@ CppTypeFor RTDEF(SetExponent10)( CppTypeFor x, std::int64_t p) { return SetExponent(x, p); } -#elif LDBL_MANT_DIG == 113 -// The __float128 implementation resides in FortranFloat128Math library. -CppTypeFor RTDEF(SetExponent16)( - CppTypeFor x, std::int64_t p) { - return SetExponent(x, p); -} #endif CppTypeFor RTDEF(Scale4)( @@ -763,12 +715,6 @@ CppTypeFor RTDEF(Scale10)( CppTypeFor x, std::int64_t p) { return Scale(x, p); } -#elif LDBL_MANT_DIG == 113 -// The __float128 implementation resides in FortranFloat128Math library. -CppTypeFor RTDEF(Scale16)( - CppTypeFor x, std::int64_t p) { - return Scale(x, p); -} #endif // SELECTED_INT_KIND @@ -823,12 +769,6 @@ CppTypeFor RTDEF(Spacing10)( CppTypeFor x) { return Spacing<64>(x); } -#elif LDBL_MANT_DIG == 113 -// The __float128 implementation resides in FortranFloat128Math library. -CppTypeFor RTDEF(Spacing16)( - CppTypeFor x) { - return Spacing<113>(x); -} #endif CppTypeFor RTDEF(FPow4i)( diff --git a/flang/test/Lower/Intrinsics/acos_complex16.f90 b/flang/test/Lower/Intrinsics/acos_complex16.f90 new file mode 100644 index 000000000000..5b3545e472e0 --- /dev/null +++ b/flang/test/Lower/Intrinsics/acos_complex16.f90 @@ -0,0 +1,8 @@ +! RUN: bbc -emit-fir %s -o - | FileCheck %s +! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s + +! CHECK: fir.call @_FortranACAcosF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16> + complex(16) :: a, b + b = acos(a) +end diff --git a/flang/test/Lower/Intrinsics/acosh_complex16.f90 b/flang/test/Lower/Intrinsics/acosh_complex16.f90 new file mode 100644 index 000000000000..a80238d72e45 --- /dev/null +++ b/flang/test/Lower/Intrinsics/acosh_complex16.f90 @@ -0,0 +1,8 @@ +! RUN: bbc -emit-fir %s -o - | FileCheck %s +! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s + +! CHECK: fir.call @_FortranACAcoshF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16> + complex(16) :: a, b + b = acosh(a) +end diff --git a/flang/test/Lower/Intrinsics/asin_complex16.f90 b/flang/test/Lower/Intrinsics/asin_complex16.f90 new file mode 100644 index 000000000000..982bf6f21b16 --- /dev/null +++ b/flang/test/Lower/Intrinsics/asin_complex16.f90 @@ -0,0 +1,8 @@ +! RUN: bbc -emit-fir %s -o - | FileCheck %s +! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s + +! CHECK: fir.call @_FortranACAsinF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16> + complex(16) :: a, b + b = asin(a) +end diff --git a/flang/test/Lower/Intrinsics/asinh_complex16.f90 b/flang/test/Lower/Intrinsics/asinh_complex16.f90 new file mode 100644 index 000000000000..2d658a68f8a2 --- /dev/null +++ b/flang/test/Lower/Intrinsics/asinh_complex16.f90 @@ -0,0 +1,8 @@ +! RUN: bbc -emit-fir %s -o - | FileCheck %s +! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s + +! CHECK: fir.call @_FortranACAsinhF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16> + complex(16) :: a, b + b = asinh(a) +end diff --git a/flang/test/Lower/Intrinsics/atan_complex16.f90 b/flang/test/Lower/Intrinsics/atan_complex16.f90 new file mode 100644 index 000000000000..315928d88568 --- /dev/null +++ b/flang/test/Lower/Intrinsics/atan_complex16.f90 @@ -0,0 +1,8 @@ +! RUN: bbc -emit-fir %s -o - | FileCheck %s +! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s + +! CHECK: fir.call @_FortranACAtanF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16> + complex(16) :: a, b + b = atan(a) +end diff --git a/flang/test/Lower/Intrinsics/atanh_complex16.f90 b/flang/test/Lower/Intrinsics/atanh_complex16.f90 new file mode 100644 index 000000000000..0d9f798ea4a2 --- /dev/null +++ b/flang/test/Lower/Intrinsics/atanh_complex16.f90 @@ -0,0 +1,8 @@ +! RUN: bbc -emit-fir %s -o - | FileCheck %s +! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s + +! CHECK: fir.call @_FortranACAtanhF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16> + complex(16) :: a, b + b = atanh(a) +end diff --git a/flang/test/Lower/Intrinsics/cos_complex16.f90 b/flang/test/Lower/Intrinsics/cos_complex16.f90 new file mode 100644 index 000000000000..79b89ce057e1 --- /dev/null +++ b/flang/test/Lower/Intrinsics/cos_complex16.f90 @@ -0,0 +1,8 @@ +! RUN: bbc -emit-fir %s -o - | FileCheck %s +! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s + +! CHECK: fir.call @_FortranACCosF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16> + complex(16) :: a, b + b = cos(a) +end diff --git a/flang/test/Lower/Intrinsics/cosh_complex16.f90 b/flang/test/Lower/Intrinsics/cosh_complex16.f90 new file mode 100644 index 000000000000..dc1723b45820 --- /dev/null +++ b/flang/test/Lower/Intrinsics/cosh_complex16.f90 @@ -0,0 +1,8 @@ +! RUN: bbc -emit-fir %s -o - | FileCheck %s +! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s + +! CHECK: fir.call @_FortranACCoshF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16> + complex(16) :: a, b + b = cosh(a) +end diff --git a/flang/test/Lower/Intrinsics/exp_complex16.f90 b/flang/test/Lower/Intrinsics/exp_complex16.f90 new file mode 100644 index 000000000000..972285c654bf --- /dev/null +++ b/flang/test/Lower/Intrinsics/exp_complex16.f90 @@ -0,0 +1,8 @@ +! RUN: bbc -emit-fir %s -o - | FileCheck %s +! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s + +! CHECK: fir.call @_FortranACExpF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16> + complex(16) :: a, b + b = exp(a) +end diff --git a/flang/test/Lower/Intrinsics/log_complex16.f90 b/flang/test/Lower/Intrinsics/log_complex16.f90 new file mode 100644 index 000000000000..d5d0eb388ca2 --- /dev/null +++ b/flang/test/Lower/Intrinsics/log_complex16.f90 @@ -0,0 +1,8 @@ +! RUN: bbc -emit-fir %s -o - | FileCheck %s +! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s + +! CHECK: fir.call @_FortranACLogF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16> + complex(16) :: a, b + b = log(a) +end diff --git a/flang/test/Lower/Intrinsics/missing-math-runtime.f90 b/flang/test/Lower/Intrinsics/missing-math-runtime.f90 deleted file mode 100644 index 699678fcf2bc..000000000000 --- a/flang/test/Lower/Intrinsics/missing-math-runtime.f90 +++ /dev/null @@ -1,12 +0,0 @@ -! If the compiler is built without 128-bit float math -! support, an appropriate error message is emitted. -! UNSUPPORTED: flang-supports-f128-math -! RUN: bbc -emit-fir %s -o /dev/null >%t 2>&1 || echo -! RUN: FileCheck %s --input-file=%t - - complex(16) :: a - real(16) :: b -! CHECK: compiler is built without support for 'ABS(COMPLEX(KIND=16))' - b = abs(a) -end - diff --git a/flang/test/Lower/Intrinsics/pow_complex16.f90 b/flang/test/Lower/Intrinsics/pow_complex16.f90 new file mode 100644 index 000000000000..db6b207aea90 --- /dev/null +++ b/flang/test/Lower/Intrinsics/pow_complex16.f90 @@ -0,0 +1,8 @@ +! RUN: bbc -emit-fir %s -o - | FileCheck %s +! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s + +! CHECK: fir.call @_FortranACPowF128({{.*}}){{.*}}: (!fir.complex<16>, !fir.complex<16>) -> !fir.complex<16> + complex(16) :: a, b + b = a ** b +end diff --git a/flang/test/Lower/Intrinsics/sin_complex16.f90 b/flang/test/Lower/Intrinsics/sin_complex16.f90 new file mode 100644 index 000000000000..5114501e48da --- /dev/null +++ b/flang/test/Lower/Intrinsics/sin_complex16.f90 @@ -0,0 +1,8 @@ +! RUN: bbc -emit-fir %s -o - | FileCheck %s +! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s + +! CHECK: fir.call @_FortranACSinF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16> + complex(16) :: a, b + b = sin(a) +end diff --git a/flang/test/Lower/Intrinsics/sinh_complex16.f90 b/flang/test/Lower/Intrinsics/sinh_complex16.f90 new file mode 100644 index 000000000000..5baf9ea8052a --- /dev/null +++ b/flang/test/Lower/Intrinsics/sinh_complex16.f90 @@ -0,0 +1,8 @@ +! RUN: bbc -emit-fir %s -o - | FileCheck %s +! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s + +! CHECK: fir.call @_FortranACSinhF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16> + complex(16) :: a, b + b = sinh(a) +end diff --git a/flang/test/Lower/Intrinsics/sqrt_complex16.f90 b/flang/test/Lower/Intrinsics/sqrt_complex16.f90 new file mode 100644 index 000000000000..75ffa22a6e2e --- /dev/null +++ b/flang/test/Lower/Intrinsics/sqrt_complex16.f90 @@ -0,0 +1,8 @@ +! RUN: bbc -emit-fir %s -o - | FileCheck %s +! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s + +! CHECK: fir.call @_FortranACSqrtF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16> + complex(16) :: a, b + b = sqrt(a) +end diff --git a/flang/test/Lower/Intrinsics/tan_complex16.f90 b/flang/test/Lower/Intrinsics/tan_complex16.f90 new file mode 100644 index 000000000000..7217145f81b0 --- /dev/null +++ b/flang/test/Lower/Intrinsics/tan_complex16.f90 @@ -0,0 +1,8 @@ +! RUN: bbc -emit-fir %s -o - | FileCheck %s +! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s + +! CHECK: fir.call @_FortranACTanF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16> + complex(16) :: a, b + b = tan(a) +end diff --git a/flang/test/Lower/Intrinsics/tanh_complex16.f90 b/flang/test/Lower/Intrinsics/tanh_complex16.f90 new file mode 100644 index 000000000000..1965094c5bce --- /dev/null +++ b/flang/test/Lower/Intrinsics/tanh_complex16.f90 @@ -0,0 +1,8 @@ +! RUN: bbc -emit-fir %s -o - | FileCheck %s +! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s + +! CHECK: fir.call @_FortranACTanhF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16> + complex(16) :: a, b + b = tanh(a) +end -- GitLab From bf631c63d01057321c070520a56a150ede32e47d Mon Sep 17 00:00:00 2001 From: Changpeng Fang Date: Tue, 5 Mar 2024 13:49:34 -0800 Subject: [PATCH 211/929] AMDGPU: Copy SubtargetPredicate from pseudo for DSDIR_Real (#84057) --- llvm/lib/Target/AMDGPU/DSDIRInstructions.td | 3 +++ 1 file changed, 3 insertions(+) diff --git a/llvm/lib/Target/AMDGPU/DSDIRInstructions.td b/llvm/lib/Target/AMDGPU/DSDIRInstructions.td index 3ef765f93395..f4f02d2cebfd 100644 --- a/llvm/lib/Target/AMDGPU/DSDIRInstructions.td +++ b/llvm/lib/Target/AMDGPU/DSDIRInstructions.td @@ -115,6 +115,9 @@ class DSDIR_Real : SIMCInstr { let isPseudo = 0; let isCodeGenOnly = 0; + + // copy SubtargetPredicate from pseudo. + let SubtargetPredicate = lds.SubtargetPredicate; } //===----------------------------------------------------------------------===// -- GitLab From 201572e34b2cb0861a60e00fd7dc2965e6af1587 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Tue, 5 Mar 2024 13:55:29 -0800 Subject: [PATCH 212/929] [AArch64] Implement -fno-plt for SelectionDAG/GlobalISel Clang sets the nonlazybind attribute for certain ObjC features. The AArch64 SelectionDAG implementation for non-intrinsic calls (46e36f0953aabb5e5cd00ed8d296d60f9f71b424) is behind a cl option. GCC implements -fno-plt for a few ELF targets. In Clang, -fno-plt also sets the nonlazybind attribute. For SelectionDAG, make the cl option not affect ELF so that non-intrinsic calls to a dso_preemptable function use GOT. Adjust AArch64TargetLowering::LowerCall to handle intrinsic calls. For FastISel, change `fastLowerCall` to bail out when a call is due to -fno-plt. For GlobalISel, handle non-intrinsic calls in CallLowering::lowerCall and intrinsic calls in AArch64CallLowering::lowerCall (where the target-independent CallLowering::lowerCall is not called). The GlobalISel test in `call-rv-marker.ll` is therefore updated. Note: the current -fno-plt -fpic implementation does not use GOT for a preemptable function. Link: #78275 Pull Request: https://github.com/llvm/llvm-project/pull/78890 --- llvm/lib/CodeGen/GlobalISel/CallLowering.cpp | 12 ++- llvm/lib/Target/AArch64/AArch64FastISel.cpp | 5 ++ .../Target/AArch64/AArch64ISelLowering.cpp | 9 ++- llvm/lib/Target/AArch64/AArch64Subtarget.cpp | 11 +-- .../AArch64/GISel/AArch64CallLowering.cpp | 11 ++- .../GISel/AArch64InstructionSelector.cpp | 16 +++- .../AArch64/GISel/AArch64LegalizerInfo.cpp | 3 + llvm/test/CodeGen/AArch64/call-rv-marker.ll | 36 +++++++-- llvm/test/CodeGen/AArch64/nonlazybind.ll | 81 +++++++++---------- 9 files changed, 116 insertions(+), 68 deletions(-) diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp index 77dc265d795d..3a37bf3cd7a8 100644 --- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp @@ -144,9 +144,15 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB, // Try looking through a bitcast from one function type to another. // Commonly happens with calls to objc_msgSend(). const Value *CalleeV = CB.getCalledOperand()->stripPointerCasts(); - if (const Function *F = dyn_cast(CalleeV)) - Info.Callee = MachineOperand::CreateGA(F, 0); - else if (isa(CalleeV) || isa(CalleeV)) { + if (const Function *F = dyn_cast(CalleeV)) { + if (F->hasFnAttribute(Attribute::NonLazyBind)) { + LLT Ty = getLLTForType(*F->getType(), DL); + Register Reg = MIRBuilder.buildGlobalValue(Ty, F).getReg(0); + Info.Callee = MachineOperand::CreateReg(Reg, false); + } else { + Info.Callee = MachineOperand::CreateGA(F, 0); + } + } else if (isa(CalleeV) || isa(CalleeV)) { // IR IFuncs and Aliases can't be forward declared (only defined), so the // callee must be in the same TU and therefore we can direct-call it without // worrying about it being out of range. diff --git a/llvm/lib/Target/AArch64/AArch64FastISel.cpp b/llvm/lib/Target/AArch64/AArch64FastISel.cpp index 49bcab588e52..87c94f10e2b1 100644 --- a/llvm/lib/Target/AArch64/AArch64FastISel.cpp +++ b/llvm/lib/Target/AArch64/AArch64FastISel.cpp @@ -3172,6 +3172,11 @@ bool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) { if (CM == CodeModel::Large && !Subtarget->isTargetMachO()) return false; + // ELF -fno-plt compiled intrinsic calls do not have the nonlazybind + // attribute. Check "RtLibUseGOT" instead. + if (MF->getFunction().getParent()->getRtLibUseGOT()) + return false; + // Let SDISel handle vararg functions. if (IsVarArg) return false; diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 8d6e92a00b5d..6061b4c1a076 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -8211,13 +8211,14 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, OpFlags); } } else if (auto *S = dyn_cast(Callee)) { - if (getTargetMachine().getCodeModel() == CodeModel::Large && - Subtarget->isTargetMachO()) { - const char *Sym = S->getSymbol(); + bool UseGot = (getTargetMachine().getCodeModel() == CodeModel::Large && + Subtarget->isTargetMachO()) || + MF.getFunction().getParent()->getRtLibUseGOT(); + const char *Sym = S->getSymbol(); + if (UseGot) { Callee = DAG.getTargetExternalSymbol(Sym, PtrVT, AArch64II::MO_GOT); Callee = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, Callee); } else { - const char *Sym = S->getSymbol(); Callee = DAG.getTargetExternalSymbol(Sym, PtrVT, 0); } } diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp index 2b01deb0cca1..2fa87f6afc7a 100644 --- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp +++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp @@ -43,10 +43,10 @@ static cl::opt UseAddressTopByteIgnored("aarch64-use-tbi", cl::desc("Assume that top byte of " "an address is ignored"), cl::init(false), cl::Hidden); -static cl::opt - UseNonLazyBind("aarch64-enable-nonlazybind", - cl::desc("Call nonlazybind functions via direct GOT load"), - cl::init(false), cl::Hidden); +static cl::opt MachOUseNonLazyBind( + "aarch64-macho-enable-nonlazybind", + cl::desc("Call nonlazybind functions via direct GOT load for Mach-O"), + cl::Hidden); static cl::opt UseAA("aarch64-use-aa", cl::init(true), cl::desc("Enable the use of AA during codegen.")); @@ -433,7 +433,8 @@ unsigned AArch64Subtarget::classifyGlobalFunctionReference( // NonLazyBind goes via GOT unless we know it's available locally. auto *F = dyn_cast(GV); - if (UseNonLazyBind && F && F->hasFnAttribute(Attribute::NonLazyBind) && + if ((!isTargetMachO() || MachOUseNonLazyBind) && F && + F->hasFnAttribute(Attribute::NonLazyBind) && !TM.shouldAssumeDSOLocal(*GV->getParent(), GV)) return AArch64II::MO_GOT; diff --git a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp index 26dbad713594..c4197ff73187 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp @@ -1301,8 +1301,17 @@ bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, !Subtarget.noBTIAtReturnTwice() && MF.getInfo()->branchTargetEnforcement()) Opc = AArch64::BLR_BTI; - else + else { + // For an intrinsic call (e.g. memset), use GOT if "RtLibUseGOT" (-fno-plt) + // is set. + if (Info.Callee.isSymbol() && F.getParent()->getRtLibUseGOT()) { + auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_GLOBAL_VALUE); + DstOp(getLLTForType(*F.getType(), DL)).addDefToMIB(MRI, MIB); + MIB.addExternalSymbol(Info.Callee.getSymbolName(), AArch64II::MO_GOT); + Info.Callee = MachineOperand::CreateReg(MIB.getReg(0), false); + } Opc = getCallOpcode(MF, Info.Callee.isReg(), false); + } auto MIB = MIRBuilder.buildInstrNoInsert(Opc); unsigned CalleeOpNo = 0; diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp index 77d6e7e93fb0..665288379239 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp @@ -2804,11 +2804,19 @@ bool AArch64InstructionSelector::select(MachineInstr &I) { } case TargetOpcode::G_GLOBAL_VALUE: { - auto GV = I.getOperand(1).getGlobal(); - if (GV->isThreadLocal()) - return selectTLSGlobalValue(I, MRI); + const GlobalValue *GV = nullptr; + unsigned OpFlags; + if (I.getOperand(1).isSymbol()) { + OpFlags = I.getOperand(1).getTargetFlags(); + // Currently only used by "RtLibUseGOT". + assert(OpFlags == AArch64II::MO_GOT); + } else { + GV = I.getOperand(1).getGlobal(); + if (GV->isThreadLocal()) + return selectTLSGlobalValue(I, MRI); + OpFlags = STI.ClassifyGlobalReference(GV, TM); + } - unsigned OpFlags = STI.ClassifyGlobalReference(GV, TM); if (OpFlags & AArch64II::MO_GOT) { I.setDesc(TII.get(AArch64::LOADgot)); I.getOperand(1).setTargetFlags(OpFlags); diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp index 33f04e6ad0c7..308364aadaf7 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp @@ -1345,6 +1345,9 @@ bool AArch64LegalizerInfo::legalizeSmallCMGlobalValue( // By splitting this here, we can optimize accesses in the small code model by // folding in the G_ADD_LOW into the load/store offset. auto &GlobalOp = MI.getOperand(1); + // Don't modify an intrinsic call. + if (GlobalOp.isSymbol()) + return true; const auto* GV = GlobalOp.getGlobal(); if (GV->isThreadLocal()) return true; // Don't want to modify TLS vars. diff --git a/llvm/test/CodeGen/AArch64/call-rv-marker.ll b/llvm/test/CodeGen/AArch64/call-rv-marker.ll index de8f5bbfb484..ee8cbe45da08 100644 --- a/llvm/test/CodeGen/AArch64/call-rv-marker.ll +++ b/llvm/test/CodeGen/AArch64/call-rv-marker.ll @@ -201,17 +201,27 @@ define dso_local void @rv_marker_3() personality ptr @__gxx_personality_v0 { ; GISEL-NEXT: bl _objc_object ; GISEL-NEXT: Ltmp1: ; GISEL-NEXT: ; %bb.1: ; %invoke.cont -; GISEL-NEXT: ldp x29, x30, [sp, #16] ; 16-byte Folded Reload +; GISEL-NEXT: Lloh0: +; GISEL-NEXT: adrp x1, _objc_release@GOTPAGE ; GISEL-NEXT: mov x0, x19 +; GISEL-NEXT: Lloh1: +; GISEL-NEXT: ldr x1, [x1, _objc_release@GOTPAGEOFF] +; GISEL-NEXT: ldp x29, x30, [sp, #16] ; 16-byte Folded Reload ; GISEL-NEXT: ldp x20, x19, [sp], #32 ; 16-byte Folded Reload -; GISEL-NEXT: b _objc_release +; GISEL-NEXT: br x1 ; GISEL-NEXT: LBB3_2: ; %lpad ; GISEL-NEXT: Ltmp2: +; GISEL-NEXT: Lloh2: +; GISEL-NEXT: adrp x8, _objc_release@GOTPAGE ; GISEL-NEXT: mov x20, x0 ; GISEL-NEXT: mov x0, x19 -; GISEL-NEXT: bl _objc_release +; GISEL-NEXT: Lloh3: +; GISEL-NEXT: ldr x8, [x8, _objc_release@GOTPAGEOFF] +; GISEL-NEXT: blr x8 ; GISEL-NEXT: mov x0, x20 ; GISEL-NEXT: bl __Unwind_Resume +; GISEL-NEXT: .loh AdrpLdrGot Lloh0, Lloh1 +; GISEL-NEXT: .loh AdrpLdrGot Lloh2, Lloh3 ; GISEL-NEXT: Lfunc_end0: ; GISEL-NEXT: .cfi_endproc ; GISEL-NEXT: .section __TEXT,__gcc_except_tab @@ -352,8 +362,12 @@ define dso_local void @rv_marker_4() personality ptr @__gxx_personality_v0 { ; GISEL-NEXT: bl _objc_object ; GISEL-NEXT: Ltmp7: ; GISEL-NEXT: ; %bb.2: ; %invoke.cont2 +; GISEL-NEXT: Lloh4: +; GISEL-NEXT: adrp x8, _objc_release@GOTPAGE ; GISEL-NEXT: mov x0, x19 -; GISEL-NEXT: bl _objc_release +; GISEL-NEXT: Lloh5: +; GISEL-NEXT: ldr x8, [x8, _objc_release@GOTPAGEOFF] +; GISEL-NEXT: blr x8 ; GISEL-NEXT: add x0, sp, #15 ; GISEL-NEXT: bl __ZN1SD1Ev ; GISEL-NEXT: ldp x29, x30, [sp, #32] ; 16-byte Folded Reload @@ -362,9 +376,13 @@ define dso_local void @rv_marker_4() personality ptr @__gxx_personality_v0 { ; GISEL-NEXT: ret ; GISEL-NEXT: LBB4_3: ; %lpad1 ; GISEL-NEXT: Ltmp8: +; GISEL-NEXT: Lloh6: +; GISEL-NEXT: adrp x8, _objc_release@GOTPAGE ; GISEL-NEXT: mov x20, x0 ; GISEL-NEXT: mov x0, x19 -; GISEL-NEXT: bl _objc_release +; GISEL-NEXT: Lloh7: +; GISEL-NEXT: ldr x8, [x8, _objc_release@GOTPAGEOFF] +; GISEL-NEXT: blr x8 ; GISEL-NEXT: b LBB4_5 ; GISEL-NEXT: LBB4_4: ; %lpad ; GISEL-NEXT: Ltmp5: @@ -374,6 +392,8 @@ define dso_local void @rv_marker_4() personality ptr @__gxx_personality_v0 { ; GISEL-NEXT: bl __ZN1SD1Ev ; GISEL-NEXT: mov x0, x20 ; GISEL-NEXT: bl __Unwind_Resume +; GISEL-NEXT: .loh AdrpLdrGot Lloh4, Lloh5 +; GISEL-NEXT: .loh AdrpLdrGot Lloh6, Lloh7 ; GISEL-NEXT: Lfunc_end1: ; GISEL-NEXT: .cfi_endproc ; GISEL-NEXT: .section __TEXT,__gcc_except_tab @@ -467,9 +487,9 @@ define dso_local ptr @rv_marker_5_indirect_call() { ; GISEL-NEXT: .cfi_offset w29, -16 ; GISEL-NEXT: .cfi_offset w19, -24 ; GISEL-NEXT: .cfi_offset w20, -32 -; GISEL-NEXT: Lloh0: +; GISEL-NEXT: Lloh8: ; GISEL-NEXT: adrp x8, _fptr@PAGE -; GISEL-NEXT: Lloh1: +; GISEL-NEXT: Lloh9: ; GISEL-NEXT: ldr x8, [x8, _fptr@PAGEOFF] ; GISEL-NEXT: blr x8 ; GISEL-NEXT: mov x29, x29 @@ -480,7 +500,7 @@ define dso_local ptr @rv_marker_5_indirect_call() { ; GISEL-NEXT: mov x0, x19 ; GISEL-NEXT: ldp x20, x19, [sp], #32 ; 16-byte Folded Reload ; GISEL-NEXT: ret -; GISEL-NEXT: .loh AdrpLdr Lloh0, Lloh1 +; GISEL-NEXT: .loh AdrpLdr Lloh8, Lloh9 entry: %0 = load ptr, ptr @fptr, align 8 %call = call ptr %0() [ "clang.arc.attachedcall"(ptr @objc_retainAutoreleasedReturnValue) ] diff --git a/llvm/test/CodeGen/AArch64/nonlazybind.ll b/llvm/test/CodeGen/AArch64/nonlazybind.ll index 669a8ee04b24..f5bb3a4ecbc9 100644 --- a/llvm/test/CodeGen/AArch64/nonlazybind.ll +++ b/llvm/test/CodeGen/AArch64/nonlazybind.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 -; RUN: llc -mtriple=aarch64-apple-ios %s -o - -aarch64-enable-nonlazybind | FileCheck %s --check-prefix=MACHO +; RUN: llc -mtriple=aarch64-apple-ios %s -o - -aarch64-macho-enable-nonlazybind | FileCheck %s --check-prefix=MACHO ; RUN: llc -mtriple=aarch64-apple-ios %s -o - | FileCheck %s --check-prefix=MACHO-NORMAL ; RUN: llc -mtriple=aarch64 -fast-isel %s -o - | FileCheck %s --check-prefixes=ELF,ELF-FI ; RUN: llc -mtriple=aarch64 -global-isel %s -o - | FileCheck %s --check-prefixes=ELF,ELF-GI @@ -19,13 +19,18 @@ define void @test_laziness(ptr %a) nounwind { ; MACHO-NEXT: Lloh1: ; MACHO-NEXT: ldr x8, [x8, _external@GOTPAGEOFF] ; MACHO-NEXT: blr x8 +; MACHO-NEXT: Lloh2: +; MACHO-NEXT: adrp x8, _memset@GOTPAGE ; MACHO-NEXT: mov x0, x19 ; MACHO-NEXT: mov w1, #1 ; =0x1 +; MACHO-NEXT: Lloh3: +; MACHO-NEXT: ldr x8, [x8, _memset@GOTPAGEOFF] ; MACHO-NEXT: mov w2, #1000 ; =0x3e8 -; MACHO-NEXT: bl _memset +; MACHO-NEXT: blr x8 ; MACHO-NEXT: ldp x29, x30, [sp, #16] ; 16-byte Folded Reload ; MACHO-NEXT: ldp x20, x19, [sp], #32 ; 16-byte Folded Reload ; MACHO-NEXT: ret +; MACHO-NEXT: .loh AdrpLdrGot Lloh2, Lloh3 ; MACHO-NEXT: .loh AdrpLdrGot Lloh0, Lloh1 ; ; MACHO-NORMAL-LABEL: test_laziness: @@ -34,50 +39,34 @@ define void @test_laziness(ptr %a) nounwind { ; MACHO-NORMAL-NEXT: stp x29, x30, [sp, #16] ; 16-byte Folded Spill ; MACHO-NORMAL-NEXT: mov x19, x0 ; MACHO-NORMAL-NEXT: bl _external +; MACHO-NORMAL-NEXT: Lloh0: +; MACHO-NORMAL-NEXT: adrp x8, _memset@GOTPAGE ; MACHO-NORMAL-NEXT: mov x0, x19 ; MACHO-NORMAL-NEXT: mov w1, #1 ; =0x1 +; MACHO-NORMAL-NEXT: Lloh1: +; MACHO-NORMAL-NEXT: ldr x8, [x8, _memset@GOTPAGEOFF] ; MACHO-NORMAL-NEXT: mov w2, #1000 ; =0x3e8 -; MACHO-NORMAL-NEXT: bl _memset +; MACHO-NORMAL-NEXT: blr x8 ; MACHO-NORMAL-NEXT: ldp x29, x30, [sp, #16] ; 16-byte Folded Reload ; MACHO-NORMAL-NEXT: ldp x20, x19, [sp], #32 ; 16-byte Folded Reload ; MACHO-NORMAL-NEXT: ret +; MACHO-NORMAL-NEXT: .loh AdrpLdrGot Lloh0, Lloh1 ; -; ELF-FI-LABEL: test_laziness: -; ELF-FI: // %bb.0: -; ELF-FI-NEXT: stp x30, x19, [sp, #-16]! // 16-byte Folded Spill -; ELF-FI-NEXT: mov x19, x0 -; ELF-FI-NEXT: bl external -; ELF-FI-NEXT: mov w8, #1 // =0x1 -; ELF-FI-NEXT: mov x0, x19 -; ELF-FI-NEXT: mov x2, #1000 // =0x3e8 -; ELF-FI-NEXT: uxtb w1, w8 -; ELF-FI-NEXT: bl memset -; ELF-FI-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload -; ELF-FI-NEXT: ret -; -; ELF-GI-LABEL: test_laziness: -; ELF-GI: // %bb.0: -; ELF-GI-NEXT: stp x30, x19, [sp, #-16]! // 16-byte Folded Spill -; ELF-GI-NEXT: mov x19, x0 -; ELF-GI-NEXT: bl external -; ELF-GI-NEXT: mov x0, x19 -; ELF-GI-NEXT: mov w1, #1 // =0x1 -; ELF-GI-NEXT: mov w2, #1000 // =0x3e8 -; ELF-GI-NEXT: bl memset -; ELF-GI-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload -; ELF-GI-NEXT: ret -; -; ELF-SDAG-LABEL: test_laziness: -; ELF-SDAG: // %bb.0: -; ELF-SDAG-NEXT: stp x30, x19, [sp, #-16]! // 16-byte Folded Spill -; ELF-SDAG-NEXT: mov x19, x0 -; ELF-SDAG-NEXT: bl external -; ELF-SDAG-NEXT: mov x0, x19 -; ELF-SDAG-NEXT: mov w1, #1 // =0x1 -; ELF-SDAG-NEXT: mov w2, #1000 // =0x3e8 -; ELF-SDAG-NEXT: bl memset -; ELF-SDAG-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload -; ELF-SDAG-NEXT: ret +; ELF-LABEL: test_laziness: +; ELF: // %bb.0: +; ELF-NEXT: stp x30, x19, [sp, #-16]! // 16-byte Folded Spill +; ELF-NEXT: adrp x8, :got:external +; ELF-NEXT: mov x19, x0 +; ELF-NEXT: ldr x8, [x8, :got_lo12:external] +; ELF-NEXT: blr x8 +; ELF-NEXT: adrp x8, :got:memset +; ELF-NEXT: mov x0, x19 +; ELF-NEXT: mov w1, #1 // =0x1 +; ELF-NEXT: ldr x8, [x8, :got_lo12:memset] +; ELF-NEXT: mov w2, #1000 // =0x3e8 +; ELF-NEXT: blr x8 +; ELF-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload +; ELF-NEXT: ret call void @external() call void @llvm.memset.p0.i64(ptr align 1 %a, i8 1, i64 1000, i1 false) ret void @@ -86,12 +75,12 @@ define void @test_laziness(ptr %a) nounwind { define void @test_laziness_tail() nounwind { ; MACHO-LABEL: test_laziness_tail: ; MACHO: ; %bb.0: -; MACHO-NEXT: Lloh2: +; MACHO-NEXT: Lloh4: ; MACHO-NEXT: adrp x0, _external@GOTPAGE -; MACHO-NEXT: Lloh3: +; MACHO-NEXT: Lloh5: ; MACHO-NEXT: ldr x0, [x0, _external@GOTPAGEOFF] ; MACHO-NEXT: br x0 -; MACHO-NEXT: .loh AdrpLdrGot Lloh2, Lloh3 +; MACHO-NEXT: .loh AdrpLdrGot Lloh4, Lloh5 ; ; MACHO-NORMAL-LABEL: test_laziness_tail: ; MACHO-NORMAL: ; %bb.0: @@ -99,7 +88,9 @@ define void @test_laziness_tail() nounwind { ; ; ELF-LABEL: test_laziness_tail: ; ELF: // %bb.0: -; ELF-NEXT: b external +; ELF-NEXT: adrp x0, :got:external +; ELF-NEXT: ldr x0, [x0, :got_lo12:external] +; ELF-NEXT: br x0 tail call void @external() ret void } @@ -108,3 +99,7 @@ declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) !llvm.module.flags = !{!0} !0 = !{i32 7, !"RtLibUseGOT", i32 1} +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; ELF-FI: {{.*}} +; ELF-GI: {{.*}} +; ELF-SDAG: {{.*}} -- GitLab From f51ade25b9205efee09a4915031848cebe772805 Mon Sep 17 00:00:00 2001 From: Maksim Panchenko Date: Tue, 5 Mar 2024 13:57:55 -0800 Subject: [PATCH 213/929] [BOLT] Add reading support for Linux kernel .parainstructions section (#83965) Read .parainstruction section and mark call instructions with ParaSite annotations. --- bolt/lib/Rewrite/LinuxKernelRewriter.cpp | 84 ++++++++++++++++++++++++ bolt/test/X86/linux-parainstructions.s | 54 +++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 bolt/test/X86/linux-parainstructions.s diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp index 145acd32356c..a78397199972 100644 --- a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp +++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp @@ -36,6 +36,10 @@ static cl::opt DumpORC("dump-orc", cl::desc("dump raw ORC unwind information (sorted)"), cl::init(false), cl::Hidden, cl::cat(BoltCategory)); +static cl::opt DumpParavirtualPatchSites( + "dump-para-sites", cl::desc("dump Linux kernel paravitual patch sites"), + cl::init(false), cl::Hidden, cl::cat(BoltCategory)); + static cl::opt DumpStaticCalls("dump-static-calls", cl::desc("dump Linux kernel static calls"), cl::init(false), cl::Hidden, @@ -147,6 +151,12 @@ class LinuxKernelRewriter final : public MetadataRewriter { /// Functions with exception handling code. DenseSet FunctionsWithExceptions; + /// Section with paravirtual patch sites. + ErrorOr ParavirtualPatchSection = std::errc::bad_address; + + /// Alignment of paravirtual patch structures. + static constexpr size_t PARA_PATCH_ALIGN = 8; + /// Insert an LKMarker for a given code pointer \p PC from a non-code section /// \p SectionName. void insertLKMarker(uint64_t PC, uint64_t SectionOffset, @@ -187,6 +197,9 @@ class LinuxKernelRewriter final : public MetadataRewriter { Error readExceptionTable(); Error rewriteExceptionTable(); + /// Paravirtual instruction patch sites. + Error readParaInstructions(); + /// Mark instructions referenced by kernel metadata. Error markInstructions(); @@ -208,6 +221,9 @@ public: if (Error E = readExceptionTable()) return E; + if (Error E = readParaInstructions()) + return E; + return Error::success(); } @@ -1013,6 +1029,74 @@ Error LinuxKernelRewriter::rewriteExceptionTable() { return Error::success(); } +/// .parainsrtuctions section contains information for patching parvirtual call +/// instructions during runtime. The entries in the section are in the form: +/// +/// struct paravirt_patch_site { +/// u8 *instr; /* original instructions */ +/// u8 type; /* type of this instruction */ +/// u8 len; /* length of original instruction */ +/// }; +/// +/// Note that the structures are aligned at 8-byte boundary. +Error LinuxKernelRewriter::readParaInstructions() { + ParavirtualPatchSection = BC.getUniqueSectionByName(".parainstructions"); + if (!ParavirtualPatchSection) + return Error::success(); + + DataExtractor DE = DataExtractor(ParavirtualPatchSection->getContents(), + BC.AsmInfo->isLittleEndian(), + BC.AsmInfo->getCodePointerSize()); + uint32_t EntryID = 0; + DataExtractor::Cursor Cursor(0); + while (Cursor && !DE.eof(Cursor)) { + const uint64_t NextOffset = alignTo(Cursor.tell(), Align(PARA_PATCH_ALIGN)); + if (!DE.isValidOffset(NextOffset)) + break; + + Cursor.seek(NextOffset); + + const uint64_t InstrLocation = DE.getU64(Cursor); + const uint8_t Type = DE.getU8(Cursor); + const uint8_t Len = DE.getU8(Cursor); + + if (!Cursor) + return createStringError(errc::executable_format_error, + "out of bounds while reading .parainstructions"); + + ++EntryID; + + if (opts::DumpParavirtualPatchSites) { + BC.outs() << "Paravirtual patch site: " << EntryID << '\n'; + BC.outs() << "\tInstr: 0x" << Twine::utohexstr(InstrLocation) + << "\n\tType: 0x" << Twine::utohexstr(Type) << "\n\tLen: 0x" + << Twine::utohexstr(Len) << '\n'; + } + + BinaryFunction *BF = BC.getBinaryFunctionContainingAddress(InstrLocation); + if (!BF && opts::Verbosity) { + BC.outs() << "BOLT-INFO: no function matches address 0x" + << Twine::utohexstr(InstrLocation) + << " referenced by paravirutal patch site\n"; + } + + if (BF && BC.shouldEmit(*BF)) { + MCInst *Inst = + BF->getInstructionAtOffset(InstrLocation - BF->getAddress()); + if (!Inst) + return createStringError(errc::executable_format_error, + "no instruction at address 0x%" PRIx64 + " in paravirtual call site %d", + InstrLocation, EntryID); + BC.MIB->addAnnotation(*Inst, "ParaSite", EntryID); + } + } + + BC.outs() << "BOLT-INFO: parsed " << EntryID << " paravirtual patch sites\n"; + + return Error::success(); +} + } // namespace std::unique_ptr diff --git a/bolt/test/X86/linux-parainstructions.s b/bolt/test/X86/linux-parainstructions.s new file mode 100644 index 000000000000..4bdfde5fb7f2 --- /dev/null +++ b/bolt/test/X86/linux-parainstructions.s @@ -0,0 +1,54 @@ +# REQUIRES: system-linux + +## Check that BOLT correctly parses the Linux kernel .parainstructions section. + +# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o +# RUN: %clang %cflags -nostdlib %t.o -o %t.exe \ +# RUN: -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr,--no-pie + +## Verify paravirtual bindings to instructions. + +# RUN: llvm-bolt %t.exe --print-normalized -o %t.out | FileCheck %s + +# CHECK: BOLT-INFO: Linux kernel binary detected +# CHECK: BOLT-INFO: parsed 2 paravirtual patch sites + + .rodata +fptr: + .quad 0 + + .text + .globl _start + .type _start, %function +_start: +# CHECK: Binary Function "_start" + nop +.L1: + call *fptr(%rip) +# CHECK: call +# CHECK-SAME: ParaSite: 1 + nop +.L2: + call *fptr(%rip) +# CHECK: call +# CHECK-SAME: ParaSite: 2 + ret + .size _start, .-_start + + +## Paravirtual patch sites. + .section .parainstructions,"a",@progbits + + .balign 8 + .quad .L1 # instruction + .byte 1 # type + .byte 7 # length + + .balign 8 + .quad .L2 # instruction + .byte 1 # type + .byte 7 # length + +## Fake Linux Kernel sections. + .section __ksymtab,"a",@progbits + .section __ksymtab_gpl,"a",@progbits -- GitLab From e96c0c1d5e0a9916098b1a31acb006ea6c1108fb Mon Sep 17 00:00:00 2001 From: Quentin Dian Date: Wed, 6 Mar 2024 06:16:28 +0800 Subject: [PATCH 214/929] [InstCombine] Fix shift calculation in InstCombineCasts (#84027) Fixes #84025. --- .../Transforms/InstCombine/InstCombineCasts.cpp | 4 ++-- llvm/test/Transforms/InstCombine/bitcast.ll | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp index 33ed1d557537..45afa6363ae0 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp @@ -2167,14 +2167,14 @@ static bool collectInsertionElements(Value *V, unsigned Shift, Type *ElementIntTy = IntegerType::get(C->getContext(), ElementSize); for (unsigned i = 0; i != NumElts; ++i) { - unsigned ShiftI = Shift + i * ElementSize; + unsigned ShiftI = i * ElementSize; Constant *Piece = ConstantFoldBinaryInstruction( Instruction::LShr, C, ConstantInt::get(C->getType(), ShiftI)); if (!Piece) return false; Piece = ConstantExpr::getTrunc(Piece, ElementIntTy); - if (!collectInsertionElements(Piece, ShiftI, Elements, VecEltTy, + if (!collectInsertionElements(Piece, ShiftI + Shift, Elements, VecEltTy, isBigEndian)) return false; } diff --git a/llvm/test/Transforms/InstCombine/bitcast.ll b/llvm/test/Transforms/InstCombine/bitcast.ll index 176b432ea0b5..5599604b666f 100644 --- a/llvm/test/Transforms/InstCombine/bitcast.ll +++ b/llvm/test/Transforms/InstCombine/bitcast.ll @@ -686,6 +686,21 @@ define ptr @bitcast_from_single_element_pointer_vector_to_pointer(<1 x ptr> %ptr ret ptr %ptr } +; Sure that we calculate the correct shift. +define <4 x i32> @bitcast_shl(i32 %arg) { +; CHECK-LABEL: @bitcast_shl( +; CHECK-NEXT: [[I5:%.*]] = insertelement <4 x i32> , i32 [[ARG:%.*]], i64 3 +; CHECK-NEXT: ret <4 x i32> [[I5]] +; + %i = zext i32 %arg to i64 + %i1 = shl i64 %i, 32 + %i2 = or i64 %i1, 65 + %i3 = zext i64 %i2 to i128 + %i4 = shl i128 %i3, 64 + %i5 = bitcast i128 %i4 to <4 x i32> + ret <4 x i32> %i5 +} + declare void @f1() declare void @f2() define ptr @select_bitcast_unsized_pointer(i1 %c) { -- GitLab From 58d8805ff9f0a9947ac122b463c00d6c0656eae6 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Tue, 5 Mar 2024 14:36:37 -0800 Subject: [PATCH 215/929] [RISCV] Always use signed APSInt in getExactInteger. (#84070) We were setting based on whether the FP value is positive/negative, but we really want to know whether the resulting integer will be treated as a signed or unsigned value. Since we use SINT_TO_FP to convert the integer to FP, we should always used signed here. Without this we convert +2147483648.0 to an integer 0x80000000 and convert it using sint_to_fp which produces -2147483648.0. --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 4 +++- llvm/test/CodeGen/RISCV/rvv/vle_vid-vfcvt.ll | 11 +++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 7da074e055a7..d56cba7de5d0 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -3233,7 +3233,9 @@ struct VIDSequence { static std::optional getExactInteger(const APFloat &APF, uint32_t BitWidth) { - APSInt ValInt(BitWidth, !APF.isNegative()); + // We will use a SINT_TO_FP to materialize this constant so we should use a + // signed APSInt here. + APSInt ValInt(BitWidth, /*IsUnsigned*/ false); // We use an arbitrary rounding mode here. If a floating-point is an exact // integer (e.g., 1.0), the rounding mode does not affect the output value. If // the rounding mode changes the output value, then it is not an exact diff --git a/llvm/test/CodeGen/RISCV/rvv/vle_vid-vfcvt.ll b/llvm/test/CodeGen/RISCV/rvv/vle_vid-vfcvt.ll index 5588dd5db598..bf330ea38a00 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vle_vid-vfcvt.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vle_vid-vfcvt.ll @@ -113,16 +113,15 @@ entry: ret void } -; FIXME: This is miscompiled. This will create -2147483648.0 instead of -; 2147483648.0 for the 4th element. +; Make sure we don't try to use vid+vsll+vfcvt. We previously flipped the sign +; of 2147483648.0. define void @foo_9(ptr nocapture noundef writeonly %t) { ; CHECK-LABEL: foo_9: ; CHECK: # %bb.0: # %entry +; CHECK-NEXT: lui a1, %hi(.LCPI8_0) +; CHECK-NEXT: addi a1, a1, %lo(.LCPI8_0) ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: vid.v v8 -; CHECK-NEXT: vsll.vi v8, v8, 31 -; CHECK-NEXT: vrsub.vi v8, v8, 0 -; CHECK-NEXT: vfcvt.f.x.v v8, v8 +; CHECK-NEXT: vle32.v v8, (a1) ; CHECK-NEXT: vse32.v v8, (a0) ; CHECK-NEXT: ret entry: -- GitLab From e77a473601314cc7e7aa912579982a38326d334c Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Tue, 5 Mar 2024 14:27:31 -0800 Subject: [PATCH 216/929] [ORC][MachO] Simplify use of LC_BUILD_VERSION in JITDylib headers. API clients can now set a MachO::HeaderOptions::BuildVersionOpts field to have MachOPlatform add an LC_BUILD_VERSION load command to the Mach header for each JITDylib. No testcase yet. In the future we'll try to add a MachO parser to the ORC runtime and extra test options to llvm-jitlink for this. This commit also incidentally fixes a bug in the MachOBuilder class that lead to a delegation cycle. --- .../llvm/ExecutionEngine/Orc/MachOBuilder.h | 2 +- .../llvm/ExecutionEngine/Orc/MachOPlatform.h | 15 ++++++++ .../lib/ExecutionEngine/Orc/MachOPlatform.cpp | 35 +++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MachOBuilder.h b/llvm/include/llvm/ExecutionEngine/Orc/MachOBuilder.h index 9a011a427db3..6ffd286c365a 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/MachOBuilder.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/MachOBuilder.h @@ -78,7 +78,7 @@ public: template MachOBuilderLoadCommand(ArgTs &&...Args) - : MachOBuilderLoadCommand(std::forward(Args)...) {} + : MachOBuilderLoadCommandImplBase(std::forward(Args)...) {} }; template <> diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h index ff1c420d047e..e928faf47885 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h @@ -58,9 +58,24 @@ public: uint32_t CompatibilityVersion; }; + struct BuildVersionOpts { + + // Derive platform from triple. + static BuildVersionOpts fromTriple(const Triple &TT, uint32_t MinOS, + uint32_t SDK); + + uint32_t Platform; // Platform. + uint32_t MinOS; // X.Y.Z is encoded in nibbles xxxx.yy.zz + uint32_t SDK; // X.Y.Z is encoded in nibbles xxxx.yy.zz + }; + /// Override for LC_IC_DYLIB. If this is nullopt, {JD.getName(), 0, 0, 0} /// will be used. std::optional IDDylib; + + /// Override for LC_BUILD_VERSION. If this is nullopt then + std::optional BuildVersion; + /// List of LC_LOAD_DYLIBs. std::vector LoadDylibs; /// List of LC_RPATHs. diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp index 377a31e63ec1..994acf584364 100644 --- a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp +++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp @@ -255,6 +255,36 @@ struct ObjCImageInfoFlags { namespace llvm { namespace orc { +MachOPlatform::HeaderOptions::BuildVersionOpts +MachOPlatform::HeaderOptions::BuildVersionOpts::fromTriple(const Triple &TT, + uint32_t MinOS, + uint32_t SDK) { + + uint32_t Platform; + switch (TT.getOS()) { + case Triple::IOS: + Platform = TT.isSimulatorEnvironment() ? MachO::PLATFORM_IOS + : MachO::PLATFORM_IOSSIMULATOR; + break; + case Triple::MacOSX: + Platform = MachO::PLATFORM_MACOS; + break; + case Triple::TvOS: + Platform = TT.isSimulatorEnvironment() ? MachO::PLATFORM_TVOS + : MachO::PLATFORM_TVOSSIMULATOR; + break; + case Triple::WatchOS: + Platform = TT.isSimulatorEnvironment() ? MachO::PLATFORM_WATCHOS + : MachO::PLATFORM_WATCHOSSIMULATOR; + break; + default: + Platform = MachO::PLATFORM_UNKNOWN; + break; + } + + return {Platform, MinOS, SDK}; +} + Expected> MachOPlatform::Create( ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD, std::unique_ptr OrcRuntime, @@ -1695,6 +1725,11 @@ jitlink::Block &createHeaderBlock(MachOPlatform &MOP, else B.template addLoadCommand(JD.getName(), 0, 0, 0); + if (Opts.BuildVersion) + B.template addLoadCommand( + Opts.BuildVersion->Platform, Opts.BuildVersion->MinOS, + Opts.BuildVersion->SDK, static_cast(0)); + for (auto &D : Opts.LoadDylibs) B.template addLoadCommand( D.Name, D.Timestamp, D.CurrentVersion, D.CompatibilityVersion); -- GitLab From d6c52c1e2d16ef79737c5666db46e872e943c18a Mon Sep 17 00:00:00 2001 From: Changpeng Fang Date: Tue, 5 Mar 2024 15:32:32 -0800 Subject: [PATCH 217/929] AMDGPU: Define HasExpOrExportInsts for export instruction definitions. (#84083) --- llvm/lib/Target/AMDGPU/AMDGPU.td | 3 +++ llvm/lib/Target/AMDGPU/EXPInstructions.td | 6 +++--- llvm/lib/Target/AMDGPU/GCNSubtarget.h | 4 ++++ llvm/lib/Target/AMDGPU/SOPInstructions.td | 2 ++ 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td index 7c278fd574ed..814ac0b93fcf 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.td +++ b/llvm/lib/Target/AMDGPU/AMDGPU.td @@ -1874,6 +1874,9 @@ def D16PreservesUnusedBits : def LDSRequiresM0Init : Predicate<"Subtarget->ldsRequiresM0Init()">; def NotLDSRequiresM0Init : Predicate<"!Subtarget->ldsRequiresM0Init()">; +def HasExpOrExportInsts : Predicate<"Subtarget->hasExpOrExportInsts()">, + AssemblerPredicate<(all_of (not FeatureGFX90AInsts))>; + def HasDSAddTid : Predicate<"Subtarget->getGeneration() >= AMDGPUSubtarget::GFX9">, AssemblerPredicate<(all_of FeatureGFX9Insts)>; diff --git a/llvm/lib/Target/AMDGPU/EXPInstructions.td b/llvm/lib/Target/AMDGPU/EXPInstructions.td index 0a1e54494910..5e3f555e4411 100644 --- a/llvm/lib/Target/AMDGPU/EXPInstructions.td +++ b/llvm/lib/Target/AMDGPU/EXPInstructions.td @@ -58,12 +58,12 @@ class EXP_Real_Row; def EXP_DONE : EXP_Pseudo<0, 1>; def EXP_ROW : EXP_Pseudo<1, 0>; def EXP_ROW_DONE : EXP_Pseudo<1, 1>; -} // let SubtargetPredicate = isNotGFX90APlus +} // let SubtargetPredicate = HasExpOrExportInsts //===----------------------------------------------------------------------===// // SI, VI, GFX10. @@ -117,7 +117,7 @@ multiclass EXP_Real_gfx11 { multiclass VEXPORT_Real_gfx12 { defvar ps = !cast(NAME); def _gfx12 : EXP_Real_Row, - EXPe_Row, MnemonicAlias<"exp", "export">, Requires<[isGFX12Plus]> { + EXPe_Row, MnemonicAlias<"exp", "export">, Requires<[isGFX12Plus, HasExpOrExportInsts]> { let AssemblerPredicate = isGFX12Only; let DecoderNamespace = "GFX12"; let row = ps.row; diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h index a933c16b6ed5..bb0ccfce324f 100644 --- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h +++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h @@ -647,6 +647,10 @@ public: // BUFFER/FLAT/GLOBAL_ATOMIC_ADD/MIN/MAX_F64 bool hasBufferFlatGlobalAtomicsF64() const { return hasGFX90AInsts(); } + bool hasExpOrExportInsts() const { + return !hasGFX940Insts(); + } + // DS_ADD_F64/DS_ADD_RTN_F64 bool hasLdsAtomicAddF64() const { return hasGFX90AInsts(); } diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td index ff79538a86ae..e14f7f95d467 100644 --- a/llvm/lib/Target/AMDGPU/SOPInstructions.td +++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td @@ -1705,6 +1705,7 @@ let SubtargetPredicate = isGFX10Plus in { } // End SubtargetPredicate = isGFX10Plus let SubtargetPredicate = isGFX11Plus in { +let OtherPredicates = [HasExpOrExportInsts] in def S_WAIT_EVENT : SOPP_Pseudo<"s_wait_event", (ins s16imm:$simm16), "$simm16"> { let hasSideEffects = 1; @@ -1737,6 +1738,7 @@ let OtherPredicates = [HasImageInsts] in { SOPP_Pseudo<"s_wait_bvhcnt", (ins s16imm:$simm16), "$simm16", [(int_amdgcn_s_wait_bvhcnt timm:$simm16)]>; } // End OtherPredicates = [HasImageInsts]. +let OtherPredicates = [HasExpOrExportInsts] in def S_WAIT_EXPCNT : SOPP_Pseudo<"s_wait_expcnt", (ins s16imm:$simm16), "$simm16", [(int_amdgcn_s_wait_expcnt timm:$simm16)]>; -- GitLab From 6f11c95d06d51e98e635d08cd3143fb88b58b6a9 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Tue, 5 Mar 2024 15:52:09 -0800 Subject: [PATCH 218/929] Revert "[AArch64] Verify ldp/stp alignment stricter" (#84096) Reverts llvm/llvm-project#83948 This broke the ASan buildbot: https://lab.llvm.org/buildbot/#/builders/168/builds/19054/steps/10/logs/stdio --- .../AArch64/AArch64LoadStoreOptimizer.cpp | 44 +++++++-------- .../CodeGen/AArch64/ldp-stp-unknown-size.mir | 56 ------------------- 2 files changed, 19 insertions(+), 81 deletions(-) delete mode 100644 llvm/test/CodeGen/AArch64/ldp-stp-unknown-size.mir diff --git a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp index abfb1f43a9c8..926a89466255 100644 --- a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp +++ b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp @@ -62,8 +62,6 @@ STATISTIC(NumUnscaledPairCreated, "Number of load/store from unscaled generated"); STATISTIC(NumZeroStoresPromoted, "Number of narrow zero stores promoted"); STATISTIC(NumLoadsFromStoresPromoted, "Number of loads from stores promoted"); -STATISTIC(NumFailedAlignmentCheck, "Number of load/store pair transformation " - "not passed the alignment check"); DEBUG_COUNTER(RegRenamingCounter, DEBUG_TYPE "-reg-renaming", "Controls which pairs are considered for renaming"); @@ -2339,6 +2337,9 @@ bool AArch64LoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) { MachineBasicBlock::iterator Paired = findMatchingInsn(MBBI, Flags, LdStLimit, /* FindNarrowMerge = */ false); if (Paired != E) { + ++NumPairCreated; + if (TII->hasUnscaledLdStOffset(MI)) + ++NumUnscaledPairCreated; // Keeping the iterator straight is a pain, so we let the merge routine tell // us what the next instruction is after it's done mucking about. auto Prev = std::prev(MBBI); @@ -2348,27 +2349,24 @@ bool AArch64LoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) { MachineMemOperand *MemOp = MI.memoperands_empty() ? nullptr : MI.memoperands().front(); - // If a load/store arrives and ldp/stp-aligned-only feature is opted, check - // that the alignment of the source pointer is at least double the alignment - // of the type. - if ((MI.mayLoad() && Subtarget->hasLdpAlignedOnly()) || - (MI.mayStore() && Subtarget->hasStpAlignedOnly())) { - // If there is no size/align information, cancel the transformation. - if (!MemOp || !MemOp->getMemoryType().isValid()) { - NumFailedAlignmentCheck++; - return false; - } + // Get the needed alignments to check them if + // ldp-aligned-only/stp-aligned-only features are opted. + uint64_t MemAlignment = MemOp ? MemOp->getAlign().value() : -1; + uint64_t TypeAlignment = MemOp ? Align(MemOp->getSize()).value() : -1; - // Get the needed alignments to check them if - // ldp-aligned-only/stp-aligned-only features are opted. - uint64_t MemAlignment = MemOp->getAlign().value(); - uint64_t TypeAlignment = Align(MemOp->getSize()).value(); + // If a load arrives and ldp-aligned-only feature is opted, check that the + // alignment of the source pointer is at least double the alignment of the + // type. + if (MI.mayLoad() && Subtarget->hasLdpAlignedOnly() && MemOp && + MemAlignment < 2 * TypeAlignment) + return false; - if (MemAlignment < 2 * TypeAlignment) { - NumFailedAlignmentCheck++; - return false; - } - } + // If a store arrives and stp-aligned-only feature is opted, check that the + // alignment of the source pointer is at least double the alignment of the + // type. + if (MI.mayStore() && Subtarget->hasStpAlignedOnly() && MemOp && + MemAlignment < 2 * TypeAlignment) + return false; MBBI = mergePairedInsns(MBBI, Paired, Flags); // Collect liveness info for instructions between Prev and the new position @@ -2376,10 +2374,6 @@ bool AArch64LoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) { for (auto I = std::next(Prev); I != MBBI; I++) updateDefinedRegisters(*I, DefinedInBB, TRI); - ++NumPairCreated; - if (TII->hasUnscaledLdStOffset(MI)) - ++NumUnscaledPairCreated; - return true; } return false; diff --git a/llvm/test/CodeGen/AArch64/ldp-stp-unknown-size.mir b/llvm/test/CodeGen/AArch64/ldp-stp-unknown-size.mir deleted file mode 100644 index 3234a7dc11f0..000000000000 --- a/llvm/test/CodeGen/AArch64/ldp-stp-unknown-size.mir +++ /dev/null @@ -1,56 +0,0 @@ -# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4 -# RUN: llc -O2 -mtriple=aarch64 -mcpu=ampere1 -simplify-mir -o - %s -run-pass=aarch64-ldst-opt | FileCheck %s --check-prefixes=CHECK -# RUN: llc -O2 -mtriple=aarch64 -simplify-mir -o - %s -run-pass=aarch64-ldst-opt | FileCheck %s --check-prefixes=CHECK-DEFAULT - ---- | - define i32 @ldp_no_size_info(ptr %0) #0 { - %2 = ptrtoint ptr %0 to i64 - %3 = and i64 %2, -64 - %4 = inttoptr i64 %3 to ptr - %5 = load i32, ptr %4, align 4 - %6 = getelementptr inbounds i32, ptr %4, i64 1 - %7 = load i32, ptr %6, align 4 - %8 = add nsw i32 %7, %5 - ret i32 %8 - } - -... ---- -name: ldp_no_size_info -alignment: 64 -tracksRegLiveness: true -tracksDebugUserValues: true -liveins: - - { reg: '$x0' } -frameInfo: - maxAlignment: 1 - maxCallFrameSize: 0 -machineFunctionInfo: - hasRedZone: false -body: | - bb.0 (%ir-block.1): - liveins: $x0 - - ; CHECK-LABEL: name: ldp_no_size_info - ; CHECK: liveins: $x0 - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: renamable $x8 = ANDXri killed renamable $x0, 7865 - ; CHECK-NEXT: renamable $w9 = LDRWui renamable $x8, 0 :: (load unknown-size from %ir.4, align 1) - ; CHECK-NEXT: renamable $w8 = LDRWui killed renamable $x8, 1 :: (load unknown-size from %ir.6, align 1) - ; CHECK-NEXT: $w0 = ADDWrs killed renamable $w8, killed renamable $w9, 0 - ; CHECK-NEXT: RET undef $lr, implicit $w0 - ; - ; CHECK-DEFAULT-LABEL: name: ldp_no_size_info - ; CHECK-DEFAULT: liveins: $x0 - ; CHECK-DEFAULT-NEXT: {{ $}} - ; CHECK-DEFAULT-NEXT: renamable $x8 = ANDXri killed renamable $x0, 7865 - ; CHECK-DEFAULT-NEXT: renamable $w9, renamable $w8 = LDPWi renamable $x8, 0 :: (load unknown-size from %ir.4, align 1), (load unknown-size from %ir.6, align 1) - ; CHECK-DEFAULT-NEXT: $w0 = ADDWrs killed renamable $w8, killed renamable $w9, 0 - ; CHECK-DEFAULT-NEXT: RET undef $lr, implicit $w0 - renamable $x8 = ANDXri killed renamable $x0, 7865 - renamable $w9 = LDRWui renamable $x8, 0 :: (load unknown-size from %ir.4) - renamable $w8 = LDRWui killed renamable $x8, 1 :: (load unknown-size from %ir.6) - $w0 = ADDWrs killed renamable $w8, killed renamable $w9, 0 - RET undef $lr, implicit $w0 - -... -- GitLab From 55c466da2f2f0baa509eb709b8de8926bd498b9b Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Wed, 6 Mar 2024 00:48:01 +0100 Subject: [PATCH 219/929] [X86][AVX512BF16] Add a few missing insert/extract patterns These are really the same as the f16 (and i16) instructions, but we need them for any type that can occur. --- llvm/lib/Target/X86/X86InstrAVX512.td | 12 ++++++++++++ llvm/test/CodeGen/X86/bfloat.ll | 21 +++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td index b610560f7fe4..a76561f092c3 100644 --- a/llvm/lib/Target/X86/X86InstrAVX512.td +++ b/llvm/lib/Target/X86/X86InstrAVX512.td @@ -494,6 +494,8 @@ defm : vinsert_for_size_lowering<"VINSERTI32x4Z256", v16i8x_info, v32i8x_info, vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX]>; defm : vinsert_for_size_lowering<"VINSERTF32x4Z256", v8f16x_info, v16f16x_info, vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX]>; +defm : vinsert_for_size_lowering<"VINSERTF32x4Z256", v8bf16x_info, v16bf16x_info, + vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX]>; // Codegen pattern with the alternative types insert VEC128 into VEC512 defm : vinsert_for_size_lowering<"VINSERTI32x4Z", v8i16x_info, v32i16_info, vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512]>; @@ -501,6 +503,8 @@ defm : vinsert_for_size_lowering<"VINSERTI32x4Z", v16i8x_info, v64i8_info, vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512]>; defm : vinsert_for_size_lowering<"VINSERTF32x4Z", v8f16x_info, v32f16_info, vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512]>; +defm : vinsert_for_size_lowering<"VINSERTF32x4Z", v8bf16x_info, v32bf16_info, + vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512]>; // Codegen pattern with the alternative types insert VEC256 into VEC512 defm : vinsert_for_size_lowering<"VINSERTI64x4Z", v16i16x_info, v32i16_info, vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512]>; @@ -508,6 +512,8 @@ defm : vinsert_for_size_lowering<"VINSERTI64x4Z", v32i8x_info, v64i8_info, vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512]>; defm : vinsert_for_size_lowering<"VINSERTF64x4Z", v16f16x_info, v32f16_info, vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512]>; +defm : vinsert_for_size_lowering<"VINSERTF64x4Z", v16bf16x_info, v32bf16_info, + vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512]>; multiclass vinsert_for_mask_cast; defm : vextract_for_size_lowering<"VEXTRACTF32x4Z256", v16f16x_info, v8f16x_info, vextract128_extract, EXTRACT_get_vextract128_imm, [HasVLX]>; +defm : vextract_for_size_lowering<"VEXTRACTF32x4Z256", v16bf16x_info, v8bf16x_info, + vextract128_extract, EXTRACT_get_vextract128_imm, [HasVLX]>; // Codegen pattern with the alternative types extract VEC128 from VEC512 defm : vextract_for_size_lowering<"VEXTRACTI32x4Z", v32i16_info, v8i16x_info, @@ -803,6 +811,8 @@ defm : vextract_for_size_lowering<"VEXTRACTI32x4Z", v64i8_info, v16i8x_info, vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512]>; defm : vextract_for_size_lowering<"VEXTRACTF32x4Z", v32f16_info, v8f16x_info, vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512]>; +defm : vextract_for_size_lowering<"VEXTRACTF32x4Z", v32bf16_info, v8bf16x_info, + vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512]>; // Codegen pattern with the alternative types extract VEC256 from VEC512 defm : vextract_for_size_lowering<"VEXTRACTI64x4Z", v32i16_info, v16i16x_info, vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512]>; @@ -810,6 +820,8 @@ defm : vextract_for_size_lowering<"VEXTRACTI64x4Z", v64i8_info, v32i8x_info, vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512]>; defm : vextract_for_size_lowering<"VEXTRACTF64x4Z", v32f16_info, v16f16x_info, vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512]>; +defm : vextract_for_size_lowering<"VEXTRACTF64x4Z", v32bf16_info, v16bf16x_info, + vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512]>; // A 128-bit extract from bits [255:128] of a 512-bit vector should use a diff --git a/llvm/test/CodeGen/X86/bfloat.ll b/llvm/test/CodeGen/X86/bfloat.ll index 0042d477f3b3..6e3020d628ed 100644 --- a/llvm/test/CodeGen/X86/bfloat.ll +++ b/llvm/test/CodeGen/X86/bfloat.ll @@ -2805,3 +2805,24 @@ define <16 x bfloat> @concat_zero_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) { %a = shufflevector <8 x bfloat> %x, <8 x bfloat> zeroinitializer, <16 x i32> ret <16 x bfloat> %a } + +define <16 x bfloat> @concat_dup_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) { +; X86-LABEL: concat_dup_v8bf16: +; X86: # %bb.0: +; X86-NEXT: vmovddup {{.*#+}} xmm0 = xmm0[0,0] +; X86-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; X86-NEXT: retl +; +; SSE2-LABEL: concat_dup_v8bf16: +; SSE2: # %bb.0: +; SSE2-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0,0] +; SSE2-NEXT: retq +; +; AVX-LABEL: concat_dup_v8bf16: +; AVX: # %bb.0: +; AVX-NEXT: vmovddup {{.*#+}} xmm0 = xmm0[0,0] +; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX-NEXT: retq + %a = shufflevector <8 x bfloat> %x, <8 x bfloat> %y, <16 x i32> + ret <16 x bfloat> %a +} -- GitLab From 0207270494cda484f80abd3b654c871dc4cf8099 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Wed, 6 Mar 2024 08:52:27 +0800 Subject: [PATCH 220/929] [RISCV] Don't remove extends for i1 indices in mgather/mscatter (#83951) --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 9 +++++---- llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll | 16 ++++++++++++++++ llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll | 15 +++++++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index d56cba7de5d0..4c3dc63afd87 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -20042,11 +20042,12 @@ Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic( bool RISCVTargetLowering::shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const { - // We have indexed loads for all legal index types. Indices are always - // zero extended + // We have indexed loads for all supported EEW types. Indices are always + // zero extended. return Extend.getOpcode() == ISD::ZERO_EXTEND && - isTypeLegal(Extend.getValueType()) && - isTypeLegal(Extend.getOperand(0).getValueType()); + isTypeLegal(Extend.getValueType()) && + isTypeLegal(Extend.getOperand(0).getValueType()) && + Extend.getOperand(0).getValueType().getVectorElementType() != MVT::i1; } bool RISCVTargetLowering::shouldConvertFpToSat(unsigned Op, EVT FPVT, diff --git a/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll index 07dcddd9c686..f3ae03af7c78 100644 --- a/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll @@ -2153,3 +2153,19 @@ define @mgather_baseidx_nxv32i8(ptr %base, %v = call @llvm.masked.gather.nxv32i8.nxv32p0( %ptrs, i32 2, %m, %passthru) ret %v } + +define @mgather_baseidx_zext_nxv1i1_nxv1i8(ptr %base, %idxs, %m, %passthru) { +; CHECK-LABEL: mgather_baseidx_zext_nxv1i1_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, mu +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmerge.vim v10, v10, 1, v0 +; CHECK-NEXT: vmv1r.v v0, v8 +; CHECK-NEXT: vluxei8.v v9, (a0), v10, v0.t +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %eidxs = zext %idxs to + %ptrs = getelementptr inbounds i8, ptr %base, %eidxs + %v = call @llvm.masked.gather.nxv1i8.nxv1p0( %ptrs, i32 1, %m, %passthru) + ret %v +} diff --git a/llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll index dc67c64f3ffd..652e7a128a96 100644 --- a/llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/mscatter-sdnode.ll @@ -1831,3 +1831,18 @@ define void @mscatter_baseidx_nxv16i16_nxv16f64( %val0, %v1, %ptrs, i32 8, %m) ret void } + +define void @mscatter_baseidx_zext_nxv1i1_nxv1i8( %val, ptr %base, %idxs, %m) { +; CHECK-LABEL: mscatter_baseidx_zext_nxv1i1_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: vmerge.vim v10, v10, 1, v0 +; CHECK-NEXT: vmv1r.v v0, v9 +; CHECK-NEXT: vsoxei8.v v8, (a0), v10, v0.t +; CHECK-NEXT: ret + %eidxs = zext %idxs to + %ptrs = getelementptr inbounds i8, ptr %base, %eidxs + call void @llvm.masked.scatter.nxv1i8.nxv1p0( %val, %ptrs, i32 1, %m) + ret void +} -- GitLab From 6c39e3fa113d2956cb5b5f6769d2ad9a266377e5 Mon Sep 17 00:00:00 2001 From: Brandon Wu Date: Wed, 6 Mar 2024 09:12:14 +0800 Subject: [PATCH 221/929] [clang][RISCV] Reorder sema check for RVV type (#83553) Currently using the command `clang -cc1 -triple riscv64` to compile the code below: ``` #include void foo() { vfloat64m1_t f64m1; } ``` would get the error message "RISC-V type 'vfloat64m1_t' ... requires the 'zve64x' extension" which is supposed to be "RISC-V type 'vfloat64m1_t' ... requires the 'zve64d' extension". --- clang/lib/Sema/SemaChecking.cpp | 11 ++++++----- clang/test/Sema/riscv-vector-float64-check.c | 3 +-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index 9a2aa2120ca2..561764edd081 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -6391,10 +6391,14 @@ void Sema::checkRVVTypeSupport(QualType Ty, SourceLocation Loc, Decl *D) { unsigned EltSize = Context.getTypeSize(Info.ElementType); unsigned MinElts = Info.EC.getKnownMinValue(); + if (Info.ElementType->isSpecificBuiltinType(BuiltinType::Double) && + !TI.hasFeature("zve64d")) + Diag(Loc, diag::err_riscv_type_requires_extension, D) << Ty << "zve64d"; // (ELEN, LMUL) pairs of (8, mf8), (16, mf4), (32, mf2), (64, m1) requires at // least zve64x - if (((EltSize == 64 && Info.ElementType->isIntegerType()) || MinElts == 1) && - !TI.hasFeature("zve64x")) + else if (((EltSize == 64 && Info.ElementType->isIntegerType()) || + MinElts == 1) && + !TI.hasFeature("zve64x")) Diag(Loc, diag::err_riscv_type_requires_extension, D) << Ty << "zve64x"; else if (Info.ElementType->isFloat16Type() && !TI.hasFeature("zvfh") && !TI.hasFeature("zvfhmin")) @@ -6406,9 +6410,6 @@ void Sema::checkRVVTypeSupport(QualType Ty, SourceLocation Loc, Decl *D) { else if (Info.ElementType->isSpecificBuiltinType(BuiltinType::Float) && !TI.hasFeature("zve32f")) Diag(Loc, diag::err_riscv_type_requires_extension, D) << Ty << "zve32f"; - else if (Info.ElementType->isSpecificBuiltinType(BuiltinType::Double) && - !TI.hasFeature("zve64d")) - Diag(Loc, diag::err_riscv_type_requires_extension, D) << Ty << "zve64d"; // Given that caller already checked isRVVType() before calling this function, // if we don't have at least zve32x supported, then we need to emit error. else if (!TI.hasFeature("zve32x")) diff --git a/clang/test/Sema/riscv-vector-float64-check.c b/clang/test/Sema/riscv-vector-float64-check.c index ee7db3266395..f21ae5c0c704 100644 --- a/clang/test/Sema/riscv-vector-float64-check.c +++ b/clang/test/Sema/riscv-vector-float64-check.c @@ -1,5 +1,4 @@ -// RUN: %clang_cc1 -triple riscv64 -target-feature +f -target-feature +d \ -// RUN: -target-feature +zve64f -target-feature +zfh \ +// RUN: %clang_cc1 -triple riscv64 -target-feature +zve32x \ // RUN: -disable-O0-optnone -o - -fsyntax-only %s -verify // REQUIRES: riscv-registered-target #include -- GitLab From 2a1b09fee4b4b22f4f7189695ce3e858b91a8d69 Mon Sep 17 00:00:00 2001 From: Brandon Wu Date: Wed, 6 Mar 2024 09:12:35 +0800 Subject: [PATCH 222/929] [RISCV][SiFive] Add RISCVUsage for SiFive Intelligence Extensions (#84010) --- llvm/docs/RISCVUsage.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst index 582b4a5b830c..a1de8596480d 100644 --- a/llvm/docs/RISCVUsage.rst +++ b/llvm/docs/RISCVUsage.rst @@ -332,6 +332,15 @@ The current vendor extensions supported are: ``XSfvcp`` LLVM implements `version 1.0.0 of the SiFive Vector Coprocessor Interface (VCIX) Software Specification `__ by SiFive. All instructions are prefixed with `sf.vc.` as described in the specification, and the riscv-toolchain-convention document linked above. +``XSfvqmaccdod``, ``XSfvqmaccqoq`` + LLVM implements `version 1.1.0 of the SiFive Int8 Matrix Multiplication Extensions Specification `__ by SiFive. All instructions are prefixed with `sf.` as described in the specification linked above. + +``Xsfvfnrclipxfqf`` + LLVM implements `version 1.0.0 of the FP32-to-int8 Ranged Clip Instructions Extension Specification `__ by SiFive. All instructions are prefixed with `sf.` as described in the specification linked above. + +``Xsfvfwmaccqqq`` + LLVM implements `version 1.0.0 of the Matrix Multiply Accumulate Instruction Extension Specification `__ by SiFive. All instructions are prefixed with `sf.` as described in the specification linked above. + ``XCVbitmanip`` LLVM implements `version 1.0.0 of the CORE-V Bit Manipulation custom instructions specification `__ by OpenHW Group. All instructions are prefixed with `cv.` as described in the specification. -- GitLab From 11f74cd4bb487e956c92132b4991e8cde1dd1a81 Mon Sep 17 00:00:00 2001 From: Brandon Wu Date: Wed, 6 Mar 2024 09:25:42 +0800 Subject: [PATCH 223/929] [RISCV] Improve error message when the extension is not supported (#83989) If the "march" has some extension with version that is not supported, it returns the error message like: "error: invalid arch name 'some_arch', unsupported version number 2.0 for extension 'some_arch'", which is not precise enough, it should return the message that only tells users "the extension is not supported". --- clang/test/Driver/riscv-arch.c | 4 ++-- llvm/lib/Support/RISCVISAInfo.cpp | 26 +++++++++++++-------- llvm/unittests/Support/RISCVISAInfoTest.cpp | 15 ++++++++++++ 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/clang/test/Driver/riscv-arch.c b/clang/test/Driver/riscv-arch.c index c9e984e07cbe..8399b4e97f86 100644 --- a/clang/test/Driver/riscv-arch.c +++ b/clang/test/Driver/riscv-arch.c @@ -306,7 +306,7 @@ // RUN: not %clang --target=riscv32-unknown-elf -march=rv32ist2p0 -### %s \ // RUN: -fsyntax-only 2>&1 | FileCheck -check-prefix=RV32-SMINOR0 %s // RV32-SMINOR0: error: invalid arch name 'rv32ist2p0', -// RV32-SMINOR0: unsupported version number 2.0 for extension 'st' +// RV32-SMINOR0: unsupported standard supervisor-level extension 'st' // RUN: not %clang --target=riscv32-unknown-elf -march=rv32ixabc_ -### %s \ // RUN: -fsyntax-only 2>&1 | FileCheck -check-prefix=RV32-XSEP %s @@ -397,7 +397,7 @@ // RUN: not %clang --target=riscv32-unknown-elf -march=rv32izbb1p0zbs1p0 -menable-experimental-extensions -### %s \ // RUN: -fsyntax-only 2>&1 | FileCheck -check-prefix=RV32-EXPERIMENTAL-ZBB-ZBS-UNDERSCORE %s -// RV32-EXPERIMENTAL-ZBB-ZBS-UNDERSCORE: error: invalid arch name 'rv32izbb1p0zbs1p0', unsupported version number 1.0 for extension 'zbb1p0zbs' +// RV32-EXPERIMENTAL-ZBB-ZBS-UNDERSCORE: error: invalid arch name 'rv32izbb1p0zbs1p0', unsupported standard user-level extension 'zbb1p0zbs' // RUN: %clang --target=riscv32-unknown-elf -march=rv32izba1p0 -### %s \ // RUN: -fsyntax-only 2>&1 | FileCheck -check-prefix=RV32-ZBA %s diff --git a/llvm/lib/Support/RISCVISAInfo.cpp b/llvm/lib/Support/RISCVISAInfo.cpp index 54b4dcb22de8..6eec03fd6f70 100644 --- a/llvm/lib/Support/RISCVISAInfo.cpp +++ b/llvm/lib/Support/RISCVISAInfo.cpp @@ -531,6 +531,17 @@ std::vector RISCVISAInfo::toFeatures(bool AddAllExtensions, return Features; } +static Error getStringErrorForInvalidExt(std::string_view ExtName) { + if (ExtName.size() == 1) { + return createStringError(errc::invalid_argument, + "unsupported standard user-level extension '" + + ExtName + "'"); + } + return createStringError(errc::invalid_argument, + "unsupported " + getExtensionTypeDesc(ExtName) + + " '" + ExtName + "'"); +} + // Extensions may have a version number, and may be separated by // an underscore '_' e.g.: rv32i2_m2. // Version number is divided into major and minor version numbers, @@ -629,6 +640,9 @@ static Error getExtensionVersion(StringRef Ext, StringRef In, unsigned &Major, if (RISCVISAInfo::isSupportedExtension(Ext, Major, Minor)) return Error::success(); + if (!RISCVISAInfo::isSupportedExtension(Ext)) + return getStringErrorForInvalidExt(Ext); + std::string Error = "unsupported version number " + std::string(MajorStr); if (!MinorStr.empty()) Error += "." + MinorStr.str(); @@ -965,16 +979,8 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension, const std::string &ExtName = SeenExtAndVers.first; RISCVISAInfo::ExtensionVersion ExtVers = SeenExtAndVers.second; - if (!RISCVISAInfo::isSupportedExtension(ExtName)) { - if (ExtName.size() == 1) { - return createStringError(errc::invalid_argument, - "unsupported standard user-level extension '" + - ExtName + "'"); - } - return createStringError(errc::invalid_argument, - "unsupported " + getExtensionTypeDesc(ExtName) + - " '" + ExtName + "'"); - } + if (!RISCVISAInfo::isSupportedExtension(ExtName)) + return getStringErrorForInvalidExt(ExtName); ISAInfo->addExtension(ExtName, ExtVers); } diff --git a/llvm/unittests/Support/RISCVISAInfoTest.cpp b/llvm/unittests/Support/RISCVISAInfoTest.cpp index 8a627eca05cb..82cf4c639b61 100644 --- a/llvm/unittests/Support/RISCVISAInfoTest.cpp +++ b/llvm/unittests/Support/RISCVISAInfoTest.cpp @@ -218,6 +218,21 @@ TEST(ParseArchString, RejectsUnrecognizedExtensionNamesByDefault) { toString( RISCVISAInfo::parseArchString("rv64g_xmadeup", true).takeError()), "unsupported non-standard user-level extension 'xmadeup'"); + EXPECT_EQ( + toString(RISCVISAInfo::parseArchString("rv64ib1p0", true).takeError()), + "unsupported standard user-level extension 'b'"); + EXPECT_EQ( + toString( + RISCVISAInfo::parseArchString("rv32i_zmadeup1p0", true).takeError()), + "unsupported standard user-level extension 'zmadeup'"); + EXPECT_EQ( + toString( + RISCVISAInfo::parseArchString("rv64g_smadeup1p0", true).takeError()), + "unsupported standard supervisor-level extension 'smadeup'"); + EXPECT_EQ( + toString( + RISCVISAInfo::parseArchString("rv64g_xmadeup1p0", true).takeError()), + "unsupported non-standard user-level extension 'xmadeup'"); } TEST(ParseArchString, IgnoresUnrecognizedExtensionNamesWithIgnoreUnknown) { -- GitLab From aeda1a6e800e0dd6c91c0332b4db95094ad5b301 Mon Sep 17 00:00:00 2001 From: wanglei Date: Tue, 5 Mar 2024 19:44:28 +0800 Subject: [PATCH 224/929] [Clang][LoongArch] Precommit test for fix wrong return value type of __iocsrrd_h. NFC --- clang/test/CodeGen/LoongArch/intrinsic-la32.c | 29 ++++++++++++++----- clang/test/CodeGen/LoongArch/intrinsic-la64.c | 21 ++++++++++++-- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/clang/test/CodeGen/LoongArch/intrinsic-la32.c b/clang/test/CodeGen/LoongArch/intrinsic-la32.c index 93d54f511a9c..6a8d99880be3 100644 --- a/clang/test/CodeGen/LoongArch/intrinsic-la32.c +++ b/clang/test/CodeGen/LoongArch/intrinsic-la32.c @@ -169,8 +169,8 @@ unsigned int cpucfg(unsigned int a) { // LA32-LABEL: @rdtime( // LA32-NEXT: entry: -// LA32-NEXT: [[TMP0:%.*]] = tail call { i32, i32 } asm sideeffect "rdtimeh.w $0, $1\0A\09", "=&r,=&r"() #[[ATTR1:[0-9]+]], !srcloc !2 -// LA32-NEXT: [[TMP1:%.*]] = tail call { i32, i32 } asm sideeffect "rdtimel.w $0, $1\0A\09", "=&r,=&r"() #[[ATTR1]], !srcloc !3 +// LA32-NEXT: [[TMP0:%.*]] = tail call { i32, i32 } asm sideeffect "rdtimeh.w $0, $1\0A\09", "=&r,=&r"() #[[ATTR1:[0-9]+]], !srcloc [[META2:![0-9]+]] +// LA32-NEXT: [[TMP1:%.*]] = tail call { i32, i32 } asm sideeffect "rdtimel.w $0, $1\0A\09", "=&r,=&r"() #[[ATTR1]], !srcloc [[META3:![0-9]+]] // LA32-NEXT: ret void // void rdtime() { @@ -201,13 +201,28 @@ void loongarch_movgr2fcsr(int a) { __builtin_loongarch_movgr2fcsr(1, a); } -// CHECK-LABEL: @cacop_w( -// CHECK-NEXT: entry: -// CHECK-NEXT: tail call void @llvm.loongarch.cacop.w(i32 1, i32 [[A:%.*]], i32 1024) -// CHECK-NEXT: tail call void @llvm.loongarch.cacop.w(i32 1, i32 [[A]], i32 1024) -// CHECK-NEXT: ret void +// LA32-LABEL: @cacop_w( +// LA32-NEXT: entry: +// LA32-NEXT: tail call void @llvm.loongarch.cacop.w(i32 1, i32 [[A:%.*]], i32 1024) +// LA32-NEXT: tail call void @llvm.loongarch.cacop.w(i32 1, i32 [[A]], i32 1024) +// LA32-NEXT: ret void // void cacop_w(unsigned long int a) { __cacop_w(1, a, 1024); __builtin_loongarch_cacop_w(1, a, 1024); } + +// LA32-LABEL: @iocsrrd_h_result( +// LA32-NEXT: entry: +// LA32-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.loongarch.iocsrrd.h(i32 [[A:%.*]]) +// LA32-NEXT: [[TMP1:%.*]] = tail call i32 @llvm.loongarch.iocsrrd.h(i32 [[A]]) +// LA32-NEXT: [[CONV2:%.*]] = and i32 [[TMP0]], 255 +// LA32-NEXT: [[ADD:%.*]] = add i32 [[TMP1]], [[CONV2]] +// LA32-NEXT: [[CONV4:%.*]] = trunc i32 [[ADD]] to i16 +// LA32-NEXT: ret i16 [[CONV4]] +// +unsigned short iocsrrd_h_result(unsigned int a) { + unsigned short b = __iocsrrd_h(a); + unsigned short c = __builtin_loongarch_iocsrrd_h(a); + return b+c; +} diff --git a/clang/test/CodeGen/LoongArch/intrinsic-la64.c b/clang/test/CodeGen/LoongArch/intrinsic-la64.c index a740882eef54..48b6a7a3d227 100644 --- a/clang/test/CodeGen/LoongArch/intrinsic-la64.c +++ b/clang/test/CodeGen/LoongArch/intrinsic-la64.c @@ -387,7 +387,7 @@ unsigned int cpucfg(unsigned int a) { // CHECK-LABEL: @rdtime_d( // CHECK-NEXT: entry: -// CHECK-NEXT: [[TMP0:%.*]] = tail call { i64, i64 } asm sideeffect "rdtime.d $0, $1\0A\09", "=&r,=&r"() #[[ATTR1:[0-9]+]], !srcloc !2 +// CHECK-NEXT: [[TMP0:%.*]] = tail call { i64, i64 } asm sideeffect "rdtime.d $0, $1\0A\09", "=&r,=&r"() #[[ATTR1:[0-9]+]], !srcloc [[META2:![0-9]+]] // CHECK-NEXT: ret void // void rdtime_d() { @@ -396,8 +396,8 @@ void rdtime_d() { // CHECK-LABEL: @rdtime( // CHECK-NEXT: entry: -// CHECK-NEXT: [[TMP0:%.*]] = tail call { i32, i32 } asm sideeffect "rdtimeh.w $0, $1\0A\09", "=&r,=&r"() #[[ATTR1]], !srcloc !3 -// CHECK-NEXT: [[TMP1:%.*]] = tail call { i32, i32 } asm sideeffect "rdtimel.w $0, $1\0A\09", "=&r,=&r"() #[[ATTR1]], !srcloc !4 +// CHECK-NEXT: [[TMP0:%.*]] = tail call { i32, i32 } asm sideeffect "rdtimeh.w $0, $1\0A\09", "=&r,=&r"() #[[ATTR1]], !srcloc [[META3:![0-9]+]] +// CHECK-NEXT: [[TMP1:%.*]] = tail call { i32, i32 } asm sideeffect "rdtimel.w $0, $1\0A\09", "=&r,=&r"() #[[ATTR1]], !srcloc [[META4:![0-9]+]] // CHECK-NEXT: ret void // void rdtime() { @@ -427,3 +427,18 @@ void loongarch_movgr2fcsr(int a) { __movgr2fcsr(1, a); __builtin_loongarch_movgr2fcsr(1, a); } + +// CHECK-LABEL: @iocsrrd_h_result( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.loongarch.iocsrrd.h(i32 [[A:%.*]]) +// CHECK-NEXT: [[TMP1:%.*]] = tail call i32 @llvm.loongarch.iocsrrd.h(i32 [[A]]) +// CHECK-NEXT: [[CONV2:%.*]] = and i32 [[TMP0]], 255 +// CHECK-NEXT: [[ADD:%.*]] = add i32 [[TMP1]], [[CONV2]] +// CHECK-NEXT: [[CONV4:%.*]] = trunc i32 [[ADD]] to i16 +// CHECK-NEXT: ret i16 [[CONV4]] +// +unsigned short iocsrrd_h_result(unsigned int a) { + unsigned short b = __iocsrrd_h(a); + unsigned short c = __builtin_loongarch_iocsrrd_h(a); + return b+c; +} -- GitLab From 8b326d59467b941831942c651c585055b3d512e4 Mon Sep 17 00:00:00 2001 From: Qizhi Hu <836744285@qq.com> Date: Wed, 6 Mar 2024 09:27:32 +0800 Subject: [PATCH 225/929] [clang-tidy] fix false negative in cppcoreguidelines-missing-std-forward (#83987) Try to fix https://github.com/llvm/llvm-project/issues/83845 When `std::forward` is invoked in a function, make sure it uses correct parameter by checking if the bounded `var` equals the parameter. Co-authored-by: huqizhi <836744285@qq.com> --- .../cppcoreguidelines/MissingStdForwardCheck.cpp | 10 ++++++---- clang-tools-extra/docs/ReleaseNotes.rst | 3 ++- .../checkers/cppcoreguidelines/missing-std-forward.cpp | 10 ++++++++++ 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.cpp index c633683570f7..87fd8adf9970 100644 --- a/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.cpp +++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.cpp @@ -112,10 +112,12 @@ void MissingStdForwardCheck::registerMatchers(MatchFinder *Finder) { auto ForwardCallMatcher = callExpr( callExpr().bind("call"), argumentCountIs(1), - hasArgument( - 0, declRefExpr(to( - varDecl(optionally(equalsBoundNode("param"))).bind("var")))), - forCallable(anyOf(equalsBoundNode("func"), CapturedInLambda)), + hasArgument(0, declRefExpr(to(varDecl().bind("var")))), + forCallable( + anyOf(allOf(equalsBoundNode("func"), + functionDecl(hasAnyParameter(parmVarDecl(allOf( + equalsBoundNode("param"), equalsBoundNode("var")))))), + CapturedInLambda)), callee(unresolvedLookupExpr(hasAnyDeclaration( namedDecl(hasUnderlyingDecl(hasName("::std::forward")))))), diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst index 143ae230fc44..1b839a35c3ed 100644 --- a/clang-tools-extra/docs/ReleaseNotes.rst +++ b/clang-tools-extra/docs/ReleaseNotes.rst @@ -156,7 +156,8 @@ Changes in existing checks - Improved :doc:`cppcoreguidelines-missing-std-forward ` check by no longer - giving false positives for deleted functions. + giving false positives for deleted functions and fix false negative when some + parameters are forwarded, but other aren't. - Cleaned up :doc:`cppcoreguidelines-prefer-member-initializer ` diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/missing-std-forward.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/missing-std-forward.cpp index 20e43f04180f..9a50eabf619b 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/missing-std-forward.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/missing-std-forward.cpp @@ -95,6 +95,16 @@ void lambda_value_capture_copy(T&& t) { [&,t]() { T other = std::forward(t); }; } +template +void use(const X &x) {} + +template +void foo(X &&x, Y &&y) { + // CHECK-MESSAGES: :[[@LINE-1]]:21: warning: forwarding reference parameter 'y' is never forwarded inside the function body [cppcoreguidelines-missing-std-forward] + use(std::forward(x)); + use(y); +} + } // namespace positive_cases namespace negative_cases { -- GitLab From 889d99a50f38150570f6fca903d61ee9770bd932 Mon Sep 17 00:00:00 2001 From: Kolya Panchenko <87679760+nikolaypanchenko@users.noreply.github.com> Date: Tue, 5 Mar 2024 17:33:56 -0800 Subject: [PATCH 226/929] [TTI] Add alignment argument to TTI for compress/expand support (#83516) Since `llvm.compressstore` and `llvm.expandload` do require memory access, it's essential for some target to check if alignment is good to be able to lower them to target-specific instructions --- llvm/include/llvm/Analysis/TargetTransformInfo.h | 16 ++++++++-------- .../llvm/Analysis/TargetTransformInfoImpl.h | 8 ++++++-- llvm/lib/Analysis/TargetTransformInfo.cpp | 10 ++++++---- llvm/lib/Target/X86/X86TargetTransformInfo.cpp | 6 +++--- llvm/lib/Target/X86/X86TargetTransformInfo.h | 4 ++-- .../Scalar/ScalarizeMaskedMemIntrin.cpp | 8 ++++++-- 6 files changed, 31 insertions(+), 21 deletions(-) diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h index 58577a6b6eb5..4eab357f1b33 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -777,9 +777,9 @@ public: bool forceScalarizeMaskedScatter(VectorType *Type, Align Alignment) const; /// Return true if the target supports masked compress store. - bool isLegalMaskedCompressStore(Type *DataType) const; + bool isLegalMaskedCompressStore(Type *DataType, Align Alignment) const; /// Return true if the target supports masked expand load. - bool isLegalMaskedExpandLoad(Type *DataType) const; + bool isLegalMaskedExpandLoad(Type *DataType, Align Alignment) const; /// Return true if the target supports strided load. bool isLegalStridedLoadStore(Type *DataType, Align Alignment) const; @@ -1863,8 +1863,8 @@ public: Align Alignment) = 0; virtual bool forceScalarizeMaskedScatter(VectorType *DataType, Align Alignment) = 0; - virtual bool isLegalMaskedCompressStore(Type *DataType) = 0; - virtual bool isLegalMaskedExpandLoad(Type *DataType) = 0; + virtual bool isLegalMaskedCompressStore(Type *DataType, Align Alignment) = 0; + virtual bool isLegalMaskedExpandLoad(Type *DataType, Align Alignment) = 0; virtual bool isLegalStridedLoadStore(Type *DataType, Align Alignment) = 0; virtual bool isLegalAltInstr(VectorType *VecTy, unsigned Opcode0, unsigned Opcode1, @@ -2358,11 +2358,11 @@ public: Align Alignment) override { return Impl.forceScalarizeMaskedScatter(DataType, Alignment); } - bool isLegalMaskedCompressStore(Type *DataType) override { - return Impl.isLegalMaskedCompressStore(DataType); + bool isLegalMaskedCompressStore(Type *DataType, Align Alignment) override { + return Impl.isLegalMaskedCompressStore(DataType, Alignment); } - bool isLegalMaskedExpandLoad(Type *DataType) override { - return Impl.isLegalMaskedExpandLoad(DataType); + bool isLegalMaskedExpandLoad(Type *DataType, Align Alignment) override { + return Impl.isLegalMaskedExpandLoad(DataType, Alignment); } bool isLegalStridedLoadStore(Type *DataType, Align Alignment) override { return Impl.isLegalStridedLoadStore(DataType, Alignment); diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h index 13379cc126a4..95fb13d1c971 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -295,14 +295,18 @@ public: return false; } - bool isLegalMaskedCompressStore(Type *DataType) const { return false; } + bool isLegalMaskedCompressStore(Type *DataType, Align Alignment) const { + return false; + } bool isLegalAltInstr(VectorType *VecTy, unsigned Opcode0, unsigned Opcode1, const SmallBitVector &OpcodeMask) const { return false; } - bool isLegalMaskedExpandLoad(Type *DataType) const { return false; } + bool isLegalMaskedExpandLoad(Type *DataType, Align Alignment) const { + return false; + } bool isLegalStridedLoadStore(Type *DataType, Align Alignment) const { return false; diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp index 1f11f0d7dd62..15311be4dba2 100644 --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -492,12 +492,14 @@ bool TargetTransformInfo::forceScalarizeMaskedScatter(VectorType *DataType, return TTIImpl->forceScalarizeMaskedScatter(DataType, Alignment); } -bool TargetTransformInfo::isLegalMaskedCompressStore(Type *DataType) const { - return TTIImpl->isLegalMaskedCompressStore(DataType); +bool TargetTransformInfo::isLegalMaskedCompressStore(Type *DataType, + Align Alignment) const { + return TTIImpl->isLegalMaskedCompressStore(DataType, Alignment); } -bool TargetTransformInfo::isLegalMaskedExpandLoad(Type *DataType) const { - return TTIImpl->isLegalMaskedExpandLoad(DataType); +bool TargetTransformInfo::isLegalMaskedExpandLoad(Type *DataType, + Align Alignment) const { + return TTIImpl->isLegalMaskedExpandLoad(DataType, Alignment); } bool TargetTransformInfo::isLegalStridedLoadStore(Type *DataType, diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp index 4cca291a2456..d336ab9d309c 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -5938,7 +5938,7 @@ bool X86TTIImpl::isLegalBroadcastLoad(Type *ElementTy, ElementTy == Type::getDoubleTy(ElementTy->getContext()); } -bool X86TTIImpl::isLegalMaskedExpandLoad(Type *DataTy) { +bool X86TTIImpl::isLegalMaskedExpandLoad(Type *DataTy, Align Alignment) { if (!isa(DataTy)) return false; @@ -5962,8 +5962,8 @@ bool X86TTIImpl::isLegalMaskedExpandLoad(Type *DataTy) { ((IntWidth == 8 || IntWidth == 16) && ST->hasVBMI2()); } -bool X86TTIImpl::isLegalMaskedCompressStore(Type *DataTy) { - return isLegalMaskedExpandLoad(DataTy); +bool X86TTIImpl::isLegalMaskedCompressStore(Type *DataTy, Align Alignment) { + return isLegalMaskedExpandLoad(DataTy, Alignment); } bool X86TTIImpl::supportsGather() const { diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h index 07a3fff4f84b..1a5e6bc886aa 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.h +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h @@ -269,8 +269,8 @@ public: bool isLegalMaskedGatherScatter(Type *DataType, Align Alignment); bool isLegalMaskedGather(Type *DataType, Align Alignment); bool isLegalMaskedScatter(Type *DataType, Align Alignment); - bool isLegalMaskedExpandLoad(Type *DataType); - bool isLegalMaskedCompressStore(Type *DataType); + bool isLegalMaskedExpandLoad(Type *DataType, Align Alignment); + bool isLegalMaskedCompressStore(Type *DataType, Align Alignment); bool isLegalAltInstr(VectorType *VecTy, unsigned Opcode0, unsigned Opcode1, const SmallBitVector &OpcodeMask) const; bool hasDivRemOp(Type *DataType, bool IsSigned); diff --git a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp index f362dc5708b7..a4111fad5d9f 100644 --- a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp +++ b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp @@ -979,12 +979,16 @@ static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT, return true; } case Intrinsic::masked_expandload: - if (TTI.isLegalMaskedExpandLoad(CI->getType())) + if (TTI.isLegalMaskedExpandLoad( + CI->getType(), + CI->getAttributes().getParamAttrs(0).getAlignment().valueOrOne())) return false; scalarizeMaskedExpandLoad(DL, CI, DTU, ModifiedDT); return true; case Intrinsic::masked_compressstore: - if (TTI.isLegalMaskedCompressStore(CI->getArgOperand(0)->getType())) + if (TTI.isLegalMaskedCompressStore( + CI->getArgOperand(0)->getType(), + CI->getAttributes().getParamAttrs(1).getAlignment().valueOrOne())) return false; scalarizeMaskedCompressStore(DL, CI, DTU, ModifiedDT); return true; -- GitLab From 31c304ba7ba8bb5c236072df55640154ea49b588 Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Tue, 5 Mar 2024 17:20:06 -0800 Subject: [PATCH 227/929] [SCEV] Migrate some tests to be autogenerated In advance of a change which needs to update these. This batch was the "easy" ones, I'll be landing the harder set a few a time for easier review. --- .../ScalarEvolution/2008-11-18-Stride1.ll | 16 ++- .../ScalarEvolution/2008-11-18-Stride2.ll | 17 ++- .../ScalarEvolution/exhaustive-trip-counts.ll | 18 ++- .../ScalarEvolution/exponential-behavior.ll | 22 ++-- .../load-with-range-metadata.ll | 27 ++++- .../max-trip-count-address-space.ll | 32 +++-- llvm/test/Analysis/ScalarEvolution/pr25369.ll | 47 ++++++-- .../test/Analysis/ScalarEvolution/shift-op.ll | 111 ++++++++++++----- llvm/test/Analysis/ScalarEvolution/sle.ll | 16 ++- .../ScalarEvolution/trip-count-pow2.ll | 63 +++++++--- .../ScalarEvolution/trip-count-switch.ll | 16 ++- .../trip-count-unknown-stride.ll | 114 +++++++++++++----- .../Analysis/ScalarEvolution/trip-count12.ll | 16 ++- .../Analysis/ScalarEvolution/trip-count13.ll | 68 ++++++++--- 14 files changed, 434 insertions(+), 149 deletions(-) diff --git a/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride1.ll b/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride1.ll index 4a799bd94ced..dbd9b83a7831 100644 --- a/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride1.ll +++ b/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride1.ll @@ -1,7 +1,5 @@ -; RUN: opt < %s -disable-output "-passes=print" 2>&1 | FileCheck %s - -; CHECK: Loop %bb: backedge-taken count is ((-5 + %x) /u 3) -; CHECK: Loop %bb: constant max backedge-taken count is 1431655764 +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -disable-output "-passes=print" -scalar-evolution-classify-expressions=0 2>&1 | FileCheck %s ; ScalarEvolution can't compute a trip count because it doesn't know if @@ -9,6 +7,16 @@ ; be teaching it how to use a more elaborate trip count computation. define i32 @f(i32 %x) nounwind readnone { +; +; CHECK-LABEL: 'f' +; CHECK-NEXT: Determining loop execution counts for: @f +; CHECK-NEXT: Loop %bb: backedge-taken count is ((-5 + %x) /u 3) +; CHECK-NEXT: Loop %bb: constant max backedge-taken count is 1431655764 +; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is ((-5 + %x) /u 3) +; CHECK-NEXT: Loop %bb: Predicated backedge-taken count is ((-5 + %x) /u 3) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %bb: Trip multiple is 1 +; entry: %0 = icmp ugt i32 %x, 4 ; [#uses=1] br i1 %0, label %bb.nph, label %bb2 diff --git a/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride2.ll b/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride2.ll index 566f74868b86..ca472d4036cb 100644 --- a/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride2.ll +++ b/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride2.ll @@ -1,13 +1,20 @@ -; RUN: opt < %s -disable-output "-passes=print" 2>&1 2>&1 | FileCheck %s - -; CHECK: Loop %bb: backedge-taken count is (((-3 + (-1 * (1 umin (-3 + (-1 * %x) + (1000 umax (3 + %x))))) + (-1 * %x) + (1000 umax (3 + %x))) /u 3) + (1 umin (-3 + (-1 * %x) + (1000 umax (3 + %x))))) -; CHECK: Loop %bb: constant max backedge-taken count is 334 - +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -disable-output "-passes=print" -scalar-evolution-classify-expressions=0 2>&1 2>&1 | FileCheck %s ; This is a tricky testcase for unsigned wrap detection which ScalarEvolution ; doesn't yet know how to do. define i32 @f(i32 %x) nounwind readnone { +; +; CHECK-LABEL: 'f' +; CHECK-NEXT: Determining loop execution counts for: @f +; CHECK-NEXT: Loop %bb: backedge-taken count is (((-3 + (-1 * (1 umin (-3 + (-1 * %x) + (1000 umax (3 + %x))))) + (-1 * %x) + (1000 umax (3 + %x))) /u 3) + (1 umin (-3 + (-1 * %x) + (1000 umax (3 + %x))))) +; CHECK-NEXT: Loop %bb: constant max backedge-taken count is 334 +; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is (((-3 + (-1 * (1 umin (-3 + (-1 * %x) + (1000 umax (3 + %x))))) + (-1 * %x) + (1000 umax (3 + %x))) /u 3) + (1 umin (-3 + (-1 * %x) + (1000 umax (3 + %x))))) +; CHECK-NEXT: Loop %bb: Predicated backedge-taken count is (((-3 + (-1 * (1 umin (-3 + (-1 * %x) + (1000 umax (3 + %x))))) + (-1 * %x) + (1000 umax (3 + %x))) /u 3) + (1 umin (-3 + (-1 * %x) + (1000 umax (3 + %x))))) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %bb: Trip multiple is 1 +; entry: %0 = icmp ugt i32 %x, 999 ; [#uses=1] br i1 %0, label %bb2, label %bb.nph diff --git a/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll b/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll index 8a84737a15db..2892b5a65b22 100644 --- a/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll +++ b/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll @@ -1,14 +1,20 @@ -; RUN: opt -disable-output "-passes=print" < %s 2>&1 | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -disable-output "-passes=print" -scalar-evolution-classify-expressions=0 < %s 2>&1 | FileCheck %s target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" define void @f_0() { -; CHECK-LABEL: Printing analysis 'Scalar Evolution Analysis' for function 'f_0': -; CHECK: Loop %for.body: backedge-taken count is 5 -; CHECK: Loop %for.body: constant max backedge-taken count is 5 -; CHECK: Loop %for.body: Predicated backedge-taken count is 5 - +; +; CHECK-LABEL: 'f_0' +; CHECK-NEXT: Determining loop execution counts for: @f_0 +; CHECK-NEXT: Loop %for.body: backedge-taken count is 5 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 5 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 5 +; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 5 +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %for.body: Trip multiple is 6 +; entry: br label %for.body diff --git a/llvm/test/Analysis/ScalarEvolution/exponential-behavior.ll b/llvm/test/Analysis/ScalarEvolution/exponential-behavior.ll index 6632e64974fd..adeaaf236d67 100644 --- a/llvm/test/Analysis/ScalarEvolution/exponential-behavior.ll +++ b/llvm/test/Analysis/ScalarEvolution/exponential-behavior.ll @@ -1,13 +1,19 @@ -; RUN: opt -disable-output "-passes=print" < %s 2>&1 | FileCheck %s - -; CHECK: Printing analysis 'Scalar Evolution Analysis' for function 'f': - -; CHECK: Loop %loop: Unpredictable backedge-taken count. -; CHECK: Loop %loop: constant max backedge-taken count is 0 -; CHECK: Loop %loop: Unpredictable predicated backedge-taken count. - +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -disable-output "-passes=print" -scalar-evolution-classify-expressions=0 < %s 2>&1 | FileCheck %s define void @f(i32 %n, ptr %ptr) { +; +; CHECK-LABEL: 'f' +; CHECK-NEXT: Determining loop execution counts for: @f +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: exit count for loop: 0 +; CHECK-NEXT: exit count for be: ***COULDNOTCOMPUTE*** +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 +; CHECK-NEXT: symbolic max exit count for loop: 0 +; CHECK-NEXT: symbolic max exit count for be: ***COULDNOTCOMPUTE*** +; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. +; entry: br label %loop diff --git a/llvm/test/Analysis/ScalarEvolution/load-with-range-metadata.ll b/llvm/test/Analysis/ScalarEvolution/load-with-range-metadata.ll index 74d109eff754..dd781731894d 100644 --- a/llvm/test/Analysis/ScalarEvolution/load-with-range-metadata.ll +++ b/llvm/test/Analysis/ScalarEvolution/load-with-range-metadata.ll @@ -1,13 +1,22 @@ -; RUN: opt -disable-output "-passes=print" < %s 2>&1 | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -disable-output "-passes=print" -scalar-evolution-classify-expressions=0 < %s 2>&1 | FileCheck %s define i32 @slt_trip_count_with_range(ptr %ptr0, ptr %ptr1) { -; CHECK-LABEL: slt_trip_count_with_range +; +; CHECK-LABEL: 'slt_trip_count_with_range' +; CHECK-NEXT: Determining loop execution counts for: @slt_trip_count_with_range +; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %limit) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 98 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %limit) +; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %limit) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; entry: %limit = load i32, ptr %ptr0, !range !0 br label %loop loop: -; CHECK: Loop %loop: constant max backedge-taken count is 98 %index = phi i32 [ 0, %entry ], [ %index.inc, %loop ] %index.inc = add i32 %index, 1 %continue = icmp slt i32 %index.inc, %limit @@ -18,13 +27,21 @@ define i32 @slt_trip_count_with_range(ptr %ptr0, ptr %ptr1) { } define i32 @ult_trip_count_with_range(ptr %ptr0, ptr %ptr1) { -; CHECK-LABEL: ult_trip_count_with_range +; +; CHECK-LABEL: 'ult_trip_count_with_range' +; CHECK-NEXT: Determining loop execution counts for: @ult_trip_count_with_range +; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %limit) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 98 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %limit) +; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %limit) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; entry: %limit = load i32, ptr %ptr0, !range !0 br label %loop loop: -; CHECK: Loop %loop: constant max backedge-taken count is 98 %index = phi i32 [ 0, %entry ], [ %index.inc, %loop ] %index.inc = add i32 %index, 1 %continue = icmp ult i32 %index.inc, %limit diff --git a/llvm/test/Analysis/ScalarEvolution/max-trip-count-address-space.ll b/llvm/test/Analysis/ScalarEvolution/max-trip-count-address-space.ll index 4376548b4c6e..5af706b201de 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-trip-count-address-space.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-trip-count-address-space.ll @@ -1,14 +1,21 @@ -; RUN: opt < %s -disable-output "-passes=print" 2>&1 | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -disable-output "-passes=print" -scalar-evolution-classify-expressions=0 2>&1 | FileCheck %s ; ScalarEvolution should be able to understand the loop and eliminate the casts. target datalayout = "e-p:32:32:32-p1:16:16:16-p2:8:8:8-p4:64:64:64-n16:32:64" -; CHECK: {%d,+,4}<%bb>{{ U: [^ ]+ S: [^ ]+}}{{ *}} Exits: (-4 + (4 * (trunc i32 %n to i16)) + %d) - - define void @foo(ptr addrspace(1) nocapture %d, i32 %n) nounwind { -; CHECK: @foo +; +; CHECK-LABEL: 'foo' +; CHECK-NEXT: Determining loop execution counts for: @foo +; CHECK-NEXT: Loop %bb: backedge-taken count is (-1 + %n) +; CHECK-NEXT: Loop %bb: constant max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is (-1 + %n) +; CHECK-NEXT: Loop %bb: Predicated backedge-taken count is (-1 + %n) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %bb: Trip multiple is 1 +; entry: %0 = icmp sgt i32 %n, 0 ; [#uses=1] br i1 %0, label %bb.nph, label %return @@ -39,7 +46,16 @@ return: ; preds = %bb1.return_crit_edge, %entry } define void @test(ptr addrspace(1) %a, i32 %n) nounwind { -; CHECK: @test +; +; CHECK-LABEL: 'test' +; CHECK-NEXT: Determining loop execution counts for: @test +; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (zext i32 %n to i64)) +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (zext i32 %n to i64)) +; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (zext i32 %n to i64)) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %for.body: Trip multiple is 1 +; entry: %cmp1 = icmp sgt i32 %n, 0 br i1 %cmp1, label %for.body.lr.ph, label %for.end @@ -62,7 +78,3 @@ for.cond.for.end_crit_edge: ; preds = %for.body for.end: ; preds = %for.cond.for.end_crit_edge, %entry ret void } - -; CHECK: Determining loop execution counts for: @test -; CHECK-NEXT: backedge-taken count is -; CHECK-NEXT: constant max backedge-taken count is 2147483646 diff --git a/llvm/test/Analysis/ScalarEvolution/pr25369.ll b/llvm/test/Analysis/ScalarEvolution/pr25369.ll index fce008bbfca1..28ab5f8d9710 100644 --- a/llvm/test/Analysis/ScalarEvolution/pr25369.ll +++ b/llvm/test/Analysis/ScalarEvolution/pr25369.ll @@ -1,10 +1,30 @@ -; RUN: opt -disable-output "-passes=print" < %s 2>&1 | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -disable-output "-passes=print" -scalar-evolution-classify-expressions=0 < %s 2>&1 | FileCheck %s target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" define void @hoge1() { -; CHECK-LABEL: Classifying expressions for: @hoge1 +; +; CHECK-LABEL: 'hoge1' +; CHECK-NEXT: Determining loop execution counts for: @hoge1 +; CHECK-NEXT: Loop %bb13: backedge-taken count is (-2 + (2 * undef) + %tmp7 + %tmp6) +; CHECK-NEXT: Loop %bb13: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %bb13: symbolic max backedge-taken count is (-2 + (2 * undef) + %tmp7 + %tmp6) +; CHECK-NEXT: Loop %bb13: Predicated backedge-taken count is (-2 + (2 * undef) + %tmp7 + %tmp6) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %bb13: Trip multiple is 1 +; CHECK-NEXT: Loop %bb4: backedge-taken count is 20 +; CHECK-NEXT: Loop %bb4: constant max backedge-taken count is 20 +; CHECK-NEXT: Loop %bb4: symbolic max backedge-taken count is 20 +; CHECK-NEXT: Loop %bb4: Predicated backedge-taken count is 20 +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %bb4: Trip multiple is 21 +; CHECK-NEXT: Loop %bb2: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %bb2: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %bb2: Unpredictable symbolic max backedge-taken count. +; CHECK-NEXT: Loop %bb2: Unpredictable predicated backedge-taken count. +; bb: br i1 undef, label %bb4, label %bb2 @@ -26,8 +46,6 @@ bb4: ; preds = %bb4, %bb2, %bb %tmp12 = icmp eq i64 %tmp11, 64 br i1 %tmp12, label %bb3, label %bb4 -; CHECK: Loop %bb4: backedge-taken count is 20 -; CHECK: Loop %bb4: constant max backedge-taken count is 20 bb13: ; preds = %bb13, %bb3 %tmp14 = phi i64 [ 0, %bb3 ], [ %tmp15, %bb13 ] @@ -41,7 +59,24 @@ bb18: ; preds = %bb13 } define void @hoge2() { -; CHECK-LABEL: Classifying expressions for: @hoge2 +; +; CHECK-LABEL: 'hoge2' +; CHECK-NEXT: Determining loop execution counts for: @hoge2 +; CHECK-NEXT: Loop %bb13: backedge-taken count is (-2 + (2 * undef) + %tmp7 + %tmp6) +; CHECK-NEXT: Loop %bb13: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %bb13: symbolic max backedge-taken count is (-2 + (2 * undef) + %tmp7 + %tmp6) +; CHECK-NEXT: Loop %bb13: Predicated backedge-taken count is (-2 + (2 * undef) + %tmp7 + %tmp6) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %bb13: Trip multiple is 1 +; CHECK-NEXT: Loop %bb4: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %bb4: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %bb4: Unpredictable symbolic max backedge-taken count. +; CHECK-NEXT: Loop %bb4: Unpredictable predicated backedge-taken count. +; CHECK-NEXT: Loop %bb2: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %bb2: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %bb2: Unpredictable symbolic max backedge-taken count. +; CHECK-NEXT: Loop %bb2: Unpredictable predicated backedge-taken count. +; bb: br i1 undef, label %bb4, label %bb2 @@ -63,8 +98,6 @@ bb4: ; preds = %bb4, %bb2, %bb %tmp12 = icmp eq i64 %tmp11, 64 br i1 %tmp12, label %bb3, label %bb4 -; CHECK: Loop %bb4: Unpredictable backedge-taken count. -; CHECK: Loop %bb4: Unpredictable constant max backedge-taken count. bb13: ; preds = %bb13, %bb3 %tmp14 = phi i64 [ 0, %bb3 ], [ %tmp15, %bb13 ] diff --git a/llvm/test/Analysis/ScalarEvolution/shift-op.ll b/llvm/test/Analysis/ScalarEvolution/shift-op.ll index c4099ac7b8dc..41e746c2f6dd 100644 --- a/llvm/test/Analysis/ScalarEvolution/shift-op.ll +++ b/llvm/test/Analysis/ScalarEvolution/shift-op.ll @@ -1,8 +1,15 @@ -; RUN: opt -disable-output "-passes=print" < %s 2>&1 | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -disable-output "-passes=print" -scalar-evolution-classify-expressions=0 < %s 2>&1 | FileCheck %s define void @test0(i32 %init) { -; CHECK-LABEL: Classifying expressions for: @test0 -; CHECK: Loop %loop: constant max backedge-taken count is 32 +; +; CHECK-LABEL: 'test0' +; CHECK-NEXT: Determining loop execution counts for: @test0 +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 32 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 32 +; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. +; entry: br label %loop @@ -17,8 +24,14 @@ define void @test0(i32 %init) { } define void @test1(i32 %init) { -; CHECK-LABEL: Classifying expressions for: @test1 -; CHECK: Loop %loop: constant max backedge-taken count is 32 +; +; CHECK-LABEL: 'test1' +; CHECK-NEXT: Determining loop execution counts for: @test1 +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 32 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 32 +; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. +; entry: br label %loop @@ -33,11 +46,16 @@ define void @test1(i32 %init) { } define void @test2(i32 %init) { -; CHECK-LABEL: Determining loop execution counts for: @test2 -; CHECK: Loop %loop: Unpredictable constant max backedge-taken count. - +; ; Unpredictable because %iv could "stabilize" to either -1 or 0, ; depending on %init. +; CHECK-LABEL: 'test2' +; CHECK-NEXT: Determining loop execution counts for: @test2 +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. +; entry: br label %loop @@ -52,8 +70,14 @@ define void @test2(i32 %init) { } define void @test3(ptr %init.ptr) { -; CHECK-LABEL: Determining loop execution counts for: @test3 -; CHECK: Loop %loop: constant max backedge-taken count is 32 +; +; CHECK-LABEL: 'test3' +; CHECK-NEXT: Determining loop execution counts for: @test3 +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 32 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 32 +; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. +; entry: %init = load i32, ptr %init.ptr, !range !0 br label %loop @@ -69,8 +93,14 @@ define void @test3(ptr %init.ptr) { } define void @test4(ptr %init.ptr) { -; CHECK-LABEL: Classifying expressions for: @test4 -; CHECK-LABEL: Loop %loop: constant max backedge-taken count is 32 +; +; CHECK-LABEL: 'test4' +; CHECK-NEXT: Determining loop execution counts for: @test4 +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 32 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 32 +; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. +; entry: %init = load i32, ptr %init.ptr, !range !1 br label %loop @@ -86,10 +116,15 @@ define void @test4(ptr %init.ptr) { } define void @test5(ptr %init.ptr) { -; CHECK-LABEL: Determining loop execution counts for: @test5 -; CHECK: Loop %loop: Unpredictable constant max backedge-taken count. - +; ; %iv will "stabilize" to -1, so this is an infinite loop +; CHECK-LABEL: 'test5' +; CHECK-NEXT: Determining loop execution counts for: @test5 +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. +; entry: %init = load i32, ptr %init.ptr, !range !1 br label %loop @@ -105,10 +140,15 @@ define void @test5(ptr %init.ptr) { } define void @test6(i32 %init, i32 %shift.amt) { -; CHECK-LABEL: Determining loop execution counts for: @test6 -; CHECK: Loop %loop: Unpredictable constant max backedge-taken count. - +; ; Potentially infinite loop, since %shift.amt could be 0 +; CHECK-LABEL: 'test6' +; CHECK-NEXT: Determining loop execution counts for: @test6 +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. +; entry: br label %loop @@ -123,9 +163,14 @@ define void @test6(i32 %init, i32 %shift.amt) { } define void @test7(i32 %init) { -; CHECK-LABEL: Classifying expressions for: @test7 -; CHECK: Loop %loop: constant max backedge-taken count is 32 - +; +; CHECK-LABEL: 'test7' +; CHECK-NEXT: Determining loop execution counts for: @test7 +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 32 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 32 +; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. +; entry: br label %loop @@ -140,12 +185,16 @@ define void @test7(i32 %init) { } define void @test8(i32 %init) { -; CHECK-LABEL: Classifying expressions for: @test8 -; CHECK: Loop %loop: Unpredictable constant max backedge-taken count. - +; ; In this test case, %iv.test stabilizes to 127, not -1, so the loop ; is infinite. - +; CHECK-LABEL: 'test8' +; CHECK-NEXT: Determining loop execution counts for: @test8 +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. +; entry: br label %loop @@ -161,11 +210,15 @@ define void @test8(i32 %init) { } define void @test9() { -; CHECK-LABEL: Determining loop execution counts for: @test9 -; CHECK: Loop %loop: Unpredictable constant max backedge-taken count. - +; ; This is an infinite loop, make sure that it recognized as such. - +; CHECK-LABEL: 'test9' +; CHECK-NEXT: Determining loop execution counts for: @test9 +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. +; entry: br label %loop diff --git a/llvm/test/Analysis/ScalarEvolution/sle.ll b/llvm/test/Analysis/ScalarEvolution/sle.ll index e36333cf85b6..a1b078113017 100644 --- a/llvm/test/Analysis/ScalarEvolution/sle.ll +++ b/llvm/test/Analysis/ScalarEvolution/sle.ll @@ -1,13 +1,19 @@ -; RUN: opt -disable-output "-passes=print" < %s 2>&1 | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -disable-output "-passes=print" -scalar-evolution-classify-expressions=0 < %s 2>&1 | FileCheck %s ; ScalarEvolution should be able to use nsw information to prove that ; this loop has a finite trip count. -; CHECK: @le -; CHECK: Loop %for.body: backedge-taken count is %n -; CHECK: Loop %for.body: constant max backedge-taken count is 9223372036854775807 - define void @le(i64 %n, ptr nocapture %p) nounwind { +; CHECK-LABEL: 'le' +; CHECK-NEXT: Determining loop execution counts for: @le +; CHECK-NEXT: Loop %for.body: backedge-taken count is %n +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 9223372036854775807 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is %n +; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is %n +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %for.body: Trip multiple is 1 +; entry: %cmp6 = icmp slt i64 %n, 0 ; [#uses=1] br i1 %cmp6, label %for.end, label %for.body diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-pow2.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-pow2.ll index 584ad28aca8c..5b754489321d 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-pow2.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-pow2.ll @@ -1,6 +1,16 @@ -; RUN: opt < %s "-passes=print" -disable-output 2>&1 | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s "-passes=print" -scalar-evolution-classify-expressions=0 -disable-output 2>&1 | FileCheck %s define void @test1(i32 %n) { +; CHECK-LABEL: 'test1' +; CHECK-NEXT: Determining loop execution counts for: @test1 +; CHECK-NEXT: Loop %loop: backedge-taken count is ((-32 + (96 * %n)) /u 32) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 134217727 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-32 + (96 * %n)) /u 32) +; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-32 + (96 * %n)) /u 32) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; entry: %s = mul i32 %n, 96 br label %loop @@ -12,13 +22,19 @@ loop: exit: ret void -; CHECK-LABEL: @test1 -; CHECK: Loop %loop: backedge-taken count is ((-32 + (96 * %n)) /u 32) -; CHECK: Loop %loop: constant max backedge-taken count is 134217727 } ; PR19183 define i32 @test2(i32 %n) { +; CHECK-LABEL: 'test2' +; CHECK-NEXT: Determining loop execution counts for: @test2 +; CHECK-NEXT: Loop %loop: backedge-taken count is ((-32 + (32 * (%n /u 32))) /u 32) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 134217727 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-32 + (32 * (%n /u 32))) /u 32) +; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-32 + (32 * (%n /u 32))) /u 32) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; entry: %s = and i32 %n, -32 br label %loop @@ -30,12 +46,18 @@ loop: exit: ret i32 %i -; CHECK-LABEL: @test2 -; CHECK: Loop %loop: backedge-taken count is ((-32 + (32 * (%n /u 32))) /u 32) -; CHECK: Loop %loop: constant max backedge-taken count is 134217727 } define void @test3(i32 %n) { +; CHECK-LABEL: 'test3' +; CHECK-NEXT: Determining loop execution counts for: @test3 +; CHECK-NEXT: Loop %loop: backedge-taken count is ((-32 + (32 * %n)) /u 32) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 134217727 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-32 + (32 * %n)) /u 32) +; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-32 + (32 * %n)) /u 32) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; entry: %s = mul i32 %n, 96 br label %loop @@ -47,12 +69,18 @@ loop: exit: ret void -; CHECK-LABEL: @test3 -; CHECK: Loop %loop: backedge-taken count is ((-32 + (32 * %n)) /u 32) -; CHECK: Loop %loop: constant max backedge-taken count is 134217727 } define void @test4(i32 %n) { +; CHECK-LABEL: 'test4' +; CHECK-NEXT: Determining loop execution counts for: @test4 +; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (-1431655764 * %n)) /u 4) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 1073741823 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (-1431655764 * %n)) /u 4) +; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (-1431655764 * %n)) /u 4) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; entry: %s = mul i32 %n, 4 br label %loop @@ -64,12 +92,18 @@ loop: exit: ret void -; CHECK-LABEL: @test4 -; CHECK: Loop %loop: backedge-taken count is ((-4 + (-1431655764 * %n)) /u 4) -; CHECK: Loop %loop: constant max backedge-taken count is 1073741823 } define void @test5(i32 %n) { +; CHECK-LABEL: 'test5' +; CHECK-NEXT: Determining loop execution counts for: @test5 +; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * %n)) /u 4) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 1073741823 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * %n)) /u 4) +; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * %n)) /u 4) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; entry: %s = mul i32 %n, 4 br label %loop @@ -81,7 +115,4 @@ loop: exit: ret void -; CHECK-LABEL: @test5 -; CHECK: Loop %loop: backedge-taken count is ((-4 + (4 * %n)) /u 4) -; CHECK: Loop %loop: constant max backedge-taken count is 1073741823 } diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-switch.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-switch.ll index d2b6e1f4e7cc..d5ab589df924 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-switch.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-switch.ll @@ -1,8 +1,19 @@ -; RUN: opt < %s -disable-output "-passes=print" 2>&1 | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -disable-output "-passes=print" -scalar-evolution-classify-expressions=0 2>&1 | FileCheck %s declare void @foo() define void @test1() nounwind { +; +; CHECK-LABEL: 'test1' +; CHECK-NEXT: Determining loop execution counts for: @test1 +; CHECK-NEXT: Loop %for.cond: backedge-taken count is 2 +; CHECK-NEXT: Loop %for.cond: constant max backedge-taken count is 2 +; CHECK-NEXT: Loop %for.cond: symbolic max backedge-taken count is 2 +; CHECK-NEXT: Loop %for.cond: Predicated backedge-taken count is 2 +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %for.cond: Trip multiple is 3 +; entry: br label %for.cond @@ -24,7 +35,4 @@ if.end: ; preds = %for.cond, %if.then for.end: ; preds = %for.cond ret void -; CHECK-LABEL: @test1 -; CHECK: Loop %for.cond: backedge-taken count is 2 -; CHECK: Loop %for.cond: constant max backedge-taken count is 2 } diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-unknown-stride.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-unknown-stride.ll index fb559ad2f4e4..63fb99d29a9b 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-unknown-stride.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-unknown-stride.ll @@ -1,18 +1,26 @@ -; RUN: opt < %s -disable-output "-passes=print" 2>&1 | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -disable-output "-passes=print" -scalar-evolution-classify-expressions=0 2>&1 | FileCheck %s ; ScalarEvolution should be able to compute trip count of the loop by proving ; that this is not an infinite loop with side effects. -; CHECK-LABEL: Determining loop execution counts for: @foo1 -; CHECK: backedge-taken count is ((-1 + (%n smax %s)) /u %s) ; We should have a conservative estimate for the max backedge taken count for ; loops with unknown stride. -; CHECK: constant max backedge-taken count is -1 target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128" define void @foo1(ptr nocapture %A, i32 %n, i32 %s) mustprogress { +; +; CHECK-LABEL: 'foo1' +; CHECK-NEXT: Determining loop execution counts for: @foo1 +; CHECK-NEXT: Loop %for.body: backedge-taken count is ((-1 + (%n smax %s)) /u %s) +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((-1 + (%n smax %s)) /u %s) +; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-1 + (%n smax %s)) /u %s) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %for.body: Trip multiple is 1 +; entry: %cmp4 = icmp sgt i32 %n, 0 br i1 %cmp4, label %for.body, label %for.end @@ -33,14 +41,21 @@ for.end: ; preds = %for.body, %entry ; Check that we are able to compute trip count of a loop without an entry guard. -; CHECK: Determining loop execution counts for: @foo2 -; CHECK: backedge-taken count is ((((-1 * (1 umin ((-1 * %s) + (%n smax %s)))) + (-1 * %s) + (%n smax %s)) /u (1 umax %s)) + (1 umin ((-1 * %s) + (%n smax %s)))) ; We should have a conservative estimate for the max backedge taken count for ; loops with unknown stride. -; CHECK: constant max backedge-taken count is -1 define void @foo2(ptr nocapture %A, i32 %n, i32 %s) mustprogress { +; +; CHECK-LABEL: 'foo2' +; CHECK-NEXT: Determining loop execution counts for: @foo2 +; CHECK-NEXT: Loop %for.body: backedge-taken count is ((((-1 * (1 umin ((-1 * %s) + (%n smax %s)))) + (-1 * %s) + (%n smax %s)) /u (1 umax %s)) + (1 umin ((-1 * %s) + (%n smax %s)))) +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((((-1 * (1 umin ((-1 * %s) + (%n smax %s)))) + (-1 * %s) + (%n smax %s)) /u (1 umax %s)) + (1 umin ((-1 * %s) + (%n smax %s)))) +; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((((-1 * (1 umin ((-1 * %s) + (%n smax %s)))) + (-1 * %s) + (%n smax %s)) /u (1 umax %s)) + (1 umin ((-1 * %s) + (%n smax %s)))) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %for.body: Trip multiple is 1 +; entry: br label %for.body @@ -60,11 +75,16 @@ for.end: ; preds = %for.body, %entry ; Check that without mustprogress we don't make assumptions about infinite ; loops being UB. -; CHECK-LABEL: Determining loop execution counts for: @foo3 -; CHECK: Loop %for.body: Unpredictable backedge-taken count. -; CHECK: Loop %for.body: Unpredictable constant max backedge-taken count. define void @foo3(ptr nocapture %A, i32 %n, i32 %s) { +; +; CHECK-LABEL: 'foo3' +; CHECK-NEXT: Determining loop execution counts for: @foo3 +; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. +; entry: br label %for.body @@ -83,11 +103,18 @@ for.end: ; preds = %for.body, %entry } ; Same as foo2, but with mustprogress on loop, not function -; CHECK: Determining loop execution counts for: @foo4 -; CHECK: backedge-taken count is ((((-1 * (1 umin ((-1 * %s) + (%n smax %s)))) + (-1 * %s) + (%n smax %s)) /u (1 umax %s)) + (1 umin ((-1 * %s) + (%n smax %s)))) -; CHECK: constant max backedge-taken count is -1 define void @foo4(ptr nocapture %A, i32 %n, i32 %s) { +; +; CHECK-LABEL: 'foo4' +; CHECK-NEXT: Determining loop execution counts for: @foo4 +; CHECK-NEXT: Loop %for.body: backedge-taken count is ((((-1 * (1 umin ((-1 * %s) + (%n smax %s)))) + (-1 * %s) + (%n smax %s)) /u (1 umax %s)) + (1 umin ((-1 * %s) + (%n smax %s)))) +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((((-1 * (1 umin ((-1 * %s) + (%n smax %s)))) + (-1 * %s) + (%n smax %s)) /u (1 umax %s)) + (1 umin ((-1 * %s) + (%n smax %s)))) +; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((((-1 * (1 umin ((-1 * %s) + (%n smax %s)))) + (-1 * %s) + (%n smax %s)) /u (1 umax %s)) + (1 umin ((-1 * %s) + (%n smax %s)))) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %for.body: Trip multiple is 1 +; entry: br label %for.body @@ -106,14 +133,21 @@ for.end: ; preds = %for.body, %entry } ; A more complex case with pre-increment compare instead of post-increment. -; CHECK-LABEL: Determining loop execution counts for: @foo5 -; CHECK: Loop %for.body: backedge-taken count is ((((-1 * (1 umin ((-1 * %start) + (%n smax %start)))) + (-1 * %start) + (%n smax %start)) /u (1 umax %s)) + (1 umin ((-1 * %start) + (%n smax %start)))) ; We should have a conservative estimate for the max backedge taken count for ; loops with unknown stride. -; CHECK: constant max backedge-taken count is -1 define void @foo5(ptr nocapture %A, i32 %n, i32 %s, i32 %start) mustprogress { +; +; CHECK-LABEL: 'foo5' +; CHECK-NEXT: Determining loop execution counts for: @foo5 +; CHECK-NEXT: Loop %for.body: backedge-taken count is ((((-1 * (1 umin ((-1 * %start) + (%n smax %start)))) + (-1 * %start) + (%n smax %start)) /u (1 umax %s)) + (1 umin ((-1 * %start) + (%n smax %start)))) +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((((-1 * (1 umin ((-1 * %start) + (%n smax %start)))) + (-1 * %start) + (%n smax %start)) /u (1 umax %s)) + (1 umin ((-1 * %start) + (%n smax %start)))) +; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((((-1 * (1 umin ((-1 * %start) + (%n smax %start)))) + (-1 * %start) + (%n smax %start)) /u (1 umax %s)) + (1 umin ((-1 * %start) + (%n smax %start)))) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %for.body: Trip multiple is 1 +; entry: br label %for.body @@ -133,12 +167,16 @@ for.end: ; preds = %for.body, %entry ; FIXME: Currently we are more conservative for known zero stride than ; for unknown but potentially zero stride. -; CHECK-LABEL: Determining loop execution counts for: @zero_stride -; CHECK: Loop %for.body: Unpredictable backedge-taken count. -; CHECK: Loop %for.body: Unpredictable constant max backedge-taken count. -; CHECK: Loop %for.body: Unpredictable predicated backedge-taken count. ; Note that this function is well defined only when %n <=s 0 define void @zero_stride(ptr nocapture %A, i32 %n) { +; +; CHECK-LABEL: 'zero_stride' +; CHECK-NEXT: Determining loop execution counts for: @zero_stride +; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. +; entry: br label %for.body @@ -156,13 +194,17 @@ for.end: ; preds = %for.body, %entry ret void } -; CHECK-LABEL: Determining loop execution counts for: @zero_stride_ub -; CHECK: Loop %for.body: Unpredictable backedge-taken count. -; CHECK: Loop %for.body: Unpredictable constant max backedge-taken count. -; CHECK: Loop %for.body: Unpredictable predicated backedge-taken count. ; Note that this function will always execute undefined behavior and thus ; any value is valid for a backedge taken count. define void @zero_stride_ub(ptr nocapture %A) { +; +; CHECK-LABEL: 'zero_stride_ub' +; CHECK-NEXT: Determining loop execution counts for: @zero_stride_ub +; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. +; entry: br label %for.body @@ -181,11 +223,18 @@ for.end: ; preds = %for.body, %entry } ; When %zero = 0, this loop is only well defined if %n < 0 and thus BTC = 0. -; CHECK-LABEL: Determining loop execution counts for: @zero_stride_symbolic -; CHECK: Loop %for.body: backedge-taken count is ((((-1 * (1 umin ((-1 * %zero) + (%n smax %zero)))) + (-1 * %zero) + (%n smax %zero)) /u (1 umax %zero)) + (1 umin ((-1 * %zero) + (%n smax %zero)))) -; CHECK: Loop %for.body: constant max backedge-taken count is -1 define void @zero_stride_symbolic(ptr nocapture %A, i32 %n, i32 %zero) { +; +; CHECK-LABEL: 'zero_stride_symbolic' +; CHECK-NEXT: Determining loop execution counts for: @zero_stride_symbolic +; CHECK-NEXT: Loop %for.body: backedge-taken count is ((((-1 * (1 umin ((-1 * %zero) + (%n smax %zero)))) + (-1 * %zero) + (%n smax %zero)) /u (1 umax %zero)) + (1 umin ((-1 * %zero) + (%n smax %zero)))) +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((((-1 * (1 umin ((-1 * %zero) + (%n smax %zero)))) + (-1 * %zero) + (%n smax %zero)) /u (1 umax %zero)) + (1 umin ((-1 * %zero) + (%n smax %zero)))) +; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((((-1 * (1 umin ((-1 * %zero) + (%n smax %zero)))) + (-1 * %zero) + (%n smax %zero)) /u (1 umax %zero)) + (1 umin ((-1 * %zero) + (%n smax %zero)))) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %for.body: Trip multiple is 1 +; entry: br label %for.body @@ -204,11 +253,16 @@ for.end: ; preds = %for.body, %entry } -; CHECK-LABEL: Determining loop execution counts for: @zero_stride_varying_rhs -; CHECK: Loop %for.body: Unpredictable backedge-taken count. -; CHECK: Loop %for.body: Unpredictable constant max backedge-taken count define void @zero_stride_varying_rhs(ptr nocapture %A, ptr %n_p, i32 %zero) { +; +; CHECK-LABEL: 'zero_stride_varying_rhs' +; CHECK-NEXT: Determining loop execution counts for: @zero_stride_varying_rhs +; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. +; entry: br label %for.body diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count12.ll b/llvm/test/Analysis/ScalarEvolution/trip-count12.ll index 6fc804ac229d..9eae3cc9b2eb 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count12.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count12.ll @@ -1,10 +1,18 @@ -; RUN: opt < %s -disable-output "-passes=print" 2>&1 | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -disable-output "-passes=print" -scalar-evolution-classify-expressions=0 2>&1 | FileCheck %s -; CHECK: Determining loop execution counts for: @test -; CHECK: Loop %for.body: backedge-taken count is ((-2 + %len) /u 2) -; CHECK: Loop %for.body: constant max backedge-taken count is 1073741823 define zeroext i16 @test(ptr nocapture %p, i32 %len) nounwind readonly { +; +; CHECK-LABEL: 'test' +; CHECK-NEXT: Determining loop execution counts for: @test +; CHECK-NEXT: Loop %for.body: backedge-taken count is ((-2 + %len) /u 2) +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 1073741823 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((-2 + %len) /u 2) +; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-2 + %len) /u 2) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %for.body: Trip multiple is 1 +; entry: %cmp2 = icmp sgt i32 %len, 1 br i1 %cmp2, label %for.body.preheader, label %for.end diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count13.ll b/llvm/test/Analysis/ScalarEvolution/trip-count13.ll index d2bd34a2424b..9dd615a04f82 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count13.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count13.ll @@ -1,7 +1,18 @@ -; RUN: opt -S -disable-output "-passes=print" < %s 2>&1 | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -disable-output "-passes=print" -scalar-evolution-classify-expressions=0 < %s 2>&1 | FileCheck %s define void @u_0(i8 %rhs) { ; E.g.: %rhs = 255, %start = 99, backedge taken 156 times +; +; CHECK-LABEL: 'u_0' +; CHECK-NEXT: Determining loop execution counts for: @u_0 +; CHECK-NEXT: Loop %loop: backedge-taken count is (-100 + (-1 * %rhs) + ((100 + %rhs) umax %rhs)) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -100, actual taken count either this or zero. +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-100 + (-1 * %rhs) + ((100 + %rhs) umax %rhs)), actual taken count either this or zero. +; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-100 + (-1 * %rhs) + ((100 + %rhs) umax %rhs)) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; entry: %start = add i8 %rhs, 100 br label %loop @@ -12,15 +23,22 @@ loop: %iv.cmp = icmp ult i8 %iv, %rhs br i1 %iv.cmp, label %loop, label %leave -; CHECK-LABEL: Determining loop execution counts for: @u_0 -; CHECK-NEXT: Loop %loop: backedge-taken count is (-100 + (-1 * %rhs) + ((100 + %rhs) umax %rhs)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -100, actual taken count either this or zero. leave: ret void } define void @u_1(i8 %start) { +; +; CHECK-LABEL: 'u_1' +; CHECK-NEXT: Determining loop execution counts for: @u_1 +; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 * %start) + ((-100 + %start) umax %start)) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -100, actual taken count either this or zero. +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 * %start) + ((-100 + %start) umax %start)), actual taken count either this or zero. +; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 * %start) + ((-100 + %start) umax %start)) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; entry: ; E.g.: %start = 99, %rhs = 255, backedge taken 156 times %rhs = add i8 %start, -100 @@ -32,15 +50,22 @@ loop: %iv.cmp = icmp ult i8 %iv, %rhs br i1 %iv.cmp, label %loop, label %leave -; CHECK-LABEL: Determining loop execution counts for: @u_1 -; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 * %start) + ((-100 + %start) umax %start)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -100, actual taken count either this or zero. leave: ret void } define void @s_0(i8 %rhs) { +; +; CHECK-LABEL: 's_0' +; CHECK-NEXT: Determining loop execution counts for: @s_0 +; CHECK-NEXT: Loop %loop: backedge-taken count is (-100 + (-1 * %rhs) + ((100 + %rhs) smax %rhs)) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -100, actual taken count either this or zero. +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-100 + (-1 * %rhs) + ((100 + %rhs) smax %rhs)), actual taken count either this or zero. +; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-100 + (-1 * %rhs) + ((100 + %rhs) smax %rhs)) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; entry: ; E.g.: %rhs = 127, %start = -29, backedge taken 156 times %start = add i8 %rhs, 100 @@ -52,15 +77,22 @@ loop: %iv.cmp = icmp slt i8 %iv, %rhs br i1 %iv.cmp, label %loop, label %leave -; CHECK-LABEL: Determining loop execution counts for: @s_0 -; CHECK-NEXT: Loop %loop: backedge-taken count is (-100 + (-1 * %rhs) + ((100 + %rhs) smax %rhs)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -100, actual taken count either this or zero. leave: ret void } define void @s_1(i8 %start) { +; +; CHECK-LABEL: 's_1' +; CHECK-NEXT: Determining loop execution counts for: @s_1 +; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 * %start) + ((-100 + %start) smax %start)) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -100, actual taken count either this or zero. +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 * %start) + ((-100 + %start) smax %start)), actual taken count either this or zero. +; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 * %start) + ((-100 + %start) smax %start)) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; entry: ; E.g.: start = -29, %rhs = 127, %backedge taken 156 times %rhs = add i8 %start, -100 @@ -72,15 +104,22 @@ loop: %iv.cmp = icmp slt i8 %iv, %rhs br i1 %iv.cmp, label %loop, label %leave -; CHECK-LABEL: Determining loop execution counts for: @s_1 -; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 * %start) + ((-100 + %start) smax %start)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -100, actual taken count either this or zero. leave: ret void } define void @s_2(i8 %start) { +; +; CHECK-LABEL: 's_2' +; CHECK-NEXT: Determining loop execution counts for: @s_2 +; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 * ((-100 + %start) smin %start)) + %start) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 * ((-100 + %start) smin %start)) + %start) +; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 * ((-100 + %start) smin %start)) + %start) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; entry: %rhs = add i8 %start, -100 br label %loop @@ -91,9 +130,6 @@ loop: %iv.cmp = icmp sgt i8 %iv, %rhs br i1 %iv.cmp, label %loop, label %leave -; CHECK-LABEL: Determining loop execution counts for: @s_2 -; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 * ((-100 + %start) smin %start)) + %start) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 leave: ret void -- GitLab From 716042a63f26cd020eb72960f72fa97b9a197382 Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Tue, 5 Mar 2024 18:00:46 -0800 Subject: [PATCH 228/929] Rename llvm::ThreadPool -> llvm::DefaultThreadPool (NFC) (#83702) The base class llvm::ThreadPoolInterface will be renamed llvm::ThreadPool in a subsequent commit. This is a breaking change: clients who use to create a ThreadPool must now create a DefaultThreadPool instead. --- bolt/lib/Core/ParallelUtilities.cpp | 4 ++-- bolt/tools/merge-fdata/merge-fdata.cpp | 2 +- .../clang-doc/tool/ClangDocMain.cpp | 2 +- .../tool/FindAllSymbolsMain.cpp | 2 +- clang/lib/Tooling/AllTUsExecution.cpp | 2 +- clang/tools/clang-scan-deps/ClangScanDeps.cpp | 2 +- lld/MachO/Writer.cpp | 2 +- lldb/source/Core/Debugger.cpp | 4 ++-- llvm/docs/ORCv2.rst | 2 +- .../SpeculativeJIT/SpeculativeJIT.cpp | 2 +- llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h | 2 +- llvm/include/llvm/Support/ThreadPool.h | 7 +++--- llvm/lib/CodeGen/ParallelCG.cpp | 2 +- llvm/lib/DWARFLinker/Classic/DWARFLinker.cpp | 2 +- .../DWARFLinker/Parallel/DWARFLinkerImpl.cpp | 2 +- llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp | 2 +- llvm/lib/ExecutionEngine/Orc/LLJIT.cpp | 4 ++-- llvm/lib/LTO/LTO.cpp | 2 +- llvm/lib/LTO/LTOBackend.cpp | 2 +- llvm/lib/LTO/ThinLTOCodeGenerator.cpp | 4 ++-- llvm/lib/Support/BalancedPartitioning.cpp | 2 +- llvm/tools/dsymutil/dsymutil.cpp | 2 +- llvm/tools/llvm-cov/CodeCoverage.cpp | 2 +- llvm/tools/llvm-cov/CoverageExporterJson.cpp | 2 +- llvm/tools/llvm-cov/CoverageReport.cpp | 4 ++-- .../tools/llvm-debuginfod/llvm-debuginfod.cpp | 2 +- llvm/tools/llvm-profdata/llvm-profdata.cpp | 2 +- llvm/tools/llvm-reduce/deltas/Delta.cpp | 2 +- llvm/unittests/ADT/LazyAtomicPointerTest.cpp | 4 ++-- llvm/unittests/Debuginfod/HTTPServerTests.cpp | 16 +++++++------- llvm/unittests/Support/ParallelTest.cpp | 2 +- llvm/unittests/Support/ThreadPool.cpp | 22 +++++++++---------- .../Support/ThreadSafeAllocatorTest.cpp | 6 ++--- mlir/include/mlir/IR/MLIRContext.h | 2 +- mlir/lib/CAPI/IR/Support.cpp | 2 +- mlir/lib/ExecutionEngine/AsyncRuntime.cpp | 2 +- mlir/lib/IR/MLIRContext.cpp | 4 ++-- 37 files changed, 65 insertions(+), 66 deletions(-) diff --git a/bolt/lib/Core/ParallelUtilities.cpp b/bolt/lib/Core/ParallelUtilities.cpp index 88d9444a6a2b..5f5e96e0e788 100644 --- a/bolt/lib/Core/ParallelUtilities.cpp +++ b/bolt/lib/Core/ParallelUtilities.cpp @@ -49,7 +49,7 @@ namespace ParallelUtilities { namespace { /// A single thread pool that is used to run parallel tasks -std::unique_ptr ThreadPoolPtr; +std::unique_ptr ThreadPoolPtr; unsigned computeCostFor(const BinaryFunction &BF, const PredicateTy &SkipPredicate, @@ -106,7 +106,7 @@ ThreadPoolInterface &getThreadPool() { if (ThreadPoolPtr.get()) return *ThreadPoolPtr; - ThreadPoolPtr = std::make_unique( + ThreadPoolPtr = std::make_unique( llvm::hardware_concurrency(opts::ThreadCount)); return *ThreadPoolPtr; } diff --git a/bolt/tools/merge-fdata/merge-fdata.cpp b/bolt/tools/merge-fdata/merge-fdata.cpp index c6dfd3cfdc56..f2ac5ad4492e 100644 --- a/bolt/tools/merge-fdata/merge-fdata.cpp +++ b/bolt/tools/merge-fdata/merge-fdata.cpp @@ -316,7 +316,7 @@ void mergeLegacyProfiles(const SmallVectorImpl &Filenames) { // least 4 tasks. ThreadPoolStrategy S = optimal_concurrency( std::max(Filenames.size() / 4, static_cast(1))); - ThreadPool Pool(S); + DefaultThreadPool Pool(S); DenseMap ParsedProfiles( Pool.getMaxConcurrency()); for (const auto &Filename : Filenames) diff --git a/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp b/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp index 22bdb5de22d8..21b581fa6df2 100644 --- a/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp +++ b/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp @@ -238,7 +238,7 @@ Example usage for a project using a compile commands database: Error = false; llvm::sys::Mutex IndexMutex; // ExecutorConcurrency is a flag exposed by AllTUsExecution.h - llvm::ThreadPool Pool(llvm::hardware_concurrency(ExecutorConcurrency)); + llvm::DefaultThreadPool Pool(llvm::hardware_concurrency(ExecutorConcurrency)); for (auto &Group : USRToBitcode) { Pool.async([&]() { std::vector> Infos; diff --git a/clang-tools-extra/clang-include-fixer/find-all-symbols/tool/FindAllSymbolsMain.cpp b/clang-tools-extra/clang-include-fixer/find-all-symbols/tool/FindAllSymbolsMain.cpp index b2d0efecc206..298b02e77cb0 100644 --- a/clang-tools-extra/clang-include-fixer/find-all-symbols/tool/FindAllSymbolsMain.cpp +++ b/clang-tools-extra/clang-include-fixer/find-all-symbols/tool/FindAllSymbolsMain.cpp @@ -89,7 +89,7 @@ bool Merge(llvm::StringRef MergeDir, llvm::StringRef OutputFile) { // Load all symbol files in MergeDir. { - llvm::ThreadPool Pool; + llvm::DefaultThreadPool Pool; for (llvm::sys::fs::directory_iterator Dir(MergeDir, EC), DirEnd; Dir != DirEnd && !EC; Dir.increment(EC)) { // Parse YAML files in parallel. diff --git a/clang/lib/Tooling/AllTUsExecution.cpp b/clang/lib/Tooling/AllTUsExecution.cpp index f327d0139941..9cad8680447b 100644 --- a/clang/lib/Tooling/AllTUsExecution.cpp +++ b/clang/lib/Tooling/AllTUsExecution.cpp @@ -115,7 +115,7 @@ llvm::Error AllTUsToolExecutor::execute( auto &Action = Actions.front(); { - llvm::ThreadPool Pool(llvm::hardware_concurrency(ThreadCount)); + llvm::DefaultThreadPool Pool(llvm::hardware_concurrency(ThreadCount)); for (std::string File : Files) { Pool.async( [&](std::string Path) { diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index 9811d2a87533..d042fecc3dbe 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -869,7 +869,7 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { DependencyScanningService Service(ScanMode, Format, OptimizeArgs, EagerLoadModules); - llvm::ThreadPool Pool(llvm::hardware_concurrency(NumThreads)); + llvm::DefaultThreadPool Pool(llvm::hardware_concurrency(NumThreads)); std::vector> WorkerTools; for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I) WorkerTools.push_back(std::make_unique(Service)); diff --git a/lld/MachO/Writer.cpp b/lld/MachO/Writer.cpp index 65b598d1d7c4..9b0a32c136e8 100644 --- a/lld/MachO/Writer.cpp +++ b/lld/MachO/Writer.cpp @@ -66,7 +66,7 @@ public: template void run(); - ThreadPool threadPool; + DefaultThreadPool threadPool; std::unique_ptr &buffer; uint64_t addr = 0; uint64_t fileOff = 0; diff --git a/lldb/source/Core/Debugger.cpp b/lldb/source/Core/Debugger.cpp index 1b25527abf98..9d62b2a908f7 100644 --- a/lldb/source/Core/Debugger.cpp +++ b/lldb/source/Core/Debugger.cpp @@ -104,7 +104,7 @@ static std::recursive_mutex *g_debugger_list_mutex_ptr = nullptr; // NOTE: intentional leak to avoid issues with C++ destructor chain static Debugger::DebuggerList *g_debugger_list_ptr = nullptr; // NOTE: intentional leak to avoid issues with C++ destructor chain -static llvm::ThreadPool *g_thread_pool = nullptr; +static llvm::DefaultThreadPoolThreadPool *g_thread_pool = nullptr; static constexpr OptionEnumValueElement g_show_disassembly_enum_values[] = { { @@ -609,7 +609,7 @@ void Debugger::Initialize(LoadPluginCallbackType load_plugin_callback) { "Debugger::Initialize called more than once!"); g_debugger_list_mutex_ptr = new std::recursive_mutex(); g_debugger_list_ptr = new DebuggerList(); - g_thread_pool = new llvm::ThreadPool(llvm::optimal_concurrency()); + g_thread_pool = new llvm::DefaultThreadPool(llvm::optimal_concurrency()); g_load_plugin_callback = load_plugin_callback; } diff --git a/llvm/docs/ORCv2.rst b/llvm/docs/ORCv2.rst index add05e05a80e..910ef5b9f3d0 100644 --- a/llvm/docs/ORCv2.rst +++ b/llvm/docs/ORCv2.rst @@ -738,7 +738,7 @@ or creating any Modules attached to it. E.g. ThreadSafeContext TSCtx(std::make_unique()); - ThreadPool TP(NumThreads); + DefaultThreadPool TP(NumThreads); JITStack J; for (auto &ModulePath : ModulePaths) { diff --git a/llvm/examples/SpeculativeJIT/SpeculativeJIT.cpp b/llvm/examples/SpeculativeJIT/SpeculativeJIT.cpp index fdd376d82da5..0d97d379d227 100644 --- a/llvm/examples/SpeculativeJIT/SpeculativeJIT.cpp +++ b/llvm/examples/SpeculativeJIT/SpeculativeJIT.cpp @@ -136,7 +136,7 @@ private: std::unique_ptr ES; DataLayout DL; MangleAndInterner Mangle{*ES, DL}; - ThreadPool CompileThreads{llvm::hardware_concurrency(NumThreads)}; + DefaultThreadPool CompileThreads{llvm::hardware_concurrency(NumThreads)}; JITDylib &MainJD; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h b/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h index 923976b182d1..76d16e63df28 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h @@ -254,7 +254,7 @@ protected: DataLayout DL; Triple TT; - std::unique_ptr CompileThreads; + std::unique_ptr CompileThreads; std::unique_ptr ObjLinkingLayer; std::unique_ptr ObjTransformLayer; diff --git a/llvm/include/llvm/Support/ThreadPool.h b/llvm/include/llvm/Support/ThreadPool.h index 93f02729f047..014b7a09d7c8 100644 --- a/llvm/include/llvm/Support/ThreadPool.h +++ b/llvm/include/llvm/Support/ThreadPool.h @@ -212,8 +212,7 @@ private: /// Maximum number of threads to potentially grow this pool to. const unsigned MaxThreadCount; }; - -#endif // LLVM_ENABLE_THREADS Disabled +#endif // LLVM_ENABLE_THREADS /// A non-threaded implementation. class SingleThreadExecutor : public ThreadPoolInterface { @@ -253,9 +252,9 @@ private: }; #if LLVM_ENABLE_THREADS -using ThreadPool = StdThreadPool; +using DefaultThreadPool = StdThreadPool; #else -using ThreadPool = SingleThreadExecutor; +using DefaultThreadPool = SingleThreadExecutor; #endif /// A group of tasks to be run on a thread pool. Thread pool tasks in different diff --git a/llvm/lib/CodeGen/ParallelCG.cpp b/llvm/lib/CodeGen/ParallelCG.cpp index 43b23368ead2..ceb64b2badab 100644 --- a/llvm/lib/CodeGen/ParallelCG.cpp +++ b/llvm/lib/CodeGen/ParallelCG.cpp @@ -52,7 +52,7 @@ void llvm::splitCodeGen( // Create ThreadPool in nested scope so that threads will be joined // on destruction. { - ThreadPool CodegenThreadPool(hardware_concurrency(OSs.size())); + DefaultThreadPool CodegenThreadPool(hardware_concurrency(OSs.size())); int ThreadCount = 0; SplitModule( diff --git a/llvm/lib/DWARFLinker/Classic/DWARFLinker.cpp b/llvm/lib/DWARFLinker/Classic/DWARFLinker.cpp index 4f5a4e2ffc70..9b581a6c9ab7 100644 --- a/llvm/lib/DWARFLinker/Classic/DWARFLinker.cpp +++ b/llvm/lib/DWARFLinker/Classic/DWARFLinker.cpp @@ -2935,7 +2935,7 @@ Error DWARFLinker::link() { } EmitLambda(); } else { - ThreadPool Pool(hardware_concurrency(2)); + DefaultThreadPool Pool(hardware_concurrency(2)); Pool.async(AnalyzeAll); Pool.async(CloneAll); Pool.wait(); diff --git a/llvm/lib/DWARFLinker/Parallel/DWARFLinkerImpl.cpp b/llvm/lib/DWARFLinker/Parallel/DWARFLinkerImpl.cpp index a052969e74c0..49b08997eb9c 100644 --- a/llvm/lib/DWARFLinker/Parallel/DWARFLinkerImpl.cpp +++ b/llvm/lib/DWARFLinker/Parallel/DWARFLinkerImpl.cpp @@ -192,7 +192,7 @@ Error DWARFLinkerImpl::link() { Context->InputDWARFFile.unload(); } } else { - ThreadPool Pool(llvm::parallel::strategy); + DefaultThreadPool Pool(llvm::parallel::strategy); for (std::unique_ptr &Context : ObjectContexts) Pool.async([&]() { // Link object file. diff --git a/llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp b/llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp index 3a28cd412de9..ff6b560d1172 100644 --- a/llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp +++ b/llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp @@ -601,7 +601,7 @@ Error DwarfTransformer::convert(uint32_t NumThreads, OutputAggregator &Out) { // Now parse all DIEs in case we have cross compile unit references in a // thread pool. - ThreadPool pool(hardware_concurrency(NumThreads)); + DefaultThreadPool pool(hardware_concurrency(NumThreads)); for (const auto &CU : DICtx.compile_units()) pool.async([&CU]() { CU->getUnitDIE(false /*CUDieOnly*/); }); pool.wait(); diff --git a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp index 833dcb9d5bf2..79adda5b7bc0 100644 --- a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp +++ b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -972,8 +972,8 @@ LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) if (S.NumCompileThreads > 0) { InitHelperTransformLayer->setCloneToNewContextOnEmit(true); - CompileThreads = - std::make_unique(hardware_concurrency(S.NumCompileThreads)); + CompileThreads = std::make_unique( + hardware_concurrency(S.NumCompileThreads)); ES->setDispatchTask([this](std::unique_ptr T) { // FIXME: We should be able to use move-capture here, but ThreadPool's // AsyncTaskTys are std::functions rather than unique_functions diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index 34a49c8588b2..9c93ec70da77 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -1409,7 +1409,7 @@ public: namespace { class InProcessThinBackend : public ThinBackendProc { - ThreadPool BackendThreadPool; + DefaultThreadPool BackendThreadPool; AddStreamFn AddStream; FileCache Cache; std::set CfiFunctionDefs; diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp index 6cfe67779b1a..71e8849dc3cc 100644 --- a/llvm/lib/LTO/LTOBackend.cpp +++ b/llvm/lib/LTO/LTOBackend.cpp @@ -431,7 +431,7 @@ static void splitCodeGen(const Config &C, TargetMachine *TM, AddStreamFn AddStream, unsigned ParallelCodeGenParallelismLevel, Module &Mod, const ModuleSummaryIndex &CombinedIndex) { - ThreadPool CodegenThreadPool( + DefaultThreadPool CodegenThreadPool( heavyweight_hardware_concurrency(ParallelCodeGenParallelismLevel)); unsigned ThreadCount = 0; const Target *T = &TM->getTarget(); diff --git a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp index 8fd181846f0c..8f517eb50dc7 100644 --- a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp +++ b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp @@ -980,7 +980,7 @@ void ThinLTOCodeGenerator::run() { if (CodeGenOnly) { // Perform only parallel codegen and return. - ThreadPool Pool; + DefaultThreadPool Pool; int count = 0; for (auto &Mod : Modules) { Pool.async([&](int count) { @@ -1126,7 +1126,7 @@ void ThinLTOCodeGenerator::run() { // Parallel optimizer + codegen { - ThreadPool Pool(heavyweight_hardware_concurrency(ThreadCount)); + DefaultThreadPool Pool(heavyweight_hardware_concurrency(ThreadCount)); for (auto IndexCount : ModulesOrdering) { auto &Mod = Modules[IndexCount]; Pool.async([&](int count) { diff --git a/llvm/lib/Support/BalancedPartitioning.cpp b/llvm/lib/Support/BalancedPartitioning.cpp index cb6ba6117994..f4254b50d26c 100644 --- a/llvm/lib/Support/BalancedPartitioning.cpp +++ b/llvm/lib/Support/BalancedPartitioning.cpp @@ -82,7 +82,7 @@ void BalancedPartitioning::run(std::vector &Nodes) const { Nodes.size(), Config.SplitDepth, Config.IterationsPerSplit)); std::optional TP; #if LLVM_ENABLE_THREADS - ThreadPool TheThreadPool; + DefaultThreadPool TheThreadPool; if (Config.TaskSplitDepth > 1) TP.emplace(TheThreadPool); #endif diff --git a/llvm/tools/dsymutil/dsymutil.cpp b/llvm/tools/dsymutil/dsymutil.cpp index b0e988c6f8e4..25e281c415e7 100644 --- a/llvm/tools/dsymutil/dsymutil.cpp +++ b/llvm/tools/dsymutil/dsymutil.cpp @@ -734,7 +734,7 @@ int dsymutil_main(int argc, char **argv, const llvm::ToolContext &) { S.ThreadsRequested = DebugMapPtrsOrErr->size(); S.Limit = true; } - ThreadPool Threads(S); + DefaultThreadPool Threads(S); // If there is more than one link to execute, we need to generate // temporary files. diff --git a/llvm/tools/llvm-cov/CodeCoverage.cpp b/llvm/tools/llvm-cov/CodeCoverage.cpp index 049e89d1a230..1e5bfbe5c3aa 100644 --- a/llvm/tools/llvm-cov/CodeCoverage.cpp +++ b/llvm/tools/llvm-cov/CodeCoverage.cpp @@ -1217,7 +1217,7 @@ int CodeCoverageTool::doShow(int argc, const char **argv, ShowFilenames); } else { // In -output-dir mode, it's safe to use multiple threads to print files. - ThreadPool Pool(S); + DefaultThreadPool Pool(S); for (const std::string &SourceFile : SourceFiles) Pool.async(&CodeCoverageTool::writeSourceFileView, this, SourceFile, Coverage.get(), Printer.get(), ShowFilenames); diff --git a/llvm/tools/llvm-cov/CoverageExporterJson.cpp b/llvm/tools/llvm-cov/CoverageExporterJson.cpp index a424bbe06e0e..9a8c7c94f061 100644 --- a/llvm/tools/llvm-cov/CoverageExporterJson.cpp +++ b/llvm/tools/llvm-cov/CoverageExporterJson.cpp @@ -277,7 +277,7 @@ json::Array renderFiles(const coverage::CoverageMapping &Coverage, S = heavyweight_hardware_concurrency(SourceFiles.size()); S.Limit = true; } - ThreadPool Pool(S); + DefaultThreadPool Pool(S); json::Array FileArray; std::mutex FileArrayMutex; diff --git a/llvm/tools/llvm-cov/CoverageReport.cpp b/llvm/tools/llvm-cov/CoverageReport.cpp index 8cc073e4def8..49a35f2a943e 100644 --- a/llvm/tools/llvm-cov/CoverageReport.cpp +++ b/llvm/tools/llvm-cov/CoverageReport.cpp @@ -465,7 +465,7 @@ std::vector CoverageReport::prepareFileReports( S = heavyweight_hardware_concurrency(Files.size()); S.Limit = true; } - ThreadPool Pool(S); + DefaultThreadPool Pool(S); std::vector FileReports; FileReports.reserve(Files.size()); @@ -580,7 +580,7 @@ Expected DirectoryCoverageReport::prepareDirectoryReports( PoolS = heavyweight_hardware_concurrency(Files.size()); PoolS.Limit = true; } - ThreadPool Pool(PoolS); + DefaultThreadPool Pool(PoolS); TPool = &Pool; LCPStack = {RootLCP}; diff --git a/llvm/tools/llvm-debuginfod/llvm-debuginfod.cpp b/llvm/tools/llvm-debuginfod/llvm-debuginfod.cpp index 9d347dbd68f3..44d656148a4e 100644 --- a/llvm/tools/llvm-debuginfod/llvm-debuginfod.cpp +++ b/llvm/tools/llvm-debuginfod/llvm-debuginfod.cpp @@ -127,7 +127,7 @@ int llvm_debuginfod_main(int argc, char **argv, const llvm::ToolContext &) { for (const std::string &Path : ScanPaths) Paths.push_back(Path); - ThreadPool Pool(hardware_concurrency(MaxConcurrency)); + DefaultThreadPool Pool(hardware_concurrency(MaxConcurrency)); DebuginfodLog Log; DebuginfodCollection Collection(Paths, Log, Pool, MinInterval); DebuginfodServer Server(Log, Collection); diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp index 577a8825fcaa..8400b0769944 100644 --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -898,7 +898,7 @@ static void mergeInstrProfile(const WeightedFileVector &Inputs, loadInput(Input, Remapper, Correlator.get(), ProfiledBinary, Contexts[0].get()); } else { - ThreadPool Pool(hardware_concurrency(NumThreads)); + DefaultThreadPool Pool(hardware_concurrency(NumThreads)); // Load the inputs in parallel (N/NumThreads serial steps). unsigned Ctx = 0; diff --git a/llvm/tools/llvm-reduce/deltas/Delta.cpp b/llvm/tools/llvm-reduce/deltas/Delta.cpp index 569117e70d6b..4b84921618e1 100644 --- a/llvm/tools/llvm-reduce/deltas/Delta.cpp +++ b/llvm/tools/llvm-reduce/deltas/Delta.cpp @@ -222,7 +222,7 @@ void llvm::runDeltaPass(TestRunner &Test, ReductionFunc ExtractChunksFromModule, std::unique_ptr ChunkThreadPoolPtr; if (NumJobs > 1) ChunkThreadPoolPtr = - std::make_unique(hardware_concurrency(NumJobs)); + std::make_unique(hardware_concurrency(NumJobs)); bool FoundAtLeastOneNewUninterestingChunkWithCurrentGranularity; do { diff --git a/llvm/unittests/ADT/LazyAtomicPointerTest.cpp b/llvm/unittests/ADT/LazyAtomicPointerTest.cpp index efead0bdf0a3..355822565426 100644 --- a/llvm/unittests/ADT/LazyAtomicPointerTest.cpp +++ b/llvm/unittests/ADT/LazyAtomicPointerTest.cpp @@ -18,7 +18,7 @@ namespace { TEST(LazyAtomicPointer, loadOrGenerate) { int Value = 0; LazyAtomicPointer Ptr; - ThreadPool Threads; + DefaultThreadPool Threads; for (unsigned I = 0; I < 4; ++I) Threads.async([&]() { Ptr.loadOrGenerate([&]() { @@ -38,7 +38,7 @@ TEST(LazyAtomicPointer, loadOrGenerate) { TEST(LazyAtomicPointer, BusyState) { int Value = 0; LazyAtomicPointer Ptr; - ThreadPool Threads; + DefaultThreadPool Threads; std::mutex BusyLock, EndLock; std::condition_variable Busy, End; diff --git a/llvm/unittests/Debuginfod/HTTPServerTests.cpp b/llvm/unittests/Debuginfod/HTTPServerTests.cpp index b0af2f850576..cd1d5f2d9fc7 100644 --- a/llvm/unittests/Debuginfod/HTTPServerTests.cpp +++ b/llvm/unittests/Debuginfod/HTTPServerTests.cpp @@ -92,7 +92,7 @@ TEST_F(HTTPClientServerTest, Hello) { Expected PortOrErr = Server.bind(); EXPECT_THAT_EXPECTED(PortOrErr, Succeeded()); unsigned Port = *PortOrErr; - ThreadPool Pool(hardware_concurrency(1)); + DefaultThreadPool Pool(hardware_concurrency(1)); Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); }); std::string Url = "http://localhost:" + utostr(Port); HTTPRequest Request(Url); @@ -116,7 +116,7 @@ TEST_F(HTTPClientServerTest, LambdaHandlerHello) { Expected PortOrErr = Server.bind(); EXPECT_THAT_EXPECTED(PortOrErr, Succeeded()); unsigned Port = *PortOrErr; - ThreadPool Pool(hardware_concurrency(1)); + DefaultThreadPool Pool(hardware_concurrency(1)); Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); }); std::string Url = "http://localhost:" + utostr(Port); HTTPRequest Request(Url); @@ -135,7 +135,7 @@ TEST_F(HTTPClientServerTest, StreamingHello) { Expected PortOrErr = Server.bind(); EXPECT_THAT_EXPECTED(PortOrErr, Succeeded()); unsigned Port = *PortOrErr; - ThreadPool Pool(hardware_concurrency(1)); + DefaultThreadPool Pool(hardware_concurrency(1)); Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); }); std::string Url = "http://localhost:" + utostr(Port); HTTPRequest Request(Url); @@ -167,7 +167,7 @@ TEST_F(HTTPClientServerTest, StreamingFileResponse) { Expected PortOrErr = Server.bind(); EXPECT_THAT_EXPECTED(PortOrErr, Succeeded()); unsigned Port = *PortOrErr; - ThreadPool Pool(hardware_concurrency(1)); + DefaultThreadPool Pool(hardware_concurrency(1)); Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); }); std::string Url = "http://localhost:" + utostr(Port); HTTPRequest Request(Url); @@ -203,7 +203,7 @@ TEST_F(HTTPClientServerTest, StreamingMissingFileResponse) { Expected PortOrErr = Server.bind(); EXPECT_THAT_EXPECTED(PortOrErr, Succeeded()); unsigned Port = *PortOrErr; - ThreadPool Pool(hardware_concurrency(1)); + DefaultThreadPool Pool(hardware_concurrency(1)); Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); }); std::string Url = "http://localhost:" + utostr(Port); HTTPRequest Request(Url); @@ -220,7 +220,7 @@ TEST_F(HTTPClientServerTest, ClientTimeout) { Expected PortOrErr = Server.bind(); EXPECT_THAT_EXPECTED(PortOrErr, Succeeded()); unsigned Port = *PortOrErr; - ThreadPool Pool(hardware_concurrency(1)); + DefaultThreadPool Pool(hardware_concurrency(1)); Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); }); std::string Url = "http://localhost:" + utostr(Port); HTTPClient Client; @@ -257,7 +257,7 @@ TEST_F(HTTPClientServerTest, PathMatching) { Expected PortOrErr = Server.bind(); EXPECT_THAT_EXPECTED(PortOrErr, Succeeded()); unsigned Port = *PortOrErr; - ThreadPool Pool(hardware_concurrency(1)); + DefaultThreadPool Pool(hardware_concurrency(1)); Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); }); std::string Url = "http://localhost:" + utostr(Port) + "/abc/1/2"; HTTPRequest Request(Url); @@ -289,7 +289,7 @@ TEST_F(HTTPClientServerTest, FirstPathMatched) { Expected PortOrErr = Server.bind(); EXPECT_THAT_EXPECTED(PortOrErr, Succeeded()); unsigned Port = *PortOrErr; - ThreadPool Pool(hardware_concurrency(1)); + DefaultThreadPool Pool(hardware_concurrency(1)); Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); }); std::string Url = "http://localhost:" + utostr(Port) + "/abc/1/2"; HTTPRequest Request(Url); diff --git a/llvm/unittests/Support/ParallelTest.cpp b/llvm/unittests/Support/ParallelTest.cpp index 53ef9fa25e82..91250f01a3c1 100644 --- a/llvm/unittests/Support/ParallelTest.cpp +++ b/llvm/unittests/Support/ParallelTest.cpp @@ -160,7 +160,7 @@ TEST(Parallel, ParallelNestedTaskGroup) { }); }; - ThreadPool Pool; + DefaultThreadPool Pool; Pool.async(Fn); Pool.async(Fn); diff --git a/llvm/unittests/Support/ThreadPool.cpp b/llvm/unittests/Support/ThreadPool.cpp index 1da8e056019d..d74c625d1229 100644 --- a/llvm/unittests/Support/ThreadPool.cpp +++ b/llvm/unittests/Support/ThreadPool.cpp @@ -140,7 +140,7 @@ TYPED_TEST(ThreadPoolTest, AsyncBarrier) { std::atomic_int checked_in{0}; - TypeParam Pool; + DefaultThreadPool Pool; for (size_t i = 0; i < 5; ++i) { Pool.async([this, &checked_in] { this->waitForMainThread(); @@ -160,7 +160,7 @@ TYPED_TEST(ThreadPoolTest, AsyncBarrierArgs) { // Test that async works with a function requiring multiple parameters. std::atomic_int checked_in{0}; - ThreadPool Pool; + DefaultThreadPool Pool; for (size_t i = 0; i < 5; ++i) { Pool.async(TestFunc, std::ref(checked_in), i); } @@ -170,7 +170,7 @@ TYPED_TEST(ThreadPoolTest, AsyncBarrierArgs) { TYPED_TEST(ThreadPoolTest, Async) { CHECK_UNSUPPORTED(); - ThreadPool Pool; + DefaultThreadPool Pool; std::atomic_int i{0}; Pool.async([this, &i] { this->waitForMainThread(); @@ -185,7 +185,7 @@ TYPED_TEST(ThreadPoolTest, Async) { TYPED_TEST(ThreadPoolTest, GetFuture) { CHECK_UNSUPPORTED(); - ThreadPool Pool(hardware_concurrency(2)); + DefaultThreadPool Pool(hardware_concurrency(2)); std::atomic_int i{0}; Pool.async([this, &i] { this->waitForMainThread(); @@ -201,7 +201,7 @@ TYPED_TEST(ThreadPoolTest, GetFuture) { TYPED_TEST(ThreadPoolTest, GetFutureWithResult) { CHECK_UNSUPPORTED(); - ThreadPool Pool(hardware_concurrency(2)); + DefaultThreadPool Pool(hardware_concurrency(2)); auto F1 = Pool.async([] { return 1; }); auto F2 = Pool.async([] { return 2; }); @@ -213,7 +213,7 @@ TYPED_TEST(ThreadPoolTest, GetFutureWithResult) { TYPED_TEST(ThreadPoolTest, GetFutureWithResultAndArgs) { CHECK_UNSUPPORTED(); - ThreadPool Pool(hardware_concurrency(2)); + DefaultThreadPool Pool(hardware_concurrency(2)); auto Fn = [](int x) { return x; }; auto F1 = Pool.async(Fn, 1); auto F2 = Pool.async(Fn, 2); @@ -229,7 +229,7 @@ TYPED_TEST(ThreadPoolTest, PoolDestruction) { // Test that we are waiting on destruction std::atomic_int checked_in{0}; { - ThreadPool Pool; + DefaultThreadPool Pool; for (size_t i = 0; i < 5; ++i) { Pool.async([this, &checked_in] { this->waitForMainThread(); @@ -250,7 +250,7 @@ TYPED_TEST(ThreadPoolTest, Groups) { ThreadPoolStrategy S = hardware_concurrency(2); if (S.compute_thread_count() < 2) GTEST_SKIP(); - ThreadPool Pool(S); + DefaultThreadPool Pool(S); typename TestFixture::PhaseResetHelper Helper(this); ThreadPoolTaskGroup Group1(Pool); ThreadPoolTaskGroup Group2(Pool); @@ -288,7 +288,7 @@ TYPED_TEST(ThreadPoolTest, Groups) { // Check recursive tasks. TYPED_TEST(ThreadPoolTest, RecursiveGroups) { CHECK_UNSUPPORTED(); - ThreadPool Pool; + DefaultThreadPool Pool; ThreadPoolTaskGroup Group(Pool); std::atomic_int checked_in1{0}; @@ -323,7 +323,7 @@ TYPED_TEST(ThreadPoolTest, RecursiveWaitDeadlock) { ThreadPoolStrategy S = hardware_concurrency(2); if (S.compute_thread_count() < 2) GTEST_SKIP(); - ThreadPool Pool(S); + DefaultThreadPool Pool(S); typename TestFixture::PhaseResetHelper Helper(this); ThreadPoolTaskGroup Group(Pool); @@ -378,7 +378,7 @@ ThreadPoolTest::RunOnAllSockets(ThreadPoolStrategy S) { std::mutex AllThreadsLock; unsigned Active = 0; - ThreadPool Pool(S); + DefaultThreadPool Pool(S); for (size_t I = 0; I < S.compute_thread_count(); ++I) { Pool.async([&] { { diff --git a/llvm/unittests/Support/ThreadSafeAllocatorTest.cpp b/llvm/unittests/Support/ThreadSafeAllocatorTest.cpp index d9a85b435ebd..b3d9430fc0f3 100644 --- a/llvm/unittests/Support/ThreadSafeAllocatorTest.cpp +++ b/llvm/unittests/Support/ThreadSafeAllocatorTest.cpp @@ -77,7 +77,7 @@ TEST(ThreadSafeAllocatorTest, AllocWait) { // Get the allocation from the allocator first since this requires a lock. Alloc.applyLocked( [&](MockAllocator &Alloc) { C = &Alloc.getAllocCondition(); }); - ThreadPool Threads; + DefaultThreadPool Threads; // First allocation of 1 byte. Threads.async([&Alloc]() { char *P = (char *)Alloc.Allocate(1, alignof(char)); @@ -104,7 +104,7 @@ TEST(ThreadSafeAllocatorTest, AllocWait) { TEST(ThreadSafeAllocatorTest, AllocWithAlign) { ThreadSafeAllocator Alloc; - ThreadPool Threads; + DefaultThreadPool Threads; for (unsigned Index = 1; Index < 100; ++Index) Threads.async( @@ -123,7 +123,7 @@ TEST(ThreadSafeAllocatorTest, AllocWithAlign) { TEST(ThreadSafeAllocatorTest, SpecificBumpPtrAllocator) { ThreadSafeAllocator> Alloc; - ThreadPool Threads; + DefaultThreadPool Threads; for (unsigned Index = 1; Index < 100; ++Index) Threads.async( diff --git a/mlir/include/mlir/IR/MLIRContext.h b/mlir/include/mlir/IR/MLIRContext.h index 2ad35d8f78ee..11e5329f43e6 100644 --- a/mlir/include/mlir/IR/MLIRContext.h +++ b/mlir/include/mlir/IR/MLIRContext.h @@ -50,7 +50,7 @@ class IRUnit; /// To control better thread spawning, an externally owned ThreadPool can be /// injected in the context. For example: /// -/// llvm::ThreadPool myThreadPool; +/// llvm::DefaultThreadPool myThreadPool; /// while (auto *request = nextCompilationRequests()) { /// MLIRContext ctx(registry, MLIRContext::Threading::DISABLED); /// ctx.setThreadPool(myThreadPool); diff --git a/mlir/lib/CAPI/IR/Support.cpp b/mlir/lib/CAPI/IR/Support.cpp index 81c9fc771926..3311131fc2bc 100644 --- a/mlir/lib/CAPI/IR/Support.cpp +++ b/mlir/lib/CAPI/IR/Support.cpp @@ -25,7 +25,7 @@ bool mlirStringRefEqual(MlirStringRef string, MlirStringRef other) { // LLVM ThreadPool API. //===----------------------------------------------------------------------===// MlirLlvmThreadPool mlirLlvmThreadPoolCreate() { - return wrap(new llvm::ThreadPool()); + return wrap(new llvm::DefaultThreadPool()); } void mlirLlvmThreadPoolDestroy(MlirLlvmThreadPool threadPool) { diff --git a/mlir/lib/ExecutionEngine/AsyncRuntime.cpp b/mlir/lib/ExecutionEngine/AsyncRuntime.cpp index ec4a81c042c2..9e6f8a721699 100644 --- a/mlir/lib/ExecutionEngine/AsyncRuntime.cpp +++ b/mlir/lib/ExecutionEngine/AsyncRuntime.cpp @@ -72,7 +72,7 @@ private: } std::atomic numRefCountedObjects; - llvm::ThreadPool threadPool; + llvm::DefaultThreadPool threadPool; }; // -------------------------------------------------------------------------- // diff --git a/mlir/lib/IR/MLIRContext.cpp b/mlir/lib/IR/MLIRContext.cpp index 92568bd311e3..e1e6d14231d9 100644 --- a/mlir/lib/IR/MLIRContext.cpp +++ b/mlir/lib/IR/MLIRContext.cpp @@ -274,7 +274,7 @@ public: MLIRContextImpl(bool threadingIsEnabled) : threadingIsEnabled(threadingIsEnabled) { if (threadingIsEnabled) { - ownedThreadPool = std::make_unique(); + ownedThreadPool = std::make_unique(); threadPool = ownedThreadPool.get(); } } @@ -621,7 +621,7 @@ void MLIRContext::disableMultithreading(bool disable) { } else if (!impl->threadPool) { // The thread pool isn't externally provided. assert(!impl->ownedThreadPool); - impl->ownedThreadPool = std::make_unique(); + impl->ownedThreadPool = std::make_unique(); impl->threadPool = impl->ownedThreadPool.get(); } } -- GitLab From 2f479b811274fede36535e34ecb545ac22e399c3 Mon Sep 17 00:00:00 2001 From: wanglei Date: Wed, 6 Mar 2024 10:03:28 +0800 Subject: [PATCH 229/929] [Clang][LoongArch] Fix wrong return value type of __iocsrrd_h (#84100) relate: https://gcc.gnu.org/pipermail/gcc-patches/2024-February/645016.html --- clang/lib/Headers/larchintrin.h | 2 +- clang/test/CodeGen/LoongArch/intrinsic-la32.c | 8 ++++---- clang/test/CodeGen/LoongArch/intrinsic-la64.c | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/clang/lib/Headers/larchintrin.h b/clang/lib/Headers/larchintrin.h index a613e5ca0e5e..f4218295919a 100644 --- a/clang/lib/Headers/larchintrin.h +++ b/clang/lib/Headers/larchintrin.h @@ -156,7 +156,7 @@ extern __inline unsigned char return (unsigned char)__builtin_loongarch_iocsrrd_b((unsigned int)_1); } -extern __inline unsigned char +extern __inline unsigned short __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __iocsrrd_h(unsigned int _1) { return (unsigned short)__builtin_loongarch_iocsrrd_h((unsigned int)_1); diff --git a/clang/test/CodeGen/LoongArch/intrinsic-la32.c b/clang/test/CodeGen/LoongArch/intrinsic-la32.c index 6a8d99880be3..eb3f8cbe7ac4 100644 --- a/clang/test/CodeGen/LoongArch/intrinsic-la32.c +++ b/clang/test/CodeGen/LoongArch/intrinsic-la32.c @@ -215,11 +215,11 @@ void cacop_w(unsigned long int a) { // LA32-LABEL: @iocsrrd_h_result( // LA32-NEXT: entry: // LA32-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.loongarch.iocsrrd.h(i32 [[A:%.*]]) +// LA32-NEXT: [[CONV_I:%.*]] = trunc i32 [[TMP0]] to i16 // LA32-NEXT: [[TMP1:%.*]] = tail call i32 @llvm.loongarch.iocsrrd.h(i32 [[A]]) -// LA32-NEXT: [[CONV2:%.*]] = and i32 [[TMP0]], 255 -// LA32-NEXT: [[ADD:%.*]] = add i32 [[TMP1]], [[CONV2]] -// LA32-NEXT: [[CONV4:%.*]] = trunc i32 [[ADD]] to i16 -// LA32-NEXT: ret i16 [[CONV4]] +// LA32-NEXT: [[TMP2:%.*]] = trunc i32 [[TMP1]] to i16 +// LA32-NEXT: [[CONV3:%.*]] = add i16 [[TMP2]], [[CONV_I]] +// LA32-NEXT: ret i16 [[CONV3]] // unsigned short iocsrrd_h_result(unsigned int a) { unsigned short b = __iocsrrd_h(a); diff --git a/clang/test/CodeGen/LoongArch/intrinsic-la64.c b/clang/test/CodeGen/LoongArch/intrinsic-la64.c index 48b6a7a3d227..50ec358f546e 100644 --- a/clang/test/CodeGen/LoongArch/intrinsic-la64.c +++ b/clang/test/CodeGen/LoongArch/intrinsic-la64.c @@ -431,11 +431,11 @@ void loongarch_movgr2fcsr(int a) { // CHECK-LABEL: @iocsrrd_h_result( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.loongarch.iocsrrd.h(i32 [[A:%.*]]) +// CHECK-NEXT: [[CONV_I:%.*]] = trunc i32 [[TMP0]] to i16 // CHECK-NEXT: [[TMP1:%.*]] = tail call i32 @llvm.loongarch.iocsrrd.h(i32 [[A]]) -// CHECK-NEXT: [[CONV2:%.*]] = and i32 [[TMP0]], 255 -// CHECK-NEXT: [[ADD:%.*]] = add i32 [[TMP1]], [[CONV2]] -// CHECK-NEXT: [[CONV4:%.*]] = trunc i32 [[ADD]] to i16 -// CHECK-NEXT: ret i16 [[CONV4]] +// CHECK-NEXT: [[TMP2:%.*]] = trunc i32 [[TMP1]] to i16 +// CHECK-NEXT: [[CONV3:%.*]] = add i16 [[TMP2]], [[CONV_I]] +// CHECK-NEXT: ret i16 [[CONV3]] // unsigned short iocsrrd_h_result(unsigned int a) { unsigned short b = __iocsrrd_h(a); -- GitLab From 987fe6fa507adb28602e84b471e6fe309830e361 Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Tue, 5 Mar 2024 17:46:24 -0800 Subject: [PATCH 230/929] [SCEV] Migrate a couple tests to be auto generated A few notes: * pr34538.ll has bitrotten. The original test printed the analysis after transforms in some cases, but this appears to been lost during migration to new pass manager. Remove the now redundant pass invocations and simplify the test setup. --- .../ScalarEvolution/max-trip-count.ll | 264 +++++++++++++----- llvm/test/Analysis/ScalarEvolution/pr34538.ll | 28 +- 2 files changed, 221 insertions(+), 71 deletions(-) diff --git a/llvm/test/Analysis/ScalarEvolution/max-trip-count.ll b/llvm/test/Analysis/ScalarEvolution/max-trip-count.ll index 1f8746d0a4a2..9cc49f2185f8 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-trip-count.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-trip-count.ll @@ -1,10 +1,19 @@ -; RUN: opt < %s -disable-output "-passes=print" 2>&1 | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -disable-output "-passes=print" -scalar-evolution-classify-expressions=0 2>&1 | FileCheck %s ; ScalarEvolution should be able to understand the loop and eliminate the casts. -; CHECK: {%d,+,4} define void @foo(ptr nocapture %d, i32 %n) nounwind { +; CHECK-LABEL: 'foo' +; CHECK-NEXT: Determining loop execution counts for: @foo +; CHECK-NEXT: Loop %bb: backedge-taken count is (-1 + %n) +; CHECK-NEXT: Loop %bb: constant max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is (-1 + %n) +; CHECK-NEXT: Loop %bb: Predicated backedge-taken count is (-1 + %n) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %bb: Trip multiple is 1 +; entry: %0 = icmp sgt i32 %n, 0 ; [#uses=1] br i1 %0, label %bb.nph, label %return @@ -39,12 +48,21 @@ return: ; preds = %bb1.return_crit_edge, %entry ; count, it should say so. ; PR7845 -; CHECK: Loop %for.cond: Unpredictable backedge-taken count. -; CHECK: Loop %for.cond: constant max backedge-taken count is 5 @.str = private constant [4 x i8] c"%d\0A\00" ; [#uses=2] define i32 @main() nounwind { +; CHECK-LABEL: 'main' +; CHECK-NEXT: Determining loop execution counts for: @main +; CHECK-NEXT: Loop %for.cond: Unpredictable backedge-taken count. +; CHECK-NEXT: exit count for for.cond: 5 +; CHECK-NEXT: exit count for for.body: ***COULDNOTCOMPUTE*** +; CHECK-NEXT: Loop %for.cond: constant max backedge-taken count is 5 +; CHECK-NEXT: Loop %for.cond: symbolic max backedge-taken count is 5 +; CHECK-NEXT: symbolic max exit count for for.cond: 5 +; CHECK-NEXT: symbolic max exit count for for.body: ***COULDNOTCOMPUTE*** +; CHECK-NEXT: Loop %for.cond: Unpredictable predicated backedge-taken count. +; entry: br label %for.cond @@ -72,6 +90,15 @@ for.end: ; preds = %for.body, %for.cond declare i32 @printf(ptr, ...) define void @test(ptr %a, i32 %n) nounwind { +; CHECK-LABEL: 'test' +; CHECK-NEXT: Determining loop execution counts for: @test +; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (zext i32 %n to i64)) +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (zext i32 %n to i64)) +; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (zext i32 %n to i64)) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %for.body: Trip multiple is 1 +; entry: %cmp1 = icmp sgt i32 %n, 0 br i1 %cmp1, label %for.body.lr.ph, label %for.end @@ -95,17 +122,22 @@ for.end: ; preds = %for.cond.for.end_cr ret void } -; CHECK: Determining loop execution counts for: @test -; CHECK-NEXT: backedge-taken count is -; CHECK-NEXT: constant max backedge-taken count is 2147483646 ; PR19799: Indvars miscompile due to an incorrect max backedge taken count from SCEV. -; CHECK-LABEL: @pr19799 -; CHECK: Loop %for.body.i: Unpredictable backedge-taken count. -; CHECK: Loop %for.body.i: constant max backedge-taken count is 1 @a = common global i32 0, align 4 define i32 @pr19799() { +; CHECK-LABEL: 'pr19799' +; CHECK-NEXT: Determining loop execution counts for: @pr19799 +; CHECK-NEXT: Loop %for.body.i: Unpredictable backedge-taken count. +; CHECK-NEXT: exit count for for.body.i: ***COULDNOTCOMPUTE*** +; CHECK-NEXT: exit count for for.cond.i: 1 +; CHECK-NEXT: Loop %for.body.i: constant max backedge-taken count is 1 +; CHECK-NEXT: Loop %for.body.i: symbolic max backedge-taken count is 1 +; CHECK-NEXT: symbolic max exit count for for.body.i: ***COULDNOTCOMPUTE*** +; CHECK-NEXT: symbolic max exit count for for.cond.i: 1 +; CHECK-NEXT: Loop %for.body.i: Unpredictable predicated backedge-taken count. +; entry: store i32 -1, ptr @a, align 4 br label %for.body.i @@ -126,12 +158,20 @@ bar.exit: ; preds = %for.cond.i, %for.bo } ; PR18886: Indvars miscompile due to an incorrect max backedge taken count from SCEV. -; CHECK-LABEL: @pr18886 -; CHECK: Loop %for.body: Unpredictable backedge-taken count. -; CHECK: Loop %for.body: constant max backedge-taken count is 3 @aa = global i64 0, align 8 define i32 @pr18886() { +; CHECK-LABEL: 'pr18886' +; CHECK-NEXT: Determining loop execution counts for: @pr18886 +; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. +; CHECK-NEXT: exit count for for.body: ***COULDNOTCOMPUTE*** +; CHECK-NEXT: exit count for for.cond: 3 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 3 +; CHECK-NEXT: symbolic max exit count for for.body: ***COULDNOTCOMPUTE*** +; CHECK-NEXT: symbolic max exit count for for.cond: 3 +; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. +; entry: store i64 -21, ptr @aa, align 8 br label %for.body @@ -156,12 +196,20 @@ return: ; may-exit early exit that can only have one non-exiting iteration ; before the check is forever skipped. ; -; CHECK-LABEL: @cannot_compute_mustexit -; CHECK: Loop %for.body.i: Unpredictable backedge-taken count. -; CHECK: Loop %for.body.i: Unpredictable constant max backedge-taken count. @b = common global i32 0, align 4 define i32 @cannot_compute_mustexit() { +; CHECK-LABEL: 'cannot_compute_mustexit' +; CHECK-NEXT: Determining loop execution counts for: @cannot_compute_mustexit +; CHECK-NEXT: Loop %for.body.i: Unpredictable backedge-taken count. +; CHECK-NEXT: exit count for for.body.i: ***COULDNOTCOMPUTE*** +; CHECK-NEXT: exit count for for.cond.i: ***COULDNOTCOMPUTE*** +; CHECK-NEXT: Loop %for.body.i: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %for.body.i: Unpredictable symbolic max backedge-taken count. +; CHECK-NEXT: symbolic max exit count for for.body.i: ***COULDNOTCOMPUTE*** +; CHECK-NEXT: symbolic max exit count for for.cond.i: ***COULDNOTCOMPUTE*** +; CHECK-NEXT: Loop %for.body.i: Unpredictable predicated backedge-taken count. +; entry: store i32 -1, ptr @a, align 4 br label %for.body.i @@ -185,10 +233,20 @@ bar.exit: ; preds = %for.cond.i, %for.bo ; This loop has two must-exits, both of which dominate the latch. The ; MaxBECount should be the minimum of them. ; -; CHECK-LABEL: @two_mustexit -; CHECK: Loop %for.body.i: backedge-taken count is 1 -; CHECK: Loop %for.body.i: constant max backedge-taken count is 1 define i32 @two_mustexit() { +; CHECK-LABEL: 'two_mustexit' +; CHECK-NEXT: Determining loop execution counts for: @two_mustexit +; CHECK-NEXT: Loop %for.body.i: backedge-taken count is 1 +; CHECK-NEXT: exit count for for.body.i: 1 +; CHECK-NEXT: exit count for for.cond.i: 2 +; CHECK-NEXT: Loop %for.body.i: constant max backedge-taken count is 1 +; CHECK-NEXT: Loop %for.body.i: symbolic max backedge-taken count is 1 +; CHECK-NEXT: symbolic max exit count for for.body.i: 1 +; CHECK-NEXT: symbolic max exit count for for.cond.i: 2 +; CHECK-NEXT: Loop %for.body.i: Predicated backedge-taken count is 1 +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %for.body.i: Trip multiple is 1 +; entry: store i32 -1, ptr @a, align 4 br label %for.body.i @@ -208,9 +266,16 @@ bar.exit: ; preds = %for.cond.i, %for.bo ret i32 0 } -; CHECK-LABEL: @ne_max_trip_count_1 -; CHECK: Loop %for.body: constant max backedge-taken count is 7 define i32 @ne_max_trip_count_1(i32 %n) { +; CHECK-LABEL: 'ne_max_trip_count_1' +; CHECK-NEXT: Determining loop execution counts for: @ne_max_trip_count_1 +; CHECK-NEXT: Loop %for.body: backedge-taken count is (zext i3 (trunc i32 %n to i3) to i32) +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 7 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (zext i3 (trunc i32 %n to i3) to i32) +; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (zext i3 (trunc i32 %n to i3) to i32) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %for.body: Trip multiple is 1 +; entry: %masked = and i32 %n, 7 br label %for.body @@ -225,9 +290,16 @@ bar.exit: ret i32 0 } -; CHECK-LABEL: @ne_max_trip_count_2 -; CHECK: Loop %for.body: constant max backedge-taken count is -1 define i32 @ne_max_trip_count_2(i32 %n) { +; CHECK-LABEL: 'ne_max_trip_count_2' +; CHECK-NEXT: Determining loop execution counts for: @ne_max_trip_count_2 +; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (zext i3 (trunc i32 %n to i3) to i32)) +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (zext i3 (trunc i32 %n to i3) to i32)) +; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (zext i3 (trunc i32 %n to i3) to i32)) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %for.body: Trip multiple is 1 +; entry: %masked = and i32 %n, 7 br label %for.body @@ -242,9 +314,16 @@ bar.exit: ret i32 0 } -; CHECK-LABEL: @ne_max_trip_count_3 -; CHECK: Loop %for.body: constant max backedge-taken count is 6 define i32 @ne_max_trip_count_3(i32 %n) { +; CHECK-LABEL: 'ne_max_trip_count_3' +; CHECK-NEXT: Determining loop execution counts for: @ne_max_trip_count_3 +; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (zext i3 (trunc i32 %n to i3) to i32)) +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 6 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (zext i3 (trunc i32 %n to i3) to i32)) +; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (zext i3 (trunc i32 %n to i3) to i32)) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %for.body: Trip multiple is 1 +; entry: %masked = and i32 %n, 7 %guard = icmp eq i32 %masked, 0 @@ -266,9 +345,16 @@ exit: ret i32 0 } -; CHECK-LABEL: @ne_max_trip_count_4 -; CHECK: Loop %for.body: constant max backedge-taken count is -2 define i32 @ne_max_trip_count_4(i32 %n) { +; CHECK-LABEL: 'ne_max_trip_count_4' +; CHECK-NEXT: Determining loop execution counts for: @ne_max_trip_count_4 +; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %n) +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %n) +; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %n) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %for.body: Trip multiple is 1 +; entry: %guard = icmp eq i32 %n, 0 br i1 %guard, label %exit, label %for.preheader @@ -292,9 +378,13 @@ exit: ; The end bound of the loop can change between iterations, so the exact trip ; count is unknown, but SCEV can calculate the max trip count. define void @changing_end_bound(ptr %n_addr, ptr %addr) { -; CHECK-LABEL: Determining loop execution counts for: @changing_end_bound -; CHECK: Loop %loop: Unpredictable backedge-taken count. -; CHECK: Loop %loop: constant max backedge-taken count is 2147483646 +; CHECK-LABEL: 'changing_end_bound' +; CHECK-NEXT: Determining loop execution counts for: @changing_end_bound +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. +; entry: br label %loop @@ -314,12 +404,16 @@ loop.exit: } ; Similar test as above, but unknown start value. -; Also, there's no nsw on the iv.next, but SCEV knows +; Also, there's no nsw on the iv.next, but SCEV knows ; the termination condition is LT, so the IV cannot wrap. define void @changing_end_bound2(i32 %start, ptr %n_addr, ptr %addr) { -; CHECK-LABEL: Determining loop execution counts for: @changing_end_bound2 -; CHECK: Loop %loop: Unpredictable backedge-taken count. -; CHECK: Loop %loop: constant max backedge-taken count is -1 +; CHECK-LABEL: 'changing_end_bound2' +; CHECK-NEXT: Determining loop execution counts for: @changing_end_bound2 +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. +; entry: br label %loop @@ -340,9 +434,13 @@ loop.exit: ; changing end bound and greater than one stride define void @changing_end_bound3(i32 %start, ptr %n_addr, ptr %addr) { -; CHECK-LABEL: Determining loop execution counts for: @changing_end_bound3 -; CHECK: Loop %loop: Unpredictable backedge-taken count. -; CHECK: Loop %loop: constant max backedge-taken count is 1073741822 +; CHECK-LABEL: 'changing_end_bound3' +; CHECK-NEXT: Determining loop execution counts for: @changing_end_bound3 +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 1073741823 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 1073741823 +; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. +; entry: br label %loop @@ -364,9 +462,13 @@ loop.exit: ; same as above test, but the IV can wrap around. ; so the max backedge taken count is unpredictable. define void @changing_end_bound4(i32 %start, ptr %n_addr, ptr %addr) { -; CHECK-LABEL: Determining loop execution counts for: @changing_end_bound4 -; CHECK: Loop %loop: Unpredictable backedge-taken count. -; CHECK: Loop %loop: Unpredictable constant max backedge-taken count. +; CHECK-LABEL: 'changing_end_bound4' +; CHECK-NEXT: Determining loop execution counts for: @changing_end_bound4 +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. +; entry: br label %loop @@ -388,9 +490,13 @@ loop.exit: ; unknown stride. Since it's not knownPositive, we do not estimate the max ; backedge taken count. define void @changing_end_bound5(i32 %stride, i32 %start, ptr %n_addr, ptr %addr) { -; CHECK-LABEL: Determining loop execution counts for: @changing_end_bound5 -; CHECK: Loop %loop: Unpredictable backedge-taken count. -; CHECK: Loop %loop: Unpredictable constant max backedge-taken count. +; CHECK-LABEL: 'changing_end_bound5' +; CHECK-NEXT: Determining loop execution counts for: @changing_end_bound5 +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. +; entry: br label %loop @@ -411,9 +517,13 @@ loop.exit: ; negative stride value define void @changing_end_bound6(i32 %start, ptr %n_addr, ptr %addr) { -; CHECK-LABEL: Determining loop execution counts for: @changing_end_bound6 -; CHECK: Loop %loop: Unpredictable backedge-taken count. -; CHECK: Loop %loop: Unpredictable constant max backedge-taken count. +; CHECK-LABEL: 'changing_end_bound6' +; CHECK-NEXT: Determining loop execution counts for: @changing_end_bound6 +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. +; entry: br label %loop @@ -434,9 +544,13 @@ loop.exit: ; sgt with negative stride define void @changing_end_bound7(i32 %start, ptr %n_addr, ptr %addr) { -; CHECK-LABEL: Determining loop execution counts for: @changing_end_bound7 -; CHECK: Loop %loop: Unpredictable backedge-taken count. -; CHECK: Loop %loop: Unpredictable constant max backedge-taken count. +; CHECK-LABEL: 'changing_end_bound7' +; CHECK-NEXT: Determining loop execution counts for: @changing_end_bound7 +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. +; entry: br label %loop @@ -456,9 +570,15 @@ loop.exit: } define void @max_overflow_se(i8 %n) mustprogress { -; CHECK-LABEL: Determining loop execution counts for: @max_overflow_se -; CHECK: Loop %loop: backedge-taken count is 0 -; CHECK: Loop %loop: constant max backedge-taken count is 0 +; CHECK-LABEL: 'max_overflow_se' +; CHECK-NEXT: Determining loop execution counts for: @max_overflow_se +; CHECK-NEXT: Loop %loop: backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 0 +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; entry: br label %loop @@ -475,11 +595,17 @@ exit: ; Show that we correctly realize that %i can overflow here as long as ; the early exit is taken before we branch on poison. define void @max_overflow_me(i8 %n) mustprogress { -; CHECK-LABEL: Determining loop execution counts for: @max_overflow_me -; CHECK: Loop %loop: Unpredictable backedge-taken count. -; CHECK: exit count for loop: 1 -; CHECK: exit count for latch: ***COULDNOTCOMPUTE*** -; CHECK: Loop %loop: constant max backedge-taken count is 1 +; CHECK-LABEL: 'max_overflow_me' +; CHECK-NEXT: Determining loop execution counts for: @max_overflow_me +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: exit count for loop: 1 +; CHECK-NEXT: exit count for latch: ***COULDNOTCOMPUTE*** +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 1 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 1 +; CHECK-NEXT: symbolic max exit count for loop: 1 +; CHECK-NEXT: symbolic max exit count for latch: ***COULDNOTCOMPUTE*** +; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. +; entry: br label %loop @@ -501,9 +627,15 @@ exit: ; Max backedge-taken count is zero. define void @bool_stride(i1 %s, i1 %n) mustprogress { -; CHECK-LABEL: Determining loop execution counts for: @bool_stride -; CHECK: Loop %loop: backedge-taken count is false -; CHECK: Loop %loop: constant max backedge-taken count is false +; CHECK-LABEL: 'bool_stride' +; CHECK-NEXT: Determining loop execution counts for: @bool_stride +; CHECK-NEXT: Loop %loop: backedge-taken count is false +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is false +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false +; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is false +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; entry: br label %loop @@ -521,9 +653,15 @@ exit: ; able to prove a zero btc, but our symbolic logic doesn't due to a lack ; of context sensativity. define void @ne_zero_max_btc(i32 %a) { -; CHECK-LABEL: Determining loop execution counts for: @ne_zero_max_btc -; CHECK: Loop %for.body: backedge-taken count is 0 -; CHECK: Loop %for.body: constant max backedge-taken count is 0 +; CHECK-LABEL: 'ne_zero_max_btc' +; CHECK-NEXT: Determining loop execution counts for: @ne_zero_max_btc +; CHECK-NEXT: Loop %for.body: backedge-taken count is 0 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 0 +; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 0 +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %for.body: Trip multiple is 1 +; entry: %cmp = icmp slt i32 %a, 1 %spec.select = select i1 %cmp, i32 %a, i32 1 diff --git a/llvm/test/Analysis/ScalarEvolution/pr34538.ll b/llvm/test/Analysis/ScalarEvolution/pr34538.ll index 3ac3d29016ce..762bbde2bf75 100644 --- a/llvm/test/Analysis/ScalarEvolution/pr34538.ll +++ b/llvm/test/Analysis/ScalarEvolution/pr34538.ll @@ -1,10 +1,16 @@ -; RUN: opt "-passes=print,loop(loop-deletion),simplifycfg" -disable-output < %s 2>&1 | FileCheck %s --check-prefix=CHECK-ANALYSIS-1 -; RUN: opt -disable-output "-passes=print" < %s 2>&1 | FileCheck %s --check-prefix=CHECK-ANALYSIS-2 +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -disable-output "-passes=print" -scalar-evolution-classify-expressions=0 < %s 2>&1 | FileCheck %s define i32 @pr34538() local_unnamed_addr #0 { -; CHECK-ANALYSIS-1: Loop %do.body: backedge-taken count is 10000 -; CHECK-ANALYSIS-1: Loop %do.body: constant max backedge-taken count is 10000 -; CHECK-ANALYSIS-1: Loop %do.body: Predicated backedge-taken count is 10000 +; CHECK-LABEL: 'pr34538' +; CHECK-NEXT: Determining loop execution counts for: @pr34538 +; CHECK-NEXT: Loop %do.body: backedge-taken count is 10000 +; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is 10000 +; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is 10000 +; CHECK-NEXT: Loop %do.body: Predicated backedge-taken count is 10000 +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %do.body: Trip multiple is 10001 +; entry: br label %do.body @@ -21,6 +27,15 @@ do.end: ; preds = %do.body define i32 @foo() { +; CHECK-LABEL: 'foo' +; CHECK-NEXT: Determining loop execution counts for: @foo +; CHECK-NEXT: Loop %do.body: backedge-taken count is 5000 +; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is 5000 +; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is 5000 +; CHECK-NEXT: Loop %do.body: Predicated backedge-taken count is 5000 +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %do.body: Trip multiple is 5001 +; entry: br label %do.body @@ -33,7 +48,4 @@ do.body: ; preds = %do.body, %entry do.end: ; preds = %do.body ret i32 0 -; CHECK-ANALYSIS-2: Loop %do.body: backedge-taken count is 5000 -; CHECK-ANALYSIS-2: Loop %do.body: constant max backedge-taken count is 5000 -; CHECK-ANALYSIS-2: Loop %do.body: Predicated backedge-taken count is 5000 } -- GitLab From d32bcbf6a7f5beb63ce435c2bea737d33d5b7468 Mon Sep 17 00:00:00 2001 From: Jie Fu Date: Wed, 6 Mar 2024 10:17:58 +0800 Subject: [PATCH 231/929] [lldb] Fix build failure in Debugger.cpp (NFC) llvm-project/lldb/source/Core/Debugger.cpp:107:14: error: no type named 'DefaultThreadPoolThreadPool' in namespace 'llvm' static llvm::DefaultThreadPoolThreadPool *g_thread_pool = nullptr; ~~~~~~^ 1 error generated. --- lldb/source/Core/Debugger.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lldb/source/Core/Debugger.cpp b/lldb/source/Core/Debugger.cpp index 9d62b2a908f7..90aabde2b764 100644 --- a/lldb/source/Core/Debugger.cpp +++ b/lldb/source/Core/Debugger.cpp @@ -104,7 +104,7 @@ static std::recursive_mutex *g_debugger_list_mutex_ptr = nullptr; // NOTE: intentional leak to avoid issues with C++ destructor chain static Debugger::DebuggerList *g_debugger_list_ptr = nullptr; // NOTE: intentional leak to avoid issues with C++ destructor chain -static llvm::DefaultThreadPoolThreadPool *g_thread_pool = nullptr; +static llvm::DefaultThreadPool *g_thread_pool = nullptr; static constexpr OptionEnumValueElement g_show_disassembly_enum_values[] = { { -- GitLab From f439c7137373575b5d8e7a2e7056b8dd59e62a26 Mon Sep 17 00:00:00 2001 From: wanglei Date: Wed, 6 Mar 2024 10:09:17 +0800 Subject: [PATCH 232/929] [InstrProf][NFC] Fix -Wimplicit-fallthrough warning in InstrProf.cpp after #82711 --- llvm/lib/ProfileData/InstrProf.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index b9afee413853..8cf97fcb1dab 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -1571,7 +1571,6 @@ size_t Header::size() const { case 12ull: return offsetOf(&Header::VTableNamesOffset) + sizeof(Header::VTableNamesOffset); - [[fallthrough]]; case 11ull: [[fallthrough]]; case 10ull: -- GitLab From 49ec8b747c83b8dec8317614c30e5610d133790e Mon Sep 17 00:00:00 2001 From: Changpeng Fang Date: Tue, 5 Mar 2024 18:26:37 -0800 Subject: [PATCH 233/929] AMDGPU: Define and Use HasInterpInsts for interp inst definitions (#84102) --- llvm/lib/Target/AMDGPU/AMDGPU.td | 3 +++ llvm/lib/Target/AMDGPU/GCNSubtarget.h | 4 ++++ llvm/lib/Target/AMDGPU/VINTERPInstructions.td | 4 ++-- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td index 814ac0b93fcf..8906c46f279e 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.td +++ b/llvm/lib/Target/AMDGPU/AMDGPU.td @@ -1877,6 +1877,9 @@ def NotLDSRequiresM0Init : Predicate<"!Subtarget->ldsRequiresM0Init()">; def HasExpOrExportInsts : Predicate<"Subtarget->hasExpOrExportInsts()">, AssemblerPredicate<(all_of (not FeatureGFX90AInsts))>; +def HasInterpInsts : Predicate<"Subtarget->hasInterpInsts()">, + AssemblerPredicate<(all_of FeatureGFX11Insts)>; + def HasDSAddTid : Predicate<"Subtarget->getGeneration() >= AMDGPUSubtarget::GFX9">, AssemblerPredicate<(all_of FeatureGFX9Insts)>; diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h index bb0ccfce324f..3283ac72aa4d 100644 --- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h +++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h @@ -651,6 +651,10 @@ public: return !hasGFX940Insts(); } + bool hasInterpInsts() const { + return GFX11Insts; + } + // DS_ADD_F64/DS_ADD_RTN_F64 bool hasLdsAtomicAddF64() const { return hasGFX90AInsts(); } diff --git a/llvm/lib/Target/AMDGPU/VINTERPInstructions.td b/llvm/lib/Target/AMDGPU/VINTERPInstructions.td index f84e163b2df0..0303d1e23a0a 100644 --- a/llvm/lib/Target/AMDGPU/VINTERPInstructions.td +++ b/llvm/lib/Target/AMDGPU/VINTERPInstructions.td @@ -105,7 +105,7 @@ class VOP3_VINTERP_F16 ArgVT> : VOPProfile { // VINTERP Pseudo Instructions //===----------------------------------------------------------------------===// -let SubtargetPredicate = isGFX11Plus in { +let SubtargetPredicate = HasInterpInsts in { let Uses = [M0, EXEC, MODE] in { def V_INTERP_P10_F32_inreg : VINTERP_Pseudo <"v_interp_p10_f32", VOP3_VINTERP_F32>; @@ -123,7 +123,7 @@ def V_INTERP_P2_RTZ_F16_F32_inreg : VINTERP_Pseudo <"v_interp_p2_rtz_f16_f32", VOP3_VINTERP_F16<[f16, f32, f32, f32]>>; } // Uses = [M0, EXEC] -} // SubtargetPredicate = isGFX11Plus +} // SubtargetPredicate = HasInterpInsts. class VInterpF32Pat : GCNPat < (f32 (op -- GitLab From 85388a06b6022d0a7bc984bcaff86cf96f045338 Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Wed, 6 Mar 2024 10:56:19 +0800 Subject: [PATCH 234/929] [RISCV] Move RISCVVType namespace to TargetParser (#83222) Clang and some middle-end optimizations may need these helper functions. This can reduce some duplications. --- .../llvm/TargetParser/RISCVTargetParser.h | 77 ++++++++++++++++ .../RISCV/MCTargetDesc/RISCVBaseInfo.cpp | 87 ------------------ .../Target/RISCV/MCTargetDesc/RISCVBaseInfo.h | 73 +-------------- llvm/lib/Target/RISCV/RISCVISelLowering.h | 1 - llvm/lib/TargetParser/RISCVTargetParser.cpp | 91 +++++++++++++++++++ llvm/unittests/Target/RISCV/CMakeLists.txt | 1 - llvm/unittests/TargetParser/CMakeLists.txt | 1 + .../RISCVTargetParserTest.cpp} | 7 +- 8 files changed, 173 insertions(+), 165 deletions(-) rename llvm/unittests/{Target/RISCV/RISCVBaseInfoTest.cpp => TargetParser/RISCVTargetParserTest.cpp} (86%) diff --git a/llvm/include/llvm/TargetParser/RISCVTargetParser.h b/llvm/include/llvm/TargetParser/RISCVTargetParser.h index 553b4efe0e30..cdd19189f8dc 100644 --- a/llvm/include/llvm/TargetParser/RISCVTargetParser.h +++ b/llvm/include/llvm/TargetParser/RISCVTargetParser.h @@ -15,6 +15,8 @@ #define LLVM_TARGETPARSER_RISCVTARGETPARSER_H #include "llvm/ADT/StringRef.h" +#include "llvm/Support/MathExtras.h" +#include "llvm/Support/raw_ostream.h" namespace llvm { @@ -36,6 +38,81 @@ void fillValidTuneCPUArchList(SmallVectorImpl &Values, bool IsRV64); bool hasFastUnalignedAccess(StringRef CPU); } // namespace RISCV + +namespace RISCVII { +enum VLMUL : uint8_t { + LMUL_1 = 0, + LMUL_2, + LMUL_4, + LMUL_8, + LMUL_RESERVED, + LMUL_F8, + LMUL_F4, + LMUL_F2 +}; + +enum { + TAIL_UNDISTURBED_MASK_UNDISTURBED = 0, + TAIL_AGNOSTIC = 1, + MASK_AGNOSTIC = 2, +}; +} // namespace RISCVII + +namespace RISCVVType { +// Is this a SEW value that can be encoded into the VTYPE format. +inline static bool isValidSEW(unsigned SEW) { + return isPowerOf2_32(SEW) && SEW >= 8 && SEW <= 1024; +} + +// Is this a LMUL value that can be encoded into the VTYPE format. +inline static bool isValidLMUL(unsigned LMUL, bool Fractional) { + return isPowerOf2_32(LMUL) && LMUL <= 8 && (!Fractional || LMUL != 1); +} + +unsigned encodeVTYPE(RISCVII::VLMUL VLMUL, unsigned SEW, bool TailAgnostic, + bool MaskAgnostic); + +inline static RISCVII::VLMUL getVLMUL(unsigned VType) { + unsigned VLMUL = VType & 0x7; + return static_cast(VLMUL); +} + +// Decode VLMUL into 1,2,4,8 and fractional indicator. +std::pair decodeVLMUL(RISCVII::VLMUL VLMUL); + +inline static RISCVII::VLMUL encodeLMUL(unsigned LMUL, bool Fractional) { + assert(isValidLMUL(LMUL, Fractional) && "Unsupported LMUL"); + unsigned LmulLog2 = Log2_32(LMUL); + return static_cast(Fractional ? 8 - LmulLog2 : LmulLog2); +} + +inline static unsigned decodeVSEW(unsigned VSEW) { + assert(VSEW < 8 && "Unexpected VSEW value"); + return 1 << (VSEW + 3); +} + +inline static unsigned encodeSEW(unsigned SEW) { + assert(isValidSEW(SEW) && "Unexpected SEW value"); + return Log2_32(SEW) - 3; +} + +inline static unsigned getSEW(unsigned VType) { + unsigned VSEW = (VType >> 3) & 0x7; + return decodeVSEW(VSEW); +} + +inline static bool isTailAgnostic(unsigned VType) { return VType & 0x40; } + +inline static bool isMaskAgnostic(unsigned VType) { return VType & 0x80; } + +void printVType(unsigned VType, raw_ostream &OS); + +unsigned getSEWLMULRatio(unsigned SEW, RISCVII::VLMUL VLMul); + +std::optional +getSameRatioLMUL(unsigned SEW, RISCVII::VLMUL VLMUL, unsigned EEW); +} // namespace RISCVVType + } // namespace llvm #endif diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp index be9c7d190b55..61f8e7171037 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp @@ -134,93 +134,6 @@ parseFeatureBits(bool IsRV64, const FeatureBitset &FeatureBits) { } // namespace RISCVFeatures -// Encode VTYPE into the binary format used by the the VSETVLI instruction which -// is used by our MC layer representation. -// -// Bits | Name | Description -// -----+------------+------------------------------------------------ -// 7 | vma | Vector mask agnostic -// 6 | vta | Vector tail agnostic -// 5:3 | vsew[2:0] | Standard element width (SEW) setting -// 2:0 | vlmul[2:0] | Vector register group multiplier (LMUL) setting -unsigned RISCVVType::encodeVTYPE(RISCVII::VLMUL VLMUL, unsigned SEW, - bool TailAgnostic, bool MaskAgnostic) { - assert(isValidSEW(SEW) && "Invalid SEW"); - unsigned VLMULBits = static_cast(VLMUL); - unsigned VSEWBits = encodeSEW(SEW); - unsigned VTypeI = (VSEWBits << 3) | (VLMULBits & 0x7); - if (TailAgnostic) - VTypeI |= 0x40; - if (MaskAgnostic) - VTypeI |= 0x80; - - return VTypeI; -} - -std::pair RISCVVType::decodeVLMUL(RISCVII::VLMUL VLMUL) { - switch (VLMUL) { - default: - llvm_unreachable("Unexpected LMUL value!"); - case RISCVII::VLMUL::LMUL_1: - case RISCVII::VLMUL::LMUL_2: - case RISCVII::VLMUL::LMUL_4: - case RISCVII::VLMUL::LMUL_8: - return std::make_pair(1 << static_cast(VLMUL), false); - case RISCVII::VLMUL::LMUL_F2: - case RISCVII::VLMUL::LMUL_F4: - case RISCVII::VLMUL::LMUL_F8: - return std::make_pair(1 << (8 - static_cast(VLMUL)), true); - } -} - -void RISCVVType::printVType(unsigned VType, raw_ostream &OS) { - unsigned Sew = getSEW(VType); - OS << "e" << Sew; - - unsigned LMul; - bool Fractional; - std::tie(LMul, Fractional) = decodeVLMUL(getVLMUL(VType)); - - if (Fractional) - OS << ", mf"; - else - OS << ", m"; - OS << LMul; - - if (isTailAgnostic(VType)) - OS << ", ta"; - else - OS << ", tu"; - - if (isMaskAgnostic(VType)) - OS << ", ma"; - else - OS << ", mu"; -} - -unsigned RISCVVType::getSEWLMULRatio(unsigned SEW, RISCVII::VLMUL VLMul) { - unsigned LMul; - bool Fractional; - std::tie(LMul, Fractional) = decodeVLMUL(VLMul); - - // Convert LMul to a fixed point value with 3 fractional bits. - LMul = Fractional ? (8 / LMul) : (LMul * 8); - - assert(SEW >= 8 && "Unexpected SEW value"); - return (SEW * 8) / LMul; -} - -std::optional -RISCVVType::getSameRatioLMUL(unsigned SEW, RISCVII::VLMUL VLMUL, unsigned EEW) { - unsigned Ratio = RISCVVType::getSEWLMULRatio(SEW, VLMUL); - unsigned EMULFixedPoint = (EEW * 8) / Ratio; - bool Fractional = EMULFixedPoint < 8; - unsigned EMUL = Fractional ? 8 / EMULFixedPoint : EMULFixedPoint / 8; - if (!isValidLMUL(EMUL, Fractional)) - return std::nullopt; - return RISCVVType::encodeLMUL(EMUL, Fractional); -} - // Include the auto-generated portion of the compress emitter. #define GEN_UNCOMPRESS_INSTR #define GEN_COMPRESS_INSTR diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h index d7f7859ce439..6d0381c30d3e 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h @@ -20,6 +20,7 @@ #include "llvm/ADT/StringSwitch.h" #include "llvm/MC/MCInstrDesc.h" #include "llvm/Support/RISCVISAInfo.h" +#include "llvm/TargetParser/RISCVTargetParser.h" #include "llvm/TargetParser/SubtargetFeature.h" namespace llvm { @@ -124,23 +125,6 @@ enum { TargetOverlapConstraintTypeMask = 3ULL << TargetOverlapConstraintTypeShift, }; -enum VLMUL : uint8_t { - LMUL_1 = 0, - LMUL_2, - LMUL_4, - LMUL_8, - LMUL_RESERVED, - LMUL_F8, - LMUL_F4, - LMUL_F2 -}; - -enum { - TAIL_UNDISTURBED_MASK_UNDISTURBED = 0, - TAIL_AGNOSTIC = 1, - MASK_AGNOSTIC = 2, -}; - // Helper functions to read TSFlags. /// \returns the format of the instruction. static inline unsigned getFormat(uint64_t TSFlags) { @@ -484,61 +468,6 @@ parseFeatureBits(bool IsRV64, const FeatureBitset &FeatureBits); } // namespace RISCVFeatures -namespace RISCVVType { -// Is this a SEW value that can be encoded into the VTYPE format. -inline static bool isValidSEW(unsigned SEW) { - return isPowerOf2_32(SEW) && SEW >= 8 && SEW <= 1024; -} - -// Is this a LMUL value that can be encoded into the VTYPE format. -inline static bool isValidLMUL(unsigned LMUL, bool Fractional) { - return isPowerOf2_32(LMUL) && LMUL <= 8 && (!Fractional || LMUL != 1); -} - -unsigned encodeVTYPE(RISCVII::VLMUL VLMUL, unsigned SEW, bool TailAgnostic, - bool MaskAgnostic); - -inline static RISCVII::VLMUL getVLMUL(unsigned VType) { - unsigned VLMUL = VType & 0x7; - return static_cast(VLMUL); -} - -// Decode VLMUL into 1,2,4,8 and fractional indicator. -std::pair decodeVLMUL(RISCVII::VLMUL VLMUL); - -inline static RISCVII::VLMUL encodeLMUL(unsigned LMUL, bool Fractional) { - assert(isValidLMUL(LMUL, Fractional) && "Unsupported LMUL"); - unsigned LmulLog2 = Log2_32(LMUL); - return static_cast(Fractional ? 8 - LmulLog2 : LmulLog2); -} - -inline static unsigned decodeVSEW(unsigned VSEW) { - assert(VSEW < 8 && "Unexpected VSEW value"); - return 1 << (VSEW + 3); -} - -inline static unsigned encodeSEW(unsigned SEW) { - assert(isValidSEW(SEW) && "Unexpected SEW value"); - return Log2_32(SEW) - 3; -} - -inline static unsigned getSEW(unsigned VType) { - unsigned VSEW = (VType >> 3) & 0x7; - return decodeVSEW(VSEW); -} - -inline static bool isTailAgnostic(unsigned VType) { return VType & 0x40; } - -inline static bool isMaskAgnostic(unsigned VType) { return VType & 0x80; } - -void printVType(unsigned VType, raw_ostream &OS); - -unsigned getSEWLMULRatio(unsigned SEW, RISCVII::VLMUL VLMul); - -std::optional -getSameRatioLMUL(unsigned SEW, RISCVII::VLMUL VLMUL, unsigned EEW); -} // namespace RISCVVType - namespace RISCVRVC { bool compress(MCInst &OutInst, const MCInst &MI, const MCSubtargetInfo &STI); bool uncompress(MCInst &OutInst, const MCInst &MI, const MCSubtargetInfo &STI); diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h index a38463f81027..f90cb4df6047 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.h +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h @@ -18,7 +18,6 @@ #include "llvm/CodeGen/CallingConvLower.h" #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/TargetLowering.h" -#include "llvm/TargetParser/RISCVTargetParser.h" #include namespace llvm { diff --git a/llvm/lib/TargetParser/RISCVTargetParser.cpp b/llvm/lib/TargetParser/RISCVTargetParser.cpp index 8036df46fb47..0d95e3a9b819 100644 --- a/llvm/lib/TargetParser/RISCVTargetParser.cpp +++ b/llvm/lib/TargetParser/RISCVTargetParser.cpp @@ -120,4 +120,95 @@ void getFeaturesForCPU(StringRef CPU, EnabledFeatures.push_back(F.substr(1)); } } // namespace RISCV + +namespace RISCVVType { +// Encode VTYPE into the binary format used by the the VSETVLI instruction which +// is used by our MC layer representation. +// +// Bits | Name | Description +// -----+------------+------------------------------------------------ +// 7 | vma | Vector mask agnostic +// 6 | vta | Vector tail agnostic +// 5:3 | vsew[2:0] | Standard element width (SEW) setting +// 2:0 | vlmul[2:0] | Vector register group multiplier (LMUL) setting +unsigned encodeVTYPE(RISCVII::VLMUL VLMUL, unsigned SEW, bool TailAgnostic, + bool MaskAgnostic) { + assert(isValidSEW(SEW) && "Invalid SEW"); + unsigned VLMULBits = static_cast(VLMUL); + unsigned VSEWBits = encodeSEW(SEW); + unsigned VTypeI = (VSEWBits << 3) | (VLMULBits & 0x7); + if (TailAgnostic) + VTypeI |= 0x40; + if (MaskAgnostic) + VTypeI |= 0x80; + + return VTypeI; +} + +std::pair decodeVLMUL(RISCVII::VLMUL VLMUL) { + switch (VLMUL) { + default: + llvm_unreachable("Unexpected LMUL value!"); + case RISCVII::VLMUL::LMUL_1: + case RISCVII::VLMUL::LMUL_2: + case RISCVII::VLMUL::LMUL_4: + case RISCVII::VLMUL::LMUL_8: + return std::make_pair(1 << static_cast(VLMUL), false); + case RISCVII::VLMUL::LMUL_F2: + case RISCVII::VLMUL::LMUL_F4: + case RISCVII::VLMUL::LMUL_F8: + return std::make_pair(1 << (8 - static_cast(VLMUL)), true); + } +} + +void printVType(unsigned VType, raw_ostream &OS) { + unsigned Sew = getSEW(VType); + OS << "e" << Sew; + + unsigned LMul; + bool Fractional; + std::tie(LMul, Fractional) = decodeVLMUL(getVLMUL(VType)); + + if (Fractional) + OS << ", mf"; + else + OS << ", m"; + OS << LMul; + + if (isTailAgnostic(VType)) + OS << ", ta"; + else + OS << ", tu"; + + if (isMaskAgnostic(VType)) + OS << ", ma"; + else + OS << ", mu"; +} + +unsigned getSEWLMULRatio(unsigned SEW, RISCVII::VLMUL VLMul) { + unsigned LMul; + bool Fractional; + std::tie(LMul, Fractional) = decodeVLMUL(VLMul); + + // Convert LMul to a fixed point value with 3 fractional bits. + LMul = Fractional ? (8 / LMul) : (LMul * 8); + + assert(SEW >= 8 && "Unexpected SEW value"); + return (SEW * 8) / LMul; +} + +std::optional +getSameRatioLMUL(unsigned SEW, RISCVII::VLMUL VLMUL, unsigned EEW) { + unsigned Ratio = RISCVVType::getSEWLMULRatio(SEW, VLMUL); + unsigned EMULFixedPoint = (EEW * 8) / Ratio; + bool Fractional = EMULFixedPoint < 8; + unsigned EMUL = Fractional ? 8 / EMULFixedPoint : EMULFixedPoint / 8; + if (!isValidLMUL(EMUL, Fractional)) + return std::nullopt; + return RISCVVType::encodeLMUL(EMUL, Fractional); +} + +} // namespace RISCVVType + } // namespace llvm diff --git a/llvm/unittests/Target/RISCV/CMakeLists.txt b/llvm/unittests/Target/RISCV/CMakeLists.txt index d80d29b7f0da..b58d605355ba 100644 --- a/llvm/unittests/Target/RISCV/CMakeLists.txt +++ b/llvm/unittests/Target/RISCV/CMakeLists.txt @@ -16,7 +16,6 @@ set(LLVM_LINK_COMPONENTS add_llvm_target_unittest(RISCVTests MCInstrAnalysisTest.cpp - RISCVBaseInfoTest.cpp RISCVInstrInfoTest.cpp ) diff --git a/llvm/unittests/TargetParser/CMakeLists.txt b/llvm/unittests/TargetParser/CMakeLists.txt index 501acc044687..3bbc74f3f8d3 100644 --- a/llvm/unittests/TargetParser/CMakeLists.txt +++ b/llvm/unittests/TargetParser/CMakeLists.txt @@ -6,6 +6,7 @@ set(LLVM_LINK_COMPONENTS add_llvm_unittest(TargetParserTests CSKYTargetParserTest.cpp Host.cpp + RISCVTargetParserTest.cpp TargetParserTest.cpp TripleTest.cpp ) diff --git a/llvm/unittests/Target/RISCV/RISCVBaseInfoTest.cpp b/llvm/unittests/TargetParser/RISCVTargetParserTest.cpp similarity index 86% rename from llvm/unittests/Target/RISCV/RISCVBaseInfoTest.cpp rename to llvm/unittests/TargetParser/RISCVTargetParserTest.cpp index 0e4c90caaaef..68338b569a20 100644 --- a/llvm/unittests/Target/RISCV/RISCVBaseInfoTest.cpp +++ b/llvm/unittests/TargetParser/RISCVTargetParserTest.cpp @@ -1,4 +1,4 @@ -//===- RISCVBaseInfoTest.cpp - RISCVBaseInfo unit tests ----------===// +//===---- RISCVTargetParserTest.cpp - RISCVTargetParser unit tests --------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,14 +6,13 @@ // //===----------------------------------------------------------------------===// -#include "MCTargetDesc/RISCVBaseInfo.h" - +#include "llvm/TargetParser/RISCVTargetParser.h" #include "gtest/gtest.h" using namespace llvm; namespace { -TEST(RISCVBaseInfo, CheckSameRatioLMUL) { +TEST(RISCVVType, CheckSameRatioLMUL) { // Smaller LMUL. EXPECT_EQ(RISCVII::LMUL_1, RISCVVType::getSameRatioLMUL(16, RISCVII::LMUL_2, 8)); -- GitLab From 96fc54828a0e72e60f90d237e571b47cad5bab87 Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Tue, 5 Mar 2024 18:57:45 -0800 Subject: [PATCH 235/929] Revert "[mlir][py] better support for arith.constant construction" (#84103) Reverts llvm/llvm-project#83259 This broke an integration test on Windows --- mlir/python/mlir/dialects/arith.py | 23 ++----------- mlir/test/python/dialects/arith_dialect.py | 38 ---------------------- 2 files changed, 2 insertions(+), 59 deletions(-) diff --git a/mlir/python/mlir/dialects/arith.py b/mlir/python/mlir/dialects/arith.py index 83a50c7ef244..61c6917393f1 100644 --- a/mlir/python/mlir/dialects/arith.py +++ b/mlir/python/mlir/dialects/arith.py @@ -5,8 +5,6 @@ from ._arith_ops_gen import * from ._arith_ops_gen import _Dialect from ._arith_enum_gen import * -from array import array as _array -from typing import overload try: from ..ir import * @@ -45,30 +43,13 @@ def _is_float_type(type: Type): class ConstantOp(ConstantOp): """Specialization for the constant op class.""" - @overload - def __init__(self, value: Attribute, *, loc=None, ip=None): - ... - - @overload def __init__( - self, result: Type, value: Union[int, float, _array], *, loc=None, ip=None + self, result: Type, value: Union[int, float, Attribute], *, loc=None, ip=None ): - ... - - def __init__(self, result, value, *, loc=None, ip=None): - if value is None: - assert isinstance(result, Attribute) - super().__init__(result, loc=loc, ip=ip) - return - if isinstance(value, int): super().__init__(IntegerAttr.get(result, value), loc=loc, ip=ip) elif isinstance(value, float): super().__init__(FloatAttr.get(result, value), loc=loc, ip=ip) - elif isinstance(value, _array) and value.typecode in ["i", "l"]: - super().__init__(DenseIntElementsAttr.get(value, type=result)) - elif isinstance(value, _array) and value.typecode in ["f", "d"]: - super().__init__(DenseFPElementsAttr.get(value, type=result)) else: super().__init__(value, loc=loc, ip=ip) @@ -98,6 +79,6 @@ class ConstantOp(ConstantOp): def constant( - result: Type, value: Union[int, float, Attribute, _array], *, loc=None, ip=None + result: Type, value: Union[int, float, Attribute], *, loc=None, ip=None ) -> Value: return _get_op_result_or_op_results(ConstantOp(result, value, loc=loc, ip=ip)) diff --git a/mlir/test/python/dialects/arith_dialect.py b/mlir/test/python/dialects/arith_dialect.py index ef0e1620bba9..8bb80eed2b81 100644 --- a/mlir/test/python/dialects/arith_dialect.py +++ b/mlir/test/python/dialects/arith_dialect.py @@ -4,7 +4,6 @@ from functools import partialmethod from mlir.ir import * import mlir.dialects.arith as arith import mlir.dialects.func as func -from array import array def run(f): @@ -93,40 +92,3 @@ def testArithValue(): b = a * a # CHECK: ArithValue(%2 = arith.mulf %cst_1, %cst_1 : f64) print(b) - - -# CHECK-LABEL: TEST: testArrayConstantConstruction -@run -def testArrayConstantConstruction(): - with Context(), Location.unknown(): - module = Module.create() - with InsertionPoint(module.body): - i32_array = array("i", [1, 2, 3, 4]) - i32 = IntegerType.get_signless(32) - vec_i32 = VectorType.get([2, 2], i32) - arith.constant(vec_i32, i32_array) - arith.ConstantOp(vec_i32, DenseIntElementsAttr.get(i32_array, type=vec_i32)) - - i64_array = array("l", [5, 6, 7, 8]) - i64 = IntegerType.get_signless(64) - vec_i64 = VectorType.get([1, 4], i64) - arith.constant(vec_i64, i64_array) - arith.ConstantOp(vec_i64, DenseIntElementsAttr.get(i64_array, type=vec_i64)) - - f32_array = array("f", [1.0, 2.0, 3.0, 4.0]) - f32 = F32Type.get() - vec_f32 = VectorType.get([4, 1], f32) - arith.constant(vec_f32, f32_array) - arith.ConstantOp(vec_f32, DenseFPElementsAttr.get(f32_array, type=vec_f32)) - - f64_array = array("d", [1.0, 2.0, 3.0, 4.0]) - f64 = F64Type.get() - vec_f64 = VectorType.get([2, 1, 2], f64) - arith.constant(vec_f64, f64_array) - arith.ConstantOp(vec_f64, DenseFPElementsAttr.get(f64_array, type=vec_f64)) - - # CHECK-COUNT-2: arith.constant dense<[{{\[}}1, 2], [3, 4]]> : vector<2x2xi32> - # CHECK-COUNT-2: arith.constant dense<[{{\[}}5, 6, 7, 8]]> : vector<1x4xi64> - # CHECK-COUNT-2: arith.constant dense<[{{\[}}1.000000e+00], [2.000000e+00], [3.000000e+00], [4.000000e+00]]> : vector<4x1xf32> - # CHECK-COUNT-2: arith.constant dense<[{{\[}}[1.000000e+00, 2.000000e+00]], [{{\[}}3.000000e+00, 4.000000e+00]]]> : vector<2x1x2xf64> - print(module) -- GitLab From 929ceec70578c24b47018e7cf11ed3bd99896515 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Wed, 6 Mar 2024 03:03:27 +0000 Subject: [PATCH 236/929] [gn build] Port 85388a06b602 --- llvm/utils/gn/secondary/llvm/unittests/Target/RISCV/BUILD.gn | 1 - llvm/utils/gn/secondary/llvm/unittests/TargetParser/BUILD.gn | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/utils/gn/secondary/llvm/unittests/Target/RISCV/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Target/RISCV/BUILD.gn index 25dd29143a14..06b147ece9f6 100644 --- a/llvm/utils/gn/secondary/llvm/unittests/Target/RISCV/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/unittests/Target/RISCV/BUILD.gn @@ -15,7 +15,6 @@ unittest("RISCVTests") { include_dirs = [ "//llvm/lib/Target/RISCV" ] sources = [ "MCInstrAnalysisTest.cpp", - "RISCVBaseInfoTest.cpp", "RISCVInstrInfoTest.cpp", ] } diff --git a/llvm/utils/gn/secondary/llvm/unittests/TargetParser/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/TargetParser/BUILD.gn index c676e06d9529..eebaf67767f4 100644 --- a/llvm/utils/gn/secondary/llvm/unittests/TargetParser/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/unittests/TargetParser/BUILD.gn @@ -10,6 +10,7 @@ unittest("TargetParserTests") { sources = [ "CSKYTargetParserTest.cpp", "Host.cpp", + "RISCVTargetParserTest.cpp", "TargetParserTest.cpp", "TripleTest.cpp", ] -- GitLab From 0fa04b6e2cd2169a8e3d22ae879394dbf07c0466 Mon Sep 17 00:00:00 2001 From: Ye Luo Date: Tue, 5 Mar 2024 21:32:48 -0600 Subject: [PATCH 237/929] [libomptarget] Fix libomptarget.rtl.amdgpu.so installation If AMD GPUs don't exist when building libomptarget, the early return causes skipping the plugin installation. --- openmp/libomptarget/plugins-nextgen/amdgpu/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/openmp/libomptarget/plugins-nextgen/amdgpu/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/amdgpu/CMakeLists.txt index 9e0ea08d8375..8fbfe4d9b13f 100644 --- a/openmp/libomptarget/plugins-nextgen/amdgpu/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/amdgpu/CMakeLists.txt @@ -109,7 +109,6 @@ if (LIBOMPTARGET_FOUND_AMDGPU_GPU OR LIBOMPTARGET_FORCE_AMDGPU_TESTS) else() libomptarget_say("Not generating AMDGPU tests, no supported devices detected." " Use 'LIBOMPTARGET_FORCE_AMDGPU_TESTS' to override.") - return() endif() # Install plugin under the lib destination folder. -- GitLab From ae709c192953be2af61f47e91a3ec9f8d5ffb027 Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Wed, 6 Mar 2024 11:49:28 +0800 Subject: [PATCH 238/929] [RISCV] Add TargetParser to MCA (#84109) This fixes buildbot failures: https://lab.llvm.org/buildbot/#/builders/57/builds/33206 --- llvm/lib/Target/RISCV/MCA/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Target/RISCV/MCA/CMakeLists.txt b/llvm/lib/Target/RISCV/MCA/CMakeLists.txt index 036cd4e0034b..b8f294b2f3e2 100644 --- a/llvm/lib/Target/RISCV/MCA/CMakeLists.txt +++ b/llvm/lib/Target/RISCV/MCA/CMakeLists.txt @@ -9,6 +9,7 @@ add_llvm_component_library(LLVMRISCVTargetMCA RISCVDesc RISCVInfo Support + TargetParser ADD_TO_COMPONENT RISCV -- GitLab From 403b9cf1bb29d550c419ac5ecd2f764bf5cfa3c0 Mon Sep 17 00:00:00 2001 From: Heejin Ahn Date: Tue, 5 Mar 2024 19:54:41 -0800 Subject: [PATCH 239/929] [WebAssembly] Use RefTypeMem2Local instead of Mem2Reg (#83196) When reference-types feature is enabled, forcing mem2reg unconditionally even in `-O0` has some problems described in #81575. This uses RefTypeMem2Local pass added in #81965 instead. This also removes `IsForced` parameter added in https://github.com/llvm/llvm-project/commit/890146b19206827bc48ee1ae1dc1534ff2ff18d7 given that we don't need it anymore. This may still hurt debug info related to reference type variables a little during the backend transformation given that they are not stored in memory anymore, but reference type variables are presumably rare and it would be still a lot less damage than forcing mem2reg on the whole program. Also this fixes the EH problem described in #81575. Fixes #81575. --- llvm/include/llvm/Transforms/Utils.h | 2 +- .../Target/WebAssembly/WebAssemblyTargetMachine.cpp | 5 +++-- llvm/lib/Transforms/Utils/Mem2Reg.cpp | 12 ++++-------- llvm/test/CodeGen/WebAssembly/ref-type-mem2local.ll | 2 +- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/llvm/include/llvm/Transforms/Utils.h b/llvm/include/llvm/Transforms/Utils.h index 0fa6de3f6713..c6a6a05f3fdd 100644 --- a/llvm/include/llvm/Transforms/Utils.h +++ b/llvm/include/llvm/Transforms/Utils.h @@ -70,7 +70,7 @@ extern char &LCSSAID; // %Y = load i32* %X // ret i32 %Y // -FunctionPass *createPromoteMemoryToRegisterPass(bool IsForced = false); +FunctionPass *createPromoteMemoryToRegisterPass(); //===----------------------------------------------------------------------===// // diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp index 4d4cae110148..70685b2e3bb2 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp @@ -478,8 +478,9 @@ void WebAssemblyPassConfig::addISelPrepare() { WasmTM->getSubtargetImpl(std::string(WasmTM->getTargetCPU()), std::string(WasmTM->getTargetFeatureString())); if (Subtarget->hasReferenceTypes()) { - // We need to remove allocas for reference types - addPass(createPromoteMemoryToRegisterPass(true)); + // We need to move reference type allocas to WASM_ADDRESS_SPACE_VAR so that + // loads and stores are promoted to local.gets/local.sets. + addPass(createWebAssemblyRefTypeMem2Local()); } // Lower atomics and TLS if necessary addPass(new CoalesceFeaturesAndStripAtomics(&getWebAssemblyTargetMachine())); diff --git a/llvm/lib/Transforms/Utils/Mem2Reg.cpp b/llvm/lib/Transforms/Utils/Mem2Reg.cpp index fbc6dd7613de..5ad7aeb463ec 100644 --- a/llvm/lib/Transforms/Utils/Mem2Reg.cpp +++ b/llvm/lib/Transforms/Utils/Mem2Reg.cpp @@ -74,19 +74,15 @@ namespace { struct PromoteLegacyPass : public FunctionPass { // Pass identification, replacement for typeid static char ID; - bool ForcePass; /// If true, forces pass to execute, instead of skipping. - PromoteLegacyPass() : FunctionPass(ID), ForcePass(false) { - initializePromoteLegacyPassPass(*PassRegistry::getPassRegistry()); - } - PromoteLegacyPass(bool IsForced) : FunctionPass(ID), ForcePass(IsForced) { + PromoteLegacyPass() : FunctionPass(ID) { initializePromoteLegacyPassPass(*PassRegistry::getPassRegistry()); } // runOnFunction - To run this pass, first we calculate the alloca // instructions that are safe for promotion, then we promote each one. bool runOnFunction(Function &F) override { - if (!ForcePass && skipFunction(F)) + if (skipFunction(F)) return false; DominatorTree &DT = getAnalysis().getDomTree(); @@ -115,6 +111,6 @@ INITIALIZE_PASS_END(PromoteLegacyPass, "mem2reg", "Promote Memory to Register", false, false) // createPromoteMemoryToRegister - Provide an entry point to create this pass. -FunctionPass *llvm::createPromoteMemoryToRegisterPass(bool IsForced) { - return new PromoteLegacyPass(IsForced); +FunctionPass *llvm::createPromoteMemoryToRegisterPass() { + return new PromoteLegacyPass(); } diff --git a/llvm/test/CodeGen/WebAssembly/ref-type-mem2local.ll b/llvm/test/CodeGen/WebAssembly/ref-type-mem2local.ll index a38243ca218c..4b32a0945e2e 100644 --- a/llvm/test/CodeGen/WebAssembly/ref-type-mem2local.ll +++ b/llvm/test/CodeGen/WebAssembly/ref-type-mem2local.ll @@ -1,4 +1,4 @@ -; RUN: opt < %s -wasm-ref-type-mem2local -S | FileCheck %s +; RUN: llc < %s -mattr=+reference-types -stop-after=wasm-ref-type-mem2local | FileCheck %s target triple = "wasm32-unknown-unknown" -- GitLab From 433b71188da9649a9040f0db5338c65369fa3e90 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Tue, 5 Mar 2024 22:03:00 -0600 Subject: [PATCH 240/929] [CUDA] Correctly set CUDA default architecture (#84017) Summary: We already had a special CUDA default that better tracked the state as of modern CUDA installations. Recently this was bumped up to `sm_52`, but there was a location that wasn't respecting this. Fix that. --- clang/lib/Driver/Driver.cpp | 2 +- clang/test/Driver/cuda-detect.cu | 5 +- clang/test/Driver/cuda-march.cu | 12 +- .../cuda-omp-unsupported-debug-options.cu | 48 ++--- clang/test/Driver/cuda-options.cu | 174 +++++++++--------- clang/test/Driver/cuda-ptxas-path.cu | 2 +- .../test/Driver/dwarf-target-version-clamp.cu | 4 +- 7 files changed, 123 insertions(+), 124 deletions(-) diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp index de8ceb2f0898..cecd34acbc92 100644 --- a/clang/lib/Driver/Driver.cpp +++ b/clang/lib/Driver/Driver.cpp @@ -3234,7 +3234,7 @@ class OffloadingActionBuilder final { CudaActionBuilder(Compilation &C, DerivedArgList &Args, const Driver::InputList &Inputs) : CudaActionBuilderBase(C, Args, Inputs, Action::OFK_Cuda) { - DefaultCudaArch = CudaArch::SM_35; + DefaultCudaArch = CudaArch::CudaDefault; } StringRef getCanonicalOffloadArch(StringRef ArchStr) override { diff --git a/clang/test/Driver/cuda-detect.cu b/clang/test/Driver/cuda-detect.cu index 077d555a3128..49e58004e672 100644 --- a/clang/test/Driver/cuda-detect.cu +++ b/clang/test/Driver/cuda-detect.cu @@ -11,7 +11,6 @@ // RUN: %clang -v --target=x86_64-apple-macosx \ // RUN: --sysroot=%S/no-cuda-there --cuda-path-ignore-env 2>&1 | FileCheck %s -check-prefix NOCUDA - // RUN: %clang -v --target=i386-unknown-linux \ // RUN: --sysroot=%S/Inputs/CUDA --cuda-path-ignore-env 2>&1 | FileCheck %s // RUN: %clang -v --target=i386-apple-macosx \ @@ -146,9 +145,9 @@ // RUN: -check-prefix NOCUDAINC // Verify that C++ include paths are passed for both host and device frontends. -// RUN: not %clang -### --target=x86_64-linux-gnu %s \ +// RUN: %clang -### --target=x86_64-linux-gnu %s \ // RUN: --stdlib=libstdc++ --sysroot=%S/Inputs/ubuntu_14.04_multiarch_tree2 \ -// RUN: --gcc-toolchain="" 2>&1 \ +// RUN: -nogpulib -nogpuinc --gcc-toolchain="" 2>&1 \ // RUN: | FileCheck %s --check-prefix CHECK-CXXINCLUDE // Verify that CUDA SDK version is propagated to the CC1 compilations. diff --git a/clang/test/Driver/cuda-march.cu b/clang/test/Driver/cuda-march.cu index 7003e9fd4198..25fd6f3a10f2 100644 --- a/clang/test/Driver/cuda-march.cu +++ b/clang/test/Driver/cuda-march.cu @@ -5,15 +5,15 @@ // REQUIRES: x86-registered-target // REQUIRES: nvptx-registered-target -// RUN: not %clang -### --target=x86_64-linux-gnu -c \ -// RUN: -march=haswell %s 2>&1 | FileCheck %s -// RUN: not %clang -### --target=x86_64-linux-gnu -c \ -// RUN: -march=haswell --cuda-gpu-arch=sm_35 %s 2>&1 | FileCheck %s +// RUN: %clang -### --target=x86_64-linux-gnu -c \ +// RUN: -nogpulib -nogpuinc -march=haswell %s 2>&1 | FileCheck %s +// RUN: %clang -### --target=x86_64-linux-gnu -c \ +// RUN: -nogpulib -nogpuinc -march=haswell --cuda-gpu-arch=sm_52 %s 2>&1 | FileCheck %s // CHECK: "-cc1"{{.*}} "-triple" "nvptx -// CHECK-SAME: "-target-cpu" "sm_35" +// CHECK-SAME: "-target-cpu" "sm_52" // CHECK: ptxas -// CHECK-SAME: "--gpu-name" "sm_35" +// CHECK-SAME: "--gpu-name" "sm_52" // CHECK: "-cc1"{{.*}} "-target-cpu" "haswell" diff --git a/clang/test/Driver/cuda-omp-unsupported-debug-options.cu b/clang/test/Driver/cuda-omp-unsupported-debug-options.cu index 77cfa6925e38..8e1bb2e496c7 100644 --- a/clang/test/Driver/cuda-omp-unsupported-debug-options.cu +++ b/clang/test/Driver/cuda-omp-unsupported-debug-options.cu @@ -2,56 +2,56 @@ // REQUIRES: nvptx-registered-target // REQUIRES: zlib -// RUN: not %clang -### --target=x86_64-linux-gnu -c %s -g -gz 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -c %s -g -gz 2>&1 \ // RUN: | FileCheck %s --check-prefixes WARN,COMMON -// RUN: not %clang -### --target=x86_64-linux-gnu -c %s -gdwarf -fdebug-info-for-profiling 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -c %s -gdwarf -fdebug-info-for-profiling 2>&1 \ // RUN: | FileCheck %s --check-prefixes WARN,COMMON -// RUN: not %clang -### --target=x86_64-linux-gnu -c %s -gdwarf-2 -gsplit-dwarf 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -c %s -gdwarf-2 -gsplit-dwarf 2>&1 \ // RUN: | FileCheck %s --check-prefixes WARN,COMMON -// RUN: not %clang -### --target=x86_64-linux-gnu -c %s -gdwarf-3 -glldb 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -c %s -gdwarf-3 -glldb 2>&1 \ // RUN: | FileCheck %s --check-prefixes WARN,COMMON -// RUN: not %clang -### --target=x86_64-linux-gnu -c %s -gdwarf-4 -gcodeview 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -c %s -gdwarf-4 -gcodeview 2>&1 \ // RUN: | FileCheck %s --check-prefixes WARN,COMMON -// RUN: not %clang -### --target=x86_64-linux-gnu -c %s -gdwarf-5 -gmodules 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -c %s -gdwarf-5 -gmodules 2>&1 \ // RUN: | FileCheck %s --check-prefixes WARN,COMMON -// RUN: not %clang -### --target=x86_64-linux-gnu -c %s -ggdb1 -fdebug-macro 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -c %s -ggdb1 -fdebug-macro 2>&1 \ // RUN: | FileCheck %s --check-prefixes WARN,COMMON -// RUN: not %clang -### --target=x86_64-linux-gnu -c %s -ggdb2 -ggnu-pubnames 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -c %s -ggdb2 -ggnu-pubnames 2>&1 \ // RUN: | FileCheck %s --check-prefixes WARN,COMMON -// RUN: not %clang -### --target=x86_64-linux-gnu -c %s -ggdb3 -gdwarf-aranges 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -c %s -ggdb3 -gdwarf-aranges 2>&1 \ // RUN: | FileCheck %s --check-prefixes WARN,COMMON -// RUN: not %clang -### --target=x86_64-linux-gnu -c %s -g -gcolumn-info -fdebug-types-section 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -c %s -g -gcolumn-info -fdebug-types-section 2>&1 \ // RUN: | FileCheck %s --check-prefixes WARN,COMMON // Same tests for OpenMP -// RUN: not %clang -### --target=x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -fopenmp=libomp -c %s \ // RUN: -fgpu-rdc -g -gz 2>&1 | FileCheck %s --check-prefixes WARN,COMMON -// RUN: not %clang -### --target=x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -fopenmp=libomp -c %s \ // RUN: -fgpu-rdc -gdwarf -fdebug-info-for-profiling 2>&1 | FileCheck %s --check-prefixes WARN,COMMON -// RUN: not %clang -### --target=x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -fopenmp=libomp -c %s \ // RUN: -fgpu-rdc -gdwarf-2 -gsplit-dwarf 2>&1 | FileCheck %s --check-prefixes WARN,COMMON -// RUN: not %clang -### --target=x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -fopenmp=libomp -c %s \ // RUN: -fgpu-rdc -gdwarf-3 -glldb 2>&1 | FileCheck %s --check-prefixes WARN,COMMON -// RUN: not %clang -### --target=x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -fopenmp=libomp -c %s \ // RUN: -fgpu-rdc -gdwarf-4 -gcodeview 2>&1 | FileCheck %s --check-prefixes WARN,COMMON -// RUN: not %clang -### --target=x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -fopenmp=libomp -c %s \ // RUN: -fgpu-rdc -gdwarf-5 -gmodules 2>&1 | FileCheck %s --check-prefixes WARN,COMMON -// RUN: not %clang -### --target=x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -fopenmp=libomp -c %s \ // RUN: -fgpu-rdc -ggdb1 -fdebug-macro 2>&1 | FileCheck %s --check-prefixes WARN,COMMON -// RUN: not %clang -### --target=x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -fopenmp=libomp -c %s \ // RUN: -fgpu-rdc -ggdb2 -ggnu-pubnames 2>&1 | FileCheck %s --check-prefixes WARN,COMMON -// RUN: not %clang -### --target=x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -fopenmp=libomp -c %s \ // RUN: -fgpu-rdc -ggdb3 -gdwarf-aranges 2>&1 | FileCheck %s --check-prefixes WARN,COMMON -// RUN: not %clang -### --target=x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -fopenmp=libomp -c %s \ // RUN: -fgpu-rdc -g -gcolumn-info -fdebug-types-section 2>&1 | FileCheck %s --check-prefixes WARN,COMMON -// RUN: not %clang -### --target=x86_64-linux-gnu -c %s -gdwarf-5 -gembed-source 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -c %s -gdwarf-5 -gembed-source 2>&1 \ // RUN: | FileCheck %s --check-prefixes WARN-GES,COMMON -// RUN: not %clang -### --target=x86_64-linux-gnu -c %s -ggdb -gembed-source -gdwarf-5 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -c %s -ggdb -gembed-source -gdwarf-5 2>&1 \ // RUN: | FileCheck %s --check-prefixes WARN-GES,COMMON -// RUN: not %clang -### --target=x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -fopenmp=libomp -c %s \ // RUN: -fgpu-rdc -gdwarf-5 -gembed-source 2>&1 | FileCheck %s --check-prefixes WARN-GES,COMMON -// RUN: not %clang -### --target=x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \ +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=sm_52 -nogpulib -nogpuinc -fopenmp=libomp -c %s \ // RUN: -fgpu-rdc -ggdb -gembed-source -gdwarf-5 2>&1 | FileCheck %s --check-prefixes WARN-GES,COMMON // COMMON: warning: debug information option '{{-gz|-fdebug-info-for-profiling|-gsplit-dwarf|-glldb|-gcodeview|-gmodules|-gembed-source|-fdebug-macro|-ggnu-pubnames|-gdwarf-aranges|-fdebug-types-section}}' is not supported diff --git a/clang/test/Driver/cuda-options.cu b/clang/test/Driver/cuda-options.cu index ad892b7839b9..8999a6618fe1 100644 --- a/clang/test/Driver/cuda-options.cu +++ b/clang/test/Driver/cuda-options.cu @@ -4,197 +4,197 @@ // Simple compilation case. Compile device-side to PTX assembly and make sure // we use it on the host side. -// RUN: %clang -### -target x86_64-linux-gnu -c --cuda-path=%S/Inputs/CUDA/usr/local/cuda %s 2>&1 \ +// RUN: %clang -### -target x86_64-linux-gnu -c -nogpulib -nogpuinc %s 2>&1 \ // RUN: | FileCheck -check-prefix DEVICE -check-prefix DEVICE-NOSAVE \ // RUN: -check-prefix HOST -check-prefix INCLUDES-DEVICE \ // RUN: -check-prefix NOLINK %s // Typical compilation + link case. -// RUN: %clang -### -target x86_64-linux-gnu --cuda-path=%S/Inputs/CUDA/usr/local/cuda %s 2>&1 \ +// RUN: %clang -### -target x86_64-linux-gnu -nogpulib -nogpuinc %s 2>&1 \ // RUN: | FileCheck -check-prefix DEVICE -check-prefix DEVICE-NOSAVE \ // RUN: -check-prefix HOST -check-prefix INCLUDES-DEVICE \ // RUN: -check-prefix LINK %s // Verify that --cuda-host-only disables device-side compilation, but doesn't // disable host-side compilation/linking. -// RUN: %clang -### -target x86_64-linux-gnu --cuda-host-only --cuda-path=%S/Inputs/CUDA/usr/local/cuda %s 2>&1 \ +// RUN: %clang -### -target x86_64-linux-gnu --cuda-host-only -nogpulib -nogpuinc %s 2>&1 \ // RUN: | FileCheck -check-prefix NODEVICE -check-prefix HOST \ // RUN: -check-prefix NOINCLUDES-DEVICE -check-prefix LINK %s // Verify that --cuda-device-only disables host-side compilation and linking. -// RUN: %clang -### -target x86_64-linux-gnu --cuda-device-only --cuda-path=%S/Inputs/CUDA/usr/local/cuda %s 2>&1 \ +// RUN: %clang -### -target x86_64-linux-gnu --cuda-device-only -nogpulib -nogpuinc %s 2>&1 \ // RUN: | FileCheck -check-prefix DEVICE -check-prefix DEVICE-NOSAVE \ // RUN: -check-prefix NOHOST -check-prefix NOLINK %s // Check that the last of --cuda-compile-host-device, --cuda-host-only, and // --cuda-device-only wins. -// RUN: %clang -### -target x86_64-linux-gnu --cuda-device-only \ -// RUN: --cuda-host-only --cuda-path=%S/Inputs/CUDA/usr/local/cuda %s 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu --cuda-device-only \ +// RUN: --cuda-host-only -nogpulib -nogpuinc %s 2>&1 \ // RUN: | FileCheck -check-prefix NODEVICE -check-prefix HOST \ // RUN: -check-prefix NOINCLUDES-DEVICE -check-prefix LINK %s -// RUN: %clang -### -target x86_64-linux-gnu --cuda-compile-host-device \ -// RUN: --cuda-host-only --cuda-path=%S/Inputs/CUDA/usr/local/cuda %s 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu --cuda-compile-host-device \ +// RUN: --cuda-host-only -nogpulib -nogpuinc %s 2>&1 \ // RUN: | FileCheck -check-prefix NODEVICE -check-prefix HOST \ // RUN: -check-prefix NOINCLUDES-DEVICE -check-prefix LINK %s -// RUN: not %clang -### --target=x86_64-linux-gnu --cuda-host-only \ -// RUN: --cuda-device-only %s 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu --cuda-host-only \ +// RUN: -nogpulib -nogpuinc --cuda-device-only %s 2>&1 \ // RUN: | FileCheck -check-prefix DEVICE -check-prefix DEVICE-NOSAVE \ // RUN: -check-prefix NOHOST -check-prefix NOLINK %s -// RUN: not %clang -### --target=x86_64-linux-gnu --cuda-compile-host-device \ -// RUN: --cuda-device-only %s 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu --cuda-compile-host-device \ +// RUN: -nogpulib -nogpuinc --cuda-device-only %s 2>&1 \ // RUN: | FileCheck -check-prefix DEVICE -check-prefix DEVICE-NOSAVE \ // RUN: -check-prefix NOHOST -check-prefix NOLINK %s -// RUN: not %clang -### --target=x86_64-linux-gnu --cuda-host-only \ -// RUN: --cuda-compile-host-device %s 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu --cuda-host-only \ +// RUN: -nogpulib -nogpuinc --cuda-compile-host-device %s 2>&1 \ // RUN: | FileCheck -check-prefix DEVICE -check-prefix DEVICE-NOSAVE \ // RUN: -check-prefix HOST -check-prefix INCLUDES-DEVICE \ // RUN: -check-prefix LINK %s -// RUN: not %clang -### --target=x86_64-linux-gnu --cuda-device-only \ -// RUN: --cuda-compile-host-device %s 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu --cuda-device-only \ +// RUN: -nogpulib -nogpuinc --cuda-compile-host-device %s 2>&1 \ // RUN: | FileCheck -check-prefix DEVICE -check-prefix DEVICE-NOSAVE \ // RUN: -check-prefix HOST -check-prefix INCLUDES-DEVICE \ // RUN: -check-prefix LINK %s // Verify that --cuda-gpu-arch option passes the correct GPU architecture to // device compilation. -// RUN: not %clang -### --target=x86_64-linux-gnu --cuda-gpu-arch=sm_30 -c %s 2>&1 \ +// RUN: %clang -### -nogpulib -nogpuinc --target=x86_64-linux-gnu --cuda-gpu-arch=sm_52 -c %s 2>&1 \ // RUN: | FileCheck -check-prefix DEVICE -check-prefix DEVICE-NOSAVE \ -// RUN: -check-prefix DEVICE-SM30 -check-prefix HOST \ +// RUN: -check-prefix DEVICE-SM52 -check-prefix HOST \ // RUN: -check-prefix INCLUDES-DEVICE -check-prefix NOLINK %s // Verify that there is one device-side compilation per --cuda-gpu-arch args // and that all results are included on the host side. -// RUN: not %clang -### --target=x86_64-linux-gnu \ -// RUN: --cuda-gpu-arch=sm_35 --cuda-gpu-arch=sm_30 -c %s 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu \ +// RUN: -nogpulib -nogpuinc --cuda-gpu-arch=sm_60 --cuda-gpu-arch=sm_52 -c %s 2>&1 \ // RUN: | FileCheck -check-prefixes DEVICE,DEVICE-NOSAVE,DEVICE2 \ -// RUN: -check-prefixes DEVICE-SM30,DEVICE2-SM35 \ +// RUN: -check-prefixes DEVICE-SM52,DEVICE2-SM60 \ // RUN: -check-prefixes INCLUDES-DEVICE,INCLUDES-DEVICE2 \ // RUN: -check-prefixes HOST,HOST-NOSAVE,NOLINK %s // Verify that device-side results are passed to the correct tool when // -save-temps is used. -// RUN: not %clang -### --target=x86_64-linux-gnu -save-temps -c %s 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -nogpuinc -save-temps -c %s 2>&1 \ // RUN: | FileCheck -check-prefix DEVICE -check-prefix DEVICE-SAVE \ // RUN: -check-prefix HOST -check-prefix HOST-SAVE -check-prefix NOLINK %s // Verify that device-side results are passed to the correct tool when // -fno-integrated-as is used. -// RUN: not %clang -### --target=x86_64-linux-gnu -fno-integrated-as -c %s 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -nogpuinc -fno-integrated-as -c %s 2>&1 \ // RUN: | FileCheck -check-prefix DEVICE -check-prefix DEVICE-NOSAVE \ // RUN: -check-prefix HOST -check-prefix HOST-NOSAVE \ // RUN: -check-prefix HOST-AS -check-prefix NOLINK %s // Verify that --[no-]cuda-gpu-arch arguments are handled correctly. // a) --no-cuda-gpu-arch=X negates preceding --cuda-gpu-arch=X -// RUN: not %clang -### --target=x86_64-linux-gnu --cuda-device-only \ -// RUN: --cuda-gpu-arch=sm_50 --cuda-gpu-arch=sm_30 \ -// RUN: --no-cuda-gpu-arch=sm_50 \ +// RUN: %clang -### --target=x86_64-linux-gnu --cuda-device-only \ +// RUN: -nogpulib -nogpuinc --cuda-gpu-arch=sm_70 --cuda-gpu-arch=sm_52 \ +// RUN: --no-cuda-gpu-arch=sm_70 \ // RUN: -c %s 2>&1 \ -// RUN: | FileCheck -check-prefixes ARCH-SM30,NOARCH-SM35,NOARCH-SM50 %s +// RUN: | FileCheck -check-prefixes ARCH-SM52,NOARCH-SM60,NOARCH-SM70 %s // b) --no-cuda-gpu-arch=X negates more than one preceding --cuda-gpu-arch=X -// RUN: not %clang -### --target=x86_64-linux-gnu --cuda-device-only \ -// RUN: --cuda-gpu-arch=sm_50 --cuda-gpu-arch=sm_50 --cuda-gpu-arch=sm_30 \ -// RUN: --no-cuda-gpu-arch=sm_50 \ +// RUN: %clang -### --target=x86_64-linux-gnu --cuda-device-only \ +// RUN: -nogpulib -nogpuinc --cuda-gpu-arch=sm_70 --cuda-gpu-arch=sm_70 --cuda-gpu-arch=sm_52 \ +// RUN: --no-cuda-gpu-arch=sm_70 \ // RUN: -c %s 2>&1 \ -// RUN: | FileCheck -check-prefixes ARCH-SM30,NOARCH-SM35,NOARCH-SM50 %s +// RUN: | FileCheck -check-prefixes ARCH-SM52,NOARCH-SM60,NOARCH-SM70 %s // c) if --no-cuda-gpu-arch=X negates all preceding --cuda-gpu-arch=X -// we default to sm_35 -- same as if no --cuda-gpu-arch were passed. -// RUN: not %clang -### --target=x86_64-linux-gnu --cuda-device-only \ -// RUN: --cuda-gpu-arch=sm_50 --cuda-gpu-arch=sm_30 \ -// RUN: --no-cuda-gpu-arch=sm_50 --no-cuda-gpu-arch=sm_30 \ +// we default to sm_52 -- same as if no --cuda-gpu-arch were passed. +// RUN: %clang -### --target=x86_64-linux-gnu --cuda-device-only \ +// RUN: -nogpulib -nogpuinc --cuda-gpu-arch=sm_70 --cuda-gpu-arch=sm_60 \ +// RUN: --no-cuda-gpu-arch=sm_70 --no-cuda-gpu-arch=sm_60 \ // RUN: -c %s 2>&1 \ -// RUN: | FileCheck -check-prefixes NOARCH-SM30,ARCH-SM35,NOARCH-SM50 %s +// RUN: | FileCheck -check-prefixes ARCH-SM52,NOARCH-SM60,NOARCH-SM70 %s // d) --no-cuda-gpu-arch=X is a no-op if there's no preceding --cuda-gpu-arch=X -// RUN: not %clang -### --target=x86_64-linux-gnu --cuda-device-only \ -// RUN: --cuda-gpu-arch=sm_35 --cuda-gpu-arch=sm_30\ -// RUN: --no-cuda-gpu-arch=sm_50 \ +// RUN: %clang -### --target=x86_64-linux-gnu --cuda-device-only \ +// RUN: -nogpulib -nogpuinc --cuda-gpu-arch=sm_60 --cuda-gpu-arch=sm_52\ +// RUN: --no-cuda-gpu-arch=sm_70 \ // RUN: -c %s 2>&1 \ -// RUN: | FileCheck -check-prefixes ARCH-SM30,ARCH-SM35,NOARCH-SM50 %s +// RUN: | FileCheck -check-prefixes ARCH-SM52,ARCH-SM60,NOARCH-SM70 %s // e) --no-cuda-gpu-arch=X does not affect following --cuda-gpu-arch=X -// RUN: not %clang -### --target=x86_64-linux-gnu --cuda-device-only \ -// RUN: --no-cuda-gpu-arch=sm_50 --no-cuda-gpu-arch=sm_30 \ -// RUN: --cuda-gpu-arch=sm_50 --cuda-gpu-arch=sm_30 \ +// RUN: %clang -### --target=x86_64-linux-gnu --cuda-device-only \ +// RUN: -nogpulib -nogpuinc --no-cuda-gpu-arch=sm_70 --no-cuda-gpu-arch=sm_52 \ +// RUN: --cuda-gpu-arch=sm_70 --cuda-gpu-arch=sm_52 \ // RUN: -c %s 2>&1 \ -// RUN: | FileCheck -check-prefixes ARCH-SM30,NOARCH-SM35,ARCH-SM50 %s +// RUN: | FileCheck -check-prefixes ARCH-SM52,NOARCH-SM60,ARCH-SM70 %s // f) --no-cuda-gpu-arch=all negates all preceding --cuda-gpu-arch=X // RUN: %clang -### -target x86_64-linux-gnu --cuda-device-only \ -// RUN: --cuda-gpu-arch=sm_35 --cuda-gpu-arch=sm_30 \ +// RUN: -nogpulib -nogpuinc --cuda-gpu-arch=sm_60 --cuda-gpu-arch=sm_52 \ // RUN: --no-cuda-gpu-arch=all \ -// RUN: --cuda-gpu-arch=sm_50 \ -// RUN: -c --cuda-path=%S/Inputs/CUDA/usr/local/cuda %s 2>&1 \ -// RUN: | FileCheck -check-prefixes NOARCH-SM30,NOARCH-SM35,ARCH-SM50 %s +// RUN: --cuda-gpu-arch=sm_70 \ +// RUN: -c -nogpulib -nogpuinc %s 2>&1 \ +// RUN: | FileCheck -check-prefixes NOARCH-SM52,NOARCH-SM60,ARCH-SM70 %s // g) There's no --cuda-gpu-arch=all // RUN: not %clang -### --target=x86_64-linux-gnu --cuda-device-only \ -// RUN: --cuda-gpu-arch=all \ +// RUN: -nogpulib -nogpuinc --cuda-gpu-arch=all \ // RUN: -c %s 2>&1 \ // RUN: | FileCheck -check-prefix ARCHALLERROR %s // Verify that --[no-]cuda-include-ptx arguments are handled correctly. // a) by default we're including PTX for all GPUs. -// RUN: not %clang -### --target=x86_64-linux-gnu \ -// RUN: --cuda-gpu-arch=sm_35 --cuda-gpu-arch=sm_30 \ +// RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -nogpuinc \ +// RUN: --cuda-gpu-arch=sm_60 --cuda-gpu-arch=sm_52 \ // RUN: -c %s 2>&1 \ -// RUN: | FileCheck -check-prefixes FATBIN-COMMON,PTX-SM35,PTX-SM30 %s +// RUN: | FileCheck -check-prefixes FATBIN-COMMON,PTX-SM60,PTX-SM52 %s // b) --no-cuda-include-ptx=all disables PTX inclusion for all GPUs -// RUN: not %clang -### --target=x86_64-linux-gnu \ -// RUN: --cuda-gpu-arch=sm_35 --cuda-gpu-arch=sm_30 \ +// RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -nogpuinc \ +// RUN: --cuda-gpu-arch=sm_60 --cuda-gpu-arch=sm_52 \ // RUN: --no-cuda-include-ptx=all \ // RUN: -c %s 2>&1 \ -// RUN: | FileCheck -check-prefixes FATBIN-COMMON,NOPTX-SM35,NOPTX-SM30 %s +// RUN: | FileCheck -check-prefixes FATBIN-COMMON,NOPTX-SM60,NOPTX-SM52 %s // c) --no-cuda-include-ptx=sm_XX disables PTX inclusion for that GPU only. -// RUN: not %clang -### --target=x86_64-linux-gnu \ -// RUN: --cuda-gpu-arch=sm_35 --cuda-gpu-arch=sm_30 \ -// RUN: --no-cuda-include-ptx=sm_35 \ +// RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -nogpuinc \ +// RUN: --cuda-gpu-arch=sm_60 --cuda-gpu-arch=sm_52 \ +// RUN: --no-cuda-include-ptx=sm_60 \ // RUN: -c %s 2>&1 \ -// RUN: | FileCheck -check-prefixes FATBIN-COMMON,NOPTX-SM35,PTX-SM30 %s -// RUN: not %clang -### --target=x86_64-linux-gnu \ -// RUN: --cuda-gpu-arch=sm_35 --cuda-gpu-arch=sm_30 \ -// RUN: --no-cuda-include-ptx=sm_30 \ +// RUN: | FileCheck -check-prefixes FATBIN-COMMON,NOPTX-SM60,PTX-SM52 %s +// RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -nogpuinc \ +// RUN: --cuda-gpu-arch=sm_60 --cuda-gpu-arch=sm_52 \ +// RUN: --no-cuda-include-ptx=sm_52 \ // RUN: -c %s 2>&1 \ -// RUN: | FileCheck -check-prefixes FATBIN-COMMON,PTX-SM35,NOPTX-SM30 %s +// RUN: | FileCheck -check-prefixes FATBIN-COMMON,PTX-SM60,NOPTX-SM52 %s // d) --cuda-include-ptx=all overrides preceding --no-cuda-include-ptx=all -// RUN: not %clang -### --target=x86_64-linux-gnu \ -// RUN: --cuda-gpu-arch=sm_35 --cuda-gpu-arch=sm_30 \ +// RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -nogpuinc \ +// RUN: --cuda-gpu-arch=sm_60 --cuda-gpu-arch=sm_52 \ // RUN: --no-cuda-include-ptx=all --cuda-include-ptx=all \ // RUN: -c %s 2>&1 \ -// RUN: | FileCheck -check-prefixes FATBIN-COMMON,PTX-SM35,PTX-SM30 %s +// RUN: | FileCheck -check-prefixes FATBIN-COMMON,PTX-SM60,PTX-SM52 %s // e) --cuda-include-ptx=all overrides preceding --no-cuda-include-ptx=sm_XX -// RUN: not %clang -### --target=x86_64-linux-gnu \ -// RUN: --cuda-gpu-arch=sm_35 --cuda-gpu-arch=sm_30 \ -// RUN: --no-cuda-include-ptx=sm_30 --cuda-include-ptx=all \ +// RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -nogpuinc \ +// RUN: --cuda-gpu-arch=sm_60 --cuda-gpu-arch=sm_52 \ +// RUN: --no-cuda-include-ptx=sm_52 --cuda-include-ptx=all \ // RUN: -c %s 2>&1 \ -// RUN: | FileCheck -check-prefixes FATBIN-COMMON,PTX-SM35,PTX-SM30 %s +// RUN: | FileCheck -check-prefixes FATBIN-COMMON,PTX-SM60,PTX-SM52 %s // Verify -flto=thin -fwhole-program-vtables handling. This should result in // both options being passed to the host compilation, with neither passed to // the device compilation. -// RUN: not %clang -### --target=x86_64-linux-gnu -c -flto=thin -fwhole-program-vtables %s 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -nogpuinc -c -flto=thin -fwhole-program-vtables %s 2>&1 \ // RUN: | FileCheck -check-prefixes DEVICE,DEVICE-NOSAVE,HOST,INCLUDES-DEVICE,NOLINK,THINLTOWPD %s // THINLTOWPD-NOT: error: invalid argument '-fwhole-program-vtables' only allowed with '-flto' -// ARCH-SM30: "-cc1"{{.*}}"-target-cpu" "sm_30" -// NOARCH-SM30-NOT: "-cc1"{{.*}}"-target-cpu" "sm_30" -// ARCH-SM35: "-cc1"{{.*}}"-target-cpu" "sm_35" -// NOARCH-SM35-NOT: "-cc1"{{.*}}"-target-cpu" "sm_35" -// ARCH-SM50: "-cc1"{{.*}}"-target-cpu" "sm_50" -// NOARCH-SM50-NOT: "-cc1"{{.*}}"-target-cpu" "sm_50" +// ARCH-SM52: "-cc1"{{.*}}"-target-cpu" "sm_52" +// NOARCH-SM52-NOT: "-cc1"{{.*}}"-target-cpu" "sm_52" +// ARCH-SM60: "-cc1"{{.*}}"-target-cpu" "sm_60" +// NOARCH-SM60-NOT: "-cc1"{{.*}}"-target-cpu" "sm_60" +// ARCH-SM70: "-cc1"{{.*}}"-target-cpu" "sm_70" +// NOARCH-SM70-NOT: "-cc1"{{.*}}"-target-cpu" "sm_70" // ARCHALLERROR: error: unsupported CUDA gpu architecture: all // Match device-side preprocessor and compiler phases with -save-temps. @@ -213,7 +213,7 @@ // DEVICE-NOSAVE-SAME: "-aux-triple" "x86_64-unknown-linux-gnu" // THINLTOWPD-NOT: "-flto=thin" // DEVICE-SAME: "-fcuda-is-device" -// DEVICE-SM30-SAME: "-target-cpu" "sm_30" +// DEVICE-SM52-SAME: "-target-cpu" "sm_52" // THINLTOWPD-NOT: "-fwhole-program-vtables" // DEVICE-SAME: "-o" "[[PTXFILE:[^"]*]]" // DEVICE-NOSAVE-SAME: "-x" "cuda" @@ -221,7 +221,7 @@ // Match the call to ptxas (which assembles PTX to SASS). // DEVICE:ptxas -// DEVICE-SM30-DAG: "--gpu-name" "sm_30" +// DEVICE-SM52-DAG: "--gpu-name" "sm_52" // DEVICE-DAG: "--output-file" "[[CUBINFILE:[^"]*]]" // DEVICE-DAG: "[[PTXFILE]]" @@ -229,13 +229,13 @@ // DEVICE2: "-cc1" "-triple" "nvptx64-nvidia-cuda" // DEVICE2-SAME: "-aux-triple" "x86_64-unknown-linux-gnu" // DEVICE2-SAME: "-fcuda-is-device" -// DEVICE2-SM35-SAME: "-target-cpu" "sm_35" +// DEVICE2-SM60-SAME: "-target-cpu" "sm_60" // DEVICE2-SAME: "-o" "[[PTXFILE2:[^"]*]]" // DEVICE2-SAME: "-x" "cuda" // Match another call to ptxas. // DEVICE2: ptxas -// DEVICE2-SM35-DAG: "--gpu-name" "sm_35" +// DEVICE2-SM60-DAG: "--gpu-name" "sm_60" // DEVICE2-DAG: "--output-file" "[[CUBINFILE2:[^"]*]]" // DEVICE2-DAG: "[[PTXFILE2]]" @@ -290,9 +290,9 @@ // FATBIN-COMMON:fatbinary // FATBIN-COMMON: "--create" "[[FATBINARY:[^"]*]]" -// FATBIN-COMMON: "--image=profile=sm_30,file= -// PTX-SM30: "--image=profile=compute_30,file= -// NOPTX-SM30-NOT: "--image=profile=compute_30,file= -// FATBIN-COMMON: "--image=profile=sm_35,file= -// PTX-SM35: "--image=profile=compute_35,file= -// NOPTX-SM35-NOT: "--image=profile=compute_35,file= +// FATBIN-COMMON: "--image=profile=sm_52,file= +// PTX-SM52: "--image=profile=compute_52,file= +// NOPTX-SM52-NOT: "--image=profile=compute_52,file= +// FATBIN-COMMON: "--image=profile=sm_60,file= +// PTX-SM60: "--image=profile=compute_60,file= +// NOPTX-SM60-NOT: "--image=profile=compute_60,file= diff --git a/clang/test/Driver/cuda-ptxas-path.cu b/clang/test/Driver/cuda-ptxas-path.cu index 09c6014a91a2..87b19d9d4d61 100644 --- a/clang/test/Driver/cuda-ptxas-path.cu +++ b/clang/test/Driver/cuda-ptxas-path.cu @@ -8,4 +8,4 @@ // CHECK-NOT: "ptxas" // CHECK: "/some/path/to/ptxas" -// CHECK-SAME: "--gpu-name" "sm_35" +// CHECK-SAME: "--gpu-name" "sm_52" diff --git a/clang/test/Driver/dwarf-target-version-clamp.cu b/clang/test/Driver/dwarf-target-version-clamp.cu index ec0386afbc57..d9dbbe62cfb2 100644 --- a/clang/test/Driver/dwarf-target-version-clamp.cu +++ b/clang/test/Driver/dwarf-target-version-clamp.cu @@ -2,9 +2,9 @@ // REQUIRES: nvptx-registered-target // Verify that DWARF version is properly clamped for nvptx, but not for the host. -// RUN: not %clang -### --target=x86_64-linux-gnu -c %s -gdwarf-5 -gembed-source 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -nogpuinc -c %s -gdwarf-5 -gembed-source 2>&1 \ // RUN: | FileCheck %s --check-prefix=DWARF-CLAMP -// RUN: not %clang -### --target=x86_64-linux-gnu -c %s -ggdb -gembed-source -gdwarf-5 2>&1 \ +// RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -nogpuinc -c %s -ggdb -gembed-source -gdwarf-5 2>&1 \ // RUN: | FileCheck %s --check-prefix=DWARF-CLAMP // DWARF-CLAMP: "-triple" "nvptx64-nvidia-cuda" -- GitLab From a30233f5765071c8b269189758e8b907e19c4724 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Tue, 5 Mar 2024 22:03:20 -0600 Subject: [PATCH 241/929] [libc] Fix standalone cross compiling build for the GPU (#84042) Summary: This patch fixes some issues with building the GPU target manually without the runtimes bootstrapping build. This fixes the install directory and sets the default namespace to something more sensible if not set. Also I got rid of the check on `-mcpu=native`. it was a neat trick, but CMake in its INFINITE wisdom does not allow you to set link flags on the compiler flag check. So I just went with the old tool usage. --- libc/CMakeLists.txt | 14 ++++++++++++-- libc/cmake/modules/prepare_libc_gpu_build.cmake | 14 ++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/libc/CMakeLists.txt b/libc/CMakeLists.txt index 0b72b1c54816..b4a2523b7788 100644 --- a/libc/CMakeLists.txt +++ b/libc/CMakeLists.txt @@ -39,7 +39,11 @@ set(LIBC_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}) set(LIBC_ENABLE_USE_BY_CLANG OFF CACHE BOOL "Whether or not to place libc in a build directory findable by a just built clang") # Defining a global namespace to enclose all libc functions. -set(LIBC_NAMESPACE "__llvm_libc_${LLVM_VERSION_MAJOR}_${LLVM_VERSION_MINOR}_${LLVM_VERSION_PATCH}_${LLVM_VERSION_SUFFIX}" +set(default_namespace "__llvm_libc") +if(LLVM_VERSION_MAJOR) + set(default_namespace "__llvm_libc_${LLVM_VERSION_MAJOR}_${LLVM_VERSION_MINOR}_${LLVM_VERSION_PATCH}_${LLVM_VERSION_SUFFIX}") +endif() +set(LIBC_NAMESPACE ${default_namespace} CACHE STRING "The namespace to use to enclose internal implementations. Must start with '__llvm_libc'." ) @@ -232,7 +236,13 @@ else() set(LIBC_LIBRARY_DIR ${CMAKE_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX}) endif() if(LIBC_TARGET_OS_IS_GPU) - set(LIBC_INSTALL_INCLUDE_DIR ${CMAKE_INSTALL_INCLUDEDIR}/${LLVM_DEFAULT_TARGET_TRIPLE}) + if(LLVM_RUNTIMES_TARGET) + set(LIBC_INSTALL_INCLUDE_DIR ${CMAKE_INSTALL_INCLUDEDIR}/${LLVM_RUNTIMES_TARGET}) + elseif(LIBC_TARGET_TRIPLE) + set(LIBC_INSTALL_INCLUDE_DIR ${CMAKE_INSTALL_INCLUDEDIR}/${LIBC_TARGET_TRIPLE}) + else() + set(LIBC_INSTALL_INCLUDE_DIR ${CMAKE_INSTALL_INCLUDEDIR}/${LLVM_DEFAULT_TARGET_TRIPLE}) + endif() else() set(LIBC_INSTALL_INCLUDE_DIR ${CMAKE_INSTALL_INCLUDEDIR}) endif() diff --git a/libc/cmake/modules/prepare_libc_gpu_build.cmake b/libc/cmake/modules/prepare_libc_gpu_build.cmake index 7e9fc746ea91..2de4cb8d82b2 100644 --- a/libc/cmake/modules/prepare_libc_gpu_build.cmake +++ b/libc/cmake/modules/prepare_libc_gpu_build.cmake @@ -48,10 +48,20 @@ endif() set(LIBC_GPU_TEST_ARCHITECTURE "" CACHE STRING "Architecture for the GPU tests") if(LIBC_TARGET_ARCHITECTURE_IS_AMDGPU) - check_cxx_compiler_flag("-nogpulib -mcpu=native" PLATFORM_HAS_GPU) + # Identify any locally installed NVIDIA GPUs on the system using 'nvptx-arch'. + find_program(LIBC_AMDGPU_ARCH + NAMES amdgpu-arch NO_DEFAULT_PATH + PATHS ${LLVM_BINARY_DIR}/bin ${compiler_path}) + if(LIBC_AMDGPU_ARCH) + execute_process(COMMAND ${LIBC_AMDGPU_ARCH} + OUTPUT_VARIABLE arch_tool_output + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) + if(arch_tool_output MATCHES "^gfx[0-9]+") + set(PLATFORM_HAS_GPU TRUE) + endif() + endif() elseif(LIBC_TARGET_ARCHITECTURE_IS_NVPTX) # Identify any locally installed NVIDIA GPUs on the system using 'nvptx-arch'. - # Using 'check_cxx_compiler_flag' does not work currently due to the link job. find_program(LIBC_NVPTX_ARCH NAMES nvptx-arch NO_DEFAULT_PATH PATHS ${LLVM_BINARY_DIR}/bin ${compiler_path}) -- GitLab From 0f3628a93749433df51b763ff675152d82a25973 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Wed, 6 Mar 2024 09:36:01 +0530 Subject: [PATCH 242/929] AMDGPU: Correct cycle counts for f64 mfma on gfx940 (#83782) --- .../lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 54 ++++++++++--------- llvm/lib/Target/AMDGPU/SISchedule.td | 4 +- llvm/test/tools/llvm-mca/AMDGPU/gfx90a-mfma.s | 38 +++++++++++++ llvm/test/tools/llvm-mca/AMDGPU/gfx940-mfma.s | 19 ++++--- 4 files changed, 79 insertions(+), 36 deletions(-) create mode 100644 llvm/test/tools/llvm-mca/AMDGPU/gfx90a-mfma.s diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp index 00fa93cc1923..7bed0d8ef0d6 100644 --- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp +++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp @@ -2538,23 +2538,24 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { break; case 8: NeedWaitStates = - ST.hasGFX940Insts() - ? isXDL(ST, *MFMA) - ? GFX940_XDL8PassWriteVgprVALUMemExpReadWaitStates - : GFX940_SMFMA8PassWriteVgprVALUMemExpReadWaitStates - : SMFMA16x16WriteVgprVALUMemExpReadWaitStates; + isDGEMM(MFMA->getOpcode()) + ? IsMemOrExport ? DMFMA16x16WriteVgprMemExpReadWaitStates + : DMFMA16x16WriteVgprVALUReadWaitStates + : ST.hasGFX940Insts() + ? isXDL(ST, *MFMA) + ? GFX940_XDL8PassWriteVgprVALUMemExpReadWaitStates + : GFX940_SMFMA8PassWriteVgprVALUMemExpReadWaitStates + : SMFMA16x16WriteVgprVALUMemExpReadWaitStates; break; case 16: [[fallthrough]]; default: + assert(!isDGEMM(MFMA->getOpcode())); NeedWaitStates = - isDGEMM(MFMA->getOpcode()) - ? IsMemOrExport ? DMFMA16x16WriteVgprMemExpReadWaitStates - : DMFMA16x16WriteVgprVALUReadWaitStates - : ST.hasGFX940Insts() - ? isXDL(ST, *MFMA) - ? GFX940_XDL16PassWriteVgprVALUMemExpReadWaitStates - : GFX940_SMFMA16PassWriteVgprVALUMemExpReadWaitStates - : SMFMA32x32WriteVgprVALUMemExpReadWaitStates; + ST.hasGFX940Insts() + ? isXDL(ST, *MFMA) + ? GFX940_XDL16PassWriteVgprVALUMemExpReadWaitStates + : GFX940_SMFMA16PassWriteVgprVALUMemExpReadWaitStates + : SMFMA32x32WriteVgprVALUMemExpReadWaitStates; break; } @@ -2633,21 +2634,24 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { : GFX940_SMFMA4PassWriteVgprVALUWawWaitStates; break; case 8: - NeedWaitStates = ST.hasGFX940Insts() - ? isXDL(ST, *MFMA) - ? GFX940_XDL8PassWriteVgprVALUWawWaitStates - : GFX940_SMFMA8PassWriteVgprVALUWawWaitStates - : SMFMA16x16WriteVgprVALUWawWaitStates; + NeedWaitStates = + isDGEMM(MFMA->getOpcode()) ? DMFMA16x16WriteVgprVALUWriteWaitStates + : + + ST.hasGFX940Insts() + ? isXDL(ST, *MFMA) ? GFX940_XDL8PassWriteVgprVALUWawWaitStates + : GFX940_SMFMA8PassWriteVgprVALUWawWaitStates + : SMFMA16x16WriteVgprVALUWawWaitStates; break; case 16: [[fallthrough]]; default: - NeedWaitStates = isDGEMM(MFMA->getOpcode()) - ? DMFMA16x16WriteVgprVALUWriteWaitStates - : ST.hasGFX940Insts() - ? isXDL(ST, *MFMA) - ? GFX940_XDL16PassWriteVgprVALUWawWaitStates - : GFX940_SMFMA16PassWriteVgprVALUWawWaitStates - : SMFMA32x32WriteVgprVALUWawWaitStates; + assert(!isDGEMM(MFMA->getOpcode())); + NeedWaitStates = + ST.hasGFX940Insts() + ? isXDL(ST, *MFMA) + ? GFX940_XDL16PassWriteVgprVALUWawWaitStates + : GFX940_SMFMA16PassWriteVgprVALUWawWaitStates + : SMFMA32x32WriteVgprVALUWawWaitStates; break; } diff --git a/llvm/lib/Target/AMDGPU/SISchedule.td b/llvm/lib/Target/AMDGPU/SISchedule.td index b0e8e4112254..a60b1f28e9d3 100644 --- a/llvm/lib/Target/AMDGPU/SISchedule.td +++ b/llvm/lib/Target/AMDGPU/SISchedule.td @@ -165,8 +165,10 @@ multiclass SICommonWriteRes { def : HWVALUWriteRes; def : HWVALUWriteRes; + let ReleaseAtCycles = [4] in def : HWVALUWriteRes; - def : HWVALUWriteRes; + let ReleaseAtCycles = [8] in + def : HWVALUWriteRes; let ReleaseAtCycles = [2] in def : HWWriteRes; diff --git a/llvm/test/tools/llvm-mca/AMDGPU/gfx90a-mfma.s b/llvm/test/tools/llvm-mca/AMDGPU/gfx90a-mfma.s new file mode 100644 index 000000000000..6b4ddb3f000c --- /dev/null +++ b/llvm/test/tools/llvm-mca/AMDGPU/gfx90a-mfma.s @@ -0,0 +1,38 @@ +# RUN: llvm-mca -mtriple=amdgcn -mcpu=gfx90a --timeline --iterations=1 --timeline-max-cycles=0 < %s | FileCheck %s + +# CHECK: Instruction Info: +# CHECK-NEXT: [1]: #uOps +# CHECK-NEXT: [2]: Latency +# CHECK-NEXT: [3]: RThroughput +# CHECK-NEXT: [4]: MayLoad +# CHECK-NEXT: [5]: MayStore +# CHECK-NEXT: [6]: HasSideEffects (U) + +# CHECK: [1] [2] [3] [4] [5] [6] Instructions: +# CHECK-NEXT: 1 8 4.00 U v_mfma_f64_4x4x4f64 a[0:1], v[0:1], v[2:3], a[0:1] +# CHECK-NEXT: 1 8 4.00 U v_mfma_f64_4x4x4f64 v[0:1], v[0:1], v[2:3], v[0:1] +# CHECK-NEXT: 1 12 8.00 U v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] +# CHECK-NEXT: 1 12 8.00 U v_mfma_f64_16x16x4f64 v[0:7], v[0:1], v[2:3], v[0:7] + + +# CHECK: Resources: +# CHECK-NEXT: [0] - HWBranch +# CHECK-NEXT: [1] - HWExport +# CHECK-NEXT: [2] - HWLGKM +# CHECK-NEXT: [3] - HWSALU +# CHECK-NEXT: [4] - HWVALU +# CHECK-NEXT: [5] - HWVMEM +# CHECK-NEXT: [6] - HWXDL + +# CHECK: [0] [1] [2] [3] [4] [5] [6] Instructions: +# CHECK-NEXT: - - - - 4.00 - - v_mfma_f64_4x4x4f64 a[0:1], v[0:1], v[2:3], a[0:1] +# CHECK-NEXT: - - - - 4.00 - - v_mfma_f64_4x4x4f64 v[0:1], v[0:1], v[2:3], v[0:1] +# CHECK-NEXT: - - - - 8.00 - - v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] +# CHECK-NEXT: - - - - 8.00 - - v_mfma_f64_16x16x4f64 v[0:7], v[0:1], v[2:3], v[0:7] +v_mfma_f64_4x4x4f64 a[0:1], v[0:1], v[2:3], a[0:1] +v_mfma_f64_4x4x4f64 v[0:1], v[0:1], v[2:3], v[0:1] + + +v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] +v_mfma_f64_16x16x4f64 v[0:7], v[0:1], v[2:3], v[0:7] + diff --git a/llvm/test/tools/llvm-mca/AMDGPU/gfx940-mfma.s b/llvm/test/tools/llvm-mca/AMDGPU/gfx940-mfma.s index e7ddeee97ea9..0e1efbe90805 100644 --- a/llvm/test/tools/llvm-mca/AMDGPU/gfx940-mfma.s +++ b/llvm/test/tools/llvm-mca/AMDGPU/gfx940-mfma.s @@ -2,7 +2,7 @@ # CHECK: Iterations: 1 # CHECK: Instructions: 78 -# CHECK: Total Cycles: 699 +# CHECK: Total Cycles: 701 # CHECK: Total uOps: 78 v_mfma_f32_16x16x4_f32 a[0:3], v0, v1, a[2:5] @@ -128,11 +128,10 @@ v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1 # CHECK-NEXT:[6]: HasSideEffects (U) # CHECK: [1] [2] [3] [4] [5] [6] Instructions: - -# CHECK: 1 8 1.00 U v_mfma_f64_4x4x4_4b_f64 a[0:1], v[0:1], a[2:3], a[2:3] -# CHECK-NEXT: 1 8 1.00 U v_mfma_f64_4x4x4_4b_f64 v[0:1], v[0:1], v[2:3], v[2:3] -# CHECK-NEXT: 1 20 1.00 U v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] -# CHECK-NEXT: 1 20 1.00 U v_mfma_f64_16x16x4_f64 v[0:7], v[0:1], v[2:3], v[0:7] +# CHECK: 1 8 4.00 U v_mfma_f64_4x4x4_4b_f64 a[0:1], v[0:1], a[2:3], a[2:3] +# CHECK-NEXT: 1 8 4.00 U v_mfma_f64_4x4x4_4b_f64 v[0:1], v[0:1], v[2:3], v[2:3] +# CHECK-NEXT: 1 12 8.00 U v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] +# CHECK-NEXT: 1 12 8.00 U v_mfma_f64_16x16x4_f64 v[0:7], v[0:1], v[2:3], v[0:7] # CHECK: Resources: # CHECK: [0] - HWBranch @@ -148,10 +147,10 @@ v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[2:3], v[4:7], v1 cbsz:3 abid:1 # CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_16x16x4_f32 v[0:3], v0, v1, v[2:5] # CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x2_f32 a[0:15], v0, v1, a[18:33] # CHECK-NEXT: - - - - - - 16.00 v_mfma_f32_32x32x2_f32 v[0:15], v0, v1, v[18:33] -# CHECK-NEXT: - - - - 1.00 - - v_mfma_f64_4x4x4_4b_f64 a[0:1], v[0:1], a[2:3], a[2:3] -# CHECK-NEXT: - - - - 1.00 - - v_mfma_f64_4x4x4_4b_f64 v[0:1], v[0:1], v[2:3], v[2:3] -# CHECK-NEXT: - - - - 1.00 - - v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] -# CHECK-NEXT: - - - - 1.00 - - v_mfma_f64_16x16x4_f64 v[0:7], v[0:1], v[2:3], v[0:7] +# CHECK-NEXT: - - - - 4.00 - - v_mfma_f64_4x4x4_4b_f64 a[0:1], v[0:1], a[2:3], a[2:3] +# CHECK-NEXT: - - - - 4.00 - - v_mfma_f64_4x4x4_4b_f64 v[0:1], v[0:1], v[2:3], v[2:3] +# CHECK-NEXT: - - - - 8.00 - - v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] +# CHECK-NEXT: - - - - 8.00 - - v_mfma_f64_16x16x4_f64 v[0:7], v[0:1], v[2:3], v[0:7] # CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x16_f16 v[0:3], v[4:5], v[6:7], v[0:3] # CHECK-NEXT: - - - - - - 4.00 v_mfma_f32_16x16x16_f16 a[0:3], v[4:5], v[6:7], a[0:3] # CHECK-NEXT: - - - - - - 8.00 v_mfma_f32_32x32x8_f16 v[0:15], v[4:5], v[6:7], v[0:15] -- GitLab From 5549b0173661155d4ca9acf4958fe6dce34c2cd5 Mon Sep 17 00:00:00 2001 From: Nathan Ridge Date: Tue, 5 Mar 2024 23:11:29 -0500 Subject: [PATCH 243/929] [clangd] Make all calls to format::getStyle() go through getFormatStyleForFile() (#82948) --- clang-tools-extra/clangd/ClangdServer.cpp | 9 ++------- clang-tools-extra/clangd/IncludeCleaner.cpp | 15 +++++---------- clang-tools-extra/clangd/IncludeCleaner.h | 1 + clang-tools-extra/clangd/ParsedAST.cpp | 7 ++++--- .../clangd/unittests/IncludeCleanerTests.cpp | 14 +++++++------- 5 files changed, 19 insertions(+), 27 deletions(-) diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp index 3f9fd0128194..2907e3ba3c30 100644 --- a/clang-tools-extra/clangd/ClangdServer.cpp +++ b/clang-tools-extra/clangd/ClangdServer.cpp @@ -551,15 +551,10 @@ void ClangdServer::formatOnType(PathRef File, Position Pos, auto Action = [File = File.str(), Code = std::move(*Code), TriggerText = TriggerText.str(), CursorPos = *CursorPos, CB = std::move(CB), this]() mutable { - auto Style = format::getStyle(format::DefaultFormatStyle, File, - format::DefaultFallbackStyle, Code, - TFS.view(/*CWD=*/std::nullopt).get()); - if (!Style) - return CB(Style.takeError()); - + auto Style = getFormatStyleForFile(File, Code, TFS); std::vector Result; for (const tooling::Replacement &R : - formatIncremental(Code, CursorPos, TriggerText, *Style)) + formatIncremental(Code, CursorPos, TriggerText, Style)) Result.push_back(replacementToEdit(Code, R)); return CB(Result); }; diff --git a/clang-tools-extra/clangd/IncludeCleaner.cpp b/clang-tools-extra/clangd/IncludeCleaner.cpp index f86a121340f7..7375b7b08609 100644 --- a/clang-tools-extra/clangd/IncludeCleaner.cpp +++ b/clang-tools-extra/clangd/IncludeCleaner.cpp @@ -111,21 +111,15 @@ bool mayConsiderUnused(const Inclusion &Inc, ParsedAST &AST, std::vector generateMissingIncludeDiagnostics( ParsedAST &AST, llvm::ArrayRef MissingIncludes, - llvm::StringRef Code, HeaderFilter IgnoreHeaders) { + llvm::StringRef Code, HeaderFilter IgnoreHeaders, const ThreadsafeFS &TFS) { std::vector Result; const SourceManager &SM = AST.getSourceManager(); const FileEntry *MainFile = SM.getFileEntryForID(SM.getMainFileID()); - auto FileStyle = format::getStyle( - format::DefaultFormatStyle, AST.tuPath(), format::DefaultFallbackStyle, - Code, &SM.getFileManager().getVirtualFileSystem()); - if (!FileStyle) { - elog("Couldn't infer style", FileStyle.takeError()); - FileStyle = format::getLLVMStyle(); - } + auto FileStyle = getFormatStyleForFile(AST.tuPath(), Code, TFS); tooling::HeaderIncludes HeaderIncludes(AST.tuPath(), Code, - FileStyle->IncludeStyle); + FileStyle.IncludeStyle); for (const auto &SymbolWithMissingInclude : MissingIncludes) { llvm::StringRef ResolvedPath = SymbolWithMissingInclude.Providers.front().resolvedPath(); @@ -459,6 +453,7 @@ bool isPreferredProvider(const Inclusion &Inc, std::vector issueIncludeCleanerDiagnostics(ParsedAST &AST, llvm::StringRef Code, const IncludeCleanerFindings &Findings, + const ThreadsafeFS &TFS, HeaderFilter IgnoreHeaders) { trace::Span Tracer("IncludeCleaner::issueIncludeCleanerDiagnostics"); std::vector UnusedIncludes = generateUnusedIncludeDiagnostics( @@ -466,7 +461,7 @@ issueIncludeCleanerDiagnostics(ParsedAST &AST, llvm::StringRef Code, std::optional RemoveAllUnused = removeAllUnusedIncludes(UnusedIncludes); std::vector MissingIncludeDiags = generateMissingIncludeDiagnostics( - AST, Findings.MissingIncludes, Code, IgnoreHeaders); + AST, Findings.MissingIncludes, Code, IgnoreHeaders, TFS); std::optional AddAllMissing = addAllMissingIncludes(MissingIncludeDiags); std::optional FixAll; diff --git a/clang-tools-extra/clangd/IncludeCleaner.h b/clang-tools-extra/clangd/IncludeCleaner.h index b3ba3a716083..387763de3407 100644 --- a/clang-tools-extra/clangd/IncludeCleaner.h +++ b/clang-tools-extra/clangd/IncludeCleaner.h @@ -59,6 +59,7 @@ using HeaderFilter = llvm::ArrayRef>; std::vector issueIncludeCleanerDiagnostics(ParsedAST &AST, llvm::StringRef Code, const IncludeCleanerFindings &Findings, + const ThreadsafeFS &TFS, HeaderFilter IgnoreHeader = {}); /// Affects whether standard library includes should be considered for diff --git a/clang-tools-extra/clangd/ParsedAST.cpp b/clang-tools-extra/clangd/ParsedAST.cpp index bbb0e2c77b3f..862f06196a71 100644 --- a/clang-tools-extra/clangd/ParsedAST.cpp +++ b/clang-tools-extra/clangd/ParsedAST.cpp @@ -360,7 +360,8 @@ void applyWarningOptions(llvm::ArrayRef ExtraArgs, } } -std::vector getIncludeCleanerDiags(ParsedAST &AST, llvm::StringRef Code) { +std::vector getIncludeCleanerDiags(ParsedAST &AST, llvm::StringRef Code, + const ThreadsafeFS &TFS) { auto &Cfg = Config::current(); if (Cfg.Diagnostics.SuppressAll) return {}; @@ -377,7 +378,7 @@ std::vector getIncludeCleanerDiags(ParsedAST &AST, llvm::StringRef Code) { Findings.MissingIncludes.clear(); if (SuppressUnused) Findings.UnusedIncludes.clear(); - return issueIncludeCleanerDiagnostics(AST, Code, Findings, + return issueIncludeCleanerDiagnostics(AST, Code, Findings, TFS, Cfg.Diagnostics.Includes.IgnoreHeader); } @@ -741,7 +742,7 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs, std::move(Clang), std::move(Action), std::move(Tokens), std::move(Macros), std::move(Marks), std::move(ParsedDecls), std::move(Diags), std::move(Includes), std::move(PI)); - llvm::move(getIncludeCleanerDiags(Result, Inputs.Contents), + llvm::move(getIncludeCleanerDiags(Result, Inputs.Contents, *Inputs.TFS), std::back_inserter(Result.Diags)); return std::move(Result); } diff --git a/clang-tools-extra/clangd/unittests/IncludeCleanerTests.cpp b/clang-tools-extra/clangd/unittests/IncludeCleanerTests.cpp index 7ed4a9103e1f..142310837bd9 100644 --- a/clang-tools-extra/clangd/unittests/IncludeCleanerTests.cpp +++ b/clang-tools-extra/clangd/unittests/IncludeCleanerTests.cpp @@ -252,7 +252,7 @@ $insert_f[[]]$insert_vector[[]] auto Findings = computeIncludeCleanerFindings(AST); Findings.UnusedIncludes.clear(); std::vector Diags = issueIncludeCleanerDiagnostics( - AST, TU.Code, Findings, + AST, TU.Code, Findings, MockFS(), {[](llvm::StringRef Header) { return Header.ends_with("buzz.h"); }}); EXPECT_THAT( Diags, @@ -505,8 +505,8 @@ TEST(IncludeCleaner, BatchFix) { )cpp"; auto AST = TU.build(); EXPECT_THAT( - issueIncludeCleanerDiagnostics(AST, TU.Code, - computeIncludeCleanerFindings(AST)), + issueIncludeCleanerDiagnostics( + AST, TU.Code, computeIncludeCleanerFindings(AST), MockFS()), UnorderedElementsAre(withFix({FixMessage("#include \"foo.h\""), FixMessage("fix all includes")}), withFix({FixMessage("remove #include directive"), @@ -520,8 +520,8 @@ TEST(IncludeCleaner, BatchFix) { )cpp"; AST = TU.build(); EXPECT_THAT( - issueIncludeCleanerDiagnostics(AST, TU.Code, - computeIncludeCleanerFindings(AST)), + issueIncludeCleanerDiagnostics( + AST, TU.Code, computeIncludeCleanerFindings(AST), MockFS()), UnorderedElementsAre(withFix({FixMessage("#include \"foo.h\""), FixMessage("fix all includes")}), withFix({FixMessage("remove #include directive"), @@ -539,8 +539,8 @@ TEST(IncludeCleaner, BatchFix) { )cpp"; AST = TU.build(); EXPECT_THAT( - issueIncludeCleanerDiagnostics(AST, TU.Code, - computeIncludeCleanerFindings(AST)), + issueIncludeCleanerDiagnostics( + AST, TU.Code, computeIncludeCleanerFindings(AST), MockFS()), UnorderedElementsAre(withFix({FixMessage("#include \"foo.h\""), FixMessage("add all missing includes"), FixMessage("fix all includes")}), -- GitLab From d1aec79a2ce077e49da7699c4ca2dee239d0a249 Mon Sep 17 00:00:00 2001 From: Nathan Ridge Date: Tue, 5 Mar 2024 23:12:43 -0500 Subject: [PATCH 244/929] [clangd] Remove calls to getFileLoc() in declToSym() (#83532) toHalfOpenFileRange() already handles translating macro locations to file locations, and it can provide a better result by knowing about both ends of the range. Fixes https://github.com/clangd/clangd/issues/1941 --- clang-tools-extra/clangd/FindSymbols.cpp | 4 ++-- clang-tools-extra/clangd/unittests/FindSymbolsTests.cpp | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/clang-tools-extra/clangd/FindSymbols.cpp b/clang-tools-extra/clangd/FindSymbols.cpp index 5b3e46a7b4dc..5244a4e89376 100644 --- a/clang-tools-extra/clangd/FindSymbols.cpp +++ b/clang-tools-extra/clangd/FindSymbols.cpp @@ -223,8 +223,8 @@ std::string getSymbolDetail(ASTContext &Ctx, const NamedDecl &ND) { std::optional declToSym(ASTContext &Ctx, const NamedDecl &ND) { auto &SM = Ctx.getSourceManager(); - SourceLocation BeginLoc = SM.getFileLoc(ND.getBeginLoc()); - SourceLocation EndLoc = SM.getFileLoc(ND.getEndLoc()); + SourceLocation BeginLoc = ND.getBeginLoc(); + SourceLocation EndLoc = ND.getEndLoc(); const auto SymbolRange = toHalfOpenFileRange(SM, Ctx.getLangOpts(), {BeginLoc, EndLoc}); if (!SymbolRange) diff --git a/clang-tools-extra/clangd/unittests/FindSymbolsTests.cpp b/clang-tools-extra/clangd/unittests/FindSymbolsTests.cpp index b1b8b4ccd184..4276a44275f5 100644 --- a/clang-tools-extra/clangd/unittests/FindSymbolsTests.cpp +++ b/clang-tools-extra/clangd/unittests/FindSymbolsTests.cpp @@ -750,6 +750,9 @@ TEST(DocumentSymbols, RangeFromMacro) { $fullDef[[FF3() { int var = 42; }]] + + #define FF4(name) int name = 0 + $FooRange[[FF4($FooSelectionRange[[foo]])]]; )"); TU.Code = Main.code().str(); EXPECT_THAT( @@ -766,7 +769,11 @@ TEST(DocumentSymbols, RangeFromMacro) { AllOf(withName("FF3"), withDetail("()"), symRange(Main.range("fullDef")), children(AllOf(withName("waldo"), withDetail("void ()"), - symRange(Main.range("fullDef"))))))); + symRange(Main.range("fullDef"))))), + AllOf( + withName("FF4"), withDetail("(foo)"), + children(AllOf(withName("foo"), symRange(Main.range("FooRange")), + symNameRange(Main.range("FooSelectionRange"))))))); } TEST(DocumentSymbols, FuncTemplates) { -- GitLab From b930b14d5d8084425cec75f2037746ab99768c0f Mon Sep 17 00:00:00 2001 From: Kai Sasaki Date: Wed, 6 Mar 2024 13:33:06 +0900 Subject: [PATCH 245/929] [mlir][complex] Support fast math flag in converting complex.atan2 op (#82101) When converting complex.atan2 op to standard, we need to keep the fast math flag given to the op. See: https://discourse.llvm.org/t/rfc-fastmath-flags-support-in-complex-dialect/71981 --- .../ComplexToStandard/ComplexToStandard.cpp | 21 +- .../convert-to-standard.mlir | 571 ++++++++++++++++++ 2 files changed, 582 insertions(+), 10 deletions(-) diff --git a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp index 20525b962ae9..12a5e2026858 100644 --- a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp +++ b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp @@ -94,34 +94,35 @@ struct Atan2OpConversion : public OpConversionPattern { auto type = cast(op.getType()); Type elementType = type.getElementType(); + arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr(); Value lhs = adaptor.getLhs(); Value rhs = adaptor.getRhs(); - Value rhsSquared = b.create(type, rhs, rhs); - Value lhsSquared = b.create(type, lhs, lhs); + Value rhsSquared = b.create(type, rhs, rhs, fmf); + Value lhsSquared = b.create(type, lhs, lhs, fmf); Value rhsSquaredPlusLhsSquared = - b.create(type, rhsSquared, lhsSquared); + b.create(type, rhsSquared, lhsSquared, fmf); Value sqrtOfRhsSquaredPlusLhsSquared = - b.create(type, rhsSquaredPlusLhsSquared); + b.create(type, rhsSquaredPlusLhsSquared, fmf); Value zero = b.create(elementType, b.getZeroAttr(elementType)); Value one = b.create(elementType, b.getFloatAttr(elementType, 1)); Value i = b.create(type, zero, one); - Value iTimesLhs = b.create(i, lhs); - Value rhsPlusILhs = b.create(rhs, iTimesLhs); + Value iTimesLhs = b.create(i, lhs, fmf); + Value rhsPlusILhs = b.create(rhs, iTimesLhs, fmf); - Value divResult = - b.create(rhsPlusILhs, sqrtOfRhsSquaredPlusLhsSquared); - Value logResult = b.create(divResult); + Value divResult = b.create( + rhsPlusILhs, sqrtOfRhsSquaredPlusLhsSquared, fmf); + Value logResult = b.create(divResult, fmf); Value negativeOne = b.create( elementType, b.getFloatAttr(elementType, -1)); Value negativeI = b.create(type, zero, negativeOne); - rewriter.replaceOpWithNewOp(op, negativeI, logResult); + rewriter.replaceOpWithNewOp(op, negativeI, logResult, fmf); return success(); } }; diff --git a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir index 39af7dd02a62..c80ba48cf8c1 100644 --- a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir +++ b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir @@ -1049,6 +1049,577 @@ func.func @complex_mul_with_fmf(%lhs: complex, %rhs: complex) -> compl // ----- +// CHECK-LABEL: func @complex_atan2_with_fmf +func.func @complex_atan2_with_fmf(%lhs: complex, + %rhs: complex) -> complex { + %atan2 = complex.atan2 %lhs, %rhs fastmath : complex + return %atan2 : complex +} + +// CHECK: %[[VAR0:.*]] = complex.re %arg1 : complex +// CHECK: %[[VAR1:.*]] = math.absf %[[VAR0]] fastmath : f32 +// CHECK: %[[VAR2:.*]] = complex.im %arg1 : complex +// CHECK: %[[VAR3:.*]] = math.absf %[[VAR2]] fastmath : f32 +// CHECK: %[[VAR4:.*]] = complex.re %arg1 : complex +// CHECK: %[[VAR5:.*]] = math.absf %[[VAR4]] fastmath : f32 +// CHECK: %[[VAR6:.*]] = complex.im %arg1 : complex +// CHECK: %[[VAR7:.*]] = math.absf %[[VAR6]] fastmath : f32 +// CHECK: %[[VAR8:.*]] = arith.mulf %[[VAR0]], %[[VAR4]] fastmath : f32 +// CHECK: %[[VAR9:.*]] = math.absf %[[VAR8]] fastmath : f32 +// CHECK: %[[VAR10:.*]] = arith.mulf %[[VAR2]], %[[VAR6]] fastmath : f32 +// CHECK: %[[VAR11:.*]] = math.absf %[[VAR10]] fastmath : f32 +// CHECK: %[[VAR12:.*]] = arith.subf %[[VAR8]], %[[VAR10]] fastmath : f32 +// CHECK: %[[VAR13:.*]] = arith.mulf %[[VAR2]], %[[VAR4]] fastmath : f32 +// CHECK: %[[VAR14:.*]] = math.absf %[[VAR13]] fastmath : f32 +// CHECK: %[[VAR15:.*]] = arith.mulf %[[VAR0]], %[[VAR6]] fastmath : f32 +// CHECK: %[[VAR16:.*]] = math.absf %[[VAR15]] fastmath : f32 +// CHECK: %[[VAR17:.*]] = arith.addf %[[VAR13]], %[[VAR15]] fastmath : f32 +// CHECK: %[[VAR18:.*]] = arith.cmpf uno, %[[VAR12]], %[[VAR12]] : f32 +// CHECK: %[[VAR19:.*]] = arith.cmpf uno, %[[VAR17]], %[[VAR17]] : f32 +// CHECK: %[[VAR20:.*]] = arith.andi %[[VAR18]], %[[VAR19]] : i1 +// CHECK: %[[CST:.*]] = arith.constant 0x7F800000 : f32 +// CHECK: %[[VAR21:.*]] = arith.cmpf oeq, %[[VAR1]], %[[CST]] : f32 +// CHECK: %[[VAR22:.*]] = arith.cmpf oeq, %[[VAR3]], %[[CST]] : f32 +// CHECK: %[[VAR23:.*]] = arith.ori %[[VAR21]], %[[VAR22]] : i1 +// CHECK: %[[VAR24:.*]] = arith.cmpf uno, %[[VAR4]], %[[VAR4]] : f32 +// CHECK: %[[VAR25:.*]] = arith.cmpf uno, %[[VAR6]], %[[VAR6]] : f32 +// CHECK: %[[CST_0:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK: %[[CST_1:.*]] = arith.constant 1.000000e+00 : f32 +// CHECK: %[[VAR26:.*]] = arith.select %[[VAR21]], %[[CST_1]], %[[CST_0]] : f32 +// CHECK: %[[VAR27:.*]] = math.copysign %[[VAR26]], %[[VAR0]] : f32 +// CHECK: %[[VAR28:.*]] = arith.select %[[VAR23]], %[[VAR27]], %[[VAR0]] : f32 +// CHECK: %[[VAR29:.*]] = arith.select %[[VAR22]], %[[CST_1]], %[[CST_0]] : f32 +// CHECK: %[[VAR30:.*]] = math.copysign %[[VAR29]], %[[VAR2]] : f32 +// CHECK: %[[VAR31:.*]] = arith.select %[[VAR23]], %[[VAR30]], %[[VAR2]] : f32 +// CHECK: %[[VAR32:.*]] = arith.andi %[[VAR23]], %[[VAR24]] : i1 +// CHECK: %[[VAR33:.*]] = math.copysign %[[CST_0]], %[[VAR4]] : f32 +// CHECK: %[[VAR34:.*]] = arith.select %[[VAR32]], %[[VAR33]], %[[VAR4]] : f32 +// CHECK: %[[VAR35:.*]] = arith.andi %[[VAR23]], %[[VAR25]] : i1 +// CHECK: %[[VAR36:.*]] = math.copysign %[[CST_0]], %[[VAR6]] : f32 +// CHECK: %[[VAR37:.*]] = arith.select %[[VAR35]], %[[VAR36]], %[[VAR6]] : f32 +// CHECK: %[[VAR38:.*]] = arith.cmpf oeq, %[[VAR5]], %cst : f32 +// CHECK: %[[VAR39:.*]] = arith.cmpf oeq, %[[VAR7]], %cst : f32 +// CHECK: %[[VAR40:.*]] = arith.ori %[[VAR38]], %[[VAR39]] : i1 +// CHECK: %[[VAR41:.*]] = arith.cmpf uno, %[[VAR28]], %[[VAR28]] : f32 +// CHECK: %[[VAR42:.*]] = arith.cmpf uno, %[[VAR31]], %[[VAR31]] : f32 +// CHECK: %[[VAR43:.*]] = arith.select %[[VAR38]], %[[CST_1]], %[[CST_0]] : f32 +// CHECK: %[[VAR44:.*]] = math.copysign %[[VAR43]], %[[VAR34]] : f32 +// CHECK: %[[VAR45:.*]] = arith.select %[[VAR40]], %[[VAR44]], %[[VAR34]] : f32 +// CHECK: %[[VAR46:.*]] = arith.select %[[VAR39]], %[[CST_1]], %[[CST_0]] : f32 +// CHECK: %[[VAR47:.*]] = math.copysign %[[VAR46]], %[[VAR37]] : f32 +// CHECK: %[[VAR48:.*]] = arith.select %[[VAR40]], %[[VAR47]], %[[VAR37]] : f32 +// CHECK: %[[VAR49:.*]] = arith.andi %[[VAR40]], %[[VAR41]] : i1 +// CHECK: %[[VAR50:.*]] = math.copysign %[[CST_0]], %[[VAR28]] : f32 +// CHECK: %[[VAR51:.*]] = arith.select %[[VAR49]], %[[VAR50]], %[[VAR28]] : f32 +// CHECK: %[[VAR52:.*]] = arith.andi %[[VAR40]], %[[VAR42]] : i1 +// CHECK: %[[VAR53:.*]] = math.copysign %[[CST_0]], %[[VAR31]] : f32 +// CHECK: %[[VAR54:.*]] = arith.select %[[VAR52]], %[[VAR53]], %[[VAR31]] : f32 +// CHECK: %[[VAR55:.*]] = arith.ori %[[VAR23]], %[[VAR40]] : i1 +// CHECK: %[[VAR56:.*]] = arith.cmpf oeq, %[[VAR9]], %[[CST]] : f32 +// CHECK: %[[VAR57:.*]] = arith.cmpf oeq, %[[VAR11]], %[[CST]] : f32 +// CHECK: %[[VAR58:.*]] = arith.ori %[[VAR56]], %[[VAR57]] : i1 +// CHECK: %[[VAR59:.*]] = arith.cmpf oeq, %[[VAR16]], %[[CST]] : f32 +// CHECK: %[[VAR60:.*]] = arith.ori %[[VAR58]], %[[VAR59]] : i1 +// CHECK: %[[VAR61:.*]] = arith.cmpf oeq, %[[VAR14]], %[[CST]] : f32 +// CHECK: %[[VAR62:.*]] = arith.ori %[[VAR60]], %[[VAR61]] : i1 +// CHECK: %[[TRUE:.*]] = arith.constant true +// CHECK: %[[VAR63:.*]] = arith.xori %[[VAR55]], %[[TRUE]] : i1 +// CHECK: %[[VAR64:.*]] = arith.andi %[[VAR62]], %[[VAR63]] : i1 +// CHECK: %[[VAR65:.*]] = arith.andi %[[VAR64]], %[[VAR41]] : i1 +// CHECK: %[[VAR66:.*]] = math.copysign %[[CST_0]], %[[VAR51]] : f32 +// CHECK: %[[VAR67:.*]] = arith.select %[[VAR65]], %[[VAR66]], %[[VAR51]] : f32 +// CHECK: %[[VAR68:.*]] = arith.andi %[[VAR64]], %[[VAR42]] : i1 +// CHECK: %[[VAR69:.*]] = math.copysign %[[CST_0]], %[[VAR54]] : f32 +// CHECK: %[[VAR70:.*]] = arith.select %[[VAR68]], %[[VAR69]], %[[VAR54]] : f32 +// CHECK: %[[VAR71:.*]] = arith.andi %[[VAR64]], %[[VAR24]] : i1 +// CHECK: %[[VAR72:.*]] = math.copysign %[[CST_0]], %[[VAR45]] : f32 +// CHECK: %[[VAR73:.*]] = arith.select %[[VAR71]], %[[VAR72]], %[[VAR45]] : f32 +// CHECK: %[[VAR74:.*]] = arith.andi %[[VAR64]], %[[VAR25]] : i1 +// CHECK: %[[VAR75:.*]] = math.copysign %[[CST_0]], %[[VAR48]] : f32 +// CHECK: %[[VAR76:.*]] = arith.select %[[VAR74]], %[[VAR75]], %[[VAR48]] : f32 +// CHECK: %[[VAR77:.*]] = arith.ori %[[VAR55]], %[[VAR64]] : i1 +// CHECK: %[[VAR78:.*]] = arith.andi %[[VAR20]], %[[VAR77]] : i1 +// CHECK: %[[VAR79:.*]] = arith.mulf %[[VAR67]], %[[VAR73]] fastmath : f32 +// CHECK: %[[VAR80:.*]] = arith.mulf %[[VAR70]], %[[VAR76]] fastmath : f32 +// CHECK: %[[VAR81:.*]] = arith.subf %[[VAR79]], %[[VAR80]] fastmath : f32 +// CHECK: %[[VAR82:.*]] = arith.mulf %[[CST]], %[[VAR81]] fastmath : f32 +// CHECK: %[[VAR83:.*]] = arith.select %[[VAR78]], %[[VAR82]], %[[VAR12]] : f32 +// CHECK: %[[VAR84:.*]] = arith.mulf %[[VAR70]], %[[VAR73]] fastmath : f32 +// CHECK: %[[VAR85:.*]] = arith.mulf %[[VAR67]], %[[VAR76]] fastmath : f32 +// CHECK: %[[VAR86:.*]] = arith.addf %[[VAR84]], %[[VAR85]] fastmath : f32 +// CHECK: %[[VAR87:.*]] = arith.mulf %[[CST]], %[[VAR86]] fastmath : f32 +// CHECK: %[[VAR88:.*]] = arith.select %[[VAR78]], %[[VAR87]], %[[VAR17]] : f32 +// CHECK: %[[VAR89:.*]] = complex.create %[[VAR83]], %[[VAR88]] : complex +// CHECK: %[[VAR90:.*]] = complex.re %arg0 : complex +// CHECK: %[[VAR91:.*]] = math.absf %[[VAR90]] fastmath : f32 +// CHECK: %[[VAR92:.*]] = complex.im %arg0 : complex +// CHECK: %[[VAR93:.*]] = math.absf %[[VAR92]] fastmath : f32 +// CHECK: %[[VAR94:.*]] = complex.re %arg0 : complex +// CHECK: %[[VAR95:.*]] = math.absf %[[VAR94]] fastmath : f32 +// CHECK: %[[VAR96:.*]] = complex.im %arg0 : complex +// CHECK: %[[VAR97:.*]] = math.absf %[[VAR96]] fastmath : f32 +// CHECK: %[[VAR98:.*]] = arith.mulf %[[VAR90]], %[[VAR94]] fastmath : f32 +// CHECK: %[[VAR99:.*]] = math.absf %[[VAR98]] fastmath : f32 +// CHECK: %[[VAR100:.*]] = arith.mulf %[[VAR92]], %[[VAR96]] fastmath : f32 +// CHECK: %[[VAR101:.*]] = math.absf %[[VAR100]] fastmath : f32 +// CHECK: %[[VAR102:.*]] = arith.subf %[[VAR98]], %[[VAR100]] fastmath : f32 +// CHECK: %[[VAR103:.*]] = arith.mulf %[[VAR92]], %[[VAR94]] fastmath : f32 +// CHECK: %[[VAR104:.*]] = math.absf %[[VAR103]] fastmath : f32 +// CHECK: %[[VAR105:.*]] = arith.mulf %[[VAR90]], %[[VAR96]] fastmath : f32 +// CHECK: %[[VAR106:.*]] = math.absf %[[VAR105]] fastmath : f32 +// CHECK: %[[VAR107:.*]] = arith.addf %[[VAR103]], %[[VAR105]] fastmath : f32 +// CHECK: %[[VAR108:.*]] = arith.cmpf uno, %[[VAR102]], %[[VAR102]] : f32 +// CHECK: %[[VAR109:.*]] = arith.cmpf uno, %[[VAR107]], %[[VAR107]] : f32 +// CHECK: %[[VAR110:.*]] = arith.andi %[[VAR108]], %[[VAR109]] : i1 +// CHECK: %[[CST_2:.*]] = arith.constant 0x7F800000 : f32 +// CHECK: %[[VAR111:.*]] = arith.cmpf oeq, %[[VAR91]], %[[CST_2]] : f32 +// CHECK: %[[VAR112:.*]] = arith.cmpf oeq, %[[VAR93]], %[[CST_2]] : f32 +// CHECK: %[[VAR113:.*]] = arith.ori %[[VAR111]], %[[VAR112]] : i1 +// CHECK: %[[VAR114:.*]] = arith.cmpf uno, %[[VAR94]], %[[VAR94]] : f32 +// CHECK: %[[VAR115:.*]] = arith.cmpf uno, %[[VAR96]], %[[VAR96]] : f32 +// CHECK: %[[CST_3:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK: %[[CST_4:.*]] = arith.constant 1.000000e+00 : f32 +// CHECK: %[[VAR116:.*]] = arith.select %[[VAR111]], %[[CST_4]], %[[CST_3]] : f32 +// CHECK: %[[VAR117:.*]] = math.copysign %[[VAR116]], %[[VAR90]] : f32 +// CHECK: %[[VAR118:.*]] = arith.select %[[VAR113]], %[[VAR117]], %[[VAR90]] : f32 +// CHECK: %[[VAR119:.*]] = arith.select %[[VAR112]], %[[CST_4]], %[[CST_3]] : f32 +// CHECK: %[[VAR120:.*]] = math.copysign %[[VAR119]], %[[VAR92]] : f32 +// CHECK: %[[VAR121:.*]] = arith.select %[[VAR113]], %[[VAR120]], %[[VAR92]] : f32 +// CHECK: %[[VAR122:.*]] = arith.andi %[[VAR113]], %[[VAR114]] : i1 +// CHECK: %[[VAR123:.*]] = math.copysign %[[CST_3]], %[[VAR94]] : f32 +// CHECK: %[[VAR124:.*]] = arith.select %[[VAR122]], %[[VAR123]], %[[VAR94]] : f32 +// CHECK: %[[VAR125:.*]] = arith.andi %[[VAR113]], %[[VAR115]] : i1 +// CHECK: %[[VAR126:.*]] = math.copysign %[[CST_3]], %[[VAR96]] : f32 +// CHECK: %[[VAR127:.*]] = arith.select %[[VAR125]], %[[VAR126]], %[[VAR96]] : f32 +// CHECK: %[[VAR128:.*]] = arith.cmpf oeq, %[[VAR95]], %[[CST_2]] : f32 +// CHECK: %[[VAR129:.*]] = arith.cmpf oeq, %[[VAR97]], %[[CST_2]] : f32 +// CHECK: %[[VAR130:.*]] = arith.ori %[[VAR128]], %[[VAR129]] : i1 +// CHECK: %[[VAR131:.*]] = arith.cmpf uno, %[[VAR118]], %[[VAR118]] : f32 +// CHECK: %[[VAR132:.*]] = arith.cmpf uno, %[[VAR121]], %[[VAR121]] : f32 +// CHECK: %[[VAR133:.*]] = arith.select %[[VAR128]], %[[CST_4]], %[[CST_3]] : f32 +// CHECK: %[[VAR134:.*]] = math.copysign %[[VAR133]], %[[VAR124]] : f32 +// CHECK: %[[VAR135:.*]] = arith.select %[[VAR130]], %[[VAR134]], %[[VAR124]] : f32 +// CHECK: %[[VAR136:.*]] = arith.select %[[VAR129]], %[[CST_4]], %[[CST_3]] : f32 +// CHECK: %[[VAR137:.*]] = math.copysign %[[VAR136]], %[[VAR127]] : f32 +// CHECK: %[[VAR138:.*]] = arith.select %[[VAR130]], %[[VAR137]], %[[VAR127]] : f32 +// CHECK: %[[VAR139:.*]] = arith.andi %[[VAR130]], %[[VAR131]] : i1 +// CHECK: %[[VAR140:.*]] = math.copysign %[[CST_3]], %[[VAR118]] : f32 +// CHECK: %[[VAR141:.*]] = arith.select %[[VAR139]], %[[VAR140]], %[[VAR118]] : f32 +// CHECK: %[[VAR142:.*]] = arith.andi %[[VAR130]], %[[VAR132]] : i1 +// CHECK: %[[VAR143:.*]] = math.copysign %[[CST_3]], %[[VAR121]] : f32 +// CHECK: %[[VAR144:.*]] = arith.select %[[VAR142]], %[[VAR143]], %[[VAR121]] : f32 +// CHECK: %[[VAR145:.*]] = arith.ori %[[VAR113]], %[[VAR130]] : i1 +// CHECK: %[[VAR146:.*]] = arith.cmpf oeq, %[[VAR99]], %[[CST_2]] : f32 +// CHECK: %[[VAR147:.*]] = arith.cmpf oeq, %[[VAR101]], %[[CST_2]] : f32 +// CHECK: %[[VAR148:.*]] = arith.ori %[[VAR146]], %[[VAR147]] : i1 +// CHECK: %[[VAR149:.*]] = arith.cmpf oeq, %[[VAR106]], %[[CST_2]] : f32 +// CHECK: %[[VAR150:.*]] = arith.ori %[[VAR148]], %[[VAR149]] : i1 +// CHECK: %[[VAR151:.*]] = arith.cmpf oeq, %[[VAR104]], %[[CST_2]] : f32 +// CHECK: %[[VAR152:.*]] = arith.ori %[[VAR150]], %[[VAR151]] : i1 +// CHECK: %[[TRUE_5:.*]] = arith.constant true +// CHECK: %[[VAR153:.*]] = arith.xori %[[VAR145]], %[[TRUE_5]] : i1 +// CHECK: %[[VAR154:.*]] = arith.andi %[[VAR152]], %[[VAR153]] : i1 +// CHECK: %[[VAR155:.*]] = arith.andi %[[VAR154]], %[[VAR131]] : i1 +// CHECK: %[[VAR156:.*]] = math.copysign %[[CST_3]], %[[VAR141]] : f32 +// CHECK: %[[VAR157:.*]] = arith.select %[[VAR155]], %[[VAR156]], %[[VAR141]] : f32 +// CHECK: %[[VAR158:.*]] = arith.andi %[[VAR154]], %[[VAR132]] : i1 +// CHECK: %[[VAR159:.*]] = math.copysign %[[CST_3]], %[[VAR144]] : f32 +// CHECK: %[[VAR160:.*]] = arith.select %[[VAR158]], %[[VAR159]], %[[VAR144]] : f32 +// CHECK: %[[VAR161:.*]] = arith.andi %[[VAR154]], %[[VAR114]] : i1 +// CHECK: %[[VAR162:.*]] = math.copysign %[[CST_3]], %[[VAR135]] : f32 +// CHECK: %[[VAR163:.*]] = arith.select %[[VAR161]], %[[VAR162]], %[[VAR135]] : f32 +// CHECK: %[[VAR164:.*]] = arith.andi %[[VAR154]], %[[VAR115]] : i1 +// CHECK: %[[VAR165:.*]] = math.copysign %[[CST_3]], %[[VAR138]] : f32 +// CHECK: %[[VAR166:.*]] = arith.select %[[VAR164]], %[[VAR165]], %[[VAR138]] : f32 +// CHECK: %[[VAR167:.*]] = arith.ori %[[VAR145]], %[[VAR154]] : i1 +// CHECK: %[[VAR168:.*]] = arith.andi %[[VAR110]], %[[VAR167]] : i1 +// CHECK: %[[VAR169:.*]] = arith.mulf %[[VAR157]], %[[VAR163]] fastmath : f32 +// CHECK: %[[VAR170:.*]] = arith.mulf %[[VAR160]], %[[VAR166]] fastmath : f32 +// CHECK: %[[VAR171:.*]] = arith.subf %[[VAR169]], %[[VAR170]] fastmath : f32 +// CHECK: %[[VAR172:.*]] = arith.mulf %[[CST_2]], %[[VAR171]] fastmath : f32 +// CHECK: %[[VAR173:.*]] = arith.select %[[VAR168]], %[[VAR172]], %[[VAR102]] : f32 +// CHECK: %[[VAR174:.*]] = arith.mulf %[[VAR160]], %[[VAR163]] fastmath : f32 +// CHECK: %[[VAR175:.*]] = arith.mulf %[[VAR157]], %[[VAR166]] fastmath : f32 +// CHECK: %[[VAR176:.*]] = arith.addf %[[VAR174]], %[[VAR175]] fastmath : f32 +// CHECK: %[[VAR177:.*]] = arith.mulf %[[CST_2]], %[[VAR176]] fastmath : f32 +// CHECK: %[[VAR178:.*]] = arith.select %[[VAR168]], %[[VAR177]], %[[VAR107]] : f32 +// CHECK: %[[VAR179:.*]] = complex.create %[[VAR173]], %[[VAR178]] : complex +// CHECK: %[[VAR180:.*]] = complex.re %[[VAR89]] : complex +// CHECK: %[[VAR181:.*]] = complex.re %[[VAR179]] : complex +// CHECK: %[[VAR182:.*]] = arith.addf %[[VAR180]], %[[VAR181]] fastmath : f32 +// CHECK: %[[VAR183:.*]] = complex.im %[[VAR89]] : complex +// CHECK: %[[VAR184:.*]] = complex.im %[[VAR179]] : complex +// CHECK: %[[VAR185:.*]] = arith.addf %[[VAR183]], %[[VAR184]] fastmath : f32 +// CHECK: %[[VAR186:.*]] = complex.create %[[VAR182]], %[[VAR185]] : complex +// CHECK: %[[CST_6:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK: %[[VAR187:.*]] = complex.re %[[VAR186]] : complex +// CHECK: %[[VAR188:.*]] = complex.im %[[VAR186]] : complex +// CHECK: %[[VAR189:.*]] = math.absf %[[VAR187]] : f32 +// CHECK: %[[CST_7:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK: %[[CST_8:.*]] = arith.constant 1.000000e+00 : f32 +// CHECK: %[[VAR190:.*]] = complex.re %[[VAR186]] : complex +// CHECK: %[[VAR191:.*]] = complex.im %[[VAR186]] : complex +// CHECK: %[[VAR192:.*]] = arith.cmpf oeq, %[[VAR190]], %[[CST_7]] : f32 +// CHECK: %[[VAR193:.*]] = arith.cmpf oeq, %[[VAR191]], %[[CST_7]] : f32 +// CHECK: %[[VAR194:.*]] = arith.divf %[[VAR191]], %[[VAR190]] : f32 +// CHECK: %[[VAR195:.*]] = arith.mulf %[[VAR194]], %[[VAR194]] : f32 +// CHECK: %[[VAR196:.*]] = arith.addf %[[VAR195]], %[[CST_8]] : f32 +// CHECK: %[[VAR197:.*]] = math.sqrt %[[VAR196]] : f32 +// CHECK: %[[VAR198:.*]] = math.absf %[[VAR190]] : f32 +// CHECK: %[[VAR199:.*]] = arith.mulf %[[VAR197]], %[[VAR198]] : f32 +// CHECK: %[[VAR200:.*]] = arith.divf %[[VAR190]], %[[VAR191]] : f32 +// CHECK: %[[VAR201:.*]] = arith.mulf %[[VAR200]], %[[VAR200]] : f32 +// CHECK: %[[VAR202:.*]] = arith.addf %[[VAR201]], %[[CST_8]] : f32 +// CHECK: %[[VAR203:.*]] = math.sqrt %[[VAR202]] : f32 +// CHECK: %[[VAR204:.*]] = math.absf %[[VAR191]] : f32 +// CHECK: %[[VAR205:.*]] = arith.mulf %[[VAR203]], %[[VAR204]] : f32 +// CHECK: %[[VAR206:.*]] = arith.cmpf ogt, %[[VAR190]], %[[VAR191]] : f32 +// CHECK: %[[VAR207:.*]] = arith.select %[[VAR206]], %[[VAR199]], %[[VAR205]] : f32 +// CHECK: %[[VAR208:.*]] = arith.select %[[VAR193]], %[[VAR198]], %[[VAR207]] : f32 +// CHECK: %[[VAR209:.*]] = arith.select %[[VAR192]], %[[VAR204]], %[[VAR208]] : f32 +// CHECK: %[[VAR210:.*]] = arith.addf %[[VAR189]], %[[VAR209]] : f32 +// CHECK: %[[CST_9:.*]] = arith.constant 5.000000e-01 : f32 +// CHECK: %[[VAR211:.*]] = arith.mulf %[[VAR210]], %[[CST_9]] : f32 +// CHECK: %[[VAR212:.*]] = math.sqrt %[[VAR211]] : f32 +// CHECK: %[[VAR213:.*]] = arith.cmpf olt, %[[VAR187]], %[[CST_6]] : f32 +// CHECK: %[[VAR214:.*]] = arith.cmpf olt, %[[VAR188]], %[[CST_6]] : f32 +// CHECK: %[[VAR215:.*]] = arith.addf %[[VAR212]], %[[VAR212]] : f32 +// CHECK: %[[VAR216:.*]] = arith.divf %[[VAR188]], %[[VAR215]] : f32 +// CHECK: %[[VAR217:.*]] = arith.negf %[[VAR212]] : f32 +// CHECK: %[[VAR218:.*]] = arith.select %[[VAR214]], %[[VAR217]], %[[VAR212]] : f32 +// CHECK: %[[VAR219:.*]] = arith.select %[[VAR213]], %[[VAR218]], %[[VAR216]] : f32 +// CHECK: %[[VAR220:.*]] = arith.addf %[[VAR219]], %[[VAR219]] : f32 +// CHECK: %[[VAR221:.*]] = arith.divf %[[VAR188]], %[[VAR220]] : f32 +// CHECK: %[[VAR222:.*]] = arith.select %[[VAR213]], %[[VAR221]], %[[VAR212]] : f32 +// CHECK: %[[VAR223:.*]] = arith.cmpf oeq, %[[VAR187]], %[[CST_6]] : f32 +// CHECK: %[[VAR224:.*]] = arith.cmpf oeq, %[[VAR188]], %[[CST_6]] : f32 +// CHECK: %[[VAR225:.*]] = arith.andi %[[VAR223]], %[[VAR224]] : i1 +// CHECK: %[[VAR226:.*]] = arith.select %[[VAR225]], %[[CST_6]], %[[VAR222]] : f32 +// CHECK: %[[VAR227:.*]] = arith.select %[[VAR225]], %[[CST_6]], %[[VAR219]] : f32 +// CHECK: %[[VAR228:.*]] = complex.create %[[VAR226]], %[[VAR227]] : complex +// CHECK: %[[CST_10:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK: %[[CST_11:.*]] = arith.constant 1.000000e+00 : f32 +// CHECK: %[[VAR229:.*]] = complex.create %[[CST_10]], %[[CST_11]] : complex +// CHECK: %[[VAR230:.*]] = complex.re %[[VAR229]] : complex +// CHECK: %[[VAR231:.*]] = math.absf %[[VAR230]] fastmath : f32 +// CHECK: %[[VAR232:.*]] = complex.im %[[VAR229]] : complex +// CHECK: %[[VAR233:.*]] = math.absf %[[VAR232]] fastmath : f32 +// CHECK: %[[VAR234:.*]] = complex.re %arg0 : complex +// CHECK: %[[VAR235:.*]] = math.absf %[[VAR234]] fastmath : f32 +// CHECK: %[[VAR236:.*]] = complex.im %arg0 : complex +// CHECK: %[[VAR237:.*]] = math.absf %[[VAR236]] fastmath : f32 +// CHECK: %[[VAR238:.*]] = arith.mulf %[[VAR230]], %[[VAR234]] fastmath : f32 +// CHECK: %[[VAR239:.*]] = math.absf %[[VAR238]] fastmath : f32 +// CHECK: %[[VAR240:.*]] = arith.mulf %[[VAR232]], %[[VAR236]] fastmath : f32 +// CHECK: %[[VAR241:.*]] = math.absf %[[VAR240]] fastmath : f32 +// CHECK: %[[VAR242:.*]] = arith.subf %[[VAR238]], %[[VAR240]] fastmath : f32 +// CHECK: %[[VAR243:.*]] = arith.mulf %[[VAR232]], %[[VAR234]] fastmath : f32 +// CHECK: %[[VAR244:.*]] = math.absf %[[VAR243]] fastmath : f32 +// CHECK: %[[VAR245:.*]] = arith.mulf %[[VAR230]], %[[VAR236]] fastmath : f32 +// CHECK: %[[VAR246:.*]] = math.absf %[[VAR245]] fastmath : f32 +// CHECK: %[[VAR247:.*]] = arith.addf %[[VAR243]], %[[VAR245]] fastmath : f32 +// CHECK: %[[VAR248:.*]] = arith.cmpf uno, %[[VAR242]], %[[VAR242]] : f32 +// CHECK: %[[VAR249:.*]] = arith.cmpf uno, %[[VAR247]], %[[VAR247]] : f32 +// CHECK: %[[VAR250:.*]] = arith.andi %[[VAR248]], %[[VAR249]] : i1 +// CHECK: %[[CST_12:.*]] = arith.constant 0x7F800000 : f32 +// CHECK: %[[VAR251:.*]] = arith.cmpf oeq, %[[VAR231]], %[[CST_12]] : f32 +// CHECK: %[[VAR252:.*]] = arith.cmpf oeq, %[[VAR233]], %[[CST_12]] : f32 +// CHECK: %[[VAR253:.*]] = arith.ori %[[VAR251]], %[[VAR252]] : i1 +// CHECK: %[[VAR254:.*]] = arith.cmpf uno, %[[VAR234]], %[[VAR234]] : f32 +// CHECK: %[[VAR255:.*]] = arith.cmpf uno, %[[VAR236]], %[[VAR236]] : f32 +// CHECK: %[[CST_13:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK: %[[CST_14:.*]] = arith.constant 1.000000e+00 : f32 +// CHECK: %[[VAR256:.*]] = arith.select %[[VAR251]], %[[CST_14]], %[[CST_13]] : f32 +// CHECK: %[[VAR257:.*]] = math.copysign %[[VAR256]], %[[VAR230]] : f32 +// CHECK: %[[VAR258:.*]] = arith.select %[[VAR253]], %[[VAR257]], %[[VAR230]] : f32 +// CHECK: %[[VAR259:.*]] = arith.select %[[VAR252]], %[[CST_14]], %[[CST_13]] : f32 +// CHECK: %[[VAR260:.*]] = math.copysign %[[VAR259]], %[[VAR232]] : f32 +// CHECK: %[[VAR261:.*]] = arith.select %[[VAR253]], %[[VAR260]], %[[VAR232]] : f32 +// CHECK: %[[VAR262:.*]] = arith.andi %[[VAR253]], %[[VAR254]] : i1 +// CHECK: %[[VAR263:.*]] = math.copysign %[[CST_13]], %[[VAR234]] : f32 +// CHECK: %[[VAR264:.*]] = arith.select %[[VAR262]], %[[VAR263]], %[[VAR234]] : f32 +// CHECK: %[[VAR265:.*]] = arith.andi %[[VAR253]], %[[VAR255]] : i1 +// CHECK: %[[VAR266:.*]] = math.copysign %[[CST_13]], %[[VAR236]] : f32 +// CHECK: %[[VAR267:.*]] = arith.select %[[VAR265]], %[[VAR266]], %[[VAR236]] : f32 +// CHECK: %[[VAR268:.*]] = arith.cmpf oeq, %[[VAR235]], %[[CST_12]] : f32 +// CHECK: %[[VAR269:.*]] = arith.cmpf oeq, %[[VAR237]], %[[CST_12]] : f32 +// CHECK: %[[VAR270:.*]] = arith.ori %[[VAR268]], %[[VAR269]] : i1 +// CHECK: %[[VAR271:.*]] = arith.cmpf uno, %[[VAR258]], %[[VAR258]] : f32 +// CHECK: %[[VAR272:.*]] = arith.cmpf uno, %[[VAR261]], %[[VAR261]] : f32 +// CHECK: %[[VAR273:.*]] = arith.select %[[VAR268]], %[[CST_14]], %[[CST_13]] : f32 +// CHECK: %[[VAR274:.*]] = math.copysign %[[VAR273]], %[[VAR264]] : f32 +// CHECK: %[[VAR275:.*]] = arith.select %[[VAR270]], %[[VAR274]], %[[VAR264]] : f32 +// CHECK: %[[VAR276:.*]] = arith.select %[[VAR269]], %[[CST_14]], %[[CST_13]] : f32 +// CHECK: %[[VAR277:.*]] = math.copysign %[[VAR276]], %[[VAR267]] : f32 +// CHECK: %[[VAR278:.*]] = arith.select %[[VAR270]], %[[VAR277]], %[[VAR267]] : f32 +// CHECK: %[[VAR279:.*]] = arith.andi %[[VAR270]], %[[VAR271]] : i1 +// CHECK: %[[VAR280:.*]] = math.copysign %[[CST_13]], %[[VAR258]] : f32 +// CHECK: %[[VAR281:.*]] = arith.select %[[VAR279]], %[[VAR280]], %[[VAR258]] : f32 +// CHECK: %[[VAR282:.*]] = arith.andi %[[VAR270]], %[[VAR272]] : i1 +// CHECK: %[[VAR283:.*]] = math.copysign %[[CST_13]], %[[VAR261]] : f32 +// CHECK: %[[VAR284:.*]] = arith.select %[[VAR282]], %[[VAR283]], %[[VAR261]] : f32 +// CHECK: %[[VAR285:.*]] = arith.ori %[[VAR253]], %[[VAR270]] : i1 +// CHECK: %[[VAR286:.*]] = arith.cmpf oeq, %[[VAR239]], %[[CST_12]] : f32 +// CHECK: %[[VAR287:.*]] = arith.cmpf oeq, %[[VAR241]], %[[CST_12]] : f32 +// CHECK: %[[VAR288:.*]] = arith.ori %[[VAR286]], %[[VAR287]] : i1 +// CHECK: %[[VAR289:.*]] = arith.cmpf oeq, %[[VAR246]], %[[CST_12]] : f32 +// CHECK: %[[VAR290:.*]] = arith.ori %[[VAR288]], %[[VAR289]] : i1 +// CHECK: %[[VAR291:.*]] = arith.cmpf oeq, %[[VAR244]], %[[CST_12]] : f32 +// CHECK: %[[VAR292:.*]] = arith.ori %[[VAR290]], %[[VAR291]] : i1 +// CHECK: %[[TRUE_15:.*]] = arith.constant true +// CHECK: %[[VAR293:.*]] = arith.xori %[[VAR285]], %[[TRUE_15]] : i1 +// CHECK: %[[VAR294:.*]] = arith.andi %[[VAR292]], %[[VAR293]] : i1 +// CHECK: %[[VAR295:.*]] = arith.andi %[[VAR294]], %[[VAR271]] : i1 +// CHECK: %[[VAR296:.*]] = math.copysign %[[CST_13]], %[[VAR281]] : f32 +// CHECK: %[[VAR297:.*]] = arith.select %[[VAR295]], %[[VAR296]], %[[VAR281]] : f32 +// CHECK: %[[VAR298:.*]] = arith.andi %[[VAR294]], %[[VAR272]] : i1 +// CHECK: %[[VAR299:.*]] = math.copysign %[[CST_13]], %[[VAR284]] : f32 +// CHECK: %[[VAR300:.*]] = arith.select %[[VAR298]], %[[VAR299]], %[[VAR284]] : f32 +// CHECK: %[[VAR301:.*]] = arith.andi %[[VAR294]], %[[VAR254]] : i1 +// CHECK: %[[VAR302:.*]] = math.copysign %[[CST_13]], %[[VAR275]] : f32 +// CHECK: %[[VAR303:.*]] = arith.select %[[VAR301]], %[[VAR302]], %[[VAR275]] : f32 +// CHECK: %[[VAR304:.*]] = arith.andi %[[VAR294]], %[[VAR255]] : i1 +// CHECK: %[[VAR305:.*]] = math.copysign %[[CST_13]], %[[VAR278]] : f32 +// CHECK: %[[VAR306:.*]] = arith.select %[[VAR304]], %[[VAR305]], %[[VAR278]] : f32 +// CHECK: %[[VAR307:.*]] = arith.ori %[[VAR285]], %[[VAR294]] : i1 +// CHECK: %[[VAR308:.*]] = arith.andi %[[VAR250]], %[[VAR307]] : i1 +// CHECK: %[[VAR309:.*]] = arith.mulf %[[VAR297]], %[[VAR303]] fastmath : f32 +// CHECK: %[[VAR310:.*]] = arith.mulf %[[VAR300]], %[[VAR306]] fastmath : f32 +// CHECK: %[[VAR311:.*]] = arith.subf %[[VAR309]], %[[VAR310]] fastmath : f32 +// CHECK: %[[VAR312:.*]] = arith.mulf %[[CST_12]], %[[VAR311]] fastmath : f32 +// CHECK: %[[VAR313:.*]] = arith.select %[[VAR308]], %[[VAR312]], %[[VAR242]] : f32 +// CHECK: %[[VAR314:.*]] = arith.mulf %[[VAR300]], %[[VAR303]] fastmath : f32 +// CHECK: %[[VAR315:.*]] = arith.mulf %[[VAR297]], %[[VAR306]] fastmath : f32 +// CHECK: %[[VAR316:.*]] = arith.addf %[[VAR314]], %[[VAR315]] fastmath : f32 +// CHECK: %[[VAR317:.*]] = arith.mulf %[[CST_12]], %[[VAR316]] fastmath : f32 +// CHECK: %[[VAR318:.*]] = arith.select %[[VAR308]], %[[VAR317]], %[[VAR247]] : f32 +// CHECK: %[[VAR319:.*]] = complex.create %[[VAR313]], %[[VAR318]] : complex +// CHECK: %[[VAR320:.*]] = complex.re %arg1 : complex +// CHECK: %[[VAR321:.*]] = complex.re %[[VAR319]] : complex +// CHECK: %[[VAR322:.*]] = arith.addf %[[VAR320]], %[[VAR321]] fastmath : f32 +// CHECK: %[[VAR323:.*]] = complex.im %arg1 : complex +// CHECK: %[[VAR324:.*]] = complex.im %[[VAR319]] : complex +// CHECK: %[[VAR325:.*]] = arith.addf %[[VAR323]], %[[VAR324]] fastmath : f32 +// CHECK: %[[VAR326:.*]] = complex.create %[[VAR322]], %[[VAR325]] : complex +// CHECK: %[[VAR327:.*]] = complex.re %[[VAR326]] : complex +// CHECK: %[[VAR328:.*]] = complex.im %[[VAR326]] : complex +// CHECK: %[[VAR329:.*]] = complex.re %[[VAR228]] : complex +// CHECK: %[[VAR330:.*]] = complex.im %[[VAR228]] : complex +// CHECK: %[[VAR331:.*]] = arith.divf %[[VAR329]], %[[VAR330]] fastmath : f32 +// CHECK: %[[VAR332:.*]] = arith.mulf %[[VAR331]], %[[VAR329]] fastmath : f32 +// CHECK: %[[VAR333:.*]] = arith.addf %[[VAR330]], %[[VAR332]] fastmath : f32 +// CHECK: %[[VAR334:.*]] = arith.mulf %[[VAR327]], %[[VAR331]] fastmath : f32 +// CHECK: %[[VAR335:.*]] = arith.addf %[[VAR334]], %[[VAR328]] fastmath : f32 +// CHECK: %[[VAR336:.*]] = arith.divf %[[VAR335]], %[[VAR333]] fastmath : f32 +// CHECK: %[[VAR337:.*]] = arith.mulf %[[VAR328]], %[[VAR331]] fastmath : f32 +// CHECK: %[[VAR338:.*]] = arith.subf %[[VAR337]], %[[VAR327]] fastmath : f32 +// CHECK: %[[VAR339:.*]] = arith.divf %[[VAR338]], %[[VAR333]] fastmath : f32 +// CHECK: %[[VAR340:.*]] = arith.divf %[[VAR330]], %[[VAR329]] fastmath : f32 +// CHECK: %[[VAR341:.*]] = arith.mulf %[[VAR340]], %[[VAR330]] fastmath : f32 +// CHECK: %[[VAR342:.*]] = arith.addf %[[VAR329]], %[[VAR341]] fastmath : f32 +// CHECK: %[[VAR343:.*]] = arith.mulf %[[VAR328]], %[[VAR340]] fastmath : f32 +// CHECK: %[[VAR344:.*]] = arith.addf %[[VAR327]], %[[VAR343]] fastmath : f32 +// CHECK: %[[VAR345:.*]] = arith.divf %[[VAR344]], %[[VAR342]] fastmath : f32 +// CHECK: %[[VAR346:.*]] = arith.mulf %[[VAR327]], %[[VAR340]] fastmath : f32 +// CHECK: %[[VAR347:.*]] = arith.subf %[[VAR328]], %[[VAR346]] fastmath : f32 +// CHECK: %[[VAR348:.*]] = arith.divf %[[VAR347]], %[[VAR342]] fastmath : f32 +// CHECK: %[[CST_16:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK: %[[VAR349:.*]] = math.absf %[[VAR329]] fastmath : f32 +// CHECK: %[[VAR350:.*]] = arith.cmpf oeq, %[[VAR349]], %[[CST_16]] : f32 +// CHECK: %[[VAR351:.*]] = math.absf %[[VAR330]] fastmath : f32 +// CHECK: %[[VAR352:.*]] = arith.cmpf oeq, %[[VAR351]], %[[CST_16]] : f32 +// CHECK: %[[VAR353:.*]] = arith.cmpf ord, %[[VAR327]], %[[CST_16]] : f32 +// CHECK: %[[VAR354:.*]] = arith.cmpf ord, %[[VAR328]], %[[CST_16]] : f32 +// CHECK: %[[VAR355:.*]] = arith.ori %[[VAR353]], %[[VAR354]] : i1 +// CHECK: %[[VAR356:.*]] = arith.andi %[[VAR350]], %[[VAR352]] : i1 +// CHECK: %[[VAR357:.*]] = arith.andi %[[VAR355]], %[[VAR356]] : i1 +// CHECK: %[[CST_17:.*]] = arith.constant 0x7F800000 : f32 +// CHECK: %[[VAR358:.*]] = math.copysign %[[CST_17]], %[[VAR329]] : f32 +// CHECK: %[[VAR359:.*]] = arith.mulf %[[VAR358]], %[[VAR327]] fastmath : f32 +// CHECK: %[[VAR360:.*]] = arith.mulf %[[VAR358]], %[[VAR328]] fastmath : f32 +// CHECK: %[[VAR361:.*]] = arith.cmpf one, %[[VAR349]], %[[CST_17]] : f32 +// CHECK: %[[VAR362:.*]] = arith.cmpf one, %[[VAR351]], %[[CST_17]] : f32 +// CHECK: %[[VAR363:.*]] = arith.andi %[[VAR361]], %[[VAR362]] : i1 +// CHECK: %[[VAR364:.*]] = math.absf %[[VAR327]] fastmath : f32 +// CHECK: %[[VAR365:.*]] = arith.cmpf oeq, %[[VAR364]], %[[CST_17]] : f32 +// CHECK: %[[VAR366:.*]] = math.absf %[[VAR328]] fastmath : f32 +// CHECK: %[[VAR367:.*]] = arith.cmpf oeq, %[[VAR366]], %[[CST_17]] : f32 +// CHECK: %[[VAR368:.*]] = arith.ori %[[VAR365]], %[[VAR367]] : i1 +// CHECK: %[[VAR369:.*]] = arith.andi %[[VAR368]], %[[VAR363]] : i1 +// CHECK: %[[CST_18:.*]] = arith.constant 1.000000e+00 : f32 +// CHECK: %[[VAR370:.*]] = arith.select %[[VAR365]], %[[CST_18]], %[[CST_16]] : f32 +// CHECK: %[[VAR371:.*]] = math.copysign %[[VAR370]], %[[VAR327]] : f32 +// CHECK: %[[VAR372:.*]] = arith.select %[[VAR367]], %[[CST_18]], %[[CST_16]] : f32 +// CHECK: %[[VAR373:.*]] = math.copysign %[[VAR372]], %[[VAR328]] : f32 +// CHECK: %[[VAR374:.*]] = arith.mulf %[[VAR371]], %[[VAR329]] fastmath : f32 +// CHECK: %[[VAR375:.*]] = arith.mulf %[[VAR373]], %[[VAR330]] fastmath : f32 +// CHECK: %[[VAR376:.*]] = arith.addf %[[VAR374]], %[[VAR375]] fastmath : f32 +// CHECK: %[[VAR377:.*]] = arith.mulf %[[CST_17]], %[[VAR376]] fastmath : f32 +// CHECK: %[[VAR378:.*]] = arith.mulf %[[VAR371]], %[[VAR330]] fastmath : f32 +// CHECK: %[[VAR379:.*]] = arith.mulf %[[VAR373]], %[[VAR329]] fastmath : f32 +// CHECK: %[[VAR380:.*]] = arith.subf %[[VAR379]], %[[VAR378]] fastmath : f32 +// CHECK: %[[VAR381:.*]] = arith.mulf %[[CST_17]], %[[VAR380]] fastmath : f32 +// CHECK: %[[VAR382:.*]] = arith.cmpf one, %[[VAR364]], %[[CST_17]] : f32 +// CHECK: %[[VAR383:.*]] = arith.cmpf one, %[[VAR366]], %[[CST_17]] : f32 +// CHECK: %[[VAR384:.*]] = arith.andi %[[VAR382]], %[[VAR383]] : i1 +// CHECK: %[[VAR385:.*]] = arith.cmpf oeq, %[[VAR349]], %[[CST_17]] : f32 +// CHECK: %[[VAR386:.*]] = arith.cmpf oeq, %[[VAR351]], %[[CST_17]] : f32 +// CHECK: %[[VAR387:.*]] = arith.ori %[[VAR385]], %[[VAR386]] : i1 +// CHECK: %[[VAR388:.*]] = arith.andi %[[VAR384]], %[[VAR387]] : i1 +// CHECK: %[[VAR389:.*]] = arith.select %[[VAR385]], %[[CST_18]], %[[CST_16]] : f32 +// CHECK: %[[VAR390:.*]] = math.copysign %[[VAR389]], %[[VAR329]] : f32 +// CHECK: %[[VAR391:.*]] = arith.select %[[VAR386]], %[[CST_18]], %[[CST_16]] : f32 +// CHECK: %[[VAR392:.*]] = math.copysign %[[VAR391]], %[[VAR330]] : f32 +// CHECK: %[[VAR393:.*]] = arith.mulf %[[VAR327]], %[[VAR390]] fastmath : f32 +// CHECK: %[[VAR394:.*]] = arith.mulf %[[VAR328]], %[[VAR392]] fastmath : f32 +// CHECK: %[[VAR395:.*]] = arith.addf %[[VAR393]], %[[VAR394]] fastmath : f32 +// CHECK: %[[VAR396:.*]] = arith.mulf %[[CST_16]], %[[VAR395]] fastmath : f32 +// CHECK: %[[VAR397:.*]] = arith.mulf %[[VAR328]], %[[VAR390]] fastmath : f32 +// CHECK: %[[VAR398:.*]] = arith.mulf %[[VAR327]], %[[VAR392]] fastmath : f32 +// CHECK: %[[VAR399:.*]] = arith.subf %[[VAR397]], %[[VAR398]] fastmath : f32 +// CHECK: %[[VAR400:.*]] = arith.mulf %[[CST_16]], %[[VAR399]] fastmath : f32 +// CHECK: %[[VAR401:.*]] = arith.cmpf olt, %[[VAR349]], %[[VAR351]] : f32 +// CHECK: %[[VAR402:.*]] = arith.select %[[VAR401]], %[[VAR336]], %[[VAR345]] : f32 +// CHECK: %[[VAR403:.*]] = arith.select %[[VAR401]], %[[VAR339]], %[[VAR348]] : f32 +// CHECK: %[[VAR404:.*]] = arith.select %[[VAR388]], %[[VAR396]], %[[VAR402]] : f32 +// CHECK: %[[VAR405:.*]] = arith.select %[[VAR388]], %[[VAR400]], %[[VAR403]] : f32 +// CHECK: %[[VAR406:.*]] = arith.select %[[VAR369]], %[[VAR377]], %[[VAR404]] : f32 +// CHECK: %[[VAR407:.*]] = arith.select %[[VAR369]], %[[VAR381]], %[[VAR405]] : f32 +// CHECK: %[[VAR408:.*]] = arith.select %[[VAR357]], %[[VAR359]], %[[VAR406]] : f32 +// CHECK: %[[VAR409:.*]] = arith.select %[[VAR357]], %[[VAR360]], %[[VAR407]] : f32 +// CHECK: %[[VAR410:.*]] = arith.cmpf uno, %[[VAR402]], %[[CST_16]] : f32 +// CHECK: %[[VAR411:.*]] = arith.cmpf uno, %[[VAR403]], %[[CST_16]] : f32 +// CHECK: %[[VAR412:.*]] = arith.andi %[[VAR410]], %[[VAR411]] : i1 +// CHECK: %[[VAR413:.*]] = arith.select %[[VAR412]], %[[VAR408]], %[[VAR402]] : f32 +// CHECK: %[[VAR414:.*]] = arith.select %[[VAR412]], %[[VAR409]], %[[VAR403]] : f32 +// CHECK: %[[VAR415:.*]] = complex.create %[[VAR413]], %[[VAR414]] : complex +// CHECK: %[[CST_19:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK: %[[CST_20:.*]] = arith.constant 1.000000e+00 : f32 +// CHECK: %[[VAR416:.*]] = complex.re %[[VAR415]] : complex +// CHECK: %[[VAR417:.*]] = complex.im %[[VAR415]] : complex +// CHECK: %[[VAR418:.*]] = arith.cmpf oeq, %[[VAR416]], %[[CST_19]] : f32 +// CHECK: %[[VAR419:.*]] = arith.cmpf oeq, %[[VAR417]], %[[CST_19]] : f32 +// CHECK: %[[VAR420:.*]] = arith.divf %[[VAR417]], %[[VAR416]] fastmath : f32 +// CHECK: %[[VAR421:.*]] = arith.mulf %[[VAR420]], %[[VAR420]] fastmath : f32 +// CHECK: %[[VAR422:.*]] = arith.addf %[[VAR421]], %[[CST_20]] fastmath : f32 +// CHECK: %[[VAR423:.*]] = math.sqrt %[[VAR422]] fastmath : f32 +// CHECK: %[[VAR424:.*]] = math.absf %[[VAR416]] fastmath : f32 +// CHECK: %[[VAR425:.*]] = arith.mulf %[[VAR423]], %[[VAR424]] fastmath : f32 +// CHECK: %[[VAR426:.*]] = arith.divf %[[VAR416]], %[[VAR417]] fastmath : f32 +// CHECK: %[[VAR427:.*]] = arith.mulf %[[VAR426]], %[[VAR426]] fastmath : f32 +// CHECK: %[[VAR428:.*]] = arith.addf %[[VAR427]], %[[CST_20]] fastmath : f32 +// CHECK: %[[VAR429:.*]] = math.sqrt %[[VAR428]] fastmath : f32 +// CHECK: %[[VAR430:.*]] = math.absf %[[VAR417]] fastmath : f32 +// CHECK: %[[VAR431:.*]] = arith.mulf %[[VAR429]], %[[VAR430]] fastmath : f32 +// CHECK: %[[VAR432:.*]] = arith.cmpf ogt, %[[VAR416]], %[[VAR417]] : f32 +// CHECK: %[[VAR433:.*]] = arith.select %[[VAR432]], %[[VAR425]], %[[VAR431]] : f32 +// CHECK: %[[VAR434:.*]] = arith.select %[[VAR419]], %[[VAR424]], %[[VAR433]] : f32 +// CHECK: %[[VAR435:.*]] = arith.select %[[VAR418]], %[[VAR430]], %[[VAR434]] : f32 +// CHECK: %[[VAR436:.*]] = math.log %[[VAR435]] fastmath : f32 +// CHECK: %[[VAR437:.*]] = complex.re %[[VAR415]] : complex +// CHECK: %[[VAR438:.*]] = complex.im %[[VAR415]] : complex +// CHECK: %[[VAR439:.*]] = math.atan2 %[[VAR438]], %[[VAR437]] fastmath : f32 +// CHECK: %[[VAR440:.*]] = complex.create %[[VAR436]], %[[VAR439]] : complex +// CHECK: %[[CST_21:.*]] = arith.constant -1.000000e+00 : f32 +// CHECK: %[[VAR441:.*]] = complex.create %[[CST_10]], %[[CST_21]] : complex +// CHECK: %[[VAR442:.*]] = complex.re %[[VAR441]] : complex +// CHECK: %[[VAR443:.*]] = math.absf %[[VAR442]] fastmath : f32 +// CHECK: %[[VAR444:.*]] = complex.im %[[VAR441]] : complex +// CHECK: %[[VAR445:.*]] = math.absf %[[VAR444]] fastmath : f32 +// CHECK: %[[VAR446:.*]] = complex.re %[[VAR440]] : complex +// CHECK: %[[VAR447:.*]] = math.absf %[[VAR446]] fastmath : f32 +// CHECK: %[[VAR448:.*]] = complex.im %[[VAR440]] : complex +// CHECK: %[[VAR449:.*]] = math.absf %[[VAR448]] fastmath : f32 +// CHECK: %[[VAR450:.*]] = arith.mulf %[[VAR442]], %[[VAR446]] fastmath : f32 +// CHECK: %[[VAR451:.*]] = math.absf %[[VAR450]] fastmath : f32 +// CHECK: %[[VAR452:.*]] = arith.mulf %[[VAR444]], %[[VAR448]] fastmath : f32 +// CHECK: %[[VAR453:.*]] = math.absf %[[VAR452]] fastmath : f32 +// CHECK: %[[VAR454:.*]] = arith.subf %[[VAR450]], %[[VAR452]] fastmath : f32 +// CHECK: %[[VAR455:.*]] = arith.mulf %[[VAR444]], %[[VAR446]] fastmath : f32 +// CHECK: %[[VAR456:.*]] = math.absf %[[VAR455]] fastmath : f32 +// CHECK: %[[VAR457:.*]] = arith.mulf %[[VAR442]], %[[VAR448]] fastmath : f32 +// CHECK: %[[VAR458:.*]] = math.absf %[[VAR457]] fastmath : f32 +// CHECK: %[[VAR459:.*]] = arith.addf %[[VAR455]], %[[VAR457]] fastmath : f32 +// CHECK: %[[VAR460:.*]] = arith.cmpf uno, %[[VAR454]], %[[VAR454]] : f32 +// CHECK: %[[VAR461:.*]] = arith.cmpf uno, %[[VAR459]], %[[VAR459]] : f32 +// CHECK: %[[VAR462:.*]] = arith.andi %[[VAR460]], %[[VAR461]] : i1 +// CHECK: %[[CST_22:.*]] = arith.constant 0x7F800000 : f32 +// CHECK: %[[VAR463:.*]] = arith.cmpf oeq, %[[VAR443]], %[[CST_22]] : f32 +// CHECK: %[[VAR464:.*]] = arith.cmpf oeq, %[[VAR445]], %[[CST_22]] : f32 +// CHECK: %[[VAR465:.*]] = arith.ori %[[VAR463]], %[[VAR464]] : i1 +// CHECK: %[[VAR466:.*]] = arith.cmpf uno, %[[VAR446]], %[[VAR446]] : f32 +// CHECK: %[[VAR467:.*]] = arith.cmpf uno, %[[VAR448]], %[[VAR448]] : f32 +// CHECK: %[[CST_23:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK: %[[CST_24:.*]] = arith.constant 1.000000e+00 : f32 +// CHECK: %[[VAR468:.*]] = arith.select %[[VAR463]], %[[CST_24]], %[[CST_23]] : f32 +// CHECK: %[[VAR469:.*]] = math.copysign %[[VAR468]], %[[VAR442]] : f32 +// CHECK: %[[VAR470:.*]] = arith.select %[[VAR465]], %[[VAR469]], %[[VAR442]] : f32 +// CHECK: %[[VAR471:.*]] = arith.select %[[VAR464]], %[[CST_24]], %[[CST_23]] : f32 +// CHECK: %[[VAR472:.*]] = math.copysign %[[VAR471]], %[[VAR444]] : f32 +// CHECK: %[[VAR473:.*]] = arith.select %[[VAR465]], %[[VAR472]], %[[VAR444]] : f32 +// CHECK: %[[VAR474:.*]] = arith.andi %[[VAR465]], %[[VAR466]] : i1 +// CHECK: %[[VAR475:.*]] = math.copysign %[[CST_23]], %[[VAR446]] : f32 +// CHECK: %[[VAR476:.*]] = arith.select %[[VAR474]], %[[VAR475]], %[[VAR446]] : f32 +// CHECK: %[[VAR477:.*]] = arith.andi %[[VAR465]], %[[VAR467]] : i1 +// CHECK: %[[VAR478:.*]] = math.copysign %[[CST_23]], %[[VAR448]] : f32 +// CHECK: %[[VAR479:.*]] = arith.select %[[VAR477]], %[[VAR478]], %[[VAR448]] : f32 +// CHECK: %[[VAR480:.*]] = arith.cmpf oeq, %[[VAR447]], %[[CST_22]] : f32 +// CHECK: %[[VAR481:.*]] = arith.cmpf oeq, %[[VAR449]], %[[CST_22]] : f32 +// CHECK: %[[VAR482:.*]] = arith.ori %[[VAR480]], %[[VAR481]] : i1 +// CHECK: %[[VAR483:.*]] = arith.cmpf uno, %[[VAR470]], %[[VAR470]] : f32 +// CHECK: %[[VAR484:.*]] = arith.cmpf uno, %[[VAR473]], %[[VAR473]] : f32 +// CHECK: %[[VAR485:.*]] = arith.select %[[VAR480]], %[[CST_24]], %[[CST_23]] : f32 +// CHECK: %[[VAR486:.*]] = math.copysign %[[VAR485]], %[[VAR476]] : f32 +// CHECK: %[[VAR487:.*]] = arith.select %[[VAR482]], %[[VAR486]], %[[VAR476]] : f32 +// CHECK: %[[VAR488:.*]] = arith.select %[[VAR481]], %[[CST_24]], %[[CST_23]] : f32 +// CHECK: %[[VAR489:.*]] = math.copysign %[[VAR488]], %[[VAR479]] : f32 +// CHECK: %[[VAR490:.*]] = arith.select %[[VAR482]], %[[VAR489]], %[[VAR479]] : f32 +// CHECK: %[[VAR491:.*]] = arith.andi %[[VAR482]], %[[VAR483]] : i1 +// CHECK: %[[VAR492:.*]] = math.copysign %[[CST_23]], %[[VAR470]] : f32 +// CHECK: %[[VAR493:.*]] = arith.select %[[VAR491]], %[[VAR492]], %[[VAR470]] : f32 +// CHECK: %[[VAR494:.*]] = arith.andi %[[VAR482]], %[[VAR484]] : i1 +// CHECK: %[[VAR495:.*]] = math.copysign %[[CST_23]], %[[VAR473]] : f32 +// CHECK: %[[VAR496:.*]] = arith.select %[[VAR494]], %[[VAR495]], %[[VAR473]] : f32 +// CHECK: %[[VAR497:.*]] = arith.ori %[[VAR465]], %[[VAR482]] : i1 +// CHECK: %[[VAR498:.*]] = arith.cmpf oeq, %[[VAR451]], %[[CST_22]] : f32 +// CHECK: %[[VAR499:.*]] = arith.cmpf oeq, %[[VAR453]], %[[CST_22]] : f32 +// CHECK: %[[VAR500:.*]] = arith.ori %[[VAR498]], %[[VAR499]] : i1 +// CHECK: %[[VAR501:.*]] = arith.cmpf oeq, %[[VAR458]], %[[CST_22]] : f32 +// CHECK: %[[VAR502:.*]] = arith.ori %[[VAR500]], %[[VAR501]] : i1 +// CHECK: %[[VAR503:.*]] = arith.cmpf oeq, %[[VAR456]], %[[CST_22]] : f32 +// CHECK: %[[VAR504:.*]] = arith.ori %[[VAR502]], %[[VAR503]] : i1 +// CHECK: %[[TRUE_25:.*]] = arith.constant true +// CHECK: %[[VAR505:.*]] = arith.xori %[[VAR497]], %[[TRUE_25]] : i1 +// CHECK: %[[VAR506:.*]] = arith.andi %[[VAR504]], %[[VAR505]] : i1 +// CHECK: %[[VAR507:.*]] = arith.andi %[[VAR506]], %[[VAR483]] : i1 +// CHECK: %[[VAR508:.*]] = math.copysign %[[CST_23]], %[[VAR493]] : f32 +// CHECK: %[[VAR509:.*]] = arith.select %[[VAR507]], %[[VAR508]], %[[VAR493]] : f32 +// CHECK: %[[VAR510:.*]] = arith.andi %[[VAR506]], %[[VAR484]] : i1 +// CHECK: %[[VAR511:.*]] = math.copysign %[[CST_23]], %[[VAR496]] : f32 +// CHECK: %[[VAR512:.*]] = arith.select %[[VAR510]], %[[VAR511]], %[[VAR496]] : f32 +// CHECK: %[[VAR513:.*]] = arith.andi %[[VAR506]], %[[VAR466]] : i1 +// CHECK: %[[VAR514:.*]] = math.copysign %[[CST_23]], %[[VAR487]] : f32 +// CHECK: %[[VAR515:.*]] = arith.select %[[VAR513]], %[[VAR514]], %[[VAR487]] : f32 +// CHECK: %[[VAR516:.*]] = arith.andi %[[VAR506]], %[[VAR467]] : i1 +// CHECK: %[[VAR517:.*]] = math.copysign %[[CST_23]], %[[VAR490]] : f32 +// CHECK: %[[VAR518:.*]] = arith.select %[[VAR516]], %[[VAR517]], %[[VAR490]] : f32 +// CHECK: %[[VAR519:.*]] = arith.ori %[[VAR497]], %[[VAR506]] : i1 +// CHECK: %[[VAR520:.*]] = arith.andi %[[VAR462]], %[[VAR519]] : i1 +// CHECK: %[[VAR521:.*]] = arith.mulf %[[VAR509]], %[[VAR515]] fastmath : f32 +// CHECK: %[[VAR522:.*]] = arith.mulf %[[VAR512]], %[[VAR518]] fastmath : f32 +// CHECK: %[[VAR523:.*]] = arith.subf %[[VAR521]], %[[VAR522]] fastmath : f32 +// CHECK: %[[VAR524:.*]] = arith.mulf %[[CST_22]], %[[VAR523]] fastmath : f32 +// CHECK: %[[VAR525:.*]] = arith.select %[[VAR520]], %[[VAR524]], %[[VAR454]] : f32 +// CHECK: %[[VAR526:.*]] = arith.mulf %[[VAR512]], %[[VAR515]] fastmath : f32 +// CHECK: %[[VAR527:.*]] = arith.mulf %[[VAR509]], %[[VAR518]] fastmath : f32 +// CHECK: %[[VAR528:.*]] = arith.addf %[[VAR526]], %[[VAR527]] fastmath : f32 +// CHECK: %[[VAR529:.*]] = arith.mulf %[[CST_22]], %[[VAR528]] fastmath : f32 +// CHECK: %[[VAR530:.*]] = arith.select %[[VAR520]], %[[VAR529]], %[[VAR459]] : f32 +// CHECK: %[[VAR531:.*]] = complex.create %[[VAR525]], %[[VAR530]] : complex +// CHECK: return %[[VAR531]] : complex + +// ----- + // CHECK-LABEL: func @complex_div_with_fmf // CHECK-SAME: (%[[LHS:.*]]: complex, %[[RHS:.*]]: complex) func.func @complex_div_with_fmf(%lhs: complex, %rhs: complex) -> complex { -- GitLab From 7bad74e66756ca2fd1fe4f5864e7123fb4553d78 Mon Sep 17 00:00:00 2001 From: Qwinci <32550582+Qwinci@users.noreply.github.com> Date: Wed, 6 Mar 2024 06:33:55 +0200 Subject: [PATCH 246/929] [clangd] Show argument names for function pointer struct fields (#69011) Show argument names in signature help when calling a function pointer struct field. --- .../clangd/unittests/CodeCompleteTests.cpp | 17 +++++++++++++++++ clang/lib/Sema/SemaCodeComplete.cpp | 6 ++++++ 2 files changed, 23 insertions(+) diff --git a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp index 6d387fec9b38..5721feecd58e 100644 --- a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp +++ b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp @@ -1462,6 +1462,23 @@ TEST(SignatureHelpTest, FunctionPointers) { typedef void (__stdcall *fn)(int x, int y); fn foo; int main() { foo(^); } + )cpp", + // Field of function pointer type + R"cpp( + struct S { + void (*foo)(int x, int y); + }; + S s; + int main() { s.foo(^); } + )cpp", + // Field of function pointer typedef type + R"cpp( + typedef void (*fn)(int x, int y); + struct S { + fn foo; + }; + S s; + int main() { s.foo(^); } )cpp"}; for (auto Test : Tests) EXPECT_THAT(signatures(Test).signatures, diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp index c44be0df9b0a..8d7523900940 100644 --- a/clang/lib/Sema/SemaCodeComplete.cpp +++ b/clang/lib/Sema/SemaCodeComplete.cpp @@ -6137,6 +6137,7 @@ ProduceSignatureHelp(Sema &SemaRef, MutableArrayRef Candidates, // so that we can recover argument names from it. static FunctionProtoTypeLoc GetPrototypeLoc(Expr *Fn) { TypeLoc Target; + if (const auto *T = Fn->getType().getTypePtr()->getAs()) { Target = T->getDecl()->getTypeSourceInfo()->getTypeLoc(); @@ -6145,6 +6146,11 @@ static FunctionProtoTypeLoc GetPrototypeLoc(Expr *Fn) { if (const auto *const VD = dyn_cast(D)) { Target = VD->getTypeSourceInfo()->getTypeLoc(); } + } else if (const auto *ME = dyn_cast(Fn)) { + const auto *MD = ME->getMemberDecl(); + if (const auto *FD = dyn_cast(MD)) { + Target = FD->getTypeSourceInfo()->getTypeLoc(); + } } if (!Target) -- GitLab From 5fb331106dbcfba21f82b2a84c22a65ee9d4d014 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Tue, 5 Mar 2024 20:59:07 -0800 Subject: [PATCH 247/929] [RISCV] Use uint32_t for NumOfVReg in getVLENFactoredAmount. (#84110) The rest of the code pretty much assumed this anyway. --- llvm/lib/Target/RISCV/RISCVInstrInfo.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp index 2abe015c9f9c..ef0d7cbc835d 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp @@ -3059,11 +3059,11 @@ void RISCVInstrInfo::getVLENFactoredAmount(MachineFunction &MF, "Reserve the stack by the multiple of one vector size."); MachineRegisterInfo &MRI = MF.getRegInfo(); - int64_t NumOfVReg = Amount / 8; + assert(isInt<32>(Amount / 8) && + "Expect the number of vector registers within 32-bits."); + uint32_t NumOfVReg = Amount / 8; BuildMI(MBB, II, DL, get(RISCV::PseudoReadVLENB), DestReg).setMIFlag(Flag); - assert(isInt<32>(NumOfVReg) && - "Expect the number of vector registers within 32-bits."); if (llvm::has_single_bit(NumOfVReg)) { uint32_t ShiftAmount = Log2_32(NumOfVReg); if (ShiftAmount == 0) @@ -3137,7 +3137,7 @@ void RISCVInstrInfo::getVLENFactoredAmount(MachineFunction &MF, .setMIFlag(Flag); uint32_t PrevShiftAmount = 0; for (uint32_t ShiftAmount = 0; NumOfVReg >> ShiftAmount; ShiftAmount++) { - if (NumOfVReg & (1LL << ShiftAmount)) { + if (NumOfVReg & (1U << ShiftAmount)) { if (ShiftAmount) BuildMI(MBB, II, DL, get(RISCV::SLLI), DestReg) .addReg(DestReg, RegState::Kill) -- GitLab From 2a13422b8bcee449405e3ebff957b4020805f91c Mon Sep 17 00:00:00 2001 From: AtariDreams <83477269+AtariDreams@users.noreply.github.com> Date: Wed, 6 Mar 2024 00:08:14 -0500 Subject: [PATCH 248/929] Convert many LivePhysRegs uses to LiveRegUnits (#83905) --- llvm/lib/CodeGen/ReachingDefAnalysis.cpp | 24 +++++----- .../Target/AArch64/AArch64FrameLowering.cpp | 28 +++++------ .../Target/AMDGPU/SIOptimizeExecMasking.cpp | 8 ++-- llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp | 6 +-- llvm/lib/Target/ARM/Thumb1FrameLowering.cpp | 6 +-- .../lib/Target/SystemZ/SystemZElimCompare.cpp | 6 +-- llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp | 6 +-- .../lib/Target/SystemZ/SystemZShortenInst.cpp | 8 ++-- llvm/lib/Target/X86/X86FloatingPoint.cpp | 8 ++-- .../CodeGen/AArch64/arm64-shrink-wrapping.ll | 48 +++++++++---------- .../AArch64/stack-probing-no-scratch-reg.mir | 38 +++++++-------- llvm/test/CodeGen/Thumb/PR35481.ll | 14 +++--- 12 files changed, 96 insertions(+), 104 deletions(-) diff --git a/llvm/lib/CodeGen/ReachingDefAnalysis.cpp b/llvm/lib/CodeGen/ReachingDefAnalysis.cpp index 61a668907be7..07fa92889d88 100644 --- a/llvm/lib/CodeGen/ReachingDefAnalysis.cpp +++ b/llvm/lib/CodeGen/ReachingDefAnalysis.cpp @@ -6,10 +6,10 @@ // //===----------------------------------------------------------------------===// -#include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/SetOperations.h" -#include "llvm/CodeGen/LivePhysRegs.h" #include "llvm/CodeGen/ReachingDefAnalysis.h" +#include "llvm/ADT/SetOperations.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/CodeGen/LiveRegUnits.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/Support/Debug.h" @@ -421,9 +421,9 @@ void ReachingDefAnalysis::getLiveOuts(MachineBasicBlock *MBB, return; VisitedBBs.insert(MBB); - LivePhysRegs LiveRegs(*TRI); + LiveRegUnits LiveRegs(*TRI); LiveRegs.addLiveOuts(*MBB); - if (LiveRegs.available(MBB->getParent()->getRegInfo(), PhysReg)) + if (LiveRegs.available(PhysReg)) return; if (auto *Def = getLocalLiveOutMIDef(MBB, PhysReg)) @@ -469,11 +469,11 @@ MachineInstr *ReachingDefAnalysis::getMIOperand(MachineInstr *MI, bool ReachingDefAnalysis::isRegUsedAfter(MachineInstr *MI, MCRegister PhysReg) const { MachineBasicBlock *MBB = MI->getParent(); - LivePhysRegs LiveRegs(*TRI); + LiveRegUnits LiveRegs(*TRI); LiveRegs.addLiveOuts(*MBB); // Yes if the register is live out of the basic block. - if (!LiveRegs.available(MBB->getParent()->getRegInfo(), PhysReg)) + if (!LiveRegs.available(PhysReg)) return true; // Walk backwards through the block to see if the register is live at some @@ -481,7 +481,7 @@ bool ReachingDefAnalysis::isRegUsedAfter(MachineInstr *MI, for (MachineInstr &Last : instructionsWithoutDebug(MBB->instr_rbegin(), MBB->instr_rend())) { LiveRegs.stepBackward(Last); - if (!LiveRegs.available(MBB->getParent()->getRegInfo(), PhysReg)) + if (!LiveRegs.available(PhysReg)) return InstIds.lookup(&Last) > InstIds.lookup(MI); } return false; @@ -504,9 +504,9 @@ bool ReachingDefAnalysis::isRegDefinedAfter(MachineInstr *MI, bool ReachingDefAnalysis::isReachingDefLiveOut(MachineInstr *MI, MCRegister PhysReg) const { MachineBasicBlock *MBB = MI->getParent(); - LivePhysRegs LiveRegs(*TRI); + LiveRegUnits LiveRegs(*TRI); LiveRegs.addLiveOuts(*MBB); - if (LiveRegs.available(MBB->getParent()->getRegInfo(), PhysReg)) + if (LiveRegs.available(PhysReg)) return false; auto Last = MBB->getLastNonDebugInstr(); @@ -525,9 +525,9 @@ bool ReachingDefAnalysis::isReachingDefLiveOut(MachineInstr *MI, MachineInstr * ReachingDefAnalysis::getLocalLiveOutMIDef(MachineBasicBlock *MBB, MCRegister PhysReg) const { - LivePhysRegs LiveRegs(*TRI); + LiveRegUnits LiveRegs(*TRI); LiveRegs.addLiveOuts(*MBB); - if (LiveRegs.available(MBB->getParent()->getRegInfo(), PhysReg)) + if (LiveRegs.available(PhysReg)) return nullptr; auto Last = MBB->getLastNonDebugInstr(); diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp index 5cc612e89162..11cf42bbc80e 100644 --- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp @@ -197,6 +197,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/LivePhysRegs.h" +#include "llvm/CodeGen/LiveRegUnits.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" @@ -988,7 +989,7 @@ void AArch64FrameLowering::emitZeroCallUsedRegs(BitVector RegsToZero, } } -static void getLiveRegsForEntryMBB(LivePhysRegs &LiveRegs, +static void getLiveRegsForEntryMBB(LiveRegUnits &LiveRegs, const MachineBasicBlock &MBB) { const MachineFunction *MF = MBB.getParent(); LiveRegs.addLiveIns(MBB); @@ -1018,16 +1019,15 @@ static Register findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB) { const AArch64Subtarget &Subtarget = MF->getSubtarget(); const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo(); - LivePhysRegs LiveRegs(TRI); + LiveRegUnits LiveRegs(TRI); getLiveRegsForEntryMBB(LiveRegs, *MBB); // Prefer X9 since it was historically used for the prologue scratch reg. - const MachineRegisterInfo &MRI = MF->getRegInfo(); - if (LiveRegs.available(MRI, AArch64::X9)) + if (LiveRegs.available(AArch64::X9)) return AArch64::X9; - for (unsigned Reg : AArch64::GPR64RegClass) { - if (LiveRegs.available(MRI, Reg)) + for (Register Reg : AArch64::GPR64RegClass) { + if (LiveRegs.available(Reg)) return Reg; } return AArch64::NoRegister; @@ -1044,13 +1044,11 @@ bool AArch64FrameLowering::canUseAsPrologue( if (AFI->hasSwiftAsyncContext()) { const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo(); - const MachineRegisterInfo &MRI = MF->getRegInfo(); - LivePhysRegs LiveRegs(TRI); + LiveRegUnits LiveRegs(TRI); getLiveRegsForEntryMBB(LiveRegs, MBB); // The StoreSwiftAsyncContext clobbers X16 and X17. Make sure they are // available. - if (!LiveRegs.available(MRI, AArch64::X16) || - !LiveRegs.available(MRI, AArch64::X17)) + if (!LiveRegs.available(AArch64::X16) || !LiveRegs.available(AArch64::X17)) return false; } @@ -1603,7 +1601,7 @@ static void emitDefineCFAWithFP(MachineFunction &MF, MachineBasicBlock &MBB, /// Collect live registers from the end of \p MI's parent up to (including) \p /// MI in \p LiveRegs. static void getLivePhysRegsUpTo(MachineInstr &MI, const TargetRegisterInfo &TRI, - LivePhysRegs &LiveRegs) { + LiveRegUnits &LiveRegs) { MachineBasicBlock &MBB = *MI.getParent(); LiveRegs.addLiveOuts(MBB); @@ -1641,7 +1639,7 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, NonFrameStart->getFlag(MachineInstr::FrameSetup)) ++NonFrameStart; - LivePhysRegs LiveRegs(*TRI); + LiveRegUnits LiveRegs(*TRI); if (NonFrameStart != MBB.end()) { getLivePhysRegsUpTo(*NonFrameStart, *TRI, LiveRegs); // Ignore registers used for stack management for now. @@ -1659,7 +1657,7 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, make_range(MBB.instr_begin(), NonFrameStart->getIterator())) { for (auto &Op : MI.operands()) if (Op.isReg() && Op.isDef()) - assert(!LiveRegs.contains(Op.getReg()) && + assert(LiveRegs.available(Op.getReg()) && "live register clobbered by inserted prologue instructions"); } }); @@ -4014,7 +4012,7 @@ MachineBasicBlock::iterator tryMergeAdjacentSTG(MachineBasicBlock::iterator II, // FIXME : This approach of bailing out from merge is conservative in // some ways like even if stg loops are not present after merge the // insert list, this liveness check is done (which is not needed). - LivePhysRegs LiveRegs(*(MBB->getParent()->getSubtarget().getRegisterInfo())); + LiveRegUnits LiveRegs(*(MBB->getParent()->getSubtarget().getRegisterInfo())); LiveRegs.addLiveOuts(*MBB); for (auto I = MBB->rbegin();; ++I) { MachineInstr &MI = *I; @@ -4023,7 +4021,7 @@ MachineBasicBlock::iterator tryMergeAdjacentSTG(MachineBasicBlock::iterator II, LiveRegs.stepBackward(*I); } InsertI++; - if (LiveRegs.contains(AArch64::NZCV)) + if (!LiveRegs.available(AArch64::NZCV)) return InsertI; llvm::stable_sort(Instrs, diff --git a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp index e3f54d01eb22..d510e7295125 100644 --- a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp +++ b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp @@ -11,7 +11,7 @@ #include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "SIRegisterInfo.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/CodeGen/LivePhysRegs.h" +#include "llvm/CodeGen/LiveRegUnits.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/TargetRegisterInfo.h" @@ -313,7 +313,7 @@ MachineBasicBlock::reverse_iterator SIOptimizeExecMasking::findExecCopy( return E; } -// XXX - Seems LivePhysRegs doesn't work correctly since it will incorrectly +// XXX - Seems LiveRegUnits doesn't work correctly since it will incorrectly // report the register as unavailable because a super-register with a lane mask // is unavailable. static bool isLiveOut(const MachineBasicBlock &MBB, unsigned Reg) { @@ -383,7 +383,7 @@ bool SIOptimizeExecMasking::isRegisterInUseBetween(MachineInstr &Stop, MCRegister Reg, bool UseLiveOuts, bool IgnoreStart) const { - LivePhysRegs LR(*TRI); + LiveRegUnits LR(*TRI); if (UseLiveOuts) LR.addLiveOuts(*Stop.getParent()); @@ -396,7 +396,7 @@ bool SIOptimizeExecMasking::isRegisterInUseBetween(MachineInstr &Stop, LR.stepBackward(*A); } - return !LR.available(*MRI, Reg); + return !LR.available(Reg); } // Determine if a register Reg is not re-defined and still in use diff --git a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp index 6121055eb021..9bcf00079744 100644 --- a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp +++ b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp @@ -31,7 +31,7 @@ #include "llvm/ADT/Statistic.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/CodeGen/LivePhysRegs.h" +#include "llvm/CodeGen/LiveRegUnits.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFrameInfo.h" @@ -109,7 +109,7 @@ namespace { const ARMSubtarget *STI; const TargetLowering *TL; ARMFunctionInfo *AFI; - LivePhysRegs LiveRegs; + LiveRegUnits LiveRegs; RegisterClassInfo RegClassInfo; MachineBasicBlock::const_iterator LiveRegPos; bool LiveRegsValid; @@ -589,7 +589,7 @@ unsigned ARMLoadStoreOpt::findFreeReg(const TargetRegisterClass &RegClass) { } for (unsigned Reg : RegClassInfo.getOrder(&RegClass)) - if (LiveRegs.available(MF->getRegInfo(), Reg)) + if (LiveRegs.available(Reg)) return Reg; return 0; } diff --git a/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp b/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp index 0f4ece64bff5..a8cf036f363c 100644 --- a/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp +++ b/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp @@ -612,11 +612,11 @@ bool Thumb1FrameLowering::needPopSpecialFixUp(const MachineFunction &MF) const { static void findTemporariesForLR(const BitVector &GPRsNoLRSP, const BitVector &PopFriendly, - const LivePhysRegs &UsedRegs, unsigned &PopReg, + const LiveRegUnits &UsedRegs, unsigned &PopReg, unsigned &TmpReg, MachineRegisterInfo &MRI) { PopReg = TmpReg = 0; for (auto Reg : GPRsNoLRSP.set_bits()) { - if (UsedRegs.available(MRI, Reg)) { + if (UsedRegs.available(Reg)) { // Remember the first pop-friendly register and exit. if (PopFriendly.test(Reg)) { PopReg = Reg; @@ -684,7 +684,7 @@ bool Thumb1FrameLowering::emitPopSpecialFixUp(MachineBasicBlock &MBB, // Look for a temporary register to use. // First, compute the liveness information. const TargetRegisterInfo &TRI = *STI.getRegisterInfo(); - LivePhysRegs UsedRegs(TRI); + LiveRegUnits UsedRegs(TRI); UsedRegs.addLiveOuts(MBB); // The semantic of pristines changed recently and now, // the callee-saved registers that are touched in the function diff --git a/llvm/lib/Target/SystemZ/SystemZElimCompare.cpp b/llvm/lib/Target/SystemZ/SystemZElimCompare.cpp index 7423ed429ffb..e58f50e471fc 100644 --- a/llvm/lib/Target/SystemZ/SystemZElimCompare.cpp +++ b/llvm/lib/Target/SystemZ/SystemZElimCompare.cpp @@ -18,7 +18,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringRef.h" -#include "llvm/CodeGen/LivePhysRegs.h" +#include "llvm/CodeGen/LiveRegUnits.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" @@ -690,9 +690,9 @@ bool SystemZElimCompare::processBlock(MachineBasicBlock &MBB) { // Walk backwards through the block looking for comparisons, recording // all CC users as we go. The subroutines can delete Compare and // instructions before it. - LivePhysRegs LiveRegs(*TRI); + LiveRegUnits LiveRegs(*TRI); LiveRegs.addLiveOuts(MBB); - bool CompleteCCUsers = !LiveRegs.contains(SystemZ::CC); + bool CompleteCCUsers = LiveRegs.available(SystemZ::CC); SmallVector CCUsers; MachineBasicBlock::iterator MBBI = MBB.end(); while (MBBI != MBB.begin()) { diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp index 046a12208467..53e9bf9a9d1b 100644 --- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp +++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp @@ -18,7 +18,7 @@ #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/LiveInterval.h" #include "llvm/CodeGen/LiveIntervals.h" -#include "llvm/CodeGen/LivePhysRegs.h" +#include "llvm/CodeGen/LiveRegUnits.h" #include "llvm/CodeGen/LiveVariables.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFrameInfo.h" @@ -1874,9 +1874,9 @@ prepareCompareSwapOperands(MachineBasicBlock::iterator const MBBI) const { } } if (CCLive) { - LivePhysRegs LiveRegs(*MBB->getParent()->getSubtarget().getRegisterInfo()); + LiveRegUnits LiveRegs(*MBB->getParent()->getSubtarget().getRegisterInfo()); LiveRegs.addLiveOuts(*MBB); - if (LiveRegs.contains(SystemZ::CC)) + if (!LiveRegs.available(SystemZ::CC)) return false; } diff --git a/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp b/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp index 30b22fa1ce92..c0adfdbf120b 100644 --- a/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp +++ b/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp @@ -13,7 +13,7 @@ //===----------------------------------------------------------------------===// #include "SystemZTargetMachine.h" -#include "llvm/CodeGen/LivePhysRegs.h" +#include "llvm/CodeGen/LiveRegUnits.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/TargetRegisterInfo.h" @@ -46,7 +46,7 @@ private: const SystemZInstrInfo *TII; const TargetRegisterInfo *TRI; - LivePhysRegs LiveRegs; + LiveRegUnits LiveRegs; }; char SystemZShortenInst::ID = 0; @@ -88,7 +88,7 @@ bool SystemZShortenInst::shortenIIF(MachineInstr &MI, unsigned LLIxL, unsigned GR64BitReg = TRI->getMatchingSuperReg(Reg, thisSubRegIdx, &SystemZ::GR64BitRegClass); Register OtherReg = TRI->getSubReg(GR64BitReg, otherSubRegIdx); - if (LiveRegs.contains(OtherReg)) + if (!LiveRegs.available(OtherReg)) return false; uint64_t Imm = MI.getOperand(1).getImm(); @@ -143,7 +143,7 @@ bool SystemZShortenInst::shortenOn001(MachineInstr &MI, unsigned Opcode) { // Calls shortenOn001 if CCLive is false. CC def operand is added in // case of success. bool SystemZShortenInst::shortenOn001AddCC(MachineInstr &MI, unsigned Opcode) { - if (!LiveRegs.contains(SystemZ::CC) && shortenOn001(MI, Opcode)) { + if (LiveRegs.available(SystemZ::CC) && shortenOn001(MI, Opcode)) { MachineInstrBuilder(*MI.getParent()->getParent(), &MI) .addReg(SystemZ::CC, RegState::ImplicitDefine | RegState::Dead); return true; diff --git a/llvm/lib/Target/X86/X86FloatingPoint.cpp b/llvm/lib/Target/X86/X86FloatingPoint.cpp index ca4d03913d09..260879ffaa4f 100644 --- a/llvm/lib/Target/X86/X86FloatingPoint.cpp +++ b/llvm/lib/Target/X86/X86FloatingPoint.cpp @@ -30,7 +30,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/EdgeBundles.h" -#include "llvm/CodeGen/LivePhysRegs.h" +#include "llvm/CodeGen/LiveRegUnits.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" @@ -1751,7 +1751,7 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &Inst) { void FPS::setKillFlags(MachineBasicBlock &MBB) const { const TargetRegisterInfo &TRI = *MBB.getParent()->getSubtarget().getRegisterInfo(); - LivePhysRegs LPR(TRI); + LiveRegUnits LPR(TRI); LPR.addLiveOuts(MBB); @@ -1773,14 +1773,14 @@ void FPS::setKillFlags(MachineBasicBlock &MBB) const { if (MO.isDef()) { Defs.set(Reg); - if (!LPR.contains(MO.getReg())) + if (LPR.available(MO.getReg())) MO.setIsDead(); } else Uses.push_back(&MO); } for (auto *MO : Uses) - if (Defs.test(getFPReg(*MO)) || !LPR.contains(MO->getReg())) + if (Defs.test(getFPReg(*MO)) || LPR.available(MO->getReg())) MO->setIsKill(); LPR.stepBackward(MI); diff --git a/llvm/test/CodeGen/AArch64/arm64-shrink-wrapping.ll b/llvm/test/CodeGen/AArch64/arm64-shrink-wrapping.ll index 5806bcf0dacf..a5fcbf764b64 100644 --- a/llvm/test/CodeGen/AArch64/arm64-shrink-wrapping.ll +++ b/llvm/test/CodeGen/AArch64/arm64-shrink-wrapping.ll @@ -1028,14 +1028,22 @@ false: ret i32 %tmp.0 } -; Re-aligned stack pointer with all caller-save regs live. See bug -; 26642. In this case we currently avoid shrink wrapping because -; ensuring we have a scratch register to re-align the stack pointer is -; too complicated. Output should be the same for both enabled and -; disabled shrink wrapping. +; Re-aligned stack pointer with all caller-save regs live. define void @stack_realign2(i32 %a, i32 %b, ptr %ptr1, ptr %ptr2, ptr %ptr3, ptr %ptr4, ptr %ptr5, ptr %ptr6) { ; ENABLE-LABEL: stack_realign2: ; ENABLE: ; %bb.0: +; ENABLE-NEXT: lsl w8, w1, w0 +; ENABLE-NEXT: lsr w9, w0, w1 +; ENABLE-NEXT: lsl w14, w0, w1 +; ENABLE-NEXT: lsr w11, w1, w0 +; ENABLE-NEXT: add w15, w1, w0 +; ENABLE-NEXT: sub w10, w8, w9 +; ENABLE-NEXT: subs w17, w1, w0 +; ENABLE-NEXT: add w16, w14, w8 +; ENABLE-NEXT: add w12, w9, w11 +; ENABLE-NEXT: add w13, w11, w15 +; ENABLE-NEXT: b.le LBB14_2 +; ENABLE-NEXT: ; %bb.1: ; %true ; ENABLE-NEXT: stp x28, x27, [sp, #-96]! ; 16-byte Folded Spill ; ENABLE-NEXT: stp x26, x25, [sp, #16] ; 16-byte Folded Spill ; ENABLE-NEXT: stp x24, x23, [sp, #32] ; 16-byte Folded Spill @@ -1043,8 +1051,8 @@ define void @stack_realign2(i32 %a, i32 %b, ptr %ptr1, ptr %ptr2, ptr %ptr3, ptr ; ENABLE-NEXT: stp x20, x19, [sp, #64] ; 16-byte Folded Spill ; ENABLE-NEXT: stp x29, x30, [sp, #80] ; 16-byte Folded Spill ; ENABLE-NEXT: add x29, sp, #80 -; ENABLE-NEXT: sub x9, sp, #32 -; ENABLE-NEXT: and sp, x9, #0xffffffffffffffe0 +; ENABLE-NEXT: sub x18, sp, #32 +; ENABLE-NEXT: and sp, x18, #0xffffffffffffffe0 ; ENABLE-NEXT: .cfi_def_cfa w29, 16 ; ENABLE-NEXT: .cfi_offset w30, -8 ; ENABLE-NEXT: .cfi_offset w29, -16 @@ -1058,22 +1066,17 @@ define void @stack_realign2(i32 %a, i32 %b, ptr %ptr1, ptr %ptr2, ptr %ptr3, ptr ; ENABLE-NEXT: .cfi_offset w26, -80 ; ENABLE-NEXT: .cfi_offset w27, -88 ; ENABLE-NEXT: .cfi_offset w28, -96 -; ENABLE-NEXT: lsl w8, w1, w0 -; ENABLE-NEXT: lsr w9, w0, w1 -; ENABLE-NEXT: lsl w14, w0, w1 -; ENABLE-NEXT: lsr w11, w1, w0 -; ENABLE-NEXT: add w15, w1, w0 -; ENABLE-NEXT: sub w10, w8, w9 -; ENABLE-NEXT: subs w17, w1, w0 -; ENABLE-NEXT: add w16, w14, w8 -; ENABLE-NEXT: add w12, w9, w11 -; ENABLE-NEXT: add w13, w11, w15 -; ENABLE-NEXT: b.le LBB14_2 -; ENABLE-NEXT: ; %bb.1: ; %true ; ENABLE-NEXT: str w0, [sp] ; ENABLE-NEXT: ; InlineAsm Start ; ENABLE-NEXT: nop ; ENABLE-NEXT: ; InlineAsm End +; ENABLE-NEXT: sub sp, x29, #80 +; ENABLE-NEXT: ldp x29, x30, [sp, #80] ; 16-byte Folded Reload +; ENABLE-NEXT: ldp x20, x19, [sp, #64] ; 16-byte Folded Reload +; ENABLE-NEXT: ldp x22, x21, [sp, #48] ; 16-byte Folded Reload +; ENABLE-NEXT: ldp x24, x23, [sp, #32] ; 16-byte Folded Reload +; ENABLE-NEXT: ldp x26, x25, [sp, #16] ; 16-byte Folded Reload +; ENABLE-NEXT: ldp x28, x27, [sp], #96 ; 16-byte Folded Reload ; ENABLE-NEXT: LBB14_2: ; %false ; ENABLE-NEXT: str w14, [x2] ; ENABLE-NEXT: str w8, [x3] @@ -1084,13 +1087,6 @@ define void @stack_realign2(i32 %a, i32 %b, ptr %ptr1, ptr %ptr2, ptr %ptr3, ptr ; ENABLE-NEXT: stp w0, w1, [x2, #4] ; ENABLE-NEXT: stp w16, w10, [x2, #12] ; ENABLE-NEXT: stp w12, w13, [x2, #20] -; ENABLE-NEXT: sub sp, x29, #80 -; ENABLE-NEXT: ldp x29, x30, [sp, #80] ; 16-byte Folded Reload -; ENABLE-NEXT: ldp x20, x19, [sp, #64] ; 16-byte Folded Reload -; ENABLE-NEXT: ldp x22, x21, [sp, #48] ; 16-byte Folded Reload -; ENABLE-NEXT: ldp x24, x23, [sp, #32] ; 16-byte Folded Reload -; ENABLE-NEXT: ldp x26, x25, [sp, #16] ; 16-byte Folded Reload -; ENABLE-NEXT: ldp x28, x27, [sp], #96 ; 16-byte Folded Reload ; ENABLE-NEXT: ret ; ; DISABLE-LABEL: stack_realign2: diff --git a/llvm/test/CodeGen/AArch64/stack-probing-no-scratch-reg.mir b/llvm/test/CodeGen/AArch64/stack-probing-no-scratch-reg.mir index f50bd9ab4b8a..078d8a5bf6b6 100644 --- a/llvm/test/CodeGen/AArch64/stack-probing-no-scratch-reg.mir +++ b/llvm/test/CodeGen/AArch64/stack-probing-no-scratch-reg.mir @@ -43,43 +43,43 @@ machineFunctionInfo: {} body: | ; CHECK-LABEL: name: f ; CHECK: bb.0.entry: - ; CHECK-NEXT: successors: %bb.3(0x80000000) + ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) ; CHECK-NEXT: liveins: $w0, $lr ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: $x9 = IMPLICIT_DEF + ; CHECK-NEXT: dead $wzr = SUBSWri killed renamable $w0, 1, 0, implicit-def $nzcv + ; CHECK-NEXT: Bcc 12, %bb.2, implicit $nzcv + ; CHECK-NEXT: B %bb.1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1.if.then1: + ; CHECK-NEXT: successors: %bb.3(0x80000000) + ; CHECK-NEXT: liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8, $x9, $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17, $x18, $x19, $x20, $x21, $x22, $x23, $x23, $x25, $x25, $x27, $x28, $lr + ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: early-clobber $sp = frame-setup STPXpre killed $fp, killed $lr, $sp, -2 :: (store (s64) into %stack.2), (store (s64) into %stack.1) - ; CHECK-NEXT: $x9 = frame-setup SUBXri $sp, 36, 12 + ; CHECK-NEXT: $xzr = frame-setup SUBXri $sp, 36, 12 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: bb.3.entry: + ; CHECK-NEXT: bb.3.if.then1: ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.3(0x40000000) - ; CHECK-NEXT: liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8, $x9, $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17, $x18, $x19, $x20, $x21, $x22, $x23, $x25, $x27, $x28 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: $sp = frame-setup SUBXri $sp, 1, 12 ; CHECK-NEXT: frame-setup STRXui $xzr, $sp, 0 - ; CHECK-NEXT: $xzr = frame-setup SUBSXrx64 $sp, $x9, 24, implicit-def $nzcv + ; CHECK-NEXT: $xzr = frame-setup SUBSXrx64 $sp, $xzr, 24, implicit-def $nzcv ; CHECK-NEXT: frame-setup Bcc 1, %bb.3, implicit $nzcv ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: bb.4.entry: - ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) - ; CHECK-NEXT: liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8, $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17, $x18, $x19, $x20, $x21, $x22, $x23, $x25, $x27, $x28 + ; CHECK-NEXT: bb.4.if.then1: + ; CHECK-NEXT: successors: %bb.2(0x80000000) ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: $sp = frame-setup SUBXri $sp, 2544, 0 ; CHECK-NEXT: frame-setup STRXui $xzr, $sp, 0 - ; CHECK-NEXT: $x9 = IMPLICIT_DEF - ; CHECK-NEXT: dead $wzr = SUBSWri killed renamable $w0, 1, 0, implicit-def $nzcv - ; CHECK-NEXT: Bcc 12, %bb.2, implicit $nzcv - ; CHECK-NEXT: B %bb.1 - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: bb.1.if.then1: - ; CHECK-NEXT: successors: %bb.2(0x80000000) - ; CHECK-NEXT: liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8, $x9, $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17, $x18, $x19, $x20, $x21, $x22, $x23, $x23, $x25, $x25, $x27, $x28 - ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: $x0 = ADDXri $sp, 0, 0 ; CHECK-NEXT: BL @g, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $x0, implicit-def $sp - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: bb.2.exit: ; CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 36, 12 ; CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 2544, 0 ; CHECK-NEXT: early-clobber $sp, $fp, $lr = frame-destroy LDPXpost $sp, 2 :: (load (s64) from %stack.2), (load (s64) from %stack.1) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2.exit: + ; CHECK-NEXT: liveins: $lr + ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: RET_ReallyLR bb.0.entry: successors: %bb.1(0x40000000), %bb.2(0x40000000) diff --git a/llvm/test/CodeGen/Thumb/PR35481.ll b/llvm/test/CodeGen/Thumb/PR35481.ll index ad3215ecb949..e48d1547782c 100644 --- a/llvm/test/CodeGen/Thumb/PR35481.ll +++ b/llvm/test/CodeGen/Thumb/PR35481.ll @@ -18,11 +18,10 @@ define <4 x i32> @f() local_unnamed_addr #0 { ; CHECK-V4T-NEXT: movs r2, #3 ; CHECK-V4T-NEXT: movs r3, #4 ; CHECK-V4T-NEXT: bl g +; CHECK-V4T-NEXT: ldr r7, [sp, #4] +; CHECK-V4T-NEXT: mov lr, r7 ; CHECK-V4T-NEXT: pop {r7} -; CHECK-V4T-NEXT: mov r12, r0 -; CHECK-V4T-NEXT: pop {r0} -; CHECK-V4T-NEXT: mov lr, r0 -; CHECK-V4T-NEXT: mov r0, r12 +; CHECK-V4T-NEXT: add sp, #4 ; CHECK-V4T-NEXT: bx lr ; ; CHECK-V8M-LABEL: f: @@ -36,11 +35,10 @@ define <4 x i32> @f() local_unnamed_addr #0 { ; CHECK-V8M-NEXT: movs r1, #2 ; CHECK-V8M-NEXT: movs r2, #3 ; CHECK-V8M-NEXT: movs r3, #4 +; CHECK-V8M-NEXT: ldr r7, [sp, #4] +; CHECK-V8M-NEXT: mov lr, r7 ; CHECK-V8M-NEXT: pop {r7} -; CHECK-V8M-NEXT: mov r12, r0 -; CHECK-V8M-NEXT: pop {r0} -; CHECK-V8M-NEXT: mov lr, r0 -; CHECK-V8M-NEXT: mov r0, r12 +; CHECK-V8M-NEXT: add sp, #4 ; CHECK-V8M-NEXT: b g entry: %call = tail call i32 @h(i32 1) -- GitLab From 3f7aa042b657671319f994ad3fb7c3eb79a6fe00 Mon Sep 17 00:00:00 2001 From: Mircea Trofin Date: Tue, 5 Mar 2024 21:10:59 -0800 Subject: [PATCH 249/929] [nfc] Fix RTTI for `InstrProf` intrinsics (#83511) --- llvm/include/llvm/IR/IntrinsicInst.h | 37 ++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h index fbaaef8ea443..c07b83a81a63 100644 --- a/llvm/include/llvm/IR/IntrinsicInst.h +++ b/llvm/include/llvm/IR/IntrinsicInst.h @@ -1429,7 +1429,35 @@ public: /// A base class for all instrprof intrinsics. class InstrProfInstBase : public IntrinsicInst { +protected: + static bool isCounterBase(const IntrinsicInst &I) { + switch (I.getIntrinsicID()) { + case Intrinsic::instrprof_cover: + case Intrinsic::instrprof_increment: + case Intrinsic::instrprof_increment_step: + case Intrinsic::instrprof_timestamp: + case Intrinsic::instrprof_value_profile: + return true; + } + return false; + } + static bool isMCDCBitmapBase(const IntrinsicInst &I) { + switch (I.getIntrinsicID()) { + case Intrinsic::instrprof_mcdc_parameters: + case Intrinsic::instrprof_mcdc_tvbitmap_update: + return true; + } + return false; + } + public: + static bool classof(const Value *V) { + if (const auto *Instr = dyn_cast(V)) + return isCounterBase(*Instr) || isMCDCBitmapBase(*Instr) || + Instr->getIntrinsicID() == + Intrinsic::instrprof_mcdc_condbitmap_update; + return false; + } // The name of the instrumented function. GlobalVariable *getName() const { return cast( @@ -1444,6 +1472,12 @@ public: /// A base class for all instrprof counter intrinsics. class InstrProfCntrInstBase : public InstrProfInstBase { public: + static bool classof(const Value *V) { + if (const auto *Instr = dyn_cast(V)) + return InstrProfInstBase::isCounterBase(*Instr); + return false; + } + // The number of counters for the instrumented function. ConstantInt *getNumCounters() const; // The index of the counter that this instruction acts on. @@ -1524,8 +1558,7 @@ public: class InstrProfMCDCBitmapInstBase : public InstrProfInstBase { public: static bool classof(const IntrinsicInst *I) { - return I->getIntrinsicID() == Intrinsic::instrprof_mcdc_parameters || - I->getIntrinsicID() == Intrinsic::instrprof_mcdc_tvbitmap_update; + return InstrProfInstBase::isMCDCBitmapBase(*I); } static bool classof(const Value *V) { return isa(V) && classof(cast(V)); -- GitLab From 9e759f3523e9e7fa955e4af58a6b4f63634dcb24 Mon Sep 17 00:00:00 2001 From: David Majnemer Date: Wed, 6 Mar 2024 05:31:34 +0000 Subject: [PATCH 250/929] [AArch64] Fix fptoi/itofp for bf16 There were a number of issues that needed to be addressed: - i64 to bf16 did not correctly round - strict rounding needed to yield a chain - fastisel did not have logic to bail on bf16 --- llvm/lib/Target/AArch64/AArch64FastISel.cpp | 6 +- .../Target/AArch64/AArch64ISelLowering.cpp | 121 +++++- .../CodeGen/AArch64/arm64-convert-v4f64.ll | 24 ++ .../arm64-fast-isel-conversion-fallback.ll | 371 ++++++++++++++++-- llvm/test/CodeGen/AArch64/cvt-fp-int-fp.ll | 104 +++++ llvm/test/CodeGen/AArch64/itofp.ll | 120 +++++- 6 files changed, 672 insertions(+), 74 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64FastISel.cpp b/llvm/lib/Target/AArch64/AArch64FastISel.cpp index 87c94f10e2b1..62cf6a2c47ac 100644 --- a/llvm/lib/Target/AArch64/AArch64FastISel.cpp +++ b/llvm/lib/Target/AArch64/AArch64FastISel.cpp @@ -2828,7 +2828,7 @@ bool AArch64FastISel::selectFPToInt(const Instruction *I, bool Signed) { return false; EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType(), true); - if (SrcVT == MVT::f128 || SrcVT == MVT::f16) + if (SrcVT == MVT::f128 || SrcVT == MVT::f16 || SrcVT == MVT::bf16) return false; unsigned Opc; @@ -2856,7 +2856,7 @@ bool AArch64FastISel::selectIntToFP(const Instruction *I, bool Signed) { if (!isTypeLegal(I->getType(), DestVT) || DestVT.isVector()) return false; // Let regular ISEL handle FP16 - if (DestVT == MVT::f16) + if (DestVT == MVT::f16 || DestVT == MVT::bf16) return false; assert((DestVT == MVT::f32 || DestVT == MVT::f64) && @@ -2978,7 +2978,7 @@ bool AArch64FastISel::fastLowerArguments() { } else if (VT == MVT::i64) { SrcReg = Registers[1][GPRIdx++]; RC = &AArch64::GPR64RegClass; - } else if (VT == MVT::f16) { + } else if (VT == MVT::f16 || VT == MVT::bf16) { SrcReg = Registers[2][FPRIdx++]; RC = &AArch64::FPR16RegClass; } else if (VT == MVT::f32) { diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 6061b4c1a076..2290223a06f8 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -4121,14 +4121,16 @@ SDValue AArch64TargetLowering::LowerFP_ROUND(SDValue Op, // Now that we have rounded, shift the bits into position. Narrow = DAG.getNode(ISD::SRL, dl, I32, Narrow, - DAG.getShiftAmountConstant(16, I32, dl)); + DAG.getShiftAmountConstant(16, I32, dl)); if (VT.isVector()) { EVT I16 = I32.changeVectorElementType(MVT::i16); Narrow = DAG.getNode(ISD::TRUNCATE, dl, I16, Narrow); return DAG.getNode(ISD::BITCAST, dl, VT, Narrow); } Narrow = DAG.getNode(ISD::BITCAST, dl, F32, Narrow); - return DAG.getTargetExtractSubreg(AArch64::hsub, dl, VT, Narrow); + SDValue Result = DAG.getTargetExtractSubreg(AArch64::hsub, dl, VT, Narrow); + return IsStrict ? DAG.getMergeValues({Result, Op.getOperand(0)}, dl) + : Result; } if (SrcVT != MVT::f128) { @@ -4487,20 +4489,121 @@ SDValue AArch64TargetLowering::LowerINT_TO_FP(SDValue Op, bool IsStrict = Op->isStrictFPOpcode(); SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0); - // f16 conversions are promoted to f32 when full fp16 is not supported. - if ((Op.getValueType() == MVT::f16 && !Subtarget->hasFullFP16()) || Op.getValueType() == MVT::bf16) { + bool IsSigned = Op->getOpcode() == ISD::STRICT_SINT_TO_FP || + Op->getOpcode() == ISD::SINT_TO_FP; + + auto IntToFpViaPromotion = [&](EVT PromoteVT) { SDLoc dl(Op); if (IsStrict) { - SDValue Val = DAG.getNode(Op.getOpcode(), dl, {MVT::f32, MVT::Other}, + SDValue Val = DAG.getNode(Op.getOpcode(), dl, {PromoteVT, MVT::Other}, {Op.getOperand(0), SrcVal}); return DAG.getNode( ISD::STRICT_FP_ROUND, dl, {Op.getValueType(), MVT::Other}, {Val.getValue(1), Val.getValue(0), DAG.getIntPtrConstant(0, dl)}); } - return DAG.getNode( - ISD::FP_ROUND, dl, Op.getValueType(), - DAG.getNode(Op.getOpcode(), dl, MVT::f32, SrcVal), - DAG.getIntPtrConstant(0, dl)); + return DAG.getNode(ISD::FP_ROUND, dl, Op.getValueType(), + DAG.getNode(Op.getOpcode(), dl, PromoteVT, SrcVal), + DAG.getIntPtrConstant(0, dl)); + }; + + if (Op.getValueType() == MVT::bf16) { + // bf16 conversions are promoted to f32 when converting from i16. + if (DAG.ComputeMaxSignificantBits(SrcVal) <= 24) { + return IntToFpViaPromotion(MVT::f32); + } + + // bf16 conversions are promoted to f64 when converting from i32. + if (DAG.ComputeMaxSignificantBits(SrcVal) <= 53) { + return IntToFpViaPromotion(MVT::f64); + } + + // We need to be careful about i64 -> bf16. + // Consider an i32 22216703. + // This number cannot be represented exactly as an f32 and so a itofp will + // turn it into 22216704.0 fptrunc to bf16 will turn this into 22282240.0 + // However, the correct bf16 was supposed to be 22151168.0 + // We need to use sticky rounding to get this correct. + if (SrcVal.getValueType() == MVT::i64) { + SDLoc DL(Op); + // This algorithm is equivalent to the following: + // uint64_t SrcHi = SrcVal & ~0xfffull; + // uint64_t SrcLo = SrcVal & 0xfffull; + // uint64_t Highest = SrcVal >> 53; + // bool HasHighest = Highest != 0; + // uint64_t ToRound = HasHighest ? SrcHi : SrcVal; + // double Rounded = static_cast(ToRound); + // uint64_t RoundedBits = std::bit_cast(Rounded); + // uint64_t HasLo = SrcLo != 0; + // bool NeedsAdjustment = HasHighest & HasLo; + // uint64_t AdjustedBits = RoundedBits | uint64_t{NeedsAdjustment}; + // double Adjusted = std::bit_cast(AdjustedBits); + // return static_cast<__bf16>(Adjusted); + // + // Essentially, what happens is that SrcVal either fits perfectly in a + // double-precision value or it is too big. If it is sufficiently small, + // we should just go u64 -> double -> bf16 in a naive way. Otherwise, we + // ensure that u64 -> double has no rounding error by only using the 52 + // MSB of the input. The low order bits will get merged into a sticky bit + // which will avoid issues incurred by double rounding. + + // Signed conversion is more or less like so: + // copysign((__bf16)abs(SrcVal), SrcVal) + SDValue SignBit; + if (IsSigned) { + SignBit = DAG.getNode(ISD::AND, DL, MVT::i64, SrcVal, + DAG.getConstant(1ull << 63, DL, MVT::i64)); + SrcVal = DAG.getNode(ISD::ABS, DL, MVT::i64, SrcVal); + } + SDValue SrcHi = DAG.getNode(ISD::AND, DL, MVT::i64, SrcVal, + DAG.getConstant(~0xfffull, DL, MVT::i64)); + SDValue SrcLo = DAG.getNode(ISD::AND, DL, MVT::i64, SrcVal, + DAG.getConstant(0xfffull, DL, MVT::i64)); + SDValue Highest = + DAG.getNode(ISD::SRL, DL, MVT::i64, SrcVal, + DAG.getShiftAmountConstant(53, MVT::i64, DL)); + SDValue Zero64 = DAG.getConstant(0, DL, MVT::i64); + SDValue ToRound = + DAG.getSelectCC(DL, Highest, Zero64, SrcHi, SrcVal, ISD::SETNE); + SDValue Rounded = + IsStrict ? DAG.getNode(Op.getOpcode(), DL, {MVT::f64, MVT::Other}, + {Op.getOperand(0), ToRound}) + : DAG.getNode(Op.getOpcode(), DL, MVT::f64, ToRound); + + SDValue RoundedBits = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Rounded); + if (SignBit) { + RoundedBits = DAG.getNode(ISD::OR, DL, MVT::i64, RoundedBits, SignBit); + } + + SDValue HasHighest = DAG.getSetCC( + DL, + getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i64), + Highest, Zero64, ISD::SETNE); + + SDValue HasLo = DAG.getSetCC( + DL, + getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i64), + SrcLo, Zero64, ISD::SETNE); + + SDValue NeedsAdjustment = + DAG.getNode(ISD::AND, DL, HasLo.getValueType(), HasHighest, HasLo); + NeedsAdjustment = DAG.getZExtOrTrunc(NeedsAdjustment, DL, MVT::i64); + + SDValue AdjustedBits = + DAG.getNode(ISD::OR, DL, MVT::i64, RoundedBits, NeedsAdjustment); + SDValue Adjusted = DAG.getNode(ISD::BITCAST, DL, MVT::f64, AdjustedBits); + return IsStrict + ? DAG.getNode(ISD::STRICT_FP_ROUND, DL, + {Op.getValueType(), MVT::Other}, + {Rounded.getValue(1), Adjusted, + DAG.getIntPtrConstant(0, DL)}) + : DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(), Adjusted, + DAG.getIntPtrConstant(0, DL, true)); + } + } + + // f16 conversions are promoted to f32 when full fp16 is not supported. + if (Op.getValueType() == MVT::f16 && !Subtarget->hasFullFP16()) { + return IntToFpViaPromotion(MVT::f32); } // i128 conversions are libcalls. diff --git a/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll b/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll index 9bf638f57a51..49325299f74a 100644 --- a/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll +++ b/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll @@ -54,6 +54,30 @@ define <4 x half> @uitofp_v4i64_to_v4f16(ptr %ptr) { ret <4 x half> %tmp2 } +define <4 x bfloat> @uitofp_v4i64_to_v4bf16(ptr %ptr) { +; CHECK-LABEL: uitofp_v4i64_to_v4bf16: +; CHECK: // %bb.0: +; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: ucvtf v0.2d, v0.2d +; CHECK-NEXT: ucvtf v1.2d, v1.2d +; CHECK-NEXT: fcvtn v0.2s, v0.2d +; CHECK-NEXT: fcvtn2 v0.4s, v1.2d +; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: ushr v3.4s, v0.4s, #16 +; CHECK-NEXT: add v1.4s, v0.4s, v1.4s +; CHECK-NEXT: and v2.16b, v3.16b, v2.16b +; CHECK-NEXT: add v1.4s, v2.4s, v1.4s +; CHECK-NEXT: fcmeq v2.4s, v0.4s, v0.4s +; CHECK-NEXT: orr v0.4s, #64, lsl #16 +; CHECK-NEXT: bit v0.16b, v1.16b, v2.16b +; CHECK-NEXT: shrn v0.4h, v0.4s, #16 +; CHECK-NEXT: ret + %tmp1 = load <4 x i64>, ptr %ptr + %tmp2 = uitofp <4 x i64> %tmp1 to <4 x bfloat> + ret <4 x bfloat> %tmp2 +} + define <4 x i16> @trunc_v4i64_to_v4i16(ptr %ptr) { ; CHECK-LABEL: trunc_v4i64_to_v4i16: ; CHECK: // %bb.0: diff --git a/llvm/test/CodeGen/AArch64/arm64-fast-isel-conversion-fallback.ll b/llvm/test/CodeGen/AArch64/arm64-fast-isel-conversion-fallback.ll index b3c073f53542..1aa28f5c2733 100644 --- a/llvm/test/CodeGen/AArch64/arm64-fast-isel-conversion-fallback.ll +++ b/llvm/test/CodeGen/AArch64/arm64-fast-isel-conversion-fallback.ll @@ -1,129 +1,416 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc -O0 -fast-isel -verify-machineinstrs -mtriple=arm64-eabi < %s | FileCheck --enable-var-scope %s ; Test fptosi define i32 @fptosi_wh(half %a) nounwind ssp { +; CHECK-LABEL: fptosi_wh: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: fcvt s0, h0 +; CHECK-NEXT: fcvtzs w0, s0 +; CHECK-NEXT: ret entry: -; CHECK-LABEL: fptosi_wh -; CHECK: fcvt [[REG:s[0-9]+]], h0 -; CHECK: fcvtzs w0, [[REG]] %conv = fptosi half %a to i32 ret i32 %conv } ; Test fptoui define i32 @fptoui_swh(half %a) nounwind ssp { +; CHECK-LABEL: fptoui_swh: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: fcvt s0, h0 +; CHECK-NEXT: fcvtzu w0, s0 +; CHECK-NEXT: ret entry: -; CHECK-LABEL: fptoui_swh -; CHECK: fcvt [[REG:s[0-9]+]], h0 -; CHECK: fcvtzu w0, [[REG]] %conv = fptoui half %a to i32 ret i32 %conv } ; Test sitofp define half @sitofp_hw_i1(i1 %a) nounwind ssp { +; CHECK-LABEL: sitofp_hw_i1: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: sbfx w8, w0, #0, #1 +; CHECK-NEXT: scvtf s0, w8 +; CHECK-NEXT: fcvt h0, s0 +; CHECK-NEXT: ret entry: -; CHECK-LABEL: sitofp_hw_i1 -; CHECK: sbfx [[REG:w[0-9]+]], w0, #0, #1 -; CHECK: scvtf s0, [[REG]] -; CHECK: fcvt h0, s0 %conv = sitofp i1 %a to half ret half %conv } ; Test sitofp define half @sitofp_hw_i8(i8 %a) nounwind ssp { +; CHECK-LABEL: sitofp_hw_i8: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: sxtb w8, w0 +; CHECK-NEXT: scvtf s0, w8 +; CHECK-NEXT: fcvt h0, s0 +; CHECK-NEXT: ret entry: -; CHECK-LABEL: sitofp_hw_i8 -; CHECK: sxtb [[REG:w[0-9]+]], w0 -; CHECK: scvtf s0, [[REG]] -; CHECK: fcvt h0, s0 %conv = sitofp i8 %a to half ret half %conv } ; Test sitofp define half @sitofp_hw_i16(i16 %a) nounwind ssp { +; CHECK-LABEL: sitofp_hw_i16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: sxth w8, w0 +; CHECK-NEXT: scvtf s0, w8 +; CHECK-NEXT: fcvt h0, s0 +; CHECK-NEXT: ret entry: -; CHECK-LABEL: sitofp_hw_i16 -; CHECK: sxth [[REG:w[0-9]+]], w0 -; CHECK: scvtf s0, [[REG]] -; CHECK: fcvt h0, s0 %conv = sitofp i16 %a to half ret half %conv } ; Test sitofp define half @sitofp_hw_i32(i32 %a) nounwind ssp { +; CHECK-LABEL: sitofp_hw_i32: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: scvtf s0, w0 +; CHECK-NEXT: fcvt h0, s0 +; CHECK-NEXT: ret entry: -; CHECK-LABEL: sitofp_hw_i32 -; CHECK: scvtf s0, w0 -; CHECK: fcvt h0, s0 %conv = sitofp i32 %a to half ret half %conv } ; Test sitofp define half @sitofp_hx(i64 %a) nounwind ssp { +; CHECK-LABEL: sitofp_hx: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: scvtf s0, x0 +; CHECK-NEXT: fcvt h0, s0 +; CHECK-NEXT: ret entry: -; CHECK-LABEL: sitofp_hx -; CHECK: scvtf s0, x0 -; CHECK: fcvt h0, s0 %conv = sitofp i64 %a to half ret half %conv } ; Test uitofp define half @uitofp_hw_i1(i1 %a) nounwind ssp { +; CHECK-LABEL: uitofp_hw_i1: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: and w8, w0, #0x1 +; CHECK-NEXT: ucvtf s0, w8 +; CHECK-NEXT: fcvt h0, s0 +; CHECK-NEXT: ret entry: -; CHECK-LABEL: uitofp_hw_i1 -; CHECK: and [[REG:w[0-9]+]], w0, #0x1 -; CHECK: ucvtf s0, [[REG]] -; CHECK: fcvt h0, s0 %conv = uitofp i1 %a to half ret half %conv } ; Test uitofp define half @uitofp_hw_i8(i8 %a) nounwind ssp { +; CHECK-LABEL: uitofp_hw_i8: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: and w8, w0, #0xff +; CHECK-NEXT: ucvtf s0, w8 +; CHECK-NEXT: fcvt h0, s0 +; CHECK-NEXT: ret entry: -; CHECK-LABEL: uitofp_hw_i8 -; CHECK: and [[REG:w[0-9]+]], w0, #0xff -; CHECK: ucvtf s0, [[REG]] -; CHECK: fcvt h0, s0 %conv = uitofp i8 %a to half ret half %conv } ; Test uitofp define half @uitofp_hw_i16(i16 %a) nounwind ssp { +; CHECK-LABEL: uitofp_hw_i16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: and w8, w0, #0xffff +; CHECK-NEXT: ucvtf s0, w8 +; CHECK-NEXT: fcvt h0, s0 +; CHECK-NEXT: ret entry: -; CHECK-LABEL: uitofp_hw_i16 -; CHECK: and [[REG:w[0-9]+]], w0, #0xffff -; CHECK: ucvtf s0, [[REG]] -; CHECK: fcvt h0, s0 %conv = uitofp i16 %a to half ret half %conv } ; Test uitofp define half @uitofp_hw_i32(i32 %a) nounwind ssp { +; CHECK-LABEL: uitofp_hw_i32: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ucvtf s0, w0 +; CHECK-NEXT: fcvt h0, s0 +; CHECK-NEXT: ret entry: -; CHECK-LABEL: uitofp_hw_i32 -; CHECK: ucvtf s0, w0 -; CHECK: fcvt h0, s0 %conv = uitofp i32 %a to half ret half %conv } ; Test uitofp define half @uitofp_hx(i64 %a) nounwind ssp { +; CHECK-LABEL: uitofp_hx: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ucvtf s0, x0 +; CHECK-NEXT: fcvt h0, s0 +; CHECK-NEXT: ret entry: -; CHECK-LABEL: uitofp_hx -; CHECK: ucvtf s0, x0 -; CHECK: fcvt h0, s0 %conv = uitofp i64 %a to half ret half %conv } +; Test fptosi +define i32 @fptosi_bf(bfloat %a) nounwind ssp { +; CHECK-LABEL: fptosi_bf: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: fmov s1, s0 +; CHECK-NEXT: // implicit-def: $s0 +; CHECK-NEXT: fmov s0, s1 +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: lsl w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: fcvtzs w0, s0 +; CHECK-NEXT: ret +entry: + %conv = fptosi bfloat %a to i32 + ret i32 %conv +} + +; Test fptoui +define i32 @fptoui_sbf(bfloat %a) nounwind ssp { +; CHECK-LABEL: fptoui_sbf: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: fmov s1, s0 +; CHECK-NEXT: // implicit-def: $s0 +; CHECK-NEXT: fmov s0, s1 +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: lsl w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: fcvtzu w0, s0 +; CHECK-NEXT: ret +entry: + %conv = fptoui bfloat %a to i32 + ret i32 %conv +} + +; Test sitofp +define bfloat @sitofp_bf_i1(i1 %a) nounwind ssp { +; CHECK-LABEL: sitofp_bf_i1: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: sbfx w8, w0, #0, #1 +; CHECK-NEXT: scvtf s0, w8 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w8, w9, #16, #1 +; CHECK-NEXT: add w8, w8, w9 +; CHECK-NEXT: mov w9, #32767 // =0x7fff +; CHECK-NEXT: add w8, w8, w9 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $s0 +; CHECK-NEXT: ret +entry: + %conv = sitofp i1 %a to bfloat + ret bfloat %conv +} + +; Test sitofp +define bfloat @sitofp_bf_i8(i8 %a) nounwind ssp { +; CHECK-LABEL: sitofp_bf_i8: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: sxtb w8, w0 +; CHECK-NEXT: scvtf s0, w8 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w8, w9, #16, #1 +; CHECK-NEXT: add w8, w8, w9 +; CHECK-NEXT: mov w9, #32767 // =0x7fff +; CHECK-NEXT: add w8, w8, w9 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $s0 +; CHECK-NEXT: ret +entry: + %conv = sitofp i8 %a to bfloat + ret bfloat %conv +} + +; Test sitofp +define bfloat @sitofp_bf_i16(i16 %a) nounwind ssp { +; CHECK-LABEL: sitofp_bf_i16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: sxth w8, w0 +; CHECK-NEXT: scvtf s0, w8 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w8, w9, #16, #1 +; CHECK-NEXT: add w8, w8, w9 +; CHECK-NEXT: mov w9, #32767 // =0x7fff +; CHECK-NEXT: add w8, w8, w9 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $s0 +; CHECK-NEXT: ret +entry: + %conv = sitofp i16 %a to bfloat + ret bfloat %conv +} + +; Test sitofp +define bfloat @sitofp_bf_i32(i32 %a) nounwind ssp { +; CHECK-LABEL: sitofp_bf_i32: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: scvtf d0, w0 +; CHECK-NEXT: fcvtxn s0, d0 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w8, w9, #16, #1 +; CHECK-NEXT: add w8, w8, w9 +; CHECK-NEXT: mov w9, #32767 // =0x7fff +; CHECK-NEXT: add w8, w8, w9 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $s0 +; CHECK-NEXT: ret +entry: + %conv = sitofp i32 %a to bfloat + ret bfloat %conv +} + +; Test sitofp +define bfloat @sitofp_bf_i164(i64 %a) nounwind ssp { +; CHECK-LABEL: sitofp_bf_i164: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: subs x8, x0, #0 +; CHECK-NEXT: cneg x10, x0, mi +; CHECK-NEXT: and x8, x10, #0xfffffffffffff000 +; CHECK-NEXT: lsr x9, x10, #53 +; CHECK-NEXT: subs x9, x9, #0 +; CHECK-NEXT: csel x8, x8, x10, ne +; CHECK-NEXT: scvtf d0, x8 +; CHECK-NEXT: fmov x8, d0 +; CHECK-NEXT: and x9, x0, #0x8000000000000000 +; CHECK-NEXT: orr x8, x8, x9 +; CHECK-NEXT: cset w9, ne +; CHECK-NEXT: ands x10, x10, #0xfff +; CHECK-NEXT: csel w9, wzr, w9, eq +; CHECK-NEXT: mov w9, w9 +; CHECK-NEXT: // kill: def $x9 killed $w9 +; CHECK-NEXT: orr x8, x8, x9 +; CHECK-NEXT: fmov d0, x8 +; CHECK-NEXT: fcvtxn s0, d0 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w8, w9, #16, #1 +; CHECK-NEXT: add w8, w8, w9 +; CHECK-NEXT: mov w9, #32767 // =0x7fff +; CHECK-NEXT: add w8, w8, w9 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $s0 +; CHECK-NEXT: ret +entry: + %conv = sitofp i64 %a to bfloat + ret bfloat %conv +} + +; Test uitofp +define bfloat @uitofp_bf_i1(i1 %a) nounwind ssp { +; CHECK-LABEL: uitofp_bf_i1: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: and w8, w0, #0x1 +; CHECK-NEXT: ucvtf s0, w8 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w8, w9, #16, #1 +; CHECK-NEXT: add w8, w8, w9 +; CHECK-NEXT: mov w9, #32767 // =0x7fff +; CHECK-NEXT: add w8, w8, w9 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $s0 +; CHECK-NEXT: ret +entry: + %conv = uitofp i1 %a to bfloat + ret bfloat %conv +} + +; Test uitofp +define bfloat @uitofp_bf_i8(i8 %a) nounwind ssp { +; CHECK-LABEL: uitofp_bf_i8: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: and w8, w0, #0xff +; CHECK-NEXT: ucvtf s0, w8 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w8, w9, #16, #1 +; CHECK-NEXT: add w8, w8, w9 +; CHECK-NEXT: mov w9, #32767 // =0x7fff +; CHECK-NEXT: add w8, w8, w9 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $s0 +; CHECK-NEXT: ret +entry: + %conv = uitofp i8 %a to bfloat + ret bfloat %conv +} + +; Test uitofp +define bfloat @uitofp_bf_i16(i16 %a) nounwind ssp { +; CHECK-LABEL: uitofp_bf_i16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: and w8, w0, #0xffff +; CHECK-NEXT: ucvtf s0, w8 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w8, w9, #16, #1 +; CHECK-NEXT: add w8, w8, w9 +; CHECK-NEXT: mov w9, #32767 // =0x7fff +; CHECK-NEXT: add w8, w8, w9 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $s0 +; CHECK-NEXT: ret +entry: + %conv = uitofp i16 %a to bfloat + ret bfloat %conv +} + +; Test uitofp +define bfloat @uitofp_bf_i32(i32 %a) nounwind ssp { +; CHECK-LABEL: uitofp_bf_i32: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ucvtf d0, w0 +; CHECK-NEXT: fcvtxn s0, d0 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w8, w9, #16, #1 +; CHECK-NEXT: add w8, w8, w9 +; CHECK-NEXT: mov w9, #32767 // =0x7fff +; CHECK-NEXT: add w8, w8, w9 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $s0 +; CHECK-NEXT: ret +entry: + %conv = uitofp i32 %a to bfloat + ret bfloat %conv +} + +; Test uitofp +define bfloat @uitofp_bf_i64(i64 %a) nounwind ssp { +; CHECK-LABEL: uitofp_bf_i64: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: and x8, x0, #0xfffffffffffff000 +; CHECK-NEXT: lsr x9, x0, #53 +; CHECK-NEXT: subs x9, x9, #0 +; CHECK-NEXT: csel x8, x8, x0, ne +; CHECK-NEXT: ucvtf d0, x8 +; CHECK-NEXT: fmov x8, d0 +; CHECK-NEXT: cset w9, ne +; CHECK-NEXT: ands x10, x0, #0xfff +; CHECK-NEXT: csel w9, wzr, w9, eq +; CHECK-NEXT: mov w9, w9 +; CHECK-NEXT: // kill: def $x9 killed $w9 +; CHECK-NEXT: orr x8, x8, x9 +; CHECK-NEXT: fmov d0, x8 +; CHECK-NEXT: fcvtxn s0, d0 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w8, w9, #16, #1 +; CHECK-NEXT: add w8, w8, w9 +; CHECK-NEXT: mov w9, #32767 // =0x7fff +; CHECK-NEXT: add w8, w8, w9 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $s0 +; CHECK-NEXT: ret +entry: + %conv = uitofp i64 %a to bfloat + ret bfloat %conv +} + diff --git a/llvm/test/CodeGen/AArch64/cvt-fp-int-fp.ll b/llvm/test/CodeGen/AArch64/cvt-fp-int-fp.ll index d7bdf2d264c4..049098ab2ae9 100644 --- a/llvm/test/CodeGen/AArch64/cvt-fp-int-fp.ll +++ b/llvm/test/CodeGen/AArch64/cvt-fp-int-fp.ll @@ -73,6 +73,56 @@ entry: ret half %conv1 } +define bfloat @t7(bfloat %x) { +; CHECK-LABEL: t7: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: mov w8, #32767 // =0x7fff +; CHECK-NEXT: lsl w9, w9, #16 +; CHECK-NEXT: fmov s0, w9 +; CHECK-NEXT: fcvtzs w9, s0 +; CHECK-NEXT: scvtf d0, w9 +; CHECK-NEXT: fcvtxn s0, d0 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w10, w9, #16, #1 +; CHECK-NEXT: add w8, w9, w8 +; CHECK-NEXT: add w8, w10, w8 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $s0 +; CHECK-NEXT: ret +entry: + %conv = fptosi bfloat %x to i32 + %conv1 = sitofp i32 %conv to bfloat + ret bfloat %conv1 +} + +define bfloat @t8(bfloat %x) { +; CHECK-LABEL: t8: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: mov w8, #32767 // =0x7fff +; CHECK-NEXT: lsl w9, w9, #16 +; CHECK-NEXT: fmov s0, w9 +; CHECK-NEXT: fcvtzu w9, s0 +; CHECK-NEXT: ucvtf d0, w9 +; CHECK-NEXT: fcvtxn s0, d0 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w10, w9, #16, #1 +; CHECK-NEXT: add w8, w9, w8 +; CHECK-NEXT: add w8, w10, w8 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $s0 +; CHECK-NEXT: ret +entry: + %conv = fptoui bfloat %x to i32 + %conv1 = uitofp i32 %conv to bfloat + ret bfloat %conv1 +} + define double @t1_strict(double %x) #0 { ; CHECK-LABEL: t1_strict: ; CHECK: // %bb.0: // %entry @@ -145,14 +195,68 @@ entry: ret half %conv1 } +define bfloat @t7_strict(bfloat %x) #0 { +; CHECK-LABEL: t7_strict: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: mov w8, #32767 // =0x7fff +; CHECK-NEXT: lsl w9, w9, #16 +; CHECK-NEXT: fmov s0, w9 +; CHECK-NEXT: fcvtzs w9, s0 +; CHECK-NEXT: scvtf d0, w9 +; CHECK-NEXT: fcvtxn s0, d0 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w10, w9, #16, #1 +; CHECK-NEXT: add w8, w9, w8 +; CHECK-NEXT: add w8, w10, w8 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $s0 +; CHECK-NEXT: ret +entry: + %conv = call i32 @llvm.experimental.constrained.fptosi.i32.bf16(bfloat %x, metadata !"fpexcept.strict") #0 + %conv1 = call bfloat @llvm.experimental.constrained.sitofp.i32.bf16(i32 %conv, metadata !"round.tonearest", metadata !"fpexcept.strict") #0 + ret bfloat %conv1 +} + +define bfloat @t8_strict(bfloat %x) #0 { +; CHECK-LABEL: t8_strict: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: mov w8, #32767 // =0x7fff +; CHECK-NEXT: lsl w9, w9, #16 +; CHECK-NEXT: fmov s0, w9 +; CHECK-NEXT: fcvtzu w9, s0 +; CHECK-NEXT: ucvtf d0, w9 +; CHECK-NEXT: fcvtxn s0, d0 +; CHECK-NEXT: fmov w9, s0 +; CHECK-NEXT: ubfx w10, w9, #16, #1 +; CHECK-NEXT: add w8, w9, w8 +; CHECK-NEXT: add w8, w10, w8 +; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: // kill: def $h0 killed $h0 killed $s0 +; CHECK-NEXT: ret +entry: + %conv = call i32 @llvm.experimental.constrained.fptoui.i32.bf16(bfloat %x, metadata !"fpexcept.strict") #0 + %conv1 = call bfloat @llvm.experimental.constrained.uitofp.i32.bf16(i32 %conv, metadata !"round.tonearest", metadata !"fpexcept.strict") #0 + ret bfloat %conv1 +} + attributes #0 = { strictfp } +declare i32 @llvm.experimental.constrained.fptosi.i32.bf16(bfloat, metadata) +declare i32 @llvm.experimental.constrained.fptoui.i32.bf16(bfloat, metadata) declare i32 @llvm.experimental.constrained.fptosi.i32.f16(half, metadata) declare i32 @llvm.experimental.constrained.fptoui.i32.f16(half, metadata) declare i32 @llvm.experimental.constrained.fptosi.i32.f32(float, metadata) declare i32 @llvm.experimental.constrained.fptoui.i32.f32(float, metadata) declare i64 @llvm.experimental.constrained.fptosi.i64.f64(double, metadata) declare i64 @llvm.experimental.constrained.fptoui.i64.f64(double, metadata) +declare bfloat @llvm.experimental.constrained.sitofp.i32.bf16(i32, metadata, metadata) +declare bfloat @llvm.experimental.constrained.uitofp.i32.bf16(i32, metadata, metadata) declare half @llvm.experimental.constrained.sitofp.i32.f16(i32, metadata, metadata) declare half @llvm.experimental.constrained.uitofp.i32.f16(i32, metadata, metadata) declare float @llvm.experimental.constrained.sitofp.i32.f32(i32, metadata, metadata) diff --git a/llvm/test/CodeGen/AArch64/itofp.ll b/llvm/test/CodeGen/AArch64/itofp.ll index be57e1e26a92..708bb43887f8 100644 --- a/llvm/test/CodeGen/AArch64/itofp.ll +++ b/llvm/test/CodeGen/AArch64/itofp.ll @@ -6206,8 +6206,23 @@ entry: define bfloat @stofp_i64_bf16(i64 %a) { ; CHECK-LABEL: stofp_i64_bf16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: scvtf s0, x0 +; CHECK-NEXT: cmp x0, #0 +; CHECK-NEXT: and x11, x0, #0x8000000000000000 ; CHECK-NEXT: mov w8, #32767 // =0x7fff +; CHECK-NEXT: cneg x9, x0, mi +; CHECK-NEXT: lsr x10, x9, #53 +; CHECK-NEXT: cmp x10, #0 +; CHECK-NEXT: and x10, x9, #0xfffffffffffff000 +; CHECK-NEXT: csel x10, x10, x9, ne +; CHECK-NEXT: scvtf d0, x10 +; CHECK-NEXT: cset w10, ne +; CHECK-NEXT: tst x9, #0xfff +; CHECK-NEXT: csel w10, wzr, w10, eq +; CHECK-NEXT: fmov x9, d0 +; CHECK-NEXT: orr x9, x9, x11 +; CHECK-NEXT: orr x9, x9, x10 +; CHECK-NEXT: fmov d0, x9 +; CHECK-NEXT: fcvtxn s0, d0 ; CHECK-NEXT: fmov w9, s0 ; CHECK-NEXT: ubfx w10, w9, #16, #1 ; CHECK-NEXT: add w8, w9, w8 @@ -6224,8 +6239,19 @@ entry: define bfloat @utofp_i64_bf16(i64 %a) { ; CHECK-LABEL: utofp_i64_bf16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ucvtf s0, x0 +; CHECK-NEXT: lsr x9, x0, #53 ; CHECK-NEXT: mov w8, #32767 // =0x7fff +; CHECK-NEXT: cmp x9, #0 +; CHECK-NEXT: and x9, x0, #0xfffffffffffff000 +; CHECK-NEXT: csel x9, x9, x0, ne +; CHECK-NEXT: ucvtf d0, x9 +; CHECK-NEXT: cset w9, ne +; CHECK-NEXT: tst x0, #0xfff +; CHECK-NEXT: csel w9, wzr, w9, eq +; CHECK-NEXT: fmov x10, d0 +; CHECK-NEXT: orr x9, x10, x9 +; CHECK-NEXT: fmov d0, x9 +; CHECK-NEXT: fcvtxn s0, d0 ; CHECK-NEXT: fmov w9, s0 ; CHECK-NEXT: ubfx w10, w9, #16, #1 ; CHECK-NEXT: add w8, w9, w8 @@ -6242,8 +6268,9 @@ entry: define bfloat @stofp_i32_bf16(i32 %a) { ; CHECK-LABEL: stofp_i32_bf16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: scvtf s0, w0 +; CHECK-NEXT: scvtf d0, w0 ; CHECK-NEXT: mov w8, #32767 // =0x7fff +; CHECK-NEXT: fcvtxn s0, d0 ; CHECK-NEXT: fmov w9, s0 ; CHECK-NEXT: ubfx w10, w9, #16, #1 ; CHECK-NEXT: add w8, w9, w8 @@ -6260,8 +6287,9 @@ entry: define bfloat @utofp_i32_bf16(i32 %a) { ; CHECK-LABEL: utofp_i32_bf16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ucvtf s0, w0 +; CHECK-NEXT: ucvtf d0, w0 ; CHECK-NEXT: mov w8, #32767 // =0x7fff +; CHECK-NEXT: fcvtxn s0, d0 ; CHECK-NEXT: fmov w9, s0 ; CHECK-NEXT: ubfx w10, w9, #16, #1 ; CHECK-NEXT: add w8, w9, w8 @@ -6355,22 +6383,52 @@ define <2 x bfloat> @stofp_v2i64_v2bf16(<2 x i64> %a) { ; CHECK-LABEL: stofp_v2i64_v2bf16: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: mov x9, v0.d[1] -; CHECK-NEXT: fmov x10, d0 ; CHECK-NEXT: mov w8, #32767 // =0x7fff -; CHECK-NEXT: scvtf s1, x10 -; CHECK-NEXT: scvtf s0, x9 -; CHECK-NEXT: fmov w10, s1 +; CHECK-NEXT: cmp x9, #0 +; CHECK-NEXT: cneg x10, x9, mi +; CHECK-NEXT: and x9, x9, #0x8000000000000000 +; CHECK-NEXT: lsr x11, x10, #53 +; CHECK-NEXT: and x12, x10, #0xfffffffffffff000 +; CHECK-NEXT: cmp x11, #0 +; CHECK-NEXT: csel x11, x12, x10, ne +; CHECK-NEXT: cset w12, ne +; CHECK-NEXT: tst x10, #0xfff +; CHECK-NEXT: fmov x10, d0 +; CHECK-NEXT: csel w12, wzr, w12, eq +; CHECK-NEXT: scvtf d0, x11 +; CHECK-NEXT: cmp x10, #0 +; CHECK-NEXT: cneg x13, x10, mi +; CHECK-NEXT: and x10, x10, #0x8000000000000000 +; CHECK-NEXT: lsr x14, x13, #53 +; CHECK-NEXT: cmp x14, #0 +; CHECK-NEXT: and x14, x13, #0xfffffffffffff000 +; CHECK-NEXT: csel x11, x14, x13, ne +; CHECK-NEXT: cset w14, ne +; CHECK-NEXT: tst x13, #0xfff +; CHECK-NEXT: scvtf d1, x11 +; CHECK-NEXT: fmov x11, d0 +; CHECK-NEXT: orr x9, x11, x9 +; CHECK-NEXT: csel w11, wzr, w14, eq +; CHECK-NEXT: fmov x13, d1 +; CHECK-NEXT: orr x9, x9, x12 +; CHECK-NEXT: fmov d0, x9 +; CHECK-NEXT: orr x10, x13, x10 +; CHECK-NEXT: orr x10, x10, x11 +; CHECK-NEXT: fcvtxn s0, d0 +; CHECK-NEXT: fmov d1, x10 +; CHECK-NEXT: fcvtxn s1, d1 ; CHECK-NEXT: fmov w9, s0 -; CHECK-NEXT: ubfx w12, w10, #16, #1 ; CHECK-NEXT: ubfx w11, w9, #16, #1 ; CHECK-NEXT: add w9, w9, w8 +; CHECK-NEXT: fmov w10, s1 +; CHECK-NEXT: add w9, w11, w9 +; CHECK-NEXT: lsr w9, w9, #16 +; CHECK-NEXT: ubfx w12, w10, #16, #1 ; CHECK-NEXT: add w8, w10, w8 +; CHECK-NEXT: fmov s1, w9 ; CHECK-NEXT: add w8, w12, w8 -; CHECK-NEXT: add w9, w11, w9 ; CHECK-NEXT: lsr w8, w8, #16 -; CHECK-NEXT: lsr w9, w9, #16 ; CHECK-NEXT: fmov s0, w8 -; CHECK-NEXT: fmov s1, w9 ; CHECK-NEXT: mov v0.h[1], v1.h[0] ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 ; CHECK-NEXT: ret @@ -6383,22 +6441,44 @@ define <2 x bfloat> @utofp_v2i64_v2bf16(<2 x i64> %a) { ; CHECK-LABEL: utofp_v2i64_v2bf16: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: mov x9, v0.d[1] -; CHECK-NEXT: fmov x10, d0 +; CHECK-NEXT: fmov x11, d0 ; CHECK-NEXT: mov w8, #32767 // =0x7fff -; CHECK-NEXT: ucvtf s1, x10 -; CHECK-NEXT: ucvtf s0, x9 -; CHECK-NEXT: fmov w10, s1 +; CHECK-NEXT: lsr x10, x9, #53 +; CHECK-NEXT: and x12, x9, #0xfffffffffffff000 +; CHECK-NEXT: cmp x10, #0 +; CHECK-NEXT: lsr x10, x11, #53 +; CHECK-NEXT: csel x12, x12, x9, ne +; CHECK-NEXT: cset w13, ne +; CHECK-NEXT: tst x9, #0xfff +; CHECK-NEXT: csel w9, wzr, w13, eq +; CHECK-NEXT: cmp x10, #0 +; CHECK-NEXT: and x10, x11, #0xfffffffffffff000 +; CHECK-NEXT: csel x10, x10, x11, ne +; CHECK-NEXT: ucvtf d0, x12 +; CHECK-NEXT: ucvtf d1, x10 +; CHECK-NEXT: cset w10, ne +; CHECK-NEXT: tst x11, #0xfff +; CHECK-NEXT: csel w10, wzr, w10, eq +; CHECK-NEXT: fmov x11, d0 +; CHECK-NEXT: fmov x12, d1 +; CHECK-NEXT: orr x9, x11, x9 +; CHECK-NEXT: orr x10, x12, x10 +; CHECK-NEXT: fmov d0, x9 +; CHECK-NEXT: fmov d1, x10 +; CHECK-NEXT: fcvtxn s0, d0 +; CHECK-NEXT: fcvtxn s1, d1 ; CHECK-NEXT: fmov w9, s0 -; CHECK-NEXT: ubfx w12, w10, #16, #1 +; CHECK-NEXT: fmov w10, s1 ; CHECK-NEXT: ubfx w11, w9, #16, #1 ; CHECK-NEXT: add w9, w9, w8 +; CHECK-NEXT: ubfx w12, w10, #16, #1 ; CHECK-NEXT: add w8, w10, w8 -; CHECK-NEXT: add w8, w12, w8 ; CHECK-NEXT: add w9, w11, w9 -; CHECK-NEXT: lsr w8, w8, #16 +; CHECK-NEXT: add w8, w12, w8 ; CHECK-NEXT: lsr w9, w9, #16 -; CHECK-NEXT: fmov s0, w8 +; CHECK-NEXT: lsr w8, w8, #16 ; CHECK-NEXT: fmov s1, w9 +; CHECK-NEXT: fmov s0, w8 ; CHECK-NEXT: mov v0.h[1], v1.h[0] ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 ; CHECK-NEXT: ret -- GitLab From eceb24c439b7a0ae5545ae2f6279e6f36e0b627e Mon Sep 17 00:00:00 2001 From: Visoiu Mistrih Francis <890283+francisvm@users.noreply.github.com> Date: Tue, 5 Mar 2024 22:41:56 -0800 Subject: [PATCH 251/929] [RISCV] Hoist immediate addresses from loads/stores (#83644) In case of loads/stores from an immediate address, avoid rematerializing the constant for every block and allow consthoist to hoist it to the entry block. --- .../Target/RISCV/RISCVTargetTransformInfo.cpp | 8 +++ .../ConstantHoisting/RISCV/immediates.ll | 58 +++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index 504970c0a9ac..ecd373649e2c 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -165,6 +165,14 @@ InstructionCost RISCVTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx, // split up large offsets in GEP into better parts than ConstantHoisting // can. return TTI::TCC_Free; + case Instruction::Store: + // If the address is a constant, use the materialization cost. + if (Idx == 1) + return getIntImmCost(Imm, Ty, CostKind); + return TTI::TCC_Free; + case Instruction::Load: + // If the address is a constant, use the materialization cost. + return getIntImmCost(Imm, Ty, CostKind); case Instruction::And: // zext.h if (Imm == UINT64_C(0xffff) && ST->hasStdExtZbb()) diff --git a/llvm/test/Transforms/ConstantHoisting/RISCV/immediates.ll b/llvm/test/Transforms/ConstantHoisting/RISCV/immediates.ll index 131ef673a61f..0a477d5b6484 100644 --- a/llvm/test/Transforms/ConstantHoisting/RISCV/immediates.ll +++ b/llvm/test/Transforms/ConstantHoisting/RISCV/immediates.ll @@ -150,3 +150,61 @@ define i64 @test16(i64 %a) nounwind { ret i64 %2 } +; Check that we hoist the absolute address of the stores to the entry block. +define void @test17(ptr %s, i32 %size) nounwind { +; CHECK-LABEL: test17 +; CHECK: %const = bitcast i32 -1073741792 to i32 +; CHECK: %0 = inttoptr i32 %const to ptr +; CHECK: store i32 20, ptr %0 +; CHECK: %1 = inttoptr i32 %const to ptr +; CHECK: store i32 10, ptr %1 +entry: + %cond = icmp eq i32 %size, 0 + br i1 %cond, label %if.true, label %exit +if.true: + store i32 20, ptr inttoptr (i32 -1073741792 to ptr) + br label %exit +exit: + store i32 10, ptr inttoptr (i32 -1073741792 to ptr) + ret void +} + +; Check that we hoist the absolute address of the loads to the entry block. +define i32 @test18(ptr %s, i32 %size) nounwind { +; CHECK-LABEL: test18 +; CHECK: %const = bitcast i32 -1073741792 to i32 +; CHECK: %0 = inttoptr i32 %const to ptr +; CHECK: %1 = load i32, ptr %0 +; CHECK: %2 = inttoptr i32 %const to ptr +; CHECK: %3 = load i32, ptr %2 +entry: + %cond = icmp eq i32 %size, 0 + br i1 %cond, label %if.true, label %if.false +if.true: + %0 = load i32, ptr inttoptr (i32 -1073741792 to ptr) + br label %return +if.false: + %1 = load i32, ptr inttoptr (i32 -1073741792 to ptr) + br label %return +return: + %val = phi i32 [%0, %if.true], [%1, %if.false] + ret i32 %val +} + + +; For addresses between [0, 2048), we can use ld/sd xN, address(zero), so don't +; hoist. +define void @test19(ptr %s, i32 %size) nounwind { +; CHECK-LABEL: test19 +; CHECK: store i32 20, ptr inttoptr (i32 2044 to ptr) +; CHECK: store i32 10, ptr inttoptr (i32 2044 to ptr) +entry: + %cond = icmp eq i32 %size, 0 + br i1 %cond, label %if.true, label %exit +if.true: + store i32 20, ptr inttoptr (i32 2044 to ptr) + br label %exit +exit: + store i32 10, ptr inttoptr (i32 2044 to ptr) + ret void +} -- GitLab From 228c737b3c2fdc22a431b65df3b9ec528db5e67f Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Sun, 18 Feb 2024 20:39:05 +0800 Subject: [PATCH 252/929] [RISCV] Add tests for vsetvli/vsetvlimax with different SEW/LMUL --- .../RISCV/riscv-vsetvli-knownbits.ll | 694 +++++++++++++++-- .../RISCV/riscv-vsetvlimax-knownbits.ll | 706 ++++++++++++++++++ 2 files changed, 1346 insertions(+), 54 deletions(-) create mode 100644 llvm/test/Transforms/InstCombine/RISCV/riscv-vsetvlimax-knownbits.ll diff --git a/llvm/test/Transforms/InstCombine/RISCV/riscv-vsetvli-knownbits.ll b/llvm/test/Transforms/InstCombine/RISCV/riscv-vsetvli-knownbits.ll index 51f78688b13e..e254d2a71b7f 100644 --- a/llvm/test/Transforms/InstCombine/RISCV/riscv-vsetvli-knownbits.ll +++ b/llvm/test/Transforms/InstCombine/RISCV/riscv-vsetvli-knownbits.ll @@ -3,10 +3,8 @@ declare i32 @llvm.riscv.vsetvli.i32(i32, i32, i32) declare i64 @llvm.riscv.vsetvli.i64(i64, i64, i64) -declare i32 @llvm.riscv.vsetvlimax.i32(i32, i32) -declare i64 @llvm.riscv.vsetvlimax.i64(i64, i64) -define i32 @vsetvli_i32() nounwind { +define i32 @vsetvli_i32() nounwind #0 { ; CHECK-LABEL: @vsetvli_i32( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 1, i32 1, i32 1) @@ -18,7 +16,7 @@ entry: ret i32 %1 } -define i64 @vsetvli_sext_i64() nounwind { +define i64 @vsetvli_sext_i64() nounwind #0 { ; CHECK-LABEL: @vsetvli_sext_i64( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 1, i64 1, i64 1) @@ -31,7 +29,7 @@ entry: ret i64 %2 } -define i64 @vsetvli_zext_i64() nounwind { +define i64 @vsetvli_zext_i64() nounwind #0 { ; CHECK-LABEL: @vsetvli_zext_i64( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 1, i64 1, i64 1) @@ -44,7 +42,29 @@ entry: ret i64 %2 } -define i32 @vsetvli_and17_i32() nounwind { +define signext i32 @vsetvl_sext() nounwind #0 { +; CHECK-LABEL: @vsetvl_sext( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 1, i64 1, i64 1) +; CHECK-NEXT: [[B:%.*]] = trunc i64 [[A]] to i32 +; CHECK-NEXT: ret i32 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 1, i64 1, i64 1) + %b = trunc i64 %a to i32 + ret i32 %b +} + +define zeroext i32 @vsetvl_zext() nounwind #0 { +; CHECK-LABEL: @vsetvl_zext( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 1, i64 1, i64 1) +; CHECK-NEXT: [[B:%.*]] = trunc i64 [[A]] to i32 +; CHECK-NEXT: ret i32 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 1, i64 1, i64 1) + %b = trunc i64 %a to i32 + ret i32 %b +} + +define i32 @vsetvli_and17_i32() nounwind #0 { ; CHECK-LABEL: @vsetvli_and17_i32( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvli.i32(i32 1, i32 1, i32 1) @@ -56,7 +76,7 @@ entry: ret i32 %1 } -define i64 @vsetvli_and17_i64() nounwind { +define i64 @vsetvli_and17_i64() nounwind #0 { ; CHECK-LABEL: @vsetvli_and17_i64( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 1, i64 1, i64 1) @@ -68,64 +88,630 @@ entry: ret i64 %1 } -define i32 @vsetvlimax_i32() nounwind { -; CHECK-LABEL: @vsetvlimax_i32( -; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 1, i32 1) -; CHECK-NEXT: ret i32 [[TMP0]] +define i64 @vsetvl_e8m1_and14bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e8m1_and14bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 0, i64 0) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 16383 +; CHECK-NEXT: ret i64 [[B]] ; -entry: - %0 = call i32 @llvm.riscv.vsetvlimax.i32(i32 1, i32 1) - %1 = and i32 %0, 2147483647 - ret i32 %1 + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 0, i64 0) + %b = and i64 %a, 16383 + ret i64 %b } -define i64 @vsetvlimax_sext_i64() nounwind { -; CHECK-LABEL: @vsetvlimax_sext_i64( -; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 1) -; CHECK-NEXT: ret i64 [[TMP0]] +define i64 @vsetvl_e8m1_and13bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e8m1_and13bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 0, i64 0) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 +; CHECK-NEXT: ret i64 [[B]] ; -entry: - %0 = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 1) - %1 = trunc i64 %0 to i32 - %2 = sext i32 %1 to i64 - ret i64 %2 + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 0, i64 0) + %b = and i64 %a, 8191 + ret i64 %b } -define i64 @vsetvlimax_zext_i64() nounwind { -; CHECK-LABEL: @vsetvlimax_zext_i64( -; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 1) -; CHECK-NEXT: ret i64 [[TMP0]] +define i64 @vsetvl_e8m1_constant_avl() nounwind #0 { +; CHECK-LABEL: @vsetvl_e8m1_constant_avl( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 1, i64 0, i64 0) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 1 +; CHECK-NEXT: ret i64 [[B]] ; -entry: - %0 = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 1) - %1 = trunc i64 %0 to i32 - %2 = zext i32 %1 to i64 - ret i64 %2 + %a = call i64 @llvm.riscv.vsetvli(i64 1, i64 0, i64 0) + %b = and i64 %a, 1 + ret i64 %b } -define i32 @vsetvlimax_and17_i32() nounwind { -; CHECK-LABEL: @vsetvlimax_and17_i32( -; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 1, i32 1) -; CHECK-NEXT: ret i32 [[TMP0]] +define i64 @vsetvl_e8m2_and15bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e8m2_and15bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 0, i64 1) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 32767 +; CHECK-NEXT: ret i64 [[B]] ; -entry: - %0 = call i32 @llvm.riscv.vsetvlimax.i32(i32 1, i32 1) - %1 = and i32 %0, 131071 - ret i32 %1 + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 0, i64 1) + %b = and i64 %a, 32767 + ret i64 %b } -define i64 @vsetvlimax_and17_i64() nounwind { -; CHECK-LABEL: @vsetvlimax_and17_i64( -; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 1) -; CHECK-NEXT: ret i64 [[TMP0]] +define i64 @vsetvl_e8m2_and14bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e8m2_and14bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 0, i64 1) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 16383 +; CHECK-NEXT: ret i64 [[B]] ; -entry: - %0 = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 1) - %1 = and i64 %0, 131071 - ret i64 %1 + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 0, i64 1) + %b = and i64 %a, 16383 + ret i64 %b +} + +define i64 @vsetvl_e8m4_and16bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e8m4_and16bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 0, i64 2) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 65535 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 0, i64 2) + %b = and i64 %a, 65535 + ret i64 %b +} + +define i64 @vsetvl_e8m4_and15bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e8m4_and15bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 0, i64 2) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 32767 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 0, i64 2) + %b = and i64 %a, 32767 + ret i64 %b +} + +define i64 @vsetvl_e8m8_and17bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e8m8_and17bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 0, i64 3) +; CHECK-NEXT: ret i64 [[A]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 0, i64 3) + %b = and i64 %a, 131071 + ret i64 %b +} + +define i64 @vsetvl_e8m8_and16bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e8m8_and16bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 0, i64 3) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 65535 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 0, i64 3) + %b = and i64 %a, 65535 + ret i64 %b +} + +define i64 @vsetvl_e8mf2_and11bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e8mf2_and11bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 0, i64 5) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 0, i64 5) + %b = and i64 %a, 2047 + ret i64 %b +} + +define i64 @vsetvl_e8mf2_and10bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e8mf2_and10bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 0, i64 5) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 1023 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 0, i64 5) + %b = and i64 %a, 1023 + ret i64 %b +} + +define i64 @vsetvl_e8mf4_and12bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e8mf4_and12bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 0, i64 6) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 0, i64 6) + %b = and i64 %a, 4095 + ret i64 %b +} + +define i64 @vsetvl_e8mf4_and11bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e8mf4_and11bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 0, i64 6) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 0, i64 6) + %b = and i64 %a, 2047 + ret i64 %b +} + +define i64 @vsetvl_e8mf8_and13bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e8mf8_and13bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 0, i64 7) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 0, i64 7) + %b = and i64 %a, 8191 + ret i64 %b +} + +define i64 @vsetvl_e8mf8_and12bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e8mf8_and12bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 0, i64 7) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 0, i64 7) + %b = and i64 %a, 4095 + ret i64 %b +} + +define i64 @vsetvl_e16m1_and13bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e16m1_and13bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 1, i64 0) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 1, i64 0) + %b = and i64 %a, 8191 + ret i64 %b +} + +define i64 @vsetvl_e16m1_and12bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e16m1_and12bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 1, i64 0) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 1, i64 0) + %b = and i64 %a, 4095 + ret i64 %b +} + +define i64 @vsetvl_e16m2_and14bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e16m2_and14bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 1, i64 1) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 16383 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 1, i64 1) + %b = and i64 %a, 16383 + ret i64 %b +} + +define i64 @vsetvl_e16m2_and13bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e16m2_and13bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 1, i64 1) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 1, i64 1) + %b = and i64 %a, 8191 + ret i64 %b +} + +define i64 @vsetvl_e16m4_and15bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e16m4_and15bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 1, i64 2) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 32767 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 1, i64 2) + %b = and i64 %a, 32767 + ret i64 %b +} + +define i64 @vsetvl_e16m4_and14bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e16m4_and14bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 1, i64 2) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 16383 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 1, i64 2) + %b = and i64 %a, 16383 + ret i64 %b +} + +define i64 @vsetvl_e16m8_and16bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e16m8_and16bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 1, i64 3) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 65535 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 1, i64 3) + %b = and i64 %a, 65535 + ret i64 %b +} + +define i64 @vsetvl_e16m8_and15bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e16m8_and15bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 1, i64 3) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 32767 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 1, i64 3) + %b = and i64 %a, 32767 + ret i64 %b +} + +define i64 @vsetvl_e16mf2_and10bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e16mf2_and10bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 1, i64 5) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 1023 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 1, i64 5) + %b = and i64 %a, 1023 + ret i64 %b +} + +define i64 @vsetvl_e16mf2_and9bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e16mf2_and9bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 1, i64 5) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 511 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 1, i64 5) + %b = and i64 %a, 511 + ret i64 %b +} + +define i64 @vsetvl_e16mf4_and11bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e16mf4_and11bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 1, i64 6) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 1, i64 6) + %b = and i64 %a, 2047 + ret i64 %b +} + +define i64 @vsetvl_e16mf4_and10bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e16mf4_and10bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 1, i64 6) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 1023 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 1, i64 6) + %b = and i64 %a, 1023 + ret i64 %b +} + +define i64 @vsetvl_e16mf8_and12bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e16mf8_and12bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 1, i64 7) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 1, i64 7) + %b = and i64 %a, 4095 + ret i64 %b +} + +define i64 @vsetvl_e16mf8_and11bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e16mf8_and11bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 1, i64 7) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 1, i64 7) + %b = and i64 %a, 2047 + ret i64 %b +} + +define i64 @vsetvl_e32m1_and12bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e32m1_and12bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 2, i64 0) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 2, i64 0) + %b = and i64 %a, 4095 + ret i64 %b +} + +define i64 @vsetvl_e32m1_and11bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e32m1_and11bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 2, i64 0) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 2, i64 0) + %b = and i64 %a, 2047 + ret i64 %b +} + +define i64 @vsetvl_e32m2_and13bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e32m2_and13bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 2, i64 1) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 2, i64 1) + %b = and i64 %a, 8191 + ret i64 %b +} + +define i64 @vsetvl_e32m2_and12bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e32m2_and12bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 2, i64 1) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 2, i64 1) + %b = and i64 %a, 4095 + ret i64 %b +} + +define i64 @vsetvl_e32m4_and14bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e32m4_and14bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 2, i64 2) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 16383 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 2, i64 2) + %b = and i64 %a, 16383 + ret i64 %b +} + +define i64 @vsetvl_e32m4_and13bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e32m4_and13bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 2, i64 2) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 2, i64 2) + %b = and i64 %a, 8191 + ret i64 %b +} + +define i64 @vsetvl_e32m8_and15bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e32m8_and15bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 2, i64 3) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 32767 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 2, i64 3) + %b = and i64 %a, 32767 + ret i64 %b +} + +define i64 @vsetvl_e32m8_and14bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e32m8_and14bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 2, i64 3) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 16383 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 2, i64 3) + %b = and i64 %a, 16383 + ret i64 %b +} + +define i64 @vsetvl_e32mf2_and9bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e32mf2_and9bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 2, i64 5) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 511 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 2, i64 5) + %b = and i64 %a, 511 + ret i64 %b +} + +define i64 @vsetvl_e32mf2_and8bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e32mf2_and8bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 2, i64 5) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 255 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 2, i64 5) + %b = and i64 %a, 255 + ret i64 %b +} + +define i64 @vsetvl_e32mf4_and10bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e32mf4_and10bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 2, i64 6) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 1023 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 2, i64 6) + %b = and i64 %a, 1023 + ret i64 %b +} + +define i64 @vsetvl_e32mf4_and9bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e32mf4_and9bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 2, i64 6) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 511 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 2, i64 6) + %b = and i64 %a, 511 + ret i64 %b } + +define i64 @vsetvl_e32mf8_and11bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e32mf8_and11bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 2, i64 7) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 2, i64 7) + %b = and i64 %a, 2047 + ret i64 %b +} + +define i64 @vsetvl_e32mf8_and10bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e32mf8_and10bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 2, i64 7) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 1023 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 2, i64 7) + %b = and i64 %a, 1023 + ret i64 %b +} + +define i64 @vsetvl_e64m1_and11bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e64m1_and11bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 3, i64 0) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 3, i64 0) + %b = and i64 %a, 2047 + ret i64 %b +} + +define i64 @vsetvl_e64m1_and10bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e64m1_and10bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 3, i64 0) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 1023 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 3, i64 0) + %b = and i64 %a, 1023 + ret i64 %b +} + +define i64 @vsetvl_e64m2_and12bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e64m2_and12bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 3, i64 1) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 3, i64 1) + %b = and i64 %a, 4095 + ret i64 %b +} + +define i64 @vsetvl_e64m2_and11bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e64m2_and11bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 3, i64 1) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 3, i64 1) + %b = and i64 %a, 2047 + ret i64 %b +} + +define i64 @vsetvl_e64m4_and13bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e64m4_and13bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 3, i64 2) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 3, i64 2) + %b = and i64 %a, 8191 + ret i64 %b +} + +define i64 @vsetvl_e64m4_and12bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e64m4_and12bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 3, i64 2) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 3, i64 2) + %b = and i64 %a, 4095 + ret i64 %b +} + +define i64 @vsetvl_e64m8_and14bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e64m8_and14bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 3, i64 3) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 16383 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 3, i64 3) + %b = and i64 %a, 16383 + ret i64 %b +} + +define i64 @vsetvl_e64m8_and13bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e64m8_and13bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 3, i64 3) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 3, i64 3) + %b = and i64 %a, 8191 + ret i64 %b +} + +define i64 @vsetvl_e64mf2_and8bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e64mf2_and8bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 3, i64 5) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 255 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 3, i64 5) + %b = and i64 %a, 255 + ret i64 %b +} + +define i64 @vsetvl_e64mf2_and7bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e64mf2_and7bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 3, i64 5) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 127 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 3, i64 5) + %b = and i64 %a, 127 + ret i64 %b +} + +define i64 @vsetvl_e64mf4_and9bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e64mf4_and9bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 3, i64 6) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 511 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 3, i64 6) + %b = and i64 %a, 511 + ret i64 %b +} + +define i64 @vsetvl_e64mf4_and8bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e64mf4_and8bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 3, i64 6) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 255 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 3, i64 6) + %b = and i64 %a, 255 + ret i64 %b +} + +define i64 @vsetvl_e64mf8_and10bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e64mf8_and10bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 3, i64 7) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 1023 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 3, i64 7) + %b = and i64 %a, 1023 + ret i64 %b +} + +define i64 @vsetvl_e64mf8_and9bits(i64 %avl) nounwind #0 { +; CHECK-LABEL: @vsetvl_e64mf8_and9bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 3, i64 7) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 511 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 3, i64 7) + %b = and i64 %a, 511 + ret i64 %b +} + +attributes #0 = { vscale_range(2,1024) } diff --git a/llvm/test/Transforms/InstCombine/RISCV/riscv-vsetvlimax-knownbits.ll b/llvm/test/Transforms/InstCombine/RISCV/riscv-vsetvlimax-knownbits.ll new file mode 100644 index 000000000000..72033b3af09f --- /dev/null +++ b/llvm/test/Transforms/InstCombine/RISCV/riscv-vsetvlimax-knownbits.ll @@ -0,0 +1,706 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -passes=instcombine -S | FileCheck %s + +declare i32 @llvm.riscv.vsetvlimax.i32(i32, i32) +declare i64 @llvm.riscv.vsetvlimax.i64(i64, i64) + +define i32 @vsetvlimax_i32() nounwind #0 { +; CHECK-LABEL: @vsetvlimax_i32( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 1, i32 1) +; CHECK-NEXT: ret i32 [[TMP0]] +; +entry: + %0 = call i32 @llvm.riscv.vsetvlimax.i32(i32 1, i32 1) + %1 = and i32 %0, 2147483647 + ret i32 %1 +} + +define i64 @vsetvlimax_sext_i64() nounwind #0 { +; CHECK-LABEL: @vsetvlimax_sext_i64( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 1) +; CHECK-NEXT: ret i64 [[TMP0]] +; +entry: + %0 = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 1) + %1 = trunc i64 %0 to i32 + %2 = sext i32 %1 to i64 + ret i64 %2 +} + +define i64 @vsetvlimax_zext_i64() nounwind #0 { +; CHECK-LABEL: @vsetvlimax_zext_i64( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 1) +; CHECK-NEXT: ret i64 [[TMP0]] +; +entry: + %0 = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 1) + %1 = trunc i64 %0 to i32 + %2 = zext i32 %1 to i64 + ret i64 %2 +} + +define signext i32 @vsetvlmax_sext() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_sext( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 1) +; CHECK-NEXT: [[B:%.*]] = trunc i64 [[A]] to i32 +; CHECK-NEXT: ret i32 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 1, i64 1) + %b = trunc i64 %a to i32 + ret i32 %b +} + +define zeroext i32 @vsetvlmax_zext() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_zext( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 1) +; CHECK-NEXT: [[B:%.*]] = trunc i64 [[A]] to i32 +; CHECK-NEXT: ret i32 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 1, i64 1) + %b = trunc i64 %a to i32 + ret i32 %b +} + +define i32 @vsetvlimax_and17_i32() nounwind #0 { +; CHECK-LABEL: @vsetvlimax_and17_i32( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.riscv.vsetvlimax.i32(i32 1, i32 1) +; CHECK-NEXT: ret i32 [[TMP0]] +; +entry: + %0 = call i32 @llvm.riscv.vsetvlimax.i32(i32 1, i32 1) + %1 = and i32 %0, 131071 + ret i32 %1 +} + +define i64 @vsetvlimax_and17_i64() nounwind #0 { +; CHECK-LABEL: @vsetvlimax_and17_i64( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 1) +; CHECK-NEXT: ret i64 [[TMP0]] +; +entry: + %0 = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 1) + %1 = and i64 %0, 131071 + ret i64 %1 +} + +define i64 @vsetvlmax_e8m1_and14bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e8m1_and14bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 0) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 16383 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 0, i64 0) + %b = and i64 %a, 16383 + ret i64 %b +} + +define i64 @vsetvlmax_e8m1_and13bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e8m1_and13bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 0) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 0, i64 0) + %b = and i64 %a, 8191 + ret i64 %b +} + +define i64 @vsetvlmax_e8m2_and15bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e8m2_and15bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 1) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 32767 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 0, i64 1) + %b = and i64 %a, 32767 + ret i64 %b +} + +define i64 @vsetvlmax_e8m2_and14bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e8m2_and14bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 1) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 16383 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 0, i64 1) + %b = and i64 %a, 16383 + ret i64 %b +} + +define i64 @vsetvlmax_e8m4_and16bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e8m4_and16bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 2) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 65535 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 0, i64 2) + %b = and i64 %a, 65535 + ret i64 %b +} + +define i64 @vsetvlmax_e8m4_and15bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e8m4_and15bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 2) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 32767 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 0, i64 2) + %b = and i64 %a, 32767 + ret i64 %b +} + +define i64 @vsetvlmax_e8m8_and17bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e8m8_and17bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 3) +; CHECK-NEXT: ret i64 [[A]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 0, i64 3) + %b = and i64 %a, 131071 + ret i64 %b +} + +define i64 @vsetvlmax_e8m8_and16bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e8m8_and16bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 3) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 65535 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 0, i64 3) + %b = and i64 %a, 65535 + ret i64 %b +} + +define i64 @vsetvlmax_e8mf2_and11bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e8mf2_and11bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 5) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 0, i64 5) + %b = and i64 %a, 2047 + ret i64 %b +} + +define i64 @vsetvlmax_e8mf2_and10bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e8mf2_and10bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 5) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 1023 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 0, i64 5) + %b = and i64 %a, 1023 + ret i64 %b +} + +define i64 @vsetvlmax_e8mf4_and12bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e8mf4_and12bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 6) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 0, i64 6) + %b = and i64 %a, 4095 + ret i64 %b +} + +define i64 @vsetvlmax_e8mf4_and11bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e8mf4_and11bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 6) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 0, i64 6) + %b = and i64 %a, 2047 + ret i64 %b +} + +define i64 @vsetvlmax_e8mf8_and13bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e8mf8_and13bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 7) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 0, i64 7) + %b = and i64 %a, 8191 + ret i64 %b +} + +define i64 @vsetvlmax_e8mf8_and12bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e8mf8_and12bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 7) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 0, i64 7) + %b = and i64 %a, 4095 + ret i64 %b +} + +define i64 @vsetvlmax_e16m1_and13bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e16m1_and13bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 0) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 1, i64 0) + %b = and i64 %a, 8191 + ret i64 %b +} + +define i64 @vsetvlmax_e16m1_and12bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e16m1_and12bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 0) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 1, i64 0) + %b = and i64 %a, 4095 + ret i64 %b +} + +define i64 @vsetvlmax_e16m2_and14bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e16m2_and14bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 1) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 16383 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 1, i64 1) + %b = and i64 %a, 16383 + ret i64 %b +} + +define i64 @vsetvlmax_e16m2_and13bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e16m2_and13bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 1) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 1, i64 1) + %b = and i64 %a, 8191 + ret i64 %b +} + +define i64 @vsetvlmax_e16m4_and15bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e16m4_and15bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 2) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 32767 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 1, i64 2) + %b = and i64 %a, 32767 + ret i64 %b +} + +define i64 @vsetvlmax_e16m4_and14bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e16m4_and14bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 2) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 16383 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 1, i64 2) + %b = and i64 %a, 16383 + ret i64 %b +} + +define i64 @vsetvlmax_e16m8_and16bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e16m8_and16bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 3) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 65535 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 1, i64 3) + %b = and i64 %a, 65535 + ret i64 %b +} + +define i64 @vsetvlmax_e16m8_and15bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e16m8_and15bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 3) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 32767 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 1, i64 3) + %b = and i64 %a, 32767 + ret i64 %b +} + +define i64 @vsetvlmax_e16mf2_and10bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e16mf2_and10bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 5) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 1023 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 1, i64 5) + %b = and i64 %a, 1023 + ret i64 %b +} + +define i64 @vsetvlmax_e16mf2_and9bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e16mf2_and9bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 5) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 511 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 1, i64 5) + %b = and i64 %a, 511 + ret i64 %b +} + +define i64 @vsetvlmax_e16mf4_and11bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e16mf4_and11bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 6) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 1, i64 6) + %b = and i64 %a, 2047 + ret i64 %b +} + +define i64 @vsetvlmax_e16mf4_and10bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e16mf4_and10bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 6) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 1023 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 1, i64 6) + %b = and i64 %a, 1023 + ret i64 %b +} + +define i64 @vsetvlmax_e16mf8_and12bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e16mf8_and12bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 7) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 1, i64 7) + %b = and i64 %a, 4095 + ret i64 %b +} + +define i64 @vsetvlmax_e16mf8_and11bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e16mf8_and11bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 7) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 1, i64 7) + %b = and i64 %a, 2047 + ret i64 %b +} + +define i64 @vsetvlmax_e32m1_and12bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e32m1_and12bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 0) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 2, i64 0) + %b = and i64 %a, 4095 + ret i64 %b +} + +define i64 @vsetvlmax_e32m1_and11bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e32m1_and11bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 0) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 2, i64 0) + %b = and i64 %a, 2047 + ret i64 %b +} + +define i64 @vsetvlmax_e32m2_and13bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e32m2_and13bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 1) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 2, i64 1) + %b = and i64 %a, 8191 + ret i64 %b +} + +define i64 @vsetvlmax_e32m2_and12bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e32m2_and12bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 1) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 2, i64 1) + %b = and i64 %a, 4095 + ret i64 %b +} + +define i64 @vsetvlmax_e32m4_and14bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e32m4_and14bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 2) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 16383 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 2, i64 2) + %b = and i64 %a, 16383 + ret i64 %b +} + +define i64 @vsetvlmax_e32m4_and13bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e32m4_and13bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 2) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 2, i64 2) + %b = and i64 %a, 8191 + ret i64 %b +} + +define i64 @vsetvlmax_e32m8_and15bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e32m8_and15bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 3) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 32767 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 2, i64 3) + %b = and i64 %a, 32767 + ret i64 %b +} + +define i64 @vsetvlmax_e32m8_and14bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e32m8_and14bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 3) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 16383 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 2, i64 3) + %b = and i64 %a, 16383 + ret i64 %b +} + +define i64 @vsetvlmax_e32mf2_and9bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e32mf2_and9bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 5) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 511 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 2, i64 5) + %b = and i64 %a, 511 + ret i64 %b +} + +define i64 @vsetvlmax_e32mf2_and8bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e32mf2_and8bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 5) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 255 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 2, i64 5) + %b = and i64 %a, 255 + ret i64 %b +} + +define i64 @vsetvlmax_e32mf4_and10bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e32mf4_and10bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 6) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 1023 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 2, i64 6) + %b = and i64 %a, 1023 + ret i64 %b +} + +define i64 @vsetvlmax_e32mf4_and9bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e32mf4_and9bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 6) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 511 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 2, i64 6) + %b = and i64 %a, 511 + ret i64 %b +} + +define i64 @vsetvlmax_e32mf8_and11bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e32mf8_and11bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 7) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 2, i64 7) + %b = and i64 %a, 2047 + ret i64 %b +} + +define i64 @vsetvlmax_e32mf8_and10bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e32mf8_and10bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 7) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 1023 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 2, i64 7) + %b = and i64 %a, 1023 + ret i64 %b +} + +define i64 @vsetvlmax_e64m1_and11bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e64m1_and11bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 0) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 3, i64 0) + %b = and i64 %a, 2047 + ret i64 %b +} + +define i64 @vsetvlmax_e64m1_and10bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e64m1_and10bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 0) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 1023 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 3, i64 0) + %b = and i64 %a, 1023 + ret i64 %b +} + +define i64 @vsetvlmax_e64m2_and12bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e64m2_and12bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 1) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 3, i64 1) + %b = and i64 %a, 4095 + ret i64 %b +} + +define i64 @vsetvlmax_e64m2_and11bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e64m2_and11bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 1) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 3, i64 1) + %b = and i64 %a, 2047 + ret i64 %b +} + +define i64 @vsetvlmax_e64m4_and13bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e64m4_and13bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 2) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 3, i64 2) + %b = and i64 %a, 8191 + ret i64 %b +} + +define i64 @vsetvlmax_e64m4_and12bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e64m4_and12bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 2) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 3, i64 2) + %b = and i64 %a, 4095 + ret i64 %b +} + +define i64 @vsetvlmax_e64m8_and14bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e64m8_and14bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 3) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 16383 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 3, i64 3) + %b = and i64 %a, 16383 + ret i64 %b +} + +define i64 @vsetvlmax_e64m8_and13bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e64m8_and13bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 3) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 3, i64 3) + %b = and i64 %a, 8191 + ret i64 %b +} + +define i64 @vsetvlmax_e64mf2_and8bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e64mf2_and8bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 5) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 255 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 3, i64 5) + %b = and i64 %a, 255 + ret i64 %b +} + +define i64 @vsetvlmax_e64mf2_and7bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e64mf2_and7bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 5) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 127 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 3, i64 5) + %b = and i64 %a, 127 + ret i64 %b +} + +define i64 @vsetvlmax_e64mf4_and9bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e64mf4_and9bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 6) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 511 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 3, i64 6) + %b = and i64 %a, 511 + ret i64 %b +} + +define i64 @vsetvlmax_e64mf4_and8bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e64mf4_and8bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 6) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 255 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 3, i64 6) + %b = and i64 %a, 255 + ret i64 %b +} + +define i64 @vsetvlmax_e64mf8_and10bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e64mf8_and10bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 7) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 1023 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 3, i64 7) + %b = and i64 %a, 1023 + ret i64 %b +} + +define i64 @vsetvlmax_e64mf8_and9bits() nounwind #0 { +; CHECK-LABEL: @vsetvlmax_e64mf8_and9bits( +; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 7) +; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 511 +; CHECK-NEXT: ret i64 [[B]] +; + %a = call i64 @llvm.riscv.vsetvlimax(i64 3, i64 7) + %b = and i64 %a, 511 + ret i64 %b +} + +attributes #0 = { vscale_range(2,1024) } -- GitLab From 60822637bf007cbaf7401a6ec25cdf2ea7b7edbd Mon Sep 17 00:00:00 2001 From: Sameer Sahasrabuddhe Date: Wed, 6 Mar 2024 12:17:37 +0530 Subject: [PATCH 253/929] Restore "Implement convergence control in MIR using SelectionDAG (#71785)" This restores commit c7fdd8c11e54585dc9d15d63de9742067e0506b9. Previously reverted in f010b1bef4dda2c7082cbb41dbabf1f149cce306. LLVM function calls carry convergence control tokens as operand bundles, where the tokens themselves are produced by convergence control intrinsics. This patch implements convergence control tokens in MIR as follows: 1. Introduce target-independent ISD opcodes and MIR opcodes for convergence control intrinsics. 2. Model token values as untyped virtual registers in MIR. The change also introduces an additional ISD opcode CONVERGENCECTRL_GLUE and a corresponding machine opcode with the same spelling. This glues the convergence control token to SDNodes that represent calls to intrinsics. The glued token is later translated to an implicit argument in the MIR. The lowering of calls to user-defined functions is target-specific. On AMDGPU, the convergence control operand bundle at a non-intrinsic call is translated to an explicit argument to the SI_CALL_ISEL instruction. Post-selection adjustment converts this explicit argument to an implicit argument on the SI_CALL instruction. --- .../llvm/ADT/GenericConvergenceVerifier.h | 6 ++ .../llvm/CodeGen/FunctionLoweringInfo.h | 10 +- llvm/include/llvm/CodeGen/ISDOpcodes.h | 9 ++ .../llvm/CodeGen/MachineConvergenceVerifier.h | 28 +++++ llvm/include/llvm/CodeGen/SelectionDAGISel.h | 4 + llvm/include/llvm/CodeGen/TargetLowering.h | 6 ++ .../llvm/IR/GenericConvergenceVerifierImpl.h | 28 ++--- llvm/include/llvm/Support/TargetOpcodes.def | 5 + llvm/include/llvm/Target/Target.td | 19 ++++ .../include/llvm/Target/TargetSelectionDAG.td | 10 ++ llvm/lib/CodeGen/CMakeLists.txt | 1 + .../CodeGen/MachineConvergenceVerifier.cpp | 100 ++++++++++++++++++ llvm/lib/CodeGen/MachineVerifier.cpp | 30 ++++++ .../SelectionDAG/FunctionLoweringInfo.cpp | 10 ++ .../lib/CodeGen/SelectionDAG/InstrEmitter.cpp | 44 +++++++- .../SelectionDAG/SelectionDAGBuilder.cpp | 50 ++++++++- .../SelectionDAG/SelectionDAGBuilder.h | 1 + .../SelectionDAG/SelectionDAGDumper.cpp | 10 ++ .../CodeGen/SelectionDAG/SelectionDAGISel.cpp | 24 +++++ llvm/lib/CodeGen/ValueTypes.cpp | 2 + llvm/lib/IR/ConvergenceVerifier.cpp | 33 +++++- llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 27 ++++- .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 12 ++- llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h | 1 + llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 24 ++++- llvm/lib/Target/AMDGPU/SIInstructions.td | 8 +- .../test/CodeGen/AMDGPU/convergence-tokens.ll | 83 +++++++++++++++ .../CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll | 18 ++++ .../kernel-vgpr-spill-mubuf-with-voffset.ll | 1 + .../AMDGPU/need-fp-from-vgpr-spills.ll | 18 ++-- .../AMDGPU/no-source-locations-in-prologue.ll | 1 + .../AMDGPU/sgpr-spills-split-regalloc.ll | 15 +-- .../CodeGen/AMDGPU/stacksave_stackrestore.ll | 78 +++++++------- llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll | 26 ++--- .../CodeGen/AMDGPU/vgpr_constant_to_sgpr.ll | 1 + .../AMDGPU/whole-wave-register-spill.ll | 1 + .../test/CodeGen/AMDGPU/wwm-reserved-spill.ll | 2 + llvm/test/CodeGen/AMDGPU/wwm-reserved.ll | 4 + llvm/test/CodeGen/PowerPC/fmf-propagation.ll | 12 +-- .../convergencectrl/AMDGPU/basic.mir | 37 +++++++ .../convergencectrl/AMDGPU/cycles.mir | 52 +++++++++ .../convergencectrl/AMDGPU/lit.local.cfg | 2 + .../convergencectrl/AMDGPU/mixed2.mir | 15 +++ .../convergencectrl/AMDGPU/region-nesting.mir | 24 +++++ .../builtins/match-table-replacerreg.td | 2 +- .../match-table-imms.td | 30 +++--- .../match-table-intrinsics.td | 2 +- .../match-table-patfrag-root.td | 2 +- .../match-table-variadics.td | 4 +- .../GlobalISelCombinerEmitter/match-table.td | 62 +++++------ 50 files changed, 833 insertions(+), 161 deletions(-) create mode 100644 llvm/include/llvm/CodeGen/MachineConvergenceVerifier.h create mode 100644 llvm/lib/CodeGen/MachineConvergenceVerifier.cpp create mode 100644 llvm/test/CodeGen/AMDGPU/convergence-tokens.ll create mode 100644 llvm/test/MachineVerifier/convergencectrl/AMDGPU/basic.mir create mode 100644 llvm/test/MachineVerifier/convergencectrl/AMDGPU/cycles.mir create mode 100644 llvm/test/MachineVerifier/convergencectrl/AMDGPU/lit.local.cfg create mode 100644 llvm/test/MachineVerifier/convergencectrl/AMDGPU/mixed2.mir create mode 100644 llvm/test/MachineVerifier/convergencectrl/AMDGPU/region-nesting.mir diff --git a/llvm/include/llvm/ADT/GenericConvergenceVerifier.h b/llvm/include/llvm/ADT/GenericConvergenceVerifier.h index 0810a0701322..dc9495ce57bf 100644 --- a/llvm/include/llvm/ADT/GenericConvergenceVerifier.h +++ b/llvm/include/llvm/ADT/GenericConvergenceVerifier.h @@ -60,6 +60,10 @@ private: NoConvergence } ConvergenceKind = NoConvergence; + /// The control token operation performed by a convergence control Intrinsic + /// in LLVM IR, or by a CONVERGENCECTRL* instruction in MIR + enum ConvOpKind { CONV_ANCHOR, CONV_ENTRY, CONV_LOOP, CONV_NONE }; + // Cache token uses found so far. Note that we track the unique definitions // and not the token values. DenseMap Tokens; @@ -68,6 +72,8 @@ private: static bool isInsideConvergentFunction(const InstructionT &I); static bool isConvergent(const InstructionT &I); + static ConvOpKind getConvOp(const InstructionT &I); + void checkConvergenceTokenProduced(const InstructionT &I); const InstructionT *findAndCheckConvergenceTokenUsed(const InstructionT &I); void reportFailure(const Twine &Message, ArrayRef Values); diff --git a/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h b/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h index cde7247aeb15..31af3014afe4 100644 --- a/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h +++ b/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h @@ -215,15 +215,7 @@ public: Register CreateRegs(Type *Ty, bool isDivergent = false); - Register InitializeRegForValue(const Value *V) { - // Tokens never live in vregs. - if (V->getType()->isTokenTy()) - return 0; - Register &R = ValueMap[V]; - assert(R == 0 && "Already initialized this value register!"); - assert(VirtReg2Value.empty()); - return R = CreateRegs(V); - } + Register InitializeRegForValue(const Value *V); /// GetLiveOutRegInfo - Gets LiveOutInfo for a register, returning NULL if the /// register is a PHI destination and the PHI's LiveOutInfo is not valid. diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h index ef0fec270a43..49d51a27e3c0 100644 --- a/llvm/include/llvm/CodeGen/ISDOpcodes.h +++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -1386,6 +1386,15 @@ enum NodeType { #define BEGIN_REGISTER_VP_SDNODE(VPSDID, ...) VPSDID, #include "llvm/IR/VPIntrinsics.def" + // The `llvm.experimental.convergence.*` intrinsics. + CONVERGENCECTRL_ANCHOR, + CONVERGENCECTRL_ENTRY, + CONVERGENCECTRL_LOOP, + // This does not correspond to any convergence control intrinsic. It is used + // to glue a convergence control token to a convergent operation in the DAG, + // which is later translated to an implicit use in the MIR. + CONVERGENCECTRL_GLUE, + /// BUILTIN_OP_END - This must be the last enum value in this list. /// The target-specific pre-isel opcode values start here. BUILTIN_OP_END diff --git a/llvm/include/llvm/CodeGen/MachineConvergenceVerifier.h b/llvm/include/llvm/CodeGen/MachineConvergenceVerifier.h new file mode 100644 index 000000000000..86aa8ce1766b --- /dev/null +++ b/llvm/include/llvm/CodeGen/MachineConvergenceVerifier.h @@ -0,0 +1,28 @@ +//===- MachineConvergenceVerifier.h - Verify convergencectrl ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// +/// This file declares the MIR specialization of the GenericConvergenceVerifier +/// template. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_MACHINECONVERGENCEVERIFIER_H +#define LLVM_CODEGEN_MACHINECONVERGENCEVERIFIER_H + +#include "llvm/ADT/GenericConvergenceVerifier.h" +#include "llvm/CodeGen/MachineSSAContext.h" + +namespace llvm { + +using MachineConvergenceVerifier = + GenericConvergenceVerifier; + +} // namespace llvm + +#endif // LLVM_CODEGEN_MACHINECONVERGENCEVERIFIER_H diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h index dbd9b391f4a4..837f8bf7263e 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h @@ -459,6 +459,10 @@ private: void Select_ARITH_FENCE(SDNode *N); void Select_MEMBARRIER(SDNode *N); + void Select_CONVERGENCECTRL_ANCHOR(SDNode *N); + void Select_CONVERGENCECTRL_ENTRY(SDNode *N); + void Select_CONVERGENCECTRL_LOOP(SDNode *N); + void pushStackMapLiveVariable(SmallVectorImpl &Ops, SDValue Operand, SDLoc DL); void Select_STACKMAP(SDNode *N); diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 7be5b5e8dd2f..2f164a460db8 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -4438,6 +4438,7 @@ public: SmallVector Ins; SmallVector InVals; const ConstantInt *CFIType = nullptr; + SDValue ConvergenceControlToken; CallLoweringInfo(SelectionDAG &DAG) : RetSExt(false), RetZExt(false), IsVarArg(false), IsInReg(false), @@ -4571,6 +4572,11 @@ public: return *this; } + CallLoweringInfo &setConvergenceControlToken(SDValue Token) { + ConvergenceControlToken = Token; + return *this; + } + ArgListTy &getArgs() { return Args; } diff --git a/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h b/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h index f6eb5066d553..a3ebde709ae6 100644 --- a/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h +++ b/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h @@ -61,12 +61,11 @@ void GenericConvergenceVerifier::visit(const BlockT &BB) { template void GenericConvergenceVerifier::visit(const InstructionT &I) { - auto ID = ContextT::getIntrinsicID(I); - auto *TokenDef = findAndCheckConvergenceTokenUsed(I); - bool IsCtrlIntrinsic = true; + ConvOpKind ConvOp = getConvOp(I); - switch (ID) { - case Intrinsic::experimental_convergence_entry: + auto *TokenDef = findAndCheckConvergenceTokenUsed(I); + switch (ConvOp) { + case CONV_ENTRY: Check(isInsideConvergentFunction(I), "Entry intrinsic can occur only in a convergent function.", {Context.print(&I)}); @@ -78,13 +77,13 @@ void GenericConvergenceVerifier::visit(const InstructionT &I) { "same basic block.", {Context.print(&I)}); LLVM_FALLTHROUGH; - case Intrinsic::experimental_convergence_anchor: + case CONV_ANCHOR: Check(!TokenDef, "Entry or anchor intrinsic cannot have a convergencectrl token " "operand.", {Context.print(&I)}); break; - case Intrinsic::experimental_convergence_loop: + case CONV_LOOP: Check(TokenDef, "Loop intrinsic must have a convergencectrl token operand.", {Context.print(&I)}); Check(!SeenFirstConvOp, @@ -93,14 +92,16 @@ void GenericConvergenceVerifier::visit(const InstructionT &I) { {Context.print(&I)}); break; default: - IsCtrlIntrinsic = false; break; } + if (ConvOp != CONV_NONE) + checkConvergenceTokenProduced(I); + if (isConvergent(I)) SeenFirstConvOp = true; - if (TokenDef || IsCtrlIntrinsic) { + if (TokenDef || ConvOp != CONV_NONE) { Check(isConvergent(I), "Convergence control token can only be used in a convergent call.", {Context.print(&I)}); @@ -143,6 +144,10 @@ void GenericConvergenceVerifier::verify(const DominatorTreeT &DT) { auto checkToken = [&](const InstructionT *Token, const InstructionT *User, SmallVectorImpl &LiveTokens) { + Check(DT.dominates(Token->getParent(), User->getParent()), + "Convergence control token must dominate all its uses.", + {Context.print(Token), Context.print(User)}); + Check(llvm::is_contained(LiveTokens, Token), "Convergence region is not well-nested.", {Context.print(Token), Context.print(User)}); @@ -161,8 +166,7 @@ void GenericConvergenceVerifier::verify(const DominatorTreeT &DT) { return; } - Check(ContextT::getIntrinsicID(*User) == - Intrinsic::experimental_convergence_loop, + Check(getConvOp(*User) == CONV_LOOP, "Convergence token used by an instruction other than " "llvm.experimental.convergence.loop in a cycle that does " "not contain the token's definition.", @@ -199,7 +203,7 @@ void GenericConvergenceVerifier::verify(const DominatorTreeT &DT) { for (auto &I : *BB) { if (auto *Token = Tokens.lookup(&I)) checkToken(Token, &I, LiveTokens); - if (isConvergenceControlIntrinsic(ContextT::getIntrinsicID(I))) + if (getConvOp(I) != CONV_NONE) LiveTokens.push_back(&I); } diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def index 42cb854d9505..6aded2ceebe1 100644 --- a/llvm/include/llvm/Support/TargetOpcodes.def +++ b/llvm/include/llvm/Support/TargetOpcodes.def @@ -225,6 +225,11 @@ HANDLE_TARGET_OPCODE(MEMBARRIER) // using. HANDLE_TARGET_OPCODE(JUMP_TABLE_DEBUG_INFO) +HANDLE_TARGET_OPCODE(CONVERGENCECTRL_ENTRY) +HANDLE_TARGET_OPCODE(CONVERGENCECTRL_ANCHOR) +HANDLE_TARGET_OPCODE(CONVERGENCECTRL_LOOP) +HANDLE_TARGET_OPCODE(CONVERGENCECTRL_GLUE) + /// The following generic opcodes are not supposed to appear after ISel. /// This is something we might want to relax, but for now, this is convenient /// to produce diagnostics. diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td index 0d97a47190b1..0577c58f8da2 100644 --- a/llvm/include/llvm/Target/Target.td +++ b/llvm/include/llvm/Target/Target.td @@ -1483,6 +1483,25 @@ def JUMP_TABLE_DEBUG_INFO : StandardPseudoInstruction { let isMeta = true; } +let hasSideEffects = false, isMeta = true, isConvergent = true in { +def CONVERGENCECTRL_ANCHOR : StandardPseudoInstruction { + let OutOperandList = (outs unknown:$dst); + let InOperandList = (ins); +} +def CONVERGENCECTRL_ENTRY : StandardPseudoInstruction { + let OutOperandList = (outs unknown:$dst); + let InOperandList = (ins); +} +def CONVERGENCECTRL_LOOP : StandardPseudoInstruction { + let OutOperandList = (outs unknown:$dst); + let InOperandList = (ins unknown:$src); +} +def CONVERGENCECTRL_GLUE : StandardPseudoInstruction { + let OutOperandList = (outs); + let InOperandList = (ins unknown:$src); +} +} + // Generic opcodes used in GlobalISel. include "llvm/Target/GenericOpcodes.td" diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td index d84c2d30e447..d7bf8c35ee10 100644 --- a/llvm/include/llvm/Target/TargetSelectionDAG.td +++ b/llvm/include/llvm/Target/TargetSelectionDAG.td @@ -789,6 +789,16 @@ def assertsext : SDNode<"ISD::AssertSext", SDT_assert>; def assertzext : SDNode<"ISD::AssertZext", SDT_assert>; def assertalign : SDNode<"ISD::AssertAlign", SDT_assert>; +def convergencectrl_anchor : SDNode<"ISD::CONVERGENCECTRL_ANCHOR", + SDTypeProfile<1, 0, [SDTCisVT<0,untyped>]>>; +def convergencectrl_entry : SDNode<"ISD::CONVERGENCECTRL_ENTRY", + SDTypeProfile<1, 0, [SDTCisVT<0,untyped>]>>; +def convergencectrl_loop : SDNode<"ISD::CONVERGENCECTRL_LOOP", + SDTypeProfile<1, 1, + [SDTCisVT<0,untyped>, SDTCisVT<1,untyped>]>>; +def convergencectrl_glue : SDNode<"ISD::CONVERGENCECTRL_GLUE", + SDTypeProfile<0, 1, [SDTCisVT<0, untyped>]>>; + //===----------------------------------------------------------------------===// // Selection DAG Condition Codes diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt index e02c1d6417e0..2c24de60edd4 100644 --- a/llvm/lib/CodeGen/CMakeLists.txt +++ b/llvm/lib/CodeGen/CMakeLists.txt @@ -110,6 +110,7 @@ add_llvm_component_library(LLVMCodeGen MachineBranchProbabilityInfo.cpp MachineCFGPrinter.cpp MachineCombiner.cpp + MachineConvergenceVerifier.cpp MachineCopyPropagation.cpp MachineCSE.cpp MachineCheckDebugify.cpp diff --git a/llvm/lib/CodeGen/MachineConvergenceVerifier.cpp b/llvm/lib/CodeGen/MachineConvergenceVerifier.cpp new file mode 100644 index 000000000000..8ec85bdf48d4 --- /dev/null +++ b/llvm/lib/CodeGen/MachineConvergenceVerifier.cpp @@ -0,0 +1,100 @@ +//===- MachineConvergenceVerifier.cpp - Verify convergencectrl ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/MachineConvergenceVerifier.h" +#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" +#include "llvm/CodeGen/MachineDominators.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/MachineSSAContext.h" +#include "llvm/IR/GenericConvergenceVerifierImpl.h" + +using namespace llvm; + +template <> +auto GenericConvergenceVerifier::getConvOp( + const MachineInstr &MI) -> ConvOpKind { + switch (MI.getOpcode()) { + default: + return CONV_NONE; + case TargetOpcode::CONVERGENCECTRL_ENTRY: + return CONV_ENTRY; + case TargetOpcode::CONVERGENCECTRL_ANCHOR: + return CONV_ANCHOR; + case TargetOpcode::CONVERGENCECTRL_LOOP: + return CONV_LOOP; + } +} + +template <> +void GenericConvergenceVerifier< + MachineSSAContext>::checkConvergenceTokenProduced(const MachineInstr &MI) { + Check(!MI.hasImplicitDef(), + "Convergence control tokens are defined explicitly.", + {Context.print(&MI)}); + const MachineOperand &Def = MI.getOperand(0); + const MachineRegisterInfo &MRI = Context.getFunction()->getRegInfo(); + Check(MRI.getUniqueVRegDef(Def.getReg()), + "Convergence control tokens must have unique definitions.", + {Context.print(&MI)}); +} + +template <> +const MachineInstr * +GenericConvergenceVerifier::findAndCheckConvergenceTokenUsed( + const MachineInstr &MI) { + const MachineRegisterInfo &MRI = Context.getFunction()->getRegInfo(); + const MachineInstr *TokenDef = nullptr; + + for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) { + const MachineOperand &MO = MI.getOperand(I); + if (!MO.isReg() || !MO.isUse()) + continue; + Register OpReg = MO.getReg(); + if (!OpReg.isVirtual()) + continue; + + const MachineInstr *Def = MRI.getUniqueVRegDef(OpReg); + if (!Def) + continue; + if (getConvOp(*Def) == CONV_NONE) + continue; + + CheckOrNull( + MI.isConvergent(), + "Convergence control tokens can only be used by convergent operations.", + {Context.print(OpReg), Context.print(&MI)}); + + CheckOrNull(!TokenDef, + "An operation can use at most one convergence control token.", + {Context.print(OpReg), Context.print(&MI)}); + + TokenDef = Def; + } + + if (TokenDef) + Tokens[&MI] = TokenDef; + + return TokenDef; +} + +template <> +bool GenericConvergenceVerifier::isInsideConvergentFunction( + const MachineInstr &MI) { + // The class MachineFunction does not have any property to indicate whether it + // is convergent. Trivially return true so that the check always passes. + return true; +} + +template <> +bool GenericConvergenceVerifier::isConvergent( + const MachineInstr &MI) { + return MI.isConvergent(); +} + +template class llvm::GenericConvergenceVerifier; diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp index 2632b5b9feac..1d0757c5d7f5 100644 --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -39,6 +39,8 @@ #include "llvm/CodeGen/LiveStacks.h" #include "llvm/CodeGen/LiveVariables.h" #include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineConvergenceVerifier.h" +#include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" @@ -220,6 +222,11 @@ namespace { LiveStacks *LiveStks = nullptr; SlotIndexes *Indexes = nullptr; + // This is calculated only when trying to verify convergence control tokens. + // Similar to the LLVM IR verifier, we calculate this locally instead of + // relying on the pass manager. + MachineDomTree DT; + void visitMachineFunctionBefore(); void visitMachineBasicBlockBefore(const MachineBasicBlock *MBB); void visitMachineBundleBefore(const MachineInstr *MI); @@ -2955,7 +2962,30 @@ void MachineVerifier::checkPHIOps(const MachineBasicBlock &MBB) { } } +static void +verifyConvergenceControl(const MachineFunction &MF, MachineDomTree &DT, + std::function FailureCB) { + MachineConvergenceVerifier CV; + CV.initialize(&errs(), FailureCB, MF); + + for (const auto &MBB : MF) { + CV.visit(MBB); + for (const auto &MI : MBB.instrs()) + CV.visit(MI); + } + + if (CV.sawTokens()) { + DT.recalculate(const_cast(MF)); + CV.verify(DT); + } +} + void MachineVerifier::visitMachineFunctionAfter() { + auto FailureCB = [this](const Twine &Message) { + report(Message.str().c_str(), MF); + }; + verifyConvergenceControl(*MF, DT, FailureCB); + calcRegsPassed(); for (const MachineBasicBlock &MBB : *MF) diff --git a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp index 4172fbc96d1e..e01cd8cbf925 100644 --- a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp @@ -395,6 +395,16 @@ Register FunctionLoweringInfo::CreateRegs(const Value *V) { !TLI->requiresUniformRegister(*MF, V)); } +Register FunctionLoweringInfo::InitializeRegForValue(const Value *V) { + // Tokens live in vregs only when used for convergence control. + if (V->getType()->isTokenTy() && !isa(V)) + return 0; + Register &R = ValueMap[V]; + assert(R == Register() && "Already initialized this value register!"); + assert(VirtReg2Value.empty()); + return R = CreateRegs(V); +} + /// GetLiveOutRegInfo - Gets LiveOutInfo for a register, returning NULL if the /// register is a PHI destination and the PHI's LiveOutInfo is not valid. If /// the register's LiveOutInfo is for a smaller bit width, it is extended to diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp index 032cff416cda..54409cbf91f1 100644 --- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -285,6 +285,30 @@ Register InstrEmitter::getVR(SDValue Op, return I->second; } +static bool isConvergenceCtrlMachineOp(SDValue Op) { + if (Op->isMachineOpcode()) { + switch (Op->getMachineOpcode()) { + case TargetOpcode::CONVERGENCECTRL_ANCHOR: + case TargetOpcode::CONVERGENCECTRL_ENTRY: + case TargetOpcode::CONVERGENCECTRL_LOOP: + case TargetOpcode::CONVERGENCECTRL_GLUE: + return true; + } + return false; + } + + // We can reach here when CopyFromReg is encountered. But rather than making a + // special case for that, we just make sure we don't reach here in some + // surprising way. + switch (Op->getOpcode()) { + case ISD::CONVERGENCECTRL_ANCHOR: + case ISD::CONVERGENCECTRL_ENTRY: + case ISD::CONVERGENCECTRL_LOOP: + case ISD::CONVERGENCECTRL_GLUE: + llvm_unreachable("Convergence control should have been selected by now."); + } + return false; +} /// AddRegisterOperand - Add the specified register as an operand to the /// specified machine instr. Insert register copies if the register is @@ -346,9 +370,12 @@ InstrEmitter::AddRegisterOperand(MachineInstrBuilder &MIB, // multiple uses. // Tied operands are never killed, so we need to check that. And that // means we need to determine the index of the operand. - bool isKill = Op.hasOneUse() && - Op.getNode()->getOpcode() != ISD::CopyFromReg && - !IsDebug && + // Don't kill convergence control tokens. Initially they are only used in glue + // nodes, and the InstrEmitter later adds implicit uses on the users of the + // glue node. This can sometimes make it seem like there is only one use, + // which is the glue node itself. + bool isKill = Op.hasOneUse() && !isConvergenceCtrlMachineOp(Op) && + Op.getNode()->getOpcode() != ISD::CopyFromReg && !IsDebug && !(IsClone || IsCloned); if (isKill) { unsigned Idx = MIB->getNumOperands(); @@ -1191,6 +1218,17 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, } } + if (SDNode *GluedNode = Node->getGluedNode()) { + // FIXME: Possibly iterate over multiple glue nodes? + if (GluedNode->getOpcode() == + ~(unsigned)TargetOpcode::CONVERGENCECTRL_GLUE) { + Register VReg = getVR(GluedNode->getOperand(0), VRBaseMap); + MachineOperand MO = MachineOperand::CreateReg(VReg, /*isDef=*/false, + /*isImp=*/true); + MIB->addOperand(MO); + } + } + // Run post-isel target hook to adjust this instruction if needed. if (II.hasPostISelHook()) TLI->AdjustInstrPostInstrSelection(*MIB, Node); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index d6ab0cf6df87..22e57d0d99e9 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -5226,6 +5226,17 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, // Create the node. SDValue Result; + + if (auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl)) { + auto *Token = Bundle->Inputs[0].get(); + SDValue ConvControlToken = getValue(Token); + assert(Ops.back().getValueType() != MVT::Glue && + "Did not expected another glue node here."); + ConvControlToken = + DAG.getNode(ISD::CONVERGENCECTRL_GLUE, {}, MVT::Glue, ConvControlToken); + Ops.push_back(ConvControlToken); + } + // In some cases, custom collection of operands from CallInst I may be needed. TLI.CollectTargetIntrinsicOperands(I, Ops, DAG); if (IsTgtIntrinsic) { @@ -6226,6 +6237,27 @@ bool SelectionDAGBuilder::visitEntryValueDbgValue( return true; } +/// Lower the call to the specified intrinsic function. +void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I, + unsigned Intrinsic) { + SDLoc sdl = getCurSDLoc(); + switch (Intrinsic) { + case Intrinsic::experimental_convergence_anchor: + setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ANCHOR, sdl, MVT::Untyped)); + break; + case Intrinsic::experimental_convergence_entry: + setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ENTRY, sdl, MVT::Untyped)); + break; + case Intrinsic::experimental_convergence_loop: { + auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl); + auto *Token = Bundle->Inputs[0].get(); + setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_LOOP, sdl, MVT::Untyped, + getValue(Token))); + break; + } + } +} + /// Lower the call to the specified intrinsic function. void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { @@ -7885,6 +7917,10 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, case Intrinsic::experimental_vector_deinterleave2: visitVectorDeinterleave(I); return; + case Intrinsic::experimental_convergence_anchor: + case Intrinsic::experimental_convergence_entry: + case Intrinsic::experimental_convergence_loop: + visitConvergenceControl(I, Intrinsic); } } @@ -8561,6 +8597,14 @@ void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee, } } + SDValue ConvControlToken; + if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) { + auto *Token = Bundle->Inputs[0].get(); + ConvControlToken = getValue(Token); + } else { + ConvControlToken = DAG.getUNDEF(MVT::Untyped); + } + TargetLowering::CallLoweringInfo CLI(DAG); CLI.setDebugLoc(getCurSDLoc()) .setChain(getRoot()) @@ -8569,7 +8613,8 @@ void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee, .setConvergent(CB.isConvergent()) .setIsPreallocated( CB.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0) - .setCFIType(CFIType); + .setCFIType(CFIType) + .setConvergenceControlToken(ConvControlToken); std::pair Result = lowerInvokable(CLI, EHPadBB); if (Result.first.getNode()) { @@ -9121,7 +9166,8 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) { assert(!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated, - LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi}) && + LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi, + LLVMContext::OB_convergencectrl}) && "Cannot lower calls with arbitrary operand bundles!"); SDValue Callee = getValue(I.getCalledOperand()); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h index 2084de473b80..dcf46e0563ff 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -623,6 +623,7 @@ private: void visitIntrinsicCall(const CallInst &I, unsigned Intrinsic); void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic); void visitConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI); + void visitConvergenceControl(const CallInst &I, unsigned Intrinsic); void visitVPLoad(const VPIntrinsic &VPIntrin, EVT VT, const SmallVectorImpl &OpValues); void visitVPStore(const VPIntrinsic &VPIntrin, diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp index 18ca17e53dac..fa71adc8da3f 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -165,6 +165,9 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { if (cast(this)->isOpaque()) return "OpaqueTargetConstant"; return "TargetConstant"; + + // clang-format off + case ISD::TargetConstantFP: return "TargetConstantFP"; case ISD::TargetGlobalAddress: return "TargetGlobalAddress"; case ISD::TargetGlobalTLSAddress: return "TargetGlobalTLSAddress"; @@ -449,6 +452,11 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::SET_FPMODE: return "set_fpmode"; case ISD::RESET_FPMODE: return "reset_fpmode"; + // Convergence control instructions + case ISD::CONVERGENCECTRL_ANCHOR: return "convergencectrl_anchor"; + case ISD::CONVERGENCECTRL_ENTRY: return "convergencectrl_entry"; + case ISD::CONVERGENCECTRL_LOOP: return "convergencectrl_loop"; + // Bit manipulation case ISD::ABS: return "abs"; case ISD::BITREVERSE: return "bitreverse"; @@ -464,6 +472,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::INIT_TRAMPOLINE: return "init_trampoline"; case ISD::ADJUST_TRAMPOLINE: return "adjust_trampoline"; + // clang-format on + case ISD::CONDCODE: switch (cast(this)->get()) { default: llvm_unreachable("Unknown setcc condition!"); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 9b5ab4267b80..1c14e4da8e9d 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -2370,6 +2370,21 @@ void SelectionDAGISel::Select_MEMBARRIER(SDNode *N) { N->getOperand(0)); } +void SelectionDAGISel::Select_CONVERGENCECTRL_ANCHOR(SDNode *N) { + CurDAG->SelectNodeTo(N, TargetOpcode::CONVERGENCECTRL_ANCHOR, + N->getValueType(0)); +} + +void SelectionDAGISel::Select_CONVERGENCECTRL_ENTRY(SDNode *N) { + CurDAG->SelectNodeTo(N, TargetOpcode::CONVERGENCECTRL_ENTRY, + N->getValueType(0)); +} + +void SelectionDAGISel::Select_CONVERGENCECTRL_LOOP(SDNode *N) { + CurDAG->SelectNodeTo(N, TargetOpcode::CONVERGENCECTRL_LOOP, + N->getValueType(0), N->getOperand(0)); +} + void SelectionDAGISel::pushStackMapLiveVariable(SmallVectorImpl &Ops, SDValue OpVal, SDLoc DL) { SDNode *OpNode = OpVal.getNode(); @@ -3117,6 +3132,15 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, case ISD::JUMP_TABLE_DEBUG_INFO: Select_JUMP_TABLE_DEBUG_INFO(NodeToMatch); return; + case ISD::CONVERGENCECTRL_ANCHOR: + Select_CONVERGENCECTRL_ANCHOR(NodeToMatch); + return; + case ISD::CONVERGENCECTRL_ENTRY: + Select_CONVERGENCECTRL_ENTRY(NodeToMatch); + return; + case ISD::CONVERGENCECTRL_LOOP: + Select_CONVERGENCECTRL_LOOP(NodeToMatch); + return; } assert(!NodeToMatch->isMachineOpcode() && "Node already selected!"); diff --git a/llvm/lib/CodeGen/ValueTypes.cpp b/llvm/lib/CodeGen/ValueTypes.cpp index 5dcabdb6a9e0..58db686ec7d5 100644 --- a/llvm/lib/CodeGen/ValueTypes.cpp +++ b/llvm/lib/CodeGen/ValueTypes.cpp @@ -629,6 +629,8 @@ EVT EVT::getEVT(Type *Ty, bool HandleUnknown){ switch (Ty->getTypeID()) { default: return MVT::getVT(Ty, HandleUnknown); + case Type::TokenTyID: + return MVT::Untyped; case Type::IntegerTyID: return getIntegerVT(Ty->getContext(), cast(Ty)->getBitWidth()); case Type::FixedVectorTyID: diff --git a/llvm/lib/IR/ConvergenceVerifier.cpp b/llvm/lib/IR/ConvergenceVerifier.cpp index 336c202b6f94..e73aeaade5f7 100644 --- a/llvm/lib/IR/ConvergenceVerifier.cpp +++ b/llvm/lib/IR/ConvergenceVerifier.cpp @@ -14,6 +14,30 @@ using namespace llvm; +template <> +auto GenericConvergenceVerifier::getConvOp(const Instruction &I) + -> ConvOpKind { + const auto *CB = dyn_cast(&I); + if (!CB) + return CONV_NONE; + switch (CB->getIntrinsicID()) { + default: + return CONV_NONE; + case Intrinsic::experimental_convergence_anchor: + return CONV_ANCHOR; + case Intrinsic::experimental_convergence_entry: + return CONV_ENTRY; + case Intrinsic::experimental_convergence_loop: + return CONV_LOOP; + } +} + +template <> +void GenericConvergenceVerifier::checkConvergenceTokenProduced( + const Instruction &I) { + return; +} + template <> const Instruction * GenericConvergenceVerifier::findAndCheckConvergenceTokenUsed( @@ -38,11 +62,10 @@ GenericConvergenceVerifier::findAndCheckConvergenceTokenUsed( auto *Token = Bundle->Inputs[0].get(); auto *Def = dyn_cast(Token); - CheckOrNull( - Def && isConvergenceControlIntrinsic(SSAContext::getIntrinsicID(*Def)), - "Convergence control tokens can only be produced by calls to the " - "convergence control intrinsics.", - {Context.print(Token), Context.print(&I)}); + CheckOrNull(Def && getConvOp(*Def) != CONV_NONE, + "Convergence control tokens can only be produced by calls to the " + "convergence control intrinsics.", + {Context.print(Token), Context.print(&I)}); if (Def) Tokens[&I] = Def; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp index 2e5b02fbe856..d1af74e6e0d3 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp @@ -2688,7 +2688,18 @@ void AMDGPUDAGToDAGISel::SelectINTRINSIC_W_CHAIN(SDNode *N) { void AMDGPUDAGToDAGISel::SelectINTRINSIC_WO_CHAIN(SDNode *N) { unsigned IntrID = N->getConstantOperandVal(0); - unsigned Opcode; + unsigned Opcode = AMDGPU::INSTRUCTION_LIST_END; + SDNode *ConvGlueNode = N->getGluedNode(); + if (ConvGlueNode) { + // FIXME: Possibly iterate over multiple glue nodes? + assert(ConvGlueNode->getOpcode() == ISD::CONVERGENCECTRL_GLUE); + ConvGlueNode = ConvGlueNode->getOperand(0).getNode(); + ConvGlueNode = + CurDAG->getMachineNode(TargetOpcode::CONVERGENCECTRL_GLUE, {}, + MVT::Glue, SDValue(ConvGlueNode, 0)); + } else { + ConvGlueNode = nullptr; + } switch (IntrID) { case Intrinsic::amdgcn_wqm: Opcode = AMDGPU::WQM; @@ -2720,11 +2731,19 @@ void AMDGPUDAGToDAGISel::SelectINTRINSIC_WO_CHAIN(SDNode *N) { break; default: SelectCode(N); - return; + break; } - SDValue Src = N->getOperand(1); - CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), {Src}); + if (Opcode != AMDGPU::INSTRUCTION_LIST_END) { + SDValue Src = N->getOperand(1); + CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), {Src}); + } + + if (ConvGlueNode) { + SmallVector NewOps(N->op_begin(), N->op_end()); + NewOps.push_back(SDValue(ConvGlueNode, 0)); + CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), NewOps); + } } void AMDGPUDAGToDAGISel::SelectINTRINSIC_VOID(SDNode *N) { diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index 76e843455bab..7467c25a84d6 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -245,6 +245,13 @@ static cl::opt LateCFGStructurize( cl::location(AMDGPUTargetMachine::EnableLateStructurizeCFG), cl::Hidden); +// Disable structurizer-based control-flow lowering in order to test convergence +// control tokens. This should eventually be replaced by the wave-transform. +static cl::opt DisableStructurizer( + "amdgpu-disable-structurizer", + cl::desc("Disable structurizer for experiments; produces unusable code"), + cl::location(AMDGPUTargetMachine::DisableStructurizer), cl::ReallyHidden); + // Enable lib calls simplifications static cl::opt EnableLibCallSimplify( "amdgpu-simplify-libcall", @@ -591,6 +598,7 @@ AMDGPUTargetMachine::AMDGPUTargetMachine(const Target &T, const Triple &TT, bool AMDGPUTargetMachine::EnableLateStructurizeCFG = false; bool AMDGPUTargetMachine::EnableFunctionCalls = false; bool AMDGPUTargetMachine::EnableLowerModuleLDS = true; +bool AMDGPUTargetMachine::DisableStructurizer = false; AMDGPUTargetMachine::~AMDGPUTargetMachine() = default; @@ -1186,7 +1194,7 @@ bool GCNPassConfig::addPreISel() { // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit // regions formed by them. addPass(&AMDGPUUnifyDivergentExitNodesID); - if (!LateCFGStructurize) { + if (!LateCFGStructurize && !DisableStructurizer) { if (EnableStructurizerWorkarounds) { addPass(createFixIrreduciblePass()); addPass(createUnifyLoopExitsPass()); @@ -1194,7 +1202,7 @@ bool GCNPassConfig::addPreISel() { addPass(createStructurizeCFGPass(false)); // true -> SkipUniformRegions } addPass(createAMDGPUAnnotateUniformValues()); - if (!LateCFGStructurize) { + if (!LateCFGStructurize && !DisableStructurizer) { addPass(createSIAnnotateControlFlowPass()); // TODO: Move this right after structurizeCFG to avoid extra divergence // analysis. This depends on stopping SIAnnotateControlFlow from making diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h index ce2dd2947daf..30ab388c7d52 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h @@ -37,6 +37,7 @@ public: static bool EnableLateStructurizeCFG; static bool EnableFunctionCalls; static bool EnableLowerModuleLDS; + static bool DisableStructurizer; AMDGPUTargetMachine(const Target &T, const Triple &TT, StringRef CPU, StringRef FS, const TargetOptions &Options, diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index f30dc06ed408..23fe0a52a74b 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -98,6 +98,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, addRegisterClass(MVT::f64, V64RegClass); addRegisterClass(MVT::v2f32, V64RegClass); + addRegisterClass(MVT::Untyped, V64RegClass); addRegisterClass(MVT::v3i32, &AMDGPU::SGPR_96RegClass); addRegisterClass(MVT::v3f32, TRI->getVGPRClassForBitWidth(96)); @@ -3828,6 +3829,9 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI, Ops.push_back(DAG.getTargetConstant(0, DL, MVT::i64)); } + if (!IsTailCall) + Ops.push_back(CLI.ConvergenceControlToken); + if (IsTailCall) { // Each tail call may have to adjust the stack by a different amount, so // this information must travel along with the operation for eventual @@ -5152,8 +5156,24 @@ MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter( MachineInstrBuilder MIB; MIB = BuildMI(*BB, MI, DL, TII->get(AMDGPU::SI_CALL), ReturnAddrReg); - for (const MachineOperand &MO : MI.operands()) - MIB.add(MO); + for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) { + MachineOperand &MO = MI.getOperand(I); + if (I != 2) { + MIB.add(MO); + continue; + } + } + + MachineOperand &MO = MI.getOperand(2); + MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); + // The token operand is always a register, whose definition is IMPLICIT_DEF + // iff there was no token on the call. + if (MachineInstr *Def = MRI.getVRegDef(MO.getReg())) { + if (Def->getOpcode() != TargetOpcode::IMPLICIT_DEF) { + MO.setImplicit(); + MIB.add(MO); + } + } MIB.cloneMemRefs(MI); MI.eraseFromParent(); diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td index 565af36bc523..33c93cdf20c4 100644 --- a/llvm/lib/Target/AMDGPU/SIInstructions.td +++ b/llvm/lib/Target/AMDGPU/SIInstructions.td @@ -618,8 +618,8 @@ def SI_RETURN : SPseudoInstSI < // This version is only needed so we can fill in the output register // in the custom inserter. def SI_CALL_ISEL : SPseudoInstSI < - (outs), (ins SSrc_b64:$src0, unknown:$callee), - [(AMDGPUcall i64:$src0, tglobaladdr:$callee)]> { + (outs), (ins SSrc_b64:$src0, unknown:$callee, unknown:$token), + [(AMDGPUcall i64:$src0, tglobaladdr:$callee, untyped:$token)]> { let Size = 4; let isCall = 1; let SchedRW = [WriteBranch]; @@ -629,8 +629,8 @@ def SI_CALL_ISEL : SPseudoInstSI < } def : GCNPat< - (AMDGPUcall i64:$src0, (i64 0)), - (SI_CALL_ISEL $src0, (i64 0)) + (AMDGPUcall i64:$src0, (i64 0), untyped:$token), + (SI_CALL_ISEL $src0, (i64 0), untyped:$token) >; // Wrapper around s_swappc_b64 with extra $callee parameter to track diff --git a/llvm/test/CodeGen/AMDGPU/convergence-tokens.ll b/llvm/test/CodeGen/AMDGPU/convergence-tokens.ll new file mode 100644 index 000000000000..2ed6d7fd0f59 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/convergence-tokens.ll @@ -0,0 +1,83 @@ +; RUN: llc --amdgpu-disable-structurizer -stop-after=amdgpu-isel -mtriple=amdgcn-- -mcpu=gfx900 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=CHECK,ISEL %s +; RUN: llc --amdgpu-disable-structurizer -stop-after=dead-mi-elimination -mtriple=amdgcn-- -mcpu=gfx900 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=CHECK,DEADMI %s + +; CHECK-LABEL: name: basic_call +; CHECK: [[TOKEN:%[0-9]+]]:sreg_64 = CONVERGENCECTRL_ENTRY +; ISEL: {{.*}} SI_CALL_ISEL {{.*}}, @foo, [[TOKEN]], csr_amdgpu, {{.*}} +; DEADMI: {{.*}} SI_CALL {{.*}}, @foo, csr_amdgpu, {{.*}}, implicit [[TOKEN]] +define i32 @basic_call(i32 %src) #0 { + %t = call token @llvm.experimental.convergence.entry() + %r = call i32 @foo(i32 %src) [ "convergencectrl"(token %t) ] + ret i32 %r +} + +; CHECK-LABEL: name: basic_intrinsic +; CHECK: [[TOKEN:%[0-9]+]]:sreg_64 = CONVERGENCECTRL_ANCHOR +; ISEL: CONVERGENCECTRL_GLUE [[TOKEN]] +; DEADMI-NOT: CONVERGENCECTRL_GLUE +; CHECK: {{.*}} = V_READFIRSTLANE_B32 {{.*}}, implicit [[TOKEN]] +define i32 @basic_intrinsic(i32 %src) #0 { + %t = call token @llvm.experimental.convergence.anchor() + %r = call i32 @llvm.amdgcn.readfirstlane(i32 %src) [ "convergencectrl"(token %t) ] + ret i32 %r +} + +; There's nothing to check here. The test is just meant to catch any crashes +; when a convergent call has no token. +define i32 @uncontrolled_call(i32 %src) #0 { + %r = call i32 @foo(i32 %src) + ret i32 %r +} + +; CHECK-LABEL: name: basic_branch +; CHECK: bb.0.entry: +; CHECK: [[TOKEN:%[0-9]+]]:sreg_64 = CONVERGENCECTRL_ANCHOR +; CHECK: bb.1.then: +; ISEL: CONVERGENCECTRL_GLUE [[TOKEN]] +; DEADMI-NOT: CONVERGENCECTRL_GLUE +; CHECK: {{.*}} = V_READFIRSTLANE_B32 {{.*}}, implicit [[TOKEN]] +define i32 @basic_branch(i32 %src, i1 %cond) #0 { +entry: + %t = call token @llvm.experimental.convergence.anchor() + %x = add i32 %src, 1 + br i1 %cond, label %then, label %else + +then: + %r = call i32 @llvm.amdgcn.readfirstlane(i32 %x) [ "convergencectrl"(token %t) ] + br label %else + +else: + %p = phi i32 [%r, %then], [%x, %entry] + ret i32 %p +} + +; CHECK-LABEL: name: basic_loop +; CHECK: [[TOKEN:%[0-9]+]]:sreg_64 = CONVERGENCECTRL_ANCHOR +; CHECK: bb.1.loop: +; CHECK: [[LOOP:%[0-9]+]]:sreg_64 = CONVERGENCECTRL_LOOP [[TOKEN]] +; ISEL: CONVERGENCECTRL_GLUE [[LOOP]] +; DEADMI-NOT: CONVERGENCECTRL_GLUE +; CHECK: {{.*}} = V_READFIRSTLANE_B32 {{.*}}, implicit [[LOOP]] +define i32 @basic_loop(i32 %src, i1 %cond) #0 { + %t1 = call token @llvm.experimental.convergence.anchor() + br label %loop + +loop: + %t2 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t1) ] + %r = call i32 @llvm.amdgcn.readfirstlane(i32 %src) [ "convergencectrl"(token %t2) ] + br i1 %cond, label %loop, label %end + +end: + ret i32 %r +} + +declare i32 @foo(i32 %x) #0 + +declare i32 @llvm.amdgcn.readfirstlane(i32) #0 + +declare token @llvm.experimental.convergence.entry() +declare token @llvm.experimental.convergence.anchor() +declare token @llvm.experimental.convergence.loop() + +attributes #0 = { nounwind readnone convergent } +attributes #1 = { nounwind } diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll index ab160ffc10ed..e015095a4884 100644 --- a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll +++ b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll @@ -92,6 +92,7 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc(<4 x i32> inreg %a, <4 x i32> %b ; DAGISEL-GFX11-NEXT: $vgpr5 = COPY [[COPY2]] ; DAGISEL-GFX11-NEXT: $vgpr6 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr7 = COPY [[COPY]] + ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -121,6 +122,7 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc(<4 x i32> inreg %a, <4 x i32> %b ; DAGISEL-GFX10-NEXT: $vgpr5 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr6 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr7 = COPY [[COPY]] + ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -232,6 +234,7 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_ptr(ptr inreg %a, ptr %b, ptr ad ; DAGISEL-GFX11-NEXT: $vgpr9 = COPY [[COPY2]] ; DAGISEL-GFX11-NEXT: $vgpr10 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr11 = COPY [[COPY]] + ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -269,6 +272,7 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_ptr(ptr inreg %a, ptr %b, ptr ad ; DAGISEL-GFX10-NEXT: $vgpr9 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr10 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr11 = COPY [[COPY]] + ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -400,6 +404,7 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_struct( {ptr, i32, <4 x i32>} in ; DAGISEL-GFX11-NEXT: $vgpr11 = COPY [[COPY2]] ; DAGISEL-GFX11-NEXT: $vgpr12 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr13 = COPY [[COPY]] + ; DAGISEL-GFX11-NEXT: [[DEF2:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -449,6 +454,7 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_struct( {ptr, i32, <4 x i32>} in ; DAGISEL-GFX10-NEXT: $vgpr11 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr12 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr13 = COPY [[COPY]] + ; DAGISEL-GFX10-NEXT: [[DEF2:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -500,6 +506,7 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_float(float inreg %a, float %b) ; DAGISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4) ; DAGISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY]] + ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -517,6 +524,7 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_float(float inreg %a, float %b) ; DAGISEL-GFX10-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr1 = COPY [[COPY]] + ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -568,6 +576,7 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_half(half inreg %a, half %b) { ; DAGISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4) ; DAGISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY]] + ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -585,6 +594,7 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_half(half inreg %a, half %b) { ; DAGISEL-GFX10-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr1 = COPY [[COPY]] + ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -636,6 +646,7 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_bfloat(bfloat inreg %a, bfloat % ; DAGISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4) ; DAGISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY]] + ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -653,6 +664,7 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_bfloat(bfloat inreg %a, bfloat % ; DAGISEL-GFX10-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr1 = COPY [[COPY]] + ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -704,6 +716,7 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_i16(i16 inreg %a, i16 %b) { ; DAGISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4) ; DAGISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY]] + ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -721,6 +734,7 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_i16(i16 inreg %a, i16 %b) { ; DAGISEL-GFX10-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr0 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr1 = COPY [[COPY]] + ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -856,6 +870,7 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_v16i16(<16 x i16> inreg %a, <16 ; DAGISEL-GFX11-NEXT: $vgpr13 = COPY [[COPY2]] ; DAGISEL-GFX11-NEXT: $vgpr14 = COPY [[COPY1]] ; DAGISEL-GFX11-NEXT: $vgpr15 = COPY [[COPY]] + ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -901,6 +916,7 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_v16i16(<16 x i16> inreg %a, <16 ; DAGISEL-GFX10-NEXT: $vgpr13 = COPY [[COPY2]] ; DAGISEL-GFX10-NEXT: $vgpr14 = COPY [[COPY1]] ; DAGISEL-GFX10-NEXT: $vgpr15 = COPY [[COPY]] + ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 @@ -2464,6 +2480,7 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_many_regs(<36 x i32> inreg %a, <128 ; DAGISEL-GFX11-NEXT: $vgpr29 = COPY [[COPY134]] ; DAGISEL-GFX11-NEXT: $vgpr30 = COPY [[COPY133]] ; DAGISEL-GFX11-NEXT: $vgpr31 = COPY [[COPY132]] + ; DAGISEL-GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $vgpr31 ; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 528, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX11-NEXT: S_ENDPGM 0 @@ -2810,6 +2827,7 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_many_regs(<36 x i32> inreg %a, <128 ; DAGISEL-GFX10-NEXT: $vgpr29 = COPY [[COPY134]] ; DAGISEL-GFX10-NEXT: $vgpr30 = COPY [[COPY133]] ; DAGISEL-GFX10-NEXT: $vgpr31 = COPY [[COPY132]] + ; DAGISEL-GFX10-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF ; DAGISEL-GFX10-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $vgpr31 ; DAGISEL-GFX10-NEXT: ADJCALLSTACKDOWN 0, 528, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32 ; DAGISEL-GFX10-NEXT: S_ENDPGM 0 diff --git a/llvm/test/CodeGen/AMDGPU/kernel-vgpr-spill-mubuf-with-voffset.ll b/llvm/test/CodeGen/AMDGPU/kernel-vgpr-spill-mubuf-with-voffset.ll index 6e905542ce53..8b6b48bcdba0 100644 --- a/llvm/test/CodeGen/AMDGPU/kernel-vgpr-spill-mubuf-with-voffset.ll +++ b/llvm/test/CodeGen/AMDGPU/kernel-vgpr-spill-mubuf-with-voffset.ll @@ -60,6 +60,7 @@ define amdgpu_kernel void @test_kernel(i32 %val) #0 { ; CHECK-NEXT: ; implicit-def: $sgpr15 ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] +; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_waitcnt lgkmcnt(0) ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: s_or_saveexec_b64 s[34:35], -1 diff --git a/llvm/test/CodeGen/AMDGPU/need-fp-from-vgpr-spills.ll b/llvm/test/CodeGen/AMDGPU/need-fp-from-vgpr-spills.ll index f70441e87a74..5f507d482eeb 100644 --- a/llvm/test/CodeGen/AMDGPU/need-fp-from-vgpr-spills.ll +++ b/llvm/test/CodeGen/AMDGPU/need-fp-from-vgpr-spills.ll @@ -27,7 +27,7 @@ define internal fastcc void @csr_vgpr_spill_fp_callee() #0 { ; CHECK-LABEL: csr_vgpr_spill_fp_callee: ; CHECK: ; %bb.0: ; %bb ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: s_mov_b32 s18, s33 +; CHECK-NEXT: s_mov_b32 s24, s33 ; CHECK-NEXT: s_mov_b32 s33, s32 ; CHECK-NEXT: s_xor_saveexec_b64 s[16:17], -1 ; CHECK-NEXT: buffer_store_dword v1, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill @@ -43,6 +43,7 @@ define internal fastcc void @csr_vgpr_spill_fp_callee() #0 { ; CHECK-NEXT: s_mov_b64 s[20:21], s[0:1] ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] +; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: ;;#ASMSTART ; CHECK-NEXT: ; clobber csr v40 @@ -54,7 +55,7 @@ define internal fastcc void @csr_vgpr_spill_fp_callee() #0 { ; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload ; CHECK-NEXT: s_mov_b64 exec, s[4:5] ; CHECK-NEXT: s_add_i32 s32, s32, 0xfffffc00 -; CHECK-NEXT: s_mov_b32 s33, s18 +; CHECK-NEXT: s_mov_b32 s33, s24 ; CHECK-NEXT: s_waitcnt vmcnt(0) ; CHECK-NEXT: s_setpc_b64 s[30:31] bb: @@ -87,6 +88,7 @@ define amdgpu_kernel void @kernel_call() { ; CHECK-NEXT: ; implicit-def: $sgpr15 ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] +; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: s_endpgm bb: @@ -146,6 +148,7 @@ define amdgpu_kernel void @kernel_tailcall() { ; CHECK-NEXT: ; implicit-def: $sgpr15 ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] +; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: s_endpgm bb: @@ -170,7 +173,7 @@ define hidden i32 @caller_save_vgpr_spill_fp_tail_call() #0 { ; CHECK-LABEL: caller_save_vgpr_spill_fp_tail_call: ; CHECK: ; %bb.0: ; %entry ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: s_mov_b32 s18, s33 +; CHECK-NEXT: s_mov_b32 s24, s33 ; CHECK-NEXT: s_mov_b32 s33, s32 ; CHECK-NEXT: s_xor_saveexec_b64 s[16:17], -1 ; CHECK-NEXT: buffer_store_dword v1, off, s[0:3], s33 ; 4-byte Folded Spill @@ -185,6 +188,7 @@ define hidden i32 @caller_save_vgpr_spill_fp_tail_call() #0 { ; CHECK-NEXT: s_mov_b64 s[20:21], s[0:1] ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] +; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: v_readlane_b32 s31, v1, 1 ; CHECK-NEXT: v_readlane_b32 s30, v1, 0 @@ -192,7 +196,7 @@ define hidden i32 @caller_save_vgpr_spill_fp_tail_call() #0 { ; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s33 ; 4-byte Folded Reload ; CHECK-NEXT: s_mov_b64 exec, s[4:5] ; CHECK-NEXT: s_add_i32 s32, s32, 0xfffffc00 -; CHECK-NEXT: s_mov_b32 s33, s18 +; CHECK-NEXT: s_mov_b32 s33, s24 ; CHECK-NEXT: s_waitcnt vmcnt(0) ; CHECK-NEXT: s_setpc_b64 s[30:31] entry: @@ -204,7 +208,7 @@ define hidden i32 @caller_save_vgpr_spill_fp() #0 { ; CHECK-LABEL: caller_save_vgpr_spill_fp: ; CHECK: ; %bb.0: ; %entry ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: s_mov_b32 s19, s33 +; CHECK-NEXT: s_mov_b32 s25, s33 ; CHECK-NEXT: s_mov_b32 s33, s32 ; CHECK-NEXT: s_xor_saveexec_b64 s[16:17], -1 ; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s33 ; 4-byte Folded Spill @@ -219,6 +223,7 @@ define hidden i32 @caller_save_vgpr_spill_fp() #0 { ; CHECK-NEXT: s_mov_b64 s[20:21], s[0:1] ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] +; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: v_readlane_b32 s31, v2, 1 ; CHECK-NEXT: v_readlane_b32 s30, v2, 0 @@ -226,7 +231,7 @@ define hidden i32 @caller_save_vgpr_spill_fp() #0 { ; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s33 ; 4-byte Folded Reload ; CHECK-NEXT: s_mov_b64 exec, s[4:5] ; CHECK-NEXT: s_add_i32 s32, s32, 0xfffffc00 -; CHECK-NEXT: s_mov_b32 s33, s19 +; CHECK-NEXT: s_mov_b32 s33, s25 ; CHECK-NEXT: s_waitcnt vmcnt(0) ; CHECK-NEXT: s_setpc_b64 s[30:31] entry: @@ -258,6 +263,7 @@ define protected amdgpu_kernel void @kernel() { ; CHECK-NEXT: ; implicit-def: $sgpr15 ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] +; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: s_endpgm entry: diff --git a/llvm/test/CodeGen/AMDGPU/no-source-locations-in-prologue.ll b/llvm/test/CodeGen/AMDGPU/no-source-locations-in-prologue.ll index 9999cb9173b5..34e67d0993fb 100644 --- a/llvm/test/CodeGen/AMDGPU/no-source-locations-in-prologue.ll +++ b/llvm/test/CodeGen/AMDGPU/no-source-locations-in-prologue.ll @@ -32,6 +32,7 @@ define hidden void @_ZL3barv() #0 !dbg !1644 { ; CHECK-NEXT: s_mov_b64 s[20:21], s[0:1] ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] +; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_waitcnt lgkmcnt(0) ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: .Ltmp1: diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spills-split-regalloc.ll b/llvm/test/CodeGen/AMDGPU/sgpr-spills-split-regalloc.ll index f523b4a2495f..764f4942cbd0 100644 --- a/llvm/test/CodeGen/AMDGPU/sgpr-spills-split-regalloc.ll +++ b/llvm/test/CodeGen/AMDGPU/sgpr-spills-split-regalloc.ll @@ -16,7 +16,7 @@ define void @spill_sgpr_with_no_lower_vgpr_available() #0 { ; GCN-LABEL: spill_sgpr_with_no_lower_vgpr_available: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_mov_b32 s18, s33 +; GCN-NEXT: s_mov_b32 s24, s33 ; GCN-NEXT: s_mov_b32 s33, s32 ; GCN-NEXT: s_or_saveexec_b64 s[16:17], -1 ; GCN-NEXT: buffer_store_dword v255, off, s[0:3], s33 offset:448 ; 4-byte Folded Spill @@ -150,6 +150,7 @@ define void @spill_sgpr_with_no_lower_vgpr_available() #0 { ; GCN-NEXT: s_mov_b64 s[20:21], s[0:1] ; GCN-NEXT: s_mov_b64 s[0:1], s[20:21] ; GCN-NEXT: s_mov_b64 s[2:3], s[22:23] +; GCN-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GCN-NEXT: v_readlane_b32 s31, v255, 1 @@ -269,7 +270,7 @@ define void @spill_sgpr_with_no_lower_vgpr_available() #0 { ; GCN-NEXT: buffer_load_dword v255, off, s[0:3], s33 offset:448 ; 4-byte Folded Reload ; GCN-NEXT: s_mov_b64 exec, s[4:5] ; GCN-NEXT: s_add_i32 s32, s32, 0xffff8c00 -; GCN-NEXT: s_mov_b32 s33, s18 +; GCN-NEXT: s_mov_b32 s33, s24 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_setpc_b64 s[30:31] %alloca = alloca i32, align 4, addrspace(5) @@ -310,7 +311,7 @@ define void @spill_to_lowest_available_vgpr() #0 { ; GCN-LABEL: spill_to_lowest_available_vgpr: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_mov_b32 s18, s33 +; GCN-NEXT: s_mov_b32 s24, s33 ; GCN-NEXT: s_mov_b32 s33, s32 ; GCN-NEXT: s_or_saveexec_b64 s[16:17], -1 ; GCN-NEXT: buffer_store_dword v254, off, s[0:3], s33 offset:444 ; 4-byte Folded Spill @@ -443,6 +444,7 @@ define void @spill_to_lowest_available_vgpr() #0 { ; GCN-NEXT: s_mov_b64 s[20:21], s[0:1] ; GCN-NEXT: s_mov_b64 s[0:1], s[20:21] ; GCN-NEXT: s_mov_b64 s[2:3], s[22:23] +; GCN-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GCN-NEXT: v_readlane_b32 s31, v254, 1 @@ -561,7 +563,7 @@ define void @spill_to_lowest_available_vgpr() #0 { ; GCN-NEXT: buffer_load_dword v254, off, s[0:3], s33 offset:444 ; 4-byte Folded Reload ; GCN-NEXT: s_mov_b64 exec, s[4:5] ; GCN-NEXT: s_add_i32 s32, s32, 0xffff8c00 -; GCN-NEXT: s_mov_b32 s33, s18 +; GCN-NEXT: s_mov_b32 s33, s24 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_setpc_b64 s[30:31] %alloca = alloca i32, align 4, addrspace(5) @@ -1528,7 +1530,7 @@ define void @spill_sgpr_no_free_vgpr_ipra() #0 { ; GCN-LABEL: spill_sgpr_no_free_vgpr_ipra: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_mov_b32 s18, s33 +; GCN-NEXT: s_mov_b32 s24, s33 ; GCN-NEXT: s_mov_b32 s33, s32 ; GCN-NEXT: s_add_i32 s32, s32, 0x7400 ; GCN-NEXT: buffer_store_dword v40, off, s[0:3], s33 offset:444 ; 4-byte Folded Spill @@ -1666,6 +1668,7 @@ define void @spill_sgpr_no_free_vgpr_ipra() #0 { ; GCN-NEXT: s_mov_b64 s[20:21], s[0:1] ; GCN-NEXT: s_mov_b64 s[0:1], s[20:21] ; GCN-NEXT: s_mov_b64 s[2:3], s[22:23] +; GCN-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GCN-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GCN-NEXT: s_mov_b64 s[4:5], exec ; GCN-NEXT: s_mov_b64 exec, 1 @@ -1798,7 +1801,7 @@ define void @spill_sgpr_no_free_vgpr_ipra() #0 { ; GCN-NEXT: buffer_load_dword v41, off, s[0:3], s33 offset:440 ; 4-byte Folded Reload ; GCN-NEXT: buffer_load_dword v40, off, s[0:3], s33 offset:444 ; 4-byte Folded Reload ; GCN-NEXT: s_add_i32 s32, s32, 0xffff8c00 -; GCN-NEXT: s_mov_b32 s33, s18 +; GCN-NEXT: s_mov_b32 s33, s24 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_setpc_b64 s[30:31] call void @child_function_ipra() diff --git a/llvm/test/CodeGen/AMDGPU/stacksave_stackrestore.ll b/llvm/test/CodeGen/AMDGPU/stacksave_stackrestore.ll index 8c5b89429bcc..33b5d6c6850b 100644 --- a/llvm/test/CodeGen/AMDGPU/stacksave_stackrestore.ll +++ b/llvm/test/CodeGen/AMDGPU/stacksave_stackrestore.ll @@ -916,13 +916,13 @@ define amdgpu_kernel void @kernel_stacksave_stackrestore_call_with_stack_objects ; WAVE32-O0-LABEL: kernel_stacksave_stackrestore_call_with_stack_objects: ; WAVE32-O0: ; %bb.0: ; WAVE32-O0-NEXT: s_mov_b32 s32, 0x1200 -; WAVE32-O0-NEXT: s_getpc_b64 s[20:21] -; WAVE32-O0-NEXT: s_mov_b32 s20, s0 -; WAVE32-O0-NEXT: s_load_dwordx4 s[20:23], s[20:21], 0x0 +; WAVE32-O0-NEXT: s_getpc_b64 s[24:25] +; WAVE32-O0-NEXT: s_mov_b32 s24, s0 +; WAVE32-O0-NEXT: s_load_dwordx4 s[24:27], s[24:25], 0x0 ; WAVE32-O0-NEXT: s_waitcnt lgkmcnt(0) -; WAVE32-O0-NEXT: s_bitset0_b32 s23, 21 -; WAVE32-O0-NEXT: s_add_u32 s20, s20, s9 -; WAVE32-O0-NEXT: s_addc_u32 s21, s21, 0 +; WAVE32-O0-NEXT: s_bitset0_b32 s27, 21 +; WAVE32-O0-NEXT: s_add_u32 s24, s24, s9 +; WAVE32-O0-NEXT: s_addc_u32 s25, s25, 0 ; WAVE32-O0-NEXT: ; implicit-def: $vgpr3 : SGPR spill to VGPR lane ; WAVE32-O0-NEXT: s_mov_b32 s14, s8 ; WAVE32-O0-NEXT: s_mov_b32 s13, s7 @@ -934,17 +934,17 @@ define amdgpu_kernel void @kernel_stacksave_stackrestore_call_with_stack_objects ; WAVE32-O0-NEXT: v_writelane_b32 v3, s0, 0 ; WAVE32-O0-NEXT: s_lshr_b32 s0, s0, 5 ; WAVE32-O0-NEXT: v_writelane_b32 v3, s0, 1 -; WAVE32-O0-NEXT: s_or_saveexec_b32 s19, -1 -; WAVE32-O0-NEXT: buffer_store_dword v3, off, s[20:23], 0 offset:128 ; 4-byte Folded Spill -; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s19 +; WAVE32-O0-NEXT: s_or_saveexec_b32 s20, -1 +; WAVE32-O0-NEXT: buffer_store_dword v3, off, s[24:27], 0 offset:128 ; 4-byte Folded Spill +; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s20 ; WAVE32-O0-NEXT: v_mov_b32_e32 v3, 42 -; WAVE32-O0-NEXT: buffer_store_dword v3, off, s[20:23], 0 +; WAVE32-O0-NEXT: buffer_store_dword v3, off, s[24:27], 0 ; WAVE32-O0-NEXT: s_waitcnt_vscnt null, 0x0 -; WAVE32-O0-NEXT: s_mov_b64 s[0:1], s[20:21] -; WAVE32-O0-NEXT: s_mov_b64 s[2:3], s[22:23] +; WAVE32-O0-NEXT: s_mov_b64 s[0:1], s[24:25] +; WAVE32-O0-NEXT: s_mov_b64 s[2:3], s[26:27] ; WAVE32-O0-NEXT: s_mov_b32 s6, s32 ; WAVE32-O0-NEXT: v_mov_b32_e32 v3, 17 -; WAVE32-O0-NEXT: buffer_store_dword v3, off, s[20:23], s6 offset:4 +; WAVE32-O0-NEXT: buffer_store_dword v3, off, s[24:27], s6 offset:4 ; WAVE32-O0-NEXT: s_mov_b32 s6, stack_passed_argument@abs32@hi ; WAVE32-O0-NEXT: s_mov_b32 s16, stack_passed_argument@abs32@lo ; WAVE32-O0-NEXT: ; kill: def $sgpr16 killed $sgpr16 def $sgpr16_sgpr17 @@ -1018,10 +1018,11 @@ define amdgpu_kernel void @kernel_stacksave_stackrestore_call_with_stack_objects ; WAVE32-O0-NEXT: v_mov_b32_e32 v29, s18 ; WAVE32-O0-NEXT: ; implicit-def: $sgpr18 ; WAVE32-O0-NEXT: v_mov_b32_e32 v30, s18 +; WAVE32-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; WAVE32-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] -; WAVE32-O0-NEXT: s_or_saveexec_b32 s19, -1 -; WAVE32-O0-NEXT: buffer_load_dword v0, off, s[20:23], 0 offset:128 ; 4-byte Folded Reload -; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s19 +; WAVE32-O0-NEXT: s_or_saveexec_b32 s20, -1 +; WAVE32-O0-NEXT: buffer_load_dword v0, off, s[24:27], 0 offset:128 ; 4-byte Folded Reload +; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s20 ; WAVE32-O0-NEXT: s_waitcnt vmcnt(0) ; WAVE32-O0-NEXT: v_readlane_b32 s1, v0, 1 ; WAVE32-O0-NEXT: v_readlane_b32 s0, v0, 0 @@ -1136,6 +1137,7 @@ define amdgpu_kernel void @kernel_stacksave_stackrestore_call_with_stack_objects ; WAVE64-O0-NEXT: v_mov_b32_e32 v29, s18 ; WAVE64-O0-NEXT: ; implicit-def: $sgpr18 ; WAVE64-O0-NEXT: v_mov_b32_e32 v30, s18 +; WAVE64-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; WAVE64-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] ; WAVE64-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 ; WAVE64-O0-NEXT: buffer_load_dword v0, off, s[24:27], 0 offset:128 ; 4-byte Folded Reload @@ -1153,13 +1155,13 @@ define amdgpu_kernel void @kernel_stacksave_stackrestore_call_with_stack_objects ; WAVE32-WWM-PREALLOC-LABEL: kernel_stacksave_stackrestore_call_with_stack_objects: ; WAVE32-WWM-PREALLOC: ; %bb.0: ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s32, 0x1200 -; WAVE32-WWM-PREALLOC-NEXT: s_getpc_b64 s[20:21] -; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s20, s0 -; WAVE32-WWM-PREALLOC-NEXT: s_load_dwordx4 s[20:23], s[20:21], 0x0 +; WAVE32-WWM-PREALLOC-NEXT: s_getpc_b64 s[24:25] +; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s24, s0 +; WAVE32-WWM-PREALLOC-NEXT: s_load_dwordx4 s[24:27], s[24:25], 0x0 ; WAVE32-WWM-PREALLOC-NEXT: s_waitcnt lgkmcnt(0) -; WAVE32-WWM-PREALLOC-NEXT: s_bitset0_b32 s23, 21 -; WAVE32-WWM-PREALLOC-NEXT: s_add_u32 s20, s20, s9 -; WAVE32-WWM-PREALLOC-NEXT: s_addc_u32 s21, s21, 0 +; WAVE32-WWM-PREALLOC-NEXT: s_bitset0_b32 s27, 21 +; WAVE32-WWM-PREALLOC-NEXT: s_add_u32 s24, s24, s9 +; WAVE32-WWM-PREALLOC-NEXT: s_addc_u32 s25, s25, 0 ; WAVE32-WWM-PREALLOC-NEXT: ; implicit-def: $vgpr32 : SGPR spill to VGPR lane ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s14, s8 ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s13, s7 @@ -1172,13 +1174,13 @@ define amdgpu_kernel void @kernel_stacksave_stackrestore_call_with_stack_objects ; WAVE32-WWM-PREALLOC-NEXT: s_lshr_b32 s0, s0, 5 ; WAVE32-WWM-PREALLOC-NEXT: v_writelane_b32 v32, s0, 1 ; WAVE32-WWM-PREALLOC-NEXT: v_mov_b32_e32 v3, 42 -; WAVE32-WWM-PREALLOC-NEXT: buffer_store_dword v3, off, s[20:23], 0 +; WAVE32-WWM-PREALLOC-NEXT: buffer_store_dword v3, off, s[24:27], 0 ; WAVE32-WWM-PREALLOC-NEXT: s_waitcnt_vscnt null, 0x0 -; WAVE32-WWM-PREALLOC-NEXT: s_mov_b64 s[0:1], s[20:21] -; WAVE32-WWM-PREALLOC-NEXT: s_mov_b64 s[2:3], s[22:23] +; WAVE32-WWM-PREALLOC-NEXT: s_mov_b64 s[0:1], s[24:25] +; WAVE32-WWM-PREALLOC-NEXT: s_mov_b64 s[2:3], s[26:27] ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s6, s32 ; WAVE32-WWM-PREALLOC-NEXT: v_mov_b32_e32 v3, 17 -; WAVE32-WWM-PREALLOC-NEXT: buffer_store_dword v3, off, s[20:23], s6 offset:4 +; WAVE32-WWM-PREALLOC-NEXT: buffer_store_dword v3, off, s[24:27], s6 offset:4 ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s6, stack_passed_argument@abs32@hi ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s16, stack_passed_argument@abs32@lo ; WAVE32-WWM-PREALLOC-NEXT: ; kill: def $sgpr16 killed $sgpr16 def $sgpr16_sgpr17 @@ -1252,6 +1254,7 @@ define amdgpu_kernel void @kernel_stacksave_stackrestore_call_with_stack_objects ; WAVE32-WWM-PREALLOC-NEXT: v_mov_b32_e32 v29, s18 ; WAVE32-WWM-PREALLOC-NEXT: ; implicit-def: $sgpr18 ; WAVE32-WWM-PREALLOC-NEXT: v_mov_b32_e32 v30, s18 +; WAVE32-WWM-PREALLOC-NEXT: ; implicit-def: $sgpr18_sgpr19 ; WAVE32-WWM-PREALLOC-NEXT: s_swappc_b64 s[30:31], s[16:17] ; WAVE32-WWM-PREALLOC-NEXT: v_readlane_b32 s1, v32, 1 ; WAVE32-WWM-PREALLOC-NEXT: v_readlane_b32 s0, v32, 0 @@ -1344,7 +1347,7 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE32-O0-LABEL: func_stacksave_stackrestore_call_with_stack_objects: ; WAVE32-O0: ; %bb.0: ; WAVE32-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; WAVE32-O0-NEXT: s_mov_b32 s25, s33 +; WAVE32-O0-NEXT: s_mov_b32 s26, s33 ; WAVE32-O0-NEXT: s_mov_b32 s33, s32 ; WAVE32-O0-NEXT: s_xor_saveexec_b32 s16, -1 ; WAVE32-O0-NEXT: buffer_store_dword v32, off, s[0:3], s33 offset:128 ; 4-byte Folded Spill @@ -1358,9 +1361,9 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE32-O0-NEXT: v_writelane_b32 v0, s16, 0 ; WAVE32-O0-NEXT: s_lshr_b32 s16, s16, 5 ; WAVE32-O0-NEXT: v_writelane_b32 v0, s16, 1 -; WAVE32-O0-NEXT: s_or_saveexec_b32 s24, -1 +; WAVE32-O0-NEXT: s_or_saveexec_b32 s25, -1 ; WAVE32-O0-NEXT: buffer_store_dword v0, off, s[0:3], s33 offset:132 ; 4-byte Folded Spill -; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s24 +; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s25 ; WAVE32-O0-NEXT: v_mov_b32_e32 v0, 42 ; WAVE32-O0-NEXT: buffer_store_dword v0, off, s[0:3], s33 ; WAVE32-O0-NEXT: s_waitcnt_vscnt null, 0x0 @@ -1437,10 +1440,11 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE32-O0-NEXT: v_mov_b32_e32 v29, s18 ; WAVE32-O0-NEXT: ; implicit-def: $sgpr18 ; WAVE32-O0-NEXT: v_mov_b32_e32 v30, s18 +; WAVE32-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; WAVE32-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] -; WAVE32-O0-NEXT: s_or_saveexec_b32 s24, -1 +; WAVE32-O0-NEXT: s_or_saveexec_b32 s25, -1 ; WAVE32-O0-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:132 ; 4-byte Folded Reload -; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s24 +; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s25 ; WAVE32-O0-NEXT: s_waitcnt vmcnt(0) ; WAVE32-O0-NEXT: v_readlane_b32 s5, v0, 1 ; WAVE32-O0-NEXT: v_readlane_b32 s4, v0, 0 @@ -1456,14 +1460,14 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE32-O0-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:136 ; 4-byte Folded Reload ; WAVE32-O0-NEXT: s_mov_b32 exec_lo, s4 ; WAVE32-O0-NEXT: s_add_i32 s32, s32, 0xffffee00 -; WAVE32-O0-NEXT: s_mov_b32 s33, s25 +; WAVE32-O0-NEXT: s_mov_b32 s33, s26 ; WAVE32-O0-NEXT: s_waitcnt vmcnt(0) ; WAVE32-O0-NEXT: s_setpc_b64 s[30:31] ; ; WAVE64-O0-LABEL: func_stacksave_stackrestore_call_with_stack_objects: ; WAVE64-O0: ; %bb.0: ; WAVE64-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; WAVE64-O0-NEXT: s_mov_b32 s19, s33 +; WAVE64-O0-NEXT: s_mov_b32 s28, s33 ; WAVE64-O0-NEXT: s_mov_b32 s33, s32 ; WAVE64-O0-NEXT: s_xor_saveexec_b64 s[16:17], -1 ; WAVE64-O0-NEXT: buffer_store_dword v32, off, s[0:3], s33 offset:128 ; 4-byte Folded Spill @@ -1556,6 +1560,7 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE64-O0-NEXT: v_mov_b32_e32 v29, s18 ; WAVE64-O0-NEXT: ; implicit-def: $sgpr18 ; WAVE64-O0-NEXT: v_mov_b32_e32 v30, s18 +; WAVE64-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; WAVE64-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] ; WAVE64-O0-NEXT: s_or_saveexec_b64 s[26:27], -1 ; WAVE64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:132 ; 4-byte Folded Reload @@ -1575,14 +1580,14 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:136 ; 4-byte Folded Reload ; WAVE64-O0-NEXT: s_mov_b64 exec, s[4:5] ; WAVE64-O0-NEXT: s_add_i32 s32, s32, 0xffffdc00 -; WAVE64-O0-NEXT: s_mov_b32 s33, s19 +; WAVE64-O0-NEXT: s_mov_b32 s33, s28 ; WAVE64-O0-NEXT: s_waitcnt vmcnt(0) ; WAVE64-O0-NEXT: s_setpc_b64 s[30:31] ; ; WAVE32-WWM-PREALLOC-LABEL: func_stacksave_stackrestore_call_with_stack_objects: ; WAVE32-WWM-PREALLOC: ; %bb.0: ; WAVE32-WWM-PREALLOC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s24, s33 +; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s25, s33 ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s33, s32 ; WAVE32-WWM-PREALLOC-NEXT: s_xor_saveexec_b32 s16, -1 ; WAVE32-WWM-PREALLOC-NEXT: buffer_store_dword v33, off, s[0:3], s33 offset:128 ; 4-byte Folded Spill @@ -1672,6 +1677,7 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE32-WWM-PREALLOC-NEXT: v_mov_b32_e32 v29, s18 ; WAVE32-WWM-PREALLOC-NEXT: ; implicit-def: $sgpr18 ; WAVE32-WWM-PREALLOC-NEXT: v_mov_b32_e32 v30, s18 +; WAVE32-WWM-PREALLOC-NEXT: ; implicit-def: $sgpr18_sgpr19 ; WAVE32-WWM-PREALLOC-NEXT: s_swappc_b64 s[30:31], s[16:17] ; WAVE32-WWM-PREALLOC-NEXT: v_readlane_b32 s5, v32, 1 ; WAVE32-WWM-PREALLOC-NEXT: v_readlane_b32 s4, v32, 0 @@ -1687,7 +1693,7 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { ; WAVE32-WWM-PREALLOC-NEXT: buffer_load_dword v32, off, s[0:3], s33 offset:132 ; 4-byte Folded Reload ; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 exec_lo, s4 ; WAVE32-WWM-PREALLOC-NEXT: s_add_i32 s32, s32, 0xffffee00 -; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s33, s24 +; WAVE32-WWM-PREALLOC-NEXT: s_mov_b32 s33, s25 ; WAVE32-WWM-PREALLOC-NEXT: s_waitcnt vmcnt(0) ; WAVE32-WWM-PREALLOC-NEXT: s_setpc_b64 s[30:31] %alloca = alloca [32 x i32], addrspace(5) diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll index bfc249e9081d..d2364a61ed68 100644 --- a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll +++ b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll @@ -233,10 +233,10 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun ; SI-NEXT: bb.1.Flow: ; SI-NEXT: successors: %bb.2(0x40000000), %bb.10(0x40000000) ; SI-NEXT: {{ $}} - ; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %47:vgpr_32, %bb.0, %4, %bb.9 - ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY4]], %bb.0, undef %49:vgpr_32, %bb.9 - ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %51:vgpr_32, %bb.9 - ; SI-NEXT: [[PHI3:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %53:vgpr_32, %bb.9 + ; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %49:vgpr_32, %bb.0, %4, %bb.9 + ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY4]], %bb.0, undef %51:vgpr_32, %bb.9 + ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %53:vgpr_32, %bb.9 + ; SI-NEXT: [[PHI3:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %55:vgpr_32, %bb.9 ; SI-NEXT: [[SI_ELSE:%[0-9]+]]:sreg_32 = SI_ELSE killed [[SI_IF]], %bb.10, implicit-def dead $exec, implicit-def dead $scc, implicit $exec ; SI-NEXT: S_BRANCH %bb.2 ; SI-NEXT: {{ $}} @@ -249,8 +249,8 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun ; SI-NEXT: bb.3: ; SI-NEXT: successors: %bb.4(0x80000000) ; SI-NEXT: {{ $}} - ; SI-NEXT: [[PHI4:%[0-9]+]]:vreg_64 = PHI undef %55:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2 - ; SI-NEXT: [[PHI5:%[0-9]+]]:vgpr_32 = PHI undef %57:vgpr_32, %bb.4, [[PHI1]], %bb.2 + ; SI-NEXT: [[PHI4:%[0-9]+]]:vreg_64 = PHI undef %57:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2 + ; SI-NEXT: [[PHI5:%[0-9]+]]:vgpr_32 = PHI undef %59:vgpr_32, %bb.4, [[PHI1]], %bb.2 ; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI4]].sub0, implicit $exec ; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI4]].sub1, implicit $exec ; SI-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_1]], %subreg.sub1 @@ -286,8 +286,8 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun ; SI-NEXT: bb.7: ; SI-NEXT: successors: %bb.8(0x80000000) ; SI-NEXT: {{ $}} - ; SI-NEXT: [[PHI6:%[0-9]+]]:vreg_64 = PHI undef %59:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6 - ; SI-NEXT: [[PHI7:%[0-9]+]]:vgpr_32 = PHI undef %61:vgpr_32, %bb.8, [[COPY4]], %bb.6 + ; SI-NEXT: [[PHI6:%[0-9]+]]:vreg_64 = PHI undef %61:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6 + ; SI-NEXT: [[PHI7:%[0-9]+]]:vgpr_32 = PHI undef %63:vgpr_32, %bb.8, [[COPY4]], %bb.6 ; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI6]].sub0, implicit $exec ; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI6]].sub1, implicit $exec ; SI-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_2]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_3]], %subreg.sub1 @@ -356,9 +356,9 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e ; SI-NEXT: bb.1.Flow: ; SI-NEXT: successors: %bb.2(0x40000000), %bb.10(0x40000000) ; SI-NEXT: {{ $}} - ; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %48:vgpr_32, %bb.0, %4, %bb.9 - ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %50:vgpr_32, %bb.9 - ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %52:vgpr_32, %bb.9 + ; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %50:vgpr_32, %bb.0, %4, %bb.9 + ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %52:vgpr_32, %bb.9 + ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %54:vgpr_32, %bb.9 ; SI-NEXT: [[SI_ELSE:%[0-9]+]]:sreg_32 = SI_ELSE killed [[SI_IF]], %bb.10, implicit-def dead $exec, implicit-def dead $scc, implicit $exec ; SI-NEXT: S_BRANCH %bb.2 ; SI-NEXT: {{ $}} @@ -371,7 +371,7 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e ; SI-NEXT: bb.3: ; SI-NEXT: successors: %bb.4(0x80000000) ; SI-NEXT: {{ $}} - ; SI-NEXT: [[PHI3:%[0-9]+]]:vreg_64 = PHI undef %54:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2 + ; SI-NEXT: [[PHI3:%[0-9]+]]:vreg_64 = PHI undef %56:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2 ; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI3]].sub0, implicit $exec ; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI3]].sub1, implicit $exec ; SI-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_1]], %subreg.sub1 @@ -407,7 +407,7 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e ; SI-NEXT: bb.7: ; SI-NEXT: successors: %bb.8(0x80000000) ; SI-NEXT: {{ $}} - ; SI-NEXT: [[PHI4:%[0-9]+]]:vreg_64 = PHI undef %56:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6 + ; SI-NEXT: [[PHI4:%[0-9]+]]:vreg_64 = PHI undef %58:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6 ; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI4]].sub0, implicit $exec ; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[PHI4]].sub1, implicit $exec ; SI-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE killed [[V_READFIRSTLANE_B32_2]], %subreg.sub0, killed [[V_READFIRSTLANE_B32_3]], %subreg.sub1 diff --git a/llvm/test/CodeGen/AMDGPU/vgpr_constant_to_sgpr.ll b/llvm/test/CodeGen/AMDGPU/vgpr_constant_to_sgpr.ll index 7840559c78eb..364ce82b2e99 100644 --- a/llvm/test/CodeGen/AMDGPU/vgpr_constant_to_sgpr.ll +++ b/llvm/test/CodeGen/AMDGPU/vgpr_constant_to_sgpr.ll @@ -47,6 +47,7 @@ define protected amdgpu_kernel void @kern(ptr %addr) !llvm.amdgcn.lds.kernel.id ; CHECK-NEXT: s_mov_b32 s15, 42 ; CHECK-NEXT: s_mov_b64 s[0:1], s[20:21] ; CHECK-NEXT: s_mov_b64 s[2:3], s[22:23] +; CHECK-NEXT: ; implicit-def: $sgpr18_sgpr19 ; CHECK-NEXT: s_waitcnt lgkmcnt(0) ; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17] ; CHECK-NEXT: s_endpgm diff --git a/llvm/test/CodeGen/AMDGPU/whole-wave-register-spill.ll b/llvm/test/CodeGen/AMDGPU/whole-wave-register-spill.ll index 7eabe982ff2b..3a33194f17c8 100644 --- a/llvm/test/CodeGen/AMDGPU/whole-wave-register-spill.ll +++ b/llvm/test/CodeGen/AMDGPU/whole-wave-register-spill.ll @@ -101,6 +101,7 @@ define void @test() #0 { ; GCN-O0-NEXT: s_mov_b64 s[20:21], s[0:1] ; GCN-O0-NEXT: s_mov_b64 s[0:1], s[20:21] ; GCN-O0-NEXT: s_mov_b64 s[2:3], s[22:23] +; GCN-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GCN-O0-NEXT: s_waitcnt lgkmcnt(0) ; GCN-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GCN-O0-NEXT: s_or_saveexec_b64 s[28:29], -1 diff --git a/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll b/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll index e79cb66dcd77..11f6a2960776 100644 --- a/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll +++ b/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll @@ -406,6 +406,7 @@ define amdgpu_gfx void @strict_wwm_call(ptr addrspace(8) inreg %tmp14, i32 inreg ; GFX9-O0-NEXT: s_mov_b64 s[0:1], s[44:45] ; GFX9-O0-NEXT: s_mov_b64 s[2:3], s[46:47] ; GFX9-O0-NEXT: v_mov_b32_e32 v0, v2 +; GFX9-O0-NEXT: ; implicit-def: $sgpr44_sgpr45 ; GFX9-O0-NEXT: s_swappc_b64 s[30:31], s[42:43] ; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0 ; GFX9-O0-NEXT: v_add_u32_e64 v1, v1, v2 @@ -632,6 +633,7 @@ define amdgpu_gfx void @strict_wwm_call_i64(ptr addrspace(8) inreg %tmp14, i64 i ; GFX9-O0-NEXT: s_mov_b64 s[2:3], s[38:39] ; GFX9-O0-NEXT: v_mov_b32_e32 v0, v2 ; GFX9-O0-NEXT: v_mov_b32_e32 v1, v3 +; GFX9-O0-NEXT: ; implicit-def: $sgpr36_sgpr37 ; GFX9-O0-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-O0-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX9-O0-NEXT: s_or_saveexec_b64 s[46:47], -1 diff --git a/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll b/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll index 47c976d2a5c3..6ac61410a0e7 100644 --- a/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll +++ b/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll @@ -413,6 +413,7 @@ define amdgpu_kernel void @call(ptr addrspace(8) inreg %tmp14, i32 inreg %arg) { ; GFX9-O0-NEXT: ; implicit-def: $sgpr15 ; GFX9-O0-NEXT: v_mov_b32_e32 v31, v3 ; GFX9-O0-NEXT: v_mov_b32_e32 v0, v6 +; GFX9-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GFX9-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 ; GFX9-O0-NEXT: buffer_load_dword v1, off, s[24:27], 0 ; 4-byte Folded Reload @@ -656,6 +657,7 @@ define amdgpu_kernel void @call_i64(ptr addrspace(8) inreg %tmp14, i64 inreg %ar ; GFX9-O0-NEXT: v_mov_b32_e32 v31, v3 ; GFX9-O0-NEXT: v_mov_b32_e32 v0, v6 ; GFX9-O0-NEXT: v_mov_b32_e32 v1, v7 +; GFX9-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GFX9-O0-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 @@ -1283,6 +1285,7 @@ define amdgpu_kernel void @strict_wwm_call(ptr addrspace(8) inreg %tmp14, i32 in ; GFX9-O0-NEXT: ; implicit-def: $sgpr15 ; GFX9-O0-NEXT: v_mov_b32_e32 v31, v3 ; GFX9-O0-NEXT: v_mov_b32_e32 v0, v6 +; GFX9-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GFX9-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 ; GFX9-O0-NEXT: buffer_load_dword v1, off, s[24:27], 0 ; 4-byte Folded Reload @@ -1526,6 +1529,7 @@ define amdgpu_kernel void @strict_wwm_call_i64(ptr addrspace(8) inreg %tmp14, i6 ; GFX9-O0-NEXT: v_mov_b32_e32 v31, v3 ; GFX9-O0-NEXT: v_mov_b32_e32 v0, v6 ; GFX9-O0-NEXT: v_mov_b32_e32 v1, v7 +; GFX9-O0-NEXT: ; implicit-def: $sgpr18_sgpr19 ; GFX9-O0-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-O0-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1 diff --git a/llvm/test/CodeGen/PowerPC/fmf-propagation.ll b/llvm/test/CodeGen/PowerPC/fmf-propagation.ll index 58b3ee485ea4..4e72a5ac5ede 100644 --- a/llvm/test/CodeGen/PowerPC/fmf-propagation.ll +++ b/llvm/test/CodeGen/PowerPC/fmf-propagation.ll @@ -577,15 +577,15 @@ define double @fcmp_nnan(double %a, double %y, double %z) { ; FP library calls can have fast-math-flags. ; FMFDEBUG-LABEL: Optimized lowered selection DAG: %bb.0 'log2_approx:' -; FMFDEBUG: ch,glue = PPCISD::CALL_NOP t11, TargetGlobalAddress:i64 -; FMFDEBUG: ch,glue = callseq_end t15, TargetConstant:i64<32>, TargetConstant:i64<0>, t15:1 -; FMFDEBUG: f64,ch,glue = CopyFromReg t16, Register:f64 $f1, t16:1 +; FMFDEBUG: ch,glue = PPCISD::CALL_NOP {{t[0-9]+}}, TargetGlobalAddress:i64 +; FMFDEBUG: ch,glue = callseq_end [[T15:t[0-9]+]], TargetConstant:i64<32>, TargetConstant:i64<0>, [[T15]]:1 +; FMFDEBUG: f64,ch,glue = CopyFromReg [[T16:t[0-9]+]], Register:f64 $f1, [[T16]]:1 ; FMFDEBUG: Type-legalized selection DAG: %bb.0 'log2_approx:' ; GLOBALDEBUG-LABEL: Optimized lowered selection DAG: %bb.0 'log2_approx:' -; GLOBALDEBUG: ch,glue = PPCISD::CALL_NOP t11, TargetGlobalAddress:i64 -; GLOBALDEBUG: ch,glue = callseq_end t15, TargetConstant:i64<32>, TargetConstant:i64<0>, t15:1 -; GLOBALDEBUG: f64,ch,glue = CopyFromReg t16, Register:f64 $f1, t16:1 +; GLOBALDEBUG: ch,glue = PPCISD::CALL_NOP {{t[0-9]+}}, TargetGlobalAddress:i64 +; GLOBALDEBUG: ch,glue = callseq_end [[T15:t[0-9]+]], TargetConstant:i64<32>, TargetConstant:i64<0>, [[T15]]:1 +; GLOBALDEBUG: f64,ch,glue = CopyFromReg [[T16:t[0-9]+]], Register:f64 $f1, [[T16]]:1 ; GLOBALDEBUG: Type-legalized selection DAG: %bb.0 'log2_approx:' declare double @log2(double) diff --git a/llvm/test/MachineVerifier/convergencectrl/AMDGPU/basic.mir b/llvm/test/MachineVerifier/convergencectrl/AMDGPU/basic.mir new file mode 100644 index 000000000000..94d0ddad2594 --- /dev/null +++ b/llvm/test/MachineVerifier/convergencectrl/AMDGPU/basic.mir @@ -0,0 +1,37 @@ +# RUN: not --crash llc -march=amdgcn -run-pass=none -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s +--- +name: basic +tracksRegLiveness: true +body: | + bb.0: + successors: %bb.1, %bb.2; + %0:sgpr_64 = CONVERGENCECTRL_ANCHOR + ; CHECK: Entry intrinsic cannot be preceded by a convergent operation in the same basic block. + ; CHECK: CONVERGENCECTRL_ENTRY + %1:sgpr_64 = CONVERGENCECTRL_ENTRY + ; CHECK: Loop intrinsic cannot be preceded by a convergent operation in the same basic block. + ; CHECK: CONVERGENCECTRL_LOOP + %2:sgpr_64 = CONVERGENCECTRL_LOOP %0:sgpr_64 + S_CBRANCH_EXECZ %bb.1, implicit $exec + S_BRANCH %bb.2 + + bb.1: + successors: %bb.2; + ; CHECK: Entry intrinsic can occur only in the entry block. + ; CHECK: CONVERGENCECTRL_ENTRY + %5:sgpr_64 = CONVERGENCECTRL_ENTRY + + bb.2: + ; CHECK: Convergence control tokens can only be used by convergent operations. + ; CHECK: G_PHI + %6:sgpr_64 = G_PHI %0:sgpr_64, %bb.0, %0:sgpr_64, %bb.1 + %7:sgpr_64 = CONVERGENCECTRL_ANCHOR + %8:sgpr_64 = IMPLICIT_DEF + %4:sgpr_64 = SI_CALL %8:sgpr_64, 1, implicit %7:sgpr_64 + ; CHECK: An operation can use at most one convergence control token. + ; CHECK: SI_CALL %{{[0-9]}}:sgpr_64, 2 + %9:sgpr_64 = SI_CALL %8:sgpr_64, 2, implicit %7:sgpr_64, implicit %7:sgpr_64 + ; CHECK: Cannot mix controlled and uncontrolled convergence in the same function. + ; CHECK: SI_CALL %{{[0-9]}}:sgpr_64, 3 + %10:sgpr_64 = SI_CALL %8:sgpr_64, 3 +... diff --git a/llvm/test/MachineVerifier/convergencectrl/AMDGPU/cycles.mir b/llvm/test/MachineVerifier/convergencectrl/AMDGPU/cycles.mir new file mode 100644 index 000000000000..87cf3e604929 --- /dev/null +++ b/llvm/test/MachineVerifier/convergencectrl/AMDGPU/cycles.mir @@ -0,0 +1,52 @@ +# RUN: not --crash llc -march=amdgcn -run-pass=none -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s +--- +name: cycles +body: | + bb.0: + %0:sgpr_64 = CONVERGENCECTRL_ANCHOR + %1:sgpr_64 = IMPLICIT_DEF + S_CBRANCH_EXECZ %bb.9, implicit $exec + S_BRANCH %bb.1 + + bb.1: + S_CBRANCH_EXECZ %bb.8, implicit $exec + S_BRANCH %bb.5 + + bb.2: + S_CBRANCH_EXECZ %bb.3, implicit $exec + S_BRANCH %bb.4 + + bb.3: + ; CHECK: Cycle heart must dominate all blocks in the cycle. + ; Irreducible cycle: entries(bb.4 bb.3) + %3:sgpr_64 = CONVERGENCECTRL_LOOP %0:sgpr_64 + S_BRANCH %bb.4 + + bb.4: + S_BRANCH %bb.3 + + bb.5: + S_CBRANCH_EXECZ %bb.6, implicit $exec + S_BRANCH %bb.2 + + bb.6: + S_BRANCH %bb.7 + + bb.7: + ; CHECK: Cycle heart must dominate all blocks in the cycle. + ; Reducible cycle: entries(bb.6) bb.7 + %4:sgpr_64 = CONVERGENCECTRL_LOOP %0:sgpr_64 + S_BRANCH %bb.6 + + bb.8: + ; CHECK: Two static convergence token uses in a cycle that does not contain either token's definition. + %5:sgpr_64 = CONVERGENCECTRL_LOOP %0:sgpr_64 + %6:sgpr_64 = CONVERGENCECTRL_LOOP %0:sgpr_64 + S_BRANCH %bb.8 + + bb.9: + ; CHECK: Convergence token used by an instruction other than llvm.experimental.convergence.loop in a cycle that does not contain the token's definition. + %7:sgpr_64 = G_SI_CALL %1:sgpr_64, 3, implicit %0:sgpr_64 + S_BRANCH %bb.9 + +... diff --git a/llvm/test/MachineVerifier/convergencectrl/AMDGPU/lit.local.cfg b/llvm/test/MachineVerifier/convergencectrl/AMDGPU/lit.local.cfg new file mode 100644 index 000000000000..7c492428aec7 --- /dev/null +++ b/llvm/test/MachineVerifier/convergencectrl/AMDGPU/lit.local.cfg @@ -0,0 +1,2 @@ +if not "AMDGPU" in config.root.targets: + config.unsupported = True diff --git a/llvm/test/MachineVerifier/convergencectrl/AMDGPU/mixed2.mir b/llvm/test/MachineVerifier/convergencectrl/AMDGPU/mixed2.mir new file mode 100644 index 000000000000..c70a48bf2130 --- /dev/null +++ b/llvm/test/MachineVerifier/convergencectrl/AMDGPU/mixed2.mir @@ -0,0 +1,15 @@ +# RUN: not --crash llc -march=amdgcn -run-pass=none -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s +--- +name: mixed2 +body: | + bb.0: + %0:sgpr_64 = IMPLICIT_DEF + %1:sgpr_64 = SI_CALL %0, 1 + ; CHECK: Cannot mix controlled and uncontrolled convergence in the same function. + ; CHECK: CONVERGENCECTRL_ANCHOR + %2:sgpr_64 = CONVERGENCECTRL_ANCHOR + ; CHECK: Cannot mix controlled and uncontrolled convergence in the same function. + ; CHECK: SI_CALL %{{[0-9]}}:sgpr_64, 2 + %3:sgpr_64 = SI_CALL %0, 2, implicit %2:sgpr_64 + +... diff --git a/llvm/test/MachineVerifier/convergencectrl/AMDGPU/region-nesting.mir b/llvm/test/MachineVerifier/convergencectrl/AMDGPU/region-nesting.mir new file mode 100644 index 000000000000..9e869acb3e93 --- /dev/null +++ b/llvm/test/MachineVerifier/convergencectrl/AMDGPU/region-nesting.mir @@ -0,0 +1,24 @@ +# RUN: not --crash llc -march=amdgcn -run-pass=none -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s +--- +name: region_nesting +body: | + bb.0: + %0:sgpr_64 = CONVERGENCECTRL_ANCHOR + %1:sgpr_64 = CONVERGENCECTRL_ANCHOR + %2:sgpr_64 = IMPLICIT_DEF + %3:sgpr_64 = SI_CALL %2, 1, implicit %0:sgpr_64 + ; CHECK: Convergence region is not well-nested. + ; CHECK: SI_CALL %{{[0-9]}}:sgpr_64, 2 + %4:sgpr_64 = SI_CALL %2, 2, implicit %1:sgpr_64 + S_CBRANCH_EXECZ %bb.1, implicit $exec + S_BRANCH %bb.2 + + bb.1: + %5:sgpr_64 = SI_CALL %2, 3, implicit %0:sgpr_64 + + bb.2: + ; CHECK: Convergence region is not well-nested. + ; CHECK: SI_CALL %{{[0-9]}}:sgpr_64, 4 + %6:sgpr_64 = SI_CALL %2, 4, implicit %1:sgpr_64 + +... diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td index 622d1df7b381..40a831d7e9e8 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td @@ -28,7 +28,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(65), GIMT_Encode2(182), /*)*//*default:*//*Label 2*/ GIMT_Encode4(562), +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(69), GIMT_Encode2(186), /*)*//*default:*//*Label 2*/ GIMT_Encode4(562), // CHECK-NEXT: /*TargetOpcode::G_UNMERGE_VALUES*//*Label 0*/ GIMT_Encode4(478), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), // CHECK-NEXT: /*TargetOpcode::G_FNEG*//*Label 1*/ GIMT_Encode4(530), // CHECK-NEXT: // Label 0: @478 diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td index f0ca65a87b76..751b1318ecc0 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td @@ -34,12 +34,12 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2(128), /*)*//*default:*//*Label 3*/ GIMT_Encode4(563), -// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4(446), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_CONSTANT*//*Label 1*/ GIMT_Encode4(477), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 2*/ GIMT_Encode4(523), -// CHECK-NEXT: // Label 0: @446 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(476), // Rule ID 0 // +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2(132), /*)*//*default:*//*Label 3*/ GIMT_Encode4(579), +// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4(462), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_CONSTANT*//*Label 1*/ GIMT_Encode4(493), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 2*/ GIMT_Encode4(539), +// CHECK-NEXT: // Label 0: @462 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(492), // Rule ID 0 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled), // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[0] a @@ -51,10 +51,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddImm8, /*InsnID*/0, /*Imm*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 4: @476 +// CHECK-NEXT: // Label 4: @492 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 1: @477 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4(522), // Rule ID 2 // +// CHECK-NEXT: // Label 1: @493 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4(538), // Rule ID 2 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled), // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[0] a @@ -66,10 +66,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddCImm, /*InsnID*/0, /*Type*/GILLT_s32, /*Imm*/GIMT_Encode8(42), // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 5: @522 +// CHECK-NEXT: // Label 5: @538 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 2: @523 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(562), // Rule ID 1 // +// CHECK-NEXT: // Label 2: @539 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(578), // Rule ID 1 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled), // CHECK-NEXT: // MIs[0] a // CHECK-NEXT: // No operand predicates @@ -83,10 +83,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 6: @562 +// CHECK-NEXT: // Label 6: @578 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 3: @563 +// CHECK-NEXT: // Label 3: @579 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: }; // Size: 564 bytes +// CHECK-NEXT: }; // Size: 580 bytes // CHECK-NEXT: return MatchTable0; // CHECK-NEXT: } diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td index a446fb72298c..e8e6d3e74f40 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td @@ -29,7 +29,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(115), GIMT_Encode2(117), /*)*//*default:*//*Label 2*/ GIMT_Encode4(132), +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(119), GIMT_Encode2(121), /*)*//*default:*//*Label 2*/ GIMT_Encode4(132), // CHECK-NEXT: /*TargetOpcode::G_INTRINSIC*//*Label 0*/ GIMT_Encode4(18), // CHECK-NEXT: /*TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS*//*Label 1*/ GIMT_Encode4(73), // CHECK-NEXT: // Label 0: @18 diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td index d3c202c4cb01..26a0ec6235e3 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td @@ -28,7 +28,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(120), GIMT_Encode2(183), /*)*//*default:*//*Label 3*/ GIMT_Encode4(380), +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(124), GIMT_Encode2(187), /*)*//*default:*//*Label 3*/ GIMT_Encode4(380), // CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 0*/ GIMT_Encode4(262), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), // CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 1*/ GIMT_Encode4(298), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), // CHECK-NEXT: /*TargetOpcode::G_FPEXT*//*Label 2*/ GIMT_Encode4(344), diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td index cc77bfdd29c3..83b77519bc73 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td @@ -37,7 +37,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(65), GIMT_Encode2(69), /*)*//*default:*//*Label 2*/ GIMT_Encode4(88), +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(69), GIMT_Encode2(73), /*)*//*default:*//*Label 2*/ GIMT_Encode4(88), // CHECK-NEXT: /*TargetOpcode::G_UNMERGE_VALUES*//*Label 0*/ GIMT_Encode4(26), GIMT_Encode4(0), GIMT_Encode4(0), // CHECK-NEXT: /*TargetOpcode::G_BUILD_VECTOR*//*Label 1*/ GIMT_Encode4(57), // CHECK-NEXT: // Label 0: @26 @@ -98,6 +98,6 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIM_Reject, // CHECK-NEXT: // Label 2: @88 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: }; +// CHECK-NEXT: }; // Size: 89 bytes // CHECK-NEXT: return MatchTable0; // CHECK-NEXT: } diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td index 57ad0009b5bd..5cf4e044a0fb 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td @@ -132,15 +132,15 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // Verify match table. // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2(128), /*)*//*default:*//*Label 6*/ GIMT_Encode4(661), -// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4(446), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_AND*//*Label 1*/ GIMT_Encode4(488), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_STORE*//*Label 2*/ GIMT_Encode4(541), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 3*/ GIMT_Encode4(583), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_SEXT*//*Label 4*/ GIMT_Encode4(608), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 5*/ GIMT_Encode4(621), -// CHECK-NEXT: // Label 0: @446 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(475), // Rule ID 4 // +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2(132), /*)*//*default:*//*Label 6*/ GIMT_Encode4(677), +// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4(462), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_AND*//*Label 1*/ GIMT_Encode4(504), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_STORE*//*Label 2*/ GIMT_Encode4(557), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 3*/ GIMT_Encode4(599), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_SEXT*//*Label 4*/ GIMT_Encode4(624), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 5*/ GIMT_Encode4(637), +// CHECK-NEXT: // Label 0: @462 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(491), // Rule ID 4 // // CHECK-NEXT: GIM_CheckFeatures, GIMT_Encode2(GIFBS_HasAnswerToEverything), // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule3Enabled), // CHECK-NEXT: // MIs[0] a @@ -155,8 +155,8 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: // Combiner Rule #3: InstTest1 // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0), // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 7: @475 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(487), // Rule ID 3 // +// CHECK-NEXT: // Label 7: @491 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(503), // Rule ID 3 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled), // CHECK-NEXT: // MIs[0] a // CHECK-NEXT: // No operand predicates @@ -165,10 +165,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: // Combiner Rule #2: InstTest0 // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner1), // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 8: @487 +// CHECK-NEXT: // Label 8: @503 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 1: @488 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 9*/ GIMT_Encode4(540), // Rule ID 6 // +// CHECK-NEXT: // Label 1: @504 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 9*/ GIMT_Encode4(556), // Rule ID 6 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule5Enabled), // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[0] dst @@ -186,10 +186,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // z // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 9: @540 +// CHECK-NEXT: // Label 9: @556 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 2: @541 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 10*/ GIMT_Encode4(582), // Rule ID 5 // +// CHECK-NEXT: // Label 2: @557 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 10*/ GIMT_Encode4(598), // Rule ID 5 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule4Enabled), // CHECK-NEXT: // MIs[0] tmp // CHECK-NEXT: GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/0, // MIs[1] @@ -207,32 +207,32 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner2), // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 10: @582 +// CHECK-NEXT: // Label 10: @598 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 3: @583 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 11*/ GIMT_Encode4(595), // Rule ID 0 // +// CHECK-NEXT: // Label 3: @599 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 11*/ GIMT_Encode4(611), // Rule ID 0 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled), // CHECK-NEXT: // Combiner Rule #0: WipOpcodeTest0; wip_match_opcode 'G_TRUNC' // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0), // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 11: @595 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 12*/ GIMT_Encode4(607), // Rule ID 1 // +// CHECK-NEXT: // Label 11: @611 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 12*/ GIMT_Encode4(623), // Rule ID 1 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled), // CHECK-NEXT: // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_TRUNC' // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0), // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 12: @607 +// CHECK-NEXT: // Label 12: @623 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 4: @608 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 13*/ GIMT_Encode4(620), // Rule ID 2 // +// CHECK-NEXT: // Label 4: @624 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 13*/ GIMT_Encode4(636), // Rule ID 2 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled), // CHECK-NEXT: // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_SEXT' // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0), // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 13: @620 +// CHECK-NEXT: // Label 13: @636 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 5: @621 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 14*/ GIMT_Encode4(660), // Rule ID 7 // +// CHECK-NEXT: // Label 5: @637 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 14*/ GIMT_Encode4(676), // Rule ID 7 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule6Enabled), // CHECK-NEXT: // MIs[0] dst // CHECK-NEXT: // No operand predicates @@ -247,10 +247,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 14: @660 +// CHECK-NEXT: // Label 14: @676 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 6: @661 +// CHECK-NEXT: // Label 6: @677 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: }; // Size: 662 bytes +// CHECK-NEXT: }; // Size: 678 bytes // CHECK-NEXT: return MatchTable0; // CHECK-NEXT: } -- GitLab From 95b52ecb7881014089a5282d308fab4eb8237f20 Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Wed, 6 Mar 2024 14:51:55 +0800 Subject: [PATCH 254/929] [RISCV] Take SEW/LMUL into account for value tracking of vsetvli[max] (#82163) So that we can benefit from some instcombine optimizations. This PR contains two commits: the first is for adding tests and the second is for the optimization. --- llvm/lib/Analysis/ValueTracking.cpp | 28 +++++-- .../RISCV/riscv-vsetvli-knownbits.ll | 84 +++++++------------ .../RISCV/riscv-vsetvlimax-knownbits.ll | 81 ++++++------------ 3 files changed, 78 insertions(+), 115 deletions(-) diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index 9d78c5d323cc..f96a60a4f547 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -73,6 +73,7 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/KnownBits.h" #include "llvm/Support/MathExtras.h" +#include "llvm/TargetParser/RISCVTargetParser.h" #include #include #include @@ -1586,12 +1587,29 @@ static void computeKnownBitsFromOperator(const Operator *I, Known.Zero.setBitsFrom(32); break; case Intrinsic::riscv_vsetvli: - case Intrinsic::riscv_vsetvlimax: - // Assume that VL output is <= 65536. - // TODO: Take SEW and LMUL into account. - if (BitWidth > 17) - Known.Zero.setBitsFrom(17); + case Intrinsic::riscv_vsetvlimax: { + bool HasAVL = II->getIntrinsicID() == Intrinsic::riscv_vsetvli; + const ConstantRange Range = getVScaleRange(II->getFunction(), BitWidth); + uint64_t SEW = RISCVVType::decodeVSEW( + cast(II->getArgOperand(HasAVL))->getZExtValue()); + RISCVII::VLMUL VLMUL = static_cast( + cast(II->getArgOperand(1 + HasAVL))->getZExtValue()); + // The Range is [Lower, Upper), so we need to subtract 1 here to get the + // real upper value. + uint64_t MaxVLEN = + (Range.getUpper().getZExtValue() - 1) * RISCV::RVVBitsPerBlock; + uint64_t MaxVL = MaxVLEN / RISCVVType::getSEWLMULRatio(SEW, VLMUL); + + // Result of vsetvli must be not larger than AVL. + if (HasAVL) + if (auto *CI = dyn_cast(II->getArgOperand(0))) + MaxVL = std::min(MaxVL, CI->getZExtValue()); + + unsigned KnownZeroFirstBit = Log2_32(MaxVL) + 1; + if (BitWidth > KnownZeroFirstBit) + Known.Zero.setBitsFrom(KnownZeroFirstBit); break; + } case Intrinsic::vscale: { if (!II->getParent() || !II->getFunction()) break; diff --git a/llvm/test/Transforms/InstCombine/RISCV/riscv-vsetvli-knownbits.ll b/llvm/test/Transforms/InstCombine/RISCV/riscv-vsetvli-knownbits.ll index e254d2a71b7f..1afae6565fe2 100644 --- a/llvm/test/Transforms/InstCombine/RISCV/riscv-vsetvli-knownbits.ll +++ b/llvm/test/Transforms/InstCombine/RISCV/riscv-vsetvli-knownbits.ll @@ -91,8 +91,7 @@ entry: define i64 @vsetvl_e8m1_and14bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e8m1_and14bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 0, i64 0) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 16383 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 0, i64 0) %b = and i64 %a, 16383 @@ -113,8 +112,7 @@ define i64 @vsetvl_e8m1_and13bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e8m1_constant_avl() nounwind #0 { ; CHECK-LABEL: @vsetvl_e8m1_constant_avl( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 1, i64 0, i64 0) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 1 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 1, i64 0, i64 0) %b = and i64 %a, 1 @@ -124,8 +122,7 @@ define i64 @vsetvl_e8m1_constant_avl() nounwind #0 { define i64 @vsetvl_e8m2_and15bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e8m2_and15bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 0, i64 1) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 32767 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 0, i64 1) %b = and i64 %a, 32767 @@ -146,8 +143,7 @@ define i64 @vsetvl_e8m2_and14bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e8m4_and16bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e8m4_and16bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 0, i64 2) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 65535 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 0, i64 2) %b = and i64 %a, 65535 @@ -189,8 +185,7 @@ define i64 @vsetvl_e8m8_and16bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e8mf2_and11bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e8mf2_and11bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 0, i64 5) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 0, i64 5) %b = and i64 %a, 2047 @@ -211,8 +206,7 @@ define i64 @vsetvl_e8mf2_and10bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e8mf4_and12bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e8mf4_and12bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 0, i64 6) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 0, i64 6) %b = and i64 %a, 4095 @@ -233,8 +227,7 @@ define i64 @vsetvl_e8mf4_and11bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e8mf8_and13bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e8mf8_and13bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 0, i64 7) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 0, i64 7) %b = and i64 %a, 8191 @@ -255,8 +248,7 @@ define i64 @vsetvl_e8mf8_and12bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e16m1_and13bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e16m1_and13bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 1, i64 0) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 1, i64 0) %b = and i64 %a, 8191 @@ -277,8 +269,7 @@ define i64 @vsetvl_e16m1_and12bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e16m2_and14bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e16m2_and14bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 1, i64 1) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 16383 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 1, i64 1) %b = and i64 %a, 16383 @@ -299,8 +290,7 @@ define i64 @vsetvl_e16m2_and13bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e16m4_and15bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e16m4_and15bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 1, i64 2) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 32767 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 1, i64 2) %b = and i64 %a, 32767 @@ -321,8 +311,7 @@ define i64 @vsetvl_e16m4_and14bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e16m8_and16bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e16m8_and16bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 1, i64 3) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 65535 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 1, i64 3) %b = and i64 %a, 65535 @@ -343,8 +332,7 @@ define i64 @vsetvl_e16m8_and15bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e16mf2_and10bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e16mf2_and10bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 1, i64 5) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 1023 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 1, i64 5) %b = and i64 %a, 1023 @@ -365,8 +353,7 @@ define i64 @vsetvl_e16mf2_and9bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e16mf4_and11bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e16mf4_and11bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 1, i64 6) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 1, i64 6) %b = and i64 %a, 2047 @@ -387,8 +374,7 @@ define i64 @vsetvl_e16mf4_and10bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e16mf8_and12bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e16mf8_and12bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 1, i64 7) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 1, i64 7) %b = and i64 %a, 4095 @@ -409,8 +395,7 @@ define i64 @vsetvl_e16mf8_and11bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e32m1_and12bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e32m1_and12bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 2, i64 0) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 2, i64 0) %b = and i64 %a, 4095 @@ -431,8 +416,7 @@ define i64 @vsetvl_e32m1_and11bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e32m2_and13bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e32m2_and13bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 2, i64 1) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 2, i64 1) %b = and i64 %a, 8191 @@ -453,8 +437,7 @@ define i64 @vsetvl_e32m2_and12bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e32m4_and14bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e32m4_and14bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 2, i64 2) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 16383 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 2, i64 2) %b = and i64 %a, 16383 @@ -475,8 +458,7 @@ define i64 @vsetvl_e32m4_and13bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e32m8_and15bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e32m8_and15bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 2, i64 3) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 32767 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 2, i64 3) %b = and i64 %a, 32767 @@ -497,8 +479,7 @@ define i64 @vsetvl_e32m8_and14bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e32mf2_and9bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e32mf2_and9bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 2, i64 5) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 511 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 2, i64 5) %b = and i64 %a, 511 @@ -519,8 +500,7 @@ define i64 @vsetvl_e32mf2_and8bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e32mf4_and10bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e32mf4_and10bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 2, i64 6) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 1023 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 2, i64 6) %b = and i64 %a, 1023 @@ -541,8 +521,7 @@ define i64 @vsetvl_e32mf4_and9bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e32mf8_and11bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e32mf8_and11bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 2, i64 7) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 2, i64 7) %b = and i64 %a, 2047 @@ -563,8 +542,7 @@ define i64 @vsetvl_e32mf8_and10bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e64m1_and11bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e64m1_and11bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 3, i64 0) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 3, i64 0) %b = and i64 %a, 2047 @@ -585,8 +563,7 @@ define i64 @vsetvl_e64m1_and10bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e64m2_and12bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e64m2_and12bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 3, i64 1) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 3, i64 1) %b = and i64 %a, 4095 @@ -607,8 +584,7 @@ define i64 @vsetvl_e64m2_and11bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e64m4_and13bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e64m4_and13bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 3, i64 2) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 3, i64 2) %b = and i64 %a, 8191 @@ -629,8 +605,7 @@ define i64 @vsetvl_e64m4_and12bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e64m8_and14bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e64m8_and14bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 3, i64 3) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 16383 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 3, i64 3) %b = and i64 %a, 16383 @@ -651,8 +626,7 @@ define i64 @vsetvl_e64m8_and13bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e64mf2_and8bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e64mf2_and8bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 3, i64 5) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 255 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 3, i64 5) %b = and i64 %a, 255 @@ -673,8 +647,7 @@ define i64 @vsetvl_e64mf2_and7bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e64mf4_and9bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e64mf4_and9bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 3, i64 6) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 511 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 3, i64 6) %b = and i64 %a, 511 @@ -695,8 +668,7 @@ define i64 @vsetvl_e64mf4_and8bits(i64 %avl) nounwind #0 { define i64 @vsetvl_e64mf8_and10bits(i64 %avl) nounwind #0 { ; CHECK-LABEL: @vsetvl_e64mf8_and10bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvli.i64(i64 [[AVL:%.*]], i64 3, i64 7) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 1023 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvli(i64 %avl, i64 3, i64 7) %b = and i64 %a, 1023 diff --git a/llvm/test/Transforms/InstCombine/RISCV/riscv-vsetvlimax-knownbits.ll b/llvm/test/Transforms/InstCombine/RISCV/riscv-vsetvlimax-knownbits.ll index 72033b3af09f..093ba75e87b5 100644 --- a/llvm/test/Transforms/InstCombine/RISCV/riscv-vsetvlimax-knownbits.ll +++ b/llvm/test/Transforms/InstCombine/RISCV/riscv-vsetvlimax-knownbits.ll @@ -91,8 +91,7 @@ entry: define i64 @vsetvlmax_e8m1_and14bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e8m1_and14bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 0) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 16383 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 0, i64 0) %b = and i64 %a, 16383 @@ -113,8 +112,7 @@ define i64 @vsetvlmax_e8m1_and13bits() nounwind #0 { define i64 @vsetvlmax_e8m2_and15bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e8m2_and15bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 1) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 32767 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 0, i64 1) %b = and i64 %a, 32767 @@ -135,8 +133,7 @@ define i64 @vsetvlmax_e8m2_and14bits() nounwind #0 { define i64 @vsetvlmax_e8m4_and16bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e8m4_and16bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 2) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 65535 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 0, i64 2) %b = and i64 %a, 65535 @@ -178,8 +175,7 @@ define i64 @vsetvlmax_e8m8_and16bits() nounwind #0 { define i64 @vsetvlmax_e8mf2_and11bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e8mf2_and11bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 5) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 0, i64 5) %b = and i64 %a, 2047 @@ -200,8 +196,7 @@ define i64 @vsetvlmax_e8mf2_and10bits() nounwind #0 { define i64 @vsetvlmax_e8mf4_and12bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e8mf4_and12bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 6) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 0, i64 6) %b = and i64 %a, 4095 @@ -222,8 +217,7 @@ define i64 @vsetvlmax_e8mf4_and11bits() nounwind #0 { define i64 @vsetvlmax_e8mf8_and13bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e8mf8_and13bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 0, i64 7) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 0, i64 7) %b = and i64 %a, 8191 @@ -244,8 +238,7 @@ define i64 @vsetvlmax_e8mf8_and12bits() nounwind #0 { define i64 @vsetvlmax_e16m1_and13bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e16m1_and13bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 0) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 1, i64 0) %b = and i64 %a, 8191 @@ -266,8 +259,7 @@ define i64 @vsetvlmax_e16m1_and12bits() nounwind #0 { define i64 @vsetvlmax_e16m2_and14bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e16m2_and14bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 1) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 16383 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 1, i64 1) %b = and i64 %a, 16383 @@ -288,8 +280,7 @@ define i64 @vsetvlmax_e16m2_and13bits() nounwind #0 { define i64 @vsetvlmax_e16m4_and15bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e16m4_and15bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 2) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 32767 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 1, i64 2) %b = and i64 %a, 32767 @@ -310,8 +301,7 @@ define i64 @vsetvlmax_e16m4_and14bits() nounwind #0 { define i64 @vsetvlmax_e16m8_and16bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e16m8_and16bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 3) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 65535 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 1, i64 3) %b = and i64 %a, 65535 @@ -332,8 +322,7 @@ define i64 @vsetvlmax_e16m8_and15bits() nounwind #0 { define i64 @vsetvlmax_e16mf2_and10bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e16mf2_and10bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 5) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 1023 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 1, i64 5) %b = and i64 %a, 1023 @@ -354,8 +343,7 @@ define i64 @vsetvlmax_e16mf2_and9bits() nounwind #0 { define i64 @vsetvlmax_e16mf4_and11bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e16mf4_and11bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 6) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 1, i64 6) %b = and i64 %a, 2047 @@ -376,8 +364,7 @@ define i64 @vsetvlmax_e16mf4_and10bits() nounwind #0 { define i64 @vsetvlmax_e16mf8_and12bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e16mf8_and12bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 1, i64 7) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 1, i64 7) %b = and i64 %a, 4095 @@ -398,8 +385,7 @@ define i64 @vsetvlmax_e16mf8_and11bits() nounwind #0 { define i64 @vsetvlmax_e32m1_and12bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e32m1_and12bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 0) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 2, i64 0) %b = and i64 %a, 4095 @@ -420,8 +406,7 @@ define i64 @vsetvlmax_e32m1_and11bits() nounwind #0 { define i64 @vsetvlmax_e32m2_and13bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e32m2_and13bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 1) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 2, i64 1) %b = and i64 %a, 8191 @@ -442,8 +427,7 @@ define i64 @vsetvlmax_e32m2_and12bits() nounwind #0 { define i64 @vsetvlmax_e32m4_and14bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e32m4_and14bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 2) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 16383 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 2, i64 2) %b = and i64 %a, 16383 @@ -464,8 +448,7 @@ define i64 @vsetvlmax_e32m4_and13bits() nounwind #0 { define i64 @vsetvlmax_e32m8_and15bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e32m8_and15bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 3) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 32767 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 2, i64 3) %b = and i64 %a, 32767 @@ -486,8 +469,7 @@ define i64 @vsetvlmax_e32m8_and14bits() nounwind #0 { define i64 @vsetvlmax_e32mf2_and9bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e32mf2_and9bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 5) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 511 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 2, i64 5) %b = and i64 %a, 511 @@ -508,8 +490,7 @@ define i64 @vsetvlmax_e32mf2_and8bits() nounwind #0 { define i64 @vsetvlmax_e32mf4_and10bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e32mf4_and10bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 6) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 1023 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 2, i64 6) %b = and i64 %a, 1023 @@ -530,8 +511,7 @@ define i64 @vsetvlmax_e32mf4_and9bits() nounwind #0 { define i64 @vsetvlmax_e32mf8_and11bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e32mf8_and11bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 2, i64 7) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 2, i64 7) %b = and i64 %a, 2047 @@ -552,8 +532,7 @@ define i64 @vsetvlmax_e32mf8_and10bits() nounwind #0 { define i64 @vsetvlmax_e64m1_and11bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e64m1_and11bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 0) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 2047 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 3, i64 0) %b = and i64 %a, 2047 @@ -574,8 +553,7 @@ define i64 @vsetvlmax_e64m1_and10bits() nounwind #0 { define i64 @vsetvlmax_e64m2_and12bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e64m2_and12bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 1) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 4095 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 3, i64 1) %b = and i64 %a, 4095 @@ -596,8 +574,7 @@ define i64 @vsetvlmax_e64m2_and11bits() nounwind #0 { define i64 @vsetvlmax_e64m4_and13bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e64m4_and13bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 2) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 8191 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 3, i64 2) %b = and i64 %a, 8191 @@ -618,8 +595,7 @@ define i64 @vsetvlmax_e64m4_and12bits() nounwind #0 { define i64 @vsetvlmax_e64m8_and14bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e64m8_and14bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 3) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 16383 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 3, i64 3) %b = and i64 %a, 16383 @@ -640,8 +616,7 @@ define i64 @vsetvlmax_e64m8_and13bits() nounwind #0 { define i64 @vsetvlmax_e64mf2_and8bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e64mf2_and8bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 5) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 255 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 3, i64 5) %b = and i64 %a, 255 @@ -662,8 +637,7 @@ define i64 @vsetvlmax_e64mf2_and7bits() nounwind #0 { define i64 @vsetvlmax_e64mf4_and9bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e64mf4_and9bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 6) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 511 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 3, i64 6) %b = and i64 %a, 511 @@ -684,8 +658,7 @@ define i64 @vsetvlmax_e64mf4_and8bits() nounwind #0 { define i64 @vsetvlmax_e64mf8_and10bits() nounwind #0 { ; CHECK-LABEL: @vsetvlmax_e64mf8_and10bits( ; CHECK-NEXT: [[A:%.*]] = call i64 @llvm.riscv.vsetvlimax.i64(i64 3, i64 7) -; CHECK-NEXT: [[B:%.*]] = and i64 [[A]], 1023 -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: ret i64 [[A]] ; %a = call i64 @llvm.riscv.vsetvlimax(i64 3, i64 7) %b = and i64 %a, 1023 -- GitLab From 6bf61283ed0c4e05ef29d9cd935e82fe2dd72a99 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Wed, 6 Mar 2024 06:59:20 +0000 Subject: [PATCH 255/929] [gn build] Port 60822637bf00 --- llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn index 327b10c085c5..b252b9fc41dd 100644 --- a/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn @@ -125,6 +125,7 @@ static_library("CodeGen") { "MachineCSE.cpp", "MachineCheckDebugify.cpp", "MachineCombiner.cpp", + "MachineConvergenceVerifier.cpp", "MachineCopyPropagation.cpp", "MachineCycleAnalysis.cpp", "MachineDebugify.cpp", -- GitLab From 5ff3f6604b3a060376db74e454b5609de35b75e1 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Tue, 5 Mar 2024 23:39:19 -0800 Subject: [PATCH 256/929] [ELF] Improve wildcard tests for input section descriptions --- lld/test/ELF/linkerscript/wildcards.s | 54 +++++++++++++++++++++------ 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/lld/test/ELF/linkerscript/wildcards.s b/lld/test/ELF/linkerscript/wildcards.s index c7fbcab44877..84a3bf2887f0 100644 --- a/lld/test/ELF/linkerscript/wildcards.s +++ b/lld/test/ELF/linkerscript/wildcards.s @@ -1,12 +1,12 @@ # REQUIRES: x86 -# RUN: split-file %s %t -# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %t/asm -o %t.o +# RUN: rm -rf %t && split-file %s %t && cd %t +# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %t/asm -o a.o ## Default case: abc and abx included in text. # RUN: echo "SECTIONS { \ # RUN: .text : { *(.abc .abx) } }" > %t.script -# RUN: ld.lld -o %t.out --script %t.script %t.o -# RUN: llvm-objdump --section-headers %t.out | \ +# RUN: ld.lld -o out --script %t.script a.o +# RUN: llvm-objdump --section-headers out | \ # RUN: FileCheck -check-prefix=SEC-DEFAULT %s # SEC-DEFAULT: Sections: # SEC-DEFAULT-NEXT: Idx Name Size @@ -23,15 +23,15 @@ ## Now replace the symbol with '?' and check that results are the same. # RUN: echo "SECTIONS { \ # RUN: .text : { *(.abc .ab?) } }" > %t.script -# RUN: ld.lld -o %t.out --script %t.script %t.o -# RUN: llvm-objdump --section-headers %t.out | \ +# RUN: ld.lld -o out --script %t.script a.o +# RUN: llvm-objdump --section-headers out | \ # RUN: FileCheck -check-prefix=SEC-DEFAULT %s ## Now see how replacing '?' with '*' will consume whole abcd. # RUN: echo "SECTIONS { \ # RUN: .text : { *(.abc .ab*) } }" > %t.script -# RUN: ld.lld -o %t.out --script %t.script %t.o -# RUN: llvm-objdump --section-headers %t.out | \ +# RUN: ld.lld -o out --script %t.script a.o +# RUN: llvm-objdump --section-headers out | \ # RUN: FileCheck -check-prefix=SEC-ALL %s # SEC-ALL: Sections: # SEC-ALL-NEXT: Idx Name Size @@ -47,8 +47,8 @@ ## All sections started with .a are merged. # RUN: echo "SECTIONS { \ # RUN: .text : { *(.a*) } }" > %t.script -# RUN: ld.lld -o %t.out --script %t.script %t.o -# RUN: llvm-objdump --section-headers %t.out | \ +# RUN: ld.lld -o out --script %t.script a.o +# RUN: llvm-objdump --section-headers out | \ # RUN: FileCheck -check-prefix=SEC-NO %s # SEC-NO: Sections: # SEC-NO-NEXT: Idx Name Size @@ -84,10 +84,40 @@ .globl _start _start: +#--- bracket.lds +# RUN: ld.lld -T bracket.lds a.o -o out +# RUN: llvm-objdump --section-headers out | FileCheck %s --check-prefix=SEC-DEFAULT +SECTIONS { + .text : { *([.]abc .ab[v-y] ) } +} + +## Test a few non-wildcard meta characters rejected by GNU ld. + +#--- lbrace.lds +# RUN: ld.lld -T lbrace.lds a.o -o out +SECTIONS { + .text : { *(.a* { ) } +} + #--- lparen.lds ## ( is recognized as a section name pattern. Note, ( is rejected by GNU ld. -# RUN: ld.lld -T %t/lparen.lds %t.o -o %t.out -# RUN: llvm-objdump --section-headers %t.out | FileCheck --check-prefix=SEC-NO %s +# RUN: ld.lld -T lparen.lds a.o -o out +# RUN: llvm-objdump --section-headers out | FileCheck --check-prefix=SEC-NO %s SECTIONS { .text : { *(.a* ( ) } } + +#--- rbrace.lds +# RUN: ld.lld -T rbrace.lds a.o -o out +SECTIONS { + .text : { *(.a* } ) } +} + +#--- rparen.lds +# RUN: not ld.lld -T rparen.lds %t.o 2>&1 | FileCheck %s --check-prefix=ERR-RPAREN --match-full-lines --strict-whitespace +# ERR-RPAREN:{{.*}}: expected filename pattern +# ERR-RPAREN-NEXT:>>> .text : { *(.a* ) ) } +# ERR-RPAREN-NEXT:>>> ^ +SECTIONS { + .text : { *(.a* ) ) } +} -- GitLab From d3e79e4cc33b89c61a8763a130f60a443eed4775 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Tue, 5 Mar 2024 23:43:39 -0800 Subject: [PATCH 257/929] [ELF] Improve wildcard test --- lld/test/ELF/linkerscript/wildcards.s | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lld/test/ELF/linkerscript/wildcards.s b/lld/test/ELF/linkerscript/wildcards.s index 84a3bf2887f0..1eea27891dfc 100644 --- a/lld/test/ELF/linkerscript/wildcards.s +++ b/lld/test/ELF/linkerscript/wildcards.s @@ -1,11 +1,11 @@ # REQUIRES: x86 # RUN: rm -rf %t && split-file %s %t && cd %t -# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %t/asm -o a.o +# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux asm -o a.o ## Default case: abc and abx included in text. # RUN: echo "SECTIONS { \ -# RUN: .text : { *(.abc .abx) } }" > %t.script -# RUN: ld.lld -o out --script %t.script a.o +# RUN: .text : { *(.abc .abx) } }" > a.t +# RUN: ld.lld -o out --script a.t a.o # RUN: llvm-objdump --section-headers out | \ # RUN: FileCheck -check-prefix=SEC-DEFAULT %s # SEC-DEFAULT: Sections: @@ -22,15 +22,15 @@ ## Now replace the symbol with '?' and check that results are the same. # RUN: echo "SECTIONS { \ -# RUN: .text : { *(.abc .ab?) } }" > %t.script -# RUN: ld.lld -o out --script %t.script a.o +# RUN: .text : { *(.abc .ab?) } }" > b.t +# RUN: ld.lld -o out -T b.t a.o # RUN: llvm-objdump --section-headers out | \ # RUN: FileCheck -check-prefix=SEC-DEFAULT %s ## Now see how replacing '?' with '*' will consume whole abcd. # RUN: echo "SECTIONS { \ -# RUN: .text : { *(.abc .ab*) } }" > %t.script -# RUN: ld.lld -o out --script %t.script a.o +# RUN: .text : { *(.abc .ab*) } }" > c.t +# RUN: ld.lld -o out --script c.t a.o # RUN: llvm-objdump --section-headers out | \ # RUN: FileCheck -check-prefix=SEC-ALL %s # SEC-ALL: Sections: @@ -46,8 +46,8 @@ ## All sections started with .a are merged. # RUN: echo "SECTIONS { \ -# RUN: .text : { *(.a*) } }" > %t.script -# RUN: ld.lld -o out --script %t.script a.o +# RUN: .text : { *(.a*) } }" > d.t +# RUN: ld.lld -o out --script d.t a.o # RUN: llvm-objdump --section-headers out | \ # RUN: FileCheck -check-prefix=SEC-NO %s # SEC-NO: Sections: @@ -114,7 +114,7 @@ SECTIONS { } #--- rparen.lds -# RUN: not ld.lld -T rparen.lds %t.o 2>&1 | FileCheck %s --check-prefix=ERR-RPAREN --match-full-lines --strict-whitespace +# RUN: not ld.lld -T rparen.lds a.o 2>&1 | FileCheck %s --check-prefix=ERR-RPAREN --match-full-lines --strict-whitespace # ERR-RPAREN:{{.*}}: expected filename pattern # ERR-RPAREN-NEXT:>>> .text : { *(.a* ) ) } # ERR-RPAREN-NEXT:>>> ^ -- GitLab From d3df2a834cf6febb44c699d109b9e7f622194837 Mon Sep 17 00:00:00 2001 From: Chuanqi Xu Date: Wed, 6 Mar 2024 15:34:37 +0800 Subject: [PATCH 258/929] [C++20] [Modules] Handle transitive import in the module properly Close https://github.com/llvm/llvm-project/issues/84002 Per [module.import]p7: > Additionally, when a module-import-declaration in a module unit of > some module M imports another module unit U of M, it also imports all > translation units imported by non-exported module-import-declarations > in the module unit purview of U. However, we only tried to implement it during the implicit import of primary module interface for module implementation unit. Also we didn't implement the last sentence from [module.import]p7 completely: > These rules can in turn lead to the importation of yet more > translation units. This patch tries to care the both issues. --- clang/docs/ReleaseNotes.rst | 5 + clang/include/clang/Basic/Module.h | 11 +-- clang/lib/Basic/Module.cpp | 8 -- clang/lib/Sema/SemaModule.cpp | 94 ++++++++++++++++++- clang/test/Modules/transitive-import.cppm | 109 ++++++++++++++++++++++ 5 files changed, 210 insertions(+), 17 deletions(-) create mode 100644 clang/test/Modules/transitive-import.cppm diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 5e0352a7eaf6..b074055a4eaf 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -86,6 +86,11 @@ C++20 Feature Support - Implemented the `__is_layout_compatible` intrinsic to support `P0466R5: Layout-compatibility and Pointer-interconvertibility Traits `_. +- Clang now implements [module.import]p7 fully. Clang now will import module + units transitively for the module units coming from the same module of the + current module units. + Fixes `#84002 `_. + C++23 Feature Support ^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h index 30ec9c993150..9f62c058ca0d 100644 --- a/clang/include/clang/Basic/Module.h +++ b/clang/include/clang/Basic/Module.h @@ -598,6 +598,11 @@ public: Kind == ModulePartitionImplementation; } + /// Is this a module partition implementation unit. + bool isModulePartitionImplementation() const { + return Kind == ModulePartitionImplementation; + } + /// Is this a module implementation. bool isModuleImplementation() const { return Kind == ModuleImplementationUnit; @@ -853,12 +858,6 @@ public: VisibleCallback Vis = [](Module *) {}, ConflictCallback Cb = [](ArrayRef, Module *, StringRef) {}); - - /// Make transitive imports visible for [module.import]/7. - void makeTransitiveImportsVisible( - Module *M, SourceLocation Loc, VisibleCallback Vis = [](Module *) {}, - ConflictCallback Cb = [](ArrayRef, Module *, StringRef) {}); - private: /// Import locations for each visible module. Indexed by the module's /// VisibilityID. diff --git a/clang/lib/Basic/Module.cpp b/clang/lib/Basic/Module.cpp index 1c5043a618ff..9f597dcf8b0f 100644 --- a/clang/lib/Basic/Module.cpp +++ b/clang/lib/Basic/Module.cpp @@ -722,14 +722,6 @@ void VisibleModuleSet::setVisible(Module *M, SourceLocation Loc, VisitModule({M, nullptr}); } -void VisibleModuleSet::makeTransitiveImportsVisible(Module *M, - SourceLocation Loc, - VisibleCallback Vis, - ConflictCallback Cb) { - for (auto *I : M->Imports) - setVisible(I, Loc, Vis, Cb); -} - ASTSourceDescriptor::ASTSourceDescriptor(Module &M) : Signature(M.Signature), ClangModule(&M) { if (M.Directory) diff --git a/clang/lib/Sema/SemaModule.cpp b/clang/lib/Sema/SemaModule.cpp index ed7f626971f3..f08c1cb3a13e 100644 --- a/clang/lib/Sema/SemaModule.cpp +++ b/clang/lib/Sema/SemaModule.cpp @@ -73,6 +73,90 @@ static std::string stringFromPath(ModuleIdPath Path) { return Name; } +/// Helper function for makeTransitiveImportsVisible to decide whether +/// the \param Imported module unit is in the same module with the \param +/// CurrentModule. +/// \param FoundPrimaryModuleInterface is a helper parameter to record the +/// primary module interface unit corresponding to the module \param +/// CurrentModule. Since currently it is expensive to decide whether two module +/// units come from the same module by comparing the module name. +static bool +isImportingModuleUnitFromSameModule(Module *Imported, Module *CurrentModule, + Module *&FoundPrimaryModuleInterface) { + if (!Imported->isNamedModule()) + return false; + + // The a partition unit we're importing must be in the same module of the + // current module. + if (Imported->isModulePartition()) + return true; + + // If we found the primary module interface during the search process, we can + // return quickly to avoid expensive string comparison. + if (FoundPrimaryModuleInterface) + return Imported == FoundPrimaryModuleInterface; + + if (!CurrentModule) + return false; + + // Then the imported module must be a primary module interface unit. It + // is only allowed to import the primary module interface unit from the same + // module in the implementation unit and the implementation partition unit. + + // Since we'll handle implementation unit above. We can only care + // about the implementation partition unit here. + if (!CurrentModule->isModulePartitionImplementation()) + return false; + + if (Imported->getPrimaryModuleInterfaceName() == + CurrentModule->getPrimaryModuleInterfaceName()) { + assert(!FoundPrimaryModuleInterface || + FoundPrimaryModuleInterface == Imported); + FoundPrimaryModuleInterface = Imported; + return true; + } + + return false; +} + +/// [module.import]p7: +/// Additionally, when a module-import-declaration in a module unit of some +/// module M imports another module unit U of M, it also imports all +/// translation units imported by non-exported module-import-declarations in +/// the module unit purview of U. These rules can in turn lead to the +/// importation of yet more translation units. +static void +makeTransitiveImportsVisible(VisibleModuleSet &VisibleModules, Module *Imported, + Module *CurrentModule, SourceLocation ImportLoc, + bool IsImportingPrimaryModuleInterface = false) { + assert(Imported->isNamedModule() && + "'makeTransitiveImportsVisible()' is intended for standard C++ named " + "modules only."); + + llvm::SmallVector Worklist; + Worklist.push_back(Imported); + + Module *FoundPrimaryModuleInterface = + IsImportingPrimaryModuleInterface ? Imported : nullptr; + + while (!Worklist.empty()) { + Module *Importing = Worklist.pop_back_val(); + + if (VisibleModules.isVisible(Importing)) + continue; + + // FIXME: The ImportLoc here is not meaningful. It may be problematic if we + // use the sourcelocation loaded from the visible modules. + VisibleModules.setVisible(Importing, ImportLoc); + + if (isImportingModuleUnitFromSameModule(Importing, CurrentModule, + FoundPrimaryModuleInterface)) + for (Module *TransImported : Importing->Imports) + if (!VisibleModules.isVisible(TransImported)) + Worklist.push_back(TransImported); + } +} + Sema::DeclGroupPtrTy Sema::ActOnGlobalModuleFragmentDecl(SourceLocation ModuleLoc) { // We start in the global module; @@ -396,8 +480,8 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc, // and return the import decl to be added to the current TU. if (Interface) { - VisibleModules.setVisible(Interface, ModuleLoc); - VisibleModules.makeTransitiveImportsVisible(Interface, ModuleLoc); + makeTransitiveImportsVisible(VisibleModules, Interface, Mod, ModuleLoc, + /*IsImportingPrimaryModuleInterface=*/true); // Make the import decl for the interface in the impl module. ImportDecl *Import = ImportDecl::Create(Context, CurContext, ModuleLoc, @@ -554,7 +638,11 @@ DeclResult Sema::ActOnModuleImport(SourceLocation StartLoc, if (Mod->isHeaderUnit()) Diag(ImportLoc, diag::warn_experimental_header_unit); - VisibleModules.setVisible(Mod, ImportLoc); + if (Mod->isNamedModule()) + makeTransitiveImportsVisible(VisibleModules, Mod, getCurrentModule(), + ImportLoc); + else + VisibleModules.setVisible(Mod, ImportLoc); checkModuleImportContext(*this, Mod, ImportLoc, CurContext); diff --git a/clang/test/Modules/transitive-import.cppm b/clang/test/Modules/transitive-import.cppm new file mode 100644 index 000000000000..0eed8cfe2f0a --- /dev/null +++ b/clang/test/Modules/transitive-import.cppm @@ -0,0 +1,109 @@ +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 %t/Invisible.cppm -emit-module-interface -o %t/Invisible.pcm +// RUN: %clang_cc1 -std=c++20 %t/Other.cppm -emit-module-interface -fprebuilt-module-path=%t \ +// RUN: -o %t/Other.pcm +// RUN: %clang_cc1 -std=c++20 %t/Another.cppm -emit-module-interface -o %t/Another.pcm +// RUN: %clang_cc1 -std=c++20 %t/A-interface.cppm -emit-module-interface \ +// RUN: -fprebuilt-module-path=%t -o %t/A-interface.pcm +// RUN: %clang_cc1 -std=c++20 %t/A-interface2.cppm -emit-module-interface \ +// RUN: -fprebuilt-module-path=%t -o %t/A-interface2.pcm +// RUN: %clang_cc1 -std=c++20 %t/A-interface3.cppm -emit-module-interface \ +// RUN: -fprebuilt-module-path=%t -o %t/A-interface3.pcm +// RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-module-interface \ +// RUN: -fprebuilt-module-path=%t -o %t/A.pcm + +// RUN: %clang_cc1 -std=c++20 %t/A.cpp -fprebuilt-module-path=%t -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 %t/A-impl.cppm -fprebuilt-module-path=%t -fsyntax-only -verify + +// RUN: %clang_cc1 -std=c++20 %t/A-impl2.cppm -fprebuilt-module-path=%t -fsyntax-only -verify + +//--- Invisible.cppm +export module Invisible; +export void invisible() {} + +//--- Other.cppm +export module Other; +import Invisible; +export void other() {} + +//--- Another.cppm +export module Another; +export void another() {} + +//--- A-interface.cppm +export module A:interface; +import Other; +export void a_interface() {} + +//--- A-interface2.cppm +export module A:interface2; +import Another; +export void a_interface2() {} + +//--- A-interface3.cppm +export module A:interface3; +import :interface; +import :interface2; +export void a_interface3() {} + +//--- A.cppm +export module A; +import Another; +import :interface; +import :interface2; +import :interface3; + +export void a() {} +export void impl(); + +//--- A.cpp +module A; +void impl() { + a_interface(); + a_interface2(); + a_interface3(); + + other(); + another(); + + invisible(); // expected-error {{declaration of 'invisible' must be imported from module 'Invisible' before it is required}} + // expected-note@* {{declaration here is not visible}} +} + +//--- A-impl.cppm +module A:impl; +import :interface3; + +void impl_part() { + a_interface(); + a_interface2(); + a_interface3(); + + other(); + another(); + + invisible(); // expected-error {{declaration of 'invisible' must be imported from module 'Invisible' before it is required}} + // expected-note@* {{declaration here is not visible}} +} + +//--- A-impl2.cppm +module A:impl2; +import A; + +void impl_part2() { + a(); + impl(); + + a_interface(); + a_interface2(); + a_interface3(); + + other(); + another(); + + invisible(); // expected-error {{declaration of 'invisible' must be imported from module 'Invisible' before it is required}} + // expected-note@* {{declaration here is not visible}} +} -- GitLab From bec7ad9fd6bceb6521448b24faddb01bc52de3a7 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Wed, 6 Mar 2024 14:58:42 +0800 Subject: [PATCH 259/929] [RISCV] Add tests for vw{add,sub,mul} with nested extend. NFC These test cases show (op (ext a), (ext b)) patterns where the dest EEW is more than 2 * source EEW. These could be lowered into widening ops where we still have extend the operands, but at a smaller EEW. --- llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll | 1088 +++++++++++++++++-- llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll | 864 ++++++++++++++- llvm/test/CodeGen/RISCV/rvv/vwsub-sdnode.ll | 1072 ++++++++++++++++-- 3 files changed, 2848 insertions(+), 176 deletions(-) diff --git a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll index 89e69565b39e..a559fbf2bc8a 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll @@ -2,8 +2,8 @@ ; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s | FileCheck %s ; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck %s -define @vwadd_vv_nxv1i64( %va, %vb) { -; CHECK-LABEL: vwadd_vv_nxv1i64: +define @vwadd_vv_nxv1i64_nxv1i32( %va, %vb) { +; CHECK-LABEL: vwadd_vv_nxv1i64_nxv1i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma ; CHECK-NEXT: vwadd.vv v10, v8, v9 @@ -15,8 +15,8 @@ define @vwadd_vv_nxv1i64( %va, %ve } -define @vwaddu_vv_nxv1i64( %va, %vb) { -; CHECK-LABEL: vwaddu_vv_nxv1i64: +define @vwaddu_vv_nxv1i64_nxv1i32( %va, %vb) { +; CHECK-LABEL: vwaddu_vv_nxv1i64_nxv1i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma ; CHECK-NEXT: vwaddu.vv v10, v8, v9 @@ -28,8 +28,8 @@ define @vwaddu_vv_nxv1i64( %va, %ve } -define @vwadd_vx_nxv1i64( %va, i32 %b) { -; CHECK-LABEL: vwadd_vx_nxv1i64: +define @vwadd_vx_nxv1i64_nxv1i32( %va, i32 %b) { +; CHECK-LABEL: vwadd_vx_nxv1i64_nxv1i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, mf2, ta, ma ; CHECK-NEXT: vwadd.vx v9, v8, a0 @@ -43,8 +43,8 @@ define @vwadd_vx_nxv1i64( %va, i32 %b) { ret %ve } -define @vwaddu_vx_nxv1i64( %va, i32 %b) { -; CHECK-LABEL: vwaddu_vx_nxv1i64: +define @vwaddu_vx_nxv1i64_nxv1i32( %va, i32 %b) { +; CHECK-LABEL: vwaddu_vx_nxv1i64_nxv1i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, mf2, ta, ma ; CHECK-NEXT: vwaddu.vx v9, v8, a0 @@ -58,8 +58,8 @@ define @vwaddu_vx_nxv1i64( %va, i32 %b) { ret %ve } -define @vwadd_wv_nxv1i64( %va, %vb) { -; CHECK-LABEL: vwadd_wv_nxv1i64: +define @vwadd_wv_nxv1i64_nxv1i32( %va, %vb) { +; CHECK-LABEL: vwadd_wv_nxv1i64_nxv1i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma ; CHECK-NEXT: vwadd.wv v8, v8, v9 @@ -69,8 +69,8 @@ define @vwadd_wv_nxv1i64( %va, %vd } -define @vwaddu_wv_nxv1i64( %va, %vb) { -; CHECK-LABEL: vwaddu_wv_nxv1i64: +define @vwaddu_wv_nxv1i64_nxv1i32( %va, %vb) { +; CHECK-LABEL: vwaddu_wv_nxv1i64_nxv1i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma ; CHECK-NEXT: vwaddu.wv v8, v8, v9 @@ -80,8 +80,8 @@ define @vwaddu_wv_nxv1i64( %va, %vd } -define @vwadd_wx_nxv1i64( %va, i32 %b) { -; CHECK-LABEL: vwadd_wx_nxv1i64: +define @vwadd_wx_nxv1i64_nxv1i32( %va, i32 %b) { +; CHECK-LABEL: vwadd_wx_nxv1i64_nxv1i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, mf2, ta, ma ; CHECK-NEXT: vwadd.wx v8, v8, a0 @@ -93,8 +93,8 @@ define @vwadd_wx_nxv1i64( %va, i32 %b) { ret %vc } -define @vwaddu_wx_nxv1i64( %va, i32 %b) { -; CHECK-LABEL: vwaddu_wx_nxv1i64: +define @vwaddu_wx_nxv1i64_nxv1i32( %va, i32 %b) { +; CHECK-LABEL: vwaddu_wx_nxv1i64_nxv1i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, mf2, ta, ma ; CHECK-NEXT: vwaddu.wx v8, v8, a0 @@ -106,8 +106,8 @@ define @vwaddu_wx_nxv1i64( %va, i32 %b) { ret %vc } -define @vwadd_vv_nxv2i64( %va, %vb) { -; CHECK-LABEL: vwadd_vv_nxv2i64: +define @vwadd_vv_nxv2i64_nxv2i32( %va, %vb) { +; CHECK-LABEL: vwadd_vv_nxv2i64_nxv2i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma ; CHECK-NEXT: vwadd.vv v10, v8, v9 @@ -119,8 +119,8 @@ define @vwadd_vv_nxv2i64( %va, %ve } -define @vwaddu_vv_nxv2i64( %va, %vb) { -; CHECK-LABEL: vwaddu_vv_nxv2i64: +define @vwaddu_vv_nxv2i64_nxv2i32( %va, %vb) { +; CHECK-LABEL: vwaddu_vv_nxv2i64_nxv2i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma ; CHECK-NEXT: vwaddu.vv v10, v8, v9 @@ -132,8 +132,8 @@ define @vwaddu_vv_nxv2i64( %va, %ve } -define @vwadd_vx_nxv2i64( %va, i32 %b) { -; CHECK-LABEL: vwadd_vx_nxv2i64: +define @vwadd_vx_nxv2i64_nxv2i32( %va, i32 %b) { +; CHECK-LABEL: vwadd_vx_nxv2i64_nxv2i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m1, ta, ma ; CHECK-NEXT: vwadd.vx v10, v8, a0 @@ -147,8 +147,8 @@ define @vwadd_vx_nxv2i64( %va, i32 %b) { ret %ve } -define @vwaddu_vx_nxv2i64( %va, i32 %b) { -; CHECK-LABEL: vwaddu_vx_nxv2i64: +define @vwaddu_vx_nxv2i64_nxv2i32( %va, i32 %b) { +; CHECK-LABEL: vwaddu_vx_nxv2i64_nxv2i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m1, ta, ma ; CHECK-NEXT: vwaddu.vx v10, v8, a0 @@ -162,8 +162,8 @@ define @vwaddu_vx_nxv2i64( %va, i32 %b) { ret %ve } -define @vwadd_wv_nxv2i64( %va, %vb) { -; CHECK-LABEL: vwadd_wv_nxv2i64: +define @vwadd_wv_nxv2i64_nxv2i32( %va, %vb) { +; CHECK-LABEL: vwadd_wv_nxv2i64_nxv2i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma ; CHECK-NEXT: vwadd.wv v8, v8, v10 @@ -173,8 +173,8 @@ define @vwadd_wv_nxv2i64( %va, %vd } -define @vwaddu_wv_nxv2i64( %va, %vb) { -; CHECK-LABEL: vwaddu_wv_nxv2i64: +define @vwaddu_wv_nxv2i64_nxv2i32( %va, %vb) { +; CHECK-LABEL: vwaddu_wv_nxv2i64_nxv2i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma ; CHECK-NEXT: vwaddu.wv v8, v8, v10 @@ -184,8 +184,8 @@ define @vwaddu_wv_nxv2i64( %va, %vd } -define @vwadd_wx_nxv2i64( %va, i32 %b) { -; CHECK-LABEL: vwadd_wx_nxv2i64: +define @vwadd_wx_nxv2i64_nxv2i32( %va, i32 %b) { +; CHECK-LABEL: vwadd_wx_nxv2i64_nxv2i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m1, ta, ma ; CHECK-NEXT: vwadd.wx v8, v8, a0 @@ -197,8 +197,8 @@ define @vwadd_wx_nxv2i64( %va, i32 %b) { ret %vc } -define @vwaddu_wx_nxv2i64( %va, i32 %b) { -; CHECK-LABEL: vwaddu_wx_nxv2i64: +define @vwaddu_wx_nxv2i64_nxv2i32( %va, i32 %b) { +; CHECK-LABEL: vwaddu_wx_nxv2i64_nxv2i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m1, ta, ma ; CHECK-NEXT: vwaddu.wx v8, v8, a0 @@ -210,8 +210,8 @@ define @vwaddu_wx_nxv2i64( %va, i32 %b) { ret %vc } -define @vwadd_vv_nxv4i64( %va, %vb) { -; CHECK-LABEL: vwadd_vv_nxv4i64: +define @vwadd_vv_nxv4i64_nxv4i32( %va, %vb) { +; CHECK-LABEL: vwadd_vv_nxv4i64_nxv4i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma ; CHECK-NEXT: vwadd.vv v12, v8, v10 @@ -223,8 +223,8 @@ define @vwadd_vv_nxv4i64( %va, %ve } -define @vwaddu_vv_nxv4i64( %va, %vb) { -; CHECK-LABEL: vwaddu_vv_nxv4i64: +define @vwaddu_vv_nxv4i64_nxv4i32( %va, %vb) { +; CHECK-LABEL: vwaddu_vv_nxv4i64_nxv4i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma ; CHECK-NEXT: vwaddu.vv v12, v8, v10 @@ -236,8 +236,8 @@ define @vwaddu_vv_nxv4i64( %va, %ve } -define @vwadd_vx_nxv4i64( %va, i32 %b) { -; CHECK-LABEL: vwadd_vx_nxv4i64: +define @vwadd_vx_nxv4i64_nxv4i32( %va, i32 %b) { +; CHECK-LABEL: vwadd_vx_nxv4i64_nxv4i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m2, ta, ma ; CHECK-NEXT: vwadd.vx v12, v8, a0 @@ -251,8 +251,8 @@ define @vwadd_vx_nxv4i64( %va, i32 %b) { ret %ve } -define @vwaddu_vx_nxv4i64( %va, i32 %b) { -; CHECK-LABEL: vwaddu_vx_nxv4i64: +define @vwaddu_vx_nxv4i64_nxv4i32( %va, i32 %b) { +; CHECK-LABEL: vwaddu_vx_nxv4i64_nxv4i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m2, ta, ma ; CHECK-NEXT: vwaddu.vx v12, v8, a0 @@ -266,8 +266,8 @@ define @vwaddu_vx_nxv4i64( %va, i32 %b) { ret %ve } -define @vwadd_wv_nxv4i64( %va, %vb) { -; CHECK-LABEL: vwadd_wv_nxv4i64: +define @vwadd_wv_nxv4i64_nxv4i32( %va, %vb) { +; CHECK-LABEL: vwadd_wv_nxv4i64_nxv4i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma ; CHECK-NEXT: vwadd.wv v8, v8, v12 @@ -277,8 +277,8 @@ define @vwadd_wv_nxv4i64( %va, %vd } -define @vwaddu_wv_nxv4i64( %va, %vb) { -; CHECK-LABEL: vwaddu_wv_nxv4i64: +define @vwaddu_wv_nxv4i64_nxv4i32( %va, %vb) { +; CHECK-LABEL: vwaddu_wv_nxv4i64_nxv4i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma ; CHECK-NEXT: vwaddu.wv v8, v8, v12 @@ -288,8 +288,8 @@ define @vwaddu_wv_nxv4i64( %va, %vd } -define @vwadd_wx_nxv4i64( %va, i32 %b) { -; CHECK-LABEL: vwadd_wx_nxv4i64: +define @vwadd_wx_nxv4i64_nxv4i32( %va, i32 %b) { +; CHECK-LABEL: vwadd_wx_nxv4i64_nxv4i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m2, ta, ma ; CHECK-NEXT: vwadd.wx v8, v8, a0 @@ -301,8 +301,8 @@ define @vwadd_wx_nxv4i64( %va, i32 %b) { ret %vc } -define @vwaddu_wx_nxv4i64( %va, i32 %b) { -; CHECK-LABEL: vwaddu_wx_nxv4i64: +define @vwaddu_wx_nxv4i64_nxv4i32( %va, i32 %b) { +; CHECK-LABEL: vwaddu_wx_nxv4i64_nxv4i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m2, ta, ma ; CHECK-NEXT: vwaddu.wx v8, v8, a0 @@ -314,8 +314,8 @@ define @vwaddu_wx_nxv4i64( %va, i32 %b) { ret %vc } -define @vwadd_vv_nxv8i64( %va, %vb) { -; CHECK-LABEL: vwadd_vv_nxv8i64: +define @vwadd_vv_nxv8i64_nxv8i32( %va, %vb) { +; CHECK-LABEL: vwadd_vv_nxv8i64_nxv8i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma ; CHECK-NEXT: vwadd.vv v16, v8, v12 @@ -327,8 +327,8 @@ define @vwadd_vv_nxv8i64( %va, %ve } -define @vwaddu_vv_nxv8i64( %va, %vb) { -; CHECK-LABEL: vwaddu_vv_nxv8i64: +define @vwaddu_vv_nxv8i64_nxv8i32( %va, %vb) { +; CHECK-LABEL: vwaddu_vv_nxv8i64_nxv8i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma ; CHECK-NEXT: vwaddu.vv v16, v8, v12 @@ -340,8 +340,8 @@ define @vwaddu_vv_nxv8i64( %va, %ve } -define @vwadd_vx_nxv8i64( %va, i32 %b) { -; CHECK-LABEL: vwadd_vx_nxv8i64: +define @vwadd_vx_nxv8i64_nxv8i32( %va, i32 %b) { +; CHECK-LABEL: vwadd_vx_nxv8i64_nxv8i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m4, ta, ma ; CHECK-NEXT: vwadd.vx v16, v8, a0 @@ -355,8 +355,8 @@ define @vwadd_vx_nxv8i64( %va, i32 %b) { ret %ve } -define @vwaddu_vx_nxv8i64( %va, i32 %b) { -; CHECK-LABEL: vwaddu_vx_nxv8i64: +define @vwaddu_vx_nxv8i64_nxv8i32( %va, i32 %b) { +; CHECK-LABEL: vwaddu_vx_nxv8i64_nxv8i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m4, ta, ma ; CHECK-NEXT: vwaddu.vx v16, v8, a0 @@ -370,8 +370,8 @@ define @vwaddu_vx_nxv8i64( %va, i32 %b) { ret %ve } -define @vwadd_wv_nxv8i64( %va, %vb) { -; CHECK-LABEL: vwadd_wv_nxv8i64: +define @vwadd_wv_nxv8i64_nxv8i32( %va, %vb) { +; CHECK-LABEL: vwadd_wv_nxv8i64_nxv8i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma ; CHECK-NEXT: vwadd.wv v8, v8, v16 @@ -381,8 +381,8 @@ define @vwadd_wv_nxv8i64( %va, %vd } -define @vwaddu_wv_nxv8i64( %va, %vb) { -; CHECK-LABEL: vwaddu_wv_nxv8i64: +define @vwaddu_wv_nxv8i64_nxv8i32( %va, %vb) { +; CHECK-LABEL: vwaddu_wv_nxv8i64_nxv8i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma ; CHECK-NEXT: vwaddu.wv v8, v8, v16 @@ -392,8 +392,8 @@ define @vwaddu_wv_nxv8i64( %va, %vd } -define @vwadd_wx_nxv8i64( %va, i32 %b) { -; CHECK-LABEL: vwadd_wx_nxv8i64: +define @vwadd_wx_nxv8i64_nxv8i32( %va, i32 %b) { +; CHECK-LABEL: vwadd_wx_nxv8i64_nxv8i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m4, ta, ma ; CHECK-NEXT: vwadd.wx v8, v8, a0 @@ -405,8 +405,8 @@ define @vwadd_wx_nxv8i64( %va, i32 %b) { ret %vc } -define @vwaddu_wx_nxv8i64( %va, i32 %b) { -; CHECK-LABEL: vwaddu_wx_nxv8i64: +define @vwaddu_wx_nxv8i64_nxv8i32( %va, i32 %b) { +; CHECK-LABEL: vwaddu_wx_nxv8i64_nxv8i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m4, ta, ma ; CHECK-NEXT: vwaddu.wx v8, v8, a0 @@ -417,3 +417,963 @@ define @vwaddu_wx_nxv8i64( %va, i32 %b) { %vc = add %va, %vb ret %vc } + +define @vwadd_vv_nxv1i64_nxv1i16( %va, %vb) { +; CHECK-LABEL: vwadd_vv_nxv1i64_nxv1i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v8, v9 +; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = add %vc, %vd + ret %ve +} + +define @vwaddu_vv_nxv1i64_nxv1i16( %va, %vb) { +; CHECK-LABEL: vwaddu_vv_nxv1i64_nxv1i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v8, v9 +; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = add %vc, %vd + ret %ve +} + +define @vwadd_vx_nxv1i64_nxv1i16( %va, i16 %b) { +; CHECK-LABEL: vwadd_vx_nxv1i64_nxv1i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v8, v9 +; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = add %vc, %vd + ret %ve +} + +define @vwaddu_vx_nxv1i64_nxv1i16( %va, i16 %b) { +; CHECK-LABEL: vwaddu_vx_nxv1i64_nxv1i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v8, v9 +; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = add %vc, %vd + ret %ve +} + +define @vwadd_wv_nxv1i64_nxv1i16( %va, %vb) { +; CHECK-LABEL: vwadd_wv_nxv1i64_nxv1i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v9 +; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: ret + %vc = sext %vb to + %vd = add %va, %vc + ret %vd +} + +define @vwaddu_wv_nxv1i64_nxv1i16( %va, %vb) { +; CHECK-LABEL: vwaddu_wv_nxv1i64_nxv1i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v9 +; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: ret + %vc = zext %vb to + %vd = add %va, %vc + ret %vd +} + +define @vwadd_wx_nxv1i64_nxv1i16( %va, i16 %b) { +; CHECK-LABEL: vwadd_wx_nxv1i64_nxv1i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v9 +; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = sext %splat to + %vc = add %va, %vb + ret %vc +} + +define @vwaddu_wx_nxv1i64_nxv1i16( %va, i16 %b) { +; CHECK-LABEL: vwaddu_wx_nxv1i64_nxv1i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v9 +; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = zext %splat to + %vc = add %va, %vb + ret %vc +} + +define @vwadd_vv_nxv2i64_nxv2i16( %va, %vb) { +; CHECK-LABEL: vwadd_vv_nxv2i64_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v12, v9 +; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = add %vc, %vd + ret %ve +} + +define @vwaddu_vv_nxv2i64_nxv2i16( %va, %vb) { +; CHECK-LABEL: vwaddu_vv_nxv2i64_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v12, v9 +; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = add %vc, %vd + ret %ve +} + +define @vwadd_vx_nxv2i64_nxv2i16( %va, i16 %b) { +; CHECK-LABEL: vwadd_vx_nxv2i64_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v12, v9 +; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = add %vc, %vd + ret %ve +} + +define @vwaddu_vx_nxv2i64_nxv2i16( %va, i16 %b) { +; CHECK-LABEL: vwaddu_vx_nxv2i64_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v12, v9 +; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = add %vc, %vd + ret %ve +} + +define @vwadd_wv_nxv2i64_nxv2i16( %va, %vb) { +; CHECK-LABEL: vwadd_wv_nxv2i64_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v10 +; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: ret + %vc = sext %vb to + %vd = add %va, %vc + ret %vd +} + +define @vwaddu_wv_nxv2i64_nxv2i16( %va, %vb) { +; CHECK-LABEL: vwaddu_wv_nxv2i64_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v12, v10 +; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: ret + %vc = zext %vb to + %vd = add %va, %vc + ret %vd +} + +define @vwadd_wx_nxv2i64_nxv2i16( %va, i16 %b) { +; CHECK-LABEL: vwadd_wx_nxv2i64_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v10 +; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = sext %splat to + %vc = add %va, %vb + ret %vc +} + +define @vwaddu_wx_nxv2i64_nxv2i16( %va, i16 %b) { +; CHECK-LABEL: vwaddu_wx_nxv2i64_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v12, v10 +; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = zext %splat to + %vc = add %va, %vb + ret %vc +} + +define @vwadd_vv_nxv4i64_nxv4i16( %va, %vb) { +; CHECK-LABEL: vwadd_vv_nxv4i64_nxv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v16, v9 +; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = add %vc, %vd + ret %ve +} + +define @vwaddu_vv_nxv4i64_nxv4i16( %va, %vb) { +; CHECK-LABEL: vwaddu_vv_nxv4i64_nxv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v16, v9 +; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = add %vc, %vd + ret %ve +} + +define @vwadd_vx_nxv4i64_nxv4i16( %va, i16 %b) { +; CHECK-LABEL: vwadd_vx_nxv4i64_nxv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v16, v9 +; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = add %vc, %vd + ret %ve +} + +define @vwaddu_vx_nxv4i64_nxv4i16( %va, i16 %b) { +; CHECK-LABEL: vwaddu_vx_nxv4i64_nxv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v16, v9 +; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = add %vc, %vd + ret %ve +} + +define @vwadd_wv_nxv4i64_nxv4i16( %va, %vb) { +; CHECK-LABEL: vwadd_wv_nxv4i64_nxv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v12 +; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: ret + %vc = sext %vb to + %vd = add %va, %vc + ret %vd +} + +define @vwaddu_wv_nxv4i64_nxv4i16( %va, %vb) { +; CHECK-LABEL: vwaddu_wv_nxv4i64_nxv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v16, v12 +; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: ret + %vc = zext %vb to + %vd = add %va, %vc + ret %vd +} + +define @vwadd_wx_nxv4i64_nxv4i16( %va, i16 %b) { +; CHECK-LABEL: vwadd_wx_nxv4i64_nxv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma +; CHECK-NEXT: vmv.v.x v12, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v12 +; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = sext %splat to + %vc = add %va, %vb + ret %vc +} + +define @vwaddu_wx_nxv4i64_nxv4i16( %va, i16 %b) { +; CHECK-LABEL: vwaddu_wx_nxv4i64_nxv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma +; CHECK-NEXT: vmv.v.x v12, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v16, v12 +; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = zext %splat to + %vc = add %va, %vb + ret %vc +} + +define @vwadd_vv_nxv8i64_nxv8i16( %va, %vb) { +; CHECK-LABEL: vwadd_vv_nxv8i64_nxv8i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v24, v10 +; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = add %vc, %vd + ret %ve +} + +define @vwaddu_vv_nxv8i64_nxv8i16( %va, %vb) { +; CHECK-LABEL: vwaddu_vv_nxv8i64_nxv8i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v24, v10 +; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = add %vc, %vd + ret %ve +} + +define @vwadd_vx_nxv8i64_nxv8i16( %va, i16 %b) { +; CHECK-LABEL: vwadd_vx_nxv8i64_nxv8i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v24, v10 +; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = add %vc, %vd + ret %ve +} + +define @vwaddu_vx_nxv8i64_nxv8i16( %va, i16 %b) { +; CHECK-LABEL: vwaddu_vx_nxv8i64_nxv8i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v24, v10 +; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = add %vc, %vd + ret %ve +} + +define @vwadd_wv_nxv8i64_nxv8i16( %va, %vb) { +; CHECK-LABEL: vwadd_wv_nxv8i64_nxv8i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf4 v24, v16 +; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: ret + %vc = sext %vb to + %vd = add %va, %vc + ret %vd +} + +define @vwaddu_wv_nxv8i64_nxv8i16( %va, %vb) { +; CHECK-LABEL: vwaddu_wv_nxv8i64_nxv8i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf4 v24, v16 +; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: ret + %vc = zext %vb to + %vd = add %va, %vc + ret %vd +} + +define @vwadd_wx_nxv8i64_nxv8i16( %va, i16 %b) { +; CHECK-LABEL: vwadd_wx_nxv8i64_nxv8i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma +; CHECK-NEXT: vmv.v.x v16, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf4 v24, v16 +; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = sext %splat to + %vc = add %va, %vb + ret %vc +} + +define @vwaddu_wx_nxv8i64_nxv8i16( %va, i16 %b) { +; CHECK-LABEL: vwaddu_wx_nxv8i64_nxv8i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma +; CHECK-NEXT: vmv.v.x v16, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf4 v24, v16 +; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = zext %splat to + %vc = add %va, %vb + ret %vc +} + +define @vwadd_vv_nxv1i64_nxv1i8( %va, %vb) { +; CHECK-LABEL: vwadd_vv_nxv1i64_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf8 v10, v8 +; CHECK-NEXT: vsext.vf8 v8, v9 +; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = add %vc, %vd + ret %ve +} + +define @vwaddu_vv_nxv1i64_nxv1i8( %va, %vb) { +; CHECK-LABEL: vwaddu_vv_nxv1i64_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf8 v10, v8 +; CHECK-NEXT: vzext.vf8 v8, v9 +; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = add %vc, %vd + ret %ve +} + +define @vwadd_vx_nxv1i64_nxv1i8( %va, i8 %b) { +; CHECK-LABEL: vwadd_vx_nxv1i64_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf8 v10, v8 +; CHECK-NEXT: vsext.vf8 v8, v9 +; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = add %vc, %vd + ret %ve +} + +define @vwaddu_vx_nxv1i64_nxv1i8( %va, i8 %b) { +; CHECK-LABEL: vwaddu_vx_nxv1i64_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf8 v10, v8 +; CHECK-NEXT: vzext.vf8 v8, v9 +; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = add %vc, %vd + ret %ve +} + +define @vwadd_wv_nxv1i64_nxv1i8( %va, %vb) { +; CHECK-LABEL: vwadd_wv_nxv1i64_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf8 v10, v9 +; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: ret + %vc = sext %vb to + %vd = add %va, %vc + ret %vd +} + +define @vwaddu_wv_nxv1i64_nxv1i8( %va, %vb) { +; CHECK-LABEL: vwaddu_wv_nxv1i64_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf8 v10, v9 +; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: ret + %vc = zext %vb to + %vd = add %va, %vc + ret %vd +} + +define @vwadd_wx_nxv1i64_nxv1i8( %va, i8 %b) { +; CHECK-LABEL: vwadd_wx_nxv1i64_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf8 v10, v9 +; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = sext %splat to + %vc = add %va, %vb + ret %vc +} + +define @vwaddu_wx_nxv1i64_nxv1i8( %va, i8 %b) { +; CHECK-LABEL: vwaddu_wx_nxv1i64_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf8 v10, v9 +; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = zext %splat to + %vc = add %va, %vb + ret %vc +} + +define @vwadd_vv_nxv2i64_nxv2i8( %va, %vb) { +; CHECK-LABEL: vwadd_vv_nxv2i64_nxv2i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf8 v10, v8 +; CHECK-NEXT: vsext.vf8 v12, v9 +; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = add %vc, %vd + ret %ve +} + +define @vwaddu_vv_nxv2i64_nxv2i8( %va, %vb) { +; CHECK-LABEL: vwaddu_vv_nxv2i64_nxv2i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf8 v10, v8 +; CHECK-NEXT: vzext.vf8 v12, v9 +; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = add %vc, %vd + ret %ve +} + +define @vwadd_vx_nxv2i64_nxv2i8( %va, i8 %b) { +; CHECK-LABEL: vwadd_vx_nxv2i64_nxv2i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf8 v10, v8 +; CHECK-NEXT: vsext.vf8 v12, v9 +; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = add %vc, %vd + ret %ve +} + +define @vwaddu_vx_nxv2i64_nxv2i8( %va, i8 %b) { +; CHECK-LABEL: vwaddu_vx_nxv2i64_nxv2i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf8 v10, v8 +; CHECK-NEXT: vzext.vf8 v12, v9 +; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = add %vc, %vd + ret %ve +} + +define @vwadd_wv_nxv2i64_nxv2i8( %va, %vb) { +; CHECK-LABEL: vwadd_wv_nxv2i64_nxv2i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf8 v12, v10 +; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: ret + %vc = sext %vb to + %vd = add %va, %vc + ret %vd +} + +define @vwaddu_wv_nxv2i64_nxv2i8( %va, %vb) { +; CHECK-LABEL: vwaddu_wv_nxv2i64_nxv2i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf8 v12, v10 +; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: ret + %vc = zext %vb to + %vd = add %va, %vc + ret %vd +} + +define @vwadd_wx_nxv2i64_nxv2i8( %va, i8 %b) { +; CHECK-LABEL: vwadd_wx_nxv2i64_nxv2i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf8 v12, v10 +; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = sext %splat to + %vc = add %va, %vb + ret %vc +} + +define @vwaddu_wx_nxv2i64_nxv2i8( %va, i8 %b) { +; CHECK-LABEL: vwaddu_wx_nxv2i64_nxv2i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf8 v12, v10 +; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = zext %splat to + %vc = add %va, %vb + ret %vc +} + +define @vwadd_vv_nxv4i64_nxv4i8( %va, %vb) { +; CHECK-LABEL: vwadd_vv_nxv4i64_nxv4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf8 v12, v8 +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = add %vc, %vd + ret %ve +} + +define @vwaddu_vv_nxv4i64_nxv4i8( %va, %vb) { +; CHECK-LABEL: vwaddu_vv_nxv4i64_nxv4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf8 v12, v8 +; CHECK-NEXT: vzext.vf8 v16, v9 +; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = add %vc, %vd + ret %ve +} + +define @vwadd_vx_nxv4i64_nxv4i8( %va, i8 %b) { +; CHECK-LABEL: vwadd_vx_nxv4i64_nxv4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf8 v12, v8 +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = add %vc, %vd + ret %ve +} + +define @vwaddu_vx_nxv4i64_nxv4i8( %va, i8 %b) { +; CHECK-LABEL: vwaddu_vx_nxv4i64_nxv4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf8 v12, v8 +; CHECK-NEXT: vzext.vf8 v16, v9 +; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = add %vc, %vd + ret %ve +} + +define @vwadd_wv_nxv4i64_nxv4i8( %va, %vb) { +; CHECK-LABEL: vwadd_wv_nxv4i64_nxv4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf8 v16, v12 +; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: ret + %vc = sext %vb to + %vd = add %va, %vc + ret %vd +} + +define @vwaddu_wv_nxv4i64_nxv4i8( %va, %vb) { +; CHECK-LABEL: vwaddu_wv_nxv4i64_nxv4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf8 v16, v12 +; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: ret + %vc = zext %vb to + %vd = add %va, %vc + ret %vd +} + +define @vwadd_wx_nxv4i64_nxv4i8( %va, i8 %b) { +; CHECK-LABEL: vwadd_wx_nxv4i64_nxv4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v12, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf8 v16, v12 +; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = sext %splat to + %vc = add %va, %vb + ret %vc +} + +define @vwaddu_wx_nxv4i64_nxv4i8( %va, i8 %b) { +; CHECK-LABEL: vwaddu_wx_nxv4i64_nxv4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v12, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf8 v16, v12 +; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = zext %splat to + %vc = add %va, %vb + ret %vc +} + +define @vwadd_vv_nxv8i64_nxv8i8( %va, %vb) { +; CHECK-LABEL: vwadd_vv_nxv8i64_nxv8i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf8 v16, v8 +; CHECK-NEXT: vsext.vf8 v24, v9 +; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = add %vc, %vd + ret %ve +} + +define @vwaddu_vv_nxv8i64_nxv8i8( %va, %vb) { +; CHECK-LABEL: vwaddu_vv_nxv8i64_nxv8i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf8 v16, v8 +; CHECK-NEXT: vzext.vf8 v24, v9 +; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = add %vc, %vd + ret %ve +} + +define @vwadd_vx_nxv8i64_nxv8i8( %va, i8 %b) { +; CHECK-LABEL: vwadd_vx_nxv8i64_nxv8i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf8 v16, v8 +; CHECK-NEXT: vsext.vf8 v24, v9 +; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = add %vc, %vd + ret %ve +} + +define @vwaddu_vx_nxv8i64_nxv8i8( %va, i8 %b) { +; CHECK-LABEL: vwaddu_vx_nxv8i64_nxv8i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf8 v16, v8 +; CHECK-NEXT: vzext.vf8 v24, v9 +; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = add %vc, %vd + ret %ve +} + +define @vwadd_wv_nxv8i64_nxv8i8( %va, %vb) { +; CHECK-LABEL: vwadd_wv_nxv8i64_nxv8i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf8 v24, v16 +; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: ret + %vc = sext %vb to + %vd = add %va, %vc + ret %vd +} + +define @vwaddu_wv_nxv8i64_nxv8i8( %va, %vb) { +; CHECK-LABEL: vwaddu_wv_nxv8i64_nxv8i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf8 v24, v16 +; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: ret + %vc = zext %vb to + %vd = add %va, %vc + ret %vd +} + +define @vwadd_wx_nxv8i64_nxv8i8( %va, i8 %b) { +; CHECK-LABEL: vwadd_wx_nxv8i64_nxv8i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.x v16, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf8 v24, v16 +; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = sext %splat to + %vc = add %va, %vb + ret %vc +} + +define @vwaddu_wx_nxv8i64_nxv8i8( %va, i8 %b) { +; CHECK-LABEL: vwaddu_wx_nxv8i64_nxv8i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.x v16, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf8 v24, v16 +; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = zext %splat to + %vc = add %va, %vb + ret %vc +} diff --git a/llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll index 98918ea21652..3634162eefd6 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll @@ -2,8 +2,8 @@ ; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s | FileCheck %s ; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck %s -define @vwmul_vv_nxv1i64( %va, %vb) { -; CHECK-LABEL: vwmul_vv_nxv1i64: +define @vwmul_vv_nxv1i64_nxv1i32( %va, %vb) { +; CHECK-LABEL: vwmul_vv_nxv1i64_nxv1i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma ; CHECK-NEXT: vwmul.vv v10, v8, v9 @@ -15,8 +15,8 @@ define @vwmul_vv_nxv1i64( %va, %ve } -define @vwmulu_vv_nxv1i64( %va, %vb) { -; CHECK-LABEL: vwmulu_vv_nxv1i64: +define @vwmulu_vv_nxv1i64_nxv1i32( %va, %vb) { +; CHECK-LABEL: vwmulu_vv_nxv1i64_nxv1i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma ; CHECK-NEXT: vwmulu.vv v10, v8, v9 @@ -28,8 +28,8 @@ define @vwmulu_vv_nxv1i64( %va, %ve } -define @vwmulsu_vv_nxv1i64( %va, %vb) { -; CHECK-LABEL: vwmulsu_vv_nxv1i64: +define @vwmulsu_vv_nxv1i64_nxv1i32( %va, %vb) { +; CHECK-LABEL: vwmulsu_vv_nxv1i64_nxv1i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma ; CHECK-NEXT: vwmulsu.vv v10, v8, v9 @@ -41,8 +41,8 @@ define @vwmulsu_vv_nxv1i64( %va, %ve } -define @vwmul_vx_nxv1i64( %va, i32 %b) { -; CHECK-LABEL: vwmul_vx_nxv1i64: +define @vwmul_vx_nxv1i64_nxv1i32( %va, i32 %b) { +; CHECK-LABEL: vwmul_vx_nxv1i64_nxv1i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, mf2, ta, ma ; CHECK-NEXT: vwmul.vx v9, v8, a0 @@ -56,8 +56,8 @@ define @vwmul_vx_nxv1i64( %va, i32 %b) { ret %ve } -define @vwmulu_vx_nxv1i64( %va, i32 %b) { -; CHECK-LABEL: vwmulu_vx_nxv1i64: +define @vwmulu_vx_nxv1i64_nxv1i32( %va, i32 %b) { +; CHECK-LABEL: vwmulu_vx_nxv1i64_nxv1i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, mf2, ta, ma ; CHECK-NEXT: vwmulu.vx v9, v8, a0 @@ -71,8 +71,8 @@ define @vwmulu_vx_nxv1i64( %va, i32 %b) { ret %ve } -define @vwmulsu_vx_nxv1i64( %va, i32 %b) { -; CHECK-LABEL: vwmulsu_vx_nxv1i64: +define @vwmulsu_vx_nxv1i64_nxv1i32( %va, i32 %b) { +; CHECK-LABEL: vwmulsu_vx_nxv1i64_nxv1i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, mf2, ta, ma ; CHECK-NEXT: vwmulsu.vx v9, v8, a0 @@ -86,8 +86,8 @@ define @vwmulsu_vx_nxv1i64( %va, i32 %b) { ret %ve } -define @vwmul_vv_nxv2i64( %va, %vb) { -; CHECK-LABEL: vwmul_vv_nxv2i64: +define @vwmul_vv_nxv2i64_nxv2i32( %va, %vb) { +; CHECK-LABEL: vwmul_vv_nxv2i64_nxv2i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma ; CHECK-NEXT: vwmul.vv v10, v8, v9 @@ -99,8 +99,8 @@ define @vwmul_vv_nxv2i64( %va, %ve } -define @vwmulu_vv_nxv2i64( %va, %vb) { -; CHECK-LABEL: vwmulu_vv_nxv2i64: +define @vwmulu_vv_nxv2i64_nxv2i32( %va, %vb) { +; CHECK-LABEL: vwmulu_vv_nxv2i64_nxv2i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma ; CHECK-NEXT: vwmulu.vv v10, v8, v9 @@ -112,8 +112,8 @@ define @vwmulu_vv_nxv2i64( %va, %ve } -define @vwmulsu_vv_nxv2i64( %va, %vb) { -; CHECK-LABEL: vwmulsu_vv_nxv2i64: +define @vwmulsu_vv_nxv2i64_nxv2i32( %va, %vb) { +; CHECK-LABEL: vwmulsu_vv_nxv2i64_nxv2i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma ; CHECK-NEXT: vwmulsu.vv v10, v8, v9 @@ -125,8 +125,8 @@ define @vwmulsu_vv_nxv2i64( %va, %ve } -define @vwmul_vx_nxv2i64( %va, i32 %b) { -; CHECK-LABEL: vwmul_vx_nxv2i64: +define @vwmul_vx_nxv2i64_nxv2i32( %va, i32 %b) { +; CHECK-LABEL: vwmul_vx_nxv2i64_nxv2i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m1, ta, ma ; CHECK-NEXT: vwmul.vx v10, v8, a0 @@ -140,8 +140,8 @@ define @vwmul_vx_nxv2i64( %va, i32 %b) { ret %ve } -define @vwmulu_vx_nxv2i64( %va, i32 %b) { -; CHECK-LABEL: vwmulu_vx_nxv2i64: +define @vwmulu_vx_nxv2i64_nxv2i32( %va, i32 %b) { +; CHECK-LABEL: vwmulu_vx_nxv2i64_nxv2i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m1, ta, ma ; CHECK-NEXT: vwmulu.vx v10, v8, a0 @@ -155,8 +155,8 @@ define @vwmulu_vx_nxv2i64( %va, i32 %b) { ret %ve } -define @vwmulsu_vx_nxv2i64( %va, i32 %b) { -; CHECK-LABEL: vwmulsu_vx_nxv2i64: +define @vwmulsu_vx_nxv2i64_nxv2i32( %va, i32 %b) { +; CHECK-LABEL: vwmulsu_vx_nxv2i64_nxv2i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m1, ta, ma ; CHECK-NEXT: vwmulsu.vx v10, v8, a0 @@ -170,8 +170,8 @@ define @vwmulsu_vx_nxv2i64( %va, i32 %b) { ret %ve } -define @vwmul_vv_nxv4i64( %va, %vb) { -; CHECK-LABEL: vwmul_vv_nxv4i64: +define @vwmul_vv_nxv4i64_nxv4i32( %va, %vb) { +; CHECK-LABEL: vwmul_vv_nxv4i64_nxv4i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma ; CHECK-NEXT: vwmul.vv v12, v8, v10 @@ -183,8 +183,8 @@ define @vwmul_vv_nxv4i64( %va, %ve } -define @vwmulu_vv_nxv4i64( %va, %vb) { -; CHECK-LABEL: vwmulu_vv_nxv4i64: +define @vwmulu_vv_nxv4i64_nxv4i32( %va, %vb) { +; CHECK-LABEL: vwmulu_vv_nxv4i64_nxv4i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma ; CHECK-NEXT: vwmulu.vv v12, v8, v10 @@ -196,8 +196,8 @@ define @vwmulu_vv_nxv4i64( %va, %ve } -define @vwmulsu_vv_nxv4i64( %va, %vb) { -; CHECK-LABEL: vwmulsu_vv_nxv4i64: +define @vwmulsu_vv_nxv4i64_nxv4i32( %va, %vb) { +; CHECK-LABEL: vwmulsu_vv_nxv4i64_nxv4i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma ; CHECK-NEXT: vwmulsu.vv v12, v8, v10 @@ -209,8 +209,8 @@ define @vwmulsu_vv_nxv4i64( %va, %ve } -define @vwmul_vx_nxv4i64( %va, i32 %b) { -; CHECK-LABEL: vwmul_vx_nxv4i64: +define @vwmul_vx_nxv4i64_nxv4i32( %va, i32 %b) { +; CHECK-LABEL: vwmul_vx_nxv4i64_nxv4i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m2, ta, ma ; CHECK-NEXT: vwmul.vx v12, v8, a0 @@ -224,8 +224,8 @@ define @vwmul_vx_nxv4i64( %va, i32 %b) { ret %ve } -define @vwmulu_vx_nxv4i64( %va, i32 %b) { -; CHECK-LABEL: vwmulu_vx_nxv4i64: +define @vwmulu_vx_nxv4i64_nxv4i32( %va, i32 %b) { +; CHECK-LABEL: vwmulu_vx_nxv4i64_nxv4i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m2, ta, ma ; CHECK-NEXT: vwmulu.vx v12, v8, a0 @@ -239,8 +239,8 @@ define @vwmulu_vx_nxv4i64( %va, i32 %b) { ret %ve } -define @vwmulsu_vx_nxv4i64( %va, i32 %b) { -; CHECK-LABEL: vwmulsu_vx_nxv4i64: +define @vwmulsu_vx_nxv4i64_nxv4i32( %va, i32 %b) { +; CHECK-LABEL: vwmulsu_vx_nxv4i64_nxv4i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m2, ta, ma ; CHECK-NEXT: vwmulsu.vx v12, v8, a0 @@ -254,8 +254,8 @@ define @vwmulsu_vx_nxv4i64( %va, i32 %b) { ret %ve } -define @vwmul_vv_nxv8i64( %va, %vb) { -; CHECK-LABEL: vwmul_vv_nxv8i64: +define @vwmul_vv_nxv8i64_nxv8i32( %va, %vb) { +; CHECK-LABEL: vwmul_vv_nxv8i64_nxv8i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma ; CHECK-NEXT: vwmul.vv v16, v8, v12 @@ -267,8 +267,8 @@ define @vwmul_vv_nxv8i64( %va, %ve } -define @vwmulu_vv_nxv8i64( %va, %vb) { -; CHECK-LABEL: vwmulu_vv_nxv8i64: +define @vwmulu_vv_nxv8i64_nxv8i32( %va, %vb) { +; CHECK-LABEL: vwmulu_vv_nxv8i64_nxv8i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma ; CHECK-NEXT: vwmulu.vv v16, v8, v12 @@ -280,8 +280,8 @@ define @vwmulu_vv_nxv8i64( %va, %ve } -define @vwmulsu_vv_nxv8i64( %va, %vb) { -; CHECK-LABEL: vwmulsu_vv_nxv8i64: +define @vwmulsu_vv_nxv8i64_nxv8i32( %va, %vb) { +; CHECK-LABEL: vwmulsu_vv_nxv8i64_nxv8i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma ; CHECK-NEXT: vwmulsu.vv v16, v8, v12 @@ -293,8 +293,8 @@ define @vwmulsu_vv_nxv8i64( %va, %ve } -define @vwmul_vx_nxv8i64( %va, i32 %b) { -; CHECK-LABEL: vwmul_vx_nxv8i64: +define @vwmul_vx_nxv8i64_nxv8i32( %va, i32 %b) { +; CHECK-LABEL: vwmul_vx_nxv8i64_nxv8i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m4, ta, ma ; CHECK-NEXT: vwmul.vx v16, v8, a0 @@ -308,8 +308,8 @@ define @vwmul_vx_nxv8i64( %va, i32 %b) { ret %ve } -define @vwmulu_vx_nxv8i64( %va, i32 %b) { -; CHECK-LABEL: vwmulu_vx_nxv8i64: +define @vwmulu_vx_nxv8i64_nxv8i32( %va, i32 %b) { +; CHECK-LABEL: vwmulu_vx_nxv8i64_nxv8i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m4, ta, ma ; CHECK-NEXT: vwmulu.vx v16, v8, a0 @@ -323,8 +323,8 @@ define @vwmulu_vx_nxv8i64( %va, i32 %b) { ret %ve } -define @vwmulsu_vx_nxv8i64( %va, i32 %b) { -; CHECK-LABEL: vwmulsu_vx_nxv8i64: +define @vwmulsu_vx_nxv8i64_nxv8i32( %va, i32 %b) { +; CHECK-LABEL: vwmulsu_vx_nxv8i64_nxv8i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m4, ta, ma ; CHECK-NEXT: vwmulsu.vx v16, v8, a0 @@ -337,3 +337,771 @@ define @vwmulsu_vx_nxv8i64( %va, i32 %b) { %ve = mul %vc, %vd ret %ve } + +define @vwmul_vv_nxv1i64_nxv1i16( %va, %vb) { +; CHECK-LABEL: vwmul_vv_nxv1i64_nxv1i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v8, v9 +; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulu_vv_nxv1i64_nxv1i16( %va, %vb) { +; CHECK-LABEL: vwmulu_vv_nxv1i64_nxv1i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v8, v9 +; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulsu_vv_nxv1i64_nxv1i16( %va, %vb) { +; CHECK-LABEL: vwmulsu_vv_nxv1i64_nxv1i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v8, v9 +; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = zext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmul_vx_nxv1i64_nxv1i16( %va, i16 %b) { +; CHECK-LABEL: vwmul_vx_nxv1i64_nxv1i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v8, v9 +; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: ret + %head = insertelement undef, i16 %b, i16 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulu_vx_nxv1i64_nxv1i16( %va, i16 %b) { +; CHECK-LABEL: vwmulu_vx_nxv1i64_nxv1i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v8, v9 +; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: ret + %head = insertelement undef, i16 %b, i16 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulsu_vx_nxv1i64_nxv1i16( %va, i16 %b) { +; CHECK-LABEL: vwmulsu_vx_nxv1i64_nxv1i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v8, v9 +; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: ret + %head = insertelement undef, i16 %b, i16 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = sext %va to + %vd = zext %splat to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmul_vv_nxv2i64_nxv2i16( %va, %vb) { +; CHECK-LABEL: vwmul_vv_nxv2i64_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v12, v9 +; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulu_vv_nxv2i64_nxv2i16( %va, %vb) { +; CHECK-LABEL: vwmulu_vv_nxv2i64_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v12, v9 +; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulsu_vv_nxv2i64_nxv2i16( %va, %vb) { +; CHECK-LABEL: vwmulsu_vv_nxv2i64_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v12, v9 +; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = zext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmul_vx_nxv2i64_nxv2i16( %va, i16 %b) { +; CHECK-LABEL: vwmul_vx_nxv2i64_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v12, v9 +; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: ret + %head = insertelement undef, i16 %b, i16 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulu_vx_nxv2i64_nxv2i16( %va, i16 %b) { +; CHECK-LABEL: vwmulu_vx_nxv2i64_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v12, v9 +; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: ret + %head = insertelement undef, i16 %b, i16 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulsu_vx_nxv2i64_nxv2i16( %va, i16 %b) { +; CHECK-LABEL: vwmulsu_vx_nxv2i64_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v12, v9 +; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: ret + %head = insertelement undef, i16 %b, i16 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = sext %va to + %vd = zext %splat to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmul_vv_nxv4i64_nxv4i16( %va, %vb) { +; CHECK-LABEL: vwmul_vv_nxv4i64_nxv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v16, v9 +; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulu_vv_nxv4i64_nxv4i16( %va, %vb) { +; CHECK-LABEL: vwmulu_vv_nxv4i64_nxv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v16, v9 +; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulsu_vv_nxv4i64_nxv4i16( %va, %vb) { +; CHECK-LABEL: vwmulsu_vv_nxv4i64_nxv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v16, v9 +; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = zext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmul_vx_nxv4i64_nxv4i16( %va, i16 %b) { +; CHECK-LABEL: vwmul_vx_nxv4i64_nxv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v16, v9 +; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: ret + %head = insertelement undef, i16 %b, i16 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulu_vx_nxv4i64_nxv4i16( %va, i16 %b) { +; CHECK-LABEL: vwmulu_vx_nxv4i64_nxv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v16, v9 +; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: ret + %head = insertelement undef, i16 %b, i16 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulsu_vx_nxv4i64_nxv4i16( %va, i16 %b) { +; CHECK-LABEL: vwmulsu_vx_nxv4i64_nxv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v16, v9 +; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: ret + %head = insertelement undef, i16 %b, i16 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = sext %va to + %vd = zext %splat to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmul_vv_nxv8i64_nxv8i16( %va, %vb) { +; CHECK-LABEL: vwmul_vv_nxv8i64_nxv8i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v24, v10 +; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulu_vv_nxv8i64_nxv8i16( %va, %vb) { +; CHECK-LABEL: vwmulu_vv_nxv8i64_nxv8i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v24, v10 +; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulsu_vv_nxv8i64_nxv8i16( %va, %vb) { +; CHECK-LABEL: vwmulsu_vv_nxv8i64_nxv8i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v24, v10 +; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = zext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmul_vx_nxv8i64_nxv8i16( %va, i16 %b) { +; CHECK-LABEL: vwmul_vx_nxv8i64_nxv8i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v24, v10 +; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: ret + %head = insertelement undef, i16 %b, i16 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulu_vx_nxv8i64_nxv8i16( %va, i16 %b) { +; CHECK-LABEL: vwmulu_vx_nxv8i64_nxv8i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v24, v10 +; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: ret + %head = insertelement undef, i16 %b, i16 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulsu_vx_nxv8i64_nxv8i16( %va, i16 %b) { +; CHECK-LABEL: vwmulsu_vx_nxv8i64_nxv8i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v24, v10 +; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: ret + %head = insertelement undef, i16 %b, i16 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = sext %va to + %vd = zext %splat to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmul_vv_nxv1i64_nxv1i8( %va, %vb) { +; CHECK-LABEL: vwmul_vv_nxv1i64_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf8 v10, v8 +; CHECK-NEXT: vsext.vf8 v8, v9 +; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulu_vv_nxv1i64_nxv1i8( %va, %vb) { +; CHECK-LABEL: vwmulu_vv_nxv1i64_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf8 v10, v8 +; CHECK-NEXT: vzext.vf8 v8, v9 +; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulsu_vv_nxv1i64_nxv1i8( %va, %vb) { +; CHECK-LABEL: vwmulsu_vv_nxv1i64_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf8 v10, v8 +; CHECK-NEXT: vzext.vf8 v8, v9 +; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = zext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmul_vx_nxv1i64_nxv1i8( %va, i8 %b) { +; CHECK-LABEL: vwmul_vx_nxv1i64_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf8 v10, v8 +; CHECK-NEXT: vsext.vf8 v8, v9 +; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: ret + %head = insertelement undef, i8 %b, i8 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulu_vx_nxv1i64_nxv1i8( %va, i8 %b) { +; CHECK-LABEL: vwmulu_vx_nxv1i64_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf8 v10, v8 +; CHECK-NEXT: vzext.vf8 v8, v9 +; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: ret + %head = insertelement undef, i8 %b, i8 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulsu_vx_nxv1i64_nxv1i8( %va, i8 %b) { +; CHECK-LABEL: vwmulsu_vx_nxv1i64_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf8 v10, v8 +; CHECK-NEXT: vzext.vf8 v8, v9 +; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: ret + %head = insertelement undef, i8 %b, i8 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = sext %va to + %vd = zext %splat to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmul_vv_nxv2i64_nxv2i8( %va, %vb) { +; CHECK-LABEL: vwmul_vv_nxv2i64_nxv2i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf8 v10, v8 +; CHECK-NEXT: vsext.vf8 v12, v9 +; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulu_vv_nxv2i64_nxv2i8( %va, %vb) { +; CHECK-LABEL: vwmulu_vv_nxv2i64_nxv2i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf8 v10, v8 +; CHECK-NEXT: vzext.vf8 v12, v9 +; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulsu_vv_nxv2i64_nxv2i8( %va, %vb) { +; CHECK-LABEL: vwmulsu_vv_nxv2i64_nxv2i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf8 v10, v8 +; CHECK-NEXT: vzext.vf8 v12, v9 +; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = zext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmul_vx_nxv2i64_nxv2i8( %va, i8 %b) { +; CHECK-LABEL: vwmul_vx_nxv2i64_nxv2i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf8 v10, v8 +; CHECK-NEXT: vsext.vf8 v12, v9 +; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: ret + %head = insertelement undef, i8 %b, i8 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulu_vx_nxv2i64_nxv2i8( %va, i8 %b) { +; CHECK-LABEL: vwmulu_vx_nxv2i64_nxv2i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf8 v10, v8 +; CHECK-NEXT: vzext.vf8 v12, v9 +; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: ret + %head = insertelement undef, i8 %b, i8 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulsu_vx_nxv2i64_nxv2i8( %va, i8 %b) { +; CHECK-LABEL: vwmulsu_vx_nxv2i64_nxv2i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf8 v10, v8 +; CHECK-NEXT: vzext.vf8 v12, v9 +; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: ret + %head = insertelement undef, i8 %b, i8 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = sext %va to + %vd = zext %splat to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmul_vv_nxv4i64_nxv4i8( %va, %vb) { +; CHECK-LABEL: vwmul_vv_nxv4i64_nxv4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf8 v12, v8 +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulu_vv_nxv4i64_nxv4i8( %va, %vb) { +; CHECK-LABEL: vwmulu_vv_nxv4i64_nxv4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf8 v12, v8 +; CHECK-NEXT: vzext.vf8 v16, v9 +; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulsu_vv_nxv4i64_nxv4i8( %va, %vb) { +; CHECK-LABEL: vwmulsu_vv_nxv4i64_nxv4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf8 v12, v8 +; CHECK-NEXT: vzext.vf8 v16, v9 +; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = zext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmul_vx_nxv4i64_nxv4i8( %va, i8 %b) { +; CHECK-LABEL: vwmul_vx_nxv4i64_nxv4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf8 v12, v8 +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: ret + %head = insertelement undef, i8 %b, i8 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulu_vx_nxv4i64_nxv4i8( %va, i8 %b) { +; CHECK-LABEL: vwmulu_vx_nxv4i64_nxv4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf8 v12, v8 +; CHECK-NEXT: vzext.vf8 v16, v9 +; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: ret + %head = insertelement undef, i8 %b, i8 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulsu_vx_nxv4i64_nxv4i8( %va, i8 %b) { +; CHECK-LABEL: vwmulsu_vx_nxv4i64_nxv4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf8 v12, v8 +; CHECK-NEXT: vzext.vf8 v16, v9 +; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: ret + %head = insertelement undef, i8 %b, i8 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = sext %va to + %vd = zext %splat to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmul_vv_nxv8i64_nxv8i8( %va, %vb) { +; CHECK-LABEL: vwmul_vv_nxv8i64_nxv8i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf8 v16, v8 +; CHECK-NEXT: vsext.vf8 v24, v9 +; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulu_vv_nxv8i64_nxv8i8( %va, %vb) { +; CHECK-LABEL: vwmulu_vv_nxv8i64_nxv8i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf8 v16, v8 +; CHECK-NEXT: vzext.vf8 v24, v9 +; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulsu_vv_nxv8i64_nxv8i8( %va, %vb) { +; CHECK-LABEL: vwmulsu_vv_nxv8i64_nxv8i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf8 v16, v8 +; CHECK-NEXT: vzext.vf8 v24, v9 +; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = zext %vb to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmul_vx_nxv8i64_nxv8i8( %va, i8 %b) { +; CHECK-LABEL: vwmul_vx_nxv8i64_nxv8i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf8 v16, v8 +; CHECK-NEXT: vsext.vf8 v24, v9 +; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: ret + %head = insertelement undef, i8 %b, i8 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulu_vx_nxv8i64_nxv8i8( %va, i8 %b) { +; CHECK-LABEL: vwmulu_vx_nxv8i64_nxv8i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf8 v16, v8 +; CHECK-NEXT: vzext.vf8 v24, v9 +; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: ret + %head = insertelement undef, i8 %b, i8 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = mul %vc, %vd + ret %ve +} + +define @vwmulsu_vx_nxv8i64_nxv8i8( %va, i8 %b) { +; CHECK-LABEL: vwmulsu_vx_nxv8i64_nxv8i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf8 v16, v8 +; CHECK-NEXT: vzext.vf8 v24, v9 +; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: ret + %head = insertelement undef, i8 %b, i8 0 + %splat = shufflevector %head, undef, zeroinitializer + %vc = sext %va to + %vd = zext %splat to + %ve = mul %vc, %vd + ret %ve +} diff --git a/llvm/test/CodeGen/RISCV/rvv/vwsub-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwsub-sdnode.ll index a6a15c164343..123469ade0ed 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vwsub-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vwsub-sdnode.ll @@ -2,8 +2,8 @@ ; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s | FileCheck %s ; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck %s -define @vwsub_vv_nxv1i64( %va, %vb) { -; CHECK-LABEL: vwsub_vv_nxv1i64: +define @vwsub_vv_nxv1i64_nxv1i32( %va, %vb) { +; CHECK-LABEL: vwsub_vv_nxv1i64_nxv1i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma ; CHECK-NEXT: vwsub.vv v10, v8, v9 @@ -15,8 +15,8 @@ define @vwsub_vv_nxv1i64( %va, %ve } -define @vwsubu_vv_nxv1i64( %va, %vb) { -; CHECK-LABEL: vwsubu_vv_nxv1i64: +define @vwsubu_vv_nxv1i64_nxv1i32( %va, %vb) { +; CHECK-LABEL: vwsubu_vv_nxv1i64_nxv1i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma ; CHECK-NEXT: vwsubu.vv v10, v8, v9 @@ -28,8 +28,8 @@ define @vwsubu_vv_nxv1i64( %va, %ve } -define @vwsub_vx_nxv1i64( %va, i32 %b) { -; CHECK-LABEL: vwsub_vx_nxv1i64: +define @vwsub_vx_nxv1i64_nxv1i32( %va, i32 %b) { +; CHECK-LABEL: vwsub_vx_nxv1i64_nxv1i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, mf2, ta, ma ; CHECK-NEXT: vwsub.vx v9, v8, a0 @@ -43,8 +43,8 @@ define @vwsub_vx_nxv1i64( %va, i32 %b) { ret %ve } -define @vwsubu_vx_nxv1i64( %va, i32 %b) { -; CHECK-LABEL: vwsubu_vx_nxv1i64: +define @vwsubu_vx_nxv1i64_nxv1i32( %va, i32 %b) { +; CHECK-LABEL: vwsubu_vx_nxv1i64_nxv1i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, mf2, ta, ma ; CHECK-NEXT: vwsubu.vx v9, v8, a0 @@ -58,8 +58,8 @@ define @vwsubu_vx_nxv1i64( %va, i32 %b) { ret %ve } -define @vwsub_wv_nxv1i64( %va, %vb) { -; CHECK-LABEL: vwsub_wv_nxv1i64: +define @vwsub_wv_nxv1i64_nxv1i32( %va, %vb) { +; CHECK-LABEL: vwsub_wv_nxv1i64_nxv1i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma ; CHECK-NEXT: vwsub.wv v8, v8, v9 @@ -69,8 +69,8 @@ define @vwsub_wv_nxv1i64( %va, %vd } -define @vwsubu_wv_nxv1i64( %va, %vb) { -; CHECK-LABEL: vwsubu_wv_nxv1i64: +define @vwsubu_wv_nxv1i64_nxv1i32( %va, %vb) { +; CHECK-LABEL: vwsubu_wv_nxv1i64_nxv1i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma ; CHECK-NEXT: vwsubu.wv v8, v8, v9 @@ -80,8 +80,8 @@ define @vwsubu_wv_nxv1i64( %va, %vd } -define @vwsub_wx_nxv1i64( %va, i32 %b) { -; CHECK-LABEL: vwsub_wx_nxv1i64: +define @vwsub_wx_nxv1i64_nxv1i32( %va, i32 %b) { +; CHECK-LABEL: vwsub_wx_nxv1i64_nxv1i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, mf2, ta, ma ; CHECK-NEXT: vwsub.wx v8, v8, a0 @@ -93,8 +93,8 @@ define @vwsub_wx_nxv1i64( %va, i32 %b) { ret %vc } -define @vwsubu_wx_nxv1i64( %va, i32 %b) { -; CHECK-LABEL: vwsubu_wx_nxv1i64: +define @vwsubu_wx_nxv1i64_nxv1i32( %va, i32 %b) { +; CHECK-LABEL: vwsubu_wx_nxv1i64_nxv1i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, mf2, ta, ma ; CHECK-NEXT: vwsubu.wx v8, v8, a0 @@ -106,8 +106,8 @@ define @vwsubu_wx_nxv1i64( %va, i32 %b) { ret %vc } -define @vwsub_vv_nxv2i64( %va, %vb) { -; CHECK-LABEL: vwsub_vv_nxv2i64: +define @vwsub_vv_nxv2i64_nxv2i32( %va, %vb) { +; CHECK-LABEL: vwsub_vv_nxv2i64_nxv2i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma ; CHECK-NEXT: vwsub.vv v10, v8, v9 @@ -119,8 +119,8 @@ define @vwsub_vv_nxv2i64( %va, %ve } -define @vwsubu_vv_nxv2i64( %va, %vb) { -; CHECK-LABEL: vwsubu_vv_nxv2i64: +define @vwsubu_vv_nxv2i64_nxv2i32( %va, %vb) { +; CHECK-LABEL: vwsubu_vv_nxv2i64_nxv2i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma ; CHECK-NEXT: vwsubu.vv v10, v8, v9 @@ -132,8 +132,8 @@ define @vwsubu_vv_nxv2i64( %va, %ve } -define @vwsub_vx_nxv2i64( %va, i32 %b) { -; CHECK-LABEL: vwsub_vx_nxv2i64: +define @vwsub_vx_nxv2i64_nxv2i32( %va, i32 %b) { +; CHECK-LABEL: vwsub_vx_nxv2i64_nxv2i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m1, ta, ma ; CHECK-NEXT: vwsub.vx v10, v8, a0 @@ -147,8 +147,8 @@ define @vwsub_vx_nxv2i64( %va, i32 %b) { ret %ve } -define @vwsubu_vx_nxv2i64( %va, i32 %b) { -; CHECK-LABEL: vwsubu_vx_nxv2i64: +define @vwsubu_vx_nxv2i64_nxv2i32( %va, i32 %b) { +; CHECK-LABEL: vwsubu_vx_nxv2i64_nxv2i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m1, ta, ma ; CHECK-NEXT: vwsubu.vx v10, v8, a0 @@ -162,8 +162,8 @@ define @vwsubu_vx_nxv2i64( %va, i32 %b) { ret %ve } -define @vwsub_wv_nxv2i64( %va, %vb) { -; CHECK-LABEL: vwsub_wv_nxv2i64: +define @vwsub_wv_nxv2i64_nxv2i32( %va, %vb) { +; CHECK-LABEL: vwsub_wv_nxv2i64_nxv2i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma ; CHECK-NEXT: vwsub.wv v8, v8, v10 @@ -173,8 +173,8 @@ define @vwsub_wv_nxv2i64( %va, %vd } -define @vwsubu_wv_nxv2i64( %va, %vb) { -; CHECK-LABEL: vwsubu_wv_nxv2i64: +define @vwsubu_wv_nxv2i64_nxv2i32( %va, %vb) { +; CHECK-LABEL: vwsubu_wv_nxv2i64_nxv2i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma ; CHECK-NEXT: vwsubu.wv v8, v8, v10 @@ -184,8 +184,8 @@ define @vwsubu_wv_nxv2i64( %va, %vd } -define @vwsub_wx_nxv2i64( %va, i32 %b) { -; CHECK-LABEL: vwsub_wx_nxv2i64: +define @vwsub_wx_nxv2i64_nxv2i32( %va, i32 %b) { +; CHECK-LABEL: vwsub_wx_nxv2i64_nxv2i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m1, ta, ma ; CHECK-NEXT: vwsub.wx v8, v8, a0 @@ -197,8 +197,8 @@ define @vwsub_wx_nxv2i64( %va, i32 %b) { ret %vc } -define @vwsubu_wx_nxv2i64( %va, i32 %b) { -; CHECK-LABEL: vwsubu_wx_nxv2i64: +define @vwsubu_wx_nxv2i64_nxv2i32( %va, i32 %b) { +; CHECK-LABEL: vwsubu_wx_nxv2i64_nxv2i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m1, ta, ma ; CHECK-NEXT: vwsubu.wx v8, v8, a0 @@ -210,8 +210,8 @@ define @vwsubu_wx_nxv2i64( %va, i32 %b) { ret %vc } -define @vwsub_vv_nxv4i64( %va, %vb) { -; CHECK-LABEL: vwsub_vv_nxv4i64: +define @vwsub_vv_nxv4i64_nxv4i32( %va, %vb) { +; CHECK-LABEL: vwsub_vv_nxv4i64_nxv4i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma ; CHECK-NEXT: vwsub.vv v12, v8, v10 @@ -223,8 +223,8 @@ define @vwsub_vv_nxv4i64( %va, %ve } -define @vwsubu_vv_nxv4i64( %va, %vb) { -; CHECK-LABEL: vwsubu_vv_nxv4i64: +define @vwsubu_vv_nxv4i64_nxv4i32( %va, %vb) { +; CHECK-LABEL: vwsubu_vv_nxv4i64_nxv4i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma ; CHECK-NEXT: vwsubu.vv v12, v8, v10 @@ -236,8 +236,8 @@ define @vwsubu_vv_nxv4i64( %va, %ve } -define @vwsub_vx_nxv4i64( %va, i32 %b) { -; CHECK-LABEL: vwsub_vx_nxv4i64: +define @vwsub_vx_nxv4i64_nxv4i32( %va, i32 %b) { +; CHECK-LABEL: vwsub_vx_nxv4i64_nxv4i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m2, ta, ma ; CHECK-NEXT: vwsub.vx v12, v8, a0 @@ -251,8 +251,8 @@ define @vwsub_vx_nxv4i64( %va, i32 %b) { ret %ve } -define @vwsubu_vx_nxv4i64( %va, i32 %b) { -; CHECK-LABEL: vwsubu_vx_nxv4i64: +define @vwsubu_vx_nxv4i64_nxv4i32( %va, i32 %b) { +; CHECK-LABEL: vwsubu_vx_nxv4i64_nxv4i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m2, ta, ma ; CHECK-NEXT: vwsubu.vx v12, v8, a0 @@ -266,8 +266,8 @@ define @vwsubu_vx_nxv4i64( %va, i32 %b) { ret %ve } -define @vwsub_wv_nxv4i64( %va, %vb) { -; CHECK-LABEL: vwsub_wv_nxv4i64: +define @vwsub_wv_nxv4i64_nxv4i32( %va, %vb) { +; CHECK-LABEL: vwsub_wv_nxv4i64_nxv4i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma ; CHECK-NEXT: vwsub.wv v8, v8, v12 @@ -277,8 +277,8 @@ define @vwsub_wv_nxv4i64( %va, %vd } -define @vwsubu_wv_nxv4i64( %va, %vb) { -; CHECK-LABEL: vwsubu_wv_nxv4i64: +define @vwsubu_wv_nxv4i64_nxv4i32( %va, %vb) { +; CHECK-LABEL: vwsubu_wv_nxv4i64_nxv4i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma ; CHECK-NEXT: vwsubu.wv v8, v8, v12 @@ -288,8 +288,8 @@ define @vwsubu_wv_nxv4i64( %va, %vd } -define @vwsub_wx_nxv4i64( %va, i32 %b) { -; CHECK-LABEL: vwsub_wx_nxv4i64: +define @vwsub_wx_nxv4i64_nxv4i32( %va, i32 %b) { +; CHECK-LABEL: vwsub_wx_nxv4i64_nxv4i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m2, ta, ma ; CHECK-NEXT: vwsub.wx v8, v8, a0 @@ -301,8 +301,8 @@ define @vwsub_wx_nxv4i64( %va, i32 %b) { ret %vc } -define @vwsubu_wx_nxv4i64( %va, i32 %b) { -; CHECK-LABEL: vwsubu_wx_nxv4i64: +define @vwsubu_wx_nxv4i64_nxv4i32( %va, i32 %b) { +; CHECK-LABEL: vwsubu_wx_nxv4i64_nxv4i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m2, ta, ma ; CHECK-NEXT: vwsubu.wx v8, v8, a0 @@ -314,8 +314,8 @@ define @vwsubu_wx_nxv4i64( %va, i32 %b) { ret %vc } -define @vwsub_vv_nxv8i64( %va, %vb) { -; CHECK-LABEL: vwsub_vv_nxv8i64: +define @vwsub_vv_nxv8i64_nxv8i32( %va, %vb) { +; CHECK-LABEL: vwsub_vv_nxv8i64_nxv8i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma ; CHECK-NEXT: vwsub.vv v16, v8, v12 @@ -327,8 +327,8 @@ define @vwsub_vv_nxv8i64( %va, %ve } -define @vwsubu_vv_nxv8i64( %va, %vb) { -; CHECK-LABEL: vwsubu_vv_nxv8i64: +define @vwsubu_vv_nxv8i64_nxv8i32( %va, %vb) { +; CHECK-LABEL: vwsubu_vv_nxv8i64_nxv8i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma ; CHECK-NEXT: vwsubu.vv v16, v8, v12 @@ -340,8 +340,8 @@ define @vwsubu_vv_nxv8i64( %va, %ve } -define @vwsub_vx_nxv8i64( %va, i32 %b) { -; CHECK-LABEL: vwsub_vx_nxv8i64: +define @vwsub_vx_nxv8i64_nxv8i32( %va, i32 %b) { +; CHECK-LABEL: vwsub_vx_nxv8i64_nxv8i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m4, ta, ma ; CHECK-NEXT: vwsub.vx v16, v8, a0 @@ -355,8 +355,8 @@ define @vwsub_vx_nxv8i64( %va, i32 %b) { ret %ve } -define @vwsubu_vx_nxv8i64( %va, i32 %b) { -; CHECK-LABEL: vwsubu_vx_nxv8i64: +define @vwsubu_vx_nxv8i64_nxv8i32( %va, i32 %b) { +; CHECK-LABEL: vwsubu_vx_nxv8i64_nxv8i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m4, ta, ma ; CHECK-NEXT: vwsubu.vx v16, v8, a0 @@ -370,8 +370,8 @@ define @vwsubu_vx_nxv8i64( %va, i32 %b) { ret %ve } -define @vwsub_wv_nxv8i64( %va, %vb) { -; CHECK-LABEL: vwsub_wv_nxv8i64: +define @vwsub_wv_nxv8i64_nxv8i32( %va, %vb) { +; CHECK-LABEL: vwsub_wv_nxv8i64_nxv8i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma ; CHECK-NEXT: vwsub.wv v8, v8, v16 @@ -381,8 +381,8 @@ define @vwsub_wv_nxv8i64( %va, %vd } -define @vwsubu_wv_nxv8i64( %va, %vb) { -; CHECK-LABEL: vwsubu_wv_nxv8i64: +define @vwsubu_wv_nxv8i64_nxv8i32( %va, %vb) { +; CHECK-LABEL: vwsubu_wv_nxv8i64_nxv8i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma ; CHECK-NEXT: vwsubu.wv v8, v8, v16 @@ -392,8 +392,8 @@ define @vwsubu_wv_nxv8i64( %va, %vd } -define @vwsub_wx_nxv8i64( %va, i32 %b) { -; CHECK-LABEL: vwsub_wx_nxv8i64: +define @vwsub_wx_nxv8i64_nxv8i32( %va, i32 %b) { +; CHECK-LABEL: vwsub_wx_nxv8i64_nxv8i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m4, ta, ma ; CHECK-NEXT: vwsub.wx v8, v8, a0 @@ -405,8 +405,8 @@ define @vwsub_wx_nxv8i64( %va, i32 %b) { ret %vc } -define @vwsubu_wx_nxv8i64( %va, i32 %b) { -; CHECK-LABEL: vwsubu_wx_nxv8i64: +define @vwsubu_wx_nxv8i64_nxv8i32( %va, i32 %b) { +; CHECK-LABEL: vwsubu_wx_nxv8i64_nxv8i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e32, m4, ta, ma ; CHECK-NEXT: vwsubu.wx v8, v8, a0 @@ -417,3 +417,947 @@ define @vwsubu_wx_nxv8i64( %va, i32 %b) { %vc = sub %va, %vb ret %vc } + +define @vwsub_vv_nxv1i64_nxv1i16( %va, %vb) { +; CHECK-LABEL: vwsub_vv_nxv1i64_nxv1i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v8, v9 +; CHECK-NEXT: vsub.vv v8, v10, v8 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsubu_vv_nxv1i64_nxv1i16( %va, %vb) { +; CHECK-LABEL: vwsubu_vv_nxv1i64_nxv1i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, mf4, ta, ma +; CHECK-NEXT: vwsubu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v8, v10 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsub_vx_nxv1i64_nxv1i16( %va, i16 %b) { +; CHECK-LABEL: vwsub_vx_nxv1i64_nxv1i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v8, v9 +; CHECK-NEXT: vsub.vv v8, v10, v8 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsubu_vx_nxv1i64_nxv1i16( %va, i16 %b) { +; CHECK-LABEL: vwsubu_vx_nxv1i64_nxv1i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma +; CHECK-NEXT: vwsubu.vx v9, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v8, v9 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsub_wv_nxv1i64_nxv1i16( %va, %vb) { +; CHECK-LABEL: vwsub_wv_nxv1i64_nxv1i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v9 +; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: ret + %vc = sext %vb to + %vd = sub %va, %vc + ret %vd +} + +define @vwsubu_wv_nxv1i64_nxv1i16( %va, %vb) { +; CHECK-LABEL: vwsubu_wv_nxv1i64_nxv1i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v9 +; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: ret + %vc = zext %vb to + %vd = sub %va, %vc + ret %vd +} + +define @vwsub_wx_nxv1i64_nxv1i16( %va, i16 %b) { +; CHECK-LABEL: vwsub_wx_nxv1i64_nxv1i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v9 +; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = sext %splat to + %vc = sub %va, %vb + ret %vc +} + +define @vwsubu_wx_nxv1i64_nxv1i16( %va, i16 %b) { +; CHECK-LABEL: vwsubu_wx_nxv1i64_nxv1i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v9 +; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = zext %splat to + %vc = sub %va, %vb + ret %vc +} + +define @vwsub_vv_nxv2i64_nxv2i16( %va, %vb) { +; CHECK-LABEL: vwsub_vv_nxv2i64_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v12, v9 +; CHECK-NEXT: vsub.vv v8, v10, v12 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsubu_vv_nxv2i64_nxv2i16( %va, %vb) { +; CHECK-LABEL: vwsubu_vv_nxv2i64_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, mf2, ta, ma +; CHECK-NEXT: vwsubu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v8, v10 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsub_vx_nxv2i64_nxv2i16( %va, i16 %b) { +; CHECK-LABEL: vwsub_vx_nxv2i64_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v12, v9 +; CHECK-NEXT: vsub.vv v8, v10, v12 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsubu_vx_nxv2i64_nxv2i16( %va, i16 %b) { +; CHECK-LABEL: vwsubu_vx_nxv2i64_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma +; CHECK-NEXT: vwsubu.vx v10, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v8, v10 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsub_wv_nxv2i64_nxv2i16( %va, %vb) { +; CHECK-LABEL: vwsub_wv_nxv2i64_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v10 +; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: ret + %vc = sext %vb to + %vd = sub %va, %vc + ret %vd +} + +define @vwsubu_wv_nxv2i64_nxv2i16( %va, %vb) { +; CHECK-LABEL: vwsubu_wv_nxv2i64_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v12, v10 +; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: ret + %vc = zext %vb to + %vd = sub %va, %vc + ret %vd +} + +define @vwsub_wx_nxv2i64_nxv2i16( %va, i16 %b) { +; CHECK-LABEL: vwsub_wx_nxv2i64_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v10 +; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = sext %splat to + %vc = sub %va, %vb + ret %vc +} + +define @vwsubu_wx_nxv2i64_nxv2i16( %va, i16 %b) { +; CHECK-LABEL: vwsubu_wx_nxv2i64_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v12, v10 +; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = zext %splat to + %vc = sub %va, %vb + ret %vc +} + +define @vwsub_vv_nxv4i64_nxv4i16( %va, %vb) { +; CHECK-LABEL: vwsub_vv_nxv4i64_nxv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v16, v9 +; CHECK-NEXT: vsub.vv v8, v12, v16 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsubu_vv_nxv4i64_nxv4i16( %va, %vb) { +; CHECK-LABEL: vwsubu_vv_nxv4i64_nxv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, ma +; CHECK-NEXT: vwsubu.vv v12, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v8, v12 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsub_vx_nxv4i64_nxv4i16( %va, i16 %b) { +; CHECK-LABEL: vwsub_vx_nxv4i64_nxv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v16, v9 +; CHECK-NEXT: vsub.vv v8, v12, v16 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsubu_vx_nxv4i64_nxv4i16( %va, i16 %b) { +; CHECK-LABEL: vwsubu_vx_nxv4i64_nxv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma +; CHECK-NEXT: vwsubu.vx v12, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v8, v12 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsub_wv_nxv4i64_nxv4i16( %va, %vb) { +; CHECK-LABEL: vwsub_wv_nxv4i64_nxv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v12 +; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: ret + %vc = sext %vb to + %vd = sub %va, %vc + ret %vd +} + +define @vwsubu_wv_nxv4i64_nxv4i16( %va, %vb) { +; CHECK-LABEL: vwsubu_wv_nxv4i64_nxv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v16, v12 +; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: ret + %vc = zext %vb to + %vd = sub %va, %vc + ret %vd +} + +define @vwsub_wx_nxv4i64_nxv4i16( %va, i16 %b) { +; CHECK-LABEL: vwsub_wx_nxv4i64_nxv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma +; CHECK-NEXT: vmv.v.x v12, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v12 +; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = sext %splat to + %vc = sub %va, %vb + ret %vc +} + +define @vwsubu_wx_nxv4i64_nxv4i16( %va, i16 %b) { +; CHECK-LABEL: vwsubu_wx_nxv4i64_nxv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma +; CHECK-NEXT: vmv.v.x v12, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v16, v12 +; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = zext %splat to + %vc = sub %va, %vb + ret %vc +} + +define @vwsub_vv_nxv8i64_nxv8i16( %va, %vb) { +; CHECK-LABEL: vwsub_vv_nxv8i64_nxv8i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v24, v10 +; CHECK-NEXT: vsub.vv v8, v16, v24 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsubu_vv_nxv8i64_nxv8i16( %va, %vb) { +; CHECK-LABEL: vwsubu_vv_nxv8i64_nxv8i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, m2, ta, ma +; CHECK-NEXT: vwsubu.vv v16, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf2 v8, v16 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsub_vx_nxv8i64_nxv8i16( %va, i16 %b) { +; CHECK-LABEL: vwsub_vx_nxv8i64_nxv8i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v24, v10 +; CHECK-NEXT: vsub.vv v8, v16, v24 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsubu_vx_nxv8i64_nxv8i16( %va, i16 %b) { +; CHECK-LABEL: vwsubu_vx_nxv8i64_nxv8i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma +; CHECK-NEXT: vwsubu.vx v16, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf2 v8, v16 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsub_wv_nxv8i64_nxv8i16( %va, %vb) { +; CHECK-LABEL: vwsub_wv_nxv8i64_nxv8i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf4 v24, v16 +; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: ret + %vc = sext %vb to + %vd = sub %va, %vc + ret %vd +} + +define @vwsubu_wv_nxv8i64_nxv8i16( %va, %vb) { +; CHECK-LABEL: vwsubu_wv_nxv8i64_nxv8i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf4 v24, v16 +; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: ret + %vc = zext %vb to + %vd = sub %va, %vc + ret %vd +} + +define @vwsub_wx_nxv8i64_nxv8i16( %va, i16 %b) { +; CHECK-LABEL: vwsub_wx_nxv8i64_nxv8i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma +; CHECK-NEXT: vmv.v.x v16, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf4 v24, v16 +; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = sext %splat to + %vc = sub %va, %vb + ret %vc +} + +define @vwsubu_wx_nxv8i64_nxv8i16( %va, i16 %b) { +; CHECK-LABEL: vwsubu_wx_nxv8i64_nxv8i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma +; CHECK-NEXT: vmv.v.x v16, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf4 v24, v16 +; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: ret + %head = insertelement poison, i16 %b, i16 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = zext %splat to + %vc = sub %va, %vb + ret %vc +} + +define @vwsub_vv_nxv1i64_nxv1i8( %va, %vb) { +; CHECK-LABEL: vwsub_vv_nxv1i64_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf8 v10, v8 +; CHECK-NEXT: vsext.vf8 v8, v9 +; CHECK-NEXT: vsub.vv v8, v10, v8 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsubu_vv_nxv1i64_nxv1i8( %va, %vb) { +; CHECK-LABEL: vwsubu_vv_nxv1i64_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e8, mf8, ta, ma +; CHECK-NEXT: vwsubu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v8, v10 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsub_vx_nxv1i64_nxv1i8( %va, i8 %b) { +; CHECK-LABEL: vwsub_vx_nxv1i64_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf8 v10, v8 +; CHECK-NEXT: vsext.vf8 v8, v9 +; CHECK-NEXT: vsub.vv v8, v10, v8 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsubu_vx_nxv1i64_nxv1i8( %va, i8 %b) { +; CHECK-LABEL: vwsubu_vx_nxv1i64_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma +; CHECK-NEXT: vwsubu.vx v9, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v8, v9 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsub_wv_nxv1i64_nxv1i8( %va, %vb) { +; CHECK-LABEL: vwsub_wv_nxv1i64_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf8 v10, v9 +; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: ret + %vc = sext %vb to + %vd = sub %va, %vc + ret %vd +} + +define @vwsubu_wv_nxv1i64_nxv1i8( %va, %vb) { +; CHECK-LABEL: vwsubu_wv_nxv1i64_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf8 v10, v9 +; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: ret + %vc = zext %vb to + %vd = sub %va, %vc + ret %vd +} + +define @vwsub_wx_nxv1i64_nxv1i8( %va, i8 %b) { +; CHECK-LABEL: vwsub_wx_nxv1i64_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vsext.vf8 v10, v9 +; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = sext %splat to + %vc = sub %va, %vb + ret %vc +} + +define @vwsubu_wx_nxv1i64_nxv1i8( %va, i8 %b) { +; CHECK-LABEL: vwsubu_wx_nxv1i64_nxv1i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma +; CHECK-NEXT: vzext.vf8 v10, v9 +; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = zext %splat to + %vc = sub %va, %vb + ret %vc +} + +define @vwsub_vv_nxv2i64_nxv2i8( %va, %vb) { +; CHECK-LABEL: vwsub_vv_nxv2i64_nxv2i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf8 v10, v8 +; CHECK-NEXT: vsext.vf8 v12, v9 +; CHECK-NEXT: vsub.vv v8, v10, v12 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsubu_vv_nxv2i64_nxv2i8( %va, %vb) { +; CHECK-LABEL: vwsubu_vv_nxv2i64_nxv2i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e8, mf4, ta, ma +; CHECK-NEXT: vwsubu.vv v10, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v8, v10 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsub_vx_nxv2i64_nxv2i8( %va, i8 %b) { +; CHECK-LABEL: vwsub_vx_nxv2i64_nxv2i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf8 v10, v8 +; CHECK-NEXT: vsext.vf8 v12, v9 +; CHECK-NEXT: vsub.vv v8, v10, v12 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsubu_vx_nxv2i64_nxv2i8( %va, i8 %b) { +; CHECK-LABEL: vwsubu_vx_nxv2i64_nxv2i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma +; CHECK-NEXT: vwsubu.vx v10, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v8, v10 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsub_wv_nxv2i64_nxv2i8( %va, %vb) { +; CHECK-LABEL: vwsub_wv_nxv2i64_nxv2i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf8 v12, v10 +; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: ret + %vc = sext %vb to + %vd = sub %va, %vc + ret %vd +} + +define @vwsubu_wv_nxv2i64_nxv2i8( %va, %vb) { +; CHECK-LABEL: vwsubu_wv_nxv2i64_nxv2i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf8 v12, v10 +; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: ret + %vc = zext %vb to + %vd = sub %va, %vc + ret %vd +} + +define @vwsub_wx_nxv2i64_nxv2i8( %va, i8 %b) { +; CHECK-LABEL: vwsub_wx_nxv2i64_nxv2i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vsext.vf8 v12, v10 +; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = sext %splat to + %vc = sub %va, %vb + ret %vc +} + +define @vwsubu_wx_nxv2i64_nxv2i8( %va, i8 %b) { +; CHECK-LABEL: vwsubu_wx_nxv2i64_nxv2i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v10, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma +; CHECK-NEXT: vzext.vf8 v12, v10 +; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = zext %splat to + %vc = sub %va, %vb + ret %vc +} + +define @vwsub_vv_nxv4i64_nxv4i8( %va, %vb) { +; CHECK-LABEL: vwsub_vv_nxv4i64_nxv4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf8 v12, v8 +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsub.vv v8, v12, v16 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsubu_vv_nxv4i64_nxv4i8( %va, %vb) { +; CHECK-LABEL: vwsubu_vv_nxv4i64_nxv4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e8, mf2, ta, ma +; CHECK-NEXT: vwsubu.vv v12, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v8, v12 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsub_vx_nxv4i64_nxv4i8( %va, i8 %b) { +; CHECK-LABEL: vwsub_vx_nxv4i64_nxv4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf8 v12, v8 +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsub.vv v8, v12, v16 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsubu_vx_nxv4i64_nxv4i8( %va, i8 %b) { +; CHECK-LABEL: vwsubu_vx_nxv4i64_nxv4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma +; CHECK-NEXT: vwsubu.vx v12, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v8, v12 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsub_wv_nxv4i64_nxv4i8( %va, %vb) { +; CHECK-LABEL: vwsub_wv_nxv4i64_nxv4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf8 v16, v12 +; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: ret + %vc = sext %vb to + %vd = sub %va, %vc + ret %vd +} + +define @vwsubu_wv_nxv4i64_nxv4i8( %va, %vb) { +; CHECK-LABEL: vwsubu_wv_nxv4i64_nxv4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf8 v16, v12 +; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: ret + %vc = zext %vb to + %vd = sub %va, %vc + ret %vd +} + +define @vwsub_wx_nxv4i64_nxv4i8( %va, i8 %b) { +; CHECK-LABEL: vwsub_wx_nxv4i64_nxv4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v12, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vsext.vf8 v16, v12 +; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = sext %splat to + %vc = sub %va, %vb + ret %vc +} + +define @vwsubu_wx_nxv4i64_nxv4i8( %va, i8 %b) { +; CHECK-LABEL: vwsubu_wx_nxv4i64_nxv4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma +; CHECK-NEXT: vmv.v.x v12, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma +; CHECK-NEXT: vzext.vf8 v16, v12 +; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = zext %splat to + %vc = sub %va, %vb + ret %vc +} + +define @vwsub_vv_nxv8i64_nxv8i8( %va, %vb) { +; CHECK-LABEL: vwsub_vv_nxv8i64_nxv8i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf8 v16, v8 +; CHECK-NEXT: vsext.vf8 v24, v9 +; CHECK-NEXT: vsub.vv v8, v16, v24 +; CHECK-NEXT: ret + %vc = sext %va to + %vd = sext %vb to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsubu_vv_nxv8i64_nxv8i8( %va, %vb) { +; CHECK-LABEL: vwsubu_vv_nxv8i64_nxv8i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e8, m1, ta, ma +; CHECK-NEXT: vwsubu.vv v16, v8, v9 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf4 v8, v16 +; CHECK-NEXT: ret + %vc = zext %va to + %vd = zext %vb to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsub_vx_nxv8i64_nxv8i8( %va, i8 %b) { +; CHECK-LABEL: vwsub_vx_nxv8i64_nxv8i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.x v9, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf8 v16, v8 +; CHECK-NEXT: vsext.vf8 v24, v9 +; CHECK-NEXT: vsub.vv v8, v16, v24 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = sext %va to + %vd = sext %splat to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsubu_vx_nxv8i64_nxv8i8( %va, i8 %b) { +; CHECK-LABEL: vwsubu_vx_nxv8i64_nxv8i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma +; CHECK-NEXT: vwsubu.vx v16, v8, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf4 v8, v16 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vc = zext %va to + %vd = zext %splat to + %ve = sub %vc, %vd + ret %ve +} + +define @vwsub_wv_nxv8i64_nxv8i8( %va, %vb) { +; CHECK-LABEL: vwsub_wv_nxv8i64_nxv8i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf8 v24, v16 +; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: ret + %vc = sext %vb to + %vd = sub %va, %vc + ret %vd +} + +define @vwsubu_wv_nxv8i64_nxv8i8( %va, %vb) { +; CHECK-LABEL: vwsubu_wv_nxv8i64_nxv8i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf8 v24, v16 +; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: ret + %vc = zext %vb to + %vd = sub %va, %vc + ret %vd +} + +define @vwsub_wx_nxv8i64_nxv8i8( %va, i8 %b) { +; CHECK-LABEL: vwsub_wx_nxv8i64_nxv8i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.x v16, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vsext.vf8 v24, v16 +; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = sext %splat to + %vc = sub %va, %vb + ret %vc +} + +define @vwsubu_wx_nxv8i64_nxv8i8( %va, i8 %b) { +; CHECK-LABEL: vwsubu_wx_nxv8i64_nxv8i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma +; CHECK-NEXT: vmv.v.x v16, a0 +; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; CHECK-NEXT: vzext.vf8 v24, v16 +; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: ret + %head = insertelement poison, i8 %b, i8 0 + %splat = shufflevector %head, poison, zeroinitializer + %vb = zext %splat to + %vc = sub %va, %vb + ret %vc +} -- GitLab From 7a0acccd81df268dc7ad4c0358c42552789f19b4 Mon Sep 17 00:00:00 2001 From: Vlad Serebrennikov Date: Wed, 6 Mar 2024 12:21:52 +0400 Subject: [PATCH 260/929] [clang] Add some CodeGen tests for CWG 4xx issues (#83715) This patch covers the following defect reports: [CWG438](https://cplusplus.github.io/CWG/issues/438.html) "Possible flaw in wording for multiple accesses to object between sequence points", [CWG439](https://cplusplus.github.io/CWG/issues/439.html) "Guarantees on casting pointer back to cv-qualified version of original type", [CWG441](https://cplusplus.github.io/CWG/issues/441.html) "Ordering of static reference initialization", [CWG462](https://cplusplus.github.io/CWG/issues/462.html) "Lifetime of temporaries bound to comma expressions", [CWG492](https://cplusplus.github.io/CWG/issues/492.html) "`typeid` constness inconsistent with example". [CWG475](https://cplusplus.github.io/CWG/issues/475.html) "When is `std::uncaught_exception()` true? (take 2)" requires a libc++abi test. As for [CWG454](https://cplusplus.github.io/CWG/issues/454.html) "When is a definition of a static data member required?", I don't feel confident in my understanding of it, so skipping over it. --- clang/test/CXX/drs/dr438.cpp | 25 ++++++++++++++++++++++++ clang/test/CXX/drs/dr439.cpp | 30 ++++++++++++++++++++++++++++ clang/test/CXX/drs/dr441.cpp | 38 ++++++++++++++++++++++++++++++++++++ clang/test/CXX/drs/dr462.cpp | 33 +++++++++++++++++++++++++++++++ clang/test/CXX/drs/dr492.cpp | 37 +++++++++++++++++++++++++++++++++++ clang/test/CXX/drs/dr4xx.cpp | 12 ++++++------ clang/www/cxx_dr_status.html | 10 +++++----- 7 files changed, 174 insertions(+), 11 deletions(-) create mode 100644 clang/test/CXX/drs/dr438.cpp create mode 100644 clang/test/CXX/drs/dr439.cpp create mode 100644 clang/test/CXX/drs/dr441.cpp create mode 100644 clang/test/CXX/drs/dr462.cpp create mode 100644 clang/test/CXX/drs/dr492.cpp diff --git a/clang/test/CXX/drs/dr438.cpp b/clang/test/CXX/drs/dr438.cpp new file mode 100644 index 000000000000..a6ed39b88c24 --- /dev/null +++ b/clang/test/CXX/drs/dr438.cpp @@ -0,0 +1,25 @@ +// RUN: %clang_cc1 -std=c++98 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++11 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++14 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++17 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++20 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++23 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++2c %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK + +namespace dr438 { // dr438: 2.7 + +void f() { + long A[2]; + A[0] = 0; + A[A[0]] = 1; +} + +} // namespace dr438 + +// CHECK-LABEL: define {{.*}} void @dr438::f()() +// CHECK: [[A:%.+]] = alloca [2 x i64] +// CHECK: {{.+}} = getelementptr inbounds [2 x i64], ptr [[A]], i64 0, i64 0 +// CHECK: [[ARRAYIDX1:%.+]] = getelementptr inbounds [2 x i64], ptr [[A]], i64 0, i64 0 +// CHECK-NEXT: [[TEMPIDX:%.+]] = load i64, ptr [[ARRAYIDX1]] +// CHECK-NEXT: [[ARRAYIDX2:%.+]] = getelementptr inbounds [2 x i64], ptr [[A]], i64 0, i64 [[TEMPIDX]] +// CHECK-LABEL: } diff --git a/clang/test/CXX/drs/dr439.cpp b/clang/test/CXX/drs/dr439.cpp new file mode 100644 index 000000000000..46960af93bb9 --- /dev/null +++ b/clang/test/CXX/drs/dr439.cpp @@ -0,0 +1,30 @@ +// RUN: %clang_cc1 -std=c++98 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++11 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++14 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++17 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++20 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++23 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++2c %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK + +namespace dr439 { // dr439: 2.7 + +void f() { + int* p1 = new int; + const int* p2 = static_cast(static_cast(p1)); + bool b = p1 == p2; // b will have the value true. +} + +} // namespace dr439 + +// We're checking that p2 was copied from p1, and then was carried over +// to the comparison without change. + +// CHECK-LABEL: define {{.*}} void @dr439::f()() +// CHECK: [[P1:%.+]] = alloca ptr, align 8 +// CHECK-NEXT: [[P2:%.+]] = alloca ptr, align 8 +// CHECK: [[TEMP0:%.+]] = load ptr, ptr [[P1]] +// CHECK-NEXT: store ptr [[TEMP0:%.+]], ptr [[P2]] +// CHECK-NEXT: [[TEMP1:%.+]] = load ptr, ptr [[P1]] +// CHECK-NEXT: [[TEMP2:%.+]] = load ptr, ptr [[P2]] +// CHECK-NEXT: {{.*}} = icmp eq ptr [[TEMP1]], [[TEMP2]] +// CHECK-LABEL: } diff --git a/clang/test/CXX/drs/dr441.cpp b/clang/test/CXX/drs/dr441.cpp new file mode 100644 index 000000000000..6504bba689d2 --- /dev/null +++ b/clang/test/CXX/drs/dr441.cpp @@ -0,0 +1,38 @@ +// RUN: %clang_cc1 -std=c++98 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++11 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++14 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++17 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++20 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++23 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++2c %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK + +namespace dr441 { // dr441: 2.7 + +struct A { + A() {} +}; + +A dynamic_init; +int i; +int& ir = i; +int* ip = &i; + +} // namespace dr441 + +// CHECK-DAG: @dr441::dynamic_init = global %"struct.dr441::A" zeroinitializer +// CHECK-DAG: @dr441::i = global i32 0 +// CHECK-DAG: @dr441::ir = constant ptr @dr441::i +// CHECK-DAG: @dr441::ip = global ptr @dr441::i +// CHECK-DAG: @llvm.global_ctors = appending global [{{.+}}] [{ {{.+}} } { {{.+}}, ptr @_GLOBAL__sub_I_dr441.cpp, {{.+}} }] + +// CHECK-LABEL: define {{.*}} void @__cxx_global_var_init() +// CHECK-NEXT: entry: +// CHECK-NEXT: call void @dr441::A::A()({{.*}} @dr441::dynamic_init) +// CHECK-NEXT: ret void +// CHECK-NEXT: } + +// CHECK-LABEL: define {{.*}} void @_GLOBAL__sub_I_dr441.cpp() +// CHECK-NEXT: entry: +// CHECK-NEXT: call void @__cxx_global_var_init() +// CHECK-NEXT: ret void +// CHECK-NEXT: } diff --git a/clang/test/CXX/drs/dr462.cpp b/clang/test/CXX/drs/dr462.cpp new file mode 100644 index 000000000000..2b268778ea10 --- /dev/null +++ b/clang/test/CXX/drs/dr462.cpp @@ -0,0 +1,33 @@ +// RUN: %clang_cc1 -std=c++98 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++11 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++14 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++17 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++20 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++23 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++2c %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK + +#if __cplusplus == 199711L +#define NOTHROW throw() +#else +#define NOTHROW noexcept(true) +#endif + +namespace dr462 { // dr462: 2.7 + +struct A { + ~A() NOTHROW {} +}; + +extern void full_expr_fence() NOTHROW; + +void f() { + const A& r = (3, A()); + full_expr_fence(); +} + +} // namespace dr462 + +// CHECK-LABEL: define {{.*}} void @dr462::f()() +// CHECK: call void @dr462::full_expr_fence()() +// CHECK: call void @dr462::A::~A() +// CHECK-LABEL: } diff --git a/clang/test/CXX/drs/dr492.cpp b/clang/test/CXX/drs/dr492.cpp new file mode 100644 index 000000000000..f53f1cb54124 --- /dev/null +++ b/clang/test/CXX/drs/dr492.cpp @@ -0,0 +1,37 @@ +// RUN: %clang_cc1 -std=c++98 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++11 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++14 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++17 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++20 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++23 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++2c %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK + +#if __cplusplus == 199711L +#define NOTHROW throw() +#else +#define NOTHROW noexcept(true) +#endif + +namespace std { +struct type_info { + const char* name() const NOTHROW; +}; +} + +namespace dr492 { // dr492: 2.7 + +void f() { + typeid(int).name(); + typeid(const int).name(); + typeid(volatile int).name(); + typeid(const volatile int).name(); +} + +} // namespace dr492 + +// CHECK-LABEL: define {{.*}} void @dr492::f()() +// CHECK: {{.*}} = call {{.*}} @std::type_info::name() const({{.*}} @typeinfo for int) +// CHECK-NEXT: {{.*}} = call {{.*}} @std::type_info::name() const({{.*}} @typeinfo for int) +// CHECK-NEXT: {{.*}} = call {{.*}} @std::type_info::name() const({{.*}} @typeinfo for int) +// CHECK-NEXT: {{.*}} = call {{.*}} @std::type_info::name() const({{.*}} @typeinfo for int) +// CHECK-LABEL: } diff --git a/clang/test/CXX/drs/dr4xx.cpp b/clang/test/CXX/drs/dr4xx.cpp index 612a152aec4c..343c4ee6f334 100644 --- a/clang/test/CXX/drs/dr4xx.cpp +++ b/clang/test/CXX/drs/dr4xx.cpp @@ -698,9 +698,9 @@ namespace dr437 { // dr437: sup 1308 }; } -// dr438 FIXME write a codegen test -// dr439 FIXME write a codegen test -// dr441 FIXME write a codegen test +// dr438 is in dr438.cpp +// dr439 is in dr439.cpp +// dr441 is in dr441.cpp // dr442: sup 348 // dr443: na @@ -943,7 +943,7 @@ namespace dr460 { // dr460: yes } // dr461: na -// dr462 FIXME write a codegen test +// dr462 is in dr462.cpp // dr463: na // dr464: na // dr465: na @@ -1089,7 +1089,7 @@ namespace dr474 { // dr474: 3.4 } } -// dr475 FIXME write a codegen test +// dr475 FIXME write a libc++abi test namespace dr477 { // dr477: 3.5 struct A { @@ -1437,7 +1437,7 @@ namespace dr491 { // dr491: dup 413 // expected-error@-1 {{excess elements in array initializer}} } -// dr492 FIXME write a codegen test +// dr492 is in dr492.cpp namespace dr493 { // dr493: dup 976 struct X { diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html index 8b638e06f4aa..b13401625a6f 100755 --- a/clang/www/cxx_dr_status.html +++ b/clang/www/cxx_dr_status.html @@ -2668,13 +2668,13 @@ of class templates 438 CD2 Possible flaw in wording for multiple accesses to object between sequence points - Unknown + Clang 2.7 439 CD1 Guarantees on casting pointer back to cv-qualified version of original type - Unknown + Clang 2.7 440 @@ -2686,7 +2686,7 @@ of class templates 441 CD1 Ordering of static reference initialization - Unknown + Clang 2.7 442 @@ -2812,7 +2812,7 @@ of class templates 462 CD3 Lifetime of temporaries bound to comma expressions - Unknown + Clang 2.7 463 @@ -2992,7 +2992,7 @@ of class templates 492 CD1 typeid constness inconsistent with example - Unknown + Clang 2.7 493 -- GitLab From 9f45c5e1a65a1abf4920b617d36ed05e73c04bea Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Wed, 6 Mar 2024 09:33:51 +0100 Subject: [PATCH 261/929] [InstCombine] Fix infinite loop in select equivalence fold (#84036) When replacing with a non-constant, it's possible that the result of the simplification is actually more complicated than the original, and may result in an infinite combine loop. Mitigate the issue by requiring that either the replacement or simplification result is constant, which should ensure that it's simpler. While this check is crude, it does not appear to cause optimization regressions in real-world code in practice. Fixes https://github.com/llvm/llvm-project/issues/83127. --- .../InstCombine/InstCombineSelect.cpp | 9 ++++- llvm/test/Transforms/InstCombine/select.ll | 38 ++++++++++++++++++- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp index c47bc33df070..aee18d770f72 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp @@ -1285,7 +1285,11 @@ Instruction *InstCombinerImpl::foldSelectValueEquivalence(SelectInst &Sel, isGuaranteedNotToBeUndefOrPoison(CmpRHS, SQ.AC, &Sel, &DT)) { if (Value *V = simplifyWithOpReplaced(TrueVal, CmpLHS, CmpRHS, SQ, /* AllowRefinement */ true)) - return replaceOperand(Sel, Swapped ? 2 : 1, V); + // Require either the replacement or the simplification result to be a + // constant to avoid infinite loops. + // FIXME: Make this check more precise. + if (isa(CmpRHS) || isa(V)) + return replaceOperand(Sel, Swapped ? 2 : 1, V); // Even if TrueVal does not simplify, we can directly replace a use of // CmpLHS with CmpRHS, as long as the instruction is not used anywhere @@ -1303,7 +1307,8 @@ Instruction *InstCombinerImpl::foldSelectValueEquivalence(SelectInst &Sel, isGuaranteedNotToBeUndefOrPoison(CmpLHS, SQ.AC, &Sel, &DT)) if (Value *V = simplifyWithOpReplaced(TrueVal, CmpRHS, CmpLHS, SQ, /* AllowRefinement */ true)) - return replaceOperand(Sel, Swapped ? 2 : 1, V); + if (isa(CmpLHS) || isa(V)) + return replaceOperand(Sel, Swapped ? 2 : 1, V); auto *FalseInst = dyn_cast(FalseVal); if (!FalseInst) diff --git a/llvm/test/Transforms/InstCombine/select.ll b/llvm/test/Transforms/InstCombine/select.ll index 4676129e3a1c..a84904106ece 100644 --- a/llvm/test/Transforms/InstCombine/select.ll +++ b/llvm/test/Transforms/InstCombine/select.ll @@ -2849,12 +2849,14 @@ define i8 @select_replacement_sub(i8 %x, i8 %y, i8 %z) { ret i8 %sel } +; FIXME: This is safe to fold. define i8 @select_replacement_shift_noundef(i8 %x, i8 %y, i8 %z) { ; CHECK-LABEL: @select_replacement_shift_noundef( ; CHECK-NEXT: [[SHR:%.*]] = lshr exact i8 [[X:%.*]], 1 ; CHECK-NEXT: call void @use_i8(i8 noundef [[SHR]]) ; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[SHR]], [[Y:%.*]] -; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], i8 [[X]], i8 [[Z:%.*]] +; CHECK-NEXT: [[SHL:%.*]] = shl i8 [[Y]], 1 +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], i8 [[SHL]], i8 [[Z:%.*]] ; CHECK-NEXT: ret i8 [[SEL]] ; %shr = lshr exact i8 %x, 1 @@ -2904,6 +2906,40 @@ define i32 @select_replacement_loop2(i32 %arg, i32 %arg2) { ret i32 %sel } +define i8 @select_replacement_loop3(i32 noundef %x) { +; CHECK-LABEL: @select_replacement_loop3( +; CHECK-NEXT: [[TRUNC:%.*]] = trunc i32 [[X:%.*]] to i8 +; CHECK-NEXT: [[REV:%.*]] = call i8 @llvm.bitreverse.i8(i8 [[TRUNC]]) +; CHECK-NEXT: [[EXT:%.*]] = zext i8 [[REV]] to i32 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[EXT]], [[X]] +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], i8 [[TRUNC]], i8 0 +; CHECK-NEXT: ret i8 [[SEL]] +; + %trunc = trunc i32 %x to i8 + %rev = call i8 @llvm.bitreverse.i8(i8 %trunc) + %ext = zext i8 %rev to i32 + %cmp = icmp eq i32 %ext, %x + %sel = select i1 %cmp, i8 %trunc, i8 0 + ret i8 %sel +} + +define i16 @select_replacement_loop4(i16 noundef %p_12) { +; CHECK-LABEL: @select_replacement_loop4( +; CHECK-NEXT: [[CMP1:%.*]] = icmp ult i16 [[P_12:%.*]], 2 +; CHECK-NEXT: [[AND1:%.*]] = and i16 [[P_12]], 1 +; CHECK-NEXT: [[AND2:%.*]] = select i1 [[CMP1]], i16 [[AND1]], i16 0 +; CHECK-NEXT: [[CMP2:%.*]] = icmp eq i16 [[AND2]], [[P_12]] +; CHECK-NEXT: [[AND3:%.*]] = select i1 [[CMP2]], i16 [[AND1]], i16 0 +; CHECK-NEXT: ret i16 [[AND3]] +; + %cmp1 = icmp ult i16 %p_12, 2 + %and1 = and i16 %p_12, 1 + %and2 = select i1 %cmp1, i16 %and1, i16 0 + %cmp2 = icmp eq i16 %and2, %p_12 + %and3 = select i1 %cmp2, i16 %and1, i16 0 + ret i16 %and3 +} + define ptr @select_replacement_gep_inbounds(ptr %base, i64 %offset) { ; CHECK-LABEL: @select_replacement_gep_inbounds( ; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[BASE:%.*]], i64 [[OFFSET:%.*]] -- GitLab From 6cdf596c52f028ea7d150e0696f967fbff443ccf Mon Sep 17 00:00:00 2001 From: hanbeom Date: Wed, 6 Mar 2024 17:42:33 +0900 Subject: [PATCH 262/929] [InstCombine] If inst in unreachable refers to an inst change it to poison (#78444) Instructions in unreachable basic blocks are removed, but terminators are not. In this case, even instructions that are only referenced by a terminator, such as a return instruction, cannot be processed properly. This patch changes the operand of a return instruction in an unreachable basic block to poison if it refers to the instruction, allowing the instruction to be properly processed. Fixes #65107. --- llvm/include/llvm/Transforms/Utils/Local.h | 7 + .../InstCombine/InstructionCombining.cpp | 9 +- llvm/lib/Transforms/Utils/Local.cpp | 22 ++- .../InstCombine/phi-select-constant.ll | 3 +- llvm/test/Transforms/InstCombine/pr63791.ll | 2 +- .../InstCombine/sink_to_unreachable.ll | 154 ++++++++++++++++++ .../InstCombine/unreachable-code.ll | 57 +++++++ 7 files changed, 246 insertions(+), 8 deletions(-) diff --git a/llvm/include/llvm/Transforms/Utils/Local.h b/llvm/include/llvm/Transforms/Utils/Local.h index 8dc843d2eaf6..cc1336287436 100644 --- a/llvm/include/llvm/Transforms/Utils/Local.h +++ b/llvm/include/llvm/Transforms/Utils/Local.h @@ -357,6 +357,13 @@ Value *salvageDebugInfoImpl(Instruction &I, uint64_t CurrentLocOps, bool replaceAllDbgUsesWith(Instruction &From, Value &To, Instruction &DomPoint, DominatorTree &DT); +/// If a terminator in an unreachable basic block has an operand of type +/// Instruction, transform it into poison. Return true if any operands +/// are changed to poison. Original Values prior to being changed to poison +/// are returned in \p PoisonedValues. +bool handleUnreachableTerminator(Instruction *I, + SmallVectorImpl &PoisonedValues); + /// Remove all instructions from a basic block other than its terminator /// and any present EH pad instructions. Returns a pair where the first element /// is the number of instructions (excluding debug info intrinsics) that have diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index 80ce0c9275b2..fec33c5ea53b 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -3241,9 +3241,12 @@ void InstCombinerImpl::handleUnreachableFrom( MadeIRChange = true; } - // RemoveDIs: to match behaviour in dbg.value mode, drop debug-info on - // terminator too. - BB->getTerminator()->dropDbgValues(); + SmallVector Changed; + if (handleUnreachableTerminator(BB->getTerminator(), Changed)) { + MadeIRChange = true; + for (Value *V : Changed) + addToWorklist(cast(V)); + } // Handle potentially dead successors. for (BasicBlock *Succ : successors(BB)) diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp index c4a8843f2840..d3bb89075015 100644 --- a/llvm/lib/Transforms/Utils/Local.cpp +++ b/llvm/lib/Transforms/Utils/Local.cpp @@ -2762,6 +2762,23 @@ bool llvm::replaceAllDbgUsesWith(Instruction &From, Value &To, return false; } +bool llvm::handleUnreachableTerminator( + Instruction *I, SmallVectorImpl &PoisonedValues) { + bool Changed = false; + // RemoveDIs: erase debug-info on this instruction manually. + I->dropDbgValues(); + for (Use &U : I->operands()) { + Value *Op = U.get(); + if (isa(Op) && !Op->getType()->isTokenTy()) { + U.set(PoisonValue::get(Op->getType())); + PoisonedValues.push_back(Op); + Changed = true; + } + } + + return Changed; +} + std::pair llvm::removeAllNonTerminatorAndEHPadInstructions(BasicBlock *BB) { unsigned NumDeadInst = 0; @@ -2769,8 +2786,9 @@ llvm::removeAllNonTerminatorAndEHPadInstructions(BasicBlock *BB) { // Delete the instructions backwards, as it has a reduced likelihood of // having to update as many def-use and use-def chains. Instruction *EndInst = BB->getTerminator(); // Last not to be deleted. - // RemoveDIs: erasing debug-info must be done manually. - EndInst->dropDbgValues(); + SmallVector Uses; + handleUnreachableTerminator(EndInst, Uses); + while (EndInst != &BB->front()) { // Delete the next to last instruction. Instruction *Inst = &*--EndInst->getIterator(); diff --git a/llvm/test/Transforms/InstCombine/phi-select-constant.ll b/llvm/test/Transforms/InstCombine/phi-select-constant.ll index 1260ef47f65e..601a3d208a08 100644 --- a/llvm/test/Transforms/InstCombine/phi-select-constant.ll +++ b/llvm/test/Transforms/InstCombine/phi-select-constant.ll @@ -140,12 +140,11 @@ end: define i16 @sink_to_unreachable_crash(i1 %a) { ; CHECK-LABEL: @sink_to_unreachable_crash( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[S:%.*]] = select i1 [[A:%.*]], i16 0, i16 5 ; CHECK-NEXT: br label [[INF_LOOP:%.*]] ; CHECK: inf_loop: ; CHECK-NEXT: br label [[INF_LOOP]] ; CHECK: unreachable: -; CHECK-NEXT: ret i16 [[S]] +; CHECK-NEXT: ret i16 poison ; entry: %s = select i1 %a, i16 0, i16 5 diff --git a/llvm/test/Transforms/InstCombine/pr63791.ll b/llvm/test/Transforms/InstCombine/pr63791.ll index a489b2e3e622..78cc1130fb33 100644 --- a/llvm/test/Transforms/InstCombine/pr63791.ll +++ b/llvm/test/Transforms/InstCombine/pr63791.ll @@ -17,7 +17,7 @@ define void @y() { ; CHECK: for.cond5.preheader.i: ; CHECK-NEXT: br i1 false, label [[FOR_INC19_I:%.*]], label [[FOR_COND1_LOOPEXIT_I:%.*]] ; CHECK: for.inc19.i: -; CHECK-NEXT: br i1 false, label [[FOR_INC19_I]], label [[FOR_COND1_LOOPEXIT_I]] +; CHECK-NEXT: br i1 poison, label [[FOR_INC19_I]], label [[FOR_COND1_LOOPEXIT_I]] ; entry: br label %for.cond.i diff --git a/llvm/test/Transforms/InstCombine/sink_to_unreachable.ll b/llvm/test/Transforms/InstCombine/sink_to_unreachable.ll index e788b634da88..02ed22217854 100644 --- a/llvm/test/Transforms/InstCombine/sink_to_unreachable.ll +++ b/llvm/test/Transforms/InstCombine/sink_to_unreachable.ll @@ -157,3 +157,157 @@ bb3: %p = phi i32 [0, %bb1], [%a, %bb2] ret i32 %p } + +define i1 @sink_to_unreachable_ret(i16 %X) { +; CHECK-LABEL: @sink_to_unreachable_ret( +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: br i1 true, label [[LOOP]], label [[UNREACH:%.*]] +; CHECK: unreach: +; CHECK-NEXT: ret i1 poison +; +entry: + br label %loop + +loop: + %p = icmp sgt i16 %X, 16 + br i1 true, label %loop, label %unreach + +unreach: + ret i1 %p +} + +define void @sink_to_unreachable_condbr(i16 %X) { +; CHECK-LABEL: @sink_to_unreachable_condbr( +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: br i1 true, label [[LOOP]], label [[UNREACH:%.*]] +; CHECK: unreach: +; CHECK-NEXT: br i1 poison, label [[DUMMY:%.*]], label [[LOOP]] +; CHECK: dummy: +; CHECK-NEXT: unreachable +; +entry: + br label %loop + +loop: + %p = icmp sgt i16 %X, 16 + br i1 true, label %loop, label %unreach + +unreach: + br i1 %p, label %dummy, label %loop + +dummy: + unreachable +} + +define void @sink_to_unreachable_switch(i16 %X) { +; CHECK-LABEL: @sink_to_unreachable_switch( +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: br i1 true, label [[LOOP]], label [[UNREACH:%.*]] +; CHECK: unreach: +; CHECK-NEXT: switch i16 poison, label [[UNREACH_RET:%.*]] [ +; CHECK-NEXT: ] +; CHECK: unreach.ret: +; CHECK-NEXT: unreachable +; +entry: + br label %loop + +loop: + %quantum = srem i16 %X, 32 + br i1 true, label %loop, label %unreach + +unreach: + switch i16 %quantum, label %unreach.ret [] + +unreach.ret: + unreachable +} + +define void @sink_to_unreachable_indirectbr(ptr %Ptr) { +; CHECK-LABEL: @sink_to_unreachable_indirectbr( +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: br i1 true, label [[LOOP]], label [[UNREACH:%.*]] +; CHECK: unreach: +; CHECK-NEXT: indirectbr ptr poison, [label %loop] +; +entry: + br label %loop + +loop: + %gep = getelementptr inbounds ptr, ptr %Ptr, i16 1 + br i1 true, label %loop, label %unreach + +unreach: + indirectbr ptr %gep, [label %loop] +} + +define void @sink_to_unreachable_invoke(ptr %Ptr) personality ptr @__CxxFrameHandler3 { +; CHECK-LABEL: @sink_to_unreachable_invoke( +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: br i1 true, label [[LOOP]], label [[UNREACH:%.*]] +; CHECK: unreach: +; CHECK-NEXT: invoke void poison(i1 false) +; CHECK-NEXT: to label [[DUMMY:%.*]] unwind label [[ICATCH_DISPATCH:%.*]] +; CHECK: unreach2: +; CHECK-NEXT: invoke void @__CxxFrameHandler3(ptr poison) +; CHECK-NEXT: to label [[DUMMY]] unwind label [[ICATCH_DISPATCH]] +; CHECK: unreach3: +; CHECK-NEXT: [[CLEAN:%.*]] = cleanuppad within none [] +; CHECK-NEXT: invoke void @__CxxFrameHandler3(ptr poison) [ "funclet"(token [[CLEAN]]) ] +; CHECK-NEXT: to label [[DUMMY]] unwind label [[ICATCH_DISPATCH]] +; CHECK: icatch.dispatch: +; CHECK-NEXT: [[TMP1:%.*]] = catchswitch within none [label %icatch] unwind to caller +; CHECK: icatch: +; CHECK-NEXT: [[TMP2:%.*]] = catchpad within [[TMP1]] [ptr null, i32 64, ptr null] +; CHECK-NEXT: catchret from [[TMP2]] to label [[DUMMY2:%.*]] +; CHECK: dummy: +; CHECK-NEXT: ret void +; CHECK: dummy2: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %gep = getelementptr inbounds ptr, ptr %Ptr, i16 1 + br i1 true, label %loop, label %unreach + +unreach: + invoke void %gep(i1 false) + to label %dummy unwind label %icatch.dispatch + +unreach2: + invoke void @__CxxFrameHandler3(ptr %gep) + to label %dummy unwind label %icatch.dispatch + +unreach3: + %clean = cleanuppad within none [] + invoke void @__CxxFrameHandler3(ptr %gep) [ "funclet"(token %clean) ] + to label %dummy unwind label %icatch.dispatch + +icatch.dispatch: + %tmp1 = catchswitch within none [label %icatch] unwind to caller + +icatch: + %tmp2 = catchpad within %tmp1 [ptr null, i32 64, ptr null] + catchret from %tmp2 to label %dummy2 + +dummy: + ret void + +dummy2: + ret void +} + +declare void @may_throw() +declare i32 @__CxxFrameHandler3(...) diff --git a/llvm/test/Transforms/InstCombine/unreachable-code.ll b/llvm/test/Transforms/InstCombine/unreachable-code.ll index 72ef1c79f3a8..bb4be9fa71f8 100644 --- a/llvm/test/Transforms/InstCombine/unreachable-code.ll +++ b/llvm/test/Transforms/InstCombine/unreachable-code.ll @@ -540,6 +540,63 @@ bb2: br label %bb } +declare void @invoke(ptr) +declare i32 @__gxx_personality_v0(...) +define void @test(i1 %x) personality ptr @__gxx_personality_v0 { +; CHECK-LABEL: define void @test +; CHECK-SAME: (i1 [[X:%.*]]) personality ptr @__gxx_personality_v0 { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[X]], label [[IF_ELSE:%.*]], label [[CLEAN1:%.*]] +; CHECK: if.else: +; CHECK-NEXT: store i32 1, ptr undef, align 4 +; CHECK-NEXT: invoke void @invoke(ptr poison) +; CHECK-NEXT: to label [[CONT:%.*]] unwind label [[LPAD5:%.*]] +; CHECK: cont: +; CHECK-NEXT: invoke void @invoke(ptr poison) +; CHECK-NEXT: to label [[CLEAN1]] unwind label [[LPAD6:%.*]] +; CHECK: lpad5: +; CHECK-NEXT: [[TMP0:%.*]] = landingpad { ptr, i32 } +; CHECK-NEXT: cleanup +; CHECK-NEXT: br label [[CLEAN1]] +; CHECK: lpad6: +; CHECK-NEXT: [[TMP1:%.*]] = landingpad { ptr, i32 } +; CHECK-NEXT: cleanup +; CHECK-NEXT: br label [[CLEAN2:%.*]] +; CHECK: clean1: +; CHECK-NEXT: ret void +; CHECK: clean2: +; CHECK-NEXT: ret void +; +entry: + %ref = alloca ptr + br i1 %x, label %if.else, label %clean1 + +if.else: + store i32 1, ptr undef + invoke void @invoke(ptr %ref) + to label %cont unwind label %lpad5 + +cont: + invoke void @invoke(ptr %ref) + to label %clean1 unwind label %lpad6 + +lpad5: + %13 = landingpad { ptr, i32 } + cleanup + br label %clean1 + +lpad6: + %14 = landingpad { ptr, i32 } + cleanup + br label %clean2 + +clean1: + ret void + +clean2: + ret void +} + ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: ; DEFAULT_ITER: {{.*}} ; MAX1: {{.*}} -- GitLab From aced81c0a5bf30dda99fde2e28364426de4c18d3 Mon Sep 17 00:00:00 2001 From: Mariya Podchishchaeva Date: Wed, 6 Mar 2024 11:46:35 +0300 Subject: [PATCH 263/929] [C23] Implement N3018: The constexpr specifier for object definitions (#73099) The implementation mostly reuses C++ code paths where possible, including narrowing check in order to provide diagnostic messages in case initializer for constexpr variable is not exactly representable in target type. The following won't work due to lack of support for other features: - Diagnosing of underspecified declarations involving constexpr - Constexpr attached to compound literals Also due to lack of support for char8_t some of examples with utf-8 strings don't work properly. Fixes https://github.com/llvm/llvm-project/issues/64742 --- clang/docs/ReleaseNotes.rst | 3 + clang/include/clang/AST/Expr.h | 11 + .../clang/Basic/DiagnosticSemaKinds.td | 12 + clang/include/clang/Basic/TokenKinds.def | 2 +- clang/lib/AST/Decl.cpp | 25 +- clang/lib/AST/ExprConstant.cpp | 10 +- clang/lib/Parse/ParseDecl.cpp | 2 + clang/lib/Sema/DeclSpec.cpp | 14 + clang/lib/Sema/SemaDecl.cpp | 79 +++- clang/lib/Sema/SemaInit.cpp | 119 +++++- clang/lib/Sema/SemaOverload.cpp | 65 +++- clang/test/C/C2x/n3018.c | 87 +++++ clang/test/Parser/c23-constexpr.c | 15 + clang/test/Sema/constexpr.c | 359 ++++++++++++++++++ clang/www/c_status.html | 2 +- 15 files changed, 766 insertions(+), 39 deletions(-) create mode 100644 clang/test/C/C2x/n3018.c create mode 100644 clang/test/Parser/c23-constexpr.c create mode 100644 clang/test/Sema/constexpr.c diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index b074055a4eaf..0ff4a93b15ea 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -143,6 +143,9 @@ C23 Feature Support macros typically exposed from ````, such as ``PRIb8``. (`#81896: `_). +- Clang now supports `N3018 The constexpr specifier for object definitions` + `_. + Non-comprehensive list of changes in this release ------------------------------------------------- diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h index bf0622bdeca3..f7857d63bdef 100644 --- a/clang/include/clang/AST/Expr.h +++ b/clang/include/clang/AST/Expr.h @@ -1875,6 +1875,17 @@ public: llvm_unreachable("Unsupported character width!"); } + // Get code unit but preserve sign info. + int64_t getCodeUnitS(size_t I, uint64_t BitWidth) const { + int64_t V = getCodeUnit(I); + if (isOrdinary() || isWide()) { + unsigned Width = getCharByteWidth() * BitWidth; + llvm::APInt AInt(Width, (uint64_t)V); + V = AInt.getSExtValue(); + } + return V; + } + unsigned getByteLength() const { return getCharByteWidth() * getLength(); } unsigned getLength() const { return *getTrailingObjects(); } unsigned getCharByteWidth() const { return StringLiteralBits.CharByteWidth; } diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index 708b3c7f1ede..b007ff7d8ccf 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -2946,6 +2946,18 @@ def warn_private_extern : Warning< def note_private_extern : Note< "use __attribute__((visibility(\"hidden\"))) attribute instead">; +// C23 constexpr +def err_c23_constexpr_not_variable : Error< + "'constexpr' can only be used in variable declarations">; +def err_c23_constexpr_invalid_type : Error< + "constexpr variable cannot have type %0">; +def err_c23_constexpr_init_not_representable : Error< + "constexpr initializer evaluates to %0 which is not exactly representable in type %1">; +def err_c23_constexpr_init_type_mismatch : Error< + "constexpr initializer for type %0 is of type %1">; +def err_c23_constexpr_pointer_not_null : Error< + "constexpr pointer initializer is not null">; + // C++ Concepts def err_concept_decls_may_only_appear_in_global_namespace_scope : Error< "concept declarations may only appear in global or namespace scope">; diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def index 1d16e4843615..3a96f8a4d22b 100644 --- a/clang/include/clang/Basic/TokenKinds.def +++ b/clang/include/clang/Basic/TokenKinds.def @@ -393,7 +393,7 @@ CXX11_KEYWORD(alignas , KEYC23) CXX11_UNARY_EXPR_OR_TYPE_TRAIT(alignof, AlignOf, KEYC23) CXX11_KEYWORD(char16_t , KEYNOMS18) CXX11_KEYWORD(char32_t , KEYNOMS18) -CXX11_KEYWORD(constexpr , 0) +CXX11_KEYWORD(constexpr , KEYC23) CXX11_KEYWORD(decltype , 0) CXX11_KEYWORD(noexcept , 0) CXX11_KEYWORD(nullptr , KEYC23) diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp index 57a92357f6e5..59c039f1f8da 100644 --- a/clang/lib/AST/Decl.cpp +++ b/clang/lib/AST/Decl.cpp @@ -2465,7 +2465,7 @@ bool VarDecl::mightBeUsableInConstantExpressions(const ASTContext &C) const { // OpenCL permits const integral variables to be used in constant // expressions, like in C++98. - if (!Lang.CPlusPlus && !Lang.OpenCL) + if (!Lang.CPlusPlus && !Lang.OpenCL && !Lang.C23) return false; // Function parameters are never usable in constant expressions. @@ -2487,14 +2487,19 @@ bool VarDecl::mightBeUsableInConstantExpressions(const ASTContext &C) const { if (!getType().isConstant(C) || getType().isVolatileQualified()) return false; - // In C++, const, non-volatile variables of integral or enumeration types - // can be used in constant expressions. - if (getType()->isIntegralOrEnumerationType()) + // In C++, but not in C, const, non-volatile variables of integral or + // enumeration types can be used in constant expressions. + if (getType()->isIntegralOrEnumerationType() && !Lang.C23) return true; + // C23 6.6p7: An identifier that is: + // ... + // - declared with storage-class specifier constexpr and has an object type, + // is a named constant, ... such a named constant is a constant expression + // with the type and value of the declared object. // Additionally, in C++11, non-volatile constexpr variables can be used in // constant expressions. - return Lang.CPlusPlus11 && isConstexpr(); + return (Lang.CPlusPlus11 || Lang.C23) && isConstexpr(); } bool VarDecl::isUsableInConstantExpressions(const ASTContext &Context) const { @@ -2572,11 +2577,11 @@ APValue *VarDecl::evaluateValueImpl(SmallVectorImpl &Notes, bool Result = Init->EvaluateAsInitializer(Eval->Evaluated, Ctx, this, Notes, IsConstantInitialization); - // In C++, this isn't a constant initializer if we produced notes. In that + // In C++/C23, this isn't a constant initializer if we produced notes. In that // case, we can't keep the result, because it may only be correct under the // assumption that the initializer is a constant context. - if (IsConstantInitialization && Ctx.getLangOpts().CPlusPlus && - !Notes.empty()) + if (IsConstantInitialization && + (Ctx.getLangOpts().CPlusPlus || Ctx.getLangOpts().C23) && !Notes.empty()) Result = false; // Ensure the computed APValue is cleaned up later if evaluation succeeded, @@ -2634,7 +2639,9 @@ bool VarDecl::checkForConstantInitialization( // std::is_constant_evaluated()). assert(!Eval->WasEvaluated && "already evaluated var value before checking for constant init"); - assert(getASTContext().getLangOpts().CPlusPlus && "only meaningful in C++"); + assert((getASTContext().getLangOpts().CPlusPlus || + getASTContext().getLangOpts().C23) && + "only meaningful in C++/C23"); assert(!getInit()->isValueDependent()); diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp index 4b6ccafbb7f0..d8ca35740fbc 100644 --- a/clang/lib/AST/ExprConstant.cpp +++ b/clang/lib/AST/ExprConstant.cpp @@ -4133,6 +4133,10 @@ static CompleteObject findCompleteObject(EvalInfo &Info, const Expr *E, } bool IsConstant = BaseType.isConstant(Info.Ctx); + bool ConstexprVar = false; + if (const auto *VD = dyn_cast_if_present( + Info.EvaluatingDecl.dyn_cast())) + ConstexprVar = VD->isConstexpr(); // Unless we're looking at a local variable or argument in a constexpr call, // the variable we're reading must be const. @@ -4152,6 +4156,9 @@ static CompleteObject findCompleteObject(EvalInfo &Info, const Expr *E, return CompleteObject(); } else if (VD->isConstexpr()) { // OK, we can read this variable. + } else if (Info.getLangOpts().C23 && ConstexprVar) { + Info.FFDiag(E); + return CompleteObject(); } else if (BaseType->isIntegralOrEnumerationType()) { if (!IsConstant) { if (!IsAccess) @@ -15826,7 +15833,8 @@ bool Expr::EvaluateAsInitializer(APValue &Value, const ASTContext &Ctx, EStatus.Diag = &Notes; EvalInfo Info(Ctx, EStatus, - (IsConstantInitialization && Ctx.getLangOpts().CPlusPlus) + (IsConstantInitialization && + (Ctx.getLangOpts().CPlusPlus || Ctx.getLangOpts().C23)) ? EvalInfo::EM_ConstantExpression : EvalInfo::EM_ConstantFold); Info.setEvaluatingDecl(VD, Value); diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp index ccbfea6a66fb..81f1c7112694 100644 --- a/clang/lib/Parse/ParseDecl.cpp +++ b/clang/lib/Parse/ParseDecl.cpp @@ -4265,6 +4265,8 @@ void Parser::ParseDeclarationSpecifiers( // constexpr, consteval, constinit specifiers case tok::kw_constexpr: + if (getLangOpts().C23) + Diag(Tok, diag::warn_c23_compat_keyword) << Tok.getName(); isInvalid = DS.SetConstexprSpec(ConstexprSpecKind::Constexpr, Loc, PrevSpec, DiagID); break; diff --git a/clang/lib/Sema/DeclSpec.cpp b/clang/lib/Sema/DeclSpec.cpp index aede602f1de8..b79683bb32a6 100644 --- a/clang/lib/Sema/DeclSpec.cpp +++ b/clang/lib/Sema/DeclSpec.cpp @@ -1377,6 +1377,20 @@ void DeclSpec::Finish(Sema &S, const PrintingPolicy &Policy) { ThreadStorageClassSpec = TSCS_unspecified; ThreadStorageClassSpecLoc = SourceLocation(); } + if (S.getLangOpts().C23 && + getConstexprSpecifier() == ConstexprSpecKind::Constexpr) { + S.Diag(ConstexprLoc, diag::err_invalid_decl_spec_combination) + << DeclSpec::getSpecifierName(getThreadStorageClassSpec()) + << SourceRange(getThreadStorageClassSpecLoc()); + } + } + + if (S.getLangOpts().C23 && + getConstexprSpecifier() == ConstexprSpecKind::Constexpr && + StorageClassSpec == SCS_extern) { + S.Diag(ConstexprLoc, diag::err_invalid_decl_spec_combination) + << DeclSpec::getSpecifierName(getStorageClassSpec()) + << SourceRange(getStorageClassSpecLoc()); } // If no type specifier was provided and we're parsing a language where diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 8fcaf6ab312b..210e2835184a 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -5147,6 +5147,8 @@ Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS, Diag(DS.getConstexprSpecLoc(), diag::err_constexpr_tag) << GetDiagnosticTypeSpecifierID(DS) << static_cast(DS.getConstexprSpecifier()); + else if (getLangOpts().C23) + Diag(DS.getConstexprSpecLoc(), diag::err_c23_constexpr_not_variable); else Diag(DS.getConstexprSpecLoc(), diag::err_constexpr_wrong_decl_kind) << static_cast(DS.getConstexprSpecifier()); @@ -8649,6 +8651,38 @@ static bool checkForConflictWithNonVisibleExternC(Sema &S, const T *ND, return false; } +static bool CheckC23ConstexprVarType(Sema &SemaRef, SourceLocation VarLoc, + QualType T) { + QualType CanonT = SemaRef.Context.getCanonicalType(T); + // C23 6.7.1p5: An object declared with storage-class specifier constexpr or + // any of its members, even recursively, shall not have an atomic type, or a + // variably modified type, or a type that is volatile or restrict qualified. + if (CanonT->isVariablyModifiedType()) { + SemaRef.Diag(VarLoc, diag::err_c23_constexpr_invalid_type) << T; + return true; + } + + // Arrays are qualified by their element type, so get the base type (this + // works on non-arrays as well). + CanonT = SemaRef.Context.getBaseElementType(CanonT); + + if (CanonT->isAtomicType() || CanonT.isVolatileQualified() || + CanonT.isRestrictQualified()) { + SemaRef.Diag(VarLoc, diag::err_c23_constexpr_invalid_type) << T; + return true; + } + + if (CanonT->isRecordType()) { + const RecordDecl *RD = CanonT->getAsRecordDecl(); + if (llvm::any_of(RD->fields(), [&SemaRef, VarLoc](const FieldDecl *F) { + return CheckC23ConstexprVarType(SemaRef, VarLoc, F->getType()); + })) + return true; + } + + return false; +} + void Sema::CheckVariableDeclarationType(VarDecl *NewVD) { // If the decl is already known invalid, don't check it. if (NewVD->isInvalidDecl()) @@ -8899,6 +8933,12 @@ void Sema::CheckVariableDeclarationType(VarDecl *NewVD) { return; } + if (getLangOpts().C23 && NewVD->isConstexpr() && + CheckC23ConstexprVarType(*this, NewVD->getLocation(), T)) { + NewVD->setInvalidDecl(); + return; + } + if (NewVD->isConstexpr() && !T->isDependentType() && RequireLiteralType(NewVD->getLocation(), T, diag::err_constexpr_var_non_literal)) { @@ -9281,6 +9321,22 @@ static FunctionDecl *CreateNewFunctionDecl(Sema &SemaRef, Declarator &D, FunctionDecl *NewFD = nullptr; bool isInline = D.getDeclSpec().isInlineSpecified(); + ConstexprSpecKind ConstexprKind = D.getDeclSpec().getConstexprSpecifier(); + if (ConstexprKind == ConstexprSpecKind::Constinit || + (SemaRef.getLangOpts().C23 && + ConstexprKind == ConstexprSpecKind::Constexpr)) { + + if (SemaRef.getLangOpts().C23) + SemaRef.Diag(D.getDeclSpec().getConstexprSpecLoc(), + diag::err_c23_constexpr_not_variable); + else + SemaRef.Diag(D.getDeclSpec().getConstexprSpecLoc(), + diag::err_constexpr_wrong_decl_kind) + << static_cast(ConstexprKind); + ConstexprKind = ConstexprSpecKind::Unspecified; + D.getMutableDeclSpec().ClearConstexprSpec(); + } + if (!SemaRef.getLangOpts().CPlusPlus) { // Determine whether the function was written with a prototype. This is // true when: @@ -9314,15 +9370,6 @@ static FunctionDecl *CreateNewFunctionDecl(Sema &SemaRef, Declarator &D, } ExplicitSpecifier ExplicitSpecifier = D.getDeclSpec().getExplicitSpecifier(); - - ConstexprSpecKind ConstexprKind = D.getDeclSpec().getConstexprSpecifier(); - if (ConstexprKind == ConstexprSpecKind::Constinit) { - SemaRef.Diag(D.getDeclSpec().getConstexprSpecLoc(), - diag::err_constexpr_wrong_decl_kind) - << static_cast(ConstexprKind); - ConstexprKind = ConstexprSpecKind::Unspecified; - D.getMutableDeclSpec().ClearConstexprSpec(); - } Expr *TrailingRequiresClause = D.getTrailingRequiresClause(); SemaRef.CheckExplicitObjectMemberFunction(DC, D, Name, R); @@ -13909,7 +13956,9 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) { VDecl->setStorageClass(SC_Extern); // C99 6.7.8p4. All file scoped initializers need to be constant. - if (!getLangOpts().CPlusPlus && !VDecl->isInvalidDecl()) + // Avoid duplicate diagnostics for constexpr variables. + if (!getLangOpts().CPlusPlus && !VDecl->isInvalidDecl() && + !VDecl->isConstexpr()) CheckForConstantInitializer(Init, DclT); } @@ -14520,9 +14569,13 @@ void Sema::CheckCompleteVariableDeclaration(VarDecl *var) { QualType baseType = Context.getBaseElementType(type); bool HasConstInit = true; + if (getLangOpts().C23 && var->isConstexpr() && !Init) + Diag(var->getLocation(), diag::err_constexpr_var_requires_const_init) + << var; + // Check whether the initializer is sufficiently constant. - if (getLangOpts().CPlusPlus && !type->isDependentType() && Init && - !Init->isValueDependent() && + if ((getLangOpts().CPlusPlus || (getLangOpts().C23 && var->isConstexpr())) && + !type->isDependentType() && Init && !Init->isValueDependent() && (GlobalStorage || var->isConstexpr() || var->mightBeUsableInConstantExpressions(Context))) { // If this variable might have a constant initializer or might be usable in @@ -14530,7 +14583,7 @@ void Sema::CheckCompleteVariableDeclaration(VarDecl *var) { // do this lazily, because the result might depend on things that change // later, such as which constexpr functions happen to be defined. SmallVector Notes; - if (!getLangOpts().CPlusPlus11) { + if (!getLangOpts().CPlusPlus11 && !getLangOpts().C23) { // Prior to C++11, in contexts where a constant initializer is required, // the set of valid constant initializers is described by syntactic rules // in [expr.const]p2-6. diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp index 060fe35ad968..011deed7a9a9 100644 --- a/clang/lib/Sema/SemaInit.cpp +++ b/clang/lib/Sema/SemaInit.cpp @@ -190,13 +190,35 @@ static void updateGNUCompoundLiteralRValue(Expr *E) { } } +static bool initializingConstexprVariable(const InitializedEntity &Entity) { + Decl *D = Entity.getDecl(); + const InitializedEntity *Parent = &Entity; + + while (Parent) { + D = Parent->getDecl(); + Parent = Parent->getParent(); + } + + if (const auto *VD = dyn_cast_if_present(D); VD && VD->isConstexpr()) + return true; + + return false; +} + +static void CheckC23ConstexprInitStringLiteral(const StringLiteral *SE, + Sema &SemaRef, QualType &TT); + static void CheckStringInit(Expr *Str, QualType &DeclT, const ArrayType *AT, - Sema &S) { + Sema &S, bool CheckC23ConstexprInit = false) { // Get the length of the string as parsed. auto *ConstantArrayTy = cast(Str->getType()->getAsArrayTypeUnsafe()); uint64_t StrLength = ConstantArrayTy->getSize().getZExtValue(); + if (CheckC23ConstexprInit) + if (const StringLiteral *SL = dyn_cast(Str->IgnoreParens())) + CheckC23ConstexprInitStringLiteral(SL, S, DeclT); + if (const IncompleteArrayType *IAT = dyn_cast(AT)) { // C99 6.7.8p14. We have an array of character type with unknown size // being initialized to a string literal. @@ -1476,7 +1498,9 @@ void InitListChecker::CheckSubElementType(const InitializedEntity &Entity, if (IsStringInit(expr, arrayType, SemaRef.Context) == SIF_None) { // FIXME: Should we do this checking in verify-only mode? if (!VerifyOnly) - CheckStringInit(expr, ElemType, arrayType, SemaRef); + CheckStringInit(expr, ElemType, arrayType, SemaRef, + SemaRef.getLangOpts().C23 && + initializingConstexprVariable(Entity)); if (StructuredList) UpdateStructuredListElement(StructuredList, StructuredIndex, expr); ++Index; @@ -1941,7 +1965,9 @@ void InitListChecker::CheckArrayType(const InitializedEntity &Entity, // constant for each string. // FIXME: Should we do these checks in verify-only mode too? if (!VerifyOnly) - CheckStringInit(IList->getInit(Index), DeclType, arrayType, SemaRef); + CheckStringInit(IList->getInit(Index), DeclType, arrayType, SemaRef, + SemaRef.getLangOpts().C23 && + initializingConstexprVariable(Entity)); if (StructuredList) { UpdateStructuredListElement(StructuredList, StructuredIndex, IList->getInit(Index)); @@ -8377,6 +8403,9 @@ static void DiagnoseNarrowingInInitList(Sema &S, QualType EntityType, const Expr *PostInit); +static void CheckC23ConstexprInitConversion(Sema &S, QualType FromType, + QualType ToType, Expr *Init); + /// Provide warnings when std::move is used on construction. static void CheckMoveOnConstruction(Sema &S, const Expr *InitExpr, bool IsReturnStmt) { @@ -9203,6 +9232,23 @@ ExprResult InitializationSequence::Perform(Sema &S, return ExprError(); CurInit = CurInitExprRes; + if (S.getLangOpts().C23 && initializingConstexprVariable(Entity)) { + CheckC23ConstexprInitConversion(S, SourceType, Entity.getType(), + CurInit.get()); + + // C23 6.7.1p6: If an object or subobject declared with storage-class + // specifier constexpr has pointer, integer, or arithmetic type, any + // explicit initializer value for it shall be null, an integer + // constant expression, or an arithmetic constant expression, + // respectively. + Expr::EvalResult ER; + if (Entity.getType()->getAs() && + CurInit.get()->EvaluateAsRValue(ER, S.Context) && + !ER.Val.isNullPointer()) { + S.Diag(Kind.getLocation(), diag::err_c23_constexpr_pointer_not_null); + } + } + bool Complained; if (S.DiagnoseAssignmentResult(ConvTy, Kind.getLocation(), Step->Type, SourceType, @@ -9220,7 +9266,9 @@ ExprResult InitializationSequence::Perform(Sema &S, QualType Ty = Step->Type; bool UpdateType = ResultType && Entity.getType()->isIncompleteArrayType(); CheckStringInit(CurInit.get(), UpdateType ? *ResultType : Ty, - S.Context.getAsArrayType(Ty), S); + S.Context.getAsArrayType(Ty), S, + S.getLangOpts().C23 && + initializingConstexprVariable(Entity)); break; } @@ -10509,6 +10557,69 @@ static void DiagnoseNarrowingInInitList(Sema &S, S.getLocForEndOfToken(PostInit->getEndLoc()), ")"); } +static void CheckC23ConstexprInitConversion(Sema &S, QualType FromType, + QualType ToType, Expr *Init) { + assert(S.getLangOpts().C23); + ImplicitConversionSequence ICS = S.TryImplicitConversion( + Init->IgnoreParenImpCasts(), ToType, /*SuppressUserConversions*/ false, + Sema::AllowedExplicit::None, + /*InOverloadResolution*/ false, + /*CStyle*/ false, + /*AllowObjCWritebackConversion=*/false); + + if (!ICS.isStandard()) + return; + + APValue Value; + QualType PreNarrowingType; + // Reuse C++ narrowing check. + switch (ICS.Standard.getNarrowingKind( + S.Context, Init, Value, PreNarrowingType, + /*IgnoreFloatToIntegralConversion*/ false)) { + // The value doesn't fit. + case NK_Constant_Narrowing: + S.Diag(Init->getBeginLoc(), diag::err_c23_constexpr_init_not_representable) + << Value.getAsString(S.Context, PreNarrowingType) << ToType; + return; + + // Conversion to a narrower type. + case NK_Type_Narrowing: + S.Diag(Init->getBeginLoc(), diag::err_c23_constexpr_init_type_mismatch) + << ToType << FromType; + return; + + // Since we only reuse narrowing check for C23 constexpr variables here, we're + // not really interested in these cases. + case NK_Dependent_Narrowing: + case NK_Variable_Narrowing: + case NK_Not_Narrowing: + return; + } + llvm_unreachable("unhandled case in switch"); +} + +static void CheckC23ConstexprInitStringLiteral(const StringLiteral *SE, + Sema &SemaRef, QualType &TT) { + assert(SemaRef.getLangOpts().C23); + // character that string literal contains fits into TT - target type. + const ArrayType *AT = SemaRef.Context.getAsArrayType(TT); + QualType CharType = AT->getElementType(); + uint32_t BitWidth = SemaRef.Context.getTypeSize(CharType); + bool isUnsigned = CharType->isUnsignedIntegerType(); + llvm::APSInt Value(BitWidth, isUnsigned); + for (unsigned I = 0, N = SE->getLength(); I != N; ++I) { + int64_t C = SE->getCodeUnitS(I, SemaRef.Context.getCharWidth()); + Value = C; + if (Value != C) { + SemaRef.Diag(SemaRef.getLocationOfStringLiteralByte(SE, I), + diag::err_c23_constexpr_init_not_representable) + << C << CharType; + return; + } + } + return; +} + //===----------------------------------------------------------------------===// // Initialization helper functions //===----------------------------------------------------------------------===// diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp index 7d38043890ca..a03f3eae5478 100644 --- a/clang/lib/Sema/SemaOverload.cpp +++ b/clang/lib/Sema/SemaOverload.cpp @@ -332,7 +332,8 @@ static const Expr *IgnoreNarrowingConversion(ASTContext &Ctx, NarrowingKind StandardConversionSequence::getNarrowingKind( ASTContext &Ctx, const Expr *Converted, APValue &ConstantValue, QualType &ConstantType, bool IgnoreFloatToIntegralConversion) const { - assert(Ctx.getLangOpts().CPlusPlus && "narrowing check outside C++"); + assert((Ctx.getLangOpts().CPlusPlus || Ctx.getLangOpts().C23) && + "narrowing check outside C++"); // C++11 [dcl.init.list]p7: // A narrowing conversion is an implicit conversion ... @@ -414,20 +415,41 @@ NarrowingKind StandardConversionSequence::getNarrowingKind( if (Initializer->isValueDependent()) return NK_Dependent_Narrowing; - if (Initializer->isCXX11ConstantExpr(Ctx, &ConstantValue)) { + Expr::EvalResult R; + if ((Ctx.getLangOpts().C23 && Initializer->EvaluateAsRValue(R, Ctx)) || + Initializer->isCXX11ConstantExpr(Ctx, &ConstantValue)) { // Constant! + if (Ctx.getLangOpts().C23) + ConstantValue = R.Val; assert(ConstantValue.isFloat()); llvm::APFloat FloatVal = ConstantValue.getFloat(); // Convert the source value into the target type. bool ignored; - llvm::APFloat::opStatus ConvertStatus = FloatVal.convert( - Ctx.getFloatTypeSemantics(ToType), - llvm::APFloat::rmNearestTiesToEven, &ignored); - // If there was no overflow, the source value is within the range of - // values that can be represented. - if (ConvertStatus & llvm::APFloat::opOverflow) { - ConstantType = Initializer->getType(); - return NK_Constant_Narrowing; + llvm::APFloat Converted = FloatVal; + llvm::APFloat::opStatus ConvertStatus = + Converted.convert(Ctx.getFloatTypeSemantics(ToType), + llvm::APFloat::rmNearestTiesToEven, &ignored); + Converted.convert(Ctx.getFloatTypeSemantics(FromType), + llvm::APFloat::rmNearestTiesToEven, &ignored); + if (Ctx.getLangOpts().C23) { + if (FloatVal.isNaN() && Converted.isNaN() && + !FloatVal.isSignaling() && !Converted.isSignaling()) { + // Quiet NaNs are considered the same value, regardless of + // payloads. + return NK_Not_Narrowing; + } + // For normal values, check exact equality. + if (!Converted.bitwiseIsEqual(FloatVal)) { + ConstantType = Initializer->getType(); + return NK_Constant_Narrowing; + } + } else { + // If there was no overflow, the source value is within the range of + // values that can be represented. + if (ConvertStatus & llvm::APFloat::opOverflow) { + ConstantType = Initializer->getType(); + return NK_Constant_Narrowing; + } } } else { return NK_Variable_Narrowing; @@ -494,7 +516,30 @@ NarrowingKind StandardConversionSequence::getNarrowingKind( } return NK_Not_Narrowing; } + case ICK_Complex_Real: + if (FromType->isComplexType() && !ToType->isComplexType()) + return NK_Type_Narrowing; + return NK_Not_Narrowing; + case ICK_Floating_Promotion: + if (Ctx.getLangOpts().C23) { + const Expr *Initializer = IgnoreNarrowingConversion(Ctx, Converted); + Expr::EvalResult R; + if (Initializer->EvaluateAsRValue(R, Ctx)) { + ConstantValue = R.Val; + assert(ConstantValue.isFloat()); + llvm::APFloat FloatVal = ConstantValue.getFloat(); + // C23 6.7.3p6 If the initializer has real type and a signaling NaN + // value, the unqualified versions of the type of the initializer and + // the corresponding real type of the object declared shall be + // compatible. + if (FloatVal.isNaN() && FloatVal.isSignaling()) { + ConstantType = Initializer->getType(); + return NK_Constant_Narrowing; + } + } + } + return NK_Not_Narrowing; default: // Other kinds of conversions are not narrowings. return NK_Not_Narrowing; diff --git a/clang/test/C/C2x/n3018.c b/clang/test/C/C2x/n3018.c new file mode 100644 index 000000000000..0d54d53b7499 --- /dev/null +++ b/clang/test/C/C2x/n3018.c @@ -0,0 +1,87 @@ +// RUN: %clang_cc1 -std=c23 -verify -triple x86_64 -pedantic -Wno-conversion -Wno-constant-conversion %s + +/* WG14 N3018: Full + * The constexpr specifier for object definitions + */ + +#define ULLONG_MAX (__LONG_LONG_MAX__*2ULL+1ULL) +#define UINT_MAX (__INT_MAX__ *2U +1U) + +void Example0() { + constexpr unsigned int minusOne = -1; + // expected-error@-1 {{constexpr initializer evaluates to -1 which is not exactly representable in type 'const unsigned int'}} + constexpr unsigned int uint_max = -1U; + constexpr double onethird = 1.0/3.0; + constexpr double onethirdtrunc = (double)(1.0/3.0); + + constexpr char string[] = { "\xFF", }; + constexpr unsigned char ucstring[] = { "\xFF", }; + // expected-error@-1 {{constexpr initializer evaluates to -1 which is not exactly representable in type 'const unsigned char'}} + constexpr char string1[] = { -1, 0, }; + constexpr unsigned char ucstring1[] = { -1, 0, }; + // expected-error@-1 {{constexpr initializer evaluates to -1 which is not exactly representable in type 'const unsigned char'}} + + // TODO: Make sure these work correctly once char8_t and _Decimal are supported + // constexpr char8_t u8string[] = { 255, 0, }; // ok + // constexpr char8_t u8string[] = { u8"\xFF", }; // ok + // constexpr _Decimal32 small = DEC64_TRUE_MIN * 0;// constraint violation +} + +void Example1() { + constexpr int K = 47; + enum { + A = K, + }; + constexpr int L = K; + static int b = K + 1; + int array[K]; + _Static_assert(K == 47); +} + +constexpr int K = 47; +static const int b = K + 1; + +void Example2() { + constexpr int A = 42LL; + constexpr signed short B = ULLONG_MAX; + // expected-error@-1 {{constexpr initializer evaluates to 18446744073709551615 which is not exactly representable in type 'const short'}} + constexpr float C = 47u; + + constexpr float D = 432000000; + constexpr float E = 1.0 / 3.0; + // expected-error@-1 {{constexpr initializer evaluates to 3.333333e-01 which is not exactly representable in type 'const float'}} + constexpr float F = 1.0f / 3.0f; +} + + +void Example3() { + constexpr static unsigned short array[] = { + 3000, + 300000, + // expected-error@-1 {{constexpr initializer evaluates to 300000 which is not exactly representable in type 'const unsigned short'}} + -1 + // expected-error@-1 {{constexpr initializer evaluates to -1 which is not exactly representable in type 'const unsigned short'}} + }; + + constexpr static unsigned short array1[] = { + 3000, + 3000, + -1 + // expected-error@-1 {{constexpr initializer evaluates to -1 which is not exactly representable in type 'const unsigned short'}} + }; + + struct S { + int x, y; + }; + constexpr struct S s = { + .x = __INT_MAX__, + .y = UINT_MAX, + // expected-error@-1 {{constexpr initializer evaluates to 4294967295 which is not exactly representable in type 'int'}} + }; +} + +void Example4() { + struct s { void *p; }; + constexpr struct s A = { nullptr }; + constexpr struct s B = A; +} diff --git a/clang/test/Parser/c23-constexpr.c b/clang/test/Parser/c23-constexpr.c new file mode 100644 index 000000000000..156128fa0745 --- /dev/null +++ b/clang/test/Parser/c23-constexpr.c @@ -0,0 +1,15 @@ +// RUN: %clang_cc1 -fsyntax-only -verify=c23 -std=c23 %s -Wpre-c2x-compat +// RUN: %clang_cc1 -fsyntax-only -verify=c17 -std=c17 %s + +constexpr int a = 0; // c17-error {{unknown type name 'constexpr'}} \ + c23-warning {{'constexpr' is incompatible with C standards before C23}} + +void func(int array[constexpr]); // c23-error {{expected expression}} \ + // c17-error {{use of undeclared}} + +_Atomic constexpr int b = 0; // c23-error {{constexpr variable cannot have type 'const _Atomic(int)'}} \ + // c23-warning {{'constexpr' is incompatible with C standards before C23}} \ + // c17-error {{unknown type name 'constexpr'}} + +int static constexpr c = 1; // c17-error {{expected ';' after top level declarator}} \ + // c23-warning {{'constexpr' is incompatible with C standards before C23}} diff --git a/clang/test/Sema/constexpr.c b/clang/test/Sema/constexpr.c new file mode 100644 index 000000000000..8286cd2107d2 --- /dev/null +++ b/clang/test/Sema/constexpr.c @@ -0,0 +1,359 @@ +// RUN: %clang_cc1 -std=c23 -verify -triple x86_64 -pedantic -Wno-conversion -Wno-constant-conversion -Wno-div-by-zero %s + +// Check that constexpr only applies to variables. +constexpr void f0() {} // expected-error {{'constexpr' can only be used in variable declarations}} +constexpr const int f1() { return 0; } // expected-error {{'constexpr' can only be used in variable declarations}} + +constexpr struct S1 { int f; }; //expected-error {{struct cannot be marked constexpr}} +constexpr struct S2 ; // expected-error {{struct cannot be marked constexpr}} +constexpr union U1; // expected-error {{union cannot be marked constexpr}} +constexpr union U2 {int a; float b;}; // expected-error {{union cannot be marked constexpr}} +constexpr enum E1 {A = 1, B = 2} ; // expected-error {{enum cannot be marked constexpr}} +struct S3 { + static constexpr int f = 0; // expected-error {{type name does not allow storage class}} + // expected-error@-1 {{type name does not allow constexpr}} + // expected-error@-2 {{expected ';' at end}} + constexpr int f1 = 0; + // expected-error@-1 {{type name does not allow constexpr}} + // expected-error@-2 {{expected ';' at end}} +}; + +constexpr; // expected-error {{'constexpr' can only be used in variable declarations}} +constexpr int V1 = 3; +constexpr float V2 = 7.0; +int V3 = (constexpr)3; // expected-error {{expected expression}} + +void f2() { + constexpr int a = 0; + constexpr float b = 1.7f; +} + +// Check how constexpr works with other storage-class specifiers. +constexpr auto V4 = 1; +constexpr static auto V5 = 1; +constexpr static const auto V6 = 1; +constexpr static const int V7 = 1; +constexpr static int V8 = 1; +constexpr auto Ulong = 1L; +constexpr auto CompoundLiteral = (int){13}; +constexpr auto DoubleCast = (double)(1 / 3); +constexpr auto String = "this is a string"; // expected-error {{constexpr pointer initializer is not null}} +constexpr signed auto Long = 1L; // expected-error {{'auto' cannot be signed or unsigned}} +_Static_assert(_Generic(Ulong, long : 1)); +_Static_assert(_Generic(CompoundLiteral, int : 1)); +_Static_assert(_Generic(DoubleCast, double : 1)); +_Static_assert(_Generic(String, char* : 1)); + +typedef constexpr int Foo; // expected-error {{typedef cannot be constexpr}} +constexpr typedef int Bar; // expected-error {{typedef cannot be constexpr}} + +void f3(constexpr register int P1) { // expected-error {{function parameter cannot be constexpr}} + constexpr register int V9 = 0; + constexpr register auto V10 = 0.0; +} + +constexpr thread_local int V11 = 38; // expected-error {{cannot combine with previous '_Thread_local' declaration specifier}} +constexpr static thread_local double V12 = 38; // expected-error {{cannot combine with previous '_Thread_local' declaration specifier}} +constexpr extern thread_local char V13; // expected-error {{cannot combine with previous '_Thread_local' declaration specifier}} +// expected-error@-1 {{cannot combine with previous 'extern' declaration specifier}} +// expected-error@-2 {{constexpr variable declaration must be a definition}} +constexpr thread_local short V14 = 38; // expected-error {{cannot combine with previous '_Thread_local' declaration specifier}} + +// Check how constexpr works with qualifiers. +constexpr _Atomic int V15 = 0; // expected-error {{constexpr variable cannot have type 'const _Atomic(int)'}} +constexpr _Atomic(int) V16 = 0; // expected-error {{constexpr variable cannot have type 'const _Atomic(int)'}} + +constexpr volatile int V17 = 0; // expected-error {{constexpr variable cannot have type 'const volatile int'}} + +constexpr int * restrict V18 = 0; // expected-error {{constexpr variable cannot have type 'int *const restrict'}} + +constexpr extern char Oops = 1; // expected-error {{cannot combine with previous 'extern' declaration specifier}} \ + // expected-warning {{'extern' variable has an initializer}} + +constexpr int * restrict * Oops1 = 0; + +typedef _Atomic(int) TheA; +typedef volatile short TheV; +typedef float * restrict TheR; + +constexpr TheA V19[3] = {}; +// expected-error@-1 {{constexpr variable cannot have type 'const TheA[3]' (aka 'const _Atomic(int)[3]')}} +constexpr TheV V20[3] = {}; +// expected-error@-1 {{constexpr variable cannot have type 'const TheV[3]' (aka 'const volatile short[3]')}} +constexpr TheR V21[3] = {}; +// expected-error@-1 {{constexpr variable cannot have type 'const TheR[3]' (aka 'float *restrict const[3]')}} + +struct HasA { + TheA f; + int b; +}; + +struct HasV { + float b; + TheV f; +}; + +struct HasR { + short b; + int a; + TheR f; +}; + +constexpr struct HasA V22[2] = {}; +// expected-error@-1 {{constexpr variable cannot have type 'TheA' (aka '_Atomic(int)')}} +constexpr struct HasV V23[2] = {}; +// expected-error@-1 {{constexpr variable cannot have type 'TheV' (aka 'volatile short')}} +constexpr struct HasR V24[2] = {}; +// expected-error@-1 {{constexpr variable cannot have type 'TheR' (aka 'float *restrict')}} + +union U3 { + float a; + union { + struct HasA f; + struct HasR f1; + }; +}; + +constexpr union U3 V25 = {}; +// expected-error@-1 {{constexpr variable cannot have type 'TheA' (aka '_Atomic(int)')}} +constexpr union U3 V26[8] = {}; +// expected-error@-1 {{constexpr variable cannot have type 'TheA' (aka '_Atomic(int)')}} + +struct S4 { + union U3 f[3]; +}; + +constexpr struct S4 V27 = {}; +// expected-error@-1 {{constexpr variable cannot have type 'TheA' (aka '_Atomic(int)')}} +constexpr const int V28 = 28; + +struct S { + union { + volatile int i; + }; + int j; +}; + +constexpr struct S s = {}; // expected-error {{constexpr variable cannot have type 'volatile int'}} + +// Check that constexpr variable must have a valid initializer which is a +// constant expression. +constexpr int V29; +// expected-error@-1 {{constexpr variable 'V29' must be initialized by a constant expression}} + +struct S5 { + int f; +}; + +constexpr struct S5 V30; +// expected-error@-1 {{constexpr variable 'V30' must be initialized by a constant expression}} +constexpr struct S5 V31 = {}; + +int randomFoo() { return 7; } + +constexpr float V32 = randomFoo(); +// expected-error@-1 {{constexpr variable 'V32' must be initialized by a constant expression}} + +const int V33 = 4; +const int V34 = 0; +const int V35 = 2; + +constexpr int V36 = V33 / V34; +// expected-error@-1 {{constexpr variable 'V36' must be initialized by a constant expression}} +constexpr int V37 = V33 / V35; +// expected-error@-1 {{constexpr variable 'V37' must be initialized by a constant expression}} +constexpr int V38 = 3; +constexpr int V39 = V38 / V38; +constexpr int V40 = V38 / 2; +constexpr int V41 = V38 / 0; +// expected-error@-1 {{constexpr variable 'V41' must be initialized by a constant expression}} +// expected-note@-2 {{division by zero}} +constexpr int V42 = V38 & 0; + +constexpr struct S5 V43 = { randomFoo() }; +// expected-error@-1 {{constexpr variable 'V43' must be initialized by a constant expression}} +constexpr struct S5 V44 = { 0 }; +constexpr struct S5 V45 = { V38 / 0 }; +// expected-error@-1 {{constexpr variable 'V45' must be initialized by a constant expression}} +// expected-note@-2 {{division by zero}} + +constexpr float V46[3] = {randomFoo() }; +// expected-error@-1 {{constexpr variable 'V46' must be initialized by a constant expression}} +constexpr struct S5 V47[3] = {randomFoo() }; +// expected-error@-1 {{constexpr variable 'V47' must be initialized by a constant expression}} + +const static int V48 = V38; +constexpr static int V49 = V48; +// expected-error@-1 {{constexpr variable 'V49' must be initialized by a constant expression}} + +void f4(const int P1) { + constexpr int V = P1; +// expected-error@-1 {{constexpr variable 'V' must be initialized by a constant expression}} + + constexpr int V1 = 12; + constexpr const int *V2 = &V1; +// expected-error@-1 {{constexpr variable 'V2' must be initialized by a constant expression}} +} + +// Check that initializer for constexpr variable should match the type of the +// variable and is exactly representable int the variable's type. + +struct S6 { + unsigned char a; +}; + +struct S7 { + union { + float a; + }; + unsigned int b; +}; + +struct S8 { + unsigned char a[3]; + unsigned int b[3]; +}; + +constexpr struct S8 DesigInit = {.b = {299, 7, 8}, .a = {-1, 7, 8}}; +// expected-error@-1 {{constexpr initializer evaluates to -1 which is not exactly representable in type 'unsigned char'}} + +void f5() { + constexpr char V50 = 300; + // expected-error@-1 {{constexpr initializer evaluates to 300 which is not exactly representable in type 'const char'}} + constexpr float V51 = 1.0 / 3.0; + // expected-error@-1 {{constexpr initializer evaluates to 3.333333e-01 which is not exactly representable in type 'const float'}} + constexpr float V52 = 0.7; + // expected-error@-1 {{constexpr initializer evaluates to 7.000000e-01 which is not exactly representable in type 'const float'}} + constexpr float V53 = 1.0f / 3.0f; + constexpr float V54 = 432000000000; + // expected-error@-1 {{constexpr initializer evaluates to 432000000000 which is not exactly representable in type 'const float'}} + constexpr unsigned char V55[] = { + "\xAF", + // expected-error@-1 {{constexpr initializer evaluates to -81 which is not exactly representable in type 'const unsigned char'}} + }; + + constexpr unsigned char V56[] = { + u8"\xAF", + }; + constexpr struct S6 V57 = {299}; + // expected-error@-1 {{constexpr initializer evaluates to 299 which is not exactly representable in type 'unsigned char'}} + constexpr struct S6 V58 = {-299}; + // expected-error@-1 {{constexpr initializer evaluates to -299 which is not exactly representable in type 'unsigned char'}} + constexpr double V59 = 0.5; + constexpr double V60 = 1.0; + constexpr float V61 = V59 / V60; + constexpr double V62 = 1.7; + constexpr float V63 = V59 / V62; + // expected-error@-1 {{constexpr initializer evaluates to 2.941176e-01 which is not exactly representable in type 'const float'}} + + constexpr unsigned char V64 = '\xAF'; + // expected-error@-1 {{constexpr initializer evaluates to -81 which is not exactly representable in type 'const unsigned char'}} + constexpr unsigned char V65 = u8'\xAF'; + + constexpr char V66[3] = {300}; + // expected-error@-1 {{constexpr initializer evaluates to 300 which is not exactly representable in type 'const char'}} + constexpr struct S6 V67[3] = {300}; + // expected-error@-1 {{constexpr initializer evaluates to 300 which is not exactly representable in type 'unsigned char'}} + + constexpr struct S7 V68 = {0.3, -1 }; + // expected-error@-1 {{constexpr initializer evaluates to 3.000000e-01 which is not exactly representable in type 'float'}} + // expected-error@-2 {{constexpr initializer evaluates to -1 which is not exactly representable in type 'unsigned int'}} + constexpr struct S7 V69 = {0.5, -1 }; + // expected-error@-1 {{constexpr initializer evaluates to -1 which is not exactly representable in type 'unsigned int'}} + constexpr struct S7 V70[3] = {{123456789}}; + // expected-error@-1 {{constexpr initializer evaluates to 123456789 which is not exactly representable in type 'float'}} + + constexpr int V71 = 0.3; + // expected-error@-1 {{constexpr initializer for type 'const int' is of type 'double'}} + constexpr int V72 = V59; + // expected-error@-1 {{constexpr initializer for type 'const int' is of type 'const double'}} + constexpr struct S6 V73 = {V59}; + // expected-error@-1 {{constexpr initializer for type 'unsigned char' is of type 'const double'}} + + constexpr float V74 = 1; + constexpr float V75 = V59; + constexpr unsigned int V76[3] = {0.5}; + // expected-error@-1 {{constexpr initializer for type 'const unsigned int' is of type 'double'}} + + constexpr _Complex float V77 = 0; + constexpr float V78 = V77; + // expected-error@-1 {{constexpr initializer for type 'const float' is of type 'const _Complex float'}} + constexpr int V79 = V77; + // expected-error@-1 {{constexpr initializer for type 'const int' is of type 'const _Complex float'}} + +} + +constexpr char string[] = "test""ing this out\xFF"; +constexpr unsigned char ustring[] = "test""ing this out\xFF"; +// expected-error@-1 {{constexpr initializer evaluates to -1 which is not exactly representable in type 'const unsigned char'}} +constexpr char u8string[] = u8"test"u8"ing this out\xFF"; +// expected-error@-1 {{constexpr initializer evaluates to 255 which is not exactly representable in type 'const char'}} +constexpr unsigned char u8ustring[] = u8"test"u8"ing this out\xFF"; +constexpr unsigned short uustring[] = u"test"u"ing this out\xFF"; +constexpr unsigned int Ustring[] = U"test"U"ing this out\xFF"; +constexpr unsigned char Arr2[6][6] = { + {"ek\xFF"}, {"ek\xFF"} +// expected-error@-1 2{{constexpr initializer evaluates to -1 which is not exactly representable in type 'const unsigned char'}} +}; + +constexpr int i = (12); +constexpr int j = (i); +constexpr unsigned jneg = (-i); +// expected-error@-1 {{constexpr initializer evaluates to -12 which is not exactly representable in type 'const unsigned int'}} + +// Check that initializer for pointer constexpr variable should be null. +constexpr int V80 = 3; +constexpr const int *V81 = &V80; +// expected-error@-1 {{constexpr pointer initializer is not null}} +constexpr int *V82 = 0; +constexpr int *V83 = V82; +constexpr int *V84 = 42; +// expected-error@-1 {{constexpr variable 'V84' must be initialized by a constant expression}} +// expected-note@-2 {{this conversion is not allowed in a constant expression}} +// expected-error@-3 {{constexpr pointer initializer is not null}} +constexpr int *V85 = nullptr; + +// Check that constexpr variables should not be VLAs. +void f6(const int P1) { + constexpr int V86[P1] = {}; +// expected-error@-1 {{constexpr variable cannot have type 'const int[P1]'}} + const int V87 = 3; + constexpr int V88[V87] = {}; +// expected-warning@-1 {{variable length array folded to constant array as an extension}} + int V89 = 7; + constexpr int V90[V89] = {}; +// expected-error@-1 {{constexpr variable cannot have type 'const int[V89]'}} +} + +void f7(int n, int array[n]) { + constexpr typeof(array) foo = 0; // Accepted because array is a pointer type, not a VLA type + int (*(*fp)(int n))[n]; + constexpr typeof(fp) bar = 0; // expected-error {{constexpr variable cannot have type 'const typeof (fp)' (aka 'int (*(*const)(int))[n]')}} +} + +// Check how constexpr works with NaNs and infinities. +#define FLT_NAN __builtin_nanf("1") +#define DBL_NAN __builtin_nan("1") +#define LD_NAN __builtin_nanf("1") +#define FLT_SNAN __builtin_nansf("1") +#define DBL_SNAN __builtin_nans("1") +#define LD_SNAN __builtin_nansl("1") +#define INF __builtin_inf() +void infsNaNs() { + // Inf and quiet NaN is always fine, signaling NaN must have the same type. + constexpr float fl0 = INF; + constexpr float fl1 = (long double)INF; + constexpr float fl2 = (long double)FLT_NAN; + constexpr float fl3 = FLT_NAN; + constexpr float fl5 = DBL_NAN; + constexpr float fl6 = LD_NAN; + constexpr float fl7 = DBL_SNAN; // expected-error {{constexpr initializer evaluates to nan which is not exactly representable in type 'const float'}} + constexpr float fl8 = LD_SNAN; // expected-error {{constexpr initializer evaluates to nan which is not exactly representable in type 'const float'}} + + constexpr double db0 = FLT_NAN; + constexpr double db2 = DBL_NAN; + constexpr double db3 = DBL_SNAN; + constexpr double db4 = FLT_SNAN; // expected-error {{constexpr initializer evaluates to nan which is not exactly representable in type 'const double'}} + constexpr double db5 = LD_SNAN; // expected-error {{constexpr initializer evaluates to nan which is not exactly representable in type 'const double'}} + constexpr double db6 = INF; +} diff --git a/clang/www/c_status.html b/clang/www/c_status.html index 3955a1d79639..7bf5e29f0639 100644 --- a/clang/www/c_status.html +++ b/clang/www/c_status.html @@ -1201,7 +1201,7 @@ conformance.

constexpr for object definitions N3018 - No + Clang 19 Introduce storage class specifiers for compound literals -- GitLab From 6e27dd47e1fc17cf661b568901fe7392ed1b8f2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingo=20M=C3=BCller?= Date: Wed, 6 Mar 2024 09:53:30 +0100 Subject: [PATCH 264/929] [mlir][gpu] Replace MLIR_GPU_TO_HSACO_PASS_ENABLE by more generic one. (#84001) This is another follow-up of #83004. The PR replaces the macro `MLIR_GPU_TO_HSACO_PASS_ENABLE` with the more generic macro `MLIR_ENABLE_ROCM_CONVERSIONS`. Until now, the former has been defined if and only if the latter evaluated to true in CMake. However, the former was not defined when the latter evaluated to false, in which case a warning was raised if compiled with `-Wundef`. Using a single macro relies on the `#cmakedefine01` mechanism that ensures the macro is always set to either 0 or 1. --- mlir/lib/Dialect/GPU/CMakeLists.txt | 1 - mlir/lib/Dialect/GPU/Transforms/SerializeToHsaco.cpp | 7 ++++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mlir/lib/Dialect/GPU/CMakeLists.txt b/mlir/lib/Dialect/GPU/CMakeLists.txt index 51cfa2216e0c..61ab298ebfb9 100644 --- a/mlir/lib/Dialect/GPU/CMakeLists.txt +++ b/mlir/lib/Dialect/GPU/CMakeLists.txt @@ -110,7 +110,6 @@ if(MLIR_ENABLE_ROCM_CONVERSIONS) target_compile_definitions(obj.MLIRGPUTransforms PRIVATE __DEFAULT_ROCM_PATH__="${DEFAULT_ROCM_PATH}" - MLIR_GPU_TO_HSACO_PASS_ENABLE=1 ) target_link_libraries(MLIRGPUTransforms diff --git a/mlir/lib/Dialect/GPU/Transforms/SerializeToHsaco.cpp b/mlir/lib/Dialect/GPU/Transforms/SerializeToHsaco.cpp index eee7a680f5b3..2af898b56774 100644 --- a/mlir/lib/Dialect/GPU/Transforms/SerializeToHsaco.cpp +++ b/mlir/lib/Dialect/GPU/Transforms/SerializeToHsaco.cpp @@ -11,11 +11,12 @@ // //===----------------------------------------------------------------------===// +#include "mlir/Config/mlir-config.h" #include "mlir/Dialect/GPU/Transforms/Passes.h" #include "mlir/IR/Location.h" #include "mlir/IR/MLIRContext.h" -#if MLIR_GPU_TO_HSACO_PASS_ENABLE +#if MLIR_ENABLE_ROCM_CONVERSIONS #include "mlir/ExecutionEngine/OptUtils.h" #include "mlir/Pass/Pass.h" #include "mlir/Support/FileUtilities.h" @@ -454,6 +455,6 @@ std::unique_ptr mlir::createGpuSerializeToHsacoPass(StringRef triple, optLevel); } -#else // MLIR_GPU_TO_HSACO_PASS_ENABLE +#else // MLIR_ENABLE_ROCM_CONVERSIONS void mlir::registerGpuSerializeToHsacoPass() {} -#endif // MLIR_GPU_TO_HSACO_PASS_ENABLE +#endif // MLIR_ENABLE_ROCM_CONVERSIONS -- GitLab From d70254a623d7c23c493cc793756a5a9d758b0a14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingo=20M=C3=BCller?= Date: Wed, 6 Mar 2024 10:13:12 +0100 Subject: [PATCH 265/929] [mlir][nvvm] Add missing include to llvm-config.h. (#83998) This is another follow-up of #83004. `NVVM/Target.cpp` uses the macro `MLIR_NVPTXCOMPILER_ENABLED`, which is defined in `llvm-config.h` but did not include that file, yielding a warning when compiled with `-Wundef`. This PR adds the include. ~~This is another follow-up of #83004, which made the same change for `MLIR_CUDA_CONVERSIONS_ENABLED`. As the previous PR, this PR commit exposes mentioned CMake variable through `mlir-config.h` and uses the macro that is introduced with the same name. This replaces the macro `MLIR_NVPTXCOMPILER_ENABLED`, which the CMake files previously defined manually.~~ --- mlir/lib/Target/LLVM/NVVM/Target.cpp | 1 + utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 1 + 2 files changed, 2 insertions(+) diff --git a/mlir/lib/Target/LLVM/NVVM/Target.cpp b/mlir/lib/Target/LLVM/NVVM/Target.cpp index d5b6645631ed..e31df99ea20d 100644 --- a/mlir/lib/Target/LLVM/NVVM/Target.cpp +++ b/mlir/lib/Target/LLVM/NVVM/Target.cpp @@ -22,6 +22,7 @@ #include "mlir/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.h" #include "mlir/Target/LLVMIR/Export.h" +#include "llvm/Config/llvm-config.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/FileUtilities.h" #include "llvm/Support/FormatVariadic.h" diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 7f33f1659922..b6d4927388b8 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -6183,6 +6183,7 @@ cc_library( ":config", "//llvm:NVPTXCodeGen", "//llvm:Support", + "//llvm:config", ], ) -- GitLab From 0e5bde021252f68b378b31fd64a885a8b28aad65 Mon Sep 17 00:00:00 2001 From: Adrian Kuegel Date: Wed, 6 Mar 2024 09:31:44 +0000 Subject: [PATCH 266/929] [mlir] Remove noexcept again from move constructors. LLVM does not have the corresponding ClangTidy check enabled, so we should not be fixing such findings. --- mlir/include/mlir/IR/BuiltinAttributeInterfaces.h | 2 +- mlir/lib/Bindings/Python/TransformInterpreter.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mlir/include/mlir/IR/BuiltinAttributeInterfaces.h b/mlir/include/mlir/IR/BuiltinAttributeInterfaces.h index 1507c62e3fe0..b0b9493c1519 100644 --- a/mlir/include/mlir/IR/BuiltinAttributeInterfaces.h +++ b/mlir/include/mlir/IR/BuiltinAttributeInterfaces.h @@ -35,7 +35,7 @@ struct ElementsAttrIndexer { public: ElementsAttrIndexer() : ElementsAttrIndexer(/*isContiguous=*/true, /*isSplat=*/true) {} - ElementsAttrIndexer(ElementsAttrIndexer &&rhs) noexcept + ElementsAttrIndexer(ElementsAttrIndexer &&rhs) : isContiguous(rhs.isContiguous), isSplat(rhs.isSplat) { if (isContiguous) conState = rhs.conState; diff --git a/mlir/lib/Bindings/Python/TransformInterpreter.cpp b/mlir/lib/Bindings/Python/TransformInterpreter.cpp index 3530f295ed82..6517f8c39dfa 100644 --- a/mlir/lib/Bindings/Python/TransformInterpreter.cpp +++ b/mlir/lib/Bindings/Python/TransformInterpreter.cpp @@ -23,7 +23,7 @@ namespace py = pybind11; namespace { struct PyMlirTransformOptions { PyMlirTransformOptions() { options = mlirTransformOptionsCreate(); }; - PyMlirTransformOptions(PyMlirTransformOptions &&other) noexcept { + PyMlirTransformOptions(PyMlirTransformOptions &&other) { options = other.options; other.options.ptr = nullptr; } -- GitLab From 52d5b8e02d258e4c32864eb1bb943cfae119b1d6 Mon Sep 17 00:00:00 2001 From: Pierre van Houtryve Date: Wed, 6 Mar 2024 10:38:20 +0100 Subject: [PATCH 267/929] [AMDGPU] Don't form sext/abs/neg fp8 cvt (#83843) gfx940 does not allow abs/sext/neg on v_cvt_fp8/bf8 & pk variants. Fixes SWDEV-447468 --- llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 14 +++ llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp | 9 ++ .../CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll | 96 +++++++++++++++++++ .../test/CodeGen/AMDGPU/verifier-sdwa-cvt.mir | 38 ++++++++ 4 files changed, 157 insertions(+) create mode 100644 llvm/test/CodeGen/AMDGPU/verifier-sdwa-cvt.mir diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index edd87e340d10..cb657c5e04e4 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -4726,6 +4726,20 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI, } } + if (Opcode == AMDGPU::V_CVT_F32_FP8_sdwa || + Opcode == AMDGPU::V_CVT_F32_BF8_sdwa || + Opcode == AMDGPU::V_CVT_PK_F32_FP8_sdwa || + Opcode == AMDGPU::V_CVT_PK_F32_BF8_sdwa) { + const MachineOperand *Src0ModsMO = + getNamedOperand(MI, AMDGPU::OpName::src0_modifiers); + unsigned Mods = Src0ModsMO->getImm(); + if (Mods & SISrcMods::ABS || Mods & SISrcMods::NEG || + Mods & SISrcMods::SEXT) { + ErrInfo = "sext, abs and neg are not allowed on this instruction"; + return false; + } + } + uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode); if (isVOPC(BasicOpcode)) { if (!ST.hasSDWASdst() && DstIdx != -1) { diff --git a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp index afc380b42034..1fadd8ce45b1 100644 --- a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp +++ b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp @@ -338,6 +338,15 @@ MachineInstr *SDWASrcOperand::potentialToConvert(const SIInstrInfo *TII) { } bool SDWASrcOperand::convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) { + switch (MI.getOpcode()) { + case AMDGPU::V_CVT_F32_FP8_sdwa: + case AMDGPU::V_CVT_F32_BF8_sdwa: + case AMDGPU::V_CVT_PK_F32_FP8_sdwa: + case AMDGPU::V_CVT_PK_F32_BF8_sdwa: + // Does not support input modifiers: noabs, noneg, nosext. + return false; + } + // Find operand in instruction that matches source operand and replace it with // target operand. Set corresponding src_sel bool IsPreserveSrc = false; diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll index fc4b663b85a6..9b8fdf901704 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll @@ -534,3 +534,99 @@ define i32 @test_cvt_sr_fp8_f32_byte3(float %x, i32 %r, i32 %old) { %ret = tail call i32 @llvm.amdgcn.cvt.sr.fp8.f32(float %x, i32 %r, i32 %old, i32 3) ret i32 %ret } + +define float @test_sext_cvt_f32_fp8(i16 %a) { +; GFX940-LABEL: test_sext_cvt_f32_fp8: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: v_bfe_i32 v0, v0, 0, 16 +; GFX940-NEXT: v_cvt_f32_fp8_sdwa v0, v0 src0_sel:BYTE_1 +; GFX940-NEXT: s_setpc_b64 s[30:31] +; +; GFX12-LABEL: test_sext_cvt_f32_fp8: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_bfe_i32 v0, v0, 0, 16 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_cvt_f32_fp8_e64 v0, v0 op_sel:[0,1] +; GFX12-NEXT: s_setpc_b64 s[30:31] + %a.sext = sext i16 %a to i32 + %ret = tail call float @llvm.amdgcn.cvt.f32.fp8(i32 %a.sext, i32 1) + ret float %ret +} + +define float @test_sext_cvt_f32_bf8(i16 %a) { +; GFX940-LABEL: test_sext_cvt_f32_bf8: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: v_bfe_i32 v0, v0, 0, 16 +; GFX940-NEXT: v_cvt_f32_bf8_sdwa v0, v0 src0_sel:BYTE_1 +; GFX940-NEXT: s_setpc_b64 s[30:31] +; +; GFX12-LABEL: test_sext_cvt_f32_bf8: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_bfe_i32 v0, v0, 0, 16 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_cvt_f32_bf8_e64 v0, v0 op_sel:[0,1] +; GFX12-NEXT: s_setpc_b64 s[30:31] + %a.sext = sext i16 %a to i32 + %ret = tail call float @llvm.amdgcn.cvt.f32.bf8(i32 %a.sext, i32 1) + ret float %ret +} + +define <2 x float> @test_sext_cvt_pk_f32_bf8_word1(i16 %a) { +; GFX940-LABEL: test_sext_cvt_pk_f32_bf8_word1: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: v_bfe_i32 v0, v0, 0, 16 +; GFX940-NEXT: v_cvt_pk_f32_bf8_sdwa v[0:1], v0 src0_sel:WORD_1 +; GFX940-NEXT: s_setpc_b64 s[30:31] +; +; GFX12-LABEL: test_sext_cvt_pk_f32_bf8_word1: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_bfe_i32 v0, v0, 0, 16 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_cvt_pk_f32_bf8_e64 v[0:1], v0 op_sel:[1,0] +; GFX12-NEXT: s_setpc_b64 s[30:31] + %a.sext = sext i16 %a to i32 + %ret = tail call <2 x float> @llvm.amdgcn.cvt.pk.f32.bf8(i32 %a.sext, i1 true) + ret <2 x float> %ret +} + +define <2 x float> @test_sext_cvt_pk_f32_fp8_word0(i16 %a) { +; GFX940-LABEL: test_sext_cvt_pk_f32_fp8_word0: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: v_bfe_i32 v0, v0, 0, 16 +; GFX940-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0 +; GFX940-NEXT: s_setpc_b64 s[30:31] +; +; GFX12-LABEL: test_sext_cvt_pk_f32_fp8_word0: +; GFX12: ; %bb.0: +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: s_wait_expcnt 0x0 +; GFX12-NEXT: s_wait_samplecnt 0x0 +; GFX12-NEXT: s_wait_bvhcnt 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_bfe_i32 v0, v0, 0, 16 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], v0 +; GFX12-NEXT: s_setpc_b64 s[30:31] + %a.sext = sext i16 %a to i32 + %ret = tail call <2 x float> @llvm.amdgcn.cvt.pk.f32.fp8(i32 %a.sext, i1 false) + ret <2 x float> %ret +} diff --git a/llvm/test/CodeGen/AMDGPU/verifier-sdwa-cvt.mir b/llvm/test/CodeGen/AMDGPU/verifier-sdwa-cvt.mir new file mode 100644 index 000000000000..2066637a34af --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/verifier-sdwa-cvt.mir @@ -0,0 +1,38 @@ +# RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx940 -run-pass machineverifier -o /dev/null %s 2>&1 | FileCheck -implicit-check-not="Bad machine code" %s + +# CHECK: *** Bad machine code: sext, abs and neg are not allowed on this instruction *** +# CHECK: $vgpr0 = V_CVT_F32_FP8_sdwa 1, $vgpr0, 0, 0, 4, implicit $mode, implicit $exec +# CHECK: *** Bad machine code: sext, abs and neg are not allowed on this instruction *** +# CHECK: $vgpr0 = V_CVT_F32_BF8_sdwa 1, $vgpr0, 0, 0, 4, implicit $mode, implicit $exec +# CHECK: *** Bad machine code: sext, abs and neg are not allowed on this instruction *** +# CHECK: $vgpr0_vgpr1 = V_CVT_PK_F32_FP8_sdwa 1, $vgpr0, 0, 0, 4, implicit $mode, implicit $exec +# CHECK: *** Bad machine code: sext, abs and neg are not allowed on this instruction *** +# CHECK: $vgpr0_vgpr1 = V_CVT_PK_F32_BF8_sdwa 1, $vgpr0, 0, 0, 4, implicit $mode, implicit $exec +# CHECK: *** Bad machine code: sext, abs and neg are not allowed on this instruction *** +# CHECK: $vgpr0 = V_CVT_F32_FP8_sdwa 2, $vgpr0, 0, 0, 4, implicit $mode, implicit $exec +# CHECK: *** Bad machine code: sext, abs and neg are not allowed on this instruction *** +# CHECK: $vgpr0 = V_CVT_F32_BF8_sdwa 2, $vgpr0, 0, 0, 4, implicit $mode, implicit $exec +# CHECK: *** Bad machine code: sext, abs and neg are not allowed on this instruction *** +# CHECK: $vgpr0_vgpr1 = V_CVT_PK_F32_FP8_sdwa 2, $vgpr0, 0, 0, 4, implicit $mode, implicit $exec +# CHECK: *** Bad machine code: sext, abs and neg are not allowed on this instruction *** +# CHECK: $vgpr0_vgpr1 = V_CVT_PK_F32_BF8_sdwa 2, $vgpr0, 0, 0, 4, implicit $mode, implicit $exec + +--- +name: test +liveins: +body: | + bb.0: + liveins: $vgpr0, $vgpr0_vgpr1 + + ; sext/neg + $vgpr0 = V_CVT_F32_FP8_sdwa 1, $vgpr0, 0, 0, 4, implicit $mode, implicit $exec + $vgpr0 = V_CVT_F32_BF8_sdwa 1, $vgpr0, 0, 0, 4, implicit $mode, implicit $exec + $vgpr0_vgpr1 = V_CVT_PK_F32_FP8_sdwa 1, $vgpr0, 0, 0, 4, implicit $mode, implicit $exec + $vgpr0_vgpr1 = V_CVT_PK_F32_BF8_sdwa 1, $vgpr0, 0, 0, 4, implicit $mode, implicit $exec + + ; abs + $vgpr0 = V_CVT_F32_FP8_sdwa 2, $vgpr0, 0, 0, 4, implicit $mode, implicit $exec + $vgpr0 = V_CVT_F32_BF8_sdwa 2, $vgpr0, 0, 0, 4, implicit $mode, implicit $exec + $vgpr0_vgpr1 = V_CVT_PK_F32_FP8_sdwa 2, $vgpr0, 0, 0, 4, implicit $mode, implicit $exec + $vgpr0_vgpr1 = V_CVT_PK_F32_BF8_sdwa 2, $vgpr0, 0, 0, 4, implicit $mode, implicit $exec +... -- GitLab From 471a6128901747130ae11ce87b6e0095f862ed9a Mon Sep 17 00:00:00 2001 From: Jacques Pienaar Date: Wed, 6 Mar 2024 10:29:57 +0000 Subject: [PATCH 268/929] [mlir][drr] Add warning for simple case of mismatched variadic. (#84040) When a variadic argument is expected but not provided the compilation fails later with a difficult to follow compilation error. Add a simple check to catch one such case. This is not yet general as it doesn't yet check leaf nodes. --- mlir/test/mlir-tblgen/rewriter-errors.td | 15 +++++++++++++ mlir/tools/mlir-tblgen/RewriterGen.cpp | 28 ++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/mlir/test/mlir-tblgen/rewriter-errors.td b/mlir/test/mlir-tblgen/rewriter-errors.td index f74fb7a69810..0b6d8c3fec61 100644 --- a/mlir/test/mlir-tblgen/rewriter-errors.td +++ b/mlir/test/mlir-tblgen/rewriter-errors.td @@ -5,6 +5,7 @@ // RUN: not mlir-tblgen -gen-rewriters -I %S/../../include -DERROR5 %s 2>&1 | FileCheck --check-prefix=ERROR5 %s // RUN: not mlir-tblgen -gen-rewriters -I %S/../../include -DERROR6 %s 2>&1 | FileCheck --check-prefix=ERROR6 %s // RUN: not mlir-tblgen -gen-rewriters -I %S/../../include -DERROR7 %s 2>&1 | FileCheck --check-prefix=ERROR7 %s +// RUN: not mlir-tblgen -gen-rewriters -I %S/../../include -DERROR8 %s 2>&1 | FileCheck --check-prefix=ERROR8 %s include "mlir/IR/OpBase.td" include "mlir/IR/PatternBase.td" @@ -64,3 +65,17 @@ def : Pat<(OpB:$result $val, $attr), (OpA $val, $val), [(AnyInteger:$result)]>; // ERROR7: [[@LINE+1]]:1: error: type constraint requires exactly one argument def : Pat<(OpB:$opB $val, $attr), (OpA $val, $val), [(AnyInteger $opB, $val)]>; #endif + +def OpC : A_Op<"op_c">, Results<(outs AnyInteger)>; +def OpD : A_Op<"op_d">, Arguments<(ins Variadic:$vargs)>, Results<(outs AnyInteger)>; + +#ifdef ERROR8 +// Check that op with variadic operand gets variadic operand in target, +// +// FIXME: this should be an error. +def : Pat<(OpB:$opB $val, $attr), (OpD $val)>; + +// ERROR8: [[@LINE+2]] +// ERROR8-SAME: op expects variadic operand `vargs`, while provided is non-variadic +def : Pat<(OpB:$opB $val, $attr), (OpD (OpC))>; +#endif diff --git a/mlir/tools/mlir-tblgen/RewriterGen.cpp b/mlir/tools/mlir-tblgen/RewriterGen.cpp index 426a3482960b..e63a065a0708 100644 --- a/mlir/tools/mlir-tblgen/RewriterGen.cpp +++ b/mlir/tools/mlir-tblgen/RewriterGen.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "mlir/Support/IndentedOstream.h" +#include "mlir/TableGen/Argument.h" #include "mlir/TableGen/Attribute.h" #include "mlir/TableGen/CodeGenHelpers.h" #include "mlir/TableGen/Format.h" @@ -18,6 +19,7 @@ #include "mlir/TableGen/Operator.h" #include "mlir/TableGen/Pattern.h" #include "mlir/TableGen/Predicate.h" +#include "mlir/TableGen/Property.h" #include "mlir/TableGen/Type.h" #include "llvm/ADT/FunctionExtras.h" #include "llvm/ADT/SetVector.h" @@ -1518,10 +1520,36 @@ std::string PatternEmitter::handleOpCreation(DagNode tree, int resultIndex, // the key. This includes both bound and unbound child nodes. ChildNodeIndexNameMap childNodeNames; + // If the argument is a type constraint, then its an operand. Check if the + // op's argument is variadic that the argument in the pattern is too. + auto checkIfMatchedVariadic = [&](int i) { + // FIXME: This does not yet check for variable/leaf case. + // FIXME: Change so that native code call can be handled. + const auto *operand = + llvm::dyn_cast_if_present(resultOp.getArg(i)); + if (!operand || !operand->isVariadic()) + return; + + auto child = tree.getArgAsNestedDag(i); + if (!child) + return; + + // Skip over replaceWithValues. + while (child.isReplaceWithValue()) { + if (!(child = child.getArgAsNestedDag(0))) + return; + } + if (!child.isNativeCodeCall() && !child.isVariadic()) + PrintFatalError(loc, formatv("op expects variadic operand `{0}`, while " + "provided is non-variadic", + resultOp.getArgName(i))); + }; + // First go through all the child nodes who are nested DAG constructs to // create ops for them and remember the symbol names for them, so that we can // use the results in the current node. This happens in a recursive manner. for (int i = 0, e = tree.getNumArgs() - tail.numDirectives; i != e; ++i) { + checkIfMatchedVariadic(i); if (auto child = tree.getArgAsNestedDag(i)) childNodeNames[i] = handleResultPattern(child, i, depth + 1); } -- GitLab From ac74d9ec0f73898713dd6e0d33f148c13a51875d Mon Sep 17 00:00:00 2001 From: Vlad Serebrennikov Date: Wed, 6 Mar 2024 14:39:39 +0400 Subject: [PATCH 269/929] [clang][NFC] Regroup declarations in `Sema` (#82217) This patch regroups declarations in `Sema` based on the file they are implemented in (e.g. `SemaChecking.cpp`). This allows to logically split `Sema` in 42 groups. No physical separation is done (e.g. splitting `Sema` into multiple classes). Table of contents added at the very beginning of `Sema`. Grouping is reflected in Doxygen commands, so structure of API reference of `Sema` is also significantly improved ([example from official documentation](https://www.doxygen.nl/manual/examples/memgrp/html/class_memgrp___test.html), [comparison of Sema API reference](https://github.com/llvm/llvm-project/pull/82217#issuecomment-1954567763)). While grouping is intentional, as well as each group consisting of `public` declarations followed by `private` ones (without changing access in-between), exact contents and order of declarations of each group is partially carried over from old structure, partially accidental due to time constrains to do the regrouping over the weekend (`Sema` is just enormously big). Data members and inline function definitions in `Sema.h` complicate the matter, since it's not obvious which group they belong to. Further work is expected to refine contents and order of declarations. What is also intentional is some kind of layering, where Concepts group follows template groups, and ObjC, code completion, CUDA, HLSL, OpenACC, OpenMP, and SYCL are all placed at the end of the file, after C and C++ parts of `Sema`. I used `clang-query` to verify that access specifiers were preserved during the process (https://gcc.godbolt.org/z/9johffY9T, thank you @ilya-biryukov). Only the following 3 member types were converted from `private` to `public` because of limitations of the new grouping: `DeclareTargetContextInfo`, `TypoExprState`, `SatisfactionStackEntryTy`. Member initializer list of `Sema` in `Sema.cpp` is rewritten to reflect new order of data members in order to avoid `-Wreorder-ctor`. Since this patch touches almost every line in `Sema.h`, it was considered appropriate to run clang-format on the whole file, and not just on changed lines. --- clang/include/clang/Sema/Sema.h | 23067 +++++++++++++++--------------- clang/lib/Sema/Sema.cpp | 38 +- 2 files changed, 11677 insertions(+), 11428 deletions(-) diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index 25c4c58ad4ae..f3d3a57104ee 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -76,168 +76,168 @@ #include namespace llvm { - class APSInt; - template class DenseSet; - class SmallBitVector; - struct InlineAsmIdentifierInfo; -} +class APSInt; +template class DenseSet; +class SmallBitVector; +struct InlineAsmIdentifierInfo; +} // namespace llvm namespace clang { - class ADLResult; - class ASTConsumer; - class ASTContext; - class ASTMutationListener; - class ASTReader; - class ASTWriter; - class ArrayType; - class ParsedAttr; - class BindingDecl; - class BlockDecl; - class CapturedDecl; - class CXXBasePath; - class CXXBasePaths; - class CXXBindTemporaryExpr; - typedef SmallVector CXXCastPath; - class CXXConstructorDecl; - class CXXConversionDecl; - class CXXDeleteExpr; - class CXXDestructorDecl; - class CXXFieldCollector; - class CXXMemberCallExpr; - class CXXMethodDecl; - class CXXScopeSpec; - class CXXTemporary; - class CXXTryStmt; - class CallExpr; - class ClassTemplateDecl; - class ClassTemplatePartialSpecializationDecl; - class ClassTemplateSpecializationDecl; - class VarTemplatePartialSpecializationDecl; - class CodeCompleteConsumer; - class CodeCompletionAllocator; - class CodeCompletionTUInfo; - class CodeCompletionResult; - class CoroutineBodyStmt; - class Decl; - class DeclAccessPair; - class DeclContext; - class DeclRefExpr; - class DeclaratorDecl; - class DeducedTemplateArgument; - class DependentDiagnostic; - class DesignatedInitExpr; - class Designation; - class EnableIfAttr; - class EnumConstantDecl; - class Expr; - class ExtVectorType; - class FormatAttr; - class FriendDecl; - class FunctionDecl; - class FunctionProtoType; - class FunctionTemplateDecl; - class ImplicitConversionSequence; - typedef MutableArrayRef ConversionSequenceList; - class InitListExpr; - class InitializationKind; - class InitializationSequence; - class InitializedEntity; - class IntegerLiteral; - class LabelStmt; - class LambdaExpr; - class LangOptions; - class LocalInstantiationScope; - class LookupResult; - class MacroInfo; - typedef ArrayRef> ModuleIdPath; - class ModuleLoader; - class MultiLevelTemplateArgumentList; - class NamedDecl; - class ObjCCategoryDecl; - class ObjCCategoryImplDecl; - class ObjCCompatibleAliasDecl; - class ObjCContainerDecl; - class ObjCImplDecl; - class ObjCImplementationDecl; - class ObjCInterfaceDecl; - class ObjCIvarDecl; - template class ObjCList; - class ObjCMessageExpr; - class ObjCMethodDecl; - class ObjCPropertyDecl; - class ObjCProtocolDecl; - class OMPThreadPrivateDecl; - class OMPRequiresDecl; - class OMPDeclareReductionDecl; - class OMPDeclareSimdDecl; - class OMPClause; - struct OMPVarListLocTy; - struct OverloadCandidate; - enum class OverloadCandidateParamOrder : char; - enum OverloadCandidateRewriteKind : unsigned; - class OverloadCandidateSet; - class OverloadExpr; - class ParenListExpr; - class ParmVarDecl; - class Preprocessor; - class PseudoDestructorTypeStorage; - class PseudoObjectExpr; - class QualType; - class StandardConversionSequence; - class Stmt; - class StringLiteral; - class SwitchStmt; - class TemplateArgument; - class TemplateArgumentList; - class TemplateArgumentLoc; - class TemplateDecl; - class TemplateInstantiationCallback; - class TemplateParameterList; - class TemplatePartialOrderingContext; - class TemplateTemplateParmDecl; - class Token; - class TypeAliasDecl; - class TypedefDecl; - class TypedefNameDecl; - class TypeLoc; - class TypoCorrectionConsumer; - class UnqualifiedId; - class UnresolvedLookupExpr; - class UnresolvedMemberExpr; - class UnresolvedSetImpl; - class UnresolvedSetIterator; - class UsingDecl; - class UsingShadowDecl; - class ValueDecl; - class VarDecl; - class VarTemplateSpecializationDecl; - class VisibilityAttr; - class VisibleDeclConsumer; - class IndirectFieldDecl; - struct DeductionFailureInfo; - class TemplateSpecCandidateSet; +class ADLResult; +class ASTConsumer; +class ASTContext; +class ASTMutationListener; +class ASTReader; +class ASTWriter; +class ArrayType; +class ParsedAttr; +class BindingDecl; +class BlockDecl; +class CapturedDecl; +class CXXBasePath; +class CXXBasePaths; +class CXXBindTemporaryExpr; +typedef SmallVector CXXCastPath; +class CXXConstructorDecl; +class CXXConversionDecl; +class CXXDeleteExpr; +class CXXDestructorDecl; +class CXXFieldCollector; +class CXXMemberCallExpr; +class CXXMethodDecl; +class CXXScopeSpec; +class CXXTemporary; +class CXXTryStmt; +class CallExpr; +class ClassTemplateDecl; +class ClassTemplatePartialSpecializationDecl; +class ClassTemplateSpecializationDecl; +class VarTemplatePartialSpecializationDecl; +class CodeCompleteConsumer; +class CodeCompletionAllocator; +class CodeCompletionTUInfo; +class CodeCompletionResult; +class CoroutineBodyStmt; +class Decl; +class DeclAccessPair; +class DeclContext; +class DeclRefExpr; +class DeclaratorDecl; +class DeducedTemplateArgument; +class DependentDiagnostic; +class DesignatedInitExpr; +class Designation; +class EnableIfAttr; +class EnumConstantDecl; +class Expr; +class ExtVectorType; +class FormatAttr; +class FriendDecl; +class FunctionDecl; +class FunctionProtoType; +class FunctionTemplateDecl; +class ImplicitConversionSequence; +typedef MutableArrayRef ConversionSequenceList; +class InitListExpr; +class InitializationKind; +class InitializationSequence; +class InitializedEntity; +class IntegerLiteral; +class LabelStmt; +class LambdaExpr; +class LangOptions; +class LocalInstantiationScope; +class LookupResult; +class MacroInfo; +typedef ArrayRef> ModuleIdPath; +class ModuleLoader; +class MultiLevelTemplateArgumentList; +class NamedDecl; +class ObjCCategoryDecl; +class ObjCCategoryImplDecl; +class ObjCCompatibleAliasDecl; +class ObjCContainerDecl; +class ObjCImplDecl; +class ObjCImplementationDecl; +class ObjCInterfaceDecl; +class ObjCIvarDecl; +template class ObjCList; +class ObjCMessageExpr; +class ObjCMethodDecl; +class ObjCPropertyDecl; +class ObjCProtocolDecl; +class OMPThreadPrivateDecl; +class OMPRequiresDecl; +class OMPDeclareReductionDecl; +class OMPDeclareSimdDecl; +class OMPClause; +struct OMPVarListLocTy; +struct OverloadCandidate; +enum class OverloadCandidateParamOrder : char; +enum OverloadCandidateRewriteKind : unsigned; +class OverloadCandidateSet; +class OverloadExpr; +class ParenListExpr; +class ParmVarDecl; +class Preprocessor; +class PseudoDestructorTypeStorage; +class PseudoObjectExpr; +class QualType; +class StandardConversionSequence; +class Stmt; +class StringLiteral; +class SwitchStmt; +class TemplateArgument; +class TemplateArgumentList; +class TemplateArgumentLoc; +class TemplateDecl; +class TemplateInstantiationCallback; +class TemplateParameterList; +class TemplatePartialOrderingContext; +class TemplateTemplateParmDecl; +class Token; +class TypeAliasDecl; +class TypedefDecl; +class TypedefNameDecl; +class TypeLoc; +class TypoCorrectionConsumer; +class UnqualifiedId; +class UnresolvedLookupExpr; +class UnresolvedMemberExpr; +class UnresolvedSetImpl; +class UnresolvedSetIterator; +class UsingDecl; +class UsingShadowDecl; +class ValueDecl; +class VarDecl; +class VarTemplateSpecializationDecl; +class VisibilityAttr; +class VisibleDeclConsumer; +class IndirectFieldDecl; +struct DeductionFailureInfo; +class TemplateSpecCandidateSet; namespace sema { - class AccessedEntity; - class BlockScopeInfo; - class Capture; - class CapturedRegionScopeInfo; - class CapturingScopeInfo; - class CompoundScopeInfo; - class DelayedDiagnostic; - class DelayedDiagnosticPool; - class FunctionScopeInfo; - class LambdaScopeInfo; - class PossiblyUnreachableDiag; - class RISCVIntrinsicManager; - class SemaPPCallbacks; - class TemplateDeductionInfo; -} +class AccessedEntity; +class BlockScopeInfo; +class Capture; +class CapturedRegionScopeInfo; +class CapturingScopeInfo; +class CompoundScopeInfo; +class DelayedDiagnostic; +class DelayedDiagnosticPool; +class FunctionScopeInfo; +class LambdaScopeInfo; +class PossiblyUnreachableDiag; +class RISCVIntrinsicManager; +class SemaPPCallbacks; +class TemplateDeductionInfo; +} // namespace sema namespace threadSafety { - class BeforeSet; - void threadSafetyCleanup(BeforeSet* Cache); -} +class BeforeSet; +void threadSafetyCleanup(BeforeSet *Cache); +} // namespace threadSafety // FIXME: No way to easily map from TemplateTypeParmTypes to // TemplateTypeParmDecls, so we have this horrible PointerUnion. @@ -422,591 +422,652 @@ enum class TemplateDeductionResult { }; /// Sema - This implements semantic analysis and AST building for C. +/// \nosubgrouping class Sema final { - Sema(const Sema &) = delete; - void operator=(const Sema &) = delete; - - ///Source of additional semantic information. - IntrusiveRefCntPtr ExternalSource; - - static bool mightHaveNonExternalLinkage(const DeclaratorDecl *FD); - - /// Determine whether two declarations should be linked together, given that - /// the old declaration might not be visible and the new declaration might - /// not have external linkage. - bool shouldLinkPossiblyHiddenDecl(const NamedDecl *Old, - const NamedDecl *New) { - if (isVisible(Old)) - return true; - // See comment in below overload for why it's safe to compute the linkage - // of the new declaration here. - if (New->isExternallyDeclarable()) { - assert(Old->isExternallyDeclarable() && - "should not have found a non-externally-declarable previous decl"); - return true; - } - return false; - } - bool shouldLinkPossiblyHiddenDecl(LookupResult &Old, const NamedDecl *New); - - void setupImplicitSpecialMemberType(CXXMethodDecl *SpecialMem, - QualType ResultTy, - ArrayRef Args); + // Table of Contents + // ----------------- + // 1. Semantic Analysis (Sema.cpp) + // 2. C++ Access Control (SemaAccess.cpp) + // 3. Attributes (SemaAttr.cpp) + // 4. Availability Attribute Handling (SemaAvailability.cpp) + // 5. Casts (SemaCast.cpp) + // 6. Extra Semantic Checking (SemaChecking.cpp) + // 7. C++ Coroutines (SemaCoroutine.cpp) + // 8. C++ Scope Specifiers (SemaCXXScopeSpec.cpp) + // 9. Declarations (SemaDecl.cpp) + // 10. Declaration Attribute Handling (SemaDeclAttr.cpp) + // 11. C++ Declarations (SemaDeclCXX.cpp) + // 12. C++ Exception Specifications (SemaExceptionSpec.cpp) + // 13. Expressions (SemaExpr.cpp) + // 14. C++ Expressions (SemaExprCXX.cpp) + // 15. Member Access Expressions (SemaExprMember.cpp) + // 16. Initializers (SemaInit.cpp) + // 17. C++ Lambda Expressions (SemaLambda.cpp) + // 18. Name Lookup (SemaLookup.cpp) + // 19. Modules (SemaModule.cpp) + // 20. C++ Overloading (SemaOverload.cpp) + // 21. Pseudo-Object (SemaPseudoObject.cpp) + // 22. Statements (SemaStmt.cpp) + // 23. `inline asm` Statement (SemaStmtAsm.cpp) + // 24. Statement Attribute Handling (SemaStmtAttr.cpp) + // 25. C++ Templates (SemaTemplate.cpp) + // 26. C++ Template Argument Deduction (SemaTemplateDeduction.cpp) + // 27. C++ Template Instantiation (SemaTemplateInstantiate.cpp) + // 28. C++ Template Declaration Instantiation + // (SemaTemplateInstantiateDecl.cpp) + // 29. C++ Variadic Templates (SemaTemplateVariadic.cpp) + // 30. Constraints and Concepts (SemaConcept.cpp) + // 31. Types (SemaType.cpp) + // 32. ObjC Declarations (SemaDeclObjC.cpp) + // 33. ObjC Expressions (SemaExprObjC.cpp) + // 34. ObjC @property and @synthesize (SemaObjCProperty.cpp) + // 35. Code Completion (SemaCodeComplete.cpp) + // 36. FixIt Helpers (SemaFixItUtils.cpp) + // 37. Name Lookup for RISC-V Vector Intrinsic (SemaRISCVVectorLookup.cpp) + // 38. CUDA (SemaCUDA.cpp) + // 39. HLSL Constructs (SemaHLSL.cpp) + // 40. OpenACC Constructs (SemaOpenACC.cpp) + // 41. OpenMP Directives and Clauses (SemaOpenMP.cpp) + // 42. SYCL Constructs (SemaSYCL.cpp) + + /// \name Semantic Analysis + /// Implementations are in Sema.cpp + ///@{ public: - /// The maximum alignment, same as in llvm::Value. We duplicate them here - /// because that allows us not to duplicate the constants in clang code, - /// which we must to since we can't directly use the llvm constants. - /// The value is verified against llvm here: lib/CodeGen/CGDecl.cpp - /// - /// This is the greatest alignment value supported by load, store, and alloca - /// instructions, and global values. - static const unsigned MaxAlignmentExponent = 32; - static const uint64_t MaximumAlignment = 1ull << MaxAlignmentExponent; - - typedef OpaquePtr DeclGroupPtrTy; - typedef OpaquePtr TemplateTy; - typedef OpaquePtr TypeTy; - - OpenCLOptions OpenCLFeatures; - FPOptions CurFPFeatures; - - const LangOptions &LangOpts; - Preprocessor &PP; - ASTContext &Context; - ASTConsumer &Consumer; - DiagnosticsEngine &Diags; - SourceManager &SourceMgr; - api_notes::APINotesManager APINotes; - - /// Flag indicating whether or not to collect detailed statistics. - bool CollectStats; - - /// Code-completion consumer. - CodeCompleteConsumer *CodeCompleter; + Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer, + TranslationUnitKind TUKind = TU_Complete, + CodeCompleteConsumer *CompletionConsumer = nullptr); + ~Sema(); - /// CurContext - This is the current declaration context of parsing. - DeclContext *CurContext; + /// Perform initialization that occurs after the parser has been + /// initialized but before it parses anything. + void Initialize(); - /// Generally null except when we temporarily switch decl contexts, - /// like in \see ActOnObjCTemporaryExitContainerContext. - DeclContext *OriginalLexicalContext; + /// This virtual key function only exists to limit the emission of debug info + /// describing the Sema class. GCC and Clang only emit debug info for a class + /// with a vtable when the vtable is emitted. Sema is final and not + /// polymorphic, but the debug info size savings are so significant that it is + /// worth adding a vtable just to take advantage of this optimization. + virtual void anchor(); - /// VAListTagName - The declaration name corresponding to __va_list_tag. - /// This is used as part of a hack to omit that class from ADL results. - DeclarationName VAListTagName; + const LangOptions &getLangOpts() const { return LangOpts; } + OpenCLOptions &getOpenCLOptions() { return OpenCLFeatures; } + FPOptions &getCurFPFeatures() { return CurFPFeatures; } - bool MSStructPragmaOn; // True when \#pragma ms_struct on + DiagnosticsEngine &getDiagnostics() const { return Diags; } + SourceManager &getSourceManager() const { return SourceMgr; } + Preprocessor &getPreprocessor() const { return PP; } + ASTContext &getASTContext() const { return Context; } + ASTConsumer &getASTConsumer() const { return Consumer; } + ASTMutationListener *getASTMutationListener() const; + ExternalSemaSource *getExternalSource() const { return ExternalSource.get(); } - /// Controls member pointer representation format under the MS ABI. - LangOptions::PragmaMSPointersToMembersKind - MSPointerToMemberRepresentationMethod; + DarwinSDKInfo *getDarwinSDKInfoForAvailabilityChecking(SourceLocation Loc, + StringRef Platform); + DarwinSDKInfo *getDarwinSDKInfoForAvailabilityChecking(); - /// Stack of active SEH __finally scopes. Can be empty. - SmallVector CurrentSEHFinally; + /// Registers an external source. If an external source already exists, + /// creates a multiplex external source and appends to it. + /// + ///\param[in] E - A non-null external sema source. + /// + void addExternalSource(ExternalSemaSource *E); - /// Source location for newly created implicit MSInheritanceAttrs - SourceLocation ImplicitMSInheritanceAttrLoc; + /// Helper class that creates diagnostics with optional + /// template instantiation stacks. + /// + /// This class provides a wrapper around the basic DiagnosticBuilder + /// class that emits diagnostics. ImmediateDiagBuilder is + /// responsible for emitting the diagnostic (as DiagnosticBuilder + /// does) and, if the diagnostic comes from inside a template + /// instantiation, printing the template instantiation stack as + /// well. + class ImmediateDiagBuilder : public DiagnosticBuilder { + Sema &SemaRef; + unsigned DiagID; - /// Holds TypoExprs that are created from `createDelayedTypo`. This is used by - /// `TransformTypos` in order to keep track of any TypoExprs that are created - /// recursively during typo correction and wipe them away if the correction - /// fails. - llvm::SmallVector TypoExprs; + public: + ImmediateDiagBuilder(DiagnosticBuilder &DB, Sema &SemaRef, unsigned DiagID) + : DiagnosticBuilder(DB), SemaRef(SemaRef), DiagID(DiagID) {} + ImmediateDiagBuilder(DiagnosticBuilder &&DB, Sema &SemaRef, unsigned DiagID) + : DiagnosticBuilder(DB), SemaRef(SemaRef), DiagID(DiagID) {} - /// pragma clang section kind - enum PragmaClangSectionKind { - PCSK_Invalid = 0, - PCSK_BSS = 1, - PCSK_Data = 2, - PCSK_Rodata = 3, - PCSK_Text = 4, - PCSK_Relro = 5 - }; + // This is a cunning lie. DiagnosticBuilder actually performs move + // construction in its copy constructor (but due to varied uses, it's not + // possible to conveniently express this as actual move construction). So + // the default copy ctor here is fine, because the base class disables the + // source anyway, so the user-defined ~ImmediateDiagBuilder is a safe no-op + // in that case anwyay. + ImmediateDiagBuilder(const ImmediateDiagBuilder &) = default; - enum PragmaClangSectionAction { - PCSA_Set = 0, - PCSA_Clear = 1 - }; + ~ImmediateDiagBuilder() { + // If we aren't active, there is nothing to do. + if (!isActive()) + return; - struct PragmaClangSection { - std::string SectionName; - bool Valid = false; - SourceLocation PragmaLocation; - }; + // Otherwise, we need to emit the diagnostic. First clear the diagnostic + // builder itself so it won't emit the diagnostic in its own destructor. + // + // This seems wasteful, in that as written the DiagnosticBuilder dtor will + // do its own needless checks to see if the diagnostic needs to be + // emitted. However, because we take care to ensure that the builder + // objects never escape, a sufficiently smart compiler will be able to + // eliminate that code. + Clear(); - PragmaClangSection PragmaClangBSSSection; - PragmaClangSection PragmaClangDataSection; - PragmaClangSection PragmaClangRodataSection; - PragmaClangSection PragmaClangRelroSection; - PragmaClangSection PragmaClangTextSection; + // Dispatch to Sema to emit the diagnostic. + SemaRef.EmitCurrentDiagnostic(DiagID); + } - enum PragmaMsStackAction { - PSK_Reset = 0x0, // #pragma () - PSK_Set = 0x1, // #pragma (value) - PSK_Push = 0x2, // #pragma (push[, id]) - PSK_Pop = 0x4, // #pragma (pop[, id]) - PSK_Show = 0x8, // #pragma (show) -- only for "pack"! - PSK_Push_Set = PSK_Push | PSK_Set, // #pragma (push[, id], value) - PSK_Pop_Set = PSK_Pop | PSK_Set, // #pragma (pop[, id], value) - }; + /// Teach operator<< to produce an object of the correct type. + template + friend const ImmediateDiagBuilder & + operator<<(const ImmediateDiagBuilder &Diag, const T &Value) { + const DiagnosticBuilder &BaseDiag = Diag; + BaseDiag << Value; + return Diag; + } - struct PragmaPackInfo { - PragmaMsStackAction Action; - StringRef SlotLabel; - Token Alignment; + // It is necessary to limit this to rvalue reference to avoid calling this + // function with a bitfield lvalue argument since non-const reference to + // bitfield is not allowed. + template ::value>> + const ImmediateDiagBuilder &operator<<(T &&V) const { + const DiagnosticBuilder &BaseDiag = *this; + BaseDiag << std::move(V); + return *this; + } }; - // #pragma pack and align. - class AlignPackInfo { + /// A generic diagnostic builder for errors which may or may not be deferred. + /// + /// In CUDA, there exist constructs (e.g. variable-length arrays, try/catch) + /// which are not allowed to appear inside __device__ functions and are + /// allowed to appear in __host__ __device__ functions only if the host+device + /// function is never codegen'ed. + /// + /// To handle this, we use the notion of "deferred diagnostics", where we + /// attach a diagnostic to a FunctionDecl that's emitted iff it's codegen'ed. + /// + /// This class lets you emit either a regular diagnostic, a deferred + /// diagnostic, or no diagnostic at all, according to an argument you pass to + /// its constructor, thus simplifying the process of creating these "maybe + /// deferred" diagnostics. + class SemaDiagnosticBuilder { public: - // `Native` represents default align mode, which may vary based on the - // platform. - enum Mode : unsigned char { Native, Natural, Packed, Mac68k }; - - // #pragma pack info constructor - AlignPackInfo(AlignPackInfo::Mode M, unsigned Num, bool IsXL) - : PackAttr(true), AlignMode(M), PackNumber(Num), XLStack(IsXL) { - assert(Num == PackNumber && "The pack number has been truncated."); - } - - // #pragma align info constructor - AlignPackInfo(AlignPackInfo::Mode M, bool IsXL) - : PackAttr(false), AlignMode(M), - PackNumber(M == Packed ? 1 : UninitPackVal), XLStack(IsXL) {} + enum Kind { + /// Emit no diagnostics. + K_Nop, + /// Emit the diagnostic immediately (i.e., behave like Sema::Diag()). + K_Immediate, + /// Emit the diagnostic immediately, and, if it's a warning or error, also + /// emit a call stack showing how this function can be reached by an a + /// priori known-emitted function. + K_ImmediateWithCallStack, + /// Create a deferred diagnostic, which is emitted only if the function + /// it's attached to is codegen'ed. Also emit a call stack as with + /// K_ImmediateWithCallStack. + K_Deferred + }; - explicit AlignPackInfo(bool IsXL) : AlignPackInfo(Native, IsXL) {} + SemaDiagnosticBuilder(Kind K, SourceLocation Loc, unsigned DiagID, + const FunctionDecl *Fn, Sema &S); + SemaDiagnosticBuilder(SemaDiagnosticBuilder &&D); + SemaDiagnosticBuilder(const SemaDiagnosticBuilder &) = default; - AlignPackInfo() : AlignPackInfo(Native, false) {} + // The copy and move assignment operator is defined as deleted pending + // further motivation. + SemaDiagnosticBuilder &operator=(const SemaDiagnosticBuilder &) = delete; + SemaDiagnosticBuilder &operator=(SemaDiagnosticBuilder &&) = delete; - // When a AlignPackInfo itself cannot be used, this returns an 32-bit - // integer encoding for it. This should only be passed to - // AlignPackInfo::getFromRawEncoding, it should not be inspected directly. - static uint32_t getRawEncoding(const AlignPackInfo &Info) { - std::uint32_t Encoding{}; - if (Info.IsXLStack()) - Encoding |= IsXLMask; + ~SemaDiagnosticBuilder(); - Encoding |= static_cast(Info.getAlignMode()) << 1; + bool isImmediate() const { return ImmediateDiag.has_value(); } - if (Info.IsPackAttr()) - Encoding |= PackAttrMask; + /// Convertible to bool: True if we immediately emitted an error, false if + /// we didn't emit an error or we created a deferred error. + /// + /// Example usage: + /// + /// if (SemaDiagnosticBuilder(...) << foo << bar) + /// return ExprError(); + /// + /// But see CUDADiagIfDeviceCode() and CUDADiagIfHostCode() -- you probably + /// want to use these instead of creating a SemaDiagnosticBuilder yourself. + operator bool() const { return isImmediate(); } - Encoding |= static_cast(Info.getPackNumber()) << 4; - - return Encoding; + template + friend const SemaDiagnosticBuilder & + operator<<(const SemaDiagnosticBuilder &Diag, const T &Value) { + if (Diag.ImmediateDiag) + *Diag.ImmediateDiag << Value; + else if (Diag.PartialDiagId) + Diag.S.DeviceDeferredDiags[Diag.Fn][*Diag.PartialDiagId].second + << Value; + return Diag; } - static AlignPackInfo getFromRawEncoding(unsigned Encoding) { - bool IsXL = static_cast(Encoding & IsXLMask); - AlignPackInfo::Mode M = - static_cast((Encoding & AlignModeMask) >> 1); - int PackNumber = (Encoding & PackNumMask) >> 4; + // It is necessary to limit this to rvalue reference to avoid calling this + // function with a bitfield lvalue argument since non-const reference to + // bitfield is not allowed. + template ::value>> + const SemaDiagnosticBuilder &operator<<(T &&V) const { + if (ImmediateDiag) + *ImmediateDiag << std::move(V); + else if (PartialDiagId) + S.DeviceDeferredDiags[Fn][*PartialDiagId].second << std::move(V); + return *this; + } - if (Encoding & PackAttrMask) - return AlignPackInfo(M, PackNumber, IsXL); + friend const SemaDiagnosticBuilder & + operator<<(const SemaDiagnosticBuilder &Diag, const PartialDiagnostic &PD) { + if (Diag.ImmediateDiag) + PD.Emit(*Diag.ImmediateDiag); + else if (Diag.PartialDiagId) + Diag.S.DeviceDeferredDiags[Diag.Fn][*Diag.PartialDiagId].second = PD; + return Diag; + } - return AlignPackInfo(M, IsXL); + void AddFixItHint(const FixItHint &Hint) const { + if (ImmediateDiag) + ImmediateDiag->AddFixItHint(Hint); + else if (PartialDiagId) + S.DeviceDeferredDiags[Fn][*PartialDiagId].second.AddFixItHint(Hint); } - bool IsPackAttr() const { return PackAttr; } + friend ExprResult ExprError(const SemaDiagnosticBuilder &) { + return ExprError(); + } + friend StmtResult StmtError(const SemaDiagnosticBuilder &) { + return StmtError(); + } + operator ExprResult() const { return ExprError(); } + operator StmtResult() const { return StmtError(); } + operator TypeResult() const { return TypeError(); } + operator DeclResult() const { return DeclResult(true); } + operator MemInitResult() const { return MemInitResult(true); } - bool IsAlignAttr() const { return !PackAttr; } + private: + Sema &S; + SourceLocation Loc; + unsigned DiagID; + const FunctionDecl *Fn; + bool ShowCallStack; - Mode getAlignMode() const { return AlignMode; } + // Invariant: At most one of these Optionals has a value. + // FIXME: Switch these to a Variant once that exists. + std::optional ImmediateDiag; + std::optional PartialDiagId; + }; - unsigned getPackNumber() const { return PackNumber; } + void PrintStats() const; - bool IsPackSet() const { - // #pragma align, #pragma pack(), and #pragma pack(0) do not set the pack - // attriute on a decl. - return PackNumber != UninitPackVal && PackNumber != 0; - } + /// Warn that the stack is nearly exhausted. + void warnStackExhausted(SourceLocation Loc); - bool IsXLStack() const { return XLStack; } + /// Run some code with "sufficient" stack space. (Currently, at least 256K is + /// guaranteed). Produces a warning if we're low on stack space and allocates + /// more in that case. Use this in code that may recurse deeply (for example, + /// in template instantiation) to avoid stack overflow. + void runWithSufficientStackSpace(SourceLocation Loc, + llvm::function_ref Fn); - bool operator==(const AlignPackInfo &Info) const { - return std::tie(AlignMode, PackNumber, PackAttr, XLStack) == - std::tie(Info.AlignMode, Info.PackNumber, Info.PackAttr, - Info.XLStack); - } + /// Returns default addr space for method qualifiers. + LangAS getDefaultCXXMethodAddrSpace() const; - bool operator!=(const AlignPackInfo &Info) const { - return !(*this == Info); - } + /// Load weak undeclared identifiers from the external source. + void LoadExternalWeakUndeclaredIdentifiers(); - private: - /// \brief True if this is a pragma pack attribute, - /// not a pragma align attribute. - bool PackAttr; + /// Determine if VD, which must be a variable or function, is an external + /// symbol that nonetheless can't be referenced from outside this translation + /// unit because its type has no linkage and it's not extern "C". + bool isExternalWithNoLinkageType(const ValueDecl *VD) const; - /// \brief The alignment mode that is in effect. - Mode AlignMode; + /// Obtain a sorted list of functions that are undefined but ODR-used. + void getUndefinedButUsed( + SmallVectorImpl> &Undefined); - /// \brief The pack number of the stack. - unsigned char PackNumber; + typedef std::pair DeleteExprLoc; + typedef llvm::SmallVector DeleteLocs; + /// Retrieves list of suspicious delete-expressions that will be checked at + /// the end of translation unit. + const llvm::MapVector & + getMismatchingDeleteExpressions() const; - /// \brief True if it is a XL #pragma align/pack stack. - bool XLStack; + /// Cause the active diagnostic on the DiagosticsEngine to be + /// emitted. This is closely coupled to the SemaDiagnosticBuilder class and + /// should not be used elsewhere. + void EmitCurrentDiagnostic(unsigned DiagID); - /// \brief Uninitialized pack value. - static constexpr unsigned char UninitPackVal = -1; + void addImplicitTypedef(StringRef Name, QualType T); - // Masks to encode and decode an AlignPackInfo. - static constexpr uint32_t IsXLMask{0x0000'0001}; - static constexpr uint32_t AlignModeMask{0x0000'0006}; - static constexpr uint32_t PackAttrMask{0x00000'0008}; - static constexpr uint32_t PackNumMask{0x0000'01F0}; - }; + /// Emit a diagnostic. + SemaDiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID, + bool DeferHint = false); - template - struct PragmaStack { - struct Slot { - llvm::StringRef StackSlotLabel; - ValueType Value; - SourceLocation PragmaLocation; - SourceLocation PragmaPushLocation; - Slot(llvm::StringRef StackSlotLabel, ValueType Value, - SourceLocation PragmaLocation, SourceLocation PragmaPushLocation) - : StackSlotLabel(StackSlotLabel), Value(Value), - PragmaLocation(PragmaLocation), - PragmaPushLocation(PragmaPushLocation) {} - }; + /// Emit a partial diagnostic. + SemaDiagnosticBuilder Diag(SourceLocation Loc, const PartialDiagnostic &PD, + bool DeferHint = false); - void Act(SourceLocation PragmaLocation, PragmaMsStackAction Action, - llvm::StringRef StackSlotLabel, ValueType Value) { - if (Action == PSK_Reset) { - CurrentValue = DefaultValue; - CurrentPragmaLocation = PragmaLocation; - return; - } - if (Action & PSK_Push) - Stack.emplace_back(StackSlotLabel, CurrentValue, CurrentPragmaLocation, - PragmaLocation); - else if (Action & PSK_Pop) { - if (!StackSlotLabel.empty()) { - // If we've got a label, try to find it and jump there. - auto I = llvm::find_if(llvm::reverse(Stack), [&](const Slot &x) { - return x.StackSlotLabel == StackSlotLabel; - }); - // If we found the label so pop from there. - if (I != Stack.rend()) { - CurrentValue = I->Value; - CurrentPragmaLocation = I->PragmaLocation; - Stack.erase(std::prev(I.base()), Stack.end()); - } - } else if (!Stack.empty()) { - // We do not have a label, just pop the last entry. - CurrentValue = Stack.back().Value; - CurrentPragmaLocation = Stack.back().PragmaLocation; - Stack.pop_back(); - } - } - if (Action & PSK_Set) { - CurrentValue = Value; - CurrentPragmaLocation = PragmaLocation; - } - } + /// Whether uncompilable error has occurred. This includes error happens + /// in deferred diagnostics. + bool hasUncompilableErrorOccurred() const; - // MSVC seems to add artificial slots to #pragma stacks on entering a C++ - // method body to restore the stacks on exit, so it works like this: - // - // struct S { - // #pragma (push, InternalPragmaSlot, ) - // void Method {} - // #pragma (pop, InternalPragmaSlot) - // }; - // - // It works even with #pragma vtordisp, although MSVC doesn't support - // #pragma vtordisp(push [, id], n) - // syntax. - // - // Push / pop a named sentinel slot. - void SentinelAction(PragmaMsStackAction Action, StringRef Label) { - assert((Action == PSK_Push || Action == PSK_Pop) && - "Can only push / pop #pragma stack sentinels!"); - Act(CurrentPragmaLocation, Action, Label, CurrentValue); - } + bool findMacroSpelling(SourceLocation &loc, StringRef name); - // Constructors. - explicit PragmaStack(const ValueType &Default) - : DefaultValue(Default), CurrentValue(Default) {} + /// Calls \c Lexer::getLocForEndOfToken() + SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset = 0); - bool hasValue() const { return CurrentValue != DefaultValue; } + /// Retrieve the module loader associated with the preprocessor. + ModuleLoader &getModuleLoader() const; - SmallVector Stack; - ValueType DefaultValue; // Value used for PSK_Reset action. - ValueType CurrentValue; - SourceLocation CurrentPragmaLocation; - }; - // FIXME: We should serialize / deserialize these if they occur in a PCH (but - // we shouldn't do so if they're in a module). + /// Invent a new identifier for parameters of abbreviated templates. + IdentifierInfo * + InventAbbreviatedTemplateParameterTypeName(IdentifierInfo *ParamName, + unsigned Index); - /// Whether to insert vtordisps prior to virtual bases in the Microsoft - /// C++ ABI. Possible values are 0, 1, and 2, which mean: - /// - /// 0: Suppress all vtordisps - /// 1: Insert vtordisps in the presence of vbase overrides and non-trivial - /// structors - /// 2: Always insert vtordisps to support RTTI on partially constructed - /// objects - PragmaStack VtorDispStack; - PragmaStack AlignPackStack; - // The current #pragma align/pack values and locations at each #include. - struct AlignPackIncludeState { - AlignPackInfo CurrentValue; - SourceLocation CurrentPragmaLocation; - bool HasNonDefaultValue, ShouldWarnOnInclude; - }; - SmallVector AlignPackIncludeStack; - // Segment #pragmas. - PragmaStack DataSegStack; - PragmaStack BSSSegStack; - PragmaStack ConstSegStack; - PragmaStack CodeSegStack; + void emitAndClearUnusedLocalTypedefWarnings(); - // #pragma strict_gs_check. - PragmaStack StrictGuardStackCheckStack; + // Emit all deferred diagnostics. + void emitDeferredDiags(); - // This stack tracks the current state of Sema.CurFPFeatures. - PragmaStack FpPragmaStack; - FPOptionsOverride CurFPFeatureOverrides() { - FPOptionsOverride result; - if (!FpPragmaStack.hasValue()) { - result = FPOptionsOverride(); - } else { - result = FpPragmaStack.CurrentValue; - } - return result; - } - - // Saves the current floating-point pragma stack and clear it in this Sema. - class FpPragmaStackSaveRAII { - public: - FpPragmaStackSaveRAII(Sema &S) - : S(S), SavedStack(std::move(S.FpPragmaStack)) { - S.FpPragmaStack.Stack.clear(); - } - ~FpPragmaStackSaveRAII() { S.FpPragmaStack = std::move(SavedStack); } - - private: - Sema &S; - PragmaStack SavedStack; + enum TUFragmentKind { + /// The global module fragment, between 'module;' and a module-declaration. + Global, + /// A normal translation unit fragment. For a non-module unit, this is the + /// entire translation unit. Otherwise, it runs from the module-declaration + /// to the private-module-fragment (if any) or the end of the TU (if not). + Normal, + /// The private module fragment, between 'module :private;' and the end of + /// the translation unit. + Private }; - void resetFPOptions(FPOptions FPO) { - CurFPFeatures = FPO; - FpPragmaStack.CurrentValue = FPO.getChangesFrom(FPOptions(LangOpts)); - } - - // RAII object to push / pop sentinel slots for all MS #pragma stacks. - // Actions should be performed only if we enter / exit a C++ method body. - class PragmaStackSentinelRAII { - public: - PragmaStackSentinelRAII(Sema &S, StringRef SlotLabel, bool ShouldAct); - ~PragmaStackSentinelRAII(); + void ActOnStartOfTranslationUnit(); + void ActOnEndOfTranslationUnit(); + void ActOnEndOfTranslationUnitFragment(TUFragmentKind Kind); - private: - Sema &S; - StringRef SlotLabel; - bool ShouldAct; - }; + Scope *getScopeForContext(DeclContext *Ctx); - /// A mapping that describes the nullability we've seen in each header file. - FileNullabilityMap NullabilityMap; + void PushFunctionScope(); + void PushBlockScope(Scope *BlockScope, BlockDecl *Block); + sema::LambdaScopeInfo *PushLambdaScope(); - /// Last section used with #pragma init_seg. - StringLiteral *CurInitSeg; - SourceLocation CurInitSegLoc; + /// This is used to inform Sema what the current TemplateParameterDepth + /// is during Parsing. Currently it is used to pass on the depth + /// when parsing generic lambda 'auto' parameters. + void RecordParsingTemplateParameterDepth(unsigned Depth); - /// Sections used with #pragma alloc_text. - llvm::StringMap> FunctionToSectionMap; + void PushCapturedRegionScope(Scope *RegionScope, CapturedDecl *CD, + RecordDecl *RD, CapturedRegionKind K, + unsigned OpenMPCaptureLevel = 0); - /// VisContext - Manages the stack for \#pragma GCC visibility. - void *VisContext; // Really a "PragmaVisStack*" + /// Custom deleter to allow FunctionScopeInfos to be kept alive for a short + /// time after they've been popped. + class PoppedFunctionScopeDeleter { + Sema *Self; - /// This an attribute introduced by \#pragma clang attribute. - struct PragmaAttributeEntry { - SourceLocation Loc; - ParsedAttr *Attribute; - SmallVector MatchRules; - bool IsUsed; + public: + explicit PoppedFunctionScopeDeleter(Sema *Self) : Self(Self) {} + void operator()(sema::FunctionScopeInfo *Scope) const; }; - /// A push'd group of PragmaAttributeEntries. - struct PragmaAttributeGroup { - /// The location of the push attribute. - SourceLocation Loc; - /// The namespace of this push group. - const IdentifierInfo *Namespace; - SmallVector Entries; - }; + using PoppedFunctionScopePtr = + std::unique_ptr; - SmallVector PragmaAttributeStack; + PoppedFunctionScopePtr + PopFunctionScopeInfo(const sema::AnalysisBasedWarnings::Policy *WP = nullptr, + const Decl *D = nullptr, + QualType BlockType = QualType()); - /// The declaration that is currently receiving an attribute from the - /// #pragma attribute stack. - const Decl *PragmaAttributeCurrentTargetDecl; + sema::FunctionScopeInfo *getEnclosingFunction() const; - /// This represents the last location of a "#pragma clang optimize off" - /// directive if such a directive has not been closed by an "on" yet. If - /// optimizations are currently "on", this is set to an invalid location. - SourceLocation OptimizeOffPragmaLocation; + void setFunctionHasBranchIntoScope(); + void setFunctionHasBranchProtectedScope(); + void setFunctionHasIndirectGoto(); + void setFunctionHasMustTail(); - /// The "on" or "off" argument passed by \#pragma optimize, that denotes - /// whether the optimizations in the list passed to the pragma should be - /// turned off or on. This boolean is true by default because command line - /// options are honored when `#pragma optimize("", on)`. - /// (i.e. `ModifyFnAttributeMSPragmaOptimze()` does nothing) - bool MSPragmaOptimizeIsOn = true; + void PushCompoundScope(bool IsStmtExpr); + void PopCompoundScope(); - /// Set of no-builtin functions listed by \#pragma function. - llvm::SmallSetVector MSFunctionNoBuiltins; + bool hasAnyUnrecoverableErrorsInThisFunction() const; - /// Flag indicating if Sema is building a recovery call expression. - /// - /// This flag is used to avoid building recovery call expressions - /// if Sema is already doing so, which would cause infinite recursions. - bool IsBuildingRecoveryCallExpr; + /// Retrieve the current block, if any. + sema::BlockScopeInfo *getCurBlock(); - /// Used to control the generation of ExprWithCleanups. - CleanupInfo Cleanup; + /// Get the innermost lambda enclosing the current location, if any. This + /// looks through intervening non-lambda scopes such as local functions and + /// blocks. + sema::LambdaScopeInfo *getEnclosingLambda() const; - /// ExprCleanupObjects - This is the stack of objects requiring - /// cleanup that are created by the current full expression. - SmallVector ExprCleanupObjects; + /// Retrieve the current lambda scope info, if any. + /// \param IgnoreNonLambdaCapturingScope true if should find the top-most + /// lambda scope info ignoring all inner capturing scopes that are not + /// lambda scopes. + sema::LambdaScopeInfo * + getCurLambda(bool IgnoreNonLambdaCapturingScope = false); - /// Store a set of either DeclRefExprs or MemberExprs that contain a reference - /// to a variable (constant) that may or may not be odr-used in this Expr, and - /// we won't know until all lvalue-to-rvalue and discarded value conversions - /// have been applied to all subexpressions of the enclosing full expression. - /// This is cleared at the end of each full expression. - using MaybeODRUseExprSet = llvm::SmallSetVector; - MaybeODRUseExprSet MaybeODRUseExprs; + /// Retrieve the current generic lambda info, if any. + sema::LambdaScopeInfo *getCurGenericLambda(); - std::unique_ptr CachedFunctionScope; + /// Retrieve the current captured region, if any. + sema::CapturedRegionScopeInfo *getCurCapturedRegion(); - /// Stack containing information about each of the nested - /// function, block, and method scopes that are currently active. - SmallVector FunctionScopes; + void ActOnComment(SourceRange Comment); - /// The index of the first FunctionScope that corresponds to the current - /// context. - unsigned FunctionScopesStart = 0; + /// Retrieve the parser's current scope. + /// + /// This routine must only be used when it is certain that semantic analysis + /// and the parser are in precisely the same context, which is not the case + /// when, e.g., we are performing any kind of template instantiation. + /// Therefore, the only safe places to use this scope are in the parser + /// itself and in routines directly invoked from the parser and *never* from + /// template substitution or instantiation. + Scope *getCurScope() const { return CurScope; } - /// Track the number of currently active capturing scopes. - unsigned CapturingFunctionScopes = 0; + IdentifierInfo *getSuperIdentifier() const; - ArrayRef getFunctionScopes() const { - return llvm::ArrayRef(FunctionScopes.begin() + FunctionScopesStart, - FunctionScopes.end()); + DeclContext *getCurLexicalContext() const { + return OriginalLexicalContext ? OriginalLexicalContext : CurContext; } - /// Stack containing information needed when in C++2a an 'auto' is encountered - /// in a function declaration parameter type specifier in order to invent a - /// corresponding template parameter in the enclosing abbreviated function - /// template. This information is also present in LambdaScopeInfo, stored in - /// the FunctionScopes stack. - SmallVector InventedParameterInfos; + SemaDiagnosticBuilder targetDiag(SourceLocation Loc, unsigned DiagID, + const FunctionDecl *FD = nullptr); + SemaDiagnosticBuilder targetDiag(SourceLocation Loc, + const PartialDiagnostic &PD, + const FunctionDecl *FD = nullptr) { + return targetDiag(Loc, PD.getDiagID(), FD) << PD; + } - /// The index of the first InventedParameterInfo that refers to the current - /// context. - unsigned InventedParameterInfosStart = 0; + /// Check if the type is allowed to be used for the current target. + void checkTypeSupport(QualType Ty, SourceLocation Loc, + ValueDecl *D = nullptr); - ArrayRef getInventedParameterInfos() const { - return llvm::ArrayRef(InventedParameterInfos.begin() + - InventedParameterInfosStart, - InventedParameterInfos.end()); - } + /// The kind of conversion being performed. + enum CheckedConversionKind { + /// An implicit conversion. + CCK_ImplicitConversion, + /// A C-style cast. + CCK_CStyleCast, + /// A functional-style cast. + CCK_FunctionalCast, + /// A cast other than a C-style cast. + CCK_OtherCast, + /// A conversion for an operand of a builtin overloaded operator. + CCK_ForBuiltinOverloadedOp + }; - typedef LazyVector - ExtVectorDeclsType; + /// ImpCastExprToType - If Expr is not of type 'Type', insert an implicit + /// cast. If there is already an implicit cast, merge into the existing one. + /// If isLvalue, the result of the cast is an lvalue. + ExprResult + ImpCastExprToType(Expr *E, QualType Type, CastKind CK, + ExprValueKind VK = VK_PRValue, + const CXXCastPath *BasePath = nullptr, + CheckedConversionKind CCK = CCK_ImplicitConversion); - /// ExtVectorDecls - This is a list all the extended vector types. This allows - /// us to associate a raw vector type with one of the ext_vector type names. - /// This is only necessary for issuing pretty diagnostics. - ExtVectorDeclsType ExtVectorDecls; + /// ScalarTypeToBooleanCastKind - Returns the cast kind corresponding + /// to the conversion from scalar type ScalarTy to the Boolean type. + static CastKind ScalarTypeToBooleanCastKind(QualType ScalarTy); - /// FieldCollector - Collects CXXFieldDecls during parsing of C++ classes. - std::unique_ptr FieldCollector; + /// If \p AllowLambda is true, treat lambda as function. + DeclContext *getFunctionLevelDeclContext(bool AllowLambda = false) const; - typedef llvm::SmallSetVector NamedDeclSetType; + /// Returns a pointer to the innermost enclosing function, or nullptr if the + /// current context is not inside a function. If \p AllowLambda is true, + /// this can return the call operator of an enclosing lambda, otherwise + /// lambdas are skipped when looking for an enclosing function. + FunctionDecl *getCurFunctionDecl(bool AllowLambda = false) const; - /// Set containing all declared private fields that are not used. - NamedDeclSetType UnusedPrivateFields; + /// getCurMethodDecl - If inside of a method body, this returns a pointer to + /// the method decl for the method being parsed. If we're currently + /// in a 'block', this returns the containing context. + ObjCMethodDecl *getCurMethodDecl(); - /// Set containing all typedefs that are likely unused. - llvm::SmallSetVector - UnusedLocalTypedefNameCandidates; + /// getCurFunctionOrMethodDecl - Return the Decl for the current ObjC method + /// or C function we're in, otherwise return null. If we're currently + /// in a 'block', this returns the containing context. + NamedDecl *getCurFunctionOrMethodDecl() const; - /// Delete-expressions to be analyzed at the end of translation unit + /// Warn if we're implicitly casting from a _Nullable pointer type to a + /// _Nonnull one. + void diagnoseNullableToNonnullConversion(QualType DstType, QualType SrcType, + SourceLocation Loc); + + /// Warn when implicitly casting 0 to nullptr. + void diagnoseZeroToNullptrConversion(CastKind Kind, const Expr *E); + + bool makeUnavailableInSystemHeader(SourceLocation loc, + UnavailableAttr::ImplicitReason reason); + + /// Retrieve a suitable printing policy for diagnostics. + PrintingPolicy getPrintingPolicy() const { + return getPrintingPolicy(Context, PP); + } + + /// Retrieve a suitable printing policy for diagnostics. + static PrintingPolicy getPrintingPolicy(const ASTContext &Ctx, + const Preprocessor &PP); + + /// Scope actions. + void ActOnTranslationUnitScope(Scope *S); + + /// Determine whether \param D is function like (function or function + /// template) for parsing. + bool isDeclaratorFunctionLike(Declarator &D); + + /// The maximum alignment, same as in llvm::Value. We duplicate them here + /// because that allows us not to duplicate the constants in clang code, + /// which we must to since we can't directly use the llvm constants. + /// The value is verified against llvm here: lib/CodeGen/CGDecl.cpp /// - /// This list contains class members, and locations of delete-expressions - /// that could not be proven as to whether they mismatch with new-expression - /// used in initializer of the field. - typedef std::pair DeleteExprLoc; - typedef llvm::SmallVector DeleteLocs; - llvm::MapVector DeleteExprs; + /// This is the greatest alignment value supported by load, store, and alloca + /// instructions, and global values. + static const unsigned MaxAlignmentExponent = 32; + static const uint64_t MaximumAlignment = 1ull << MaxAlignmentExponent; - typedef llvm::SmallPtrSet RecordDeclSetTy; + /// Flag indicating whether or not to collect detailed statistics. + bool CollectStats; - /// PureVirtualClassDiagSet - a set of class declarations which we have - /// emitted a list of pure virtual functions. Used to prevent emitting the - /// same list more than once. - std::unique_ptr PureVirtualClassDiagSet; + std::unique_ptr CachedFunctionScope; - /// ParsingInitForAutoVars - a set of declarations with auto types for which - /// we are currently parsing the initializer. - llvm::SmallPtrSet ParsingInitForAutoVars; + /// Stack containing information about each of the nested + /// function, block, and method scopes that are currently active. + SmallVector FunctionScopes; - /// Look for a locally scoped extern "C" declaration by the given name. - NamedDecl *findLocallyScopedExternCDecl(DeclarationName Name); + /// The index of the first FunctionScope that corresponds to the current + /// context. + unsigned FunctionScopesStart = 0; - typedef LazyVector - TentativeDefinitionsType; + /// Track the number of currently active capturing scopes. + unsigned CapturingFunctionScopes = 0; - /// All the tentative definitions encountered in the TU. - TentativeDefinitionsType TentativeDefinitions; + llvm::BumpPtrAllocator BumpAlloc; - /// All the external declarations encoutered and used in the TU. - SmallVector ExternalDeclarations; + /// The kind of translation unit we are processing. + /// + /// When we're processing a complete translation unit, Sema will perform + /// end-of-translation-unit semantic tasks (such as creating + /// initializers for tentative definitions in C) once parsing has + /// completed. Modules and precompiled headers perform different kinds of + /// checks. + const TranslationUnitKind TUKind; - typedef LazyVector - UnusedFileScopedDeclsType; + /// Translation Unit Scope - useful to Objective-C actions that need + /// to lookup file scope declarations in the "ordinary" C decl namespace. + /// For example, user-defined classes, built-in "id" type, etc. + Scope *TUScope; - /// The set of file scoped decls seen so far that have not been used - /// and must warn if not used. Only contains the first declaration. - UnusedFileScopedDeclsType UnusedFileScopedDecls; + bool WarnedStackExhausted = false; - typedef LazyVector - DelegatingCtorDeclsType; + void incrementMSManglingNumber() const { + return CurScope->incrementMSManglingNumber(); + } - /// All the delegating constructors seen so far in the file, used for - /// cycle detection at the end of the TU. - DelegatingCtorDeclsType DelegatingCtorDecls; + /// Try to recover by turning the given expression into a + /// call. Returns true if recovery was attempted or an error was + /// emitted; this may also leave the ExprResult invalid. + bool tryToRecoverWithCall(ExprResult &E, const PartialDiagnostic &PD, + bool ForceComplain = false, + bool (*IsPlausibleResult)(QualType) = nullptr); - /// All the overriding functions seen during a class definition - /// that had their exception spec checks delayed, plus the overridden - /// function. - SmallVector, 2> - DelayedOverridingExceptionSpecChecks; + /// Figure out if an expression could be turned into a call. + bool tryExprAsCall(Expr &E, QualType &ZeroArgCallReturnTy, + UnresolvedSetImpl &NonTemplateOverloads); - /// All the function redeclarations seen during a class definition that had - /// their exception spec checks delayed, plus the prior declaration they - /// should be checked against. Except during error recovery, the new decl - /// should always be a friend declaration, as that's the only valid way to - /// redeclare a special member before its class is complete. - SmallVector, 2> - DelayedEquivalentExceptionSpecChecks; + typedef OpaquePtr DeclGroupPtrTy; + typedef OpaquePtr TemplateTy; + typedef OpaquePtr TypeTy; - typedef llvm::MapVector> - LateParsedTemplateMapT; - LateParsedTemplateMapT LateParsedTemplateMap; + OpenCLOptions OpenCLFeatures; + FPOptions CurFPFeatures; + + const LangOptions &LangOpts; + Preprocessor &PP; + ASTContext &Context; + ASTConsumer &Consumer; + DiagnosticsEngine &Diags; + SourceManager &SourceMgr; + api_notes::APINotesManager APINotes; + + /// A RAII object to enter scope of a compound statement. + class CompoundScopeRAII { + public: + CompoundScopeRAII(Sema &S, bool IsStmtExpr = false) : S(S) { + S.ActOnStartOfCompoundStmt(IsStmtExpr); + } + + ~CompoundScopeRAII() { S.ActOnFinishOfCompoundStmt(); } + + private: + Sema &S; + }; + + /// An RAII helper that pops function a function scope on exit. + struct FunctionScopeRAII { + Sema &S; + bool Active; + FunctionScopeRAII(Sema &S) : S(S), Active(true) {} + ~FunctionScopeRAII() { + if (Active) + S.PopFunctionScopeInfo(); + } + void disable() { Active = false; } + }; + + /// Build a partial diagnostic. + PartialDiagnostic PDiag(unsigned DiagID = 0); // in SemaInternal.h + + sema::FunctionScopeInfo *getCurFunction() const { + return FunctionScopes.empty() ? nullptr : FunctionScopes.back(); + } + + /// Worker object for performing CFG-based warnings. + sema::AnalysisBasedWarnings AnalysisWarnings; + threadSafety::BeforeSet *ThreadSafetyDeclCache; /// Callback to the parser to parse templated functions when needed. typedef void LateTemplateParserCB(void *P, LateParsedTemplate &LPT); @@ -1016,8 +1077,7 @@ public: void *OpaqueParser; void SetLateTemplateParser(LateTemplateParserCB *LTP, - LateTemplateParserCleanupCB *LTPCleanup, - void *P) { + LateTemplateParserCleanupCB *LTPCleanup, void *P) { LateTemplateParser = LTP; LateTemplateParserCleanup = LTPCleanup; OpaqueParser = P; @@ -1027,6 +1087,14 @@ public: std::function ParseTypeFromStringCallback; + /// VAListTagName - The declaration name corresponding to __va_list_tag. + /// This is used as part of a hack to omit that class from ADL results. + DeclarationName VAListTagName; + + /// Is the last error level diagnostic immediate. This is used to determined + /// whether the next info diagnostic should be immediate. + bool IsLastErrorImmediate = true; + class DelayedDiagnostics; class DelayedDiagnosticsState { @@ -1053,9 +1121,7 @@ public: bool shouldDelayDiagnostics() { return CurPool != nullptr; } /// Returns the current delayed-diagnostics pool. - sema::DelayedDiagnosticPool *getCurrentPool() const { - return CurPool; - } + sema::DelayedDiagnosticPool *getCurrentPool() const { return CurPool; } /// Enter a new scope. Access and deprecation diagnostics will be /// collected in this pool. @@ -1089,1657 +1155,1626 @@ public: } } DelayedDiagnostics; - enum CUDAFunctionTarget { - CFT_Device, - CFT_Global, - CFT_Host, - CFT_HostDevice, - CFT_InvalidTarget - }; + ParsingDeclState PushParsingDeclaration(sema::DelayedDiagnosticPool &pool) { + return DelayedDiagnostics.push(pool); + } - /// A RAII object to temporarily push a declaration context. - class ContextRAII { - private: - Sema &S; - DeclContext *SavedContext; - ProcessingContextState SavedContextState; - QualType SavedCXXThisTypeOverride; - unsigned SavedFunctionScopesStart; - unsigned SavedInventedParameterInfosStart; + /// CurContext - This is the current declaration context of parsing. + DeclContext *CurContext; - public: - ContextRAII(Sema &S, DeclContext *ContextToPush, bool NewThisContext = true) - : S(S), SavedContext(S.CurContext), - SavedContextState(S.DelayedDiagnostics.pushUndelayed()), - SavedCXXThisTypeOverride(S.CXXThisTypeOverride), - SavedFunctionScopesStart(S.FunctionScopesStart), - SavedInventedParameterInfosStart(S.InventedParameterInfosStart) - { - assert(ContextToPush && "pushing null context"); - S.CurContext = ContextToPush; - if (NewThisContext) - S.CXXThisTypeOverride = QualType(); - // Any saved FunctionScopes do not refer to this context. - S.FunctionScopesStart = S.FunctionScopes.size(); - S.InventedParameterInfosStart = S.InventedParameterInfos.size(); - } +protected: + friend class Parser; + friend class InitializationSequence; + friend class ASTReader; + friend class ASTDeclReader; + friend class ASTWriter; - void pop() { - if (!SavedContext) return; - S.CurContext = SavedContext; - S.DelayedDiagnostics.popUndelayed(SavedContextState); - S.CXXThisTypeOverride = SavedCXXThisTypeOverride; - S.FunctionScopesStart = SavedFunctionScopesStart; - S.InventedParameterInfosStart = SavedInventedParameterInfosStart; - SavedContext = nullptr; - } +private: + std::optional> CachedDarwinSDKInfo; + bool WarnedDarwinSDKInfoMissing = false; - ~ContextRAII() { - pop(); - } - }; + Sema(const Sema &) = delete; + void operator=(const Sema &) = delete; - /// RAII object to handle the state changes required to synthesize - /// a function body. - class SynthesizedFunctionScope { - Sema &S; - Sema::ContextRAII SavedContext; - bool PushedCodeSynthesisContext = false; + /// Source of additional semantic information. + IntrusiveRefCntPtr ExternalSource; - public: - SynthesizedFunctionScope(Sema &S, DeclContext *DC) - : S(S), SavedContext(S, DC) { - auto *FD = dyn_cast(DC); - S.PushFunctionScope(); - S.PushExpressionEvaluationContext( - (FD && FD->isConsteval()) - ? ExpressionEvaluationContext::ImmediateFunctionContext - : ExpressionEvaluationContext::PotentiallyEvaluated); - if (FD) { - FD->setWillHaveBody(true); - S.ExprEvalContexts.back().InImmediateFunctionContext = - FD->isImmediateFunction() || - S.ExprEvalContexts[S.ExprEvalContexts.size() - 2] - .isConstantEvaluated(); - S.ExprEvalContexts.back().InImmediateEscalatingFunctionContext = - S.getLangOpts().CPlusPlus20 && FD->isImmediateEscalating(); - } else - assert(isa(DC)); - } + /// The handler for the FileChanged preprocessor events. + /// + /// Used for diagnostics that implement custom semantic analysis for #include + /// directives, like -Wpragma-pack. + sema::SemaPPCallbacks *SemaPPCallbackHandler; - void addContextNote(SourceLocation UseLoc) { - assert(!PushedCodeSynthesisContext); + /// The parser's current scope. + /// + /// The parser maintains this state here. + Scope *CurScope; - Sema::CodeSynthesisContext Ctx; - Ctx.Kind = Sema::CodeSynthesisContext::DefiningSynthesizedFunction; - Ctx.PointOfInstantiation = UseLoc; - Ctx.Entity = cast(S.CurContext); - S.pushCodeSynthesisContext(Ctx); + mutable IdentifierInfo *Ident_super; - PushedCodeSynthesisContext = true; - } + ///@} - ~SynthesizedFunctionScope() { - if (PushedCodeSynthesisContext) - S.popCodeSynthesisContext(); - if (auto *FD = dyn_cast(S.CurContext)) { - FD->setWillHaveBody(false); - S.CheckImmediateEscalatingFunctionDefinition(FD, S.getCurFunction()); - } - S.PopExpressionEvaluationContext(); - S.PopFunctionScopeInfo(); - } + // + // + // ------------------------------------------------------------------------- + // + // + + /// \name C++ Access Control + /// Implementations are in SemaAccess.cpp + ///@{ + +public: + enum AccessResult { + AR_accessible, + AR_inaccessible, + AR_dependent, + AR_delayed }; - /// WeakUndeclaredIdentifiers - Identifiers contained in \#pragma weak before - /// declared. Rare. May alias another identifier, declared or undeclared. - /// - /// For aliases, the target identifier is used as a key for eventual - /// processing when the target is declared. For the single-identifier form, - /// the sole identifier is used as the key. Each entry is a `SetVector` - /// (ordered by parse order) of aliases (identified by the alias name) in case - /// of multiple aliases to the same undeclared identifier. - llvm::MapVector< - IdentifierInfo *, - llvm::SetVector< - WeakInfo, llvm::SmallVector, - llvm::SmallDenseSet>> - WeakUndeclaredIdentifiers; + bool SetMemberAccessSpecifier(NamedDecl *MemberDecl, + NamedDecl *PrevMemberDecl, + AccessSpecifier LexicalAS); - /// ExtnameUndeclaredIdentifiers - Identifiers contained in - /// \#pragma redefine_extname before declared. Used in Solaris system headers - /// to define functions that occur in multiple standards to call the version - /// in the currently selected standard. - llvm::DenseMap ExtnameUndeclaredIdentifiers; + AccessResult CheckUnresolvedMemberAccess(UnresolvedMemberExpr *E, + DeclAccessPair FoundDecl); + AccessResult CheckUnresolvedLookupAccess(UnresolvedLookupExpr *E, + DeclAccessPair FoundDecl); + AccessResult CheckAllocationAccess(SourceLocation OperatorLoc, + SourceRange PlacementRange, + CXXRecordDecl *NamingClass, + DeclAccessPair FoundDecl, + bool Diagnose = true); + AccessResult CheckConstructorAccess(SourceLocation Loc, CXXConstructorDecl *D, + DeclAccessPair FoundDecl, + const InitializedEntity &Entity, + bool IsCopyBindingRefToTemp = false); + AccessResult CheckConstructorAccess(SourceLocation Loc, CXXConstructorDecl *D, + DeclAccessPair FoundDecl, + const InitializedEntity &Entity, + const PartialDiagnostic &PDiag); + AccessResult CheckDestructorAccess(SourceLocation Loc, + CXXDestructorDecl *Dtor, + const PartialDiagnostic &PDiag, + QualType objectType = QualType()); + AccessResult CheckFriendAccess(NamedDecl *D); + AccessResult CheckMemberAccess(SourceLocation UseLoc, + CXXRecordDecl *NamingClass, + DeclAccessPair Found); + AccessResult + CheckStructuredBindingMemberAccess(SourceLocation UseLoc, + CXXRecordDecl *DecomposedClass, + DeclAccessPair Field); + AccessResult CheckMemberOperatorAccess(SourceLocation Loc, Expr *ObjectExpr, + const SourceRange &, + DeclAccessPair FoundDecl); + AccessResult CheckMemberOperatorAccess(SourceLocation Loc, Expr *ObjectExpr, + Expr *ArgExpr, + DeclAccessPair FoundDecl); + AccessResult CheckMemberOperatorAccess(SourceLocation Loc, Expr *ObjectExpr, + ArrayRef ArgExprs, + DeclAccessPair FoundDecl); + AccessResult CheckAddressOfMemberAccess(Expr *OvlExpr, + DeclAccessPair FoundDecl); + AccessResult CheckBaseClassAccess(SourceLocation AccessLoc, QualType Base, + QualType Derived, const CXXBasePath &Path, + unsigned DiagID, bool ForceCheck = false, + bool ForceUnprivileged = false); + void CheckLookupAccess(const LookupResult &R); + bool IsSimplyAccessible(NamedDecl *Decl, CXXRecordDecl *NamingClass, + QualType BaseType); + bool isMemberAccessibleForDeletion(CXXRecordDecl *NamingClass, + DeclAccessPair Found, QualType ObjectType, + SourceLocation Loc, + const PartialDiagnostic &Diag); + bool isMemberAccessibleForDeletion(CXXRecordDecl *NamingClass, + DeclAccessPair Found, + QualType ObjectType) { + return isMemberAccessibleForDeletion(NamingClass, Found, ObjectType, + SourceLocation(), PDiag()); + } + void HandleDependentAccessCheck( + const DependentDiagnostic &DD, + const MultiLevelTemplateArgumentList &TemplateArgs); + void HandleDelayedAccessCheck(sema::DelayedDiagnostic &DD, Decl *Ctx); - /// Load weak undeclared identifiers from the external source. - void LoadExternalWeakUndeclaredIdentifiers(); + ///@} - /// WeakTopLevelDecl - Translation-unit scoped declarations generated by - /// \#pragma weak during processing of other Decls. - /// I couldn't figure out a clean way to generate these in-line, so - /// we store them here and handle separately -- which is a hack. - /// It would be best to refactor this. - SmallVector WeakTopLevelDecl; + // + // + // ------------------------------------------------------------------------- + // + // - IdentifierResolver IdResolver; + /// \name Attributes + /// Implementations are in SemaAttr.cpp + ///@{ - /// Translation Unit Scope - useful to Objective-C actions that need - /// to lookup file scope declarations in the "ordinary" C decl namespace. - /// For example, user-defined classes, built-in "id" type, etc. - Scope *TUScope; +public: + /// Controls member pointer representation format under the MS ABI. + LangOptions::PragmaMSPointersToMembersKind + MSPointerToMemberRepresentationMethod; - /// The C++ "std" namespace, where the standard library resides. - LazyDeclPtr StdNamespace; + bool MSStructPragmaOn; // True when \#pragma ms_struct on - /// The C++ "std::bad_alloc" class, which is defined by the C++ - /// standard library. - LazyDeclPtr StdBadAlloc; + /// Source location for newly created implicit MSInheritanceAttrs + SourceLocation ImplicitMSInheritanceAttrLoc; - /// The C++ "std::align_val_t" enum class, which is defined by the C++ - /// standard library. - LazyDeclPtr StdAlignValT; + /// pragma clang section kind + enum PragmaClangSectionKind { + PCSK_Invalid = 0, + PCSK_BSS = 1, + PCSK_Data = 2, + PCSK_Rodata = 3, + PCSK_Text = 4, + PCSK_Relro = 5 + }; - /// The C++ "std::initializer_list" template, which is defined in - /// \. - ClassTemplateDecl *StdInitializerList; + enum PragmaClangSectionAction { PCSA_Set = 0, PCSA_Clear = 1 }; - /// The C++ "std::coroutine_traits" template, which is defined in - /// \ - ClassTemplateDecl *StdCoroutineTraitsCache; + struct PragmaClangSection { + std::string SectionName; + bool Valid = false; + SourceLocation PragmaLocation; + }; - /// The C++ "type_info" declaration, which is defined in \. - RecordDecl *CXXTypeInfoDecl; + PragmaClangSection PragmaClangBSSSection; + PragmaClangSection PragmaClangDataSection; + PragmaClangSection PragmaClangRodataSection; + PragmaClangSection PragmaClangRelroSection; + PragmaClangSection PragmaClangTextSection; - /// The C++ "std::source_location::__impl" struct, defined in - /// \. - RecordDecl *StdSourceLocationImplDecl; + enum PragmaMsStackAction { + PSK_Reset = 0x0, // #pragma () + PSK_Set = 0x1, // #pragma (value) + PSK_Push = 0x2, // #pragma (push[, id]) + PSK_Pop = 0x4, // #pragma (pop[, id]) + PSK_Show = 0x8, // #pragma (show) -- only for "pack"! + PSK_Push_Set = PSK_Push | PSK_Set, // #pragma (push[, id], value) + PSK_Pop_Set = PSK_Pop | PSK_Set, // #pragma (pop[, id], value) + }; - /// Caches identifiers/selectors for NSFoundation APIs. - std::unique_ptr NSAPIObj; + struct PragmaPackInfo { + PragmaMsStackAction Action; + StringRef SlotLabel; + Token Alignment; + }; - /// The declaration of the Objective-C NSNumber class. - ObjCInterfaceDecl *NSNumberDecl; + // #pragma pack and align. + class AlignPackInfo { + public: + // `Native` represents default align mode, which may vary based on the + // platform. + enum Mode : unsigned char { Native, Natural, Packed, Mac68k }; - /// The declaration of the Objective-C NSValue class. - ObjCInterfaceDecl *NSValueDecl; + // #pragma pack info constructor + AlignPackInfo(AlignPackInfo::Mode M, unsigned Num, bool IsXL) + : PackAttr(true), AlignMode(M), PackNumber(Num), XLStack(IsXL) { + assert(Num == PackNumber && "The pack number has been truncated."); + } - /// Pointer to NSNumber type (NSNumber *). - QualType NSNumberPointer; + // #pragma align info constructor + AlignPackInfo(AlignPackInfo::Mode M, bool IsXL) + : PackAttr(false), AlignMode(M), + PackNumber(M == Packed ? 1 : UninitPackVal), XLStack(IsXL) {} - /// Pointer to NSValue type (NSValue *). - QualType NSValuePointer; + explicit AlignPackInfo(bool IsXL) : AlignPackInfo(Native, IsXL) {} - /// The Objective-C NSNumber methods used to create NSNumber literals. - ObjCMethodDecl *NSNumberLiteralMethods[NSAPI::NumNSNumberLiteralMethods]; + AlignPackInfo() : AlignPackInfo(Native, false) {} - /// The declaration of the Objective-C NSString class. - ObjCInterfaceDecl *NSStringDecl; + // When a AlignPackInfo itself cannot be used, this returns an 32-bit + // integer encoding for it. This should only be passed to + // AlignPackInfo::getFromRawEncoding, it should not be inspected directly. + static uint32_t getRawEncoding(const AlignPackInfo &Info) { + std::uint32_t Encoding{}; + if (Info.IsXLStack()) + Encoding |= IsXLMask; - /// Pointer to NSString type (NSString *). - QualType NSStringPointer; + Encoding |= static_cast(Info.getAlignMode()) << 1; - /// The declaration of the stringWithUTF8String: method. - ObjCMethodDecl *StringWithUTF8StringMethod; + if (Info.IsPackAttr()) + Encoding |= PackAttrMask; - /// The declaration of the valueWithBytes:objCType: method. - ObjCMethodDecl *ValueWithBytesObjCTypeMethod; + Encoding |= static_cast(Info.getPackNumber()) << 4; - /// The declaration of the Objective-C NSArray class. - ObjCInterfaceDecl *NSArrayDecl; + return Encoding; + } - /// The declaration of the arrayWithObjects:count: method. - ObjCMethodDecl *ArrayWithObjectsMethod; + static AlignPackInfo getFromRawEncoding(unsigned Encoding) { + bool IsXL = static_cast(Encoding & IsXLMask); + AlignPackInfo::Mode M = + static_cast((Encoding & AlignModeMask) >> 1); + int PackNumber = (Encoding & PackNumMask) >> 4; - /// The declaration of the Objective-C NSDictionary class. - ObjCInterfaceDecl *NSDictionaryDecl; + if (Encoding & PackAttrMask) + return AlignPackInfo(M, PackNumber, IsXL); - /// The declaration of the dictionaryWithObjects:forKeys:count: method. - ObjCMethodDecl *DictionaryWithObjectsMethod; + return AlignPackInfo(M, IsXL); + } - /// id type. - QualType QIDNSCopying; + bool IsPackAttr() const { return PackAttr; } - /// will hold 'respondsToSelector:' - Selector RespondsToSelectorSel; + bool IsAlignAttr() const { return !PackAttr; } - /// A flag to remember whether the implicit forms of operator new and delete - /// have been declared. - bool GlobalNewDeleteDeclared; + Mode getAlignMode() const { return AlignMode; } - /// Describes how the expressions currently being parsed are - /// evaluated at run-time, if at all. - enum class ExpressionEvaluationContext { - /// The current expression and its subexpressions occur within an - /// unevaluated operand (C++11 [expr]p7), such as the subexpression of - /// \c sizeof, where the type of the expression may be significant but - /// no code will be generated to evaluate the value of the expression at - /// run time. - Unevaluated, + unsigned getPackNumber() const { return PackNumber; } - /// The current expression occurs within a braced-init-list within - /// an unevaluated operand. This is mostly like a regular unevaluated - /// context, except that we still instantiate constexpr functions that are - /// referenced here so that we can perform narrowing checks correctly. - UnevaluatedList, - - /// The current expression occurs within a discarded statement. - /// This behaves largely similarly to an unevaluated operand in preventing - /// definitions from being required, but not in other ways. - DiscardedStatement, + bool IsPackSet() const { + // #pragma align, #pragma pack(), and #pragma pack(0) do not set the pack + // attriute on a decl. + return PackNumber != UninitPackVal && PackNumber != 0; + } - /// The current expression occurs within an unevaluated - /// operand that unconditionally permits abstract references to - /// fields, such as a SIZE operator in MS-style inline assembly. - UnevaluatedAbstract, + bool IsXLStack() const { return XLStack; } - /// The current context is "potentially evaluated" in C++11 terms, - /// but the expression is evaluated at compile-time (like the values of - /// cases in a switch statement). - ConstantEvaluated, + bool operator==(const AlignPackInfo &Info) const { + return std::tie(AlignMode, PackNumber, PackAttr, XLStack) == + std::tie(Info.AlignMode, Info.PackNumber, Info.PackAttr, + Info.XLStack); + } - /// In addition of being constant evaluated, the current expression - /// occurs in an immediate function context - either a consteval function - /// or a consteval if statement. - ImmediateFunctionContext, + bool operator!=(const AlignPackInfo &Info) const { + return !(*this == Info); + } - /// The current expression is potentially evaluated at run time, - /// which means that code may be generated to evaluate the value of the - /// expression at run time. - PotentiallyEvaluated, + private: + /// \brief True if this is a pragma pack attribute, + /// not a pragma align attribute. + bool PackAttr; - /// The current expression is potentially evaluated, but any - /// declarations referenced inside that expression are only used if - /// in fact the current expression is used. - /// - /// This value is used when parsing default function arguments, for which - /// we would like to provide diagnostics (e.g., passing non-POD arguments - /// through varargs) but do not want to mark declarations as "referenced" - /// until the default argument is used. - PotentiallyEvaluatedIfUsed - }; + /// \brief The alignment mode that is in effect. + Mode AlignMode; - using ImmediateInvocationCandidate = llvm::PointerIntPair; + /// \brief The pack number of the stack. + unsigned char PackNumber; - /// Data structure used to record current or nested - /// expression evaluation contexts. - struct ExpressionEvaluationContextRecord { - /// The expression evaluation context. - ExpressionEvaluationContext Context; + /// \brief True if it is a XL #pragma align/pack stack. + bool XLStack; - /// Whether the enclosing context needed a cleanup. - CleanupInfo ParentCleanup; + /// \brief Uninitialized pack value. + static constexpr unsigned char UninitPackVal = -1; - /// The number of active cleanup objects when we entered - /// this expression evaluation context. - unsigned NumCleanupObjects; + // Masks to encode and decode an AlignPackInfo. + static constexpr uint32_t IsXLMask{0x0000'0001}; + static constexpr uint32_t AlignModeMask{0x0000'0006}; + static constexpr uint32_t PackAttrMask{0x00000'0008}; + static constexpr uint32_t PackNumMask{0x0000'01F0}; + }; - /// The number of typos encountered during this expression evaluation - /// context (i.e. the number of TypoExprs created). - unsigned NumTypos; + template struct PragmaStack { + struct Slot { + llvm::StringRef StackSlotLabel; + ValueType Value; + SourceLocation PragmaLocation; + SourceLocation PragmaPushLocation; + Slot(llvm::StringRef StackSlotLabel, ValueType Value, + SourceLocation PragmaLocation, SourceLocation PragmaPushLocation) + : StackSlotLabel(StackSlotLabel), Value(Value), + PragmaLocation(PragmaLocation), + PragmaPushLocation(PragmaPushLocation) {} + }; - MaybeODRUseExprSet SavedMaybeODRUseExprs; + void Act(SourceLocation PragmaLocation, PragmaMsStackAction Action, + llvm::StringRef StackSlotLabel, ValueType Value) { + if (Action == PSK_Reset) { + CurrentValue = DefaultValue; + CurrentPragmaLocation = PragmaLocation; + return; + } + if (Action & PSK_Push) + Stack.emplace_back(StackSlotLabel, CurrentValue, CurrentPragmaLocation, + PragmaLocation); + else if (Action & PSK_Pop) { + if (!StackSlotLabel.empty()) { + // If we've got a label, try to find it and jump there. + auto I = llvm::find_if(llvm::reverse(Stack), [&](const Slot &x) { + return x.StackSlotLabel == StackSlotLabel; + }); + // If we found the label so pop from there. + if (I != Stack.rend()) { + CurrentValue = I->Value; + CurrentPragmaLocation = I->PragmaLocation; + Stack.erase(std::prev(I.base()), Stack.end()); + } + } else if (!Stack.empty()) { + // We do not have a label, just pop the last entry. + CurrentValue = Stack.back().Value; + CurrentPragmaLocation = Stack.back().PragmaLocation; + Stack.pop_back(); + } + } + if (Action & PSK_Set) { + CurrentValue = Value; + CurrentPragmaLocation = PragmaLocation; + } + } - /// The lambdas that are present within this context, if it - /// is indeed an unevaluated context. - SmallVector Lambdas; + // MSVC seems to add artificial slots to #pragma stacks on entering a C++ + // method body to restore the stacks on exit, so it works like this: + // + // struct S { + // #pragma (push, InternalPragmaSlot, ) + // void Method {} + // #pragma (pop, InternalPragmaSlot) + // }; + // + // It works even with #pragma vtordisp, although MSVC doesn't support + // #pragma vtordisp(push [, id], n) + // syntax. + // + // Push / pop a named sentinel slot. + void SentinelAction(PragmaMsStackAction Action, StringRef Label) { + assert((Action == PSK_Push || Action == PSK_Pop) && + "Can only push / pop #pragma stack sentinels!"); + Act(CurrentPragmaLocation, Action, Label, CurrentValue); + } - /// The declaration that provides context for lambda expressions - /// and block literals if the normal declaration context does not - /// suffice, e.g., in a default function argument. - Decl *ManglingContextDecl; + // Constructors. + explicit PragmaStack(const ValueType &Default) + : DefaultValue(Default), CurrentValue(Default) {} - /// If we are processing a decltype type, a set of call expressions - /// for which we have deferred checking the completeness of the return type. - SmallVector DelayedDecltypeCalls; + bool hasValue() const { return CurrentValue != DefaultValue; } - /// If we are processing a decltype type, a set of temporary binding - /// expressions for which we have deferred checking the destructor. - SmallVector DelayedDecltypeBinds; + SmallVector Stack; + ValueType DefaultValue; // Value used for PSK_Reset action. + ValueType CurrentValue; + SourceLocation CurrentPragmaLocation; + }; + // FIXME: We should serialize / deserialize these if they occur in a PCH (but + // we shouldn't do so if they're in a module). - llvm::SmallPtrSet PossibleDerefs; + /// Whether to insert vtordisps prior to virtual bases in the Microsoft + /// C++ ABI. Possible values are 0, 1, and 2, which mean: + /// + /// 0: Suppress all vtordisps + /// 1: Insert vtordisps in the presence of vbase overrides and non-trivial + /// structors + /// 2: Always insert vtordisps to support RTTI on partially constructed + /// objects + PragmaStack VtorDispStack; + PragmaStack AlignPackStack; + // The current #pragma align/pack values and locations at each #include. + struct AlignPackIncludeState { + AlignPackInfo CurrentValue; + SourceLocation CurrentPragmaLocation; + bool HasNonDefaultValue, ShouldWarnOnInclude; + }; + SmallVector AlignPackIncludeStack; + // Segment #pragmas. + PragmaStack DataSegStack; + PragmaStack BSSSegStack; + PragmaStack ConstSegStack; + PragmaStack CodeSegStack; - /// Expressions appearing as the LHS of a volatile assignment in this - /// context. We produce a warning for these when popping the context if - /// they are not discarded-value expressions nor unevaluated operands. - SmallVector VolatileAssignmentLHSs; + // #pragma strict_gs_check. + PragmaStack StrictGuardStackCheckStack; - /// Set of candidates for starting an immediate invocation. - llvm::SmallVector ImmediateInvocationCandidates; + // This stack tracks the current state of Sema.CurFPFeatures. + PragmaStack FpPragmaStack; + FPOptionsOverride CurFPFeatureOverrides() { + FPOptionsOverride result; + if (!FpPragmaStack.hasValue()) { + result = FPOptionsOverride(); + } else { + result = FpPragmaStack.CurrentValue; + } + return result; + } - /// Set of DeclRefExprs referencing a consteval function when used in a - /// context not already known to be immediately invoked. - llvm::SmallPtrSet ReferenceToConsteval; + enum PragmaSectionKind { + PSK_DataSeg, + PSK_BSSSeg, + PSK_ConstSeg, + PSK_CodeSeg, + }; - /// P2718R0 - Lifetime extension in range-based for loops. - /// MaterializeTemporaryExprs in for-range-init expressions which need to - /// extend lifetime. Add MaterializeTemporaryExpr* if the value of - /// InLifetimeExtendingContext is true. - SmallVector ForRangeLifetimeExtendTemps; + // RAII object to push / pop sentinel slots for all MS #pragma stacks. + // Actions should be performed only if we enter / exit a C++ method body. + class PragmaStackSentinelRAII { + public: + PragmaStackSentinelRAII(Sema &S, StringRef SlotLabel, bool ShouldAct); + ~PragmaStackSentinelRAII(); - /// \brief Describes whether we are in an expression constext which we have - /// to handle differently. - enum ExpressionKind { - EK_Decltype, EK_TemplateArgument, EK_Other - } ExprContext; + private: + Sema &S; + StringRef SlotLabel; + bool ShouldAct; + }; - // A context can be nested in both a discarded statement context and - // an immediate function context, so they need to be tracked independently. - bool InDiscardedStatement; - bool InImmediateFunctionContext; - bool InImmediateEscalatingFunctionContext; + /// Last section used with #pragma init_seg. + StringLiteral *CurInitSeg; + SourceLocation CurInitSegLoc; - bool IsCurrentlyCheckingDefaultArgumentOrInitializer = false; + /// Sections used with #pragma alloc_text. + llvm::StringMap> FunctionToSectionMap; - // We are in a constant context, but we also allow - // non constant expressions, for example for array bounds (which may be - // VLAs). - bool InConditionallyConstantEvaluateContext = false; + /// VisContext - Manages the stack for \#pragma GCC visibility. + void *VisContext; // Really a "PragmaVisStack*" - /// Whether we are currently in a context in which all temporaries must be - /// lifetime-extended, even if they're not bound to a reference (for - /// example, in a for-range initializer). - bool InLifetimeExtendingContext = false; + /// This an attribute introduced by \#pragma clang attribute. + struct PragmaAttributeEntry { + SourceLocation Loc; + ParsedAttr *Attribute; + SmallVector MatchRules; + bool IsUsed; + }; - /// Whether we are currently in a context in which all temporaries must be - /// materialized. - /// - /// [class.temporary]/p2: - /// The materialization of a temporary object is generally delayed as long - /// as possible in order to avoid creating unnecessary temporary objects. - /// - /// Temporary objects are materialized: - /// (2.1) when binding a reference to a prvalue ([dcl.init.ref], - /// [expr.type.conv], [expr.dynamic.cast], [expr.static.cast], - /// [expr.const.cast], [expr.cast]), - /// - /// (2.2) when performing member access on a class prvalue ([expr.ref], - /// [expr.mptr.oper]), - /// - /// (2.3) when performing an array-to-pointer conversion or subscripting - /// on an array prvalue ([conv.array], [expr.sub]), - /// - /// (2.4) when initializing an object of type - /// std​::​initializer_list from a braced-init-list - /// ([dcl.init.list]), - /// - /// (2.5) for certain unevaluated operands ([expr.typeid], [expr.sizeof]) - /// - /// (2.6) when a prvalue that has type other than cv void appears as a - /// discarded-value expression ([expr.context]). - bool InMaterializeTemporaryObjectContext = false; + /// A push'd group of PragmaAttributeEntries. + struct PragmaAttributeGroup { + /// The location of the push attribute. + SourceLocation Loc; + /// The namespace of this push group. + const IdentifierInfo *Namespace; + SmallVector Entries; + }; - // When evaluating immediate functions in the initializer of a default - // argument or default member initializer, this is the declaration whose - // default initializer is being evaluated and the location of the call - // or constructor definition. - struct InitializationContext { - InitializationContext(SourceLocation Loc, ValueDecl *Decl, - DeclContext *Context) - : Loc(Loc), Decl(Decl), Context(Context) { - assert(Decl && Context && "invalid initialization context"); - } + SmallVector PragmaAttributeStack; - SourceLocation Loc; - ValueDecl *Decl = nullptr; - DeclContext *Context = nullptr; - }; - std::optional DelayedDefaultInitializationContext; + /// The declaration that is currently receiving an attribute from the + /// #pragma attribute stack. + const Decl *PragmaAttributeCurrentTargetDecl; - ExpressionEvaluationContextRecord(ExpressionEvaluationContext Context, - unsigned NumCleanupObjects, - CleanupInfo ParentCleanup, - Decl *ManglingContextDecl, - ExpressionKind ExprContext) - : Context(Context), ParentCleanup(ParentCleanup), - NumCleanupObjects(NumCleanupObjects), NumTypos(0), - ManglingContextDecl(ManglingContextDecl), ExprContext(ExprContext), - InDiscardedStatement(false), InImmediateFunctionContext(false), - InImmediateEscalatingFunctionContext(false) {} + /// This represents the last location of a "#pragma clang optimize off" + /// directive if such a directive has not been closed by an "on" yet. If + /// optimizations are currently "on", this is set to an invalid location. + SourceLocation OptimizeOffPragmaLocation; - bool isUnevaluated() const { - return Context == ExpressionEvaluationContext::Unevaluated || - Context == ExpressionEvaluationContext::UnevaluatedAbstract || - Context == ExpressionEvaluationContext::UnevaluatedList; - } + /// Get the location for the currently active "\#pragma clang optimize + /// off". If this location is invalid, then the state of the pragma is "on". + SourceLocation getOptimizeOffPragmaLocation() const { + return OptimizeOffPragmaLocation; + } - bool isConstantEvaluated() const { - return Context == ExpressionEvaluationContext::ConstantEvaluated || - Context == ExpressionEvaluationContext::ImmediateFunctionContext; - } + /// The "on" or "off" argument passed by \#pragma optimize, that denotes + /// whether the optimizations in the list passed to the pragma should be + /// turned off or on. This boolean is true by default because command line + /// options are honored when `#pragma optimize("", on)`. + /// (i.e. `ModifyFnAttributeMSPragmaOptimze()` does nothing) + bool MSPragmaOptimizeIsOn = true; - bool isImmediateFunctionContext() const { - return Context == ExpressionEvaluationContext::ImmediateFunctionContext || - (Context == ExpressionEvaluationContext::DiscardedStatement && - InImmediateFunctionContext) || - // C++23 [expr.const]p14: - // An expression or conversion is in an immediate function - // context if it is potentially evaluated and either: - // * its innermost enclosing non-block scope is a function - // parameter scope of an immediate function, or - // * its enclosing statement is enclosed by the compound- - // statement of a consteval if statement. - (Context == ExpressionEvaluationContext::PotentiallyEvaluated && - InImmediateFunctionContext); - } + /// Set of no-builtin functions listed by \#pragma function. + llvm::SmallSetVector MSFunctionNoBuiltins; - bool isDiscardedStatementContext() const { - return Context == ExpressionEvaluationContext::DiscardedStatement || - (Context == - ExpressionEvaluationContext::ImmediateFunctionContext && - InDiscardedStatement); - } - }; + /// AddAlignmentAttributesForRecord - Adds any needed alignment attributes to + /// a the record decl, to handle '\#pragma pack' and '\#pragma options align'. + void AddAlignmentAttributesForRecord(RecordDecl *RD); - /// A stack of expression evaluation contexts. - SmallVector ExprEvalContexts; + /// AddMsStructLayoutForRecord - Adds ms_struct layout attribute to record. + void AddMsStructLayoutForRecord(RecordDecl *RD); - // Set of failed immediate invocations to avoid double diagnosing. - llvm::SmallPtrSet FailedImmediateInvocations; + /// Add gsl::Pointer attribute to std::container::iterator + /// \param ND The declaration that introduces the name + /// std::container::iterator. \param UnderlyingRecord The record named by ND. + void inferGslPointerAttribute(NamedDecl *ND, CXXRecordDecl *UnderlyingRecord); - /// Emit a warning for all pending noderef expressions that we recorded. - void WarnOnPendingNoDerefs(ExpressionEvaluationContextRecord &Rec); + /// Add [[gsl::Owner]] and [[gsl::Pointer]] attributes for std:: types. + void inferGslOwnerPointerAttribute(CXXRecordDecl *Record); - /// Compute the mangling number context for a lambda expression or - /// block literal. Also return the extra mangling decl if any. - /// - /// \param DC - The DeclContext containing the lambda expression or - /// block literal. - std::tuple - getCurrentMangleNumberContext(const DeclContext *DC); + /// Add [[gsl::Pointer]] attributes for std:: types. + void inferGslPointerAttribute(TypedefNameDecl *TD); + enum PragmaOptionsAlignKind { + POAK_Native, // #pragma options align=native + POAK_Natural, // #pragma options align=natural + POAK_Packed, // #pragma options align=packed + POAK_Power, // #pragma options align=power + POAK_Mac68k, // #pragma options align=mac68k + POAK_Reset // #pragma options align=reset + }; - /// SpecialMemberOverloadResult - The overloading result for a special member - /// function. - /// - /// This is basically a wrapper around PointerIntPair. The lowest bits of the - /// integer are used to determine whether overload resolution succeeded. - class SpecialMemberOverloadResult { - public: - enum Kind { - NoMemberOrDeleted, - Ambiguous, - Success - }; + /// ActOnPragmaClangSection - Called on well formed \#pragma clang section + void ActOnPragmaClangSection(SourceLocation PragmaLoc, + PragmaClangSectionAction Action, + PragmaClangSectionKind SecKind, + StringRef SecName); - private: - llvm::PointerIntPair Pair; + /// ActOnPragmaOptionsAlign - Called on well formed \#pragma options align. + void ActOnPragmaOptionsAlign(PragmaOptionsAlignKind Kind, + SourceLocation PragmaLoc); - public: - SpecialMemberOverloadResult() {} - SpecialMemberOverloadResult(CXXMethodDecl *MD) - : Pair(MD, MD->isDeleted() ? NoMemberOrDeleted : Success) {} + /// ActOnPragmaPack - Called on well formed \#pragma pack(...). + void ActOnPragmaPack(SourceLocation PragmaLoc, PragmaMsStackAction Action, + StringRef SlotLabel, Expr *Alignment); - CXXMethodDecl *getMethod() const { return Pair.getPointer(); } - void setMethod(CXXMethodDecl *MD) { Pair.setPointer(MD); } + /// ConstantFoldAttrArgs - Folds attribute arguments into ConstantExprs + /// (unless they are value dependent or type dependent). Returns false + /// and emits a diagnostic if one or more of the arguments could not be + /// folded into a constant. + bool ConstantFoldAttrArgs(const AttributeCommonInfo &CI, + MutableArrayRef Args); - Kind getKind() const { return static_cast(Pair.getInt()); } - void setKind(Kind K) { Pair.setInt(K); } + enum class PragmaAlignPackDiagnoseKind { + NonDefaultStateAtInclude, + ChangedStateAtExit }; - class SpecialMemberOverloadResultEntry - : public llvm::FastFoldingSetNode, - public SpecialMemberOverloadResult { - public: - SpecialMemberOverloadResultEntry(const llvm::FoldingSetNodeID &ID) - : FastFoldingSetNode(ID) - {} - }; + void DiagnoseNonDefaultPragmaAlignPack(PragmaAlignPackDiagnoseKind Kind, + SourceLocation IncludeLoc); + void DiagnoseUnterminatedPragmaAlignPack(); - /// A cache of special member function overload resolution results - /// for C++ records. - llvm::FoldingSet SpecialMemberCache; + /// ActOnPragmaMSStruct - Called on well formed \#pragma ms_struct [on|off]. + void ActOnPragmaMSStruct(PragmaMSStructKind Kind); - /// A cache of the flags available in enumerations with the flag_bits - /// attribute. - mutable llvm::DenseMap FlagBitsCache; + /// ActOnPragmaMSComment - Called on well formed + /// \#pragma comment(kind, "arg"). + void ActOnPragmaMSComment(SourceLocation CommentLoc, PragmaMSCommentKind Kind, + StringRef Arg); - /// The kind of translation unit we are processing. - /// - /// When we're processing a complete translation unit, Sema will perform - /// end-of-translation-unit semantic tasks (such as creating - /// initializers for tentative definitions in C) once parsing has - /// completed. Modules and precompiled headers perform different kinds of - /// checks. - const TranslationUnitKind TUKind; + /// ActOnPragmaDetectMismatch - Call on well-formed \#pragma detect_mismatch + void ActOnPragmaDetectMismatch(SourceLocation Loc, StringRef Name, + StringRef Value); - llvm::BumpPtrAllocator BumpAlloc; + /// Are precise floating point semantics currently enabled? + bool isPreciseFPEnabled() { + return !CurFPFeatures.getAllowFPReassociate() && + !CurFPFeatures.getNoSignedZero() && + !CurFPFeatures.getAllowReciprocal() && + !CurFPFeatures.getAllowApproxFunc(); + } - /// The number of SFINAE diagnostics that have been trapped. - unsigned NumSFINAEErrors; + void ActOnPragmaFPEvalMethod(SourceLocation Loc, + LangOptions::FPEvalMethodKind Value); - typedef llvm::DenseMap> - UnparsedDefaultArgInstantiationsMap; + /// ActOnPragmaFloatControl - Call on well-formed \#pragma float_control + void ActOnPragmaFloatControl(SourceLocation Loc, PragmaMsStackAction Action, + PragmaFloatControlKind Value); - /// A mapping from parameters with unparsed default arguments to the - /// set of instantiations of each parameter. - /// - /// This mapping is a temporary data structure used when parsing - /// nested class templates or nested classes of class templates, - /// where we might end up instantiating an inner class before the - /// default arguments of its methods have been parsed. - UnparsedDefaultArgInstantiationsMap UnparsedDefaultArgInstantiations; + /// ActOnPragmaMSPointersToMembers - called on well formed \#pragma + /// pointers_to_members(representation method[, general purpose + /// representation]). + void ActOnPragmaMSPointersToMembers( + LangOptions::PragmaMSPointersToMembersKind Kind, + SourceLocation PragmaLoc); - // Contains the locations of the beginning of unparsed default - // argument locations. - llvm::DenseMap UnparsedDefaultArgLocs; + /// Called on well formed \#pragma vtordisp(). + void ActOnPragmaMSVtorDisp(PragmaMsStackAction Action, + SourceLocation PragmaLoc, MSVtorDispMode Value); - /// UndefinedInternals - all the used, undefined objects which require a - /// definition in this translation unit. - llvm::MapVector UndefinedButUsed; - - /// Determine if VD, which must be a variable or function, is an external - /// symbol that nonetheless can't be referenced from outside this translation - /// unit because its type has no linkage and it's not extern "C". - bool isExternalWithNoLinkageType(const ValueDecl *VD) const; + bool UnifySection(StringRef SectionName, int SectionFlags, + NamedDecl *TheDecl); + bool UnifySection(StringRef SectionName, int SectionFlags, + SourceLocation PragmaSectionLocation); - /// Obtain a sorted list of functions that are undefined but ODR-used. - void getUndefinedButUsed( - SmallVectorImpl > &Undefined); + /// Called on well formed \#pragma bss_seg/data_seg/const_seg/code_seg. + void ActOnPragmaMSSeg(SourceLocation PragmaLocation, + PragmaMsStackAction Action, + llvm::StringRef StackSlotLabel, + StringLiteral *SegmentName, llvm::StringRef PragmaName); - /// Retrieves list of suspicious delete-expressions that will be checked at - /// the end of translation unit. - const llvm::MapVector & - getMismatchingDeleteExpressions() const; + /// Called on well formed \#pragma section(). + void ActOnPragmaMSSection(SourceLocation PragmaLocation, int SectionFlags, + StringLiteral *SegmentName); - class GlobalMethodPool { - public: - using Lists = std::pair; - using iterator = llvm::DenseMap::iterator; - iterator begin() { return Methods.begin(); } - iterator end() { return Methods.end(); } - iterator find(Selector Sel) { return Methods.find(Sel); } - std::pair insert(std::pair &&Val) { - return Methods.insert(Val); - } - int count(Selector Sel) const { return Methods.count(Sel); } - bool empty() const { return Methods.empty(); } + /// Called on well-formed \#pragma init_seg(). + void ActOnPragmaMSInitSeg(SourceLocation PragmaLocation, + StringLiteral *SegmentName); - private: - llvm::DenseMap Methods; - }; + /// Called on well-formed \#pragma alloc_text(). + void ActOnPragmaMSAllocText( + SourceLocation PragmaLocation, StringRef Section, + const SmallVector> + &Functions); - /// Method Pool - allows efficient lookup when typechecking messages to "id". - /// We need to maintain a list, since selectors can have differing signatures - /// across classes. In Cocoa, this happens to be extremely uncommon (only 1% - /// of selectors are "overloaded"). - /// At the head of the list it is recorded whether there were 0, 1, or >= 2 - /// methods inside categories with a particular selector. - GlobalMethodPool MethodPool; + /// ActOnPragmaMSStrictGuardStackCheck - Called on well formed \#pragma + /// strict_gs_check. + void ActOnPragmaMSStrictGuardStackCheck(SourceLocation PragmaLocation, + PragmaMsStackAction Action, + bool Value); - /// Method selectors used in a \@selector expression. Used for implementation - /// of -Wselector. - llvm::MapVector ReferencedSelectors; + /// ActOnPragmaUnused - Called on well-formed '\#pragma unused'. + void ActOnPragmaUnused(const Token &Identifier, Scope *curScope, + SourceLocation PragmaLoc); - /// List of SourceLocations where 'self' is implicitly retained inside a - /// block. - llvm::SmallVector, 1> - ImplicitlyRetainedSelfLocs; + /// AddCFAuditedAttribute - Check whether we're currently within + /// '\#pragma clang arc_cf_code_audited' and, if so, consider adding + /// the appropriate attribute. + void AddCFAuditedAttribute(Decl *D); - /// Kinds of C++ special members. - enum CXXSpecialMember { - CXXDefaultConstructor, - CXXCopyConstructor, - CXXMoveConstructor, - CXXCopyAssignment, - CXXMoveAssignment, - CXXDestructor, - CXXInvalid - }; + void ActOnPragmaAttributeAttribute(ParsedAttr &Attribute, + SourceLocation PragmaLoc, + attr::ParsedSubjectMatchRuleSet Rules); + void ActOnPragmaAttributeEmptyPush(SourceLocation PragmaLoc, + const IdentifierInfo *Namespace); - typedef llvm::PointerIntPair - SpecialMemberDecl; + /// Called on well-formed '\#pragma clang attribute pop'. + void ActOnPragmaAttributePop(SourceLocation PragmaLoc, + const IdentifierInfo *Namespace); - /// The C++ special members which we are currently in the process of - /// declaring. If this process recursively triggers the declaration of the - /// same special member, we should act as if it is not yet declared. - llvm::SmallPtrSet SpecialMembersBeingDeclared; + /// Adds the attributes that have been specified using the + /// '\#pragma clang attribute push' directives to the given declaration. + void AddPragmaAttributes(Scope *S, Decl *D); - /// Kinds of defaulted comparison operator functions. - enum class DefaultedComparisonKind : unsigned char { - /// This is not a defaultable comparison operator. - None, - /// This is an operator== that should be implemented as a series of - /// subobject comparisons. - Equal, - /// This is an operator<=> that should be implemented as a series of - /// subobject comparisons. - ThreeWay, - /// This is an operator!= that should be implemented as a rewrite in terms - /// of a == comparison. - NotEqual, - /// This is an <, <=, >, or >= that should be implemented as a rewrite in - /// terms of a <=> comparison. - Relational, - }; + void PrintPragmaAttributeInstantiationPoint(); - /// The function definitions which were renamed as part of typo-correction - /// to match their respective declarations. We want to keep track of them - /// to ensure that we don't emit a "redefinition" error if we encounter a - /// correctly named definition after the renamed definition. - llvm::SmallPtrSet TypoCorrectedFunctionDefinitions; + void DiagnoseUnterminatedPragmaAttribute(); - /// Stack of types that correspond to the parameter entities that are - /// currently being copy-initialized. Can be empty. - llvm::SmallVector CurrentParameterCopyTypes; + /// Called on well formed \#pragma clang optimize. + void ActOnPragmaOptimize(bool On, SourceLocation PragmaLoc); - void ReadMethodPool(Selector Sel); - void updateOutOfDateSelector(Selector Sel); + /// #pragma optimize("[optimization-list]", on | off). + void ActOnPragmaMSOptimize(SourceLocation Loc, bool IsOn); - /// Private Helper predicate to check for 'self'. - bool isSelfExpr(Expr *RExpr); - bool isSelfExpr(Expr *RExpr, const ObjCMethodDecl *Method); + /// Call on well formed \#pragma function. + void + ActOnPragmaMSFunction(SourceLocation Loc, + const llvm::SmallVectorImpl &NoBuiltins); - /// Cause the active diagnostic on the DiagosticsEngine to be - /// emitted. This is closely coupled to the SemaDiagnosticBuilder class and - /// should not be used elsewhere. - void EmitCurrentDiagnostic(unsigned DiagID); + /// Only called on function definitions; if there is a pragma in scope + /// with the effect of a range-based optnone, consider marking the function + /// with attribute optnone. + void AddRangeBasedOptnone(FunctionDecl *FD); - /// Records and restores the CurFPFeatures state on entry/exit of compound - /// statements. - class FPFeaturesStateRAII { - public: - FPFeaturesStateRAII(Sema &S); - ~FPFeaturesStateRAII(); - FPOptionsOverride getOverrides() { return OldOverrides; } + /// Only called on function definitions; if there is a `#pragma alloc_text` + /// that decides which code section the function should be in, add + /// attribute section to the function. + void AddSectionMSAllocText(FunctionDecl *FD); - private: - Sema& S; - FPOptions OldFPFeaturesState; - FPOptionsOverride OldOverrides; - LangOptions::FPEvalMethodKind OldEvalMethod; - SourceLocation OldFPPragmaLocation; - }; + /// Adds the 'optnone' attribute to the function declaration if there + /// are no conflicts; Loc represents the location causing the 'optnone' + /// attribute to be added (usually because of a pragma). + void AddOptnoneAttributeIfNoConflicts(FunctionDecl *FD, SourceLocation Loc); - void addImplicitTypedef(StringRef Name, QualType T); + /// Only called on function definitions; if there is a MSVC #pragma optimize + /// in scope, consider changing the function's attributes based on the + /// optimization list passed to the pragma. + void ModifyFnAttributesMSPragmaOptimize(FunctionDecl *FD); - bool WarnedStackExhausted = false; + /// Only called on function definitions; if there is a pragma in scope + /// with the effect of a range-based no_builtin, consider marking the function + /// with attribute no_builtin. + void AddImplicitMSFunctionNoBuiltinAttr(FunctionDecl *FD); - /// Increment when we find a reference; decrement when we find an ignored - /// assignment. Ultimately the value is 0 if every reference is an ignored - /// assignment. - llvm::DenseMap RefsMinusAssignments; + /// AddPushedVisibilityAttribute - If '\#pragma GCC visibility' was used, + /// add an appropriate visibility attribute. + void AddPushedVisibilityAttribute(Decl *RD); - /// Indicate RISC-V vector builtin functions enabled or not. - bool DeclareRISCVVBuiltins = false; + /// FreeVisContext - Deallocate and null out VisContext. + void FreeVisContext(); - /// Indicate RISC-V SiFive vector builtin functions enabled or not. - bool DeclareRISCVSiFiveVectorBuiltins = false; + /// ActOnPragmaVisibility - Called on well formed \#pragma GCC visibility... . + void ActOnPragmaVisibility(const IdentifierInfo *VisType, + SourceLocation PragmaLoc); -private: - std::unique_ptr RVIntrinsicManager; + /// ActOnPragmaFPContract - Called on well formed + /// \#pragma {STDC,OPENCL} FP_CONTRACT and + /// \#pragma clang fp contract + void ActOnPragmaFPContract(SourceLocation Loc, LangOptions::FPModeKind FPC); - std::optional> CachedDarwinSDKInfo; + /// Called on well formed + /// \#pragma clang fp reassociate + /// or + /// \#pragma clang fp reciprocal + void ActOnPragmaFPValueChangingOption(SourceLocation Loc, PragmaFPKind Kind, + bool IsEnabled); - bool WarnedDarwinSDKInfoMissing = false; + /// ActOnPragmaFenvAccess - Called on well formed + /// \#pragma STDC FENV_ACCESS + void ActOnPragmaFEnvAccess(SourceLocation Loc, bool IsEnabled); -public: - Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer, - TranslationUnitKind TUKind = TU_Complete, - CodeCompleteConsumer *CompletionConsumer = nullptr); - ~Sema(); + /// ActOnPragmaCXLimitedRange - Called on well formed + /// \#pragma STDC CX_LIMITED_RANGE + void ActOnPragmaCXLimitedRange(SourceLocation Loc, + LangOptions::ComplexRangeKind Range); - /// Perform initialization that occurs after the parser has been - /// initialized but before it parses anything. - void Initialize(); + /// Called on well formed '\#pragma clang fp' that has option 'exceptions'. + void ActOnPragmaFPExceptions(SourceLocation Loc, + LangOptions::FPExceptionModeKind); - /// This virtual key function only exists to limit the emission of debug info - /// describing the Sema class. GCC and Clang only emit debug info for a class - /// with a vtable when the vtable is emitted. Sema is final and not - /// polymorphic, but the debug info size savings are so significant that it is - /// worth adding a vtable just to take advantage of this optimization. - virtual void anchor(); + /// Called to set constant rounding mode for floating point operations. + void ActOnPragmaFEnvRound(SourceLocation Loc, llvm::RoundingMode); - const LangOptions &getLangOpts() const { return LangOpts; } - OpenCLOptions &getOpenCLOptions() { return OpenCLFeatures; } - FPOptions &getCurFPFeatures() { return CurFPFeatures; } + /// Called to set exception behavior for floating point operations. + void setExceptionMode(SourceLocation Loc, LangOptions::FPExceptionModeKind); - DiagnosticsEngine &getDiagnostics() const { return Diags; } - SourceManager &getSourceManager() const { return SourceMgr; } - Preprocessor &getPreprocessor() const { return PP; } - ASTContext &getASTContext() const { return Context; } - ASTConsumer &getASTConsumer() const { return Consumer; } - ASTMutationListener *getASTMutationListener() const; - ExternalSemaSource *getExternalSource() const { return ExternalSource.get(); } + /// PushNamespaceVisibilityAttr - Note that we've entered a + /// namespace with a visibility attribute. + void PushNamespaceVisibilityAttr(const VisibilityAttr *Attr, + SourceLocation Loc); - DarwinSDKInfo *getDarwinSDKInfoForAvailabilityChecking(SourceLocation Loc, - StringRef Platform); - DarwinSDKInfo *getDarwinSDKInfoForAvailabilityChecking(); + /// PopPragmaVisibility - Pop the top element of the visibility stack; used + /// for '\#pragma GCC visibility' and visibility attributes on namespaces. + void PopPragmaVisibility(bool IsNamespaceEnd, SourceLocation EndLoc); - ///Registers an external source. If an external source already exists, - /// creates a multiplex external source and appends to it. - /// - ///\param[in] E - A non-null external sema source. - /// - void addExternalSource(ExternalSemaSource *E); + /// Handles semantic checking for features that are common to all attributes, + /// such as checking whether a parameter was properly specified, or the + /// correct number of arguments were passed, etc. Returns true if the + /// attribute has been diagnosed. + bool checkCommonAttributeFeatures(const Decl *D, const ParsedAttr &A, + bool SkipArgCountCheck = false); + bool checkCommonAttributeFeatures(const Stmt *S, const ParsedAttr &A, + bool SkipArgCountCheck = false); - void PrintStats() const; + ///@} - /// Warn that the stack is nearly exhausted. - void warnStackExhausted(SourceLocation Loc); + // + // + // ------------------------------------------------------------------------- + // + // - /// Run some code with "sufficient" stack space. (Currently, at least 256K is - /// guaranteed). Produces a warning if we're low on stack space and allocates - /// more in that case. Use this in code that may recurse deeply (for example, - /// in template instantiation) to avoid stack overflow. - void runWithSufficientStackSpace(SourceLocation Loc, - llvm::function_ref Fn); + /// \name Availability Attribute Handling + /// Implementations are in SemaAvailability.cpp + ///@{ - /// Helper class that creates diagnostics with optional - /// template instantiation stacks. - /// - /// This class provides a wrapper around the basic DiagnosticBuilder - /// class that emits diagnostics. ImmediateDiagBuilder is - /// responsible for emitting the diagnostic (as DiagnosticBuilder - /// does) and, if the diagnostic comes from inside a template - /// instantiation, printing the template instantiation stack as - /// well. - class ImmediateDiagBuilder : public DiagnosticBuilder { - Sema &SemaRef; - unsigned DiagID; +public: + /// Issue any -Wunguarded-availability warnings in \c FD + void DiagnoseUnguardedAvailabilityViolations(Decl *FD); - public: - ImmediateDiagBuilder(DiagnosticBuilder &DB, Sema &SemaRef, unsigned DiagID) - : DiagnosticBuilder(DB), SemaRef(SemaRef), DiagID(DiagID) {} - ImmediateDiagBuilder(DiagnosticBuilder &&DB, Sema &SemaRef, unsigned DiagID) - : DiagnosticBuilder(DB), SemaRef(SemaRef), DiagID(DiagID) {} + void handleDelayedAvailabilityCheck(sema::DelayedDiagnostic &DD, Decl *Ctx); - // This is a cunning lie. DiagnosticBuilder actually performs move - // construction in its copy constructor (but due to varied uses, it's not - // possible to conveniently express this as actual move construction). So - // the default copy ctor here is fine, because the base class disables the - // source anyway, so the user-defined ~ImmediateDiagBuilder is a safe no-op - // in that case anwyay. - ImmediateDiagBuilder(const ImmediateDiagBuilder &) = default; + /// Retrieve the current function, if any, that should be analyzed for + /// potential availability violations. + sema::FunctionScopeInfo *getCurFunctionAvailabilityContext(); - ~ImmediateDiagBuilder() { - // If we aren't active, there is nothing to do. - if (!isActive()) return; + void DiagnoseAvailabilityOfDecl(NamedDecl *D, ArrayRef Locs, + const ObjCInterfaceDecl *UnknownObjCClass, + bool ObjCPropertyAccess, + bool AvoidPartialAvailabilityChecks = false, + ObjCInterfaceDecl *ClassReceiver = nullptr); - // Otherwise, we need to emit the diagnostic. First clear the diagnostic - // builder itself so it won't emit the diagnostic in its own destructor. - // - // This seems wasteful, in that as written the DiagnosticBuilder dtor will - // do its own needless checks to see if the diagnostic needs to be - // emitted. However, because we take care to ensure that the builder - // objects never escape, a sufficiently smart compiler will be able to - // eliminate that code. - Clear(); + ///@} - // Dispatch to Sema to emit the diagnostic. - SemaRef.EmitCurrentDiagnostic(DiagID); - } + // + // + // ------------------------------------------------------------------------- + // + // - /// Teach operator<< to produce an object of the correct type. - template - friend const ImmediateDiagBuilder & - operator<<(const ImmediateDiagBuilder &Diag, const T &Value) { - const DiagnosticBuilder &BaseDiag = Diag; - BaseDiag << Value; - return Diag; - } + /// \name Casts + /// Implementations are in SemaCast.cpp + ///@{ - // It is necessary to limit this to rvalue reference to avoid calling this - // function with a bitfield lvalue argument since non-const reference to - // bitfield is not allowed. - template ::value>> - const ImmediateDiagBuilder &operator<<(T &&V) const { - const DiagnosticBuilder &BaseDiag = *this; - BaseDiag << std::move(V); - return *this; - } - }; +public: + static bool isCast(CheckedConversionKind CCK) { + return CCK == CCK_CStyleCast || CCK == CCK_FunctionalCast || + CCK == CCK_OtherCast; + } - /// A generic diagnostic builder for errors which may or may not be deferred. - /// - /// In CUDA, there exist constructs (e.g. variable-length arrays, try/catch) - /// which are not allowed to appear inside __device__ functions and are - /// allowed to appear in __host__ __device__ functions only if the host+device - /// function is never codegen'ed. - /// - /// To handle this, we use the notion of "deferred diagnostics", where we - /// attach a diagnostic to a FunctionDecl that's emitted iff it's codegen'ed. - /// - /// This class lets you emit either a regular diagnostic, a deferred - /// diagnostic, or no diagnostic at all, according to an argument you pass to - /// its constructor, thus simplifying the process of creating these "maybe - /// deferred" diagnostics. - class SemaDiagnosticBuilder { - public: - enum Kind { - /// Emit no diagnostics. - K_Nop, - /// Emit the diagnostic immediately (i.e., behave like Sema::Diag()). - K_Immediate, - /// Emit the diagnostic immediately, and, if it's a warning or error, also - /// emit a call stack showing how this function can be reached by an a - /// priori known-emitted function. - K_ImmediateWithCallStack, - /// Create a deferred diagnostic, which is emitted only if the function - /// it's attached to is codegen'ed. Also emit a call stack as with - /// K_ImmediateWithCallStack. - K_Deferred - }; + /// ActOnCXXNamedCast - Parse + /// {dynamic,static,reinterpret,const,addrspace}_cast's. + ExprResult ActOnCXXNamedCast(SourceLocation OpLoc, tok::TokenKind Kind, + SourceLocation LAngleBracketLoc, Declarator &D, + SourceLocation RAngleBracketLoc, + SourceLocation LParenLoc, Expr *E, + SourceLocation RParenLoc); - SemaDiagnosticBuilder(Kind K, SourceLocation Loc, unsigned DiagID, - const FunctionDecl *Fn, Sema &S); - SemaDiagnosticBuilder(SemaDiagnosticBuilder &&D); - SemaDiagnosticBuilder(const SemaDiagnosticBuilder &) = default; + ExprResult BuildCXXNamedCast(SourceLocation OpLoc, tok::TokenKind Kind, + TypeSourceInfo *Ty, Expr *E, + SourceRange AngleBrackets, SourceRange Parens); - // The copy and move assignment operator is defined as deleted pending - // further motivation. - SemaDiagnosticBuilder &operator=(const SemaDiagnosticBuilder &) = delete; - SemaDiagnosticBuilder &operator=(SemaDiagnosticBuilder &&) = delete; + ExprResult ActOnBuiltinBitCastExpr(SourceLocation KWLoc, Declarator &Dcl, + ExprResult Operand, + SourceLocation RParenLoc); - ~SemaDiagnosticBuilder(); + ExprResult BuildBuiltinBitCastExpr(SourceLocation KWLoc, TypeSourceInfo *TSI, + Expr *Operand, SourceLocation RParenLoc); - bool isImmediate() const { return ImmediateDiag.has_value(); } + // Checks that reinterpret casts don't have undefined behavior. + void CheckCompatibleReinterpretCast(QualType SrcType, QualType DestType, + bool IsDereference, SourceRange Range); - /// Convertible to bool: True if we immediately emitted an error, false if - /// we didn't emit an error or we created a deferred error. - /// - /// Example usage: - /// - /// if (SemaDiagnosticBuilder(...) << foo << bar) - /// return ExprError(); - /// - /// But see CUDADiagIfDeviceCode() and CUDADiagIfHostCode() -- you probably - /// want to use these instead of creating a SemaDiagnosticBuilder yourself. - operator bool() const { return isImmediate(); } + // Checks that the vector type should be initialized from a scalar + // by splatting the value rather than populating a single element. + // This is the case for AltiVecVector types as well as with + // AltiVecPixel and AltiVecBool when -faltivec-src-compat=xl is specified. + bool ShouldSplatAltivecScalarInCast(const VectorType *VecTy); - template - friend const SemaDiagnosticBuilder & - operator<<(const SemaDiagnosticBuilder &Diag, const T &Value) { - if (Diag.ImmediateDiag) - *Diag.ImmediateDiag << Value; - else if (Diag.PartialDiagId) - Diag.S.DeviceDeferredDiags[Diag.Fn][*Diag.PartialDiagId].second - << Value; - return Diag; - } + // Checks if the -faltivec-src-compat=gcc option is specified. + // If so, AltiVecVector, AltiVecBool and AltiVecPixel types are + // treated the same way as they are when trying to initialize + // these vectors on gcc (an error is emitted). + bool CheckAltivecInitFromScalar(SourceRange R, QualType VecTy, + QualType SrcTy); - // It is necessary to limit this to rvalue reference to avoid calling this - // function with a bitfield lvalue argument since non-const reference to - // bitfield is not allowed. - template ::value>> - const SemaDiagnosticBuilder &operator<<(T &&V) const { - if (ImmediateDiag) - *ImmediateDiag << std::move(V); - else if (PartialDiagId) - S.DeviceDeferredDiags[Fn][*PartialDiagId].second << std::move(V); - return *this; - } + ExprResult BuildCStyleCastExpr(SourceLocation LParenLoc, TypeSourceInfo *Ty, + SourceLocation RParenLoc, Expr *Op); - friend const SemaDiagnosticBuilder & - operator<<(const SemaDiagnosticBuilder &Diag, const PartialDiagnostic &PD) { - if (Diag.ImmediateDiag) - PD.Emit(*Diag.ImmediateDiag); - else if (Diag.PartialDiagId) - Diag.S.DeviceDeferredDiags[Diag.Fn][*Diag.PartialDiagId].second = PD; - return Diag; - } + ExprResult BuildCXXFunctionalCastExpr(TypeSourceInfo *TInfo, QualType Type, + SourceLocation LParenLoc, + Expr *CastExpr, + SourceLocation RParenLoc); - void AddFixItHint(const FixItHint &Hint) const { - if (ImmediateDiag) - ImmediateDiag->AddFixItHint(Hint); - else if (PartialDiagId) - S.DeviceDeferredDiags[Fn][*PartialDiagId].second.AddFixItHint(Hint); - } + ///@} - friend ExprResult ExprError(const SemaDiagnosticBuilder &) { - return ExprError(); - } - friend StmtResult StmtError(const SemaDiagnosticBuilder &) { - return StmtError(); - } - operator ExprResult() const { return ExprError(); } - operator StmtResult() const { return StmtError(); } - operator TypeResult() const { return TypeError(); } - operator DeclResult() const { return DeclResult(true); } - operator MemInitResult() const { return MemInitResult(true); } + // + // + // ------------------------------------------------------------------------- + // + // - private: - Sema &S; - SourceLocation Loc; - unsigned DiagID; - const FunctionDecl *Fn; - bool ShowCallStack; + /// \name Extra Semantic Checking + /// Implementations are in SemaChecking.cpp + ///@{ - // Invariant: At most one of these Optionals has a value. - // FIXME: Switch these to a Variant once that exists. - std::optional ImmediateDiag; - std::optional PartialDiagId; - }; +public: + /// Used to change context to isConstantEvaluated without pushing a heavy + /// ExpressionEvaluationContextRecord object. + bool isConstantEvaluatedOverride = false; - /// Is the last error level diagnostic immediate. This is used to determined - /// whether the next info diagnostic should be immediate. - bool IsLastErrorImmediate = true; + bool isConstantEvaluatedContext() const { + return currentEvaluationContext().isConstantEvaluated() || + isConstantEvaluatedOverride; + } - /// Emit a diagnostic. - SemaDiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID, - bool DeferHint = false); + SourceLocation getLocationOfStringLiteralByte(const StringLiteral *SL, + unsigned ByteNo) const; - /// Emit a partial diagnostic. - SemaDiagnosticBuilder Diag(SourceLocation Loc, const PartialDiagnostic &PD, - bool DeferHint = false); + enum FormatArgumentPassingKind { + FAPK_Fixed, // values to format are fixed (no C-style variadic arguments) + FAPK_Variadic, // values to format are passed as variadic arguments + FAPK_VAList, // values to format are passed in a va_list + }; - /// Build a partial diagnostic. - PartialDiagnostic PDiag(unsigned DiagID = 0); // in SemaInternal.h + // Used to grab the relevant information from a FormatAttr and a + // FunctionDeclaration. + struct FormatStringInfo { + unsigned FormatIdx; + unsigned FirstDataArg; + FormatArgumentPassingKind ArgPassingKind; + }; - /// Whether deferrable diagnostics should be deferred. - bool DeferDiags = false; + static bool getFormatStringInfo(const FormatAttr *Format, bool IsCXXMember, + bool IsVariadic, FormatStringInfo *FSI); - /// RAII class to control scope of DeferDiags. - class DeferDiagsRAII { - Sema &S; - bool SavedDeferDiags = false; + // Used by C++ template instantiation. + ExprResult SemaBuiltinShuffleVector(CallExpr *TheCall); + ExprResult SemaConvertVectorExpr(Expr *E, TypeSourceInfo *TInfo, + SourceLocation BuiltinLoc, + SourceLocation RParenLoc); - public: - DeferDiagsRAII(Sema &S, bool DeferDiags) - : S(S), SavedDeferDiags(S.DeferDiags) { - S.DeferDiags = DeferDiags; - } - ~DeferDiagsRAII() { S.DeferDiags = SavedDeferDiags; } + enum FormatStringType { + FST_Scanf, + FST_Printf, + FST_NSString, + FST_Strftime, + FST_Strfmon, + FST_Kprintf, + FST_FreeBSDKPrintf, + FST_OSTrace, + FST_OSLog, + FST_Unknown }; + static FormatStringType GetFormatStringType(const FormatAttr *Format); - /// Whether uncompilable error has occurred. This includes error happens - /// in deferred diagnostics. - bool hasUncompilableErrorOccurred() const; + bool FormatStringHasSArg(const StringLiteral *FExpr); - bool findMacroSpelling(SourceLocation &loc, StringRef name); + static bool GetFormatNSStringIdx(const FormatAttr *Format, unsigned &Idx); - /// Get a string to suggest for zero-initialization of a type. - std::string - getFixItZeroInitializerForType(QualType T, SourceLocation Loc) const; - std::string getFixItZeroLiteralForType(QualType T, SourceLocation Loc) const; + void CheckFloatComparison(SourceLocation Loc, Expr *LHS, Expr *RHS, + BinaryOperatorKind Opcode); - /// Calls \c Lexer::getLocForEndOfToken() - SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset = 0); + /// Register a magic integral constant to be used as a type tag. + void RegisterTypeTagForDatatype(const IdentifierInfo *ArgumentKind, + uint64_t MagicValue, QualType Type, + bool LayoutCompatible, bool MustBeNull); - /// Retrieve the module loader associated with the preprocessor. - ModuleLoader &getModuleLoader() const; + struct TypeTagData { + TypeTagData() {} - /// Invent a new identifier for parameters of abbreviated templates. - IdentifierInfo * - InventAbbreviatedTemplateParameterTypeName(IdentifierInfo *ParamName, - unsigned Index); + TypeTagData(QualType Type, bool LayoutCompatible, bool MustBeNull) + : Type(Type), LayoutCompatible(LayoutCompatible), + MustBeNull(MustBeNull) {} - void emitAndClearUnusedLocalTypedefWarnings(); + QualType Type; - private: - /// Function or variable declarations to be checked for whether the deferred - /// diagnostics should be emitted. - llvm::SmallSetVector DeclsToCheckForDeferredDiags; + /// If true, \c Type should be compared with other expression's types for + /// layout-compatibility. + LLVM_PREFERRED_TYPE(bool) + unsigned LayoutCompatible : 1; + LLVM_PREFERRED_TYPE(bool) + unsigned MustBeNull : 1; + }; - public: - // Emit all deferred diagnostics. - void emitDeferredDiags(); + /// A pair of ArgumentKind identifier and magic value. This uniquely + /// identifies the magic value. + typedef std::pair TypeTagMagicValue; - enum TUFragmentKind { - /// The global module fragment, between 'module;' and a module-declaration. - Global, - /// A normal translation unit fragment. For a non-module unit, this is the - /// entire translation unit. Otherwise, it runs from the module-declaration - /// to the private-module-fragment (if any) or the end of the TU (if not). - Normal, - /// The private module fragment, between 'module :private;' and the end of - /// the translation unit. - Private - }; + /// Diagnoses the current set of gathered accesses. This typically + /// happens at full expression level. The set is cleared after emitting the + /// diagnostics. + void DiagnoseMisalignedMembers(); - void ActOnStartOfTranslationUnit(); - void ActOnEndOfTranslationUnit(); - void ActOnEndOfTranslationUnitFragment(TUFragmentKind Kind); + /// This function checks if the expression is in the sef of potentially + /// misaligned members and it is converted to some pointer type T with lower + /// or equal alignment requirements. If so it removes it. This is used when + /// we do not want to diagnose such misaligned access (e.g. in conversions to + /// void*). + void DiscardMisalignedMemberAddress(const Type *T, Expr *E); - void CheckDelegatingCtorCycles(); + /// This function calls Action when it determines that E designates a + /// misaligned member due to the packed attribute. This is used to emit + /// local diagnostics like in reference binding. + void RefersToMemberWithReducedAlignment( + Expr *E, + llvm::function_ref + Action); - Scope *getScopeForContext(DeclContext *Ctx); + enum class AtomicArgumentOrder { API, AST }; + ExprResult + BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, + SourceLocation RParenLoc, MultiExprArg Args, + AtomicExpr::AtomicOp Op, + AtomicArgumentOrder ArgOrder = AtomicArgumentOrder::API); - void PushFunctionScope(); - void PushBlockScope(Scope *BlockScope, BlockDecl *Block); - sema::LambdaScopeInfo *PushLambdaScope(); + /// Check to see if a given expression could have '.c_str()' called on it. + bool hasCStrMethod(const Expr *E); - /// This is used to inform Sema what the current TemplateParameterDepth - /// is during Parsing. Currently it is used to pass on the depth - /// when parsing generic lambda 'auto' parameters. - void RecordParsingTemplateParameterDepth(unsigned Depth); + void DiagnoseAlwaysNonNullPointer(Expr *E, + Expr::NullPointerConstantKind NullType, + bool IsEqual, SourceRange Range); - void PushCapturedRegionScope(Scope *RegionScope, CapturedDecl *CD, - RecordDecl *RD, CapturedRegionKind K, - unsigned OpenMPCaptureLevel = 0); + bool CheckParmsForFunctionDef(ArrayRef Parameters, + bool CheckParameterNames); - /// Custom deleter to allow FunctionScopeInfos to be kept alive for a short - /// time after they've been popped. - class PoppedFunctionScopeDeleter { - Sema *Self; + void CheckCastAlign(Expr *Op, QualType T, SourceRange TRange); - public: - explicit PoppedFunctionScopeDeleter(Sema *Self) : Self(Self) {} - void operator()(sema::FunctionScopeInfo *Scope) const; - }; + /// checkRetainCycles - Check whether an Objective-C message send + /// might create an obvious retain cycle. + void checkRetainCycles(ObjCMessageExpr *msg); + void checkRetainCycles(Expr *receiver, Expr *argument); + void checkRetainCycles(VarDecl *Var, Expr *Init); - using PoppedFunctionScopePtr = - std::unique_ptr; + /// checkUnsafeAssigns - Check whether +1 expr is being assigned + /// to weak/__unsafe_unretained type. + bool checkUnsafeAssigns(SourceLocation Loc, QualType LHS, Expr *RHS); - PoppedFunctionScopePtr - PopFunctionScopeInfo(const sema::AnalysisBasedWarnings::Policy *WP = nullptr, - const Decl *D = nullptr, - QualType BlockType = QualType()); + /// checkUnsafeExprAssigns - Check whether +1 expr is being assigned + /// to weak/__unsafe_unretained expression. + void checkUnsafeExprAssigns(SourceLocation Loc, Expr *LHS, Expr *RHS); - sema::FunctionScopeInfo *getCurFunction() const { - return FunctionScopes.empty() ? nullptr : FunctionScopes.back(); - } + /// Emit \p DiagID if statement located on \p StmtLoc has a suspicious null + /// statement as a \p Body, and it is located on the same line. + /// + /// This helps prevent bugs due to typos, such as: + /// if (condition); + /// do_stuff(); + void DiagnoseEmptyStmtBody(SourceLocation StmtLoc, const Stmt *Body, + unsigned DiagID); - sema::FunctionScopeInfo *getEnclosingFunction() const; + /// Warn if a for/while loop statement \p S, which is followed by + /// \p PossibleBody, has a suspicious null statement as a body. + void DiagnoseEmptyLoopBody(const Stmt *S, const Stmt *PossibleBody); - void setFunctionHasBranchIntoScope(); - void setFunctionHasBranchProtectedScope(); - void setFunctionHasIndirectGoto(); - void setFunctionHasMustTail(); + /// Warn if a value is moved to itself. + void DiagnoseSelfMove(const Expr *LHSExpr, const Expr *RHSExpr, + SourceLocation OpLoc); - void PushCompoundScope(bool IsStmtExpr); - void PopCompoundScope(); + // Used for emitting the right warning by DefaultVariadicArgumentPromotion + enum VariadicCallType { + VariadicFunction, + VariadicBlock, + VariadicMethod, + VariadicConstructor, + VariadicDoesNotApply + }; - sema::CompoundScopeInfo &getCurCompoundScope() const; + bool IsLayoutCompatible(QualType T1, QualType T2) const; - bool hasAnyUnrecoverableErrorsInThisFunction() const; +private: + void CheckArrayAccess(const Expr *BaseExpr, const Expr *IndexExpr, + const ArraySubscriptExpr *ASE = nullptr, + bool AllowOnePastEnd = true, bool IndexNegated = false); + void CheckArrayAccess(const Expr *E); - /// Retrieve the current block, if any. - sema::BlockScopeInfo *getCurBlock(); + bool CheckFunctionCall(FunctionDecl *FDecl, CallExpr *TheCall, + const FunctionProtoType *Proto); + bool CheckObjCMethodCall(ObjCMethodDecl *Method, SourceLocation loc, + ArrayRef Args); + bool CheckPointerCall(NamedDecl *NDecl, CallExpr *TheCall, + const FunctionProtoType *Proto); + bool CheckOtherCall(CallExpr *TheCall, const FunctionProtoType *Proto); + void CheckConstructorCall(FunctionDecl *FDecl, QualType ThisType, + ArrayRef Args, + const FunctionProtoType *Proto, SourceLocation Loc); - /// Get the innermost lambda enclosing the current location, if any. This - /// looks through intervening non-lambda scopes such as local functions and - /// blocks. - sema::LambdaScopeInfo *getEnclosingLambda() const; + void checkAIXMemberAlignment(SourceLocation Loc, const Expr *Arg); - /// Retrieve the current lambda scope info, if any. - /// \param IgnoreNonLambdaCapturingScope true if should find the top-most - /// lambda scope info ignoring all inner capturing scopes that are not - /// lambda scopes. - sema::LambdaScopeInfo * - getCurLambda(bool IgnoreNonLambdaCapturingScope = false); + void CheckArgAlignment(SourceLocation Loc, NamedDecl *FDecl, + StringRef ParamName, QualType ArgTy, QualType ParamTy); - /// Retrieve the current generic lambda info, if any. - sema::LambdaScopeInfo *getCurGenericLambda(); + void checkCall(NamedDecl *FDecl, const FunctionProtoType *Proto, + const Expr *ThisArg, ArrayRef Args, + bool IsMemberFunction, SourceLocation Loc, SourceRange Range, + VariadicCallType CallType); - /// Retrieve the current captured region, if any. - sema::CapturedRegionScopeInfo *getCurCapturedRegion(); + bool CheckObjCString(Expr *Arg); + ExprResult CheckOSLogFormatStringArg(Expr *Arg); - /// Retrieve the current function, if any, that should be analyzed for - /// potential availability violations. - sema::FunctionScopeInfo *getCurFunctionAvailabilityContext(); + ExprResult CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID, + CallExpr *TheCall); - /// WeakTopLevelDeclDecls - access to \#pragma weak-generated Decls - SmallVectorImpl &WeakTopLevelDecls() { return WeakTopLevelDecl; } + bool CheckTSBuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID, + CallExpr *TheCall); - /// Called before parsing a function declarator belonging to a function - /// declaration. - void ActOnStartFunctionDeclarationDeclarator(Declarator &D, - unsigned TemplateParameterDepth); + void checkFortifiedBuiltinMemoryFunction(FunctionDecl *FD, CallExpr *TheCall); - /// Called after parsing a function declarator belonging to a function - /// declaration. - void ActOnFinishFunctionDeclarationDeclarator(Declarator &D); + bool CheckARMBuiltinExclusiveCall(unsigned BuiltinID, CallExpr *TheCall, + unsigned MaxWidth); + bool CheckNeonBuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID, + CallExpr *TheCall); + bool CheckMVEBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); + bool CheckSVEBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); + bool ParseSVEImmChecks(CallExpr *TheCall, + SmallVector, 3> &ImmChecks); + bool CheckSMEBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); + bool CheckCDEBuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID, + CallExpr *TheCall); + bool CheckARMCoprocessorImmediate(const TargetInfo &TI, const Expr *CoprocArg, + bool WantCDE); + bool CheckARMBuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID, + CallExpr *TheCall); - void ActOnComment(SourceRange Comment); + bool CheckAArch64BuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID, + CallExpr *TheCall); + bool CheckBPFBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); + bool CheckHexagonBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); + bool CheckHexagonBuiltinArgument(unsigned BuiltinID, CallExpr *TheCall); + bool CheckMipsBuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID, + CallExpr *TheCall); + bool CheckMipsBuiltinCpu(const TargetInfo &TI, unsigned BuiltinID, + CallExpr *TheCall); + bool CheckMipsBuiltinArgument(unsigned BuiltinID, CallExpr *TheCall); + bool CheckSystemZBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); + bool CheckX86BuiltinRoundingOrSAE(unsigned BuiltinID, CallExpr *TheCall); + bool CheckX86BuiltinGatherScatterScale(unsigned BuiltinID, CallExpr *TheCall); + bool CheckX86BuiltinTileArguments(unsigned BuiltinID, CallExpr *TheCall); + bool CheckX86BuiltinTileArgumentsRange(CallExpr *TheCall, + ArrayRef ArgNums); + bool CheckX86BuiltinTileDuplicate(CallExpr *TheCall, ArrayRef ArgNums); + bool CheckX86BuiltinTileRangeAndDuplicate(CallExpr *TheCall, + ArrayRef ArgNums); + bool CheckX86BuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID, + CallExpr *TheCall); + bool CheckPPCBuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID, + CallExpr *TheCall); + bool CheckAMDGCNBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); + bool CheckRISCVLMUL(CallExpr *TheCall, unsigned ArgNum); + bool CheckRISCVBuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID, + CallExpr *TheCall); + void checkRVVTypeSupport(QualType Ty, SourceLocation Loc, Decl *D); + bool CheckLoongArchBuiltinFunctionCall(const TargetInfo &TI, + unsigned BuiltinID, CallExpr *TheCall); + bool CheckWebAssemblyBuiltinFunctionCall(const TargetInfo &TI, + unsigned BuiltinID, + CallExpr *TheCall); + bool CheckNVPTXBuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID, + CallExpr *TheCall); - //===--------------------------------------------------------------------===// - // Type Analysis / Processing: SemaType.cpp. - // + bool SemaBuiltinVAStart(unsigned BuiltinID, CallExpr *TheCall); + bool SemaBuiltinVAStartARMMicrosoft(CallExpr *Call); + bool SemaBuiltinUnorderedCompare(CallExpr *TheCall, unsigned BuiltinID); + bool SemaBuiltinFPClassification(CallExpr *TheCall, unsigned NumArgs, + unsigned BuiltinID); + bool SemaBuiltinComplex(CallExpr *TheCall); + bool SemaBuiltinVSX(CallExpr *TheCall); + bool SemaBuiltinOSLogFormat(CallExpr *TheCall); + bool SemaValueIsRunOfOnes(CallExpr *TheCall, unsigned ArgNum); - QualType BuildQualifiedType(QualType T, SourceLocation Loc, Qualifiers Qs, - const DeclSpec *DS = nullptr); - QualType BuildQualifiedType(QualType T, SourceLocation Loc, unsigned CVRA, - const DeclSpec *DS = nullptr); - QualType BuildPointerType(QualType T, - SourceLocation Loc, DeclarationName Entity); - QualType BuildReferenceType(QualType T, bool LValueRef, - SourceLocation Loc, DeclarationName Entity); - QualType BuildArrayType(QualType T, ArraySizeModifier ASM, Expr *ArraySize, - unsigned Quals, SourceRange Brackets, - DeclarationName Entity); - QualType BuildVectorType(QualType T, Expr *VecSize, SourceLocation AttrLoc); - QualType BuildExtVectorType(QualType T, Expr *ArraySize, - SourceLocation AttrLoc); - QualType BuildMatrixType(QualType T, Expr *NumRows, Expr *NumColumns, - SourceLocation AttrLoc); + bool SemaBuiltinPrefetch(CallExpr *TheCall); + bool SemaBuiltinAllocaWithAlign(CallExpr *TheCall); + bool SemaBuiltinArithmeticFence(CallExpr *TheCall); + bool SemaBuiltinAssume(CallExpr *TheCall); + bool SemaBuiltinAssumeAligned(CallExpr *TheCall); + bool SemaBuiltinLongjmp(CallExpr *TheCall); + bool SemaBuiltinSetjmp(CallExpr *TheCall); + ExprResult SemaBuiltinAtomicOverloaded(ExprResult TheCallResult); + ExprResult SemaBuiltinNontemporalOverloaded(ExprResult TheCallResult); + ExprResult SemaAtomicOpsOverloaded(ExprResult TheCallResult, + AtomicExpr::AtomicOp Op); + bool SemaBuiltinConstantArg(CallExpr *TheCall, int ArgNum, + llvm::APSInt &Result); + bool SemaBuiltinConstantArgRange(CallExpr *TheCall, int ArgNum, int Low, + int High, bool RangeIsError = true); + bool SemaBuiltinConstantArgMultiple(CallExpr *TheCall, int ArgNum, + unsigned Multiple); + bool SemaBuiltinConstantArgPower2(CallExpr *TheCall, int ArgNum); + bool SemaBuiltinConstantArgShiftedByte(CallExpr *TheCall, int ArgNum, + unsigned ArgBits); + bool SemaBuiltinConstantArgShiftedByteOrXXFF(CallExpr *TheCall, int ArgNum, + unsigned ArgBits); + bool SemaBuiltinARMSpecialReg(unsigned BuiltinID, CallExpr *TheCall, + int ArgNum, unsigned ExpectedFieldNum, + bool AllowName); + bool SemaBuiltinARMMemoryTaggingCall(unsigned BuiltinID, CallExpr *TheCall); + bool SemaBuiltinPPCMMACall(CallExpr *TheCall, unsigned BuiltinID, + const char *TypeDesc); - QualType BuildAddressSpaceAttr(QualType &T, LangAS ASIdx, Expr *AddrSpace, - SourceLocation AttrLoc); + bool CheckPPCMMAType(QualType Type, SourceLocation TypeLoc); - /// Same as above, but constructs the AddressSpace index if not provided. - QualType BuildAddressSpaceAttr(QualType &T, Expr *AddrSpace, - SourceLocation AttrLoc); + bool SemaBuiltinElementwiseMath(CallExpr *TheCall); + bool SemaBuiltinElementwiseTernaryMath(CallExpr *TheCall, + bool CheckForFloatArgs = true); + bool PrepareBuiltinElementwiseMathOneArgCall(CallExpr *TheCall); + bool PrepareBuiltinReduceMathOneArgCall(CallExpr *TheCall); - CodeAlignAttr *BuildCodeAlignAttr(const AttributeCommonInfo &CI, Expr *E); - bool CheckRebuiltStmtAttributes(ArrayRef Attrs); + bool SemaBuiltinNonDeterministicValue(CallExpr *TheCall); - bool CheckQualifiedFunctionForTypeId(QualType T, SourceLocation Loc); + // Matrix builtin handling. + ExprResult SemaBuiltinMatrixTranspose(CallExpr *TheCall, + ExprResult CallResult); + ExprResult SemaBuiltinMatrixColumnMajorLoad(CallExpr *TheCall, + ExprResult CallResult); + ExprResult SemaBuiltinMatrixColumnMajorStore(CallExpr *TheCall, + ExprResult CallResult); - bool CheckFunctionReturnType(QualType T, SourceLocation Loc); + // WebAssembly builtin handling. + bool BuiltinWasmRefNullExtern(CallExpr *TheCall); + bool BuiltinWasmRefNullFunc(CallExpr *TheCall); + bool BuiltinWasmTableGet(CallExpr *TheCall); + bool BuiltinWasmTableSet(CallExpr *TheCall); + bool BuiltinWasmTableSize(CallExpr *TheCall); + bool BuiltinWasmTableGrow(CallExpr *TheCall); + bool BuiltinWasmTableFill(CallExpr *TheCall); + bool BuiltinWasmTableCopy(CallExpr *TheCall); - /// Check an argument list for placeholders that we won't try to - /// handle later. - bool CheckArgsForPlaceholders(MultiExprArg args); + bool CheckFormatArguments(const FormatAttr *Format, + ArrayRef Args, bool IsCXXMember, + VariadicCallType CallType, SourceLocation Loc, + SourceRange Range, + llvm::SmallBitVector &CheckedVarArgs); + bool CheckFormatArguments(ArrayRef Args, + FormatArgumentPassingKind FAPK, unsigned format_idx, + unsigned firstDataArg, FormatStringType Type, + VariadicCallType CallType, SourceLocation Loc, + SourceRange range, + llvm::SmallBitVector &CheckedVarArgs); - /// Build a function type. - /// - /// This routine checks the function type according to C++ rules and - /// under the assumption that the result type and parameter types have - /// just been instantiated from a template. It therefore duplicates - /// some of the behavior of GetTypeForDeclarator, but in a much - /// simpler form that is only suitable for this narrow use case. - /// - /// \param T The return type of the function. - /// - /// \param ParamTypes The parameter types of the function. This array - /// will be modified to account for adjustments to the types of the - /// function parameters. - /// - /// \param Loc The location of the entity whose type involves this - /// function type or, if there is no such entity, the location of the - /// type that will have function type. - /// - /// \param Entity The name of the entity that involves the function - /// type, if known. - /// - /// \param EPI Extra information about the function type. Usually this will - /// be taken from an existing function with the same prototype. - /// - /// \returns A suitable function type, if there are no errors. The - /// unqualified type will always be a FunctionProtoType. - /// Otherwise, returns a NULL type. - QualType BuildFunctionType(QualType T, - MutableArrayRef ParamTypes, - SourceLocation Loc, DeclarationName Entity, - const FunctionProtoType::ExtProtoInfo &EPI); + void CheckInfNaNFunction(const CallExpr *Call, const FunctionDecl *FDecl); - QualType BuildMemberPointerType(QualType T, QualType Class, - SourceLocation Loc, - DeclarationName Entity); - QualType BuildBlockPointerType(QualType T, - SourceLocation Loc, DeclarationName Entity); - QualType BuildParenType(QualType T); - QualType BuildAtomicType(QualType T, SourceLocation Loc); - QualType BuildReadPipeType(QualType T, - SourceLocation Loc); - QualType BuildWritePipeType(QualType T, - SourceLocation Loc); - QualType BuildBitIntType(bool IsUnsigned, Expr *BitWidth, SourceLocation Loc); + void CheckAbsoluteValueFunction(const CallExpr *Call, + const FunctionDecl *FDecl); - TypeSourceInfo *GetTypeForDeclarator(Declarator &D); - TypeSourceInfo *GetTypeForDeclaratorCast(Declarator &D, QualType FromTy); + void CheckMaxUnsignedZero(const CallExpr *Call, const FunctionDecl *FDecl); - /// Package the given type and TSI into a ParsedType. - ParsedType CreateParsedType(QualType T, TypeSourceInfo *TInfo); - DeclarationNameInfo GetNameForDeclarator(Declarator &D); - DeclarationNameInfo GetNameFromUnqualifiedId(const UnqualifiedId &Name); - static QualType GetTypeFromParser(ParsedType Ty, - TypeSourceInfo **TInfo = nullptr); - CanThrowResult canThrow(const Stmt *E); - /// Determine whether the callee of a particular function call can throw. - /// E, D and Loc are all optional. - static CanThrowResult canCalleeThrow(Sema &S, const Expr *E, const Decl *D, - SourceLocation Loc = SourceLocation()); - const FunctionProtoType *ResolveExceptionSpec(SourceLocation Loc, - const FunctionProtoType *FPT); - void UpdateExceptionSpec(FunctionDecl *FD, - const FunctionProtoType::ExceptionSpecInfo &ESI); - bool CheckSpecifiedExceptionType(QualType &T, SourceRange Range); - bool CheckDistantExceptionSpec(QualType T); - bool CheckEquivalentExceptionSpec(FunctionDecl *Old, FunctionDecl *New); - bool CheckEquivalentExceptionSpec( - const FunctionProtoType *Old, SourceLocation OldLoc, - const FunctionProtoType *New, SourceLocation NewLoc); - bool CheckEquivalentExceptionSpec( - const PartialDiagnostic &DiagID, const PartialDiagnostic & NoteID, - const FunctionProtoType *Old, SourceLocation OldLoc, - const FunctionProtoType *New, SourceLocation NewLoc); - bool handlerCanCatch(QualType HandlerType, QualType ExceptionType); - bool CheckExceptionSpecSubset( - const PartialDiagnostic &DiagID, const PartialDiagnostic &NestedDiagID, - const PartialDiagnostic &NoteID, const PartialDiagnostic &NoThrowDiagID, - const FunctionProtoType *Superset, bool SkipSupersetFirstParameter, - SourceLocation SuperLoc, const FunctionProtoType *Subset, - bool SkipSubsetFirstParameter, SourceLocation SubLoc); - bool CheckParamExceptionSpec( - const PartialDiagnostic &NestedDiagID, const PartialDiagnostic &NoteID, - const FunctionProtoType *Target, bool SkipTargetFirstParameter, - SourceLocation TargetLoc, const FunctionProtoType *Source, - bool SkipSourceFirstParameter, SourceLocation SourceLoc); + void CheckMemaccessArguments(const CallExpr *Call, unsigned BId, + IdentifierInfo *FnName); - TypeResult ActOnTypeName(Declarator &D); + void CheckStrlcpycatArguments(const CallExpr *Call, IdentifierInfo *FnName); - /// The parser has parsed the context-sensitive type 'instancetype' - /// in an Objective-C message declaration. Return the appropriate type. - ParsedType ActOnObjCInstanceType(SourceLocation Loc); + void CheckStrncatArguments(const CallExpr *Call, IdentifierInfo *FnName); - /// Abstract class used to diagnose incomplete types. - struct TypeDiagnoser { - TypeDiagnoser() {} + void CheckFreeArguments(const CallExpr *E); - virtual void diagnose(Sema &S, SourceLocation Loc, QualType T) = 0; - virtual ~TypeDiagnoser() {} - }; + void CheckReturnValExpr(Expr *RetValExp, QualType lhsType, + SourceLocation ReturnLoc, bool isObjCMethod = false, + const AttrVec *Attrs = nullptr, + const FunctionDecl *FD = nullptr); - static int getPrintable(int I) { return I; } - static unsigned getPrintable(unsigned I) { return I; } - static bool getPrintable(bool B) { return B; } - static const char * getPrintable(const char *S) { return S; } - static StringRef getPrintable(StringRef S) { return S; } - static const std::string &getPrintable(const std::string &S) { return S; } - static const IdentifierInfo *getPrintable(const IdentifierInfo *II) { - return II; - } - static DeclarationName getPrintable(DeclarationName N) { return N; } - static QualType getPrintable(QualType T) { return T; } - static SourceRange getPrintable(SourceRange R) { return R; } - static SourceRange getPrintable(SourceLocation L) { return L; } - static SourceRange getPrintable(const Expr *E) { return E->getSourceRange(); } - static SourceRange getPrintable(TypeLoc TL) { return TL.getSourceRange();} + void CheckImplicitConversions(Expr *E, SourceLocation CC = SourceLocation()); + void CheckBoolLikeConversion(Expr *E, SourceLocation CC); + void CheckForIntOverflow(const Expr *E); + void CheckUnsequencedOperations(const Expr *E); - template class BoundTypeDiagnoser : public TypeDiagnoser { - protected: - unsigned DiagID; - std::tuple Args; - - template - void emit(const SemaDiagnosticBuilder &DB, - std::index_sequence) const { - // Apply all tuple elements to the builder in order. - bool Dummy[] = {false, (DB << getPrintable(std::get(Args)))...}; - (void)Dummy; - } + /// Perform semantic checks on a completed expression. This will either + /// be a full-expression or a default argument expression. + void CheckCompletedExpr(Expr *E, SourceLocation CheckLoc = SourceLocation(), + bool IsConstexpr = false); - public: - BoundTypeDiagnoser(unsigned DiagID, const Ts &...Args) - : TypeDiagnoser(), DiagID(DiagID), Args(Args...) { - assert(DiagID != 0 && "no diagnostic for type diagnoser"); - } + void CheckBitFieldInitialization(SourceLocation InitLoc, FieldDecl *Field, + Expr *Init); + /// Check whether receiver is mutable ObjC container which + /// attempts to add itself into the container + void CheckObjCCircularContainer(ObjCMessageExpr *Message); - void diagnose(Sema &S, SourceLocation Loc, QualType T) override { - const SemaDiagnosticBuilder &DB = S.Diag(Loc, DiagID); - emit(DB, std::index_sequence_for()); - DB << T; - } - }; + void CheckTCBEnforcement(const SourceLocation CallExprLoc, + const NamedDecl *Callee); - /// Do a check to make sure \p Name looks like a legal argument for the - /// swift_name attribute applied to decl \p D. Raise a diagnostic if the name - /// is invalid for the given declaration. - /// - /// \p AL is used to provide caret diagnostics in case of a malformed name. - /// - /// \returns true if the name is a valid swift name for \p D, false otherwise. - bool DiagnoseSwiftName(Decl *D, StringRef Name, SourceLocation Loc, - const ParsedAttr &AL, bool IsAsync); + /// A map from magic value to type information. + std::unique_ptr> + TypeTagForDatatypeMagicValues; - /// A derivative of BoundTypeDiagnoser for which the diagnostic's type - /// parameter is preceded by a 0/1 enum that is 1 if the type is sizeless. - /// For example, a diagnostic with no other parameters would generally have - /// the form "...%select{incomplete|sizeless}0 type %1...". - template - class SizelessTypeDiagnoser : public BoundTypeDiagnoser { - public: - SizelessTypeDiagnoser(unsigned DiagID, const Ts &... Args) - : BoundTypeDiagnoser(DiagID, Args...) {} + /// Peform checks on a call of a function with argument_with_type_tag + /// or pointer_with_type_tag attributes. + void CheckArgumentWithTypeTag(const ArgumentWithTypeTagAttr *Attr, + const ArrayRef ExprArgs, + SourceLocation CallSiteLoc); - void diagnose(Sema &S, SourceLocation Loc, QualType T) override { - const SemaDiagnosticBuilder &DB = S.Diag(Loc, this->DiagID); - this->emit(DB, std::index_sequence_for()); - DB << T->isSizelessType() << T; - } - }; + /// Check if we are taking the address of a packed field + /// as this may be a problem if the pointer value is dereferenced. + void CheckAddressOfPackedMember(Expr *rhs); - enum class CompleteTypeKind { - /// Apply the normal rules for complete types. In particular, - /// treat all sizeless types as incomplete. - Normal, + /// Helper class that collects misaligned member designations and + /// their location info for delayed diagnostics. + struct MisalignedMember { + Expr *E; + RecordDecl *RD; + ValueDecl *MD; + CharUnits Alignment; - /// Relax the normal rules for complete types so that they include - /// sizeless built-in types. - AcceptSizeless, + MisalignedMember() : E(), RD(), MD() {} + MisalignedMember(Expr *E, RecordDecl *RD, ValueDecl *MD, + CharUnits Alignment) + : E(E), RD(RD), MD(MD), Alignment(Alignment) {} + explicit MisalignedMember(Expr *E) + : MisalignedMember(E, nullptr, nullptr, CharUnits()) {} - // FIXME: Eventually we should flip the default to Normal and opt in - // to AcceptSizeless rather than opt out of it. - Default = AcceptSizeless + bool operator==(const MisalignedMember &m) { return this->E == m.E; } }; + /// Small set of gathered accesses to potentially misaligned members + /// due to the packed attribute. + SmallVector MisalignedMembers; - enum class AcceptableKind { Visible, Reachable }; + /// Adds an expression to the set of gathered misaligned members. + void AddPotentialMisalignedMembers(Expr *E, RecordDecl *RD, ValueDecl *MD, + CharUnits Alignment); + ///@} -private: - /// Methods for marking which expressions involve dereferencing a pointer - /// marked with the 'noderef' attribute. Expressions are checked bottom up as - /// they are parsed, meaning that a noderef pointer may not be accessed. For - /// example, in `&*p` where `p` is a noderef pointer, we will first parse the - /// `*p`, but need to check that `address of` is called on it. This requires - /// keeping a container of all pending expressions and checking if the address - /// of them are eventually taken. - void CheckSubscriptAccessOfNoDeref(const ArraySubscriptExpr *E); - void CheckAddressOfNoDeref(const Expr *E); - void CheckMemberAccessOfNoDeref(const MemberExpr *E); + // + // + // ------------------------------------------------------------------------- + // + // - bool RequireCompleteTypeImpl(SourceLocation Loc, QualType T, - CompleteTypeKind Kind, TypeDiagnoser *Diagnoser); + /// \name C++ Coroutines + /// Implementations are in SemaCoroutine.cpp + ///@{ - struct ModuleScope { - SourceLocation BeginLoc; - clang::Module *Module = nullptr; - VisibleModuleSet OuterVisibleModules; - }; - /// The modules we're currently parsing. - llvm::SmallVector ModuleScopes; +public: + /// The C++ "std::coroutine_traits" template, which is defined in + /// \ + ClassTemplateDecl *StdCoroutineTraitsCache; - /// For an interface unit, this is the implicitly imported interface unit. - clang::Module *ThePrimaryInterface = nullptr; + bool ActOnCoroutineBodyStart(Scope *S, SourceLocation KwLoc, + StringRef Keyword); + ExprResult ActOnCoawaitExpr(Scope *S, SourceLocation KwLoc, Expr *E); + ExprResult ActOnCoyieldExpr(Scope *S, SourceLocation KwLoc, Expr *E); + StmtResult ActOnCoreturnStmt(Scope *S, SourceLocation KwLoc, Expr *E); - /// The explicit global module fragment of the current translation unit. - /// The explicit Global Module Fragment, as specified in C++ - /// [module.global.frag]. - clang::Module *TheGlobalModuleFragment = nullptr; + ExprResult BuildOperatorCoawaitLookupExpr(Scope *S, SourceLocation Loc); + ExprResult BuildOperatorCoawaitCall(SourceLocation Loc, Expr *E, + UnresolvedLookupExpr *Lookup); + ExprResult BuildResolvedCoawaitExpr(SourceLocation KwLoc, Expr *Operand, + Expr *Awaiter, bool IsImplicit = false); + ExprResult BuildUnresolvedCoawaitExpr(SourceLocation KwLoc, Expr *Operand, + UnresolvedLookupExpr *Lookup); + ExprResult BuildCoyieldExpr(SourceLocation KwLoc, Expr *E); + StmtResult BuildCoreturnStmt(SourceLocation KwLoc, Expr *E, + bool IsImplicit = false); + StmtResult BuildCoroutineBodyStmt(CoroutineBodyStmt::CtorArgs); + bool buildCoroutineParameterMoves(SourceLocation Loc); + VarDecl *buildCoroutinePromise(SourceLocation Loc); + void CheckCompletedCoroutineBody(FunctionDecl *FD, Stmt *&Body); - /// The implicit global module fragments of the current translation unit. - /// - /// The contents in the implicit global module fragment can't be discarded. - clang::Module *TheImplicitGlobalModuleFragment = nullptr; + // As a clang extension, enforces that a non-coroutine function must be marked + // with [[clang::coro_wrapper]] if it returns a type marked with + // [[clang::coro_return_type]]. + // Expects that FD is not a coroutine. + void CheckCoroutineWrapper(FunctionDecl *FD); + /// Lookup 'coroutine_traits' in std namespace and std::experimental + /// namespace. The namespace found is recorded in Namespace. + ClassTemplateDecl *lookupCoroutineTraits(SourceLocation KwLoc, + SourceLocation FuncLoc); + /// Check that the expression co_await promise.final_suspend() shall not be + /// potentially-throwing. + bool checkFinalSuspendNoThrow(const Stmt *FinalSuspend); - /// Namespace definitions that we will export when they finish. - llvm::SmallPtrSet DeferredExportedNamespaces; + ///@} - /// In a C++ standard module, inline declarations require a definition to be - /// present at the end of a definition domain. This set holds the decls to - /// be checked at the end of the TU. - llvm::SmallPtrSet PendingInlineFuncDecls; + // + // + // ------------------------------------------------------------------------- + // + // - /// Helper function to judge if we are in module purview. - /// Return false if we are not in a module. - bool isCurrentModulePurview() const; + /// \name C++ Scope Specifiers + /// Implementations are in SemaCXXScopeSpec.cpp + ///@{ - /// Enter the scope of the explicit global module fragment. - Module *PushGlobalModuleFragment(SourceLocation BeginLoc); - /// Leave the scope of the explicit global module fragment. - void PopGlobalModuleFragment(); +public: + // Marks SS invalid if it represents an incomplete type. + bool RequireCompleteDeclContext(CXXScopeSpec &SS, DeclContext *DC); + // Complete an enum decl, maybe without a scope spec. + bool RequireCompleteEnumDecl(EnumDecl *D, SourceLocation L, + CXXScopeSpec *SS = nullptr); - /// Enter the scope of an implicit global module fragment. - Module *PushImplicitGlobalModuleFragment(SourceLocation BeginLoc); - /// Leave the scope of an implicit global module fragment. - void PopImplicitGlobalModuleFragment(); + DeclContext *computeDeclContext(QualType T); + DeclContext *computeDeclContext(const CXXScopeSpec &SS, + bool EnteringContext = false); + bool isDependentScopeSpecifier(const CXXScopeSpec &SS); + CXXRecordDecl *getCurrentInstantiationOf(NestedNameSpecifier *NNS); - VisibleModuleSet VisibleModules; + /// The parser has parsed a global nested-name-specifier '::'. + /// + /// \param CCLoc The location of the '::'. + /// + /// \param SS The nested-name-specifier, which will be updated in-place + /// to reflect the parsed nested-name-specifier. + /// + /// \returns true if an error occurred, false otherwise. + bool ActOnCXXGlobalScopeSpecifier(SourceLocation CCLoc, CXXScopeSpec &SS); - /// Cache for module units which is usable for current module. - llvm::DenseSet UsableModuleUnitsCache; + /// The parser has parsed a '__super' nested-name-specifier. + /// + /// \param SuperLoc The location of the '__super' keyword. + /// + /// \param ColonColonLoc The location of the '::'. + /// + /// \param SS The nested-name-specifier, which will be updated in-place + /// to reflect the parsed nested-name-specifier. + /// + /// \returns true if an error occurred, false otherwise. + bool ActOnSuperScopeSpecifier(SourceLocation SuperLoc, + SourceLocation ColonColonLoc, CXXScopeSpec &SS); - bool isUsableModule(const Module *M); + bool isAcceptableNestedNameSpecifier(const NamedDecl *SD, + bool *CanCorrect = nullptr); + NamedDecl *FindFirstQualifierInScope(Scope *S, NestedNameSpecifier *NNS); - bool isAcceptableSlow(const NamedDecl *D, AcceptableKind Kind); + /// Keeps information about an identifier in a nested-name-spec. + /// + struct NestedNameSpecInfo { + /// The type of the object, if we're parsing nested-name-specifier in + /// a member access expression. + ParsedType ObjectType; -public: - /// Get the module unit whose scope we are currently within. - Module *getCurrentModule() const { - return ModuleScopes.empty() ? nullptr : ModuleScopes.back().Module; - } + /// The identifier preceding the '::'. + IdentifierInfo *Identifier; - /// Is the module scope we are an implementation unit? - bool currentModuleIsImplementation() const { - return ModuleScopes.empty() - ? false - : ModuleScopes.back().Module->isModuleImplementation(); - } + /// The location of the identifier. + SourceLocation IdentifierLoc; - /// Is the module scope we are in a C++ Header Unit? - bool currentModuleIsHeaderUnit() const { - return ModuleScopes.empty() ? false - : ModuleScopes.back().Module->isHeaderUnit(); - } + /// The location of the '::'. + SourceLocation CCLoc; - /// Get the module owning an entity. - Module *getOwningModule(const Decl *Entity) { - return Entity->getOwningModule(); - } + /// Creates info object for the most typical case. + NestedNameSpecInfo(IdentifierInfo *II, SourceLocation IdLoc, + SourceLocation ColonColonLoc, + ParsedType ObjectType = ParsedType()) + : ObjectType(ObjectType), Identifier(II), IdentifierLoc(IdLoc), + CCLoc(ColonColonLoc) {} - /// Make a merged definition of an existing hidden definition \p ND - /// visible at the specified location. - void makeMergedDefinitionVisible(NamedDecl *ND); + NestedNameSpecInfo(IdentifierInfo *II, SourceLocation IdLoc, + SourceLocation ColonColonLoc, QualType ObjectType) + : ObjectType(ParsedType::make(ObjectType)), Identifier(II), + IdentifierLoc(IdLoc), CCLoc(ColonColonLoc) {} + }; - bool isModuleVisible(const Module *M, bool ModulePrivate = false); + bool BuildCXXNestedNameSpecifier(Scope *S, NestedNameSpecInfo &IdInfo, + bool EnteringContext, CXXScopeSpec &SS, + NamedDecl *ScopeLookupResult, + bool ErrorRecoveryLookup, + bool *IsCorrectedToColon = nullptr, + bool OnlyNamespace = false); - // When loading a non-modular PCH files, this is used to restore module - // visibility. - void makeModuleVisible(Module *Mod, SourceLocation ImportLoc) { - VisibleModules.setVisible(Mod, ImportLoc); - } + /// The parser has parsed a nested-name-specifier 'identifier::'. + /// + /// \param S The scope in which this nested-name-specifier occurs. + /// + /// \param IdInfo Parser information about an identifier in the + /// nested-name-spec. + /// + /// \param EnteringContext Whether we're entering the context nominated by + /// this nested-name-specifier. + /// + /// \param SS The nested-name-specifier, which is both an input + /// parameter (the nested-name-specifier before this type) and an + /// output parameter (containing the full nested-name-specifier, + /// including this new type). + /// + /// \param IsCorrectedToColon If not null, suggestions to replace '::' -> ':' + /// are allowed. The bool value pointed by this parameter is set to 'true' + /// if the identifier is treated as if it was followed by ':', not '::'. + /// + /// \param OnlyNamespace If true, only considers namespaces in lookup. + /// + /// \returns true if an error occurred, false otherwise. + bool ActOnCXXNestedNameSpecifier(Scope *S, NestedNameSpecInfo &IdInfo, + bool EnteringContext, CXXScopeSpec &SS, + bool *IsCorrectedToColon = nullptr, + bool OnlyNamespace = false); - /// Determine whether a declaration is visible to name lookup. - bool isVisible(const NamedDecl *D) { - return D->isUnconditionallyVisible() || - isAcceptableSlow(D, AcceptableKind::Visible); - } + /// The parser has parsed a nested-name-specifier + /// 'template[opt] template-name < template-args >::'. + /// + /// \param S The scope in which this nested-name-specifier occurs. + /// + /// \param SS The nested-name-specifier, which is both an input + /// parameter (the nested-name-specifier before this type) and an + /// output parameter (containing the full nested-name-specifier, + /// including this new type). + /// + /// \param TemplateKWLoc the location of the 'template' keyword, if any. + /// \param TemplateName the template name. + /// \param TemplateNameLoc The location of the template name. + /// \param LAngleLoc The location of the opening angle bracket ('<'). + /// \param TemplateArgs The template arguments. + /// \param RAngleLoc The location of the closing angle bracket ('>'). + /// \param CCLoc The location of the '::'. + /// + /// \param EnteringContext Whether we're entering the context of the + /// nested-name-specifier. + /// + /// + /// \returns true if an error occurred, false otherwise. + bool ActOnCXXNestedNameSpecifier( + Scope *S, CXXScopeSpec &SS, SourceLocation TemplateKWLoc, + TemplateTy TemplateName, SourceLocation TemplateNameLoc, + SourceLocation LAngleLoc, ASTTemplateArgsPtr TemplateArgs, + SourceLocation RAngleLoc, SourceLocation CCLoc, bool EnteringContext); - /// Determine whether a declaration is reachable. - bool isReachable(const NamedDecl *D) { - // All visible declarations are reachable. - return D->isUnconditionallyVisible() || - isAcceptableSlow(D, AcceptableKind::Reachable); - } + bool ActOnCXXNestedNameSpecifierDecltype(CXXScopeSpec &SS, const DeclSpec &DS, + SourceLocation ColonColonLoc); - /// Determine whether a declaration is acceptable (visible/reachable). - bool isAcceptable(const NamedDecl *D, AcceptableKind Kind) { - return Kind == AcceptableKind::Visible ? isVisible(D) : isReachable(D); - } + bool ActOnCXXNestedNameSpecifierIndexedPack(CXXScopeSpec &SS, + const DeclSpec &DS, + SourceLocation ColonColonLoc, + QualType Type); - /// Determine whether any declaration of an entity is visible. - bool - hasVisibleDeclaration(const NamedDecl *D, - llvm::SmallVectorImpl *Modules = nullptr) { - return isVisible(D) || hasVisibleDeclarationSlow(D, Modules); - } + bool IsInvalidUnlessNestedName(Scope *S, CXXScopeSpec &SS, + NestedNameSpecInfo &IdInfo, + bool EnteringContext); - bool hasVisibleDeclarationSlow(const NamedDecl *D, - llvm::SmallVectorImpl *Modules); - /// Determine whether any declaration of an entity is reachable. - bool - hasReachableDeclaration(const NamedDecl *D, - llvm::SmallVectorImpl *Modules = nullptr) { - return isReachable(D) || hasReachableDeclarationSlow(D, Modules); - } - bool hasReachableDeclarationSlow( - const NamedDecl *D, llvm::SmallVectorImpl *Modules = nullptr); + /// Given a C++ nested-name-specifier, produce an annotation value + /// that the parser can use later to reconstruct the given + /// nested-name-specifier. + /// + /// \param SS A nested-name-specifier. + /// + /// \returns A pointer containing all of the information in the + /// nested-name-specifier \p SS. + void *SaveNestedNameSpecifierAnnotation(CXXScopeSpec &SS); - bool hasVisibleMergedDefinition(const NamedDecl *Def); - bool hasMergedDefinitionInCurrentModule(const NamedDecl *Def); + /// Given an annotation pointer for a nested-name-specifier, restore + /// the nested-name-specifier structure. + /// + /// \param Annotation The annotation pointer, produced by + /// \c SaveNestedNameSpecifierAnnotation(). + /// + /// \param AnnotationRange The source range corresponding to the annotation. + /// + /// \param SS The nested-name-specifier that will be updated with the contents + /// of the annotation pointer. + void RestoreNestedNameSpecifierAnnotation(void *Annotation, + SourceRange AnnotationRange, + CXXScopeSpec &SS); - /// Determine if \p D and \p Suggested have a structurally compatible - /// layout as described in C11 6.2.7/1. - bool hasStructuralCompatLayout(Decl *D, Decl *Suggested); + bool ShouldEnterDeclaratorScope(Scope *S, const CXXScopeSpec &SS); - /// Determine if \p D has a visible definition. If not, suggest a declaration - /// that should be made visible to expose the definition. - bool hasVisibleDefinition(NamedDecl *D, NamedDecl **Suggested, - bool OnlyNeedComplete = false); - bool hasVisibleDefinition(const NamedDecl *D) { - NamedDecl *Hidden; - return hasVisibleDefinition(const_cast(D), &Hidden); - } + /// ActOnCXXEnterDeclaratorScope - Called when a C++ scope specifier (global + /// scope or nested-name-specifier) is parsed, part of a declarator-id. + /// After this method is called, according to [C++ 3.4.3p3], names should be + /// looked up in the declarator-id's scope, until the declarator is parsed and + /// ActOnCXXExitDeclaratorScope is called. + /// The 'SS' should be a non-empty valid CXXScopeSpec. + bool ActOnCXXEnterDeclaratorScope(Scope *S, CXXScopeSpec &SS); - /// Determine if \p D has a reachable definition. If not, suggest a - /// declaration that should be made reachable to expose the definition. - bool hasReachableDefinition(NamedDecl *D, NamedDecl **Suggested, - bool OnlyNeedComplete = false); - bool hasReachableDefinition(NamedDecl *D) { - NamedDecl *Hidden; - return hasReachableDefinition(D, &Hidden); - } + /// ActOnCXXExitDeclaratorScope - Called when a declarator that previously + /// invoked ActOnCXXEnterDeclaratorScope(), is finished. 'SS' is the same + /// CXXScopeSpec that was passed to ActOnCXXEnterDeclaratorScope as well. + /// Used to indicate that names should revert to being looked up in the + /// defining scope. + void ActOnCXXExitDeclaratorScope(Scope *S, const CXXScopeSpec &SS); - bool hasAcceptableDefinition(NamedDecl *D, NamedDecl **Suggested, - AcceptableKind Kind, - bool OnlyNeedComplete = false); - bool hasAcceptableDefinition(NamedDecl *D, AcceptableKind Kind) { - NamedDecl *Hidden; - return hasAcceptableDefinition(D, &Hidden, Kind); - } + ///@} - /// Determine if the template parameter \p D has a visible default argument. - bool - hasVisibleDefaultArgument(const NamedDecl *D, - llvm::SmallVectorImpl *Modules = nullptr); - /// Determine if the template parameter \p D has a reachable default argument. - bool hasReachableDefaultArgument( - const NamedDecl *D, llvm::SmallVectorImpl *Modules = nullptr); - /// Determine if the template parameter \p D has a reachable default argument. - bool hasAcceptableDefaultArgument(const NamedDecl *D, - llvm::SmallVectorImpl *Modules, - Sema::AcceptableKind Kind); + // + // + // ------------------------------------------------------------------------- + // + // - /// Determine if there is a visible declaration of \p D that is an explicit - /// specialization declaration for a specialization of a template. (For a - /// member specialization, use hasVisibleMemberSpecialization.) - bool hasVisibleExplicitSpecialization( - const NamedDecl *D, llvm::SmallVectorImpl *Modules = nullptr); - /// Determine if there is a reachable declaration of \p D that is an explicit - /// specialization declaration for a specialization of a template. (For a - /// member specialization, use hasReachableMemberSpecialization.) - bool hasReachableExplicitSpecialization( - const NamedDecl *D, llvm::SmallVectorImpl *Modules = nullptr); + /// \name Declarations + /// Implementations are in SemaDecl.cpp + ///@{ - /// Determine if there is a visible declaration of \p D that is a member - /// specialization declaration (as opposed to an instantiated declaration). - bool hasVisibleMemberSpecialization( - const NamedDecl *D, llvm::SmallVectorImpl *Modules = nullptr); - /// Determine if there is a reachable declaration of \p D that is a member - /// specialization declaration (as opposed to an instantiated declaration). - bool hasReachableMemberSpecialization( - const NamedDecl *D, llvm::SmallVectorImpl *Modules = nullptr); +public: + IdentifierResolver IdResolver; - /// Determine if \p A and \p B are equivalent internal linkage declarations - /// from different modules, and thus an ambiguity error can be downgraded to - /// an extension warning. - bool isEquivalentInternalLinkageDeclaration(const NamedDecl *A, - const NamedDecl *B); - void diagnoseEquivalentInternalLinkageDeclarations( - SourceLocation Loc, const NamedDecl *D, - ArrayRef Equiv); + /// The index of the first InventedParameterInfo that refers to the current + /// context. + unsigned InventedParameterInfosStart = 0; - bool isUsualDeallocationFunction(const CXXMethodDecl *FD); + /// A RAII object to temporarily push a declaration context. + class ContextRAII { + private: + Sema &S; + DeclContext *SavedContext; + ProcessingContextState SavedContextState; + QualType SavedCXXThisTypeOverride; + unsigned SavedFunctionScopesStart; + unsigned SavedInventedParameterInfosStart; - // Check whether the size of array element of type \p EltTy is a multiple of - // its alignment and return false if it isn't. - bool checkArrayElementAlignment(QualType EltTy, SourceLocation Loc); + public: + ContextRAII(Sema &S, DeclContext *ContextToPush, bool NewThisContext = true) + : S(S), SavedContext(S.CurContext), + SavedContextState(S.DelayedDiagnostics.pushUndelayed()), + SavedCXXThisTypeOverride(S.CXXThisTypeOverride), + SavedFunctionScopesStart(S.FunctionScopesStart), + SavedInventedParameterInfosStart(S.InventedParameterInfosStart) { + assert(ContextToPush && "pushing null context"); + S.CurContext = ContextToPush; + if (NewThisContext) + S.CXXThisTypeOverride = QualType(); + // Any saved FunctionScopes do not refer to this context. + S.FunctionScopesStart = S.FunctionScopes.size(); + S.InventedParameterInfosStart = S.InventedParameterInfos.size(); + } - bool isCompleteType(SourceLocation Loc, QualType T, - CompleteTypeKind Kind = CompleteTypeKind::Default) { - return !RequireCompleteTypeImpl(Loc, T, Kind, nullptr); - } - bool RequireCompleteType(SourceLocation Loc, QualType T, - CompleteTypeKind Kind, TypeDiagnoser &Diagnoser); - bool RequireCompleteType(SourceLocation Loc, QualType T, - CompleteTypeKind Kind, unsigned DiagID); + void pop() { + if (!SavedContext) + return; + S.CurContext = SavedContext; + S.DelayedDiagnostics.popUndelayed(SavedContextState); + S.CXXThisTypeOverride = SavedCXXThisTypeOverride; + S.FunctionScopesStart = SavedFunctionScopesStart; + S.InventedParameterInfosStart = SavedInventedParameterInfosStart; + SavedContext = nullptr; + } - bool RequireCompleteType(SourceLocation Loc, QualType T, - TypeDiagnoser &Diagnoser) { - return RequireCompleteType(Loc, T, CompleteTypeKind::Default, Diagnoser); - } - bool RequireCompleteType(SourceLocation Loc, QualType T, unsigned DiagID) { - return RequireCompleteType(Loc, T, CompleteTypeKind::Default, DiagID); - } + ~ContextRAII() { pop(); } + }; - template - bool RequireCompleteType(SourceLocation Loc, QualType T, unsigned DiagID, - const Ts &...Args) { - BoundTypeDiagnoser Diagnoser(DiagID, Args...); - return RequireCompleteType(Loc, T, Diagnoser); - } + void DiagnoseInvalidJumps(Stmt *Body); - template - bool RequireCompleteSizedType(SourceLocation Loc, QualType T, unsigned DiagID, - const Ts &... Args) { - SizelessTypeDiagnoser Diagnoser(DiagID, Args...); - return RequireCompleteType(Loc, T, CompleteTypeKind::Normal, Diagnoser); - } + /// The function definitions which were renamed as part of typo-correction + /// to match their respective declarations. We want to keep track of them + /// to ensure that we don't emit a "redefinition" error if we encounter a + /// correctly named definition after the renamed definition. + llvm::SmallPtrSet TypoCorrectedFunctionDefinitions; - /// Get the type of expression E, triggering instantiation to complete the - /// type if necessary -- that is, if the expression refers to a templated - /// static data member of incomplete array type. - /// - /// May still return an incomplete type if instantiation was not possible or - /// if the type is incomplete for a different reason. Use - /// RequireCompleteExprType instead if a diagnostic is expected for an - /// incomplete expression type. - QualType getCompletedType(Expr *E); + /// A cache of the flags available in enumerations with the flag_bits + /// attribute. + mutable llvm::DenseMap FlagBitsCache; - void completeExprArrayBound(Expr *E); - bool RequireCompleteExprType(Expr *E, CompleteTypeKind Kind, - TypeDiagnoser &Diagnoser); - bool RequireCompleteExprType(Expr *E, unsigned DiagID); + /// WeakUndeclaredIdentifiers - Identifiers contained in \#pragma weak before + /// declared. Rare. May alias another identifier, declared or undeclared. + /// + /// For aliases, the target identifier is used as a key for eventual + /// processing when the target is declared. For the single-identifier form, + /// the sole identifier is used as the key. Each entry is a `SetVector` + /// (ordered by parse order) of aliases (identified by the alias name) in case + /// of multiple aliases to the same undeclared identifier. + llvm::MapVector< + IdentifierInfo *, + llvm::SetVector< + WeakInfo, llvm::SmallVector, + llvm::SmallDenseSet>> + WeakUndeclaredIdentifiers; - template - bool RequireCompleteExprType(Expr *E, unsigned DiagID, const Ts &...Args) { - BoundTypeDiagnoser Diagnoser(DiagID, Args...); - return RequireCompleteExprType(E, CompleteTypeKind::Default, Diagnoser); - } + /// ExtnameUndeclaredIdentifiers - Identifiers contained in + /// \#pragma redefine_extname before declared. Used in Solaris system headers + /// to define functions that occur in multiple standards to call the version + /// in the currently selected standard. + llvm::DenseMap ExtnameUndeclaredIdentifiers; - template - bool RequireCompleteSizedExprType(Expr *E, unsigned DiagID, - const Ts &... Args) { - SizelessTypeDiagnoser Diagnoser(DiagID, Args...); - return RequireCompleteExprType(E, CompleteTypeKind::Normal, Diagnoser); - } + /// Set containing all typedefs that are likely unused. + llvm::SmallSetVector + UnusedLocalTypedefNameCandidates; - bool RequireLiteralType(SourceLocation Loc, QualType T, - TypeDiagnoser &Diagnoser); - bool RequireLiteralType(SourceLocation Loc, QualType T, unsigned DiagID); + typedef LazyVector + UnusedFileScopedDeclsType; - template - bool RequireLiteralType(SourceLocation Loc, QualType T, unsigned DiagID, - const Ts &...Args) { - BoundTypeDiagnoser Diagnoser(DiagID, Args...); - return RequireLiteralType(Loc, T, Diagnoser); - } + /// The set of file scoped decls seen so far that have not been used + /// and must warn if not used. Only contains the first declaration. + UnusedFileScopedDeclsType UnusedFileScopedDecls; - QualType getElaboratedType(ElaboratedTypeKeyword Keyword, - const CXXScopeSpec &SS, QualType T, - TagDecl *OwnedTagDecl = nullptr); + typedef LazyVector + TentativeDefinitionsType; - // Returns the underlying type of a decltype with the given expression. - QualType getDecltypeForExpr(Expr *E); + /// All the tentative definitions encountered in the TU. + TentativeDefinitionsType TentativeDefinitions; - QualType BuildTypeofExprType(Expr *E, TypeOfKind Kind); - /// If AsUnevaluated is false, E is treated as though it were an evaluated - /// context, such as when building a type for decltype(auto). - QualType BuildDecltypeType(Expr *E, bool AsUnevaluated = true); + /// All the external declarations encoutered and used in the TU. + SmallVector ExternalDeclarations; - QualType ActOnPackIndexingType(QualType Pattern, Expr *IndexExpr, - SourceLocation Loc, - SourceLocation EllipsisLoc); - QualType BuildPackIndexingType(QualType Pattern, Expr *IndexExpr, - SourceLocation Loc, SourceLocation EllipsisLoc, - bool FullySubstituted = false, - ArrayRef Expansions = {}); + /// Generally null except when we temporarily switch decl contexts, + /// like in \see ActOnObjCTemporaryExitContainerContext. + DeclContext *OriginalLexicalContext; - using UTTKind = UnaryTransformType::UTTKind; - QualType BuildUnaryTransformType(QualType BaseType, UTTKind UKind, - SourceLocation Loc); - QualType BuiltinEnumUnderlyingType(QualType BaseType, SourceLocation Loc); - QualType BuiltinAddPointer(QualType BaseType, SourceLocation Loc); - QualType BuiltinRemovePointer(QualType BaseType, SourceLocation Loc); - QualType BuiltinDecay(QualType BaseType, SourceLocation Loc); - QualType BuiltinAddReference(QualType BaseType, UTTKind UKind, - SourceLocation Loc); - QualType BuiltinRemoveExtent(QualType BaseType, UTTKind UKind, - SourceLocation Loc); - QualType BuiltinRemoveReference(QualType BaseType, UTTKind UKind, - SourceLocation Loc); - QualType BuiltinChangeCVRQualifiers(QualType BaseType, UTTKind UKind, - SourceLocation Loc); - QualType BuiltinChangeSignedness(QualType BaseType, UTTKind UKind, - SourceLocation Loc); + /// Is the module scope we are in a C++ Header Unit? + bool currentModuleIsHeaderUnit() const { + return ModuleScopes.empty() ? false + : ModuleScopes.back().Module->isHeaderUnit(); + } - //===--------------------------------------------------------------------===// - // Symbol table / Decl tracking callbacks: SemaDecl.cpp. - // + /// Get the module owning an entity. + Module *getOwningModule(const Decl *Entity) { + return Entity->getOwningModule(); + } struct SkipBodyInfo { SkipBodyInfo() = default; @@ -2751,8 +2786,6 @@ public: DeclGroupPtrTy ConvertDeclToDeclGroup(Decl *Ptr, Decl *OwnedType = nullptr); - void DiagnoseUseOfUnimplementedSelectors(); - ParsedType getTypeName(const IdentifierInfo &II, SourceLocation NameLoc, Scope *S, CXXScopeSpec *SS = nullptr, bool isClassName = false, bool HasTrailingDot = false, @@ -2765,10 +2798,8 @@ public: IdentifierInfo **CorrectedII = nullptr); TypeSpecifierType isTagName(IdentifierInfo &II, Scope *S); bool isMicrosoftMissingTypename(const CXXScopeSpec *SS, Scope *S); - void DiagnoseUnknownTypeName(IdentifierInfo *&II, - SourceLocation IILoc, - Scope *S, - CXXScopeSpec *SS, + void DiagnoseUnknownTypeName(IdentifierInfo *&II, SourceLocation IILoc, + Scope *S, CXXScopeSpec *SS, ParsedType &SuggestedType, bool IsTemplateName = false); @@ -2837,9 +2868,7 @@ public: NameClassification(const IdentifierInfo *Keyword) : Kind(NC_Keyword) {} - static NameClassification Error() { - return NameClassification(NC_Error); - } + static NameClassification Error() { return NameClassification(NC_Error); } static NameClassification Unknown() { return NameClassification(NC_Unknown); @@ -3013,9 +3042,6 @@ public: // diagnoseExprIntendedAsTemplateName. return false; } - void diagnoseExprIntendedAsTemplateName(Scope *S, ExprResult TemplateName, - SourceLocation Less, - SourceLocation Greater); void warnOnReservedIdentifier(const NamedDecl *D); @@ -3023,8 +3049,8 @@ public: NamedDecl *HandleDeclarator(Scope *S, Declarator &D, MultiTemplateParamsArg TemplateParameterLists); - bool tryToFixVariablyModifiedVarType(TypeSourceInfo *&TInfo, - QualType &T, SourceLocation Loc, + bool tryToFixVariablyModifiedVarType(TypeSourceInfo *&TInfo, QualType &T, + SourceLocation Loc, unsigned FailedFoldDiagID); void RegisterLocallyScopedExternCDecl(NamedDecl *ND, Scope *S); bool DiagnoseClassNameShadow(DeclContext *DC, DeclarationNameInfo Info); @@ -3032,16 +3058,7 @@ public: DeclarationName Name, SourceLocation Loc, TemplateIdAnnotation *TemplateId, bool IsMemberSpecialization); - void - diagnoseIgnoredQualifiers(unsigned DiagID, unsigned Quals, - SourceLocation FallbackLoc, - SourceLocation ConstQualLoc = SourceLocation(), - SourceLocation VolatileQualLoc = SourceLocation(), - SourceLocation RestrictQualLoc = SourceLocation(), - SourceLocation AtomicQualLoc = SourceLocation(), - SourceLocation UnalignedQualLoc = SourceLocation()); - static bool adjustContextForLocalExternDecl(DeclContext *&DC); void DiagnoseFunctionSpecifiers(const DeclSpec &DS); NamedDecl *getShadowedDeclaration(const TypedefNameDecl *D, const LookupResult &R); @@ -3058,68 +3075,35 @@ public: void DiagnoseShadowingLambdaDecls(const sema::LambdaScopeInfo *LSI); -private: - /// Map of current shadowing declarations to shadowed declarations. Warn if - /// it looks like the user is trying to modify the shadowing declaration. - llvm::DenseMap ShadowingDecls; - -public: - void CheckCastAlign(Expr *Op, QualType T, SourceRange TRange); void handleTagNumbering(const TagDecl *Tag, Scope *TagScope); void setTagNameForLinkagePurposes(TagDecl *TagFromDeclSpec, TypedefNameDecl *NewTD); void CheckTypedefForVariablyModifiedType(Scope *S, TypedefNameDecl *D); - NamedDecl* ActOnTypedefDeclarator(Scope* S, Declarator& D, DeclContext* DC, + NamedDecl *ActOnTypedefDeclarator(Scope *S, Declarator &D, DeclContext *DC, TypeSourceInfo *TInfo, LookupResult &Previous); - NamedDecl* ActOnTypedefNameDecl(Scope* S, DeclContext* DC, TypedefNameDecl *D, + NamedDecl *ActOnTypedefNameDecl(Scope *S, DeclContext *DC, TypedefNameDecl *D, LookupResult &Previous, bool &Redeclaration); NamedDecl *ActOnVariableDeclarator( Scope *S, Declarator &D, DeclContext *DC, TypeSourceInfo *TInfo, LookupResult &Previous, MultiTemplateParamsArg TemplateParamLists, bool &AddToScope, ArrayRef Bindings = std::nullopt); - NamedDecl * - ActOnDecompositionDeclarator(Scope *S, Declarator &D, - MultiTemplateParamsArg TemplateParamLists); - void DiagPlaceholderVariableDefinition(SourceLocation Loc); - bool DiagRedefinedPlaceholderFieldDecl(SourceLocation Loc, - RecordDecl *ClassDecl, - const IdentifierInfo *Name); + // Returns true if the variable declaration is a redeclaration bool CheckVariableDeclaration(VarDecl *NewVD, LookupResult &Previous); void CheckVariableDeclarationType(VarDecl *NewVD); - bool DeduceVariableDeclarationType(VarDecl *VDecl, bool DirectInit, - Expr *Init); void CheckCompleteVariableDeclaration(VarDecl *VD); - void CheckCompleteDecompositionDeclaration(DecompositionDecl *DD); - void MaybeSuggestAddingStaticToDecl(const FunctionDecl *D); - NamedDecl* ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC, + NamedDecl *ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC, TypeSourceInfo *TInfo, LookupResult &Previous, MultiTemplateParamsArg TemplateParamLists, bool &AddToScope); bool AddOverriddenMethods(CXXRecordDecl *DC, CXXMethodDecl *MD); - enum class CheckConstexprKind { - /// Diagnose issues that are non-constant or that are extensions. - Diagnose, - /// Identify whether this function satisfies the formal rules for constexpr - /// functions in the current lanugage mode (with no extensions). - CheckValid - }; - - bool CheckConstexprFunctionDefinition(const FunctionDecl *FD, - CheckConstexprKind Kind); - - void DiagnoseHiddenVirtualMethods(CXXMethodDecl *MD); - void FindHiddenVirtualMethods(CXXMethodDecl *MD, - SmallVectorImpl &OverloadedMethods); - void NoteHiddenVirtualMethods(CXXMethodDecl *MD, - SmallVectorImpl &OverloadedMethods); // Returns true if the function declaration is a redeclaration - bool CheckFunctionDeclaration(Scope *S, - FunctionDecl *NewFD, LookupResult &Previous, + bool CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD, + LookupResult &Previous, bool IsMemberSpecialization, bool DeclIsDefn); bool shouldLinkDependentDeclWithPrevious(Decl *D, Decl *OldDecl); bool canFullyTypeCheckRedeclaration(ValueDecl *NewD, ValueDecl *OldD, @@ -3138,8 +3122,7 @@ public: void CheckFunctionOrTemplateParamDeclarator(Scope *S, Declarator &D); Decl *ActOnParamDeclarator(Scope *S, Declarator &D, SourceLocation ExplicitThisLoc = {}); - ParmVarDecl *BuildParmVarDeclForTypedef(DeclContext *DC, - SourceLocation Loc, + ParmVarDecl *BuildParmVarDeclForTypedef(DeclContext *DC, SourceLocation Loc, QualType T); QualType AdjustParameterTypeForObjCAutoRefCount(QualType T, SourceLocation NameLoc, @@ -3148,17 +3131,6 @@ public: SourceLocation NameLoc, IdentifierInfo *Name, QualType T, TypeSourceInfo *TSInfo, StorageClass SC); - void ActOnParamDefaultArgument(Decl *param, - SourceLocation EqualLoc, - Expr *defarg); - void ActOnParamUnparsedDefaultArgument(Decl *param, SourceLocation EqualLoc, - SourceLocation ArgLoc); - void ActOnParamDefaultArgumentError(Decl *param, SourceLocation EqualLoc, - Expr* DefaultArg); - ExprResult ConvertParamDefaultArgument(ParmVarDecl *Param, Expr *DefaultArg, - SourceLocation EqualLoc); - void SetParamDefaultArgument(ParmVarDecl *Param, Expr *DefaultArg, - SourceLocation EqualLoc); // Contexts where using non-trivial C union types can be disallowed. This is // passed to err_non_trivial_c_union_in_invalid_context. @@ -3206,13 +3178,11 @@ public: void ActOnUninitializedDecl(Decl *dcl); void ActOnInitializerError(Decl *Dcl); - void ActOnPureSpecifier(Decl *D, SourceLocation PureSpecLoc); void ActOnCXXForRangeDecl(Decl *D); StmtResult ActOnCXXForRangeIdentifier(Scope *S, SourceLocation IdentLoc, IdentifierInfo *Ident, ParsedAttributes &Attrs); - void SetDeclDeleted(Decl *dcl, SourceLocation DelLoc); - void SetDeclDefaulted(Decl *dcl, SourceLocation DefaultLoc); + void CheckStaticLocalForDllExport(VarDecl *VD); void CheckThreadLocalForLargeAlignment(VarDecl *VD); void FinalizeDeclaration(Decl *D); @@ -3249,14 +3219,6 @@ public: Decl *ActOnStartOfFunctionDef(Scope *S, Decl *D, SkipBodyInfo *SkipBody = nullptr, FnBodyKind BodyKind = FnBodyKind::Other); - void SetFunctionBodyKind(Decl *D, SourceLocation Loc, FnBodyKind BodyKind); - void ActOnStartTrailingRequiresClause(Scope *S, Declarator &D); - ExprResult ActOnFinishTrailingRequiresClause(ExprResult ConstraintExpr); - ExprResult ActOnRequiresClause(ExprResult ConstraintExpr); - void ActOnStartOfObjCMethodDef(Scope *S, Decl *D); - bool isObjCMethodDecl(Decl *D) { - return D && isa(D); - } /// Determine whether we can delay parsing the body of a function or /// function template until it is used, assuming we don't care about emitting @@ -3277,10 +3239,6 @@ public: /// \c constexpr in C++11 or has an 'auto' return type in C++14). bool canSkipFunctionBody(Decl *D); - /// Determine whether \param D is function like (function or function - /// template) for parsing. - bool isDeclaratorFunctionLike(Declarator &D); - void computeNRVO(Stmt *Body, sema::FunctionScopeInfo *Scope); Decl *ActOnFinishFunctionBody(Decl *Decl, Stmt *Body); Decl *ActOnFinishFunctionBody(Decl *Decl, Stmt *Body, bool IsInstantiation); @@ -3302,159 +3260,29 @@ public: DiagnoseSizeOfParametersAndReturnValue(ArrayRef Parameters, QualType ReturnTy, NamedDecl *D); - void DiagnoseInvalidJumps(Stmt *Body); - Decl *ActOnFileScopeAsmDecl(Expr *expr, - SourceLocation AsmLoc, + Decl *ActOnFileScopeAsmDecl(Expr *expr, SourceLocation AsmLoc, SourceLocation RParenLoc); Decl *ActOnTopLevelStmtDecl(Stmt *Statement); - /// Handle a C++11 empty-declaration and attribute-declaration. - Decl *ActOnEmptyDeclaration(Scope *S, const ParsedAttributesView &AttrList, - SourceLocation SemiLoc); + void ActOnPopScope(SourceLocation Loc, Scope *S); - enum class ModuleDeclKind { - Interface, ///< 'export module X;' - Implementation, ///< 'module X;' - PartitionInterface, ///< 'export module X:Y;' - PartitionImplementation, ///< 'module X:Y;' - }; + Decl *ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS, DeclSpec &DS, + const ParsedAttributesView &DeclAttrs, + RecordDecl *&AnonRecord); + Decl *ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS, DeclSpec &DS, + const ParsedAttributesView &DeclAttrs, + MultiTemplateParamsArg TemplateParams, + bool IsExplicitInstantiation, + RecordDecl *&AnonRecord); - /// An enumeration to represent the transition of states in parsing module - /// fragments and imports. If we are not parsing a C++20 TU, or we find - /// an error in state transition, the state is set to NotACXX20Module. - enum class ModuleImportState { - FirstDecl, ///< Parsing the first decl in a TU. - GlobalFragment, ///< after 'module;' but before 'module X;' - ImportAllowed, ///< after 'module X;' but before any non-import decl. - ImportFinished, ///< after any non-import decl. - PrivateFragmentImportAllowed, ///< after 'module :private;' but before any - ///< non-import decl. - PrivateFragmentImportFinished, ///< after 'module :private;' but a - ///< non-import decl has already been seen. - NotACXX20Module ///< Not a C++20 TU, or an invalid state was found. - }; + Decl *BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS, AccessSpecifier AS, + RecordDecl *Record, + const PrintingPolicy &Policy); -private: - /// The parser has begun a translation unit to be compiled as a C++20 - /// Header Unit, helper for ActOnStartOfTranslationUnit() only. - void HandleStartOfHeaderUnit(); - -public: - /// The parser has processed a module-declaration that begins the definition - /// of a module interface or implementation. - DeclGroupPtrTy ActOnModuleDecl(SourceLocation StartLoc, - SourceLocation ModuleLoc, ModuleDeclKind MDK, - ModuleIdPath Path, ModuleIdPath Partition, - ModuleImportState &ImportState); - - /// The parser has processed a global-module-fragment declaration that begins - /// the definition of the global module fragment of the current module unit. - /// \param ModuleLoc The location of the 'module' keyword. - DeclGroupPtrTy ActOnGlobalModuleFragmentDecl(SourceLocation ModuleLoc); - - /// The parser has processed a private-module-fragment declaration that begins - /// the definition of the private module fragment of the current module unit. - /// \param ModuleLoc The location of the 'module' keyword. - /// \param PrivateLoc The location of the 'private' keyword. - DeclGroupPtrTy ActOnPrivateModuleFragmentDecl(SourceLocation ModuleLoc, - SourceLocation PrivateLoc); - - /// The parser has processed a module import declaration. - /// - /// \param StartLoc The location of the first token in the declaration. This - /// could be the location of an '@', 'export', or 'import'. - /// \param ExportLoc The location of the 'export' keyword, if any. - /// \param ImportLoc The location of the 'import' keyword. - /// \param Path The module toplevel name as an access path. - /// \param IsPartition If the name is for a partition. - DeclResult ActOnModuleImport(SourceLocation StartLoc, - SourceLocation ExportLoc, - SourceLocation ImportLoc, ModuleIdPath Path, - bool IsPartition = false); - DeclResult ActOnModuleImport(SourceLocation StartLoc, - SourceLocation ExportLoc, - SourceLocation ImportLoc, Module *M, - ModuleIdPath Path = {}); - - /// The parser has processed a module import translated from a - /// #include or similar preprocessing directive. - void ActOnModuleInclude(SourceLocation DirectiveLoc, Module *Mod); - void BuildModuleInclude(SourceLocation DirectiveLoc, Module *Mod); - - /// The parsed has entered a submodule. - void ActOnModuleBegin(SourceLocation DirectiveLoc, Module *Mod); - /// The parser has left a submodule. - void ActOnModuleEnd(SourceLocation DirectiveLoc, Module *Mod); - - /// Create an implicit import of the given module at the given - /// source location, for error recovery, if possible. - /// - /// This routine is typically used when an entity found by name lookup - /// is actually hidden within a module that we know about but the user - /// has forgotten to import. - void createImplicitModuleImportForErrorRecovery(SourceLocation Loc, - Module *Mod); - - /// Kinds of missing import. Note, the values of these enumerators correspond - /// to %select values in diagnostics. - enum class MissingImportKind { - Declaration, - Definition, - DefaultArgument, - ExplicitSpecialization, - PartialSpecialization - }; - - /// Diagnose that the specified declaration needs to be visible but - /// isn't, and suggest a module import that would resolve the problem. - void diagnoseMissingImport(SourceLocation Loc, const NamedDecl *Decl, - MissingImportKind MIK, bool Recover = true); - void diagnoseMissingImport(SourceLocation Loc, const NamedDecl *Decl, - SourceLocation DeclLoc, ArrayRef Modules, - MissingImportKind MIK, bool Recover); - - Decl *ActOnStartExportDecl(Scope *S, SourceLocation ExportLoc, - SourceLocation LBraceLoc); - Decl *ActOnFinishExportDecl(Scope *S, Decl *ExportDecl, - SourceLocation RBraceLoc); - - /// We've found a use of a templated declaration that would trigger an - /// implicit instantiation. Check that any relevant explicit specializations - /// and partial specializations are visible/reachable, and diagnose if not. - void checkSpecializationVisibility(SourceLocation Loc, NamedDecl *Spec); - void checkSpecializationReachability(SourceLocation Loc, NamedDecl *Spec); - - /// Retrieve a suitable printing policy for diagnostics. - PrintingPolicy getPrintingPolicy() const { - return getPrintingPolicy(Context, PP); - } - - /// Retrieve a suitable printing policy for diagnostics. - static PrintingPolicy getPrintingPolicy(const ASTContext &Ctx, - const Preprocessor &PP); - - /// Scope actions. - void ActOnPopScope(SourceLocation Loc, Scope *S); - void ActOnTranslationUnitScope(Scope *S); - - Decl *ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS, DeclSpec &DS, - const ParsedAttributesView &DeclAttrs, - RecordDecl *&AnonRecord); - Decl *ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS, DeclSpec &DS, - const ParsedAttributesView &DeclAttrs, - MultiTemplateParamsArg TemplateParams, - bool IsExplicitInstantiation, - RecordDecl *&AnonRecord); - - Decl *BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS, - AccessSpecifier AS, - RecordDecl *Record, - const PrintingPolicy &Policy); - - /// Called once it is known whether - /// a tag declaration is an anonymous union or struct. - void ActOnDefinedDeclarationSpecifier(Decl *D); + /// Called once it is known whether + /// a tag declaration is an anonymous union or struct. + void ActOnDefinedDeclarationSpecifier(Decl *D); void DiagPlaceholderFieldDeclDefinitions(RecordDecl *Record); @@ -3479,9 +3307,8 @@ public: /// what kind of non-tag type this is. NonTagKind getNonTagTypeDeclKind(const Decl *D, TagTypeKind TTK); - bool isAcceptableTagRedeclaration(const TagDecl *Previous, - TagTypeKind NewTag, bool isDefinition, - SourceLocation NewTagLoc, + bool isAcceptableTagRedeclaration(const TagDecl *Previous, TagTypeKind NewTag, + bool isDefinition, SourceLocation NewTagLoc, const IdentifierInfo *Name); enum TagUseKind { @@ -3513,106 +3340,21 @@ public: bool IsTypeSpecifier, bool IsTemplateParamOrArg, OffsetOfKind OOK, SkipBodyInfo *SkipBody = nullptr); - DeclResult ActOnTemplatedFriendTag(Scope *S, SourceLocation FriendLoc, - unsigned TagSpec, SourceLocation TagLoc, - CXXScopeSpec &SS, IdentifierInfo *Name, - SourceLocation NameLoc, - const ParsedAttributesView &Attr, - MultiTemplateParamsArg TempParamLists); - - TypeResult ActOnDependentTag(Scope *S, - unsigned TagSpec, - TagUseKind TUK, - const CXXScopeSpec &SS, - IdentifierInfo *Name, - SourceLocation TagLoc, - SourceLocation NameLoc); - - void ActOnDefs(Scope *S, Decl *TagD, SourceLocation DeclStart, - IdentifierInfo *ClassName, - SmallVectorImpl &Decls); Decl *ActOnField(Scope *S, Decl *TagD, SourceLocation DeclStart, Declarator &D, Expr *BitfieldWidth); FieldDecl *HandleField(Scope *S, RecordDecl *TagD, SourceLocation DeclStart, Declarator &D, Expr *BitfieldWidth, - InClassInitStyle InitStyle, - AccessSpecifier AS); - MSPropertyDecl *HandleMSProperty(Scope *S, RecordDecl *TagD, - SourceLocation DeclStart, Declarator &D, - Expr *BitfieldWidth, - InClassInitStyle InitStyle, - AccessSpecifier AS, - const ParsedAttr &MSPropertyAttr); + InClassInitStyle InitStyle, AccessSpecifier AS); FieldDecl *CheckFieldDecl(DeclarationName Name, QualType T, - TypeSourceInfo *TInfo, - RecordDecl *Record, SourceLocation Loc, - bool Mutable, Expr *BitfieldWidth, - InClassInitStyle InitStyle, - SourceLocation TSSL, - AccessSpecifier AS, NamedDecl *PrevDecl, - Declarator *D = nullptr); + TypeSourceInfo *TInfo, RecordDecl *Record, + SourceLocation Loc, bool Mutable, + Expr *BitfieldWidth, InClassInitStyle InitStyle, + SourceLocation TSSL, AccessSpecifier AS, + NamedDecl *PrevDecl, Declarator *D = nullptr); bool CheckNontrivialField(FieldDecl *FD); - void DiagnoseNontrivial(const CXXRecordDecl *Record, CXXSpecialMember CSM); - - enum TrivialABIHandling { - /// The triviality of a method unaffected by "trivial_abi". - TAH_IgnoreTrivialABI, - - /// The triviality of a method affected by "trivial_abi". - TAH_ConsiderTrivialABI - }; - - bool SpecialMemberIsTrivial(CXXMethodDecl *MD, CXXSpecialMember CSM, - TrivialABIHandling TAH = TAH_IgnoreTrivialABI, - bool Diagnose = false); - - /// For a defaulted function, the kind of defaulted function that it is. - class DefaultedFunctionKind { - unsigned SpecialMember : 8; - unsigned Comparison : 8; - - public: - DefaultedFunctionKind() - : SpecialMember(CXXInvalid), Comparison(llvm::to_underlying(DefaultedComparisonKind::None)) { - } - DefaultedFunctionKind(CXXSpecialMember CSM) - : SpecialMember(CSM), Comparison(llvm::to_underlying(DefaultedComparisonKind::None)) {} - DefaultedFunctionKind(DefaultedComparisonKind Comp) - : SpecialMember(CXXInvalid), Comparison(llvm::to_underlying(Comp)) {} - - bool isSpecialMember() const { return SpecialMember != CXXInvalid; } - bool isComparison() const { - return static_cast(Comparison) != DefaultedComparisonKind::None; - } - - explicit operator bool() const { - return isSpecialMember() || isComparison(); - } - - CXXSpecialMember asSpecialMember() const { return static_cast(SpecialMember); } - DefaultedComparisonKind asComparison() const { return static_cast(Comparison); } - - /// Get the index of this function kind for use in diagnostics. - unsigned getDiagnosticIndex() const { - static_assert(CXXInvalid > CXXDestructor, - "invalid should have highest index"); - static_assert((unsigned)DefaultedComparisonKind::None == 0, - "none should be equal to zero"); - return SpecialMember + Comparison; - } - }; - - DefaultedFunctionKind getDefaultedFunctionKind(const FunctionDecl *FD); - - CXXSpecialMember getSpecialMember(const CXXMethodDecl *MD) { - return getDefaultedFunctionKind(MD).asSpecialMember(); - } - DefaultedComparisonKind getDefaultedComparisonKind(const FunctionDecl *FD) { - return getDefaultedFunctionKind(FD).asComparison(); - } void ActOnLastBitfield(SourceLocation DeclStart, SmallVectorImpl &AllIvarDecls); @@ -3634,20 +3376,6 @@ public: /// in case of a structural mismatch. bool ActOnDuplicateDefinition(Decl *Prev, SkipBodyInfo &SkipBody); - /// Check ODR hashes for C/ObjC when merging types from modules. - /// Differently from C++, actually parse the body and reject in case - /// of a mismatch. - template ::value>> - bool ActOnDuplicateODRHashDefinition(T *Duplicate, T *Previous) { - if (Duplicate->getODRHash() != Previous->getODRHash()) - return false; - - // Make the previous decl visible. - makeMergedDefinitionVisible(Previous); - return true; - } - typedef void *SkippedDefinitionContext; /// Invoked when we enter a tag definition that we're skipping. @@ -3686,8 +3414,7 @@ public: EnumConstantDecl *CheckEnumConstant(EnumDecl *Enum, EnumConstantDecl *LastEnumConst, - SourceLocation IdLoc, - IdentifierInfo *Id, + SourceLocation IdLoc, IdentifierInfo *Id, Expr *val); bool CheckEnumUnderlyingType(TypeSourceInfo *TI); bool CheckEnumRedeclaration(SourceLocation EnumLoc, bool IsScoped, @@ -3716,34 +3443,15 @@ public: void EnterDeclaratorContext(Scope *S, DeclContext *DC); void ExitDeclaratorContext(Scope *S); - /// Enter a template parameter scope, after it's been associated with a particular - /// DeclContext. Causes lookup within the scope to chain through enclosing contexts - /// in the correct order. + /// Enter a template parameter scope, after it's been associated with a + /// particular DeclContext. Causes lookup within the scope to chain through + /// enclosing contexts in the correct order. void EnterTemplatedContext(Scope *S, DeclContext *DC); /// Push the parameters of D, which must be a function, into scope. - void ActOnReenterFunctionContext(Scope* S, Decl* D); + void ActOnReenterFunctionContext(Scope *S, Decl *D); void ActOnExitFunctionContext(); - /// If \p AllowLambda is true, treat lambda as function. - DeclContext *getFunctionLevelDeclContext(bool AllowLambda = false) const; - - /// Returns a pointer to the innermost enclosing function, or nullptr if the - /// current context is not inside a function. If \p AllowLambda is true, - /// this can return the call operator of an enclosing lambda, otherwise - /// lambdas are skipped when looking for an enclosing function. - FunctionDecl *getCurFunctionDecl(bool AllowLambda = false) const; - - /// getCurMethodDecl - If inside of a method body, this returns a pointer to - /// the method decl for the method being parsed. If we're currently - /// in a 'block', this returns the containing context. - ObjCMethodDecl *getCurMethodDecl(); - - /// getCurFunctionOrMethodDecl - Return the Decl for the current ObjC method - /// or C function we're in, otherwise return null. If we're currently - /// in a 'block', this returns the containing context. - NamedDecl *getCurFunctionOrMethodDecl() const; - /// Add this decl to the scope shadowed decl chains. void PushOnScopeChains(NamedDecl *D, Scope *S, bool AddToContext = true); @@ -3785,9516 +3493,9111 @@ public: AMK_OptionalProtocolImplementation }; - /// Describes the kind of priority given to an availability attribute. - /// - /// The sum of priorities deteremines the final priority of the attribute. - /// The final priority determines how the attribute will be merged. - /// An attribute with a lower priority will always remove higher priority - /// attributes for the specified platform when it is being applied. An - /// attribute with a higher priority will not be applied if the declaration - /// already has an availability attribute with a lower priority for the - /// specified platform. The final prirority values are not expected to match - /// the values in this enumeration, but instead should be treated as a plain - /// integer value. This enumeration just names the priority weights that are - /// used to calculate that final vaue. - enum AvailabilityPriority : int { - /// The availability attribute was specified explicitly next to the - /// declaration. - AP_Explicit = 0, + void mergeDeclAttributes(NamedDecl *New, Decl *Old, + AvailabilityMergeKind AMK = AMK_Redeclaration); + void MergeTypedefNameDecl(Scope *S, TypedefNameDecl *New, + LookupResult &OldDecls); + bool MergeFunctionDecl(FunctionDecl *New, NamedDecl *&Old, Scope *S, + bool MergeTypeWithOld, bool NewDeclIsDefn); + bool MergeCompatibleFunctionDecls(FunctionDecl *New, FunctionDecl *Old, + Scope *S, bool MergeTypeWithOld); + void mergeObjCMethodDecls(ObjCMethodDecl *New, ObjCMethodDecl *Old); + void MergeVarDecl(VarDecl *New, LookupResult &Previous); + void MergeVarDeclTypes(VarDecl *New, VarDecl *Old, bool MergeTypeWithOld); + bool checkVarDeclRedefinition(VarDecl *OldDefn, VarDecl *NewDefn); + void notePreviousDefinition(const NamedDecl *Old, SourceLocation New); - /// The availability attribute was applied using '#pragma clang attribute'. - AP_PragmaClangAttribute = 1, + void FilterLookupForScope(LookupResult &R, DeclContext *Ctx, Scope *S, + bool ConsiderLinkage, bool AllowInlineNamespace); - /// The availability attribute for a specific platform was inferred from - /// an availability attribute for another platform. - AP_InferredFromOtherPlatform = 2 - }; + bool CheckRedeclarationModuleOwnership(NamedDecl *New, NamedDecl *Old); + bool CheckRedeclarationExported(NamedDecl *New, NamedDecl *Old); + bool CheckRedeclarationInModule(NamedDecl *New, NamedDecl *Old); + bool IsRedefinitionInModule(const NamedDecl *New, const NamedDecl *Old) const; - /// Attribute merging methods. Return true if a new attribute was added. - AvailabilityAttr * - mergeAvailabilityAttr(NamedDecl *D, const AttributeCommonInfo &CI, - IdentifierInfo *Platform, bool Implicit, - VersionTuple Introduced, VersionTuple Deprecated, - VersionTuple Obsoleted, bool IsUnavailable, - StringRef Message, bool IsStrict, StringRef Replacement, - AvailabilityMergeKind AMK, int Priority); - TypeVisibilityAttr * - mergeTypeVisibilityAttr(Decl *D, const AttributeCommonInfo &CI, - TypeVisibilityAttr::VisibilityType Vis); - VisibilityAttr *mergeVisibilityAttr(Decl *D, const AttributeCommonInfo &CI, - VisibilityAttr::VisibilityType Vis); - UuidAttr *mergeUuidAttr(Decl *D, const AttributeCommonInfo &CI, - StringRef UuidAsWritten, MSGuidDecl *GuidDecl); - DLLImportAttr *mergeDLLImportAttr(Decl *D, const AttributeCommonInfo &CI); - DLLExportAttr *mergeDLLExportAttr(Decl *D, const AttributeCommonInfo &CI); - MSInheritanceAttr *mergeMSInheritanceAttr(Decl *D, - const AttributeCommonInfo &CI, - bool BestCase, - MSInheritanceModel Model); - ErrorAttr *mergeErrorAttr(Decl *D, const AttributeCommonInfo &CI, - StringRef NewUserDiagnostic); - FormatAttr *mergeFormatAttr(Decl *D, const AttributeCommonInfo &CI, - IdentifierInfo *Format, int FormatIdx, - int FirstArg); - SectionAttr *mergeSectionAttr(Decl *D, const AttributeCommonInfo &CI, - StringRef Name); - CodeSegAttr *mergeCodeSegAttr(Decl *D, const AttributeCommonInfo &CI, - StringRef Name); - AlwaysInlineAttr *mergeAlwaysInlineAttr(Decl *D, - const AttributeCommonInfo &CI, - const IdentifierInfo *Ident); - MinSizeAttr *mergeMinSizeAttr(Decl *D, const AttributeCommonInfo &CI); - SwiftNameAttr *mergeSwiftNameAttr(Decl *D, const SwiftNameAttr &SNA, - StringRef Name); - OptimizeNoneAttr *mergeOptimizeNoneAttr(Decl *D, - const AttributeCommonInfo &CI); - InternalLinkageAttr *mergeInternalLinkageAttr(Decl *D, const ParsedAttr &AL); - InternalLinkageAttr *mergeInternalLinkageAttr(Decl *D, - const InternalLinkageAttr &AL); - WebAssemblyImportNameAttr *mergeImportNameAttr( - Decl *D, const WebAssemblyImportNameAttr &AL); - WebAssemblyImportModuleAttr *mergeImportModuleAttr( - Decl *D, const WebAssemblyImportModuleAttr &AL); - EnforceTCBAttr *mergeEnforceTCBAttr(Decl *D, const EnforceTCBAttr &AL); - EnforceTCBLeafAttr *mergeEnforceTCBLeafAttr(Decl *D, - const EnforceTCBLeafAttr &AL); - BTFDeclTagAttr *mergeBTFDeclTagAttr(Decl *D, const BTFDeclTagAttr &AL); - HLSLNumThreadsAttr *mergeHLSLNumThreadsAttr(Decl *D, - const AttributeCommonInfo &AL, - int X, int Y, int Z); - HLSLShaderAttr *mergeHLSLShaderAttr(Decl *D, const AttributeCommonInfo &AL, - HLSLShaderAttr::ShaderType ShaderType); - HLSLParamModifierAttr * - mergeHLSLParamModifierAttr(Decl *D, const AttributeCommonInfo &AL, - HLSLParamModifierAttr::Spelling Spelling); - - void mergeDeclAttributes(NamedDecl *New, Decl *Old, - AvailabilityMergeKind AMK = AMK_Redeclaration); - void MergeTypedefNameDecl(Scope *S, TypedefNameDecl *New, - LookupResult &OldDecls); - bool MergeFunctionDecl(FunctionDecl *New, NamedDecl *&Old, Scope *S, - bool MergeTypeWithOld, bool NewDeclIsDefn); - bool MergeCompatibleFunctionDecls(FunctionDecl *New, FunctionDecl *Old, - Scope *S, bool MergeTypeWithOld); - void mergeObjCMethodDecls(ObjCMethodDecl *New, ObjCMethodDecl *Old); - void MergeVarDecl(VarDecl *New, LookupResult &Previous); - void MergeVarDeclTypes(VarDecl *New, VarDecl *Old, bool MergeTypeWithOld); - void MergeVarDeclExceptionSpecs(VarDecl *New, VarDecl *Old); - bool checkVarDeclRedefinition(VarDecl *OldDefn, VarDecl *NewDefn); - void notePreviousDefinition(const NamedDecl *Old, SourceLocation New); - bool MergeCXXFunctionDecl(FunctionDecl *New, FunctionDecl *Old, Scope *S); + bool ShouldWarnIfUnusedFileScopedDecl(const DeclaratorDecl *D) const; - // AssignmentAction - This is used by all the assignment diagnostic functions - // to represent what is actually causing the operation - enum AssignmentAction { - AA_Assigning, - AA_Passing, - AA_Returning, - AA_Converting, - AA_Initializing, - AA_Sending, - AA_Casting, - AA_Passing_CFAudited - }; + /// If it's a file scoped decl that must warn if not used, keep track + /// of it. + void MarkUnusedFileScopedDecl(const DeclaratorDecl *D); - /// C++ Overloading. - enum OverloadKind { - /// This is a legitimate overload: the existing declarations are - /// functions or function templates with different signatures. - Ovl_Overload, + typedef llvm::function_ref + DiagReceiverTy; - /// This is not an overload because the signature exactly matches - /// an existing declaration. - Ovl_Match, + void DiagnoseUnusedNestedTypedefs(const RecordDecl *D); + void DiagnoseUnusedNestedTypedefs(const RecordDecl *D, + DiagReceiverTy DiagReceiver); + void DiagnoseUnusedDecl(const NamedDecl *ND); + void DiagnoseUnusedDecl(const NamedDecl *ND, DiagReceiverTy DiagReceiver); - /// This is not an overload because the lookup results contain a - /// non-function. - Ovl_NonFunction - }; - OverloadKind CheckOverload(Scope *S, - FunctionDecl *New, - const LookupResult &OldDecls, - NamedDecl *&OldDecl, - bool UseMemberUsingDeclRules); - bool IsOverload(FunctionDecl *New, FunctionDecl *Old, - bool UseMemberUsingDeclRules, bool ConsiderCudaAttrs = true); + /// If VD is set but not otherwise used, diagnose, for a parameter or a + /// variable. + void DiagnoseUnusedButSetDecl(const VarDecl *VD, DiagReceiverTy DiagReceiver); - // Checks whether MD constitutes an override the base class method BaseMD. - // When checking for overrides, the object object members are ignored. - bool IsOverride(FunctionDecl *MD, FunctionDecl *BaseMD, - bool UseMemberUsingDeclRules, bool ConsiderCudaAttrs = true); + ObjCInterfaceDecl *getObjCInterfaceDecl(IdentifierInfo *&Id, + SourceLocation IdLoc, + bool TypoCorrection = false); - // Calculates whether the expression Constraint depends on an enclosing - // template, for the purposes of [temp.friend] p9. - // TemplateDepth is the 'depth' of the friend function, which is used to - // compare whether a declaration reference is referring to a containing - // template, or just the current friend function. A 'lower' TemplateDepth in - // the AST refers to a 'containing' template. As the constraint is - // uninstantiated, this is relative to the 'top' of the TU. - bool - ConstraintExpressionDependsOnEnclosingTemplate(const FunctionDecl *Friend, - unsigned TemplateDepth, - const Expr *Constraint); + Scope *getNonFieldDeclScope(Scope *S); - // Calculates whether the friend function depends on an enclosing template for - // the purposes of [temp.friend] p9. - bool FriendConstraintsDependOnEnclosingTemplate(const FunctionDecl *FD); + FunctionDecl *CreateBuiltin(IdentifierInfo *II, QualType Type, unsigned ID, + SourceLocation Loc); + NamedDecl *LazilyCreateBuiltin(IdentifierInfo *II, unsigned ID, Scope *S, + bool ForRedeclaration, SourceLocation Loc); - enum class AllowedExplicit { - /// Allow no explicit functions to be used. - None, - /// Allow explicit conversion functions but not explicit constructors. - Conversions, - /// Allow both explicit conversion functions and explicit constructors. - All - }; + /// Get the outermost AttributedType node that sets a calling convention. + /// Valid types should not have multiple attributes with different CCs. + const AttributedType *getCallingConvAttributedType(QualType T) const; - ImplicitConversionSequence - TryImplicitConversion(Expr *From, QualType ToType, - bool SuppressUserConversions, - AllowedExplicit AllowExplicit, - bool InOverloadResolution, - bool CStyle, - bool AllowObjCWritebackConversion); + DeclarationNameInfo GetNameForDeclarator(Declarator &D); + DeclarationNameInfo GetNameFromUnqualifiedId(const UnqualifiedId &Name); - bool IsIntegralPromotion(Expr *From, QualType FromType, QualType ToType); - bool IsFloatingPointPromotion(QualType FromType, QualType ToType); - bool IsComplexPromotion(QualType FromType, QualType ToType); - bool IsPointerConversion(Expr *From, QualType FromType, QualType ToType, - bool InOverloadResolution, - QualType& ConvertedType, bool &IncompatibleObjC); - bool isObjCPointerConversion(QualType FromType, QualType ToType, - QualType& ConvertedType, bool &IncompatibleObjC); - bool isObjCWritebackConversion(QualType FromType, QualType ToType, - QualType &ConvertedType); - bool IsBlockPointerConversion(QualType FromType, QualType ToType, - QualType& ConvertedType); + /// ParsingInitForAutoVars - a set of declarations with auto types for which + /// we are currently parsing the initializer. + llvm::SmallPtrSet ParsingInitForAutoVars; - bool FunctionParamTypesAreEqual(ArrayRef Old, - ArrayRef New, - unsigned *ArgPos = nullptr, - bool Reversed = false); + /// Look for a locally scoped extern "C" declaration by the given name. + NamedDecl *findLocallyScopedExternCDecl(DeclarationName Name); - bool FunctionParamTypesAreEqual(const FunctionProtoType *OldType, - const FunctionProtoType *NewType, - unsigned *ArgPos = nullptr, - bool Reversed = false); + bool inferObjCARCLifetime(ValueDecl *decl); - bool FunctionNonObjectParamTypesAreEqual(const FunctionDecl *OldFunction, - const FunctionDecl *NewFunction, - unsigned *ArgPos = nullptr, - bool Reversed = false); + void deduceOpenCLAddressSpace(ValueDecl *decl); - void HandleFunctionTypeMismatch(PartialDiagnostic &PDiag, - QualType FromType, QualType ToType); + static bool adjustContextForLocalExternDecl(DeclContext *&DC); - void maybeExtendBlockObject(ExprResult &E); - CastKind PrepareCastToObjCObjectPointer(ExprResult &E); - bool CheckPointerConversion(Expr *From, QualType ToType, - CastKind &Kind, - CXXCastPath& BasePath, - bool IgnoreBaseAccess, - bool Diagnose = true); - bool IsMemberPointerConversion(Expr *From, QualType FromType, QualType ToType, - bool InOverloadResolution, - QualType &ConvertedType); - bool CheckMemberPointerConversion(Expr *From, QualType ToType, - CastKind &Kind, - CXXCastPath &BasePath, - bool IgnoreBaseAccess); - bool IsQualificationConversion(QualType FromType, QualType ToType, - bool CStyle, bool &ObjCLifetimeConversion); - bool IsFunctionConversion(QualType FromType, QualType ToType, - QualType &ResultTy); - bool DiagnoseMultipleUserDefinedConversion(Expr *From, QualType ToType); - bool isSameOrCompatibleFunctionType(QualType Param, QualType Arg); + void MarkTypoCorrectedFunctionDefinition(const NamedDecl *F); - bool CanPerformAggregateInitializationForOverloadResolution( - const InitializedEntity &Entity, InitListExpr *From); + /// Checks if the variant/multiversion functions are compatible. + bool areMultiversionVariantFunctionsCompatible( + const FunctionDecl *OldFD, const FunctionDecl *NewFD, + const PartialDiagnostic &NoProtoDiagID, + const PartialDiagnosticAt &NoteCausedDiagIDAt, + const PartialDiagnosticAt &NoSupportDiagIDAt, + const PartialDiagnosticAt &DiffDiagIDAt, bool TemplatesSupported, + bool ConstexprSupported, bool CLinkageMayDiffer); - bool IsStringInit(Expr *Init, const ArrayType *AT); + /// type checking declaration initializers (C99 6.7.8) + bool CheckForConstantInitializer(Expr *e, QualType t); - bool CanPerformCopyInitialization(const InitializedEntity &Entity, - ExprResult Init); - ExprResult PerformCopyInitialization(const InitializedEntity &Entity, - SourceLocation EqualLoc, - ExprResult Init, - bool TopLevelOfInitList = false, - bool AllowExplicit = false); - ExprResult InitializeExplicitObjectArgument(Sema &S, Expr *Obj, - FunctionDecl *Fun); - ExprResult PerformImplicitObjectArgumentInitialization( - Expr *From, NestedNameSpecifier *Qualifier, NamedDecl *FoundDecl, - CXXMethodDecl *Method); + QualType deduceVarTypeFromInitializer(VarDecl *VDecl, DeclarationName Name, + QualType Type, TypeSourceInfo *TSI, + SourceRange Range, bool DirectInit, + Expr *Init); - /// Check that the lifetime of the initializer (and its subobjects) is - /// sufficient for initializing the entity, and perform lifetime extension - /// (when permitted) if not. - void checkInitializerLifetime(const InitializedEntity &Entity, Expr *Init); + bool DeduceVariableDeclarationType(VarDecl *VDecl, bool DirectInit, + Expr *Init); - ExprResult PerformContextuallyConvertToBool(Expr *From); - ExprResult PerformContextuallyConvertToObjCPointer(Expr *From); + sema::LambdaScopeInfo *RebuildLambdaScopeInfo(CXXMethodDecl *CallOperator); - /// Contexts in which a converted constant expression is required. - enum CCEKind { - CCEK_CaseValue, ///< Expression in a case label. - CCEK_Enumerator, ///< Enumerator value with fixed underlying type. - CCEK_TemplateArg, ///< Value of a non-type template parameter. - CCEK_ArrayBound, ///< Array bound in array declarator or new-expression. - CCEK_ExplicitBool, ///< Condition in an explicit(bool) specifier. - CCEK_Noexcept, ///< Condition in a noexcept(bool) specifier. - CCEK_StaticAssertMessageSize, ///< Call to size() in a static assert - ///< message. - CCEK_StaticAssertMessageData, ///< Call to data() in a static assert - ///< message. - }; + /// The declarator \p D defines a function in the scope \p S which is nested + /// in an `omp begin/end declare variant` scope. In this method we create a + /// declaration for \p D and rename \p D according to the OpenMP context + /// selector of the surrounding scope. Return all base functions in \p Bases. + void ActOnStartOfFunctionDefinitionInOpenMPDeclareVariantScope( + Scope *S, Declarator &D, MultiTemplateParamsArg TemplateParameterLists, + SmallVectorImpl &Bases); - ExprResult BuildConvertedConstantExpression(Expr *From, QualType T, - CCEKind CCE, - NamedDecl *Dest = nullptr); + // Heuristically tells if the function is `get_return_object` member of a + // coroutine promise_type by matching the function name. + static bool CanBeGetReturnObject(const FunctionDecl *FD); + static bool CanBeGetReturnTypeOnAllocFailure(const FunctionDecl *FD); - ExprResult CheckConvertedConstantExpression(Expr *From, QualType T, - llvm::APSInt &Value, CCEKind CCE); - ExprResult CheckConvertedConstantExpression(Expr *From, QualType T, - APValue &Value, CCEKind CCE, - NamedDecl *Dest = nullptr); + NamedDecl *ImplicitlyDefineFunction(SourceLocation Loc, IdentifierInfo &II, + Scope *S); + void AddKnownFunctionAttributesForReplaceableGlobalAllocationFunction( + FunctionDecl *FD); + void AddKnownFunctionAttributes(FunctionDecl *FD); - ExprResult - EvaluateConvertedConstantExpression(Expr *E, QualType T, APValue &Value, - CCEKind CCE, bool RequireInt, - const APValue &PreNarrowingValue); + /// VerifyBitField - verifies that a bit field expression is an ICE and has + /// the correct width, and that the field type is valid. + /// Returns false on success. + ExprResult VerifyBitField(SourceLocation FieldLoc, IdentifierInfo *FieldName, + QualType FieldTy, bool IsMsStruct, Expr *BitWidth); - /// Abstract base class used to perform a contextual implicit - /// conversion from an expression to any type passing a filter. - class ContextualImplicitConverter { - public: - bool Suppress; - bool SuppressConversion; + /// IsValueInFlagEnum - Determine if a value is allowed as part of a flag + /// enum. If AllowMask is true, then we also allow the complement of a valid + /// value, to be used as a mask. + bool IsValueInFlagEnum(const EnumDecl *ED, const llvm::APInt &Val, + bool AllowMask) const; - ContextualImplicitConverter(bool Suppress = false, - bool SuppressConversion = false) - : Suppress(Suppress), SuppressConversion(SuppressConversion) {} + /// ActOnPragmaWeakID - Called on well formed \#pragma weak ident. + void ActOnPragmaWeakID(IdentifierInfo *WeakName, SourceLocation PragmaLoc, + SourceLocation WeakNameLoc); - /// Determine whether the specified type is a valid destination type - /// for this conversion. - virtual bool match(QualType T) = 0; + /// ActOnPragmaRedefineExtname - Called on well formed + /// \#pragma redefine_extname oldname newname. + void ActOnPragmaRedefineExtname(IdentifierInfo *WeakName, + IdentifierInfo *AliasName, + SourceLocation PragmaLoc, + SourceLocation WeakNameLoc, + SourceLocation AliasNameLoc); - /// Emits a diagnostic complaining that the expression does not have - /// integral or enumeration type. - virtual SemaDiagnosticBuilder - diagnoseNoMatch(Sema &S, SourceLocation Loc, QualType T) = 0; + /// ActOnPragmaWeakAlias - Called on well formed \#pragma weak ident = ident. + void ActOnPragmaWeakAlias(IdentifierInfo *WeakName, IdentifierInfo *AliasName, + SourceLocation PragmaLoc, + SourceLocation WeakNameLoc, + SourceLocation AliasNameLoc); - /// Emits a diagnostic when the expression has incomplete class type. - virtual SemaDiagnosticBuilder - diagnoseIncomplete(Sema &S, SourceLocation Loc, QualType T) = 0; + ObjCContainerDecl *getObjCDeclContext() const; - /// Emits a diagnostic when the only matching conversion function - /// is explicit. - virtual SemaDiagnosticBuilder diagnoseExplicitConv( - Sema &S, SourceLocation Loc, QualType T, QualType ConvTy) = 0; + /// Status of the function emission on the CUDA/HIP/OpenMP host/device attrs. + enum class FunctionEmissionStatus { + Emitted, + CUDADiscarded, // Discarded due to CUDA/HIP hostness + OMPDiscarded, // Discarded due to OpenMP hostness + TemplateDiscarded, // Discarded due to uninstantiated templates + Unknown, + }; + FunctionEmissionStatus getEmissionStatus(const FunctionDecl *Decl, + bool Final = false); - /// Emits a note for the explicit conversion function. - virtual SemaDiagnosticBuilder - noteExplicitConv(Sema &S, CXXConversionDecl *Conv, QualType ConvTy) = 0; + // Whether the callee should be ignored in CUDA/HIP/OpenMP host/device check. + bool shouldIgnoreInHostDeviceCheck(FunctionDecl *Callee); - /// Emits a diagnostic when there are multiple possible conversion - /// functions. - virtual SemaDiagnosticBuilder - diagnoseAmbiguous(Sema &S, SourceLocation Loc, QualType T) = 0; +private: + /// Function or variable declarations to be checked for whether the deferred + /// diagnostics should be emitted. + llvm::SmallSetVector DeclsToCheckForDeferredDiags; - /// Emits a note for one of the candidate conversions. - virtual SemaDiagnosticBuilder - noteAmbiguous(Sema &S, CXXConversionDecl *Conv, QualType ConvTy) = 0; + /// Map of current shadowing declarations to shadowed declarations. Warn if + /// it looks like the user is trying to modify the shadowing declaration. + llvm::DenseMap ShadowingDecls; - /// Emits a diagnostic when we picked a conversion function - /// (for cases when we are not allowed to pick a conversion function). - virtual SemaDiagnosticBuilder diagnoseConversion( - Sema &S, SourceLocation Loc, QualType T, QualType ConvTy) = 0; + static bool mightHaveNonExternalLinkage(const DeclaratorDecl *FD); - virtual ~ContextualImplicitConverter() {} - }; + ///@} - class ICEConvertDiagnoser : public ContextualImplicitConverter { - bool AllowScopedEnumerations; + // + // + // ------------------------------------------------------------------------- + // + // - public: - ICEConvertDiagnoser(bool AllowScopedEnumerations, - bool Suppress, bool SuppressConversion) - : ContextualImplicitConverter(Suppress, SuppressConversion), - AllowScopedEnumerations(AllowScopedEnumerations) {} + /// \name Declaration Attribute Handling + /// Implementations are in SemaDeclAttr.cpp + ///@{ - /// Match an integral or (possibly scoped) enumeration type. - bool match(QualType T) override; +public: + /// Describes the kind of priority given to an availability attribute. + /// + /// The sum of priorities deteremines the final priority of the attribute. + /// The final priority determines how the attribute will be merged. + /// An attribute with a lower priority will always remove higher priority + /// attributes for the specified platform when it is being applied. An + /// attribute with a higher priority will not be applied if the declaration + /// already has an availability attribute with a lower priority for the + /// specified platform. The final prirority values are not expected to match + /// the values in this enumeration, but instead should be treated as a plain + /// integer value. This enumeration just names the priority weights that are + /// used to calculate that final vaue. + enum AvailabilityPriority : int { + /// The availability attribute was specified explicitly next to the + /// declaration. + AP_Explicit = 0, - SemaDiagnosticBuilder - diagnoseNoMatch(Sema &S, SourceLocation Loc, QualType T) override { - return diagnoseNotInt(S, Loc, T); - } + /// The availability attribute was applied using '#pragma clang attribute'. + AP_PragmaClangAttribute = 1, - /// Emits a diagnostic complaining that the expression does not have - /// integral or enumeration type. - virtual SemaDiagnosticBuilder - diagnoseNotInt(Sema &S, SourceLocation Loc, QualType T) = 0; + /// The availability attribute for a specific platform was inferred from + /// an availability attribute for another platform. + AP_InferredFromOtherPlatform = 2 }; - /// Perform a contextual implicit conversion. - ExprResult PerformContextualImplicitConversion( - SourceLocation Loc, Expr *FromE, ContextualImplicitConverter &Converter); - - - enum ObjCSubscriptKind { - OS_Array, - OS_Dictionary, - OS_Error + /// Describes the reason a calling convention specification was ignored, used + /// for diagnostics. + enum class CallingConventionIgnoredReason { + ForThisTarget = 0, + VariadicFunction, + ConstructorDestructor, + BuiltinFunction }; - ObjCSubscriptKind CheckSubscriptingKind(Expr *FromE); - // Note that LK_String is intentionally after the other literals, as - // this is used for diagnostics logic. - enum ObjCLiteralKind { - LK_Array, - LK_Dictionary, - LK_Numeric, - LK_Boxed, - LK_String, - LK_Block, - LK_None - }; - ObjCLiteralKind CheckLiteralKind(Expr *FromE); + /// WeakTopLevelDecl - Translation-unit scoped declarations generated by + /// \#pragma weak during processing of other Decls. + /// I couldn't figure out a clean way to generate these in-line, so + /// we store them here and handle separately -- which is a hack. + /// It would be best to refactor this. + SmallVector WeakTopLevelDecl; - ExprResult PerformObjectMemberConversion(Expr *From, - NestedNameSpecifier *Qualifier, - NamedDecl *FoundDecl, - NamedDecl *Member); + /// WeakTopLevelDeclDecls - access to \#pragma weak-generated Decls + SmallVectorImpl &WeakTopLevelDecls() { return WeakTopLevelDecl; } - // Members have to be NamespaceDecl* or TranslationUnitDecl*. - // TODO: make this is a typesafe union. - typedef llvm::SmallSetVector AssociatedNamespaceSet; - typedef llvm::SmallSetVector AssociatedClassSet; + typedef LazyVector + ExtVectorDeclsType; - using ADLCallKind = CallExpr::ADLCallKind; + /// ExtVectorDecls - This is a list all the extended vector types. This allows + /// us to associate a raw vector type with one of the ext_vector type names. + /// This is only necessary for issuing pretty diagnostics. + ExtVectorDeclsType ExtVectorDecls; - void AddOverloadCandidate( - FunctionDecl *Function, DeclAccessPair FoundDecl, ArrayRef Args, - OverloadCandidateSet &CandidateSet, bool SuppressUserConversions = false, - bool PartialOverloading = false, bool AllowExplicit = true, - bool AllowExplicitConversion = false, - ADLCallKind IsADLCandidate = ADLCallKind::NotADL, - ConversionSequenceList EarlyConversions = std::nullopt, - OverloadCandidateParamOrder PO = {}, - bool AggregateCandidateDeduction = false); - void AddFunctionCandidates(const UnresolvedSetImpl &Functions, - ArrayRef Args, - OverloadCandidateSet &CandidateSet, - TemplateArgumentListInfo *ExplicitTemplateArgs = nullptr, - bool SuppressUserConversions = false, - bool PartialOverloading = false, - bool FirstArgumentIsBase = false); - void AddMethodCandidate(DeclAccessPair FoundDecl, - QualType ObjectType, - Expr::Classification ObjectClassification, - ArrayRef Args, - OverloadCandidateSet& CandidateSet, - bool SuppressUserConversion = false, - OverloadCandidateParamOrder PO = {}); - void - AddMethodCandidate(CXXMethodDecl *Method, DeclAccessPair FoundDecl, - CXXRecordDecl *ActingContext, QualType ObjectType, - Expr::Classification ObjectClassification, - ArrayRef Args, OverloadCandidateSet &CandidateSet, - bool SuppressUserConversions = false, - bool PartialOverloading = false, - ConversionSequenceList EarlyConversions = std::nullopt, - OverloadCandidateParamOrder PO = {}); - void AddMethodTemplateCandidate(FunctionTemplateDecl *MethodTmpl, - DeclAccessPair FoundDecl, - CXXRecordDecl *ActingContext, - TemplateArgumentListInfo *ExplicitTemplateArgs, - QualType ObjectType, - Expr::Classification ObjectClassification, - ArrayRef Args, - OverloadCandidateSet& CandidateSet, - bool SuppressUserConversions = false, - bool PartialOverloading = false, - OverloadCandidateParamOrder PO = {}); - void AddTemplateOverloadCandidate( - FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl, - TemplateArgumentListInfo *ExplicitTemplateArgs, ArrayRef Args, - OverloadCandidateSet &CandidateSet, bool SuppressUserConversions = false, - bool PartialOverloading = false, bool AllowExplicit = true, - ADLCallKind IsADLCandidate = ADLCallKind::NotADL, - OverloadCandidateParamOrder PO = {}, - bool AggregateCandidateDeduction = false); - bool CheckNonDependentConversions( - FunctionTemplateDecl *FunctionTemplate, ArrayRef ParamTypes, - ArrayRef Args, OverloadCandidateSet &CandidateSet, - ConversionSequenceList &Conversions, bool SuppressUserConversions, - CXXRecordDecl *ActingContext = nullptr, QualType ObjectType = QualType(), - Expr::Classification ObjectClassification = {}, - OverloadCandidateParamOrder PO = {}); - void AddConversionCandidate( - CXXConversionDecl *Conversion, DeclAccessPair FoundDecl, - CXXRecordDecl *ActingContext, Expr *From, QualType ToType, - OverloadCandidateSet &CandidateSet, bool AllowObjCConversionOnExplicit, - bool AllowExplicit, bool AllowResultConversion = true); - void AddTemplateConversionCandidate( - FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl, - CXXRecordDecl *ActingContext, Expr *From, QualType ToType, - OverloadCandidateSet &CandidateSet, bool AllowObjCConversionOnExplicit, - bool AllowExplicit, bool AllowResultConversion = true); - void AddSurrogateCandidate(CXXConversionDecl *Conversion, - DeclAccessPair FoundDecl, - CXXRecordDecl *ActingContext, - const FunctionProtoType *Proto, - Expr *Object, ArrayRef Args, - OverloadCandidateSet& CandidateSet); - void AddNonMemberOperatorCandidates( - const UnresolvedSetImpl &Functions, ArrayRef Args, - OverloadCandidateSet &CandidateSet, - TemplateArgumentListInfo *ExplicitTemplateArgs = nullptr); - void AddMemberOperatorCandidates(OverloadedOperatorKind Op, - SourceLocation OpLoc, ArrayRef Args, - OverloadCandidateSet &CandidateSet, - OverloadCandidateParamOrder PO = {}); - void AddBuiltinCandidate(QualType *ParamTys, ArrayRef Args, - OverloadCandidateSet& CandidateSet, - bool IsAssignmentOperator = false, - unsigned NumContextualBoolArguments = 0); - void AddBuiltinOperatorCandidates(OverloadedOperatorKind Op, - SourceLocation OpLoc, ArrayRef Args, - OverloadCandidateSet& CandidateSet); - void AddArgumentDependentLookupCandidates(DeclarationName Name, - SourceLocation Loc, - ArrayRef Args, - TemplateArgumentListInfo *ExplicitTemplateArgs, - OverloadCandidateSet& CandidateSet, - bool PartialOverloading = false); + bool checkStringLiteralArgumentAttr(const AttributeCommonInfo &CI, + const Expr *E, StringRef &Str, + SourceLocation *ArgLocation = nullptr); + bool checkStringLiteralArgumentAttr(const ParsedAttr &Attr, unsigned ArgNum, + StringRef &Str, + SourceLocation *ArgLocation = nullptr); - // Emit as a 'note' the specific overload candidate - void NoteOverloadCandidate( - const NamedDecl *Found, const FunctionDecl *Fn, - OverloadCandidateRewriteKind RewriteKind = OverloadCandidateRewriteKind(), - QualType DestType = QualType(), bool TakingAddress = false); + /// Determine if type T is a valid subject for a nonnull and similar + /// attributes. By default, we look through references (the behavior used by + /// nonnull), but if the second parameter is true, then we treat a reference + /// type as valid. + bool isValidPointerAttrType(QualType T, bool RefOkay = false); - // Emit as a series of 'note's all template and non-templates identified by - // the expression Expr - void NoteAllOverloadCandidates(Expr *E, QualType DestType = QualType(), - bool TakingAddress = false); + /// AddAssumeAlignedAttr - Adds an assume_aligned attribute to a particular + /// declaration. + void AddAssumeAlignedAttr(Decl *D, const AttributeCommonInfo &CI, Expr *E, + Expr *OE); - /// Check the enable_if expressions on the given function. Returns the first - /// failing attribute, or NULL if they were all successful. - EnableIfAttr *CheckEnableIf(FunctionDecl *Function, SourceLocation CallLoc, - ArrayRef Args, - bool MissingImplicitThis = false); + /// AddAllocAlignAttr - Adds an alloc_align attribute to a particular + /// declaration. + void AddAllocAlignAttr(Decl *D, const AttributeCommonInfo &CI, + Expr *ParamExpr); - /// Find the failed Boolean condition within a given Boolean - /// constant expression, and describe it with a string. - std::pair findFailedBooleanCondition(Expr *Cond); - - /// Emit diagnostics for the diagnose_if attributes on Function, ignoring any - /// non-ArgDependent DiagnoseIfAttrs. - /// - /// Argument-dependent diagnose_if attributes should be checked each time a - /// function is used as a direct callee of a function call. - /// - /// Returns true if any errors were emitted. - bool diagnoseArgDependentDiagnoseIfAttrs(const FunctionDecl *Function, - const Expr *ThisArg, - ArrayRef Args, - SourceLocation Loc); + bool CheckAttrTarget(const ParsedAttr &CurrAttr); + bool CheckAttrNoArgs(const ParsedAttr &CurrAttr); - /// Emit diagnostics for the diagnose_if attributes on Function, ignoring any - /// ArgDependent DiagnoseIfAttrs. - /// - /// Argument-independent diagnose_if attributes should be checked on every use - /// of a function. - /// - /// Returns true if any errors were emitted. - bool diagnoseArgIndependentDiagnoseIfAttrs(const NamedDecl *ND, - SourceLocation Loc); + AvailabilityAttr * + mergeAvailabilityAttr(NamedDecl *D, const AttributeCommonInfo &CI, + IdentifierInfo *Platform, bool Implicit, + VersionTuple Introduced, VersionTuple Deprecated, + VersionTuple Obsoleted, bool IsUnavailable, + StringRef Message, bool IsStrict, StringRef Replacement, + AvailabilityMergeKind AMK, int Priority); + TypeVisibilityAttr * + mergeTypeVisibilityAttr(Decl *D, const AttributeCommonInfo &CI, + TypeVisibilityAttr::VisibilityType Vis); + VisibilityAttr *mergeVisibilityAttr(Decl *D, const AttributeCommonInfo &CI, + VisibilityAttr::VisibilityType Vis); + SectionAttr *mergeSectionAttr(Decl *D, const AttributeCommonInfo &CI, + StringRef Name); - /// Returns whether the given function's address can be taken or not, - /// optionally emitting a diagnostic if the address can't be taken. - /// - /// Returns false if taking the address of the function is illegal. - bool checkAddressOfFunctionIsAvailable(const FunctionDecl *Function, - bool Complain = false, - SourceLocation Loc = SourceLocation()); + llvm::Error isValidSectionSpecifier(StringRef Str); + bool checkSectionName(SourceLocation LiteralLoc, StringRef Str); + CodeSegAttr *mergeCodeSegAttr(Decl *D, const AttributeCommonInfo &CI, + StringRef Name); - // [PossiblyAFunctionType] --> [Return] - // NonFunctionType --> NonFunctionType - // R (A) --> R(A) - // R (*)(A) --> R (A) - // R (&)(A) --> R (A) - // R (S::*)(A) --> R (A) - QualType ExtractUnqualifiedFunctionType(QualType PossiblyAFunctionType); + bool checkTargetAttr(SourceLocation LiteralLoc, StringRef Str); + bool checkTargetVersionAttr(SourceLocation LiteralLoc, Decl *D, + StringRef &Str, bool &isDefault); + bool checkTargetClonesAttrString( + SourceLocation LiteralLoc, StringRef Str, const StringLiteral *Literal, + Decl *D, bool &HasDefault, bool &HasCommas, bool &HasNotDefault, + SmallVectorImpl> &StringsBuffer); - FunctionDecl * - ResolveAddressOfOverloadedFunction(Expr *AddressOfExpr, - QualType TargetType, - bool Complain, - DeclAccessPair &Found, - bool *pHadMultipleCandidates = nullptr); + ErrorAttr *mergeErrorAttr(Decl *D, const AttributeCommonInfo &CI, + StringRef NewUserDiagnostic); + FormatAttr *mergeFormatAttr(Decl *D, const AttributeCommonInfo &CI, + IdentifierInfo *Format, int FormatIdx, + int FirstArg); - FunctionDecl * - resolveAddressOfSingleOverloadCandidate(Expr *E, DeclAccessPair &FoundResult); + /// AddAlignedAttr - Adds an aligned attribute to a particular declaration. + void AddAlignedAttr(Decl *D, const AttributeCommonInfo &CI, Expr *E, + bool IsPackExpansion); + void AddAlignedAttr(Decl *D, const AttributeCommonInfo &CI, TypeSourceInfo *T, + bool IsPackExpansion); - bool resolveAndFixAddressOfSingleOverloadCandidate( - ExprResult &SrcExpr, bool DoFunctionPointerConversion = false); + /// AddAlignValueAttr - Adds an align_value attribute to a particular + /// declaration. + void AddAlignValueAttr(Decl *D, const AttributeCommonInfo &CI, Expr *E); - FunctionDecl *ResolveSingleFunctionTemplateSpecialization( - OverloadExpr *ovl, bool Complain = false, DeclAccessPair *Found = nullptr, - TemplateSpecCandidateSet *FailedTSC = nullptr); + /// AddAnnotationAttr - Adds an annotation Annot with Args arguments to D. + void AddAnnotationAttr(Decl *D, const AttributeCommonInfo &CI, + StringRef Annot, MutableArrayRef Args); - bool ResolveAndFixSingleFunctionTemplateSpecialization( - ExprResult &SrcExpr, bool DoFunctionPointerConversion = false, - bool Complain = false, SourceRange OpRangeForComplaining = SourceRange(), - QualType DestTypeForComplaining = QualType(), - unsigned DiagIDForComplaining = 0); + bool checkMSInheritanceAttrOnDefinition(CXXRecordDecl *RD, SourceRange Range, + bool BestCase, + MSInheritanceModel SemanticSpelling); - ExprResult FixOverloadedFunctionReference(Expr *E, DeclAccessPair FoundDecl, - FunctionDecl *Fn); - ExprResult FixOverloadedFunctionReference(ExprResult, - DeclAccessPair FoundDecl, - FunctionDecl *Fn); + void CheckAlignasUnderalignment(Decl *D); - void AddOverloadedCallCandidates(UnresolvedLookupExpr *ULE, - ArrayRef Args, - OverloadCandidateSet &CandidateSet, - bool PartialOverloading = false); - void AddOverloadedCallCandidates( - LookupResult &R, TemplateArgumentListInfo *ExplicitTemplateArgs, - ArrayRef Args, OverloadCandidateSet &CandidateSet); + /// AddModeAttr - Adds a mode attribute to a particular declaration. + void AddModeAttr(Decl *D, const AttributeCommonInfo &CI, IdentifierInfo *Name, + bool InInstantiation = false); + AlwaysInlineAttr *mergeAlwaysInlineAttr(Decl *D, + const AttributeCommonInfo &CI, + const IdentifierInfo *Ident); + MinSizeAttr *mergeMinSizeAttr(Decl *D, const AttributeCommonInfo &CI); + SwiftNameAttr *mergeSwiftNameAttr(Decl *D, const SwiftNameAttr &SNA, + StringRef Name); + OptimizeNoneAttr *mergeOptimizeNoneAttr(Decl *D, + const AttributeCommonInfo &CI); + InternalLinkageAttr *mergeInternalLinkageAttr(Decl *D, const ParsedAttr &AL); + InternalLinkageAttr *mergeInternalLinkageAttr(Decl *D, + const InternalLinkageAttr &AL); - // An enum used to represent the different possible results of building a - // range-based for loop. - enum ForRangeStatus { - FRS_Success, - FRS_NoViableFunction, - FRS_DiagnosticIssued + enum CUDAFunctionTarget { + CFT_Device, + CFT_Global, + CFT_Host, + CFT_HostDevice, + CFT_InvalidTarget }; - ForRangeStatus BuildForRangeBeginEndCall(SourceLocation Loc, - SourceLocation RangeLoc, - const DeclarationNameInfo &NameInfo, - LookupResult &MemberLookup, - OverloadCandidateSet *CandidateSet, - Expr *Range, ExprResult *CallExpr); + /// Check validaty of calling convention attribute \p attr. If \p FD + /// is not null pointer, use \p FD to determine the CUDA/HIP host/device + /// target. Otherwise, it is specified by \p CFT. + bool CheckCallingConvAttr(const ParsedAttr &attr, CallingConv &CC, + const FunctionDecl *FD = nullptr, + CUDAFunctionTarget CFT = CFT_InvalidTarget); - ExprResult BuildOverloadedCallExpr(Scope *S, Expr *Fn, - UnresolvedLookupExpr *ULE, - SourceLocation LParenLoc, - MultiExprArg Args, - SourceLocation RParenLoc, - Expr *ExecConfig, - bool AllowTypoCorrection=true, - bool CalleesAddressIsTaken=false); + void AddParameterABIAttr(Decl *D, const AttributeCommonInfo &CI, + ParameterABI ABI); + bool CheckRegparmAttr(const ParsedAttr &attr, unsigned &value); - bool buildOverloadedCallSet(Scope *S, Expr *Fn, UnresolvedLookupExpr *ULE, - MultiExprArg Args, SourceLocation RParenLoc, - OverloadCandidateSet *CandidateSet, - ExprResult *Result); + /// Create an CUDALaunchBoundsAttr attribute. + CUDALaunchBoundsAttr *CreateLaunchBoundsAttr(const AttributeCommonInfo &CI, + Expr *MaxThreads, + Expr *MinBlocks, + Expr *MaxBlocks); - ExprResult CreateUnresolvedLookupExpr(CXXRecordDecl *NamingClass, - NestedNameSpecifierLoc NNSLoc, - DeclarationNameInfo DNI, - const UnresolvedSetImpl &Fns, - bool PerformADL = true); + /// AddLaunchBoundsAttr - Adds a launch_bounds attribute to a particular + /// declaration. + void AddLaunchBoundsAttr(Decl *D, const AttributeCommonInfo &CI, + Expr *MaxThreads, Expr *MinBlocks, Expr *MaxBlocks); - ExprResult CreateOverloadedUnaryOp(SourceLocation OpLoc, - UnaryOperatorKind Opc, - const UnresolvedSetImpl &Fns, - Expr *input, bool RequiresADL = true); + enum class RetainOwnershipKind { NS, CF, OS }; + void AddXConsumedAttr(Decl *D, const AttributeCommonInfo &CI, + RetainOwnershipKind K, bool IsTemplateInstantiation); - void LookupOverloadedBinOp(OverloadCandidateSet &CandidateSet, - OverloadedOperatorKind Op, - const UnresolvedSetImpl &Fns, - ArrayRef Args, bool RequiresADL = true); - ExprResult CreateOverloadedBinOp(SourceLocation OpLoc, - BinaryOperatorKind Opc, - const UnresolvedSetImpl &Fns, - Expr *LHS, Expr *RHS, - bool RequiresADL = true, - bool AllowRewrittenCandidates = true, - FunctionDecl *DefaultedFn = nullptr); - ExprResult BuildSynthesizedThreeWayComparison(SourceLocation OpLoc, - const UnresolvedSetImpl &Fns, - Expr *LHS, Expr *RHS, - FunctionDecl *DefaultedFn); + bool checkNSReturnsRetainedReturnType(SourceLocation loc, QualType type); - ExprResult CreateOverloadedArraySubscriptExpr(SourceLocation LLoc, - SourceLocation RLoc, Expr *Base, - MultiExprArg Args); + /// Do a check to make sure \p Name looks like a legal argument for the + /// swift_name attribute applied to decl \p D. Raise a diagnostic if the name + /// is invalid for the given declaration. + /// + /// \p AL is used to provide caret diagnostics in case of a malformed name. + /// + /// \returns true if the name is a valid swift name for \p D, false otherwise. + bool DiagnoseSwiftName(Decl *D, StringRef Name, SourceLocation Loc, + const ParsedAttr &AL, bool IsAsync); - ExprResult BuildCallToMemberFunction(Scope *S, Expr *MemExpr, - SourceLocation LParenLoc, - MultiExprArg Args, - SourceLocation RParenLoc, - Expr *ExecConfig = nullptr, - bool IsExecConfig = false, - bool AllowRecovery = false); - ExprResult - BuildCallToObjectOfClassType(Scope *S, Expr *Object, SourceLocation LParenLoc, - MultiExprArg Args, - SourceLocation RParenLoc); + UuidAttr *mergeUuidAttr(Decl *D, const AttributeCommonInfo &CI, + StringRef UuidAsWritten, MSGuidDecl *GuidDecl); - ExprResult BuildOverloadedArrowExpr(Scope *S, Expr *Base, - SourceLocation OpLoc, - bool *NoArrowOperatorFound = nullptr); + BTFDeclTagAttr *mergeBTFDeclTagAttr(Decl *D, const BTFDeclTagAttr &AL); + HLSLNumThreadsAttr *mergeHLSLNumThreadsAttr(Decl *D, + const AttributeCommonInfo &AL, + int X, int Y, int Z); + HLSLShaderAttr *mergeHLSLShaderAttr(Decl *D, const AttributeCommonInfo &AL, + HLSLShaderAttr::ShaderType ShaderType); + HLSLParamModifierAttr * + mergeHLSLParamModifierAttr(Decl *D, const AttributeCommonInfo &AL, + HLSLParamModifierAttr::Spelling Spelling); - /// CheckCallReturnType - Checks that a call expression's return type is - /// complete. Returns true on failure. The location passed in is the location - /// that best represents the call. - bool CheckCallReturnType(QualType ReturnType, SourceLocation Loc, - CallExpr *CE, FunctionDecl *FD); + WebAssemblyImportNameAttr * + mergeImportNameAttr(Decl *D, const WebAssemblyImportNameAttr &AL); + WebAssemblyImportModuleAttr * + mergeImportModuleAttr(Decl *D, const WebAssemblyImportModuleAttr &AL); - /// Helpers for dealing with blocks and functions. - bool CheckParmsForFunctionDef(ArrayRef Parameters, - bool CheckParameterNames); - void CheckCXXDefaultArguments(FunctionDecl *FD); - void CheckExtraCXXDefaultArguments(Declarator &D); - Scope *getNonFieldDeclScope(Scope *S); + /// Create an AMDGPUWavesPerEUAttr attribute. + AMDGPUFlatWorkGroupSizeAttr * + CreateAMDGPUFlatWorkGroupSizeAttr(const AttributeCommonInfo &CI, Expr *Min, + Expr *Max); - /// \name Name lookup - /// - /// These routines provide name lookup that is used during semantic - /// analysis to resolve the various kinds of names (identifiers, - /// overloaded operator names, constructor names, etc.) into zero or - /// more declarations within a particular scope. The major entry - /// points are LookupName, which performs unqualified name lookup, - /// and LookupQualifiedName, which performs qualified name lookup. - /// - /// All name lookup is performed based on some specific criteria, - /// which specify what names will be visible to name lookup and how - /// far name lookup should work. These criteria are important both - /// for capturing language semantics (certain lookups will ignore - /// certain names, for example) and for performance, since name - /// lookup is often a bottleneck in the compilation of C++. Name - /// lookup criteria is specified via the LookupCriteria enumeration. - /// - /// The results of name lookup can vary based on the kind of name - /// lookup performed, the current language, and the translation - /// unit. In C, for example, name lookup will either return nothing - /// (no entity found) or a single declaration. In C++, name lookup - /// can additionally refer to a set of overloaded functions or - /// result in an ambiguity. All of the possible results of name - /// lookup are captured by the LookupResult class, which provides - /// the ability to distinguish among them. - //@{ + /// addAMDGPUFlatWorkGroupSizeAttr - Adds an amdgpu_flat_work_group_size + /// attribute to a particular declaration. + void addAMDGPUFlatWorkGroupSizeAttr(Decl *D, const AttributeCommonInfo &CI, + Expr *Min, Expr *Max); - /// Describes the kind of name lookup to perform. - enum LookupNameKind { - /// Ordinary name lookup, which finds ordinary names (functions, - /// variables, typedefs, etc.) in C and most kinds of names - /// (functions, variables, members, types, etc.) in C++. - LookupOrdinaryName = 0, - /// Tag name lookup, which finds the names of enums, classes, - /// structs, and unions. - LookupTagName, - /// Label name lookup. - LookupLabel, - /// Member name lookup, which finds the names of - /// class/struct/union members. - LookupMemberName, - /// Look up of an operator name (e.g., operator+) for use with - /// operator overloading. This lookup is similar to ordinary name - /// lookup, but will ignore any declarations that are class members. - LookupOperatorName, - /// Look up a name following ~ in a destructor name. This is an ordinary - /// lookup, but prefers tags to typedefs. - LookupDestructorName, - /// Look up of a name that precedes the '::' scope resolution - /// operator in C++. This lookup completely ignores operator, object, - /// function, and enumerator names (C++ [basic.lookup.qual]p1). - LookupNestedNameSpecifierName, - /// Look up a namespace name within a C++ using directive or - /// namespace alias definition, ignoring non-namespace names (C++ - /// [basic.lookup.udir]p1). - LookupNamespaceName, - /// Look up all declarations in a scope with the given name, - /// including resolved using declarations. This is appropriate - /// for checking redeclarations for a using declaration. - LookupUsingDeclName, - /// Look up an ordinary name that is going to be redeclared as a - /// name with linkage. This lookup ignores any declarations that - /// are outside of the current scope unless they have linkage. See - /// C99 6.2.2p4-5 and C++ [basic.link]p6. - LookupRedeclarationWithLinkage, - /// Look up a friend of a local class. This lookup does not look - /// outside the innermost non-class scope. See C++11 [class.friend]p11. - LookupLocalFriendName, - /// Look up the name of an Objective-C protocol. - LookupObjCProtocolName, - /// Look up implicit 'self' parameter of an objective-c method. - LookupObjCImplicitSelfParam, - /// Look up the name of an OpenMP user-defined reduction operation. - LookupOMPReductionName, - /// Look up the name of an OpenMP user-defined mapper. - LookupOMPMapperName, - /// Look up any declaration with any name. - LookupAnyName - }; + /// Create an AMDGPUWavesPerEUAttr attribute. + AMDGPUWavesPerEUAttr * + CreateAMDGPUWavesPerEUAttr(const AttributeCommonInfo &CI, Expr *Min, + Expr *Max); - /// Specifies whether (or how) name lookup is being performed for a - /// redeclaration (vs. a reference). - enum RedeclarationKind { - /// The lookup is a reference to this name that is not for the - /// purpose of redeclaring the name. - NotForRedeclaration = 0, - /// The lookup results will be used for redeclaration of a name, - /// if an entity by that name already exists and is visible. - ForVisibleRedeclaration, - /// The lookup results will be used for redeclaration of a name - /// with external linkage; non-visible lookup results with external linkage - /// may also be found. - ForExternalRedeclaration - }; + /// addAMDGPUWavePersEUAttr - Adds an amdgpu_waves_per_eu attribute to a + /// particular declaration. + void addAMDGPUWavesPerEUAttr(Decl *D, const AttributeCommonInfo &CI, + Expr *Min, Expr *Max); - RedeclarationKind forRedeclarationInCurContext() const { - // A declaration with an owning module for linkage can never link against - // anything that is not visible. We don't need to check linkage here; if - // the context has internal linkage, redeclaration lookup won't find things - // from other TUs, and we can't safely compute linkage yet in general. - if (cast(CurContext) - ->getOwningModuleForLinkage(/*IgnoreLinkage*/true)) - return ForVisibleRedeclaration; - return ForExternalRedeclaration; - } + DLLImportAttr *mergeDLLImportAttr(Decl *D, const AttributeCommonInfo &CI); + DLLExportAttr *mergeDLLExportAttr(Decl *D, const AttributeCommonInfo &CI); + MSInheritanceAttr *mergeMSInheritanceAttr(Decl *D, + const AttributeCommonInfo &CI, + bool BestCase, + MSInheritanceModel Model); - /// The possible outcomes of name lookup for a literal operator. - enum LiteralOperatorLookupResult { - /// The lookup resulted in an error. - LOLR_Error, - /// The lookup found no match but no diagnostic was issued. - LOLR_ErrorNoDiagnostic, - /// The lookup found a single 'cooked' literal operator, which - /// expects a normal literal to be built and passed to it. - LOLR_Cooked, - /// The lookup found a single 'raw' literal operator, which expects - /// a string literal containing the spelling of the literal token. - LOLR_Raw, - /// The lookup found an overload set of literal operator templates, - /// which expect the characters of the spelling of the literal token to be - /// passed as a non-type template argument pack. - LOLR_Template, - /// The lookup found an overload set of literal operator templates, - /// which expect the character type and characters of the spelling of the - /// string literal token to be passed as template arguments. - LOLR_StringTemplatePack, - }; + bool CheckCountedByAttr(Scope *Scope, const FieldDecl *FD); - SpecialMemberOverloadResult LookupSpecialMember(CXXRecordDecl *D, - CXXSpecialMember SM, - bool ConstArg, - bool VolatileArg, - bool RValueThis, - bool ConstThis, - bool VolatileThis); + EnforceTCBAttr *mergeEnforceTCBAttr(Decl *D, const EnforceTCBAttr &AL); + EnforceTCBLeafAttr *mergeEnforceTCBLeafAttr(Decl *D, + const EnforceTCBLeafAttr &AL); - typedef std::function TypoDiagnosticGenerator; - typedef std::function - TypoRecoveryCallback; + // Helper for delayed processing of attributes. + void ProcessDeclAttributeDelayed(Decl *D, + const ParsedAttributesView &AttrList); -private: - bool CppLookupName(LookupResult &R, Scope *S); + // Options for ProcessDeclAttributeList(). + struct ProcessDeclAttributeOptions { + ProcessDeclAttributeOptions() + : IncludeCXX11Attributes(true), IgnoreTypeAttributes(false) {} - struct TypoExprState { - std::unique_ptr Consumer; - TypoDiagnosticGenerator DiagHandler; - TypoRecoveryCallback RecoveryHandler; - TypoExprState(); - TypoExprState(TypoExprState &&other) noexcept; - TypoExprState &operator=(TypoExprState &&other) noexcept; - }; + ProcessDeclAttributeOptions WithIncludeCXX11Attributes(bool Val) { + ProcessDeclAttributeOptions Result = *this; + Result.IncludeCXX11Attributes = Val; + return Result; + } - /// The set of unhandled TypoExprs and their associated state. - llvm::MapVector DelayedTypos; + ProcessDeclAttributeOptions WithIgnoreTypeAttributes(bool Val) { + ProcessDeclAttributeOptions Result = *this; + Result.IgnoreTypeAttributes = Val; + return Result; + } - /// Creates a new TypoExpr AST node. - TypoExpr *createDelayedTypo(std::unique_ptr TCC, - TypoDiagnosticGenerator TDG, - TypoRecoveryCallback TRC, SourceLocation TypoLoc); + // Should C++11 attributes be processed? + bool IncludeCXX11Attributes; - // The set of known/encountered (unique, canonicalized) NamespaceDecls. - // - // The boolean value will be true to indicate that the namespace was loaded - // from an AST/PCH file, or false otherwise. - llvm::MapVector KnownNamespaces; + // Should any type attributes encountered be ignored? + // If this option is false, a diagnostic will be emitted for any type + // attributes of a kind that does not "slide" from the declaration to + // the decl-specifier-seq. + bool IgnoreTypeAttributes; + }; - /// Whether we have already loaded known namespaces from an extenal - /// source. - bool LoadedExternalKnownNamespaces; + void ProcessDeclAttributeList(Scope *S, Decl *D, + const ParsedAttributesView &AttrList, + const ProcessDeclAttributeOptions &Options = + ProcessDeclAttributeOptions()); + bool ProcessAccessDeclAttributeList(AccessSpecDecl *ASDecl, + const ParsedAttributesView &AttrList); - /// Helper for CorrectTypo and CorrectTypoDelayed used to create and - /// populate a new TypoCorrectionConsumer. Returns nullptr if typo correction - /// should be skipped entirely. - std::unique_ptr - makeTypoCorrectionConsumer(const DeclarationNameInfo &Typo, - Sema::LookupNameKind LookupKind, Scope *S, - CXXScopeSpec *SS, - CorrectionCandidateCallback &CCC, - DeclContext *MemberContext, bool EnteringContext, - const ObjCObjectPointerType *OPT, - bool ErrorRecovery); + void checkUnusedDeclAttributes(Declarator &D); -public: - const TypoExprState &getTypoExprState(TypoExpr *TE) const; + NamedDecl *DeclClonePragmaWeak(NamedDecl *ND, const IdentifierInfo *II, + SourceLocation Loc); + void DeclApplyPragmaWeak(Scope *S, NamedDecl *ND, const WeakInfo &W); - /// Clears the state of the given TypoExpr. - void clearDelayedTypo(TypoExpr *TE); + void ProcessPragmaWeak(Scope *S, Decl *D); + // Decl attributes - this routine is the top level dispatcher. + void ProcessDeclAttributes(Scope *S, Decl *D, const Declarator &PD); - /// Look up a name, looking for a single declaration. Return - /// null if the results were absent, ambiguous, or overloaded. - /// - /// It is preferable to use the elaborated form and explicitly handle - /// ambiguity and overloaded. - NamedDecl *LookupSingleName(Scope *S, DeclarationName Name, - SourceLocation Loc, - LookupNameKind NameKind, - RedeclarationKind Redecl - = NotForRedeclaration); - bool LookupBuiltin(LookupResult &R); - void LookupNecessaryTypesForBuiltin(Scope *S, unsigned ID); - bool LookupName(LookupResult &R, Scope *S, bool AllowBuiltinCreation = false, - bool ForceNoCPlusPlus = false); - bool LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx, - bool InUnqualifiedLookup = false); - bool LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx, - CXXScopeSpec &SS); - bool LookupParsedName(LookupResult &R, Scope *S, CXXScopeSpec *SS, - bool AllowBuiltinCreation = false, - bool EnteringContext = false); - ObjCProtocolDecl *LookupProtocol(IdentifierInfo *II, SourceLocation IdLoc, - RedeclarationKind Redecl - = NotForRedeclaration); - bool LookupInSuper(LookupResult &R, CXXRecordDecl *Class); + void PopParsingDeclaration(ParsingDeclState state, Decl *decl); - void LookupOverloadedOperatorName(OverloadedOperatorKind Op, Scope *S, - UnresolvedSetImpl &Functions); + void redelayDiagnostics(sema::DelayedDiagnosticPool &pool); - LabelDecl *LookupOrCreateLabel(IdentifierInfo *II, SourceLocation IdentLoc, - SourceLocation GnuLabelLoc = SourceLocation()); + ///@} - DeclContextLookupResult LookupConstructors(CXXRecordDecl *Class); - CXXConstructorDecl *LookupDefaultConstructor(CXXRecordDecl *Class); - CXXConstructorDecl *LookupCopyingConstructor(CXXRecordDecl *Class, - unsigned Quals); - CXXMethodDecl *LookupCopyingAssignment(CXXRecordDecl *Class, unsigned Quals, - bool RValueThis, unsigned ThisQuals); - CXXConstructorDecl *LookupMovingConstructor(CXXRecordDecl *Class, - unsigned Quals); - CXXMethodDecl *LookupMovingAssignment(CXXRecordDecl *Class, unsigned Quals, - bool RValueThis, unsigned ThisQuals); - CXXDestructorDecl *LookupDestructor(CXXRecordDecl *Class); + // + // + // ------------------------------------------------------------------------- + // + // - bool checkLiteralOperatorId(const CXXScopeSpec &SS, const UnqualifiedId &Id, - bool IsUDSuffix); - LiteralOperatorLookupResult - LookupLiteralOperator(Scope *S, LookupResult &R, ArrayRef ArgTys, - bool AllowRaw, bool AllowTemplate, - bool AllowStringTemplate, bool DiagnoseMissing, - StringLiteral *StringLit = nullptr); - bool isKnownName(StringRef name); + /// \name C++ Declarations + /// Implementations are in SemaDeclCXX.cpp + ///@{ - /// Status of the function emission on the CUDA/HIP/OpenMP host/device attrs. - enum class FunctionEmissionStatus { - Emitted, - CUDADiscarded, // Discarded due to CUDA/HIP hostness - OMPDiscarded, // Discarded due to OpenMP hostness - TemplateDiscarded, // Discarded due to uninstantiated templates - Unknown, - }; - FunctionEmissionStatus getEmissionStatus(const FunctionDecl *Decl, - bool Final = false); +public: + void CheckDelegatingCtorCycles(); - // Whether the callee should be ignored in CUDA/HIP/OpenMP host/device check. - bool shouldIgnoreInHostDeviceCheck(FunctionDecl *Callee); + /// Called before parsing a function declarator belonging to a function + /// declaration. + void ActOnStartFunctionDeclarationDeclarator(Declarator &D, + unsigned TemplateParameterDepth); - void ArgumentDependentLookup(DeclarationName Name, SourceLocation Loc, - ArrayRef Args, ADLResult &Functions); + /// Called after parsing a function declarator belonging to a function + /// declaration. + void ActOnFinishFunctionDeclarationDeclarator(Declarator &D); - void LookupVisibleDecls(Scope *S, LookupNameKind Kind, - VisibleDeclConsumer &Consumer, - bool IncludeGlobalScope = true, - bool LoadExternal = true); - void LookupVisibleDecls(DeclContext *Ctx, LookupNameKind Kind, - VisibleDeclConsumer &Consumer, - bool IncludeGlobalScope = true, - bool IncludeDependentBases = false, - bool LoadExternal = true); + // Act on C++ namespaces + Decl *ActOnStartNamespaceDef(Scope *S, SourceLocation InlineLoc, + SourceLocation NamespaceLoc, + SourceLocation IdentLoc, IdentifierInfo *Ident, + SourceLocation LBrace, + const ParsedAttributesView &AttrList, + UsingDirectiveDecl *&UsingDecl, bool IsNested); + void ActOnFinishNamespaceDef(Decl *Dcl, SourceLocation RBrace); - enum CorrectTypoKind { - CTK_NonError, // CorrectTypo used in a non error recovery situation. - CTK_ErrorRecovery // CorrectTypo used in normal error recovery. - }; + NamespaceDecl *getStdNamespace() const; + NamespaceDecl *getOrCreateStdNamespace(); - TypoCorrection CorrectTypo(const DeclarationNameInfo &Typo, - Sema::LookupNameKind LookupKind, - Scope *S, CXXScopeSpec *SS, - CorrectionCandidateCallback &CCC, - CorrectTypoKind Mode, - DeclContext *MemberContext = nullptr, - bool EnteringContext = false, - const ObjCObjectPointerType *OPT = nullptr, - bool RecordFailure = true); + CXXRecordDecl *getStdBadAlloc() const; + EnumDecl *getStdAlignValT() const; - TypoExpr *CorrectTypoDelayed(const DeclarationNameInfo &Typo, - Sema::LookupNameKind LookupKind, Scope *S, - CXXScopeSpec *SS, - CorrectionCandidateCallback &CCC, - TypoDiagnosticGenerator TDG, - TypoRecoveryCallback TRC, CorrectTypoKind Mode, - DeclContext *MemberContext = nullptr, - bool EnteringContext = false, - const ObjCObjectPointerType *OPT = nullptr); + ValueDecl *tryLookupUnambiguousFieldDecl(RecordDecl *ClassDecl, + const IdentifierInfo *MemberOrBase); - /// Process any TypoExprs in the given Expr and its children, - /// generating diagnostics as appropriate and returning a new Expr if there - /// were typos that were all successfully corrected and ExprError if one or - /// more typos could not be corrected. - /// - /// \param E The Expr to check for TypoExprs. - /// - /// \param InitDecl A VarDecl to avoid because the Expr being corrected is its - /// initializer. + enum class ComparisonCategoryUsage { + /// The '<=>' operator was used in an expression and a builtin operator + /// was selected. + OperatorInExpression, + /// A defaulted 'operator<=>' needed the comparison category. This + /// typically only applies to 'std::strong_ordering', due to the implicit + /// fallback return value. + DefaultedOperator, + }; + + /// Lookup the specified comparison category types in the standard + /// library, an check the VarDecls possibly returned by the operator<=> + /// builtins for that type. /// - /// \param RecoverUncorrectedTypos If true, when typo correction fails, it - /// will rebuild the given Expr with all TypoExprs degraded to RecoveryExprs. + /// \return The type of the comparison category type corresponding to the + /// specified Kind, or a null type if an error occurs + QualType CheckComparisonCategoryType(ComparisonCategoryType Kind, + SourceLocation Loc, + ComparisonCategoryUsage Usage); + + /// Tests whether Ty is an instance of std::initializer_list and, if + /// it is and Element is not NULL, assigns the element type to Element. + bool isStdInitializerList(QualType Ty, QualType *Element); + + /// Looks for the std::initializer_list template and instantiates it + /// with Element, or emits an error if it's not found. /// - /// \param Filter A function applied to a newly rebuilt Expr to determine if - /// it is an acceptable/usable result from a single combination of typo - /// corrections. As long as the filter returns ExprError, different - /// combinations of corrections will be tried until all are exhausted. - ExprResult CorrectDelayedTyposInExpr( - Expr *E, VarDecl *InitDecl = nullptr, - bool RecoverUncorrectedTypos = false, - llvm::function_ref Filter = - [](Expr *E) -> ExprResult { return E; }); + /// \returns The instantiated template, or null on error. + QualType BuildStdInitializerList(QualType Element, SourceLocation Loc); - ExprResult CorrectDelayedTyposInExpr( - ExprResult ER, VarDecl *InitDecl = nullptr, - bool RecoverUncorrectedTypos = false, - llvm::function_ref Filter = - [](Expr *E) -> ExprResult { return E; }) { - return ER.isInvalid() - ? ER - : CorrectDelayedTyposInExpr(ER.get(), InitDecl, - RecoverUncorrectedTypos, Filter); - } + /// Determine whether Ctor is an initializer-list constructor, as + /// defined in [dcl.init.list]p2. + bool isInitListConstructor(const FunctionDecl *Ctor); - void diagnoseTypo(const TypoCorrection &Correction, - const PartialDiagnostic &TypoDiag, - bool ErrorRecovery = true); + Decl *ActOnUsingDirective(Scope *CurScope, SourceLocation UsingLoc, + SourceLocation NamespcLoc, CXXScopeSpec &SS, + SourceLocation IdentLoc, + IdentifierInfo *NamespcName, + const ParsedAttributesView &AttrList); - void diagnoseTypo(const TypoCorrection &Correction, - const PartialDiagnostic &TypoDiag, - const PartialDiagnostic &PrevNote, - bool ErrorRecovery = true); + void PushUsingDirective(Scope *S, UsingDirectiveDecl *UDir); - void MarkTypoCorrectedFunctionDefinition(const NamedDecl *F); + Decl *ActOnNamespaceAliasDef(Scope *CurScope, SourceLocation NamespaceLoc, + SourceLocation AliasLoc, IdentifierInfo *Alias, + CXXScopeSpec &SS, SourceLocation IdentLoc, + IdentifierInfo *Ident); - void FindAssociatedClassesAndNamespaces(SourceLocation InstantiationLoc, - ArrayRef Args, - AssociatedNamespaceSet &AssociatedNamespaces, - AssociatedClassSet &AssociatedClasses); + void FilterUsingLookup(Scope *S, LookupResult &lookup); + void HideUsingShadowDecl(Scope *S, UsingShadowDecl *Shadow); + bool CheckUsingShadowDecl(BaseUsingDecl *BUD, NamedDecl *Target, + const LookupResult &PreviousDecls, + UsingShadowDecl *&PrevShadow); + UsingShadowDecl *BuildUsingShadowDecl(Scope *S, BaseUsingDecl *BUD, + NamedDecl *Target, + UsingShadowDecl *PrevDecl); - void FilterLookupForScope(LookupResult &R, DeclContext *Ctx, Scope *S, - bool ConsiderLinkage, bool AllowInlineNamespace); + bool CheckUsingDeclRedeclaration(SourceLocation UsingLoc, + bool HasTypenameKeyword, + const CXXScopeSpec &SS, + SourceLocation NameLoc, + const LookupResult &Previous); + bool CheckUsingDeclQualifier(SourceLocation UsingLoc, bool HasTypename, + const CXXScopeSpec &SS, + const DeclarationNameInfo &NameInfo, + SourceLocation NameLoc, + const LookupResult *R = nullptr, + const UsingDecl *UD = nullptr); - bool CheckRedeclarationModuleOwnership(NamedDecl *New, NamedDecl *Old); - bool CheckRedeclarationExported(NamedDecl *New, NamedDecl *Old); - bool CheckRedeclarationInModule(NamedDecl *New, NamedDecl *Old); - bool IsRedefinitionInModule(const NamedDecl *New, - const NamedDecl *Old) const; + NamedDecl *BuildUsingDeclaration(Scope *S, AccessSpecifier AS, + SourceLocation UsingLoc, + bool HasTypenameKeyword, + SourceLocation TypenameLoc, CXXScopeSpec &SS, + DeclarationNameInfo NameInfo, + SourceLocation EllipsisLoc, + const ParsedAttributesView &AttrList, + bool IsInstantiation, bool IsUsingIfExists); + NamedDecl *BuildUsingEnumDeclaration(Scope *S, AccessSpecifier AS, + SourceLocation UsingLoc, + SourceLocation EnumLoc, + SourceLocation NameLoc, + TypeSourceInfo *EnumType, EnumDecl *ED); + NamedDecl *BuildUsingPackDecl(NamedDecl *InstantiatedFrom, + ArrayRef Expansions); - void DiagnoseAmbiguousLookup(LookupResult &Result); - //@} + bool CheckInheritingConstructorUsingDecl(UsingDecl *UD); - /// Attempts to produce a RecoveryExpr after some AST node cannot be created. - ExprResult CreateRecoveryExpr(SourceLocation Begin, SourceLocation End, - ArrayRef SubExprs, - QualType T = QualType()); + /// Given a derived-class using shadow declaration for a constructor and the + /// correspnding base class constructor, find or create the implicit + /// synthesized derived class constructor to use for this initialization. + CXXConstructorDecl * + findInheritingConstructor(SourceLocation Loc, CXXConstructorDecl *BaseCtor, + ConstructorUsingShadowDecl *DerivedShadow); - ObjCInterfaceDecl *getObjCInterfaceDecl(IdentifierInfo *&Id, - SourceLocation IdLoc, - bool TypoCorrection = false); - FunctionDecl *CreateBuiltin(IdentifierInfo *II, QualType Type, unsigned ID, - SourceLocation Loc); - NamedDecl *LazilyCreateBuiltin(IdentifierInfo *II, unsigned ID, - Scope *S, bool ForRedeclaration, - SourceLocation Loc); - NamedDecl *ImplicitlyDefineFunction(SourceLocation Loc, IdentifierInfo &II, - Scope *S); - void AddKnownFunctionAttributesForReplaceableGlobalAllocationFunction( - FunctionDecl *FD); - void AddKnownFunctionAttributes(FunctionDecl *FD); + Decl *ActOnUsingDeclaration(Scope *CurScope, AccessSpecifier AS, + SourceLocation UsingLoc, + SourceLocation TypenameLoc, CXXScopeSpec &SS, + UnqualifiedId &Name, SourceLocation EllipsisLoc, + const ParsedAttributesView &AttrList); + Decl *ActOnUsingEnumDeclaration(Scope *CurScope, AccessSpecifier AS, + SourceLocation UsingLoc, + SourceLocation EnumLoc, + SourceLocation IdentLoc, IdentifierInfo &II, + CXXScopeSpec *SS = nullptr); + Decl *ActOnAliasDeclaration(Scope *CurScope, AccessSpecifier AS, + MultiTemplateParamsArg TemplateParams, + SourceLocation UsingLoc, UnqualifiedId &Name, + const ParsedAttributesView &AttrList, + TypeResult Type, Decl *DeclFromDeclSpec); - // More parsing and symbol table subroutines. + /// BuildCXXConstructExpr - Creates a complete call to a constructor, + /// including handling of its default argument expressions. + /// + /// \param ConstructKind - a CXXConstructExpr::ConstructionKind + ExprResult BuildCXXConstructExpr( + SourceLocation ConstructLoc, QualType DeclInitType, NamedDecl *FoundDecl, + CXXConstructorDecl *Constructor, MultiExprArg Exprs, + bool HadMultipleCandidates, bool IsListInitialization, + bool IsStdInitListInitialization, bool RequiresZeroInit, + CXXConstructionKind ConstructKind, SourceRange ParenRange); - void ProcessPragmaWeak(Scope *S, Decl *D); - // Decl attributes - this routine is the top level dispatcher. - void ProcessDeclAttributes(Scope *S, Decl *D, const Declarator &PD); - // Helper for delayed processing of attributes. - void ProcessDeclAttributeDelayed(Decl *D, - const ParsedAttributesView &AttrList); + /// Build a CXXConstructExpr whose constructor has already been resolved if + /// it denotes an inherited constructor. + ExprResult BuildCXXConstructExpr( + SourceLocation ConstructLoc, QualType DeclInitType, + CXXConstructorDecl *Constructor, bool Elidable, MultiExprArg Exprs, + bool HadMultipleCandidates, bool IsListInitialization, + bool IsStdInitListInitialization, bool RequiresZeroInit, + CXXConstructionKind ConstructKind, SourceRange ParenRange); - // Options for ProcessDeclAttributeList(). - struct ProcessDeclAttributeOptions { - ProcessDeclAttributeOptions() - : IncludeCXX11Attributes(true), IgnoreTypeAttributes(false) {} + // FIXME: Can we remove this and have the above BuildCXXConstructExpr check if + // the constructor can be elidable? + ExprResult BuildCXXConstructExpr( + SourceLocation ConstructLoc, QualType DeclInitType, NamedDecl *FoundDecl, + CXXConstructorDecl *Constructor, bool Elidable, MultiExprArg Exprs, + bool HadMultipleCandidates, bool IsListInitialization, + bool IsStdInitListInitialization, bool RequiresZeroInit, + CXXConstructionKind ConstructKind, SourceRange ParenRange); - ProcessDeclAttributeOptions WithIncludeCXX11Attributes(bool Val) { - ProcessDeclAttributeOptions Result = *this; - Result.IncludeCXX11Attributes = Val; - return Result; + ExprResult ConvertMemberDefaultInitExpression(FieldDecl *FD, Expr *InitExpr, + SourceLocation InitLoc); + + /// FinalizeVarWithDestructor - Prepare for calling destructor on the + /// constructed variable. + void FinalizeVarWithDestructor(VarDecl *VD, const RecordType *DeclInitType); + + /// Helper class that collects exception specifications for + /// implicitly-declared special member functions. + class ImplicitExceptionSpecification { + // Pointer to allow copying + Sema *Self; + // We order exception specifications thus: + // noexcept is the most restrictive, but is only used in C++11. + // throw() comes next. + // Then a throw(collected exceptions) + // Finally no specification, which is expressed as noexcept(false). + // throw(...) is used instead if any called function uses it. + ExceptionSpecificationType ComputedEST; + llvm::SmallPtrSet ExceptionsSeen; + SmallVector Exceptions; + + void ClearExceptions() { + ExceptionsSeen.clear(); + Exceptions.clear(); } - ProcessDeclAttributeOptions WithIgnoreTypeAttributes(bool Val) { - ProcessDeclAttributeOptions Result = *this; - Result.IgnoreTypeAttributes = Val; - return Result; + public: + explicit ImplicitExceptionSpecification(Sema &Self) + : Self(&Self), ComputedEST(EST_BasicNoexcept) { + if (!Self.getLangOpts().CPlusPlus11) + ComputedEST = EST_DynamicNone; } - // Should C++11 attributes be processed? - bool IncludeCXX11Attributes; + /// Get the computed exception specification type. + ExceptionSpecificationType getExceptionSpecType() const { + assert(!isComputedNoexcept(ComputedEST) && + "noexcept(expr) should not be a possible result"); + return ComputedEST; + } - // Should any type attributes encountered be ignored? - // If this option is false, a diagnostic will be emitted for any type - // attributes of a kind that does not "slide" from the declaration to - // the decl-specifier-seq. - bool IgnoreTypeAttributes; - }; - - void ProcessDeclAttributeList(Scope *S, Decl *D, - const ParsedAttributesView &AttrList, - const ProcessDeclAttributeOptions &Options = - ProcessDeclAttributeOptions()); - bool ProcessAccessDeclAttributeList(AccessSpecDecl *ASDecl, - const ParsedAttributesView &AttrList); + /// The number of exceptions in the exception specification. + unsigned size() const { return Exceptions.size(); } - void checkUnusedDeclAttributes(Declarator &D); + /// The set of exceptions in the exception specification. + const QualType *data() const { return Exceptions.data(); } - /// Handles semantic checking for features that are common to all attributes, - /// such as checking whether a parameter was properly specified, or the - /// correct number of arguments were passed, etc. Returns true if the - /// attribute has been diagnosed. - bool checkCommonAttributeFeatures(const Decl *D, const ParsedAttr &A, - bool SkipArgCountCheck = false); - bool checkCommonAttributeFeatures(const Stmt *S, const ParsedAttr &A, - bool SkipArgCountCheck = false); + /// Integrate another called method into the collected data. + void CalledDecl(SourceLocation CallLoc, const CXXMethodDecl *Method); - /// Map any API notes provided for this declaration to attributes on the - /// declaration. - /// - /// Triggered by declaration-attribute processing. - void ProcessAPINotes(Decl *D); + /// Integrate an invoked expression into the collected data. + void CalledExpr(Expr *E) { CalledStmt(E); } - /// Determine if type T is a valid subject for a nonnull and similar - /// attributes. By default, we look through references (the behavior used by - /// nonnull), but if the second parameter is true, then we treat a reference - /// type as valid. - bool isValidPointerAttrType(QualType T, bool RefOkay = false); + /// Integrate an invoked statement into the collected data. + void CalledStmt(Stmt *S); - bool CheckRegparmAttr(const ParsedAttr &attr, unsigned &value); + /// Overwrite an EPI's exception specification with this + /// computed exception specification. + FunctionProtoType::ExceptionSpecInfo getExceptionSpec() const { + FunctionProtoType::ExceptionSpecInfo ESI; + ESI.Type = getExceptionSpecType(); + if (ESI.Type == EST_Dynamic) { + ESI.Exceptions = Exceptions; + } else if (ESI.Type == EST_None) { + /// C++11 [except.spec]p14: + /// The exception-specification is noexcept(false) if the set of + /// potential exceptions of the special member function contains "any" + ESI.Type = EST_NoexceptFalse; + ESI.NoexceptExpr = + Self->ActOnCXXBoolLiteral(SourceLocation(), tok::kw_false).get(); + } + return ESI; + } + }; - /// Check validaty of calling convention attribute \p attr. If \p FD - /// is not null pointer, use \p FD to determine the CUDA/HIP host/device - /// target. Otherwise, it is specified by \p CFT. - bool CheckCallingConvAttr(const ParsedAttr &attr, CallingConv &CC, - const FunctionDecl *FD = nullptr, - CUDAFunctionTarget CFT = CFT_InvalidTarget); - bool CheckAttrTarget(const ParsedAttr &CurrAttr); - bool CheckAttrNoArgs(const ParsedAttr &CurrAttr); - bool checkStringLiteralArgumentAttr(const AttributeCommonInfo &CI, - const Expr *E, StringRef &Str, - SourceLocation *ArgLocation = nullptr); - bool checkStringLiteralArgumentAttr(const ParsedAttr &Attr, unsigned ArgNum, - StringRef &Str, - SourceLocation *ArgLocation = nullptr); - llvm::Error isValidSectionSpecifier(StringRef Str); - bool checkSectionName(SourceLocation LiteralLoc, StringRef Str); - bool checkTargetAttr(SourceLocation LiteralLoc, StringRef Str); - bool checkTargetVersionAttr(SourceLocation LiteralLoc, Decl *D, - StringRef &Str, bool &isDefault); - bool checkTargetClonesAttrString( - SourceLocation LiteralLoc, StringRef Str, const StringLiteral *Literal, - Decl *D, bool &HasDefault, bool &HasCommas, bool &HasNotDefault, - SmallVectorImpl> &StringsBuffer); - bool checkMSInheritanceAttrOnDefinition( - CXXRecordDecl *RD, SourceRange Range, bool BestCase, - MSInheritanceModel SemanticSpelling); + /// Evaluate the implicit exception specification for a defaulted + /// special member function. + void EvaluateImplicitExceptionSpec(SourceLocation Loc, FunctionDecl *FD); - void CheckAlignasUnderalignment(Decl *D); + /// Check the given exception-specification and update the + /// exception specification information with the results. + void checkExceptionSpecification(bool IsTopLevel, + ExceptionSpecificationType EST, + ArrayRef DynamicExceptions, + ArrayRef DynamicExceptionRanges, + Expr *NoexceptExpr, + SmallVectorImpl &Exceptions, + FunctionProtoType::ExceptionSpecInfo &ESI); - bool CheckNoInlineAttr(const Stmt *OrigSt, const Stmt *CurSt, - const AttributeCommonInfo &A); - bool CheckAlwaysInlineAttr(const Stmt *OrigSt, const Stmt *CurSt, - const AttributeCommonInfo &A); + /// Add an exception-specification to the given member function + /// (or member function template). The exception-specification was parsed + /// after the method itself was declared. + void actOnDelayedExceptionSpecification( + Decl *Method, ExceptionSpecificationType EST, + SourceRange SpecificationRange, ArrayRef DynamicExceptions, + ArrayRef DynamicExceptionRanges, Expr *NoexceptExpr); - bool CheckCountedByAttr(Scope *Scope, const FieldDecl *FD); + /// Kinds of C++ special members. + enum CXXSpecialMember { + CXXDefaultConstructor, + CXXCopyConstructor, + CXXMoveConstructor, + CXXCopyAssignment, + CXXMoveAssignment, + CXXDestructor, + CXXInvalid + }; - /// Adjust the calling convention of a method to be the ABI default if it - /// wasn't specified explicitly. This handles method types formed from - /// function type typedefs and typename template arguments. - void adjustMemberFunctionCC(QualType &T, bool HasThisPointer, - bool IsCtorOrDtor, SourceLocation Loc); + class InheritedConstructorInfo; - // Check if there is an explicit attribute, but only look through parens. - // The intent is to look for an attribute on the current declarator, but not - // one that came from a typedef. - bool hasExplicitCallingConv(QualType T); + /// Determine if a special member function should have a deleted + /// definition when it is defaulted. + bool ShouldDeleteSpecialMember(CXXMethodDecl *MD, CXXSpecialMember CSM, + InheritedConstructorInfo *ICI = nullptr, + bool Diagnose = false); - /// Get the outermost AttributedType node that sets a calling convention. - /// Valid types should not have multiple attributes with different CCs. - const AttributedType *getCallingConvAttributedType(QualType T) const; + /// Produce notes explaining why a defaulted function was defined as deleted. + void DiagnoseDeletedDefaultedFunction(FunctionDecl *FD); - /// Check whether a nullability type specifier can be added to the given - /// type through some means not written in source (e.g. API notes). + /// Declare the implicit default constructor for the given class. /// - /// \param Type The type to which the nullability specifier will be - /// added. On success, this type will be updated appropriately. + /// \param ClassDecl The class declaration into which the implicit + /// default constructor will be added. /// - /// \param Nullability The nullability specifier to add. + /// \returns The implicitly-declared default constructor. + CXXConstructorDecl * + DeclareImplicitDefaultConstructor(CXXRecordDecl *ClassDecl); + + /// DefineImplicitDefaultConstructor - Checks for feasibility of + /// defining this constructor as the default constructor. + void DefineImplicitDefaultConstructor(SourceLocation CurrentLocation, + CXXConstructorDecl *Constructor); + + /// Declare the implicit destructor for the given class. /// - /// \param DiagLoc The location to use for diagnostics. + /// \param ClassDecl The class declaration into which the implicit + /// destructor will be added. /// - /// \param AllowArrayTypes Whether to accept nullability specifiers on an - /// array type (e.g., because it will decay to a pointer). + /// \returns The implicitly-declared destructor. + CXXDestructorDecl *DeclareImplicitDestructor(CXXRecordDecl *ClassDecl); + + /// DefineImplicitDestructor - Checks for feasibility of + /// defining this destructor as the default destructor. + void DefineImplicitDestructor(SourceLocation CurrentLocation, + CXXDestructorDecl *Destructor); + + /// Build an exception spec for destructors that don't have one. /// - /// \param OverrideExisting Whether to override an existing, locally-specified - /// nullability specifier rather than complaining about the conflict. + /// C++11 says that user-defined destructors with no exception spec get one + /// that looks as if the destructor was implicitly declared. + void AdjustDestructorExceptionSpec(CXXDestructorDecl *Destructor); + + /// Define the specified inheriting constructor. + void DefineInheritingConstructor(SourceLocation UseLoc, + CXXConstructorDecl *Constructor); + + /// Declare the implicit copy constructor for the given class. /// - /// \returns true if nullability cannot be applied, false otherwise. - bool CheckImplicitNullabilityTypeSpecifier(QualType &Type, - NullabilityKind Nullability, - SourceLocation DiagLoc, - bool AllowArrayTypes, - bool OverrideExisting); + /// \param ClassDecl The class declaration into which the implicit + /// copy constructor will be added. + /// + /// \returns The implicitly-declared copy constructor. + CXXConstructorDecl *DeclareImplicitCopyConstructor(CXXRecordDecl *ClassDecl); - /// Process the attributes before creating an attributed statement. Returns - /// the semantic attributes that have been processed. - void ProcessStmtAttributes(Stmt *Stmt, const ParsedAttributes &InAttrs, - SmallVectorImpl &OutAttrs); + /// DefineImplicitCopyConstructor - Checks for feasibility of + /// defining this constructor as the copy constructor. + void DefineImplicitCopyConstructor(SourceLocation CurrentLocation, + CXXConstructorDecl *Constructor); - void WarnConflictingTypedMethods(ObjCMethodDecl *Method, - ObjCMethodDecl *MethodDecl, - bool IsProtocolMethodDecl); + /// Declare the implicit move constructor for the given class. + /// + /// \param ClassDecl The Class declaration into which the implicit + /// move constructor will be added. + /// + /// \returns The implicitly-declared move constructor, or NULL if it wasn't + /// declared. + CXXConstructorDecl *DeclareImplicitMoveConstructor(CXXRecordDecl *ClassDecl); - void CheckConflictingOverridingMethod(ObjCMethodDecl *Method, - ObjCMethodDecl *Overridden, - bool IsProtocolMethodDecl); + /// DefineImplicitMoveConstructor - Checks for feasibility of + /// defining this constructor as the move constructor. + void DefineImplicitMoveConstructor(SourceLocation CurrentLocation, + CXXConstructorDecl *Constructor); - /// WarnExactTypedMethods - This routine issues a warning if method - /// implementation declaration matches exactly that of its declaration. - void WarnExactTypedMethods(ObjCMethodDecl *Method, - ObjCMethodDecl *MethodDecl, - bool IsProtocolMethodDecl); + /// Declare the implicit copy assignment operator for the given class. + /// + /// \param ClassDecl The class declaration into which the implicit + /// copy assignment operator will be added. + /// + /// \returns The implicitly-declared copy assignment operator. + CXXMethodDecl *DeclareImplicitCopyAssignment(CXXRecordDecl *ClassDecl); - typedef llvm::SmallPtrSet SelectorSet; + /// Defines an implicitly-declared copy assignment operator. + void DefineImplicitCopyAssignment(SourceLocation CurrentLocation, + CXXMethodDecl *MethodDecl); - /// CheckImplementationIvars - This routine checks if the instance variables - /// listed in the implelementation match those listed in the interface. - void CheckImplementationIvars(ObjCImplementationDecl *ImpDecl, - ObjCIvarDecl **Fields, unsigned nIvars, - SourceLocation Loc); + /// Declare the implicit move assignment operator for the given class. + /// + /// \param ClassDecl The Class declaration into which the implicit + /// move assignment operator will be added. + /// + /// \returns The implicitly-declared move assignment operator, or NULL if it + /// wasn't declared. + CXXMethodDecl *DeclareImplicitMoveAssignment(CXXRecordDecl *ClassDecl); - /// ImplMethodsVsClassMethods - This is main routine to warn if any method - /// remains unimplemented in the class or category \@implementation. - void ImplMethodsVsClassMethods(Scope *S, ObjCImplDecl* IMPDecl, - ObjCContainerDecl* IDecl, - bool IncompleteImpl = false); + /// Defines an implicitly-declared move assignment operator. + void DefineImplicitMoveAssignment(SourceLocation CurrentLocation, + CXXMethodDecl *MethodDecl); - /// DiagnoseUnimplementedProperties - This routine warns on those properties - /// which must be implemented by this implementation. - void DiagnoseUnimplementedProperties(Scope *S, ObjCImplDecl* IMPDecl, - ObjCContainerDecl *CDecl, - bool SynthesizeProperties); + /// Check a completed declaration of an implicit special member. + void CheckImplicitSpecialMemberDeclaration(Scope *S, FunctionDecl *FD); - /// Diagnose any null-resettable synthesized setters. - void diagnoseNullResettableSynthesizedSetters(const ObjCImplDecl *impDecl); + /// Determine whether the given function is an implicitly-deleted + /// special member function. + bool isImplicitlyDeleted(FunctionDecl *FD); - /// DefaultSynthesizeProperties - This routine default synthesizes all - /// properties which must be synthesized in the class's \@implementation. - void DefaultSynthesizeProperties(Scope *S, ObjCImplDecl *IMPDecl, - ObjCInterfaceDecl *IDecl, - SourceLocation AtEnd); - void DefaultSynthesizeProperties(Scope *S, Decl *D, SourceLocation AtEnd); + /// Check whether 'this' shows up in the type of a static member + /// function after the (naturally empty) cv-qualifier-seq would be. + /// + /// \returns true if an error occurred. + bool checkThisInStaticMemberFunctionType(CXXMethodDecl *Method); - /// IvarBacksCurrentMethodAccessor - This routine returns 'true' if 'IV' is - /// an ivar synthesized for 'Method' and 'Method' is a property accessor - /// declared in class 'IFace'. - bool IvarBacksCurrentMethodAccessor(ObjCInterfaceDecl *IFace, - ObjCMethodDecl *Method, ObjCIvarDecl *IV); + /// Whether this' shows up in the exception specification of a static + /// member function. + bool checkThisInStaticMemberFunctionExceptionSpec(CXXMethodDecl *Method); - /// DiagnoseUnusedBackingIvarInAccessor - Issue an 'unused' warning if ivar which - /// backs the property is not used in the property's accessor. - void DiagnoseUnusedBackingIvarInAccessor(Scope *S, - const ObjCImplementationDecl *ImplD); + /// Check whether 'this' shows up in the attributes of the given + /// static member function. + /// + /// \returns true if an error occurred. + bool checkThisInStaticMemberFunctionAttributes(CXXMethodDecl *Method); - /// GetIvarBackingPropertyAccessor - If method is a property setter/getter and - /// it property has a backing ivar, returns this ivar; otherwise, returns NULL. - /// It also returns ivar's property on success. - ObjCIvarDecl *GetIvarBackingPropertyAccessor(const ObjCMethodDecl *Method, - const ObjCPropertyDecl *&PDecl) const; + bool CheckImmediateEscalatingFunctionDefinition( + FunctionDecl *FD, const sema::FunctionScopeInfo *FSI); - /// Called by ActOnProperty to handle \@property declarations in - /// class extensions. - ObjCPropertyDecl *HandlePropertyInClassExtension(Scope *S, - SourceLocation AtLoc, - SourceLocation LParenLoc, - FieldDeclarator &FD, - Selector GetterSel, - SourceLocation GetterNameLoc, - Selector SetterSel, - SourceLocation SetterNameLoc, - const bool isReadWrite, - unsigned &Attributes, - const unsigned AttributesAsWritten, - QualType T, - TypeSourceInfo *TSI, - tok::ObjCKeywordKind MethodImplKind); + void DiagnoseImmediateEscalatingReason(FunctionDecl *FD); - /// Called by ActOnProperty and HandlePropertyInClassExtension to - /// handle creating the ObjcPropertyDecl for a category or \@interface. - ObjCPropertyDecl *CreatePropertyDecl(Scope *S, - ObjCContainerDecl *CDecl, - SourceLocation AtLoc, - SourceLocation LParenLoc, - FieldDeclarator &FD, - Selector GetterSel, - SourceLocation GetterNameLoc, - Selector SetterSel, - SourceLocation SetterNameLoc, - const bool isReadWrite, - const unsigned Attributes, - const unsigned AttributesAsWritten, - QualType T, - TypeSourceInfo *TSI, - tok::ObjCKeywordKind MethodImplKind, - DeclContext *lexicalDC = nullptr); + bool CompleteConstructorCall(CXXConstructorDecl *Constructor, + QualType DeclInitType, MultiExprArg ArgsPtr, + SourceLocation Loc, + SmallVectorImpl &ConvertedArgs, + bool AllowExplicit = false, + bool IsListInitialization = false); - /// AtomicPropertySetterGetterRules - This routine enforces the rule (via - /// warning) when atomic property has one but not the other user-declared - /// setter or getter. - void AtomicPropertySetterGetterRules(ObjCImplDecl* IMPDecl, - ObjCInterfaceDecl* IDecl); + /// ActOnCXXEnterDeclInitializer - Invoked when we are about to parse an + /// initializer for the declaration 'Dcl'. + /// After this method is called, according to [C++ 3.4.1p13], if 'Dcl' is a + /// static data member of class X, names should be looked up in the scope of + /// class X. + void ActOnCXXEnterDeclInitializer(Scope *S, Decl *Dcl); - void DiagnoseOwningPropertyGetterSynthesis(const ObjCImplementationDecl *D); + /// ActOnCXXExitDeclInitializer - Invoked after we are finished parsing an + /// initializer for the declaration 'Dcl'. + void ActOnCXXExitDeclInitializer(Scope *S, Decl *Dcl); - void DiagnoseMissingDesignatedInitOverrides( - const ObjCImplementationDecl *ImplD, - const ObjCInterfaceDecl *IFD); + /// Define the "body" of the conversion from a lambda object to a + /// function pointer. + /// + /// This routine doesn't actually define a sensible body; rather, it fills + /// in the initialization expression needed to copy the lambda object into + /// the block, and IR generation actually generates the real body of the + /// block pointer conversion. + void + DefineImplicitLambdaToFunctionPointerConversion(SourceLocation CurrentLoc, + CXXConversionDecl *Conv); - void DiagnoseDuplicateIvars(ObjCInterfaceDecl *ID, ObjCInterfaceDecl *SID); + /// Define the "body" of the conversion from a lambda object to a + /// block pointer. + /// + /// This routine doesn't actually define a sensible body; rather, it fills + /// in the initialization expression needed to copy the lambda object into + /// the block, and IR generation actually generates the real body of the + /// block pointer conversion. + void DefineImplicitLambdaToBlockPointerConversion(SourceLocation CurrentLoc, + CXXConversionDecl *Conv); - enum MethodMatchStrategy { - MMS_loose, - MMS_strict - }; + Decl *ActOnStartLinkageSpecification(Scope *S, SourceLocation ExternLoc, + Expr *LangStr, SourceLocation LBraceLoc); + Decl *ActOnFinishLinkageSpecification(Scope *S, Decl *LinkageSpec, + SourceLocation RBraceLoc); - /// MatchTwoMethodDeclarations - Checks if two methods' type match and returns - /// true, or false, accordingly. - bool MatchTwoMethodDeclarations(const ObjCMethodDecl *Method, - const ObjCMethodDecl *PrevMethod, - MethodMatchStrategy strategy = MMS_strict); + //===--------------------------------------------------------------------===// + // C++ Classes + // + CXXRecordDecl *getCurrentClass(Scope *S, const CXXScopeSpec *SS); + bool isCurrentClassName(const IdentifierInfo &II, Scope *S, + const CXXScopeSpec *SS = nullptr); + bool isCurrentClassNameTypo(IdentifierInfo *&II, const CXXScopeSpec *SS); - /// MatchAllMethodDeclarations - Check methods declaraed in interface or - /// or protocol against those declared in their implementations. - void MatchAllMethodDeclarations(const SelectorSet &InsMap, - const SelectorSet &ClsMap, - SelectorSet &InsMapSeen, - SelectorSet &ClsMapSeen, - ObjCImplDecl* IMPDecl, - ObjCContainerDecl* IDecl, - bool &IncompleteImpl, - bool ImmediateClass, - bool WarnCategoryMethodImpl=false); + bool ActOnAccessSpecifier(AccessSpecifier Access, SourceLocation ASLoc, + SourceLocation ColonLoc, + const ParsedAttributesView &Attrs); - /// CheckCategoryVsClassMethodMatches - Checks that methods implemented in - /// category matches with those implemented in its primary class and - /// warns each time an exact match is found. - void CheckCategoryVsClassMethodMatches(ObjCCategoryImplDecl *CatIMP); + NamedDecl * + ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D, + MultiTemplateParamsArg TemplateParameterLists, + Expr *BitfieldWidth, const VirtSpecifiers &VS, + InClassInitStyle InitStyle); - /// Add the given method to the list of globally-known methods. - void addMethodToGlobalList(ObjCMethodList *List, ObjCMethodDecl *Method); + void ActOnStartCXXInClassMemberInitializer(); + void ActOnFinishCXXInClassMemberInitializer(Decl *VarDecl, + SourceLocation EqualLoc, + Expr *Init); - /// Returns default addr space for method qualifiers. - LangAS getDefaultCXXMethodAddrSpace() const; + MemInitResult + ActOnMemInitializer(Decl *ConstructorD, Scope *S, CXXScopeSpec &SS, + IdentifierInfo *MemberOrBase, ParsedType TemplateTypeTy, + const DeclSpec &DS, SourceLocation IdLoc, + SourceLocation LParenLoc, ArrayRef Args, + SourceLocation RParenLoc, SourceLocation EllipsisLoc); -private: - /// AddMethodToGlobalPool - Add an instance or factory method to the global - /// pool. See descriptoin of AddInstanceMethodToGlobalPool. - void AddMethodToGlobalPool(ObjCMethodDecl *Method, bool impl, bool instance); + MemInitResult ActOnMemInitializer(Decl *ConstructorD, Scope *S, + CXXScopeSpec &SS, + IdentifierInfo *MemberOrBase, + ParsedType TemplateTypeTy, + const DeclSpec &DS, SourceLocation IdLoc, + Expr *InitList, SourceLocation EllipsisLoc); - /// LookupMethodInGlobalPool - Returns the instance or factory method and - /// optionally warns if there are multiple signatures. - ObjCMethodDecl *LookupMethodInGlobalPool(Selector Sel, SourceRange R, - bool receiverIdOrClass, - bool instance); + MemInitResult BuildMemInitializer(Decl *ConstructorD, Scope *S, + CXXScopeSpec &SS, + IdentifierInfo *MemberOrBase, + ParsedType TemplateTypeTy, + const DeclSpec &DS, SourceLocation IdLoc, + Expr *Init, SourceLocation EllipsisLoc); -public: - /// - Returns instance or factory methods in global method pool for - /// given selector. It checks the desired kind first, if none is found, and - /// parameter checkTheOther is set, it then checks the other kind. If no such - /// method or only one method is found, function returns false; otherwise, it - /// returns true. - bool - CollectMultipleMethodsInGlobalPool(Selector Sel, - SmallVectorImpl& Methods, - bool InstanceFirst, bool CheckTheOther, - const ObjCObjectType *TypeBound = nullptr); + MemInitResult BuildMemberInitializer(ValueDecl *Member, Expr *Init, + SourceLocation IdLoc); - bool - AreMultipleMethodsInGlobalPool(Selector Sel, ObjCMethodDecl *BestMethod, - SourceRange R, bool receiverIdOrClass, - SmallVectorImpl& Methods); + MemInitResult BuildBaseInitializer(QualType BaseType, + TypeSourceInfo *BaseTInfo, Expr *Init, + CXXRecordDecl *ClassDecl, + SourceLocation EllipsisLoc); - void - DiagnoseMultipleMethodInGlobalPool(SmallVectorImpl &Methods, - Selector Sel, SourceRange R, - bool receiverIdOrClass); + MemInitResult BuildDelegatingInitializer(TypeSourceInfo *TInfo, Expr *Init, + CXXRecordDecl *ClassDecl); -private: - /// - Returns a selector which best matches given argument list or - /// nullptr if none could be found - ObjCMethodDecl *SelectBestMethod(Selector Sel, MultiExprArg Args, - bool IsInstance, - SmallVectorImpl& Methods); + bool SetDelegatingInitializer(CXXConstructorDecl *Constructor, + CXXCtorInitializer *Initializer); + bool SetCtorInitializers( + CXXConstructorDecl *Constructor, bool AnyErrors, + ArrayRef Initializers = std::nullopt); - /// Record the typo correction failure and return an empty correction. - TypoCorrection FailedCorrection(IdentifierInfo *Typo, SourceLocation TypoLoc, - bool RecordFailure = true) { - if (RecordFailure) - TypoCorrectionFailures[Typo].insert(TypoLoc); - return TypoCorrection(); - } + void SetIvarInitializers(ObjCImplementationDecl *ObjCImplementation); -public: - /// AddInstanceMethodToGlobalPool - All instance methods in a translation - /// unit are added to a global pool. This allows us to efficiently associate - /// a selector with a method declaraation for purposes of typechecking - /// messages sent to "id" (where the class of the object is unknown). - void AddInstanceMethodToGlobalPool(ObjCMethodDecl *Method, bool impl=false) { - AddMethodToGlobalPool(Method, impl, /*instance*/true); - } + /// MarkBaseAndMemberDestructorsReferenced - Given a record decl, + /// mark all the non-trivial destructors of its members and bases as + /// referenced. + void MarkBaseAndMemberDestructorsReferenced(SourceLocation Loc, + CXXRecordDecl *Record); - /// AddFactoryMethodToGlobalPool - Same as above, but for factory methods. - void AddFactoryMethodToGlobalPool(ObjCMethodDecl *Method, bool impl=false) { - AddMethodToGlobalPool(Method, impl, /*instance*/false); - } + /// Mark destructors of virtual bases of this class referenced. In the Itanium + /// C++ ABI, this is done when emitting a destructor for any non-abstract + /// class. In the Microsoft C++ ABI, this is done any time a class's + /// destructor is referenced. + void MarkVirtualBaseDestructorsReferenced( + SourceLocation Location, CXXRecordDecl *ClassDecl, + llvm::SmallPtrSetImpl *DirectVirtualBases = nullptr); - /// AddAnyMethodToGlobalPool - Add any method, instance or factory to global - /// pool. - void AddAnyMethodToGlobalPool(Decl *D); + /// Do semantic checks to allow the complete destructor variant to be emitted + /// when the destructor is defined in another translation unit. In the Itanium + /// C++ ABI, destructor variants are emitted together. In the MS C++ ABI, they + /// can be emitted in separate TUs. To emit the complete variant, run a subset + /// of the checks performed when emitting a regular destructor. + void CheckCompleteDestructorVariant(SourceLocation CurrentLocation, + CXXDestructorDecl *Dtor); - /// LookupInstanceMethodInGlobalPool - Returns the method and warns if - /// there are multiple signatures. - ObjCMethodDecl *LookupInstanceMethodInGlobalPool(Selector Sel, SourceRange R, - bool receiverIdOrClass=false) { - return LookupMethodInGlobalPool(Sel, R, receiverIdOrClass, - /*instance*/true); - } + /// The list of classes whose vtables have been used within + /// this translation unit, and the source locations at which the + /// first use occurred. + typedef std::pair VTableUse; - /// LookupFactoryMethodInGlobalPool - Returns the method and warns if - /// there are multiple signatures. - ObjCMethodDecl *LookupFactoryMethodInGlobalPool(Selector Sel, SourceRange R, - bool receiverIdOrClass=false) { - return LookupMethodInGlobalPool(Sel, R, receiverIdOrClass, - /*instance*/false); - } + /// The list of vtables that are required but have not yet been + /// materialized. + SmallVector VTableUses; - const ObjCMethodDecl *SelectorsForTypoCorrection(Selector Sel, - QualType ObjectType=QualType()); - /// LookupImplementedMethodInGlobalPool - Returns the method which has an - /// implementation. - ObjCMethodDecl *LookupImplementedMethodInGlobalPool(Selector Sel); + /// The set of classes whose vtables have been used within + /// this translation unit, and a bit that will be true if the vtable is + /// required to be emitted (otherwise, it should be emitted only if needed + /// by code generation). + llvm::DenseMap VTablesUsed; - /// CollectIvarsToConstructOrDestruct - Collect those ivars which require - /// initialization. - void CollectIvarsToConstructOrDestruct(ObjCInterfaceDecl *OI, - SmallVectorImpl &Ivars); + /// Load any externally-stored vtable uses. + void LoadExternalVTableUses(); - //===--------------------------------------------------------------------===// - // Statement Parsing Callbacks: SemaStmt.cpp. -public: - class FullExprArg { - public: - FullExprArg() : E(nullptr) { } - FullExprArg(Sema &actions) : E(nullptr) { } + /// Note that the vtable for the given class was used at the + /// given location. + void MarkVTableUsed(SourceLocation Loc, CXXRecordDecl *Class, + bool DefinitionRequired = false); - ExprResult release() { - return E; - } + /// Mark the exception specifications of all virtual member functions + /// in the given class as needed. + void MarkVirtualMemberExceptionSpecsNeeded(SourceLocation Loc, + const CXXRecordDecl *RD); - Expr *get() const { return E; } + /// MarkVirtualMembersReferenced - Will mark all members of the given + /// CXXRecordDecl referenced. + void MarkVirtualMembersReferenced(SourceLocation Loc, const CXXRecordDecl *RD, + bool ConstexprOnly = false); - Expr *operator->() { - return E; - } + /// Define all of the vtables that have been used in this + /// translation unit and reference any virtual members used by those + /// vtables. + /// + /// \returns true if any work was done, false otherwise. + bool DefineUsedVTables(); - private: - // FIXME: No need to make the entire Sema class a friend when it's just - // Sema::MakeFullExpr that needs access to the constructor below. - friend class Sema; + void AddImplicitlyDeclaredMembersToClass(CXXRecordDecl *ClassDecl); - explicit FullExprArg(Expr *expr) : E(expr) {} + void ActOnMemInitializers(Decl *ConstructorDecl, SourceLocation ColonLoc, + ArrayRef MemInits, + bool AnyErrors); - Expr *E; - }; + /// Check class-level dllimport/dllexport attribute. The caller must + /// ensure that referenceDLLExportedClassMethods is called some point later + /// when all outer classes of Class are complete. + void checkClassLevelDLLAttribute(CXXRecordDecl *Class); + void checkClassLevelCodeSegAttribute(CXXRecordDecl *Class); - FullExprArg MakeFullExpr(Expr *Arg) { - return MakeFullExpr(Arg, Arg ? Arg->getExprLoc() : SourceLocation()); - } - FullExprArg MakeFullExpr(Expr *Arg, SourceLocation CC) { - return FullExprArg( - ActOnFinishFullExpr(Arg, CC, /*DiscardedValue*/ false).get()); - } - FullExprArg MakeFullDiscardedValueExpr(Expr *Arg) { - ExprResult FE = - ActOnFinishFullExpr(Arg, Arg ? Arg->getExprLoc() : SourceLocation(), - /*DiscardedValue*/ true); - return FullExprArg(FE.get()); - } + void referenceDLLExportedClassMethods(); - StmtResult ActOnExprStmt(ExprResult Arg, bool DiscardedValue = true); - StmtResult ActOnExprStmtError(); + void propagateDLLAttrToBaseClassTemplate( + CXXRecordDecl *Class, Attr *ClassAttr, + ClassTemplateSpecializationDecl *BaseTemplateSpec, + SourceLocation BaseLoc); - StmtResult ActOnNullStmt(SourceLocation SemiLoc, - bool HasLeadingEmptyMacro = false); + void CheckCompletedCXXClass(Scope *S, CXXRecordDecl *Record); - void ActOnStartOfCompoundStmt(bool IsStmtExpr); - void ActOnAfterCompoundStatementLeadingPragmas(); - void ActOnFinishOfCompoundStmt(); - StmtResult ActOnCompoundStmt(SourceLocation L, SourceLocation R, - ArrayRef Elts, bool isStmtExpr); + /// Check that the C++ class annoated with "trivial_abi" satisfies all the + /// conditions that are needed for the attribute to have an effect. + void checkIllFormedTrivialABIStruct(CXXRecordDecl &RD); - /// A RAII object to enter scope of a compound statement. - class CompoundScopeRAII { - public: - CompoundScopeRAII(Sema &S, bool IsStmtExpr = false) : S(S) { - S.ActOnStartOfCompoundStmt(IsStmtExpr); - } + void ActOnFinishCXXMemberSpecification(Scope *S, SourceLocation RLoc, + Decl *TagDecl, SourceLocation LBrac, + SourceLocation RBrac, + const ParsedAttributesView &AttrList); + void ActOnFinishCXXMemberDecls(); + void ActOnFinishCXXNonNestedClass(); - ~CompoundScopeRAII() { - S.ActOnFinishOfCompoundStmt(); - } + void ActOnReenterCXXMethodParameter(Scope *S, ParmVarDecl *Param); + unsigned ActOnReenterTemplateScope(Decl *Template, + llvm::function_ref EnterScope); + void ActOnStartDelayedMemberDeclarations(Scope *S, Decl *Record); + void ActOnStartDelayedCXXMethodDeclaration(Scope *S, Decl *Method); + void ActOnDelayedCXXMethodParameter(Scope *S, Decl *Param); + void ActOnFinishDelayedMemberDeclarations(Scope *S, Decl *Record); + void ActOnFinishDelayedCXXMethodDeclaration(Scope *S, Decl *Method); + void ActOnFinishDelayedMemberInitializers(Decl *Record); - private: - Sema &S; - }; + bool EvaluateStaticAssertMessageAsString(Expr *Message, std::string &Result, + ASTContext &Ctx, + bool ErrorOnInvalidMessage); + Decl *ActOnStaticAssertDeclaration(SourceLocation StaticAssertLoc, + Expr *AssertExpr, Expr *AssertMessageExpr, + SourceLocation RParenLoc); + Decl *BuildStaticAssertDeclaration(SourceLocation StaticAssertLoc, + Expr *AssertExpr, Expr *AssertMessageExpr, + SourceLocation RParenLoc, bool Failed); + void DiagnoseStaticAssertDetails(const Expr *E); - /// An RAII helper that pops function a function scope on exit. - struct FunctionScopeRAII { - Sema &S; - bool Active; - FunctionScopeRAII(Sema &S) : S(S), Active(true) {} - ~FunctionScopeRAII() { - if (Active) - S.PopFunctionScopeInfo(); - } - void disable() { Active = false; } - }; + Decl *ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS, + MultiTemplateParamsArg TemplateParams); + NamedDecl *ActOnFriendFunctionDecl(Scope *S, Declarator &D, + MultiTemplateParamsArg TemplateParams); - StmtResult ActOnDeclStmt(DeclGroupPtrTy Decl, - SourceLocation StartLoc, - SourceLocation EndLoc); - void ActOnForEachDeclStmt(DeclGroupPtrTy Decl); - StmtResult ActOnForEachLValueExpr(Expr *E); - ExprResult ActOnCaseExpr(SourceLocation CaseLoc, ExprResult Val); - StmtResult ActOnCaseStmt(SourceLocation CaseLoc, ExprResult LHS, - SourceLocation DotDotDotLoc, ExprResult RHS, - SourceLocation ColonLoc); - void ActOnCaseStmtBody(Stmt *CaseStmt, Stmt *SubStmt); + QualType CheckConstructorDeclarator(Declarator &D, QualType R, + StorageClass &SC); + void CheckConstructor(CXXConstructorDecl *Constructor); + QualType CheckDestructorDeclarator(Declarator &D, QualType R, + StorageClass &SC); + bool CheckDestructor(CXXDestructorDecl *Destructor); + void CheckConversionDeclarator(Declarator &D, QualType &R, StorageClass &SC); + Decl *ActOnConversionDeclarator(CXXConversionDecl *Conversion); + bool CheckDeductionGuideDeclarator(Declarator &D, QualType &R, + StorageClass &SC); - StmtResult ActOnDefaultStmt(SourceLocation DefaultLoc, - SourceLocation ColonLoc, - Stmt *SubStmt, Scope *CurScope); - StmtResult ActOnLabelStmt(SourceLocation IdentLoc, LabelDecl *TheDecl, - SourceLocation ColonLoc, Stmt *SubStmt); + void CheckExplicitlyDefaultedFunction(Scope *S, FunctionDecl *MD); - StmtResult BuildAttributedStmt(SourceLocation AttrsLoc, - ArrayRef Attrs, Stmt *SubStmt); - StmtResult ActOnAttributedStmt(const ParsedAttributes &AttrList, - Stmt *SubStmt); + bool CheckExplicitlyDefaultedSpecialMember(CXXMethodDecl *MD, + CXXSpecialMember CSM, + SourceLocation DefaultLoc); + void CheckDelayedMemberExceptionSpecs(); - class ConditionResult; + /// Kinds of defaulted comparison operator functions. + enum class DefaultedComparisonKind : unsigned char { + /// This is not a defaultable comparison operator. + None, + /// This is an operator== that should be implemented as a series of + /// subobject comparisons. + Equal, + /// This is an operator<=> that should be implemented as a series of + /// subobject comparisons. + ThreeWay, + /// This is an operator!= that should be implemented as a rewrite in terms + /// of a == comparison. + NotEqual, + /// This is an <, <=, >, or >= that should be implemented as a rewrite in + /// terms of a <=> comparison. + Relational, + }; - StmtResult ActOnIfStmt(SourceLocation IfLoc, IfStatementKind StatementKind, - SourceLocation LParenLoc, Stmt *InitStmt, - ConditionResult Cond, SourceLocation RParenLoc, - Stmt *ThenVal, SourceLocation ElseLoc, Stmt *ElseVal); - StmtResult BuildIfStmt(SourceLocation IfLoc, IfStatementKind StatementKind, - SourceLocation LParenLoc, Stmt *InitStmt, - ConditionResult Cond, SourceLocation RParenLoc, - Stmt *ThenVal, SourceLocation ElseLoc, Stmt *ElseVal); - StmtResult ActOnStartOfSwitchStmt(SourceLocation SwitchLoc, - SourceLocation LParenLoc, Stmt *InitStmt, - ConditionResult Cond, - SourceLocation RParenLoc); - StmtResult ActOnFinishSwitchStmt(SourceLocation SwitchLoc, - Stmt *Switch, Stmt *Body); - StmtResult ActOnWhileStmt(SourceLocation WhileLoc, SourceLocation LParenLoc, - ConditionResult Cond, SourceLocation RParenLoc, - Stmt *Body); - StmtResult ActOnDoStmt(SourceLocation DoLoc, Stmt *Body, - SourceLocation WhileLoc, SourceLocation CondLParen, - Expr *Cond, SourceLocation CondRParen); + bool CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *MD, + DefaultedComparisonKind DCK); + void DeclareImplicitEqualityComparison(CXXRecordDecl *RD, + FunctionDecl *Spaceship); + void DefineDefaultedComparison(SourceLocation Loc, FunctionDecl *FD, + DefaultedComparisonKind DCK); - StmtResult ActOnForStmt(SourceLocation ForLoc, - SourceLocation LParenLoc, - Stmt *First, - ConditionResult Second, - FullExprArg Third, - SourceLocation RParenLoc, - Stmt *Body); - ExprResult CheckObjCForCollectionOperand(SourceLocation forLoc, - Expr *collection); - StmtResult ActOnObjCForCollectionStmt(SourceLocation ForColLoc, - Stmt *First, Expr *collection, - SourceLocation RParenLoc); - StmtResult FinishObjCForCollectionStmt(Stmt *ForCollection, Stmt *Body); + void CheckExplicitObjectMemberFunction(Declarator &D, DeclarationName Name, + QualType R, bool IsLambda, + DeclContext *DC = nullptr); + void CheckExplicitObjectMemberFunction(DeclContext *DC, Declarator &D, + DeclarationName Name, QualType R); + void CheckExplicitObjectLambda(Declarator &D); - enum BuildForRangeKind { - /// Initial building of a for-range statement. - BFRK_Build, - /// Instantiation or recovery rebuild of a for-range statement. Don't - /// attempt any typo-correction. - BFRK_Rebuild, - /// Determining whether a for-range statement could be built. Avoid any - /// unnecessary or irreversible actions. - BFRK_Check - }; + //===--------------------------------------------------------------------===// + // C++ Derived Classes + // - StmtResult ActOnCXXForRangeStmt( - Scope *S, SourceLocation ForLoc, SourceLocation CoawaitLoc, - Stmt *InitStmt, Stmt *LoopVar, SourceLocation ColonLoc, Expr *Collection, - SourceLocation RParenLoc, BuildForRangeKind Kind, - ArrayRef LifetimeExtendTemps = {}); - StmtResult BuildCXXForRangeStmt( - SourceLocation ForLoc, SourceLocation CoawaitLoc, Stmt *InitStmt, - SourceLocation ColonLoc, Stmt *RangeDecl, Stmt *Begin, Stmt *End, - Expr *Cond, Expr *Inc, Stmt *LoopVarDecl, SourceLocation RParenLoc, - BuildForRangeKind Kind, - ArrayRef LifetimeExtendTemps = {}); - StmtResult FinishCXXForRangeStmt(Stmt *ForRange, Stmt *Body); + /// ActOnBaseSpecifier - Parsed a base specifier + CXXBaseSpecifier *CheckBaseSpecifier(CXXRecordDecl *Class, + SourceRange SpecifierRange, bool Virtual, + AccessSpecifier Access, + TypeSourceInfo *TInfo, + SourceLocation EllipsisLoc); - StmtResult ActOnGotoStmt(SourceLocation GotoLoc, - SourceLocation LabelLoc, - LabelDecl *TheDecl); - StmtResult ActOnIndirectGotoStmt(SourceLocation GotoLoc, - SourceLocation StarLoc, - Expr *DestExp); - StmtResult ActOnContinueStmt(SourceLocation ContinueLoc, Scope *CurScope); - StmtResult ActOnBreakStmt(SourceLocation BreakLoc, Scope *CurScope); + BaseResult ActOnBaseSpecifier(Decl *classdecl, SourceRange SpecifierRange, + const ParsedAttributesView &Attrs, bool Virtual, + AccessSpecifier Access, ParsedType basetype, + SourceLocation BaseLoc, + SourceLocation EllipsisLoc); - void ActOnCapturedRegionStart(SourceLocation Loc, Scope *CurScope, - CapturedRegionKind Kind, unsigned NumParams); - typedef std::pair CapturedParamNameType; - void ActOnCapturedRegionStart(SourceLocation Loc, Scope *CurScope, - CapturedRegionKind Kind, - ArrayRef Params, - unsigned OpenMPCaptureLevel = 0); - StmtResult ActOnCapturedRegionEnd(Stmt *S); - void ActOnCapturedRegionError(); - RecordDecl *CreateCapturedStmtRecordDecl(CapturedDecl *&CD, - SourceLocation Loc, - unsigned NumParams); + bool AttachBaseSpecifiers(CXXRecordDecl *Class, + MutableArrayRef Bases); + void ActOnBaseSpecifiers(Decl *ClassDecl, + MutableArrayRef Bases); - struct NamedReturnInfo { - const VarDecl *Candidate; + bool IsDerivedFrom(SourceLocation Loc, QualType Derived, QualType Base); + bool IsDerivedFrom(SourceLocation Loc, QualType Derived, QualType Base, + CXXBasePaths &Paths); - enum Status : uint8_t { None, MoveEligible, MoveEligibleAndCopyElidable }; - Status S; + // FIXME: I don't like this name. + void BuildBasePathArray(const CXXBasePaths &Paths, CXXCastPath &BasePath); - bool isMoveEligible() const { return S != None; }; - bool isCopyElidable() const { return S == MoveEligibleAndCopyElidable; } - }; - enum class SimplerImplicitMoveMode { ForceOff, Normal, ForceOn }; - NamedReturnInfo getNamedReturnInfo( - Expr *&E, SimplerImplicitMoveMode Mode = SimplerImplicitMoveMode::Normal); - NamedReturnInfo getNamedReturnInfo(const VarDecl *VD); - const VarDecl *getCopyElisionCandidate(NamedReturnInfo &Info, - QualType ReturnType); + bool CheckDerivedToBaseConversion(QualType Derived, QualType Base, + SourceLocation Loc, SourceRange Range, + CXXCastPath *BasePath = nullptr, + bool IgnoreAccess = false); + bool CheckDerivedToBaseConversion(QualType Derived, QualType Base, + unsigned InaccessibleBaseID, + unsigned AmbiguousBaseConvID, + SourceLocation Loc, SourceRange Range, + DeclarationName Name, CXXCastPath *BasePath, + bool IgnoreAccess = false); - ExprResult - PerformMoveOrCopyInitialization(const InitializedEntity &Entity, - const NamedReturnInfo &NRInfo, Expr *Value, - bool SupressSimplerImplicitMoves = false); + std::string getAmbiguousPathsDisplayString(CXXBasePaths &Paths); - StmtResult ActOnReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp, - Scope *CurScope); - StmtResult BuildReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp, - bool AllowRecovery = false); - StmtResult ActOnCapScopeReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp, - NamedReturnInfo &NRInfo, - bool SupressSimplerImplicitMoves); + bool CheckOverridingFunctionAttributes(const CXXMethodDecl *New, + const CXXMethodDecl *Old); - StmtResult ActOnGCCAsmStmt(SourceLocation AsmLoc, bool IsSimple, - bool IsVolatile, unsigned NumOutputs, - unsigned NumInputs, IdentifierInfo **Names, - MultiExprArg Constraints, MultiExprArg Exprs, - Expr *AsmString, MultiExprArg Clobbers, - unsigned NumLabels, - SourceLocation RParenLoc); + /// CheckOverridingFunctionReturnType - Checks whether the return types are + /// covariant, according to C++ [class.virtual]p5. + bool CheckOverridingFunctionReturnType(const CXXMethodDecl *New, + const CXXMethodDecl *Old); - void FillInlineAsmIdentifierInfo(Expr *Res, - llvm::InlineAsmIdentifierInfo &Info); - ExprResult LookupInlineAsmIdentifier(CXXScopeSpec &SS, - SourceLocation TemplateKWLoc, - UnqualifiedId &Id, - bool IsUnevaluatedContext); - bool LookupInlineAsmField(StringRef Base, StringRef Member, - unsigned &Offset, SourceLocation AsmLoc); - ExprResult LookupInlineAsmVarDeclField(Expr *RefExpr, StringRef Member, - SourceLocation AsmLoc); - StmtResult ActOnMSAsmStmt(SourceLocation AsmLoc, SourceLocation LBraceLoc, - ArrayRef AsmToks, - StringRef AsmString, - unsigned NumOutputs, unsigned NumInputs, - ArrayRef Constraints, - ArrayRef Clobbers, - ArrayRef Exprs, - SourceLocation EndLoc); - LabelDecl *GetOrCreateMSAsmLabel(StringRef ExternalLabelName, - SourceLocation Location, - bool AlwaysCreate); + // Check that the overriding method has no explicit object parameter. + bool CheckExplicitObjectOverride(CXXMethodDecl *New, + const CXXMethodDecl *Old); - VarDecl *BuildObjCExceptionDecl(TypeSourceInfo *TInfo, QualType ExceptionType, - SourceLocation StartLoc, - SourceLocation IdLoc, IdentifierInfo *Id, - bool Invalid = false); + bool CheckPureMethod(CXXMethodDecl *Method, SourceRange InitRange); - Decl *ActOnObjCExceptionDecl(Scope *S, Declarator &D); + /// CheckOverrideControl - Check C++11 override control semantics. + void CheckOverrideControl(NamedDecl *D); - StmtResult ActOnObjCAtCatchStmt(SourceLocation AtLoc, SourceLocation RParen, - Decl *Parm, Stmt *Body); + /// DiagnoseAbsenceOfOverrideControl - Diagnose if 'override' keyword was + /// not used in the declaration of an overriding method. + void DiagnoseAbsenceOfOverrideControl(NamedDecl *D, bool Inconsistent); - StmtResult ActOnObjCAtFinallyStmt(SourceLocation AtLoc, Stmt *Body); + /// CheckForFunctionMarkedFinal - Checks whether a virtual member function + /// overrides a virtual member function marked 'final', according to + /// C++11 [class.virtual]p4. + bool CheckIfOverriddenFunctionIsMarkedFinal(const CXXMethodDecl *New, + const CXXMethodDecl *Old); - StmtResult ActOnObjCAtTryStmt(SourceLocation AtLoc, Stmt *Try, - MultiStmtArg Catch, Stmt *Finally); + enum AbstractDiagSelID { + AbstractNone = -1, + AbstractReturnType, + AbstractParamType, + AbstractVariableType, + AbstractFieldType, + AbstractIvarType, + AbstractSynthesizedIvarType, + AbstractArrayType + }; - StmtResult BuildObjCAtThrowStmt(SourceLocation AtLoc, Expr *Throw); - StmtResult ActOnObjCAtThrowStmt(SourceLocation AtLoc, Expr *Throw, - Scope *CurScope); - ExprResult ActOnObjCAtSynchronizedOperand(SourceLocation atLoc, - Expr *operand); - StmtResult ActOnObjCAtSynchronizedStmt(SourceLocation AtLoc, - Expr *SynchExpr, - Stmt *SynchBody); + struct TypeDiagnoser; - StmtResult ActOnObjCAutoreleasePoolStmt(SourceLocation AtLoc, Stmt *Body); + bool isAbstractType(SourceLocation Loc, QualType T); + bool RequireNonAbstractType(SourceLocation Loc, QualType T, + TypeDiagnoser &Diagnoser); + template + bool RequireNonAbstractType(SourceLocation Loc, QualType T, unsigned DiagID, + const Ts &...Args) { + BoundTypeDiagnoser Diagnoser(DiagID, Args...); + return RequireNonAbstractType(Loc, T, Diagnoser); + } - VarDecl *BuildExceptionDeclaration(Scope *S, TypeSourceInfo *TInfo, - SourceLocation StartLoc, - SourceLocation IdLoc, - IdentifierInfo *Id); + void DiagnoseAbstractType(const CXXRecordDecl *RD); - Decl *ActOnExceptionDeclarator(Scope *S, Declarator &D); + //===--------------------------------------------------------------------===// + // C++ Overloaded Operators [C++ 13.5] + // - StmtResult ActOnCXXCatchBlock(SourceLocation CatchLoc, - Decl *ExDecl, Stmt *HandlerBlock); - StmtResult ActOnCXXTryBlock(SourceLocation TryLoc, Stmt *TryBlock, - ArrayRef Handlers); + bool CheckOverloadedOperatorDeclaration(FunctionDecl *FnDecl); - StmtResult ActOnSEHTryBlock(bool IsCXXTry, // try (true) or __try (false) ? - SourceLocation TryLoc, Stmt *TryBlock, - Stmt *Handler); - StmtResult ActOnSEHExceptBlock(SourceLocation Loc, - Expr *FilterExpr, - Stmt *Block); - void ActOnStartSEHFinallyBlock(); - void ActOnAbortSEHFinallyBlock(); - StmtResult ActOnFinishSEHFinallyBlock(SourceLocation Loc, Stmt *Block); - StmtResult ActOnSEHLeaveStmt(SourceLocation Loc, Scope *CurScope); + bool CheckLiteralOperatorDeclaration(FunctionDecl *FnDecl); - void DiagnoseReturnInConstructorExceptionHandler(CXXTryStmt *TryBlock); + /// ActOnExplicitBoolSpecifier - Build an ExplicitSpecifier from an expression + /// found in an explicit(bool) specifier. + ExplicitSpecifier ActOnExplicitBoolSpecifier(Expr *E); - bool ShouldWarnIfUnusedFileScopedDecl(const DeclaratorDecl *D) const; + /// tryResolveExplicitSpecifier - Attempt to resolve the explict specifier. + /// Returns true if the explicit specifier is now resolved. + bool tryResolveExplicitSpecifier(ExplicitSpecifier &ExplicitSpec); - /// If it's a file scoped decl that must warn if not used, keep track - /// of it. - void MarkUnusedFileScopedDecl(const DeclaratorDecl *D); + DeclResult ActOnCXXConditionDeclaration(Scope *S, Declarator &D); - typedef llvm::function_ref - DiagReceiverTy; + // Emitting members of dllexported classes is delayed until the class + // (including field initializers) is fully parsed. + SmallVector DelayedDllExportClasses; + SmallVector DelayedDllExportMemberFunctions; - /// DiagnoseUnusedExprResult - If the statement passed in is an expression - /// whose result is unused, warn. - void DiagnoseUnusedExprResult(const Stmt *S, unsigned DiagID); - void DiagnoseUnusedNestedTypedefs(const RecordDecl *D); - void DiagnoseUnusedNestedTypedefs(const RecordDecl *D, - DiagReceiverTy DiagReceiver); - void DiagnoseUnusedDecl(const NamedDecl *ND); - void DiagnoseUnusedDecl(const NamedDecl *ND, DiagReceiverTy DiagReceiver); + void MergeVarDeclExceptionSpecs(VarDecl *New, VarDecl *Old); + bool MergeCXXFunctionDecl(FunctionDecl *New, FunctionDecl *Old, Scope *S); - /// If VD is set but not otherwise used, diagnose, for a parameter or a - /// variable. - void DiagnoseUnusedButSetDecl(const VarDecl *VD, DiagReceiverTy DiagReceiver); + /// Helpers for dealing with blocks and functions. + void CheckCXXDefaultArguments(FunctionDecl *FD); + void CheckExtraCXXDefaultArguments(Declarator &D); - /// Emit \p DiagID if statement located on \p StmtLoc has a suspicious null - /// statement as a \p Body, and it is located on the same line. - /// - /// This helps prevent bugs due to typos, such as: - /// if (condition); - /// do_stuff(); - void DiagnoseEmptyStmtBody(SourceLocation StmtLoc, - const Stmt *Body, - unsigned DiagID); + CXXSpecialMember getSpecialMember(const CXXMethodDecl *MD) { + return getDefaultedFunctionKind(MD).asSpecialMember(); + } - /// Warn if a for/while loop statement \p S, which is followed by - /// \p PossibleBody, has a suspicious null statement as a body. - void DiagnoseEmptyLoopBody(const Stmt *S, - const Stmt *PossibleBody); + VarDecl *BuildExceptionDeclaration(Scope *S, TypeSourceInfo *TInfo, + SourceLocation StartLoc, + SourceLocation IdLoc, IdentifierInfo *Id); - /// Warn if a value is moved to itself. - void DiagnoseSelfMove(const Expr *LHSExpr, const Expr *RHSExpr, - SourceLocation OpLoc); + Decl *ActOnExceptionDeclarator(Scope *S, Declarator &D); - /// Returns a field in a CXXRecordDecl that has the same name as the decl \p - /// SelfAssigned when inside a CXXMethodDecl. - const FieldDecl * - getSelfAssignmentClassMemberCandidate(const ValueDecl *SelfAssigned); + void DiagnoseReturnInConstructorExceptionHandler(CXXTryStmt *TryBlock); - /// Warn if we're implicitly casting from a _Nullable pointer type to a - /// _Nonnull one. - void diagnoseNullableToNonnullConversion(QualType DstType, QualType SrcType, - SourceLocation Loc); + DeclResult ActOnTemplatedFriendTag(Scope *S, SourceLocation FriendLoc, + unsigned TagSpec, SourceLocation TagLoc, + CXXScopeSpec &SS, IdentifierInfo *Name, + SourceLocation NameLoc, + const ParsedAttributesView &Attr, + MultiTemplateParamsArg TempParamLists); - /// Warn when implicitly casting 0 to nullptr. - void diagnoseZeroToNullptrConversion(CastKind Kind, const Expr *E); + MSPropertyDecl *HandleMSProperty(Scope *S, RecordDecl *TagD, + SourceLocation DeclStart, Declarator &D, + Expr *BitfieldWidth, + InClassInitStyle InitStyle, + AccessSpecifier AS, + const ParsedAttr &MSPropertyAttr); - ParsingDeclState PushParsingDeclaration(sema::DelayedDiagnosticPool &pool) { - return DelayedDiagnostics.push(pool); - } - void PopParsingDeclaration(ParsingDeclState state, Decl *decl); + void DiagnoseNontrivial(const CXXRecordDecl *Record, CXXSpecialMember CSM); - typedef ProcessingContextState ParsingClassState; - ParsingClassState PushParsingClass() { - ParsingClassDepth++; - return DelayedDiagnostics.pushUndelayed(); - } - void PopParsingClass(ParsingClassState state) { - ParsingClassDepth--; - DelayedDiagnostics.popUndelayed(state); - } + enum TrivialABIHandling { + /// The triviality of a method unaffected by "trivial_abi". + TAH_IgnoreTrivialABI, - void redelayDiagnostics(sema::DelayedDiagnosticPool &pool); + /// The triviality of a method affected by "trivial_abi". + TAH_ConsiderTrivialABI + }; - void DiagnoseAvailabilityOfDecl(NamedDecl *D, ArrayRef Locs, - const ObjCInterfaceDecl *UnknownObjCClass, - bool ObjCPropertyAccess, - bool AvoidPartialAvailabilityChecks = false, - ObjCInterfaceDecl *ClassReceiver = nullptr); + bool SpecialMemberIsTrivial(CXXMethodDecl *MD, CXXSpecialMember CSM, + TrivialABIHandling TAH = TAH_IgnoreTrivialABI, + bool Diagnose = false); - bool makeUnavailableInSystemHeader(SourceLocation loc, - UnavailableAttr::ImplicitReason reason); + /// For a defaulted function, the kind of defaulted function that it is. + class DefaultedFunctionKind { + unsigned SpecialMember : 8; + unsigned Comparison : 8; - /// Issue any -Wunguarded-availability warnings in \c FD - void DiagnoseUnguardedAvailabilityViolations(Decl *FD); + public: + DefaultedFunctionKind() + : SpecialMember(CXXInvalid), + Comparison(llvm::to_underlying(DefaultedComparisonKind::None)) {} + DefaultedFunctionKind(CXXSpecialMember CSM) + : SpecialMember(CSM), + Comparison(llvm::to_underlying(DefaultedComparisonKind::None)) {} + DefaultedFunctionKind(DefaultedComparisonKind Comp) + : SpecialMember(CXXInvalid), Comparison(llvm::to_underlying(Comp)) {} - void handleDelayedAvailabilityCheck(sema::DelayedDiagnostic &DD, Decl *Ctx); + bool isSpecialMember() const { return SpecialMember != CXXInvalid; } + bool isComparison() const { + return static_cast(Comparison) != + DefaultedComparisonKind::None; + } - //===--------------------------------------------------------------------===// - // Expression Parsing Callbacks: SemaExpr.cpp. + explicit operator bool() const { + return isSpecialMember() || isComparison(); + } - bool CanUseDecl(NamedDecl *D, bool TreatUnavailableAsInvalid); - // A version of DiagnoseUseOfDecl that should be used if overload resolution - // has been used to find this declaration, which means we don't have to bother - // checking the trailing requires clause. - bool DiagnoseUseOfOverloadedDecl(NamedDecl *D, SourceLocation Loc) { - return DiagnoseUseOfDecl( - D, Loc, /*UnknownObjCClass=*/nullptr, /*ObjCPropertyAccess=*/false, - /*AvoidPartialAvailabilityChecks=*/false, /*ClassReceiver=*/nullptr, - /*SkipTrailingRequiresClause=*/true); - } + CXXSpecialMember asSpecialMember() const { + return static_cast(SpecialMember); + } + DefaultedComparisonKind asComparison() const { + return static_cast(Comparison); + } - bool DiagnoseUseOfDecl(NamedDecl *D, ArrayRef Locs, - const ObjCInterfaceDecl *UnknownObjCClass = nullptr, - bool ObjCPropertyAccess = false, - bool AvoidPartialAvailabilityChecks = false, - ObjCInterfaceDecl *ClassReciever = nullptr, - bool SkipTrailingRequiresClause = false); - void NoteDeletedFunction(FunctionDecl *FD); - void NoteDeletedInheritingConstructor(CXXConstructorDecl *CD); - bool DiagnosePropertyAccessorMismatch(ObjCPropertyDecl *PD, - ObjCMethodDecl *Getter, - SourceLocation Loc); - void DiagnoseSentinelCalls(const NamedDecl *D, SourceLocation Loc, - ArrayRef Args); + /// Get the index of this function kind for use in diagnostics. + unsigned getDiagnosticIndex() const { + static_assert(CXXInvalid > CXXDestructor, + "invalid should have highest index"); + static_assert((unsigned)DefaultedComparisonKind::None == 0, + "none should be equal to zero"); + return SpecialMember + Comparison; + } + }; - void PushExpressionEvaluationContext( - ExpressionEvaluationContext NewContext, Decl *LambdaContextDecl = nullptr, - ExpressionEvaluationContextRecord::ExpressionKind Type = - ExpressionEvaluationContextRecord::EK_Other); - enum ReuseLambdaContextDecl_t { ReuseLambdaContextDecl }; - void PushExpressionEvaluationContext( - ExpressionEvaluationContext NewContext, ReuseLambdaContextDecl_t, - ExpressionEvaluationContextRecord::ExpressionKind Type = - ExpressionEvaluationContextRecord::EK_Other); - void PopExpressionEvaluationContext(); + DefaultedFunctionKind getDefaultedFunctionKind(const FunctionDecl *FD); - void DiscardCleanupsInEvaluationContext(); - - ExprResult TransformToPotentiallyEvaluated(Expr *E); - TypeSourceInfo *TransformToPotentiallyEvaluated(TypeSourceInfo *TInfo); - ExprResult HandleExprEvaluationContextForTypeof(Expr *E); + /// Handle a C++11 empty-declaration and attribute-declaration. + Decl *ActOnEmptyDeclaration(Scope *S, const ParsedAttributesView &AttrList, + SourceLocation SemiLoc); - ExprResult CheckUnevaluatedOperand(Expr *E); - void CheckUnusedVolatileAssignment(Expr *E); + enum class CheckConstexprKind { + /// Diagnose issues that are non-constant or that are extensions. + Diagnose, + /// Identify whether this function satisfies the formal rules for constexpr + /// functions in the current lanugage mode (with no extensions). + CheckValid + }; - ExprResult ActOnConstantExpression(ExprResult Res); + bool CheckConstexprFunctionDefinition(const FunctionDecl *FD, + CheckConstexprKind Kind); - // Functions for marking a declaration referenced. These functions also - // contain the relevant logic for marking if a reference to a function or - // variable is an odr-use (in the C++11 sense). There are separate variants - // for expressions referring to a decl; these exist because odr-use marking - // needs to be delayed for some constant variables when we build one of the - // named expressions. - // - // MightBeOdrUse indicates whether the use could possibly be an odr-use, and - // should usually be true. This only needs to be set to false if the lack of - // odr-use cannot be determined from the current context (for instance, - // because the name denotes a virtual function and was written without an - // explicit nested-name-specifier). - void MarkAnyDeclReferenced(SourceLocation Loc, Decl *D, bool MightBeOdrUse); - void MarkFunctionReferenced(SourceLocation Loc, FunctionDecl *Func, - bool MightBeOdrUse = true); - void MarkVariableReferenced(SourceLocation Loc, VarDecl *Var); - void MarkDeclRefReferenced(DeclRefExpr *E, const Expr *Base = nullptr); - void MarkMemberReferenced(MemberExpr *E); - void MarkFunctionParmPackReferenced(FunctionParmPackExpr *E); - void MarkCaptureUsedInEnclosingContext(ValueDecl *Capture, SourceLocation Loc, - unsigned CapturingScopeIndex); + void DiagnoseHiddenVirtualMethods(CXXMethodDecl *MD); + void + FindHiddenVirtualMethods(CXXMethodDecl *MD, + SmallVectorImpl &OverloadedMethods); + void + NoteHiddenVirtualMethods(CXXMethodDecl *MD, + SmallVectorImpl &OverloadedMethods); + void ActOnParamDefaultArgument(Decl *param, SourceLocation EqualLoc, + Expr *defarg); + void ActOnParamUnparsedDefaultArgument(Decl *param, SourceLocation EqualLoc, + SourceLocation ArgLoc); + void ActOnParamDefaultArgumentError(Decl *param, SourceLocation EqualLoc, + Expr *DefaultArg); + ExprResult ConvertParamDefaultArgument(ParmVarDecl *Param, Expr *DefaultArg, + SourceLocation EqualLoc); + void SetParamDefaultArgument(ParmVarDecl *Param, Expr *DefaultArg, + SourceLocation EqualLoc); - ExprResult CheckLValueToRValueConversionOperand(Expr *E); - void CleanupVarDeclMarking(); + void ActOnPureSpecifier(Decl *D, SourceLocation PureSpecLoc); + void SetDeclDeleted(Decl *dcl, SourceLocation DelLoc); + void SetDeclDefaulted(Decl *dcl, SourceLocation DefaultLoc); - enum TryCaptureKind { - TryCapture_Implicit, TryCapture_ExplicitByVal, TryCapture_ExplicitByRef - }; + void SetFunctionBodyKind(Decl *D, SourceLocation Loc, FnBodyKind BodyKind); + void ActOnStartTrailingRequiresClause(Scope *S, Declarator &D); + ExprResult ActOnFinishTrailingRequiresClause(ExprResult ConstraintExpr); + ExprResult ActOnRequiresClause(ExprResult ConstraintExpr); - /// Try to capture the given variable. - /// - /// \param Var The variable to capture. - /// - /// \param Loc The location at which the capture occurs. - /// - /// \param Kind The kind of capture, which may be implicit (for either a - /// block or a lambda), or explicit by-value or by-reference (for a lambda). - /// - /// \param EllipsisLoc The location of the ellipsis, if one is provided in - /// an explicit lambda capture. - /// - /// \param BuildAndDiagnose Whether we are actually supposed to add the - /// captures or diagnose errors. If false, this routine merely check whether - /// the capture can occur without performing the capture itself or complaining - /// if the variable cannot be captured. - /// - /// \param CaptureType Will be set to the type of the field used to capture - /// this variable in the innermost block or lambda. Only valid when the - /// variable can be captured. - /// - /// \param DeclRefType Will be set to the type of a reference to the capture - /// from within the current scope. Only valid when the variable can be - /// captured. - /// - /// \param FunctionScopeIndexToStopAt If non-null, it points to the index - /// of the FunctionScopeInfo stack beyond which we do not attempt to capture. - /// This is useful when enclosing lambdas must speculatively capture - /// variables that may or may not be used in certain specializations of - /// a nested generic lambda. - /// - /// \returns true if an error occurred (i.e., the variable cannot be - /// captured) and false if the capture succeeded. - bool tryCaptureVariable(ValueDecl *Var, SourceLocation Loc, - TryCaptureKind Kind, SourceLocation EllipsisLoc, - bool BuildAndDiagnose, QualType &CaptureType, - QualType &DeclRefType, - const unsigned *const FunctionScopeIndexToStopAt); + NamedDecl * + ActOnDecompositionDeclarator(Scope *S, Declarator &D, + MultiTemplateParamsArg TemplateParamLists); + void DiagPlaceholderVariableDefinition(SourceLocation Loc); + bool DiagRedefinedPlaceholderFieldDecl(SourceLocation Loc, + RecordDecl *ClassDecl, + const IdentifierInfo *Name); - /// Try to capture the given variable. - bool tryCaptureVariable(ValueDecl *Var, SourceLocation Loc, - TryCaptureKind Kind = TryCapture_Implicit, - SourceLocation EllipsisLoc = SourceLocation()); + void CheckCompleteDecompositionDeclaration(DecompositionDecl *DD); - /// Checks if the variable must be captured. - bool NeedToCaptureVariable(ValueDecl *Var, SourceLocation Loc); + /// Stack containing information needed when in C++2a an 'auto' is encountered + /// in a function declaration parameter type specifier in order to invent a + /// corresponding template parameter in the enclosing abbreviated function + /// template. This information is also present in LambdaScopeInfo, stored in + /// the FunctionScopes stack. + SmallVector InventedParameterInfos; - /// Given a variable, determine the type that a reference to that - /// variable will have in the given scope. - QualType getCapturedDeclRefType(ValueDecl *Var, SourceLocation Loc); + /// FieldCollector - Collects CXXFieldDecls during parsing of C++ classes. + std::unique_ptr FieldCollector; - /// Mark all of the declarations referenced within a particular AST node as - /// referenced. Used when template instantiation instantiates a non-dependent - /// type -- entities referenced by the type are now referenced. - void MarkDeclarationsReferencedInType(SourceLocation Loc, QualType T); - void MarkDeclarationsReferencedInExpr( - Expr *E, bool SkipLocalVariables = false, - ArrayRef StopAt = std::nullopt); + typedef llvm::SmallSetVector NamedDeclSetType; + /// Set containing all declared private fields that are not used. + NamedDeclSetType UnusedPrivateFields; - /// Try to recover by turning the given expression into a - /// call. Returns true if recovery was attempted or an error was - /// emitted; this may also leave the ExprResult invalid. - bool tryToRecoverWithCall(ExprResult &E, const PartialDiagnostic &PD, - bool ForceComplain = false, - bool (*IsPlausibleResult)(QualType) = nullptr); + typedef llvm::SmallPtrSet RecordDeclSetTy; - /// Figure out if an expression could be turned into a call. - bool tryExprAsCall(Expr &E, QualType &ZeroArgCallReturnTy, - UnresolvedSetImpl &NonTemplateOverloads); + /// PureVirtualClassDiagSet - a set of class declarations which we have + /// emitted a list of pure virtual functions. Used to prevent emitting the + /// same list more than once. + std::unique_ptr PureVirtualClassDiagSet; - /// Try to convert an expression \p E to type \p Ty. Returns the result of the - /// conversion. - ExprResult tryConvertExprToType(Expr *E, QualType Ty); + typedef LazyVector + DelegatingCtorDeclsType; - /// Conditionally issue a diagnostic based on the statements's reachability - /// analysis. - /// - /// \param Stmts If Stmts is non-empty, delay reporting the diagnostic until - /// the function body is parsed, and then do a basic reachability analysis to - /// determine if the statement is reachable. If it is unreachable, the - /// diagnostic will not be emitted. - bool DiagIfReachable(SourceLocation Loc, ArrayRef Stmts, - const PartialDiagnostic &PD); + /// All the delegating constructors seen so far in the file, used for + /// cycle detection at the end of the TU. + DelegatingCtorDeclsType DelegatingCtorDecls; - /// Conditionally issue a diagnostic based on the current - /// evaluation context. - /// - /// \param Statement If Statement is non-null, delay reporting the - /// diagnostic until the function body is parsed, and then do a basic - /// reachability analysis to determine if the statement is reachable. - /// If it is unreachable, the diagnostic will not be emitted. - bool DiagRuntimeBehavior(SourceLocation Loc, const Stmt *Statement, - const PartialDiagnostic &PD); - /// Similar, but diagnostic is only produced if all the specified statements - /// are reachable. - bool DiagRuntimeBehavior(SourceLocation Loc, ArrayRef Stmts, - const PartialDiagnostic &PD); + /// The C++ "std" namespace, where the standard library resides. + LazyDeclPtr StdNamespace; - // Primary Expressions. - SourceRange getExprRange(Expr *E) const; + /// The C++ "std::initializer_list" template, which is defined in + /// \. + ClassTemplateDecl *StdInitializerList; - ExprResult ActOnIdExpression( - Scope *S, CXXScopeSpec &SS, SourceLocation TemplateKWLoc, - UnqualifiedId &Id, bool HasTrailingLParen, bool IsAddressOfOperand, - CorrectionCandidateCallback *CCC = nullptr, - bool IsInlineAsmIdentifier = false, Token *KeywordReplacement = nullptr); + // Contains the locations of the beginning of unparsed default + // argument locations. + llvm::DenseMap UnparsedDefaultArgLocs; - void DecomposeUnqualifiedId(const UnqualifiedId &Id, - TemplateArgumentListInfo &Buffer, - DeclarationNameInfo &NameInfo, - const TemplateArgumentListInfo *&TemplateArgs); + /// UndefinedInternals - all the used, undefined objects which require a + /// definition in this translation unit. + llvm::MapVector UndefinedButUsed; - bool DiagnoseDependentMemberLookup(const LookupResult &R); + typedef llvm::PointerIntPair + SpecialMemberDecl; - bool - DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R, - CorrectionCandidateCallback &CCC, - TemplateArgumentListInfo *ExplicitTemplateArgs = nullptr, - ArrayRef Args = std::nullopt, - DeclContext *LookupCtx = nullptr, - TypoExpr **Out = nullptr); + /// The C++ special members which we are currently in the process of + /// declaring. If this process recursively triggers the declaration of the + /// same special member, we should act as if it is not yet declared. + llvm::SmallPtrSet SpecialMembersBeingDeclared; - DeclResult LookupIvarInObjCMethod(LookupResult &Lookup, Scope *S, - IdentifierInfo *II); - ExprResult BuildIvarRefExpr(Scope *S, SourceLocation Loc, ObjCIvarDecl *IV); + void NoteDeletedInheritingConstructor(CXXConstructorDecl *CD); - ExprResult LookupInObjCMethod(LookupResult &LookUp, Scope *S, - IdentifierInfo *II, - bool AllowBuiltinCreation=false); + void ActOnDefaultCtorInitializers(Decl *CDtorDecl); - ExprResult ActOnDependentIdExpression(const CXXScopeSpec &SS, - SourceLocation TemplateKWLoc, - const DeclarationNameInfo &NameInfo, - bool isAddressOfOperand, - const TemplateArgumentListInfo *TemplateArgs); + typedef ProcessingContextState ParsingClassState; + ParsingClassState PushParsingClass() { + ParsingClassDepth++; + return DelayedDiagnostics.pushUndelayed(); + } + void PopParsingClass(ParsingClassState state) { + ParsingClassDepth--; + DelayedDiagnostics.popUndelayed(state); + } - /// If \p D cannot be odr-used in the current expression evaluation context, - /// return a reason explaining why. Otherwise, return NOUR_None. - NonOdrUseReason getNonOdrUseReasonInCurrentContext(ValueDecl *D); +private: + void setupImplicitSpecialMemberType(CXXMethodDecl *SpecialMem, + QualType ResultTy, + ArrayRef Args); - DeclRefExpr *BuildDeclRefExpr(ValueDecl *D, QualType Ty, ExprValueKind VK, - SourceLocation Loc, - const CXXScopeSpec *SS = nullptr); - DeclRefExpr * - BuildDeclRefExpr(ValueDecl *D, QualType Ty, ExprValueKind VK, - const DeclarationNameInfo &NameInfo, - const CXXScopeSpec *SS = nullptr, - NamedDecl *FoundD = nullptr, - SourceLocation TemplateKWLoc = SourceLocation(), - const TemplateArgumentListInfo *TemplateArgs = nullptr); - DeclRefExpr * - BuildDeclRefExpr(ValueDecl *D, QualType Ty, ExprValueKind VK, - const DeclarationNameInfo &NameInfo, - NestedNameSpecifierLoc NNS, - NamedDecl *FoundD = nullptr, - SourceLocation TemplateKWLoc = SourceLocation(), - const TemplateArgumentListInfo *TemplateArgs = nullptr); + // A cache representing if we've fully checked the various comparison category + // types stored in ASTContext. The bit-index corresponds to the integer value + // of a ComparisonCategoryType enumerator. + llvm::SmallBitVector FullyCheckedComparisonCategories; - ExprResult - BuildAnonymousStructUnionMemberReference( - const CXXScopeSpec &SS, - SourceLocation nameLoc, - IndirectFieldDecl *indirectField, - DeclAccessPair FoundDecl = DeclAccessPair::make(nullptr, AS_none), - Expr *baseObjectExpr = nullptr, - SourceLocation opLoc = SourceLocation()); + ValueDecl *tryLookupCtorInitMemberDecl(CXXRecordDecl *ClassDecl, + CXXScopeSpec &SS, + ParsedType TemplateTypeTy, + IdentifierInfo *MemberOrBase); - ExprResult BuildPossibleImplicitMemberExpr( - const CXXScopeSpec &SS, SourceLocation TemplateKWLoc, LookupResult &R, - const TemplateArgumentListInfo *TemplateArgs, const Scope *S, - UnresolvedLookupExpr *AsULE = nullptr); - ExprResult BuildImplicitMemberExpr(const CXXScopeSpec &SS, - SourceLocation TemplateKWLoc, - LookupResult &R, - const TemplateArgumentListInfo *TemplateArgs, - bool IsDefiniteInstance, - const Scope *S); - bool UseArgumentDependentLookup(const CXXScopeSpec &SS, - const LookupResult &R, - bool HasTrailingLParen); + /// Check if there is a field shadowing. + void CheckShadowInheritedFields(const SourceLocation &Loc, + DeclarationName FieldName, + const CXXRecordDecl *RD, + bool DeclIsField = true); - ExprResult - BuildQualifiedDeclarationNameExpr(CXXScopeSpec &SS, - const DeclarationNameInfo &NameInfo, - bool IsAddressOfOperand, const Scope *S, - TypeSourceInfo **RecoveryTSI = nullptr); + ///@} - ExprResult BuildDependentDeclRefExpr(const CXXScopeSpec &SS, - SourceLocation TemplateKWLoc, - const DeclarationNameInfo &NameInfo, - const TemplateArgumentListInfo *TemplateArgs); + // + // + // ------------------------------------------------------------------------- + // + // - ExprResult BuildDeclarationNameExpr(const CXXScopeSpec &SS, - LookupResult &R, - bool NeedsADL, - bool AcceptInvalidDecl = false); - ExprResult BuildDeclarationNameExpr( - const CXXScopeSpec &SS, const DeclarationNameInfo &NameInfo, NamedDecl *D, - NamedDecl *FoundD = nullptr, - const TemplateArgumentListInfo *TemplateArgs = nullptr, - bool AcceptInvalidDecl = false); + /// \name C++ Exception Specifications + /// Implementations are in SemaExceptionSpec.cpp + ///@{ - ExprResult BuildLiteralOperatorCall(LookupResult &R, - DeclarationNameInfo &SuffixInfo, - ArrayRef Args, - SourceLocation LitEndLoc, - TemplateArgumentListInfo *ExplicitTemplateArgs = nullptr); +public: + /// All the overriding functions seen during a class definition + /// that had their exception spec checks delayed, plus the overridden + /// function. + SmallVector, 2> + DelayedOverridingExceptionSpecChecks; - // ExpandFunctionLocalPredefinedMacros - Returns a new vector of Tokens, - // where Tokens representing function local predefined macros (such as - // __FUNCTION__) are replaced (expanded) with string-literal Tokens. - std::vector ExpandFunctionLocalPredefinedMacros(ArrayRef Toks); + /// All the function redeclarations seen during a class definition that had + /// their exception spec checks delayed, plus the prior declaration they + /// should be checked against. Except during error recovery, the new decl + /// should always be a friend declaration, as that's the only valid way to + /// redeclare a special member before its class is complete. + SmallVector, 2> + DelayedEquivalentExceptionSpecChecks; - ExprResult BuildPredefinedExpr(SourceLocation Loc, PredefinedIdentKind IK); - ExprResult ActOnPredefinedExpr(SourceLocation Loc, tok::TokenKind Kind); - ExprResult ActOnIntegerConstant(SourceLocation Loc, uint64_t Val); + /// Determine if we're in a case where we need to (incorrectly) eagerly + /// parse an exception specification to work around a libstdc++ bug. + bool isLibstdcxxEagerExceptionSpecHack(const Declarator &D); - ExprResult BuildSYCLUniqueStableNameExpr(SourceLocation OpLoc, - SourceLocation LParen, - SourceLocation RParen, - TypeSourceInfo *TSI); - ExprResult ActOnSYCLUniqueStableNameExpr(SourceLocation OpLoc, - SourceLocation LParen, - SourceLocation RParen, - ParsedType ParsedTy); + /// Check the given noexcept-specifier, convert its expression, and compute + /// the appropriate ExceptionSpecificationType. + ExprResult ActOnNoexceptSpec(Expr *NoexceptExpr, + ExceptionSpecificationType &EST); - bool CheckLoopHintExpr(Expr *E, SourceLocation Loc); + CanThrowResult canThrow(const Stmt *E); + /// Determine whether the callee of a particular function call can throw. + /// E, D and Loc are all optional. + static CanThrowResult canCalleeThrow(Sema &S, const Expr *E, const Decl *D, + SourceLocation Loc = SourceLocation()); + const FunctionProtoType *ResolveExceptionSpec(SourceLocation Loc, + const FunctionProtoType *FPT); + void UpdateExceptionSpec(FunctionDecl *FD, + const FunctionProtoType::ExceptionSpecInfo &ESI); + bool CheckSpecifiedExceptionType(QualType &T, SourceRange Range); + bool CheckDistantExceptionSpec(QualType T); + bool CheckEquivalentExceptionSpec(FunctionDecl *Old, FunctionDecl *New); + bool CheckEquivalentExceptionSpec(const FunctionProtoType *Old, + SourceLocation OldLoc, + const FunctionProtoType *New, + SourceLocation NewLoc); + bool CheckEquivalentExceptionSpec(const PartialDiagnostic &DiagID, + const PartialDiagnostic &NoteID, + const FunctionProtoType *Old, + SourceLocation OldLoc, + const FunctionProtoType *New, + SourceLocation NewLoc); + bool handlerCanCatch(QualType HandlerType, QualType ExceptionType); + bool CheckExceptionSpecSubset( + const PartialDiagnostic &DiagID, const PartialDiagnostic &NestedDiagID, + const PartialDiagnostic &NoteID, const PartialDiagnostic &NoThrowDiagID, + const FunctionProtoType *Superset, bool SkipSupersetFirstParameter, + SourceLocation SuperLoc, const FunctionProtoType *Subset, + bool SkipSubsetFirstParameter, SourceLocation SubLoc); + bool CheckParamExceptionSpec( + const PartialDiagnostic &NestedDiagID, const PartialDiagnostic &NoteID, + const FunctionProtoType *Target, bool SkipTargetFirstParameter, + SourceLocation TargetLoc, const FunctionProtoType *Source, + bool SkipSourceFirstParameter, SourceLocation SourceLoc); - ExprResult ActOnNumericConstant(const Token &Tok, Scope *UDLScope = nullptr); - ExprResult ActOnCharacterConstant(const Token &Tok, - Scope *UDLScope = nullptr); - ExprResult ActOnParenExpr(SourceLocation L, SourceLocation R, Expr *E); - ExprResult ActOnParenListExpr(SourceLocation L, - SourceLocation R, - MultiExprArg Val); + bool CheckExceptionSpecCompatibility(Expr *From, QualType ToType); - /// ActOnStringLiteral - The specified tokens were lexed as pasted string - /// fragments (e.g. "foo" "bar" L"baz"). - ExprResult ActOnStringLiteral(ArrayRef StringToks, - Scope *UDLScope = nullptr); + /// CheckOverridingFunctionExceptionSpec - Checks whether the exception + /// spec is a subset of base spec. + bool CheckOverridingFunctionExceptionSpec(const CXXMethodDecl *New, + const CXXMethodDecl *Old); - ExprResult ActOnUnevaluatedStringLiteral(ArrayRef StringToks); + ///@} - /// ControllingExprOrType is either an opaque pointer coming out of a - /// ParsedType or an Expr *. FIXME: it'd be better to split this interface - /// into two so we don't take a void *, but that's awkward because one of - /// the operands is either a ParsedType or an Expr *, which doesn't lend - /// itself to generic code very well. - ExprResult ActOnGenericSelectionExpr(SourceLocation KeyLoc, - SourceLocation DefaultLoc, - SourceLocation RParenLoc, - bool PredicateIsExpr, - void *ControllingExprOrType, - ArrayRef ArgTypes, - ArrayRef ArgExprs); - /// ControllingExprOrType is either a TypeSourceInfo * or an Expr *. FIXME: - /// it'd be better to split this interface into two so we don't take a - /// void *, but see the FIXME on ActOnGenericSelectionExpr as to why that - /// isn't a trivial change. - ExprResult CreateGenericSelectionExpr(SourceLocation KeyLoc, - SourceLocation DefaultLoc, - SourceLocation RParenLoc, - bool PredicateIsExpr, - void *ControllingExprOrType, - ArrayRef Types, - ArrayRef Exprs); + // + // + // ------------------------------------------------------------------------- + // + // - // Binary/Unary Operators. 'Tok' is the token for the operator. - ExprResult CreateBuiltinUnaryOp(SourceLocation OpLoc, UnaryOperatorKind Opc, - Expr *InputExpr, bool IsAfterAmp = false); - ExprResult BuildUnaryOp(Scope *S, SourceLocation OpLoc, UnaryOperatorKind Opc, - Expr *Input, bool IsAfterAmp = false); - ExprResult ActOnUnaryOp(Scope *S, SourceLocation OpLoc, tok::TokenKind Op, - Expr *Input, bool IsAfterAmp = false); + /// \name Expressions + /// Implementations are in SemaExpr.cpp + ///@{ - bool isQualifiedMemberAccess(Expr *E); - bool CheckUseOfCXXMethodAsAddressOfOperand(SourceLocation OpLoc, - const Expr *Op, - const CXXMethodDecl *MD); +public: + /// Describes how the expressions currently being parsed are + /// evaluated at run-time, if at all. + enum class ExpressionEvaluationContext { + /// The current expression and its subexpressions occur within an + /// unevaluated operand (C++11 [expr]p7), such as the subexpression of + /// \c sizeof, where the type of the expression may be significant but + /// no code will be generated to evaluate the value of the expression at + /// run time. + Unevaluated, - QualType CheckAddressOfOperand(ExprResult &Operand, SourceLocation OpLoc); + /// The current expression occurs within a braced-init-list within + /// an unevaluated operand. This is mostly like a regular unevaluated + /// context, except that we still instantiate constexpr functions that are + /// referenced here so that we can perform narrowing checks correctly. + UnevaluatedList, - bool CheckTypeTraitArity(unsigned Arity, SourceLocation Loc, size_t N); + /// The current expression occurs within a discarded statement. + /// This behaves largely similarly to an unevaluated operand in preventing + /// definitions from being required, but not in other ways. + DiscardedStatement, - bool ActOnAlignasTypeArgument(StringRef KWName, ParsedType Ty, - SourceLocation OpLoc, SourceRange R); - bool CheckAlignasTypeArgument(StringRef KWName, TypeSourceInfo *TInfo, - SourceLocation OpLoc, SourceRange R); + /// The current expression occurs within an unevaluated + /// operand that unconditionally permits abstract references to + /// fields, such as a SIZE operator in MS-style inline assembly. + UnevaluatedAbstract, - ExprResult CreateUnaryExprOrTypeTraitExpr(TypeSourceInfo *TInfo, - SourceLocation OpLoc, - UnaryExprOrTypeTrait ExprKind, - SourceRange R); - ExprResult CreateUnaryExprOrTypeTraitExpr(Expr *E, SourceLocation OpLoc, - UnaryExprOrTypeTrait ExprKind); - ExprResult - ActOnUnaryExprOrTypeTraitExpr(SourceLocation OpLoc, - UnaryExprOrTypeTrait ExprKind, - bool IsType, void *TyOrEx, - SourceRange ArgRange); + /// The current context is "potentially evaluated" in C++11 terms, + /// but the expression is evaluated at compile-time (like the values of + /// cases in a switch statement). + ConstantEvaluated, - ExprResult CheckPlaceholderExpr(Expr *E); - bool CheckVecStepExpr(Expr *E); + /// In addition of being constant evaluated, the current expression + /// occurs in an immediate function context - either a consteval function + /// or a consteval if statement. + ImmediateFunctionContext, - bool CheckUnaryExprOrTypeTraitOperand(Expr *E, UnaryExprOrTypeTrait ExprKind); - bool CheckUnaryExprOrTypeTraitOperand(QualType ExprType, SourceLocation OpLoc, - SourceRange ExprRange, - UnaryExprOrTypeTrait ExprKind, - StringRef KWName); - ExprResult ActOnSizeofParameterPackExpr(Scope *S, - SourceLocation OpLoc, - IdentifierInfo &Name, - SourceLocation NameLoc, - SourceLocation RParenLoc); + /// The current expression is potentially evaluated at run time, + /// which means that code may be generated to evaluate the value of the + /// expression at run time. + PotentiallyEvaluated, - ExprResult ActOnPackIndexingExpr(Scope *S, Expr *PackExpression, - SourceLocation EllipsisLoc, - SourceLocation LSquareLoc, Expr *IndexExpr, - SourceLocation RSquareLoc); + /// The current expression is potentially evaluated, but any + /// declarations referenced inside that expression are only used if + /// in fact the current expression is used. + /// + /// This value is used when parsing default function arguments, for which + /// we would like to provide diagnostics (e.g., passing non-POD arguments + /// through varargs) but do not want to mark declarations as "referenced" + /// until the default argument is used. + PotentiallyEvaluatedIfUsed + }; - ExprResult BuildPackIndexingExpr(Expr *PackExpression, - SourceLocation EllipsisLoc, Expr *IndexExpr, - SourceLocation RSquareLoc, - ArrayRef ExpandedExprs = {}, - bool EmptyPack = false); + /// Store a set of either DeclRefExprs or MemberExprs that contain a reference + /// to a variable (constant) that may or may not be odr-used in this Expr, and + /// we won't know until all lvalue-to-rvalue and discarded value conversions + /// have been applied to all subexpressions of the enclosing full expression. + /// This is cleared at the end of each full expression. + using MaybeODRUseExprSet = llvm::SmallSetVector; + MaybeODRUseExprSet MaybeODRUseExprs; - ExprResult ActOnPostfixUnaryOp(Scope *S, SourceLocation OpLoc, - tok::TokenKind Kind, Expr *Input); + using ImmediateInvocationCandidate = llvm::PointerIntPair; - ExprResult ActOnArraySubscriptExpr(Scope *S, Expr *Base, SourceLocation LLoc, - MultiExprArg ArgExprs, - SourceLocation RLoc); - ExprResult CreateBuiltinArraySubscriptExpr(Expr *Base, SourceLocation LLoc, - Expr *Idx, SourceLocation RLoc); + /// Data structure used to record current or nested + /// expression evaluation contexts. + struct ExpressionEvaluationContextRecord { + /// The expression evaluation context. + ExpressionEvaluationContext Context; - ExprResult CreateBuiltinMatrixSubscriptExpr(Expr *Base, Expr *RowIdx, - Expr *ColumnIdx, - SourceLocation RBLoc); + /// Whether the enclosing context needed a cleanup. + CleanupInfo ParentCleanup; - ExprResult ActOnOMPArraySectionExpr(Expr *Base, SourceLocation LBLoc, - Expr *LowerBound, - SourceLocation ColonLocFirst, - SourceLocation ColonLocSecond, - Expr *Length, Expr *Stride, - SourceLocation RBLoc); - ExprResult ActOnOMPArrayShapingExpr(Expr *Base, SourceLocation LParenLoc, - SourceLocation RParenLoc, - ArrayRef Dims, - ArrayRef Brackets); + /// The number of active cleanup objects when we entered + /// this expression evaluation context. + unsigned NumCleanupObjects; - /// Data structure for iterator expression. - struct OMPIteratorData { - IdentifierInfo *DeclIdent = nullptr; - SourceLocation DeclIdentLoc; - ParsedType Type; - OMPIteratorExpr::IteratorRange Range; - SourceLocation AssignLoc; - SourceLocation ColonLoc; - SourceLocation SecColonLoc; - }; + /// The number of typos encountered during this expression evaluation + /// context (i.e. the number of TypoExprs created). + unsigned NumTypos; - ExprResult ActOnOMPIteratorExpr(Scope *S, SourceLocation IteratorKwLoc, - SourceLocation LLoc, SourceLocation RLoc, - ArrayRef Data); + MaybeODRUseExprSet SavedMaybeODRUseExprs; - // This struct is for use by ActOnMemberAccess to allow - // BuildMemberReferenceExpr to be able to reinvoke ActOnMemberAccess after - // changing the access operator from a '.' to a '->' (to see if that is the - // change needed to fix an error about an unknown member, e.g. when the class - // defines a custom operator->). - struct ActOnMemberAccessExtraArgs { - Scope *S; - UnqualifiedId &Id; - Decl *ObjCImpDecl; - }; + /// The lambdas that are present within this context, if it + /// is indeed an unevaluated context. + SmallVector Lambdas; - ExprResult BuildMemberReferenceExpr( - Expr *Base, QualType BaseType, SourceLocation OpLoc, bool IsArrow, - CXXScopeSpec &SS, SourceLocation TemplateKWLoc, - NamedDecl *FirstQualifierInScope, const DeclarationNameInfo &NameInfo, - const TemplateArgumentListInfo *TemplateArgs, - const Scope *S, - ActOnMemberAccessExtraArgs *ExtraArgs = nullptr); + /// The declaration that provides context for lambda expressions + /// and block literals if the normal declaration context does not + /// suffice, e.g., in a default function argument. + Decl *ManglingContextDecl; - ExprResult - BuildMemberReferenceExpr(Expr *Base, QualType BaseType, SourceLocation OpLoc, - bool IsArrow, const CXXScopeSpec &SS, - SourceLocation TemplateKWLoc, - NamedDecl *FirstQualifierInScope, LookupResult &R, - const TemplateArgumentListInfo *TemplateArgs, - const Scope *S, - bool SuppressQualifierCheck = false, - ActOnMemberAccessExtraArgs *ExtraArgs = nullptr); + /// If we are processing a decltype type, a set of call expressions + /// for which we have deferred checking the completeness of the return type. + SmallVector DelayedDecltypeCalls; - ExprResult BuildFieldReferenceExpr(Expr *BaseExpr, bool IsArrow, - SourceLocation OpLoc, - const CXXScopeSpec &SS, FieldDecl *Field, - DeclAccessPair FoundDecl, - const DeclarationNameInfo &MemberNameInfo); + /// If we are processing a decltype type, a set of temporary binding + /// expressions for which we have deferred checking the destructor. + SmallVector DelayedDecltypeBinds; - ExprResult PerformMemberExprBaseConversion(Expr *Base, bool IsArrow); + llvm::SmallPtrSet PossibleDerefs; - bool CheckQualifiedMemberReference(Expr *BaseExpr, QualType BaseType, - const CXXScopeSpec &SS, - const LookupResult &R); + /// Expressions appearing as the LHS of a volatile assignment in this + /// context. We produce a warning for these when popping the context if + /// they are not discarded-value expressions nor unevaluated operands. + SmallVector VolatileAssignmentLHSs; - ExprResult ActOnDependentMemberExpr(Expr *Base, QualType BaseType, - bool IsArrow, SourceLocation OpLoc, - const CXXScopeSpec &SS, - SourceLocation TemplateKWLoc, - NamedDecl *FirstQualifierInScope, - const DeclarationNameInfo &NameInfo, - const TemplateArgumentListInfo *TemplateArgs); + /// Set of candidates for starting an immediate invocation. + llvm::SmallVector + ImmediateInvocationCandidates; - ExprResult ActOnMemberAccessExpr(Scope *S, Expr *Base, - SourceLocation OpLoc, - tok::TokenKind OpKind, - CXXScopeSpec &SS, - SourceLocation TemplateKWLoc, - UnqualifiedId &Member, - Decl *ObjCImpDecl); + /// Set of DeclRefExprs referencing a consteval function when used in a + /// context not already known to be immediately invoked. + llvm::SmallPtrSet ReferenceToConsteval; - MemberExpr * - BuildMemberExpr(Expr *Base, bool IsArrow, SourceLocation OpLoc, - const CXXScopeSpec *SS, SourceLocation TemplateKWLoc, - ValueDecl *Member, DeclAccessPair FoundDecl, - bool HadMultipleCandidates, - const DeclarationNameInfo &MemberNameInfo, QualType Ty, - ExprValueKind VK, ExprObjectKind OK, - const TemplateArgumentListInfo *TemplateArgs = nullptr); - MemberExpr * - BuildMemberExpr(Expr *Base, bool IsArrow, SourceLocation OpLoc, - NestedNameSpecifierLoc NNS, SourceLocation TemplateKWLoc, - ValueDecl *Member, DeclAccessPair FoundDecl, - bool HadMultipleCandidates, - const DeclarationNameInfo &MemberNameInfo, QualType Ty, - ExprValueKind VK, ExprObjectKind OK, - const TemplateArgumentListInfo *TemplateArgs = nullptr); + /// P2718R0 - Lifetime extension in range-based for loops. + /// MaterializeTemporaryExprs in for-range-init expressions which need to + /// extend lifetime. Add MaterializeTemporaryExpr* if the value of + /// InLifetimeExtendingContext is true. + SmallVector ForRangeLifetimeExtendTemps; - void ActOnDefaultCtorInitializers(Decl *CDtorDecl); - bool ConvertArgumentsForCall(CallExpr *Call, Expr *Fn, - FunctionDecl *FDecl, - const FunctionProtoType *Proto, - ArrayRef Args, - SourceLocation RParenLoc, - bool ExecConfig = false); - void CheckStaticArrayArgument(SourceLocation CallLoc, - ParmVarDecl *Param, - const Expr *ArgExpr); + /// \brief Describes whether we are in an expression constext which we have + /// to handle differently. + enum ExpressionKind { + EK_Decltype, + EK_TemplateArgument, + EK_Other + } ExprContext; - /// ActOnCallExpr - Handle a call to Fn with the specified array of arguments. - /// This provides the location of the left/right parens and a list of comma - /// locations. - ExprResult ActOnCallExpr(Scope *S, Expr *Fn, SourceLocation LParenLoc, - MultiExprArg ArgExprs, SourceLocation RParenLoc, - Expr *ExecConfig = nullptr); - ExprResult BuildCallExpr(Scope *S, Expr *Fn, SourceLocation LParenLoc, - MultiExprArg ArgExprs, SourceLocation RParenLoc, - Expr *ExecConfig = nullptr, - bool IsExecConfig = false, - bool AllowRecovery = false); - Expr *BuildBuiltinCallExpr(SourceLocation Loc, Builtin::ID Id, - MultiExprArg CallArgs); - enum class AtomicArgumentOrder { API, AST }; - ExprResult - BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, - SourceLocation RParenLoc, MultiExprArg Args, - AtomicExpr::AtomicOp Op, - AtomicArgumentOrder ArgOrder = AtomicArgumentOrder::API); - ExprResult - BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl, SourceLocation LParenLoc, - ArrayRef Arg, SourceLocation RParenLoc, - Expr *Config = nullptr, bool IsExecConfig = false, - ADLCallKind UsesADL = ADLCallKind::NotADL); + // A context can be nested in both a discarded statement context and + // an immediate function context, so they need to be tracked independently. + bool InDiscardedStatement; + bool InImmediateFunctionContext; + bool InImmediateEscalatingFunctionContext; - ExprResult ActOnCUDAExecConfigExpr(Scope *S, SourceLocation LLLLoc, - MultiExprArg ExecConfig, - SourceLocation GGGLoc); + bool IsCurrentlyCheckingDefaultArgumentOrInitializer = false; - ExprResult ActOnCastExpr(Scope *S, SourceLocation LParenLoc, - Declarator &D, ParsedType &Ty, - SourceLocation RParenLoc, Expr *CastExpr); - ExprResult BuildCStyleCastExpr(SourceLocation LParenLoc, - TypeSourceInfo *Ty, - SourceLocation RParenLoc, - Expr *Op); - CastKind PrepareScalarCast(ExprResult &src, QualType destType); + // We are in a constant context, but we also allow + // non constant expressions, for example for array bounds (which may be + // VLAs). + bool InConditionallyConstantEvaluateContext = false; - /// Build an altivec or OpenCL literal. - ExprResult BuildVectorLiteral(SourceLocation LParenLoc, - SourceLocation RParenLoc, Expr *E, - TypeSourceInfo *TInfo); + /// Whether we are currently in a context in which all temporaries must be + /// lifetime-extended, even if they're not bound to a reference (for + /// example, in a for-range initializer). + bool InLifetimeExtendingContext = false; - ExprResult MaybeConvertParenListExprToParenExpr(Scope *S, Expr *ME); + /// Whether we are currently in a context in which all temporaries must be + /// materialized. + /// + /// [class.temporary]/p2: + /// The materialization of a temporary object is generally delayed as long + /// as possible in order to avoid creating unnecessary temporary objects. + /// + /// Temporary objects are materialized: + /// (2.1) when binding a reference to a prvalue ([dcl.init.ref], + /// [expr.type.conv], [expr.dynamic.cast], [expr.static.cast], + /// [expr.const.cast], [expr.cast]), + /// + /// (2.2) when performing member access on a class prvalue ([expr.ref], + /// [expr.mptr.oper]), + /// + /// (2.3) when performing an array-to-pointer conversion or subscripting + /// on an array prvalue ([conv.array], [expr.sub]), + /// + /// (2.4) when initializing an object of type + /// std​::​initializer_list from a braced-init-list + /// ([dcl.init.list]), + /// + /// (2.5) for certain unevaluated operands ([expr.typeid], [expr.sizeof]) + /// + /// (2.6) when a prvalue that has type other than cv void appears as a + /// discarded-value expression ([expr.context]). + bool InMaterializeTemporaryObjectContext = false; - ExprResult ActOnCompoundLiteral(SourceLocation LParenLoc, - ParsedType Ty, - SourceLocation RParenLoc, - Expr *InitExpr); + // When evaluating immediate functions in the initializer of a default + // argument or default member initializer, this is the declaration whose + // default initializer is being evaluated and the location of the call + // or constructor definition. + struct InitializationContext { + InitializationContext(SourceLocation Loc, ValueDecl *Decl, + DeclContext *Context) + : Loc(Loc), Decl(Decl), Context(Context) { + assert(Decl && Context && "invalid initialization context"); + } - ExprResult BuildCompoundLiteralExpr(SourceLocation LParenLoc, - TypeSourceInfo *TInfo, - SourceLocation RParenLoc, - Expr *LiteralExpr); + SourceLocation Loc; + ValueDecl *Decl = nullptr; + DeclContext *Context = nullptr; + }; + std::optional DelayedDefaultInitializationContext; - ExprResult ActOnInitList(SourceLocation LBraceLoc, - MultiExprArg InitArgList, - SourceLocation RBraceLoc); + ExpressionEvaluationContextRecord(ExpressionEvaluationContext Context, + unsigned NumCleanupObjects, + CleanupInfo ParentCleanup, + Decl *ManglingContextDecl, + ExpressionKind ExprContext) + : Context(Context), ParentCleanup(ParentCleanup), + NumCleanupObjects(NumCleanupObjects), NumTypos(0), + ManglingContextDecl(ManglingContextDecl), ExprContext(ExprContext), + InDiscardedStatement(false), InImmediateFunctionContext(false), + InImmediateEscalatingFunctionContext(false) {} - ExprResult BuildInitList(SourceLocation LBraceLoc, - MultiExprArg InitArgList, - SourceLocation RBraceLoc); + bool isUnevaluated() const { + return Context == ExpressionEvaluationContext::Unevaluated || + Context == ExpressionEvaluationContext::UnevaluatedAbstract || + Context == ExpressionEvaluationContext::UnevaluatedList; + } - ExprResult ActOnDesignatedInitializer(Designation &Desig, - SourceLocation EqualOrColonLoc, - bool GNUSyntax, - ExprResult Init); + bool isConstantEvaluated() const { + return Context == ExpressionEvaluationContext::ConstantEvaluated || + Context == ExpressionEvaluationContext::ImmediateFunctionContext; + } -private: - static BinaryOperatorKind ConvertTokenKindToBinaryOpcode(tok::TokenKind Kind); + bool isImmediateFunctionContext() const { + return Context == ExpressionEvaluationContext::ImmediateFunctionContext || + (Context == ExpressionEvaluationContext::DiscardedStatement && + InImmediateFunctionContext) || + // C++23 [expr.const]p14: + // An expression or conversion is in an immediate function + // context if it is potentially evaluated and either: + // * its innermost enclosing non-block scope is a function + // parameter scope of an immediate function, or + // * its enclosing statement is enclosed by the compound- + // statement of a consteval if statement. + (Context == ExpressionEvaluationContext::PotentiallyEvaluated && + InImmediateFunctionContext); + } -public: - ExprResult ActOnBinOp(Scope *S, SourceLocation TokLoc, - tok::TokenKind Kind, Expr *LHSExpr, Expr *RHSExpr); - ExprResult BuildBinOp(Scope *S, SourceLocation OpLoc, - BinaryOperatorKind Opc, Expr *LHSExpr, Expr *RHSExpr); - ExprResult CreateBuiltinBinOp(SourceLocation OpLoc, BinaryOperatorKind Opc, - Expr *LHSExpr, Expr *RHSExpr); - void LookupBinOp(Scope *S, SourceLocation OpLoc, BinaryOperatorKind Opc, - UnresolvedSetImpl &Functions); + bool isDiscardedStatementContext() const { + return Context == ExpressionEvaluationContext::DiscardedStatement || + (Context == + ExpressionEvaluationContext::ImmediateFunctionContext && + InDiscardedStatement); + } + }; - void DiagnoseCommaOperator(const Expr *LHS, SourceLocation Loc); + const ExpressionEvaluationContextRecord ¤tEvaluationContext() const { + assert(!ExprEvalContexts.empty() && + "Must be in an expression evaluation context"); + return ExprEvalContexts.back(); + }; - /// ActOnConditionalOp - Parse a ?: operation. Note that 'LHS' may be null - /// in the case of a the GNU conditional expr extension. - ExprResult ActOnConditionalOp(SourceLocation QuestionLoc, - SourceLocation ColonLoc, - Expr *CondExpr, Expr *LHSExpr, Expr *RHSExpr); + /// Increment when we find a reference; decrement when we find an ignored + /// assignment. Ultimately the value is 0 if every reference is an ignored + /// assignment. + llvm::DenseMap RefsMinusAssignments; - /// ActOnAddrLabel - Parse the GNU address of label extension: "&&foo". - ExprResult ActOnAddrLabel(SourceLocation OpLoc, SourceLocation LabLoc, - LabelDecl *TheDecl); + /// Used to control the generation of ExprWithCleanups. + CleanupInfo Cleanup; - void ActOnStartStmtExpr(); - ExprResult ActOnStmtExpr(Scope *S, SourceLocation LPLoc, Stmt *SubStmt, - SourceLocation RPLoc); - ExprResult BuildStmtExpr(SourceLocation LPLoc, Stmt *SubStmt, - SourceLocation RPLoc, unsigned TemplateDepth); - // Handle the final expression in a statement expression. - ExprResult ActOnStmtExprResult(ExprResult E); - void ActOnStmtExprError(); + /// ExprCleanupObjects - This is the stack of objects requiring + /// cleanup that are created by the current full expression. + SmallVector ExprCleanupObjects; - // __builtin_offsetof(type, identifier(.identifier|[expr])*) - struct OffsetOfComponent { - SourceLocation LocStart, LocEnd; - bool isBrackets; // true if [expr], false if .ident - union { - IdentifierInfo *IdentInfo; - Expr *E; - } U; + // AssignmentAction - This is used by all the assignment diagnostic functions + // to represent what is actually causing the operation + enum AssignmentAction { + AA_Assigning, + AA_Passing, + AA_Returning, + AA_Converting, + AA_Initializing, + AA_Sending, + AA_Casting, + AA_Passing_CFAudited }; - /// __builtin_offsetof(type, a.b[123][456].c) - ExprResult BuildBuiltinOffsetOf(SourceLocation BuiltinLoc, - TypeSourceInfo *TInfo, - ArrayRef Components, - SourceLocation RParenLoc); - ExprResult ActOnBuiltinOffsetOf(Scope *S, - SourceLocation BuiltinLoc, - SourceLocation TypeLoc, - ParsedType ParsedArgTy, - ArrayRef Components, - SourceLocation RParenLoc); + bool CanUseDecl(NamedDecl *D, bool TreatUnavailableAsInvalid); + // A version of DiagnoseUseOfDecl that should be used if overload resolution + // has been used to find this declaration, which means we don't have to bother + // checking the trailing requires clause. + bool DiagnoseUseOfOverloadedDecl(NamedDecl *D, SourceLocation Loc) { + return DiagnoseUseOfDecl( + D, Loc, /*UnknownObjCClass=*/nullptr, /*ObjCPropertyAccess=*/false, + /*AvoidPartialAvailabilityChecks=*/false, /*ClassReceiver=*/nullptr, + /*SkipTrailingRequiresClause=*/true); + } - // __builtin_choose_expr(constExpr, expr1, expr2) - ExprResult ActOnChooseExpr(SourceLocation BuiltinLoc, - Expr *CondExpr, Expr *LHSExpr, - Expr *RHSExpr, SourceLocation RPLoc); + bool DiagnoseUseOfDecl(NamedDecl *D, ArrayRef Locs, + const ObjCInterfaceDecl *UnknownObjCClass = nullptr, + bool ObjCPropertyAccess = false, + bool AvoidPartialAvailabilityChecks = false, + ObjCInterfaceDecl *ClassReciever = nullptr, + bool SkipTrailingRequiresClause = false); + void NoteDeletedFunction(FunctionDecl *FD); - // __builtin_va_arg(expr, type) - ExprResult ActOnVAArg(SourceLocation BuiltinLoc, Expr *E, ParsedType Ty, - SourceLocation RPLoc); - ExprResult BuildVAArgExpr(SourceLocation BuiltinLoc, Expr *E, - TypeSourceInfo *TInfo, SourceLocation RPLoc); + void DiagnoseSentinelCalls(const NamedDecl *D, SourceLocation Loc, + ArrayRef Args); - // __builtin_LINE(), __builtin_FUNCTION(), __builtin_FUNCSIG(), - // __builtin_FILE(), __builtin_COLUMN(), __builtin_source_location() - ExprResult ActOnSourceLocExpr(SourceLocIdentKind Kind, - SourceLocation BuiltinLoc, - SourceLocation RPLoc); + void PushExpressionEvaluationContext( + ExpressionEvaluationContext NewContext, Decl *LambdaContextDecl = nullptr, + ExpressionEvaluationContextRecord::ExpressionKind Type = + ExpressionEvaluationContextRecord::EK_Other); + enum ReuseLambdaContextDecl_t { ReuseLambdaContextDecl }; + void PushExpressionEvaluationContext( + ExpressionEvaluationContext NewContext, ReuseLambdaContextDecl_t, + ExpressionEvaluationContextRecord::ExpressionKind Type = + ExpressionEvaluationContextRecord::EK_Other); + void PopExpressionEvaluationContext(); - // Build a potentially resolved SourceLocExpr. - ExprResult BuildSourceLocExpr(SourceLocIdentKind Kind, QualType ResultTy, - SourceLocation BuiltinLoc, SourceLocation RPLoc, - DeclContext *ParentContext); + void DiscardCleanupsInEvaluationContext(); - // __null - ExprResult ActOnGNUNullExpr(SourceLocation TokenLoc); + ExprResult TransformToPotentiallyEvaluated(Expr *E); + TypeSourceInfo *TransformToPotentiallyEvaluated(TypeSourceInfo *TInfo); + ExprResult HandleExprEvaluationContextForTypeof(Expr *E); - bool CheckCaseExpression(Expr *E); + void CheckUnusedVolatileAssignment(Expr *E); - /// Describes the result of an "if-exists" condition check. - enum IfExistsResult { - /// The symbol exists. - IER_Exists, + ExprResult ActOnConstantExpression(ExprResult Res); - /// The symbol does not exist. - IER_DoesNotExist, + // Functions for marking a declaration referenced. These functions also + // contain the relevant logic for marking if a reference to a function or + // variable is an odr-use (in the C++11 sense). There are separate variants + // for expressions referring to a decl; these exist because odr-use marking + // needs to be delayed for some constant variables when we build one of the + // named expressions. + // + // MightBeOdrUse indicates whether the use could possibly be an odr-use, and + // should usually be true. This only needs to be set to false if the lack of + // odr-use cannot be determined from the current context (for instance, + // because the name denotes a virtual function and was written without an + // explicit nested-name-specifier). + void MarkAnyDeclReferenced(SourceLocation Loc, Decl *D, bool MightBeOdrUse); + void MarkFunctionReferenced(SourceLocation Loc, FunctionDecl *Func, + bool MightBeOdrUse = true); + void MarkVariableReferenced(SourceLocation Loc, VarDecl *Var); + void MarkDeclRefReferenced(DeclRefExpr *E, const Expr *Base = nullptr); + void MarkMemberReferenced(MemberExpr *E); + void MarkFunctionParmPackReferenced(FunctionParmPackExpr *E); + void MarkCaptureUsedInEnclosingContext(ValueDecl *Capture, SourceLocation Loc, + unsigned CapturingScopeIndex); - /// The name is a dependent name, so the results will differ - /// from one instantiation to the next. - IER_Dependent, + ExprResult CheckLValueToRValueConversionOperand(Expr *E); + void CleanupVarDeclMarking(); - /// An error occurred. - IER_Error + enum TryCaptureKind { + TryCapture_Implicit, + TryCapture_ExplicitByVal, + TryCapture_ExplicitByRef }; - IfExistsResult - CheckMicrosoftIfExistsSymbol(Scope *S, CXXScopeSpec &SS, - const DeclarationNameInfo &TargetNameInfo); + /// Try to capture the given variable. + /// + /// \param Var The variable to capture. + /// + /// \param Loc The location at which the capture occurs. + /// + /// \param Kind The kind of capture, which may be implicit (for either a + /// block or a lambda), or explicit by-value or by-reference (for a lambda). + /// + /// \param EllipsisLoc The location of the ellipsis, if one is provided in + /// an explicit lambda capture. + /// + /// \param BuildAndDiagnose Whether we are actually supposed to add the + /// captures or diagnose errors. If false, this routine merely check whether + /// the capture can occur without performing the capture itself or complaining + /// if the variable cannot be captured. + /// + /// \param CaptureType Will be set to the type of the field used to capture + /// this variable in the innermost block or lambda. Only valid when the + /// variable can be captured. + /// + /// \param DeclRefType Will be set to the type of a reference to the capture + /// from within the current scope. Only valid when the variable can be + /// captured. + /// + /// \param FunctionScopeIndexToStopAt If non-null, it points to the index + /// of the FunctionScopeInfo stack beyond which we do not attempt to capture. + /// This is useful when enclosing lambdas must speculatively capture + /// variables that may or may not be used in certain specializations of + /// a nested generic lambda. + /// + /// \returns true if an error occurred (i.e., the variable cannot be + /// captured) and false if the capture succeeded. + bool tryCaptureVariable(ValueDecl *Var, SourceLocation Loc, + TryCaptureKind Kind, SourceLocation EllipsisLoc, + bool BuildAndDiagnose, QualType &CaptureType, + QualType &DeclRefType, + const unsigned *const FunctionScopeIndexToStopAt); - IfExistsResult - CheckMicrosoftIfExistsSymbol(Scope *S, SourceLocation KeywordLoc, - bool IsIfExists, CXXScopeSpec &SS, - UnqualifiedId &Name); + /// Try to capture the given variable. + bool tryCaptureVariable(ValueDecl *Var, SourceLocation Loc, + TryCaptureKind Kind = TryCapture_Implicit, + SourceLocation EllipsisLoc = SourceLocation()); - StmtResult BuildMSDependentExistsStmt(SourceLocation KeywordLoc, - bool IsIfExists, - NestedNameSpecifierLoc QualifierLoc, - DeclarationNameInfo NameInfo, - Stmt *Nested); - StmtResult ActOnMSDependentExistsStmt(SourceLocation KeywordLoc, - bool IsIfExists, - CXXScopeSpec &SS, UnqualifiedId &Name, - Stmt *Nested); + /// Checks if the variable must be captured. + bool NeedToCaptureVariable(ValueDecl *Var, SourceLocation Loc); - //===------------------------- "Block" Extension ------------------------===// + /// Given a variable, determine the type that a reference to that + /// variable will have in the given scope. + QualType getCapturedDeclRefType(ValueDecl *Var, SourceLocation Loc); - /// ActOnBlockStart - This callback is invoked when a block literal is - /// started. - void ActOnBlockStart(SourceLocation CaretLoc, Scope *CurScope); + /// Mark all of the declarations referenced within a particular AST node as + /// referenced. Used when template instantiation instantiates a non-dependent + /// type -- entities referenced by the type are now referenced. + void MarkDeclarationsReferencedInType(SourceLocation Loc, QualType T); + void MarkDeclarationsReferencedInExpr( + Expr *E, bool SkipLocalVariables = false, + ArrayRef StopAt = std::nullopt); - /// ActOnBlockArguments - This callback allows processing of block arguments. - /// If there are no arguments, this is still invoked. - void ActOnBlockArguments(SourceLocation CaretLoc, Declarator &ParamInfo, - Scope *CurScope); + /// Try to convert an expression \p E to type \p Ty. Returns the result of the + /// conversion. + ExprResult tryConvertExprToType(Expr *E, QualType Ty); - /// ActOnBlockError - If there is an error parsing a block, this callback - /// is invoked to pop the information about the block from the action impl. - void ActOnBlockError(SourceLocation CaretLoc, Scope *CurScope); + /// Conditionally issue a diagnostic based on the statements's reachability + /// analysis. + /// + /// \param Stmts If Stmts is non-empty, delay reporting the diagnostic until + /// the function body is parsed, and then do a basic reachability analysis to + /// determine if the statement is reachable. If it is unreachable, the + /// diagnostic will not be emitted. + bool DiagIfReachable(SourceLocation Loc, ArrayRef Stmts, + const PartialDiagnostic &PD); - /// ActOnBlockStmtExpr - This is called when the body of a block statement - /// literal was successfully completed. ^(int x){...} - ExprResult ActOnBlockStmtExpr(SourceLocation CaretLoc, Stmt *Body, - Scope *CurScope); + /// Conditionally issue a diagnostic based on the current + /// evaluation context. + /// + /// \param Statement If Statement is non-null, delay reporting the + /// diagnostic until the function body is parsed, and then do a basic + /// reachability analysis to determine if the statement is reachable. + /// If it is unreachable, the diagnostic will not be emitted. + bool DiagRuntimeBehavior(SourceLocation Loc, const Stmt *Statement, + const PartialDiagnostic &PD); + /// Similar, but diagnostic is only produced if all the specified statements + /// are reachable. + bool DiagRuntimeBehavior(SourceLocation Loc, ArrayRef Stmts, + const PartialDiagnostic &PD); - //===---------------------------- Clang Extensions ----------------------===// + // Primary Expressions. + SourceRange getExprRange(Expr *E) const; - /// __builtin_convertvector(...) - ExprResult ActOnConvertVectorExpr(Expr *E, ParsedType ParsedDestTy, - SourceLocation BuiltinLoc, - SourceLocation RParenLoc); + ExprResult ActOnIdExpression(Scope *S, CXXScopeSpec &SS, + SourceLocation TemplateKWLoc, UnqualifiedId &Id, + bool HasTrailingLParen, bool IsAddressOfOperand, + CorrectionCandidateCallback *CCC = nullptr, + bool IsInlineAsmIdentifier = false, + Token *KeywordReplacement = nullptr); - //===---------------------------- OpenCL Features -----------------------===// + void DecomposeUnqualifiedId(const UnqualifiedId &Id, + TemplateArgumentListInfo &Buffer, + DeclarationNameInfo &NameInfo, + const TemplateArgumentListInfo *&TemplateArgs); - /// __builtin_astype(...) - ExprResult ActOnAsTypeExpr(Expr *E, ParsedType ParsedDestTy, - SourceLocation BuiltinLoc, - SourceLocation RParenLoc); - ExprResult BuildAsTypeExpr(Expr *E, QualType DestTy, - SourceLocation BuiltinLoc, - SourceLocation RParenLoc); + bool DiagnoseDependentMemberLookup(const LookupResult &R); - //===---------------------------- HLSL Features -------------------------===// - Decl *ActOnStartHLSLBuffer(Scope *BufferScope, bool CBuffer, - SourceLocation KwLoc, IdentifierInfo *Ident, - SourceLocation IdentLoc, SourceLocation LBrace); - void ActOnFinishHLSLBuffer(Decl *Dcl, SourceLocation RBrace); + bool + DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R, + CorrectionCandidateCallback &CCC, + TemplateArgumentListInfo *ExplicitTemplateArgs = nullptr, + ArrayRef Args = std::nullopt, + DeclContext *LookupCtx = nullptr, + TypoExpr **Out = nullptr); - //===---------------------------- C++ Features --------------------------===// + DeclResult LookupIvarInObjCMethod(LookupResult &Lookup, Scope *S, + IdentifierInfo *II); + ExprResult BuildIvarRefExpr(Scope *S, SourceLocation Loc, ObjCIvarDecl *IV); - // Act on C++ namespaces - Decl *ActOnStartNamespaceDef(Scope *S, SourceLocation InlineLoc, - SourceLocation NamespaceLoc, - SourceLocation IdentLoc, IdentifierInfo *Ident, - SourceLocation LBrace, - const ParsedAttributesView &AttrList, - UsingDirectiveDecl *&UsingDecl, bool IsNested); - void ActOnFinishNamespaceDef(Decl *Dcl, SourceLocation RBrace); + ExprResult LookupInObjCMethod(LookupResult &LookUp, Scope *S, + IdentifierInfo *II, + bool AllowBuiltinCreation = false); - NamespaceDecl *getStdNamespace() const; - NamespaceDecl *getOrCreateStdNamespace(); + /// If \p D cannot be odr-used in the current expression evaluation context, + /// return a reason explaining why. Otherwise, return NOUR_None. + NonOdrUseReason getNonOdrUseReasonInCurrentContext(ValueDecl *D); - CXXRecordDecl *getStdBadAlloc() const; - EnumDecl *getStdAlignValT() const; + DeclRefExpr *BuildDeclRefExpr(ValueDecl *D, QualType Ty, ExprValueKind VK, + SourceLocation Loc, + const CXXScopeSpec *SS = nullptr); + DeclRefExpr * + BuildDeclRefExpr(ValueDecl *D, QualType Ty, ExprValueKind VK, + const DeclarationNameInfo &NameInfo, + const CXXScopeSpec *SS = nullptr, + NamedDecl *FoundD = nullptr, + SourceLocation TemplateKWLoc = SourceLocation(), + const TemplateArgumentListInfo *TemplateArgs = nullptr); + DeclRefExpr * + BuildDeclRefExpr(ValueDecl *D, QualType Ty, ExprValueKind VK, + const DeclarationNameInfo &NameInfo, + NestedNameSpecifierLoc NNS, NamedDecl *FoundD = nullptr, + SourceLocation TemplateKWLoc = SourceLocation(), + const TemplateArgumentListInfo *TemplateArgs = nullptr); - ValueDecl *tryLookupUnambiguousFieldDecl(RecordDecl *ClassDecl, - const IdentifierInfo *MemberOrBase); + bool UseArgumentDependentLookup(const CXXScopeSpec &SS, const LookupResult &R, + bool HasTrailingLParen); -private: - // A cache representing if we've fully checked the various comparison category - // types stored in ASTContext. The bit-index corresponds to the integer value - // of a ComparisonCategoryType enumerator. - llvm::SmallBitVector FullyCheckedComparisonCategories; + ExprResult + BuildQualifiedDeclarationNameExpr(CXXScopeSpec &SS, + const DeclarationNameInfo &NameInfo, + bool IsAddressOfOperand, const Scope *S, + TypeSourceInfo **RecoveryTSI = nullptr); - ValueDecl *tryLookupCtorInitMemberDecl(CXXRecordDecl *ClassDecl, - CXXScopeSpec &SS, - ParsedType TemplateTypeTy, - IdentifierInfo *MemberOrBase); + ExprResult BuildDeclarationNameExpr(const CXXScopeSpec &SS, LookupResult &R, + bool NeedsADL, + bool AcceptInvalidDecl = false); + ExprResult BuildDeclarationNameExpr( + const CXXScopeSpec &SS, const DeclarationNameInfo &NameInfo, NamedDecl *D, + NamedDecl *FoundD = nullptr, + const TemplateArgumentListInfo *TemplateArgs = nullptr, + bool AcceptInvalidDecl = false); -public: - enum class ComparisonCategoryUsage { - /// The '<=>' operator was used in an expression and a builtin operator - /// was selected. - OperatorInExpression, - /// A defaulted 'operator<=>' needed the comparison category. This - /// typically only applies to 'std::strong_ordering', due to the implicit - /// fallback return value. - DefaultedOperator, - }; + // ExpandFunctionLocalPredefinedMacros - Returns a new vector of Tokens, + // where Tokens representing function local predefined macros (such as + // __FUNCTION__) are replaced (expanded) with string-literal Tokens. + std::vector ExpandFunctionLocalPredefinedMacros(ArrayRef Toks); - /// Lookup the specified comparison category types in the standard - /// library, an check the VarDecls possibly returned by the operator<=> - /// builtins for that type. - /// - /// \return The type of the comparison category type corresponding to the - /// specified Kind, or a null type if an error occurs - QualType CheckComparisonCategoryType(ComparisonCategoryType Kind, - SourceLocation Loc, - ComparisonCategoryUsage Usage); + ExprResult BuildPredefinedExpr(SourceLocation Loc, PredefinedIdentKind IK); + ExprResult ActOnPredefinedExpr(SourceLocation Loc, tok::TokenKind Kind); + ExprResult ActOnIntegerConstant(SourceLocation Loc, uint64_t Val); - /// Tests whether Ty is an instance of std::initializer_list and, if - /// it is and Element is not NULL, assigns the element type to Element. - bool isStdInitializerList(QualType Ty, QualType *Element); + ExprResult BuildSYCLUniqueStableNameExpr(SourceLocation OpLoc, + SourceLocation LParen, + SourceLocation RParen, + TypeSourceInfo *TSI); + ExprResult ActOnSYCLUniqueStableNameExpr(SourceLocation OpLoc, + SourceLocation LParen, + SourceLocation RParen, + ParsedType ParsedTy); - /// Looks for the std::initializer_list template and instantiates it - /// with Element, or emits an error if it's not found. - /// - /// \returns The instantiated template, or null on error. - QualType BuildStdInitializerList(QualType Element, SourceLocation Loc); + bool CheckLoopHintExpr(Expr *E, SourceLocation Loc); - /// Determine whether Ctor is an initializer-list constructor, as - /// defined in [dcl.init.list]p2. - bool isInitListConstructor(const FunctionDecl *Ctor); + ExprResult ActOnNumericConstant(const Token &Tok, Scope *UDLScope = nullptr); + ExprResult ActOnCharacterConstant(const Token &Tok, + Scope *UDLScope = nullptr); + ExprResult ActOnParenExpr(SourceLocation L, SourceLocation R, Expr *E); + ExprResult ActOnParenListExpr(SourceLocation L, SourceLocation R, + MultiExprArg Val); - Decl *ActOnUsingDirective(Scope *CurScope, SourceLocation UsingLoc, - SourceLocation NamespcLoc, CXXScopeSpec &SS, - SourceLocation IdentLoc, - IdentifierInfo *NamespcName, - const ParsedAttributesView &AttrList); + /// ActOnStringLiteral - The specified tokens were lexed as pasted string + /// fragments (e.g. "foo" "bar" L"baz"). + ExprResult ActOnStringLiteral(ArrayRef StringToks, + Scope *UDLScope = nullptr); - void PushUsingDirective(Scope *S, UsingDirectiveDecl *UDir); + ExprResult ActOnUnevaluatedStringLiteral(ArrayRef StringToks); - Decl *ActOnNamespaceAliasDef(Scope *CurScope, - SourceLocation NamespaceLoc, - SourceLocation AliasLoc, - IdentifierInfo *Alias, - CXXScopeSpec &SS, - SourceLocation IdentLoc, - IdentifierInfo *Ident); + /// ControllingExprOrType is either an opaque pointer coming out of a + /// ParsedType or an Expr *. FIXME: it'd be better to split this interface + /// into two so we don't take a void *, but that's awkward because one of + /// the operands is either a ParsedType or an Expr *, which doesn't lend + /// itself to generic code very well. + ExprResult ActOnGenericSelectionExpr(SourceLocation KeyLoc, + SourceLocation DefaultLoc, + SourceLocation RParenLoc, + bool PredicateIsExpr, + void *ControllingExprOrType, + ArrayRef ArgTypes, + ArrayRef ArgExprs); + /// ControllingExprOrType is either a TypeSourceInfo * or an Expr *. FIXME: + /// it'd be better to split this interface into two so we don't take a + /// void *, but see the FIXME on ActOnGenericSelectionExpr as to why that + /// isn't a trivial change. + ExprResult CreateGenericSelectionExpr(SourceLocation KeyLoc, + SourceLocation DefaultLoc, + SourceLocation RParenLoc, + bool PredicateIsExpr, + void *ControllingExprOrType, + ArrayRef Types, + ArrayRef Exprs); - void FilterUsingLookup(Scope *S, LookupResult &lookup); - void HideUsingShadowDecl(Scope *S, UsingShadowDecl *Shadow); - bool CheckUsingShadowDecl(BaseUsingDecl *BUD, NamedDecl *Target, - const LookupResult &PreviousDecls, - UsingShadowDecl *&PrevShadow); - UsingShadowDecl *BuildUsingShadowDecl(Scope *S, BaseUsingDecl *BUD, - NamedDecl *Target, - UsingShadowDecl *PrevDecl); + // Binary/Unary Operators. 'Tok' is the token for the operator. + ExprResult CreateBuiltinUnaryOp(SourceLocation OpLoc, UnaryOperatorKind Opc, + Expr *InputExpr, bool IsAfterAmp = false); + ExprResult BuildUnaryOp(Scope *S, SourceLocation OpLoc, UnaryOperatorKind Opc, + Expr *Input, bool IsAfterAmp = false); + ExprResult ActOnUnaryOp(Scope *S, SourceLocation OpLoc, tok::TokenKind Op, + Expr *Input, bool IsAfterAmp = false); - bool CheckUsingDeclRedeclaration(SourceLocation UsingLoc, - bool HasTypenameKeyword, - const CXXScopeSpec &SS, - SourceLocation NameLoc, - const LookupResult &Previous); - bool CheckUsingDeclQualifier(SourceLocation UsingLoc, bool HasTypename, - const CXXScopeSpec &SS, - const DeclarationNameInfo &NameInfo, - SourceLocation NameLoc, - const LookupResult *R = nullptr, - const UsingDecl *UD = nullptr); + bool isQualifiedMemberAccess(Expr *E); + bool CheckUseOfCXXMethodAsAddressOfOperand(SourceLocation OpLoc, + const Expr *Op, + const CXXMethodDecl *MD); - NamedDecl *BuildUsingDeclaration( - Scope *S, AccessSpecifier AS, SourceLocation UsingLoc, - bool HasTypenameKeyword, SourceLocation TypenameLoc, CXXScopeSpec &SS, - DeclarationNameInfo NameInfo, SourceLocation EllipsisLoc, - const ParsedAttributesView &AttrList, bool IsInstantiation, - bool IsUsingIfExists); - NamedDecl *BuildUsingEnumDeclaration(Scope *S, AccessSpecifier AS, - SourceLocation UsingLoc, - SourceLocation EnumLoc, - SourceLocation NameLoc, - TypeSourceInfo *EnumType, EnumDecl *ED); - NamedDecl *BuildUsingPackDecl(NamedDecl *InstantiatedFrom, - ArrayRef Expansions); + QualType CheckAddressOfOperand(ExprResult &Operand, SourceLocation OpLoc); - bool CheckInheritingConstructorUsingDecl(UsingDecl *UD); + bool ActOnAlignasTypeArgument(StringRef KWName, ParsedType Ty, + SourceLocation OpLoc, SourceRange R); + bool CheckAlignasTypeArgument(StringRef KWName, TypeSourceInfo *TInfo, + SourceLocation OpLoc, SourceRange R); - /// Given a derived-class using shadow declaration for a constructor and the - /// correspnding base class constructor, find or create the implicit - /// synthesized derived class constructor to use for this initialization. - CXXConstructorDecl * - findInheritingConstructor(SourceLocation Loc, CXXConstructorDecl *BaseCtor, - ConstructorUsingShadowDecl *DerivedShadow); + ExprResult CreateUnaryExprOrTypeTraitExpr(TypeSourceInfo *TInfo, + SourceLocation OpLoc, + UnaryExprOrTypeTrait ExprKind, + SourceRange R); + ExprResult CreateUnaryExprOrTypeTraitExpr(Expr *E, SourceLocation OpLoc, + UnaryExprOrTypeTrait ExprKind); + ExprResult ActOnUnaryExprOrTypeTraitExpr(SourceLocation OpLoc, + UnaryExprOrTypeTrait ExprKind, + bool IsType, void *TyOrEx, + SourceRange ArgRange); - Decl *ActOnUsingDeclaration(Scope *CurScope, AccessSpecifier AS, - SourceLocation UsingLoc, - SourceLocation TypenameLoc, CXXScopeSpec &SS, - UnqualifiedId &Name, SourceLocation EllipsisLoc, - const ParsedAttributesView &AttrList); - Decl *ActOnUsingEnumDeclaration(Scope *CurScope, AccessSpecifier AS, - SourceLocation UsingLoc, - SourceLocation EnumLoc, - SourceLocation IdentLoc, IdentifierInfo &II, - CXXScopeSpec *SS = nullptr); - Decl *ActOnAliasDeclaration(Scope *CurScope, AccessSpecifier AS, - MultiTemplateParamsArg TemplateParams, - SourceLocation UsingLoc, UnqualifiedId &Name, - const ParsedAttributesView &AttrList, - TypeResult Type, Decl *DeclFromDeclSpec); + ExprResult CheckPlaceholderExpr(Expr *E); + bool CheckVecStepExpr(Expr *E); - /// BuildCXXConstructExpr - Creates a complete call to a constructor, - /// including handling of its default argument expressions. - /// - /// \param ConstructKind - a CXXConstructExpr::ConstructionKind - ExprResult BuildCXXConstructExpr( - SourceLocation ConstructLoc, QualType DeclInitType, NamedDecl *FoundDecl, - CXXConstructorDecl *Constructor, MultiExprArg Exprs, - bool HadMultipleCandidates, bool IsListInitialization, - bool IsStdInitListInitialization, bool RequiresZeroInit, - CXXConstructionKind ConstructKind, SourceRange ParenRange); + bool CheckUnaryExprOrTypeTraitOperand(Expr *E, UnaryExprOrTypeTrait ExprKind); + bool CheckUnaryExprOrTypeTraitOperand(QualType ExprType, SourceLocation OpLoc, + SourceRange ExprRange, + UnaryExprOrTypeTrait ExprKind, + StringRef KWName); - /// Build a CXXConstructExpr whose constructor has already been resolved if - /// it denotes an inherited constructor. - ExprResult BuildCXXConstructExpr( - SourceLocation ConstructLoc, QualType DeclInitType, - CXXConstructorDecl *Constructor, bool Elidable, MultiExprArg Exprs, - bool HadMultipleCandidates, bool IsListInitialization, - bool IsStdInitListInitialization, bool RequiresZeroInit, - CXXConstructionKind ConstructKind, SourceRange ParenRange); + ExprResult ActOnPostfixUnaryOp(Scope *S, SourceLocation OpLoc, + tok::TokenKind Kind, Expr *Input); - // FIXME: Can we remove this and have the above BuildCXXConstructExpr check if - // the constructor can be elidable? - ExprResult BuildCXXConstructExpr( - SourceLocation ConstructLoc, QualType DeclInitType, NamedDecl *FoundDecl, - CXXConstructorDecl *Constructor, bool Elidable, MultiExprArg Exprs, - bool HadMultipleCandidates, bool IsListInitialization, - bool IsStdInitListInitialization, bool RequiresZeroInit, - CXXConstructionKind ConstructKind, SourceRange ParenRange); + ExprResult ActOnArraySubscriptExpr(Scope *S, Expr *Base, SourceLocation LLoc, + MultiExprArg ArgExprs, + SourceLocation RLoc); + ExprResult CreateBuiltinArraySubscriptExpr(Expr *Base, SourceLocation LLoc, + Expr *Idx, SourceLocation RLoc); - ExprResult ConvertMemberDefaultInitExpression(FieldDecl *FD, Expr *InitExpr, - SourceLocation InitLoc); + ExprResult CreateBuiltinMatrixSubscriptExpr(Expr *Base, Expr *RowIdx, + Expr *ColumnIdx, + SourceLocation RBLoc); - ExprResult BuildCXXDefaultInitExpr(SourceLocation Loc, FieldDecl *Field); + ExprResult ActOnOMPArraySectionExpr(Expr *Base, SourceLocation LBLoc, + Expr *LowerBound, + SourceLocation ColonLocFirst, + SourceLocation ColonLocSecond, + Expr *Length, Expr *Stride, + SourceLocation RBLoc); + ExprResult ActOnOMPArrayShapingExpr(Expr *Base, SourceLocation LParenLoc, + SourceLocation RParenLoc, + ArrayRef Dims, + ArrayRef Brackets); + /// Data structure for iterator expression. + struct OMPIteratorData { + IdentifierInfo *DeclIdent = nullptr; + SourceLocation DeclIdentLoc; + ParsedType Type; + OMPIteratorExpr::IteratorRange Range; + SourceLocation AssignLoc; + SourceLocation ColonLoc; + SourceLocation SecColonLoc; + }; - /// Instantiate or parse a C++ default argument expression as necessary. - /// Return true on error. - bool CheckCXXDefaultArgExpr(SourceLocation CallLoc, FunctionDecl *FD, - ParmVarDecl *Param, Expr *Init = nullptr, - bool SkipImmediateInvocations = true); + ExprResult ActOnOMPIteratorExpr(Scope *S, SourceLocation IteratorKwLoc, + SourceLocation LLoc, SourceLocation RLoc, + ArrayRef Data); - /// BuildCXXDefaultArgExpr - Creates a CXXDefaultArgExpr, instantiating - /// the default expr if needed. - ExprResult BuildCXXDefaultArgExpr(SourceLocation CallLoc, FunctionDecl *FD, - ParmVarDecl *Param, Expr *Init = nullptr); + bool ConvertArgumentsForCall(CallExpr *Call, Expr *Fn, FunctionDecl *FDecl, + const FunctionProtoType *Proto, + ArrayRef Args, SourceLocation RParenLoc, + bool ExecConfig = false); + void CheckStaticArrayArgument(SourceLocation CallLoc, ParmVarDecl *Param, + const Expr *ArgExpr); - /// FinalizeVarWithDestructor - Prepare for calling destructor on the - /// constructed variable. - void FinalizeVarWithDestructor(VarDecl *VD, const RecordType *DeclInitType); + /// ActOnCallExpr - Handle a call to Fn with the specified array of arguments. + /// This provides the location of the left/right parens and a list of comma + /// locations. + ExprResult ActOnCallExpr(Scope *S, Expr *Fn, SourceLocation LParenLoc, + MultiExprArg ArgExprs, SourceLocation RParenLoc, + Expr *ExecConfig = nullptr); + ExprResult BuildCallExpr(Scope *S, Expr *Fn, SourceLocation LParenLoc, + MultiExprArg ArgExprs, SourceLocation RParenLoc, + Expr *ExecConfig = nullptr, + bool IsExecConfig = false, + bool AllowRecovery = false); + Expr *BuildBuiltinCallExpr(SourceLocation Loc, Builtin::ID Id, + MultiExprArg CallArgs); - /// Helper class that collects exception specifications for - /// implicitly-declared special member functions. - class ImplicitExceptionSpecification { - // Pointer to allow copying - Sema *Self; - // We order exception specifications thus: - // noexcept is the most restrictive, but is only used in C++11. - // throw() comes next. - // Then a throw(collected exceptions) - // Finally no specification, which is expressed as noexcept(false). - // throw(...) is used instead if any called function uses it. - ExceptionSpecificationType ComputedEST; - llvm::SmallPtrSet ExceptionsSeen; - SmallVector Exceptions; + using ADLCallKind = CallExpr::ADLCallKind; - void ClearExceptions() { - ExceptionsSeen.clear(); - Exceptions.clear(); - } + ExprResult + BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl, SourceLocation LParenLoc, + ArrayRef Arg, SourceLocation RParenLoc, + Expr *Config = nullptr, bool IsExecConfig = false, + ADLCallKind UsesADL = ADLCallKind::NotADL); - public: - explicit ImplicitExceptionSpecification(Sema &Self) - : Self(&Self), ComputedEST(EST_BasicNoexcept) { - if (!Self.getLangOpts().CPlusPlus11) - ComputedEST = EST_DynamicNone; - } + ExprResult ActOnCastExpr(Scope *S, SourceLocation LParenLoc, Declarator &D, + ParsedType &Ty, SourceLocation RParenLoc, + Expr *CastExpr); - /// Get the computed exception specification type. - ExceptionSpecificationType getExceptionSpecType() const { - assert(!isComputedNoexcept(ComputedEST) && - "noexcept(expr) should not be a possible result"); - return ComputedEST; - } + CastKind PrepareScalarCast(ExprResult &src, QualType destType); - /// The number of exceptions in the exception specification. - unsigned size() const { return Exceptions.size(); } + /// Build an altivec or OpenCL literal. + ExprResult BuildVectorLiteral(SourceLocation LParenLoc, + SourceLocation RParenLoc, Expr *E, + TypeSourceInfo *TInfo); - /// The set of exceptions in the exception specification. - const QualType *data() const { return Exceptions.data(); } + ExprResult MaybeConvertParenListExprToParenExpr(Scope *S, Expr *ME); - /// Integrate another called method into the collected data. - void CalledDecl(SourceLocation CallLoc, const CXXMethodDecl *Method); + ExprResult ActOnCompoundLiteral(SourceLocation LParenLoc, ParsedType Ty, + SourceLocation RParenLoc, Expr *InitExpr); - /// Integrate an invoked expression into the collected data. - void CalledExpr(Expr *E) { CalledStmt(E); } + ExprResult BuildCompoundLiteralExpr(SourceLocation LParenLoc, + TypeSourceInfo *TInfo, + SourceLocation RParenLoc, + Expr *LiteralExpr); - /// Integrate an invoked statement into the collected data. - void CalledStmt(Stmt *S); + ExprResult ActOnInitList(SourceLocation LBraceLoc, MultiExprArg InitArgList, + SourceLocation RBraceLoc); - /// Overwrite an EPI's exception specification with this - /// computed exception specification. - FunctionProtoType::ExceptionSpecInfo getExceptionSpec() const { - FunctionProtoType::ExceptionSpecInfo ESI; - ESI.Type = getExceptionSpecType(); - if (ESI.Type == EST_Dynamic) { - ESI.Exceptions = Exceptions; - } else if (ESI.Type == EST_None) { - /// C++11 [except.spec]p14: - /// The exception-specification is noexcept(false) if the set of - /// potential exceptions of the special member function contains "any" - ESI.Type = EST_NoexceptFalse; - ESI.NoexceptExpr = Self->ActOnCXXBoolLiteral(SourceLocation(), - tok::kw_false).get(); - } - return ESI; - } - }; + ExprResult BuildInitList(SourceLocation LBraceLoc, MultiExprArg InitArgList, + SourceLocation RBraceLoc); - /// Evaluate the implicit exception specification for a defaulted - /// special member function. - void EvaluateImplicitExceptionSpec(SourceLocation Loc, FunctionDecl *FD); + ExprResult ActOnBinOp(Scope *S, SourceLocation TokLoc, tok::TokenKind Kind, + Expr *LHSExpr, Expr *RHSExpr); + ExprResult BuildBinOp(Scope *S, SourceLocation OpLoc, BinaryOperatorKind Opc, + Expr *LHSExpr, Expr *RHSExpr); + ExprResult CreateBuiltinBinOp(SourceLocation OpLoc, BinaryOperatorKind Opc, + Expr *LHSExpr, Expr *RHSExpr); + void LookupBinOp(Scope *S, SourceLocation OpLoc, BinaryOperatorKind Opc, + UnresolvedSetImpl &Functions); - /// Check the given noexcept-specifier, convert its expression, and compute - /// the appropriate ExceptionSpecificationType. - ExprResult ActOnNoexceptSpec(Expr *NoexceptExpr, - ExceptionSpecificationType &EST); + void DiagnoseCommaOperator(const Expr *LHS, SourceLocation Loc); - /// Check the given exception-specification and update the - /// exception specification information with the results. - void checkExceptionSpecification(bool IsTopLevel, - ExceptionSpecificationType EST, - ArrayRef DynamicExceptions, - ArrayRef DynamicExceptionRanges, - Expr *NoexceptExpr, - SmallVectorImpl &Exceptions, - FunctionProtoType::ExceptionSpecInfo &ESI); + /// ActOnConditionalOp - Parse a ?: operation. Note that 'LHS' may be null + /// in the case of a the GNU conditional expr extension. + ExprResult ActOnConditionalOp(SourceLocation QuestionLoc, + SourceLocation ColonLoc, Expr *CondExpr, + Expr *LHSExpr, Expr *RHSExpr); - /// Determine if we're in a case where we need to (incorrectly) eagerly - /// parse an exception specification to work around a libstdc++ bug. - bool isLibstdcxxEagerExceptionSpecHack(const Declarator &D); + /// ActOnAddrLabel - Parse the GNU address of label extension: "&&foo". + ExprResult ActOnAddrLabel(SourceLocation OpLoc, SourceLocation LabLoc, + LabelDecl *TheDecl); - /// Add an exception-specification to the given member function - /// (or member function template). The exception-specification was parsed - /// after the method itself was declared. - void actOnDelayedExceptionSpecification(Decl *Method, - ExceptionSpecificationType EST, - SourceRange SpecificationRange, - ArrayRef DynamicExceptions, - ArrayRef DynamicExceptionRanges, - Expr *NoexceptExpr); + void ActOnStartStmtExpr(); + ExprResult ActOnStmtExpr(Scope *S, SourceLocation LPLoc, Stmt *SubStmt, + SourceLocation RPLoc); + ExprResult BuildStmtExpr(SourceLocation LPLoc, Stmt *SubStmt, + SourceLocation RPLoc, unsigned TemplateDepth); + // Handle the final expression in a statement expression. + ExprResult ActOnStmtExprResult(ExprResult E); + void ActOnStmtExprError(); - class InheritedConstructorInfo; + // __builtin_offsetof(type, identifier(.identifier|[expr])*) + struct OffsetOfComponent { + SourceLocation LocStart, LocEnd; + bool isBrackets; // true if [expr], false if .ident + union { + IdentifierInfo *IdentInfo; + Expr *E; + } U; + }; - /// Determine if a special member function should have a deleted - /// definition when it is defaulted. - bool ShouldDeleteSpecialMember(CXXMethodDecl *MD, CXXSpecialMember CSM, - InheritedConstructorInfo *ICI = nullptr, - bool Diagnose = false); + /// __builtin_offsetof(type, a.b[123][456].c) + ExprResult BuildBuiltinOffsetOf(SourceLocation BuiltinLoc, + TypeSourceInfo *TInfo, + ArrayRef Components, + SourceLocation RParenLoc); + ExprResult ActOnBuiltinOffsetOf(Scope *S, SourceLocation BuiltinLoc, + SourceLocation TypeLoc, + ParsedType ParsedArgTy, + ArrayRef Components, + SourceLocation RParenLoc); - /// Produce notes explaining why a defaulted function was defined as deleted. - void DiagnoseDeletedDefaultedFunction(FunctionDecl *FD); + // __builtin_choose_expr(constExpr, expr1, expr2) + ExprResult ActOnChooseExpr(SourceLocation BuiltinLoc, Expr *CondExpr, + Expr *LHSExpr, Expr *RHSExpr, + SourceLocation RPLoc); - /// Declare the implicit default constructor for the given class. - /// - /// \param ClassDecl The class declaration into which the implicit - /// default constructor will be added. - /// - /// \returns The implicitly-declared default constructor. - CXXConstructorDecl *DeclareImplicitDefaultConstructor( - CXXRecordDecl *ClassDecl); + // __builtin_va_arg(expr, type) + ExprResult ActOnVAArg(SourceLocation BuiltinLoc, Expr *E, ParsedType Ty, + SourceLocation RPLoc); + ExprResult BuildVAArgExpr(SourceLocation BuiltinLoc, Expr *E, + TypeSourceInfo *TInfo, SourceLocation RPLoc); - /// DefineImplicitDefaultConstructor - Checks for feasibility of - /// defining this constructor as the default constructor. - void DefineImplicitDefaultConstructor(SourceLocation CurrentLocation, - CXXConstructorDecl *Constructor); + // __builtin_LINE(), __builtin_FUNCTION(), __builtin_FUNCSIG(), + // __builtin_FILE(), __builtin_COLUMN(), __builtin_source_location() + ExprResult ActOnSourceLocExpr(SourceLocIdentKind Kind, + SourceLocation BuiltinLoc, + SourceLocation RPLoc); - /// Declare the implicit destructor for the given class. - /// - /// \param ClassDecl The class declaration into which the implicit - /// destructor will be added. - /// - /// \returns The implicitly-declared destructor. - CXXDestructorDecl *DeclareImplicitDestructor(CXXRecordDecl *ClassDecl); + // Build a potentially resolved SourceLocExpr. + ExprResult BuildSourceLocExpr(SourceLocIdentKind Kind, QualType ResultTy, + SourceLocation BuiltinLoc, SourceLocation RPLoc, + DeclContext *ParentContext); - /// DefineImplicitDestructor - Checks for feasibility of - /// defining this destructor as the default destructor. - void DefineImplicitDestructor(SourceLocation CurrentLocation, - CXXDestructorDecl *Destructor); + // __null + ExprResult ActOnGNUNullExpr(SourceLocation TokenLoc); - /// Build an exception spec for destructors that don't have one. - /// - /// C++11 says that user-defined destructors with no exception spec get one - /// that looks as if the destructor was implicitly declared. - void AdjustDestructorExceptionSpec(CXXDestructorDecl *Destructor); + bool CheckCaseExpression(Expr *E); - /// Define the specified inheriting constructor. - void DefineInheritingConstructor(SourceLocation UseLoc, - CXXConstructorDecl *Constructor); + //===------------------------- "Block" Extension ------------------------===// - /// Declare the implicit copy constructor for the given class. - /// - /// \param ClassDecl The class declaration into which the implicit - /// copy constructor will be added. - /// - /// \returns The implicitly-declared copy constructor. - CXXConstructorDecl *DeclareImplicitCopyConstructor(CXXRecordDecl *ClassDecl); + /// ActOnBlockStart - This callback is invoked when a block literal is + /// started. + void ActOnBlockStart(SourceLocation CaretLoc, Scope *CurScope); - /// DefineImplicitCopyConstructor - Checks for feasibility of - /// defining this constructor as the copy constructor. - void DefineImplicitCopyConstructor(SourceLocation CurrentLocation, - CXXConstructorDecl *Constructor); + /// ActOnBlockArguments - This callback allows processing of block arguments. + /// If there are no arguments, this is still invoked. + void ActOnBlockArguments(SourceLocation CaretLoc, Declarator &ParamInfo, + Scope *CurScope); - /// Declare the implicit move constructor for the given class. - /// - /// \param ClassDecl The Class declaration into which the implicit - /// move constructor will be added. - /// - /// \returns The implicitly-declared move constructor, or NULL if it wasn't - /// declared. - CXXConstructorDecl *DeclareImplicitMoveConstructor(CXXRecordDecl *ClassDecl); + /// ActOnBlockError - If there is an error parsing a block, this callback + /// is invoked to pop the information about the block from the action impl. + void ActOnBlockError(SourceLocation CaretLoc, Scope *CurScope); - /// DefineImplicitMoveConstructor - Checks for feasibility of - /// defining this constructor as the move constructor. - void DefineImplicitMoveConstructor(SourceLocation CurrentLocation, - CXXConstructorDecl *Constructor); + /// ActOnBlockStmtExpr - This is called when the body of a block statement + /// literal was successfully completed. ^(int x){...} + ExprResult ActOnBlockStmtExpr(SourceLocation CaretLoc, Stmt *Body, + Scope *CurScope); - /// Declare the implicit copy assignment operator for the given class. - /// - /// \param ClassDecl The class declaration into which the implicit - /// copy assignment operator will be added. - /// - /// \returns The implicitly-declared copy assignment operator. - CXXMethodDecl *DeclareImplicitCopyAssignment(CXXRecordDecl *ClassDecl); + //===---------------------------- Clang Extensions ----------------------===// - /// Defines an implicitly-declared copy assignment operator. - void DefineImplicitCopyAssignment(SourceLocation CurrentLocation, - CXXMethodDecl *MethodDecl); + /// __builtin_convertvector(...) + ExprResult ActOnConvertVectorExpr(Expr *E, ParsedType ParsedDestTy, + SourceLocation BuiltinLoc, + SourceLocation RParenLoc); - /// Declare the implicit move assignment operator for the given class. - /// - /// \param ClassDecl The Class declaration into which the implicit - /// move assignment operator will be added. - /// - /// \returns The implicitly-declared move assignment operator, or NULL if it - /// wasn't declared. - CXXMethodDecl *DeclareImplicitMoveAssignment(CXXRecordDecl *ClassDecl); + //===---------------------------- OpenCL Features -----------------------===// - /// Defines an implicitly-declared move assignment operator. - void DefineImplicitMoveAssignment(SourceLocation CurrentLocation, - CXXMethodDecl *MethodDecl); + /// __builtin_astype(...) + ExprResult ActOnAsTypeExpr(Expr *E, ParsedType ParsedDestTy, + SourceLocation BuiltinLoc, + SourceLocation RParenLoc); + ExprResult BuildAsTypeExpr(Expr *E, QualType DestTy, + SourceLocation BuiltinLoc, + SourceLocation RParenLoc); - /// Force the declaration of any implicitly-declared members of this - /// class. - void ForceDeclarationOfImplicitMembers(CXXRecordDecl *Class); + /// Attempts to produce a RecoveryExpr after some AST node cannot be created. + ExprResult CreateRecoveryExpr(SourceLocation Begin, SourceLocation End, + ArrayRef SubExprs, + QualType T = QualType()); - /// Check a completed declaration of an implicit special member. - void CheckImplicitSpecialMemberDeclaration(Scope *S, FunctionDecl *FD); + // Note that LK_String is intentionally after the other literals, as + // this is used for diagnostics logic. + enum ObjCLiteralKind { + LK_Array, + LK_Dictionary, + LK_Numeric, + LK_Boxed, + LK_String, + LK_Block, + LK_None + }; + ObjCLiteralKind CheckLiteralKind(Expr *FromE); - /// Determine whether the given function is an implicitly-deleted - /// special member function. - bool isImplicitlyDeleted(FunctionDecl *FD); + ExprResult PerformObjectMemberConversion(Expr *From, + NestedNameSpecifier *Qualifier, + NamedDecl *FoundDecl, + NamedDecl *Member); - /// Check whether 'this' shows up in the type of a static member - /// function after the (naturally empty) cv-qualifier-seq would be. - /// - /// \returns true if an error occurred. - bool checkThisInStaticMemberFunctionType(CXXMethodDecl *Method); + /// CheckCallReturnType - Checks that a call expression's return type is + /// complete. Returns true on failure. The location passed in is the location + /// that best represents the call. + bool CheckCallReturnType(QualType ReturnType, SourceLocation Loc, + CallExpr *CE, FunctionDecl *FD); - /// Whether this' shows up in the exception specification of a static - /// member function. - bool checkThisInStaticMemberFunctionExceptionSpec(CXXMethodDecl *Method); + /// Emit a warning for all pending noderef expressions that we recorded. + void WarnOnPendingNoDerefs(ExpressionEvaluationContextRecord &Rec); - /// Check whether 'this' shows up in the attributes of the given - /// static member function. - /// - /// \returns true if an error occurred. - bool checkThisInStaticMemberFunctionAttributes(CXXMethodDecl *Method); + ExprResult BuildCXXDefaultInitExpr(SourceLocation Loc, FieldDecl *Field); - /// MaybeBindToTemporary - If the passed in expression has a record type with - /// a non-trivial destructor, this will return CXXBindTemporaryExpr. Otherwise - /// it simply returns the passed in expression. - ExprResult MaybeBindToTemporary(Expr *E); + /// Instantiate or parse a C++ default argument expression as necessary. + /// Return true on error. + bool CheckCXXDefaultArgExpr(SourceLocation CallLoc, FunctionDecl *FD, + ParmVarDecl *Param, Expr *Init = nullptr, + bool SkipImmediateInvocations = true); + + /// BuildCXXDefaultArgExpr - Creates a CXXDefaultArgExpr, instantiating + /// the default expr if needed. + ExprResult BuildCXXDefaultArgExpr(SourceLocation CallLoc, FunctionDecl *FD, + ParmVarDecl *Param, Expr *Init = nullptr); /// Wrap the expression in a ConstantExpr if it is a potential immediate /// invocation. ExprResult CheckForImmediateInvocation(ExprResult E, FunctionDecl *Decl); - bool CheckImmediateEscalatingFunctionDefinition( - FunctionDecl *FD, const sema::FunctionScopeInfo *FSI); - void MarkExpressionAsImmediateEscalating(Expr *E); - void DiagnoseImmediateEscalatingReason(FunctionDecl *FD); + bool IsInvalidSMECallConversion(QualType FromType, QualType ToType); - bool CompleteConstructorCall(CXXConstructorDecl *Constructor, - QualType DeclInitType, MultiExprArg ArgsPtr, - SourceLocation Loc, - SmallVectorImpl &ConvertedArgs, - bool AllowExplicit = false, - bool IsListInitialization = false); + const DeclContext *getCurObjCLexicalContext() const { + const DeclContext *DC = getCurLexicalContext(); + // A category implicitly has the attribute of the interface. + if (const ObjCCategoryDecl *CatD = dyn_cast(DC)) + DC = CatD->getClassInterface(); + return DC; + } - ParsedType getInheritingConstructorName(CXXScopeSpec &SS, - SourceLocation NameLoc, - IdentifierInfo &Name); + /// Abstract base class used for diagnosing integer constant + /// expression violations. + class VerifyICEDiagnoser { + public: + bool Suppress; - ParsedType getConstructorName(IdentifierInfo &II, SourceLocation NameLoc, - Scope *S, CXXScopeSpec &SS, - bool EnteringContext); - ParsedType getDestructorName(IdentifierInfo &II, SourceLocation NameLoc, - Scope *S, CXXScopeSpec &SS, - ParsedType ObjectType, bool EnteringContext); + VerifyICEDiagnoser(bool Suppress = false) : Suppress(Suppress) {} - ParsedType getDestructorTypeForDecltype(const DeclSpec &DS, - ParsedType ObjectType); + virtual SemaDiagnosticBuilder + diagnoseNotICEType(Sema &S, SourceLocation Loc, QualType T); + virtual SemaDiagnosticBuilder diagnoseNotICE(Sema &S, + SourceLocation Loc) = 0; + virtual SemaDiagnosticBuilder diagnoseFold(Sema &S, SourceLocation Loc); + virtual ~VerifyICEDiagnoser() {} + }; - // Checks that reinterpret casts don't have undefined behavior. - void CheckCompatibleReinterpretCast(QualType SrcType, QualType DestType, - bool IsDereference, SourceRange Range); + enum AllowFoldKind { + NoFold, + AllowFold, + }; - // Checks that the vector type should be initialized from a scalar - // by splatting the value rather than populating a single element. - // This is the case for AltiVecVector types as well as with - // AltiVecPixel and AltiVecBool when -faltivec-src-compat=xl is specified. - bool ShouldSplatAltivecScalarInCast(const VectorType *VecTy); + /// VerifyIntegerConstantExpression - Verifies that an expression is an ICE, + /// and reports the appropriate diagnostics. Returns false on success. + /// Can optionally return the value of the expression. + ExprResult VerifyIntegerConstantExpression(Expr *E, llvm::APSInt *Result, + VerifyICEDiagnoser &Diagnoser, + AllowFoldKind CanFold = NoFold); + ExprResult VerifyIntegerConstantExpression(Expr *E, llvm::APSInt *Result, + unsigned DiagID, + AllowFoldKind CanFold = NoFold); + ExprResult VerifyIntegerConstantExpression(Expr *E, + llvm::APSInt *Result = nullptr, + AllowFoldKind CanFold = NoFold); + ExprResult VerifyIntegerConstantExpression(Expr *E, + AllowFoldKind CanFold = NoFold) { + return VerifyIntegerConstantExpression(E, nullptr, CanFold); + } - // Checks if the -faltivec-src-compat=gcc option is specified. - // If so, AltiVecVector, AltiVecBool and AltiVecPixel types are - // treated the same way as they are when trying to initialize - // these vectors on gcc (an error is emitted). - bool CheckAltivecInitFromScalar(SourceRange R, QualType VecTy, - QualType SrcTy); + /// DiagnoseAssignmentAsCondition - Given that an expression is + /// being used as a boolean condition, warn if it's an assignment. + void DiagnoseAssignmentAsCondition(Expr *E); - /// ActOnCXXNamedCast - Parse - /// {dynamic,static,reinterpret,const,addrspace}_cast's. - ExprResult ActOnCXXNamedCast(SourceLocation OpLoc, - tok::TokenKind Kind, - SourceLocation LAngleBracketLoc, - Declarator &D, - SourceLocation RAngleBracketLoc, - SourceLocation LParenLoc, - Expr *E, - SourceLocation RParenLoc); + /// Redundant parentheses over an equality comparison can indicate + /// that the user intended an assignment used as condition. + void DiagnoseEqualityWithExtraParens(ParenExpr *ParenE); - ExprResult BuildCXXNamedCast(SourceLocation OpLoc, - tok::TokenKind Kind, - TypeSourceInfo *Ty, - Expr *E, - SourceRange AngleBrackets, - SourceRange Parens); + class FullExprArg { + public: + FullExprArg() : E(nullptr) {} + FullExprArg(Sema &actions) : E(nullptr) {} - ExprResult ActOnBuiltinBitCastExpr(SourceLocation KWLoc, Declarator &Dcl, - ExprResult Operand, - SourceLocation RParenLoc); + ExprResult release() { return E; } - ExprResult BuildBuiltinBitCastExpr(SourceLocation KWLoc, TypeSourceInfo *TSI, - Expr *Operand, SourceLocation RParenLoc); + Expr *get() const { return E; } - ExprResult BuildCXXTypeId(QualType TypeInfoType, - SourceLocation TypeidLoc, - TypeSourceInfo *Operand, - SourceLocation RParenLoc); - ExprResult BuildCXXTypeId(QualType TypeInfoType, - SourceLocation TypeidLoc, - Expr *Operand, - SourceLocation RParenLoc); + Expr *operator->() { return E; } - /// ActOnCXXTypeid - Parse typeid( something ). - ExprResult ActOnCXXTypeid(SourceLocation OpLoc, - SourceLocation LParenLoc, bool isType, - void *TyOrExpr, - SourceLocation RParenLoc); + private: + // FIXME: No need to make the entire Sema class a friend when it's just + // Sema::MakeFullExpr that needs access to the constructor below. + friend class Sema; - ExprResult BuildCXXUuidof(QualType TypeInfoType, - SourceLocation TypeidLoc, - TypeSourceInfo *Operand, - SourceLocation RParenLoc); - ExprResult BuildCXXUuidof(QualType TypeInfoType, - SourceLocation TypeidLoc, - Expr *Operand, - SourceLocation RParenLoc); + explicit FullExprArg(Expr *expr) : E(expr) {} - /// ActOnCXXUuidof - Parse __uuidof( something ). - ExprResult ActOnCXXUuidof(SourceLocation OpLoc, - SourceLocation LParenLoc, bool isType, - void *TyOrExpr, - SourceLocation RParenLoc); + Expr *E; + }; - /// Handle a C++1z fold-expression: ( expr op ... op expr ). - ExprResult ActOnCXXFoldExpr(Scope *S, SourceLocation LParenLoc, Expr *LHS, - tok::TokenKind Operator, - SourceLocation EllipsisLoc, Expr *RHS, - SourceLocation RParenLoc); - ExprResult BuildCXXFoldExpr(UnresolvedLookupExpr *Callee, - SourceLocation LParenLoc, Expr *LHS, - BinaryOperatorKind Operator, - SourceLocation EllipsisLoc, Expr *RHS, - SourceLocation RParenLoc, - std::optional NumExpansions); - ExprResult BuildEmptyCXXFoldExpr(SourceLocation EllipsisLoc, - BinaryOperatorKind Operator); - - //// ActOnCXXThis - Parse 'this' pointer. - ExprResult ActOnCXXThis(SourceLocation loc); - - /// Build a CXXThisExpr and mark it referenced in the current context. - Expr *BuildCXXThisExpr(SourceLocation Loc, QualType Type, bool IsImplicit); - void MarkThisReferenced(CXXThisExpr *This); - - /// Try to retrieve the type of the 'this' pointer. - /// - /// \returns The type of 'this', if possible. Otherwise, returns a NULL type. - QualType getCurrentThisType(); + FullExprArg MakeFullExpr(Expr *Arg) { + return MakeFullExpr(Arg, Arg ? Arg->getExprLoc() : SourceLocation()); + } + FullExprArg MakeFullExpr(Expr *Arg, SourceLocation CC) { + return FullExprArg( + ActOnFinishFullExpr(Arg, CC, /*DiscardedValue*/ false).get()); + } + FullExprArg MakeFullDiscardedValueExpr(Expr *Arg) { + ExprResult FE = + ActOnFinishFullExpr(Arg, Arg ? Arg->getExprLoc() : SourceLocation(), + /*DiscardedValue*/ true); + return FullExprArg(FE.get()); + } - /// When non-NULL, the C++ 'this' expression is allowed despite the - /// current context not being a non-static member function. In such cases, - /// this provides the type used for 'this'. - QualType CXXThisTypeOverride; + class ConditionResult { + Decl *ConditionVar; + FullExprArg Condition; + bool Invalid; + std::optional KnownValue; - /// RAII object used to temporarily allow the C++ 'this' expression - /// to be used, with the given qualifiers on the current class type. - class CXXThisScopeRAII { - Sema &S; - QualType OldCXXThisTypeOverride; - bool Enabled; + friend class Sema; + ConditionResult(Sema &S, Decl *ConditionVar, FullExprArg Condition, + bool IsConstexpr) + : ConditionVar(ConditionVar), Condition(Condition), Invalid(false) { + if (IsConstexpr && Condition.get()) { + if (std::optional Val = + Condition.get()->getIntegerConstantExpr(S.Context)) { + KnownValue = !!(*Val); + } + } + } + explicit ConditionResult(bool Invalid) + : ConditionVar(nullptr), Condition(nullptr), Invalid(Invalid), + KnownValue(std::nullopt) {} public: - /// Introduce a new scope where 'this' may be allowed (when enabled), - /// using the given declaration (which is either a class template or a - /// class) along with the given qualifiers. - /// along with the qualifiers placed on '*this'. - CXXThisScopeRAII(Sema &S, Decl *ContextDecl, Qualifiers CXXThisTypeQuals, - bool Enabled = true); - - ~CXXThisScopeRAII(); + ConditionResult() : ConditionResult(false) {} + bool isInvalid() const { return Invalid; } + std::pair get() const { + return std::make_pair(cast_or_null(ConditionVar), + Condition.get()); + } + std::optional getKnownValue() const { return KnownValue; } }; + static ConditionResult ConditionError() { return ConditionResult(true); } - /// Make sure the value of 'this' is actually available in the current - /// context, if it is a potentially evaluated context. - /// - /// \param Loc The location at which the capture of 'this' occurs. - /// - /// \param Explicit Whether 'this' is explicitly captured in a lambda - /// capture list. + /// CheckBooleanCondition - Diagnose problems involving the use of + /// the given expression as a boolean condition (e.g. in an if + /// statement). Also performs the standard function and array + /// decays, possibly changing the input variable. /// - /// \param FunctionScopeIndexToStopAt If non-null, it points to the index - /// of the FunctionScopeInfo stack beyond which we do not attempt to capture. - /// This is useful when enclosing lambdas must speculatively capture - /// 'this' that may or may not be used in certain specializations of - /// a nested generic lambda (depending on whether the name resolves to - /// a non-static member function or a static function). - /// \return returns 'true' if failed, 'false' if success. - bool CheckCXXThisCapture(SourceLocation Loc, bool Explicit = false, - bool BuildAndDiagnose = true, - const unsigned *const FunctionScopeIndexToStopAt = nullptr, - bool ByCopy = false); + /// \param Loc - A location associated with the condition, e.g. the + /// 'if' keyword. + /// \return true iff there were any errors + ExprResult CheckBooleanCondition(SourceLocation Loc, Expr *E, + bool IsConstexpr = false); - /// Determine whether the given type is the type of *this that is used - /// outside of the body of a member function for a type that is currently - /// being defined. - bool isThisOutsideMemberFunctionBody(QualType BaseType); + enum class ConditionKind { + Boolean, ///< A boolean condition, from 'if', 'while', 'for', or 'do'. + ConstexprIf, ///< A constant boolean condition from 'if constexpr'. + Switch ///< An integral condition for a 'switch' statement. + }; - /// ActOnCXXBoolLiteral - Parse {true,false} literals. - ExprResult ActOnCXXBoolLiteral(SourceLocation OpLoc, tok::TokenKind Kind); + ConditionResult ActOnCondition(Scope *S, SourceLocation Loc, Expr *SubExpr, + ConditionKind CK, bool MissingOK = false); + QualType CheckConditionalOperands( // C99 6.5.15 + ExprResult &Cond, ExprResult &LHS, ExprResult &RHS, ExprValueKind &VK, + ExprObjectKind &OK, SourceLocation QuestionLoc); - /// ActOnObjCBoolLiteral - Parse {__objc_yes,__objc_no} literals. - ExprResult ActOnObjCBoolLiteral(SourceLocation OpLoc, tok::TokenKind Kind); + QualType FindCompositeObjCPointerType(ExprResult &LHS, ExprResult &RHS, + SourceLocation QuestionLoc); - ExprResult - ActOnObjCAvailabilityCheckExpr(llvm::ArrayRef AvailSpecs, - SourceLocation AtLoc, SourceLocation RParen); + bool DiagnoseConditionalForNull(const Expr *LHSExpr, const Expr *RHSExpr, + SourceLocation QuestionLoc); - /// ActOnCXXNullPtrLiteral - Parse 'nullptr'. - ExprResult ActOnCXXNullPtrLiteral(SourceLocation Loc); + /// type checking for vector binary operators. + QualType CheckVectorOperands(ExprResult &LHS, ExprResult &RHS, + SourceLocation Loc, bool IsCompAssign, + bool AllowBothBool, bool AllowBoolConversion, + bool AllowBoolOperation, bool ReportInvalid); + QualType GetSignedVectorType(QualType V); + QualType GetSignedSizelessVectorType(QualType V); + QualType CheckVectorCompareOperands(ExprResult &LHS, ExprResult &RHS, + SourceLocation Loc, + BinaryOperatorKind Opc); + QualType CheckSizelessVectorCompareOperands(ExprResult &LHS, ExprResult &RHS, + SourceLocation Loc, + BinaryOperatorKind Opc); + QualType CheckVectorLogicalOperands(ExprResult &LHS, ExprResult &RHS, + SourceLocation Loc); - //// ActOnCXXThrow - Parse throw expressions. - ExprResult ActOnCXXThrow(Scope *S, SourceLocation OpLoc, Expr *expr); - ExprResult BuildCXXThrow(SourceLocation OpLoc, Expr *Ex, - bool IsThrownVarInScope); - bool CheckCXXThrowOperand(SourceLocation ThrowLoc, QualType ThrowTy, Expr *E); + /// Context in which we're performing a usual arithmetic conversion. + enum ArithConvKind { + /// An arithmetic operation. + ACK_Arithmetic, + /// A bitwise operation. + ACK_BitwiseOp, + /// A comparison. + ACK_Comparison, + /// A conditional (?:) operator. + ACK_Conditional, + /// A compound assignment expression. + ACK_CompAssign, + }; - /// ActOnCXXTypeConstructExpr - Parse construction of a specified type. - /// Can be interpreted either as function-style casting ("int(x)") - /// or class type construction ("ClassType(x,y,z)") - /// or creation of a value-initialized type ("int()"). - ExprResult ActOnCXXTypeConstructExpr(ParsedType TypeRep, - SourceLocation LParenOrBraceLoc, - MultiExprArg Exprs, - SourceLocation RParenOrBraceLoc, - bool ListInitialization); + // type checking for sizeless vector binary operators. + QualType CheckSizelessVectorOperands(ExprResult &LHS, ExprResult &RHS, + SourceLocation Loc, bool IsCompAssign, + ArithConvKind OperationKind); - ExprResult BuildCXXTypeConstructExpr(TypeSourceInfo *Type, - SourceLocation LParenLoc, - MultiExprArg Exprs, - SourceLocation RParenLoc, - bool ListInitialization); + /// Type checking for matrix binary operators. + QualType CheckMatrixElementwiseOperands(ExprResult &LHS, ExprResult &RHS, + SourceLocation Loc, + bool IsCompAssign); + QualType CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS, + SourceLocation Loc, bool IsCompAssign); - /// ActOnCXXNew - Parsed a C++ 'new' expression. - ExprResult ActOnCXXNew(SourceLocation StartLoc, bool UseGlobal, - SourceLocation PlacementLParen, - MultiExprArg PlacementArgs, - SourceLocation PlacementRParen, - SourceRange TypeIdParens, Declarator &D, - Expr *Initializer); - ExprResult - BuildCXXNew(SourceRange Range, bool UseGlobal, SourceLocation PlacementLParen, - MultiExprArg PlacementArgs, SourceLocation PlacementRParen, - SourceRange TypeIdParens, QualType AllocType, - TypeSourceInfo *AllocTypeInfo, std::optional ArraySize, - SourceRange DirectInitRange, Expr *Initializer); + bool isValidSveBitcast(QualType srcType, QualType destType); + bool isValidRVVBitcast(QualType srcType, QualType destType); - /// Determine whether \p FD is an aligned allocation or deallocation - /// function that is unavailable. - bool isUnavailableAlignedAllocationFunction(const FunctionDecl &FD) const; + bool areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy); - /// Produce diagnostics if \p FD is an aligned allocation or deallocation - /// function that is unavailable. - void diagnoseUnavailableAlignedAllocation(const FunctionDecl &FD, - SourceLocation Loc); + bool areVectorTypesSameSize(QualType srcType, QualType destType); + bool areLaxCompatibleVectorTypes(QualType srcType, QualType destType); + bool isLaxVectorConversion(QualType srcType, QualType destType); + bool anyAltivecTypes(QualType srcType, QualType destType); - bool CheckAllocatedType(QualType AllocType, SourceLocation Loc, - SourceRange R); + // type checking C++ declaration initializers (C++ [dcl.init]). - /// The scope in which to find allocation functions. - enum AllocationFunctionScope { - /// Only look for allocation functions in the global scope. - AFS_Global, - /// Only look for allocation functions in the scope of the - /// allocated class. - AFS_Class, - /// Look for allocation functions in both the global scope - /// and in the scope of the allocated class. - AFS_Both - }; + ExprResult checkUnknownAnyCast(SourceRange TypeRange, QualType CastType, + Expr *CastExpr, CastKind &CastKind, + ExprValueKind &VK, CXXCastPath &Path); - /// Finds the overloads of operator new and delete that are appropriate - /// for the allocation. - bool FindAllocationFunctions(SourceLocation StartLoc, SourceRange Range, - AllocationFunctionScope NewScope, - AllocationFunctionScope DeleteScope, - QualType AllocType, bool IsArray, - bool &PassAlignment, MultiExprArg PlaceArgs, - FunctionDecl *&OperatorNew, - FunctionDecl *&OperatorDelete, - bool Diagnose = true); - void DeclareGlobalNewDelete(); - void DeclareGlobalAllocationFunction(DeclarationName Name, QualType Return, - ArrayRef Params); + /// Force an expression with unknown-type to an expression of the + /// given type. + ExprResult forceUnknownAnyToType(Expr *E, QualType ToType); - bool FindDeallocationFunction(SourceLocation StartLoc, CXXRecordDecl *RD, - DeclarationName Name, FunctionDecl *&Operator, - bool Diagnose = true, bool WantSize = false, - bool WantAligned = false); - FunctionDecl *FindUsualDeallocationFunction(SourceLocation StartLoc, - bool CanProvideSize, - bool Overaligned, - DeclarationName Name); - FunctionDecl *FindDeallocationFunctionForDestructor(SourceLocation StartLoc, - CXXRecordDecl *RD); + /// Type-check an expression that's being passed to an + /// __unknown_anytype parameter. + ExprResult checkUnknownAnyArg(SourceLocation callLoc, Expr *result, + QualType ¶mType); - /// ActOnCXXDelete - Parsed a C++ 'delete' expression - ExprResult ActOnCXXDelete(SourceLocation StartLoc, - bool UseGlobal, bool ArrayForm, - Expr *Operand); - void CheckVirtualDtorCall(CXXDestructorDecl *dtor, SourceLocation Loc, - bool IsDelete, bool CallCanBeVirtual, - bool WarnOnNonAbstractTypes, - SourceLocation DtorLoc); - - ExprResult ActOnNoexceptExpr(SourceLocation KeyLoc, SourceLocation LParen, - Expr *Operand, SourceLocation RParen); - ExprResult BuildCXXNoexceptExpr(SourceLocation KeyLoc, Expr *Operand, - SourceLocation RParen); + // CheckMatrixCast - Check type constraints for matrix casts. + // We allow casting between matrixes of the same dimensions i.e. when they + // have the same number of rows and column. Returns true if the cast is + // invalid. + bool CheckMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy, + CastKind &Kind); - /// Parsed one of the type trait support pseudo-functions. - ExprResult ActOnTypeTrait(TypeTrait Kind, SourceLocation KWLoc, - ArrayRef Args, - SourceLocation RParenLoc); - ExprResult BuildTypeTrait(TypeTrait Kind, SourceLocation KWLoc, - ArrayRef Args, - SourceLocation RParenLoc); + // CheckVectorCast - check type constraints for vectors. + // Since vectors are an extension, there are no C standard reference for this. + // We allow casting between vectors and integer datatypes of the same size. + // returns true if the cast is invalid + bool CheckVectorCast(SourceRange R, QualType VectorTy, QualType Ty, + CastKind &Kind); - /// ActOnArrayTypeTrait - Parsed one of the binary type trait support - /// pseudo-functions. - ExprResult ActOnArrayTypeTrait(ArrayTypeTrait ATT, - SourceLocation KWLoc, - ParsedType LhsTy, - Expr *DimExpr, - SourceLocation RParen); + /// Prepare `SplattedExpr` for a vector splat operation, adding + /// implicit casts if necessary. + ExprResult prepareVectorSplat(QualType VectorTy, Expr *SplattedExpr); - ExprResult BuildArrayTypeTrait(ArrayTypeTrait ATT, - SourceLocation KWLoc, - TypeSourceInfo *TSInfo, - Expr *DimExpr, - SourceLocation RParen); + // CheckExtVectorCast - check type constraints for extended vectors. + // Since vectors are an extension, there are no C standard reference for this. + // We allow casting between vectors and integer datatypes of the same size, + // or vectors and the element type of that vector. + // returns the cast expr + ExprResult CheckExtVectorCast(SourceRange R, QualType DestTy, Expr *CastExpr, + CastKind &Kind); - /// ActOnExpressionTrait - Parsed one of the unary type trait support - /// pseudo-functions. - ExprResult ActOnExpressionTrait(ExpressionTrait OET, - SourceLocation KWLoc, - Expr *Queried, - SourceLocation RParen); + QualType PreferredConditionType(ConditionKind K) const { + return K == ConditionKind::Switch ? Context.IntTy : Context.BoolTy; + } - ExprResult BuildExpressionTrait(ExpressionTrait OET, - SourceLocation KWLoc, - Expr *Queried, - SourceLocation RParen); + // UsualUnaryConversions - promotes integers (C99 6.3.1.1p2) and converts + // functions and arrays to their respective pointers (C99 6.3.2.1). + ExprResult UsualUnaryConversions(Expr *E); - ExprResult ActOnStartCXXMemberReference(Scope *S, - Expr *Base, - SourceLocation OpLoc, - tok::TokenKind OpKind, - ParsedType &ObjectType, - bool &MayBePseudoDestructor); + /// CallExprUnaryConversions - a special case of an unary conversion + /// performed on a function designator of a call expression. + ExprResult CallExprUnaryConversions(Expr *E); - ExprResult BuildPseudoDestructorExpr(Expr *Base, - SourceLocation OpLoc, - tok::TokenKind OpKind, - const CXXScopeSpec &SS, - TypeSourceInfo *ScopeType, - SourceLocation CCLoc, - SourceLocation TildeLoc, - PseudoDestructorTypeStorage DestroyedType); + // DefaultFunctionArrayConversion - converts functions and arrays + // to their respective pointers (C99 6.3.2.1). + ExprResult DefaultFunctionArrayConversion(Expr *E, bool Diagnose = true); - ExprResult ActOnPseudoDestructorExpr(Scope *S, Expr *Base, - SourceLocation OpLoc, - tok::TokenKind OpKind, - CXXScopeSpec &SS, - UnqualifiedId &FirstTypeName, - SourceLocation CCLoc, - SourceLocation TildeLoc, - UnqualifiedId &SecondTypeName); + // DefaultFunctionArrayLvalueConversion - converts functions and + // arrays to their respective pointers and performs the + // lvalue-to-rvalue conversion. + ExprResult DefaultFunctionArrayLvalueConversion(Expr *E, + bool Diagnose = true); - ExprResult ActOnPseudoDestructorExpr(Scope *S, Expr *Base, - SourceLocation OpLoc, - tok::TokenKind OpKind, - SourceLocation TildeLoc, - const DeclSpec& DS); + // DefaultLvalueConversion - performs lvalue-to-rvalue conversion on + // the operand. This function is a no-op if the operand has a function type + // or an array type. + ExprResult DefaultLvalueConversion(Expr *E); - /// MaybeCreateExprWithCleanups - If the current full-expression - /// requires any cleanups, surround it with a ExprWithCleanups node. - /// Otherwise, just returns the passed-in expression. - Expr *MaybeCreateExprWithCleanups(Expr *SubExpr); - Stmt *MaybeCreateStmtWithCleanups(Stmt *SubStmt); - ExprResult MaybeCreateExprWithCleanups(ExprResult SubExpr); + // DefaultArgumentPromotion (C99 6.5.2.2p6). Used for function calls that + // do not have a prototype. Integer promotions are performed on each + // argument, and arguments that have type float are promoted to double. + ExprResult DefaultArgumentPromotion(Expr *E); - MaterializeTemporaryExpr * - CreateMaterializeTemporaryExpr(QualType T, Expr *Temporary, - bool BoundToLvalueReference); + VariadicCallType getVariadicCallType(FunctionDecl *FDecl, + const FunctionProtoType *Proto, + Expr *Fn); - ExprResult ActOnFinishFullExpr(Expr *Expr, bool DiscardedValue) { - return ActOnFinishFullExpr( - Expr, Expr ? Expr->getExprLoc() : SourceLocation(), DiscardedValue); - } - ExprResult ActOnFinishFullExpr(Expr *Expr, SourceLocation CC, - bool DiscardedValue, bool IsConstexpr = false, - bool IsTemplateArgument = false); - StmtResult ActOnFinishFullStmt(Stmt *Stmt); + // Used for determining in which context a type is allowed to be passed to a + // vararg function. + enum VarArgKind { + VAK_Valid, + VAK_ValidInCXX11, + VAK_Undefined, + VAK_MSVCUndefined, + VAK_Invalid + }; - // Marks SS invalid if it represents an incomplete type. - bool RequireCompleteDeclContext(CXXScopeSpec &SS, DeclContext *DC); - // Complete an enum decl, maybe without a scope spec. - bool RequireCompleteEnumDecl(EnumDecl *D, SourceLocation L, - CXXScopeSpec *SS = nullptr); + // Determines which VarArgKind fits an expression. + VarArgKind isValidVarArgType(const QualType &Ty); - DeclContext *computeDeclContext(QualType T); - DeclContext *computeDeclContext(const CXXScopeSpec &SS, - bool EnteringContext = false); - bool isDependentScopeSpecifier(const CXXScopeSpec &SS); - CXXRecordDecl *getCurrentInstantiationOf(NestedNameSpecifier *NNS); + /// Check to see if the given expression is a valid argument to a variadic + /// function, issuing a diagnostic if not. + void checkVariadicArgument(const Expr *E, VariadicCallType CT); - /// The parser has parsed a global nested-name-specifier '::'. - /// - /// \param CCLoc The location of the '::'. - /// - /// \param SS The nested-name-specifier, which will be updated in-place - /// to reflect the parsed nested-name-specifier. - /// - /// \returns true if an error occurred, false otherwise. - bool ActOnCXXGlobalScopeSpecifier(SourceLocation CCLoc, CXXScopeSpec &SS); + /// GatherArgumentsForCall - Collector argument expressions for various + /// form of call prototypes. + bool GatherArgumentsForCall(SourceLocation CallLoc, FunctionDecl *FDecl, + const FunctionProtoType *Proto, + unsigned FirstParam, ArrayRef Args, + SmallVectorImpl &AllArgs, + VariadicCallType CallType = VariadicDoesNotApply, + bool AllowExplicit = false, + bool IsListInitialization = false); - /// The parser has parsed a '__super' nested-name-specifier. - /// - /// \param SuperLoc The location of the '__super' keyword. - /// - /// \param ColonColonLoc The location of the '::'. - /// - /// \param SS The nested-name-specifier, which will be updated in-place - /// to reflect the parsed nested-name-specifier. - /// - /// \returns true if an error occurred, false otherwise. - bool ActOnSuperScopeSpecifier(SourceLocation SuperLoc, - SourceLocation ColonColonLoc, CXXScopeSpec &SS); + // DefaultVariadicArgumentPromotion - Like DefaultArgumentPromotion, but + // will create a runtime trap if the resulting type is not a POD type. + ExprResult DefaultVariadicArgumentPromotion(Expr *E, VariadicCallType CT, + FunctionDecl *FDecl); - bool isAcceptableNestedNameSpecifier(const NamedDecl *SD, - bool *CanCorrect = nullptr); - NamedDecl *FindFirstQualifierInScope(Scope *S, NestedNameSpecifier *NNS); + // UsualArithmeticConversions - performs the UsualUnaryConversions on it's + // operands and then handles various conversions that are common to binary + // operators (C99 6.3.1.8). If both operands aren't arithmetic, this + // routine returns the first non-arithmetic type found. The client is + // responsible for emitting appropriate error diagnostics. + QualType UsualArithmeticConversions(ExprResult &LHS, ExprResult &RHS, + SourceLocation Loc, ArithConvKind ACK); - /// Keeps information about an identifier in a nested-name-spec. - /// - struct NestedNameSpecInfo { - /// The type of the object, if we're parsing nested-name-specifier in - /// a member access expression. - ParsedType ObjectType; + /// AssignConvertType - All of the 'assignment' semantic checks return this + /// enum to indicate whether the assignment was allowed. These checks are + /// done for simple assignments, as well as initialization, return from + /// function, argument passing, etc. The query is phrased in terms of a + /// source and destination type. + enum AssignConvertType { + /// Compatible - the types are compatible according to the standard. + Compatible, - /// The identifier preceding the '::'. - IdentifierInfo *Identifier; + /// PointerToInt - The assignment converts a pointer to an int, which we + /// accept as an extension. + PointerToInt, - /// The location of the identifier. - SourceLocation IdentifierLoc; + /// IntToPointer - The assignment converts an int to a pointer, which we + /// accept as an extension. + IntToPointer, - /// The location of the '::'. - SourceLocation CCLoc; + /// FunctionVoidPointer - The assignment is between a function pointer and + /// void*, which the standard doesn't allow, but we accept as an extension. + FunctionVoidPointer, - /// Creates info object for the most typical case. - NestedNameSpecInfo(IdentifierInfo *II, SourceLocation IdLoc, - SourceLocation ColonColonLoc, ParsedType ObjectType = ParsedType()) - : ObjectType(ObjectType), Identifier(II), IdentifierLoc(IdLoc), - CCLoc(ColonColonLoc) { - } + /// IncompatiblePointer - The assignment is between two pointers types that + /// are not compatible, but we accept them as an extension. + IncompatiblePointer, - NestedNameSpecInfo(IdentifierInfo *II, SourceLocation IdLoc, - SourceLocation ColonColonLoc, QualType ObjectType) - : ObjectType(ParsedType::make(ObjectType)), Identifier(II), - IdentifierLoc(IdLoc), CCLoc(ColonColonLoc) { - } - }; + /// IncompatibleFunctionPointer - The assignment is between two function + /// pointers types that are not compatible, but we accept them as an + /// extension. + IncompatibleFunctionPointer, - bool BuildCXXNestedNameSpecifier(Scope *S, - NestedNameSpecInfo &IdInfo, - bool EnteringContext, - CXXScopeSpec &SS, - NamedDecl *ScopeLookupResult, - bool ErrorRecoveryLookup, - bool *IsCorrectedToColon = nullptr, - bool OnlyNamespace = false); + /// IncompatibleFunctionPointerStrict - The assignment is between two + /// function pointer types that are not identical, but are compatible, + /// unless compiled with -fsanitize=cfi, in which case the type mismatch + /// may trip an indirect call runtime check. + IncompatibleFunctionPointerStrict, - /// The parser has parsed a nested-name-specifier 'identifier::'. - /// - /// \param S The scope in which this nested-name-specifier occurs. - /// - /// \param IdInfo Parser information about an identifier in the - /// nested-name-spec. - /// - /// \param EnteringContext Whether we're entering the context nominated by - /// this nested-name-specifier. - /// - /// \param SS The nested-name-specifier, which is both an input - /// parameter (the nested-name-specifier before this type) and an - /// output parameter (containing the full nested-name-specifier, - /// including this new type). - /// - /// \param IsCorrectedToColon If not null, suggestions to replace '::' -> ':' - /// are allowed. The bool value pointed by this parameter is set to 'true' - /// if the identifier is treated as if it was followed by ':', not '::'. - /// - /// \param OnlyNamespace If true, only considers namespaces in lookup. - /// - /// \returns true if an error occurred, false otherwise. - bool ActOnCXXNestedNameSpecifier(Scope *S, - NestedNameSpecInfo &IdInfo, - bool EnteringContext, - CXXScopeSpec &SS, - bool *IsCorrectedToColon = nullptr, - bool OnlyNamespace = false); + /// IncompatiblePointerSign - The assignment is between two pointers types + /// which point to integers which have a different sign, but are otherwise + /// identical. This is a subset of the above, but broken out because it's by + /// far the most common case of incompatible pointers. + IncompatiblePointerSign, - ExprResult ActOnDecltypeExpression(Expr *E); - - bool ActOnCXXNestedNameSpecifierDecltype(CXXScopeSpec &SS, - const DeclSpec &DS, - SourceLocation ColonColonLoc); + /// CompatiblePointerDiscardsQualifiers - The assignment discards + /// c/v/r qualifiers, which we accept as an extension. + CompatiblePointerDiscardsQualifiers, - bool ActOnCXXNestedNameSpecifierIndexedPack(CXXScopeSpec &SS, - const DeclSpec &DS, - SourceLocation ColonColonLoc, - QualType Type); + /// IncompatiblePointerDiscardsQualifiers - The assignment + /// discards qualifiers that we don't permit to be discarded, + /// like address spaces. + IncompatiblePointerDiscardsQualifiers, - bool IsInvalidUnlessNestedName(Scope *S, CXXScopeSpec &SS, - NestedNameSpecInfo &IdInfo, - bool EnteringContext); + /// IncompatibleNestedPointerAddressSpaceMismatch - The assignment + /// changes address spaces in nested pointer types which is not allowed. + /// For instance, converting __private int ** to __generic int ** is + /// illegal even though __private could be converted to __generic. + IncompatibleNestedPointerAddressSpaceMismatch, - bool IsInvalidSMECallConversion(QualType FromType, QualType ToType); + /// IncompatibleNestedPointerQualifiers - The assignment is between two + /// nested pointer types, and the qualifiers other than the first two + /// levels differ e.g. char ** -> const char **, but we accept them as an + /// extension. + IncompatibleNestedPointerQualifiers, - /// The parser has parsed a nested-name-specifier - /// 'template[opt] template-name < template-args >::'. - /// - /// \param S The scope in which this nested-name-specifier occurs. - /// - /// \param SS The nested-name-specifier, which is both an input - /// parameter (the nested-name-specifier before this type) and an - /// output parameter (containing the full nested-name-specifier, - /// including this new type). - /// - /// \param TemplateKWLoc the location of the 'template' keyword, if any. - /// \param TemplateName the template name. - /// \param TemplateNameLoc The location of the template name. - /// \param LAngleLoc The location of the opening angle bracket ('<'). - /// \param TemplateArgs The template arguments. - /// \param RAngleLoc The location of the closing angle bracket ('>'). - /// \param CCLoc The location of the '::'. - /// - /// \param EnteringContext Whether we're entering the context of the - /// nested-name-specifier. - /// - /// - /// \returns true if an error occurred, false otherwise. - bool ActOnCXXNestedNameSpecifier(Scope *S, - CXXScopeSpec &SS, - SourceLocation TemplateKWLoc, - TemplateTy TemplateName, - SourceLocation TemplateNameLoc, - SourceLocation LAngleLoc, - ASTTemplateArgsPtr TemplateArgs, - SourceLocation RAngleLoc, - SourceLocation CCLoc, - bool EnteringContext); + /// IncompatibleVectors - The assignment is between two vector types that + /// have the same size, which we accept as an extension. + IncompatibleVectors, - /// Given a C++ nested-name-specifier, produce an annotation value - /// that the parser can use later to reconstruct the given - /// nested-name-specifier. - /// - /// \param SS A nested-name-specifier. - /// - /// \returns A pointer containing all of the information in the - /// nested-name-specifier \p SS. - void *SaveNestedNameSpecifierAnnotation(CXXScopeSpec &SS); + /// IntToBlockPointer - The assignment converts an int to a block + /// pointer. We disallow this. + IntToBlockPointer, - /// Given an annotation pointer for a nested-name-specifier, restore - /// the nested-name-specifier structure. - /// - /// \param Annotation The annotation pointer, produced by - /// \c SaveNestedNameSpecifierAnnotation(). - /// - /// \param AnnotationRange The source range corresponding to the annotation. - /// - /// \param SS The nested-name-specifier that will be updated with the contents - /// of the annotation pointer. - void RestoreNestedNameSpecifierAnnotation(void *Annotation, - SourceRange AnnotationRange, - CXXScopeSpec &SS); + /// IncompatibleBlockPointer - The assignment is between two block + /// pointers types that are not compatible. + IncompatibleBlockPointer, - bool ShouldEnterDeclaratorScope(Scope *S, const CXXScopeSpec &SS); + /// IncompatibleObjCQualifiedId - The assignment is between a qualified + /// id type and something else (that is incompatible with it). For example, + /// "id " = "Foo *", where "Foo *" doesn't implement the XXX protocol. + IncompatibleObjCQualifiedId, - /// ActOnCXXEnterDeclaratorScope - Called when a C++ scope specifier (global - /// scope or nested-name-specifier) is parsed, part of a declarator-id. - /// After this method is called, according to [C++ 3.4.3p3], names should be - /// looked up in the declarator-id's scope, until the declarator is parsed and - /// ActOnCXXExitDeclaratorScope is called. - /// The 'SS' should be a non-empty valid CXXScopeSpec. - bool ActOnCXXEnterDeclaratorScope(Scope *S, CXXScopeSpec &SS); + /// IncompatibleObjCWeakRef - Assigning a weak-unavailable object to an + /// object with __weak qualifier. + IncompatibleObjCWeakRef, - /// ActOnCXXExitDeclaratorScope - Called when a declarator that previously - /// invoked ActOnCXXEnterDeclaratorScope(), is finished. 'SS' is the same - /// CXXScopeSpec that was passed to ActOnCXXEnterDeclaratorScope as well. - /// Used to indicate that names should revert to being looked up in the - /// defining scope. - void ActOnCXXExitDeclaratorScope(Scope *S, const CXXScopeSpec &SS); + /// Incompatible - We reject this conversion outright, it is invalid to + /// represent it in the AST. + Incompatible + }; - /// ActOnCXXEnterDeclInitializer - Invoked when we are about to parse an - /// initializer for the declaration 'Dcl'. - /// After this method is called, according to [C++ 3.4.1p13], if 'Dcl' is a - /// static data member of class X, names should be looked up in the scope of - /// class X. - void ActOnCXXEnterDeclInitializer(Scope *S, Decl *Dcl); + /// DiagnoseAssignmentResult - Emit a diagnostic, if required, for the + /// assignment conversion type specified by ConvTy. This returns true if the + /// conversion was invalid or false if the conversion was accepted. + bool DiagnoseAssignmentResult(AssignConvertType ConvTy, SourceLocation Loc, + QualType DstType, QualType SrcType, + Expr *SrcExpr, AssignmentAction Action, + bool *Complained = nullptr); - /// ActOnCXXExitDeclInitializer - Invoked after we are finished parsing an - /// initializer for the declaration 'Dcl'. - void ActOnCXXExitDeclInitializer(Scope *S, Decl *Dcl); + /// CheckAssignmentConstraints - Perform type checking for assignment, + /// argument passing, variable initialization, and function return values. + /// C99 6.5.16. + AssignConvertType CheckAssignmentConstraints(SourceLocation Loc, + QualType LHSType, + QualType RHSType); - /// Create a new lambda closure type. - CXXRecordDecl *createLambdaClosureType(SourceRange IntroducerRange, - TypeSourceInfo *Info, - unsigned LambdaDependencyKind, - LambdaCaptureDefault CaptureDefault); + /// Check assignment constraints and optionally prepare for a conversion of + /// the RHS to the LHS type. The conversion is prepared for if ConvertRHS + /// is true. + AssignConvertType CheckAssignmentConstraints(QualType LHSType, + ExprResult &RHS, CastKind &Kind, + bool ConvertRHS = true); - /// Number lambda for linkage purposes if necessary. - void handleLambdaNumbering(CXXRecordDecl *Class, CXXMethodDecl *Method, - std::optional - NumberingOverride = std::nullopt); + /// Check assignment constraints for an assignment of RHS to LHSType. + /// + /// \param LHSType The destination type for the assignment. + /// \param RHS The source expression for the assignment. + /// \param Diagnose If \c true, diagnostics may be produced when checking + /// for assignability. If a diagnostic is produced, \p RHS will be + /// set to ExprError(). Note that this function may still return + /// without producing a diagnostic, even for an invalid assignment. + /// \param DiagnoseCFAudited If \c true, the target is a function parameter + /// in an audited Core Foundation API and does not need to be checked + /// for ARC retain issues. + /// \param ConvertRHS If \c true, \p RHS will be updated to model the + /// conversions necessary to perform the assignment. If \c false, + /// \p Diagnose must also be \c false. + AssignConvertType CheckSingleAssignmentConstraints( + QualType LHSType, ExprResult &RHS, bool Diagnose = true, + bool DiagnoseCFAudited = false, bool ConvertRHS = true); - /// Endow the lambda scope info with the relevant properties. - void buildLambdaScope(sema::LambdaScopeInfo *LSI, CXXMethodDecl *CallOperator, - SourceRange IntroducerRange, - LambdaCaptureDefault CaptureDefault, - SourceLocation CaptureDefaultLoc, bool ExplicitParams, - bool Mutable); + // If the lhs type is a transparent union, check whether we + // can initialize the transparent union with the given expression. + AssignConvertType CheckTransparentUnionArgumentConstraints(QualType ArgType, + ExprResult &RHS); - CXXMethodDecl *CreateLambdaCallOperator(SourceRange IntroducerRange, - CXXRecordDecl *Class); + /// the following "Check" methods will return a valid/converted QualType + /// or a null QualType (indicating an error diagnostic was issued). - void AddTemplateParametersToLambdaCallOperator( - CXXMethodDecl *CallOperator, CXXRecordDecl *Class, - TemplateParameterList *TemplateParams); + /// type checking binary operators (subroutines of CreateBuiltinBinOp). + QualType InvalidOperands(SourceLocation Loc, ExprResult &LHS, + ExprResult &RHS); + QualType InvalidLogicalVectorOperands(SourceLocation Loc, ExprResult &LHS, + ExprResult &RHS); - void CompleteLambdaCallOperator( - CXXMethodDecl *Method, SourceLocation LambdaLoc, - SourceLocation CallOperatorLoc, Expr *TrailingRequiresClause, - TypeSourceInfo *MethodTyInfo, ConstexprSpecKind ConstexprKind, - StorageClass SC, ArrayRef Params, - bool HasExplicitResultType); + QualType CheckMultiplyDivideOperands( // C99 6.5.5 + ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, bool IsCompAssign, + bool IsDivide); + QualType CheckRemainderOperands( // C99 6.5.5 + ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, + bool IsCompAssign = false); + QualType CheckAdditionOperands( // C99 6.5.6 + ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, + BinaryOperatorKind Opc, QualType *CompLHSTy = nullptr); + QualType CheckSubtractionOperands( // C99 6.5.6 + ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, + QualType *CompLHSTy = nullptr); + QualType CheckShiftOperands( // C99 6.5.7 + ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, + BinaryOperatorKind Opc, bool IsCompAssign = false); + void CheckPtrComparisonWithNullChar(ExprResult &E, ExprResult &NullE); + QualType CheckCompareOperands( // C99 6.5.8/9 + ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, + BinaryOperatorKind Opc); + QualType CheckBitwiseOperands( // C99 6.5.[10...12] + ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, + BinaryOperatorKind Opc); + QualType CheckLogicalOperands( // C99 6.5.[13,14] + ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, + BinaryOperatorKind Opc); + // CheckAssignmentOperands is used for both simple and compound assignment. + // For simple assignment, pass both expressions and a null converted type. + // For compound assignment, pass both expressions and the converted type. + QualType CheckAssignmentOperands( // C99 6.5.16.[1,2] + Expr *LHSExpr, ExprResult &RHS, SourceLocation Loc, QualType CompoundType, + BinaryOperatorKind Opc); - void DiagnoseInvalidExplicitObjectParameterInLambda(CXXMethodDecl *Method); + bool CheckConversionToObjCLiteral(QualType DstType, Expr *&SrcExpr, + bool Diagnose = true); - /// Perform initialization analysis of the init-capture and perform - /// any implicit conversions such as an lvalue-to-rvalue conversion if - /// not being used to initialize a reference. - ParsedType actOnLambdaInitCaptureInitialization( - SourceLocation Loc, bool ByRef, SourceLocation EllipsisLoc, - IdentifierInfo *Id, LambdaCaptureInitKind InitKind, Expr *&Init) { - return ParsedType::make(buildLambdaInitCaptureInitialization( - Loc, ByRef, EllipsisLoc, std::nullopt, Id, - InitKind != LambdaCaptureInitKind::CopyInit, Init)); + /// To be used for checking whether the arguments being passed to + /// function exceeds the number of parameters expected for it. + static bool TooManyArguments(size_t NumParams, size_t NumArgs, + bool PartialOverloading = false) { + // We check whether we're just after a comma in code-completion. + if (NumArgs > 0 && PartialOverloading) + return NumArgs + 1 > NumParams; // If so, we view as an extra argument. + return NumArgs > NumParams; } - QualType buildLambdaInitCaptureInitialization( - SourceLocation Loc, bool ByRef, SourceLocation EllipsisLoc, - std::optional NumExpansions, IdentifierInfo *Id, - bool DirectInit, Expr *&Init); - /// Create a dummy variable within the declcontext of the lambda's - /// call operator, for name lookup purposes for a lambda init capture. - /// - /// CodeGen handles emission of lambda captures, ignoring these dummy - /// variables appropriately. - VarDecl *createLambdaInitCaptureVarDecl( - SourceLocation Loc, QualType InitCaptureType, SourceLocation EllipsisLoc, - IdentifierInfo *Id, unsigned InitStyle, Expr *Init, DeclContext *DeclCtx); + /// Whether the AST is currently being rebuilt to correct immediate + /// invocations. Immediate invocation candidates and references to consteval + /// functions aren't tracked when this is set. + bool RebuildingImmediateInvocation = false; - /// Add an init-capture to a lambda scope. - void addInitCapture(sema::LambdaScopeInfo *LSI, VarDecl *Var, bool ByRef); + bool isAlwaysConstantEvaluatedContext() const { + const ExpressionEvaluationContextRecord &Ctx = currentEvaluationContext(); + return (Ctx.isConstantEvaluated() || isConstantEvaluatedOverride) && + !Ctx.InConditionallyConstantEvaluateContext; + } - /// Note that we have finished the explicit captures for the - /// given lambda. - void finishLambdaExplicitCaptures(sema::LambdaScopeInfo *LSI); + /// Determines whether we are currently in a context that + /// is not evaluated as per C++ [expr] p5. + bool isUnevaluatedContext() const { + return currentEvaluationContext().isUnevaluated(); + } - /// Deduce a block or lambda's return type based on the return - /// statements present in the body. - void deduceClosureReturnType(sema::CapturingScopeInfo &CSI); + bool isImmediateFunctionContext() const { + return currentEvaluationContext().isImmediateFunctionContext(); + } - /// Once the Lambdas capture are known, we can start to create the closure, - /// call operator method, and keep track of the captures. - /// We do the capture lookup here, but they are not actually captured until - /// after we know what the qualifiers of the call operator are. - void ActOnLambdaExpressionAfterIntroducer(LambdaIntroducer &Intro, - Scope *CurContext); + bool isInLifetimeExtendingContext() const { + assert(!ExprEvalContexts.empty() && + "Must be in an expression evaluation context"); + return ExprEvalContexts.back().InLifetimeExtendingContext; + } - /// This is called after parsing the explicit template parameter list - /// on a lambda (if it exists) in C++2a. - void ActOnLambdaExplicitTemplateParameterList(LambdaIntroducer &Intro, - SourceLocation LAngleLoc, - ArrayRef TParams, - SourceLocation RAngleLoc, - ExprResult RequiresClause); + bool isCheckingDefaultArgumentOrInitializer() const { + const ExpressionEvaluationContextRecord &Ctx = currentEvaluationContext(); + return (Ctx.Context == + ExpressionEvaluationContext::PotentiallyEvaluatedIfUsed) || + Ctx.IsCurrentlyCheckingDefaultArgumentOrInitializer; + } - void ActOnLambdaClosureQualifiers(LambdaIntroducer &Intro, - SourceLocation MutableLoc); + std::optional + InnermostDeclarationWithDelayedImmediateInvocations() const { + assert(!ExprEvalContexts.empty() && + "Must be in an expression evaluation context"); + for (const auto &Ctx : llvm::reverse(ExprEvalContexts)) { + if (Ctx.Context == ExpressionEvaluationContext::PotentiallyEvaluated && + Ctx.DelayedDefaultInitializationContext) + return Ctx.DelayedDefaultInitializationContext; + if (Ctx.isConstantEvaluated() || Ctx.isImmediateFunctionContext() || + Ctx.isUnevaluated()) + break; + } + return std::nullopt; + } - void ActOnLambdaClosureParameters( - Scope *LambdaScope, - MutableArrayRef ParamInfo); + std::optional + OutermostDeclarationWithDelayedImmediateInvocations() const { + assert(!ExprEvalContexts.empty() && + "Must be in an expression evaluation context"); + std::optional Res; + for (auto &Ctx : llvm::reverse(ExprEvalContexts)) { + if (Ctx.Context == ExpressionEvaluationContext::PotentiallyEvaluated && + !Ctx.DelayedDefaultInitializationContext && Res) + break; + if (Ctx.isConstantEvaluated() || Ctx.isImmediateFunctionContext() || + Ctx.isUnevaluated()) + break; + Res = Ctx.DelayedDefaultInitializationContext; + } + return Res; + } - /// ActOnStartOfLambdaDefinition - This is called just before we start - /// parsing the body of a lambda; it analyzes the explicit captures and - /// arguments, and sets up various data-structures for the body of the - /// lambda. - void ActOnStartOfLambdaDefinition(LambdaIntroducer &Intro, - Declarator &ParamInfo, const DeclSpec &DS); + /// keepInLifetimeExtendingContext - Pull down InLifetimeExtendingContext + /// flag from previous context. + void keepInLifetimeExtendingContext() { + if (ExprEvalContexts.size() > 2 && + ExprEvalContexts[ExprEvalContexts.size() - 2] + .InLifetimeExtendingContext) { + auto &LastRecord = ExprEvalContexts.back(); + auto &PrevRecord = ExprEvalContexts[ExprEvalContexts.size() - 2]; + LastRecord.InLifetimeExtendingContext = + PrevRecord.InLifetimeExtendingContext; + } + } - /// ActOnLambdaError - If there is an error parsing a lambda, this callback - /// is invoked to pop the information about the lambda. - void ActOnLambdaError(SourceLocation StartLoc, Scope *CurScope, - bool IsInstantiation = false); + /// keepInMaterializeTemporaryObjectContext - Pull down + /// InMaterializeTemporaryObjectContext flag from previous context. + void keepInMaterializeTemporaryObjectContext() { + if (ExprEvalContexts.size() > 2 && + ExprEvalContexts[ExprEvalContexts.size() - 2] + .InMaterializeTemporaryObjectContext) { + auto &LastRecord = ExprEvalContexts.back(); + auto &PrevRecord = ExprEvalContexts[ExprEvalContexts.size() - 2]; + LastRecord.InMaterializeTemporaryObjectContext = + PrevRecord.InMaterializeTemporaryObjectContext; + } + } - /// ActOnLambdaExpr - This is called when the body of a lambda expression - /// was successfully completed. - ExprResult ActOnLambdaExpr(SourceLocation StartLoc, Stmt *Body); + DefaultedComparisonKind getDefaultedComparisonKind(const FunctionDecl *FD) { + return getDefaultedFunctionKind(FD).asComparison(); + } - /// Does copying/destroying the captured variable have side effects? - bool CaptureHasSideEffects(const sema::Capture &From); + /// Returns a field in a CXXRecordDecl that has the same name as the decl \p + /// SelfAssigned when inside a CXXMethodDecl. + const FieldDecl * + getSelfAssignmentClassMemberCandidate(const ValueDecl *SelfAssigned); - /// Diagnose if an explicit lambda capture is unused. Returns true if a - /// diagnostic is emitted. - bool DiagnoseUnusedLambdaCapture(SourceRange CaptureRange, - const sema::Capture &From); + void MaybeSuggestAddingStaticToDecl(const FunctionDecl *D); - /// Build a FieldDecl suitable to hold the given capture. - FieldDecl *BuildCaptureField(RecordDecl *RD, const sema::Capture &Capture); + template + bool RequireCompleteSizedType(SourceLocation Loc, QualType T, unsigned DiagID, + const Ts &...Args) { + SizelessTypeDiagnoser Diagnoser(DiagID, Args...); + return RequireCompleteType(Loc, T, CompleteTypeKind::Normal, Diagnoser); + } - /// Initialize the given capture with a suitable expression. - ExprResult BuildCaptureInit(const sema::Capture &Capture, - SourceLocation ImplicitCaptureLoc, - bool IsOpenMPMapping = false); + template + bool RequireCompleteSizedExprType(Expr *E, unsigned DiagID, + const Ts &...Args) { + SizelessTypeDiagnoser Diagnoser(DiagID, Args...); + return RequireCompleteExprType(E, CompleteTypeKind::Normal, Diagnoser); + } - /// Complete a lambda-expression having processed and attached the - /// lambda body. - ExprResult BuildLambdaExpr(SourceLocation StartLoc, SourceLocation EndLoc, - sema::LambdaScopeInfo *LSI); + /// Abstract class used to diagnose incomplete types. + struct TypeDiagnoser { + TypeDiagnoser() {} - /// Get the return type to use for a lambda's conversion function(s) to - /// function pointer type, given the type of the call operator. - QualType - getLambdaConversionFunctionResultType(const FunctionProtoType *CallOpType, - CallingConv CC); + virtual void diagnose(Sema &S, SourceLocation Loc, QualType T) = 0; + virtual ~TypeDiagnoser() {} + }; - /// Define the "body" of the conversion from a lambda object to a - /// function pointer. - /// - /// This routine doesn't actually define a sensible body; rather, it fills - /// in the initialization expression needed to copy the lambda object into - /// the block, and IR generation actually generates the real body of the - /// block pointer conversion. - void DefineImplicitLambdaToFunctionPointerConversion( - SourceLocation CurrentLoc, CXXConversionDecl *Conv); + template class BoundTypeDiagnoser : public TypeDiagnoser { + protected: + unsigned DiagID; + std::tuple Args; - /// Define the "body" of the conversion from a lambda object to a - /// block pointer. - /// - /// This routine doesn't actually define a sensible body; rather, it fills - /// in the initialization expression needed to copy the lambda object into - /// the block, and IR generation actually generates the real body of the - /// block pointer conversion. - void DefineImplicitLambdaToBlockPointerConversion(SourceLocation CurrentLoc, - CXXConversionDecl *Conv); + template + void emit(const SemaDiagnosticBuilder &DB, + std::index_sequence) const { + // Apply all tuple elements to the builder in order. + bool Dummy[] = {false, (DB << getPrintable(std::get(Args)))...}; + (void)Dummy; + } - ExprResult BuildBlockForLambdaConversion(SourceLocation CurrentLocation, - SourceLocation ConvLocation, - CXXConversionDecl *Conv, - Expr *Src); + public: + BoundTypeDiagnoser(unsigned DiagID, const Ts &...Args) + : TypeDiagnoser(), DiagID(DiagID), Args(Args...) { + assert(DiagID != 0 && "no diagnostic for type diagnoser"); + } - sema::LambdaScopeInfo *RebuildLambdaScopeInfo(CXXMethodDecl *CallOperator); + void diagnose(Sema &S, SourceLocation Loc, QualType T) override { + const SemaDiagnosticBuilder &DB = S.Diag(Loc, DiagID); + emit(DB, std::index_sequence_for()); + DB << T; + } + }; - class LambdaScopeForCallOperatorInstantiationRAII - : private FunctionScopeRAII { + /// A derivative of BoundTypeDiagnoser for which the diagnostic's type + /// parameter is preceded by a 0/1 enum that is 1 if the type is sizeless. + /// For example, a diagnostic with no other parameters would generally have + /// the form "...%select{incomplete|sizeless}0 type %1...". + template + class SizelessTypeDiagnoser : public BoundTypeDiagnoser { public: - LambdaScopeForCallOperatorInstantiationRAII( - Sema &SemasRef, FunctionDecl *FD, MultiLevelTemplateArgumentList MLTAL, - LocalInstantiationScope &Scope, - bool ShouldAddDeclsFromParentScope = true); + SizelessTypeDiagnoser(unsigned DiagID, const Ts &...Args) + : BoundTypeDiagnoser(DiagID, Args...) {} + + void diagnose(Sema &S, SourceLocation Loc, QualType T) override { + const SemaDiagnosticBuilder &DB = S.Diag(Loc, this->DiagID); + this->emit(DB, std::index_sequence_for()); + DB << T->isSizelessType() << T; + } }; - /// Check whether the given expression is a valid constraint expression. - /// A diagnostic is emitted if it is not, false is returned, and - /// PossibleNonPrimary will be set to true if the failure might be due to a - /// non-primary expression being used as an atomic constraint. - bool CheckConstraintExpression(const Expr *CE, Token NextToken = Token(), - bool *PossibleNonPrimary = nullptr, - bool IsTrailingRequiresClause = false); + /// Check an argument list for placeholders that we won't try to + /// handle later. + bool CheckArgsForPlaceholders(MultiExprArg args); -private: - /// Caches pairs of template-like decls whose associated constraints were - /// checked for subsumption and whether or not the first's constraints did in - /// fact subsume the second's. - llvm::DenseMap, bool> SubsumptionCache; - /// Caches the normalized associated constraints of declarations (concepts or - /// constrained declarations). If an error occurred while normalizing the - /// associated constraints of the template or concept, nullptr will be cached - /// here. - llvm::DenseMap - NormalizationCache; + /// The C++ "std::source_location::__impl" struct, defined in + /// \. + RecordDecl *StdSourceLocationImplDecl; - llvm::ContextualFoldingSet - SatisfactionCache; + /// A stack of expression evaluation contexts. + SmallVector ExprEvalContexts; - /// Introduce the instantiated local variables into the local - /// instantiation scope. - void addInstantiatedLocalVarsToScope(FunctionDecl *Function, - const FunctionDecl *PatternDecl, - LocalInstantiationScope &Scope); - /// Introduce the instantiated function parameters into the local - /// instantiation scope, and set the parameter names to those used - /// in the template. - bool addInstantiatedParametersToScope( - FunctionDecl *Function, const FunctionDecl *PatternDecl, - LocalInstantiationScope &Scope, - const MultiLevelTemplateArgumentList &TemplateArgs); + // Set of failed immediate invocations to avoid double diagnosing. + llvm::SmallPtrSet FailedImmediateInvocations; - /// Introduce the instantiated captures of the lambda into the local - /// instantiation scope. - bool addInstantiatedCapturesToScope( - FunctionDecl *Function, const FunctionDecl *PatternDecl, - LocalInstantiationScope &Scope, - const MultiLevelTemplateArgumentList &TemplateArgs); - - /// used by SetupConstraintCheckingTemplateArgumentsAndScope to recursively(in - /// the case of lambdas) set up the LocalInstantiationScope of the current - /// function. - bool SetupConstraintScope( - FunctionDecl *FD, std::optional> TemplateArgs, - MultiLevelTemplateArgumentList MLTAL, LocalInstantiationScope &Scope); - - /// Used during constraint checking, sets up the constraint template argument - /// lists, and calls SetupConstraintScope to set up the - /// LocalInstantiationScope to have the proper set of ParVarDecls configured. - std::optional - SetupConstraintCheckingTemplateArgumentsAndScope( - FunctionDecl *FD, std::optional> TemplateArgs, - LocalInstantiationScope &Scope); + /// List of SourceLocations where 'self' is implicitly retained inside a + /// block. + llvm::SmallVector, 1> + ImplicitlyRetainedSelfLocs; private: - // The current stack of constraint satisfactions, so we can exit-early. - using SatisfactionStackEntryTy = - std::pair; - llvm::SmallVector - SatisfactionStack; + static BinaryOperatorKind ConvertTokenKindToBinaryOpcode(tok::TokenKind Kind); -public: - void PushSatisfactionStackEntry(const NamedDecl *D, - const llvm::FoldingSetNodeID &ID) { - const NamedDecl *Can = cast(D->getCanonicalDecl()); - SatisfactionStack.emplace_back(Can, ID); - } + /// Methods for marking which expressions involve dereferencing a pointer + /// marked with the 'noderef' attribute. Expressions are checked bottom up as + /// they are parsed, meaning that a noderef pointer may not be accessed. For + /// example, in `&*p` where `p` is a noderef pointer, we will first parse the + /// `*p`, but need to check that `address of` is called on it. This requires + /// keeping a container of all pending expressions and checking if the address + /// of them are eventually taken. + void CheckSubscriptAccessOfNoDeref(const ArraySubscriptExpr *E); + void CheckAddressOfNoDeref(const Expr *E); - void PopSatisfactionStackEntry() { SatisfactionStack.pop_back(); } + ///@} - bool SatisfactionStackContains(const NamedDecl *D, - const llvm::FoldingSetNodeID &ID) const { - const NamedDecl *Can = cast(D->getCanonicalDecl()); - return llvm::find(SatisfactionStack, - SatisfactionStackEntryTy{Can, ID}) != - SatisfactionStack.end(); - } + // + // + // ------------------------------------------------------------------------- + // + // - // Resets the current SatisfactionStack for cases where we are instantiating - // constraints as a 'side effect' of normal instantiation in a way that is not - // indicative of recursive definition. - class SatisfactionStackResetRAII { - llvm::SmallVector - BackupSatisfactionStack; - Sema &SemaRef; + /// \name C++ Expressions + /// Implementations are in SemaExprCXX.cpp + ///@{ - public: - SatisfactionStackResetRAII(Sema &S) : SemaRef(S) { - SemaRef.SwapSatisfactionStack(BackupSatisfactionStack); - } +public: + /// The C++ "std::bad_alloc" class, which is defined by the C++ + /// standard library. + LazyDeclPtr StdBadAlloc; - ~SatisfactionStackResetRAII() { - SemaRef.SwapSatisfactionStack(BackupSatisfactionStack); - } - }; + /// The C++ "std::align_val_t" enum class, which is defined by the C++ + /// standard library. + LazyDeclPtr StdAlignValT; - void SwapSatisfactionStack( - llvm::SmallVectorImpl &NewSS) { - SatisfactionStack.swap(NewSS); - } + /// The C++ "type_info" declaration, which is defined in \. + RecordDecl *CXXTypeInfoDecl; - const NormalizedConstraint * - getNormalizedAssociatedConstraints( - NamedDecl *ConstrainedDecl, ArrayRef AssociatedConstraints); + /// A flag to remember whether the implicit forms of operator new and delete + /// have been declared. + bool GlobalNewDeleteDeclared; - /// \brief Check whether the given declaration's associated constraints are - /// at least as constrained than another declaration's according to the - /// partial ordering of constraints. - /// - /// \param Result If no error occurred, receives the result of true if D1 is - /// at least constrained than D2, and false otherwise. + /// Delete-expressions to be analyzed at the end of translation unit /// - /// \returns true if an error occurred, false otherwise. - bool IsAtLeastAsConstrained(NamedDecl *D1, MutableArrayRef AC1, - NamedDecl *D2, MutableArrayRef AC2, - bool &Result); - - /// If D1 was not at least as constrained as D2, but would've been if a pair - /// of atomic constraints involved had been declared in a concept and not - /// repeated in two separate places in code. - /// \returns true if such a diagnostic was emitted, false otherwise. - bool MaybeEmitAmbiguousAtomicConstraintsDiagnostic(NamedDecl *D1, - ArrayRef AC1, NamedDecl *D2, ArrayRef AC2); + /// This list contains class members, and locations of delete-expressions + /// that could not be proven as to whether they mismatch with new-expression + /// used in initializer of the field. + llvm::MapVector DeleteExprs; - /// \brief Check whether the given list of constraint expressions are - /// satisfied (as if in a 'conjunction') given template arguments. - /// \param Template the template-like entity that triggered the constraints - /// check (either a concept or a constrained entity). - /// \param ConstraintExprs a list of constraint expressions, treated as if - /// they were 'AND'ed together. - /// \param TemplateArgLists the list of template arguments to substitute into - /// the constraint expression. - /// \param TemplateIDRange The source range of the template id that - /// caused the constraints check. - /// \param Satisfaction if true is returned, will contain details of the - /// satisfaction, with enough information to diagnose an unsatisfied - /// expression. - /// \returns true if an error occurred and satisfaction could not be checked, - /// false otherwise. - bool CheckConstraintSatisfaction( - const NamedDecl *Template, ArrayRef ConstraintExprs, - const MultiLevelTemplateArgumentList &TemplateArgLists, - SourceRange TemplateIDRange, ConstraintSatisfaction &Satisfaction) { - llvm::SmallVector Converted; - return CheckConstraintSatisfaction(Template, ConstraintExprs, Converted, - TemplateArgLists, TemplateIDRange, - Satisfaction); + bool isInMaterializeTemporaryObjectContext() const { + assert(!ExprEvalContexts.empty() && + "Must be in an expression evaluation context"); + return ExprEvalContexts.back().InMaterializeTemporaryObjectContext; } - /// \brief Check whether the given list of constraint expressions are - /// satisfied (as if in a 'conjunction') given template arguments. - /// Additionally, takes an empty list of Expressions which is populated with - /// the instantiated versions of the ConstraintExprs. - /// \param Template the template-like entity that triggered the constraints - /// check (either a concept or a constrained entity). - /// \param ConstraintExprs a list of constraint expressions, treated as if - /// they were 'AND'ed together. - /// \param ConvertedConstraints a out parameter that will get populated with - /// the instantiated version of the ConstraintExprs if we successfully checked - /// satisfaction. - /// \param TemplateArgList the multi-level list of template arguments to - /// substitute into the constraint expression. This should be relative to the - /// top-level (hence multi-level), since we need to instantiate fully at the - /// time of checking. - /// \param TemplateIDRange The source range of the template id that - /// caused the constraints check. - /// \param Satisfaction if true is returned, will contain details of the - /// satisfaction, with enough information to diagnose an unsatisfied - /// expression. - /// \returns true if an error occurred and satisfaction could not be checked, - /// false otherwise. - bool CheckConstraintSatisfaction( - const NamedDecl *Template, ArrayRef ConstraintExprs, - llvm::SmallVectorImpl &ConvertedConstraints, - const MultiLevelTemplateArgumentList &TemplateArgList, - SourceRange TemplateIDRange, ConstraintSatisfaction &Satisfaction); - - /// \brief Check whether the given non-dependent constraint expression is - /// satisfied. Returns false and updates Satisfaction with the satisfaction - /// verdict if successful, emits a diagnostic and returns true if an error - /// occurred and satisfaction could not be determined. - /// - /// \returns true if an error occurred, false otherwise. - bool CheckConstraintSatisfaction(const Expr *ConstraintExpr, - ConstraintSatisfaction &Satisfaction); + ParsedType getInheritingConstructorName(CXXScopeSpec &SS, + SourceLocation NameLoc, + IdentifierInfo &Name); - /// Check whether the given function decl's trailing requires clause is - /// satisfied, if any. Returns false and updates Satisfaction with the - /// satisfaction verdict if successful, emits a diagnostic and returns true if - /// an error occurred and satisfaction could not be determined. - /// - /// \returns true if an error occurred, false otherwise. - bool CheckFunctionConstraints(const FunctionDecl *FD, - ConstraintSatisfaction &Satisfaction, - SourceLocation UsageLoc = SourceLocation(), - bool ForOverloadResolution = false); + ParsedType getConstructorName(IdentifierInfo &II, SourceLocation NameLoc, + Scope *S, CXXScopeSpec &SS, + bool EnteringContext); + ParsedType getDestructorName(IdentifierInfo &II, SourceLocation NameLoc, + Scope *S, CXXScopeSpec &SS, + ParsedType ObjectType, bool EnteringContext); - /// \brief Ensure that the given template arguments satisfy the constraints - /// associated with the given template, emitting a diagnostic if they do not. - /// - /// \param Template The template to which the template arguments are being - /// provided. - /// - /// \param TemplateArgs The converted, canonicalized template arguments. - /// - /// \param TemplateIDRange The source range of the template id that - /// caused the constraints check. - /// - /// \returns true if the constrains are not satisfied or could not be checked - /// for satisfaction, false if the constraints are satisfied. - bool EnsureTemplateArgumentListConstraints( - TemplateDecl *Template, - const MultiLevelTemplateArgumentList &TemplateArgs, - SourceRange TemplateIDRange); + ParsedType getDestructorTypeForDecltype(const DeclSpec &DS, + ParsedType ObjectType); - /// \brief Emit diagnostics explaining why a constraint expression was deemed - /// unsatisfied. - /// \param First whether this is the first time an unsatisfied constraint is - /// diagnosed for this error. - void - DiagnoseUnsatisfiedConstraint(const ConstraintSatisfaction &Satisfaction, - bool First = true); + ExprResult BuildCXXTypeId(QualType TypeInfoType, SourceLocation TypeidLoc, + TypeSourceInfo *Operand, SourceLocation RParenLoc); + ExprResult BuildCXXTypeId(QualType TypeInfoType, SourceLocation TypeidLoc, + Expr *Operand, SourceLocation RParenLoc); - /// \brief Emit diagnostics explaining why a constraint expression was deemed - /// unsatisfied. - void - DiagnoseUnsatisfiedConstraint(const ASTConstraintSatisfaction &Satisfaction, - bool First = true); + /// ActOnCXXTypeid - Parse typeid( something ). + ExprResult ActOnCXXTypeid(SourceLocation OpLoc, SourceLocation LParenLoc, + bool isType, void *TyOrExpr, + SourceLocation RParenLoc); - // ParseObjCStringLiteral - Parse Objective-C string literals. - ExprResult ParseObjCStringLiteral(SourceLocation *AtLocs, - ArrayRef Strings); + ExprResult BuildCXXUuidof(QualType TypeInfoType, SourceLocation TypeidLoc, + TypeSourceInfo *Operand, SourceLocation RParenLoc); + ExprResult BuildCXXUuidof(QualType TypeInfoType, SourceLocation TypeidLoc, + Expr *Operand, SourceLocation RParenLoc); - ExprResult BuildObjCStringLiteral(SourceLocation AtLoc, StringLiteral *S); + /// ActOnCXXUuidof - Parse __uuidof( something ). + ExprResult ActOnCXXUuidof(SourceLocation OpLoc, SourceLocation LParenLoc, + bool isType, void *TyOrExpr, + SourceLocation RParenLoc); - /// BuildObjCNumericLiteral - builds an ObjCBoxedExpr AST node for the - /// numeric literal expression. Type of the expression will be "NSNumber *" - /// or "id" if NSNumber is unavailable. - ExprResult BuildObjCNumericLiteral(SourceLocation AtLoc, Expr *Number); - ExprResult ActOnObjCBoolLiteral(SourceLocation AtLoc, SourceLocation ValueLoc, - bool Value); - ExprResult BuildObjCArrayLiteral(SourceRange SR, MultiExprArg Elements); + //// ActOnCXXThis - Parse 'this' pointer. + ExprResult ActOnCXXThis(SourceLocation loc); - /// BuildObjCBoxedExpr - builds an ObjCBoxedExpr AST node for the - /// '@' prefixed parenthesized expression. The type of the expression will - /// either be "NSNumber *", "NSString *" or "NSValue *" depending on the type - /// of ValueType, which is allowed to be a built-in numeric type, "char *", - /// "const char *" or C structure with attribute 'objc_boxable'. - ExprResult BuildObjCBoxedExpr(SourceRange SR, Expr *ValueExpr); + /// Build a CXXThisExpr and mark it referenced in the current context. + Expr *BuildCXXThisExpr(SourceLocation Loc, QualType Type, bool IsImplicit); + void MarkThisReferenced(CXXThisExpr *This); - ExprResult BuildObjCSubscriptExpression(SourceLocation RB, Expr *BaseExpr, - Expr *IndexExpr, - ObjCMethodDecl *getterMethod, - ObjCMethodDecl *setterMethod); + /// Try to retrieve the type of the 'this' pointer. + /// + /// \returns The type of 'this', if possible. Otherwise, returns a NULL type. + QualType getCurrentThisType(); - ExprResult BuildObjCDictionaryLiteral(SourceRange SR, - MutableArrayRef Elements); + /// When non-NULL, the C++ 'this' expression is allowed despite the + /// current context not being a non-static member function. In such cases, + /// this provides the type used for 'this'. + QualType CXXThisTypeOverride; - ExprResult BuildObjCEncodeExpression(SourceLocation AtLoc, - TypeSourceInfo *EncodedTypeInfo, - SourceLocation RParenLoc); - ExprResult BuildCXXMemberCallExpr(Expr *Exp, NamedDecl *FoundDecl, - CXXConversionDecl *Method, - bool HadMultipleCandidates); + /// RAII object used to temporarily allow the C++ 'this' expression + /// to be used, with the given qualifiers on the current class type. + class CXXThisScopeRAII { + Sema &S; + QualType OldCXXThisTypeOverride; + bool Enabled; - ExprResult ParseObjCEncodeExpression(SourceLocation AtLoc, - SourceLocation EncodeLoc, - SourceLocation LParenLoc, - ParsedType Ty, - SourceLocation RParenLoc); + public: + /// Introduce a new scope where 'this' may be allowed (when enabled), + /// using the given declaration (which is either a class template or a + /// class) along with the given qualifiers. + /// along with the qualifiers placed on '*this'. + CXXThisScopeRAII(Sema &S, Decl *ContextDecl, Qualifiers CXXThisTypeQuals, + bool Enabled = true); - /// ParseObjCSelectorExpression - Build selector expression for \@selector - ExprResult ParseObjCSelectorExpression(Selector Sel, - SourceLocation AtLoc, - SourceLocation SelLoc, - SourceLocation LParenLoc, - SourceLocation RParenLoc, - bool WarnMultipleSelectors); + ~CXXThisScopeRAII(); + }; - /// ParseObjCProtocolExpression - Build protocol expression for \@protocol - ExprResult ParseObjCProtocolExpression(IdentifierInfo * ProtocolName, - SourceLocation AtLoc, - SourceLocation ProtoLoc, - SourceLocation LParenLoc, - SourceLocation ProtoIdLoc, - SourceLocation RParenLoc); + /// Make sure the value of 'this' is actually available in the current + /// context, if it is a potentially evaluated context. + /// + /// \param Loc The location at which the capture of 'this' occurs. + /// + /// \param Explicit Whether 'this' is explicitly captured in a lambda + /// capture list. + /// + /// \param FunctionScopeIndexToStopAt If non-null, it points to the index + /// of the FunctionScopeInfo stack beyond which we do not attempt to capture. + /// This is useful when enclosing lambdas must speculatively capture + /// 'this' that may or may not be used in certain specializations of + /// a nested generic lambda (depending on whether the name resolves to + /// a non-static member function or a static function). + /// \return returns 'true' if failed, 'false' if success. + bool CheckCXXThisCapture( + SourceLocation Loc, bool Explicit = false, bool BuildAndDiagnose = true, + const unsigned *const FunctionScopeIndexToStopAt = nullptr, + bool ByCopy = false); - //===--------------------------------------------------------------------===// - // C++ Declarations - // - Decl *ActOnStartLinkageSpecification(Scope *S, - SourceLocation ExternLoc, - Expr *LangStr, - SourceLocation LBraceLoc); - Decl *ActOnFinishLinkageSpecification(Scope *S, - Decl *LinkageSpec, - SourceLocation RBraceLoc); + /// Determine whether the given type is the type of *this that is used + /// outside of the body of a member function for a type that is currently + /// being defined. + bool isThisOutsideMemberFunctionBody(QualType BaseType); + /// ActOnCXXBoolLiteral - Parse {true,false} literals. + ExprResult ActOnCXXBoolLiteral(SourceLocation OpLoc, tok::TokenKind Kind); - //===--------------------------------------------------------------------===// - // C++ Classes - // - CXXRecordDecl *getCurrentClass(Scope *S, const CXXScopeSpec *SS); - bool isCurrentClassName(const IdentifierInfo &II, Scope *S, - const CXXScopeSpec *SS = nullptr); - bool isCurrentClassNameTypo(IdentifierInfo *&II, const CXXScopeSpec *SS); + /// ActOnObjCBoolLiteral - Parse {__objc_yes,__objc_no} literals. + ExprResult ActOnObjCBoolLiteral(SourceLocation OpLoc, tok::TokenKind Kind); - bool ActOnAccessSpecifier(AccessSpecifier Access, SourceLocation ASLoc, - SourceLocation ColonLoc, - const ParsedAttributesView &Attrs); + ExprResult + ActOnObjCAvailabilityCheckExpr(llvm::ArrayRef AvailSpecs, + SourceLocation AtLoc, SourceLocation RParen); - NamedDecl *ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, - Declarator &D, - MultiTemplateParamsArg TemplateParameterLists, - Expr *BitfieldWidth, const VirtSpecifiers &VS, - InClassInitStyle InitStyle); + /// ActOnCXXNullPtrLiteral - Parse 'nullptr'. + ExprResult ActOnCXXNullPtrLiteral(SourceLocation Loc); - void ActOnStartCXXInClassMemberInitializer(); - void ActOnFinishCXXInClassMemberInitializer(Decl *VarDecl, - SourceLocation EqualLoc, - Expr *Init); + //// ActOnCXXThrow - Parse throw expressions. + ExprResult ActOnCXXThrow(Scope *S, SourceLocation OpLoc, Expr *expr); + ExprResult BuildCXXThrow(SourceLocation OpLoc, Expr *Ex, + bool IsThrownVarInScope); + bool CheckCXXThrowOperand(SourceLocation ThrowLoc, QualType ThrowTy, Expr *E); - MemInitResult ActOnMemInitializer(Decl *ConstructorD, - Scope *S, - CXXScopeSpec &SS, - IdentifierInfo *MemberOrBase, - ParsedType TemplateTypeTy, - const DeclSpec &DS, - SourceLocation IdLoc, - SourceLocation LParenLoc, - ArrayRef Args, - SourceLocation RParenLoc, - SourceLocation EllipsisLoc); + /// ActOnCXXTypeConstructExpr - Parse construction of a specified type. + /// Can be interpreted either as function-style casting ("int(x)") + /// or class type construction ("ClassType(x,y,z)") + /// or creation of a value-initialized type ("int()"). + ExprResult ActOnCXXTypeConstructExpr(ParsedType TypeRep, + SourceLocation LParenOrBraceLoc, + MultiExprArg Exprs, + SourceLocation RParenOrBraceLoc, + bool ListInitialization); - MemInitResult ActOnMemInitializer(Decl *ConstructorD, - Scope *S, - CXXScopeSpec &SS, - IdentifierInfo *MemberOrBase, - ParsedType TemplateTypeTy, - const DeclSpec &DS, - SourceLocation IdLoc, - Expr *InitList, - SourceLocation EllipsisLoc); + ExprResult BuildCXXTypeConstructExpr(TypeSourceInfo *Type, + SourceLocation LParenLoc, + MultiExprArg Exprs, + SourceLocation RParenLoc, + bool ListInitialization); - MemInitResult BuildMemInitializer(Decl *ConstructorD, - Scope *S, - CXXScopeSpec &SS, - IdentifierInfo *MemberOrBase, - ParsedType TemplateTypeTy, - const DeclSpec &DS, - SourceLocation IdLoc, - Expr *Init, - SourceLocation EllipsisLoc); + /// ActOnCXXNew - Parsed a C++ 'new' expression. + ExprResult ActOnCXXNew(SourceLocation StartLoc, bool UseGlobal, + SourceLocation PlacementLParen, + MultiExprArg PlacementArgs, + SourceLocation PlacementRParen, + SourceRange TypeIdParens, Declarator &D, + Expr *Initializer); + ExprResult + BuildCXXNew(SourceRange Range, bool UseGlobal, SourceLocation PlacementLParen, + MultiExprArg PlacementArgs, SourceLocation PlacementRParen, + SourceRange TypeIdParens, QualType AllocType, + TypeSourceInfo *AllocTypeInfo, std::optional ArraySize, + SourceRange DirectInitRange, Expr *Initializer); - MemInitResult BuildMemberInitializer(ValueDecl *Member, - Expr *Init, - SourceLocation IdLoc); + /// Determine whether \p FD is an aligned allocation or deallocation + /// function that is unavailable. + bool isUnavailableAlignedAllocationFunction(const FunctionDecl &FD) const; - MemInitResult BuildBaseInitializer(QualType BaseType, - TypeSourceInfo *BaseTInfo, - Expr *Init, - CXXRecordDecl *ClassDecl, - SourceLocation EllipsisLoc); + /// Produce diagnostics if \p FD is an aligned allocation or deallocation + /// function that is unavailable. + void diagnoseUnavailableAlignedAllocation(const FunctionDecl &FD, + SourceLocation Loc); - MemInitResult BuildDelegatingInitializer(TypeSourceInfo *TInfo, - Expr *Init, - CXXRecordDecl *ClassDecl); + bool CheckAllocatedType(QualType AllocType, SourceLocation Loc, + SourceRange R); - bool SetDelegatingInitializer(CXXConstructorDecl *Constructor, - CXXCtorInitializer *Initializer); + /// The scope in which to find allocation functions. + enum AllocationFunctionScope { + /// Only look for allocation functions in the global scope. + AFS_Global, + /// Only look for allocation functions in the scope of the + /// allocated class. + AFS_Class, + /// Look for allocation functions in both the global scope + /// and in the scope of the allocated class. + AFS_Both + }; - bool SetCtorInitializers( - CXXConstructorDecl *Constructor, bool AnyErrors, - ArrayRef Initializers = std::nullopt); + /// Finds the overloads of operator new and delete that are appropriate + /// for the allocation. + bool FindAllocationFunctions(SourceLocation StartLoc, SourceRange Range, + AllocationFunctionScope NewScope, + AllocationFunctionScope DeleteScope, + QualType AllocType, bool IsArray, + bool &PassAlignment, MultiExprArg PlaceArgs, + FunctionDecl *&OperatorNew, + FunctionDecl *&OperatorDelete, + bool Diagnose = true); + void DeclareGlobalNewDelete(); + void DeclareGlobalAllocationFunction(DeclarationName Name, QualType Return, + ArrayRef Params); - void SetIvarInitializers(ObjCImplementationDecl *ObjCImplementation); + bool FindDeallocationFunction(SourceLocation StartLoc, CXXRecordDecl *RD, + DeclarationName Name, FunctionDecl *&Operator, + bool Diagnose = true, bool WantSize = false, + bool WantAligned = false); + FunctionDecl *FindUsualDeallocationFunction(SourceLocation StartLoc, + bool CanProvideSize, + bool Overaligned, + DeclarationName Name); + FunctionDecl *FindDeallocationFunctionForDestructor(SourceLocation StartLoc, + CXXRecordDecl *RD); + /// ActOnCXXDelete - Parsed a C++ 'delete' expression + ExprResult ActOnCXXDelete(SourceLocation StartLoc, bool UseGlobal, + bool ArrayForm, Expr *Operand); + void CheckVirtualDtorCall(CXXDestructorDecl *dtor, SourceLocation Loc, + bool IsDelete, bool CallCanBeVirtual, + bool WarnOnNonAbstractTypes, + SourceLocation DtorLoc); - /// MarkBaseAndMemberDestructorsReferenced - Given a record decl, - /// mark all the non-trivial destructors of its members and bases as - /// referenced. - void MarkBaseAndMemberDestructorsReferenced(SourceLocation Loc, - CXXRecordDecl *Record); + ExprResult ActOnNoexceptExpr(SourceLocation KeyLoc, SourceLocation LParen, + Expr *Operand, SourceLocation RParen); + ExprResult BuildCXXNoexceptExpr(SourceLocation KeyLoc, Expr *Operand, + SourceLocation RParen); - /// Mark destructors of virtual bases of this class referenced. In the Itanium - /// C++ ABI, this is done when emitting a destructor for any non-abstract - /// class. In the Microsoft C++ ABI, this is done any time a class's - /// destructor is referenced. - void MarkVirtualBaseDestructorsReferenced( - SourceLocation Location, CXXRecordDecl *ClassDecl, - llvm::SmallPtrSetImpl *DirectVirtualBases = nullptr); + ExprResult ActOnStartCXXMemberReference(Scope *S, Expr *Base, + SourceLocation OpLoc, + tok::TokenKind OpKind, + ParsedType &ObjectType, + bool &MayBePseudoDestructor); - /// Do semantic checks to allow the complete destructor variant to be emitted - /// when the destructor is defined in another translation unit. In the Itanium - /// C++ ABI, destructor variants are emitted together. In the MS C++ ABI, they - /// can be emitted in separate TUs. To emit the complete variant, run a subset - /// of the checks performed when emitting a regular destructor. - void CheckCompleteDestructorVariant(SourceLocation CurrentLocation, - CXXDestructorDecl *Dtor); + ExprResult BuildPseudoDestructorExpr( + Expr *Base, SourceLocation OpLoc, tok::TokenKind OpKind, + const CXXScopeSpec &SS, TypeSourceInfo *ScopeType, SourceLocation CCLoc, + SourceLocation TildeLoc, PseudoDestructorTypeStorage DestroyedType); - /// The list of classes whose vtables have been used within - /// this translation unit, and the source locations at which the - /// first use occurred. - typedef std::pair VTableUse; + ExprResult ActOnPseudoDestructorExpr( + Scope *S, Expr *Base, SourceLocation OpLoc, tok::TokenKind OpKind, + CXXScopeSpec &SS, UnqualifiedId &FirstTypeName, SourceLocation CCLoc, + SourceLocation TildeLoc, UnqualifiedId &SecondTypeName); - /// The list of vtables that are required but have not yet been - /// materialized. - SmallVector VTableUses; + ExprResult ActOnPseudoDestructorExpr(Scope *S, Expr *Base, + SourceLocation OpLoc, + tok::TokenKind OpKind, + SourceLocation TildeLoc, + const DeclSpec &DS); - /// The set of classes whose vtables have been used within - /// this translation unit, and a bit that will be true if the vtable is - /// required to be emitted (otherwise, it should be emitted only if needed - /// by code generation). - llvm::DenseMap VTablesUsed; + /// MaybeCreateExprWithCleanups - If the current full-expression + /// requires any cleanups, surround it with a ExprWithCleanups node. + /// Otherwise, just returns the passed-in expression. + Expr *MaybeCreateExprWithCleanups(Expr *SubExpr); + Stmt *MaybeCreateStmtWithCleanups(Stmt *SubStmt); + ExprResult MaybeCreateExprWithCleanups(ExprResult SubExpr); - /// Load any externally-stored vtable uses. - void LoadExternalVTableUses(); + ExprResult ActOnFinishFullExpr(Expr *Expr, bool DiscardedValue) { + return ActOnFinishFullExpr( + Expr, Expr ? Expr->getExprLoc() : SourceLocation(), DiscardedValue); + } + ExprResult ActOnFinishFullExpr(Expr *Expr, SourceLocation CC, + bool DiscardedValue, bool IsConstexpr = false, + bool IsTemplateArgument = false); + StmtResult ActOnFinishFullStmt(Stmt *Stmt); - /// Note that the vtable for the given class was used at the - /// given location. - void MarkVTableUsed(SourceLocation Loc, CXXRecordDecl *Class, - bool DefinitionRequired = false); + ExprResult ActOnDecltypeExpression(Expr *E); - /// Mark the exception specifications of all virtual member functions - /// in the given class as needed. - void MarkVirtualMemberExceptionSpecsNeeded(SourceLocation Loc, - const CXXRecordDecl *RD); + bool checkLiteralOperatorId(const CXXScopeSpec &SS, const UnqualifiedId &Id, + bool IsUDSuffix); - /// MarkVirtualMembersReferenced - Will mark all members of the given - /// CXXRecordDecl referenced. - void MarkVirtualMembersReferenced(SourceLocation Loc, const CXXRecordDecl *RD, - bool ConstexprOnly = false); + bool isUsualDeallocationFunction(const CXXMethodDecl *FD); - /// Define all of the vtables that have been used in this - /// translation unit and reference any virtual members used by those - /// vtables. - /// - /// \returns true if any work was done, false otherwise. - bool DefineUsedVTables(); + ConditionResult ActOnConditionVariable(Decl *ConditionVar, + SourceLocation StmtLoc, + ConditionKind CK); - void AddImplicitlyDeclaredMembersToClass(CXXRecordDecl *ClassDecl); + ExprResult CheckConditionVariable(VarDecl *ConditionVar, + SourceLocation StmtLoc, ConditionKind CK); - void ActOnMemInitializers(Decl *ConstructorDecl, - SourceLocation ColonLoc, - ArrayRef MemInits, - bool AnyErrors); + /// CheckCXXBooleanCondition - Returns true if conversion to bool is invalid. + ExprResult CheckCXXBooleanCondition(Expr *CondExpr, bool IsConstexpr = false); - /// Check class-level dllimport/dllexport attribute. The caller must - /// ensure that referenceDLLExportedClassMethods is called some point later - /// when all outer classes of Class are complete. - void checkClassLevelDLLAttribute(CXXRecordDecl *Class); - void checkClassLevelCodeSegAttribute(CXXRecordDecl *Class); + bool IsStringLiteralToNonConstPointerConversion(Expr *From, QualType ToType); - void referenceDLLExportedClassMethods(); + ExprResult + PerformImplicitConversion(Expr *From, QualType ToType, + const ImplicitConversionSequence &ICS, + AssignmentAction Action, + CheckedConversionKind CCK = CCK_ImplicitConversion); + ExprResult PerformImplicitConversion(Expr *From, QualType ToType, + const StandardConversionSequence &SCS, + AssignmentAction Action, + CheckedConversionKind CCK); - void propagateDLLAttrToBaseClassTemplate( - CXXRecordDecl *Class, Attr *ClassAttr, - ClassTemplateSpecializationDecl *BaseTemplateSpec, - SourceLocation BaseLoc); + bool CheckTypeTraitArity(unsigned Arity, SourceLocation Loc, size_t N); - /// Add gsl::Pointer attribute to std::container::iterator - /// \param ND The declaration that introduces the name - /// std::container::iterator. \param UnderlyingRecord The record named by ND. - void inferGslPointerAttribute(NamedDecl *ND, CXXRecordDecl *UnderlyingRecord); + /// Parsed one of the type trait support pseudo-functions. + ExprResult ActOnTypeTrait(TypeTrait Kind, SourceLocation KWLoc, + ArrayRef Args, + SourceLocation RParenLoc); + ExprResult BuildTypeTrait(TypeTrait Kind, SourceLocation KWLoc, + ArrayRef Args, + SourceLocation RParenLoc); - /// Add [[gsl::Owner]] and [[gsl::Pointer]] attributes for std:: types. - void inferGslOwnerPointerAttribute(CXXRecordDecl *Record); + /// ActOnArrayTypeTrait - Parsed one of the binary type trait support + /// pseudo-functions. + ExprResult ActOnArrayTypeTrait(ArrayTypeTrait ATT, SourceLocation KWLoc, + ParsedType LhsTy, Expr *DimExpr, + SourceLocation RParen); - /// Add [[gsl::Pointer]] attributes for std:: types. - void inferGslPointerAttribute(TypedefNameDecl *TD); + ExprResult BuildArrayTypeTrait(ArrayTypeTrait ATT, SourceLocation KWLoc, + TypeSourceInfo *TSInfo, Expr *DimExpr, + SourceLocation RParen); - void CheckCompletedCXXClass(Scope *S, CXXRecordDecl *Record); + /// ActOnExpressionTrait - Parsed one of the unary type trait support + /// pseudo-functions. + ExprResult ActOnExpressionTrait(ExpressionTrait OET, SourceLocation KWLoc, + Expr *Queried, SourceLocation RParen); - /// Check that the C++ class annoated with "trivial_abi" satisfies all the - /// conditions that are needed for the attribute to have an effect. - void checkIllFormedTrivialABIStruct(CXXRecordDecl &RD); + ExprResult BuildExpressionTrait(ExpressionTrait OET, SourceLocation KWLoc, + Expr *Queried, SourceLocation RParen); - void ActOnFinishCXXMemberSpecification(Scope *S, SourceLocation RLoc, - Decl *TagDecl, SourceLocation LBrac, - SourceLocation RBrac, - const ParsedAttributesView &AttrList); - void ActOnFinishCXXMemberDecls(); - void ActOnFinishCXXNonNestedClass(); + QualType CheckPointerToMemberOperands( // C++ 5.5 + ExprResult &LHS, ExprResult &RHS, ExprValueKind &VK, SourceLocation OpLoc, + bool isIndirect); + QualType CheckVectorConditionalTypes(ExprResult &Cond, ExprResult &LHS, + ExprResult &RHS, + SourceLocation QuestionLoc); - void ActOnReenterCXXMethodParameter(Scope *S, ParmVarDecl *Param); - unsigned ActOnReenterTemplateScope(Decl *Template, - llvm::function_ref EnterScope); - void ActOnStartDelayedMemberDeclarations(Scope *S, Decl *Record); - void ActOnStartDelayedCXXMethodDeclaration(Scope *S, Decl *Method); - void ActOnDelayedCXXMethodParameter(Scope *S, Decl *Param); - void ActOnFinishDelayedMemberDeclarations(Scope *S, Decl *Record); - void ActOnFinishDelayedCXXMethodDeclaration(Scope *S, Decl *Method); - void ActOnFinishDelayedMemberInitializers(Decl *Record); - void MarkAsLateParsedTemplate(FunctionDecl *FD, Decl *FnD, - CachedTokens &Toks); - void UnmarkAsLateParsedTemplate(FunctionDecl *FD); - bool IsInsideALocalClassWithinATemplateFunction(); + QualType CheckSizelessVectorConditionalTypes(ExprResult &Cond, + ExprResult &LHS, ExprResult &RHS, + SourceLocation QuestionLoc); - bool EvaluateStaticAssertMessageAsString(Expr *Message, std::string &Result, - ASTContext &Ctx, - bool ErrorOnInvalidMessage); - Decl *ActOnStaticAssertDeclaration(SourceLocation StaticAssertLoc, - Expr *AssertExpr, - Expr *AssertMessageExpr, - SourceLocation RParenLoc); - Decl *BuildStaticAssertDeclaration(SourceLocation StaticAssertLoc, - Expr *AssertExpr, Expr *AssertMessageExpr, - SourceLocation RParenLoc, bool Failed); - void DiagnoseStaticAssertDetails(const Expr *E); + QualType CXXCheckConditionalOperands( // C++ 5.16 + ExprResult &cond, ExprResult &lhs, ExprResult &rhs, ExprValueKind &VK, + ExprObjectKind &OK, SourceLocation questionLoc); - Decl *ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS, - MultiTemplateParamsArg TemplateParams); - NamedDecl *ActOnFriendFunctionDecl(Scope *S, Declarator &D, - MultiTemplateParamsArg TemplateParams); + QualType FindCompositePointerType(SourceLocation Loc, Expr *&E1, Expr *&E2, + bool ConvertArgs = true); + QualType FindCompositePointerType(SourceLocation Loc, ExprResult &E1, + ExprResult &E2, bool ConvertArgs = true) { + Expr *E1Tmp = E1.get(), *E2Tmp = E2.get(); + QualType Composite = + FindCompositePointerType(Loc, E1Tmp, E2Tmp, ConvertArgs); + E1 = E1Tmp; + E2 = E2Tmp; + return Composite; + } - QualType CheckConstructorDeclarator(Declarator &D, QualType R, - StorageClass& SC); - void CheckConstructor(CXXConstructorDecl *Constructor); - QualType CheckDestructorDeclarator(Declarator &D, QualType R, - StorageClass& SC); - bool CheckDestructor(CXXDestructorDecl *Destructor); - void CheckConversionDeclarator(Declarator &D, QualType &R, - StorageClass& SC); - Decl *ActOnConversionDeclarator(CXXConversionDecl *Conversion); - bool CheckDeductionGuideDeclarator(Declarator &D, QualType &R, - StorageClass &SC); - void CheckDeductionGuideTemplate(FunctionTemplateDecl *TD); + /// MaybeBindToTemporary - If the passed in expression has a record type with + /// a non-trivial destructor, this will return CXXBindTemporaryExpr. Otherwise + /// it simply returns the passed in expression. + ExprResult MaybeBindToTemporary(Expr *E); - void CheckExplicitlyDefaultedFunction(Scope *S, FunctionDecl *MD); + /// IgnoredValueConversions - Given that an expression's result is + /// syntactically ignored, perform any conversions that are + /// required. + ExprResult IgnoredValueConversions(Expr *E); - bool CheckExplicitlyDefaultedSpecialMember(CXXMethodDecl *MD, - CXXSpecialMember CSM, - SourceLocation DefaultLoc); - void CheckDelayedMemberExceptionSpecs(); + ExprResult CheckUnevaluatedOperand(Expr *E); - bool CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *MD, - DefaultedComparisonKind DCK); - void DeclareImplicitEqualityComparison(CXXRecordDecl *RD, - FunctionDecl *Spaceship); - void DefineDefaultedComparison(SourceLocation Loc, FunctionDecl *FD, - DefaultedComparisonKind DCK); + /// Process any TypoExprs in the given Expr and its children, + /// generating diagnostics as appropriate and returning a new Expr if there + /// were typos that were all successfully corrected and ExprError if one or + /// more typos could not be corrected. + /// + /// \param E The Expr to check for TypoExprs. + /// + /// \param InitDecl A VarDecl to avoid because the Expr being corrected is its + /// initializer. + /// + /// \param RecoverUncorrectedTypos If true, when typo correction fails, it + /// will rebuild the given Expr with all TypoExprs degraded to RecoveryExprs. + /// + /// \param Filter A function applied to a newly rebuilt Expr to determine if + /// it is an acceptable/usable result from a single combination of typo + /// corrections. As long as the filter returns ExprError, different + /// combinations of corrections will be tried until all are exhausted. + ExprResult CorrectDelayedTyposInExpr( + Expr *E, VarDecl *InitDecl = nullptr, + bool RecoverUncorrectedTypos = false, + llvm::function_ref Filter = + [](Expr *E) -> ExprResult { return E; }); - void CheckExplicitObjectMemberFunction(Declarator &D, DeclarationName Name, - QualType R, bool IsLambda, - DeclContext *DC = nullptr); - void CheckExplicitObjectMemberFunction(DeclContext *DC, Declarator &D, - DeclarationName Name, QualType R); - void CheckExplicitObjectLambda(Declarator &D); + ExprResult CorrectDelayedTyposInExpr( + ExprResult ER, VarDecl *InitDecl = nullptr, + bool RecoverUncorrectedTypos = false, + llvm::function_ref Filter = + [](Expr *E) -> ExprResult { return E; }) { + return ER.isInvalid() + ? ER + : CorrectDelayedTyposInExpr(ER.get(), InitDecl, + RecoverUncorrectedTypos, Filter); + } - //===--------------------------------------------------------------------===// - // C++ Derived Classes - // + /// Describes the result of an "if-exists" condition check. + enum IfExistsResult { + /// The symbol exists. + IER_Exists, - /// ActOnBaseSpecifier - Parsed a base specifier - CXXBaseSpecifier *CheckBaseSpecifier(CXXRecordDecl *Class, - SourceRange SpecifierRange, - bool Virtual, AccessSpecifier Access, - TypeSourceInfo *TInfo, - SourceLocation EllipsisLoc); + /// The symbol does not exist. + IER_DoesNotExist, - BaseResult ActOnBaseSpecifier(Decl *classdecl, SourceRange SpecifierRange, - const ParsedAttributesView &Attrs, bool Virtual, - AccessSpecifier Access, ParsedType basetype, - SourceLocation BaseLoc, - SourceLocation EllipsisLoc); + /// The name is a dependent name, so the results will differ + /// from one instantiation to the next. + IER_Dependent, - bool AttachBaseSpecifiers(CXXRecordDecl *Class, - MutableArrayRef Bases); - void ActOnBaseSpecifiers(Decl *ClassDecl, - MutableArrayRef Bases); + /// An error occurred. + IER_Error + }; - bool IsDerivedFrom(SourceLocation Loc, QualType Derived, QualType Base); - bool IsDerivedFrom(SourceLocation Loc, QualType Derived, QualType Base, - CXXBasePaths &Paths); + IfExistsResult + CheckMicrosoftIfExistsSymbol(Scope *S, CXXScopeSpec &SS, + const DeclarationNameInfo &TargetNameInfo); - // FIXME: I don't like this name. - void BuildBasePathArray(const CXXBasePaths &Paths, CXXCastPath &BasePath); + IfExistsResult CheckMicrosoftIfExistsSymbol(Scope *S, + SourceLocation KeywordLoc, + bool IsIfExists, CXXScopeSpec &SS, + UnqualifiedId &Name); - bool CheckDerivedToBaseConversion(QualType Derived, QualType Base, - SourceLocation Loc, SourceRange Range, - CXXCastPath *BasePath = nullptr, - bool IgnoreAccess = false); - bool CheckDerivedToBaseConversion(QualType Derived, QualType Base, - unsigned InaccessibleBaseID, - unsigned AmbiguousBaseConvID, - SourceLocation Loc, SourceRange Range, - DeclarationName Name, - CXXCastPath *BasePath, - bool IgnoreAccess = false); + RequiresExprBodyDecl * + ActOnStartRequiresExpr(SourceLocation RequiresKWLoc, + ArrayRef LocalParameters, + Scope *BodyScope); + void ActOnFinishRequiresExpr(); + concepts::Requirement *ActOnSimpleRequirement(Expr *E); + concepts::Requirement *ActOnTypeRequirement(SourceLocation TypenameKWLoc, + CXXScopeSpec &SS, + SourceLocation NameLoc, + IdentifierInfo *TypeName, + TemplateIdAnnotation *TemplateId); + concepts::Requirement *ActOnCompoundRequirement(Expr *E, + SourceLocation NoexceptLoc); + concepts::Requirement *ActOnCompoundRequirement( + Expr *E, SourceLocation NoexceptLoc, CXXScopeSpec &SS, + TemplateIdAnnotation *TypeConstraint, unsigned Depth); + concepts::Requirement *ActOnNestedRequirement(Expr *Constraint); + concepts::ExprRequirement *BuildExprRequirement( + Expr *E, bool IsSatisfied, SourceLocation NoexceptLoc, + concepts::ExprRequirement::ReturnTypeRequirement ReturnTypeRequirement); + concepts::ExprRequirement *BuildExprRequirement( + concepts::Requirement::SubstitutionDiagnostic *ExprSubstDiag, + bool IsSatisfied, SourceLocation NoexceptLoc, + concepts::ExprRequirement::ReturnTypeRequirement ReturnTypeRequirement); + concepts::TypeRequirement *BuildTypeRequirement(TypeSourceInfo *Type); + concepts::TypeRequirement *BuildTypeRequirement( + concepts::Requirement::SubstitutionDiagnostic *SubstDiag); + concepts::NestedRequirement *BuildNestedRequirement(Expr *E); + concepts::NestedRequirement * + BuildNestedRequirement(StringRef InvalidConstraintEntity, + const ASTConstraintSatisfaction &Satisfaction); + ExprResult ActOnRequiresExpr(SourceLocation RequiresKWLoc, + RequiresExprBodyDecl *Body, + SourceLocation LParenLoc, + ArrayRef LocalParameters, + SourceLocation RParenLoc, + ArrayRef Requirements, + SourceLocation ClosingBraceLoc); - std::string getAmbiguousPathsDisplayString(CXXBasePaths &Paths); +private: + ExprResult SemaBuiltinOperatorNewDeleteOverloaded(ExprResult TheCallResult, + bool IsDelete); - bool CheckOverridingFunctionAttributes(const CXXMethodDecl *New, - const CXXMethodDecl *Old); + void AnalyzeDeleteExprMismatch(const CXXDeleteExpr *DE); + void AnalyzeDeleteExprMismatch(FieldDecl *Field, SourceLocation DeleteLoc, + bool DeleteWasArrayForm); - /// CheckOverridingFunctionReturnType - Checks whether the return types are - /// covariant, according to C++ [class.virtual]p5. - bool CheckOverridingFunctionReturnType(const CXXMethodDecl *New, - const CXXMethodDecl *Old); + ///@} - // Check that the overriding method has no explicit object parameter. - bool CheckExplicitObjectOverride(CXXMethodDecl *New, - const CXXMethodDecl *Old); + // + // + // ------------------------------------------------------------------------- + // + // - /// CheckOverridingFunctionExceptionSpec - Checks whether the exception - /// spec is a subset of base spec. - bool CheckOverridingFunctionExceptionSpec(const CXXMethodDecl *New, - const CXXMethodDecl *Old); + /// \name Member Access Expressions + /// Implementations are in SemaExprMember.cpp + ///@{ - bool CheckPureMethod(CXXMethodDecl *Method, SourceRange InitRange); +public: + ExprResult BuildPossibleImplicitMemberExpr( + const CXXScopeSpec &SS, SourceLocation TemplateKWLoc, LookupResult &R, + const TemplateArgumentListInfo *TemplateArgs, const Scope *S, + UnresolvedLookupExpr *AsULE = nullptr); + ExprResult + BuildImplicitMemberExpr(const CXXScopeSpec &SS, SourceLocation TemplateKWLoc, + LookupResult &R, + const TemplateArgumentListInfo *TemplateArgs, + bool IsDefiniteInstance, const Scope *S); + + ExprResult ActOnDependentMemberExpr( + Expr *Base, QualType BaseType, bool IsArrow, SourceLocation OpLoc, + const CXXScopeSpec &SS, SourceLocation TemplateKWLoc, + NamedDecl *FirstQualifierInScope, const DeclarationNameInfo &NameInfo, + const TemplateArgumentListInfo *TemplateArgs); - /// CheckOverrideControl - Check C++11 override control semantics. - void CheckOverrideControl(NamedDecl *D); + ExprResult ActOnMemberAccessExpr(Scope *S, Expr *Base, SourceLocation OpLoc, + tok::TokenKind OpKind, CXXScopeSpec &SS, + SourceLocation TemplateKWLoc, + UnqualifiedId &Member, Decl *ObjCImpDecl); + + MemberExpr *BuildMemberExpr( + Expr *Base, bool IsArrow, SourceLocation OpLoc, const CXXScopeSpec *SS, + SourceLocation TemplateKWLoc, ValueDecl *Member, DeclAccessPair FoundDecl, + bool HadMultipleCandidates, const DeclarationNameInfo &MemberNameInfo, + QualType Ty, ExprValueKind VK, ExprObjectKind OK, + const TemplateArgumentListInfo *TemplateArgs = nullptr); + MemberExpr * + BuildMemberExpr(Expr *Base, bool IsArrow, SourceLocation OpLoc, + NestedNameSpecifierLoc NNS, SourceLocation TemplateKWLoc, + ValueDecl *Member, DeclAccessPair FoundDecl, + bool HadMultipleCandidates, + const DeclarationNameInfo &MemberNameInfo, QualType Ty, + ExprValueKind VK, ExprObjectKind OK, + const TemplateArgumentListInfo *TemplateArgs = nullptr); - /// DiagnoseAbsenceOfOverrideControl - Diagnose if 'override' keyword was - /// not used in the declaration of an overriding method. - void DiagnoseAbsenceOfOverrideControl(NamedDecl *D, bool Inconsistent); + bool CheckQualifiedMemberReference(Expr *BaseExpr, QualType BaseType, + const CXXScopeSpec &SS, + const LookupResult &R); - /// CheckForFunctionMarkedFinal - Checks whether a virtual member function - /// overrides a virtual member function marked 'final', according to - /// C++11 [class.virtual]p4. - bool CheckIfOverriddenFunctionIsMarkedFinal(const CXXMethodDecl *New, - const CXXMethodDecl *Old); + // This struct is for use by ActOnMemberAccess to allow + // BuildMemberReferenceExpr to be able to reinvoke ActOnMemberAccess after + // changing the access operator from a '.' to a '->' (to see if that is the + // change needed to fix an error about an unknown member, e.g. when the class + // defines a custom operator->). + struct ActOnMemberAccessExtraArgs { + Scope *S; + UnqualifiedId &Id; + Decl *ObjCImpDecl; + }; + ExprResult BuildMemberReferenceExpr( + Expr *Base, QualType BaseType, SourceLocation OpLoc, bool IsArrow, + CXXScopeSpec &SS, SourceLocation TemplateKWLoc, + NamedDecl *FirstQualifierInScope, const DeclarationNameInfo &NameInfo, + const TemplateArgumentListInfo *TemplateArgs, const Scope *S, + ActOnMemberAccessExtraArgs *ExtraArgs = nullptr); - //===--------------------------------------------------------------------===// - // C++ Access Control + ExprResult + BuildMemberReferenceExpr(Expr *Base, QualType BaseType, SourceLocation OpLoc, + bool IsArrow, const CXXScopeSpec &SS, + SourceLocation TemplateKWLoc, + NamedDecl *FirstQualifierInScope, LookupResult &R, + const TemplateArgumentListInfo *TemplateArgs, + const Scope *S, bool SuppressQualifierCheck = false, + ActOnMemberAccessExtraArgs *ExtraArgs = nullptr); + + ExprResult BuildFieldReferenceExpr(Expr *BaseExpr, bool IsArrow, + SourceLocation OpLoc, + const CXXScopeSpec &SS, FieldDecl *Field, + DeclAccessPair FoundDecl, + const DeclarationNameInfo &MemberNameInfo); + + ExprResult PerformMemberExprBaseConversion(Expr *Base, bool IsArrow); + + ExprResult BuildAnonymousStructUnionMemberReference( + const CXXScopeSpec &SS, SourceLocation nameLoc, + IndirectFieldDecl *indirectField, + DeclAccessPair FoundDecl = DeclAccessPair::make(nullptr, AS_none), + Expr *baseObjectExpr = nullptr, SourceLocation opLoc = SourceLocation()); + +private: + void CheckMemberAccessOfNoDeref(const MemberExpr *E); + + ///@} + + // + // + // ------------------------------------------------------------------------- + // // - enum AccessResult { - AR_accessible, - AR_inaccessible, - AR_dependent, - AR_delayed - }; + /// \name Initializers + /// Implementations are in SemaInit.cpp + ///@{ - bool SetMemberAccessSpecifier(NamedDecl *MemberDecl, - NamedDecl *PrevMemberDecl, - AccessSpecifier LexicalAS); +public: + /// Stack of types that correspond to the parameter entities that are + /// currently being copy-initialized. Can be empty. + llvm::SmallVector CurrentParameterCopyTypes; - AccessResult CheckUnresolvedMemberAccess(UnresolvedMemberExpr *E, - DeclAccessPair FoundDecl); - AccessResult CheckUnresolvedLookupAccess(UnresolvedLookupExpr *E, - DeclAccessPair FoundDecl); - AccessResult CheckAllocationAccess(SourceLocation OperatorLoc, - SourceRange PlacementRange, - CXXRecordDecl *NamingClass, - DeclAccessPair FoundDecl, - bool Diagnose = true); - AccessResult CheckConstructorAccess(SourceLocation Loc, - CXXConstructorDecl *D, - DeclAccessPair FoundDecl, - const InitializedEntity &Entity, - bool IsCopyBindingRefToTemp = false); - AccessResult CheckConstructorAccess(SourceLocation Loc, - CXXConstructorDecl *D, - DeclAccessPair FoundDecl, - const InitializedEntity &Entity, - const PartialDiagnostic &PDiag); - AccessResult CheckDestructorAccess(SourceLocation Loc, - CXXDestructorDecl *Dtor, - const PartialDiagnostic &PDiag, - QualType objectType = QualType()); - AccessResult CheckFriendAccess(NamedDecl *D); - AccessResult CheckMemberAccess(SourceLocation UseLoc, - CXXRecordDecl *NamingClass, - DeclAccessPair Found); - AccessResult - CheckStructuredBindingMemberAccess(SourceLocation UseLoc, - CXXRecordDecl *DecomposedClass, - DeclAccessPair Field); - AccessResult CheckMemberOperatorAccess(SourceLocation Loc, Expr *ObjectExpr, - const SourceRange &, - DeclAccessPair FoundDecl); - AccessResult CheckMemberOperatorAccess(SourceLocation Loc, - Expr *ObjectExpr, - Expr *ArgExpr, - DeclAccessPair FoundDecl); - AccessResult CheckMemberOperatorAccess(SourceLocation Loc, Expr *ObjectExpr, - ArrayRef ArgExprs, - DeclAccessPair FoundDecl); - AccessResult CheckAddressOfMemberAccess(Expr *OvlExpr, - DeclAccessPair FoundDecl); - AccessResult CheckBaseClassAccess(SourceLocation AccessLoc, - QualType Base, QualType Derived, - const CXXBasePath &Path, - unsigned DiagID, - bool ForceCheck = false, - bool ForceUnprivileged = false); - void CheckLookupAccess(const LookupResult &R); - bool IsSimplyAccessible(NamedDecl *Decl, CXXRecordDecl *NamingClass, - QualType BaseType); - bool isMemberAccessibleForDeletion(CXXRecordDecl *NamingClass, - DeclAccessPair Found, QualType ObjectType, - SourceLocation Loc, - const PartialDiagnostic &Diag); - bool isMemberAccessibleForDeletion(CXXRecordDecl *NamingClass, - DeclAccessPair Found, - QualType ObjectType) { - return isMemberAccessibleForDeletion(NamingClass, Found, ObjectType, - SourceLocation(), PDiag()); - } + llvm::DenseMap + AggregateDeductionCandidates; - void HandleDependentAccessCheck(const DependentDiagnostic &DD, - const MultiLevelTemplateArgumentList &TemplateArgs); - void PerformDependentDiagnostics(const DeclContext *Pattern, - const MultiLevelTemplateArgumentList &TemplateArgs); + bool IsStringInit(Expr *Init, const ArrayType *AT); - void HandleDelayedAccessCheck(sema::DelayedDiagnostic &DD, Decl *Ctx); + bool CanPerformAggregateInitializationForOverloadResolution( + const InitializedEntity &Entity, InitListExpr *From); - /// When true, access checking violations are treated as SFINAE - /// failures rather than hard errors. - bool AccessCheckingSFINAE; + ExprResult ActOnDesignatedInitializer(Designation &Desig, + SourceLocation EqualOrColonLoc, + bool GNUSyntax, ExprResult Init); - enum AbstractDiagSelID { - AbstractNone = -1, - AbstractReturnType, - AbstractParamType, - AbstractVariableType, - AbstractFieldType, - AbstractIvarType, - AbstractSynthesizedIvarType, - AbstractArrayType - }; + /// Check that the lifetime of the initializer (and its subobjects) is + /// sufficient for initializing the entity, and perform lifetime extension + /// (when permitted) if not. + void checkInitializerLifetime(const InitializedEntity &Entity, Expr *Init); - bool isAbstractType(SourceLocation Loc, QualType T); - bool RequireNonAbstractType(SourceLocation Loc, QualType T, - TypeDiagnoser &Diagnoser); - template - bool RequireNonAbstractType(SourceLocation Loc, QualType T, unsigned DiagID, - const Ts &...Args) { - BoundTypeDiagnoser Diagnoser(DiagID, Args...); - return RequireNonAbstractType(Loc, T, Diagnoser); - } + MaterializeTemporaryExpr * + CreateMaterializeTemporaryExpr(QualType T, Expr *Temporary, + bool BoundToLvalueReference); - void DiagnoseAbstractType(const CXXRecordDecl *RD); + /// If \p E is a prvalue denoting an unmaterialized temporary, materialize + /// it as an xvalue. In C++98, the result will still be a prvalue, because + /// we don't have xvalues there. + ExprResult TemporaryMaterializationConversion(Expr *E); - //===--------------------------------------------------------------------===// - // C++ Overloaded Operators [C++ 13.5] - // + ExprResult PerformQualificationConversion( + Expr *E, QualType Ty, ExprValueKind VK = VK_PRValue, + CheckedConversionKind CCK = CCK_ImplicitConversion); - bool CheckOverloadedOperatorDeclaration(FunctionDecl *FnDecl); + bool CanPerformCopyInitialization(const InitializedEntity &Entity, + ExprResult Init); + ExprResult PerformCopyInitialization(const InitializedEntity &Entity, + SourceLocation EqualLoc, ExprResult Init, + bool TopLevelOfInitList = false, + bool AllowExplicit = false); - bool CheckLiteralOperatorDeclaration(FunctionDecl *FnDecl); + QualType DeduceTemplateSpecializationFromInitializer( + TypeSourceInfo *TInfo, const InitializedEntity &Entity, + const InitializationKind &Kind, MultiExprArg Init); + + ///@} - //===--------------------------------------------------------------------===// - // C++ Templates [C++ 14] // - void FilterAcceptableTemplateNames(LookupResult &R, - bool AllowFunctionTemplates = true, - bool AllowDependent = true); - bool hasAnyAcceptableTemplateNames(LookupResult &R, - bool AllowFunctionTemplates = true, - bool AllowDependent = true, - bool AllowNonTemplateFunctions = false); - /// Try to interpret the lookup result D as a template-name. - /// - /// \param D A declaration found by name lookup. - /// \param AllowFunctionTemplates Whether function templates should be - /// considered valid results. - /// \param AllowDependent Whether unresolved using declarations (that might - /// name templates) should be considered valid results. - static NamedDecl *getAsTemplateNameDecl(NamedDecl *D, - bool AllowFunctionTemplates = true, - bool AllowDependent = true); + // + // ------------------------------------------------------------------------- + // + // - enum TemplateNameIsRequiredTag { TemplateNameIsRequired }; - /// Whether and why a template name is required in this lookup. - class RequiredTemplateKind { - public: - /// Template name is required if TemplateKWLoc is valid. - RequiredTemplateKind(SourceLocation TemplateKWLoc = SourceLocation()) - : TemplateKW(TemplateKWLoc) {} - /// Template name is unconditionally required. - RequiredTemplateKind(TemplateNameIsRequiredTag) {} + /// \name C++ Lambda Expressions + /// Implementations are in SemaLambda.cpp + ///@{ - SourceLocation getTemplateKeywordLoc() const { - return TemplateKW.value_or(SourceLocation()); - } - bool hasTemplateKeyword() const { return getTemplateKeywordLoc().isValid(); } - bool isRequired() const { return TemplateKW != SourceLocation(); } - explicit operator bool() const { return isRequired(); } +public: + /// Create a new lambda closure type. + CXXRecordDecl *createLambdaClosureType(SourceRange IntroducerRange, + TypeSourceInfo *Info, + unsigned LambdaDependencyKind, + LambdaCaptureDefault CaptureDefault); - private: - std::optional TemplateKW; - }; + /// Number lambda for linkage purposes if necessary. + void handleLambdaNumbering(CXXRecordDecl *Class, CXXMethodDecl *Method, + std::optional + NumberingOverride = std::nullopt); - enum class AssumedTemplateKind { - /// This is not assumed to be a template name. - None, - /// This is assumed to be a template name because lookup found nothing. - FoundNothing, - /// This is assumed to be a template name because lookup found one or more - /// functions (but no function templates). - FoundFunctions, - }; - bool LookupTemplateName( - LookupResult &R, Scope *S, CXXScopeSpec &SS, QualType ObjectType, - bool EnteringContext, bool &MemberOfUnknownSpecialization, - RequiredTemplateKind RequiredTemplate = SourceLocation(), - AssumedTemplateKind *ATK = nullptr, bool AllowTypoCorrection = true); - - TemplateNameKind isTemplateName(Scope *S, - CXXScopeSpec &SS, - bool hasTemplateKeyword, - const UnqualifiedId &Name, - ParsedType ObjectType, - bool EnteringContext, - TemplateTy &Template, - bool &MemberOfUnknownSpecialization, - bool Disambiguation = false); + /// Endow the lambda scope info with the relevant properties. + void buildLambdaScope(sema::LambdaScopeInfo *LSI, CXXMethodDecl *CallOperator, + SourceRange IntroducerRange, + LambdaCaptureDefault CaptureDefault, + SourceLocation CaptureDefaultLoc, bool ExplicitParams, + bool Mutable); - /// Try to resolve an undeclared template name as a type template. - /// - /// Sets II to the identifier corresponding to the template name, and updates - /// Name to a corresponding (typo-corrected) type template name and TNK to - /// the corresponding kind, if possible. - void ActOnUndeclaredTypeTemplateName(Scope *S, TemplateTy &Name, - TemplateNameKind &TNK, - SourceLocation NameLoc, - IdentifierInfo *&II); + CXXMethodDecl *CreateLambdaCallOperator(SourceRange IntroducerRange, + CXXRecordDecl *Class); - bool resolveAssumedTemplateNameAsType(Scope *S, TemplateName &Name, - SourceLocation NameLoc, - bool Diagnose = true); + void AddTemplateParametersToLambdaCallOperator( + CXXMethodDecl *CallOperator, CXXRecordDecl *Class, + TemplateParameterList *TemplateParams); - /// Determine whether a particular identifier might be the name in a C++1z - /// deduction-guide declaration. - bool isDeductionGuideName(Scope *S, const IdentifierInfo &Name, - SourceLocation NameLoc, CXXScopeSpec &SS, - ParsedTemplateTy *Template = nullptr); + void CompleteLambdaCallOperator( + CXXMethodDecl *Method, SourceLocation LambdaLoc, + SourceLocation CallOperatorLoc, Expr *TrailingRequiresClause, + TypeSourceInfo *MethodTyInfo, ConstexprSpecKind ConstexprKind, + StorageClass SC, ArrayRef Params, + bool HasExplicitResultType); - bool DiagnoseUnknownTemplateName(const IdentifierInfo &II, - SourceLocation IILoc, - Scope *S, - const CXXScopeSpec *SS, - TemplateTy &SuggestedTemplate, - TemplateNameKind &SuggestedKind); + void DiagnoseInvalidExplicitObjectParameterInLambda(CXXMethodDecl *Method); - bool DiagnoseUninstantiableTemplate(SourceLocation PointOfInstantiation, - NamedDecl *Instantiation, - bool InstantiatedFromMember, - const NamedDecl *Pattern, - const NamedDecl *PatternDef, - TemplateSpecializationKind TSK, - bool Complain = true); + /// Perform initialization analysis of the init-capture and perform + /// any implicit conversions such as an lvalue-to-rvalue conversion if + /// not being used to initialize a reference. + ParsedType actOnLambdaInitCaptureInitialization( + SourceLocation Loc, bool ByRef, SourceLocation EllipsisLoc, + IdentifierInfo *Id, LambdaCaptureInitKind InitKind, Expr *&Init) { + return ParsedType::make(buildLambdaInitCaptureInitialization( + Loc, ByRef, EllipsisLoc, std::nullopt, Id, + InitKind != LambdaCaptureInitKind::CopyInit, Init)); + } + QualType buildLambdaInitCaptureInitialization( + SourceLocation Loc, bool ByRef, SourceLocation EllipsisLoc, + std::optional NumExpansions, IdentifierInfo *Id, + bool DirectInit, Expr *&Init); - /// DiagnoseTemplateParameterShadow - Produce a diagnostic complaining - /// that the template parameter 'PrevDecl' is being shadowed by a new - /// declaration at location Loc. Returns true to indicate that this is - /// an error, and false otherwise. - /// - /// \param Loc The location of the declaration that shadows a template - /// parameter. - /// - /// \param PrevDecl The template parameter that the declaration shadows. + /// Create a dummy variable within the declcontext of the lambda's + /// call operator, for name lookup purposes for a lambda init capture. /// - /// \param SupportedForCompatibility Whether to issue the diagnostic as - /// a warning for compatibility with older versions of clang. - /// Ignored when MSVC compatibility is enabled. - void DiagnoseTemplateParameterShadow(SourceLocation Loc, Decl *PrevDecl, - bool SupportedForCompatibility = false); - TemplateDecl *AdjustDeclIfTemplate(Decl *&Decl); - - NamedDecl *ActOnTypeParameter(Scope *S, bool Typename, - SourceLocation EllipsisLoc, - SourceLocation KeyLoc, - IdentifierInfo *ParamName, - SourceLocation ParamNameLoc, - unsigned Depth, unsigned Position, - SourceLocation EqualLoc, - ParsedType DefaultArg, bool HasTypeConstraint); + /// CodeGen handles emission of lambda captures, ignoring these dummy + /// variables appropriately. + VarDecl *createLambdaInitCaptureVarDecl( + SourceLocation Loc, QualType InitCaptureType, SourceLocation EllipsisLoc, + IdentifierInfo *Id, unsigned InitStyle, Expr *Init, DeclContext *DeclCtx); - bool CheckTypeConstraint(TemplateIdAnnotation *TypeConstraint); + /// Add an init-capture to a lambda scope. + void addInitCapture(sema::LambdaScopeInfo *LSI, VarDecl *Var, bool ByRef); - bool ActOnTypeConstraint(const CXXScopeSpec &SS, - TemplateIdAnnotation *TypeConstraint, - TemplateTypeParmDecl *ConstrainedParameter, - SourceLocation EllipsisLoc); - bool BuildTypeConstraint(const CXXScopeSpec &SS, - TemplateIdAnnotation *TypeConstraint, - TemplateTypeParmDecl *ConstrainedParameter, - SourceLocation EllipsisLoc, - bool AllowUnexpandedPack); + /// Note that we have finished the explicit captures for the + /// given lambda. + void finishLambdaExplicitCaptures(sema::LambdaScopeInfo *LSI); - bool AttachTypeConstraint(NestedNameSpecifierLoc NS, - DeclarationNameInfo NameInfo, - ConceptDecl *NamedConcept, - const TemplateArgumentListInfo *TemplateArgs, - TemplateTypeParmDecl *ConstrainedParameter, - SourceLocation EllipsisLoc); + /// Deduce a block or lambda's return type based on the return + /// statements present in the body. + void deduceClosureReturnType(sema::CapturingScopeInfo &CSI); - bool AttachTypeConstraint(AutoTypeLoc TL, - NonTypeTemplateParmDecl *NewConstrainedParm, - NonTypeTemplateParmDecl *OrigConstrainedParm, - SourceLocation EllipsisLoc); + /// Once the Lambdas capture are known, we can start to create the closure, + /// call operator method, and keep track of the captures. + /// We do the capture lookup here, but they are not actually captured until + /// after we know what the qualifiers of the call operator are. + void ActOnLambdaExpressionAfterIntroducer(LambdaIntroducer &Intro, + Scope *CurContext); - bool RequireStructuralType(QualType T, SourceLocation Loc); + /// This is called after parsing the explicit template parameter list + /// on a lambda (if it exists) in C++2a. + void ActOnLambdaExplicitTemplateParameterList(LambdaIntroducer &Intro, + SourceLocation LAngleLoc, + ArrayRef TParams, + SourceLocation RAngleLoc, + ExprResult RequiresClause); - QualType CheckNonTypeTemplateParameterType(TypeSourceInfo *&TSI, - SourceLocation Loc); - QualType CheckNonTypeTemplateParameterType(QualType T, SourceLocation Loc); + void ActOnLambdaClosureQualifiers(LambdaIntroducer &Intro, + SourceLocation MutableLoc); - NamedDecl *ActOnNonTypeTemplateParameter(Scope *S, Declarator &D, - unsigned Depth, - unsigned Position, - SourceLocation EqualLoc, - Expr *DefaultArg); - NamedDecl *ActOnTemplateTemplateParameter(Scope *S, - SourceLocation TmpLoc, - TemplateParameterList *Params, - SourceLocation EllipsisLoc, - IdentifierInfo *ParamName, - SourceLocation ParamNameLoc, - unsigned Depth, - unsigned Position, - SourceLocation EqualLoc, - ParsedTemplateArgument DefaultArg); + void ActOnLambdaClosureParameters( + Scope *LambdaScope, + MutableArrayRef ParamInfo); - TemplateParameterList * - ActOnTemplateParameterList(unsigned Depth, - SourceLocation ExportLoc, - SourceLocation TemplateLoc, - SourceLocation LAngleLoc, - ArrayRef Params, - SourceLocation RAngleLoc, - Expr *RequiresClause); + /// ActOnStartOfLambdaDefinition - This is called just before we start + /// parsing the body of a lambda; it analyzes the explicit captures and + /// arguments, and sets up various data-structures for the body of the + /// lambda. + void ActOnStartOfLambdaDefinition(LambdaIntroducer &Intro, + Declarator &ParamInfo, const DeclSpec &DS); - /// The context in which we are checking a template parameter list. - enum TemplateParamListContext { - TPC_ClassTemplate, - TPC_VarTemplate, - TPC_FunctionTemplate, - TPC_ClassTemplateMember, - TPC_FriendClassTemplate, - TPC_FriendFunctionTemplate, - TPC_FriendFunctionTemplateDefinition, - TPC_TypeAliasTemplate - }; + /// ActOnLambdaError - If there is an error parsing a lambda, this callback + /// is invoked to pop the information about the lambda. + void ActOnLambdaError(SourceLocation StartLoc, Scope *CurScope, + bool IsInstantiation = false); - bool CheckTemplateParameterList(TemplateParameterList *NewParams, - TemplateParameterList *OldParams, - TemplateParamListContext TPC, - SkipBodyInfo *SkipBody = nullptr); - TemplateParameterList *MatchTemplateParametersToScopeSpecifier( - SourceLocation DeclStartLoc, SourceLocation DeclLoc, - const CXXScopeSpec &SS, TemplateIdAnnotation *TemplateId, - ArrayRef ParamLists, - bool IsFriend, bool &IsMemberSpecialization, bool &Invalid, - bool SuppressDiagnostic = false); + /// ActOnLambdaExpr - This is called when the body of a lambda expression + /// was successfully completed. + ExprResult ActOnLambdaExpr(SourceLocation StartLoc, Stmt *Body); - DeclResult CheckClassTemplate( - Scope *S, unsigned TagSpec, TagUseKind TUK, SourceLocation KWLoc, - CXXScopeSpec &SS, IdentifierInfo *Name, SourceLocation NameLoc, - const ParsedAttributesView &Attr, TemplateParameterList *TemplateParams, - AccessSpecifier AS, SourceLocation ModulePrivateLoc, - SourceLocation FriendLoc, unsigned NumOuterTemplateParamLists, - TemplateParameterList **OuterTemplateParamLists, - SkipBodyInfo *SkipBody = nullptr); + /// Does copying/destroying the captured variable have side effects? + bool CaptureHasSideEffects(const sema::Capture &From); - TemplateArgumentLoc getTrivialTemplateArgumentLoc(const TemplateArgument &Arg, - QualType NTTPType, - SourceLocation Loc); + /// Diagnose if an explicit lambda capture is unused. Returns true if a + /// diagnostic is emitted. + bool DiagnoseUnusedLambdaCapture(SourceRange CaptureRange, + const sema::Capture &From); - /// Get a template argument mapping the given template parameter to itself, - /// e.g. for X in \c template, this would return an expression template - /// argument referencing X. - TemplateArgumentLoc getIdentityTemplateArgumentLoc(NamedDecl *Param, - SourceLocation Location); + /// Build a FieldDecl suitable to hold the given capture. + FieldDecl *BuildCaptureField(RecordDecl *RD, const sema::Capture &Capture); - void translateTemplateArguments(const ASTTemplateArgsPtr &In, - TemplateArgumentListInfo &Out); + /// Initialize the given capture with a suitable expression. + ExprResult BuildCaptureInit(const sema::Capture &Capture, + SourceLocation ImplicitCaptureLoc, + bool IsOpenMPMapping = false); - ParsedTemplateArgument ActOnTemplateTypeArgument(TypeResult ParsedType); + /// Complete a lambda-expression having processed and attached the + /// lambda body. + ExprResult BuildLambdaExpr(SourceLocation StartLoc, SourceLocation EndLoc, + sema::LambdaScopeInfo *LSI); - void NoteAllFoundTemplates(TemplateName Name); + /// Get the return type to use for a lambda's conversion function(s) to + /// function pointer type, given the type of the call operator. + QualType + getLambdaConversionFunctionResultType(const FunctionProtoType *CallOpType, + CallingConv CC); - QualType CheckTemplateIdType(TemplateName Template, - SourceLocation TemplateLoc, - TemplateArgumentListInfo &TemplateArgs); + ExprResult BuildBlockForLambdaConversion(SourceLocation CurrentLocation, + SourceLocation ConvLocation, + CXXConversionDecl *Conv, Expr *Src); - TypeResult - ActOnTemplateIdType(Scope *S, CXXScopeSpec &SS, SourceLocation TemplateKWLoc, - TemplateTy Template, IdentifierInfo *TemplateII, - SourceLocation TemplateIILoc, SourceLocation LAngleLoc, - ASTTemplateArgsPtr TemplateArgs, SourceLocation RAngleLoc, - bool IsCtorOrDtorName = false, bool IsClassName = false, - ImplicitTypenameContext AllowImplicitTypename = - ImplicitTypenameContext::No); + class LambdaScopeForCallOperatorInstantiationRAII + : private FunctionScopeRAII { + public: + LambdaScopeForCallOperatorInstantiationRAII( + Sema &SemasRef, FunctionDecl *FD, MultiLevelTemplateArgumentList MLTAL, + LocalInstantiationScope &Scope, + bool ShouldAddDeclsFromParentScope = true); + }; - /// Parsed an elaborated-type-specifier that refers to a template-id, - /// such as \c class T::template apply. - TypeResult ActOnTagTemplateIdType(TagUseKind TUK, - TypeSpecifierType TagSpec, - SourceLocation TagLoc, - CXXScopeSpec &SS, - SourceLocation TemplateKWLoc, - TemplateTy TemplateD, - SourceLocation TemplateLoc, - SourceLocation LAngleLoc, - ASTTemplateArgsPtr TemplateArgsIn, - SourceLocation RAngleLoc); - - DeclResult ActOnVarTemplateSpecialization( - Scope *S, Declarator &D, TypeSourceInfo *DI, LookupResult &Previous, - SourceLocation TemplateKWLoc, TemplateParameterList *TemplateParams, - StorageClass SC, bool IsPartialSpecialization); - - /// Get the specialization of the given variable template corresponding to - /// the specified argument list, or a null-but-valid result if the arguments - /// are dependent. - DeclResult CheckVarTemplateId(VarTemplateDecl *Template, - SourceLocation TemplateLoc, - SourceLocation TemplateNameLoc, - const TemplateArgumentListInfo &TemplateArgs); - - /// Form a reference to the specialization of the given variable template - /// corresponding to the specified argument list, or a null-but-valid result - /// if the arguments are dependent. - ExprResult CheckVarTemplateId(const CXXScopeSpec &SS, - const DeclarationNameInfo &NameInfo, - VarTemplateDecl *Template, NamedDecl *FoundD, - SourceLocation TemplateLoc, - const TemplateArgumentListInfo *TemplateArgs); + /// Compute the mangling number context for a lambda expression or + /// block literal. Also return the extra mangling decl if any. + /// + /// \param DC - The DeclContext containing the lambda expression or + /// block literal. + std::tuple + getCurrentMangleNumberContext(const DeclContext *DC); - ExprResult - CheckConceptTemplateId(const CXXScopeSpec &SS, - SourceLocation TemplateKWLoc, - const DeclarationNameInfo &ConceptNameInfo, - NamedDecl *FoundDecl, ConceptDecl *NamedConcept, - const TemplateArgumentListInfo *TemplateArgs); + ///@} - void diagnoseMissingTemplateArguments(TemplateName Name, SourceLocation Loc); + // + // + // ------------------------------------------------------------------------- + // + // - ExprResult BuildTemplateIdExpr(const CXXScopeSpec &SS, - SourceLocation TemplateKWLoc, - LookupResult &R, - bool RequiresADL, - const TemplateArgumentListInfo *TemplateArgs); + /// \name Name Lookup + /// + /// These routines provide name lookup that is used during semantic + /// analysis to resolve the various kinds of names (identifiers, + /// overloaded operator names, constructor names, etc.) into zero or + /// more declarations within a particular scope. The major entry + /// points are LookupName, which performs unqualified name lookup, + /// and LookupQualifiedName, which performs qualified name lookup. + /// + /// All name lookup is performed based on some specific criteria, + /// which specify what names will be visible to name lookup and how + /// far name lookup should work. These criteria are important both + /// for capturing language semantics (certain lookups will ignore + /// certain names, for example) and for performance, since name + /// lookup is often a bottleneck in the compilation of C++. Name + /// lookup criteria is specified via the LookupCriteria enumeration. + /// + /// The results of name lookup can vary based on the kind of name + /// lookup performed, the current language, and the translation + /// unit. In C, for example, name lookup will either return nothing + /// (no entity found) or a single declaration. In C++, name lookup + /// can additionally refer to a set of overloaded functions or + /// result in an ambiguity. All of the possible results of name + /// lookup are captured by the LookupResult class, which provides + /// the ability to distinguish among them. + /// + /// Implementations are in SemaLookup.cpp + ///@{ - ExprResult BuildQualifiedTemplateIdExpr(CXXScopeSpec &SS, - SourceLocation TemplateKWLoc, - const DeclarationNameInfo &NameInfo, - const TemplateArgumentListInfo *TemplateArgs); +public: + /// Tracks whether we are in a context where typo correction is + /// disabled. + bool DisableTypoCorrection; - TemplateNameKind ActOnTemplateName( - Scope *S, CXXScopeSpec &SS, SourceLocation TemplateKWLoc, - const UnqualifiedId &Name, ParsedType ObjectType, bool EnteringContext, - TemplateTy &Template, bool AllowInjectedClassName = false); + /// The number of typos corrected by CorrectTypo. + unsigned TyposCorrected; - DeclResult ActOnClassTemplateSpecialization( - Scope *S, unsigned TagSpec, TagUseKind TUK, SourceLocation KWLoc, - SourceLocation ModulePrivateLoc, CXXScopeSpec &SS, - TemplateIdAnnotation &TemplateId, const ParsedAttributesView &Attr, - MultiTemplateParamsArg TemplateParameterLists, - SkipBodyInfo *SkipBody = nullptr); + typedef llvm::SmallSet SrcLocSet; + typedef llvm::DenseMap IdentifierSourceLocations; - bool CheckTemplatePartialSpecializationArgs(SourceLocation Loc, - TemplateDecl *PrimaryTemplate, - unsigned NumExplicitArgs, - ArrayRef Args); - void CheckTemplatePartialSpecialization( - ClassTemplatePartialSpecializationDecl *Partial); - void CheckTemplatePartialSpecialization( - VarTemplatePartialSpecializationDecl *Partial); + /// A cache containing identifiers for which typo correction failed and + /// their locations, so that repeated attempts to correct an identifier in a + /// given location are ignored if typo correction already failed for it. + IdentifierSourceLocations TypoCorrectionFailures; - Decl *ActOnTemplateDeclarator(Scope *S, - MultiTemplateParamsArg TemplateParameterLists, - Declarator &D); + /// SpecialMemberOverloadResult - The overloading result for a special member + /// function. + /// + /// This is basically a wrapper around PointerIntPair. The lowest bits of the + /// integer are used to determine whether overload resolution succeeded. + class SpecialMemberOverloadResult { + public: + enum Kind { NoMemberOrDeleted, Ambiguous, Success }; - bool - CheckSpecializationInstantiationRedecl(SourceLocation NewLoc, - TemplateSpecializationKind NewTSK, - NamedDecl *PrevDecl, - TemplateSpecializationKind PrevTSK, - SourceLocation PrevPtOfInstantiation, - bool &SuppressNew); + private: + llvm::PointerIntPair Pair; - bool CheckDependentFunctionTemplateSpecialization( - FunctionDecl *FD, const TemplateArgumentListInfo *ExplicitTemplateArgs, - LookupResult &Previous); + public: + SpecialMemberOverloadResult() {} + SpecialMemberOverloadResult(CXXMethodDecl *MD) + : Pair(MD, MD->isDeleted() ? NoMemberOrDeleted : Success) {} - bool CheckFunctionTemplateSpecialization( - FunctionDecl *FD, TemplateArgumentListInfo *ExplicitTemplateArgs, - LookupResult &Previous, bool QualifiedFriend = false); - bool CheckMemberSpecialization(NamedDecl *Member, LookupResult &Previous); - void CompleteMemberSpecialization(NamedDecl *Member, LookupResult &Previous); + CXXMethodDecl *getMethod() const { return Pair.getPointer(); } + void setMethod(CXXMethodDecl *MD) { Pair.setPointer(MD); } - DeclResult ActOnExplicitInstantiation( - Scope *S, SourceLocation ExternLoc, SourceLocation TemplateLoc, - unsigned TagSpec, SourceLocation KWLoc, const CXXScopeSpec &SS, - TemplateTy Template, SourceLocation TemplateNameLoc, - SourceLocation LAngleLoc, ASTTemplateArgsPtr TemplateArgs, - SourceLocation RAngleLoc, const ParsedAttributesView &Attr); + Kind getKind() const { return static_cast(Pair.getInt()); } + void setKind(Kind K) { Pair.setInt(K); } + }; - DeclResult ActOnExplicitInstantiation(Scope *S, SourceLocation ExternLoc, - SourceLocation TemplateLoc, - unsigned TagSpec, SourceLocation KWLoc, - CXXScopeSpec &SS, IdentifierInfo *Name, - SourceLocation NameLoc, - const ParsedAttributesView &Attr); + class SpecialMemberOverloadResultEntry : public llvm::FastFoldingSetNode, + public SpecialMemberOverloadResult { + public: + SpecialMemberOverloadResultEntry(const llvm::FoldingSetNodeID &ID) + : FastFoldingSetNode(ID) {} + }; - DeclResult ActOnExplicitInstantiation(Scope *S, - SourceLocation ExternLoc, - SourceLocation TemplateLoc, - Declarator &D); + /// A cache of special member function overload resolution results + /// for C++ records. + llvm::FoldingSet SpecialMemberCache; - TemplateArgumentLoc SubstDefaultTemplateArgumentIfAvailable( - TemplateDecl *Template, SourceLocation TemplateLoc, - SourceLocation RAngleLoc, Decl *Param, - ArrayRef SugaredConverted, - ArrayRef CanonicalConverted, bool &HasDefaultArg); + /// Holds TypoExprs that are created from `createDelayedTypo`. This is used by + /// `TransformTypos` in order to keep track of any TypoExprs that are created + /// recursively during typo correction and wipe them away if the correction + /// fails. + llvm::SmallVector TypoExprs; - SourceLocation getTopMostPointOfInstantiation(const NamedDecl *) const; + enum class AcceptableKind { Visible, Reachable }; - /// Specifies the context in which a particular template - /// argument is being checked. - enum CheckTemplateArgumentKind { - /// The template argument was specified in the code or was - /// instantiated with some deduced template arguments. - CTAK_Specified, + // Members have to be NamespaceDecl* or TranslationUnitDecl*. + // TODO: make this is a typesafe union. + typedef llvm::SmallSetVector AssociatedNamespaceSet; + typedef llvm::SmallSetVector AssociatedClassSet; - /// The template argument was deduced via template argument - /// deduction. - CTAK_Deduced, + /// Describes the kind of name lookup to perform. + enum LookupNameKind { + /// Ordinary name lookup, which finds ordinary names (functions, + /// variables, typedefs, etc.) in C and most kinds of names + /// (functions, variables, members, types, etc.) in C++. + LookupOrdinaryName = 0, + /// Tag name lookup, which finds the names of enums, classes, + /// structs, and unions. + LookupTagName, + /// Label name lookup. + LookupLabel, + /// Member name lookup, which finds the names of + /// class/struct/union members. + LookupMemberName, + /// Look up of an operator name (e.g., operator+) for use with + /// operator overloading. This lookup is similar to ordinary name + /// lookup, but will ignore any declarations that are class members. + LookupOperatorName, + /// Look up a name following ~ in a destructor name. This is an ordinary + /// lookup, but prefers tags to typedefs. + LookupDestructorName, + /// Look up of a name that precedes the '::' scope resolution + /// operator in C++. This lookup completely ignores operator, object, + /// function, and enumerator names (C++ [basic.lookup.qual]p1). + LookupNestedNameSpecifierName, + /// Look up a namespace name within a C++ using directive or + /// namespace alias definition, ignoring non-namespace names (C++ + /// [basic.lookup.udir]p1). + LookupNamespaceName, + /// Look up all declarations in a scope with the given name, + /// including resolved using declarations. This is appropriate + /// for checking redeclarations for a using declaration. + LookupUsingDeclName, + /// Look up an ordinary name that is going to be redeclared as a + /// name with linkage. This lookup ignores any declarations that + /// are outside of the current scope unless they have linkage. See + /// C99 6.2.2p4-5 and C++ [basic.link]p6. + LookupRedeclarationWithLinkage, + /// Look up a friend of a local class. This lookup does not look + /// outside the innermost non-class scope. See C++11 [class.friend]p11. + LookupLocalFriendName, + /// Look up the name of an Objective-C protocol. + LookupObjCProtocolName, + /// Look up implicit 'self' parameter of an objective-c method. + LookupObjCImplicitSelfParam, + /// Look up the name of an OpenMP user-defined reduction operation. + LookupOMPReductionName, + /// Look up the name of an OpenMP user-defined mapper. + LookupOMPMapperName, + /// Look up any declaration with any name. + LookupAnyName + }; - /// The template argument was deduced from an array bound - /// via template argument deduction. - CTAK_DeducedFromArrayBound + /// The possible outcomes of name lookup for a literal operator. + enum LiteralOperatorLookupResult { + /// The lookup resulted in an error. + LOLR_Error, + /// The lookup found no match but no diagnostic was issued. + LOLR_ErrorNoDiagnostic, + /// The lookup found a single 'cooked' literal operator, which + /// expects a normal literal to be built and passed to it. + LOLR_Cooked, + /// The lookup found a single 'raw' literal operator, which expects + /// a string literal containing the spelling of the literal token. + LOLR_Raw, + /// The lookup found an overload set of literal operator templates, + /// which expect the characters of the spelling of the literal token to be + /// passed as a non-type template argument pack. + LOLR_Template, + /// The lookup found an overload set of literal operator templates, + /// which expect the character type and characters of the spelling of the + /// string literal token to be passed as template arguments. + LOLR_StringTemplatePack, }; - bool - CheckTemplateArgument(NamedDecl *Param, TemplateArgumentLoc &Arg, - NamedDecl *Template, SourceLocation TemplateLoc, - SourceLocation RAngleLoc, unsigned ArgumentPackIndex, - SmallVectorImpl &SugaredConverted, - SmallVectorImpl &CanonicalConverted, - CheckTemplateArgumentKind CTAK); + SpecialMemberOverloadResult + LookupSpecialMember(CXXRecordDecl *D, CXXSpecialMember SM, bool ConstArg, + bool VolatileArg, bool RValueThis, bool ConstThis, + bool VolatileThis); - /// Check that the given template arguments can be provided to - /// the given template, converting the arguments along the way. - /// - /// \param Template The template to which the template arguments are being - /// provided. - /// - /// \param TemplateLoc The location of the template name in the source. - /// - /// \param TemplateArgs The list of template arguments. If the template is - /// a template template parameter, this function may extend the set of - /// template arguments to also include substituted, defaulted template - /// arguments. - /// - /// \param PartialTemplateArgs True if the list of template arguments is - /// intentionally partial, e.g., because we're checking just the initial - /// set of template arguments. - /// - /// \param Converted Will receive the converted, canonicalized template - /// arguments. - /// - /// \param UpdateArgsWithConversions If \c true, update \p TemplateArgs to - /// contain the converted forms of the template arguments as written. - /// Otherwise, \p TemplateArgs will not be modified. - /// - /// \param ConstraintsNotSatisfied If provided, and an error occurred, will - /// receive true if the cause for the error is the associated constraints of - /// the template not being satisfied by the template arguments. - /// - /// \returns true if an error occurred, false otherwise. - bool CheckTemplateArgumentList( - TemplateDecl *Template, SourceLocation TemplateLoc, - TemplateArgumentListInfo &TemplateArgs, bool PartialTemplateArgs, - SmallVectorImpl &SugaredConverted, - SmallVectorImpl &CanonicalConverted, - bool UpdateArgsWithConversions = true, - bool *ConstraintsNotSatisfied = nullptr); + typedef std::function TypoDiagnosticGenerator; + typedef std::function + TypoRecoveryCallback; - bool CheckTemplateTypeArgument( - TemplateTypeParmDecl *Param, TemplateArgumentLoc &Arg, - SmallVectorImpl &SugaredConverted, - SmallVectorImpl &CanonicalConverted); + /// Specifies whether (or how) name lookup is being performed for a + /// redeclaration (vs. a reference). + enum RedeclarationKind { + /// The lookup is a reference to this name that is not for the + /// purpose of redeclaring the name. + NotForRedeclaration = 0, + /// The lookup results will be used for redeclaration of a name, + /// if an entity by that name already exists and is visible. + ForVisibleRedeclaration, + /// The lookup results will be used for redeclaration of a name + /// with external linkage; non-visible lookup results with external linkage + /// may also be found. + ForExternalRedeclaration + }; - bool CheckTemplateArgument(TypeSourceInfo *Arg); - ExprResult CheckTemplateArgument(NonTypeTemplateParmDecl *Param, - QualType InstantiatedParamType, Expr *Arg, - TemplateArgument &SugaredConverted, - TemplateArgument &CanonicalConverted, - CheckTemplateArgumentKind CTAK); - bool CheckTemplateTemplateArgument(TemplateTemplateParmDecl *Param, - TemplateParameterList *Params, - TemplateArgumentLoc &Arg); + RedeclarationKind forRedeclarationInCurContext() const { + // A declaration with an owning module for linkage can never link against + // anything that is not visible. We don't need to check linkage here; if + // the context has internal linkage, redeclaration lookup won't find things + // from other TUs, and we can't safely compute linkage yet in general. + if (cast(CurContext) + ->getOwningModuleForLinkage(/*IgnoreLinkage*/ true)) + return ForVisibleRedeclaration; + return ForExternalRedeclaration; + } - void NoteTemplateLocation(const NamedDecl &Decl, - std::optional ParamRange = {}); - void NoteTemplateParameterLocation(const NamedDecl &Decl); + /// Look up a name, looking for a single declaration. Return + /// null if the results were absent, ambiguous, or overloaded. + /// + /// It is preferable to use the elaborated form and explicitly handle + /// ambiguity and overloaded. + NamedDecl *LookupSingleName(Scope *S, DeclarationName Name, + SourceLocation Loc, LookupNameKind NameKind, + RedeclarationKind Redecl = NotForRedeclaration); + bool LookupBuiltin(LookupResult &R); + void LookupNecessaryTypesForBuiltin(Scope *S, unsigned ID); + bool LookupName(LookupResult &R, Scope *S, bool AllowBuiltinCreation = false, + bool ForceNoCPlusPlus = false); + bool LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx, + bool InUnqualifiedLookup = false); + bool LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx, + CXXScopeSpec &SS); + bool LookupParsedName(LookupResult &R, Scope *S, CXXScopeSpec *SS, + bool AllowBuiltinCreation = false, + bool EnteringContext = false); + ObjCProtocolDecl * + LookupProtocol(IdentifierInfo *II, SourceLocation IdLoc, + RedeclarationKind Redecl = NotForRedeclaration); + bool LookupInSuper(LookupResult &R, CXXRecordDecl *Class); - ExprResult - BuildExpressionFromDeclTemplateArgument(const TemplateArgument &Arg, - QualType ParamType, - SourceLocation Loc); - ExprResult - BuildExpressionFromNonTypeTemplateArgument(const TemplateArgument &Arg, - SourceLocation Loc); + void LookupOverloadedOperatorName(OverloadedOperatorKind Op, Scope *S, + UnresolvedSetImpl &Functions); - /// Enumeration describing how template parameter lists are compared - /// for equality. - enum TemplateParameterListEqualKind { - /// We are matching the template parameter lists of two templates - /// that might be redeclarations. - /// - /// \code - /// template struct X; - /// template struct X; - /// \endcode - TPL_TemplateMatch, + LabelDecl *LookupOrCreateLabel(IdentifierInfo *II, SourceLocation IdentLoc, + SourceLocation GnuLabelLoc = SourceLocation()); - /// We are matching the template parameter lists of two template - /// template parameters as part of matching the template parameter lists - /// of two templates that might be redeclarations. - /// - /// \code - /// template class TT> struct X; - /// template class Other> struct X; - /// \endcode - TPL_TemplateTemplateParmMatch, + DeclContextLookupResult LookupConstructors(CXXRecordDecl *Class); + CXXConstructorDecl *LookupDefaultConstructor(CXXRecordDecl *Class); + CXXConstructorDecl *LookupCopyingConstructor(CXXRecordDecl *Class, + unsigned Quals); + CXXMethodDecl *LookupCopyingAssignment(CXXRecordDecl *Class, unsigned Quals, + bool RValueThis, unsigned ThisQuals); + CXXConstructorDecl *LookupMovingConstructor(CXXRecordDecl *Class, + unsigned Quals); + CXXMethodDecl *LookupMovingAssignment(CXXRecordDecl *Class, unsigned Quals, + bool RValueThis, unsigned ThisQuals); + CXXDestructorDecl *LookupDestructor(CXXRecordDecl *Class); - /// We are matching the template parameter lists of a template - /// template argument against the template parameter lists of a template - /// template parameter. - /// - /// \code - /// template class Metafun> struct X; - /// template struct integer_c; - /// X xic; - /// \endcode - TPL_TemplateTemplateArgumentMatch, + /// Force the declaration of any implicitly-declared members of this + /// class. + void ForceDeclarationOfImplicitMembers(CXXRecordDecl *Class); - /// We are determining whether the template-parameters are equivalent - /// according to C++ [temp.over.link]/6. This comparison does not consider - /// constraints. - /// - /// \code - /// template void f(T); - /// template void f(T); - /// \endcode - TPL_TemplateParamsEquivalent, - }; + /// Make a merged definition of an existing hidden definition \p ND + /// visible at the specified location. + void makeMergedDefinitionVisible(NamedDecl *ND); - // A struct to represent the 'new' declaration, which is either itself just - // the named decl, or the important information we need about it in order to - // do constraint comparisons. - class TemplateCompareNewDeclInfo { - const NamedDecl *ND = nullptr; - const DeclContext *DC = nullptr; - const DeclContext *LexicalDC = nullptr; - SourceLocation Loc; + /// Get the set of additional modules that should be checked during + /// name lookup. A module and its imports become visible when instanting a + /// template defined within it. + llvm::DenseSet &getLookupModules(); - public: - TemplateCompareNewDeclInfo(const NamedDecl *ND) : ND(ND) {} - TemplateCompareNewDeclInfo(const DeclContext *DeclCtx, - const DeclContext *LexicalDeclCtx, - SourceLocation Loc) + bool hasVisibleMergedDefinition(const NamedDecl *Def); + bool hasMergedDefinitionInCurrentModule(const NamedDecl *Def); - : DC(DeclCtx), LexicalDC(LexicalDeclCtx), Loc(Loc) { - assert(DC && LexicalDC && - "Constructor only for cases where we have the information to put " - "in here"); - } + /// Determine if the template parameter \p D has a visible default argument. + bool + hasVisibleDefaultArgument(const NamedDecl *D, + llvm::SmallVectorImpl *Modules = nullptr); + /// Determine if the template parameter \p D has a reachable default argument. + bool hasReachableDefaultArgument( + const NamedDecl *D, llvm::SmallVectorImpl *Modules = nullptr); + /// Determine if the template parameter \p D has a reachable default argument. + bool hasAcceptableDefaultArgument(const NamedDecl *D, + llvm::SmallVectorImpl *Modules, + Sema::AcceptableKind Kind); - // If this was constructed with no information, we cannot do substitution - // for constraint comparison, so make sure we can check that. - bool isInvalid() const { return !ND && !DC; } + /// Determine if there is a visible declaration of \p D that is an explicit + /// specialization declaration for a specialization of a template. (For a + /// member specialization, use hasVisibleMemberSpecialization.) + bool hasVisibleExplicitSpecialization( + const NamedDecl *D, llvm::SmallVectorImpl *Modules = nullptr); + /// Determine if there is a reachable declaration of \p D that is an explicit + /// specialization declaration for a specialization of a template. (For a + /// member specialization, use hasReachableMemberSpecialization.) + bool hasReachableExplicitSpecialization( + const NamedDecl *D, llvm::SmallVectorImpl *Modules = nullptr); - const NamedDecl *getDecl() const { return ND; } + /// Determine if there is a visible declaration of \p D that is a member + /// specialization declaration (as opposed to an instantiated declaration). + bool hasVisibleMemberSpecialization( + const NamedDecl *D, llvm::SmallVectorImpl *Modules = nullptr); + /// Determine if there is a reachable declaration of \p D that is a member + /// specialization declaration (as opposed to an instantiated declaration). + bool hasReachableMemberSpecialization( + const NamedDecl *D, llvm::SmallVectorImpl *Modules = nullptr); - bool ContainsDecl(const NamedDecl *ND) const { return this->ND == ND; } + bool isModuleVisible(const Module *M, bool ModulePrivate = false); - const DeclContext *getLexicalDeclContext() const { - return ND ? ND->getLexicalDeclContext() : LexicalDC; - } + /// Determine whether any declaration of an entity is visible. + bool + hasVisibleDeclaration(const NamedDecl *D, + llvm::SmallVectorImpl *Modules = nullptr) { + return isVisible(D) || hasVisibleDeclarationSlow(D, Modules); + } - const DeclContext *getDeclContext() const { - return ND ? ND->getDeclContext() : DC; - } + bool hasVisibleDeclarationSlow(const NamedDecl *D, + llvm::SmallVectorImpl *Modules); + /// Determine whether any declaration of an entity is reachable. + bool + hasReachableDeclaration(const NamedDecl *D, + llvm::SmallVectorImpl *Modules = nullptr) { + return isReachable(D) || hasReachableDeclarationSlow(D, Modules); + } + bool hasReachableDeclarationSlow( + const NamedDecl *D, llvm::SmallVectorImpl *Modules = nullptr); - SourceLocation getLocation() const { return ND ? ND->getLocation() : Loc; } - }; + void diagnoseTypo(const TypoCorrection &Correction, + const PartialDiagnostic &TypoDiag, + bool ErrorRecovery = true); - bool TemplateParameterListsAreEqual( - const TemplateCompareNewDeclInfo &NewInstFrom, TemplateParameterList *New, - const NamedDecl *OldInstFrom, TemplateParameterList *Old, bool Complain, - TemplateParameterListEqualKind Kind, - SourceLocation TemplateArgLoc = SourceLocation()); + void diagnoseTypo(const TypoCorrection &Correction, + const PartialDiagnostic &TypoDiag, + const PartialDiagnostic &PrevNote, + bool ErrorRecovery = true); - bool TemplateParameterListsAreEqual( - TemplateParameterList *New, TemplateParameterList *Old, bool Complain, - TemplateParameterListEqualKind Kind, - SourceLocation TemplateArgLoc = SourceLocation()) { - return TemplateParameterListsAreEqual(nullptr, New, nullptr, Old, Complain, - Kind, TemplateArgLoc); - } - - // Calculates whether two constraint expressions are equal irrespective of a - // difference in 'depth'. This takes a pair of optional 'NamedDecl's 'Old' and - // 'New', which are the "source" of the constraint, since this is necessary - // for figuring out the relative 'depth' of the constraint. The depth of the - // 'primary template' and the 'instantiated from' templates aren't necessarily - // the same, such as a case when one is a 'friend' defined in a class. - bool AreConstraintExpressionsEqual(const NamedDecl *Old, - const Expr *OldConstr, - const TemplateCompareNewDeclInfo &New, - const Expr *NewConstr); + void FindAssociatedClassesAndNamespaces( + SourceLocation InstantiationLoc, ArrayRef Args, + AssociatedNamespaceSet &AssociatedNamespaces, + AssociatedClassSet &AssociatedClasses); - bool CheckTemplateDeclScope(Scope *S, TemplateParameterList *TemplateParams); + void DiagnoseAmbiguousLookup(LookupResult &Result); - /// Called when the parser has parsed a C++ typename - /// specifier, e.g., "typename T::type". - /// - /// \param S The scope in which this typename type occurs. - /// \param TypenameLoc the location of the 'typename' keyword - /// \param SS the nested-name-specifier following the typename (e.g., 'T::'). - /// \param II the identifier we're retrieving (e.g., 'type' in the example). - /// \param IdLoc the location of the identifier. - /// \param IsImplicitTypename context where T::type refers to a type. - TypeResult ActOnTypenameType( - Scope *S, SourceLocation TypenameLoc, const CXXScopeSpec &SS, - const IdentifierInfo &II, SourceLocation IdLoc, - ImplicitTypenameContext IsImplicitTypename = ImplicitTypenameContext::No); + LiteralOperatorLookupResult + LookupLiteralOperator(Scope *S, LookupResult &R, ArrayRef ArgTys, + bool AllowRaw, bool AllowTemplate, + bool AllowStringTemplate, bool DiagnoseMissing, + StringLiteral *StringLit = nullptr); - /// Called when the parser has parsed a C++ typename - /// specifier that ends in a template-id, e.g., - /// "typename MetaFun::template apply". - /// - /// \param S The scope in which this typename type occurs. - /// \param TypenameLoc the location of the 'typename' keyword - /// \param SS the nested-name-specifier following the typename (e.g., 'T::'). - /// \param TemplateLoc the location of the 'template' keyword, if any. - /// \param TemplateName The template name. - /// \param TemplateII The identifier used to name the template. - /// \param TemplateIILoc The location of the template name. - /// \param LAngleLoc The location of the opening angle bracket ('<'). - /// \param TemplateArgs The template arguments. - /// \param RAngleLoc The location of the closing angle bracket ('>'). - TypeResult - ActOnTypenameType(Scope *S, SourceLocation TypenameLoc, - const CXXScopeSpec &SS, - SourceLocation TemplateLoc, - TemplateTy TemplateName, - IdentifierInfo *TemplateII, - SourceLocation TemplateIILoc, - SourceLocation LAngleLoc, - ASTTemplateArgsPtr TemplateArgs, - SourceLocation RAngleLoc); + void ArgumentDependentLookup(DeclarationName Name, SourceLocation Loc, + ArrayRef Args, ADLResult &Functions); - QualType CheckTypenameType(ElaboratedTypeKeyword Keyword, - SourceLocation KeywordLoc, - NestedNameSpecifierLoc QualifierLoc, - const IdentifierInfo &II, - SourceLocation IILoc, - TypeSourceInfo **TSI, - bool DeducedTSTContext); + void LookupVisibleDecls(Scope *S, LookupNameKind Kind, + VisibleDeclConsumer &Consumer, + bool IncludeGlobalScope = true, + bool LoadExternal = true); + void LookupVisibleDecls(DeclContext *Ctx, LookupNameKind Kind, + VisibleDeclConsumer &Consumer, + bool IncludeGlobalScope = true, + bool IncludeDependentBases = false, + bool LoadExternal = true); - QualType CheckTypenameType(ElaboratedTypeKeyword Keyword, - SourceLocation KeywordLoc, - NestedNameSpecifierLoc QualifierLoc, - const IdentifierInfo &II, - SourceLocation IILoc, - bool DeducedTSTContext = true); + enum CorrectTypoKind { + CTK_NonError, // CorrectTypo used in a non error recovery situation. + CTK_ErrorRecovery // CorrectTypo used in normal error recovery. + }; + TypoCorrection CorrectTypo(const DeclarationNameInfo &Typo, + Sema::LookupNameKind LookupKind, Scope *S, + CXXScopeSpec *SS, CorrectionCandidateCallback &CCC, + CorrectTypoKind Mode, + DeclContext *MemberContext = nullptr, + bool EnteringContext = false, + const ObjCObjectPointerType *OPT = nullptr, + bool RecordFailure = true); - TypeSourceInfo *RebuildTypeInCurrentInstantiation(TypeSourceInfo *T, - SourceLocation Loc, - DeclarationName Name); - bool RebuildNestedNameSpecifierInCurrentInstantiation(CXXScopeSpec &SS); + TypoExpr *CorrectTypoDelayed( + const DeclarationNameInfo &Typo, Sema::LookupNameKind LookupKind, + Scope *S, CXXScopeSpec *SS, CorrectionCandidateCallback &CCC, + TypoDiagnosticGenerator TDG, TypoRecoveryCallback TRC, + CorrectTypoKind Mode, DeclContext *MemberContext = nullptr, + bool EnteringContext = false, const ObjCObjectPointerType *OPT = nullptr); - ExprResult RebuildExprInCurrentInstantiation(Expr *E); - bool RebuildTemplateParamsInCurrentInstantiation( - TemplateParameterList *Params); + /// Kinds of missing import. Note, the values of these enumerators correspond + /// to %select values in diagnostics. + enum class MissingImportKind { + Declaration, + Definition, + DefaultArgument, + ExplicitSpecialization, + PartialSpecialization + }; - std::string - getTemplateArgumentBindingsText(const TemplateParameterList *Params, - const TemplateArgumentList &Args); + /// Diagnose that the specified declaration needs to be visible but + /// isn't, and suggest a module import that would resolve the problem. + void diagnoseMissingImport(SourceLocation Loc, const NamedDecl *Decl, + MissingImportKind MIK, bool Recover = true); + void diagnoseMissingImport(SourceLocation Loc, const NamedDecl *Decl, + SourceLocation DeclLoc, ArrayRef Modules, + MissingImportKind MIK, bool Recover); - std::string - getTemplateArgumentBindingsText(const TemplateParameterList *Params, - const TemplateArgument *Args, - unsigned NumArgs); + struct TypoExprState { + std::unique_ptr Consumer; + TypoDiagnosticGenerator DiagHandler; + TypoRecoveryCallback RecoveryHandler; + TypoExprState(); + TypoExprState(TypoExprState &&other) noexcept; + TypoExprState &operator=(TypoExprState &&other) noexcept; + }; - //===--------------------------------------------------------------------===// - // C++ Concepts - //===--------------------------------------------------------------------===// - Decl *ActOnConceptDefinition( - Scope *S, MultiTemplateParamsArg TemplateParameterLists, - IdentifierInfo *Name, SourceLocation NameLoc, Expr *ConstraintExpr); + const TypoExprState &getTypoExprState(TypoExpr *TE) const; - void CheckConceptRedefinition(ConceptDecl *NewDecl, LookupResult &Previous, - bool &AddToScope); + /// Clears the state of the given TypoExpr. + void clearDelayedTypo(TypoExpr *TE); - RequiresExprBodyDecl * - ActOnStartRequiresExpr(SourceLocation RequiresKWLoc, - ArrayRef LocalParameters, - Scope *BodyScope); - void ActOnFinishRequiresExpr(); - concepts::Requirement *ActOnSimpleRequirement(Expr *E); - concepts::Requirement *ActOnTypeRequirement( - SourceLocation TypenameKWLoc, CXXScopeSpec &SS, SourceLocation NameLoc, - IdentifierInfo *TypeName, TemplateIdAnnotation *TemplateId); - concepts::Requirement *ActOnCompoundRequirement(Expr *E, - SourceLocation NoexceptLoc); - concepts::Requirement * - ActOnCompoundRequirement( - Expr *E, SourceLocation NoexceptLoc, CXXScopeSpec &SS, - TemplateIdAnnotation *TypeConstraint, unsigned Depth); - concepts::Requirement *ActOnNestedRequirement(Expr *Constraint); - concepts::ExprRequirement * - BuildExprRequirement( - Expr *E, bool IsSatisfied, SourceLocation NoexceptLoc, - concepts::ExprRequirement::ReturnTypeRequirement ReturnTypeRequirement); - concepts::ExprRequirement * - BuildExprRequirement( - concepts::Requirement::SubstitutionDiagnostic *ExprSubstDiag, - bool IsSatisfied, SourceLocation NoexceptLoc, - concepts::ExprRequirement::ReturnTypeRequirement ReturnTypeRequirement); - concepts::TypeRequirement *BuildTypeRequirement(TypeSourceInfo *Type); - concepts::TypeRequirement * - BuildTypeRequirement( - concepts::Requirement::SubstitutionDiagnostic *SubstDiag); - concepts::NestedRequirement *BuildNestedRequirement(Expr *E); - concepts::NestedRequirement * - BuildNestedRequirement(StringRef InvalidConstraintEntity, - const ASTConstraintSatisfaction &Satisfaction); - ExprResult ActOnRequiresExpr(SourceLocation RequiresKWLoc, - RequiresExprBodyDecl *Body, - SourceLocation LParenLoc, - ArrayRef LocalParameters, - SourceLocation RParenLoc, - ArrayRef Requirements, - SourceLocation ClosingBraceLoc); + /// Called on #pragma clang __debug dump II + void ActOnPragmaDump(Scope *S, SourceLocation Loc, IdentifierInfo *II); - //===--------------------------------------------------------------------===// - // C++ Variadic Templates (C++0x [temp.variadic]) - //===--------------------------------------------------------------------===// + /// Called on #pragma clang __debug dump E + void ActOnPragmaDump(Expr *E); - /// Determine whether an unexpanded parameter pack might be permitted in this - /// location. Useful for error recovery. - bool isUnexpandedParameterPackPermitted(); +private: + // The set of known/encountered (unique, canonicalized) NamespaceDecls. + // + // The boolean value will be true to indicate that the namespace was loaded + // from an AST/PCH file, or false otherwise. + llvm::MapVector KnownNamespaces; - /// The context in which an unexpanded parameter pack is - /// being diagnosed. - /// - /// Note that the values of this enumeration line up with the first - /// argument to the \c err_unexpanded_parameter_pack diagnostic. - enum UnexpandedParameterPackContext { - /// An arbitrary expression. - UPPC_Expression = 0, + /// Whether we have already loaded known namespaces from an extenal + /// source. + bool LoadedExternalKnownNamespaces; - /// The base type of a class type. - UPPC_BaseType, + bool CppLookupName(LookupResult &R, Scope *S); - /// The type of an arbitrary declaration. - UPPC_DeclarationType, + bool isUsableModule(const Module *M); - /// The type of a data member. - UPPC_DataMemberType, + /// Helper for CorrectTypo and CorrectTypoDelayed used to create and + /// populate a new TypoCorrectionConsumer. Returns nullptr if typo correction + /// should be skipped entirely. + std::unique_ptr makeTypoCorrectionConsumer( + const DeclarationNameInfo &Typo, Sema::LookupNameKind LookupKind, + Scope *S, CXXScopeSpec *SS, CorrectionCandidateCallback &CCC, + DeclContext *MemberContext, bool EnteringContext, + const ObjCObjectPointerType *OPT, bool ErrorRecovery); - /// The size of a bit-field. - UPPC_BitFieldWidth, + /// The set of unhandled TypoExprs and their associated state. + llvm::MapVector DelayedTypos; - /// The expression in a static assertion. - UPPC_StaticAssertExpression, + /// Creates a new TypoExpr AST node. + TypoExpr *createDelayedTypo(std::unique_ptr TCC, + TypoDiagnosticGenerator TDG, + TypoRecoveryCallback TRC, SourceLocation TypoLoc); - /// The fixed underlying type of an enumeration. - UPPC_FixedUnderlyingType, + /// Cache for module units which is usable for current module. + llvm::DenseSet UsableModuleUnitsCache; - /// The enumerator value. - UPPC_EnumeratorValue, + /// Record the typo correction failure and return an empty correction. + TypoCorrection FailedCorrection(IdentifierInfo *Typo, SourceLocation TypoLoc, + bool RecordFailure = true) { + if (RecordFailure) + TypoCorrectionFailures[Typo].insert(TypoLoc); + return TypoCorrection(); + } - /// A using declaration. - UPPC_UsingDeclaration, + bool isAcceptableSlow(const NamedDecl *D, AcceptableKind Kind); - /// A friend declaration. - UPPC_FriendDeclaration, + /// Determine whether two declarations should be linked together, given that + /// the old declaration might not be visible and the new declaration might + /// not have external linkage. + bool shouldLinkPossiblyHiddenDecl(const NamedDecl *Old, + const NamedDecl *New) { + if (isVisible(Old)) + return true; + // See comment in below overload for why it's safe to compute the linkage + // of the new declaration here. + if (New->isExternallyDeclarable()) { + assert(Old->isExternallyDeclarable() && + "should not have found a non-externally-declarable previous decl"); + return true; + } + return false; + } + bool shouldLinkPossiblyHiddenDecl(LookupResult &Old, const NamedDecl *New); - /// A declaration qualifier. - UPPC_DeclarationQualifier, + ///@} - /// An initializer. - UPPC_Initializer, + // + // + // ------------------------------------------------------------------------- + // + // - /// A default argument. - UPPC_DefaultArgument, + /// \name Modules + /// Implementations are in SemaModule.cpp + ///@{ - /// The type of a non-type template parameter. - UPPC_NonTypeTemplateParameterType, +public: + /// Get the module unit whose scope we are currently within. + Module *getCurrentModule() const { + return ModuleScopes.empty() ? nullptr : ModuleScopes.back().Module; + } - /// The type of an exception. - UPPC_ExceptionType, + /// Is the module scope we are an implementation unit? + bool currentModuleIsImplementation() const { + return ModuleScopes.empty() + ? false + : ModuleScopes.back().Module->isModuleImplementation(); + } - /// Explicit specialization. - UPPC_ExplicitSpecialization, + // When loading a non-modular PCH files, this is used to restore module + // visibility. + void makeModuleVisible(Module *Mod, SourceLocation ImportLoc) { + VisibleModules.setVisible(Mod, ImportLoc); + } - /// Partial specialization. - UPPC_PartialSpecialization, + enum class ModuleDeclKind { + Interface, ///< 'export module X;' + Implementation, ///< 'module X;' + PartitionInterface, ///< 'export module X:Y;' + PartitionImplementation, ///< 'module X:Y;' + }; - /// Microsoft __if_exists. - UPPC_IfExists, + /// An enumeration to represent the transition of states in parsing module + /// fragments and imports. If we are not parsing a C++20 TU, or we find + /// an error in state transition, the state is set to NotACXX20Module. + enum class ModuleImportState { + FirstDecl, ///< Parsing the first decl in a TU. + GlobalFragment, ///< after 'module;' but before 'module X;' + ImportAllowed, ///< after 'module X;' but before any non-import decl. + ImportFinished, ///< after any non-import decl. + PrivateFragmentImportAllowed, ///< after 'module :private;' but before any + ///< non-import decl. + PrivateFragmentImportFinished, ///< after 'module :private;' but a + ///< non-import decl has already been seen. + NotACXX20Module ///< Not a C++20 TU, or an invalid state was found. + }; - /// Microsoft __if_not_exists. - UPPC_IfNotExists, + /// The parser has processed a module-declaration that begins the definition + /// of a module interface or implementation. + DeclGroupPtrTy ActOnModuleDecl(SourceLocation StartLoc, + SourceLocation ModuleLoc, ModuleDeclKind MDK, + ModuleIdPath Path, ModuleIdPath Partition, + ModuleImportState &ImportState); - /// Lambda expression. - UPPC_Lambda, + /// The parser has processed a global-module-fragment declaration that begins + /// the definition of the global module fragment of the current module unit. + /// \param ModuleLoc The location of the 'module' keyword. + DeclGroupPtrTy ActOnGlobalModuleFragmentDecl(SourceLocation ModuleLoc); - /// Block expression. - UPPC_Block, + /// The parser has processed a private-module-fragment declaration that begins + /// the definition of the private module fragment of the current module unit. + /// \param ModuleLoc The location of the 'module' keyword. + /// \param PrivateLoc The location of the 'private' keyword. + DeclGroupPtrTy ActOnPrivateModuleFragmentDecl(SourceLocation ModuleLoc, + SourceLocation PrivateLoc); - /// A type constraint. - UPPC_TypeConstraint, + /// The parser has processed a module import declaration. + /// + /// \param StartLoc The location of the first token in the declaration. This + /// could be the location of an '@', 'export', or 'import'. + /// \param ExportLoc The location of the 'export' keyword, if any. + /// \param ImportLoc The location of the 'import' keyword. + /// \param Path The module toplevel name as an access path. + /// \param IsPartition If the name is for a partition. + DeclResult ActOnModuleImport(SourceLocation StartLoc, + SourceLocation ExportLoc, + SourceLocation ImportLoc, ModuleIdPath Path, + bool IsPartition = false); + DeclResult ActOnModuleImport(SourceLocation StartLoc, + SourceLocation ExportLoc, + SourceLocation ImportLoc, Module *M, + ModuleIdPath Path = {}); - // A requirement in a requires-expression. - UPPC_Requirement, + /// The parser has processed a module import translated from a + /// #include or similar preprocessing directive. + void ActOnModuleInclude(SourceLocation DirectiveLoc, Module *Mod); + void BuildModuleInclude(SourceLocation DirectiveLoc, Module *Mod); - // A requires-clause. - UPPC_RequiresClause, - }; + /// The parsed has entered a submodule. + void ActOnModuleBegin(SourceLocation DirectiveLoc, Module *Mod); + /// The parser has left a submodule. + void ActOnModuleEnd(SourceLocation DirectiveLoc, Module *Mod); - /// Diagnose unexpanded parameter packs. - /// - /// \param Loc The location at which we should emit the diagnostic. - /// - /// \param UPPC The context in which we are diagnosing unexpanded - /// parameter packs. - /// - /// \param Unexpanded the set of unexpanded parameter packs. + /// Create an implicit import of the given module at the given + /// source location, for error recovery, if possible. /// - /// \returns true if an error occurred, false otherwise. - bool DiagnoseUnexpandedParameterPacks(SourceLocation Loc, - UnexpandedParameterPackContext UPPC, - ArrayRef Unexpanded); + /// This routine is typically used when an entity found by name lookup + /// is actually hidden within a module that we know about but the user + /// has forgotten to import. + void createImplicitModuleImportForErrorRecovery(SourceLocation Loc, + Module *Mod); - /// If the given type contains an unexpanded parameter pack, - /// diagnose the error. - /// - /// \param Loc The source location where a diagnostc should be emitted. - /// - /// \param T The type that is being checked for unexpanded parameter - /// packs. - /// - /// \returns true if an error occurred, false otherwise. - bool DiagnoseUnexpandedParameterPack(SourceLocation Loc, TypeSourceInfo *T, - UnexpandedParameterPackContext UPPC); + Decl *ActOnStartExportDecl(Scope *S, SourceLocation ExportLoc, + SourceLocation LBraceLoc); + Decl *ActOnFinishExportDecl(Scope *S, Decl *ExportDecl, + SourceLocation RBraceLoc); - /// If the given expression contains an unexpanded parameter - /// pack, diagnose the error. - /// - /// \param E The expression that is being checked for unexpanded - /// parameter packs. - /// - /// \returns true if an error occurred, false otherwise. - bool DiagnoseUnexpandedParameterPack(Expr *E, - UnexpandedParameterPackContext UPPC = UPPC_Expression); +private: + /// The parser has begun a translation unit to be compiled as a C++20 + /// Header Unit, helper for ActOnStartOfTranslationUnit() only. + void HandleStartOfHeaderUnit(); - /// If the given requirees-expression contains an unexpanded reference to one - /// of its own parameter packs, diagnose the error. - /// - /// \param RE The requiress-expression that is being checked for unexpanded - /// parameter packs. - /// - /// \returns true if an error occurred, false otherwise. - bool DiagnoseUnexpandedParameterPackInRequiresExpr(RequiresExpr *RE); + struct ModuleScope { + SourceLocation BeginLoc; + clang::Module *Module = nullptr; + VisibleModuleSet OuterVisibleModules; + }; + /// The modules we're currently parsing. + llvm::SmallVector ModuleScopes; - /// If the given nested-name-specifier contains an unexpanded - /// parameter pack, diagnose the error. - /// - /// \param SS The nested-name-specifier that is being checked for - /// unexpanded parameter packs. - /// - /// \returns true if an error occurred, false otherwise. - bool DiagnoseUnexpandedParameterPack(const CXXScopeSpec &SS, - UnexpandedParameterPackContext UPPC); + /// For an interface unit, this is the implicitly imported interface unit. + clang::Module *ThePrimaryInterface = nullptr; - /// If the given name contains an unexpanded parameter pack, - /// diagnose the error. - /// - /// \param NameInfo The name (with source location information) that - /// is being checked for unexpanded parameter packs. - /// - /// \returns true if an error occurred, false otherwise. - bool DiagnoseUnexpandedParameterPack(const DeclarationNameInfo &NameInfo, - UnexpandedParameterPackContext UPPC); + /// The explicit global module fragment of the current translation unit. + /// The explicit Global Module Fragment, as specified in C++ + /// [module.global.frag]. + clang::Module *TheGlobalModuleFragment = nullptr; - /// If the given template name contains an unexpanded parameter pack, - /// diagnose the error. - /// - /// \param Loc The location of the template name. - /// - /// \param Template The template name that is being checked for unexpanded - /// parameter packs. + /// The implicit global module fragments of the current translation unit. /// - /// \returns true if an error occurred, false otherwise. - bool DiagnoseUnexpandedParameterPack(SourceLocation Loc, - TemplateName Template, - UnexpandedParameterPackContext UPPC); + /// The contents in the implicit global module fragment can't be discarded. + clang::Module *TheImplicitGlobalModuleFragment = nullptr; - /// If the given template argument contains an unexpanded parameter - /// pack, diagnose the error. - /// - /// \param Arg The template argument that is being checked for unexpanded - /// parameter packs. - /// - /// \returns true if an error occurred, false otherwise. - bool DiagnoseUnexpandedParameterPack(TemplateArgumentLoc Arg, - UnexpandedParameterPackContext UPPC); + /// Namespace definitions that we will export when they finish. + llvm::SmallPtrSet DeferredExportedNamespaces; - /// Collect the set of unexpanded parameter packs within the given - /// template argument. - /// - /// \param Arg The template argument that will be traversed to find - /// unexpanded parameter packs. - void collectUnexpandedParameterPacks(TemplateArgument Arg, - SmallVectorImpl &Unexpanded); + /// In a C++ standard module, inline declarations require a definition to be + /// present at the end of a definition domain. This set holds the decls to + /// be checked at the end of the TU. + llvm::SmallPtrSet PendingInlineFuncDecls; - /// Collect the set of unexpanded parameter packs within the given - /// template argument. - /// - /// \param Arg The template argument that will be traversed to find - /// unexpanded parameter packs. - void collectUnexpandedParameterPacks(TemplateArgumentLoc Arg, - SmallVectorImpl &Unexpanded); + /// Helper function to judge if we are in module purview. + /// Return false if we are not in a module. + bool isCurrentModulePurview() const; - /// Collect the set of unexpanded parameter packs within the given - /// type. - /// - /// \param T The type that will be traversed to find - /// unexpanded parameter packs. - void collectUnexpandedParameterPacks(QualType T, - SmallVectorImpl &Unexpanded); + /// Enter the scope of the explicit global module fragment. + Module *PushGlobalModuleFragment(SourceLocation BeginLoc); + /// Leave the scope of the explicit global module fragment. + void PopGlobalModuleFragment(); - /// Collect the set of unexpanded parameter packs within the given - /// type. - /// - /// \param TL The type that will be traversed to find - /// unexpanded parameter packs. - void collectUnexpandedParameterPacks(TypeLoc TL, - SmallVectorImpl &Unexpanded); + /// Enter the scope of an implicit global module fragment. + Module *PushImplicitGlobalModuleFragment(SourceLocation BeginLoc); + /// Leave the scope of an implicit global module fragment. + void PopImplicitGlobalModuleFragment(); - /// Collect the set of unexpanded parameter packs within the given - /// nested-name-specifier. - /// - /// \param NNS The nested-name-specifier that will be traversed to find - /// unexpanded parameter packs. - void collectUnexpandedParameterPacks(NestedNameSpecifierLoc NNS, - SmallVectorImpl &Unexpanded); + VisibleModuleSet VisibleModules; - /// Collect the set of unexpanded parameter packs within the given - /// name. - /// - /// \param NameInfo The name that will be traversed to find - /// unexpanded parameter packs. - void collectUnexpandedParameterPacks(const DeclarationNameInfo &NameInfo, - SmallVectorImpl &Unexpanded); + ///@} - /// Invoked when parsing a template argument followed by an - /// ellipsis, which creates a pack expansion. - /// - /// \param Arg The template argument preceding the ellipsis, which - /// may already be invalid. - /// - /// \param EllipsisLoc The location of the ellipsis. - ParsedTemplateArgument ActOnPackExpansion(const ParsedTemplateArgument &Arg, - SourceLocation EllipsisLoc); + // + // + // ------------------------------------------------------------------------- + // + // - /// Invoked when parsing a type followed by an ellipsis, which - /// creates a pack expansion. - /// - /// \param Type The type preceding the ellipsis, which will become - /// the pattern of the pack expansion. - /// - /// \param EllipsisLoc The location of the ellipsis. - TypeResult ActOnPackExpansion(ParsedType Type, SourceLocation EllipsisLoc); + /// \name C++ Overloading + /// Implementations are in SemaOverload.cpp + ///@{ - /// Construct a pack expansion type from the pattern of the pack - /// expansion. - TypeSourceInfo *CheckPackExpansion(TypeSourceInfo *Pattern, - SourceLocation EllipsisLoc, - std::optional NumExpansions); - - /// Construct a pack expansion type from the pattern of the pack - /// expansion. - QualType CheckPackExpansion(QualType Pattern, SourceRange PatternRange, - SourceLocation EllipsisLoc, - std::optional NumExpansions); +public: + /// Whether deferrable diagnostics should be deferred. + bool DeferDiags = false; - /// Invoked when parsing an expression followed by an ellipsis, which - /// creates a pack expansion. - /// - /// \param Pattern The expression preceding the ellipsis, which will become - /// the pattern of the pack expansion. - /// - /// \param EllipsisLoc The location of the ellipsis. - ExprResult ActOnPackExpansion(Expr *Pattern, SourceLocation EllipsisLoc); + /// RAII class to control scope of DeferDiags. + class DeferDiagsRAII { + Sema &S; + bool SavedDeferDiags = false; - /// Invoked when parsing an expression followed by an ellipsis, which - /// creates a pack expansion. - /// - /// \param Pattern The expression preceding the ellipsis, which will become - /// the pattern of the pack expansion. - /// - /// \param EllipsisLoc The location of the ellipsis. - ExprResult CheckPackExpansion(Expr *Pattern, SourceLocation EllipsisLoc, - std::optional NumExpansions); + public: + DeferDiagsRAII(Sema &S, bool DeferDiags) + : S(S), SavedDeferDiags(S.DeferDiags) { + S.DeferDiags = DeferDiags; + } + ~DeferDiagsRAII() { S.DeferDiags = SavedDeferDiags; } + }; - /// Determine whether we could expand a pack expansion with the - /// given set of parameter packs into separate arguments by repeatedly - /// transforming the pattern. - /// - /// \param EllipsisLoc The location of the ellipsis that identifies the - /// pack expansion. - /// - /// \param PatternRange The source range that covers the entire pattern of - /// the pack expansion. - /// - /// \param Unexpanded The set of unexpanded parameter packs within the - /// pattern. - /// - /// \param ShouldExpand Will be set to \c true if the transformer should - /// expand the corresponding pack expansions into separate arguments. When - /// set, \c NumExpansions must also be set. - /// - /// \param RetainExpansion Whether the caller should add an unexpanded - /// pack expansion after all of the expanded arguments. This is used - /// when extending explicitly-specified template argument packs per - /// C++0x [temp.arg.explicit]p9. - /// - /// \param NumExpansions The number of separate arguments that will be in - /// the expanded form of the corresponding pack expansion. This is both an - /// input and an output parameter, which can be set by the caller if the - /// number of expansions is known a priori (e.g., due to a prior substitution) - /// and will be set by the callee when the number of expansions is known. - /// The callee must set this value when \c ShouldExpand is \c true; it may - /// set this value in other cases. + /// Flag indicating if Sema is building a recovery call expression. /// - /// \returns true if an error occurred (e.g., because the parameter packs - /// are to be instantiated with arguments of different lengths), false - /// otherwise. If false, \c ShouldExpand (and possibly \c NumExpansions) - /// must be set. - bool CheckParameterPacksForExpansion( - SourceLocation EllipsisLoc, SourceRange PatternRange, - ArrayRef Unexpanded, - const MultiLevelTemplateArgumentList &TemplateArgs, bool &ShouldExpand, - bool &RetainExpansion, std::optional &NumExpansions); + /// This flag is used to avoid building recovery call expressions + /// if Sema is already doing so, which would cause infinite recursions. + bool IsBuildingRecoveryCallExpr; - /// Determine the number of arguments in the given pack expansion - /// type. - /// - /// This routine assumes that the number of arguments in the expansion is - /// consistent across all of the unexpanded parameter packs in its pattern. - /// - /// Returns an empty Optional if the type can't be expanded. - std::optional getNumArgumentsInExpansion( - QualType T, const MultiLevelTemplateArgumentList &TemplateArgs); + enum OverloadKind { + /// This is a legitimate overload: the existing declarations are + /// functions or function templates with different signatures. + Ovl_Overload, - /// Determine whether the given declarator contains any unexpanded - /// parameter packs. - /// - /// This routine is used by the parser to disambiguate function declarators - /// with an ellipsis prior to the ')', e.g., - /// - /// \code - /// void f(T...); - /// \endcode - /// - /// To determine whether we have an (unnamed) function parameter pack or - /// a variadic function. - /// - /// \returns true if the declarator contains any unexpanded parameter packs, - /// false otherwise. - bool containsUnexpandedParameterPacks(Declarator &D); + /// This is not an overload because the signature exactly matches + /// an existing declaration. + Ovl_Match, - /// Returns the pattern of the pack expansion for a template argument. - /// - /// \param OrigLoc The template argument to expand. - /// - /// \param Ellipsis Will be set to the location of the ellipsis. - /// - /// \param NumExpansions Will be set to the number of expansions that will - /// be generated from this pack expansion, if known a priori. - TemplateArgumentLoc getTemplateArgumentPackExpansionPattern( - TemplateArgumentLoc OrigLoc, SourceLocation &Ellipsis, - std::optional &NumExpansions) const; + /// This is not an overload because the lookup results contain a + /// non-function. + Ovl_NonFunction + }; + OverloadKind CheckOverload(Scope *S, FunctionDecl *New, + const LookupResult &OldDecls, NamedDecl *&OldDecl, + bool UseMemberUsingDeclRules); + bool IsOverload(FunctionDecl *New, FunctionDecl *Old, + bool UseMemberUsingDeclRules, bool ConsiderCudaAttrs = true); - /// Given a template argument that contains an unexpanded parameter pack, but - /// which has already been substituted, attempt to determine the number of - /// elements that will be produced once this argument is fully-expanded. - /// - /// This is intended for use when transforming 'sizeof...(Arg)' in order to - /// avoid actually expanding the pack where possible. - std::optional getFullyPackExpandedSize(TemplateArgument Arg); + // Checks whether MD constitutes an override the base class method BaseMD. + // When checking for overrides, the object object members are ignored. + bool IsOverride(FunctionDecl *MD, FunctionDecl *BaseMD, + bool UseMemberUsingDeclRules, bool ConsiderCudaAttrs = true); - //===--------------------------------------------------------------------===// - // C++ Template Argument Deduction (C++ [temp.deduct]) - //===--------------------------------------------------------------------===// + enum class AllowedExplicit { + /// Allow no explicit functions to be used. + None, + /// Allow explicit conversion functions but not explicit constructors. + Conversions, + /// Allow both explicit conversion functions and explicit constructors. + All + }; - /// Adjust the type \p ArgFunctionType to match the calling convention, - /// noreturn, and optionally the exception specification of \p FunctionType. - /// Deduction often wants to ignore these properties when matching function - /// types. - QualType adjustCCAndNoReturn(QualType ArgFunctionType, QualType FunctionType, - bool AdjustExceptionSpec = false); + ImplicitConversionSequence TryImplicitConversion( + Expr *From, QualType ToType, bool SuppressUserConversions, + AllowedExplicit AllowExplicit, bool InOverloadResolution, bool CStyle, + bool AllowObjCWritebackConversion); - TemplateDeductionResult - DeduceTemplateArguments(ClassTemplatePartialSpecializationDecl *Partial, - ArrayRef TemplateArgs, - sema::TemplateDeductionInfo &Info); + ExprResult PerformImplicitConversion(Expr *From, QualType ToType, + AssignmentAction Action, + bool AllowExplicit = false); - TemplateDeductionResult - DeduceTemplateArguments(VarTemplatePartialSpecializationDecl *Partial, - ArrayRef TemplateArgs, - sema::TemplateDeductionInfo &Info); + bool IsIntegralPromotion(Expr *From, QualType FromType, QualType ToType); + bool IsFloatingPointPromotion(QualType FromType, QualType ToType); + bool IsComplexPromotion(QualType FromType, QualType ToType); + bool IsPointerConversion(Expr *From, QualType FromType, QualType ToType, + bool InOverloadResolution, QualType &ConvertedType, + bool &IncompatibleObjC); + bool isObjCPointerConversion(QualType FromType, QualType ToType, + QualType &ConvertedType, bool &IncompatibleObjC); + bool isObjCWritebackConversion(QualType FromType, QualType ToType, + QualType &ConvertedType); + bool IsBlockPointerConversion(QualType FromType, QualType ToType, + QualType &ConvertedType); - TemplateDeductionResult SubstituteExplicitTemplateArguments( - FunctionTemplateDecl *FunctionTemplate, - TemplateArgumentListInfo &ExplicitTemplateArgs, - SmallVectorImpl &Deduced, - SmallVectorImpl &ParamTypes, QualType *FunctionType, - sema::TemplateDeductionInfo &Info); + bool FunctionParamTypesAreEqual(ArrayRef Old, + ArrayRef New, + unsigned *ArgPos = nullptr, + bool Reversed = false); - /// brief A function argument from which we performed template argument - // deduction for a call. - struct OriginalCallArg { - OriginalCallArg(QualType OriginalParamType, bool DecomposedParam, - unsigned ArgIdx, QualType OriginalArgType) - : OriginalParamType(OriginalParamType), - DecomposedParam(DecomposedParam), ArgIdx(ArgIdx), - OriginalArgType(OriginalArgType) {} + bool FunctionParamTypesAreEqual(const FunctionProtoType *OldType, + const FunctionProtoType *NewType, + unsigned *ArgPos = nullptr, + bool Reversed = false); - QualType OriginalParamType; - bool DecomposedParam; - unsigned ArgIdx; - QualType OriginalArgType; - }; + bool FunctionNonObjectParamTypesAreEqual(const FunctionDecl *OldFunction, + const FunctionDecl *NewFunction, + unsigned *ArgPos = nullptr, + bool Reversed = false); - TemplateDeductionResult FinishTemplateArgumentDeduction( - FunctionTemplateDecl *FunctionTemplate, - SmallVectorImpl &Deduced, - unsigned NumExplicitlySpecified, FunctionDecl *&Specialization, - sema::TemplateDeductionInfo &Info, - SmallVectorImpl const *OriginalCallArgs = nullptr, - bool PartialOverloading = false, - llvm::function_ref CheckNonDependent = []{ return false; }); + void HandleFunctionTypeMismatch(PartialDiagnostic &PDiag, QualType FromType, + QualType ToType); - TemplateDeductionResult DeduceTemplateArguments( - FunctionTemplateDecl *FunctionTemplate, - TemplateArgumentListInfo *ExplicitTemplateArgs, ArrayRef Args, - FunctionDecl *&Specialization, sema::TemplateDeductionInfo &Info, - bool PartialOverloading, bool AggregateDeductionCandidate, - QualType ObjectType, Expr::Classification ObjectClassification, - llvm::function_ref)> CheckNonDependent); + bool CheckPointerConversion(Expr *From, QualType ToType, CastKind &Kind, + CXXCastPath &BasePath, bool IgnoreBaseAccess, + bool Diagnose = true); + bool IsMemberPointerConversion(Expr *From, QualType FromType, QualType ToType, + bool InOverloadResolution, + QualType &ConvertedType); + bool CheckMemberPointerConversion(Expr *From, QualType ToType, CastKind &Kind, + CXXCastPath &BasePath, + bool IgnoreBaseAccess); + bool IsQualificationConversion(QualType FromType, QualType ToType, + bool CStyle, bool &ObjCLifetimeConversion); + bool IsFunctionConversion(QualType FromType, QualType ToType, + QualType &ResultTy); + bool DiagnoseMultipleUserDefinedConversion(Expr *From, QualType ToType); - TemplateDeductionResult - DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate, - TemplateArgumentListInfo *ExplicitTemplateArgs, - QualType ArgFunctionType, - FunctionDecl *&Specialization, - sema::TemplateDeductionInfo &Info, - bool IsAddressOfFunction = false); + ExprResult InitializeExplicitObjectArgument(Sema &S, Expr *Obj, + FunctionDecl *Fun); + ExprResult PerformImplicitObjectArgumentInitialization( + Expr *From, NestedNameSpecifier *Qualifier, NamedDecl *FoundDecl, + CXXMethodDecl *Method); - TemplateDeductionResult DeduceTemplateArguments( - FunctionTemplateDecl *FunctionTemplate, QualType ObjectType, - Expr::Classification ObjectClassification, QualType ToType, - CXXConversionDecl *&Specialization, sema::TemplateDeductionInfo &Info); + ExprResult PerformContextuallyConvertToBool(Expr *From); + ExprResult PerformContextuallyConvertToObjCPointer(Expr *From); - TemplateDeductionResult - DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate, - TemplateArgumentListInfo *ExplicitTemplateArgs, - FunctionDecl *&Specialization, - sema::TemplateDeductionInfo &Info, - bool IsAddressOfFunction = false); - - /// Substitute Replacement for \p auto in \p TypeWithAuto - QualType SubstAutoType(QualType TypeWithAuto, QualType Replacement); - /// Substitute Replacement for auto in TypeWithAuto - TypeSourceInfo* SubstAutoTypeSourceInfo(TypeSourceInfo *TypeWithAuto, - QualType Replacement); - - // Substitute auto in TypeWithAuto for a Dependent auto type - QualType SubstAutoTypeDependent(QualType TypeWithAuto); - - // Substitute auto in TypeWithAuto for a Dependent auto type - TypeSourceInfo * - SubstAutoTypeSourceInfoDependent(TypeSourceInfo *TypeWithAuto); - - /// Completely replace the \c auto in \p TypeWithAuto by - /// \p Replacement. This does not retain any \c auto type sugar. - QualType ReplaceAutoType(QualType TypeWithAuto, QualType Replacement); - TypeSourceInfo *ReplaceAutoTypeSourceInfo(TypeSourceInfo *TypeWithAuto, - QualType Replacement); - - TemplateDeductionResult - DeduceAutoType(TypeLoc AutoTypeLoc, Expr *Initializer, QualType &Result, - sema::TemplateDeductionInfo &Info, - bool DependentDeduction = false, - bool IgnoreConstraints = false, - TemplateSpecCandidateSet *FailedTSC = nullptr); - void DiagnoseAutoDeductionFailure(VarDecl *VDecl, Expr *Init); - bool DeduceReturnType(FunctionDecl *FD, SourceLocation Loc, - bool Diagnose = true); - - bool CheckIfFunctionSpecializationIsImmediate(FunctionDecl *FD, - SourceLocation Loc); - - /// Declare implicit deduction guides for a class template if we've - /// not already done so. - void DeclareImplicitDeductionGuides(TemplateDecl *Template, - SourceLocation Loc); - FunctionTemplateDecl *DeclareImplicitDeductionGuideFromInitList( - TemplateDecl *Template, MutableArrayRef ParamTypes, - SourceLocation Loc); - llvm::DenseMap - AggregateDeductionCandidates; - - QualType DeduceTemplateSpecializationFromInitializer( - TypeSourceInfo *TInfo, const InitializedEntity &Entity, - const InitializationKind &Kind, MultiExprArg Init); - - QualType deduceVarTypeFromInitializer(VarDecl *VDecl, DeclarationName Name, - QualType Type, TypeSourceInfo *TSI, - SourceRange Range, bool DirectInit, - Expr *Init); - - TypeLoc getReturnTypeLoc(FunctionDecl *FD) const; - - bool DeduceFunctionTypeFromReturnExpr(FunctionDecl *FD, - SourceLocation ReturnLoc, Expr *RetExpr, - const AutoType *AT); - - FunctionTemplateDecl *getMoreSpecializedTemplate( - FunctionTemplateDecl *FT1, FunctionTemplateDecl *FT2, SourceLocation Loc, - TemplatePartialOrderingContext TPOC, unsigned NumCallArguments1, - unsigned NumCallArguments2, bool Reversed = false); - UnresolvedSetIterator - getMostSpecialized(UnresolvedSetIterator SBegin, UnresolvedSetIterator SEnd, - TemplateSpecCandidateSet &FailedCandidates, - SourceLocation Loc, - const PartialDiagnostic &NoneDiag, - const PartialDiagnostic &AmbigDiag, - const PartialDiagnostic &CandidateDiag, - bool Complain = true, QualType TargetType = QualType()); + /// Contexts in which a converted constant expression is required. + enum CCEKind { + CCEK_CaseValue, ///< Expression in a case label. + CCEK_Enumerator, ///< Enumerator value with fixed underlying type. + CCEK_TemplateArg, ///< Value of a non-type template parameter. + CCEK_ArrayBound, ///< Array bound in array declarator or new-expression. + CCEK_ExplicitBool, ///< Condition in an explicit(bool) specifier. + CCEK_Noexcept, ///< Condition in a noexcept(bool) specifier. + CCEK_StaticAssertMessageSize, ///< Call to size() in a static assert + ///< message. + CCEK_StaticAssertMessageData, ///< Call to data() in a static assert + ///< message. + }; - ClassTemplatePartialSpecializationDecl * - getMoreSpecializedPartialSpecialization( - ClassTemplatePartialSpecializationDecl *PS1, - ClassTemplatePartialSpecializationDecl *PS2, - SourceLocation Loc); + ExprResult BuildConvertedConstantExpression(Expr *From, QualType T, + CCEKind CCE, + NamedDecl *Dest = nullptr); - bool isMoreSpecializedThanPrimary(ClassTemplatePartialSpecializationDecl *T, - sema::TemplateDeductionInfo &Info); + ExprResult CheckConvertedConstantExpression(Expr *From, QualType T, + llvm::APSInt &Value, CCEKind CCE); + ExprResult CheckConvertedConstantExpression(Expr *From, QualType T, + APValue &Value, CCEKind CCE, + NamedDecl *Dest = nullptr); - VarTemplatePartialSpecializationDecl *getMoreSpecializedPartialSpecialization( - VarTemplatePartialSpecializationDecl *PS1, - VarTemplatePartialSpecializationDecl *PS2, SourceLocation Loc); + ExprResult + EvaluateConvertedConstantExpression(Expr *E, QualType T, APValue &Value, + CCEKind CCE, bool RequireInt, + const APValue &PreNarrowingValue); - bool isMoreSpecializedThanPrimary(VarTemplatePartialSpecializationDecl *T, - sema::TemplateDeductionInfo &Info); + /// Abstract base class used to perform a contextual implicit + /// conversion from an expression to any type passing a filter. + class ContextualImplicitConverter { + public: + bool Suppress; + bool SuppressConversion; - bool isTemplateTemplateParameterAtLeastAsSpecializedAs( - TemplateParameterList *PParam, TemplateDecl *AArg, SourceLocation Loc); + ContextualImplicitConverter(bool Suppress = false, + bool SuppressConversion = false) + : Suppress(Suppress), SuppressConversion(SuppressConversion) {} - void MarkUsedTemplateParameters(const Expr *E, bool OnlyDeduced, - unsigned Depth, llvm::SmallBitVector &Used); + /// Determine whether the specified type is a valid destination type + /// for this conversion. + virtual bool match(QualType T) = 0; - void MarkUsedTemplateParameters(const TemplateArgumentList &TemplateArgs, - bool OnlyDeduced, - unsigned Depth, - llvm::SmallBitVector &Used); - void MarkDeducedTemplateParameters( - const FunctionTemplateDecl *FunctionTemplate, - llvm::SmallBitVector &Deduced) { - return MarkDeducedTemplateParameters(Context, FunctionTemplate, Deduced); - } - static void MarkDeducedTemplateParameters(ASTContext &Ctx, - const FunctionTemplateDecl *FunctionTemplate, - llvm::SmallBitVector &Deduced); + /// Emits a diagnostic complaining that the expression does not have + /// integral or enumeration type. + virtual SemaDiagnosticBuilder diagnoseNoMatch(Sema &S, SourceLocation Loc, + QualType T) = 0; - //===--------------------------------------------------------------------===// - // C++ Template Instantiation - // + /// Emits a diagnostic when the expression has incomplete class type. + virtual SemaDiagnosticBuilder + diagnoseIncomplete(Sema &S, SourceLocation Loc, QualType T) = 0; - MultiLevelTemplateArgumentList getTemplateInstantiationArgs( - const NamedDecl *D, const DeclContext *DC = nullptr, bool Final = false, - std::optional> Innermost = std::nullopt, - bool RelativeToPrimary = false, const FunctionDecl *Pattern = nullptr, - bool ForConstraintInstantiation = false, - bool SkipForSpecialization = false); + /// Emits a diagnostic when the only matching conversion function + /// is explicit. + virtual SemaDiagnosticBuilder diagnoseExplicitConv(Sema &S, + SourceLocation Loc, + QualType T, + QualType ConvTy) = 0; - /// A context in which code is being synthesized (where a source location - /// alone is not sufficient to identify the context). This covers template - /// instantiation and various forms of implicitly-generated functions. - struct CodeSynthesisContext { - /// The kind of template instantiation we are performing - enum SynthesisKind { - /// We are instantiating a template declaration. The entity is - /// the declaration we're instantiating (e.g., a CXXRecordDecl). - TemplateInstantiation, + /// Emits a note for the explicit conversion function. + virtual SemaDiagnosticBuilder + noteExplicitConv(Sema &S, CXXConversionDecl *Conv, QualType ConvTy) = 0; - /// We are instantiating a default argument for a template - /// parameter. The Entity is the template parameter whose argument is - /// being instantiated, the Template is the template, and the - /// TemplateArgs/NumTemplateArguments provide the template arguments as - /// specified. - DefaultTemplateArgumentInstantiation, + /// Emits a diagnostic when there are multiple possible conversion + /// functions. + virtual SemaDiagnosticBuilder diagnoseAmbiguous(Sema &S, SourceLocation Loc, + QualType T) = 0; - /// We are instantiating a default argument for a function. - /// The Entity is the ParmVarDecl, and TemplateArgs/NumTemplateArgs - /// provides the template arguments as specified. - DefaultFunctionArgumentInstantiation, + /// Emits a note for one of the candidate conversions. + virtual SemaDiagnosticBuilder + noteAmbiguous(Sema &S, CXXConversionDecl *Conv, QualType ConvTy) = 0; - /// We are substituting explicit template arguments provided for - /// a function template. The entity is a FunctionTemplateDecl. - ExplicitTemplateArgumentSubstitution, + /// Emits a diagnostic when we picked a conversion function + /// (for cases when we are not allowed to pick a conversion function). + virtual SemaDiagnosticBuilder diagnoseConversion(Sema &S, + SourceLocation Loc, + QualType T, + QualType ConvTy) = 0; - /// We are substituting template argument determined as part of - /// template argument deduction for either a class template - /// partial specialization or a function template. The - /// Entity is either a {Class|Var}TemplatePartialSpecializationDecl or - /// a TemplateDecl. - DeducedTemplateArgumentSubstitution, + virtual ~ContextualImplicitConverter() {} + }; - /// We are substituting into a lambda expression. - LambdaExpressionSubstitution, + class ICEConvertDiagnoser : public ContextualImplicitConverter { + bool AllowScopedEnumerations; - /// We are substituting prior template arguments into a new - /// template parameter. The template parameter itself is either a - /// NonTypeTemplateParmDecl or a TemplateTemplateParmDecl. - PriorTemplateArgumentSubstitution, + public: + ICEConvertDiagnoser(bool AllowScopedEnumerations, bool Suppress, + bool SuppressConversion) + : ContextualImplicitConverter(Suppress, SuppressConversion), + AllowScopedEnumerations(AllowScopedEnumerations) {} - /// We are checking the validity of a default template argument that - /// has been used when naming a template-id. - DefaultTemplateArgumentChecking, + /// Match an integral or (possibly scoped) enumeration type. + bool match(QualType T) override; - /// We are computing the exception specification for a defaulted special - /// member function. - ExceptionSpecEvaluation, + SemaDiagnosticBuilder diagnoseNoMatch(Sema &S, SourceLocation Loc, + QualType T) override { + return diagnoseNotInt(S, Loc, T); + } - /// We are instantiating the exception specification for a function - /// template which was deferred until it was needed. - ExceptionSpecInstantiation, + /// Emits a diagnostic complaining that the expression does not have + /// integral or enumeration type. + virtual SemaDiagnosticBuilder diagnoseNotInt(Sema &S, SourceLocation Loc, + QualType T) = 0; + }; - /// We are instantiating a requirement of a requires expression. - RequirementInstantiation, + /// Perform a contextual implicit conversion. + ExprResult + PerformContextualImplicitConversion(SourceLocation Loc, Expr *FromE, + ContextualImplicitConverter &Converter); - /// We are checking the satisfaction of a nested requirement of a requires - /// expression. - NestedRequirementConstraintsCheck, + /// ReferenceCompareResult - Expresses the result of comparing two + /// types (cv1 T1 and cv2 T2) to determine their compatibility for the + /// purposes of initialization by reference (C++ [dcl.init.ref]p4). + enum ReferenceCompareResult { + /// Ref_Incompatible - The two types are incompatible, so direct + /// reference binding is not possible. + Ref_Incompatible = 0, + /// Ref_Related - The two types are reference-related, which means + /// that their unqualified forms (T1 and T2) are either the same + /// or T1 is a base class of T2. + Ref_Related, + /// Ref_Compatible - The two types are reference-compatible. + Ref_Compatible + }; - /// We are declaring an implicit special member function. - DeclaringSpecialMember, + // Fake up a scoped enumeration that still contextually converts to bool. + struct ReferenceConversionsScope { + /// The conversions that would be performed on an lvalue of type T2 when + /// binding a reference of type T1 to it, as determined when evaluating + /// whether T1 is reference-compatible with T2. + enum ReferenceConversions { + Qualification = 0x1, + NestedQualification = 0x2, + Function = 0x4, + DerivedToBase = 0x8, + ObjC = 0x10, + ObjCLifetime = 0x20, - /// We are declaring an implicit 'operator==' for a defaulted - /// 'operator<=>'. - DeclaringImplicitEqualityComparison, + LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/ObjCLifetime) + }; + }; + using ReferenceConversions = ReferenceConversionsScope::ReferenceConversions; - /// We are defining a synthesized function (such as a defaulted special - /// member). - DefiningSynthesizedFunction, + ReferenceCompareResult + CompareReferenceRelationship(SourceLocation Loc, QualType T1, QualType T2, + ReferenceConversions *Conv = nullptr); - // We are checking the constraints associated with a constrained entity or - // the constraint expression of a concept. This includes the checks that - // atomic constraints have the type 'bool' and that they can be constant - // evaluated. - ConstraintsCheck, + void AddOverloadCandidate( + FunctionDecl *Function, DeclAccessPair FoundDecl, ArrayRef Args, + OverloadCandidateSet &CandidateSet, bool SuppressUserConversions = false, + bool PartialOverloading = false, bool AllowExplicit = true, + bool AllowExplicitConversion = false, + ADLCallKind IsADLCandidate = ADLCallKind::NotADL, + ConversionSequenceList EarlyConversions = std::nullopt, + OverloadCandidateParamOrder PO = {}, + bool AggregateCandidateDeduction = false); + void AddFunctionCandidates( + const UnresolvedSetImpl &Functions, ArrayRef Args, + OverloadCandidateSet &CandidateSet, + TemplateArgumentListInfo *ExplicitTemplateArgs = nullptr, + bool SuppressUserConversions = false, bool PartialOverloading = false, + bool FirstArgumentIsBase = false); + void AddMethodCandidate(DeclAccessPair FoundDecl, QualType ObjectType, + Expr::Classification ObjectClassification, + ArrayRef Args, + OverloadCandidateSet &CandidateSet, + bool SuppressUserConversion = false, + OverloadCandidateParamOrder PO = {}); + void + AddMethodCandidate(CXXMethodDecl *Method, DeclAccessPair FoundDecl, + CXXRecordDecl *ActingContext, QualType ObjectType, + Expr::Classification ObjectClassification, + ArrayRef Args, OverloadCandidateSet &CandidateSet, + bool SuppressUserConversions = false, + bool PartialOverloading = false, + ConversionSequenceList EarlyConversions = std::nullopt, + OverloadCandidateParamOrder PO = {}); + void AddMethodTemplateCandidate( + FunctionTemplateDecl *MethodTmpl, DeclAccessPair FoundDecl, + CXXRecordDecl *ActingContext, + TemplateArgumentListInfo *ExplicitTemplateArgs, QualType ObjectType, + Expr::Classification ObjectClassification, ArrayRef Args, + OverloadCandidateSet &CandidateSet, bool SuppressUserConversions = false, + bool PartialOverloading = false, OverloadCandidateParamOrder PO = {}); + void AddTemplateOverloadCandidate( + FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl, + TemplateArgumentListInfo *ExplicitTemplateArgs, ArrayRef Args, + OverloadCandidateSet &CandidateSet, bool SuppressUserConversions = false, + bool PartialOverloading = false, bool AllowExplicit = true, + ADLCallKind IsADLCandidate = ADLCallKind::NotADL, + OverloadCandidateParamOrder PO = {}, + bool AggregateCandidateDeduction = false); + bool CheckNonDependentConversions( + FunctionTemplateDecl *FunctionTemplate, ArrayRef ParamTypes, + ArrayRef Args, OverloadCandidateSet &CandidateSet, + ConversionSequenceList &Conversions, bool SuppressUserConversions, + CXXRecordDecl *ActingContext = nullptr, QualType ObjectType = QualType(), + Expr::Classification ObjectClassification = {}, + OverloadCandidateParamOrder PO = {}); + void AddConversionCandidate( + CXXConversionDecl *Conversion, DeclAccessPair FoundDecl, + CXXRecordDecl *ActingContext, Expr *From, QualType ToType, + OverloadCandidateSet &CandidateSet, bool AllowObjCConversionOnExplicit, + bool AllowExplicit, bool AllowResultConversion = true); + void AddTemplateConversionCandidate( + FunctionTemplateDecl *FunctionTemplate, DeclAccessPair FoundDecl, + CXXRecordDecl *ActingContext, Expr *From, QualType ToType, + OverloadCandidateSet &CandidateSet, bool AllowObjCConversionOnExplicit, + bool AllowExplicit, bool AllowResultConversion = true); + void AddSurrogateCandidate(CXXConversionDecl *Conversion, + DeclAccessPair FoundDecl, + CXXRecordDecl *ActingContext, + const FunctionProtoType *Proto, Expr *Object, + ArrayRef Args, + OverloadCandidateSet &CandidateSet); + void AddNonMemberOperatorCandidates( + const UnresolvedSetImpl &Functions, ArrayRef Args, + OverloadCandidateSet &CandidateSet, + TemplateArgumentListInfo *ExplicitTemplateArgs = nullptr); + void AddMemberOperatorCandidates(OverloadedOperatorKind Op, + SourceLocation OpLoc, ArrayRef Args, + OverloadCandidateSet &CandidateSet, + OverloadCandidateParamOrder PO = {}); + void AddBuiltinCandidate(QualType *ParamTys, ArrayRef Args, + OverloadCandidateSet &CandidateSet, + bool IsAssignmentOperator = false, + unsigned NumContextualBoolArguments = 0); + void AddBuiltinOperatorCandidates(OverloadedOperatorKind Op, + SourceLocation OpLoc, ArrayRef Args, + OverloadCandidateSet &CandidateSet); + void AddArgumentDependentLookupCandidates( + DeclarationName Name, SourceLocation Loc, ArrayRef Args, + TemplateArgumentListInfo *ExplicitTemplateArgs, + OverloadCandidateSet &CandidateSet, bool PartialOverloading = false); - // We are substituting template arguments into a constraint expression. - ConstraintSubstitution, + /// Check the enable_if expressions on the given function. Returns the first + /// failing attribute, or NULL if they were all successful. + EnableIfAttr *CheckEnableIf(FunctionDecl *Function, SourceLocation CallLoc, + ArrayRef Args, + bool MissingImplicitThis = false); - // We are normalizing a constraint expression. - ConstraintNormalization, + /// Emit diagnostics for the diagnose_if attributes on Function, ignoring any + /// non-ArgDependent DiagnoseIfAttrs. + /// + /// Argument-dependent diagnose_if attributes should be checked each time a + /// function is used as a direct callee of a function call. + /// + /// Returns true if any errors were emitted. + bool diagnoseArgDependentDiagnoseIfAttrs(const FunctionDecl *Function, + const Expr *ThisArg, + ArrayRef Args, + SourceLocation Loc); - // Instantiating a Requires Expression parameter clause. - RequirementParameterInstantiation, + /// Emit diagnostics for the diagnose_if attributes on Function, ignoring any + /// ArgDependent DiagnoseIfAttrs. + /// + /// Argument-independent diagnose_if attributes should be checked on every use + /// of a function. + /// + /// Returns true if any errors were emitted. + bool diagnoseArgIndependentDiagnoseIfAttrs(const NamedDecl *ND, + SourceLocation Loc); - // We are substituting into the parameter mapping of an atomic constraint - // during normalization. - ParameterMappingSubstitution, + /// Determine if \p A and \p B are equivalent internal linkage declarations + /// from different modules, and thus an ambiguity error can be downgraded to + /// an extension warning. + bool isEquivalentInternalLinkageDeclaration(const NamedDecl *A, + const NamedDecl *B); + void diagnoseEquivalentInternalLinkageDeclarations( + SourceLocation Loc, const NamedDecl *D, + ArrayRef Equiv); - /// We are rewriting a comparison operator in terms of an operator<=>. - RewritingOperatorAsSpaceship, + // Emit as a 'note' the specific overload candidate + void NoteOverloadCandidate( + const NamedDecl *Found, const FunctionDecl *Fn, + OverloadCandidateRewriteKind RewriteKind = OverloadCandidateRewriteKind(), + QualType DestType = QualType(), bool TakingAddress = false); - /// We are initializing a structured binding. - InitializingStructuredBinding, + // Emit as a series of 'note's all template and non-templates identified by + // the expression Expr + void NoteAllOverloadCandidates(Expr *E, QualType DestType = QualType(), + bool TakingAddress = false); - /// We are marking a class as __dllexport. - MarkingClassDllexported, + /// Returns whether the given function's address can be taken or not, + /// optionally emitting a diagnostic if the address can't be taken. + /// + /// Returns false if taking the address of the function is illegal. + bool checkAddressOfFunctionIsAvailable(const FunctionDecl *Function, + bool Complain = false, + SourceLocation Loc = SourceLocation()); - /// We are building an implied call from __builtin_dump_struct. The - /// arguments are in CallArgs. - BuildingBuiltinDumpStructCall, + // [PossiblyAFunctionType] --> [Return] + // NonFunctionType --> NonFunctionType + // R (A) --> R(A) + // R (*)(A) --> R (A) + // R (&)(A) --> R (A) + // R (S::*)(A) --> R (A) + QualType ExtractUnqualifiedFunctionType(QualType PossiblyAFunctionType); - /// Added for Template instantiation observation. - /// Memoization means we are _not_ instantiating a template because - /// it is already instantiated (but we entered a context where we - /// would have had to if it was not already instantiated). - Memoization, + FunctionDecl * + ResolveAddressOfOverloadedFunction(Expr *AddressOfExpr, QualType TargetType, + bool Complain, DeclAccessPair &Found, + bool *pHadMultipleCandidates = nullptr); - /// We are building deduction guides for a class. - BuildingDeductionGuides, - } Kind; + FunctionDecl * + resolveAddressOfSingleOverloadCandidate(Expr *E, DeclAccessPair &FoundResult); - /// Was the enclosing context a non-instantiation SFINAE context? - bool SavedInNonInstantiationSFINAEContext; + bool resolveAndFixAddressOfSingleOverloadCandidate( + ExprResult &SrcExpr, bool DoFunctionPointerConversion = false); - /// The point of instantiation or synthesis within the source code. - SourceLocation PointOfInstantiation; + FunctionDecl *ResolveSingleFunctionTemplateSpecialization( + OverloadExpr *ovl, bool Complain = false, DeclAccessPair *Found = nullptr, + TemplateSpecCandidateSet *FailedTSC = nullptr); - /// The entity that is being synthesized. - Decl *Entity; + bool ResolveAndFixSingleFunctionTemplateSpecialization( + ExprResult &SrcExpr, bool DoFunctionPointerConversion = false, + bool Complain = false, SourceRange OpRangeForComplaining = SourceRange(), + QualType DestTypeForComplaining = QualType(), + unsigned DiagIDForComplaining = 0); - /// The template (or partial specialization) in which we are - /// performing the instantiation, for substitutions of prior template - /// arguments. - NamedDecl *Template; + void AddOverloadedCallCandidates(UnresolvedLookupExpr *ULE, + ArrayRef Args, + OverloadCandidateSet &CandidateSet, + bool PartialOverloading = false); + void AddOverloadedCallCandidates( + LookupResult &R, TemplateArgumentListInfo *ExplicitTemplateArgs, + ArrayRef Args, OverloadCandidateSet &CandidateSet); - union { - /// The list of template arguments we are substituting, if they - /// are not part of the entity. - const TemplateArgument *TemplateArgs; + // An enum used to represent the different possible results of building a + // range-based for loop. + enum ForRangeStatus { + FRS_Success, + FRS_NoViableFunction, + FRS_DiagnosticIssued + }; - /// The list of argument expressions in a synthesized call. - const Expr *const *CallArgs; - }; + ForRangeStatus BuildForRangeBeginEndCall(SourceLocation Loc, + SourceLocation RangeLoc, + const DeclarationNameInfo &NameInfo, + LookupResult &MemberLookup, + OverloadCandidateSet *CandidateSet, + Expr *Range, ExprResult *CallExpr); - // FIXME: Wrap this union around more members, or perhaps store the - // kind-specific members in the RAII object owning the context. - union { - /// The number of template arguments in TemplateArgs. - unsigned NumTemplateArgs; + ExprResult BuildOverloadedCallExpr( + Scope *S, Expr *Fn, UnresolvedLookupExpr *ULE, SourceLocation LParenLoc, + MultiExprArg Args, SourceLocation RParenLoc, Expr *ExecConfig, + bool AllowTypoCorrection = true, bool CalleesAddressIsTaken = false); - /// The number of expressions in CallArgs. - unsigned NumCallArgs; + bool buildOverloadedCallSet(Scope *S, Expr *Fn, UnresolvedLookupExpr *ULE, + MultiExprArg Args, SourceLocation RParenLoc, + OverloadCandidateSet *CandidateSet, + ExprResult *Result); - /// The special member being declared or defined. - CXXSpecialMember SpecialMember; - }; + ExprResult CreateUnresolvedLookupExpr(CXXRecordDecl *NamingClass, + NestedNameSpecifierLoc NNSLoc, + DeclarationNameInfo DNI, + const UnresolvedSetImpl &Fns, + bool PerformADL = true); - ArrayRef template_arguments() const { - assert(Kind != DeclaringSpecialMember); - return {TemplateArgs, NumTemplateArgs}; - } + ExprResult CreateOverloadedUnaryOp(SourceLocation OpLoc, + UnaryOperatorKind Opc, + const UnresolvedSetImpl &Fns, Expr *input, + bool RequiresADL = true); - /// The template deduction info object associated with the - /// substitution or checking of explicit or deduced template arguments. - sema::TemplateDeductionInfo *DeductionInfo; + void LookupOverloadedBinOp(OverloadCandidateSet &CandidateSet, + OverloadedOperatorKind Op, + const UnresolvedSetImpl &Fns, + ArrayRef Args, bool RequiresADL = true); + ExprResult CreateOverloadedBinOp(SourceLocation OpLoc, BinaryOperatorKind Opc, + const UnresolvedSetImpl &Fns, Expr *LHS, + Expr *RHS, bool RequiresADL = true, + bool AllowRewrittenCandidates = true, + FunctionDecl *DefaultedFn = nullptr); + ExprResult BuildSynthesizedThreeWayComparison(SourceLocation OpLoc, + const UnresolvedSetImpl &Fns, + Expr *LHS, Expr *RHS, + FunctionDecl *DefaultedFn); - /// The source range that covers the construct that cause - /// the instantiation, e.g., the template-id that causes a class - /// template instantiation. - SourceRange InstantiationRange; + ExprResult CreateOverloadedArraySubscriptExpr(SourceLocation LLoc, + SourceLocation RLoc, Expr *Base, + MultiExprArg Args); - CodeSynthesisContext() - : Kind(TemplateInstantiation), - SavedInNonInstantiationSFINAEContext(false), Entity(nullptr), - Template(nullptr), TemplateArgs(nullptr), NumTemplateArgs(0), - DeductionInfo(nullptr) {} + ExprResult BuildCallToMemberFunction( + Scope *S, Expr *MemExpr, SourceLocation LParenLoc, MultiExprArg Args, + SourceLocation RParenLoc, Expr *ExecConfig = nullptr, + bool IsExecConfig = false, bool AllowRecovery = false); + ExprResult BuildCallToObjectOfClassType(Scope *S, Expr *Object, + SourceLocation LParenLoc, + MultiExprArg Args, + SourceLocation RParenLoc); - /// Determines whether this template is an actual instantiation - /// that should be counted toward the maximum instantiation depth. - bool isInstantiationRecord() const; - }; - - /// List of active code synthesis contexts. - /// - /// This vector is treated as a stack. As synthesis of one entity requires - /// synthesis of another, additional contexts are pushed onto the stack. - SmallVector CodeSynthesisContexts; + ExprResult BuildOverloadedArrowExpr(Scope *S, Expr *Base, + SourceLocation OpLoc, + bool *NoArrowOperatorFound = nullptr); - /// Specializations whose definitions are currently being instantiated. - llvm::DenseSet> InstantiatingSpecializations; + ExprResult BuildCXXMemberCallExpr(Expr *Exp, NamedDecl *FoundDecl, + CXXConversionDecl *Method, + bool HadMultipleCandidates); - /// Non-dependent types used in templates that have already been instantiated - /// by some template instantiation. - llvm::DenseSet InstantiatedNonDependentTypes; + ExprResult BuildLiteralOperatorCall( + LookupResult &R, DeclarationNameInfo &SuffixInfo, ArrayRef Args, + SourceLocation LitEndLoc, + TemplateArgumentListInfo *ExplicitTemplateArgs = nullptr); - /// Extra modules inspected when performing a lookup during a template - /// instantiation. Computed lazily. - SmallVector CodeSynthesisContextLookupModules; + ExprResult FixOverloadedFunctionReference(Expr *E, DeclAccessPair FoundDecl, + FunctionDecl *Fn); + ExprResult FixOverloadedFunctionReference(ExprResult, + DeclAccessPair FoundDecl, + FunctionDecl *Fn); - /// Cache of additional modules that should be used for name lookup - /// within the current template instantiation. Computed lazily; use - /// getLookupModules() to get a complete set. - llvm::DenseSet LookupModulesCache; +private: + /// - Returns a selector which best matches given argument list or + /// nullptr if none could be found + ObjCMethodDecl *SelectBestMethod(Selector Sel, MultiExprArg Args, + bool IsInstance, + SmallVectorImpl &Methods); - /// Get the set of additional modules that should be checked during - /// name lookup. A module and its imports become visible when instanting a - /// template defined within it. - llvm::DenseSet &getLookupModules(); + ///@} - /// Map from the most recent declaration of a namespace to the most - /// recent visible declaration of that namespace. - llvm::DenseMap VisibleNamespaceCache; + // + // + // ------------------------------------------------------------------------- + // + // - /// Whether we are in a SFINAE context that is not associated with - /// template instantiation. - /// - /// This is used when setting up a SFINAE trap (\c see SFINAETrap) outside - /// of a template instantiation or template argument deduction. - bool InNonInstantiationSFINAEContext; + /// \name Pseudo-Object + /// Implementations are in SemaPseudoObject.cpp + ///@{ - /// The number of \p CodeSynthesisContexts that are not template - /// instantiations and, therefore, should not be counted as part of the - /// instantiation depth. - /// - /// When the instantiation depth reaches the user-configurable limit - /// \p LangOptions::InstantiationDepth we will abort instantiation. - // FIXME: Should we have a similar limit for other forms of synthesis? - unsigned NonInstantiationEntries; +public: + void maybeExtendBlockObject(ExprResult &E); + CastKind PrepareCastToObjCObjectPointer(ExprResult &E); - /// The depth of the context stack at the point when the most recent - /// error or warning was produced. - /// - /// This value is used to suppress printing of redundant context stacks - /// when there are multiple errors or warnings in the same instantiation. - // FIXME: Does this belong in Sema? It's tough to implement it anywhere else. - unsigned LastEmittedCodeSynthesisContextDepth = 0; + enum ObjCSubscriptKind { OS_Array, OS_Dictionary, OS_Error }; + ObjCSubscriptKind CheckSubscriptingKind(Expr *FromE); - /// The template instantiation callbacks to trace or track - /// instantiations (objects can be chained). - /// - /// This callbacks is used to print, trace or track template - /// instantiations as they are being constructed. - std::vector> - TemplateInstCallbacks; + ExprResult checkPseudoObjectIncDec(Scope *S, SourceLocation OpLoc, + UnaryOperatorKind Opcode, Expr *Op); + ExprResult checkPseudoObjectAssignment(Scope *S, SourceLocation OpLoc, + BinaryOperatorKind Opcode, Expr *LHS, + Expr *RHS); + ExprResult checkPseudoObjectRValue(Expr *E); + Expr *recreateSyntacticForm(PseudoObjectExpr *E); - /// The current index into pack expansion arguments that will be - /// used for substitution of parameter packs. - /// - /// The pack expansion index will be -1 to indicate that parameter packs - /// should be instantiated as themselves. Otherwise, the index specifies - /// which argument within the parameter pack will be used for substitution. - int ArgumentPackSubstitutionIndex; + ///@} - /// RAII object used to change the argument pack substitution index - /// within a \c Sema object. - /// - /// See \c ArgumentPackSubstitutionIndex for more information. - class ArgumentPackSubstitutionIndexRAII { - Sema &Self; - int OldSubstitutionIndex; + // + // + // ------------------------------------------------------------------------- + // + // - public: - ArgumentPackSubstitutionIndexRAII(Sema &Self, int NewSubstitutionIndex) - : Self(Self), OldSubstitutionIndex(Self.ArgumentPackSubstitutionIndex) { - Self.ArgumentPackSubstitutionIndex = NewSubstitutionIndex; - } + /// \name Statements + /// Implementations are in SemaStmt.cpp + ///@{ - ~ArgumentPackSubstitutionIndexRAII() { - Self.ArgumentPackSubstitutionIndex = OldSubstitutionIndex; - } - }; +public: + /// Stack of active SEH __finally scopes. Can be empty. + SmallVector CurrentSEHFinally; - friend class ArgumentPackSubstitutionRAII; + StmtResult ActOnExprStmt(ExprResult Arg, bool DiscardedValue = true); + StmtResult ActOnExprStmtError(); - /// For each declaration that involved template argument deduction, the - /// set of diagnostics that were suppressed during that template argument - /// deduction. - /// - /// FIXME: Serialize this structure to the AST file. - typedef llvm::DenseMap > - SuppressedDiagnosticsMap; - SuppressedDiagnosticsMap SuppressedDiagnostics; + StmtResult ActOnNullStmt(SourceLocation SemiLoc, + bool HasLeadingEmptyMacro = false); - /// A stack object to be created when performing template - /// instantiation. - /// - /// Construction of an object of type \c InstantiatingTemplate - /// pushes the current instantiation onto the stack of active - /// instantiations. If the size of this stack exceeds the maximum - /// number of recursive template instantiations, construction - /// produces an error and evaluates true. - /// - /// Destruction of this object will pop the named instantiation off - /// the stack. - struct InstantiatingTemplate { - /// Note that we are instantiating a class template, - /// function template, variable template, alias template, - /// or a member thereof. - InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, - Decl *Entity, - SourceRange InstantiationRange = SourceRange()); + StmtResult ActOnDeclStmt(DeclGroupPtrTy Decl, SourceLocation StartLoc, + SourceLocation EndLoc); + void ActOnForEachDeclStmt(DeclGroupPtrTy Decl); - struct ExceptionSpecification {}; - /// Note that we are instantiating an exception specification - /// of a function template. - InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, - FunctionDecl *Entity, ExceptionSpecification, - SourceRange InstantiationRange = SourceRange()); + /// DiagnoseUnusedExprResult - If the statement passed in is an expression + /// whose result is unused, warn. + void DiagnoseUnusedExprResult(const Stmt *S, unsigned DiagID); - /// Note that we are instantiating a default argument in a - /// template-id. - InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, - TemplateParameter Param, TemplateDecl *Template, - ArrayRef TemplateArgs, - SourceRange InstantiationRange = SourceRange()); + void ActOnStartOfCompoundStmt(bool IsStmtExpr); + void ActOnAfterCompoundStatementLeadingPragmas(); + void ActOnFinishOfCompoundStmt(); + StmtResult ActOnCompoundStmt(SourceLocation L, SourceLocation R, + ArrayRef Elts, bool isStmtExpr); - /// Note that we are substituting either explicitly-specified or - /// deduced template arguments during function template argument deduction. - InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, - FunctionTemplateDecl *FunctionTemplate, - ArrayRef TemplateArgs, - CodeSynthesisContext::SynthesisKind Kind, - sema::TemplateDeductionInfo &DeductionInfo, - SourceRange InstantiationRange = SourceRange()); + sema::CompoundScopeInfo &getCurCompoundScope() const; - /// Note that we are instantiating as part of template - /// argument deduction for a class template declaration. - InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, - TemplateDecl *Template, - ArrayRef TemplateArgs, - sema::TemplateDeductionInfo &DeductionInfo, - SourceRange InstantiationRange = SourceRange()); + ExprResult ActOnCaseExpr(SourceLocation CaseLoc, ExprResult Val); + StmtResult ActOnCaseStmt(SourceLocation CaseLoc, ExprResult LHS, + SourceLocation DotDotDotLoc, ExprResult RHS, + SourceLocation ColonLoc); + void ActOnCaseStmtBody(Stmt *CaseStmt, Stmt *SubStmt); - /// Note that we are instantiating as part of template - /// argument deduction for a class template partial - /// specialization. - InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, - ClassTemplatePartialSpecializationDecl *PartialSpec, - ArrayRef TemplateArgs, - sema::TemplateDeductionInfo &DeductionInfo, - SourceRange InstantiationRange = SourceRange()); + StmtResult ActOnDefaultStmt(SourceLocation DefaultLoc, + SourceLocation ColonLoc, Stmt *SubStmt, + Scope *CurScope); + StmtResult ActOnLabelStmt(SourceLocation IdentLoc, LabelDecl *TheDecl, + SourceLocation ColonLoc, Stmt *SubStmt); - /// Note that we are instantiating as part of template - /// argument deduction for a variable template partial - /// specialization. - InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, - VarTemplatePartialSpecializationDecl *PartialSpec, - ArrayRef TemplateArgs, - sema::TemplateDeductionInfo &DeductionInfo, - SourceRange InstantiationRange = SourceRange()); + StmtResult BuildAttributedStmt(SourceLocation AttrsLoc, + ArrayRef Attrs, Stmt *SubStmt); + StmtResult ActOnAttributedStmt(const ParsedAttributes &AttrList, + Stmt *SubStmt); - /// Note that we are instantiating a default argument for a function - /// parameter. - InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, - ParmVarDecl *Param, - ArrayRef TemplateArgs, - SourceRange InstantiationRange = SourceRange()); + /// Check whether the given statement can have musttail applied to it, + /// issuing a diagnostic and returning false if not. In the success case, + /// the statement is rewritten to remove implicit nodes from the return + /// value. + bool checkAndRewriteMustTailAttr(Stmt *St, const Attr &MTA); - /// Note that we are substituting prior template arguments into a - /// non-type parameter. - InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, - NamedDecl *Template, - NonTypeTemplateParmDecl *Param, - ArrayRef TemplateArgs, - SourceRange InstantiationRange); + StmtResult ActOnIfStmt(SourceLocation IfLoc, IfStatementKind StatementKind, + SourceLocation LParenLoc, Stmt *InitStmt, + ConditionResult Cond, SourceLocation RParenLoc, + Stmt *ThenVal, SourceLocation ElseLoc, Stmt *ElseVal); + StmtResult BuildIfStmt(SourceLocation IfLoc, IfStatementKind StatementKind, + SourceLocation LParenLoc, Stmt *InitStmt, + ConditionResult Cond, SourceLocation RParenLoc, + Stmt *ThenVal, SourceLocation ElseLoc, Stmt *ElseVal); - /// Note that we are substituting prior template arguments into a - /// template template parameter. - InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, - NamedDecl *Template, - TemplateTemplateParmDecl *Param, - ArrayRef TemplateArgs, - SourceRange InstantiationRange); + ExprResult CheckSwitchCondition(SourceLocation SwitchLoc, Expr *Cond); - /// Note that we are checking the default template argument - /// against the template parameter for a given template-id. - InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, - TemplateDecl *Template, - NamedDecl *Param, - ArrayRef TemplateArgs, - SourceRange InstantiationRange); + StmtResult ActOnStartOfSwitchStmt(SourceLocation SwitchLoc, + SourceLocation LParenLoc, Stmt *InitStmt, + ConditionResult Cond, + SourceLocation RParenLoc); + StmtResult ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch, + Stmt *Body); - struct ConstraintsCheck {}; - /// \brief Note that we are checking the constraints associated with some - /// constrained entity (a concept declaration or a template with associated - /// constraints). - InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, - ConstraintsCheck, NamedDecl *Template, - ArrayRef TemplateArgs, - SourceRange InstantiationRange); + /// DiagnoseAssignmentEnum - Warn if assignment to enum is a constant + /// integer not in the range of enum values. + void DiagnoseAssignmentEnum(QualType DstType, QualType SrcType, + Expr *SrcExpr); - struct ConstraintSubstitution {}; - /// \brief Note that we are checking a constraint expression associated - /// with a template declaration or as part of the satisfaction check of a - /// concept. - InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, - ConstraintSubstitution, NamedDecl *Template, - sema::TemplateDeductionInfo &DeductionInfo, - SourceRange InstantiationRange); + StmtResult ActOnWhileStmt(SourceLocation WhileLoc, SourceLocation LParenLoc, + ConditionResult Cond, SourceLocation RParenLoc, + Stmt *Body); + StmtResult ActOnDoStmt(SourceLocation DoLoc, Stmt *Body, + SourceLocation WhileLoc, SourceLocation CondLParen, + Expr *Cond, SourceLocation CondRParen); - struct ConstraintNormalization {}; - /// \brief Note that we are normalizing a constraint expression. - InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, - ConstraintNormalization, NamedDecl *Template, - SourceRange InstantiationRange); + StmtResult ActOnForStmt(SourceLocation ForLoc, SourceLocation LParenLoc, + Stmt *First, ConditionResult Second, + FullExprArg Third, SourceLocation RParenLoc, + Stmt *Body); - struct ParameterMappingSubstitution {}; - /// \brief Note that we are subtituting into the parameter mapping of an - /// atomic constraint during constraint normalization. - InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, - ParameterMappingSubstitution, NamedDecl *Template, - SourceRange InstantiationRange); + StmtResult ActOnForEachLValueExpr(Expr *E); - /// \brief Note that we are substituting template arguments into a part of - /// a requirement of a requires expression. - InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, - concepts::Requirement *Req, - sema::TemplateDeductionInfo &DeductionInfo, - SourceRange InstantiationRange = SourceRange()); + ExprResult CheckObjCForCollectionOperand(SourceLocation forLoc, + Expr *collection); + StmtResult ActOnObjCForCollectionStmt(SourceLocation ForColLoc, Stmt *First, + Expr *collection, + SourceLocation RParenLoc); + StmtResult FinishObjCForCollectionStmt(Stmt *ForCollection, Stmt *Body); - /// \brief Note that we are checking the satisfaction of the constraint - /// expression inside of a nested requirement. - InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, - concepts::NestedRequirement *Req, ConstraintsCheck, - SourceRange InstantiationRange = SourceRange()); + enum BuildForRangeKind { + /// Initial building of a for-range statement. + BFRK_Build, + /// Instantiation or recovery rebuild of a for-range statement. Don't + /// attempt any typo-correction. + BFRK_Rebuild, + /// Determining whether a for-range statement could be built. Avoid any + /// unnecessary or irreversible actions. + BFRK_Check + }; - /// \brief Note that we are checking a requires clause. - InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, - const RequiresExpr *E, - sema::TemplateDeductionInfo &DeductionInfo, - SourceRange InstantiationRange); + StmtResult ActOnCXXForRangeStmt( + Scope *S, SourceLocation ForLoc, SourceLocation CoawaitLoc, + Stmt *InitStmt, Stmt *LoopVar, SourceLocation ColonLoc, Expr *Collection, + SourceLocation RParenLoc, BuildForRangeKind Kind, + ArrayRef LifetimeExtendTemps = {}); + StmtResult BuildCXXForRangeStmt( + SourceLocation ForLoc, SourceLocation CoawaitLoc, Stmt *InitStmt, + SourceLocation ColonLoc, Stmt *RangeDecl, Stmt *Begin, Stmt *End, + Expr *Cond, Expr *Inc, Stmt *LoopVarDecl, SourceLocation RParenLoc, + BuildForRangeKind Kind, + ArrayRef LifetimeExtendTemps = {}); + StmtResult FinishCXXForRangeStmt(Stmt *ForRange, Stmt *Body); - struct BuildingDeductionGuidesTag {}; - /// \brief Note that we are building deduction guides. - InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, - TemplateDecl *Entity, BuildingDeductionGuidesTag, - SourceRange InstantiationRange = SourceRange()); + StmtResult ActOnGotoStmt(SourceLocation GotoLoc, SourceLocation LabelLoc, + LabelDecl *TheDecl); + StmtResult ActOnIndirectGotoStmt(SourceLocation GotoLoc, + SourceLocation StarLoc, Expr *DestExp); + StmtResult ActOnContinueStmt(SourceLocation ContinueLoc, Scope *CurScope); + StmtResult ActOnBreakStmt(SourceLocation BreakLoc, Scope *CurScope); - /// Note that we have finished instantiating this template. - void Clear(); + struct NamedReturnInfo { + const VarDecl *Candidate; - ~InstantiatingTemplate() { Clear(); } + enum Status : uint8_t { None, MoveEligible, MoveEligibleAndCopyElidable }; + Status S; - /// Determines whether we have exceeded the maximum - /// recursive template instantiations. - bool isInvalid() const { return Invalid; } + bool isMoveEligible() const { return S != None; }; + bool isCopyElidable() const { return S == MoveEligibleAndCopyElidable; } + }; + enum class SimplerImplicitMoveMode { ForceOff, Normal, ForceOn }; + NamedReturnInfo getNamedReturnInfo( + Expr *&E, SimplerImplicitMoveMode Mode = SimplerImplicitMoveMode::Normal); + NamedReturnInfo getNamedReturnInfo(const VarDecl *VD); + const VarDecl *getCopyElisionCandidate(NamedReturnInfo &Info, + QualType ReturnType); - /// Determine whether we are already instantiating this - /// specialization in some surrounding active instantiation. - bool isAlreadyInstantiating() const { return AlreadyInstantiating; } + ExprResult + PerformMoveOrCopyInitialization(const InitializedEntity &Entity, + const NamedReturnInfo &NRInfo, Expr *Value, + bool SupressSimplerImplicitMoves = false); - private: - Sema &SemaRef; - bool Invalid; - bool AlreadyInstantiating; - bool CheckInstantiationDepth(SourceLocation PointOfInstantiation, - SourceRange InstantiationRange); + TypeLoc getReturnTypeLoc(FunctionDecl *FD) const; - InstantiatingTemplate( - Sema &SemaRef, CodeSynthesisContext::SynthesisKind Kind, - SourceLocation PointOfInstantiation, SourceRange InstantiationRange, - Decl *Entity, NamedDecl *Template = nullptr, - ArrayRef TemplateArgs = std::nullopt, - sema::TemplateDeductionInfo *DeductionInfo = nullptr); + bool DeduceFunctionTypeFromReturnExpr(FunctionDecl *FD, + SourceLocation ReturnLoc, Expr *RetExpr, + const AutoType *AT); - InstantiatingTemplate(const InstantiatingTemplate&) = delete; + StmtResult ActOnReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp, + Scope *CurScope); + StmtResult BuildReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp, + bool AllowRecovery = false); + StmtResult ActOnCapScopeReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp, + NamedReturnInfo &NRInfo, + bool SupressSimplerImplicitMoves); - InstantiatingTemplate& - operator=(const InstantiatingTemplate&) = delete; - }; + StmtResult ActOnObjCAtCatchStmt(SourceLocation AtLoc, SourceLocation RParen, + Decl *Parm, Stmt *Body); - void pushCodeSynthesisContext(CodeSynthesisContext Ctx); - void popCodeSynthesisContext(); + StmtResult ActOnObjCAtFinallyStmt(SourceLocation AtLoc, Stmt *Body); - /// Determine whether we are currently performing template instantiation. - bool inTemplateInstantiation() const { - return CodeSynthesisContexts.size() > NonInstantiationEntries; - } + StmtResult ActOnObjCAtTryStmt(SourceLocation AtLoc, Stmt *Try, + MultiStmtArg Catch, Stmt *Finally); - void PrintContextStack() { - if (!CodeSynthesisContexts.empty() && - CodeSynthesisContexts.size() != LastEmittedCodeSynthesisContextDepth) { - PrintInstantiationStack(); - LastEmittedCodeSynthesisContextDepth = CodeSynthesisContexts.size(); - } - if (PragmaAttributeCurrentTargetDecl) - PrintPragmaAttributeInstantiationPoint(); - } - void PrintInstantiationStack(); + StmtResult BuildObjCAtThrowStmt(SourceLocation AtLoc, Expr *Throw); + StmtResult ActOnObjCAtThrowStmt(SourceLocation AtLoc, Expr *Throw, + Scope *CurScope); + ExprResult ActOnObjCAtSynchronizedOperand(SourceLocation atLoc, + Expr *operand); + StmtResult ActOnObjCAtSynchronizedStmt(SourceLocation AtLoc, Expr *SynchExpr, + Stmt *SynchBody); - void PrintPragmaAttributeInstantiationPoint(); + StmtResult ActOnObjCAutoreleasePoolStmt(SourceLocation AtLoc, Stmt *Body); - /// Determines whether we are currently in a context where - /// template argument substitution failures are not considered - /// errors. - /// - /// \returns An empty \c Optional if we're not in a SFINAE context. - /// Otherwise, contains a pointer that, if non-NULL, contains the nearest - /// template-deduction context object, which can be used to capture - /// diagnostics that will be suppressed. - std::optional isSFINAEContext() const; + StmtResult ActOnCXXCatchBlock(SourceLocation CatchLoc, Decl *ExDecl, + Stmt *HandlerBlock); + StmtResult ActOnCXXTryBlock(SourceLocation TryLoc, Stmt *TryBlock, + ArrayRef Handlers); - /// Whether the AST is currently being rebuilt to correct immediate - /// invocations. Immediate invocation candidates and references to consteval - /// functions aren't tracked when this is set. - bool RebuildingImmediateInvocation = false; + StmtResult ActOnSEHTryBlock(bool IsCXXTry, // try (true) or __try (false) ? + SourceLocation TryLoc, Stmt *TryBlock, + Stmt *Handler); + StmtResult ActOnSEHExceptBlock(SourceLocation Loc, Expr *FilterExpr, + Stmt *Block); + void ActOnStartSEHFinallyBlock(); + void ActOnAbortSEHFinallyBlock(); + StmtResult ActOnFinishSEHFinallyBlock(SourceLocation Loc, Stmt *Block); + StmtResult ActOnSEHLeaveStmt(SourceLocation Loc, Scope *CurScope); - /// Used to change context to isConstantEvaluated without pushing a heavy - /// ExpressionEvaluationContextRecord object. - bool isConstantEvaluatedOverride = false; + StmtResult BuildMSDependentExistsStmt(SourceLocation KeywordLoc, + bool IsIfExists, + NestedNameSpecifierLoc QualifierLoc, + DeclarationNameInfo NameInfo, + Stmt *Nested); + StmtResult ActOnMSDependentExistsStmt(SourceLocation KeywordLoc, + bool IsIfExists, CXXScopeSpec &SS, + UnqualifiedId &Name, Stmt *Nested); - const ExpressionEvaluationContextRecord ¤tEvaluationContext() const { - assert(!ExprEvalContexts.empty() && - "Must be in an expression evaluation context"); - return ExprEvalContexts.back(); - }; + void ActOnCapturedRegionStart(SourceLocation Loc, Scope *CurScope, + CapturedRegionKind Kind, unsigned NumParams); + typedef std::pair CapturedParamNameType; + void ActOnCapturedRegionStart(SourceLocation Loc, Scope *CurScope, + CapturedRegionKind Kind, + ArrayRef Params, + unsigned OpenMPCaptureLevel = 0); + StmtResult ActOnCapturedRegionEnd(Stmt *S); + void ActOnCapturedRegionError(); + RecordDecl *CreateCapturedStmtRecordDecl(CapturedDecl *&CD, + SourceLocation Loc, + unsigned NumParams); - bool isConstantEvaluatedContext() const { - return currentEvaluationContext().isConstantEvaluated() || - isConstantEvaluatedOverride; - } +private: + /// Check whether the given statement can have musttail applied to it, + /// issuing a diagnostic and returning false if not. + bool checkMustTailAttr(const Stmt *St, const Attr &MTA); - bool isAlwaysConstantEvaluatedContext() const { - const ExpressionEvaluationContextRecord &Ctx = currentEvaluationContext(); - return (Ctx.isConstantEvaluated() || isConstantEvaluatedOverride) && - !Ctx.InConditionallyConstantEvaluateContext; - } + /// Check if the given expression contains 'break' or 'continue' + /// statement that produces control flow different from GCC. + void CheckBreakContinueBinding(Expr *E); - /// Determines whether we are currently in a context that - /// is not evaluated as per C++ [expr] p5. - bool isUnevaluatedContext() const { - return currentEvaluationContext().isUnevaluated(); - } + ///@} - bool isImmediateFunctionContext() const { - return currentEvaluationContext().isImmediateFunctionContext(); - } + // + // + // ------------------------------------------------------------------------- + // + // - bool isInLifetimeExtendingContext() const { - assert(!ExprEvalContexts.empty() && - "Must be in an expression evaluation context"); - return ExprEvalContexts.back().InLifetimeExtendingContext; - } + /// \name `inline asm` Statement + /// Implementations are in SemaStmtAsm.cpp + ///@{ - bool isInMaterializeTemporaryObjectContext() const { - assert(!ExprEvalContexts.empty() && - "Must be in an expression evaluation context"); - return ExprEvalContexts.back().InMaterializeTemporaryObjectContext; - } +public: + StmtResult ActOnGCCAsmStmt(SourceLocation AsmLoc, bool IsSimple, + bool IsVolatile, unsigned NumOutputs, + unsigned NumInputs, IdentifierInfo **Names, + MultiExprArg Constraints, MultiExprArg Exprs, + Expr *AsmString, MultiExprArg Clobbers, + unsigned NumLabels, SourceLocation RParenLoc); - bool isCheckingDefaultArgumentOrInitializer() const { - const ExpressionEvaluationContextRecord &Ctx = currentEvaluationContext(); - return (Ctx.Context == - ExpressionEvaluationContext::PotentiallyEvaluatedIfUsed) || - Ctx.IsCurrentlyCheckingDefaultArgumentOrInitializer; - } + void FillInlineAsmIdentifierInfo(Expr *Res, + llvm::InlineAsmIdentifierInfo &Info); + ExprResult LookupInlineAsmIdentifier(CXXScopeSpec &SS, + SourceLocation TemplateKWLoc, + UnqualifiedId &Id, + bool IsUnevaluatedContext); + bool LookupInlineAsmField(StringRef Base, StringRef Member, unsigned &Offset, + SourceLocation AsmLoc); + ExprResult LookupInlineAsmVarDeclField(Expr *RefExpr, StringRef Member, + SourceLocation AsmLoc); + StmtResult ActOnMSAsmStmt(SourceLocation AsmLoc, SourceLocation LBraceLoc, + ArrayRef AsmToks, StringRef AsmString, + unsigned NumOutputs, unsigned NumInputs, + ArrayRef Constraints, + ArrayRef Clobbers, + ArrayRef Exprs, SourceLocation EndLoc); + LabelDecl *GetOrCreateMSAsmLabel(StringRef ExternalLabelName, + SourceLocation Location, bool AlwaysCreate); - std::optional - InnermostDeclarationWithDelayedImmediateInvocations() const { - assert(!ExprEvalContexts.empty() && - "Must be in an expression evaluation context"); - for (const auto &Ctx : llvm::reverse(ExprEvalContexts)) { - if (Ctx.Context == ExpressionEvaluationContext::PotentiallyEvaluated && - Ctx.DelayedDefaultInitializationContext) - return Ctx.DelayedDefaultInitializationContext; - if (Ctx.isConstantEvaluated() || Ctx.isImmediateFunctionContext() || - Ctx.isUnevaluated()) - break; - } - return std::nullopt; - } + ///@} - std::optional - OutermostDeclarationWithDelayedImmediateInvocations() const { - assert(!ExprEvalContexts.empty() && - "Must be in an expression evaluation context"); - std::optional Res; - for (auto &Ctx : llvm::reverse(ExprEvalContexts)) { - if (Ctx.Context == ExpressionEvaluationContext::PotentiallyEvaluated && - !Ctx.DelayedDefaultInitializationContext && Res) - break; - if (Ctx.isConstantEvaluated() || Ctx.isImmediateFunctionContext() || - Ctx.isUnevaluated()) - break; - Res = Ctx.DelayedDefaultInitializationContext; - } - return Res; - } + // + // + // ------------------------------------------------------------------------- + // + // - /// keepInLifetimeExtendingContext - Pull down InLifetimeExtendingContext - /// flag from previous context. - void keepInLifetimeExtendingContext() { - if (ExprEvalContexts.size() > 2 && - ExprEvalContexts[ExprEvalContexts.size() - 2] - .InLifetimeExtendingContext) { - auto &LastRecord = ExprEvalContexts.back(); - auto &PrevRecord = ExprEvalContexts[ExprEvalContexts.size() - 2]; - LastRecord.InLifetimeExtendingContext = - PrevRecord.InLifetimeExtendingContext; - } - } + /// \name Statement Attribute Handling + /// Implementations are in SemaStmtAttr.cpp + ///@{ - /// keepInMaterializeTemporaryObjectContext - Pull down - /// InMaterializeTemporaryObjectContext flag from previous context. - void keepInMaterializeTemporaryObjectContext() { - if (ExprEvalContexts.size() > 2 && - ExprEvalContexts[ExprEvalContexts.size() - 2] - .InMaterializeTemporaryObjectContext) { - auto &LastRecord = ExprEvalContexts.back(); - auto &PrevRecord = ExprEvalContexts[ExprEvalContexts.size() - 2]; - LastRecord.InMaterializeTemporaryObjectContext = - PrevRecord.InMaterializeTemporaryObjectContext; - } - } +public: + bool CheckNoInlineAttr(const Stmt *OrigSt, const Stmt *CurSt, + const AttributeCommonInfo &A); + bool CheckAlwaysInlineAttr(const Stmt *OrigSt, const Stmt *CurSt, + const AttributeCommonInfo &A); - /// RAII class used to determine whether SFINAE has - /// trapped any errors that occur during template argument - /// deduction. - class SFINAETrap { - Sema &SemaRef; - unsigned PrevSFINAEErrors; - bool PrevInNonInstantiationSFINAEContext; - bool PrevAccessCheckingSFINAE; - bool PrevLastDiagnosticIgnored; + CodeAlignAttr *BuildCodeAlignAttr(const AttributeCommonInfo &CI, Expr *E); + bool CheckRebuiltStmtAttributes(ArrayRef Attrs); - public: - explicit SFINAETrap(Sema &SemaRef, bool AccessCheckingSFINAE = false) - : SemaRef(SemaRef), PrevSFINAEErrors(SemaRef.NumSFINAEErrors), - PrevInNonInstantiationSFINAEContext( - SemaRef.InNonInstantiationSFINAEContext), - PrevAccessCheckingSFINAE(SemaRef.AccessCheckingSFINAE), - PrevLastDiagnosticIgnored( - SemaRef.getDiagnostics().isLastDiagnosticIgnored()) - { - if (!SemaRef.isSFINAEContext()) - SemaRef.InNonInstantiationSFINAEContext = true; - SemaRef.AccessCheckingSFINAE = AccessCheckingSFINAE; - } + /// Process the attributes before creating an attributed statement. Returns + /// the semantic attributes that have been processed. + void ProcessStmtAttributes(Stmt *Stmt, const ParsedAttributes &InAttrs, + SmallVectorImpl &OutAttrs); - ~SFINAETrap() { - SemaRef.NumSFINAEErrors = PrevSFINAEErrors; - SemaRef.InNonInstantiationSFINAEContext - = PrevInNonInstantiationSFINAEContext; - SemaRef.AccessCheckingSFINAE = PrevAccessCheckingSFINAE; - SemaRef.getDiagnostics().setLastDiagnosticIgnored( - PrevLastDiagnosticIgnored); - } + ///@} - /// Determine whether any SFINAE errors have been trapped. - bool hasErrorOccurred() const { - return SemaRef.NumSFINAEErrors > PrevSFINAEErrors; - } - }; + // + // + // ------------------------------------------------------------------------- + // + // - /// RAII class used to indicate that we are performing provisional - /// semantic analysis to determine the validity of a construct, so - /// typo-correction and diagnostics in the immediate context (not within - /// implicitly-instantiated templates) should be suppressed. - class TentativeAnalysisScope { - Sema &SemaRef; - // FIXME: Using a SFINAETrap for this is a hack. - SFINAETrap Trap; - bool PrevDisableTypoCorrection; + /// \name C++ Templates + /// Implementations are in SemaTemplate.cpp + ///@{ + +public: + // Saves the current floating-point pragma stack and clear it in this Sema. + class FpPragmaStackSaveRAII { public: - explicit TentativeAnalysisScope(Sema &SemaRef) - : SemaRef(SemaRef), Trap(SemaRef, true), - PrevDisableTypoCorrection(SemaRef.DisableTypoCorrection) { - SemaRef.DisableTypoCorrection = true; - } - ~TentativeAnalysisScope() { - SemaRef.DisableTypoCorrection = PrevDisableTypoCorrection; + FpPragmaStackSaveRAII(Sema &S) + : S(S), SavedStack(std::move(S.FpPragmaStack)) { + S.FpPragmaStack.Stack.clear(); } + ~FpPragmaStackSaveRAII() { S.FpPragmaStack = std::move(SavedStack); } + + private: + Sema &S; + PragmaStack SavedStack; }; - /// The current instantiation scope used to store local - /// variables. - LocalInstantiationScope *CurrentInstantiationScope; + void resetFPOptions(FPOptions FPO) { + CurFPFeatures = FPO; + FpPragmaStack.CurrentValue = FPO.getChangesFrom(FPOptions(LangOpts)); + } - /// Tracks whether we are in a context where typo correction is - /// disabled. - bool DisableTypoCorrection; + ArrayRef getInventedParameterInfos() const { + return llvm::ArrayRef(InventedParameterInfos.begin() + + InventedParameterInfosStart, + InventedParameterInfos.end()); + } - /// The number of typos corrected by CorrectTypo. - unsigned TyposCorrected; + /// The number of SFINAE diagnostics that have been trapped. + unsigned NumSFINAEErrors; - typedef llvm::SmallSet SrcLocSet; - typedef llvm::DenseMap IdentifierSourceLocations; + ArrayRef getFunctionScopes() const { + return llvm::ArrayRef(FunctionScopes.begin() + FunctionScopesStart, + FunctionScopes.end()); + } - /// A cache containing identifiers for which typo correction failed and - /// their locations, so that repeated attempts to correct an identifier in a - /// given location are ignored if typo correction already failed for it. - IdentifierSourceLocations TypoCorrectionFailures; + typedef llvm::MapVector> + LateParsedTemplateMapT; + LateParsedTemplateMapT LateParsedTemplateMap; - /// Worker object for performing CFG-based warnings. - sema::AnalysisBasedWarnings AnalysisWarnings; - threadSafety::BeforeSet *ThreadSafetyDeclCache; + /// Determine the number of levels of enclosing template parameters. This is + /// only usable while parsing. Note that this does not include dependent + /// contexts in which no template parameters have yet been declared, such as + /// in a terse function template or generic lambda before the first 'auto' is + /// encountered. + unsigned getTemplateDepth(Scope *S) const; - /// An entity for which implicit template instantiation is required. + void FilterAcceptableTemplateNames(LookupResult &R, + bool AllowFunctionTemplates = true, + bool AllowDependent = true); + bool hasAnyAcceptableTemplateNames(LookupResult &R, + bool AllowFunctionTemplates = true, + bool AllowDependent = true, + bool AllowNonTemplateFunctions = false); + /// Try to interpret the lookup result D as a template-name. /// - /// The source location associated with the declaration is the first place in - /// the source code where the declaration was "used". It is not necessarily - /// the point of instantiation (which will be either before or after the - /// namespace-scope declaration that triggered this implicit instantiation), - /// However, it is the location that diagnostics should generally refer to, - /// because users will need to know what code triggered the instantiation. - typedef std::pair PendingImplicitInstantiation; - - /// The queue of implicit template instantiations that are required - /// but have not yet been performed. - std::deque PendingInstantiations; - - /// Queue of implicit template instantiations that cannot be performed - /// eagerly. - SmallVector LateParsedInstantiations; - - SmallVector, 8> SavedVTableUses; - SmallVector, 8> - SavedPendingInstantiations; + /// \param D A declaration found by name lookup. + /// \param AllowFunctionTemplates Whether function templates should be + /// considered valid results. + /// \param AllowDependent Whether unresolved using declarations (that might + /// name templates) should be considered valid results. + static NamedDecl *getAsTemplateNameDecl(NamedDecl *D, + bool AllowFunctionTemplates = true, + bool AllowDependent = true); - class GlobalEagerInstantiationScope { + enum TemplateNameIsRequiredTag { TemplateNameIsRequired }; + /// Whether and why a template name is required in this lookup. + class RequiredTemplateKind { public: - GlobalEagerInstantiationScope(Sema &S, bool Enabled) - : S(S), Enabled(Enabled) { - if (!Enabled) return; - - S.SavedPendingInstantiations.emplace_back(); - S.SavedPendingInstantiations.back().swap(S.PendingInstantiations); + /// Template name is required if TemplateKWLoc is valid. + RequiredTemplateKind(SourceLocation TemplateKWLoc = SourceLocation()) + : TemplateKW(TemplateKWLoc) {} + /// Template name is unconditionally required. + RequiredTemplateKind(TemplateNameIsRequiredTag) {} - S.SavedVTableUses.emplace_back(); - S.SavedVTableUses.back().swap(S.VTableUses); + SourceLocation getTemplateKeywordLoc() const { + return TemplateKW.value_or(SourceLocation()); } - - void perform() { - if (Enabled) { - S.DefineUsedVTables(); - S.PerformPendingInstantiations(); - } + bool hasTemplateKeyword() const { + return getTemplateKeywordLoc().isValid(); } + bool isRequired() const { return TemplateKW != SourceLocation(); } + explicit operator bool() const { return isRequired(); } - ~GlobalEagerInstantiationScope() { - if (!Enabled) return; - - // Restore the set of pending vtables. - assert(S.VTableUses.empty() && - "VTableUses should be empty before it is discarded."); - S.VTableUses.swap(S.SavedVTableUses.back()); - S.SavedVTableUses.pop_back(); + private: + std::optional TemplateKW; + }; - // Restore the set of pending implicit instantiations. - if (S.TUKind != TU_Prefix || !S.LangOpts.PCHInstantiateTemplates) { - assert(S.PendingInstantiations.empty() && - "PendingInstantiations should be empty before it is discarded."); - S.PendingInstantiations.swap(S.SavedPendingInstantiations.back()); - S.SavedPendingInstantiations.pop_back(); - } else { - // Template instantiations in the PCH may be delayed until the TU. - S.PendingInstantiations.swap(S.SavedPendingInstantiations.back()); - S.PendingInstantiations.insert( - S.PendingInstantiations.end(), - S.SavedPendingInstantiations.back().begin(), - S.SavedPendingInstantiations.back().end()); - S.SavedPendingInstantiations.pop_back(); - } - } - - private: - Sema &S; - bool Enabled; + enum class AssumedTemplateKind { + /// This is not assumed to be a template name. + None, + /// This is assumed to be a template name because lookup found nothing. + FoundNothing, + /// This is assumed to be a template name because lookup found one or more + /// functions (but no function templates). + FoundFunctions, }; + bool LookupTemplateName( + LookupResult &R, Scope *S, CXXScopeSpec &SS, QualType ObjectType, + bool EnteringContext, bool &MemberOfUnknownSpecialization, + RequiredTemplateKind RequiredTemplate = SourceLocation(), + AssumedTemplateKind *ATK = nullptr, bool AllowTypoCorrection = true); - /// The queue of implicit template instantiations that are required - /// and must be performed within the current local scope. - /// - /// This queue is only used for member functions of local classes in - /// templates, which must be instantiated in the same scope as their - /// enclosing function, so that they can reference function-local - /// types, static variables, enumerators, etc. - std::deque PendingLocalImplicitInstantiations; + TemplateNameKind isTemplateName(Scope *S, CXXScopeSpec &SS, + bool hasTemplateKeyword, + const UnqualifiedId &Name, + ParsedType ObjectType, bool EnteringContext, + TemplateTy &Template, + bool &MemberOfUnknownSpecialization, + bool Disambiguation = false); - class LocalEagerInstantiationScope { - public: - LocalEagerInstantiationScope(Sema &S) : S(S) { - SavedPendingLocalImplicitInstantiations.swap( - S.PendingLocalImplicitInstantiations); - } + /// Try to resolve an undeclared template name as a type template. + /// + /// Sets II to the identifier corresponding to the template name, and updates + /// Name to a corresponding (typo-corrected) type template name and TNK to + /// the corresponding kind, if possible. + void ActOnUndeclaredTypeTemplateName(Scope *S, TemplateTy &Name, + TemplateNameKind &TNK, + SourceLocation NameLoc, + IdentifierInfo *&II); - void perform() { S.PerformPendingInstantiations(/*LocalOnly=*/true); } + bool resolveAssumedTemplateNameAsType(Scope *S, TemplateName &Name, + SourceLocation NameLoc, + bool Diagnose = true); - ~LocalEagerInstantiationScope() { - assert(S.PendingLocalImplicitInstantiations.empty() && - "there shouldn't be any pending local implicit instantiations"); - SavedPendingLocalImplicitInstantiations.swap( - S.PendingLocalImplicitInstantiations); - } + /// Determine whether a particular identifier might be the name in a C++1z + /// deduction-guide declaration. + bool isDeductionGuideName(Scope *S, const IdentifierInfo &Name, + SourceLocation NameLoc, CXXScopeSpec &SS, + ParsedTemplateTy *Template = nullptr); - private: - Sema &S; - std::deque - SavedPendingLocalImplicitInstantiations; - }; + bool DiagnoseUnknownTemplateName(const IdentifierInfo &II, + SourceLocation IILoc, Scope *S, + const CXXScopeSpec *SS, + TemplateTy &SuggestedTemplate, + TemplateNameKind &SuggestedKind); - /// A helper class for building up ExtParameterInfos. - class ExtParameterInfoBuilder { - SmallVector Infos; - bool HasInteresting = false; + bool DiagnoseUninstantiableTemplate(SourceLocation PointOfInstantiation, + NamedDecl *Instantiation, + bool InstantiatedFromMember, + const NamedDecl *Pattern, + const NamedDecl *PatternDef, + TemplateSpecializationKind TSK, + bool Complain = true); - public: - /// Set the ExtParameterInfo for the parameter at the given index, - /// - void set(unsigned index, FunctionProtoType::ExtParameterInfo info) { - assert(Infos.size() <= index); - Infos.resize(index); - Infos.push_back(info); + /// DiagnoseTemplateParameterShadow - Produce a diagnostic complaining + /// that the template parameter 'PrevDecl' is being shadowed by a new + /// declaration at location Loc. Returns true to indicate that this is + /// an error, and false otherwise. + /// + /// \param Loc The location of the declaration that shadows a template + /// parameter. + /// + /// \param PrevDecl The template parameter that the declaration shadows. + /// + /// \param SupportedForCompatibility Whether to issue the diagnostic as + /// a warning for compatibility with older versions of clang. + /// Ignored when MSVC compatibility is enabled. + void DiagnoseTemplateParameterShadow(SourceLocation Loc, Decl *PrevDecl, + bool SupportedForCompatibility = false); + TemplateDecl *AdjustDeclIfTemplate(Decl *&Decl); - if (!HasInteresting) - HasInteresting = (info != FunctionProtoType::ExtParameterInfo()); - } + NamedDecl *ActOnTypeParameter(Scope *S, bool Typename, + SourceLocation EllipsisLoc, + SourceLocation KeyLoc, + IdentifierInfo *ParamName, + SourceLocation ParamNameLoc, unsigned Depth, + unsigned Position, SourceLocation EqualLoc, + ParsedType DefaultArg, bool HasTypeConstraint); - /// Return a pointer (suitable for setting in an ExtProtoInfo) to the - /// ExtParameterInfo array we've built up. - const FunctionProtoType::ExtParameterInfo * - getPointerOrNull(unsigned numParams) { - if (!HasInteresting) return nullptr; - Infos.resize(numParams); - return Infos.data(); - } - }; + bool CheckTypeConstraint(TemplateIdAnnotation *TypeConstraint); - void PerformPendingInstantiations(bool LocalOnly = false); + bool ActOnTypeConstraint(const CXXScopeSpec &SS, + TemplateIdAnnotation *TypeConstraint, + TemplateTypeParmDecl *ConstrainedParameter, + SourceLocation EllipsisLoc); + bool BuildTypeConstraint(const CXXScopeSpec &SS, + TemplateIdAnnotation *TypeConstraint, + TemplateTypeParmDecl *ConstrainedParameter, + SourceLocation EllipsisLoc, + bool AllowUnexpandedPack); - TypeSourceInfo *SubstType(TypeSourceInfo *T, - const MultiLevelTemplateArgumentList &TemplateArgs, - SourceLocation Loc, DeclarationName Entity, - bool AllowDeducedTST = false); + bool AttachTypeConstraint(NestedNameSpecifierLoc NS, + DeclarationNameInfo NameInfo, + ConceptDecl *NamedConcept, + const TemplateArgumentListInfo *TemplateArgs, + TemplateTypeParmDecl *ConstrainedParameter, + SourceLocation EllipsisLoc); - QualType SubstType(QualType T, - const MultiLevelTemplateArgumentList &TemplateArgs, - SourceLocation Loc, DeclarationName Entity); + bool AttachTypeConstraint(AutoTypeLoc TL, + NonTypeTemplateParmDecl *NewConstrainedParm, + NonTypeTemplateParmDecl *OrigConstrainedParm, + SourceLocation EllipsisLoc); - TypeSourceInfo *SubstType(TypeLoc TL, - const MultiLevelTemplateArgumentList &TemplateArgs, - SourceLocation Loc, DeclarationName Entity); + bool RequireStructuralType(QualType T, SourceLocation Loc); - TypeSourceInfo *SubstFunctionDeclType( - TypeSourceInfo *T, const MultiLevelTemplateArgumentList &TemplateArgs, - SourceLocation Loc, DeclarationName Entity, CXXRecordDecl *ThisContext, - Qualifiers ThisTypeQuals, bool EvaluateConstraints = true); - void SubstExceptionSpec(FunctionDecl *New, const FunctionProtoType *Proto, - const MultiLevelTemplateArgumentList &Args); - bool SubstExceptionSpec(SourceLocation Loc, - FunctionProtoType::ExceptionSpecInfo &ESI, - SmallVectorImpl &ExceptionStorage, - const MultiLevelTemplateArgumentList &Args); - ParmVarDecl * - SubstParmVarDecl(ParmVarDecl *D, - const MultiLevelTemplateArgumentList &TemplateArgs, - int indexAdjustment, std::optional NumExpansions, - bool ExpectParameterPack, bool EvaluateConstraints = true); - bool SubstParmTypes(SourceLocation Loc, ArrayRef Params, - const FunctionProtoType::ExtParameterInfo *ExtParamInfos, - const MultiLevelTemplateArgumentList &TemplateArgs, - SmallVectorImpl &ParamTypes, - SmallVectorImpl *OutParams, - ExtParameterInfoBuilder &ParamInfos); - bool SubstDefaultArgument(SourceLocation Loc, ParmVarDecl *Param, - const MultiLevelTemplateArgumentList &TemplateArgs, - bool ForCallExpr = false); - ExprResult SubstExpr(Expr *E, - const MultiLevelTemplateArgumentList &TemplateArgs); + QualType CheckNonTypeTemplateParameterType(TypeSourceInfo *&TSI, + SourceLocation Loc); + QualType CheckNonTypeTemplateParameterType(QualType T, SourceLocation Loc); - // A RAII type used by the TemplateDeclInstantiator and TemplateInstantiator - // to disable constraint evaluation, then restore the state. - template struct ConstraintEvalRAII { - InstTy &TI; - bool OldValue; + NamedDecl *ActOnNonTypeTemplateParameter(Scope *S, Declarator &D, + unsigned Depth, unsigned Position, + SourceLocation EqualLoc, + Expr *DefaultArg); + NamedDecl *ActOnTemplateTemplateParameter( + Scope *S, SourceLocation TmpLoc, TemplateParameterList *Params, + SourceLocation EllipsisLoc, IdentifierInfo *ParamName, + SourceLocation ParamNameLoc, unsigned Depth, unsigned Position, + SourceLocation EqualLoc, ParsedTemplateArgument DefaultArg); + + TemplateParameterList *ActOnTemplateParameterList( + unsigned Depth, SourceLocation ExportLoc, SourceLocation TemplateLoc, + SourceLocation LAngleLoc, ArrayRef Params, + SourceLocation RAngleLoc, Expr *RequiresClause); - ConstraintEvalRAII(InstTy &TI) - : TI(TI), OldValue(TI.getEvaluateConstraints()) { - TI.setEvaluateConstraints(false); - } - ~ConstraintEvalRAII() { TI.setEvaluateConstraints(OldValue); } + /// The context in which we are checking a template parameter list. + enum TemplateParamListContext { + TPC_ClassTemplate, + TPC_VarTemplate, + TPC_FunctionTemplate, + TPC_ClassTemplateMember, + TPC_FriendClassTemplate, + TPC_FriendFunctionTemplate, + TPC_FriendFunctionTemplateDefinition, + TPC_TypeAliasTemplate }; - // Must be used instead of SubstExpr at 'constraint checking' time. - ExprResult - SubstConstraintExpr(Expr *E, - const MultiLevelTemplateArgumentList &TemplateArgs); - // Unlike the above, this does not evaluates constraints. - ExprResult SubstConstraintExprWithoutSatisfaction( - Expr *E, const MultiLevelTemplateArgumentList &TemplateArgs); - - /// Substitute the given template arguments into a list of - /// expressions, expanding pack expansions if required. - /// - /// \param Exprs The list of expressions to substitute into. - /// - /// \param IsCall Whether this is some form of call, in which case - /// default arguments will be dropped. - /// - /// \param TemplateArgs The set of template arguments to substitute. - /// - /// \param Outputs Will receive all of the substituted arguments. - /// - /// \returns true if an error occurred, false otherwise. - bool SubstExprs(ArrayRef Exprs, bool IsCall, - const MultiLevelTemplateArgumentList &TemplateArgs, - SmallVectorImpl &Outputs); + bool CheckTemplateParameterList(TemplateParameterList *NewParams, + TemplateParameterList *OldParams, + TemplateParamListContext TPC, + SkipBodyInfo *SkipBody = nullptr); + TemplateParameterList *MatchTemplateParametersToScopeSpecifier( + SourceLocation DeclStartLoc, SourceLocation DeclLoc, + const CXXScopeSpec &SS, TemplateIdAnnotation *TemplateId, + ArrayRef ParamLists, bool IsFriend, + bool &IsMemberSpecialization, bool &Invalid, + bool SuppressDiagnostic = false); - StmtResult SubstStmt(Stmt *S, - const MultiLevelTemplateArgumentList &TemplateArgs); + DeclResult CheckClassTemplate( + Scope *S, unsigned TagSpec, TagUseKind TUK, SourceLocation KWLoc, + CXXScopeSpec &SS, IdentifierInfo *Name, SourceLocation NameLoc, + const ParsedAttributesView &Attr, TemplateParameterList *TemplateParams, + AccessSpecifier AS, SourceLocation ModulePrivateLoc, + SourceLocation FriendLoc, unsigned NumOuterTemplateParamLists, + TemplateParameterList **OuterTemplateParamLists, + SkipBodyInfo *SkipBody = nullptr); - TemplateParameterList * - SubstTemplateParams(TemplateParameterList *Params, DeclContext *Owner, - const MultiLevelTemplateArgumentList &TemplateArgs, - bool EvaluateConstraints = true); - - bool - SubstTemplateArguments(ArrayRef Args, - const MultiLevelTemplateArgumentList &TemplateArgs, - TemplateArgumentListInfo &Outputs); - - Decl *SubstDecl(Decl *D, DeclContext *Owner, - const MultiLevelTemplateArgumentList &TemplateArgs); - - /// Substitute the name and return type of a defaulted 'operator<=>' to form - /// an implicit 'operator=='. - FunctionDecl *SubstSpaceshipAsEqualEqual(CXXRecordDecl *RD, - FunctionDecl *Spaceship); - - ExprResult SubstInitializer(Expr *E, - const MultiLevelTemplateArgumentList &TemplateArgs, - bool CXXDirectInit); - - bool - SubstBaseSpecifiers(CXXRecordDecl *Instantiation, - CXXRecordDecl *Pattern, - const MultiLevelTemplateArgumentList &TemplateArgs); + void translateTemplateArguments(const ASTTemplateArgsPtr &In, + TemplateArgumentListInfo &Out); - bool - InstantiateClass(SourceLocation PointOfInstantiation, - CXXRecordDecl *Instantiation, CXXRecordDecl *Pattern, - const MultiLevelTemplateArgumentList &TemplateArgs, - TemplateSpecializationKind TSK, - bool Complain = true); + ParsedTemplateArgument ActOnTemplateTypeArgument(TypeResult ParsedType); - bool InstantiateEnum(SourceLocation PointOfInstantiation, - EnumDecl *Instantiation, EnumDecl *Pattern, - const MultiLevelTemplateArgumentList &TemplateArgs, - TemplateSpecializationKind TSK); + void NoteAllFoundTemplates(TemplateName Name); - bool InstantiateInClassInitializer( - SourceLocation PointOfInstantiation, FieldDecl *Instantiation, - FieldDecl *Pattern, const MultiLevelTemplateArgumentList &TemplateArgs); + QualType CheckTemplateIdType(TemplateName Template, + SourceLocation TemplateLoc, + TemplateArgumentListInfo &TemplateArgs); - struct LateInstantiatedAttribute { - const Attr *TmplAttr; - LocalInstantiationScope *Scope; - Decl *NewDecl; + TypeResult + ActOnTemplateIdType(Scope *S, CXXScopeSpec &SS, SourceLocation TemplateKWLoc, + TemplateTy Template, IdentifierInfo *TemplateII, + SourceLocation TemplateIILoc, SourceLocation LAngleLoc, + ASTTemplateArgsPtr TemplateArgs, SourceLocation RAngleLoc, + bool IsCtorOrDtorName = false, bool IsClassName = false, + ImplicitTypenameContext AllowImplicitTypename = + ImplicitTypenameContext::No); - LateInstantiatedAttribute(const Attr *A, LocalInstantiationScope *S, - Decl *D) - : TmplAttr(A), Scope(S), NewDecl(D) - { } - }; - typedef SmallVector LateInstantiatedAttrVec; + /// Parsed an elaborated-type-specifier that refers to a template-id, + /// such as \c class T::template apply. + TypeResult ActOnTagTemplateIdType( + TagUseKind TUK, TypeSpecifierType TagSpec, SourceLocation TagLoc, + CXXScopeSpec &SS, SourceLocation TemplateKWLoc, TemplateTy TemplateD, + SourceLocation TemplateLoc, SourceLocation LAngleLoc, + ASTTemplateArgsPtr TemplateArgsIn, SourceLocation RAngleLoc); - void InstantiateAttrs(const MultiLevelTemplateArgumentList &TemplateArgs, - const Decl *Pattern, Decl *Inst, - LateInstantiatedAttrVec *LateAttrs = nullptr, - LocalInstantiationScope *OuterMostScope = nullptr); - void updateAttrsForLateParsedTemplate(const Decl *Pattern, Decl *Inst); + DeclResult ActOnVarTemplateSpecialization( + Scope *S, Declarator &D, TypeSourceInfo *DI, LookupResult &Previous, + SourceLocation TemplateKWLoc, TemplateParameterList *TemplateParams, + StorageClass SC, bool IsPartialSpecialization); - void - InstantiateAttrsForDecl(const MultiLevelTemplateArgumentList &TemplateArgs, - const Decl *Pattern, Decl *Inst, - LateInstantiatedAttrVec *LateAttrs = nullptr, - LocalInstantiationScope *OuterMostScope = nullptr); + /// Get the specialization of the given variable template corresponding to + /// the specified argument list, or a null-but-valid result if the arguments + /// are dependent. + DeclResult CheckVarTemplateId(VarTemplateDecl *Template, + SourceLocation TemplateLoc, + SourceLocation TemplateNameLoc, + const TemplateArgumentListInfo &TemplateArgs); - void InstantiateDefaultCtorDefaultArgs(CXXConstructorDecl *Ctor); + /// Form a reference to the specialization of the given variable template + /// corresponding to the specified argument list, or a null-but-valid result + /// if the arguments are dependent. + ExprResult CheckVarTemplateId(const CXXScopeSpec &SS, + const DeclarationNameInfo &NameInfo, + VarTemplateDecl *Template, NamedDecl *FoundD, + SourceLocation TemplateLoc, + const TemplateArgumentListInfo *TemplateArgs); - bool usesPartialOrExplicitSpecialization( - SourceLocation Loc, ClassTemplateSpecializationDecl *ClassTemplateSpec); + ExprResult + CheckConceptTemplateId(const CXXScopeSpec &SS, SourceLocation TemplateKWLoc, + const DeclarationNameInfo &ConceptNameInfo, + NamedDecl *FoundDecl, ConceptDecl *NamedConcept, + const TemplateArgumentListInfo *TemplateArgs); - bool - InstantiateClassTemplateSpecialization(SourceLocation PointOfInstantiation, - ClassTemplateSpecializationDecl *ClassTemplateSpec, - TemplateSpecializationKind TSK, - bool Complain = true); + void diagnoseMissingTemplateArguments(TemplateName Name, SourceLocation Loc); - void InstantiateClassMembers(SourceLocation PointOfInstantiation, - CXXRecordDecl *Instantiation, - const MultiLevelTemplateArgumentList &TemplateArgs, - TemplateSpecializationKind TSK); + ExprResult BuildTemplateIdExpr(const CXXScopeSpec &SS, + SourceLocation TemplateKWLoc, LookupResult &R, + bool RequiresADL, + const TemplateArgumentListInfo *TemplateArgs); - void InstantiateClassTemplateSpecializationMembers( - SourceLocation PointOfInstantiation, - ClassTemplateSpecializationDecl *ClassTemplateSpec, - TemplateSpecializationKind TSK); + ExprResult + BuildQualifiedTemplateIdExpr(CXXScopeSpec &SS, SourceLocation TemplateKWLoc, + const DeclarationNameInfo &NameInfo, + const TemplateArgumentListInfo *TemplateArgs); - NestedNameSpecifierLoc - SubstNestedNameSpecifierLoc(NestedNameSpecifierLoc NNS, - const MultiLevelTemplateArgumentList &TemplateArgs); + TemplateNameKind ActOnTemplateName(Scope *S, CXXScopeSpec &SS, + SourceLocation TemplateKWLoc, + const UnqualifiedId &Name, + ParsedType ObjectType, + bool EnteringContext, TemplateTy &Template, + bool AllowInjectedClassName = false); - DeclarationNameInfo - SubstDeclarationNameInfo(const DeclarationNameInfo &NameInfo, - const MultiLevelTemplateArgumentList &TemplateArgs); - TemplateName - SubstTemplateName(NestedNameSpecifierLoc QualifierLoc, TemplateName Name, - SourceLocation Loc, - const MultiLevelTemplateArgumentList &TemplateArgs); + DeclResult ActOnClassTemplateSpecialization( + Scope *S, unsigned TagSpec, TagUseKind TUK, SourceLocation KWLoc, + SourceLocation ModulePrivateLoc, CXXScopeSpec &SS, + TemplateIdAnnotation &TemplateId, const ParsedAttributesView &Attr, + MultiTemplateParamsArg TemplateParameterLists, + SkipBodyInfo *SkipBody = nullptr); - bool SubstTypeConstraint(TemplateTypeParmDecl *Inst, const TypeConstraint *TC, - const MultiLevelTemplateArgumentList &TemplateArgs, - bool EvaluateConstraint); + bool CheckTemplatePartialSpecializationArgs(SourceLocation Loc, + TemplateDecl *PrimaryTemplate, + unsigned NumExplicitArgs, + ArrayRef Args); + void CheckTemplatePartialSpecialization( + ClassTemplatePartialSpecializationDecl *Partial); + void CheckTemplatePartialSpecialization( + VarTemplatePartialSpecializationDecl *Partial); - bool InstantiateDefaultArgument(SourceLocation CallLoc, FunctionDecl *FD, - ParmVarDecl *Param); - void InstantiateExceptionSpec(SourceLocation PointOfInstantiation, - FunctionDecl *Function); - bool CheckInstantiatedFunctionTemplateConstraints( - SourceLocation PointOfInstantiation, FunctionDecl *Decl, - ArrayRef TemplateArgs, - ConstraintSatisfaction &Satisfaction); - FunctionDecl *InstantiateFunctionDeclaration(FunctionTemplateDecl *FTD, - const TemplateArgumentList *Args, - SourceLocation Loc); - void InstantiateFunctionDefinition(SourceLocation PointOfInstantiation, - FunctionDecl *Function, - bool Recursive = false, - bool DefinitionRequired = false, - bool AtEndOfTU = false); - VarTemplateSpecializationDecl *BuildVarTemplateInstantiation( - VarTemplateDecl *VarTemplate, VarDecl *FromVar, - const TemplateArgumentList *PartialSpecArgs, - const TemplateArgumentListInfo &TemplateArgsInfo, - SmallVectorImpl &Converted, - SourceLocation PointOfInstantiation, - LateInstantiatedAttrVec *LateAttrs = nullptr, - LocalInstantiationScope *StartingScope = nullptr); - VarTemplateSpecializationDecl *CompleteVarTemplateSpecializationDecl( - VarTemplateSpecializationDecl *VarSpec, VarDecl *PatternDecl, - const MultiLevelTemplateArgumentList &TemplateArgs); - void - BuildVariableInstantiation(VarDecl *NewVar, VarDecl *OldVar, - const MultiLevelTemplateArgumentList &TemplateArgs, - LateInstantiatedAttrVec *LateAttrs, - DeclContext *Owner, - LocalInstantiationScope *StartingScope, - bool InstantiatingVarTemplate = false, - VarTemplateSpecializationDecl *PrevVTSD = nullptr); + Decl *ActOnTemplateDeclarator(Scope *S, + MultiTemplateParamsArg TemplateParameterLists, + Declarator &D); - void InstantiateVariableInitializer( - VarDecl *Var, VarDecl *OldVar, - const MultiLevelTemplateArgumentList &TemplateArgs); - void InstantiateVariableDefinition(SourceLocation PointOfInstantiation, - VarDecl *Var, bool Recursive = false, - bool DefinitionRequired = false, - bool AtEndOfTU = false); + bool CheckSpecializationInstantiationRedecl( + SourceLocation NewLoc, + TemplateSpecializationKind ActOnExplicitInstantiationNewTSK, + NamedDecl *PrevDecl, TemplateSpecializationKind PrevTSK, + SourceLocation PrevPtOfInstantiation, bool &SuppressNew); - void InstantiateMemInitializers(CXXConstructorDecl *New, - const CXXConstructorDecl *Tmpl, - const MultiLevelTemplateArgumentList &TemplateArgs); + bool CheckDependentFunctionTemplateSpecialization( + FunctionDecl *FD, const TemplateArgumentListInfo *ExplicitTemplateArgs, + LookupResult &Previous); - ExplicitSpecifier instantiateExplicitSpecifier( - const MultiLevelTemplateArgumentList &TemplateArgs, ExplicitSpecifier ES); + bool CheckFunctionTemplateSpecialization( + FunctionDecl *FD, TemplateArgumentListInfo *ExplicitTemplateArgs, + LookupResult &Previous, bool QualifiedFriend = false); + bool CheckMemberSpecialization(NamedDecl *Member, LookupResult &Previous); + void CompleteMemberSpecialization(NamedDecl *Member, LookupResult &Previous); - NamedDecl *FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D, - const MultiLevelTemplateArgumentList &TemplateArgs, - bool FindingInstantiatedContext = false); - DeclContext *FindInstantiatedContext(SourceLocation Loc, DeclContext *DC, - const MultiLevelTemplateArgumentList &TemplateArgs); + DeclResult ActOnExplicitInstantiation( + Scope *S, SourceLocation ExternLoc, SourceLocation TemplateLoc, + unsigned TagSpec, SourceLocation KWLoc, const CXXScopeSpec &SS, + TemplateTy Template, SourceLocation TemplateNameLoc, + SourceLocation LAngleLoc, ASTTemplateArgsPtr TemplateArgs, + SourceLocation RAngleLoc, const ParsedAttributesView &Attr); - // Objective-C declarations. - enum ObjCContainerKind { - OCK_None = -1, - OCK_Interface = 0, - OCK_Protocol, - OCK_Category, - OCK_ClassExtension, - OCK_Implementation, - OCK_CategoryImplementation - }; - ObjCContainerKind getObjCContainerKind() const; + DeclResult ActOnExplicitInstantiation(Scope *S, SourceLocation ExternLoc, + SourceLocation TemplateLoc, + unsigned TagSpec, SourceLocation KWLoc, + CXXScopeSpec &SS, IdentifierInfo *Name, + SourceLocation NameLoc, + const ParsedAttributesView &Attr); - DeclResult actOnObjCTypeParam(Scope *S, - ObjCTypeParamVariance variance, - SourceLocation varianceLoc, - unsigned index, - IdentifierInfo *paramName, - SourceLocation paramLoc, - SourceLocation colonLoc, - ParsedType typeBound); + DeclResult ActOnExplicitInstantiation(Scope *S, SourceLocation ExternLoc, + SourceLocation TemplateLoc, + Declarator &D); - ObjCTypeParamList *actOnObjCTypeParamList(Scope *S, SourceLocation lAngleLoc, - ArrayRef typeParams, - SourceLocation rAngleLoc); - void popObjCTypeParamList(Scope *S, ObjCTypeParamList *typeParamList); + TemplateArgumentLoc SubstDefaultTemplateArgumentIfAvailable( + TemplateDecl *Template, SourceLocation TemplateLoc, + SourceLocation RAngleLoc, Decl *Param, + ArrayRef SugaredConverted, + ArrayRef CanonicalConverted, bool &HasDefaultArg); - ObjCInterfaceDecl *ActOnStartClassInterface( - Scope *S, SourceLocation AtInterfaceLoc, IdentifierInfo *ClassName, - SourceLocation ClassLoc, ObjCTypeParamList *typeParamList, - IdentifierInfo *SuperName, SourceLocation SuperLoc, - ArrayRef SuperTypeArgs, SourceRange SuperTypeArgsRange, - Decl *const *ProtoRefs, unsigned NumProtoRefs, - const SourceLocation *ProtoLocs, SourceLocation EndProtoLoc, - const ParsedAttributesView &AttrList, SkipBodyInfo *SkipBody); + SourceLocation getTopMostPointOfInstantiation(const NamedDecl *) const; - void ActOnSuperClassOfClassInterface(Scope *S, - SourceLocation AtInterfaceLoc, - ObjCInterfaceDecl *IDecl, - IdentifierInfo *ClassName, - SourceLocation ClassLoc, - IdentifierInfo *SuperName, - SourceLocation SuperLoc, - ArrayRef SuperTypeArgs, - SourceRange SuperTypeArgsRange); + /// Specifies the context in which a particular template + /// argument is being checked. + enum CheckTemplateArgumentKind { + /// The template argument was specified in the code or was + /// instantiated with some deduced template arguments. + CTAK_Specified, - void ActOnTypedefedProtocols(SmallVectorImpl &ProtocolRefs, - SmallVectorImpl &ProtocolLocs, - IdentifierInfo *SuperName, - SourceLocation SuperLoc); + /// The template argument was deduced via template argument + /// deduction. + CTAK_Deduced, - Decl *ActOnCompatibilityAlias( - SourceLocation AtCompatibilityAliasLoc, - IdentifierInfo *AliasName, SourceLocation AliasLocation, - IdentifierInfo *ClassName, SourceLocation ClassLocation); + /// The template argument was deduced from an array bound + /// via template argument deduction. + CTAK_DeducedFromArrayBound + }; - bool CheckForwardProtocolDeclarationForCircularDependency( - IdentifierInfo *PName, - SourceLocation &PLoc, SourceLocation PrevLoc, - const ObjCList &PList); + bool + CheckTemplateArgument(NamedDecl *Param, TemplateArgumentLoc &Arg, + NamedDecl *Template, SourceLocation TemplateLoc, + SourceLocation RAngleLoc, unsigned ArgumentPackIndex, + SmallVectorImpl &SugaredConverted, + SmallVectorImpl &CanonicalConverted, + CheckTemplateArgumentKind CTAK); - ObjCProtocolDecl *ActOnStartProtocolInterface( - SourceLocation AtProtoInterfaceLoc, IdentifierInfo *ProtocolName, - SourceLocation ProtocolLoc, Decl *const *ProtoRefNames, - unsigned NumProtoRefs, const SourceLocation *ProtoLocs, - SourceLocation EndProtoLoc, const ParsedAttributesView &AttrList, - SkipBodyInfo *SkipBody); - - ObjCCategoryDecl *ActOnStartCategoryInterface( - SourceLocation AtInterfaceLoc, IdentifierInfo *ClassName, - SourceLocation ClassLoc, ObjCTypeParamList *typeParamList, - IdentifierInfo *CategoryName, SourceLocation CategoryLoc, - Decl *const *ProtoRefs, unsigned NumProtoRefs, - const SourceLocation *ProtoLocs, SourceLocation EndProtoLoc, - const ParsedAttributesView &AttrList); - - ObjCImplementationDecl *ActOnStartClassImplementation( - SourceLocation AtClassImplLoc, IdentifierInfo *ClassName, - SourceLocation ClassLoc, IdentifierInfo *SuperClassname, - SourceLocation SuperClassLoc, const ParsedAttributesView &AttrList); - - ObjCCategoryImplDecl *ActOnStartCategoryImplementation( - SourceLocation AtCatImplLoc, IdentifierInfo *ClassName, - SourceLocation ClassLoc, IdentifierInfo *CatName, SourceLocation CatLoc, - const ParsedAttributesView &AttrList); - - DeclGroupPtrTy ActOnFinishObjCImplementation(Decl *ObjCImpDecl, - ArrayRef Decls); - - DeclGroupPtrTy ActOnForwardClassDeclaration(SourceLocation Loc, - IdentifierInfo **IdentList, - SourceLocation *IdentLocs, - ArrayRef TypeParamLists, - unsigned NumElts); + /// Check that the given template arguments can be provided to + /// the given template, converting the arguments along the way. + /// + /// \param Template The template to which the template arguments are being + /// provided. + /// + /// \param TemplateLoc The location of the template name in the source. + /// + /// \param TemplateArgs The list of template arguments. If the template is + /// a template template parameter, this function may extend the set of + /// template arguments to also include substituted, defaulted template + /// arguments. + /// + /// \param PartialTemplateArgs True if the list of template arguments is + /// intentionally partial, e.g., because we're checking just the initial + /// set of template arguments. + /// + /// \param Converted Will receive the converted, canonicalized template + /// arguments. + /// + /// \param UpdateArgsWithConversions If \c true, update \p TemplateArgs to + /// contain the converted forms of the template arguments as written. + /// Otherwise, \p TemplateArgs will not be modified. + /// + /// \param ConstraintsNotSatisfied If provided, and an error occurred, will + /// receive true if the cause for the error is the associated constraints of + /// the template not being satisfied by the template arguments. + /// + /// \returns true if an error occurred, false otherwise. + bool CheckTemplateArgumentList( + TemplateDecl *Template, SourceLocation TemplateLoc, + TemplateArgumentListInfo &TemplateArgs, bool PartialTemplateArgs, + SmallVectorImpl &SugaredConverted, + SmallVectorImpl &CanonicalConverted, + bool UpdateArgsWithConversions = true, + bool *ConstraintsNotSatisfied = nullptr); - DeclGroupPtrTy - ActOnForwardProtocolDeclaration(SourceLocation AtProtoclLoc, - ArrayRef IdentList, - const ParsedAttributesView &attrList); + bool CheckTemplateTypeArgument( + TemplateTypeParmDecl *Param, TemplateArgumentLoc &Arg, + SmallVectorImpl &SugaredConverted, + SmallVectorImpl &CanonicalConverted); - void FindProtocolDeclaration(bool WarnOnDeclarations, bool ForObjCContainer, - ArrayRef ProtocolId, - SmallVectorImpl &Protocols); + bool CheckTemplateArgument(TypeSourceInfo *Arg); + ExprResult CheckTemplateArgument(NonTypeTemplateParmDecl *Param, + QualType InstantiatedParamType, Expr *Arg, + TemplateArgument &SugaredConverted, + TemplateArgument &CanonicalConverted, + CheckTemplateArgumentKind CTAK); + bool CheckTemplateTemplateArgument(TemplateTemplateParmDecl *Param, + TemplateParameterList *Params, + TemplateArgumentLoc &Arg); - void DiagnoseTypeArgsAndProtocols(IdentifierInfo *ProtocolId, - SourceLocation ProtocolLoc, - IdentifierInfo *TypeArgId, - SourceLocation TypeArgLoc, - bool SelectProtocolFirst = false); + void NoteTemplateLocation(const NamedDecl &Decl, + std::optional ParamRange = {}); + void NoteTemplateParameterLocation(const NamedDecl &Decl); - /// Given a list of identifiers (and their locations), resolve the - /// names to either Objective-C protocol qualifiers or type - /// arguments, as appropriate. - void actOnObjCTypeArgsOrProtocolQualifiers( - Scope *S, - ParsedType baseType, - SourceLocation lAngleLoc, - ArrayRef identifiers, - ArrayRef identifierLocs, - SourceLocation rAngleLoc, - SourceLocation &typeArgsLAngleLoc, - SmallVectorImpl &typeArgs, - SourceLocation &typeArgsRAngleLoc, - SourceLocation &protocolLAngleLoc, - SmallVectorImpl &protocols, - SourceLocation &protocolRAngleLoc, - bool warnOnIncompleteProtocols); + ExprResult BuildExpressionFromDeclTemplateArgument( + const TemplateArgument &Arg, QualType ParamType, SourceLocation Loc); + ExprResult + BuildExpressionFromNonTypeTemplateArgument(const TemplateArgument &Arg, + SourceLocation Loc); - /// Build a an Objective-C protocol-qualified 'id' type where no - /// base type was specified. - TypeResult actOnObjCProtocolQualifierType( - SourceLocation lAngleLoc, - ArrayRef protocols, - ArrayRef protocolLocs, - SourceLocation rAngleLoc); + /// Enumeration describing how template parameter lists are compared + /// for equality. + enum TemplateParameterListEqualKind { + /// We are matching the template parameter lists of two templates + /// that might be redeclarations. + /// + /// \code + /// template struct X; + /// template struct X; + /// \endcode + TPL_TemplateMatch, - /// Build a specialized and/or protocol-qualified Objective-C type. - TypeResult actOnObjCTypeArgsAndProtocolQualifiers( - Scope *S, - SourceLocation Loc, - ParsedType BaseType, - SourceLocation TypeArgsLAngleLoc, - ArrayRef TypeArgs, - SourceLocation TypeArgsRAngleLoc, - SourceLocation ProtocolLAngleLoc, - ArrayRef Protocols, - ArrayRef ProtocolLocs, - SourceLocation ProtocolRAngleLoc); + /// We are matching the template parameter lists of two template + /// template parameters as part of matching the template parameter lists + /// of two templates that might be redeclarations. + /// + /// \code + /// template class TT> struct X; + /// template class Other> struct X; + /// \endcode + TPL_TemplateTemplateParmMatch, - /// Build an Objective-C type parameter type. - QualType BuildObjCTypeParamType(const ObjCTypeParamDecl *Decl, - SourceLocation ProtocolLAngleLoc, - ArrayRef Protocols, - ArrayRef ProtocolLocs, - SourceLocation ProtocolRAngleLoc, - bool FailOnError = false); + /// We are matching the template parameter lists of a template + /// template argument against the template parameter lists of a template + /// template parameter. + /// + /// \code + /// template class Metafun> struct X; + /// template struct integer_c; + /// X xic; + /// \endcode + TPL_TemplateTemplateArgumentMatch, - /// Build an Objective-C object pointer type. - QualType BuildObjCObjectType( - QualType BaseType, SourceLocation Loc, SourceLocation TypeArgsLAngleLoc, - ArrayRef TypeArgs, SourceLocation TypeArgsRAngleLoc, - SourceLocation ProtocolLAngleLoc, ArrayRef Protocols, - ArrayRef ProtocolLocs, SourceLocation ProtocolRAngleLoc, - bool FailOnError, bool Rebuilding); + /// We are determining whether the template-parameters are equivalent + /// according to C++ [temp.over.link]/6. This comparison does not consider + /// constraints. + /// + /// \code + /// template void f(T); + /// template void f(T); + /// \endcode + TPL_TemplateParamsEquivalent, + }; - /// Ensure attributes are consistent with type. - /// \param [in, out] Attributes The attributes to check; they will - /// be modified to be consistent with \p PropertyTy. - void CheckObjCPropertyAttributes(Decl *PropertyPtrTy, - SourceLocation Loc, - unsigned &Attributes, - bool propertyInPrimaryClass); + // A struct to represent the 'new' declaration, which is either itself just + // the named decl, or the important information we need about it in order to + // do constraint comparisons. + class TemplateCompareNewDeclInfo { + const NamedDecl *ND = nullptr; + const DeclContext *DC = nullptr; + const DeclContext *LexicalDC = nullptr; + SourceLocation Loc; - /// Process the specified property declaration and create decls for the - /// setters and getters as needed. - /// \param property The property declaration being processed - void ProcessPropertyDecl(ObjCPropertyDecl *property); + public: + TemplateCompareNewDeclInfo(const NamedDecl *ND) : ND(ND) {} + TemplateCompareNewDeclInfo(const DeclContext *DeclCtx, + const DeclContext *LexicalDeclCtx, + SourceLocation Loc) + : DC(DeclCtx), LexicalDC(LexicalDeclCtx), Loc(Loc) { + assert(DC && LexicalDC && + "Constructor only for cases where we have the information to put " + "in here"); + } - void DiagnosePropertyMismatch(ObjCPropertyDecl *Property, - ObjCPropertyDecl *SuperProperty, - const IdentifierInfo *Name, - bool OverridingProtocolProperty); + // If this was constructed with no information, we cannot do substitution + // for constraint comparison, so make sure we can check that. + bool isInvalid() const { return !ND && !DC; } - void DiagnoseClassExtensionDupMethods(ObjCCategoryDecl *CAT, - ObjCInterfaceDecl *ID); + const NamedDecl *getDecl() const { return ND; } - Decl *ActOnAtEnd(Scope *S, SourceRange AtEnd, - ArrayRef allMethods = std::nullopt, - ArrayRef allTUVars = std::nullopt); + bool ContainsDecl(const NamedDecl *ND) const { return this->ND == ND; } - Decl *ActOnProperty(Scope *S, SourceLocation AtLoc, - SourceLocation LParenLoc, - FieldDeclarator &FD, ObjCDeclSpec &ODS, - Selector GetterSel, Selector SetterSel, - tok::ObjCKeywordKind MethodImplKind, - DeclContext *lexicalDC = nullptr); + const DeclContext *getLexicalDeclContext() const { + return ND ? ND->getLexicalDeclContext() : LexicalDC; + } - Decl *ActOnPropertyImplDecl(Scope *S, - SourceLocation AtLoc, - SourceLocation PropertyLoc, - bool ImplKind, - IdentifierInfo *PropertyId, - IdentifierInfo *PropertyIvar, - SourceLocation PropertyIvarLoc, - ObjCPropertyQueryKind QueryKind); + const DeclContext *getDeclContext() const { + return ND ? ND->getDeclContext() : DC; + } - enum ObjCSpecialMethodKind { - OSMK_None, - OSMK_Alloc, - OSMK_New, - OSMK_Copy, - OSMK_RetainingInit, - OSMK_NonRetainingInit + SourceLocation getLocation() const { return ND ? ND->getLocation() : Loc; } }; - struct ObjCArgInfo { - IdentifierInfo *Name; - SourceLocation NameLoc; - // The Type is null if no type was specified, and the DeclSpec is invalid - // in this case. - ParsedType Type; - ObjCDeclSpec DeclSpec; + bool TemplateParameterListsAreEqual( + const TemplateCompareNewDeclInfo &NewInstFrom, TemplateParameterList *New, + const NamedDecl *OldInstFrom, TemplateParameterList *Old, bool Complain, + TemplateParameterListEqualKind Kind, + SourceLocation TemplateArgLoc = SourceLocation()); - /// ArgAttrs - Attribute list for this argument. - ParsedAttributesView ArgAttrs; - }; + bool TemplateParameterListsAreEqual( + TemplateParameterList *New, TemplateParameterList *Old, bool Complain, + TemplateParameterListEqualKind Kind, + SourceLocation TemplateArgLoc = SourceLocation()) { + return TemplateParameterListsAreEqual(nullptr, New, nullptr, Old, Complain, + Kind, TemplateArgLoc); + } - Decl *ActOnMethodDeclaration( - Scope *S, - SourceLocation BeginLoc, // location of the + or -. - SourceLocation EndLoc, // location of the ; or {. - tok::TokenKind MethodType, ObjCDeclSpec &ReturnQT, ParsedType ReturnType, - ArrayRef SelectorLocs, Selector Sel, - // optional arguments. The number of types/arguments is obtained - // from the Sel.getNumArgs(). - ObjCArgInfo *ArgInfo, DeclaratorChunk::ParamInfo *CParamInfo, - unsigned CNumArgs, // c-style args - const ParsedAttributesView &AttrList, tok::ObjCKeywordKind MethodImplKind, - bool isVariadic, bool MethodDefinition); + bool CheckTemplateDeclScope(Scope *S, TemplateParameterList *TemplateParams); - ObjCMethodDecl *LookupMethodInQualifiedType(Selector Sel, - const ObjCObjectPointerType *OPT, - bool IsInstance); - ObjCMethodDecl *LookupMethodInObjectType(Selector Sel, QualType Ty, - bool IsInstance); - - bool CheckARCMethodDecl(ObjCMethodDecl *method); - bool inferObjCARCLifetime(ValueDecl *decl); + /// Called when the parser has parsed a C++ typename + /// specifier, e.g., "typename T::type". + /// + /// \param S The scope in which this typename type occurs. + /// \param TypenameLoc the location of the 'typename' keyword + /// \param SS the nested-name-specifier following the typename (e.g., 'T::'). + /// \param II the identifier we're retrieving (e.g., 'type' in the example). + /// \param IdLoc the location of the identifier. + /// \param IsImplicitTypename context where T::type refers to a type. + TypeResult ActOnTypenameType( + Scope *S, SourceLocation TypenameLoc, const CXXScopeSpec &SS, + const IdentifierInfo &II, SourceLocation IdLoc, + ImplicitTypenameContext IsImplicitTypename = ImplicitTypenameContext::No); - void deduceOpenCLAddressSpace(ValueDecl *decl); + /// Called when the parser has parsed a C++ typename + /// specifier that ends in a template-id, e.g., + /// "typename MetaFun::template apply". + /// + /// \param S The scope in which this typename type occurs. + /// \param TypenameLoc the location of the 'typename' keyword + /// \param SS the nested-name-specifier following the typename (e.g., 'T::'). + /// \param TemplateLoc the location of the 'template' keyword, if any. + /// \param TemplateName The template name. + /// \param TemplateII The identifier used to name the template. + /// \param TemplateIILoc The location of the template name. + /// \param LAngleLoc The location of the opening angle bracket ('<'). + /// \param TemplateArgs The template arguments. + /// \param RAngleLoc The location of the closing angle bracket ('>'). + TypeResult + ActOnTypenameType(Scope *S, SourceLocation TypenameLoc, + const CXXScopeSpec &SS, SourceLocation TemplateLoc, + TemplateTy TemplateName, IdentifierInfo *TemplateII, + SourceLocation TemplateIILoc, SourceLocation LAngleLoc, + ASTTemplateArgsPtr TemplateArgs, SourceLocation RAngleLoc); - ExprResult - HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT, - Expr *BaseExpr, - SourceLocation OpLoc, - DeclarationName MemberName, - SourceLocation MemberLoc, - SourceLocation SuperLoc, QualType SuperType, - bool Super); + QualType CheckTypenameType(ElaboratedTypeKeyword Keyword, + SourceLocation KeywordLoc, + NestedNameSpecifierLoc QualifierLoc, + const IdentifierInfo &II, SourceLocation IILoc, + TypeSourceInfo **TSI, bool DeducedTSTContext); - ExprResult - ActOnClassPropertyRefExpr(IdentifierInfo &receiverName, - IdentifierInfo &propertyName, - SourceLocation receiverNameLoc, - SourceLocation propertyNameLoc); + QualType CheckTypenameType(ElaboratedTypeKeyword Keyword, + SourceLocation KeywordLoc, + NestedNameSpecifierLoc QualifierLoc, + const IdentifierInfo &II, SourceLocation IILoc, + bool DeducedTSTContext = true); - ObjCMethodDecl *tryCaptureObjCSelf(SourceLocation Loc); + TypeSourceInfo *RebuildTypeInCurrentInstantiation(TypeSourceInfo *T, + SourceLocation Loc, + DeclarationName Name); + bool RebuildNestedNameSpecifierInCurrentInstantiation(CXXScopeSpec &SS); - /// Describes the kind of message expression indicated by a message - /// send that starts with an identifier. - enum ObjCMessageKind { - /// The message is sent to 'super'. - ObjCSuperMessage, - /// The message is an instance message. - ObjCInstanceMessage, - /// The message is a class message, and the identifier is a type - /// name. - ObjCClassMessage - }; + ExprResult RebuildExprInCurrentInstantiation(Expr *E); + bool + RebuildTemplateParamsInCurrentInstantiation(TemplateParameterList *Params); - ObjCMessageKind getObjCMessageKind(Scope *S, - IdentifierInfo *Name, - SourceLocation NameLoc, - bool IsSuper, - bool HasTrailingDot, - ParsedType &ReceiverType); + std::string + getTemplateArgumentBindingsText(const TemplateParameterList *Params, + const TemplateArgumentList &Args); - ExprResult ActOnSuperMessage(Scope *S, SourceLocation SuperLoc, - Selector Sel, - SourceLocation LBracLoc, - ArrayRef SelectorLocs, - SourceLocation RBracLoc, - MultiExprArg Args); + std::string + getTemplateArgumentBindingsText(const TemplateParameterList *Params, + const TemplateArgument *Args, + unsigned NumArgs); - ExprResult BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo, - QualType ReceiverType, - SourceLocation SuperLoc, - Selector Sel, - ObjCMethodDecl *Method, - SourceLocation LBracLoc, - ArrayRef SelectorLocs, - SourceLocation RBracLoc, - MultiExprArg Args, - bool isImplicit = false); + void diagnoseExprIntendedAsTemplateName(Scope *S, ExprResult TemplateName, + SourceLocation Less, + SourceLocation Greater); - ExprResult BuildClassMessageImplicit(QualType ReceiverType, - bool isSuperReceiver, - SourceLocation Loc, - Selector Sel, - ObjCMethodDecl *Method, - MultiExprArg Args); + ExprResult ActOnDependentIdExpression( + const CXXScopeSpec &SS, SourceLocation TemplateKWLoc, + const DeclarationNameInfo &NameInfo, bool isAddressOfOperand, + const TemplateArgumentListInfo *TemplateArgs); - ExprResult ActOnClassMessage(Scope *S, - ParsedType Receiver, - Selector Sel, - SourceLocation LBracLoc, - ArrayRef SelectorLocs, - SourceLocation RBracLoc, - MultiExprArg Args); + ExprResult + BuildDependentDeclRefExpr(const CXXScopeSpec &SS, + SourceLocation TemplateKWLoc, + const DeclarationNameInfo &NameInfo, + const TemplateArgumentListInfo *TemplateArgs); - ExprResult BuildInstanceMessage(Expr *Receiver, - QualType ReceiverType, - SourceLocation SuperLoc, - Selector Sel, - ObjCMethodDecl *Method, - SourceLocation LBracLoc, - ArrayRef SelectorLocs, - SourceLocation RBracLoc, - MultiExprArg Args, - bool isImplicit = false); + // Calculates whether the expression Constraint depends on an enclosing + // template, for the purposes of [temp.friend] p9. + // TemplateDepth is the 'depth' of the friend function, which is used to + // compare whether a declaration reference is referring to a containing + // template, or just the current friend function. A 'lower' TemplateDepth in + // the AST refers to a 'containing' template. As the constraint is + // uninstantiated, this is relative to the 'top' of the TU. + bool + ConstraintExpressionDependsOnEnclosingTemplate(const FunctionDecl *Friend, + unsigned TemplateDepth, + const Expr *Constraint); - ExprResult BuildInstanceMessageImplicit(Expr *Receiver, - QualType ReceiverType, - SourceLocation Loc, - Selector Sel, - ObjCMethodDecl *Method, - MultiExprArg Args); + /// Declare implicit deduction guides for a class template if we've + /// not already done so. + void DeclareImplicitDeductionGuides(TemplateDecl *Template, + SourceLocation Loc); + FunctionTemplateDecl *DeclareImplicitDeductionGuideFromInitList( + TemplateDecl *Template, MutableArrayRef ParamTypes, + SourceLocation Loc); - ExprResult ActOnInstanceMessage(Scope *S, - Expr *Receiver, - Selector Sel, - SourceLocation LBracLoc, - ArrayRef SelectorLocs, - SourceLocation RBracLoc, - MultiExprArg Args); + /// Find the failed Boolean condition within a given Boolean + /// constant expression, and describe it with a string. + std::pair findFailedBooleanCondition(Expr *Cond); - ExprResult BuildObjCBridgedCast(SourceLocation LParenLoc, - ObjCBridgeCastKind Kind, - SourceLocation BridgeKeywordLoc, - TypeSourceInfo *TSInfo, - Expr *SubExpr); + void CheckDeductionGuideTemplate(FunctionTemplateDecl *TD); - ExprResult ActOnObjCBridgedCast(Scope *S, - SourceLocation LParenLoc, - ObjCBridgeCastKind Kind, - SourceLocation BridgeKeywordLoc, - ParsedType Type, - SourceLocation RParenLoc, - Expr *SubExpr); + Decl *ActOnConceptDefinition(Scope *S, + MultiTemplateParamsArg TemplateParameterLists, + IdentifierInfo *Name, SourceLocation NameLoc, + Expr *ConstraintExpr); - void CheckTollFreeBridgeCast(QualType castType, Expr *castExpr); + void CheckConceptRedefinition(ConceptDecl *NewDecl, LookupResult &Previous, + bool &AddToScope); - void CheckObjCBridgeRelatedCast(QualType castType, Expr *castExpr); + TypeResult ActOnDependentTag(Scope *S, unsigned TagSpec, TagUseKind TUK, + const CXXScopeSpec &SS, IdentifierInfo *Name, + SourceLocation TagLoc, SourceLocation NameLoc); - bool CheckTollFreeBridgeStaticCast(QualType castType, Expr *castExpr, - CastKind &Kind); + void MarkAsLateParsedTemplate(FunctionDecl *FD, Decl *FnD, + CachedTokens &Toks); + void UnmarkAsLateParsedTemplate(FunctionDecl *FD); + bool IsInsideALocalClassWithinATemplateFunction(); - bool checkObjCBridgeRelatedComponents(SourceLocation Loc, - QualType DestType, QualType SrcType, - ObjCInterfaceDecl *&RelatedClass, - ObjCMethodDecl *&ClassMethod, - ObjCMethodDecl *&InstanceMethod, - TypedefNameDecl *&TDNDecl, - bool CfToNs, bool Diagnose = true); + /// We've found a use of a templated declaration that would trigger an + /// implicit instantiation. Check that any relevant explicit specializations + /// and partial specializations are visible/reachable, and diagnose if not. + void checkSpecializationVisibility(SourceLocation Loc, NamedDecl *Spec); + void checkSpecializationReachability(SourceLocation Loc, NamedDecl *Spec); - bool CheckObjCBridgeRelatedConversions(SourceLocation Loc, - QualType DestType, QualType SrcType, - Expr *&SrcExpr, bool Diagnose = true); + ///@} - bool CheckConversionToObjCLiteral(QualType DstType, Expr *&SrcExpr, - bool Diagnose = true); + // + // + // ------------------------------------------------------------------------- + // + // - bool checkInitMethod(ObjCMethodDecl *method, QualType receiverTypeIfCall); + /// \name C++ Template Argument Deduction + /// Implementations are in SemaTemplateDeduction.cpp + ///@{ - /// Check whether the given new method is a valid override of the - /// given overridden method, and set any properties that should be inherited. - void CheckObjCMethodOverride(ObjCMethodDecl *NewMethod, - const ObjCMethodDecl *Overridden); +public: + /// When true, access checking violations are treated as SFINAE + /// failures rather than hard errors. + bool AccessCheckingSFINAE; - /// Describes the compatibility of a result type with its method. - enum ResultTypeCompatibilityKind { - RTC_Compatible, - RTC_Incompatible, - RTC_Unknown - }; + /// RAII class used to determine whether SFINAE has + /// trapped any errors that occur during template argument + /// deduction. + class SFINAETrap { + Sema &SemaRef; + unsigned PrevSFINAEErrors; + bool PrevInNonInstantiationSFINAEContext; + bool PrevAccessCheckingSFINAE; + bool PrevLastDiagnosticIgnored; - void CheckObjCMethodDirectOverrides(ObjCMethodDecl *method, - ObjCMethodDecl *overridden); + public: + explicit SFINAETrap(Sema &SemaRef, bool AccessCheckingSFINAE = false) + : SemaRef(SemaRef), PrevSFINAEErrors(SemaRef.NumSFINAEErrors), + PrevInNonInstantiationSFINAEContext( + SemaRef.InNonInstantiationSFINAEContext), + PrevAccessCheckingSFINAE(SemaRef.AccessCheckingSFINAE), + PrevLastDiagnosticIgnored( + SemaRef.getDiagnostics().isLastDiagnosticIgnored()) { + if (!SemaRef.isSFINAEContext()) + SemaRef.InNonInstantiationSFINAEContext = true; + SemaRef.AccessCheckingSFINAE = AccessCheckingSFINAE; + } - void CheckObjCMethodOverrides(ObjCMethodDecl *ObjCMethod, - ObjCInterfaceDecl *CurrentClass, - ResultTypeCompatibilityKind RTC); + ~SFINAETrap() { + SemaRef.NumSFINAEErrors = PrevSFINAEErrors; + SemaRef.InNonInstantiationSFINAEContext = + PrevInNonInstantiationSFINAEContext; + SemaRef.AccessCheckingSFINAE = PrevAccessCheckingSFINAE; + SemaRef.getDiagnostics().setLastDiagnosticIgnored( + PrevLastDiagnosticIgnored); + } - enum PragmaOptionsAlignKind { - POAK_Native, // #pragma options align=native - POAK_Natural, // #pragma options align=natural - POAK_Packed, // #pragma options align=packed - POAK_Power, // #pragma options align=power - POAK_Mac68k, // #pragma options align=mac68k - POAK_Reset // #pragma options align=reset + /// Determine whether any SFINAE errors have been trapped. + bool hasErrorOccurred() const { + return SemaRef.NumSFINAEErrors > PrevSFINAEErrors; + } }; - /// ActOnPragmaClangSection - Called on well formed \#pragma clang section - void ActOnPragmaClangSection(SourceLocation PragmaLoc, - PragmaClangSectionAction Action, - PragmaClangSectionKind SecKind, StringRef SecName); - - /// ActOnPragmaOptionsAlign - Called on well formed \#pragma options align. - void ActOnPragmaOptionsAlign(PragmaOptionsAlignKind Kind, - SourceLocation PragmaLoc); - - /// ActOnPragmaPack - Called on well formed \#pragma pack(...). - void ActOnPragmaPack(SourceLocation PragmaLoc, PragmaMsStackAction Action, - StringRef SlotLabel, Expr *Alignment); + /// RAII class used to indicate that we are performing provisional + /// semantic analysis to determine the validity of a construct, so + /// typo-correction and diagnostics in the immediate context (not within + /// implicitly-instantiated templates) should be suppressed. + class TentativeAnalysisScope { + Sema &SemaRef; + // FIXME: Using a SFINAETrap for this is a hack. + SFINAETrap Trap; + bool PrevDisableTypoCorrection; - enum class PragmaAlignPackDiagnoseKind { - NonDefaultStateAtInclude, - ChangedStateAtExit + public: + explicit TentativeAnalysisScope(Sema &SemaRef) + : SemaRef(SemaRef), Trap(SemaRef, true), + PrevDisableTypoCorrection(SemaRef.DisableTypoCorrection) { + SemaRef.DisableTypoCorrection = true; + } + ~TentativeAnalysisScope() { + SemaRef.DisableTypoCorrection = PrevDisableTypoCorrection; + } }; - void DiagnoseNonDefaultPragmaAlignPack(PragmaAlignPackDiagnoseKind Kind, - SourceLocation IncludeLoc); - void DiagnoseUnterminatedPragmaAlignPack(); + /// For each declaration that involved template argument deduction, the + /// set of diagnostics that were suppressed during that template argument + /// deduction. + /// + /// FIXME: Serialize this structure to the AST file. + typedef llvm::DenseMap> + SuppressedDiagnosticsMap; + SuppressedDiagnosticsMap SuppressedDiagnostics; - /// ActOnPragmaMSStrictGuardStackCheck - Called on well formed \#pragma - /// strict_gs_check. - void ActOnPragmaMSStrictGuardStackCheck(SourceLocation PragmaLocation, - PragmaMsStackAction Action, - bool Value); + bool isSameOrCompatibleFunctionType(QualType Param, QualType Arg); - /// ActOnPragmaMSStruct - Called on well formed \#pragma ms_struct [on|off]. - void ActOnPragmaMSStruct(PragmaMSStructKind Kind); + TemplateArgumentLoc getTrivialTemplateArgumentLoc(const TemplateArgument &Arg, + QualType NTTPType, + SourceLocation Loc); - /// ActOnPragmaMSComment - Called on well formed - /// \#pragma comment(kind, "arg"). - void ActOnPragmaMSComment(SourceLocation CommentLoc, PragmaMSCommentKind Kind, - StringRef Arg); + /// Get a template argument mapping the given template parameter to itself, + /// e.g. for X in \c template, this would return an expression template + /// argument referencing X. + TemplateArgumentLoc getIdentityTemplateArgumentLoc(NamedDecl *Param, + SourceLocation Location); - /// ActOnPragmaMSPointersToMembers - called on well formed \#pragma - /// pointers_to_members(representation method[, general purpose - /// representation]). - void ActOnPragmaMSPointersToMembers( - LangOptions::PragmaMSPointersToMembersKind Kind, - SourceLocation PragmaLoc); + /// Adjust the type \p ArgFunctionType to match the calling convention, + /// noreturn, and optionally the exception specification of \p FunctionType. + /// Deduction often wants to ignore these properties when matching function + /// types. + QualType adjustCCAndNoReturn(QualType ArgFunctionType, QualType FunctionType, + bool AdjustExceptionSpec = false); - /// Called on well formed \#pragma vtordisp(). - void ActOnPragmaMSVtorDisp(PragmaMsStackAction Action, - SourceLocation PragmaLoc, - MSVtorDispMode Value); + TemplateDeductionResult + DeduceTemplateArguments(ClassTemplatePartialSpecializationDecl *Partial, + ArrayRef TemplateArgs, + sema::TemplateDeductionInfo &Info); - enum PragmaSectionKind { - PSK_DataSeg, - PSK_BSSSeg, - PSK_ConstSeg, - PSK_CodeSeg, - }; + TemplateDeductionResult + DeduceTemplateArguments(VarTemplatePartialSpecializationDecl *Partial, + ArrayRef TemplateArgs, + sema::TemplateDeductionInfo &Info); - bool UnifySection(StringRef SectionName, int SectionFlags, - NamedDecl *TheDecl); - bool UnifySection(StringRef SectionName, - int SectionFlags, - SourceLocation PragmaSectionLocation); + TemplateDeductionResult SubstituteExplicitTemplateArguments( + FunctionTemplateDecl *FunctionTemplate, + TemplateArgumentListInfo &ExplicitTemplateArgs, + SmallVectorImpl &Deduced, + SmallVectorImpl &ParamTypes, QualType *FunctionType, + sema::TemplateDeductionInfo &Info); - /// Called on well formed \#pragma bss_seg/data_seg/const_seg/code_seg. - void ActOnPragmaMSSeg(SourceLocation PragmaLocation, - PragmaMsStackAction Action, - llvm::StringRef StackSlotLabel, - StringLiteral *SegmentName, - llvm::StringRef PragmaName); + /// brief A function argument from which we performed template argument + // deduction for a call. + struct OriginalCallArg { + OriginalCallArg(QualType OriginalParamType, bool DecomposedParam, + unsigned ArgIdx, QualType OriginalArgType) + : OriginalParamType(OriginalParamType), + DecomposedParam(DecomposedParam), ArgIdx(ArgIdx), + OriginalArgType(OriginalArgType) {} - /// Called on well formed \#pragma section(). - void ActOnPragmaMSSection(SourceLocation PragmaLocation, - int SectionFlags, StringLiteral *SegmentName); + QualType OriginalParamType; + bool DecomposedParam; + unsigned ArgIdx; + QualType OriginalArgType; + }; - /// Called on well-formed \#pragma init_seg(). - void ActOnPragmaMSInitSeg(SourceLocation PragmaLocation, - StringLiteral *SegmentName); + TemplateDeductionResult FinishTemplateArgumentDeduction( + FunctionTemplateDecl *FunctionTemplate, + SmallVectorImpl &Deduced, + unsigned NumExplicitlySpecified, FunctionDecl *&Specialization, + sema::TemplateDeductionInfo &Info, + SmallVectorImpl const *OriginalCallArgs = nullptr, + bool PartialOverloading = false, + llvm::function_ref CheckNonDependent = [] { return false; }); - /// Called on well-formed \#pragma alloc_text(). - void ActOnPragmaMSAllocText( - SourceLocation PragmaLocation, StringRef Section, - const SmallVector> - &Functions); + TemplateDeductionResult DeduceTemplateArguments( + FunctionTemplateDecl *FunctionTemplate, + TemplateArgumentListInfo *ExplicitTemplateArgs, ArrayRef Args, + FunctionDecl *&Specialization, sema::TemplateDeductionInfo &Info, + bool PartialOverloading, bool AggregateDeductionCandidate, + QualType ObjectType, Expr::Classification ObjectClassification, + llvm::function_ref)> CheckNonDependent); - /// Called on #pragma clang __debug dump II - void ActOnPragmaDump(Scope *S, SourceLocation Loc, IdentifierInfo *II); + TemplateDeductionResult DeduceTemplateArguments( + FunctionTemplateDecl *FunctionTemplate, + TemplateArgumentListInfo *ExplicitTemplateArgs, QualType ArgFunctionType, + FunctionDecl *&Specialization, sema::TemplateDeductionInfo &Info, + bool IsAddressOfFunction = false); - /// Called on #pragma clang __debug dump E - void ActOnPragmaDump(Expr *E); + TemplateDeductionResult DeduceTemplateArguments( + FunctionTemplateDecl *FunctionTemplate, QualType ObjectType, + Expr::Classification ObjectClassification, QualType ToType, + CXXConversionDecl *&Specialization, sema::TemplateDeductionInfo &Info); - /// ActOnPragmaDetectMismatch - Call on well-formed \#pragma detect_mismatch - void ActOnPragmaDetectMismatch(SourceLocation Loc, StringRef Name, - StringRef Value); + TemplateDeductionResult + DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate, + TemplateArgumentListInfo *ExplicitTemplateArgs, + FunctionDecl *&Specialization, + sema::TemplateDeductionInfo &Info, + bool IsAddressOfFunction = false); - /// Are precise floating point semantics currently enabled? - bool isPreciseFPEnabled() { - return !CurFPFeatures.getAllowFPReassociate() && - !CurFPFeatures.getNoSignedZero() && - !CurFPFeatures.getAllowReciprocal() && - !CurFPFeatures.getAllowApproxFunc(); - } + /// Substitute Replacement for \p auto in \p TypeWithAuto + QualType SubstAutoType(QualType TypeWithAuto, QualType Replacement); + /// Substitute Replacement for auto in TypeWithAuto + TypeSourceInfo *SubstAutoTypeSourceInfo(TypeSourceInfo *TypeWithAuto, + QualType Replacement); - void ActOnPragmaFPEvalMethod(SourceLocation Loc, - LangOptions::FPEvalMethodKind Value); + // Substitute auto in TypeWithAuto for a Dependent auto type + QualType SubstAutoTypeDependent(QualType TypeWithAuto); - /// ActOnPragmaFloatControl - Call on well-formed \#pragma float_control - void ActOnPragmaFloatControl(SourceLocation Loc, PragmaMsStackAction Action, - PragmaFloatControlKind Value); + // Substitute auto in TypeWithAuto for a Dependent auto type + TypeSourceInfo * + SubstAutoTypeSourceInfoDependent(TypeSourceInfo *TypeWithAuto); - /// ActOnPragmaUnused - Called on well-formed '\#pragma unused'. - void ActOnPragmaUnused(const Token &Identifier, - Scope *curScope, - SourceLocation PragmaLoc); + /// Completely replace the \c auto in \p TypeWithAuto by + /// \p Replacement. This does not retain any \c auto type sugar. + QualType ReplaceAutoType(QualType TypeWithAuto, QualType Replacement); + TypeSourceInfo *ReplaceAutoTypeSourceInfo(TypeSourceInfo *TypeWithAuto, + QualType Replacement); - /// ActOnPragmaVisibility - Called on well formed \#pragma GCC visibility... . - void ActOnPragmaVisibility(const IdentifierInfo* VisType, - SourceLocation PragmaLoc); + TemplateDeductionResult + DeduceAutoType(TypeLoc AutoTypeLoc, Expr *Initializer, QualType &Result, + sema::TemplateDeductionInfo &Info, + bool DependentDeduction = false, + bool IgnoreConstraints = false, + TemplateSpecCandidateSet *FailedTSC = nullptr); + void DiagnoseAutoDeductionFailure(VarDecl *VDecl, Expr *Init); + bool DeduceReturnType(FunctionDecl *FD, SourceLocation Loc, + bool Diagnose = true); - NamedDecl *DeclClonePragmaWeak(NamedDecl *ND, const IdentifierInfo *II, - SourceLocation Loc); - void DeclApplyPragmaWeak(Scope *S, NamedDecl *ND, const WeakInfo &W); + bool CheckIfFunctionSpecializationIsImmediate(FunctionDecl *FD, + SourceLocation Loc); - /// ActOnPragmaWeakID - Called on well formed \#pragma weak ident. - void ActOnPragmaWeakID(IdentifierInfo* WeakName, - SourceLocation PragmaLoc, - SourceLocation WeakNameLoc); + ClassTemplatePartialSpecializationDecl * + getMoreSpecializedPartialSpecialization( + ClassTemplatePartialSpecializationDecl *PS1, + ClassTemplatePartialSpecializationDecl *PS2, SourceLocation Loc); - /// ActOnPragmaRedefineExtname - Called on well formed - /// \#pragma redefine_extname oldname newname. - void ActOnPragmaRedefineExtname(IdentifierInfo* WeakName, - IdentifierInfo* AliasName, - SourceLocation PragmaLoc, - SourceLocation WeakNameLoc, - SourceLocation AliasNameLoc); + bool isMoreSpecializedThanPrimary(ClassTemplatePartialSpecializationDecl *T, + sema::TemplateDeductionInfo &Info); - /// ActOnPragmaWeakAlias - Called on well formed \#pragma weak ident = ident. - void ActOnPragmaWeakAlias(IdentifierInfo* WeakName, - IdentifierInfo* AliasName, - SourceLocation PragmaLoc, - SourceLocation WeakNameLoc, - SourceLocation AliasNameLoc); + VarTemplatePartialSpecializationDecl *getMoreSpecializedPartialSpecialization( + VarTemplatePartialSpecializationDecl *PS1, + VarTemplatePartialSpecializationDecl *PS2, SourceLocation Loc); - /// ActOnPragmaFPContract - Called on well formed - /// \#pragma {STDC,OPENCL} FP_CONTRACT and - /// \#pragma clang fp contract - void ActOnPragmaFPContract(SourceLocation Loc, LangOptions::FPModeKind FPC); + bool isMoreSpecializedThanPrimary(VarTemplatePartialSpecializationDecl *T, + sema::TemplateDeductionInfo &Info); - /// Called on well formed - /// \#pragma clang fp reassociate - /// or - /// \#pragma clang fp reciprocal - void ActOnPragmaFPValueChangingOption(SourceLocation Loc, PragmaFPKind Kind, - bool IsEnabled); + bool isTemplateTemplateParameterAtLeastAsSpecializedAs( + TemplateParameterList *PParam, TemplateDecl *AArg, SourceLocation Loc); - /// ActOnPragmaFenvAccess - Called on well formed - /// \#pragma STDC FENV_ACCESS - void ActOnPragmaFEnvAccess(SourceLocation Loc, bool IsEnabled); + void MarkUsedTemplateParameters(const Expr *E, bool OnlyDeduced, + unsigned Depth, llvm::SmallBitVector &Used); - /// ActOnPragmaCXLimitedRange - Called on well formed - /// \#pragma STDC CX_LIMITED_RANGE - void ActOnPragmaCXLimitedRange(SourceLocation Loc, - LangOptions::ComplexRangeKind Range); + void MarkUsedTemplateParameters(const TemplateArgumentList &TemplateArgs, + bool OnlyDeduced, unsigned Depth, + llvm::SmallBitVector &Used); + void + MarkDeducedTemplateParameters(const FunctionTemplateDecl *FunctionTemplate, + llvm::SmallBitVector &Deduced) { + return MarkDeducedTemplateParameters(Context, FunctionTemplate, Deduced); + } + static void + MarkDeducedTemplateParameters(ASTContext &Ctx, + const FunctionTemplateDecl *FunctionTemplate, + llvm::SmallBitVector &Deduced); - /// Called on well formed '\#pragma clang fp' that has option 'exceptions'. - void ActOnPragmaFPExceptions(SourceLocation Loc, - LangOptions::FPExceptionModeKind); + FunctionTemplateDecl *getMoreSpecializedTemplate( + FunctionTemplateDecl *FT1, FunctionTemplateDecl *FT2, SourceLocation Loc, + TemplatePartialOrderingContext TPOC, unsigned NumCallArguments1, + unsigned NumCallArguments2, bool Reversed = false); + UnresolvedSetIterator + getMostSpecialized(UnresolvedSetIterator SBegin, UnresolvedSetIterator SEnd, + TemplateSpecCandidateSet &FailedCandidates, + SourceLocation Loc, const PartialDiagnostic &NoneDiag, + const PartialDiagnostic &AmbigDiag, + const PartialDiagnostic &CandidateDiag, + bool Complain = true, QualType TargetType = QualType()); - /// Called to set constant rounding mode for floating point operations. - void ActOnPragmaFEnvRound(SourceLocation Loc, llvm::RoundingMode); + ///@} - /// Called to set exception behavior for floating point operations. - void setExceptionMode(SourceLocation Loc, LangOptions::FPExceptionModeKind); + // + // + // ------------------------------------------------------------------------- + // + // - /// AddAlignmentAttributesForRecord - Adds any needed alignment attributes to - /// a the record decl, to handle '\#pragma pack' and '\#pragma options align'. - void AddAlignmentAttributesForRecord(RecordDecl *RD); + /// \name C++ Template Instantiation + /// Implementations are in SemaTemplateInstantiate.cpp + ///@{ - /// AddMsStructLayoutForRecord - Adds ms_struct layout attribute to record. - void AddMsStructLayoutForRecord(RecordDecl *RD); +public: + /// A helper class for building up ExtParameterInfos. + class ExtParameterInfoBuilder { + SmallVector Infos; + bool HasInteresting = false; - /// PushNamespaceVisibilityAttr - Note that we've entered a - /// namespace with a visibility attribute. - void PushNamespaceVisibilityAttr(const VisibilityAttr *Attr, - SourceLocation Loc); + public: + /// Set the ExtParameterInfo for the parameter at the given index, + /// + void set(unsigned index, FunctionProtoType::ExtParameterInfo info) { + assert(Infos.size() <= index); + Infos.resize(index); + Infos.push_back(info); - /// AddPushedVisibilityAttribute - If '\#pragma GCC visibility' was used, - /// add an appropriate visibility attribute. - void AddPushedVisibilityAttribute(Decl *RD); + if (!HasInteresting) + HasInteresting = (info != FunctionProtoType::ExtParameterInfo()); + } - /// PopPragmaVisibility - Pop the top element of the visibility stack; used - /// for '\#pragma GCC visibility' and visibility attributes on namespaces. - void PopPragmaVisibility(bool IsNamespaceEnd, SourceLocation EndLoc); + /// Return a pointer (suitable for setting in an ExtProtoInfo) to the + /// ExtParameterInfo array we've built up. + const FunctionProtoType::ExtParameterInfo * + getPointerOrNull(unsigned numParams) { + if (!HasInteresting) + return nullptr; + Infos.resize(numParams); + return Infos.data(); + } + }; - /// FreeVisContext - Deallocate and null out VisContext. - void FreeVisContext(); + /// The current instantiation scope used to store local + /// variables. + LocalInstantiationScope *CurrentInstantiationScope; - /// AddCFAuditedAttribute - Check whether we're currently within - /// '\#pragma clang arc_cf_code_audited' and, if so, consider adding - /// the appropriate attribute. - void AddCFAuditedAttribute(Decl *D); + typedef llvm::DenseMap> + UnparsedDefaultArgInstantiationsMap; - void ActOnPragmaAttributeAttribute(ParsedAttr &Attribute, - SourceLocation PragmaLoc, - attr::ParsedSubjectMatchRuleSet Rules); - void ActOnPragmaAttributeEmptyPush(SourceLocation PragmaLoc, - const IdentifierInfo *Namespace); + /// A mapping from parameters with unparsed default arguments to the + /// set of instantiations of each parameter. + /// + /// This mapping is a temporary data structure used when parsing + /// nested class templates or nested classes of class templates, + /// where we might end up instantiating an inner class before the + /// default arguments of its methods have been parsed. + UnparsedDefaultArgInstantiationsMap UnparsedDefaultArgInstantiations; - /// Called on well-formed '\#pragma clang attribute pop'. - void ActOnPragmaAttributePop(SourceLocation PragmaLoc, - const IdentifierInfo *Namespace); + /// A context in which code is being synthesized (where a source location + /// alone is not sufficient to identify the context). This covers template + /// instantiation and various forms of implicitly-generated functions. + struct CodeSynthesisContext { + /// The kind of template instantiation we are performing + enum SynthesisKind { + /// We are instantiating a template declaration. The entity is + /// the declaration we're instantiating (e.g., a CXXRecordDecl). + TemplateInstantiation, - /// Adds the attributes that have been specified using the - /// '\#pragma clang attribute push' directives to the given declaration. - void AddPragmaAttributes(Scope *S, Decl *D); + /// We are instantiating a default argument for a template + /// parameter. The Entity is the template parameter whose argument is + /// being instantiated, the Template is the template, and the + /// TemplateArgs/NumTemplateArguments provide the template arguments as + /// specified. + DefaultTemplateArgumentInstantiation, - void DiagnoseUnterminatedPragmaAttribute(); + /// We are instantiating a default argument for a function. + /// The Entity is the ParmVarDecl, and TemplateArgs/NumTemplateArgs + /// provides the template arguments as specified. + DefaultFunctionArgumentInstantiation, - /// Called on well formed \#pragma clang optimize. - void ActOnPragmaOptimize(bool On, SourceLocation PragmaLoc); + /// We are substituting explicit template arguments provided for + /// a function template. The entity is a FunctionTemplateDecl. + ExplicitTemplateArgumentSubstitution, - /// #pragma optimize("[optimization-list]", on | off). - void ActOnPragmaMSOptimize(SourceLocation Loc, bool IsOn); + /// We are substituting template argument determined as part of + /// template argument deduction for either a class template + /// partial specialization or a function template. The + /// Entity is either a {Class|Var}TemplatePartialSpecializationDecl or + /// a TemplateDecl. + DeducedTemplateArgumentSubstitution, - /// Call on well formed \#pragma function. - void - ActOnPragmaMSFunction(SourceLocation Loc, - const llvm::SmallVectorImpl &NoBuiltins); + /// We are substituting into a lambda expression. + LambdaExpressionSubstitution, - /// Get the location for the currently active "\#pragma clang optimize - /// off". If this location is invalid, then the state of the pragma is "on". - SourceLocation getOptimizeOffPragmaLocation() const { - return OptimizeOffPragmaLocation; - } + /// We are substituting prior template arguments into a new + /// template parameter. The template parameter itself is either a + /// NonTypeTemplateParmDecl or a TemplateTemplateParmDecl. + PriorTemplateArgumentSubstitution, - /// Only called on function definitions; if there is a pragma in scope - /// with the effect of a range-based optnone, consider marking the function - /// with attribute optnone. - void AddRangeBasedOptnone(FunctionDecl *FD); + /// We are checking the validity of a default template argument that + /// has been used when naming a template-id. + DefaultTemplateArgumentChecking, - /// Only called on function definitions; if there is a `#pragma alloc_text` - /// that decides which code section the function should be in, add - /// attribute section to the function. - void AddSectionMSAllocText(FunctionDecl *FD); + /// We are computing the exception specification for a defaulted special + /// member function. + ExceptionSpecEvaluation, - /// Adds the 'optnone' attribute to the function declaration if there - /// are no conflicts; Loc represents the location causing the 'optnone' - /// attribute to be added (usually because of a pragma). - void AddOptnoneAttributeIfNoConflicts(FunctionDecl *FD, SourceLocation Loc); + /// We are instantiating the exception specification for a function + /// template which was deferred until it was needed. + ExceptionSpecInstantiation, - /// Only called on function definitions; if there is a MSVC #pragma optimize - /// in scope, consider changing the function's attributes based on the - /// optimization list passed to the pragma. - void ModifyFnAttributesMSPragmaOptimize(FunctionDecl *FD); + /// We are instantiating a requirement of a requires expression. + RequirementInstantiation, - /// Only called on function definitions; if there is a pragma in scope - /// with the effect of a range-based no_builtin, consider marking the function - /// with attribute no_builtin. - void AddImplicitMSFunctionNoBuiltinAttr(FunctionDecl *FD); + /// We are checking the satisfaction of a nested requirement of a requires + /// expression. + NestedRequirementConstraintsCheck, - /// AddAlignedAttr - Adds an aligned attribute to a particular declaration. - void AddAlignedAttr(Decl *D, const AttributeCommonInfo &CI, Expr *E, - bool IsPackExpansion); - void AddAlignedAttr(Decl *D, const AttributeCommonInfo &CI, TypeSourceInfo *T, - bool IsPackExpansion); + /// We are declaring an implicit special member function. + DeclaringSpecialMember, - /// AddAssumeAlignedAttr - Adds an assume_aligned attribute to a particular - /// declaration. - void AddAssumeAlignedAttr(Decl *D, const AttributeCommonInfo &CI, Expr *E, - Expr *OE); + /// We are declaring an implicit 'operator==' for a defaulted + /// 'operator<=>'. + DeclaringImplicitEqualityComparison, - /// AddAllocAlignAttr - Adds an alloc_align attribute to a particular - /// declaration. - void AddAllocAlignAttr(Decl *D, const AttributeCommonInfo &CI, - Expr *ParamExpr); + /// We are defining a synthesized function (such as a defaulted special + /// member). + DefiningSynthesizedFunction, - /// AddAlignValueAttr - Adds an align_value attribute to a particular - /// declaration. - void AddAlignValueAttr(Decl *D, const AttributeCommonInfo &CI, Expr *E); + // We are checking the constraints associated with a constrained entity or + // the constraint expression of a concept. This includes the checks that + // atomic constraints have the type 'bool' and that they can be constant + // evaluated. + ConstraintsCheck, - /// AddAnnotationAttr - Adds an annotation Annot with Args arguments to D. - void AddAnnotationAttr(Decl *D, const AttributeCommonInfo &CI, - StringRef Annot, MutableArrayRef Args); + // We are substituting template arguments into a constraint expression. + ConstraintSubstitution, - /// ConstantFoldAttrArgs - Folds attribute arguments into ConstantExprs - /// (unless they are value dependent or type dependent). Returns false - /// and emits a diagnostic if one or more of the arguments could not be - /// folded into a constant. - bool ConstantFoldAttrArgs(const AttributeCommonInfo &CI, - MutableArrayRef Args); + // We are normalizing a constraint expression. + ConstraintNormalization, - /// Create an CUDALaunchBoundsAttr attribute. - CUDALaunchBoundsAttr *CreateLaunchBoundsAttr(const AttributeCommonInfo &CI, - Expr *MaxThreads, - Expr *MinBlocks, - Expr *MaxBlocks); + // Instantiating a Requires Expression parameter clause. + RequirementParameterInstantiation, - /// AddLaunchBoundsAttr - Adds a launch_bounds attribute to a particular - /// declaration. - void AddLaunchBoundsAttr(Decl *D, const AttributeCommonInfo &CI, - Expr *MaxThreads, Expr *MinBlocks, Expr *MaxBlocks); + // We are substituting into the parameter mapping of an atomic constraint + // during normalization. + ParameterMappingSubstitution, - /// AddModeAttr - Adds a mode attribute to a particular declaration. - void AddModeAttr(Decl *D, const AttributeCommonInfo &CI, IdentifierInfo *Name, - bool InInstantiation = false); + /// We are rewriting a comparison operator in terms of an operator<=>. + RewritingOperatorAsSpaceship, - void AddParameterABIAttr(Decl *D, const AttributeCommonInfo &CI, - ParameterABI ABI); + /// We are initializing a structured binding. + InitializingStructuredBinding, - enum class RetainOwnershipKind {NS, CF, OS}; - void AddXConsumedAttr(Decl *D, const AttributeCommonInfo &CI, - RetainOwnershipKind K, bool IsTemplateInstantiation); + /// We are marking a class as __dllexport. + MarkingClassDllexported, - /// Create an AMDGPUWavesPerEUAttr attribute. - AMDGPUFlatWorkGroupSizeAttr * - CreateAMDGPUFlatWorkGroupSizeAttr(const AttributeCommonInfo &CI, Expr *Min, - Expr *Max); + /// We are building an implied call from __builtin_dump_struct. The + /// arguments are in CallArgs. + BuildingBuiltinDumpStructCall, - /// addAMDGPUFlatWorkGroupSizeAttr - Adds an amdgpu_flat_work_group_size - /// attribute to a particular declaration. - void addAMDGPUFlatWorkGroupSizeAttr(Decl *D, const AttributeCommonInfo &CI, - Expr *Min, Expr *Max); + /// Added for Template instantiation observation. + /// Memoization means we are _not_ instantiating a template because + /// it is already instantiated (but we entered a context where we + /// would have had to if it was not already instantiated). + Memoization, - /// Create an AMDGPUWavesPerEUAttr attribute. - AMDGPUWavesPerEUAttr * - CreateAMDGPUWavesPerEUAttr(const AttributeCommonInfo &CI, Expr *Min, - Expr *Max); + /// We are building deduction guides for a class. + BuildingDeductionGuides, + } Kind; - /// addAMDGPUWavePersEUAttr - Adds an amdgpu_waves_per_eu attribute to a - /// particular declaration. - void addAMDGPUWavesPerEUAttr(Decl *D, const AttributeCommonInfo &CI, - Expr *Min, Expr *Max); + /// Was the enclosing context a non-instantiation SFINAE context? + bool SavedInNonInstantiationSFINAEContext; - bool checkNSReturnsRetainedReturnType(SourceLocation loc, QualType type); + /// The point of instantiation or synthesis within the source code. + SourceLocation PointOfInstantiation; - //===--------------------------------------------------------------------===// - // C++ Coroutines - // - bool ActOnCoroutineBodyStart(Scope *S, SourceLocation KwLoc, - StringRef Keyword); - ExprResult ActOnCoawaitExpr(Scope *S, SourceLocation KwLoc, Expr *E); - ExprResult ActOnCoyieldExpr(Scope *S, SourceLocation KwLoc, Expr *E); - StmtResult ActOnCoreturnStmt(Scope *S, SourceLocation KwLoc, Expr *E); + /// The entity that is being synthesized. + Decl *Entity; - ExprResult BuildOperatorCoawaitLookupExpr(Scope *S, SourceLocation Loc); - ExprResult BuildOperatorCoawaitCall(SourceLocation Loc, Expr *E, - UnresolvedLookupExpr *Lookup); - ExprResult BuildResolvedCoawaitExpr(SourceLocation KwLoc, Expr *Operand, - Expr *Awaiter, bool IsImplicit = false); - ExprResult BuildUnresolvedCoawaitExpr(SourceLocation KwLoc, Expr *Operand, - UnresolvedLookupExpr *Lookup); - ExprResult BuildCoyieldExpr(SourceLocation KwLoc, Expr *E); - StmtResult BuildCoreturnStmt(SourceLocation KwLoc, Expr *E, - bool IsImplicit = false); - StmtResult BuildCoroutineBodyStmt(CoroutineBodyStmt::CtorArgs); - bool buildCoroutineParameterMoves(SourceLocation Loc); - VarDecl *buildCoroutinePromise(SourceLocation Loc); - void CheckCompletedCoroutineBody(FunctionDecl *FD, Stmt *&Body); + /// The template (or partial specialization) in which we are + /// performing the instantiation, for substitutions of prior template + /// arguments. + NamedDecl *Template; - // Heuristically tells if the function is `get_return_object` member of a - // coroutine promise_type by matching the function name. - static bool CanBeGetReturnObject(const FunctionDecl *FD); - static bool CanBeGetReturnTypeOnAllocFailure(const FunctionDecl *FD); + union { + /// The list of template arguments we are substituting, if they + /// are not part of the entity. + const TemplateArgument *TemplateArgs; - // As a clang extension, enforces that a non-coroutine function must be marked - // with [[clang::coro_wrapper]] if it returns a type marked with - // [[clang::coro_return_type]]. - // Expects that FD is not a coroutine. - void CheckCoroutineWrapper(FunctionDecl *FD); - /// Lookup 'coroutine_traits' in std namespace and std::experimental - /// namespace. The namespace found is recorded in Namespace. - ClassTemplateDecl *lookupCoroutineTraits(SourceLocation KwLoc, - SourceLocation FuncLoc); - /// Check that the expression co_await promise.final_suspend() shall not be - /// potentially-throwing. - bool checkFinalSuspendNoThrow(const Stmt *FinalSuspend); + /// The list of argument expressions in a synthesized call. + const Expr *const *CallArgs; + }; - //===--------------------------------------------------------------------===// - // OpenMP directives and clauses. - // -private: - void *VarDataSharingAttributesStack; + // FIXME: Wrap this union around more members, or perhaps store the + // kind-specific members in the RAII object owning the context. + union { + /// The number of template arguments in TemplateArgs. + unsigned NumTemplateArgs; - struct DeclareTargetContextInfo { - struct MapInfo { - OMPDeclareTargetDeclAttr::MapTypeTy MT; - SourceLocation Loc; + /// The number of expressions in CallArgs. + unsigned NumCallArgs; + + /// The special member being declared or defined. + CXXSpecialMember SpecialMember; }; - /// Explicitly listed variables and functions in a 'to' or 'link' clause. - llvm::DenseMap ExplicitlyMapped; - /// The 'device_type' as parsed from the clause. - OMPDeclareTargetDeclAttr::DevTypeTy DT = OMPDeclareTargetDeclAttr::DT_Any; + ArrayRef template_arguments() const { + assert(Kind != DeclaringSpecialMember); + return {TemplateArgs, NumTemplateArgs}; + } - /// The directive kind, `begin declare target` or `declare target`. - OpenMPDirectiveKind Kind; + /// The template deduction info object associated with the + /// substitution or checking of explicit or deduced template arguments. + sema::TemplateDeductionInfo *DeductionInfo; - /// The directive with indirect clause. - std::optional Indirect; + /// The source range that covers the construct that cause + /// the instantiation, e.g., the template-id that causes a class + /// template instantiation. + SourceRange InstantiationRange; - /// The directive location. - SourceLocation Loc; + CodeSynthesisContext() + : Kind(TemplateInstantiation), + SavedInNonInstantiationSFINAEContext(false), Entity(nullptr), + Template(nullptr), TemplateArgs(nullptr), NumTemplateArgs(0), + DeductionInfo(nullptr) {} - DeclareTargetContextInfo(OpenMPDirectiveKind Kind, SourceLocation Loc) - : Kind(Kind), Loc(Loc) {} + /// Determines whether this template is an actual instantiation + /// that should be counted toward the maximum instantiation depth. + bool isInstantiationRecord() const; }; - /// Number of nested '#pragma omp declare target' directives. - SmallVector DeclareTargetNesting; - - /// Initialization of data-sharing attributes stack. - void InitDataSharingAttributesStack(); - void DestroyDataSharingAttributesStack(); + /// A stack object to be created when performing template + /// instantiation. + /// + /// Construction of an object of type \c InstantiatingTemplate + /// pushes the current instantiation onto the stack of active + /// instantiations. If the size of this stack exceeds the maximum + /// number of recursive template instantiations, construction + /// produces an error and evaluates true. + /// + /// Destruction of this object will pop the named instantiation off + /// the stack. + struct InstantiatingTemplate { + /// Note that we are instantiating a class template, + /// function template, variable template, alias template, + /// or a member thereof. + InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, + Decl *Entity, + SourceRange InstantiationRange = SourceRange()); - /// Returns OpenMP nesting level for current directive. - unsigned getOpenMPNestingLevel() const; + struct ExceptionSpecification {}; + /// Note that we are instantiating an exception specification + /// of a function template. + InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, + FunctionDecl *Entity, ExceptionSpecification, + SourceRange InstantiationRange = SourceRange()); - /// Adjusts the function scopes index for the target-based regions. - void adjustOpenMPTargetScopeIndex(unsigned &FunctionScopesIndex, - unsigned Level) const; + /// Note that we are instantiating a default argument in a + /// template-id. + InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, + TemplateParameter Param, TemplateDecl *Template, + ArrayRef TemplateArgs, + SourceRange InstantiationRange = SourceRange()); - /// Returns the number of scopes associated with the construct on the given - /// OpenMP level. - int getNumberOfConstructScopes(unsigned Level) const; + /// Note that we are substituting either explicitly-specified or + /// deduced template arguments during function template argument deduction. + InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, + FunctionTemplateDecl *FunctionTemplate, + ArrayRef TemplateArgs, + CodeSynthesisContext::SynthesisKind Kind, + sema::TemplateDeductionInfo &DeductionInfo, + SourceRange InstantiationRange = SourceRange()); - /// Push new OpenMP function region for non-capturing function. - void pushOpenMPFunctionRegion(); + /// Note that we are instantiating as part of template + /// argument deduction for a class template declaration. + InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, + TemplateDecl *Template, + ArrayRef TemplateArgs, + sema::TemplateDeductionInfo &DeductionInfo, + SourceRange InstantiationRange = SourceRange()); - /// Pop OpenMP function region for non-capturing function. - void popOpenMPFunctionRegion(const sema::FunctionScopeInfo *OldFSI); + /// Note that we are instantiating as part of template + /// argument deduction for a class template partial + /// specialization. + InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, + ClassTemplatePartialSpecializationDecl *PartialSpec, + ArrayRef TemplateArgs, + sema::TemplateDeductionInfo &DeductionInfo, + SourceRange InstantiationRange = SourceRange()); - /// Analyzes and checks a loop nest for use by a loop transformation. - /// - /// \param Kind The loop transformation directive kind. - /// \param NumLoops How many nested loops the directive is expecting. - /// \param AStmt Associated statement of the transformation directive. - /// \param LoopHelpers [out] The loop analysis result. - /// \param Body [out] The body code nested in \p NumLoops loop. - /// \param OriginalInits [out] Collection of statements and declarations that - /// must have been executed/declared before entering the - /// loop. - /// - /// \return Whether there was any error. - bool checkTransformableLoopNest( - OpenMPDirectiveKind Kind, Stmt *AStmt, int NumLoops, - SmallVectorImpl &LoopHelpers, - Stmt *&Body, - SmallVectorImpl, 0>> - &OriginalInits); + /// Note that we are instantiating as part of template + /// argument deduction for a variable template partial + /// specialization. + InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, + VarTemplatePartialSpecializationDecl *PartialSpec, + ArrayRef TemplateArgs, + sema::TemplateDeductionInfo &DeductionInfo, + SourceRange InstantiationRange = SourceRange()); - /// Helper to keep information about the current `omp begin/end declare - /// variant` nesting. - struct OMPDeclareVariantScope { - /// The associated OpenMP context selector. - OMPTraitInfo *TI; + /// Note that we are instantiating a default argument for a function + /// parameter. + InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, + ParmVarDecl *Param, + ArrayRef TemplateArgs, + SourceRange InstantiationRange = SourceRange()); - /// The associated OpenMP context selector mangling. - std::string NameSuffix; + /// Note that we are substituting prior template arguments into a + /// non-type parameter. + InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, + NamedDecl *Template, NonTypeTemplateParmDecl *Param, + ArrayRef TemplateArgs, + SourceRange InstantiationRange); - OMPDeclareVariantScope(OMPTraitInfo &TI); - }; + /// Note that we are substituting prior template arguments into a + /// template template parameter. + InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, + NamedDecl *Template, TemplateTemplateParmDecl *Param, + ArrayRef TemplateArgs, + SourceRange InstantiationRange); - /// Return the OMPTraitInfo for the surrounding scope, if any. - OMPTraitInfo *getOMPTraitInfoForSurroundingScope() { - return OMPDeclareVariantScopes.empty() ? nullptr - : OMPDeclareVariantScopes.back().TI; - } + /// Note that we are checking the default template argument + /// against the template parameter for a given template-id. + InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, + TemplateDecl *Template, NamedDecl *Param, + ArrayRef TemplateArgs, + SourceRange InstantiationRange); - /// The current `omp begin/end declare variant` scopes. - SmallVector OMPDeclareVariantScopes; + struct ConstraintsCheck {}; + /// \brief Note that we are checking the constraints associated with some + /// constrained entity (a concept declaration or a template with associated + /// constraints). + InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, + ConstraintsCheck, NamedDecl *Template, + ArrayRef TemplateArgs, + SourceRange InstantiationRange); - /// The current `omp begin/end assumes` scopes. - SmallVector OMPAssumeScoped; + struct ConstraintSubstitution {}; + /// \brief Note that we are checking a constraint expression associated + /// with a template declaration or as part of the satisfaction check of a + /// concept. + InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, + ConstraintSubstitution, NamedDecl *Template, + sema::TemplateDeductionInfo &DeductionInfo, + SourceRange InstantiationRange); - /// All `omp assumes` we encountered so far. - SmallVector OMPAssumeGlobal; + struct ConstraintNormalization {}; + /// \brief Note that we are normalizing a constraint expression. + InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, + ConstraintNormalization, NamedDecl *Template, + SourceRange InstantiationRange); - /// OMPD_loop is mapped to OMPD_for, OMPD_distribute or OMPD_simd depending - /// on the parameter of the bind clause. In the methods for the - /// mapped directives, check the parameters of the lastprivate clause. - bool checkLastPrivateForMappedDirectives(ArrayRef Clauses); - /// Depending on the bind clause of OMPD_loop map the directive to new - /// directives. - /// 1) loop bind(parallel) --> OMPD_for - /// 2) loop bind(teams) --> OMPD_distribute - /// 3) loop bind(thread) --> OMPD_simd - /// This is being handled in Sema instead of Codegen because of the need for - /// rigorous semantic checking in the new mapped directives. - bool mapLoopConstruct(llvm::SmallVector &ClausesWithoutBind, - ArrayRef Clauses, - OpenMPBindClauseKind &BindKind, - OpenMPDirectiveKind &Kind, - OpenMPDirectiveKind &PrevMappedDirective, - SourceLocation StartLoc, SourceLocation EndLoc, - const DeclarationNameInfo &DirName, - OpenMPDirectiveKind CancelRegion); + struct ParameterMappingSubstitution {}; + /// \brief Note that we are subtituting into the parameter mapping of an + /// atomic constraint during constraint normalization. + InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, + ParameterMappingSubstitution, NamedDecl *Template, + SourceRange InstantiationRange); -public: - /// The declarator \p D defines a function in the scope \p S which is nested - /// in an `omp begin/end declare variant` scope. In this method we create a - /// declaration for \p D and rename \p D according to the OpenMP context - /// selector of the surrounding scope. Return all base functions in \p Bases. - void ActOnStartOfFunctionDefinitionInOpenMPDeclareVariantScope( - Scope *S, Declarator &D, MultiTemplateParamsArg TemplateParameterLists, - SmallVectorImpl &Bases); + /// \brief Note that we are substituting template arguments into a part of + /// a requirement of a requires expression. + InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, + concepts::Requirement *Req, + sema::TemplateDeductionInfo &DeductionInfo, + SourceRange InstantiationRange = SourceRange()); - /// Register \p D as specialization of all base functions in \p Bases in the - /// current `omp begin/end declare variant` scope. - void ActOnFinishedFunctionDefinitionInOpenMPDeclareVariantScope( - Decl *D, SmallVectorImpl &Bases); + /// \brief Note that we are checking the satisfaction of the constraint + /// expression inside of a nested requirement. + InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, + concepts::NestedRequirement *Req, ConstraintsCheck, + SourceRange InstantiationRange = SourceRange()); - /// Act on \p D, a function definition inside of an `omp [begin/end] assumes`. - void ActOnFinishedFunctionDefinitionInOpenMPAssumeScope(Decl *D); + /// \brief Note that we are checking a requires clause. + InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, + const RequiresExpr *E, + sema::TemplateDeductionInfo &DeductionInfo, + SourceRange InstantiationRange); - /// Can we exit an OpenMP declare variant scope at the moment. - bool isInOpenMPDeclareVariantScope() const { - return !OMPDeclareVariantScopes.empty(); - } + struct BuildingDeductionGuidesTag {}; + /// \brief Note that we are building deduction guides. + InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation, + TemplateDecl *Entity, BuildingDeductionGuidesTag, + SourceRange InstantiationRange = SourceRange()); - ExprResult - VerifyPositiveIntegerConstantInClause(Expr *Op, OpenMPClauseKind CKind, - bool StrictlyPositive = true, - bool SuppressExprDiags = false); + /// Note that we have finished instantiating this template. + void Clear(); - /// Given the potential call expression \p Call, determine if there is a - /// specialization via the OpenMP declare variant mechanism available. If - /// there is, return the specialized call expression, otherwise return the - /// original \p Call. - ExprResult ActOnOpenMPCall(ExprResult Call, Scope *Scope, - SourceLocation LParenLoc, MultiExprArg ArgExprs, - SourceLocation RParenLoc, Expr *ExecConfig); + ~InstantiatingTemplate() { Clear(); } - /// Handle a `omp begin declare variant`. - void ActOnOpenMPBeginDeclareVariant(SourceLocation Loc, OMPTraitInfo &TI); + /// Determines whether we have exceeded the maximum + /// recursive template instantiations. + bool isInvalid() const { return Invalid; } - /// Handle a `omp end declare variant`. - void ActOnOpenMPEndDeclareVariant(); + /// Determine whether we are already instantiating this + /// specialization in some surrounding active instantiation. + bool isAlreadyInstantiating() const { return AlreadyInstantiating; } - /// Checks if the variant/multiversion functions are compatible. - bool areMultiversionVariantFunctionsCompatible( - const FunctionDecl *OldFD, const FunctionDecl *NewFD, - const PartialDiagnostic &NoProtoDiagID, - const PartialDiagnosticAt &NoteCausedDiagIDAt, - const PartialDiagnosticAt &NoSupportDiagIDAt, - const PartialDiagnosticAt &DiffDiagIDAt, bool TemplatesSupported, - bool ConstexprSupported, bool CLinkageMayDiffer); + private: + Sema &SemaRef; + bool Invalid; + bool AlreadyInstantiating; + bool CheckInstantiationDepth(SourceLocation PointOfInstantiation, + SourceRange InstantiationRange); - /// Function tries to capture lambda's captured variables in the OpenMP region - /// before the original lambda is captured. - void tryCaptureOpenMPLambdas(ValueDecl *V); + InstantiatingTemplate( + Sema &SemaRef, CodeSynthesisContext::SynthesisKind Kind, + SourceLocation PointOfInstantiation, SourceRange InstantiationRange, + Decl *Entity, NamedDecl *Template = nullptr, + ArrayRef TemplateArgs = std::nullopt, + sema::TemplateDeductionInfo *DeductionInfo = nullptr); - /// Return true if the provided declaration \a VD should be captured by - /// reference. - /// \param Level Relative level of nested OpenMP construct for that the check - /// is performed. - /// \param OpenMPCaptureLevel Capture level within an OpenMP construct. - bool isOpenMPCapturedByRef(const ValueDecl *D, unsigned Level, - unsigned OpenMPCaptureLevel) const; + InstantiatingTemplate(const InstantiatingTemplate &) = delete; - /// Check if the specified variable is used in one of the private - /// clauses (private, firstprivate, lastprivate, reduction etc.) in OpenMP - /// constructs. - VarDecl *isOpenMPCapturedDecl(ValueDecl *D, bool CheckScopeInfo = false, - unsigned StopAt = 0); + InstantiatingTemplate &operator=(const InstantiatingTemplate &) = delete; + }; - /// The member expression(this->fd) needs to be rebuilt in the template - /// instantiation to generate private copy for OpenMP when default - /// clause is used. The function will return true if default - /// cluse is used. - bool isOpenMPRebuildMemberExpr(ValueDecl *D); + bool + SubstTemplateArguments(ArrayRef Args, + const MultiLevelTemplateArgumentList &TemplateArgs, + TemplateArgumentListInfo &Outputs); - ExprResult getOpenMPCapturedExpr(VarDecl *Capture, ExprValueKind VK, - ExprObjectKind OK, SourceLocation Loc); + MultiLevelTemplateArgumentList getTemplateInstantiationArgs( + const NamedDecl *D, const DeclContext *DC = nullptr, bool Final = false, + std::optional> Innermost = std::nullopt, + bool RelativeToPrimary = false, const FunctionDecl *Pattern = nullptr, + bool ForConstraintInstantiation = false, + bool SkipForSpecialization = false); - /// If the current region is a loop-based region, mark the start of the loop - /// construct. - void startOpenMPLoop(); + /// RAII object to handle the state changes required to synthesize + /// a function body. + class SynthesizedFunctionScope { + Sema &S; + Sema::ContextRAII SavedContext; + bool PushedCodeSynthesisContext = false; - /// If the current region is a range loop-based region, mark the start of the - /// loop construct. - void startOpenMPCXXRangeFor(); + public: + SynthesizedFunctionScope(Sema &S, DeclContext *DC) + : S(S), SavedContext(S, DC) { + auto *FD = dyn_cast(DC); + S.PushFunctionScope(); + S.PushExpressionEvaluationContext( + (FD && FD->isConsteval()) + ? ExpressionEvaluationContext::ImmediateFunctionContext + : ExpressionEvaluationContext::PotentiallyEvaluated); + if (FD) { + FD->setWillHaveBody(true); + S.ExprEvalContexts.back().InImmediateFunctionContext = + FD->isImmediateFunction() || + S.ExprEvalContexts[S.ExprEvalContexts.size() - 2] + .isConstantEvaluated(); + S.ExprEvalContexts.back().InImmediateEscalatingFunctionContext = + S.getLangOpts().CPlusPlus20 && FD->isImmediateEscalating(); + } else + assert(isa(DC)); + } - /// Check if the specified variable is used in 'private' clause. - /// \param Level Relative level of nested OpenMP construct for that the check - /// is performed. - OpenMPClauseKind isOpenMPPrivateDecl(ValueDecl *D, unsigned Level, - unsigned CapLevel) const; + void addContextNote(SourceLocation UseLoc) { + assert(!PushedCodeSynthesisContext); - /// Sets OpenMP capture kind (OMPC_private, OMPC_firstprivate, OMPC_map etc.) - /// for \p FD based on DSA for the provided corresponding captured declaration - /// \p D. - void setOpenMPCaptureKind(FieldDecl *FD, const ValueDecl *D, unsigned Level); + Sema::CodeSynthesisContext Ctx; + Ctx.Kind = Sema::CodeSynthesisContext::DefiningSynthesizedFunction; + Ctx.PointOfInstantiation = UseLoc; + Ctx.Entity = cast(S.CurContext); + S.pushCodeSynthesisContext(Ctx); - /// Check if the specified variable is captured by 'target' directive. - /// \param Level Relative level of nested OpenMP construct for that the check - /// is performed. - bool isOpenMPTargetCapturedDecl(const ValueDecl *D, unsigned Level, - unsigned CaptureLevel) const; + PushedCodeSynthesisContext = true; + } - /// Check if the specified global variable must be captured by outer capture - /// regions. - /// \param Level Relative level of nested OpenMP construct for that - /// the check is performed. - bool isOpenMPGlobalCapturedDecl(ValueDecl *D, unsigned Level, - unsigned CaptureLevel) const; + ~SynthesizedFunctionScope() { + if (PushedCodeSynthesisContext) + S.popCodeSynthesisContext(); + if (auto *FD = dyn_cast(S.CurContext)) { + FD->setWillHaveBody(false); + S.CheckImmediateEscalatingFunctionDefinition(FD, S.getCurFunction()); + } + S.PopExpressionEvaluationContext(); + S.PopFunctionScopeInfo(); + } + }; - ExprResult PerformOpenMPImplicitIntegerConversion(SourceLocation OpLoc, - Expr *Op); - /// Called on start of new data sharing attribute block. - void StartOpenMPDSABlock(OpenMPDirectiveKind K, - const DeclarationNameInfo &DirName, Scope *CurScope, - SourceLocation Loc); - /// Start analysis of clauses. - void StartOpenMPClause(OpenMPClauseKind K); - /// End analysis of clauses. - void EndOpenMPClause(); - /// Called on end of data sharing attribute block. - void EndOpenMPDSABlock(Stmt *CurDirective); + /// List of active code synthesis contexts. + /// + /// This vector is treated as a stack. As synthesis of one entity requires + /// synthesis of another, additional contexts are pushed onto the stack. + SmallVector CodeSynthesisContexts; - /// Check if the current region is an OpenMP loop region and if it is, - /// mark loop control variable, used in \p Init for loop initialization, as - /// private by default. - /// \param Init First part of the for loop. - void ActOnOpenMPLoopInitialization(SourceLocation ForLoc, Stmt *Init); + /// Specializations whose definitions are currently being instantiated. + llvm::DenseSet> InstantiatingSpecializations; - /// Called on well-formed '\#pragma omp metadirective' after parsing - /// of the associated statement. - StmtResult ActOnOpenMPMetaDirective(ArrayRef Clauses, - Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc); + /// Non-dependent types used in templates that have already been instantiated + /// by some template instantiation. + llvm::DenseSet InstantiatedNonDependentTypes; - // OpenMP directives and clauses. - /// Called on correct id-expression from the '#pragma omp - /// threadprivate'. - ExprResult ActOnOpenMPIdExpression(Scope *CurScope, CXXScopeSpec &ScopeSpec, - const DeclarationNameInfo &Id, - OpenMPDirectiveKind Kind); - /// Called on well-formed '#pragma omp threadprivate'. - DeclGroupPtrTy ActOnOpenMPThreadprivateDirective( - SourceLocation Loc, - ArrayRef VarList); - /// Builds a new OpenMPThreadPrivateDecl and checks its correctness. - OMPThreadPrivateDecl *CheckOMPThreadPrivateDecl(SourceLocation Loc, - ArrayRef VarList); - /// Called on well-formed '#pragma omp allocate'. - DeclGroupPtrTy ActOnOpenMPAllocateDirective(SourceLocation Loc, - ArrayRef VarList, - ArrayRef Clauses, - DeclContext *Owner = nullptr); + /// Extra modules inspected when performing a lookup during a template + /// instantiation. Computed lazily. + SmallVector CodeSynthesisContextLookupModules; - /// Called on well-formed '#pragma omp [begin] assume[s]'. - void ActOnOpenMPAssumesDirective(SourceLocation Loc, - OpenMPDirectiveKind DKind, - ArrayRef Assumptions, - bool SkippedClauses); + /// Cache of additional modules that should be used for name lookup + /// within the current template instantiation. Computed lazily; use + /// getLookupModules() to get a complete set. + llvm::DenseSet LookupModulesCache; - /// Check if there is an active global `omp begin assumes` directive. - bool isInOpenMPAssumeScope() const { return !OMPAssumeScoped.empty(); } + /// Map from the most recent declaration of a namespace to the most + /// recent visible declaration of that namespace. + llvm::DenseMap VisibleNamespaceCache; - /// Check if there is an active global `omp assumes` directive. - bool hasGlobalOpenMPAssumes() const { return !OMPAssumeGlobal.empty(); } + /// Whether we are in a SFINAE context that is not associated with + /// template instantiation. + /// + /// This is used when setting up a SFINAE trap (\c see SFINAETrap) outside + /// of a template instantiation or template argument deduction. + bool InNonInstantiationSFINAEContext; - /// Called on well-formed '#pragma omp end assumes'. - void ActOnOpenMPEndAssumesDirective(); + /// The number of \p CodeSynthesisContexts that are not template + /// instantiations and, therefore, should not be counted as part of the + /// instantiation depth. + /// + /// When the instantiation depth reaches the user-configurable limit + /// \p LangOptions::InstantiationDepth we will abort instantiation. + // FIXME: Should we have a similar limit for other forms of synthesis? + unsigned NonInstantiationEntries; - /// Called on well-formed '#pragma omp requires'. - DeclGroupPtrTy ActOnOpenMPRequiresDirective(SourceLocation Loc, - ArrayRef ClauseList); - /// Check restrictions on Requires directive - OMPRequiresDecl *CheckOMPRequiresDecl(SourceLocation Loc, - ArrayRef Clauses); - /// Check if the specified type is allowed to be used in 'omp declare - /// reduction' construct. - QualType ActOnOpenMPDeclareReductionType(SourceLocation TyLoc, - TypeResult ParsedType); - /// Called on start of '#pragma omp declare reduction'. - DeclGroupPtrTy ActOnOpenMPDeclareReductionDirectiveStart( - Scope *S, DeclContext *DC, DeclarationName Name, - ArrayRef> ReductionTypes, - AccessSpecifier AS, Decl *PrevDeclInScope = nullptr); - /// Initialize declare reduction construct initializer. - void ActOnOpenMPDeclareReductionCombinerStart(Scope *S, Decl *D); - /// Finish current declare reduction construct initializer. - void ActOnOpenMPDeclareReductionCombinerEnd(Decl *D, Expr *Combiner); - /// Initialize declare reduction construct initializer. - /// \return omp_priv variable. - VarDecl *ActOnOpenMPDeclareReductionInitializerStart(Scope *S, Decl *D); - /// Finish current declare reduction construct initializer. - void ActOnOpenMPDeclareReductionInitializerEnd(Decl *D, Expr *Initializer, - VarDecl *OmpPrivParm); - /// Called at the end of '#pragma omp declare reduction'. - DeclGroupPtrTy ActOnOpenMPDeclareReductionDirectiveEnd( - Scope *S, DeclGroupPtrTy DeclReductions, bool IsValid); + /// The depth of the context stack at the point when the most recent + /// error or warning was produced. + /// + /// This value is used to suppress printing of redundant context stacks + /// when there are multiple errors or warnings in the same instantiation. + // FIXME: Does this belong in Sema? It's tough to implement it anywhere else. + unsigned LastEmittedCodeSynthesisContextDepth = 0; - /// Check variable declaration in 'omp declare mapper' construct. - TypeResult ActOnOpenMPDeclareMapperVarDecl(Scope *S, Declarator &D); - /// Check if the specified type is allowed to be used in 'omp declare - /// mapper' construct. - QualType ActOnOpenMPDeclareMapperType(SourceLocation TyLoc, - TypeResult ParsedType); - /// Called on start of '#pragma omp declare mapper'. - DeclGroupPtrTy ActOnOpenMPDeclareMapperDirective( - Scope *S, DeclContext *DC, DeclarationName Name, QualType MapperType, - SourceLocation StartLoc, DeclarationName VN, AccessSpecifier AS, - Expr *MapperVarRef, ArrayRef Clauses, - Decl *PrevDeclInScope = nullptr); - /// Build the mapper variable of '#pragma omp declare mapper'. - ExprResult ActOnOpenMPDeclareMapperDirectiveVarDecl(Scope *S, - QualType MapperType, - SourceLocation StartLoc, - DeclarationName VN); - void ActOnOpenMPIteratorVarDecl(VarDecl *VD); - bool isOpenMPDeclareMapperVarDeclAllowed(const VarDecl *VD) const; - const ValueDecl *getOpenMPDeclareMapperVarName() const; + /// The template instantiation callbacks to trace or track + /// instantiations (objects can be chained). + /// + /// This callbacks is used to print, trace or track template + /// instantiations as they are being constructed. + std::vector> + TemplateInstCallbacks; - /// Called on the start of target region i.e. '#pragma omp declare target'. - bool ActOnStartOpenMPDeclareTargetContext(DeclareTargetContextInfo &DTCI); + /// The current index into pack expansion arguments that will be + /// used for substitution of parameter packs. + /// + /// The pack expansion index will be -1 to indicate that parameter packs + /// should be instantiated as themselves. Otherwise, the index specifies + /// which argument within the parameter pack will be used for substitution. + int ArgumentPackSubstitutionIndex; - /// Called at the end of target region i.e. '#pragma omp end declare target'. - const DeclareTargetContextInfo ActOnOpenMPEndDeclareTargetDirective(); + /// RAII object used to change the argument pack substitution index + /// within a \c Sema object. + /// + /// See \c ArgumentPackSubstitutionIndex for more information. + class ArgumentPackSubstitutionIndexRAII { + Sema &Self; + int OldSubstitutionIndex; - /// Called once a target context is completed, that can be when a - /// '#pragma omp end declare target' was encountered or when a - /// '#pragma omp declare target' without declaration-definition-seq was - /// encountered. - void ActOnFinishedOpenMPDeclareTargetContext(DeclareTargetContextInfo &DTCI); + public: + ArgumentPackSubstitutionIndexRAII(Sema &Self, int NewSubstitutionIndex) + : Self(Self), OldSubstitutionIndex(Self.ArgumentPackSubstitutionIndex) { + Self.ArgumentPackSubstitutionIndex = NewSubstitutionIndex; + } - /// Report unterminated 'omp declare target' or 'omp begin declare target' at - /// the end of a compilation unit. - void DiagnoseUnterminatedOpenMPDeclareTarget(); + ~ArgumentPackSubstitutionIndexRAII() { + Self.ArgumentPackSubstitutionIndex = OldSubstitutionIndex; + } + }; - /// Searches for the provided declaration name for OpenMP declare target - /// directive. - NamedDecl *lookupOpenMPDeclareTargetName(Scope *CurScope, - CXXScopeSpec &ScopeSpec, - const DeclarationNameInfo &Id); + friend class ArgumentPackSubstitutionRAII; - /// Called on correct id-expression from the '#pragma omp declare target'. - void ActOnOpenMPDeclareTargetName(NamedDecl *ND, SourceLocation Loc, - OMPDeclareTargetDeclAttr::MapTypeTy MT, - DeclareTargetContextInfo &DTCI); + void pushCodeSynthesisContext(CodeSynthesisContext Ctx); + void popCodeSynthesisContext(); - /// Check declaration inside target region. - void - checkDeclIsAllowedInOpenMPTarget(Expr *E, Decl *D, - SourceLocation IdLoc = SourceLocation()); + void PrintContextStack() { + if (!CodeSynthesisContexts.empty() && + CodeSynthesisContexts.size() != LastEmittedCodeSynthesisContextDepth) { + PrintInstantiationStack(); + LastEmittedCodeSynthesisContextDepth = CodeSynthesisContexts.size(); + } + if (PragmaAttributeCurrentTargetDecl) + PrintPragmaAttributeInstantiationPoint(); + } + void PrintInstantiationStack(); - /// Adds OMPDeclareTargetDeclAttr to referenced variables in declare target - /// directive. - void ActOnOpenMPDeclareTargetInitializer(Decl *D); + /// Determines whether we are currently in a context where + /// template argument substitution failures are not considered + /// errors. + /// + /// \returns An empty \c Optional if we're not in a SFINAE context. + /// Otherwise, contains a pointer that, if non-NULL, contains the nearest + /// template-deduction context object, which can be used to capture + /// diagnostics that will be suppressed. + std::optional isSFINAEContext() const; - /// Finishes analysis of the deferred functions calls that may be declared as - /// host/nohost during device/host compilation. - void finalizeOpenMPDelayedAnalysis(const FunctionDecl *Caller, - const FunctionDecl *Callee, - SourceLocation Loc); + TypeSourceInfo *SubstType(TypeSourceInfo *T, + const MultiLevelTemplateArgumentList &TemplateArgs, + SourceLocation Loc, DeclarationName Entity, + bool AllowDeducedTST = false); - /// Return true if currently in OpenMP task with untied clause context. - bool isInOpenMPTaskUntiedContext() const; + QualType SubstType(QualType T, + const MultiLevelTemplateArgumentList &TemplateArgs, + SourceLocation Loc, DeclarationName Entity); - /// Return true inside OpenMP declare target region. - bool isInOpenMPDeclareTargetContext() const { - return !DeclareTargetNesting.empty(); - } - /// Return true inside OpenMP target region. - bool isInOpenMPTargetExecutionDirective() const; + TypeSourceInfo *SubstType(TypeLoc TL, + const MultiLevelTemplateArgumentList &TemplateArgs, + SourceLocation Loc, DeclarationName Entity); - /// Return the number of captured regions created for an OpenMP directive. - static int getOpenMPCaptureLevels(OpenMPDirectiveKind Kind); + TypeSourceInfo *SubstFunctionDeclType( + TypeSourceInfo *T, const MultiLevelTemplateArgumentList &TemplateArgs, + SourceLocation Loc, DeclarationName Entity, CXXRecordDecl *ThisContext, + Qualifiers ThisTypeQuals, bool EvaluateConstraints = true); + void SubstExceptionSpec(FunctionDecl *New, const FunctionProtoType *Proto, + const MultiLevelTemplateArgumentList &Args); + bool SubstExceptionSpec(SourceLocation Loc, + FunctionProtoType::ExceptionSpecInfo &ESI, + SmallVectorImpl &ExceptionStorage, + const MultiLevelTemplateArgumentList &Args); + ParmVarDecl * + SubstParmVarDecl(ParmVarDecl *D, + const MultiLevelTemplateArgumentList &TemplateArgs, + int indexAdjustment, std::optional NumExpansions, + bool ExpectParameterPack, bool EvaluateConstraints = true); + bool SubstParmTypes(SourceLocation Loc, ArrayRef Params, + const FunctionProtoType::ExtParameterInfo *ExtParamInfos, + const MultiLevelTemplateArgumentList &TemplateArgs, + SmallVectorImpl &ParamTypes, + SmallVectorImpl *OutParams, + ExtParameterInfoBuilder &ParamInfos); + bool SubstDefaultArgument(SourceLocation Loc, ParmVarDecl *Param, + const MultiLevelTemplateArgumentList &TemplateArgs, + bool ForCallExpr = false); + ExprResult SubstExpr(Expr *E, + const MultiLevelTemplateArgumentList &TemplateArgs); - /// Initialization of captured region for OpenMP region. - void ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope); + // A RAII type used by the TemplateDeclInstantiator and TemplateInstantiator + // to disable constraint evaluation, then restore the state. + template struct ConstraintEvalRAII { + InstTy &TI; + bool OldValue; - /// Called for syntactical loops (ForStmt or CXXForRangeStmt) associated to - /// an OpenMP loop directive. - StmtResult ActOnOpenMPCanonicalLoop(Stmt *AStmt); + ConstraintEvalRAII(InstTy &TI) + : TI(TI), OldValue(TI.getEvaluateConstraints()) { + TI.setEvaluateConstraints(false); + } + ~ConstraintEvalRAII() { TI.setEvaluateConstraints(OldValue); } + }; - /// Process a canonical OpenMP loop nest that can either be a canonical - /// literal loop (ForStmt or CXXForRangeStmt), or the generated loop of an - /// OpenMP loop transformation construct. - StmtResult ActOnOpenMPLoopnest(Stmt *AStmt); + // Must be used instead of SubstExpr at 'constraint checking' time. + ExprResult + SubstConstraintExpr(Expr *E, + const MultiLevelTemplateArgumentList &TemplateArgs); + // Unlike the above, this does not evaluates constraints. + ExprResult SubstConstraintExprWithoutSatisfaction( + Expr *E, const MultiLevelTemplateArgumentList &TemplateArgs); - /// End of OpenMP region. + /// Substitute the given template arguments into a list of + /// expressions, expanding pack expansions if required. /// - /// \param S Statement associated with the current OpenMP region. - /// \param Clauses List of clauses for the current OpenMP region. + /// \param Exprs The list of expressions to substitute into. /// - /// \returns Statement for finished OpenMP region. - StmtResult ActOnOpenMPRegionEnd(StmtResult S, ArrayRef Clauses); - StmtResult ActOnOpenMPExecutableDirective( - OpenMPDirectiveKind Kind, const DeclarationNameInfo &DirName, - OpenMPDirectiveKind CancelRegion, ArrayRef Clauses, - Stmt *AStmt, SourceLocation StartLoc, SourceLocation EndLoc, - OpenMPDirectiveKind PrevMappedDirective = llvm::omp::OMPD_unknown); - /// Called on well-formed '\#pragma omp parallel' after parsing - /// of the associated statement. - StmtResult ActOnOpenMPParallelDirective(ArrayRef Clauses, - Stmt *AStmt, - SourceLocation StartLoc, - SourceLocation EndLoc); - using VarsWithInheritedDSAType = - llvm::SmallDenseMap; - /// Called on well-formed '\#pragma omp simd' after parsing - /// of the associated statement. - StmtResult - ActOnOpenMPSimdDirective(ArrayRef Clauses, Stmt *AStmt, - SourceLocation StartLoc, SourceLocation EndLoc, - VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '#pragma omp tile' after parsing of its clauses and - /// the associated statement. - StmtResult ActOnOpenMPTileDirective(ArrayRef Clauses, - Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '#pragma omp unroll' after parsing of its clauses - /// and the associated statement. - StmtResult ActOnOpenMPUnrollDirective(ArrayRef Clauses, - Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp for' after parsing - /// of the associated statement. - StmtResult - ActOnOpenMPForDirective(ArrayRef Clauses, Stmt *AStmt, - SourceLocation StartLoc, SourceLocation EndLoc, - VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp for simd' after parsing - /// of the associated statement. - StmtResult - ActOnOpenMPForSimdDirective(ArrayRef Clauses, Stmt *AStmt, - SourceLocation StartLoc, SourceLocation EndLoc, - VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp sections' after parsing - /// of the associated statement. - StmtResult ActOnOpenMPSectionsDirective(ArrayRef Clauses, - Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp section' after parsing of the - /// associated statement. - StmtResult ActOnOpenMPSectionDirective(Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp scope' after parsing of the - /// associated statement. - StmtResult ActOnOpenMPScopeDirective(ArrayRef Clauses, - Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp single' after parsing of the - /// associated statement. - StmtResult ActOnOpenMPSingleDirective(ArrayRef Clauses, - Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp master' after parsing of the - /// associated statement. - StmtResult ActOnOpenMPMasterDirective(Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp critical' after parsing of the - /// associated statement. - StmtResult ActOnOpenMPCriticalDirective(const DeclarationNameInfo &DirName, - ArrayRef Clauses, - Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp parallel for' after parsing - /// of the associated statement. - StmtResult ActOnOpenMPParallelForDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp parallel for simd' after - /// parsing of the associated statement. - StmtResult ActOnOpenMPParallelForSimdDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp parallel master' after - /// parsing of the associated statement. - StmtResult ActOnOpenMPParallelMasterDirective(ArrayRef Clauses, - Stmt *AStmt, - SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp parallel masked' after - /// parsing of the associated statement. - StmtResult ActOnOpenMPParallelMaskedDirective(ArrayRef Clauses, - Stmt *AStmt, - SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp parallel sections' after - /// parsing of the associated statement. - StmtResult ActOnOpenMPParallelSectionsDirective(ArrayRef Clauses, - Stmt *AStmt, - SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp task' after parsing of the - /// associated statement. - StmtResult ActOnOpenMPTaskDirective(ArrayRef Clauses, - Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp taskyield'. - StmtResult ActOnOpenMPTaskyieldDirective(SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp error'. - /// Error direcitive is allowed in both declared and excutable contexts. - /// Adding InExContext to identify which context is called from. - StmtResult ActOnOpenMPErrorDirective(ArrayRef Clauses, - SourceLocation StartLoc, - SourceLocation EndLoc, - bool InExContext = true); - /// Called on well-formed '\#pragma omp barrier'. - StmtResult ActOnOpenMPBarrierDirective(SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp taskwait'. - StmtResult ActOnOpenMPTaskwaitDirective(ArrayRef Clauses, - SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp taskgroup'. - StmtResult ActOnOpenMPTaskgroupDirective(ArrayRef Clauses, - Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp flush'. - StmtResult ActOnOpenMPFlushDirective(ArrayRef Clauses, - SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp depobj'. - StmtResult ActOnOpenMPDepobjDirective(ArrayRef Clauses, - SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp scan'. - StmtResult ActOnOpenMPScanDirective(ArrayRef Clauses, - SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp ordered' after parsing of the - /// associated statement. - StmtResult ActOnOpenMPOrderedDirective(ArrayRef Clauses, - Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp atomic' after parsing of the - /// associated statement. - StmtResult ActOnOpenMPAtomicDirective(ArrayRef Clauses, - Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp target' after parsing of the - /// associated statement. - StmtResult ActOnOpenMPTargetDirective(ArrayRef Clauses, - Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp target data' after parsing of - /// the associated statement. - StmtResult ActOnOpenMPTargetDataDirective(ArrayRef Clauses, - Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp target enter data' after - /// parsing of the associated statement. - StmtResult ActOnOpenMPTargetEnterDataDirective(ArrayRef Clauses, - SourceLocation StartLoc, - SourceLocation EndLoc, - Stmt *AStmt); - /// Called on well-formed '\#pragma omp target exit data' after - /// parsing of the associated statement. - StmtResult ActOnOpenMPTargetExitDataDirective(ArrayRef Clauses, - SourceLocation StartLoc, - SourceLocation EndLoc, - Stmt *AStmt); - /// Called on well-formed '\#pragma omp target parallel' after - /// parsing of the associated statement. - StmtResult ActOnOpenMPTargetParallelDirective(ArrayRef Clauses, - Stmt *AStmt, - SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp target parallel for' after - /// parsing of the associated statement. - StmtResult ActOnOpenMPTargetParallelForDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp teams' after parsing of the - /// associated statement. - StmtResult ActOnOpenMPTeamsDirective(ArrayRef Clauses, - Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp teams loop' after parsing of the - /// associated statement. - StmtResult ActOnOpenMPTeamsGenericLoopDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp target teams loop' after parsing of - /// the associated statement. - StmtResult ActOnOpenMPTargetTeamsGenericLoopDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp parallel loop' after parsing of the - /// associated statement. - StmtResult ActOnOpenMPParallelGenericLoopDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp target parallel loop' after parsing - /// of the associated statement. - StmtResult ActOnOpenMPTargetParallelGenericLoopDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp cancellation point'. - StmtResult - ActOnOpenMPCancellationPointDirective(SourceLocation StartLoc, - SourceLocation EndLoc, - OpenMPDirectiveKind CancelRegion); - /// Called on well-formed '\#pragma omp cancel'. - StmtResult ActOnOpenMPCancelDirective(ArrayRef Clauses, - SourceLocation StartLoc, - SourceLocation EndLoc, - OpenMPDirectiveKind CancelRegion); - /// Called on well-formed '\#pragma omp taskloop' after parsing of the - /// associated statement. - StmtResult - ActOnOpenMPTaskLoopDirective(ArrayRef Clauses, Stmt *AStmt, - SourceLocation StartLoc, SourceLocation EndLoc, - VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp taskloop simd' after parsing of - /// the associated statement. - StmtResult ActOnOpenMPTaskLoopSimdDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp master taskloop' after parsing of the - /// associated statement. - StmtResult ActOnOpenMPMasterTaskLoopDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp master taskloop simd' after parsing of - /// the associated statement. - StmtResult ActOnOpenMPMasterTaskLoopSimdDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp parallel master taskloop' after - /// parsing of the associated statement. - StmtResult ActOnOpenMPParallelMasterTaskLoopDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp parallel master taskloop simd' after - /// parsing of the associated statement. - StmtResult ActOnOpenMPParallelMasterTaskLoopSimdDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp masked taskloop' after parsing of the - /// associated statement. - StmtResult ActOnOpenMPMaskedTaskLoopDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp masked taskloop simd' after parsing of - /// the associated statement. - StmtResult ActOnOpenMPMaskedTaskLoopSimdDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp parallel masked taskloop' after - /// parsing of the associated statement. - StmtResult ActOnOpenMPParallelMaskedTaskLoopDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp parallel masked taskloop simd' after - /// parsing of the associated statement. - StmtResult ActOnOpenMPParallelMaskedTaskLoopSimdDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp distribute' after parsing - /// of the associated statement. - StmtResult - ActOnOpenMPDistributeDirective(ArrayRef Clauses, Stmt *AStmt, - SourceLocation StartLoc, SourceLocation EndLoc, - VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp target update'. - StmtResult ActOnOpenMPTargetUpdateDirective(ArrayRef Clauses, - SourceLocation StartLoc, - SourceLocation EndLoc, - Stmt *AStmt); - /// Called on well-formed '\#pragma omp distribute parallel for' after - /// parsing of the associated statement. - StmtResult ActOnOpenMPDistributeParallelForDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp distribute parallel for simd' - /// after parsing of the associated statement. - StmtResult ActOnOpenMPDistributeParallelForSimdDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp distribute simd' after - /// parsing of the associated statement. - StmtResult ActOnOpenMPDistributeSimdDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp target parallel for simd' after - /// parsing of the associated statement. - StmtResult ActOnOpenMPTargetParallelForSimdDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp target simd' after parsing of - /// the associated statement. - StmtResult - ActOnOpenMPTargetSimdDirective(ArrayRef Clauses, Stmt *AStmt, - SourceLocation StartLoc, SourceLocation EndLoc, - VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp teams distribute' after parsing of - /// the associated statement. - StmtResult ActOnOpenMPTeamsDistributeDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp teams distribute simd' after parsing - /// of the associated statement. - StmtResult ActOnOpenMPTeamsDistributeSimdDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp teams distribute parallel for simd' - /// after parsing of the associated statement. - StmtResult ActOnOpenMPTeamsDistributeParallelForSimdDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp teams distribute parallel for' - /// after parsing of the associated statement. - StmtResult ActOnOpenMPTeamsDistributeParallelForDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp target teams' after parsing of the - /// associated statement. - StmtResult ActOnOpenMPTargetTeamsDirective(ArrayRef Clauses, - Stmt *AStmt, - SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp target teams distribute' after parsing - /// of the associated statement. - StmtResult ActOnOpenMPTargetTeamsDistributeDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp target teams distribute parallel for' - /// after parsing of the associated statement. - StmtResult ActOnOpenMPTargetTeamsDistributeParallelForDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp target teams distribute parallel for - /// simd' after parsing of the associated statement. - StmtResult ActOnOpenMPTargetTeamsDistributeParallelForSimdDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp target teams distribute simd' after - /// parsing of the associated statement. - StmtResult ActOnOpenMPTargetTeamsDistributeSimdDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - /// Called on well-formed '\#pragma omp interop'. - StmtResult ActOnOpenMPInteropDirective(ArrayRef Clauses, - SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp dispatch' after parsing of the - // /associated statement. - StmtResult ActOnOpenMPDispatchDirective(ArrayRef Clauses, - Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed '\#pragma omp masked' after parsing of the - // /associated statement. - StmtResult ActOnOpenMPMaskedDirective(ArrayRef Clauses, - Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc); + /// \param IsCall Whether this is some form of call, in which case + /// default arguments will be dropped. + /// + /// \param TemplateArgs The set of template arguments to substitute. + /// + /// \param Outputs Will receive all of the substituted arguments. + /// + /// \returns true if an error occurred, false otherwise. + bool SubstExprs(ArrayRef Exprs, bool IsCall, + const MultiLevelTemplateArgumentList &TemplateArgs, + SmallVectorImpl &Outputs); + + StmtResult SubstStmt(Stmt *S, + const MultiLevelTemplateArgumentList &TemplateArgs); + + ExprResult + SubstInitializer(Expr *E, const MultiLevelTemplateArgumentList &TemplateArgs, + bool CXXDirectInit); + + bool SubstBaseSpecifiers(CXXRecordDecl *Instantiation, CXXRecordDecl *Pattern, + const MultiLevelTemplateArgumentList &TemplateArgs); + + bool InstantiateClass(SourceLocation PointOfInstantiation, + CXXRecordDecl *Instantiation, CXXRecordDecl *Pattern, + const MultiLevelTemplateArgumentList &TemplateArgs, + TemplateSpecializationKind TSK, bool Complain = true); + + bool InstantiateEnum(SourceLocation PointOfInstantiation, + EnumDecl *Instantiation, EnumDecl *Pattern, + const MultiLevelTemplateArgumentList &TemplateArgs, + TemplateSpecializationKind TSK); + + bool InstantiateInClassInitializer( + SourceLocation PointOfInstantiation, FieldDecl *Instantiation, + FieldDecl *Pattern, const MultiLevelTemplateArgumentList &TemplateArgs); + + bool usesPartialOrExplicitSpecialization( + SourceLocation Loc, ClassTemplateSpecializationDecl *ClassTemplateSpec); + + bool InstantiateClassTemplateSpecialization( + SourceLocation PointOfInstantiation, + ClassTemplateSpecializationDecl *ClassTemplateSpec, + TemplateSpecializationKind TSK, bool Complain = true); + + void + InstantiateClassMembers(SourceLocation PointOfInstantiation, + CXXRecordDecl *Instantiation, + const MultiLevelTemplateArgumentList &TemplateArgs, + TemplateSpecializationKind TSK); + + void InstantiateClassTemplateSpecializationMembers( + SourceLocation PointOfInstantiation, + ClassTemplateSpecializationDecl *ClassTemplateSpec, + TemplateSpecializationKind TSK); + + NestedNameSpecifierLoc SubstNestedNameSpecifierLoc( + NestedNameSpecifierLoc NNS, + const MultiLevelTemplateArgumentList &TemplateArgs); + + DeclarationNameInfo + SubstDeclarationNameInfo(const DeclarationNameInfo &NameInfo, + const MultiLevelTemplateArgumentList &TemplateArgs); + TemplateName + SubstTemplateName(NestedNameSpecifierLoc QualifierLoc, TemplateName Name, + SourceLocation Loc, + const MultiLevelTemplateArgumentList &TemplateArgs); + + bool SubstTypeConstraint(TemplateTypeParmDecl *Inst, const TypeConstraint *TC, + const MultiLevelTemplateArgumentList &TemplateArgs, + bool EvaluateConstraint); + + /// Determine whether we are currently performing template instantiation. + bool inTemplateInstantiation() const { + return CodeSynthesisContexts.size() > NonInstantiationEntries; + } + + ///@} + + // + // + // ------------------------------------------------------------------------- + // + // + + /// \name C++ Template Declaration Instantiation + /// Implementations are in SemaTemplateInstantiateDecl.cpp + ///@{ + +public: + /// An entity for which implicit template instantiation is required. + /// + /// The source location associated with the declaration is the first place in + /// the source code where the declaration was "used". It is not necessarily + /// the point of instantiation (which will be either before or after the + /// namespace-scope declaration that triggered this implicit instantiation), + /// However, it is the location that diagnostics should generally refer to, + /// because users will need to know what code triggered the instantiation. + typedef std::pair PendingImplicitInstantiation; + + /// The queue of implicit template instantiations that are required + /// but have not yet been performed. + std::deque PendingInstantiations; + + /// Queue of implicit template instantiations that cannot be performed + /// eagerly. + SmallVector LateParsedInstantiations; + + SmallVector, 8> SavedVTableUses; + SmallVector, 8> + SavedPendingInstantiations; + + /// The queue of implicit template instantiations that are required + /// and must be performed within the current local scope. + /// + /// This queue is only used for member functions of local classes in + /// templates, which must be instantiated in the same scope as their + /// enclosing function, so that they can reference function-local + /// types, static variables, enumerators, etc. + std::deque PendingLocalImplicitInstantiations; + + class LocalEagerInstantiationScope { + public: + LocalEagerInstantiationScope(Sema &S) : S(S) { + SavedPendingLocalImplicitInstantiations.swap( + S.PendingLocalImplicitInstantiations); + } + + void perform() { S.PerformPendingInstantiations(/*LocalOnly=*/true); } + + ~LocalEagerInstantiationScope() { + assert(S.PendingLocalImplicitInstantiations.empty() && + "there shouldn't be any pending local implicit instantiations"); + SavedPendingLocalImplicitInstantiations.swap( + S.PendingLocalImplicitInstantiations); + } + + private: + Sema &S; + std::deque + SavedPendingLocalImplicitInstantiations; + }; + + /// Records and restores the CurFPFeatures state on entry/exit of compound + /// statements. + class FPFeaturesStateRAII { + public: + FPFeaturesStateRAII(Sema &S); + ~FPFeaturesStateRAII(); + FPOptionsOverride getOverrides() { return OldOverrides; } + + private: + Sema &S; + FPOptions OldFPFeaturesState; + FPOptionsOverride OldOverrides; + LangOptions::FPEvalMethodKind OldEvalMethod; + SourceLocation OldFPPragmaLocation; + }; + + class GlobalEagerInstantiationScope { + public: + GlobalEagerInstantiationScope(Sema &S, bool Enabled) + : S(S), Enabled(Enabled) { + if (!Enabled) + return; + + S.SavedPendingInstantiations.emplace_back(); + S.SavedPendingInstantiations.back().swap(S.PendingInstantiations); + + S.SavedVTableUses.emplace_back(); + S.SavedVTableUses.back().swap(S.VTableUses); + } + + void perform() { + if (Enabled) { + S.DefineUsedVTables(); + S.PerformPendingInstantiations(); + } + } + + ~GlobalEagerInstantiationScope() { + if (!Enabled) + return; + + // Restore the set of pending vtables. + assert(S.VTableUses.empty() && + "VTableUses should be empty before it is discarded."); + S.VTableUses.swap(S.SavedVTableUses.back()); + S.SavedVTableUses.pop_back(); + + // Restore the set of pending implicit instantiations. + if (S.TUKind != TU_Prefix || !S.LangOpts.PCHInstantiateTemplates) { + assert(S.PendingInstantiations.empty() && + "PendingInstantiations should be empty before it is discarded."); + S.PendingInstantiations.swap(S.SavedPendingInstantiations.back()); + S.SavedPendingInstantiations.pop_back(); + } else { + // Template instantiations in the PCH may be delayed until the TU. + S.PendingInstantiations.swap(S.SavedPendingInstantiations.back()); + S.PendingInstantiations.insert( + S.PendingInstantiations.end(), + S.SavedPendingInstantiations.back().begin(), + S.SavedPendingInstantiations.back().end()); + S.SavedPendingInstantiations.pop_back(); + } + } + + private: + Sema &S; + bool Enabled; + }; + + ExplicitSpecifier instantiateExplicitSpecifier( + const MultiLevelTemplateArgumentList &TemplateArgs, ExplicitSpecifier ES); + + struct LateInstantiatedAttribute { + const Attr *TmplAttr; + LocalInstantiationScope *Scope; + Decl *NewDecl; + + LateInstantiatedAttribute(const Attr *A, LocalInstantiationScope *S, + Decl *D) + : TmplAttr(A), Scope(S), NewDecl(D) {} + }; + typedef SmallVector LateInstantiatedAttrVec; + + void InstantiateAttrs(const MultiLevelTemplateArgumentList &TemplateArgs, + const Decl *Pattern, Decl *Inst, + LateInstantiatedAttrVec *LateAttrs = nullptr, + LocalInstantiationScope *OuterMostScope = nullptr); + void updateAttrsForLateParsedTemplate(const Decl *Pattern, Decl *Inst); + + void + InstantiateAttrsForDecl(const MultiLevelTemplateArgumentList &TemplateArgs, + const Decl *Pattern, Decl *Inst, + LateInstantiatedAttrVec *LateAttrs = nullptr, + LocalInstantiationScope *OuterMostScope = nullptr); + + void InstantiateDefaultCtorDefaultArgs(CXXConstructorDecl *Ctor); + + bool InstantiateDefaultArgument(SourceLocation CallLoc, FunctionDecl *FD, + ParmVarDecl *Param); + void InstantiateExceptionSpec(SourceLocation PointOfInstantiation, + FunctionDecl *Function); + FunctionDecl *InstantiateFunctionDeclaration(FunctionTemplateDecl *FTD, + const TemplateArgumentList *Args, + SourceLocation Loc); + void InstantiateFunctionDefinition(SourceLocation PointOfInstantiation, + FunctionDecl *Function, + bool Recursive = false, + bool DefinitionRequired = false, + bool AtEndOfTU = false); + VarTemplateSpecializationDecl *BuildVarTemplateInstantiation( + VarTemplateDecl *VarTemplate, VarDecl *FromVar, + const TemplateArgumentList *PartialSpecArgs, + const TemplateArgumentListInfo &TemplateArgsInfo, + SmallVectorImpl &Converted, + SourceLocation PointOfInstantiation, + LateInstantiatedAttrVec *LateAttrs = nullptr, + LocalInstantiationScope *StartingScope = nullptr); + VarTemplateSpecializationDecl *CompleteVarTemplateSpecializationDecl( + VarTemplateSpecializationDecl *VarSpec, VarDecl *PatternDecl, + const MultiLevelTemplateArgumentList &TemplateArgs); + void + BuildVariableInstantiation(VarDecl *NewVar, VarDecl *OldVar, + const MultiLevelTemplateArgumentList &TemplateArgs, + LateInstantiatedAttrVec *LateAttrs, + DeclContext *Owner, + LocalInstantiationScope *StartingScope, + bool InstantiatingVarTemplate = false, + VarTemplateSpecializationDecl *PrevVTSD = nullptr); + + void InstantiateVariableInitializer( + VarDecl *Var, VarDecl *OldVar, + const MultiLevelTemplateArgumentList &TemplateArgs); + void InstantiateVariableDefinition(SourceLocation PointOfInstantiation, + VarDecl *Var, bool Recursive = false, + bool DefinitionRequired = false, + bool AtEndOfTU = false); + + void InstantiateMemInitializers( + CXXConstructorDecl *New, const CXXConstructorDecl *Tmpl, + const MultiLevelTemplateArgumentList &TemplateArgs); + + NamedDecl * + FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D, + const MultiLevelTemplateArgumentList &TemplateArgs, + bool FindingInstantiatedContext = false); + DeclContext * + FindInstantiatedContext(SourceLocation Loc, DeclContext *DC, + const MultiLevelTemplateArgumentList &TemplateArgs); + + Decl *SubstDecl(Decl *D, DeclContext *Owner, + const MultiLevelTemplateArgumentList &TemplateArgs); + + /// Substitute the name and return type of a defaulted 'operator<=>' to form + /// an implicit 'operator=='. + FunctionDecl *SubstSpaceshipAsEqualEqual(CXXRecordDecl *RD, + FunctionDecl *Spaceship); + + void PerformPendingInstantiations(bool LocalOnly = false); + + TemplateParameterList * + SubstTemplateParams(TemplateParameterList *Params, DeclContext *Owner, + const MultiLevelTemplateArgumentList &TemplateArgs, + bool EvaluateConstraints = true); + + void PerformDependentDiagnostics( + const DeclContext *Pattern, + const MultiLevelTemplateArgumentList &TemplateArgs); + +private: + /// Introduce the instantiated local variables into the local + /// instantiation scope. + void addInstantiatedLocalVarsToScope(FunctionDecl *Function, + const FunctionDecl *PatternDecl, + LocalInstantiationScope &Scope); + /// Introduce the instantiated function parameters into the local + /// instantiation scope, and set the parameter names to those used + /// in the template. + bool addInstantiatedParametersToScope( + FunctionDecl *Function, const FunctionDecl *PatternDecl, + LocalInstantiationScope &Scope, + const MultiLevelTemplateArgumentList &TemplateArgs); + + int ParsingClassDepth = 0; + + class SavePendingParsedClassStateRAII { + public: + SavePendingParsedClassStateRAII(Sema &S) : S(S) { swapSavedState(); } + + ~SavePendingParsedClassStateRAII() { + assert(S.DelayedOverridingExceptionSpecChecks.empty() && + "there shouldn't be any pending delayed exception spec checks"); + assert(S.DelayedEquivalentExceptionSpecChecks.empty() && + "there shouldn't be any pending delayed exception spec checks"); + swapSavedState(); + } + + private: + Sema &S; + decltype(DelayedOverridingExceptionSpecChecks) + SavedOverridingExceptionSpecChecks; + decltype(DelayedEquivalentExceptionSpecChecks) + SavedEquivalentExceptionSpecChecks; + + void swapSavedState() { + SavedOverridingExceptionSpecChecks.swap( + S.DelayedOverridingExceptionSpecChecks); + SavedEquivalentExceptionSpecChecks.swap( + S.DelayedEquivalentExceptionSpecChecks); + } + }; + + ///@} + + // + // + // ------------------------------------------------------------------------- + // + // + + /// \name C++ Variadic Templates + /// Implementations are in SemaTemplateVariadic.cpp + ///@{ + +public: + /// Determine whether an unexpanded parameter pack might be permitted in this + /// location. Useful for error recovery. + bool isUnexpandedParameterPackPermitted(); + + /// The context in which an unexpanded parameter pack is + /// being diagnosed. + /// + /// Note that the values of this enumeration line up with the first + /// argument to the \c err_unexpanded_parameter_pack diagnostic. + enum UnexpandedParameterPackContext { + /// An arbitrary expression. + UPPC_Expression = 0, + + /// The base type of a class type. + UPPC_BaseType, + + /// The type of an arbitrary declaration. + UPPC_DeclarationType, + + /// The type of a data member. + UPPC_DataMemberType, + + /// The size of a bit-field. + UPPC_BitFieldWidth, + + /// The expression in a static assertion. + UPPC_StaticAssertExpression, + + /// The fixed underlying type of an enumeration. + UPPC_FixedUnderlyingType, + + /// The enumerator value. + UPPC_EnumeratorValue, + + /// A using declaration. + UPPC_UsingDeclaration, + + /// A friend declaration. + UPPC_FriendDeclaration, + + /// A declaration qualifier. + UPPC_DeclarationQualifier, + + /// An initializer. + UPPC_Initializer, + + /// A default argument. + UPPC_DefaultArgument, + + /// The type of a non-type template parameter. + UPPC_NonTypeTemplateParameterType, + + /// The type of an exception. + UPPC_ExceptionType, + + /// Explicit specialization. + UPPC_ExplicitSpecialization, + + /// Partial specialization. + UPPC_PartialSpecialization, + + /// Microsoft __if_exists. + UPPC_IfExists, + + /// Microsoft __if_not_exists. + UPPC_IfNotExists, + + /// Lambda expression. + UPPC_Lambda, + + /// Block expression. + UPPC_Block, + + /// A type constraint. + UPPC_TypeConstraint, + + // A requirement in a requires-expression. + UPPC_Requirement, + + // A requires-clause. + UPPC_RequiresClause, + }; + + /// Diagnose unexpanded parameter packs. + /// + /// \param Loc The location at which we should emit the diagnostic. + /// + /// \param UPPC The context in which we are diagnosing unexpanded + /// parameter packs. + /// + /// \param Unexpanded the set of unexpanded parameter packs. + /// + /// \returns true if an error occurred, false otherwise. + bool DiagnoseUnexpandedParameterPacks( + SourceLocation Loc, UnexpandedParameterPackContext UPPC, + ArrayRef Unexpanded); + + /// If the given type contains an unexpanded parameter pack, + /// diagnose the error. + /// + /// \param Loc The source location where a diagnostc should be emitted. + /// + /// \param T The type that is being checked for unexpanded parameter + /// packs. + /// + /// \returns true if an error occurred, false otherwise. + bool DiagnoseUnexpandedParameterPack(SourceLocation Loc, TypeSourceInfo *T, + UnexpandedParameterPackContext UPPC); + + /// If the given expression contains an unexpanded parameter + /// pack, diagnose the error. + /// + /// \param E The expression that is being checked for unexpanded + /// parameter packs. + /// + /// \returns true if an error occurred, false otherwise. + bool DiagnoseUnexpandedParameterPack( + Expr *E, UnexpandedParameterPackContext UPPC = UPPC_Expression); + + /// If the given requirees-expression contains an unexpanded reference to one + /// of its own parameter packs, diagnose the error. + /// + /// \param RE The requiress-expression that is being checked for unexpanded + /// parameter packs. + /// + /// \returns true if an error occurred, false otherwise. + bool DiagnoseUnexpandedParameterPackInRequiresExpr(RequiresExpr *RE); + + /// If the given nested-name-specifier contains an unexpanded + /// parameter pack, diagnose the error. + /// + /// \param SS The nested-name-specifier that is being checked for + /// unexpanded parameter packs. + /// + /// \returns true if an error occurred, false otherwise. + bool DiagnoseUnexpandedParameterPack(const CXXScopeSpec &SS, + UnexpandedParameterPackContext UPPC); + + /// If the given name contains an unexpanded parameter pack, + /// diagnose the error. + /// + /// \param NameInfo The name (with source location information) that + /// is being checked for unexpanded parameter packs. + /// + /// \returns true if an error occurred, false otherwise. + bool DiagnoseUnexpandedParameterPack(const DeclarationNameInfo &NameInfo, + UnexpandedParameterPackContext UPPC); + + /// If the given template name contains an unexpanded parameter pack, + /// diagnose the error. + /// + /// \param Loc The location of the template name. + /// + /// \param Template The template name that is being checked for unexpanded + /// parameter packs. + /// + /// \returns true if an error occurred, false otherwise. + bool DiagnoseUnexpandedParameterPack(SourceLocation Loc, + TemplateName Template, + UnexpandedParameterPackContext UPPC); + + /// If the given template argument contains an unexpanded parameter + /// pack, diagnose the error. + /// + /// \param Arg The template argument that is being checked for unexpanded + /// parameter packs. + /// + /// \returns true if an error occurred, false otherwise. + bool DiagnoseUnexpandedParameterPack(TemplateArgumentLoc Arg, + UnexpandedParameterPackContext UPPC); + + /// Collect the set of unexpanded parameter packs within the given + /// template argument. + /// + /// \param Arg The template argument that will be traversed to find + /// unexpanded parameter packs. + void collectUnexpandedParameterPacks( + TemplateArgument Arg, + SmallVectorImpl &Unexpanded); + + /// Collect the set of unexpanded parameter packs within the given + /// template argument. + /// + /// \param Arg The template argument that will be traversed to find + /// unexpanded parameter packs. + void collectUnexpandedParameterPacks( + TemplateArgumentLoc Arg, + SmallVectorImpl &Unexpanded); + + /// Collect the set of unexpanded parameter packs within the given + /// type. + /// + /// \param T The type that will be traversed to find + /// unexpanded parameter packs. + void collectUnexpandedParameterPacks( + QualType T, SmallVectorImpl &Unexpanded); + + /// Collect the set of unexpanded parameter packs within the given + /// type. + /// + /// \param TL The type that will be traversed to find + /// unexpanded parameter packs. + void collectUnexpandedParameterPacks( + TypeLoc TL, SmallVectorImpl &Unexpanded); + + /// Collect the set of unexpanded parameter packs within the given + /// nested-name-specifier. + /// + /// \param NNS The nested-name-specifier that will be traversed to find + /// unexpanded parameter packs. + void collectUnexpandedParameterPacks( + NestedNameSpecifierLoc NNS, + SmallVectorImpl &Unexpanded); + + /// Collect the set of unexpanded parameter packs within the given + /// name. + /// + /// \param NameInfo The name that will be traversed to find + /// unexpanded parameter packs. + void collectUnexpandedParameterPacks( + const DeclarationNameInfo &NameInfo, + SmallVectorImpl &Unexpanded); + + /// Invoked when parsing a template argument followed by an + /// ellipsis, which creates a pack expansion. + /// + /// \param Arg The template argument preceding the ellipsis, which + /// may already be invalid. + /// + /// \param EllipsisLoc The location of the ellipsis. + ParsedTemplateArgument ActOnPackExpansion(const ParsedTemplateArgument &Arg, + SourceLocation EllipsisLoc); + + /// Invoked when parsing a type followed by an ellipsis, which + /// creates a pack expansion. + /// + /// \param Type The type preceding the ellipsis, which will become + /// the pattern of the pack expansion. + /// + /// \param EllipsisLoc The location of the ellipsis. + TypeResult ActOnPackExpansion(ParsedType Type, SourceLocation EllipsisLoc); + + /// Construct a pack expansion type from the pattern of the pack + /// expansion. + TypeSourceInfo *CheckPackExpansion(TypeSourceInfo *Pattern, + SourceLocation EllipsisLoc, + std::optional NumExpansions); + + /// Construct a pack expansion type from the pattern of the pack + /// expansion. + QualType CheckPackExpansion(QualType Pattern, SourceRange PatternRange, + SourceLocation EllipsisLoc, + std::optional NumExpansions); + + /// Invoked when parsing an expression followed by an ellipsis, which + /// creates a pack expansion. + /// + /// \param Pattern The expression preceding the ellipsis, which will become + /// the pattern of the pack expansion. + /// + /// \param EllipsisLoc The location of the ellipsis. + ExprResult ActOnPackExpansion(Expr *Pattern, SourceLocation EllipsisLoc); + + /// Invoked when parsing an expression followed by an ellipsis, which + /// creates a pack expansion. + /// + /// \param Pattern The expression preceding the ellipsis, which will become + /// the pattern of the pack expansion. + /// + /// \param EllipsisLoc The location of the ellipsis. + ExprResult CheckPackExpansion(Expr *Pattern, SourceLocation EllipsisLoc, + std::optional NumExpansions); + + /// Determine whether we could expand a pack expansion with the + /// given set of parameter packs into separate arguments by repeatedly + /// transforming the pattern. + /// + /// \param EllipsisLoc The location of the ellipsis that identifies the + /// pack expansion. + /// + /// \param PatternRange The source range that covers the entire pattern of + /// the pack expansion. + /// + /// \param Unexpanded The set of unexpanded parameter packs within the + /// pattern. + /// + /// \param ShouldExpand Will be set to \c true if the transformer should + /// expand the corresponding pack expansions into separate arguments. When + /// set, \c NumExpansions must also be set. + /// + /// \param RetainExpansion Whether the caller should add an unexpanded + /// pack expansion after all of the expanded arguments. This is used + /// when extending explicitly-specified template argument packs per + /// C++0x [temp.arg.explicit]p9. + /// + /// \param NumExpansions The number of separate arguments that will be in + /// the expanded form of the corresponding pack expansion. This is both an + /// input and an output parameter, which can be set by the caller if the + /// number of expansions is known a priori (e.g., due to a prior substitution) + /// and will be set by the callee when the number of expansions is known. + /// The callee must set this value when \c ShouldExpand is \c true; it may + /// set this value in other cases. + /// + /// \returns true if an error occurred (e.g., because the parameter packs + /// are to be instantiated with arguments of different lengths), false + /// otherwise. If false, \c ShouldExpand (and possibly \c NumExpansions) + /// must be set. + bool CheckParameterPacksForExpansion( + SourceLocation EllipsisLoc, SourceRange PatternRange, + ArrayRef Unexpanded, + const MultiLevelTemplateArgumentList &TemplateArgs, bool &ShouldExpand, + bool &RetainExpansion, std::optional &NumExpansions); + + /// Determine the number of arguments in the given pack expansion + /// type. + /// + /// This routine assumes that the number of arguments in the expansion is + /// consistent across all of the unexpanded parameter packs in its pattern. + /// + /// Returns an empty Optional if the type can't be expanded. + std::optional getNumArgumentsInExpansion( + QualType T, const MultiLevelTemplateArgumentList &TemplateArgs); + + /// Determine whether the given declarator contains any unexpanded + /// parameter packs. + /// + /// This routine is used by the parser to disambiguate function declarators + /// with an ellipsis prior to the ')', e.g., + /// + /// \code + /// void f(T...); + /// \endcode + /// + /// To determine whether we have an (unnamed) function parameter pack or + /// a variadic function. + /// + /// \returns true if the declarator contains any unexpanded parameter packs, + /// false otherwise. + bool containsUnexpandedParameterPacks(Declarator &D); + + /// Returns the pattern of the pack expansion for a template argument. + /// + /// \param OrigLoc The template argument to expand. + /// + /// \param Ellipsis Will be set to the location of the ellipsis. + /// + /// \param NumExpansions Will be set to the number of expansions that will + /// be generated from this pack expansion, if known a priori. + TemplateArgumentLoc getTemplateArgumentPackExpansionPattern( + TemplateArgumentLoc OrigLoc, SourceLocation &Ellipsis, + std::optional &NumExpansions) const; + + /// Given a template argument that contains an unexpanded parameter pack, but + /// which has already been substituted, attempt to determine the number of + /// elements that will be produced once this argument is fully-expanded. + /// + /// This is intended for use when transforming 'sizeof...(Arg)' in order to + /// avoid actually expanding the pack where possible. + std::optional getFullyPackExpandedSize(TemplateArgument Arg); + + ExprResult ActOnSizeofParameterPackExpr(Scope *S, SourceLocation OpLoc, + IdentifierInfo &Name, + SourceLocation NameLoc, + SourceLocation RParenLoc); + + ExprResult ActOnPackIndexingExpr(Scope *S, Expr *PackExpression, + SourceLocation EllipsisLoc, + SourceLocation LSquareLoc, Expr *IndexExpr, + SourceLocation RSquareLoc); + + ExprResult BuildPackIndexingExpr(Expr *PackExpression, + SourceLocation EllipsisLoc, Expr *IndexExpr, + SourceLocation RSquareLoc, + ArrayRef ExpandedExprs = {}, + bool EmptyPack = false); + + /// Handle a C++1z fold-expression: ( expr op ... op expr ). + ExprResult ActOnCXXFoldExpr(Scope *S, SourceLocation LParenLoc, Expr *LHS, + tok::TokenKind Operator, + SourceLocation EllipsisLoc, Expr *RHS, + SourceLocation RParenLoc); + ExprResult BuildCXXFoldExpr(UnresolvedLookupExpr *Callee, + SourceLocation LParenLoc, Expr *LHS, + BinaryOperatorKind Operator, + SourceLocation EllipsisLoc, Expr *RHS, + SourceLocation RParenLoc, + std::optional NumExpansions); + ExprResult BuildEmptyCXXFoldExpr(SourceLocation EllipsisLoc, + BinaryOperatorKind Operator); + + ///@} + + // + // + // ------------------------------------------------------------------------- + // + // + + /// \name Constraints and Concepts + /// Implementations are in SemaConcept.cpp + ///@{ + +public: + void PushSatisfactionStackEntry(const NamedDecl *D, + const llvm::FoldingSetNodeID &ID) { + const NamedDecl *Can = cast(D->getCanonicalDecl()); + SatisfactionStack.emplace_back(Can, ID); + } + + void PopSatisfactionStackEntry() { SatisfactionStack.pop_back(); } + + bool SatisfactionStackContains(const NamedDecl *D, + const llvm::FoldingSetNodeID &ID) const { + const NamedDecl *Can = cast(D->getCanonicalDecl()); + return llvm::find(SatisfactionStack, SatisfactionStackEntryTy{Can, ID}) != + SatisfactionStack.end(); + } + + using SatisfactionStackEntryTy = + std::pair; + + // Resets the current SatisfactionStack for cases where we are instantiating + // constraints as a 'side effect' of normal instantiation in a way that is not + // indicative of recursive definition. + class SatisfactionStackResetRAII { + llvm::SmallVector BackupSatisfactionStack; + Sema &SemaRef; + + public: + SatisfactionStackResetRAII(Sema &S) : SemaRef(S) { + SemaRef.SwapSatisfactionStack(BackupSatisfactionStack); + } + + ~SatisfactionStackResetRAII() { + SemaRef.SwapSatisfactionStack(BackupSatisfactionStack); + } + }; + + void SwapSatisfactionStack( + llvm::SmallVectorImpl &NewSS) { + SatisfactionStack.swap(NewSS); + } + + /// Check whether the given expression is a valid constraint expression. + /// A diagnostic is emitted if it is not, false is returned, and + /// PossibleNonPrimary will be set to true if the failure might be due to a + /// non-primary expression being used as an atomic constraint. + bool CheckConstraintExpression(const Expr *CE, Token NextToken = Token(), + bool *PossibleNonPrimary = nullptr, + bool IsTrailingRequiresClause = false); + + /// \brief Check whether the given list of constraint expressions are + /// satisfied (as if in a 'conjunction') given template arguments. + /// \param Template the template-like entity that triggered the constraints + /// check (either a concept or a constrained entity). + /// \param ConstraintExprs a list of constraint expressions, treated as if + /// they were 'AND'ed together. + /// \param TemplateArgLists the list of template arguments to substitute into + /// the constraint expression. + /// \param TemplateIDRange The source range of the template id that + /// caused the constraints check. + /// \param Satisfaction if true is returned, will contain details of the + /// satisfaction, with enough information to diagnose an unsatisfied + /// expression. + /// \returns true if an error occurred and satisfaction could not be checked, + /// false otherwise. + bool CheckConstraintSatisfaction( + const NamedDecl *Template, ArrayRef ConstraintExprs, + const MultiLevelTemplateArgumentList &TemplateArgLists, + SourceRange TemplateIDRange, ConstraintSatisfaction &Satisfaction) { + llvm::SmallVector Converted; + return CheckConstraintSatisfaction(Template, ConstraintExprs, Converted, + TemplateArgLists, TemplateIDRange, + Satisfaction); + } + + /// \brief Check whether the given list of constraint expressions are + /// satisfied (as if in a 'conjunction') given template arguments. + /// Additionally, takes an empty list of Expressions which is populated with + /// the instantiated versions of the ConstraintExprs. + /// \param Template the template-like entity that triggered the constraints + /// check (either a concept or a constrained entity). + /// \param ConstraintExprs a list of constraint expressions, treated as if + /// they were 'AND'ed together. + /// \param ConvertedConstraints a out parameter that will get populated with + /// the instantiated version of the ConstraintExprs if we successfully checked + /// satisfaction. + /// \param TemplateArgList the multi-level list of template arguments to + /// substitute into the constraint expression. This should be relative to the + /// top-level (hence multi-level), since we need to instantiate fully at the + /// time of checking. + /// \param TemplateIDRange The source range of the template id that + /// caused the constraints check. + /// \param Satisfaction if true is returned, will contain details of the + /// satisfaction, with enough information to diagnose an unsatisfied + /// expression. + /// \returns true if an error occurred and satisfaction could not be checked, + /// false otherwise. + bool CheckConstraintSatisfaction( + const NamedDecl *Template, ArrayRef ConstraintExprs, + llvm::SmallVectorImpl &ConvertedConstraints, + const MultiLevelTemplateArgumentList &TemplateArgList, + SourceRange TemplateIDRange, ConstraintSatisfaction &Satisfaction); + + /// \brief Check whether the given non-dependent constraint expression is + /// satisfied. Returns false and updates Satisfaction with the satisfaction + /// verdict if successful, emits a diagnostic and returns true if an error + /// occurred and satisfaction could not be determined. + /// + /// \returns true if an error occurred, false otherwise. + bool CheckConstraintSatisfaction(const Expr *ConstraintExpr, + ConstraintSatisfaction &Satisfaction); + + /// Check whether the given function decl's trailing requires clause is + /// satisfied, if any. Returns false and updates Satisfaction with the + /// satisfaction verdict if successful, emits a diagnostic and returns true if + /// an error occurred and satisfaction could not be determined. + /// + /// \returns true if an error occurred, false otherwise. + bool CheckFunctionConstraints(const FunctionDecl *FD, + ConstraintSatisfaction &Satisfaction, + SourceLocation UsageLoc = SourceLocation(), + bool ForOverloadResolution = false); + + // Calculates whether two constraint expressions are equal irrespective of a + // difference in 'depth'. This takes a pair of optional 'NamedDecl's 'Old' and + // 'New', which are the "source" of the constraint, since this is necessary + // for figuring out the relative 'depth' of the constraint. The depth of the + // 'primary template' and the 'instantiated from' templates aren't necessarily + // the same, such as a case when one is a 'friend' defined in a class. + bool AreConstraintExpressionsEqual(const NamedDecl *Old, + const Expr *OldConstr, + const TemplateCompareNewDeclInfo &New, + const Expr *NewConstr); + + // Calculates whether the friend function depends on an enclosing template for + // the purposes of [temp.friend] p9. + bool FriendConstraintsDependOnEnclosingTemplate(const FunctionDecl *FD); + + /// \brief Ensure that the given template arguments satisfy the constraints + /// associated with the given template, emitting a diagnostic if they do not. + /// + /// \param Template The template to which the template arguments are being + /// provided. + /// + /// \param TemplateArgs The converted, canonicalized template arguments. + /// + /// \param TemplateIDRange The source range of the template id that + /// caused the constraints check. + /// + /// \returns true if the constrains are not satisfied or could not be checked + /// for satisfaction, false if the constraints are satisfied. + bool EnsureTemplateArgumentListConstraints( + TemplateDecl *Template, + const MultiLevelTemplateArgumentList &TemplateArgs, + SourceRange TemplateIDRange); + + bool CheckInstantiatedFunctionTemplateConstraints( + SourceLocation PointOfInstantiation, FunctionDecl *Decl, + ArrayRef TemplateArgs, + ConstraintSatisfaction &Satisfaction); + + /// \brief Emit diagnostics explaining why a constraint expression was deemed + /// unsatisfied. + /// \param First whether this is the first time an unsatisfied constraint is + /// diagnosed for this error. + void DiagnoseUnsatisfiedConstraint(const ConstraintSatisfaction &Satisfaction, + bool First = true); + + /// \brief Emit diagnostics explaining why a constraint expression was deemed + /// unsatisfied. + void + DiagnoseUnsatisfiedConstraint(const ASTConstraintSatisfaction &Satisfaction, + bool First = true); + + const NormalizedConstraint *getNormalizedAssociatedConstraints( + NamedDecl *ConstrainedDecl, ArrayRef AssociatedConstraints); + + /// \brief Check whether the given declaration's associated constraints are + /// at least as constrained than another declaration's according to the + /// partial ordering of constraints. + /// + /// \param Result If no error occurred, receives the result of true if D1 is + /// at least constrained than D2, and false otherwise. + /// + /// \returns true if an error occurred, false otherwise. + bool IsAtLeastAsConstrained(NamedDecl *D1, MutableArrayRef AC1, + NamedDecl *D2, MutableArrayRef AC2, + bool &Result); + + /// If D1 was not at least as constrained as D2, but would've been if a pair + /// of atomic constraints involved had been declared in a concept and not + /// repeated in two separate places in code. + /// \returns true if such a diagnostic was emitted, false otherwise. + bool MaybeEmitAmbiguousAtomicConstraintsDiagnostic( + NamedDecl *D1, ArrayRef AC1, NamedDecl *D2, + ArrayRef AC2); + +private: + /// Caches pairs of template-like decls whose associated constraints were + /// checked for subsumption and whether or not the first's constraints did in + /// fact subsume the second's. + llvm::DenseMap, bool> SubsumptionCache; + /// Caches the normalized associated constraints of declarations (concepts or + /// constrained declarations). If an error occurred while normalizing the + /// associated constraints of the template or concept, nullptr will be cached + /// here. + llvm::DenseMap NormalizationCache; + + llvm::ContextualFoldingSet + SatisfactionCache; + + // The current stack of constraint satisfactions, so we can exit-early. + llvm::SmallVector SatisfactionStack; + + /// Introduce the instantiated captures of the lambda into the local + /// instantiation scope. + bool addInstantiatedCapturesToScope( + FunctionDecl *Function, const FunctionDecl *PatternDecl, + LocalInstantiationScope &Scope, + const MultiLevelTemplateArgumentList &TemplateArgs); + + /// used by SetupConstraintCheckingTemplateArgumentsAndScope to recursively(in + /// the case of lambdas) set up the LocalInstantiationScope of the current + /// function. + bool SetupConstraintScope( + FunctionDecl *FD, std::optional> TemplateArgs, + MultiLevelTemplateArgumentList MLTAL, LocalInstantiationScope &Scope); + + /// Used during constraint checking, sets up the constraint template argument + /// lists, and calls SetupConstraintScope to set up the + /// LocalInstantiationScope to have the proper set of ParVarDecls configured. + std::optional + SetupConstraintCheckingTemplateArgumentsAndScope( + FunctionDecl *FD, std::optional> TemplateArgs, + LocalInstantiationScope &Scope); + + ///@} + + // + // + // ------------------------------------------------------------------------- + // + // + + /// \name Types + /// Implementations are in SemaType.cpp + ///@{ + +public: + /// A mapping that describes the nullability we've seen in each header file. + FileNullabilityMap NullabilityMap; + + static int getPrintable(int I) { return I; } + static unsigned getPrintable(unsigned I) { return I; } + static bool getPrintable(bool B) { return B; } + static const char *getPrintable(const char *S) { return S; } + static StringRef getPrintable(StringRef S) { return S; } + static const std::string &getPrintable(const std::string &S) { return S; } + static const IdentifierInfo *getPrintable(const IdentifierInfo *II) { + return II; + } + static DeclarationName getPrintable(DeclarationName N) { return N; } + static QualType getPrintable(QualType T) { return T; } + static SourceRange getPrintable(SourceRange R) { return R; } + static SourceRange getPrintable(SourceLocation L) { return L; } + static SourceRange getPrintable(const Expr *E) { return E->getSourceRange(); } + static SourceRange getPrintable(TypeLoc TL) { return TL.getSourceRange(); } + + enum class CompleteTypeKind { + /// Apply the normal rules for complete types. In particular, + /// treat all sizeless types as incomplete. + Normal, + + /// Relax the normal rules for complete types so that they include + /// sizeless built-in types. + AcceptSizeless, + + // FIXME: Eventually we should flip the default to Normal and opt in + // to AcceptSizeless rather than opt out of it. + Default = AcceptSizeless + }; + + /// Build a an Objective-C protocol-qualified 'id' type where no + /// base type was specified. + TypeResult actOnObjCProtocolQualifierType( + SourceLocation lAngleLoc, ArrayRef protocols, + ArrayRef protocolLocs, SourceLocation rAngleLoc); + + /// Build a specialized and/or protocol-qualified Objective-C type. + TypeResult actOnObjCTypeArgsAndProtocolQualifiers( + Scope *S, SourceLocation Loc, ParsedType BaseType, + SourceLocation TypeArgsLAngleLoc, ArrayRef TypeArgs, + SourceLocation TypeArgsRAngleLoc, SourceLocation ProtocolLAngleLoc, + ArrayRef Protocols, ArrayRef ProtocolLocs, + SourceLocation ProtocolRAngleLoc); - /// Called on well-formed '\#pragma omp loop' after parsing of the - /// associated statement. - StmtResult ActOnOpenMPGenericLoopDirective( - ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, - SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Build an Objective-C type parameter type. + QualType BuildObjCTypeParamType(const ObjCTypeParamDecl *Decl, + SourceLocation ProtocolLAngleLoc, + ArrayRef Protocols, + ArrayRef ProtocolLocs, + SourceLocation ProtocolRAngleLoc, + bool FailOnError = false); - /// Checks correctness of linear modifiers. - bool CheckOpenMPLinearModifier(OpenMPLinearClauseKind LinKind, - SourceLocation LinLoc); - /// Checks that the specified declaration matches requirements for the linear - /// decls. - bool CheckOpenMPLinearDecl(const ValueDecl *D, SourceLocation ELoc, - OpenMPLinearClauseKind LinKind, QualType Type, - bool IsDeclareSimd = false); + /// Build an Objective-C object pointer type. + QualType BuildObjCObjectType( + QualType BaseType, SourceLocation Loc, SourceLocation TypeArgsLAngleLoc, + ArrayRef TypeArgs, SourceLocation TypeArgsRAngleLoc, + SourceLocation ProtocolLAngleLoc, ArrayRef Protocols, + ArrayRef ProtocolLocs, SourceLocation ProtocolRAngleLoc, + bool FailOnError, bool Rebuilding); - /// Called on well-formed '\#pragma omp declare simd' after parsing of - /// the associated method/function. - DeclGroupPtrTy ActOnOpenMPDeclareSimdDirective( - DeclGroupPtrTy DG, OMPDeclareSimdDeclAttr::BranchStateTy BS, - Expr *Simdlen, ArrayRef Uniforms, ArrayRef Aligneds, - ArrayRef Alignments, ArrayRef Linears, - ArrayRef LinModifiers, ArrayRef Steps, SourceRange SR); + QualType BuildQualifiedType(QualType T, SourceLocation Loc, Qualifiers Qs, + const DeclSpec *DS = nullptr); + QualType BuildQualifiedType(QualType T, SourceLocation Loc, unsigned CVRA, + const DeclSpec *DS = nullptr); + QualType BuildPointerType(QualType T, SourceLocation Loc, + DeclarationName Entity); + QualType BuildReferenceType(QualType T, bool LValueRef, SourceLocation Loc, + DeclarationName Entity); + QualType BuildArrayType(QualType T, ArraySizeModifier ASM, Expr *ArraySize, + unsigned Quals, SourceRange Brackets, + DeclarationName Entity); + QualType BuildVectorType(QualType T, Expr *VecSize, SourceLocation AttrLoc); + QualType BuildExtVectorType(QualType T, Expr *ArraySize, + SourceLocation AttrLoc); + QualType BuildMatrixType(QualType T, Expr *NumRows, Expr *NumColumns, + SourceLocation AttrLoc); - /// Checks '\#pragma omp declare variant' variant function and original - /// functions after parsing of the associated method/function. - /// \param DG Function declaration to which declare variant directive is - /// applied to. - /// \param VariantRef Expression that references the variant function, which - /// must be used instead of the original one, specified in \p DG. - /// \param TI The trait info object representing the match clause. - /// \param NumAppendArgs The number of omp_interop_t arguments to account for - /// in checking. - /// \returns std::nullopt, if the function/variant function are not compatible - /// with the pragma, pair of original function/variant ref expression - /// otherwise. - std::optional> - checkOpenMPDeclareVariantFunction(DeclGroupPtrTy DG, Expr *VariantRef, - OMPTraitInfo &TI, unsigned NumAppendArgs, - SourceRange SR); + QualType BuildAddressSpaceAttr(QualType &T, LangAS ASIdx, Expr *AddrSpace, + SourceLocation AttrLoc); - /// Called on well-formed '\#pragma omp declare variant' after parsing of - /// the associated method/function. - /// \param FD Function declaration to which declare variant directive is - /// applied to. - /// \param VariantRef Expression that references the variant function, which - /// must be used instead of the original one, specified in \p DG. - /// \param TI The context traits associated with the function variant. - /// \param AdjustArgsNothing The list of 'nothing' arguments. - /// \param AdjustArgsNeedDevicePtr The list of 'need_device_ptr' arguments. - /// \param AppendArgs The list of 'append_args' arguments. - /// \param AdjustArgsLoc The Location of an 'adjust_args' clause. - /// \param AppendArgsLoc The Location of an 'append_args' clause. - /// \param SR The SourceRange of the 'declare variant' directive. - void ActOnOpenMPDeclareVariantDirective( - FunctionDecl *FD, Expr *VariantRef, OMPTraitInfo &TI, - ArrayRef AdjustArgsNothing, - ArrayRef AdjustArgsNeedDevicePtr, - ArrayRef AppendArgs, SourceLocation AdjustArgsLoc, - SourceLocation AppendArgsLoc, SourceRange SR); + /// Same as above, but constructs the AddressSpace index if not provided. + QualType BuildAddressSpaceAttr(QualType &T, Expr *AddrSpace, + SourceLocation AttrLoc); - OMPClause *ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind, - Expr *Expr, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'allocator' clause. - OMPClause *ActOnOpenMPAllocatorClause(Expr *Allocator, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'if' clause. - OMPClause *ActOnOpenMPIfClause(OpenMPDirectiveKind NameModifier, - Expr *Condition, SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation NameModifierLoc, - SourceLocation ColonLoc, - SourceLocation EndLoc); - /// Called on well-formed 'final' clause. - OMPClause *ActOnOpenMPFinalClause(Expr *Condition, SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'num_threads' clause. - OMPClause *ActOnOpenMPNumThreadsClause(Expr *NumThreads, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'align' clause. - OMPClause *ActOnOpenMPAlignClause(Expr *Alignment, SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'safelen' clause. - OMPClause *ActOnOpenMPSafelenClause(Expr *Length, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'simdlen' clause. - OMPClause *ActOnOpenMPSimdlenClause(Expr *Length, SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-form 'sizes' clause. - OMPClause *ActOnOpenMPSizesClause(ArrayRef SizeExprs, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-form 'full' clauses. - OMPClause *ActOnOpenMPFullClause(SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-form 'partial' clauses. - OMPClause *ActOnOpenMPPartialClause(Expr *FactorExpr, SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'collapse' clause. - OMPClause *ActOnOpenMPCollapseClause(Expr *NumForLoops, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'ordered' clause. - OMPClause * - ActOnOpenMPOrderedClause(SourceLocation StartLoc, SourceLocation EndLoc, - SourceLocation LParenLoc = SourceLocation(), - Expr *NumForLoops = nullptr); - /// Called on well-formed 'grainsize' clause. - OMPClause *ActOnOpenMPGrainsizeClause(OpenMPGrainsizeClauseModifier Modifier, - Expr *Size, SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation ModifierLoc, - SourceLocation EndLoc); - /// Called on well-formed 'num_tasks' clause. - OMPClause *ActOnOpenMPNumTasksClause(OpenMPNumTasksClauseModifier Modifier, - Expr *NumTasks, SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation ModifierLoc, - SourceLocation EndLoc); - /// Called on well-formed 'hint' clause. - OMPClause *ActOnOpenMPHintClause(Expr *Hint, SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'detach' clause. - OMPClause *ActOnOpenMPDetachClause(Expr *Evt, SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); + bool CheckQualifiedFunctionForTypeId(QualType T, SourceLocation Loc); + + bool CheckFunctionReturnType(QualType T, SourceLocation Loc); + + /// Build a function type. + /// + /// This routine checks the function type according to C++ rules and + /// under the assumption that the result type and parameter types have + /// just been instantiated from a template. It therefore duplicates + /// some of the behavior of GetTypeForDeclarator, but in a much + /// simpler form that is only suitable for this narrow use case. + /// + /// \param T The return type of the function. + /// + /// \param ParamTypes The parameter types of the function. This array + /// will be modified to account for adjustments to the types of the + /// function parameters. + /// + /// \param Loc The location of the entity whose type involves this + /// function type or, if there is no such entity, the location of the + /// type that will have function type. + /// + /// \param Entity The name of the entity that involves the function + /// type, if known. + /// + /// \param EPI Extra information about the function type. Usually this will + /// be taken from an existing function with the same prototype. + /// + /// \returns A suitable function type, if there are no errors. The + /// unqualified type will always be a FunctionProtoType. + /// Otherwise, returns a NULL type. + QualType BuildFunctionType(QualType T, MutableArrayRef ParamTypes, + SourceLocation Loc, DeclarationName Entity, + const FunctionProtoType::ExtProtoInfo &EPI); + + QualType BuildMemberPointerType(QualType T, QualType Class, + SourceLocation Loc, DeclarationName Entity); + QualType BuildBlockPointerType(QualType T, SourceLocation Loc, + DeclarationName Entity); + QualType BuildParenType(QualType T); + QualType BuildAtomicType(QualType T, SourceLocation Loc); + QualType BuildReadPipeType(QualType T, SourceLocation Loc); + QualType BuildWritePipeType(QualType T, SourceLocation Loc); + QualType BuildBitIntType(bool IsUnsigned, Expr *BitWidth, SourceLocation Loc); + + TypeSourceInfo *GetTypeForDeclarator(Declarator &D); + TypeSourceInfo *GetTypeForDeclaratorCast(Declarator &D, QualType FromTy); + + /// Package the given type and TSI into a ParsedType. + ParsedType CreateParsedType(QualType T, TypeSourceInfo *TInfo); + static QualType GetTypeFromParser(ParsedType Ty, + TypeSourceInfo **TInfo = nullptr); + + TypeResult ActOnTypeName(Declarator &D); + + /// The parser has parsed the context-sensitive type 'instancetype' + /// in an Objective-C message declaration. Return the appropriate type. + ParsedType ActOnObjCInstanceType(SourceLocation Loc); + + // Check whether the size of array element of type \p EltTy is a multiple of + // its alignment and return false if it isn't. + bool checkArrayElementAlignment(QualType EltTy, SourceLocation Loc); + + void + diagnoseIgnoredQualifiers(unsigned DiagID, unsigned Quals, + SourceLocation FallbackLoc, + SourceLocation ConstQualLoc = SourceLocation(), + SourceLocation VolatileQualLoc = SourceLocation(), + SourceLocation RestrictQualLoc = SourceLocation(), + SourceLocation AtomicQualLoc = SourceLocation(), + SourceLocation UnalignedQualLoc = SourceLocation()); + + /// Retrieve the keyword associated + IdentifierInfo *getNullabilityKeyword(NullabilityKind nullability); + + /// The struct behind the CFErrorRef pointer. + RecordDecl *CFError = nullptr; + bool isCFError(RecordDecl *D); + + /// Retrieve the identifier "NSError". + IdentifierInfo *getNSErrorIdent(); + + /// Adjust the calling convention of a method to be the ABI default if it + /// wasn't specified explicitly. This handles method types formed from + /// function type typedefs and typename template arguments. + void adjustMemberFunctionCC(QualType &T, bool HasThisPointer, + bool IsCtorOrDtor, SourceLocation Loc); + + // Check if there is an explicit attribute, but only look through parens. + // The intent is to look for an attribute on the current declarator, but not + // one that came from a typedef. + bool hasExplicitCallingConv(QualType T); + + /// Check whether a nullability type specifier can be added to the given + /// type through some means not written in source (e.g. API notes). + /// + /// \param Type The type to which the nullability specifier will be + /// added. On success, this type will be updated appropriately. + /// + /// \param Nullability The nullability specifier to add. + /// + /// \param DiagLoc The location to use for diagnostics. + /// + /// \param AllowArrayTypes Whether to accept nullability specifiers on an + /// array type (e.g., because it will decay to a pointer). + /// + /// \param OverrideExisting Whether to override an existing, locally-specified + /// nullability specifier rather than complaining about the conflict. + /// + /// \returns true if nullability cannot be applied, false otherwise. + bool CheckImplicitNullabilityTypeSpecifier(QualType &Type, + NullabilityKind Nullability, + SourceLocation DiagLoc, + bool AllowArrayTypes, + bool OverrideExisting); - OMPClause *ActOnOpenMPSimpleClause(OpenMPClauseKind Kind, - unsigned Argument, - SourceLocation ArgumentLoc, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'when' clause. - OMPClause *ActOnOpenMPWhenClause(OMPTraitInfo &TI, SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'default' clause. - OMPClause *ActOnOpenMPDefaultClause(llvm::omp::DefaultKind Kind, - SourceLocation KindLoc, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'proc_bind' clause. - OMPClause *ActOnOpenMPProcBindClause(llvm::omp::ProcBindKind Kind, - SourceLocation KindLoc, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'order' clause. - OMPClause *ActOnOpenMPOrderClause(OpenMPOrderClauseModifier Modifier, - OpenMPOrderClauseKind Kind, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation MLoc, SourceLocation KindLoc, - SourceLocation EndLoc); - /// Called on well-formed 'update' clause. - OMPClause *ActOnOpenMPUpdateClause(OpenMPDependClauseKind Kind, - SourceLocation KindLoc, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); + /// Get the type of expression E, triggering instantiation to complete the + /// type if necessary -- that is, if the expression refers to a templated + /// static data member of incomplete array type. + /// + /// May still return an incomplete type if instantiation was not possible or + /// if the type is incomplete for a different reason. Use + /// RequireCompleteExprType instead if a diagnostic is expected for an + /// incomplete expression type. + QualType getCompletedType(Expr *E); - OMPClause *ActOnOpenMPSingleExprWithArgClause( - OpenMPClauseKind Kind, ArrayRef Arguments, Expr *Expr, - SourceLocation StartLoc, SourceLocation LParenLoc, - ArrayRef ArgumentsLoc, SourceLocation DelimLoc, - SourceLocation EndLoc); - /// Called on well-formed 'schedule' clause. - OMPClause *ActOnOpenMPScheduleClause( - OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2, - OpenMPScheduleClauseKind Kind, Expr *ChunkSize, SourceLocation StartLoc, - SourceLocation LParenLoc, SourceLocation M1Loc, SourceLocation M2Loc, - SourceLocation KindLoc, SourceLocation CommaLoc, SourceLocation EndLoc); + void completeExprArrayBound(Expr *E); + bool RequireCompleteExprType(Expr *E, CompleteTypeKind Kind, + TypeDiagnoser &Diagnoser); + bool RequireCompleteExprType(Expr *E, unsigned DiagID); - OMPClause *ActOnOpenMPClause(OpenMPClauseKind Kind, SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed 'nowait' clause. - OMPClause *ActOnOpenMPNowaitClause(SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed 'untied' clause. - OMPClause *ActOnOpenMPUntiedClause(SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed 'mergeable' clause. - OMPClause *ActOnOpenMPMergeableClause(SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed 'read' clause. - OMPClause *ActOnOpenMPReadClause(SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed 'write' clause. - OMPClause *ActOnOpenMPWriteClause(SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed 'update' clause. - OMPClause *ActOnOpenMPUpdateClause(SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed 'capture' clause. - OMPClause *ActOnOpenMPCaptureClause(SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed 'compare' clause. - OMPClause *ActOnOpenMPCompareClause(SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed 'fail' clause. - OMPClause *ActOnOpenMPFailClause(SourceLocation StartLoc, - SourceLocation EndLoc); - OMPClause *ActOnOpenMPFailClause( - OpenMPClauseKind Kind, SourceLocation KindLoc, - SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation EndLoc); + template + bool RequireCompleteExprType(Expr *E, unsigned DiagID, const Ts &...Args) { + BoundTypeDiagnoser Diagnoser(DiagID, Args...); + return RequireCompleteExprType(E, CompleteTypeKind::Default, Diagnoser); + } - /// Called on well-formed 'seq_cst' clause. - OMPClause *ActOnOpenMPSeqCstClause(SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed 'acq_rel' clause. - OMPClause *ActOnOpenMPAcqRelClause(SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed 'acquire' clause. - OMPClause *ActOnOpenMPAcquireClause(SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed 'release' clause. - OMPClause *ActOnOpenMPReleaseClause(SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed 'relaxed' clause. - OMPClause *ActOnOpenMPRelaxedClause(SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed 'weak' clause. - OMPClause *ActOnOpenMPWeakClause(SourceLocation StartLoc, - SourceLocation EndLoc); + QualType getElaboratedType(ElaboratedTypeKeyword Keyword, + const CXXScopeSpec &SS, QualType T, + TagDecl *OwnedTagDecl = nullptr); - /// Called on well-formed 'init' clause. - OMPClause * - ActOnOpenMPInitClause(Expr *InteropVar, OMPInteropInfo &InteropInfo, - SourceLocation StartLoc, SourceLocation LParenLoc, - SourceLocation VarLoc, SourceLocation EndLoc); + // Returns the underlying type of a decltype with the given expression. + QualType getDecltypeForExpr(Expr *E); - /// Called on well-formed 'use' clause. - OMPClause *ActOnOpenMPUseClause(Expr *InteropVar, SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation VarLoc, SourceLocation EndLoc); + QualType BuildTypeofExprType(Expr *E, TypeOfKind Kind); + /// If AsUnevaluated is false, E is treated as though it were an evaluated + /// context, such as when building a type for decltype(auto). + QualType BuildDecltypeType(Expr *E, bool AsUnevaluated = true); - /// Called on well-formed 'destroy' clause. - OMPClause *ActOnOpenMPDestroyClause(Expr *InteropVar, SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation VarLoc, - SourceLocation EndLoc); - /// Called on well-formed 'novariants' clause. - OMPClause *ActOnOpenMPNovariantsClause(Expr *Condition, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'nocontext' clause. - OMPClause *ActOnOpenMPNocontextClause(Expr *Condition, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'filter' clause. - OMPClause *ActOnOpenMPFilterClause(Expr *ThreadID, SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'threads' clause. - OMPClause *ActOnOpenMPThreadsClause(SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed 'simd' clause. - OMPClause *ActOnOpenMPSIMDClause(SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed 'nogroup' clause. - OMPClause *ActOnOpenMPNogroupClause(SourceLocation StartLoc, - SourceLocation EndLoc); - /// Called on well-formed 'unified_address' clause. - OMPClause *ActOnOpenMPUnifiedAddressClause(SourceLocation StartLoc, - SourceLocation EndLoc); + QualType ActOnPackIndexingType(QualType Pattern, Expr *IndexExpr, + SourceLocation Loc, + SourceLocation EllipsisLoc); + QualType BuildPackIndexingType(QualType Pattern, Expr *IndexExpr, + SourceLocation Loc, SourceLocation EllipsisLoc, + bool FullySubstituted = false, + ArrayRef Expansions = {}); - /// Called on well-formed 'unified_address' clause. - OMPClause *ActOnOpenMPUnifiedSharedMemoryClause(SourceLocation StartLoc, - SourceLocation EndLoc); + using UTTKind = UnaryTransformType::UTTKind; + QualType BuildUnaryTransformType(QualType BaseType, UTTKind UKind, + SourceLocation Loc); + QualType BuiltinEnumUnderlyingType(QualType BaseType, SourceLocation Loc); + QualType BuiltinAddPointer(QualType BaseType, SourceLocation Loc); + QualType BuiltinRemovePointer(QualType BaseType, SourceLocation Loc); + QualType BuiltinDecay(QualType BaseType, SourceLocation Loc); + QualType BuiltinAddReference(QualType BaseType, UTTKind UKind, + SourceLocation Loc); + QualType BuiltinRemoveExtent(QualType BaseType, UTTKind UKind, + SourceLocation Loc); + QualType BuiltinRemoveReference(QualType BaseType, UTTKind UKind, + SourceLocation Loc); + QualType BuiltinChangeCVRQualifiers(QualType BaseType, UTTKind UKind, + SourceLocation Loc); + QualType BuiltinChangeSignedness(QualType BaseType, UTTKind UKind, + SourceLocation Loc); + + bool RequireLiteralType(SourceLocation Loc, QualType T, + TypeDiagnoser &Diagnoser); + bool RequireLiteralType(SourceLocation Loc, QualType T, unsigned DiagID); + + template + bool RequireLiteralType(SourceLocation Loc, QualType T, unsigned DiagID, + const Ts &...Args) { + BoundTypeDiagnoser Diagnoser(DiagID, Args...); + return RequireLiteralType(Loc, T, Diagnoser); + } + + bool isCompleteType(SourceLocation Loc, QualType T, + CompleteTypeKind Kind = CompleteTypeKind::Default) { + return !RequireCompleteTypeImpl(Loc, T, Kind, nullptr); + } + bool RequireCompleteType(SourceLocation Loc, QualType T, + CompleteTypeKind Kind, TypeDiagnoser &Diagnoser); + bool RequireCompleteType(SourceLocation Loc, QualType T, + CompleteTypeKind Kind, unsigned DiagID); + + bool RequireCompleteType(SourceLocation Loc, QualType T, + TypeDiagnoser &Diagnoser) { + return RequireCompleteType(Loc, T, CompleteTypeKind::Default, Diagnoser); + } + bool RequireCompleteType(SourceLocation Loc, QualType T, unsigned DiagID) { + return RequireCompleteType(Loc, T, CompleteTypeKind::Default, DiagID); + } + + template + bool RequireCompleteType(SourceLocation Loc, QualType T, unsigned DiagID, + const Ts &...Args) { + BoundTypeDiagnoser Diagnoser(DiagID, Args...); + return RequireCompleteType(Loc, T, Diagnoser); + } + + /// Determine whether a declaration is visible to name lookup. + bool isVisible(const NamedDecl *D) { + return D->isUnconditionallyVisible() || + isAcceptableSlow(D, AcceptableKind::Visible); + } + + /// Determine whether a declaration is reachable. + bool isReachable(const NamedDecl *D) { + // All visible declarations are reachable. + return D->isUnconditionallyVisible() || + isAcceptableSlow(D, AcceptableKind::Reachable); + } + + /// Determine whether a declaration is acceptable (visible/reachable). + bool isAcceptable(const NamedDecl *D, AcceptableKind Kind) { + return Kind == AcceptableKind::Visible ? isVisible(D) : isReachable(D); + } - /// Called on well-formed 'reverse_offload' clause. - OMPClause *ActOnOpenMPReverseOffloadClause(SourceLocation StartLoc, - SourceLocation EndLoc); + /// Determine if \p D and \p Suggested have a structurally compatible + /// layout as described in C11 6.2.7/1. + bool hasStructuralCompatLayout(Decl *D, Decl *Suggested); - /// Called on well-formed 'dynamic_allocators' clause. - OMPClause *ActOnOpenMPDynamicAllocatorsClause(SourceLocation StartLoc, - SourceLocation EndLoc); + /// Determine if \p D has a visible definition. If not, suggest a declaration + /// that should be made visible to expose the definition. + bool hasVisibleDefinition(NamedDecl *D, NamedDecl **Suggested, + bool OnlyNeedComplete = false); + bool hasVisibleDefinition(const NamedDecl *D) { + NamedDecl *Hidden; + return hasVisibleDefinition(const_cast(D), &Hidden); + } - /// Called on well-formed 'atomic_default_mem_order' clause. - OMPClause *ActOnOpenMPAtomicDefaultMemOrderClause( - OpenMPAtomicDefaultMemOrderClauseKind Kind, SourceLocation KindLoc, - SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation EndLoc); + /// Determine if \p D has a reachable definition. If not, suggest a + /// declaration that should be made reachable to expose the definition. + bool hasReachableDefinition(NamedDecl *D, NamedDecl **Suggested, + bool OnlyNeedComplete = false); + bool hasReachableDefinition(NamedDecl *D) { + NamedDecl *Hidden; + return hasReachableDefinition(D, &Hidden); + } - /// Called on well-formed 'at' clause. - OMPClause *ActOnOpenMPAtClause(OpenMPAtClauseKind Kind, - SourceLocation KindLoc, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); + bool hasAcceptableDefinition(NamedDecl *D, NamedDecl **Suggested, + AcceptableKind Kind, + bool OnlyNeedComplete = false); + bool hasAcceptableDefinition(NamedDecl *D, AcceptableKind Kind) { + NamedDecl *Hidden; + return hasAcceptableDefinition(D, &Hidden, Kind); + } - /// Called on well-formed 'severity' clause. - OMPClause *ActOnOpenMPSeverityClause(OpenMPSeverityClauseKind Kind, - SourceLocation KindLoc, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); +private: + bool RequireCompleteTypeImpl(SourceLocation Loc, QualType T, + CompleteTypeKind Kind, TypeDiagnoser *Diagnoser); - /// Called on well-formed 'message' clause. - /// passing string for message. - OMPClause *ActOnOpenMPMessageClause(Expr *MS, SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); + /// Nullability type specifiers. + IdentifierInfo *Ident__Nonnull = nullptr; + IdentifierInfo *Ident__Nullable = nullptr; + IdentifierInfo *Ident__Nullable_result = nullptr; + IdentifierInfo *Ident__Null_unspecified = nullptr; - /// Data used for processing a list of variables in OpenMP clauses. - struct OpenMPVarListDataTy final { - Expr *DepModOrTailExpr = nullptr; - Expr *IteratorExpr = nullptr; - SourceLocation ColonLoc; - SourceLocation RLoc; - CXXScopeSpec ReductionOrMapperIdScopeSpec; - DeclarationNameInfo ReductionOrMapperId; - int ExtraModifier = -1; ///< Additional modifier for linear, map, depend or - ///< lastprivate clause. - SmallVector - MapTypeModifiers; - SmallVector - MapTypeModifiersLoc; - SmallVector - MotionModifiers; - SmallVector MotionModifiersLoc; - bool IsMapTypeImplicit = false; - SourceLocation ExtraModifierLoc; - SourceLocation OmpAllMemoryLoc; - SourceLocation - StepModifierLoc; /// 'step' modifier location for linear clause - }; + IdentifierInfo *Ident_NSError = nullptr; - OMPClause *ActOnOpenMPVarListClause(OpenMPClauseKind Kind, - ArrayRef Vars, - const OMPVarListLocTy &Locs, - OpenMPVarListDataTy &Data); - /// Called on well-formed 'inclusive' clause. - OMPClause *ActOnOpenMPInclusiveClause(ArrayRef VarList, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'exclusive' clause. - OMPClause *ActOnOpenMPExclusiveClause(ArrayRef VarList, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'allocate' clause. - OMPClause * - ActOnOpenMPAllocateClause(Expr *Allocator, ArrayRef VarList, - SourceLocation StartLoc, SourceLocation ColonLoc, - SourceLocation LParenLoc, SourceLocation EndLoc); - /// Called on well-formed 'private' clause. - OMPClause *ActOnOpenMPPrivateClause(ArrayRef VarList, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'firstprivate' clause. - OMPClause *ActOnOpenMPFirstprivateClause(ArrayRef VarList, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'lastprivate' clause. - OMPClause *ActOnOpenMPLastprivateClause( - ArrayRef VarList, OpenMPLastprivateModifier LPKind, - SourceLocation LPKindLoc, SourceLocation ColonLoc, - SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation EndLoc); - /// Called on well-formed 'shared' clause. - OMPClause *ActOnOpenMPSharedClause(ArrayRef VarList, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'reduction' clause. - OMPClause *ActOnOpenMPReductionClause( - ArrayRef VarList, OpenMPReductionClauseModifier Modifier, - SourceLocation StartLoc, SourceLocation LParenLoc, - SourceLocation ModifierLoc, SourceLocation ColonLoc, - SourceLocation EndLoc, CXXScopeSpec &ReductionIdScopeSpec, - const DeclarationNameInfo &ReductionId, - ArrayRef UnresolvedReductions = std::nullopt); - /// Called on well-formed 'task_reduction' clause. - OMPClause *ActOnOpenMPTaskReductionClause( - ArrayRef VarList, SourceLocation StartLoc, - SourceLocation LParenLoc, SourceLocation ColonLoc, SourceLocation EndLoc, - CXXScopeSpec &ReductionIdScopeSpec, - const DeclarationNameInfo &ReductionId, - ArrayRef UnresolvedReductions = std::nullopt); - /// Called on well-formed 'in_reduction' clause. - OMPClause *ActOnOpenMPInReductionClause( - ArrayRef VarList, SourceLocation StartLoc, - SourceLocation LParenLoc, SourceLocation ColonLoc, SourceLocation EndLoc, - CXXScopeSpec &ReductionIdScopeSpec, - const DeclarationNameInfo &ReductionId, - ArrayRef UnresolvedReductions = std::nullopt); - /// Called on well-formed 'linear' clause. - OMPClause *ActOnOpenMPLinearClause( - ArrayRef VarList, Expr *Step, SourceLocation StartLoc, - SourceLocation LParenLoc, OpenMPLinearClauseKind LinKind, - SourceLocation LinLoc, SourceLocation ColonLoc, - SourceLocation StepModifierLoc, SourceLocation EndLoc); - /// Called on well-formed 'aligned' clause. - OMPClause *ActOnOpenMPAlignedClause(ArrayRef VarList, - Expr *Alignment, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation ColonLoc, - SourceLocation EndLoc); - /// Called on well-formed 'copyin' clause. - OMPClause *ActOnOpenMPCopyinClause(ArrayRef VarList, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'copyprivate' clause. - OMPClause *ActOnOpenMPCopyprivateClause(ArrayRef VarList, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'flush' pseudo clause. - OMPClause *ActOnOpenMPFlushClause(ArrayRef VarList, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'depobj' pseudo clause. - OMPClause *ActOnOpenMPDepobjClause(Expr *Depobj, SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'depend' clause. - OMPClause *ActOnOpenMPDependClause(const OMPDependClause::DependDataTy &Data, - Expr *DepModifier, - ArrayRef VarList, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'device' clause. - OMPClause *ActOnOpenMPDeviceClause(OpenMPDeviceClauseModifier Modifier, - Expr *Device, SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation ModifierLoc, - SourceLocation EndLoc); - /// Called on well-formed 'map' clause. - OMPClause *ActOnOpenMPMapClause( - Expr *IteratorModifier, ArrayRef MapTypeModifiers, - ArrayRef MapTypeModifiersLoc, - CXXScopeSpec &MapperIdScopeSpec, DeclarationNameInfo &MapperId, - OpenMPMapClauseKind MapType, bool IsMapTypeImplicit, - SourceLocation MapLoc, SourceLocation ColonLoc, ArrayRef VarList, - const OMPVarListLocTy &Locs, bool NoDiagnose = false, - ArrayRef UnresolvedMappers = std::nullopt); - /// Called on well-formed 'num_teams' clause. - OMPClause *ActOnOpenMPNumTeamsClause(Expr *NumTeams, SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'thread_limit' clause. - OMPClause *ActOnOpenMPThreadLimitClause(Expr *ThreadLimit, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'priority' clause. - OMPClause *ActOnOpenMPPriorityClause(Expr *Priority, SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on well-formed 'dist_schedule' clause. - OMPClause *ActOnOpenMPDistScheduleClause( - OpenMPDistScheduleClauseKind Kind, Expr *ChunkSize, - SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation KindLoc, - SourceLocation CommaLoc, SourceLocation EndLoc); - /// Called on well-formed 'defaultmap' clause. - OMPClause *ActOnOpenMPDefaultmapClause( - OpenMPDefaultmapClauseModifier M, OpenMPDefaultmapClauseKind Kind, - SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation MLoc, - SourceLocation KindLoc, SourceLocation EndLoc); - /// Called on well-formed 'to' clause. - OMPClause * - ActOnOpenMPToClause(ArrayRef MotionModifiers, - ArrayRef MotionModifiersLoc, - CXXScopeSpec &MapperIdScopeSpec, - DeclarationNameInfo &MapperId, SourceLocation ColonLoc, - ArrayRef VarList, const OMPVarListLocTy &Locs, - ArrayRef UnresolvedMappers = std::nullopt); - /// Called on well-formed 'from' clause. - OMPClause * - ActOnOpenMPFromClause(ArrayRef MotionModifiers, - ArrayRef MotionModifiersLoc, - CXXScopeSpec &MapperIdScopeSpec, - DeclarationNameInfo &MapperId, SourceLocation ColonLoc, - ArrayRef VarList, const OMPVarListLocTy &Locs, - ArrayRef UnresolvedMappers = std::nullopt); - /// Called on well-formed 'use_device_ptr' clause. - OMPClause *ActOnOpenMPUseDevicePtrClause(ArrayRef VarList, - const OMPVarListLocTy &Locs); - /// Called on well-formed 'use_device_addr' clause. - OMPClause *ActOnOpenMPUseDeviceAddrClause(ArrayRef VarList, - const OMPVarListLocTy &Locs); - /// Called on well-formed 'is_device_ptr' clause. - OMPClause *ActOnOpenMPIsDevicePtrClause(ArrayRef VarList, - const OMPVarListLocTy &Locs); - /// Called on well-formed 'has_device_addr' clause. - OMPClause *ActOnOpenMPHasDeviceAddrClause(ArrayRef VarList, - const OMPVarListLocTy &Locs); - /// Called on well-formed 'nontemporal' clause. - OMPClause *ActOnOpenMPNontemporalClause(ArrayRef VarList, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); + ///@} + + // + // + // ------------------------------------------------------------------------- + // + // - /// Data for list of allocators. - struct UsesAllocatorsData { - /// Allocator. - Expr *Allocator = nullptr; - /// Allocator traits. - Expr *AllocatorTraits = nullptr; - /// Locations of '(' and ')' symbols. - SourceLocation LParenLoc, RParenLoc; + /// \name ObjC Declarations + /// Implementations are in SemaDeclObjC.cpp + ///@{ + +public: + enum ObjCSpecialMethodKind { + OSMK_None, + OSMK_Alloc, + OSMK_New, + OSMK_Copy, + OSMK_RetainingInit, + OSMK_NonRetainingInit }; - /// Called on well-formed 'uses_allocators' clause. - OMPClause *ActOnOpenMPUsesAllocatorClause(SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc, - ArrayRef Data); - /// Called on well-formed 'affinity' clause. - OMPClause *ActOnOpenMPAffinityClause(SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation ColonLoc, - SourceLocation EndLoc, Expr *Modifier, - ArrayRef Locators); - /// Called on a well-formed 'bind' clause. - OMPClause *ActOnOpenMPBindClause(OpenMPBindClauseKind Kind, - SourceLocation KindLoc, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); - /// Called on a well-formed 'ompx_dyn_cgroup_mem' clause. - OMPClause *ActOnOpenMPXDynCGroupMemClause(Expr *Size, SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); + /// Method selectors used in a \@selector expression. Used for implementation + /// of -Wselector. + llvm::MapVector ReferencedSelectors; - /// Called on well-formed 'doacross' clause. - OMPClause * - ActOnOpenMPDoacrossClause(OpenMPDoacrossClauseModifier DepType, - SourceLocation DepLoc, SourceLocation ColonLoc, - ArrayRef VarList, SourceLocation StartLoc, - SourceLocation LParenLoc, SourceLocation EndLoc); + class GlobalMethodPool { + public: + using Lists = std::pair; + using iterator = llvm::DenseMap::iterator; + iterator begin() { return Methods.begin(); } + iterator end() { return Methods.end(); } + iterator find(Selector Sel) { return Methods.find(Sel); } + std::pair insert(std::pair &&Val) { + return Methods.insert(Val); + } + int count(Selector Sel) const { return Methods.count(Sel); } + bool empty() const { return Methods.empty(); } - /// Called on a well-formed 'ompx_attribute' clause. - OMPClause *ActOnOpenMPXAttributeClause(ArrayRef Attrs, - SourceLocation StartLoc, - SourceLocation LParenLoc, - SourceLocation EndLoc); + private: + llvm::DenseMap Methods; + }; - /// Called on a well-formed 'ompx_bare' clause. - OMPClause *ActOnOpenMPXBareClause(SourceLocation StartLoc, - SourceLocation EndLoc); + /// Method Pool - allows efficient lookup when typechecking messages to "id". + /// We need to maintain a list, since selectors can have differing signatures + /// across classes. In Cocoa, this happens to be extremely uncommon (only 1% + /// of selectors are "overloaded"). + /// At the head of the list it is recorded whether there were 0, 1, or >= 2 + /// methods inside categories with a particular selector. + GlobalMethodPool MethodPool; - //===--------------------------------------------------------------------===// - // OpenACC directives and clauses. + /// Check ODR hashes for C/ObjC when merging types from modules. + /// Differently from C++, actually parse the body and reject in case + /// of a mismatch. + template ::value>> + bool ActOnDuplicateODRHashDefinition(T *Duplicate, T *Previous) { + if (Duplicate->getODRHash() != Previous->getODRHash()) + return false; - /// Called after parsing an OpenACC Clause so that it can be checked. - bool ActOnOpenACCClause(OpenACCClauseKind ClauseKind, - SourceLocation StartLoc); + // Make the previous decl visible. + makeMergedDefinitionVisible(Previous); + return true; + } - /// Called after the construct has been parsed, but clauses haven't been - /// parsed. This allows us to diagnose not-implemented, as well as set up any - /// state required for parsing the clauses. - void ActOnOpenACCConstruct(OpenACCDirectiveKind K, SourceLocation StartLoc); + typedef llvm::SmallPtrSet SelectorSet; - /// Called after the directive, including its clauses, have been parsed and - /// parsing has consumed the 'annot_pragma_openacc_end' token. This DOES - /// happen before any associated declarations or statements have been parsed. - /// This function is only called when we are parsing a 'statement' context. - bool ActOnStartOpenACCStmtDirective(OpenACCDirectiveKind K, - SourceLocation StartLoc); + enum MethodMatchStrategy { MMS_loose, MMS_strict }; - /// Called after the directive, including its clauses, have been parsed and - /// parsing has consumed the 'annot_pragma_openacc_end' token. This DOES - /// happen before any associated declarations or statements have been parsed. - /// This function is only called when we are parsing a 'Decl' context. - bool ActOnStartOpenACCDeclDirective(OpenACCDirectiveKind K, - SourceLocation StartLoc); - /// Called when we encounter an associated statement for our construct, this - /// should check legality of the statement as it appertains to this Construct. - StmtResult ActOnOpenACCAssociatedStmt(OpenACCDirectiveKind K, - StmtResult AssocStmt); + enum ObjCContainerKind { + OCK_None = -1, + OCK_Interface = 0, + OCK_Protocol, + OCK_Category, + OCK_ClassExtension, + OCK_Implementation, + OCK_CategoryImplementation + }; + ObjCContainerKind getObjCContainerKind() const; - /// Called after the directive has been completely parsed, including the - /// declaration group or associated statement. - StmtResult ActOnEndOpenACCStmtDirective(OpenACCDirectiveKind K, - SourceLocation StartLoc, - SourceLocation EndLoc, - StmtResult AssocStmt); - /// Called after the directive has been completely parsed, including the - /// declaration group or associated statement. - DeclGroupRef ActOnEndOpenACCDeclDirective(); + DeclResult actOnObjCTypeParam(Scope *S, ObjCTypeParamVariance variance, + SourceLocation varianceLoc, unsigned index, + IdentifierInfo *paramName, + SourceLocation paramLoc, + SourceLocation colonLoc, ParsedType typeBound); - /// The kind of conversion being performed. - enum CheckedConversionKind { - /// An implicit conversion. - CCK_ImplicitConversion, - /// A C-style cast. - CCK_CStyleCast, - /// A functional-style cast. - CCK_FunctionalCast, - /// A cast other than a C-style cast. - CCK_OtherCast, - /// A conversion for an operand of a builtin overloaded operator. - CCK_ForBuiltinOverloadedOp + ObjCTypeParamList *actOnObjCTypeParamList(Scope *S, SourceLocation lAngleLoc, + ArrayRef typeParams, + SourceLocation rAngleLoc); + void popObjCTypeParamList(Scope *S, ObjCTypeParamList *typeParamList); + + ObjCInterfaceDecl *ActOnStartClassInterface( + Scope *S, SourceLocation AtInterfaceLoc, IdentifierInfo *ClassName, + SourceLocation ClassLoc, ObjCTypeParamList *typeParamList, + IdentifierInfo *SuperName, SourceLocation SuperLoc, + ArrayRef SuperTypeArgs, SourceRange SuperTypeArgsRange, + Decl *const *ProtoRefs, unsigned NumProtoRefs, + const SourceLocation *ProtoLocs, SourceLocation EndProtoLoc, + const ParsedAttributesView &AttrList, SkipBodyInfo *SkipBody); + + void ActOnSuperClassOfClassInterface( + Scope *S, SourceLocation AtInterfaceLoc, ObjCInterfaceDecl *IDecl, + IdentifierInfo *ClassName, SourceLocation ClassLoc, + IdentifierInfo *SuperName, SourceLocation SuperLoc, + ArrayRef SuperTypeArgs, SourceRange SuperTypeArgsRange); + + void ActOnTypedefedProtocols(SmallVectorImpl &ProtocolRefs, + SmallVectorImpl &ProtocolLocs, + IdentifierInfo *SuperName, + SourceLocation SuperLoc); + + Decl *ActOnCompatibilityAlias(SourceLocation AtCompatibilityAliasLoc, + IdentifierInfo *AliasName, + SourceLocation AliasLocation, + IdentifierInfo *ClassName, + SourceLocation ClassLocation); + + bool CheckForwardProtocolDeclarationForCircularDependency( + IdentifierInfo *PName, SourceLocation &PLoc, SourceLocation PrevLoc, + const ObjCList &PList); + + ObjCProtocolDecl *ActOnStartProtocolInterface( + SourceLocation AtProtoInterfaceLoc, IdentifierInfo *ProtocolName, + SourceLocation ProtocolLoc, Decl *const *ProtoRefNames, + unsigned NumProtoRefs, const SourceLocation *ProtoLocs, + SourceLocation EndProtoLoc, const ParsedAttributesView &AttrList, + SkipBodyInfo *SkipBody); + + ObjCCategoryDecl *ActOnStartCategoryInterface( + SourceLocation AtInterfaceLoc, IdentifierInfo *ClassName, + SourceLocation ClassLoc, ObjCTypeParamList *typeParamList, + IdentifierInfo *CategoryName, SourceLocation CategoryLoc, + Decl *const *ProtoRefs, unsigned NumProtoRefs, + const SourceLocation *ProtoLocs, SourceLocation EndProtoLoc, + const ParsedAttributesView &AttrList); + + ObjCImplementationDecl *ActOnStartClassImplementation( + SourceLocation AtClassImplLoc, IdentifierInfo *ClassName, + SourceLocation ClassLoc, IdentifierInfo *SuperClassname, + SourceLocation SuperClassLoc, const ParsedAttributesView &AttrList); + + ObjCCategoryImplDecl *ActOnStartCategoryImplementation( + SourceLocation AtCatImplLoc, IdentifierInfo *ClassName, + SourceLocation ClassLoc, IdentifierInfo *CatName, SourceLocation CatLoc, + const ParsedAttributesView &AttrList); + + DeclGroupPtrTy ActOnFinishObjCImplementation(Decl *ObjCImpDecl, + ArrayRef Decls); + + DeclGroupPtrTy + ActOnForwardProtocolDeclaration(SourceLocation AtProtoclLoc, + ArrayRef IdentList, + const ParsedAttributesView &attrList); + + void FindProtocolDeclaration(bool WarnOnDeclarations, bool ForObjCContainer, + ArrayRef ProtocolId, + SmallVectorImpl &Protocols); + + void DiagnoseTypeArgsAndProtocols(IdentifierInfo *ProtocolId, + SourceLocation ProtocolLoc, + IdentifierInfo *TypeArgId, + SourceLocation TypeArgLoc, + bool SelectProtocolFirst = false); + + /// Given a list of identifiers (and their locations), resolve the + /// names to either Objective-C protocol qualifiers or type + /// arguments, as appropriate. + void actOnObjCTypeArgsOrProtocolQualifiers( + Scope *S, ParsedType baseType, SourceLocation lAngleLoc, + ArrayRef identifiers, + ArrayRef identifierLocs, SourceLocation rAngleLoc, + SourceLocation &typeArgsLAngleLoc, SmallVectorImpl &typeArgs, + SourceLocation &typeArgsRAngleLoc, SourceLocation &protocolLAngleLoc, + SmallVectorImpl &protocols, SourceLocation &protocolRAngleLoc, + bool warnOnIncompleteProtocols); + + void DiagnoseClassExtensionDupMethods(ObjCCategoryDecl *CAT, + ObjCInterfaceDecl *ID); + + Decl *ActOnAtEnd(Scope *S, SourceRange AtEnd, + ArrayRef allMethods = std::nullopt, + ArrayRef allTUVars = std::nullopt); + + struct ObjCArgInfo { + IdentifierInfo *Name; + SourceLocation NameLoc; + // The Type is null if no type was specified, and the DeclSpec is invalid + // in this case. + ParsedType Type; + ObjCDeclSpec DeclSpec; + + /// ArgAttrs - Attribute list for this argument. + ParsedAttributesView ArgAttrs; + }; + + Decl *ActOnMethodDeclaration( + Scope *S, + SourceLocation BeginLoc, // location of the + or -. + SourceLocation EndLoc, // location of the ; or {. + tok::TokenKind MethodType, ObjCDeclSpec &ReturnQT, ParsedType ReturnType, + ArrayRef SelectorLocs, Selector Sel, + // optional arguments. The number of types/arguments is obtained + // from the Sel.getNumArgs(). + ObjCArgInfo *ArgInfo, DeclaratorChunk::ParamInfo *CParamInfo, + unsigned CNumArgs, // c-style args + const ParsedAttributesView &AttrList, tok::ObjCKeywordKind MethodImplKind, + bool isVariadic, bool MethodDefinition); + + bool CheckARCMethodDecl(ObjCMethodDecl *method); + + bool checkInitMethod(ObjCMethodDecl *method, QualType receiverTypeIfCall); + + /// Check whether the given new method is a valid override of the + /// given overridden method, and set any properties that should be inherited. + void CheckObjCMethodOverride(ObjCMethodDecl *NewMethod, + const ObjCMethodDecl *Overridden); + + /// Describes the compatibility of a result type with its method. + enum ResultTypeCompatibilityKind { + RTC_Compatible, + RTC_Incompatible, + RTC_Unknown }; - static bool isCast(CheckedConversionKind CCK) { - return CCK == CCK_CStyleCast || CCK == CCK_FunctionalCast || - CCK == CCK_OtherCast; - } + void CheckObjCMethodDirectOverrides(ObjCMethodDecl *method, + ObjCMethodDecl *overridden); - /// ImpCastExprToType - If Expr is not of type 'Type', insert an implicit - /// cast. If there is already an implicit cast, merge into the existing one. - /// If isLvalue, the result of the cast is an lvalue. - ExprResult - ImpCastExprToType(Expr *E, QualType Type, CastKind CK, - ExprValueKind VK = VK_PRValue, - const CXXCastPath *BasePath = nullptr, - CheckedConversionKind CCK = CCK_ImplicitConversion); + void CheckObjCMethodOverrides(ObjCMethodDecl *ObjCMethod, + ObjCInterfaceDecl *CurrentClass, + ResultTypeCompatibilityKind RTC); - /// ScalarTypeToBooleanCastKind - Returns the cast kind corresponding - /// to the conversion from scalar type ScalarTy to the Boolean type. - static CastKind ScalarTypeToBooleanCastKind(QualType ScalarTy); + /// AddAnyMethodToGlobalPool - Add any method, instance or factory to global + /// pool. + void AddAnyMethodToGlobalPool(Decl *D); - /// IgnoredValueConversions - Given that an expression's result is - /// syntactically ignored, perform any conversions that are - /// required. - ExprResult IgnoredValueConversions(Expr *E); + void ActOnStartOfObjCMethodDef(Scope *S, Decl *D); + bool isObjCMethodDecl(Decl *D) { return D && isa(D); } - // UsualUnaryConversions - promotes integers (C99 6.3.1.1p2) and converts - // functions and arrays to their respective pointers (C99 6.3.2.1). - ExprResult UsualUnaryConversions(Expr *E); + /// CheckImplementationIvars - This routine checks if the instance variables + /// listed in the implelementation match those listed in the interface. + void CheckImplementationIvars(ObjCImplementationDecl *ImpDecl, + ObjCIvarDecl **Fields, unsigned nIvars, + SourceLocation Loc); - /// CallExprUnaryConversions - a special case of an unary conversion - /// performed on a function designator of a call expression. - ExprResult CallExprUnaryConversions(Expr *E); + void WarnConflictingTypedMethods(ObjCMethodDecl *Method, + ObjCMethodDecl *MethodDecl, + bool IsProtocolMethodDecl); - // DefaultFunctionArrayConversion - converts functions and arrays - // to their respective pointers (C99 6.3.2.1). - ExprResult DefaultFunctionArrayConversion(Expr *E, bool Diagnose = true); + void CheckConflictingOverridingMethod(ObjCMethodDecl *Method, + ObjCMethodDecl *Overridden, + bool IsProtocolMethodDecl); - // DefaultFunctionArrayLvalueConversion - converts functions and - // arrays to their respective pointers and performs the - // lvalue-to-rvalue conversion. - ExprResult DefaultFunctionArrayLvalueConversion(Expr *E, - bool Diagnose = true); + /// WarnExactTypedMethods - This routine issues a warning if method + /// implementation declaration matches exactly that of its declaration. + void WarnExactTypedMethods(ObjCMethodDecl *Method, ObjCMethodDecl *MethodDecl, + bool IsProtocolMethodDecl); - // DefaultLvalueConversion - performs lvalue-to-rvalue conversion on - // the operand. This function is a no-op if the operand has a function type - // or an array type. - ExprResult DefaultLvalueConversion(Expr *E); + /// MatchAllMethodDeclarations - Check methods declaraed in interface or + /// or protocol against those declared in their implementations. + void MatchAllMethodDeclarations( + const SelectorSet &InsMap, const SelectorSet &ClsMap, + SelectorSet &InsMapSeen, SelectorSet &ClsMapSeen, ObjCImplDecl *IMPDecl, + ObjCContainerDecl *IDecl, bool &IncompleteImpl, bool ImmediateClass, + bool WarnCategoryMethodImpl = false); - // DefaultArgumentPromotion (C99 6.5.2.2p6). Used for function calls that - // do not have a prototype. Integer promotions are performed on each - // argument, and arguments that have type float are promoted to double. - ExprResult DefaultArgumentPromotion(Expr *E); + /// CheckCategoryVsClassMethodMatches - Checks that methods implemented in + /// category matches with those implemented in its primary class and + /// warns each time an exact match is found. + void CheckCategoryVsClassMethodMatches(ObjCCategoryImplDecl *CatIMP); - /// If \p E is a prvalue denoting an unmaterialized temporary, materialize - /// it as an xvalue. In C++98, the result will still be a prvalue, because - /// we don't have xvalues there. - ExprResult TemporaryMaterializationConversion(Expr *E); + /// ImplMethodsVsClassMethods - This is main routine to warn if any method + /// remains unimplemented in the class or category \@implementation. + void ImplMethodsVsClassMethods(Scope *S, ObjCImplDecl *IMPDecl, + ObjCContainerDecl *IDecl, + bool IncompleteImpl = false); - // Used for emitting the right warning by DefaultVariadicArgumentPromotion - enum VariadicCallType { - VariadicFunction, - VariadicBlock, - VariadicMethod, - VariadicConstructor, - VariadicDoesNotApply - }; + DeclGroupPtrTy ActOnForwardClassDeclaration( + SourceLocation Loc, IdentifierInfo **IdentList, SourceLocation *IdentLocs, + ArrayRef TypeParamLists, unsigned NumElts); - VariadicCallType getVariadicCallType(FunctionDecl *FDecl, - const FunctionProtoType *Proto, - Expr *Fn); + /// MatchTwoMethodDeclarations - Checks if two methods' type match and returns + /// true, or false, accordingly. + bool MatchTwoMethodDeclarations(const ObjCMethodDecl *Method, + const ObjCMethodDecl *PrevMethod, + MethodMatchStrategy strategy = MMS_strict); - // Used for determining in which context a type is allowed to be passed to a - // vararg function. - enum VarArgKind { - VAK_Valid, - VAK_ValidInCXX11, - VAK_Undefined, - VAK_MSVCUndefined, - VAK_Invalid - }; + /// Add the given method to the list of globally-known methods. + void addMethodToGlobalList(ObjCMethodList *List, ObjCMethodDecl *Method); - // Determines which VarArgKind fits an expression. - VarArgKind isValidVarArgType(const QualType &Ty); + void ReadMethodPool(Selector Sel); + void updateOutOfDateSelector(Selector Sel); - /// Check to see if the given expression is a valid argument to a variadic - /// function, issuing a diagnostic if not. - void checkVariadicArgument(const Expr *E, VariadicCallType CT); + /// - Returns instance or factory methods in global method pool for + /// given selector. It checks the desired kind first, if none is found, and + /// parameter checkTheOther is set, it then checks the other kind. If no such + /// method or only one method is found, function returns false; otherwise, it + /// returns true. + bool + CollectMultipleMethodsInGlobalPool(Selector Sel, + SmallVectorImpl &Methods, + bool InstanceFirst, bool CheckTheOther, + const ObjCObjectType *TypeBound = nullptr); - /// Check whether the given statement can have musttail applied to it, - /// issuing a diagnostic and returning false if not. In the success case, - /// the statement is rewritten to remove implicit nodes from the return - /// value. - bool checkAndRewriteMustTailAttr(Stmt *St, const Attr &MTA); + bool + AreMultipleMethodsInGlobalPool(Selector Sel, ObjCMethodDecl *BestMethod, + SourceRange R, bool receiverIdOrClass, + SmallVectorImpl &Methods); -private: - /// Check whether the given statement can have musttail applied to it, - /// issuing a diagnostic and returning false if not. - bool checkMustTailAttr(const Stmt *St, const Attr &MTA); + void + DiagnoseMultipleMethodInGlobalPool(SmallVectorImpl &Methods, + Selector Sel, SourceRange R, + bool receiverIdOrClass); -public: - /// Check to see if a given expression could have '.c_str()' called on it. - bool hasCStrMethod(const Expr *E); + const ObjCMethodDecl * + SelectorsForTypoCorrection(Selector Sel, QualType ObjectType = QualType()); + /// LookupImplementedMethodInGlobalPool - Returns the method which has an + /// implementation. + ObjCMethodDecl *LookupImplementedMethodInGlobalPool(Selector Sel); - /// GatherArgumentsForCall - Collector argument expressions for various - /// form of call prototypes. - bool GatherArgumentsForCall(SourceLocation CallLoc, FunctionDecl *FDecl, - const FunctionProtoType *Proto, - unsigned FirstParam, ArrayRef Args, - SmallVectorImpl &AllArgs, - VariadicCallType CallType = VariadicDoesNotApply, - bool AllowExplicit = false, - bool IsListInitialization = false); + void DiagnoseDuplicateIvars(ObjCInterfaceDecl *ID, ObjCInterfaceDecl *SID); - // DefaultVariadicArgumentPromotion - Like DefaultArgumentPromotion, but - // will create a runtime trap if the resulting type is not a POD type. - ExprResult DefaultVariadicArgumentPromotion(Expr *E, VariadicCallType CT, - FunctionDecl *FDecl); + /// Checks that the Objective-C declaration is declared in the global scope. + /// Emits an error and marks the declaration as invalid if it's not declared + /// in the global scope. + bool CheckObjCDeclScope(Decl *D); - /// Context in which we're performing a usual arithmetic conversion. - enum ArithConvKind { - /// An arithmetic operation. - ACK_Arithmetic, - /// A bitwise operation. - ACK_BitwiseOp, - /// A comparison. - ACK_Comparison, - /// A conditional (?:) operator. - ACK_Conditional, - /// A compound assignment expression. - ACK_CompAssign, - }; + void ActOnDefs(Scope *S, Decl *TagD, SourceLocation DeclStart, + IdentifierInfo *ClassName, SmallVectorImpl &Decls); - // UsualArithmeticConversions - performs the UsualUnaryConversions on it's - // operands and then handles various conversions that are common to binary - // operators (C99 6.3.1.8). If both operands aren't arithmetic, this - // routine returns the first non-arithmetic type found. The client is - // responsible for emitting appropriate error diagnostics. - QualType UsualArithmeticConversions(ExprResult &LHS, ExprResult &RHS, - SourceLocation Loc, ArithConvKind ACK); + VarDecl *BuildObjCExceptionDecl(TypeSourceInfo *TInfo, QualType ExceptionType, + SourceLocation StartLoc, SourceLocation IdLoc, + IdentifierInfo *Id, bool Invalid = false); - /// AssignConvertType - All of the 'assignment' semantic checks return this - /// enum to indicate whether the assignment was allowed. These checks are - /// done for simple assignments, as well as initialization, return from - /// function, argument passing, etc. The query is phrased in terms of a - /// source and destination type. - enum AssignConvertType { - /// Compatible - the types are compatible according to the standard. - Compatible, + Decl *ActOnObjCExceptionDecl(Scope *S, Declarator &D); - /// PointerToInt - The assignment converts a pointer to an int, which we - /// accept as an extension. - PointerToInt, + /// CollectIvarsToConstructOrDestruct - Collect those ivars which require + /// initialization. + void + CollectIvarsToConstructOrDestruct(ObjCInterfaceDecl *OI, + SmallVectorImpl &Ivars); - /// IntToPointer - The assignment converts an int to a pointer, which we - /// accept as an extension. - IntToPointer, + void DiagnoseUseOfUnimplementedSelectors(); - /// FunctionVoidPointer - The assignment is between a function pointer and - /// void*, which the standard doesn't allow, but we accept as an extension. - FunctionVoidPointer, + /// DiagnoseUnusedBackingIvarInAccessor - Issue an 'unused' warning if ivar + /// which backs the property is not used in the property's accessor. + void DiagnoseUnusedBackingIvarInAccessor(Scope *S, + const ObjCImplementationDecl *ImplD); - /// IncompatiblePointer - The assignment is between two pointers types that - /// are not compatible, but we accept them as an extension. - IncompatiblePointer, + /// GetIvarBackingPropertyAccessor - If method is a property setter/getter and + /// it property has a backing ivar, returns this ivar; otherwise, returns + /// NULL. It also returns ivar's property on success. + ObjCIvarDecl * + GetIvarBackingPropertyAccessor(const ObjCMethodDecl *Method, + const ObjCPropertyDecl *&PDecl) const; - /// IncompatibleFunctionPointer - The assignment is between two function - /// pointers types that are not compatible, but we accept them as an - /// extension. - IncompatibleFunctionPointer, + /// AddInstanceMethodToGlobalPool - All instance methods in a translation + /// unit are added to a global pool. This allows us to efficiently associate + /// a selector with a method declaraation for purposes of typechecking + /// messages sent to "id" (where the class of the object is unknown). + void AddInstanceMethodToGlobalPool(ObjCMethodDecl *Method, + bool impl = false) { + AddMethodToGlobalPool(Method, impl, /*instance*/ true); + } - /// IncompatibleFunctionPointerStrict - The assignment is between two - /// function pointer types that are not identical, but are compatible, - /// unless compiled with -fsanitize=cfi, in which case the type mismatch - /// may trip an indirect call runtime check. - IncompatibleFunctionPointerStrict, + /// AddFactoryMethodToGlobalPool - Same as above, but for factory methods. + void AddFactoryMethodToGlobalPool(ObjCMethodDecl *Method, bool impl = false) { + AddMethodToGlobalPool(Method, impl, /*instance*/ false); + } - /// IncompatiblePointerSign - The assignment is between two pointers types - /// which point to integers which have a different sign, but are otherwise - /// identical. This is a subset of the above, but broken out because it's by - /// far the most common case of incompatible pointers. - IncompatiblePointerSign, +private: + /// AddMethodToGlobalPool - Add an instance or factory method to the global + /// pool. See descriptoin of AddInstanceMethodToGlobalPool. + void AddMethodToGlobalPool(ObjCMethodDecl *Method, bool impl, bool instance); - /// CompatiblePointerDiscardsQualifiers - The assignment discards - /// c/v/r qualifiers, which we accept as an extension. - CompatiblePointerDiscardsQualifiers, + /// LookupMethodInGlobalPool - Returns the instance or factory method and + /// optionally warns if there are multiple signatures. + ObjCMethodDecl *LookupMethodInGlobalPool(Selector Sel, SourceRange R, + bool receiverIdOrClass, + bool instance); + + ///@} + + // + // + // ------------------------------------------------------------------------- + // + // - /// IncompatiblePointerDiscardsQualifiers - The assignment - /// discards qualifiers that we don't permit to be discarded, - /// like address spaces. - IncompatiblePointerDiscardsQualifiers, + /// \name ObjC Expressions + /// Implementations are in SemaExprObjC.cpp + ///@{ - /// IncompatibleNestedPointerAddressSpaceMismatch - The assignment - /// changes address spaces in nested pointer types which is not allowed. - /// For instance, converting __private int ** to __generic int ** is - /// illegal even though __private could be converted to __generic. - IncompatibleNestedPointerAddressSpaceMismatch, +public: + /// Caches identifiers/selectors for NSFoundation APIs. + std::unique_ptr NSAPIObj; - /// IncompatibleNestedPointerQualifiers - The assignment is between two - /// nested pointer types, and the qualifiers other than the first two - /// levels differ e.g. char ** -> const char **, but we accept them as an - /// extension. - IncompatibleNestedPointerQualifiers, + /// The declaration of the Objective-C NSNumber class. + ObjCInterfaceDecl *NSNumberDecl; - /// IncompatibleVectors - The assignment is between two vector types that - /// have the same size, which we accept as an extension. - IncompatibleVectors, + /// The declaration of the Objective-C NSValue class. + ObjCInterfaceDecl *NSValueDecl; - /// IntToBlockPointer - The assignment converts an int to a block - /// pointer. We disallow this. - IntToBlockPointer, + /// Pointer to NSNumber type (NSNumber *). + QualType NSNumberPointer; - /// IncompatibleBlockPointer - The assignment is between two block - /// pointers types that are not compatible. - IncompatibleBlockPointer, + /// Pointer to NSValue type (NSValue *). + QualType NSValuePointer; - /// IncompatibleObjCQualifiedId - The assignment is between a qualified - /// id type and something else (that is incompatible with it). For example, - /// "id " = "Foo *", where "Foo *" doesn't implement the XXX protocol. - IncompatibleObjCQualifiedId, + /// The Objective-C NSNumber methods used to create NSNumber literals. + ObjCMethodDecl *NSNumberLiteralMethods[NSAPI::NumNSNumberLiteralMethods]; - /// IncompatibleObjCWeakRef - Assigning a weak-unavailable object to an - /// object with __weak qualifier. - IncompatibleObjCWeakRef, + /// The declaration of the Objective-C NSString class. + ObjCInterfaceDecl *NSStringDecl; - /// Incompatible - We reject this conversion outright, it is invalid to - /// represent it in the AST. - Incompatible - }; + /// Pointer to NSString type (NSString *). + QualType NSStringPointer; - /// DiagnoseAssignmentResult - Emit a diagnostic, if required, for the - /// assignment conversion type specified by ConvTy. This returns true if the - /// conversion was invalid or false if the conversion was accepted. - bool DiagnoseAssignmentResult(AssignConvertType ConvTy, - SourceLocation Loc, - QualType DstType, QualType SrcType, - Expr *SrcExpr, AssignmentAction Action, - bool *Complained = nullptr); + /// The declaration of the stringWithUTF8String: method. + ObjCMethodDecl *StringWithUTF8StringMethod; - /// IsValueInFlagEnum - Determine if a value is allowed as part of a flag - /// enum. If AllowMask is true, then we also allow the complement of a valid - /// value, to be used as a mask. - bool IsValueInFlagEnum(const EnumDecl *ED, const llvm::APInt &Val, - bool AllowMask) const; + /// The declaration of the valueWithBytes:objCType: method. + ObjCMethodDecl *ValueWithBytesObjCTypeMethod; - /// DiagnoseAssignmentEnum - Warn if assignment to enum is a constant - /// integer not in the range of enum values. - void DiagnoseAssignmentEnum(QualType DstType, QualType SrcType, - Expr *SrcExpr); + /// The declaration of the Objective-C NSArray class. + ObjCInterfaceDecl *NSArrayDecl; - /// CheckAssignmentConstraints - Perform type checking for assignment, - /// argument passing, variable initialization, and function return values. - /// C99 6.5.16. - AssignConvertType CheckAssignmentConstraints(SourceLocation Loc, - QualType LHSType, - QualType RHSType); + /// The declaration of the arrayWithObjects:count: method. + ObjCMethodDecl *ArrayWithObjectsMethod; - /// Check assignment constraints and optionally prepare for a conversion of - /// the RHS to the LHS type. The conversion is prepared for if ConvertRHS - /// is true. - AssignConvertType CheckAssignmentConstraints(QualType LHSType, - ExprResult &RHS, - CastKind &Kind, - bool ConvertRHS = true); + /// The declaration of the Objective-C NSDictionary class. + ObjCInterfaceDecl *NSDictionaryDecl; - /// Check assignment constraints for an assignment of RHS to LHSType. - /// - /// \param LHSType The destination type for the assignment. - /// \param RHS The source expression for the assignment. - /// \param Diagnose If \c true, diagnostics may be produced when checking - /// for assignability. If a diagnostic is produced, \p RHS will be - /// set to ExprError(). Note that this function may still return - /// without producing a diagnostic, even for an invalid assignment. - /// \param DiagnoseCFAudited If \c true, the target is a function parameter - /// in an audited Core Foundation API and does not need to be checked - /// for ARC retain issues. - /// \param ConvertRHS If \c true, \p RHS will be updated to model the - /// conversions necessary to perform the assignment. If \c false, - /// \p Diagnose must also be \c false. - AssignConvertType CheckSingleAssignmentConstraints( - QualType LHSType, ExprResult &RHS, bool Diagnose = true, - bool DiagnoseCFAudited = false, bool ConvertRHS = true); + /// The declaration of the dictionaryWithObjects:forKeys:count: method. + ObjCMethodDecl *DictionaryWithObjectsMethod; - // If the lhs type is a transparent union, check whether we - // can initialize the transparent union with the given expression. - AssignConvertType CheckTransparentUnionArgumentConstraints(QualType ArgType, - ExprResult &RHS); + /// id type. + QualType QIDNSCopying; - bool IsStringLiteralToNonConstPointerConversion(Expr *From, QualType ToType); + /// will hold 'respondsToSelector:' + Selector RespondsToSelectorSel; - bool CheckExceptionSpecCompatibility(Expr *From, QualType ToType); + ExprResult HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT, + Expr *BaseExpr, SourceLocation OpLoc, + DeclarationName MemberName, + SourceLocation MemberLoc, + SourceLocation SuperLoc, + QualType SuperType, bool Super); - ExprResult PerformImplicitConversion(Expr *From, QualType ToType, - AssignmentAction Action, - bool AllowExplicit = false); - ExprResult PerformImplicitConversion(Expr *From, QualType ToType, - const ImplicitConversionSequence& ICS, - AssignmentAction Action, - CheckedConversionKind CCK - = CCK_ImplicitConversion); - ExprResult PerformImplicitConversion(Expr *From, QualType ToType, - const StandardConversionSequence& SCS, - AssignmentAction Action, - CheckedConversionKind CCK); + ExprResult ActOnClassPropertyRefExpr(IdentifierInfo &receiverName, + IdentifierInfo &propertyName, + SourceLocation receiverNameLoc, + SourceLocation propertyNameLoc); - ExprResult PerformQualificationConversion( - Expr *E, QualType Ty, ExprValueKind VK = VK_PRValue, - CheckedConversionKind CCK = CCK_ImplicitConversion); + // ParseObjCStringLiteral - Parse Objective-C string literals. + ExprResult ParseObjCStringLiteral(SourceLocation *AtLocs, + ArrayRef Strings); - /// the following "Check" methods will return a valid/converted QualType - /// or a null QualType (indicating an error diagnostic was issued). + ExprResult BuildObjCStringLiteral(SourceLocation AtLoc, StringLiteral *S); - /// type checking binary operators (subroutines of CreateBuiltinBinOp). - QualType InvalidOperands(SourceLocation Loc, ExprResult &LHS, - ExprResult &RHS); - QualType InvalidLogicalVectorOperands(SourceLocation Loc, ExprResult &LHS, - ExprResult &RHS); - QualType CheckPointerToMemberOperands( // C++ 5.5 - ExprResult &LHS, ExprResult &RHS, ExprValueKind &VK, - SourceLocation OpLoc, bool isIndirect); - QualType CheckMultiplyDivideOperands( // C99 6.5.5 - ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, bool IsCompAssign, - bool IsDivide); - QualType CheckRemainderOperands( // C99 6.5.5 - ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, - bool IsCompAssign = false); - QualType CheckAdditionOperands( // C99 6.5.6 - ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, - BinaryOperatorKind Opc, QualType* CompLHSTy = nullptr); - QualType CheckSubtractionOperands( // C99 6.5.6 - ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, - QualType* CompLHSTy = nullptr); - QualType CheckShiftOperands( // C99 6.5.7 - ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, - BinaryOperatorKind Opc, bool IsCompAssign = false); - void CheckPtrComparisonWithNullChar(ExprResult &E, ExprResult &NullE); - QualType CheckCompareOperands( // C99 6.5.8/9 - ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, - BinaryOperatorKind Opc); - QualType CheckBitwiseOperands( // C99 6.5.[10...12] - ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, - BinaryOperatorKind Opc); - QualType CheckLogicalOperands( // C99 6.5.[13,14] - ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, - BinaryOperatorKind Opc); - // CheckAssignmentOperands is used for both simple and compound assignment. - // For simple assignment, pass both expressions and a null converted type. - // For compound assignment, pass both expressions and the converted type. - QualType CheckAssignmentOperands( // C99 6.5.16.[1,2] - Expr *LHSExpr, ExprResult &RHS, SourceLocation Loc, QualType CompoundType, - BinaryOperatorKind Opc); + /// BuildObjCNumericLiteral - builds an ObjCBoxedExpr AST node for the + /// numeric literal expression. Type of the expression will be "NSNumber *" + /// or "id" if NSNumber is unavailable. + ExprResult BuildObjCNumericLiteral(SourceLocation AtLoc, Expr *Number); + ExprResult ActOnObjCBoolLiteral(SourceLocation AtLoc, SourceLocation ValueLoc, + bool Value); + ExprResult BuildObjCArrayLiteral(SourceRange SR, MultiExprArg Elements); - ExprResult checkPseudoObjectIncDec(Scope *S, SourceLocation OpLoc, - UnaryOperatorKind Opcode, Expr *Op); - ExprResult checkPseudoObjectAssignment(Scope *S, SourceLocation OpLoc, - BinaryOperatorKind Opcode, - Expr *LHS, Expr *RHS); - ExprResult checkPseudoObjectRValue(Expr *E); - Expr *recreateSyntacticForm(PseudoObjectExpr *E); + /// BuildObjCBoxedExpr - builds an ObjCBoxedExpr AST node for the + /// '@' prefixed parenthesized expression. The type of the expression will + /// either be "NSNumber *", "NSString *" or "NSValue *" depending on the type + /// of ValueType, which is allowed to be a built-in numeric type, "char *", + /// "const char *" or C structure with attribute 'objc_boxable'. + ExprResult BuildObjCBoxedExpr(SourceRange SR, Expr *ValueExpr); - QualType CheckConditionalOperands( // C99 6.5.15 - ExprResult &Cond, ExprResult &LHS, ExprResult &RHS, - ExprValueKind &VK, ExprObjectKind &OK, SourceLocation QuestionLoc); - QualType CXXCheckConditionalOperands( // C++ 5.16 - ExprResult &cond, ExprResult &lhs, ExprResult &rhs, - ExprValueKind &VK, ExprObjectKind &OK, SourceLocation questionLoc); - QualType CheckVectorConditionalTypes(ExprResult &Cond, ExprResult &LHS, - ExprResult &RHS, - SourceLocation QuestionLoc); + ExprResult BuildObjCSubscriptExpression(SourceLocation RB, Expr *BaseExpr, + Expr *IndexExpr, + ObjCMethodDecl *getterMethod, + ObjCMethodDecl *setterMethod); - QualType CheckSizelessVectorConditionalTypes(ExprResult &Cond, - ExprResult &LHS, ExprResult &RHS, - SourceLocation QuestionLoc); - QualType FindCompositePointerType(SourceLocation Loc, Expr *&E1, Expr *&E2, - bool ConvertArgs = true); - QualType FindCompositePointerType(SourceLocation Loc, - ExprResult &E1, ExprResult &E2, - bool ConvertArgs = true) { - Expr *E1Tmp = E1.get(), *E2Tmp = E2.get(); - QualType Composite = - FindCompositePointerType(Loc, E1Tmp, E2Tmp, ConvertArgs); - E1 = E1Tmp; - E2 = E2Tmp; - return Composite; - } + ExprResult + BuildObjCDictionaryLiteral(SourceRange SR, + MutableArrayRef Elements); - QualType FindCompositeObjCPointerType(ExprResult &LHS, ExprResult &RHS, - SourceLocation QuestionLoc); + ExprResult BuildObjCEncodeExpression(SourceLocation AtLoc, + TypeSourceInfo *EncodedTypeInfo, + SourceLocation RParenLoc); - bool DiagnoseConditionalForNull(const Expr *LHSExpr, const Expr *RHSExpr, - SourceLocation QuestionLoc); + ExprResult ParseObjCEncodeExpression(SourceLocation AtLoc, + SourceLocation EncodeLoc, + SourceLocation LParenLoc, ParsedType Ty, + SourceLocation RParenLoc); - void DiagnoseAlwaysNonNullPointer(Expr *E, - Expr::NullPointerConstantKind NullType, - bool IsEqual, SourceRange Range); + /// ParseObjCSelectorExpression - Build selector expression for \@selector + ExprResult ParseObjCSelectorExpression(Selector Sel, SourceLocation AtLoc, + SourceLocation SelLoc, + SourceLocation LParenLoc, + SourceLocation RParenLoc, + bool WarnMultipleSelectors); - /// type checking for vector binary operators. - QualType CheckVectorOperands(ExprResult &LHS, ExprResult &RHS, - SourceLocation Loc, bool IsCompAssign, - bool AllowBothBool, bool AllowBoolConversion, - bool AllowBoolOperation, bool ReportInvalid); - QualType GetSignedVectorType(QualType V); - QualType GetSignedSizelessVectorType(QualType V); - QualType CheckVectorCompareOperands(ExprResult &LHS, ExprResult &RHS, - SourceLocation Loc, - BinaryOperatorKind Opc); - QualType CheckSizelessVectorCompareOperands(ExprResult &LHS, ExprResult &RHS, - SourceLocation Loc, - BinaryOperatorKind Opc); - QualType CheckVectorLogicalOperands(ExprResult &LHS, ExprResult &RHS, - SourceLocation Loc); + /// ParseObjCProtocolExpression - Build protocol expression for \@protocol + ExprResult ParseObjCProtocolExpression(IdentifierInfo *ProtocolName, + SourceLocation AtLoc, + SourceLocation ProtoLoc, + SourceLocation LParenLoc, + SourceLocation ProtoIdLoc, + SourceLocation RParenLoc); - // type checking for sizeless vector binary operators. - QualType CheckSizelessVectorOperands(ExprResult &LHS, ExprResult &RHS, - SourceLocation Loc, bool IsCompAssign, - ArithConvKind OperationKind); + ObjCMethodDecl *tryCaptureObjCSelf(SourceLocation Loc); - /// Type checking for matrix binary operators. - QualType CheckMatrixElementwiseOperands(ExprResult &LHS, ExprResult &RHS, - SourceLocation Loc, - bool IsCompAssign); - QualType CheckMatrixMultiplyOperands(ExprResult &LHS, ExprResult &RHS, - SourceLocation Loc, bool IsCompAssign); + /// Describes the kind of message expression indicated by a message + /// send that starts with an identifier. + enum ObjCMessageKind { + /// The message is sent to 'super'. + ObjCSuperMessage, + /// The message is an instance message. + ObjCInstanceMessage, + /// The message is a class message, and the identifier is a type + /// name. + ObjCClassMessage + }; - bool isValidSveBitcast(QualType srcType, QualType destType); - bool isValidRVVBitcast(QualType srcType, QualType destType); + ObjCMessageKind getObjCMessageKind(Scope *S, IdentifierInfo *Name, + SourceLocation NameLoc, bool IsSuper, + bool HasTrailingDot, + ParsedType &ReceiverType); - bool areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy); + ExprResult ActOnSuperMessage(Scope *S, SourceLocation SuperLoc, Selector Sel, + SourceLocation LBracLoc, + ArrayRef SelectorLocs, + SourceLocation RBracLoc, MultiExprArg Args); - bool areVectorTypesSameSize(QualType srcType, QualType destType); - bool areLaxCompatibleVectorTypes(QualType srcType, QualType destType); - bool isLaxVectorConversion(QualType srcType, QualType destType); - bool anyAltivecTypes(QualType srcType, QualType destType); + ExprResult BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo, + QualType ReceiverType, SourceLocation SuperLoc, + Selector Sel, ObjCMethodDecl *Method, + SourceLocation LBracLoc, + ArrayRef SelectorLocs, + SourceLocation RBracLoc, MultiExprArg Args, + bool isImplicit = false); - /// type checking declaration initializers (C99 6.7.8) - bool CheckForConstantInitializer(Expr *e, QualType t); + ExprResult BuildClassMessageImplicit(QualType ReceiverType, + bool isSuperReceiver, SourceLocation Loc, + Selector Sel, ObjCMethodDecl *Method, + MultiExprArg Args); - // type checking C++ declaration initializers (C++ [dcl.init]). + ExprResult ActOnClassMessage(Scope *S, ParsedType Receiver, Selector Sel, + SourceLocation LBracLoc, + ArrayRef SelectorLocs, + SourceLocation RBracLoc, MultiExprArg Args); - /// ReferenceCompareResult - Expresses the result of comparing two - /// types (cv1 T1 and cv2 T2) to determine their compatibility for the - /// purposes of initialization by reference (C++ [dcl.init.ref]p4). - enum ReferenceCompareResult { - /// Ref_Incompatible - The two types are incompatible, so direct - /// reference binding is not possible. - Ref_Incompatible = 0, - /// Ref_Related - The two types are reference-related, which means - /// that their unqualified forms (T1 and T2) are either the same - /// or T1 is a base class of T2. - Ref_Related, - /// Ref_Compatible - The two types are reference-compatible. - Ref_Compatible - }; + ExprResult BuildInstanceMessage(Expr *Receiver, QualType ReceiverType, + SourceLocation SuperLoc, Selector Sel, + ObjCMethodDecl *Method, + SourceLocation LBracLoc, + ArrayRef SelectorLocs, + SourceLocation RBracLoc, MultiExprArg Args, + bool isImplicit = false); - // Fake up a scoped enumeration that still contextually converts to bool. - struct ReferenceConversionsScope { - /// The conversions that would be performed on an lvalue of type T2 when - /// binding a reference of type T1 to it, as determined when evaluating - /// whether T1 is reference-compatible with T2. - enum ReferenceConversions { - Qualification = 0x1, - NestedQualification = 0x2, - Function = 0x4, - DerivedToBase = 0x8, - ObjC = 0x10, - ObjCLifetime = 0x20, + ExprResult BuildInstanceMessageImplicit(Expr *Receiver, QualType ReceiverType, + SourceLocation Loc, Selector Sel, + ObjCMethodDecl *Method, + MultiExprArg Args); - LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/ObjCLifetime) - }; - }; - using ReferenceConversions = ReferenceConversionsScope::ReferenceConversions; + ExprResult ActOnInstanceMessage(Scope *S, Expr *Receiver, Selector Sel, + SourceLocation LBracLoc, + ArrayRef SelectorLocs, + SourceLocation RBracLoc, MultiExprArg Args); - ReferenceCompareResult - CompareReferenceRelationship(SourceLocation Loc, QualType T1, QualType T2, - ReferenceConversions *Conv = nullptr); + ExprResult BuildObjCBridgedCast(SourceLocation LParenLoc, + ObjCBridgeCastKind Kind, + SourceLocation BridgeKeywordLoc, + TypeSourceInfo *TSInfo, Expr *SubExpr); - ExprResult checkUnknownAnyCast(SourceRange TypeRange, QualType CastType, - Expr *CastExpr, CastKind &CastKind, - ExprValueKind &VK, CXXCastPath &Path); + ExprResult ActOnObjCBridgedCast(Scope *S, SourceLocation LParenLoc, + ObjCBridgeCastKind Kind, + SourceLocation BridgeKeywordLoc, + ParsedType Type, SourceLocation RParenLoc, + Expr *SubExpr); - /// Force an expression with unknown-type to an expression of the - /// given type. - ExprResult forceUnknownAnyToType(Expr *E, QualType ToType); + void CheckTollFreeBridgeCast(QualType castType, Expr *castExpr); - /// Type-check an expression that's being passed to an - /// __unknown_anytype parameter. - ExprResult checkUnknownAnyArg(SourceLocation callLoc, - Expr *result, QualType ¶mType); + void CheckObjCBridgeRelatedCast(QualType castType, Expr *castExpr); - // CheckMatrixCast - Check type constraints for matrix casts. - // We allow casting between matrixes of the same dimensions i.e. when they - // have the same number of rows and column. Returns true if the cast is - // invalid. - bool CheckMatrixCast(SourceRange R, QualType DestTy, QualType SrcTy, - CastKind &Kind); + bool CheckTollFreeBridgeStaticCast(QualType castType, Expr *castExpr, + CastKind &Kind); - // CheckVectorCast - check type constraints for vectors. - // Since vectors are an extension, there are no C standard reference for this. - // We allow casting between vectors and integer datatypes of the same size. - // returns true if the cast is invalid - bool CheckVectorCast(SourceRange R, QualType VectorTy, QualType Ty, - CastKind &Kind); + bool checkObjCBridgeRelatedComponents(SourceLocation Loc, QualType DestType, + QualType SrcType, + ObjCInterfaceDecl *&RelatedClass, + ObjCMethodDecl *&ClassMethod, + ObjCMethodDecl *&InstanceMethod, + TypedefNameDecl *&TDNDecl, bool CfToNs, + bool Diagnose = true); - /// Prepare `SplattedExpr` for a vector splat operation, adding - /// implicit casts if necessary. - ExprResult prepareVectorSplat(QualType VectorTy, Expr *SplattedExpr); + bool CheckObjCBridgeRelatedConversions(SourceLocation Loc, QualType DestType, + QualType SrcType, Expr *&SrcExpr, + bool Diagnose = true); - // CheckExtVectorCast - check type constraints for extended vectors. - // Since vectors are an extension, there are no C standard reference for this. - // We allow casting between vectors and integer datatypes of the same size, - // or vectors and the element type of that vector. - // returns the cast expr - ExprResult CheckExtVectorCast(SourceRange R, QualType DestTy, Expr *CastExpr, - CastKind &Kind); + /// Private Helper predicate to check for 'self'. + bool isSelfExpr(Expr *RExpr); + bool isSelfExpr(Expr *RExpr, const ObjCMethodDecl *Method); - ExprResult BuildCXXFunctionalCastExpr(TypeSourceInfo *TInfo, QualType Type, - SourceLocation LParenLoc, - Expr *CastExpr, - SourceLocation RParenLoc); + ObjCMethodDecl *LookupMethodInQualifiedType(Selector Sel, + const ObjCObjectPointerType *OPT, + bool IsInstance); + ObjCMethodDecl *LookupMethodInObjectType(Selector Sel, QualType Ty, + bool IsInstance); + + bool isKnownName(StringRef name); enum ARCConversionResult { ACR_okay, ACR_unbridged, ACR_error }; @@ -13305,8 +12608,7 @@ public: CheckedConversionKind CCK, bool Diagnose = true, bool DiagnoseCFAudited = false, - BinaryOperatorKind Opc = BO_PtrMemD - ); + BinaryOperatorKind Opc = BO_PtrMemD); Expr *stripARCUnbridgedCast(Expr *e); void diagnoseARCUnbridgedCast(Expr *e); @@ -13314,20 +12616,6 @@ public: bool CheckObjCARCUnavailableWeakConversion(QualType castType, QualType ExprType); - /// checkRetainCycles - Check whether an Objective-C message send - /// might create an obvious retain cycle. - void checkRetainCycles(ObjCMessageExpr *msg); - void checkRetainCycles(Expr *receiver, Expr *argument); - void checkRetainCycles(VarDecl *Var, Expr *Init); - - /// checkUnsafeAssigns - Check whether +1 expr is being assigned - /// to weak/__unsafe_unretained type. - bool checkUnsafeAssigns(SourceLocation Loc, QualType LHS, Expr *RHS); - - /// checkUnsafeExprAssigns - Check whether +1 expr is being assigned - /// to weak/__unsafe_unretained expression. - void checkUnsafeExprAssigns(SourceLocation Loc, Expr *LHS, Expr *RHS); - /// CheckMessageArgumentTypes - Check types in an Obj-C message send. /// \param Method - May be null. /// \param [out] ReturnType - The return type of the send. @@ -13356,142 +12644,398 @@ public: /// type, and if so, emit a note describing what happened. void EmitRelatedResultTypeNoteForReturn(QualType destType); - class ConditionResult { - Decl *ConditionVar; - FullExprArg Condition; - bool Invalid; - std::optional KnownValue; + /// LookupInstanceMethodInGlobalPool - Returns the method and warns if + /// there are multiple signatures. + ObjCMethodDecl * + LookupInstanceMethodInGlobalPool(Selector Sel, SourceRange R, + bool receiverIdOrClass = false) { + return LookupMethodInGlobalPool(Sel, R, receiverIdOrClass, + /*instance*/ true); + } - friend class Sema; - ConditionResult(Sema &S, Decl *ConditionVar, FullExprArg Condition, - bool IsConstexpr) - : ConditionVar(ConditionVar), Condition(Condition), Invalid(false) { - if (IsConstexpr && Condition.get()) { - if (std::optional Val = - Condition.get()->getIntegerConstantExpr(S.Context)) { - KnownValue = !!(*Val); - } - } - } - explicit ConditionResult(bool Invalid) - : ConditionVar(nullptr), Condition(nullptr), Invalid(Invalid), - KnownValue(std::nullopt) {} + /// LookupFactoryMethodInGlobalPool - Returns the method and warns if + /// there are multiple signatures. + ObjCMethodDecl * + LookupFactoryMethodInGlobalPool(Selector Sel, SourceRange R, + bool receiverIdOrClass = false) { + return LookupMethodInGlobalPool(Sel, R, receiverIdOrClass, + /*instance*/ false); + } - public: - ConditionResult() : ConditionResult(false) {} - bool isInvalid() const { return Invalid; } - std::pair get() const { - return std::make_pair(cast_or_null(ConditionVar), - Condition.get()); - } - std::optional getKnownValue() const { return KnownValue; } - }; - static ConditionResult ConditionError() { return ConditionResult(true); } + ///@} - enum class ConditionKind { - Boolean, ///< A boolean condition, from 'if', 'while', 'for', or 'do'. - ConstexprIf, ///< A constant boolean condition from 'if constexpr'. - Switch ///< An integral condition for a 'switch' statement. - }; - QualType PreferredConditionType(ConditionKind K) const { - return K == ConditionKind::Switch ? Context.IntTy : Context.BoolTy; - } + // + // + // ------------------------------------------------------------------------- + // + // + + /// \name ObjC @property and @synthesize + /// Implementations are in SemaObjCProperty.cpp + ///@{ + +public: + /// Ensure attributes are consistent with type. + /// \param [in, out] Attributes The attributes to check; they will + /// be modified to be consistent with \p PropertyTy. + void CheckObjCPropertyAttributes(Decl *PropertyPtrTy, SourceLocation Loc, + unsigned &Attributes, + bool propertyInPrimaryClass); + + /// Process the specified property declaration and create decls for the + /// setters and getters as needed. + /// \param property The property declaration being processed + void ProcessPropertyDecl(ObjCPropertyDecl *property); + + Decl *ActOnProperty(Scope *S, SourceLocation AtLoc, SourceLocation LParenLoc, + FieldDeclarator &FD, ObjCDeclSpec &ODS, + Selector GetterSel, Selector SetterSel, + tok::ObjCKeywordKind MethodImplKind, + DeclContext *lexicalDC = nullptr); + + Decl *ActOnPropertyImplDecl(Scope *S, SourceLocation AtLoc, + SourceLocation PropertyLoc, bool ImplKind, + IdentifierInfo *PropertyId, + IdentifierInfo *PropertyIvar, + SourceLocation PropertyIvarLoc, + ObjCPropertyQueryKind QueryKind); + + /// Called by ActOnProperty to handle \@property declarations in + /// class extensions. + ObjCPropertyDecl *HandlePropertyInClassExtension( + Scope *S, SourceLocation AtLoc, SourceLocation LParenLoc, + FieldDeclarator &FD, Selector GetterSel, SourceLocation GetterNameLoc, + Selector SetterSel, SourceLocation SetterNameLoc, const bool isReadWrite, + unsigned &Attributes, const unsigned AttributesAsWritten, QualType T, + TypeSourceInfo *TSI, tok::ObjCKeywordKind MethodImplKind); + + /// Called by ActOnProperty and HandlePropertyInClassExtension to + /// handle creating the ObjcPropertyDecl for a category or \@interface. + ObjCPropertyDecl * + CreatePropertyDecl(Scope *S, ObjCContainerDecl *CDecl, SourceLocation AtLoc, + SourceLocation LParenLoc, FieldDeclarator &FD, + Selector GetterSel, SourceLocation GetterNameLoc, + Selector SetterSel, SourceLocation SetterNameLoc, + const bool isReadWrite, const unsigned Attributes, + const unsigned AttributesAsWritten, QualType T, + TypeSourceInfo *TSI, tok::ObjCKeywordKind MethodImplKind, + DeclContext *lexicalDC = nullptr); + + void DiagnosePropertyMismatch(ObjCPropertyDecl *Property, + ObjCPropertyDecl *SuperProperty, + const IdentifierInfo *Name, + bool OverridingProtocolProperty); + + bool DiagnosePropertyAccessorMismatch(ObjCPropertyDecl *PD, + ObjCMethodDecl *Getter, + SourceLocation Loc); + + /// DiagnoseUnimplementedProperties - This routine warns on those properties + /// which must be implemented by this implementation. + void DiagnoseUnimplementedProperties(Scope *S, ObjCImplDecl *IMPDecl, + ObjCContainerDecl *CDecl, + bool SynthesizeProperties); + + /// Diagnose any null-resettable synthesized setters. + void diagnoseNullResettableSynthesizedSetters(const ObjCImplDecl *impDecl); + + /// DefaultSynthesizeProperties - This routine default synthesizes all + /// properties which must be synthesized in the class's \@implementation. + void DefaultSynthesizeProperties(Scope *S, ObjCImplDecl *IMPDecl, + ObjCInterfaceDecl *IDecl, + SourceLocation AtEnd); + void DefaultSynthesizeProperties(Scope *S, Decl *D, SourceLocation AtEnd); + + /// IvarBacksCurrentMethodAccessor - This routine returns 'true' if 'IV' is + /// an ivar synthesized for 'Method' and 'Method' is a property accessor + /// declared in class 'IFace'. + bool IvarBacksCurrentMethodAccessor(ObjCInterfaceDecl *IFace, + ObjCMethodDecl *Method, ObjCIvarDecl *IV); + + void DiagnoseOwningPropertyGetterSynthesis(const ObjCImplementationDecl *D); + + void + DiagnoseMissingDesignatedInitOverrides(const ObjCImplementationDecl *ImplD, + const ObjCInterfaceDecl *IFD); + + /// AtomicPropertySetterGetterRules - This routine enforces the rule (via + /// warning) when atomic property has one but not the other user-declared + /// setter or getter. + void AtomicPropertySetterGetterRules(ObjCImplDecl *IMPDecl, + ObjCInterfaceDecl *IDecl); + + ///@} - ConditionResult ActOnCondition(Scope *S, SourceLocation Loc, Expr *SubExpr, - ConditionKind CK, bool MissingOK = false); + // + // + // ------------------------------------------------------------------------- + // + // - ConditionResult ActOnConditionVariable(Decl *ConditionVar, - SourceLocation StmtLoc, - ConditionKind CK); + /// \name Code Completion + /// Implementations are in SemaCodeComplete.cpp + ///@{ - DeclResult ActOnCXXConditionDeclaration(Scope *S, Declarator &D); +public: + /// Code-completion consumer. + CodeCompleteConsumer *CodeCompleter; - ExprResult CheckConditionVariable(VarDecl *ConditionVar, - SourceLocation StmtLoc, - ConditionKind CK); - ExprResult CheckSwitchCondition(SourceLocation SwitchLoc, Expr *Cond); + /// Describes the context in which code completion occurs. + enum ParserCompletionContext { + /// Code completion occurs at top-level or namespace context. + PCC_Namespace, + /// Code completion occurs within a class, struct, or union. + PCC_Class, + /// Code completion occurs within an Objective-C interface, protocol, + /// or category. + PCC_ObjCInterface, + /// Code completion occurs within an Objective-C implementation or + /// category implementation + PCC_ObjCImplementation, + /// Code completion occurs within the list of instance variables + /// in an Objective-C interface, protocol, category, or implementation. + PCC_ObjCInstanceVariableList, + /// Code completion occurs following one or more template + /// headers. + PCC_Template, + /// Code completion occurs following one or more template + /// headers within a class. + PCC_MemberTemplate, + /// Code completion occurs within an expression. + PCC_Expression, + /// Code completion occurs within a statement, which may + /// also be an expression or a declaration. + PCC_Statement, + /// Code completion occurs at the beginning of the + /// initialization statement (or expression) in a for loop. + PCC_ForInit, + /// Code completion occurs within the condition of an if, + /// while, switch, or for statement. + PCC_Condition, + /// Code completion occurs within the body of a function on a + /// recovery path, where we do not have a specific handle on our position + /// in the grammar. + PCC_RecoveryInFunction, + /// Code completion occurs where only a type is permitted. + PCC_Type, + /// Code completion occurs in a parenthesized expression, which + /// might also be a type cast. + PCC_ParenthesizedExpression, + /// Code completion occurs within a sequence of declaration + /// specifiers within a function, method, or block. + PCC_LocalDeclarationSpecifiers, + /// Code completion occurs at top-level in a REPL session + PCC_TopLevelOrExpression, + }; - /// CheckBooleanCondition - Diagnose problems involving the use of - /// the given expression as a boolean condition (e.g. in an if - /// statement). Also performs the standard function and array - /// decays, possibly changing the input variable. + void CodeCompleteModuleImport(SourceLocation ImportLoc, ModuleIdPath Path); + void CodeCompleteOrdinaryName(Scope *S, + ParserCompletionContext CompletionContext); + void CodeCompleteDeclSpec(Scope *S, DeclSpec &DS, bool AllowNonIdentifiers, + bool AllowNestedNameSpecifiers); + + struct CodeCompleteExpressionData; + void CodeCompleteExpression(Scope *S, const CodeCompleteExpressionData &Data); + void CodeCompleteExpression(Scope *S, QualType PreferredType, + bool IsParenthesized = false); + void CodeCompleteMemberReferenceExpr(Scope *S, Expr *Base, Expr *OtherOpBase, + SourceLocation OpLoc, bool IsArrow, + bool IsBaseExprStatement, + QualType PreferredType); + void CodeCompletePostfixExpression(Scope *S, ExprResult LHS, + QualType PreferredType); + void CodeCompleteTag(Scope *S, unsigned TagSpec); + void CodeCompleteTypeQualifiers(DeclSpec &DS); + void CodeCompleteFunctionQualifiers(DeclSpec &DS, Declarator &D, + const VirtSpecifiers *VS = nullptr); + void CodeCompleteBracketDeclarator(Scope *S); + void CodeCompleteCase(Scope *S); + enum class AttributeCompletion { + Attribute, + Scope, + None, + }; + void CodeCompleteAttribute( + AttributeCommonInfo::Syntax Syntax, + AttributeCompletion Completion = AttributeCompletion::Attribute, + const IdentifierInfo *Scope = nullptr); + /// Determines the preferred type of the current function argument, by + /// examining the signatures of all possible overloads. + /// Returns null if unknown or ambiguous, or if code completion is off. /// - /// \param Loc - A location associated with the condition, e.g. the - /// 'if' keyword. - /// \return true iff there were any errors - ExprResult CheckBooleanCondition(SourceLocation Loc, Expr *E, - bool IsConstexpr = false); + /// If the code completion point has been reached, also reports the function + /// signatures that were considered. + /// + /// FIXME: rename to GuessCallArgumentType to reduce confusion. + QualType ProduceCallSignatureHelp(Expr *Fn, ArrayRef Args, + SourceLocation OpenParLoc); + QualType ProduceConstructorSignatureHelp(QualType Type, SourceLocation Loc, + ArrayRef Args, + SourceLocation OpenParLoc, + bool Braced); + QualType ProduceCtorInitMemberSignatureHelp( + Decl *ConstructorDecl, CXXScopeSpec SS, ParsedType TemplateTypeTy, + ArrayRef ArgExprs, IdentifierInfo *II, SourceLocation OpenParLoc, + bool Braced); + QualType ProduceTemplateArgumentSignatureHelp( + TemplateTy, ArrayRef, SourceLocation LAngleLoc); + void CodeCompleteInitializer(Scope *S, Decl *D); + /// Trigger code completion for a record of \p BaseType. \p InitExprs are + /// expressions in the initializer list seen so far and \p D is the current + /// Designation being parsed. + void CodeCompleteDesignator(const QualType BaseType, + llvm::ArrayRef InitExprs, + const Designation &D); + void CodeCompleteAfterIf(Scope *S, bool IsBracedThen); - /// ActOnExplicitBoolSpecifier - Build an ExplicitSpecifier from an expression - /// found in an explicit(bool) specifier. - ExplicitSpecifier ActOnExplicitBoolSpecifier(Expr *E); + void CodeCompleteQualifiedId(Scope *S, CXXScopeSpec &SS, bool EnteringContext, + bool IsUsingDeclaration, QualType BaseType, + QualType PreferredType); + void CodeCompleteUsing(Scope *S); + void CodeCompleteUsingDirective(Scope *S); + void CodeCompleteNamespaceDecl(Scope *S); + void CodeCompleteNamespaceAliasDecl(Scope *S); + void CodeCompleteOperatorName(Scope *S); + void CodeCompleteConstructorInitializer( + Decl *Constructor, ArrayRef Initializers); - /// tryResolveExplicitSpecifier - Attempt to resolve the explict specifier. - /// Returns true if the explicit specifier is now resolved. - bool tryResolveExplicitSpecifier(ExplicitSpecifier &ExplicitSpec); + void CodeCompleteLambdaIntroducer(Scope *S, LambdaIntroducer &Intro, + bool AfterAmpersand); + void CodeCompleteAfterFunctionEquals(Declarator &D); - /// DiagnoseAssignmentAsCondition - Given that an expression is - /// being used as a boolean condition, warn if it's an assignment. - void DiagnoseAssignmentAsCondition(Expr *E); + void CodeCompleteObjCAtDirective(Scope *S); + void CodeCompleteObjCAtVisibility(Scope *S); + void CodeCompleteObjCAtStatement(Scope *S); + void CodeCompleteObjCAtExpression(Scope *S); + void CodeCompleteObjCPropertyFlags(Scope *S, ObjCDeclSpec &ODS); + void CodeCompleteObjCPropertyGetter(Scope *S); + void CodeCompleteObjCPropertySetter(Scope *S); + void CodeCompleteObjCPassingType(Scope *S, ObjCDeclSpec &DS, + bool IsParameter); + void CodeCompleteObjCMessageReceiver(Scope *S); + void CodeCompleteObjCSuperMessage(Scope *S, SourceLocation SuperLoc, + ArrayRef SelIdents, + bool AtArgumentExpression); + void CodeCompleteObjCClassMessage(Scope *S, ParsedType Receiver, + ArrayRef SelIdents, + bool AtArgumentExpression, + bool IsSuper = false); + void CodeCompleteObjCInstanceMessage(Scope *S, Expr *Receiver, + ArrayRef SelIdents, + bool AtArgumentExpression, + ObjCInterfaceDecl *Super = nullptr); + void CodeCompleteObjCForCollection(Scope *S, DeclGroupPtrTy IterationVar); + void CodeCompleteObjCSelector(Scope *S, ArrayRef SelIdents); + void + CodeCompleteObjCProtocolReferences(ArrayRef Protocols); + void CodeCompleteObjCProtocolDecl(Scope *S); + void CodeCompleteObjCInterfaceDecl(Scope *S); + void CodeCompleteObjCClassForwardDecl(Scope *S); + void CodeCompleteObjCSuperclass(Scope *S, IdentifierInfo *ClassName, + SourceLocation ClassNameLoc); + void CodeCompleteObjCImplementationDecl(Scope *S); + void CodeCompleteObjCInterfaceCategory(Scope *S, IdentifierInfo *ClassName, + SourceLocation ClassNameLoc); + void CodeCompleteObjCImplementationCategory(Scope *S, + IdentifierInfo *ClassName, + SourceLocation ClassNameLoc); + void CodeCompleteObjCPropertyDefinition(Scope *S); + void CodeCompleteObjCPropertySynthesizeIvar(Scope *S, + IdentifierInfo *PropertyName); + void CodeCompleteObjCMethodDecl(Scope *S, + std::optional IsInstanceMethod, + ParsedType ReturnType); + void CodeCompleteObjCMethodDeclSelector(Scope *S, bool IsInstanceMethod, + bool AtParameterName, + ParsedType ReturnType, + ArrayRef SelIdents); + void CodeCompleteObjCClassPropertyRefExpr(Scope *S, IdentifierInfo &ClassName, + SourceLocation ClassNameLoc, + bool IsBaseExprStatement); + void CodeCompletePreprocessorDirective(bool InConditional); + void CodeCompleteInPreprocessorConditionalExclusion(Scope *S); + void CodeCompletePreprocessorMacroName(bool IsDefinition); + void CodeCompletePreprocessorExpression(); + void CodeCompletePreprocessorMacroArgument(Scope *S, IdentifierInfo *Macro, + MacroInfo *MacroInfo, + unsigned Argument); + void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled); + void CodeCompleteNaturalLanguage(); + void CodeCompleteAvailabilityPlatformName(); + void + GatherGlobalCodeCompletions(CodeCompletionAllocator &Allocator, + CodeCompletionTUInfo &CCTUInfo, + SmallVectorImpl &Results); - /// Redundant parentheses over an equality comparison can indicate - /// that the user intended an assignment used as condition. - void DiagnoseEqualityWithExtraParens(ParenExpr *ParenE); + ///@} - /// CheckCXXBooleanCondition - Returns true if conversion to bool is invalid. - ExprResult CheckCXXBooleanCondition(Expr *CondExpr, bool IsConstexpr = false); + // + // + // ------------------------------------------------------------------------- + // + // - /// Checks that the Objective-C declaration is declared in the global scope. - /// Emits an error and marks the declaration as invalid if it's not declared - /// in the global scope. - bool CheckObjCDeclScope(Decl *D); + /// \name FixIt Helpers + /// Implementations are in SemaFixItUtils.cpp + ///@{ - /// Abstract base class used for diagnosing integer constant - /// expression violations. - class VerifyICEDiagnoser { - public: - bool Suppress; +public: + /// Get a string to suggest for zero-initialization of a type. + std::string getFixItZeroInitializerForType(QualType T, + SourceLocation Loc) const; + std::string getFixItZeroLiteralForType(QualType T, SourceLocation Loc) const; - VerifyICEDiagnoser(bool Suppress = false) : Suppress(Suppress) { } + ///@} - virtual SemaDiagnosticBuilder - diagnoseNotICEType(Sema &S, SourceLocation Loc, QualType T); - virtual SemaDiagnosticBuilder diagnoseNotICE(Sema &S, - SourceLocation Loc) = 0; - virtual SemaDiagnosticBuilder diagnoseFold(Sema &S, SourceLocation Loc); - virtual ~VerifyICEDiagnoser() {} - }; + // + // + // ------------------------------------------------------------------------- + // + // - enum AllowFoldKind { - NoFold, - AllowFold, - }; + /// \name API Notes + /// Implementations are in SemaAPINotes.cpp + ///@{ - /// VerifyIntegerConstantExpression - Verifies that an expression is an ICE, - /// and reports the appropriate diagnostics. Returns false on success. - /// Can optionally return the value of the expression. - ExprResult VerifyIntegerConstantExpression(Expr *E, llvm::APSInt *Result, - VerifyICEDiagnoser &Diagnoser, - AllowFoldKind CanFold = NoFold); - ExprResult VerifyIntegerConstantExpression(Expr *E, llvm::APSInt *Result, - unsigned DiagID, - AllowFoldKind CanFold = NoFold); - ExprResult VerifyIntegerConstantExpression(Expr *E, - llvm::APSInt *Result = nullptr, - AllowFoldKind CanFold = NoFold); - ExprResult VerifyIntegerConstantExpression(Expr *E, - AllowFoldKind CanFold = NoFold) { - return VerifyIntegerConstantExpression(E, nullptr, CanFold); - } +public: + /// Map any API notes provided for this declaration to attributes on the + /// declaration. + /// + /// Triggered by declaration-attribute processing. + void ProcessAPINotes(Decl *D); - /// VerifyBitField - verifies that a bit field expression is an ICE and has - /// the correct width, and that the field type is valid. - /// Returns false on success. - ExprResult VerifyBitField(SourceLocation FieldLoc, IdentifierInfo *FieldName, - QualType FieldTy, bool IsMsStruct, Expr *BitWidth); + ///@} + // + // + // ------------------------------------------------------------------------- + // + // + + /// \name Name Lookup for RISC-V Vector Intrinsic + /// Implementations are in SemaRISCVVectorLookup.cpp + ///@{ + +public: + /// Indicate RISC-V vector builtin functions enabled or not. + bool DeclareRISCVVBuiltins = false; + + /// Indicate RISC-V SiFive vector builtin functions enabled or not. + bool DeclareRISCVSiFiveVectorBuiltins = false; private: - unsigned ForceCUDAHostDeviceDepth = 0; + std::unique_ptr RVIntrinsicManager; + + ///@} + + // + // + // ------------------------------------------------------------------------- + // + // + + /// \name CUDA + /// Implementations are in SemaCUDA.cpp + ///@{ public: /// Increments our count of the number of times we've seen a pragma forcing @@ -13504,6 +13048,10 @@ public: /// before incrementing, so you can emit an error. bool PopForceCUDAHostDevice(); + ExprResult ActOnCUDAExecConfigExpr(Scope *S, SourceLocation LLLLoc, + MultiExprArg ExecConfig, + SourceLocation GGGLoc); + /// Diagnostics that are emitted only if we discover that the given function /// must be codegen'ed. Because handling these correctly adds overhead to /// compilation, this is currently only enabled for CUDA compilations. @@ -13558,54 +13106,6 @@ public: /// Same as CUDADiagIfDeviceCode, with "host" and "device" switched. SemaDiagnosticBuilder CUDADiagIfHostCode(SourceLocation Loc, unsigned DiagID); - /// Creates a SemaDiagnosticBuilder that emits the diagnostic if the current - /// context is "used as device code". - /// - /// - If CurContext is a `declare target` function or it is known that the - /// function is emitted for the device, emits the diagnostics immediately. - /// - If CurContext is a non-`declare target` function and we are compiling - /// for the device, creates a diagnostic which is emitted if and when we - /// realize that the function will be codegen'ed. - /// - /// Example usage: - /// - /// // Variable-length arrays are not allowed in NVPTX device code. - /// if (diagIfOpenMPDeviceCode(Loc, diag::err_vla_unsupported)) - /// return ExprError(); - /// // Otherwise, continue parsing as normal. - SemaDiagnosticBuilder diagIfOpenMPDeviceCode(SourceLocation Loc, - unsigned DiagID, - const FunctionDecl *FD); - - /// Creates a SemaDiagnosticBuilder that emits the diagnostic if the current - /// context is "used as host code". - /// - /// - If CurContext is a `declare target` function or it is known that the - /// function is emitted for the host, emits the diagnostics immediately. - /// - If CurContext is a non-host function, just ignore it. - /// - /// Example usage: - /// - /// // Variable-length arrays are not allowed in NVPTX device code. - /// if (diagIfOpenMPHostode(Loc, diag::err_vla_unsupported)) - /// return ExprError(); - /// // Otherwise, continue parsing as normal. - SemaDiagnosticBuilder diagIfOpenMPHostCode(SourceLocation Loc, - unsigned DiagID, - const FunctionDecl *FD); - - SemaDiagnosticBuilder targetDiag(SourceLocation Loc, unsigned DiagID, - const FunctionDecl *FD = nullptr); - SemaDiagnosticBuilder targetDiag(SourceLocation Loc, - const PartialDiagnostic &PD, - const FunctionDecl *FD = nullptr) { - return targetDiag(Loc, PD.getDiagID(), FD) << PD; - } - - /// Check if the type is allowed to be used for the current target. - void checkTypeSupport(QualType Ty, SourceLocation Loc, - ValueDecl *D = nullptr); - /// Determines whether the given function is a CUDA device/host/kernel/etc. /// function. /// @@ -13696,7 +13196,6 @@ public: /// and current compilation settings. void MaybeAddCUDAConstantAttr(VarDecl *VD); -public: /// Check whether we're allowed to call Callee from the current context. /// /// - If the call is never allowed in a semantically-correct program @@ -13747,743 +13246,1491 @@ public: bool inferCUDATargetForImplicitSpecialMember(CXXRecordDecl *ClassDecl, CXXSpecialMember CSM, CXXMethodDecl *MemberDecl, - bool ConstRHS, - bool Diagnose); + bool ConstRHS, bool Diagnose); + + /// \return true if \p CD can be considered empty according to CUDA + /// (E.2.3.1 in CUDA 7.5 Programming guide). + bool isEmptyCudaConstructor(SourceLocation Loc, CXXConstructorDecl *CD); + bool isEmptyCudaDestructor(SourceLocation Loc, CXXDestructorDecl *CD); + + // \brief Checks that initializers of \p Var satisfy CUDA restrictions. In + // case of error emits appropriate diagnostic and invalidates \p Var. + // + // \details CUDA allows only empty constructors as initializers for global + // variables (see E.2.3.1, CUDA 7.5). The same restriction also applies to all + // __shared__ variables whether they are local or not (they all are implicitly + // static in CUDA). One exception is that CUDA allows constant initializers + // for __constant__ and __device__ variables. + void checkAllowedCUDAInitializer(VarDecl *VD); + + /// Check whether NewFD is a valid overload for CUDA. Emits + /// diagnostics and invalidates NewFD if not. + void checkCUDATargetOverload(FunctionDecl *NewFD, + const LookupResult &Previous); + /// Copies target attributes from the template TD to the function FD. + void inheritCUDATargetAttrs(FunctionDecl *FD, const FunctionTemplateDecl &TD); + + /// Returns the name of the launch configuration function. This is the name + /// of the function that will be called to configure kernel call, with the + /// parameters specified via <<<>>>. + std::string getCudaConfigureFuncName() const; + +private: + unsigned ForceCUDAHostDeviceDepth = 0; + + ///@} + + // + // + // ------------------------------------------------------------------------- + // + // + + /// \name HLSL Constructs + /// Implementations are in SemaHLSL.cpp + ///@{ + +public: + Decl *ActOnStartHLSLBuffer(Scope *BufferScope, bool CBuffer, + SourceLocation KwLoc, IdentifierInfo *Ident, + SourceLocation IdentLoc, SourceLocation LBrace); + void ActOnFinishHLSLBuffer(Decl *Dcl, SourceLocation RBrace); + + bool CheckHLSLBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); + + bool SemaBuiltinVectorMath(CallExpr *TheCall, QualType &Res); + bool SemaBuiltinVectorToScalarMath(CallExpr *TheCall); + + ///@} + + // + // + // ------------------------------------------------------------------------- + // + // + + /// \name OpenACC Constructs + /// Implementations are in SemaOpenACC.cpp + ///@{ + +public: + /// Called after parsing an OpenACC Clause so that it can be checked. + bool ActOnOpenACCClause(OpenACCClauseKind ClauseKind, + SourceLocation StartLoc); + + /// Called after the construct has been parsed, but clauses haven't been + /// parsed. This allows us to diagnose not-implemented, as well as set up any + /// state required for parsing the clauses. + void ActOnOpenACCConstruct(OpenACCDirectiveKind K, SourceLocation StartLoc); + + /// Called after the directive, including its clauses, have been parsed and + /// parsing has consumed the 'annot_pragma_openacc_end' token. This DOES + /// happen before any associated declarations or statements have been parsed. + /// This function is only called when we are parsing a 'statement' context. + bool ActOnStartOpenACCStmtDirective(OpenACCDirectiveKind K, + SourceLocation StartLoc); + + /// Called after the directive, including its clauses, have been parsed and + /// parsing has consumed the 'annot_pragma_openacc_end' token. This DOES + /// happen before any associated declarations or statements have been parsed. + /// This function is only called when we are parsing a 'Decl' context. + bool ActOnStartOpenACCDeclDirective(OpenACCDirectiveKind K, + SourceLocation StartLoc); + /// Called when we encounter an associated statement for our construct, this + /// should check legality of the statement as it appertains to this Construct. + StmtResult ActOnOpenACCAssociatedStmt(OpenACCDirectiveKind K, + StmtResult AssocStmt); + + /// Called after the directive has been completely parsed, including the + /// declaration group or associated statement. + StmtResult ActOnEndOpenACCStmtDirective(OpenACCDirectiveKind K, + SourceLocation StartLoc, + SourceLocation EndLoc, + StmtResult AssocStmt); + /// Called after the directive has been completely parsed, including the + /// declaration group or associated statement. + DeclGroupRef ActOnEndOpenACCDeclDirective(); + + ///@} + + // + // + // ------------------------------------------------------------------------- + // + // + + /// \name OpenMP Directives and Clauses + /// Implementations are in SemaOpenMP.cpp + ///@{ + +public: + /// Creates a SemaDiagnosticBuilder that emits the diagnostic if the current + /// context is "used as device code". + /// + /// - If CurContext is a `declare target` function or it is known that the + /// function is emitted for the device, emits the diagnostics immediately. + /// - If CurContext is a non-`declare target` function and we are compiling + /// for the device, creates a diagnostic which is emitted if and when we + /// realize that the function will be codegen'ed. + /// + /// Example usage: + /// + /// // Variable-length arrays are not allowed in NVPTX device code. + /// if (diagIfOpenMPDeviceCode(Loc, diag::err_vla_unsupported)) + /// return ExprError(); + /// // Otherwise, continue parsing as normal. + SemaDiagnosticBuilder diagIfOpenMPDeviceCode(SourceLocation Loc, + unsigned DiagID, + const FunctionDecl *FD); + + /// Creates a SemaDiagnosticBuilder that emits the diagnostic if the current + /// context is "used as host code". + /// + /// - If CurContext is a `declare target` function or it is known that the + /// function is emitted for the host, emits the diagnostics immediately. + /// - If CurContext is a non-host function, just ignore it. + /// + /// Example usage: + /// + /// // Variable-length arrays are not allowed in NVPTX device code. + /// if (diagIfOpenMPHostode(Loc, diag::err_vla_unsupported)) + /// return ExprError(); + /// // Otherwise, continue parsing as normal. + SemaDiagnosticBuilder diagIfOpenMPHostCode(SourceLocation Loc, + unsigned DiagID, + const FunctionDecl *FD); + + /// Register \p D as specialization of all base functions in \p Bases in the + /// current `omp begin/end declare variant` scope. + void ActOnFinishedFunctionDefinitionInOpenMPDeclareVariantScope( + Decl *D, SmallVectorImpl &Bases); + + /// Act on \p D, a function definition inside of an `omp [begin/end] assumes`. + void ActOnFinishedFunctionDefinitionInOpenMPAssumeScope(Decl *D); + + /// Can we exit an OpenMP declare variant scope at the moment. + bool isInOpenMPDeclareVariantScope() const { + return !OMPDeclareVariantScopes.empty(); + } + + ExprResult + VerifyPositiveIntegerConstantInClause(Expr *Op, OpenMPClauseKind CKind, + bool StrictlyPositive = true, + bool SuppressExprDiags = false); + + /// Given the potential call expression \p Call, determine if there is a + /// specialization via the OpenMP declare variant mechanism available. If + /// there is, return the specialized call expression, otherwise return the + /// original \p Call. + ExprResult ActOnOpenMPCall(ExprResult Call, Scope *Scope, + SourceLocation LParenLoc, MultiExprArg ArgExprs, + SourceLocation RParenLoc, Expr *ExecConfig); + + /// Handle a `omp begin declare variant`. + void ActOnOpenMPBeginDeclareVariant(SourceLocation Loc, OMPTraitInfo &TI); + + /// Handle a `omp end declare variant`. + void ActOnOpenMPEndDeclareVariant(); + + /// Function tries to capture lambda's captured variables in the OpenMP region + /// before the original lambda is captured. + void tryCaptureOpenMPLambdas(ValueDecl *V); + + /// Return true if the provided declaration \a VD should be captured by + /// reference. + /// \param Level Relative level of nested OpenMP construct for that the check + /// is performed. + /// \param OpenMPCaptureLevel Capture level within an OpenMP construct. + bool isOpenMPCapturedByRef(const ValueDecl *D, unsigned Level, + unsigned OpenMPCaptureLevel) const; - /// \return true if \p CD can be considered empty according to CUDA - /// (E.2.3.1 in CUDA 7.5 Programming guide). - bool isEmptyCudaConstructor(SourceLocation Loc, CXXConstructorDecl *CD); - bool isEmptyCudaDestructor(SourceLocation Loc, CXXDestructorDecl *CD); + /// Check if the specified variable is used in one of the private + /// clauses (private, firstprivate, lastprivate, reduction etc.) in OpenMP + /// constructs. + VarDecl *isOpenMPCapturedDecl(ValueDecl *D, bool CheckScopeInfo = false, + unsigned StopAt = 0); - // \brief Checks that initializers of \p Var satisfy CUDA restrictions. In - // case of error emits appropriate diagnostic and invalidates \p Var. - // - // \details CUDA allows only empty constructors as initializers for global - // variables (see E.2.3.1, CUDA 7.5). The same restriction also applies to all - // __shared__ variables whether they are local or not (they all are implicitly - // static in CUDA). One exception is that CUDA allows constant initializers - // for __constant__ and __device__ variables. - void checkAllowedCUDAInitializer(VarDecl *VD); + /// The member expression(this->fd) needs to be rebuilt in the template + /// instantiation to generate private copy for OpenMP when default + /// clause is used. The function will return true if default + /// cluse is used. + bool isOpenMPRebuildMemberExpr(ValueDecl *D); - /// Check whether NewFD is a valid overload for CUDA. Emits - /// diagnostics and invalidates NewFD if not. - void checkCUDATargetOverload(FunctionDecl *NewFD, - const LookupResult &Previous); - /// Copies target attributes from the template TD to the function FD. - void inheritCUDATargetAttrs(FunctionDecl *FD, const FunctionTemplateDecl &TD); + ExprResult getOpenMPCapturedExpr(VarDecl *Capture, ExprValueKind VK, + ExprObjectKind OK, SourceLocation Loc); - /// Returns the name of the launch configuration function. This is the name - /// of the function that will be called to configure kernel call, with the - /// parameters specified via <<<>>>. - std::string getCudaConfigureFuncName() const; + /// If the current region is a loop-based region, mark the start of the loop + /// construct. + void startOpenMPLoop(); - /// \name Code completion - //@{ - /// Describes the context in which code completion occurs. - enum ParserCompletionContext { - /// Code completion occurs at top-level or namespace context. - PCC_Namespace, - /// Code completion occurs within a class, struct, or union. - PCC_Class, - /// Code completion occurs within an Objective-C interface, protocol, - /// or category. - PCC_ObjCInterface, - /// Code completion occurs within an Objective-C implementation or - /// category implementation - PCC_ObjCImplementation, - /// Code completion occurs within the list of instance variables - /// in an Objective-C interface, protocol, category, or implementation. - PCC_ObjCInstanceVariableList, - /// Code completion occurs following one or more template - /// headers. - PCC_Template, - /// Code completion occurs following one or more template - /// headers within a class. - PCC_MemberTemplate, - /// Code completion occurs within an expression. - PCC_Expression, - /// Code completion occurs within a statement, which may - /// also be an expression or a declaration. - PCC_Statement, - /// Code completion occurs at the beginning of the - /// initialization statement (or expression) in a for loop. - PCC_ForInit, - /// Code completion occurs within the condition of an if, - /// while, switch, or for statement. - PCC_Condition, - /// Code completion occurs within the body of a function on a - /// recovery path, where we do not have a specific handle on our position - /// in the grammar. - PCC_RecoveryInFunction, - /// Code completion occurs where only a type is permitted. - PCC_Type, - /// Code completion occurs in a parenthesized expression, which - /// might also be a type cast. - PCC_ParenthesizedExpression, - /// Code completion occurs within a sequence of declaration - /// specifiers within a function, method, or block. - PCC_LocalDeclarationSpecifiers, - /// Code completion occurs at top-level in a REPL session - PCC_TopLevelOrExpression, - }; + /// If the current region is a range loop-based region, mark the start of the + /// loop construct. + void startOpenMPCXXRangeFor(); - void CodeCompleteModuleImport(SourceLocation ImportLoc, ModuleIdPath Path); - void CodeCompleteOrdinaryName(Scope *S, - ParserCompletionContext CompletionContext); - void CodeCompleteDeclSpec(Scope *S, DeclSpec &DS, - bool AllowNonIdentifiers, - bool AllowNestedNameSpecifiers); + /// Check if the specified variable is used in 'private' clause. + /// \param Level Relative level of nested OpenMP construct for that the check + /// is performed. + OpenMPClauseKind isOpenMPPrivateDecl(ValueDecl *D, unsigned Level, + unsigned CapLevel) const; - struct CodeCompleteExpressionData; - void CodeCompleteExpression(Scope *S, - const CodeCompleteExpressionData &Data); - void CodeCompleteExpression(Scope *S, QualType PreferredType, - bool IsParenthesized = false); - void CodeCompleteMemberReferenceExpr(Scope *S, Expr *Base, Expr *OtherOpBase, - SourceLocation OpLoc, bool IsArrow, - bool IsBaseExprStatement, - QualType PreferredType); - void CodeCompletePostfixExpression(Scope *S, ExprResult LHS, - QualType PreferredType); - void CodeCompleteTag(Scope *S, unsigned TagSpec); - void CodeCompleteTypeQualifiers(DeclSpec &DS); - void CodeCompleteFunctionQualifiers(DeclSpec &DS, Declarator &D, - const VirtSpecifiers *VS = nullptr); - void CodeCompleteBracketDeclarator(Scope *S); - void CodeCompleteCase(Scope *S); - enum class AttributeCompletion { - Attribute, - Scope, - None, - }; - void CodeCompleteAttribute( - AttributeCommonInfo::Syntax Syntax, - AttributeCompletion Completion = AttributeCompletion::Attribute, - const IdentifierInfo *Scope = nullptr); - /// Determines the preferred type of the current function argument, by - /// examining the signatures of all possible overloads. - /// Returns null if unknown or ambiguous, or if code completion is off. - /// - /// If the code completion point has been reached, also reports the function - /// signatures that were considered. - /// - /// FIXME: rename to GuessCallArgumentType to reduce confusion. - QualType ProduceCallSignatureHelp(Expr *Fn, ArrayRef Args, - SourceLocation OpenParLoc); - QualType ProduceConstructorSignatureHelp(QualType Type, SourceLocation Loc, - ArrayRef Args, - SourceLocation OpenParLoc, - bool Braced); - QualType ProduceCtorInitMemberSignatureHelp( - Decl *ConstructorDecl, CXXScopeSpec SS, ParsedType TemplateTypeTy, - ArrayRef ArgExprs, IdentifierInfo *II, SourceLocation OpenParLoc, - bool Braced); - QualType ProduceTemplateArgumentSignatureHelp( - TemplateTy, ArrayRef, SourceLocation LAngleLoc); - void CodeCompleteInitializer(Scope *S, Decl *D); - /// Trigger code completion for a record of \p BaseType. \p InitExprs are - /// expressions in the initializer list seen so far and \p D is the current - /// Designation being parsed. - void CodeCompleteDesignator(const QualType BaseType, - llvm::ArrayRef InitExprs, - const Designation &D); - void CodeCompleteAfterIf(Scope *S, bool IsBracedThen); + /// Sets OpenMP capture kind (OMPC_private, OMPC_firstprivate, OMPC_map etc.) + /// for \p FD based on DSA for the provided corresponding captured declaration + /// \p D. + void setOpenMPCaptureKind(FieldDecl *FD, const ValueDecl *D, unsigned Level); - void CodeCompleteQualifiedId(Scope *S, CXXScopeSpec &SS, bool EnteringContext, - bool IsUsingDeclaration, QualType BaseType, - QualType PreferredType); - void CodeCompleteUsing(Scope *S); - void CodeCompleteUsingDirective(Scope *S); - void CodeCompleteNamespaceDecl(Scope *S); - void CodeCompleteNamespaceAliasDecl(Scope *S); - void CodeCompleteOperatorName(Scope *S); - void CodeCompleteConstructorInitializer( - Decl *Constructor, - ArrayRef Initializers); + /// Check if the specified variable is captured by 'target' directive. + /// \param Level Relative level of nested OpenMP construct for that the check + /// is performed. + bool isOpenMPTargetCapturedDecl(const ValueDecl *D, unsigned Level, + unsigned CaptureLevel) const; - void CodeCompleteLambdaIntroducer(Scope *S, LambdaIntroducer &Intro, - bool AfterAmpersand); - void CodeCompleteAfterFunctionEquals(Declarator &D); + /// Check if the specified global variable must be captured by outer capture + /// regions. + /// \param Level Relative level of nested OpenMP construct for that + /// the check is performed. + bool isOpenMPGlobalCapturedDecl(ValueDecl *D, unsigned Level, + unsigned CaptureLevel) const; - void CodeCompleteObjCAtDirective(Scope *S); - void CodeCompleteObjCAtVisibility(Scope *S); - void CodeCompleteObjCAtStatement(Scope *S); - void CodeCompleteObjCAtExpression(Scope *S); - void CodeCompleteObjCPropertyFlags(Scope *S, ObjCDeclSpec &ODS); - void CodeCompleteObjCPropertyGetter(Scope *S); - void CodeCompleteObjCPropertySetter(Scope *S); - void CodeCompleteObjCPassingType(Scope *S, ObjCDeclSpec &DS, - bool IsParameter); - void CodeCompleteObjCMessageReceiver(Scope *S); - void CodeCompleteObjCSuperMessage(Scope *S, SourceLocation SuperLoc, - ArrayRef SelIdents, - bool AtArgumentExpression); - void CodeCompleteObjCClassMessage(Scope *S, ParsedType Receiver, - ArrayRef SelIdents, - bool AtArgumentExpression, - bool IsSuper = false); - void CodeCompleteObjCInstanceMessage(Scope *S, Expr *Receiver, - ArrayRef SelIdents, - bool AtArgumentExpression, - ObjCInterfaceDecl *Super = nullptr); - void CodeCompleteObjCForCollection(Scope *S, - DeclGroupPtrTy IterationVar); - void CodeCompleteObjCSelector(Scope *S, - ArrayRef SelIdents); - void CodeCompleteObjCProtocolReferences( - ArrayRef Protocols); - void CodeCompleteObjCProtocolDecl(Scope *S); - void CodeCompleteObjCInterfaceDecl(Scope *S); - void CodeCompleteObjCClassForwardDecl(Scope *S); - void CodeCompleteObjCSuperclass(Scope *S, - IdentifierInfo *ClassName, - SourceLocation ClassNameLoc); - void CodeCompleteObjCImplementationDecl(Scope *S); - void CodeCompleteObjCInterfaceCategory(Scope *S, - IdentifierInfo *ClassName, - SourceLocation ClassNameLoc); - void CodeCompleteObjCImplementationCategory(Scope *S, - IdentifierInfo *ClassName, - SourceLocation ClassNameLoc); - void CodeCompleteObjCPropertyDefinition(Scope *S); - void CodeCompleteObjCPropertySynthesizeIvar(Scope *S, - IdentifierInfo *PropertyName); - void CodeCompleteObjCMethodDecl(Scope *S, - std::optional IsInstanceMethod, - ParsedType ReturnType); - void CodeCompleteObjCMethodDeclSelector(Scope *S, - bool IsInstanceMethod, - bool AtParameterName, - ParsedType ReturnType, - ArrayRef SelIdents); - void CodeCompleteObjCClassPropertyRefExpr(Scope *S, IdentifierInfo &ClassName, - SourceLocation ClassNameLoc, - bool IsBaseExprStatement); - void CodeCompletePreprocessorDirective(bool InConditional); - void CodeCompleteInPreprocessorConditionalExclusion(Scope *S); - void CodeCompletePreprocessorMacroName(bool IsDefinition); - void CodeCompletePreprocessorExpression(); - void CodeCompletePreprocessorMacroArgument(Scope *S, - IdentifierInfo *Macro, - MacroInfo *MacroInfo, - unsigned Argument); - void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled); - void CodeCompleteNaturalLanguage(); - void CodeCompleteAvailabilityPlatformName(); - void GatherGlobalCodeCompletions(CodeCompletionAllocator &Allocator, - CodeCompletionTUInfo &CCTUInfo, - SmallVectorImpl &Results); - //@} + ExprResult PerformOpenMPImplicitIntegerConversion(SourceLocation OpLoc, + Expr *Op); + /// Called on start of new data sharing attribute block. + void StartOpenMPDSABlock(OpenMPDirectiveKind K, + const DeclarationNameInfo &DirName, Scope *CurScope, + SourceLocation Loc); + /// Start analysis of clauses. + void StartOpenMPClause(OpenMPClauseKind K); + /// End analysis of clauses. + void EndOpenMPClause(); + /// Called on end of data sharing attribute block. + void EndOpenMPDSABlock(Stmt *CurDirective); - //===--------------------------------------------------------------------===// - // Extra semantic analysis beyond the C type system + /// Check if the current region is an OpenMP loop region and if it is, + /// mark loop control variable, used in \p Init for loop initialization, as + /// private by default. + /// \param Init First part of the for loop. + void ActOnOpenMPLoopInitialization(SourceLocation ForLoc, Stmt *Init); -public: - SourceLocation getLocationOfStringLiteralByte(const StringLiteral *SL, - unsigned ByteNo) const; + /// Called on well-formed '\#pragma omp metadirective' after parsing + /// of the associated statement. + StmtResult ActOnOpenMPMetaDirective(ArrayRef Clauses, + Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc); - enum FormatArgumentPassingKind { - FAPK_Fixed, // values to format are fixed (no C-style variadic arguments) - FAPK_Variadic, // values to format are passed as variadic arguments - FAPK_VAList, // values to format are passed in a va_list - }; + // OpenMP directives and clauses. + /// Called on correct id-expression from the '#pragma omp + /// threadprivate'. + ExprResult ActOnOpenMPIdExpression(Scope *CurScope, CXXScopeSpec &ScopeSpec, + const DeclarationNameInfo &Id, + OpenMPDirectiveKind Kind); + /// Called on well-formed '#pragma omp threadprivate'. + DeclGroupPtrTy ActOnOpenMPThreadprivateDirective(SourceLocation Loc, + ArrayRef VarList); + /// Builds a new OpenMPThreadPrivateDecl and checks its correctness. + OMPThreadPrivateDecl *CheckOMPThreadPrivateDecl(SourceLocation Loc, + ArrayRef VarList); + /// Called on well-formed '#pragma omp allocate'. + DeclGroupPtrTy ActOnOpenMPAllocateDirective(SourceLocation Loc, + ArrayRef VarList, + ArrayRef Clauses, + DeclContext *Owner = nullptr); - // Used to grab the relevant information from a FormatAttr and a - // FunctionDeclaration. - struct FormatStringInfo { - unsigned FormatIdx; - unsigned FirstDataArg; - FormatArgumentPassingKind ArgPassingKind; - }; + /// Called on well-formed '#pragma omp [begin] assume[s]'. + void ActOnOpenMPAssumesDirective(SourceLocation Loc, + OpenMPDirectiveKind DKind, + ArrayRef Assumptions, + bool SkippedClauses); - static bool getFormatStringInfo(const FormatAttr *Format, bool IsCXXMember, - bool IsVariadic, FormatStringInfo *FSI); + /// Check if there is an active global `omp begin assumes` directive. + bool isInOpenMPAssumeScope() const { return !OMPAssumeScoped.empty(); } -private: - void CheckArrayAccess(const Expr *BaseExpr, const Expr *IndexExpr, - const ArraySubscriptExpr *ASE = nullptr, - bool AllowOnePastEnd = true, bool IndexNegated = false); - void CheckArrayAccess(const Expr *E); + /// Check if there is an active global `omp assumes` directive. + bool hasGlobalOpenMPAssumes() const { return !OMPAssumeGlobal.empty(); } - bool CheckFunctionCall(FunctionDecl *FDecl, CallExpr *TheCall, - const FunctionProtoType *Proto); - bool CheckObjCMethodCall(ObjCMethodDecl *Method, SourceLocation loc, - ArrayRef Args); - bool CheckPointerCall(NamedDecl *NDecl, CallExpr *TheCall, - const FunctionProtoType *Proto); - bool CheckOtherCall(CallExpr *TheCall, const FunctionProtoType *Proto); - void CheckConstructorCall(FunctionDecl *FDecl, QualType ThisType, - ArrayRef Args, - const FunctionProtoType *Proto, SourceLocation Loc); + /// Called on well-formed '#pragma omp end assumes'. + void ActOnOpenMPEndAssumesDirective(); - void checkAIXMemberAlignment(SourceLocation Loc, const Expr *Arg); + /// Called on well-formed '#pragma omp requires'. + DeclGroupPtrTy ActOnOpenMPRequiresDirective(SourceLocation Loc, + ArrayRef ClauseList); + /// Check restrictions on Requires directive + OMPRequiresDecl *CheckOMPRequiresDecl(SourceLocation Loc, + ArrayRef Clauses); + /// Check if the specified type is allowed to be used in 'omp declare + /// reduction' construct. + QualType ActOnOpenMPDeclareReductionType(SourceLocation TyLoc, + TypeResult ParsedType); + /// Called on start of '#pragma omp declare reduction'. + DeclGroupPtrTy ActOnOpenMPDeclareReductionDirectiveStart( + Scope *S, DeclContext *DC, DeclarationName Name, + ArrayRef> ReductionTypes, + AccessSpecifier AS, Decl *PrevDeclInScope = nullptr); + /// Initialize declare reduction construct initializer. + void ActOnOpenMPDeclareReductionCombinerStart(Scope *S, Decl *D); + /// Finish current declare reduction construct initializer. + void ActOnOpenMPDeclareReductionCombinerEnd(Decl *D, Expr *Combiner); + /// Initialize declare reduction construct initializer. + /// \return omp_priv variable. + VarDecl *ActOnOpenMPDeclareReductionInitializerStart(Scope *S, Decl *D); + /// Finish current declare reduction construct initializer. + void ActOnOpenMPDeclareReductionInitializerEnd(Decl *D, Expr *Initializer, + VarDecl *OmpPrivParm); + /// Called at the end of '#pragma omp declare reduction'. + DeclGroupPtrTy ActOnOpenMPDeclareReductionDirectiveEnd( + Scope *S, DeclGroupPtrTy DeclReductions, bool IsValid); - void CheckArgAlignment(SourceLocation Loc, NamedDecl *FDecl, - StringRef ParamName, QualType ArgTy, QualType ParamTy); + /// Check variable declaration in 'omp declare mapper' construct. + TypeResult ActOnOpenMPDeclareMapperVarDecl(Scope *S, Declarator &D); + /// Check if the specified type is allowed to be used in 'omp declare + /// mapper' construct. + QualType ActOnOpenMPDeclareMapperType(SourceLocation TyLoc, + TypeResult ParsedType); + /// Called on start of '#pragma omp declare mapper'. + DeclGroupPtrTy ActOnOpenMPDeclareMapperDirective( + Scope *S, DeclContext *DC, DeclarationName Name, QualType MapperType, + SourceLocation StartLoc, DeclarationName VN, AccessSpecifier AS, + Expr *MapperVarRef, ArrayRef Clauses, + Decl *PrevDeclInScope = nullptr); + /// Build the mapper variable of '#pragma omp declare mapper'. + ExprResult ActOnOpenMPDeclareMapperDirectiveVarDecl(Scope *S, + QualType MapperType, + SourceLocation StartLoc, + DeclarationName VN); + void ActOnOpenMPIteratorVarDecl(VarDecl *VD); + bool isOpenMPDeclareMapperVarDeclAllowed(const VarDecl *VD) const; + const ValueDecl *getOpenMPDeclareMapperVarName() const; - void checkCall(NamedDecl *FDecl, const FunctionProtoType *Proto, - const Expr *ThisArg, ArrayRef Args, - bool IsMemberFunction, SourceLocation Loc, SourceRange Range, - VariadicCallType CallType); + struct DeclareTargetContextInfo { + struct MapInfo { + OMPDeclareTargetDeclAttr::MapTypeTy MT; + SourceLocation Loc; + }; + /// Explicitly listed variables and functions in a 'to' or 'link' clause. + llvm::DenseMap ExplicitlyMapped; - bool CheckObjCString(Expr *Arg); - ExprResult CheckOSLogFormatStringArg(Expr *Arg); + /// The 'device_type' as parsed from the clause. + OMPDeclareTargetDeclAttr::DevTypeTy DT = OMPDeclareTargetDeclAttr::DT_Any; - ExprResult CheckBuiltinFunctionCall(FunctionDecl *FDecl, - unsigned BuiltinID, CallExpr *TheCall); + /// The directive kind, `begin declare target` or `declare target`. + OpenMPDirectiveKind Kind; - bool CheckTSBuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID, - CallExpr *TheCall); + /// The directive with indirect clause. + std::optional Indirect; - void checkFortifiedBuiltinMemoryFunction(FunctionDecl *FD, CallExpr *TheCall); + /// The directive location. + SourceLocation Loc; - bool CheckARMBuiltinExclusiveCall(unsigned BuiltinID, CallExpr *TheCall, - unsigned MaxWidth); - bool CheckNeonBuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID, - CallExpr *TheCall); - bool CheckMVEBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); - bool CheckSVEBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); - bool ParseSVEImmChecks(CallExpr *TheCall, - SmallVector, 3> &ImmChecks); - bool CheckSMEBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); - bool CheckCDEBuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID, - CallExpr *TheCall); - bool CheckARMCoprocessorImmediate(const TargetInfo &TI, const Expr *CoprocArg, - bool WantCDE); - bool CheckARMBuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID, - CallExpr *TheCall); + DeclareTargetContextInfo(OpenMPDirectiveKind Kind, SourceLocation Loc) + : Kind(Kind), Loc(Loc) {} + }; - bool CheckAArch64BuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID, - CallExpr *TheCall); - bool CheckBPFBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); - bool CheckHexagonBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); - bool CheckHexagonBuiltinArgument(unsigned BuiltinID, CallExpr *TheCall); - bool CheckMipsBuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID, - CallExpr *TheCall); - bool CheckMipsBuiltinCpu(const TargetInfo &TI, unsigned BuiltinID, - CallExpr *TheCall); - bool CheckMipsBuiltinArgument(unsigned BuiltinID, CallExpr *TheCall); - bool CheckSystemZBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); - bool CheckX86BuiltinRoundingOrSAE(unsigned BuiltinID, CallExpr *TheCall); - bool CheckX86BuiltinGatherScatterScale(unsigned BuiltinID, CallExpr *TheCall); - bool CheckX86BuiltinTileArguments(unsigned BuiltinID, CallExpr *TheCall); - bool CheckX86BuiltinTileArgumentsRange(CallExpr *TheCall, - ArrayRef ArgNums); - bool CheckX86BuiltinTileDuplicate(CallExpr *TheCall, ArrayRef ArgNums); - bool CheckX86BuiltinTileRangeAndDuplicate(CallExpr *TheCall, - ArrayRef ArgNums); - bool CheckX86BuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID, - CallExpr *TheCall); - bool CheckPPCBuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID, - CallExpr *TheCall); - bool CheckAMDGCNBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); - bool CheckHLSLBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); - bool CheckRISCVLMUL(CallExpr *TheCall, unsigned ArgNum); - bool CheckRISCVBuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID, - CallExpr *TheCall); - void checkRVVTypeSupport(QualType Ty, SourceLocation Loc, Decl *D); - bool CheckLoongArchBuiltinFunctionCall(const TargetInfo &TI, - unsigned BuiltinID, CallExpr *TheCall); - bool CheckWebAssemblyBuiltinFunctionCall(const TargetInfo &TI, - unsigned BuiltinID, - CallExpr *TheCall); - bool CheckNVPTXBuiltinFunctionCall(const TargetInfo &TI, unsigned BuiltinID, - CallExpr *TheCall); + /// Called on the start of target region i.e. '#pragma omp declare target'. + bool ActOnStartOpenMPDeclareTargetContext(DeclareTargetContextInfo &DTCI); - bool SemaBuiltinVAStart(unsigned BuiltinID, CallExpr *TheCall); - bool SemaBuiltinVAStartARMMicrosoft(CallExpr *Call); - bool SemaBuiltinUnorderedCompare(CallExpr *TheCall, unsigned BuiltinID); - bool SemaBuiltinFPClassification(CallExpr *TheCall, unsigned NumArgs, - unsigned BuiltinID); - bool SemaBuiltinComplex(CallExpr *TheCall); - bool SemaBuiltinVSX(CallExpr *TheCall); - bool SemaBuiltinOSLogFormat(CallExpr *TheCall); - bool SemaValueIsRunOfOnes(CallExpr *TheCall, unsigned ArgNum); + /// Called at the end of target region i.e. '#pragma omp end declare target'. + const DeclareTargetContextInfo ActOnOpenMPEndDeclareTargetDirective(); -public: - bool IsLayoutCompatible(QualType T1, QualType T2) const; + /// Called once a target context is completed, that can be when a + /// '#pragma omp end declare target' was encountered or when a + /// '#pragma omp declare target' without declaration-definition-seq was + /// encountered. + void ActOnFinishedOpenMPDeclareTargetContext(DeclareTargetContextInfo &DTCI); - // Used by C++ template instantiation. - ExprResult SemaBuiltinShuffleVector(CallExpr *TheCall); - ExprResult SemaConvertVectorExpr(Expr *E, TypeSourceInfo *TInfo, - SourceLocation BuiltinLoc, - SourceLocation RParenLoc); + /// Report unterminated 'omp declare target' or 'omp begin declare target' at + /// the end of a compilation unit. + void DiagnoseUnterminatedOpenMPDeclareTarget(); -private: - bool SemaBuiltinPrefetch(CallExpr *TheCall); - bool SemaBuiltinAllocaWithAlign(CallExpr *TheCall); - bool SemaBuiltinArithmeticFence(CallExpr *TheCall); - bool SemaBuiltinAssume(CallExpr *TheCall); - bool SemaBuiltinAssumeAligned(CallExpr *TheCall); - bool SemaBuiltinLongjmp(CallExpr *TheCall); - bool SemaBuiltinSetjmp(CallExpr *TheCall); - ExprResult SemaBuiltinAtomicOverloaded(ExprResult TheCallResult); - ExprResult SemaBuiltinNontemporalOverloaded(ExprResult TheCallResult); - ExprResult SemaAtomicOpsOverloaded(ExprResult TheCallResult, - AtomicExpr::AtomicOp Op); - ExprResult SemaBuiltinOperatorNewDeleteOverloaded(ExprResult TheCallResult, - bool IsDelete); - bool SemaBuiltinConstantArg(CallExpr *TheCall, int ArgNum, - llvm::APSInt &Result); - bool SemaBuiltinConstantArgRange(CallExpr *TheCall, int ArgNum, int Low, - int High, bool RangeIsError = true); - bool SemaBuiltinConstantArgMultiple(CallExpr *TheCall, int ArgNum, - unsigned Multiple); - bool SemaBuiltinConstantArgPower2(CallExpr *TheCall, int ArgNum); - bool SemaBuiltinConstantArgShiftedByte(CallExpr *TheCall, int ArgNum, - unsigned ArgBits); - bool SemaBuiltinConstantArgShiftedByteOrXXFF(CallExpr *TheCall, int ArgNum, - unsigned ArgBits); - bool SemaBuiltinARMSpecialReg(unsigned BuiltinID, CallExpr *TheCall, - int ArgNum, unsigned ExpectedFieldNum, - bool AllowName); - bool SemaBuiltinARMMemoryTaggingCall(unsigned BuiltinID, CallExpr *TheCall); - bool SemaBuiltinPPCMMACall(CallExpr *TheCall, unsigned BuiltinID, - const char *TypeDesc); + /// Searches for the provided declaration name for OpenMP declare target + /// directive. + NamedDecl *lookupOpenMPDeclareTargetName(Scope *CurScope, + CXXScopeSpec &ScopeSpec, + const DeclarationNameInfo &Id); - bool CheckPPCMMAType(QualType Type, SourceLocation TypeLoc); + /// Called on correct id-expression from the '#pragma omp declare target'. + void ActOnOpenMPDeclareTargetName(NamedDecl *ND, SourceLocation Loc, + OMPDeclareTargetDeclAttr::MapTypeTy MT, + DeclareTargetContextInfo &DTCI); - bool SemaBuiltinVectorMath(CallExpr *TheCall, QualType &Res); - bool SemaBuiltinVectorToScalarMath(CallExpr *TheCall); - bool SemaBuiltinElementwiseMath(CallExpr *TheCall); - bool SemaBuiltinElementwiseTernaryMath(CallExpr *TheCall, - bool CheckForFloatArgs = true); - bool PrepareBuiltinElementwiseMathOneArgCall(CallExpr *TheCall); - bool PrepareBuiltinReduceMathOneArgCall(CallExpr *TheCall); + /// Check declaration inside target region. + void + checkDeclIsAllowedInOpenMPTarget(Expr *E, Decl *D, + SourceLocation IdLoc = SourceLocation()); - bool SemaBuiltinNonDeterministicValue(CallExpr *TheCall); + /// Adds OMPDeclareTargetDeclAttr to referenced variables in declare target + /// directive. + void ActOnOpenMPDeclareTargetInitializer(Decl *D); - // Matrix builtin handling. - ExprResult SemaBuiltinMatrixTranspose(CallExpr *TheCall, - ExprResult CallResult); - ExprResult SemaBuiltinMatrixColumnMajorLoad(CallExpr *TheCall, - ExprResult CallResult); - ExprResult SemaBuiltinMatrixColumnMajorStore(CallExpr *TheCall, - ExprResult CallResult); + /// Finishes analysis of the deferred functions calls that may be declared as + /// host/nohost during device/host compilation. + void finalizeOpenMPDelayedAnalysis(const FunctionDecl *Caller, + const FunctionDecl *Callee, + SourceLocation Loc); - // WebAssembly builtin handling. - bool BuiltinWasmRefNullExtern(CallExpr *TheCall); - bool BuiltinWasmRefNullFunc(CallExpr *TheCall); - bool BuiltinWasmTableGet(CallExpr *TheCall); - bool BuiltinWasmTableSet(CallExpr *TheCall); - bool BuiltinWasmTableSize(CallExpr *TheCall); - bool BuiltinWasmTableGrow(CallExpr *TheCall); - bool BuiltinWasmTableFill(CallExpr *TheCall); - bool BuiltinWasmTableCopy(CallExpr *TheCall); + /// Return true if currently in OpenMP task with untied clause context. + bool isInOpenMPTaskUntiedContext() const; -public: - enum FormatStringType { - FST_Scanf, - FST_Printf, - FST_NSString, - FST_Strftime, - FST_Strfmon, - FST_Kprintf, - FST_FreeBSDKPrintf, - FST_OSTrace, - FST_OSLog, - FST_Unknown - }; - static FormatStringType GetFormatStringType(const FormatAttr *Format); + /// Return true inside OpenMP declare target region. + bool isInOpenMPDeclareTargetContext() const { + return !DeclareTargetNesting.empty(); + } + /// Return true inside OpenMP target region. + bool isInOpenMPTargetExecutionDirective() const; - bool FormatStringHasSArg(const StringLiteral *FExpr); + /// Return the number of captured regions created for an OpenMP directive. + static int getOpenMPCaptureLevels(OpenMPDirectiveKind Kind); - static bool GetFormatNSStringIdx(const FormatAttr *Format, unsigned &Idx); + /// Initialization of captured region for OpenMP region. + void ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope); -private: - bool CheckFormatArguments(const FormatAttr *Format, - ArrayRef Args, bool IsCXXMember, - VariadicCallType CallType, SourceLocation Loc, - SourceRange Range, - llvm::SmallBitVector &CheckedVarArgs); - bool CheckFormatArguments(ArrayRef Args, - FormatArgumentPassingKind FAPK, unsigned format_idx, - unsigned firstDataArg, FormatStringType Type, - VariadicCallType CallType, SourceLocation Loc, - SourceRange range, - llvm::SmallBitVector &CheckedVarArgs); + /// Called for syntactical loops (ForStmt or CXXForRangeStmt) associated to + /// an OpenMP loop directive. + StmtResult ActOnOpenMPCanonicalLoop(Stmt *AStmt); - void CheckInfNaNFunction(const CallExpr *Call, const FunctionDecl *FDecl); + /// Process a canonical OpenMP loop nest that can either be a canonical + /// literal loop (ForStmt or CXXForRangeStmt), or the generated loop of an + /// OpenMP loop transformation construct. + StmtResult ActOnOpenMPLoopnest(Stmt *AStmt); - void CheckAbsoluteValueFunction(const CallExpr *Call, - const FunctionDecl *FDecl); + /// End of OpenMP region. + /// + /// \param S Statement associated with the current OpenMP region. + /// \param Clauses List of clauses for the current OpenMP region. + /// + /// \returns Statement for finished OpenMP region. + StmtResult ActOnOpenMPRegionEnd(StmtResult S, ArrayRef Clauses); + StmtResult ActOnOpenMPExecutableDirective( + OpenMPDirectiveKind Kind, const DeclarationNameInfo &DirName, + OpenMPDirectiveKind CancelRegion, ArrayRef Clauses, + Stmt *AStmt, SourceLocation StartLoc, SourceLocation EndLoc, + OpenMPDirectiveKind PrevMappedDirective = llvm::omp::OMPD_unknown); + /// Called on well-formed '\#pragma omp parallel' after parsing + /// of the associated statement. + StmtResult ActOnOpenMPParallelDirective(ArrayRef Clauses, + Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc); + using VarsWithInheritedDSAType = + llvm::SmallDenseMap; + /// Called on well-formed '\#pragma omp simd' after parsing + /// of the associated statement. + StmtResult + ActOnOpenMPSimdDirective(ArrayRef Clauses, Stmt *AStmt, + SourceLocation StartLoc, SourceLocation EndLoc, + VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '#pragma omp tile' after parsing of its clauses and + /// the associated statement. + StmtResult ActOnOpenMPTileDirective(ArrayRef Clauses, + Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '#pragma omp unroll' after parsing of its clauses + /// and the associated statement. + StmtResult ActOnOpenMPUnrollDirective(ArrayRef Clauses, + Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp for' after parsing + /// of the associated statement. + StmtResult + ActOnOpenMPForDirective(ArrayRef Clauses, Stmt *AStmt, + SourceLocation StartLoc, SourceLocation EndLoc, + VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp for simd' after parsing + /// of the associated statement. + StmtResult + ActOnOpenMPForSimdDirective(ArrayRef Clauses, Stmt *AStmt, + SourceLocation StartLoc, SourceLocation EndLoc, + VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp sections' after parsing + /// of the associated statement. + StmtResult ActOnOpenMPSectionsDirective(ArrayRef Clauses, + Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp section' after parsing of the + /// associated statement. + StmtResult ActOnOpenMPSectionDirective(Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp scope' after parsing of the + /// associated statement. + StmtResult ActOnOpenMPScopeDirective(ArrayRef Clauses, + Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp single' after parsing of the + /// associated statement. + StmtResult ActOnOpenMPSingleDirective(ArrayRef Clauses, + Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp master' after parsing of the + /// associated statement. + StmtResult ActOnOpenMPMasterDirective(Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp critical' after parsing of the + /// associated statement. + StmtResult ActOnOpenMPCriticalDirective(const DeclarationNameInfo &DirName, + ArrayRef Clauses, + Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp parallel for' after parsing + /// of the associated statement. + StmtResult ActOnOpenMPParallelForDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp parallel for simd' after + /// parsing of the associated statement. + StmtResult ActOnOpenMPParallelForSimdDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp parallel master' after + /// parsing of the associated statement. + StmtResult ActOnOpenMPParallelMasterDirective(ArrayRef Clauses, + Stmt *AStmt, + SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp parallel masked' after + /// parsing of the associated statement. + StmtResult ActOnOpenMPParallelMaskedDirective(ArrayRef Clauses, + Stmt *AStmt, + SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp parallel sections' after + /// parsing of the associated statement. + StmtResult ActOnOpenMPParallelSectionsDirective(ArrayRef Clauses, + Stmt *AStmt, + SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp task' after parsing of the + /// associated statement. + StmtResult ActOnOpenMPTaskDirective(ArrayRef Clauses, + Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp taskyield'. + StmtResult ActOnOpenMPTaskyieldDirective(SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp error'. + /// Error direcitive is allowed in both declared and excutable contexts. + /// Adding InExContext to identify which context is called from. + StmtResult ActOnOpenMPErrorDirective(ArrayRef Clauses, + SourceLocation StartLoc, + SourceLocation EndLoc, + bool InExContext = true); + /// Called on well-formed '\#pragma omp barrier'. + StmtResult ActOnOpenMPBarrierDirective(SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp taskwait'. + StmtResult ActOnOpenMPTaskwaitDirective(ArrayRef Clauses, + SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp taskgroup'. + StmtResult ActOnOpenMPTaskgroupDirective(ArrayRef Clauses, + Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp flush'. + StmtResult ActOnOpenMPFlushDirective(ArrayRef Clauses, + SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp depobj'. + StmtResult ActOnOpenMPDepobjDirective(ArrayRef Clauses, + SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp scan'. + StmtResult ActOnOpenMPScanDirective(ArrayRef Clauses, + SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp ordered' after parsing of the + /// associated statement. + StmtResult ActOnOpenMPOrderedDirective(ArrayRef Clauses, + Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp atomic' after parsing of the + /// associated statement. + StmtResult ActOnOpenMPAtomicDirective(ArrayRef Clauses, + Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp target' after parsing of the + /// associated statement. + StmtResult ActOnOpenMPTargetDirective(ArrayRef Clauses, + Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp target data' after parsing of + /// the associated statement. + StmtResult ActOnOpenMPTargetDataDirective(ArrayRef Clauses, + Stmt *AStmt, + SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp target enter data' after + /// parsing of the associated statement. + StmtResult ActOnOpenMPTargetEnterDataDirective(ArrayRef Clauses, + SourceLocation StartLoc, + SourceLocation EndLoc, + Stmt *AStmt); + /// Called on well-formed '\#pragma omp target exit data' after + /// parsing of the associated statement. + StmtResult ActOnOpenMPTargetExitDataDirective(ArrayRef Clauses, + SourceLocation StartLoc, + SourceLocation EndLoc, + Stmt *AStmt); + /// Called on well-formed '\#pragma omp target parallel' after + /// parsing of the associated statement. + StmtResult ActOnOpenMPTargetParallelDirective(ArrayRef Clauses, + Stmt *AStmt, + SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp target parallel for' after + /// parsing of the associated statement. + StmtResult ActOnOpenMPTargetParallelForDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp teams' after parsing of the + /// associated statement. + StmtResult ActOnOpenMPTeamsDirective(ArrayRef Clauses, + Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp teams loop' after parsing of the + /// associated statement. + StmtResult ActOnOpenMPTeamsGenericLoopDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp target teams loop' after parsing of + /// the associated statement. + StmtResult ActOnOpenMPTargetTeamsGenericLoopDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp parallel loop' after parsing of the + /// associated statement. + StmtResult ActOnOpenMPParallelGenericLoopDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp target parallel loop' after parsing + /// of the associated statement. + StmtResult ActOnOpenMPTargetParallelGenericLoopDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp cancellation point'. + StmtResult + ActOnOpenMPCancellationPointDirective(SourceLocation StartLoc, + SourceLocation EndLoc, + OpenMPDirectiveKind CancelRegion); + /// Called on well-formed '\#pragma omp cancel'. + StmtResult ActOnOpenMPCancelDirective(ArrayRef Clauses, + SourceLocation StartLoc, + SourceLocation EndLoc, + OpenMPDirectiveKind CancelRegion); + /// Called on well-formed '\#pragma omp taskloop' after parsing of the + /// associated statement. + StmtResult + ActOnOpenMPTaskLoopDirective(ArrayRef Clauses, Stmt *AStmt, + SourceLocation StartLoc, SourceLocation EndLoc, + VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp taskloop simd' after parsing of + /// the associated statement. + StmtResult ActOnOpenMPTaskLoopSimdDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp master taskloop' after parsing of the + /// associated statement. + StmtResult ActOnOpenMPMasterTaskLoopDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp master taskloop simd' after parsing of + /// the associated statement. + StmtResult ActOnOpenMPMasterTaskLoopSimdDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp parallel master taskloop' after + /// parsing of the associated statement. + StmtResult ActOnOpenMPParallelMasterTaskLoopDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp parallel master taskloop simd' after + /// parsing of the associated statement. + StmtResult ActOnOpenMPParallelMasterTaskLoopSimdDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp masked taskloop' after parsing of the + /// associated statement. + StmtResult ActOnOpenMPMaskedTaskLoopDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp masked taskloop simd' after parsing of + /// the associated statement. + StmtResult ActOnOpenMPMaskedTaskLoopSimdDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp parallel masked taskloop' after + /// parsing of the associated statement. + StmtResult ActOnOpenMPParallelMaskedTaskLoopDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp parallel masked taskloop simd' after + /// parsing of the associated statement. + StmtResult ActOnOpenMPParallelMaskedTaskLoopSimdDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp distribute' after parsing + /// of the associated statement. + StmtResult + ActOnOpenMPDistributeDirective(ArrayRef Clauses, Stmt *AStmt, + SourceLocation StartLoc, SourceLocation EndLoc, + VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp target update'. + StmtResult ActOnOpenMPTargetUpdateDirective(ArrayRef Clauses, + SourceLocation StartLoc, + SourceLocation EndLoc, + Stmt *AStmt); + /// Called on well-formed '\#pragma omp distribute parallel for' after + /// parsing of the associated statement. + StmtResult ActOnOpenMPDistributeParallelForDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp distribute parallel for simd' + /// after parsing of the associated statement. + StmtResult ActOnOpenMPDistributeParallelForSimdDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp distribute simd' after + /// parsing of the associated statement. + StmtResult ActOnOpenMPDistributeSimdDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp target parallel for simd' after + /// parsing of the associated statement. + StmtResult ActOnOpenMPTargetParallelForSimdDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp target simd' after parsing of + /// the associated statement. + StmtResult + ActOnOpenMPTargetSimdDirective(ArrayRef Clauses, Stmt *AStmt, + SourceLocation StartLoc, SourceLocation EndLoc, + VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp teams distribute' after parsing of + /// the associated statement. + StmtResult ActOnOpenMPTeamsDistributeDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp teams distribute simd' after parsing + /// of the associated statement. + StmtResult ActOnOpenMPTeamsDistributeSimdDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp teams distribute parallel for simd' + /// after parsing of the associated statement. + StmtResult ActOnOpenMPTeamsDistributeParallelForSimdDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp teams distribute parallel for' + /// after parsing of the associated statement. + StmtResult ActOnOpenMPTeamsDistributeParallelForDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp target teams' after parsing of the + /// associated statement. + StmtResult ActOnOpenMPTargetTeamsDirective(ArrayRef Clauses, + Stmt *AStmt, + SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp target teams distribute' after parsing + /// of the associated statement. + StmtResult ActOnOpenMPTargetTeamsDistributeDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp target teams distribute parallel for' + /// after parsing of the associated statement. + StmtResult ActOnOpenMPTargetTeamsDistributeParallelForDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp target teams distribute parallel for + /// simd' after parsing of the associated statement. + StmtResult ActOnOpenMPTargetTeamsDistributeParallelForSimdDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp target teams distribute simd' after + /// parsing of the associated statement. + StmtResult ActOnOpenMPTargetTeamsDistributeSimdDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); + /// Called on well-formed '\#pragma omp interop'. + StmtResult ActOnOpenMPInteropDirective(ArrayRef Clauses, + SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp dispatch' after parsing of the + // /associated statement. + StmtResult ActOnOpenMPDispatchDirective(ArrayRef Clauses, + Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed '\#pragma omp masked' after parsing of the + // /associated statement. + StmtResult ActOnOpenMPMaskedDirective(ArrayRef Clauses, + Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc); - void CheckMaxUnsignedZero(const CallExpr *Call, const FunctionDecl *FDecl); + /// Called on well-formed '\#pragma omp loop' after parsing of the + /// associated statement. + StmtResult ActOnOpenMPGenericLoopDirective( + ArrayRef Clauses, Stmt *AStmt, SourceLocation StartLoc, + SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA); - void CheckMemaccessArguments(const CallExpr *Call, - unsigned BId, - IdentifierInfo *FnName); + /// Checks correctness of linear modifiers. + bool CheckOpenMPLinearModifier(OpenMPLinearClauseKind LinKind, + SourceLocation LinLoc); + /// Checks that the specified declaration matches requirements for the linear + /// decls. + bool CheckOpenMPLinearDecl(const ValueDecl *D, SourceLocation ELoc, + OpenMPLinearClauseKind LinKind, QualType Type, + bool IsDeclareSimd = false); - void CheckStrlcpycatArguments(const CallExpr *Call, - IdentifierInfo *FnName); + /// Called on well-formed '\#pragma omp declare simd' after parsing of + /// the associated method/function. + DeclGroupPtrTy ActOnOpenMPDeclareSimdDirective( + DeclGroupPtrTy DG, OMPDeclareSimdDeclAttr::BranchStateTy BS, + Expr *Simdlen, ArrayRef Uniforms, ArrayRef Aligneds, + ArrayRef Alignments, ArrayRef Linears, + ArrayRef LinModifiers, ArrayRef Steps, SourceRange SR); - void CheckStrncatArguments(const CallExpr *Call, - IdentifierInfo *FnName); + /// Checks '\#pragma omp declare variant' variant function and original + /// functions after parsing of the associated method/function. + /// \param DG Function declaration to which declare variant directive is + /// applied to. + /// \param VariantRef Expression that references the variant function, which + /// must be used instead of the original one, specified in \p DG. + /// \param TI The trait info object representing the match clause. + /// \param NumAppendArgs The number of omp_interop_t arguments to account for + /// in checking. + /// \returns std::nullopt, if the function/variant function are not compatible + /// with the pragma, pair of original function/variant ref expression + /// otherwise. + std::optional> + checkOpenMPDeclareVariantFunction(DeclGroupPtrTy DG, Expr *VariantRef, + OMPTraitInfo &TI, unsigned NumAppendArgs, + SourceRange SR); - void CheckFreeArguments(const CallExpr *E); + /// Called on well-formed '\#pragma omp declare variant' after parsing of + /// the associated method/function. + /// \param FD Function declaration to which declare variant directive is + /// applied to. + /// \param VariantRef Expression that references the variant function, which + /// must be used instead of the original one, specified in \p DG. + /// \param TI The context traits associated with the function variant. + /// \param AdjustArgsNothing The list of 'nothing' arguments. + /// \param AdjustArgsNeedDevicePtr The list of 'need_device_ptr' arguments. + /// \param AppendArgs The list of 'append_args' arguments. + /// \param AdjustArgsLoc The Location of an 'adjust_args' clause. + /// \param AppendArgsLoc The Location of an 'append_args' clause. + /// \param SR The SourceRange of the 'declare variant' directive. + void ActOnOpenMPDeclareVariantDirective( + FunctionDecl *FD, Expr *VariantRef, OMPTraitInfo &TI, + ArrayRef AdjustArgsNothing, + ArrayRef AdjustArgsNeedDevicePtr, + ArrayRef AppendArgs, SourceLocation AdjustArgsLoc, + SourceLocation AppendArgsLoc, SourceRange SR); - void CheckReturnValExpr(Expr *RetValExp, QualType lhsType, - SourceLocation ReturnLoc, - bool isObjCMethod = false, - const AttrVec *Attrs = nullptr, - const FunctionDecl *FD = nullptr); + OMPClause *ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind, Expr *Expr, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'allocator' clause. + OMPClause *ActOnOpenMPAllocatorClause(Expr *Allocator, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'if' clause. + OMPClause *ActOnOpenMPIfClause(OpenMPDirectiveKind NameModifier, + Expr *Condition, SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation NameModifierLoc, + SourceLocation ColonLoc, + SourceLocation EndLoc); + /// Called on well-formed 'final' clause. + OMPClause *ActOnOpenMPFinalClause(Expr *Condition, SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'num_threads' clause. + OMPClause *ActOnOpenMPNumThreadsClause(Expr *NumThreads, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'align' clause. + OMPClause *ActOnOpenMPAlignClause(Expr *Alignment, SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'safelen' clause. + OMPClause *ActOnOpenMPSafelenClause(Expr *Length, SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'simdlen' clause. + OMPClause *ActOnOpenMPSimdlenClause(Expr *Length, SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-form 'sizes' clause. + OMPClause *ActOnOpenMPSizesClause(ArrayRef SizeExprs, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-form 'full' clauses. + OMPClause *ActOnOpenMPFullClause(SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-form 'partial' clauses. + OMPClause *ActOnOpenMPPartialClause(Expr *FactorExpr, SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'collapse' clause. + OMPClause *ActOnOpenMPCollapseClause(Expr *NumForLoops, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'ordered' clause. + OMPClause * + ActOnOpenMPOrderedClause(SourceLocation StartLoc, SourceLocation EndLoc, + SourceLocation LParenLoc = SourceLocation(), + Expr *NumForLoops = nullptr); + /// Called on well-formed 'grainsize' clause. + OMPClause *ActOnOpenMPGrainsizeClause(OpenMPGrainsizeClauseModifier Modifier, + Expr *Size, SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation ModifierLoc, + SourceLocation EndLoc); + /// Called on well-formed 'num_tasks' clause. + OMPClause *ActOnOpenMPNumTasksClause(OpenMPNumTasksClauseModifier Modifier, + Expr *NumTasks, SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation ModifierLoc, + SourceLocation EndLoc); + /// Called on well-formed 'hint' clause. + OMPClause *ActOnOpenMPHintClause(Expr *Hint, SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'detach' clause. + OMPClause *ActOnOpenMPDetachClause(Expr *Evt, SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); -public: - void CheckFloatComparison(SourceLocation Loc, Expr *LHS, Expr *RHS, - BinaryOperatorKind Opcode); + OMPClause *ActOnOpenMPSimpleClause(OpenMPClauseKind Kind, unsigned Argument, + SourceLocation ArgumentLoc, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'when' clause. + OMPClause *ActOnOpenMPWhenClause(OMPTraitInfo &TI, SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'default' clause. + OMPClause *ActOnOpenMPDefaultClause(llvm::omp::DefaultKind Kind, + SourceLocation KindLoc, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'proc_bind' clause. + OMPClause *ActOnOpenMPProcBindClause(llvm::omp::ProcBindKind Kind, + SourceLocation KindLoc, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'order' clause. + OMPClause *ActOnOpenMPOrderClause(OpenMPOrderClauseModifier Modifier, + OpenMPOrderClauseKind Kind, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation MLoc, SourceLocation KindLoc, + SourceLocation EndLoc); + /// Called on well-formed 'update' clause. + OMPClause *ActOnOpenMPUpdateClause(OpenMPDependClauseKind Kind, + SourceLocation KindLoc, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); -private: - void CheckImplicitConversions(Expr *E, SourceLocation CC = SourceLocation()); - void CheckBoolLikeConversion(Expr *E, SourceLocation CC); - void CheckForIntOverflow(const Expr *E); - void CheckUnsequencedOperations(const Expr *E); + OMPClause *ActOnOpenMPSingleExprWithArgClause( + OpenMPClauseKind Kind, ArrayRef Arguments, Expr *Expr, + SourceLocation StartLoc, SourceLocation LParenLoc, + ArrayRef ArgumentsLoc, SourceLocation DelimLoc, + SourceLocation EndLoc); + /// Called on well-formed 'schedule' clause. + OMPClause *ActOnOpenMPScheduleClause( + OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2, + OpenMPScheduleClauseKind Kind, Expr *ChunkSize, SourceLocation StartLoc, + SourceLocation LParenLoc, SourceLocation M1Loc, SourceLocation M2Loc, + SourceLocation KindLoc, SourceLocation CommaLoc, SourceLocation EndLoc); - /// Perform semantic checks on a completed expression. This will either - /// be a full-expression or a default argument expression. - void CheckCompletedExpr(Expr *E, SourceLocation CheckLoc = SourceLocation(), - bool IsConstexpr = false); + OMPClause *ActOnOpenMPClause(OpenMPClauseKind Kind, SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed 'nowait' clause. + OMPClause *ActOnOpenMPNowaitClause(SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed 'untied' clause. + OMPClause *ActOnOpenMPUntiedClause(SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed 'mergeable' clause. + OMPClause *ActOnOpenMPMergeableClause(SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed 'read' clause. + OMPClause *ActOnOpenMPReadClause(SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed 'write' clause. + OMPClause *ActOnOpenMPWriteClause(SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed 'update' clause. + OMPClause *ActOnOpenMPUpdateClause(SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed 'capture' clause. + OMPClause *ActOnOpenMPCaptureClause(SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed 'compare' clause. + OMPClause *ActOnOpenMPCompareClause(SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed 'fail' clause. + OMPClause *ActOnOpenMPFailClause(SourceLocation StartLoc, + SourceLocation EndLoc); + OMPClause *ActOnOpenMPFailClause(OpenMPClauseKind Kind, + SourceLocation KindLoc, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); - void CheckBitFieldInitialization(SourceLocation InitLoc, FieldDecl *Field, - Expr *Init); + /// Called on well-formed 'seq_cst' clause. + OMPClause *ActOnOpenMPSeqCstClause(SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed 'acq_rel' clause. + OMPClause *ActOnOpenMPAcqRelClause(SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed 'acquire' clause. + OMPClause *ActOnOpenMPAcquireClause(SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed 'release' clause. + OMPClause *ActOnOpenMPReleaseClause(SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed 'relaxed' clause. + OMPClause *ActOnOpenMPRelaxedClause(SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed 'weak' clause. + OMPClause *ActOnOpenMPWeakClause(SourceLocation StartLoc, + SourceLocation EndLoc); - /// Check if there is a field shadowing. - void CheckShadowInheritedFields(const SourceLocation &Loc, - DeclarationName FieldName, - const CXXRecordDecl *RD, - bool DeclIsField = true); + /// Called on well-formed 'init' clause. + OMPClause * + ActOnOpenMPInitClause(Expr *InteropVar, OMPInteropInfo &InteropInfo, + SourceLocation StartLoc, SourceLocation LParenLoc, + SourceLocation VarLoc, SourceLocation EndLoc); - /// Check if the given expression contains 'break' or 'continue' - /// statement that produces control flow different from GCC. - void CheckBreakContinueBinding(Expr *E); + /// Called on well-formed 'use' clause. + OMPClause *ActOnOpenMPUseClause(Expr *InteropVar, SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation VarLoc, SourceLocation EndLoc); - /// Check whether receiver is mutable ObjC container which - /// attempts to add itself into the container - void CheckObjCCircularContainer(ObjCMessageExpr *Message); + /// Called on well-formed 'destroy' clause. + OMPClause *ActOnOpenMPDestroyClause(Expr *InteropVar, SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation VarLoc, + SourceLocation EndLoc); + /// Called on well-formed 'novariants' clause. + OMPClause *ActOnOpenMPNovariantsClause(Expr *Condition, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'nocontext' clause. + OMPClause *ActOnOpenMPNocontextClause(Expr *Condition, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'filter' clause. + OMPClause *ActOnOpenMPFilterClause(Expr *ThreadID, SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'threads' clause. + OMPClause *ActOnOpenMPThreadsClause(SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed 'simd' clause. + OMPClause *ActOnOpenMPSIMDClause(SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed 'nogroup' clause. + OMPClause *ActOnOpenMPNogroupClause(SourceLocation StartLoc, + SourceLocation EndLoc); + /// Called on well-formed 'unified_address' clause. + OMPClause *ActOnOpenMPUnifiedAddressClause(SourceLocation StartLoc, + SourceLocation EndLoc); - void CheckTCBEnforcement(const SourceLocation CallExprLoc, - const NamedDecl *Callee); + /// Called on well-formed 'unified_address' clause. + OMPClause *ActOnOpenMPUnifiedSharedMemoryClause(SourceLocation StartLoc, + SourceLocation EndLoc); - void AnalyzeDeleteExprMismatch(const CXXDeleteExpr *DE); - void AnalyzeDeleteExprMismatch(FieldDecl *Field, SourceLocation DeleteLoc, - bool DeleteWasArrayForm); -public: - /// Register a magic integral constant to be used as a type tag. - void RegisterTypeTagForDatatype(const IdentifierInfo *ArgumentKind, - uint64_t MagicValue, QualType Type, - bool LayoutCompatible, bool MustBeNull); + /// Called on well-formed 'reverse_offload' clause. + OMPClause *ActOnOpenMPReverseOffloadClause(SourceLocation StartLoc, + SourceLocation EndLoc); - struct TypeTagData { - TypeTagData() {} + /// Called on well-formed 'dynamic_allocators' clause. + OMPClause *ActOnOpenMPDynamicAllocatorsClause(SourceLocation StartLoc, + SourceLocation EndLoc); - TypeTagData(QualType Type, bool LayoutCompatible, bool MustBeNull) : - Type(Type), LayoutCompatible(LayoutCompatible), - MustBeNull(MustBeNull) - {} + /// Called on well-formed 'atomic_default_mem_order' clause. + OMPClause *ActOnOpenMPAtomicDefaultMemOrderClause( + OpenMPAtomicDefaultMemOrderClauseKind Kind, SourceLocation KindLoc, + SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation EndLoc); - QualType Type; + /// Called on well-formed 'at' clause. + OMPClause *ActOnOpenMPAtClause(OpenMPAtClauseKind Kind, + SourceLocation KindLoc, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); - /// If true, \c Type should be compared with other expression's types for - /// layout-compatibility. - LLVM_PREFERRED_TYPE(bool) - unsigned LayoutCompatible : 1; - LLVM_PREFERRED_TYPE(bool) - unsigned MustBeNull : 1; - }; + /// Called on well-formed 'severity' clause. + OMPClause *ActOnOpenMPSeverityClause(OpenMPSeverityClauseKind Kind, + SourceLocation KindLoc, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); - /// A pair of ArgumentKind identifier and magic value. This uniquely - /// identifies the magic value. - typedef std::pair TypeTagMagicValue; + /// Called on well-formed 'message' clause. + /// passing string for message. + OMPClause *ActOnOpenMPMessageClause(Expr *MS, SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); -private: - /// A map from magic value to type information. - std::unique_ptr> - TypeTagForDatatypeMagicValues; + /// Data used for processing a list of variables in OpenMP clauses. + struct OpenMPVarListDataTy final { + Expr *DepModOrTailExpr = nullptr; + Expr *IteratorExpr = nullptr; + SourceLocation ColonLoc; + SourceLocation RLoc; + CXXScopeSpec ReductionOrMapperIdScopeSpec; + DeclarationNameInfo ReductionOrMapperId; + int ExtraModifier = -1; ///< Additional modifier for linear, map, depend or + ///< lastprivate clause. + SmallVector + MapTypeModifiers; + SmallVector + MapTypeModifiersLoc; + SmallVector + MotionModifiers; + SmallVector MotionModifiersLoc; + bool IsMapTypeImplicit = false; + SourceLocation ExtraModifierLoc; + SourceLocation OmpAllMemoryLoc; + SourceLocation + StepModifierLoc; /// 'step' modifier location for linear clause + }; - /// Peform checks on a call of a function with argument_with_type_tag - /// or pointer_with_type_tag attributes. - void CheckArgumentWithTypeTag(const ArgumentWithTypeTagAttr *Attr, - const ArrayRef ExprArgs, - SourceLocation CallSiteLoc); + OMPClause *ActOnOpenMPVarListClause(OpenMPClauseKind Kind, + ArrayRef Vars, + const OMPVarListLocTy &Locs, + OpenMPVarListDataTy &Data); + /// Called on well-formed 'inclusive' clause. + OMPClause *ActOnOpenMPInclusiveClause(ArrayRef VarList, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'exclusive' clause. + OMPClause *ActOnOpenMPExclusiveClause(ArrayRef VarList, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'allocate' clause. + OMPClause * + ActOnOpenMPAllocateClause(Expr *Allocator, ArrayRef VarList, + SourceLocation StartLoc, SourceLocation ColonLoc, + SourceLocation LParenLoc, SourceLocation EndLoc); + /// Called on well-formed 'private' clause. + OMPClause *ActOnOpenMPPrivateClause(ArrayRef VarList, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'firstprivate' clause. + OMPClause *ActOnOpenMPFirstprivateClause(ArrayRef VarList, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'lastprivate' clause. + OMPClause *ActOnOpenMPLastprivateClause( + ArrayRef VarList, OpenMPLastprivateModifier LPKind, + SourceLocation LPKindLoc, SourceLocation ColonLoc, + SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation EndLoc); + /// Called on well-formed 'shared' clause. + OMPClause *ActOnOpenMPSharedClause(ArrayRef VarList, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'reduction' clause. + OMPClause *ActOnOpenMPReductionClause( + ArrayRef VarList, OpenMPReductionClauseModifier Modifier, + SourceLocation StartLoc, SourceLocation LParenLoc, + SourceLocation ModifierLoc, SourceLocation ColonLoc, + SourceLocation EndLoc, CXXScopeSpec &ReductionIdScopeSpec, + const DeclarationNameInfo &ReductionId, + ArrayRef UnresolvedReductions = std::nullopt); + /// Called on well-formed 'task_reduction' clause. + OMPClause *ActOnOpenMPTaskReductionClause( + ArrayRef VarList, SourceLocation StartLoc, + SourceLocation LParenLoc, SourceLocation ColonLoc, SourceLocation EndLoc, + CXXScopeSpec &ReductionIdScopeSpec, + const DeclarationNameInfo &ReductionId, + ArrayRef UnresolvedReductions = std::nullopt); + /// Called on well-formed 'in_reduction' clause. + OMPClause *ActOnOpenMPInReductionClause( + ArrayRef VarList, SourceLocation StartLoc, + SourceLocation LParenLoc, SourceLocation ColonLoc, SourceLocation EndLoc, + CXXScopeSpec &ReductionIdScopeSpec, + const DeclarationNameInfo &ReductionId, + ArrayRef UnresolvedReductions = std::nullopt); + /// Called on well-formed 'linear' clause. + OMPClause *ActOnOpenMPLinearClause( + ArrayRef VarList, Expr *Step, SourceLocation StartLoc, + SourceLocation LParenLoc, OpenMPLinearClauseKind LinKind, + SourceLocation LinLoc, SourceLocation ColonLoc, + SourceLocation StepModifierLoc, SourceLocation EndLoc); + /// Called on well-formed 'aligned' clause. + OMPClause *ActOnOpenMPAlignedClause(ArrayRef VarList, Expr *Alignment, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation ColonLoc, + SourceLocation EndLoc); + /// Called on well-formed 'copyin' clause. + OMPClause *ActOnOpenMPCopyinClause(ArrayRef VarList, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'copyprivate' clause. + OMPClause *ActOnOpenMPCopyprivateClause(ArrayRef VarList, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'flush' pseudo clause. + OMPClause *ActOnOpenMPFlushClause(ArrayRef VarList, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'depobj' pseudo clause. + OMPClause *ActOnOpenMPDepobjClause(Expr *Depobj, SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'depend' clause. + OMPClause *ActOnOpenMPDependClause(const OMPDependClause::DependDataTy &Data, + Expr *DepModifier, + ArrayRef VarList, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'device' clause. + OMPClause *ActOnOpenMPDeviceClause(OpenMPDeviceClauseModifier Modifier, + Expr *Device, SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation ModifierLoc, + SourceLocation EndLoc); + /// Called on well-formed 'map' clause. + OMPClause *ActOnOpenMPMapClause( + Expr *IteratorModifier, ArrayRef MapTypeModifiers, + ArrayRef MapTypeModifiersLoc, + CXXScopeSpec &MapperIdScopeSpec, DeclarationNameInfo &MapperId, + OpenMPMapClauseKind MapType, bool IsMapTypeImplicit, + SourceLocation MapLoc, SourceLocation ColonLoc, ArrayRef VarList, + const OMPVarListLocTy &Locs, bool NoDiagnose = false, + ArrayRef UnresolvedMappers = std::nullopt); + /// Called on well-formed 'num_teams' clause. + OMPClause *ActOnOpenMPNumTeamsClause(Expr *NumTeams, SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'thread_limit' clause. + OMPClause *ActOnOpenMPThreadLimitClause(Expr *ThreadLimit, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'priority' clause. + OMPClause *ActOnOpenMPPriorityClause(Expr *Priority, SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); + /// Called on well-formed 'dist_schedule' clause. + OMPClause *ActOnOpenMPDistScheduleClause( + OpenMPDistScheduleClauseKind Kind, Expr *ChunkSize, + SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation KindLoc, + SourceLocation CommaLoc, SourceLocation EndLoc); + /// Called on well-formed 'defaultmap' clause. + OMPClause *ActOnOpenMPDefaultmapClause( + OpenMPDefaultmapClauseModifier M, OpenMPDefaultmapClauseKind Kind, + SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation MLoc, + SourceLocation KindLoc, SourceLocation EndLoc); + /// Called on well-formed 'to' clause. + OMPClause * + ActOnOpenMPToClause(ArrayRef MotionModifiers, + ArrayRef MotionModifiersLoc, + CXXScopeSpec &MapperIdScopeSpec, + DeclarationNameInfo &MapperId, SourceLocation ColonLoc, + ArrayRef VarList, const OMPVarListLocTy &Locs, + ArrayRef UnresolvedMappers = std::nullopt); + /// Called on well-formed 'from' clause. + OMPClause * + ActOnOpenMPFromClause(ArrayRef MotionModifiers, + ArrayRef MotionModifiersLoc, + CXXScopeSpec &MapperIdScopeSpec, + DeclarationNameInfo &MapperId, SourceLocation ColonLoc, + ArrayRef VarList, const OMPVarListLocTy &Locs, + ArrayRef UnresolvedMappers = std::nullopt); + /// Called on well-formed 'use_device_ptr' clause. + OMPClause *ActOnOpenMPUseDevicePtrClause(ArrayRef VarList, + const OMPVarListLocTy &Locs); + /// Called on well-formed 'use_device_addr' clause. + OMPClause *ActOnOpenMPUseDeviceAddrClause(ArrayRef VarList, + const OMPVarListLocTy &Locs); + /// Called on well-formed 'is_device_ptr' clause. + OMPClause *ActOnOpenMPIsDevicePtrClause(ArrayRef VarList, + const OMPVarListLocTy &Locs); + /// Called on well-formed 'has_device_addr' clause. + OMPClause *ActOnOpenMPHasDeviceAddrClause(ArrayRef VarList, + const OMPVarListLocTy &Locs); + /// Called on well-formed 'nontemporal' clause. + OMPClause *ActOnOpenMPNontemporalClause(ArrayRef VarList, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); - /// Check if we are taking the address of a packed field - /// as this may be a problem if the pointer value is dereferenced. - void CheckAddressOfPackedMember(Expr *rhs); + /// Data for list of allocators. + struct UsesAllocatorsData { + /// Allocator. + Expr *Allocator = nullptr; + /// Allocator traits. + Expr *AllocatorTraits = nullptr; + /// Locations of '(' and ')' symbols. + SourceLocation LParenLoc, RParenLoc; + }; + /// Called on well-formed 'uses_allocators' clause. + OMPClause *ActOnOpenMPUsesAllocatorClause(SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc, + ArrayRef Data); + /// Called on well-formed 'affinity' clause. + OMPClause *ActOnOpenMPAffinityClause(SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation ColonLoc, + SourceLocation EndLoc, Expr *Modifier, + ArrayRef Locators); + /// Called on a well-formed 'bind' clause. + OMPClause *ActOnOpenMPBindClause(OpenMPBindClauseKind Kind, + SourceLocation KindLoc, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); - /// The parser's current scope. - /// - /// The parser maintains this state here. - Scope *CurScope; + /// Called on a well-formed 'ompx_dyn_cgroup_mem' clause. + OMPClause *ActOnOpenMPXDynCGroupMemClause(Expr *Size, SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); - mutable IdentifierInfo *Ident_super; + /// Called on well-formed 'doacross' clause. + OMPClause * + ActOnOpenMPDoacrossClause(OpenMPDoacrossClauseModifier DepType, + SourceLocation DepLoc, SourceLocation ColonLoc, + ArrayRef VarList, SourceLocation StartLoc, + SourceLocation LParenLoc, SourceLocation EndLoc); - /// Nullability type specifiers. - IdentifierInfo *Ident__Nonnull = nullptr; - IdentifierInfo *Ident__Nullable = nullptr; - IdentifierInfo *Ident__Nullable_result = nullptr; - IdentifierInfo *Ident__Null_unspecified = nullptr; + /// Called on a well-formed 'ompx_attribute' clause. + OMPClause *ActOnOpenMPXAttributeClause(ArrayRef Attrs, + SourceLocation StartLoc, + SourceLocation LParenLoc, + SourceLocation EndLoc); - IdentifierInfo *Ident_NSError = nullptr; + /// Called on a well-formed 'ompx_bare' clause. + OMPClause *ActOnOpenMPXBareClause(SourceLocation StartLoc, + SourceLocation EndLoc); - /// The handler for the FileChanged preprocessor events. - /// - /// Used for diagnostics that implement custom semantic analysis for #include - /// directives, like -Wpragma-pack. - sema::SemaPPCallbacks *SemaPPCallbackHandler; +private: + void *VarDataSharingAttributesStack; -protected: - friend class Parser; - friend class InitializationSequence; - friend class ASTReader; - friend class ASTDeclReader; - friend class ASTWriter; + /// Number of nested '#pragma omp declare target' directives. + SmallVector DeclareTargetNesting; -public: - /// Retrieve the keyword associated - IdentifierInfo *getNullabilityKeyword(NullabilityKind nullability); + /// Initialization of data-sharing attributes stack. + void InitDataSharingAttributesStack(); + void DestroyDataSharingAttributesStack(); - /// The struct behind the CFErrorRef pointer. - RecordDecl *CFError = nullptr; - bool isCFError(RecordDecl *D); + /// Returns OpenMP nesting level for current directive. + unsigned getOpenMPNestingLevel() const; - /// Retrieve the identifier "NSError". - IdentifierInfo *getNSErrorIdent(); + /// Adjusts the function scopes index for the target-based regions. + void adjustOpenMPTargetScopeIndex(unsigned &FunctionScopesIndex, + unsigned Level) const; - /// Retrieve the parser's current scope. - /// - /// This routine must only be used when it is certain that semantic analysis - /// and the parser are in precisely the same context, which is not the case - /// when, e.g., we are performing any kind of template instantiation. - /// Therefore, the only safe places to use this scope are in the parser - /// itself and in routines directly invoked from the parser and *never* from - /// template substitution or instantiation. - Scope *getCurScope() const { return CurScope; } + /// Returns the number of scopes associated with the construct on the given + /// OpenMP level. + int getNumberOfConstructScopes(unsigned Level) const; - void incrementMSManglingNumber() const { - return CurScope->incrementMSManglingNumber(); - } + /// Push new OpenMP function region for non-capturing function. + void pushOpenMPFunctionRegion(); - IdentifierInfo *getSuperIdentifier() const; + /// Pop OpenMP function region for non-capturing function. + void popOpenMPFunctionRegion(const sema::FunctionScopeInfo *OldFSI); - ObjCContainerDecl *getObjCDeclContext() const; + /// Analyzes and checks a loop nest for use by a loop transformation. + /// + /// \param Kind The loop transformation directive kind. + /// \param NumLoops How many nested loops the directive is expecting. + /// \param AStmt Associated statement of the transformation directive. + /// \param LoopHelpers [out] The loop analysis result. + /// \param Body [out] The body code nested in \p NumLoops loop. + /// \param OriginalInits [out] Collection of statements and declarations that + /// must have been executed/declared before entering the + /// loop. + /// + /// \return Whether there was any error. + bool checkTransformableLoopNest( + OpenMPDirectiveKind Kind, Stmt *AStmt, int NumLoops, + SmallVectorImpl &LoopHelpers, + Stmt *&Body, + SmallVectorImpl, 0>> + &OriginalInits); - DeclContext *getCurLexicalContext() const { - return OriginalLexicalContext ? OriginalLexicalContext : CurContext; - } + /// Helper to keep information about the current `omp begin/end declare + /// variant` nesting. + struct OMPDeclareVariantScope { + /// The associated OpenMP context selector. + OMPTraitInfo *TI; - const DeclContext *getCurObjCLexicalContext() const { - const DeclContext *DC = getCurLexicalContext(); - // A category implicitly has the attribute of the interface. - if (const ObjCCategoryDecl *CatD = dyn_cast(DC)) - DC = CatD->getClassInterface(); - return DC; - } + /// The associated OpenMP context selector mangling. + std::string NameSuffix; - /// Determine the number of levels of enclosing template parameters. This is - /// only usable while parsing. Note that this does not include dependent - /// contexts in which no template parameters have yet been declared, such as - /// in a terse function template or generic lambda before the first 'auto' is - /// encountered. - unsigned getTemplateDepth(Scope *S) const; + OMPDeclareVariantScope(OMPTraitInfo &TI); + }; - /// To be used for checking whether the arguments being passed to - /// function exceeds the number of parameters expected for it. - static bool TooManyArguments(size_t NumParams, size_t NumArgs, - bool PartialOverloading = false) { - // We check whether we're just after a comma in code-completion. - if (NumArgs > 0 && PartialOverloading) - return NumArgs + 1 > NumParams; // If so, we view as an extra argument. - return NumArgs > NumParams; + /// Return the OMPTraitInfo for the surrounding scope, if any. + OMPTraitInfo *getOMPTraitInfoForSurroundingScope() { + return OMPDeclareVariantScopes.empty() ? nullptr + : OMPDeclareVariantScopes.back().TI; } - // Emitting members of dllexported classes is delayed until the class - // (including field initializers) is fully parsed. - SmallVector DelayedDllExportClasses; - SmallVector DelayedDllExportMemberFunctions; - -private: - int ParsingClassDepth = 0; - - class SavePendingParsedClassStateRAII { - public: - SavePendingParsedClassStateRAII(Sema &S) : S(S) { swapSavedState(); } - - ~SavePendingParsedClassStateRAII() { - assert(S.DelayedOverridingExceptionSpecChecks.empty() && - "there shouldn't be any pending delayed exception spec checks"); - assert(S.DelayedEquivalentExceptionSpecChecks.empty() && - "there shouldn't be any pending delayed exception spec checks"); - swapSavedState(); - } + /// The current `omp begin/end declare variant` scopes. + SmallVector OMPDeclareVariantScopes; - private: - Sema &S; - decltype(DelayedOverridingExceptionSpecChecks) - SavedOverridingExceptionSpecChecks; - decltype(DelayedEquivalentExceptionSpecChecks) - SavedEquivalentExceptionSpecChecks; + /// The current `omp begin/end assumes` scopes. + SmallVector OMPAssumeScoped; - void swapSavedState() { - SavedOverridingExceptionSpecChecks.swap( - S.DelayedOverridingExceptionSpecChecks); - SavedEquivalentExceptionSpecChecks.swap( - S.DelayedEquivalentExceptionSpecChecks); - } - }; + /// All `omp assumes` we encountered so far. + SmallVector OMPAssumeGlobal; - /// Helper class that collects misaligned member designations and - /// their location info for delayed diagnostics. - struct MisalignedMember { - Expr *E; - RecordDecl *RD; - ValueDecl *MD; - CharUnits Alignment; + /// OMPD_loop is mapped to OMPD_for, OMPD_distribute or OMPD_simd depending + /// on the parameter of the bind clause. In the methods for the + /// mapped directives, check the parameters of the lastprivate clause. + bool checkLastPrivateForMappedDirectives(ArrayRef Clauses); + /// Depending on the bind clause of OMPD_loop map the directive to new + /// directives. + /// 1) loop bind(parallel) --> OMPD_for + /// 2) loop bind(teams) --> OMPD_distribute + /// 3) loop bind(thread) --> OMPD_simd + /// This is being handled in Sema instead of Codegen because of the need for + /// rigorous semantic checking in the new mapped directives. + bool mapLoopConstruct(llvm::SmallVector &ClausesWithoutBind, + ArrayRef Clauses, + OpenMPBindClauseKind &BindKind, + OpenMPDirectiveKind &Kind, + OpenMPDirectiveKind &PrevMappedDirective, + SourceLocation StartLoc, SourceLocation EndLoc, + const DeclarationNameInfo &DirName, + OpenMPDirectiveKind CancelRegion); - MisalignedMember() : E(), RD(), MD() {} - MisalignedMember(Expr *E, RecordDecl *RD, ValueDecl *MD, - CharUnits Alignment) - : E(E), RD(RD), MD(MD), Alignment(Alignment) {} - explicit MisalignedMember(Expr *E) - : MisalignedMember(E, nullptr, nullptr, CharUnits()) {} + ///@} - bool operator==(const MisalignedMember &m) { return this->E == m.E; } - }; - /// Small set of gathered accesses to potentially misaligned members - /// due to the packed attribute. - SmallVector MisalignedMembers; + // + // + // ------------------------------------------------------------------------- + // + // - /// Adds an expression to the set of gathered misaligned members. - void AddPotentialMisalignedMembers(Expr *E, RecordDecl *RD, ValueDecl *MD, - CharUnits Alignment); + /// \name SYCL Constructs + /// Implementations are in SemaSYCL.cpp + ///@{ public: - /// Diagnoses the current set of gathered accesses. This typically - /// happens at full expression level. The set is cleared after emitting the - /// diagnostics. - void DiagnoseMisalignedMembers(); - - /// This function checks if the expression is in the sef of potentially - /// misaligned members and it is converted to some pointer type T with lower - /// or equal alignment requirements. If so it removes it. This is used when - /// we do not want to diagnose such misaligned access (e.g. in conversions to - /// void*). - void DiscardMisalignedMemberAddress(const Type *T, Expr *E); - - /// This function calls Action when it determines that E designates a - /// misaligned member due to the packed attribute. This is used to emit - /// local diagnostics like in reference binding. - void RefersToMemberWithReducedAlignment( - Expr *E, - llvm::function_ref - Action); - - /// Describes the reason a calling convention specification was ignored, used - /// for diagnostics. - enum class CallingConventionIgnoredReason { - ForThisTarget = 0, - VariadicFunction, - ConstructorDestructor, - BuiltinFunction - }; /// Creates a SemaDiagnosticBuilder that emits the diagnostic if the current /// context is "used as device code". /// @@ -14508,6 +14755,8 @@ public: void deepTypeCheckForSYCLDevice(SourceLocation UsedAt, llvm::DenseSet Visited, ValueDecl *DeclToCheck); + + ///@} }; DeductionFailureInfo diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp index cfb653e665ea..720d5fd5f042 100644 --- a/clang/lib/Sema/Sema.cpp +++ b/clang/lib/Sema/Sema.cpp @@ -188,38 +188,38 @@ const uint64_t Sema::MaximumAlignment; Sema::Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer, TranslationUnitKind TUKind, CodeCompleteConsumer *CodeCompleter) - : ExternalSource(nullptr), CurFPFeatures(pp.getLangOpts()), + : CollectStats(false), TUKind(TUKind), CurFPFeatures(pp.getLangOpts()), LangOpts(pp.getLangOpts()), PP(pp), Context(ctxt), Consumer(consumer), Diags(PP.getDiagnostics()), SourceMgr(PP.getSourceManager()), - APINotes(SourceMgr, LangOpts), CollectStats(false), - CodeCompleter(CodeCompleter), CurContext(nullptr), - OriginalLexicalContext(nullptr), MSStructPragmaOn(false), + APINotes(SourceMgr, LangOpts), AnalysisWarnings(*this), + ThreadSafetyDeclCache(nullptr), LateTemplateParser(nullptr), + LateTemplateParserCleanup(nullptr), OpaqueParser(nullptr), + CurContext(nullptr), ExternalSource(nullptr), CurScope(nullptr), + Ident_super(nullptr), MSPointerToMemberRepresentationMethod( LangOpts.getMSPointerToMemberRepresentationMethod()), - VtorDispStack(LangOpts.getVtorDispMode()), + MSStructPragmaOn(false), VtorDispStack(LangOpts.getVtorDispMode()), AlignPackStack(AlignPackInfo(getLangOpts().XLPragmaPack)), DataSegStack(nullptr), BSSSegStack(nullptr), ConstSegStack(nullptr), CodeSegStack(nullptr), StrictGuardStackCheckStack(false), FpPragmaStack(FPOptionsOverride()), CurInitSeg(nullptr), VisContext(nullptr), PragmaAttributeCurrentTargetDecl(nullptr), - IsBuildingRecoveryCallExpr(false), LateTemplateParser(nullptr), - LateTemplateParserCleanup(nullptr), OpaqueParser(nullptr), IdResolver(pp), - StdInitializerList(nullptr), StdCoroutineTraitsCache(nullptr), - CXXTypeInfoDecl(nullptr), StdSourceLocationImplDecl(nullptr), + StdCoroutineTraitsCache(nullptr), IdResolver(pp), + OriginalLexicalContext(nullptr), StdInitializerList(nullptr), + FullyCheckedComparisonCategories( + static_cast(ComparisonCategoryType::Last) + 1), + StdSourceLocationImplDecl(nullptr), CXXTypeInfoDecl(nullptr), + GlobalNewDeleteDeclared(false), DisableTypoCorrection(false), + TyposCorrected(0), IsBuildingRecoveryCallExpr(false), NumSFINAEErrors(0), + AccessCheckingSFINAE(false), CurrentInstantiationScope(nullptr), + InNonInstantiationSFINAEContext(false), NonInstantiationEntries(0), + ArgumentPackSubstitutionIndex(-1), SatisfactionCache(Context), NSNumberDecl(nullptr), NSValueDecl(nullptr), NSStringDecl(nullptr), StringWithUTF8StringMethod(nullptr), ValueWithBytesObjCTypeMethod(nullptr), NSArrayDecl(nullptr), ArrayWithObjectsMethod(nullptr), NSDictionaryDecl(nullptr), - DictionaryWithObjectsMethod(nullptr), GlobalNewDeleteDeclared(false), - TUKind(TUKind), NumSFINAEErrors(0), - FullyCheckedComparisonCategories( - static_cast(ComparisonCategoryType::Last) + 1), - SatisfactionCache(Context), AccessCheckingSFINAE(false), - InNonInstantiationSFINAEContext(false), NonInstantiationEntries(0), - ArgumentPackSubstitutionIndex(-1), CurrentInstantiationScope(nullptr), - DisableTypoCorrection(false), TyposCorrected(0), AnalysisWarnings(*this), - ThreadSafetyDeclCache(nullptr), VarDataSharingAttributesStack(nullptr), - CurScope(nullptr), Ident_super(nullptr) { + DictionaryWithObjectsMethod(nullptr), CodeCompleter(CodeCompleter), + VarDataSharingAttributesStack(nullptr) { assert(pp.TUKind == TUKind); TUScope = nullptr; -- GitLab From 20459ddc82514249501c3517583c4ec8dd23cecd Mon Sep 17 00:00:00 2001 From: Jonathan Schilling <130992531+jons-pf@users.noreply.github.com> Date: Wed, 6 Mar 2024 11:41:56 +0100 Subject: [PATCH 270/929] [openmp] Clarify error message if TSan is missing (#70916) For an uninformed user, the error message might refer to a missing "TSan stopping operation", rather than indicating that TSan is missing **and therefore** operation is stopped. --- openmp/tools/archer/ompt-tsan.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openmp/tools/archer/ompt-tsan.cpp b/openmp/tools/archer/ompt-tsan.cpp index 8b338f6b18b6..de77e25db2d3 100644 --- a/openmp/tools/archer/ompt-tsan.cpp +++ b/openmp/tools/archer/ompt-tsan.cpp @@ -1248,7 +1248,7 @@ ompt_start_tool(unsigned int omp_version, const char *runtime_version) { // tool the chance to be loaded { if (archer_flags->verbose) - std::cout << "Archer detected OpenMP application without TSan " + std::cout << "Archer detected OpenMP application without TSan; " "stopping operation" << std::endl; delete archer_flags; -- GitLab From 92529eb4aeb2dbb706aedebc1ed9fc4f8045231a Mon Sep 17 00:00:00 2001 From: Alexandros Lamprineas Date: Wed, 6 Mar 2024 10:42:39 +0000 Subject: [PATCH 271/929] [FMV] Allow target version definitions in any order. (#83887) This patch fixes #71698. It allows defining the default target version prior to other version definitions without raising semantic errors. --- clang/lib/Sema/SemaDecl.cpp | 15 +++++-- clang/test/CodeGen/attr-target-version.c | 56 ++++++++++++------------ clang/test/Sema/attr-target-version.c | 25 ++++++++--- 3 files changed, 59 insertions(+), 37 deletions(-) diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 210e2835184a..6b81ee183cc4 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -11479,6 +11479,16 @@ static bool CheckTargetCausesMultiVersioning(Sema &S, FunctionDecl *OldFD, bool &Redeclaration, NamedDecl *&OldDecl, LookupResult &Previous) { + assert(!OldFD->isMultiVersion() && "Unexpected MultiVersion"); + + // The definitions should be allowed in any order. If we have discovered + // a new target version and the preceeding was the default, then add the + // corresponding attribute to it. + if (OldFD->getMultiVersionKind() == MultiVersionKind::None && + NewFD->getMultiVersionKind() == MultiVersionKind::TargetVersion) + OldFD->addAttr(TargetVersionAttr::CreateImplicit(S.Context, "default", + OldFD->getSourceRange())); + const auto *NewTA = NewFD->getAttr(); const auto *NewTVA = NewFD->getAttr(); const auto *OldTA = OldFD->getAttr(); @@ -11505,9 +11515,8 @@ static bool CheckTargetCausesMultiVersioning(Sema &S, FunctionDecl *OldFD, } // If this is 'default', permit the forward declaration. - if (!OldFD->isMultiVersion() && - ((NewTA && NewTA->isDefaultVersion() && !OldTA) || - (NewTVA && NewTVA->isDefaultVersion() && !OldTVA))) { + if ((NewTA && NewTA->isDefaultVersion() && !OldTA) || + (NewTVA && NewTVA->isDefaultVersion() && !OldTVA)) { Redeclaration = true; OldDecl = OldFD; OldFD->setIsMultiVersion(); diff --git a/clang/test/CodeGen/attr-target-version.c b/clang/test/CodeGen/attr-target-version.c index 56a42499d0a7..ae1a8772f6cc 100644 --- a/clang/test/CodeGen/attr-target-version.c +++ b/clang/test/CodeGen/attr-target-version.c @@ -94,16 +94,16 @@ int hoo(void) { // CHECK: @fmv_one.ifunc = weak_odr alias i32 (), ptr @fmv_one // CHECK: @fmv_two.ifunc = weak_odr alias i32 (), ptr @fmv_two // CHECK: @fmv_e.ifunc = weak_odr alias i32 (), ptr @fmv_e +// CHECK: @fmv_c.ifunc = weak_odr alias void (), ptr @fmv_c // CHECK: @fmv_inline.ifunc = weak_odr alias i32 (), ptr @fmv_inline // CHECK: @fmv_d.ifunc = internal alias i32 (), ptr @fmv_d -// CHECK: @fmv_c.ifunc = weak_odr alias void (), ptr @fmv_c // CHECK: @fmv = weak_odr ifunc i32 (), ptr @fmv.resolver // CHECK: @fmv_one = weak_odr ifunc i32 (), ptr @fmv_one.resolver // CHECK: @fmv_two = weak_odr ifunc i32 (), ptr @fmv_two.resolver // CHECK: @fmv_e = weak_odr ifunc i32 (), ptr @fmv_e.resolver +// CHECK: @fmv_c = weak_odr ifunc void (), ptr @fmv_c.resolver // CHECK: @fmv_inline = weak_odr ifunc i32 (), ptr @fmv_inline.resolver // CHECK: @fmv_d = internal ifunc i32 (), ptr @fmv_d.resolver -// CHECK: @fmv_c = weak_odr ifunc void (), ptr @fmv_c.resolver //. // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv._MflagmMfp16fmlMrng @@ -238,11 +238,18 @@ int hoo(void) { // CHECK-NEXT: ret i32 111 // // -// CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_c._Mssbs -// CHECK-SAME: () #[[ATTR2]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: ret void +// CHECK-LABEL: define {{[^@]+}}@fmv_c.resolver() comdat { +// CHECK-NEXT: resolver_entry: +// CHECK-NEXT: call void @__init_cpu_features_resolver() +// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 281474976710656 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 281474976710656 +// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] +// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] +// CHECK: resolver_return: +// CHECK-NEXT: ret ptr @fmv_c._Mssbs +// CHECK: resolver_else: +// CHECK-NEXT: ret ptr @fmv_c.default // // // CHECK: Function Attrs: noinline nounwind optnone @@ -405,20 +412,6 @@ int hoo(void) { // CHECK-NEXT: ret ptr @fmv_d.default // // -// CHECK-LABEL: define {{[^@]+}}@fmv_c.resolver() comdat { -// CHECK-NEXT: resolver_entry: -// CHECK-NEXT: call void @__init_cpu_features_resolver() -// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 281474976710656 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 281474976710656 -// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] -// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] -// CHECK: resolver_return: -// CHECK-NEXT: ret ptr @fmv_c._Mssbs -// CHECK: resolver_else: -// CHECK-NEXT: ret ptr @fmv_c.default -// -// // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@recur // CHECK-SAME: () #[[ATTR2]] { @@ -568,6 +561,20 @@ int hoo(void) { // // // CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_c.default +// CHECK-SAME: () #[[ATTR2]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret void +// +// +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_c._Mssbs +// CHECK-SAME: () #[[ATTR2]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret void +// +// +// CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mf64mmMpmullMsha1 // CHECK-SAME: () #[[ATTR12:[0-9]+]] { // CHECK-NEXT: entry: @@ -700,13 +707,6 @@ int hoo(void) { // CHECK-NEXT: ret i32 1 // // -// CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_c.default -// CHECK-SAME: () #[[ATTR2]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: ret void -// -// // CHECK-NOFMV: Function Attrs: noinline nounwind optnone // CHECK-NOFMV-LABEL: define {{[^@]+}}@fmv // CHECK-NOFMV-SAME: () #[[ATTR0:[0-9]+]] { diff --git a/clang/test/Sema/attr-target-version.c b/clang/test/Sema/attr-target-version.c index 587c721de5e3..e2940c434c2f 100644 --- a/clang/test/Sema/attr-target-version.c +++ b/clang/test/Sema/attr-target-version.c @@ -42,12 +42,25 @@ void __attribute__((target_version("ssbs+fp16fml"))) two(void) {} //expected-error@+1 {{'main' cannot be a multiversioned function}} int __attribute__((target_version("lse"))) main(void) { return 1; } -//expected-note@+1 {{previous definition is here}} -int hoo(void) { return 1; } -//expected-note@-1 {{previous definition is here}} -//expected-warning@+2 {{attribute declaration must precede definition}} -//expected-error@+1 {{redefinition of 'hoo'}} -int __attribute__((target_version("dit"))) hoo(void) { return 2; } +// It is ok for the default version to appear first. +int default_first(void) { return 1; } +int __attribute__((target_version("dit"))) default_first(void) { return 2; } +int __attribute__((target_version("mops"))) default_first(void) { return 3; } + +// It is ok if the default version is between other versions. +int __attribute__((target_version("simd"))) default_middle(void) {return 0; } +int __attribute__((target_version("default"))) default_middle(void) { return 1; } +int __attribute__((target_version("aes"))) default_middle(void) { return 2; } + +// It is ok for the default version to be the last one. +int __attribute__((target_version("rdm"))) default_last(void) {return 0; } +int __attribute__((target_version("lse+aes"))) default_last(void) { return 1; } +int __attribute__((target_version("default"))) default_last(void) { return 2; } + +// It is also ok to forward declare the default. +int __attribute__((target_version("default"))) default_fwd_declare(void); +int __attribute__((target_version("sve"))) default_fwd_declare(void) { return 0; } +int default_fwd_declare(void) { return 1; } //expected-warning@+1 {{unsupported '' in the 'target_version' attribute string; 'target_version' attribute ignored}} int __attribute__((target_version(""))) unsup1(void) { return 1; } -- GitLab From e85470232ba2fa49aaee83240741de0bc82a3ffa Mon Sep 17 00:00:00 2001 From: Vlad Serebrennikov Date: Wed, 6 Mar 2024 13:49:43 +0300 Subject: [PATCH 272/929] Subscribe @Endilll to changes in Clang's `Sema.h` --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3fe0cbbcb84d..85850848b220 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -27,6 +27,7 @@ /llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp @nikic /llvm/lib/Transforms/InstCombine/ @nikic +/clang/include/clang/Sema/Sema.h @Endilll /clang/test/CXX/drs/ @Endilll /clang/www/cxx_dr_status.html @Endilll /clang/www/make_cxx_dr_status @Endilll -- GitLab From 239312e49b2fdf35e23d75197a621ee6087743ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20=C3=81lvarez=20Ayll=C3=B3n?= Date: Wed, 6 Mar 2024 12:10:07 +0100 Subject: [PATCH 273/929] Reapply "[clang][analyzer] StreamChecker: Model getc, vfscanf, putc, vfprintf" (#83281) `va_list` is a platform-specific type. On some, it is a struct instead of a pointer to a struct, so `lookupFn` was ignoring calls to `vfprintf` and `vfscanf`. `stream.c` now runs in four different platforms to make sure the logic works across targets. --- .../StaticAnalyzer/Checkers/StreamChecker.cpp | 33 ++++++++++++--- ...ystem-header-simulator-for-simple-stream.h | 2 +- .../system-header-simulator-for-valist.h | 6 +++ .../Analysis/Inputs/system-header-simulator.h | 3 ++ clang/test/Analysis/stream-invalidate.c | 42 +++++++++++++++++++ clang/test/Analysis/stream.c | 39 ++++++++++++++++- 6 files changed, 118 insertions(+), 7 deletions(-) diff --git a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp index 0208f94e1b5a..2ec47bf55df7 100644 --- a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp @@ -318,18 +318,30 @@ private: {{{"fgets"}, 3}, {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, true), std::bind(&StreamChecker::evalFgetx, _1, _2, _3, _4, false), 2}}, + {{{"getc"}, 1}, + {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, true), + std::bind(&StreamChecker::evalFgetx, _1, _2, _3, _4, true), 0}}, {{{"fputc"}, 2}, {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, false), std::bind(&StreamChecker::evalFputx, _1, _2, _3, _4, true), 1}}, {{{"fputs"}, 2}, {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, false), std::bind(&StreamChecker::evalFputx, _1, _2, _3, _4, false), 1}}, + {{{"putc"}, 2}, + {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, false), + std::bind(&StreamChecker::evalFputx, _1, _2, _3, _4, true), 1}}, {{{"fprintf"}}, {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, false), std::bind(&StreamChecker::evalFprintf, _1, _2, _3, _4), 0}}, + {{{"vfprintf"}, 3}, + {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, false), + std::bind(&StreamChecker::evalFprintf, _1, _2, _3, _4), 0}}, {{{"fscanf"}}, {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, true), std::bind(&StreamChecker::evalFscanf, _1, _2, _3, _4), 0}}, + {{{"vfscanf"}, 3}, + {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, true), + std::bind(&StreamChecker::evalFscanf, _1, _2, _3, _4), 0}}, {{{"ungetc"}, 2}, {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, false), std::bind(&StreamChecker::evalUngetc, _1, _2, _3, _4), 1}}, @@ -389,6 +401,8 @@ private: mutable int SeekCurVal = 1; /// Expanded value of SEEK_END, 2 if not found. mutable int SeekEndVal = 2; + /// The built-in va_list type is platform-specific + mutable QualType VaListType; void evalFopen(const FnDescription *Desc, const CallEvent &Call, CheckerContext &C) const; @@ -518,7 +532,8 @@ private: return nullptr; for (auto *P : Call.parameters()) { QualType T = P->getType(); - if (!T->isIntegralOrEnumerationType() && !T->isPointerType()) + if (!T->isIntegralOrEnumerationType() && !T->isPointerType() && + T.getCanonicalType() != VaListType) return nullptr; } @@ -557,6 +572,10 @@ private: SeekCurVal = *OptInt; } + void initVaListType(CheckerContext &C) const { + VaListType = C.getASTContext().getBuiltinVaListType().getCanonicalType(); + } + /// Searches for the ExplodedNode where the file descriptor was acquired for /// StreamSym. static const ExplodedNode *getAcquisitionSite(const ExplodedNode *N, @@ -705,6 +724,7 @@ static ProgramStateRef escapeArgs(ProgramStateRef State, CheckerContext &C, void StreamChecker::checkPreCall(const CallEvent &Call, CheckerContext &C) const { initMacroValues(C); + initVaListType(C); const FnDescription *Desc = lookupFn(Call); if (!Desc || !Desc->PreFn) @@ -1085,10 +1105,13 @@ void StreamChecker::evalFscanf(const FnDescription *Desc, const CallEvent &Call, if (!StateNotFailed) return; - SmallVector EscArgs; - for (auto EscArg : llvm::seq(2u, Call.getNumArgs())) - EscArgs.push_back(EscArg); - StateNotFailed = escapeArgs(StateNotFailed, C, Call, EscArgs); + if (auto const *Callee = Call.getCalleeIdentifier(); + !Callee || !Callee->getName().equals("vfscanf")) { + SmallVector EscArgs; + for (auto EscArg : llvm::seq(2u, Call.getNumArgs())) + EscArgs.push_back(EscArg); + StateNotFailed = escapeArgs(StateNotFailed, C, Call, EscArgs); + } if (StateNotFailed) C.addTransition(StateNotFailed); diff --git a/clang/test/Analysis/Inputs/system-header-simulator-for-simple-stream.h b/clang/test/Analysis/Inputs/system-header-simulator-for-simple-stream.h index 098a2208fecb..c26d35821491 100644 --- a/clang/test/Analysis/Inputs/system-header-simulator-for-simple-stream.h +++ b/clang/test/Analysis/Inputs/system-header-simulator-for-simple-stream.h @@ -5,7 +5,7 @@ // suppressed. #pragma clang system_header -typedef struct __sFILE { +typedef struct _FILE { unsigned char *_p; } FILE; FILE *fopen(const char *restrict, const char *restrict) __asm("_" "fopen" ); diff --git a/clang/test/Analysis/Inputs/system-header-simulator-for-valist.h b/clang/test/Analysis/Inputs/system-header-simulator-for-valist.h index 7299b61353d4..720944abb8ad 100644 --- a/clang/test/Analysis/Inputs/system-header-simulator-for-valist.h +++ b/clang/test/Analysis/Inputs/system-header-simulator-for-valist.h @@ -10,6 +10,8 @@ #define restrict /*restrict*/ #endif +typedef struct _FILE FILE; + typedef __builtin_va_list va_list; #define va_start(ap, param) __builtin_va_start(ap, param) @@ -21,6 +23,10 @@ int vprintf (const char *restrict format, va_list arg); int vsprintf (char *restrict s, const char *restrict format, va_list arg); +int vfprintf(FILE *stream, const char *format, va_list ap); + +int vfscanf(FILE *stream, const char *format, va_list ap); + int some_library_function(int n, va_list arg); // No warning from system header. diff --git a/clang/test/Analysis/Inputs/system-header-simulator.h b/clang/test/Analysis/Inputs/system-header-simulator.h index 15986984802c..8fd51449ecc0 100644 --- a/clang/test/Analysis/Inputs/system-header-simulator.h +++ b/clang/test/Analysis/Inputs/system-header-simulator.h @@ -73,6 +73,9 @@ int ferror(FILE *stream); int fileno(FILE *stream); int fflush(FILE *stream); + +int getc(FILE *stream); + size_t strlen(const char *); char *strcpy(char *restrict, const char *restrict); diff --git a/clang/test/Analysis/stream-invalidate.c b/clang/test/Analysis/stream-invalidate.c index 6745d11a2fe7..5046a356d058 100644 --- a/clang/test/Analysis/stream-invalidate.c +++ b/clang/test/Analysis/stream-invalidate.c @@ -4,6 +4,7 @@ // RUN: -analyzer-checker=debug.ExprInspection #include "Inputs/system-header-simulator.h" +#include "Inputs/system-header-simulator-for-valist.h" void clang_analyzer_eval(int); void clang_analyzer_dump(int); @@ -145,3 +146,44 @@ void test_fgetpos() { fclose(F); } + +void test_fprintf() { + FILE *F1 = tmpfile(); + if (!F1) + return; + + unsigned a = 42; + char *output = "HELLO"; + int r = fprintf(F1, "%s\t%u\n", output, a); + // fprintf does not invalidate any of its input + // 69 is ascii for 'E' + clang_analyzer_dump(a); // expected-warning {{42 S32b}} + clang_analyzer_dump(output[1]); // expected-warning {{69 S32b}} + fclose(F1); +} + +int test_vfscanf_inner(const char *fmt, ...) { + FILE *F1 = tmpfile(); + if (!F1) + return EOF; + + va_list ap; + va_start(ap, fmt); + + int r = vfscanf(F1, fmt, ap); + + fclose(F1); + va_end(ap); + return r; +} + +void test_vfscanf() { + int i = 42; + int j = 43; + int r = test_vfscanf_inner("%d", &i); + if (r != EOF) { + // i gets invalidated by the call to test_vfscanf_inner, not by vfscanf. + clang_analyzer_dump(i); // expected-warning {{conj_$}} + clang_analyzer_dump(j); // expected-warning {{43 S32b}} + } +} diff --git a/clang/test/Analysis/stream.c b/clang/test/Analysis/stream.c index 378c9154f8f6..7ba27740a937 100644 --- a/clang/test/Analysis/stream.c +++ b/clang/test/Analysis/stream.c @@ -1,6 +1,10 @@ -// RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.unix.Stream,debug.ExprInspection -verify %s +// RUN: %clang_analyze_cc1 -triple=x86_64-pc-linux-gnu -analyzer-checker=core,alpha.unix.Stream,debug.ExprInspection -verify %s +// RUN: %clang_analyze_cc1 -triple=armv8-none-linux-eabi -analyzer-checker=core,alpha.unix.Stream,debug.ExprInspection -verify %s +// RUN: %clang_analyze_cc1 -triple=aarch64-linux-gnu -analyzer-checker=core,alpha.unix.Stream,debug.ExprInspection -verify %s +// RUN: %clang_analyze_cc1 -triple=hexagon -analyzer-checker=core,alpha.unix.Stream,debug.ExprInspection -verify %s #include "Inputs/system-header-simulator.h" +#include "Inputs/system-header-simulator-for-valist.h" void clang_analyzer_eval(int); @@ -65,12 +69,24 @@ void check_fseek(void) { fclose(fp); } +void check_fseeko(void) { + FILE *fp = tmpfile(); + fseeko(fp, 0, 0); // expected-warning {{Stream pointer might be NULL}} + fclose(fp); +} + void check_ftell(void) { FILE *fp = tmpfile(); ftell(fp); // expected-warning {{Stream pointer might be NULL}} fclose(fp); } +void check_ftello(void) { + FILE *fp = tmpfile(); + ftello(fp); // expected-warning {{Stream pointer might be NULL}} + fclose(fp); +} + void check_rewind(void) { FILE *fp = tmpfile(); rewind(fp); // expected-warning {{Stream pointer might be NULL}} @@ -129,6 +145,18 @@ void f_dopen(int fd) { fclose(F); } +void f_vfprintf(int fd, va_list args) { + FILE *F = fdopen(fd, "r"); + vfprintf(F, "%d", args); // expected-warning {{Stream pointer might be NULL}} + fclose(F); +} + +void f_vfscanf(int fd, va_list args) { + FILE *F = fdopen(fd, "r"); + vfscanf(F, "%u", args); // expected-warning {{Stream pointer might be NULL}} + fclose(F); +} + void f_seek(void) { FILE *p = fopen("foo", "r"); if (!p) @@ -138,6 +166,15 @@ void f_seek(void) { fclose(p); } +void f_seeko(void) { + FILE *p = fopen("foo", "r"); + if (!p) + return; + fseeko(p, 1, SEEK_SET); // no-warning + fseeko(p, 1, 3); // expected-warning {{The whence argument to fseek() should be SEEK_SET, SEEK_END, or SEEK_CUR}} + fclose(p); +} + void f_double_close(void) { FILE *p = fopen("foo", "r"); if (!p) -- GitLab From ea23761429faa4e61d7a601da9a313dfdc058cb0 Mon Sep 17 00:00:00 2001 From: Yuta Mukai Date: Wed, 6 Mar 2024 20:19:56 +0900 Subject: [PATCH 274/929] [AArch64] Verify ldp/stp alignment stricter (#84124) When ldp-aligned-only/stp-aligned-only is specified, modified to cancel ldp/stp transformation if MachineMemOperand is not present or the access size is unknown. In the previous implementation, the test passed when there was no MachineMemOperand. Also, if the size was unknown, an incorrect value was used or an assertion failed. (But actually, if there is no MachineMemOperand, it will be excluded from the target by isCandidateToMergeOrPair() before reaching the part.) A statistic NumFailedAlignmentCheck is added. NumPairCreated is modified so that it only counts if it is not canceled. --- .../AArch64/AArch64LoadStoreOptimizer.cpp | 44 ++++++++------- .../CodeGen/AArch64/ldp-stp-unknown-size.mir | 56 +++++++++++++++++++ 2 files changed, 81 insertions(+), 19 deletions(-) create mode 100644 llvm/test/CodeGen/AArch64/ldp-stp-unknown-size.mir diff --git a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp index 926a89466255..0ab2c401b174 100644 --- a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp +++ b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp @@ -62,6 +62,8 @@ STATISTIC(NumUnscaledPairCreated, "Number of load/store from unscaled generated"); STATISTIC(NumZeroStoresPromoted, "Number of narrow zero stores promoted"); STATISTIC(NumLoadsFromStoresPromoted, "Number of loads from stores promoted"); +STATISTIC(NumFailedAlignmentCheck, "Number of load/store pair transformation " + "not passed the alignment check"); DEBUG_COUNTER(RegRenamingCounter, DEBUG_TYPE "-reg-renaming", "Controls which pairs are considered for renaming"); @@ -2337,9 +2339,6 @@ bool AArch64LoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) { MachineBasicBlock::iterator Paired = findMatchingInsn(MBBI, Flags, LdStLimit, /* FindNarrowMerge = */ false); if (Paired != E) { - ++NumPairCreated; - if (TII->hasUnscaledLdStOffset(MI)) - ++NumUnscaledPairCreated; // Keeping the iterator straight is a pain, so we let the merge routine tell // us what the next instruction is after it's done mucking about. auto Prev = std::prev(MBBI); @@ -2349,24 +2348,31 @@ bool AArch64LoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) { MachineMemOperand *MemOp = MI.memoperands_empty() ? nullptr : MI.memoperands().front(); - // Get the needed alignments to check them if - // ldp-aligned-only/stp-aligned-only features are opted. - uint64_t MemAlignment = MemOp ? MemOp->getAlign().value() : -1; - uint64_t TypeAlignment = MemOp ? Align(MemOp->getSize()).value() : -1; + // If a load/store arrives and ldp/stp-aligned-only feature is opted, check + // that the alignment of the source pointer is at least double the alignment + // of the type. + if ((MI.mayLoad() && Subtarget->hasLdpAlignedOnly()) || + (MI.mayStore() && Subtarget->hasStpAlignedOnly())) { + // If there is no size/align information, cancel the transformation. + if (!MemOp || !MemOp->getMemoryType().isValid()) { + NumFailedAlignmentCheck++; + return false; + } - // If a load arrives and ldp-aligned-only feature is opted, check that the - // alignment of the source pointer is at least double the alignment of the - // type. - if (MI.mayLoad() && Subtarget->hasLdpAlignedOnly() && MemOp && - MemAlignment < 2 * TypeAlignment) - return false; + // Get the needed alignments to check them if + // ldp-aligned-only/stp-aligned-only features are opted. + uint64_t MemAlignment = MemOp->getAlign().value(); + uint64_t TypeAlignment = Align(MemOp->getSize()).value(); - // If a store arrives and stp-aligned-only feature is opted, check that the - // alignment of the source pointer is at least double the alignment of the - // type. - if (MI.mayStore() && Subtarget->hasStpAlignedOnly() && MemOp && - MemAlignment < 2 * TypeAlignment) - return false; + if (MemAlignment < 2 * TypeAlignment) { + NumFailedAlignmentCheck++; + return false; + } + } + + ++NumPairCreated; + if (TII->hasUnscaledLdStOffset(MI)) + ++NumUnscaledPairCreated; MBBI = mergePairedInsns(MBBI, Paired, Flags); // Collect liveness info for instructions between Prev and the new position diff --git a/llvm/test/CodeGen/AArch64/ldp-stp-unknown-size.mir b/llvm/test/CodeGen/AArch64/ldp-stp-unknown-size.mir new file mode 100644 index 000000000000..3234a7dc11f0 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/ldp-stp-unknown-size.mir @@ -0,0 +1,56 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4 +# RUN: llc -O2 -mtriple=aarch64 -mcpu=ampere1 -simplify-mir -o - %s -run-pass=aarch64-ldst-opt | FileCheck %s --check-prefixes=CHECK +# RUN: llc -O2 -mtriple=aarch64 -simplify-mir -o - %s -run-pass=aarch64-ldst-opt | FileCheck %s --check-prefixes=CHECK-DEFAULT + +--- | + define i32 @ldp_no_size_info(ptr %0) #0 { + %2 = ptrtoint ptr %0 to i64 + %3 = and i64 %2, -64 + %4 = inttoptr i64 %3 to ptr + %5 = load i32, ptr %4, align 4 + %6 = getelementptr inbounds i32, ptr %4, i64 1 + %7 = load i32, ptr %6, align 4 + %8 = add nsw i32 %7, %5 + ret i32 %8 + } + +... +--- +name: ldp_no_size_info +alignment: 64 +tracksRegLiveness: true +tracksDebugUserValues: true +liveins: + - { reg: '$x0' } +frameInfo: + maxAlignment: 1 + maxCallFrameSize: 0 +machineFunctionInfo: + hasRedZone: false +body: | + bb.0 (%ir-block.1): + liveins: $x0 + + ; CHECK-LABEL: name: ldp_no_size_info + ; CHECK: liveins: $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $x8 = ANDXri killed renamable $x0, 7865 + ; CHECK-NEXT: renamable $w9 = LDRWui renamable $x8, 0 :: (load unknown-size from %ir.4, align 1) + ; CHECK-NEXT: renamable $w8 = LDRWui killed renamable $x8, 1 :: (load unknown-size from %ir.6, align 1) + ; CHECK-NEXT: $w0 = ADDWrs killed renamable $w8, killed renamable $w9, 0 + ; CHECK-NEXT: RET undef $lr, implicit $w0 + ; + ; CHECK-DEFAULT-LABEL: name: ldp_no_size_info + ; CHECK-DEFAULT: liveins: $x0 + ; CHECK-DEFAULT-NEXT: {{ $}} + ; CHECK-DEFAULT-NEXT: renamable $x8 = ANDXri killed renamable $x0, 7865 + ; CHECK-DEFAULT-NEXT: renamable $w9, renamable $w8 = LDPWi renamable $x8, 0 :: (load unknown-size from %ir.4, align 1), (load unknown-size from %ir.6, align 1) + ; CHECK-DEFAULT-NEXT: $w0 = ADDWrs killed renamable $w8, killed renamable $w9, 0 + ; CHECK-DEFAULT-NEXT: RET undef $lr, implicit $w0 + renamable $x8 = ANDXri killed renamable $x0, 7865 + renamable $w9 = LDRWui renamable $x8, 0 :: (load unknown-size from %ir.4) + renamable $w8 = LDRWui killed renamable $x8, 1 :: (load unknown-size from %ir.6) + $w0 = ADDWrs killed renamable $w8, killed renamable $w9, 0 + RET undef $lr, implicit $w0 + +... -- GitLab From 86b630092f9a5139c4c4d627f1e0504e8f549bce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Wed, 6 Mar 2024 11:28:19 +0100 Subject: [PATCH 275/929] [clang][Interp] Allow getFieldPtr ops on null pointers in C --- clang/lib/AST/Interp/Interp.h | 3 ++- clang/test/AST/Interp/c.c | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h index baa1b8e9071a..9c3d58d5619c 100644 --- a/clang/lib/AST/Interp/Interp.h +++ b/clang/lib/AST/Interp/Interp.h @@ -1224,7 +1224,8 @@ inline bool GetPtrGlobal(InterpState &S, CodePtr OpPC, uint32_t I) { inline bool GetPtrField(InterpState &S, CodePtr OpPC, uint32_t Off) { const Pointer &Ptr = S.Stk.pop(); - if (S.inConstantContext() && !CheckNull(S, OpPC, Ptr, CSK_Field)) + if (S.getLangOpts().CPlusPlus && S.inConstantContext() && + !CheckNull(S, OpPC, Ptr, CSK_Field)) return false; if (CheckDummy(S, OpPC, Ptr)) { diff --git a/clang/test/AST/Interp/c.c b/clang/test/AST/Interp/c.c index a5270172cfc7..8de6139efbea 100644 --- a/clang/test/AST/Interp/c.c +++ b/clang/test/AST/Interp/c.c @@ -79,6 +79,11 @@ int a2[(intptr_t)&((struct y*)0)->y]; // all-warning {{folded to constant array} const struct y *yy = (struct y*)0; const intptr_t L = (intptr_t)(&(yy->y)); // all-error {{not a compile-time constant}} +_Static_assert((long)&((struct y*)0)->y > 0, ""); // pedantic-ref-warning {{GNU extension}} \ + // pedantic-ref-note {{this conversion is not allowed in a constant expression}} \ + // pedantic-expected-warning {{GNU extension}} \ + // pedantic-expected-note {{this conversion is not allowed in a constant expression}} + const ptrdiff_t m = &m + 137 - &m; _Static_assert(m == 137, ""); // pedantic-ref-warning {{GNU extension}} \ // pedantic-expected-warning {{GNU extension}} -- GitLab From a9ff5c0b75abc3b7aa5c357b226e067b4ddc04bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Wed, 6 Mar 2024 12:33:43 +0100 Subject: [PATCH 276/929] [clang][Docs] Add my Discourse + Discord name And escape the @ in my email --- clang/CodeOwners.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/CodeOwners.rst b/clang/CodeOwners.rst index 995a2f50226a..2ae04c129eb7 100644 --- a/clang/CodeOwners.rst +++ b/clang/CodeOwners.rst @@ -60,7 +60,7 @@ Analysis & CFG Experimental new constant interpreter ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ | Timm Bäder -| tbaeder@redhat.com (email), tbaeder (Phabricator), tbaederr (GitHub) +| tbaeder\@redhat.com (email), tbaeder (Phabricator), tbaederr (GitHub), tbaeder (Discourse), tbaeder (Discord) Modules & serialization -- GitLab From 5ddc5b8525a9256716cda1398249f0ab2dd968eb Mon Sep 17 00:00:00 2001 From: Kirill Chibisov Date: Wed, 6 Mar 2024 15:41:20 +0400 Subject: [PATCH 277/929] [mlir][emitc] Fix `emitc.expression` example (#84060) Make it use and refer to `emitc.yield` and also fix type issues. --- mlir/include/mlir/Dialect/EmitC/IR/EmitC.td | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td index 5679742bfa16..6bef395e94eb 100644 --- a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td +++ b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td @@ -389,17 +389,17 @@ def EmitC_ExpressionOp : EmitC_Op<"expression", As the operation is to be emitted as a C expression, the operations within its body must form a single Def-Use tree of emitc ops whose result is - yielded by a terminating `yield`. + yielded by a terminating `emitc.yield`. Example: ```mlir - %r = emitc.expression : () -> i32 { + %r = emitc.expression : i32 { %0 = emitc.add %a, %b : (i32, i32) -> i32 - %1 = emitc.call "foo"(%0) : () -> i32 + %1 = emitc.call_opaque "foo"(%0) : (i32) -> i32 %2 = emitc.add %c, %d : (i32, i32) -> i32 %3 = emitc.mul %1, %2 : (i32, i32) -> i32 - yield %3 + emitc.yield %3 : i32 } ``` @@ -409,9 +409,9 @@ def EmitC_ExpressionOp : EmitC_Op<"expression", int32_t v7 = foo(v1 + v2) * (v3 + v4); ``` - The operations allowed within expression body are emitc.add, emitc.apply, - emitc.call, emitc.cast, emitc.cmp, emitc.div, emitc.mul, emitc.rem and - emitc.sub. + The operations allowed within expression body are `emitc.add`, + `emitc.apply`, `emitc.call_opaque`, `emitc.cast`, `emitc.cmp`, `emitc.div`, + `emitc.mul`, `emitc.rem`, and `emitc.sub`. When specified, the optional `do_not_inline` indicates that the expression is to be emitted as seen above, i.e. as the rhs of an EmitC SSA value -- GitLab From cf922e51b850410a3a2944035432f44f1e741b21 Mon Sep 17 00:00:00 2001 From: Yingchi Long Date: Wed, 6 Mar 2024 19:47:44 +0800 Subject: [PATCH 278/929] [BPF] lowering target address leaf nodes tconstpool (#73667) Adds custom lowering for tconstpool. Please ref: https://github.com/llvm/llvm-project/pull/73668 for test coverage --- llvm/lib/Target/BPF/BPFISelLowering.cpp | 37 +++++++++++++++++++++---- llvm/lib/Target/BPF/BPFISelLowering.h | 5 ++++ llvm/lib/Target/BPF/BPFInstrInfo.td | 1 + llvm/lib/Target/BPF/BPFMCInstLower.cpp | 3 ++ 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Target/BPF/BPFISelLowering.cpp b/llvm/lib/Target/BPF/BPFISelLowering.cpp index 4d8ace7c1ece..b8ca8ec98c06 100644 --- a/llvm/lib/Target/BPF/BPFISelLowering.cpp +++ b/llvm/lib/Target/BPF/BPFISelLowering.cpp @@ -69,7 +69,7 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM, setOperationAction(ISD::BRIND, MVT::Other, Expand); setOperationAction(ISD::BRCOND, MVT::Other, Expand); - setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); + setOperationAction({ISD::GlobalAddress, ISD::ConstantPool}, MVT::i64, Custom); setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Custom); setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); @@ -308,6 +308,8 @@ SDValue BPFTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { return LowerBR_CC(Op, DAG); case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); + case ISD::ConstantPool: + return LowerConstantPool(Op, DAG); case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG); case ISD::SDIV: @@ -691,18 +693,41 @@ const char *BPFTargetLowering::getTargetNodeName(unsigned Opcode) const { return nullptr; } +static SDValue getTargetNode(GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty, + SelectionDAG &DAG, unsigned Flags) { + return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags); +} + +static SDValue getTargetNode(ConstantPoolSDNode *N, const SDLoc &DL, EVT Ty, + SelectionDAG &DAG, unsigned Flags) { + return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlign(), + N->getOffset(), Flags); +} + +template +SDValue BPFTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG, + unsigned Flags) const { + SDLoc DL(N); + + SDValue GA = getTargetNode(N, DL, MVT::i64, DAG, Flags); + + return DAG.getNode(BPFISD::Wrapper, DL, MVT::i64, GA); +} + SDValue BPFTargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const { - auto *N = cast(Op); + GlobalAddressSDNode *N = cast(Op); if (N->getOffset() != 0) report_fatal_error("invalid offset for global address: " + Twine(N->getOffset())); + return getAddr(N, DAG); +} - SDLoc DL(Op); - const GlobalValue *GV = N->getGlobal(); - SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i64); +SDValue BPFTargetLowering::LowerConstantPool(SDValue Op, + SelectionDAG &DAG) const { + ConstantPoolSDNode *N = cast(Op); - return DAG.getNode(BPFISD::Wrapper, DL, MVT::i64, GA); + return getAddr(N, DAG); } unsigned diff --git a/llvm/lib/Target/BPF/BPFISelLowering.h b/llvm/lib/Target/BPF/BPFISelLowering.h index 819711b650c1..42707949e864 100644 --- a/llvm/lib/Target/BPF/BPFISelLowering.h +++ b/llvm/lib/Target/BPF/BPFISelLowering.h @@ -77,8 +77,13 @@ private: SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const; SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const; SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const; + + SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const; SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const; + template + SDValue getAddr(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const; + // Lower the result values of a call, copying them out of physregs into vregs SDValue LowerCallResult(SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool IsVarArg, diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.td b/llvm/lib/Target/BPF/BPFInstrInfo.td index 690d53420718..82d347023106 100644 --- a/llvm/lib/Target/BPF/BPFInstrInfo.td +++ b/llvm/lib/Target/BPF/BPFInstrInfo.td @@ -727,6 +727,7 @@ let usesCustomInserter = 1, isCodeGenOnly = 1 in { // load 64-bit global addr into register def : Pat<(BPFWrapper tglobaladdr:$in), (LD_imm64 tglobaladdr:$in)>; +def : Pat<(BPFWrapper tconstpool:$in), (LD_imm64 tconstpool:$in)>; // 0xffffFFFF doesn't fit into simm32, optimize common case def : Pat<(i64 (and (i64 GPR:$src), 0xffffFFFF)), diff --git a/llvm/lib/Target/BPF/BPFMCInstLower.cpp b/llvm/lib/Target/BPF/BPFMCInstLower.cpp index 2ce9c386f24c..040a1fb75070 100644 --- a/llvm/lib/Target/BPF/BPFMCInstLower.cpp +++ b/llvm/lib/Target/BPF/BPFMCInstLower.cpp @@ -74,6 +74,9 @@ void BPFMCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const { case MachineOperand::MO_GlobalAddress: MCOp = LowerSymbolOperand(MO, GetGlobalAddressSymbol(MO)); break; + case MachineOperand::MO_ConstantPoolIndex: + MCOp = LowerSymbolOperand(MO, Printer.GetCPISymbol(MO.getIndex())); + break; } OutMI.addOperand(MCOp); -- GitLab From 716a23c412704c0224bbe97c1a55ced1b1624e0c Mon Sep 17 00:00:00 2001 From: Hristo Hristov Date: Wed, 6 Mar 2024 14:09:26 +0200 Subject: [PATCH 279/929] [libc++][NFC] Status page: fix minor formatting issues (#83969) Fixes several minor formatting issues and omissions I introduced on the status pages in previous patches. --- libcxx/docs/ReleaseNotes/19.rst | 2 +- libcxx/docs/Status/Cxx2cPapers.csv | 2 +- libcxx/docs/Status/SpaceshipPapers.csv | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst index b068325b35c9..04f16610f811 100644 --- a/libcxx/docs/ReleaseNotes/19.rst +++ b/libcxx/docs/ReleaseNotes/19.rst @@ -41,7 +41,7 @@ Implemented Papers - P2637R3 - Member ``visit`` - P2652R2 - Disallow User Specialization of ``allocator_traits`` - P2819R2 - Add ``tuple`` protocol to ``complex`` -- P2495R3 - Interfacing ``stringstream``s with ``string_view`` +- P2495R3 - Interfacing ``stringstream``\s with ``string_view`` - P2302R4 - ``std::ranges::contains`` - P1659R3 - ``std::ranges::starts_with`` and ``std::ranges::ends_with`` diff --git a/libcxx/docs/Status/Cxx2cPapers.csv b/libcxx/docs/Status/Cxx2cPapers.csv index febb0977255b..4a5443dea115 100644 --- a/libcxx/docs/Status/Cxx2cPapers.csv +++ b/libcxx/docs/Status/Cxx2cPapers.csv @@ -6,7 +6,7 @@ "`P2545R4 `__","LWG","Read-Copy Update (RCU)","Varna June 2023","","","" "`P2530R3 `__","LWG","Hazard Pointers for C++26","Varna June 2023","","","" "`P2538R1 `__","LWG","ADL-proof ``std::projected``","Varna June 2023","|Complete|","18.0","|ranges|" -"`P2495R3 `__","LWG","Interfacing ``stringstream``s with ``string_view``","Varna June 2023","|Complete|","19.0","" +"`P2495R3 `__","LWG","Interfacing ``stringstream``\s with ``string_view``","Varna June 2023","|Complete|","19.0","" "`P2510R3 `__","LWG","Formatting pointers","Varna June 2023","|Complete| [#note-P2510R3]_","17.0","|format|" "`P2198R7 `__","LWG","Freestanding Feature-Test Macros and Implementation-Defined Extensions","Varna June 2023","","","" "`P2338R4 `__","LWG","Freestanding Library: Character primitives and the C library","Varna June 2023","","","" diff --git a/libcxx/docs/Status/SpaceshipPapers.csv b/libcxx/docs/Status/SpaceshipPapers.csv index 48fe417bc7dc..21b788ee4292 100644 --- a/libcxx/docs/Status/SpaceshipPapers.csv +++ b/libcxx/docs/Status/SpaceshipPapers.csv @@ -3,10 +3,10 @@ `P2404R3 `_,"Relaxing ``equality_comparable_with``'s, ``totally_ordered_with``'s, and ``three_way_comparable_with``'s common reference requirements to support move-only types",, `LWG3330 `_,Include ```` from most library headers,"|Complete|","13.0" `LWG3347 `_,"``std::pair`` now requires ``T`` and ``U`` to be *less-than-comparable*",|Nothing To Do|, -`LWG3350 `_,Simplify return type of lexicographical_compare_three_way,|Nothing To Do|, +`LWG3350 `_,Simplify return type of ``lexicographical_compare_three_way``,|Nothing To Do|, `LWG3360 `_,``three_way_comparable_with`` is inconsistent with similar concepts,|Nothing To Do|, -`LWG3380 `_,common_type and comparison categories,|Nothing To Do|, -`LWG3395 `_,Definition for three-way comparison needs to be updated,|Nothing To Do|, +`LWG3380 `_,``common_type`` and comparison categories,|Nothing To Do|, +`LWG3395 `_,Definition for *three-way* comparison needs to be updated,|Nothing To Do|, `P0905R1 `_,Symmetry for spaceship,, `P1120R0 `_,Consistency improvements for ``<=>`` and other comparison operators,, `LWG3431 `_,``<=>`` for containers should require ``three_way_comparable`` instead of ``<=>``,, -- GitLab From c2e34a60c433f5dff79c4c2bdb2df5d8ad1f201f Mon Sep 17 00:00:00 2001 From: David Spickett Date: Wed, 6 Mar 2024 12:30:53 +0000 Subject: [PATCH 280/929] [lldb][lldb-server] Fix pipe spelling mistake in log message --- lldb/tools/lldb-server/lldb-gdbserver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lldb/tools/lldb-server/lldb-gdbserver.cpp b/lldb/tools/lldb-server/lldb-gdbserver.cpp index 38c00e83e7b0..563284730bc7 100644 --- a/lldb/tools/lldb-server/lldb-gdbserver.cpp +++ b/lldb/tools/lldb-server/lldb-gdbserver.cpp @@ -235,7 +235,7 @@ void ConnectToRemote(MainLoop &mainloop, Status error = writeSocketIdToPipe(named_pipe_path, socket_id); if (error.Fail()) llvm::errs() << llvm::formatv( - "failed to write to the named peipe '{0}': {1}\n", + "failed to write to the named pipe '{0}': {1}\n", named_pipe_path, error.AsCString()); } // If we have an unnamed pipe to write the socket id back to, do -- GitLab From a888f5e4d7d979617a47262a875f5433ee161074 Mon Sep 17 00:00:00 2001 From: Ivan Kosarev Date: Wed, 6 Mar 2024 14:44:19 +0200 Subject: [PATCH 281/929] [AMDGPU][NFC] Update tests to use -triple= instead of -arch=. (#84153) --- llvm/test/MC/AMDGPU/bf16_imm.s | 4 ++-- llvm/test/MC/AMDGPU/gfx1150_asm_features.s | 4 ++-- llvm/test/MC/AMDGPU/gfx1150_asm_salu_float.s | 2 +- llvm/test/MC/AMDGPU/gfx1150_asm_sopp.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_ds.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_ds_alias.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_exp.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_features.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_global_load_tr.s | 8 ++++---- llvm/test/MC/AMDGPU/gfx12_asm_smem.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_sop1.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_sop2.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_sop2_alias.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_sopc.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_sopk.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_sopk_alias.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_sopp.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mtbuf.s | 4 ++-- llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mtbuf_alias.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mubuf.s | 4 ++-- llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mubuf_alias.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vdsdir.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vflat.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vflat_alias.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vimage.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vimage_alias.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vop1.s | 4 ++-- llvm/test/MC/AMDGPU/gfx12_asm_vop1_dpp16.s | 4 ++-- llvm/test/MC/AMDGPU/gfx12_asm_vop1_dpp8.s | 4 ++-- llvm/test/MC/AMDGPU/gfx12_asm_vop1_t16_err.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vop1_t16_promote.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vop2.s | 8 ++++---- llvm/test/MC/AMDGPU/gfx12_asm_vop2_aliases.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp16.s | 8 ++++---- llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp8.s | 8 ++++---- llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_err.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_promote.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vop3.s | 8 ++++---- llvm/test/MC/AMDGPU/gfx12_asm_vop3_aliases.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s | 8 ++++---- llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s | 8 ++++---- llvm/test/MC/AMDGPU/gfx12_asm_vop3_err.s | 4 ++-- llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1_dpp16.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1_dpp8.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2.s | 8 ++++---- llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp16.s | 8 ++++---- llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp8.s | 8 ++++---- llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s | 8 ++++---- llvm/test/MC/AMDGPU/gfx12_asm_vop3c_dpp16.s | 8 ++++---- llvm/test/MC/AMDGPU/gfx12_asm_vop3c_dpp8.s | 8 ++++---- llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s | 4 ++-- llvm/test/MC/AMDGPU/gfx12_asm_vop3cx_dpp16.s | 4 ++-- llvm/test/MC/AMDGPU/gfx12_asm_vop3cx_dpp8.s | 4 ++-- llvm/test/MC/AMDGPU/gfx12_asm_vop3p.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vop3p_aliases.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vop3p_dpp16.s | 4 ++-- llvm/test/MC/AMDGPU/gfx12_asm_vop3p_dpp16_err.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vop3p_dpp8.s | 4 ++-- llvm/test/MC/AMDGPU/gfx12_asm_vop3p_dpp8_err.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vop3p_err.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vop3p_features.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vopc.s | 8 ++++---- llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp16.s | 8 ++++---- llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp8.s | 8 ++++---- llvm/test/MC/AMDGPU/gfx12_asm_vopc_t16_err.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vopc_t16_promote.s | 8 ++++---- llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s | 4 ++-- llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp16.s | 4 ++-- llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp8.s | 4 ++-- llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_err.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_promote.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vopd.s | 6 +++--- llvm/test/MC/AMDGPU/gfx12_asm_vopd_errs.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vopd_features.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_vsample.s | 2 +- llvm/test/MC/AMDGPU/gfx12_asm_wmma_w32.s | 4 ++-- llvm/test/MC/AMDGPU/gfx12_asm_wmma_w64.s | 4 ++-- llvm/test/MC/AMDGPU/gfx12_err.s | 2 +- llvm/test/MC/AMDGPU/gfx12_unsupported.s | 2 +- .../test/MC/Disassembler/AMDGPU/gfx1150_dasm_features.txt | 2 +- .../MC/Disassembler/AMDGPU/gfx1150_dasm_salu_float.txt | 2 +- llvm/test/MC/Disassembler/AMDGPU/gfx1150_dasm_sopp.txt | 2 +- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_ds.txt | 2 +- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_exp.txt | 2 +- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_features.txt | 2 +- .../MC/Disassembler/AMDGPU/gfx12_dasm_global_load_tr.txt | 4 ++-- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_smem.txt | 2 +- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt | 2 +- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop2.txt | 4 ++-- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopc.txt | 2 +- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopk.txt | 4 ++-- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopp.txt | 2 +- .../MC/Disassembler/AMDGPU/gfx12_dasm_vbuffer_mtbuf.txt | 2 +- .../MC/Disassembler/AMDGPU/gfx12_dasm_vbuffer_mubuf.txt | 2 +- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vdsdir.txt | 2 +- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vflat.txt | 2 +- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vimage.txt | 2 +- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vinterp.txt | 2 +- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1.txt | 4 ++-- .../test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1_dpp16.txt | 4 ++-- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1_dpp8.txt | 4 ++-- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2.txt | 4 ++-- .../test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp16.txt | 4 ++-- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp8.txt | 4 ++-- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt | 4 ++-- .../test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt | 4 ++-- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt | 4 ++-- .../MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop1.txt | 2 +- .../AMDGPU/gfx12_dasm_vop3_from_vop1_dpp16.txt | 2 +- .../AMDGPU/gfx12_dasm_vop3_from_vop1_dpp8.txt | 2 +- .../MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop2.txt | 4 ++-- .../AMDGPU/gfx12_dasm_vop3_from_vop2_dpp16.txt | 4 ++-- .../AMDGPU/gfx12_dasm_vop3_from_vop2_dpp8.txt | 4 ++-- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3c.txt | 4 ++-- .../MC/Disassembler/AMDGPU/gfx12_dasm_vop3c_dpp16.txt | 4 ++-- .../test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3c_dpp8.txt | 4 ++-- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx.txt | 4 ++-- .../MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx_dpp16.txt | 4 ++-- .../MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx_dpp8.txt | 4 ++-- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p.txt | 4 ++-- .../MC/Disassembler/AMDGPU/gfx12_dasm_vop3p_dpp16.txt | 4 ++-- .../test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p_dpp8.txt | 4 ++-- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p_err.txt | 2 +- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc.txt | 4 ++-- .../test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc_dpp16.txt | 4 ++-- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc_dpp8.txt | 4 ++-- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx.txt | 4 ++-- .../MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp16.txt | 4 ++-- .../test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp8.txt | 4 ++-- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopd.txt | 2 +- .../MC/Disassembler/AMDGPU/gfx12_dasm_vopd_features.txt | 2 +- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vsample.txt | 2 +- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_wmma_w32.txt | 4 ++-- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_wmma_w64.txt | 4 ++-- 135 files changed, 238 insertions(+), 238 deletions(-) diff --git a/llvm/test/MC/AMDGPU/bf16_imm.s b/llvm/test/MC/AMDGPU/bf16_imm.s index 1bf76dfa1731..654248064106 100644 --- a/llvm/test/MC/AMDGPU/bf16_imm.s +++ b/llvm/test/MC/AMDGPU/bf16_imm.s @@ -1,5 +1,5 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1100 -show-encoding %s | FileCheck %s -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -show-encoding %s | FileCheck %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck %s v_dot2_bf16_bf16 v5, v1, v2, 100.0 // CHECK: v_dot2_bf16_bf16 v5, v1, v2, 0x42c8 ; encoding: [0x05,0x00,0x67,0xd6,0x01,0x05,0xfe,0x03,0xc8,0x42,0x00,0x00] diff --git a/llvm/test/MC/AMDGPU/gfx1150_asm_features.s b/llvm/test/MC/AMDGPU/gfx1150_asm_features.s index 55c855175a89..056221fd7eda 100644 --- a/llvm/test/MC/AMDGPU/gfx1150_asm_features.s +++ b/llvm/test/MC/AMDGPU/gfx1150_asm_features.s @@ -1,5 +1,5 @@ -// RUN: llvm-mc -arch=amdgcn -show-encoding -mcpu=gfx1150 %s | FileCheck --check-prefix=GFX1150 %s -// RUN: llvm-mc -arch=amdgcn -show-encoding -mcpu=gfx1151 %s | FileCheck --check-prefix=GFX1150 %s +// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1150 %s | FileCheck --check-prefix=GFX1150 %s +// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1151 %s | FileCheck --check-prefix=GFX1150 %s // // Subtargets allow src1 of VOP3 DPP instructions to be SGPR or inlinable diff --git a/llvm/test/MC/AMDGPU/gfx1150_asm_salu_float.s b/llvm/test/MC/AMDGPU/gfx1150_asm_salu_float.s index 0c097bf24a14..9ea01cd2ceaa 100644 --- a/llvm/test/MC/AMDGPU/gfx1150_asm_salu_float.s +++ b/llvm/test/MC/AMDGPU/gfx1150_asm_salu_float.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1150 -show-encoding %s | FileCheck --check-prefixes=GFX1150 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1150 -show-encoding %s | FileCheck --check-prefixes=GFX1150 %s s_cvt_f32_i32 s5, s1 // GFX1150: encoding: [0x01,0x64,0x85,0xbe] diff --git a/llvm/test/MC/AMDGPU/gfx1150_asm_sopp.s b/llvm/test/MC/AMDGPU/gfx1150_asm_sopp.s index 463e9cade7f9..044ce48c2678 100644 --- a/llvm/test/MC/AMDGPU/gfx1150_asm_sopp.s +++ b/llvm/test/MC/AMDGPU/gfx1150_asm_sopp.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1150 -show-encoding %s | FileCheck --check-prefixes=GFX1150 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1150 -show-encoding %s | FileCheck --check-prefixes=GFX1150 %s s_singleuse_vdst 0x0000 // GFX1150: encoding: [0x00,0x00,0x93,0xbf] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_ds.s b/llvm/test/MC/AMDGPU/gfx12_asm_ds.s index 54c4d15e701c..a0e6a3a61355 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_ds.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_ds.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s ds_nop // GFX12: [0x00,0x00,0x50,0xd8,0x00,0x00,0x00,0x00] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_ds_alias.s b/llvm/test/MC/AMDGPU/gfx12_asm_ds_alias.s index 057e99330bca..c10b96a29217 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_ds_alias.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_ds_alias.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s ds_max_f32 v1, v2 // GFX12: ds_max_num_f32 v1, v2 ; encoding: [0x00,0x00,0x4c,0xd8,0x01,0x02,0x00,0x00] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_exp.s b/llvm/test/MC/AMDGPU/gfx12_asm_exp.s index 2390813954a2..a614168325d7 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_exp.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_exp.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck -check-prefix=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck -check-prefix=GFX12 %s export mrt0 off, off, off, off // GFX12: export mrt0 off, off, off, off ; encoding: [0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x00] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_features.s b/llvm/test/MC/AMDGPU/gfx12_asm_features.s index d0540d2199b1..bb911c65c2bd 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_features.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_features.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefix=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefix=GFX12 %s // // Subtargets allow src1 of VOP3 DPP instructions to be SGPR or inlinable diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_global_load_tr.s b/llvm/test/MC/AMDGPU/gfx12_asm_global_load_tr.s index 597e0d29b43a..842cc5180340 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_global_load_tr.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_global_load_tr.s @@ -1,7 +1,7 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W64 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W32 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W64 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W32 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s global_load_tr_b128 v[1:4], v0, s[0:1] offset:-64 // W64-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_smem.s b/llvm/test/MC/AMDGPU/gfx12_asm_smem.s index eb59607755da..a64f35337d0e 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_smem.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_smem.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefix=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefix=GFX12 %s //===----------------------------------------------------------------------===// // ENC_SMEM. diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s b/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s index 6f32e6cccfbc..4fd355f10f34 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefix=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefix=GFX12 %s s_sleep_var 0x1234 // GFX12: encoding: [0xff,0x58,0x80,0xbe,0x34,0x12,0x00,0x00] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_sop2.s b/llvm/test/MC/AMDGPU/gfx12_asm_sop2.s index d1d920872418..c9b58d892fea 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_sop2.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_sop2.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck -check-prefix=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck -check-prefix=GFX12 %s s_add_nc_u64 s[0:1], s[2:3], s[4:5] // GFX12: encoding: [0x02,0x04,0x80,0xa9] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_sop2_alias.s b/llvm/test/MC/AMDGPU/gfx12_asm_sop2_alias.s index e3ad0198cae5..e30e640d0fc0 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_sop2_alias.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_sop2_alias.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefix=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefix=GFX12 %s s_add_i32 s0, s1, s2 // GFX12: s_add_co_i32 s0, s1, s2 ; encoding: [0x01,0x02,0x00,0x81] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_sopc.s b/llvm/test/MC/AMDGPU/gfx12_asm_sopc.s index 6887ec318851..56a95d504b93 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_sopc.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_sopc.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s s_cmp_lt_f32 s1, s2 // GFX12: encoding: [0x01,0x02,0x41,0xbf] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_sopk.s b/llvm/test/MC/AMDGPU/gfx12_asm_sopk.s index 3d8d59cfada3..69566bb45338 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_sopk.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_sopk.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefix=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefix=GFX12 %s s_movk_i32 s0, 0x1234 // GFX12: encoding: [0x34,0x12,0x00,0xb0] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_sopk_alias.s b/llvm/test/MC/AMDGPU/gfx12_asm_sopk_alias.s index aae08fda8d6e..4a25922f956d 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_sopk_alias.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_sopk_alias.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefix=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefix=GFX12 %s s_addk_i32 s0, 0x1234 // GFX12: s_addk_co_i32 s0, 0x1234 ; encoding: [0x34,0x12,0x80,0xb7] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_sopp.s b/llvm/test/MC/AMDGPU/gfx12_asm_sopp.s index 73d04f18e860..e98659208d5a 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_sopp.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_sopp.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefix=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefix=GFX12 %s s_wait_loadcnt 0x1234 // GFX12: encoding: [0x34,0x12,0xc0,0xbf] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mtbuf.s b/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mtbuf.s index d2cd09ad1d40..9f43ce37c307 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mtbuf.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mtbuf.s @@ -1,5 +1,5 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefixes=GFX12-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefixes=GFX12-ERR --implicit-check-not=error: %s tbuffer_load_d16_format_x v4, off, s[8:11], s3 format:[BUF_FMT_8_UNORM] offset:8388607 // GFX12: encoding: [0x03,0x00,0x22,0xc4,0x04,0x10,0x80,0x00,0x00,0xff,0xff,0x7f] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mtbuf_alias.s b/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mtbuf_alias.s index ef00edcf846e..cb98c9824e19 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mtbuf_alias.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mtbuf_alias.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s tbuffer_load_format_d16_x v4, off, s[8:11], s3 format:[BUF_FMT_8_UNORM] offset:8388607 // GFX12: tbuffer_load_d16_format_x v4, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x22,0xc4,0x04,0x10,0x80,0x00,0x00,0xff,0xff,0x7f] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mubuf.s b/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mubuf.s index 6ce4f1952abe..08ec5b3f6a52 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mubuf.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mubuf.s @@ -1,5 +1,5 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefixes=GFX12-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefixes=GFX12-ERR --implicit-check-not=error: %s buffer_load_b32 v5, off, s[8:11], s3 offset:8388607 // GFX12: encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mubuf_alias.s b/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mubuf_alias.s index 7363750900c0..69ff0211b9fd 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mubuf_alias.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mubuf_alias.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s buffer_load_dword v5, off, s[8:11], s3 offset:8388607 // GFX12: buffer_load_b32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vdsdir.s b/llvm/test/MC/AMDGPU/gfx12_asm_vdsdir.s index dbd732f99992..ada18da8e388 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vdsdir.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vdsdir.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --strict-whitespace -check-prefix=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --strict-whitespace -check-prefix=GFX12 %s ds_direct_load v1 wait_va_vdst:15 // GFX12: ds_direct_load v1 wait_va_vdst:15 wait_vm_vsrc:0 ; encoding: [0x01,0x00,0x1f,0xce] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vflat.s b/llvm/test/MC/AMDGPU/gfx12_asm_vflat.s index c814b42c1275..575bc1a8255a 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vflat.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vflat.s @@ -1,4 +1,4 @@ -; RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s +; RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s flat_atomic_add_f32 v1, v[0:1], v2 offset:-8000000 th:TH_ATOMIC_RETURN // GFX12: encoding: [0x7c,0x80,0x15,0xec,0x01,0x00,0x10,0x01,0x00,0x00,0xee,0x85] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vflat_alias.s b/llvm/test/MC/AMDGPU/gfx12_asm_vflat_alias.s index df453d96117a..6fe999ddaafc 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vflat_alias.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vflat_alias.s @@ -1,4 +1,4 @@ -; RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck -check-prefix=GFX12 %s +; RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck -check-prefix=GFX12 %s global_atomic_csub v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN // GFX12: global_atomic_sub_clamp_u32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0xc0,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vimage.s b/llvm/test/MC/AMDGPU/gfx12_asm_vimage.s index c06fb614b88b..39010883a3c0 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vimage.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vimage.s @@ -1,4 +1,4 @@ -; RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +; RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D // GFX12: encoding: [0x00,0x00,0xc0,0xd3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vimage_alias.s b/llvm/test/MC/AMDGPU/gfx12_asm_vimage_alias.s index 43c462c9b6d4..a88a3ef100fb 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vimage_alias.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vimage_alias.s @@ -1,4 +1,4 @@ -; RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +; RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s image_atomic_add v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D // GFX12: image_atomic_add_uint v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x00,0x43,0xd0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s index c9c4fceffaeb..a21ef11bfb87 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s @@ -1,5 +1,5 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=GFX12 %s -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=GFX12 %s v_bfrev_b32_e32 v5, v1 // GFX12: encoding: [0x01,0x71,0x0a,0x7e] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop1_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop1_dpp16.s index 5e0e1b688bc5..29ae941a4e85 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop1_dpp16.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop1_dpp16.s @@ -1,5 +1,5 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s v_bfrev_b32_dpp v5, v1 quad_perm:[3,2,1,0] // GFX12: encoding: [0xfa,0x70,0x0a,0x7e,0x01,0x1b,0x00,0xff] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop1_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop1_dpp8.s index 36c89710ce8f..9c9e63f0335b 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop1_dpp8.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop1_dpp8.s @@ -1,5 +1,5 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s v_bfrev_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] // GFX12: encoding: [0xe9,0x70,0x0a,0x7e,0x01,0x77,0x39,0x05] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop1_t16_err.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop1_t16_err.s index f34d029e02da..37edf627e366 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop1_t16_err.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop1_t16_err.s @@ -1,4 +1,4 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=error %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=error %s v_ceil_f16_e32 v128, 0xfe0b // GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop1_t16_promote.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop1_t16_promote.s index dce0b0378993..648a1e02428b 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop1_t16_promote.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop1_t16_promote.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 --implicit-check-not=_e32 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 --implicit-check-not=_e32 %s v_ceil_f16 v128, 0xfe0b // GFX12: v_ceil_f16_e64 diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop2.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop2.s index 2cf98c9e6106..233af7e1b5d3 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop2.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop2.s @@ -1,7 +1,7 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s v_add_co_ci_u32_e32 v5, vcc_lo, v1, v2, vcc_lo // W32: encoding: [0x01,0x05,0x0a,0x40] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_aliases.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_aliases.s index 53bd4649fd3a..5976d8af01f3 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_aliases.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_aliases.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s v_min_f32 v5, v1, v2 // GFX12: v_min_num_f32_e32 v5, v1, v2 ; encoding: [0x01,0x05,0x0a,0x2a] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp16.s index 0c9a3f4f3359..548c9186ec78 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp16.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp16.s @@ -1,7 +1,7 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo quad_perm:[3,2,1,0] // W32: encoding: [0xfa,0x04,0x0a,0x40,0x01,0x1b,0x00,0xff] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp8.s index 423fcf307470..72bf613e2a37 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp8.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp8.s @@ -1,7 +1,7 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] // W32: encoding: [0xe9,0x04,0x0a,0x40,0x01,0x77,0x39,0x05] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_err.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_err.s index d25411b5bfd2..045d698bd504 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_err.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_err.s @@ -1,4 +1,4 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=error %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=error %s v_add_f16_e32 v255, v1, v2 // GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_promote.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_promote.s index febc6311c18a..13939842f730 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_promote.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_t16_promote.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=_e32 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=_e32 %s v_add_f16 v255, v1, v2 // GFX12: v_add_f16_e64 diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s index beb57999b855..9d0488fd2f1c 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s @@ -1,7 +1,7 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s v_add3_u32 v5, v1, v2, s3 // GFX12: encoding: [0x05,0x00,0x55,0xd6,0x01,0x05,0x0e,0x00] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_aliases.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_aliases.s index f2ae7dd8c35a..3829e0601b1e 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_aliases.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_aliases.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s v_min3_f32 v5, v1, v2, v3 // GFX12: v_min3_num_f32 v5, v1, v2, v3 ; encoding: [0x05,0x00,0x29,0xd6,0x01,0x05,0x0e,0x04] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s index df3430f376f6..88bdb7ef571e 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s @@ -1,7 +1,7 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefixes=GFX12-ERR,W32-ERR --implicit-check-not=error: %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefixes=GFX12-ERR,W64-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefixes=GFX12-ERR,W32-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefixes=GFX12-ERR,W64-ERR --implicit-check-not=error: %s v_add3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] // GFX12: [0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s index 09dd6df618c5..0e84765d9ff0 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s @@ -1,7 +1,7 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefixes=GFX12-ERR,W32-ERR --implicit-check-not=error: %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefixes=GFX12-ERR,W64-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefixes=GFX12-ERR,W32-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefixes=GFX12-ERR,W64-ERR --implicit-check-not=error: %s v_add3_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] // GFX12: [0x05,0x00,0x55,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_err.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_err.s index d8236c302b6d..55c5fcabea73 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_err.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_err.s @@ -1,5 +1,5 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --strict-whitespace --implicit-check-not=error %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --strict-whitespace --implicit-check-not=error %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --strict-whitespace --implicit-check-not=error %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --strict-whitespace --implicit-check-not=error %s v_permlane16_b32 v5, v1, s2, s3 op_sel:[0, 0, 0, 1] // GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid op_sel operand diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1.s index 7ee60262a5c1..9c6a3b8e68cd 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=GFX12 %s v_bfrev_b32_e64 v5, v1 // GFX12: encoding: [0x05,0x00,0xb8,0xd5,0x01,0x01,0x00,0x00] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1_dpp16.s index 808f941197c4..706e9e3717dc 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1_dpp16.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1_dpp16.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=GFX12 %s v_bfrev_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0] // GFX12: [0x05,0x00,0xb8,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1_dpp8.s index f7b51cfb6bda..002cd778a997 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1_dpp8.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1_dpp8.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=GFX12 %s v_bfrev_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] // GFX12: [0x05,0x00,0xb8,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2.s index 7ba33bc9170e..f78acd074d87 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2.s @@ -1,7 +1,7 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s v_add_co_ci_u32_e64 v5, s6, v1, 0xaf123456, s3 // W32: encoding: [0x05,0x06,0x20,0xd5,0x01,0xff,0x0d,0x00,0x56,0x34,0x12,0xaf] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp16.s index 1b8eb5a7f169..ab88ec813e2f 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp16.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp16.s @@ -1,7 +1,7 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[3,2,1,0] // W32: [0x05,0x06,0x20,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x1b,0x00,0xff] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp8.s index cbe7d774cf62..dc151d664e3f 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp8.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp8.s @@ -1,7 +1,7 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] // W32: [0x05,0x06,0x20,0xd5,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s index c674f120ec4d..8bc776498c06 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s @@ -1,7 +1,7 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s v_cmp_class_f16_e64 s5, v1, v2 // W32: encoding: [0x05,0x00,0x7d,0xd4,0x01,0x05,0x02,0x00] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3c_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3c_dpp16.s index ed4718ab29aa..b50b18e7e381 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3c_dpp16.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3c_dpp16.s @@ -1,7 +1,7 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s v_cmp_class_f16_e64_dpp s5, v1, v2 quad_perm:[3,2,1,0] // W32: [0x05,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3c_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3c_dpp8.s index d3474a339d55..b9dc61420ccb 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3c_dpp8.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3c_dpp8.s @@ -1,7 +1,7 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s v_cmp_class_f16_e64_dpp s5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] // W32: [0x05,0x00,0x7d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s index c6ee72a99c02..46ecb9f4d2ca 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s @@ -1,5 +1,5 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s v_cmpx_class_f16_e64 v1, v2 // GFX12: encoding: [0x7e,0x00,0xfd,0xd4,0x01,0x05,0x02,0x00] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx_dpp16.s index bae502f8210d..03958ba09c70 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx_dpp16.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx_dpp16.s @@ -1,5 +1,5 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s v_cmpx_class_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] // GFX12: [0x7e,0x00,0xfd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx_dpp8.s index 9f4b825ae47d..efc6168018f0 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx_dpp8.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx_dpp8.s @@ -1,5 +1,5 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s v_cmpx_class_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] // GFX12: [0x7e,0x00,0xfd,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3p.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3p.s index 567fc61d0b93..db9ad3d2a841 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3p.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3p.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefix=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefix=GFX12 %s v_dot2_f32_bf16 v5, v1, v2, v3 // GFX12: [0x05,0x40,0x1a,0xcc,0x01,0x05,0x0e,0x1c] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_aliases.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_aliases.s index eca35691af50..79054e9c4063 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_aliases.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_aliases.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s v_pk_min_f16 v0, v1, v2 // GFX12: v_pk_min_num_f16 v0, v1, v2 ; encoding: [0x00,0x40,0x1b,0xcc,0x01,0x05,0x02,0x18] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_dpp16.s index a6360684f1d0..32378c545ab0 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_dpp16.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_dpp16.s @@ -1,5 +1,5 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s v_dot2_f32_f16 v0, v1, v2, v3 neg_lo:[0,0,0] neg_hi:[0,0,0] quad_perm:[2,2,3,1] bound_ctrl:0 fi:1 // GFX12: v_dot2_f32_f16_e64_dpp v0, v1, v2, v3 quad_perm:[2,2,3,1] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x00,0x40,0x13,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x7a,0x04,0xff] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_dpp16_err.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_dpp16_err.s index b76754123207..491d0ad50255 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_dpp16_err.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_dpp16_err.s @@ -1,4 +1,4 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=error: %s // check for error with sgpr or imm operands diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_dpp8.s index 299339339e8c..dec1a7c512b8 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_dpp8.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_dpp8.s @@ -1,5 +1,5 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s v_fma_mix_f32 v0, v1, v2, v3 dpp8:[2,2,2,2,4,4,4,4] // GFX12: encoding: [0x00,0x00,0x20,0xcc,0xe9,0x04,0x0e,0x04,0x01,0x92,0x44,0x92] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_dpp8_err.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_dpp8_err.s index 50d3b6aca41b..fc5243d5f65f 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_dpp8_err.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_dpp8_err.s @@ -1,4 +1,4 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=error: %s // check for error with sgpr or imm operands diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_err.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_err.s index 269989a26e28..a7d42c6b923a 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_err.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_err.s @@ -1,4 +1,4 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=error: %s v_dot4_f32_fp8_bf8 v0, v1, v2, v3 clamp // GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_features.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_features.s index f52dff263c63..5d49575b3c9a 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_features.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3p_features.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefix=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefix=GFX12 %s // // Test op_sel/op_sel_hi diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopc.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopc.s index 18d83ad7ef4a..e0d48972575e 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vopc.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopc.s @@ -1,7 +1,7 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W32 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W64 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W32 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W64 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s v_cmp_class_f16_e32 vcc_lo, v1, v2 // W32: encoding: [0x01,0x05,0xfa,0x7c] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp16.s index bebf149a9b85..acccb95a2be0 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp16.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp16.s @@ -1,7 +1,7 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W32 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W64 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W32 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W64 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s v_cmp_class_f16_dpp vcc_lo, v1, v2 quad_perm:[3,2,1,0] // W32: encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x1b,0x00,0xff] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp8.s index 57a30964a848..9e5e7e3f88ea 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp8.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopc_dpp8.s @@ -1,7 +1,7 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W32 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W64 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W32 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W64 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s v_cmp_class_f16_dpp vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] // W32: encoding: [0xe9,0x04,0xfa,0x7c,0x01,0x77,0x39,0x05] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopc_t16_err.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopc_t16_err.s index 3b76c26a2f40..ebdd8adc0adf 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vopc_t16_err.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopc_t16_err.s @@ -1,4 +1,4 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=error %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=error %s v_cmp_class_f16_e32 vcc, v1, v255 // GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopc_t16_promote.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopc_t16_promote.s index bec31a47ec37..47e9b3a7ad7e 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vopc_t16_promote.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopc_t16_promote.s @@ -1,7 +1,7 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W32 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W64 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W32 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W64 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s v_cmp_class_f16 vcc, v1, v255 // W64: v_cmp_class_f16_e64 vcc, v1, v255 diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s index 73fee99592a1..ffbf50cce76e 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s @@ -1,5 +1,5 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s v_cmpx_class_f16_e32 v1, v2 // GFX12: encoding: [0x01,0x05,0xfa,0x7d] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp16.s index 1243e6b0cecb..63974efa9fbd 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp16.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp16.s @@ -1,5 +1,5 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s v_cmpx_class_f16_dpp v1, v2 quad_perm:[3,2,1,0] // GFX12: encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x1b,0x00,0xff] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp8.s index 91a6e2e1de51..1a3e1fb1866a 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp8.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_dpp8.s @@ -1,5 +1,5 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s v_cmpx_class_f16_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] // GFX12: encoding: [0xe9,0x04,0xfa,0x7d,0x01,0x77,0x39,0x05] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_err.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_err.s index b3b7f1340517..8d6bff152101 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_err.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_err.s @@ -1,4 +1,4 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=error %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 --implicit-check-not=error %s v_cmpx_class_f16_e32 v1, v255 // GFX12: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_promote.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_promote.s index 1d9603b884e4..9d7b60b91730 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_promote.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx_t16_promote.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12 %s v_cmpx_class_f16 v1, v255 // GFX12: v_cmpx_class_f16_e64 diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopd.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopd.s index 5fb0d2f0d4eb..8df347cbc119 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vopd.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopd.s @@ -1,6 +1,6 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --check-prefixes=W64-ERR --implicit-check-not=error: %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --check-prefixes=W64-ERR --implicit-check-not=error: %s v_dual_add_f32 v255, v4, v2 :: v_dual_add_f32 v6, v1, v3 // GFX12: encoding: [0x04,0x05,0x08,0xc9,0x01,0x07,0x06,0xff] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopd_errs.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopd_errs.s index 0712814db955..5751258fe85d 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vopd_errs.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopd_errs.s @@ -1,4 +1,4 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 %s 2>&1 | FileCheck %s -check-prefix=GFX12 --implicit-check-not=error: --strict-whitespace +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 %s 2>&1 | FileCheck %s -check-prefix=GFX12 --implicit-check-not=error: --strict-whitespace //===----------------------------------------------------------------------===// // A VOPD instruction can use only one literal. diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopd_features.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopd_features.s index 840dc3f82f2b..7b5e6dd3da75 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vopd_features.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopd_features.s @@ -1,4 +1,4 @@ -// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck -check-prefix=GFX12 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck -check-prefix=GFX12 %s //===----------------------------------------------------------------------===// // A VOPD instruction can use one or more literals, diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vsample.s b/llvm/test/MC/AMDGPU/gfx12_asm_vsample.s index 260761261177..bb036332e4b5 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_vsample.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_vsample.s @@ -1,4 +1,4 @@ -; RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +; RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s image_sample v64, v32, s[4:11], s[100:103] dmask:0x1 dim:SQ_RSRC_IMG_1D // GFX12: encoding: [0x00,0xc0,0x46,0xe4,0x40,0x08,0x00,0x32,0x20,0x00,0x00,0x00] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_wmma_w32.s b/llvm/test/MC/AMDGPU/gfx12_asm_wmma_w32.s index e1cd0cab6634..6907619313ca 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_wmma_w32.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_wmma_w32.s @@ -1,5 +1,5 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: %s v_wmma_f32_16x16x16_f16 v[8:15], v[0:3], v[4:7], v[8:15] // GFX12: v_wmma_f32_16x16x16_f16 v[8:15], v[0:3], v[4:7], v[8:15] ; encoding: [0x08,0x40,0x40,0xcc,0x00,0x09,0x22,0x1c] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_wmma_w64.s b/llvm/test/MC/AMDGPU/gfx12_asm_wmma_w64.s index 8bd9e5039b72..6a52cf5c2af8 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_wmma_w64.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_wmma_w64.s @@ -1,5 +1,5 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=GFX12 %s -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=GFX12 %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: %s v_wmma_f32_16x16x16_f16 v[4:7], v[0:1], v[2:3], v[4:7] // GFX12: v_wmma_f32_16x16x16_f16 v[4:7], v[0:1], v[2:3], v[4:7] ; encoding: [0x04,0x40,0x40,0xcc,0x00,0x05,0x12,0x1c] diff --git a/llvm/test/MC/AMDGPU/gfx12_err.s b/llvm/test/MC/AMDGPU/gfx12_err.s index edc24f4cf4fe..8b2565cb7f56 100644 --- a/llvm/test/MC/AMDGPU/gfx12_err.s +++ b/llvm/test/MC/AMDGPU/gfx12_err.s @@ -1,4 +1,4 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefixes=GFX12-ERR --implicit-check-not=error: -strict-whitespace %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefixes=GFX12-ERR --implicit-check-not=error: -strict-whitespace %s v_cubesc_f32_e64_dpp v5, v1, v2, 12345678 row_shr:4 row_mask:0xf bank_mask:0xf // GFX12-ERR: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction diff --git a/llvm/test/MC/AMDGPU/gfx12_unsupported.s b/llvm/test/MC/AMDGPU/gfx12_unsupported.s index 6cd0fe2ad008..f0debbf052bc 100644 --- a/llvm/test/MC/AMDGPU/gfx12_unsupported.s +++ b/llvm/test/MC/AMDGPU/gfx12_unsupported.s @@ -1,4 +1,4 @@ -// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 %s 2>&1 | FileCheck --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 %s 2>&1 | FileCheck --implicit-check-not=error: %s //===----------------------------------------------------------------------===// // Unsupported instructions. diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1150_dasm_features.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1150_dasm_features.txt index 5ed9a673db2e..6ab3e08d8691 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx1150_dasm_features.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1150_dasm_features.txt @@ -1,4 +1,4 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1150 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1150 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1150 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1150 %s # GFX1150: v_add3_u32_e64_dpp v5, v1, s2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0x0c,0x04,0x01,0x1b,0x00,0xff] 0x05,0x00,0x55,0xd6,0xfa,0x04,0x0c,0x04,0x01,0x1b,0x00,0xff diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1150_dasm_salu_float.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1150_dasm_salu_float.txt index 14732def4bfe..07a71b6986d7 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx1150_dasm_salu_float.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1150_dasm_salu_float.txt @@ -1,4 +1,4 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1150 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1150 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1150 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1150 %s # GFX1150: s_cvt_f32_i32 s5, s1 ; encoding: [0x01,0x64,0x85,0xbe] 0x01,0x64,0x85,0xbe diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1150_dasm_sopp.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1150_dasm_sopp.txt index ddeb70a55969..8fa266a73ff8 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx1150_dasm_sopp.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1150_dasm_sopp.txt @@ -1,4 +1,4 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1150 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1150 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1150 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1150 %s # GFX1150: s_singleuse_vdst 0x0 ; encoding: [0x00,0x00,0x93,0xbf] 0x00,0x00,0x93,0xbf diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_ds.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_ds.txt index f7e1e8dd2fd4..36c58d4c6732 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_ds.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_ds.txt @@ -1,4 +1,4 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefix=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefix=GFX12 %s # GFX12: ds_add_f32 v0, v1 ; encoding: [0x00,0x00,0x54,0xd8,0x00,0x01,0x00,0x00] 0x00,0x00,0x54,0xd8,0x00,0x01,0x00,0x00 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_exp.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_exp.txt index 3f173123983b..8d131da08b6b 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_exp.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_exp.txt @@ -1,4 +1,4 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck %s -check-prefix=GFX12 +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck %s -check-prefix=GFX12 # GFX12: export dual_src_blend0 v4, v3, v2, v1 ; encoding: [0x5f,0x01,0x00,0xf8,0x04,0x03,0x02,0x01] 0x5f,0x01,0x00,0xf8,0x04,0x03,0x02,0x01 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_features.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_features.txt index 4ea6d8b7e27f..0c4427cff63e 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_features.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_features.txt @@ -1,4 +1,4 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s # GFX12: v_add3_u32_e64_dpp v5, v1, s2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0x0c,0x04,0x01,0x1b,0x00,0xff] 0x05,0x00,0x55,0xd6,0xfa,0x04,0x0c,0x04,0x01,0x1b,0x00,0xff diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_global_load_tr.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_global_load_tr.txt index e8498d4aef0a..b561c76571af 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_global_load_tr.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_global_load_tr.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -disassemble -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W64 %s -# RUN: llvm-mc -disassemble -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W32 %s +# RUN: llvm-mc -disassemble -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W64 %s +# RUN: llvm-mc -disassemble -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=W32 %s # W32: global_load_tr_b128 v[1:4], v0, s[0:1] offset:64 ; encoding: [0x00,0xc0,0x15,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00] # W64: global_load_tr_b128 v[1:2], v0, s[0:1] offset:64 ; encoding: [0x00,0xc0,0x15,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00] diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_smem.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_smem.txt index 81cd930d2261..28decdd4c5b1 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_smem.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_smem.txt @@ -1,4 +1,4 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s # GFX12: s_load_i8 s5, s[2:3], s0 offset:0x0 ; encoding: [0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0x00] 0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0x00 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt index d15a329c8ead..c87cea120568 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt @@ -1,4 +1,4 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s # GFX12: s_sleep_var 0x1234 ; encoding: [0xff,0x58,0x80,0xbe,0x34,0x12,0x00,0x00] 0xff,0x58,0x80,0xbe,0x34,0x12,0x00,0x00 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop2.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop2.txt index 297b6b083550..349aedcb78a8 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop2.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop2.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefix=GFX12 %s -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-WavefrontSize32,+WavefrontSize64 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefix=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefix=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-WavefrontSize32,+WavefrontSize64 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefix=GFX12 %s # GFX12: s_add_nc_u64 s[0:1], s[2:3], s[4:5] ; encoding: [0x02,0x04,0x80,0xa9] 0x02,0x04,0x80,0xa9 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopc.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopc.txt index ad418db854c8..ac023f10aed6 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopc.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopc.txt @@ -1,4 +1,4 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s # GFX12: s_cmp_lt_f32 s1, s2 ; encoding: [0x01,0x02,0x41,0xbf] 0x01,0x02,0x41,0xbf diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopk.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopk.txt index ebf65a45d176..f2b9858388d9 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopk.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopk.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefix=GFX12 %s -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-WavefrontSize32,+WavefrontSize64 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefix=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefix=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-WavefrontSize32,+WavefrontSize64 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefix=GFX12 %s # GFX12: s_addk_co_i32 exec_hi, 0x1234 ; encoding: [0x34,0x12,0xff,0xb7] 0x34,0x12,0xff,0xb7 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopp.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopp.txt index e9371d14864a..d42f920aa61d 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopp.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopp.txt @@ -1,4 +1,4 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefixes=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefixes=GFX12 %s # GFX12: s_wait_alu depctr_hold_cnt(0) depctr_sa_sdst(0) depctr_va_vdst(0) depctr_va_sdst(0) depctr_va_ssrc(0) depctr_va_vcc(0) depctr_vm_vsrc(0) ; encoding: [0x00,0x00,0x88,0xbf] 0x00,0x00,0x88,0xbf diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vbuffer_mtbuf.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vbuffer_mtbuf.txt index affea2e081c1..8d8cfc172ad7 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vbuffer_mtbuf.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vbuffer_mtbuf.txt @@ -1,4 +1,4 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s # GFX12: tbuffer_load_d16_format_x v4, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x22,0xc4,0x04,0x10,0x80,0x00,0x00,0xff,0xff,0x7f] 0x03,0x00,0x22,0xc4,0x04,0x10,0x80,0x00,0x00,0xff,0xff,0x7f diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vbuffer_mubuf.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vbuffer_mubuf.txt index 8b74ff25ba50..cb3b125041b9 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vbuffer_mubuf.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vbuffer_mubuf.txt @@ -1,4 +1,4 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s # GFX12: buffer_load_b32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f] 0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vdsdir.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vdsdir.txt index 705dd6438124..9e43aee375fb 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vdsdir.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vdsdir.txt @@ -1,4 +1,4 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck --strict-whitespace -check-prefix=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck --strict-whitespace -check-prefix=GFX12 %s # GFX12: ds_direct_load v10 wait_va_vdst:6 wait_vm_vsrc:0 ; encoding: [0x0a,0x00,0x16,0xce] 0x0a,0x00,0x16,0xce diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vflat.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vflat.txt index 181f2bd64606..e0b658b1fda3 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vflat.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vflat.txt @@ -1,4 +1,4 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck %s -check-prefix=GFX12 +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck %s -check-prefix=GFX12 # GFX12: flat_atomic_add_f32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x15,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00] 0x7c,0x80,0x15,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vimage.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vimage.txt index bb09f602c0ac..aa49caacb4fc 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vimage.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vimage.txt @@ -1,4 +1,4 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding %s | FileCheck --check-prefixes=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding %s | FileCheck --check-prefixes=GFX12 %s # GFX12: image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x00,0xc0,0xd3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00] 0x00,0x00,0xc0,0xd3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vinterp.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vinterp.txt index bd1db69625ca..977cd732947c 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vinterp.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vinterp.txt @@ -1,4 +1,4 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble %s | FileCheck -strict-whitespace -check-prefix=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble %s | FileCheck -strict-whitespace -check-prefix=GFX12 %s # GFX12: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0{{$}} 0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1.txt index 39bb7338c807..d54f819c5a06 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s # GFX12: v_bfrev_b32_e32 v5, v1 ; encoding: [0x01,0x71,0x0a,0x7e] 0x01,0x71,0x0a,0x7e diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1_dpp16.txt index 5848333f41ef..09e0a925d90e 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1_dpp16.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1_dpp16.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s # GFX12: v_bfrev_b32_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x70,0x0a,0x7e,0x01,0x1b,0x00,0xff] 0xfa,0x70,0x0a,0x7e,0x01,0x1b,0x00,0xff diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1_dpp8.txt index d42e9ae25039..8329ae4cefc9 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1_dpp8.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1_dpp8.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s # GFX12: v_bfrev_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x70,0x0a,0x7e,0x01,0x77,0x39,0x05] 0xe9,0x70,0x0a,0x7e,0x01,0x77,0x39,0x05 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2.txt index 05f4f184c903..1021298b276e 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s # W32: v_add_co_ci_u32_e32 v5, vcc_lo, v1, v2, vcc_lo ; encoding: [0x01,0x05,0x0a,0x40] # W64: v_add_co_ci_u32_e32 v5, vcc, v1, v2, vcc ; encoding: [0x01,0x05,0x0a,0x40] diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp16.txt index 7f7d76819306..337ae1a13970 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp16.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp16.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x1b,0x00,0xff] # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x1b,0x00,0xff] diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp8.txt index 27d59eced837..0cb02f4de30b 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp8.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop2_dpp8.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s # W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x40,0x01,0x77,0x39,0x05] # W64: v_add_co_ci_u32_dpp v5, vcc, v1, v2, vcc dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x40,0x01,0x77,0x39,0x05] diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt index f86903b8de44..6acaa8152720 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s # GFX12: v_add3_u32 v5, v1, v2, s3 ; encoding: [0x05,0x00,0x55,0xd6,0x01,0x05,0x0e,0x00] 0x05,0x00,0x55,0xd6,0x01,0x05,0x0e,0x00 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt index 1be1d6e91ad8..4303c6d2c880 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s # GFX12: v_add3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff] 0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt index 44b3f7594029..c73ffe7e2cf4 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s # GFX12: v_add3_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05] 0x05,0x00,0x55,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop1.txt index 9a8368a65f3d..5c7cc3a8e223 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop1.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop1.txt @@ -1,4 +1,4 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s # GFX12: v_bfrev_b32_e64 v5, v1 ; encoding: [0x05,0x00,0xb8,0xd5,0x01,0x01,0x00,0x00] 0x05,0x00,0xb8,0xd5,0x01,0x01,0x00,0x00 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop1_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop1_dpp16.txt index 8af274e0b402..ac745c543324 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop1_dpp16.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop1_dpp16.txt @@ -1,4 +1,4 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s # GFX12: v_bfrev_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xb8,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff] 0x05,0x00,0xb8,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop1_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop1_dpp8.txt index 3d48d58c775b..fdeda3bb272d 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop1_dpp8.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop1_dpp8.txt @@ -1,4 +1,4 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s # GFX12: v_bfrev_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xb8,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05] 0x05,0x00,0xb8,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop2.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop2.txt index c14f82f9d985..6802d790f576 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop2.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop2.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s # W32: v_add_co_ci_u32_e64 v5, s12, v1, 0xaf123456, s6 ; encoding: [0x05,0x0c,0x20,0xd5,0x01,0xff,0x19,0x00,0x56,0x34,0x12,0xaf] # W64: v_add_co_ci_u32_e64 v5, s[12:13], v1, 0xaf123456, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0x01,0xff,0x19,0x00,0x56,0x34,0x12,0xaf] diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop2_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop2_dpp16.txt index 49cd84b9e662..56d7805bde8d 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop2_dpp16.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop2_dpp16.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s # W32: v_add_co_ci_u32_e64_dpp v5, s12, v1, v2, s6 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x20,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x1b,0x00,0xff] # W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x20,0xd5,0xfa,0x04,0x1a,0x00,0x01,0x1b,0x00,0xff] diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop2_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop2_dpp8.txt index 3972cc7b9a61..da7faa82290f 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop2_dpp8.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop2_dpp8.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s # W32: v_add_co_ci_u32_e64_dpp v5, s12, v1, v2, s6 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0c,0x20,0xd5,0xe9,0x04,0x1a,0x00,0x01,0x77,0x39,0x05] # W64: v_add_co_ci_u32_e64_dpp v5, s[12:13], v1, v2, s[6:7] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0c,0x20,0xd5,0xe9,0x04,0x1a,0x00,0x01,0x77,0x39,0x05] diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3c.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3c.txt index 3062124e36b8..90dca108a36b 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3c.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3c.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s # W32: v_cmp_class_f16_e64 s10, v1, v2 ; encoding: [0x0a,0x00,0x7d,0xd4,0x01,0x05,0x02,0x00] # W64: v_cmp_class_f16_e64 s[10:11], v1, v2 ; encoding: [0x0a,0x00,0x7d,0xd4,0x01,0x05,0x02,0x00] diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3c_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3c_dpp16.txt index 3edfb4a97862..e6ea6da1e259 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3c_dpp16.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3c_dpp16.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s # W32: v_cmp_class_f16_e64_dpp s10, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff] # W64: v_cmp_class_f16_e64_dpp s[10:11], v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x0a,0x00,0x7d,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff] diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3c_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3c_dpp8.txt index 23be05d295a7..98f8fd959116 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3c_dpp8.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3c_dpp8.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64 %s # W32: v_cmp_class_f16_e64_dpp s10, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x7d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05] # W64: v_cmp_class_f16_e64_dpp s[10:11], v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x0a,0x00,0x7d,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05] diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx.txt index 47a76cc3ef89..28f2bea07e57 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s # GFX12: v_cmpx_class_f16_e64 v1, v2 ; encoding: [0x7e,0x00,0xfd,0xd4,0x01,0x05,0x02,0x00] 0x7e,0x00,0xfd,0xd4,0x01,0x05,0x02,0x00 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx_dpp16.txt index 77888c0e1553..eb7675f677f4 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx_dpp16.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx_dpp16.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s # GFX12: v_cmpx_class_f16_e64_dpp v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x7e,0x00,0xfd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff] 0x7e,0x00,0xfd,0xd4,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx_dpp8.txt index 11972f4ba655..d5e112eb3424 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx_dpp8.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3cx_dpp8.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s # GFX12: v_cmpx_class_f16_e64_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x7e,0x00,0xfd,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05] 0x7e,0x00,0xfd,0xd4,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p.txt index 0f6debcbd9e3..53c4350a7dc1 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s # GFX12: v_dot2_f32_bf16 v5, v1, v2, v3 ; encoding: [0x05,0x40,0x1a,0xcc,0x01,0x05,0x0e,0x1c] 0x05,0x40,0x1a,0xcc,0x01,0x05,0x0e,0x1c diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p_dpp16.txt index 10f438465d65..426e70103d93 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p_dpp16.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p_dpp16.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s # GFX12: v_dot2_f32_f16_e64_dpp v0, v1, v2, v3 neg_lo:[1,1,0] neg_hi:[1,0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xe ; encoding: [0x00,0x45,0x13,0xcc,0xfa,0x04,0x0e,0x7c,0x01,0x1b,0x00,0xfe] 0x00,0x45,0x13,0xcc,0xfa,0x04,0x0e,0x7c,0x01,0x1b,0x00,0xfe diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p_dpp8.txt index 2fb9c23ed5ec..6f4290cfdb12 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p_dpp8.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p_dpp8.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s # GFX12: v_dot2_f32_f16_e64_dpp v0, v1, v2, v3 neg_lo:[0,1,1] neg_hi:[1,0,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x00,0x45,0x13,0xcc,0xe9,0x04,0x0e,0xdc,0x01,0x77,0x39,0x05] 0x00,0x45,0x13,0xcc,0xe9,0x04,0x0e,0xdc,0x01,0x77,0x39,0x05 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p_err.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p_err.txt index 36de7251f377..75f3f4e2dfed 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p_err.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p_err.txt @@ -1,4 +1,4 @@ -# RUN: not llvm-mc -disassemble -arch=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --implicit-check-not=warning: --check-prefix=GFX12 %s +# RUN: not llvm-mc -disassemble -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --implicit-check-not=warning: --check-prefix=GFX12 %s # v_dot4_f32_fp8_bf8 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc.txt index 902a378a9392..d9a75d25c1f2 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W32 -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W64 +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W32 +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W64 # W32: v_cmp_class_f16_e32 vcc_lo, v1, v2 ; encoding: [0x01,0x05,0xfa,0x7c] # W64: v_cmp_class_f16_e32 vcc, v1, v2 ; encoding: [0x01,0x05,0xfa,0x7c] diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc_dpp16.txt index 3ec29cf78ff3..4afd8b1c8a32 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc_dpp16.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc_dpp16.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W32 -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W64 +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W32 +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W64 # W32: v_cmp_class_f16 vcc_lo, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x1b,0x00,0xff] # W64: v_cmp_class_f16 vcc, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0xfa,0x7c,0x01,0x1b,0x00,0xff] diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc_dpp8.txt index 10518e0af8af..906da92b9afc 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc_dpp8.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopc_dpp8.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W32 -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W64 +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W32 +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=W64 # W32: v_cmp_class_f16 vcc_lo, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0xfa,0x7c,0x01,0x77,0x39,0x05] # W64: v_cmp_class_f16 vcc, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0xfa,0x7c,0x01,0x77,0x39,0x05] diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx.txt index 96d5e82e23b3..6588cdd0fba0 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12 -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12 +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12 +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12 # GFX12: v_cmpx_class_f16_e32 v1, v2 ; encoding: [0x01,0x05,0xfa,0x7d] 0x01,0x05,0xfa,0x7d diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp16.txt index 2ed92dbe192c..14f0ca729671 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp16.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp16.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12 -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12 +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12 +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12 # GFX12: v_cmpx_class_f16 v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0xfa,0x7d,0x01,0x1b,0x00,0xff] 0xfa,0x04,0xfa,0x7d,0x01,0x1b,0x00,0xff diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp8.txt index 9374b8b9287d..7ca10f5bf3d3 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp8.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopcx_dpp8.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12 -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12 +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12 +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12 # GFX12: v_cmpx_class_f16 v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0xfa,0x7d,0x01,0x77,0x39,0x05] 0xe9,0x04,0xfa,0x7d,0x01,0x77,0x39,0x05 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopd.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopd.txt index afd29bcdeb08..26bb8b8f92b0 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopd.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopd.txt @@ -1,4 +1,4 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12 +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX12 # GFX12: v_dual_add_f32 v255, v4, v2 :: v_dual_add_f32 v6, v1, v3 ; encoding: [0x04,0x05,0x08,0xc9,0x01,0x07,0x06,0xff] 0x04,0x05,0x08,0xc9,0x01,0x07,0x06,0xff diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopd_features.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopd_features.txt index fc943f7f003e..31d88c55f433 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopd_features.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vopd_features.txt @@ -1,4 +1,4 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX12 %s #===------------------------------------------------------------------------===# # Check instructions with several literals diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vsample.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vsample.txt index e777d541502e..e03849ffa838 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vsample.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vsample.txt @@ -1,4 +1,4 @@ -# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s # GFX12: image_sample v64, v32, s[4:11], s[100:103] dmask:0x1 dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0xc0,0x46,0xe4,0x40,0x08,0x00,0x32,0x20,0x00,0x00,0x00] 0x00,0xc0,0x46,0xe4,0x40,0x08,0x00,0x32,0x20,0x00,0x00,0x00 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_wmma_w32.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_wmma_w32.txt index 5079d2f08965..653a6a4be491 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_wmma_w32.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_wmma_w32.txt @@ -1,5 +1,5 @@ -# RUN: not llvm-mc -disassemble -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s -# RUN: not llvm-mc -disassemble -arch=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --implicit-check-not=warning: --check-prefix=GFX12-ERR %s +# RUN: not llvm-mc -disassemble -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s +# RUN: not llvm-mc -disassemble -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --implicit-check-not=warning: --check-prefix=GFX12-ERR %s [0x08,0x40,0x40,0xcc,0x00,0x09,0x22,0x1c] # GFX12: v_wmma_f32_16x16x16_f16 v[8:15], v[0:3], v[4:7], v[8:15] ; encoding: [0x08,0x40,0x40,0xcc,0x00,0x09,0x22,0x1c] diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_wmma_w64.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_wmma_w64.txt index 61700faa8e60..4fc236da131b 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_wmma_w64.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_wmma_w64.txt @@ -1,5 +1,5 @@ -# RUN: not llvm-mc -disassemble -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=GFX12 %s -# RUN: not llvm-mc -disassemble -arch=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --implicit-check-not=warning: --check-prefix=GFX12-ERR %s +# RUN: not llvm-mc -disassemble -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s | FileCheck --check-prefix=GFX12 %s +# RUN: not llvm-mc -disassemble -triple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -show-encoding %s 2>&1 | FileCheck --implicit-check-not=warning: --check-prefix=GFX12-ERR %s [0x04,0x40,0x40,0xcc,0x00,0x05,0x12,0x1c] # GFX12: v_wmma_f32_16x16x16_f16 v[4:7], v[0:1], v[2:3], v[4:7] ; encoding: [0x04,0x40,0x40,0xcc,0x00,0x05,0x12,0x1c] -- GitLab From df267fe32759658bebd846f98fdf5db61312837d Mon Sep 17 00:00:00 2001 From: Marius Brehler Date: Wed, 6 Mar 2024 12:46:29 +0000 Subject: [PATCH 282/929] [mlir][EmitC] Correct comment (NFC) --- mlir/lib/Conversion/FuncToEmitC/FuncToEmitCPass.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlir/lib/Conversion/FuncToEmitC/FuncToEmitCPass.cpp b/mlir/lib/Conversion/FuncToEmitC/FuncToEmitCPass.cpp index 26d32e29bef8..0b97f2641ad0 100644 --- a/mlir/lib/Conversion/FuncToEmitC/FuncToEmitCPass.cpp +++ b/mlir/lib/Conversion/FuncToEmitC/FuncToEmitCPass.cpp @@ -1,4 +1,4 @@ -//===- FuncToEmitC.cpp - Func to EmitC Pass ---------------------*- C++ -*-===// +//===- FuncToEmitCPass.cpp - Func to EmitC Pass -----------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. -- GitLab From a6ee0adb724a3f1705fc54937e734ffc2a032096 Mon Sep 17 00:00:00 2001 From: Michael Maitland Date: Wed, 6 Mar 2024 07:59:29 -0500 Subject: [PATCH 283/929] [llvm-mca][AMDGPU] Retire instructions that have issue carry over correctly (#83881) https://github.com/llvm/llvm-project/issues/83775 shows llvm-mca hits sanitizer error in cycleEnd. There was an instruction that takes multiple cycles to issue and is finished executing directly after issue. Prior to this patch, the instruction is retired on the first issue cycle, despite taking multiple cycles to issue. To fix this, if an instruction takes multiple cycles to issue and is done executing after issue, let updateCarriedOver retire the instruction when it is fully issued. --- llvm/lib/MCA/Stages/InOrderIssueStage.cpp | 36 +++++--- .../test/tools/llvm-mca/AMDGPU/carried-over.s | 87 +++++++++++++++++++ 2 files changed, 113 insertions(+), 10 deletions(-) create mode 100644 llvm/test/tools/llvm-mca/AMDGPU/carried-over.s diff --git a/llvm/lib/MCA/Stages/InOrderIssueStage.cpp b/llvm/lib/MCA/Stages/InOrderIssueStage.cpp index 0f1737dc3cbc..8f720dbd82a7 100644 --- a/llvm/lib/MCA/Stages/InOrderIssueStage.cpp +++ b/llvm/lib/MCA/Stages/InOrderIssueStage.cpp @@ -257,13 +257,13 @@ llvm::Error InOrderIssueStage::tryIssue(InstRef &IR) { } // If the instruction has a latency of 0, we need to handle - // the execution and retirement now. - if (IS.isExecuted()) { + // the execution and retirement now. If the instruction is issued in multiple + // cycles, we cannot handle the instruction being executed here so we make + // updateCarriedOver responsible. + if (IS.isExecuted() && !ShouldCarryOver) { PRF.onInstructionExecuted(&IS); LSU.onInstructionExecuted(IR); - notifyEvent( - HWInstructionEvent(HWInstructionEvent::Executed, IR)); - LLVM_DEBUG(dbgs() << "[E] Instruction #" << IR << " is executed\n"); + notifyInstructionExecuted(IR); retireInstruction(IR); return llvm::ErrorSuccess(); @@ -294,12 +294,18 @@ void InOrderIssueStage::updateIssuedInst() { continue; } - PRF.onInstructionExecuted(&IS); - LSU.onInstructionExecuted(IR); - notifyInstructionExecuted(IR); - ++NumExecuted; + // If the instruction takes multiple cycles to issue, defer these calls + // to updateCarriedOver. We still remove from IssuedInst even if there is + // carry over to avoid an extra call to cycleEvent in the next cycle. + if (!CarriedOver) { + PRF.onInstructionExecuted(&IS); + LSU.onInstructionExecuted(IR); + notifyInstructionExecuted(IR); - retireInstruction(*I); + retireInstruction(*I); + } + + ++NumExecuted; std::iter_swap(I, E - NumExecuted); } @@ -329,6 +335,16 @@ void InOrderIssueStage::updateCarriedOver() { else Bandwidth -= CarryOver; + // updateIssuedInst defered these calls to updateCarriedOver when there was + // a carry over. + if (CarriedOver.getInstruction()->isExecuted()) { + PRF.onInstructionExecuted(CarriedOver.getInstruction()); + LSU.onInstructionExecuted(CarriedOver); + notifyInstructionExecuted(CarriedOver); + + retireInstruction(CarriedOver); + } + CarriedOver = InstRef(); CarryOver = 0; } diff --git a/llvm/test/tools/llvm-mca/AMDGPU/carried-over.s b/llvm/test/tools/llvm-mca/AMDGPU/carried-over.s new file mode 100644 index 000000000000..8317469699a1 --- /dev/null +++ b/llvm/test/tools/llvm-mca/AMDGPU/carried-over.s @@ -0,0 +1,87 @@ +# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py +# RUN: llvm-mca -mtriple=amdgcn -mcpu=gfx940 --timeline --iterations=1 --timeline-max-cycles=0 < %s | FileCheck %s + +v_pk_mov_b32 v[0:1], v[2:3], v[4:5] +v_pk_add_f32 v[0:1], v[0:1], v[0:1] +v_pk_mul_f32 v[0:1], v[0:1], v[0:1] +v_add_co_u32 v5, s[0:1], v1, v2 +v_sub_co_u32 v5, s[0:1], v1, v2 +v_add_u32 v5, v1, v2 +v_sub_u32 v5, v1, v2 + +# CHECK: Iterations: 1 +# CHECK-NEXT: Instructions: 7 +# CHECK-NEXT: Total Cycles: 10 +# CHECK-NEXT: Total uOps: 9 + +# CHECK: Dispatch Width: 1 +# CHECK-NEXT: uOps Per Cycle: 0.90 +# CHECK-NEXT: IPC: 0.70 +# CHECK-NEXT: Block RThroughput: 9.0 + +# CHECK: Instruction Info: +# CHECK-NEXT: [1]: #uOps +# CHECK-NEXT: [2]: Latency +# CHECK-NEXT: [3]: RThroughput +# CHECK-NEXT: [4]: MayLoad +# CHECK-NEXT: [5]: MayStore +# CHECK-NEXT: [6]: HasSideEffects (U) + +# CHECK: [1] [2] [3] [4] [5] [6] Instructions: +# CHECK-NEXT: 1 1 1.00 U v_pk_mov_b32 v[0:1], v[2:3], v[4:5] +# CHECK-NEXT: 1 1 1.00 U v_pk_add_f32 v[0:1], v[0:1], v[0:1] +# CHECK-NEXT: 1 1 1.00 U v_pk_mul_f32 v[0:1], v[0:1], v[0:1] +# CHECK-NEXT: 2 1 1.00 U v_add_co_u32_e64 v5, s[0:1], v1, v2 +# CHECK-NEXT: 2 1 1.00 U v_sub_co_u32_e64 v5, s[0:1], v1, v2 +# CHECK-NEXT: 1 1 1.00 U v_add_u32_e32 v5, v1, v2 +# CHECK-NEXT: 1 1 1.00 U v_sub_u32_e32 v5, v1, v2 + +# CHECK: Resources: +# CHECK-NEXT: [0] - HWBranch +# CHECK-NEXT: [1] - HWExport +# CHECK-NEXT: [2] - HWLGKM +# CHECK-NEXT: [3] - HWSALU +# CHECK-NEXT: [4] - HWVALU +# CHECK-NEXT: [5] - HWVMEM +# CHECK-NEXT: [6] - HWXDL + +# CHECK: Resource pressure per iteration: +# CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] +# CHECK-NEXT: - - - 2.00 7.00 - - + +# CHECK: Resource pressure by instruction: +# CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] Instructions: +# CHECK-NEXT: - - - - 1.00 - - v_pk_mov_b32 v[0:1], v[2:3], v[4:5] +# CHECK-NEXT: - - - - 1.00 - - v_pk_add_f32 v[0:1], v[0:1], v[0:1] +# CHECK-NEXT: - - - - 1.00 - - v_pk_mul_f32 v[0:1], v[0:1], v[0:1] +# CHECK-NEXT: - - - 1.00 1.00 - - v_add_co_u32_e64 v5, s[0:1], v1, v2 +# CHECK-NEXT: - - - 1.00 1.00 - - v_sub_co_u32_e64 v5, s[0:1], v1, v2 +# CHECK-NEXT: - - - - 1.00 - - v_add_u32_e32 v5, v1, v2 +# CHECK-NEXT: - - - - 1.00 - - v_sub_u32_e32 v5, v1, v2 + +# CHECK: Timeline view: +# CHECK-NEXT: Index 0123456789 + +# CHECK: [0,0] DE . . v_pk_mov_b32 v[0:1], v[2:3], v[4:5] +# CHECK-NEXT: [0,1] .DE . . v_pk_add_f32 v[0:1], v[0:1], v[0:1] +# CHECK-NEXT: [0,2] . DE . . v_pk_mul_f32 v[0:1], v[0:1], v[0:1] +# CHECK-NEXT: [0,3] . DE. . v_add_co_u32_e64 v5, s[0:1], v1, v2 +# CHECK-NEXT: [0,4] . DeE . v_sub_co_u32_e64 v5, s[0:1], v1, v2 +# CHECK-NEXT: [0,5] . . DE. v_add_u32_e32 v5, v1, v2 +# CHECK-NEXT: [0,6] . . DE v_sub_u32_e32 v5, v1, v2 + +# CHECK: Average Wait times (based on the timeline view): +# CHECK-NEXT: [0]: Executions +# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue +# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready +# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage + +# CHECK: [0] [1] [2] [3] +# CHECK-NEXT: 0. 1 0.0 0.0 0.0 v_pk_mov_b32 v[0:1], v[2:3], v[4:5] +# CHECK-NEXT: 1. 1 0.0 0.0 0.0 v_pk_add_f32 v[0:1], v[0:1], v[0:1] +# CHECK-NEXT: 2. 1 0.0 0.0 0.0 v_pk_mul_f32 v[0:1], v[0:1], v[0:1] +# CHECK-NEXT: 3. 1 0.0 0.0 0.0 v_add_co_u32_e64 v5, s[0:1], v1, v2 +# CHECK-NEXT: 4. 1 0.0 0.0 0.0 v_sub_co_u32_e64 v5, s[0:1], v1, v2 +# CHECK-NEXT: 5. 1 0.0 0.0 0.0 v_add_u32_e32 v5, v1, v2 +# CHECK-NEXT: 6. 1 0.0 0.0 0.0 v_sub_u32_e32 v5, v1, v2 +# CHECK-NEXT: 1 0.0 0.0 0.0 -- GitLab From c486d90bdcbd50046fbd8ac874e4b0e62e287193 Mon Sep 17 00:00:00 2001 From: Balazs Benics Date: Wed, 6 Mar 2024 14:02:16 +0100 Subject: [PATCH 284/929] [analyzer][NFC] Document check::ASTCodeBody checker callback (#84160) Fixes #73764 With this patch, now all the callbacks are demonstrated here. --- .../Checkers/CheckerDocumentation.cpp | 61 +++++++++++-------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/clang/lib/StaticAnalyzer/Checkers/CheckerDocumentation.cpp b/clang/lib/StaticAnalyzer/Checkers/CheckerDocumentation.cpp index 01e0bed54cc6..153a1b1acbfa 100644 --- a/clang/lib/StaticAnalyzer/Checkers/CheckerDocumentation.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/CheckerDocumentation.cpp @@ -33,30 +33,36 @@ namespace ento { /// checking. /// /// \sa CheckerContext -class CheckerDocumentation : public Checker< check::PreStmt, - check::PostStmt, - check::PreObjCMessage, - check::PostObjCMessage, - check::ObjCMessageNil, - check::PreCall, - check::PostCall, - check::BranchCondition, - check::NewAllocator, - check::Location, - check::Bind, - check::DeadSymbols, - check::BeginFunction, - check::EndFunction, - check::EndAnalysis, - check::EndOfTranslationUnit, - eval::Call, - eval::Assume, - check::LiveSymbols, - check::RegionChanges, - check::PointerEscape, - check::ConstPointerEscape, - check::Event, - check::ASTDecl > { +class CheckerDocumentation + : public Checker< + // clang-format off + check::ASTCodeBody, + check::ASTDecl, + check::BeginFunction, + check::Bind, + check::BranchCondition, + check::ConstPointerEscape, + check::DeadSymbols, + check::EndAnalysis, + check::EndFunction, + check::EndOfTranslationUnit, + check::Event, + check::LiveSymbols, + check::Location, + check::NewAllocator, + check::ObjCMessageNil, + check::PointerEscape, + check::PostCall, + check::PostObjCMessage, + check::PostStmt, + check::PreCall, + check::PreObjCMessage, + check::PreStmt, + check::RegionChanges, + eval::Assume, + eval::Call + // clang-format on + > { public: /// Pre-visit the Statement. /// @@ -321,6 +327,13 @@ public: void checkASTDecl(const FunctionDecl *D, AnalysisManager &Mgr, BugReporter &BR) const {} + + /// Check every declaration that has a statement body in the AST. + /// + /// As AST traversal callback, which should only be used when the checker is + /// not path sensitive. It will be called for every Declaration in the AST. + void checkASTCodeBody(const Decl *D, AnalysisManager &Mgr, + BugReporter &BR) const {} }; void CheckerDocumentation::checkPostStmt(const DeclStmt *DS, -- GitLab From be15a6b3b68bed7b9d982f25b2e10a273302002a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingo=20M=C3=BCller?= Date: Wed, 6 Mar 2024 14:14:07 +0100 Subject: [PATCH 285/929] [mlir][opt] Expose MLIR_ENABLE_DEPRECATED_GPU_SER... in mlir-config.h. (#84006) This is another follow-up of #83004, which made the same change for `MLIR_CUDA_CONVERSIONS_ENABLED`. As the previous PR, this PR commit exposes mentioned CMake variable through `mlir-config.h` and uses the macro that is introduced with the same name. This replaces the macro `MLIR_ENABLE_DEPRECATED_GPU_SERIALIZATION`, which the CMake files previously defined manually. --- mlir/include/mlir/Config/mlir-config.h.cmake | 3 +++ mlir/tools/mlir-opt/CMakeLists.txt | 8 -------- mlir/tools/mlir-opt/mlir-opt.cpp | 3 ++- utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 2 ++ 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/mlir/include/mlir/Config/mlir-config.h.cmake b/mlir/include/mlir/Config/mlir-config.h.cmake index a0176063b474..dab83be28ddc 100644 --- a/mlir/include/mlir/Config/mlir-config.h.cmake +++ b/mlir/include/mlir/Config/mlir-config.h.cmake @@ -13,6 +13,9 @@ #ifndef MLIR_CONFIG_H #define MLIR_CONFIG_H +/* If set, enable deprecated serialization passes. */ +#cmakedefine01 MLIR_DEPRECATED_GPU_SERIALIZATION_ENABLE + /* Enable expensive checks to detect invalid pattern API usage. Failed checks manifest as fatal errors or invalid memory accesses (e.g., accessing deallocated memory) that cause a crash. Running with ASAN is recommended for diff --git a/mlir/tools/mlir-opt/CMakeLists.txt b/mlir/tools/mlir-opt/CMakeLists.txt index 701fc461b3b4..4389840af10e 100644 --- a/mlir/tools/mlir-opt/CMakeLists.txt +++ b/mlir/tools/mlir-opt/CMakeLists.txt @@ -106,11 +106,3 @@ llvm_update_compile_flags(mlir-opt) mlir_check_all_link_libraries(mlir-opt) export_executable_symbols_for_plugins(mlir-opt) - -if(MLIR_ENABLE_DEPRECATED_GPU_SERIALIZATION) - # Enable deprecated serialization passes. - target_compile_definitions(mlir-opt - PRIVATE - MLIR_DEPRECATED_GPU_SERIALIZATION_ENABLE=1 - ) -endif() diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp index 0ba1a3a534e3..e4d05631c35f 100644 --- a/mlir/tools/mlir-opt/mlir-opt.cpp +++ b/mlir/tools/mlir-opt/mlir-opt.cpp @@ -10,6 +10,7 @@ // //===----------------------------------------------------------------------===// +#include "mlir/Config/mlir-config.h" #include "mlir/IR/AsmState.h" #include "mlir/IR/Dialect.h" #include "mlir/IR/MLIRContext.h" @@ -277,7 +278,7 @@ void registerTestPasses() { int main(int argc, char **argv) { registerAllPasses(); -#if MLIR_DEPRECATED_GPU_SERIALIZATION_ENABLE == 1 +#if MLIR_DEPRECATED_GPU_SERIALIZATION_ENABLE registerGpuSerializeToCubinPass(); registerGpuSerializeToHsacoPass(); #endif diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index b6d4927388b8..2037e307190f 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -32,6 +32,7 @@ expand_template( name = "mlir_config_h_gen", out = "include/mlir/Config/mlir-config.h", substitutions = { + "#cmakedefine01 MLIR_DEPRECATED_GPU_SERIALIZATION_ENABLE": "#define MLIR_DEPRECATED_GPU_SERIALIZATION_ENABLE 0", "#cmakedefine01 MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS": "#define MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS 0", "#cmakedefine MLIR_GREEDY_REWRITE_RANDOMIZER_SEED ${MLIR_GREEDY_REWRITE_RANDOMIZER_SEED}": "/* #undef MLIR_GREEDY_REWRITE_RANDOMIZER_SEED */", "#cmakedefine01 MLIR_ENABLE_PDL_IN_PATTERNMATCH": "#define MLIR_ENABLE_PDL_IN_PATTERNMATCH 1", @@ -9172,6 +9173,7 @@ cc_binary( ":SCFToGPU", ":Support", ":Transforms", + ":config", "//llvm:AllTargetsCodeGens", "//llvm:Support", "//mlir/test:TestAffine", -- GitLab From 099045a0456a77c38aed504d0b50d44c4d3df6ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingo=20M=C3=BCller?= Date: Wed, 6 Mar 2024 14:14:53 +0100 Subject: [PATCH 286/929] [mlir][nvvm] Expose MLIR_NVPTXCOMPILER_ENABLED in mlir-config.h. (#84007) This is another follow-up of #83004, which made the same change for `MLIR_CUDA_CONVERSIONS_ENABLED`. As the previous PR, this PR commit exposes mentioned CMake variable through `mlir-config.h` and uses the macro that is introduced with the same name. This replaces the macro `MLIR_NVPTXCOMPILER_ENABLED`, which the CMake files previously defined manually. --- mlir/include/mlir/Config/mlir-config.h.cmake | 3 +++ mlir/lib/Target/LLVM/CMakeLists.txt | 1 - mlir/lib/Target/LLVM/NVVM/Target.cpp | 10 +++++----- utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 10 ++++++++-- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/mlir/include/mlir/Config/mlir-config.h.cmake b/mlir/include/mlir/Config/mlir-config.h.cmake index dab83be28ddc..25483467eea2 100644 --- a/mlir/include/mlir/Config/mlir-config.h.cmake +++ b/mlir/include/mlir/Config/mlir-config.h.cmake @@ -36,6 +36,9 @@ and targets. */ #cmakedefine01 MLIR_ENABLE_CUDA_CONVERSIONS +/* If set, enables features that depend on the NVIDIA's PTX compiler. */ +#cmakedefine01 MLIR_ENABLE_NVPTXCOMPILER + /* If set, enables ROCm-related features in ROCM-related transforms, pipelines, and targets. */ #cmakedefine01 MLIR_ENABLE_ROCM_CONVERSIONS diff --git a/mlir/lib/Target/LLVM/CMakeLists.txt b/mlir/lib/Target/LLVM/CMakeLists.txt index cc2c3a00a02e..e0657c895e8a 100644 --- a/mlir/lib/Target/LLVM/CMakeLists.txt +++ b/mlir/lib/Target/LLVM/CMakeLists.txt @@ -93,7 +93,6 @@ if(MLIR_ENABLE_CUDA_CONVERSIONS) # Define the `CUDAToolkit` path. target_compile_definitions(obj.MLIRNVVMTarget PRIVATE - MLIR_NVPTXCOMPILER_ENABLED=${MLIR_ENABLE_NVPTXCOMPILER} __DEFAULT_CUDATOOLKIT_PATH__="${MLIR_CUDAToolkit_ROOT}" ) endif() diff --git a/mlir/lib/Target/LLVM/NVVM/Target.cpp b/mlir/lib/Target/LLVM/NVVM/Target.cpp index e31df99ea20d..e438ce84af1b 100644 --- a/mlir/lib/Target/LLVM/NVVM/Target.cpp +++ b/mlir/lib/Target/LLVM/NVVM/Target.cpp @@ -432,7 +432,7 @@ NVPTXSerializer::compileToBinary(const std::string &ptxCode) { return SmallVector(fatbin.begin(), fatbin.end()); } -#if MLIR_NVPTXCOMPILER_ENABLED == 1 +#if MLIR_ENABLE_NVPTXCOMPILER #include "nvPTXCompiler.h" #define RETURN_ON_NVPTXCOMPILER_ERROR(expr) \ @@ -511,7 +511,7 @@ NVPTXSerializer::compileToBinaryNVPTX(const std::string &ptxCode) { RETURN_ON_NVPTXCOMPILER_ERROR(nvPTXCompilerDestroy(&compiler)); return binary; } -#endif // MLIR_NVPTXCOMPILER_ENABLED == 1 +#endif // MLIR_ENABLE_NVPTXCOMPILER std::optional> NVPTXSerializer::moduleToObject(llvm::Module &llvmModule) { @@ -557,12 +557,12 @@ NVPTXSerializer::moduleToObject(llvm::Module &llvmModule) { return SmallVector(bin.begin(), bin.end()); } - // Compile to binary. -#if MLIR_NVPTXCOMPILER_ENABLED == 1 + // Compile to binary. +#if MLIR_ENABLE_NVPTXCOMPILER return compileToBinaryNVPTX(*serializedISA); #else return compileToBinary(*serializedISA); -#endif // MLIR_NVPTXCOMPILER_ENABLED == 1 +#endif // MLIR_ENABLE_NVPTXCOMPILER } #endif // MLIR_ENABLE_CUDA_CONVERSIONS diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 2037e307190f..04b1a210b2e2 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -38,8 +38,14 @@ expand_template( "#cmakedefine01 MLIR_ENABLE_PDL_IN_PATTERNMATCH": "#define MLIR_ENABLE_PDL_IN_PATTERNMATCH 1", "#cmakedefine01 MLIR_ENABLE_ROCM_CONVERSIONS": "#define MLIR_ENABLE_ROCM_CONVERSIONS 0", } | if_cuda_available( - {"#cmakedefine01 MLIR_ENABLE_CUDA_CONVERSIONS": "#define MLIR_ENABLE_CUDA_CONVERSIONS 1"}, - {"#cmakedefine01 MLIR_ENABLE_CUDA_CONVERSIONS": "#define MLIR_ENABLE_CUDA_CONVERSIONS 0"}, + { + "#cmakedefine01 MLIR_ENABLE_CUDA_CONVERSIONS": "#define MLIR_ENABLE_CUDA_CONVERSIONS 1", + "#cmakedefine01 MLIR_ENABLE_NVPTXCOMPILER": "#define MLIR_ENABLE_NVPTXCOMPILER 1", + }, + { + "#cmakedefine01 MLIR_ENABLE_CUDA_CONVERSIONS": "#define MLIR_ENABLE_CUDA_CONVERSIONS 0", + "#cmakedefine01 MLIR_ENABLE_NVPTXCOMPILER": "#define MLIR_ENABLE_NVPTXCOMPILER 0", + }, ), template = "include/mlir/Config/mlir-config.h.cmake", ) -- GitLab From 8406f8023dc08a1697acd2375e120670c0233a8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingo=20M=C3=BCller?= Date: Wed, 6 Mar 2024 14:15:17 +0100 Subject: [PATCH 287/929] [mlir][cmake] Enable -Wundef. (#84011) This is another follow-up of #83004, which fixed a bug due to some macros not being defined in some situations. By raising warnings on undefined macros, this kind of bug is less likely to be introduced. Similar to #83004, the fix is probably adding an include to `mlir-config.h` (and potentially defining the macro there). --- mlir/CMakeLists.txt | 6 ++++++ mlir/include/mlir/Config/mlir-config.h.cmake | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/mlir/CMakeLists.txt b/mlir/CMakeLists.txt index e37cb465b919..5c4301af040b 100644 --- a/mlir/CMakeLists.txt +++ b/mlir/CMakeLists.txt @@ -70,6 +70,12 @@ endif() check_c_compiler_flag("-Werror=implicit-function-declaration" C_SUPPORTS_WERROR_IMPLICIT_FUNCTION_DECLARATION) append_if(C_SUPPORTS_WERROR_IMPLICIT_FUNCTION_DECLARATION "-Werror=implicit-function-declaration" CMAKE_C_FLAGS) +# Warn on undefined macros. This is often an indication that an include to +# `mlir-config.h` or similar is missing. +check_c_compiler_flag("-Wundef" C_SUPPORTS_WUNDEF) +append_if(C_SUPPORTS_WUNDEF "-Wundef" CMAKE_C_FLAGS) +append_if(C_SUPPORTS_WUNDEF "-Wundef" CMAKE_CXX_FLAGS) + # Forbid mismatch between declaration and definition for class vs struct. This is # harmless on Unix systems, but it'll be a ticking bomb for MSVC/Windows systems # where it creeps into the ABI. diff --git a/mlir/include/mlir/Config/mlir-config.h.cmake b/mlir/include/mlir/Config/mlir-config.h.cmake index 25483467eea2..9339ce07bdfd 100644 --- a/mlir/include/mlir/Config/mlir-config.h.cmake +++ b/mlir/include/mlir/Config/mlir-config.h.cmake @@ -8,6 +8,13 @@ /* This file enumerates variables from the MLIR configuration so that they can be in exported headers and won't override package specific directives. + Defining the variables here is preferable over specifying them in CMake files + via `target_compile_definitions` because it is easier to ensure that they are + defined consistently across all targets: They are guaranteed to be 0/1 + variables thanks to #cmakedefine01, so we can test with `#if` and find + missing definitions or includes with `-Wundef`. With `#ifdef`, these mistakes + can go unnoticed. + This is a C header that can be included in the mlir-c headers. */ #ifndef MLIR_CONFIG_H -- GitLab From 38763be6ab706e5661e94b68c3aa2069f4c736d8 Mon Sep 17 00:00:00 2001 From: jeanPerier Date: Wed, 6 Mar 2024 14:29:27 +0100 Subject: [PATCH 288/929] [flang] Do not traverse selectors in FindImpureCall and HasVectorSubscript (#84041) In presence of symbols with AssocEntityDetails in an expression, `Traverse`, `AnyTraverse`, `AllTraverse`, and `SetTraverse` automatically visit the selector expression or variable. This is most often the desired behavior but can be surprising, and was not correct for FindImpureCall and HasVectorSubscript. Add a default template option to flag the behavior to someone willing to use the Traverse helper for a new utility, and set this template to false for FindImpureCall and HasVectorSubscript. --- flang/include/flang/Evaluate/traverse.h | 24 +++++++++++++++--------- flang/lib/Evaluate/tools.cpp | 11 +++++++---- flang/test/Semantics/forall01.f90 | 11 +++++++++++ flang/test/Semantics/selecttype03.f90 | 2 +- 4 files changed, 34 insertions(+), 14 deletions(-) diff --git a/flang/include/flang/Evaluate/traverse.h b/flang/include/flang/Evaluate/traverse.h index 8d75cc2df724..7f4a67d97e64 100644 --- a/flang/include/flang/Evaluate/traverse.h +++ b/flang/include/flang/Evaluate/traverse.h @@ -45,7 +45,9 @@ #include namespace Fortran::evaluate { -template class Traverse { +template +class Traverse { public: explicit Traverse(Visitor &v) : visitor_{v} {} @@ -108,12 +110,13 @@ public: } Result operator()(const Symbol &symbol) const { const Symbol &ultimate{symbol.GetUltimate()}; - if (const auto *assoc{ - ultimate.detailsIf()}) { - return visitor_(assoc->expr()); - } else { - return visitor_.Default(); + if constexpr (TraverseAssocEntityDetails) { + if (const auto *assoc{ + ultimate.detailsIf()}) { + return visitor_(assoc->expr()); + } } + return visitor_.Default(); } Result operator()(const StaticDataObject &) const { return visitor_.Default(); @@ -284,7 +287,8 @@ private: // For validity checks across an expression: if any operator() result is // false, so is the overall result. template > + bool TraverseAssocEntityDetails = true, + typename Base = Traverse> struct AllTraverse : public Base { explicit AllTraverse(Visitor &v) : Base{v} {} using Base::operator(); @@ -296,7 +300,8 @@ struct AllTraverse : public Base { // is truthful is the final result. Works for Booleans, pointers, // and std::optional<>. template > + bool TraverseAssocEntityDetails = true, + typename Base = Traverse> class AnyTraverse : public Base { public: explicit AnyTraverse(Visitor &v) : Base{v} {} @@ -315,7 +320,8 @@ private: }; template > + bool TraverseAssocEntityDetails = true, + typename Base = Traverse> struct SetTraverse : public Base { explicit SetTraverse(Visitor &v) : Base{v} {} using Base::operator(); diff --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp index e7fc651b9173..f514a25b0102 100644 --- a/flang/lib/Evaluate/tools.cpp +++ b/flang/lib/Evaluate/tools.cpp @@ -995,8 +995,10 @@ template semantics::UnorderedSymbolSet CollectSymbols( const Expr &); // HasVectorSubscript() -struct HasVectorSubscriptHelper : public AnyTraverse { - using Base = AnyTraverse; +struct HasVectorSubscriptHelper + : public AnyTraverse { + using Base = AnyTraverse; HasVectorSubscriptHelper() : Base{*this} {} using Base::operator(); bool operator()(const Subscript &ss) const { @@ -1045,9 +1047,10 @@ parser::Message *AttachDeclaration( } class FindImpureCallHelper - : public AnyTraverse> { + : public AnyTraverse, + /*TraverseAssocEntityDetails=*/false> { using Result = std::optional; - using Base = AnyTraverse; + using Base = AnyTraverse; public: explicit FindImpureCallHelper(FoldingContext &c) : Base{*this}, context_{c} {} diff --git a/flang/test/Semantics/forall01.f90 b/flang/test/Semantics/forall01.f90 index a81eb9621e77..72ad9ecd3947 100644 --- a/flang/test/Semantics/forall01.f90 +++ b/flang/test/Semantics/forall01.f90 @@ -135,3 +135,14 @@ subroutine forall7(x) end forall end select end subroutine + +subroutine forall8(x) + real :: x(10) + real, external :: foo + !ERROR: Impure procedure 'foo' may not be referenced in a FORALL + forall(i=1:10) x(i) = foo() + i + !OK + associate(y => foo()) + forall (i=1:10) x(i) = y + i + end associate +end subroutine diff --git a/flang/test/Semantics/selecttype03.f90 b/flang/test/Semantics/selecttype03.f90 index eb343c4ccc53..c440960f404a 100644 --- a/flang/test/Semantics/selecttype03.f90 +++ b/flang/test/Semantics/selecttype03.f90 @@ -65,7 +65,7 @@ select type (b => array1(V,2)) b%i = 1 !VDC type is (t2) !ERROR: Actual argument associated with INTENT(IN OUT) dummy argument 'z=' is not definable - !BECAUSE: Variable 'b' has a vector subscript + !BECAUSE: Construct association 'b' has a vector subscript call sub_with_in_and_inout_param_vector(b,b) !VDC end select select type(b => foo(1) ) -- GitLab From c4979c935f778eca623afd17dad6f97facba34bf Mon Sep 17 00:00:00 2001 From: Benjamin Maxwell Date: Wed, 6 Mar 2024 13:36:29 +0000 Subject: [PATCH 289/929] [mlir][VectorOps] Add fold vector.shuffle -> vector.interleave (#80968) This folds fixed-size vector.shuffle ops that perform a 1-D interleave to a vector.interleave operation. For example: ```mlir %0 = vector.shuffle %a, %b [0, 2, 1, 4] : vector<2xi32>, vector<2xi32> ``` folds to: ```mlir %0 = vector.interleave %a, %b : vector<2xi32> ``` Depends on: #80967 --- mlir/lib/Dialect/Vector/IR/VectorOps.cpp | 42 +++++++++++++++++++++- mlir/test/Dialect/Vector/canonicalize.mlir | 23 ++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp index 5be6a628904c..75f6220ad8f3 100644 --- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp +++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp @@ -2479,11 +2479,51 @@ public: } }; +/// Pattern to rewrite a fixed-size interleave via vector.shuffle to +/// vector.interleave. +class ShuffleInterleave : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(ShuffleOp op, + PatternRewriter &rewriter) const override { + VectorType resultType = op.getResultVectorType(); + if (resultType.isScalable()) + return rewriter.notifyMatchFailure( + op, "ShuffleOp can't represent a scalable interleave"); + + if (resultType.getRank() != 1) + return rewriter.notifyMatchFailure( + op, "ShuffleOp can't represent an n-D interleave"); + + VectorType sourceType = op.getV1VectorType(); + if (sourceType != op.getV2VectorType() || + sourceType.getNumElements() * 2 != resultType.getNumElements()) { + return rewriter.notifyMatchFailure( + op, "ShuffleOp types don't match an interleave"); + } + + ArrayAttr shuffleMask = op.getMask(); + int64_t resultVectorSize = resultType.getNumElements(); + for (int i = 0, e = resultVectorSize / 2; i < e; ++i) { + int64_t maskValueA = cast(shuffleMask[i * 2]).getInt(); + int64_t maskValueB = cast(shuffleMask[(i * 2) + 1]).getInt(); + if (maskValueA != i || maskValueB != (resultVectorSize / 2) + i) + return rewriter.notifyMatchFailure(op, + "ShuffleOp mask not interleaving"); + } + + rewriter.replaceOpWithNewOp(op, op.getV1(), op.getV2()); + return success(); + } +}; + } // namespace void ShuffleOp::getCanonicalizationPatterns(RewritePatternSet &results, MLIRContext *context) { - results.add(context); + results.add( + context); } //===----------------------------------------------------------------------===// diff --git a/mlir/test/Dialect/Vector/canonicalize.mlir b/mlir/test/Dialect/Vector/canonicalize.mlir index e6f045e12e51..4c73a6271786 100644 --- a/mlir/test/Dialect/Vector/canonicalize.mlir +++ b/mlir/test/Dialect/Vector/canonicalize.mlir @@ -2567,3 +2567,26 @@ func.func @load_store_forwarding_rank_mismatch(%v0: vector<4x1x1xf32>, %arg0: te tensor<4x4x4xf32>, vector<1x100x4x5xf32> return %r : vector<1x100x4x5xf32> } + +// ----- + +// CHECK-LABEL: func.func @rank_0_shuffle_to_interleave( +// CHECK-SAME: %[[LHS:.*]]: vector, %[[RHS:.*]]: vector) +func.func @rank_0_shuffle_to_interleave(%arg0: vector, %arg1: vector) -> vector<2xf64> +{ + // CHECK: %[[ZIP:.*]] = vector.interleave %[[LHS]], %[[RHS]] : vector + // CHECK: return %[[ZIP]] + %0 = vector.shuffle %arg0, %arg1 [0, 1] : vector, vector + return %0 : vector<2xf64> +} + +// ----- + +// CHECK-LABEL: func.func @rank_1_shuffle_to_interleave( +// CHECK-SAME: %[[LHS:.*]]: vector<6xi32>, %[[RHS:.*]]: vector<6xi32>) +func.func @rank_1_shuffle_to_interleave(%arg0: vector<6xi32>, %arg1: vector<6xi32>) -> vector<12xi32> { + // CHECK: %[[ZIP:.*]] = vector.interleave %[[LHS]], %[[RHS]] : vector<6xi32> + // CHECK: return %[[ZIP]] + %0 = vector.shuffle %arg0, %arg1 [0, 6, 1, 7, 2, 8, 3, 9, 4, 10, 5, 11] : vector<6xi32>, vector<6xi32> + return %0 : vector<12xi32> +} -- GitLab From e9c1dbb408c51115289a135e721de0bf5efec0c1 Mon Sep 17 00:00:00 2001 From: Shilei Tian Date: Wed, 6 Mar 2024 08:42:54 -0500 Subject: [PATCH 290/929] Revert "[AMDGPU] Replace `isInlinableLiteral16` with specific version (#81345)" This reverts commit 530f0e64ec11327879c44f2fd55c7c28efdbaa2d because it breaks downstream. --- .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp | 54 ++----- .../AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp | 18 ++- .../AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h | 4 +- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 28 +--- llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 25 +-- llvm/lib/Target/AMDGPU/SIInstrInfo.h | 4 +- .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 10 +- llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 8 +- llvm/test/CodeGen/AMDGPU/immv216.ll | 42 ++--- .../test/CodeGen/AMDGPU/inline-constraints.ll | 12 ++ .../CodeGen/AMDGPU/shrink-add-sub-constant.ll | 144 ++++++------------ 11 files changed, 119 insertions(+), 230 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp index 10999d846e3b..cb4eddfe5320 100644 --- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp +++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp @@ -2006,12 +2006,8 @@ static bool isInlineableLiteralOp16(int64_t Val, MVT VT, bool HasInv2Pi) { return isInlinableIntLiteral(Val); } - if (VT.getScalarType() == MVT::f16) - return AMDGPU::isInlinableLiteralFP16(Val, HasInv2Pi); - - assert(VT.getScalarType() == MVT::bf16); - - return AMDGPU::isInlinableLiteralBF16(Val, HasInv2Pi); + // f16/v2f16 operands work correctly for all values. + return AMDGPU::isInlinableLiteral16(Val, HasInv2Pi); } bool AMDGPUOperand::isInlinableImm(MVT type) const { @@ -2379,26 +2375,15 @@ void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyMo return; case AMDGPU::OPERAND_REG_IMM_INT16: - case AMDGPU::OPERAND_REG_INLINE_C_INT16: - case AMDGPU::OPERAND_REG_INLINE_AC_INT16: - if (isSafeTruncation(Val, 16) && - AMDGPU::isInlinableIntLiteral(static_cast(Val))) { - Inst.addOperand(MCOperand::createImm(Val)); - setImmKindConst(); - return; - } - - Inst.addOperand(MCOperand::createImm(Val & 0xffff)); - setImmKindLiteral(); - return; - - case AMDGPU::OPERAND_REG_INLINE_C_FP16: case AMDGPU::OPERAND_REG_IMM_FP16: case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED: + case AMDGPU::OPERAND_REG_INLINE_C_INT16: + case AMDGPU::OPERAND_REG_INLINE_C_FP16: + case AMDGPU::OPERAND_REG_INLINE_AC_INT16: case AMDGPU::OPERAND_REG_INLINE_AC_FP16: if (isSafeTruncation(Val, 16) && - AMDGPU::isInlinableLiteralFP16(static_cast(Val), - AsmParser->hasInv2PiInlineImm())) { + AMDGPU::isInlinableLiteral16(static_cast(Val), + AsmParser->hasInv2PiInlineImm())) { Inst.addOperand(MCOperand::createImm(Val)); setImmKindConst(); return; @@ -2425,17 +2410,12 @@ void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyMo return; case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: - case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16: { - assert(isSafeTruncation(Val, 16)); - assert(AMDGPU::isInlinableIntLiteral(static_cast(Val))); - Inst.addOperand(MCOperand::createImm(Val)); - return; - } case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: + case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16: case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: { assert(isSafeTruncation(Val, 16)); - assert(AMDGPU::isInlinableLiteralFP16(static_cast(Val), - AsmParser->hasInv2PiInlineImm())); + assert(AMDGPU::isInlinableLiteral16(static_cast(Val), + AsmParser->hasInv2PiInlineImm())); Inst.addOperand(MCOperand::createImm(Val)); return; @@ -3579,19 +3559,7 @@ bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst, OperandType == AMDGPU::OPERAND_REG_IMM_V2BF16) return AMDGPU::isInlinableLiteralV2BF16(Val); - if (OperandType == AMDGPU::OPERAND_REG_IMM_FP16 || - OperandType == AMDGPU::OPERAND_REG_INLINE_C_FP16 || - OperandType == AMDGPU::OPERAND_REG_INLINE_AC_FP16 || - OperandType == AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED) - return AMDGPU::isInlinableLiteralFP16(Val, hasInv2PiInlineImm()); - - if (OperandType == AMDGPU::OPERAND_REG_IMM_BF16 || - OperandType == AMDGPU::OPERAND_REG_INLINE_C_BF16 || - OperandType == AMDGPU::OPERAND_REG_INLINE_AC_BF16 || - OperandType == AMDGPU::OPERAND_REG_IMM_BF16_DEFERRED) - return AMDGPU::isInlinableLiteralBF16(Val, hasInv2PiInlineImm()); - - llvm_unreachable("invalid operand type"); + return AMDGPU::isInlinableLiteral16(Val, hasInv2PiInlineImm()); } default: llvm_unreachable("invalid operand size"); diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp index 683e8dad796c..a32be1e50a60 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp @@ -460,8 +460,10 @@ void AMDGPUInstPrinter::printImmediateInt16(uint32_t Imm, } } -static bool printImmediateFP16(uint32_t Imm, const MCSubtargetInfo &STI, - raw_ostream &O) { +// This must accept a 32-bit immediate value to correctly handle packed 16-bit +// operations. +static bool printImmediateFloat16(uint32_t Imm, const MCSubtargetInfo &STI, + raw_ostream &O) { if (Imm == 0x3C00) O << "1.0"; else if (Imm == 0xBC00) @@ -527,9 +529,9 @@ void AMDGPUInstPrinter::printImmediateBF16(uint32_t Imm, O << formatHex(static_cast(Imm)); } -void AMDGPUInstPrinter::printImmediateF16(uint32_t Imm, - const MCSubtargetInfo &STI, - raw_ostream &O) { +void AMDGPUInstPrinter::printImmediate16(uint32_t Imm, + const MCSubtargetInfo &STI, + raw_ostream &O) { int16_t SImm = static_cast(Imm); if (isInlinableIntLiteral(SImm)) { O << SImm; @@ -537,7 +539,7 @@ void AMDGPUInstPrinter::printImmediateF16(uint32_t Imm, } uint16_t HImm = static_cast(Imm); - if (printImmediateFP16(HImm, STI, O)) + if (printImmediateFloat16(HImm, STI, O)) return; uint64_t Imm16 = static_cast(Imm); @@ -564,7 +566,7 @@ void AMDGPUInstPrinter::printImmediateV216(uint32_t Imm, uint8_t OpType, case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: if (isUInt<16>(Imm) && - printImmediateFP16(static_cast(Imm), STI, O)) + printImmediateFloat16(static_cast(Imm), STI, O)) return; break; case AMDGPU::OPERAND_REG_IMM_V2BF16: @@ -843,7 +845,7 @@ void AMDGPUInstPrinter::printRegularOperand(const MCInst *MI, unsigned OpNo, case AMDGPU::OPERAND_REG_INLINE_AC_FP16: case AMDGPU::OPERAND_REG_IMM_FP16: case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED: - printImmediateF16(Op.getImm(), STI, O); + printImmediate16(Op.getImm(), STI, O); break; case AMDGPU::OPERAND_REG_INLINE_C_BF16: case AMDGPU::OPERAND_REG_INLINE_AC_BF16: diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h index c801eaf1111e..15ecbf2e5e59 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h @@ -86,10 +86,10 @@ private: raw_ostream &O); void printImmediateInt16(uint32_t Imm, const MCSubtargetInfo &STI, raw_ostream &O); + void printImmediate16(uint32_t Imm, const MCSubtargetInfo &STI, + raw_ostream &O); void printImmediateBF16(uint32_t Imm, const MCSubtargetInfo &STI, raw_ostream &O); - void printImmediateF16(uint32_t Imm, const MCSubtargetInfo &STI, - raw_ostream &O); void printImmediateV216(uint32_t Imm, uint8_t OpType, const MCSubtargetInfo &STI, raw_ostream &O); bool printImmediateFloat32(uint32_t Imm, const MCSubtargetInfo &STI, diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 23fe0a52a74b..5f6ce12ae1fb 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -15423,32 +15423,16 @@ bool SITargetLowering::checkAsmConstraintVal(SDValue Op, StringRef Constraint, llvm_unreachable("Invalid asm constraint"); } -bool SITargetLowering::checkAsmConstraintValA(SDValue Op, uint64_t Val, +bool SITargetLowering::checkAsmConstraintValA(SDValue Op, + uint64_t Val, unsigned MaxSize) const { unsigned Size = std::min(Op.getScalarValueSizeInBits(), MaxSize); bool HasInv2Pi = Subtarget->hasInv2PiInlineImm(); - if (Size == 16) { - MVT VT = Op.getSimpleValueType(); - switch (VT.SimpleTy) { - default: - return false; - case MVT::i16: - return AMDGPU::isInlinableLiteralI16(Val, HasInv2Pi); - case MVT::f16: - return AMDGPU::isInlinableLiteralFP16(Val, HasInv2Pi); - case MVT::bf16: - return AMDGPU::isInlinableLiteralBF16(Val, HasInv2Pi); - case MVT::v2i16: - return AMDGPU::getInlineEncodingV2I16(Val).has_value(); - case MVT::v2f16: - return AMDGPU::getInlineEncodingV2F16(Val).has_value(); - case MVT::v2bf16: - return AMDGPU::getInlineEncodingV2BF16(Val).has_value(); - } - } - if ((Size == 32 && AMDGPU::isInlinableLiteral32(Val, HasInv2Pi)) || - (Size == 64 && AMDGPU::isInlinableLiteral64(Val, HasInv2Pi))) + if ((Size == 16 && AMDGPU::isInlinableLiteral16(Val, HasInv2Pi)) || + (Size == 32 && AMDGPU::isInlinableLiteral32(Val, HasInv2Pi)) || + (Size == 64 && AMDGPU::isInlinableLiteral64(Val, HasInv2Pi))) { return true; + } return false; } diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index cb657c5e04e4..a5b6c582a78c 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -4121,32 +4121,13 @@ bool SIInstrInfo::isInlineConstant(const APInt &Imm) const { ST.hasInv2PiInlineImm()); case 16: return ST.has16BitInsts() && - AMDGPU::isInlinableLiteralI16(Imm.getSExtValue(), - ST.hasInv2PiInlineImm()); + AMDGPU::isInlinableLiteral16(Imm.getSExtValue(), + ST.hasInv2PiInlineImm()); default: llvm_unreachable("invalid bitwidth"); } } -bool SIInstrInfo::isInlineConstant(const APFloat &Imm) const { - APInt IntImm = Imm.bitcastToAPInt(); - int64_t IntImmVal = IntImm.getSExtValue(); - bool HasInv2Pi = ST.hasInv2PiInlineImm(); - switch (APFloat::SemanticsToEnum(Imm.getSemantics())) { - default: - llvm_unreachable("invalid fltSemantics"); - case APFloatBase::S_IEEEsingle: - case APFloatBase::S_IEEEdouble: - return isInlineConstant(IntImm); - case APFloatBase::S_BFloat: - return ST.has16BitInsts() && - AMDGPU::isInlinableLiteralBF16(IntImmVal, HasInv2Pi); - case APFloatBase::S_IEEEhalf: - return ST.has16BitInsts() && - AMDGPU::isInlinableLiteralFP16(IntImmVal, HasInv2Pi); - } -} - bool SIInstrInfo::isInlineConstant(const MachineOperand &MO, uint8_t OperandType) const { assert(!MO.isReg() && "isInlineConstant called on register operand!"); @@ -4219,7 +4200,7 @@ bool SIInstrInfo::isInlineConstant(const MachineOperand &MO, // constants in these cases int16_t Trunc = static_cast(Imm); return ST.has16BitInsts() && - AMDGPU::isInlinableLiteralFP16(Trunc, ST.hasInv2PiInlineImm()); + AMDGPU::isInlinableLiteral16(Trunc, ST.hasInv2PiInlineImm()); } return false; diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h index dab2cb2946ac..82c6117292ae 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h @@ -984,7 +984,9 @@ public: bool isInlineConstant(const APInt &Imm) const; - bool isInlineConstant(const APFloat &Imm) const; + bool isInlineConstant(const APFloat &Imm) const { + return isInlineConstant(Imm.bitcastToAPInt()); + } // Returns true if this non-register operand definitely does not need to be // encoded as a 32-bit literal. Note that this function handles all kinds of diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index 63285c06edaf..963dc2882fcc 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -2647,19 +2647,13 @@ bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi) { Val == 0x3E22; // 1.0 / (2.0 * pi) } -bool isInlinableLiteralI16(int16_t Literal, bool HasInv2Pi) { +bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi) { if (!HasInv2Pi) return false; - if (isInlinableIntLiteral(Literal)) - return true; - return Literal == static_cast(0x3e22f983); -} -bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi) { - if (!HasInv2Pi) - return false; if (isInlinableIntLiteral(Literal)) return true; + uint16_t Val = static_cast(Literal); return Val == 0x3C00 || // 1.0 Val == 0xBC00 || // -1.0 diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h index 9fcb4caca30b..6edf01d1217f 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h @@ -1397,13 +1397,7 @@ LLVM_READNONE bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi); LLVM_READNONE -bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi); - -LLVM_READNONE -bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi); - -LLVM_READNONE -bool isInlinableLiteralI16(int16_t Literal, bool HasInv2Pi); +bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi); LLVM_READNONE std::optional getInlineEncodingV2I16(uint32_t Literal); diff --git a/llvm/test/CodeGen/AMDGPU/immv216.ll b/llvm/test/CodeGen/AMDGPU/immv216.ll index ae51c3edf1c7..b66ca71a3274 100644 --- a/llvm/test/CodeGen/AMDGPU/immv216.ll +++ b/llvm/test/CodeGen/AMDGPU/immv216.ll @@ -577,40 +577,40 @@ define amdgpu_kernel void @add_inline_imm_64_v2f16(ptr addrspace(1) %out, <2 x h } ; GCN-LABEL: {{^}}mul_inline_imm_0.5_v2i16: -; GFX9: s_movk_i32 [[K:s[0-9]+]], 0x3800 -; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] op_sel_hi:[1,0] +; GFX9: s_mov_b32 [[K:s[0-9]+]], 0x38003800 +; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] -; GFX10: v_pk_mul_lo_u16 v0, 0x3800, v0 op_sel_hi:[0,1] ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0x38,0x00,0x00] +; GFX10: v_pk_mul_lo_u16 v0, 0x38003800, v0 ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0x38,0x00,0x38] define <2 x i16> @mul_inline_imm_0.5_v2i16(<2 x i16> %x) { %y = mul <2 x i16> %x, bitcast (<2 x half> to <2 x i16>) ret <2 x i16> %y } ; GCN-LABEL: {{^}}mul_inline_imm_neg_0.5_v2i16: -; GFX9: s_movk_i32 [[K:s[0-9]+]], 0xb800 -; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] op_sel_hi:[1,0] +; GFX9: s_mov_b32 [[K:s[0-9]+]], 0xb800b800 +; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] -; GFX10: v_pk_mul_lo_u16 v0, 0xffffb800, v0 op_sel_hi:[0,1] ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0xb8,0xff,0xff] +; GFX10: v_pk_mul_lo_u16 v0, 0xb800b800, v0 ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0xb8,0x00,0xb8] define <2 x i16> @mul_inline_imm_neg_0.5_v2i16(<2 x i16> %x) { %y = mul <2 x i16> %x, bitcast (<2 x half> to <2 x i16>) ret <2 x i16> %y } ; GCN-LABEL: {{^}}mul_inline_imm_1.0_v2i16: -; GFX9: s_movk_i32 [[K:s[0-9]+]], 0x3c00 -; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] op_sel_hi:[1,0] +; GFX9: s_mov_b32 [[K:s[0-9]+]], 0x3c003c00 +; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] -; GFX10: v_pk_mul_lo_u16 v0, 0x3c00, v0 op_sel_hi:[0,1] ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0x3c,0x00,0x00] +; GFX10: v_pk_mul_lo_u16 v0, 0x3c003c00, v0 ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0x3c,0x00,0x3c] define <2 x i16> @mul_inline_imm_1.0_v2i16(<2 x i16> %x) { %y = mul <2 x i16> %x, bitcast (<2 x half> to <2 x i16>) ret <2 x i16> %y } ; GCN-LABEL: {{^}}mul_inline_imm_neg_1.0_v2i16: -; GFX9: s_movk_i32 [[K:s[0-9]+]], 0xbc00 -; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] op_sel_hi:[1,0] +; GFX9: s_mov_b32 [[K:s[0-9]+]], 0xbc00bc00 +; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] -; GFX10: v_pk_mul_lo_u16 v0, 0xffffbc00, v0 op_sel_hi:[0,1] ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0xbc,0xff,0xff] +; GFX10: v_pk_mul_lo_u16 v0, 0xbc00bc00, v0 ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0xbc,0x00,0xbc] define <2 x i16> @mul_inline_imm_neg_1.0_v2i16(<2 x i16> %x) { %y = mul <2 x i16> %x, bitcast (<2 x half> to <2 x i16>) ret <2 x i16> %y @@ -635,10 +635,10 @@ define <2 x i16> @shl_inline_imm_neg_2.0_v2i16(<2 x i16> %x) { } ; GCN-LABEL: {{^}}mul_inline_imm_4.0_v2i16: -; GFX9: s_movk_i32 [[K:s[0-9]+]], 0x4400 -; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] op_sel_hi:[1,0] +; GFX9: s_mov_b32 [[K:s[0-9]+]], 0x44004400 +; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] -; GFX10: v_pk_mul_lo_u16 v0, 0x4400, v0 op_sel_hi:[0,1] ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0x44,0x00,0x00] +; GFX10: v_pk_mul_lo_u16 v0, 0x44004400, v0 ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0x44,0x00,0x44] define <2 x i16> @mul_inline_imm_4.0_v2i16(<2 x i16> %x) { %y = mul <2 x i16> %x, bitcast (<2 x half> to <2 x i16>) ret <2 x i16> %y @@ -646,20 +646,20 @@ define <2 x i16> @mul_inline_imm_4.0_v2i16(<2 x i16> %x) { } ; GCN-LABEL: {{^}}mul_inline_imm_neg_4.0_v2i16: -; GFX9: s_movk_i32 [[K:s[0-9]+]], 0xc400 -; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] op_sel_hi:[1,0] +; GFX9: s_mov_b32 [[K:s[0-9]+]], 0xc400c400 +; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] -; GFX10: v_pk_mul_lo_u16 v0, 0xffffc400, v0 op_sel_hi:[0,1] ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0xc4,0xff,0xff] +; GFX10: v_pk_mul_lo_u16 v0, 0xc400c400, v0 ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0xc4,0x00,0xc4] define <2 x i16> @mul_inline_imm_neg_4.0_v2i16(<2 x i16> %x) { %y = mul <2 x i16> %x, bitcast (<2 x half> to <2 x i16>) ret <2 x i16> %y } ; GCN-LABEL: {{^}}mul_inline_imm_inv2pi_v2i16: -; GFX9: s_movk_i32 [[K:s[0-9]+]], 0x3118 -; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] op_sel_hi:[1,0] +; GFX9: s_mov_b32 [[K:s[0-9]+]], 0x31183118 +; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] -; GFX10: v_pk_mul_lo_u16 v0, 0x3118, v0 op_sel_hi:[0,1] ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x18,0x31,0x00,0x00] +; GFX10: v_pk_mul_lo_u16 v0, 0x31183118, v0 ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x18,0x31,0x18,0x31] define <2 x i16> @mul_inline_imm_inv2pi_v2i16(<2 x i16> %x) { %y = mul <2 x i16> %x, bitcast (<2 x half> to <2 x i16>) ret <2 x i16> %y diff --git a/llvm/test/CodeGen/AMDGPU/inline-constraints.ll b/llvm/test/CodeGen/AMDGPU/inline-constraints.ll index 7bd6b037386b..9ef246fe2e10 100644 --- a/llvm/test/CodeGen/AMDGPU/inline-constraints.ll +++ b/llvm/test/CodeGen/AMDGPU/inline-constraints.ll @@ -97,6 +97,7 @@ define i32 @inline_A_constraint_H1() { ; NOSI: error: invalid operand for inline asm constraint 'A' ; VI-LABEL: {{^}}inline_A_constraint_H2: +; VI: v_mov_b32 {{v[0-9]+}}, 0x3c00 define i32 @inline_A_constraint_H2() { %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,A"(i16 bitcast (half 1.0 to i16)) ret i32 %v0 @@ -104,6 +105,7 @@ define i32 @inline_A_constraint_H2() { ; NOSI: error: invalid operand for inline asm constraint 'A' ; VI-LABEL: {{^}}inline_A_constraint_H3: +; VI: v_mov_b32 {{v[0-9]+}}, 0xbc00 define i32 @inline_A_constraint_H3() { %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,A"(i16 bitcast (half -1.0 to i16)) ret i32 %v0 @@ -111,6 +113,7 @@ define i32 @inline_A_constraint_H3() { ; NOSI: error: invalid operand for inline asm constraint 'A' ; VI-LABEL: {{^}}inline_A_constraint_H4: +; VI: v_mov_b32 {{v[0-9]+}}, 0x3118 define i32 @inline_A_constraint_H4() { %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,A"(half 0xH3118) ret i32 %v0 @@ -118,6 +121,7 @@ define i32 @inline_A_constraint_H4() { ; NOSI: error: invalid operand for inline asm constraint 'A' ; VI-LABEL: {{^}}inline_A_constraint_H5: +; VI: v_mov_b32 {{v[0-9]+}}, 0x3118 define i32 @inline_A_constraint_H5() { %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,A"(i16 bitcast (half 0xH3118 to i16)) ret i32 %v0 @@ -125,6 +129,7 @@ define i32 @inline_A_constraint_H5() { ; NOSI: error: invalid operand for inline asm constraint 'A' ; VI-LABEL: {{^}}inline_A_constraint_H6: +; VI: v_mov_b32 {{v[0-9]+}}, 0xb800 define i32 @inline_A_constraint_H6() { %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,A"(half -0.5) ret i32 %v0 @@ -288,6 +293,7 @@ define i32 @inline_A_constraint_V0() { ; NOSI: error: invalid operand for inline asm constraint 'A' ; VI-LABEL: {{^}}inline_A_constraint_V1: +; VI: v_mov_b32 {{v[0-9]+}}, 0xb800 define i32 @inline_A_constraint_V1() { %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,A"(<2 x half> ) ret i32 %v0 @@ -964,6 +970,7 @@ define i32 @inline_DA_constraint_H1() { ; NOSI: error: invalid operand for inline asm constraint 'DA' ; VI-LABEL: {{^}}inline_DA_constraint_H2: +; VI: v_mov_b32 {{v[0-9]+}}, 0x3c00 define i32 @inline_DA_constraint_H2() { %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i16 bitcast (half 1.0 to i16)) ret i32 %v0 @@ -971,6 +978,7 @@ define i32 @inline_DA_constraint_H2() { ; NOSI: error: invalid operand for inline asm constraint 'DA' ; VI-LABEL: {{^}}inline_DA_constraint_H3: +; VI: v_mov_b32 {{v[0-9]+}}, 0xbc00 define i32 @inline_DA_constraint_H3() { %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i16 bitcast (half -1.0 to i16)) ret i32 %v0 @@ -978,6 +986,7 @@ define i32 @inline_DA_constraint_H3() { ; NOSI: error: invalid operand for inline asm constraint 'DA' ; VI-LABEL: {{^}}inline_DA_constraint_H4: +; VI: v_mov_b32 {{v[0-9]+}}, 0x3118 define i32 @inline_DA_constraint_H4() { %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(half 0xH3118) ret i32 %v0 @@ -985,6 +994,7 @@ define i32 @inline_DA_constraint_H4() { ; NOSI: error: invalid operand for inline asm constraint 'DA' ; VI-LABEL: {{^}}inline_DA_constraint_H5: +; VI: v_mov_b32 {{v[0-9]+}}, 0x3118 define i32 @inline_DA_constraint_H5() { %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i16 bitcast (half 0xH3118 to i16)) ret i32 %v0 @@ -992,6 +1002,7 @@ define i32 @inline_DA_constraint_H5() { ; NOSI: error: invalid operand for inline asm constraint 'DA' ; VI-LABEL: {{^}}inline_DA_constraint_H6: +; VI: v_mov_b32 {{v[0-9]+}}, 0xb800 define i32 @inline_DA_constraint_H6() { %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(half -0.5) ret i32 %v0 @@ -1153,6 +1164,7 @@ define i32 @inline_DA_constraint_V0() { ; NOSI: error: invalid operand for inline asm constraint 'DA' ; VI-LABEL: {{^}}inline_DA_constraint_V1: +; VI: v_mov_b32 {{v[0-9]+}}, 0xb800 define i32 @inline_DA_constraint_V1() { %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(<2 x half> ) ret i32 %v0 diff --git a/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll b/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll index 1a55bf608ebf..5de9b0b92c9a 100644 --- a/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll +++ b/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll @@ -3400,9 +3400,9 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_fpone(ptr addrspace(1) %out, p ; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0 ; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-SDAG-NEXT: global_load_dword v1, v0, s[2:3] -; GFX9-SDAG-NEXT: s_movk_i32 s2, 0xc400 +; GFX9-SDAG-NEXT: s_mov_b32 s2, 0xc400c400 ; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) -; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v1, s2 op_sel_hi:[1,0] +; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v1, s2 ; GFX9-SDAG-NEXT: global_store_dword v0, v1, s[0:1] ; GFX9-SDAG-NEXT: s_endpgm ; @@ -3418,53 +3418,29 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_fpone(ptr addrspace(1) %out, p ; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[0:1] ; GFX9-GISEL-NEXT: s_endpgm ; -; GFX10-SDAG-LABEL: v_test_v2i16_x_add_neg_fpone: -; GFX10-SDAG: ; %bb.0: -; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 -; GFX10-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0 -; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-SDAG-NEXT: global_load_dword v1, v0, s[2:3] -; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) -; GFX10-SDAG-NEXT: v_pk_add_u16 v1, 0xffffc400, v1 op_sel_hi:[0,1] -; GFX10-SDAG-NEXT: global_store_dword v0, v1, s[0:1] -; GFX10-SDAG-NEXT: s_endpgm -; -; GFX10-GISEL-LABEL: v_test_v2i16_x_add_neg_fpone: -; GFX10-GISEL: ; %bb.0: -; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 -; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0 -; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-GISEL-NEXT: global_load_dword v1, v0, s[2:3] -; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) -; GFX10-GISEL-NEXT: v_pk_add_u16 v1, 0xc400c400, v1 -; GFX10-GISEL-NEXT: global_store_dword v0, v1, s[0:1] -; GFX10-GISEL-NEXT: s_endpgm -; -; GFX11-SDAG-LABEL: v_test_v2i16_x_add_neg_fpone: -; GFX11-SDAG: ; %bb.0: -; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[0:1], 0x24 -; GFX11-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0 -; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-SDAG-NEXT: global_load_b32 v1, v0, s[2:3] -; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) -; GFX11-SDAG-NEXT: v_pk_add_u16 v1, 0xffffc400, v1 op_sel_hi:[0,1] -; GFX11-SDAG-NEXT: global_store_b32 v0, v1, s[0:1] -; GFX11-SDAG-NEXT: s_nop 0 -; GFX11-SDAG-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) -; GFX11-SDAG-NEXT: s_endpgm +; GFX10-LABEL: v_test_v2i16_x_add_neg_fpone: +; GFX10: ; %bb.0: +; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: global_load_dword v1, v0, s[2:3] +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: v_pk_add_u16 v1, 0xc400c400, v1 +; GFX10-NEXT: global_store_dword v0, v1, s[0:1] +; GFX10-NEXT: s_endpgm ; -; GFX11-GISEL-LABEL: v_test_v2i16_x_add_neg_fpone: -; GFX11-GISEL: ; %bb.0: -; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[0:1], 0x24 -; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0 -; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-GISEL-NEXT: global_load_b32 v1, v0, s[2:3] -; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) -; GFX11-GISEL-NEXT: v_pk_add_u16 v1, 0xc400c400, v1 -; GFX11-GISEL-NEXT: global_store_b32 v0, v1, s[0:1] -; GFX11-GISEL-NEXT: s_nop 0 -; GFX11-GISEL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) -; GFX11-GISEL-NEXT: s_endpgm +; GFX11-LABEL: v_test_v2i16_x_add_neg_fpone: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24 +; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: global_load_b32 v1, v0, s[2:3] +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_pk_add_u16 v1, 0xc400c400, v1 +; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] +; GFX11-NEXT: s_nop 0 +; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX11-NEXT: s_endpgm %tid = call i32 @llvm.amdgcn.workitem.id.x() %tid.ext = sext i32 %tid to i64 %gep = getelementptr inbounds <2 x i16>, ptr addrspace(1) %in, i64 %tid.ext @@ -3565,9 +3541,9 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_negfpone(ptr addrspace(1) %out ; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0 ; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-SDAG-NEXT: global_load_dword v1, v0, s[2:3] -; GFX9-SDAG-NEXT: s_movk_i32 s2, 0x4400 +; GFX9-SDAG-NEXT: s_mov_b32 s2, 0x44004400 ; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) -; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v1, s2 op_sel_hi:[1,0] +; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v1, s2 ; GFX9-SDAG-NEXT: global_store_dword v0, v1, s[0:1] ; GFX9-SDAG-NEXT: s_endpgm ; @@ -3583,53 +3559,29 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_negfpone(ptr addrspace(1) %out ; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[0:1] ; GFX9-GISEL-NEXT: s_endpgm ; -; GFX10-SDAG-LABEL: v_test_v2i16_x_add_neg_negfpone: -; GFX10-SDAG: ; %bb.0: -; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 -; GFX10-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0 -; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-SDAG-NEXT: global_load_dword v1, v0, s[2:3] -; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) -; GFX10-SDAG-NEXT: v_pk_add_u16 v1, 0x4400, v1 op_sel_hi:[0,1] -; GFX10-SDAG-NEXT: global_store_dword v0, v1, s[0:1] -; GFX10-SDAG-NEXT: s_endpgm -; -; GFX10-GISEL-LABEL: v_test_v2i16_x_add_neg_negfpone: -; GFX10-GISEL: ; %bb.0: -; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 -; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0 -; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-GISEL-NEXT: global_load_dword v1, v0, s[2:3] -; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) -; GFX10-GISEL-NEXT: v_pk_add_u16 v1, 0x44004400, v1 -; GFX10-GISEL-NEXT: global_store_dword v0, v1, s[0:1] -; GFX10-GISEL-NEXT: s_endpgm -; -; GFX11-SDAG-LABEL: v_test_v2i16_x_add_neg_negfpone: -; GFX11-SDAG: ; %bb.0: -; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[0:1], 0x24 -; GFX11-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0 -; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-SDAG-NEXT: global_load_b32 v1, v0, s[2:3] -; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) -; GFX11-SDAG-NEXT: v_pk_add_u16 v1, 0x4400, v1 op_sel_hi:[0,1] -; GFX11-SDAG-NEXT: global_store_b32 v0, v1, s[0:1] -; GFX11-SDAG-NEXT: s_nop 0 -; GFX11-SDAG-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) -; GFX11-SDAG-NEXT: s_endpgm +; GFX10-LABEL: v_test_v2i16_x_add_neg_negfpone: +; GFX10: ; %bb.0: +; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: global_load_dword v1, v0, s[2:3] +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: v_pk_add_u16 v1, 0x44004400, v1 +; GFX10-NEXT: global_store_dword v0, v1, s[0:1] +; GFX10-NEXT: s_endpgm ; -; GFX11-GISEL-LABEL: v_test_v2i16_x_add_neg_negfpone: -; GFX11-GISEL: ; %bb.0: -; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[0:1], 0x24 -; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0 -; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-GISEL-NEXT: global_load_b32 v1, v0, s[2:3] -; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) -; GFX11-GISEL-NEXT: v_pk_add_u16 v1, 0x44004400, v1 -; GFX11-GISEL-NEXT: global_store_b32 v0, v1, s[0:1] -; GFX11-GISEL-NEXT: s_nop 0 -; GFX11-GISEL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) -; GFX11-GISEL-NEXT: s_endpgm +; GFX11-LABEL: v_test_v2i16_x_add_neg_negfpone: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24 +; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: global_load_b32 v1, v0, s[2:3] +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_pk_add_u16 v1, 0x44004400, v1 +; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] +; GFX11-NEXT: s_nop 0 +; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX11-NEXT: s_endpgm %tid = call i32 @llvm.amdgcn.workitem.id.x() %tid.ext = sext i32 %tid to i64 %gep = getelementptr inbounds <2 x i16>, ptr addrspace(1) %in, i64 %tid.ext -- GitLab From 33e312137b065ba330b187f56ddd60df70927241 Mon Sep 17 00:00:00 2001 From: Corentin Jabot Date: Wed, 6 Mar 2024 14:47:30 +0100 Subject: [PATCH 291/929] [Clang] Mark papers & core issues implemented in clang 18 as released --- clang/www/cxx_dr_status.html | 14 +++++++------- clang/www/cxx_status.html | 10 +++++----- clang/www/make_cxx_dr_status | 3 +-- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html index b13401625a6f..0b810b50c529 100755 --- a/clang/www/cxx_dr_status.html +++ b/clang/www/cxx_dr_status.html @@ -11076,7 +11076,7 @@ and POD class 1878 CD4 operator auto template - Clang 18 + Clang 18 1879 @@ -12384,7 +12384,7 @@ and POD class 2096 CD4 Constraints on literal unions - Duplicate of 2598 + Duplicate of 2598 2097 @@ -15396,7 +15396,7 @@ and POD class 2598 C++23 Unions should not require a non-static data member of literal type - Clang 18 + Clang 18 2599 @@ -15726,7 +15726,7 @@ and POD class 2653 C++23 Can an explicit object parameter have a default argument? - Clang 18 + Clang 18 2654 @@ -15840,7 +15840,7 @@ and POD class 2672 DR Lambda body SFINAE is still required, contrary to intent and note - Clang 18 + Clang 18 2673 @@ -15930,7 +15930,7 @@ and POD class 2687 C++23 Calling an explicit object member function via an address-of-overload-set - Clang 18 + Clang 18 2688 @@ -16542,7 +16542,7 @@ and POD class 2789 DR Overload resolution with implicit and explicit object member functions - Clang 18 + Clang 18 2790 diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html index 301f141b2f2b..421b3426b006 100755 --- a/clang/www/cxx_status.html +++ b/clang/www/cxx_status.html @@ -115,7 +115,7 @@ C++23, informally referred to as C++26.

Unevaluated strings P2361R6 - Clang 18 + Clang 18 Add @, $, and ` to the basic character set @@ -145,13 +145,13 @@ C++23, informally referred to as C++26.

Placeholder variables with no name P2169R4 - Clang 18 + Clang 18 Template parameter initialization P2308R1 (DR) - Clang 18 + Clang 18 Pack Indexing @@ -161,7 +161,7 @@ C++23, informally referred to as C++26.

Remove Deprecated Arithmetic Conversion on Enumerations P2864R2 - Clang 18 + Clang 18 @@ -239,7 +239,7 @@ C++23, informally referred to as C++26.

Deducing this P0847R7 - Clang 18 + Clang 18 P2797R0 diff --git a/clang/www/make_cxx_dr_status b/clang/www/make_cxx_dr_status index 38f847cdc1b7..7183e1a6d2be 100755 --- a/clang/www/make_cxx_dr_status +++ b/clang/www/make_cxx_dr_status @@ -1,6 +1,7 @@ #! /usr/bin/env python3 import sys, os, re, urllib.request +latest_release = 18 clang_www_dir = os.path.dirname(__file__) default_issue_list_path = os.path.join(clang_www_dir, 'cwg_index.html') @@ -127,8 +128,6 @@ out_file.write('''\ Available in Clang? ''') -latest_release = 17 - class AvailabilityError(RuntimeError): pass -- GitLab From ae79854e4dcfb923130d3cd751363dbee19cb5b5 Mon Sep 17 00:00:00 2001 From: Aaron Ballman Date: Wed, 6 Mar 2024 08:58:04 -0500 Subject: [PATCH 292/929] Update C status page now that Clang 18.x is released --- clang/www/c_status.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/clang/www/c_status.html b/clang/www/c_status.html index 7bf5e29f0639..9e4600b3e66a 100644 --- a/clang/www/c_status.html +++ b/clang/www/c_status.html @@ -784,7 +784,7 @@ conformance.

Free positioning of labels inside compound statements N2508 - Clang 18 + Clang 18 Clarification request for C17 example of undefined behavior @@ -1156,7 +1156,7 @@ conformance.

Remove trigraphs??! N2940 - Clang 18 + Clang 18 Improved normal enumerations @@ -1196,7 +1196,7 @@ conformance.

Type inference for object declarations N3007 - Clang 18 + Clang 18 constexpr for object definitions -- GitLab From a8dd99eb36bbf000bb1b42e27df8644eb0762fb8 Mon Sep 17 00:00:00 2001 From: eric Date: Wed, 6 Mar 2024 08:55:46 -0500 Subject: [PATCH 293/929] Bump the github actions runner base version. Yesterday, one of the issues the libc++ builders encountered was that they were using a client that was too old; too old to even update automatically. To get things working, i had to push a testing image with this change. The testing image has been working for 12 hours now, so it's time to commit to it :-) --- libcxx/utils/ci/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcxx/utils/ci/docker-compose.yml b/libcxx/utils/ci/docker-compose.yml index 26b341ced415..5cefcb28bd86 100644 --- a/libcxx/utils/ci/docker-compose.yml +++ b/libcxx/utils/ci/docker-compose.yml @@ -21,7 +21,7 @@ services: dockerfile: Dockerfile target: actions-builder args: - BASE_IMAGE: ghcr.io/actions/actions-runner:2.311.0 + BASE_IMAGE: ghcr.io/actions/actions-runner:2.314.1 <<: *compiler_versions android-buildkite-builder: image: ghcr.io/libcxx/android-buildkite-builder:${TAG:-latest} -- GitLab From 63afcbbeeaf68751b57acc1f2911afee767d86bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Botond=20Istv=C3=A1n=20Horv=C3=A1th?= Date: Wed, 6 Mar 2024 15:03:21 +0100 Subject: [PATCH 294/929] [clang][Sema] Bugfix for choosing the more specialized overload (#83279) There was a bug in Clang where it couldn't choose which overload candidate was more specialized if it was comparing a member-function to a non-member function. Previously, this was detected as an ambiguity, now Clang chooses correctly. This patch fixes the bug by fully implementing CWG2445 and moving the template transformation described in `[temp.func.order]` paragraph 3 from `isAtLeastAsSpecializedAs()` to `Sema::getMoreSpecializedTemplate()` so we have the transformed parameter list during the whole comparison. Also, to be able to add the correct type for the implicit object parameter `Sema::getMoreSpecializedTemplate()` has new parameters for the object type. Fixes #74494, fixes #82509 --- clang/docs/ReleaseNotes.rst | 6 + clang/include/clang/Sema/Sema.h | 3 +- clang/lib/Sema/SemaOverload.cpp | 13 +- clang/lib/Sema/SemaTemplateDeduction.cpp | 236 ++++++++++++--------- clang/test/CXX/drs/dr24xx.cpp | 61 ++++++ clang/test/SemaCXX/overload-template.cpp | 25 +++ clang/test/SemaCXX/overloaded-operator.cpp | 37 ++++ clang/www/cxx_dr_status.html | 2 +- 8 files changed, 273 insertions(+), 110 deletions(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 0ff4a93b15ea..942820a52685 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -319,6 +319,12 @@ Bug Fixes to C++ Support Fixes (#GH80630) - Fix a crash when an explicit template argument list is used with a name for which lookup finds a non-template function and a dependent using declarator. +- Fix a bug where overload resolution falsely reported an ambiguity when it was comparing + a member-function against a non member function or a member-function with an + explicit object parameter against a member function with no explicit object parameter + when one of the function had more specialized templates. + Fixes (`#82509 `_) + and (`#74494 `_) Bug Fixes to AST Handling ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index f3d3a57104ee..2d949f3fc9a7 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -9952,7 +9952,8 @@ public: FunctionTemplateDecl *getMoreSpecializedTemplate( FunctionTemplateDecl *FT1, FunctionTemplateDecl *FT2, SourceLocation Loc, TemplatePartialOrderingContext TPOC, unsigned NumCallArguments1, - unsigned NumCallArguments2, bool Reversed = false); + QualType RawObj1Ty = {}, QualType RawObj2Ty = {}, bool Reversed = false); + UnresolvedSetIterator getMostSpecialized(UnresolvedSetIterator SBegin, UnresolvedSetIterator SEnd, TemplateSpecCandidateSet &FailedCandidates, diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp index a03f3eae5478..b0c693f078ef 100644 --- a/clang/lib/Sema/SemaOverload.cpp +++ b/clang/lib/Sema/SemaOverload.cpp @@ -10571,14 +10571,23 @@ bool clang::isBetterOverloadCandidate( // according to the partial ordering rules described in 14.5.5.2, or, // if not that, if (Cand1IsSpecialization && Cand2IsSpecialization) { + const auto *Obj1Context = + dyn_cast(Cand1.FoundDecl->getDeclContext()); + const auto *Obj2Context = + dyn_cast(Cand2.FoundDecl->getDeclContext()); if (FunctionTemplateDecl *BetterTemplate = S.getMoreSpecializedTemplate( Cand1.Function->getPrimaryTemplate(), Cand2.Function->getPrimaryTemplate(), Loc, isa(Cand1.Function) ? TPOC_Conversion : TPOC_Call, - Cand1.ExplicitCallArguments, Cand2.ExplicitCallArguments, - Cand1.isReversed() ^ Cand2.isReversed())) + Cand1.ExplicitCallArguments, + Obj1Context ? QualType(Obj1Context->getTypeForDecl(), 0) + : QualType{}, + Obj2Context ? QualType(Obj2Context->getTypeForDecl(), 0) + : QualType{}, + Cand1.isReversed() ^ Cand2.isReversed())) { return BetterTemplate == Cand1.Function->getPrimaryTemplate(); + } } // -— F1 and F2 are non-template functions with the same diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp index 563491f76f54..65f7fa15b20d 100644 --- a/clang/lib/Sema/SemaTemplateDeduction.cpp +++ b/clang/lib/Sema/SemaTemplateDeduction.cpp @@ -5333,38 +5333,38 @@ bool Sema::CheckIfFunctionSpecializationIsImmediate(FunctionDecl *FD, return false; } -/// If this is a non-static member function, -static void -AddImplicitObjectParameterType(ASTContext &Context, - CXXMethodDecl *Method, - SmallVectorImpl &ArgTypes) { - // C++11 [temp.func.order]p3: - // [...] The new parameter is of type "reference to cv A," where cv are - // the cv-qualifiers of the function template (if any) and A is - // the class of which the function template is a member. +static QualType GetImplicitObjectParameterType(ASTContext &Context, + const CXXMethodDecl *Method, + QualType RawType, + bool IsOtherRvr) { + // C++20 [temp.func.order]p3.1, p3.2: + // - The type X(M) is "rvalue reference to cv A" if the optional + // ref-qualifier of M is && or if M has no ref-qualifier and the + // positionally-corresponding parameter of the other transformed template + // has rvalue reference type; if this determination depends recursively + // upon whether X(M) is an rvalue reference type, it is not considered to + // have rvalue reference type. // - // The standard doesn't say explicitly, but we pick the appropriate kind of - // reference type based on [over.match.funcs]p4. - assert(Method && Method->isImplicitObjectMemberFunction() && - "expected an implicit objet function"); - QualType ArgTy = Context.getTypeDeclType(Method->getParent()); - ArgTy = Context.getQualifiedType(ArgTy, Method->getMethodQualifiers()); - if (Method->getRefQualifier() == RQ_RValue) - ArgTy = Context.getRValueReferenceType(ArgTy); - else - ArgTy = Context.getLValueReferenceType(ArgTy); - ArgTypes.push_back(ArgTy); + // - Otherwise, X(M) is "lvalue reference to cv A". + assert(Method && !Method->isExplicitObjectMemberFunction() && + "expected a member function with no explicit object parameter"); + + RawType = Context.getQualifiedType(RawType, Method->getMethodQualifiers()); + if (Method->getRefQualifier() == RQ_RValue || + (IsOtherRvr && Method->getRefQualifier() == RQ_None)) + return Context.getRValueReferenceType(RawType); + return Context.getLValueReferenceType(RawType); } /// Determine whether the function template \p FT1 is at least as /// specialized as \p FT2. -static bool isAtLeastAsSpecializedAs(Sema &S, - SourceLocation Loc, - FunctionTemplateDecl *FT1, - FunctionTemplateDecl *FT2, +static bool isAtLeastAsSpecializedAs(Sema &S, SourceLocation Loc, + const FunctionTemplateDecl *FT1, + const FunctionTemplateDecl *FT2, TemplatePartialOrderingContext TPOC, - unsigned NumCallArguments1, - bool Reversed) { + bool Reversed, + const SmallVector &Args1, + const SmallVector &Args2) { assert(!Reversed || TPOC == TPOC_Call); FunctionDecl *FD1 = FT1->getTemplatedDecl(); @@ -5381,66 +5381,8 @@ static bool isAtLeastAsSpecializedAs(Sema &S, // The types used to determine the ordering depend on the context in which // the partial ordering is done: TemplateDeductionInfo Info(Loc); - SmallVector Args2; switch (TPOC) { - case TPOC_Call: { - // - In the context of a function call, the function parameter types are - // used. - CXXMethodDecl *Method1 = dyn_cast(FD1); - CXXMethodDecl *Method2 = dyn_cast(FD2); - - // C++11 [temp.func.order]p3: - // [...] If only one of the function templates is a non-static - // member, that function template is considered to have a new - // first parameter inserted in its function parameter list. The - // new parameter is of type "reference to cv A," where cv are - // the cv-qualifiers of the function template (if any) and A is - // the class of which the function template is a member. - // - // Note that we interpret this to mean "if one of the function - // templates is a non-static member and the other is a non-member"; - // otherwise, the ordering rules for static functions against non-static - // functions don't make any sense. - // - // C++98/03 doesn't have this provision but we've extended DR532 to cover - // it as wording was broken prior to it. - SmallVector Args1; - - unsigned NumComparedArguments = NumCallArguments1; - - if (!Method2 && Method1 && Method1->isImplicitObjectMemberFunction()) { - // Compare 'this' from Method1 against first parameter from Method2. - AddImplicitObjectParameterType(S.Context, Method1, Args1); - ++NumComparedArguments; - } else if (!Method1 && Method2 && - Method2->isImplicitObjectMemberFunction()) { - // Compare 'this' from Method2 against first parameter from Method1. - AddImplicitObjectParameterType(S.Context, Method2, Args2); - } else if (Method1 && Method2 && Reversed && - Method1->isImplicitObjectMemberFunction() && - Method2->isImplicitObjectMemberFunction()) { - // Compare 'this' from Method1 against second parameter from Method2 - // and 'this' from Method2 against second parameter from Method1. - AddImplicitObjectParameterType(S.Context, Method1, Args1); - AddImplicitObjectParameterType(S.Context, Method2, Args2); - ++NumComparedArguments; - } - - Args1.insert(Args1.end(), Proto1->param_type_begin(), - Proto1->param_type_end()); - Args2.insert(Args2.end(), Proto2->param_type_begin(), - Proto2->param_type_end()); - - // C++ [temp.func.order]p5: - // The presence of unused ellipsis and default arguments has no effect on - // the partial ordering of function templates. - if (Args1.size() > NumComparedArguments) - Args1.resize(NumComparedArguments); - if (Args2.size() > NumComparedArguments) - Args2.resize(NumComparedArguments); - if (Reversed) - std::reverse(Args2.begin(), Args2.end()); - + case TPOC_Call: if (DeduceTemplateArguments(S, TemplateParams, Args2.data(), Args2.size(), Args1.data(), Args1.size(), Info, Deduced, TDF_None, /*PartialOrdering=*/true) != @@ -5448,7 +5390,6 @@ static bool isAtLeastAsSpecializedAs(Sema &S, return false; break; - } case TPOC_Conversion: // - In the context of a call to a conversion operator, the return types @@ -5536,8 +5477,13 @@ static bool isAtLeastAsSpecializedAs(Sema &S, /// \param NumCallArguments1 The number of arguments in the call to FT1, used /// only when \c TPOC is \c TPOC_Call. /// -/// \param NumCallArguments2 The number of arguments in the call to FT2, used -/// only when \c TPOC is \c TPOC_Call. +/// \param RawObj1Ty The type of the object parameter of FT1 if a member +/// function only used if \c TPOC is \c TPOC_Call and FT1 is a Function +/// template from a member function +/// +/// \param RawObj2Ty The type of the object parameter of FT2 if a member +/// function only used if \c TPOC is \c TPOC_Call and FT2 is a Function +/// template from a member function /// /// \param Reversed If \c true, exactly one of FT1 and FT2 is an overload /// candidate with a reversed parameter order. In this case, the corresponding @@ -5548,13 +5494,76 @@ static bool isAtLeastAsSpecializedAs(Sema &S, FunctionTemplateDecl *Sema::getMoreSpecializedTemplate( FunctionTemplateDecl *FT1, FunctionTemplateDecl *FT2, SourceLocation Loc, TemplatePartialOrderingContext TPOC, unsigned NumCallArguments1, - unsigned NumCallArguments2, bool Reversed) { + QualType RawObj1Ty, QualType RawObj2Ty, bool Reversed) { + SmallVector Args1; + SmallVector Args2; + const FunctionDecl *FD1 = FT1->getTemplatedDecl(); + const FunctionDecl *FD2 = FT2->getTemplatedDecl(); + bool ShouldConvert1 = false; + bool ShouldConvert2 = false; + QualType Obj1Ty; + QualType Obj2Ty; + if (TPOC == TPOC_Call) { + const FunctionProtoType *Proto1 = + FD1->getType()->getAs(); + const FunctionProtoType *Proto2 = + FD2->getType()->getAs(); + + // - In the context of a function call, the function parameter types are + // used. + const CXXMethodDecl *Method1 = dyn_cast(FD1); + const CXXMethodDecl *Method2 = dyn_cast(FD2); + // C++20 [temp.func.order]p3 + // [...] Each function template M that is a member function is + // considered to have a new first parameter of type + // X(M), described below, inserted in its function parameter list. + // + // Note that we interpret "that is a member function" as + // "that is a member function with no expicit object argument". + // Otherwise the ordering rules for methods with expicit objet arguments + // against anything else make no sense. + ShouldConvert1 = Method1 && !Method1->isExplicitObjectMemberFunction(); + ShouldConvert2 = Method2 && !Method2->isExplicitObjectMemberFunction(); + if (ShouldConvert1) { + bool IsRValRef2 = + ShouldConvert2 + ? Method2->getRefQualifier() == RQ_RValue + : Proto2->param_type_begin()[0]->isRValueReferenceType(); + // Compare 'this' from Method1 against first parameter from Method2. + Obj1Ty = GetImplicitObjectParameterType(this->Context, Method1, RawObj1Ty, + IsRValRef2); + Args1.push_back(Obj1Ty); + } + if (ShouldConvert2) { + bool IsRValRef1 = + ShouldConvert1 + ? Method1->getRefQualifier() == RQ_RValue + : Proto1->param_type_begin()[0]->isRValueReferenceType(); + // Compare 'this' from Method2 against first parameter from Method1. + Obj2Ty = GetImplicitObjectParameterType(this->Context, Method2, RawObj2Ty, + IsRValRef1); + Args2.push_back(Obj2Ty); + } + size_t NumComparedArguments = NumCallArguments1 + ShouldConvert1; + + Args1.insert(Args1.end(), Proto1->param_type_begin(), + Proto1->param_type_end()); + Args2.insert(Args2.end(), Proto2->param_type_begin(), + Proto2->param_type_end()); - bool Better1 = isAtLeastAsSpecializedAs(*this, Loc, FT1, FT2, TPOC, - NumCallArguments1, Reversed); - bool Better2 = isAtLeastAsSpecializedAs(*this, Loc, FT2, FT1, TPOC, - NumCallArguments2, Reversed); + // C++ [temp.func.order]p5: + // The presence of unused ellipsis and default arguments has no effect on + // the partial ordering of function templates. + Args1.resize(std::min(Args1.size(), NumComparedArguments)); + Args2.resize(std::min(Args2.size(), NumComparedArguments)); + if (Reversed) + std::reverse(Args2.begin(), Args2.end()); + } + bool Better1 = isAtLeastAsSpecializedAs(*this, Loc, FT1, FT2, TPOC, Reversed, + Args1, Args2); + bool Better2 = isAtLeastAsSpecializedAs(*this, Loc, FT2, FT1, TPOC, Reversed, + Args2, Args1); // C++ [temp.deduct.partial]p10: // F is more specialized than G if F is at least as specialized as G and G // is not at least as specialized as F. @@ -5568,12 +5577,28 @@ FunctionTemplateDecl *Sema::getMoreSpecializedTemplate( // ... and if G has a trailing function parameter pack for which F does not // have a corresponding parameter, and if F does not have a trailing // function parameter pack, then F is more specialized than G. - FunctionDecl *FD1 = FT1->getTemplatedDecl(); - FunctionDecl *FD2 = FT2->getTemplatedDecl(); - unsigned NumParams1 = FD1->getNumParams(); - unsigned NumParams2 = FD2->getNumParams(); - bool Variadic1 = NumParams1 && FD1->parameters().back()->isParameterPack(); - bool Variadic2 = NumParams2 && FD2->parameters().back()->isParameterPack(); + + SmallVector Param1; + Param1.reserve(FD1->param_size() + ShouldConvert1); + if (ShouldConvert1) + Param1.push_back(Obj1Ty); + for (const auto &P : FD1->parameters()) + Param1.push_back(P->getType()); + + SmallVector Param2; + Param2.reserve(FD2->param_size() + ShouldConvert2); + if (ShouldConvert2) + Param2.push_back(Obj2Ty); + for (const auto &P : FD2->parameters()) + Param2.push_back(P->getType()); + + unsigned NumParams1 = Param1.size(); + unsigned NumParams2 = Param2.size(); + + bool Variadic1 = + FD1->param_size() && FD1->parameters().back()->isParameterPack(); + bool Variadic2 = + FD2->param_size() && FD2->parameters().back()->isParameterPack(); if (Variadic1 != Variadic2) { if (Variadic1 && NumParams1 > NumParams2) return FT2; @@ -5584,8 +5609,8 @@ FunctionTemplateDecl *Sema::getMoreSpecializedTemplate( // This a speculative fix for CWG1432 (Similar to the fix for CWG1395) that // there is no wording or even resolution for this issue. for (int i = 0, e = std::min(NumParams1, NumParams2); i < e; ++i) { - QualType T1 = FD1->getParamDecl(i)->getType().getCanonicalType(); - QualType T2 = FD2->getParamDecl(i)->getType().getCanonicalType(); + QualType T1 = Param1[i].getCanonicalType(); + QualType T2 = Param2[i].getCanonicalType(); auto *TST1 = dyn_cast(T1); auto *TST2 = dyn_cast(T2); if (!TST1 || !TST2) @@ -5644,8 +5669,7 @@ FunctionTemplateDecl *Sema::getMoreSpecializedTemplate( // Any top-level cv-qualifiers modifying a parameter type are deleted when // forming the function type. for (unsigned i = 0; i < NumParams1; ++i) - if (!Context.hasSameUnqualifiedType(FD1->getParamDecl(i)->getType(), - FD2->getParamDecl(i)->getType())) + if (!Context.hasSameUnqualifiedType(Param1[i], Param2[i])) return nullptr; // C++20 [temp.func.order]p6.3: @@ -5733,8 +5757,8 @@ UnresolvedSetIterator Sema::getMostSpecialized( FunctionTemplateDecl *Challenger = cast(*I)->getPrimaryTemplate(); assert(Challenger && "Not a function template specialization?"); - if (isSameTemplate(getMoreSpecializedTemplate(BestTemplate, Challenger, - Loc, TPOC_Other, 0, 0), + if (isSameTemplate(getMoreSpecializedTemplate(BestTemplate, Challenger, Loc, + TPOC_Other, 0), Challenger)) { Best = I; BestTemplate = Challenger; @@ -5749,7 +5773,7 @@ UnresolvedSetIterator Sema::getMostSpecialized( = cast(*I)->getPrimaryTemplate(); if (I != Best && !isSameTemplate(getMoreSpecializedTemplate(BestTemplate, Challenger, - Loc, TPOC_Other, 0, 0), + Loc, TPOC_Other, 0), BestTemplate)) { Ambiguous = true; break; diff --git a/clang/test/CXX/drs/dr24xx.cpp b/clang/test/CXX/drs/dr24xx.cpp index ae8dda3351f4..4534ed26e56d 100644 --- a/clang/test/CXX/drs/dr24xx.cpp +++ b/clang/test/CXX/drs/dr24xx.cpp @@ -68,3 +68,64 @@ template struct X {}; X<1> x; #endif } + +namespace dr2445 { // dr2445: 19 +#if __cplusplus >= 202002L + template constexpr bool F = false; + template struct A { }; + + template + bool operator==(T, A); + + template + bool operator!=(A, U) { + static_assert(F, "Isn't this less specialized?"); + return false; + } + + bool f(A ax, A ay) { return ay != ax; } + + template concept AlwaysTrue=true; + template struct B { + template + bool operator==(const B&)const; + }; + + + template + bool operator==(const B&,const B&) { + static_assert(F, "Isn't this less specialized?"); + return false; + } + + bool g(B bx, B by) { return bx == by; } + + struct C{ + template + int operator+(T){return 0;} + template + void operator-(T){} + }; + template + void operator+(C&&,T){} + template + int operator-(C&&,T){return 0;} + + void t(int* iptr){ + int x1 = C{} + iptr; + int x2 = C{} - iptr; + } + + struct D{ + template + int operator+(T) volatile {return 1;} + }; + + template + void operator+(volatile D&,T) {} + + int foo(volatile D& d){ + return d + 1; + } +#endif +} diff --git a/clang/test/SemaCXX/overload-template.cpp b/clang/test/SemaCXX/overload-template.cpp index 0a23788ef3da..0fe13c479cce 100644 --- a/clang/test/SemaCXX/overload-template.cpp +++ b/clang/test/SemaCXX/overload-template.cpp @@ -1,4 +1,5 @@ // RUN: %clang_cc1 -fsyntax-only -verify %s +// RUN: %clang_cc1 -std=c++23 -verify -fsyntax-only %s enum copy_traits { movable = 1 }; @@ -33,3 +34,27 @@ void ReproducesBugSimply() { InsertRow(3, B{}); // expected-error {{no matching function for call to 'InsertRow'}} } +#if __cplusplus >= 202302L +namespace overloadCheck{ + template + concept AlwaysTrue = true; + + struct S { + int f(AlwaysTrue auto) { return 1; } + void f(this S&&, auto) {} + + void g(auto) {} + int g(this S&&,AlwaysTrue auto) {return 1;} + + int h(AlwaysTrue auto) { return 1; } //expected-note {{previous definition is here}} + int h(this S&&,AlwaysTrue auto) { // expected-error {{class member cannot be redeclared}} + return 1; + } + }; + + int main() { + int x = S{}.f(0); + int y = S{}.g(0); + } +} +#endif diff --git a/clang/test/SemaCXX/overloaded-operator.cpp b/clang/test/SemaCXX/overloaded-operator.cpp index 887848c29b83..49311625d7ab 100644 --- a/clang/test/SemaCXX/overloaded-operator.cpp +++ b/clang/test/SemaCXX/overloaded-operator.cpp @@ -645,3 +645,40 @@ class b { } + +#if __cplusplus >= 202002L +namespace nw{ + template + concept AlwaysTrue=true; + + struct S{ + template + void operator+(const T&)const{} + + template + int operator-(const T&)const{return 0;} + + template + int operator*(const T&)const{ // expected-note {{candidate function}} + return 0; + } + }; + + template + int operator+(const S&, const T&){return 0;} + + template + void operator-(const S&, const T&){} + + template + int operator*(const S&, const T&){ // expected-note {{candidate function}} + return 0; + } + + void foo(){ + int a = S{} + 1; + int b = S{} - 1; + int c = S{} * 1; // expected-error {{use of overloaded operator '*' is ambiguous (with operand types 'S' and 'int')}} + } +} +#endif diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html index 0b810b50c529..774c71bc1cb6 100755 --- a/clang/www/cxx_dr_status.html +++ b/clang/www/cxx_dr_status.html @@ -14478,7 +14478,7 @@ and POD class 2445 C++20 Partial ordering with rewritten candidates - Unknown + Clang 19 2446 -- GitLab From 1fc5e50cebf1e423bce105930c3a075044998f68 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Wed, 6 Mar 2024 08:11:54 -0600 Subject: [PATCH 295/929] [AMDGPU] Implement 'llvm.get.fpenv' and 'llvm.set.fpenv' (#83906) Summary: This patch implements the LLVM floating point environment control intrinsics and also exposes it through clang. We encode the floating point environment as a 64-bit value that simply concatenates the values of the mode registers and the current trap status. We only fetch the bits relevant for floating point instructions. That is, rounding mode, denormalization mode, ieee, dx10 clamp, debug, enabled traps, f16 overflow, and active exceptions. --- clang/include/clang/Basic/BuiltinsAMDGPU.def | 3 + clang/lib/CodeGen/CGBuiltin.cpp | 11 + clang/lib/Sema/SemaChecking.cpp | 3 + clang/test/CodeGenOpenCL/builtins-amdgcn.cl | 12 + llvm/docs/AMDGPUUsage.rst | 7 + llvm/docs/LangRef.rst | 2 + llvm/docs/ReleaseNotes.rst | 2 + .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 52 +++ llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h | 5 + llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 72 ++++ llvm/lib/Target/AMDGPU/SIISelLowering.h | 2 + llvm/test/CodeGen/AMDGPU/fpenv.ll | 374 ++++++++++++++++++ 12 files changed, 545 insertions(+) create mode 100644 llvm/test/CodeGen/AMDGPU/fpenv.ll diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def index 213311b96df7..6628e8f265fe 100644 --- a/clang/include/clang/Basic/BuiltinsAMDGPU.def +++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def @@ -325,6 +325,9 @@ BUILTIN(__builtin_amdgcn_read_exec_hi, "Ui", "nc") BUILTIN(__builtin_amdgcn_endpgm, "v", "nr") +BUILTIN(__builtin_amdgcn_get_fpenv, "WUi", "n") +BUILTIN(__builtin_amdgcn_set_fpenv, "vWUi", "n") + //===----------------------------------------------------------------------===// // R600-NI only builtins. //===----------------------------------------------------------------------===// diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index cc30665a4eee..20c357579391 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -18439,6 +18439,17 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, CGM.getIntrinsic(Intrinsic::amdgcn_global_load_tr, {ArgTy}); return Builder.CreateCall(F, {Addr}); } + case AMDGPU::BI__builtin_amdgcn_get_fpenv: { + Function *F = CGM.getIntrinsic(Intrinsic::get_fpenv, + {llvm::Type::getInt64Ty(getLLVMContext())}); + return Builder.CreateCall(F); + } + case AMDGPU::BI__builtin_amdgcn_set_fpenv: { + Function *F = CGM.getIntrinsic(Intrinsic::set_fpenv, + {llvm::Type::getInt64Ty(getLLVMContext())}); + llvm::Value *Env = EmitScalarExpr(E->getArg(0)); + return Builder.CreateCall(F, {Env}); + } case AMDGPU::BI__builtin_amdgcn_read_exec: return EmitAMDGCNBallotForExec(*this, E, Int64Ty, Int64Ty, false); case AMDGPU::BI__builtin_amdgcn_read_exec_lo: diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index 561764edd081..82a7c16106f6 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -5321,6 +5321,9 @@ bool Sema::CheckAMDGCNBuiltinFunctionCall(unsigned BuiltinID, // position of memory order and scope arguments in the builtin unsigned OrderIndex, ScopeIndex; switch (BuiltinID) { + case AMDGPU::BI__builtin_amdgcn_get_fpenv: + case AMDGPU::BI__builtin_amdgcn_set_fpenv: + return false; case AMDGPU::BI__builtin_amdgcn_atomic_inc32: case AMDGPU::BI__builtin_amdgcn_atomic_inc64: case AMDGPU::BI__builtin_amdgcn_atomic_dec32: diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn.cl index 7d9010ee9067..8a4533633706 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn.cl @@ -839,6 +839,18 @@ unsigned test_wavefrontsize() { return __builtin_amdgcn_wavefrontsize(); } +// CHECK-LABEL test_get_fpenv( +unsigned long test_get_fpenv() { + // CHECK: call i64 @llvm.get.fpenv.i64() + return __builtin_amdgcn_get_fpenv(); +} + +// CHECK-LABEL test_set_fpenv( +void test_set_fpenv(unsigned long env) { + // CHECK: call void @llvm.set.fpenv.i64(i64 %[[ENV:.+]]) + __builtin_amdgcn_set_fpenv(env); +} + // CHECK-DAG: [[$WI_RANGE]] = !{i32 0, i32 1024} // CHECK-DAG: [[$WS_RANGE]] = !{i16 1, i16 1025} // CHECK-DAG: attributes #[[$NOUNWIND_READONLY]] = { convergent mustprogress nocallback nofree nounwind willreturn memory(none) } diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst index 0c588c849586..ed41be4b08de 100644 --- a/llvm/docs/AMDGPUUsage.rst +++ b/llvm/docs/AMDGPUUsage.rst @@ -1151,6 +1151,13 @@ The AMDGPU backend implements the following LLVM IR intrinsics. register do not exactly match the FLT_ROUNDS values, so a conversion is performed. + :ref:`llvm.get.fpenv` Returns the current value of the AMDGPU floating point environment. + This stores information related to the current rounding mode, + denormalization mode, enabled traps, and floating point exceptions. + The format is a 64-bit concatenation of the MODE and TRAPSTS registers. + + :ref:`llvm.set.fpenv` Sets the floating point environment to the specifies state. + llvm.amdgcn.wave.reduce.umin Performs an arithmetic unsigned min reduction on the unsigned values provided by each lane in the wavefront. Intrinsic takes a hint for reduction strategy using second operand diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index 0e3f6f8daaba..e0f301aa06dc 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -26432,6 +26432,7 @@ similar to C library function 'fesetround', however this intrinsic does not return any value and uses platform-independent representation of IEEE rounding modes. +.. _int_get_fpenv: '``llvm.get.fpenv``' Intrinsic ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -26455,6 +26456,7 @@ Semantics: The '``llvm.get.fpenv``' intrinsic reads the current floating-point environment and returns it as an integer value. +.. _int_set_fpenv: '``llvm.set.fpenv``' Intrinsic ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst index 8ce6ee5cebb2..56abe7fe4027 100644 --- a/llvm/docs/ReleaseNotes.rst +++ b/llvm/docs/ReleaseNotes.rst @@ -70,6 +70,8 @@ Changes to the AArch64 Backend Changes to the AMDGPU Backend ----------------------------- +* Implemented the ``llvm.get.fpenv`` and ``llvm.set.fpenv`` intrinsics. + Changes to the ARM Backend -------------------------- diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index 4c3b983f2960..0029c51231f2 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -905,6 +905,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, getActionDefinitionsBuilder(G_STACKRESTORE) .legalFor({PrivatePtr}); + getActionDefinitionsBuilder({G_GET_FPENV, G_SET_FPENV}).customFor({S64}); + getActionDefinitionsBuilder(G_GLOBAL_VALUE) .customIf(typeIsNot(0, PrivatePtr)); @@ -2128,6 +2130,10 @@ bool AMDGPULegalizerInfo::legalizeCustom( return legalizeFPTruncRound(MI, B); case TargetOpcode::G_STACKSAVE: return legalizeStackSave(MI, B); + case TargetOpcode::G_GET_FPENV: + return legalizeGetFPEnv(MI, MRI, B); + case TargetOpcode::G_SET_FPENV: + return legalizeSetFPEnv(MI, MRI, B); default: return false; } @@ -6940,6 +6946,52 @@ bool AMDGPULegalizerInfo::legalizeWaveID(MachineInstr &MI, return true; } +static constexpr unsigned FPEnvModeBitField = + AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_MODE, 0, 23); + +static constexpr unsigned FPEnvTrapBitField = + AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_TRAPSTS, 0, 5); + +bool AMDGPULegalizerInfo::legalizeGetFPEnv(MachineInstr &MI, + MachineRegisterInfo &MRI, + MachineIRBuilder &B) const { + Register Src = MI.getOperand(0).getReg(); + if (MRI.getType(Src) != S64) + return false; + + auto ModeReg = + B.buildIntrinsic(Intrinsic::amdgcn_s_getreg, {S32}, + /*HasSideEffects=*/true, /*isConvergent=*/false) + .addImm(FPEnvModeBitField); + auto TrapReg = + B.buildIntrinsic(Intrinsic::amdgcn_s_getreg, {S32}, + /*HasSideEffects=*/true, /*isConvergent=*/false) + .addImm(FPEnvTrapBitField); + B.buildMergeLikeInstr(Src, {ModeReg, TrapReg}); + MI.eraseFromParent(); + return true; +} + +bool AMDGPULegalizerInfo::legalizeSetFPEnv(MachineInstr &MI, + MachineRegisterInfo &MRI, + MachineIRBuilder &B) const { + Register Src = MI.getOperand(0).getReg(); + if (MRI.getType(Src) != S64) + return false; + + auto Unmerge = B.buildUnmerge({S32, S32}, MI.getOperand(0)); + B.buildIntrinsic(Intrinsic::amdgcn_s_setreg, ArrayRef(), + /*HasSideEffects=*/true, /*isConvergent=*/false) + .addImm(static_cast(FPEnvModeBitField)) + .addReg(Unmerge.getReg(0)); + B.buildIntrinsic(Intrinsic::amdgcn_s_setreg, ArrayRef(), + /*HasSideEffects=*/true, /*isConvergent=*/false) + .addImm(static_cast(FPEnvTrapBitField)) + .addReg(Unmerge.getReg(1)); + MI.eraseFromParent(); + return true; +} + bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const { MachineIRBuilder &B = Helper.MIRBuilder; diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h index ecbe42681c66..9661646fffc9 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h @@ -214,6 +214,11 @@ public: bool legalizeStackSave(MachineInstr &MI, MachineIRBuilder &B) const; bool legalizeWaveID(MachineInstr &MI, MachineIRBuilder &B) const; + bool legalizeGetFPEnv(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &B) const; + bool legalizeSetFPEnv(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &B) const; + bool legalizeImageIntrinsic( MachineInstr &MI, MachineIRBuilder &B, GISelChangeObserver &Observer, diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 5f6ce12ae1fb..bf1463f93df6 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -877,6 +877,8 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, setOperationAction(ISD::STACKSAVE, MVT::Other, Custom); setOperationAction(ISD::GET_ROUNDING, MVT::i32, Custom); + setOperationAction(ISD::GET_FPENV, MVT::i64, Custom); + setOperationAction(ISD::SET_FPENV, MVT::i64, Custom); // TODO: Could move this to custom lowering, could benefit from combines on // extract of relevant bits. @@ -4081,6 +4083,72 @@ SDValue SITargetLowering::lowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const { return DAG.getNode(ISD::BF16_TO_FP, SL, DstVT, BitCast); } +SDValue SITargetLowering::lowerGET_FPENV(SDValue Op, SelectionDAG &DAG) const { + SDLoc SL(Op); + if (Op.getValueType() != MVT::i64) + return Op; + + uint32_t ModeHwReg = + AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_MODE, 0, 23); + SDValue ModeHwRegImm = DAG.getTargetConstant(ModeHwReg, SL, MVT::i32); + uint32_t TrapHwReg = + AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_TRAPSTS, 0, 5); + SDValue TrapHwRegImm = DAG.getTargetConstant(TrapHwReg, SL, MVT::i32); + + SDVTList VTList = DAG.getVTList(MVT::i32, MVT::Other); + SDValue IntrinID = + DAG.getTargetConstant(Intrinsic::amdgcn_s_getreg, SL, MVT::i32); + SDValue GetModeReg = DAG.getNode(ISD::INTRINSIC_W_CHAIN, SL, VTList, + Op.getOperand(0), IntrinID, ModeHwRegImm); + SDValue GetTrapReg = DAG.getNode(ISD::INTRINSIC_W_CHAIN, SL, VTList, + Op.getOperand(0), IntrinID, TrapHwRegImm); + SDValue TokenReg = + DAG.getNode(ISD::TokenFactor, SL, MVT::Other, GetModeReg.getValue(1), + GetTrapReg.getValue(1)); + + SDValue CvtPtr = + DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, GetModeReg, GetTrapReg); + SDValue Result = DAG.getNode(ISD::BITCAST, SL, MVT::i64, CvtPtr); + + return DAG.getMergeValues({Result, TokenReg}, SL); +} + +SDValue SITargetLowering::lowerSET_FPENV(SDValue Op, SelectionDAG &DAG) const { + SDLoc SL(Op); + if (Op.getOperand(1).getValueType() != MVT::i64) + return Op; + + SDValue Input = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op.getOperand(1)); + SDValue NewModeReg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Input, + DAG.getConstant(0, SL, MVT::i32)); + SDValue NewTrapReg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Input, + DAG.getConstant(1, SL, MVT::i32)); + + SDValue ReadFirstLaneID = + DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, SL, MVT::i32); + NewModeReg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32, + ReadFirstLaneID, NewModeReg); + NewTrapReg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32, + ReadFirstLaneID, NewTrapReg); + + unsigned ModeHwReg = + AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_MODE, 0, 23); + SDValue ModeHwRegImm = DAG.getTargetConstant(ModeHwReg, SL, MVT::i32); + unsigned TrapHwReg = + AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_TRAPSTS, 0, 5); + SDValue TrapHwRegImm = DAG.getTargetConstant(TrapHwReg, SL, MVT::i32); + + SDValue IntrinID = + DAG.getTargetConstant(Intrinsic::amdgcn_s_setreg, SL, MVT::i32); + SDValue SetModeReg = + DAG.getNode(ISD::INTRINSIC_VOID, SL, MVT::Other, Op.getOperand(0), + IntrinID, ModeHwRegImm, NewModeReg); + SDValue SetTrapReg = + DAG.getNode(ISD::INTRINSIC_VOID, SL, MVT::Other, Op.getOperand(0), + IntrinID, TrapHwRegImm, NewTrapReg); + return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, SetTrapReg, SetModeReg); +} + Register SITargetLowering::getRegisterByName(const char* RegName, LLT VT, const MachineFunction &MF) const { Register Reg = StringSwitch(RegName) @@ -5681,6 +5749,10 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { case ISD::FP_EXTEND: case ISD::STRICT_FP_EXTEND: return lowerFP_EXTEND(Op, DAG); + case ISD::GET_FPENV: + return lowerGET_FPENV(Op, DAG); + case ISD::SET_FPENV: + return lowerSET_FPENV(Op, DAG); } return SDValue(); } diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h index fc90a208fa0b..a20442e3737e 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.h +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h @@ -425,6 +425,8 @@ public: SDValue lowerPREFETCH(SDValue Op, SelectionDAG &DAG) const; SDValue lowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const; + SDValue lowerGET_FPENV(SDValue Op, SelectionDAG &DAG) const; + SDValue lowerSET_FPENV(SDValue Op, SelectionDAG &DAG) const; Register getRegisterByName(const char* RegName, LLT VT, const MachineFunction &MF) const override; diff --git a/llvm/test/CodeGen/AMDGPU/fpenv.ll b/llvm/test/CodeGen/AMDGPU/fpenv.ll new file mode 100644 index 000000000000..80d5c9ad0ceb --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/fpenv.ll @@ -0,0 +1,374 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=amdgcn -global-isel=0 -mcpu=tahiti < %s | FileCheck -check-prefixes=GFX6-SDAG %s +; RUN: llc -mtriple=amdgcn -global-isel=1 -mcpu=tahiti < %s | FileCheck -check-prefixes=GFX6-ISEL %s +; RUN: llc -mtriple=amdgcn -global-isel=0 -mcpu=fiji < %s | FileCheck -check-prefixes=GFX8-SDAG %s +; RUN: llc -mtriple=amdgcn -global-isel=1 -mcpu=fiji < %s | FileCheck -check-prefixes=GFX8-ISEL %s +; RUN: llc -mtriple=amdgcn -global-isel=0 -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9-SDAG %s +; RUN: llc -mtriple=amdgcn -global-isel=1 -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9-ISEL %s +; RUN: llc -mtriple=amdgcn -global-isel=0 -mcpu=gfx1030 < %s | FileCheck -check-prefixes=GFX10-SDAG %s +; RUN: llc -mtriple=amdgcn -global-isel=1 -mcpu=gfx1030 < %s | FileCheck -check-prefixes=GFX10-ISEL %s +; RUN: llc -mtriple=amdgcn -amdgpu-enable-delay-alu=0 -global-isel=0 -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11-SDAG %s +; RUN: llc -mtriple=amdgcn -amdgpu-enable-delay-alu=0 -global-isel=1 -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11-ISEL %s + +declare i64 @llvm.get.fpenv.i64() + +declare void @llvm.set.fpenv.i64(i64) + +define i64 @get_fpenv() { +; GFX6-SDAG-LABEL: get_fpenv: +; GFX6-SDAG: ; %bb.0: ; %entry +; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX6-SDAG-NEXT: s_getreg_b32 s4, hwreg(HW_REG_TRAPSTS, 0, 5) +; GFX6-SDAG-NEXT: s_getreg_b32 s5, hwreg(HW_REG_MODE, 0, 23) +; GFX6-SDAG-NEXT: v_mov_b32_e32 v0, s5 +; GFX6-SDAG-NEXT: v_mov_b32_e32 v1, s4 +; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX6-ISEL-LABEL: get_fpenv: +; GFX6-ISEL: ; %bb.0: ; %entry +; GFX6-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX6-ISEL-NEXT: s_getreg_b32 s4, hwreg(HW_REG_MODE, 0, 23) +; GFX6-ISEL-NEXT: s_getreg_b32 s5, hwreg(HW_REG_TRAPSTS, 0, 5) +; GFX6-ISEL-NEXT: v_mov_b32_e32 v0, s4 +; GFX6-ISEL-NEXT: v_mov_b32_e32 v1, s5 +; GFX6-ISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX8-SDAG-LABEL: get_fpenv: +; GFX8-SDAG: ; %bb.0: ; %entry +; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-SDAG-NEXT: s_getreg_b32 s4, hwreg(HW_REG_TRAPSTS, 0, 5) +; GFX8-SDAG-NEXT: s_getreg_b32 s5, hwreg(HW_REG_MODE, 0, 23) +; GFX8-SDAG-NEXT: v_mov_b32_e32 v0, s5 +; GFX8-SDAG-NEXT: v_mov_b32_e32 v1, s4 +; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX8-ISEL-LABEL: get_fpenv: +; GFX8-ISEL: ; %bb.0: ; %entry +; GFX8-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-ISEL-NEXT: s_getreg_b32 s4, hwreg(HW_REG_MODE, 0, 23) +; GFX8-ISEL-NEXT: s_getreg_b32 s5, hwreg(HW_REG_TRAPSTS, 0, 5) +; GFX8-ISEL-NEXT: v_mov_b32_e32 v0, s4 +; GFX8-ISEL-NEXT: v_mov_b32_e32 v1, s5 +; GFX8-ISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-SDAG-LABEL: get_fpenv: +; GFX9-SDAG: ; %bb.0: ; %entry +; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-SDAG-NEXT: s_getreg_b32 s4, hwreg(HW_REG_TRAPSTS, 0, 5) +; GFX9-SDAG-NEXT: s_getreg_b32 s5, hwreg(HW_REG_MODE, 0, 23) +; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s5 +; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s4 +; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-ISEL-LABEL: get_fpenv: +; GFX9-ISEL: ; %bb.0: ; %entry +; GFX9-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-ISEL-NEXT: s_getreg_b32 s4, hwreg(HW_REG_MODE, 0, 23) +; GFX9-ISEL-NEXT: s_getreg_b32 s5, hwreg(HW_REG_TRAPSTS, 0, 5) +; GFX9-ISEL-NEXT: v_mov_b32_e32 v0, s4 +; GFX9-ISEL-NEXT: v_mov_b32_e32 v1, s5 +; GFX9-ISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-SDAG-LABEL: get_fpenv: +; GFX10-SDAG: ; %bb.0: ; %entry +; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-SDAG-NEXT: s_getreg_b32 s4, hwreg(HW_REG_TRAPSTS, 0, 5) +; GFX10-SDAG-NEXT: s_getreg_b32 s5, hwreg(HW_REG_MODE, 0, 23) +; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4 +; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s5 +; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-ISEL-LABEL: get_fpenv: +; GFX10-ISEL: ; %bb.0: ; %entry +; GFX10-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-ISEL-NEXT: s_getreg_b32 s4, hwreg(HW_REG_MODE, 0, 23) +; GFX10-ISEL-NEXT: s_getreg_b32 s5, hwreg(HW_REG_TRAPSTS, 0, 5) +; GFX10-ISEL-NEXT: v_mov_b32_e32 v0, s4 +; GFX10-ISEL-NEXT: v_mov_b32_e32 v1, s5 +; GFX10-ISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: get_fpenv: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: s_getreg_b32 s0, hwreg(HW_REG_TRAPSTS, 0, 5) +; GFX11-SDAG-NEXT: s_getreg_b32 s1, hwreg(HW_REG_MODE, 0, 23) +; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v0, s1 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-ISEL-LABEL: get_fpenv: +; GFX11-ISEL: ; %bb.0: ; %entry +; GFX11-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-ISEL-NEXT: s_getreg_b32 s0, hwreg(HW_REG_MODE, 0, 23) +; GFX11-ISEL-NEXT: s_getreg_b32 s1, hwreg(HW_REG_TRAPSTS, 0, 5) +; GFX11-ISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 +; GFX11-ISEL-NEXT: s_setpc_b64 s[30:31] +entry: + %0 = call i64 @llvm.get.fpenv.i64() + ret i64 %0 +} + +define void @set_fpenv(i64 %env) { +; GFX6-SDAG-LABEL: set_fpenv: +; GFX6-SDAG: ; %bb.0: ; %entry +; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX6-SDAG-NEXT: v_readfirstlane_b32 s4, v0 +; GFX6-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 23), s4 +; GFX6-SDAG-NEXT: v_readfirstlane_b32 s4, v1 +; GFX6-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_TRAPSTS, 0, 5), s4 +; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX6-ISEL-LABEL: set_fpenv: +; GFX6-ISEL: ; %bb.0: ; %entry +; GFX6-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX6-ISEL-NEXT: v_readfirstlane_b32 s4, v0 +; GFX6-ISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 23), s4 +; GFX6-ISEL-NEXT: v_readfirstlane_b32 s4, v1 +; GFX6-ISEL-NEXT: s_setreg_b32 hwreg(HW_REG_TRAPSTS, 0, 5), s4 +; GFX6-ISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX8-SDAG-LABEL: set_fpenv: +; GFX8-SDAG: ; %bb.0: ; %entry +; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-SDAG-NEXT: v_readfirstlane_b32 s4, v0 +; GFX8-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 23), s4 +; GFX8-SDAG-NEXT: v_readfirstlane_b32 s4, v1 +; GFX8-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_TRAPSTS, 0, 5), s4 +; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX8-ISEL-LABEL: set_fpenv: +; GFX8-ISEL: ; %bb.0: ; %entry +; GFX8-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-ISEL-NEXT: v_readfirstlane_b32 s4, v0 +; GFX8-ISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 23), s4 +; GFX8-ISEL-NEXT: v_readfirstlane_b32 s4, v1 +; GFX8-ISEL-NEXT: s_setreg_b32 hwreg(HW_REG_TRAPSTS, 0, 5), s4 +; GFX8-ISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-SDAG-LABEL: set_fpenv: +; GFX9-SDAG: ; %bb.0: ; %entry +; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-SDAG-NEXT: v_readfirstlane_b32 s4, v0 +; GFX9-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 23), s4 +; GFX9-SDAG-NEXT: v_readfirstlane_b32 s4, v1 +; GFX9-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_TRAPSTS, 0, 5), s4 +; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-ISEL-LABEL: set_fpenv: +; GFX9-ISEL: ; %bb.0: ; %entry +; GFX9-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-ISEL-NEXT: v_readfirstlane_b32 s4, v0 +; GFX9-ISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 23), s4 +; GFX9-ISEL-NEXT: v_readfirstlane_b32 s4, v1 +; GFX9-ISEL-NEXT: s_setreg_b32 hwreg(HW_REG_TRAPSTS, 0, 5), s4 +; GFX9-ISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-SDAG-LABEL: set_fpenv: +; GFX10-SDAG: ; %bb.0: ; %entry +; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v0 +; GFX10-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 23), s4 +; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v1 +; GFX10-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_TRAPSTS, 0, 5), s4 +; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-ISEL-LABEL: set_fpenv: +; GFX10-ISEL: ; %bb.0: ; %entry +; GFX10-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-ISEL-NEXT: v_readfirstlane_b32 s4, v0 +; GFX10-ISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 23), s4 +; GFX10-ISEL-NEXT: v_readfirstlane_b32 s4, v1 +; GFX10-ISEL-NEXT: s_setreg_b32 hwreg(HW_REG_TRAPSTS, 0, 5), s4 +; GFX10-ISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: set_fpenv: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_readfirstlane_b32 s0, v0 +; GFX11-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 23), s0 +; GFX11-SDAG-NEXT: v_readfirstlane_b32 s0, v1 +; GFX11-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_TRAPSTS, 0, 5), s0 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-ISEL-LABEL: set_fpenv: +; GFX11-ISEL: ; %bb.0: ; %entry +; GFX11-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-ISEL-NEXT: v_readfirstlane_b32 s0, v0 +; GFX11-ISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 23), s0 +; GFX11-ISEL-NEXT: v_readfirstlane_b32 s0, v1 +; GFX11-ISEL-NEXT: s_setreg_b32 hwreg(HW_REG_TRAPSTS, 0, 5), s0 +; GFX11-ISEL-NEXT: s_setpc_b64 s[30:31] +entry: + call void @llvm.set.fpenv.i64(i64 %env) + ret void +} + +define void @set_fpenv_constant() { +; GFX6-SDAG-LABEL: set_fpenv_constant: +; GFX6-SDAG: ; %bb.0: ; %entry +; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX6-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 23), 0 +; GFX6-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 0, 5), 0 +; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX6-ISEL-LABEL: set_fpenv_constant: +; GFX6-ISEL: ; %bb.0: ; %entry +; GFX6-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX6-ISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 23), 0 +; GFX6-ISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 0, 5), 0 +; GFX6-ISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX8-SDAG-LABEL: set_fpenv_constant: +; GFX8-SDAG: ; %bb.0: ; %entry +; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 23), 0 +; GFX8-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 0, 5), 0 +; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX8-ISEL-LABEL: set_fpenv_constant: +; GFX8-ISEL: ; %bb.0: ; %entry +; GFX8-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-ISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 23), 0 +; GFX8-ISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 0, 5), 0 +; GFX8-ISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-SDAG-LABEL: set_fpenv_constant: +; GFX9-SDAG: ; %bb.0: ; %entry +; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 23), 0 +; GFX9-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 0, 5), 0 +; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-ISEL-LABEL: set_fpenv_constant: +; GFX9-ISEL: ; %bb.0: ; %entry +; GFX9-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-ISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 23), 0 +; GFX9-ISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 0, 5), 0 +; GFX9-ISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-SDAG-LABEL: set_fpenv_constant: +; GFX10-SDAG: ; %bb.0: ; %entry +; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 23), 0 +; GFX10-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 0, 5), 0 +; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-ISEL-LABEL: set_fpenv_constant: +; GFX10-ISEL: ; %bb.0: ; %entry +; GFX10-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-ISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 23), 0 +; GFX10-ISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 0, 5), 0 +; GFX10-ISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: set_fpenv_constant: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 23), 0 +; GFX11-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 0, 5), 0 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-ISEL-LABEL: set_fpenv_constant: +; GFX11-ISEL: ; %bb.0: ; %entry +; GFX11-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-ISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 23), 0 +; GFX11-ISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 0, 5), 0 +; GFX11-ISEL-NEXT: s_setpc_b64 s[30:31] +entry: + call void @llvm.set.fpenv.i64(i64 0) + ret void +} + +define void @get_set_fpenv() { +; GFX6-SDAG-LABEL: get_set_fpenv: +; GFX6-SDAG: ; %bb.0: ; %entry +; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX6-SDAG-NEXT: s_getreg_b32 s4, hwreg(HW_REG_TRAPSTS, 0, 5) +; GFX6-SDAG-NEXT: s_getreg_b32 s5, hwreg(HW_REG_MODE, 0, 23) +; GFX6-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 23), s5 +; GFX6-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_TRAPSTS, 0, 5), s4 +; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX6-ISEL-LABEL: get_set_fpenv: +; GFX6-ISEL: ; %bb.0: ; %entry +; GFX6-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX6-ISEL-NEXT: s_getreg_b32 s4, hwreg(HW_REG_MODE, 0, 23) +; GFX6-ISEL-NEXT: s_getreg_b32 s5, hwreg(HW_REG_TRAPSTS, 0, 5) +; GFX6-ISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 23), s4 +; GFX6-ISEL-NEXT: s_setreg_b32 hwreg(HW_REG_TRAPSTS, 0, 5), s5 +; GFX6-ISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX8-SDAG-LABEL: get_set_fpenv: +; GFX8-SDAG: ; %bb.0: ; %entry +; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-SDAG-NEXT: s_getreg_b32 s4, hwreg(HW_REG_TRAPSTS, 0, 5) +; GFX8-SDAG-NEXT: s_getreg_b32 s5, hwreg(HW_REG_MODE, 0, 23) +; GFX8-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 23), s5 +; GFX8-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_TRAPSTS, 0, 5), s4 +; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX8-ISEL-LABEL: get_set_fpenv: +; GFX8-ISEL: ; %bb.0: ; %entry +; GFX8-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-ISEL-NEXT: s_getreg_b32 s4, hwreg(HW_REG_MODE, 0, 23) +; GFX8-ISEL-NEXT: s_getreg_b32 s5, hwreg(HW_REG_TRAPSTS, 0, 5) +; GFX8-ISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 23), s4 +; GFX8-ISEL-NEXT: s_setreg_b32 hwreg(HW_REG_TRAPSTS, 0, 5), s5 +; GFX8-ISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-SDAG-LABEL: get_set_fpenv: +; GFX9-SDAG: ; %bb.0: ; %entry +; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-SDAG-NEXT: s_getreg_b32 s4, hwreg(HW_REG_TRAPSTS, 0, 5) +; GFX9-SDAG-NEXT: s_getreg_b32 s5, hwreg(HW_REG_MODE, 0, 23) +; GFX9-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 23), s5 +; GFX9-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_TRAPSTS, 0, 5), s4 +; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-ISEL-LABEL: get_set_fpenv: +; GFX9-ISEL: ; %bb.0: ; %entry +; GFX9-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-ISEL-NEXT: s_getreg_b32 s4, hwreg(HW_REG_MODE, 0, 23) +; GFX9-ISEL-NEXT: s_getreg_b32 s5, hwreg(HW_REG_TRAPSTS, 0, 5) +; GFX9-ISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 23), s4 +; GFX9-ISEL-NEXT: s_setreg_b32 hwreg(HW_REG_TRAPSTS, 0, 5), s5 +; GFX9-ISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-SDAG-LABEL: get_set_fpenv: +; GFX10-SDAG: ; %bb.0: ; %entry +; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-SDAG-NEXT: s_getreg_b32 s4, hwreg(HW_REG_TRAPSTS, 0, 5) +; GFX10-SDAG-NEXT: s_getreg_b32 s5, hwreg(HW_REG_MODE, 0, 23) +; GFX10-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 23), s5 +; GFX10-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_TRAPSTS, 0, 5), s4 +; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-ISEL-LABEL: get_set_fpenv: +; GFX10-ISEL: ; %bb.0: ; %entry +; GFX10-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-ISEL-NEXT: s_getreg_b32 s4, hwreg(HW_REG_MODE, 0, 23) +; GFX10-ISEL-NEXT: s_getreg_b32 s5, hwreg(HW_REG_TRAPSTS, 0, 5) +; GFX10-ISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 23), s4 +; GFX10-ISEL-NEXT: s_setreg_b32 hwreg(HW_REG_TRAPSTS, 0, 5), s5 +; GFX10-ISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: get_set_fpenv: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: s_getreg_b32 s0, hwreg(HW_REG_TRAPSTS, 0, 5) +; GFX11-SDAG-NEXT: s_getreg_b32 s1, hwreg(HW_REG_MODE, 0, 23) +; GFX11-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 23), s1 +; GFX11-SDAG-NEXT: s_setreg_b32 hwreg(HW_REG_TRAPSTS, 0, 5), s0 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-ISEL-LABEL: get_set_fpenv: +; GFX11-ISEL: ; %bb.0: ; %entry +; GFX11-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-ISEL-NEXT: s_getreg_b32 s0, hwreg(HW_REG_MODE, 0, 23) +; GFX11-ISEL-NEXT: s_getreg_b32 s1, hwreg(HW_REG_TRAPSTS, 0, 5) +; GFX11-ISEL-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 23), s0 +; GFX11-ISEL-NEXT: s_setreg_b32 hwreg(HW_REG_TRAPSTS, 0, 5), s1 +; GFX11-ISEL-NEXT: s_setpc_b64 s[30:31] +entry: + %0 = call i64 @llvm.get.fpenv.i64() + call void @llvm.set.fpenv.i64(i64 %0) + ret void +} -- GitLab From 6d3bb854713e43daad34ffc84d0132d7fe010abf Mon Sep 17 00:00:00 2001 From: Sandeep Kosuri <66305775+sandeepkosuri@users.noreply.github.com> Date: Wed, 6 Mar 2024 19:46:23 +0530 Subject: [PATCH 296/929] [OpenMP] Parse and Sema support for declare target in local scope (#83223) - adds Parse and Sema support for the `declare target` directive inside a function scope. --- .../clang/Basic/DiagnosticSemaKinds.td | 2 ++ clang/lib/Parse/ParseOpenMP.cpp | 23 ++++++++++++++++++- clang/lib/Sema/SemaOpenMP.cpp | 9 ++++++++ .../test/OpenMP/declare_target_ast_print.cpp | 19 +++++++++++++++ clang/test/OpenMP/declare_target_messages.cpp | 9 ++++++++ 5 files changed, 61 insertions(+), 1 deletion(-) diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index b007ff7d8ccf..5a90e631a894 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -11352,6 +11352,8 @@ def err_omp_device_type_mismatch : Error< def err_omp_wrong_device_function_call : Error< "function with 'device_type(%0)' is not available on %select{device|host}1">; def note_omp_marked_device_type_here : Note<"marked as 'device_type(%0)' here">; +def err_omp_declare_target_has_local_vars : Error< + "local variable '%0' should not be used in 'declare target' directive; ">; def warn_omp_declare_target_after_first_use : Warning< "declaration marked as declare target after first use, it may lead to incorrect results">, InGroup; diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp index bfc31f2653c2..814126e321d3 100644 --- a/clang/lib/Parse/ParseOpenMP.cpp +++ b/clang/lib/Parse/ParseOpenMP.cpp @@ -2984,8 +2984,29 @@ StmtResult Parser::ParseOpenMPDeclarativeOrExecutableDirective( OMPDirectiveScope.Exit(); break; } + case OMPD_declare_target: { + SourceLocation DTLoc = ConsumeAnyToken(); + bool HasClauses = Tok.isNot(tok::annot_pragma_openmp_end); + Sema::DeclareTargetContextInfo DTCI(DKind, DTLoc); + if (HasClauses) + ParseOMPDeclareTargetClauses(DTCI); + bool HasImplicitMappings = + !HasClauses || (DTCI.ExplicitlyMapped.empty() && DTCI.Indirect); + + if (HasImplicitMappings) { + Diag(Tok, diag::err_omp_unexpected_directive) + << 1 << getOpenMPDirectiveName(DKind); + SkipUntil(tok::annot_pragma_openmp_end); + break; + } + + // Skip the last annot_pragma_openmp_end. + ConsumeAnyToken(); + + Actions.ActOnFinishedOpenMPDeclareTargetContext(DTCI); + break; + } case OMPD_declare_simd: - case OMPD_declare_target: case OMPD_begin_declare_target: case OMPD_end_declare_target: case OMPD_requires: diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp index f4364a259ad5..afffd371c58d 100644 --- a/clang/lib/Sema/SemaOpenMP.cpp +++ b/clang/lib/Sema/SemaOpenMP.cpp @@ -23353,6 +23353,15 @@ void Sema::ActOnOpenMPDeclareTargetName(NamedDecl *ND, SourceLocation Loc, isa(ND)) && "Expected variable, function or function template."); + if (auto *VD = dyn_cast(ND)) { + // Only global variables can be marked as declare target. + if (!VD->isFileVarDecl() && !VD->isStaticLocal() && + !VD->isStaticDataMember()) { + Diag(Loc, diag::err_omp_declare_target_has_local_vars) + << VD->getNameAsString(); + return; + } + } // Diagnose marking after use as it may lead to incorrect diagnosis and // codegen. if (LangOpts.OpenMP >= 50 && diff --git a/clang/test/OpenMP/declare_target_ast_print.cpp b/clang/test/OpenMP/declare_target_ast_print.cpp index 40c5dd299abd..43cccf763e97 100644 --- a/clang/test/OpenMP/declare_target_ast_print.cpp +++ b/clang/test/OpenMP/declare_target_ast_print.cpp @@ -360,6 +360,17 @@ int inner_link; // CHECK-NEXT: int inner_link; // CHECK-NEXT: #pragma omp end declare target +void foo2() { return ;} +// CHECK: #pragma omp declare target +// CHECK-NEXT: void foo2() { +// CHECK-NEXT: return; +// CHECK-NEXT: } + +int x; +// CHECK: #pragma omp declare target link +// CHECK-NEXT: int x; +// CHECK-NEXT: #pragma omp end declare target + int main (int argc, char **argv) { foo(); foo_c(); @@ -367,6 +378,14 @@ int main (int argc, char **argv) { test1(); baz(); baz(); + +#if _OPENMP == 202111 +#pragma omp declare target enter(foo2) +#else +#pragma omp declare target to (foo2) +#endif + + #pragma omp declare target link(x) return (0); } diff --git a/clang/test/OpenMP/declare_target_messages.cpp b/clang/test/OpenMP/declare_target_messages.cpp index cf034aca7c91..de831f8575ee 100644 --- a/clang/test/OpenMP/declare_target_messages.cpp +++ b/clang/test/OpenMP/declare_target_messages.cpp @@ -182,11 +182,20 @@ struct S { #pragma omp end declare target }; +void foo3() { + return; +} + +int *y; +int **w = &y; int main (int argc, char **argv) { + int a = 2; #pragma omp declare target // expected-error {{unexpected OpenMP directive '#pragma omp declare target'}} int v; #pragma omp end declare target // expected-error {{unexpected OpenMP directive '#pragma omp end declare target'}} foo(v); +#pragma omp declare target to(foo3) link(w) // omp52-error {{unexpected 'to' clause, use 'enter' instead}} omp52-error {{expected at least one 'enter', 'link' or 'indirect' clause}} +#pragma omp declare target to(a) //omp45-error {{local variable 'a' should not be used in 'declare target' directive}} omp5-error {{local variable 'a' should not be used in 'declare target' directive}} omp51-error {{local variable 'a' should not be used in 'declare target' directive}} omp52-error {{unexpected 'to' clause, use 'enter' instead}} omp52-error {{expected at least one 'enter', 'link' or 'indirect' clause}} return (0); } -- GitLab From f7d354af5714358d7ad83e58a1016c3f385416dd Mon Sep 17 00:00:00 2001 From: yandalur Date: Wed, 6 Mar 2024 19:47:02 +0530 Subject: [PATCH 297/929] [Hexagon] Fix shift value when folding shl DAG node (#83853) When folding (or (shl xx, s), (zext y)) to (COMBINE (shl xx, s-32), y), fix resulting shift value in HexagonISD::COMBINE node to not generate negative values. --------- Co-authored-by: Yashas Andaluri --- .../Target/Hexagon/HexagonISelLowering.cpp | 2 +- .../CodeGen/Hexagon/isel-fold-shl-zext.ll | 37 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 llvm/test/CodeGen/Hexagon/isel-fold-shl-zext.ll diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp index 13691053ddd7..eda1150835a1 100644 --- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp +++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp @@ -3543,7 +3543,7 @@ HexagonTargetLowering::PerformDAGCombine(SDNode *N, unsigned A = Amt->getZExtValue(); SDValue S = Shl.getOperand(0); SDValue T0 = DCI.DAG.getNode(ISD::SHL, dl, ty(S), S, - DCI.DAG.getConstant(32 - A, dl, MVT::i32)); + DCI.DAG.getConstant(A - 32, dl, MVT::i32)); SDValue T1 = DCI.DAG.getZExtOrTrunc(T0, dl, MVT::i32); SDValue T2 = DCI.DAG.getZExtOrTrunc(Z, dl, MVT::i32); return DCI.DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64, {T1, T2}); diff --git a/llvm/test/CodeGen/Hexagon/isel-fold-shl-zext.ll b/llvm/test/CodeGen/Hexagon/isel-fold-shl-zext.ll new file mode 100644 index 000000000000..5fa502310051 --- /dev/null +++ b/llvm/test/CodeGen/Hexagon/isel-fold-shl-zext.ll @@ -0,0 +1,37 @@ +; RUN: llc -mtriple=hexagon-unknown-elf < %s | FileCheck %s + +; In ISelLowering, when folding nodes (or (shl xx, s), (zext y)) +; to (COMBINE (shl xx, s-32), y) where s >= 32, +; check that resulting shift value does not create an undef + + +target datalayout = "e-m:e-p:32:32:32-a:0-n16:32-i64:64:64-i32:32:32-i16:16:16-i1:8:8-f32:32:32-f64:64:64-v32:32:32-v64:64:64-v512:512:512-v1024:1024:1024-v2048:2048:2048" +target triple = "hexagon" + +; Function Attrs: nofree nosync nounwind memory(readwrite, inaccessiblemem: none) +define dso_local void @foo(i64* nocapture noundef %buf, i32 %a, i32 %b) local_unnamed_addr { +; CHECK-LABEL: foo: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: { +; CHECK-NEXT: r[[REG0:[0-9]+]] = addasl(r2,r1,#1) +; CHECK-NEXT: r[[REG2:[0-9]+]] = asl(r1,#1) +; CHECK-NEXT: } +; CHECK-NEXT: { +; CHECK-NEXT: r[[REG1:[0-9]+]] = addasl(r[[REG0]],r1,#1) +; CHECK-NEXT: } +; CHECK-NEXT: { +; CHECK-NEXT: jumpr r31 +; CHECK-NEXT: memd(r0+#8) = r[[REG2]]:[[REG1]] +; CHECK-NEXT: } +entry: + %arrayidx = getelementptr inbounds i64, i64* %buf, i32 1 + %add0 = shl nsw i32 %a, 1 + %add1 = add nsw i32 %add0, %b + %add2 = add nsw i32 %add1, %add0 + %ext0 = zext i32 %add0 to i64 + %shift0 = shl nuw i64 %ext0, 32 + %ext1 = zext i32 %add2 to i64 + %or0 = or i64 %shift0, %ext1 + store i64 %or0, i64* %arrayidx, align 8 + ret void +} -- GitLab From 8fdec5d3badb94807d757c0c58a3900704320919 Mon Sep 17 00:00:00 2001 From: Jakub Kuderski Date: Wed, 6 Mar 2024 09:45:07 -0500 Subject: [PATCH 298/929] [Support][Casting] Add predicates for `isa*` functions (#83753) Expose function objects that call into `llvm::isa` and `llvm::isa_and_present`, such that these type checks can be used as predicates in generic algorithms. Before this change, `llvm::isa*` functions cannot be easily used without knowing both the argument type and the checked types, which leads to them being wrapped in lambdas. For example: ```c++ llvm::all_of(myTypes, [](auto type) { return llvm::isa(type); }); ``` With this PR the example above becomes: ```c++ llvm::all_of(myTypes, llvm::IsaPred); ``` As an alternative solution, I considered redefining `isa*` as function objects, but I decided against doing that because it would create asymmetry with other cast *functions* and could break code that depends on them being actual functions. --- llvm/include/llvm/Support/Casting.h | 46 +++++++++++++++++++++++++++++ llvm/unittests/Support/Casting.cpp | 15 ++++++++++ 2 files changed, 61 insertions(+) diff --git a/llvm/include/llvm/Support/Casting.h b/llvm/include/llvm/Support/Casting.h index 2391c1a531a5..14a32ccd0e0d 100644 --- a/llvm/include/llvm/Support/Casting.h +++ b/llvm/include/llvm/Support/Casting.h @@ -801,6 +801,52 @@ template return unique_dyn_cast_or_null(Val); } +//===----------------------------------------------------------------------===// +// Isa Predicates +//===----------------------------------------------------------------------===// + +/// These are wrappers over isa* function that allow them to be used in generic +/// algorithms such as `llvm:all_of`, `llvm::none_of`, etc. This is accomplished +/// by exposing the isa* functions through function objects with a generic +/// function call operator. + +namespace detail { +template struct IsaCheckPredicate { + template [[nodiscard]] bool operator()(const T &Val) const { + return isa(Val); + } +}; + +template struct IsaAndPresentCheckPredicate { + template [[nodiscard]] bool operator()(const T &Val) const { + return isa_and_present(Val); + } +}; +} // namespace detail + +/// Function object wrapper for the `llvm::isa` type check. The function call +/// operator returns true when the value can be cast to any type in `Types`. +/// Example: +/// ``` +/// SmallVector myTypes = ...; +/// if (llvm::all_of(myTypes, llvm::IsaPred)) +/// ... +/// ``` +template +inline constexpr detail::IsaCheckPredicate IsaPred{}; + +/// Function object wrapper for the `llvm::isa_and_present` type check. The +/// function call operator returns true when the value can be cast to any type +/// in `Types`, or if the value is not present (e.g., nullptr). Example: +/// ``` +/// SmallVector myTypes = ...; +/// if (llvm::all_of(myTypes, llvm::IsaAndPresentPred)) +/// ... +/// ``` +template +inline constexpr detail::IsaAndPresentCheckPredicate + IsaAndPresentPred{}; + } // end namespace llvm #endif // LLVM_SUPPORT_CASTING_H diff --git a/llvm/unittests/Support/Casting.cpp b/llvm/unittests/Support/Casting.cpp index a92565481459..a128cedaf398 100644 --- a/llvm/unittests/Support/Casting.cpp +++ b/llvm/unittests/Support/Casting.cpp @@ -282,6 +282,21 @@ TEST(CastingTest, dyn_cast_if_present) { EXPECT_FALSE(t4.hasValue); } +TEST(CastingTest, isa_check_predicates) { + auto IsaFoo = IsaPred; + EXPECT_TRUE(IsaFoo(B1)); + EXPECT_TRUE(IsaFoo(B2)); + EXPECT_TRUE(IsaFoo(B3)); + EXPECT_TRUE(IsaPred(B4)); + EXPECT_TRUE((IsaPred(B4))); + + auto IsaAndPresentFoo = IsaAndPresentPred; + EXPECT_TRUE(IsaAndPresentFoo(B2)); + EXPECT_TRUE(IsaAndPresentFoo(B4)); + EXPECT_FALSE(IsaAndPresentPred(fub())); + EXPECT_FALSE((IsaAndPresentPred(fub()))); +} + std::unique_ptr newd() { return std::make_unique(); } std::unique_ptr newb() { return std::make_unique(); } -- GitLab From 4490003a22658dcd12527029b2c8682b63d8a9d6 Mon Sep 17 00:00:00 2001 From: Emma Pilkington Date: Wed, 6 Mar 2024 09:51:48 -0500 Subject: [PATCH 299/929] [AMDGPU] Rename COV module flag to amdhsa_code_object_version (#79905) The previous name 'amdgpu_code_object_version', was misleading since this is really a property of the HSA OS. The new spelling also matches the asm directive I added in bc82cfb. --- clang/lib/CodeGen/CodeGenModule.cpp | 4 ++-- .../CodeGenCUDA/amdgpu-code-object-version-linking.cu | 6 +++--- clang/test/CodeGenCUDA/amdgpu-code-object-version.cu | 8 ++++---- clang/test/CodeGenHIP/default-attributes.hip | 4 ++-- clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel.cl | 4 ++-- lld/test/ELF/lto/amdgcn-oses.ll | 2 +- llvm/lib/IR/AutoUpgrade.cpp | 9 +++++++++ llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 2 +- llvm/test/Bitcode/upgrade-module-flag.ll | 6 ++++-- .../CodeGen/AMDGPU/GlobalISel/crash-stack-address-O0.ll | 2 +- .../AMDGPU/GlobalISel/dropped_debug_info_assert.ll | 2 +- .../implicit-kernarg-backend-usage-global-isel.ll | 2 +- .../AMDGPU/GlobalISel/irtranslator-assert-align.ll | 2 +- .../CodeGen/AMDGPU/GlobalISel/irtranslator-atomicrmw.ll | 2 +- .../GlobalISel/irtranslator-call-abi-attribute-hints.ll | 2 +- .../AMDGPU/GlobalISel/irtranslator-call-implicit-args.ll | 2 +- .../AMDGPU/GlobalISel/irtranslator-call-return-values.ll | 2 +- .../CodeGen/AMDGPU/GlobalISel/irtranslator-call-sret.ll | 2 +- llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll | 2 +- .../GlobalISel/irtranslator-constant-fold-vector-op.ll | 2 +- .../AMDGPU/GlobalISel/irtranslator-function-args.ll | 2 +- .../AMDGPU/GlobalISel/irtranslator-indirect-call.ll | 2 +- .../CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll | 2 +- .../AMDGPU/GlobalISel/irtranslator-sibling-call.ll | 2 +- .../CodeGen/AMDGPU/GlobalISel/irtranslator-tail-call.ll | 2 +- .../AMDGPU/GlobalISel/llvm.amdgcn.dispatch.ptr.ll | 2 +- .../CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.is.private.ll | 2 +- .../CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.is.shared.ll | 2 +- .../AMDGPU/GlobalISel/llvm.amdgcn.kernarg.segment.ptr.ll | 2 +- .../CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.queue.ptr.ll | 2 +- .../AMDGPU/GlobalISel/llvm.amdgcn.workgroup.id.ll | 2 +- .../CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workitem.id.ll | 2 +- llvm/test/CodeGen/AMDGPU/GlobalISel/non-entry-alloca.ll | 2 +- .../AMDGPU/abi-attribute-hints-undefined-behavior.ll | 2 +- llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll | 6 +++--- llvm/test/CodeGen/AMDGPU/addrspacecast.gfx6.ll | 2 +- llvm/test/CodeGen/AMDGPU/addrspacecast.ll | 2 +- .../AMDGPU/amdgpu-simplify-libcall-pow-codegen.ll | 2 +- llvm/test/CodeGen/AMDGPU/amdgpu.private-memory.ll | 2 +- .../CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll | 6 +++--- llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll | 6 +++--- .../CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll | 2 +- llvm/test/CodeGen/AMDGPU/attributor-noopt.ll | 2 +- .../blender-no-live-segment-at-def-implicit-def.ll | 2 +- .../CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll | 2 +- .../CodeGen/AMDGPU/call-alias-register-usage-agpr.ll | 2 +- llvm/test/CodeGen/AMDGPU/call-alias-register-usage0.ll | 2 +- llvm/test/CodeGen/AMDGPU/call-alias-register-usage1.ll | 2 +- llvm/test/CodeGen/AMDGPU/call-alias-register-usage2.ll | 2 +- llvm/test/CodeGen/AMDGPU/call-alias-register-usage3.ll | 2 +- llvm/test/CodeGen/AMDGPU/call-args-inreg.ll | 2 +- llvm/test/CodeGen/AMDGPU/call-argument-types.ll | 2 +- llvm/test/CodeGen/AMDGPU/call-graph-register-usage.ll | 2 +- llvm/test/CodeGen/AMDGPU/call-waitcnt.ll | 2 +- .../AMDGPU/callee-special-input-sgprs-fixed-abi.ll | 2 +- llvm/test/CodeGen/AMDGPU/callee-special-input-vgprs.ll | 2 +- llvm/test/CodeGen/AMDGPU/cc-update.ll | 2 +- llvm/test/CodeGen/AMDGPU/cf-loop-on-constant.ll | 2 +- llvm/test/CodeGen/AMDGPU/codegen-internal-only-func.ll | 2 +- llvm/test/CodeGen/AMDGPU/collapse-endcf.ll | 2 +- llvm/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll | 2 +- .../CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll | 2 +- llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll | 2 +- llvm/test/CodeGen/AMDGPU/dagcombine-lshr-and-cmp.ll | 2 +- llvm/test/CodeGen/AMDGPU/ds_read2.ll | 2 +- .../CodeGen/AMDGPU/dwarf-multi-register-use-crash.ll | 2 +- llvm/test/CodeGen/AMDGPU/elf-notes.ll | 2 +- .../CodeGen/AMDGPU/enable-scratch-only-dynamic-stack.ll | 2 +- .../CodeGen/AMDGPU/flat-for-global-subtarget-feature.ll | 2 +- llvm/test/CodeGen/AMDGPU/flat-scratch-init.ll | 2 +- llvm/test/CodeGen/AMDGPU/flat-scratch-reg.ll | 2 +- llvm/test/CodeGen/AMDGPU/fneg-fabs.ll | 2 +- llvm/test/CodeGen/AMDGPU/gfx11-user-sgpr-init16-bug.ll | 2 +- llvm/test/CodeGen/AMDGPU/gfx902-without-xnack.ll | 2 +- llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll | 2 +- llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll | 2 +- llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll | 2 +- llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll | 2 +- llvm/test/CodeGen/AMDGPU/hsa-default-device.ll | 2 +- llvm/test/CodeGen/AMDGPU/hsa-fp-mode.ll | 2 +- llvm/test/CodeGen/AMDGPU/hsa-func.ll | 2 +- llvm/test/CodeGen/AMDGPU/hsa-generic-target-features.ll | 2 +- .../AMDGPU/hsa-metadata-dynlds-func-hidden-args-v5.ll | 2 +- .../AMDGPU/hsa-metadata-dynlds-funcarg-hidden-args-v5.ll | 2 +- .../AMDGPU/hsa-metadata-dynlds-kernarg-hidden-args-v5.ll | 2 +- llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel-.ll | 2 +- .../AMDGPU/hsa-metadata-from-llvm-ctor-dtor-list.ll | 2 +- .../CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full.ll | 2 +- llvm/test/CodeGen/AMDGPU/hsa-metadata-heap-v5.ll | 2 +- llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args-v4.ll | 2 +- llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args-v5.ll | 2 +- .../CodeGen/AMDGPU/hsa-metadata-hostcall-present-asan.ll | 2 +- llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-v4.ll | 2 +- llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-v5.ll | 2 +- llvm/test/CodeGen/AMDGPU/hsa-metadata-images.ll | 2 +- .../CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-1.ll | 2 +- .../CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-3.ll | 2 +- .../CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll | 2 +- .../CodeGen/AMDGPU/hsa-metadata-multigrid-sync-arg-v5.ll | 2 +- llvm/test/CodeGen/AMDGPU/hsa-metadata-queue-ptr-v5.ll | 2 +- llvm/test/CodeGen/AMDGPU/hsa-metadata-queueptr-v5.ll | 2 +- .../hsa-metadata-resource-usage-function-ordering.ll | 2 +- .../AMDGPU/hsa-metadata-uniform-workgroup-size-v5.ll | 2 +- .../AMDGPU/hsa-metadata-workgroup-processor-mode-v5.ll | 2 +- llvm/test/CodeGen/AMDGPU/hsa-note-no-func.ll | 2 +- llvm/test/CodeGen/AMDGPU/hsa.ll | 2 +- llvm/test/CodeGen/AMDGPU/implicit-arg-v5-opt.ll | 2 +- .../CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll | 2 +- .../CodeGen/AMDGPU/implicit-kernel-argument-alignment.ll | 2 +- .../test/CodeGen/AMDGPU/implicitarg-offset-attributes.ll | 8 ++++---- llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll | 2 +- llvm/test/CodeGen/AMDGPU/indirect-call.ll | 2 +- llvm/test/CodeGen/AMDGPU/insert-delay-alu-bug.ll | 2 +- llvm/test/CodeGen/AMDGPU/kernarg-size.ll | 2 +- llvm/test/CodeGen/AMDGPU/kernel-argument-dag-lowering.ll | 2 +- .../AMDGPU/kernel-vgpr-spill-mubuf-with-voffset.ll | 2 +- llvm/test/CodeGen/AMDGPU/large-alloca-compute.ll | 2 +- llvm/test/CodeGen/AMDGPU/lds-alignment.ll | 2 +- llvm/test/CodeGen/AMDGPU/lds-frame-extern.ll | 2 +- llvm/test/CodeGen/AMDGPU/lds-global-non-entry-func.ll | 2 +- llvm/test/CodeGen/AMDGPU/lds-size.ll | 2 +- llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dispatch.id.ll | 2 +- llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dispatch.ptr.ll | 2 +- llvm/test/CodeGen/AMDGPU/llvm.amdgcn.implicitarg.ptr.ll | 2 +- llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll | 2 +- llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll | 2 +- .../CodeGen/AMDGPU/llvm.amdgcn.kernarg.segment.ptr.ll | 2 +- llvm/test/CodeGen/AMDGPU/llvm.amdgcn.queue.ptr.ll | 2 +- llvm/test/CodeGen/AMDGPU/llvm.amdgcn.workgroup.id.ll | 2 +- llvm/test/CodeGen/AMDGPU/llvm.amdgcn.workitem.id.ll | 2 +- llvm/test/CodeGen/AMDGPU/llvm.dbg.value.ll | 2 +- llvm/test/CodeGen/AMDGPU/lower-kernargs.ll | 6 +++--- llvm/test/CodeGen/AMDGPU/lower-module-lds-via-hybrid.ll | 4 ++-- llvm/test/CodeGen/AMDGPU/lower-module-lds-via-table.ll | 2 +- .../machine-sink-temporal-divergence-swdev407790.ll | 2 +- llvm/test/CodeGen/AMDGPU/module-lds-false-sharing.ll | 2 +- llvm/test/CodeGen/AMDGPU/need-fp-from-vgpr-spills.ll | 2 +- llvm/test/CodeGen/AMDGPU/no-hsa-graphics-shaders.ll | 2 +- llvm/test/CodeGen/AMDGPU/non-entry-alloca.ll | 2 +- llvm/test/CodeGen/AMDGPU/nop-data.ll | 2 +- llvm/test/CodeGen/AMDGPU/partial-sgpr-to-vgpr-spills.ll | 2 +- llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll | 2 +- llvm/test/CodeGen/AMDGPU/promote-alloca-calling-conv.ll | 2 +- llvm/test/CodeGen/AMDGPU/promote-alloca-no-opts.ll | 2 +- .../AMDGPU/promote-alloca-padding-size-estimate.ll | 2 +- llvm/test/CodeGen/AMDGPU/recursion.ll | 2 +- .../test/CodeGen/AMDGPU/remove-no-kernel-id-attribute.ll | 2 +- llvm/test/CodeGen/AMDGPU/reqd-work-group-size.ll | 2 +- .../test/CodeGen/AMDGPU/resource-optimization-remarks.ll | 2 +- llvm/test/CodeGen/AMDGPU/resource-usage-dead-function.ll | 2 +- llvm/test/CodeGen/AMDGPU/sgpr-spill-no-vgprs.ll | 2 +- .../AMDGPU/sgpr-spill-update-only-slot-indexes.ll | 2 +- llvm/test/CodeGen/AMDGPU/simple-indirect-call.ll | 2 +- llvm/test/CodeGen/AMDGPU/sopk-no-literal.ll | 2 +- llvm/test/CodeGen/AMDGPU/spill-m0.ll | 2 +- llvm/test/CodeGen/AMDGPU/stack-realign-kernel.ll | 2 +- llvm/test/CodeGen/AMDGPU/stacksave_stackrestore.ll | 2 +- llvm/test/CodeGen/AMDGPU/tid-kd-xnack-any.ll | 2 +- llvm/test/CodeGen/AMDGPU/tid-kd-xnack-off.ll | 2 +- llvm/test/CodeGen/AMDGPU/tid-kd-xnack-on.ll | 2 +- llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-any.ll | 2 +- .../AMDGPU/tid-mul-func-xnack-all-not-supported.ll | 2 +- llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-off.ll | 2 +- llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-on.ll | 2 +- llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-1.ll | 2 +- llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-2.ll | 2 +- llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-1.ll | 2 +- llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-2.ll | 2 +- .../AMDGPU/tid-mul-func-xnack-invalid-any-off-on.ll | 2 +- llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-any.ll | 2 +- .../CodeGen/AMDGPU/tid-one-func-xnack-not-supported.ll | 2 +- llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-off.ll | 2 +- llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-on.ll | 2 +- llvm/test/CodeGen/AMDGPU/trap-abis.ll | 2 +- llvm/test/CodeGen/AMDGPU/trap.ll | 2 +- llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll | 2 +- .../CodeGen/AMDGPU/unstructured-cfg-def-use-issue.ll | 2 +- .../CodeGen/AMDGPU/unsupported-code-object-version.ll | 2 +- .../CodeGen/AMDGPU/vgpr-spill-placement-issue61083.ll | 2 +- llvm/test/CodeGen/AMDGPU/vgpr_constant_to_sgpr.ll | 2 +- llvm/test/CodeGen/AMDGPU/wwm-reserved.ll | 2 +- llvm/test/tools/llvm-reduce/reduce-module-flags.ll | 2 +- 182 files changed, 213 insertions(+), 202 deletions(-) diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index 82a97ecfaa00..d02875c6a86d 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -872,12 +872,12 @@ void CodeGenModule::Release() { EmitMainVoidAlias(); if (getTriple().isAMDGPU()) { - // Emit amdgpu_code_object_version module flag, which is code object version + // Emit amdhsa_code_object_version module flag, which is code object version // times 100. if (getTarget().getTargetOpts().CodeObjectVersion != llvm::CodeObjectVersionKind::COV_None) { getModule().addModuleFlag(llvm::Module::Error, - "amdgpu_code_object_version", + "amdhsa_code_object_version", getTarget().getTargetOpts().CodeObjectVersion); } diff --git a/clang/test/CodeGenCUDA/amdgpu-code-object-version-linking.cu b/clang/test/CodeGenCUDA/amdgpu-code-object-version-linking.cu index d33acdf7eb8b..cb467886c016 100644 --- a/clang/test/CodeGenCUDA/amdgpu-code-object-version-linking.cu +++ b/clang/test/CodeGenCUDA/amdgpu-code-object-version-linking.cu @@ -52,7 +52,7 @@ // LINKED4: [[GEP_4_Z:%.*]] = getelementptr i8, ptr addrspace(4) %{{.*}}, i32 8 // LINKED4: select i1 false, ptr addrspace(4) [[GEP_5_Z]], ptr addrspace(4) [[GEP_4_Z]] // LINKED4: load i16, ptr addrspace(4) %{{.*}}, align 2, !range [[$WS_RANGE:![0-9]*]], !invariant.load{{.*}}, !noundef -// LINKED4: "amdgpu_code_object_version", i32 400 +// LINKED4: "amdhsa_code_object_version", i32 400 // LINKED5: __oclc_ABI_version = weak_odr hidden local_unnamed_addr addrspace(4) constant i32 500 // LINKED5-LABEL: bar @@ -82,7 +82,7 @@ // LINKED5: [[GEP_4_Z:%.*]] = getelementptr i8, ptr addrspace(4) %{{.*}}, i32 8 // LINKED5: select i1 true, ptr addrspace(4) [[GEP_5_Z]], ptr addrspace(4) [[GEP_4_Z]] // LINKED5: load i16, ptr addrspace(4) %{{.*}}, align 2, !range [[$WS_RANGE:![0-9]*]], !invariant.load{{.*}}, !noundef -// LINKED5: "amdgpu_code_object_version", i32 500 +// LINKED5: "amdhsa_code_object_version", i32 500 // LINKED6: __oclc_ABI_version = weak_odr hidden local_unnamed_addr addrspace(4) constant i32 600 // LINKED6-LABEL: bar @@ -112,7 +112,7 @@ // LINKED6: [[GEP_4_Z:%.*]] = getelementptr i8, ptr addrspace(4) %{{.*}}, i32 8 // LINKED6: select i1 true, ptr addrspace(4) [[GEP_5_Z]], ptr addrspace(4) [[GEP_4_Z]] // LINKED6: load i16, ptr addrspace(4) %{{.*}}, align 2, !range [[$WS_RANGE:![0-9]*]], !invariant.load{{.*}}, !noundef -// LINKED6: "amdgpu_code_object_version", i32 600 +// LINKED6: "amdhsa_code_object_version", i32 600 #ifdef DEVICELIB __device__ void bar(int *x, int *y, int *z) diff --git a/clang/test/CodeGenCUDA/amdgpu-code-object-version.cu b/clang/test/CodeGenCUDA/amdgpu-code-object-version.cu index d3450a105df3..ffe12544917f 100644 --- a/clang/test/CodeGenCUDA/amdgpu-code-object-version.cu +++ b/clang/test/CodeGenCUDA/amdgpu-code-object-version.cu @@ -18,8 +18,8 @@ // RUN: not %clang_cc1 -fcuda-is-device -triple amdgcn-amd-amdhsa -emit-llvm \ // RUN: -mcode-object-version=4.1 -o - %s 2>&1| FileCheck %s -check-prefix=INV -// V4: !{{.*}} = !{i32 1, !"amdgpu_code_object_version", i32 400} -// V5: !{{.*}} = !{i32 1, !"amdgpu_code_object_version", i32 500} -// V6: !{{.*}} = !{i32 1, !"amdgpu_code_object_version", i32 600} -// NONE-NOT: !{{.*}} = !{i32 1, !"amdgpu_code_object_version", +// V4: !{{.*}} = !{i32 1, !"amdhsa_code_object_version", i32 400} +// V5: !{{.*}} = !{i32 1, !"amdhsa_code_object_version", i32 500} +// V6: !{{.*}} = !{i32 1, !"amdhsa_code_object_version", i32 600} +// NONE-NOT: !{{.*}} = !{i32 1, !"amdhsa_code_object_version", // INV: error: invalid value '4.1' in '-mcode-object-version=4.1' diff --git a/clang/test/CodeGenHIP/default-attributes.hip b/clang/test/CodeGenHIP/default-attributes.hip index 9c9ea521271b..63572bfd242b 100644 --- a/clang/test/CodeGenHIP/default-attributes.hip +++ b/clang/test/CodeGenHIP/default-attributes.hip @@ -46,11 +46,11 @@ __global__ void kernel() { // OPT: attributes #0 = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) "no-trapping-math"="true" "stack-protector-buffer-size"="8" } // OPT: attributes #1 = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) "amdgpu-flat-work-group-size"="1,1024" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "uniform-work-group-size"="true" } //. -// OPTNONE: !0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +// OPTNONE: !0 = !{i32 1, !"amdhsa_code_object_version", i32 500} // OPTNONE: !1 = !{i32 1, !"amdgpu_printf_kind", !"hostcall"} // OPTNONE: !2 = !{i32 1, !"wchar_size", i32 4} //. -// OPT: !0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +// OPT: !0 = !{i32 1, !"amdhsa_code_object_version", i32 500} // OPT: !1 = !{i32 1, !"amdgpu_printf_kind", !"hostcall"} // OPT: !2 = !{i32 1, !"wchar_size", i32 4} //. diff --git a/clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel.cl b/clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel.cl index 2cf1286e2b54..a5c9f69bc2de 100644 --- a/clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel.cl +++ b/clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel.cl @@ -703,7 +703,7 @@ kernel void test_target_features_kernel(global int *i) { // GFX900: attributes #8 = { nounwind } // GFX900: attributes #9 = { convergent nounwind } //. -// NOCPU: !0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +// NOCPU: !0 = !{i32 1, !"amdhsa_code_object_version", i32 500} // NOCPU: !1 = !{i32 1, !"wchar_size", i32 4} // NOCPU: !2 = !{i32 2, i32 0} // NOCPU: !3 = !{i32 1, i32 0, i32 1, i32 0} @@ -721,7 +721,7 @@ kernel void test_target_features_kernel(global int *i) { // NOCPU: !15 = !{i32 1} // NOCPU: !16 = !{!"int*"} //. -// GFX900: !0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +// GFX900: !0 = !{i32 1, !"amdhsa_code_object_version", i32 500} // GFX900: !1 = !{i32 1, !"wchar_size", i32 4} // GFX900: !2 = !{i32 2, i32 0} // GFX900: !3 = !{!4, !4, i64 0} diff --git a/lld/test/ELF/lto/amdgcn-oses.ll b/lld/test/ELF/lto/amdgcn-oses.ll index 0fd0ce4b9477..7a74d0317f2b 100644 --- a/lld/test/ELF/lto/amdgcn-oses.ll +++ b/lld/test/ELF/lto/amdgcn-oses.ll @@ -28,7 +28,7 @@ target triple = "amdgcn-amd-amdhsa" target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5" !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} define void @_start() { ret void diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp index 3b784b063974..25992395e471 100644 --- a/llvm/lib/IR/AutoUpgrade.cpp +++ b/llvm/lib/IR/AutoUpgrade.cpp @@ -5072,6 +5072,15 @@ bool llvm::UpgradeModuleFlags(Module &M) { Changed = true; } } + + if (ID->getString() == "amdgpu_code_object_version") { + Metadata *Ops[3] = { + Op->getOperand(0), + MDString::get(M.getContext(), "amdhsa_code_object_version"), + Op->getOperand(2)}; + ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops)); + Changed = true; + } } // "Objective-C Class Properties" is recently added for Objective-C. We diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index 963dc2882fcc..0eab7acc9ebc 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -164,7 +164,7 @@ bool isHsaAbi(const MCSubtargetInfo &STI) { unsigned getAMDHSACodeObjectVersion(const Module &M) { if (auto Ver = mdconst::extract_or_null( - M.getModuleFlag("amdgpu_code_object_version"))) { + M.getModuleFlag("amdhsa_code_object_version"))) { return (unsigned)Ver->getZExtValue() / 100; } diff --git a/llvm/test/Bitcode/upgrade-module-flag.ll b/llvm/test/Bitcode/upgrade-module-flag.ll index 1004fd88d185..6a523be93fb1 100644 --- a/llvm/test/Bitcode/upgrade-module-flag.ll +++ b/llvm/test/Bitcode/upgrade-module-flag.ll @@ -1,15 +1,17 @@ ; RUN: llvm-as < %s | llvm-dis | FileCheck %s ; RUN: verify-uselistorder < %s -!llvm.module.flags = !{!0, !1, !2, !3} +!llvm.module.flags = !{!0, !1, !2, !3, !4} !0 = !{i32 1, !"PIC Level", i32 1} !1 = !{i32 1, !"PIE Level", i32 1} !2 = !{i32 1, !"Objective-C Image Info Version", i32 0} !3 = !{i32 1, !"Objective-C Image Info Section", !"__DATA, __objc_imageinfo, regular, no_dead_strip"} +!4 = !{i32 1, !"amdgpu_code_object_version", i32 500} ; CHECK: !0 = !{i32 8, !"PIC Level", i32 1} ; CHECK: !1 = !{i32 7, !"PIE Level", i32 1} ; CHECK: !2 = !{i32 1, !"Objective-C Image Info Version", i32 0} ; CHECK: !3 = !{i32 1, !"Objective-C Image Info Section", !"__DATA,__objc_imageinfo,regular,no_dead_strip"} -; CHECK: !4 = !{i32 4, !"Objective-C Class Properties", i32 0} +; CHECK: !4 = !{i32 1, !"amdhsa_code_object_version", i32 500} +; CHECK: !5 = !{i32 4, !"Objective-C Class Properties", i32 0} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/crash-stack-address-O0.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/crash-stack-address-O0.ll index 0d793654f7ea..48916d8d9b2c 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/crash-stack-address-O0.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/crash-stack-address-O0.ll @@ -25,4 +25,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/dropped_debug_info_assert.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/dropped_debug_info_assert.ll index 44c4910bac7e..56bd7ddde6f5 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/dropped_debug_info_assert.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/dropped_debug_info_assert.ll @@ -89,4 +89,4 @@ attributes #0 = { nofree nosync nounwind readnone willreturn } !7 = distinct !DISubprogram(name: "call_debug_loc", scope: !1, file: !1, line: 8, type: !8, scopeLine: 9, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !9) !8 = !DISubroutineType(types: !9) !9 = !{} -!10 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!10 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/implicit-kernarg-backend-usage-global-isel.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/implicit-kernarg-backend-usage-global-isel.ll index 03374e62e7e9..8859ac69923a 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/implicit-kernarg-backend-usage-global-isel.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/implicit-kernarg-backend-usage-global-isel.ll @@ -370,4 +370,4 @@ declare void @llvm.trap() declare void @llvm.debugtrap() !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-assert-align.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-assert-align.ll index 0576d9781e3d..3150f8cac128 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-assert-align.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-assert-align.ll @@ -211,4 +211,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-atomicrmw.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-atomicrmw.ll index 1e1c632ee96f..fa49b26847e5 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-atomicrmw.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-atomicrmw.ll @@ -50,4 +50,4 @@ define float @test_atomicrmw_fsub(ptr addrspace(3) %addr) { } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-abi-attribute-hints.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-abi-attribute-hints.ll index ca889de30c65..ca33eae14881 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-abi-attribute-hints.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-abi-attribute-hints.ll @@ -226,4 +226,4 @@ define void @func_call_no_other_sgprs() { } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-implicit-args.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-implicit-args.ll index 213598b2e812..a5f59b15c11b 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-implicit-args.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-implicit-args.ll @@ -1228,4 +1228,4 @@ attributes #1 = { nounwind readnone speculatable willreturn } !3 = !{i32 32, i32 2, i32 1} !4 = !{i32 1, i32 32, i32 2} !5 = !{i32 32, i32 1, i32 2} -!6 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!6 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-return-values.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-return-values.ll index 8b0a006e29c0..37f2118572d8 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-return-values.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-return-values.ll @@ -2969,4 +2969,4 @@ attributes #1 = { nounwind readnone } attributes #2 = { nounwind noinline } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-sret.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-sret.ll index ed68d82997f5..854f3463b64d 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-sret.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-sret.ll @@ -90,4 +90,4 @@ define amdgpu_kernel void @test_call_external_void_func_sret_struct_i8_i32_byval } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll index cb0efc19169d..392b0ae6823e 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll @@ -6059,4 +6059,4 @@ attributes #1 = { nounwind readnone } attributes #2 = { nounwind noinline } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-constant-fold-vector-op.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-constant-fold-vector-op.ll index ab407079abc6..ce0e2e40e5d1 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-constant-fold-vector-op.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-constant-fold-vector-op.ll @@ -24,4 +24,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll index 2f0156d67bdf..0b21c2112f05 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll @@ -3236,4 +3236,4 @@ define void @void_func_v2p3_inreg(<2 x ptr addrspace(3)> inreg %arg0) #0 { attributes #0 = { nounwind } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-indirect-call.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-indirect-call.ll index 0fb13bf9c8a2..0c918def3dc5 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-indirect-call.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-indirect-call.ll @@ -74,4 +74,4 @@ define amdgpu_gfx void @test_gfx_indirect_call_sgpr_ptr(ptr %fptr) { } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll index ceff84ea1812..326df0750cfb 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll @@ -333,4 +333,4 @@ define amdgpu_kernel void @asm_constraint_n_n() { !llvm.module.flags = !{!1} !0 = !{i32 70} -!1 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!1 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-sibling-call.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-sibling-call.ll index b45b307f890d..02bf77250151 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-sibling-call.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-sibling-call.ll @@ -1516,4 +1516,4 @@ attributes #0 = { nounwind } attributes #1 = { nounwind noinline } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-tail-call.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-tail-call.ll index 1ead1e443dfe..81d2f36ac874 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-tail-call.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-tail-call.ll @@ -44,4 +44,4 @@ define void @tail_call_void_func_void() { } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.dispatch.ptr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.dispatch.ptr.ll index 26ab4408a5f5..d165fb577efc 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.dispatch.ptr.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.dispatch.ptr.ll @@ -17,4 +17,4 @@ declare noalias ptr addrspace(4) @llvm.amdgcn.dispatch.ptr() #0 attributes #0 = { readnone } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.is.private.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.is.private.ll index a0f54bfee2dc..303dc46e2c88 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.is.private.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.is.private.ll @@ -151,4 +151,4 @@ declare i1 @llvm.amdgcn.is.private(ptr nocapture) #0 attributes #0 = { nounwind readnone speculatable } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.is.shared.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.is.shared.ll index 4b3a47589103..63702d258757 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.is.shared.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.is.shared.ll @@ -151,4 +151,4 @@ declare i1 @llvm.amdgcn.is.shared(ptr nocapture) #0 attributes #0 = { nounwind readnone speculatable } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.kernarg.segment.ptr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.kernarg.segment.ptr.ll index 9d0ede60c7cf..7fc9842824b0 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.kernarg.segment.ptr.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.kernarg.segment.ptr.ll @@ -128,4 +128,4 @@ attributes #2 = { nounwind "amdgpu-implicitarg-num-bytes"="48" } attributes #3 = { nounwind "amdgpu-implicitarg-num-bytes"="38" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.queue.ptr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.queue.ptr.ll index 37578f0ba26e..1eb0c2a87742 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.queue.ptr.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.queue.ptr.ll @@ -17,4 +17,4 @@ declare noalias ptr addrspace(4) @llvm.amdgcn.queue.ptr() #0 attributes #0 = { nounwind readnone } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workgroup.id.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workgroup.id.ll index 4235e1b35d39..df201c1903b6 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workgroup.id.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workgroup.id.ll @@ -104,4 +104,4 @@ attributes #0 = { nounwind readnone } attributes #1 = { nounwind } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workitem.id.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workitem.id.ll index 9698a3894db6..2e62d13f1e69 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workitem.id.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workitem.id.ll @@ -201,4 +201,4 @@ attributes #1 = { nounwind } !2 = !{i32 1, i32 1, i32 64} !llvm.module.flags = !{!99} -!99 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!99 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/non-entry-alloca.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/non-entry-alloca.ll index e3779ecb1da6..b940dc74839b 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/non-entry-alloca.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/non-entry-alloca.ll @@ -267,7 +267,7 @@ declare i32 @llvm.amdgcn.workitem.id.x() #0 attributes #0 = { nounwind readnone speculatable } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: ; ASSUME1024: {{.*}} ; DEFAULTSIZE: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/abi-attribute-hints-undefined-behavior.ll b/llvm/test/CodeGen/AMDGPU/abi-attribute-hints-undefined-behavior.ll index a439c0f51ffe..ae20ab1de3a2 100644 --- a/llvm/test/CodeGen/AMDGPU/abi-attribute-hints-undefined-behavior.ll +++ b/llvm/test/CodeGen/AMDGPU/abi-attribute-hints-undefined-behavior.ll @@ -402,4 +402,4 @@ declare void @llvm.debugtrap() attributes #0 = { "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-queue-ptr" "amdgpu-no-work-group-id-x" "amdgpu-no-work-group-id-y" "amdgpu-no-work-group-id-z" "amdgpu-no-work-item-id-x" "amdgpu-no-work-item-id-y" "amdgpu-no-work-item-id-z" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll b/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll index b356b9af479e..66034af5c351 100644 --- a/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll +++ b/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll @@ -227,7 +227,7 @@ attributes #0 = { argmemonly nounwind } attributes #1 = { nounwind } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} ;. ; AKF_HSA: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nounwind willreturn memory(argmem: readwrite) } ; AKF_HSA: attributes #[[ATTR1]] = { nounwind } @@ -237,7 +237,7 @@ attributes #1 = { nounwind } ; ATTRIBUTOR_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } ; ATTRIBUTOR_HSA: attributes #[[ATTR3]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ;. -; AKF_HSA: [[META0:![0-9]+]] = !{i32 1, !"amdgpu_code_object_version", i32 500} +; AKF_HSA: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500} ;. -; ATTRIBUTOR_HSA: [[META0:![0-9]+]] = !{i32 1, !"amdgpu_code_object_version", i32 500} +; ATTRIBUTOR_HSA: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500} ;. diff --git a/llvm/test/CodeGen/AMDGPU/addrspacecast.gfx6.ll b/llvm/test/CodeGen/AMDGPU/addrspacecast.gfx6.ll index ed574d012bc6..4e0fc580afdd 100644 --- a/llvm/test/CodeGen/AMDGPU/addrspacecast.gfx6.ll +++ b/llvm/test/CodeGen/AMDGPU/addrspacecast.gfx6.ll @@ -208,4 +208,4 @@ define ptr addrspace(6) @addrspacecast_flat_null_to_constant32bit() { attributes #0 = { "amdgpu-32bit-address-high-bits"="0xffff8000" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/addrspacecast.ll b/llvm/test/CodeGen/AMDGPU/addrspacecast.ll index 4b3165f57546..50423c59eabe 100644 --- a/llvm/test/CodeGen/AMDGPU/addrspacecast.ll +++ b/llvm/test/CodeGen/AMDGPU/addrspacecast.ll @@ -424,4 +424,4 @@ attributes #2 = { nounwind readnone } attributes #3 = { nounwind "amdgpu-32bit-address-high-bits"="0xffff8000" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow-codegen.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow-codegen.ll index bdd7ff11fde6..6eb7a4a9a90d 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow-codegen.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow-codegen.ll @@ -775,4 +775,4 @@ define double @test_pown_fast_f64_known_odd(double %x, i32 %y.arg) { } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu.private-memory.ll b/llvm/test/CodeGen/AMDGPU/amdgpu.private-memory.ll index 93c18de8ca62..d6841d40f231 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgpu.private-memory.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgpu.private-memory.ll @@ -530,7 +530,7 @@ attributes #0 = { nounwind "amdgpu-waves-per-eu"="1,2" "amdgpu-flat-work-group-s attributes #1 = { nounwind "amdgpu-flat-work-group-size"="1,256" } !llvm.module.flags = !{!99} -!99 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!99 = !{i32 1, !"amdhsa_code_object_version", i32 400} ; HSAOPT: !1 = !{} ; HSAOPT: !2 = !{i32 0, i32 257} diff --git a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll index 1396dab69c13..af0eb23d8e99 100644 --- a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll +++ b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll @@ -1012,7 +1012,7 @@ attributes #6 = { "enqueued-block" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} ;. ; AKF_HSA: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } ; AKF_HSA: attributes #[[ATTR1]] = { nounwind "target-cpu"="fiji" } @@ -1055,7 +1055,7 @@ attributes #6 = { "enqueued-block" } ; ATTRIBUTOR_HSA: attributes #[[ATTR28]] = { nounwind } ; ATTRIBUTOR_HSA: attributes #[[ATTR29]] = { "enqueued-block" } ;. -; AKF_HSA: [[META0:![0-9]+]] = !{i32 1, !"amdgpu_code_object_version", i32 500} +; AKF_HSA: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500} ;. -; ATTRIBUTOR_HSA: [[META0:![0-9]+]] = !{i32 1, !"amdgpu_code_object_version", i32 500} +; ATTRIBUTOR_HSA: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500} ;. diff --git a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll index e0139fda653f..9a9c28ac632f 100644 --- a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll +++ b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll @@ -635,7 +635,7 @@ attributes #0 = { nounwind readnone speculatable } attributes #1 = { nounwind } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} ;. ; AKF_HSA: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } @@ -657,7 +657,7 @@ attributes #1 = { nounwind } ; ATTRIBUTOR_HSA: attributes #[[ATTR12]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } ; ATTRIBUTOR_HSA: attributes #[[ATTR13]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ;. -; AKF_HSA: [[META0:![0-9]+]] = !{i32 1, !"amdgpu_code_object_version", i32 500} +; AKF_HSA: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500} ;. -; ATTRIBUTOR_HSA: [[META0:![0-9]+]] = !{i32 1, !"amdgpu_code_object_version", i32 500} +; ATTRIBUTOR_HSA: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500} ;. diff --git a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll index f6c1862a2674..fc13b86566f7 100644 --- a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll +++ b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll @@ -130,7 +130,7 @@ define amdgpu_kernel void @min_1024_max_1024() #3 { attributes #3 = {"amdgpu-flat-work-group-size"="1024,1024"} !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} ; HSAMD: amdhsa.kernels ; HSAMD: .max_flat_workgroup_size: 64 diff --git a/llvm/test/CodeGen/AMDGPU/attributor-noopt.ll b/llvm/test/CodeGen/AMDGPU/attributor-noopt.ll index a374689da573..b2f01660201d 100644 --- a/llvm/test/CodeGen/AMDGPU/attributor-noopt.ll +++ b/llvm/test/CodeGen/AMDGPU/attributor-noopt.ll @@ -36,4 +36,4 @@ define amdgpu_kernel void @foo() { } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/blender-no-live-segment-at-def-implicit-def.ll b/llvm/test/CodeGen/AMDGPU/blender-no-live-segment-at-def-implicit-def.ll index 7c8d40c49bb8..6dfc832ff3ac 100644 --- a/llvm/test/CodeGen/AMDGPU/blender-no-live-segment-at-def-implicit-def.ll +++ b/llvm/test/CodeGen/AMDGPU/blender-no-live-segment-at-def-implicit-def.ll @@ -124,4 +124,4 @@ kernel_direct_lighting.exit: ; preds = %if.end294.i.i, %ent declare float @_Z3dotDv3_fS_(<3 x float>) !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll index 24ef8ce1beb2..384715a849c1 100644 --- a/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll +++ b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll @@ -1332,4 +1332,4 @@ declare void @f2(i64) declare i32 @llvm.amdgcn.workitem.id.x() !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage-agpr.ll b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage-agpr.ll index 72bb515ba57e..a795e9956034 100644 --- a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage-agpr.ll +++ b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage-agpr.ll @@ -31,4 +31,4 @@ attributes #1 = { noinline norecurse nounwind readnone willreturn } attributes #2 = { nounwind readnone willreturn } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage0.ll b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage0.ll index 6afc90639dce..c976cc3d53b5 100644 --- a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage0.ll +++ b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage0.ll @@ -26,4 +26,4 @@ attributes #1 = { noinline norecurse nounwind readnone willreturn } attributes #2 = { nounwind readnone willreturn } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage1.ll b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage1.ll index 137bb13330fd..edef71ef143d 100644 --- a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage1.ll +++ b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage1.ll @@ -29,4 +29,4 @@ attributes #1 = { noinline norecurse nounwind readnone willreturn "amdgpu-waves- attributes #2 = { nounwind readnone willreturn } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage2.ll b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage2.ll index 2800ed635bdb..bb34ef1a15d2 100644 --- a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage2.ll +++ b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage2.ll @@ -26,4 +26,4 @@ attributes #1 = { noinline norecurse nounwind readnone willreturn "amdgpu-waves- attributes #2 = { nounwind readnone willreturn } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage3.ll b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage3.ll index f7c0a57f5217..8a88eb7e51ad 100644 --- a/llvm/test/CodeGen/AMDGPU/call-alias-register-usage3.ll +++ b/llvm/test/CodeGen/AMDGPU/call-alias-register-usage3.ll @@ -26,4 +26,4 @@ attributes #1 = { noinline norecurse nounwind readnone willreturn "amdgpu-flat-w attributes #2 = { nounwind readnone willreturn } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/call-args-inreg.ll b/llvm/test/CodeGen/AMDGPU/call-args-inreg.ll index 42e9dce37477..8766303d7ee6 100644 --- a/llvm/test/CodeGen/AMDGPU/call-args-inreg.ll +++ b/llvm/test/CodeGen/AMDGPU/call-args-inreg.ll @@ -1580,4 +1580,4 @@ attributes #0 = { nounwind } attributes #1 = { nounwind "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-queue-ptr" "amdgpu-no-work-group-id-x" "amdgpu-no-work-group-id-y" "amdgpu-no-work-group-id-z" "amdgpu-no-work-item-id-x" "amdgpu-no-work-item-id-y" "amdgpu-no-work-item-id-z" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll index 863bd0d8c752..725c2d71ac5e 100644 --- a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll +++ b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll @@ -7016,4 +7016,4 @@ attributes #1 = { nounwind readnone } attributes #2 = { nounwind noinline } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/call-graph-register-usage.ll b/llvm/test/CodeGen/AMDGPU/call-graph-register-usage.ll index 2e43f685fd70..ed418070ecb5 100644 --- a/llvm/test/CodeGen/AMDGPU/call-graph-register-usage.ll +++ b/llvm/test/CodeGen/AMDGPU/call-graph-register-usage.ll @@ -278,4 +278,4 @@ attributes #1 = { nounwind noinline norecurse } attributes #2 = { nounwind noinline } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/call-waitcnt.ll b/llvm/test/CodeGen/AMDGPU/call-waitcnt.ll index 616e5f00fc1e..60f2dc1ce414 100644 --- a/llvm/test/CodeGen/AMDGPU/call-waitcnt.ll +++ b/llvm/test/CodeGen/AMDGPU/call-waitcnt.ll @@ -152,4 +152,4 @@ declare void @got.func(i32) #0 attributes #0 = { nounwind "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/callee-special-input-sgprs-fixed-abi.ll b/llvm/test/CodeGen/AMDGPU/callee-special-input-sgprs-fixed-abi.ll index e1828042752e..b711542be5a7 100644 --- a/llvm/test/CodeGen/AMDGPU/callee-special-input-sgprs-fixed-abi.ll +++ b/llvm/test/CodeGen/AMDGPU/callee-special-input-sgprs-fixed-abi.ll @@ -582,4 +582,4 @@ attributes #1 = { nounwind noinline } attributes #2 = { nounwind noinline "amdgpu-implicitarg-num-bytes"="0" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/callee-special-input-vgprs.ll b/llvm/test/CodeGen/AMDGPU/callee-special-input-vgprs.ll index 9f535a94e61f..5e6f377da28e 100644 --- a/llvm/test/CodeGen/AMDGPU/callee-special-input-vgprs.ll +++ b/llvm/test/CodeGen/AMDGPU/callee-special-input-vgprs.ll @@ -813,4 +813,4 @@ attributes #1 = { nounwind noinline } attributes #2 = { nounwind "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/cc-update.ll b/llvm/test/CodeGen/AMDGPU/cc-update.ll index 6f42fd0aff13..c674aebabcc8 100644 --- a/llvm/test/CodeGen/AMDGPU/cc-update.ll +++ b/llvm/test/CodeGen/AMDGPU/cc-update.ll @@ -595,4 +595,4 @@ attributes #1 = { nounwind "amdgpu-num-vgpr"="8" } attributes #2 = { nounwind "frame-pointer"="all" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/cf-loop-on-constant.ll b/llvm/test/CodeGen/AMDGPU/cf-loop-on-constant.ll index 56159195a963..3c8ea61b0d43 100644 --- a/llvm/test/CodeGen/AMDGPU/cf-loop-on-constant.ll +++ b/llvm/test/CodeGen/AMDGPU/cf-loop-on-constant.ll @@ -504,4 +504,4 @@ for.body: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/codegen-internal-only-func.ll b/llvm/test/CodeGen/AMDGPU/codegen-internal-only-func.ll index bc8f3ebfd5d2..aa1ad16b2a56 100644 --- a/llvm/test/CodeGen/AMDGPU/codegen-internal-only-func.ll +++ b/llvm/test/CodeGen/AMDGPU/codegen-internal-only-func.ll @@ -27,4 +27,4 @@ define internal i32 @func() { } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll b/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll index 6422beeec088..6bc8d29b3bf7 100644 --- a/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll +++ b/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll @@ -1439,4 +1439,4 @@ attributes #1 = { nounwind convergent } attributes #2 = { nounwind } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll b/llvm/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll index 2b5a8d956034..789150f690d5 100644 --- a/llvm/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll +++ b/llvm/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll @@ -277,4 +277,4 @@ attributes #0 = { nounwind } attributes #1 = { nounwind readnone } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll b/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll index 11871db1ef65..09dc6d6bff9e 100644 --- a/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll +++ b/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll @@ -283,4 +283,4 @@ declare hidden { <4 x i32>, <4 x half> } @func_struct() #0 attributes #0 = { nounwind} !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll index e157c69dff36..028a28ed9a23 100644 --- a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll +++ b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll @@ -3021,4 +3021,4 @@ for.body.i: ; preds = %for.body.i, %entry } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-lshr-and-cmp.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-lshr-and-cmp.ll index ce478d41380a..5cadb65c9c94 100644 --- a/llvm/test/CodeGen/AMDGPU/dagcombine-lshr-and-cmp.ll +++ b/llvm/test/CodeGen/AMDGPU/dagcombine-lshr-and-cmp.ll @@ -96,4 +96,4 @@ out.else: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/ds_read2.ll b/llvm/test/CodeGen/AMDGPU/ds_read2.ll index 1b0ffb97ecb1..777a8f3fef1c 100644 --- a/llvm/test/CodeGen/AMDGPU/ds_read2.ll +++ b/llvm/test/CodeGen/AMDGPU/ds_read2.ll @@ -1542,4 +1542,4 @@ attributes #2 = { convergent nounwind } attributes #3 = { nounwind noinline } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/dwarf-multi-register-use-crash.ll b/llvm/test/CodeGen/AMDGPU/dwarf-multi-register-use-crash.ll index 764c40ebc714..16f16f56248c 100644 --- a/llvm/test/CodeGen/AMDGPU/dwarf-multi-register-use-crash.ll +++ b/llvm/test/CodeGen/AMDGPU/dwarf-multi-register-use-crash.ll @@ -155,4 +155,4 @@ attributes #0 = { nocallback nofree nosync nounwind readnone speculatable willre !41 = !DILocalVariable(name: "dummy", arg: 3, scope: !34, file: !1, line: 49, type: !5) !42 = !DILocalVariable(name: "dummy", arg: 4, scope: !34, file: !1, line: 49, type: !5) !43 = !DILocation(line: 49, column: 9, scope: !34) -!44 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!44 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/elf-notes.ll b/llvm/test/CodeGen/AMDGPU/elf-notes.ll index ef4243413fc3..d958dde01c3f 100644 --- a/llvm/test/CodeGen/AMDGPU/elf-notes.ll +++ b/llvm/test/CodeGen/AMDGPU/elf-notes.ll @@ -85,4 +85,4 @@ define amdgpu_kernel void @elf_notes() { } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/enable-scratch-only-dynamic-stack.ll b/llvm/test/CodeGen/AMDGPU/enable-scratch-only-dynamic-stack.ll index d91c899a27eb..78ac2f9eaff0 100644 --- a/llvm/test/CodeGen/AMDGPU/enable-scratch-only-dynamic-stack.ll +++ b/llvm/test/CodeGen/AMDGPU/enable-scratch-only-dynamic-stack.ll @@ -21,4 +21,4 @@ define amdgpu_kernel void @test_indirect_call() { } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/flat-for-global-subtarget-feature.ll b/llvm/test/CodeGen/AMDGPU/flat-for-global-subtarget-feature.ll index 0b2133755e81..fee6540f43c6 100644 --- a/llvm/test/CodeGen/AMDGPU/flat-for-global-subtarget-feature.ll +++ b/llvm/test/CodeGen/AMDGPU/flat-for-global-subtarget-feature.ll @@ -52,4 +52,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/flat-scratch-init.ll b/llvm/test/CodeGen/AMDGPU/flat-scratch-init.ll index 57991d6cba94..b9583a73295e 100644 --- a/llvm/test/CodeGen/AMDGPU/flat-scratch-init.ll +++ b/llvm/test/CodeGen/AMDGPU/flat-scratch-init.ll @@ -418,4 +418,4 @@ define amdgpu_kernel void @kernel_no_calls_no_stack() { attributes #0 = { nounwind } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/flat-scratch-reg.ll b/llvm/test/CodeGen/AMDGPU/flat-scratch-reg.ll index 268b88f6a487..1633d21c41d5 100644 --- a/llvm/test/CodeGen/AMDGPU/flat-scratch-reg.ll +++ b/llvm/test/CodeGen/AMDGPU/flat-scratch-reg.ll @@ -144,4 +144,4 @@ entry: attributes #0 = { nounwind } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll b/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll index 3be2d9435119..b0c17828cb13 100644 --- a/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll +++ b/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll @@ -111,4 +111,4 @@ declare <2 x float> @llvm.fabs.v2f32(<2 x float>) readnone declare <4 x float> @llvm.fabs.v4f32(<4 x float>) readnone !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/gfx11-user-sgpr-init16-bug.ll b/llvm/test/CodeGen/AMDGPU/gfx11-user-sgpr-init16-bug.ll index 3973cf1eec83..81239e841e09 100644 --- a/llvm/test/CodeGen/AMDGPU/gfx11-user-sgpr-init16-bug.ll +++ b/llvm/test/CodeGen/AMDGPU/gfx11-user-sgpr-init16-bug.ll @@ -194,4 +194,4 @@ declare i64 @llvm.amdgcn.dispatch.id() #0 attributes #0 = { nounwind readnone speculatable willreturn } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/gfx902-without-xnack.ll b/llvm/test/CodeGen/AMDGPU/gfx902-without-xnack.ll index 9ab03da4b979..a83cde14892b 100644 --- a/llvm/test/CodeGen/AMDGPU/gfx902-without-xnack.ll +++ b/llvm/test/CodeGen/AMDGPU/gfx902-without-xnack.ll @@ -7,4 +7,4 @@ define amdgpu_kernel void @test_kernel(ptr addrspace(1) %out0, ptr addrspace(1) } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll index 9c7ce392f09e..96c615b974ce 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll @@ -5413,4 +5413,4 @@ attributes #1 = { strictfp "denormal-fp-math-f32"="preserve-sign,preserve-sign" attributes #2 = { strictfp} !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll index 0cbc8533db98..3cc5a4cd1d0a 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll @@ -3553,4 +3553,4 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_defalut_scop attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll index c9952c2594b6..314c52a71d93 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll @@ -3553,4 +3553,4 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_defalut_scop attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll index 11d35c5a5958..bc9125e326c4 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll @@ -5621,4 +5621,4 @@ attributes #1 = { strictfp "denormal-fp-math-f32"="preserve-sign,preserve-sign" attributes #2 = { strictfp} !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-default-device.ll b/llvm/test/CodeGen/AMDGPU/hsa-default-device.ll index 04c64dce57bc..0f1a784eba19 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-default-device.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-default-device.ll @@ -10,4 +10,4 @@ define amdgpu_kernel void @test_kernel(ptr addrspace(1) %out0, ptr addrspace(1) } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-fp-mode.ll b/llvm/test/CodeGen/AMDGPU/hsa-fp-mode.ll index fc9a8d96be6a..af7b57a9f67b 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-fp-mode.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-fp-mode.ll @@ -101,4 +101,4 @@ attributes #7 = { nounwind "amdgpu-ieee"="false" "target-cpu"="fiji" } attributes #8 = { nounwind "amdgpu-dx10-clamp"="false" "amdgpu-ieee"="false" "target-cpu"="fiji" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-func.ll b/llvm/test/CodeGen/AMDGPU/hsa-func.ll index e92f9160e769..4ef3c994d062 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-func.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-func.ll @@ -69,4 +69,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-generic-target-features.ll b/llvm/test/CodeGen/AMDGPU/hsa-generic-target-features.ll index a2d9bbf575b4..b69fd884ee4a 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-generic-target-features.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-generic-target-features.ll @@ -28,4 +28,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 600} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 600} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-dynlds-func-hidden-args-v5.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-dynlds-func-hidden-args-v5.ll index cb15ff9fcb1b..cd46747370ad 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-dynlds-func-hidden-args-v5.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-dynlds-func-hidden-args-v5.ll @@ -115,7 +115,7 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} !llvm.printf.fmts = !{!1, !2} !1 = !{!"1:1:4:%d\5Cn"} !2 = !{!"2:1:8:%g\5Cn"} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-dynlds-funcarg-hidden-args-v5.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-dynlds-funcarg-hidden-args-v5.ll index 16bfe5f01968..2fe96975bb92 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-dynlds-funcarg-hidden-args-v5.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-dynlds-funcarg-hidden-args-v5.ll @@ -115,7 +115,7 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} !llvm.printf.fmts = !{!1, !2} !1 = !{!"1:1:4:%d\5Cn"} !2 = !{!"2:1:8:%g\5Cn"} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-dynlds-kernarg-hidden-args-v5.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-dynlds-kernarg-hidden-args-v5.ll index d457c61b8d40..b3ed362052bb 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-dynlds-kernarg-hidden-args-v5.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-dynlds-kernarg-hidden-args-v5.ll @@ -116,7 +116,7 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} !llvm.printf.fmts = !{!1, !2} !1 = !{!"1:1:4:%d\5Cn"} !2 = !{!"2:1:8:%g\5Cn"} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel-.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel-.ll index 042abe382283..3d0e061d3328 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel-.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel-.ll @@ -151,7 +151,7 @@ attributes #2 = { optnone noinline "amdgpu-no-completion-action" "amdgpu-implici attributes #3 = { optnone noinline "amdgpu-no-default-queue" "amdgpu-implicitarg-num-bytes"="48" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} !1 = !{i32 0} !2 = !{!"none"} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ctor-dtor-list.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ctor-dtor-list.ll index fb08fd2c4508..daac1105b531 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ctor-dtor-list.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ctor-dtor-list.ll @@ -39,4 +39,4 @@ define internal void @bar.5() { ; PARSER: AMDGPU HSA Metadata Parser Test: PASS !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full.ll index dc3a6e8b633b..28246d7f9e6f 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full.ll @@ -1746,7 +1746,7 @@ attributes #1 = { optnone noinline "amdgpu-no-completion-action" "amdgpu-no-defa attributes #2 = { optnone noinline "amdgpu-implicitarg-num-bytes"="56" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} !llvm.printf.fmts = !{!100, !101} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-heap-v5.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-heap-v5.ll index 8486033818aa..6a49eac134a6 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-heap-v5.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-heap-v5.ll @@ -294,4 +294,4 @@ attributes #1 = { nounwind readnone speculatable willreturn } attributes #2 = { noinline } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args-v4.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args-v4.ll index f4892ebdc9c9..ccdcb523ef0b 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args-v4.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args-v4.ll @@ -298,4 +298,4 @@ attributes #4 = { optnone noinline "amdgpu-implicitarg-num-bytes"="48" } attributes #5 = { optnone noinline "amdgpu-implicitarg-num-bytes"="56" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args-v5.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args-v5.ll index 1a2ce636c733..e10f050b8e7a 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args-v5.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args-v5.ll @@ -115,7 +115,7 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} !llvm.printf.fmts = !{!1, !2} !1 = !{!"1:1:4:%d\5Cn"} !2 = !{!"2:1:8:%g\5Cn"} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present-asan.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present-asan.ll index 22c6e1477622..48988a8aead8 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present-asan.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present-asan.ll @@ -40,7 +40,7 @@ define amdgpu_kernel void @test_kernel(i8 %a) #0 attributes #0 = { sanitize_address "amdgpu-implicitarg-num-bytes"="48" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} !1 = !{i32 0} !2 = !{!"none"} !3 = !{!"char"} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-v4.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-v4.ll index 8f90025fe8e2..6f4c8911efd3 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-v4.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-v4.ll @@ -296,4 +296,4 @@ attributes #3 = { "amdgpu-implicitarg-num-bytes"="48" "amdgpu-no-hostcall-ptr" } attributes #4 = { noinline } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-v5.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-v5.ll index bc9b43716642..01f8fbfd7631 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-v5.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-v5.ll @@ -294,4 +294,4 @@ attributes #1 = { nounwind readnone speculatable willreturn } attributes #2 = { noinline } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-images.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-images.ll index 6d49f22eb429..99a5bccc4ff6 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-images.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-images.ll @@ -99,7 +99,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %a, ; CHECK-NEXT: - 1 !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} !1 = !{!"image1d_t", !"image1d_array_t", !"image1d_buffer_t", !"image2d_t", !"image2d_array_t", !"image2d_array_depth_t", diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-1.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-1.ll index fc5e6e273125..8d7824c56ba1 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-1.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-1.ll @@ -10,4 +10,4 @@ !opencl.ocl.version = !{} !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-3.ll index 1ec79c95bc2a..cf1759fffa99 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-3.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-3.ll @@ -11,4 +11,4 @@ !opencl.ocl.version = !{!0} !llvm.module.flags = !{!1} !0 = !{i32 1} -!1 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!1 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll index 82ee23dca3f6..7a9f4ae8a20f 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll @@ -165,4 +165,4 @@ attributes #1 = { "amdgpu-num-vgpr"="20" } attributes #2 = { "amdgpu-flat-work-group-size"="1,256" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-multigrid-sync-arg-v5.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-multigrid-sync-arg-v5.ll index e6dfee2bbfa8..689619227b8d 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-multigrid-sync-arg-v5.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-multigrid-sync-arg-v5.ll @@ -294,4 +294,4 @@ attributes #1 = { nounwind readnone speculatable willreturn } attributes #2 = { noinline } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-queue-ptr-v5.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-queue-ptr-v5.ll index 62fe65f3161b..9854977c2f30 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-queue-ptr-v5.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-queue-ptr-v5.ll @@ -78,4 +78,4 @@ declare void @llvm.trap() declare void @llvm.debugtrap() !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-queueptr-v5.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-queueptr-v5.ll index 83306b156a94..cf26a427aec3 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-queueptr-v5.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-queueptr-v5.ll @@ -294,4 +294,4 @@ attributes #1 = { nounwind readnone speculatable willreturn } attributes #2 = { noinline } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-resource-usage-function-ordering.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-resource-usage-function-ordering.ll index ac655f9a6bc2..7986368e2a35 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-resource-usage-function-ordering.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-resource-usage-function-ordering.ll @@ -137,4 +137,4 @@ define amdgpu_kernel void @test4() { attributes #0 = { norecurse } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-uniform-workgroup-size-v5.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-uniform-workgroup-size-v5.ll index 328e7bfd4bc9..d1152b8ae7de 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-uniform-workgroup-size-v5.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-uniform-workgroup-size-v5.ll @@ -30,4 +30,4 @@ attributes #0 = { "uniform-work-group-size"="true" } attributes #1 = { "uniform-work-group-size"="false" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-workgroup-processor-mode-v5.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-workgroup-processor-mode-v5.ll index e1b0d9fb3f43..e6c3fe139ffb 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-workgroup-processor-mode-v5.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-workgroup-processor-mode-v5.ll @@ -14,4 +14,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/hsa-note-no-func.ll b/llvm/test/CodeGen/AMDGPU/hsa-note-no-func.ll index 0b93cf29f90c..ea578fc64c69 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa-note-no-func.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa-note-no-func.ll @@ -58,4 +58,4 @@ ; HSA-GFX907: .amdgcn_target "amdgcn-unknown-amdhsa--gfx906" !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/hsa.ll b/llvm/test/CodeGen/AMDGPU/hsa.ll index fa0c06a61c32..de484677bf5e 100644 --- a/llvm/test/CodeGen/AMDGPU/hsa.ll +++ b/llvm/test/CodeGen/AMDGPU/hsa.ll @@ -121,4 +121,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/implicit-arg-v5-opt.ll b/llvm/test/CodeGen/AMDGPU/implicit-arg-v5-opt.ll index 954218f339fa..e37b6ff10ffa 100644 --- a/llvm/test/CodeGen/AMDGPU/implicit-arg-v5-opt.ll +++ b/llvm/test/CodeGen/AMDGPU/implicit-arg-v5-opt.ll @@ -200,4 +200,4 @@ declare i32 @llvm.amdgcn.workgroup.id.z() #1 attributes #0 = { nounwind "uniform-work-group-size"="true" } attributes #1 = { nounwind readnone speculatable } !0 = !{i32 8, i32 16, i32 2} -!1 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!1 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll b/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll index 30fe4a80e693..72f10ea892e5 100644 --- a/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll +++ b/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll @@ -362,4 +362,4 @@ declare void @llvm.trap() declare void @llvm.debugtrap() !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/implicit-kernel-argument-alignment.ll b/llvm/test/CodeGen/AMDGPU/implicit-kernel-argument-alignment.ll index ba0156213ae4..6b2080305e73 100644 --- a/llvm/test/CodeGen/AMDGPU/implicit-kernel-argument-alignment.ll +++ b/llvm/test/CodeGen/AMDGPU/implicit-kernel-argument-alignment.ll @@ -58,4 +58,4 @@ define amdgpu_kernel void @test_aligned_to_eight(i64 %eight) { declare ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr() !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/implicitarg-offset-attributes.ll b/llvm/test/CodeGen/AMDGPU/implicitarg-offset-attributes.ll index a8263a317baa..a5792bf29ddc 100644 --- a/llvm/test/CodeGen/AMDGPU/implicitarg-offset-attributes.ll +++ b/llvm/test/CodeGen/AMDGPU/implicitarg-offset-attributes.ll @@ -253,7 +253,7 @@ define amdgpu_kernel void @test_default_queue_completion_action_offset_v5_0(ptr attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} ;. @@ -278,9 +278,9 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo ; V6: attributes #[[ATTR4]] = { "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } ; V6: attributes #[[ATTR5]] = { "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } ;. -; V4: [[META0:![0-9]+]] = !{i32 1, !"amdgpu_code_object_version", i32 400} +; V4: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 400} ;. -; V5: [[META0:![0-9]+]] = !{i32 1, !"amdgpu_code_object_version", i32 500} +; V5: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500} ;. -; V6: [[META0:![0-9]+]] = !{i32 1, !"amdgpu_code_object_version", i32 600} +; V6: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 600} ;. diff --git a/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll b/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll index fcc5a68f90eb..3cabe41afb05 100644 --- a/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll +++ b/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll @@ -111,4 +111,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/indirect-call.ll b/llvm/test/CodeGen/AMDGPU/indirect-call.ll index 408199bbc922..7799b9509ceb 100644 --- a/llvm/test/CodeGen/AMDGPU/indirect-call.ll +++ b/llvm/test/CodeGen/AMDGPU/indirect-call.ll @@ -1627,4 +1627,4 @@ define void @test_indirect_tail_call_vgpr_ptr(ptr %fptr) { } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/insert-delay-alu-bug.ll b/llvm/test/CodeGen/AMDGPU/insert-delay-alu-bug.ll index 807c19001fd9..cddfb21a6fbd 100644 --- a/llvm/test/CodeGen/AMDGPU/insert-delay-alu-bug.ll +++ b/llvm/test/CodeGen/AMDGPU/insert-delay-alu-bug.ll @@ -248,4 +248,4 @@ bb43: attributes #0 = { noinline optnone } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/kernarg-size.ll b/llvm/test/CodeGen/AMDGPU/kernarg-size.ll index a04fe28dbfff..2370ceff89bd 100644 --- a/llvm/test/CodeGen/AMDGPU/kernarg-size.ll +++ b/llvm/test/CodeGen/AMDGPU/kernarg-size.ll @@ -20,4 +20,4 @@ define amdgpu_kernel void @trap(ptr addrspace(1) nocapture readonly %arg0) { } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/kernel-argument-dag-lowering.ll b/llvm/test/CodeGen/AMDGPU/kernel-argument-dag-lowering.ll index 3015d6887401..1a73df341108 100644 --- a/llvm/test/CodeGen/AMDGPU/kernel-argument-dag-lowering.ll +++ b/llvm/test/CodeGen/AMDGPU/kernel-argument-dag-lowering.ll @@ -280,4 +280,4 @@ define amdgpu_kernel void @byref_constant_i32_arg_offset0(ptr addrspace(4) byref } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/kernel-vgpr-spill-mubuf-with-voffset.ll b/llvm/test/CodeGen/AMDGPU/kernel-vgpr-spill-mubuf-with-voffset.ll index 8b6b48bcdba0..1acbb0911828 100644 --- a/llvm/test/CodeGen/AMDGPU/kernel-vgpr-spill-mubuf-with-voffset.ll +++ b/llvm/test/CodeGen/AMDGPU/kernel-vgpr-spill-mubuf-with-voffset.ll @@ -116,4 +116,4 @@ declare void @device_func(ptr addrspace(5)) attributes #0 = { nounwind "frame-pointer"="all" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/large-alloca-compute.ll b/llvm/test/CodeGen/AMDGPU/large-alloca-compute.ll index bec8492667ea..cb6073e9341e 100644 --- a/llvm/test/CodeGen/AMDGPU/large-alloca-compute.ll +++ b/llvm/test/CodeGen/AMDGPU/large-alloca-compute.ll @@ -70,4 +70,4 @@ define amdgpu_kernel void @large_alloca_compute_shader(i32 %x, i32 %y) #0 { attributes #0 = { nounwind } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/lds-alignment.ll b/llvm/test/CodeGen/AMDGPU/lds-alignment.ll index 2222bfee6c78..8c23ace9b014 100644 --- a/llvm/test/CodeGen/AMDGPU/lds-alignment.ll +++ b/llvm/test/CodeGen/AMDGPU/lds-alignment.ll @@ -216,4 +216,4 @@ attributes #1 = { nounwind } attributes #2 = { convergent nounwind } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/lds-frame-extern.ll b/llvm/test/CodeGen/AMDGPU/lds-frame-extern.ll index 66f31bbf7afe..9619cb73b153 100644 --- a/llvm/test/CodeGen/AMDGPU/lds-frame-extern.ll +++ b/llvm/test/CodeGen/AMDGPU/lds-frame-extern.ll @@ -613,4 +613,4 @@ define amdgpu_kernel void @module_1_kernel_overalign_indirect_extern_overalign(i attributes #0 = { noinline } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/lds-global-non-entry-func.ll b/llvm/test/CodeGen/AMDGPU/lds-global-non-entry-func.ll index d7c42040eabf..433a836e7ca0 100644 --- a/llvm/test/CodeGen/AMDGPU/lds-global-non-entry-func.ll +++ b/llvm/test/CodeGen/AMDGPU/lds-global-non-entry-func.ll @@ -618,4 +618,4 @@ ret: ; GFX9: {{.*}} !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/lds-size.ll b/llvm/test/CodeGen/AMDGPU/lds-size.ll index ebbc645ea91f..1a9d15a00297 100644 --- a/llvm/test/CodeGen/AMDGPU/lds-size.ll +++ b/llvm/test/CodeGen/AMDGPU/lds-size.ll @@ -36,4 +36,4 @@ endif: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dispatch.id.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dispatch.id.ll index 04ac24948b8b..f8a1388c9415 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dispatch.id.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dispatch.id.ll @@ -19,4 +19,4 @@ attributes #0 = { nounwind } attributes #1 = { nounwind readnone } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dispatch.ptr.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dispatch.ptr.ll index 2b1705d974f1..4fe6eed0ef1f 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dispatch.ptr.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dispatch.ptr.ll @@ -33,4 +33,4 @@ declare noalias ptr addrspace(4) @llvm.amdgcn.dispatch.ptr() #0 attributes #0 = { readnone } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.implicitarg.ptr.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.implicitarg.ptr.ll index ebbbe8aaa3a1..70eff4945015 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.implicitarg.ptr.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.implicitarg.ptr.ll @@ -316,4 +316,4 @@ attributes #2 = { nounwind readnone speculatable } attributes #3 = { nounwind noinline "amdgpu-implicitarg-num-bytes"="0" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll index eab6ebd689fa..bc10eb68d75c 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll @@ -55,4 +55,4 @@ declare i1 @llvm.amdgcn.is.private(ptr nocapture) #0 attributes #0 = { nounwind readnone speculatable } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll index 2672c12ecf1f..aad4d924952f 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll @@ -54,4 +54,4 @@ declare i1 @llvm.amdgcn.is.shared(ptr nocapture) #0 attributes #0 = { nounwind readnone speculatable } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.kernarg.segment.ptr.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.kernarg.segment.ptr.ll index 40dafcf09133..8dba22312ac8 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.kernarg.segment.ptr.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.kernarg.segment.ptr.ll @@ -125,4 +125,4 @@ attributes #2 = { nounwind "amdgpu-implicitarg-num-bytes"="48" } attributes #3 = { nounwind "amdgpu-implicitarg-num-bytes"="38" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.queue.ptr.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.queue.ptr.ll index 96bd49a66efd..36d231978871 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.queue.ptr.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.queue.ptr.ll @@ -18,4 +18,4 @@ declare noalias ptr addrspace(4) @llvm.amdgcn.queue.ptr() #0 attributes #0 = { nounwind readnone } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.workgroup.id.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.workgroup.id.ll index 17c076e656cc..ab29ca4a9973 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.workgroup.id.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.workgroup.id.ll @@ -104,4 +104,4 @@ attributes #0 = { nounwind readnone } attributes #1 = { nounwind } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.workitem.id.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.workitem.id.ll index 168777549f1d..a1835ea176d5 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.workitem.id.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.workitem.id.ll @@ -135,4 +135,4 @@ attributes #1 = { nounwind } !0 = !{i32 64, i32 1, i32 1} !1 = !{i32 1, i32 64, i32 1} !2 = !{i32 1, i32 1, i32 64} -!3 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!3 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/llvm.dbg.value.ll b/llvm/test/CodeGen/AMDGPU/llvm.dbg.value.ll index a496e1c3ae6f..6c8fccd54b81 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.dbg.value.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.dbg.value.ll @@ -55,4 +55,4 @@ attributes #1 = { nounwind readnone } !12 = !{i32 2, !"Debug Info Version", i32 3} !13 = !DIExpression() !14 = !DILocation(line: 1, column: 42, scope: !4) -!15 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!15 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/lower-kernargs.ll b/llvm/test/CodeGen/AMDGPU/lower-kernargs.ll index daabcd36915a..7408d3776ae2 100644 --- a/llvm/test/CodeGen/AMDGPU/lower-kernargs.ll +++ b/llvm/test/CodeGen/AMDGPU/lower-kernargs.ll @@ -1696,7 +1696,7 @@ attributes #2 = { nounwind "target-cpu"="tahiti" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} ;. ; HSA: attributes #[[ATTR0:[0-9]+]] = { nounwind "target-cpu"="kaveri" } ; HSA: attributes #[[ATTR1:[0-9]+]] = { nounwind "amdgpu-implicitarg-num-bytes"="40" "target-cpu"="kaveri" } @@ -1708,13 +1708,13 @@ attributes #2 = { nounwind "target-cpu"="tahiti" } ; MESA: attributes #[[ATTR2:[0-9]+]] = { nounwind "target-cpu"="tahiti" } ; MESA: attributes #[[ATTR3:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } ;. -; HSA: [[META0:![0-9]+]] = !{i32 1, !"amdgpu_code_object_version", i32 500} +; HSA: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500} ; HSA: [[META1]] = !{} ; HSA: [[META2]] = !{i64 42} ; HSA: [[META3:![0-9]+]] = !{i64 128} ; HSA: [[META4]] = !{i64 1024} ;. -; MESA: [[META0:![0-9]+]] = !{i32 1, !"amdgpu_code_object_version", i32 500} +; MESA: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500} ; MESA: [[META1]] = !{} ; MESA: [[META2]] = !{i64 42} ; MESA: [[META3:![0-9]+]] = !{i64 128} diff --git a/llvm/test/CodeGen/AMDGPU/lower-module-lds-via-hybrid.ll b/llvm/test/CodeGen/AMDGPU/lower-module-lds-via-hybrid.ll index bb7c43f76c8a..00d01a080ad1 100644 --- a/llvm/test/CodeGen/AMDGPU/lower-module-lds-via-hybrid.ll +++ b/llvm/test/CodeGen/AMDGPU/lower-module-lds-via-hybrid.ll @@ -283,7 +283,7 @@ define amdgpu_kernel void @k123() { ; OPT: !0 = !{i32 0, i32 1} ; OPT: !1 = !{i32 4, i32 5} ; OPT: !2 = !{i32 8, i32 9} -; OPT: !3 = !{i32 1, !"amdgpu_code_object_version", i32 500} +; OPT: !3 = !{i32 1, !"amdhsa_code_object_version", i32 500} ; OPT: !4 = !{i32 1} ; OPT: !5 = !{!6} ; OPT: !6 = distinct !{!6, !7} @@ -313,4 +313,4 @@ attributes #4 = { nocallback nofree nosync nounwind speculatable willreturn memo ; GCN-NEXT: .size llvm.amdgcn.lds.offset.table, 8 !llvm.module.flags = !{!3} -!3 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!3 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/lower-module-lds-via-table.ll b/llvm/test/CodeGen/AMDGPU/lower-module-lds-via-table.ll index 4d73436c519b..d3cc60c501fd 100644 --- a/llvm/test/CodeGen/AMDGPU/lower-module-lds-via-table.ll +++ b/llvm/test/CodeGen/AMDGPU/lower-module-lds-via-table.ll @@ -373,4 +373,4 @@ define amdgpu_kernel void @k123() { ; GCN-NEXT: .size llvm.amdgcn.lds.offset.table, 48 !llvm.module.flags = !{!4} -!4 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!4 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll index 0bb5288f43ef..6672568b98a2 100644 --- a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll +++ b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll @@ -1087,7 +1087,7 @@ attributes #5 = { convergent nounwind } !opencl.ocl.version = !{!3} !llvm.ident = !{!4} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} !1 = !{i32 1, !"wchar_size", i32 4} !2 = !{i32 8, !"PIC Level", i32 2} !3 = !{i32 1, i32 2} diff --git a/llvm/test/CodeGen/AMDGPU/module-lds-false-sharing.ll b/llvm/test/CodeGen/AMDGPU/module-lds-false-sharing.ll index b398e86403ff..46036256780b 100644 --- a/llvm/test/CodeGen/AMDGPU/module-lds-false-sharing.ll +++ b/llvm/test/CodeGen/AMDGPU/module-lds-false-sharing.ll @@ -190,4 +190,4 @@ define void @nonkernel() { } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/need-fp-from-vgpr-spills.ll b/llvm/test/CodeGen/AMDGPU/need-fp-from-vgpr-spills.ll index 5f507d482eeb..cbdc7bb45634 100644 --- a/llvm/test/CodeGen/AMDGPU/need-fp-from-vgpr-spills.ll +++ b/llvm/test/CodeGen/AMDGPU/need-fp-from-vgpr-spills.ll @@ -275,4 +275,4 @@ attributes #0 = { "frame-pointer"="none" noinline } attributes #1 = { "frame-pointer"="all" noinline } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/no-hsa-graphics-shaders.ll b/llvm/test/CodeGen/AMDGPU/no-hsa-graphics-shaders.ll index c128b1729bfc..ee6a578c7285 100644 --- a/llvm/test/CodeGen/AMDGPU/no-hsa-graphics-shaders.ll +++ b/llvm/test/CodeGen/AMDGPU/no-hsa-graphics-shaders.ll @@ -16,4 +16,4 @@ define amdgpu_gs void @geometry_shader() #0 { } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/non-entry-alloca.ll b/llvm/test/CodeGen/AMDGPU/non-entry-alloca.ll index 494ace8a641e..125e6bc0f787 100644 --- a/llvm/test/CodeGen/AMDGPU/non-entry-alloca.ll +++ b/llvm/test/CodeGen/AMDGPU/non-entry-alloca.ll @@ -405,4 +405,4 @@ attributes #0 = { nounwind readnone speculatable } attributes #1 = { nounwind "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/nop-data.ll b/llvm/test/CodeGen/AMDGPU/nop-data.ll index d75bd8fb22e4..b541a893cac2 100644 --- a/llvm/test/CodeGen/AMDGPU/nop-data.ll +++ b/llvm/test/CodeGen/AMDGPU/nop-data.ll @@ -87,4 +87,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/partial-sgpr-to-vgpr-spills.ll b/llvm/test/CodeGen/AMDGPU/partial-sgpr-to-vgpr-spills.ll index d898a13cc619..5b0354e63c23 100644 --- a/llvm/test/CodeGen/AMDGPU/partial-sgpr-to-vgpr-spills.ll +++ b/llvm/test/CodeGen/AMDGPU/partial-sgpr-to-vgpr-spills.ll @@ -1246,4 +1246,4 @@ attributes #0 = { nounwind } attributes #1 = { nounwind "amdgpu-waves-per-eu"="8,8" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll b/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll index 5d580f838e0e..1be041c8dc9b 100644 --- a/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll +++ b/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll @@ -830,4 +830,4 @@ declare void @foo() attributes #0 = { "amdgpu-num-vgpr"="42" "amdgpu-num-sgpr"="40"} !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-calling-conv.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-calling-conv.ll index cfe822f2e135..ad2db7485808 100644 --- a/llvm/test/CodeGen/AMDGPU/promote-alloca-calling-conv.ll +++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-calling-conv.ll @@ -96,4 +96,4 @@ attributes #0 = { nounwind "amdgpu-flat-work-group-size"="64,64" } attributes #1 = { nounwind readnone } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-no-opts.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-no-opts.ll index 7ae400663ab8..28b923243b6d 100644 --- a/llvm/test/CodeGen/AMDGPU/promote-alloca-no-opts.ll +++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-no-opts.ll @@ -36,4 +36,4 @@ attributes #0 = { nounwind "amdgpu-flat-work-group-size"="64,64" } attributes #1 = { nounwind optnone noinline "amdgpu-flat-work-group-size"="64,64" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-padding-size-estimate.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-padding-size-estimate.ll index 665c2a3caa95..20a8cfc2a279 100644 --- a/llvm/test/CodeGen/AMDGPU/promote-alloca-padding-size-estimate.ll +++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-padding-size-estimate.ll @@ -129,4 +129,4 @@ entry: attributes #0 = { nounwind "amdgpu-flat-work-group-size"="64,64" "amdgpu-waves-per-eu"="1,7" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/recursion.ll b/llvm/test/CodeGen/AMDGPU/recursion.ll index ccf30b5a593f..d58477c194ea 100644 --- a/llvm/test/CodeGen/AMDGPU/recursion.ll +++ b/llvm/test/CodeGen/AMDGPU/recursion.ll @@ -79,4 +79,4 @@ define amdgpu_kernel void @kernel_calls_tail_recursive_with_stack() { } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/remove-no-kernel-id-attribute.ll b/llvm/test/CodeGen/AMDGPU/remove-no-kernel-id-attribute.ll index 2c1a6b43c847..297a056526ca 100644 --- a/llvm/test/CodeGen/AMDGPU/remove-no-kernel-id-attribute.ll +++ b/llvm/test/CodeGen/AMDGPU/remove-no-kernel-id-attribute.ll @@ -188,7 +188,7 @@ define amdgpu_kernel void @kernel_lds_recursion() { } !llvm.module.flags = !{!1} -!1 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!1 = !{i32 1, !"amdhsa_code_object_version", i32 400} ;. ; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } diff --git a/llvm/test/CodeGen/AMDGPU/reqd-work-group-size.ll b/llvm/test/CodeGen/AMDGPU/reqd-work-group-size.ll index 7d7917e0b20c..8c584a1890c9 100644 --- a/llvm/test/CodeGen/AMDGPU/reqd-work-group-size.ll +++ b/llvm/test/CodeGen/AMDGPU/reqd-work-group-size.ll @@ -458,4 +458,4 @@ attributes #3 = { nounwind "uniform-work-group-size"="false" } !3 = !{i16 8, i16 16, i16 2} !llvm.module.flags = !{!4} -!4 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!4 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/resource-optimization-remarks.ll b/llvm/test/CodeGen/AMDGPU/resource-optimization-remarks.ll index 9b52695fefb7..a640ac985ade 100644 --- a/llvm/test/CodeGen/AMDGPU/resource-optimization-remarks.ll +++ b/llvm/test/CodeGen/AMDGPU/resource-optimization-remarks.ll @@ -211,4 +211,4 @@ define amdgpu_kernel void @test_indirect_w_static_stack() !dbg !10 { !8 = distinct !DISubprogram(name: "empty_func", scope: !1, file: !1, type: !4, scopeLine: 52, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0) !9 = distinct !DISubprogram(name: "test_indirect_call", scope: !1, file: !1, type: !4, scopeLine: 64, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0) !10 = distinct !DISubprogram(name: "test_indirect_w_static_stack", scope: !1, file: !1, type: !4, scopeLine: 74, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0) -!11 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!11 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/resource-usage-dead-function.ll b/llvm/test/CodeGen/AMDGPU/resource-usage-dead-function.ll index 503b33487579..bba59ba4d803 100644 --- a/llvm/test/CodeGen/AMDGPU/resource-usage-dead-function.ll +++ b/llvm/test/CodeGen/AMDGPU/resource-usage-dead-function.ll @@ -37,4 +37,4 @@ bb2: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spill-no-vgprs.ll b/llvm/test/CodeGen/AMDGPU/sgpr-spill-no-vgprs.ll index 242ecd89b5c7..d430ba758572 100644 --- a/llvm/test/CodeGen/AMDGPU/sgpr-spill-no-vgprs.ll +++ b/llvm/test/CodeGen/AMDGPU/sgpr-spill-no-vgprs.ll @@ -251,4 +251,4 @@ attributes #0 = { nounwind } attributes #1 = { nounwind "amdgpu-waves-per-eu"="10,10" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spill-update-only-slot-indexes.ll b/llvm/test/CodeGen/AMDGPU/sgpr-spill-update-only-slot-indexes.ll index 28804923fb31..b339915edd20 100644 --- a/llvm/test/CodeGen/AMDGPU/sgpr-spill-update-only-slot-indexes.ll +++ b/llvm/test/CodeGen/AMDGPU/sgpr-spill-update-only-slot-indexes.ll @@ -40,4 +40,4 @@ define amdgpu_kernel void @kernel() { } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/simple-indirect-call.ll b/llvm/test/CodeGen/AMDGPU/simple-indirect-call.ll index e7c5aaf043ef..f229f33664e1 100644 --- a/llvm/test/CodeGen/AMDGPU/simple-indirect-call.ll +++ b/llvm/test/CodeGen/AMDGPU/simple-indirect-call.ll @@ -78,4 +78,4 @@ define amdgpu_kernel void @test_simple_indirect_call() { ;. !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/sopk-no-literal.ll b/llvm/test/CodeGen/AMDGPU/sopk-no-literal.ll index ef868372990b..fc700c55d7ee 100644 --- a/llvm/test/CodeGen/AMDGPU/sopk-no-literal.ll +++ b/llvm/test/CodeGen/AMDGPU/sopk-no-literal.ll @@ -24,4 +24,4 @@ define amdgpu_kernel void @test_sopk_size(i32 %var.mode) { declare void @llvm.amdgcn.s.setreg(i32 immarg, i32) !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/spill-m0.ll b/llvm/test/CodeGen/AMDGPU/spill-m0.ll index f192f25b2388..b2235544686f 100644 --- a/llvm/test/CodeGen/AMDGPU/spill-m0.ll +++ b/llvm/test/CodeGen/AMDGPU/spill-m0.ll @@ -192,4 +192,4 @@ attributes #0 = { nounwind } attributes #1 = { nounwind readnone } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/stack-realign-kernel.ll b/llvm/test/CodeGen/AMDGPU/stack-realign-kernel.ll index eb211f779fb3..5c6f0019f1ed 100644 --- a/llvm/test/CodeGen/AMDGPU/stack-realign-kernel.ll +++ b/llvm/test/CodeGen/AMDGPU/stack-realign-kernel.ll @@ -341,4 +341,4 @@ attributes #1 = { nounwind "stackrealign" } attributes #2 = { nounwind alignstack=128 } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/stacksave_stackrestore.ll b/llvm/test/CodeGen/AMDGPU/stacksave_stackrestore.ll index 33b5d6c6850b..b8bc01e0b879 100644 --- a/llvm/test/CodeGen/AMDGPU/stacksave_stackrestore.ll +++ b/llvm/test/CodeGen/AMDGPU/stacksave_stackrestore.ll @@ -1706,7 +1706,7 @@ define void @func_stacksave_stackrestore_call_with_stack_objects() { } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: ; WAVE32: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-any.ll b/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-any.ll index 2d2d64910c4f..1be420eccb35 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-any.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-any.ll @@ -24,4 +24,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-off.ll b/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-off.ll index e676f4f8de74..acdcd16a1f9e 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-off.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-off.ll @@ -24,4 +24,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-on.ll b/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-on.ll index 705bedf45097..0aac07342db8 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-on.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-kd-xnack-on.ll @@ -24,4 +24,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-any.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-any.ll index 89d89a7e9f60..560b0e2c81cf 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-any.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-any.ll @@ -42,4 +42,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-not-supported.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-not-supported.ll index bc57c9986592..0741ec4ffac4 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-not-supported.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-not-supported.ll @@ -41,4 +41,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-off.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-off.ll index 51351c32d29a..08dd90250d0b 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-off.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-off.ll @@ -43,4 +43,4 @@ entry: attributes #0 = { "target-features"="-xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-on.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-on.ll index f408cbefaca1..a8340ddadaaf 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-on.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-on.ll @@ -44,4 +44,4 @@ entry: attributes #0 = { "target-features"="+xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-1.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-1.ll index 78b337640202..aefcfac23ff5 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-1.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-1.ll @@ -44,4 +44,4 @@ entry: attributes #0 = { "target-features"="-xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-2.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-2.ll index d1c98c7da357..6005c3162240 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-2.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-2.ll @@ -44,4 +44,4 @@ entry: attributes #0 = { "target-features"="-xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-1.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-1.ll index adf84db400c1..328f56fb841b 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-1.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-1.ll @@ -44,4 +44,4 @@ entry: attributes #0 = { "target-features"="+xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-2.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-2.ll index 210b2e84af10..c50dd8b2fec7 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-2.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-2.ll @@ -44,4 +44,4 @@ entry: attributes #0 = { "target-features"="+xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-invalid-any-off-on.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-invalid-any-off-on.ll index ec812689ac3e..0f54d783484d 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-invalid-any-off-on.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-invalid-any-off-on.ll @@ -21,4 +21,4 @@ attributes #0 = { "target-features"="-xnack" } attributes #1 = { "target-features"="+xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-any.ll b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-any.ll index 321c20bc91de..cb2c07c7f9f4 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-any.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-any.ll @@ -27,4 +27,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-not-supported.ll b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-not-supported.ll index 44e77a23fba0..fed493b630a4 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-not-supported.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-not-supported.ll @@ -31,4 +31,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-off.ll b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-off.ll index 3205dbeb9e66..60ff8b2dbb5e 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-off.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-off.ll @@ -34,4 +34,4 @@ entry: attributes #0 = { "target-features"="-xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-on.ll b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-on.ll index 6e7c575f1a96..e04629a24209 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-on.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-on.ll @@ -34,4 +34,4 @@ entry: attributes #0 = { "target-features"="+xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/trap-abis.ll b/llvm/test/CodeGen/AMDGPU/trap-abis.ll index 54a15513cf0a..3cd6c98ef4b8 100644 --- a/llvm/test/CodeGen/AMDGPU/trap-abis.ll +++ b/llvm/test/CodeGen/AMDGPU/trap-abis.ll @@ -207,4 +207,4 @@ attributes #0 = { nounwind noreturn } attributes #1 = { nounwind } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/trap.ll b/llvm/test/CodeGen/AMDGPU/trap.ll index 937de1e581a3..2f687295af73 100644 --- a/llvm/test/CodeGen/AMDGPU/trap.ll +++ b/llvm/test/CodeGen/AMDGPU/trap.ll @@ -146,4 +146,4 @@ attributes #0 = { nounwind noreturn } attributes #1 = { nounwind } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} diff --git a/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll b/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll index 1118cc3b1646..837b46f0ce57 100644 --- a/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll +++ b/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll @@ -719,4 +719,4 @@ bb73.i: ; preds = %bb70.i } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/unstructured-cfg-def-use-issue.ll b/llvm/test/CodeGen/AMDGPU/unstructured-cfg-def-use-issue.ll index ed7182914cd9..a5e1506114f2 100644 --- a/llvm/test/CodeGen/AMDGPU/unstructured-cfg-def-use-issue.ll +++ b/llvm/test/CodeGen/AMDGPU/unstructured-cfg-def-use-issue.ll @@ -523,4 +523,4 @@ declare i32 @llvm.amdgcn.workitem.id.x() declare hidden float @spam() !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/unsupported-code-object-version.ll b/llvm/test/CodeGen/AMDGPU/unsupported-code-object-version.ll index 4ee7538d7885..da7bc3a85d73 100644 --- a/llvm/test/CodeGen/AMDGPU/unsupported-code-object-version.ll +++ b/llvm/test/CodeGen/AMDGPU/unsupported-code-object-version.ll @@ -5,4 +5,4 @@ ; HSA-ERROR: Unexpected code object version !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-spill-placement-issue61083.ll b/llvm/test/CodeGen/AMDGPU/vgpr-spill-placement-issue61083.ll index a1d3e2a093fa..20dc5ad5c866 100644 --- a/llvm/test/CodeGen/AMDGPU/vgpr-spill-placement-issue61083.ll +++ b/llvm/test/CodeGen/AMDGPU/vgpr-spill-placement-issue61083.ll @@ -104,4 +104,4 @@ declare void @llvm.amdgcn.s.barrier() declare void @llvm.trap() !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/vgpr_constant_to_sgpr.ll b/llvm/test/CodeGen/AMDGPU/vgpr_constant_to_sgpr.ll index 364ce82b2e99..37f207fc7a54 100644 --- a/llvm/test/CodeGen/AMDGPU/vgpr_constant_to_sgpr.ll +++ b/llvm/test/CodeGen/AMDGPU/vgpr_constant_to_sgpr.ll @@ -58,4 +58,4 @@ define protected amdgpu_kernel void @kern(ptr %addr) !llvm.amdgcn.lds.kernel.id !llvm.module.flags = !{!1} !0 = !{i32 42} -!1 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!1 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll b/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll index 6ac61410a0e7..e5cebc1c3183 100644 --- a/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll +++ b/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll @@ -1764,4 +1764,4 @@ declare <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32>, i32, i32) declare <4 x i32> @llvm.amdgcn.s.buffer.load.v4i32(<4 x i32>, i32, i32) !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 500} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 500} diff --git a/llvm/test/tools/llvm-reduce/reduce-module-flags.ll b/llvm/test/tools/llvm-reduce/reduce-module-flags.ll index a98ad6bff600..e4dc84211856 100644 --- a/llvm/test/tools/llvm-reduce/reduce-module-flags.ll +++ b/llvm/test/tools/llvm-reduce/reduce-module-flags.ll @@ -32,7 +32,7 @@ !llvm.module.flags = !{!0, !1, !2, !3, !4} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdhsa_code_object_version", i32 400} !1 = !{i32 1, !"wchar_size", i32 4} !2 = !{i32 7, !"openmp", i32 50} !3 = !{i32 7, !"openmp-device", i32 50} -- GitLab From fec471649fffaa3ec44e17801e5c9605825e58bb Mon Sep 17 00:00:00 2001 From: Haojian Wu Date: Wed, 6 Mar 2024 15:55:41 +0100 Subject: [PATCH 300/929] [clang] Use getDefaultArgRange instead of getDefaultArg to retrieve the (#79296) source location in` AliasTemplateDeductionGuideTransform`. I don't have a reproducible testcase, but this should be a safe and non-functional change. We have checked the `hasDefaultArg` before calling `getDefaultArg()`, but `hasDefaultArg` allows unparsed/uninstantiated default arg which is prohibited in `getDefaultArg()`. Since we're only interested in the source location, we switch to use `getDefaultArgRange()` API. --- clang/lib/Sema/SemaTemplate.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp index df20f83ac8af..7e91815c2d52 100644 --- a/clang/lib/Sema/SemaTemplate.cpp +++ b/clang/lib/Sema/SemaTemplate.cpp @@ -2598,7 +2598,7 @@ private: // placeholder to indicate there is a default argument. QualType ParamTy = NewDI->getType(); NewDefArg = new (SemaRef.Context) - OpaqueValueExpr(OldParam->getDefaultArg()->getBeginLoc(), + OpaqueValueExpr(OldParam->getDefaultArgRange().getBegin(), ParamTy.getNonLValueExprType(SemaRef.Context), ParamTy->isLValueReferenceType() ? VK_LValue : ParamTy->isRValueReferenceType() ? VK_XValue -- GitLab From f448b8ec03bbd0e1b47d56f461b324fb2af8c113 Mon Sep 17 00:00:00 2001 From: Joe Nash Date: Wed, 6 Mar 2024 10:11:10 -0500 Subject: [PATCH 301/929] [AMDGPU] Make gfx11 vop1 disassembler tests use strict-whitespace (#84078) NFC. The whitespace needs to be consistently formatted in some manner. Might as well use -strict-whitespace as the standard. Adds -strict-whitespace to RUN lines and adjust CHECK line space padding accordingly. Also test REAL16 and FAKE16 CHECK lines with wave64. --- .../Disassembler/AMDGPU/gfx11_dasm_vop1.txt | 428 +++++++++--------- .../AMDGPU/gfx11_dasm_vop3_from_vop1.txt | 398 ++++++++-------- 2 files changed, 414 insertions(+), 412 deletions(-) diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop1.txt index c8f7bf8dc01d..b176a57d70f8 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop1.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop1.txt @@ -1,7 +1,7 @@ -# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,GFX11-REAL16 %s -# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s -# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+real-true16,-WavefrontSize32,+WavefrontSize64 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX11 %s -# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16,-WavefrontSize32,+WavefrontSize64 -disassemble -show-encoding < %s | FileCheck -check-prefix=GFX11 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefixes=GFX11,GFX11-REAL16 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefixes=GFX11,GFX11-FAKE16 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+real-true16,-WavefrontSize32,+WavefrontSize64 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefixes=GFX11,GFX11-REAL16 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16,-WavefrontSize32,+WavefrontSize64 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefixes=GFX11,GFX11-FAKE16 %s # GFX11: v_bfrev_b32_e32 v5, v1 ; encoding: [0x01,0x71,0x0a,0x7e] 0x01,0x71,0x0a,0x7e @@ -48,72 +48,72 @@ # GFX11: v_bfrev_b32_e32 v255, 0xaf123456 ; encoding: [0xff,0x70,0xfe,0x7f,0x56,0x34,0x12,0xaf] 0xff,0x70,0xfe,0x7f,0x56,0x34,0x12,0xaf -# GFX11-FAKE16: v_ceil_f16_e32 v5, v1 ; encoding: [0x01,0xb9,0x0a,0x7e] -# GFX11-REAL16: v_ceil_f16_e32 v5.l, v1.l ; encoding: [0x01,0xb9,0x0a,0x7e] +# GFX11-FAKE16: v_ceil_f16_e32 v5, v1 ; encoding: [0x01,0xb9,0x0a,0x7e] +# GFX11-REAL16: v_ceil_f16_e32 v5.l, v1.l ; encoding: [0x01,0xb9,0x0a,0x7e] 0x01,0xb9,0x0a,0x7e -# GFX11-FAKE16: v_ceil_f16_e32 v5, v127 ; encoding: [0x7f,0xb9,0x0a,0x7e] -# GFX11-REAL16: v_ceil_f16_e32 v5.l, v127.l ; encoding: [0x7f,0xb9,0x0a,0x7e] +# GFX11-FAKE16: v_ceil_f16_e32 v5, v127 ; encoding: [0x7f,0xb9,0x0a,0x7e] +# GFX11-REAL16: v_ceil_f16_e32 v5.l, v127.l ; encoding: [0x7f,0xb9,0x0a,0x7e] 0x7f,0xb9,0x0a,0x7e # GFX11-FAKE16: v_ceil_f16_e32 v5, v129/*Invalid register, operand has 'VS_32_Lo128' register class*/ ; encoding: [0x81,0xb9,0x0a,0x7e] -# GFX11-REAL16: v_ceil_f16_e32 v5.l, v1.h ; encoding: [0x81,0xb9,0x0a,0x7e] +# GFX11-REAL16: v_ceil_f16_e32 v5.l, v1.h ; encoding: [0x81,0xb9,0x0a,0x7e] 0x81,0xb9,0x0a,0x7e # GFX11-FAKE16: v_ceil_f16_e32 v5, v255/*Invalid register, operand has 'VS_32_Lo128' register class*/ ; encoding: [0xff,0xb9,0x0a,0x7e] -# GFX11-REAL16: v_ceil_f16_e32 v5.l, v127.h ; encoding: [0xff,0xb9,0x0a,0x7e] +# GFX11-REAL16: v_ceil_f16_e32 v5.l, v127.h ; encoding: [0xff,0xb9,0x0a,0x7e] 0xff,0xb9,0x0a,0x7e -# GFX11-FAKE16: v_ceil_f16_e32 v5, s1 ; encoding: [0x01,0xb8,0x0a,0x7e] -# GFX11-REAL16: v_ceil_f16_e32 v5.l, s1 ; encoding: [0x01,0xb8,0x0a,0x7e] +# GFX11-FAKE16: v_ceil_f16_e32 v5, s1 ; encoding: [0x01,0xb8,0x0a,0x7e] +# GFX11-REAL16: v_ceil_f16_e32 v5.l, s1 ; encoding: [0x01,0xb8,0x0a,0x7e] 0x01,0xb8,0x0a,0x7e -# GFX11-FAKE16: v_ceil_f16_e32 v5, s105 ; encoding: [0x69,0xb8,0x0a,0x7e] -# GFX11-REAL16: v_ceil_f16_e32 v5.l, s105 ; encoding: [0x69,0xb8,0x0a,0x7e] +# GFX11-FAKE16: v_ceil_f16_e32 v5, s105 ; encoding: [0x69,0xb8,0x0a,0x7e] +# GFX11-REAL16: v_ceil_f16_e32 v5.l, s105 ; encoding: [0x69,0xb8,0x0a,0x7e] 0x69,0xb8,0x0a,0x7e -# GFX11-FAKE16: v_ceil_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xb8,0x0a,0x7e] -# GFX11-REAL16: v_ceil_f16_e32 v5.l, vcc_lo ; encoding: [0x6a,0xb8,0x0a,0x7e] +# GFX11-FAKE16: v_ceil_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xb8,0x0a,0x7e] +# GFX11-REAL16: v_ceil_f16_e32 v5.l, vcc_lo ; encoding: [0x6a,0xb8,0x0a,0x7e] 0x6a,0xb8,0x0a,0x7e -# GFX11-FAKE16: v_ceil_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xb8,0x0a,0x7e] -# GFX11-REAL16: v_ceil_f16_e32 v5.l, vcc_hi ; encoding: [0x6b,0xb8,0x0a,0x7e] +# GFX11-FAKE16: v_ceil_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xb8,0x0a,0x7e] +# GFX11-REAL16: v_ceil_f16_e32 v5.l, vcc_hi ; encoding: [0x6b,0xb8,0x0a,0x7e] 0x6b,0xb8,0x0a,0x7e -# GFX11-FAKE16: v_ceil_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xb8,0x0a,0x7e] -# GFX11-REAL16: v_ceil_f16_e32 v5.l, ttmp15 ; encoding: [0x7b,0xb8,0x0a,0x7e] +# GFX11-FAKE16: v_ceil_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xb8,0x0a,0x7e] +# GFX11-REAL16: v_ceil_f16_e32 v5.l, ttmp15 ; encoding: [0x7b,0xb8,0x0a,0x7e] 0x7b,0xb8,0x0a,0x7e -# GFX11-FAKE16: v_ceil_f16_e32 v5, m0 ; encoding: [0x7d,0xb8,0x0a,0x7e] -# GFX11-REAL16: v_ceil_f16_e32 v5.l, m0 ; encoding: [0x7d,0xb8,0x0a,0x7e] +# GFX11-FAKE16: v_ceil_f16_e32 v5, m0 ; encoding: [0x7d,0xb8,0x0a,0x7e] +# GFX11-REAL16: v_ceil_f16_e32 v5.l, m0 ; encoding: [0x7d,0xb8,0x0a,0x7e] 0x7d,0xb8,0x0a,0x7e -# GFX11-FAKE16: v_ceil_f16_e32 v5, exec_lo ; encoding: [0x7e,0xb8,0x0a,0x7e] -# GFX11-REAL16: v_ceil_f16_e32 v5.l, exec_lo ; encoding: [0x7e,0xb8,0x0a,0x7e] +# GFX11-FAKE16: v_ceil_f16_e32 v5, exec_lo ; encoding: [0x7e,0xb8,0x0a,0x7e] +# GFX11-REAL16: v_ceil_f16_e32 v5.l, exec_lo ; encoding: [0x7e,0xb8,0x0a,0x7e] 0x7e,0xb8,0x0a,0x7e -# GFX11-FAKE16: v_ceil_f16_e32 v5, exec_hi ; encoding: [0x7f,0xb8,0x0a,0x7e] -# GFX11-REAL16: v_ceil_f16_e32 v5.l, exec_hi ; encoding: [0x7f,0xb8,0x0a,0x7e] +# GFX11-FAKE16: v_ceil_f16_e32 v5, exec_hi ; encoding: [0x7f,0xb8,0x0a,0x7e] +# GFX11-REAL16: v_ceil_f16_e32 v5.l, exec_hi ; encoding: [0x7f,0xb8,0x0a,0x7e] 0x7f,0xb8,0x0a,0x7e -# GFX11-FAKE16: v_ceil_f16_e32 v5, null ; encoding: [0x7c,0xb8,0x0a,0x7e] -# GFX11-REAL16: v_ceil_f16_e32 v5.l, null ; encoding: [0x7c,0xb8,0x0a,0x7e] +# GFX11-FAKE16: v_ceil_f16_e32 v5, null ; encoding: [0x7c,0xb8,0x0a,0x7e] +# GFX11-REAL16: v_ceil_f16_e32 v5.l, null ; encoding: [0x7c,0xb8,0x0a,0x7e] 0x7c,0xb8,0x0a,0x7e -# GFX11-FAKE16: v_ceil_f16_e32 v5, -1 ; encoding: [0xc1,0xb8,0x0a,0x7e] -# GFX11-REAL16: v_ceil_f16_e32 v5.l, -1 ; encoding: [0xc1,0xb8,0x0a,0x7e] +# GFX11-FAKE16: v_ceil_f16_e32 v5, -1 ; encoding: [0xc1,0xb8,0x0a,0x7e] +# GFX11-REAL16: v_ceil_f16_e32 v5.l, -1 ; encoding: [0xc1,0xb8,0x0a,0x7e] 0xc1,0xb8,0x0a,0x7e -# GFX11-FAKE16: v_ceil_f16_e32 v127, 0.5 ; encoding: [0xf0,0xb8,0xfe,0x7e] -# GFX11-REAL16: v_ceil_f16_e32 v127.l, 0.5 ; encoding: [0xf0,0xb8,0xfe,0x7e] +# GFX11-FAKE16: v_ceil_f16_e32 v127, 0.5 ; encoding: [0xf0,0xb8,0xfe,0x7e] +# GFX11-REAL16: v_ceil_f16_e32 v127.l, 0.5 ; encoding: [0xf0,0xb8,0xfe,0x7e] 0xf0,0xb8,0xfe,0x7e # COM: GFX11-FAKE16: warning: invalid instruction encoding 0xfd,0xb8,0x0a,0x7f -# GFX11-REAL16: v_ceil_f16_e32 v5.h, src_scc ; encoding: [0xfd,0xb8,0x0a,0x7f] +# GFX11-REAL16: v_ceil_f16_e32 v5.h, src_scc ; encoding: [0xfd,0xb8,0x0a,0x7f] 0xfd,0xb8,0x0a,0x7f # COM: GFX11-FAKE16: warning: invalid instruction encoding 0xff,0xb8,0xfe,0x7f,0x0b,0xfe,0x00,0x00 -# GFX11-REAL16: v_ceil_f16_e32 v127.h, 0xfe0b ; encoding: [0xff,0xb8,0xfe,0x7f,0x0b,0xfe,0x00,0x00] +# GFX11-REAL16: v_ceil_f16_e32 v127.h, 0xfe0b ; encoding: [0xff,0xb8,0xfe,0x7f,0x0b,0xfe,0x00,0x00] 0xff,0xb8,0xfe,0x7f,0x0b,0xfe,0x00,0x00 # GFX11: v_ceil_f32_e32 v5, v1 ; encoding: [0x01,0x45,0x0a,0x7e] @@ -1610,72 +1610,72 @@ # GFX11: v_cvt_u32_u16_e32 v255, 0xfe0b ; encoding: [0xff,0xd6,0xfe,0x7f,0x0b,0xfe,0x00,0x00] 0xff,0xd6,0xfe,0x7f,0x0b,0xfe,0x00,0x00 -# GFX11-FAKE16: v_exp_f16_e32 v5, v1 ; encoding: [0x01,0xb1,0x0a,0x7e] -# GFX11-REAL16: v_exp_f16_e32 v5.l, v1.l ; encoding: [0x01,0xb1,0x0a,0x7e] +# GFX11-FAKE16: v_exp_f16_e32 v5, v1 ; encoding: [0x01,0xb1,0x0a,0x7e] +# GFX11-REAL16: v_exp_f16_e32 v5.l, v1.l ; encoding: [0x01,0xb1,0x0a,0x7e] 0x01,0xb1,0x0a,0x7e -# GFX11-FAKE16: v_exp_f16_e32 v5, v127 ; encoding: [0x7f,0xb1,0x0a,0x7e] -# GFX11-REAL16: v_exp_f16_e32 v5.l, v127.l ; encoding: [0x7f,0xb1,0x0a,0x7e] +# GFX11-FAKE16: v_exp_f16_e32 v5, v127 ; encoding: [0x7f,0xb1,0x0a,0x7e] +# GFX11-REAL16: v_exp_f16_e32 v5.l, v127.l ; encoding: [0x7f,0xb1,0x0a,0x7e] 0x7f,0xb1,0x0a,0x7e # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_exp_f16_e32 v5.l, v1.h ; encoding: [0x81,0xb1,0x0a,0x7e] +# GFX11-REAL16: v_exp_f16_e32 v5.l, v1.h ; encoding: [0x81,0xb1,0x0a,0x7e] 0x81,0xb1,0x0a,0x7e # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_exp_f16_e32 v5.l, v127.h ; encoding: [0xff,0xb1,0x0a,0x7e] +# GFX11-REAL16: v_exp_f16_e32 v5.l, v127.h ; encoding: [0xff,0xb1,0x0a,0x7e] 0xff,0xb1,0x0a,0x7e -# GFX11-FAKE16: v_exp_f16_e32 v5, s1 ; encoding: [0x01,0xb0,0x0a,0x7e] -# GFX11-REAL16: v_exp_f16_e32 v5.l, s1 ; encoding: [0x01,0xb0,0x0a,0x7e] +# GFX11-FAKE16: v_exp_f16_e32 v5, s1 ; encoding: [0x01,0xb0,0x0a,0x7e] +# GFX11-REAL16: v_exp_f16_e32 v5.l, s1 ; encoding: [0x01,0xb0,0x0a,0x7e] 0x01,0xb0,0x0a,0x7e -# GFX11-FAKE16: v_exp_f16_e32 v5, s105 ; encoding: [0x69,0xb0,0x0a,0x7e] -# GFX11-REAL16: v_exp_f16_e32 v5.l, s105 ; encoding: [0x69,0xb0,0x0a,0x7e] +# GFX11-FAKE16: v_exp_f16_e32 v5, s105 ; encoding: [0x69,0xb0,0x0a,0x7e] +# GFX11-REAL16: v_exp_f16_e32 v5.l, s105 ; encoding: [0x69,0xb0,0x0a,0x7e] 0x69,0xb0,0x0a,0x7e -# GFX11-FAKE16: v_exp_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xb0,0x0a,0x7e] -# GFX11-REAL16: v_exp_f16_e32 v5.l, vcc_lo ; encoding: [0x6a,0xb0,0x0a,0x7e] +# GFX11-FAKE16: v_exp_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xb0,0x0a,0x7e] +# GFX11-REAL16: v_exp_f16_e32 v5.l, vcc_lo ; encoding: [0x6a,0xb0,0x0a,0x7e] 0x6a,0xb0,0x0a,0x7e -# GFX11-FAKE16: v_exp_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xb0,0x0a,0x7e] -# GFX11-REAL16: v_exp_f16_e32 v5.l, vcc_hi ; encoding: [0x6b,0xb0,0x0a,0x7e] +# GFX11-FAKE16: v_exp_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xb0,0x0a,0x7e] +# GFX11-REAL16: v_exp_f16_e32 v5.l, vcc_hi ; encoding: [0x6b,0xb0,0x0a,0x7e] 0x6b,0xb0,0x0a,0x7e -# GFX11-FAKE16: v_exp_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xb0,0x0a,0x7e] -# GFX11-REAL16: v_exp_f16_e32 v5.l, ttmp15 ; encoding: [0x7b,0xb0,0x0a,0x7e] +# GFX11-FAKE16: v_exp_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xb0,0x0a,0x7e] +# GFX11-REAL16: v_exp_f16_e32 v5.l, ttmp15 ; encoding: [0x7b,0xb0,0x0a,0x7e] 0x7b,0xb0,0x0a,0x7e -# GFX11-FAKE16: v_exp_f16_e32 v5, m0 ; encoding: [0x7d,0xb0,0x0a,0x7e] -# GFX11-REAL16: v_exp_f16_e32 v5.l, m0 ; encoding: [0x7d,0xb0,0x0a,0x7e] +# GFX11-FAKE16: v_exp_f16_e32 v5, m0 ; encoding: [0x7d,0xb0,0x0a,0x7e] +# GFX11-REAL16: v_exp_f16_e32 v5.l, m0 ; encoding: [0x7d,0xb0,0x0a,0x7e] 0x7d,0xb0,0x0a,0x7e -# GFX11-FAKE16: v_exp_f16_e32 v5, exec_lo ; encoding: [0x7e,0xb0,0x0a,0x7e] -# GFX11-REAL16: v_exp_f16_e32 v5.l, exec_lo ; encoding: [0x7e,0xb0,0x0a,0x7e] +# GFX11-FAKE16: v_exp_f16_e32 v5, exec_lo ; encoding: [0x7e,0xb0,0x0a,0x7e] +# GFX11-REAL16: v_exp_f16_e32 v5.l, exec_lo ; encoding: [0x7e,0xb0,0x0a,0x7e] 0x7e,0xb0,0x0a,0x7e -# GFX11-FAKE16: v_exp_f16_e32 v5, exec_hi ; encoding: [0x7f,0xb0,0x0a,0x7e] -# GFX11-REAL16: v_exp_f16_e32 v5.l, exec_hi ; encoding: [0x7f,0xb0,0x0a,0x7e] +# GFX11-FAKE16: v_exp_f16_e32 v5, exec_hi ; encoding: [0x7f,0xb0,0x0a,0x7e] +# GFX11-REAL16: v_exp_f16_e32 v5.l, exec_hi ; encoding: [0x7f,0xb0,0x0a,0x7e] 0x7f,0xb0,0x0a,0x7e -# GFX11-FAKE16: v_exp_f16_e32 v5, null ; encoding: [0x7c,0xb0,0x0a,0x7e] -# GFX11-REAL16: v_exp_f16_e32 v5.l, null ; encoding: [0x7c,0xb0,0x0a,0x7e] +# GFX11-FAKE16: v_exp_f16_e32 v5, null ; encoding: [0x7c,0xb0,0x0a,0x7e] +# GFX11-REAL16: v_exp_f16_e32 v5.l, null ; encoding: [0x7c,0xb0,0x0a,0x7e] 0x7c,0xb0,0x0a,0x7e -# GFX11-FAKE16: v_exp_f16_e32 v5, -1 ; encoding: [0xc1,0xb0,0x0a,0x7e] -# GFX11-REAL16: v_exp_f16_e32 v5.l, -1 ; encoding: [0xc1,0xb0,0x0a,0x7e] +# GFX11-FAKE16: v_exp_f16_e32 v5, -1 ; encoding: [0xc1,0xb0,0x0a,0x7e] +# GFX11-REAL16: v_exp_f16_e32 v5.l, -1 ; encoding: [0xc1,0xb0,0x0a,0x7e] 0xc1,0xb0,0x0a,0x7e -# GFX11-FAKE16: v_exp_f16_e32 v127, 0.5 ; encoding: [0xf0,0xb0,0xfe,0x7e] -# GFX11-REAL16: v_exp_f16_e32 v127.l, 0.5 ; encoding: [0xf0,0xb0,0xfe,0x7e] +# GFX11-FAKE16: v_exp_f16_e32 v127, 0.5 ; encoding: [0xf0,0xb0,0xfe,0x7e] +# GFX11-REAL16: v_exp_f16_e32 v127.l, 0.5 ; encoding: [0xf0,0xb0,0xfe,0x7e] 0xf0,0xb0,0xfe,0x7e # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_exp_f16_e32 v5.h, src_scc ; encoding: [0xfd,0xb0,0x0a,0x7f] +# GFX11-REAL16: v_exp_f16_e32 v5.h, src_scc ; encoding: [0xfd,0xb0,0x0a,0x7f] 0xfd,0xb0,0x0a,0x7f # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_exp_f16_e32 v127.h, 0xfe0b ; encoding: [0xff,0xb0,0xfe,0x7f,0x0b,0xfe,0x00,0x00] +# GFX11-REAL16: v_exp_f16_e32 v127.h, 0xfe0b ; encoding: [0xff,0xb0,0xfe,0x7f,0x0b,0xfe,0x00,0x00] 0xff,0xb0,0xfe,0x7f,0x0b,0xfe,0x00,0x00 # GFX11: v_exp_f32_e32 v5, v1 ; encoding: [0x01,0x4b,0x0a,0x7e] @@ -1723,72 +1723,72 @@ # GFX11: v_exp_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x4a,0xfe,0x7f,0x56,0x34,0x12,0xaf] 0xff,0x4a,0xfe,0x7f,0x56,0x34,0x12,0xaf -# GFX11-FAKE16: v_floor_f16_e32 v5, v1 ; encoding: [0x01,0xb7,0x0a,0x7e] -# GFX11-REAL16: v_floor_f16_e32 v5.l, v1.l ; encoding: [0x01,0xb7,0x0a,0x7e] +# GFX11-FAKE16: v_floor_f16_e32 v5, v1 ; encoding: [0x01,0xb7,0x0a,0x7e] +# GFX11-REAL16: v_floor_f16_e32 v5.l, v1.l ; encoding: [0x01,0xb7,0x0a,0x7e] 0x01,0xb7,0x0a,0x7e -# GFX11-FAKE16: v_floor_f16_e32 v5, v127 ; encoding: [0x7f,0xb7,0x0a,0x7e] -# GFX11-REAL16: v_floor_f16_e32 v5.l, v127.l ; encoding: [0x7f,0xb7,0x0a,0x7e] +# GFX11-FAKE16: v_floor_f16_e32 v5, v127 ; encoding: [0x7f,0xb7,0x0a,0x7e] +# GFX11-REAL16: v_floor_f16_e32 v5.l, v127.l ; encoding: [0x7f,0xb7,0x0a,0x7e] 0x7f,0xb7,0x0a,0x7e # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_floor_f16_e32 v5.l, v1.h ; encoding: [0x81,0xb7,0x0a,0x7e] +# GFX11-REAL16: v_floor_f16_e32 v5.l, v1.h ; encoding: [0x81,0xb7,0x0a,0x7e] 0x81,0xb7,0x0a,0x7e # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_floor_f16_e32 v5.l, v127.h ; encoding: [0xff,0xb7,0x0a,0x7e] +# GFX11-REAL16: v_floor_f16_e32 v5.l, v127.h ; encoding: [0xff,0xb7,0x0a,0x7e] 0xff,0xb7,0x0a,0x7e -# GFX11-FAKE16: v_floor_f16_e32 v5, s1 ; encoding: [0x01,0xb6,0x0a,0x7e] -# GFX11-REAL16: v_floor_f16_e32 v5.l, s1 ; encoding: [0x01,0xb6,0x0a,0x7e] +# GFX11-FAKE16: v_floor_f16_e32 v5, s1 ; encoding: [0x01,0xb6,0x0a,0x7e] +# GFX11-REAL16: v_floor_f16_e32 v5.l, s1 ; encoding: [0x01,0xb6,0x0a,0x7e] 0x01,0xb6,0x0a,0x7e -# GFX11-FAKE16: v_floor_f16_e32 v5, s105 ; encoding: [0x69,0xb6,0x0a,0x7e] -# GFX11-REAL16: v_floor_f16_e32 v5.l, s105 ; encoding: [0x69,0xb6,0x0a,0x7e] +# GFX11-FAKE16: v_floor_f16_e32 v5, s105 ; encoding: [0x69,0xb6,0x0a,0x7e] +# GFX11-REAL16: v_floor_f16_e32 v5.l, s105 ; encoding: [0x69,0xb6,0x0a,0x7e] 0x69,0xb6,0x0a,0x7e -# GFX11-FAKE16: v_floor_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xb6,0x0a,0x7e] -# GFX11-REAL16: v_floor_f16_e32 v5.l, vcc_lo ; encoding: [0x6a,0xb6,0x0a,0x7e] +# GFX11-FAKE16: v_floor_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xb6,0x0a,0x7e] +# GFX11-REAL16: v_floor_f16_e32 v5.l, vcc_lo ; encoding: [0x6a,0xb6,0x0a,0x7e] 0x6a,0xb6,0x0a,0x7e -# GFX11-FAKE16: v_floor_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xb6,0x0a,0x7e] -# GFX11-REAL16: v_floor_f16_e32 v5.l, vcc_hi ; encoding: [0x6b,0xb6,0x0a,0x7e] +# GFX11-FAKE16: v_floor_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xb6,0x0a,0x7e] +# GFX11-REAL16: v_floor_f16_e32 v5.l, vcc_hi ; encoding: [0x6b,0xb6,0x0a,0x7e] 0x6b,0xb6,0x0a,0x7e -# GFX11-FAKE16: v_floor_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xb6,0x0a,0x7e] -# GFX11-REAL16: v_floor_f16_e32 v5.l, ttmp15 ; encoding: [0x7b,0xb6,0x0a,0x7e] +# GFX11-FAKE16: v_floor_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xb6,0x0a,0x7e] +# GFX11-REAL16: v_floor_f16_e32 v5.l, ttmp15 ; encoding: [0x7b,0xb6,0x0a,0x7e] 0x7b,0xb6,0x0a,0x7e -# GFX11-FAKE16: v_floor_f16_e32 v5, m0 ; encoding: [0x7d,0xb6,0x0a,0x7e] -# GFX11-REAL16: v_floor_f16_e32 v5.l, m0 ; encoding: [0x7d,0xb6,0x0a,0x7e] +# GFX11-FAKE16: v_floor_f16_e32 v5, m0 ; encoding: [0x7d,0xb6,0x0a,0x7e] +# GFX11-REAL16: v_floor_f16_e32 v5.l, m0 ; encoding: [0x7d,0xb6,0x0a,0x7e] 0x7d,0xb6,0x0a,0x7e -# GFX11-FAKE16: v_floor_f16_e32 v5, exec_lo ; encoding: [0x7e,0xb6,0x0a,0x7e] -# GFX11-REAL16: v_floor_f16_e32 v5.l, exec_lo ; encoding: [0x7e,0xb6,0x0a,0x7e] +# GFX11-FAKE16: v_floor_f16_e32 v5, exec_lo ; encoding: [0x7e,0xb6,0x0a,0x7e] +# GFX11-REAL16: v_floor_f16_e32 v5.l, exec_lo ; encoding: [0x7e,0xb6,0x0a,0x7e] 0x7e,0xb6,0x0a,0x7e -# GFX11-FAKE16: v_floor_f16_e32 v5, exec_hi ; encoding: [0x7f,0xb6,0x0a,0x7e] -# GFX11-REAL16: v_floor_f16_e32 v5.l, exec_hi ; encoding: [0x7f,0xb6,0x0a,0x7e] +# GFX11-FAKE16: v_floor_f16_e32 v5, exec_hi ; encoding: [0x7f,0xb6,0x0a,0x7e] +# GFX11-REAL16: v_floor_f16_e32 v5.l, exec_hi ; encoding: [0x7f,0xb6,0x0a,0x7e] 0x7f,0xb6,0x0a,0x7e -# GFX11-FAKE16: v_floor_f16_e32 v5, null ; encoding: [0x7c,0xb6,0x0a,0x7e] -# GFX11-REAL16: v_floor_f16_e32 v5.l, null ; encoding: [0x7c,0xb6,0x0a,0x7e] +# GFX11-FAKE16: v_floor_f16_e32 v5, null ; encoding: [0x7c,0xb6,0x0a,0x7e] +# GFX11-REAL16: v_floor_f16_e32 v5.l, null ; encoding: [0x7c,0xb6,0x0a,0x7e] 0x7c,0xb6,0x0a,0x7e -# GFX11-FAKE16: v_floor_f16_e32 v5, -1 ; encoding: [0xc1,0xb6,0x0a,0x7e] -# GFX11-REAL16: v_floor_f16_e32 v5.l, -1 ; encoding: [0xc1,0xb6,0x0a,0x7e] +# GFX11-FAKE16: v_floor_f16_e32 v5, -1 ; encoding: [0xc1,0xb6,0x0a,0x7e] +# GFX11-REAL16: v_floor_f16_e32 v5.l, -1 ; encoding: [0xc1,0xb6,0x0a,0x7e] 0xc1,0xb6,0x0a,0x7e -# GFX11-FAKE16: v_floor_f16_e32 v127, 0.5 ; encoding: [0xf0,0xb6,0xfe,0x7e] -# GFX11-REAL16: v_floor_f16_e32 v127.l, 0.5 ; encoding: [0xf0,0xb6,0xfe,0x7e] +# GFX11-FAKE16: v_floor_f16_e32 v127, 0.5 ; encoding: [0xf0,0xb6,0xfe,0x7e] +# GFX11-REAL16: v_floor_f16_e32 v127.l, 0.5 ; encoding: [0xf0,0xb6,0xfe,0x7e] 0xf0,0xb6,0xfe,0x7e # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_floor_f16_e32 v5.h, src_scc ; encoding: [0xfd,0xb6,0x0a,0x7f] +# GFX11-REAL16: v_floor_f16_e32 v5.h, src_scc ; encoding: [0xfd,0xb6,0x0a,0x7f] 0xfd,0xb6,0x0a,0x7f # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_floor_f16_e32 v127.h, 0xfe0b ; encoding: [0xff,0xb6,0xfe,0x7f,0x0b,0xfe,0x00,0x00] +# GFX11-REAL16: v_floor_f16_e32 v127.h, 0xfe0b ; encoding: [0xff,0xb6,0xfe,0x7f,0x0b,0xfe,0x00,0x00] 0xff,0xb6,0xfe,0x7f,0x0b,0xfe,0x00,0x00 # GFX11: v_floor_f32_e32 v5, v1 ; encoding: [0x01,0x49,0x0a,0x7e] @@ -2250,72 +2250,72 @@ # GFX11: v_frexp_mant_f64_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x7a,0xfc,0x7f,0x56,0x34,0x12,0xaf] 0xff,0x7a,0xfc,0x7f,0x56,0x34,0x12,0xaf -# GFX11-FAKE16: v_log_f16_e32 v5, v1 ; encoding: [0x01,0xaf,0x0a,0x7e] -# GFX11-REAL16: v_log_f16_e32 v5.l, v1.l ; encoding: [0x01,0xaf,0x0a,0x7e] +# GFX11-FAKE16: v_log_f16_e32 v5, v1 ; encoding: [0x01,0xaf,0x0a,0x7e] +# GFX11-REAL16: v_log_f16_e32 v5.l, v1.l ; encoding: [0x01,0xaf,0x0a,0x7e] 0x01,0xaf,0x0a,0x7e -# GFX11-FAKE16: v_log_f16_e32 v5, v127 ; encoding: [0x7f,0xaf,0x0a,0x7e] -# GFX11-REAL16: v_log_f16_e32 v5.l, v127.l ; encoding: [0x7f,0xaf,0x0a,0x7e] +# GFX11-FAKE16: v_log_f16_e32 v5, v127 ; encoding: [0x7f,0xaf,0x0a,0x7e] +# GFX11-REAL16: v_log_f16_e32 v5.l, v127.l ; encoding: [0x7f,0xaf,0x0a,0x7e] 0x7f,0xaf,0x0a,0x7e # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_log_f16_e32 v5.l, v1.h ; encoding: [0x81,0xaf,0x0a,0x7e] +# GFX11-REAL16: v_log_f16_e32 v5.l, v1.h ; encoding: [0x81,0xaf,0x0a,0x7e] 0x81,0xaf,0x0a,0x7e # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_log_f16_e32 v5.l, v127.h ; encoding: [0xff,0xaf,0x0a,0x7e] +# GFX11-REAL16: v_log_f16_e32 v5.l, v127.h ; encoding: [0xff,0xaf,0x0a,0x7e] 0xff,0xaf,0x0a,0x7e -# GFX11-FAKE16: v_log_f16_e32 v5, s1 ; encoding: [0x01,0xae,0x0a,0x7e] -# GFX11-REAL16: v_log_f16_e32 v5.l, s1 ; encoding: [0x01,0xae,0x0a,0x7e] +# GFX11-FAKE16: v_log_f16_e32 v5, s1 ; encoding: [0x01,0xae,0x0a,0x7e] +# GFX11-REAL16: v_log_f16_e32 v5.l, s1 ; encoding: [0x01,0xae,0x0a,0x7e] 0x01,0xae,0x0a,0x7e -# GFX11-FAKE16: v_log_f16_e32 v5, s105 ; encoding: [0x69,0xae,0x0a,0x7e] -# GFX11-REAL16: v_log_f16_e32 v5.l, s105 ; encoding: [0x69,0xae,0x0a,0x7e] +# GFX11-FAKE16: v_log_f16_e32 v5, s105 ; encoding: [0x69,0xae,0x0a,0x7e] +# GFX11-REAL16: v_log_f16_e32 v5.l, s105 ; encoding: [0x69,0xae,0x0a,0x7e] 0x69,0xae,0x0a,0x7e -# GFX11-FAKE16: v_log_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xae,0x0a,0x7e] -# GFX11-REAL16: v_log_f16_e32 v5.l, vcc_lo ; encoding: [0x6a,0xae,0x0a,0x7e] +# GFX11-FAKE16: v_log_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xae,0x0a,0x7e] +# GFX11-REAL16: v_log_f16_e32 v5.l, vcc_lo ; encoding: [0x6a,0xae,0x0a,0x7e] 0x6a,0xae,0x0a,0x7e -# GFX11-FAKE16: v_log_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xae,0x0a,0x7e] -# GFX11-REAL16: v_log_f16_e32 v5.l, vcc_hi ; encoding: [0x6b,0xae,0x0a,0x7e] +# GFX11-FAKE16: v_log_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xae,0x0a,0x7e] +# GFX11-REAL16: v_log_f16_e32 v5.l, vcc_hi ; encoding: [0x6b,0xae,0x0a,0x7e] 0x6b,0xae,0x0a,0x7e -# GFX11-FAKE16: v_log_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xae,0x0a,0x7e] -# GFX11-REAL16: v_log_f16_e32 v5.l, ttmp15 ; encoding: [0x7b,0xae,0x0a,0x7e] +# GFX11-FAKE16: v_log_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xae,0x0a,0x7e] +# GFX11-REAL16: v_log_f16_e32 v5.l, ttmp15 ; encoding: [0x7b,0xae,0x0a,0x7e] 0x7b,0xae,0x0a,0x7e -# GFX11-FAKE16: v_log_f16_e32 v5, m0 ; encoding: [0x7d,0xae,0x0a,0x7e] -# GFX11-REAL16: v_log_f16_e32 v5.l, m0 ; encoding: [0x7d,0xae,0x0a,0x7e] +# GFX11-FAKE16: v_log_f16_e32 v5, m0 ; encoding: [0x7d,0xae,0x0a,0x7e] +# GFX11-REAL16: v_log_f16_e32 v5.l, m0 ; encoding: [0x7d,0xae,0x0a,0x7e] 0x7d,0xae,0x0a,0x7e -# GFX11-FAKE16: v_log_f16_e32 v5, exec_lo ; encoding: [0x7e,0xae,0x0a,0x7e] -# GFX11-REAL16: v_log_f16_e32 v5.l, exec_lo ; encoding: [0x7e,0xae,0x0a,0x7e] +# GFX11-FAKE16: v_log_f16_e32 v5, exec_lo ; encoding: [0x7e,0xae,0x0a,0x7e] +# GFX11-REAL16: v_log_f16_e32 v5.l, exec_lo ; encoding: [0x7e,0xae,0x0a,0x7e] 0x7e,0xae,0x0a,0x7e -# GFX11-FAKE16: v_log_f16_e32 v5, exec_hi ; encoding: [0x7f,0xae,0x0a,0x7e] -# GFX11-REAL16: v_log_f16_e32 v5.l, exec_hi ; encoding: [0x7f,0xae,0x0a,0x7e] +# GFX11-FAKE16: v_log_f16_e32 v5, exec_hi ; encoding: [0x7f,0xae,0x0a,0x7e] +# GFX11-REAL16: v_log_f16_e32 v5.l, exec_hi ; encoding: [0x7f,0xae,0x0a,0x7e] 0x7f,0xae,0x0a,0x7e -# GFX11-FAKE16: v_log_f16_e32 v5, null ; encoding: [0x7c,0xae,0x0a,0x7e] -# GFX11-REAL16: v_log_f16_e32 v5.l, null ; encoding: [0x7c,0xae,0x0a,0x7e] +# GFX11-FAKE16: v_log_f16_e32 v5, null ; encoding: [0x7c,0xae,0x0a,0x7e] +# GFX11-REAL16: v_log_f16_e32 v5.l, null ; encoding: [0x7c,0xae,0x0a,0x7e] 0x7c,0xae,0x0a,0x7e -# GFX11-FAKE16: v_log_f16_e32 v5, -1 ; encoding: [0xc1,0xae,0x0a,0x7e] -# GFX11-REAL16: v_log_f16_e32 v5.l, -1 ; encoding: [0xc1,0xae,0x0a,0x7e] +# GFX11-FAKE16: v_log_f16_e32 v5, -1 ; encoding: [0xc1,0xae,0x0a,0x7e] +# GFX11-REAL16: v_log_f16_e32 v5.l, -1 ; encoding: [0xc1,0xae,0x0a,0x7e] 0xc1,0xae,0x0a,0x7e -# GFX11-FAKE16: v_log_f16_e32 v127, 0.5 ; encoding: [0xf0,0xae,0xfe,0x7e] -# GFX11-REAL16: v_log_f16_e32 v127.l, 0.5 ; encoding: [0xf0,0xae,0xfe,0x7e] +# GFX11-FAKE16: v_log_f16_e32 v127, 0.5 ; encoding: [0xf0,0xae,0xfe,0x7e] +# GFX11-REAL16: v_log_f16_e32 v127.l, 0.5 ; encoding: [0xf0,0xae,0xfe,0x7e] 0xf0,0xae,0xfe,0x7e # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_log_f16_e32 v5.h, src_scc ; encoding: [0xfd,0xae,0x0a,0x7f] +# GFX11-REAL16: v_log_f16_e32 v5.h, src_scc ; encoding: [0xfd,0xae,0x0a,0x7f] 0xfd,0xae,0x0a,0x7f # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_log_f16_e32 v127.h, 0xfe0b ; encoding: [0xff,0xae,0xfe,0x7f,0x0b,0xfe,0x00,0x00] +# GFX11-REAL16: v_log_f16_e32 v127.h, 0xfe0b ; encoding: [0xff,0xae,0xfe,0x7f,0x0b,0xfe,0x00,0x00] 0xff,0xae,0xfe,0x7f,0x0b,0xfe,0x00,0x00 # GFX11: v_log_f32_e32 v5, v1 ; encoding: [0x01,0x4f,0x0a,0x7e] @@ -2573,72 +2573,72 @@ # GFX11: v_pipeflush ; encoding: [0x00,0x36,0x00,0x7e] 0x00,0x36,0x00,0x7e -# GFX11-FAKE16: v_rcp_f16_e32 v5, v1 ; encoding: [0x01,0xa9,0x0a,0x7e] -# GFX11-REAL16: v_rcp_f16_e32 v5.l, v1.l ; encoding: [0x01,0xa9,0x0a,0x7e] +# GFX11-FAKE16: v_rcp_f16_e32 v5, v1 ; encoding: [0x01,0xa9,0x0a,0x7e] +# GFX11-REAL16: v_rcp_f16_e32 v5.l, v1.l ; encoding: [0x01,0xa9,0x0a,0x7e] 0x01,0xa9,0x0a,0x7e -# GFX11-FAKE16: v_rcp_f16_e32 v5, v127 ; encoding: [0x7f,0xa9,0x0a,0x7e] -# GFX11-REAL16: v_rcp_f16_e32 v5.l, v127.l ; encoding: [0x7f,0xa9,0x0a,0x7e] +# GFX11-FAKE16: v_rcp_f16_e32 v5, v127 ; encoding: [0x7f,0xa9,0x0a,0x7e] +# GFX11-REAL16: v_rcp_f16_e32 v5.l, v127.l ; encoding: [0x7f,0xa9,0x0a,0x7e] 0x7f,0xa9,0x0a,0x7e # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_rcp_f16_e32 v5.l, v1.h ; encoding: [0x81,0xa9,0x0a,0x7e] +# GFX11-REAL16: v_rcp_f16_e32 v5.l, v1.h ; encoding: [0x81,0xa9,0x0a,0x7e] 0x81,0xa9,0x0a,0x7e # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_rcp_f16_e32 v5.l, v127.h ; encoding: [0xff,0xa9,0x0a,0x7e] +# GFX11-REAL16: v_rcp_f16_e32 v5.l, v127.h ; encoding: [0xff,0xa9,0x0a,0x7e] 0xff,0xa9,0x0a,0x7e -# GFX11-FAKE16: v_rcp_f16_e32 v5, s1 ; encoding: [0x01,0xa8,0x0a,0x7e] -# GFX11-REAL16: v_rcp_f16_e32 v5.l, s1 ; encoding: [0x01,0xa8,0x0a,0x7e] +# GFX11-FAKE16: v_rcp_f16_e32 v5, s1 ; encoding: [0x01,0xa8,0x0a,0x7e] +# GFX11-REAL16: v_rcp_f16_e32 v5.l, s1 ; encoding: [0x01,0xa8,0x0a,0x7e] 0x01,0xa8,0x0a,0x7e -# GFX11-FAKE16: v_rcp_f16_e32 v5, s105 ; encoding: [0x69,0xa8,0x0a,0x7e] -# GFX11-REAL16: v_rcp_f16_e32 v5.l, s105 ; encoding: [0x69,0xa8,0x0a,0x7e] +# GFX11-FAKE16: v_rcp_f16_e32 v5, s105 ; encoding: [0x69,0xa8,0x0a,0x7e] +# GFX11-REAL16: v_rcp_f16_e32 v5.l, s105 ; encoding: [0x69,0xa8,0x0a,0x7e] 0x69,0xa8,0x0a,0x7e -# GFX11-FAKE16: v_rcp_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xa8,0x0a,0x7e] -# GFX11-REAL16: v_rcp_f16_e32 v5.l, vcc_lo ; encoding: [0x6a,0xa8,0x0a,0x7e] +# GFX11-FAKE16: v_rcp_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xa8,0x0a,0x7e] +# GFX11-REAL16: v_rcp_f16_e32 v5.l, vcc_lo ; encoding: [0x6a,0xa8,0x0a,0x7e] 0x6a,0xa8,0x0a,0x7e -# GFX11-FAKE16: v_rcp_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xa8,0x0a,0x7e] -# GFX11-REAL16: v_rcp_f16_e32 v5.l, vcc_hi ; encoding: [0x6b,0xa8,0x0a,0x7e] +# GFX11-FAKE16: v_rcp_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xa8,0x0a,0x7e] +# GFX11-REAL16: v_rcp_f16_e32 v5.l, vcc_hi ; encoding: [0x6b,0xa8,0x0a,0x7e] 0x6b,0xa8,0x0a,0x7e -# GFX11-FAKE16: v_rcp_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xa8,0x0a,0x7e] -# GFX11-REAL16: v_rcp_f16_e32 v5.l, ttmp15 ; encoding: [0x7b,0xa8,0x0a,0x7e] +# GFX11-FAKE16: v_rcp_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xa8,0x0a,0x7e] +# GFX11-REAL16: v_rcp_f16_e32 v5.l, ttmp15 ; encoding: [0x7b,0xa8,0x0a,0x7e] 0x7b,0xa8,0x0a,0x7e -# GFX11-FAKE16: v_rcp_f16_e32 v5, m0 ; encoding: [0x7d,0xa8,0x0a,0x7e] -# GFX11-REAL16: v_rcp_f16_e32 v5.l, m0 ; encoding: [0x7d,0xa8,0x0a,0x7e] +# GFX11-FAKE16: v_rcp_f16_e32 v5, m0 ; encoding: [0x7d,0xa8,0x0a,0x7e] +# GFX11-REAL16: v_rcp_f16_e32 v5.l, m0 ; encoding: [0x7d,0xa8,0x0a,0x7e] 0x7d,0xa8,0x0a,0x7e -# GFX11-FAKE16: v_rcp_f16_e32 v5, exec_lo ; encoding: [0x7e,0xa8,0x0a,0x7e] -# GFX11-REAL16: v_rcp_f16_e32 v5.l, exec_lo ; encoding: [0x7e,0xa8,0x0a,0x7e] +# GFX11-FAKE16: v_rcp_f16_e32 v5, exec_lo ; encoding: [0x7e,0xa8,0x0a,0x7e] +# GFX11-REAL16: v_rcp_f16_e32 v5.l, exec_lo ; encoding: [0x7e,0xa8,0x0a,0x7e] 0x7e,0xa8,0x0a,0x7e -# GFX11-FAKE16: v_rcp_f16_e32 v5, exec_hi ; encoding: [0x7f,0xa8,0x0a,0x7e] -# GFX11-REAL16: v_rcp_f16_e32 v5.l, exec_hi ; encoding: [0x7f,0xa8,0x0a,0x7e] +# GFX11-FAKE16: v_rcp_f16_e32 v5, exec_hi ; encoding: [0x7f,0xa8,0x0a,0x7e] +# GFX11-REAL16: v_rcp_f16_e32 v5.l, exec_hi ; encoding: [0x7f,0xa8,0x0a,0x7e] 0x7f,0xa8,0x0a,0x7e -# GFX11-FAKE16: v_rcp_f16_e32 v5, null ; encoding: [0x7c,0xa8,0x0a,0x7e] -# GFX11-REAL16: v_rcp_f16_e32 v5.l, null ; encoding: [0x7c,0xa8,0x0a,0x7e] +# GFX11-FAKE16: v_rcp_f16_e32 v5, null ; encoding: [0x7c,0xa8,0x0a,0x7e] +# GFX11-REAL16: v_rcp_f16_e32 v5.l, null ; encoding: [0x7c,0xa8,0x0a,0x7e] 0x7c,0xa8,0x0a,0x7e -# GFX11-FAKE16: v_rcp_f16_e32 v5, -1 ; encoding: [0xc1,0xa8,0x0a,0x7e] -# GFX11-REAL16: v_rcp_f16_e32 v5.l, -1 ; encoding: [0xc1,0xa8,0x0a,0x7e] +# GFX11-FAKE16: v_rcp_f16_e32 v5, -1 ; encoding: [0xc1,0xa8,0x0a,0x7e] +# GFX11-REAL16: v_rcp_f16_e32 v5.l, -1 ; encoding: [0xc1,0xa8,0x0a,0x7e] 0xc1,0xa8,0x0a,0x7e -# GFX11-FAKE16: v_rcp_f16_e32 v127, 0.5 ; encoding: [0xf0,0xa8,0xfe,0x7e] -# GFX11-REAL16: v_rcp_f16_e32 v127.l, 0.5 ; encoding: [0xf0,0xa8,0xfe,0x7e] +# GFX11-FAKE16: v_rcp_f16_e32 v127, 0.5 ; encoding: [0xf0,0xa8,0xfe,0x7e] +# GFX11-REAL16: v_rcp_f16_e32 v127.l, 0.5 ; encoding: [0xf0,0xa8,0xfe,0x7e] 0xf0,0xa8,0xfe,0x7e # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_rcp_f16_e32 v5.h, src_scc ; encoding: [0xfd,0xa8,0x0a,0x7f] +# GFX11-REAL16: v_rcp_f16_e32 v5.h, src_scc ; encoding: [0xfd,0xa8,0x0a,0x7f] 0xfd,0xa8,0x0a,0x7f # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_rcp_f16_e32 v127.h, 0xfe0b ; encoding: [0xff,0xa8,0xfe,0x7f,0x0b,0xfe,0x00,0x00] +# GFX11-REAL16: v_rcp_f16_e32 v127.h, 0xfe0b ; encoding: [0xff,0xa8,0xfe,0x7f,0x0b,0xfe,0x00,0x00] 0xff,0xa8,0xfe,0x7f,0x0b,0xfe,0x00,0x00 # GFX11: v_rcp_f32_e32 v5, v1 ; encoding: [0x01,0x55,0x0a,0x7e] @@ -2911,72 +2911,72 @@ # GFX11: v_rndne_f64_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x32,0xfc,0x7f,0x56,0x34,0x12,0xaf] 0xff,0x32,0xfc,0x7f,0x56,0x34,0x12,0xaf -# GFX11-FAKE16: v_rsq_f16_e32 v5, v1 ; encoding: [0x01,0xad,0x0a,0x7e] -# GFX11-REAL16: v_rsq_f16_e32 v5.l, v1.l ; encoding: [0x01,0xad,0x0a,0x7e] +# GFX11-FAKE16: v_rsq_f16_e32 v5, v1 ; encoding: [0x01,0xad,0x0a,0x7e] +# GFX11-REAL16: v_rsq_f16_e32 v5.l, v1.l ; encoding: [0x01,0xad,0x0a,0x7e] 0x01,0xad,0x0a,0x7e -# GFX11-FAKE16: v_rsq_f16_e32 v5, v127 ; encoding: [0x7f,0xad,0x0a,0x7e] -# GFX11-REAL16: v_rsq_f16_e32 v5.l, v127.l ; encoding: [0x7f,0xad,0x0a,0x7e] +# GFX11-FAKE16: v_rsq_f16_e32 v5, v127 ; encoding: [0x7f,0xad,0x0a,0x7e] +# GFX11-REAL16: v_rsq_f16_e32 v5.l, v127.l ; encoding: [0x7f,0xad,0x0a,0x7e] 0x7f,0xad,0x0a,0x7e # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_rsq_f16_e32 v5.l, v1.h ; encoding: [0x81,0xad,0x0a,0x7e] +# GFX11-REAL16: v_rsq_f16_e32 v5.l, v1.h ; encoding: [0x81,0xad,0x0a,0x7e] 0x81,0xad,0x0a,0x7e # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_rsq_f16_e32 v5.l, v127.h ; encoding: [0xff,0xad,0x0a,0x7e] +# GFX11-REAL16: v_rsq_f16_e32 v5.l, v127.h ; encoding: [0xff,0xad,0x0a,0x7e] 0xff,0xad,0x0a,0x7e -# GFX11-FAKE16: v_rsq_f16_e32 v5, s1 ; encoding: [0x01,0xac,0x0a,0x7e] -# GFX11-REAL16: v_rsq_f16_e32 v5.l, s1 ; encoding: [0x01,0xac,0x0a,0x7e] +# GFX11-FAKE16: v_rsq_f16_e32 v5, s1 ; encoding: [0x01,0xac,0x0a,0x7e] +# GFX11-REAL16: v_rsq_f16_e32 v5.l, s1 ; encoding: [0x01,0xac,0x0a,0x7e] 0x01,0xac,0x0a,0x7e -# GFX11-FAKE16: v_rsq_f16_e32 v5, s105 ; encoding: [0x69,0xac,0x0a,0x7e] -# GFX11-REAL16: v_rsq_f16_e32 v5.l, s105 ; encoding: [0x69,0xac,0x0a,0x7e] +# GFX11-FAKE16: v_rsq_f16_e32 v5, s105 ; encoding: [0x69,0xac,0x0a,0x7e] +# GFX11-REAL16: v_rsq_f16_e32 v5.l, s105 ; encoding: [0x69,0xac,0x0a,0x7e] 0x69,0xac,0x0a,0x7e -# GFX11-FAKE16: v_rsq_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xac,0x0a,0x7e] -# GFX11-REAL16: v_rsq_f16_e32 v5.l, vcc_lo ; encoding: [0x6a,0xac,0x0a,0x7e] +# GFX11-FAKE16: v_rsq_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xac,0x0a,0x7e] +# GFX11-REAL16: v_rsq_f16_e32 v5.l, vcc_lo ; encoding: [0x6a,0xac,0x0a,0x7e] 0x6a,0xac,0x0a,0x7e -# GFX11-FAKE16: v_rsq_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xac,0x0a,0x7e] -# GFX11-REAL16: v_rsq_f16_e32 v5.l, vcc_hi ; encoding: [0x6b,0xac,0x0a,0x7e] +# GFX11-FAKE16: v_rsq_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xac,0x0a,0x7e] +# GFX11-REAL16: v_rsq_f16_e32 v5.l, vcc_hi ; encoding: [0x6b,0xac,0x0a,0x7e] 0x6b,0xac,0x0a,0x7e -# GFX11-FAKE16: v_rsq_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xac,0x0a,0x7e] -# GFX11-REAL16: v_rsq_f16_e32 v5.l, ttmp15 ; encoding: [0x7b,0xac,0x0a,0x7e] +# GFX11-FAKE16: v_rsq_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xac,0x0a,0x7e] +# GFX11-REAL16: v_rsq_f16_e32 v5.l, ttmp15 ; encoding: [0x7b,0xac,0x0a,0x7e] 0x7b,0xac,0x0a,0x7e -# GFX11-FAKE16: v_rsq_f16_e32 v5, m0 ; encoding: [0x7d,0xac,0x0a,0x7e] -# GFX11-REAL16: v_rsq_f16_e32 v5.l, m0 ; encoding: [0x7d,0xac,0x0a,0x7e] +# GFX11-FAKE16: v_rsq_f16_e32 v5, m0 ; encoding: [0x7d,0xac,0x0a,0x7e] +# GFX11-REAL16: v_rsq_f16_e32 v5.l, m0 ; encoding: [0x7d,0xac,0x0a,0x7e] 0x7d,0xac,0x0a,0x7e -# GFX11-FAKE16: v_rsq_f16_e32 v5, exec_lo ; encoding: [0x7e,0xac,0x0a,0x7e] -# GFX11-REAL16: v_rsq_f16_e32 v5.l, exec_lo ; encoding: [0x7e,0xac,0x0a,0x7e] +# GFX11-FAKE16: v_rsq_f16_e32 v5, exec_lo ; encoding: [0x7e,0xac,0x0a,0x7e] +# GFX11-REAL16: v_rsq_f16_e32 v5.l, exec_lo ; encoding: [0x7e,0xac,0x0a,0x7e] 0x7e,0xac,0x0a,0x7e -# GFX11-FAKE16: v_rsq_f16_e32 v5, exec_hi ; encoding: [0x7f,0xac,0x0a,0x7e] -# GFX11-REAL16: v_rsq_f16_e32 v5.l, exec_hi ; encoding: [0x7f,0xac,0x0a,0x7e] +# GFX11-FAKE16: v_rsq_f16_e32 v5, exec_hi ; encoding: [0x7f,0xac,0x0a,0x7e] +# GFX11-REAL16: v_rsq_f16_e32 v5.l, exec_hi ; encoding: [0x7f,0xac,0x0a,0x7e] 0x7f,0xac,0x0a,0x7e -# GFX11-FAKE16: v_rsq_f16_e32 v5, null ; encoding: [0x7c,0xac,0x0a,0x7e] -# GFX11-REAL16: v_rsq_f16_e32 v5.l, null ; encoding: [0x7c,0xac,0x0a,0x7e] +# GFX11-FAKE16: v_rsq_f16_e32 v5, null ; encoding: [0x7c,0xac,0x0a,0x7e] +# GFX11-REAL16: v_rsq_f16_e32 v5.l, null ; encoding: [0x7c,0xac,0x0a,0x7e] 0x7c,0xac,0x0a,0x7e -# GFX11-FAKE16: v_rsq_f16_e32 v5, -1 ; encoding: [0xc1,0xac,0x0a,0x7e] -# GFX11-REAL16: v_rsq_f16_e32 v5.l, -1 ; encoding: [0xc1,0xac,0x0a,0x7e] +# GFX11-FAKE16: v_rsq_f16_e32 v5, -1 ; encoding: [0xc1,0xac,0x0a,0x7e] +# GFX11-REAL16: v_rsq_f16_e32 v5.l, -1 ; encoding: [0xc1,0xac,0x0a,0x7e] 0xc1,0xac,0x0a,0x7e -# GFX11-FAKE16: v_rsq_f16_e32 v127, 0.5 ; encoding: [0xf0,0xac,0xfe,0x7e] -# GFX11-REAL16: v_rsq_f16_e32 v127.l, 0.5 ; encoding: [0xf0,0xac,0xfe,0x7e] +# GFX11-FAKE16: v_rsq_f16_e32 v127, 0.5 ; encoding: [0xf0,0xac,0xfe,0x7e] +# GFX11-REAL16: v_rsq_f16_e32 v127.l, 0.5 ; encoding: [0xf0,0xac,0xfe,0x7e] 0xf0,0xac,0xfe,0x7e # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_rsq_f16_e32 v5.h, src_scc ; encoding: [0xfd,0xac,0x0a,0x7f] +# GFX11-REAL16: v_rsq_f16_e32 v5.h, src_scc ; encoding: [0xfd,0xac,0x0a,0x7f] 0xfd,0xac,0x0a,0x7f # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_rsq_f16_e32 v127.h, 0xfe0b ; encoding: [0xff,0xac,0xfe,0x7f,0x0b,0xfe,0x00,0x00] +# GFX11-REAL16: v_rsq_f16_e32 v127.h, 0xfe0b ; encoding: [0xff,0xac,0xfe,0x7f,0x0b,0xfe,0x00,0x00] 0xff,0xac,0xfe,0x7f,0x0b,0xfe,0x00,0x00 # GFX11: v_rsq_f32_e32 v5, v1 ; encoding: [0x01,0x5d,0x0a,0x7e] @@ -3195,72 +3195,72 @@ # GFX11: v_sin_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x6a,0xfe,0x7f,0x56,0x34,0x12,0xaf] 0xff,0x6a,0xfe,0x7f,0x56,0x34,0x12,0xaf -# GFX11-FAKE16: v_sqrt_f16_e32 v5, v1 ; encoding: [0x01,0xab,0x0a,0x7e] -# GFX11-REAL16: v_sqrt_f16_e32 v5.l, v1.l ; encoding: [0x01,0xab,0x0a,0x7e] +# GFX11-FAKE16: v_sqrt_f16_e32 v5, v1 ; encoding: [0x01,0xab,0x0a,0x7e] +# GFX11-REAL16: v_sqrt_f16_e32 v5.l, v1.l ; encoding: [0x01,0xab,0x0a,0x7e] 0x01,0xab,0x0a,0x7e -# GFX11-FAKE16: v_sqrt_f16_e32 v5, v127 ; encoding: [0x7f,0xab,0x0a,0x7e] -# GFX11-REAL16: v_sqrt_f16_e32 v5.l, v127.l ; encoding: [0x7f,0xab,0x0a,0x7e] +# GFX11-FAKE16: v_sqrt_f16_e32 v5, v127 ; encoding: [0x7f,0xab,0x0a,0x7e] +# GFX11-REAL16: v_sqrt_f16_e32 v5.l, v127.l ; encoding: [0x7f,0xab,0x0a,0x7e] 0x7f,0xab,0x0a,0x7e # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_sqrt_f16_e32 v5.l, v1.h ; encoding: [0x81,0xab,0x0a,0x7e] +# GFX11-REAL16: v_sqrt_f16_e32 v5.l, v1.h ; encoding: [0x81,0xab,0x0a,0x7e] 0x81,0xab,0x0a,0x7e # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_sqrt_f16_e32 v5.l, v127.h ; encoding: [0xff,0xab,0x0a,0x7e] +# GFX11-REAL16: v_sqrt_f16_e32 v5.l, v127.h ; encoding: [0xff,0xab,0x0a,0x7e] 0xff,0xab,0x0a,0x7e -# GFX11-FAKE16: v_sqrt_f16_e32 v5, s1 ; encoding: [0x01,0xaa,0x0a,0x7e] -# GFX11-REAL16: v_sqrt_f16_e32 v5.l, s1 ; encoding: [0x01,0xaa,0x0a,0x7e] +# GFX11-FAKE16: v_sqrt_f16_e32 v5, s1 ; encoding: [0x01,0xaa,0x0a,0x7e] +# GFX11-REAL16: v_sqrt_f16_e32 v5.l, s1 ; encoding: [0x01,0xaa,0x0a,0x7e] 0x01,0xaa,0x0a,0x7e -# GFX11-FAKE16: v_sqrt_f16_e32 v5, s105 ; encoding: [0x69,0xaa,0x0a,0x7e] -# GFX11-REAL16: v_sqrt_f16_e32 v5.l, s105 ; encoding: [0x69,0xaa,0x0a,0x7e] +# GFX11-FAKE16: v_sqrt_f16_e32 v5, s105 ; encoding: [0x69,0xaa,0x0a,0x7e] +# GFX11-REAL16: v_sqrt_f16_e32 v5.l, s105 ; encoding: [0x69,0xaa,0x0a,0x7e] 0x69,0xaa,0x0a,0x7e -# GFX11-FAKE16: v_sqrt_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xaa,0x0a,0x7e] -# GFX11-REAL16: v_sqrt_f16_e32 v5.l, vcc_lo ; encoding: [0x6a,0xaa,0x0a,0x7e] +# GFX11-FAKE16: v_sqrt_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xaa,0x0a,0x7e] +# GFX11-REAL16: v_sqrt_f16_e32 v5.l, vcc_lo ; encoding: [0x6a,0xaa,0x0a,0x7e] 0x6a,0xaa,0x0a,0x7e -# GFX11-FAKE16: v_sqrt_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xaa,0x0a,0x7e] -# GFX11-REAL16: v_sqrt_f16_e32 v5.l, vcc_hi ; encoding: [0x6b,0xaa,0x0a,0x7e] +# GFX11-FAKE16: v_sqrt_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xaa,0x0a,0x7e] +# GFX11-REAL16: v_sqrt_f16_e32 v5.l, vcc_hi ; encoding: [0x6b,0xaa,0x0a,0x7e] 0x6b,0xaa,0x0a,0x7e -# GFX11-FAKE16: v_sqrt_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xaa,0x0a,0x7e] -# GFX11-REAL16: v_sqrt_f16_e32 v5.l, ttmp15 ; encoding: [0x7b,0xaa,0x0a,0x7e] +# GFX11-FAKE16: v_sqrt_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xaa,0x0a,0x7e] +# GFX11-REAL16: v_sqrt_f16_e32 v5.l, ttmp15 ; encoding: [0x7b,0xaa,0x0a,0x7e] 0x7b,0xaa,0x0a,0x7e -# GFX11-FAKE16: v_sqrt_f16_e32 v5, m0 ; encoding: [0x7d,0xaa,0x0a,0x7e] -# GFX11-REAL16: v_sqrt_f16_e32 v5.l, m0 ; encoding: [0x7d,0xaa,0x0a,0x7e] +# GFX11-FAKE16: v_sqrt_f16_e32 v5, m0 ; encoding: [0x7d,0xaa,0x0a,0x7e] +# GFX11-REAL16: v_sqrt_f16_e32 v5.l, m0 ; encoding: [0x7d,0xaa,0x0a,0x7e] 0x7d,0xaa,0x0a,0x7e -# GFX11-FAKE16: v_sqrt_f16_e32 v5, exec_lo ; encoding: [0x7e,0xaa,0x0a,0x7e] -# GFX11-REAL16: v_sqrt_f16_e32 v5.l, exec_lo ; encoding: [0x7e,0xaa,0x0a,0x7e] +# GFX11-FAKE16: v_sqrt_f16_e32 v5, exec_lo ; encoding: [0x7e,0xaa,0x0a,0x7e] +# GFX11-REAL16: v_sqrt_f16_e32 v5.l, exec_lo ; encoding: [0x7e,0xaa,0x0a,0x7e] 0x7e,0xaa,0x0a,0x7e -# GFX11-FAKE16: v_sqrt_f16_e32 v5, exec_hi ; encoding: [0x7f,0xaa,0x0a,0x7e] -# GFX11-REAL16: v_sqrt_f16_e32 v5.l, exec_hi ; encoding: [0x7f,0xaa,0x0a,0x7e] +# GFX11-FAKE16: v_sqrt_f16_e32 v5, exec_hi ; encoding: [0x7f,0xaa,0x0a,0x7e] +# GFX11-REAL16: v_sqrt_f16_e32 v5.l, exec_hi ; encoding: [0x7f,0xaa,0x0a,0x7e] 0x7f,0xaa,0x0a,0x7e -# GFX11-FAKE16: v_sqrt_f16_e32 v5, null ; encoding: [0x7c,0xaa,0x0a,0x7e] -# GFX11-REAL16: v_sqrt_f16_e32 v5.l, null ; encoding: [0x7c,0xaa,0x0a,0x7e] +# GFX11-FAKE16: v_sqrt_f16_e32 v5, null ; encoding: [0x7c,0xaa,0x0a,0x7e] +# GFX11-REAL16: v_sqrt_f16_e32 v5.l, null ; encoding: [0x7c,0xaa,0x0a,0x7e] 0x7c,0xaa,0x0a,0x7e -# GFX11-FAKE16: v_sqrt_f16_e32 v5, -1 ; encoding: [0xc1,0xaa,0x0a,0x7e] -# GFX11-REAL16: v_sqrt_f16_e32 v5.l, -1 ; encoding: [0xc1,0xaa,0x0a,0x7e] +# GFX11-FAKE16: v_sqrt_f16_e32 v5, -1 ; encoding: [0xc1,0xaa,0x0a,0x7e] +# GFX11-REAL16: v_sqrt_f16_e32 v5.l, -1 ; encoding: [0xc1,0xaa,0x0a,0x7e] 0xc1,0xaa,0x0a,0x7e -# GFX11-FAKE16: v_sqrt_f16_e32 v127, 0.5 ; encoding: [0xf0,0xaa,0xfe,0x7e] -# GFX11-REAL16: v_sqrt_f16_e32 v127.l, 0.5 ; encoding: [0xf0,0xaa,0xfe,0x7e] +# GFX11-FAKE16: v_sqrt_f16_e32 v127, 0.5 ; encoding: [0xf0,0xaa,0xfe,0x7e] +# GFX11-REAL16: v_sqrt_f16_e32 v127.l, 0.5 ; encoding: [0xf0,0xaa,0xfe,0x7e] 0xf0,0xaa,0xfe,0x7e # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_sqrt_f16_e32 v5.h, src_scc ; encoding: [0xfd,0xaa,0x0a,0x7f] +# GFX11-REAL16: v_sqrt_f16_e32 v5.h, src_scc ; encoding: [0xfd,0xaa,0x0a,0x7f] 0xfd,0xaa,0x0a,0x7f # COM: GFX11-FAKE16: warning: invalid instruction encoding -# GFX11-REAL16: v_sqrt_f16_e32 v127.h, 0xfe0b ; encoding: [0xff,0xaa,0xfe,0x7f,0x0b,0xfe,0x00,0x00] +# GFX11-REAL16: v_sqrt_f16_e32 v127.h, 0xfe0b ; encoding: [0xff,0xaa,0xfe,0x7f,0x0b,0xfe,0x00,0x00] 0xff,0xaa,0xfe,0x7f,0x0b,0xfe,0x00,0x00 # GFX11: v_sqrt_f32_e32 v5, v1 ; encoding: [0x01,0x67,0x0a,0x7e] diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_from_vop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_from_vop1.txt index cb6c2ac768c2..139c097a2223 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_from_vop1.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_from_vop1.txt @@ -1,5 +1,7 @@ -# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,GFX11-REAL16 %s -# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+real-true16,+wavefrontsize32,-wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefixes=GFX11,GFX11-REAL16 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+real-true16,-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefixes=GFX11,GFX11-REAL16 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16,+wavefrontsize32,-wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefixes=GFX11,GFX11-FAKE16 %s +# R UN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16,-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefixes=GFX11,GFX11-FAKE16 %s # GFX11: v_bfrev_b32_e64 v5, v1 ; encoding: [0x05,0x00,0xb8,0xd5,0x01,0x01,0x00,0x00] 0x05,0x00,0xb8,0xd5,0x01,0x01,0x00,0x00 @@ -46,60 +48,60 @@ # GFX11: v_bfrev_b32_e64 v255, 0xaf123456 ; encoding: [0xff,0x00,0xb8,0xd5,0xff,0x00,0x00,0x00,0x56,0x34,0x12,0xaf] 0xff,0x00,0xb8,0xd5,0xff,0x00,0x00,0x00,0x56,0x34,0x12,0xaf -# GFX11-REAL16: v_ceil_f16_e64 v5.l, v1.l ; encoding: [0x05,0x00,0xdc,0xd5,0x01,0x01,0x00,0x00] -# GFX11-FAKE16: v_ceil_f16_e64 v5, v1 ; encoding: [0x05,0x00,0xdc,0xd5,0x01,0x01,0x00,0x00] +# GFX11-REAL16: v_ceil_f16_e64 v5.l, v1.l ; encoding: [0x05,0x00,0xdc,0xd5,0x01,0x01,0x00,0x00] +# GFX11-FAKE16: v_ceil_f16_e64 v5, v1 ; encoding: [0x05,0x00,0xdc,0xd5,0x01,0x01,0x00,0x00] 0x05,0x00,0xdc,0xd5,0x01,0x01,0x00,0x00 -# GFX11-REAL16: v_ceil_f16_e64 v5.l, v255.l ; encoding: [0x05,0x00,0xdc,0xd5,0xff,0x01,0x00,0x00] -# GFX11-FAKE16: v_ceil_f16_e64 v5, v255 ; encoding: [0x05,0x00,0xdc,0xd5,0xff,0x01,0x00,0x00] +# GFX11-REAL16: v_ceil_f16_e64 v5.l, v255.l ; encoding: [0x05,0x00,0xdc,0xd5,0xff,0x01,0x00,0x00] +# GFX11-FAKE16: v_ceil_f16_e64 v5, v255 ; encoding: [0x05,0x00,0xdc,0xd5,0xff,0x01,0x00,0x00] 0x05,0x00,0xdc,0xd5,0xff,0x01,0x00,0x00 -# GFX11-REAL16: v_ceil_f16_e64 v5.l, s1 ; encoding: [0x05,0x00,0xdc,0xd5,0x01,0x00,0x00,0x00] -# GFX11-FAKE16: v_ceil_f16_e64 v5, s1 ; encoding: [0x05,0x00,0xdc,0xd5,0x01,0x00,0x00,0x00] +# GFX11-REAL16: v_ceil_f16_e64 v5.l, s1 ; encoding: [0x05,0x00,0xdc,0xd5,0x01,0x00,0x00,0x00] +# GFX11-FAKE16: v_ceil_f16_e64 v5, s1 ; encoding: [0x05,0x00,0xdc,0xd5,0x01,0x00,0x00,0x00] 0x05,0x00,0xdc,0xd5,0x01,0x00,0x00,0x00 -# GFX11-REAL16: v_ceil_f16_e64 v5.l, s105 ; encoding: [0x05,0x00,0xdc,0xd5,0x69,0x00,0x00,0x00] -# GFX11-FAKE16: v_ceil_f16_e64 v5, s105 ; encoding: [0x05,0x00,0xdc,0xd5,0x69,0x00,0x00,0x00] +# GFX11-REAL16: v_ceil_f16_e64 v5.l, s105 ; encoding: [0x05,0x00,0xdc,0xd5,0x69,0x00,0x00,0x00] +# GFX11-FAKE16: v_ceil_f16_e64 v5, s105 ; encoding: [0x05,0x00,0xdc,0xd5,0x69,0x00,0x00,0x00] 0x05,0x00,0xdc,0xd5,0x69,0x00,0x00,0x00 -# GFX11-REAL16: v_ceil_f16_e64 v5.l, vcc_lo ; encoding: [0x05,0x00,0xdc,0xd5,0x6a,0x00,0x00,0x00] -# GFX11-FAKE16: v_ceil_f16_e64 v5, vcc_lo ; encoding: [0x05,0x00,0xdc,0xd5,0x6a,0x00,0x00,0x00] +# GFX11-REAL16: v_ceil_f16_e64 v5.l, vcc_lo ; encoding: [0x05,0x00,0xdc,0xd5,0x6a,0x00,0x00,0x00] +# GFX11-FAKE16: v_ceil_f16_e64 v5, vcc_lo ; encoding: [0x05,0x00,0xdc,0xd5,0x6a,0x00,0x00,0x00] 0x05,0x00,0xdc,0xd5,0x6a,0x00,0x00,0x00 -# GFX11-REAL16: v_ceil_f16_e64 v5.l, vcc_hi ; encoding: [0x05,0x00,0xdc,0xd5,0x6b,0x00,0x00,0x00] -# GFX11-FAKE16: v_ceil_f16_e64 v5, vcc_hi ; encoding: [0x05,0x00,0xdc,0xd5,0x6b,0x00,0x00,0x00] +# GFX11-REAL16: v_ceil_f16_e64 v5.l, vcc_hi ; encoding: [0x05,0x00,0xdc,0xd5,0x6b,0x00,0x00,0x00] +# GFX11-FAKE16: v_ceil_f16_e64 v5, vcc_hi ; encoding: [0x05,0x00,0xdc,0xd5,0x6b,0x00,0x00,0x00] 0x05,0x00,0xdc,0xd5,0x6b,0x00,0x00,0x00 -# GFX11-REAL16: v_ceil_f16_e64 v5.l, ttmp15 ; encoding: [0x05,0x00,0xdc,0xd5,0x7b,0x00,0x00,0x00] -# GFX11-FAKE16: v_ceil_f16_e64 v5, ttmp15 ; encoding: [0x05,0x00,0xdc,0xd5,0x7b,0x00,0x00,0x00] +# GFX11-REAL16: v_ceil_f16_e64 v5.l, ttmp15 ; encoding: [0x05,0x00,0xdc,0xd5,0x7b,0x00,0x00,0x00] +# GFX11-FAKE16: v_ceil_f16_e64 v5, ttmp15 ; encoding: [0x05,0x00,0xdc,0xd5,0x7b,0x00,0x00,0x00] 0x05,0x00,0xdc,0xd5,0x7b,0x00,0x00,0x00 -# GFX11-REAL16: v_ceil_f16_e64 v5.l, m0 ; encoding: [0x05,0x00,0xdc,0xd5,0x7d,0x00,0x00,0x00] -# GFX11-FAKE16: v_ceil_f16_e64 v5, m0 ; encoding: [0x05,0x00,0xdc,0xd5,0x7d,0x00,0x00,0x00] +# GFX11-REAL16: v_ceil_f16_e64 v5.l, m0 ; encoding: [0x05,0x00,0xdc,0xd5,0x7d,0x00,0x00,0x00] +# GFX11-FAKE16: v_ceil_f16_e64 v5, m0 ; encoding: [0x05,0x00,0xdc,0xd5,0x7d,0x00,0x00,0x00] 0x05,0x00,0xdc,0xd5,0x7d,0x00,0x00,0x00 -# GFX11-REAL16: v_ceil_f16_e64 v5.l, exec_lo ; encoding: [0x05,0x00,0xdc,0xd5,0x7e,0x00,0x00,0x00] -# GFX11-FAKE16: v_ceil_f16_e64 v5, exec_lo ; encoding: [0x05,0x00,0xdc,0xd5,0x7e,0x00,0x00,0x00] +# GFX11-REAL16: v_ceil_f16_e64 v5.l, exec_lo ; encoding: [0x05,0x00,0xdc,0xd5,0x7e,0x00,0x00,0x00] +# GFX11-FAKE16: v_ceil_f16_e64 v5, exec_lo ; encoding: [0x05,0x00,0xdc,0xd5,0x7e,0x00,0x00,0x00] 0x05,0x00,0xdc,0xd5,0x7e,0x00,0x00,0x00 -# GFX11-REAL16: v_ceil_f16_e64 v5.l, exec_hi ; encoding: [0x05,0x00,0xdc,0xd5,0x7f,0x00,0x00,0x00] -# GFX11-FAKE16: v_ceil_f16_e64 v5, exec_hi ; encoding: [0x05,0x00,0xdc,0xd5,0x7f,0x00,0x00,0x00] +# GFX11-REAL16: v_ceil_f16_e64 v5.l, exec_hi ; encoding: [0x05,0x00,0xdc,0xd5,0x7f,0x00,0x00,0x00] +# GFX11-FAKE16: v_ceil_f16_e64 v5, exec_hi ; encoding: [0x05,0x00,0xdc,0xd5,0x7f,0x00,0x00,0x00] 0x05,0x00,0xdc,0xd5,0x7f,0x00,0x00,0x00 -# GFX11-REAL16: v_ceil_f16_e64 v5.l, null ; encoding: [0x05,0x00,0xdc,0xd5,0x7c,0x00,0x00,0x00] -# GFX11-FAKE16: v_ceil_f16_e64 v5, null ; encoding: [0x05,0x00,0xdc,0xd5,0x7c,0x00,0x00,0x00] +# GFX11-REAL16: v_ceil_f16_e64 v5.l, null ; encoding: [0x05,0x00,0xdc,0xd5,0x7c,0x00,0x00,0x00] +# GFX11-FAKE16: v_ceil_f16_e64 v5, null ; encoding: [0x05,0x00,0xdc,0xd5,0x7c,0x00,0x00,0x00] 0x05,0x00,0xdc,0xd5,0x7c,0x00,0x00,0x00 -# GFX11-REAL16: v_ceil_f16_e64 v5.l, -1 ; encoding: [0x05,0x00,0xdc,0xd5,0xc1,0x00,0x00,0x00] -# GFX11-FAKE16: v_ceil_f16_e64 v5, -1 ; encoding: [0x05,0x00,0xdc,0xd5,0xc1,0x00,0x00,0x00] +# GFX11-REAL16: v_ceil_f16_e64 v5.l, -1 ; encoding: [0x05,0x00,0xdc,0xd5,0xc1,0x00,0x00,0x00] +# GFX11-FAKE16: v_ceil_f16_e64 v5, -1 ; encoding: [0x05,0x00,0xdc,0xd5,0xc1,0x00,0x00,0x00] 0x05,0x00,0xdc,0xd5,0xc1,0x00,0x00,0x00 -# GFX11-REAL16: v_ceil_f16_e64 v5.l, 0.5 mul:2 ; encoding: [0x05,0x00,0xdc,0xd5,0xf0,0x00,0x00,0x08] -# GFX11-FAKE16: v_ceil_f16_e64 v5, 0.5 mul:2 ; encoding: [0x05,0x00,0xdc,0xd5,0xf0,0x00,0x00,0x08] +# GFX11-REAL16: v_ceil_f16_e64 v5.l, 0.5 mul:2 ; encoding: [0x05,0x00,0xdc,0xd5,0xf0,0x00,0x00,0x08] +# GFX11-FAKE16: v_ceil_f16_e64 v5, 0.5 mul:2 ; encoding: [0x05,0x00,0xdc,0xd5,0xf0,0x00,0x00,0x08] 0x05,0x00,0xdc,0xd5,0xf0,0x00,0x00,0x08 -# GFX11-REAL16: v_ceil_f16_e64 v5.l, src_scc mul:4 ; encoding: [0x05,0x00,0xdc,0xd5,0xfd,0x00,0x00,0x10] -# GFX11-FAKE16: v_ceil_f16_e64 v5, src_scc mul:4 ; encoding: [0x05,0x00,0xdc,0xd5,0xfd,0x00,0x00,0x10] +# GFX11-REAL16: v_ceil_f16_e64 v5.l, src_scc mul:4 ; encoding: [0x05,0x00,0xdc,0xd5,0xfd,0x00,0x00,0x10] +# GFX11-FAKE16: v_ceil_f16_e64 v5, src_scc mul:4 ; encoding: [0x05,0x00,0xdc,0xd5,0xfd,0x00,0x00,0x10] 0x05,0x00,0xdc,0xd5,0xfd,0x00,0x00,0x10 # GFX11-REAL16: v_ceil_f16_e64 v255.l, -|0xfe0b| clamp div:2 ; encoding: [0xff,0x81,0xdc,0xd5,0xff,0x00,0x00,0x38,0x0b,0xfe,0x00,0x00] @@ -1600,60 +1602,60 @@ # GFX11: v_cvt_u32_u16_e64 v255, 0xfe0b ; encoding: [0xff,0x00,0xeb,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00] 0xff,0x00,0xeb,0xd5,0xff,0x00,0x00,0x00,0x0b,0xfe,0x00,0x00 -# GFX11-REAL16: v_exp_f16_e64 v5.l, v1.l ; encoding: [0x05,0x00,0xd8,0xd5,0x01,0x01,0x00,0x00] -# GFX11-FAKE16: v_exp_f16_e64 v5, v1 ; encoding: [0x05,0x00,0xd8,0xd5,0x01,0x01,0x00,0x00] +# GFX11-REAL16: v_exp_f16_e64 v5.l, v1.l ; encoding: [0x05,0x00,0xd8,0xd5,0x01,0x01,0x00,0x00] +# GFX11-FAKE16: v_exp_f16_e64 v5, v1 ; encoding: [0x05,0x00,0xd8,0xd5,0x01,0x01,0x00,0x00] 0x05,0x00,0xd8,0xd5,0x01,0x01,0x00,0x00 -# GFX11-REAL16: v_exp_f16_e64 v5.l, v255.l ; encoding: [0x05,0x00,0xd8,0xd5,0xff,0x01,0x00,0x00] -# GFX11-FAKE16: v_exp_f16_e64 v5, v255 ; encoding: [0x05,0x00,0xd8,0xd5,0xff,0x01,0x00,0x00] +# GFX11-REAL16: v_exp_f16_e64 v5.l, v255.l ; encoding: [0x05,0x00,0xd8,0xd5,0xff,0x01,0x00,0x00] +# GFX11-FAKE16: v_exp_f16_e64 v5, v255 ; encoding: [0x05,0x00,0xd8,0xd5,0xff,0x01,0x00,0x00] 0x05,0x00,0xd8,0xd5,0xff,0x01,0x00,0x00 -# GFX11-REAL16: v_exp_f16_e64 v5.l, s1 ; encoding: [0x05,0x00,0xd8,0xd5,0x01,0x00,0x00,0x00] -# GFX11-FAKE16: v_exp_f16_e64 v5, s1 ; encoding: [0x05,0x00,0xd8,0xd5,0x01,0x00,0x00,0x00] +# GFX11-REAL16: v_exp_f16_e64 v5.l, s1 ; encoding: [0x05,0x00,0xd8,0xd5,0x01,0x00,0x00,0x00] +# GFX11-FAKE16: v_exp_f16_e64 v5, s1 ; encoding: [0x05,0x00,0xd8,0xd5,0x01,0x00,0x00,0x00] 0x05,0x00,0xd8,0xd5,0x01,0x00,0x00,0x00 -# GFX11-REAL16: v_exp_f16_e64 v5.l, s105 ; encoding: [0x05,0x00,0xd8,0xd5,0x69,0x00,0x00,0x00] -# GFX11-FAKE16: v_exp_f16_e64 v5, s105 ; encoding: [0x05,0x00,0xd8,0xd5,0x69,0x00,0x00,0x00] +# GFX11-REAL16: v_exp_f16_e64 v5.l, s105 ; encoding: [0x05,0x00,0xd8,0xd5,0x69,0x00,0x00,0x00] +# GFX11-FAKE16: v_exp_f16_e64 v5, s105 ; encoding: [0x05,0x00,0xd8,0xd5,0x69,0x00,0x00,0x00] 0x05,0x00,0xd8,0xd5,0x69,0x00,0x00,0x00 -# GFX11-REAL16: v_exp_f16_e64 v5.l, vcc_lo ; encoding: [0x05,0x00,0xd8,0xd5,0x6a,0x00,0x00,0x00] -# GFX11-FAKE16: v_exp_f16_e64 v5, vcc_lo ; encoding: [0x05,0x00,0xd8,0xd5,0x6a,0x00,0x00,0x00] +# GFX11-REAL16: v_exp_f16_e64 v5.l, vcc_lo ; encoding: [0x05,0x00,0xd8,0xd5,0x6a,0x00,0x00,0x00] +# GFX11-FAKE16: v_exp_f16_e64 v5, vcc_lo ; encoding: [0x05,0x00,0xd8,0xd5,0x6a,0x00,0x00,0x00] 0x05,0x00,0xd8,0xd5,0x6a,0x00,0x00,0x00 -# GFX11-REAL16: v_exp_f16_e64 v5.l, vcc_hi ; encoding: [0x05,0x00,0xd8,0xd5,0x6b,0x00,0x00,0x00] -# GFX11-FAKE16: v_exp_f16_e64 v5, vcc_hi ; encoding: [0x05,0x00,0xd8,0xd5,0x6b,0x00,0x00,0x00] +# GFX11-REAL16: v_exp_f16_e64 v5.l, vcc_hi ; encoding: [0x05,0x00,0xd8,0xd5,0x6b,0x00,0x00,0x00] +# GFX11-FAKE16: v_exp_f16_e64 v5, vcc_hi ; encoding: [0x05,0x00,0xd8,0xd5,0x6b,0x00,0x00,0x00] 0x05,0x00,0xd8,0xd5,0x6b,0x00,0x00,0x00 -# GFX11-REAL16: v_exp_f16_e64 v5.l, ttmp15 ; encoding: [0x05,0x00,0xd8,0xd5,0x7b,0x00,0x00,0x00] -# GFX11-FAKE16: v_exp_f16_e64 v5, ttmp15 ; encoding: [0x05,0x00,0xd8,0xd5,0x7b,0x00,0x00,0x00] +# GFX11-REAL16: v_exp_f16_e64 v5.l, ttmp15 ; encoding: [0x05,0x00,0xd8,0xd5,0x7b,0x00,0x00,0x00] +# GFX11-FAKE16: v_exp_f16_e64 v5, ttmp15 ; encoding: [0x05,0x00,0xd8,0xd5,0x7b,0x00,0x00,0x00] 0x05,0x00,0xd8,0xd5,0x7b,0x00,0x00,0x00 -# GFX11-REAL16: v_exp_f16_e64 v5.l, m0 ; encoding: [0x05,0x00,0xd8,0xd5,0x7d,0x00,0x00,0x00] -# GFX11-FAKE16: v_exp_f16_e64 v5, m0 ; encoding: [0x05,0x00,0xd8,0xd5,0x7d,0x00,0x00,0x00] +# GFX11-REAL16: v_exp_f16_e64 v5.l, m0 ; encoding: [0x05,0x00,0xd8,0xd5,0x7d,0x00,0x00,0x00] +# GFX11-FAKE16: v_exp_f16_e64 v5, m0 ; encoding: [0x05,0x00,0xd8,0xd5,0x7d,0x00,0x00,0x00] 0x05,0x00,0xd8,0xd5,0x7d,0x00,0x00,0x00 -# GFX11-REAL16: v_exp_f16_e64 v5.l, exec_lo ; encoding: [0x05,0x00,0xd8,0xd5,0x7e,0x00,0x00,0x00] -# GFX11-FAKE16: v_exp_f16_e64 v5, exec_lo ; encoding: [0x05,0x00,0xd8,0xd5,0x7e,0x00,0x00,0x00] +# GFX11-REAL16: v_exp_f16_e64 v5.l, exec_lo ; encoding: [0x05,0x00,0xd8,0xd5,0x7e,0x00,0x00,0x00] +# GFX11-FAKE16: v_exp_f16_e64 v5, exec_lo ; encoding: [0x05,0x00,0xd8,0xd5,0x7e,0x00,0x00,0x00] 0x05,0x00,0xd8,0xd5,0x7e,0x00,0x00,0x00 -# GFX11-REAL16: v_exp_f16_e64 v5.l, exec_hi ; encoding: [0x05,0x00,0xd8,0xd5,0x7f,0x00,0x00,0x00] -# GFX11-FAKE16: v_exp_f16_e64 v5, exec_hi ; encoding: [0x05,0x00,0xd8,0xd5,0x7f,0x00,0x00,0x00] +# GFX11-REAL16: v_exp_f16_e64 v5.l, exec_hi ; encoding: [0x05,0x00,0xd8,0xd5,0x7f,0x00,0x00,0x00] +# GFX11-FAKE16: v_exp_f16_e64 v5, exec_hi ; encoding: [0x05,0x00,0xd8,0xd5,0x7f,0x00,0x00,0x00] 0x05,0x00,0xd8,0xd5,0x7f,0x00,0x00,0x00 -# GFX11-REAL16: v_exp_f16_e64 v5.l, null ; encoding: [0x05,0x00,0xd8,0xd5,0x7c,0x00,0x00,0x00] -# GFX11-FAKE16: v_exp_f16_e64 v5, null ; encoding: [0x05,0x00,0xd8,0xd5,0x7c,0x00,0x00,0x00] +# GFX11-REAL16: v_exp_f16_e64 v5.l, null ; encoding: [0x05,0x00,0xd8,0xd5,0x7c,0x00,0x00,0x00] +# GFX11-FAKE16: v_exp_f16_e64 v5, null ; encoding: [0x05,0x00,0xd8,0xd5,0x7c,0x00,0x00,0x00] 0x05,0x00,0xd8,0xd5,0x7c,0x00,0x00,0x00 -# GFX11-REAL16: v_exp_f16_e64 v5.l, -1 ; encoding: [0x05,0x00,0xd8,0xd5,0xc1,0x00,0x00,0x00] -# GFX11-FAKE16: v_exp_f16_e64 v5, -1 ; encoding: [0x05,0x00,0xd8,0xd5,0xc1,0x00,0x00,0x00] +# GFX11-REAL16: v_exp_f16_e64 v5.l, -1 ; encoding: [0x05,0x00,0xd8,0xd5,0xc1,0x00,0x00,0x00] +# GFX11-FAKE16: v_exp_f16_e64 v5, -1 ; encoding: [0x05,0x00,0xd8,0xd5,0xc1,0x00,0x00,0x00] 0x05,0x00,0xd8,0xd5,0xc1,0x00,0x00,0x00 -# GFX11-REAL16: v_exp_f16_e64 v5.l, 0.5 mul:2 ; encoding: [0x05,0x00,0xd8,0xd5,0xf0,0x00,0x00,0x08] -# GFX11-FAKE16: v_exp_f16_e64 v5, 0.5 mul:2 ; encoding: [0x05,0x00,0xd8,0xd5,0xf0,0x00,0x00,0x08] +# GFX11-REAL16: v_exp_f16_e64 v5.l, 0.5 mul:2 ; encoding: [0x05,0x00,0xd8,0xd5,0xf0,0x00,0x00,0x08] +# GFX11-FAKE16: v_exp_f16_e64 v5, 0.5 mul:2 ; encoding: [0x05,0x00,0xd8,0xd5,0xf0,0x00,0x00,0x08] 0x05,0x00,0xd8,0xd5,0xf0,0x00,0x00,0x08 -# GFX11-REAL16: v_exp_f16_e64 v5.l, src_scc mul:4 ; encoding: [0x05,0x00,0xd8,0xd5,0xfd,0x00,0x00,0x10] -# GFX11-FAKE16: v_exp_f16_e64 v5, src_scc mul:4 ; encoding: [0x05,0x00,0xd8,0xd5,0xfd,0x00,0x00,0x10] +# GFX11-REAL16: v_exp_f16_e64 v5.l, src_scc mul:4 ; encoding: [0x05,0x00,0xd8,0xd5,0xfd,0x00,0x00,0x10] +# GFX11-FAKE16: v_exp_f16_e64 v5, src_scc mul:4 ; encoding: [0x05,0x00,0xd8,0xd5,0xfd,0x00,0x00,0x10] 0x05,0x00,0xd8,0xd5,0xfd,0x00,0x00,0x10 # GFX11-REAL16: v_exp_f16_e64 v255.l, -|0xfe0b| clamp div:2 ; encoding: [0xff,0x81,0xd8,0xd5,0xff,0x00,0x00,0x38,0x0b,0xfe,0x00,0x00] @@ -1705,60 +1707,60 @@ # GFX11: v_exp_f32_e64 v255, -|0xaf123456| clamp div:2 ; encoding: [0xff,0x81,0xa5,0xd5,0xff,0x00,0x00,0x38,0x56,0x34,0x12,0xaf] 0xff,0x81,0xa5,0xd5,0xff,0x00,0x00,0x38,0x56,0x34,0x12,0xaf -# GFX11-REAL16: v_floor_f16_e64 v5.l, v1.l ; encoding: [0x05,0x00,0xdb,0xd5,0x01,0x01,0x00,0x00] -# GFX11-FAKE16: v_floor_f16_e64 v5, v1 ; encoding: [0x05,0x00,0xdb,0xd5,0x01,0x01,0x00,0x00] +# GFX11-REAL16: v_floor_f16_e64 v5.l, v1.l ; encoding: [0x05,0x00,0xdb,0xd5,0x01,0x01,0x00,0x00] +# GFX11-FAKE16: v_floor_f16_e64 v5, v1 ; encoding: [0x05,0x00,0xdb,0xd5,0x01,0x01,0x00,0x00] 0x05,0x00,0xdb,0xd5,0x01,0x01,0x00,0x00 -# GFX11-REAL16: v_floor_f16_e64 v5.l, v255.l ; encoding: [0x05,0x00,0xdb,0xd5,0xff,0x01,0x00,0x00] -# GFX11-FAKE16: v_floor_f16_e64 v5, v255 ; encoding: [0x05,0x00,0xdb,0xd5,0xff,0x01,0x00,0x00] +# GFX11-REAL16: v_floor_f16_e64 v5.l, v255.l ; encoding: [0x05,0x00,0xdb,0xd5,0xff,0x01,0x00,0x00] +# GFX11-FAKE16: v_floor_f16_e64 v5, v255 ; encoding: [0x05,0x00,0xdb,0xd5,0xff,0x01,0x00,0x00] 0x05,0x00,0xdb,0xd5,0xff,0x01,0x00,0x00 -# GFX11-REAL16: v_floor_f16_e64 v5.l, s1 ; encoding: [0x05,0x00,0xdb,0xd5,0x01,0x00,0x00,0x00] -# GFX11-FAKE16: v_floor_f16_e64 v5, s1 ; encoding: [0x05,0x00,0xdb,0xd5,0x01,0x00,0x00,0x00] +# GFX11-REAL16: v_floor_f16_e64 v5.l, s1 ; encoding: [0x05,0x00,0xdb,0xd5,0x01,0x00,0x00,0x00] +# GFX11-FAKE16: v_floor_f16_e64 v5, s1 ; encoding: [0x05,0x00,0xdb,0xd5,0x01,0x00,0x00,0x00] 0x05,0x00,0xdb,0xd5,0x01,0x00,0x00,0x00 -# GFX11-REAL16: v_floor_f16_e64 v5.l, s105 ; encoding: [0x05,0x00,0xdb,0xd5,0x69,0x00,0x00,0x00] -# GFX11-FAKE16: v_floor_f16_e64 v5, s105 ; encoding: [0x05,0x00,0xdb,0xd5,0x69,0x00,0x00,0x00] +# GFX11-REAL16: v_floor_f16_e64 v5.l, s105 ; encoding: [0x05,0x00,0xdb,0xd5,0x69,0x00,0x00,0x00] +# GFX11-FAKE16: v_floor_f16_e64 v5, s105 ; encoding: [0x05,0x00,0xdb,0xd5,0x69,0x00,0x00,0x00] 0x05,0x00,0xdb,0xd5,0x69,0x00,0x00,0x00 -# GFX11-REAL16: v_floor_f16_e64 v5.l, vcc_lo ; encoding: [0x05,0x00,0xdb,0xd5,0x6a,0x00,0x00,0x00] -# GFX11-FAKE16: v_floor_f16_e64 v5, vcc_lo ; encoding: [0x05,0x00,0xdb,0xd5,0x6a,0x00,0x00,0x00] +# GFX11-REAL16: v_floor_f16_e64 v5.l, vcc_lo ; encoding: [0x05,0x00,0xdb,0xd5,0x6a,0x00,0x00,0x00] +# GFX11-FAKE16: v_floor_f16_e64 v5, vcc_lo ; encoding: [0x05,0x00,0xdb,0xd5,0x6a,0x00,0x00,0x00] 0x05,0x00,0xdb,0xd5,0x6a,0x00,0x00,0x00 -# GFX11-REAL16: v_floor_f16_e64 v5.l, vcc_hi ; encoding: [0x05,0x00,0xdb,0xd5,0x6b,0x00,0x00,0x00] -# GFX11-FAKE16: v_floor_f16_e64 v5, vcc_hi ; encoding: [0x05,0x00,0xdb,0xd5,0x6b,0x00,0x00,0x00] +# GFX11-REAL16: v_floor_f16_e64 v5.l, vcc_hi ; encoding: [0x05,0x00,0xdb,0xd5,0x6b,0x00,0x00,0x00] +# GFX11-FAKE16: v_floor_f16_e64 v5, vcc_hi ; encoding: [0x05,0x00,0xdb,0xd5,0x6b,0x00,0x00,0x00] 0x05,0x00,0xdb,0xd5,0x6b,0x00,0x00,0x00 -# GFX11-REAL16: v_floor_f16_e64 v5.l, ttmp15 ; encoding: [0x05,0x00,0xdb,0xd5,0x7b,0x00,0x00,0x00] -# GFX11-FAKE16: v_floor_f16_e64 v5, ttmp15 ; encoding: [0x05,0x00,0xdb,0xd5,0x7b,0x00,0x00,0x00] +# GFX11-REAL16: v_floor_f16_e64 v5.l, ttmp15 ; encoding: [0x05,0x00,0xdb,0xd5,0x7b,0x00,0x00,0x00] +# GFX11-FAKE16: v_floor_f16_e64 v5, ttmp15 ; encoding: [0x05,0x00,0xdb,0xd5,0x7b,0x00,0x00,0x00] 0x05,0x00,0xdb,0xd5,0x7b,0x00,0x00,0x00 -# GFX11-REAL16: v_floor_f16_e64 v5.l, m0 ; encoding: [0x05,0x00,0xdb,0xd5,0x7d,0x00,0x00,0x00] -# GFX11-FAKE16: v_floor_f16_e64 v5, m0 ; encoding: [0x05,0x00,0xdb,0xd5,0x7d,0x00,0x00,0x00] +# GFX11-REAL16: v_floor_f16_e64 v5.l, m0 ; encoding: [0x05,0x00,0xdb,0xd5,0x7d,0x00,0x00,0x00] +# GFX11-FAKE16: v_floor_f16_e64 v5, m0 ; encoding: [0x05,0x00,0xdb,0xd5,0x7d,0x00,0x00,0x00] 0x05,0x00,0xdb,0xd5,0x7d,0x00,0x00,0x00 -# GFX11-REAL16: v_floor_f16_e64 v5.l, exec_lo ; encoding: [0x05,0x00,0xdb,0xd5,0x7e,0x00,0x00,0x00] -# GFX11-FAKE16: v_floor_f16_e64 v5, exec_lo ; encoding: [0x05,0x00,0xdb,0xd5,0x7e,0x00,0x00,0x00] +# GFX11-REAL16: v_floor_f16_e64 v5.l, exec_lo ; encoding: [0x05,0x00,0xdb,0xd5,0x7e,0x00,0x00,0x00] +# GFX11-FAKE16: v_floor_f16_e64 v5, exec_lo ; encoding: [0x05,0x00,0xdb,0xd5,0x7e,0x00,0x00,0x00] 0x05,0x00,0xdb,0xd5,0x7e,0x00,0x00,0x00 -# GFX11-REAL16: v_floor_f16_e64 v5.l, exec_hi ; encoding: [0x05,0x00,0xdb,0xd5,0x7f,0x00,0x00,0x00] -# GFX11-FAKE16: v_floor_f16_e64 v5, exec_hi ; encoding: [0x05,0x00,0xdb,0xd5,0x7f,0x00,0x00,0x00] +# GFX11-REAL16: v_floor_f16_e64 v5.l, exec_hi ; encoding: [0x05,0x00,0xdb,0xd5,0x7f,0x00,0x00,0x00] +# GFX11-FAKE16: v_floor_f16_e64 v5, exec_hi ; encoding: [0x05,0x00,0xdb,0xd5,0x7f,0x00,0x00,0x00] 0x05,0x00,0xdb,0xd5,0x7f,0x00,0x00,0x00 -# GFX11-REAL16: v_floor_f16_e64 v5.l, null ; encoding: [0x05,0x00,0xdb,0xd5,0x7c,0x00,0x00,0x00] -# GFX11-FAKE16: v_floor_f16_e64 v5, null ; encoding: [0x05,0x00,0xdb,0xd5,0x7c,0x00,0x00,0x00] +# GFX11-REAL16: v_floor_f16_e64 v5.l, null ; encoding: [0x05,0x00,0xdb,0xd5,0x7c,0x00,0x00,0x00] +# GFX11-FAKE16: v_floor_f16_e64 v5, null ; encoding: [0x05,0x00,0xdb,0xd5,0x7c,0x00,0x00,0x00] 0x05,0x00,0xdb,0xd5,0x7c,0x00,0x00,0x00 -# GFX11-REAL16: v_floor_f16_e64 v5.l, -1 ; encoding: [0x05,0x00,0xdb,0xd5,0xc1,0x00,0x00,0x00] -# GFX11-FAKE16: v_floor_f16_e64 v5, -1 ; encoding: [0x05,0x00,0xdb,0xd5,0xc1,0x00,0x00,0x00] +# GFX11-REAL16: v_floor_f16_e64 v5.l, -1 ; encoding: [0x05,0x00,0xdb,0xd5,0xc1,0x00,0x00,0x00] +# GFX11-FAKE16: v_floor_f16_e64 v5, -1 ; encoding: [0x05,0x00,0xdb,0xd5,0xc1,0x00,0x00,0x00] 0x05,0x00,0xdb,0xd5,0xc1,0x00,0x00,0x00 -# GFX11-REAL16: v_floor_f16_e64 v5.l, 0.5 mul:2 ; encoding: [0x05,0x00,0xdb,0xd5,0xf0,0x00,0x00,0x08] -# GFX11-FAKE16: v_floor_f16_e64 v5, 0.5 mul:2 ; encoding: [0x05,0x00,0xdb,0xd5,0xf0,0x00,0x00,0x08] +# GFX11-REAL16: v_floor_f16_e64 v5.l, 0.5 mul:2 ; encoding: [0x05,0x00,0xdb,0xd5,0xf0,0x00,0x00,0x08] +# GFX11-FAKE16: v_floor_f16_e64 v5, 0.5 mul:2 ; encoding: [0x05,0x00,0xdb,0xd5,0xf0,0x00,0x00,0x08] 0x05,0x00,0xdb,0xd5,0xf0,0x00,0x00,0x08 -# GFX11-REAL16: v_floor_f16_e64 v5.l, src_scc mul:4 ; encoding: [0x05,0x00,0xdb,0xd5,0xfd,0x00,0x00,0x10] -# GFX11-FAKE16: v_floor_f16_e64 v5, src_scc mul:4 ; encoding: [0x05,0x00,0xdb,0xd5,0xfd,0x00,0x00,0x10] +# GFX11-REAL16: v_floor_f16_e64 v5.l, src_scc mul:4 ; encoding: [0x05,0x00,0xdb,0xd5,0xfd,0x00,0x00,0x10] +# GFX11-FAKE16: v_floor_f16_e64 v5, src_scc mul:4 ; encoding: [0x05,0x00,0xdb,0xd5,0xfd,0x00,0x00,0x10] 0x05,0x00,0xdb,0xd5,0xfd,0x00,0x00,0x10 # GFX11-REAL16: v_floor_f16_e64 v255.l, -|0xfe0b| clamp div:2 ; encoding: [0xff,0x81,0xdb,0xd5,0xff,0x00,0x00,0x38,0x0b,0xfe,0x00,0x00] @@ -2224,60 +2226,60 @@ # GFX11: v_frexp_mant_f64_e64 v[254:255], 0xaf123456 clamp div:2 ; encoding: [0xfe,0x80,0xbd,0xd5,0xff,0x00,0x00,0x18,0x56,0x34,0x12,0xaf] 0xfe,0x80,0xbd,0xd5,0xff,0x00,0x00,0x18,0x56,0x34,0x12,0xaf -# GFX11-REAL16: v_log_f16_e64 v5.l, v1.l ; encoding: [0x05,0x00,0xd7,0xd5,0x01,0x01,0x00,0x00] -# GFX11-FAKE16: v_log_f16_e64 v5, v1 ; encoding: [0x05,0x00,0xd7,0xd5,0x01,0x01,0x00,0x00] +# GFX11-REAL16: v_log_f16_e64 v5.l, v1.l ; encoding: [0x05,0x00,0xd7,0xd5,0x01,0x01,0x00,0x00] +# GFX11-FAKE16: v_log_f16_e64 v5, v1 ; encoding: [0x05,0x00,0xd7,0xd5,0x01,0x01,0x00,0x00] 0x05,0x00,0xd7,0xd5,0x01,0x01,0x00,0x00 -# GFX11-REAL16: v_log_f16_e64 v5.l, v255.l ; encoding: [0x05,0x00,0xd7,0xd5,0xff,0x01,0x00,0x00] -# GFX11-FAKE16: v_log_f16_e64 v5, v255 ; encoding: [0x05,0x00,0xd7,0xd5,0xff,0x01,0x00,0x00] +# GFX11-REAL16: v_log_f16_e64 v5.l, v255.l ; encoding: [0x05,0x00,0xd7,0xd5,0xff,0x01,0x00,0x00] +# GFX11-FAKE16: v_log_f16_e64 v5, v255 ; encoding: [0x05,0x00,0xd7,0xd5,0xff,0x01,0x00,0x00] 0x05,0x00,0xd7,0xd5,0xff,0x01,0x00,0x00 -# GFX11-REAL16: v_log_f16_e64 v5.l, s1 ; encoding: [0x05,0x00,0xd7,0xd5,0x01,0x00,0x00,0x00] -# GFX11-FAKE16: v_log_f16_e64 v5, s1 ; encoding: [0x05,0x00,0xd7,0xd5,0x01,0x00,0x00,0x00] +# GFX11-REAL16: v_log_f16_e64 v5.l, s1 ; encoding: [0x05,0x00,0xd7,0xd5,0x01,0x00,0x00,0x00] +# GFX11-FAKE16: v_log_f16_e64 v5, s1 ; encoding: [0x05,0x00,0xd7,0xd5,0x01,0x00,0x00,0x00] 0x05,0x00,0xd7,0xd5,0x01,0x00,0x00,0x00 -# GFX11-REAL16: v_log_f16_e64 v5.l, s105 ; encoding: [0x05,0x00,0xd7,0xd5,0x69,0x00,0x00,0x00] -# GFX11-FAKE16: v_log_f16_e64 v5, s105 ; encoding: [0x05,0x00,0xd7,0xd5,0x69,0x00,0x00,0x00] +# GFX11-REAL16: v_log_f16_e64 v5.l, s105 ; encoding: [0x05,0x00,0xd7,0xd5,0x69,0x00,0x00,0x00] +# GFX11-FAKE16: v_log_f16_e64 v5, s105 ; encoding: [0x05,0x00,0xd7,0xd5,0x69,0x00,0x00,0x00] 0x05,0x00,0xd7,0xd5,0x69,0x00,0x00,0x00 -# GFX11-REAL16: v_log_f16_e64 v5.l, vcc_lo ; encoding: [0x05,0x00,0xd7,0xd5,0x6a,0x00,0x00,0x00] -# GFX11-FAKE16: v_log_f16_e64 v5, vcc_lo ; encoding: [0x05,0x00,0xd7,0xd5,0x6a,0x00,0x00,0x00] +# GFX11-REAL16: v_log_f16_e64 v5.l, vcc_lo ; encoding: [0x05,0x00,0xd7,0xd5,0x6a,0x00,0x00,0x00] +# GFX11-FAKE16: v_log_f16_e64 v5, vcc_lo ; encoding: [0x05,0x00,0xd7,0xd5,0x6a,0x00,0x00,0x00] 0x05,0x00,0xd7,0xd5,0x6a,0x00,0x00,0x00 -# GFX11-REAL16: v_log_f16_e64 v5.l, vcc_hi ; encoding: [0x05,0x00,0xd7,0xd5,0x6b,0x00,0x00,0x00] -# GFX11-FAKE16: v_log_f16_e64 v5, vcc_hi ; encoding: [0x05,0x00,0xd7,0xd5,0x6b,0x00,0x00,0x00] +# GFX11-REAL16: v_log_f16_e64 v5.l, vcc_hi ; encoding: [0x05,0x00,0xd7,0xd5,0x6b,0x00,0x00,0x00] +# GFX11-FAKE16: v_log_f16_e64 v5, vcc_hi ; encoding: [0x05,0x00,0xd7,0xd5,0x6b,0x00,0x00,0x00] 0x05,0x00,0xd7,0xd5,0x6b,0x00,0x00,0x00 -# GFX11-REAL16: v_log_f16_e64 v5.l, ttmp15 ; encoding: [0x05,0x00,0xd7,0xd5,0x7b,0x00,0x00,0x00] -# GFX11-FAKE16: v_log_f16_e64 v5, ttmp15 ; encoding: [0x05,0x00,0xd7,0xd5,0x7b,0x00,0x00,0x00] +# GFX11-REAL16: v_log_f16_e64 v5.l, ttmp15 ; encoding: [0x05,0x00,0xd7,0xd5,0x7b,0x00,0x00,0x00] +# GFX11-FAKE16: v_log_f16_e64 v5, ttmp15 ; encoding: [0x05,0x00,0xd7,0xd5,0x7b,0x00,0x00,0x00] 0x05,0x00,0xd7,0xd5,0x7b,0x00,0x00,0x00 -# GFX11-REAL16: v_log_f16_e64 v5.l, m0 ; encoding: [0x05,0x00,0xd7,0xd5,0x7d,0x00,0x00,0x00] -# GFX11-FAKE16: v_log_f16_e64 v5, m0 ; encoding: [0x05,0x00,0xd7,0xd5,0x7d,0x00,0x00,0x00] +# GFX11-REAL16: v_log_f16_e64 v5.l, m0 ; encoding: [0x05,0x00,0xd7,0xd5,0x7d,0x00,0x00,0x00] +# GFX11-FAKE16: v_log_f16_e64 v5, m0 ; encoding: [0x05,0x00,0xd7,0xd5,0x7d,0x00,0x00,0x00] 0x05,0x00,0xd7,0xd5,0x7d,0x00,0x00,0x00 -# GFX11-REAL16: v_log_f16_e64 v5.l, exec_lo ; encoding: [0x05,0x00,0xd7,0xd5,0x7e,0x00,0x00,0x00] -# GFX11-FAKE16: v_log_f16_e64 v5, exec_lo ; encoding: [0x05,0x00,0xd7,0xd5,0x7e,0x00,0x00,0x00] +# GFX11-REAL16: v_log_f16_e64 v5.l, exec_lo ; encoding: [0x05,0x00,0xd7,0xd5,0x7e,0x00,0x00,0x00] +# GFX11-FAKE16: v_log_f16_e64 v5, exec_lo ; encoding: [0x05,0x00,0xd7,0xd5,0x7e,0x00,0x00,0x00] 0x05,0x00,0xd7,0xd5,0x7e,0x00,0x00,0x00 -# GFX11-REAL16: v_log_f16_e64 v5.l, exec_hi ; encoding: [0x05,0x00,0xd7,0xd5,0x7f,0x00,0x00,0x00] -# GFX11-FAKE16: v_log_f16_e64 v5, exec_hi ; encoding: [0x05,0x00,0xd7,0xd5,0x7f,0x00,0x00,0x00] +# GFX11-REAL16: v_log_f16_e64 v5.l, exec_hi ; encoding: [0x05,0x00,0xd7,0xd5,0x7f,0x00,0x00,0x00] +# GFX11-FAKE16: v_log_f16_e64 v5, exec_hi ; encoding: [0x05,0x00,0xd7,0xd5,0x7f,0x00,0x00,0x00] 0x05,0x00,0xd7,0xd5,0x7f,0x00,0x00,0x00 -# GFX11-REAL16: v_log_f16_e64 v5.l, null ; encoding: [0x05,0x00,0xd7,0xd5,0x7c,0x00,0x00,0x00] -# GFX11-FAKE16: v_log_f16_e64 v5, null ; encoding: [0x05,0x00,0xd7,0xd5,0x7c,0x00,0x00,0x00] +# GFX11-REAL16: v_log_f16_e64 v5.l, null ; encoding: [0x05,0x00,0xd7,0xd5,0x7c,0x00,0x00,0x00] +# GFX11-FAKE16: v_log_f16_e64 v5, null ; encoding: [0x05,0x00,0xd7,0xd5,0x7c,0x00,0x00,0x00] 0x05,0x00,0xd7,0xd5,0x7c,0x00,0x00,0x00 -# GFX11-REAL16: v_log_f16_e64 v5.l, -1 ; encoding: [0x05,0x00,0xd7,0xd5,0xc1,0x00,0x00,0x00] -# GFX11-FAKE16: v_log_f16_e64 v5, -1 ; encoding: [0x05,0x00,0xd7,0xd5,0xc1,0x00,0x00,0x00] +# GFX11-REAL16: v_log_f16_e64 v5.l, -1 ; encoding: [0x05,0x00,0xd7,0xd5,0xc1,0x00,0x00,0x00] +# GFX11-FAKE16: v_log_f16_e64 v5, -1 ; encoding: [0x05,0x00,0xd7,0xd5,0xc1,0x00,0x00,0x00] 0x05,0x00,0xd7,0xd5,0xc1,0x00,0x00,0x00 -# GFX11-REAL16: v_log_f16_e64 v5.l, 0.5 mul:2 ; encoding: [0x05,0x00,0xd7,0xd5,0xf0,0x00,0x00,0x08] -# GFX11-FAKE16: v_log_f16_e64 v5, 0.5 mul:2 ; encoding: [0x05,0x00,0xd7,0xd5,0xf0,0x00,0x00,0x08] +# GFX11-REAL16: v_log_f16_e64 v5.l, 0.5 mul:2 ; encoding: [0x05,0x00,0xd7,0xd5,0xf0,0x00,0x00,0x08] +# GFX11-FAKE16: v_log_f16_e64 v5, 0.5 mul:2 ; encoding: [0x05,0x00,0xd7,0xd5,0xf0,0x00,0x00,0x08] 0x05,0x00,0xd7,0xd5,0xf0,0x00,0x00,0x08 -# GFX11-REAL16: v_log_f16_e64 v5.l, src_scc mul:4 ; encoding: [0x05,0x00,0xd7,0xd5,0xfd,0x00,0x00,0x10] -# GFX11-FAKE16: v_log_f16_e64 v5, src_scc mul:4 ; encoding: [0x05,0x00,0xd7,0xd5,0xfd,0x00,0x00,0x10] +# GFX11-REAL16: v_log_f16_e64 v5.l, src_scc mul:4 ; encoding: [0x05,0x00,0xd7,0xd5,0xfd,0x00,0x00,0x10] +# GFX11-FAKE16: v_log_f16_e64 v5, src_scc mul:4 ; encoding: [0x05,0x00,0xd7,0xd5,0xfd,0x00,0x00,0x10] 0x05,0x00,0xd7,0xd5,0xfd,0x00,0x00,0x10 # GFX11-REAL16: v_log_f16_e64 v255.l, -|0xfe0b| clamp div:2 ; encoding: [0xff,0x81,0xd7,0xd5,0xff,0x00,0x00,0x38,0x0b,0xfe,0x00,0x00] @@ -2533,60 +2535,60 @@ # GFX11: v_pipeflush ; encoding: [0x00,0x00,0x9b,0xd5,0x00,0x00,0x00,0x00] 0x00,0x00,0x9b,0xd5,0x00,0x00,0x00,0x00 -# GFX11-REAL16: v_rcp_f16_e64 v5.l, v1.l ; encoding: [0x05,0x00,0xd4,0xd5,0x01,0x01,0x00,0x00] -# GFX11-FAKE16: v_rcp_f16_e64 v5, v1 ; encoding: [0x05,0x00,0xd4,0xd5,0x01,0x01,0x00,0x00] +# GFX11-REAL16: v_rcp_f16_e64 v5.l, v1.l ; encoding: [0x05,0x00,0xd4,0xd5,0x01,0x01,0x00,0x00] +# GFX11-FAKE16: v_rcp_f16_e64 v5, v1 ; encoding: [0x05,0x00,0xd4,0xd5,0x01,0x01,0x00,0x00] 0x05,0x00,0xd4,0xd5,0x01,0x01,0x00,0x00 -# GFX11-REAL16: v_rcp_f16_e64 v5.l, v255.l ; encoding: [0x05,0x00,0xd4,0xd5,0xff,0x01,0x00,0x00] -# GFX11-FAKE16: v_rcp_f16_e64 v5, v255 ; encoding: [0x05,0x00,0xd4,0xd5,0xff,0x01,0x00,0x00] +# GFX11-REAL16: v_rcp_f16_e64 v5.l, v255.l ; encoding: [0x05,0x00,0xd4,0xd5,0xff,0x01,0x00,0x00] +# GFX11-FAKE16: v_rcp_f16_e64 v5, v255 ; encoding: [0x05,0x00,0xd4,0xd5,0xff,0x01,0x00,0x00] 0x05,0x00,0xd4,0xd5,0xff,0x01,0x00,0x00 -# GFX11-REAL16: v_rcp_f16_e64 v5.l, s1 ; encoding: [0x05,0x00,0xd4,0xd5,0x01,0x00,0x00,0x00] -# GFX11-FAKE16: v_rcp_f16_e64 v5, s1 ; encoding: [0x05,0x00,0xd4,0xd5,0x01,0x00,0x00,0x00] +# GFX11-REAL16: v_rcp_f16_e64 v5.l, s1 ; encoding: [0x05,0x00,0xd4,0xd5,0x01,0x00,0x00,0x00] +# GFX11-FAKE16: v_rcp_f16_e64 v5, s1 ; encoding: [0x05,0x00,0xd4,0xd5,0x01,0x00,0x00,0x00] 0x05,0x00,0xd4,0xd5,0x01,0x00,0x00,0x00 -# GFX11-REAL16: v_rcp_f16_e64 v5.l, s105 ; encoding: [0x05,0x00,0xd4,0xd5,0x69,0x00,0x00,0x00] -# GFX11-FAKE16: v_rcp_f16_e64 v5, s105 ; encoding: [0x05,0x00,0xd4,0xd5,0x69,0x00,0x00,0x00] +# GFX11-REAL16: v_rcp_f16_e64 v5.l, s105 ; encoding: [0x05,0x00,0xd4,0xd5,0x69,0x00,0x00,0x00] +# GFX11-FAKE16: v_rcp_f16_e64 v5, s105 ; encoding: [0x05,0x00,0xd4,0xd5,0x69,0x00,0x00,0x00] 0x05,0x00,0xd4,0xd5,0x69,0x00,0x00,0x00 -# GFX11-REAL16: v_rcp_f16_e64 v5.l, vcc_lo ; encoding: [0x05,0x00,0xd4,0xd5,0x6a,0x00,0x00,0x00] -# GFX11-FAKE16: v_rcp_f16_e64 v5, vcc_lo ; encoding: [0x05,0x00,0xd4,0xd5,0x6a,0x00,0x00,0x00] +# GFX11-REAL16: v_rcp_f16_e64 v5.l, vcc_lo ; encoding: [0x05,0x00,0xd4,0xd5,0x6a,0x00,0x00,0x00] +# GFX11-FAKE16: v_rcp_f16_e64 v5, vcc_lo ; encoding: [0x05,0x00,0xd4,0xd5,0x6a,0x00,0x00,0x00] 0x05,0x00,0xd4,0xd5,0x6a,0x00,0x00,0x00 -# GFX11-REAL16: v_rcp_f16_e64 v5.l, vcc_hi ; encoding: [0x05,0x00,0xd4,0xd5,0x6b,0x00,0x00,0x00] -# GFX11-FAKE16: v_rcp_f16_e64 v5, vcc_hi ; encoding: [0x05,0x00,0xd4,0xd5,0x6b,0x00,0x00,0x00] +# GFX11-REAL16: v_rcp_f16_e64 v5.l, vcc_hi ; encoding: [0x05,0x00,0xd4,0xd5,0x6b,0x00,0x00,0x00] +# GFX11-FAKE16: v_rcp_f16_e64 v5, vcc_hi ; encoding: [0x05,0x00,0xd4,0xd5,0x6b,0x00,0x00,0x00] 0x05,0x00,0xd4,0xd5,0x6b,0x00,0x00,0x00 -# GFX11-REAL16: v_rcp_f16_e64 v5.l, ttmp15 ; encoding: [0x05,0x00,0xd4,0xd5,0x7b,0x00,0x00,0x00] -# GFX11-FAKE16: v_rcp_f16_e64 v5, ttmp15 ; encoding: [0x05,0x00,0xd4,0xd5,0x7b,0x00,0x00,0x00] +# GFX11-REAL16: v_rcp_f16_e64 v5.l, ttmp15 ; encoding: [0x05,0x00,0xd4,0xd5,0x7b,0x00,0x00,0x00] +# GFX11-FAKE16: v_rcp_f16_e64 v5, ttmp15 ; encoding: [0x05,0x00,0xd4,0xd5,0x7b,0x00,0x00,0x00] 0x05,0x00,0xd4,0xd5,0x7b,0x00,0x00,0x00 -# GFX11-REAL16: v_rcp_f16_e64 v5.l, m0 ; encoding: [0x05,0x00,0xd4,0xd5,0x7d,0x00,0x00,0x00] -# GFX11-FAKE16: v_rcp_f16_e64 v5, m0 ; encoding: [0x05,0x00,0xd4,0xd5,0x7d,0x00,0x00,0x00] +# GFX11-REAL16: v_rcp_f16_e64 v5.l, m0 ; encoding: [0x05,0x00,0xd4,0xd5,0x7d,0x00,0x00,0x00] +# GFX11-FAKE16: v_rcp_f16_e64 v5, m0 ; encoding: [0x05,0x00,0xd4,0xd5,0x7d,0x00,0x00,0x00] 0x05,0x00,0xd4,0xd5,0x7d,0x00,0x00,0x00 -# GFX11-REAL16: v_rcp_f16_e64 v5.l, exec_lo ; encoding: [0x05,0x00,0xd4,0xd5,0x7e,0x00,0x00,0x00] -# GFX11-FAKE16: v_rcp_f16_e64 v5, exec_lo ; encoding: [0x05,0x00,0xd4,0xd5,0x7e,0x00,0x00,0x00] +# GFX11-REAL16: v_rcp_f16_e64 v5.l, exec_lo ; encoding: [0x05,0x00,0xd4,0xd5,0x7e,0x00,0x00,0x00] +# GFX11-FAKE16: v_rcp_f16_e64 v5, exec_lo ; encoding: [0x05,0x00,0xd4,0xd5,0x7e,0x00,0x00,0x00] 0x05,0x00,0xd4,0xd5,0x7e,0x00,0x00,0x00 -# GFX11-REAL16: v_rcp_f16_e64 v5.l, exec_hi ; encoding: [0x05,0x00,0xd4,0xd5,0x7f,0x00,0x00,0x00] -# GFX11-FAKE16: v_rcp_f16_e64 v5, exec_hi ; encoding: [0x05,0x00,0xd4,0xd5,0x7f,0x00,0x00,0x00] +# GFX11-REAL16: v_rcp_f16_e64 v5.l, exec_hi ; encoding: [0x05,0x00,0xd4,0xd5,0x7f,0x00,0x00,0x00] +# GFX11-FAKE16: v_rcp_f16_e64 v5, exec_hi ; encoding: [0x05,0x00,0xd4,0xd5,0x7f,0x00,0x00,0x00] 0x05,0x00,0xd4,0xd5,0x7f,0x00,0x00,0x00 -# GFX11-REAL16: v_rcp_f16_e64 v5.l, null ; encoding: [0x05,0x00,0xd4,0xd5,0x7c,0x00,0x00,0x00] -# GFX11-FAKE16: v_rcp_f16_e64 v5, null ; encoding: [0x05,0x00,0xd4,0xd5,0x7c,0x00,0x00,0x00] +# GFX11-REAL16: v_rcp_f16_e64 v5.l, null ; encoding: [0x05,0x00,0xd4,0xd5,0x7c,0x00,0x00,0x00] +# GFX11-FAKE16: v_rcp_f16_e64 v5, null ; encoding: [0x05,0x00,0xd4,0xd5,0x7c,0x00,0x00,0x00] 0x05,0x00,0xd4,0xd5,0x7c,0x00,0x00,0x00 -# GFX11-REAL16: v_rcp_f16_e64 v5.l, -1 ; encoding: [0x05,0x00,0xd4,0xd5,0xc1,0x00,0x00,0x00] -# GFX11-FAKE16: v_rcp_f16_e64 v5, -1 ; encoding: [0x05,0x00,0xd4,0xd5,0xc1,0x00,0x00,0x00] +# GFX11-REAL16: v_rcp_f16_e64 v5.l, -1 ; encoding: [0x05,0x00,0xd4,0xd5,0xc1,0x00,0x00,0x00] +# GFX11-FAKE16: v_rcp_f16_e64 v5, -1 ; encoding: [0x05,0x00,0xd4,0xd5,0xc1,0x00,0x00,0x00] 0x05,0x00,0xd4,0xd5,0xc1,0x00,0x00,0x00 -# GFX11-REAL16: v_rcp_f16_e64 v5.l, 0.5 mul:2 ; encoding: [0x05,0x00,0xd4,0xd5,0xf0,0x00,0x00,0x08] -# GFX11-FAKE16: v_rcp_f16_e64 v5, 0.5 mul:2 ; encoding: [0x05,0x00,0xd4,0xd5,0xf0,0x00,0x00,0x08] +# GFX11-REAL16: v_rcp_f16_e64 v5.l, 0.5 mul:2 ; encoding: [0x05,0x00,0xd4,0xd5,0xf0,0x00,0x00,0x08] +# GFX11-FAKE16: v_rcp_f16_e64 v5, 0.5 mul:2 ; encoding: [0x05,0x00,0xd4,0xd5,0xf0,0x00,0x00,0x08] 0x05,0x00,0xd4,0xd5,0xf0,0x00,0x00,0x08 -# GFX11-REAL16: v_rcp_f16_e64 v5.l, src_scc mul:4 ; encoding: [0x05,0x00,0xd4,0xd5,0xfd,0x00,0x00,0x10] -# GFX11-FAKE16: v_rcp_f16_e64 v5, src_scc mul:4 ; encoding: [0x05,0x00,0xd4,0xd5,0xfd,0x00,0x00,0x10] +# GFX11-REAL16: v_rcp_f16_e64 v5.l, src_scc mul:4 ; encoding: [0x05,0x00,0xd4,0xd5,0xfd,0x00,0x00,0x10] +# GFX11-FAKE16: v_rcp_f16_e64 v5, src_scc mul:4 ; encoding: [0x05,0x00,0xd4,0xd5,0xfd,0x00,0x00,0x10] 0x05,0x00,0xd4,0xd5,0xfd,0x00,0x00,0x10 # GFX11-REAL16: v_rcp_f16_e64 v255.l, -|0xfe0b| clamp div:2 ; encoding: [0xff,0x81,0xd4,0xd5,0xff,0x00,0x00,0x38,0x0b,0xfe,0x00,0x00] @@ -2845,60 +2847,60 @@ # GFX11: v_rndne_f64_e64 v[254:255], 0xaf123456 clamp div:2 ; encoding: [0xfe,0x80,0x99,0xd5,0xff,0x00,0x00,0x18,0x56,0x34,0x12,0xaf] 0xfe,0x80,0x99,0xd5,0xff,0x00,0x00,0x18,0x56,0x34,0x12,0xaf -# GFX11-REAL16: v_rsq_f16_e64 v5.l, v1.l ; encoding: [0x05,0x00,0xd6,0xd5,0x01,0x01,0x00,0x00] -# GFX11-FAKE16: v_rsq_f16_e64 v5, v1 ; encoding: [0x05,0x00,0xd6,0xd5,0x01,0x01,0x00,0x00] +# GFX11-REAL16: v_rsq_f16_e64 v5.l, v1.l ; encoding: [0x05,0x00,0xd6,0xd5,0x01,0x01,0x00,0x00] +# GFX11-FAKE16: v_rsq_f16_e64 v5, v1 ; encoding: [0x05,0x00,0xd6,0xd5,0x01,0x01,0x00,0x00] 0x05,0x00,0xd6,0xd5,0x01,0x01,0x00,0x00 -# GFX11-REAL16: v_rsq_f16_e64 v5.l, v255.l ; encoding: [0x05,0x00,0xd6,0xd5,0xff,0x01,0x00,0x00] -# GFX11-FAKE16: v_rsq_f16_e64 v5, v255 ; encoding: [0x05,0x00,0xd6,0xd5,0xff,0x01,0x00,0x00] +# GFX11-REAL16: v_rsq_f16_e64 v5.l, v255.l ; encoding: [0x05,0x00,0xd6,0xd5,0xff,0x01,0x00,0x00] +# GFX11-FAKE16: v_rsq_f16_e64 v5, v255 ; encoding: [0x05,0x00,0xd6,0xd5,0xff,0x01,0x00,0x00] 0x05,0x00,0xd6,0xd5,0xff,0x01,0x00,0x00 -# GFX11-REAL16: v_rsq_f16_e64 v5.l, s1 ; encoding: [0x05,0x00,0xd6,0xd5,0x01,0x00,0x00,0x00] -# GFX11-FAKE16: v_rsq_f16_e64 v5, s1 ; encoding: [0x05,0x00,0xd6,0xd5,0x01,0x00,0x00,0x00] +# GFX11-REAL16: v_rsq_f16_e64 v5.l, s1 ; encoding: [0x05,0x00,0xd6,0xd5,0x01,0x00,0x00,0x00] +# GFX11-FAKE16: v_rsq_f16_e64 v5, s1 ; encoding: [0x05,0x00,0xd6,0xd5,0x01,0x00,0x00,0x00] 0x05,0x00,0xd6,0xd5,0x01,0x00,0x00,0x00 -# GFX11-REAL16: v_rsq_f16_e64 v5.l, s105 ; encoding: [0x05,0x00,0xd6,0xd5,0x69,0x00,0x00,0x00] -# GFX11-FAKE16: v_rsq_f16_e64 v5, s105 ; encoding: [0x05,0x00,0xd6,0xd5,0x69,0x00,0x00,0x00] +# GFX11-REAL16: v_rsq_f16_e64 v5.l, s105 ; encoding: [0x05,0x00,0xd6,0xd5,0x69,0x00,0x00,0x00] +# GFX11-FAKE16: v_rsq_f16_e64 v5, s105 ; encoding: [0x05,0x00,0xd6,0xd5,0x69,0x00,0x00,0x00] 0x05,0x00,0xd6,0xd5,0x69,0x00,0x00,0x00 -# GFX11-REAL16: v_rsq_f16_e64 v5.l, vcc_lo ; encoding: [0x05,0x00,0xd6,0xd5,0x6a,0x00,0x00,0x00] -# GFX11-FAKE16: v_rsq_f16_e64 v5, vcc_lo ; encoding: [0x05,0x00,0xd6,0xd5,0x6a,0x00,0x00,0x00] +# GFX11-REAL16: v_rsq_f16_e64 v5.l, vcc_lo ; encoding: [0x05,0x00,0xd6,0xd5,0x6a,0x00,0x00,0x00] +# GFX11-FAKE16: v_rsq_f16_e64 v5, vcc_lo ; encoding: [0x05,0x00,0xd6,0xd5,0x6a,0x00,0x00,0x00] 0x05,0x00,0xd6,0xd5,0x6a,0x00,0x00,0x00 -# GFX11-REAL16: v_rsq_f16_e64 v5.l, vcc_hi ; encoding: [0x05,0x00,0xd6,0xd5,0x6b,0x00,0x00,0x00] -# GFX11-FAKE16: v_rsq_f16_e64 v5, vcc_hi ; encoding: [0x05,0x00,0xd6,0xd5,0x6b,0x00,0x00,0x00] +# GFX11-REAL16: v_rsq_f16_e64 v5.l, vcc_hi ; encoding: [0x05,0x00,0xd6,0xd5,0x6b,0x00,0x00,0x00] +# GFX11-FAKE16: v_rsq_f16_e64 v5, vcc_hi ; encoding: [0x05,0x00,0xd6,0xd5,0x6b,0x00,0x00,0x00] 0x05,0x00,0xd6,0xd5,0x6b,0x00,0x00,0x00 -# GFX11-REAL16: v_rsq_f16_e64 v5.l, ttmp15 ; encoding: [0x05,0x00,0xd6,0xd5,0x7b,0x00,0x00,0x00] -# GFX11-FAKE16: v_rsq_f16_e64 v5, ttmp15 ; encoding: [0x05,0x00,0xd6,0xd5,0x7b,0x00,0x00,0x00] +# GFX11-REAL16: v_rsq_f16_e64 v5.l, ttmp15 ; encoding: [0x05,0x00,0xd6,0xd5,0x7b,0x00,0x00,0x00] +# GFX11-FAKE16: v_rsq_f16_e64 v5, ttmp15 ; encoding: [0x05,0x00,0xd6,0xd5,0x7b,0x00,0x00,0x00] 0x05,0x00,0xd6,0xd5,0x7b,0x00,0x00,0x00 -# GFX11-REAL16: v_rsq_f16_e64 v5.l, m0 ; encoding: [0x05,0x00,0xd6,0xd5,0x7d,0x00,0x00,0x00] -# GFX11-FAKE16: v_rsq_f16_e64 v5, m0 ; encoding: [0x05,0x00,0xd6,0xd5,0x7d,0x00,0x00,0x00] +# GFX11-REAL16: v_rsq_f16_e64 v5.l, m0 ; encoding: [0x05,0x00,0xd6,0xd5,0x7d,0x00,0x00,0x00] +# GFX11-FAKE16: v_rsq_f16_e64 v5, m0 ; encoding: [0x05,0x00,0xd6,0xd5,0x7d,0x00,0x00,0x00] 0x05,0x00,0xd6,0xd5,0x7d,0x00,0x00,0x00 -# GFX11-REAL16: v_rsq_f16_e64 v5.l, exec_lo ; encoding: [0x05,0x00,0xd6,0xd5,0x7e,0x00,0x00,0x00] -# GFX11-FAKE16: v_rsq_f16_e64 v5, exec_lo ; encoding: [0x05,0x00,0xd6,0xd5,0x7e,0x00,0x00,0x00] +# GFX11-REAL16: v_rsq_f16_e64 v5.l, exec_lo ; encoding: [0x05,0x00,0xd6,0xd5,0x7e,0x00,0x00,0x00] +# GFX11-FAKE16: v_rsq_f16_e64 v5, exec_lo ; encoding: [0x05,0x00,0xd6,0xd5,0x7e,0x00,0x00,0x00] 0x05,0x00,0xd6,0xd5,0x7e,0x00,0x00,0x00 -# GFX11-REAL16: v_rsq_f16_e64 v5.l, exec_hi ; encoding: [0x05,0x00,0xd6,0xd5,0x7f,0x00,0x00,0x00] -# GFX11-FAKE16: v_rsq_f16_e64 v5, exec_hi ; encoding: [0x05,0x00,0xd6,0xd5,0x7f,0x00,0x00,0x00] +# GFX11-REAL16: v_rsq_f16_e64 v5.l, exec_hi ; encoding: [0x05,0x00,0xd6,0xd5,0x7f,0x00,0x00,0x00] +# GFX11-FAKE16: v_rsq_f16_e64 v5, exec_hi ; encoding: [0x05,0x00,0xd6,0xd5,0x7f,0x00,0x00,0x00] 0x05,0x00,0xd6,0xd5,0x7f,0x00,0x00,0x00 -# GFX11-REAL16: v_rsq_f16_e64 v5.l, null ; encoding: [0x05,0x00,0xd6,0xd5,0x7c,0x00,0x00,0x00] -# GFX11-FAKE16: v_rsq_f16_e64 v5, null ; encoding: [0x05,0x00,0xd6,0xd5,0x7c,0x00,0x00,0x00] +# GFX11-REAL16: v_rsq_f16_e64 v5.l, null ; encoding: [0x05,0x00,0xd6,0xd5,0x7c,0x00,0x00,0x00] +# GFX11-FAKE16: v_rsq_f16_e64 v5, null ; encoding: [0x05,0x00,0xd6,0xd5,0x7c,0x00,0x00,0x00] 0x05,0x00,0xd6,0xd5,0x7c,0x00,0x00,0x00 -# GFX11-REAL16: v_rsq_f16_e64 v5.l, -1 ; encoding: [0x05,0x00,0xd6,0xd5,0xc1,0x00,0x00,0x00] -# GFX11-FAKE16: v_rsq_f16_e64 v5, -1 ; encoding: [0x05,0x00,0xd6,0xd5,0xc1,0x00,0x00,0x00] +# GFX11-REAL16: v_rsq_f16_e64 v5.l, -1 ; encoding: [0x05,0x00,0xd6,0xd5,0xc1,0x00,0x00,0x00] +# GFX11-FAKE16: v_rsq_f16_e64 v5, -1 ; encoding: [0x05,0x00,0xd6,0xd5,0xc1,0x00,0x00,0x00] 0x05,0x00,0xd6,0xd5,0xc1,0x00,0x00,0x00 -# GFX11-REAL16: v_rsq_f16_e64 v5.l, 0.5 mul:2 ; encoding: [0x05,0x00,0xd6,0xd5,0xf0,0x00,0x00,0x08] -# GFX11-FAKE16: v_rsq_f16_e64 v5, 0.5 mul:2 ; encoding: [0x05,0x00,0xd6,0xd5,0xf0,0x00,0x00,0x08] +# GFX11-REAL16: v_rsq_f16_e64 v5.l, 0.5 mul:2 ; encoding: [0x05,0x00,0xd6,0xd5,0xf0,0x00,0x00,0x08] +# GFX11-FAKE16: v_rsq_f16_e64 v5, 0.5 mul:2 ; encoding: [0x05,0x00,0xd6,0xd5,0xf0,0x00,0x00,0x08] 0x05,0x00,0xd6,0xd5,0xf0,0x00,0x00,0x08 -# GFX11-REAL16: v_rsq_f16_e64 v5.l, src_scc mul:4 ; encoding: [0x05,0x00,0xd6,0xd5,0xfd,0x00,0x00,0x10] -# GFX11-FAKE16: v_rsq_f16_e64 v5, src_scc mul:4 ; encoding: [0x05,0x00,0xd6,0xd5,0xfd,0x00,0x00,0x10] +# GFX11-REAL16: v_rsq_f16_e64 v5.l, src_scc mul:4 ; encoding: [0x05,0x00,0xd6,0xd5,0xfd,0x00,0x00,0x10] +# GFX11-FAKE16: v_rsq_f16_e64 v5, src_scc mul:4 ; encoding: [0x05,0x00,0xd6,0xd5,0xfd,0x00,0x00,0x10] 0x05,0x00,0xd6,0xd5,0xfd,0x00,0x00,0x10 # GFX11-REAL16: v_rsq_f16_e64 v255.l, -|0xfe0b| clamp div:2 ; encoding: [0xff,0x81,0xd6,0xd5,0xff,0x00,0x00,0x38,0x0b,0xfe,0x00,0x00] @@ -3121,60 +3123,60 @@ # GFX11: v_sin_f32_e64 v255, -|0xaf123456| clamp div:2 ; encoding: [0xff,0x81,0xb5,0xd5,0xff,0x00,0x00,0x38,0x56,0x34,0x12,0xaf] 0xff,0x81,0xb5,0xd5,0xff,0x00,0x00,0x38,0x56,0x34,0x12,0xaf -# GFX11-REAL16: v_sqrt_f16_e64 v5.l, v1.l ; encoding: [0x05,0x00,0xd5,0xd5,0x01,0x01,0x00,0x00] -# GFX11-FAKE16: v_sqrt_f16_e64 v5, v1 ; encoding: [0x05,0x00,0xd5,0xd5,0x01,0x01,0x00,0x00] +# GFX11-REAL16: v_sqrt_f16_e64 v5.l, v1.l ; encoding: [0x05,0x00,0xd5,0xd5,0x01,0x01,0x00,0x00] +# GFX11-FAKE16: v_sqrt_f16_e64 v5, v1 ; encoding: [0x05,0x00,0xd5,0xd5,0x01,0x01,0x00,0x00] 0x05,0x00,0xd5,0xd5,0x01,0x01,0x00,0x00 -# GFX11-REAL16: v_sqrt_f16_e64 v5.l, v255.l ; encoding: [0x05,0x00,0xd5,0xd5,0xff,0x01,0x00,0x00] -# GFX11-FAKE16: v_sqrt_f16_e64 v5, v255 ; encoding: [0x05,0x00,0xd5,0xd5,0xff,0x01,0x00,0x00] +# GFX11-REAL16: v_sqrt_f16_e64 v5.l, v255.l ; encoding: [0x05,0x00,0xd5,0xd5,0xff,0x01,0x00,0x00] +# GFX11-FAKE16: v_sqrt_f16_e64 v5, v255 ; encoding: [0x05,0x00,0xd5,0xd5,0xff,0x01,0x00,0x00] 0x05,0x00,0xd5,0xd5,0xff,0x01,0x00,0x00 -# GFX11-REAL16: v_sqrt_f16_e64 v5.l, s1 ; encoding: [0x05,0x00,0xd5,0xd5,0x01,0x00,0x00,0x00] -# GFX11-FAKE16: v_sqrt_f16_e64 v5, s1 ; encoding: [0x05,0x00,0xd5,0xd5,0x01,0x00,0x00,0x00] +# GFX11-REAL16: v_sqrt_f16_e64 v5.l, s1 ; encoding: [0x05,0x00,0xd5,0xd5,0x01,0x00,0x00,0x00] +# GFX11-FAKE16: v_sqrt_f16_e64 v5, s1 ; encoding: [0x05,0x00,0xd5,0xd5,0x01,0x00,0x00,0x00] 0x05,0x00,0xd5,0xd5,0x01,0x00,0x00,0x00 -# GFX11-REAL16: v_sqrt_f16_e64 v5.l, s105 ; encoding: [0x05,0x00,0xd5,0xd5,0x69,0x00,0x00,0x00] -# GFX11-FAKE16: v_sqrt_f16_e64 v5, s105 ; encoding: [0x05,0x00,0xd5,0xd5,0x69,0x00,0x00,0x00] +# GFX11-REAL16: v_sqrt_f16_e64 v5.l, s105 ; encoding: [0x05,0x00,0xd5,0xd5,0x69,0x00,0x00,0x00] +# GFX11-FAKE16: v_sqrt_f16_e64 v5, s105 ; encoding: [0x05,0x00,0xd5,0xd5,0x69,0x00,0x00,0x00] 0x05,0x00,0xd5,0xd5,0x69,0x00,0x00,0x00 -# GFX11-REAL16: v_sqrt_f16_e64 v5.l, vcc_lo ; encoding: [0x05,0x00,0xd5,0xd5,0x6a,0x00,0x00,0x00] -# GFX11-FAKE16: v_sqrt_f16_e64 v5, vcc_lo ; encoding: [0x05,0x00,0xd5,0xd5,0x6a,0x00,0x00,0x00] +# GFX11-REAL16: v_sqrt_f16_e64 v5.l, vcc_lo ; encoding: [0x05,0x00,0xd5,0xd5,0x6a,0x00,0x00,0x00] +# GFX11-FAKE16: v_sqrt_f16_e64 v5, vcc_lo ; encoding: [0x05,0x00,0xd5,0xd5,0x6a,0x00,0x00,0x00] 0x05,0x00,0xd5,0xd5,0x6a,0x00,0x00,0x00 -# GFX11-REAL16: v_sqrt_f16_e64 v5.l, vcc_hi ; encoding: [0x05,0x00,0xd5,0xd5,0x6b,0x00,0x00,0x00] -# GFX11-FAKE16: v_sqrt_f16_e64 v5, vcc_hi ; encoding: [0x05,0x00,0xd5,0xd5,0x6b,0x00,0x00,0x00] +# GFX11-REAL16: v_sqrt_f16_e64 v5.l, vcc_hi ; encoding: [0x05,0x00,0xd5,0xd5,0x6b,0x00,0x00,0x00] +# GFX11-FAKE16: v_sqrt_f16_e64 v5, vcc_hi ; encoding: [0x05,0x00,0xd5,0xd5,0x6b,0x00,0x00,0x00] 0x05,0x00,0xd5,0xd5,0x6b,0x00,0x00,0x00 -# GFX11-REAL16: v_sqrt_f16_e64 v5.l, ttmp15 ; encoding: [0x05,0x00,0xd5,0xd5,0x7b,0x00,0x00,0x00] -# GFX11-FAKE16: v_sqrt_f16_e64 v5, ttmp15 ; encoding: [0x05,0x00,0xd5,0xd5,0x7b,0x00,0x00,0x00] +# GFX11-REAL16: v_sqrt_f16_e64 v5.l, ttmp15 ; encoding: [0x05,0x00,0xd5,0xd5,0x7b,0x00,0x00,0x00] +# GFX11-FAKE16: v_sqrt_f16_e64 v5, ttmp15 ; encoding: [0x05,0x00,0xd5,0xd5,0x7b,0x00,0x00,0x00] 0x05,0x00,0xd5,0xd5,0x7b,0x00,0x00,0x00 -# GFX11-REAL16: v_sqrt_f16_e64 v5.l, m0 ; encoding: [0x05,0x00,0xd5,0xd5,0x7d,0x00,0x00,0x00] -# GFX11-FAKE16: v_sqrt_f16_e64 v5, m0 ; encoding: [0x05,0x00,0xd5,0xd5,0x7d,0x00,0x00,0x00] +# GFX11-REAL16: v_sqrt_f16_e64 v5.l, m0 ; encoding: [0x05,0x00,0xd5,0xd5,0x7d,0x00,0x00,0x00] +# GFX11-FAKE16: v_sqrt_f16_e64 v5, m0 ; encoding: [0x05,0x00,0xd5,0xd5,0x7d,0x00,0x00,0x00] 0x05,0x00,0xd5,0xd5,0x7d,0x00,0x00,0x00 -# GFX11-REAL16: v_sqrt_f16_e64 v5.l, exec_lo ; encoding: [0x05,0x00,0xd5,0xd5,0x7e,0x00,0x00,0x00] -# GFX11-FAKE16: v_sqrt_f16_e64 v5, exec_lo ; encoding: [0x05,0x00,0xd5,0xd5,0x7e,0x00,0x00,0x00] +# GFX11-REAL16: v_sqrt_f16_e64 v5.l, exec_lo ; encoding: [0x05,0x00,0xd5,0xd5,0x7e,0x00,0x00,0x00] +# GFX11-FAKE16: v_sqrt_f16_e64 v5, exec_lo ; encoding: [0x05,0x00,0xd5,0xd5,0x7e,0x00,0x00,0x00] 0x05,0x00,0xd5,0xd5,0x7e,0x00,0x00,0x00 -# GFX11-REAL16: v_sqrt_f16_e64 v5.l, exec_hi ; encoding: [0x05,0x00,0xd5,0xd5,0x7f,0x00,0x00,0x00] -# GFX11-FAKE16: v_sqrt_f16_e64 v5, exec_hi ; encoding: [0x05,0x00,0xd5,0xd5,0x7f,0x00,0x00,0x00] +# GFX11-REAL16: v_sqrt_f16_e64 v5.l, exec_hi ; encoding: [0x05,0x00,0xd5,0xd5,0x7f,0x00,0x00,0x00] +# GFX11-FAKE16: v_sqrt_f16_e64 v5, exec_hi ; encoding: [0x05,0x00,0xd5,0xd5,0x7f,0x00,0x00,0x00] 0x05,0x00,0xd5,0xd5,0x7f,0x00,0x00,0x00 -# GFX11-REAL16: v_sqrt_f16_e64 v5.l, null ; encoding: [0x05,0x00,0xd5,0xd5,0x7c,0x00,0x00,0x00] -# GFX11-FAKE16: v_sqrt_f16_e64 v5, null ; encoding: [0x05,0x00,0xd5,0xd5,0x7c,0x00,0x00,0x00] +# GFX11-REAL16: v_sqrt_f16_e64 v5.l, null ; encoding: [0x05,0x00,0xd5,0xd5,0x7c,0x00,0x00,0x00] +# GFX11-FAKE16: v_sqrt_f16_e64 v5, null ; encoding: [0x05,0x00,0xd5,0xd5,0x7c,0x00,0x00,0x00] 0x05,0x00,0xd5,0xd5,0x7c,0x00,0x00,0x00 -# GFX11-REAL16: v_sqrt_f16_e64 v5.l, -1 ; encoding: [0x05,0x00,0xd5,0xd5,0xc1,0x00,0x00,0x00] -# GFX11-FAKE16: v_sqrt_f16_e64 v5, -1 ; encoding: [0x05,0x00,0xd5,0xd5,0xc1,0x00,0x00,0x00] +# GFX11-REAL16: v_sqrt_f16_e64 v5.l, -1 ; encoding: [0x05,0x00,0xd5,0xd5,0xc1,0x00,0x00,0x00] +# GFX11-FAKE16: v_sqrt_f16_e64 v5, -1 ; encoding: [0x05,0x00,0xd5,0xd5,0xc1,0x00,0x00,0x00] 0x05,0x00,0xd5,0xd5,0xc1,0x00,0x00,0x00 -# GFX11-REAL16: v_sqrt_f16_e64 v5.l, 0.5 mul:2 ; encoding: [0x05,0x00,0xd5,0xd5,0xf0,0x00,0x00,0x08] -# GFX11-FAKE16: v_sqrt_f16_e64 v5, 0.5 mul:2 ; encoding: [0x05,0x00,0xd5,0xd5,0xf0,0x00,0x00,0x08] +# GFX11-REAL16: v_sqrt_f16_e64 v5.l, 0.5 mul:2 ; encoding: [0x05,0x00,0xd5,0xd5,0xf0,0x00,0x00,0x08] +# GFX11-FAKE16: v_sqrt_f16_e64 v5, 0.5 mul:2 ; encoding: [0x05,0x00,0xd5,0xd5,0xf0,0x00,0x00,0x08] 0x05,0x00,0xd5,0xd5,0xf0,0x00,0x00,0x08 -# GFX11-REAL16: v_sqrt_f16_e64 v5.l, src_scc mul:4 ; encoding: [0x05,0x00,0xd5,0xd5,0xfd,0x00,0x00,0x10] -# GFX11-FAKE16: v_sqrt_f16_e64 v5, src_scc mul:4 ; encoding: [0x05,0x00,0xd5,0xd5,0xfd,0x00,0x00,0x10] +# GFX11-REAL16: v_sqrt_f16_e64 v5.l, src_scc mul:4 ; encoding: [0x05,0x00,0xd5,0xd5,0xfd,0x00,0x00,0x10] +# GFX11-FAKE16: v_sqrt_f16_e64 v5, src_scc mul:4 ; encoding: [0x05,0x00,0xd5,0xd5,0xfd,0x00,0x00,0x10] 0x05,0x00,0xd5,0xd5,0xfd,0x00,0x00,0x10 # GFX11-REAL16: v_sqrt_f16_e64 v255.l, -|0xfe0b| clamp div:2 ; encoding: [0xff,0x81,0xd5,0xd5,0xff,0x00,0x00,0x38,0x0b,0xfe,0x00,0x00] -- GitLab From a87dc23a6271cadd6d3f93561f2d8b1661399e6d Mon Sep 17 00:00:00 2001 From: Balazs Benics Date: Wed, 6 Mar 2024 16:32:14 +0100 Subject: [PATCH 302/929] [clang][NFC] Trim license header comments to 81 characters (#82919) clang-format would format these headers poorly by splitting it into multiple lines. --- clang/include/clang/AST/DeclOpenMP.h | 2 +- clang/include/clang/AST/ParentMapContext.h | 2 +- clang/include/clang/Basic/OpenCLExtensionTypes.def | 2 +- clang/include/clang/Basic/RISCVVTypes.def | 8 ++++---- clang/include/clang/Basic/arm_neon_incl.td | 2 +- .../include/clang/Serialization/PCHContainerOperations.h | 2 +- .../clang/StaticAnalyzer/Core/PathDiagnosticConsumers.h | 2 +- clang/lib/ARCMigrate/TransGCAttrs.cpp | 2 +- clang/lib/AST/Interp/ByteCodeEmitter.h | 2 +- clang/lib/AST/Interp/FunctionPointer.h | 2 +- clang/lib/AST/Interp/PrimType.h | 2 +- clang/lib/Driver/ToolChains/Arch/Mips.h | 2 +- clang/lib/Driver/ToolChains/Arch/Sparc.h | 2 +- clang/lib/Headers/llvm_libc_wrappers/assert.h | 2 +- clang/lib/Sema/AnalysisBasedWarnings.cpp | 2 +- clang/lib/Sema/SemaChecking.cpp | 2 +- .../lib/StaticAnalyzer/Checkers/DirectIvarAssignment.cpp | 2 +- .../Checkers/ObjCAutoreleaseWriteChecker.cpp | 2 +- .../lib/StaticAnalyzer/Checkers/STLAlgorithmModeling.cpp | 2 +- clang/lib/StaticAnalyzer/Checkers/SimpleStreamChecker.cpp | 2 +- clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp | 2 +- clang/lib/Tooling/Refactoring/AtomicChange.cpp | 2 +- clang/test/Analysis/misc-ps-region-store.mm | 4 ++-- clang/test/Rewriter/rewrite-modern-class.mm | 2 +- .../tools/clang-fuzzer/proto-to-llvm/loop_proto_to_llvm.h | 2 +- clang/tools/diagtool/DiagTool.cpp | 2 +- clang/unittests/AST/ASTImporterODRStrategiesTest.cpp | 2 +- clang/unittests/ASTMatchers/Dynamic/RegistryTest.cpp | 4 ++-- clang/unittests/ASTMatchers/Dynamic/VariantValueTest.cpp | 4 ++-- .../CXXOperatorCallExprTraverser.cpp | 2 +- .../RecursiveASTVisitorTests/CallbacksBinaryOperator.cpp | 2 +- .../CallbacksCompoundAssignOperator.cpp | 2 +- .../RecursiveASTVisitorTests/CallbacksUnaryOperator.cpp | 2 +- .../InitListExprPostOrderNoQueue.cpp | 2 +- .../InitListExprPreOrderNoQueue.cpp | 2 +- .../TemplateArgumentLocTraverser.cpp | 2 +- 36 files changed, 42 insertions(+), 42 deletions(-) diff --git a/clang/include/clang/AST/DeclOpenMP.h b/clang/include/clang/AST/DeclOpenMP.h index 73725e6e8566..8fdfddb6c1fd 100644 --- a/clang/include/clang/AST/DeclOpenMP.h +++ b/clang/include/clang/AST/DeclOpenMP.h @@ -1,4 +1,4 @@ -//===- DeclOpenMP.h - Classes for representing OpenMP directives -*- C++ -*-===// +//===- DeclOpenMP.h - Classes for representing OpenMP directives -*- C++ -*-==// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/include/clang/AST/ParentMapContext.h b/clang/include/clang/AST/ParentMapContext.h index d3b2e3986a99..6f79038627d9 100644 --- a/clang/include/clang/AST/ParentMapContext.h +++ b/clang/include/clang/AST/ParentMapContext.h @@ -1,4 +1,4 @@ -//===- ParentMapContext.h - Map of parents using DynTypedNode -------*- C++ -*-===// +//===- ParentMapContext.h - Map of parents using DynTypedNode ---*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/include/clang/Basic/OpenCLExtensionTypes.def b/clang/include/clang/Basic/OpenCLExtensionTypes.def index 17c72d69a020..50ea826c18a7 100644 --- a/clang/include/clang/Basic/OpenCLExtensionTypes.def +++ b/clang/include/clang/Basic/OpenCLExtensionTypes.def @@ -1,4 +1,4 @@ -//===-- OpenCLExtensionTypes.def - Metadata about BuiltinTypes ------*- C++ -*-===// +//===-- OpenCLExtensionTypes.def - Metadata about BuiltinTypes --*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/include/clang/Basic/RISCVVTypes.def b/clang/include/clang/Basic/RISCVVTypes.def index 6620de8ad50e..ccb8cb39068e 100644 --- a/clang/include/clang/Basic/RISCVVTypes.def +++ b/clang/include/clang/Basic/RISCVVTypes.def @@ -383,7 +383,7 @@ RVV_VECTOR_TYPE_INT("__rvv_uint64m2x4_t", RvvUint64m2x4, RvvUint64m2x4Ty, 2, 64, RVV_VECTOR_TYPE_INT("__rvv_uint64m4x2_t", RvvUint64m4x2, RvvUint64m4x2Ty, 4, 64, 2, false) -//===- Float16 tuple types --------------------------------------------------===// +//===- Float16 tuple types ------------------------------------------------===// RVV_VECTOR_TYPE_FLOAT("__rvv_float16mf4x2_t", RvvFloat16mf4x2, RvvFloat16mf4x2Ty, 1, 16, 2) RVV_VECTOR_TYPE_FLOAT("__rvv_float16mf4x3_t", RvvFloat16mf4x3, RvvFloat16mf4x3Ty, 1, 16, 3) RVV_VECTOR_TYPE_FLOAT("__rvv_float16mf4x4_t", RvvFloat16mf4x4, RvvFloat16mf4x4Ty, 1, 16, 4) @@ -414,7 +414,7 @@ RVV_VECTOR_TYPE_FLOAT("__rvv_float16m2x4_t", RvvFloat16m2x4, RvvFloat16m2x4Ty, 8 RVV_VECTOR_TYPE_FLOAT("__rvv_float16m4x2_t", RvvFloat16m4x2, RvvFloat16m4x2Ty, 16, 16, 2) -//===- Float32 tuple types --------------------------------------------------===// +//===- Float32 tuple types ------------------------------------------------===// RVV_VECTOR_TYPE_FLOAT("__rvv_float32mf2x2_t", RvvFloat32mf2x2, RvvFloat32mf2x2Ty, 1, 32, 2) RVV_VECTOR_TYPE_FLOAT("__rvv_float32mf2x3_t", RvvFloat32mf2x3, RvvFloat32mf2x3Ty, 1, 32, 3) RVV_VECTOR_TYPE_FLOAT("__rvv_float32mf2x4_t", RvvFloat32mf2x4, RvvFloat32mf2x4Ty, 1, 32, 4) @@ -437,7 +437,7 @@ RVV_VECTOR_TYPE_FLOAT("__rvv_float32m2x4_t", RvvFloat32m2x4, RvvFloat32m2x4Ty, 4 RVV_VECTOR_TYPE_FLOAT("__rvv_float32m4x2_t", RvvFloat32m4x2, RvvFloat32m4x2Ty, 8, 32, 2) -//===- Float64 tuple types -------------------------------------------------===// +//===- Float64 tuple types ------------------------------------------------===// RVV_VECTOR_TYPE_FLOAT("__rvv_float64m1x2_t", RvvFloat64m1x2, RvvFloat64m1x2Ty, 1, 64, 2) RVV_VECTOR_TYPE_FLOAT("__rvv_float64m1x3_t", RvvFloat64m1x3, RvvFloat64m1x3Ty, 1, 64, 3) RVV_VECTOR_TYPE_FLOAT("__rvv_float64m1x4_t", RvvFloat64m1x4, RvvFloat64m1x4Ty, 1, 64, 4) @@ -452,7 +452,7 @@ RVV_VECTOR_TYPE_FLOAT("__rvv_float64m2x4_t", RvvFloat64m2x4, RvvFloat64m2x4Ty, 2 RVV_VECTOR_TYPE_FLOAT("__rvv_float64m4x2_t", RvvFloat64m4x2, RvvFloat64m4x2Ty, 4, 64, 2) -//===- BFloat16 tuple types -------------------------------------------------===// +//===- BFloat16 tuple types -----------------------------------------------===// RVV_VECTOR_TYPE_BFLOAT("__rvv_bfloat16mf4x2_t", RvvBFloat16mf4x2, RvvBFloat16mf4x2Ty, 1, 16, 2) RVV_VECTOR_TYPE_BFLOAT("__rvv_bfloat16mf4x3_t", RvvBFloat16mf4x3, RvvBFloat16mf4x3Ty, diff --git a/clang/include/clang/Basic/arm_neon_incl.td b/clang/include/clang/Basic/arm_neon_incl.td index 4f969ac1c78a..b8155c187d1b 100644 --- a/clang/include/clang/Basic/arm_neon_incl.td +++ b/clang/include/clang/Basic/arm_neon_incl.td @@ -1,4 +1,4 @@ -//===--- arm_neon_incl.td - ARM NEON compiler interface ------------------------===// +//===--- arm_neon_incl.td - ARM NEON compiler interface -------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/include/clang/Serialization/PCHContainerOperations.h b/clang/include/clang/Serialization/PCHContainerOperations.h index be10feb5e351..ddfddf2dafad 100644 --- a/clang/include/clang/Serialization/PCHContainerOperations.h +++ b/clang/include/clang/Serialization/PCHContainerOperations.h @@ -1,4 +1,4 @@ -//===--- Serialization/PCHContainerOperations.h - PCH Containers --*- C++ -*-===// +//===-- PCHContainerOperations.h - PCH Containers ---------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/include/clang/StaticAnalyzer/Core/PathDiagnosticConsumers.h b/clang/include/clang/StaticAnalyzer/Core/PathDiagnosticConsumers.h index 2694aac478cd..88a9d76f2488 100644 --- a/clang/include/clang/StaticAnalyzer/Core/PathDiagnosticConsumers.h +++ b/clang/include/clang/StaticAnalyzer/Core/PathDiagnosticConsumers.h @@ -1,4 +1,4 @@ -//===--- PathDiagnosticConsumers.h - Path Diagnostic Clients ------*- C++ -*-===// +//===--- PathDiagnosticConsumers.h - Path Diagnostic Clients ----*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/lib/ARCMigrate/TransGCAttrs.cpp b/clang/lib/ARCMigrate/TransGCAttrs.cpp index 28d1db7f4376..85e3fe77660b 100644 --- a/clang/lib/ARCMigrate/TransGCAttrs.cpp +++ b/clang/lib/ARCMigrate/TransGCAttrs.cpp @@ -1,4 +1,4 @@ -//===--- TransGCAttrs.cpp - Transformations to ARC mode --------------------===// +//===--- TransGCAttrs.cpp - Transformations to ARC mode -------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/lib/AST/Interp/ByteCodeEmitter.h b/clang/lib/AST/Interp/ByteCodeEmitter.h index 548769329b7f..5612a8c1481a 100644 --- a/clang/lib/AST/Interp/ByteCodeEmitter.h +++ b/clang/lib/AST/Interp/ByteCodeEmitter.h @@ -1,4 +1,4 @@ -//===--- ByteCodeEmitter.h - Instruction emitter for the VM ---------*- C++ -*-===// +//===--- ByteCodeEmitter.h - Instruction emitter for the VM -----*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/lib/AST/Interp/FunctionPointer.h b/clang/lib/AST/Interp/FunctionPointer.h index bb3da9b50aa5..2ff691b1cd3e 100644 --- a/clang/lib/AST/Interp/FunctionPointer.h +++ b/clang/lib/AST/Interp/FunctionPointer.h @@ -1,4 +1,4 @@ -//===--- FunctionPointer.h - Types for the constexpr VM ----------*- C++ -*-===// +//===--- FunctionPointer.h - Types for the constexpr VM ---------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/lib/AST/Interp/PrimType.h b/clang/lib/AST/Interp/PrimType.h index 24a24a71a07b..2bc83b334643 100644 --- a/clang/lib/AST/Interp/PrimType.h +++ b/clang/lib/AST/Interp/PrimType.h @@ -1,4 +1,4 @@ -//===--- PrimType.h - Types for the constexpr VM --------------------*- C++ -*-===// +//===--- PrimType.h - Types for the constexpr VM ----------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/lib/Driver/ToolChains/Arch/Mips.h b/clang/lib/Driver/ToolChains/Arch/Mips.h index 62211c711420..674c21744b52 100644 --- a/clang/lib/Driver/ToolChains/Arch/Mips.h +++ b/clang/lib/Driver/ToolChains/Arch/Mips.h @@ -1,4 +1,4 @@ -//===--- Mips.h - Mips-specific Tool Helpers ----------------------*- C++ -*-===// +//===--- Mips.h - Mips-specific Tool Helpers --------------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/lib/Driver/ToolChains/Arch/Sparc.h b/clang/lib/Driver/ToolChains/Arch/Sparc.h index 44658c4259c6..2b178d9df1ee 100644 --- a/clang/lib/Driver/ToolChains/Arch/Sparc.h +++ b/clang/lib/Driver/ToolChains/Arch/Sparc.h @@ -1,4 +1,4 @@ -//===--- Sparc.h - Sparc-specific Tool Helpers ----------------------*- C++ -*-===// +//===--- Sparc.h - Sparc-specific Tool Helpers ------------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/lib/Headers/llvm_libc_wrappers/assert.h b/clang/lib/Headers/llvm_libc_wrappers/assert.h index de650ca8442a..610ed96a458c 100644 --- a/clang/lib/Headers/llvm_libc_wrappers/assert.h +++ b/clang/lib/Headers/llvm_libc_wrappers/assert.h @@ -1,4 +1,4 @@ -//===-- Wrapper for C standard assert.h declarations on the GPU ------------===// +//===-- Wrapper for C standard assert.h declarations on the GPU -*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/lib/Sema/AnalysisBasedWarnings.cpp b/clang/lib/Sema/AnalysisBasedWarnings.cpp index 8239ba49429d..6992ba9ad9a7 100644 --- a/clang/lib/Sema/AnalysisBasedWarnings.cpp +++ b/clang/lib/Sema/AnalysisBasedWarnings.cpp @@ -1,4 +1,4 @@ -//=- AnalysisBasedWarnings.cpp - Sema warnings based on libAnalysis -*- C++ -*-=// +//=== AnalysisBasedWarnings.cpp - Sema warnings based on libAnalysis ------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index 82a7c16106f6..6ed61a19ff17 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -11527,7 +11527,7 @@ void CheckFormatHandler::EmitFormatDiagnostic( } } -//===--- CHECK: Printf format string checking ------------------------------===// +//===--- CHECK: Printf format string checking -----------------------------===// namespace { diff --git a/clang/lib/StaticAnalyzer/Checkers/DirectIvarAssignment.cpp b/clang/lib/StaticAnalyzer/Checkers/DirectIvarAssignment.cpp index 49486ea796c2..fc174e29be47 100644 --- a/clang/lib/StaticAnalyzer/Checkers/DirectIvarAssignment.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/DirectIvarAssignment.cpp @@ -1,4 +1,4 @@ -//=- DirectIvarAssignment.cpp - Check rules on ObjC properties -*- C++ ----*-==// +//===- DirectIvarAssignment.cpp - Check rules on ObjC properties -*- C++ -*-==// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/lib/StaticAnalyzer/Checkers/ObjCAutoreleaseWriteChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ObjCAutoreleaseWriteChecker.cpp index 514f53b4804f..e7fd14d4558b 100644 --- a/clang/lib/StaticAnalyzer/Checkers/ObjCAutoreleaseWriteChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/ObjCAutoreleaseWriteChecker.cpp @@ -1,4 +1,4 @@ -//===- ObjCAutoreleaseWriteChecker.cpp ----------------------------*- C++ -*-==// +//===- ObjCAutoreleaseWriteChecker.cpp ---------------------------*- C++ -*-==// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/lib/StaticAnalyzer/Checkers/STLAlgorithmModeling.cpp b/clang/lib/StaticAnalyzer/Checkers/STLAlgorithmModeling.cpp index 788f2875863c..a5173a05636a 100644 --- a/clang/lib/StaticAnalyzer/Checkers/STLAlgorithmModeling.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/STLAlgorithmModeling.cpp @@ -1,4 +1,4 @@ -//===-- STLAlgorithmModeling.cpp -----------------------------------*- C++ -*--// +//===-- STLAlgorithmModeling.cpp ----------------------------------*- C++ -*--// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/lib/StaticAnalyzer/Checkers/SimpleStreamChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/SimpleStreamChecker.cpp index 7cbe271dfbf9..50d50562d3e7 100644 --- a/clang/lib/StaticAnalyzer/Checkers/SimpleStreamChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/SimpleStreamChecker.cpp @@ -1,4 +1,4 @@ -//===-- SimpleStreamChecker.cpp -----------------------------------------*- C++ -*--// +//===-- SimpleStreamChecker.cpp -----------------------------------*- C++ -*--// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp b/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp index 2d8498e36015..c6f87b45ab88 100644 --- a/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp +++ b/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp @@ -3038,7 +3038,7 @@ ProgramStateRef RangeConstraintManager::setRange(ProgramStateRef State, //===------------------------------------------------------------------------=== // assumeSymX methods: protected interface for RangeConstraintManager. -//===------------------------------------------------------------------------===/ +//===------------------------------------------------------------------------=== // The syntax for ranges below is mathematical, using [x, y] for closed ranges // and (x, y) for open ranges. These ranges are modular, corresponding with diff --git a/clang/lib/Tooling/Refactoring/AtomicChange.cpp b/clang/lib/Tooling/Refactoring/AtomicChange.cpp index 3d5ae2fed014..dfc98355c664 100644 --- a/clang/lib/Tooling/Refactoring/AtomicChange.cpp +++ b/clang/lib/Tooling/Refactoring/AtomicChange.cpp @@ -1,4 +1,4 @@ -//===--- AtomicChange.cpp - AtomicChange implementation -----------------*- C++ -*-===// +//===--- AtomicChange.cpp - AtomicChange implementation ---------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/test/Analysis/misc-ps-region-store.mm b/clang/test/Analysis/misc-ps-region-store.mm index 7722f01e35bc..3a2df83d4f41 100644 --- a/clang/test/Analysis/misc-ps-region-store.mm +++ b/clang/test/Analysis/misc-ps-region-store.mm @@ -2,9 +2,9 @@ // RUN: %clang_analyze_cc1 -triple x86_64-apple-darwin9 -analyzer-checker=core,alpha.core -verify -fblocks %s // expected-no-diagnostics -//===------------------------------------------------------------------------------------------===// +//===----------------------------------------------------------------------===// // This files tests our path-sensitive handling of Objective-c++ files. -//===------------------------------------------------------------------------------------------===// +//===----------------------------------------------------------------------===// // Test basic handling of references. char &test1_aux(); diff --git a/clang/test/Rewriter/rewrite-modern-class.mm b/clang/test/Rewriter/rewrite-modern-class.mm index cf2143e86463..7d75a51502cd 100644 --- a/clang/test/Rewriter/rewrite-modern-class.mm +++ b/clang/test/Rewriter/rewrite-modern-class.mm @@ -44,7 +44,7 @@ @implementation class_has_no_ivar @end -//============================class needs to be synthesized here===================== +//===================== class needs to be synthesized here ===================== @interface SUPER { @public double divar; diff --git a/clang/tools/clang-fuzzer/proto-to-llvm/loop_proto_to_llvm.h b/clang/tools/clang-fuzzer/proto-to-llvm/loop_proto_to_llvm.h index 173b937e5270..0614835dcc81 100644 --- a/clang/tools/clang-fuzzer/proto-to-llvm/loop_proto_to_llvm.h +++ b/clang/tools/clang-fuzzer/proto-to-llvm/loop_proto_to_llvm.h @@ -1,4 +1,4 @@ -//==-- loop_proto_to_llvm.h - Protobuf-C++ conversion ----------------------------==// +//===- loop_proto_to_llvm.h - Protobuf-C++ conversion -----------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/tools/diagtool/DiagTool.cpp b/clang/tools/diagtool/DiagTool.cpp index 99abe5755f71..384eef560c6c 100644 --- a/clang/tools/diagtool/DiagTool.cpp +++ b/clang/tools/diagtool/DiagTool.cpp @@ -1,4 +1,4 @@ -//===- DiagTool.cpp - Classes for defining diagtool tools -------------------===// +//===- DiagTool.cpp - Classes for defining diagtool tools -----------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/unittests/AST/ASTImporterODRStrategiesTest.cpp b/clang/unittests/AST/ASTImporterODRStrategiesTest.cpp index 3f21a149f7e1..db0ba5d7e1ed 100644 --- a/clang/unittests/AST/ASTImporterODRStrategiesTest.cpp +++ b/clang/unittests/AST/ASTImporterODRStrategiesTest.cpp @@ -1,4 +1,4 @@ -//===- unittest/AST/ASTImporterODRStrategiesTest.cpp -----------------------===// +//===- ASTImporterODRStrategiesTest.cpp -----------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/unittests/ASTMatchers/Dynamic/RegistryTest.cpp b/clang/unittests/ASTMatchers/Dynamic/RegistryTest.cpp index 9428f06ade58..013bb912dfb8 100644 --- a/clang/unittests/ASTMatchers/Dynamic/RegistryTest.cpp +++ b/clang/unittests/ASTMatchers/Dynamic/RegistryTest.cpp @@ -1,10 +1,10 @@ -//===- unittest/ASTMatchers/Dynamic/RegistryTest.cpp - Registry unit tests -===// +//===- RegistryTest.cpp - Registry unit tests -----------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -//===-----------------------------------------------------------------------===// +//===----------------------------------------------------------------------===// #include "../ASTMatchersTest.h" #include "clang/ASTMatchers/Dynamic/Registry.h" diff --git a/clang/unittests/ASTMatchers/Dynamic/VariantValueTest.cpp b/clang/unittests/ASTMatchers/Dynamic/VariantValueTest.cpp index c62a6b385e28..2449a2907ac8 100644 --- a/clang/unittests/ASTMatchers/Dynamic/VariantValueTest.cpp +++ b/clang/unittests/ASTMatchers/Dynamic/VariantValueTest.cpp @@ -1,10 +1,10 @@ -//===- unittest/ASTMatchers/Dynamic/VariantValueTest.cpp - VariantValue unit tests -===// +//===- VariantValueTest.cpp - VariantValue unit tests ---------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -//===-----------------------------------------------------------------------------===// +//===----------------------------------------------------------------------===// #include "../ASTMatchersTest.h" #include "clang/ASTMatchers/Dynamic/VariantValue.h" diff --git a/clang/unittests/Tooling/RecursiveASTVisitorTests/CXXOperatorCallExprTraverser.cpp b/clang/unittests/Tooling/RecursiveASTVisitorTests/CXXOperatorCallExprTraverser.cpp index 91de8d17c9d5..376874eb351d 100644 --- a/clang/unittests/Tooling/RecursiveASTVisitorTests/CXXOperatorCallExprTraverser.cpp +++ b/clang/unittests/Tooling/RecursiveASTVisitorTests/CXXOperatorCallExprTraverser.cpp @@ -1,4 +1,4 @@ -//===- unittest/Tooling/RecursiveASTVisitorTests/CXXOperatorCallExprTraverser.cpp -===// +//===- CXXOperatorCallExprTraverser.cpp -----------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/unittests/Tooling/RecursiveASTVisitorTests/CallbacksBinaryOperator.cpp b/clang/unittests/Tooling/RecursiveASTVisitorTests/CallbacksBinaryOperator.cpp index 4c5b2b66c068..3d7a02872541 100644 --- a/clang/unittests/Tooling/RecursiveASTVisitorTests/CallbacksBinaryOperator.cpp +++ b/clang/unittests/Tooling/RecursiveASTVisitorTests/CallbacksBinaryOperator.cpp @@ -1,4 +1,4 @@ -//===- unittests/Tooling/RecursiveASTVisitorTests/CallbacksBinaryOperator.cpp -===// +//===- CallbacksBinaryOperator.cpp ----------------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/unittests/Tooling/RecursiveASTVisitorTests/CallbacksCompoundAssignOperator.cpp b/clang/unittests/Tooling/RecursiveASTVisitorTests/CallbacksCompoundAssignOperator.cpp index f381d5e88a59..34a39d2f8b7b 100644 --- a/clang/unittests/Tooling/RecursiveASTVisitorTests/CallbacksCompoundAssignOperator.cpp +++ b/clang/unittests/Tooling/RecursiveASTVisitorTests/CallbacksCompoundAssignOperator.cpp @@ -1,4 +1,4 @@ -//===- unittests/Tooling/RecursiveASTVisitorTests/CallbacksCompoundAssignOperator.cpp -===// +//===- CallbacksCompoundAssignOperator.cpp --------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/unittests/Tooling/RecursiveASTVisitorTests/CallbacksUnaryOperator.cpp b/clang/unittests/Tooling/RecursiveASTVisitorTests/CallbacksUnaryOperator.cpp index 26c755599be8..c0c7de474d0d 100644 --- a/clang/unittests/Tooling/RecursiveASTVisitorTests/CallbacksUnaryOperator.cpp +++ b/clang/unittests/Tooling/RecursiveASTVisitorTests/CallbacksUnaryOperator.cpp @@ -1,4 +1,4 @@ -//===- unittests/Tooling/RecursiveASTVisitorTests/CallbacksUnaryOperator.cpp -===// +//===- CallbacksUnaryOperator.cpp -----------------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/unittests/Tooling/RecursiveASTVisitorTests/InitListExprPostOrderNoQueue.cpp b/clang/unittests/Tooling/RecursiveASTVisitorTests/InitListExprPostOrderNoQueue.cpp index a15f4c83c5eb..8750f7834944 100644 --- a/clang/unittests/Tooling/RecursiveASTVisitorTests/InitListExprPostOrderNoQueue.cpp +++ b/clang/unittests/Tooling/RecursiveASTVisitorTests/InitListExprPostOrderNoQueue.cpp @@ -1,4 +1,4 @@ -//===- unittest/Tooling/RecursiveASTVisitorTests/InitListExprPostOrderNoQueue.cpp -===// +//===- InitListExprPostOrderNoQueue.cpp -----------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/unittests/Tooling/RecursiveASTVisitorTests/InitListExprPreOrderNoQueue.cpp b/clang/unittests/Tooling/RecursiveASTVisitorTests/InitListExprPreOrderNoQueue.cpp index 1dafeef7cda7..8db88e1e0639 100644 --- a/clang/unittests/Tooling/RecursiveASTVisitorTests/InitListExprPreOrderNoQueue.cpp +++ b/clang/unittests/Tooling/RecursiveASTVisitorTests/InitListExprPreOrderNoQueue.cpp @@ -1,4 +1,4 @@ -//===- unittest/Tooling/RecursiveASTVisitorTests/InitListExprPreOrderNoQueue.cpp -===// +//===- InitListExprPreOrderNoQueue.cpp ------------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/clang/unittests/Tooling/RecursiveASTVisitorTests/TemplateArgumentLocTraverser.cpp b/clang/unittests/Tooling/RecursiveASTVisitorTests/TemplateArgumentLocTraverser.cpp index 5eee19f0730e..f068e53ae9c2 100644 --- a/clang/unittests/Tooling/RecursiveASTVisitorTests/TemplateArgumentLocTraverser.cpp +++ b/clang/unittests/Tooling/RecursiveASTVisitorTests/TemplateArgumentLocTraverser.cpp @@ -1,4 +1,4 @@ -//===- unittest/Tooling/RecursiveASTVisitorTests/TemplateArgumentLocTraverser.cpp -===// +//===- TemplateArgumentLocTraverser.cpp -----------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. -- GitLab From 1fd1f4c0e1f229a2eceb94fc4e41bdd4b9ca7d5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mirko=20Brku=C5=A1anin?= Date: Wed, 6 Mar 2024 16:33:52 +0100 Subject: [PATCH 303/929] [AMDGPU] Handle amdgpu.last.use metadata (#83816) Convert !amdgpu.last.use metadata into MachineMemOperand for last use and handle it in SIMemoryLegalizer similar to nontemporal and volatile. --- llvm/docs/AMDGPUUsage.rst | 15 +++ llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 7 +- llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp | 99 ++++++++++--------- .../AMDGPU/memory-legalizer-flat-lastuse.ll | 83 ++++++++++++++++ .../AMDGPU/memory-legalizer-global-lastuse.ll | 84 ++++++++++++++++ .../memory-legalizer-private-lastuse.ll | 90 +++++++++++++++++ 6 files changed, 332 insertions(+), 46 deletions(-) create mode 100644 llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll create mode 100644 llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll create mode 100644 llvm/test/CodeGen/AMDGPU/memory-legalizer-private-lastuse.ll diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst index ed41be4b08de..7f39f69cae60 100644 --- a/llvm/docs/AMDGPUUsage.rst +++ b/llvm/docs/AMDGPUUsage.rst @@ -1306,6 +1306,21 @@ The AMDGPU backend implements the following LLVM IR intrinsics. List AMDGPU intrinsics. +LLVM IR Metadata +------------------ + +The AMDGPU backend implements the following LLVM IR metadata. + +.. table:: AMDGPU LLVM IR Metadata + :name: amdgpu-llvm-ir-metadata-table + + ============================================== ========================================================== + LLVM IR Metadata Description + ============================================== ========================================================== + !amdgpu.last.use Sets TH_LOAD_LU temporal hint on load instructions that support it. + Takes priority over nontemporal hint (TH_LOAD_NT). + ============================================== ========================================================== + LLVM IR Attributes ------------------ diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index bf1463f93df6..bbb5f8d912a5 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -16251,9 +16251,12 @@ bool SITargetLowering::isReassocProfitable(MachineRegisterInfo &MRI, MachineMemOperand::Flags SITargetLowering::getTargetMMOFlags(const Instruction &I) const { // Propagate metadata set by AMDGPUAnnotateUniformValues to the MMO of a load. + MachineMemOperand::Flags Flags = MachineMemOperand::MONone; if (I.getMetadata("amdgpu.noclobber")) - return MONoClobber; - return MachineMemOperand::MONone; + Flags |= MONoClobber; + if (I.getMetadata("amdgpu.last.use")) + Flags |= MOLastUse; + return Flags; } bool SITargetLowering::checkForPhysRegDependency( diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp index 73c23f0f987c..62306fa667b3 100644 --- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp +++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp @@ -98,22 +98,22 @@ private: bool IsCrossAddressSpaceOrdering = false; bool IsVolatile = false; bool IsNonTemporal = false; - - SIMemOpInfo(AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent, - SIAtomicScope Scope = SIAtomicScope::SYSTEM, - SIAtomicAddrSpace OrderingAddrSpace = SIAtomicAddrSpace::ATOMIC, - SIAtomicAddrSpace InstrAddrSpace = SIAtomicAddrSpace::ALL, - bool IsCrossAddressSpaceOrdering = true, - AtomicOrdering FailureOrdering = - AtomicOrdering::SequentiallyConsistent, - bool IsVolatile = false, - bool IsNonTemporal = false) - : Ordering(Ordering), FailureOrdering(FailureOrdering), - Scope(Scope), OrderingAddrSpace(OrderingAddrSpace), - InstrAddrSpace(InstrAddrSpace), - IsCrossAddressSpaceOrdering(IsCrossAddressSpaceOrdering), - IsVolatile(IsVolatile), - IsNonTemporal(IsNonTemporal) { + bool IsLastUse = false; + + SIMemOpInfo( + AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent, + SIAtomicScope Scope = SIAtomicScope::SYSTEM, + SIAtomicAddrSpace OrderingAddrSpace = SIAtomicAddrSpace::ATOMIC, + SIAtomicAddrSpace InstrAddrSpace = SIAtomicAddrSpace::ALL, + bool IsCrossAddressSpaceOrdering = true, + AtomicOrdering FailureOrdering = AtomicOrdering::SequentiallyConsistent, + bool IsVolatile = false, bool IsNonTemporal = false, + bool IsLastUse = false) + : Ordering(Ordering), FailureOrdering(FailureOrdering), Scope(Scope), + OrderingAddrSpace(OrderingAddrSpace), InstrAddrSpace(InstrAddrSpace), + IsCrossAddressSpaceOrdering(IsCrossAddressSpaceOrdering), + IsVolatile(IsVolatile), IsNonTemporal(IsNonTemporal), + IsLastUse(IsLastUse) { if (Ordering == AtomicOrdering::NotAtomic) { assert(Scope == SIAtomicScope::NONE && @@ -201,6 +201,10 @@ public: return IsNonTemporal; } + /// \returns True if memory access of the machine instruction used to + /// create this SIMemOpInfo is last use, false otherwise. + bool isLastUse() const { return IsLastUse; } + /// \returns True if ordering constraint of the machine instruction used to /// create this SIMemOpInfo is unordered or higher, false otherwise. bool isAtomic() const { @@ -305,12 +309,13 @@ public: SIAtomicAddrSpace AddrSpace) const = 0; /// Update \p MI memory instruction of kind \p Op associated with address - /// spaces \p AddrSpace to indicate it is volatile and/or nontemporal. Return - /// true iff the instruction was modified. + /// spaces \p AddrSpace to indicate it is volatile and/or + /// nontemporal/last-use. Return true iff the instruction was modified. virtual bool enableVolatileAndOrNonTemporal(MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op, bool IsVolatile, - bool IsNonTemporal) const = 0; + bool IsNonTemporal, + bool IsLastUse = false) const = 0; virtual bool expandSystemScopeStore(MachineBasicBlock::iterator &MI) const { return false; @@ -394,8 +399,8 @@ public: bool enableVolatileAndOrNonTemporal(MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op, - bool IsVolatile, - bool IsNonTemporal) const override; + bool IsVolatile, bool IsNonTemporal, + bool IsLastUse) const override; bool insertWait(MachineBasicBlock::iterator &MI, SIAtomicScope Scope, @@ -447,8 +452,8 @@ public: bool enableVolatileAndOrNonTemporal(MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op, - bool IsVolatile, - bool IsNonTemporal) const override; + bool IsVolatile, bool IsNonTemporal, + bool IsLastUse) const override; bool insertWait(MachineBasicBlock::iterator &MI, SIAtomicScope Scope, @@ -508,8 +513,8 @@ public: bool enableVolatileAndOrNonTemporal(MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op, - bool IsVolatile, - bool IsNonTemporal) const override; + bool IsVolatile, bool IsNonTemporal, + bool IsLastUse) const override; bool insertAcquire(MachineBasicBlock::iterator &MI, SIAtomicScope Scope, SIAtomicAddrSpace AddrSpace, Position Pos) const override; @@ -552,8 +557,8 @@ public: bool enableVolatileAndOrNonTemporal(MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op, - bool IsVolatile, - bool IsNonTemporal) const override; + bool IsVolatile, bool IsNonTemporal, + bool IsLastUse) const override; bool insertWait(MachineBasicBlock::iterator &MI, SIAtomicScope Scope, @@ -578,8 +583,8 @@ public: bool enableVolatileAndOrNonTemporal(MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op, - bool IsVolatile, - bool IsNonTemporal) const override; + bool IsVolatile, bool IsNonTemporal, + bool IsLastUse) const override; }; class SIGfx12CacheControl : public SIGfx11CacheControl { @@ -614,8 +619,8 @@ public: bool enableVolatileAndOrNonTemporal(MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op, - bool IsVolatile, - bool IsNonTemporal) const override; + bool IsVolatile, bool IsNonTemporal, + bool IsLastUse) const override; bool expandSystemScopeStore(MachineBasicBlock::iterator &MI) const override; }; @@ -745,12 +750,14 @@ std::optional SIMemOpAccess::constructFromMIWithMMO( SIAtomicAddrSpace InstrAddrSpace = SIAtomicAddrSpace::NONE; bool IsNonTemporal = true; bool IsVolatile = false; + bool IsLastUse = false; // Validator should check whether or not MMOs cover the entire set of // locations accessed by the memory instruction. for (const auto &MMO : MI->memoperands()) { IsNonTemporal &= MMO->isNonTemporal(); IsVolatile |= MMO->isVolatile(); + IsLastUse |= MMO->getFlags() & MOLastUse; InstrAddrSpace |= toSIAtomicAddrSpace(MMO->getPointerInfo().getAddrSpace()); AtomicOrdering OpOrdering = MMO->getSuccessOrdering(); @@ -792,7 +799,7 @@ std::optional SIMemOpAccess::constructFromMIWithMMO( } return SIMemOpInfo(Ordering, Scope, OrderingAddrSpace, InstrAddrSpace, IsCrossAddressSpaceOrdering, FailureOrdering, IsVolatile, - IsNonTemporal); + IsNonTemporal, IsLastUse); } std::optional @@ -969,7 +976,7 @@ bool SIGfx6CacheControl::enableRMWCacheBypass( bool SIGfx6CacheControl::enableVolatileAndOrNonTemporal( MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op, - bool IsVolatile, bool IsNonTemporal) const { + bool IsVolatile, bool IsNonTemporal, bool IsLastUse = false) const { // Only handle load and store, not atomic read-modify-write insructions. The // latter use glc to indicate if the atomic returns a result and so must not // be used for cache control. @@ -1322,7 +1329,7 @@ bool SIGfx90ACacheControl::enableRMWCacheBypass( bool SIGfx90ACacheControl::enableVolatileAndOrNonTemporal( MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op, - bool IsVolatile, bool IsNonTemporal) const { + bool IsVolatile, bool IsNonTemporal, bool IsLastUse = false) const { // Only handle load and store, not atomic read-modify-write insructions. The // latter use glc to indicate if the atomic returns a result and so must not // be used for cache control. @@ -1624,7 +1631,7 @@ bool SIGfx940CacheControl::enableRMWCacheBypass( bool SIGfx940CacheControl::enableVolatileAndOrNonTemporal( MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op, - bool IsVolatile, bool IsNonTemporal) const { + bool IsVolatile, bool IsNonTemporal, bool IsLastUse = false) const { // Only handle load and store, not atomic read-modify-write insructions. The // latter use glc to indicate if the atomic returns a result and so must not // be used for cache control. @@ -1856,7 +1863,7 @@ bool SIGfx10CacheControl::enableLoadCacheBypass( bool SIGfx10CacheControl::enableVolatileAndOrNonTemporal( MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op, - bool IsVolatile, bool IsNonTemporal) const { + bool IsVolatile, bool IsNonTemporal, bool IsLastUse = false) const { // Only handle load and store, not atomic read-modify-write insructions. The // latter use glc to indicate if the atomic returns a result and so must not @@ -2127,7 +2134,7 @@ bool SIGfx11CacheControl::enableLoadCacheBypass( bool SIGfx11CacheControl::enableVolatileAndOrNonTemporal( MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op, - bool IsVolatile, bool IsNonTemporal) const { + bool IsVolatile, bool IsNonTemporal, bool IsLastUse = false) const { // Only handle load and store, not atomic read-modify-write insructions. The // latter use glc to indicate if the atomic returns a result and so must not @@ -2379,7 +2386,7 @@ bool SIGfx12CacheControl::insertAcquire(MachineBasicBlock::iterator &MI, bool SIGfx12CacheControl::enableVolatileAndOrNonTemporal( MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op, - bool IsVolatile, bool IsNonTemporal) const { + bool IsVolatile, bool IsNonTemporal, bool IsLastUse = false) const { // Only handle load and store, not atomic read-modify-write instructions. assert(MI->mayLoad() ^ MI->mayStore()); @@ -2392,7 +2399,10 @@ bool SIGfx12CacheControl::enableVolatileAndOrNonTemporal( bool Changed = false; - if (IsNonTemporal) { + if (IsLastUse) { + // Set last-use hint. + Changed |= setTH(MI, AMDGPU::CPol::TH_LU); + } else if (IsNonTemporal) { // Set non-temporal hint for all cache levels. Changed |= setTH(MI, AMDGPU::CPol::TH_NT); } @@ -2472,11 +2482,12 @@ bool SIMemoryLegalizer::expandLoad(const SIMemOpInfo &MOI, } // Atomic instructions already bypass caches to the scope specified by the - // SyncScope operand. Only non-atomic volatile and nontemporal instructions - // need additional treatment. - Changed |= CC->enableVolatileAndOrNonTemporal(MI, MOI.getInstrAddrSpace(), - SIMemOp::LOAD, MOI.isVolatile(), - MOI.isNonTemporal()); + // SyncScope operand. Only non-atomic volatile and nontemporal/last-use + // instructions need additional treatment. + Changed |= CC->enableVolatileAndOrNonTemporal( + MI, MOI.getInstrAddrSpace(), SIMemOp::LOAD, MOI.isVolatile(), + MOI.isNonTemporal(), MOI.isLastUse()); + return Changed; } diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll new file mode 100644 index 000000000000..71ec5512c72d --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll @@ -0,0 +1,83 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12,GFX12-WGP %s +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12,GFX12-CU %s + +define amdgpu_kernel void @flat_last_use_load_0(ptr %in, ptr %out) { +; GFX12-LABEL: flat_last_use_load_0: +; GFX12: ; %bb.0: ; %entry +; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 +; GFX12-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_LU +; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: flat_store_b32 v[0:1], v2 +; GFX12-NEXT: s_endpgm +entry: + %val = load i32, ptr %in, align 4, !amdgpu.last.use !{} + store i32 %val, ptr %out + ret void +} + +define amdgpu_kernel void @flat_last_use_load_1(ptr %in, ptr %out) { +; GFX12-LABEL: flat_last_use_load_1: +; GFX12: ; %bb.0: ; %entry +; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 +; GFX12-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add_co_u32 v0, s0, s0, v0 +; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0 +; GFX12-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_LU +; GFX12-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2 +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: flat_store_b32 v[0:1], v2 +; GFX12-NEXT: s_endpgm +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %val.gep = getelementptr inbounds i32, ptr %in, i32 %tid + %val = load i32, ptr %val.gep, align 4, !amdgpu.last.use !{} + store i32 %val, ptr %out + ret void +} + +define amdgpu_kernel void @flat_last_use_and_volatile_load(ptr %in, ptr %out) { +; GFX12-LABEL: flat_last_use_and_volatile_load: +; GFX12: ; %bb.0: ; %entry +; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 +; GFX12-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_BYPASS scope:SCOPE_SYS +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 +; GFX12-NEXT: s_wait_dscnt 0x0 +; GFX12-NEXT: flat_store_b32 v[0:1], v2 +; GFX12-NEXT: s_endpgm +entry: + %val = load volatile i32, ptr %in, align 4, !amdgpu.last.use !{} + store i32 %val, ptr %out + ret void +} + +define amdgpu_kernel void @flat_last_use_and_nontemporal_load(ptr %in, ptr %out) { +; GFX12-LABEL: flat_last_use_and_nontemporal_load: +; GFX12: ; %bb.0: ; %entry +; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1 +; GFX12-NEXT: flat_load_b32 v2, v[0:1] th:TH_LOAD_LU +; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3 +; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-NEXT: flat_store_b32 v[0:1], v2 +; GFX12-NEXT: s_endpgm +entry: + %val = load i32, ptr %in, align 4, !amdgpu.last.use !{}, !nontemporal !0 + store i32 %val, ptr %out + ret void +} + +!0 = !{i32 1} +declare i32 @llvm.amdgcn.workitem.id.x() +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; GFX12-CU: {{.*}} +; GFX12-WGP: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll new file mode 100644 index 000000000000..fa2fc3c42369 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll @@ -0,0 +1,84 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12,GFX12-WGP %s +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12,GFX12-CU %s + +define amdgpu_kernel void @global_last_use_load_0(ptr addrspace(1) %in, ptr addrspace(1) %out) { +; GFX12-LABEL: global_last_use_load_0: +; GFX12: ; %bb.0: ; %entry +; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: s_load_b32 s0, s[0:1], 0x0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0 +; GFX12-NEXT: global_store_b32 v0, v1, s[2:3] +; GFX12-NEXT: s_nop 0 +; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX12-NEXT: s_endpgm +entry: + %val = load i32, ptr addrspace(1) %in, align 4, !amdgpu.last.use !{} + store i32 %val, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @global_last_use_load_1(ptr addrspace(1) %in, ptr addrspace(1) %out) { +; GFX12-LABEL: global_last_use_load_1: +; GFX12: ; %bb.0: ; %entry +; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 +; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: global_load_b32 v0, v0, s[0:1] th:TH_LOAD_LU +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_store_b32 v1, v0, s[2:3] +; GFX12-NEXT: s_nop 0 +; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX12-NEXT: s_endpgm +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %val.gep = getelementptr inbounds i32, ptr addrspace(1) %in, i32 %tid + %val = load i32, ptr addrspace(1) %val.gep, align 4, !amdgpu.last.use !{} + store i32 %val, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @global_last_use_and_volatile_load(ptr addrspace(1) %in, ptr addrspace(1) %out) { +; GFX12-LABEL: global_last_use_and_volatile_load: +; GFX12: ; %bb.0: ; %entry +; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 +; GFX12-NEXT: v_mov_b32_e32 v0, 0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: global_load_b32 v1, v0, s[0:1] th:TH_LOAD_BYPASS scope:SCOPE_SYS +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_store_b32 v0, v1, s[2:3] +; GFX12-NEXT: s_nop 0 +; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX12-NEXT: s_endpgm +entry: + %val = load volatile i32, ptr addrspace(1) %in, align 4, !amdgpu.last.use !{} + store i32 %val, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @global_last_use_and_nontemporal_load(ptr addrspace(1) %in, ptr addrspace(1) %out) { +; GFX12-LABEL: global_last_use_and_nontemporal_load: +; GFX12: ; %bb.0: ; %entry +; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 +; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: global_load_b32 v0, v0, s[0:1] th:TH_LOAD_LU +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_store_b32 v1, v0, s[2:3] +; GFX12-NEXT: s_nop 0 +; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX12-NEXT: s_endpgm +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %val.gep = getelementptr inbounds i32, ptr addrspace(1) %in, i32 %tid + %val = load i32, ptr addrspace(1) %val.gep, align 4, !amdgpu.last.use !{}, !nontemporal !0 + store i32 %val, ptr addrspace(1) %out + ret void +} +!0 = !{i32 1} +declare i32 @llvm.amdgcn.workitem.id.x() +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; GFX12-CU: {{.*}} +; GFX12-WGP: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-lastuse.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-lastuse.ll new file mode 100644 index 000000000000..1ce9fc308af3 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-lastuse.ll @@ -0,0 +1,90 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12,GFX12-WGP %s +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12,GFX12-CU %s + +define amdgpu_kernel void @private_last_use_load_0(ptr addrspace(5) %in, ptr addrspace(1) %out) { +; GFX12-LABEL: private_last_use_load_0: +; GFX12: ; %bb.0: ; %entry +; GFX12-NEXT: s_clause 0x1 +; GFX12-NEXT: s_load_b32 s2, s[0:1], 0x0 +; GFX12-NEXT: s_load_b64 s[0:1], s[0:1], 0x8 +; GFX12-NEXT: v_mov_b32_e32 v1, 0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: scratch_load_b32 v0, off, s2 th:TH_LOAD_LU +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_store_b32 v1, v0, s[0:1] +; GFX12-NEXT: s_nop 0 +; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX12-NEXT: s_endpgm +entry: + %val = load i32, ptr addrspace(5) %in, align 4, !amdgpu.last.use !{} + store i32 %val, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @private_last_use_load_1(ptr addrspace(5) %in, ptr addrspace(1) %out) { +; GFX12-LABEL: private_last_use_load_1: +; GFX12: ; %bb.0: ; %entry +; GFX12-NEXT: s_load_b32 s2, s[0:1], 0x0 +; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0 +; GFX12-NEXT: s_load_b64 s[0:1], s[0:1], 0x8 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: scratch_load_b32 v0, v0, s2 th:TH_LOAD_LU +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_store_b32 v1, v0, s[0:1] +; GFX12-NEXT: s_nop 0 +; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX12-NEXT: s_endpgm +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %val.gep = getelementptr inbounds i32, ptr addrspace(5) %in, i32 %tid + %val = load i32, ptr addrspace(5) %val.gep, align 4, !amdgpu.last.use !{} + store i32 %val, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @private_last_use_and_volatile_load(ptr addrspace(5) %in, ptr addrspace(1) %out) { +; GFX12-LABEL: private_last_use_and_volatile_load: +; GFX12: ; %bb.0: ; %entry +; GFX12-NEXT: s_clause 0x1 +; GFX12-NEXT: s_load_b32 s2, s[0:1], 0x0 +; GFX12-NEXT: s_load_b64 s[0:1], s[0:1], 0x8 +; GFX12-NEXT: v_mov_b32_e32 v1, 0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: scratch_load_b32 v0, off, s2 th:TH_LOAD_BYPASS scope:SCOPE_SYS +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_store_b32 v1, v0, s[0:1] +; GFX12-NEXT: s_nop 0 +; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX12-NEXT: s_endpgm +entry: + %val = load volatile i32, ptr addrspace(5) %in, align 4, !amdgpu.last.use !{} + store i32 %val, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @private_last_use_and_nontemporal_load(ptr addrspace(5) %in, ptr addrspace(1) %out) { +; GFX12-LABEL: private_last_use_and_nontemporal_load: +; GFX12: ; %bb.0: ; %entry +; GFX12-NEXT: s_clause 0x1 +; GFX12-NEXT: s_load_b32 s2, s[0:1], 0x0 +; GFX12-NEXT: s_load_b64 s[0:1], s[0:1], 0x8 +; GFX12-NEXT: v_mov_b32_e32 v1, 0 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: scratch_load_b32 v0, off, s2 th:TH_LOAD_LU +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_store_b32 v1, v0, s[0:1] +; GFX12-NEXT: s_nop 0 +; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX12-NEXT: s_endpgm +entry: + %val = load i32, ptr addrspace(5) %in, align 4, !amdgpu.last.use !{}, !nontemporal !0 + store i32 %val, ptr addrspace(1) %out + ret void +} + +!0 = !{i32 1} +declare i32 @llvm.amdgcn.workitem.id.x() +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; GFX12-CU: {{.*}} +; GFX12-WGP: {{.*}} -- GitLab From ad1b2a812973d584f56121b953bf9698b32386ae Mon Sep 17 00:00:00 2001 From: NagyDonat Date: Wed, 6 Mar 2024 16:42:31 +0100 Subject: [PATCH 304/929] [analyzer] Demonstrate superfluous unsigned >= 0 assumption (#78442) This commit adds a testcase which highlights the current incorrect behavior of the CSA diagnostic generation: it produces a note which says "Assuming 'arg' is >= 0" in a situation where this is not a fresh assumption because 'arg' is an unsigned integer. I also created ticket 78440 to track this bug. --- .../Core/BugReporterVisitors.cpp | 10 ++++++++++ clang/test/Analysis/assuming-unsigned-ge-0.c | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 clang/test/Analysis/assuming-unsigned-ge-0.c diff --git a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp index 2f9965036b9e..a0822513a6d0 100644 --- a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp +++ b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp @@ -2883,6 +2883,16 @@ ConditionBRVisitor::VisitTrueTest(const Expr *Cond, BugReporterContext &BRC, // previous program state we assuming the newly seen constraint information. // If we cannot evaluate the condition (and the constraints are the same) // the analyzer has no information about the value and just assuming it. + // FIXME: This logic is not entirely correct, because e.g. in code like + // void f(unsigned arg) { + // if (arg >= 0) { + // // ... + // } + // } + // it will say that the "arg >= 0" check is _assuming_ something new because + // the constraint that "$arg >= 0" is 1 was added to the list of known + // constraints. However, the unsigned value is always >= 0 so semantically + // this is not a "real" assumption. bool IsAssuming = !BRC.getStateManager().haveEqualConstraints(CurrentState, PrevState) || CurrentState->getSVal(Cond, LCtx).isUnknownOrUndef(); diff --git a/clang/test/Analysis/assuming-unsigned-ge-0.c b/clang/test/Analysis/assuming-unsigned-ge-0.c new file mode 100644 index 000000000000..553e68cb96c6 --- /dev/null +++ b/clang/test/Analysis/assuming-unsigned-ge-0.c @@ -0,0 +1,19 @@ +// RUN: %clang_analyze_cc1 -analyzer-output=text \ +// RUN: -analyzer-checker=core -verify %s + +int assuming_unsigned_ge_0(unsigned arg) { + // TODO This testcase demonstrates the current incorrect behavior of Clang + // Static Analyzer: here 'arg' is unsigned, so "arg >= 0" is not a fresh + // assumption, but it still appears in the diagnostics as if it's fresh: + // expected-note@+2 {{Assuming 'arg' is >= 0}} + // expected-note@+1 {{Taking false branch}} + if (arg < 0) + return 0; + // expected-note@+2 {{Assuming 'arg' is <= 0}} + // expected-note@+1 {{Taking false branch}} + if (arg > 0) + return 0; + // expected-note@+2 {{Division by zero}} + // expected-warning@+1 {{Division by zero}} + return 100 / arg; +} -- GitLab From f53c2f66a76d8a7dcc649d334cd60374b883196f Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 6 Mar 2024 14:54:06 +0000 Subject: [PATCH 305/929] [X86] combineSetCC - use getZExtOrTrunc() to perform the constant folding. NFCI --- llvm/lib/Target/X86/X86ISelLowering.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 6eaaec407dbb..697e8064dc44 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -53266,14 +53266,10 @@ static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG, APInt UpperBits = APInt::getBitsSetFrom(SrcVT.getScalarSizeInBits(), OpVT.getScalarSizeInBits()); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); - auto *C = cast(RHS); if (DAG.MaskedValueIsZero(LHS.getOperand(0), UpperBits) && TLI.isTypeLegal(LHS.getOperand(0).getValueType())) return DAG.getSetCC(DL, VT, LHS.getOperand(0), - DAG.getConstant(C->getAPIntValue().zextOrTrunc( - SrcVT.getScalarSizeInBits()), - DL, SrcVT), - CC); + DAG.getZExtOrTrunc(RHS, DL, SrcVT), CC); } // With C as a power of 2 and C != 0 and C != INT_MIN: -- GitLab From 5a896c66e3074621235a17f9dff95cbbed900c10 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 6 Mar 2024 15:07:49 +0000 Subject: [PATCH 306/929] [X86] Merge repeated getTargetLoweringInfo() calls. NFC. --- llvm/lib/Target/X86/X86ISelLowering.cpp | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 697e8064dc44..eebb7cf91c42 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -7200,6 +7200,7 @@ static SDValue lowerBuildVectorAsBroadcast(BuildVectorSDNode *BVOp, MVT VT = BVOp->getSimpleValueType(0); unsigned NumElts = VT.getVectorNumElements(); + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) && "Unsupported vector type for broadcast."); @@ -7265,7 +7266,6 @@ static SDValue lowerBuildVectorAsBroadcast(BuildVectorSDNode *BVOp, if (isFoldableUseOfShuffle(BVOp)) return SDValue(); // replace BUILD_VECTOR with broadcast of the repeated constants. - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); LLVMContext *Ctx = DAG.getContext(); MVT PVT = TLI.getPointerTy(DAG.getDataLayout()); if (SplatBitSize == 32 || SplatBitSize == 64 || @@ -7363,7 +7363,6 @@ static SDValue lowerBuildVectorAsBroadcast(BuildVectorSDNode *BVOp, assert(C && "Invalid constant type"); - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDValue CP = DAG.getConstantPool(C, TLI.getPointerTy(DAG.getDataLayout())); Align Alignment = cast(CP)->getAlign(); @@ -38219,6 +38218,7 @@ static SDValue combineX86ShuffleChain(ArrayRef Inputs, SDValue Root, unsigned NumMaskElts = Mask.size(); unsigned MaskEltSizeInBits = RootSizeInBits / NumMaskElts; + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); // Determine the effective mask value type. FloatDomain &= (32 <= MaskEltSizeInBits); @@ -38227,7 +38227,7 @@ static SDValue combineX86ShuffleChain(ArrayRef Inputs, SDValue Root, MaskVT = MVT::getVectorVT(MaskVT, NumMaskElts); // Only allow legal mask types. - if (!DAG.getTargetLoweringInfo().isTypeLegal(MaskVT)) + if (!TLI.isTypeLegal(MaskVT)) return SDValue(); // Attempt to match the mask against known shuffle patterns. @@ -38533,7 +38533,7 @@ static SDValue combineX86ShuffleChain(ArrayRef Inputs, SDValue Root, // which is much simpler than any shuffle. if (UnaryShuffle && MaskContainsZeros && AllowVariablePerLaneMask && isSequentialOrUndefOrZeroInRange(Mask, 0, NumMaskElts, 0) && - DAG.getTargetLoweringInfo().isTypeLegal(MaskVT)) { + TLI.isTypeLegal(MaskVT)) { APInt Zero = APInt::getZero(MaskEltSizeInBits); APInt AllOnes = APInt::getAllOnes(MaskEltSizeInBits); APInt UndefElts(NumMaskElts, 0); @@ -40073,6 +40073,7 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL, MVT VT = N.getSimpleValueType(); SmallVector Mask; unsigned Opcode = N.getOpcode(); + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); if (SDValue R = combineCommutableSHUFP(N, VT, DL, DAG)) return R; @@ -40124,7 +40125,7 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL, // 32-bit targets have to bitcast i64 to f64, so better to bitcast upward. if (Src.getOpcode() == ISD::BITCAST && SrcVT.getScalarSizeInBits() == BCVT.getScalarSizeInBits() && - DAG.getTargetLoweringInfo().isTypeLegal(BCVT) && + TLI.isTypeLegal(BCVT) && FixedVectorType::isValidElementType( BCVT.getScalarType().getTypeForEVT(*DAG.getContext()))) { EVT NewVT = EVT::getVectorVT(*DAG.getContext(), BCVT.getScalarType(), @@ -40162,8 +40163,7 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL, isNullConstant(Src.getOperand(1)) && Src.getValueType() == Src.getOperand(0).getValueType().getScalarType() && - DAG.getTargetLoweringInfo().isTypeLegal( - Src.getOperand(0).getValueType())) + TLI.isTypeLegal(Src.getOperand(0).getValueType())) return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Src.getOperand(0)); // Share broadcast with the longest vector and extract low subvector (free). @@ -40377,7 +40377,7 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL, ConstantVec[0] = const_cast(C->getConstantIntValue()); // Load the vector constant from constant pool. - MVT PVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); + MVT PVT = TLI.getPointerTy(DAG.getDataLayout()); SDValue CP = DAG.getConstantPool(ConstantVector::get(ConstantVec), PVT); MachinePointerInfo MPI = MachinePointerInfo::getConstantPool(DAG.getMachineFunction()); @@ -44555,7 +44555,6 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG, if (LoadVec && CIdx && ISD::isNormalLoad(LoadVec) && VT.isInteger() && SrcVT.getVectorElementType() == VT && DCI.isAfterLegalizeDAG() && !LikelyUsedAsVector && LoadVec->isSimple()) { - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDValue NewPtr = TLI.getVectorElementPointer(DAG, LoadVec->getBasePtr(), SrcVT, EltIdx); unsigned PtrOff = VT.getSizeInBits() * CIdx->getZExtValue() / 8; @@ -45957,6 +45956,7 @@ static SDValue combinePTESTCC(SDValue EFLAGS, X86::CondCode &CC, SDValue Op0 = EFLAGS.getOperand(0); SDValue Op1 = EFLAGS.getOperand(1); MVT OpVT = Op0.getSimpleValueType(); + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); // TEST*(~X,Y) == TEST*(X,Y) if (SDValue NotOp0 = IsNOT(Op0, DAG)) { @@ -46040,12 +46040,11 @@ static SDValue combinePTESTCC(SDValue EFLAGS, X86::CondCode &CC, // TODO: Handle TESTC with comparison inversion. // TODO: Can we remove SimplifyMultipleUseDemandedBits and rely on // TESTP/MOVMSK combines to make sure its never worse than PTEST? - if (BCVT.isVector() && DAG.getTargetLoweringInfo().isTypeLegal(BCVT)) { + if (BCVT.isVector() && TLI.isTypeLegal(BCVT)) { unsigned EltBits = BCVT.getScalarSizeInBits(); if (DAG.ComputeNumSignBits(BC) == EltBits) { assert(VT == MVT::i32 && "Expected i32 EFLAGS comparison result"); APInt SignMask = APInt::getSignMask(EltBits); - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); if (SDValue Res = TLI.SimplifyMultipleUseDemandedBits(BC, SignMask, DAG)) { // For vXi16 cases we need to use pmovmksb and extract every other @@ -53605,6 +53604,7 @@ static SDValue combineGatherScatter(SDNode *N, SelectionDAG &DAG, SDValue Index = GorS->getIndex(); SDValue Base = GorS->getBasePtr(); SDValue Scale = GorS->getScale(); + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); if (DCI.isBeforeLegalize()) { unsigned IndexWidth = Index.getScalarValueSizeInBits(); @@ -53639,7 +53639,6 @@ static SDValue combineGatherScatter(SDNode *N, SelectionDAG &DAG, } } - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); // Try to move splat constant adders from the index operand to the base // pointer operand. Taking care to multiply by the scale. We can only do @@ -53696,7 +53695,6 @@ static SDValue combineGatherScatter(SDNode *N, SelectionDAG &DAG, // With vector masks we only demand the upper bit of the mask. SDValue Mask = GorS->getMask(); if (Mask.getScalarValueSizeInBits() != 1) { - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); APInt DemandedMask(APInt::getSignMask(Mask.getScalarValueSizeInBits())); if (TLI.SimplifyDemandedBits(Mask, DemandedMask, DCI)) { if (N->getOpcode() != ISD::DELETED_NODE) -- GitLab From 371132962b3566659d533bfe99c3e3cfb0dd37bd Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 6 Mar 2024 15:18:24 +0000 Subject: [PATCH 307/929] [TTI] getInstructionCost - pull out repeated getShuffleMask() calls. NFC. Pulled out of #84156 --- .../llvm/Analysis/TargetTransformInfoImpl.h | 51 ++++++++----------- 1 file changed, 21 insertions(+), 30 deletions(-) diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h index 95fb13d1c971..71573b6b3b8f 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -1324,6 +1324,7 @@ public: auto *VecTy = cast(U->getType()); auto *VecSrcTy = cast(Operands[0]->getType()); + ArrayRef Mask = Shuffle->getShuffleMask(); int NumSubElts, SubIndex; if (Shuffle->changesLength()) { @@ -1333,21 +1334,19 @@ public: if (Shuffle->isExtractSubvectorMask(SubIndex)) return TargetTTI->getShuffleCost(TTI::SK_ExtractSubvector, VecSrcTy, - Shuffle->getShuffleMask(), CostKind, - SubIndex, VecTy, Operands); + Mask, CostKind, SubIndex, VecTy, + Operands); if (Shuffle->isInsertSubvectorMask(NumSubElts, SubIndex)) return TargetTTI->getShuffleCost( - TTI::SK_InsertSubvector, VecTy, Shuffle->getShuffleMask(), - CostKind, SubIndex, + TTI::SK_InsertSubvector, VecTy, Mask, CostKind, SubIndex, FixedVectorType::get(VecTy->getScalarType(), NumSubElts), Operands); int ReplicationFactor, VF; if (Shuffle->isReplicationMask(ReplicationFactor, VF)) { - APInt DemandedDstElts = - APInt::getZero(Shuffle->getShuffleMask().size()); - for (auto I : enumerate(Shuffle->getShuffleMask())) { + APInt DemandedDstElts = APInt::getZero(Mask.size()); + for (auto I : enumerate(Mask)) { if (I.value() != PoisonMaskElem) DemandedDstElts.setBit(I.index()); } @@ -1363,44 +1362,36 @@ public: return 0; if (Shuffle->isReverse()) - return TargetTTI->getShuffleCost(TTI::SK_Reverse, VecTy, - Shuffle->getShuffleMask(), CostKind, 0, - nullptr, Operands); + return TargetTTI->getShuffleCost(TTI::SK_Reverse, VecTy, Mask, CostKind, + 0, nullptr, Operands); if (Shuffle->isSelect()) - return TargetTTI->getShuffleCost(TTI::SK_Select, VecTy, - Shuffle->getShuffleMask(), CostKind, 0, - nullptr, Operands); + return TargetTTI->getShuffleCost(TTI::SK_Select, VecTy, Mask, CostKind, + 0, nullptr, Operands); if (Shuffle->isTranspose()) - return TargetTTI->getShuffleCost(TTI::SK_Transpose, VecTy, - Shuffle->getShuffleMask(), CostKind, 0, - nullptr, Operands); + return TargetTTI->getShuffleCost(TTI::SK_Transpose, VecTy, Mask, + CostKind, 0, nullptr, Operands); if (Shuffle->isZeroEltSplat()) - return TargetTTI->getShuffleCost(TTI::SK_Broadcast, VecTy, - Shuffle->getShuffleMask(), CostKind, 0, - nullptr, Operands); + return TargetTTI->getShuffleCost(TTI::SK_Broadcast, VecTy, Mask, + CostKind, 0, nullptr, Operands); if (Shuffle->isSingleSource()) - return TargetTTI->getShuffleCost(TTI::SK_PermuteSingleSrc, VecTy, - Shuffle->getShuffleMask(), CostKind, 0, - nullptr, Operands); + return TargetTTI->getShuffleCost(TTI::SK_PermuteSingleSrc, VecTy, Mask, + CostKind, 0, nullptr, Operands); if (Shuffle->isInsertSubvectorMask(NumSubElts, SubIndex)) return TargetTTI->getShuffleCost( - TTI::SK_InsertSubvector, VecTy, Shuffle->getShuffleMask(), CostKind, - SubIndex, FixedVectorType::get(VecTy->getScalarType(), NumSubElts), - Operands); + TTI::SK_InsertSubvector, VecTy, Mask, CostKind, SubIndex, + FixedVectorType::get(VecTy->getScalarType(), NumSubElts), Operands); if (Shuffle->isSplice(SubIndex)) - return TargetTTI->getShuffleCost(TTI::SK_Splice, VecTy, - Shuffle->getShuffleMask(), CostKind, + return TargetTTI->getShuffleCost(TTI::SK_Splice, VecTy, Mask, CostKind, SubIndex, nullptr, Operands); - return TargetTTI->getShuffleCost(TTI::SK_PermuteTwoSrc, VecTy, - Shuffle->getShuffleMask(), CostKind, 0, - nullptr, Operands); + return TargetTTI->getShuffleCost(TTI::SK_PermuteTwoSrc, VecTy, Mask, + CostKind, 0, nullptr, Operands); } case Instruction::ExtractElement: { auto *EEI = dyn_cast(U); -- GitLab From 6540f1635a6566aef93d9b4f568ac648474c39e7 Mon Sep 17 00:00:00 2001 From: Krzysztof Drewniak Date: Wed, 6 Mar 2024 09:49:58 -0600 Subject: [PATCH 308/929] [AMDGPU] Add IR-level pass to rewrite away address space 7 (#77952) This commit adds the -lower-buffer-fat-pointers pass, which is applicable to all AMDGCN compilations. The purpose of this pass is to remove the type `ptr addrspace(7)` from incoming IR. This must be done at the LLVM IR level because `ptr addrspace(7)`, as a 160-bit primitive type, cannot be correctly handled by SelectionDAG. The detailed operation of the pass is described in comments, but, in summary, the removal proceeds by: 1. Rewriting loads and stores of ptr addrspace(7) to loads and stores of i160 (including vectors and aggregates). This is needed because the in-register representation of these pointers will stop matching their in-memory representation in step 2, and so ptrtoint/inttoptr operations are used to preserve the expected memory layout 2. Mutating the IR to replace all occurrences of `ptr addrspace(7)` with the type `{ptr addrspace(8), ptr addrspace(6) }`, which makes the two parts of a buffer fat pointer (the 128-bit address space 8 resource and the 32-bit address space 6 offset) visible in the IR. This also impacts the argument and return types of functions. 3. *Splitting* the resource and offset parts. All instructions that produce or consume buffer fat pointers (like GEP or load) are rewritten to produce or consume the resource and offset parts separately. For example, GEP updates the offset part of the result and a load uses the resource and offset parts to populate the relevant llvm.amdgcn.raw.ptr.buffer.load intrinsic call. At the end of this process, the original mutated instructions are replaced by their new split counterparts, ensuring no invalidly-typed IR escapes this pass. (For operations like call, where the struct form is needed, insertelement operations are inserted). Compared to LGC's PatchBufferOp ( https://github.com/GPUOpen-Drivers/llpc/blob/32cda89776980202597d5bf4ed4447a1bae64047/lgc/patch/PatchBufferOp.cpp ): this pass - Also handles vectors of ptr addrspace(7)s - Also handles function boundaries - Includes the same uniform buffer optimization for loops and conditionals - Does *not* handle memcpy() and friends (this is future work) - Does *not* break up large loads and stores into smaller parts. This should be handled by extending the legalization of *.buffer.{load,store} to handle larger types by producing multiple instructions (the same way ordinary LOAD and STORE are legalized). That work is planned for a followup commit. - Does *not* have special logic for handling divergent buffer descriptors. The logic in LGC is, as far as I can tell, incorrect in general, and, per discussions with @nhaehnle, isn't widely used. Therefore, divergent descriptors are handled with waterfall loops later in legalization. As a final matter, this commit updates atomic expansion to treat buffer operations analogously to global ones. (One question for reviewers: is the new pass is the right place? Should it be later in the pipeline?) Differential Revision: https://reviews.llvm.org/D158463 --- llvm/lib/Target/AMDGPU/AMDGPU.h | 13 + .../AMDGPU/AMDGPULowerBufferFatPointers.cpp | 2012 +++++++++++++++++ .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 29 + llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 + llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 14 +- ...tor-non-integral-address-spaces-vectors.ll | 68 +- ...rtranslator-non-integral-address-spaces.ll | 7 +- llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 25 + .../AMDGPU/lower-buffer-fat-pointers-calls.ll | 114 + .../lower-buffer-fat-pointers-constants.ll | 220 ++ .../lower-buffer-fat-pointers-control-flow.ll | 458 ++++ .../lower-buffer-fat-pointers-memops.ll | 189 ++ .../lower-buffer-fat-pointers-p7-in-memory.ll | 154 ++ .../lower-buffer-fat-pointers-pointer-ops.ll | 480 ++++ ...fer-fat-pointers-unoptimized-debug-data.ll | 127 ++ .../secondary/llvm/lib/Target/AMDGPU/BUILD.gn | 1 + 16 files changed, 3898 insertions(+), 14 deletions(-) create mode 100644 llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp create mode 100644 llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-calls.ll create mode 100644 llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll create mode 100644 llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-control-flow.ll create mode 100644 llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memops.ll create mode 100644 llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-p7-in-memory.ll create mode 100644 llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-pointer-ops.ll create mode 100644 llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h index 36af767a70b0..6016bd5187d8 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.h +++ b/llvm/lib/Target/AMDGPU/AMDGPU.h @@ -59,6 +59,7 @@ FunctionPass *createAMDGPUMachineCFGStructurizerPass(); FunctionPass *createAMDGPURewriteOutArgumentsPass(); ModulePass * createAMDGPULowerModuleLDSLegacyPass(const AMDGPUTargetMachine *TM = nullptr); +ModulePass *createAMDGPULowerBufferFatPointersPass(); FunctionPass *createSIModeRegisterPass(); FunctionPass *createGCNPreRAOptimizationsPass(); @@ -136,6 +137,18 @@ struct AMDGPULowerModuleLDSPass : PassInfoMixin { PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); }; +void initializeAMDGPULowerBufferFatPointersPass(PassRegistry &); +extern char &AMDGPULowerBufferFatPointersID; + +struct AMDGPULowerBufferFatPointersPass + : PassInfoMixin { + AMDGPULowerBufferFatPointersPass(const TargetMachine &TM) : TM(TM) {} + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); + +private: + const TargetMachine &TM; +}; + void initializeAMDGPURewriteOutArgumentsPass(PassRegistry &); extern char &AMDGPURewriteOutArgumentsID; diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp new file mode 100644 index 000000000000..34fcb9aec38f --- /dev/null +++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp @@ -0,0 +1,2012 @@ +//===-- AMDGPULowerBufferFatPointers.cpp ---------------------------=// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This pass lowers operations on buffer fat pointers (addrspace 7) to +// operations on buffer resources (addrspace 8) and is needed for correct +// codegen. +// +// # Background +// +// Address space 7 (the buffer fat pointer) is a 160-bit pointer that consists +// of a 128-bit buffer descriptor and a 32-bit offset into that descriptor. +// The buffer resource part needs to be it needs to be a "raw" buffer resource +// (it must have a stride of 0 and bounds checks must be in raw buffer mode +// or disabled). +// +// When these requirements are met, a buffer resource can be treated as a +// typical (though quite wide) pointer that follows typical LLVM pointer +// semantics. This allows the frontend to reason about such buffers (which are +// often encountered in the context of SPIR-V kernels). +// +// However, because of their non-power-of-2 size, these fat pointers cannot be +// present during translation to MIR (though this restriction may be lifted +// during the transition to GlobalISel). Therefore, this pass is needed in order +// to correctly implement these fat pointers. +// +// The resource intrinsics take the resource part (the address space 8 pointer) +// and the offset part (the 32-bit integer) as separate arguments. In addition, +// many users of these buffers manipulate the offset while leaving the resource +// part alone. For these reasons, we want to typically separate the resource +// and offset parts into separate variables, but combine them together when +// encountering cases where this is required, such as by inserting these values +// into aggretates or moving them to memory. +// +// Therefore, at a high level, `ptr addrspace(7) %x` becomes `ptr addrspace(8) +// %x.rsrc` and `i32 %x.off`, which will be combined into `{ptr addrspace(8), +// i32} %x = {%x.rsrc, %x.off}` if needed. Similarly, `vector` becomes +// `{vector, vector}` and its component parts. +// +// # Implementation +// +// This pass proceeds in three main phases: +// +// ## Rewriting loads and stores of p7 +// +// The first phase is to rewrite away all loads and stors of `ptr addrspace(7)`, +// including aggregates containing such pointers, to ones that use `i160`. This +// is handled by `StoreFatPtrsAsIntsVisitor` , which visits loads, stores, and +// allocas and, if the loaded or stored type contains `ptr addrspace(7)`, +// rewrites that type to one where the p7s are replaced by i160s, copying other +// parts of aggregates as needed. In the case of a store, each pointer is +// `ptrtoint`d to i160 before storing, and load integers are `inttoptr`d back. +// This same transformation is applied to vectors of pointers. +// +// Such a transformation allows the later phases of the pass to not need +// to handle buffer fat pointers moving to and from memory, where we load +// have to handle the incompatibility between a `{Nxp8, Nxi32}` representation +// and `Nxi60` directly. Instead, that transposing action (where the vectors +// of resources and vectors of offsets are concatentated before being stored to +// memory) are handled through implementing `inttoptr` and `ptrtoint` only. +// +// Atomics operations on `ptr addrspace(7)` values are not suppported, as the +// hardware does not include a 160-bit atomic. +// +// ## Type remapping +// +// We use a `ValueMapper` to mangle uses of [vectors of] buffer fat pointers +// to the corresponding struct type, which has a resource part and an offset +// part. +// +// This uses a `BufferFatPtrToStructTypeMap` and a `FatPtrConstMaterializer` +// to, usually by way of `setType`ing values. Constants are handled here +// because there isn't a good way to fix them up later. +// +// This has the downside of leaving the IR in an invalid state (for example, +// the instruction `getelementptr {ptr addrspace(8), i32} %p, ...` will exist), +// but all such invalid states will be resolved by the third phase. +// +// Functions that don't take buffer fat pointers are modified in place. Those +// that do take such pointers have their basic blocks moved to a new function +// with arguments that are {ptr addrspace(8), i32} arguments and return values. +// This phase also records intrinsics so that they can be remangled or deleted +// later. +// +// +// ## Splitting pointer structs +// +// The meat of this pass consists of defining semantics for operations that +// produce or consume [vectors of] buffer fat pointers in terms of their +// resource and offset parts. This is accomplished throgh the `SplitPtrStructs` +// visitor. +// +// In the first pass through each function that is being lowered, the splitter +// inserts new instructions to implement the split-structures behavior, which is +// needed for correctness and performance. It records a list of "split users", +// instructions that are being replaced by operations on the resource and offset +// parts. +// +// Split users do not necessarily need to produce parts themselves ( +// a `load float, ptr addrspace(7)` does not, for example), but, if they do not +// generate fat buffer pointers, they must RAUW in their replacement +// instructions during the initial visit. +// +// When these new instructions are created, they use the split parts recorded +// for their initial arguments in order to generate their replacements, creating +// a parallel set of instructions that does not refer to the original fat +// pointer values but instead to their resource and offset components. +// +// Instructions, such as `extractvalue`, that produce buffer fat pointers from +// sources that do not have split parts, have such parts generated using +// `extractvalue`. This is also the initial handling of PHI nodes, which +// are then cleaned up. +// +// ### Conditionals +// +// PHI nodes are initially given resource parts via `extractvalue`. However, +// this is not an efficient rewrite of such nodes, as, in most cases, the +// resource part in a conditional or loop remains constant throughout the loop +// and only the offset varies. Failing to optimize away these constant resources +// would cause additional registers to be sent around loops and might lead to +// waterfall loops being generated for buffer operations due to the +// "non-uniform" resource argument. +// +// Therefore, after all instructions have been visited, the pointer splitter +// post-processes all encountered conditionals. Given a PHI node or select, +// getPossibleRsrcRoots() collects all values that the resource parts of that +// conditional's input could come from as well as collecting all conditional +// instructions encountered during the search. If, after filtering out the +// initial node itself, the set of encountered conditionals is a subset of the +// potential roots and there is a single potential resource that isn't in the +// conditional set, that value is the only possible value the resource argument +// could have throughout the control flow. +// +// If that condition is met, then a PHI node can have its resource part changed +// to the singleton value and then be replaced by a PHI on the offsets. +// Otherwise, each PHI node is split into two, one for the resource part and one +// for the offset part, which replace the temporary `extractvalue` instructions +// that were added during the first pass. +// +// Similar logic applies to `select`, where +// `%z = select i1 %cond, %cond, ptr addrspace(7) %x, ptr addrspace(7) %y` +// can be split into `%z.rsrc = %x.rsrc` and +// `%z.off = select i1 %cond, ptr i32 %x.off, i32 %y.off` +// if both `%x` and `%y` have the same resource part, but two `select` +// operations will be needed if they do not. +// +// ### Final processing +// +// After conditionals have been cleaned up, the IR for each function is +// rewritten to remove all the old instructions that have been split up. +// +// Any instruction that used to produce a buffer fat pointer (and therefore now +// produces a resource-and-offset struct after type remapping) is +// replaced as follows: +// 1. All debug value annotations are cloned to reflect that the resource part +// and offset parts are computed separately and constitute different +// fragments of the underlying source language variable. +// 2. All uses that were themselves split are replaced by a `poison` of the +// struct type, as they will themselves be erased soon. This rule, combined +// with debug handling, should leave the use lists of split instructions +// empty in almost all cases. +// 3. If a user of the original struct-valued result remains, the structure +// needed for the new types to work is constructed out of the newly-defined +// parts, and the original instruction is replaced by this structure +// before being erased. Instructions requiring this construction include +// `ret` and `insertvalue`. +// +// # Consequences +// +// This pass does not alter the CFG. +// +// Alias analysis information will become coarser, as the LLVM alias analyzer +// cannot handle the buffer intrinsics. Specifically, while we can determine +// that the following two loads do not alias: +// ``` +// %y = getelementptr i32, ptr addrspace(7) %x, i32 1 +// %a = load i32, ptr addrspace(7) %x +// %b = load i32, ptr addrspace(7) %y +// ``` +// we cannot (except through some code that runs during scheduling) determine +// that the rewritten loads below do not alias. +// ``` +// %y.off = add i32 %x.off, 1 +// %a = call @llvm.amdgcn.raw.ptr.buffer.load(ptr addrspace(8) %x.rsrc, i32 +// %x.off, ...) +// %b = call @llvm.amdgcn.raw.ptr.buffer.load(ptr addrspace(8) +// %x.rsrc, i32 %y.off, ...) +// ``` +// However, existing alias information is preserved. +//===----------------------------------------------------------------------===// + +#include "AMDGPU.h" +#include "AMDGPUTargetMachine.h" +#include "GCNSubtarget.h" +#include "SIDefines.h" +#include "llvm/ADT/SetOperations.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/ConstantFolding.h" +#include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/IR/AttributeMask.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DebugInfo.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/InstIterator.h" +#include "llvm/IR/InstVisitor.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicsAMDGPU.h" +#include "llvm/IR/Metadata.h" +#include "llvm/IR/Operator.h" +#include "llvm/IR/PatternMatch.h" +#include "llvm/InitializePasses.h" +#include "llvm/Pass.h" +#include "llvm/Support/AtomicOrdering.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Transforms/Utils/Cloning.h" +#include "llvm/Transforms/Utils/Local.h" +#include "llvm/Transforms/Utils/ValueMapper.h" + +#define DEBUG_TYPE "amdgpu-lower-buffer-fat-pointers" + +using namespace llvm; + +static constexpr unsigned BufferOffsetWidth = 32; + +namespace { +/// Recursively replace instances of ptr addrspace(7) and vector with some other type as defined by the relevant subclass. +class BufferFatPtrTypeLoweringBase : public ValueMapTypeRemapper { + DenseMap Map; + + Type *remapTypeImpl(Type *Ty, SmallPtrSetImpl &Seen); + +protected: + virtual Type *remapScalar(PointerType *PT) = 0; + virtual Type *remapVector(VectorType *VT) = 0; + + const DataLayout &DL; + +public: + BufferFatPtrTypeLoweringBase(const DataLayout &DL) : DL(DL) {} + Type *remapType(Type *SrcTy) override; + void clear() { Map.clear(); } +}; + +/// Remap ptr addrspace(7) to i160 and vector to +/// vector in order to correctly handling loading/storing these values +/// from memory. +class BufferFatPtrToIntTypeMap : public BufferFatPtrTypeLoweringBase { + using BufferFatPtrTypeLoweringBase::BufferFatPtrTypeLoweringBase; + +protected: + Type *remapScalar(PointerType *PT) override { return DL.getIntPtrType(PT); } + Type *remapVector(VectorType *VT) override { return DL.getIntPtrType(VT); } +}; + +/// Remap ptr addrspace(7) to {ptr addrspace(8), i32} (the resource and offset +/// parts of the pointer) so that we can easily rewrite operations on these +/// values that aren't loading them from or storing them to memory. +class BufferFatPtrToStructTypeMap : public BufferFatPtrTypeLoweringBase { + using BufferFatPtrTypeLoweringBase::BufferFatPtrTypeLoweringBase; + +protected: + Type *remapScalar(PointerType *PT) override; + Type *remapVector(VectorType *VT) override; +}; +} // namespace + +// This code is adapted from the type remapper in lib/Linker/IRMover.cpp +Type *BufferFatPtrTypeLoweringBase::remapTypeImpl( + Type *Ty, SmallPtrSetImpl &Seen) { + Type **Entry = &Map[Ty]; + if (*Entry) + return *Entry; + if (auto *PT = dyn_cast(Ty)) { + if (PT->getAddressSpace() == AMDGPUAS::BUFFER_FAT_POINTER) { + return *Entry = remapScalar(PT); + } + } + if (auto *VT = dyn_cast(Ty)) { + auto *PT = dyn_cast(VT->getElementType()); + if (PT && PT->getAddressSpace() == AMDGPUAS::BUFFER_FAT_POINTER) { + return *Entry = remapVector(VT); + } + return *Entry = Ty; + } + // Whether the type is one that is structurally uniqued - that is, if it is + // not a named struct (the only kind of type where multiple structurally + // identical types that have a distinct `Type*`) + StructType *TyAsStruct = dyn_cast(Ty); + bool IsUniqued = !TyAsStruct || TyAsStruct->isLiteral(); + // Base case for ints, floats, opaque pointers, and so on, which don't + // require recursion. + if (Ty->getNumContainedTypes() == 0 && IsUniqued) + return *Entry = Ty; + if (!IsUniqued) { + // Create a dummy type for recursion purposes. + if (!Seen.insert(TyAsStruct).second) { + StructType *Placeholder = StructType::create(Ty->getContext()); + return *Entry = Placeholder; + } + } + bool Changed = false; + SmallVector ElementTypes(Ty->getNumContainedTypes(), nullptr); + for (unsigned int I = 0, E = Ty->getNumContainedTypes(); I < E; ++I) { + Type *OldElem = Ty->getContainedType(I); + Type *NewElem = remapTypeImpl(OldElem, Seen); + ElementTypes[I] = NewElem; + Changed |= (OldElem != NewElem); + } + // Recursive calls to remapTypeImpl() may have invalidated pointer. + Entry = &Map[Ty]; + if (!Changed) { + return *Entry = Ty; + } + if (auto *ArrTy = dyn_cast(Ty)) + return *Entry = ArrayType::get(ElementTypes[0], ArrTy->getNumElements()); + if (auto *FnTy = dyn_cast(Ty)) + return *Entry = FunctionType::get(ElementTypes[0], + ArrayRef(ElementTypes).slice(1), + FnTy->isVarArg()); + if (auto *STy = dyn_cast(Ty)) { + // Genuine opaque types don't have a remapping. + if (STy->isOpaque()) + return *Entry = Ty; + bool IsPacked = STy->isPacked(); + if (IsUniqued) + return *Entry = StructType::get(Ty->getContext(), ElementTypes, IsPacked); + SmallString<16> Name(STy->getName()); + STy->setName(""); + Type **RecursionEntry = &Map[Ty]; + if (*RecursionEntry) { + auto *Placeholder = cast(*RecursionEntry); + Placeholder->setBody(ElementTypes, IsPacked); + Placeholder->setName(Name); + return *Entry = Placeholder; + } + return *Entry = StructType::create(Ty->getContext(), ElementTypes, Name, + IsPacked); + } + llvm_unreachable("Unknown type of type that contains elements"); +} + +Type *BufferFatPtrTypeLoweringBase::remapType(Type *SrcTy) { + SmallPtrSet Visited; + return remapTypeImpl(SrcTy, Visited); +} + +Type *BufferFatPtrToStructTypeMap::remapScalar(PointerType *PT) { + LLVMContext &Ctx = PT->getContext(); + return StructType::get(PointerType::get(Ctx, AMDGPUAS::BUFFER_RESOURCE), + IntegerType::get(Ctx, BufferOffsetWidth)); +} + +Type *BufferFatPtrToStructTypeMap::remapVector(VectorType *VT) { + ElementCount EC = VT->getElementCount(); + LLVMContext &Ctx = VT->getContext(); + Type *RsrcVec = + VectorType::get(PointerType::get(Ctx, AMDGPUAS::BUFFER_RESOURCE), EC); + Type *OffVec = VectorType::get(IntegerType::get(Ctx, BufferOffsetWidth), EC); + return StructType::get(RsrcVec, OffVec); +} + +static bool isBufferFatPtrOrVector(Type *Ty) { + if (auto *PT = dyn_cast(Ty->getScalarType())) + return PT->getAddressSpace() == AMDGPUAS::BUFFER_FAT_POINTER; + return false; +} + +// True if the type is {ptr addrspace(8), i32} or a struct containing vectors of +// those types. Used to quickly skip instructions we don't need to process. +static bool isSplitFatPtr(Type *Ty) { + auto *ST = dyn_cast(Ty); + if (!ST) + return false; + if (!ST->isLiteral() || ST->getNumElements() != 2) + return false; + auto *MaybeRsrc = + dyn_cast(ST->getElementType(0)->getScalarType()); + auto *MaybeOff = + dyn_cast(ST->getElementType(1)->getScalarType()); + return MaybeRsrc && MaybeOff && + MaybeRsrc->getAddressSpace() == AMDGPUAS::BUFFER_RESOURCE && + MaybeOff->getBitWidth() == BufferOffsetWidth; +} + +// True if the result type or any argument types are buffer fat pointers. +static bool isBufferFatPtrConst(Constant *C) { + Type *T = C->getType(); + return isBufferFatPtrOrVector(T) || any_of(C->operands(), [](const Use &U) { + return isBufferFatPtrOrVector(U.get()->getType()); + }); +} + +namespace { +/// Convert [vectors of] buffer fat pointers to integers when they are read from +/// or stored to memory. This ensures that these pointers will have the same +/// memory layout as before they are lowered, even though they will no longer +/// have their previous layout in registers/in the program (they'll be broken +/// down into resource and offset parts). This has the downside of imposing +/// marshalling costs when reading or storing these values, but since placing +/// such pointers into memory is an uncommon operation at best, we feel that +/// this cost is acceptable for better performance in the common case. +class StoreFatPtrsAsIntsVisitor + : public InstVisitor { + BufferFatPtrToIntTypeMap *TypeMap; + + ValueToValueMapTy ConvertedForStore; + + IRBuilder<> IRB; + + // Convert all the buffer fat pointers within the input value to inttegers + // so that it can be stored in memory. + Value *fatPtrsToInts(Value *V, Type *From, Type *To, const Twine &Name); + // Convert all the i160s that need to be buffer fat pointers (as specified) + // by the To type) into those pointers to preserve the semantics of the rest + // of the program. + Value *intsToFatPtrs(Value *V, Type *From, Type *To, const Twine &Name); + +public: + StoreFatPtrsAsIntsVisitor(BufferFatPtrToIntTypeMap *TypeMap, LLVMContext &Ctx) + : TypeMap(TypeMap), IRB(Ctx) {} + bool processFunction(Function &F); + + bool visitInstruction(Instruction &I) { return false; } + bool visitAllocaInst(AllocaInst &I); + bool visitLoadInst(LoadInst &LI); + bool visitStoreInst(StoreInst &SI); + bool visitGetElementPtrInst(GetElementPtrInst &I); +}; +} // namespace + +Value *StoreFatPtrsAsIntsVisitor::fatPtrsToInts(Value *V, Type *From, Type *To, + const Twine &Name) { + if (From == To) + return V; + ValueToValueMapTy::iterator Find = ConvertedForStore.find(V); + if (Find != ConvertedForStore.end()) + return Find->second; + if (isBufferFatPtrOrVector(From)) { + Value *Cast = IRB.CreatePtrToInt(V, To, Name + ".int"); + ConvertedForStore[V] = Cast; + return Cast; + } + if (From->getNumContainedTypes() == 0) + return V; + // Structs, arrays, and other compound types. + Value *Ret = PoisonValue::get(To); + if (auto *AT = dyn_cast(From)) { + Type *FromPart = AT->getArrayElementType(); + Type *ToPart = cast(To)->getElementType(); + for (uint64_t I = 0, E = AT->getArrayNumElements(); I < E; ++I) { + Value *Field = IRB.CreateExtractValue(V, I); + Value *NewField = + fatPtrsToInts(Field, FromPart, ToPart, Name + "." + Twine(I)); + Ret = IRB.CreateInsertValue(Ret, NewField, I); + } + } else { + for (auto [Idx, FromPart, ToPart] : + enumerate(From->subtypes(), To->subtypes())) { + Value *Field = IRB.CreateExtractValue(V, Idx); + Value *NewField = + fatPtrsToInts(Field, FromPart, ToPart, Name + "." + Twine(Idx)); + Ret = IRB.CreateInsertValue(Ret, NewField, Idx); + } + } + ConvertedForStore[V] = Ret; + return Ret; +} + +Value *StoreFatPtrsAsIntsVisitor::intsToFatPtrs(Value *V, Type *From, Type *To, + const Twine &Name) { + if (From == To) + return V; + if (isBufferFatPtrOrVector(To)) { + Value *Cast = IRB.CreateIntToPtr(V, To, Name + ".ptr"); + return Cast; + } + if (From->getNumContainedTypes() == 0) + return V; + // Structs, arrays, and other compound types. + Value *Ret = PoisonValue::get(To); + if (auto *AT = dyn_cast(From)) { + Type *FromPart = AT->getArrayElementType(); + Type *ToPart = cast(To)->getElementType(); + for (uint64_t I = 0, E = AT->getArrayNumElements(); I < E; ++I) { + Value *Field = IRB.CreateExtractValue(V, I); + Value *NewField = + intsToFatPtrs(Field, FromPart, ToPart, Name + "." + Twine(I)); + Ret = IRB.CreateInsertValue(Ret, NewField, I); + } + } else { + for (auto [Idx, FromPart, ToPart] : + enumerate(From->subtypes(), To->subtypes())) { + Value *Field = IRB.CreateExtractValue(V, Idx); + Value *NewField = + intsToFatPtrs(Field, FromPart, ToPart, Name + "." + Twine(Idx)); + Ret = IRB.CreateInsertValue(Ret, NewField, Idx); + } + } + return Ret; +} + +bool StoreFatPtrsAsIntsVisitor::processFunction(Function &F) { + bool Changed = false; + // The visitors will mutate GEPs and allocas, but will push loads and stores + // to the worklist to avoid invalidation. + for (Instruction &I : make_early_inc_range(instructions(F))) { + Changed |= visit(I); + } + ConvertedForStore.clear(); + return Changed; +} + +bool StoreFatPtrsAsIntsVisitor::visitAllocaInst(AllocaInst &I) { + Type *Ty = I.getAllocatedType(); + Type *NewTy = TypeMap->remapType(Ty); + if (Ty == NewTy) + return false; + I.setAllocatedType(NewTy); + return true; +} + +bool StoreFatPtrsAsIntsVisitor::visitGetElementPtrInst(GetElementPtrInst &I) { + Type *Ty = I.getSourceElementType(); + Type *NewTy = TypeMap->remapType(Ty); + if (Ty == NewTy) + return false; + // We'll be rewriting the type `ptr addrspace(7)` out of existence soon, so + // make sure GEPs don't have different semantics with the new type. + I.setSourceElementType(NewTy); + I.setResultElementType(TypeMap->remapType(I.getResultElementType())); + return true; +} + +bool StoreFatPtrsAsIntsVisitor::visitLoadInst(LoadInst &LI) { + Type *Ty = LI.getType(); + Type *IntTy = TypeMap->remapType(Ty); + if (Ty == IntTy) + return false; + + IRB.SetInsertPoint(&LI); + auto *NLI = cast(LI.clone()); + NLI->mutateType(IntTy); + NLI = IRB.Insert(NLI); + copyMetadataForLoad(*NLI, LI); + NLI->takeName(&LI); + + Value *CastBack = intsToFatPtrs(NLI, IntTy, Ty, NLI->getName()); + LI.replaceAllUsesWith(CastBack); + LI.eraseFromParent(); + return true; +} + +bool StoreFatPtrsAsIntsVisitor::visitStoreInst(StoreInst &SI) { + Value *V = SI.getValueOperand(); + Type *Ty = V->getType(); + Type *IntTy = TypeMap->remapType(Ty); + if (Ty == IntTy) + return false; + + IRB.SetInsertPoint(&SI); + Value *IntV = fatPtrsToInts(V, Ty, IntTy, V->getName()); + for (auto *Dbg : at::getAssignmentMarkers(&SI)) + Dbg->setValue(IntV); + + SI.setOperand(0, IntV); + return true; +} + +/// Return the ptr addrspace(8) and i32 (resource and offset parts) in a lowered +/// buffer fat pointer constant. +static std::pair +splitLoweredFatBufferConst(Constant *C) { + if (auto *AZ = dyn_cast(C)) + return std::make_pair(AZ->getStructElement(0), AZ->getStructElement(1)); + if (auto *SC = dyn_cast(C)) + return std::make_pair(SC->getOperand(0), SC->getOperand(1)); + llvm_unreachable("Conversion should've created a {p8, i32} struct"); +} + +namespace { +/// Handle the remapping of ptr addrspace(7) constants. +class FatPtrConstMaterializer final : public ValueMaterializer { + BufferFatPtrToStructTypeMap *TypeMap; + BufferFatPtrToIntTypeMap *IntTypeMap; + // An internal mapper that is used to recurse into the arguments of constants. + // While the documentation for `ValueMapper` specifies not to use it + // recursively, examination of the logic in mapValue() shows that it can + // safely be used recursively when handling constants, like it does in its own + // logic. + ValueMapper InternalMapper; + + Constant *materializeBufferFatPtrConst(Constant *C); + + const DataLayout &DL; + +public: + // UnderlyingMap is the value map this materializer will be filling. + FatPtrConstMaterializer(BufferFatPtrToStructTypeMap *TypeMap, + ValueToValueMapTy &UnderlyingMap, + BufferFatPtrToIntTypeMap *IntTypeMap, + const DataLayout &DL) + : TypeMap(TypeMap), IntTypeMap(IntTypeMap), + InternalMapper(UnderlyingMap, RF_None, TypeMap, this), DL(DL) {} + virtual ~FatPtrConstMaterializer() = default; + + Value *materialize(Value *V) override; +}; +} // namespace + +Constant *FatPtrConstMaterializer::materializeBufferFatPtrConst(Constant *C) { + Type *SrcTy = C->getType(); + auto *NewTy = dyn_cast(TypeMap->remapType(SrcTy)); + if (C->isNullValue()) + return ConstantAggregateZero::getNullValue(NewTy); + if (isa(C)) { + return ConstantStruct::get(NewTy, + {PoisonValue::get(NewTy->getElementType(0)), + PoisonValue::get(NewTy->getElementType(1))}); + } + if (isa(C)) { + return ConstantStruct::get(NewTy, + {UndefValue::get(NewTy->getElementType(0)), + UndefValue::get(NewTy->getElementType(1))}); + } + + if (isa(C)) + report_fatal_error("Global values containing ptr addrspace(7) (buffer " + "fat pointer) values are not supported"); + + if (auto *VC = dyn_cast(C)) { + if (Constant *S = VC->getSplatValue()) { + Constant *NewS = InternalMapper.mapConstant(*S); + if (!NewS) + return nullptr; + auto [Rsrc, Off] = splitLoweredFatBufferConst(NewS); + auto EC = VC->getType()->getElementCount(); + return ConstantStruct::get(NewTy, {ConstantVector::getSplat(EC, Rsrc), + ConstantVector::getSplat(EC, Off)}); + } + SmallVector Rsrcs; + SmallVector Offs; + for (Value *Op : VC->operand_values()) { + auto *NewOp = dyn_cast_or_null(InternalMapper.mapValue(*Op)); + if (!NewOp) + return nullptr; + auto [Rsrc, Off] = splitLoweredFatBufferConst(NewOp); + Rsrcs.push_back(Rsrc); + Offs.push_back(Off); + } + Constant *RsrcVec = ConstantVector::get(Rsrcs); + Constant *OffVec = ConstantVector::get(Offs); + return ConstantStruct::get(NewTy, {RsrcVec, OffVec}); + } + + // Constant expressions. This code mirrors how we fix up the equivalent + // instructions later. + auto *CE = dyn_cast(C); + if (!CE) + return nullptr; + if (auto *GEPO = dyn_cast(C)) { + Constant *RemappedPtr = + InternalMapper.mapConstant(*cast(GEPO->getPointerOperand())); + auto [Rsrc, Off] = splitLoweredFatBufferConst(RemappedPtr); + Type *OffTy = Off->getType(); + bool InBounds = GEPO->isInBounds(); + + MapVector VariableOffs; + APInt NewConstOffVal = APInt::getZero(BufferOffsetWidth); + if (!GEPO->collectOffset(DL, BufferOffsetWidth, VariableOffs, + NewConstOffVal)) + report_fatal_error( + "Scalable vector or unsized struct in fat pointer GEP"); + Constant *OffAccum = nullptr; + // Accumulate offsets together before adding to the base in order to + // preserve as many of the inbounds properties as possible. + for (auto [Arg, Multiple] : VariableOffs) { + Constant *NewArg = InternalMapper.mapConstant(*cast(Arg)); + NewArg = ConstantFoldIntegerCast(NewArg, OffTy, /*IsSigned=*/true, DL); + if (!Multiple.isOne()) { + if (Multiple.isPowerOf2()) { + NewArg = ConstantExpr::getShl( + NewArg, + CE->getIntegerValue( + OffTy, APInt(BufferOffsetWidth, Multiple.logBase2())), + /*hasNUW=*/InBounds, /*HasNSW=*/InBounds); + } else { + NewArg = + ConstantExpr::getMul(NewArg, CE->getIntegerValue(OffTy, Multiple), + /*hasNUW=*/InBounds, /*hasNSW=*/InBounds); + } + } + if (OffAccum) { + OffAccum = ConstantExpr::getAdd(OffAccum, NewArg, /*hasNUW=*/InBounds, + /*hasNSW=*/InBounds); + } else { + OffAccum = NewArg; + } + } + Constant *NewConstOff = CE->getIntegerValue(OffTy, NewConstOffVal); + if (OffAccum) + OffAccum = ConstantExpr::getAdd(OffAccum, NewConstOff, + /*hasNUW=*/InBounds, /*hasNSW=*/InBounds); + else + OffAccum = NewConstOff; + bool HasNonNegativeOff = false; + if (auto *CI = dyn_cast(OffAccum)) { + HasNonNegativeOff = !CI->isNegative(); + } + Constant *NewOff = ConstantExpr::getAdd( + Off, OffAccum, /*hasNUW=*/InBounds && HasNonNegativeOff, + /*hasNSW=*/false); + return ConstantStruct::get(NewTy, {Rsrc, NewOff}); + } + + if (auto *PI = dyn_cast(CE)) { + Constant *Parts = + InternalMapper.mapConstant(*cast(PI->getPointerOperand())); + auto [Rsrc, Off] = splitLoweredFatBufferConst(Parts); + // Here, we take advantage of the fact that ptrtoint has a built-in + // zero-extension behavior. + unsigned FatPtrWidth = + DL.getPointerSizeInBits(AMDGPUAS::BUFFER_FAT_POINTER); + Constant *RsrcInt = CE->getPtrToInt(Rsrc, SrcTy); + unsigned Width = SrcTy->getScalarSizeInBits(); + Constant *Shift = + CE->getIntegerValue(SrcTy, APInt(Width, BufferOffsetWidth)); + Constant *OffCast = + ConstantFoldIntegerCast(Off, SrcTy, /*IsSigned=*/false, DL); + Constant *RsrcHi = ConstantExpr::getShl( + RsrcInt, Shift, Width >= FatPtrWidth, Width > FatPtrWidth); + // This should be an or, but those got recently removed. + Constant *Result = ConstantExpr::getAdd(RsrcHi, OffCast, true, true); + return Result; + } + + if (CE->getOpcode() == Instruction::IntToPtr) { + auto *Arg = cast(CE->getOperand(0)); + unsigned FatPtrWidth = + DL.getPointerSizeInBits(AMDGPUAS::BUFFER_FAT_POINTER); + unsigned RsrcPtrWidth = DL.getPointerSizeInBits(AMDGPUAS::BUFFER_RESOURCE); + auto *WantedTy = Arg->getType()->getWithNewBitWidth(FatPtrWidth); + Arg = ConstantFoldIntegerCast(Arg, WantedTy, /*IsSigned=*/false, DL); + + Constant *Shift = + CE->getIntegerValue(WantedTy, APInt(FatPtrWidth, BufferOffsetWidth)); + Type *RsrcIntType = WantedTy->getWithNewBitWidth(RsrcPtrWidth); + Type *RsrcTy = NewTy->getElementType(0); + Type *OffTy = WantedTy->getWithNewBitWidth(BufferOffsetWidth); + Constant *RsrcInt = CE->getTrunc( + ConstantFoldBinaryOpOperands(Instruction::LShr, Arg, Shift, DL), + RsrcIntType); + Constant *Rsrc = CE->getIntToPtr(RsrcInt, RsrcTy); + Constant *Off = ConstantFoldIntegerCast(Arg, OffTy, /*isSigned=*/false, DL); + + return ConstantStruct::get(NewTy, {Rsrc, Off}); + } + + if (auto *AC = dyn_cast(CE)) { + unsigned SrcAS = AC->getSrcAddressSpace(); + unsigned DstAS = AC->getDestAddressSpace(); + auto *Arg = cast(AC->getPointerOperand()); + auto *NewArg = InternalMapper.mapConstant(*Arg); + if (!NewArg) + return nullptr; + if (SrcAS == AMDGPUAS::BUFFER_FAT_POINTER && + DstAS == AMDGPUAS::BUFFER_FAT_POINTER) + return NewArg; + if (SrcAS == AMDGPUAS::BUFFER_RESOURCE && + DstAS == AMDGPUAS::BUFFER_FAT_POINTER) { + auto *NullOff = CE->getNullValue(NewTy->getElementType(1)); + return ConstantStruct::get(NewTy, {NewArg, NullOff}); + } + report_fatal_error( + "Unsupported address space cast for a buffer fat pointer"); + } + return nullptr; +} + +Value *FatPtrConstMaterializer::materialize(Value *V) { + Constant *C = dyn_cast(V); + if (!C) + return nullptr; + if (auto *GEPO = dyn_cast(C)) { + // As a special case, adjust GEP constants that have a ptr addrspace(7) in + // their source types here, since the earlier local changes didn't handle + // htis. + Type *SrcTy = GEPO->getSourceElementType(); + Type *NewSrcTy = IntTypeMap->remapType(SrcTy); + if (SrcTy != NewSrcTy) { + SmallVector Ops; + Ops.reserve(GEPO->getNumOperands()); + for (const Use &U : GEPO->operands()) + Ops.push_back(cast(U.get())); + auto *NewGEP = ConstantExpr::getGetElementPtr( + NewSrcTy, Ops[0], ArrayRef(Ops).slice(1), + GEPO->isInBounds(), GEPO->getInRangeIndex()); + LLVM_DEBUG(dbgs() << "p7-getting GEP: " << *GEPO << " becomes " << *NewGEP + << "\n"); + Value *FurtherMap = materialize(NewGEP); + return FurtherMap ? FurtherMap : NewGEP; + } + } + // Structs and other types that happen to contain fat pointers get remapped + // by the mapValue() logic. + if (!isBufferFatPtrConst(C)) + return nullptr; + return materializeBufferFatPtrConst(C); +} + +using PtrParts = std::pair; +namespace { +// The visitor returns the resource and offset parts for an instruction if they +// can be computed, or (nullptr, nullptr) for cases that don't have a meaningful +// value mapping. +class SplitPtrStructs : public InstVisitor { + ValueToValueMapTy RsrcParts; + ValueToValueMapTy OffParts; + + // Track instructions that have been rewritten into a user of the component + // parts of their ptr addrspace(7) input. Instructions that produced + // ptr addrspace(7) parts should **not** be RAUW'd before being added to this + // set, as that replacement will be handled in a post-visit step. However, + // instructions that yield values that aren't fat pointers (ex. ptrtoint) + // should RAUW themselves with new instructions that use the split parts + // of their arguments during processing. + DenseSet SplitUsers; + + // Nodes that need a second look once we've computed the parts for all other + // instructions to see if, for example, we really need to phi on the resource + // part. + SmallVector Conditionals; + // Temporary instructions produced while lowering conditionals that should be + // killed. + SmallVector ConditionalTemps; + + // Subtarget info, needed for determining what cache control bits to set. + const TargetMachine *TM; + const GCNSubtarget *ST; + + IRBuilder<> IRB; + + // Copy metadata between instructions if applicable. + void copyMetadata(Value *Dest, Value *Src); + + // Get the resource and offset parts of the value V, inserting appropriate + // extractvalue calls if needed. + PtrParts getPtrParts(Value *V); + + // Given an instruction that could produce multiple resource parts (a PHI or + // select), collect the set of possible instructions that could have provided + // its resource parts that it could have (the `Roots`) and the set of + // conditional instructions visited during the search (`Seen`). If, after + // removing the root of the search from `Seen` and `Roots`, `Seen` is a subset + // of `Roots` and `Roots - Seen` contains one element, the resource part of + // that element can replace the resource part of all other elements in `Seen`. + void getPossibleRsrcRoots(Instruction *I, SmallPtrSetImpl &Roots, + SmallPtrSetImpl &Seen); + void processConditionals(); + + // If an instruction hav been split into resource and offset parts, + // delete that instruction. If any of its uses have not themselves been split + // into parts (for example, an insertvalue), construct the structure + // that the type rewrites declared should be produced by the dying instruction + // and use that. + // Also, kill the temporary extractvalue operations produced by the two-stage + // lowering of PHIs and conditionals. + void killAndReplaceSplitInstructions(SmallVectorImpl &Origs); + + void setAlign(CallInst *Intr, Align A, unsigned RsrcArgIdx); + void insertPreMemOpFence(AtomicOrdering Order, SyncScope::ID SSID); + void insertPostMemOpFence(AtomicOrdering Order, SyncScope::ID SSID); + Value *handleMemoryInst(Instruction *I, Value *Arg, Value *Ptr, Type *Ty, + Align Alignment, AtomicOrdering Order, + bool IsVolatile, SyncScope::ID SSID); + +public: + SplitPtrStructs(LLVMContext &Ctx, const TargetMachine *TM) + : TM(TM), ST(nullptr), IRB(Ctx) {} + + void processFunction(Function &F); + + PtrParts visitInstruction(Instruction &I); + PtrParts visitLoadInst(LoadInst &LI); + PtrParts visitStoreInst(StoreInst &SI); + PtrParts visitAtomicRMWInst(AtomicRMWInst &AI); + PtrParts visitAtomicCmpXchgInst(AtomicCmpXchgInst &AI); + PtrParts visitGetElementPtrInst(GetElementPtrInst &GEP); + + PtrParts visitPtrToIntInst(PtrToIntInst &PI); + PtrParts visitIntToPtrInst(IntToPtrInst &IP); + PtrParts visitAddrSpaceCastInst(AddrSpaceCastInst &I); + PtrParts visitICmpInst(ICmpInst &Cmp); + PtrParts visitFreezeInst(FreezeInst &I); + + PtrParts visitExtractElementInst(ExtractElementInst &I); + PtrParts visitInsertElementInst(InsertElementInst &I); + PtrParts visitShuffleVectorInst(ShuffleVectorInst &I); + + PtrParts visitPHINode(PHINode &PHI); + PtrParts visitSelectInst(SelectInst &SI); + + PtrParts visitIntrinsicInst(IntrinsicInst &II); +}; +} // namespace + +void SplitPtrStructs::copyMetadata(Value *Dest, Value *Src) { + auto *DestI = dyn_cast(Dest); + auto *SrcI = dyn_cast(Src); + + if (!DestI || !SrcI) + return; + + DestI->copyMetadata(*SrcI); +} + +PtrParts SplitPtrStructs::getPtrParts(Value *V) { + assert(isSplitFatPtr(V->getType()) && "it's not meaningful to get the parts " + "of something that wasn't rewritten"); + auto *RsrcEntry = &RsrcParts[V]; + auto *OffEntry = &OffParts[V]; + if (*RsrcEntry && *OffEntry) + return {*RsrcEntry, *OffEntry}; + + if (auto *C = dyn_cast(V)) { + auto [Rsrc, Off] = splitLoweredFatBufferConst(C); + return {*RsrcEntry = Rsrc, *OffEntry = Off}; + } + + IRBuilder<>::InsertPointGuard Guard(IRB); + if (auto *I = dyn_cast(V)) { + LLVM_DEBUG(dbgs() << "Recursing to split parts of " << *I << "\n"); + auto [Rsrc, Off] = visit(*I); + if (Rsrc && Off) + return {*RsrcEntry = Rsrc, *OffEntry = Off}; + // We'll be creating the new values after the relevant instruction. + // This instruction generates a value and so isn't a terminator. + IRB.SetInsertPoint(*I->getInsertionPointAfterDef()); + IRB.SetCurrentDebugLocation(I->getDebugLoc()); + } else if (auto *A = dyn_cast(V)) { + IRB.SetInsertPointPastAllocas(A->getParent()); + IRB.SetCurrentDebugLocation(DebugLoc()); + } + Value *Rsrc = IRB.CreateExtractValue(V, 0, V->getName() + ".rsrc"); + Value *Off = IRB.CreateExtractValue(V, 1, V->getName() + ".off"); + return {*RsrcEntry = Rsrc, *OffEntry = Off}; +} + +/// Returns the instruction that defines the resource part of the value V. +/// Note that this is not getUnderlyingObject(), since that looks through +/// operations like ptrmask which might modify the resource part. +/// +/// We can limit ourselves to just looking through GEPs followed by looking +/// through addrspacecasts because only those two operations preserve the +/// resource part, and because operations on an `addrspace(8)` (which is the +/// legal input to this addrspacecast) would produce a different resource part. +static Value *rsrcPartRoot(Value *V) { + while (auto *GEP = dyn_cast(V)) + V = GEP->getPointerOperand(); + while (auto *ASC = dyn_cast(V)) + V = ASC->getPointerOperand(); + return V; +} + +void SplitPtrStructs::getPossibleRsrcRoots(Instruction *I, + SmallPtrSetImpl &Roots, + SmallPtrSetImpl &Seen) { + if (auto *PHI = dyn_cast(I)) { + if (!Seen.insert(I).second) + return; + for (Value *In : PHI->incoming_values()) { + In = rsrcPartRoot(In); + Roots.insert(In); + if (isa(In)) + getPossibleRsrcRoots(cast(In), Roots, Seen); + } + } else if (auto *SI = dyn_cast(I)) { + if (!Seen.insert(SI).second) + return; + Value *TrueVal = rsrcPartRoot(SI->getTrueValue()); + Value *FalseVal = rsrcPartRoot(SI->getFalseValue()); + Roots.insert(TrueVal); + Roots.insert(FalseVal); + if (isa(TrueVal)) + getPossibleRsrcRoots(cast(TrueVal), Roots, Seen); + if (isa(FalseVal)) + getPossibleRsrcRoots(cast(FalseVal), Roots, Seen); + } else { + llvm_unreachable("getPossibleRsrcParts() only works on phi and select"); + } +} + +void SplitPtrStructs::processConditionals() { + SmallDenseMap FoundRsrcs; + SmallPtrSet Roots; + SmallPtrSet Seen; + for (Instruction *I : Conditionals) { + // These have to exist by now because we've visited these nodes. + Value *Rsrc = RsrcParts[I]; + Value *Off = OffParts[I]; + assert(Rsrc && Off && "must have visited conditionals by now"); + + std::optional MaybeRsrc; + auto MaybeFoundRsrc = FoundRsrcs.find(I); + if (MaybeFoundRsrc != FoundRsrcs.end()) { + MaybeRsrc = MaybeFoundRsrc->second; + } else { + IRBuilder<>::InsertPointGuard Guard(IRB); + Roots.clear(); + Seen.clear(); + getPossibleRsrcRoots(I, Roots, Seen); + LLVM_DEBUG(dbgs() << "Processing conditional: " << *I << "\n"); +#ifndef NDEBUG + for (Value *V : Roots) + LLVM_DEBUG(dbgs() << "Root: " << *V << "\n"); + for (Value *V : Seen) + LLVM_DEBUG(dbgs() << "Seen: " << *V << "\n"); +#endif + // If we are our own possible root, then we shouldn't block our + // replacement with a valid incoming value. + Roots.erase(I); + // We don't want to block the optimization for conditionals that don't + // refer to themselves but did see themselves during the traversal. + Seen.erase(I); + + if (set_is_subset(Seen, Roots)) { + auto Diff = set_difference(Roots, Seen); + if (Diff.size() == 1) { + Value *RootVal = *Diff.begin(); + // Handle the case where previous loops already looked through + // an addrspacecast. + if (isSplitFatPtr(RootVal->getType())) + MaybeRsrc = std::get<0>(getPtrParts(RootVal)); + else + MaybeRsrc = RootVal; + } + } + } + + if (auto *PHI = dyn_cast(I)) { + Value *NewRsrc; + StructType *PHITy = cast(PHI->getType()); + IRB.SetInsertPoint(*PHI->getInsertionPointAfterDef()); + IRB.SetCurrentDebugLocation(PHI->getDebugLoc()); + if (MaybeRsrc) { + NewRsrc = *MaybeRsrc; + } else { + Type *RsrcTy = PHITy->getElementType(0); + auto *RsrcPHI = IRB.CreatePHI(RsrcTy, PHI->getNumIncomingValues()); + RsrcPHI->takeName(Rsrc); + for (auto [V, BB] : llvm::zip(PHI->incoming_values(), PHI->blocks())) { + Value *VRsrc = std::get<0>(getPtrParts(V)); + RsrcPHI->addIncoming(VRsrc, BB); + } + copyMetadata(RsrcPHI, PHI); + NewRsrc = RsrcPHI; + } + + Type *OffTy = PHITy->getElementType(1); + auto *NewOff = IRB.CreatePHI(OffTy, PHI->getNumIncomingValues()); + NewOff->takeName(Off); + for (auto [V, BB] : llvm::zip(PHI->incoming_values(), PHI->blocks())) { + assert(OffParts.count(V) && "An offset part had to be created by now"); + Value *VOff = std::get<1>(getPtrParts(V)); + NewOff->addIncoming(VOff, BB); + } + copyMetadata(NewOff, PHI); + + // Note: We don't eraseFromParent() the temporaries because we don't want + // to put the corrections maps in an inconstent state. That'll be handed + // during the rest of the killing. Also, `ValueToValueMapTy` guarantees + // that references in that map will be updated as well. + ConditionalTemps.push_back(cast(Rsrc)); + ConditionalTemps.push_back(cast(Off)); + Rsrc->replaceAllUsesWith(NewRsrc); + Off->replaceAllUsesWith(NewOff); + + // Save on recomputing the cycle traversals in known-root cases. + if (MaybeRsrc) + for (Value *V : Seen) + FoundRsrcs[cast(V)] = NewRsrc; + } else if (auto *SI = dyn_cast(I)) { + if (MaybeRsrc) { + ConditionalTemps.push_back(cast(Rsrc)); + Rsrc->replaceAllUsesWith(*MaybeRsrc); + for (Value *V : Seen) + FoundRsrcs[cast(V)] = *MaybeRsrc; + } + } else { + llvm_unreachable("Only PHIs and selects go in the conditionals list"); + } + } +} + +void SplitPtrStructs::killAndReplaceSplitInstructions( + SmallVectorImpl &Origs) { + for (Instruction *I : ConditionalTemps) + I->eraseFromParent(); + + for (Instruction *I : Origs) { + if (!SplitUsers.contains(I)) + continue; + + SmallVector Dbgs; + findDbgValues(Dbgs, I); + for (auto *Dbg : Dbgs) { + IRB.SetInsertPoint(Dbg); + auto &DL = I->getModule()->getDataLayout(); + assert(isSplitFatPtr(I->getType()) && + "We should've RAUW'd away loads, stores, etc. at this point"); + auto *OffDbg = cast(Dbg->clone()); + copyMetadata(OffDbg, Dbg); + auto [Rsrc, Off] = getPtrParts(I); + + int64_t RsrcSz = DL.getTypeSizeInBits(Rsrc->getType()); + int64_t OffSz = DL.getTypeSizeInBits(Off->getType()); + + std::optional RsrcExpr = + DIExpression::createFragmentExpression(Dbg->getExpression(), 0, + RsrcSz); + std::optional OffExpr = + DIExpression::createFragmentExpression(Dbg->getExpression(), RsrcSz, + OffSz); + if (OffExpr) { + OffDbg->setExpression(*OffExpr); + OffDbg->replaceVariableLocationOp(I, Off); + IRB.Insert(OffDbg); + } else { + OffDbg->deleteValue(); + } + if (RsrcExpr) { + Dbg->setExpression(*RsrcExpr); + Dbg->replaceVariableLocationOp(I, Rsrc); + } else { + Dbg->replaceVariableLocationOp(I, UndefValue::get(I->getType())); + } + } + + Value *Poison = PoisonValue::get(I->getType()); + I->replaceUsesWithIf(Poison, [&](const Use &U) -> bool { + if (const auto *UI = dyn_cast(U.getUser())) + return SplitUsers.contains(UI); + return false; + }); + + if (I->use_empty()) { + I->eraseFromParent(); + continue; + } + IRB.SetInsertPoint(*I->getInsertionPointAfterDef()); + IRB.SetCurrentDebugLocation(I->getDebugLoc()); + auto [Rsrc, Off] = getPtrParts(I); + Value *Struct = PoisonValue::get(I->getType()); + Struct = IRB.CreateInsertValue(Struct, Rsrc, 0); + Struct = IRB.CreateInsertValue(Struct, Off, 1); + copyMetadata(Struct, I); + Struct->takeName(I); + I->replaceAllUsesWith(Struct); + I->eraseFromParent(); + } +} + +void SplitPtrStructs::setAlign(CallInst *Intr, Align A, unsigned RsrcArgIdx) { + LLVMContext &Ctx = Intr->getContext(); + Intr->addParamAttr(RsrcArgIdx, Attribute::getWithAlignment(Ctx, A)); +} + +void SplitPtrStructs::insertPreMemOpFence(AtomicOrdering Order, + SyncScope::ID SSID) { + switch (Order) { + case AtomicOrdering::Release: + case AtomicOrdering::AcquireRelease: + case AtomicOrdering::SequentiallyConsistent: + IRB.CreateFence(AtomicOrdering::Release, SSID); + break; + default: + break; + } +} + +void SplitPtrStructs::insertPostMemOpFence(AtomicOrdering Order, + SyncScope::ID SSID) { + switch (Order) { + case AtomicOrdering::Acquire: + case AtomicOrdering::AcquireRelease: + case AtomicOrdering::SequentiallyConsistent: + IRB.CreateFence(AtomicOrdering::Acquire, SSID); + break; + default: + break; + } +} + +Value *SplitPtrStructs::handleMemoryInst(Instruction *I, Value *Arg, Value *Ptr, + Type *Ty, Align Alignment, + AtomicOrdering Order, bool IsVolatile, + SyncScope::ID SSID) { + IRB.SetInsertPoint(I); + + auto [Rsrc, Off] = getPtrParts(Ptr); + SmallVector Args; + if (Arg) + Args.push_back(Arg); + Args.push_back(Rsrc); + Args.push_back(Off); + insertPreMemOpFence(Order, SSID); + // soffset is always 0 for these cases, where we always want any offset to be + // part of bounds checking and we don't know which parts of the GEPs is + // uniform. + Args.push_back(IRB.getInt32(0)); + + uint32_t Aux = 0; + bool IsInvariant = + (isa(I) && I->getMetadata(LLVMContext::MD_invariant_load)); + bool IsNonTemporal = I->getMetadata(LLVMContext::MD_nontemporal); + // Atomic loads and stores need glc, atomic read-modify-write doesn't. + bool IsOneWayAtomic = + !isa(I) && Order != AtomicOrdering::NotAtomic; + if (IsOneWayAtomic) + Aux |= AMDGPU::CPol::GLC; + if (IsNonTemporal && !IsInvariant) + Aux |= AMDGPU::CPol::SLC; + if (isa(I) && ST->getGeneration() == AMDGPUSubtarget::GFX10) + Aux |= (Aux & AMDGPU::CPol::GLC ? AMDGPU::CPol::DLC : 0); + if (IsVolatile) + Aux |= AMDGPU::CPol::VOLATILE; + Args.push_back(IRB.getInt32(Aux)); + + Intrinsic::ID IID = Intrinsic::not_intrinsic; + if (isa(I)) + // TODO: Do we need to do something about atomic loads? + IID = Intrinsic::amdgcn_raw_ptr_buffer_load; + else if (isa(I)) + IID = Intrinsic::amdgcn_raw_ptr_buffer_store; + else if (auto *RMW = dyn_cast(I)) { + switch (RMW->getOperation()) { + case AtomicRMWInst::Xchg: + IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap; + break; + case AtomicRMWInst::Add: + IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_add; + break; + case AtomicRMWInst::Sub: + IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub; + break; + case AtomicRMWInst::And: + IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_and; + break; + case AtomicRMWInst::Or: + IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_or; + break; + case AtomicRMWInst::Xor: + IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor; + break; + case AtomicRMWInst::Max: + IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax; + break; + case AtomicRMWInst::Min: + IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin; + break; + case AtomicRMWInst::UMax: + IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax; + break; + case AtomicRMWInst::UMin: + IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin; + break; + case AtomicRMWInst::FAdd: + IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd; + break; + case AtomicRMWInst::FMax: + IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax; + break; + case AtomicRMWInst::FMin: + IID = Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin; + break; + case AtomicRMWInst::FSub: { + report_fatal_error("atomic floating point subtraction not supported for " + "buffer resources and should've been expanded away"); + break; + } + case AtomicRMWInst::Nand: + report_fatal_error("atomic nand not supported for buffer resources and " + "should've been expanded away"); + break; + case AtomicRMWInst::UIncWrap: + case AtomicRMWInst::UDecWrap: + report_fatal_error("wrapping increment/decrement not supported for " + "buffer resources and should've ben expanded away"); + break; + case AtomicRMWInst::BAD_BINOP: + llvm_unreachable("Not sure how we got a bad binop"); + } + } + + auto *Call = IRB.CreateIntrinsic(IID, Ty, Args); + copyMetadata(Call, I); + setAlign(Call, Alignment, Arg ? 1 : 0); + Call->takeName(I); + + insertPostMemOpFence(Order, SSID); + // The "no moving p7 directly" rewrites ensure that this load or store won't + // itself need to be split into parts. + SplitUsers.insert(I); + I->replaceAllUsesWith(Call); + return Call; +} + +PtrParts SplitPtrStructs::visitInstruction(Instruction &I) { + return {nullptr, nullptr}; +} + +PtrParts SplitPtrStructs::visitLoadInst(LoadInst &LI) { + if (!isSplitFatPtr(LI.getPointerOperandType())) + return {nullptr, nullptr}; + handleMemoryInst(&LI, nullptr, LI.getPointerOperand(), LI.getType(), + LI.getAlign(), LI.getOrdering(), LI.isVolatile(), + LI.getSyncScopeID()); + return {nullptr, nullptr}; +} + +PtrParts SplitPtrStructs::visitStoreInst(StoreInst &SI) { + if (!isSplitFatPtr(SI.getPointerOperandType())) + return {nullptr, nullptr}; + Value *Arg = SI.getValueOperand(); + handleMemoryInst(&SI, Arg, SI.getPointerOperand(), Arg->getType(), + SI.getAlign(), SI.getOrdering(), SI.isVolatile(), + SI.getSyncScopeID()); + return {nullptr, nullptr}; +} + +PtrParts SplitPtrStructs::visitAtomicRMWInst(AtomicRMWInst &AI) { + if (!isSplitFatPtr(AI.getPointerOperand()->getType())) + return {nullptr, nullptr}; + Value *Arg = AI.getValOperand(); + handleMemoryInst(&AI, Arg, AI.getPointerOperand(), Arg->getType(), + AI.getAlign(), AI.getOrdering(), AI.isVolatile(), + AI.getSyncScopeID()); + return {nullptr, nullptr}; +} + +// Unlike load, store, and RMW, cmpxchg needs special handling to account +// for the boolean argument. +PtrParts SplitPtrStructs::visitAtomicCmpXchgInst(AtomicCmpXchgInst &AI) { + Value *Ptr = AI.getPointerOperand(); + if (!isSplitFatPtr(Ptr->getType())) + return {nullptr, nullptr}; + IRB.SetInsertPoint(&AI); + + Type *Ty = AI.getNewValOperand()->getType(); + AtomicOrdering Order = AI.getMergedOrdering(); + SyncScope::ID SSID = AI.getSyncScopeID(); + bool IsNonTemporal = AI.getMetadata(LLVMContext::MD_nontemporal); + + auto [Rsrc, Off] = getPtrParts(Ptr); + insertPreMemOpFence(Order, SSID); + + uint32_t Aux = 0; + if (IsNonTemporal) + Aux |= AMDGPU::CPol::SLC; + if (AI.isVolatile()) + Aux |= AMDGPU::CPol::VOLATILE; + auto *Call = + IRB.CreateIntrinsic(Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap, Ty, + {AI.getNewValOperand(), AI.getCompareOperand(), Rsrc, + Off, IRB.getInt32(0), IRB.getInt32(Aux)}); + copyMetadata(Call, &AI); + setAlign(Call, AI.getAlign(), 2); + Call->takeName(&AI); + insertPostMemOpFence(Order, SSID); + + Value *Res = PoisonValue::get(AI.getType()); + Res = IRB.CreateInsertValue(Res, Call, 0); + if (!AI.isWeak()) { + Value *Succeeded = IRB.CreateICmpEQ(Call, AI.getCompareOperand()); + Res = IRB.CreateInsertValue(Res, Succeeded, 1); + } + SplitUsers.insert(&AI); + AI.replaceAllUsesWith(Res); + return {nullptr, nullptr}; +} + +PtrParts SplitPtrStructs::visitGetElementPtrInst(GetElementPtrInst &GEP) { + Value *Ptr = GEP.getPointerOperand(); + if (!isSplitFatPtr(Ptr->getType())) + return {nullptr, nullptr}; + IRB.SetInsertPoint(&GEP); + + auto [Rsrc, Off] = getPtrParts(Ptr); + Type *OffTy = Off->getType(); + const DataLayout &DL = GEP.getModule()->getDataLayout(); + bool InBounds = GEP.isInBounds(); + + // In order to call collectOffset() and thus not have to reimplement it, + // we need the GEP's pointer operand to have ptr addrspace(7) type + GEP.setOperand(GEP.getPointerOperandIndex(), + PoisonValue::get(IRB.getPtrTy(AMDGPUAS::BUFFER_FAT_POINTER))); + MapVector VariableOffs; + APInt ConstOffVal = APInt::getZero(BufferOffsetWidth); + if (!GEP.collectOffset(DL, BufferOffsetWidth, VariableOffs, ConstOffVal)) + report_fatal_error("Scalable vector or unsized struct in fat pointer GEP"); + GEP.setOperand(GEP.getPointerOperandIndex(), Ptr); + Value *OffAccum = nullptr; + // Accumulate offsets together before adding to the base in order to preserve + // as many of the inbounds properties as possible. + for (auto [Arg, Multiple] : VariableOffs) { + if (auto *OffVecTy = dyn_cast(OffTy)) + if (!Arg->getType()->isVectorTy()) + Arg = IRB.CreateVectorSplat(OffVecTy->getElementCount(), Arg); + Arg = IRB.CreateIntCast(Arg, OffTy, /*isSigned=*/true); + if (!Multiple.isOne()) { + if (Multiple.isPowerOf2()) + Arg = IRB.CreateShl(Arg, Multiple.logBase2(), "", /*hasNUW=*/InBounds, + /*HasNSW=*/InBounds); + else + Arg = IRB.CreateMul(Arg, ConstantExpr::getIntegerValue(OffTy, Multiple), + "", /*hasNUW=*/InBounds, /*hasNSW=*/InBounds); + } + if (OffAccum) + OffAccum = IRB.CreateAdd(OffAccum, Arg, "", /*hasNUW=*/InBounds, + /*hasNSW=*/InBounds); + else + OffAccum = Arg; + } + if (!ConstOffVal.isZero()) { + Constant *ConstOff = ConstantExpr::getIntegerValue(OffTy, ConstOffVal); + if (OffAccum) + OffAccum = IRB.CreateAdd(OffAccum, ConstOff, "", /*hasNUW=*/InBounds, + /*hasNSW=*/InBounds); + else + OffAccum = ConstOff; + } + + if (!OffAccum) { // Constant-zero offset + SplitUsers.insert(&GEP); + return {Rsrc, Off}; + } + + bool HasNonNegativeOff = false; + if (auto *CI = dyn_cast(OffAccum)) { + HasNonNegativeOff = !CI->isNegative(); + } + Value *NewOff; + if (PatternMatch::match(Off, PatternMatch::is_zero())) { + NewOff = OffAccum; + } else { + NewOff = IRB.CreateAdd(Off, OffAccum, "", + /*hasNUW=*/InBounds && HasNonNegativeOff, + /*hasNSW=*/false); + } + copyMetadata(NewOff, &GEP); + NewOff->takeName(&GEP); + SplitUsers.insert(&GEP); + return {Rsrc, NewOff}; +} + +PtrParts SplitPtrStructs::visitPtrToIntInst(PtrToIntInst &PI) { + Value *Ptr = PI.getPointerOperand(); + if (!isSplitFatPtr(Ptr->getType())) + return {nullptr, nullptr}; + IRB.SetInsertPoint(&PI); + + Type *ResTy = PI.getType(); + unsigned Width = ResTy->getScalarSizeInBits(); + + auto [Rsrc, Off] = getPtrParts(Ptr); + const DataLayout &DL = PI.getModule()->getDataLayout(); + unsigned FatPtrWidth = DL.getPointerSizeInBits(AMDGPUAS::BUFFER_FAT_POINTER); + + Value *RsrcInt; + if (Width <= BufferOffsetWidth) + RsrcInt = ConstantExpr::getIntegerValue(ResTy, APInt::getZero(Width)); + else + RsrcInt = IRB.CreatePtrToInt(Rsrc, ResTy, PI.getName() + ".rsrc"); + copyMetadata(RsrcInt, &PI); + + Value *Shl = IRB.CreateShl( + RsrcInt, + ConstantExpr::getIntegerValue(ResTy, APInt(Width, BufferOffsetWidth)), "", + Width >= FatPtrWidth, Width > FatPtrWidth); + Value *OffCast = + IRB.CreateIntCast(Off, ResTy, /*isSigned=*/false, PI.getName() + ".off"); + Value *Res = IRB.CreateOr(Shl, OffCast); + Res->takeName(&PI); + SplitUsers.insert(&PI); + PI.replaceAllUsesWith(Res); + return {nullptr, nullptr}; +} + +PtrParts SplitPtrStructs::visitIntToPtrInst(IntToPtrInst &IP) { + if (!isSplitFatPtr(IP.getType())) + return {nullptr, nullptr}; + IRB.SetInsertPoint(&IP); + const DataLayout &DL = IP.getModule()->getDataLayout(); + unsigned RsrcPtrWidth = DL.getPointerSizeInBits(AMDGPUAS::BUFFER_RESOURCE); + Value *Int = IP.getOperand(0); + Type *IntTy = Int->getType(); + Type *RsrcIntTy = IntTy->getWithNewBitWidth(RsrcPtrWidth); + unsigned Width = IntTy->getScalarSizeInBits(); + + auto *RetTy = cast(IP.getType()); + Type *RsrcTy = RetTy->getElementType(0); + Type *OffTy = RetTy->getElementType(1); + Value *RsrcPart = IRB.CreateLShr( + Int, + ConstantExpr::getIntegerValue(IntTy, APInt(Width, BufferOffsetWidth))); + Value *RsrcInt = IRB.CreateIntCast(RsrcPart, RsrcIntTy, /*isSigned=*/false); + Value *Rsrc = IRB.CreateIntToPtr(RsrcInt, RsrcTy, IP.getName() + ".rsrc"); + Value *Off = + IRB.CreateIntCast(Int, OffTy, /*IsSigned=*/false, IP.getName() + ".off"); + + copyMetadata(Rsrc, &IP); + SplitUsers.insert(&IP); + return {Rsrc, Off}; +} + +PtrParts SplitPtrStructs::visitAddrSpaceCastInst(AddrSpaceCastInst &I) { + if (!isSplitFatPtr(I.getType())) + return {nullptr, nullptr}; + IRB.SetInsertPoint(&I); + Value *In = I.getPointerOperand(); + // No-op casts preserve parts + if (In->getType() == I.getType()) { + auto [Rsrc, Off] = getPtrParts(In); + SplitUsers.insert(&I); + return {Rsrc, Off}; + } + if (I.getSrcAddressSpace() != AMDGPUAS::BUFFER_RESOURCE) + report_fatal_error("Only buffer resources (addrspace 8) can be cast to " + "buffer fat pointers (addrspace 7)"); + Type *OffTy = cast(I.getType())->getElementType(1); + Value *ZeroOff = Constant::getNullValue(OffTy); + SplitUsers.insert(&I); + return {In, ZeroOff}; +} + +PtrParts SplitPtrStructs::visitICmpInst(ICmpInst &Cmp) { + Value *Lhs = Cmp.getOperand(0); + if (!isSplitFatPtr(Lhs->getType())) + return {nullptr, nullptr}; + Value *Rhs = Cmp.getOperand(1); + IRB.SetInsertPoint(&Cmp); + ICmpInst::Predicate Pred = Cmp.getPredicate(); + + assert((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) && + "Pointer comparison is only equal or unequal"); + auto [LhsRsrc, LhsOff] = getPtrParts(Lhs); + auto [RhsRsrc, RhsOff] = getPtrParts(Rhs); + Value *RsrcCmp = + IRB.CreateICmp(Pred, LhsRsrc, RhsRsrc, Cmp.getName() + ".rsrc"); + copyMetadata(RsrcCmp, &Cmp); + Value *OffCmp = IRB.CreateICmp(Pred, LhsOff, RhsOff, Cmp.getName() + ".off"); + copyMetadata(OffCmp, &Cmp); + + Value *Res = nullptr; + if (Pred == ICmpInst::ICMP_EQ) + Res = IRB.CreateAnd(RsrcCmp, OffCmp); + else if (Pred == ICmpInst::ICMP_NE) + Res = IRB.CreateOr(RsrcCmp, OffCmp); + copyMetadata(Res, &Cmp); + Res->takeName(&Cmp); + SplitUsers.insert(&Cmp); + Cmp.replaceAllUsesWith(Res); + return {nullptr, nullptr}; +} + +PtrParts SplitPtrStructs::visitFreezeInst(FreezeInst &I) { + if (!isSplitFatPtr(I.getType())) + return {nullptr, nullptr}; + IRB.SetInsertPoint(&I); + auto [Rsrc, Off] = getPtrParts(I.getOperand(0)); + + Value *RsrcRes = IRB.CreateFreeze(Rsrc, I.getName() + ".rsrc"); + copyMetadata(RsrcRes, &I); + Value *OffRes = IRB.CreateFreeze(Off, I.getName() + ".off"); + copyMetadata(OffRes, &I); + SplitUsers.insert(&I); + return {RsrcRes, OffRes}; +} + +PtrParts SplitPtrStructs::visitExtractElementInst(ExtractElementInst &I) { + if (!isSplitFatPtr(I.getType())) + return {nullptr, nullptr}; + IRB.SetInsertPoint(&I); + Value *Vec = I.getVectorOperand(); + Value *Idx = I.getIndexOperand(); + auto [Rsrc, Off] = getPtrParts(Vec); + + Value *RsrcRes = IRB.CreateExtractElement(Rsrc, Idx, I.getName() + ".rsrc"); + copyMetadata(RsrcRes, &I); + Value *OffRes = IRB.CreateExtractElement(Off, Idx, I.getName() + ".off"); + copyMetadata(OffRes, &I); + SplitUsers.insert(&I); + return {RsrcRes, OffRes}; +} + +PtrParts SplitPtrStructs::visitInsertElementInst(InsertElementInst &I) { + // The mutated instructions temporarily don't return vectors, and so + // we need the generic getType() here to avoid crashes. + if (!isSplitFatPtr(cast(I).getType())) + return {nullptr, nullptr}; + IRB.SetInsertPoint(&I); + Value *Vec = I.getOperand(0); + Value *Elem = I.getOperand(1); + Value *Idx = I.getOperand(2); + auto [VecRsrc, VecOff] = getPtrParts(Vec); + auto [ElemRsrc, ElemOff] = getPtrParts(Elem); + + Value *RsrcRes = + IRB.CreateInsertElement(VecRsrc, ElemRsrc, Idx, I.getName() + ".rsrc"); + copyMetadata(RsrcRes, &I); + Value *OffRes = + IRB.CreateInsertElement(VecOff, ElemOff, Idx, I.getName() + ".off"); + copyMetadata(OffRes, &I); + SplitUsers.insert(&I); + return {RsrcRes, OffRes}; +} + +PtrParts SplitPtrStructs::visitShuffleVectorInst(ShuffleVectorInst &I) { + // Cast is needed for the same reason as insertelement's. + if (!isSplitFatPtr(cast(I).getType())) + return {nullptr, nullptr}; + IRB.SetInsertPoint(&I); + + Value *V1 = I.getOperand(0); + Value *V2 = I.getOperand(1); + ArrayRef Mask = I.getShuffleMask(); + auto [V1Rsrc, V1Off] = getPtrParts(V1); + auto [V2Rsrc, V2Off] = getPtrParts(V2); + + Value *RsrcRes = + IRB.CreateShuffleVector(V1Rsrc, V2Rsrc, Mask, I.getName() + ".rsrc"); + copyMetadata(RsrcRes, &I); + Value *OffRes = + IRB.CreateShuffleVector(V1Off, V2Off, Mask, I.getName() + ".off"); + copyMetadata(OffRes, &I); + SplitUsers.insert(&I); + return {RsrcRes, OffRes}; +} + +PtrParts SplitPtrStructs::visitPHINode(PHINode &PHI) { + if (!isSplitFatPtr(PHI.getType())) + return {nullptr, nullptr}; + IRB.SetInsertPoint(*PHI.getInsertionPointAfterDef()); + // Phi nodes will be handled in post-processing after we've visited every + // instruction. However, instead of just returning {nullptr, nullptr}, + // we explicitly create the temporary extractvalue operations that are our + // temporary results so that they end up at the beginning of the block with + // the PHIs. + Value *TmpRsrc = IRB.CreateExtractValue(&PHI, 0, PHI.getName() + ".rsrc"); + Value *TmpOff = IRB.CreateExtractValue(&PHI, 1, PHI.getName() + ".off"); + Conditionals.push_back(&PHI); + SplitUsers.insert(&PHI); + return {TmpRsrc, TmpOff}; +} + +PtrParts SplitPtrStructs::visitSelectInst(SelectInst &SI) { + if (!isSplitFatPtr(SI.getType())) + return {nullptr, nullptr}; + IRB.SetInsertPoint(&SI); + + Value *Cond = SI.getCondition(); + Value *True = SI.getTrueValue(); + Value *False = SI.getFalseValue(); + auto [TrueRsrc, TrueOff] = getPtrParts(True); + auto [FalseRsrc, FalseOff] = getPtrParts(False); + + Value *RsrcRes = RsrcRes = + IRB.CreateSelect(Cond, TrueRsrc, FalseRsrc, SI.getName() + ".rsrc", &SI); + copyMetadata(RsrcRes, &SI); + Conditionals.push_back(&SI); + Value *OffRes = + IRB.CreateSelect(Cond, TrueOff, FalseOff, SI.getName() + ".off", &SI); + copyMetadata(OffRes, &SI); + SplitUsers.insert(&SI); + return {RsrcRes, OffRes}; +} + +/// Returns true if this intrinsic needs to be removed when it is +/// applied to `ptr addrspace(7)` values. Calls to these intrinsics are +/// rewritten into calls to versions of that intrinsic on the resource +/// descriptor. +static bool isRemovablePointerIntrinsic(Intrinsic::ID IID) { + switch (IID) { + default: + return false; + case Intrinsic::ptrmask: + case Intrinsic::invariant_start: + case Intrinsic::invariant_end: + case Intrinsic::launder_invariant_group: + case Intrinsic::strip_invariant_group: + return true; + } +} + +PtrParts SplitPtrStructs::visitIntrinsicInst(IntrinsicInst &I) { + Intrinsic::ID IID = I.getIntrinsicID(); + switch (IID) { + default: + break; + case Intrinsic::ptrmask: { + Value *Ptr = I.getArgOperand(0); + if (!isSplitFatPtr(Ptr->getType())) + return {nullptr, nullptr}; + Value *Mask = I.getArgOperand(1); + IRB.SetInsertPoint(&I); + auto [Rsrc, Off] = getPtrParts(Ptr); + if (Mask->getType() != Off->getType()) + report_fatal_error("offset width is not equal to index width of fat " + "pointer (data layout not set up correctly?)"); + Value *OffRes = IRB.CreateAnd(Off, Mask, I.getName() + ".off"); + copyMetadata(OffRes, &I); + SplitUsers.insert(&I); + return {Rsrc, OffRes}; + } + // Pointer annotation intrinsics that, given their object-wide nature + // operate on the resource part. + case Intrinsic::invariant_start: { + Value *Ptr = I.getArgOperand(1); + if (!isSplitFatPtr(Ptr->getType())) + return {nullptr, nullptr}; + IRB.SetInsertPoint(&I); + auto [Rsrc, Off] = getPtrParts(Ptr); + Type *NewTy = PointerType::get(I.getContext(), AMDGPUAS::BUFFER_RESOURCE); + auto *NewRsrc = IRB.CreateIntrinsic(IID, {NewTy}, {I.getOperand(0), Rsrc}); + copyMetadata(NewRsrc, &I); + NewRsrc->takeName(&I); + SplitUsers.insert(&I); + I.replaceAllUsesWith(NewRsrc); + return {nullptr, nullptr}; + } + case Intrinsic::invariant_end: { + Value *RealPtr = I.getArgOperand(2); + if (!isSplitFatPtr(RealPtr->getType())) + return {nullptr, nullptr}; + IRB.SetInsertPoint(&I); + Value *RealRsrc = getPtrParts(RealPtr).first; + Value *InvPtr = I.getArgOperand(0); + Value *Size = I.getArgOperand(1); + Value *NewRsrc = IRB.CreateIntrinsic(IID, {RealRsrc->getType()}, + {InvPtr, Size, RealRsrc}); + copyMetadata(NewRsrc, &I); + NewRsrc->takeName(&I); + SplitUsers.insert(&I); + I.replaceAllUsesWith(NewRsrc); + return {nullptr, nullptr}; + } + case Intrinsic::launder_invariant_group: + case Intrinsic::strip_invariant_group: { + Value *Ptr = I.getArgOperand(0); + if (!isSplitFatPtr(Ptr->getType())) + return {nullptr, nullptr}; + IRB.SetInsertPoint(&I); + auto [Rsrc, Off] = getPtrParts(Ptr); + Value *NewRsrc = IRB.CreateIntrinsic(IID, {Rsrc->getType()}, {Rsrc}); + copyMetadata(NewRsrc, &I); + NewRsrc->takeName(&I); + SplitUsers.insert(&I); + return {NewRsrc, Off}; + } + } + return {nullptr, nullptr}; +} + +void SplitPtrStructs::processFunction(Function &F) { + ST = &TM->getSubtarget(F); + SmallVector Originals; + LLVM_DEBUG(dbgs() << "Splitting pointer structs in function: " << F.getName() + << "\n"); + for (Instruction &I : instructions(F)) + Originals.push_back(&I); + for (Instruction *I : Originals) { + auto [Rsrc, Off] = visit(I); + assert((Rsrc && Off) || + (!Rsrc && !Off) && "Can't have a resource but no offset"); + if (Rsrc) + RsrcParts[I] = Rsrc; + if (Off) + OffParts[I] = Off; + } + processConditionals(); + killAndReplaceSplitInstructions(Originals); + + // Clean up after ourselves to save on memory. + RsrcParts.clear(); + OffParts.clear(); + SplitUsers.clear(); + Conditionals.clear(); + ConditionalTemps.clear(); +} + +namespace { +class AMDGPULowerBufferFatPointers : public ModulePass { +public: + static char ID; + + AMDGPULowerBufferFatPointers() : ModulePass(ID) { + initializeAMDGPULowerBufferFatPointersPass( + *PassRegistry::getPassRegistry()); + } + + bool run(Module &M, const TargetMachine &TM); + bool runOnModule(Module &M) override; + + void getAnalysisUsage(AnalysisUsage &AU) const override; +}; +} // namespace + +/// Returns true if there are values that have a buffer fat pointer in them, +/// which means we'll need to perform rewrites on this function. As a side +/// effect, this will populate the type remapping cache. +static bool containsBufferFatPointers(const Function &F, + BufferFatPtrToStructTypeMap *TypeMap) { + bool HasFatPointers = false; + for (const BasicBlock &BB : F) { + for (const Instruction &I : BB) { + HasFatPointers |= (I.getType() != TypeMap->remapType(I.getType())); + for (const Use &U : I.operands()) + if (auto *C = dyn_cast(U.get())) + HasFatPointers |= isBufferFatPtrConst(C); + } + } + return HasFatPointers; +} + +static bool hasFatPointerInterface(const Function &F, + BufferFatPtrToStructTypeMap *TypeMap) { + Type *Ty = F.getFunctionType(); + return Ty != TypeMap->remapType(Ty); +} + +/// Move the body of `OldF` into a new function, returning it. +static Function *moveFunctionAdaptingType(Function *OldF, FunctionType *NewTy, + ValueToValueMapTy &CloneMap) { + bool IsIntrinsic = OldF->isIntrinsic(); + Function *NewF = + Function::Create(NewTy, OldF->getLinkage(), OldF->getAddressSpace()); + NewF->copyAttributesFrom(OldF); + NewF->copyMetadata(OldF, 0); + NewF->takeName(OldF); + NewF->updateAfterNameChange(); + NewF->setDLLStorageClass(OldF->getDLLStorageClass()); + OldF->getParent()->getFunctionList().insertAfter(OldF->getIterator(), NewF); + + while (!OldF->empty()) { + BasicBlock *BB = &OldF->front(); + BB->removeFromParent(); + BB->insertInto(NewF); + CloneMap[BB] = BB; + for (Instruction &I : *BB) { + CloneMap[&I] = &I; + } + } + + AttributeMask PtrOnlyAttrs; + for (auto K : + {Attribute::Dereferenceable, Attribute::DereferenceableOrNull, + Attribute::NoAlias, Attribute::NoCapture, Attribute::NoFree, + Attribute::NonNull, Attribute::NullPointerIsValid, Attribute::ReadNone, + Attribute::ReadOnly, Attribute::WriteOnly}) { + PtrOnlyAttrs.addAttribute(K); + } + SmallVector ArgAttrs; + AttributeList OldAttrs = OldF->getAttributes(); + + for (auto [I, OldArg, NewArg] : enumerate(OldF->args(), NewF->args())) { + CloneMap[&NewArg] = &OldArg; + NewArg.takeName(&OldArg); + Type *OldArgTy = OldArg.getType(), *NewArgTy = NewArg.getType(); + // Temporarily mutate type of `NewArg` to allow RAUW to work. + NewArg.mutateType(OldArgTy); + OldArg.replaceAllUsesWith(&NewArg); + NewArg.mutateType(NewArgTy); + + AttributeSet ArgAttr = OldAttrs.getParamAttrs(I); + // Intrinsics get their attributes fixed later. + if (OldArgTy != NewArgTy && !IsIntrinsic) + ArgAttr = ArgAttr.removeAttributes(NewF->getContext(), PtrOnlyAttrs); + ArgAttrs.push_back(ArgAttr); + } + AttributeSet RetAttrs = OldAttrs.getRetAttrs(); + if (OldF->getReturnType() != NewF->getReturnType() && !IsIntrinsic) + RetAttrs = RetAttrs.removeAttributes(NewF->getContext(), PtrOnlyAttrs); + NewF->setAttributes(AttributeList::get( + NewF->getContext(), OldAttrs.getFnAttrs(), RetAttrs, ArgAttrs)); + return NewF; +} + +static void makeCloneInPraceMap(Function *F, ValueToValueMapTy &CloneMap) { + for (Argument &A : F->args()) + CloneMap[&A] = &A; + for (BasicBlock &BB : *F) { + CloneMap[&BB] = &BB; + for (Instruction &I : BB) + CloneMap[&I] = &I; + } +} + +bool AMDGPULowerBufferFatPointers::run(Module &M, const TargetMachine &TM) { + bool Changed = false; + const DataLayout &DL = M.getDataLayout(); + // Record the functions which need to be remapped. + // The second element of the pair indicates whether the function has to have + // its arguments or return types adjusted. + SmallVector> NeedsRemap; + + BufferFatPtrToStructTypeMap StructTM(DL); + BufferFatPtrToIntTypeMap IntTM(DL); + for (const GlobalVariable &GV : M.globals()) { + if (GV.getAddressSpace() == AMDGPUAS::BUFFER_FAT_POINTER) + report_fatal_error("Global variables with a buffer fat pointer address " + "space (7) are not supported"); + Type *VT = GV.getValueType(); + if (VT != StructTM.remapType(VT)) + report_fatal_error("Global variables that contain buffer fat pointers " + "(address space 7 pointers) are unsupported. Use " + "buffer resource pointers (address space 8) instead."); + } + + StoreFatPtrsAsIntsVisitor MemOpsRewrite(&IntTM, M.getContext()); + for (Function &F : M.functions()) { + bool InterfaceChange = hasFatPointerInterface(F, &StructTM); + bool BodyChanges = containsBufferFatPointers(F, &StructTM); + Changed |= MemOpsRewrite.processFunction(F); + if (InterfaceChange || BodyChanges) + NeedsRemap.push_back(std::make_pair(&F, InterfaceChange)); + } + if (NeedsRemap.empty()) + return Changed; + + SmallVector NeedsPostProcess; + SmallVector Intrinsics; + // Keep one big map so as to memoize constants across functions. + ValueToValueMapTy CloneMap; + FatPtrConstMaterializer Materializer(&StructTM, CloneMap, &IntTM, DL); + + ValueMapper LowerInFuncs(CloneMap, RF_None, &StructTM, &Materializer); + for (auto [F, InterfaceChange] : NeedsRemap) { + Function *NewF = F; + if (InterfaceChange) + NewF = moveFunctionAdaptingType( + F, cast(StructTM.remapType(F->getFunctionType())), + CloneMap); + else + makeCloneInPraceMap(F, CloneMap); + LowerInFuncs.remapFunction(*NewF); + if (NewF->isIntrinsic()) + Intrinsics.push_back(NewF); + else + NeedsPostProcess.push_back(NewF); + if (InterfaceChange) { + F->replaceAllUsesWith(NewF); + F->eraseFromParent(); + } + Changed = true; + } + StructTM.clear(); + IntTM.clear(); + CloneMap.clear(); + + SplitPtrStructs Splitter(M.getContext(), &TM); + for (Function *F : NeedsPostProcess) + Splitter.processFunction(*F); + for (Function *F : Intrinsics) { + if (isRemovablePointerIntrinsic(F->getIntrinsicID())) { + F->eraseFromParent(); + } else { + std::optional NewF = Intrinsic::remangleIntrinsicFunction(F); + if (NewF) + F->replaceAllUsesWith(*NewF); + } + } + return Changed; +} + +bool AMDGPULowerBufferFatPointers::runOnModule(Module &M) { + TargetPassConfig &TPC = getAnalysis(); + const TargetMachine &TM = TPC.getTM(); + return run(M, TM); +} + +char AMDGPULowerBufferFatPointers::ID = 0; + +char &llvm::AMDGPULowerBufferFatPointersID = AMDGPULowerBufferFatPointers::ID; + +void AMDGPULowerBufferFatPointers::getAnalysisUsage(AnalysisUsage &AU) const { + AU.addRequired(); +} + +#define PASS_DESC "Lower buffer fat pointer operations to buffer resources" +INITIALIZE_PASS_BEGIN(AMDGPULowerBufferFatPointers, DEBUG_TYPE, PASS_DESC, + false, false) +INITIALIZE_PASS_DEPENDENCY(TargetPassConfig) +INITIALIZE_PASS_END(AMDGPULowerBufferFatPointers, DEBUG_TYPE, PASS_DESC, false, + false) +#undef PASS_DESC + +ModulePass *llvm::createAMDGPULowerBufferFatPointersPass() { + return new AMDGPULowerBufferFatPointers(); +} + +PreservedAnalyses +AMDGPULowerBufferFatPointersPass::run(Module &M, ModuleAnalysisManager &MA) { + return AMDGPULowerBufferFatPointers().run(M, TM) ? PreservedAnalyses::none() + : PreservedAnalyses::all(); +} diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index 7467c25a84d6..2b457fe519d9 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -34,6 +34,7 @@ #include "TargetInfo/AMDGPUTargetInfo.h" #include "Utils/AMDGPUBaseInfo.h" #include "llvm/Analysis/CGSCCPassManager.h" +#include "llvm/Analysis/CallGraphSCCPass.h" #include "llvm/CodeGen/GlobalISel/CSEInfo.h" #include "llvm/CodeGen/GlobalISel/IRTranslator.h" #include "llvm/CodeGen/GlobalISel/InstructionSelect.h" @@ -420,6 +421,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() { initializeAMDGPULateCodeGenPreparePass(*PR); initializeAMDGPURemoveIncompatibleFunctionsPass(*PR); initializeAMDGPULowerModuleLDSLegacyPass(*PR); + initializeAMDGPULowerBufferFatPointersPass(*PR); initializeAMDGPURewriteOutArgumentsPass(*PR); initializeAMDGPURewriteUndefForPHILegacyPass(*PR); initializeAMDGPUUnifyMetadataPass(*PR); @@ -654,6 +656,10 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks( PM.addPass(AMDGPULowerModuleLDSPass(*this)); return true; } + if (PassName == "amdgpu-lower-buffer-fat-pointers") { + PM.addPass(AMDGPULowerBufferFatPointersPass(*this)); + return true; + } if (PassName == "amdgpu-lower-ctor-dtor") { PM.addPass(AMDGPUCtorDtorLoweringPass()); return true; @@ -1121,6 +1127,29 @@ void AMDGPUPassConfig::addCodeGenPrepare() { EnableLowerKernelArguments) addPass(createAMDGPULowerKernelArgumentsPass()); + if (TM->getTargetTriple().getArch() == Triple::amdgcn) { + // This lowering has been placed after codegenprepare to take advantage of + // address mode matching (which is why it isn't put with the LDS lowerings). + // It could be placed anywhere before uniformity annotations (an analysis + // that it changes by splitting up fat pointers into their components) + // but has been put before switch lowering and CFG flattening so that those + // passes can run on the more optimized control flow this pass creates in + // many cases. + // + // FIXME: This should ideally be put after the LoadStoreVectorizer. + // However, due to some annoying facts about ResourceUsageAnalysis, + // (especially as exercised in the resource-usage-dead-function test), + // we need all the function passes codegenprepare all the way through + // said resource usage analysis to run on the call graph produced + // before codegenprepare runs (because codegenprepare will knock some + // nodes out of the graph, which leads to function-level passes not + // being run on them, which causes crashes in the resource usage analysis). + addPass(createAMDGPULowerBufferFatPointersPass()); + // In accordance with the above FIXME, manually force all the + // function-level passes into a CGSCCPassManager. + addPass(new DummyCGSCCPass()); + } + TargetPassConfig::addCodeGenPrepare(); if (isPassEnabled(EnableLoadStoreVectorizer)) diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt index 9a974eaf50d2..48325a0928f9 100644 --- a/llvm/lib/Target/AMDGPU/CMakeLists.txt +++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt @@ -69,6 +69,7 @@ add_llvm_target(AMDGPUCodeGen AMDGPULibCalls.cpp AMDGPUImageIntrinsicOptimizer.cpp AMDGPULibFunc.cpp + AMDGPULowerBufferFatPointers.cpp AMDGPULowerKernelArguments.cpp AMDGPULowerKernelAttributes.cpp AMDGPULowerModuleLDSPass.cpp diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index bbb5f8d912a5..e105db313548 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -16013,7 +16013,8 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const { if (!Ty->isFloatTy() && (!Subtarget->hasGFX90AInsts() || !Ty->isDoubleTy())) return AtomicExpansionKind::CmpXChg; - if (AMDGPU::isFlatGlobalAddrSpace(AS) && + if ((AMDGPU::isFlatGlobalAddrSpace(AS) || + AS == AMDGPUAS::BUFFER_FAT_POINTER) && Subtarget->hasAtomicFaddNoRtnInsts()) { if (Subtarget->hasGFX940Insts()) return AtomicExpansionKind::None; @@ -16025,11 +16026,13 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const { if (HasSystemScope) return AtomicExpansionKind::CmpXChg; - if (AS == AMDGPUAS::GLOBAL_ADDRESS && Ty->isFloatTy()) { - // global atomic fadd f32 no-rtn: gfx908, gfx90a, gfx940, gfx11+. + if ((AS == AMDGPUAS::GLOBAL_ADDRESS || + AS == AMDGPUAS::BUFFER_FAT_POINTER) && + Ty->isFloatTy()) { + // global/buffer atomic fadd f32 no-rtn: gfx908, gfx90a, gfx940, gfx11+. if (RMW->use_empty() && Subtarget->hasAtomicFaddNoRtnInsts()) return ReportUnsafeHWInst(AtomicExpansionKind::None); - // global atomic fadd f32 rtn: gfx90a, gfx940, gfx11+. + // global/buffer atomic fadd f32 rtn: gfx90a, gfx940, gfx11+. if (!RMW->use_empty() && Subtarget->hasAtomicFaddRtnInsts()) return ReportUnsafeHWInst(AtomicExpansionKind::None); } @@ -16084,7 +16087,8 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const { case AtomicRMWInst::Max: case AtomicRMWInst::UMin: case AtomicRMWInst::UMax: { - if (AMDGPU::isFlatGlobalAddrSpace(AS)) { + if (AMDGPU::isFlatGlobalAddrSpace(AS) || + AS == AMDGPUAS::BUFFER_FAT_POINTER) { if (RMW->getType()->isFloatTy() && unsafeFPAtomicsDisabled(RMW->getFunction())) return AtomicExpansionKind::CmpXChg; diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-non-integral-address-spaces-vectors.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-non-integral-address-spaces-vectors.ll index ed86cc2d4307..ae0556c1b21f 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-non-integral-address-spaces-vectors.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-non-integral-address-spaces-vectors.ll @@ -1,15 +1,73 @@ -; RUN: not --crash llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -o - -stop-after=irtranslator < %s -; REQUIRES: asserts - -; Confirm that no one's gotten vectors of addrspace(7) pointers to go through the -; IR translater incidentally. +; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2 +; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -o - -stop-after=irtranslator < %s | FileCheck %s define <2 x ptr addrspace(7)> @no_auto_constfold_gep_vector() { + ; CHECK-LABEL: name: no_auto_constfold_gep_vector + ; CHECK: bb.1 (%ir-block.0): + ; CHECK-NEXT: [[C:%[0-9]+]]:_(p8) = G_CONSTANT i128 0 + ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p8>) = G_BUILD_VECTOR [[C]](p8), [[C]](p8) + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 123 + ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[C1]](s32), [[C1]](s32) + ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<2 x p8>) + ; CHECK-NEXT: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<2 x s32>) + ; CHECK-NEXT: $vgpr0 = COPY [[UV]](s32) + ; CHECK-NEXT: $vgpr1 = COPY [[UV1]](s32) + ; CHECK-NEXT: $vgpr2 = COPY [[UV2]](s32) + ; CHECK-NEXT: $vgpr3 = COPY [[UV3]](s32) + ; CHECK-NEXT: $vgpr4 = COPY [[UV4]](s32) + ; CHECK-NEXT: $vgpr5 = COPY [[UV5]](s32) + ; CHECK-NEXT: $vgpr6 = COPY [[UV6]](s32) + ; CHECK-NEXT: $vgpr7 = COPY [[UV7]](s32) + ; CHECK-NEXT: $vgpr8 = COPY [[UV8]](s32) + ; CHECK-NEXT: $vgpr9 = COPY [[UV9]](s32) + ; CHECK-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9 %gep = getelementptr i8, <2 x ptr addrspace(7)> zeroinitializer, <2 x i32> ret <2 x ptr addrspace(7)> %gep } define <2 x ptr addrspace(7)> @gep_vector_splat(<2 x ptr addrspace(7)> %ptrs, i64 %idx) { + ; CHECK-LABEL: name: gep_vector_splat + ; CHECK: bb.1 (%ir-block.0): + ; CHECK-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1 + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4 + ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6 + ; CHECK-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7 + ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32) + ; CHECK-NEXT: [[MV1:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32) + ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p8>) = G_BUILD_VECTOR [[MV]](p8), [[MV1]](p8) + ; CHECK-NEXT: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8 + ; CHECK-NEXT: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9 + ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32) + ; CHECK-NEXT: [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10 + ; CHECK-NEXT: [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11 + ; CHECK-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY10]](s32), [[COPY11]](s32) + ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(<2 x s64>) = G_IMPLICIT_DEF + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<2 x p8>) = G_IMPLICIT_DEF + ; CHECK-NEXT: [[DEF2:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF + ; CHECK-NEXT: [[IVEC:%[0-9]+]]:_(<2 x s64>) = G_INSERT_VECTOR_ELT [[DEF]], [[MV2]](s64), [[C]](s64) + ; CHECK-NEXT: [[SHUF:%[0-9]+]]:_(<2 x s64>) = G_SHUFFLE_VECTOR [[IVEC]](<2 x s64>), [[DEF]], shufflemask(0, 0) + ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(<2 x s32>) = G_TRUNC [[SHUF]](<2 x s64>) + ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(<2 x s32>) = G_ADD [[BUILD_VECTOR1]], [[TRUNC]] + ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<2 x p8>) + ; CHECK-NEXT: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ADD]](<2 x s32>) + ; CHECK-NEXT: $vgpr0 = COPY [[UV]](s32) + ; CHECK-NEXT: $vgpr1 = COPY [[UV1]](s32) + ; CHECK-NEXT: $vgpr2 = COPY [[UV2]](s32) + ; CHECK-NEXT: $vgpr3 = COPY [[UV3]](s32) + ; CHECK-NEXT: $vgpr4 = COPY [[UV4]](s32) + ; CHECK-NEXT: $vgpr5 = COPY [[UV5]](s32) + ; CHECK-NEXT: $vgpr6 = COPY [[UV6]](s32) + ; CHECK-NEXT: $vgpr7 = COPY [[UV7]](s32) + ; CHECK-NEXT: $vgpr8 = COPY [[UV8]](s32) + ; CHECK-NEXT: $vgpr9 = COPY [[UV9]](s32) + ; CHECK-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9 %gep = getelementptr i8, <2 x ptr addrspace(7)> %ptrs, i64 %idx ret <2 x ptr addrspace(7)> %gep } diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-non-integral-address-spaces.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-non-integral-address-spaces.ll index aff716c5abfd..d5c68e772d0b 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-non-integral-address-spaces.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-non-integral-address-spaces.ll @@ -5,15 +5,14 @@ define ptr addrspace(7) @no_auto_constfold_gep() { ; CHECK-LABEL: name: no_auto_constfold_gep ; CHECK: bb.1 (%ir-block.0): - ; CHECK-NEXT: [[C:%[0-9]+]]:_(p7) = G_CONSTANT i160 0 + ; CHECK-NEXT: [[C:%[0-9]+]]:_(p8) = G_CONSTANT i128 0 ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 123 - ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p7) = G_PTR_ADD [[C]], [[C1]](s32) - ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[PTR_ADD]](p7) + ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C]](p8) ; CHECK-NEXT: $vgpr0 = COPY [[UV]](s32) ; CHECK-NEXT: $vgpr1 = COPY [[UV1]](s32) ; CHECK-NEXT: $vgpr2 = COPY [[UV2]](s32) ; CHECK-NEXT: $vgpr3 = COPY [[UV3]](s32) - ; CHECK-NEXT: $vgpr4 = COPY [[UV4]](s32) + ; CHECK-NEXT: $vgpr4 = COPY [[C1]](s32) ; CHECK-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4 %gep = getelementptr i8, ptr addrspace(7) null, i32 123 ret ptr addrspace(7) %gep diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll index c67328a025b8..5007f77316f5 100644 --- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll +++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll @@ -51,6 +51,11 @@ ; GCN-O0-NEXT: AMDGPU Annotate Kernel Features ; GCN-O0-NEXT: FunctionPass Manager ; GCN-O0-NEXT: AMDGPU Lower Kernel Arguments +; GCN-O0-NEXT: Lower buffer fat pointer operations to buffer resources +; GCN-O0-NEXT: CallGraph Construction +; GCN-O0-NEXT: Call Graph SCC Pass Manager +; GCN-O0-NEXT: DummyCGSCCPass +; GCN-O0-NEXT: FunctionPass Manager ; GCN-O0-NEXT: Lazy Value Information Analysis ; GCN-O0-NEXT: Lower SwitchInst's to branches ; GCN-O0-NEXT: Lower invoke and unwind, for unwindless code generators @@ -229,6 +234,11 @@ ; GCN-O1-NEXT: AMDGPU Annotate Kernel Features ; GCN-O1-NEXT: FunctionPass Manager ; GCN-O1-NEXT: AMDGPU Lower Kernel Arguments +; GCN-O1-NEXT: Lower buffer fat pointer operations to buffer resources +; GCN-O1-NEXT: CallGraph Construction +; GCN-O1-NEXT: Call Graph SCC Pass Manager +; GCN-O1-NEXT: DummyCGSCCPass +; GCN-O1-NEXT: FunctionPass Manager ; GCN-O1-NEXT: Dominator Tree Construction ; GCN-O1-NEXT: Natural Loop Information ; GCN-O1-NEXT: CodeGen Prepare @@ -513,6 +523,11 @@ ; GCN-O1-OPTS-NEXT: AMDGPU Annotate Kernel Features ; GCN-O1-OPTS-NEXT: FunctionPass Manager ; GCN-O1-OPTS-NEXT: AMDGPU Lower Kernel Arguments +; GCN-O1-OPTS-NEXT: Lower buffer fat pointer operations to buffer resources +; GCN-O1-OPTS-NEXT: CallGraph Construction +; GCN-O1-OPTS-NEXT: Call Graph SCC Pass Manager +; GCN-O1-OPTS-NEXT: DummyCGSCCPass +; GCN-O1-OPTS-NEXT: FunctionPass Manager ; GCN-O1-OPTS-NEXT: Dominator Tree Construction ; GCN-O1-OPTS-NEXT: Natural Loop Information ; GCN-O1-OPTS-NEXT: CodeGen Prepare @@ -815,6 +830,11 @@ ; GCN-O2-NEXT: AMDGPU Annotate Kernel Features ; GCN-O2-NEXT: FunctionPass Manager ; GCN-O2-NEXT: AMDGPU Lower Kernel Arguments +; GCN-O2-NEXT: Lower buffer fat pointer operations to buffer resources +; GCN-O2-NEXT: CallGraph Construction +; GCN-O2-NEXT: Call Graph SCC Pass Manager +; GCN-O2-NEXT: DummyCGSCCPass +; GCN-O2-NEXT: FunctionPass Manager ; GCN-O2-NEXT: Dominator Tree Construction ; GCN-O2-NEXT: Natural Loop Information ; GCN-O2-NEXT: CodeGen Prepare @@ -1131,6 +1151,11 @@ ; GCN-O3-NEXT: AMDGPU Annotate Kernel Features ; GCN-O3-NEXT: FunctionPass Manager ; GCN-O3-NEXT: AMDGPU Lower Kernel Arguments +; GCN-O3-NEXT: Lower buffer fat pointer operations to buffer resources +; GCN-O3-NEXT: CallGraph Construction +; GCN-O3-NEXT: Call Graph SCC Pass Manager +; GCN-O3-NEXT: DummyCGSCCPass +; GCN-O3-NEXT: FunctionPass Manager ; GCN-O3-NEXT: Dominator Tree Construction ; GCN-O3-NEXT: Natural Loop Information ; GCN-O3-NEXT: CodeGen Prepare diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-calls.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-calls.ll new file mode 100644 index 000000000000..6f0d51a02773 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-calls.ll @@ -0,0 +1,114 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 +; RUN: opt -S -mcpu=gfx900 -amdgpu-lower-buffer-fat-pointers < %s | FileCheck %s +; RUN: opt -S -mcpu=gfx900 -passes=amdgpu-lower-buffer-fat-pointers < %s | FileCheck %s + +target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8" +target triple = "amdgcn--" + +define ptr addrspace(7) @recur.inner.1(ptr addrspace(7) %x, i32 %v) { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @recur.inner.1 +; CHECK-SAME: ({ ptr addrspace(8), i32 } [[X:%.*]], i32 [[V:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: bb: +; CHECK-NEXT: [[X_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[X]], 0 +; CHECK-NEXT: [[X_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[X]], 1 +; CHECK-NEXT: [[ISBASE:%.*]] = icmp sgt i32 [[V]], 0 +; CHECK-NEXT: br i1 [[ISBASE]], label [[RECUR:%.*]], label [[ELSE:%.*]] +; CHECK: recur: +; CHECK-NEXT: [[DEC:%.*]] = sub i32 [[V]], 1 +; CHECK-NEXT: [[INC:%.*]] = call { ptr addrspace(8), i32 } @recur.inner.2(i32 [[DEC]], { ptr addrspace(8), i32 } [[X]]) +; CHECK-NEXT: [[INC_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[INC]], 0 +; CHECK-NEXT: [[INC_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[INC]], 1 +; CHECK-NEXT: br label [[END:%.*]] +; CHECK: else: +; CHECK-NEXT: br label [[END]] +; CHECK: end: +; CHECK-NEXT: [[RET_RSRC:%.*]] = phi ptr addrspace(8) [ [[INC_RSRC]], [[RECUR]] ], [ [[X_RSRC]], [[ELSE]] ] +; CHECK-NEXT: [[RET_OFF:%.*]] = phi i32 [ [[INC_OFF]], [[RECUR]] ], [ [[X_OFF]], [[ELSE]] ] +; CHECK-NEXT: [[TMP0:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[RET_RSRC]], 0 +; CHECK-NEXT: [[RET:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP0]], i32 [[RET_OFF]], 1 +; CHECK-NEXT: ret { ptr addrspace(8), i32 } [[RET]] +; +bb: + %isBase = icmp sgt i32 %v, 0 + br i1 %isBase, label %recur, label %else +recur: + %dec = sub i32 %v, 1 + %inc = call ptr addrspace(7) @recur.inner.2(i32 %dec, ptr addrspace(7) %x) + br label %end +else: + br label %end +end: + %ret = phi ptr addrspace(7) [%inc, %recur], [%x, %else] + ret ptr addrspace(7) %ret +} + +define ptr addrspace(7) @recur.inner.2(i32 %v, ptr addrspace(7) %x) { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @recur.inner.2 +; CHECK-SAME: (i32 [[V:%.*]], { ptr addrspace(8), i32 } [[X:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[X_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[X]], 0 +; CHECK-NEXT: [[X_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[X]], 1 +; CHECK-NEXT: [[INC:%.*]] = add i32 [[X_OFF]], 4 +; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[X_RSRC]], 0 +; CHECK-NEXT: [[TMP2:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP1]], i32 [[INC]], 1 +; CHECK-NEXT: [[RET:%.*]] = call { ptr addrspace(8), i32 } @recur.inner.1({ ptr addrspace(8), i32 } [[TMP2]], i32 [[V]]) +; CHECK-NEXT: ret { ptr addrspace(8), i32 } [[RET]] +; + %inc = getelementptr i32, ptr addrspace(7) %x, i32 1 + %ret = call ptr addrspace(7) @recur.inner.1(ptr addrspace(7) %inc, i32 %v) + ret ptr addrspace(7) %ret +} + +define void @recur.outer(ptr addrspace(7) %x, ptr %arg) { +; CHECK-LABEL: define void @recur.outer +; CHECK-SAME: ({ ptr addrspace(8), i32 } [[X:%.*]], ptr [[ARG:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[BOUND:%.*]] = load i32, ptr [[ARG]], align 4 +; CHECK-NEXT: [[RET:%.*]] = call { ptr addrspace(8), i32 } @recur.inner.1({ ptr addrspace(8), i32 } [[X]], i32 [[BOUND]]) +; CHECK-NEXT: [[RET_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[RET]], 0 +; CHECK-NEXT: [[RET_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[RET]], 1 +; CHECK-NEXT: [[RET_INT_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[RET_RSRC]] to i160 +; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i160 [[RET_INT_RSRC]], 32 +; CHECK-NEXT: [[RET_INT_OFF:%.*]] = zext i32 [[RET_OFF]] to i160 +; CHECK-NEXT: [[RET_INT:%.*]] = or i160 [[TMP1]], [[RET_INT_OFF]] +; CHECK-NEXT: store i160 [[RET_INT]], ptr [[ARG]], align 32 +; CHECK-NEXT: ret void +; + %bound = load i32, ptr %arg + %ret = call ptr addrspace(7) @recur.inner.1(ptr addrspace(7) %x, i32 %bound) + store ptr addrspace(7) %ret, ptr %arg + ret void +} + +declare ptr addrspace(7) @extern(ptr addrspace(7) %arg) +define void @caller(ptr addrspace(7) noundef nonnull %arg) { +; CHECK-LABEL: define void @caller +; CHECK-SAME: ({ ptr addrspace(8), i32 } noundef [[ARG:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ARG_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[ARG]], 0 +; CHECK-NEXT: [[ARG_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[ARG]], 1 +; CHECK-NEXT: [[V:%.*]] = call { ptr addrspace(8), i32 } @extern({ ptr addrspace(8), i32 } [[ARG]]) +; CHECK-NEXT: [[V_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[V]], 0 +; CHECK-NEXT: [[V_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[V]], 1 +; CHECK-NEXT: [[V_INT_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[V_RSRC]] to i160 +; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i160 [[V_INT_RSRC]], 32 +; CHECK-NEXT: [[V_INT_OFF:%.*]] = zext i32 [[V_OFF]] to i160 +; CHECK-NEXT: [[V_INT:%.*]] = or i160 [[TMP1]], [[V_INT_OFF]] +; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i160(i160 [[V_INT]], ptr addrspace(8) align 32 [[ARG_RSRC]], i32 [[ARG_OFF]], i32 0, i32 0) +; CHECK-NEXT: ret void +; + %v = call ptr addrspace(7) @extern(ptr addrspace(7) %arg) + store ptr addrspace(7) %v, ptr addrspace(7) %arg + ret void +} + +define internal noalias noundef nonnull ptr addrspace(7) @foo(ptr addrspace(7) noalias noundef nonnull %arg) { +; CHECK-LABEL: define internal noundef { ptr addrspace(8), i32 } @foo +; CHECK-SAME: ({ ptr addrspace(8), i32 } noundef [[ARG:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ARG_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[ARG]], 0 +; CHECK-NEXT: [[ARG_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[ARG]], 1 +; CHECK-NEXT: [[RET:%.*]] = add nuw i32 [[ARG_OFF]], 4 +; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[ARG_RSRC]], 0 +; CHECK-NEXT: [[TMP2:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP1]], i32 [[RET]], 1 +; CHECK-NEXT: ret { ptr addrspace(8), i32 } [[TMP2]] +; + %ret = getelementptr inbounds i32, ptr addrspace(7) %arg, i32 1 + ret ptr addrspace(7) %ret +} diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll new file mode 100644 index 000000000000..77ca227bcf66 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll @@ -0,0 +1,220 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 +; RUN: opt -S -mcpu=gfx900 -amdgpu-lower-buffer-fat-pointers < %s | FileCheck %s +; RUN: opt -S -mcpu=gfx900 -passes=amdgpu-lower-buffer-fat-pointers < %s | FileCheck %s + +target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8" +target triple = "amdgcn--" + +@buf = external addrspace(8) global i8 +@flat = external global i8 + +define ptr addrspace(7) @null() { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @null +; CHECK-SAME: () #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: ret { ptr addrspace(8), i32 } zeroinitializer +; + ret ptr addrspace(7) null +} + +define <2 x ptr addrspace(7)> @null_vector() { +; CHECK-LABEL: define { <2 x ptr addrspace(8)>, <2 x i32> } @null_vector +; CHECK-SAME: () #[[ATTR0]] { +; CHECK-NEXT: ret { <2 x ptr addrspace(8)>, <2 x i32> } zeroinitializer +; + ret <2 x ptr addrspace(7)> zeroinitializer +} + +define ptr addrspace(7) @undef() { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @undef +; CHECK-SAME: () #[[ATTR0]] { +; CHECK-NEXT: ret { ptr addrspace(8), i32 } undef +; + ret ptr addrspace(7) undef +} + +define <2 x ptr addrspace(7)> @undef_vec() { +; CHECK-LABEL: define { <2 x ptr addrspace(8)>, <2 x i32> } @undef_vec +; CHECK-SAME: () #[[ATTR0]] { +; CHECK-NEXT: ret { <2 x ptr addrspace(8)>, <2 x i32> } undef +; + ret <2 x ptr addrspace(7)> undef +} + +define ptr addrspace(7) @poison() { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @poison +; CHECK-SAME: () #[[ATTR0]] { +; CHECK-NEXT: ret { ptr addrspace(8), i32 } poison +; + ret ptr addrspace(7) poison +} + +define <2 x ptr addrspace(7)> @poison_vec() { +; CHECK-LABEL: define { <2 x ptr addrspace(8)>, <2 x i32> } @poison_vec +; CHECK-SAME: () #[[ATTR0]] { +; CHECK-NEXT: ret { <2 x ptr addrspace(8)>, <2 x i32> } poison +; + ret <2 x ptr addrspace(7)> poison +} + +define ptr addrspace(7) @cast_global() { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @cast_global +; CHECK-SAME: () #[[ATTR0]] { +; CHECK-NEXT: ret { ptr addrspace(8), i32 } { ptr addrspace(8) @buf, i32 0 } +; + ret ptr addrspace(7) addrspacecast (ptr addrspace(8) @buf to ptr addrspace(7)) +} + +define ptr addrspace(7) @cast_null() { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @cast_null +; CHECK-SAME: () #[[ATTR0]] { +; CHECK-NEXT: ret { ptr addrspace(8), i32 } zeroinitializer +; + ret ptr addrspace(7) addrspacecast (ptr addrspace(8) null to ptr addrspace(7)) +} + +define <2 x ptr addrspace(7)> @cast_vec() { +; CHECK-LABEL: define { <2 x ptr addrspace(8)>, <2 x i32> } @cast_vec +; CHECK-SAME: () #[[ATTR0]] { +; CHECK-NEXT: ret { <2 x ptr addrspace(8)>, <2 x i32> } { <2 x ptr addrspace(8)> , <2 x i32> zeroinitializer } +; + ret <2 x ptr addrspace(7)> addrspacecast ( + <2 x ptr addrspace(8)> + to <2 x ptr addrspace(7)>) +} + +define ptr addrspace(7) @gep() { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @gep +; CHECK-SAME: () #[[ATTR0]] { +; CHECK-NEXT: ret { ptr addrspace(8), i32 } { ptr addrspace(8) @buf, i32 36 } +; + ret ptr addrspace(7) getelementptr inbounds ( + [4 x i32], + ptr addrspace(7) addrspacecast (ptr addrspace(8) @buf to ptr addrspace(7)), + i64 2, i32 1) +} + +define <2 x ptr addrspace(7)> @gep_vector() { +; CHECK-LABEL: define { <2 x ptr addrspace(8)>, <2 x i32> } @gep_vector +; CHECK-SAME: () #[[ATTR0]] { +; CHECK-NEXT: ret { <2 x ptr addrspace(8)>, <2 x i32> } { <2 x ptr addrspace(8)> , <2 x i32> } +; + ret <2 x ptr addrspace(7)> getelementptr ( + i32, + <2 x ptr addrspace(7)> + , + <2 x i32> ) +} + +define ptr @gep_of_p7() { +; CHECK-LABEL: define ptr @gep_of_p7 +; CHECK-SAME: () #[[ATTR0]] { +; CHECK-NEXT: ret ptr getelementptr inbounds (ptr addrspace(7), ptr @flat, i64 2) +; + ret ptr getelementptr inbounds (ptr addrspace(7), ptr @flat, i64 2) +} + +define ptr @gep_of_p7_vector() { +; CHECK-LABEL: define ptr @gep_of_p7_vector +; CHECK-SAME: () #[[ATTR0]] { +; CHECK-NEXT: ret ptr getelementptr (<2 x ptr addrspace(7)>, ptr @flat, i64 2) +; + ret ptr getelementptr (<2 x ptr addrspace(7)>, ptr @flat, i64 2) +} + +define ptr @gep_of_p7_struct() { +; CHECK-LABEL: define ptr @gep_of_p7_struct +; CHECK-SAME: () #[[ATTR0]] { +; CHECK-NEXT: ret ptr getelementptr ({ ptr addrspace(7), i32 }, ptr @flat, i64 2) +; + ret ptr getelementptr ({ptr addrspace(7), i32}, ptr @flat, i64 2) +} + +define ptr addrspace(7) @gep_p7_from_p7() { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @gep_p7_from_p7 +; CHECK-SAME: () #[[ATTR0]] { +; CHECK-NEXT: ret { ptr addrspace(8), i32 } { ptr addrspace(8) @buf, i32 48 } +; + ret ptr addrspace(7) getelementptr (ptr addrspace(7), + ptr addrspace(7) addrspacecast (ptr addrspace(8) @buf to ptr addrspace(7)), + i64 2) +} + +define i160 @ptrtoint() { +; CHECK-LABEL: define i160 @ptrtoint +; CHECK-SAME: () #[[ATTR0]] { +; CHECK-NEXT: ret i160 add nuw nsw (i160 shl nuw (i160 ptrtoint (ptr addrspace(8) @buf to i160), i160 32), i160 12) +; + ret i160 ptrtoint( + ptr addrspace(7) getelementptr( + i32, ptr addrspace(7) addrspacecast (ptr addrspace(8) @buf to ptr addrspace(7)), + i32 3) to i160) +} + +define i256 @ptrtoint_long() { +; CHECK-LABEL: define i256 @ptrtoint_long +; CHECK-SAME: () #[[ATTR0]] { +; CHECK-NEXT: ret i256 add nuw nsw (i256 shl nuw nsw (i256 ptrtoint (ptr addrspace(8) @buf to i256), i256 32), i256 12) +; + ret i256 ptrtoint( + ptr addrspace(7) getelementptr( + i32, ptr addrspace(7) addrspacecast (ptr addrspace(8) @buf to ptr addrspace(7)), + i32 3) to i256) +} + +define i64 @ptrtoint_short() { +; CHECK-LABEL: define i64 @ptrtoint_short +; CHECK-SAME: () #[[ATTR0]] { +; CHECK-NEXT: ret i64 add nuw nsw (i64 shl (i64 ptrtoint (ptr addrspace(8) @buf to i64), i64 32), i64 12) +; + ret i64 ptrtoint( + ptr addrspace(7) getelementptr( + i32, ptr addrspace(7) addrspacecast (ptr addrspace(8) @buf to ptr addrspace(7)), + i32 3) to i64) +} + +define i32 @ptrtoint_very_short() { +; CHECK-LABEL: define i32 @ptrtoint_very_short +; CHECK-SAME: () #[[ATTR0]] { +; CHECK-NEXT: ret i32 add nuw nsw (i32 shl (i32 ptrtoint (ptr addrspace(8) @buf to i32), i32 32), i32 12) +; + ret i32 ptrtoint( + ptr addrspace(7) getelementptr( + i32, ptr addrspace(7) addrspacecast (ptr addrspace(8) @buf to ptr addrspace(7)), + i32 3) to i32) +} + + +define <2 x i160> @ptrtoint_vec() { +; CHECK-LABEL: define <2 x i160> @ptrtoint_vec +; CHECK-SAME: () #[[ATTR0]] { +; CHECK-NEXT: ret <2 x i160> zeroinitializer +; + ret <2 x i160> ptrtoint (<2 x ptr addrspace(7)> zeroinitializer to <2 x i160>) +} + +define ptr addrspace(7) @inttoptr() { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @inttoptr +; CHECK-SAME: () #[[ATTR0]] { +; CHECK-NEXT: ret { ptr addrspace(8), i32 } zeroinitializer +; + ret ptr addrspace(7) inttoptr (i160 0 to ptr addrspace(7)) +} + +define <2 x ptr addrspace(7)> @inttoptr_vec() { +; CHECK-LABEL: define { <2 x ptr addrspace(8)>, <2 x i32> } @inttoptr_vec +; CHECK-SAME: () #[[ATTR0]] { +; CHECK-NEXT: ret { <2 x ptr addrspace(8)>, <2 x i32> } { <2 x ptr addrspace(8)> zeroinitializer, <2 x i32> } +; + ret <2 x ptr addrspace(7)> inttoptr (<2 x i160> to <2 x ptr addrspace(7)>) +} + +define i32 @fancy_zero() { +; CHECK-LABEL: define i32 @fancy_zero +; CHECK-SAME: () #[[ATTR0]] { +; CHECK-NEXT: ret i32 shl (i32 ptrtoint (ptr addrspace(8) @buf to i32), i32 32) +; + ret i32 ptrtoint ( + ptr addrspace(7) addrspacecast (ptr addrspace(8) @buf to ptr addrspace(7)) + to i32) +} diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-control-flow.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-control-flow.ll new file mode 100644 index 000000000000..e2ecbb700d04 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-control-flow.ll @@ -0,0 +1,458 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 +; RUN: opt -S -mcpu=gfx900 -amdgpu-lower-buffer-fat-pointers < %s | FileCheck %s +; RUN: opt -S -mcpu=gfx900 -passes=amdgpu-lower-buffer-fat-pointers < %s | FileCheck %s + +target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8" +target triple = "amdgcn--" + +;; This should optimize to just the offset part +define float @sum(ptr addrspace(8) %buf, i32 %len) { +; CHECK-LABEL: define float @sum +; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], i32 [[LEN:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[SUM_PREV:%.*]] = phi float [ [[SUM:%.*]], [[LOOP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ] +; CHECK-NEXT: [[PTR_PREV_OFF:%.*]] = phi i32 [ [[PTR:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ] +; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 [[PTR_PREV_OFF]], i32 0, i32 0) +; CHECK-NEXT: [[SUM]] = fadd float [[SUM_PREV]], [[VAL]] +; CHECK-NEXT: [[PTR]] = add i32 [[PTR_PREV_OFF]], 4 +; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1 +; CHECK-NEXT: [[TEST:%.*]] = icmp ult i32 [[I_NEXT]], [[LEN]] +; CHECK-NEXT: br i1 [[TEST]], label [[LOOP]], label [[EXIT:%.*]] +; CHECK: exit: +; CHECK-NEXT: ret float [[SUM]] +; +entry: + %start = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) + br label %loop +loop: + %sum.prev = phi float [ %sum, %loop ], [ 0.0, %entry ] + %ptr.prev = phi ptr addrspace(7) [ %ptr, %loop ], [ %start, %entry ] + %i = phi i32 [ %i.next, %loop ], [ 0, %entry ] + + %val = load float, ptr addrspace(7) %ptr.prev + %sum = fadd float %sum.prev, %val + + %ptr = getelementptr float, ptr addrspace(7) %ptr.prev, i32 1 + %i.next = add i32 %i, 1 + %test = icmp ult i32 %i.next, %len + br i1 %test, label %loop, label %exit +exit: + ret float %sum +} + +;; But this should not +define float @sum_integer_ops(ptr addrspace(8) %buf, i32 %len) { +; CHECK-LABEL: define float @sum_integer_ops +; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], i32 [[LEN:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[SUM_PREV:%.*]] = phi float [ [[SUM:%.*]], [[LOOP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ] +; CHECK-NEXT: [[PTR_PREV_RSRC:%.*]] = phi ptr addrspace(8) [ [[PTR_RSRC:%.*]], [[LOOP]] ], [ [[BUF]], [[ENTRY]] ] +; CHECK-NEXT: [[PTR_PREV_OFF:%.*]] = phi i32 [ [[PTR_OFF:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ] +; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[PTR_PREV_RSRC]], i32 [[PTR_PREV_OFF]], i32 0, i32 0) +; CHECK-NEXT: [[SUM]] = fadd float [[SUM_PREV]], [[VAL]] +; CHECK-NEXT: [[PTR_PREV_INT_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[PTR_PREV_RSRC]] to i160 +; CHECK-NEXT: [[TMP0:%.*]] = shl nuw i160 [[PTR_PREV_INT_RSRC]], 32 +; CHECK-NEXT: [[PTR_PREV_INT_OFF:%.*]] = zext i32 [[PTR_PREV_OFF]] to i160 +; CHECK-NEXT: [[PTR_PREV_INT:%.*]] = or i160 [[TMP0]], [[PTR_PREV_INT_OFF]] +; CHECK-NEXT: [[PTR_INT:%.*]] = add i160 [[PTR_PREV_INT]], 4 +; CHECK-NEXT: [[TMP1:%.*]] = lshr i160 [[PTR_INT]], 32 +; CHECK-NEXT: [[TMP2:%.*]] = trunc i160 [[TMP1]] to i128 +; CHECK-NEXT: [[PTR_RSRC]] = inttoptr i128 [[TMP2]] to ptr addrspace(8) +; CHECK-NEXT: [[PTR_OFF]] = trunc i160 [[PTR_INT]] to i32 +; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1 +; CHECK-NEXT: [[TEST:%.*]] = icmp ult i32 [[I_NEXT]], [[LEN]] +; CHECK-NEXT: br i1 [[TEST]], label [[LOOP]], label [[EXIT:%.*]] +; CHECK: exit: +; CHECK-NEXT: ret float [[SUM]] +; +entry: + %start = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) + br label %loop +loop: + %sum.prev = phi float [ %sum, %loop ], [ 0.0, %entry ] + %ptr.prev = phi ptr addrspace(7) [ %ptr, %loop ], [ %start, %entry ] + %i = phi i32 [ %i.next, %loop ], [ 0, %entry ] + + %val = load float, ptr addrspace(7) %ptr.prev + %sum = fadd float %sum.prev, %val + + %ptr.prev.int = ptrtoint ptr addrspace(7) %ptr.prev to i160 + %ptr.int = add i160 %ptr.prev.int, 4 + %ptr = inttoptr i160 %ptr.int to ptr addrspace(7) + %i.next = add i32 %i, 1 + %test = icmp ult i32 %i.next, %len + br i1 %test, label %loop, label %exit +exit: + ret float %sum +} + +;; Should go to offsets only +define float @sum_2d(ptr addrspace(8) %buf, i32 %ii, i32 %jj) { +; CHECK-LABEL: define float @sum_2d +; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], i32 [[II:%.*]], i32 [[JJ:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[LOOP1_ENTRY:%.*]] +; CHECK: loop1.entry: +; CHECK-NEXT: [[SUM1_PREV:%.*]] = phi float [ [[SUM:%.*]], [[LOOP1_EXIT:%.*]] ], [ 0.000000e+00, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP1_EXIT]] ], [ 0, [[ENTRY]] ] +; CHECK-NEXT: [[PTR1_PREV_OFF:%.*]] = phi i32 [ [[PTR1:%.*]], [[LOOP1_EXIT]] ], [ 0, [[ENTRY]] ] +; CHECK-NEXT: br label [[LOOP2:%.*]] +; CHECK: loop2: +; CHECK-NEXT: [[SUM2_PREV:%.*]] = phi float [ [[SUM]], [[LOOP2]] ], [ [[SUM1_PREV]], [[LOOP1_ENTRY]] ] +; CHECK-NEXT: [[J:%.*]] = phi i32 [ [[J_NEXT:%.*]], [[LOOP2]] ], [ 0, [[LOOP1_ENTRY]] ] +; CHECK-NEXT: [[PTR2_PREV_OFF:%.*]] = phi i32 [ [[PTR2:%.*]], [[LOOP2]] ], [ [[PTR1_PREV_OFF]], [[LOOP1_ENTRY]] ] +; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 [[PTR2_PREV_OFF]], i32 0, i32 0) +; CHECK-NEXT: [[SUM]] = fadd float [[SUM2_PREV]], [[VAL]] +; CHECK-NEXT: [[PTR2]] = add i32 [[PTR2_PREV_OFF]], 4 +; CHECK-NEXT: [[J_NEXT]] = add i32 [[J]], 1 +; CHECK-NEXT: [[TEST2:%.*]] = icmp ult i32 [[J_NEXT]], [[JJ]] +; CHECK-NEXT: br i1 [[TEST2]], label [[LOOP2]], label [[LOOP1_EXIT]] +; CHECK: loop1.exit: +; CHECK-NEXT: [[PTR1]] = add i32 [[PTR2]], 4 +; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1 +; CHECK-NEXT: [[TEST1:%.*]] = icmp ult i32 [[I_NEXT]], [[II]] +; CHECK-NEXT: br i1 [[TEST1]], label [[LOOP1_ENTRY]], label [[EXIT:%.*]] +; CHECK: exit: +; CHECK-NEXT: ret float [[SUM]] +; +entry: + %start = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) + br label %loop1.entry +loop1.entry: + %sum1.prev = phi float [ %sum, %loop1.exit ], [ 0.0, %entry ] + %ptr1.prev = phi ptr addrspace(7) [ %ptr1, %loop1.exit ], [ %start, %entry ] + %i = phi i32 [ %i.next, %loop1.exit ], [ 0, %entry ] + + br label %loop2 +loop2: + %sum2.prev = phi float [ %sum, %loop2 ], [ %sum1.prev, %loop1.entry ] + %ptr2.prev = phi ptr addrspace(7) [ %ptr2, %loop2 ], [ %ptr1.prev, %loop1.entry ] + %j = phi i32 [ %j.next, %loop2 ], [ 0, %loop1.entry ] + + %val = load float, ptr addrspace(7) %ptr2.prev + %sum = fadd float %sum2.prev, %val + + %ptr2 = getelementptr float, ptr addrspace(7) %ptr2.prev, i32 1 + %j.next = add i32 %j, 1 + %test2 = icmp ult i32 %j.next, %jj + + br i1 %test2, label %loop2, label %loop1.exit +loop1.exit: + %ptr1 = getelementptr float, ptr addrspace(7) %ptr2, i32 1 + %i.next = add i32 %i, 1 + %test1 = icmp ult i32 %i.next, %ii + br i1 %test1, label %loop1.entry, label %exit +exit: + ret float %sum +} + +;; This should optimize to just the offset parts since all the arguments to the +;; select point to the same buffer. +define float @sum_jump_on_negative(ptr addrspace(8) %buf, i32 %len) { +; CHECK-LABEL: define float @sum_jump_on_negative +; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], i32 [[LEN:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[SUM_PREV:%.*]] = phi float [ [[SUM:%.*]], [[LOOP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ] +; CHECK-NEXT: [[PTR_PREV_OFF:%.*]] = phi i32 [ [[PTR_OFF:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ] +; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 [[PTR_PREV_OFF]], i32 0, i32 0) +; CHECK-NEXT: [[SUM]] = fadd float [[SUM_PREV]], [[VAL]] +; CHECK-NEXT: [[SKIP_NEXT:%.*]] = fcmp olt float [[VAL]], 0.000000e+00 +; CHECK-NEXT: [[SMALL_JUMP:%.*]] = add i32 [[PTR_PREV_OFF]], 4 +; CHECK-NEXT: [[LARGE_JUMP:%.*]] = add i32 [[PTR_PREV_OFF]], 8 +; CHECK-NEXT: [[PTR_OFF]] = select i1 [[SKIP_NEXT]], i32 [[LARGE_JUMP]], i32 [[SMALL_JUMP]] +; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1 +; CHECK-NEXT: [[TEST:%.*]] = icmp ult i32 [[I_NEXT]], [[LEN]] +; CHECK-NEXT: br i1 [[TEST]], label [[LOOP]], label [[EXIT:%.*]] +; CHECK: exit: +; CHECK-NEXT: ret float [[SUM]] +; +entry: + %start = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) + br label %loop +loop: + %sum.prev = phi float [ %sum, %loop ], [ 0.0, %entry ] + %ptr.prev = phi ptr addrspace(7) [ %ptr, %loop ], [ %start, %entry ] + %i = phi i32 [ %i.next, %loop ], [ 0, %entry ] + + %val = load float, ptr addrspace(7) %ptr.prev + %sum = fadd float %sum.prev, %val + + %skip.next = fcmp olt float %val, 0.0 + %small.jump = getelementptr float, ptr addrspace(7) %ptr.prev, i32 1 + %large.jump = getelementptr float, ptr addrspace(7) %ptr.prev, i32 2 + %ptr = select i1 %skip.next, ptr addrspace(7) %large.jump, ptr addrspace(7) %small.jump + + %i.next = add i32 %i, 1 + %test = icmp ult i32 %i.next, %len + br i1 %test, label %loop, label %exit +exit: + ret float %sum +} + +define float @sum_jump_on_negative_with_phi(ptr addrspace(8) %buf, i32 %len) { +; CHECK-LABEL: define float @sum_jump_on_negative_with_phi +; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], i32 [[LEN:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[SUM_PREV:%.*]] = phi float [ [[SUM:%.*]], [[LOOP_EXIT:%.*]] ], [ 0.000000e+00, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP_EXIT]] ], [ 0, [[ENTRY]] ] +; CHECK-NEXT: [[PTR_PREV_OFF:%.*]] = phi i32 [ [[PTR_OFF:%.*]], [[LOOP_EXIT]] ], [ 0, [[ENTRY]] ] +; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 [[PTR_PREV_OFF]], i32 0, i32 0) +; CHECK-NEXT: [[SUM]] = fadd float [[SUM_PREV]], [[VAL]] +; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1 +; CHECK-NEXT: [[TEST:%.*]] = icmp ult i32 [[I_NEXT]], [[LEN]] +; CHECK-NEXT: [[SKIP_NEXT:%.*]] = fcmp olt float [[VAL]], 0.000000e+00 +; CHECK-NEXT: br i1 [[SKIP_NEXT]], label [[THEN:%.*]], label [[ELSE:%.*]] +; CHECK: then: +; CHECK-NEXT: [[LARGE_JUMP:%.*]] = add i32 [[PTR_PREV_OFF]], 8 +; CHECK-NEXT: br label [[LOOP_EXIT]] +; CHECK: else: +; CHECK-NEXT: [[SMALL_JUMP:%.*]] = add i32 [[PTR_PREV_OFF]], 4 +; CHECK-NEXT: br label [[LOOP_EXIT]] +; CHECK: loop.exit: +; CHECK-NEXT: [[PTR_OFF]] = phi i32 [ [[LARGE_JUMP]], [[THEN]] ], [ [[SMALL_JUMP]], [[ELSE]] ] +; CHECK-NEXT: br i1 [[TEST]], label [[LOOP]], label [[EXIT:%.*]] +; CHECK: exit: +; CHECK-NEXT: ret float [[SUM]] +; +entry: + %start = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) + br label %loop +loop: + %sum.prev = phi float [ %sum, %loop.exit ], [ 0.0, %entry ] + %ptr.prev = phi ptr addrspace(7) [ %ptr, %loop.exit ], [ %start, %entry ] + %i = phi i32 [ %i.next, %loop.exit ], [ 0, %entry ] + + %val = load float, ptr addrspace(7) %ptr.prev + %sum = fadd float %sum.prev, %val + + %i.next = add i32 %i, 1 + %test = icmp ult i32 %i.next, %len + + %skip.next = fcmp olt float %val, 0.0 + br i1 %skip.next, label %then, label %else +then: + %large.jump = getelementptr float, ptr addrspace(7) %ptr.prev, i32 2 + br label %loop.exit +else: + %small.jump = getelementptr float, ptr addrspace(7) %ptr.prev, i32 1 + br label %loop.exit +loop.exit: + %ptr = phi ptr addrspace(7) [ %large.jump, %then ], [ %small.jump, %else ] + br i1 %test, label %loop, label %exit +exit: + ret float %sum +} + +;; But this has a shifting resource part. +define float @sum_new_buffer_on_negative(ptr addrspace(8) %buf1, ptr addrspace(8) %buf2, i32 %len) { +; CHECK-LABEL: define float @sum_new_buffer_on_negative +; CHECK-SAME: (ptr addrspace(8) [[BUF1:%.*]], ptr addrspace(8) [[BUF2:%.*]], i32 [[LEN:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[SUM_PREV:%.*]] = phi float [ [[SUM:%.*]], [[LOOP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ] +; CHECK-NEXT: [[PTR_PREV_RSRC:%.*]] = phi ptr addrspace(8) [ [[PTR_RSRC:%.*]], [[LOOP]] ], [ [[BUF1]], [[ENTRY]] ] +; CHECK-NEXT: [[PTR_PREV_OFF:%.*]] = phi i32 [ [[PTR_OFF:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ] +; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[PTR_PREV_RSRC]], i32 [[PTR_PREV_OFF]], i32 0, i32 0) +; CHECK-NEXT: [[SUM]] = fadd float [[SUM_PREV]], [[VAL]] +; CHECK-NEXT: [[HOP:%.*]] = fcmp olt float [[VAL]], 0.000000e+00 +; CHECK-NEXT: [[THIS_NEXT:%.*]] = add i32 [[PTR_PREV_OFF]], 4 +; CHECK-NEXT: [[PTR_RSRC]] = select i1 [[HOP]], ptr addrspace(8) [[PTR_PREV_RSRC]], ptr addrspace(8) [[BUF2]] +; CHECK-NEXT: [[PTR_OFF]] = select i1 [[HOP]], i32 [[THIS_NEXT]], i32 0 +; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1 +; CHECK-NEXT: [[TEST:%.*]] = icmp ult i32 [[I_NEXT]], [[LEN]] +; CHECK-NEXT: br i1 [[TEST]], label [[LOOP]], label [[EXIT:%.*]] +; CHECK: exit: +; CHECK-NEXT: ret float [[SUM]] +; +entry: + %start = addrspacecast ptr addrspace(8) %buf1 to ptr addrspace(7) + %start2 = addrspacecast ptr addrspace(8) %buf2 to ptr addrspace(7) + br label %loop +loop: + %sum.prev = phi float [ %sum, %loop ], [ 0.0, %entry ] + %ptr.prev = phi ptr addrspace(7) [ %ptr, %loop ], [ %start, %entry ] + %i = phi i32 [ %i.next, %loop ], [ 0, %entry ] + + %val = load float, ptr addrspace(7) %ptr.prev + %sum = fadd float %sum.prev, %val + + %hop = fcmp olt float %val, 0.0 + %this.next = getelementptr float, ptr addrspace(7) %ptr.prev, i32 1 + %ptr = select i1 %hop, ptr addrspace(7) %this.next, ptr addrspace(7) %start2 + + %i.next = add i32 %i, 1 + %test = icmp ult i32 %i.next, %len + br i1 %test, label %loop, label %exit +exit: + ret float %sum +} + +;; As does this. +define float @sum_new_buffer_on_negative_with_phi(ptr addrspace(8) %buf1, ptr addrspace(8) %buf2, i32 %len) { +; CHECK-LABEL: define float @sum_new_buffer_on_negative_with_phi +; CHECK-SAME: (ptr addrspace(8) [[BUF1:%.*]], ptr addrspace(8) [[BUF2:%.*]], i32 [[LEN:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[SUM_PREV:%.*]] = phi float [ [[SUM:%.*]], [[LOOP_EXIT:%.*]] ], [ 0.000000e+00, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP_EXIT]] ], [ 0, [[ENTRY]] ] +; CHECK-NEXT: [[PTR_PREV_RSRC:%.*]] = phi ptr addrspace(8) [ [[PTR_RSRC:%.*]], [[LOOP_EXIT]] ], [ [[BUF1]], [[ENTRY]] ] +; CHECK-NEXT: [[PTR_PREV_OFF:%.*]] = phi i32 [ [[PTR_OFF:%.*]], [[LOOP_EXIT]] ], [ 0, [[ENTRY]] ] +; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[PTR_PREV_RSRC]], i32 [[PTR_PREV_OFF]], i32 0, i32 0) +; CHECK-NEXT: [[SUM]] = fadd float [[SUM_PREV]], [[VAL]] +; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1 +; CHECK-NEXT: [[TEST:%.*]] = icmp ult i32 [[I_NEXT]], [[LEN]] +; CHECK-NEXT: [[HOP:%.*]] = fcmp olt float [[VAL]], 0.000000e+00 +; CHECK-NEXT: br i1 [[HOP]], label [[THEN:%.*]], label [[LOOP_EXIT]] +; CHECK: then: +; CHECK-NEXT: [[THIS_NEXT:%.*]] = add i32 [[PTR_PREV_OFF]], 4 +; CHECK-NEXT: br label [[LOOP_EXIT]] +; CHECK: loop.exit: +; CHECK-NEXT: [[PTR_RSRC]] = phi ptr addrspace(8) [ [[PTR_PREV_RSRC]], [[THEN]] ], [ [[BUF2]], [[LOOP]] ] +; CHECK-NEXT: [[PTR_OFF]] = phi i32 [ [[THIS_NEXT]], [[THEN]] ], [ 0, [[LOOP]] ] +; CHECK-NEXT: br i1 [[TEST]], label [[LOOP]], label [[EXIT:%.*]] +; CHECK: exit: +; CHECK-NEXT: ret float [[SUM]] +; +entry: + %start = addrspacecast ptr addrspace(8) %buf1 to ptr addrspace(7) + %start2 = addrspacecast ptr addrspace(8) %buf2 to ptr addrspace(7) + br label %loop +loop: + %sum.prev = phi float [ %sum, %loop.exit ], [ 0.0, %entry ] + %ptr.prev = phi ptr addrspace(7) [ %ptr, %loop.exit ], [ %start, %entry ] + %i = phi i32 [ %i.next, %loop.exit ], [ 0, %entry ] + + %val = load float, ptr addrspace(7) %ptr.prev + %sum = fadd float %sum.prev, %val + + %i.next = add i32 %i, 1 + %test = icmp ult i32 %i.next, %len + %hop = fcmp olt float %val, 0.0 + br i1 %hop, label %then, label %loop.exit +then: + %this.next = getelementptr float, ptr addrspace(7) %ptr.prev, i32 1 + br label %loop.exit +loop.exit: + %ptr = phi ptr addrspace(7) [ %this.next, %then ], [ %start2, %loop ] + br i1 %test, label %loop, label %exit +exit: + ret float %sum +} + +;; Test that the uniform buffer descriptor optimization works correctly for phi +;; nodes that repeat the same predecessor multiple times. +define float @sum_duplicate_preds(ptr addrspace(8) %buf, i32 %len) { +; CHECK-LABEL: define float @sum_duplicate_preds +; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], i32 [[LEN:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[SUM_PREV:%.*]] = phi float [ [[SUM:%.*]], [[LOOP]] ], [ [[SUM]], [[LOOP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ], [ [[I_NEXT]], [[LOOP]] ] +; CHECK-NEXT: [[PTR_PREV_OFF:%.*]] = phi i32 [ [[PTR:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ], [ [[PTR]], [[LOOP]] ] +; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 [[PTR_PREV_OFF]], i32 0, i32 0) +; CHECK-NEXT: [[SUM]] = fadd float [[SUM_PREV]], [[VAL]] +; CHECK-NEXT: [[PTR]] = add i32 [[PTR_PREV_OFF]], 4 +; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1 +; CHECK-NEXT: [[TEST:%.*]] = icmp ult i32 [[I_NEXT]], [[LEN]] +; CHECK-NEXT: [[TEST_EXT:%.*]] = zext i1 [[TEST]] to i32 +; CHECK-NEXT: switch i32 [[TEST_EXT]], label [[LOOP]] [ +; CHECK-NEXT: i32 1, label [[LOOP]] +; CHECK-NEXT: i32 0, label [[EXIT:%.*]] +; CHECK-NEXT: ] +; CHECK: exit: +; CHECK-NEXT: ret float [[SUM]] +; +entry: + %start = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) + br label %loop +loop: + %sum.prev = phi float [ %sum, %loop ], [ %sum, %loop ], [ 0.0, %entry ] + %ptr.prev = phi ptr addrspace(7) [ %ptr, %loop ], [ %start, %entry ], [ %ptr, %loop ] + %i = phi i32 [ %i.next, %loop ], [ 0, %entry ], [ %i.next, %loop ] + + %val = load float, ptr addrspace(7) %ptr.prev + %sum = fadd float %sum.prev, %val + + %ptr = getelementptr float, ptr addrspace(7) %ptr.prev, i32 1 + %i.next = add i32 %i, 1 + %test = icmp ult i32 %i.next, %len + %test.ext = zext i1 %test to i32 + switch i32 %test.ext, label %loop [ + i32 1, label %loop + i32 0, label %exit + ] +exit: + ret float %sum +} + +;; And similirly check the "might not be uniform" case. +define float @sum_integer_ops_duplicate_preds(ptr addrspace(8) %buf, i32 %len) { +; CHECK-LABEL: define float @sum_integer_ops_duplicate_preds +; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], i32 [[LEN:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[SUM_PREV:%.*]] = phi float [ [[SUM:%.*]], [[LOOP]] ], [ [[SUM]], [[LOOP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ], [ [[I_NEXT]], [[LOOP]] ] +; CHECK-NEXT: [[PTR_PREV_RSRC:%.*]] = phi ptr addrspace(8) [ [[PTR_RSRC:%.*]], [[LOOP]] ], [ [[BUF]], [[ENTRY]] ], [ [[PTR_RSRC]], [[LOOP]] ] +; CHECK-NEXT: [[PTR_PREV_OFF:%.*]] = phi i32 [ [[PTR_OFF:%.*]], [[LOOP]] ], [ 0, [[ENTRY]] ], [ [[PTR_OFF]], [[LOOP]] ] +; CHECK-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[PTR_PREV_RSRC]], i32 [[PTR_PREV_OFF]], i32 0, i32 0) +; CHECK-NEXT: [[SUM]] = fadd float [[SUM_PREV]], [[VAL]] +; CHECK-NEXT: [[PTR_PREV_INT_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[PTR_PREV_RSRC]] to i160 +; CHECK-NEXT: [[TMP0:%.*]] = shl nuw i160 [[PTR_PREV_INT_RSRC]], 32 +; CHECK-NEXT: [[PTR_PREV_INT_OFF:%.*]] = zext i32 [[PTR_PREV_OFF]] to i160 +; CHECK-NEXT: [[PTR_PREV_INT:%.*]] = or i160 [[TMP0]], [[PTR_PREV_INT_OFF]] +; CHECK-NEXT: [[PTR_INT:%.*]] = add i160 [[PTR_PREV_INT]], 4 +; CHECK-NEXT: [[TMP1:%.*]] = lshr i160 [[PTR_INT]], 32 +; CHECK-NEXT: [[TMP2:%.*]] = trunc i160 [[TMP1]] to i128 +; CHECK-NEXT: [[PTR_RSRC]] = inttoptr i128 [[TMP2]] to ptr addrspace(8) +; CHECK-NEXT: [[PTR_OFF]] = trunc i160 [[PTR_INT]] to i32 +; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1 +; CHECK-NEXT: [[TEST:%.*]] = icmp ult i32 [[I_NEXT]], [[LEN]] +; CHECK-NEXT: [[TEST_EXT:%.*]] = zext i1 [[TEST]] to i32 +; CHECK-NEXT: switch i32 [[TEST_EXT]], label [[LOOP]] [ +; CHECK-NEXT: i32 1, label [[LOOP]] +; CHECK-NEXT: i32 0, label [[EXIT:%.*]] +; CHECK-NEXT: ] +; CHECK: exit: +; CHECK-NEXT: ret float [[SUM]] +; +entry: + %start = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) + br label %loop +loop: + %sum.prev = phi float [ %sum, %loop ], [ %sum, %loop ], [ 0.0, %entry ] + %ptr.prev = phi ptr addrspace(7) [ %ptr, %loop ], [ %start, %entry ], [ %ptr, %loop ] + %i = phi i32 [ %i.next, %loop ], [ 0, %entry ], [ %i.next, %loop ] + + %val = load float, ptr addrspace(7) %ptr.prev + %sum = fadd float %sum.prev, %val + + %ptr.prev.int = ptrtoint ptr addrspace(7) %ptr.prev to i160 + %ptr.int = add i160 %ptr.prev.int, 4 + %ptr = inttoptr i160 %ptr.int to ptr addrspace(7) + %i.next = add i32 %i, 1 + %test = icmp ult i32 %i.next, %len + %test.ext = zext i1 %test to i32 + switch i32 %test.ext, label %loop [ + i32 1, label %loop + i32 0, label %exit + ] +exit: + ret float %sum +} diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memops.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memops.ll new file mode 100644 index 000000000000..66c68f7cc731 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memops.ll @@ -0,0 +1,189 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 +; RUN: opt -S -mcpu=gfx900 -amdgpu-lower-buffer-fat-pointers < %s | FileCheck %s +; RUN: opt -S -mcpu=gfx900 -passes=amdgpu-lower-buffer-fat-pointers < %s | FileCheck %s + +target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8" +target triple = "amdgcn--" + +define void @loads(ptr addrspace(8) %buf) { +; CHECK-LABEL: define void @loads +; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[SCALAR:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0) +; CHECK-NEXT: [[VEC2:%.*]] = call <2 x float> @llvm.amdgcn.raw.ptr.buffer.load.v2f32(ptr addrspace(8) align 8 [[BUF]], i32 16, i32 0, i32 0) +; CHECK-NEXT: [[VEC4:%.*]] = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0) +; CHECK-NEXT: [[NONTEMPORAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 2), !nontemporal [[META0:![0-9]+]] +; CHECK-NEXT: [[INVARIANT:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0), !invariant.load [[META1:![0-9]+]] +; CHECK-NEXT: [[NONTEMPORAL_INVARIANT:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0), !invariant.load [[META1]], !nontemporal [[META0]] +; CHECK-NEXT: [[VOLATILE:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648) +; CHECK-NEXT: [[VOLATILE_NONTEMPORAL:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483646), !nontemporal [[META0]] +; CHECK-NEXT: fence syncscope("wavefront") release +; CHECK-NEXT: [[ATOMIC:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483647) +; CHECK-NEXT: fence syncscope("wavefront") acquire +; CHECK-NEXT: [[ATOMIC_MONOTONIC:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 1) +; CHECK-NEXT: [[ATOMIC_ACQUIRE:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 1) +; CHECK-NEXT: fence acquire +; CHECK-NEXT: ret void +; + %base = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) + %p = getelementptr float, ptr addrspace(7) %base, i32 4 + + %scalar = load float, ptr addrspace(7) %p, align 4 + %vec2 = load <2 x float>, ptr addrspace(7) %p, align 8 + %vec4 = load <4 x float>, ptr addrspace(7) %p, align 16 + + %nontemporal = load float, ptr addrspace(7) %p, !nontemporal !0 + %invariant = load float, ptr addrspace(7) %p, !invariant.load !1 + %nontemporal.invariant = load float, ptr addrspace(7) %p, !nontemporal !0, !invariant.load !1 + + %volatile = load volatile float, ptr addrspace(7) %p + %volatile.nontemporal = load volatile float, ptr addrspace(7) %p, !nontemporal !0 + + %atomic = load atomic volatile float, ptr addrspace(7) %p syncscope("wavefront") seq_cst, align 4 + %atomic.monotonic = load atomic float, ptr addrspace(7) %p syncscope("wavefront") monotonic, align 4 + %atomic.acquire = load atomic float, ptr addrspace(7) %p acquire, align 4 + + ret void +} + +define void @stores(ptr addrspace(8) %buf, float %f, <4 x float> %f4) { +; CHECK-LABEL: define void @stores +; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], float [[F:%.*]], <4 x float> [[F4:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0) +; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> [[F4]], ptr addrspace(8) align 16 [[BUF]], i32 16, i32 0, i32 0) +; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 2), !nontemporal [[META0]] +; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648) +; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483646), !nontemporal [[META0]] +; CHECK-NEXT: fence syncscope("wavefront") release +; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483647) +; CHECK-NEXT: fence syncscope("wavefront") acquire +; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 1) +; CHECK-NEXT: fence release +; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 1) +; CHECK-NEXT: ret void +; + %base = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) + %p = getelementptr float, ptr addrspace(7) %base, i32 4 + + store float %f, ptr addrspace(7) %p, align 4 + store <4 x float> %f4, ptr addrspace(7) %p, align 16 + + store float %f, ptr addrspace(7) %p, !nontemporal !0 + + store volatile float %f, ptr addrspace(7) %p + store volatile float %f, ptr addrspace(7) %p, !nontemporal !0 + + store atomic volatile float %f, ptr addrspace(7) %p syncscope("wavefront") seq_cst, align 4 + store atomic float %f, ptr addrspace(7) %p syncscope("wavefront") monotonic, align 4 + store atomic float %f, ptr addrspace(7) %p release, align 4 + + ret void +} + +define void @atomicrmw(ptr addrspace(8) %buf, float %f, i32 %i) { +; CHECK-LABEL: define void @atomicrmw +; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], float [[F:%.*]], i32 [[I:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: fence syncscope("wavefront") release +; CHECK-NEXT: [[XCHG:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.swap.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0) +; CHECK-NEXT: fence syncscope("wavefront") acquire +; CHECK-NEXT: fence syncscope("wavefront") release +; CHECK-NEXT: [[ADD:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0) +; CHECK-NEXT: fence syncscope("wavefront") acquire +; CHECK-NEXT: fence syncscope("wavefront") release +; CHECK-NEXT: [[SUB:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.sub.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0) +; CHECK-NEXT: fence syncscope("wavefront") acquire +; CHECK-NEXT: fence syncscope("wavefront") release +; CHECK-NEXT: [[AND:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.and.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0) +; CHECK-NEXT: fence syncscope("wavefront") acquire +; CHECK-NEXT: fence syncscope("wavefront") release +; CHECK-NEXT: [[OR:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.or.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0) +; CHECK-NEXT: fence syncscope("wavefront") acquire +; CHECK-NEXT: fence syncscope("wavefront") release +; CHECK-NEXT: [[XOR:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.xor.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0) +; CHECK-NEXT: fence syncscope("wavefront") acquire +; CHECK-NEXT: fence syncscope("wavefront") release +; CHECK-NEXT: [[MIN:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.smin.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0) +; CHECK-NEXT: fence syncscope("wavefront") acquire +; CHECK-NEXT: fence syncscope("wavefront") release +; CHECK-NEXT: [[MAX:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.smax.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0) +; CHECK-NEXT: fence syncscope("wavefront") acquire +; CHECK-NEXT: fence syncscope("wavefront") release +; CHECK-NEXT: [[UMIN:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.umin.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0) +; CHECK-NEXT: fence syncscope("wavefront") acquire +; CHECK-NEXT: fence syncscope("wavefront") release +; CHECK-NEXT: [[UMAX:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.umax.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0) +; CHECK-NEXT: fence syncscope("wavefront") acquire +; CHECK-NEXT: fence syncscope("wavefront") release +; CHECK-NEXT: [[FADD:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0) +; CHECK-NEXT: fence syncscope("wavefront") acquire +; CHECK-NEXT: fence syncscope("wavefront") release +; CHECK-NEXT: [[FMAX:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0) +; CHECK-NEXT: fence syncscope("wavefront") acquire +; CHECK-NEXT: fence syncscope("wavefront") release +; CHECK-NEXT: [[FMIN:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f32(float [[F]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0) +; CHECK-NEXT: fence syncscope("wavefront") acquire +; CHECK-NEXT: fence syncscope("wavefront") release +; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 [[I]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0) +; CHECK-NEXT: fence syncscope("wavefront") acquire +; CHECK-NEXT: ret void +; + %base = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) + %p = getelementptr float, ptr addrspace(7) %base, i32 4 + + ; Fence insertion is tested by loads and stores + %xchg = atomicrmw xchg ptr addrspace(7) %p, i32 %i syncscope("wavefront") seq_cst, align 4 + %add = atomicrmw add ptr addrspace(7) %p, i32 %i syncscope("wavefront") seq_cst, align 4 + %sub = atomicrmw sub ptr addrspace(7) %p, i32 %i syncscope("wavefront") seq_cst, align 4 + %and = atomicrmw and ptr addrspace(7) %p, i32 %i syncscope("wavefront") seq_cst, align 4 + %or = atomicrmw or ptr addrspace(7) %p, i32 %i syncscope("wavefront") seq_cst, align 4 + %xor = atomicrmw xor ptr addrspace(7) %p, i32 %i syncscope("wavefront") seq_cst, align 4 + %min = atomicrmw min ptr addrspace(7) %p, i32 %i syncscope("wavefront") seq_cst, align 4 + %max = atomicrmw max ptr addrspace(7) %p, i32 %i syncscope("wavefront") seq_cst, align 4 + %umin = atomicrmw umin ptr addrspace(7) %p, i32 %i syncscope("wavefront") seq_cst, align 4 + %umax = atomicrmw umax ptr addrspace(7) %p, i32 %i syncscope("wavefront") seq_cst, align 4 + + %fadd = atomicrmw fadd ptr addrspace(7) %p, float %f syncscope("wavefront") seq_cst, align 4 + %fmax = atomicrmw fmax ptr addrspace(7) %p, float %f syncscope("wavefront") seq_cst, align 4 + %fmin = atomicrmw fmin ptr addrspace(7) %p, float %f syncscope("wavefront") seq_cst, align 4 + + ; Check a no-return atomic + atomicrmw add ptr addrspace(7) %p, i32 %i syncscope("wavefront") seq_cst, align 4 + + ret void +} + +define {i32, i1} @cmpxchg(ptr addrspace(8) %buf, i32 %wanted, i32 %new) { +; CHECK-LABEL: define { i32, i1 } @cmpxchg +; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], i32 [[WANTED:%.*]], i32 [[NEW:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: fence syncscope("wavefront") release +; CHECK-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 [[NEW]], i32 [[WANTED]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 -2147483648) +; CHECK-NEXT: fence syncscope("wavefront") acquire +; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { i32, i1 } poison, i32 [[RET]], 0 +; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[RET]], [[WANTED]] +; CHECK-NEXT: [[TMP3:%.*]] = insertvalue { i32, i1 } [[TMP1]], i1 [[TMP2]], 1 +; CHECK-NEXT: ret { i32, i1 } [[TMP3]] +; + %base = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) + %p = getelementptr i32, ptr addrspace(7) %base, i32 4 + + %ret = cmpxchg volatile ptr addrspace(7) %p, i32 %wanted, i32 %new syncscope("wavefront") acq_rel monotonic, align 4 + ret {i32, i1} %ret +} + +define {i32, i1} @cmpxchg_weak(ptr addrspace(8) %buf, i32 %wanted, i32 %new) { +; CHECK-LABEL: define { i32, i1 } @cmpxchg_weak +; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], i32 [[WANTED:%.*]], i32 [[NEW:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: fence syncscope("wavefront") release +; CHECK-NEXT: [[RET:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.cmpswap.i32(i32 [[NEW]], i32 [[WANTED]], ptr addrspace(8) align 4 [[BUF]], i32 16, i32 0, i32 0) +; CHECK-NEXT: fence syncscope("wavefront") acquire +; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { i32, i1 } poison, i32 [[RET]], 0 +; CHECK-NEXT: ret { i32, i1 } [[TMP1]] +; + %base = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) + %p = getelementptr i32, ptr addrspace(7) %base, i32 4 + + %ret = cmpxchg weak ptr addrspace(7) %p, i32 %wanted, i32 %new syncscope("wavefront") acq_rel monotonic, align 4 + ret {i32, i1} %ret +} + +!0 = ! { i32 1 } +!1 = ! { } diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-p7-in-memory.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-p7-in-memory.ll new file mode 100644 index 000000000000..5ff08a5e8227 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-p7-in-memory.ll @@ -0,0 +1,154 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 +; RUN: opt -S -mcpu=gfx900 -amdgpu-lower-buffer-fat-pointers < %s | FileCheck %s +; RUN: opt -S -mcpu=gfx900 -passes=amdgpu-lower-buffer-fat-pointers < %s | FileCheck %s + +target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8" +target triple = "amdgcn--" + +define void @scalar_copy(ptr %a, ptr %b) { +; CHECK-LABEL: define void @scalar_copy +; CHECK-SAME: (ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[X:%.*]] = load i160, ptr [[A]], align 32 +; CHECK-NEXT: [[TMP1:%.*]] = lshr i160 [[X]], 32 +; CHECK-NEXT: [[TMP2:%.*]] = trunc i160 [[TMP1]] to i128 +; CHECK-NEXT: [[X_PTR_RSRC:%.*]] = inttoptr i128 [[TMP2]] to ptr addrspace(8) +; CHECK-NEXT: [[X_PTR_OFF:%.*]] = trunc i160 [[X]] to i32 +; CHECK-NEXT: [[B1:%.*]] = getelementptr i160, ptr [[B]], i64 1 +; CHECK-NEXT: [[X_PTR_INT_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[X_PTR_RSRC]] to i160 +; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i160 [[X_PTR_INT_RSRC]], 32 +; CHECK-NEXT: [[X_PTR_INT_OFF:%.*]] = zext i32 [[X_PTR_OFF]] to i160 +; CHECK-NEXT: [[X_PTR_INT:%.*]] = or i160 [[TMP3]], [[X_PTR_INT_OFF]] +; CHECK-NEXT: store i160 [[X_PTR_INT]], ptr [[B1]], align 32 +; CHECK-NEXT: ret void +; + %x = load ptr addrspace(7), ptr %a + %b1 = getelementptr ptr addrspace(7), ptr %b, i64 1 + store ptr addrspace(7) %x, ptr %b1 + ret void +} + +define void @vector_copy(ptr %a, ptr %b) { +; CHECK-LABEL: define void @vector_copy +; CHECK-SAME: (ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[X:%.*]] = load <4 x i160>, ptr [[A]], align 128 +; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i160> [[X]], +; CHECK-NEXT: [[TMP2:%.*]] = trunc <4 x i160> [[TMP1]] to <4 x i128> +; CHECK-NEXT: [[X_PTR_RSRC:%.*]] = inttoptr <4 x i128> [[TMP2]] to <4 x ptr addrspace(8)> +; CHECK-NEXT: [[X_PTR_OFF:%.*]] = trunc <4 x i160> [[X]] to <4 x i32> +; CHECK-NEXT: [[B1:%.*]] = getelementptr <4 x i160>, ptr [[B]], i64 2 +; CHECK-NEXT: [[X_PTR_INT_RSRC:%.*]] = ptrtoint <4 x ptr addrspace(8)> [[X_PTR_RSRC]] to <4 x i160> +; CHECK-NEXT: [[TMP3:%.*]] = shl nuw <4 x i160> [[X_PTR_INT_RSRC]], +; CHECK-NEXT: [[X_PTR_INT_OFF:%.*]] = zext <4 x i32> [[X_PTR_OFF]] to <4 x i160> +; CHECK-NEXT: [[X_PTR_INT:%.*]] = or <4 x i160> [[TMP3]], [[X_PTR_INT_OFF]] +; CHECK-NEXT: store <4 x i160> [[X_PTR_INT]], ptr [[B1]], align 128 +; CHECK-NEXT: ret void +; + %x = load <4 x ptr addrspace(7)>, ptr %a + %b1 = getelementptr <4 x ptr addrspace(7)>, ptr %b, i64 2 + store <4 x ptr addrspace(7)> %x, ptr %b1 + ret void +} + +define void @alloca(ptr %a, ptr %b) { +; CHECK-LABEL: define void @alloca +; CHECK-SAME: (ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ALLOCA:%.*]] = alloca [5 x i160], align 32, addrspace(5) +; CHECK-NEXT: [[X:%.*]] = load i160, ptr [[A]], align 32 +; CHECK-NEXT: [[TMP1:%.*]] = lshr i160 [[X]], 32 +; CHECK-NEXT: [[TMP2:%.*]] = trunc i160 [[TMP1]] to i128 +; CHECK-NEXT: [[X_PTR_RSRC:%.*]] = inttoptr i128 [[TMP2]] to ptr addrspace(8) +; CHECK-NEXT: [[X_PTR_OFF:%.*]] = trunc i160 [[X]] to i32 +; CHECK-NEXT: [[L:%.*]] = getelementptr i160, ptr addrspace(5) [[ALLOCA]], i32 1 +; CHECK-NEXT: [[X_PTR_INT_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[X_PTR_RSRC]] to i160 +; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i160 [[X_PTR_INT_RSRC]], 32 +; CHECK-NEXT: [[X_PTR_INT_OFF:%.*]] = zext i32 [[X_PTR_OFF]] to i160 +; CHECK-NEXT: [[X_PTR_INT:%.*]] = or i160 [[TMP3]], [[X_PTR_INT_OFF]] +; CHECK-NEXT: store i160 [[X_PTR_INT]], ptr addrspace(5) [[L]], align 32 +; CHECK-NEXT: [[Y:%.*]] = load i160, ptr addrspace(5) [[L]], align 32 +; CHECK-NEXT: [[TMP4:%.*]] = lshr i160 [[Y]], 32 +; CHECK-NEXT: [[TMP5:%.*]] = trunc i160 [[TMP4]] to i128 +; CHECK-NEXT: [[Y_PTR_RSRC:%.*]] = inttoptr i128 [[TMP5]] to ptr addrspace(8) +; CHECK-NEXT: [[Y_PTR_OFF:%.*]] = trunc i160 [[Y]] to i32 +; CHECK-NEXT: [[Y_PTR_INT_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[Y_PTR_RSRC]] to i160 +; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i160 [[Y_PTR_INT_RSRC]], 32 +; CHECK-NEXT: [[Y_PTR_INT_OFF:%.*]] = zext i32 [[Y_PTR_OFF]] to i160 +; CHECK-NEXT: [[Y_PTR_INT:%.*]] = or i160 [[TMP6]], [[Y_PTR_INT_OFF]] +; CHECK-NEXT: store i160 [[Y_PTR_INT]], ptr [[B]], align 32 +; CHECK-NEXT: ret void +; + %alloca = alloca [5 x ptr addrspace(7)], addrspace(5) + %x = load ptr addrspace(7), ptr %a + %l = getelementptr ptr addrspace(7), ptr addrspace(5) %alloca, i32 1 + store ptr addrspace(7) %x, ptr addrspace(5) %l + %y = load ptr addrspace(7), ptr addrspace(5) %l + store ptr addrspace(7) %y, ptr %b + ret void +} + +define void @complex_copy(ptr %a, ptr %b) { +; CHECK-LABEL: define void @complex_copy +; CHECK-SAME: (ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[X:%.*]] = load { [2 x i160], i32, i160 }, ptr [[A]], align 32 +; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { [2 x i160], i32, i160 } [[X]], 0 +; CHECK-NEXT: [[TMP2:%.*]] = extractvalue [2 x i160] [[TMP1]], 0 +; CHECK-NEXT: [[TMP3:%.*]] = lshr i160 [[TMP2]], 32 +; CHECK-NEXT: [[TMP4:%.*]] = trunc i160 [[TMP3]] to i128 +; CHECK-NEXT: [[X_0_0_PTR_RSRC:%.*]] = inttoptr i128 [[TMP4]] to ptr addrspace(8) +; CHECK-NEXT: [[X_0_0_PTR_OFF:%.*]] = trunc i160 [[TMP2]] to i32 +; CHECK-NEXT: [[TMP5:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[X_0_0_PTR_RSRC]], 0 +; CHECK-NEXT: [[X_0_0_PTR:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP5]], i32 [[X_0_0_PTR_OFF]], 1 +; CHECK-NEXT: [[TMP6:%.*]] = insertvalue [2 x { ptr addrspace(8), i32 }] poison, { ptr addrspace(8), i32 } [[X_0_0_PTR]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = extractvalue [2 x i160] [[TMP1]], 1 +; CHECK-NEXT: [[TMP8:%.*]] = lshr i160 [[TMP7]], 32 +; CHECK-NEXT: [[TMP9:%.*]] = trunc i160 [[TMP8]] to i128 +; CHECK-NEXT: [[X_0_1_PTR_RSRC:%.*]] = inttoptr i128 [[TMP9]] to ptr addrspace(8) +; CHECK-NEXT: [[X_0_1_PTR_OFF:%.*]] = trunc i160 [[TMP7]] to i32 +; CHECK-NEXT: [[TMP10:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[X_0_1_PTR_RSRC]], 0 +; CHECK-NEXT: [[X_0_1_PTR:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP10]], i32 [[X_0_1_PTR_OFF]], 1 +; CHECK-NEXT: [[TMP11:%.*]] = insertvalue [2 x { ptr addrspace(8), i32 }] [[TMP6]], { ptr addrspace(8), i32 } [[X_0_1_PTR]], 1 +; CHECK-NEXT: [[TMP12:%.*]] = insertvalue { [2 x { ptr addrspace(8), i32 }], i32, { ptr addrspace(8), i32 } } poison, [2 x { ptr addrspace(8), i32 }] [[TMP11]], 0 +; CHECK-NEXT: [[TMP13:%.*]] = extractvalue { [2 x i160], i32, i160 } [[X]], 1 +; CHECK-NEXT: [[TMP14:%.*]] = insertvalue { [2 x { ptr addrspace(8), i32 }], i32, { ptr addrspace(8), i32 } } [[TMP12]], i32 [[TMP13]], 1 +; CHECK-NEXT: [[TMP15:%.*]] = extractvalue { [2 x i160], i32, i160 } [[X]], 2 +; CHECK-NEXT: [[TMP16:%.*]] = lshr i160 [[TMP15]], 32 +; CHECK-NEXT: [[TMP17:%.*]] = trunc i160 [[TMP16]] to i128 +; CHECK-NEXT: [[X_2_PTR_RSRC:%.*]] = inttoptr i128 [[TMP17]] to ptr addrspace(8) +; CHECK-NEXT: [[X_2_PTR_OFF:%.*]] = trunc i160 [[TMP15]] to i32 +; CHECK-NEXT: [[TMP18:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[X_2_PTR_RSRC]], 0 +; CHECK-NEXT: [[X_2_PTR:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP18]], i32 [[X_2_PTR_OFF]], 1 +; CHECK-NEXT: [[TMP19:%.*]] = insertvalue { [2 x { ptr addrspace(8), i32 }], i32, { ptr addrspace(8), i32 } } [[TMP14]], { ptr addrspace(8), i32 } [[X_2_PTR]], 2 +; CHECK-NEXT: [[TMP20:%.*]] = extractvalue { [2 x { ptr addrspace(8), i32 }], i32, { ptr addrspace(8), i32 } } [[TMP19]], 0 +; CHECK-NEXT: [[TMP21:%.*]] = extractvalue [2 x { ptr addrspace(8), i32 }] [[TMP20]], 0 +; CHECK-NEXT: [[DOTRSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[TMP21]], 0 +; CHECK-NEXT: [[DOTOFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[TMP21]], 1 +; CHECK-NEXT: [[DOT0_0_INT_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[DOTRSRC]] to i160 +; CHECK-NEXT: [[TMP22:%.*]] = shl nuw i160 [[DOT0_0_INT_RSRC]], 32 +; CHECK-NEXT: [[DOT0_0_INT_OFF:%.*]] = zext i32 [[DOTOFF]] to i160 +; CHECK-NEXT: [[DOT0_0_INT:%.*]] = or i160 [[TMP22]], [[DOT0_0_INT_OFF]] +; CHECK-NEXT: [[TMP23:%.*]] = insertvalue [2 x i160] poison, i160 [[DOT0_0_INT]], 0 +; CHECK-NEXT: [[TMP24:%.*]] = extractvalue [2 x { ptr addrspace(8), i32 }] [[TMP20]], 1 +; CHECK-NEXT: [[DOTRSRC1:%.*]] = extractvalue { ptr addrspace(8), i32 } [[TMP24]], 0 +; CHECK-NEXT: [[DOTOFF2:%.*]] = extractvalue { ptr addrspace(8), i32 } [[TMP24]], 1 +; CHECK-NEXT: [[DOT0_1_INT_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[DOTRSRC1]] to i160 +; CHECK-NEXT: [[TMP25:%.*]] = shl nuw i160 [[DOT0_1_INT_RSRC]], 32 +; CHECK-NEXT: [[DOT0_1_INT_OFF:%.*]] = zext i32 [[DOTOFF2]] to i160 +; CHECK-NEXT: [[DOT0_1_INT:%.*]] = or i160 [[TMP25]], [[DOT0_1_INT_OFF]] +; CHECK-NEXT: [[TMP26:%.*]] = insertvalue [2 x i160] [[TMP23]], i160 [[DOT0_1_INT]], 1 +; CHECK-NEXT: [[TMP27:%.*]] = insertvalue { [2 x i160], i32, i160 } poison, [2 x i160] [[TMP26]], 0 +; CHECK-NEXT: [[TMP28:%.*]] = extractvalue { [2 x { ptr addrspace(8), i32 }], i32, { ptr addrspace(8), i32 } } [[TMP19]], 1 +; CHECK-NEXT: [[TMP29:%.*]] = insertvalue { [2 x i160], i32, i160 } [[TMP27]], i32 [[TMP28]], 1 +; CHECK-NEXT: [[TMP30:%.*]] = extractvalue { [2 x { ptr addrspace(8), i32 }], i32, { ptr addrspace(8), i32 } } [[TMP19]], 2 +; CHECK-NEXT: [[DOTRSRC3:%.*]] = extractvalue { ptr addrspace(8), i32 } [[TMP30]], 0 +; CHECK-NEXT: [[DOTOFF4:%.*]] = extractvalue { ptr addrspace(8), i32 } [[TMP30]], 1 +; CHECK-NEXT: [[DOT2_INT_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[DOTRSRC3]] to i160 +; CHECK-NEXT: [[TMP31:%.*]] = shl nuw i160 [[DOT2_INT_RSRC]], 32 +; CHECK-NEXT: [[DOT2_INT_OFF:%.*]] = zext i32 [[DOTOFF4]] to i160 +; CHECK-NEXT: [[DOT2_INT:%.*]] = or i160 [[TMP31]], [[DOT2_INT_OFF]] +; CHECK-NEXT: [[TMP32:%.*]] = insertvalue { [2 x i160], i32, i160 } [[TMP29]], i160 [[DOT2_INT]], 2 +; CHECK-NEXT: store { [2 x i160], i32, i160 } [[TMP32]], ptr [[B]], align 32 +; CHECK-NEXT: ret void +; + %x = load {[2 x ptr addrspace(7)], i32, ptr addrspace(7)}, ptr %a + store {[2 x ptr addrspace(7)], i32, ptr addrspace(7)} %x, ptr %b + ret void +} diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-pointer-ops.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-pointer-ops.ll new file mode 100644 index 000000000000..85cc810988a7 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-pointer-ops.ll @@ -0,0 +1,480 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 +; RUN: opt -S -mcpu=gfx900 -amdgpu-lower-buffer-fat-pointers < %s | FileCheck %s +; RUN: opt -S -mcpu=gfx900 -passes=amdgpu-lower-buffer-fat-pointers < %s | FileCheck %s + +target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8" +target triple = "amdgcn--" + +define ptr addrspace(7) @gep(ptr addrspace(7) %in, i32 %idx) { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @gep +; CHECK-SAME: ({ ptr addrspace(8), i32 } [[IN:%.*]], i32 [[IDX:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[IN_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[IN]], 0 +; CHECK-NEXT: [[IN_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[IN]], 1 +; CHECK-NEXT: [[TMP1:%.*]] = mul nuw nsw i32 [[IDX]], 40 +; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i32 [[TMP1]], 32 +; CHECK-NEXT: [[RET:%.*]] = add i32 [[IN_OFF]], [[TMP2]] +; CHECK-NEXT: [[TMP3:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[IN_RSRC]], 0 +; CHECK-NEXT: [[TMP4:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP3]], i32 [[RET]], 1 +; CHECK-NEXT: ret { ptr addrspace(8), i32 } [[TMP4]] +; + %ret = getelementptr inbounds {i32, [4 x ptr]}, ptr addrspace(7) %in, i32 %idx, i32 1, i32 3 + ret ptr addrspace(7) %ret +} + +define <2 x ptr addrspace(7)> @gep_vectors(<2 x ptr addrspace(7)> %in, <2 x i32> %idx) { +; CHECK-LABEL: define { <2 x ptr addrspace(8)>, <2 x i32> } @gep_vectors +; CHECK-SAME: ({ <2 x ptr addrspace(8)>, <2 x i32> } [[IN:%.*]], <2 x i32> [[IDX:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[IN_RSRC:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[IN]], 0 +; CHECK-NEXT: [[IN_OFF:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[IN]], 1 +; CHECK-NEXT: [[TMP1:%.*]] = mul nuw nsw <2 x i32> [[IDX]], +; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw <2 x i32> [[TMP1]], +; CHECK-NEXT: [[RET:%.*]] = add <2 x i32> [[IN_OFF]], [[TMP2]] +; CHECK-NEXT: [[TMP3:%.*]] = insertvalue { <2 x ptr addrspace(8)>, <2 x i32> } poison, <2 x ptr addrspace(8)> [[IN_RSRC]], 0 +; CHECK-NEXT: [[TMP4:%.*]] = insertvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[TMP3]], <2 x i32> [[RET]], 1 +; CHECK-NEXT: ret { <2 x ptr addrspace(8)>, <2 x i32> } [[TMP4]] +; + %ret = getelementptr inbounds {i32, [4 x ptr]}, <2 x ptr addrspace(7)> %in, <2 x i32> %idx, i32 1, i32 3 + ret <2 x ptr addrspace(7)> %ret +} + +define <2 x ptr addrspace(7)> @gep_vector_scalar(<2 x ptr addrspace(7)> %in, i64 %idx) { +; CHECK-LABEL: define { <2 x ptr addrspace(8)>, <2 x i32> } @gep_vector_scalar +; CHECK-SAME: ({ <2 x ptr addrspace(8)>, <2 x i32> } [[IN:%.*]], i64 [[IDX:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[IN_RSRC:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[IN]], 0 +; CHECK-NEXT: [[IN_OFF:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[IN]], 1 +; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[IDX]], i64 0 +; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i64> [[DOTSPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer +; CHECK-NEXT: [[TMP1:%.*]] = trunc <2 x i64> [[DOTSPLAT]] to <2 x i32> +; CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw <2 x i32> [[TMP1]], +; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw <2 x i32> [[TMP2]], +; CHECK-NEXT: [[RET:%.*]] = add <2 x i32> [[IN_OFF]], [[TMP3]] +; CHECK-NEXT: [[TMP4:%.*]] = insertvalue { <2 x ptr addrspace(8)>, <2 x i32> } poison, <2 x ptr addrspace(8)> [[IN_RSRC]], 0 +; CHECK-NEXT: [[TMP5:%.*]] = insertvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[TMP4]], <2 x i32> [[RET]], 1 +; CHECK-NEXT: ret { <2 x ptr addrspace(8)>, <2 x i32> } [[TMP5]] +; + %ret = getelementptr inbounds {i32, [4 x ptr]}, <2 x ptr addrspace(7)> %in, i64 %idx, i32 1, i32 3 + ret <2 x ptr addrspace(7)> %ret +} + +define ptr addrspace(7) @simple_gep(ptr addrspace(7) %ptr, i32 %off) { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @simple_gep +; CHECK-SAME: ({ ptr addrspace(8), i32 } [[PTR:%.*]], i32 [[OFF:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0 +; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1 +; CHECK-NEXT: [[TMP1:%.*]] = shl i32 [[OFF]], 2 +; CHECK-NEXT: [[RET:%.*]] = add i32 [[PTR_OFF]], [[TMP1]] +; CHECK-NEXT: [[TMP2:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[PTR_RSRC]], 0 +; CHECK-NEXT: [[TMP3:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP2]], i32 [[RET]], 1 +; CHECK-NEXT: ret { ptr addrspace(8), i32 } [[TMP3]] +; + %ret = getelementptr i32, ptr addrspace(7) %ptr, i32 %off + ret ptr addrspace(7) %ret +} + +define ptr addrspace(7) @simple_inbounds_gep(ptr addrspace(7) %ptr, i32 %off) { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @simple_inbounds_gep +; CHECK-SAME: ({ ptr addrspace(8), i32 } [[PTR:%.*]], i32 [[OFF:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0 +; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1 +; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i32 [[OFF]], 2 +; CHECK-NEXT: [[RET:%.*]] = add i32 [[PTR_OFF]], [[TMP1]] +; CHECK-NEXT: [[TMP2:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[PTR_RSRC]], 0 +; CHECK-NEXT: [[TMP3:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP2]], i32 [[RET]], 1 +; CHECK-NEXT: ret { ptr addrspace(8), i32 } [[TMP3]] +; + %ret = getelementptr inbounds i32, ptr addrspace(7) %ptr, i32 %off + ret ptr addrspace(7) %ret +} + +define ptr addrspace(7) @zero_gep(ptr addrspace(7) %ptr) { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @zero_gep +; CHECK-SAME: ({ ptr addrspace(8), i32 } [[PTR:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0 +; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1 +; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[PTR_RSRC]], 0 +; CHECK-NEXT: [[RET:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP1]], i32 [[PTR_OFF]], 1 +; CHECK-NEXT: ret { ptr addrspace(8), i32 } [[RET]] +; + %ret = getelementptr i8, ptr addrspace(7) %ptr, i32 0 + ret ptr addrspace(7) %ret +} + +define ptr addrspace(7) @zero_gep_goes_second(ptr addrspace(7) %v0, i32 %arg) { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @zero_gep_goes_second +; CHECK-SAME: ({ ptr addrspace(8), i32 } [[V0:%.*]], i32 [[ARG:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[V0_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[V0]], 0 +; CHECK-NEXT: [[V0_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[V0]], 1 +; CHECK-NEXT: [[V1:%.*]] = add i32 [[V0_OFF]], [[ARG]] +; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[V0_RSRC]], 0 +; CHECK-NEXT: [[V2:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP1]], i32 [[V1]], 1 +; CHECK-NEXT: ret { ptr addrspace(8), i32 } [[V2]] +; + %v1 = getelementptr i8, ptr addrspace(7) %v0, i32 %arg + %v2 = getelementptr i8, ptr addrspace(7) %v1, i32 0 + ret ptr addrspace(7) %v2 +} + +define ptr addrspace(7) @zero_gep_goes_first(ptr addrspace(7) %v0, i32 %arg) { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @zero_gep_goes_first +; CHECK-SAME: ({ ptr addrspace(8), i32 } [[V0:%.*]], i32 [[ARG:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[V0_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[V0]], 0 +; CHECK-NEXT: [[V0_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[V0]], 1 +; CHECK-NEXT: [[V2:%.*]] = add i32 [[V0_OFF]], [[ARG]] +; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[V0_RSRC]], 0 +; CHECK-NEXT: [[TMP2:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP1]], i32 [[V2]], 1 +; CHECK-NEXT: ret { ptr addrspace(8), i32 } [[TMP2]] +; + %v1 = getelementptr i8, ptr addrspace(7) %v0, i32 0 + %v2 = getelementptr i8, ptr addrspace(7) %v1, i32 %arg + ret ptr addrspace(7) %v2 +} + +define i160 @ptrtoint(ptr addrspace(7) %ptr) { +; CHECK-LABEL: define i160 @ptrtoint +; CHECK-SAME: ({ ptr addrspace(8), i32 } [[PTR:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0 +; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1 +; CHECK-NEXT: [[RET_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[PTR_RSRC]] to i160 +; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i160 [[RET_RSRC]], 32 +; CHECK-NEXT: [[RET_OFF:%.*]] = zext i32 [[PTR_OFF]] to i160 +; CHECK-NEXT: [[RET:%.*]] = or i160 [[TMP1]], [[RET_OFF]] +; CHECK-NEXT: ret i160 [[RET]] +; + %ret = ptrtoint ptr addrspace(7) %ptr to i160 + ret i160 %ret +} + +define <2 x i160> @ptrtoint_vec(<2 x ptr addrspace(7)> %ptr) { +; CHECK-LABEL: define <2 x i160> @ptrtoint_vec +; CHECK-SAME: ({ <2 x ptr addrspace(8)>, <2 x i32> } [[PTR:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[PTR]], 0 +; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[PTR]], 1 +; CHECK-NEXT: [[RET_RSRC:%.*]] = ptrtoint <2 x ptr addrspace(8)> [[PTR_RSRC]] to <2 x i160> +; CHECK-NEXT: [[TMP1:%.*]] = shl nuw <2 x i160> [[RET_RSRC]], +; CHECK-NEXT: [[RET_OFF:%.*]] = zext <2 x i32> [[PTR_OFF]] to <2 x i160> +; CHECK-NEXT: [[RET:%.*]] = or <2 x i160> [[TMP1]], [[RET_OFF]] +; CHECK-NEXT: ret <2 x i160> [[RET]] +; + %ret = ptrtoint <2 x ptr addrspace(7)> %ptr to <2 x i160> + ret <2 x i160> %ret +} + +define i256 @ptrtoint_long(ptr addrspace(7) %ptr) { +; CHECK-LABEL: define i256 @ptrtoint_long +; CHECK-SAME: ({ ptr addrspace(8), i32 } [[PTR:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0 +; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1 +; CHECK-NEXT: [[RET_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[PTR_RSRC]] to i256 +; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i256 [[RET_RSRC]], 32 +; CHECK-NEXT: [[RET_OFF:%.*]] = zext i32 [[PTR_OFF]] to i256 +; CHECK-NEXT: [[RET:%.*]] = or i256 [[TMP1]], [[RET_OFF]] +; CHECK-NEXT: ret i256 [[RET]] +; + %ret = ptrtoint ptr addrspace(7) %ptr to i256 + ret i256 %ret +} + +define i64 @ptrtoint_short(ptr addrspace(7) %ptr) { +; CHECK-LABEL: define i64 @ptrtoint_short +; CHECK-SAME: ({ ptr addrspace(8), i32 } [[PTR:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0 +; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1 +; CHECK-NEXT: [[RET_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[PTR_RSRC]] to i64 +; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[RET_RSRC]], 32 +; CHECK-NEXT: [[RET_OFF:%.*]] = zext i32 [[PTR_OFF]] to i64 +; CHECK-NEXT: [[RET:%.*]] = or i64 [[TMP1]], [[RET_OFF]] +; CHECK-NEXT: ret i64 [[RET]] +; + %ret = ptrtoint ptr addrspace(7) %ptr to i64 + ret i64 %ret +} + +define i32 @ptrtoint_offset(ptr addrspace(7) %ptr) { +; CHECK-LABEL: define i32 @ptrtoint_offset +; CHECK-SAME: ({ ptr addrspace(8), i32 } [[PTR:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0 +; CHECK-NEXT: [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1 +; CHECK-NEXT: [[RET:%.*]] = or i32 poison, [[PTR_OFF]] +; CHECK-NEXT: ret i32 [[RET]] +; + %ret = ptrtoint ptr addrspace(7) %ptr to i32 + ret i32 %ret +} + +define ptr addrspace(7) @inttoptr(i160 %v) { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @inttoptr +; CHECK-SAME: (i160 [[V:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[TMP1:%.*]] = lshr i160 [[V]], 32 +; CHECK-NEXT: [[TMP2:%.*]] = trunc i160 [[TMP1]] to i128 +; CHECK-NEXT: [[RET_RSRC:%.*]] = inttoptr i128 [[TMP2]] to ptr addrspace(8) +; CHECK-NEXT: [[RET_OFF:%.*]] = trunc i160 [[V]] to i32 +; CHECK-NEXT: [[TMP3:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[RET_RSRC]], 0 +; CHECK-NEXT: [[RET:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP3]], i32 [[RET_OFF]], 1 +; CHECK-NEXT: ret { ptr addrspace(8), i32 } [[RET]] +; + %ret = inttoptr i160 %v to ptr addrspace(7) + ret ptr addrspace(7) %ret +} + +define <2 x ptr addrspace(7)> @inttoptr_vec(<2 x i160> %v) { +; CHECK-LABEL: define { <2 x ptr addrspace(8)>, <2 x i32> } @inttoptr_vec +; CHECK-SAME: (<2 x i160> [[V:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[TMP1:%.*]] = lshr <2 x i160> [[V]], +; CHECK-NEXT: [[TMP2:%.*]] = trunc <2 x i160> [[TMP1]] to <2 x i128> +; CHECK-NEXT: [[RET_RSRC:%.*]] = inttoptr <2 x i128> [[TMP2]] to <2 x ptr addrspace(8)> +; CHECK-NEXT: [[RET_OFF:%.*]] = trunc <2 x i160> [[V]] to <2 x i32> +; CHECK-NEXT: [[TMP3:%.*]] = insertvalue { <2 x ptr addrspace(8)>, <2 x i32> } poison, <2 x ptr addrspace(8)> [[RET_RSRC]], 0 +; CHECK-NEXT: [[RET:%.*]] = insertvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[TMP3]], <2 x i32> [[RET_OFF]], 1 +; CHECK-NEXT: ret { <2 x ptr addrspace(8)>, <2 x i32> } [[RET]] +; + %ret = inttoptr <2 x i160> %v to <2 x ptr addrspace(7)> + ret <2 x ptr addrspace(7)> %ret +} + +define ptr addrspace(7) @inttoptr_long(i256 %v) { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @inttoptr_long +; CHECK-SAME: (i256 [[V:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[TMP1:%.*]] = lshr i256 [[V]], 32 +; CHECK-NEXT: [[TMP2:%.*]] = trunc i256 [[TMP1]] to i128 +; CHECK-NEXT: [[RET_RSRC:%.*]] = inttoptr i128 [[TMP2]] to ptr addrspace(8) +; CHECK-NEXT: [[RET_OFF:%.*]] = trunc i256 [[V]] to i32 +; CHECK-NEXT: [[TMP3:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[RET_RSRC]], 0 +; CHECK-NEXT: [[RET:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP3]], i32 [[RET_OFF]], 1 +; CHECK-NEXT: ret { ptr addrspace(8), i32 } [[RET]] +; + %ret = inttoptr i256 %v to ptr addrspace(7) + ret ptr addrspace(7) %ret +} + +define ptr addrspace(7) @inttoptr_offset(i32 %v) { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @inttoptr_offset +; CHECK-SAME: (i32 [[V:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[TMP1:%.*]] = lshr i32 [[V]], 32 +; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i128 +; CHECK-NEXT: [[RET_RSRC:%.*]] = inttoptr i128 [[TMP2]] to ptr addrspace(8) +; CHECK-NEXT: [[TMP3:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[RET_RSRC]], 0 +; CHECK-NEXT: [[RET:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP3]], i32 [[V]], 1 +; CHECK-NEXT: ret { ptr addrspace(8), i32 } [[RET]] +; + %ret = inttoptr i32 %v to ptr addrspace(7) + ret ptr addrspace(7) %ret +} + +define ptr addrspace(7) @addrspacecast(ptr addrspace(8) %buf) { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @addrspacecast +; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[BUF]], 0 +; CHECK-NEXT: [[RET:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP1]], i32 0, 1 +; CHECK-NEXT: ret { ptr addrspace(8), i32 } [[RET]] +; + %ret = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7) + ret ptr addrspace(7) %ret +} + +define <2 x ptr addrspace(7)> @addrspacecast_vec(<2 x ptr addrspace(8)> %buf) { +; CHECK-LABEL: define { <2 x ptr addrspace(8)>, <2 x i32> } @addrspacecast_vec +; CHECK-SAME: (<2 x ptr addrspace(8)> [[BUF:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { <2 x ptr addrspace(8)>, <2 x i32> } poison, <2 x ptr addrspace(8)> [[BUF]], 0 +; CHECK-NEXT: [[RET:%.*]] = insertvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[TMP1]], <2 x i32> zeroinitializer, 1 +; CHECK-NEXT: ret { <2 x ptr addrspace(8)>, <2 x i32> } [[RET]] +; + %ret = addrspacecast <2 x ptr addrspace(8)> %buf to <2 x ptr addrspace(7)> + ret <2 x ptr addrspace(7)> %ret +} + +define i1 @icmp_eq(ptr addrspace(7) %a, ptr addrspace(7) %b) { +; CHECK-LABEL: define i1 @icmp_eq +; CHECK-SAME: ({ ptr addrspace(8), i32 } [[A:%.*]], { ptr addrspace(8), i32 } [[B:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[B_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[B]], 0 +; CHECK-NEXT: [[B_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[B]], 1 +; CHECK-NEXT: [[A_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[A]], 0 +; CHECK-NEXT: [[A_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[A]], 1 +; CHECK-NEXT: [[RET_RSRC:%.*]] = icmp eq ptr addrspace(8) [[A_RSRC]], [[B_RSRC]] +; CHECK-NEXT: [[RET_OFF:%.*]] = icmp eq i32 [[A_OFF]], [[B_OFF]] +; CHECK-NEXT: [[RET:%.*]] = and i1 [[RET_RSRC]], [[RET_OFF]] +; CHECK-NEXT: ret i1 [[RET]] +; + %ret = icmp eq ptr addrspace(7) %a, %b + ret i1 %ret +} + +define i1 @icmp_ne(ptr addrspace(7) %a, ptr addrspace(7) %b) { +; CHECK-LABEL: define i1 @icmp_ne +; CHECK-SAME: ({ ptr addrspace(8), i32 } [[A:%.*]], { ptr addrspace(8), i32 } [[B:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[B_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[B]], 0 +; CHECK-NEXT: [[B_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[B]], 1 +; CHECK-NEXT: [[A_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[A]], 0 +; CHECK-NEXT: [[A_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[A]], 1 +; CHECK-NEXT: [[RET_RSRC:%.*]] = icmp ne ptr addrspace(8) [[A_RSRC]], [[B_RSRC]] +; CHECK-NEXT: [[RET_OFF:%.*]] = icmp ne i32 [[A_OFF]], [[B_OFF]] +; CHECK-NEXT: [[RET:%.*]] = or i1 [[RET_RSRC]], [[RET_OFF]] +; CHECK-NEXT: ret i1 [[RET]] +; + %ret = icmp ne ptr addrspace(7) %a, %b + ret i1 %ret +} + +define <2 x i1> @icmp_eq_vec(<2 x ptr addrspace(7)> %a, <2 x ptr addrspace(7)> %b) { +; CHECK-LABEL: define <2 x i1> @icmp_eq_vec +; CHECK-SAME: ({ <2 x ptr addrspace(8)>, <2 x i32> } [[A:%.*]], { <2 x ptr addrspace(8)>, <2 x i32> } [[B:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[B_RSRC:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[B]], 0 +; CHECK-NEXT: [[B_OFF:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[B]], 1 +; CHECK-NEXT: [[A_RSRC:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[A]], 0 +; CHECK-NEXT: [[A_OFF:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[A]], 1 +; CHECK-NEXT: [[RET_RSRC:%.*]] = icmp eq <2 x ptr addrspace(8)> [[A_RSRC]], [[B_RSRC]] +; CHECK-NEXT: [[RET_OFF:%.*]] = icmp eq <2 x i32> [[A_OFF]], [[B_OFF]] +; CHECK-NEXT: [[RET:%.*]] = and <2 x i1> [[RET_RSRC]], [[RET_OFF]] +; CHECK-NEXT: ret <2 x i1> [[RET]] +; + %ret = icmp eq <2 x ptr addrspace(7)> %a, %b + ret <2 x i1> %ret +} + +define <2 x i1> @icmp_ne_vec(<2 x ptr addrspace(7)> %a, <2 x ptr addrspace(7)> %b) { +; CHECK-LABEL: define <2 x i1> @icmp_ne_vec +; CHECK-SAME: ({ <2 x ptr addrspace(8)>, <2 x i32> } [[A:%.*]], { <2 x ptr addrspace(8)>, <2 x i32> } [[B:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[B_RSRC:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[B]], 0 +; CHECK-NEXT: [[B_OFF:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[B]], 1 +; CHECK-NEXT: [[A_RSRC:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[A]], 0 +; CHECK-NEXT: [[A_OFF:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[A]], 1 +; CHECK-NEXT: [[RET_RSRC:%.*]] = icmp ne <2 x ptr addrspace(8)> [[A_RSRC]], [[B_RSRC]] +; CHECK-NEXT: [[RET_OFF:%.*]] = icmp ne <2 x i32> [[A_OFF]], [[B_OFF]] +; CHECK-NEXT: [[RET:%.*]] = or <2 x i1> [[RET_RSRC]], [[RET_OFF]] +; CHECK-NEXT: ret <2 x i1> [[RET]] +; + %ret = icmp ne <2 x ptr addrspace(7)> %a, %b + ret <2 x i1> %ret +} + +define ptr addrspace(7) @freeze(ptr addrspace(7) %p) { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @freeze +; CHECK-SAME: ({ ptr addrspace(8), i32 } [[P:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[P_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[P]], 0 +; CHECK-NEXT: [[P_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[P]], 1 +; CHECK-NEXT: [[RET_RSRC:%.*]] = freeze ptr addrspace(8) [[P_RSRC]] +; CHECK-NEXT: [[RET_OFF:%.*]] = freeze i32 [[P_OFF]] +; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[RET_RSRC]], 0 +; CHECK-NEXT: [[RET:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP1]], i32 [[RET_OFF]], 1 +; CHECK-NEXT: ret { ptr addrspace(8), i32 } [[RET]] +; + %ret = freeze ptr addrspace(7) %p + ret ptr addrspace(7) %ret +} + +define <2 x ptr addrspace(7)> @freeze_vec(<2 x ptr addrspace(7)> %p) { +; CHECK-LABEL: define { <2 x ptr addrspace(8)>, <2 x i32> } @freeze_vec +; CHECK-SAME: ({ <2 x ptr addrspace(8)>, <2 x i32> } [[P:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[P_RSRC:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[P]], 0 +; CHECK-NEXT: [[P_OFF:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[P]], 1 +; CHECK-NEXT: [[RET_RSRC:%.*]] = freeze <2 x ptr addrspace(8)> [[P_RSRC]] +; CHECK-NEXT: [[RET_OFF:%.*]] = freeze <2 x i32> [[P_OFF]] +; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { <2 x ptr addrspace(8)>, <2 x i32> } poison, <2 x ptr addrspace(8)> [[RET_RSRC]], 0 +; CHECK-NEXT: [[RET:%.*]] = insertvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[TMP1]], <2 x i32> [[RET_OFF]], 1 +; CHECK-NEXT: ret { <2 x ptr addrspace(8)>, <2 x i32> } [[RET]] +; + %ret = freeze <2 x ptr addrspace(7)> %p + ret <2 x ptr addrspace(7)> %ret +} + +define ptr addrspace(7) @extractelement(<2 x ptr addrspace(7)> %v, i32 %i) { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @extractelement +; CHECK-SAME: ({ <2 x ptr addrspace(8)>, <2 x i32> } [[V:%.*]], i32 [[I:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[V_RSRC:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[V]], 0 +; CHECK-NEXT: [[V_OFF:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[V]], 1 +; CHECK-NEXT: [[RET_RSRC:%.*]] = extractelement <2 x ptr addrspace(8)> [[V_RSRC]], i32 [[I]] +; CHECK-NEXT: [[RET_OFF:%.*]] = extractelement <2 x i32> [[V_OFF]], i32 [[I]] +; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[RET_RSRC]], 0 +; CHECK-NEXT: [[RET:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP1]], i32 [[RET_OFF]], 1 +; CHECK-NEXT: ret { ptr addrspace(8), i32 } [[RET]] +; + %ret = extractelement <2 x ptr addrspace(7)> %v, i32 %i + ret ptr addrspace(7) %ret +} + +define <2 x ptr addrspace(7)> @insertelement(<2 x ptr addrspace(7)> %v, ptr addrspace(7) %s, i32 %i) { +; CHECK-LABEL: define { <2 x ptr addrspace(8)>, <2 x i32> } @insertelement +; CHECK-SAME: ({ <2 x ptr addrspace(8)>, <2 x i32> } [[V:%.*]], { ptr addrspace(8), i32 } [[S:%.*]], i32 [[I:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[S_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[S]], 0 +; CHECK-NEXT: [[S_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[S]], 1 +; CHECK-NEXT: [[V_RSRC:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[V]], 0 +; CHECK-NEXT: [[V_OFF:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[V]], 1 +; CHECK-NEXT: [[RET_RSRC:%.*]] = insertelement <2 x ptr addrspace(8)> [[V_RSRC]], ptr addrspace(8) [[S_RSRC]], i32 [[I]] +; CHECK-NEXT: [[RET_OFF:%.*]] = insertelement <2 x i32> [[V_OFF]], i32 [[S_OFF]], i32 [[I]] +; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { <2 x ptr addrspace(8)>, <2 x i32> } poison, <2 x ptr addrspace(8)> [[RET_RSRC]], 0 +; CHECK-NEXT: [[RET:%.*]] = insertvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[TMP1]], <2 x i32> [[RET_OFF]], 1 +; CHECK-NEXT: ret { <2 x ptr addrspace(8)>, <2 x i32> } [[RET]] +; + %ret = insertelement <2 x ptr addrspace(7)> %v, ptr addrspace(7) %s, i32 %i + ret <2 x ptr addrspace(7)> %ret +} + +define <4 x ptr addrspace(7)> @shufflenvector(<2 x ptr addrspace(7)> %a, <2 x ptr addrspace(7)> %b) { +; CHECK-LABEL: define { <4 x ptr addrspace(8)>, <4 x i32> } @shufflenvector +; CHECK-SAME: ({ <2 x ptr addrspace(8)>, <2 x i32> } [[A:%.*]], { <2 x ptr addrspace(8)>, <2 x i32> } [[B:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[B_RSRC:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[B]], 0 +; CHECK-NEXT: [[B_OFF:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[B]], 1 +; CHECK-NEXT: [[A_RSRC:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[A]], 0 +; CHECK-NEXT: [[A_OFF:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[A]], 1 +; CHECK-NEXT: [[RET_RSRC:%.*]] = shufflevector <2 x ptr addrspace(8)> [[A_RSRC]], <2 x ptr addrspace(8)> [[B_RSRC]], <4 x i32> +; CHECK-NEXT: [[RET_OFF:%.*]] = shufflevector <2 x i32> [[A_OFF]], <2 x i32> [[B_OFF]], <4 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { <4 x ptr addrspace(8)>, <4 x i32> } poison, <4 x ptr addrspace(8)> [[RET_RSRC]], 0 +; CHECK-NEXT: [[RET:%.*]] = insertvalue { <4 x ptr addrspace(8)>, <4 x i32> } [[TMP1]], <4 x i32> [[RET_OFF]], 1 +; CHECK-NEXT: ret { <4 x ptr addrspace(8)>, <4 x i32> } [[RET]] +; + %ret = shufflevector <2 x ptr addrspace(7)> %a, <2 x ptr addrspace(7)> %b, <4 x i32> + ret <4 x ptr addrspace(7)> %ret +} + +declare ptr addrspace(7) @llvm.ptrmask.p7.i32(ptr addrspace(7), i32) + +define ptr addrspace(7) @ptrmask(ptr addrspace(7) %p, i32 %mask) { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @ptrmask +; CHECK-SAME: ({ ptr addrspace(8), i32 } [[P:%.*]], i32 [[MASK:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[P_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[P]], 0 +; CHECK-NEXT: [[P_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[P]], 1 +; CHECK-NEXT: [[RET_OFF:%.*]] = and i32 [[P_OFF]], [[MASK]] +; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[P_RSRC]], 0 +; CHECK-NEXT: [[RET:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP1]], i32 [[RET_OFF]], 1 +; CHECK-NEXT: ret { ptr addrspace(8), i32 } [[RET]] +; + %ret = call ptr addrspace(7) @llvm.ptrmask.p7.i32(ptr addrspace(7) %p, i32 %mask) + ret ptr addrspace(7) %ret +} + +declare ptr @llvm.invariant.start.p7(i64, ptr addrspace(7) nocapture) +declare void @llvm.invariant.end.p7(ptr, i64, ptr addrspace(7) nocapture) + +define i32 @invariant_start_end(ptr addrspace(7) %p) { +; CHECK-LABEL: define i32 @invariant_start_end +; CHECK-SAME: ({ ptr addrspace(8), i32 } [[P:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[P_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[P]], 0 +; CHECK-NEXT: [[P_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[P]], 1 +; CHECK-NEXT: [[INV:%.*]] = call ptr @llvm.invariant.start.p8(i64 256, ptr addrspace(8) [[P_RSRC]]) +; CHECK-NEXT: [[V:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 [[P_RSRC]], i32 [[P_OFF]], i32 0, i32 0) +; CHECK-NEXT: call void @llvm.invariant.end.p8(ptr [[INV]], i64 256, ptr addrspace(8) [[P_RSRC]]) +; CHECK-NEXT: ret i32 [[V]] +; + %inv = call ptr @llvm.invariant.start.p7(i64 256, ptr addrspace(7) %p) + %v = load i32, ptr addrspace(7) %p + call void @llvm.invariant.end.p7(ptr %inv, i64 256, ptr addrspace(7) %p) + ret i32 %v +} + +declare ptr addrspace(7) @llvm.launder.invariant.group.p7(ptr addrspace(7) nocapture) +declare ptr addrspace(7) @llvm.strip.invariant.group.p7(ptr addrspace(7) nocapture) + +define ptr addrspace(7) @invariant_group(ptr addrspace(7) %p) { +; CHECK-LABEL: define { ptr addrspace(8), i32 } @invariant_group +; CHECK-SAME: ({ ptr addrspace(8), i32 } [[P:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[P_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[P]], 0 +; CHECK-NEXT: [[P_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[P]], 1 +; CHECK-NEXT: [[LAUNDERED:%.*]] = call ptr addrspace(8) @llvm.launder.invariant.group.p8(ptr addrspace(8) [[P_RSRC]]) +; CHECK-NEXT: [[STRIPPED:%.*]] = call ptr addrspace(8) @llvm.strip.invariant.group.p8(ptr addrspace(8) [[LAUNDERED]]) +; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[STRIPPED]], 0 +; CHECK-NEXT: [[TMP2:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP1]], i32 [[P_OFF]], 1 +; CHECK-NEXT: ret { ptr addrspace(8), i32 } [[TMP2]] +; + %laundered = call ptr addrspace(7) @llvm.launder.invariant.group.p7(ptr addrspace(7) %p) + %stripped = call ptr addrspace(7) @llvm.strip.invariant.group.p7(ptr addrspace(7) %laundered) + ret ptr addrspace(7) %stripped +} diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll new file mode 100644 index 000000000000..69387e67c1c7 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll @@ -0,0 +1,127 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 +; RUN: opt -S -mcpu=gfx900 -amdgpu-lower-buffer-fat-pointers -check-debugify < %s | FileCheck %s +; RUN: opt -S -mcpu=gfx900 -passes=amdgpu-lower-buffer-fat-pointers,check-debugify < %s | FileCheck %s + +target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8" +target triple = "amdgcn--" + +define float @debug_stash_pointer(ptr addrspace(8) %buf, i32 %idx, ptr addrspace(8) %aux) !dbg !5 { +; CHECK-LABEL: define float @debug_stash_pointer +; CHECK-SAME: (ptr addrspace(8) [[BUF:%.*]], i32 [[IDX:%.*]], ptr addrspace(8) [[AUX:%.*]]) #[[ATTR0:[0-9]+]] !dbg [[DBG5:![0-9]+]] { +; CHECK-NEXT: [[BUF_PTR_VAR:%.*]] = alloca i160, align 32, addrspace(5), !dbg [[DBG21:![0-9]+]] +; CHECK-NEXT: tail call void @llvm.dbg.value(metadata ptr addrspace(5) [[BUF_PTR_VAR]], metadata [[META10:![0-9]+]], metadata !DIExpression()), !dbg [[DBG21]] +; CHECK-NEXT: [[AUX_PTR_VAR:%.*]] = alloca i160, align 32, addrspace(5), !dbg [[DBG22:![0-9]+]] +; CHECK-NEXT: tail call void @llvm.dbg.value(metadata ptr addrspace(5) [[AUX_PTR_VAR]], metadata [[META12:![0-9]+]], metadata !DIExpression()), !dbg [[DBG22]] +; CHECK-NEXT: tail call void @llvm.dbg.value(metadata { ptr addrspace(8), i32 } undef, metadata [[META13:![0-9]+]], metadata !DIExpression()), !dbg [[DBG23:![0-9]+]] +; CHECK-NEXT: [[BUF_PTR_INT_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[BUF]] to i160, !dbg [[DBG24:![0-9]+]] +; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i160 [[BUF_PTR_INT_RSRC]], 32, !dbg [[DBG24]] +; CHECK-NEXT: [[BUF_PTR_INT:%.*]] = or i160 [[TMP1]], 0, !dbg [[DBG24]] +; CHECK-NEXT: store i160 [[BUF_PTR_INT]], ptr addrspace(5) [[BUF_PTR_VAR]], align 32, !dbg [[DBG24]] +; CHECK-NEXT: tail call void @llvm.dbg.value(metadata { ptr addrspace(8), i32 } undef, metadata [[META15:![0-9]+]], metadata !DIExpression()), !dbg [[DBG25:![0-9]+]] +; CHECK-NEXT: [[AUX_PTR_INT_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[AUX]] to i160, !dbg [[DBG26:![0-9]+]] +; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i160 [[AUX_PTR_INT_RSRC]], 32, !dbg [[DBG26]] +; CHECK-NEXT: [[AUX_PTR_INT:%.*]] = or i160 [[TMP2]], 0, !dbg [[DBG26]] +; CHECK-NEXT: store i160 [[AUX_PTR_INT]], ptr addrspace(5) [[AUX_PTR_VAR]], align 32, !dbg [[DBG26]] +; CHECK-NEXT: [[BUF_PTR_2:%.*]] = load i160, ptr addrspace(5) [[BUF_PTR_VAR]], align 32, !dbg [[DBG27:![0-9]+]] +; CHECK-NEXT: [[TMP3:%.*]] = lshr i160 [[BUF_PTR_2]], 32, !dbg [[DBG27]] +; CHECK-NEXT: [[TMP4:%.*]] = trunc i160 [[TMP3]] to i128, !dbg [[DBG27]] +; CHECK-NEXT: [[BUF_PTR_2_PTR_RSRC:%.*]] = inttoptr i128 [[TMP4]] to ptr addrspace(8), !dbg [[DBG27]] +; CHECK-NEXT: [[BUF_PTR_2_PTR_OFF:%.*]] = trunc i160 [[BUF_PTR_2]] to i32, !dbg [[DBG27]] +; CHECK-NEXT: tail call void @llvm.dbg.value(metadata { ptr addrspace(8), i32 } undef, metadata [[META16:![0-9]+]], metadata !DIExpression()), !dbg [[DBG27]] +; CHECK-NEXT: [[TMP5:%.*]] = shl i32 [[IDX]], 2, !dbg [[DBG28:![0-9]+]] +; CHECK-NEXT: [[BUF_PTR_3:%.*]] = add i32 [[BUF_PTR_2_PTR_OFF]], [[TMP5]], !dbg [[DBG28]] +; CHECK-NEXT: tail call void @llvm.dbg.value(metadata { ptr addrspace(8), i32 } undef, metadata [[META17:![0-9]+]], metadata !DIExpression()), !dbg [[DBG28]] +; CHECK-NEXT: [[BUF_PTR_3_INT_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[BUF_PTR_2_PTR_RSRC]] to i160, !dbg [[DBG29:![0-9]+]] +; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i160 [[BUF_PTR_3_INT_RSRC]], 32, !dbg [[DBG29]] +; CHECK-NEXT: [[BUF_PTR_3_INT_OFF:%.*]] = zext i32 [[BUF_PTR_3]] to i160, !dbg [[DBG29]] +; CHECK-NEXT: [[BUF_PTR_3_INT:%.*]] = or i160 [[TMP6]], [[BUF_PTR_3_INT_OFF]], !dbg [[DBG29]] +; CHECK-NEXT: store i160 [[BUF_PTR_3_INT]], ptr addrspace(5) [[BUF_PTR_VAR]], align 32, !dbg [[DBG29]] +; CHECK-NEXT: [[BUF_PTR_4:%.*]] = load i160, ptr addrspace(5) [[BUF_PTR_VAR]], align 32, !dbg [[DBG30:![0-9]+]] +; CHECK-NEXT: [[TMP7:%.*]] = lshr i160 [[BUF_PTR_4]], 32, !dbg [[DBG30]] +; CHECK-NEXT: [[TMP8:%.*]] = trunc i160 [[TMP7]] to i128, !dbg [[DBG30]] +; CHECK-NEXT: [[BUF_PTR_4_PTR_RSRC:%.*]] = inttoptr i128 [[TMP8]] to ptr addrspace(8), !dbg [[DBG30]] +; CHECK-NEXT: [[BUF_PTR_4_PTR_OFF:%.*]] = trunc i160 [[BUF_PTR_4]] to i32, !dbg [[DBG30]] +; CHECK-NEXT: tail call void @llvm.dbg.value(metadata { ptr addrspace(8), i32 } undef, metadata [[META18:![0-9]+]], metadata !DIExpression()), !dbg [[DBG30]] +; CHECK-NEXT: [[RET:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF_PTR_4_PTR_RSRC]], i32 [[BUF_PTR_4_PTR_OFF]], i32 0, i32 0), !dbg [[DBG31:![0-9]+]] +; CHECK-NEXT: tail call void @llvm.dbg.value(metadata float [[RET]], metadata [[META19:![0-9]+]], metadata !DIExpression()), !dbg [[DBG31]] +; CHECK-NEXT: [[AUX_PTR_2:%.*]] = load i160, ptr addrspace(5) [[AUX_PTR_VAR]], align 32, !dbg [[DBG32:![0-9]+]] +; CHECK-NEXT: [[TMP9:%.*]] = lshr i160 [[AUX_PTR_2]], 32, !dbg [[DBG32]] +; CHECK-NEXT: [[TMP10:%.*]] = trunc i160 [[TMP9]] to i128, !dbg [[DBG32]] +; CHECK-NEXT: [[AUX_PTR_2_PTR_RSRC:%.*]] = inttoptr i128 [[TMP10]] to ptr addrspace(8), !dbg [[DBG32]] +; CHECK-NEXT: [[AUX_PTR_2_PTR_OFF:%.*]] = trunc i160 [[AUX_PTR_2]] to i32, !dbg [[DBG32]] +; CHECK-NEXT: tail call void @llvm.dbg.value(metadata { ptr addrspace(8), i32 } undef, metadata [[META20:![0-9]+]], metadata !DIExpression()), !dbg [[DBG32]] +; CHECK-NEXT: [[BUF_PTR_4_PTR_INT_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[BUF_PTR_4_PTR_RSRC]] to i160, !dbg [[DBG33:![0-9]+]] +; CHECK-NEXT: [[TMP11:%.*]] = shl nuw i160 [[BUF_PTR_4_PTR_INT_RSRC]], 32, !dbg [[DBG33]] +; CHECK-NEXT: [[BUF_PTR_4_PTR_INT_OFF:%.*]] = zext i32 [[BUF_PTR_4_PTR_OFF]] to i160, !dbg [[DBG33]] +; CHECK-NEXT: [[BUF_PTR_4_PTR_INT:%.*]] = or i160 [[TMP11]], [[BUF_PTR_4_PTR_INT_OFF]], !dbg [[DBG33]] +; CHECK-NEXT: call void @llvm.amdgcn.raw.ptr.buffer.store.i160(i160 [[BUF_PTR_4_PTR_INT]], ptr addrspace(8) align 32 [[AUX_PTR_2_PTR_RSRC]], i32 [[AUX_PTR_2_PTR_OFF]], i32 0, i32 0), !dbg [[DBG33]] +; CHECK-NEXT: ret float [[RET]], !dbg [[DBG34:![0-9]+]] +; + %buf.ptr.var = alloca ptr addrspace(7), align 32, addrspace(5), !dbg !20 + call void @llvm.dbg.value(metadata ptr addrspace(5) %buf.ptr.var, metadata !9, metadata !DIExpression()), !dbg !20 + %aux.ptr.var = alloca ptr addrspace(7), align 32, addrspace(5), !dbg !21 + call void @llvm.dbg.value(metadata ptr addrspace(5) %aux.ptr.var, metadata !11, metadata !DIExpression()), !dbg !21 + %buf.ptr = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7), !dbg !22 + call void @llvm.dbg.value(metadata ptr addrspace(7) %buf.ptr, metadata !12, metadata !DIExpression()), !dbg !22 + store ptr addrspace(7) %buf.ptr, ptr addrspace(5) %buf.ptr.var, align 32, !dbg !23 + %aux.ptr = addrspacecast ptr addrspace(8) %aux to ptr addrspace(7), !dbg !24 + call void @llvm.dbg.value(metadata ptr addrspace(7) %aux.ptr, metadata !14, metadata !DIExpression()), !dbg !24 + store ptr addrspace(7) %aux.ptr, ptr addrspace(5) %aux.ptr.var, align 32, !dbg !25 + %buf.ptr.2 = load ptr addrspace(7), ptr addrspace(5) %buf.ptr.var, align 32, !dbg !26 + call void @llvm.dbg.value(metadata ptr addrspace(7) %buf.ptr.2, metadata !15, metadata !DIExpression()), !dbg !26 + %buf.ptr.3 = getelementptr float, ptr addrspace(7) %buf.ptr.2, i32 %idx, !dbg !27 + call void @llvm.dbg.value(metadata ptr addrspace(7) %buf.ptr.3, metadata !16, metadata !DIExpression()), !dbg !27 + store ptr addrspace(7) %buf.ptr.3, ptr addrspace(5) %buf.ptr.var, align 32, !dbg !28 + %buf.ptr.4 = load ptr addrspace(7), ptr addrspace(5) %buf.ptr.var, align 32, !dbg !29 + call void @llvm.dbg.value(metadata ptr addrspace(7) %buf.ptr.4, metadata !17, metadata !DIExpression()), !dbg !29 + %ret = load float, ptr addrspace(7) %buf.ptr.4, align 4, !dbg !30 + call void @llvm.dbg.value(metadata float %ret, metadata !18, metadata !DIExpression()), !dbg !30 + %aux.ptr.2 = load ptr addrspace(7), ptr addrspace(5) %aux.ptr.var, align 32, !dbg !31 + call void @llvm.dbg.value(metadata ptr addrspace(7) %aux.ptr.2, metadata !19, metadata !DIExpression()), !dbg !31 + store ptr addrspace(7) %buf.ptr.4, ptr addrspace(7) %aux.ptr.2, align 32, !dbg !32 + ret float %ret, !dbg !33 +} + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare void @llvm.dbg.value(metadata, metadata, metadata) #0 + +attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + +!llvm.dbg.cu = !{!0} +!llvm.debugify = !{!2, !3} +!llvm.module.flags = !{!4} + +!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug) +!1 = !DIFile(filename: "", directory: "/") +!2 = !{i32 14} +!3 = !{i32 9} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = distinct !DISubprogram(name: "debug_stash_pointer", linkageName: "debug_stash_pointer", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !8) +!6 = !DISubroutineType(types: !7) +!7 = !{} +!8 = !{!9, !11, !12, !14, !15, !16, !17, !18, !19} +!9 = !DILocalVariable(name: "1", scope: !5, file: !1, line: 1, type: !10) +!10 = !DIBasicType(name: "ty32", size: 32, encoding: DW_ATE_unsigned) +!11 = !DILocalVariable(name: "2", scope: !5, file: !1, line: 2, type: !10) +!12 = !DILocalVariable(name: "3", scope: !5, file: !1, line: 3, type: !13) +!13 = !DIBasicType(name: "ty256", size: 256, encoding: DW_ATE_unsigned) +!14 = !DILocalVariable(name: "4", scope: !5, file: !1, line: 5, type: !13) +!15 = !DILocalVariable(name: "5", scope: !5, file: !1, line: 7, type: !13) +!16 = !DILocalVariable(name: "6", scope: !5, file: !1, line: 8, type: !13) +!17 = !DILocalVariable(name: "7", scope: !5, file: !1, line: 10, type: !13) +!18 = !DILocalVariable(name: "8", scope: !5, file: !1, line: 11, type: !10) +!19 = !DILocalVariable(name: "9", scope: !5, file: !1, line: 12, type: !13) +!20 = !DILocation(line: 1, column: 1, scope: !5) +!21 = !DILocation(line: 2, column: 1, scope: !5) +!22 = !DILocation(line: 3, column: 1, scope: !5) +!23 = !DILocation(line: 4, column: 1, scope: !5) +!24 = !DILocation(line: 5, column: 1, scope: !5) +!25 = !DILocation(line: 6, column: 1, scope: !5) +!26 = !DILocation(line: 7, column: 1, scope: !5) +!27 = !DILocation(line: 8, column: 1, scope: !5) +!28 = !DILocation(line: 9, column: 1, scope: !5) +!29 = !DILocation(line: 10, column: 1, scope: !5) +!30 = !DILocation(line: 11, column: 1, scope: !5) +!31 = !DILocation(line: 12, column: 1, scope: !5) +!32 = !DILocation(line: 13, column: 1, scope: !5) +!33 = !DILocation(line: 14, column: 1, scope: !5) diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn index af79c8eae5ec..e1b867bf70ba 100644 --- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn @@ -157,6 +157,7 @@ static_library("LLVMAMDGPUCodeGen") { "AMDGPULegalizerInfo.cpp", "AMDGPULibCalls.cpp", "AMDGPULibFunc.cpp", + "AMDGPULowerBufferFatPointers.cpp", "AMDGPULowerKernelArguments.cpp", "AMDGPULowerKernelAttributes.cpp", "AMDGPULowerModuleLDSPass.cpp", -- GitLab From 67c6ad6f30e35c7670bce9bca902caa4b1c8c0e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20=C3=81lvarez=20Ayll=C3=B3n?= Date: Wed, 6 Mar 2024 16:52:18 +0100 Subject: [PATCH 309/929] [clang][analyzer] Model allocation behavior or getdelim/geline (#83138) `getdelim` and `getline` may free, allocate, or re-allocate the input buffer, ensuring its size is enough to hold the incoming line, the delimiter, and the null terminator. `*lineptr` must be a valid argument to `free`, which means it can be either 1. `NULL`, in which case these functions perform an allocation equivalent to a call to `malloc` even on failure. 2. A pointer returned by the `malloc` family of functions. Other pointers are UB (`alloca`, a pointer to a static, to a stack variable, etc.) --- .../Core/PathSensitive/CheckerHelpers.h | 6 ++ .../StaticAnalyzer/Checkers/MallocChecker.cpp | 77 +++++++++++++-- .../StaticAnalyzer/Core/CheckerHelpers.cpp | 9 ++ .../system-header-simulator-for-malloc.h | 1 + clang/test/Analysis/getline-alloc.c | 95 +++++++++++++++++++ 5 files changed, 182 insertions(+), 6 deletions(-) create mode 100644 clang/test/Analysis/getline-alloc.c diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h index 65982457ad83..60421e5437d8 100644 --- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h +++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h @@ -13,6 +13,9 @@ #ifndef LLVM_CLANG_STATICANALYZER_CORE_PATHSENSITIVE_CHECKERHELPERS_H #define LLVM_CLANG_STATICANALYZER_CORE_PATHSENSITIVE_CHECKERHELPERS_H +#include "ProgramState_Fwd.h" +#include "SVals.h" + #include "clang/AST/OperationKinds.h" #include "clang/AST/Stmt.h" #include "clang/Basic/OperatorKinds.h" @@ -110,6 +113,9 @@ public: OperatorKind operationKindFromOverloadedOperator(OverloadedOperatorKind OOK, bool IsBinary); +std::optional getPointeeDefVal(SVal PtrSVal, + ProgramStateRef State); + } // namespace ento } // namespace clang diff --git a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp index b27ca6a44959..03cb7696707f 100644 --- a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp @@ -382,6 +382,8 @@ private: CHECK_FN(checkGMemdup) CHECK_FN(checkGMallocN) CHECK_FN(checkGMallocN0) + CHECK_FN(preGetdelim) + CHECK_FN(checkGetdelim) CHECK_FN(checkReallocN) CHECK_FN(checkOwnershipAttr) @@ -391,6 +393,11 @@ private: using CheckFn = std::function; + const CallDescriptionMap PreFnMap{ + {{{"getline"}, 3}, &MallocChecker::preGetdelim}, + {{{"getdelim"}, 4}, &MallocChecker::preGetdelim}, + }; + const CallDescriptionMap FreeingMemFnMap{ {{{"free"}, 1}, &MallocChecker::checkFree}, {{{"if_freenameindex"}, 1}, &MallocChecker::checkIfFreeNameIndex}, @@ -439,6 +446,8 @@ private: std::bind(&MallocChecker::checkRealloc, _1, _2, _3, false)}, {{{"g_realloc_n"}, 3}, &MallocChecker::checkReallocN}, {{{"g_try_realloc_n"}, 3}, &MallocChecker::checkReallocN}, + {{{"getline"}, 3}, &MallocChecker::checkGetdelim}, + {{{"getdelim"}, 4}, &MallocChecker::checkGetdelim}, }; bool isMemCall(const CallEvent &Call) const; @@ -588,11 +597,14 @@ private: /// } /// \param [in] ReturnsNullOnFailure Whether the memory deallocation function /// we're modeling returns with Null on failure. + /// \param [in] ArgValOpt Optional value to use for the argument instead of + /// the one obtained from ArgExpr. /// \returns The ProgramState right after deallocation. [[nodiscard]] ProgramStateRef FreeMemAux(CheckerContext &C, const Expr *ArgExpr, const CallEvent &Call, ProgramStateRef State, bool Hold, bool &IsKnownToBeAllocated, - AllocationFamily Family, bool ReturnsNullOnFailure = false) const; + AllocationFamily Family, bool ReturnsNullOnFailure = false, + std::optional ArgValOpt = {}) const; // TODO: Needs some refactoring, as all other deallocation modeling // functions are suffering from out parameters and messy code due to how @@ -1423,6 +1435,50 @@ void MallocChecker::checkGMallocN0(const CallEvent &Call, C.addTransition(State); } +void MallocChecker::preGetdelim(const CallEvent &Call, + CheckerContext &C) const { + if (!Call.isGlobalCFunction()) + return; + + ProgramStateRef State = C.getState(); + const auto LinePtr = getPointeeDefVal(Call.getArgSVal(0), State); + if (!LinePtr) + return; + + // FreeMemAux takes IsKnownToBeAllocated as an output parameter, and it will + // be true after the call if the symbol was registered by this checker. + // We do not need this value here, as FreeMemAux will take care + // of reporting any violation of the preconditions. + bool IsKnownToBeAllocated = false; + State = FreeMemAux(C, Call.getArgExpr(0), Call, State, false, + IsKnownToBeAllocated, AF_Malloc, false, LinePtr); + if (State) + C.addTransition(State); +} + +void MallocChecker::checkGetdelim(const CallEvent &Call, + CheckerContext &C) const { + if (!Call.isGlobalCFunction()) + return; + + ProgramStateRef State = C.getState(); + // Handle the post-conditions of getline and getdelim: + // Register the new conjured value as an allocated buffer. + const CallExpr *CE = dyn_cast_or_null(Call.getOriginExpr()); + if (!CE) + return; + + SValBuilder &SVB = C.getSValBuilder(); + + const auto LinePtr = getPointeeDefVal(Call.getArgSVal(0), State); + const auto Size = getPointeeDefVal(Call.getArgSVal(1), State); + if (!LinePtr || !Size || !LinePtr->getAsRegion()) + return; + + State = setDynamicExtent(State, LinePtr->getAsRegion(), *Size, SVB); + C.addTransition(MallocUpdateRefState(C, CE, State, AF_Malloc, *LinePtr)); +} + void MallocChecker::checkReallocN(const CallEvent &Call, CheckerContext &C) const { ProgramStateRef State = C.getState(); @@ -1895,15 +1951,17 @@ static void printExpectedDeallocName(raw_ostream &os, AllocationFamily Family) { } } -ProgramStateRef MallocChecker::FreeMemAux( - CheckerContext &C, const Expr *ArgExpr, const CallEvent &Call, - ProgramStateRef State, bool Hold, bool &IsKnownToBeAllocated, - AllocationFamily Family, bool ReturnsNullOnFailure) const { +ProgramStateRef +MallocChecker::FreeMemAux(CheckerContext &C, const Expr *ArgExpr, + const CallEvent &Call, ProgramStateRef State, + bool Hold, bool &IsKnownToBeAllocated, + AllocationFamily Family, bool ReturnsNullOnFailure, + std::optional ArgValOpt) const { if (!State) return nullptr; - SVal ArgVal = C.getSVal(ArgExpr); + SVal ArgVal = ArgValOpt.value_or(C.getSVal(ArgExpr)); if (!isa(ArgVal)) return nullptr; DefinedOrUnknownSVal location = ArgVal.castAs(); @@ -2881,6 +2939,13 @@ void MallocChecker::checkPreCall(const CallEvent &Call, return; } + // We need to handle getline pre-conditions here before the pointed region + // gets invalidated by StreamChecker + if (const auto *PreFN = PreFnMap.lookup(Call)) { + (*PreFN)(this, Call, C); + return; + } + // We will check for double free in the post visit. if (const AnyFunctionCall *FC = dyn_cast(&Call)) { const FunctionDecl *FD = FC->getDecl(); diff --git a/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp b/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp index 84ad20a54807..364c87e910b7 100644 --- a/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp +++ b/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp @@ -14,6 +14,7 @@ #include "clang/AST/Decl.h" #include "clang/AST/Expr.h" #include "clang/Lex/Preprocessor.h" +#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h" #include namespace clang { @@ -182,5 +183,13 @@ OperatorKind operationKindFromOverloadedOperator(OverloadedOperatorKind OOK, } } +std::optional getPointeeDefVal(SVal PtrSVal, + ProgramStateRef State) { + if (const auto *Ptr = PtrSVal.getAsRegion()) { + return State->getSVal(Ptr).getAs(); + } + return std::nullopt; +} + } // namespace ento } // namespace clang diff --git a/clang/test/Analysis/Inputs/system-header-simulator-for-malloc.h b/clang/test/Analysis/Inputs/system-header-simulator-for-malloc.h index e76455655e9e..bc7009eb0d1b 100644 --- a/clang/test/Analysis/Inputs/system-header-simulator-for-malloc.h +++ b/clang/test/Analysis/Inputs/system-header-simulator-for-malloc.h @@ -9,6 +9,7 @@ typedef __typeof(sizeof(int)) size_t; void *malloc(size_t); void *calloc(size_t, size_t); void free(void *); +void *alloca(size_t); #if __OBJC__ diff --git a/clang/test/Analysis/getline-alloc.c b/clang/test/Analysis/getline-alloc.c new file mode 100644 index 000000000000..5b5c716cb605 --- /dev/null +++ b/clang/test/Analysis/getline-alloc.c @@ -0,0 +1,95 @@ +// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix,debug.ExprInspection -verify %s + +// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix,alpha.unix,debug.ExprInspection -verify %s + +#include "Inputs/system-header-simulator.h" +#include "Inputs/system-header-simulator-for-malloc.h" + +void test_getline_null_buffer() { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + size_t n = 0; + if (getline(&buffer, &n, F1) > 0) { + char c = buffer[0]; // ok + } + free(buffer); + fclose(F1); +} + +void test_getline_malloc_buffer() { + FILE *F1 = tmpfile(); + if (!F1) + return; + + size_t n = 10; + char *buffer = malloc(n); + char *ptr = buffer; + + ssize_t r = getdelim(&buffer, &n, '\r', F1); + // ptr may be dangling + free(ptr); // expected-warning {{Attempt to free released memory}} + free(buffer); // ok + fclose(F1); +} + +void test_getline_alloca() { + FILE *F1 = tmpfile(); + if (!F1) + return; + size_t n = 10; + char *buffer = alloca(n); + getline(&buffer, &n, F1); // expected-warning {{Memory allocated by alloca() should not be deallocated}} + fclose(F1); +} + +void test_getline_invalid_ptr() { + FILE *F1 = tmpfile(); + if (!F1) + return; + size_t n = 10; + char *buffer = (char*)test_getline_invalid_ptr; + getline(&buffer, &n, F1); // expected-warning {{Argument to getline() is the address of the function 'test_getline_invalid_ptr', which is not memory allocated by malloc()}} + fclose(F1); +} + +void test_getline_leak() { + FILE *F1 = tmpfile(); + if (!F1) + return; + + char *buffer = NULL; + size_t n = 0; + ssize_t read; + + while ((read = getline(&buffer, &n, F1)) != -1) { + printf("%s\n", buffer); + } + + fclose(F1); // expected-warning {{Potential memory leak}} +} + +void test_getline_stack() { + size_t n = 10; + char buffer[10]; + char *ptr = buffer; + + FILE *F1 = tmpfile(); + if (!F1) + return; + + getline(&ptr, &n, F1); // expected-warning {{Argument to getline() is the address of the local variable 'buffer', which is not memory allocated by malloc()}} +} + +void test_getline_static() { + static size_t n = 10; + static char buffer[10]; + char *ptr = buffer; + + FILE *F1 = tmpfile(); + if (!F1) + return; + + getline(&ptr, &n, F1); // expected-warning {{Argument to getline() is the address of the static variable 'buffer', which is not memory allocated by malloc()}} +} -- GitLab From 8e4887ff6de135680512f21fb41e949f73cbf262 Mon Sep 17 00:00:00 2001 From: Nikolas Klauser Date: Wed, 6 Mar 2024 16:56:18 +0100 Subject: [PATCH 310/929] [libc++][NFC] Remove redundant preprocessor directive --- libcxx/include/span | 2 -- 1 file changed, 2 deletions(-) diff --git a/libcxx/include/span b/libcxx/include/span index 9efaac517fc8..cfeef35d2d80 100644 --- a/libcxx/include/span +++ b/libcxx/include/span @@ -564,10 +564,8 @@ _LIBCPP_HIDE_FROM_ABI auto as_writable_bytes(span<_Tp, _Extent> __s) noexcept { return __s.__as_writable_bytes(); } -# if _LIBCPP_STD_VER >= 20 template span(_It, _EndOrSize) -> span>>; -# endif // _LIBCPP_STD_VER >= 20 template span(_Tp (&)[_Sz]) -> span<_Tp, _Sz>; -- GitLab From 12fdabc7908d3acbec42ce6172a225db85cb4f23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Wed, 6 Mar 2024 16:56:11 +0100 Subject: [PATCH 311/929] [clang][Interp] Diagnose dummy pointers used in Inc/Dec ops For example for unknown parameter decls. --- clang/lib/AST/Interp/Interp.h | 11 +++++------ clang/test/AST/Interp/literals.cpp | 12 ++++++++++++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h index 9c3d58d5619c..43cbc2ff292c 100644 --- a/clang/lib/AST/Interp/Interp.h +++ b/clang/lib/AST/Interp/Interp.h @@ -525,8 +525,7 @@ enum class IncDecOp { template bool IncDecHelper(InterpState &S, CodePtr OpPC, const Pointer &Ptr) { - if (Ptr.isDummy()) - return false; + assert(!Ptr.isDummy()); if constexpr (std::is_same_v) { if (!S.getLangOpts().CPlusPlus14) @@ -585,7 +584,7 @@ bool IncDecHelper(InterpState &S, CodePtr OpPC, const Pointer &Ptr) { template ::T> bool Inc(InterpState &S, CodePtr OpPC) { const Pointer &Ptr = S.Stk.pop(); - if (Ptr.isDummy()) + if (!CheckDummy(S, OpPC, Ptr)) return false; if (!CheckInitialized(S, OpPC, Ptr, AK_Increment)) return false; @@ -599,7 +598,7 @@ bool Inc(InterpState &S, CodePtr OpPC) { template ::T> bool IncPop(InterpState &S, CodePtr OpPC) { const Pointer &Ptr = S.Stk.pop(); - if (Ptr.isDummy()) + if (!CheckDummy(S, OpPC, Ptr)) return false; if (!CheckInitialized(S, OpPC, Ptr, AK_Increment)) return false; @@ -614,7 +613,7 @@ bool IncPop(InterpState &S, CodePtr OpPC) { template ::T> bool Dec(InterpState &S, CodePtr OpPC) { const Pointer &Ptr = S.Stk.pop(); - if (Ptr.isDummy()) + if (!CheckDummy(S, OpPC, Ptr)) return false; if (!CheckInitialized(S, OpPC, Ptr, AK_Decrement)) return false; @@ -628,7 +627,7 @@ bool Dec(InterpState &S, CodePtr OpPC) { template ::T> bool DecPop(InterpState &S, CodePtr OpPC) { const Pointer &Ptr = S.Stk.pop(); - if (Ptr.isDummy()) + if (!CheckDummy(S, OpPC, Ptr)) return false; if (!CheckInitialized(S, OpPC, Ptr, AK_Decrement)) return false; diff --git a/clang/test/AST/Interp/literals.cpp b/clang/test/AST/Interp/literals.cpp index d86609108ca4..7ae8499b1156 100644 --- a/clang/test/AST/Interp/literals.cpp +++ b/clang/test/AST/Interp/literals.cpp @@ -839,6 +839,18 @@ namespace IncDec { return a[1]; } static_assert(f() == 3, ""); + + int nonconst(int a) { // both-note 4{{declared here}} + static_assert(a++, ""); // both-error {{not an integral constant expression}} \ + // both-note {{function parameter 'a' with unknown value cannot be used in a constant expression}} + static_assert(a--, ""); // both-error {{not an integral constant expression}} \ + // both-note {{function parameter 'a' with unknown value cannot be used in a constant expression}} + static_assert(++a, ""); // both-error {{not an integral constant expression}} \ + // both-note {{function parameter 'a' with unknown value cannot be used in a constant expression}} + static_assert(--a, ""); // both-error {{not an integral constant expression}} \ + // both-note {{function parameter 'a' with unknown value cannot be used in a constant expression}} + } + }; #endif -- GitLab From d9d9301eec6dfefcf53fd04b61324f140f273033 Mon Sep 17 00:00:00 2001 From: Max Winkler <82551778+MaxEW707@users.noreply.github.com> Date: Wed, 6 Mar 2024 10:58:12 -0500 Subject: [PATCH 312/929] [clang][Sema] Warn on self move for inlined static cast (#76646) There are code bases that inline `std::move` manually via `static_cast`. Treat a static cast to an xvalue as an inlined `std::move` call and warn on a self move. --- clang/lib/Sema/SemaChecking.cpp | 19 +++++++++---------- clang/test/SemaCXX/warn-self-move.cpp | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index 6ed61a19ff17..3597f93a0171 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -19093,18 +19093,17 @@ void Sema::DiagnoseSelfMove(const Expr *LHSExpr, const Expr *RHSExpr, LHSExpr = LHSExpr->IgnoreParenImpCasts(); RHSExpr = RHSExpr->IgnoreParenImpCasts(); - // Check for a call expression - const CallExpr *CE = dyn_cast(RHSExpr); - if (!CE || CE->getNumArgs() != 1) - return; - - // Check for a call to std::move - if (!CE->isCallToStdMove()) + // Check for a call to std::move or for a static_cast(..) to an xvalue + // which we can treat as an inlined std::move + if (const auto *CE = dyn_cast(RHSExpr); + CE && CE->getNumArgs() == 1 && CE->isCallToStdMove()) + RHSExpr = CE->getArg(0); + else if (const auto *CXXSCE = dyn_cast(RHSExpr); + CXXSCE && CXXSCE->isXValue()) + RHSExpr = CXXSCE->getSubExpr(); + else return; - // Get argument from std::move - RHSExpr = CE->getArg(0); - const DeclRefExpr *LHSDeclRef = dyn_cast(LHSExpr); const DeclRefExpr *RHSDeclRef = dyn_cast(RHSExpr); diff --git a/clang/test/SemaCXX/warn-self-move.cpp b/clang/test/SemaCXX/warn-self-move.cpp index 0987e9b6bf60..5937bb705ed2 100644 --- a/clang/test/SemaCXX/warn-self-move.cpp +++ b/clang/test/SemaCXX/warn-self-move.cpp @@ -16,6 +16,9 @@ void int_test() { x = std::move(x); // expected-warning{{explicitly moving}} (x) = std::move(x); // expected-warning{{explicitly moving}} + x = static_cast(x); // expected-warning{{explicitly moving}} + (x) = static_cast(x); // expected-warning{{explicitly moving}} + using std::move; x = move(x); // expected-warning{{explicitly moving}} \ expected-warning {{unqualified call to 'std::move}} @@ -26,6 +29,9 @@ void global_int_test() { global = std::move(global); // expected-warning{{explicitly moving}} (global) = std::move(global); // expected-warning{{explicitly moving}} + global = static_cast(global); // expected-warning{{explicitly moving}} + (global) = static_cast(global); // expected-warning{{explicitly moving}} + using std::move; global = move(global); // expected-warning{{explicitly moving}} \ expected-warning {{unqualified call to 'std::move}} @@ -35,11 +41,16 @@ class field_test { int x; field_test(field_test&& other) { x = std::move(x); // expected-warning{{explicitly moving}} + x = static_cast(x); // expected-warning{{explicitly moving}} x = std::move(other.x); + x = static_cast(other.x); other.x = std::move(x); + other.x = static_cast(x); other.x = std::move(other.x); // expected-warning{{explicitly moving}} + other.x = static_cast(other.x); // expected-warning{{explicitly moving}} } void withSuggest(int x) { + x = static_cast(x); // expected-warning{{explicitly moving variable of type 'int' to itself; did you mean to move to member 'x'?}} x = std::move(x); // expected-warning{{explicitly moving variable of type 'int' to itself; did you mean to move to member 'x'?}} } }; @@ -50,11 +61,15 @@ struct C { C() {}; ~C() {} }; void struct_test() { A a; a = std::move(a); // expected-warning{{explicitly moving}} + a = static_cast(a); // expected-warning{{explicitly moving}} B b; b = std::move(b); // expected-warning{{explicitly moving}} + b = static_cast(b); // expected-warning{{explicitly moving}} b.a = std::move(b.a); // expected-warning{{explicitly moving}} + b.a = static_cast(b.a); // expected-warning{{explicitly moving}} C c; c = std::move(c); // expected-warning{{explicitly moving}} + c = static_cast(c); // expected-warning{{explicitly moving}} } -- GitLab From d4687fe7d1639ea5d16190c89a54de1f2c6e2a9a Mon Sep 17 00:00:00 2001 From: Exile <2094247798@qq.com> Date: Thu, 7 Mar 2024 00:01:30 +0800 Subject: [PATCH 313/929] [analyzer] Fix crash on dereference invalid return value of getAdjustedParameterIndex() (#83585) Fixes #78810 Thanks for Snape3058 's comment --------- Co-authored-by: miaozhiyuan --- clang/lib/StaticAnalyzer/Core/CallEvent.cpp | 2 +- clang/test/Analysis/cxx2b-deducing-this.cpp | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp index 0ac1d91b79be..bc14aea27f67 100644 --- a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp +++ b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp @@ -1409,7 +1409,7 @@ CallEventManager::getSimpleCall(const CallExpr *CE, ProgramStateRef State, if (const auto *OpCE = dyn_cast(CE)) { const FunctionDecl *DirectCallee = OpCE->getDirectCallee(); if (const auto *MD = dyn_cast(DirectCallee)) - if (MD->isInstance()) + if (MD->isImplicitObjectMemberFunction()) return create(OpCE, State, LCtx, ElemRef); } else if (CE->getCallee()->getType()->isBlockPointerType()) { diff --git a/clang/test/Analysis/cxx2b-deducing-this.cpp b/clang/test/Analysis/cxx2b-deducing-this.cpp index d22a897097be..2ec9e96bf0f8 100644 --- a/clang/test/Analysis/cxx2b-deducing-this.cpp +++ b/clang/test/Analysis/cxx2b-deducing-this.cpp @@ -60,3 +60,14 @@ void top() { s.c(); s.c(11); } + + +struct S2 { + bool operator==(this auto, S2) { + return true; + } +}; +void use_deducing_this() { + int result = S2{} == S2{}; // no-crash + clang_analyzer_dump(result); // expected-warning {{1 S32b}} +} -- GitLab From 28e30b4042742bb79cc66b92a16a8508aa35cb92 Mon Sep 17 00:00:00 2001 From: ZERO-N <1416312228@qq.com> Date: Thu, 7 Mar 2024 00:07:15 +0800 Subject: [PATCH 314/929] [Clang][Sema][NFC]Use isa_and_nonnull to simplify the code and avoid build warning (#84181) Signed-off-by: nwh --- clang/lib/Sema/SemaExceptionSpec.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clang/lib/Sema/SemaExceptionSpec.cpp b/clang/lib/Sema/SemaExceptionSpec.cpp index 3563b4f683f0..00384f9dc16a 100644 --- a/clang/lib/Sema/SemaExceptionSpec.cpp +++ b/clang/lib/Sema/SemaExceptionSpec.cpp @@ -1017,13 +1017,13 @@ CanThrowResult Sema::canCalleeThrow(Sema &S, const Expr *E, const Decl *D, SourceLocation Loc) { // As an extension, we assume that __attribute__((nothrow)) functions don't // throw. - if (D && isa(D) && D->hasAttr()) + if (isa_and_nonnull(D) && D->hasAttr()) return CT_Cannot; QualType T; // In C++1z, just look at the function type of the callee. - if (S.getLangOpts().CPlusPlus17 && E && isa(E)) { + if (S.getLangOpts().CPlusPlus17 && isa_and_nonnull(E)) { E = cast(E)->getCallee(); T = E->getType(); if (T->isSpecificPlaceholderType(BuiltinType::BoundMember)) { -- GitLab From 1c6e09c27f2216070abe5c268fb6269aeea97e68 Mon Sep 17 00:00:00 2001 From: Slava Zakharin Date: Wed, 6 Mar 2024 08:17:09 -0800 Subject: [PATCH 315/929] [flang] Added COMPLEX(16) ** INTEGER(4/8) lowering and runtime. (#84115) --- flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 8 ++++ flang/runtime/complex-powi.cpp | 45 ++++++++++++++++++- .../test/Lower/Intrinsics/pow_complex16i.f90 | 9 ++++ .../test/Lower/Intrinsics/pow_complex16k.f90 | 9 ++++ 4 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 flang/test/Lower/Intrinsics/pow_complex16i.f90 create mode 100644 flang/test/Lower/Intrinsics/pow_complex16k.f90 diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp index 25598ed16831..2f7ace658e47 100644 --- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp +++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp @@ -930,6 +930,10 @@ constexpr auto FuncTypeComplex16Complex16 = genFuncType, Ty::Complex<16>>; constexpr auto FuncTypeComplex16Complex16Complex16 = genFuncType, Ty::Complex<16>, Ty::Complex<16>>; +constexpr auto FuncTypeComplex16Complex16Integer4 = + genFuncType, Ty::Complex<16>, Ty::Integer<4>>; +constexpr auto FuncTypeComplex16Complex16Integer8 = + genFuncType, Ty::Complex<16>, Ty::Integer<8>>; static constexpr MathOperation mathOperations[] = { {"abs", "fabsf", genFuncType, Ty::Real<4>>, @@ -1226,10 +1230,14 @@ static constexpr MathOperation mathOperations[] = { genFuncType, Ty::Complex<4>, Ty::Integer<4>>, genLibCall}, {"pow", RTNAME_STRING(zpowi), genFuncType, Ty::Complex<8>, Ty::Integer<4>>, genLibCall}, + {"pow", RTNAME_STRING(cqpowi), FuncTypeComplex16Complex16Integer4, + genLibF128Call}, {"pow", RTNAME_STRING(cpowk), genFuncType, Ty::Complex<4>, Ty::Integer<8>>, genLibCall}, {"pow", RTNAME_STRING(zpowk), genFuncType, Ty::Complex<8>, Ty::Integer<8>>, genLibCall}, + {"pow", RTNAME_STRING(cqpowk), FuncTypeComplex16Complex16Integer8, + genLibF128Call}, {"sign", "copysignf", genFuncType, Ty::Real<4>, Ty::Real<4>>, genMathOp}, {"sign", "copysign", genFuncType, Ty::Real<8>, Ty::Real<8>>, diff --git a/flang/runtime/complex-powi.cpp b/flang/runtime/complex-powi.cpp index 18723bb93cbf..77031e402427 100644 --- a/flang/runtime/complex-powi.cpp +++ b/flang/runtime/complex-powi.cpp @@ -6,6 +6,7 @@ * * ===-----------------------------------------------------------------------=== */ +#include "flang/Common/float128.h" #include "flang/Runtime/entry-names.h" #include #include @@ -79,6 +80,30 @@ extern "C" double _Complex RTNAME(zpowk)( double _Complex base, std::int64_t exp) { return tgpowi(base, exp); } + +#if LDBL_MANT_DIG == 113 || HAS_FLOAT128 +// Duplicate CFloat128ComplexType definition from flang/Common/float128.h. +// float128.h does not define it for C++, because _Complex triggers +// c99-extension warnings. We decided to disable warnings for this +// particular file, so we can use _Complex here. +#if LDBL_MANT_DIG == 113 +typedef long double _Complex Qcomplex; +#elif HAS_FLOAT128 +#if !defined(_ARCH_PPC) || defined(__LONG_DOUBLE_IEEE128__) +typedef _Complex float __attribute__((mode(TC))) Qcomplex; +#else +typedef _Complex float __attribute__((mode(KC))) Qcomplex; +#endif +#endif + +extern "C" Qcomplex RTNAME(cqpowi)(Qcomplex base, std::int32_t exp) { + return tgpowi(base, exp); +} +extern "C" Qcomplex RTNAME(cqpowk)(Qcomplex base, std::int64_t exp) { + return tgpowi(base, exp); +} +#endif + #else // on MSVC, C complex is always just a struct of two members as it is not // supported as a builtin type. So we use C++ complex here as that has the @@ -116,10 +141,28 @@ extern "C" Fcomplex RTNAME(cpowk)(Fcomplex base, std::int64_t exp) { return *(Fcomplex *)(&cppres); } -extern "C" Dcomplex RTNAME(zpowk)(Dcomplex base, std::int32_t exp) { +extern "C" Dcomplex RTNAME(zpowk)(Dcomplex base, std::int64_t exp) { auto cppbase = *(std::complex *)(&base); auto cppres = tgpowi(cppbase, exp); return *(Dcomplex *)(&cppres); } +#if LDBL_MANT_DIG == 113 || HAS_FLOAT128 +struct Qcomplex { + CFloat128Type re; + CFloat128Type im; +}; + +extern "C" Dcomplex RTNAME(cqpowi)(Qcomplex base, std::int32_t exp) { + auto cppbase = *(std::complex *)(&base); + auto cppres = tgpowi(cppbase, exp); + return *(Qcomplex *)(&cppres); +} + +extern "C" Dcomplex RTNAME(cqpowk)(Qcomplex base, std::int64_t exp) { + auto cppbase = *(std::complex *)(&base); + auto cppres = tgpowi(cppbase, exp); + return *(Qcomplex *)(&cppres); +} +#endif #endif diff --git a/flang/test/Lower/Intrinsics/pow_complex16i.f90 b/flang/test/Lower/Intrinsics/pow_complex16i.f90 new file mode 100644 index 000000000000..1cabaf94f8a6 --- /dev/null +++ b/flang/test/Lower/Intrinsics/pow_complex16i.f90 @@ -0,0 +1,9 @@ +! RUN: bbc -emit-fir %s -o - | FileCheck %s +! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s + +! CHECK: fir.call @_FortranAcqpowi({{.*}}){{.*}}: (!fir.complex<16>, i32) -> !fir.complex<16> + complex(16) :: a + integer(4) :: b + b = a ** b +end diff --git a/flang/test/Lower/Intrinsics/pow_complex16k.f90 b/flang/test/Lower/Intrinsics/pow_complex16k.f90 new file mode 100644 index 000000000000..e3b1b4d31afa --- /dev/null +++ b/flang/test/Lower/Intrinsics/pow_complex16k.f90 @@ -0,0 +1,9 @@ +! RUN: bbc -emit-fir %s -o - | FileCheck %s +! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s +! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s + +! CHECK: fir.call @_FortranAcqpowk({{.*}}){{.*}}: (!fir.complex<16>, i64) -> !fir.complex<16> + complex(16) :: a + integer(8) :: b + b = a ** b +end -- GitLab From 8ee7ef6a373eb36f6945bc5cf0adcb3b94daf120 Mon Sep 17 00:00:00 2001 From: David Green Date: Wed, 6 Mar 2024 16:36:20 +0000 Subject: [PATCH 316/929] [GlobalISel] Use LocationSize in GISelAddressing. NFC (#83885) This is similar to #83017 but for the areas in GlobalISel's LoadStoreOpt, and should help simplify #70452 a little. It will likely change a little again once the sizes can be scalable. --- llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp | 51 +++++++++++--------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp index 246aa88b09ac..b5c9d3e912cc 100644 --- a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp @@ -117,8 +117,12 @@ bool GISelAddressing::aliasIsKnownForLoadStore(const MachineInstr &MI1, if (!BasePtr0.BaseReg.isValid() || !BasePtr1.BaseReg.isValid()) return false; - int64_t Size1 = LdSt1->getMemSize(); - int64_t Size2 = LdSt2->getMemSize(); + LocationSize Size1 = LdSt1->getMemSize() != MemoryLocation::UnknownSize + ? LdSt1->getMemSize() + : LocationSize::beforeOrAfterPointer(); + LocationSize Size2 = LdSt2->getMemSize() != MemoryLocation::UnknownSize + ? LdSt2->getMemSize() + : LocationSize::beforeOrAfterPointer(); int64_t PtrDiff; if (BasePtr0.BaseReg == BasePtr1.BaseReg) { @@ -128,20 +132,18 @@ bool GISelAddressing::aliasIsKnownForLoadStore(const MachineInstr &MI1, // vector objects on the stack. // BasePtr1 is PtrDiff away from BasePtr0. They alias if none of the // following situations arise: - if (PtrDiff >= 0 && - Size1 != static_cast(MemoryLocation::UnknownSize)) { + if (PtrDiff >= 0 && Size1.hasValue()) { // [----BasePtr0----] // [---BasePtr1--] // ========PtrDiff========> - IsAlias = !(Size1 <= PtrDiff); + IsAlias = !((int64_t)Size1.getValue() <= PtrDiff); return true; } - if (PtrDiff < 0 && - Size2 != static_cast(MemoryLocation::UnknownSize)) { + if (PtrDiff < 0 && Size2.hasValue()) { // [----BasePtr0----] // [---BasePtr1--] // =====(-PtrDiff)====> - IsAlias = !((PtrDiff + Size2) <= 0); + IsAlias = !((PtrDiff + (int64_t)Size2.getValue()) <= 0); return true; } return false; @@ -196,7 +198,7 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI, bool IsAtomic; Register BasePtr; int64_t Offset; - uint64_t NumBytes; + LocationSize NumBytes; MachineMemOperand *MMO; }; @@ -212,16 +214,22 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI, Offset = 0; } - uint64_t Size = MemoryLocation::getSizeOrUnknown( - LS->getMMO().getMemoryType().getSizeInBytes()); - return {LS->isVolatile(), LS->isAtomic(), BaseReg, - Offset /*base offset*/, Size, &LS->getMMO()}; + TypeSize Size = LS->getMMO().getMemoryType().getSizeInBytes(); + return {LS->isVolatile(), + LS->isAtomic(), + BaseReg, + Offset /*base offset*/, + Size.isScalable() ? LocationSize::beforeOrAfterPointer() + : LocationSize::precise(Size), + &LS->getMMO()}; } // FIXME: support recognizing lifetime instructions. // Default. return {false /*isvolatile*/, - /*isAtomic*/ false, Register(), - (int64_t)0 /*offset*/, 0 /*size*/, + /*isAtomic*/ false, + Register(), + (int64_t)0 /*offset*/, + LocationSize::beforeOrAfterPointer() /*size*/, (MachineMemOperand *)nullptr}; }; MemUseCharacteristics MUC0 = getCharacteristics(&MI), @@ -262,15 +270,14 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI, // FIXME: port the alignment based alias analysis from SDAG's isAlias(). int64_t SrcValOffset0 = MUC0.MMO->getOffset(); int64_t SrcValOffset1 = MUC1.MMO->getOffset(); - uint64_t Size0 = MUC0.NumBytes; - uint64_t Size1 = MUC1.NumBytes; - if (AA && MUC0.MMO->getValue() && MUC1.MMO->getValue() && - Size0 != MemoryLocation::UnknownSize && - Size1 != MemoryLocation::UnknownSize) { + LocationSize Size0 = MUC0.NumBytes; + LocationSize Size1 = MUC1.NumBytes; + if (AA && MUC0.MMO->getValue() && MUC1.MMO->getValue() && Size0.hasValue() && + Size1.hasValue()) { // Use alias analysis information. int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1); - int64_t Overlap0 = Size0 + SrcValOffset0 - MinOffset; - int64_t Overlap1 = Size1 + SrcValOffset1 - MinOffset; + int64_t Overlap0 = Size0.getValue() + SrcValOffset0 - MinOffset; + int64_t Overlap1 = Size1.getValue() + SrcValOffset1 - MinOffset; if (AA->isNoAlias(MemoryLocation(MUC0.MMO->getValue(), Overlap0, MUC0.MMO->getAAInfo()), MemoryLocation(MUC1.MMO->getValue(), Overlap1, -- GitLab From 571d5af5aa3ebfdf59d2eccb6a90c2b97c6d7ea8 Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Wed, 6 Mar 2024 16:37:43 +0000 Subject: [PATCH 317/929] [DAGCombiner] Improve comment on reassociateOps and its helper --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 932944d4b6f3..cdcb71146404 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -1145,8 +1145,8 @@ bool DAGCombiner::reassociationCanBreakAddressingModePattern(unsigned Opc, return false; } -// Helper for DAGCombiner::reassociateOps. Try to reassociate an expression -// such as (Opc N0, N1), if \p N0 is the same kind of operation as \p Opc. +/// Helper for DAGCombiner::reassociateOps. Try to reassociate (Opc N0, N1) if +/// \p N0 is the same kind of operation as \p Opc. SDValue DAGCombiner::reassociateOpsCommutative(unsigned Opc, const SDLoc &DL, SDValue N0, SDValue N1, SDNodeFlags Flags) { @@ -1244,7 +1244,8 @@ SDValue DAGCombiner::reassociateOpsCommutative(unsigned Opc, const SDLoc &DL, return SDValue(); } -// Try to reassociate commutative binops. +/// Try to reassociate commutative (Opc N0, N1) if either \p N0 or \p N1 is the +/// same kind of operation as \p Opc. SDValue DAGCombiner::reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0, SDValue N1, SDNodeFlags Flags) { assert(TLI.isCommutativeBinOp(Opc) && "Operation not commutative."); -- GitLab From 67c82d6ffb4bbc21212116a11f390761d859297a Mon Sep 17 00:00:00 2001 From: Krzysztof Parzyszek Date: Wed, 6 Mar 2024 10:46:26 -0600 Subject: [PATCH 318/929] [Frontend] Add leaf constructs and association to OpenMP/ACC directives (#83625) Add members "leafConstructs" and "association" to .td describing OpenMP/ACC directives. The naming follows the terminology used in the OpenMP standard: a "leaf" construct is a construct that is itself not a composition or a combination of other constructs, and "association" is the source language construct to which the directive applies (e.g. loop, block, etc.) The tblgen-generated output then contains two additional functions - getLeafConstructs(D), and - getDirectiveAssociation(D) plus "enum class Association", all in namespaces "llvm::omp" and "llvm::acc". Note: getLeafConstructs returns an empty sequence for a construct that is itself a leaf construct. Use the new functions to simplify a few OpenMP-related functions in clang. --- clang/lib/Basic/OpenMPKinds.cpp | 129 +++------ .../llvm/Frontend/Directive/DirectiveBase.td | 36 +++ llvm/include/llvm/Frontend/OpenACC/ACC.td | 27 +- llvm/include/llvm/Frontend/OpenMP/OMP.td | 172 ++++++++++-- llvm/include/llvm/TableGen/DirectiveEmitter.h | 10 + llvm/test/TableGen/directive1.td | 33 +++ llvm/test/TableGen/directive2.td | 34 +++ llvm/utils/TableGen/DirectiveEmitter.cpp | 244 +++++++++++++++++- 8 files changed, 562 insertions(+), 123 deletions(-) diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp index 6c31b0824eb8..b3e9affbb3e5 100644 --- a/clang/lib/Basic/OpenMPKinds.cpp +++ b/clang/lib/Basic/OpenMPKinds.cpp @@ -574,31 +574,7 @@ const char *clang::getOpenMPSimpleClauseTypeName(OpenMPClauseKind Kind, } bool clang::isOpenMPLoopDirective(OpenMPDirectiveKind DKind) { - return DKind == OMPD_simd || DKind == OMPD_for || DKind == OMPD_for_simd || - DKind == OMPD_parallel_for || DKind == OMPD_parallel_for_simd || - DKind == OMPD_taskloop || DKind == OMPD_taskloop_simd || - DKind == OMPD_master_taskloop || DKind == OMPD_master_taskloop_simd || - DKind == OMPD_parallel_master_taskloop || - DKind == OMPD_parallel_master_taskloop_simd || - DKind == OMPD_masked_taskloop || DKind == OMPD_masked_taskloop_simd || - DKind == OMPD_parallel_masked_taskloop || DKind == OMPD_distribute || - DKind == OMPD_parallel_masked_taskloop_simd || - DKind == OMPD_target_parallel_for || - DKind == OMPD_distribute_parallel_for || - DKind == OMPD_distribute_parallel_for_simd || - DKind == OMPD_distribute_simd || - DKind == OMPD_target_parallel_for_simd || DKind == OMPD_target_simd || - DKind == OMPD_teams_distribute || - DKind == OMPD_teams_distribute_simd || - DKind == OMPD_teams_distribute_parallel_for_simd || - DKind == OMPD_teams_distribute_parallel_for || - DKind == OMPD_target_teams_distribute || - DKind == OMPD_target_teams_distribute_parallel_for || - DKind == OMPD_target_teams_distribute_parallel_for_simd || - DKind == OMPD_target_teams_distribute_simd || DKind == OMPD_tile || - DKind == OMPD_unroll || DKind == OMPD_loop || - DKind == OMPD_teams_loop || DKind == OMPD_target_teams_loop || - DKind == OMPD_parallel_loop || DKind == OMPD_target_parallel_loop; + return getDirectiveAssociation(DKind) == Association::Loop; } bool clang::isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind) { @@ -619,44 +595,20 @@ bool clang::isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind) { } bool clang::isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind) { - return DKind == OMPD_taskloop || DKind == OMPD_taskloop_simd || - DKind == OMPD_master_taskloop || DKind == OMPD_master_taskloop_simd || - DKind == OMPD_parallel_master_taskloop || - DKind == OMPD_masked_taskloop || DKind == OMPD_masked_taskloop_simd || - DKind == OMPD_parallel_masked_taskloop || - DKind == OMPD_parallel_masked_taskloop_simd || - DKind == OMPD_parallel_master_taskloop_simd; + return DKind == OMPD_taskloop || + llvm::is_contained(getLeafConstructs(DKind), OMPD_taskloop); } bool clang::isOpenMPParallelDirective(OpenMPDirectiveKind DKind) { - return DKind == OMPD_parallel || DKind == OMPD_parallel_for || - DKind == OMPD_parallel_for_simd || DKind == OMPD_parallel_sections || - DKind == OMPD_target_parallel || DKind == OMPD_target_parallel_for || - DKind == OMPD_distribute_parallel_for || - DKind == OMPD_distribute_parallel_for_simd || - DKind == OMPD_target_parallel_for_simd || - DKind == OMPD_teams_distribute_parallel_for || - DKind == OMPD_teams_distribute_parallel_for_simd || - DKind == OMPD_target_teams_distribute_parallel_for || - DKind == OMPD_target_teams_distribute_parallel_for_simd || - DKind == OMPD_parallel_master || DKind == OMPD_parallel_masked || - DKind == OMPD_parallel_master_taskloop || - DKind == OMPD_parallel_master_taskloop_simd || - DKind == OMPD_parallel_masked_taskloop || - DKind == OMPD_parallel_masked_taskloop_simd || - DKind == OMPD_parallel_loop || DKind == OMPD_target_parallel_loop || - DKind == OMPD_teams_loop; + if (DKind == OMPD_teams_loop) + return true; + return DKind == OMPD_parallel || + llvm::is_contained(getLeafConstructs(DKind), OMPD_parallel); } bool clang::isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind) { - return DKind == OMPD_target || DKind == OMPD_target_parallel || - DKind == OMPD_target_parallel_for || - DKind == OMPD_target_parallel_for_simd || DKind == OMPD_target_simd || - DKind == OMPD_target_teams || DKind == OMPD_target_teams_distribute || - DKind == OMPD_target_teams_distribute_parallel_for || - DKind == OMPD_target_teams_distribute_parallel_for_simd || - DKind == OMPD_target_teams_distribute_simd || - DKind == OMPD_target_teams_loop || DKind == OMPD_target_parallel_loop; + return DKind == OMPD_target || + llvm::is_contained(getLeafConstructs(DKind), OMPD_target); } bool clang::isOpenMPTargetDataManagementDirective(OpenMPDirectiveKind DKind) { @@ -665,60 +617,45 @@ bool clang::isOpenMPTargetDataManagementDirective(OpenMPDirectiveKind DKind) { } bool clang::isOpenMPNestingTeamsDirective(OpenMPDirectiveKind DKind) { - return DKind == OMPD_teams || DKind == OMPD_teams_distribute || - DKind == OMPD_teams_distribute_simd || - DKind == OMPD_teams_distribute_parallel_for_simd || - DKind == OMPD_teams_distribute_parallel_for || - DKind == OMPD_teams_loop; + if (DKind == OMPD_teams) + return true; + ArrayRef Leaves = getLeafConstructs(DKind); + return !Leaves.empty() && Leaves.front() == OMPD_teams; } bool clang::isOpenMPTeamsDirective(OpenMPDirectiveKind DKind) { - return isOpenMPNestingTeamsDirective(DKind) || DKind == OMPD_target_teams || - DKind == OMPD_target_teams_distribute || - DKind == OMPD_target_teams_distribute_parallel_for || - DKind == OMPD_target_teams_distribute_parallel_for_simd || - DKind == OMPD_target_teams_distribute_simd || - DKind == OMPD_target_teams_loop; + return DKind == OMPD_teams || + llvm::is_contained(getLeafConstructs(DKind), OMPD_teams); } bool clang::isOpenMPSimdDirective(OpenMPDirectiveKind DKind) { - return DKind == OMPD_simd || DKind == OMPD_for_simd || - DKind == OMPD_parallel_for_simd || DKind == OMPD_taskloop_simd || - DKind == OMPD_master_taskloop_simd || - DKind == OMPD_masked_taskloop_simd || - DKind == OMPD_parallel_master_taskloop_simd || - DKind == OMPD_parallel_masked_taskloop_simd || - DKind == OMPD_distribute_parallel_for_simd || - DKind == OMPD_distribute_simd || DKind == OMPD_target_simd || - DKind == OMPD_teams_distribute_simd || - DKind == OMPD_teams_distribute_parallel_for_simd || - DKind == OMPD_target_teams_distribute_parallel_for_simd || - DKind == OMPD_target_teams_distribute_simd || - DKind == OMPD_target_parallel_for_simd; + // Avoid OMPD_declare_simd + if (getDirectiveAssociation(DKind) != Association::Loop) + return false; + // Formally, OMPD_end_do_simd also has a loop association, but + // it's a Fortran-specific directive. + + return DKind == OMPD_simd || + llvm::is_contained(getLeafConstructs(DKind), OMPD_simd); } bool clang::isOpenMPNestingDistributeDirective(OpenMPDirectiveKind Kind) { - return Kind == OMPD_distribute || Kind == OMPD_distribute_parallel_for || - Kind == OMPD_distribute_parallel_for_simd || - Kind == OMPD_distribute_simd; - // TODO add next directives. + if (Kind == OMPD_distribute) + return true; + ArrayRef Leaves = getLeafConstructs(Kind); + return !Leaves.empty() && Leaves.front() == OMPD_distribute; } bool clang::isOpenMPDistributeDirective(OpenMPDirectiveKind Kind) { - return isOpenMPNestingDistributeDirective(Kind) || - Kind == OMPD_teams_distribute || Kind == OMPD_teams_distribute_simd || - Kind == OMPD_teams_distribute_parallel_for_simd || - Kind == OMPD_teams_distribute_parallel_for || - Kind == OMPD_target_teams_distribute || - Kind == OMPD_target_teams_distribute_parallel_for || - Kind == OMPD_target_teams_distribute_parallel_for_simd || - Kind == OMPD_target_teams_distribute_simd; + return Kind == OMPD_distribute || + llvm::is_contained(getLeafConstructs(Kind), OMPD_distribute); } bool clang::isOpenMPGenericLoopDirective(OpenMPDirectiveKind Kind) { - return Kind == OMPD_loop || Kind == OMPD_teams_loop || - Kind == OMPD_target_teams_loop || Kind == OMPD_parallel_loop || - Kind == OMPD_target_parallel_loop; + if (Kind == OMPD_loop) + return true; + ArrayRef Leaves = getLeafConstructs(Kind); + return !Leaves.empty() && Leaves.back() == OMPD_loop; } bool clang::isOpenMPPrivate(OpenMPClauseKind Kind) { diff --git a/llvm/include/llvm/Frontend/Directive/DirectiveBase.td b/llvm/include/llvm/Frontend/Directive/DirectiveBase.td index 31578710365b..ce532e0cfae2 100644 --- a/llvm/include/llvm/Frontend/Directive/DirectiveBase.td +++ b/llvm/include/llvm/Frontend/Directive/DirectiveBase.td @@ -127,6 +127,35 @@ class VersionedClause { int maxVersion = max; } +// Kinds of directive associations. +class Association { + string name = n; // Name of the enum value in enum class Association. +} +// All of the AS_Xyz names are recognized by TableGen in order to calculate +// the association in the AS_FromLeaves case. +def AS_None : Association<"None"> {} // No association +def AS_Block : Association<"Block"> {} // Block (incl. single + // statement) +def AS_Declaration : Association<"Declaration"> {} // Declaration +def AS_Delimited : Association<"Delimited"> {} // Region delimited with + // begin/end +def AS_Loop : Association<"Loop"> {} // Loop +def AS_Separating : Association<"Separating"> {} // Separates parts of a + // construct + +def AS_FromLeaves : Association<"FromLeaves"> {} // See below +// AS_FromLeaves can be used for combined/composite directives, and the actual +// association will be computed based on associations of the leaf constructs: +// (x + y) + z = x + (y + z) +// x + y = y + x +// x + x = x +// AS_None + x = x +// AS_Block + AS_Loop = AS_Loop +// Other combinations are not allowed. +// This association is not valid for leaf constructs. +// The name "AS_FromLeaves" is recognized by TableGen, and there is no enum +// generated for it. + // Information about a specific directive. class Directive { // Name of the directive. Can be composite directive sepearted by whitespace. @@ -152,6 +181,13 @@ class Directive { // List of clauses that are required. list requiredClauses = []; + // List of leaf constituent directives in the order in which they appear + // in the combined/composite directive. + list leafConstructs = []; + // Set directive used by default when unknown. bit isDefault = false; + + // What the directive is associated with. + Association association = AS_FromLeaves; } diff --git a/llvm/include/llvm/Frontend/OpenACC/ACC.td b/llvm/include/llvm/Frontend/OpenACC/ACC.td index 0dbd934d83f0..dfa6a222e9f7 100644 --- a/llvm/include/llvm/Frontend/OpenACC/ACC.td +++ b/llvm/include/llvm/Frontend/OpenACC/ACC.td @@ -266,7 +266,9 @@ def ACCC_Unknown : Clause<"unknown"> { //===----------------------------------------------------------------------===// // 2.12 -def ACC_Atomic : Directive<"atomic"> {} +def ACC_Atomic : Directive<"atomic"> { + let association = AS_Block; +} // 2.6.5 def ACC_Data : Directive<"data"> { @@ -290,6 +292,7 @@ def ACC_Data : Directive<"data"> { VersionedClause, VersionedClause ]; + let association = AS_Block; } // 2.13 @@ -304,6 +307,7 @@ def ACC_Declare : Directive<"declare"> { VersionedClause, VersionedClause ]; + let association = AS_None; } // 2.5.3 @@ -329,6 +333,7 @@ def ACC_Kernels : Directive<"kernels"> { VersionedClause, VersionedClause ]; + let association = AS_Block; } // 2.5.1 @@ -357,6 +362,7 @@ def ACC_Parallel : Directive<"parallel"> { VersionedClause, VersionedClause ]; + let association = AS_Block; } // 2.5.2 @@ -384,6 +390,7 @@ def ACC_Serial : Directive<"serial"> { VersionedClause, VersionedClause ]; + let association = AS_Block; } // 2.9 @@ -403,10 +410,13 @@ def ACC_Loop : Directive<"loop"> { VersionedClause, VersionedClause ]; + let association = AS_Loop; } // 2.10 -def ACC_Cache : Directive<"cache"> {} +def ACC_Cache : Directive<"cache"> { + let association = AS_None; +} // 2.14.1 def ACC_Init : Directive<"init"> { @@ -415,6 +425,7 @@ def ACC_Init : Directive<"init"> { VersionedClause, VersionedClause ]; + let association = AS_None; } // 2.15.1 @@ -430,6 +441,7 @@ def ACC_Routine : Directive<"routine"> { let allowedOnceClauses = [ VersionedClause ]; + let association = AS_Declaration; } // 2.14.3 @@ -448,6 +460,7 @@ def ACC_Set : Directive<"set"> { VersionedClause, VersionedClause ]; + let association = AS_None; } // 2.14.2 @@ -457,6 +470,7 @@ def ACC_Shutdown : Directive<"shutdown"> { VersionedClause, VersionedClause ]; + let association = AS_None; } // 2.14.4 @@ -475,6 +489,7 @@ def ACC_Update : Directive<"update"> { VersionedClause, VersionedClause ]; + let association = AS_None; } // 2.16.3 @@ -483,6 +498,7 @@ def ACC_Wait : Directive<"wait"> { VersionedClause, VersionedClause ]; + let association = AS_None; } // 2.14.6 @@ -499,6 +515,7 @@ def ACC_EnterData : Directive<"enter data"> { VersionedClause, VersionedClause ]; + let association = AS_None; } // 2.14.7 @@ -516,6 +533,7 @@ def ACC_ExitData : Directive<"exit data"> { VersionedClause, VersionedClause ]; + let association = AS_None; } // 2.8 @@ -527,6 +545,7 @@ def ACC_HostData : Directive<"host_data"> { let requiredClauses = [ VersionedClause ]; + let association = AS_Block; } // 2.11 @@ -564,6 +583,7 @@ def ACC_KernelsLoop : Directive<"kernels loop"> { VersionedClause, VersionedClause ]; + let leafConstructs = [ACC_Kernels, ACC_Loop]; } // 2.11 @@ -602,6 +622,7 @@ def ACC_ParallelLoop : Directive<"parallel loop"> { VersionedClause, VersionedClause ]; + let leafConstructs = [ACC_Parallel, ACC_Loop]; } // 2.11 @@ -637,8 +658,10 @@ def ACC_SerialLoop : Directive<"serial loop"> { VersionedClause, VersionedClause ]; + let leafConstructs = [ACC_Serial, ACC_Loop]; } def ACC_Unknown : Directive<"unknown"> { let isDefault = true; + let association = AS_None; } diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td index 77d207f2b10a..d9a931438b42 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMP.td +++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td @@ -463,7 +463,9 @@ def OMPC_OMX_Bare : Clause<"ompx_bare"> { // Definition of OpenMP directives //===----------------------------------------------------------------------===// -def OMP_ThreadPrivate : Directive<"threadprivate"> {} +def OMP_ThreadPrivate : Directive<"threadprivate"> { + let association = AS_None; +} def OMP_Parallel : Directive<"parallel"> { let allowedClauses = [ VersionedClause, @@ -480,6 +482,7 @@ def OMP_Parallel : Directive<"parallel"> { VersionedClause, VersionedClause, ]; + let association = AS_Block; } def OMP_Task : Directive<"task"> { let allowedClauses = [ @@ -500,6 +503,7 @@ def OMP_Task : Directive<"task"> { VersionedClause, VersionedClause ]; + let association = AS_Block; } def OMP_Simd : Directive<"simd"> { let allowedClauses = [ @@ -518,17 +522,20 @@ def OMP_Simd : Directive<"simd"> { VersionedClause, VersionedClause ]; + let association = AS_Loop; } def OMP_Tile : Directive<"tile"> { let allowedOnceClauses = [ VersionedClause, ]; + let association = AS_Loop; } def OMP_Unroll : Directive<"unroll"> { let allowedOnceClauses = [ VersionedClause, VersionedClause, ]; + let association = AS_Loop; } def OMP_For : Directive<"for"> { let allowedClauses = [ @@ -544,6 +551,7 @@ def OMP_For : Directive<"for"> { VersionedClause, VersionedClause ]; + let association = AS_Loop; } def OMP_Do : Directive<"do"> { let allowedClauses = [ @@ -560,6 +568,7 @@ def OMP_Do : Directive<"do"> { VersionedClause, VersionedClause ]; + let association = AS_Loop; } def OMP_Sections : Directive<"sections"> { let allowedClauses = [ @@ -570,8 +579,11 @@ def OMP_Sections : Directive<"sections"> { VersionedClause, VersionedClause ]; + let association = AS_Block; +} +def OMP_Section : Directive<"section"> { + let association = AS_Separating; } -def OMP_Section : Directive<"section"> {} def OMP_Single : Directive<"single"> { let allowedClauses = [ VersionedClause, @@ -580,33 +592,44 @@ def OMP_Single : Directive<"single"> { VersionedClause, VersionedClause ]; + let association = AS_Block; +} +def OMP_Master : Directive<"master"> { + let association = AS_Block; } -def OMP_Master : Directive<"master"> {} def OMP_Critical : Directive<"critical"> { let allowedClauses = [ VersionedClause ]; + let association = AS_Block; +} +def OMP_TaskYield : Directive<"taskyield"> { + let association = AS_None; +} +def OMP_Barrier : Directive<"barrier"> { + let association = AS_None; } -def OMP_TaskYield : Directive<"taskyield"> {} -def OMP_Barrier : Directive<"barrier"> {} def OMP_Error : Directive<"error"> { let allowedClauses = [ VersionedClause, VersionedClause, VersionedClause ]; + let association = AS_None; } def OMP_TaskWait : Directive<"taskwait"> { let allowedClauses = [ VersionedClause, VersionedClause ]; + let association = AS_None; } def OMP_TaskGroup : Directive<"taskgroup"> { let allowedClauses = [ VersionedClause, VersionedClause ]; + let association = AS_Block; } def OMP_Flush : Directive<"flush"> { let allowedOnceClauses = [ @@ -617,6 +640,7 @@ def OMP_Flush : Directive<"flush"> { // OMPKinds.def. VersionedClause ]; + let association = AS_None; } def OMP_Ordered : Directive<"ordered"> { let allowedClauses = [ @@ -627,6 +651,8 @@ def OMP_Ordered : Directive<"ordered"> { VersionedClause, VersionedClause ]; + let association = AS_None; + // There is also a block-associated "ordered" directive. } def OMP_Atomic : Directive<"atomic"> { let allowedClauses = [ @@ -646,6 +672,7 @@ def OMP_Atomic : Directive<"atomic"> { VersionedClause, VersionedClause ]; + let association = AS_Block; } def OMP_Target : Directive<"target"> { let allowedClauses = [ @@ -669,6 +696,7 @@ def OMP_Target : Directive<"target"> { VersionedClause, VersionedClause, ]; + let association = AS_Block; } def OMP_Teams : Directive<"teams"> { let allowedClauses = [ @@ -685,11 +713,13 @@ def OMP_Teams : Directive<"teams"> { VersionedClause, VersionedClause ]; + let association = AS_Block; } def OMP_Cancel : Directive<"cancel"> { let allowedOnceClauses = [ VersionedClause ]; + let association = AS_None; } def OMP_Requires : Directive<"requires"> { let allowedOnceClauses = [ @@ -707,8 +737,11 @@ def OMP_Requires : Directive<"requires"> { VersionedClause, VersionedClause ]; + let association = AS_None; +} +def OMP_Nothing : Directive<"nothing"> { + let association = AS_None; } -def OMP_Nothing : Directive<"nothing"> {} def OMP_TargetData : Directive<"target data"> { let allowedOnceClauses = [ VersionedClause, @@ -719,6 +752,7 @@ def OMP_TargetData : Directive<"target data"> { VersionedClause, VersionedClause ]; + let association = AS_Block; } def OMP_TargetEnterData : Directive<"target enter data"> { let allowedClauses = [ @@ -732,6 +766,7 @@ def OMP_TargetEnterData : Directive<"target enter data"> { let requiredClauses = [ VersionedClause ]; + let association = AS_None; } def OMP_TargetExitData : Directive<"target exit data"> { let allowedClauses = [ @@ -745,6 +780,7 @@ def OMP_TargetExitData : Directive<"target exit data"> { let requiredClauses = [ VersionedClause ]; + let association = AS_None; } def OMP_TargetParallel : Directive<"target parallel"> { let allowedClauses = [ @@ -771,6 +807,7 @@ def OMP_TargetParallel : Directive<"target parallel"> { VersionedClause, VersionedClause, ]; + let leafConstructs = [OMP_Target, OMP_Parallel]; } def OMP_TargetParallelFor : Directive<"target parallel for"> { let allowedClauses = [ @@ -803,6 +840,7 @@ def OMP_TargetParallelFor : Directive<"target parallel for"> { VersionedClause, VersionedClause, ]; + let leafConstructs = [OMP_Target, OMP_Parallel, OMP_For]; } def OMP_TargetParallelDo : Directive<"target parallel do"> { let allowedClauses = [ @@ -833,6 +871,7 @@ def OMP_TargetParallelDo : Directive<"target parallel do"> { VersionedClause, VersionedClause ]; + let leafConstructs = [OMP_Target, OMP_Parallel, OMP_Do]; } def OMP_TargetUpdate : Directive<"target update"> { let allowedClauses = [ @@ -845,6 +884,13 @@ def OMP_TargetUpdate : Directive<"target update"> { VersionedClause, VersionedClause ]; + let association = AS_None; +} +def OMP_masked : Directive<"masked"> { + let allowedOnceClauses = [ + VersionedClause + ]; + let association = AS_Block; } def OMP_ParallelFor : Directive<"parallel for"> { let allowedClauses = [ @@ -866,6 +912,7 @@ def OMP_ParallelFor : Directive<"parallel for"> { VersionedClause, VersionedClause, ]; + let leafConstructs = [OMP_Parallel, OMP_For]; } def OMP_ParallelDo : Directive<"parallel do"> { let allowedClauses = [ @@ -887,6 +934,7 @@ def OMP_ParallelDo : Directive<"parallel do"> { VersionedClause, VersionedClause ]; + let leafConstructs = [OMP_Parallel, OMP_Do]; } def OMP_ParallelForSimd : Directive<"parallel for simd"> { let allowedClauses = [ @@ -912,6 +960,7 @@ def OMP_ParallelForSimd : Directive<"parallel for simd"> { VersionedClause, VersionedClause, ]; + let leafConstructs = [OMP_Parallel, OMP_For, OMP_Simd]; } def OMP_ParallelDoSimd : Directive<"parallel do simd"> { let allowedClauses = [ @@ -938,6 +987,7 @@ def OMP_ParallelDoSimd : Directive<"parallel do simd"> { VersionedClause, VersionedClause ]; + let leafConstructs = [OMP_Parallel, OMP_Do, OMP_Simd]; } def OMP_ParallelMaster : Directive<"parallel master"> { let allowedClauses = [ @@ -953,6 +1003,7 @@ def OMP_ParallelMaster : Directive<"parallel master"> { VersionedClause, VersionedClause, ]; + let leafConstructs = [OMP_Parallel, OMP_Master]; } def OMP_ParallelMasked : Directive<"parallel masked"> { let allowedClauses = [ @@ -969,6 +1020,7 @@ def OMP_ParallelMasked : Directive<"parallel masked"> { VersionedClause, VersionedClause, ]; + let leafConstructs = [OMP_Parallel, OMP_masked]; } def OMP_ParallelSections : Directive<"parallel sections"> { let allowedClauses = [ @@ -987,6 +1039,7 @@ def OMP_ParallelSections : Directive<"parallel sections"> { VersionedClause, VersionedClause ]; + let leafConstructs = [OMP_Parallel, OMP_Sections]; } def OMP_ForSimd : Directive<"for simd"> { let allowedClauses = [ @@ -1007,6 +1060,7 @@ def OMP_ForSimd : Directive<"for simd"> { VersionedClause, VersionedClause ]; + let leafConstructs = [OMP_For, OMP_Simd]; } def OMP_DoSimd : Directive<"do simd"> { let allowedClauses = [ @@ -1027,13 +1081,19 @@ def OMP_DoSimd : Directive<"do simd"> { VersionedClause, VersionedClause ]; + let leafConstructs = [OMP_Do, OMP_Simd]; +} +def OMP_CancellationPoint : Directive<"cancellation point"> { + let association = AS_None; +} +def OMP_DeclareReduction : Directive<"declare reduction"> { + let association = AS_None; } -def OMP_CancellationPoint : Directive<"cancellation point"> {} -def OMP_DeclareReduction : Directive<"declare reduction"> {} def OMP_DeclareMapper : Directive<"declare mapper"> { let allowedClauses = [ VersionedClause ]; + let association = AS_None; } def OMP_DeclareSimd : Directive<"declare simd"> { let allowedClauses = [ @@ -1048,6 +1108,7 @@ def OMP_DeclareSimd : Directive<"declare simd"> { VersionedClause, VersionedClause ]; + let association = AS_Declaration; } def OMP_TaskLoop : Directive<"taskloop"> { let allowedClauses = [ @@ -1073,6 +1134,7 @@ def OMP_TaskLoop : Directive<"taskloop"> { VersionedClause, VersionedClause ]; + let association = AS_Loop; } def OMP_TaskLoopSimd : Directive<"taskloop simd"> { let allowedClauses = [ @@ -1104,6 +1166,7 @@ def OMP_TaskLoopSimd : Directive<"taskloop simd"> { VersionedClause, VersionedClause ]; + let leafConstructs = [OMP_TaskLoop, OMP_Simd]; } def OMP_Distribute : Directive<"distribute"> { let allowedClauses = [ @@ -1116,6 +1179,7 @@ def OMP_Distribute : Directive<"distribute"> { VersionedClause, VersionedClause ]; + let association = AS_Loop; } def OMP_BeginDeclareTarget : Directive<"begin declare target"> { let allowedClauses = [ @@ -1124,6 +1188,7 @@ def OMP_BeginDeclareTarget : Directive<"begin declare target"> { VersionedClause, VersionedClause ]; + let association = AS_Delimited; } def OMP_DeclareTarget : Directive<"declare target"> { let allowedClauses = [ @@ -1135,8 +1200,11 @@ def OMP_DeclareTarget : Directive<"declare target"> { let allowedOnceClauses = [ VersionedClause ]; + let association = AS_None; +} +def OMP_EndDeclareTarget : Directive<"end declare target"> { + let association = AS_Delimited; } -def OMP_EndDeclareTarget : Directive<"end declare target"> {} def OMP_DistributeParallelFor : Directive<"distribute parallel for"> { let allowedClauses = [ VersionedClause, @@ -1156,6 +1224,7 @@ def OMP_DistributeParallelFor : Directive<"distribute parallel for"> { VersionedClause, VersionedClause, ]; + let leafConstructs = [OMP_Distribute, OMP_Parallel, OMP_For]; } def OMP_DistributeParallelDo : Directive<"distribute parallel do"> { let allowedClauses = [ @@ -1179,6 +1248,7 @@ def OMP_DistributeParallelDo : Directive<"distribute parallel do"> { VersionedClause, VersionedClause ]; + let leafConstructs = [OMP_Distribute, OMP_Parallel, OMP_Do]; } def OMP_DistributeParallelForSimd : Directive<"distribute parallel for simd"> { let allowedClauses = [ @@ -1204,6 +1274,7 @@ def OMP_DistributeParallelForSimd : Directive<"distribute parallel for simd"> { VersionedClause, VersionedClause, ]; + let leafConstructs = [OMP_Distribute, OMP_Parallel, OMP_For, OMP_Simd]; } def OMP_DistributeParallelDoSimd : Directive<"distribute parallel do simd"> { let allowedClauses = [ @@ -1228,6 +1299,7 @@ def OMP_DistributeParallelDoSimd : Directive<"distribute parallel do simd"> { VersionedClause, VersionedClause ]; + let leafConstructs = [OMP_Distribute, OMP_Parallel, OMP_Do, OMP_Simd]; } def OMP_DistributeSimd : Directive<"distribute simd"> { let allowedClauses = [ @@ -1254,6 +1326,7 @@ def OMP_DistributeSimd : Directive<"distribute simd"> { VersionedClause, VersionedClause ]; + let leafConstructs = [OMP_Distribute, OMP_Simd]; } def OMP_TargetParallelForSimd : Directive<"target parallel for simd"> { @@ -1291,6 +1364,7 @@ def OMP_TargetParallelForSimd : Directive<"target parallel for simd"> { VersionedClause, VersionedClause, ]; + let leafConstructs = [OMP_Target, OMP_Parallel, OMP_For, OMP_Simd]; } def OMP_TargetParallelDoSimd : Directive<"target parallel do simd"> { let allowedClauses = [ @@ -1322,6 +1396,7 @@ def OMP_TargetParallelDoSimd : Directive<"target parallel do simd"> { VersionedClause, VersionedClause ]; + let leafConstructs = [OMP_Target, OMP_Parallel, OMP_Do, OMP_Simd]; } def OMP_TargetSimd : Directive<"target simd"> { let allowedClauses = [ @@ -1356,6 +1431,7 @@ def OMP_TargetSimd : Directive<"target simd"> { VersionedClause, VersionedClause, ]; + let leafConstructs = [OMP_Target, OMP_Simd]; } def OMP_TeamsDistribute : Directive<"teams distribute"> { let allowedClauses = [ @@ -1375,6 +1451,7 @@ def OMP_TeamsDistribute : Directive<"teams distribute"> { let allowedOnceClauses = [ VersionedClause ]; + let leafConstructs = [OMP_Teams, OMP_Distribute]; } def OMP_TeamsDistributeSimd : Directive<"teams distribute simd"> { let allowedClauses = [ @@ -1400,6 +1477,7 @@ def OMP_TeamsDistributeSimd : Directive<"teams distribute simd"> { VersionedClause, VersionedClause ]; + let leafConstructs = [OMP_Teams, OMP_Distribute, OMP_Simd]; } def OMP_TeamsDistributeParallelForSimd : @@ -1428,6 +1506,7 @@ def OMP_TeamsDistributeParallelForSimd : VersionedClause, VersionedClause, ]; + let leafConstructs = [OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_For, OMP_Simd]; } def OMP_TeamsDistributeParallelDoSimd : Directive<"teams distribute parallel do simd"> { @@ -1456,6 +1535,7 @@ def OMP_TeamsDistributeParallelDoSimd : VersionedClause, VersionedClause ]; + let leafConstructs = [OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_Do, OMP_Simd]; } def OMP_TeamsDistributeParallelFor : Directive<"teams distribute parallel for"> { @@ -1479,6 +1559,7 @@ def OMP_TeamsDistributeParallelFor : VersionedClause, VersionedClause, ]; + let leafConstructs = [OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_For]; } def OMP_TeamsDistributeParallelDo : Directive<"teams distribute parallel do"> { @@ -1505,6 +1586,7 @@ let allowedOnceClauses = [ VersionedClause, VersionedClause ]; + let leafConstructs = [OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_Do]; } def OMP_TargetTeams : Directive<"target teams"> { let allowedClauses = [ @@ -1532,6 +1614,7 @@ def OMP_TargetTeams : Directive<"target teams"> { VersionedClause, VersionedClause, ]; + let leafConstructs = [OMP_Target, OMP_Teams]; } def OMP_TargetTeamsDistribute : Directive<"target teams distribute"> { let allowedClauses = [ @@ -1560,6 +1643,7 @@ def OMP_TargetTeamsDistribute : Directive<"target teams distribute"> { VersionedClause, VersionedClause, ]; + let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute]; } def OMP_TargetTeamsDistributeParallelFor : @@ -1594,6 +1678,7 @@ def OMP_TargetTeamsDistributeParallelFor : let allowedOnceClauses = [ VersionedClause, ]; + let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_For]; } def OMP_TargetTeamsDistributeParallelDo : Directive<"target teams distribute parallel do"> { @@ -1628,6 +1713,7 @@ def OMP_TargetTeamsDistributeParallelDo : VersionedClause, VersionedClause ]; + let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_Do]; } def OMP_TargetTeamsDistributeParallelForSimd : Directive<"target teams distribute parallel for simd"> { @@ -1666,6 +1752,7 @@ def OMP_TargetTeamsDistributeParallelForSimd : let allowedOnceClauses = [ VersionedClause, ]; + let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_For, OMP_Simd]; } def OMP_TargetTeamsDistributeParallelDoSimd : Directive<"target teams distribute parallel do simd"> { @@ -1704,6 +1791,7 @@ def OMP_TargetTeamsDistributeParallelDoSimd : VersionedClause, VersionedClause ]; + let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Parallel, OMP_Do, OMP_Simd]; } def OMP_TargetTeamsDistributeSimd : Directive<"target teams distribute simd"> { @@ -1738,17 +1826,20 @@ def OMP_TargetTeamsDistributeSimd : VersionedClause, VersionedClause ]; + let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Simd]; } def OMP_Allocate : Directive<"allocate"> { let allowedOnceClauses = [ VersionedClause, VersionedClause ]; + let association = AS_None; } def OMP_Allocators : Directive<"allocators"> { let allowedClauses = [ VersionedClause ]; + let association = AS_Block; } def OMP_DeclareVariant : Directive<"declare variant"> { let allowedClauses = [ @@ -1758,6 +1849,7 @@ def OMP_DeclareVariant : Directive<"declare variant"> { VersionedClause, VersionedClause ]; + let association = AS_Declaration; } def OMP_MasterTaskloop : Directive<"master taskloop"> { let allowedClauses = [ @@ -1779,6 +1871,7 @@ def OMP_MasterTaskloop : Directive<"master taskloop"> { VersionedClause, VersionedClause ]; + let leafConstructs = [OMP_Master, OMP_TaskLoop]; } def OMP_MaskedTaskloop : Directive<"masked taskloop"> { let allowedClauses = [ @@ -1801,6 +1894,7 @@ def OMP_MaskedTaskloop : Directive<"masked taskloop"> { VersionedClause, VersionedClause ]; + let leafConstructs = [OMP_masked, OMP_TaskLoop]; } def OMP_ParallelMasterTaskloop : Directive<"parallel master taskloop"> { @@ -1826,6 +1920,7 @@ def OMP_ParallelMasterTaskloop : VersionedClause, VersionedClause, ]; + let leafConstructs = [OMP_Parallel, OMP_Master, OMP_TaskLoop]; } def OMP_ParallelMaskedTaskloop : Directive<"parallel masked taskloop"> { @@ -1852,6 +1947,7 @@ def OMP_ParallelMaskedTaskloop : VersionedClause, VersionedClause, ]; + let leafConstructs = [OMP_Parallel, OMP_masked, OMP_TaskLoop]; } def OMP_MasterTaskloopSimd : Directive<"master taskloop simd"> { let allowedClauses = [ @@ -1879,6 +1975,7 @@ def OMP_MasterTaskloopSimd : Directive<"master taskloop simd"> { VersionedClause, VersionedClause ]; + let leafConstructs = [OMP_Master, OMP_TaskLoop, OMP_Simd]; } def OMP_MaskedTaskloopSimd : Directive<"masked taskloop simd"> { let allowedClauses = [ @@ -1907,6 +2004,7 @@ def OMP_MaskedTaskloopSimd : Directive<"masked taskloop simd"> { VersionedClause, VersionedClause ]; + let leafConstructs = [OMP_masked, OMP_TaskLoop, OMP_Simd]; } def OMP_ParallelMasterTaskloopSimd : Directive<"parallel master taskloop simd"> { @@ -1938,6 +2036,7 @@ def OMP_ParallelMasterTaskloopSimd : VersionedClause, VersionedClause, ]; + let leafConstructs = [OMP_Parallel, OMP_Master, OMP_TaskLoop, OMP_Simd]; } def OMP_ParallelMaskedTaskloopSimd : Directive<"parallel masked taskloop simd"> { @@ -1970,6 +2069,7 @@ def OMP_ParallelMaskedTaskloopSimd : VersionedClause, VersionedClause, ]; + let leafConstructs = [OMP_Parallel, OMP_masked, OMP_TaskLoop, OMP_Simd]; } def OMP_Depobj : Directive<"depobj"> { let allowedClauses = [ @@ -1980,18 +2080,30 @@ def OMP_Depobj : Directive<"depobj"> { // OMPKinds.def. VersionedClause ]; + let association = AS_None; } def OMP_Scan : Directive<"scan"> { let allowedClauses = [ VersionedClause, VersionedClause ]; + let association = AS_Separating; +} +def OMP_Assumes : Directive<"assumes"> { + let association = AS_None; +} +def OMP_BeginAssumes : Directive<"begin assumes"> { + let association = AS_Delimited; +} +def OMP_EndAssumes : Directive<"end assumes"> { + let association = AS_Delimited; +} +def OMP_BeginDeclareVariant : Directive<"begin declare variant"> { + let association = AS_Delimited; +} +def OMP_EndDeclareVariant : Directive<"end declare variant"> { + let association = AS_Delimited; } -def OMP_Assumes : Directive<"assumes"> {} -def OMP_BeginAssumes : Directive<"begin assumes"> {} -def OMP_EndAssumes : Directive<"end assumes"> {} -def OMP_BeginDeclareVariant : Directive<"begin declare variant"> {} -def OMP_EndDeclareVariant : Directive<"end declare variant"> {} def OMP_scope : Directive<"scope"> { let allowedClauses = [ VersionedClause, @@ -2000,6 +2112,10 @@ def OMP_scope : Directive<"scope"> { let allowedOnceClauses = [ VersionedClause ]; + let association = AS_Block; +} +def OMP_Workshare : Directive<"workshare"> { + let association = AS_Block; } def OMP_ParallelWorkshare : Directive<"parallel workshare"> { let allowedClauses = [ @@ -2016,22 +2132,29 @@ def OMP_ParallelWorkshare : Directive<"parallel workshare"> { VersionedClause, VersionedClause ]; + let leafConstructs = [OMP_Parallel, OMP_Workshare]; } -def OMP_Workshare : Directive<"workshare"> {} def OMP_EndDo : Directive<"end do"> { let allowedOnceClauses = [ VersionedClause ]; + // Needed for association computation, since OMP_Do has it "from leafConstructs". + let leafConstructs = OMP_Do.leafConstructs; + let association = OMP_Do.association; } def OMP_EndDoSimd : Directive<"end do simd"> { let allowedOnceClauses = [ VersionedClause ]; + let leafConstructs = OMP_DoSimd.leafConstructs; + let association = OMP_DoSimd.association; } def OMP_EndSections : Directive<"end sections"> { let allowedOnceClauses = [ VersionedClause ]; + let leafConstructs = OMP_Sections.leafConstructs; + let association = OMP_Sections.association; } def OMP_EndSingle : Directive<"end single"> { let allowedClauses = [ @@ -2040,11 +2163,15 @@ def OMP_EndSingle : Directive<"end single"> { let allowedOnceClauses = [ VersionedClause ]; + let leafConstructs = OMP_Single.leafConstructs; + let association = OMP_Single.association; } def OMP_EndWorkshare : Directive<"end workshare"> { let allowedClauses = [ VersionedClause ]; + let leafConstructs = OMP_Workshare.leafConstructs; + let association = OMP_Workshare.association; } def OMP_interop : Directive<"interop"> { let allowedClauses = [ @@ -2055,6 +2182,7 @@ def OMP_interop : Directive<"interop"> { VersionedClause, VersionedClause, ]; + let association = AS_None; } def OMP_dispatch : Directive<"dispatch"> { let allowedClauses = [ @@ -2066,11 +2194,7 @@ def OMP_dispatch : Directive<"dispatch"> { VersionedClause, VersionedClause ]; -} -def OMP_masked : Directive<"masked"> { - let allowedOnceClauses = [ - VersionedClause - ]; + let association = AS_Block; } def OMP_loop : Directive<"loop"> { let allowedClauses = [ @@ -2083,6 +2207,7 @@ def OMP_loop : Directive<"loop"> { VersionedClause, VersionedClause ]; + let association = AS_Loop; } def OMP_teams_loop : Directive<"teams loop"> { let allowedClauses = [ @@ -2102,6 +2227,7 @@ def OMP_teams_loop : Directive<"teams loop"> { VersionedClause, VersionedClause, ]; + let leafConstructs = [OMP_Teams, OMP_loop]; } def OMP_target_teams_loop : Directive<"target teams loop"> { let allowedClauses = [ @@ -2131,6 +2257,7 @@ def OMP_target_teams_loop : Directive<"target teams loop"> { VersionedClause, VersionedClause, ]; + let leafConstructs = [OMP_Target, OMP_Teams, OMP_loop]; } def OMP_parallel_loop : Directive<"parallel loop"> { let allowedClauses = [ @@ -2152,6 +2279,7 @@ def OMP_parallel_loop : Directive<"parallel loop"> { VersionedClause, VersionedClause, ]; + let leafConstructs = [OMP_Parallel, OMP_loop]; } def OMP_target_parallel_loop : Directive<"target parallel loop"> { let allowedClauses = [ @@ -2183,11 +2311,15 @@ def OMP_target_parallel_loop : Directive<"target parallel loop"> { VersionedClause, VersionedClause, ]; + let leafConstructs = [OMP_Target, OMP_Parallel, OMP_loop]; } def OMP_Metadirective : Directive<"metadirective"> { let allowedClauses = [VersionedClause]; let allowedOnceClauses = [VersionedClause]; + let association = AS_None; } + def OMP_Unknown : Directive<"unknown"> { let isDefault = true; + let association = AS_None; } diff --git a/llvm/include/llvm/TableGen/DirectiveEmitter.h b/llvm/include/llvm/TableGen/DirectiveEmitter.h index c86018715a48..bd536d4a2b7f 100644 --- a/llvm/include/llvm/TableGen/DirectiveEmitter.h +++ b/llvm/include/llvm/TableGen/DirectiveEmitter.h @@ -51,6 +51,10 @@ public: return Def->getValueAsBit("enableBitmaskEnumInNamespace"); } + std::vector getAssociations() const { + return Records.getAllDerivedDefinitions("Association"); + } + std::vector getDirectives() const { return Records.getAllDerivedDefinitions("Directive"); } @@ -121,6 +125,12 @@ public: std::vector getRequiredClauses() const { return Def->getValueAsListOfDefs("requiredClauses"); } + + std::vector getLeafConstructs() const { + return Def->getValueAsListOfDefs("leafConstructs"); + } + + Record *getAssociation() const { return Def->getValueAsDef("association"); } }; // Wrapper class that contains Clause's information defined in DirectiveBase.td diff --git a/llvm/test/TableGen/directive1.td b/llvm/test/TableGen/directive1.td index bcd9fa34c99d..aa47d7ee666d 100644 --- a/llvm/test/TableGen/directive1.td +++ b/llvm/test/TableGen/directive1.td @@ -44,11 +44,13 @@ def TDL_DirA : Directive<"dira"> { VersionedClause ]; let isDefault = 1; + let association = AS_None; } // CHECK: #ifndef LLVM_Tdl_INC // CHECK-NEXT: #define LLVM_Tdl_INC // CHECK-EMPTY: +// CHECK-NEXT: #include "llvm/ADT/ArrayRef.h" // CHECK-NEXT: #include "llvm/ADT/BitmaskEnum.h" // CHECK-EMPTY: // CHECK-NEXT: namespace llvm { @@ -57,6 +59,17 @@ def TDL_DirA : Directive<"dira"> { // CHECK-EMPTY: // CHECK-NEXT: LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE(); // CHECK-EMPTY: +// CHECK-NEXT: enum class Association { +// CHECK-NEXT: Block, +// CHECK-NEXT: Declaration, +// CHECK-NEXT: Delimited, +// CHECK-NEXT: Loop, +// CHECK-NEXT: None, +// CHECK-NEXT: Separating, +// CHECK-NEXT: }; +// CHECK-EMPTY: +// CHECK-NEXT: static constexpr std::size_t Association_enumSize = 6; +// CHECK-EMPTY: // CHECK-NEXT: enum class Directive { // CHECK-NEXT: TDLD_dira, // CHECK-NEXT: }; @@ -99,6 +112,8 @@ def TDL_DirA : Directive<"dira"> { // CHECK-NEXT: /// Return true if \p C is a valid clause for \p D in version \p Version. // CHECK-NEXT: bool isAllowedClauseForDirective(Directive D, Clause C, unsigned Version); // CHECK-EMPTY: +// CHECK-NEXT: llvm::ArrayRef getLeafConstructs(Directive D); +// CHECK-NEXT: Association getDirectiveAssociation(Directive D); // CHECK-NEXT: AKind getAKind(StringRef); // CHECK-NEXT: llvm::StringRef getTdlAKindName(AKind); // CHECK-EMPTY: @@ -270,6 +285,8 @@ def TDL_DirA : Directive<"dira"> { // IMPL: #ifdef GEN_DIRECTIVES_IMPL // IMPL-NEXT: #undef GEN_DIRECTIVES_IMPL // IMPL-EMPTY: +// IMPL-NEXT: #include "llvm/Support/ErrorHandling.h" +// IMPL-EMPTY: // IMPL-NEXT: Directive llvm::tdl::getTdlDirectiveKind(llvm::StringRef Str) { // IMPL-NEXT: return llvm::StringSwitch(Str) // IMPL-NEXT: .Case("dira",TDLD_dira) @@ -342,4 +359,20 @@ def TDL_DirA : Directive<"dira"> { // IMPL-NEXT: llvm_unreachable("Invalid Tdl Directive kind"); // IMPL-NEXT: } // IMPL-EMPTY: +// IMPL-NEXT: llvm::ArrayRef llvm::tdl::getLeafConstructs(llvm::tdl::Directive Dir) { +// IMPL-NEXT: switch (Dir) { +// IMPL-NEXT: default: +// IMPL-NEXT: return ArrayRef{}; +// IMPL-NEXT: } // switch (Dir) +// IMPL-NEXT: } +// IMPL-EMPTY: +// IMPL-NEXT: llvm::tdl::Association llvm::tdl::getDirectiveAssociation(llvm::tdl::Directive Dir) { +// IMPL-NEXT: switch (Dir) { +// IMPL-NEXT: case llvm::tdl::Directive::TDLD_dira: +// IMPL-NEXT: return llvm::tdl::Association::None; +// IMPL-NEXT: default: +// IMPL-NEXT: llvm_unreachable("Unexpected directive"); +// IMPL-NEXT: } // switch(Dir) +// IMPL-NEXT: } +// IMPL-EMPTY: // IMPL-NEXT: #endif // GEN_DIRECTIVES_IMPL diff --git a/llvm/test/TableGen/directive2.td b/llvm/test/TableGen/directive2.td index 01741152956d..90f9c45b5e15 100644 --- a/llvm/test/TableGen/directive2.td +++ b/llvm/test/TableGen/directive2.td @@ -38,15 +38,29 @@ def TDL_DirA : Directive<"dira"> { VersionedClause ]; let isDefault = 1; + let association = AS_Block; } // CHECK: #ifndef LLVM_Tdl_INC // CHECK-NEXT: #define LLVM_Tdl_INC // CHECK-EMPTY: +// CHECK-NEXT: #include "llvm/ADT/ArrayRef.h" +// CHECK-EMPTY: // CHECK-NEXT: namespace llvm { // CHECK-NEXT: class StringRef; // CHECK-NEXT: namespace tdl { // CHECK-EMPTY: +// CHECK-NEXT: enum class Association { +// CHECK-NEXT: Block, +// CHECK-NEXT: Declaration, +// CHECK-NEXT: Delimited, +// CHECK-NEXT: Loop, +// CHECK-NEXT: None, +// CHECK-NEXT: Separating, +// CHECK-NEXT: }; +// CHECK-EMPTY: +// CHECK-NEXT: static constexpr std::size_t Association_enumSize = 6; +// CHECK-EMPTY: // CHECK-NEXT: enum class Directive { // CHECK-NEXT: TDLD_dira, // CHECK-NEXT: }; @@ -74,6 +88,8 @@ def TDL_DirA : Directive<"dira"> { // CHECK-NEXT: /// Return true if \p C is a valid clause for \p D in version \p Version. // CHECK-NEXT: bool isAllowedClauseForDirective(Directive D, Clause C, unsigned Version); // CHECK-EMPTY: +// CHECK-NEXT: llvm::ArrayRef getLeafConstructs(Directive D); +// CHECK-NEXT: Association getDirectiveAssociation(Directive D); // CHECK-NEXT: } // namespace tdl // CHECK-NEXT: } // namespace llvm // CHECK-NEXT: #endif // LLVM_Tdl_INC @@ -217,6 +233,8 @@ def TDL_DirA : Directive<"dira"> { // IMPL: #ifdef GEN_DIRECTIVES_IMPL // IMPL-NEXT: #undef GEN_DIRECTIVES_IMPL // IMPL-EMPTY: +// IMPL-NEXT: #include "llvm/Support/ErrorHandling.h" +// IMPL-EMPTY: // IMPL-NEXT: Directive llvm::tdl::getTdlDirectiveKind(llvm::StringRef Str) { // IMPL-NEXT: return llvm::StringSwitch(Str) // IMPL-NEXT: .Case("dira",TDLD_dira) @@ -272,4 +290,20 @@ def TDL_DirA : Directive<"dira"> { // IMPL-NEXT: llvm_unreachable("Invalid Tdl Directive kind"); // IMPL-NEXT: } // IMPL-EMPTY: +// IMPL-NEXT: llvm::ArrayRef llvm::tdl::getLeafConstructs(llvm::tdl::Directive Dir) { +// IMPL-NEXT: switch (Dir) { +// IMPL-NEXT: default: +// IMPL-NEXT: return ArrayRef{}; +// IMPL-NEXT: } // switch (Dir) +// IMPL-NEXT: } +// IMPL-EMPTY: +// IMPL-NEXT: llvm::tdl::Association llvm::tdl::getDirectiveAssociation(llvm::tdl::Directive Dir) { +// IMPL-NEXT: switch (Dir) { +// IMPL-NEXT: case llvm::tdl::Directive::TDLD_dira: +// IMPL-NEXT: return llvm::tdl::Association::Block; +// IMPL-NEXT: default: +// IMPL-NEXT: llvm_unreachable("Unexpected directive"); +// IMPL-NEXT: } // switch(Dir) +// IMPL-NEXT: } +// IMPL-EMPTY: // IMPL-NEXT: #endif // GEN_DIRECTIVES_IMPL diff --git a/llvm/utils/TableGen/DirectiveEmitter.cpp b/llvm/utils/TableGen/DirectiveEmitter.cpp index 99eebacd6b3f..5a35d320227d 100644 --- a/llvm/utils/TableGen/DirectiveEmitter.cpp +++ b/llvm/utils/TableGen/DirectiveEmitter.cpp @@ -42,7 +42,8 @@ private: // Generate enum class static void GenerateEnumClass(const std::vector &Records, raw_ostream &OS, StringRef Enum, StringRef Prefix, - const DirectiveLanguage &DirLang) { + const DirectiveLanguage &DirLang, + bool ExportEnums) { OS << "\n"; OS << "enum class " << Enum << " {\n"; for (const auto &R : Records) { @@ -59,7 +60,7 @@ static void GenerateEnumClass(const std::vector &Records, // At the same time we do not loose the strong type guarantees of the enum // class, that is we cannot pass an unsigned as Directive without an explicit // cast. - if (DirLang.hasMakeEnumAvailableInNamespace()) { + if (ExportEnums) { OS << "\n"; for (const auto &R : Records) { BaseRecord Rec{R}; @@ -183,9 +184,10 @@ static void EmitDirectivesDecl(RecordKeeper &Records, raw_ostream &OS) { OS << "#ifndef LLVM_" << DirLang.getName() << "_INC\n"; OS << "#define LLVM_" << DirLang.getName() << "_INC\n"; + OS << "\n#include \"llvm/ADT/ArrayRef.h\"\n"; if (DirLang.hasEnableBitmaskEnumInNamespace()) - OS << "\n#include \"llvm/ADT/BitmaskEnum.h\"\n"; + OS << "#include \"llvm/ADT/BitmaskEnum.h\"\n"; OS << "\n"; OS << "namespace llvm {\n"; @@ -200,13 +202,24 @@ static void EmitDirectivesDecl(RecordKeeper &Records, raw_ostream &OS) { if (DirLang.hasEnableBitmaskEnumInNamespace()) OS << "\nLLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();\n"; + // Emit Directive associations + std::vector associations; + llvm::copy_if( + DirLang.getAssociations(), std::back_inserter(associations), + // Skip the "special" value + [](const Record *Def) { return Def->getName() != "AS_FromLeaves"; }); + GenerateEnumClass(associations, OS, "Association", + /*Prefix=*/"", DirLang, /*ExportEnums=*/false); + // Emit Directive enumeration GenerateEnumClass(DirLang.getDirectives(), OS, "Directive", - DirLang.getDirectivePrefix(), DirLang); + DirLang.getDirectivePrefix(), DirLang, + DirLang.hasMakeEnumAvailableInNamespace()); // Emit Clause enumeration GenerateEnumClass(DirLang.getClauses(), OS, "Clause", - DirLang.getClausePrefix(), DirLang); + DirLang.getClausePrefix(), DirLang, + DirLang.hasMakeEnumAvailableInNamespace()); // Emit ClauseVal enumeration std::string EnumHelperFuncs; @@ -231,6 +244,8 @@ static void EmitDirectivesDecl(RecordKeeper &Records, raw_ostream &OS) { OS << "bool isAllowedClauseForDirective(Directive D, " << "Clause C, unsigned Version);\n"; OS << "\n"; + OS << "llvm::ArrayRef getLeafConstructs(Directive D);\n"; + OS << "Association getDirectiveAssociation(Directive D);\n"; if (EnumHelperFuncs.length() > 0) { OS << EnumHelperFuncs; OS << "\n"; @@ -435,6 +450,217 @@ static void GenerateIsAllowedClause(const DirectiveLanguage &DirLang, OS << "}\n"; // End of function isAllowedClauseForDirective } +// Generate the getLeafConstructs function implementation. +static void GenerateGetLeafConstructs(const DirectiveLanguage &DirLang, + raw_ostream &OS) { + auto getQualifiedName = [&](StringRef Formatted) -> std::string { + return (llvm::Twine("llvm::") + DirLang.getCppNamespace() + + "::Directive::" + DirLang.getDirectivePrefix() + Formatted) + .str(); + }; + + // For each list of leaves, generate a static local object, then + // return a reference to that object for a given directive, e.g. + // + // static ListTy leafConstructs_A_B = { A, B }; + // static ListTy leafConstructs_C_D_E = { C, D, E }; + // switch (Dir) { + // case A_B: + // return leafConstructs_A_B; + // case C_D_E: + // return leafConstructs_C_D_E; + // } + + // Map from a record that defines a directive to the name of the + // local object with the list of its leaves. + DenseMap ListNames; + + std::string DirectiveTypeName = + std::string("llvm::") + DirLang.getCppNamespace().str() + "::Directive"; + + OS << '\n'; + + // ArrayRef<...> llvm::::GetLeafConstructs(llvm::::Directive Dir) + OS << "llvm::ArrayRef<" << DirectiveTypeName + << "> llvm::" << DirLang.getCppNamespace() << "::getLeafConstructs(" + << DirectiveTypeName << " Dir) "; + OS << "{\n"; + + // Generate the locals. + for (Record *R : DirLang.getDirectives()) { + Directive Dir{R}; + + std::vector LeafConstructs = Dir.getLeafConstructs(); + if (LeafConstructs.empty()) + continue; + + std::string ListName = "leafConstructs_" + Dir.getFormattedName(); + OS << " static const " << DirectiveTypeName << ' ' << ListName + << "[] = {\n"; + for (Record *L : LeafConstructs) { + Directive LeafDir{L}; + OS << " " << getQualifiedName(LeafDir.getFormattedName()) << ",\n"; + } + OS << " };\n"; + ListNames.insert(std::make_pair(R, std::move(ListName))); + } + + if (!ListNames.empty()) + OS << '\n'; + OS << " switch (Dir) {\n"; + for (Record *R : DirLang.getDirectives()) { + auto F = ListNames.find(R); + if (F == ListNames.end()) + continue; + + Directive Dir{R}; + OS << " case " << getQualifiedName(Dir.getFormattedName()) << ":\n"; + OS << " return " << F->second << ";\n"; + } + OS << " default:\n"; + OS << " return ArrayRef<" << DirectiveTypeName << ">{};\n"; + OS << " } // switch (Dir)\n"; + OS << "}\n"; +} + +static void GenerateGetDirectiveAssociation(const DirectiveLanguage &DirLang, + raw_ostream &OS) { + enum struct Association { + None = 0, // None should be the smallest value. + Block, // The values of the rest don't matter. + Declaration, + Delimited, + Loop, + Separating, + FromLeaves, + Invalid, + }; + + std::vector associations = DirLang.getAssociations(); + + auto getAssocValue = [](StringRef name) -> Association { + return StringSwitch(name) + .Case("AS_Block", Association::Block) + .Case("AS_Declaration", Association::Declaration) + .Case("AS_Delimited", Association::Delimited) + .Case("AS_Loop", Association::Loop) + .Case("AS_None", Association::None) + .Case("AS_Separating", Association::Separating) + .Case("AS_FromLeaves", Association::FromLeaves) + .Default(Association::Invalid); + }; + + auto getAssocName = [&](Association A) -> StringRef { + if (A != Association::Invalid && A != Association::FromLeaves) { + auto F = llvm::find_if(associations, [&](const Record *R) { + return getAssocValue(R->getName()) == A; + }); + if (F != associations.end()) + return (*F)->getValueAsString("name"); // enum name + } + llvm_unreachable("Unexpected association value"); + }; + + auto errorPrefixFor = [&](Directive D) -> std::string { + return (Twine("Directive '") + D.getName() + "' in namespace '" + + DirLang.getCppNamespace() + "' ") + .str(); + }; + + auto reduce = [&](Association A, Association B) -> Association { + if (A > B) + std::swap(A, B); + + // Calculate the result using the following rules: + // x + x = x + // AS_None + x = x + // AS_Block + AS_Loop = AS_Loop + if (A == Association::None || A == B) + return B; + if (A == Association::Block && B == Association::Loop) + return B; + if (A == Association::Loop && B == Association::Block) + return A; + return Association::Invalid; + }; + + llvm::DenseMap AsMap; + + auto compAssocImpl = [&](const Record *R, auto &&Self) -> Association { + if (auto F = AsMap.find(R); F != AsMap.end()) + return F->second; + + Directive D{R}; + Association AS = getAssocValue(D.getAssociation()->getName()); + if (AS == Association::Invalid) { + PrintFatalError(errorPrefixFor(D) + + "has an unrecognized value for association: '" + + D.getAssociation()->getName() + "'"); + } + if (AS != Association::FromLeaves) { + AsMap.insert(std::make_pair(R, AS)); + return AS; + } + // Compute the association from leaf constructs. + std::vector leaves = D.getLeafConstructs(); + if (leaves.empty()) { + llvm::errs() << D.getName() << '\n'; + PrintFatalError(errorPrefixFor(D) + + "requests association to be computed from leaves, " + "but it has no leaves"); + } + + Association Result = Self(leaves[0], Self); + for (int I = 1, E = leaves.size(); I < E; ++I) { + Association A = Self(leaves[I], Self); + Association R = reduce(Result, A); + if (R == Association::Invalid) { + PrintFatalError(errorPrefixFor(D) + + "has leaves with incompatible association values: " + + getAssocName(A) + " and " + getAssocName(R)); + } + Result = R; + } + + assert(Result != Association::Invalid); + assert(Result != Association::FromLeaves); + AsMap.insert(std::make_pair(R, Result)); + return Result; + }; + + for (Record *R : DirLang.getDirectives()) + compAssocImpl(R, compAssocImpl); // Updates AsMap. + + OS << '\n'; + + auto getQualifiedName = [&](StringRef Formatted) -> std::string { + return (llvm::Twine("llvm::") + DirLang.getCppNamespace() + + "::Directive::" + DirLang.getDirectivePrefix() + Formatted) + .str(); + }; + + std::string DirectiveTypeName = + std::string("llvm::") + DirLang.getCppNamespace().str() + "::Directive"; + std::string AssociationTypeName = + std::string("llvm::") + DirLang.getCppNamespace().str() + "::Association"; + + OS << AssociationTypeName << " llvm::" << DirLang.getCppNamespace() + << "::getDirectiveAssociation(" << DirectiveTypeName << " Dir) {\n"; + OS << " switch (Dir) {\n"; + for (Record *R : DirLang.getDirectives()) { + if (auto F = AsMap.find(R); F != AsMap.end()) { + Directive Dir{R}; + OS << " case " << getQualifiedName(Dir.getFormattedName()) << ":\n"; + OS << " return " << AssociationTypeName + << "::" << getAssocName(F->second) << ";\n"; + } + } + OS << " default:\n"; + OS << " llvm_unreachable(\"Unexpected directive\");\n"; + OS << " } // switch(Dir)\n"; + OS << "}\n"; +} + // Generate a simple enum set with the give clauses. static void GenerateClauseSet(const std::vector &Clauses, raw_ostream &OS, StringRef ClauseSetPrefix, @@ -855,6 +1081,8 @@ void EmitDirectivesBasicImpl(const DirectiveLanguage &DirLang, raw_ostream &OS) { IfDefScope Scope("GEN_DIRECTIVES_IMPL", OS); + OS << "\n#include \"llvm/Support/ErrorHandling.h\"\n"; + // getDirectiveKind(StringRef Str) GenerateGetKind(DirLang.getDirectives(), OS, "Directive", DirLang, DirLang.getDirectivePrefix(), /*ImplicitAsUnknown=*/false); @@ -877,6 +1105,12 @@ void EmitDirectivesBasicImpl(const DirectiveLanguage &DirLang, // isAllowedClauseForDirective(Directive D, Clause C, unsigned Version) GenerateIsAllowedClause(DirLang, OS); + + // getLeafConstructs(Directive D) + GenerateGetLeafConstructs(DirLang, OS); + + // getDirectiveAssociation(Directive D) + GenerateGetDirectiveAssociation(DirLang, OS); } // Generate the implemenation section for the enumeration in the directive -- GitLab From 7755c26195900fa5611cd87582acd2f1875d3c40 Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Wed, 6 Mar 2024 07:49:58 -0800 Subject: [PATCH 319/929] [SCEV] Include type when printing constant max backedge taken count When printing the result of the analysis, i8 -1 and i64 -1 are quite different in terms of analysis quality. In a recent conversion with a new contributor, we ran into exactly this confusion. Adding the type for constant scevs more globally seems worthwhile, but introduces a much larger test diff. I'm splitting this off first since it addresses the immediate need, and then going to do some further changes to clarify a few related bits of analysis result output. --- llvm/lib/Analysis/ScalarEvolution.cpp | 3 +- .../ScalarEvolution/2008-11-18-Stride1.ll | 2 +- .../ScalarEvolution/2008-11-18-Stride2.ll | 2 +- .../Analysis/ScalarEvolution/add-like-or.ll | 2 +- ...drec-computed-during-addrec-calculation.ll | 4 +- .../ScalarEvolution/addrec-sub-nsw.ll | 4 +- llvm/test/Analysis/ScalarEvolution/alloca.ll | 2 +- .../backedge-taken-count-guard-info.ll | 4 +- .../ScalarEvolution/becount-invalidation.ll | 2 +- .../ScalarEvolution/decrementing_addrecs.ll | 4 +- .../exact-exit-count-more-precise.ll | 4 +- .../ScalarEvolution/exhaustive-trip-counts.ll | 2 +- .../ScalarEvolution/exit-count-select-safe.ll | 78 ++++++++-------- .../ScalarEvolution/exit-count-select.ll | 12 +-- .../ScalarEvolution/exponential-behavior.ll | 2 +- .../ScalarEvolution/finite-trip-count.ll | 18 ++-- .../flags-from-poison-noautogen.ll | 4 +- .../ScalarEvolution/flags-from-poison.ll | 70 +++++++-------- llvm/test/Analysis/ScalarEvolution/fold.ll | 2 +- .../ScalarEvolution/huge-trip-multiple.ll | 10 +-- .../ScalarEvolution/incorrect-exit-count.ll | 6 +- .../increasing-or-decreasing-iv.ll | 16 ++-- .../Analysis/ScalarEvolution/invalidation.ll | 2 +- .../load-with-range-metadata.ll | 4 +- llvm/test/Analysis/ScalarEvolution/load.ll | 4 +- .../ScalarEvolution/logical-operations.ll | 2 +- ...en-count-guard-info-rewrite-expressions.ll | 36 ++++---- .../max-backedge-taken-count-guard-info.ll | 90 +++++++++---------- ...-backedge-taken-count-limit-by-wrapping.ll | 8 +- .../max-be-count-not-constant.ll | 2 +- .../ScalarEvolution/max-expr-cache.ll | 4 +- .../max-trip-count-address-space.ll | 4 +- .../ScalarEvolution/max-trip-count.ll | 34 +++---- .../Analysis/ScalarEvolution/min-max-exprs.ll | 2 +- .../Analysis/ScalarEvolution/ne-overflow.ll | 10 +-- .../no-wrap-symbolic-becount.ll | 6 +- .../ScalarEvolution/nsw-offset-assume.ll | 2 +- .../Analysis/ScalarEvolution/nsw-offset.ll | 2 +- llvm/test/Analysis/ScalarEvolution/nsw.ll | 18 ++-- .../Analysis/ScalarEvolution/outer_phi.ll | 4 +- .../overflow-intrinsics-trip-count.ll | 22 ++--- .../ScalarEvolution/overflow-intrinsics.ll | 10 +-- llvm/test/Analysis/ScalarEvolution/pr25369.ll | 6 +- llvm/test/Analysis/ScalarEvolution/pr34538.ll | 4 +- llvm/test/Analysis/ScalarEvolution/pr48225.ll | 4 +- .../ptrtoint-constantexpr-loop.ll | 6 +- .../test/Analysis/ScalarEvolution/ptrtoint.ll | 16 ++-- .../ScalarEvolution/range-signedness.ll | 2 +- .../Analysis/ScalarEvolution/range_nw_flag.ll | 12 +-- llvm/test/Analysis/ScalarEvolution/ranges.ll | 12 +-- llvm/test/Analysis/ScalarEvolution/sdiv.ll | 2 +- .../ScalarEvolution/sext-add-inreg-loop.ll | 2 +- .../test/Analysis/ScalarEvolution/sext-mul.ll | 4 +- .../Analysis/ScalarEvolution/sext-to-zext.ll | 2 +- .../test/Analysis/ScalarEvolution/shift-op.ll | 10 +-- .../ScalarEvolution/shift-recurrences.ll | 34 +++---- llvm/test/Analysis/ScalarEvolution/sle.ll | 2 +- .../ScalarEvolution/smax-br-phi-idioms.ll | 2 +- .../ScalarEvolution/smin-smax-folds.ll | 6 +- .../ScalarEvolution/solve-quadratic-i1.ll | 4 +- .../solve-quadratic-overflow.ll | 2 +- llvm/test/Analysis/ScalarEvolution/srem.ll | 2 +- .../symbolic_max_exit_count.ll | 20 ++--- .../trip-count-andor-selectform.ll | 24 ++--- .../ScalarEvolution/trip-count-andor.ll | 24 ++--- .../trip-count-implied-addrec.ll | 12 +-- .../ScalarEvolution/trip-count-minmax.ll | 14 +-- .../trip-count-negative-stride.ll | 42 ++++----- .../ScalarEvolution/trip-count-pow2.ll | 10 +-- .../ScalarEvolution/trip-count-switch.ll | 2 +- .../trip-count-unknown-stride.ll | 10 +-- .../Analysis/ScalarEvolution/trip-count.ll | 8 +- .../Analysis/ScalarEvolution/trip-count10.ll | 8 +- .../Analysis/ScalarEvolution/trip-count12.ll | 2 +- .../Analysis/ScalarEvolution/trip-count13.ll | 10 +-- .../Analysis/ScalarEvolution/trip-count14.ll | 16 ++-- .../Analysis/ScalarEvolution/trip-count15.ll | 6 +- .../Analysis/ScalarEvolution/trip-count2.ll | 2 +- .../Analysis/ScalarEvolution/trip-count3.ll | 2 +- .../Analysis/ScalarEvolution/trip-count4.ll | 2 +- .../Analysis/ScalarEvolution/trip-count6.ll | 2 +- .../Analysis/ScalarEvolution/trip-count7.ll | 2 +- .../Analysis/ScalarEvolution/trip-count8.ll | 2 +- .../Analysis/ScalarEvolution/trip-count9.ll | 42 ++++----- .../trip-multiple-guard-info.ll | 32 +++---- .../Analysis/ScalarEvolution/trip-multiple.ll | 16 ++-- .../ScalarEvolution/umin-umax-folds.ll | 18 ++-- .../Analysis/ScalarEvolution/unknown_phis.ll | 2 +- .../ScalarEvolution/widenable-condition.ll | 2 +- .../invalidate-scev-after-hoisting.ll | 5 -- .../Transforms/LoopSimplify/preserve-scev.ll | 12 +-- 91 files changed, 499 insertions(+), 503 deletions(-) diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index e1e6742e50ef..096091db8019 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -13501,7 +13501,8 @@ static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE, auto *ConstantBTC = SE->getConstantMaxBackedgeTakenCount(L); if (!isa(ConstantBTC)) { - OS << "constant max backedge-taken count is " << *ConstantBTC; + OS << "constant max backedge-taken count is " + << *ConstantBTC->getType() << " " << *ConstantBTC; if (SE->isBackedgeTakenCountMaxOrZero(L)) OS << ", actual taken count either this or zero."; } else { diff --git a/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride1.ll b/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride1.ll index dbd9b83a7831..104c3177db1f 100644 --- a/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride1.ll +++ b/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride1.ll @@ -11,7 +11,7 @@ define i32 @f(i32 %x) nounwind readnone { ; CHECK-LABEL: 'f' ; CHECK-NEXT: Determining loop execution counts for: @f ; CHECK-NEXT: Loop %bb: backedge-taken count is ((-5 + %x) /u 3) -; CHECK-NEXT: Loop %bb: constant max backedge-taken count is 1431655764 +; CHECK-NEXT: Loop %bb: constant max backedge-taken count is i32 1431655764 ; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is ((-5 + %x) /u 3) ; CHECK-NEXT: Loop %bb: Predicated backedge-taken count is ((-5 + %x) /u 3) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride2.ll b/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride2.ll index ca472d4036cb..6f5e60d9bc2e 100644 --- a/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride2.ll +++ b/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride2.ll @@ -9,7 +9,7 @@ define i32 @f(i32 %x) nounwind readnone { ; CHECK-LABEL: 'f' ; CHECK-NEXT: Determining loop execution counts for: @f ; CHECK-NEXT: Loop %bb: backedge-taken count is (((-3 + (-1 * (1 umin (-3 + (-1 * %x) + (1000 umax (3 + %x))))) + (-1 * %x) + (1000 umax (3 + %x))) /u 3) + (1 umin (-3 + (-1 * %x) + (1000 umax (3 + %x))))) -; CHECK-NEXT: Loop %bb: constant max backedge-taken count is 334 +; CHECK-NEXT: Loop %bb: constant max backedge-taken count is i32 334 ; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is (((-3 + (-1 * (1 umin (-3 + (-1 * %x) + (1000 umax (3 + %x))))) + (-1 * %x) + (1000 umax (3 + %x))) /u 3) + (1 umin (-3 + (-1 * %x) + (1000 umax (3 + %x))))) ; CHECK-NEXT: Loop %bb: Predicated backedge-taken count is (((-3 + (-1 * (1 umin (-3 + (-1 * %x) + (1000 umax (3 + %x))))) + (-1 * %x) + (1000 umax (3 + %x))) /u 3) + (1 umin (-3 + (-1 * %x) + (1000 umax (3 + %x))))) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/add-like-or.ll b/llvm/test/Analysis/ScalarEvolution/add-like-or.ll index 86bb9058832b..a525f9f17f07 100644 --- a/llvm/test/Analysis/ScalarEvolution/add-like-or.ll +++ b/llvm/test/Analysis/ScalarEvolution/add-like-or.ll @@ -66,7 +66,7 @@ define void @mask-high(i64 %arg, ptr dereferenceable(4) %arg1) { ; CHECK-NEXT: --> {(2 + (16 * (%arg /u 16))),+,1}<%bb6> U: full-set S: full-set Exits: (1 + ((sext i32 %i to i64) smax (1 + (16 * (%arg /u 16))))) LoopDispositions: { %bb6: Computable } ; CHECK-NEXT: Determining loop execution counts for: @mask-high ; CHECK-NEXT: Loop %bb6: backedge-taken count is (-1 + (-16 * (%arg /u 16)) + ((sext i32 %i to i64) smax (1 + (16 * (%arg /u 16))))) -; CHECK-NEXT: Loop %bb6: constant max backedge-taken count is -9223372034707292162 +; CHECK-NEXT: Loop %bb6: constant max backedge-taken count is i64 -9223372034707292162 ; CHECK-NEXT: Loop %bb6: symbolic max backedge-taken count is (-1 + (-16 * (%arg /u 16)) + ((sext i32 %i to i64) smax (1 + (16 * (%arg /u 16))))) ; CHECK-NEXT: Loop %bb6: Predicated backedge-taken count is (-1 + (-16 * (%arg /u 16)) + ((sext i32 %i to i64) smax (1 + (16 * (%arg /u 16))))) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/addrec-computed-during-addrec-calculation.ll b/llvm/test/Analysis/ScalarEvolution/addrec-computed-during-addrec-calculation.ll index 9dfde18b6fda..ceb293dfa6c0 100644 --- a/llvm/test/Analysis/ScalarEvolution/addrec-computed-during-addrec-calculation.ll +++ b/llvm/test/Analysis/ScalarEvolution/addrec-computed-during-addrec-calculation.ll @@ -27,11 +27,11 @@ define void @test(ptr %p) { ; CHECK-NEXT: --> {{\{\{}}%iv,+,1}<%loop2>,+,1}<%loop3> U: full-set S: full-set --> {%iv,+,1}<%loop2> U: full-set S: full-set Exits: <> LoopDispositions: { %loop.header: Variant, %loop2: Variant, %loop3: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test ; CHECK-NEXT: Loop %loop2: Unpredictable backedge-taken count. -; CHECK-NEXT: Loop %loop2: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop2: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop2: symbolic max backedge-taken count is -1 ; CHECK-NEXT: Loop %loop2: Unpredictable predicated backedge-taken count. ; CHECK-NEXT: Loop %loop3: backedge-taken count is false -; CHECK-NEXT: Loop %loop3: constant max backedge-taken count is false +; CHECK-NEXT: Loop %loop3: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop3: symbolic max backedge-taken count is false ; CHECK-NEXT: Loop %loop3: Predicated backedge-taken count is false ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/addrec-sub-nsw.ll b/llvm/test/Analysis/ScalarEvolution/addrec-sub-nsw.ll index cf59ff8812bb..a021b1790bdb 100644 --- a/llvm/test/Analysis/ScalarEvolution/addrec-sub-nsw.ll +++ b/llvm/test/Analysis/ScalarEvolution/addrec-sub-nsw.ll @@ -12,7 +12,7 @@ define i32 @test_1_non_negative(i32 %n) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,-2147483648) S: [1,-2147483648) Exits: (1 smax %n) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_1_non_negative ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (1 smax %n)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (1 smax %n)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (1 smax %n)) ; CHECK-NEXT: Predicates: @@ -51,7 +51,7 @@ define i32 @test_2_non_positive(i32 %n) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,-2147483648) S: [1,-2147483648) Exits: (1 smax %n) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_2_non_positive ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (1 smax %n)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (1 smax %n)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (1 smax %n)) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/alloca.ll b/llvm/test/Analysis/ScalarEvolution/alloca.ll index 1a704322bf05..cf299104198e 100644 --- a/llvm/test/Analysis/ScalarEvolution/alloca.ll +++ b/llvm/test/Analysis/ScalarEvolution/alloca.ll @@ -44,7 +44,7 @@ define void @alloca_icmp_null_exit_count() { ; CHECK-NEXT: --> (%cmp2 umin %cmp1) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @alloca_icmp_null_exit_count ; CHECK-NEXT: Loop %loop: backedge-taken count is 2 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 2 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 2 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 2 ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info.ll index 2c29447358b3..2ba00196464f 100644 --- a/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info.ll +++ b/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info.ll @@ -14,7 +14,7 @@ define void @loop_guard_improves_exact_backedge_taken_count_1(i32 %conv) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,2) S: [1,2) Exits: 1 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @loop_guard_improves_exact_backedge_taken_count_1 ; CHECK-NEXT: Loop %loop: backedge-taken count is 0 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 0 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 0 ; CHECK-NEXT: Predicates: @@ -50,7 +50,7 @@ define void @loop_guard_improves_exact_backedge_taken_count_2(i32 %conv) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,3) S: [1,3) Exits: (1 + (zext i1 (trunc i32 %conv to i1) to i64)) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @loop_guard_improves_exact_backedge_taken_count_2 ; CHECK-NEXT: Loop %loop: backedge-taken count is (zext i1 (trunc i32 %conv to i1) to i64) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i1 (trunc i32 %conv to i1) to i64) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (zext i1 (trunc i32 %conv to i1) to i64) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/becount-invalidation.ll b/llvm/test/Analysis/ScalarEvolution/becount-invalidation.ll index 418f85613ef5..a87f07716fa6 100644 --- a/llvm/test/Analysis/ScalarEvolution/becount-invalidation.ll +++ b/llvm/test/Analysis/ScalarEvolution/becount-invalidation.ll @@ -30,7 +30,7 @@ define void @test(ptr %arg) { ; CHECK-NEXT: Loop %loop2.header: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for loop2.header: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for loop2.latch: false -; CHECK-NEXT: Loop %loop2.header: constant max backedge-taken count is false +; CHECK-NEXT: Loop %loop2.header: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop2.header: symbolic max backedge-taken count is false ; CHECK-NEXT: symbolic max exit count for loop2.header: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for loop2.latch: false diff --git a/llvm/test/Analysis/ScalarEvolution/decrementing_addrecs.ll b/llvm/test/Analysis/ScalarEvolution/decrementing_addrecs.ll index e92f66cf6e18..c681df485e27 100644 --- a/llvm/test/Analysis/ScalarEvolution/decrementing_addrecs.ll +++ b/llvm/test/Analysis/ScalarEvolution/decrementing_addrecs.ll @@ -47,7 +47,7 @@ define i32 @test_step_1_flags(i32 %n) { ; DEFAULT-NEXT: --> {(-2 + %n),+,-1}<%loop> U: full-set S: full-set Exits: -1 LoopDispositions: { %loop: Computable } ; DEFAULT-NEXT: Determining loop execution counts for: @test_step_1_flags ; DEFAULT-NEXT: Loop %loop: backedge-taken count is (-1 + %n) -; DEFAULT-NEXT: Loop %loop: constant max backedge-taken count is 2147483646 +; DEFAULT-NEXT: Loop %loop: constant max backedge-taken count is i32 2147483646 ; DEFAULT-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %n) ; DEFAULT-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %n) ; DEFAULT-NEXT: Predicates: @@ -73,7 +73,7 @@ define i32 @test_step_1_flags(i32 %n) { ; EXPENSIVE_SHARPENING-NEXT: --> {(-2 + %n),+,-1}<%loop> U: full-set S: [-1,2147483646) Exits: -1 LoopDispositions: { %loop: Computable } ; EXPENSIVE_SHARPENING-NEXT: Determining loop execution counts for: @test_step_1_flags ; EXPENSIVE_SHARPENING-NEXT: Loop %loop: backedge-taken count is (-1 + %n) -; EXPENSIVE_SHARPENING-NEXT: Loop %loop: constant max backedge-taken count is 2147483646 +; EXPENSIVE_SHARPENING-NEXT: Loop %loop: constant max backedge-taken count is i32 2147483646 ; EXPENSIVE_SHARPENING-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %n) ; EXPENSIVE_SHARPENING-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %n) ; EXPENSIVE_SHARPENING-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/exact-exit-count-more-precise.ll b/llvm/test/Analysis/ScalarEvolution/exact-exit-count-more-precise.ll index 19a65b4aa371..27d30c524f21 100644 --- a/llvm/test/Analysis/ScalarEvolution/exact-exit-count-more-precise.ll +++ b/llvm/test/Analysis/ScalarEvolution/exact-exit-count-more-precise.ll @@ -14,7 +14,7 @@ define void @test_and(i16 %in) { ; CHECK-NEXT: --> (%_tmp10.i umin %exitcond.i) U: full-set S: full-set Exits: true LoopDispositions: { %bb1.i: Variant, %bb2: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_and ; CHECK-NEXT: Loop %bb1.i: backedge-taken count is (1 + (-1 * %in)) -; CHECK-NEXT: Loop %bb1.i: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %bb1.i: constant max backedge-taken count is i16 -1 ; CHECK-NEXT: Loop %bb1.i: symbolic max backedge-taken count is (1 + (-1 * %in)) ; CHECK-NEXT: Loop %bb1.i: Predicated backedge-taken count is (1 + (-1 * %in)) ; CHECK-NEXT: Predicates: @@ -56,7 +56,7 @@ define void @test_or() { ; CHECK-NEXT: --> (%C11 umax %C5) U: full-set S: full-set Exits: false LoopDispositions: { %BB: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_or ; CHECK-NEXT: Loop %BB: backedge-taken count is undef -; CHECK-NEXT: Loop %BB: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %BB: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %BB: symbolic max backedge-taken count is undef ; CHECK-NEXT: Loop %BB: Predicated backedge-taken count is undef ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll b/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll index 2892b5a65b22..37f99cf9f24f 100644 --- a/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll +++ b/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll @@ -9,7 +9,7 @@ define void @f_0() { ; CHECK-LABEL: 'f_0' ; CHECK-NEXT: Determining loop execution counts for: @f_0 ; CHECK-NEXT: Loop %for.body: backedge-taken count is 5 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 5 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 5 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 5 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 5 ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/exit-count-select-safe.ll b/llvm/test/Analysis/ScalarEvolution/exit-count-select-safe.ll index d3cec77982af..6bf01fcde5f2 100644 --- a/llvm/test/Analysis/ScalarEvolution/exit-count-select-safe.ll +++ b/llvm/test/Analysis/ScalarEvolution/exit-count-select-safe.ll @@ -12,7 +12,7 @@ define i32 @logical_and_2ops(i32 %n, i32 %m) { ; CHECK-NEXT: --> (%cond_p0 umin_seq %cond_p1) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_2ops ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin_seq %m) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin_seq %m) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin_seq %m) ; CHECK-NEXT: Predicates: @@ -42,7 +42,7 @@ define i32 @logical_or_2ops(i32 %n, i32 %m) { ; CHECK-NEXT: --> (true + ((true + %cond_p0) umin_seq (true + %cond_p1))) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_or_2ops ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin_seq %m) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin_seq %m) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin_seq %m) ; CHECK-NEXT: Predicates: @@ -74,7 +74,7 @@ define i32 @logical_and_3ops(i32 %n, i32 %m, i32 %k) { ; CHECK-NEXT: --> (%cond_p0 umin_seq %cond_p1 umin_seq %cond_p2) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_3ops ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin_seq %m umin_seq %k) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin_seq %m umin_seq %k) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin_seq %m umin_seq %k) ; CHECK-NEXT: Predicates: @@ -108,7 +108,7 @@ define i32 @logical_or_3ops(i32 %n, i32 %m, i32 %k) { ; CHECK-NEXT: --> (true + ((true + %cond_p0) umin_seq (true + %cond_p1) umin_seq (true + %cond_p2))) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_or_3ops ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin_seq %m umin_seq %k) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin_seq %m umin_seq %k) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin_seq %m umin_seq %k) ; CHECK-NEXT: Predicates: @@ -144,7 +144,7 @@ define i32 @logical_or_3ops_duplicate(i32 %n, i32 %m, i32 %k) { ; CHECK-NEXT: --> (true + ((true + %cond_p0) umin_seq ((true + %cond_p1) umin (true + %cond_p2)) umin_seq (true + %cond_p3))) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_or_3ops_duplicate ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin_seq %m umin_seq %k) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin_seq %m umin_seq %k) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin_seq %m umin_seq %k) ; CHECK-NEXT: Predicates: @@ -182,7 +182,7 @@ define i32 @logical_or_3ops_redundant_uminseq_operand(i32 %n, i32 %m, i32 %k) { ; CHECK-NEXT: --> (true + (((true + %cond_p0) umin (true + %cond_p1)) umin_seq (true + %cond_p2))) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_or_3ops_redundant_uminseq_operand ; CHECK-NEXT: Loop %loop: backedge-taken count is ((%n umin %m) umin_seq %k) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((%n umin %m) umin_seq %k) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((%n umin %m) umin_seq %k) ; CHECK-NEXT: Predicates: @@ -219,7 +219,7 @@ define i32 @logical_or_3ops_redundant_umin_operand(i32 %n, i32 %m, i32 %k) { ; CHECK-NEXT: --> (true + ((true + %cond_p0) umin_seq (true + %cond_p1) umin_seq (true + %cond_p2))) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_or_3ops_redundant_umin_operand ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin_seq %k umin_seq %m) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin_seq %k umin_seq %m) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin_seq %k umin_seq %m) ; CHECK-NEXT: Predicates: @@ -258,7 +258,7 @@ define i32 @logical_or_4ops_redundant_operand_across_umins(i32 %n, i32 %m, i32 % ; CHECK-NEXT: --> (true + ((true + %cond_p0) umin_seq (true + %cond_p1) umin_seq (true + %cond_p2))) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_or_4ops_redundant_operand_across_umins ; CHECK-NEXT: Loop %loop: backedge-taken count is ((%n umin %m) umin_seq %k umin_seq %q) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((%n umin %m) umin_seq %k umin_seq %q) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((%n umin %m) umin_seq %k umin_seq %q) ; CHECK-NEXT: Predicates: @@ -298,7 +298,7 @@ define i32 @logical_or_3ops_operand_wise_redundant_umin(i32 %n, i32 %m, i32 %k) ; CHECK-NEXT: --> (true + ((true + %cond_p0) umin_seq (true + %cond_p1) umin_seq (true + %cond_p2))) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_or_3ops_operand_wise_redundant_umin ; CHECK-NEXT: Loop %loop: backedge-taken count is ((%n umin %m) umin_seq %k) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((%n umin %m) umin_seq %k) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((%n umin %m) umin_seq %k) ; CHECK-NEXT: Predicates: @@ -336,7 +336,7 @@ define i32 @logical_or_3ops_partially_redundant_umin(i32 %n, i32 %m, i32 %k) { ; CHECK-NEXT: --> (true + ((true + %cond_p0) umin_seq (true + %cond_p1))) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_or_3ops_partially_redundant_umin ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin_seq (%m umin %k)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin_seq (%m umin %k)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin_seq (%m umin %k)) ; CHECK-NEXT: Predicates: @@ -382,13 +382,13 @@ define i32 @logical_or_5ops_redundant_opearand_of_inner_uminseq(i32 %a, i32 %b, ; CHECK-NEXT: --> (true + ((true + %cond_p5) umin_seq (true + %cond_p6) umin_seq (true + %cond_p7))) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_or_5ops_redundant_opearand_of_inner_uminseq ; CHECK-NEXT: Loop %loop: backedge-taken count is (%a umin_seq %b umin_seq ((%e umin_seq %d) umin %c)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%a umin_seq %b umin_seq ((%e umin_seq %d) umin %c)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%a umin_seq %b umin_seq ((%e umin_seq %d) umin %c)) ; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; CHECK-NEXT: Loop %first.loop: backedge-taken count is (%e umin_seq %d umin_seq %a) -; CHECK-NEXT: Loop %first.loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %first.loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %first.loop: symbolic max backedge-taken count is (%e umin_seq %d umin_seq %a) ; CHECK-NEXT: Loop %first.loop: Predicated backedge-taken count is (%e umin_seq %d umin_seq %a) ; CHECK-NEXT: Predicates: @@ -435,7 +435,7 @@ define i32 @logical_and_2ops_and_constant(i32 %n, i32 %m, i32 %k) { ; CHECK-NEXT: --> (true + ((true + %cond_p1) umin (true + %cond_p0))) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_2ops_and_constant ; CHECK-NEXT: Loop %loop: backedge-taken count is (42 umin %n) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 42 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 42 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (42 umin %n) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (42 umin %n) ; CHECK-NEXT: Predicates: @@ -474,13 +474,13 @@ define i32 @computeSCEVAtScope(i32 %d.0) { ; CHECK-NEXT: --> {1,+,1}<%for.cond4> U: [1,2) S: [1,2) Exits: 1 LoopDispositions: { %for.cond4: Computable, %while.cond: Variant } ; CHECK-NEXT: Determining loop execution counts for: @computeSCEVAtScope ; CHECK-NEXT: Loop %for.cond: backedge-taken count is (-1 * %d.0) -; CHECK-NEXT: Loop %for.cond: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.cond: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %for.cond: symbolic max backedge-taken count is (-1 * %d.0) ; CHECK-NEXT: Loop %for.cond: Predicated backedge-taken count is (-1 * %d.0) ; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.cond: Trip multiple is 1 ; CHECK-NEXT: Loop %for.cond4: backedge-taken count is 0 -; CHECK-NEXT: Loop %for.cond4: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %for.cond4: constant max backedge-taken count is i32 0 ; CHECK-NEXT: Loop %for.cond4: symbolic max backedge-taken count is 0 ; CHECK-NEXT: Loop %for.cond4: Predicated backedge-taken count is 0 ; CHECK-NEXT: Predicates: @@ -543,7 +543,7 @@ define i64 @uminseq_vs_ptrtoint_complexity(i64 %n, i64 %m, ptr %ptr) { ; CHECK-NEXT: --> {(ptrtoint ptr %ptr to i64),+,1}<%loop> U: full-set S: full-set --> ((%n umin_seq %m) + (ptrtoint ptr %ptr to i64)) U: full-set S: full-set ; CHECK-NEXT: Determining loop execution counts for: @uminseq_vs_ptrtoint_complexity ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin_seq %m) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin_seq %m) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin_seq %m) ; CHECK-NEXT: Predicates: @@ -577,7 +577,7 @@ define i32 @logical_and_implies_poison1(i32 %n) { ; CHECK-NEXT: --> (%cond_p0 umin %cond_p1) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_implies_poison1 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((1 + %n) umin %n) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((1 + %n) umin %n) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((1 + %n) umin %n) ; CHECK-NEXT: Predicates: @@ -610,7 +610,7 @@ define i32 @logical_and_implies_poison2(i32 %n) { ; CHECK-NEXT: --> (%cond_p1 umin %cond_p0) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_implies_poison2 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((1 + %n) umin %n) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((1 + %n) umin %n) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((1 + %n) umin %n) ; CHECK-NEXT: Predicates: @@ -643,7 +643,7 @@ define i32 @logical_and_implies_poison3(i32 %n, i32 %m) { ; CHECK-NEXT: --> (%cond_p1 umin %cond_p0) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_implies_poison3 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((%n + %m) umin %n) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((%n + %m) umin %n) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((%n + %m) umin %n) ; CHECK-NEXT: Predicates: @@ -676,7 +676,7 @@ define i32 @logical_and_implies_poison_wrong_direction(i32 %n, i32 %m) { ; CHECK-NEXT: --> (%cond_p0 umin_seq %cond_p1) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_implies_poison_wrong_direction ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin_seq (%n + %m)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin_seq (%n + %m)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin_seq (%n + %m)) ; CHECK-NEXT: Predicates: @@ -707,7 +707,7 @@ define i32 @logical_and_implies_poison_noundef(i32 %n, i32 noundef %m) { ; CHECK-NEXT: --> (%cond_p0 umin %cond_p1) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_implies_poison_noundef ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin %m) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin %m) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin %m) ; CHECK-NEXT: Predicates: @@ -737,7 +737,7 @@ define i32 @logical_and_implies_poison_noundef_wrong_direction(i32 %n, i32 nound ; CHECK-NEXT: --> (%cond_p0 umin_seq %cond_p1) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_implies_poison_noundef_wrong_direction ; CHECK-NEXT: Loop %loop: backedge-taken count is (%m umin_seq %n) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%m umin_seq %n) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%m umin_seq %n) ; CHECK-NEXT: Predicates: @@ -771,7 +771,7 @@ define i32 @logical_and_implies_poison_complex1(i32 %n, i32 %m) { ; CHECK-NEXT: --> (%cond_p0 umin %cond_p1) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_implies_poison_complex1 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((%n + %m) umin (1 + %n + %m)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((%n + %m) umin (1 + %n + %m)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((%n + %m) umin (1 + %n + %m)) ; CHECK-NEXT: Predicates: @@ -807,7 +807,7 @@ define i32 @logical_and_implies_poison_complex2(i32 %n, i32 %m, i32 %l) { ; CHECK-NEXT: --> (%cond_p0 umin %cond_p1) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_implies_poison_complex2 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((%n + %m) umin (%n + %m + %l)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((%n + %m) umin (%n + %m + %l)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((%n + %m) umin (%n + %m + %l)) ; CHECK-NEXT: Predicates: @@ -843,7 +843,7 @@ define i32 @logical_and_implies_poison_complex_wrong_direction(i32 %n, i32 %m, i ; CHECK-NEXT: --> (%cond_p0 umin_seq %cond_p1) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_implies_poison_complex_wrong_direction ; CHECK-NEXT: Loop %loop: backedge-taken count is ((%n + %m) umin_seq (%n + %m + %l)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((%n + %m) umin_seq (%n + %m + %l)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((%n + %m) umin_seq (%n + %m + %l)) ; CHECK-NEXT: Predicates: @@ -879,7 +879,7 @@ define i32 @logical_and_implies_multiple_ops(i32 %n, i32 %m) { ; CHECK-NEXT: --> ((%cond_p0 umin %cond_p1) umin_seq %cond_p2) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_implies_multiple_ops ; CHECK-NEXT: Loop %loop: backedge-taken count is (((1 + %n) umin %n) umin_seq %m) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (((1 + %n) umin %n) umin_seq %m) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (((1 + %n) umin %n) umin_seq %m) ; CHECK-NEXT: Predicates: @@ -916,7 +916,7 @@ define i32 @logical_and_implies_multiple_ops2(i32 %n, i32 %m) { ; CHECK-NEXT: --> (%cond_p0 umin_seq (%cond_p1 umin %cond_p2)) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_implies_multiple_ops2 ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin_seq ((1 + %n) umin %m)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin_seq ((1 + %n) umin %m)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin_seq ((1 + %n) umin %m)) ; CHECK-NEXT: Predicates: @@ -953,7 +953,7 @@ define i32 @logical_and_implies_multiple_ops3(i32 %n, i32 %m) { ; CHECK-NEXT: --> (%cond_p0 umin_seq %cond_p1 umin_seq %cond_p2) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_implies_multiple_ops3 ; CHECK-NEXT: Loop %loop: backedge-taken count is (%m umin_seq ((1 + %n) umin %n)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%m umin_seq ((1 + %n) umin %n)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%m umin_seq ((1 + %n) umin %n)) ; CHECK-NEXT: Predicates: @@ -990,7 +990,7 @@ define i32 @logical_and_not_zero(i16 %n, i32 %m) { ; CHECK-NEXT: --> (%cond_p0 umin_seq %cond_p1) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_not_zero ; CHECK-NEXT: Loop %loop: backedge-taken count is ((1 + (zext i16 %n to i32)) umin %m) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 65536 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 65536 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((1 + (zext i16 %n to i32)) umin %m) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((1 + (zext i16 %n to i32)) umin %m) ; CHECK-NEXT: Predicates: @@ -1026,7 +1026,7 @@ define i32 @logical_and_not_zero_wrong_order(i16 %n, i32 %m) { ; CHECK-NEXT: --> (%cond_p0 umin_seq %cond_p1) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_not_zero_wrong_order ; CHECK-NEXT: Loop %loop: backedge-taken count is (%m umin_seq (1 + (zext i16 %n to i32))) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 65536 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 65536 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%m umin_seq (1 + (zext i16 %n to i32))) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%m umin_seq (1 + (zext i16 %n to i32))) ; CHECK-NEXT: Predicates: @@ -1058,7 +1058,7 @@ define i32 @logical_and_not_zero_needs_context(i32 %n, i32 %m) { ; CHECK-NEXT: --> (%cond_p0 umin_seq %cond_p1) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_not_zero_needs_context ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin_seq %m) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin_seq %m) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin_seq %m) ; CHECK-NEXT: Predicates: @@ -1097,7 +1097,7 @@ define i32 @logical_and_known_smaller(i16 %n, i16 %m) { ; CHECK-NEXT: --> (%cond_p0 umin_seq %cond_p1) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_known_smaller ; CHECK-NEXT: Loop %loop: backedge-taken count is (zext i16 %n to i32) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 65535 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 65535 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i16 %n to i32) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (zext i16 %n to i32) ; CHECK-NEXT: Predicates: @@ -1136,7 +1136,7 @@ define i32 @logical_and_known_smaller_equal(i16 %n, i16 %m) { ; CHECK-NEXT: --> (%cond_p0 umin_seq %cond_p1) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_known_smaller_equal ; CHECK-NEXT: Loop %loop: backedge-taken count is (zext i16 %n to i32) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 65535 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 65535 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i16 %n to i32) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (zext i16 %n to i32) ; CHECK-NEXT: Predicates: @@ -1175,7 +1175,7 @@ define i32 @logical_and_not_known_smaller_equal(i16 %n, i16 %m) { ; CHECK-NEXT: --> (%cond_p0 umin_seq %cond_p1) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_not_known_smaller_equal ; CHECK-NEXT: Loop %loop: backedge-taken count is ((zext i16 %n to i32) umin_seq (65534 + (zext i16 %m to i32))) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 65535 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 65535 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((zext i16 %n to i32) umin_seq (65534 + (zext i16 %m to i32))) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((zext i16 %n to i32) umin_seq (65534 + (zext i16 %m to i32))) ; CHECK-NEXT: Predicates: @@ -1214,7 +1214,7 @@ define i32 @logical_and_known_greater(i16 %n, i16 %m) { ; CHECK-NEXT: --> (%cond_p0 umin_seq %cond_p1) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_known_greater ; CHECK-NEXT: Loop %loop: backedge-taken count is (zext i16 %n to i32) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 65535 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 65535 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i16 %n to i32) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (zext i16 %n to i32) ; CHECK-NEXT: Predicates: @@ -1253,7 +1253,7 @@ define i32 @logical_and_known_greater_equal(i16 %n, i16 %m) { ; CHECK-NEXT: --> (%cond_p0 umin_seq %cond_p1) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_known_greater_equal ; CHECK-NEXT: Loop %loop: backedge-taken count is (zext i16 %n to i32) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 65535 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 65535 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i16 %n to i32) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (zext i16 %n to i32) ; CHECK-NEXT: Predicates: @@ -1292,7 +1292,7 @@ define i32 @logical_and_not_known_greater_equal(i16 %n, i16 %m) { ; CHECK-NEXT: --> (%cond_p0 umin_seq %cond_p1) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_not_known_greater_equal ; CHECK-NEXT: Loop %loop: backedge-taken count is ((zext i16 %n to i32) umin (65534 + (zext i16 %m to i32))) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 65535 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 65535 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((zext i16 %n to i32) umin (65534 + (zext i16 %m to i32))) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((zext i16 %n to i32) umin (65534 + (zext i16 %m to i32))) ; CHECK-NEXT: Predicates: @@ -1325,7 +1325,7 @@ define i32 @logical_and_zero_arg1(i32 %n) { ; CHECK-NEXT: --> (%cond_p0 umin_seq %cond_p1) U: full-set S: full-set Exits: false LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_zero_arg1 ; CHECK-NEXT: Loop %loop: backedge-taken count is 0 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 0 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 0 ; CHECK-NEXT: Predicates: @@ -1355,7 +1355,7 @@ define i32 @logical_and_zero_arg2(i32 %n) { ; CHECK-NEXT: --> (%cond_p1 umin %cond_p0) U: full-set S: full-set Exits: false LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_zero_arg2 ; CHECK-NEXT: Loop %loop: backedge-taken count is 0 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 0 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 0 ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/exit-count-select.ll b/llvm/test/Analysis/ScalarEvolution/exit-count-select.ll index f9de34e54b9e..060fb7967120 100644 --- a/llvm/test/Analysis/ScalarEvolution/exit-count-select.ll +++ b/llvm/test/Analysis/ScalarEvolution/exit-count-select.ll @@ -14,7 +14,7 @@ define void @logical_and_m_const(i32 %n) { ; CHECK-NEXT: --> (%cond_i2 umin %cond_i) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_m_const ; CHECK-NEXT: Loop %loop: backedge-taken count is (2 umin %n) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 2 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (2 umin %n) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (2 umin %n) ; CHECK-NEXT: Predicates: @@ -46,7 +46,7 @@ define void @logical_and_nonzero(i32 %m) { ; CHECK-NEXT: --> (%cond_i umin_seq %cond_i2) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_nonzero ; CHECK-NEXT: Loop %loop: backedge-taken count is (2 umin %m) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 2 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (2 umin %m) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (2 umin %m) ; CHECK-NEXT: Predicates: @@ -79,7 +79,7 @@ define void @logical_and_zero(i32 %m) { ; CHECK-NEXT: --> (%cond_i umin_seq %cond_i2) U: full-set S: full-set Exits: false LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_zero ; CHECK-NEXT: Loop %loop: backedge-taken count is 0 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 0 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 0 ; CHECK-NEXT: Predicates: @@ -143,7 +143,7 @@ define void @logical_or_m_const(i32 %n) { ; CHECK-NEXT: --> (true + ((true + %cond_i) umin (true + %cond_i2))) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_or_m_const ; CHECK-NEXT: Loop %loop: backedge-taken count is (2 umin %n) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 2 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (2 umin %n) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (2 umin %n) ; CHECK-NEXT: Predicates: @@ -175,7 +175,7 @@ define void @logical_or_nonzero(i32 %m) { ; CHECK-NEXT: --> (true + ((true + %cond_i) umin_seq (true + %cond_i2))) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_or_nonzero ; CHECK-NEXT: Loop %loop: backedge-taken count is (2 umin %m) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 2 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (2 umin %m) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (2 umin %m) ; CHECK-NEXT: Predicates: @@ -208,7 +208,7 @@ define void @logical_or_zero(i32 %m) { ; CHECK-NEXT: --> (true + ((true + %cond_i) umin_seq (true + %cond_i2))) U: full-set S: full-set Exits: true LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_or_zero ; CHECK-NEXT: Loop %loop: backedge-taken count is 0 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 0 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 0 ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/exponential-behavior.ll b/llvm/test/Analysis/ScalarEvolution/exponential-behavior.ll index adeaaf236d67..1de7aa5e870e 100644 --- a/llvm/test/Analysis/ScalarEvolution/exponential-behavior.ll +++ b/llvm/test/Analysis/ScalarEvolution/exponential-behavior.ll @@ -8,7 +8,7 @@ define void @f(i32 %n, ptr %ptr) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for loop: 0 ; CHECK-NEXT: exit count for be: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 0 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 ; CHECK-NEXT: symbolic max exit count for loop: 0 ; CHECK-NEXT: symbolic max exit count for be: ***COULDNOTCOMPUTE*** diff --git a/llvm/test/Analysis/ScalarEvolution/finite-trip-count.ll b/llvm/test/Analysis/ScalarEvolution/finite-trip-count.ll index af7454a3570a..339c3ffa224e 100644 --- a/llvm/test/Analysis/ScalarEvolution/finite-trip-count.ll +++ b/llvm/test/Analysis/ScalarEvolution/finite-trip-count.ll @@ -10,7 +10,7 @@ define void @sle_pre_inc(i32 %len) willreturn { ; CHECK-LABEL: 'sle_pre_inc' ; CHECK-NEXT: Determining loop execution counts for: @sle_pre_inc ; CHECK-NEXT: Loop %for.body: backedge-taken count is (0 smax (1 + %len)) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 2147483647 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483647 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (0 smax (1 + %len)) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (0 smax (1 + %len)) ; CHECK-NEXT: Predicates: @@ -34,7 +34,7 @@ define void @sle_post_inc(i32 %len) willreturn { ; CHECK-LABEL: 'sle_post_inc' ; CHECK-NEXT: Determining loop execution counts for: @sle_post_inc ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (1 smax (1 + %len))) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (1 smax (1 + %len))) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (1 smax (1 + %len))) ; CHECK-NEXT: Predicates: @@ -80,7 +80,7 @@ define void @ule_pre_inc(i32 %len) willreturn { ; CHECK-LABEL: 'ule_pre_inc' ; CHECK-NEXT: Determining loop execution counts for: @ule_pre_inc ; CHECK-NEXT: Loop %for.body: backedge-taken count is (1 + %len) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (1 + %len) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (1 + %len) ; CHECK-NEXT: Predicates: @@ -104,7 +104,7 @@ define void @ule_post_inc(i32 %len) willreturn { ; CHECK-LABEL: 'ule_post_inc' ; CHECK-NEXT: Determining loop execution counts for: @ule_post_inc ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (1 umax (1 + %len))) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (1 umax (1 + %len))) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (1 umax (1 + %len))) ; CHECK-NEXT: Predicates: @@ -150,7 +150,7 @@ define void @sge_pre_inc(i32 %end) willreturn { ; CHECK-LABEL: 'sge_pre_inc' ; CHECK-NEXT: Determining loop execution counts for: @sge_pre_inc ; CHECK-NEXT: Loop %for.body: backedge-taken count is (100 + (-1 * (100 smin (-1 + %end)))) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -2147483548 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2147483548 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (100 + (-1 * (100 smin (-1 + %end)))) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (100 + (-1 * (100 smin (-1 + %end)))) ; CHECK-NEXT: Predicates: @@ -174,7 +174,7 @@ define void @sge_post_inc(i32 %end) willreturn { ; CHECK-LABEL: 'sge_post_inc' ; CHECK-NEXT: Determining loop execution counts for: @sge_post_inc ; CHECK-NEXT: Loop %for.body: backedge-taken count is (100 + (-1 * (100 smin (-1 + %end)))) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -2147483548 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2147483548 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (100 + (-1 * (100 smin (-1 + %end)))) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (100 + (-1 * (100 smin (-1 + %end)))) ; CHECK-NEXT: Predicates: @@ -220,7 +220,7 @@ define void @use_pre_inc(i32 %end) willreturn { ; CHECK-LABEL: 'use_pre_inc' ; CHECK-NEXT: Determining loop execution counts for: @use_pre_inc ; CHECK-NEXT: Loop %for.body: backedge-taken count is (100 + (-1 * (100 umin (-1 + %end)))) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 100 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 100 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (100 + (-1 * (100 umin (-1 + %end)))) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (100 + (-1 * (100 umin (-1 + %end)))) ; CHECK-NEXT: Predicates: @@ -244,7 +244,7 @@ define void @use_post_inc(i32 %end) willreturn { ; CHECK-LABEL: 'use_post_inc' ; CHECK-NEXT: Determining loop execution counts for: @use_post_inc ; CHECK-NEXT: Loop %for.body: backedge-taken count is (99 + (-1 * (99 umin (-1 + %end)))) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 99 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 99 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (99 + (-1 * (99 umin (-1 + %end)))) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (99 + (-1 * (99 umin (-1 + %end)))) ; CHECK-NEXT: Predicates: @@ -290,7 +290,7 @@ define void @pr54191(i64 %n) mustprogress { ; CHECK-LABEL: 'pr54191' ; CHECK-NEXT: Determining loop execution counts for: @pr54191 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-2 + %n) /u 2) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4611686018427387902 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4611686018427387902 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-2 + %n) /u 2) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-2 + %n) /u 2) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/flags-from-poison-noautogen.ll b/llvm/test/Analysis/ScalarEvolution/flags-from-poison-noautogen.ll index 0f49ad4aabc4..eed81eec97ce 100644 --- a/llvm/test/Analysis/ScalarEvolution/flags-from-poison-noautogen.ll +++ b/llvm/test/Analysis/ScalarEvolution/flags-from-poison-noautogen.ll @@ -24,13 +24,13 @@ define void @subrecurrences(i32 %outer_l, i32 %inner_l, i32 %val) { ; CHECK-NEXT: --> (1 /u {{\{\{}}-1,+,-1}<%outer>,+,1}<%inner>) U: [0,2) S: [0,2) Exits: (1 /u {(-1 + %inner_l),+,-1}<%outer>) LoopDispositions: { %inner: Computable, %outer: Variant } ; CHECK-NEXT: Determining loop execution counts for: @subrecurrences ; CHECK-NEXT: Loop %inner: backedge-taken count is %inner_l -; CHECK-NEXT: Loop %inner: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %inner: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %inner: symbolic max backedge-taken count is %inner_l ; CHECK-NEXT: Loop %inner: Predicated backedge-taken count is %inner_l ; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %inner: Trip multiple is 1 ; CHECK-NEXT: Loop %outer: backedge-taken count is %outer_l -; CHECK-NEXT: Loop %outer: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %outer: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %outer: symbolic max backedge-taken count is %outer_l ; CHECK-NEXT: Loop %outer: Predicated backedge-taken count is %outer_l ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/flags-from-poison.ll b/llvm/test/Analysis/ScalarEvolution/flags-from-poison.ll index 84e68dd17577..16ffb54b01f6 100644 --- a/llvm/test/Analysis/ScalarEvolution/flags-from-poison.ll +++ b/llvm/test/Analysis/ScalarEvolution/flags-from-poison.ll @@ -30,7 +30,7 @@ define void @test-add-nsw(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,0) S: [1,0) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-add-nsw ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -69,7 +69,7 @@ define void @test-add-nuw(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,0) S: [1,0) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-add-nuw ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -109,7 +109,7 @@ define void @test-add-scope-invariant(ptr %input, i32 %needle) { ; CHECK-NEXT: --> ((4 * (sext i32 {(1 + %offset),+,1}<%loop> to i64)) + %input) U: full-set S: full-set Exits: ((4 * (sext i32 %needle to i64)) + %input) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-add-scope-invariant ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %offset) + %needle) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %offset) + %needle) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %offset) + %needle) ; CHECK-NEXT: Predicates: @@ -392,7 +392,7 @@ define void @test-add-nuw-from-icmp(ptr %input, i32 %offset, ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,0) S: [1,0) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-add-nuw-from-icmp ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -432,7 +432,7 @@ define void @test-add-no-load(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,0) S: [1,0) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-add-no-load ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -470,7 +470,7 @@ define void @test-add-not-header(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,-2147483648) S: [1,-2147483648) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-add-not-header ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -509,7 +509,7 @@ define void @test-add-not-header2(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,-2147483648) S: [1,-2147483648) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-add-not-header2 ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -552,7 +552,7 @@ define void @test-add-not-header3(ptr %input, i32 %offset, i32 %numIterations, ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for loop2: (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: symbolic max exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for loop2: (-1 + %numIterations) @@ -593,7 +593,7 @@ define void @test-add-not-header4(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,-2147483648) S: [1,-2147483648) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-add-not-header4 ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -670,7 +670,7 @@ define void @test-add-not-header6(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: --> {((4 * (sext i32 (1 + %offset) to i64)) + %input),+,4}<%loop> U: full-set S: full-set Exits: ((4 * (zext i32 (-1 + %numIterations) to i64)) + (4 * (sext i32 (1 + %offset) to i64)) + %input) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-add-not-header6 ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -709,7 +709,7 @@ define void @test-add-not-header7(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: --> {((4 * (sext i32 (1 + %offset) to i64)) + %input),+,4}<%loop> U: full-set S: full-set Exits: ((4 * (zext i32 (-1 + %numIterations) to i64)) + (4 * (sext i32 (1 + %offset) to i64)) + %input) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-add-not-header7 ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -750,7 +750,7 @@ define void @test-add-not-header8(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: --> ((4 * (sext i32 {(1 + %offset),+,1}<%loop> to i64)) + %input) U: full-set S: full-set Exits: ((4 * (sext i32 (%offset + %numIterations) to i64)) + %input) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-add-not-header8 ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -791,7 +791,7 @@ define void @test-add-call(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,0) S: [1,0) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-add-call ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -829,7 +829,7 @@ define void @test-add-call2(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,0) S: [1,0) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-add-call2 ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -866,7 +866,7 @@ define void @test-gep-propagates-poison(ptr %input, i32 %offset, i32 %numIterati ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,-2147483648) S: [1,-2147483648) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-gep-propagates-poison ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -905,7 +905,7 @@ define void @test-add-mul-propagates(ptr %input, i32 %offset, i32 %numIterations ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,-2147483648) S: [1,-2147483648) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-add-mul-propagates ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -944,7 +944,7 @@ define void @test-mul-propagates-poison(ptr %input, i32 %offset, i32 %numIterati ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,-2147483648) S: [1,-2147483648) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-mul-propagates-poison ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -982,7 +982,7 @@ define void @test-mul-propagates-poison-2(ptr %input, i32 %offset, i32 %numItera ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,-2147483648) S: [1,-2147483648) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-mul-propagates-poison-2 ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -1019,7 +1019,7 @@ define void @test-add-div(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,-2147483648) S: [1,-2147483648) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-add-div ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -1054,7 +1054,7 @@ define void @test-add-div2(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,-2147483648) S: [1,-2147483648) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-add-div2 ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -1089,7 +1089,7 @@ define void @test-add-store(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,-2147483648) S: [1,-2147483648) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-add-store ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -1132,7 +1132,7 @@ define void @test-add-twice(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,-2147483648) S: [1,-2147483648) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-add-twice ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -1176,7 +1176,7 @@ define void @test-mul-nsw(ptr %input, i32 %stride, i32 %numIterations) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,-2147483648) S: [1,-2147483648) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-mul-nsw ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -1214,7 +1214,7 @@ define void @test-mul-nuw(ptr %input, i32 %stride, i32 %numIterations) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,0) S: [1,0) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-mul-nuw ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -1254,7 +1254,7 @@ define void @test-shl-nsw(ptr %input, i32 %start, i32 %numIterations) { ; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: [-2147483647,-2147483648) S: [-2147483647,-2147483648) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-shl-nsw ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %start) + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %start) + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %start) + %numIterations) ; CHECK-NEXT: Predicates: @@ -1294,7 +1294,7 @@ define void @test-shl-nuw-edgecase(ptr %input, i32 %start, i32 %numIterations) { ; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: [-2147483647,-2147483648) S: [-2147483647,-2147483648) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-shl-nuw-edgecase ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %start) + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %start) + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %start) + %numIterations) ; CHECK-NEXT: Predicates: @@ -1334,7 +1334,7 @@ define void @test-shl-nuw-nsw(ptr %input, i32 %start, i32 %numIterations) { ; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: [-2147483647,-2147483648) S: [-2147483647,-2147483648) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-shl-nuw-nsw ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %start) + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %start) + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %start) + %numIterations) ; CHECK-NEXT: Predicates: @@ -1374,7 +1374,7 @@ define void @test-shl-no-nsw(ptr %input, i32 %start, i32 %numIterations) { ; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: [-2147483647,-2147483648) S: [-2147483647,-2147483648) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-shl-no-nsw ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %start) + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %start) + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %start) + %numIterations) ; CHECK-NEXT: Predicates: @@ -1414,7 +1414,7 @@ define void @test-shl-nsw-edgecase(ptr %input, i32 %start, i32 %numIterations) { ; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: [-2147483647,-2147483648) S: [-2147483647,-2147483648) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-shl-nsw-edgecase ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %start) + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %start) + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %start) + %numIterations) ; CHECK-NEXT: Predicates: @@ -1452,7 +1452,7 @@ define void @test-shl-nuw(ptr %input, i32 %numIterations) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,0) S: [1,0) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-shl-nuw ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -1493,7 +1493,7 @@ define void @test-sub-no-nsw(ptr %input, i32 %start, i32 %sub, i32 %numIteration ; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: [-2147483647,-2147483648) S: [-2147483647,-2147483648) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-sub-no-nsw ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %start) + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %start) + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %start) + %numIterations) ; CHECK-NEXT: Predicates: @@ -1535,7 +1535,7 @@ define void @test-sub-nsw(ptr %input, i32 %start, i32 %sub, i32 %numIterations) ; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: [-2147483647,-2147483648) S: [-2147483647,-2147483648) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-sub-nsw ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %start) + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %start) + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %start) + %numIterations) ; CHECK-NEXT: Predicates: @@ -1576,7 +1576,7 @@ define void @test-sub-nsw-lhs-non-negative(ptr %input, i32 %sub, i32 %numIterati ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,-2147483648) S: [1,-2147483648) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-sub-nsw-lhs-non-negative ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -1621,7 +1621,7 @@ define void @test-sext-sub(ptr %input, i32 %sub, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for cont: (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: symbolic max exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for cont: (-1 + %numIterations) @@ -1674,7 +1674,7 @@ define void @test-sub-with-add(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,-2147483648) S: [1,-2147483648) Exits: %numIterations LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-sub-with-add ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/fold.ll b/llvm/test/Analysis/ScalarEvolution/fold.ll index e8e3dc97c751..485458b8ec84 100644 --- a/llvm/test/Analysis/ScalarEvolution/fold.ll +++ b/llvm/test/Analysis/ScalarEvolution/fold.ll @@ -78,7 +78,7 @@ define void @test4(i32 %x, i32 %y) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,22) S: [1,22) Exits: 21 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test4 ; CHECK-NEXT: Loop %loop: backedge-taken count is 20 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 20 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 20 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 20 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 20 ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/huge-trip-multiple.ll b/llvm/test/Analysis/ScalarEvolution/huge-trip-multiple.ll index 55fcea22b584..463f54fb549f 100644 --- a/llvm/test/Analysis/ScalarEvolution/huge-trip-multiple.ll +++ b/llvm/test/Analysis/ScalarEvolution/huge-trip-multiple.ll @@ -15,7 +15,7 @@ define void @trip_count_4294967295() { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,4294967296) S: [1,4294967296) Exits: 4294967295 LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @trip_count_4294967295 ; CHECK-NEXT: Loop %for.body: backedge-taken count is 4294967294 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 4294967294 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 4294967294 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 4294967294 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 4294967294 ; CHECK-NEXT: Predicates: @@ -44,7 +44,7 @@ define void @trip_count_4294967296() { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,4294967297) S: [1,4294967297) Exits: 4294967296 LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @trip_count_4294967296 ; CHECK-NEXT: Loop %for.body: backedge-taken count is 4294967295 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 4294967295 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 4294967295 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 4294967295 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 4294967295 ; CHECK-NEXT: Predicates: @@ -73,7 +73,7 @@ define void @trip_count_8589935692() { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,8589934593) S: [1,8589934593) Exits: 8589934592 LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @trip_count_8589935692 ; CHECK-NEXT: Loop %for.body: backedge-taken count is 8589934591 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 8589934591 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 8589934591 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 8589934591 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 8589934591 ; CHECK-NEXT: Predicates: @@ -102,7 +102,7 @@ define void @trip_count_9223372036854775808() { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-9223372036854775807) S: [1,-9223372036854775807) Exits: -9223372036854775808 LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @trip_count_9223372036854775808 ; CHECK-NEXT: Loop %for.body: backedge-taken count is 9223372036854775807 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 9223372036854775807 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 9223372036854775807 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 9223372036854775807 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 9223372036854775807 ; CHECK-NEXT: Predicates: @@ -131,7 +131,7 @@ define void @trip_count_18446744073709551615() { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,0) S: [1,0) Exits: -1 LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @trip_count_18446744073709551615 ; CHECK-NEXT: Loop %for.body: backedge-taken count is -2 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is -2 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is -2 ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/incorrect-exit-count.ll b/llvm/test/Analysis/ScalarEvolution/incorrect-exit-count.ll index 4ddf64c2c123..cb8429d9ecd9 100644 --- a/llvm/test/Analysis/ScalarEvolution/incorrect-exit-count.ll +++ b/llvm/test/Analysis/ScalarEvolution/incorrect-exit-count.ll @@ -60,7 +60,7 @@ define dso_local i32 @f() { ; CHECK-NEXT: Loop %for.cond6: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for for.cond6: 0 ; CHECK-NEXT: exit count for for.end: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %for.cond6: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %for.cond6: constant max backedge-taken count is i32 0 ; CHECK-NEXT: Loop %for.cond6: symbolic max backedge-taken count is 0 ; CHECK-NEXT: symbolic max exit count for for.cond6: 0 ; CHECK-NEXT: symbolic max exit count for for.end: ***COULDNOTCOMPUTE*** @@ -68,7 +68,7 @@ define dso_local i32 @f() { ; CHECK-NEXT: Loop %inner.loop: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for inner.loop: 0 ; CHECK-NEXT: exit count for for.end.3: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %inner.loop: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %inner.loop: constant max backedge-taken count is i32 0 ; CHECK-NEXT: Loop %inner.loop: symbolic max backedge-taken count is 0 ; CHECK-NEXT: symbolic max exit count for inner.loop: 0 ; CHECK-NEXT: symbolic max exit count for for.end.3: ***COULDNOTCOMPUTE*** @@ -77,7 +77,7 @@ define dso_local i32 @f() { ; CHECK-NEXT: exit count for for.cond6: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for inner.loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for for.inc13.3: 2 -; CHECK-NEXT: Loop %outer.loop: constant max backedge-taken count is 2 +; CHECK-NEXT: Loop %outer.loop: constant max backedge-taken count is i32 2 ; CHECK-NEXT: Loop %outer.loop: symbolic max backedge-taken count is 2 ; CHECK-NEXT: symbolic max exit count for for.cond6: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for inner.loop: ***COULDNOTCOMPUTE*** diff --git a/llvm/test/Analysis/ScalarEvolution/increasing-or-decreasing-iv.ll b/llvm/test/Analysis/ScalarEvolution/increasing-or-decreasing-iv.ll index a264a7bf4979..9ec4f5eb6688 100644 --- a/llvm/test/Analysis/ScalarEvolution/increasing-or-decreasing-iv.ll +++ b/llvm/test/Analysis/ScalarEvolution/increasing-or-decreasing-iv.ll @@ -18,7 +18,7 @@ define void @f0(i1 %c) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,129) S: [1,129) Exits: 128 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @f0 ; CHECK-NEXT: Loop %loop: backedge-taken count is 127 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 127 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 127 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 127 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 127 ; CHECK-NEXT: Predicates: @@ -86,7 +86,7 @@ define void @f1(i1 %c) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,17) S: [1,17) Exits: 16 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @f1 ; CHECK-NEXT: Loop %loop: backedge-taken count is 15 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 15 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 15 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 15 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 15 ; CHECK-NEXT: Predicates: @@ -147,7 +147,7 @@ define void @f2(i1 %c) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,129) S: [1,129) Exits: 128 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @f2 ; CHECK-NEXT: Loop %loop: backedge-taken count is 127 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 127 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 127 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 127 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 127 ; CHECK-NEXT: Predicates: @@ -190,7 +190,7 @@ define void @f3(i1 %c) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,129) S: [1,129) Exits: 128 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @f3 ; CHECK-NEXT: Loop %loop: backedge-taken count is 127 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 127 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i16 127 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 127 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 127 ; CHECK-NEXT: Predicates: @@ -240,7 +240,7 @@ define void @f4(i1 %c) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,129) S: [1,129) Exits: 128 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @f4 ; CHECK-NEXT: Loop %loop: backedge-taken count is 127 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 127 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 127 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 127 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 127 ; CHECK-NEXT: Predicates: @@ -284,7 +284,7 @@ define void @f5(i1 %c) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,129) S: [1,129) Exits: 128 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @f5 ; CHECK-NEXT: Loop %loop: backedge-taken count is 127 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 127 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i16 127 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 127 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 127 ; CHECK-NEXT: Predicates: @@ -330,7 +330,7 @@ define void @f6(i1 %c) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,129) S: [1,129) Exits: 128 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @f6 ; CHECK-NEXT: Loop %loop: backedge-taken count is 127 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 127 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i16 127 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 127 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 127 ; CHECK-NEXT: Predicates: @@ -379,7 +379,7 @@ define void @f7(i1 %c) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,129) S: [1,129) Exits: 128 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @f7 ; CHECK-NEXT: Loop %loop: backedge-taken count is 127 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 127 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i16 127 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 127 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 127 ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/invalidation.ll b/llvm/test/Analysis/ScalarEvolution/invalidation.ll index 0aa0aa68842b..f5d9c2b083d5 100644 --- a/llvm/test/Analysis/ScalarEvolution/invalidation.ll +++ b/llvm/test/Analysis/ScalarEvolution/invalidation.ll @@ -40,7 +40,7 @@ target triple = "x86_64-unknown-linux-gnu" define void @test(i32 %n) { ; CHECK-LABEL: Classifying expressions for: @test ; CHECK: Loop %loop: backedge-taken count is 14 -; CHECK: Loop %loop: constant max backedge-taken count is 14 +; CHECK: Loop %loop: constant max backedge-taken count is i32 14 ; CHECK: Loop %loop: Predicated backedge-taken count is 14 entry: diff --git a/llvm/test/Analysis/ScalarEvolution/load-with-range-metadata.ll b/llvm/test/Analysis/ScalarEvolution/load-with-range-metadata.ll index dd781731894d..6d8416cc3e89 100644 --- a/llvm/test/Analysis/ScalarEvolution/load-with-range-metadata.ll +++ b/llvm/test/Analysis/ScalarEvolution/load-with-range-metadata.ll @@ -6,7 +6,7 @@ define i32 @slt_trip_count_with_range(ptr %ptr0, ptr %ptr1) { ; CHECK-LABEL: 'slt_trip_count_with_range' ; CHECK-NEXT: Determining loop execution counts for: @slt_trip_count_with_range ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %limit) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 98 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 98 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %limit) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %limit) ; CHECK-NEXT: Predicates: @@ -31,7 +31,7 @@ define i32 @ult_trip_count_with_range(ptr %ptr0, ptr %ptr1) { ; CHECK-LABEL: 'ult_trip_count_with_range' ; CHECK-NEXT: Determining loop execution counts for: @ult_trip_count_with_range ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %limit) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 98 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 98 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %limit) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %limit) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/load.ll b/llvm/test/Analysis/ScalarEvolution/load.ll index 83b60c58d517..f31dea4db446 100644 --- a/llvm/test/Analysis/ScalarEvolution/load.ll +++ b/llvm/test/Analysis/ScalarEvolution/load.ll @@ -31,7 +31,7 @@ define i32 @test1() nounwind readnone { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,51) S: [1,51) Exits: 50 LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test1 ; CHECK-NEXT: Loop %for.body: backedge-taken count is 49 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 49 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 49 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 49 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 49 ; CHECK-NEXT: Predicates: @@ -85,7 +85,7 @@ define i32 @test2() nounwind uwtable readonly { ; CHECK-NEXT: --> %1 U: full-set S: full-set Exits: null LoopDispositions: { %for.body: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test2 ; CHECK-NEXT: Loop %for.body: backedge-taken count is 4 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 4 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 4 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 4 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 4 ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/logical-operations.ll b/llvm/test/Analysis/ScalarEvolution/logical-operations.ll index 3fe1bd1e1646..7f4e6a0bd95f 100644 --- a/llvm/test/Analysis/ScalarEvolution/logical-operations.ll +++ b/llvm/test/Analysis/ScalarEvolution/logical-operations.ll @@ -432,7 +432,7 @@ define ptr @tautological_select_like_phi(i32 %tc) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,102) S: [1,102) Exits: 101 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @tautological_select_like_phi ; CHECK-NEXT: Loop %loop: backedge-taken count is 100 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 100 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 100 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 100 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 100 ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-rewrite-expressions.ll b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-rewrite-expressions.ll index 483ce64072ed..bc01ade3a219 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-rewrite-expressions.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-rewrite-expressions.ll @@ -17,7 +17,7 @@ define void @rewrite_zext(i32 %n) { ; CHECK-NEXT: --> {8,+,8}<%loop> U: [8,25) S: [8,25) Exits: (8 + (8 * ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_zext ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 2 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8) ; CHECK-NEXT: Predicates: @@ -61,7 +61,7 @@ define i32 @rewrite_zext_min_max(i32 %N, ptr %arr) { ; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_zext_min_max ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) ; CHECK-NEXT: Predicates: @@ -109,7 +109,7 @@ define i32 @rewrite_min_max_zext(i32 %N, ptr %arr) { ; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_min_max_zext ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) ; CHECK-NEXT: Predicates: @@ -157,7 +157,7 @@ define i32 @rewrite_sext_min_max(i32 %N, ptr %arr) { ; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_sext_min_max ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Predicates: @@ -205,7 +205,7 @@ define i32 @rewrite_min_max_sext(i32 %N, ptr %arr) { ; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_min_max_sext ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Predicates: @@ -255,7 +255,7 @@ define i32 @rewrite_zext_with_info_from_icmp_ne(i32 %N) { ; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,5) S: [4,5) Exits: 4 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_zext_with_info_from_icmp_ne ; CHECK-NEXT: Loop %loop: backedge-taken count is 0 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 0 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 0 ; CHECK-NEXT: Predicates: @@ -305,7 +305,7 @@ define i32 @rewrite_zext_no_icmp_ne(i32 %N) { ; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,4294967297) S: [4,4294967297) Exits: (4 + (4 * ((-4 + (4 * ((4 + (zext i32 (-1 + (zext i2 (trunc i32 %N to i2) to i32)) to i64)) /u 4))) /u 4))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_zext_no_icmp_ne ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * ((4 + (zext i32 (-1 + (zext i2 (trunc i32 %N to i2) to i32)) to i64)) /u 4))) /u 4) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 1073741823 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 1073741823 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * ((4 + (zext i32 (-1 + (zext i2 (trunc i32 %N to i2) to i32)) to i64)) /u 4))) /u 4) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * ((4 + (zext i32 (-1 + (zext i2 (trunc i32 %N to i2) to i32)) to i64)) /u 4))) /u 4) ; CHECK-NEXT: Predicates: @@ -347,7 +347,7 @@ define void @rewrite_zext_and_base_1(i32 %n) { ; CHECK-NEXT: --> {8,+,8}<%loop> U: [8,33) S: [8,33) Exits: (8 + (8 * ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_zext_and_base_1 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8) ; CHECK-NEXT: Predicates: @@ -392,7 +392,7 @@ define void @rewrite_zext_and_base_2(i32 %n) { ; CHECK-NEXT: --> {8,+,8}<%loop> U: [8,33) S: [8,33) Exits: (8 + (8 * ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_zext_and_base_2 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8) ; CHECK-NEXT: Predicates: @@ -435,7 +435,7 @@ define void @guard_pessimizes_analysis_step2(i1 %c, i32 %N) { ; CHECK-NEXT: --> {(2 + %init),+,2}<%loop> U: [4,19) S: [4,19) Exits: (2 + (2 * ((14 + (-1 * %init)) /u 2)) + %init) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @guard_pessimizes_analysis_step2 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((14 + (-1 * %init)) /u 2) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 6 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 6 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((14 + (-1 * %init)) /u 2) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((14 + (-1 * %init)) /u 2) ; CHECK-NEXT: Predicates: @@ -483,7 +483,7 @@ define i32 @rewrite_sext_slt_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) /u 4))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_sext_slt_narrow_check ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Predicates: @@ -529,7 +529,7 @@ define i32 @rewrite_zext_ult_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) /u 4))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_zext_ult_narrow_check ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Predicates: @@ -575,7 +575,7 @@ define i32 @rewrite_zext_ule_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) /u 4))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_zext_ule_narrow_check ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Predicates: @@ -621,7 +621,7 @@ define i32 @rewrite_zext_sle_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) /u 4))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_zext_sle_narrow_check ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Predicates: @@ -667,7 +667,7 @@ define i32 @rewrite_zext_uge_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_zext_uge_narrow_check ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) ; CHECK-NEXT: Predicates: @@ -713,7 +713,7 @@ define i32 @rewrite_sext_sge_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_sext_sge_narrow_check ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Predicates: @@ -759,7 +759,7 @@ define i32 @rewrite_zext_ugt_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_zext_ugt_narrow_check ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) ; CHECK-NEXT: Predicates: @@ -805,7 +805,7 @@ define i32 @rewrite_sext_sgt_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_sext_sgt_narrow_check ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll index 33baa996faf8..2f293147f0cc 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll @@ -14,7 +14,7 @@ define void @test_guard_less_than_16(ptr nocapture %a, i64 %i) { ; CHECK-NEXT: --> {(1 + %i),+,1}<%loop> U: full-set S: full-set Exits: 16 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_less_than_16 ; CHECK-NEXT: Loop %loop: backedge-taken count is (15 + (-1 * %i)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 15 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 15 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (15 + (-1 * %i)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (15 + (-1 * %i)) ; CHECK-NEXT: Predicates: @@ -47,7 +47,7 @@ define void @test_guard_less_than_16_operands_swapped(ptr nocapture %a, i64 %i) ; CHECK-NEXT: --> {(1 + %i),+,1}<%loop> U: full-set S: full-set Exits: 16 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_less_than_16_operands_swapped ; CHECK-NEXT: Loop %loop: backedge-taken count is (15 + (-1 * %i)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 15 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 15 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (15 + (-1 * %i)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (15 + (-1 * %i)) ; CHECK-NEXT: Predicates: @@ -80,7 +80,7 @@ define void @test_guard_less_than_16_branches_flipped(ptr nocapture %a, i64 %i) ; CHECK-NEXT: --> {(1 + %i),+,1}<%loop> U: full-set S: full-set Exits: 16 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_less_than_16_branches_flipped ; CHECK-NEXT: Loop %loop: backedge-taken count is (15 + (-1 * %i)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (15 + (-1 * %i)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (15 + (-1 * %i)) ; CHECK-NEXT: Predicates: @@ -113,7 +113,7 @@ define void @test_guard_uge_16_branches_flipped(ptr nocapture %a, i64 %i) { ; CHECK-NEXT: --> {(1 + %i),+,1}<%loop> U: full-set S: full-set Exits: 16 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_uge_16_branches_flipped ; CHECK-NEXT: Loop %loop: backedge-taken count is (15 + (-1 * %i)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 15 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 15 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (15 + (-1 * %i)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (15 + (-1 * %i)) ; CHECK-NEXT: Predicates: @@ -146,7 +146,7 @@ define void @test_guard_eq_12(ptr nocapture %a, i64 %N) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,14) S: [1,14) Exits: (1 + %N) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_eq_12 ; CHECK-NEXT: Loop %loop: backedge-taken count is %N -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 12 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 12 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %N ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %N ; CHECK-NEXT: Predicates: @@ -179,7 +179,7 @@ define void @test_guard_ule_12(ptr nocapture %a, i64 %N) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,14) S: [1,14) Exits: (1 + %N) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_ule_12 ; CHECK-NEXT: Loop %loop: backedge-taken count is %N -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 12 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 12 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %N ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %N ; CHECK-NEXT: Predicates: @@ -212,7 +212,7 @@ define void @test_guard_ule_12_step2(ptr nocapture %a, i64 %N) { ; CHECK-NEXT: --> {2,+,2}<%loop> U: [2,15) S: [2,15) Exits: (2 + (2 * (%N /u 2))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_ule_12_step2 ; CHECK-NEXT: Loop %loop: backedge-taken count is (%N /u 2) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 6 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 6 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%N /u 2) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%N /u 2) ; CHECK-NEXT: Predicates: @@ -245,7 +245,7 @@ define void @test_multiple_const_guards_order1(ptr nocapture %a, i64 %i) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,11) S: [1,11) Exits: (1 + %i) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_multiple_const_guards_order1 ; CHECK-NEXT: Loop %loop: backedge-taken count is %i -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 9 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 9 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %i ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %i ; CHECK-NEXT: Predicates: @@ -282,7 +282,7 @@ define void @test_multiple_const_guards_order2(ptr nocapture %a, i64 %i) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,11) S: [1,11) Exits: (1 + %i) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_multiple_const_guards_order2 ; CHECK-NEXT: Loop %loop: backedge-taken count is %i -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 9 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 9 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %i ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %i ; CHECK-NEXT: Predicates: @@ -319,7 +319,7 @@ define void @test_multiple_var_guards_order1(ptr nocapture %a, i64 %i, i64 %N) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,12) S: [1,12) Exits: (1 + %i) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_multiple_var_guards_order1 ; CHECK-NEXT: Loop %loop: backedge-taken count is %i -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 10 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 10 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %i ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %i ; CHECK-NEXT: Predicates: @@ -356,7 +356,7 @@ define void @test_multiple_var_guards_order2(ptr nocapture %a, i64 %i, i64 %N) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,12) S: [1,12) Exits: (1 + %i) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_multiple_var_guards_order2 ; CHECK-NEXT: Loop %loop: backedge-taken count is %i -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 10 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 10 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %i ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %i ; CHECK-NEXT: Predicates: @@ -394,7 +394,7 @@ define void @test_multiple_var_guards_cycle(ptr nocapture %a, i64 %i, i64 %N) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,-1) S: [1,-1) Exits: (1 + %N) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_multiple_var_guards_cycle ; CHECK-NEXT: Loop %loop: backedge-taken count is %N -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %N ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %N ; CHECK-NEXT: Predicates: @@ -431,7 +431,7 @@ define void @test_guard_ult_ne(ptr nocapture readonly %data, i64 %count) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,5) S: [1,5) Exits: %count LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_ult_ne ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) ; CHECK-NEXT: Predicates: @@ -468,7 +468,7 @@ define void @test_guard_ne_ult(ptr nocapture readonly %data, i64 %count) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,5) S: [1,5) Exits: %count LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_ne_ult ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) ; CHECK-NEXT: Predicates: @@ -507,7 +507,7 @@ define void @test_guard_if_and_enter(ptr nocapture readonly %data, i64 %count) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,5) S: [1,5) Exits: %count LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_if_and_enter ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) ; CHECK-NEXT: Predicates: @@ -544,7 +544,7 @@ define void @test_guard_if_and_skip(ptr nocapture readonly %data, i64 %count) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,0) S: [1,0) Exits: %count LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_if_and_skip ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) ; CHECK-NEXT: Predicates: @@ -583,7 +583,7 @@ define void @test_guard_if_and_and(ptr nocapture readonly %data, i64 %count, i1 ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,5) S: [1,5) Exits: %count LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_if_and_and ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) ; CHECK-NEXT: Predicates: @@ -623,7 +623,7 @@ define void @test_guard_if_and_or(ptr nocapture readonly %data, i64 %count, i1 % ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,0) S: [1,0) Exits: %count LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_if_and_or ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) ; CHECK-NEXT: Predicates: @@ -661,7 +661,7 @@ define void @test_guard_if_or_skip(ptr nocapture readonly %data, i64 %count) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,5) S: [1,5) Exits: %count LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_if_or_skip ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) ; CHECK-NEXT: Predicates: @@ -698,7 +698,7 @@ define void @test_guard_if_or_enter(ptr nocapture readonly %data, i64 %count) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,0) S: [1,0) Exits: %count LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_if_or_enter ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) ; CHECK-NEXT: Predicates: @@ -737,7 +737,7 @@ define void @test_guard_if_or_or(ptr nocapture readonly %data, i64 %count, i1 %c ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,5) S: [1,5) Exits: %count LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_if_or_or ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) ; CHECK-NEXT: Predicates: @@ -777,7 +777,7 @@ define void @test_guard_if_or_and(ptr nocapture readonly %data, i64 %count, i1 % ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,0) S: [1,0) Exits: %count LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_if_or_and ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) ; CHECK-NEXT: Predicates: @@ -816,7 +816,7 @@ define void @test_guard_and_assume(ptr nocapture readonly %data, i64 %count) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,5) S: [1,5) Exits: %count LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_and_assume ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) ; CHECK-NEXT: Predicates: @@ -853,7 +853,7 @@ define void @test_guard_assume_and(ptr nocapture readonly %data, i64 %count) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,5) S: [1,5) Exits: %count LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_assume_and ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) ; CHECK-NEXT: Predicates: @@ -892,7 +892,7 @@ define void @guard_pessimizes_analysis_step1(i1 %c, i32 %N) { ; CHECK-NEXT: --> {(1 + %init),+,1}<%loop> U: [3,12) S: [3,12) Exits: 10 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @guard_pessimizes_analysis_step1 ; CHECK-NEXT: Loop %loop: backedge-taken count is (9 + (-1 * %init)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 7 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 7 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (9 + (-1 * %init)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (9 + (-1 * %init)) ; CHECK-NEXT: Predicates: @@ -933,7 +933,7 @@ define void @guard_pessimizes_analysis_step2(i1 %c, i32 %N) { ; CHECK-NEXT: --> {(2 + %init),+,2}<%loop> U: [4,12) S: [4,12) Exits: (2 + (2 * ((8 + (-1 * %init)) /u 2)) + %init) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @guard_pessimizes_analysis_step2 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((8 + (-1 * %init)) /u 2) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((8 + (-1 * %init)) /u 2) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((8 + (-1 * %init)) /u 2) ; CHECK-NEXT: Predicates: @@ -975,7 +975,7 @@ define void @crash(ptr %ptr) { ; CHECK-NEXT: --> {(1 + %ptr),+,1}<%while.body125> U: full-set S: full-set Exits: {(-1 + (-1 * (ptrtoint ptr %ptr to i64)) + %ptr),+,-1}<%while.cond111> LoopDispositions: { %while.body125: Computable } ; CHECK-NEXT: Determining loop execution counts for: @crash ; CHECK-NEXT: Loop %while.body125: backedge-taken count is {(-2 + (-1 * (ptrtoint ptr %ptr to i64))),+,-1}<%while.cond111> -; CHECK-NEXT: Loop %while.body125: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %while.body125: constant max backedge-taken count is i64 -2 ; CHECK-NEXT: Loop %while.body125: symbolic max backedge-taken count is {(-2 + (-1 * (ptrtoint ptr %ptr to i64))),+,-1}<%while.cond111> ; CHECK-NEXT: Loop %while.body125: Predicated backedge-taken count is {(-2 + (-1 * (ptrtoint ptr %ptr to i64))),+,-1}<%while.cond111> ; CHECK-NEXT: Predicates: @@ -1029,7 +1029,7 @@ define void @test_guard_uge(i32 %blockSize) { ; CHECK-NEXT: --> {(-1 + (%blockSize /u 4)),+,-1}<%while.body> U: [-1073741823,1073741823) S: [-1073741823,1073741823) Exits: 0 LoopDispositions: { %while.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_uge ; CHECK-NEXT: Loop %while.body: backedge-taken count is (-1 + (%blockSize /u 4)) -; CHECK-NEXT: Loop %while.body: constant max backedge-taken count is 1073741822 +; CHECK-NEXT: Loop %while.body: constant max backedge-taken count is i32 1073741822 ; CHECK-NEXT: Loop %while.body: symbolic max backedge-taken count is (-1 + (%blockSize /u 4)) ; CHECK-NEXT: Loop %while.body: Predicated backedge-taken count is (-1 + (%blockSize /u 4)) ; CHECK-NEXT: Predicates: @@ -1066,7 +1066,7 @@ define void @test_guard_ugt(i32 %blockSize) { ; CHECK-NEXT: --> {(-1 + (%blockSize /u 4)),+,-1}<%while.body> U: [-1073741823,1073741823) S: [-1073741823,1073741823) Exits: 0 LoopDispositions: { %while.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_ugt ; CHECK-NEXT: Loop %while.body: backedge-taken count is (-1 + (%blockSize /u 4)) -; CHECK-NEXT: Loop %while.body: constant max backedge-taken count is 1073741822 +; CHECK-NEXT: Loop %while.body: constant max backedge-taken count is i32 1073741822 ; CHECK-NEXT: Loop %while.body: symbolic max backedge-taken count is (-1 + (%blockSize /u 4)) ; CHECK-NEXT: Loop %while.body: Predicated backedge-taken count is (-1 + (%blockSize /u 4)) ; CHECK-NEXT: Predicates: @@ -1103,7 +1103,7 @@ define void @test_guard_uge_and_ule(i32 %blockSize) { ; CHECK-NEXT: --> {(-1 + (%blockSize /u 4)),+,-1}<%while.body> U: [-256,1073741823) S: [-256,1073741823) Exits: 0 LoopDispositions: { %while.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_uge_and_ule ; CHECK-NEXT: Loop %while.body: backedge-taken count is (-1 + (%blockSize /u 4)) -; CHECK-NEXT: Loop %while.body: constant max backedge-taken count is 255 +; CHECK-NEXT: Loop %while.body: constant max backedge-taken count is i32 255 ; CHECK-NEXT: Loop %while.body: symbolic max backedge-taken count is (-1 + (%blockSize /u 4)) ; CHECK-NEXT: Loop %while.body: Predicated backedge-taken count is (-1 + (%blockSize /u 4)) ; CHECK-NEXT: Predicates: @@ -1144,7 +1144,7 @@ define void @test_guard_ugt_and_ult(i32 %blockSize) { ; CHECK-NEXT: --> {(-1 + (%blockSize /u 4)),+,-1}<%while.body> U: [-256,1073741823) S: [-256,1073741823) Exits: 0 LoopDispositions: { %while.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_ugt_and_ult ; CHECK-NEXT: Loop %while.body: backedge-taken count is (-1 + (%blockSize /u 4)) -; CHECK-NEXT: Loop %while.body: constant max backedge-taken count is 255 +; CHECK-NEXT: Loop %while.body: constant max backedge-taken count is i32 255 ; CHECK-NEXT: Loop %while.body: symbolic max backedge-taken count is (-1 + (%blockSize /u 4)) ; CHECK-NEXT: Loop %while.body: Predicated backedge-taken count is (-1 + (%blockSize /u 4)) ; CHECK-NEXT: Predicates: @@ -1187,7 +1187,7 @@ define void @test_guard_slt_sgt_1(ptr nocapture %a, i64 %N) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,12) S: [1,12) Exits: %N LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_slt_sgt_1 ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %N) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 10 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 10 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %N) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %N) ; CHECK-NEXT: Predicates: @@ -1224,7 +1224,7 @@ define void @test_guard_slt_sgt_2(ptr nocapture %a, i64 %i) { ; CHECK-NEXT: --> {(1 + %i),+,1}<%loop> U: full-set S: full-set Exits: 18 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_slt_sgt_2 ; CHECK-NEXT: Loop %loop: backedge-taken count is (17 + (-1 * %i)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 12 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 12 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (17 + (-1 * %i)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (17 + (-1 * %i)) ; CHECK-NEXT: Predicates: @@ -1261,7 +1261,7 @@ define void @test_guard_sle_sge_1(ptr nocapture %a, i64 %N) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,13) S: [1,13) Exits: %N LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_sle_sge_1 ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %N) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 11 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 11 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %N) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %N) ; CHECK-NEXT: Predicates: @@ -1298,7 +1298,7 @@ define void @test_guard_sle_sge_2(ptr nocapture %a, i64 %i) { ; CHECK-NEXT: --> {(1 + %i),+,1}<%loop> U: full-set S: full-set Exits: 18 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_guard_sle_sge_2 ; CHECK-NEXT: Loop %loop: backedge-taken count is (17 + (-1 * %i)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 13 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 13 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (17 + (-1 * %i)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (17 + (-1 * %i)) ; CHECK-NEXT: Predicates: @@ -1338,7 +1338,7 @@ define void @optimized_range_check_unsigned(ptr %pred, i32 %N) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,8) S: [1,8) Exits: %N LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @optimized_range_check_unsigned ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %N) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 6 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 6 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %N) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %N) ; CHECK-NEXT: Predicates: @@ -1375,7 +1375,7 @@ define void @optimized_range_check_unsigned_icmp_ops_swapped(ptr %pred, i32 %N) ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,8) S: [1,8) Exits: %N LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @optimized_range_check_unsigned_icmp_ops_swapped ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %N) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 6 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 6 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %N) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %N) ; CHECK-NEXT: Predicates: @@ -1414,7 +1414,7 @@ define void @optimized_range_check_unsigned2(ptr %pred, i32 %N) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,22) S: [1,22) Exits: %N LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @optimized_range_check_unsigned2 ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %N) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 20 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 20 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %N) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %N) ; CHECK-NEXT: Predicates: @@ -1454,7 +1454,7 @@ define void @optimized_range_check_unsigned3(ptr %pred, i1 %c) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,4) S: [1,4) Exits: %N LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @optimized_range_check_unsigned3 ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %N) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 2 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %N) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %N) ; CHECK-NEXT: Predicates: @@ -1493,7 +1493,7 @@ define void @not_optimized_range_check_unsigned1(ptr %pred, i32 %N) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,-2147483648) S: [1,-2147483648) Exits: %N LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @not_optimized_range_check_unsigned1 ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %N) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %N) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %N) ; CHECK-NEXT: Predicates: @@ -1531,7 +1531,7 @@ define void @not_optimized_range_check_unsigned2(ptr %pred, i32 %N) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,-2147483648) S: [1,-2147483648) Exits: %N LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @not_optimized_range_check_unsigned2 ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %N) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %N) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %N) ; CHECK-NEXT: Predicates: @@ -1563,7 +1563,7 @@ define i32 @sle_sgt_ult_umax_to_smax(i32 %num) { ; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,29) S: [4,29) Exits: (4 + (4 * ((-4 + %num) /u 4))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @sle_sgt_ult_umax_to_smax ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + %num) /u 4) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 6 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 6 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + %num) /u 4) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + %num) /u 4) ; CHECK-NEXT: Predicates: @@ -1601,7 +1601,7 @@ define i32 @ult_sle_sgt_umax_to_smax(i32 %num) { ; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,-3) S: [-2147483648,2147483645) Exits: (4 + (4 * ((-4 + %num) /u 4))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @ult_sle_sgt_umax_to_smax ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + %num) /u 4) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 1073741823 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 1073741823 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + %num) /u 4) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + %num) /u 4) ; CHECK-NEXT: Predicates: @@ -1638,7 +1638,7 @@ define i32 @ptr_induction_ult_1(ptr %a, ptr %b) { ; CHECK-NEXT: --> {(4 + %a),+,4}<%loop> U: full-set S: full-set Exits: (4 + %a) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @ptr_induction_ult_1 ; CHECK-NEXT: Loop %loop: backedge-taken count is 0 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 0 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 0 ; CHECK-NEXT: Predicates: @@ -1699,7 +1699,7 @@ define void @gep_addrec_nw(ptr %a) { ; CHECK-NEXT: --> {(4 + %a),+,4}<%for.body> U: full-set S: full-set Exits: (1516 + %a) LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @gep_addrec_nw ; CHECK-NEXT: Loop %for.body: backedge-taken count is 378 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 378 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 378 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 378 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 378 ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-limit-by-wrapping.ll b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-limit-by-wrapping.ll index 9e0214e12746..45e6337a9b35 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-limit-by-wrapping.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-limit-by-wrapping.ll @@ -8,7 +8,7 @@ define void @max_backedge_taken_count_by_wrapping1_nsw_nuw(i8 %N, ptr %ptr) { ; CHECK-LABEL: 'max_backedge_taken_count_by_wrapping1_nsw_nuw' ; CHECK-NEXT: Determining loop execution counts for: @max_backedge_taken_count_by_wrapping1_nsw_nuw ; CHECK-NEXT: Loop %loop: backedge-taken count is (%N /u 4) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 63 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 63 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%N /u 4) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%N /u 4) ; CHECK-NEXT: Predicates: @@ -33,7 +33,7 @@ define void @max_backedge_taken_count_by_wrapping1_nuw(i8 %N, ptr %ptr) { ; CHECK-LABEL: 'max_backedge_taken_count_by_wrapping1_nuw' ; CHECK-NEXT: Determining loop execution counts for: @max_backedge_taken_count_by_wrapping1_nuw ; CHECK-NEXT: Loop %loop: backedge-taken count is (%N /u 4) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 63 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 63 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%N /u 4) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%N /u 4) ; CHECK-NEXT: Predicates: @@ -58,7 +58,7 @@ define void @max_backedge_taken_count_by_wrapping2_nsw_nuw(i8 %N, ptr %ptr) { ; CHECK-LABEL: 'max_backedge_taken_count_by_wrapping2_nsw_nuw' ; CHECK-NEXT: Determining loop execution counts for: @max_backedge_taken_count_by_wrapping2_nsw_nuw ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-64 + %N) /u 4) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 63 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 63 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-64 + %N) /u 4) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-64 + %N) /u 4) ; CHECK-NEXT: Predicates: @@ -83,7 +83,7 @@ define void @max_backedge_taken_count_by_wrapping2_nuw(i8 %N, ptr %ptr) { ; CHECK-LABEL: 'max_backedge_taken_count_by_wrapping2_nuw' ; CHECK-NEXT: Determining loop execution counts for: @max_backedge_taken_count_by_wrapping2_nuw ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-64 + %N) /u 4) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 63 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 63 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-64 + %N) /u 4) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-64 + %N) /u 4) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/max-be-count-not-constant.ll b/llvm/test/Analysis/ScalarEvolution/max-be-count-not-constant.ll index f91b923e4acd..385864b98887 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-be-count-not-constant.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-be-count-not-constant.ll @@ -21,7 +21,7 @@ define void @pluto(i32 %arg) { ; CHECK-NEXT: --> {(2 + %tmp),+,(2 + %tmp)}<%bb2> U: [1,5) S: [1,5) Exits: (2 + ((2 + %tmp) * (1 /u (2 + %tmp))) + %tmp) LoopDispositions: { %bb2: Computable } ; CHECK-NEXT: Determining loop execution counts for: @pluto ; CHECK-NEXT: Loop %bb2: backedge-taken count is (1 /u (2 + %tmp)) -; CHECK-NEXT: Loop %bb2: constant max backedge-taken count is 1 +; CHECK-NEXT: Loop %bb2: constant max backedge-taken count is i32 1 ; CHECK-NEXT: Loop %bb2: symbolic max backedge-taken count is (1 /u (2 + %tmp)) ; CHECK-NEXT: Loop %bb2: Predicated backedge-taken count is (1 /u (2 + %tmp)) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/max-expr-cache.ll b/llvm/test/Analysis/ScalarEvolution/max-expr-cache.ll index c2d905375069..46e65a36dc65 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-expr-cache.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-expr-cache.ll @@ -88,7 +88,7 @@ define void @smax(i32 %tmp3) { ; CHECK-NEXT: --> {1,+,1}<%bb4> U: [1,0) S: [1,0) Exits: <> LoopDispositions: { %bb4: Computable, %bb53: Invariant } ; CHECK-NEXT: Determining loop execution counts for: @smax ; CHECK-NEXT: Loop %bb53: backedge-taken count is (-1 + (zext i32 (0 smax %tmp49) to i64) + (-1 * undef)) -; CHECK-NEXT: Loop %bb53: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %bb53: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %bb53: symbolic max backedge-taken count is (-1 + (zext i32 (0 smax %tmp49) to i64) + (-1 * undef)) ; CHECK-NEXT: Loop %bb53: Predicated backedge-taken count is (-1 + (zext i32 (0 smax %tmp49) to i64) + (-1 * undef)) ; CHECK-NEXT: Predicates: @@ -250,7 +250,7 @@ define void @umax(i32 %tmp3) { ; CHECK-NEXT: --> {1,+,1}<%bb4> U: [1,0) S: [1,0) Exits: <> LoopDispositions: { %bb4: Computable, %bb53: Invariant } ; CHECK-NEXT: Determining loop execution counts for: @umax ; CHECK-NEXT: Loop %bb53: backedge-taken count is (-1 + (zext i32 %tmp49 to i64) + (-1 * undef)) -; CHECK-NEXT: Loop %bb53: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %bb53: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %bb53: symbolic max backedge-taken count is (-1 + (zext i32 %tmp49 to i64) + (-1 * undef)) ; CHECK-NEXT: Loop %bb53: Predicated backedge-taken count is (-1 + (zext i32 %tmp49 to i64) + (-1 * undef)) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/max-trip-count-address-space.ll b/llvm/test/Analysis/ScalarEvolution/max-trip-count-address-space.ll index 5af706b201de..802b76b4072a 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-trip-count-address-space.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-trip-count-address-space.ll @@ -10,7 +10,7 @@ define void @foo(ptr addrspace(1) nocapture %d, i32 %n) nounwind { ; CHECK-LABEL: 'foo' ; CHECK-NEXT: Determining loop execution counts for: @foo ; CHECK-NEXT: Loop %bb: backedge-taken count is (-1 + %n) -; CHECK-NEXT: Loop %bb: constant max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %bb: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is (-1 + %n) ; CHECK-NEXT: Loop %bb: Predicated backedge-taken count is (-1 + %n) ; CHECK-NEXT: Predicates: @@ -50,7 +50,7 @@ define void @test(ptr addrspace(1) %a, i32 %n) nounwind { ; CHECK-LABEL: 'test' ; CHECK-NEXT: Determining loop execution counts for: @test ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (zext i32 %n to i64)) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (zext i32 %n to i64)) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (zext i32 %n to i64)) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/max-trip-count.ll b/llvm/test/Analysis/ScalarEvolution/max-trip-count.ll index 9cc49f2185f8..7f08bec162a5 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-trip-count.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-trip-count.ll @@ -8,7 +8,7 @@ define void @foo(ptr nocapture %d, i32 %n) nounwind { ; CHECK-LABEL: 'foo' ; CHECK-NEXT: Determining loop execution counts for: @foo ; CHECK-NEXT: Loop %bb: backedge-taken count is (-1 + %n) -; CHECK-NEXT: Loop %bb: constant max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %bb: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is (-1 + %n) ; CHECK-NEXT: Loop %bb: Predicated backedge-taken count is (-1 + %n) ; CHECK-NEXT: Predicates: @@ -57,7 +57,7 @@ define i32 @main() nounwind { ; CHECK-NEXT: Loop %for.cond: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for for.cond: 5 ; CHECK-NEXT: exit count for for.body: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %for.cond: constant max backedge-taken count is 5 +; CHECK-NEXT: Loop %for.cond: constant max backedge-taken count is i32 5 ; CHECK-NEXT: Loop %for.cond: symbolic max backedge-taken count is 5 ; CHECK-NEXT: symbolic max exit count for for.cond: 5 ; CHECK-NEXT: symbolic max exit count for for.body: ***COULDNOTCOMPUTE*** @@ -93,7 +93,7 @@ define void @test(ptr %a, i32 %n) nounwind { ; CHECK-LABEL: 'test' ; CHECK-NEXT: Determining loop execution counts for: @test ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (zext i32 %n to i64)) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (zext i32 %n to i64)) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (zext i32 %n to i64)) ; CHECK-NEXT: Predicates: @@ -132,7 +132,7 @@ define i32 @pr19799() { ; CHECK-NEXT: Loop %for.body.i: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for for.body.i: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for for.cond.i: 1 -; CHECK-NEXT: Loop %for.body.i: constant max backedge-taken count is 1 +; CHECK-NEXT: Loop %for.body.i: constant max backedge-taken count is i32 1 ; CHECK-NEXT: Loop %for.body.i: symbolic max backedge-taken count is 1 ; CHECK-NEXT: symbolic max exit count for for.body.i: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for for.cond.i: 1 @@ -166,7 +166,7 @@ define i32 @pr18886() { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for for.body: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for for.cond: 3 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 3 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 3 ; CHECK-NEXT: symbolic max exit count for for.body: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for for.cond: 3 @@ -239,7 +239,7 @@ define i32 @two_mustexit() { ; CHECK-NEXT: Loop %for.body.i: backedge-taken count is 1 ; CHECK-NEXT: exit count for for.body.i: 1 ; CHECK-NEXT: exit count for for.cond.i: 2 -; CHECK-NEXT: Loop %for.body.i: constant max backedge-taken count is 1 +; CHECK-NEXT: Loop %for.body.i: constant max backedge-taken count is i32 1 ; CHECK-NEXT: Loop %for.body.i: symbolic max backedge-taken count is 1 ; CHECK-NEXT: symbolic max exit count for for.body.i: 1 ; CHECK-NEXT: symbolic max exit count for for.cond.i: 2 @@ -270,7 +270,7 @@ define i32 @ne_max_trip_count_1(i32 %n) { ; CHECK-LABEL: 'ne_max_trip_count_1' ; CHECK-NEXT: Determining loop execution counts for: @ne_max_trip_count_1 ; CHECK-NEXT: Loop %for.body: backedge-taken count is (zext i3 (trunc i32 %n to i3) to i32) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 7 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 7 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (zext i3 (trunc i32 %n to i3) to i32) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (zext i3 (trunc i32 %n to i3) to i32) ; CHECK-NEXT: Predicates: @@ -294,7 +294,7 @@ define i32 @ne_max_trip_count_2(i32 %n) { ; CHECK-LABEL: 'ne_max_trip_count_2' ; CHECK-NEXT: Determining loop execution counts for: @ne_max_trip_count_2 ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (zext i3 (trunc i32 %n to i3) to i32)) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (zext i3 (trunc i32 %n to i3) to i32)) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (zext i3 (trunc i32 %n to i3) to i32)) ; CHECK-NEXT: Predicates: @@ -318,7 +318,7 @@ define i32 @ne_max_trip_count_3(i32 %n) { ; CHECK-LABEL: 'ne_max_trip_count_3' ; CHECK-NEXT: Determining loop execution counts for: @ne_max_trip_count_3 ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (zext i3 (trunc i32 %n to i3) to i32)) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 6 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 6 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (zext i3 (trunc i32 %n to i3) to i32)) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (zext i3 (trunc i32 %n to i3) to i32)) ; CHECK-NEXT: Predicates: @@ -349,7 +349,7 @@ define i32 @ne_max_trip_count_4(i32 %n) { ; CHECK-LABEL: 'ne_max_trip_count_4' ; CHECK-NEXT: Determining loop execution counts for: @ne_max_trip_count_4 ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %n) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %n) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %n) ; CHECK-NEXT: Predicates: @@ -381,7 +381,7 @@ define void @changing_end_bound(ptr %n_addr, ptr %addr) { ; CHECK-LABEL: 'changing_end_bound' ; CHECK-NEXT: Determining loop execution counts for: @changing_end_bound ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 2147483646 ; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; @@ -410,7 +410,7 @@ define void @changing_end_bound2(i32 %start, ptr %n_addr, ptr %addr) { ; CHECK-LABEL: 'changing_end_bound2' ; CHECK-NEXT: Determining loop execution counts for: @changing_end_bound2 ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is -1 ; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; @@ -437,7 +437,7 @@ define void @changing_end_bound3(i32 %start, ptr %n_addr, ptr %addr) { ; CHECK-LABEL: 'changing_end_bound3' ; CHECK-NEXT: Determining loop execution counts for: @changing_end_bound3 ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 1073741823 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 1073741823 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 1073741823 ; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; @@ -573,7 +573,7 @@ define void @max_overflow_se(i8 %n) mustprogress { ; CHECK-LABEL: 'max_overflow_se' ; CHECK-NEXT: Determining loop execution counts for: @max_overflow_se ; CHECK-NEXT: Loop %loop: backedge-taken count is 0 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 0 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 0 ; CHECK-NEXT: Predicates: @@ -600,7 +600,7 @@ define void @max_overflow_me(i8 %n) mustprogress { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for loop: 1 ; CHECK-NEXT: exit count for latch: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 1 ; CHECK-NEXT: symbolic max exit count for loop: 1 ; CHECK-NEXT: symbolic max exit count for latch: ***COULDNOTCOMPUTE*** @@ -630,7 +630,7 @@ define void @bool_stride(i1 %s, i1 %n) mustprogress { ; CHECK-LABEL: 'bool_stride' ; CHECK-NEXT: Determining loop execution counts for: @bool_stride ; CHECK-NEXT: Loop %loop: backedge-taken count is false -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is false +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is false ; CHECK-NEXT: Predicates: @@ -656,7 +656,7 @@ define void @ne_zero_max_btc(i32 %a) { ; CHECK-LABEL: 'ne_zero_max_btc' ; CHECK-NEXT: Determining loop execution counts for: @ne_zero_max_btc ; CHECK-NEXT: Loop %for.body: backedge-taken count is 0 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 0 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 0 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 0 ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/min-max-exprs.ll b/llvm/test/Analysis/ScalarEvolution/min-max-exprs.ll index bc0c05aafc6b..8616c98ef1ed 100644 --- a/llvm/test/Analysis/ScalarEvolution/min-max-exprs.ll +++ b/llvm/test/Analysis/ScalarEvolution/min-max-exprs.ll @@ -47,7 +47,7 @@ define void @f(ptr %A, i32 %N) { ; CHECK-NEXT: --> {1,+,1}<%bb1> U: [1,-2147483647) S: [1,-2147483647) Exits: (1 + (0 smax %N)) LoopDispositions: { %bb1: Computable } ; CHECK-NEXT: Determining loop execution counts for: @f ; CHECK-NEXT: Loop %bb1: backedge-taken count is (0 smax %N) -; CHECK-NEXT: Loop %bb1: constant max backedge-taken count is 2147483647 +; CHECK-NEXT: Loop %bb1: constant max backedge-taken count is i32 2147483647 ; CHECK-NEXT: Loop %bb1: symbolic max backedge-taken count is (0 smax %N) ; CHECK-NEXT: Loop %bb1: Predicated backedge-taken count is (0 smax %N) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/ne-overflow.ll b/llvm/test/Analysis/ScalarEvolution/ne-overflow.ll index 6ae241c85f65..9f508f6ddafe 100644 --- a/llvm/test/Analysis/ScalarEvolution/ne-overflow.ll +++ b/llvm/test/Analysis/ScalarEvolution/ne-overflow.ll @@ -11,7 +11,7 @@ define void @test(i32 %N) mustprogress { ; CHECK-LABEL: 'test' ; CHECK-NEXT: Determining loop execution counts for: @test ; CHECK-NEXT: Loop %for.body: backedge-taken count is ((-2 + %N) /u 2) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 2147483647 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483647 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((-2 + %N) /u 2) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-2 + %N) /u 2) ; CHECK-NEXT: Predicates: @@ -34,7 +34,7 @@ define void @test_preinc(i32 %N) mustprogress { ; CHECK-LABEL: 'test_preinc' ; CHECK-NEXT: Determining loop execution counts for: @test_preinc ; CHECK-NEXT: Loop %for.body: backedge-taken count is (%N /u 2) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 2147483647 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483647 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (%N /u 2) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (%N /u 2) ; CHECK-NEXT: Predicates: @@ -127,7 +127,7 @@ define void @test_1024(i32 %N) mustprogress { ; CHECK-LABEL: 'test_1024' ; CHECK-NEXT: Determining loop execution counts for: @test_1024 ; CHECK-NEXT: Loop %for.body: backedge-taken count is ((-1024 + %N) /u 1024) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 4194303 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 4194303 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((-1024 + %N) /u 1024) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-1024 + %N) /u 1024) ; CHECK-NEXT: Predicates: @@ -150,7 +150,7 @@ define void @test_uneven_divide(i32 %N) mustprogress { ; CHECK-LABEL: 'test_uneven_divide' ; CHECK-NEXT: Determining loop execution counts for: @test_uneven_divide ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (-1431655765 * %N)) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (-1431655765 * %N)) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (-1431655765 * %N)) ; CHECK-NEXT: Predicates: @@ -222,7 +222,7 @@ define void @test_other_exit(i32 %N) mustprogress { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for for.body: 9 ; CHECK-NEXT: exit count for for.latch: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 9 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 9 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 9 ; CHECK-NEXT: symbolic max exit count for for.body: 9 ; CHECK-NEXT: symbolic max exit count for for.latch: ***COULDNOTCOMPUTE*** diff --git a/llvm/test/Analysis/ScalarEvolution/no-wrap-symbolic-becount.ll b/llvm/test/Analysis/ScalarEvolution/no-wrap-symbolic-becount.ll index 1831d81cc99f..eec7f16148eb 100644 --- a/llvm/test/Analysis/ScalarEvolution/no-wrap-symbolic-becount.ll +++ b/llvm/test/Analysis/ScalarEvolution/no-wrap-symbolic-becount.ll @@ -26,7 +26,7 @@ define i32 @test_01(i32 %start, ptr %p, ptr %q) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for loop: (zext i32 %start to i64) ; CHECK-NEXT: exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4294967295 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4294967295 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i32 %start to i64) ; CHECK-NEXT: symbolic max exit count for loop: (zext i32 %start to i64) ; CHECK-NEXT: symbolic max exit count for backedge: ***COULDNOTCOMPUTE*** @@ -74,7 +74,7 @@ define i32 @test_02(i32 %start, ptr %p, ptr %q) { ; CHECK-NEXT: --> {(-1 + (2147483648 * (zext i32 %start to i64))),+,-1}<%loop> U: full-set S: full-set Exits: -9223372036854775807 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_02 ; CHECK-NEXT: Loop %loop: backedge-taken count is (9223372036854775806 + (2147483648 * (zext i32 %start to i64))) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -2147483650 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -2147483650 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (9223372036854775806 + (2147483648 * (zext i32 %start to i64))) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (9223372036854775806 + (2147483648 * (zext i32 %start to i64))) ; CHECK-NEXT: Predicates: @@ -135,7 +135,7 @@ define void @pointer_iv_nowrap_guard(ptr %startptr, ptr %endptr) local_unnamed_a ; CHECK-NEXT: --> {(8004 + %startptr),+,4}<%loop> U: full-set S: full-set Exits: (8004 + (4 * ((-8001 + (-1 * (ptrtoint ptr %startptr to i64)) + ((8004 + (ptrtoint ptr %startptr to i64)) umax (ptrtoint ptr %endptr to i64))) /u 4)) + %startptr) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @pointer_iv_nowrap_guard ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-8001 + (-1 * (ptrtoint ptr %startptr to i64)) + ((8004 + (ptrtoint ptr %startptr to i64)) umax (ptrtoint ptr %endptr to i64))) /u 4) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4611686018427387903 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4611686018427387903 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-8001 + (-1 * (ptrtoint ptr %startptr to i64)) + ((8004 + (ptrtoint ptr %startptr to i64)) umax (ptrtoint ptr %endptr to i64))) /u 4) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-8001 + (-1 * (ptrtoint ptr %startptr to i64)) + ((8004 + (ptrtoint ptr %startptr to i64)) umax (ptrtoint ptr %endptr to i64))) /u 4) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/nsw-offset-assume.ll b/llvm/test/Analysis/ScalarEvolution/nsw-offset-assume.ll index 60809fcf31de..e8462fc7459e 100644 --- a/llvm/test/Analysis/ScalarEvolution/nsw-offset-assume.ll +++ b/llvm/test/Analysis/ScalarEvolution/nsw-offset-assume.ll @@ -45,7 +45,7 @@ define void @foo(i32 %no, ptr nocapture %d, ptr nocapture %q) nounwind { ; CHECK-NEXT: --> {2,+,2}<%bb> U: [2,2147483647) S: [2,2147483647) Exits: (2 + (2 * ((-1 + (2 * (%no /u 2))) /u 2))) LoopDispositions: { %bb: Computable } ; CHECK-NEXT: Determining loop execution counts for: @foo ; CHECK-NEXT: Loop %bb: backedge-taken count is ((-1 + (2 * (%no /u 2))) /u 2) -; CHECK-NEXT: Loop %bb: constant max backedge-taken count is 1073741822 +; CHECK-NEXT: Loop %bb: constant max backedge-taken count is i32 1073741822 ; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is ((-1 + (2 * (%no /u 2))) /u 2) ; CHECK-NEXT: Loop %bb: Predicated backedge-taken count is ((-1 + (2 * (%no /u 2))) /u 2) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/nsw-offset.ll b/llvm/test/Analysis/ScalarEvolution/nsw-offset.ll index fab408ea372e..f27018ffe17c 100644 --- a/llvm/test/Analysis/ScalarEvolution/nsw-offset.ll +++ b/llvm/test/Analysis/ScalarEvolution/nsw-offset.ll @@ -42,7 +42,7 @@ define void @foo(i32 %no, ptr nocapture %d, ptr nocapture %q) nounwind { ; CHECK-NEXT: --> {2,+,2}<%bb> U: [2,2147483647) S: [2,2147483647) Exits: (2 + (2 * ((-1 + (2 * (%no /u 2))) /u 2))) LoopDispositions: { %bb: Computable } ; CHECK-NEXT: Determining loop execution counts for: @foo ; CHECK-NEXT: Loop %bb: backedge-taken count is ((-1 + (2 * (%no /u 2))) /u 2) -; CHECK-NEXT: Loop %bb: constant max backedge-taken count is 1073741822 +; CHECK-NEXT: Loop %bb: constant max backedge-taken count is i32 1073741822 ; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is ((-1 + (2 * (%no /u 2))) /u 2) ; CHECK-NEXT: Loop %bb: Predicated backedge-taken count is ((-1 + (2 * (%no /u 2))) /u 2) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/nsw.ll b/llvm/test/Analysis/ScalarEvolution/nsw.ll index 5c4c9fcd8a0d..0240902a18fb 100644 --- a/llvm/test/Analysis/ScalarEvolution/nsw.ll +++ b/llvm/test/Analysis/ScalarEvolution/nsw.ll @@ -77,7 +77,7 @@ define void @test2(ptr %begin, ptr %end) ssp { ; CHECK-NEXT: --> {(4 + %begin),+,4}<%for.body.i.i> U: full-set S: full-set Exits: (4 + (4 * ((-4 + (-1 * (ptrtoint ptr %begin to i64)) + (ptrtoint ptr %end to i64)) /u 4)) + %begin) LoopDispositions: { %for.body.i.i: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test2 ; CHECK-NEXT: Loop %for.body.i.i: backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %begin to i64)) + (ptrtoint ptr %end to i64)) /u 4) -; CHECK-NEXT: Loop %for.body.i.i: constant max backedge-taken count is 4611686018427387903 +; CHECK-NEXT: Loop %for.body.i.i: constant max backedge-taken count is i64 4611686018427387903 ; CHECK-NEXT: Loop %for.body.i.i: symbolic max backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %begin to i64)) + (ptrtoint ptr %end to i64)) /u 4) ; CHECK-NEXT: Loop %for.body.i.i: Predicated backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %begin to i64)) + (ptrtoint ptr %end to i64)) /u 4) ; CHECK-NEXT: Predicates: @@ -118,7 +118,7 @@ define void @test3(ptr %begin, ptr %end) nounwind ssp { ; CHECK-NEXT: --> {%begin,+,4}<%for.body.i.i> U: full-set S: full-set Exits: ((4 * ((-4 + (-1 * (ptrtoint ptr %begin to i64)) + (ptrtoint ptr %end to i64)) /u 4)) + %begin) LoopDispositions: { %for.body.i.i: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test3 ; CHECK-NEXT: Loop %for.body.i.i: backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %begin to i64)) + (ptrtoint ptr %end to i64)) /u 4) -; CHECK-NEXT: Loop %for.body.i.i: constant max backedge-taken count is 4611686018427387903 +; CHECK-NEXT: Loop %for.body.i.i: constant max backedge-taken count is i64 4611686018427387903 ; CHECK-NEXT: Loop %for.body.i.i: symbolic max backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %begin to i64)) + (ptrtoint ptr %end to i64)) /u 4) ; CHECK-NEXT: Loop %for.body.i.i: Predicated backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %begin to i64)) + (ptrtoint ptr %end to i64)) /u 4) ; CHECK-NEXT: Predicates: @@ -181,7 +181,7 @@ define i32 @PR12375(ptr readnone %arg) { ; CHECK-NEXT: --> {(4 + %arg),+,4}<%bb1> U: [4,0) S: [4,0) Exits: (8 + %arg) LoopDispositions: { %bb1: Computable } ; CHECK-NEXT: Determining loop execution counts for: @PR12375 ; CHECK-NEXT: Loop %bb1: backedge-taken count is 1 -; CHECK-NEXT: Loop %bb1: constant max backedge-taken count is 1 +; CHECK-NEXT: Loop %bb1: constant max backedge-taken count is i64 1 ; CHECK-NEXT: Loop %bb1: symbolic max backedge-taken count is 1 ; CHECK-NEXT: Loop %bb1: Predicated backedge-taken count is 1 ; CHECK-NEXT: Predicates: @@ -212,7 +212,7 @@ define void @PR12376(ptr nocapture %arg, ptr nocapture %arg1) { ; CHECK-NEXT: --> {(4 + %arg),+,4}<%bb2> U: [4,0) S: [4,0) Exits: (4 + (4 * ((-1 + (-1 * (ptrtoint ptr %arg to i64)) + ((4 + (ptrtoint ptr %arg to i64)) umax (ptrtoint ptr %arg1 to i64))) /u 4)) + %arg) LoopDispositions: { %bb2: Computable } ; CHECK-NEXT: Determining loop execution counts for: @PR12376 ; CHECK-NEXT: Loop %bb2: backedge-taken count is ((-1 + (-1 * (ptrtoint ptr %arg to i64)) + ((4 + (ptrtoint ptr %arg to i64)) umax (ptrtoint ptr %arg1 to i64))) /u 4) -; CHECK-NEXT: Loop %bb2: constant max backedge-taken count is 4611686018427387902 +; CHECK-NEXT: Loop %bb2: constant max backedge-taken count is i64 4611686018427387902 ; CHECK-NEXT: Loop %bb2: symbolic max backedge-taken count is ((-1 + (-1 * (ptrtoint ptr %arg to i64)) + ((4 + (ptrtoint ptr %arg to i64)) umax (ptrtoint ptr %arg1 to i64))) /u 4) ; CHECK-NEXT: Loop %bb2: Predicated backedge-taken count is ((-1 + (-1 * (ptrtoint ptr %arg to i64)) + ((4 + (ptrtoint ptr %arg to i64)) umax (ptrtoint ptr %arg1 to i64))) /u 4) ; CHECK-NEXT: Predicates: @@ -248,7 +248,7 @@ define void @nswnowrap(i32 %v, ptr %buf) { ; CHECK-NEXT: --> %buf.val U: full-set S: full-set Exits: <> LoopDispositions: { %for.body: Variant } ; CHECK-NEXT: Determining loop execution counts for: @nswnowrap ; CHECK-NEXT: Loop %for.body: backedge-taken count is ((-1 * %v) + ((1 + %v) smax %v)) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 1, actual taken count either this or zero. +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 1, actual taken count either this or zero. ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((-1 * %v) + ((1 + %v) smax %v)), actual taken count either this or zero. ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-1 * %v) + ((1 + %v) smax %v)) ; CHECK-NEXT: Predicates: @@ -291,7 +291,7 @@ define void @test4(i32 %arg) { ; CHECK-NEXT: --> {(1 + %arg),+,1}<%for.body> U: [-2147483647,-2147483648) S: [-2147483647,-2147483648) Exits: (10 smax (1 + %arg)) LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test4 ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (-1 * %arg) + (10 smax (1 + %arg))) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -2147483639 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2147483639 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (-1 * %arg) + (10 smax (1 + %arg))) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (-1 * %arg) + (10 smax (1 + %arg))) ; CHECK-NEXT: Predicates: @@ -325,7 +325,7 @@ define void @bad_postinc_nsw_a(i32 %n) { ; CHECK-NEXT: --> {7,+,7}<%loop> U: [7,-3) S: [7,0) Exits: (7 + (7 * ((((-1 * (1 umin %n)) + %n) /u 7) + (1 umin %n)))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @bad_postinc_nsw_a ; CHECK-NEXT: Loop %loop: backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 7) + (1 umin %n)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 613566756 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 613566756 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 7) + (1 umin %n)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 7) + (1 umin %n)) ; CHECK-NEXT: Predicates: @@ -386,7 +386,7 @@ define void @pr28012(i32 %n) { ; CHECK-NEXT: --> {7,+,7}<%loop> U: [7,-3) S: [7,-3) Exits: (7 + (7 * ((-1 + (7 umax %n)) /u 7))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @pr28012 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 + (7 umax %n)) /u 7) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 613566755 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 613566755 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 + (7 umax %n)) /u 7) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 + (7 umax %n)) /u 7) ; CHECK-NEXT: Predicates: @@ -451,7 +451,7 @@ define void @pr66066() { ; CHECK-NEXT: --> {-128,+,-128}<%loop> U: [0,-127) S: [-128,1) Exits: 0 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @pr66066 ; CHECK-NEXT: Loop %loop: backedge-taken count is 1 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 1 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 1 ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/outer_phi.ll b/llvm/test/Analysis/ScalarEvolution/outer_phi.ll index 4e2249201e89..c1faa5a8ac59 100644 --- a/llvm/test/Analysis/ScalarEvolution/outer_phi.ll +++ b/llvm/test/Analysis/ScalarEvolution/outer_phi.ll @@ -20,7 +20,7 @@ define i32 @test_01(i32 %a, i32 %b) { ; CHECK-NEXT: Loop %inner: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for inner: %b ; CHECK-NEXT: exit count for inner.backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %inner: constant max backedge-taken count is 2147483647 +; CHECK-NEXT: Loop %inner: constant max backedge-taken count is i32 2147483647 ; CHECK-NEXT: Loop %inner: symbolic max backedge-taken count is %b ; CHECK-NEXT: symbolic max exit count for inner: %b ; CHECK-NEXT: symbolic max exit count for inner.backedge: ***COULDNOTCOMPUTE*** @@ -84,7 +84,7 @@ define i32 @test_02(i32 %a, i32 %b) { ; CHECK-NEXT: Loop %inner: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for inner: ((-1 * %outer.iv) + (%b smax %outer.iv)) ; CHECK-NEXT: exit count for inner.backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %inner: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %inner: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %inner: symbolic max backedge-taken count is ((-1 * %outer.iv) + (%b smax %outer.iv)) ; CHECK-NEXT: symbolic max exit count for inner: ((-1 * %outer.iv) + (%b smax %outer.iv)) ; CHECK-NEXT: symbolic max exit count for inner.backedge: ***COULDNOTCOMPUTE*** diff --git a/llvm/test/Analysis/ScalarEvolution/overflow-intrinsics-trip-count.ll b/llvm/test/Analysis/ScalarEvolution/overflow-intrinsics-trip-count.ll index 769db5ae0d5b..821572af55c2 100644 --- a/llvm/test/Analysis/ScalarEvolution/overflow-intrinsics-trip-count.ll +++ b/llvm/test/Analysis/ScalarEvolution/overflow-intrinsics-trip-count.ll @@ -12,7 +12,7 @@ define void @uadd_exhaustive() { ; CHECK-LABEL: 'uadd_exhaustive' ; CHECK-NEXT: Determining loop execution counts for: @uadd_exhaustive ; CHECK-NEXT: Loop %for.body: backedge-taken count is 35 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 35 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 35 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 35 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 35 ; CHECK-NEXT: Predicates: @@ -39,7 +39,7 @@ define void @sadd_exhaustive() { ; CHECK-LABEL: 'sadd_exhaustive' ; CHECK-NEXT: Determining loop execution counts for: @sadd_exhaustive ; CHECK-NEXT: Loop %for.body: backedge-taken count is 67 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 67 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 67 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 67 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 67 ; CHECK-NEXT: Predicates: @@ -66,7 +66,7 @@ define void @usub_exhaustive() { ; CHECK-LABEL: 'usub_exhaustive' ; CHECK-NEXT: Determining loop execution counts for: @usub_exhaustive ; CHECK-NEXT: Loop %for.body: backedge-taken count is 50 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 50 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 50 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 50 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 50 ; CHECK-NEXT: Predicates: @@ -93,7 +93,7 @@ define void @ssub_exhaustive() { ; CHECK-LABEL: 'ssub_exhaustive' ; CHECK-NEXT: Determining loop execution counts for: @ssub_exhaustive ; CHECK-NEXT: Loop %for.body: backedge-taken count is 68 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 68 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 68 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 68 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 68 ; CHECK-NEXT: Predicates: @@ -120,7 +120,7 @@ define void @smul_exhaustive() { ; CHECK-LABEL: 'smul_exhaustive' ; CHECK-NEXT: Determining loop execution counts for: @smul_exhaustive ; CHECK-NEXT: Loop %for.body: backedge-taken count is 14 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 14 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 14 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 14 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 14 ; CHECK-NEXT: Predicates: @@ -147,7 +147,7 @@ define void @umul_exhaustive() { ; CHECK-LABEL: 'umul_exhaustive' ; CHECK-NEXT: Determining loop execution counts for: @umul_exhaustive ; CHECK-NEXT: Loop %for.body: backedge-taken count is 15 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 15 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 15 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 15 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 15 ; CHECK-NEXT: Predicates: @@ -174,7 +174,7 @@ define void @uadd_symbolic_start(i16 %start) { ; CHECK-LABEL: 'uadd_symbolic_start' ; CHECK-NEXT: Determining loop execution counts for: @uadd_symbolic_start ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (-1 * %start)) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (-1 * %start)) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (-1 * %start)) ; CHECK-NEXT: Predicates: @@ -201,7 +201,7 @@ define void @sadd_symbolic_start(i16 %start) { ; CHECK-LABEL: 'sadd_symbolic_start' ; CHECK-NEXT: Determining loop execution counts for: @sadd_symbolic_start ; CHECK-NEXT: Loop %for.body: backedge-taken count is (32767 + (-1 * %start)) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (32767 + (-1 * %start)) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (32767 + (-1 * %start)) ; CHECK-NEXT: Predicates: @@ -279,7 +279,7 @@ define void @usub_symbolic_start(i16 %start) { ; CHECK-LABEL: 'usub_symbolic_start' ; CHECK-NEXT: Determining loop execution counts for: @usub_symbolic_start ; CHECK-NEXT: Loop %for.body: backedge-taken count is %start -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is %start ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is %start ; CHECK-NEXT: Predicates: @@ -306,7 +306,7 @@ define void @ssub_symbolic_start(i16 %start) { ; CHECK-LABEL: 'ssub_symbolic_start' ; CHECK-NEXT: Determining loop execution counts for: @ssub_symbolic_start ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-32768 + %start) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-32768 + %start) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-32768 + %start) ; CHECK-NEXT: Predicates: @@ -385,7 +385,7 @@ define void @sadd_symbolic_non_latch(i16 %start) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is ((230 + (-1 * %start)) umin (32767 + (-1 * %start))) ; CHECK-NEXT: exit count for for.body: (32767 + (-1 * %start)) ; CHECK-NEXT: exit count for for.latch: (230 + (-1 * %start)) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((230 + (-1 * %start)) umin (32767 + (-1 * %start))) ; CHECK-NEXT: symbolic max exit count for for.body: (32767 + (-1 * %start)) ; CHECK-NEXT: symbolic max exit count for for.latch: (230 + (-1 * %start)) diff --git a/llvm/test/Analysis/ScalarEvolution/overflow-intrinsics.ll b/llvm/test/Analysis/ScalarEvolution/overflow-intrinsics.ll index cd740531f833..9985ddd6803d 100644 --- a/llvm/test/Analysis/ScalarEvolution/overflow-intrinsics.ll +++ b/llvm/test/Analysis/ScalarEvolution/overflow-intrinsics.ll @@ -31,7 +31,7 @@ cont: ; preds = %for.body %tmp2 = extractvalue { i32, i1 } %tmp0, 0 %cmp = icmp slt i32 %tmp2, 16 br i1 %cmp, label %for.body, label %for.cond.cleanup -; CHECK: Loop %for.body: constant max backedge-taken count is 15 +; CHECK: Loop %for.body: constant max backedge-taken count is i32 15 } define void @f_sadd_1(ptr %a) { @@ -66,7 +66,7 @@ cont: ; preds = %for.body %tmp2 = extractvalue { i32, i1 } %tmp0, 0 %cmp = icmp slt i32 %tmp2, 16 br i1 %cmp, label %for.body, label %for.cond.cleanup -; CHECK: Loop %for.body: constant max backedge-taken count is 15 +; CHECK: Loop %for.body: constant max backedge-taken count is i32 15 } define void @f_sadd_2(ptr %a, ptr %c) { @@ -216,7 +216,7 @@ cont: ; preds = %for.body %tmp2 = extractvalue { i32, i1 } %tmp0, 0 %cmp = icmp slt i32 %tmp2, 16 br i1 %cmp, label %for.body, label %for.cond.cleanup -; CHECK: Loop %for.body: constant max backedge-taken count is 15 +; CHECK: Loop %for.body: constant max backedge-taken count is i32 15 } define void @f_ssub(ptr nocapture %a) { @@ -247,7 +247,7 @@ cont: ; preds = %for.body %tmp2 = extractvalue { i32, i1 } %tmp0, 0 %cmp = icmp sgt i32 %tmp2, -1 br i1 %cmp, label %for.body, label %for.cond.cleanup -; CHECK: Loop %for.body: constant max backedge-taken count is 15 +; CHECK: Loop %for.body: constant max backedge-taken count is i32 15 } define void @f_usub(ptr nocapture %a) { @@ -278,7 +278,7 @@ cont: ; preds = %for.body %tmp2 = extractvalue { i32, i1 } %tmp0, 0 %cmp = icmp sgt i32 %tmp2, -1 br i1 %cmp, label %for.body, label %for.cond.cleanup -; CHECK: Loop %for.body: constant max backedge-taken count is 15 +; CHECK: Loop %for.body: constant max backedge-taken count is i32 15 } define i32 @f_smul(i32 %val_a, i32 %val_b) { diff --git a/llvm/test/Analysis/ScalarEvolution/pr25369.ll b/llvm/test/Analysis/ScalarEvolution/pr25369.ll index 28ab5f8d9710..6b5fc6d0a596 100644 --- a/llvm/test/Analysis/ScalarEvolution/pr25369.ll +++ b/llvm/test/Analysis/ScalarEvolution/pr25369.ll @@ -9,13 +9,13 @@ define void @hoge1() { ; CHECK-LABEL: 'hoge1' ; CHECK-NEXT: Determining loop execution counts for: @hoge1 ; CHECK-NEXT: Loop %bb13: backedge-taken count is (-2 + (2 * undef) + %tmp7 + %tmp6) -; CHECK-NEXT: Loop %bb13: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %bb13: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %bb13: symbolic max backedge-taken count is (-2 + (2 * undef) + %tmp7 + %tmp6) ; CHECK-NEXT: Loop %bb13: Predicated backedge-taken count is (-2 + (2 * undef) + %tmp7 + %tmp6) ; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %bb13: Trip multiple is 1 ; CHECK-NEXT: Loop %bb4: backedge-taken count is 20 -; CHECK-NEXT: Loop %bb4: constant max backedge-taken count is 20 +; CHECK-NEXT: Loop %bb4: constant max backedge-taken count is i64 20 ; CHECK-NEXT: Loop %bb4: symbolic max backedge-taken count is 20 ; CHECK-NEXT: Loop %bb4: Predicated backedge-taken count is 20 ; CHECK-NEXT: Predicates: @@ -63,7 +63,7 @@ define void @hoge2() { ; CHECK-LABEL: 'hoge2' ; CHECK-NEXT: Determining loop execution counts for: @hoge2 ; CHECK-NEXT: Loop %bb13: backedge-taken count is (-2 + (2 * undef) + %tmp7 + %tmp6) -; CHECK-NEXT: Loop %bb13: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %bb13: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %bb13: symbolic max backedge-taken count is (-2 + (2 * undef) + %tmp7 + %tmp6) ; CHECK-NEXT: Loop %bb13: Predicated backedge-taken count is (-2 + (2 * undef) + %tmp7 + %tmp6) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/pr34538.ll b/llvm/test/Analysis/ScalarEvolution/pr34538.ll index 762bbde2bf75..9386c0eb5457 100644 --- a/llvm/test/Analysis/ScalarEvolution/pr34538.ll +++ b/llvm/test/Analysis/ScalarEvolution/pr34538.ll @@ -5,7 +5,7 @@ define i32 @pr34538() local_unnamed_addr #0 { ; CHECK-LABEL: 'pr34538' ; CHECK-NEXT: Determining loop execution counts for: @pr34538 ; CHECK-NEXT: Loop %do.body: backedge-taken count is 10000 -; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is 10000 +; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is i32 10000 ; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is 10000 ; CHECK-NEXT: Loop %do.body: Predicated backedge-taken count is 10000 ; CHECK-NEXT: Predicates: @@ -30,7 +30,7 @@ define i32 @foo() { ; CHECK-LABEL: 'foo' ; CHECK-NEXT: Determining loop execution counts for: @foo ; CHECK-NEXT: Loop %do.body: backedge-taken count is 5000 -; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is 5000 +; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is i32 5000 ; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is 5000 ; CHECK-NEXT: Loop %do.body: Predicated backedge-taken count is 5000 ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/pr48225.ll b/llvm/test/Analysis/ScalarEvolution/pr48225.ll index e0eccaa8c040..ca4b1bd42bae 100644 --- a/llvm/test/Analysis/ScalarEvolution/pr48225.ll +++ b/llvm/test/Analysis/ScalarEvolution/pr48225.ll @@ -24,7 +24,7 @@ define void @test_and(i1 %boolcond) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for loop: 2 ; CHECK-NEXT: exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 2 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 2 ; CHECK-NEXT: symbolic max exit count for loop: 2 ; CHECK-NEXT: symbolic max exit count for backedge: ***COULDNOTCOMPUTE*** @@ -74,7 +74,7 @@ define void @test_or(i1 %boolcond) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for loop: 2 ; CHECK-NEXT: exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 2 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 2 ; CHECK-NEXT: symbolic max exit count for loop: 2 ; CHECK-NEXT: symbolic max exit count for backedge: ***COULDNOTCOMPUTE*** diff --git a/llvm/test/Analysis/ScalarEvolution/ptrtoint-constantexpr-loop.ll b/llvm/test/Analysis/ScalarEvolution/ptrtoint-constantexpr-loop.ll index 1af1cc67a702..20037b58bd20 100644 --- a/llvm/test/Analysis/ScalarEvolution/ptrtoint-constantexpr-loop.ll +++ b/llvm/test/Analysis/ScalarEvolution/ptrtoint-constantexpr-loop.ll @@ -349,7 +349,7 @@ define i64 @sext_like_noop(i32 %n) { ; PTR64_IDX64-NEXT: --> {2,+,1}<%for.body> U: [2,0) S: [2,0) Exits: (trunc i64 (ptrtoint ptr @sext_like_noop to i64) to i32) LoopDispositions: { %for.body: Computable } ; PTR64_IDX64-NEXT: Determining loop execution counts for: @sext_like_noop ; PTR64_IDX64-NEXT: Loop %for.body: backedge-taken count is (-2 + (trunc i64 (ptrtoint ptr @sext_like_noop to i64) to i32)) -; PTR64_IDX64-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; PTR64_IDX64-NEXT: Loop %for.body: constant max backedge-taken count is i32 -1 ; PTR64_IDX64-NEXT: Loop %for.body: symbolic max backedge-taken count is (-2 + (trunc i64 (ptrtoint ptr @sext_like_noop to i64) to i32)) ; PTR64_IDX64-NEXT: Loop %for.body: Predicated backedge-taken count is (-2 + (trunc i64 (ptrtoint ptr @sext_like_noop to i64) to i32)) ; PTR64_IDX64-NEXT: Predicates: @@ -367,7 +367,7 @@ define i64 @sext_like_noop(i32 %n) { ; PTR64_IDX32-NEXT: --> {2,+,1}<%for.body> U: [2,0) S: [2,0) Exits: ptrtoint (ptr @sext_like_noop to i32) LoopDispositions: { %for.body: Computable } ; PTR64_IDX32-NEXT: Determining loop execution counts for: @sext_like_noop ; PTR64_IDX32-NEXT: Loop %for.body: backedge-taken count is (-2 + ptrtoint (ptr @sext_like_noop to i32)) -; PTR64_IDX32-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; PTR64_IDX32-NEXT: Loop %for.body: constant max backedge-taken count is i32 -1 ; PTR64_IDX32-NEXT: Loop %for.body: symbolic max backedge-taken count is (-2 + ptrtoint (ptr @sext_like_noop to i32)) ; PTR64_IDX32-NEXT: Loop %for.body: Predicated backedge-taken count is (-2 + ptrtoint (ptr @sext_like_noop to i32)) ; PTR64_IDX32-NEXT: Predicates: @@ -385,7 +385,7 @@ define i64 @sext_like_noop(i32 %n) { ; PTR16_IDX16-NEXT: --> {2,+,1}<%for.body> U: [2,0) S: [2,0) Exits: (zext i16 (ptrtoint ptr @sext_like_noop to i16) to i32) LoopDispositions: { %for.body: Computable } ; PTR16_IDX16-NEXT: Determining loop execution counts for: @sext_like_noop ; PTR16_IDX16-NEXT: Loop %for.body: backedge-taken count is (-2 + (zext i16 (ptrtoint ptr @sext_like_noop to i16) to i32)) -; PTR16_IDX16-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; PTR16_IDX16-NEXT: Loop %for.body: constant max backedge-taken count is i32 -1 ; PTR16_IDX16-NEXT: Loop %for.body: symbolic max backedge-taken count is (-2 + (zext i16 (ptrtoint ptr @sext_like_noop to i16) to i32)) ; PTR16_IDX16-NEXT: Loop %for.body: Predicated backedge-taken count is (-2 + (zext i16 (ptrtoint ptr @sext_like_noop to i16) to i32)) ; PTR16_IDX16-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/ptrtoint.ll b/llvm/test/Analysis/ScalarEvolution/ptrtoint.ll index 4451b7ee6bac..f30b3752d7b3 100644 --- a/llvm/test/Analysis/ScalarEvolution/ptrtoint.ll +++ b/llvm/test/Analysis/ScalarEvolution/ptrtoint.ll @@ -229,7 +229,7 @@ define void @ptrtoint_of_addrec(ptr %in, i32 %count) { ; X64-NEXT: --> {1,+,1}<%loop> U: [1,0) S: [1,0) Exits: (zext i32 %count to i64) LoopDispositions: { %loop: Computable } ; X64-NEXT: Determining loop execution counts for: @ptrtoint_of_addrec ; X64-NEXT: Loop %loop: backedge-taken count is (-1 + (zext i32 %count to i64)) -; X64-NEXT: Loop %loop: constant max backedge-taken count is -1 +; X64-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; X64-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (zext i32 %count to i64)) ; X64-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (zext i32 %count to i64)) ; X64-NEXT: Predicates: @@ -249,7 +249,7 @@ define void @ptrtoint_of_addrec(ptr %in, i32 %count) { ; X32-NEXT: --> {1,+,1}<%loop> U: [1,0) S: [1,0) Exits: (zext i32 %count to i64) LoopDispositions: { %loop: Computable } ; X32-NEXT: Determining loop execution counts for: @ptrtoint_of_addrec ; X32-NEXT: Loop %loop: backedge-taken count is (-1 + (zext i32 %count to i64)) -; X32-NEXT: Loop %loop: constant max backedge-taken count is -1 +; X32-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; X32-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (zext i32 %count to i64)) ; X32-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (zext i32 %count to i64)) ; X32-NEXT: Predicates: @@ -397,7 +397,7 @@ define void @pr46786_c26_char(ptr %arg, ptr %arg1, ptr %arg2) { ; X64-NEXT: --> {(1 + %arg),+,1}<%bb6> U: full-set S: full-set Exits: ((-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64) + %arg) LoopDispositions: { %bb6: Computable } ; X64-NEXT: Determining loop execution counts for: @pr46786_c26_char ; X64-NEXT: Loop %bb6: backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) -; X64-NEXT: Loop %bb6: constant max backedge-taken count is -1 +; X64-NEXT: Loop %bb6: constant max backedge-taken count is i64 -1 ; X64-NEXT: Loop %bb6: symbolic max backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) ; X64-NEXT: Loop %bb6: Predicated backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) ; X64-NEXT: Predicates: @@ -425,7 +425,7 @@ define void @pr46786_c26_char(ptr %arg, ptr %arg1, ptr %arg2) { ; X32-NEXT: --> {(1 + %arg),+,1}<%bb6> U: full-set S: full-set Exits: ((-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32) + %arg) LoopDispositions: { %bb6: Computable } ; X32-NEXT: Determining loop execution counts for: @pr46786_c26_char ; X32-NEXT: Loop %bb6: backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) -; X32-NEXT: Loop %bb6: constant max backedge-taken count is -1 +; X32-NEXT: Loop %bb6: constant max backedge-taken count is i32 -1 ; X32-NEXT: Loop %bb6: symbolic max backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) ; X32-NEXT: Loop %bb6: Predicated backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) ; X32-NEXT: Predicates: @@ -486,7 +486,7 @@ define void @pr46786_c26_int(ptr %arg, ptr %arg1, ptr %arg2) { ; X64-NEXT: --> {(4 + %arg),+,4}<%bb6> U: full-set S: full-set Exits: (4 + (4 * ((-4 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) /u 4)) + %arg) LoopDispositions: { %bb6: Computable } ; X64-NEXT: Determining loop execution counts for: @pr46786_c26_int ; X64-NEXT: Loop %bb6: backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) /u 4) -; X64-NEXT: Loop %bb6: constant max backedge-taken count is 4611686018427387903 +; X64-NEXT: Loop %bb6: constant max backedge-taken count is i64 4611686018427387903 ; X64-NEXT: Loop %bb6: symbolic max backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) /u 4) ; X64-NEXT: Loop %bb6: Predicated backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) /u 4) ; X64-NEXT: Predicates: @@ -516,7 +516,7 @@ define void @pr46786_c26_int(ptr %arg, ptr %arg1, ptr %arg2) { ; X32-NEXT: --> {(4 + %arg),+,4}<%bb6> U: full-set S: full-set Exits: (4 + (4 * ((-4 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) /u 4)) + %arg) LoopDispositions: { %bb6: Computable } ; X32-NEXT: Determining loop execution counts for: @pr46786_c26_int ; X32-NEXT: Loop %bb6: backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) /u 4) -; X32-NEXT: Loop %bb6: constant max backedge-taken count is 1073741823 +; X32-NEXT: Loop %bb6: constant max backedge-taken count is i32 1073741823 ; X32-NEXT: Loop %bb6: symbolic max backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) /u 4) ; X32-NEXT: Loop %bb6: Predicated backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) /u 4) ; X32-NEXT: Predicates: @@ -564,7 +564,7 @@ define void @ptrtoint_of_integer(ptr %arg, i64 %arg1, i1 %arg2) local_unnamed_ad ; X64-NEXT: Loop %bb8: Unpredictable backedge-taken count. ; X64-NEXT: exit count for bb8: ***COULDNOTCOMPUTE*** ; X64-NEXT: exit count for bb10: (-2 + (-1 * %arg1) + (ptrtoint ptr %arg to i64)) -; X64-NEXT: Loop %bb8: constant max backedge-taken count is -1 +; X64-NEXT: Loop %bb8: constant max backedge-taken count is i64 -1 ; X64-NEXT: Loop %bb8: symbolic max backedge-taken count is (-2 + (-1 * %arg1) + (ptrtoint ptr %arg to i64)) ; X64-NEXT: symbolic max exit count for bb8: ***COULDNOTCOMPUTE*** ; X64-NEXT: symbolic max exit count for bb10: (-2 + (-1 * %arg1) + (ptrtoint ptr %arg to i64)) @@ -584,7 +584,7 @@ define void @ptrtoint_of_integer(ptr %arg, i64 %arg1, i1 %arg2) local_unnamed_ad ; X32-NEXT: Loop %bb8: Unpredictable backedge-taken count. ; X32-NEXT: exit count for bb8: ***COULDNOTCOMPUTE*** ; X32-NEXT: exit count for bb10: (-2 + (zext i32 (ptrtoint ptr %arg to i32) to i64) + (-1 * %arg1)) -; X32-NEXT: Loop %bb8: constant max backedge-taken count is -1 +; X32-NEXT: Loop %bb8: constant max backedge-taken count is i64 -1 ; X32-NEXT: Loop %bb8: symbolic max backedge-taken count is (-2 + (zext i32 (ptrtoint ptr %arg to i32) to i64) + (-1 * %arg1)) ; X32-NEXT: symbolic max exit count for bb8: ***COULDNOTCOMPUTE*** ; X32-NEXT: symbolic max exit count for bb10: (-2 + (zext i32 (ptrtoint ptr %arg to i32) to i64) + (-1 * %arg1)) diff --git a/llvm/test/Analysis/ScalarEvolution/range-signedness.ll b/llvm/test/Analysis/ScalarEvolution/range-signedness.ll index 119bfc39afe6..74b78b35513e 100644 --- a/llvm/test/Analysis/ScalarEvolution/range-signedness.ll +++ b/llvm/test/Analysis/ScalarEvolution/range-signedness.ll @@ -40,7 +40,7 @@ define void @y(ptr %addr) { ; CHECK-NEXT: --> {-4,+,1}<%loop> U: [-4,7) S: [-4,7) Exits: 6 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @y ; CHECK-NEXT: Loop %loop: backedge-taken count is 10 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 10 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 10 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 10 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 10 ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/range_nw_flag.ll b/llvm/test/Analysis/ScalarEvolution/range_nw_flag.ll index 1272975406f7..dd487be904d5 100644 --- a/llvm/test/Analysis/ScalarEvolution/range_nw_flag.ll +++ b/llvm/test/Analysis/ScalarEvolution/range_nw_flag.ll @@ -15,7 +15,7 @@ define void @test-add-nuw(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: --> ((4 * (sext i32 {(1 + %offset),+,1}<%loop> to i64)) + %input) U: full-set S: full-set Exits: ((4 * (sext i32 (%offset + %numIterations) to i64)) + %input) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-add-nuw ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -51,7 +51,7 @@ define void @test-addrec-nuw(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: --> ((4 * (sext i32 {(1 + (10 smax %offset)),+,1}<%loop> to i64)) + %input) U: full-set S: full-set Exits: ((4 * (sext i32 ((10 smax %offset) + %numIterations) to i64)) + %input) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-addrec-nuw ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -89,7 +89,7 @@ define void @test-addrec-nsw-start-neg-strip-neg(ptr %input, i32 %offset, i32 %n ; CHECK-NEXT: --> {(-4 + (4 * (-10 smin (sext i32 %offset to i64))) + %input),+,-4}<%loop> U: full-set S: full-set Exits: (-4 + (4 * (-10 smin (sext i32 %offset to i64))) + (-4 * (zext i32 (-1 + (-1 * %numIterations)) to i64)) + %input) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-addrec-nsw-start-neg-strip-neg ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %numIterations)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %numIterations)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %numIterations)) ; CHECK-NEXT: Predicates: @@ -127,7 +127,7 @@ define void @test-addrec-nsw-start-pos-strip-neg(ptr %input, i32 %offset, i32 %n ; CHECK-NEXT: --> {(-4 + (4 * (10 smin (sext i32 %offset to i64))) + %input),+,-4}<%loop> U: full-set S: full-set Exits: (-4 + (4 * (10 smin (sext i32 %offset to i64))) + (-4 * (zext i32 (-1 + (-1 * %numIterations)) to i64)) + %input) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-addrec-nsw-start-pos-strip-neg ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %numIterations)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %numIterations)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %numIterations)) ; CHECK-NEXT: Predicates: @@ -165,7 +165,7 @@ define void @test-addrec-nsw-start-pos-strip-pos(ptr %input, i32 %offset, i32 %n ; CHECK-NEXT: --> {(4 + (4 * (zext i32 (10 smax %offset) to i64)) + %input),+,4}<%loop> U: [44,0) S: [44,0) Exits: (4 + (4 * (zext i32 (-1 + %numIterations) to i64)) + (4 * (zext i32 (10 smax %offset) to i64)) + %input) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-addrec-nsw-start-pos-strip-pos ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: @@ -203,7 +203,7 @@ define void @test-addrec-nsw-start-neg-strip-pos(ptr %input, i32 %offset, i32 %n ; CHECK-NEXT: --> {(4 + (4 * (-10 smax (sext i32 %offset to i64))) + %input),+,4}<%loop> U: full-set S: full-set Exits: (4 + (4 * (zext i32 (-1 + %numIterations) to i64)) + (4 * (-10 smax (sext i32 %offset to i64))) + %input) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test-addrec-nsw-start-neg-strip-pos ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/ranges.ll b/llvm/test/Analysis/ScalarEvolution/ranges.ll index 3b43c525d2ce..e6e280c676b9 100644 --- a/llvm/test/Analysis/ScalarEvolution/ranges.ll +++ b/llvm/test/Analysis/ScalarEvolution/ranges.ll @@ -136,7 +136,7 @@ define void @add_6(i32 %n) { ; CHECK-NEXT: --> {6,+,6}<%loop> U: [6,-3) S: [-2147483648,2147483647) Exits: (6 + (6 * ((((-1 * (1 umin %n)) + %n) /u 6) + (1 umin %n)))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @add_6 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 6) + (1 umin %n)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 715827882 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 715827882 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 6) + (1 umin %n)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 6) + (1 umin %n)) ; CHECK-NEXT: Predicates: @@ -163,7 +163,7 @@ define void @add_7(i32 %n) { ; CHECK-NEXT: --> {7,+,7}<%loop> U: [7,-3) S: [7,0) Exits: (7 + (7 * ((((-1 * (1 umin %n)) + %n) /u 7) + (1 umin %n)))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @add_7 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 7) + (1 umin %n)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 613566756 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 613566756 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 7) + (1 umin %n)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 7) + (1 umin %n)) ; CHECK-NEXT: Predicates: @@ -190,7 +190,7 @@ define void @add_8(i32 %n) { ; CHECK-NEXT: --> {8,+,8}<%loop> U: [8,-7) S: [-2147483648,2147483641) Exits: (8 + (8 * ((7 + %n) /u 8))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @add_8 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((7 + %n) /u 8) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 536870911 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 536870911 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((7 + %n) /u 8) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((7 + %n) /u 8) ; CHECK-NEXT: Predicates: @@ -218,7 +218,7 @@ define void @add_9(i32 %n) { ; CHECK-NEXT: --> {9,+,9}<%loop> U: [9,-3) S: [9,0) Exits: (9 + (9 * ((((-1 * (1 umin %n)) + %n) /u 9) + (1 umin %n)))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @add_9 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 9) + (1 umin %n)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 477218588 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 477218588 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 9) + (1 umin %n)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 9) + (1 umin %n)) ; CHECK-NEXT: Predicates: @@ -246,7 +246,7 @@ define void @add_10(i32 %n) { ; CHECK-NEXT: --> {10,+,10}<%loop> U: [10,-5) S: [-2147483648,2147483647) Exits: (10 + (10 * ((((-1 * (1 umin %n)) + %n) /u 10) + (1 umin %n)))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @add_10 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 10) + (1 umin %n)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 429496729 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 429496729 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 10) + (1 umin %n)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 10) + (1 umin %n)) ; CHECK-NEXT: Predicates: @@ -511,7 +511,7 @@ define void @truncate(i16 %n) { ; CHECK-NEXT: --> {9,+,9}<%loop> U: full-set S: full-set Exits: (9 + (9 * (trunc i16 ((((-1 * (1 umin %n)) + %n) /u 9) + (1 umin %n)) to i8))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @truncate ; CHECK-NEXT: Loop %loop: backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 9) + (1 umin %n)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 7281 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i16 7281 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 9) + (1 umin %n)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 9) + (1 umin %n)) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/sdiv.ll b/llvm/test/Analysis/ScalarEvolution/sdiv.ll index 3d4aaa29536e..48856d00b1c2 100644 --- a/llvm/test/Analysis/ScalarEvolution/sdiv.ll +++ b/llvm/test/Analysis/ScalarEvolution/sdiv.ll @@ -31,7 +31,7 @@ define dso_local void @_Z4loopi(i32 %width) local_unnamed_addr #0 { ; CHECK-NEXT: --> {1,+,1}<%for.cond> U: [1,0) S: [1,0) Exits: (1 + %width) LoopDispositions: { %for.cond: Computable } ; CHECK-NEXT: Determining loop execution counts for: @_Z4loopi ; CHECK-NEXT: Loop %for.cond: backedge-taken count is %width -; CHECK-NEXT: Loop %for.cond: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.cond: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %for.cond: symbolic max backedge-taken count is %width ; CHECK-NEXT: Loop %for.cond: Predicated backedge-taken count is %width ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/sext-add-inreg-loop.ll b/llvm/test/Analysis/ScalarEvolution/sext-add-inreg-loop.ll index 1345aa4e6411..3de4dcfc4046 100644 --- a/llvm/test/Analysis/ScalarEvolution/sext-add-inreg-loop.ll +++ b/llvm/test/Analysis/ScalarEvolution/sext-add-inreg-loop.ll @@ -24,7 +24,7 @@ define dso_local i32 @test_loop(ptr nocapture noundef readonly %x) { ; CHECK-NEXT: --> {2,+,1}<%for.body> U: [2,11) S: [2,11) Exits: 10 LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_loop ; CHECK-NEXT: Loop %for.body: backedge-taken count is 8 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 8 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 8 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 8 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 8 ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/sext-mul.ll b/llvm/test/Analysis/ScalarEvolution/sext-mul.ll index 2340d5cb6fb6..e375e7a066c9 100644 --- a/llvm/test/Analysis/ScalarEvolution/sext-mul.ll +++ b/llvm/test/Analysis/ScalarEvolution/sext-mul.ll @@ -30,7 +30,7 @@ define void @foo(ptr nocapture %arg, i32 %arg1, i32 %arg2) { ; CHECK-NEXT: --> {1,+,1}<%bb7> U: [1,2147483648) S: [1,2147483648) Exits: (zext i32 %arg2 to i64) LoopDispositions: { %bb7: Computable } ; CHECK-NEXT: Determining loop execution counts for: @foo ; CHECK-NEXT: Loop %bb7: backedge-taken count is (-1 + (zext i32 %arg2 to i64)) -; CHECK-NEXT: Loop %bb7: constant max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %bb7: constant max backedge-taken count is i64 2147483646 ; CHECK-NEXT: Loop %bb7: symbolic max backedge-taken count is (-1 + (zext i32 %arg2 to i64)) ; CHECK-NEXT: Loop %bb7: Predicated backedge-taken count is (-1 + (zext i32 %arg2 to i64)) ; CHECK-NEXT: Predicates: @@ -97,7 +97,7 @@ define void @goo(ptr nocapture %arg3, i32 %arg4, i32 %arg5) { ; CHECK-NEXT: --> {1,+,1}<%bb7> U: [1,2147483648) S: [1,2147483648) Exits: (zext i32 %arg5 to i128) LoopDispositions: { %bb7: Computable } ; CHECK-NEXT: Determining loop execution counts for: @goo ; CHECK-NEXT: Loop %bb7: backedge-taken count is (-1 + (zext i32 %arg5 to i128)) -; CHECK-NEXT: Loop %bb7: constant max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %bb7: constant max backedge-taken count is i128 2147483646 ; CHECK-NEXT: Loop %bb7: symbolic max backedge-taken count is (-1 + (zext i32 %arg5 to i128)) ; CHECK-NEXT: Loop %bb7: Predicated backedge-taken count is (-1 + (zext i32 %arg5 to i128)) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/sext-to-zext.ll b/llvm/test/Analysis/ScalarEvolution/sext-to-zext.ll index 18f20c371789..7db59734e6be 100644 --- a/llvm/test/Analysis/ScalarEvolution/sext-to-zext.ll +++ b/llvm/test/Analysis/ScalarEvolution/sext-to-zext.ll @@ -20,7 +20,7 @@ define void @f(i1 %c) { ; CHECK-NEXT: --> {(zext i32 %start to i64),+,(sext i32 %step to i64)}<%loop> U: [0,101) S: [0,101) Exits: ((zext i32 %start to i64) + (99 * (sext i32 %step to i64))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @f ; CHECK-NEXT: Loop %loop: backedge-taken count is 99 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 99 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 99 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 99 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 99 ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/shift-op.ll b/llvm/test/Analysis/ScalarEvolution/shift-op.ll index 41e746c2f6dd..5a1e45431aaf 100644 --- a/llvm/test/Analysis/ScalarEvolution/shift-op.ll +++ b/llvm/test/Analysis/ScalarEvolution/shift-op.ll @@ -6,7 +6,7 @@ define void @test0(i32 %init) { ; CHECK-LABEL: 'test0' ; CHECK-NEXT: Determining loop execution counts for: @test0 ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 32 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 32 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 32 ; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; @@ -28,7 +28,7 @@ define void @test1(i32 %init) { ; CHECK-LABEL: 'test1' ; CHECK-NEXT: Determining loop execution counts for: @test1 ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 32 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 32 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 32 ; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; @@ -74,7 +74,7 @@ define void @test3(ptr %init.ptr) { ; CHECK-LABEL: 'test3' ; CHECK-NEXT: Determining loop execution counts for: @test3 ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 32 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 32 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 32 ; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; @@ -97,7 +97,7 @@ define void @test4(ptr %init.ptr) { ; CHECK-LABEL: 'test4' ; CHECK-NEXT: Determining loop execution counts for: @test4 ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 32 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 32 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 32 ; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; @@ -167,7 +167,7 @@ define void @test7(i32 %init) { ; CHECK-LABEL: 'test7' ; CHECK-NEXT: Determining loop execution counts for: @test7 ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 32 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 32 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 32 ; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; diff --git a/llvm/test/Analysis/ScalarEvolution/shift-recurrences.ll b/llvm/test/Analysis/ScalarEvolution/shift-recurrences.ll index bbb1f46197a8..0a2e121c0502 100644 --- a/llvm/test/Analysis/ScalarEvolution/shift-recurrences.ll +++ b/llvm/test/Analysis/ScalarEvolution/shift-recurrences.ll @@ -207,7 +207,7 @@ define void @test_shl2() { ; CHECK-NEXT: --> (2 * %iv.shl) U: [8,129) S: [8,129) Exits: 128 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_shl2 ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 ; CHECK-NEXT: Predicates: @@ -242,7 +242,7 @@ define void @test_shl3(i1 %c) { ; CHECK-NEXT: --> %iv.shl.next U: [0,-3) S: [-9223372036854775808,9223372036854775805) Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_shl3 ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 ; CHECK-NEXT: Predicates: @@ -276,7 +276,7 @@ define void @test_shl4() { ; CHECK-NEXT: --> (2 * %iv.shl) U: [8,-9223372036854775807) S: [-9223372036854775808,9223372036854775801) Exits: -9223372036854775808 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_shl4 ; CHECK-NEXT: Loop %loop: backedge-taken count is 60 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 60 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 60 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 60 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 60 ; CHECK-NEXT: Predicates: @@ -309,7 +309,7 @@ define void @test_shl5() { ; CHECK-NEXT: --> (2 * %iv.shl) U: [0,-7) S: [-9223372036854775808,9223372036854775801) Exits: 0 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_shl5 ; CHECK-NEXT: Loop %loop: backedge-taken count is 61 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 61 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 61 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 61 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 61 ; CHECK-NEXT: Predicates: @@ -344,7 +344,7 @@ define void @test_shl6(i1 %c) { ; CHECK-NEXT: --> %iv.shl.next U: [0,-3) S: [-9223372036854775808,9223372036854775805) Exits: 16 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_shl6 ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 ; CHECK-NEXT: Predicates: @@ -378,7 +378,7 @@ define void @test_shl7(i1 %c, i64 %shiftamt) { ; CHECK-NEXT: --> %iv.shl.next U: [0,-3) S: [-9223372036854775808,9223372036854775805) Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_shl7 ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 ; CHECK-NEXT: Predicates: @@ -525,7 +525,7 @@ define void @test_ashr_tc_positive() { ; CHECK-NEXT: --> %iv.ashr.next U: [0,512) S: [0,512) Exits: 31 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_ashr_tc_positive ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 ; CHECK-NEXT: Predicates: @@ -557,7 +557,7 @@ define void @test_ashr_tc_negative() { ; CHECK-NEXT: --> %iv.ashr.next U: [-64,0) S: [-64,0) Exits: -4 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_ashr_tc_negative ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 ; CHECK-NEXT: Predicates: @@ -591,7 +591,7 @@ define void @test_ashr_tc_either(i1 %a) { ; CHECK-NEXT: --> %iv.ashr.next U: [-16,16) S: [-16,16) Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_ashr_tc_either ; CHECK-NEXT: Loop %loop: backedge-taken count is 60 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 60 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 60 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 60 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 60 ; CHECK-NEXT: Predicates: @@ -624,7 +624,7 @@ define void @test_ashr_zero_shift() { ; CHECK-NEXT: --> %iv.ashr U: [1023,1024) S: [1023,1024) Exits: 1023 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_ashr_zero_shift ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 ; CHECK-NEXT: Predicates: @@ -656,7 +656,7 @@ define void @test_lshr_tc_positive() { ; CHECK-NEXT: --> (%iv.lshr /u 2) U: [31,512) S: [31,512) Exits: 31 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_lshr_tc_positive ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 ; CHECK-NEXT: Predicates: @@ -688,7 +688,7 @@ define void @test_lshr_tc_negative() { ; CHECK-NEXT: --> (%iv.lshr /u 2) U: [7,-128) S: [7,-128) Exits: 7 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_lshr_tc_negative ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 ; CHECK-NEXT: Predicates: @@ -722,7 +722,7 @@ define void @test_lshr_tc_either(i1 %a) { ; CHECK-NEXT: --> (%iv.lshr /u 2) U: [0,-128) S: [0,-128) Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_lshr_tc_either ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 ; CHECK-NEXT: Predicates: @@ -755,7 +755,7 @@ define void @test_lshr_zero_shift() { ; CHECK-NEXT: --> %iv.lshr U: [1023,1024) S: [1023,1024) Exits: 1023 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_lshr_zero_shift ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 ; CHECK-NEXT: Predicates: @@ -788,7 +788,7 @@ define void @test_lshr_power_of_2_start() { ; CHECK-NEXT: --> (%iv.lshr /u 4) U: [1,257) S: [1,257) Exits: 1 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_lshr_power_of_2_start ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 ; CHECK-NEXT: Predicates: @@ -821,7 +821,7 @@ define void @test_lshr_arbitrary_start() { ; CHECK-NEXT: --> (%iv.lshr /u 4) U: [0,240) S: [0,240) Exits: 0 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_lshr_arbitrary_start ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 ; CHECK-NEXT: Predicates: @@ -853,7 +853,7 @@ define void @test_lshr_start_power_of_2_plus_one() { ; CHECK-NEXT: --> (%iv.lshr /u 4) U: [1,257) S: [1,257) Exits: 1 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_lshr_start_power_of_2_plus_one ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/sle.ll b/llvm/test/Analysis/ScalarEvolution/sle.ll index a1b078113017..16264d463f61 100644 --- a/llvm/test/Analysis/ScalarEvolution/sle.ll +++ b/llvm/test/Analysis/ScalarEvolution/sle.ll @@ -8,7 +8,7 @@ define void @le(i64 %n, ptr nocapture %p) nounwind { ; CHECK-LABEL: 'le' ; CHECK-NEXT: Determining loop execution counts for: @le ; CHECK-NEXT: Loop %for.body: backedge-taken count is %n -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 9223372036854775807 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 9223372036854775807 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is %n ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is %n ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/smax-br-phi-idioms.ll b/llvm/test/Analysis/ScalarEvolution/smax-br-phi-idioms.ll index 9ec124ed333c..2f0e547b03e5 100644 --- a/llvm/test/Analysis/ScalarEvolution/smax-br-phi-idioms.ll +++ b/llvm/test/Analysis/ScalarEvolution/smax-br-phi-idioms.ll @@ -157,7 +157,7 @@ define i32 @f5(ptr %val) { ; CHECK-NEXT: Loop %for.end: backedge-taken count is false ; CHECK-NEXT: exit count for for.end: false ; CHECK-NEXT: exit count for for.condt: false -; CHECK-NEXT: Loop %for.end: constant max backedge-taken count is false +; CHECK-NEXT: Loop %for.end: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %for.end: symbolic max backedge-taken count is false ; CHECK-NEXT: symbolic max exit count for for.end: false ; CHECK-NEXT: symbolic max exit count for for.condt: false diff --git a/llvm/test/Analysis/ScalarEvolution/smin-smax-folds.ll b/llvm/test/Analysis/ScalarEvolution/smin-smax-folds.ll index 5fc871433dd6..2de71ee40304 100644 --- a/llvm/test/Analysis/ScalarEvolution/smin-smax-folds.ll +++ b/llvm/test/Analysis/ScalarEvolution/smin-smax-folds.ll @@ -34,7 +34,7 @@ define void @smin_simplify_with_guard(i32 %n) { ; CHECK-NEXT: --> {(-1 + %n),+,-1}<%for.body> U: full-set S: full-set Exits: -1 LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @smin_simplify_with_guard ; CHECK-NEXT: Loop %for.body: backedge-taken count is %n -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 2147483647 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483647 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is %n ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is %n ; CHECK-NEXT: Predicates: @@ -71,7 +71,7 @@ define void @smin_to_smax(i32 %n) { ; CHECK-NEXT: --> {(-1 + %n),+,-1}<%for.body> U: full-set S: full-set Exits: (-1 + (0 smin %n)) LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @smin_to_smax ; CHECK-NEXT: Loop %for.body: backedge-taken count is ((-1 * (0 smin %n)) + %n) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 2147483647 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483647 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((-1 * (0 smin %n)) + %n) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-1 * (0 smin %n)) + %n) ; CHECK-NEXT: Predicates: @@ -107,7 +107,7 @@ define void @smax_simplify_with_guard(i32 %start, i32 %n) { ; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: full-set S: full-set Exits: (1 + %n) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @smax_simplify_with_guard ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 * %start) + %n) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 * %start) + %n) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 * %start) + %n) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/solve-quadratic-i1.ll b/llvm/test/Analysis/ScalarEvolution/solve-quadratic-i1.ll index 89f15fa87376..0c718fe4f880 100644 --- a/llvm/test/Analysis/ScalarEvolution/solve-quadratic-i1.ll +++ b/llvm/test/Analysis/ScalarEvolution/solve-quadratic-i1.ll @@ -18,7 +18,7 @@ define void @f0() { ; CHECK-NEXT: --> (zext i1 {true,+,true,+,true}<%b1> to i16) U: [0,2) S: [0,2) Exits: 0 LoopDispositions: { %b1: Computable } ; CHECK-NEXT: Determining loop execution counts for: @f0 ; CHECK-NEXT: Loop %b1: backedge-taken count is 1 -; CHECK-NEXT: Loop %b1: constant max backedge-taken count is 1 +; CHECK-NEXT: Loop %b1: constant max backedge-taken count is i6 1 ; CHECK-NEXT: Loop %b1: symbolic max backedge-taken count is 1 ; CHECK-NEXT: Loop %b1: Predicated backedge-taken count is 1 ; CHECK-NEXT: Predicates: @@ -69,7 +69,7 @@ define void @f1() #0 { ; CHECK-NEXT: Loop %b3: Unpredictable symbolic max backedge-taken count. ; CHECK-NEXT: Loop %b3: Unpredictable predicated backedge-taken count. ; CHECK-NEXT: Loop %b1: backedge-taken count is 2 -; CHECK-NEXT: Loop %b1: constant max backedge-taken count is 2 +; CHECK-NEXT: Loop %b1: constant max backedge-taken count is i6 2 ; CHECK-NEXT: Loop %b1: symbolic max backedge-taken count is 2 ; CHECK-NEXT: Loop %b1: Predicated backedge-taken count is 2 ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/solve-quadratic-overflow.ll b/llvm/test/Analysis/ScalarEvolution/solve-quadratic-overflow.ll index e04d019b819c..76c5f13a7992 100644 --- a/llvm/test/Analysis/ScalarEvolution/solve-quadratic-overflow.ll +++ b/llvm/test/Analysis/ScalarEvolution/solve-quadratic-overflow.ll @@ -24,7 +24,7 @@ define signext i32 @f0() { ; CHECK-NEXT: --> {-1,+,-1}<%b1> U: [-256,0) S: [-256,0) --> -256 U: [-256,-255) S: [-256,-255) ; CHECK-NEXT: Determining loop execution counts for: @f0 ; CHECK-NEXT: Loop %b1: backedge-taken count is 255 -; CHECK-NEXT: Loop %b1: constant max backedge-taken count is 255 +; CHECK-NEXT: Loop %b1: constant max backedge-taken count is i16 255 ; CHECK-NEXT: Loop %b1: symbolic max backedge-taken count is 255 ; CHECK-NEXT: Loop %b1: Predicated backedge-taken count is 255 ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/srem.ll b/llvm/test/Analysis/ScalarEvolution/srem.ll index 3a7f7e736607..cc67865f66ce 100644 --- a/llvm/test/Analysis/ScalarEvolution/srem.ll +++ b/llvm/test/Analysis/ScalarEvolution/srem.ll @@ -31,7 +31,7 @@ define dso_local void @_Z4loopi(i32 %width) local_unnamed_addr #0 { ; CHECK-NEXT: --> {1,+,1}<%for.cond> U: [1,0) S: [1,0) Exits: (1 + %width) LoopDispositions: { %for.cond: Computable } ; CHECK-NEXT: Determining loop execution counts for: @_Z4loopi ; CHECK-NEXT: Loop %for.cond: backedge-taken count is %width -; CHECK-NEXT: Loop %for.cond: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.cond: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %for.cond: symbolic max backedge-taken count is %width ; CHECK-NEXT: Loop %for.cond: Predicated backedge-taken count is %width ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/symbolic_max_exit_count.ll b/llvm/test/Analysis/ScalarEvolution/symbolic_max_exit_count.ll index d0be0082e713..da3fd481dd79 100644 --- a/llvm/test/Analysis/ScalarEvolution/symbolic_max_exit_count.ll +++ b/llvm/test/Analysis/ScalarEvolution/symbolic_max_exit_count.ll @@ -19,7 +19,7 @@ define i32 @test_simple_case(i32 %start, i32 %len) { ; CHECK-NEXT: exit count for loop: %start ; CHECK-NEXT: exit count for range_check_block: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %start ; CHECK-NEXT: symbolic max exit count for loop: %start ; CHECK-NEXT: symbolic max exit count for range_check_block: ***COULDNOTCOMPUTE*** @@ -78,7 +78,7 @@ define i32 @test_litter_conditions(i32 %start, i32 %len) { ; CHECK-NEXT: exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for range_check_block: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %start ; CHECK-NEXT: symbolic max exit count for loop: %start ; CHECK-NEXT: symbolic max exit count for range_check_block: ***COULDNOTCOMPUTE*** @@ -141,7 +141,7 @@ define i32 @test_litter_conditions_bad_context(i32 %start, i32 %len) { ; CHECK-NEXT: exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for range_check_block: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %start ; CHECK-NEXT: symbolic max exit count for loop: %start ; CHECK-NEXT: symbolic max exit count for range_check_block: ***COULDNOTCOMPUTE*** @@ -197,7 +197,7 @@ define i32 @test_and_conditions(i32 %start, i32 %len) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %start ; CHECK-NEXT: symbolic max exit count for loop: %start ; CHECK-NEXT: symbolic max exit count for backedge: ***COULDNOTCOMPUTE*** @@ -240,7 +240,7 @@ define i32 @test_mixup_constant_symbolic(i32 %end, i32 %len) { ; CHECK-NEXT: exit count for loop: %end ; CHECK-NEXT: exit count for range_check_block: 1000 ; CHECK-NEXT: exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 1000 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 1000 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (1000 umin %end) ; CHECK-NEXT: symbolic max exit count for loop: %end ; CHECK-NEXT: symbolic max exit count for range_check_block: 1000 @@ -289,7 +289,7 @@ define i32 @test_mixup_constant_symbolic_merged(i32 %end, i32 %len) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for loop: (1000 umin %end) ; CHECK-NEXT: exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 1000 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 1000 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (1000 umin %end) ; CHECK-NEXT: symbolic max exit count for loop: (1000 umin %end) ; CHECK-NEXT: symbolic max exit count for backedge: ***COULDNOTCOMPUTE*** @@ -346,7 +346,7 @@ define i32 @test_two_phis(i32 %start_1, i32 %start_2, i32 %len) { ; CHECK-NEXT: exit count for zero_check_block: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for range_check_block: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%start_1 umin_seq %start_2) ; CHECK-NEXT: symbolic max exit count for loop: %start_1 ; CHECK-NEXT: symbolic max exit count for zero_check_block: %start_2 @@ -407,7 +407,7 @@ define i32 @test_two_phis_simple(i32 %start_1, i32 %start_2, i32 %len) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (%start_1 umin_seq %start_2) ; CHECK-NEXT: exit count for loop: %start_1 ; CHECK-NEXT: exit count for backedge: %start_2 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%start_1 umin_seq %start_2) ; CHECK-NEXT: symbolic max exit count for loop: %start_1 ; CHECK-NEXT: symbolic max exit count for backedge: %start_2 @@ -464,7 +464,7 @@ define i32 @test_two_phis_arithmetic_and(i32 %start_1, i32 %start_2, i32 %len) { ; CHECK-NEXT: exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for range_check_block: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%start_1 umin %start_2) ; CHECK-NEXT: symbolic max exit count for loop: (%start_1 umin %start_2) ; CHECK-NEXT: symbolic max exit count for range_check_block: ***COULDNOTCOMPUTE*** @@ -611,7 +611,7 @@ define i32 @test_two_phis_logical_and(i32 %start_1, i32 %start_2, i32 %len) { ; CHECK-NEXT: exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for range_check_block: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%start_1 umin_seq %start_2) ; CHECK-NEXT: symbolic max exit count for loop: (%start_1 umin_seq %start_2) ; CHECK-NEXT: symbolic max exit count for range_check_block: ***COULDNOTCOMPUTE*** diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-andor-selectform.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-andor-selectform.ll index 42fca4adbca5..4527211f3ab9 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-andor-selectform.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-andor-selectform.ll @@ -8,7 +8,7 @@ define void @unsimplified_and1(i32 %n) { ; CHECK-LABEL: 'unsimplified_and1' ; CHECK-NEXT: Determining loop execution counts for: @unsimplified_and1 ; CHECK-NEXT: Loop %loop: backedge-taken count is %n -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n ; CHECK-NEXT: Predicates: @@ -32,7 +32,7 @@ define void @unsimplified_and2(i32 %n) { ; CHECK-LABEL: 'unsimplified_and2' ; CHECK-NEXT: Determining loop execution counts for: @unsimplified_and2 ; CHECK-NEXT: Loop %loop: backedge-taken count is %n -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n ; CHECK-NEXT: Predicates: @@ -56,7 +56,7 @@ define void @unsimplified_and3(i32 %n) { ; CHECK-LABEL: 'unsimplified_and3' ; CHECK-NEXT: Determining loop execution counts for: @unsimplified_and3 ; CHECK-NEXT: Loop %loop: backedge-taken count is false -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is false +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is false ; CHECK-NEXT: Predicates: @@ -80,7 +80,7 @@ define void @unsimplified_and4(i32 %n) { ; CHECK-LABEL: 'unsimplified_and4' ; CHECK-NEXT: Determining loop execution counts for: @unsimplified_and4 ; CHECK-NEXT: Loop %loop: backedge-taken count is false -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is false +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is false ; CHECK-NEXT: Predicates: @@ -148,7 +148,7 @@ define void @unsimplified_or3(i32 %n) { ; CHECK-LABEL: 'unsimplified_or3' ; CHECK-NEXT: Determining loop execution counts for: @unsimplified_or3 ; CHECK-NEXT: Loop %loop: backedge-taken count is %n -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n ; CHECK-NEXT: Predicates: @@ -172,7 +172,7 @@ define void @unsimplified_or4(i32 %n) { ; CHECK-LABEL: 'unsimplified_or4' ; CHECK-NEXT: Determining loop execution counts for: @unsimplified_or4 ; CHECK-NEXT: Loop %loop: backedge-taken count is %n -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n ; CHECK-NEXT: Predicates: @@ -196,7 +196,7 @@ define void @reversed_and1(i32 %n) { ; CHECK-LABEL: 'reversed_and1' ; CHECK-NEXT: Determining loop execution counts for: @reversed_and1 ; CHECK-NEXT: Loop %loop: backedge-taken count is %n -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n ; CHECK-NEXT: Predicates: @@ -220,7 +220,7 @@ define void @reversed_and2(i32 %n) { ; CHECK-LABEL: 'reversed_and2' ; CHECK-NEXT: Determining loop execution counts for: @reversed_and2 ; CHECK-NEXT: Loop %loop: backedge-taken count is %n -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n ; CHECK-NEXT: Predicates: @@ -288,7 +288,7 @@ define void @reversed_or1(i32 %n) { ; CHECK-LABEL: 'reversed_or1' ; CHECK-NEXT: Determining loop execution counts for: @reversed_or1 ; CHECK-NEXT: Loop %loop: backedge-taken count is false -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is false +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is false ; CHECK-NEXT: Predicates: @@ -312,7 +312,7 @@ define void @reversed_or2(i32 %n) { ; CHECK-LABEL: 'reversed_or2' ; CHECK-NEXT: Determining loop execution counts for: @reversed_or2 ; CHECK-NEXT: Loop %loop: backedge-taken count is false -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is false +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is false ; CHECK-NEXT: Predicates: @@ -336,7 +336,7 @@ define void @reversed_or3(i32 %n) { ; CHECK-LABEL: 'reversed_or3' ; CHECK-NEXT: Determining loop execution counts for: @reversed_or3 ; CHECK-NEXT: Loop %loop: backedge-taken count is %n -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n ; CHECK-NEXT: Predicates: @@ -360,7 +360,7 @@ define void @reversed_or4(i32 %n) { ; CHECK-LABEL: 'reversed_or4' ; CHECK-NEXT: Determining loop execution counts for: @reversed_or4 ; CHECK-NEXT: Loop %loop: backedge-taken count is %n -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-andor.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-andor.ll index c689cb4e3b86..aa1fad42b6fb 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-andor.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-andor.ll @@ -8,7 +8,7 @@ define void @unsimplified_and1(i32 %n) { ; CHECK-LABEL: 'unsimplified_and1' ; CHECK-NEXT: Determining loop execution counts for: @unsimplified_and1 ; CHECK-NEXT: Loop %loop: backedge-taken count is %n -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n ; CHECK-NEXT: Predicates: @@ -32,7 +32,7 @@ define void @unsimplified_and2(i32 %n) { ; CHECK-LABEL: 'unsimplified_and2' ; CHECK-NEXT: Determining loop execution counts for: @unsimplified_and2 ; CHECK-NEXT: Loop %loop: backedge-taken count is %n -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n ; CHECK-NEXT: Predicates: @@ -56,7 +56,7 @@ define void @unsimplified_and3(i32 %n) { ; CHECK-LABEL: 'unsimplified_and3' ; CHECK-NEXT: Determining loop execution counts for: @unsimplified_and3 ; CHECK-NEXT: Loop %loop: backedge-taken count is false -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is false +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is false ; CHECK-NEXT: Predicates: @@ -80,7 +80,7 @@ define void @unsimplified_and4(i32 %n) { ; CHECK-LABEL: 'unsimplified_and4' ; CHECK-NEXT: Determining loop execution counts for: @unsimplified_and4 ; CHECK-NEXT: Loop %loop: backedge-taken count is false -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is false +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is false ; CHECK-NEXT: Predicates: @@ -148,7 +148,7 @@ define void @unsimplified_or3(i32 %n) { ; CHECK-LABEL: 'unsimplified_or3' ; CHECK-NEXT: Determining loop execution counts for: @unsimplified_or3 ; CHECK-NEXT: Loop %loop: backedge-taken count is %n -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n ; CHECK-NEXT: Predicates: @@ -172,7 +172,7 @@ define void @unsimplified_or4(i32 %n) { ; CHECK-LABEL: 'unsimplified_or4' ; CHECK-NEXT: Determining loop execution counts for: @unsimplified_or4 ; CHECK-NEXT: Loop %loop: backedge-taken count is %n -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n ; CHECK-NEXT: Predicates: @@ -196,7 +196,7 @@ define void @reversed_and1(i32 %n) { ; CHECK-LABEL: 'reversed_and1' ; CHECK-NEXT: Determining loop execution counts for: @reversed_and1 ; CHECK-NEXT: Loop %loop: backedge-taken count is %n -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n ; CHECK-NEXT: Predicates: @@ -220,7 +220,7 @@ define void @reversed_and2(i32 %n) { ; CHECK-LABEL: 'reversed_and2' ; CHECK-NEXT: Determining loop execution counts for: @reversed_and2 ; CHECK-NEXT: Loop %loop: backedge-taken count is %n -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n ; CHECK-NEXT: Predicates: @@ -288,7 +288,7 @@ define void @reversed_or1(i32 %n) { ; CHECK-LABEL: 'reversed_or1' ; CHECK-NEXT: Determining loop execution counts for: @reversed_or1 ; CHECK-NEXT: Loop %loop: backedge-taken count is false -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is false +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is false ; CHECK-NEXT: Predicates: @@ -312,7 +312,7 @@ define void @reversed_or2(i32 %n) { ; CHECK-LABEL: 'reversed_or2' ; CHECK-NEXT: Determining loop execution counts for: @reversed_or2 ; CHECK-NEXT: Loop %loop: backedge-taken count is false -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is false +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is false ; CHECK-NEXT: Predicates: @@ -336,7 +336,7 @@ define void @reversed_or3(i32 %n) { ; CHECK-LABEL: 'reversed_or3' ; CHECK-NEXT: Determining loop execution counts for: @reversed_or3 ; CHECK-NEXT: Loop %loop: backedge-taken count is %n -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n ; CHECK-NEXT: Predicates: @@ -360,7 +360,7 @@ define void @reversed_or4(i32 %n) { ; CHECK-LABEL: 'reversed_or4' ; CHECK-NEXT: Determining loop execution counts for: @reversed_or4 ; CHECK-NEXT: Loop %loop: backedge-taken count is %n -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-implied-addrec.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-implied-addrec.ll index e04563207d78..e67c7dff14c2 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-implied-addrec.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-implied-addrec.ll @@ -13,7 +13,7 @@ define void @nw_implies_nuw(i16 %n) mustprogress { ; CHECK-LABEL: 'nw_implies_nuw' ; CHECK-NEXT: Determining loop execution counts for: @nw_implies_nuw ; CHECK-NEXT: Loop %for.body: backedge-taken count is %n -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is %n ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is %n ; CHECK-NEXT: Predicates: @@ -291,7 +291,7 @@ define void @rhs_narrow_range(i16 %n.raw) { ; CHECK-LABEL: 'rhs_narrow_range' ; CHECK-NEXT: Determining loop execution counts for: @rhs_narrow_range ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (1 umax (2 * (zext i7 (trunc i16 (%n.raw /u 2) to i7) to i16)))) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 253 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 253 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (1 umax (2 * (zext i7 (trunc i16 (%n.raw /u 2) to i7) to i16)))) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (1 umax (2 * (zext i7 (trunc i16 (%n.raw /u 2) to i7) to i16)))) ; CHECK-NEXT: Predicates: @@ -341,7 +341,7 @@ define void @ult_constant_rhs(i16 %n.raw, i8 %start) { ; CHECK-LABEL: 'ult_constant_rhs' ; CHECK-NEXT: Determining loop execution counts for: @ult_constant_rhs ; CHECK-NEXT: Loop %for.body: backedge-taken count is (255 + (-1 * (zext i8 (1 + %start) to i16))) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 255 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 255 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (255 + (-1 * (zext i8 (1 + %start) to i16))) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (255 + (-1 * (zext i8 (1 + %start) to i16))) ; CHECK-NEXT: Predicates: @@ -366,7 +366,7 @@ define void @ult_constant_rhs_stride2(i16 %n.raw, i8 %start) { ; CHECK-LABEL: 'ult_constant_rhs_stride2' ; CHECK-NEXT: Determining loop execution counts for: @ult_constant_rhs_stride2 ; CHECK-NEXT: Loop %for.body: backedge-taken count is ((1 + (-1 * (zext i8 (2 + %start) to i16)) + (254 umax (zext i8 (2 + %start) to i16))) /u 2) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 127 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 127 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((1 + (-1 * (zext i8 (2 + %start) to i16)) + (254 umax (zext i8 (2 + %start) to i16))) /u 2) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((1 + (-1 * (zext i8 (2 + %start) to i16)) + (254 umax (zext i8 (2 + %start) to i16))) /u 2) ; CHECK-NEXT: Predicates: @@ -416,7 +416,7 @@ define void @ult_restricted_rhs(i16 %n.raw) { ; CHECK-LABEL: 'ult_restricted_rhs' ; CHECK-NEXT: Determining loop execution counts for: @ult_restricted_rhs ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (1 umax (zext i8 (trunc i16 %n.raw to i8) to i16))) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 254 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 254 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (1 umax (zext i8 (trunc i16 %n.raw to i8) to i16))) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (1 umax (zext i8 (trunc i16 %n.raw to i8) to i16))) ; CHECK-NEXT: Predicates: @@ -441,7 +441,7 @@ define void @ult_guarded_rhs(i16 %n) {; ; CHECK-LABEL: 'ult_guarded_rhs' ; CHECK-NEXT: Determining loop execution counts for: @ult_guarded_rhs ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (1 umax %n)) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (1 umax %n)) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (1 umax %n)) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-minmax.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-minmax.ll index 7d4876baa9e5..f9d490d5ee38 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-minmax.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-minmax.ll @@ -15,7 +15,7 @@ define void @nomulitply(i32 noundef %a, i32 noundef %b) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-2147483648) S: [1,-2147483648) Exits: (%a umin %b) LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @nomulitply ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (%a umin %b)) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (%a umin %b)) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (%a umin %b)) ; CHECK-NEXT: Predicates: @@ -61,7 +61,7 @@ define void @umin(i32 noundef %a, i32 noundef %b) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-2147483648) S: [1,-2147483648) Exits: ((2 * %a) umin (4 * %b)) LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @umin ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + ((2 * %a) umin (4 * %b))) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + ((2 * %a) umin (4 * %b))) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + ((2 * %a) umin (4 * %b))) ; CHECK-NEXT: Predicates: @@ -111,7 +111,7 @@ define void @umax(i32 noundef %a, i32 noundef %b) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-1) S: [1,-1) Exits: ((2 * %a) umax (4 * %b)) LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @umax ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + ((2 * %a) umax (4 * %b))) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -3 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -3 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + ((2 * %a) umax (4 * %b))) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + ((2 * %a) umax (4 * %b))) ; CHECK-NEXT: Predicates: @@ -161,7 +161,7 @@ define void @smin(i32 noundef %a, i32 noundef %b) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-2147483648) S: [1,-2147483648) Exits: ((2 * %a) smin (4 * %b)) LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @smin ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + ((2 * %a) smin (4 * %b))) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + ((2 * %a) smin (4 * %b))) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + ((2 * %a) smin (4 * %b))) ; CHECK-NEXT: Predicates: @@ -210,7 +210,7 @@ define void @smax(i32 noundef %a, i32 noundef %b) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-1) S: [1,-1) Exits: ((2 * %a) smax (4 * %b)) LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @smax ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + ((2 * %a) smax (4 * %b))) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -3 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -3 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + ((2 * %a) smax (4 * %b))) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + ((2 * %a) smax (4 * %b))) ; CHECK-NEXT: Predicates: @@ -259,7 +259,7 @@ define void @umin_seq2(i32 %n, i32 %m) { ; CHECK-NEXT: --> (%cond_p0 umin_seq %cond_p1) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @umin_seq2 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 + (1 umax (2 * %n))) umin_seq (-1 + (1 umax (16 * %m)))) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -17 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -17 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 + (1 umax (2 * %n))) umin_seq (-1 + (1 umax (16 * %m)))) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 + (1 umax (2 * %n))) umin_seq (-1 + (1 umax (16 * %m)))) ; CHECK-NEXT: Predicates: @@ -297,7 +297,7 @@ define void @umin-3and6(i32 noundef %a, i32 noundef %b) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-2147483648) S: [1,-2147483648) Exits: ((3 * %a) umin (6 * %b)) LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @umin-3and6 ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + ((3 * %a) umin (6 * %b))) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + ((3 * %a) umin (6 * %b))) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + ((3 * %a) umin (6 * %b))) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-negative-stride.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-negative-stride.ll index f43dfe35d39b..3cd0904fc712 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-negative-stride.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-negative-stride.ll @@ -59,7 +59,7 @@ define void @ult_infinite_ub() mustprogress { ; CHECK-LABEL: 'ult_infinite_ub' ; CHECK-NEXT: Determining loop execution counts for: @ult_infinite_ub ; CHECK-NEXT: Loop %for.body: backedge-taken count is 1 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 1 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 1 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 1 ; CHECK-NEXT: Predicates: @@ -84,7 +84,7 @@ define void @ult_129_not_taken() { ; CHECK-LABEL: 'ult_129_not_taken' ; CHECK-NEXT: Determining loop execution counts for: @ult_129_not_taken ; CHECK-NEXT: Loop %for.body: backedge-taken count is 0 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 0 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 0 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 0 ; CHECK-NEXT: Predicates: @@ -107,7 +107,7 @@ define void @ult_129_unknown_start(i8 %start) mustprogress { ; CHECK-LABEL: 'ult_129_unknown_start' ; CHECK-NEXT: Determining loop execution counts for: @ult_129_unknown_start ; CHECK-NEXT: Loop %for.body: backedge-taken count is 0 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 0 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 0 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 0 ; CHECK-NEXT: Predicates: @@ -159,7 +159,7 @@ define void @ult_ub1() { ; CHECK-LABEL: 'ult_ub1' ; CHECK-NEXT: Determining loop execution counts for: @ult_ub1 ; CHECK-NEXT: Loop %for.body: backedge-taken count is 2 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 2 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 2 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 2 ; CHECK-NEXT: Predicates: @@ -184,7 +184,7 @@ define void @ult_ub2() { ; CHECK-LABEL: 'ult_ub2' ; CHECK-NEXT: Determining loop execution counts for: @ult_ub2 ; CHECK-NEXT: Loop %for.body: backedge-taken count is 0 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 0 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 0 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 0 ; CHECK-NEXT: Predicates: @@ -209,7 +209,7 @@ define void @ult_129_preinc() { ; CHECK-LABEL: 'ult_129_preinc' ; CHECK-NEXT: Determining loop execution counts for: @ult_129_preinc ; CHECK-NEXT: Loop %for.body: backedge-taken count is 1 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 1 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 1 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 1 ; CHECK-NEXT: Predicates: @@ -232,7 +232,7 @@ define void @ult_preinc(i8 %step) { ; CHECK-LABEL: 'ult_preinc' ; CHECK-NEXT: Determining loop execution counts for: @ult_preinc ; CHECK-NEXT: Loop %for.body: backedge-taken count is 1 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 1 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 1 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 1 ; CHECK-NEXT: Predicates: @@ -309,7 +309,7 @@ define void @slt_wrap() { ; CHECK-LABEL: 'slt_wrap' ; CHECK-NEXT: Determining loop execution counts for: @slt_wrap ; CHECK-NEXT: Loop %for.body: backedge-taken count is 63 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 63 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 63 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 63 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 63 ; CHECK-NEXT: Predicates: @@ -357,7 +357,7 @@ define void @slt_infinite_ub() mustprogress { ; CHECK-LABEL: 'slt_infinite_ub' ; CHECK-NEXT: Determining loop execution counts for: @slt_infinite_ub ; CHECK-NEXT: Loop %for.body: backedge-taken count is 0 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 0 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 0 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 0 ; CHECK-NEXT: Predicates: @@ -382,7 +382,7 @@ define void @slt_129_not_taken() { ; CHECK-LABEL: 'slt_129_not_taken' ; CHECK-NEXT: Determining loop execution counts for: @slt_129_not_taken ; CHECK-NEXT: Loop %for.body: backedge-taken count is 0 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 0 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 0 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 0 ; CHECK-NEXT: Predicates: @@ -429,7 +429,7 @@ define void @slt_129_unknown_start(i8 %start) mustprogress { ; CHECK-LABEL: 'slt_129_unknown_start' ; CHECK-NEXT: Determining loop execution counts for: @slt_129_unknown_start ; CHECK-NEXT: Loop %for.body: backedge-taken count is (((127 + (-1 * (1 umin (127 + (-1 * %start) + (0 smax (-127 + %start))))) + (-1 * %start) + (0 smax (-127 + %start))) /u -127) + (1 umin (127 + (-1 * %start) + (0 smax (-127 + %start))))) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 2 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (((127 + (-1 * (1 umin (127 + (-1 * %start) + (0 smax (-127 + %start))))) + (-1 * %start) + (0 smax (-127 + %start))) /u -127) + (1 umin (127 + (-1 * %start) + (0 smax (-127 + %start))))) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (((127 + (-1 * (1 umin (127 + (-1 * %start) + (0 smax (-127 + %start))))) + (-1 * %start) + (0 smax (-127 + %start))) /u -127) + (1 umin (127 + (-1 * %start) + (0 smax (-127 + %start))))) ; CHECK-NEXT: Predicates: @@ -455,7 +455,7 @@ define void @slt_ub1() { ; CHECK-LABEL: 'slt_ub1' ; CHECK-NEXT: Determining loop execution counts for: @slt_ub1 ; CHECK-NEXT: Loop %for.body: backedge-taken count is false -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is false +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is false ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is false ; CHECK-NEXT: Predicates: @@ -480,7 +480,7 @@ define void @slt_ub2() { ; CHECK-LABEL: 'slt_ub2' ; CHECK-NEXT: Determining loop execution counts for: @slt_ub2 ; CHECK-NEXT: Loop %for.body: backedge-taken count is false -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is false +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is false ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is false ; CHECK-NEXT: Predicates: @@ -505,7 +505,7 @@ define void @slt_129_preinc() { ; CHECK-LABEL: 'slt_129_preinc' ; CHECK-NEXT: Determining loop execution counts for: @slt_129_preinc ; CHECK-NEXT: Loop %for.body: backedge-taken count is 1 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 1 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 1 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 1 ; CHECK-NEXT: Predicates: @@ -528,7 +528,7 @@ define void @slt_preinc(i8 %step) { ; CHECK-LABEL: 'slt_preinc' ; CHECK-NEXT: Determining loop execution counts for: @slt_preinc ; CHECK-NEXT: Loop %for.body: backedge-taken count is 1 -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 1 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 1 ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 1 ; CHECK-NEXT: Predicates: @@ -602,13 +602,13 @@ define void @step_is_neg_addrec_slt_8(i64 %n) { ; CHECK-LABEL: 'step_is_neg_addrec_slt_8' ; CHECK-NEXT: Determining loop execution counts for: @step_is_neg_addrec_slt_8 ; CHECK-NEXT: Loop %inner: backedge-taken count is (7 /u {0,+,-1}<%outer.header>) -; CHECK-NEXT: Loop %inner: constant max backedge-taken count is 8 +; CHECK-NEXT: Loop %inner: constant max backedge-taken count is i32 8 ; CHECK-NEXT: Loop %inner: symbolic max backedge-taken count is (7 /u {0,+,-1}<%outer.header>) ; CHECK-NEXT: Loop %inner: Predicated backedge-taken count is (7 /u {0,+,-1}<%outer.header>) ; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %inner: Trip multiple is 1 ; CHECK-NEXT: Loop %outer.header: backedge-taken count is 0 -; CHECK-NEXT: Loop %outer.header: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %outer.header: constant max backedge-taken count is i64 0 ; CHECK-NEXT: Loop %outer.header: symbolic max backedge-taken count is 0 ; CHECK-NEXT: Loop %outer.header: Predicated backedge-taken count is 0 ; CHECK-NEXT: Predicates: @@ -644,13 +644,13 @@ define void @step_is_neg_addrec_slt_var(i32 %n) { ; CHECK-LABEL: 'step_is_neg_addrec_slt_var' ; CHECK-NEXT: Determining loop execution counts for: @step_is_neg_addrec_slt_var ; CHECK-NEXT: Loop %inner: backedge-taken count is ({0,+,1}<%outer.header> + ({0,+,-1}<%outer.header> smax %n)) -; CHECK-NEXT: Loop %inner: constant max backedge-taken count is 2147483647 +; CHECK-NEXT: Loop %inner: constant max backedge-taken count is i32 2147483647 ; CHECK-NEXT: Loop %inner: symbolic max backedge-taken count is ({0,+,1}<%outer.header> + ({0,+,-1}<%outer.header> smax %n)) ; CHECK-NEXT: Loop %inner: Predicated backedge-taken count is ({0,+,1}<%outer.header> + ({0,+,-1}<%outer.header> smax %n)) ; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %inner: Trip multiple is 1 ; CHECK-NEXT: Loop %outer.header: backedge-taken count is 0 -; CHECK-NEXT: Loop %outer.header: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %outer.header: constant max backedge-taken count is i64 0 ; CHECK-NEXT: Loop %outer.header: symbolic max backedge-taken count is 0 ; CHECK-NEXT: Loop %outer.header: Predicated backedge-taken count is 0 ; CHECK-NEXT: Predicates: @@ -686,13 +686,13 @@ define void @step_is_neg_addrec_unknown_start(i32 %n) { ; CHECK-LABEL: 'step_is_neg_addrec_unknown_start' ; CHECK-NEXT: Determining loop execution counts for: @step_is_neg_addrec_unknown_start ; CHECK-NEXT: Loop %inner: backedge-taken count is ({(-1 * %n),+,1}<%outer.header> + (8 smax {%n,+,-1}<%outer.header>)) -; CHECK-NEXT: Loop %inner: constant max backedge-taken count is -2147483640 +; CHECK-NEXT: Loop %inner: constant max backedge-taken count is i32 -2147483640 ; CHECK-NEXT: Loop %inner: symbolic max backedge-taken count is ({(-1 * %n),+,1}<%outer.header> + (8 smax {%n,+,-1}<%outer.header>)) ; CHECK-NEXT: Loop %inner: Predicated backedge-taken count is ({(-1 * %n),+,1}<%outer.header> + (8 smax {%n,+,-1}<%outer.header>)) ; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %inner: Trip multiple is 1 ; CHECK-NEXT: Loop %outer.header: backedge-taken count is 0 -; CHECK-NEXT: Loop %outer.header: constant max backedge-taken count is 0 +; CHECK-NEXT: Loop %outer.header: constant max backedge-taken count is i64 0 ; CHECK-NEXT: Loop %outer.header: symbolic max backedge-taken count is 0 ; CHECK-NEXT: Loop %outer.header: Predicated backedge-taken count is 0 ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-pow2.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-pow2.ll index 5b754489321d..8feb7a87b34b 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-pow2.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-pow2.ll @@ -5,7 +5,7 @@ define void @test1(i32 %n) { ; CHECK-LABEL: 'test1' ; CHECK-NEXT: Determining loop execution counts for: @test1 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-32 + (96 * %n)) /u 32) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 134217727 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 134217727 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-32 + (96 * %n)) /u 32) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-32 + (96 * %n)) /u 32) ; CHECK-NEXT: Predicates: @@ -29,7 +29,7 @@ define i32 @test2(i32 %n) { ; CHECK-LABEL: 'test2' ; CHECK-NEXT: Determining loop execution counts for: @test2 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-32 + (32 * (%n /u 32))) /u 32) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 134217727 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 134217727 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-32 + (32 * (%n /u 32))) /u 32) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-32 + (32 * (%n /u 32))) /u 32) ; CHECK-NEXT: Predicates: @@ -52,7 +52,7 @@ define void @test3(i32 %n) { ; CHECK-LABEL: 'test3' ; CHECK-NEXT: Determining loop execution counts for: @test3 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-32 + (32 * %n)) /u 32) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 134217727 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 134217727 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-32 + (32 * %n)) /u 32) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-32 + (32 * %n)) /u 32) ; CHECK-NEXT: Predicates: @@ -75,7 +75,7 @@ define void @test4(i32 %n) { ; CHECK-LABEL: 'test4' ; CHECK-NEXT: Determining loop execution counts for: @test4 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (-1431655764 * %n)) /u 4) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 1073741823 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 1073741823 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (-1431655764 * %n)) /u 4) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (-1431655764 * %n)) /u 4) ; CHECK-NEXT: Predicates: @@ -98,7 +98,7 @@ define void @test5(i32 %n) { ; CHECK-LABEL: 'test5' ; CHECK-NEXT: Determining loop execution counts for: @test5 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * %n)) /u 4) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 1073741823 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 1073741823 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * %n)) /u 4) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * %n)) /u 4) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-switch.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-switch.ll index d5ab589df924..b315deb5a87c 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-switch.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-switch.ll @@ -8,7 +8,7 @@ define void @test1() nounwind { ; CHECK-LABEL: 'test1' ; CHECK-NEXT: Determining loop execution counts for: @test1 ; CHECK-NEXT: Loop %for.cond: backedge-taken count is 2 -; CHECK-NEXT: Loop %for.cond: constant max backedge-taken count is 2 +; CHECK-NEXT: Loop %for.cond: constant max backedge-taken count is i32 2 ; CHECK-NEXT: Loop %for.cond: symbolic max backedge-taken count is 2 ; CHECK-NEXT: Loop %for.cond: Predicated backedge-taken count is 2 ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-unknown-stride.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-unknown-stride.ll index 63fb99d29a9b..ef7084130335 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-unknown-stride.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-unknown-stride.ll @@ -15,7 +15,7 @@ define void @foo1(ptr nocapture %A, i32 %n, i32 %s) mustprogress { ; CHECK-LABEL: 'foo1' ; CHECK-NEXT: Determining loop execution counts for: @foo1 ; CHECK-NEXT: Loop %for.body: backedge-taken count is ((-1 + (%n smax %s)) /u %s) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((-1 + (%n smax %s)) /u %s) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-1 + (%n smax %s)) /u %s) ; CHECK-NEXT: Predicates: @@ -50,7 +50,7 @@ define void @foo2(ptr nocapture %A, i32 %n, i32 %s) mustprogress { ; CHECK-LABEL: 'foo2' ; CHECK-NEXT: Determining loop execution counts for: @foo2 ; CHECK-NEXT: Loop %for.body: backedge-taken count is ((((-1 * (1 umin ((-1 * %s) + (%n smax %s)))) + (-1 * %s) + (%n smax %s)) /u (1 umax %s)) + (1 umin ((-1 * %s) + (%n smax %s)))) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((((-1 * (1 umin ((-1 * %s) + (%n smax %s)))) + (-1 * %s) + (%n smax %s)) /u (1 umax %s)) + (1 umin ((-1 * %s) + (%n smax %s)))) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((((-1 * (1 umin ((-1 * %s) + (%n smax %s)))) + (-1 * %s) + (%n smax %s)) /u (1 umax %s)) + (1 umin ((-1 * %s) + (%n smax %s)))) ; CHECK-NEXT: Predicates: @@ -109,7 +109,7 @@ define void @foo4(ptr nocapture %A, i32 %n, i32 %s) { ; CHECK-LABEL: 'foo4' ; CHECK-NEXT: Determining loop execution counts for: @foo4 ; CHECK-NEXT: Loop %for.body: backedge-taken count is ((((-1 * (1 umin ((-1 * %s) + (%n smax %s)))) + (-1 * %s) + (%n smax %s)) /u (1 umax %s)) + (1 umin ((-1 * %s) + (%n smax %s)))) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((((-1 * (1 umin ((-1 * %s) + (%n smax %s)))) + (-1 * %s) + (%n smax %s)) /u (1 umax %s)) + (1 umin ((-1 * %s) + (%n smax %s)))) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((((-1 * (1 umin ((-1 * %s) + (%n smax %s)))) + (-1 * %s) + (%n smax %s)) /u (1 umax %s)) + (1 umin ((-1 * %s) + (%n smax %s)))) ; CHECK-NEXT: Predicates: @@ -142,7 +142,7 @@ define void @foo5(ptr nocapture %A, i32 %n, i32 %s, i32 %start) mustprogress { ; CHECK-LABEL: 'foo5' ; CHECK-NEXT: Determining loop execution counts for: @foo5 ; CHECK-NEXT: Loop %for.body: backedge-taken count is ((((-1 * (1 umin ((-1 * %start) + (%n smax %start)))) + (-1 * %start) + (%n smax %start)) /u (1 umax %s)) + (1 umin ((-1 * %start) + (%n smax %start)))) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((((-1 * (1 umin ((-1 * %start) + (%n smax %start)))) + (-1 * %start) + (%n smax %start)) /u (1 umax %s)) + (1 umin ((-1 * %start) + (%n smax %start)))) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((((-1 * (1 umin ((-1 * %start) + (%n smax %start)))) + (-1 * %start) + (%n smax %start)) /u (1 umax %s)) + (1 umin ((-1 * %start) + (%n smax %start)))) ; CHECK-NEXT: Predicates: @@ -229,7 +229,7 @@ define void @zero_stride_symbolic(ptr nocapture %A, i32 %n, i32 %zero) { ; CHECK-LABEL: 'zero_stride_symbolic' ; CHECK-NEXT: Determining loop execution counts for: @zero_stride_symbolic ; CHECK-NEXT: Loop %for.body: backedge-taken count is ((((-1 * (1 umin ((-1 * %zero) + (%n smax %zero)))) + (-1 * %zero) + (%n smax %zero)) /u (1 umax %zero)) + (1 umin ((-1 * %zero) + (%n smax %zero)))) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((((-1 * (1 umin ((-1 * %zero) + (%n smax %zero)))) + (-1 * %zero) + (%n smax %zero)) /u (1 umax %zero)) + (1 umin ((-1 * %zero) + (%n smax %zero)))) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((((-1 * (1 umin ((-1 * %zero) + (%n smax %zero)))) + (-1 * %zero) + (%n smax %zero)) /u (1 umax %zero)) + (1 umin ((-1 * %zero) + (%n smax %zero)))) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count.ll b/llvm/test/Analysis/ScalarEvolution/trip-count.ll index 36b42c62dd39..9f34c43b907c 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count.ll @@ -10,7 +10,7 @@ define void @PR1101(i32 %N) { ; CHECK-LABEL: 'PR1101' ; CHECK-NEXT: Determining loop execution counts for: @PR1101 ; CHECK-NEXT: Loop %bb3: backedge-taken count is 10000 -; CHECK-NEXT: Loop %bb3: constant max backedge-taken count is 10000 +; CHECK-NEXT: Loop %bb3: constant max backedge-taken count is i32 10000 ; CHECK-NEXT: Loop %bb3: symbolic max backedge-taken count is 10000 ; CHECK-NEXT: Loop %bb3: Predicated backedge-taken count is 10000 ; CHECK-NEXT: Predicates: @@ -41,7 +41,7 @@ define i32 @PR22795() { ; CHECK-LABEL: 'PR22795' ; CHECK-NEXT: Determining loop execution counts for: @PR22795 ; CHECK-NEXT: Loop %preheader: backedge-taken count is 7 -; CHECK-NEXT: Loop %preheader: constant max backedge-taken count is 7 +; CHECK-NEXT: Loop %preheader: constant max backedge-taken count is i64 7 ; CHECK-NEXT: Loop %preheader: symbolic max backedge-taken count is 7 ; CHECK-NEXT: Loop %preheader: Predicated backedge-taken count is 7 ; CHECK-NEXT: Predicates: @@ -104,7 +104,7 @@ define void @pr28012(i32 %n) { ; CHECK-LABEL: 'pr28012' ; CHECK-NEXT: Determining loop execution counts for: @pr28012 ; CHECK-NEXT: Loop %loop: backedge-taken count is -1431655751 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1431655751 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1431655751 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is -1431655751 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is -1431655751 ; CHECK-NEXT: Predicates: @@ -128,7 +128,7 @@ define void @non_zero_from_loop_guard(i16 %n) { ; CHECK-LABEL: 'non_zero_from_loop_guard' ; CHECK-NEXT: Determining loop execution counts for: @non_zero_from_loop_guard ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (%n /u 2)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 32766 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i16 32766 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (%n /u 2)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (%n /u 2)) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count10.ll b/llvm/test/Analysis/ScalarEvolution/trip-count10.ll index 2b2439c0ac46..acce7a81e0ab 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count10.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count10.ll @@ -28,7 +28,7 @@ define void @b(i64 %n) nounwind { ; CHECK-LABEL: 'b' ; CHECK-NEXT: Determining loop execution counts for: @b ; CHECK-NEXT: Loop %loop: backedge-taken count is false -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is false +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is false ; CHECK-NEXT: Predicates: @@ -51,7 +51,7 @@ define void @c(i64 %n) nounwind { ; CHECK-LABEL: 'c' ; CHECK-NEXT: Determining loop execution counts for: @c ; CHECK-NEXT: Loop %loop: backedge-taken count is false -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is false +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is false ; CHECK-NEXT: Predicates: @@ -132,7 +132,7 @@ define void @constant_phi_operands() nounwind { ; CHECK-LABEL: 'constant_phi_operands' ; CHECK-NEXT: Determining loop execution counts for: @constant_phi_operands ; CHECK-NEXT: Loop %loop: backedge-taken count is 1 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 1 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 1 ; CHECK-NEXT: Predicates: @@ -158,7 +158,7 @@ define void @exit_orcond_nsw(ptr %a) nounwind { ; CHECK-LABEL: 'exit_orcond_nsw' ; CHECK-NEXT: Determining loop execution counts for: @exit_orcond_nsw ; CHECK-NEXT: Loop %for.body.i: Unpredictable backedge-taken count. -; CHECK-NEXT: Loop %for.body.i: constant max backedge-taken count is 1 +; CHECK-NEXT: Loop %for.body.i: constant max backedge-taken count is i32 1 ; CHECK-NEXT: Loop %for.body.i: symbolic max backedge-taken count is 1 ; CHECK-NEXT: Loop %for.body.i: Unpredictable predicated backedge-taken count. ; diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count12.ll b/llvm/test/Analysis/ScalarEvolution/trip-count12.ll index 9eae3cc9b2eb..0eb903d64b2a 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count12.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count12.ll @@ -7,7 +7,7 @@ define zeroext i16 @test(ptr nocapture %p, i32 %len) nounwind readonly { ; CHECK-LABEL: 'test' ; CHECK-NEXT: Determining loop execution counts for: @test ; CHECK-NEXT: Loop %for.body: backedge-taken count is ((-2 + %len) /u 2) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 1073741823 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 1073741823 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((-2 + %len) /u 2) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-2 + %len) /u 2) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count13.ll b/llvm/test/Analysis/ScalarEvolution/trip-count13.ll index 9dd615a04f82..ef8640e65968 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count13.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count13.ll @@ -7,7 +7,7 @@ define void @u_0(i8 %rhs) { ; CHECK-LABEL: 'u_0' ; CHECK-NEXT: Determining loop execution counts for: @u_0 ; CHECK-NEXT: Loop %loop: backedge-taken count is (-100 + (-1 * %rhs) + ((100 + %rhs) umax %rhs)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -100, actual taken count either this or zero. +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 -100, actual taken count either this or zero. ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-100 + (-1 * %rhs) + ((100 + %rhs) umax %rhs)), actual taken count either this or zero. ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-100 + (-1 * %rhs) + ((100 + %rhs) umax %rhs)) ; CHECK-NEXT: Predicates: @@ -33,7 +33,7 @@ define void @u_1(i8 %start) { ; CHECK-LABEL: 'u_1' ; CHECK-NEXT: Determining loop execution counts for: @u_1 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 * %start) + ((-100 + %start) umax %start)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -100, actual taken count either this or zero. +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 -100, actual taken count either this or zero. ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 * %start) + ((-100 + %start) umax %start)), actual taken count either this or zero. ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 * %start) + ((-100 + %start) umax %start)) ; CHECK-NEXT: Predicates: @@ -60,7 +60,7 @@ define void @s_0(i8 %rhs) { ; CHECK-LABEL: 's_0' ; CHECK-NEXT: Determining loop execution counts for: @s_0 ; CHECK-NEXT: Loop %loop: backedge-taken count is (-100 + (-1 * %rhs) + ((100 + %rhs) smax %rhs)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -100, actual taken count either this or zero. +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 -100, actual taken count either this or zero. ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-100 + (-1 * %rhs) + ((100 + %rhs) smax %rhs)), actual taken count either this or zero. ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-100 + (-1 * %rhs) + ((100 + %rhs) smax %rhs)) ; CHECK-NEXT: Predicates: @@ -87,7 +87,7 @@ define void @s_1(i8 %start) { ; CHECK-LABEL: 's_1' ; CHECK-NEXT: Determining loop execution counts for: @s_1 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 * %start) + ((-100 + %start) smax %start)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -100, actual taken count either this or zero. +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 -100, actual taken count either this or zero. ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 * %start) + ((-100 + %start) smax %start)), actual taken count either this or zero. ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 * %start) + ((-100 + %start) smax %start)) ; CHECK-NEXT: Predicates: @@ -114,7 +114,7 @@ define void @s_2(i8 %start) { ; CHECK-LABEL: 's_2' ; CHECK-NEXT: Determining loop execution counts for: @s_2 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 * ((-100 + %start) smin %start)) + %start) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 * ((-100 + %start) smin %start)) + %start) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 * ((-100 + %start) smin %start)) + %start) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count14.ll b/llvm/test/Analysis/ScalarEvolution/trip-count14.ll index 7c551afc79e4..6561e080bee1 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count14.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count14.ll @@ -5,7 +5,7 @@ define void @s32_max1(i32 %n, ptr %p) { ; CHECK-LABEL: 's32_max1' ; CHECK-NEXT: Determining loop execution counts for: @s32_max1 ; CHECK-NEXT: Loop %do.body: backedge-taken count is ((-1 * %n) + ((1 + %n) smax %n)) -; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is 1, actual taken count either this or zero. +; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is i32 1, actual taken count either this or zero. ; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is ((-1 * %n) + ((1 + %n) smax %n)), actual taken count either this or zero. ; CHECK-NEXT: Loop %do.body: Predicated backedge-taken count is ((-1 * %n) + ((1 + %n) smax %n)) ; CHECK-NEXT: Predicates: @@ -31,7 +31,7 @@ define void @s32_max2(i32 %n, ptr %p) { ; CHECK-LABEL: 's32_max2' ; CHECK-NEXT: Determining loop execution counts for: @s32_max2 ; CHECK-NEXT: Loop %do.body: backedge-taken count is ((-1 * %n) + ((2 + %n) smax %n)) -; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is 2, actual taken count either this or zero. +; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is i32 2, actual taken count either this or zero. ; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is ((-1 * %n) + ((2 + %n) smax %n)), actual taken count either this or zero. ; CHECK-NEXT: Loop %do.body: Predicated backedge-taken count is ((-1 * %n) + ((2 + %n) smax %n)) ; CHECK-NEXT: Predicates: @@ -57,7 +57,7 @@ define void @s32_maxx(i32 %n, i32 %x, ptr %p) { ; CHECK-LABEL: 's32_maxx' ; CHECK-NEXT: Determining loop execution counts for: @s32_maxx ; CHECK-NEXT: Loop %do.body: backedge-taken count is ((-1 * %n) + ((%n + %x) smax %n)) -; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is ((-1 * %n) + ((%n + %x) smax %n)) ; CHECK-NEXT: Loop %do.body: Predicated backedge-taken count is ((-1 * %n) + ((%n + %x) smax %n)) ; CHECK-NEXT: Predicates: @@ -85,7 +85,7 @@ define void @s32_max2_unpredictable_exit(i32 %n, i32 %x, ptr %p) { ; CHECK-NEXT: Loop %do.body: backedge-taken count is (((-1 * %n) + ((2 + %n) smax %n)) umin ((-1 * %n) + %x)) ; CHECK-NEXT: exit count for do.body: ((-1 * %n) + %x) ; CHECK-NEXT: exit count for if.end: ((-1 * %n) + ((2 + %n) smax %n)) -; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is 2 +; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is i32 2 ; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is (((-1 * %n) + ((2 + %n) smax %n)) umin ((-1 * %n) + %x)) ; CHECK-NEXT: symbolic max exit count for do.body: ((-1 * %n) + %x) ; CHECK-NEXT: symbolic max exit count for if.end: ((-1 * %n) + ((2 + %n) smax %n)) @@ -117,7 +117,7 @@ define void @u32_max1(i32 %n, ptr %p) { ; CHECK-LABEL: 'u32_max1' ; CHECK-NEXT: Determining loop execution counts for: @u32_max1 ; CHECK-NEXT: Loop %do.body: backedge-taken count is ((-1 * %n) + ((1 + %n) umax %n)) -; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is 1, actual taken count either this or zero. +; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is i32 1, actual taken count either this or zero. ; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is ((-1 * %n) + ((1 + %n) umax %n)), actual taken count either this or zero. ; CHECK-NEXT: Loop %do.body: Predicated backedge-taken count is ((-1 * %n) + ((1 + %n) umax %n)) ; CHECK-NEXT: Predicates: @@ -143,7 +143,7 @@ define void @u32_max2(i32 %n, ptr %p) { ; CHECK-LABEL: 'u32_max2' ; CHECK-NEXT: Determining loop execution counts for: @u32_max2 ; CHECK-NEXT: Loop %do.body: backedge-taken count is ((-1 * %n) + ((2 + %n) umax %n)) -; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is 2, actual taken count either this or zero. +; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is i32 2, actual taken count either this or zero. ; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is ((-1 * %n) + ((2 + %n) umax %n)), actual taken count either this or zero. ; CHECK-NEXT: Loop %do.body: Predicated backedge-taken count is ((-1 * %n) + ((2 + %n) umax %n)) ; CHECK-NEXT: Predicates: @@ -169,7 +169,7 @@ define void @u32_maxx(i32 %n, i32 %x, ptr %p) { ; CHECK-LABEL: 'u32_maxx' ; CHECK-NEXT: Determining loop execution counts for: @u32_maxx ; CHECK-NEXT: Loop %do.body: backedge-taken count is ((-1 * %n) + ((%n + %x) umax %n)) -; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is ((-1 * %n) + ((%n + %x) umax %n)) ; CHECK-NEXT: Loop %do.body: Predicated backedge-taken count is ((-1 * %n) + ((%n + %x) umax %n)) ; CHECK-NEXT: Predicates: @@ -197,7 +197,7 @@ define void @u32_max2_unpredictable_exit(i32 %n, i32 %x, ptr %p) { ; CHECK-NEXT: Loop %do.body: backedge-taken count is (((-1 * %n) + ((2 + %n) umax %n)) umin ((-1 * %n) + %x)) ; CHECK-NEXT: exit count for do.body: ((-1 * %n) + %x) ; CHECK-NEXT: exit count for if.end: ((-1 * %n) + ((2 + %n) umax %n)) -; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is 2 +; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is i32 2 ; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is (((-1 * %n) + ((2 + %n) umax %n)) umin ((-1 * %n) + %x)) ; CHECK-NEXT: symbolic max exit count for do.body: ((-1 * %n) + %x) ; CHECK-NEXT: symbolic max exit count for if.end: ((-1 * %n) + ((2 + %n) umax %n)) diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count15.ll b/llvm/test/Analysis/ScalarEvolution/trip-count15.ll index 2a995c635ca7..3b379510701f 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count15.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count15.ll @@ -12,7 +12,7 @@ define void @umin_unsigned_check(i64 %n) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,4099) S: [1,4099) Exits: (2 + (4096 umin %n)) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @umin_unsigned_check ; CHECK-NEXT: Loop %loop: backedge-taken count is (1 + (4096 umin %n)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4097 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4097 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (1 + (4096 umin %n)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (1 + (4096 umin %n)) ; CHECK-NEXT: Predicates: @@ -43,7 +43,7 @@ define void @umin_signed_check(i64 %n) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,4099) S: [1,4099) Exits: (2 + (4096 umin %n)) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @umin_signed_check ; CHECK-NEXT: Loop %loop: backedge-taken count is (1 + (4096 umin %n)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4097 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4097 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (1 + (4096 umin %n)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (1 + (4096 umin %n)) ; CHECK-NEXT: Predicates: @@ -74,7 +74,7 @@ define void @smin_signed_check(i64 %n) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,4099) S: [1,4099) Exits: (1 + (0 smax (1 + (4096 smin %n)))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @smin_signed_check ; CHECK-NEXT: Loop %loop: backedge-taken count is (0 smax (1 + (4096 smin %n))) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4097 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4097 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (0 smax (1 + (4096 smin %n))) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (0 smax (1 + (4096 smin %n))) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count2.ll b/llvm/test/Analysis/ScalarEvolution/trip-count2.ll index adc15a118368..247385dad6dc 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count2.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count2.ll @@ -7,7 +7,7 @@ define void @PR1101(i32 %N) { ; CHECK-LABEL: 'PR1101' ; CHECK-NEXT: Determining loop execution counts for: @PR1101 ; CHECK-NEXT: Loop %bb3: backedge-taken count is 4 -; CHECK-NEXT: Loop %bb3: constant max backedge-taken count is 4 +; CHECK-NEXT: Loop %bb3: constant max backedge-taken count is i32 4 ; CHECK-NEXT: Loop %bb3: symbolic max backedge-taken count is 4 ; CHECK-NEXT: Loop %bb3: Predicated backedge-taken count is 4 ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count3.ll b/llvm/test/Analysis/ScalarEvolution/trip-count3.ll index 232878ed3083..de5b9ecf9013 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count3.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count3.ll @@ -40,7 +40,7 @@ define void @sha_stream_bb3_2E_i(ptr %sha_info, ptr %data1, i32, ptr %buffer_add ; CHECK-LABEL: 'sha_stream_bb3_2E_i' ; CHECK-NEXT: Determining loop execution counts for: @sha_stream_bb3_2E_i ; CHECK-NEXT: Loop %bb3.i: backedge-taken count is ((63 + (-1 * (63 smin %0)) + %0) /u 64) -; CHECK-NEXT: Loop %bb3.i: constant max backedge-taken count is 33554431 +; CHECK-NEXT: Loop %bb3.i: constant max backedge-taken count is i32 33554431 ; CHECK-NEXT: Loop %bb3.i: symbolic max backedge-taken count is ((63 + (-1 * (63 smin %0)) + %0) /u 64) ; CHECK-NEXT: Loop %bb3.i: Predicated backedge-taken count is ((63 + (-1 * (63 smin %0)) + %0) /u 64) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count4.ll b/llvm/test/Analysis/ScalarEvolution/trip-count4.ll index c8e83ffd757a..c289557d6fb8 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count4.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count4.ll @@ -7,7 +7,7 @@ define void @another_count_down_signed(ptr %d, i64 %n) nounwind { ; CHECK-LABEL: 'another_count_down_signed' ; CHECK-NEXT: Determining loop execution counts for: @another_count_down_signed ; CHECK-NEXT: Loop %loop: backedge-taken count is (-11 + %n) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-11 + %n) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-11 + %n) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count6.ll b/llvm/test/Analysis/ScalarEvolution/trip-count6.ll index e0ab31b1fd10..a0bed8fdb0a1 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count6.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count6.ll @@ -9,7 +9,7 @@ define i8 @f() { ; CHECK-NEXT: Loop %bb: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for bb: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for bb2: 1 -; CHECK-NEXT: Loop %bb: constant max backedge-taken count is 1 +; CHECK-NEXT: Loop %bb: constant max backedge-taken count is i8 1 ; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is 1 ; CHECK-NEXT: symbolic max exit count for bb: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for bb2: 1 diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count7.ll b/llvm/test/Analysis/ScalarEvolution/trip-count7.ll index 3eba6b3c1740..b143b252f27a 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count7.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count7.ll @@ -64,7 +64,7 @@ define void @Doit_bb7_2E_i(ptr %x1, ptr %c, ptr %b, ptr %a, ptr %q, ptr %x1.sub, ; CHECK-LABEL: 'Doit_bb7_2E_i' ; CHECK-NEXT: Determining loop execution counts for: @Doit_bb7_2E_i ; CHECK-NEXT: Loop %bb7.i: Unpredictable backedge-taken count. -; CHECK-NEXT: Loop %bb7.i: constant max backedge-taken count is 8 +; CHECK-NEXT: Loop %bb7.i: constant max backedge-taken count is i32 8 ; CHECK-NEXT: Loop %bb7.i: symbolic max backedge-taken count is 8 ; CHECK-NEXT: Loop %bb7.i: Unpredictable predicated backedge-taken count. ; diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count8.ll b/llvm/test/Analysis/ScalarEvolution/trip-count8.ll index a0bb7a8e0632..69dec1c5d3c8 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count8.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count8.ll @@ -8,7 +8,7 @@ define i32 @foo(i32 %ecx) nounwind { ; CHECK-LABEL: 'foo' ; CHECK-NEXT: Determining loop execution counts for: @foo ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %ecx) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %ecx) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %ecx) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count9.ll b/llvm/test/Analysis/ScalarEvolution/trip-count9.ll index 5301ac5f9eb3..07fdb508be99 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count9.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count9.ll @@ -13,7 +13,7 @@ define void @foo(i4 %n) { ; CHECK-LABEL: 'foo' ; CHECK-NEXT: Determining loop execution counts for: @foo ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %n) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 6 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 6 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %n) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %n) ; CHECK-NEXT: Predicates: @@ -55,7 +55,7 @@ define void @start1(i4 %n) { ; CHECK-LABEL: 'start1' ; CHECK-NEXT: Determining loop execution counts for: @start1 ; CHECK-NEXT: Loop %loop: backedge-taken count is (-2 + (2 smax %n)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 5 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 5 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-2 + (2 smax %n)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-2 + (2 smax %n)) ; CHECK-NEXT: Predicates: @@ -97,7 +97,7 @@ define void @startx(i4 %n, i4 %x) { ; CHECK-LABEL: 'startx' ; CHECK-NEXT: Determining loop execution counts for: @startx ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax %n)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax %n)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax %n)) ; CHECK-NEXT: Predicates: @@ -139,7 +139,7 @@ define void @nsw(i4 %n) { ; CHECK-LABEL: 'nsw' ; CHECK-NEXT: Determining loop execution counts for: @nsw ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %n) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 6 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 6 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %n) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %n) ; CHECK-NEXT: Predicates: @@ -169,7 +169,7 @@ define void @nsw_step2(i4 %n) { ; CHECK-LABEL: 'nsw_step2' ; CHECK-NEXT: Determining loop execution counts for: @nsw_step2 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 + %n) /u 2) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 2 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 + %n) /u 2) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 + %n) /u 2) ; CHECK-NEXT: Predicates: @@ -191,7 +191,7 @@ define void @nsw_start1(i4 %n) { ; CHECK-LABEL: 'nsw_start1' ; CHECK-NEXT: Determining loop execution counts for: @nsw_start1 ; CHECK-NEXT: Loop %loop: backedge-taken count is (-2 + (2 smax %n)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 5 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 5 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-2 + (2 smax %n)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-2 + (2 smax %n)) ; CHECK-NEXT: Predicates: @@ -213,7 +213,7 @@ define void @nsw_start1_step2(i4 %n) { ; CHECK-LABEL: 'nsw_start1_step2' ; CHECK-NEXT: Determining loop execution counts for: @nsw_start1_step2 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-2 + (3 smax %n)) /u 2) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 2 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-2 + (3 smax %n)) /u 2) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-2 + (3 smax %n)) /u 2) ; CHECK-NEXT: Predicates: @@ -235,7 +235,7 @@ define void @nsw_startx(i4 %n, i4 %x) { ; CHECK-LABEL: 'nsw_startx' ; CHECK-NEXT: Determining loop execution counts for: @nsw_startx ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax %n)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax %n)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax %n)) ; CHECK-NEXT: Predicates: @@ -257,7 +257,7 @@ define void @nsw_startx_step2(i4 %n, i4 %x) { ; CHECK-LABEL: 'nsw_startx_step2' ; CHECK-NEXT: Determining loop execution counts for: @nsw_startx_step2 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax %n)) /u 2) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 7 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 7 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax %n)) /u 2) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax %n)) /u 2) ; CHECK-NEXT: Predicates: @@ -279,7 +279,7 @@ define void @even(i4 %n) { ; CHECK-LABEL: 'even' ; CHECK-NEXT: Determining loop execution counts for: @even ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (2 * %n)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 5 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 5 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (2 * %n)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (2 * %n)) ; CHECK-NEXT: Predicates: @@ -302,7 +302,7 @@ define void @even_step2(i4 %n) { ; CHECK-LABEL: 'even_step2' ; CHECK-NEXT: Determining loop execution counts for: @even_step2 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 + (2 * %n)) /u 2) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 2 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 + (2 * %n)) /u 2) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 + (2 * %n)) /u 2) ; CHECK-NEXT: Predicates: @@ -325,7 +325,7 @@ define void @even_start1(i4 %n) { ; CHECK-LABEL: 'even_start1' ; CHECK-NEXT: Determining loop execution counts for: @even_start1 ; CHECK-NEXT: Loop %loop: backedge-taken count is (-2 + (2 * %n)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-2 + (2 * %n)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-2 + (2 * %n)) ; CHECK-NEXT: Predicates: @@ -348,7 +348,7 @@ define void @even_start1_step2(i4 %n) { ; CHECK-LABEL: 'even_start1_step2' ; CHECK-NEXT: Determining loop execution counts for: @even_start1_step2 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-2 + (2 * %n)) /u 2) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 2 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-2 + (2 * %n)) /u 2) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-2 + (2 * %n)) /u 2) ; CHECK-NEXT: Predicates: @@ -371,7 +371,7 @@ define void @even_startx(i4 %n, i4 %x) { ; CHECK-LABEL: 'even_startx' ; CHECK-NEXT: Determining loop execution counts for: @even_startx ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax (2 * %n))) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 -2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax (2 * %n))) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax (2 * %n))) ; CHECK-NEXT: Predicates: @@ -394,7 +394,7 @@ define void @even_startx_step2(i4 %n, i4 %x) { ; CHECK-LABEL: 'even_startx_step2' ; CHECK-NEXT: Determining loop execution counts for: @even_startx_step2 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax (2 * %n))) /u 2) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 7 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 7 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax (2 * %n))) /u 2) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax (2 * %n))) /u 2) ; CHECK-NEXT: Predicates: @@ -417,7 +417,7 @@ define void @even_nsw(i4 %n) { ; CHECK-LABEL: 'even_nsw' ; CHECK-NEXT: Determining loop execution counts for: @even_nsw ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (2 * %n)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 5 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 5 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (2 * %n)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (2 * %n)) ; CHECK-NEXT: Predicates: @@ -440,7 +440,7 @@ define void @even_nsw_step2(i4 %n) { ; CHECK-LABEL: 'even_nsw_step2' ; CHECK-NEXT: Determining loop execution counts for: @even_nsw_step2 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 + (2 * %n)) /u 2) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 2 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 + (2 * %n)) /u 2) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 + (2 * %n)) /u 2) ; CHECK-NEXT: Predicates: @@ -463,7 +463,7 @@ define void @even_nsw_start1(i4 %n) { ; CHECK-LABEL: 'even_nsw_start1' ; CHECK-NEXT: Determining loop execution counts for: @even_nsw_start1 ; CHECK-NEXT: Loop %loop: backedge-taken count is (-2 + (2 * %n)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-2 + (2 * %n)) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-2 + (2 * %n)) ; CHECK-NEXT: Predicates: @@ -486,7 +486,7 @@ define void @even_nsw_start1_step2(i4 %n) { ; CHECK-LABEL: 'even_nsw_start1_step2' ; CHECK-NEXT: Determining loop execution counts for: @even_nsw_start1_step2 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-2 + (2 * %n)) /u 2) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 2 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-2 + (2 * %n)) /u 2) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-2 + (2 * %n)) /u 2) ; CHECK-NEXT: Predicates: @@ -509,7 +509,7 @@ define void @even_nsw_startx(i4 %n, i4 %x) { ; CHECK-LABEL: 'even_nsw_startx' ; CHECK-NEXT: Determining loop execution counts for: @even_nsw_startx ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax (2 * %n))) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 -2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax (2 * %n))) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax (2 * %n))) ; CHECK-NEXT: Predicates: @@ -532,7 +532,7 @@ define void @even_nsw_startx_step2(i4 %n, i4 %x) { ; CHECK-LABEL: 'even_nsw_startx_step2' ; CHECK-NEXT: Determining loop execution counts for: @even_nsw_startx_step2 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax (2 * %n))) /u 2) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 7 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 7 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax (2 * %n))) /u 2) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax (2 * %n))) /u 2) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll index a0a5158bdff1..7cb65d23764e 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll @@ -14,7 +14,7 @@ define void @test_trip_multiple_4(i32 %num) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-2147483648) S: [1,-2147483648) Exits: %num LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_trip_multiple_4 ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) ; CHECK-NEXT: Predicates: @@ -49,7 +49,7 @@ define void @test_trip_multiple_4_guard(i32 %num) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-2147483648) S: [1,-2147483648) Exits: %num LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_trip_multiple_4_guard ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) ; CHECK-NEXT: Predicates: @@ -85,7 +85,7 @@ define void @test_trip_multiple_4_ugt_5(i32 %num) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-2147483648) S: [1,-2147483648) Exits: %num LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_trip_multiple_4_ugt_5 ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) ; CHECK-NEXT: Predicates: @@ -121,7 +121,7 @@ define void @test_trip_multiple_4_ugt_5_order_swapped(i32 %num) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-2147483648) S: [1,-2147483648) Exits: %num LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_trip_multiple_4_ugt_5_order_swapped ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) ; CHECK-NEXT: Predicates: @@ -156,7 +156,7 @@ define void @test_trip_multiple_4_sgt_5(i32 %num) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-2147483648) S: [1,-2147483648) Exits: %num LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_trip_multiple_4_sgt_5 ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) ; CHECK-NEXT: Predicates: @@ -192,7 +192,7 @@ define void @test_trip_multiple_4_sgt_5_order_swapped(i32 %num) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-2147483648) S: [1,-2147483648) Exits: %num LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_trip_multiple_4_sgt_5_order_swapped ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) ; CHECK-NEXT: Predicates: @@ -227,7 +227,7 @@ define void @test_trip_multiple_4_uge_5(i32 %num) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-2147483648) S: [1,-2147483648) Exits: %num LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_trip_multiple_4_uge_5 ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) ; CHECK-NEXT: Predicates: @@ -263,7 +263,7 @@ define void @test_trip_multiple_4_uge_5_order_swapped(i32 %num) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-2147483648) S: [1,-2147483648) Exits: %num LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_trip_multiple_4_uge_5_order_swapped ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) ; CHECK-NEXT: Predicates: @@ -299,7 +299,7 @@ define void @test_trip_multiple_4_sge_5(i32 %num) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-2147483648) S: [1,-2147483648) Exits: %num LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_trip_multiple_4_sge_5 ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) ; CHECK-NEXT: Predicates: @@ -334,7 +334,7 @@ define void @test_trip_multiple_4_sge_5_order_swapped(i32 %num) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-2147483648) S: [1,-2147483648) Exits: %num LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_trip_multiple_4_sge_5_order_swapped ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) ; CHECK-NEXT: Predicates: @@ -370,7 +370,7 @@ define void @test_trip_multiple_4_icmp_ops_swapped(i32 %num) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-2147483648) S: [1,-2147483648) Exits: %num LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_trip_multiple_4_icmp_ops_swapped ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) ; CHECK-NEXT: Predicates: @@ -405,7 +405,7 @@ define void @test_trip_multiple_4_upper_lower_bounds(i32 %num) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-2147483648) S: [1,-2147483648) Exits: %num LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_trip_multiple_4_upper_lower_bounds ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) ; CHECK-NEXT: Predicates: @@ -442,7 +442,7 @@ define void @test_trip_multiple_4_upper_lower_bounds_swapped1(i32 %num) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-2147483648) S: [1,-2147483648) Exits: %num LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_trip_multiple_4_upper_lower_bounds_swapped1 ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) ; CHECK-NEXT: Predicates: @@ -479,7 +479,7 @@ define void @test_trip_multiple_4_upper_lower_bounds_swapped2(i32 %num) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-2147483648) S: [1,-2147483648) Exits: %num LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_trip_multiple_4_upper_lower_bounds_swapped2 ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) ; CHECK-NEXT: Predicates: @@ -516,7 +516,7 @@ define void @test_trip_multiple_5(i32 %num) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-2147483648) S: [1,-2147483648) Exits: %num LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_trip_multiple_5 ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) ; CHECK-NEXT: Predicates: @@ -553,7 +553,7 @@ define void @test_trunc_operand_larger_than_urem_expr(i64 %N) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-9223372036854775808) S: [1,-9223372036854775808) Exits: %N LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_trunc_operand_larger_than_urem_expr ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %N) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %N) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %N) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-multiple.ll b/llvm/test/Analysis/ScalarEvolution/trip-multiple.ll index a292dcec7796..f0065b9c2d8b 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-multiple.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-multiple.ll @@ -25,7 +25,7 @@ define void @trip_multiple_3(i32 noundef %num) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,0) S: [1,0) Exits: %num LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @trip_multiple_3 ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) ; CHECK-NEXT: Predicates: @@ -61,7 +61,7 @@ define void @trip_multiple_4(i32 noundef %num) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-3) S: [1,-3) Exits: %num LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @trip_multiple_4 ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -5 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -5 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) ; CHECK-NEXT: Predicates: @@ -98,7 +98,7 @@ define void @trip_multiple_5(i32 noundef %num) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,0) S: [1,0) Exits: %num LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @trip_multiple_5 ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -2 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) ; CHECK-NEXT: Predicates: @@ -135,7 +135,7 @@ define void @trip_multiple_6(i32 noundef %num) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-3) S: [1,-3) Exits: %num LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @trip_multiple_6 ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -5 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -5 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) ; CHECK-NEXT: Predicates: @@ -172,7 +172,7 @@ define void @trip_multiple_7(i32 noundef %num) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-3) S: [1,-3) Exits: %num LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @trip_multiple_7 ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -5 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -5 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) ; CHECK-NEXT: Predicates: @@ -209,7 +209,7 @@ define void @trip_multiple_8(i32 noundef %num) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-7) S: [1,-7) Exits: %num LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @trip_multiple_8 ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -9 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -9 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) ; CHECK-NEXT: Predicates: @@ -245,7 +245,7 @@ define void @trip_multiple_9(i32 noundef %num) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-3) S: [1,-3) Exits: %num LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @trip_multiple_9 ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -5 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -5 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) ; CHECK-NEXT: Predicates: @@ -281,7 +281,7 @@ define void @trip_multiple_10(i32 noundef %num) { ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-5) S: [1,-5) Exits: %num LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @trip_multiple_10 ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is -7 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -7 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/umin-umax-folds.ll b/llvm/test/Analysis/ScalarEvolution/umin-umax-folds.ll index 94f4acc0c5d1..917503b23a74 100644 --- a/llvm/test/Analysis/ScalarEvolution/umin-umax-folds.ll +++ b/llvm/test/Analysis/ScalarEvolution/umin-umax-folds.ll @@ -16,7 +16,7 @@ define void @umin_sext_x_zext_x(i32 %len) { ; CHECK-NEXT: --> (%cmp1 umin %cmp2) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @umin_sext_x_zext_x ; CHECK-NEXT: Loop %loop: backedge-taken count is (zext i32 %len to i64) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4294967295 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4294967295 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i32 %len to i64) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (zext i32 %len to i64) ; CHECK-NEXT: Predicates: @@ -52,7 +52,7 @@ define void @ule_sext_x_zext_x(i32 %len) { ; CHECK-NEXT: --> (zext i32 %len to i64) U: [0,4294967296) S: [0,4294967296) Exits: (zext i32 %len to i64) LoopDispositions: { %loop: Invariant } ; CHECK-NEXT: Determining loop execution counts for: @ule_sext_x_zext_x ; CHECK-NEXT: Loop %loop: backedge-taken count is (zext i32 %len to i64) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4294967295 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4294967295 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i32 %len to i64) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (zext i32 %len to i64) ; CHECK-NEXT: Predicates: @@ -88,7 +88,7 @@ define void @uge_sext_x_zext_x(i32 %len) { ; CHECK-NEXT: --> (sext i32 %len to i64) U: [-2147483648,2147483648) S: [-2147483648,2147483648) Exits: (sext i32 %len to i64) LoopDispositions: { %loop: Invariant } ; CHECK-NEXT: Determining loop execution counts for: @uge_sext_x_zext_x ; CHECK-NEXT: Loop %loop: backedge-taken count is (sext i32 %len to i64) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (sext i32 %len to i64) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (sext i32 %len to i64) ; CHECK-NEXT: Predicates: @@ -124,7 +124,7 @@ define void @ult_sext_x_zext_x(i32 %len) { ; CHECK-NEXT: --> (zext i32 %len to i64) U: [0,4294967296) S: [0,4294967296) Exits: (zext i32 %len to i64) LoopDispositions: { %loop: Invariant } ; CHECK-NEXT: Determining loop execution counts for: @ult_sext_x_zext_x ; CHECK-NEXT: Loop %loop: backedge-taken count is (zext i32 %len to i64) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4294967295 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4294967295 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i32 %len to i64) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (zext i32 %len to i64) ; CHECK-NEXT: Predicates: @@ -160,7 +160,7 @@ define void @ugt_sext_x_zext_x(i32 %len) { ; CHECK-NEXT: --> (sext i32 %len to i64) U: [-2147483648,2147483648) S: [-2147483648,2147483648) Exits: (sext i32 %len to i64) LoopDispositions: { %loop: Invariant } ; CHECK-NEXT: Determining loop execution counts for: @ugt_sext_x_zext_x ; CHECK-NEXT: Loop %loop: backedge-taken count is (sext i32 %len to i64) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (sext i32 %len to i64) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (sext i32 %len to i64) ; CHECK-NEXT: Predicates: @@ -196,7 +196,7 @@ define void @sle_sext_x_zext_x(i32 %len) { ; CHECK-NEXT: --> (zext i32 %len to i64) U: [0,4294967296) S: [0,4294967296) Exits: (zext i32 %len to i64) LoopDispositions: { %loop: Invariant } ; CHECK-NEXT: Determining loop execution counts for: @sle_sext_x_zext_x ; CHECK-NEXT: Loop %loop: backedge-taken count is (zext i32 %len to i64) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4294967295 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4294967295 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i32 %len to i64) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (zext i32 %len to i64) ; CHECK-NEXT: Predicates: @@ -232,7 +232,7 @@ define void @sge_sext_x_zext_x(i32 %len) { ; CHECK-NEXT: --> (zext i32 %len to i64) U: [0,4294967296) S: [0,4294967296) Exits: (zext i32 %len to i64) LoopDispositions: { %loop: Invariant } ; CHECK-NEXT: Determining loop execution counts for: @sge_sext_x_zext_x ; CHECK-NEXT: Loop %loop: backedge-taken count is (zext i32 %len to i64) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4294967295 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4294967295 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i32 %len to i64) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (zext i32 %len to i64) ; CHECK-NEXT: Predicates: @@ -268,7 +268,7 @@ define void @slt_sext_x_zext_x(i32 %len) { ; CHECK-NEXT: --> (sext i32 %len to i64) U: [-2147483648,2147483648) S: [-2147483648,2147483648) Exits: (sext i32 %len to i64) LoopDispositions: { %loop: Invariant } ; CHECK-NEXT: Determining loop execution counts for: @slt_sext_x_zext_x ; CHECK-NEXT: Loop %loop: backedge-taken count is (sext i32 %len to i64) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (sext i32 %len to i64) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (sext i32 %len to i64) ; CHECK-NEXT: Predicates: @@ -304,7 +304,7 @@ define void @sgt_sext_x_zext_x(i32 %len) { ; CHECK-NEXT: --> (zext i32 %len to i64) U: [0,4294967296) S: [0,4294967296) Exits: (zext i32 %len to i64) LoopDispositions: { %loop: Invariant } ; CHECK-NEXT: Determining loop execution counts for: @sgt_sext_x_zext_x ; CHECK-NEXT: Loop %loop: backedge-taken count is (zext i32 %len to i64) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 4294967295 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4294967295 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i32 %len to i64) ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (zext i32 %len to i64) ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/unknown_phis.ll b/llvm/test/Analysis/ScalarEvolution/unknown_phis.ll index 7bd7a5eb2cfc..685a3d182dc4 100644 --- a/llvm/test/Analysis/ScalarEvolution/unknown_phis.ll +++ b/llvm/test/Analysis/ScalarEvolution/unknown_phis.ll @@ -48,7 +48,7 @@ define void @merge_values_with_ranges_looped(ptr %a_len_ptr, ptr %b_len_ptr) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,101) S: [1,101) Exits: 100 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @merge_values_with_ranges_looped ; CHECK-NEXT: Loop %loop: backedge-taken count is 99 -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 99 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 99 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 99 ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 99 ; CHECK-NEXT: Predicates: diff --git a/llvm/test/Analysis/ScalarEvolution/widenable-condition.ll b/llvm/test/Analysis/ScalarEvolution/widenable-condition.ll index 7d8b01b9effd..867b06bd24ca 100644 --- a/llvm/test/Analysis/ScalarEvolution/widenable-condition.ll +++ b/llvm/test/Analysis/ScalarEvolution/widenable-condition.ll @@ -23,7 +23,7 @@ define i32 @wc_max() { ; CHECK-NEXT: --> (%cond_1 umin %widenable_cond3) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @wc_max ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 1999 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 1999 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 1999 ; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; diff --git a/llvm/test/Transforms/LoopDeletion/invalidate-scev-after-hoisting.ll b/llvm/test/Transforms/LoopDeletion/invalidate-scev-after-hoisting.ll index d2d085b8c7ad..4827fb50dab2 100644 --- a/llvm/test/Transforms/LoopDeletion/invalidate-scev-after-hoisting.ll +++ b/llvm/test/Transforms/LoopDeletion/invalidate-scev-after-hoisting.ll @@ -4,11 +4,6 @@ ; Make sure the SCEV for %invar is invalidated properly when the instruction is ; moved by LoopDeletion. -; CHECK: Determining loop execution counts for: @test -; CHECK-NEXT: Loop %inner: backedge-taken count is (405 + %invar) -; CHECK-NEXT: Loop %inner: constant max backedge-taken count is 405 -; CHECK-NEXT: Loop %inner: symbolic max backedge-taken count is (405 + %invar) -; CHECK-NEXT: Loop %inner: Predicated backedge-taken count is (405 + %invar) define void @test_pr57837() { ; CHECK-LABEL: @test_pr57837( diff --git a/llvm/test/Transforms/LoopSimplify/preserve-scev.ll b/llvm/test/Transforms/LoopSimplify/preserve-scev.ll index 039fc16a6322..2763ef44b8d5 100644 --- a/llvm/test/Transforms/LoopSimplify/preserve-scev.ll +++ b/llvm/test/Transforms/LoopSimplify/preserve-scev.ll @@ -13,7 +13,7 @@ target datalayout = "n8:16:32:64" ; CHECK: %[[PHI:.*]] = phi i32 [ 0, %entry ], [ %{{.*}}, %if.then5 ], [ %[[PHI]], %if.end ] ; CHECK-LABEL: Determining loop execution counts for: @test ; CHECK: Loop %for.body18: Unpredictable backedge-taken count. -; CHECK: Loop %for.body18: constant max backedge-taken count is 2147483646 +; CHECK: Loop %for.body18: constant max backedge-taken count is i32 2147483646 ; CHECK: Loop %for.body18: Unpredictable predicated backedge-taken count. ; CHECK: Loop %for.cond: Unpredictable backedge-taken count. ; CHECK: Loop %for.cond: Unpredictable constant max backedge-taken count. @@ -25,10 +25,10 @@ target datalayout = "n8:16:32:64" ; CHECK: phi i32 [ %{{.*}}, %if.then5 ], [ 0, %entry ] ; CHECK-LABEL: Determining loop execution counts for: @test ; CHECK: Loop %for.body18: Unpredictable backedge-taken count. -; CHECK: Loop %for.body18: constant max backedge-taken count is 2147483646 +; CHECK: Loop %for.body18: constant max backedge-taken count is i32 2147483646 ; CHECK: Loop %for.body18: Unpredictable predicated backedge-taken count. ; CHECK: Loop %for.cond: Unpredictable backedge-taken count. -; CHECK: Loop %for.cond: constant max backedge-taken count is -2147483647 +; CHECK: Loop %for.cond: constant max backedge-taken count is i32 -2147483647 ; CHECK: Loop %for.cond: Unpredictable predicated backedge-taken count. ; CHECK: Loop %for.cond.outer: Unpredictable backedge-taken count. ; CHECK: Loop %for.cond.outer: Unpredictable constant max backedge-taken count. @@ -82,7 +82,7 @@ declare void @foo() nounwind ; First SCEV print: ; CHECK-LABEL: Determining loop execution counts for: @mergeExit ; CHECK: Loop %while.cond191: Unpredictable backedge-taken count. -; CHECK: Loop %while.cond191: constant max backedge-taken count is -1 +; CHECK: Loop %while.cond191: constant max backedge-taken count is i32 -1 ; CHECK: Loop %while.cond191: Unpredictable predicated backedge-taken count. ; CHECK: Loop %while.cond191.outer: Unpredictable backedge-taken count. ; CHECK: Loop %while.cond191.outer: Unpredictable constant max backedge-taken count. @@ -92,10 +92,10 @@ declare void @foo() nounwind ; Second SCEV print: ; CHECK-LABEL: Determining loop execution counts for: @mergeExit ; CHECK: Loop %while.cond191: backedge-taken count is 0 -; CHECK: Loop %while.cond191: constant max backedge-taken count is 0 +; CHECK: Loop %while.cond191: constant max backedge-taken count is i32 0 ; CHECK: Loop %while.cond191: Predicated backedge-taken count is 0 ; CHECK: Loop %while.cond191.outer: Unpredictable backedge-taken count. -; CHECK: Loop %while.cond191.outer: constant max backedge-taken count is false +; CHECK: Loop %while.cond191.outer: constant max backedge-taken count is i1 false ; CHECK: Loop %while.cond191.outer: Unpredictable predicated backedge-taken count. define void @mergeExit(i32 %MapAttrCount) nounwind uwtable ssp { entry: -- GitLab From c161720ab49ed426e50f70c09da9b0c04be0f50f Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 6 Mar 2024 08:56:37 -0800 Subject: [PATCH 320/929] [RISCV] Slightly improve expanded multiply emulation in getVLENFactoredAmount. (#84113) Instead of initializing the accumulator to 0. Initialize it on first assignment with a mv from the register that holds VLENB << ShiftAmount. Fix a missing kill flag on the final Add. I have no real interest in this case, just an easy optimization I noticed. --- llvm/lib/Target/RISCV/RISCVInstrInfo.cpp | 28 +++++++++++-------- .../CodeGen/RISCV/rvv/allocate-lmul-2-4-8.ll | 9 ++---- .../RISCV/rvv/fixed-vectors-reduction-fp.ll | 24 ++++++---------- .../RISCV/rvv/fixed-vectors-shuffle-concat.ll | 6 ++-- 4 files changed, 30 insertions(+), 37 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp index ef0d7cbc835d..e0564172182c 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp @@ -3130,11 +3130,7 @@ void RISCVInstrInfo::getVLENFactoredAmount(MachineFunction &MF, .addReg(N, RegState::Kill) .setMIFlag(Flag); } else { - Register Acc = MRI.createVirtualRegister(&RISCV::GPRRegClass); - BuildMI(MBB, II, DL, get(RISCV::ADDI), Acc) - .addReg(RISCV::X0) - .addImm(0) - .setMIFlag(Flag); + Register Acc; uint32_t PrevShiftAmount = 0; for (uint32_t ShiftAmount = 0; NumOfVReg >> ShiftAmount; ShiftAmount++) { if (NumOfVReg & (1U << ShiftAmount)) { @@ -3143,17 +3139,27 @@ void RISCVInstrInfo::getVLENFactoredAmount(MachineFunction &MF, .addReg(DestReg, RegState::Kill) .addImm(ShiftAmount - PrevShiftAmount) .setMIFlag(Flag); - if (NumOfVReg >> (ShiftAmount + 1)) - BuildMI(MBB, II, DL, get(RISCV::ADD), Acc) - .addReg(Acc, RegState::Kill) - .addReg(DestReg) - .setMIFlag(Flag); + if (NumOfVReg >> (ShiftAmount + 1)) { + // If we don't have an accmulator yet, create it and copy DestReg. + if (!Acc) { + Acc = MRI.createVirtualRegister(&RISCV::GPRRegClass); + BuildMI(MBB, II, DL, get(TargetOpcode::COPY), Acc) + .addReg(DestReg) + .setMIFlag(Flag); + } else { + BuildMI(MBB, II, DL, get(RISCV::ADD), Acc) + .addReg(Acc, RegState::Kill) + .addReg(DestReg) + .setMIFlag(Flag); + } + } PrevShiftAmount = ShiftAmount; } } + assert(Acc && "Expected valid accumulator"); BuildMI(MBB, II, DL, get(RISCV::ADD), DestReg) .addReg(DestReg, RegState::Kill) - .addReg(Acc) + .addReg(Acc, RegState::Kill) .setMIFlag(Flag); } } diff --git a/llvm/test/CodeGen/RISCV/rvv/allocate-lmul-2-4-8.ll b/llvm/test/CodeGen/RISCV/rvv/allocate-lmul-2-4-8.ll index 78bec6c68c3f..466ab085b266 100644 --- a/llvm/test/CodeGen/RISCV/rvv/allocate-lmul-2-4-8.ll +++ b/llvm/test/CodeGen/RISCV/rvv/allocate-lmul-2-4-8.ll @@ -253,9 +253,8 @@ define void @lmul4_and_2_x2_1() nounwind { ; NOMUL-NEXT: sd s0, 32(sp) # 8-byte Folded Spill ; NOMUL-NEXT: addi s0, sp, 48 ; NOMUL-NEXT: csrr a0, vlenb -; NOMUL-NEXT: li a1, 0 ; NOMUL-NEXT: slli a0, a0, 2 -; NOMUL-NEXT: add a1, a1, a0 +; NOMUL-NEXT: mv a1, a0 ; NOMUL-NEXT: slli a0, a0, 1 ; NOMUL-NEXT: add a0, a0, a1 ; NOMUL-NEXT: sub sp, sp, a0 @@ -455,9 +454,8 @@ define void @lmul_8_x5() nounwind { ; NOMUL-NEXT: sd s0, 64(sp) # 8-byte Folded Spill ; NOMUL-NEXT: addi s0, sp, 80 ; NOMUL-NEXT: csrr a0, vlenb -; NOMUL-NEXT: li a1, 0 ; NOMUL-NEXT: slli a0, a0, 3 -; NOMUL-NEXT: add a1, a1, a0 +; NOMUL-NEXT: mv a1, a0 ; NOMUL-NEXT: slli a0, a0, 2 ; NOMUL-NEXT: add a0, a0, a1 ; NOMUL-NEXT: sub sp, sp, a0 @@ -517,9 +515,8 @@ define void @lmul_8_x9() nounwind { ; NOMUL-NEXT: sd s0, 64(sp) # 8-byte Folded Spill ; NOMUL-NEXT: addi s0, sp, 80 ; NOMUL-NEXT: csrr a0, vlenb -; NOMUL-NEXT: li a1, 0 ; NOMUL-NEXT: slli a0, a0, 3 -; NOMUL-NEXT: add a1, a1, a0 +; NOMUL-NEXT: mv a1, a0 ; NOMUL-NEXT: slli a0, a0, 3 ; NOMUL-NEXT: add a0, a0, a1 ; NOMUL-NEXT: sub sp, sp, a0 diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-fp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-fp.ll index 855e280164a2..68740eec56e4 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-fp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-fp.ll @@ -2133,9 +2133,8 @@ define float @vreduce_fminimum_v128f32(ptr %x) { ; CHECK-NEXT: addi sp, sp, -16 ; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: csrr a1, vlenb -; CHECK-NEXT: li a2, 0 ; CHECK-NEXT: slli a1, a1, 3 -; CHECK-NEXT: add a2, a2, a1 +; CHECK-NEXT: mv a2, a1 ; CHECK-NEXT: slli a1, a1, 1 ; CHECK-NEXT: add a1, a1, a2 ; CHECK-NEXT: sub sp, sp, a1 @@ -2256,9 +2255,8 @@ define float @vreduce_fminimum_v128f32(ptr %x) { ; CHECK-NEXT: vfmin.vv v8, v11, v8 ; CHECK-NEXT: vfmv.f.s fa0, v8 ; CHECK-NEXT: csrr a0, vlenb -; CHECK-NEXT: li a1, 0 ; CHECK-NEXT: slli a0, a0, 3 -; CHECK-NEXT: add a1, a1, a0 +; CHECK-NEXT: mv a1, a0 ; CHECK-NEXT: slli a0, a0, 1 ; CHECK-NEXT: add a0, a0, a1 ; CHECK-NEXT: add sp, sp, a0 @@ -2739,9 +2737,8 @@ define double @vreduce_fminimum_v64f64(ptr %x) { ; CHECK-NEXT: addi sp, sp, -16 ; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: csrr a1, vlenb -; CHECK-NEXT: li a2, 0 ; CHECK-NEXT: slli a1, a1, 3 -; CHECK-NEXT: add a2, a2, a1 +; CHECK-NEXT: mv a2, a1 ; CHECK-NEXT: slli a1, a1, 1 ; CHECK-NEXT: add a1, a1, a2 ; CHECK-NEXT: sub sp, sp, a1 @@ -2852,9 +2849,8 @@ define double @vreduce_fminimum_v64f64(ptr %x) { ; CHECK-NEXT: vfmin.vv v8, v11, v8 ; CHECK-NEXT: vfmv.f.s fa0, v8 ; CHECK-NEXT: csrr a0, vlenb -; CHECK-NEXT: li a1, 0 ; CHECK-NEXT: slli a0, a0, 3 -; CHECK-NEXT: add a1, a1, a0 +; CHECK-NEXT: mv a1, a0 ; CHECK-NEXT: slli a0, a0, 1 ; CHECK-NEXT: add a0, a0, a1 ; CHECK-NEXT: add sp, sp, a0 @@ -3461,9 +3457,8 @@ define float @vreduce_fmaximum_v128f32(ptr %x) { ; CHECK-NEXT: addi sp, sp, -16 ; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: csrr a1, vlenb -; CHECK-NEXT: li a2, 0 ; CHECK-NEXT: slli a1, a1, 3 -; CHECK-NEXT: add a2, a2, a1 +; CHECK-NEXT: mv a2, a1 ; CHECK-NEXT: slli a1, a1, 1 ; CHECK-NEXT: add a1, a1, a2 ; CHECK-NEXT: sub sp, sp, a1 @@ -3584,9 +3579,8 @@ define float @vreduce_fmaximum_v128f32(ptr %x) { ; CHECK-NEXT: vfmax.vv v8, v11, v8 ; CHECK-NEXT: vfmv.f.s fa0, v8 ; CHECK-NEXT: csrr a0, vlenb -; CHECK-NEXT: li a1, 0 ; CHECK-NEXT: slli a0, a0, 3 -; CHECK-NEXT: add a1, a1, a0 +; CHECK-NEXT: mv a1, a0 ; CHECK-NEXT: slli a0, a0, 1 ; CHECK-NEXT: add a0, a0, a1 ; CHECK-NEXT: add sp, sp, a0 @@ -4067,9 +4061,8 @@ define double @vreduce_fmaximum_v64f64(ptr %x) { ; CHECK-NEXT: addi sp, sp, -16 ; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: csrr a1, vlenb -; CHECK-NEXT: li a2, 0 ; CHECK-NEXT: slli a1, a1, 3 -; CHECK-NEXT: add a2, a2, a1 +; CHECK-NEXT: mv a2, a1 ; CHECK-NEXT: slli a1, a1, 1 ; CHECK-NEXT: add a1, a1, a2 ; CHECK-NEXT: sub sp, sp, a1 @@ -4180,9 +4173,8 @@ define double @vreduce_fmaximum_v64f64(ptr %x) { ; CHECK-NEXT: vfmax.vv v8, v11, v8 ; CHECK-NEXT: vfmv.f.s fa0, v8 ; CHECK-NEXT: csrr a0, vlenb -; CHECK-NEXT: li a1, 0 ; CHECK-NEXT: slli a0, a0, 3 -; CHECK-NEXT: add a1, a1, a0 +; CHECK-NEXT: mv a1, a0 ; CHECK-NEXT: slli a0, a0, 1 ; CHECK-NEXT: add a0, a0, a1 ; CHECK-NEXT: add sp, sp, a0 diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll index 6a7ec6dc5bd7..e5bef20fd9e2 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll @@ -193,9 +193,8 @@ define <32 x i32> @concat_8xv4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x ; VLA-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb ; VLA-NEXT: vmv1r.v v16, v15 ; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: li a1, 0 ; VLA-NEXT: slli a0, a0, 3 -; VLA-NEXT: add a1, a1, a0 +; VLA-NEXT: mv a1, a0 ; VLA-NEXT: slli a0, a0, 1 ; VLA-NEXT: add a0, a0, a1 ; VLA-NEXT: add a0, sp, a0 @@ -245,9 +244,8 @@ define <32 x i32> @concat_8xv4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x ; VLA-NEXT: li a0, 32 ; VLA-NEXT: vsetvli zero, a0, e32, m8, ta, ma ; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: li a1, 0 ; VLA-NEXT: slli a0, a0, 3 -; VLA-NEXT: add a1, a1, a0 +; VLA-NEXT: mv a1, a0 ; VLA-NEXT: slli a0, a0, 1 ; VLA-NEXT: add a0, a0, a1 ; VLA-NEXT: add a0, sp, a0 -- GitLab From 0cbbcf1ef006ce13a1fa94960067723982ae955a Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Wed, 6 Mar 2024 10:58:39 -0600 Subject: [PATCH 321/929] [libc] Update GPU documentation pages (#84076) Summary: After the overhaul of the GPU build the documentation pages were a little stale. This updates them with more in-depth information on building the GPU runtimes and using them. Specifically using them goes through the differences between the offloading and direct compilation modes. --- libc/docs/full_cross_build.rst | 11 ++ libc/docs/gpu/building.rst | 246 +++++++++++++++++++++++++++++++ libc/docs/gpu/index.rst | 3 +- libc/docs/gpu/rpc.rst | 2 + libc/docs/gpu/using.rst | 261 ++++++++++++++++++++++++++------- 5 files changed, 473 insertions(+), 50 deletions(-) create mode 100644 libc/docs/gpu/building.rst diff --git a/libc/docs/full_cross_build.rst b/libc/docs/full_cross_build.rst index f06464534f15..100e17a977e7 100644 --- a/libc/docs/full_cross_build.rst +++ b/libc/docs/full_cross_build.rst @@ -94,6 +94,8 @@ The above ``ninja`` command will build the libc static archives ``libc.a`` and ``libm.a`` for the target specified with ``-DLIBC_TARGET_TRIPLE`` in the CMake configure step. +.. _runtimes_cross_build: + Runtimes cross build ==================== @@ -230,3 +232,12 @@ component of the target triple as ``none``. For example, to build for a 32-bit arm target on bare metal, one can use a target triple like ``arm-none-eabi``. Other than that, the libc for a bare metal target can be built using any of the three recipes described above. + +Building for the GPU +==================== + +To build for a GPU architecture, it should only be necessary to specify the +target triple as one of the supported GPU targets. Currently, this is either +``nvptx64-nvidia-cuda`` for NVIDIA GPUs or ``amdgcn-amd-amdhsa`` for AMD GPUs. +More detailed information is provided in the :ref:`GPU +documentation`. diff --git a/libc/docs/gpu/building.rst b/libc/docs/gpu/building.rst new file mode 100644 index 000000000000..dab21e1324d2 --- /dev/null +++ b/libc/docs/gpu/building.rst @@ -0,0 +1,246 @@ +.. _libc_gpu_building: + +====================== +Building libs for GPUs +====================== + +.. contents:: Table of Contents + :depth: 4 + :local: + +Building the GPU C library +========================== + +This document will present recipes to build the LLVM C library targeting a GPU +architecture. The GPU build uses the same :ref:`cross build` +support as the other targets. However, the GPU target has the restriction that +it *must* be built with an up-to-date ``clang`` compiler. This is because the +GPU target uses several compiler extensions to target GPU architectures. + +The LLVM C library currently supports two GPU targets. This is either +``nvptx64-nvidia-cuda`` for NVIDIA GPUs or ``amdgcn-amd-amdhsa`` for AMD GPUs. +Targeting these architectures is done through ``clang``'s cross-compiling +support using the ``--target=`` flag. The following sections will +describe how to build the GPU support specifically. + +Once you have finished building, refer to :ref:`libc_gpu_usage` to get started +with the newly built C library. + +Standard runtimes build +----------------------- + +The simplest way to build the GPU libc is to use the existing LLVM runtimes +support. This will automatically handle bootstrapping an up-to-date ``clang`` +compiler and using it to build the C library. The following CMake invocation +will instruct it to build the ``libc`` runtime targeting both AMD and NVIDIA +GPUs. + +.. code-block:: sh + + $> cd llvm-project # The llvm-project checkout + $> mkdir build + $> cd build + $> cmake ../llvm -G Ninja \ + -DLLVM_ENABLE_PROJECTS="clang;lld" \ + -DLLVM_ENABLE_RUNTIMES="openmp" \ + -DCMAKE_BUILD_TYPE= \ # Select build type + -DCMAKE_INSTALL_PREFIX= \ # Where the libraries will live + -DRUNTIMES_nvptx64-nvidia-cuda_LLVM_ENABLE_RUNTIMES=libc \ + -DRUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES=libc \ + -DLLVM_RUNTIME_TARGETS="default;amdgcn-amd-amdhsa;nvptx64-nvidia-cuda" + $> ninja install + +We need ``clang`` to build the GPU C library and ``lld`` to link AMDGPU +executables, so we enable them in ``LLVM_ENABLE_PROJECTS``. We add ``openmp`` to +``LLVM_ENABLED_RUNTIMES`` so it is built for the default target and provides +OpenMP support. We then set ``RUNTIMES__LLVM_ENABLE_RUNTIMES`` to enable +``libc`` for the GPU targets. The ``LLVM_RUNTIME_TARGETS`` sets the enabled +targets to build, in this case we want the default target and the GPU targets. +Note that if ``libc`` were included in ``LLVM_ENABLE_RUNTIMES`` it would build +targeting the default host environment as well. + +Runtimes cross build +-------------------- + +For users wanting more direct control over the build process, the build steps +can be done manually instead. This build closely follows the instructions in the +:ref:`main documentation` but is specialized for the GPU +build. We follow the same steps to first build the libc tools and a suitable +compiler. These tools must all be up-to-date with the libc source. + +.. code-block:: sh + + $> cd llvm-project # The llvm-project checkout + $> mkdir build-libc-tools # A different build directory for the build tools + $> cd build-libc-tools + $> HOST_C_COMPILER= # For example "clang" + $> HOST_CXX_COMPILER= # For example "clang++" + $> cmake ../llvm \ + -G Ninja \ + -DLLVM_ENABLE_PROJECTS="clang;libc" \ + -DCMAKE_C_COMPILER=$HOST_C_COMPILER \ + -DCMAKE_CXX_COMPILER=$HOST_CXX_COMPILER \ + -DLLVM_LIBC_FULL_BUILD=ON \ + -DLIBC_HDRGEN_ONLY=ON \ # Only build the 'libc-hdrgen' tool + -DCMAKE_BUILD_TYPE=Release # Release suggested to make "clang" fast + $> ninja # Build the 'clang' compiler + $> ninja libc-hdrgen # Build the 'libc-hdrgen' tool + +Once this has finished the build directory should contain the ``clang`` compiler +and the ``libc-hdrgen`` executable. We will use the ``clang`` compiler to build +the GPU code and the ``libc-hdrgen`` tool to create the necessary headers. We +use these tools to bootstrap the build out of the runtimes directory targeting a +GPU architecture. + +.. code-block:: sh + + $> cd llvm-project # The llvm-project checkout + $> mkdir build # A different build directory for the build tools + $> cd build + $> TARGET_TRIPLE= + $> TARGET_C_COMPILER= + $> TARGET_CXX_COMPILER= + $> HDRGEN= + $> cmake ../runtimes \ # Point to the runtimes build + -G Ninja \ + -DLLVM_ENABLE_RUNTIMES=libc \ + -DCMAKE_C_COMPILER=$TARGET_C_COMPILER \ + -DCMAKE_CXX_COMPILER=$TARGET_CXX_COMPILER \ + -DLLVM_LIBC_FULL_BUILD=ON \ + -DLLVM_RUNTIMES_TARGET=$TARGET_TRIPLE \ + -DLIBC_HDRGEN_EXE=$HDRGEN \ + -DCMAKE_BUILD_TYPE=Release + $> ninja install + +The above steps will result in a build targeting one of the supported GPU +architectures. Building for multiple targets requires separate CMake +invocations. + +Standalone cross build +---------------------- + +The GPU build can also be targeted directly as long as the compiler used is a +supported ``clang`` compiler. This method is generally not recommended as it can +only target a single GPU architecture. + +.. code-block:: sh + + $> cd llvm-project # The llvm-project checkout + $> mkdir build # A different build directory for the build tools + $> cd build + $> CLANG_C_COMPILER= # Must be a trunk build + $> CLANG_CXX_COMPILER= # Must be a trunk build + $> TARGET_TRIPLE= + $> cmake ../llvm \ # Point to the llvm directory + -G Ninja \ + -DLLVM_ENABLE_PROJECTS=libc \ + -DCMAKE_C_COMPILER=$CLANG_C_COMPILER \ + -DCMAKE_CXX_COMPILER=$CLANG_CXX_COMPILER \ + -DLLVM_LIBC_FULL_BUILD=ON \ + -DLIBC_TARGET_TRIPLE=$TARGET_TRIPLE \ + -DCMAKE_BUILD_TYPE=Release + $> ninja install + +This will build and install the GPU C library along with all the other LLVM +libraries. + +Build overview +============== + +Once installed, the GPU build will create several files used for different +targets. This section will briefly describe their purpose. + +**lib//libcgpu-amdgpu.a or lib/libcgpu-amdgpu.a** + A static library containing fat binaries supporting AMD GPUs. These are built + using the support described in the `clang documentation + `_. These are intended to + be static libraries included natively for offloading languages like CUDA, HIP, + or OpenMP. This implements the standard C library. + +**lib//libmgpu-amdgpu.a or lib/libmgpu-amdgpu.a** + A static library containing fat binaries that implements the standard math + library for AMD GPUs. + +**lib//libcgpu-nvptx.a or lib/libcgpu-nvptx.a** + A static library containing fat binaries that implement the standard C library + for NVIDIA GPUs. + +**lib//libmgpu-nvptx.a or lib/libmgpu-nvptx.a** + A static library containing fat binaries that implement the standard math + library for NVIDIA GPUs. + +**include/** + The include directory where all of the generated headers for the target will + go. These definitions are strictly for the GPU when being targeted directly. + +**lib/clang//include/llvm-libc-wrappers/llvm-libc-decls** + These are wrapper headers created for offloading languages like CUDA, HIP, or + OpenMP. They contain functions supported in the GPU libc along with attributes + and metadata that declare them on the target device and make them compatible + with the host headers. + +**lib//libc.a** + The main C library static archive containing LLVM-IR targeting the given GPU. + It can be linked directly or inspected depending on the target support. + +**lib//libm.a** + The C library static archive providing implementations of the standard math + functions. + +**lib//libc.bc** + An alternate form of the library provided as a single LLVM-IR bitcode blob. + This can be used similarly to NVIDIA's or AMD's device libraries. + +**lib//libm.bc** + An alternate form of the library provided as a single LLVM-IR bitcode blob + containing the standard math functions. + +**lib//crt1.o** + An LLVM-IR file containing startup code to call the ``main`` function on the + GPU. This is used similarly to the standard C library startup object. + +**bin/amdhsa-loader** + A binary utility used to launch executables compiled targeting the AMD GPU. + This will be included if the build system found the ``hsa-runtime64`` library + either in ``/opt/rocm`` or the current CMake installation directory. This is + required to build the GPU tests .See the :ref:`libc GPU usage` + for more information. + +**bin/nvptx-loader** + A binary utility used to launch executables compiled targeting the NVIDIA GPU. + This will be included if the build system found the CUDA driver API. This is + required for building tests. + +**include/llvm-libc-rpc-server.h** + A header file containing definitions that can be used to interface with the + :ref:`RPC server`. + +**lib/libllvmlibc_rpc_server.a** + The static library containing the implementation of the RPC server. This can + be used to enable host services for anyone looking to interface with the + :ref:`RPC client`. + +CMake options +============= + +This section briefly lists a few of the CMake variables that specifically +control the GPU build of the C library. + +**LLVM_LIBC_FULL_BUILD**:BOOL + This flag controls whether or not the libc build will generate its own + headers. This must always be on when targeting the GPU. + +**LIBC_GPU_TEST_ARCHITECTURE**:STRING + Sets the architecture used to build the GPU tests for, such as ``gfx90a`` or + ``sm_80`` for AMD and NVIDIA GPUs respectively. The default behavior is to + detect the system's GPU architecture using the ``native`` option. If this + option is not set and a GPU was not detected the tests will not be built. + +**LIBC_GPU_TEST_JOBS**:STRING + Sets the number of threads used to run GPU tests. The GPU test suite will + commonly run out of resources if this is not constrained so it is recommended + to keep it low. The default value is a single thread. + +**LIBC_GPU_LOADER_EXECUTABLE**:STRING + Overrides the default loader used for running GPU tests. If this is not + provided the standard one will be built. diff --git a/libc/docs/gpu/index.rst b/libc/docs/gpu/index.rst index 2d7654866650..1fca67205acb 100644 --- a/libc/docs/gpu/index.rst +++ b/libc/docs/gpu/index.rst @@ -12,8 +12,9 @@ learn more about this project. .. toctree:: + building using support - testing rpc + testing motivation diff --git a/libc/docs/gpu/rpc.rst b/libc/docs/gpu/rpc.rst index 7b0b35af4da8..9d6d8099db95 100644 --- a/libc/docs/gpu/rpc.rst +++ b/libc/docs/gpu/rpc.rst @@ -188,6 +188,8 @@ in the GPU executable as an indicator for whether or not the server can be checked. These details should ideally be handled by the GPU language runtime, but the following example shows how it can be used by a standard user. +.. _libc_gpu_cuda_server: + .. code-block:: cuda #include diff --git a/libc/docs/gpu/using.rst b/libc/docs/gpu/using.rst index 1a48c8a3bcba..11a00cd620d8 100644 --- a/libc/docs/gpu/using.rst +++ b/libc/docs/gpu/using.rst @@ -1,6 +1,5 @@ .. _libc_gpu_usage: - =================== Using libc for GPUs =================== @@ -9,54 +8,98 @@ Using libc for GPUs :depth: 4 :local: -Building the GPU library -======================== - -LLVM's libc GPU support *must* be built with an up-to-date ``clang`` compiler -due to heavy reliance on ``clang``'s GPU support. This can be done automatically -using the LLVM runtimes support. The GPU build is done using cross-compilation -to the GPU architecture. This project currently supports AMD and NVIDIA GPUs -which can be targeted using the appropriate target name. The following -invocation will enable a cross-compiling build for the GPU architecture and -enable the ``libc`` project only for them. +Using the GPU C library +======================= + +Once you have finished :ref:`building` the GPU C library it +can be used to run libc or libm functions directly on the GPU. Currently, not +all C standard functions are supported on the GPU. Consult the :ref:`list of +supported functions` for a comprehensive list. + +The GPU C library supports two main usage modes. The first is as a supplementary +library for offloading languages such as OpenMP, CUDA, or HIP. These aim to +provide standard system utilities similarly to existing vendor libraries. The +second method treats the GPU as a hosted target by compiling C or C++ for it +directly. This is more similar to targeting OpenCL and is primarily used for +exported functions on the GPU and testing. + +Offloading usage +---------------- + +Offloading languages like CUDA, HIP, or OpenMP work by compiling a single source +file for both the host target and a list of offloading devices. In order to +support standard compilation flows, the ``clang`` driver uses fat binaries, +described in the `clang documentation +`_. This linking mode is used +by the OpenMP toolchain, but is currently opt-in for the CUDA and HIP toolchains +through the ``--offload-new-driver``` and ``-fgpu-rdc`` flags. + +The installation should contain a static library called ``libcgpu-amdgpu.a`` or +``libcgpu-nvptx.a`` depending on which GPU architectures your build targeted. +These contain fat binaries compatible with the offloading toolchain such that +they can be used directly. .. code-block:: sh - $> cd llvm-project # The llvm-project checkout - $> mkdir build - $> cd build - $> cmake ../llvm -G Ninja \ - -DLLVM_ENABLE_PROJECTS="clang;lld;compiler-rt" \ - -DLLVM_ENABLE_RUNTIMES="openmp" \ - -DCMAKE_BUILD_TYPE= \ # Select build type - -DCMAKE_INSTALL_PREFIX= \ # Where 'libcgpu.a' will live - -DRUNTIMES_nvptx64-nvidia-cuda_LLVM_ENABLE_RUNTIMES=libc \ - -DRUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES=libc \ - -DLLVM_RUNTIME_TARGETS=default;amdgcn-amd-amdhsa;nvptx64-nvidia-cuda - $> ninja install - -Since we want to include ``clang``, ``lld`` and ``compiler-rt`` in our -toolchain, we list them in ``LLVM_ENABLE_PROJECTS``. To ensure ``libc`` is built -using a compatible compiler and to support ``openmp`` offloading, we list them -in ``LLVM_ENABLE_RUNTIMES`` to build them after the enabled projects using the -newly built compiler. ``CMAKE_INSTALL_PREFIX`` specifies the installation -directory in which to install the ``libcgpu-nvptx.a`` and ``libcgpu-amdgpu.a`` -libraries and headers along with LLVM. The generated headers will be placed in -``include/``. - -Usage -===== - -Once the static archive has been built it can be linked directly -with offloading applications as a standard library. This process is described in -the `clang documentation `_. -This linking mode is used by the OpenMP toolchain, but is currently opt-in for -the CUDA and HIP toolchains through the ``--offload-new-driver``` and -``-fgpu-rdc`` flags. A typical usage will look this this: + $> clang opnemp.c -fopenmp --offload-arch=gfx90a -lcgpu-amdgpu + $> clang cuda.cu --offload-arch=sm_80 --offload-new-driver -fgpu-rdc -lcgpu-nvptx + $> clang hip.hip --offload-arch=gfx940 --offload-new-driver -fgpu-rdc -lcgpu-amdgpu + +This will automatically link in the needed function definitions if they were +required by the user's application. Normally using the ``-fgpu-rdc`` option +results in sub-par performance due to ABA linking. However, the offloading +toolchain supports the ``--foffload-lto`` option to support LTO on the target +device. + +Offloading languages require that functions present on the device be declared as +such. This is done with the ``__device__`` keyword in CUDA and HIP or the +``declare target`` pragma in OpenMP. This requires that the LLVM C library +exposes its implemented functions to the compiler when it is used to build. We +support this by providing wrapper headers in the compiler's resource directory. +These are located in ``/include/llvm-libc-wrappers`` in your +installation. + +The support for HIP and CUDA is more experimental, requiring manual intervention +to link and use the facilities. An example of this is shown in the :ref:`CUDA +server example`. The OpenMP Offloading toolchain is +completely integrated with the LLVM C library however. It will automatically +handle including the necessary libraries, define device-side interfaces, and run +the RPC server. + +OpenMP Offloading example +^^^^^^^^^^^^^^^^^^^^^^^^^ + +This section provides a simple example of compiling an OpenMP program with the +GPU C library. + +.. code-block:: c++ + + #include + + int main() { + FILE *file = stderr; + #pragma omp target teams num_teams(2) thread_limit(2) + #pragma omp parallel num_threads(2) + { fputs("Hello from OpenMP!\n", file); } + } + +This can simply be compiled like any other OpenMP application to print from two +threads and two blocks. .. code-block:: sh - $> clang foo.c -fopenmp --offload-arch=gfx90a -lcgpu + $> clang openmp.c -fopenmp --offload-arch=gfx90a + $> ./a.out + Hello from OpenMP! + Hello from OpenMP! + Hello from OpenMP! + Hello from OpenMP! + +Including the wrapper headers, linking the C library, and running the :ref:`RPC +server` are all handled automatically by the compiler and runtime. + +Binary format +^^^^^^^^^^^^^ The ``libcgpu.a`` static archive is a fat-binary containing LLVM-IR for each supported target device. The supported architectures can be seen using LLVM's @@ -64,25 +107,145 @@ supported target device. The supported architectures can be seen using LLVM's .. code-block:: sh - $> llvm-objdump --offloading libcgpu.a - libcgpu.a(strcmp.cpp.o): file format elf64-x86-64 + $> llvm-objdump --offloading libcgpu-amdgpu.a + libcgpu-amdgpu.a(strcmp.cpp.o): file format elf64-x86-64 OFFLOADING IMAGE [0]: kind llvm ir arch generic triple amdgcn-amd-amdhsa producer none + ... Because the device code is stored inside a fat binary, it can be difficult to inspect the resulting code. This can be done using the following utilities: .. code-block:: sh - $> llvm-ar x libcgpu.a strcmp.cpp.o - $> clang-offload-packager strcmp.cpp.o --image=arch=gfx90a,file=gfx90a.bc - $> opt -S out.bc - ... + $> llvm-ar x libcgpu.a strcmp.cpp.o + $> clang-offload-packager strcmp.cpp.o --image=arch=generic,file=strcmp.bc + $> opt -S out.bc + ... Please note that this fat binary format is provided for compatibility with existing offloading toolchains. The implementation in ``libc`` does not depend on any existing offloading languages and is completely freestanding. + +Direct compilation +------------------ + +Instead of using standard offloading languages, we can also target the CPU +directly using C and C++ to create a GPU executable similarly to OpenCL. This is +done by targeting the GPU architecture using `clang's cross compilation +support `_. This is the +method that the GPU C library uses both to build the library and to run tests. + +This allows us to easily define GPU specific libraries and programs that fit +well into existing tools. In order to target the GPU effectively we rely heavily +on the compiler's intrinsic and built-in functions. For example, the following +function gets the thread identifier in the 'x' dimension on both GPUs supported +GPUs. + +.. code-block:: c++ + + uint32_t get_thread_id_x() { + #if defined(__AMDGPU__) + return __builtin_amdgcn_workitem_id_x(); + #elif defined(__NVPTX__) + return __nvvm_read_ptx_sreg_tid_x(); + #else + #error "Unsupported platform" + #endif + } + +We can then compile this for both NVPTX and AMDGPU into LLVM-IR using the +following commands. + +.. code-block:: sh + + $> clang id.c --target=amdgcn-amd-amdhsa -mcpu=native -nogpulib -flto -c + $> clang id.c --target=nvptx64-nvidia-cuda -march=native -nogpulib -flto -c + +We use this support to treat the GPU as a hosted environment by providing a C +library and startup object just like a standard C library running on the host +machine. Then, in order to execute these programs, we provide a loader utility +to launch the executable on the GPU similar to a cross-compiling emulator. + +Building for AMDGPU targets +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The AMDGPU target supports several features natively by virtue of using ``lld`` +as its linker. The installation will include the ``include/amdgcn-amd-amdhsa`` +and ``lib/amdgcn-amd-amdha`` directories that contain the necessary code to use +the library. We can directly link against ``libc.a`` and use LTO to generate the +final executable. + +.. code-block:: c++ + + #include + + int main() { fputs("Hello from AMDGPU!\n", stdout); } + +This program can then be compiled using the ``clang`` compiler. Note that +``-flto`` and ``-mcpu=`` should be defined. This is because the GPU +sub-architectures do not have strict backwards compatibility. Use ``-mcpu=help`` +for accepted arguments or ``-mcpu=native`` to target the system's installed GPUs +if present. Additionally, the AMDGPU target always uses ``-flto`` because we +currently do not fully support ELF linking in ``lld``. Once built, we use the +``amdhsa-loader`` utility to launch execution on the GPU. This will be built if +the ``hsa_runtime64`` library was found during build time. + +.. code-block:: sh + + $> clang hello.c --target=amdgcn-amd-amdhsa -mcpu=native -flto -lc /lib/amdgcn-amd-amdhsa/crt1.o + $> amdhsa-loader --threads 2 --blocks 2 a.out + Hello from AMDGPU! + Hello from AMDGPU! + Hello from AMDGPU! + Hello from AMDGPU! + +This will include the ``stdio.h`` header, which is found in the +``include/amdgcn-amd-amdhsa`` directory. We define out ``main`` function like a +standard application. The startup utility in ``lib/amdgcn-amd-amdhsa/crt1.o`` +will handle the necessary steps to execute the ``main`` function along with +global initializers and command line arguments. Finally, we link in the +``libc.a`` library stored in ``lib/amdgcn-amd-amdhsa`` to define the standard C +functions. + +The search paths for the include directories and libraries are automatically +handled by the compiler. We use this support internally to run unit tests on the +GPU directly. See :ref:`libc_gpu_testing` for more information. The installation +also provides ``libc.bc`` which is a single LLVM-IR bitcode blob that can be +used instead of the static library. + +Building for NVPTX targets +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The infrastructure is the same as the AMDGPU example. However, the NVPTX binary +utilities are very limited and must be targeted directly. There is no linker +support for static libraries so we need to link in the ``libc.bc`` bitcode and +inform the compiler driver of the file's contents. + +.. code-block:: c++ + + #include + + int main(int argc, char **argv, char **envp) { + fputs("Hello from NVPTX!\n", stdout); + } + +Additionally, the NVPTX ABI requires that every function signature matches. This +requires us to pass the full prototype from ``main``. The installation will +contain the ``nvptx-loader`` utility if the CUDA driver was found during +compilation. + +.. code-block:: sh + + $> clang hello.c --target=nvptx64-nvidia-cuda -march=native \ + -x ir /lib/nvptx64-nvidia-cuda/libc.bc \ + -x ir /lib/nvptx64-nvidia-cuda/crt1.o + $> nvptx-loader --threads 2 --blocks 2 a.out + Hello from NVPTX! + Hello from NVPTX! + Hello from NVPTX! + Hello from NVPTX! -- GitLab From 3589cacfa8da89b9b5051e4dba659caa575e6b3f Mon Sep 17 00:00:00 2001 From: Yingwei Zheng Date: Thu, 7 Mar 2024 01:05:39 +0800 Subject: [PATCH 322/929] [ValueTracking] Handle `icmp pred (trunc X), C` in `computeKnownBitsFromCmp` (#82803) This patch handles the pattern `icmp pred (trunc X), C` in `computeKnownBitsFromCmp` to infer low bits of `X` from dominating conditions. --- llvm/lib/Analysis/ValueTracking.cpp | 28 +++- .../test/Transforms/InstCombine/known-bits.ll | 140 ++++++++++++++++++ 2 files changed, 162 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index f96a60a4f547..53c6326b8ec2 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -708,6 +708,25 @@ static void computeKnownBitsFromCmp(const Value *V, CmpInst::Predicate Pred, } } +static void computeKnownBitsFromICmpCond(const Value *V, ICmpInst *Cmp, + KnownBits &Known, + const SimplifyQuery &SQ, bool Invert) { + ICmpInst::Predicate Pred = + Invert ? Cmp->getInversePredicate() : Cmp->getPredicate(); + Value *LHS = Cmp->getOperand(0); + Value *RHS = Cmp->getOperand(1); + + // Handle icmp pred (trunc V), C + if (match(LHS, m_Trunc(m_Specific(V)))) { + KnownBits DstKnown(LHS->getType()->getScalarSizeInBits()); + computeKnownBitsFromCmp(LHS, Pred, LHS, RHS, DstKnown, SQ); + Known = Known.unionWith(DstKnown.anyext(Known.getBitWidth())); + return; + } + + computeKnownBitsFromCmp(V, Pred, LHS, RHS, Known, SQ); +} + static void computeKnownBitsFromCond(const Value *V, Value *Cond, KnownBits &Known, unsigned Depth, const SimplifyQuery &SQ, bool Invert) { @@ -727,9 +746,7 @@ static void computeKnownBitsFromCond(const Value *V, Value *Cond, } if (auto *Cmp = dyn_cast(Cond)) - computeKnownBitsFromCmp( - V, Invert ? Cmp->getInversePredicate() : Cmp->getPredicate(), - Cmp->getOperand(0), Cmp->getOperand(1), Known, SQ); + computeKnownBitsFromICmpCond(V, Cmp, Known, SQ, Invert); } void llvm::computeKnownBitsFromContext(const Value *V, KnownBits &Known, @@ -815,8 +832,7 @@ void llvm::computeKnownBitsFromContext(const Value *V, KnownBits &Known, if (!isValidAssumeForContext(I, Q.CxtI, Q.DT)) continue; - computeKnownBitsFromCmp(V, Cmp->getPredicate(), Cmp->getOperand(0), - Cmp->getOperand(1), Known, Q); + computeKnownBitsFromICmpCond(V, Cmp, Known, Q, /*Invert=*/false); } // Conflicting assumption: Undefined behavior will occur on this execution @@ -9133,7 +9149,7 @@ addValueAffectedByCondition(Value *V, // Peek through unary operators to find the source of the condition. Value *Op; - if (match(I, m_PtrToInt(m_Value(Op)))) { + if (match(I, m_CombineOr(m_PtrToInt(m_Value(Op)), m_Trunc(m_Value(Op))))) { if (isa(Op) || isa(Op)) InsertAffected(Op); } diff --git a/llvm/test/Transforms/InstCombine/known-bits.ll b/llvm/test/Transforms/InstCombine/known-bits.ll index b658ee0d2ef4..58c283815cf9 100644 --- a/llvm/test/Transforms/InstCombine/known-bits.ll +++ b/llvm/test/Transforms/InstCombine/known-bits.ll @@ -343,5 +343,145 @@ exit: ret i8 %or2 } +define i32 @test_icmp_trunc1(i32 %x){ +; CHECK-LABEL: @test_icmp_trunc1( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[Y:%.*]] = trunc i32 [[X:%.*]] to i16 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i16 [[Y]], 7 +; CHECK-NEXT: br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]] +; CHECK: then: +; CHECK-NEXT: ret i32 7 +; CHECK: else: +; CHECK-NEXT: ret i32 0 +; +entry: + %y = trunc i32 %x to i16 + %cmp = icmp eq i16 %y, 7 + br i1 %cmp, label %then, label %else +then: + %z = and i32 %x, 15 + ret i32 %z +else: + ret i32 0 +} + +define i32 @test_icmp_trunc_assume(i32 %x){ +; CHECK-LABEL: @test_icmp_trunc_assume( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[Y:%.*]] = trunc i32 [[X:%.*]] to i16 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i16 [[Y]], 7 +; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]]) +; CHECK-NEXT: ret i32 7 +; +entry: + %y = trunc i32 %x to i16 + %cmp = icmp eq i16 %y, 7 + call void @llvm.assume(i1 %cmp) + %z = and i32 %x, 15 + ret i32 %z +} + +define i64 @test_icmp_trunc2(i64 %x) { +; CHECK-LABEL: @test_icmp_trunc2( +; CHECK-NEXT: [[CONV:%.*]] = trunc i64 [[X:%.*]] to i32 +; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[CONV]], 12 +; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] +; CHECK: if.then: +; CHECK-NEXT: [[SEXT:%.*]] = and i64 [[X]], 2147483647 +; CHECK-NEXT: ret i64 [[SEXT]] +; CHECK: if.else: +; CHECK-NEXT: ret i64 0 +; + %conv = trunc i64 %x to i32 + %cmp = icmp sgt i32 %conv, 12 + br i1 %cmp, label %if.then, label %if.else + +if.then: + %sext = shl i64 %x, 32 + %ret = ashr exact i64 %sext, 32 + ret i64 %ret +if.else: + ret i64 0 +} + +define i64 @test_icmp_trunc3(i64 %n) { +; CHECK-LABEL: @test_icmp_trunc3( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CONV:%.*]] = trunc i64 [[N:%.*]] to i32 +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[CONV]], 96 +; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] +; CHECK: if.then: +; CHECK-NEXT: [[RET:%.*]] = and i64 [[N]], 127 +; CHECK-NEXT: ret i64 [[RET]] +; CHECK: if.else: +; CHECK-NEXT: ret i64 0 +; +entry: + %conv = trunc i64 %n to i32 + %cmp = icmp ult i32 %conv, 96 + br i1 %cmp, label %if.then, label %if.else + +if.then: + %ret = and i64 %n, 4294967295 + ret i64 %ret + +if.else: + ret i64 0 +} + +define i8 @test_icmp_trunc4(i64 %n) { +; CHECK-LABEL: @test_icmp_trunc4( +; CHECK-NEXT: [[CONV:%.*]] = trunc i64 [[N:%.*]] to i32 +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[CONV]], 10 +; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] +; CHECK: if.then: +; CHECK-NEXT: [[CONV2:%.*]] = trunc i64 [[N]] to i8 +; CHECK-NEXT: [[ADD:%.*]] = or disjoint i8 [[CONV2]], 48 +; CHECK-NEXT: ret i8 [[ADD]] +; CHECK: if.else: +; CHECK-NEXT: ret i8 0 +; + %conv = trunc i64 %n to i32 + %cmp = icmp ult i32 %conv, 10 + br i1 %cmp, label %if.then, label %if.else + +if.then: + %conv2 = trunc i64 %n to i8 + %add = add i8 %conv2, 48 + ret i8 %add + +if.else: + ret i8 0 +} + +define i64 @test_icmp_trunc5(i64 %n) { +; CHECK-LABEL: @test_icmp_trunc5( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[SHR:%.*]] = ashr i64 [[N:%.*]], 47 +; CHECK-NEXT: [[CONV1:%.*]] = trunc i64 [[SHR]] to i32 +; CHECK-NEXT: [[CMP:%.*]] = icmp ugt i32 [[CONV1]], -13 +; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] +; CHECK: if.then: +; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[SHR]], 15 +; CHECK-NEXT: [[NOT:%.*]] = xor i64 [[TMP0]], 15 +; CHECK-NEXT: ret i64 [[NOT]] +; CHECK: if.else: +; CHECK-NEXT: ret i64 13 +; +entry: + %shr = ashr i64 %n, 47 + %conv1 = trunc i64 %shr to i32 + %cmp = icmp ugt i32 %conv1, -13 + br i1 %cmp, label %if.then, label %if.else + +if.then: + %and = and i64 %shr, 4294967295 + %not = xor i64 %and, 4294967295 + ret i64 %not + +if.else: + ret i64 13 +} + declare void @use(i1) declare void @sink(i8) -- GitLab From c371ee9da9c4fa4b682baf47ce5c2a6b6f4c74bc Mon Sep 17 00:00:00 2001 From: Zequan Wu Date: Wed, 6 Mar 2024 12:08:50 -0500 Subject: [PATCH 323/929] [Profile][Windows] Fix flakyness when checking existence of binary id (#84196) There is a small chance that binary id starting with 0 (1/256). It is not sufficient to just check the first byte. --- compiler-rt/lib/profile/InstrProfilingPlatformWindows.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c index b9642ca7f681..9421f67b768e 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c @@ -96,9 +96,10 @@ ValueProfNode *EndVNode = &VNodesEnd; /* lld-link provides __buildid symbol which ponits to the 16 bytes build id when * using /build-id flag. https://lld.llvm.org/windows_support.html#lld-flags */ #define BUILD_ID_LEN 16 -COMPILER_RT_WEAK uint8_t __buildid[BUILD_ID_LEN]; +COMPILER_RT_WEAK uint8_t __buildid[BUILD_ID_LEN] = {0}; COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) { - if (*__buildid) { + static const uint8_t zeros[BUILD_ID_LEN] = {0}; + if (memcmp(__buildid, zeros, BUILD_ID_LEN) != 0) { if (Writer && lprofWriteOneBinaryId(Writer, BUILD_ID_LEN, __buildid, 0) == -1) return -1; -- GitLab From 0bd9255f8ad9f321dd606365e2bc28447a9976cb Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 6 Mar 2024 17:23:15 +0000 Subject: [PATCH 324/929] [X86] Improve KnownBits for X86ISD::PSADBW nodes (#83830) Don't just return the known zero upperbits, compute the absdiff Knownbits and perform the horizontal sum. Add implementations that handle both the X86ISD::PSADBW nodes and the INTRINSIC_WO_CHAIN intrinsics (pre-legalization). --- llvm/lib/Target/X86/X86ISelLowering.cpp | 46 ++++++++++++++-- llvm/test/CodeGen/X86/psadbw.ll | 72 +++++++------------------ llvm/test/CodeGen/X86/sad.ll | 16 ++---- 3 files changed, 63 insertions(+), 71 deletions(-) diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index eebb7cf91c42..94c4bbc4a099 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -36738,6 +36738,26 @@ X86TargetLowering::targetShrinkDemandedConstant(SDValue Op, return TLO.CombineTo(Op, NewOp); } +static void computeKnownBitsForPSADBW(SDValue LHS, SDValue RHS, + KnownBits &Known, + const APInt &DemandedElts, + const SelectionDAG &DAG, unsigned Depth) { + KnownBits Known2; + unsigned NumSrcElts = LHS.getValueType().getVectorNumElements(); + APInt DemandedSrcElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts); + Known = DAG.computeKnownBits(RHS, DemandedSrcElts, Depth + 1); + Known2 = DAG.computeKnownBits(LHS, DemandedSrcElts, Depth + 1); + Known = KnownBits::absdiff(Known, Known2).zext(16); + // Known = (((D0 + D1) + (D2 + D3)) + ((D4 + D5) + (D6 + D7))) + Known = KnownBits::computeForAddSub(/*Add=*/true, /*NSW=*/true, /*NUW=*/true, + Known, Known); + Known = KnownBits::computeForAddSub(/*Add=*/true, /*NSW=*/true, /*NUW=*/true, + Known, Known); + Known = KnownBits::computeForAddSub(/*Add=*/true, /*NSW=*/true, /*NUW=*/true, + Known, Known); + Known = Known.zext(64); +} + void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, @@ -36887,12 +36907,13 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op, break; } case X86ISD::PSADBW: { + SDValue LHS = Op.getOperand(0); + SDValue RHS = Op.getOperand(1); assert(VT.getScalarType() == MVT::i64 && - Op.getOperand(0).getValueType().getScalarType() == MVT::i8 && + LHS.getValueType() == RHS.getValueType() && + LHS.getValueType().getScalarType() == MVT::i8 && "Unexpected PSADBW types"); - - // PSADBW - fills low 16 bits and zeros upper 48 bits of each i64 result. - Known.Zero.setBitsFrom(16); + computeKnownBitsForPSADBW(LHS, RHS, Known, DemandedElts, DAG, Depth); break; } case X86ISD::PCMPGT: @@ -37046,6 +37067,23 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op, } break; } + case ISD::INTRINSIC_WO_CHAIN: { + switch (Op->getConstantOperandVal(0)) { + case Intrinsic::x86_sse2_psad_bw: + case Intrinsic::x86_avx2_psad_bw: + case Intrinsic::x86_avx512_psad_bw_512: { + SDValue LHS = Op.getOperand(1); + SDValue RHS = Op.getOperand(2); + assert(VT.getScalarType() == MVT::i64 && + LHS.getValueType() == RHS.getValueType() && + LHS.getValueType().getScalarType() == MVT::i8 && + "Unexpected PSADBW types"); + computeKnownBitsForPSADBW(LHS, RHS, Known, DemandedElts, DAG, Depth); + break; + } + } + break; + } } // Handle target shuffles. diff --git a/llvm/test/CodeGen/X86/psadbw.ll b/llvm/test/CodeGen/X86/psadbw.ll index 8141b22d321f..354fb8b92e9d 100644 --- a/llvm/test/CodeGen/X86/psadbw.ll +++ b/llvm/test/CodeGen/X86/psadbw.ll @@ -50,21 +50,14 @@ define i64 @combine_psadbw_demandedelt(<16 x i8> %0, <16 x i8> %1) nounwind { define <2 x i64> @combine_psadbw_cmp_knownbits(<16 x i8> %a0) nounwind { ; X86-SSE-LABEL: combine_psadbw_cmp_knownbits: ; X86-SSE: # %bb.0: -; X86-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 -; X86-SSE-NEXT: pxor %xmm1, %xmm1 -; X86-SSE-NEXT: psadbw %xmm0, %xmm1 -; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] -; X86-SSE-NEXT: por {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 -; X86-SSE-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 +; X86-SSE-NEXT: xorps %xmm0, %xmm0 ; X86-SSE-NEXT: retl ; ; X64-SSE-LABEL: combine_psadbw_cmp_knownbits: ; X64-SSE: # %bb.0: ; X64-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; X64-SSE-NEXT: pxor %xmm1, %xmm1 -; X64-SSE-NEXT: psadbw %xmm0, %xmm1 -; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2] -; X64-SSE-NEXT: por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; X64-SSE-NEXT: psadbw %xmm1, %xmm0 ; X64-SSE-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; X64-SSE-NEXT: retq ; @@ -82,28 +75,15 @@ define <2 x i64> @combine_psadbw_cmp_knownbits(<16 x i8> %a0) nounwind { ret <2 x i64> %ext } -; TODO: No need to scalarize the sitofp as the PSADBW results are smaller than i32. +; No need to scalarize the sitofp as the PSADBW results are smaller than i32. define <2 x double> @combine_psadbw_sitofp_knownbits(<16 x i8> %a0) nounwind { ; X86-SSE-LABEL: combine_psadbw_sitofp_knownbits: ; X86-SSE: # %bb.0: -; X86-SSE-NEXT: pushl %ebp -; X86-SSE-NEXT: movl %esp, %ebp -; X86-SSE-NEXT: andl $-8, %esp -; X86-SSE-NEXT: subl $32, %esp ; X86-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 ; X86-SSE-NEXT: pxor %xmm1, %xmm1 ; X86-SSE-NEXT: psadbw %xmm0, %xmm1 -; X86-SSE-NEXT: movq %xmm1, {{[0-9]+}}(%esp) -; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3] -; X86-SSE-NEXT: movq %xmm0, {{[0-9]+}}(%esp) -; X86-SSE-NEXT: fildll {{[0-9]+}}(%esp) -; X86-SSE-NEXT: fstpl {{[0-9]+}}(%esp) -; X86-SSE-NEXT: fildll {{[0-9]+}}(%esp) -; X86-SSE-NEXT: fstpl (%esp) -; X86-SSE-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero -; X86-SSE-NEXT: movhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1] -; X86-SSE-NEXT: movl %ebp, %esp -; X86-SSE-NEXT: popl %ebp +; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3] +; X86-SSE-NEXT: cvtdq2pd %xmm0, %xmm0 ; X86-SSE-NEXT: retl ; ; X64-SSE-LABEL: combine_psadbw_sitofp_knownbits: @@ -111,14 +91,8 @@ define <2 x double> @combine_psadbw_sitofp_knownbits(<16 x i8> %a0) nounwind { ; X64-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; X64-SSE-NEXT: pxor %xmm1, %xmm1 ; X64-SSE-NEXT: psadbw %xmm0, %xmm1 -; X64-SSE-NEXT: movd %xmm1, %eax -; X64-SSE-NEXT: xorps %xmm0, %xmm0 -; X64-SSE-NEXT: cvtsi2sd %eax, %xmm0 -; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3] -; X64-SSE-NEXT: movd %xmm1, %eax -; X64-SSE-NEXT: xorps %xmm1, %xmm1 -; X64-SSE-NEXT: cvtsi2sd %eax, %xmm1 -; X64-SSE-NEXT: unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3] +; X64-SSE-NEXT: cvtdq2pd %xmm0, %xmm0 ; X64-SSE-NEXT: retq ; ; AVX2-LABEL: combine_psadbw_sitofp_knownbits: @@ -126,10 +100,8 @@ define <2 x double> @combine_psadbw_sitofp_knownbits(<16 x i8> %a0) nounwind { ; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 ; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0 -; AVX2-NEXT: vcvtdq2pd %xmm0, %xmm1 -; AVX2-NEXT: vpextrq $1, %xmm0, %rax -; AVX2-NEXT: vcvtsi2sd %eax, %xmm2, %xmm0 -; AVX2-NEXT: vunpcklpd {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; AVX2-NEXT: vcvtdq2pd %xmm0, %xmm0 ; AVX2-NEXT: retq %mask = and <16 x i8> %a0, %sad = tail call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> %mask, <16 x i8> zeroinitializer) @@ -137,28 +109,24 @@ define <2 x double> @combine_psadbw_sitofp_knownbits(<16 x i8> %a0) nounwind { ret <2 x double> %cvt } -; TODO: Convert from uitofp to sitofp as the PSADBW results are zero-extended. +; Convert from uitofp to sitofp as the PSADBW results are zero-extended. define <2 x double> @combine_psadbw_uitofp_knownbits(<16 x i8> %a0) nounwind { ; X86-SSE-LABEL: combine_psadbw_uitofp_knownbits: ; X86-SSE: # %bb.0: ; X86-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 ; X86-SSE-NEXT: pxor %xmm1, %xmm1 -; X86-SSE-NEXT: psadbw %xmm1, %xmm0 -; X86-SSE-NEXT: por {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 -; X86-SSE-NEXT: movapd {{.*#+}} xmm1 = [0,1160773632,0,1160773632] -; X86-SSE-NEXT: subpd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1 -; X86-SSE-NEXT: addpd %xmm1, %xmm0 +; X86-SSE-NEXT: psadbw %xmm0, %xmm1 +; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3] +; X86-SSE-NEXT: cvtdq2pd %xmm0, %xmm0 ; X86-SSE-NEXT: retl ; ; X64-SSE-LABEL: combine_psadbw_uitofp_knownbits: ; X64-SSE: # %bb.0: ; X64-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; X64-SSE-NEXT: pxor %xmm1, %xmm1 -; X64-SSE-NEXT: psadbw %xmm1, %xmm0 -; X64-SSE-NEXT: por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; X64-SSE-NEXT: movapd {{.*#+}} xmm1 = [4985484787499139072,4985484787499139072] -; X64-SSE-NEXT: subpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; X64-SSE-NEXT: addpd %xmm1, %xmm0 +; X64-SSE-NEXT: psadbw %xmm0, %xmm1 +; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3] +; X64-SSE-NEXT: cvtdq2pd %xmm0, %xmm0 ; X64-SSE-NEXT: retq ; ; AVX2-LABEL: combine_psadbw_uitofp_knownbits: @@ -166,12 +134,8 @@ define <2 x double> @combine_psadbw_uitofp_knownbits(<16 x i8> %a0) nounwind { ; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 ; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0 -; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] -; AVX2-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX2-NEXT: vmovddup {{.*#+}} xmm1 = [4985484787499139072,4985484787499139072] -; AVX2-NEXT: # xmm1 = mem[0,0] -; AVX2-NEXT: vsubpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 -; AVX2-NEXT: vaddpd %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; AVX2-NEXT: vcvtdq2pd %xmm0, %xmm0 ; AVX2-NEXT: retq %mask = and <16 x i8> %a0, %sad = tail call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> %mask, <16 x i8> zeroinitializer) diff --git a/llvm/test/CodeGen/X86/sad.ll b/llvm/test/CodeGen/X86/sad.ll index 2a33e75a8357..ca319687da54 100644 --- a/llvm/test/CodeGen/X86/sad.ll +++ b/llvm/test/CodeGen/X86/sad.ll @@ -989,9 +989,7 @@ define dso_local i32 @sad_unroll_nonzero_initial(ptr %arg, ptr %arg1, ptr %arg2, ; SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3] ; SSE2-NEXT: paddd %xmm2, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1] -; SSE2-NEXT: paddd %xmm0, %xmm1 -; SSE2-NEXT: movd %xmm1, %eax +; SSE2-NEXT: movd %xmm0, %eax ; SSE2-NEXT: retq ; ; AVX-LABEL: sad_unroll_nonzero_initial: @@ -1053,9 +1051,7 @@ define dso_local i32 @sad_double_reduction(ptr %arg, ptr %arg1, ptr %arg2, ptr % ; SSE2-NEXT: paddd %xmm1, %xmm2 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3] ; SSE2-NEXT: paddd %xmm2, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1] -; SSE2-NEXT: por %xmm0, %xmm1 -; SSE2-NEXT: movd %xmm1, %eax +; SSE2-NEXT: movd %xmm0, %eax ; SSE2-NEXT: retq ; ; AVX-LABEL: sad_double_reduction: @@ -1067,8 +1063,6 @@ define dso_local i32 @sad_double_reduction(ptr %arg, ptr %arg1, ptr %arg2, ptr % ; AVX-NEXT: vpaddd %xmm0, %xmm1, %xmm0 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1] -; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 ; AVX-NEXT: vmovd %xmm0, %eax ; AVX-NEXT: retq bb: @@ -1115,9 +1109,7 @@ define dso_local i32 @sad_double_reduction_abs(ptr %arg, ptr %arg1, ptr %arg2, p ; SSE2-NEXT: paddd %xmm1, %xmm2 ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3] ; SSE2-NEXT: paddd %xmm2, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1] -; SSE2-NEXT: por %xmm0, %xmm1 -; SSE2-NEXT: movd %xmm1, %eax +; SSE2-NEXT: movd %xmm0, %eax ; SSE2-NEXT: retq ; ; AVX-LABEL: sad_double_reduction_abs: @@ -1129,8 +1121,6 @@ define dso_local i32 @sad_double_reduction_abs(ptr %arg, ptr %arg1, ptr %arg2, p ; AVX-NEXT: vpaddd %xmm0, %xmm1, %xmm0 ; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1] -; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 ; AVX-NEXT: vmovd %xmm0, %eax ; AVX-NEXT: retq bb: -- GitLab From deff460b46dfcc8d6d5917a2b78c0d52edbe4afb Mon Sep 17 00:00:00 2001 From: Krzysztof Parzyszek Date: Wed, 6 Mar 2024 11:33:57 -0600 Subject: [PATCH 325/929] [Frontend] Fix build break after 67c82d6ffb4b Remove the `default` label from a switch that covers all cases. --- llvm/utils/TableGen/DirectiveEmitter.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llvm/utils/TableGen/DirectiveEmitter.cpp b/llvm/utils/TableGen/DirectiveEmitter.cpp index 5a35d320227d..e0edf1720f8a 100644 --- a/llvm/utils/TableGen/DirectiveEmitter.cpp +++ b/llvm/utils/TableGen/DirectiveEmitter.cpp @@ -655,9 +655,8 @@ static void GenerateGetDirectiveAssociation(const DirectiveLanguage &DirLang, << "::" << getAssocName(F->second) << ";\n"; } } - OS << " default:\n"; - OS << " llvm_unreachable(\"Unexpected directive\");\n"; OS << " } // switch(Dir)\n"; + OS << " llvm_unreachable(\"Unexpected directive\");\n"; OS << "}\n"; } -- GitLab From ecea949f0ac713a625a3c42495b93b472315b500 Mon Sep 17 00:00:00 2001 From: Krzysztof Parzyszek Date: Wed, 6 Mar 2024 11:36:57 -0600 Subject: [PATCH 326/929] Revert "[Frontend] Fix build break after 67c82d6ffb4b" This reverts commit deff460b46dfcc8d6d5917a2b78c0d52edbe4afb. Committed this from a different computer with my old (invalid) email address. --- llvm/utils/TableGen/DirectiveEmitter.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llvm/utils/TableGen/DirectiveEmitter.cpp b/llvm/utils/TableGen/DirectiveEmitter.cpp index e0edf1720f8a..5a35d320227d 100644 --- a/llvm/utils/TableGen/DirectiveEmitter.cpp +++ b/llvm/utils/TableGen/DirectiveEmitter.cpp @@ -655,8 +655,9 @@ static void GenerateGetDirectiveAssociation(const DirectiveLanguage &DirLang, << "::" << getAssocName(F->second) << ";\n"; } } + OS << " default:\n"; + OS << " llvm_unreachable(\"Unexpected directive\");\n"; OS << " } // switch(Dir)\n"; - OS << " llvm_unreachable(\"Unexpected directive\");\n"; OS << "}\n"; } -- GitLab From 5dc5bfbb70f96cc001dc1173cbecfadab2e48fbf Mon Sep 17 00:00:00 2001 From: Krzysztof Parzyszek Date: Wed, 6 Mar 2024 11:33:57 -0600 Subject: [PATCH 327/929] [Frontend] Fix build break after 67c82d6ffb4b Remove the `default` label from a switch that covers all cases. --- llvm/utils/TableGen/DirectiveEmitter.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llvm/utils/TableGen/DirectiveEmitter.cpp b/llvm/utils/TableGen/DirectiveEmitter.cpp index 5a35d320227d..e0edf1720f8a 100644 --- a/llvm/utils/TableGen/DirectiveEmitter.cpp +++ b/llvm/utils/TableGen/DirectiveEmitter.cpp @@ -655,9 +655,8 @@ static void GenerateGetDirectiveAssociation(const DirectiveLanguage &DirLang, << "::" << getAssocName(F->second) << ";\n"; } } - OS << " default:\n"; - OS << " llvm_unreachable(\"Unexpected directive\");\n"; OS << " } // switch(Dir)\n"; + OS << " llvm_unreachable(\"Unexpected directive\");\n"; OS << "}\n"; } -- GitLab From 44be5a7fdc20a7f90d63dc18699a470e900bd3ba Mon Sep 17 00:00:00 2001 From: David Green Date: Wed, 6 Mar 2024 17:40:13 +0000 Subject: [PATCH 328/929] [Codegen] Make Width in getMemOperandsWithOffsetWidth a LocationSize. (#83875) This is another part of #70452 which makes getMemOperandsWithOffsetWidth use a LocationSize for Width, as opposed to the unsigned it currently uses. The advantages on it's own are not super high if getMemOperandsWithOffsetWidth usually uses known sizes, but if the values can come from an MMO it can help be more accurate in case they are Unknown (and in the future, scalable). --- llvm/include/llvm/CodeGen/TargetInstrInfo.h | 2 +- llvm/lib/CodeGen/MachineScheduler.cpp | 13 ++++---- llvm/lib/CodeGen/TargetInstrInfo.cpp | 2 +- llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 2 +- llvm/lib/Target/AArch64/AArch64InstrInfo.h | 2 +- .../lib/Target/AMDGPU/SIInsertHardClauses.cpp | 2 +- llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 6 ++-- llvm/lib/Target/AMDGPU/SIInstrInfo.h | 2 +- llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp | 8 ++--- llvm/lib/Target/Hexagon/HexagonInstrInfo.h | 4 +-- llvm/lib/Target/Hexagon/HexagonSubtarget.cpp | 11 ++++--- llvm/lib/Target/Lanai/LanaiInstrInfo.cpp | 11 ++++--- llvm/lib/Target/Lanai/LanaiInstrInfo.h | 4 +-- llvm/lib/Target/PowerPC/PPCInstrInfo.cpp | 15 +++++---- llvm/lib/Target/PowerPC/PPCInstrInfo.h | 4 +-- llvm/lib/Target/RISCV/RISCVInstrInfo.cpp | 11 ++++--- llvm/lib/Target/RISCV/RISCVInstrInfo.h | 4 +-- llvm/lib/Target/X86/X86InstrInfo.cpp | 2 +- llvm/lib/Target/X86/X86InstrInfo.h | 2 +- llvm/test/CodeGen/AMDGPU/cluster_stores.ll | 32 +++++++++---------- .../Target/RISCV/RISCVInstrInfoTest.cpp | 2 +- 21 files changed, 73 insertions(+), 68 deletions(-) diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h index e7787aafb98e..be4ee5b6f9e2 100644 --- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h +++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h @@ -1446,7 +1446,7 @@ public: /// abstraction that supports negative offsets. virtual bool getMemOperandsWithOffsetWidth( const MachineInstr &MI, SmallVectorImpl &BaseOps, - int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, + int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const { return false; } diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp index 3bbd126bdaf1..0d5bf3299387 100644 --- a/llvm/lib/CodeGen/MachineScheduler.cpp +++ b/llvm/lib/CodeGen/MachineScheduler.cpp @@ -1729,11 +1729,11 @@ class BaseMemOpClusterMutation : public ScheduleDAGMutation { SUnit *SU; SmallVector BaseOps; int64_t Offset; - unsigned Width; + LocationSize Width; bool OffsetIsScalable; MemOpInfo(SUnit *SU, ArrayRef BaseOps, - int64_t Offset, bool OffsetIsScalable, unsigned Width) + int64_t Offset, bool OffsetIsScalable, LocationSize Width) : SU(SU), BaseOps(BaseOps.begin(), BaseOps.end()), Offset(Offset), Width(Width), OffsetIsScalable(OffsetIsScalable) {} @@ -1866,11 +1866,12 @@ void BaseMemOpClusterMutation::clusterNeighboringMemOps( auto MemOpb = MemOpRecords[NextIdx]; unsigned ClusterLength = 2; - unsigned CurrentClusterBytes = MemOpa.Width + MemOpb.Width; + unsigned CurrentClusterBytes = MemOpa.Width.getValue().getKnownMinValue() + + MemOpb.Width.getValue().getKnownMinValue(); if (SUnit2ClusterInfo.count(MemOpa.SU->NodeNum)) { ClusterLength = SUnit2ClusterInfo[MemOpa.SU->NodeNum].first + 1; - CurrentClusterBytes = - SUnit2ClusterInfo[MemOpa.SU->NodeNum].second + MemOpb.Width; + CurrentClusterBytes = SUnit2ClusterInfo[MemOpa.SU->NodeNum].second + + MemOpb.Width.getValue().getKnownMinValue(); } if (!TII->shouldClusterMemOps(MemOpa.BaseOps, MemOpa.Offset, @@ -1940,7 +1941,7 @@ void BaseMemOpClusterMutation::collectMemOpRecords( SmallVector BaseOps; int64_t Offset; bool OffsetIsScalable; - unsigned Width; + LocationSize Width = 0; if (TII->getMemOperandsWithOffsetWidth(MI, BaseOps, Offset, OffsetIsScalable, Width, TRI)) { MemOpRecords.push_back( diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp index 4783742a14ad..5b02c1bc39c0 100644 --- a/llvm/lib/CodeGen/TargetInstrInfo.cpp +++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp @@ -1365,7 +1365,7 @@ bool TargetInstrInfo::getMemOperandWithOffset( const MachineInstr &MI, const MachineOperand *&BaseOp, int64_t &Offset, bool &OffsetIsScalable, const TargetRegisterInfo *TRI) const { SmallVector BaseOps; - unsigned Width; + LocationSize Width = 0; if (!getMemOperandsWithOffsetWidth(MI, BaseOps, Offset, OffsetIsScalable, Width, TRI) || BaseOps.size() != 1) diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp index 17e0e36ee682..5df691f35275 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -2675,7 +2675,7 @@ bool AArch64InstrInfo::isCandidateToMergeOrPair(const MachineInstr &MI) const { bool AArch64InstrInfo::getMemOperandsWithOffsetWidth( const MachineInstr &LdSt, SmallVectorImpl &BaseOps, - int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, + int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const { if (!LdSt.mayLoadOrStore()) return false; diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h index 6c6689091ead..2f10f80f4bdf 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h @@ -155,7 +155,7 @@ public: bool getMemOperandsWithOffsetWidth( const MachineInstr &MI, SmallVectorImpl &BaseOps, - int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, + int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const override; /// If \p OffsetIsScalable is set to 'true', the offset is scaled by `vscale`. diff --git a/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp b/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp index 01580fe345ba..dcc60765cc20 100644 --- a/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp +++ b/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp @@ -208,7 +208,7 @@ public: int64_t Dummy1; bool Dummy2; - unsigned Dummy3; + LocationSize Dummy3 = 0; SmallVector BaseOps; if (Type <= LAST_REAL_HARDCLAUSE_TYPE) { if (!SII->getMemOperandsWithOffsetWidth(MI, BaseOps, Dummy1, Dummy2, diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index a5b6c582a78c..ca2c2d87009f 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -360,7 +360,7 @@ static bool isStride64(unsigned Opc) { bool SIInstrInfo::getMemOperandsWithOffsetWidth( const MachineInstr &LdSt, SmallVectorImpl &BaseOps, - int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, + int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const { if (!LdSt.mayLoadOrStore()) return false; @@ -424,7 +424,7 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth( DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); Width = getOpSize(LdSt, DataOpIdx); DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1); - Width += getOpSize(LdSt, DataOpIdx); + Width = Width.getValue() + getOpSize(LdSt, DataOpIdx); } else { Width = getOpSize(LdSt, DataOpIdx); } @@ -3647,7 +3647,7 @@ bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa, const MachineInstr &MIb) const { SmallVector BaseOps0, BaseOps1; int64_t Offset0, Offset1; - unsigned Dummy0, Dummy1; + LocationSize Dummy0 = 0, Dummy1 = 0; bool Offset0IsScalable, Offset1IsScalable; if (!getMemOperandsWithOffsetWidth(MIa, BaseOps0, Offset0, Offset0IsScalable, Dummy0, &RI) || diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h index 82c6117292ae..4200e0c8a29e 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h @@ -240,7 +240,7 @@ public: bool getMemOperandsWithOffsetWidth( const MachineInstr &LdSt, SmallVectorImpl &BaseOps, int64_t &Offset, - bool &OffsetIsScalable, unsigned &Width, + bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const final; bool shouldClusterMemOps(ArrayRef BaseOps1, diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp index 619c7dc69f9b..b9bf26ba7cca 100644 --- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp +++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp @@ -3070,7 +3070,7 @@ bool HexagonInstrInfo::addLatencyToSchedule(const MachineInstr &MI1, /// Get the base register and byte offset of a load/store instr. bool HexagonInstrInfo::getMemOperandsWithOffsetWidth( const MachineInstr &LdSt, SmallVectorImpl &BaseOps, - int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, + int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const { OffsetIsScalable = false; const MachineOperand *BaseOp = getBaseAndOffset(LdSt, Offset, Width); @@ -3286,9 +3286,9 @@ unsigned HexagonInstrInfo::getAddrMode(const MachineInstr &MI) const { // returned in Offset and the access size is returned in AccessSize. // If the base operand has a subregister or the offset field does not contain // an immediate value, return nullptr. -MachineOperand *HexagonInstrInfo::getBaseAndOffset(const MachineInstr &MI, - int64_t &Offset, - unsigned &AccessSize) const { +MachineOperand * +HexagonInstrInfo::getBaseAndOffset(const MachineInstr &MI, int64_t &Offset, + LocationSize &AccessSize) const { // Return if it is not a base+offset type instruction or a MemOp. if (getAddrMode(MI) != HexagonII::BaseImmOffset && getAddrMode(MI) != HexagonII::BaseLongOffset && diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h index e496995d3ff1..4efc62fd717c 100644 --- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h +++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h @@ -208,7 +208,7 @@ public: bool getMemOperandsWithOffsetWidth( const MachineInstr &LdSt, SmallVectorImpl &BaseOps, int64_t &Offset, - bool &OffsetIsScalable, unsigned &Width, + bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const override; /// Reverses the branch condition of the specified condition list, @@ -437,7 +437,7 @@ public: unsigned getAddrMode(const MachineInstr &MI) const; MachineOperand *getBaseAndOffset(const MachineInstr &MI, int64_t &Offset, - unsigned &AccessSize) const; + LocationSize &AccessSize) const; SmallVector getBranchingInstrs(MachineBasicBlock& MBB) const; unsigned getCExtOpNum(const MachineInstr &MI) const; HexagonII::CompoundGroup diff --git a/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp b/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp index 4df811f188df..2d320e6b0cad 100644 --- a/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp +++ b/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp @@ -395,10 +395,11 @@ void HexagonSubtarget::BankConflictMutation::apply(ScheduleDAGInstrs *DAG) { HII.getAddrMode(L0) != HexagonII::BaseImmOffset) continue; int64_t Offset0; - unsigned Size0; + LocationSize Size0 = 0; MachineOperand *BaseOp0 = HII.getBaseAndOffset(L0, Offset0, Size0); // Is the access size is longer than the L1 cache line, skip the check. - if (BaseOp0 == nullptr || !BaseOp0->isReg() || Size0 >= 32) + if (BaseOp0 == nullptr || !BaseOp0->isReg() || !Size0.hasValue() || + Size0.getValue() >= 32) continue; // Scan only up to 32 instructions ahead (to avoid n^2 complexity). for (unsigned j = i+1, m = std::min(i+32, e); j != m; ++j) { @@ -408,10 +409,10 @@ void HexagonSubtarget::BankConflictMutation::apply(ScheduleDAGInstrs *DAG) { HII.getAddrMode(L1) != HexagonII::BaseImmOffset) continue; int64_t Offset1; - unsigned Size1; + LocationSize Size1 = 0; MachineOperand *BaseOp1 = HII.getBaseAndOffset(L1, Offset1, Size1); - if (BaseOp1 == nullptr || !BaseOp1->isReg() || Size1 >= 32 || - BaseOp0->getReg() != BaseOp1->getReg()) + if (BaseOp1 == nullptr || !BaseOp1->isReg() || !Size0.hasValue() || + Size1.getValue() >= 32 || BaseOp0->getReg() != BaseOp1->getReg()) continue; // Check bits 3 and 4 of the offset: if they differ, a bank conflict // is unlikely. diff --git a/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp b/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp index 4fe725b9457f..b8a37435f5a6 100644 --- a/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp +++ b/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp @@ -102,14 +102,15 @@ bool LanaiInstrInfo::areMemAccessesTriviallyDisjoint( const TargetRegisterInfo *TRI = &getRegisterInfo(); const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr; int64_t OffsetA = 0, OffsetB = 0; - unsigned int WidthA = 0, WidthB = 0; + LocationSize WidthA = 0, WidthB = 0; if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) && getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) { if (BaseOpA->isIdenticalTo(*BaseOpB)) { int LowOffset = std::min(OffsetA, OffsetB); int HighOffset = std::max(OffsetA, OffsetB); - int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB; - if (LowOffset + LowWidth <= HighOffset) + LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB; + if (LowWidth.hasValue() && + LowOffset + (int)LowWidth.getValue() <= HighOffset) return true; } } @@ -752,7 +753,7 @@ Register LanaiInstrInfo::isStoreToStackSlot(const MachineInstr &MI, bool LanaiInstrInfo::getMemOperandWithOffsetWidth( const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset, - unsigned &Width, const TargetRegisterInfo * /*TRI*/) const { + LocationSize &Width, const TargetRegisterInfo * /*TRI*/) const { // Handle only loads/stores with base register followed by immediate offset // and with add as ALU op. if (LdSt.getNumOperands() != 4) @@ -793,7 +794,7 @@ bool LanaiInstrInfo::getMemOperandWithOffsetWidth( bool LanaiInstrInfo::getMemOperandsWithOffsetWidth( const MachineInstr &LdSt, SmallVectorImpl &BaseOps, - int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, + int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const { switch (LdSt.getOpcode()) { default: diff --git a/llvm/lib/Target/Lanai/LanaiInstrInfo.h b/llvm/lib/Target/Lanai/LanaiInstrInfo.h index 189aedf07120..8ad2b9237c92 100644 --- a/llvm/lib/Target/Lanai/LanaiInstrInfo.h +++ b/llvm/lib/Target/Lanai/LanaiInstrInfo.h @@ -70,12 +70,12 @@ public: bool getMemOperandsWithOffsetWidth( const MachineInstr &LdSt, SmallVectorImpl &BaseOps, int64_t &Offset, - bool &OffsetIsScalable, unsigned &Width, + bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const override; bool getMemOperandWithOffsetWidth(const MachineInstr &LdSt, const MachineOperand *&BaseOp, - int64_t &Offset, unsigned &Width, + int64_t &Offset, LocationSize &Width, const TargetRegisterInfo *TRI) const; std::pair diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp index 1c610b269d32..5d37e929f875 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp @@ -2821,7 +2821,7 @@ bool PPCInstrInfo::optimizeCmpPostRA(MachineInstr &CmpMI) const { bool PPCInstrInfo::getMemOperandsWithOffsetWidth( const MachineInstr &LdSt, SmallVectorImpl &BaseOps, - int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, + int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const { const MachineOperand *BaseOp; OffsetIsScalable = false; @@ -2913,7 +2913,7 @@ bool PPCInstrInfo::shouldClusterMemOps( return false; int64_t Offset1 = 0, Offset2 = 0; - unsigned Width1 = 0, Width2 = 0; + LocationSize Width1 = 0, Width2 = 0; const MachineOperand *Base1 = nullptr, *Base2 = nullptr; if (!getMemOperandWithOffsetWidth(FirstLdSt, Base1, Offset1, Width1, TRI) || !getMemOperandWithOffsetWidth(SecondLdSt, Base2, Offset2, Width2, TRI) || @@ -2924,7 +2924,7 @@ bool PPCInstrInfo::shouldClusterMemOps( "getMemOperandWithOffsetWidth return incorrect base op"); // The caller should already have ordered FirstMemOp/SecondMemOp by offset. assert(Offset1 <= Offset2 && "Caller should have ordered offsets."); - return Offset1 + Width1 == Offset2; + return Offset1 + (int64_t)Width1.getValue() == Offset2; } /// GetInstSize - Return the number of bytes of code the specified @@ -5504,7 +5504,7 @@ MachineInstr *PPCInstrInfo::findLoopInstr( // memory width. Width is the size of memory that is being loaded/stored. bool PPCInstrInfo::getMemOperandWithOffsetWidth( const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset, - unsigned &Width, const TargetRegisterInfo *TRI) const { + LocationSize &Width, const TargetRegisterInfo *TRI) const { if (!LdSt.mayLoadOrStore() || LdSt.getNumExplicitOperands() != 3) return false; @@ -5542,14 +5542,15 @@ bool PPCInstrInfo::areMemAccessesTriviallyDisjoint( const TargetRegisterInfo *TRI = &getRegisterInfo(); const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr; int64_t OffsetA = 0, OffsetB = 0; - unsigned int WidthA = 0, WidthB = 0; + LocationSize WidthA = 0, WidthB = 0; if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) && getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) { if (BaseOpA->isIdenticalTo(*BaseOpB)) { int LowOffset = std::min(OffsetA, OffsetB); int HighOffset = std::max(OffsetA, OffsetB); - int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB; - if (LowOffset + LowWidth <= HighOffset) + LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB; + if (LowWidth.hasValue() && + LowOffset + (int)LowWidth.getValue() <= HighOffset) return true; } } diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.h b/llvm/lib/Target/PowerPC/PPCInstrInfo.h index 43ed668a941b..045932dc0d3b 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.h +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.h @@ -543,7 +543,7 @@ public: /// loaded/stored (e.g. 1, 2, 4, 8). bool getMemOperandWithOffsetWidth(const MachineInstr &LdSt, const MachineOperand *&BaseOp, - int64_t &Offset, unsigned &Width, + int64_t &Offset, LocationSize &Width, const TargetRegisterInfo *TRI) const; bool optimizeCmpPostRA(MachineInstr &MI) const; @@ -553,7 +553,7 @@ public: bool getMemOperandsWithOffsetWidth( const MachineInstr &LdSt, SmallVectorImpl &BaseOps, int64_t &Offset, - bool &OffsetIsScalable, unsigned &Width, + bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const override; /// Returns true if the two given memory operations should be scheduled diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp index e0564172182c..37a8079dcbf1 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp @@ -2195,7 +2195,7 @@ MachineInstr *RISCVInstrInfo::emitLdStWithAddr(MachineInstr &MemI, bool RISCVInstrInfo::getMemOperandsWithOffsetWidth( const MachineInstr &LdSt, SmallVectorImpl &BaseOps, - int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, + int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const { if (!LdSt.mayLoadOrStore()) return false; @@ -2300,7 +2300,7 @@ bool RISCVInstrInfo::shouldClusterMemOps( // function) and set it as appropriate. bool RISCVInstrInfo::getMemOperandWithOffsetWidth( const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset, - unsigned &Width, const TargetRegisterInfo *TRI) const { + LocationSize &Width, const TargetRegisterInfo *TRI) const { if (!LdSt.mayLoadOrStore()) return false; @@ -2339,14 +2339,15 @@ bool RISCVInstrInfo::areMemAccessesTriviallyDisjoint( const TargetRegisterInfo *TRI = STI.getRegisterInfo(); const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr; int64_t OffsetA = 0, OffsetB = 0; - unsigned int WidthA = 0, WidthB = 0; + LocationSize WidthA = 0, WidthB = 0; if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) && getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) { if (BaseOpA->isIdenticalTo(*BaseOpB)) { int LowOffset = std::min(OffsetA, OffsetB); int HighOffset = std::max(OffsetA, OffsetB); - int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB; - if (LowOffset + LowWidth <= HighOffset) + LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB; + if (LowWidth.hasValue() && + LowOffset + (int)LowWidth.getValue() <= HighOffset) return true; } } diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h index 2d5673425996..8a312ee5e779 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h @@ -156,7 +156,7 @@ public: bool getMemOperandsWithOffsetWidth( const MachineInstr &MI, SmallVectorImpl &BaseOps, - int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, + int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const override; bool shouldClusterMemOps(ArrayRef BaseOps1, @@ -168,7 +168,7 @@ public: bool getMemOperandWithOffsetWidth(const MachineInstr &LdSt, const MachineOperand *&BaseOp, - int64_t &Offset, unsigned &Width, + int64_t &Offset, LocationSize &Width, const TargetRegisterInfo *TRI) const; bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp index 25be1e2f6833..3f0557e651f8 100644 --- a/llvm/lib/Target/X86/X86InstrInfo.cpp +++ b/llvm/lib/Target/X86/X86InstrInfo.cpp @@ -4519,7 +4519,7 @@ bool X86InstrInfo::preservesZeroValueInReg( bool X86InstrInfo::getMemOperandsWithOffsetWidth( const MachineInstr &MemOp, SmallVectorImpl &BaseOps, - int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, + int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const { const MCInstrDesc &Desc = MemOp.getDesc(); int MemRefBegin = X86II::getMemoryOperandNo(Desc.TSFlags); diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h index 996a24d9e8a9..0e5fcbeda08f 100644 --- a/llvm/lib/Target/X86/X86InstrInfo.h +++ b/llvm/lib/Target/X86/X86InstrInfo.h @@ -375,7 +375,7 @@ public: bool getMemOperandsWithOffsetWidth( const MachineInstr &LdSt, SmallVectorImpl &BaseOps, int64_t &Offset, - bool &OffsetIsScalable, unsigned &Width, + bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const override; bool analyzeBranchPredicate(MachineBasicBlock &MBB, TargetInstrInfo::MachineBranchPredicate &MBP, diff --git a/llvm/test/CodeGen/AMDGPU/cluster_stores.ll b/llvm/test/CodeGen/AMDGPU/cluster_stores.ll index 46b237b71069..b6948dab6bf9 100644 --- a/llvm/test/CodeGen/AMDGPU/cluster_stores.ll +++ b/llvm/test/CodeGen/AMDGPU/cluster_stores.ll @@ -11,11 +11,11 @@ ; DBG-LABEL: cluster_load_cluster_store: -; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16 -; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4 -; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4 -; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4 -; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4 +; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(16) +; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(4) +; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(4) +; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(4) +; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(4) ; DBG: Cluster ld/st SU([[L1:[0-9]+]]) - SU([[L2:[0-9]+]]) ; DBG: Cluster ld/st SU([[L2]]) - SU([[L3:[0-9]+]]) @@ -136,11 +136,11 @@ bb: ; DBG-LABEL: cluster_load_valu_cluster_store: -; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16 -; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4 -; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4 -; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4 -; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4 +; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(16) +; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(4) +; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(4) +; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(4) +; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(4) ; DBG: Cluster ld/st SU([[L1:[0-9]+]]) - SU([[L2:[0-9]+]]) ; DBG: Cluster ld/st SU([[L2]]) - SU([[L3:[0-9]+]]) @@ -266,8 +266,8 @@ bb: ; Cluster loads from the same texture with different coordinates ; DBG-LABEL: cluster_image_load: -; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16 -; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16 +; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(16) +; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(16) ; DBG: {{^}}Cluster ld/st [[SU1:SU\([0-9]+\)]] - [[SU2:SU\([0-9]+\)]] ; DBG: {{^}}[[SU1]]: {{.*}} IMAGE_LOAD ; DBG: {{^}}[[SU2]]: {{.*}} IMAGE_LOAD @@ -335,8 +335,8 @@ entry: ; Don't cluster loads from different textures ; DBG-LABEL: no_cluster_image_load: -; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16 -; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16 +; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(16) +; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(16) ; DBG-NOT: {{^}}Cluster ld/st define amdgpu_ps void @no_cluster_image_load(<8 x i32> inreg %src1, <8 x i32> inreg %src2, <8 x i32> inreg %dst, i32 %x, i32 %y) { ; GFX9-LABEL: no_cluster_image_load: @@ -387,8 +387,8 @@ entry: ; Cluster loads from the same texture and sampler with different coordinates ; DBG-LABEL: cluster_image_sample: -; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16 -; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16 +; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(16) +; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: LocationSize::precise(16) ; DBG: {{^}}Cluster ld/st [[SU1:SU\([0-9]+\)]] - [[SU2:SU\([0-9]+\)]] ; DBG: {{^}}[[SU1]]: {{.*}} IMAGE_SAMPLE ; DBG: {{^}}[[SU2]]: {{.*}} IMAGE_SAMPLE diff --git a/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp b/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp index 5f3ce53f5d27..c690466071a1 100644 --- a/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp +++ b/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp @@ -163,7 +163,7 @@ TEST_P(RISCVInstrInfoTest, GetMemOperandsWithOffsetWidth) { DebugLoc DL; SmallVector BaseOps; - unsigned Width; + LocationSize Width = 0; int64_t Offset; bool OffsetIsScalable; -- GitLab From 8277e308c04b2157ea1076276e3eb5d08f533bac Mon Sep 17 00:00:00 2001 From: Tulio Magno Quites Machado Filho Date: Wed, 6 Mar 2024 14:53:05 -0300 Subject: [PATCH 329/929] [clang] Fix standalone execution in TableGen/target-builtins-prototype-parser.td (#84186) Use a path that works for both standalone as well as full repository builds. Fixes: 9b672de99760 ("[clang][Builtins] Parse clang extended vectors types. (#83584)") --- .../TableGen/target-builtins-prototype-parser.td | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/clang/test/TableGen/target-builtins-prototype-parser.td b/clang/test/TableGen/target-builtins-prototype-parser.td index 3d6c92341ac4..555aebb3ccfb 100644 --- a/clang/test/TableGen/target-builtins-prototype-parser.td +++ b/clang/test/TableGen/target-builtins-prototype-parser.td @@ -1,11 +1,11 @@ -// RUN: clang-tblgen -I %p/../../../clang/include/ %s --gen-clang-builtins | FileCheck %s -// RUN: not clang-tblgen -I %p/../../../clang/include/ %s --gen-clang-builtins -DERROR_EXPECTED_LANES 2>&1 | FileCheck %s --check-prefix=ERROR_EXPECTED_LANES -// RUN: not clang-tblgen -I %p/../../../clang/include/ %s --gen-clang-builtins -DERROR_EXPECTED_COMMA 2>&1 | FileCheck %s --check-prefix=ERROR_EXPECTED_COMMA -// RUN: not clang-tblgen -I %p/../../../clang/include/ %s --gen-clang-builtins -DERROR_EXPECTED_TYPE 2>&1 | FileCheck %s --check-prefix=ERROR_EXPECTED_TYPE -// RUN: not clang-tblgen -I %p/../../../clang/include/ %s --gen-clang-builtins -DERROR_EXPECTED_A 2>&1 | FileCheck %s --check-prefix=ERROR_EXPECTED_A -// RUN: not clang-tblgen -I %p/../../../clang/include/ %s --gen-clang-builtins -DERROR_EXPECTED_B 2>&1 | FileCheck %s --check-prefix=ERROR_EXPECTED_B -// RUN: not clang-tblgen -I %p/../../../clang/include/ %s --gen-clang-builtins -DERROR_EXPECTED_C 2>&1 | FileCheck %s --check-prefix=ERROR_EXPECTED_C -// RUN: not clang-tblgen -I %p/../../../clang/include/ %s --gen-clang-builtins -DERROR_EXPECTED_D 2>&1 | FileCheck %s --check-prefix=ERROR_EXPECTED_D +// RUN: clang-tblgen -I %p/../../include/ %s --gen-clang-builtins | FileCheck %s +// RUN: not clang-tblgen -I %p/../../include/ %s --gen-clang-builtins -DERROR_EXPECTED_LANES 2>&1 | FileCheck %s --check-prefix=ERROR_EXPECTED_LANES +// RUN: not clang-tblgen -I %p/../../include/ %s --gen-clang-builtins -DERROR_EXPECTED_COMMA 2>&1 | FileCheck %s --check-prefix=ERROR_EXPECTED_COMMA +// RUN: not clang-tblgen -I %p/../../include/ %s --gen-clang-builtins -DERROR_EXPECTED_TYPE 2>&1 | FileCheck %s --check-prefix=ERROR_EXPECTED_TYPE +// RUN: not clang-tblgen -I %p/../../include/ %s --gen-clang-builtins -DERROR_EXPECTED_A 2>&1 | FileCheck %s --check-prefix=ERROR_EXPECTED_A +// RUN: not clang-tblgen -I %p/../../include/ %s --gen-clang-builtins -DERROR_EXPECTED_B 2>&1 | FileCheck %s --check-prefix=ERROR_EXPECTED_B +// RUN: not clang-tblgen -I %p/../../include/ %s --gen-clang-builtins -DERROR_EXPECTED_C 2>&1 | FileCheck %s --check-prefix=ERROR_EXPECTED_C +// RUN: not clang-tblgen -I %p/../../include/ %s --gen-clang-builtins -DERROR_EXPECTED_D 2>&1 | FileCheck %s --check-prefix=ERROR_EXPECTED_D include "clang/Basic/BuiltinsBase.td" -- GitLab From fc9f1d49aae4328ef36e1d9ba606e93703fde970 Mon Sep 17 00:00:00 2001 From: Peiming Liu Date: Wed, 6 Mar 2024 09:57:41 -0800 Subject: [PATCH 330/929] =?UTF-8?q?[mlir][sparse]=20use=20a=20consistent?= =?UTF-8?q?=20order=20between=20[dis]assembleOp=20and=20sto=E2=80=A6=20(#8?= =?UTF-8?q?4079)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …rage layout. --- .../SparseTensor/IR/SparseTensorOps.td | 31 +++++---- .../Transforms/SparseAssembler.cpp | 33 +++------- .../Transforms/SparseGPUCodegen.cpp | 4 +- .../Transforms/SparseTensorCodegen.cpp | 10 +-- .../Transforms/SparseTensorConversion.cpp | 27 ++++---- .../SparseTensor/GPU/gpu_spgemm_lib.mlir | 2 +- mlir/test/Dialect/SparseTensor/external.mlir | 66 +++++++++---------- mlir/test/Dialect/SparseTensor/invalid.mlir | 31 +++++---- mlir/test/Dialect/SparseTensor/pack_copy.mlir | 38 +++++------ mlir/test/Dialect/SparseTensor/roundtrip.mlir | 25 +++---- .../Dialect/SparseTensor/sparse_pack.mlir | 17 ++--- .../SparseTensor/sparse_reinterpret_map.mlir | 15 +++-- .../Dialect/SparseTensor/CPU/sparse_pack.mlir | 31 +++++---- .../SparseTensor/CPU/sparse_pack_d.mlir | 13 ++-- 14 files changed, 168 insertions(+), 175 deletions(-) diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td index 3a5447d29f86..feed15d6af05 100644 --- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td +++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td @@ -55,8 +55,8 @@ def SparseTensor_NewOp : SparseTensor_Op<"new", [Pure]>, } def SparseTensor_AssembleOp : SparseTensor_Op<"assemble", [Pure]>, - Arguments<(ins TensorOf<[AnyType]>:$values, - Variadic>:$levels)>, + Arguments<(ins Variadic>:$levels, + TensorOf<[AnyType]>:$values)>, Results<(outs AnySparseTensor: $result)> { let summary = "Returns a sparse tensor assembled from the given values and levels"; @@ -96,20 +96,20 @@ def SparseTensor_AssembleOp : SparseTensor_Op<"assemble", [Pure]>, }]; let assemblyFormat = - "$values `,` $levels attr-dict" - "`:` type($values) `,` type($levels) `to` type($result)"; + "` ` `(` $levels `)` `,` $values attr-dict" + " `:` `(` type($levels) `)` `,` type($values) `to` type($result)"; let hasVerifier = 1; } def SparseTensor_DisassembleOp : SparseTensor_Op<"disassemble", [Pure, SameVariadicResultSize]>, Arguments<(ins AnySparseTensor:$tensor, - TensorOf<[AnyType]>:$out_values, - Variadic>:$out_levels)>, - Results<(outs TensorOf<[AnyType]>:$ret_values, - Variadic>:$ret_levels, - AnyIndexingScalarLike:$val_len, - Variadic:$lvl_lens)> { + Variadic>:$out_levels, + TensorOf<[AnyType]>:$out_values)>, + Results<(outs Variadic>:$ret_levels, + TensorOf<[AnyType]>:$ret_values, + Variadic:$lvl_lens, + AnyIndexingScalarLike:$val_len)> { let summary = "Returns the (values, coordinates) pair disassembled from the input tensor"; let description = [{ @@ -134,8 +134,9 @@ def SparseTensor_DisassembleOp : SparseTensor_Op<"disassemble", [Pure, SameVaria // |0.0, 0.0, 0.0, 0.0| %v, %p, %c, %v_len, %p_len, %c_len = sparse_tensor.disassemble %sp : tensor<3x4xf64, #COO> - outs(%od, %op, %oi : tensor<3xf64>, tensor<2xindex>, tensor<3x2xindex>) - -> tensor<3xf64>, (tensor<2xindex>, tensor<3x2xindex>), index, (index, index) + out_lvls(%op, %oi) : tensor<2xindex>, tensor<3x2xindex>, + out_vals(%od) : tensor<3xf64> -> + tensor<3xf64>, (tensor<2xindex>, tensor<3x2xindex>), index, (index, index) // %v = arith.constant dense<[ 1.1, 2.2, 3.3 ]> : tensor<3xf64> // %p = arith.constant dense<[ 0, 3 ]> : tensor<2xindex> // %c = arith.constant dense<[[0,0], [1,2], [1,3]]> : tensor<3x2xindex> @@ -147,8 +148,10 @@ def SparseTensor_DisassembleOp : SparseTensor_Op<"disassemble", [Pure, SameVaria let assemblyFormat = "$tensor `:` type($tensor) " - "`outs` `(` $out_values `,` $out_levels `:` type($out_values) `,` type($out_levels) `)` attr-dict" - "`->` type($ret_values) `,` `(` type($ret_levels) `)` `,` type($val_len) `,` `(` type($lvl_lens) `)`"; + "`out_lvls` `(` $out_levels `:` type($out_levels) `)` " + "`out_vals` `(` $out_values `:` type($out_values) `)` attr-dict" + "`->` `(` type($ret_levels) `)` `,` type($ret_values) `,` " + "`(` type($lvl_lens) `)` `,` type($val_len)"; let hasVerifier = 1; } diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp index b39a2d9c57d8..a91d32a23cac 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseAssembler.cpp @@ -31,22 +31,16 @@ static void convTypes(TypeRange types, SmallVectorImpl &convTypes, convTypes.push_back(type); continue; } - // Convert the external representation of the values array. + + // Convert the external representation of the position/coordinate array const SparseTensorType stt(cast(type)); - auto shape = stt.getBatchLvlShape(); - shape.push_back(ShapedType::kDynamic); - auto vtp = RankedTensorType::get(shape, stt.getElementType()); - convTypes.push_back(vtp); - if (extraTypes) - extraTypes->push_back(vtp); - - // Convert the external representation of the position/coordinate array. foreachFieldAndTypeInSparseTensor(stt, [&convTypes, extraTypes]( Type t, FieldIndex, SparseTensorFieldKind kind, Level, LevelType) { if (kind == SparseTensorFieldKind::CrdMemRef || - kind == SparseTensorFieldKind::PosMemRef) { + kind == SparseTensorFieldKind::PosMemRef || + kind == SparseTensorFieldKind::ValMemRef) { ShapedType st = t.cast(); auto rtp = RankedTensorType::get(st.getShape(), st.getElementType()); convTypes.push_back(rtp); @@ -70,29 +64,22 @@ static void convVals(OpBuilder &builder, Location loc, TypeRange types, toVals.push_back(fromVals[idx++]); continue; } - // Convert the external representation of the values array. + // Handle sparse data. auto rtp = cast(type); const SparseTensorType stt(rtp); - auto shape = stt.getBatchLvlShape(); - shape.push_back(ShapedType::kDynamic); SmallVector inputs; SmallVector retTypes; SmallVector cntTypes; - // Collect the external representation of the values array for - // input or the outgoing sparse tensor for output. - inputs.push_back(fromVals[idx++]); - if (!isIn) { - inputs.push_back(extraVals[extra++]); - retTypes.push_back(RankedTensorType::get(shape, stt.getElementType())); - cntTypes.push_back(builder.getIndexType()); // nnz - } + if (!isIn) + inputs.push_back(fromVals[idx++]); // The sparse tensor to disassemble // Collect the external representations of the pos/crd arrays. foreachFieldAndTypeInSparseTensor(stt, [&, isIn](Type t, FieldIndex, SparseTensorFieldKind kind, Level, LevelType) { if (kind == SparseTensorFieldKind::CrdMemRef || - kind == SparseTensorFieldKind::PosMemRef) { + kind == SparseTensorFieldKind::PosMemRef || + kind == SparseTensorFieldKind::ValMemRef) { if (isIn) { inputs.push_back(fromVals[idx++]); } else { @@ -100,7 +87,7 @@ static void convVals(OpBuilder &builder, Location loc, TypeRange types, auto rtp = RankedTensorType::get(st.getShape(), st.getElementType()); inputs.push_back(extraVals[extra++]); retTypes.push_back(rtp); - cntTypes.push_back(rtp.getElementType()); + cntTypes.push_back(builder.getIndexType()); } } return true; diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp index cb75f6a0ea88..8be76cac87f2 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp @@ -928,8 +928,8 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter, Value vt = rewriter.create(loc, valH); Value rt = rewriter.create(loc, rowH); Value ct = rewriter.create(loc, colH); - rewriter.replaceOpWithNewOp(op, c.getType(), vt, - ValueRange{rt, ct}); + rewriter.replaceOpWithNewOp(op, c.getType(), ValueRange{rt, ct}, + vt); return success(); } diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp index eb45a29fb389..44c5d4dbe485 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp @@ -1409,14 +1409,10 @@ struct SparseDisassembleOpConverter sz = desc.getValMemSize(rewriter, loc); src = desc.getValMemRef(); dst = genToMemref(rewriter, loc, op.getOutValues()); - // Values is the last field in descriptor, but it is the first - // operand in unpack operation. - // TODO: maybe change unpack/pack operation instead to be - // consistent. - retMem.insert(retMem.begin(), dst); + + retMem.push_back(dst); Type valLenTp = op.getValLen().getType(); - retLen.insert(retLen.begin(), - genScalarToTensor(rewriter, loc, sz, valLenTp)); + retLen.push_back(genScalarToTensor(rewriter, loc, sz, valLenTp)); } else { assert(fKind == SparseTensorFieldKind::PosMemRef || fKind == SparseTensorFieldKind::CrdMemRef); diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp index b0447b243661..010c3aa58b72 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp @@ -738,13 +738,7 @@ public: auto stt = getSparseTensorType(op.getTensor()); SmallVector retVal; SmallVector retLen; - // Get the values buffer first. - auto vals = genValuesCall(rewriter, loc, stt, adaptor.getTensor()); - auto valLenTp = op.getValLen().getType(); - auto valLen = linalg::createOrFoldDimOp(rewriter, loc, vals, 0); - retVal.push_back(vals); - retLen.push_back(genScalarToTensor(rewriter, loc, valLen, valLenTp)); - // Then get the positions and coordinates buffers. + // Get the positions and coordinates buffers. const Level lvlRank = stt.getLvlRank(); Level trailCOOLen = 0; for (Level l = 0; l < lvlRank; l++) { @@ -761,7 +755,7 @@ public: auto poss = genPositionsCall(rewriter, loc, stt, adaptor.getTensor(), l); auto posLen = linalg::createOrFoldDimOp(rewriter, loc, poss, 0); - auto posLenTp = op.getLvlLens().getTypes()[retLen.size() - 1]; + auto posLenTp = op.getLvlLens().getTypes()[retLen.size()]; retVal.push_back(poss); retLen.push_back(genScalarToTensor(rewriter, loc, posLen, posLenTp)); } @@ -769,7 +763,7 @@ public: auto crds = genCoordinatesCall(rewriter, loc, stt, adaptor.getTensor(), l); auto crdLen = linalg::createOrFoldDimOp(rewriter, loc, crds, 0); - auto crdLenTp = op.getLvlLens().getTypes()[retLen.size() - 1]; + auto crdLenTp = op.getLvlLens().getTypes()[retLen.size()]; retVal.push_back(crds); retLen.push_back(genScalarToTensor(rewriter, loc, crdLen, crdLenTp)); } @@ -784,14 +778,13 @@ public: auto poss = genPositionsCall(rewriter, loc, stt, adaptor.getTensor(), cooStartLvl); auto posLen = linalg::createOrFoldDimOp(rewriter, loc, poss, 0); - auto posLenTp = op.getLvlLens().getTypes()[retLen.size() - 1]; + auto posLenTp = op.getLvlLens().getTypes()[retLen.size()]; retVal.push_back(poss); retLen.push_back(genScalarToTensor(rewriter, loc, posLen, posLenTp)); // Coordinates, copied over with: // for (i = 0; i < crdLen; i++) // buf[i][0] = crd0[i]; buf[i][1] = crd1[i]; - auto buf = - genToMemref(rewriter, loc, op.getOutLevels()[retLen.size() - 1]); + auto buf = genToMemref(rewriter, loc, op.getOutLevels()[retLen.size()]); auto crds0 = genCoordinatesCall(rewriter, loc, stt, adaptor.getTensor(), cooStartLvl); auto crds1 = genCoordinatesCall(rewriter, loc, stt, adaptor.getTensor(), @@ -814,10 +807,17 @@ public: args[1] = one; rewriter.create(loc, c1, buf, args); rewriter.setInsertionPointAfter(forOp); - auto bufLenTp = op.getLvlLens().getTypes()[retLen.size() - 1]; + auto bufLenTp = op.getLvlLens().getTypes()[retLen.size()]; retVal.push_back(buf); retLen.push_back(genScalarToTensor(rewriter, loc, bufLen, bufLenTp)); } + // Get the values buffer last. + auto vals = genValuesCall(rewriter, loc, stt, adaptor.getTensor()); + auto valLenTp = op.getValLen().getType(); + auto valLen = linalg::createOrFoldDimOp(rewriter, loc, vals, 0); + retVal.push_back(vals); + retLen.push_back(genScalarToTensor(rewriter, loc, valLen, valLenTp)); + // Converts MemRefs back to Tensors. assert(retVal.size() + retLen.size() == op.getNumResults()); for (unsigned i = 0, sz = retVal.size(); i < sz; i++) { @@ -825,6 +825,7 @@ public: retVal[i] = rewriter.create(loc, op.getResultTypes()[i], tensor); } + // Appends the actual memory length used in each buffer returned. retVal.append(retLen.begin(), retLen.end()); rewriter.replaceOp(op, retVal); diff --git a/mlir/test/Dialect/SparseTensor/GPU/gpu_spgemm_lib.mlir b/mlir/test/Dialect/SparseTensor/GPU/gpu_spgemm_lib.mlir index 7ac37c1c4950..fa8ad1cc5060 100644 --- a/mlir/test/Dialect/SparseTensor/GPU/gpu_spgemm_lib.mlir +++ b/mlir/test/Dialect/SparseTensor/GPU/gpu_spgemm_lib.mlir @@ -85,7 +85,7 @@ // CHECK: %[[VAL_a2:.*]] = bufferization.to_tensor %[[VAL_83]] : memref // CHECK: %[[VAL_a3:.*]] = bufferization.to_tensor %[[VAL_81]] : memref // CHECK: %[[VAL_a4:.*]] = bufferization.to_tensor %[[VAL_82]] : memref -// CHECK: %[[VAL_a5:.*]] = sparse_tensor.assemble %[[VAL_a2]], %[[VAL_a3]], %[[VAL_a4]] : tensor, tensor, tensor to tensor<8x8xf32, #{{.*}}> +// CHECK: %[[VAL_a5:.*]] = sparse_tensor.assemble (%[[VAL_a3]], %[[VAL_a4]]), %[[VAL_a2]] : (tensor, tensor), tensor to tensor<8x8xf32, #{{.*}}> // CHECK: return %[[VAL_a5]] : tensor<8x8xf32, #{{.*}}> // CHECK: } func.func @matmulCSR(%A: tensor<8x8xf32, #CSR>, diff --git a/mlir/test/Dialect/SparseTensor/external.mlir b/mlir/test/Dialect/SparseTensor/external.mlir index b5701ad20242..435737fc0979 100644 --- a/mlir/test/Dialect/SparseTensor/external.mlir +++ b/mlir/test/Dialect/SparseTensor/external.mlir @@ -13,10 +13,10 @@ func.func @nop(%arg0: tensor<100xf32>) -> tensor<100xf32> { // ----- // CHECK-LABEL: func.func @sparse_in( -// CHECK-SAME: %[[A:.*]]: tensor, -// CHECK-SAME: %[[B:.*]]: tensor, -// CHECK-SAME: %[[C:.*]]: tensor) -> tensor<64x64xf32> { -// CHECK: %[[I:.*]] = sparse_tensor.assemble %[[A]], %[[B]], %[[C]] +// CHECK-SAME: %[[B:.*0]]: tensor, +// CHECK-SAME: %[[C:.*1]]: tensor, +// CHECK-SAME: %[[A:.*]]: tensor) -> tensor<64x64xf32> { +// CHECK: %[[I:.*]] = sparse_tensor.assemble (%[[B]], %[[C]]), %[[A]] // CHECK: %[[F:.*]] = call @_internal_sparse_in(%[[I]]) // CHECK: return %[[F]] : tensor<64x64xf32> // CHECK: } @@ -30,11 +30,11 @@ func.func @sparse_in(%arg0: tensor<64x64xf32, #sparse>) -> tensor<64x64xf32> { // ----- // CHECK-LABEL: func.func @sparse_in2( -// CHECK-SAME: %[[X:.*]]: tensor<100xf32>, -// CHECK-SAME: %[[A:.*]]: tensor, -// CHECK-SAME: %[[B:.*]]: tensor, -// CHECK-SAME: %[[C:.*]]: tensor) -> tensor<64x64xf32> { -// CHECK: %[[I:.*]] = sparse_tensor.assemble %[[A]], %[[B]], %[[C]] +// CHECK-SAME: %[[X:.*0]]: tensor<100xf32>, +// CHECK-SAME: %[[B:.*1]]: tensor, +// CHECK-SAME: %[[C:.*2]]: tensor, +// CHECK-SAME: %[[A:.*3]]: tensor) -> tensor<64x64xf32> { +// CHECK: %[[I:.*]] = sparse_tensor.assemble (%[[B]], %[[C]]), %[[A]] // CHECK: %[[F:.*]] = call @_internal_sparse_in2(%[[X]], %[[I]]) // CHECK: return %[[F]] : tensor<64x64xf32> // CHECK: } @@ -48,10 +48,10 @@ func.func @sparse_in2(%arg0: tensor<100xf32>, %arg1: tensor<64x64xf32, #sparse>) // ----- // CHECK-LABEL: func.func @sparse_out( -// CHECK-SAME: %[[X:.*]]: tensor<64x64xf32>, -// CHECK-SAME: %[[A:.*]]: tensor, -// CHECK-SAME: %[[B:.*]]: tensor, -// CHECK-SAME: %[[C:.*]]: tensor) -> (tensor, tensor, tensor) { +// CHECK-SAME: %[[X:.*0]]: tensor<64x64xf32>, +// CHECK-SAME: %[[B:.*1]]: tensor, +// CHECK-SAME: %[[C:.*2]]: tensor, +// CHECK-SAME: %[[A:.*3]]: tensor) // CHECK: %[[F:.*]] = call @_internal_sparse_out(%[[X]]) // CHECK: sparse_tensor.disassemble %[[F]] // CHECK: return @@ -66,10 +66,10 @@ func.func @sparse_out(%arg0: tensor<64x64xf32>) -> tensor<64x64xf32, #sparse> { // ----- // CHECK-LABEL: func.func @sparse_out2( -// CHECK-SAME: %[[X:.*]]: tensor<64x64xf32>, -// CHECK-SAME: %[[A:.*]]: tensor, -// CHECK-SAME: %[[B:.*]]: tensor, -// CHECK-SAME: %[[C:.*]]: tensor) -> (tensor<64x64xf32>, tensor, tensor, tensor) { +// CHECK-SAME: %[[X:.*0]]: tensor<64x64xf32>, +// CHECK-SAME: %[[B:.*1]]: tensor, +// CHECK-SAME: %[[C:.*2]]: tensor, +// CHECK-SAME: %[[A:.*3]]: tensor) // CHECK: %[[F:.*]]:2 = call @_internal_sparse_out2(%[[X]]) // CHECK: sparse_tensor.disassemble %[[F]]#1 // CHECK: return %[[F]]#0 @@ -84,13 +84,13 @@ func.func @sparse_out2(%arg0: tensor<64x64xf32>) -> (tensor<64x64xf32>, tensor<6 // ----- // CHECK-LABEL: func.func @sparse_inout( -// CHECK-SAME: %[[A:.*0]]: tensor, -// CHECK-SAME: %[[B:.*1]]: tensor, -// CHECK-SAME: %[[C:.*2]]: tensor, -// CHECK-SAME: %[[D:.*3]]: tensor, -// CHECK-SAME: %[[E:.*4]]: tensor, -// CHECK-SAME: %[[F:.*5]]: tensor) -> (tensor, tensor, tensor) { -// CHECK: %[[I:.*]] = sparse_tensor.assemble %[[A]], %[[B]], %[[C]] +// CHECK-SAME: %[[B:.*0]]: tensor, +// CHECK-SAME: %[[C:.*1]]: tensor, +// CHECK-SAME: %[[A:.*2]]: tensor, +// CHECK-SAME: %[[E:.*3]]: tensor, +// CHECK-SAME: %[[F:.*4]]: tensor, +// CHECK-SAME: %[[D:.*5]]: tensor) +// CHECK: %[[I:.*]] = sparse_tensor.assemble (%[[B]], %[[C]]), %[[A]] // CHECK: %[[F:.*]] = call @_internal_sparse_inout(%[[I]]) // CHECK: sparse_tensor.disassemble %[[F]] // CHECK: return @@ -104,15 +104,15 @@ func.func @sparse_inout(%arg0: tensor<64x64xf32, #sparse>) -> tensor<64x64xf32, // ----- // CHECK-LABEL: func.func @sparse_inout_coo_soa( -// CHECK-SAME: %[[A:.*0]]: tensor, -// CHECK-SAME: %[[B:.*1]]: tensor, -// CHECK-SAME: %[[C:.*2]]: tensor, -// CHECK-SAME: %[[D:.*3]]: tensor, -// CHECK-SAME: %[[E:.*4]]: tensor, -// CHECK-SAME: %[[F:.*5]]: tensor, -// CHECK-SAME: %[[G:.*6]]: tensor, -// CHECK-SAME: %[[H:.*7]]: tensor) -> (tensor, tensor, tensor, tensor) { -// CHECK: %[[I:.*]] = sparse_tensor.assemble %[[A]], %[[B]], %[[C]], %[[D]] +// CHECK-SAME: %[[B:.*0]]: tensor, +// CHECK-SAME: %[[C:.*1]]: tensor, +// CHECK-SAME: %[[D:.*2]]: tensor, +// CHECK-SAME: %[[A:.*3]]: tensor, +// CHECK-SAME: %[[F:.*4]]: tensor, +// CHECK-SAME: %[[G:.*5]]: tensor, +// CHECK-SAME: %[[H:.*6]]: tensor, +// CHECK-SAME: %[[E:.*7]]: tensor) +// CHECK: %[[I:.*]] = sparse_tensor.assemble (%[[B]], %[[C]], %[[D]]), %[[A]] // CHECK: %[[F:.*]] = call @_internal_sparse_inout_coo_soa(%[[I]]) // CHECK: sparse_tensor.disassemble %[[F]] // CHECK: return diff --git a/mlir/test/Dialect/SparseTensor/invalid.mlir b/mlir/test/Dialect/SparseTensor/invalid.mlir index 395b812a7685..eac97f702f58 100644 --- a/mlir/test/Dialect/SparseTensor/invalid.mlir +++ b/mlir/test/Dialect/SparseTensor/invalid.mlir @@ -13,8 +13,8 @@ func.func @invalid_new_dense(%arg0: !llvm.ptr) -> tensor<32xf32> { func.func @non_static_pack_ret(%values: tensor<6xf64>, %pos: tensor<2xi32>, %coordinates: tensor<6x1xi32>) -> tensor { // expected-error@+1 {{the sparse-tensor must have static shape}} - %0 = sparse_tensor.assemble %values, %pos, %coordinates - : tensor<6xf64>, tensor<2xi32>, tensor<6x1xi32> to tensor + %0 = sparse_tensor.assemble (%pos, %coordinates), %values + : (tensor<2xi32>, tensor<6x1xi32>), tensor<6xf64> to tensor return %0 : tensor } @@ -25,8 +25,8 @@ func.func @non_static_pack_ret(%values: tensor<6xf64>, %pos: tensor<2xi32>, %coo func.func @invalid_pack_type(%values: tensor<6xf64>, %pos: tensor<2xi32>, %coordinates: tensor<6x1xi32>) -> tensor<100xf32, #SparseVector> { // expected-error@+1 {{input/output element-types don't match}} - %0 = sparse_tensor.assemble %values, %pos, %coordinates - : tensor<6xf64>, tensor<2xi32>, tensor<6x1xi32> to tensor<100xf32, #SparseVector> + %0 = sparse_tensor.assemble (%pos, %coordinates), %values + : (tensor<2xi32>, tensor<6x1xi32>), tensor<6xf64> to tensor<100xf32, #SparseVector> return %0 : tensor<100xf32, #SparseVector> } @@ -37,8 +37,8 @@ func.func @invalid_pack_type(%values: tensor<6xf64>, %pos: tensor<2xi32>, %coord func.func @invalid_pack_type(%values: tensor<6xf64>, %pos: tensor<2xi32>, %coordinates: tensor<6x3xi32>) -> tensor<100x2xf64, #SparseVector> { // expected-error@+1 {{input/output trailing COO level-ranks don't match}} - %0 = sparse_tensor.assemble %values, %pos, %coordinates - : tensor<6xf64>, tensor<2xi32>, tensor<6x3xi32> to tensor<100x2xf64, #SparseVector> + %0 = sparse_tensor.assemble (%pos, %coordinates), %values + : (tensor<2xi32>, tensor<6x3xi32>), tensor<6xf64> to tensor<100x2xf64, #SparseVector> return %0 : tensor<100x2xf64, #SparseVector> } @@ -49,8 +49,8 @@ func.func @invalid_pack_type(%values: tensor<6xf64>, %pos: tensor<2xi32>, %coord func.func @invalid_pack_mis_position(%values: tensor<6xf64>, %coordinates: tensor<6xi32>) -> tensor<2x100xf64, #CSR> { // expected-error@+1 {{inconsistent number of fields between input/output}} - %0 = sparse_tensor.assemble %values, %coordinates - : tensor<6xf64>, tensor<6xi32> to tensor<2x100xf64, #CSR> + %0 = sparse_tensor.assemble (%coordinates), %values + : (tensor<6xi32>), tensor<6xf64> to tensor<2x100xf64, #CSR> return %0 : tensor<2x100xf64, #CSR> } @@ -61,8 +61,9 @@ func.func @invalid_pack_mis_position(%values: tensor<6xf64>, %coordinates: tenso func.func @invalid_unpack_type(%sp: tensor<100xf32, #SparseVector>, %values: tensor<6xf64>, %pos: tensor<2xi32>, %coordinates: tensor<6x1xi32>) { // expected-error@+1 {{input/output element-types don't match}} %rv, %rp, %rc, %vl, %pl, %cl = sparse_tensor.disassemble %sp : tensor<100xf32, #SparseVector> - outs(%values, %pos, %coordinates : tensor<6xf64>, tensor<2xi32>, tensor<6x1xi32>) - -> tensor<6xf64>, (tensor<2xi32>, tensor<6x1xi32>), index, (index, index) + out_lvls(%pos, %coordinates : tensor<2xi32>, tensor<6x1xi32>) + out_vals(%values : tensor<6xf64>) + -> (tensor<2xi32>, tensor<6x1xi32>), tensor<6xf64>, (index, index), index return } @@ -73,8 +74,9 @@ func.func @invalid_unpack_type(%sp: tensor<100xf32, #SparseVector>, %values: ten func.func @invalid_unpack_type(%sp: tensor<100x2xf64, #SparseVector>, %values: tensor<6xf64>, %pos: tensor<2xi32>, %coordinates: tensor<6x3xi32>) { // expected-error@+1 {{input/output trailing COO level-ranks don't match}} %rv, %rp, %rc, %vl, %pl, %cl = sparse_tensor.disassemble %sp : tensor<100x2xf64, #SparseVector> - outs(%values, %pos, %coordinates : tensor<6xf64>, tensor<2xi32>, tensor<6x3xi32>) - -> tensor<6xf64>, (tensor<2xi32>, tensor<6x3xi32>), index, (index, index) + out_lvls(%pos, %coordinates : tensor<2xi32>, tensor<6x3xi32> ) + out_vals(%values : tensor<6xf64>) + -> (tensor<2xi32>, tensor<6x3xi32>), tensor<6xf64>, (index, index), index return } @@ -85,8 +87,9 @@ func.func @invalid_unpack_type(%sp: tensor<100x2xf64, #SparseVector>, %values: t func.func @invalid_unpack_mis_position(%sp: tensor<2x100xf64, #CSR>, %values: tensor<6xf64>, %coordinates: tensor<6xi32>) { // expected-error@+1 {{inconsistent number of fields between input/output}} %rv, %rc, %vl, %pl = sparse_tensor.disassemble %sp : tensor<2x100xf64, #CSR> - outs(%values, %coordinates : tensor<6xf64>, tensor<6xi32>) - -> tensor<6xf64>, (tensor<6xi32>), index, (index) + out_lvls(%coordinates : tensor<6xi32>) + out_vals(%values : tensor<6xf64>) + -> (tensor<6xi32>), tensor<6xf64>, (index), index return } diff --git a/mlir/test/Dialect/SparseTensor/pack_copy.mlir b/mlir/test/Dialect/SparseTensor/pack_copy.mlir index e60f9bb7149b..ec8f0b531fb2 100644 --- a/mlir/test/Dialect/SparseTensor/pack_copy.mlir +++ b/mlir/test/Dialect/SparseTensor/pack_copy.mlir @@ -19,26 +19,25 @@ // This forces a copy for the values and positions. // // CHECK-LABEL: func.func @foo( -// CHECK-SAME: %[[VAL:.*]]: memref<3xf64>, // CHECK-SAME: %[[CRD:.*]]: memref<3xi32>, -// CHECK-SAME: %[[POS:.*]]: memref<11xi32>) -// CHECK: %[[ALLOC1:.*]] = memref.alloc() {alignment = 64 : i64} : memref<3xf64> -// CHECK: memref.copy %[[VAL]], %[[ALLOC1]] : memref<3xf64> to memref<3xf64> +// CHECK-SAME: %[[POS:.*]]: memref<11xi32>, +// CHECK-SAME: %[[VAL:.*]]: memref<3xf64>) // CHECK: %[[ALLOC2:.*]] = memref.alloc() {alignment = 64 : i64} : memref<11xi32> // CHECK: memref.copy %[[POS]], %[[ALLOC2]] : memref<11xi32> to memref<11xi32> +// CHECK: %[[ALLOC1:.*]] = memref.alloc() {alignment = 64 : i64} : memref<3xf64> +// CHECK: memref.copy %[[VAL]], %[[ALLOC1]] : memref<3xf64> to memref<3xf64> // CHECK-NOT: memref.copy // CHECK: return // -func.func @foo(%arg0: tensor<3xf64> {bufferization.writable = false}, - %arg1: tensor<3xi32> {bufferization.writable = false}, - %arg2: tensor<11xi32> {bufferization.writable = false}) -> (index) { +func.func @foo(%arg1: tensor<3xi32> {bufferization.writable = false}, + %arg2: tensor<11xi32> {bufferization.writable = false}, + %arg0: tensor<3xf64> {bufferization.writable = false}) -> (index) { // // Pack the buffers into a sparse tensors. // - %pack = sparse_tensor.assemble %arg0, %arg2, %arg1 - : tensor<3xf64>, - tensor<11xi32>, - tensor<3xi32> to tensor<10x10xf64, #CSR> + %pack = sparse_tensor.assemble (%arg2, %arg1), %arg0 + : (tensor<11xi32>, tensor<3xi32>), + tensor<3xf64> to tensor<10x10xf64, #CSR> // // Scale the sparse tensor "in-place" (this has no impact on the final @@ -64,22 +63,21 @@ func.func @foo(%arg0: tensor<3xf64> {bufferization.writable = false}, // Pass in the buffers of the sparse tensor, marked writable. // // CHECK-LABEL: func.func @bar( -// CHECK-SAME: %[[VAL:.*]]: memref<3xf64>, // CHECK-SAME: %[[CRD:.*]]: memref<3xi32>, -// CHECK-SAME: %[[POS:.*]]: memref<11xi32>) +// CHECK-SAME: %[[POS:.*]]: memref<11xi32>, +// CHECK-SAME: %[[VAL:.*]]: memref<3xf64>) // CHECK-NOT: memref.copy // CHECK: return // -func.func @bar(%arg0: tensor<3xf64> {bufferization.writable = true}, - %arg1: tensor<3xi32> {bufferization.writable = true}, - %arg2: tensor<11xi32> {bufferization.writable = true}) -> (index) { +func.func @bar(%arg1: tensor<3xi32> {bufferization.writable = true}, + %arg2: tensor<11xi32> {bufferization.writable = true}, + %arg0: tensor<3xf64> {bufferization.writable = true}) -> (index) { // // Pack the buffers into a sparse tensors. // - %pack = sparse_tensor.assemble %arg0, %arg2, %arg1 - : tensor<3xf64>, - tensor<11xi32>, - tensor<3xi32> to tensor<10x10xf64, #CSR> + %pack = sparse_tensor.assemble (%arg2, %arg1), %arg0 + : (tensor<11xi32>, tensor<3xi32>), + tensor<3xf64> to tensor<10x10xf64, #CSR> // // Scale the sparse tensor "in-place" (this has no impact on the final diff --git a/mlir/test/Dialect/SparseTensor/roundtrip.mlir b/mlir/test/Dialect/SparseTensor/roundtrip.mlir index f4a58df1d4d2..41094fbad921 100644 --- a/mlir/test/Dialect/SparseTensor/roundtrip.mlir +++ b/mlir/test/Dialect/SparseTensor/roundtrip.mlir @@ -16,14 +16,14 @@ func.func @sparse_new(%arg0: !llvm.ptr) -> tensor<128xf64, #SparseVector> { #SparseVector = #sparse_tensor.encoding<{map = (d0) -> (d0 : compressed), posWidth=32, crdWidth=32}> // CHECK-LABEL: func @sparse_pack( -// CHECK-SAME: %[[D:.*]]: tensor<6xf64>, // CHECK-SAME: %[[P:.*]]: tensor<2xi32>, -// CHECK-SAME: %[[I:.*]]: tensor<6x1xi32>) -// CHECK: %[[R:.*]] = sparse_tensor.assemble %[[D]], %[[P]], %[[I]] +// CHECK-SAME: %[[I:.*]]: tensor<6x1xi32>, +// CHECK-SAME: %[[D:.*]]: tensor<6xf64>) +// CHECK: %[[R:.*]] = sparse_tensor.assemble (%[[P]], %[[I]]), %[[D]] // CHECK: return %[[R]] : tensor<100xf64, #{{.*}}> -func.func @sparse_pack(%data: tensor<6xf64>, %pos: tensor<2xi32>, %index: tensor<6x1xi32>) +func.func @sparse_pack(%pos: tensor<2xi32>, %index: tensor<6x1xi32>, %data: tensor<6xf64>) -> tensor<100xf64, #SparseVector> { - %0 = sparse_tensor.assemble %data, %pos, %index : tensor<6xf64>, tensor<2xi32>, tensor<6x1xi32> + %0 = sparse_tensor.assemble (%pos, %index), %data: (tensor<2xi32>, tensor<6x1xi32>), tensor<6xf64> to tensor<100xf64, #SparseVector> return %0 : tensor<100xf64, #SparseVector> } @@ -36,17 +36,18 @@ func.func @sparse_pack(%data: tensor<6xf64>, %pos: tensor<2xi32>, %index: tensor // CHECK-SAME: %[[OD:.*]]: tensor<6xf64> // CHECK-SAME: %[[OP:.*]]: tensor<2xindex> // CHECK-SAME: %[[OI:.*]]: tensor<6x1xi32> -// CHECK: %[[D:.*]], %[[P:.*]]:2, %[[DL:.*]], %[[PL:.*]]:2 = sparse_tensor.disassemble %[[T]] -// CHECK: return %[[D]], %[[P]]#0, %[[P]]#1 +// CHECK: %[[P:.*]]:2, %[[D:.*]], %[[PL:.*]]:2, %[[DL:.*]] = sparse_tensor.disassemble %[[T]] +// CHECK: return %[[P]]#0, %[[P]]#1, %[[D]] func.func @sparse_unpack(%sp : tensor<100xf64, #SparseVector>, %od : tensor<6xf64>, %op : tensor<2xindex>, %oi : tensor<6x1xi32>) - -> (tensor<6xf64>, tensor<2xindex>, tensor<6x1xi32>) { - %rd, %rp, %ri, %vl, %pl, %cl = sparse_tensor.disassemble %sp : tensor<100xf64, #SparseVector> - outs(%od, %op, %oi : tensor<6xf64>, tensor<2xindex>, tensor<6x1xi32>) - -> tensor<6xf64>, (tensor<2xindex>, tensor<6x1xi32>), index, (index, index) - return %rd, %rp, %ri : tensor<6xf64>, tensor<2xindex>, tensor<6x1xi32> + -> (tensor<2xindex>, tensor<6x1xi32>, tensor<6xf64>) { + %rp, %ri, %rd, %vl, %pl, %cl = sparse_tensor.disassemble %sp : tensor<100xf64, #SparseVector> + out_lvls(%op, %oi : tensor<2xindex>, tensor<6x1xi32>) + out_vals(%od : tensor<6xf64>) + -> (tensor<2xindex>, tensor<6x1xi32>), tensor<6xf64>, (index, index), index + return %rp, %ri, %rd : tensor<2xindex>, tensor<6x1xi32>, tensor<6xf64> } // ----- diff --git a/mlir/test/Dialect/SparseTensor/sparse_pack.mlir b/mlir/test/Dialect/SparseTensor/sparse_pack.mlir index 7cb699092f88..a90194a74ee4 100644 --- a/mlir/test/Dialect/SparseTensor/sparse_pack.mlir +++ b/mlir/test/Dialect/SparseTensor/sparse_pack.mlir @@ -31,8 +31,8 @@ // CHECK: } func.func @sparse_pack(%values: tensor<6xf64>, %pos:tensor<2xindex>, %coordinates: tensor<6x2xi32>) -> tensor<100x100xf64, #COO> { - %0 = sparse_tensor.assemble %values, %pos, %coordinates - : tensor<6xf64>, tensor<2xindex>, tensor<6x2xi32> to tensor<100x100xf64, #COO> + %0 = sparse_tensor.assemble (%pos, %coordinates), %values + : (tensor<2xindex>, tensor<6x2xi32>), tensor<6xf64> to tensor<100x100xf64, #COO> return %0 : tensor<100x100xf64, #COO> } @@ -60,9 +60,9 @@ func.func @sparse_pack(%values: tensor<6xf64>, %pos:tensor<2xindex>, %coordinate // CHECK: %[[VAL_18:.*]] = memref.subview %[[VAL_17]][0] {{\[}}%[[VAL_16]]] [1] : memref<6xf64> to memref // CHECK: %[[VAL_19:.*]] = memref.subview %[[VAL_2]][0] {{\[}}%[[VAL_16]]] [1] : memref to memref // CHECK: memref.copy %[[VAL_19]], %[[VAL_18]] : memref to memref -// CHECK: %[[VAL_20:.*]] = bufferization.to_tensor %[[VAL_17]] : memref<6xf64> -// CHECK: %[[VAL_21:.*]] = bufferization.to_tensor %[[VAL_8]] : memref<2xindex> -// CHECK: %[[VAL_22:.*]] = bufferization.to_tensor %[[VAL_12]] : memref<6x2xi32> +// CHECK-DAG: %[[VAL_20:.*]] = bufferization.to_tensor %[[VAL_17]] : memref<6xf64> +// CHECK-DAG: %[[VAL_21:.*]] = bufferization.to_tensor %[[VAL_8]] : memref<2xindex> +// CHECK-DAG: %[[VAL_22:.*]] = bufferization.to_tensor %[[VAL_12]] : memref<6x2xi32> // CHECK: return %[[VAL_20]], %[[VAL_21]], %[[VAL_22]] : tensor<6xf64>, tensor<2xindex>, tensor<6x2xi32> // CHECK: } func.func @sparse_unpack(%sp : tensor<100x100xf64, #COO>, @@ -70,8 +70,9 @@ func.func @sparse_unpack(%sp : tensor<100x100xf64, #COO>, %op : tensor<2xindex>, %oi : tensor<6x2xi32>) -> (tensor<6xf64>, tensor<2xindex>, tensor<6x2xi32>) { - %rd, %rp, %ri, %dl, %pl, %il = sparse_tensor.disassemble %sp : tensor<100x100xf64, #COO> - outs(%od, %op, %oi : tensor<6xf64>, tensor<2xindex>, tensor<6x2xi32>) - -> tensor<6xf64>, (tensor<2xindex>, tensor<6x2xi32>), index, (index, index) + %rp, %ri, %rd, %dl, %pl, %il = sparse_tensor.disassemble %sp : tensor<100x100xf64, #COO> + out_lvls(%op, %oi : tensor<2xindex>, tensor<6x2xi32>) + out_vals(%od : tensor<6xf64>) + -> (tensor<2xindex>, tensor<6x2xi32>), tensor<6xf64>, (index, index), index return %rd, %rp, %ri : tensor<6xf64>, tensor<2xindex>, tensor<6x2xi32> } diff --git a/mlir/test/Dialect/SparseTensor/sparse_reinterpret_map.mlir b/mlir/test/Dialect/SparseTensor/sparse_reinterpret_map.mlir index 54de1024323b..aa17261724db 100644 --- a/mlir/test/Dialect/SparseTensor/sparse_reinterpret_map.mlir +++ b/mlir/test/Dialect/SparseTensor/sparse_reinterpret_map.mlir @@ -99,13 +99,13 @@ func.func @sparse_foreach_reinterpret_map(%6 : tensor<2x4xf64, #BSR>) -> tensor< // CHECK-SAME: %[[VAL_0:.*]]: tensor, // CHECK-SAME: %[[VAL_1:.*]]: tensor, // CHECK-SAME: %[[VAL_2:.*]]: tensor) -> tensor<2x4xf64, #[[$remap]]> { -// CHECK: %[[VAL_3:.*]] = sparse_tensor.assemble %[[VAL_0]], %[[VAL_1]], %[[VAL_2]] : tensor, tensor, tensor to tensor<1x2x2x2xf64, #[[$demap]]> +// CHECK: %[[VAL_3:.*]] = sparse_tensor.assemble {{.*}} to tensor<1x2x2x2xf64, #[[$demap]]> // CHECK: %[[VAL_4:.*]] = sparse_tensor.reinterpret_map %[[VAL_3]] : tensor<1x2x2x2xf64, #[[$demap]]> to tensor<2x4xf64, #[[$remap]]> // CHECK: return %[[VAL_4]] : tensor<2x4xf64, #[[$remap]]> // CHECK: } func.func @sparse_assemble_reinterpret_map(%val : tensor, %pos:tensor, %crd:tensor) -> tensor<2x4xf64, #BSR> { - %0 = sparse_tensor.assemble %val, %pos, %crd - : tensor, tensor, tensor to tensor<2x4xf64, #BSR> + %0 = sparse_tensor.assemble (%pos, %crd), %val + : (tensor, tensor), tensor to tensor<2x4xf64, #BSR> return %0 : tensor<2x4xf64, #BSR> } @@ -115,7 +115,7 @@ func.func @sparse_assemble_reinterpret_map(%val : tensor, %pos:tensor, // CHECK-SAME: %[[VAL_3:.*]]: tensor) -> (tensor, tensor, tensor) { // CHECK: %[[VAL_4:.*]] = sparse_tensor.reinterpret_map %[[VAL_0]] : tensor<2x4xf64, #[[$remap]]> to tensor<1x2x2x2xf64, #[[$demap]]> -// CHECK: %[[VAL_5:.*]], %[[VAL_6:.*]]:2, %[[VAL_7:.*]], %[[VAL_8:.*]]:2 = sparse_tensor.disassemble %[[VAL_4]] : tensor<1x2x2x2xf64, #[[$demap]]> +// CHECK: %{{.*}} = sparse_tensor.disassemble %[[VAL_4]] : tensor<1x2x2x2xf64, #[[$demap]]> // CHECK: return // CHECK: } func.func @sparse_disassemble_reinterpret_map(%sp : tensor<2x4xf64, #BSR>, @@ -123,8 +123,9 @@ func.func @sparse_disassemble_reinterpret_map(%sp : tensor<2x4xf64, #BSR>, %op : tensor, %oi : tensor) -> (tensor, tensor, tensor) { - %rd, %rp, %ri, %dl, %pl, %il = sparse_tensor.disassemble %sp : tensor<2x4xf64, #BSR> - outs(%od, %op, %oi : tensor, tensor, tensor) - -> tensor, (tensor, tensor), index, (index, index) + %rp, %ri, %rd, %dl, %pl, %il = sparse_tensor.disassemble %sp : tensor<2x4xf64, #BSR> + out_lvls(%op, %oi : tensor, tensor) + out_vals(%od : tensor) + -> (tensor, tensor), tensor, (index, index), index return %rd, %rp, %ri : tensor, tensor, tensor } diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir index 2b9b73a1990e..b792d00681dd 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir @@ -87,9 +87,9 @@ module { [ 7, 8]] > : tensor<3x2xi32> - %s4 = sparse_tensor.assemble %data, %pos, %index : tensor<3xf64>, tensor<2xindex>, tensor<3x2xindex> + %s4 = sparse_tensor.assemble (%pos, %index), %data : (tensor<2xindex>, tensor<3x2xindex>), tensor<3xf64> to tensor<10x10xf64, #SortedCOO> - %s5 = sparse_tensor.assemble %data, %pos32, %index32 : tensor<3xf64>, tensor<2xi32>, tensor<3x2xi32> + %s5 = sparse_tensor.assemble (%pos32, %index32), %data : (tensor<2xi32>, tensor<3x2xi32>), tensor<3xf64> to tensor<10x10xf64, #SortedCOOI32> // @@ -107,7 +107,7 @@ module { %csr_index32 = arith.constant dense< [1, 0, 1] > : tensor<3xi32> - %csr = sparse_tensor.assemble %csr_data, %csr_pos32, %csr_index32 : tensor<3xf64>, tensor<3xi32>, tensor<3xi32> + %csr = sparse_tensor.assemble (%csr_pos32, %csr_index32), %csr_data : (tensor<3xi32>, tensor<3xi32>), tensor<3xf64> to tensor<2x2xf64, #CSR> // @@ -131,8 +131,8 @@ module { [ 10, 10]] > : tensor<6x2xindex> - %bs = sparse_tensor.assemble %bdata, %bpos, %bindex : - tensor<5xf64>, tensor<4xindex>, tensor<6x2xindex> to tensor<2x10x10xf64, #BCOO> + %bs = sparse_tensor.assemble (%bpos, %bindex), %bdata : + (tensor<4xindex>, tensor<6x2xindex>), tensor<5xf64> to tensor<2x10x10xf64, #BCOO> // // Verify results. @@ -231,9 +231,10 @@ module { %od = tensor.empty() : tensor<3xf64> %op = tensor.empty() : tensor<2xi32> %oi = tensor.empty() : tensor<3x2xi32> - %d, %p, %i, %dl, %pl, %il = sparse_tensor.disassemble %s5 : tensor<10x10xf64, #SortedCOOI32> - outs(%od, %op, %oi : tensor<3xf64>, tensor<2xi32>, tensor<3x2xi32>) - -> tensor<3xf64>, (tensor<2xi32>, tensor<3x2xi32>), index, (i32, i64) + %p, %i, %d, %dl, %pl, %il = sparse_tensor.disassemble %s5 : tensor<10x10xf64, #SortedCOOI32> + out_lvls(%op, %oi : tensor<2xi32>, tensor<3x2xi32>) + out_vals(%od : tensor<3xf64>) + -> (tensor<2xi32>, tensor<3x2xi32>), tensor<3xf64>, (i32, i64), index // CHECK-NEXT: ( 1, 2, 3 ) %vd = vector.transfer_read %d[%c0], %f0 : tensor<3xf64>, vector<3xf64> @@ -246,9 +247,10 @@ module { %d_csr = tensor.empty() : tensor<4xf64> %p_csr = tensor.empty() : tensor<3xi32> %i_csr = tensor.empty() : tensor<3xi32> - %rd_csr, %rp_csr, %ri_csr, %ld_csr, %lp_csr, %li_csr = sparse_tensor.disassemble %csr : tensor<2x2xf64, #CSR> - outs(%d_csr, %p_csr, %i_csr : tensor<4xf64>, tensor<3xi32>, tensor<3xi32>) - -> tensor<4xf64>, (tensor<3xi32>, tensor<3xi32>), index, (i32, i64) + %rp_csr, %ri_csr, %rd_csr, %ld_csr, %lp_csr, %li_csr = sparse_tensor.disassemble %csr : tensor<2x2xf64, #CSR> + out_lvls(%p_csr, %i_csr : tensor<3xi32>, tensor<3xi32>) + out_vals(%d_csr : tensor<4xf64>) + -> (tensor<3xi32>, tensor<3xi32>), tensor<4xf64>, (i32, i64), index // CHECK-NEXT: ( 1, 2, 3 ) %vd_csr = vector.transfer_read %rd_csr[%c0], %f0 : tensor<4xf64>, vector<3xf64> @@ -257,9 +259,10 @@ module { %bod = tensor.empty() : tensor<6xf64> %bop = tensor.empty() : tensor<4xindex> %boi = tensor.empty() : tensor<6x2xindex> - %bd, %bp, %bi, %ld, %lp, %li = sparse_tensor.disassemble %bs : tensor<2x10x10xf64, #BCOO> - outs(%bod, %bop, %boi : tensor<6xf64>, tensor<4xindex>, tensor<6x2xindex>) - -> tensor<6xf64>, (tensor<4xindex>, tensor<6x2xindex>), index, (i32, tensor) + %bp, %bi, %bd, %lp, %li, %ld = sparse_tensor.disassemble %bs : tensor<2x10x10xf64, #BCOO> + out_lvls(%bop, %boi : tensor<4xindex>, tensor<6x2xindex>) + out_vals(%bod : tensor<6xf64>) + -> (tensor<4xindex>, tensor<6x2xindex>), tensor<6xf64>, (i32, tensor), index // CHECK-NEXT: ( 1, 2, 3, 4, 5 ) %vbd = vector.transfer_read %bd[%c0], %f0 : tensor<6xf64>, vector<5xf64> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir index da816c7fbb11..8a65e2449c15 100755 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir @@ -71,11 +71,10 @@ module { %crd02 = arith.constant dense< [ 0, 1, 0, 1, 0, 0, 1, 0 ]> : tensor<8xi32> - %s0 = sparse_tensor.assemble %data0, %pos00, %crd00, %pos01, %crd01, %pos02, %crd02 : - tensor<8xf32>, - tensor<2xi64>, tensor<3xi32>, - tensor<4xi64>, tensor<5xi32>, - tensor<6xi64>, tensor<8xi32> to tensor<4x3x2xf32, #CCC> + %s0 = sparse_tensor.assemble (%pos00, %crd00, %pos01, %crd01, %pos02, %crd02), %data0 : + (tensor<2xi64>, tensor<3xi32>, + tensor<4xi64>, tensor<5xi32>, + tensor<6xi64>, tensor<8xi32>), tensor<8xf32> to tensor<4x3x2xf32, #CCC> // // Setup BatchedCSR. @@ -89,7 +88,7 @@ module { %crd1 = arith.constant dense< [ 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1]> : tensor<16xi32> - %s1 = sparse_tensor.assemble %data1, %pos1, %crd1 : tensor<16xf32>, tensor<13xi64>, tensor<16xi32> to tensor<4x3x2xf32, #BatchedCSR> + %s1 = sparse_tensor.assemble (%pos1, %crd1), %data1 : (tensor<13xi64>, tensor<16xi32>), tensor<16xf32> to tensor<4x3x2xf32, #BatchedCSR> // // Setup CSRDense. @@ -103,7 +102,7 @@ module { %crd2 = arith.constant dense< [ 0, 1, 2, 0, 2, 0, 1, 2, 0, 1, 2 ]> : tensor<11xi32> - %s2 = sparse_tensor.assemble %data2, %pos2, %crd2 : tensor<22xf32>, tensor<5xi64>, tensor<11xi32> to tensor<4x3x2xf32, #CSRDense> + %s2 = sparse_tensor.assemble (%pos2, %crd2), %data2 : (tensor<5xi64>, tensor<11xi32>), tensor<22xf32> to tensor<4x3x2xf32, #CSRDense> // // Verify. -- GitLab From dcd08daed5fb9928fedb09c2a84d1e9d2ab59630 Mon Sep 17 00:00:00 2001 From: Pavel Iliin Date: Wed, 6 Mar 2024 17:59:31 +0000 Subject: [PATCH 331/929] [NFC][Docs] Documenting __builtin_cpu_supports. (#84098) Co-authored-by: Jon Roelofs --- clang/docs/LanguageExtensions.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst index bcd69198eafd..2b54dffd058a 100644 --- a/clang/docs/LanguageExtensions.rst +++ b/clang/docs/LanguageExtensions.rst @@ -2799,6 +2799,34 @@ counter's true frequency will need to be provided by the user. Query for this feature with ``__has_builtin(__builtin_readsteadycounter)``. +``__builtin_cpu_supports`` +-------------------------- + +**Syntax**: + +.. code-block:: c++ + + int __builtin_cpu_supports(const char *features); + +**Example of Use:**: + +.. code-block:: c++ + + if (__builtin_cpu_supports("sve")) + sve_code(); + +**Description**: + +The ``__builtin_cpu_supports`` function detects if the run-time CPU supports +features specified in string argument. It returns a positive integer if all +features are supported and 0 otherwise. Feature names are target specific. On +AArch64 features are combined using ``+`` like this +``__builtin_cpu_supports("flagm+sha3+lse+rcpc2+fcma+memtag+bti+sme2")``. +If a feature name is not supported, Clang will issue a warning and replace +builtin by the constant 0. + +Query for this feature with ``__has_builtin(__builtin_cpu_supports)``. + ``__builtin_dump_struct`` ------------------------- -- GitLab From aeaa11aeac0aedf32cafec6532d303fea595c5fc Mon Sep 17 00:00:00 2001 From: Jason Molenda Date: Wed, 6 Mar 2024 10:06:56 -0800 Subject: [PATCH 332/929] [lldb] Address mask sbprocess apis and new mask invalid const (#83663) [lldb] Add SBProcess methods for get/set/use address masks (#83095) I'm reviving a patch from phabracator, https://reviews.llvm.org/D155905 which was approved but I wasn't thrilled with all the API I was adding to SBProcess for all of the address mask types / memory regions. In this update, I added enums to control type address mask type (code, data, any) and address space specifiers (low, high, all) with defaulted arguments for the most common case. I originally landed this via https://github.com/llvm/llvm-project/pull/83095 but it failed on CIs outside of arm64 Darwin so I had to debug it on more environments and update the patch. This patch is also fixing a bug in the "addressable bits to address mask" calculation I added in AddressableBits::SetProcessMasks. If lldb were told that 64 bits are valid for addressing, this method would overflow the calculation and set an invalid mask. Added tests to check this specific bug while I was adding these APIs. This patch changes the value of "no mask set" from 0 to LLDB_INVALID_ADDRESS_MASK, which is UINT64_MAX. A mask of all 1's means "no bits are used for addressing" which is an impossible mask, whereas a mask of 0 means "all bits are used for addressing" which is possible. I added a base class implementation of ABI::FixCodeAddress and ABI::FixDataAddress that will apply the Process mask values if they are set to a value other than LLDB_INVALID_ADDRESS_MASK. I updated all the callers/users of the Mask methods which were handling a value of 0 to mean invalid mask to use LLDB_INVALID_ADDRESS_MASK. I added code to the all AArch64 ABI Fix* methods to apply the Highmem masks if they have been set. These will not be set on a Linux environment, but in TestAddressMasks.py I test the highmem masks feature for any AArch64 target, so all AArch64 ABI plugins must handle it. rdar://123530562 --- lldb/include/lldb/API/SBProcess.h | 114 +++++++++++++++ lldb/include/lldb/Target/ABI.h | 8 +- lldb/include/lldb/Target/Process.h | 36 +++-- lldb/include/lldb/Utility/AddressableBits.h | 3 + lldb/include/lldb/lldb-defines.h | 5 + lldb/include/lldb/lldb-enumerations.h | 16 +++ lldb/source/API/SBProcess.cpp | 92 ++++++++++++ lldb/source/Commands/CommandObjectProcess.cpp | 2 +- .../source/Plugins/ABI/AArch64/ABIAArch64.cpp | 33 ++++- .../Plugins/ABI/AArch64/ABIMacOSX_arm64.cpp | 8 +- .../Plugins/ABI/AArch64/ABISysV_arm64.cpp | 40 ++++-- .../DynamicLoaderDarwinKernel.cpp | 3 +- .../ObjectFile/Mach-O/ObjectFileMachO.cpp | 4 +- lldb/source/Target/ABI.cpp | 33 +++++ lldb/source/Target/Process.cpp | 14 +- lldb/source/Utility/AddressableBits.cpp | 12 +- .../python_api/process/address-masks/Makefile | 3 + .../process/address-masks/TestAddressMasks.py | 131 ++++++++++++++++++ .../python_api/process/address-masks/main.c | 5 + 19 files changed, 517 insertions(+), 45 deletions(-) create mode 100644 lldb/test/API/python_api/process/address-masks/Makefile create mode 100644 lldb/test/API/python_api/process/address-masks/TestAddressMasks.py create mode 100644 lldb/test/API/python_api/process/address-masks/main.c diff --git a/lldb/include/lldb/API/SBProcess.h b/lldb/include/lldb/API/SBProcess.h index 4f92a41f3028..7da3335a7234 100644 --- a/lldb/include/lldb/API/SBProcess.h +++ b/lldb/include/lldb/API/SBProcess.h @@ -407,6 +407,120 @@ public: /// the process isn't loaded from a core file. lldb::SBFileSpec GetCoreFile(); + /// \{ + /// \group Mask Address Methods + /// + /// \a type + /// All of the methods in this group take \a type argument + /// which is an AddressMaskType enum value. + /// There can be different address masks for code addresses and + /// data addresses, this argument can select which to get/set, + /// or to use when clearing non-addressable bits from an address. + /// This choice of mask can be important for example on AArch32 + /// systems. Where instructions where instructions start on even addresses, + /// the 0th bit may be used to indicate that a function is thumb code. On + /// such a target, the eAddressMaskTypeCode may clear the 0th bit from an + /// address to get the actual address Whereas eAddressMaskTypeData would not. + /// + /// \a addr_range + /// Many of the methods in this group take an \a addr_range argument + /// which is an AddressMaskRange enum value. + /// Needing to specify the address range is highly unusual, and the + /// default argument can be used in nearly all circumstances. + /// On some architectures (e.g., AArch64), it is possible to have + /// different page table setups for low and high memory, so different + /// numbers of bits relevant to addressing. It is possible to have + /// a program running in one half of memory and accessing the other + /// as heap, so we need to maintain two different sets of address masks + /// to debug this correctly. + + /// Get the current address mask that will be applied to addresses + /// before reading from memory. + /// + /// \param[in] type + /// See \ref Mask Address Methods description of this argument. + /// eAddressMaskTypeAny is often a suitable value when code and + /// data masks are the same on a given target. + /// + /// \param[in] addr_range + /// See \ref Mask Address Methods description of this argument. + /// This will default to eAddressMaskRangeLow which is the + /// only set of masks used normally. + /// + /// \return + /// The address mask currently in use. Bits which are not used + /// for addressing will be set to 1 in the mask. + lldb::addr_t GetAddressMask( + lldb::AddressMaskType type, + lldb::AddressMaskRange addr_range = lldb::eAddressMaskRangeLow); + + /// Set the current address mask that can be applied to addresses + /// before reading from memory. + /// + /// \param[in] type + /// See \ref Mask Address Methods description of this argument. + /// eAddressMaskTypeAll is often a suitable value when the + /// same mask is being set for both code and data. + /// + /// \param[in] mask + /// The address mask to set. Bits which are not used for addressing + /// should be set to 1 in the mask. + /// + /// \param[in] addr_range + /// See \ref Mask Address Methods description of this argument. + /// This will default to eAddressMaskRangeLow which is the + /// only set of masks used normally. + void SetAddressMask( + lldb::AddressMaskType type, lldb::addr_t mask, + lldb::AddressMaskRange addr_range = lldb::eAddressMaskRangeLow); + + /// Set the number of bits used for addressing in this Process. + /// + /// On Darwin and similar systems, the addressable bits are expressed + /// as the number of low order bits that are relevant to addressing, + /// instead of a more general address mask. + /// This method calculates the correct mask value for a given number + /// of low order addressable bits. + /// + /// \param[in] type + /// See \ref Mask Address Methods description of this argument. + /// eAddressMaskTypeAll is often a suitable value when the + /// same mask is being set for both code and data. + /// + /// \param[in] num_bits + /// Number of bits that are used for addressing. + /// For example, a value of 42 indicates that the low 42 bits + /// are relevant for addressing, and that higher-order bits may + /// be used for various metadata like pointer authentication, + /// Type Byte Ignore, etc. + /// + /// \param[in] addr_range + /// See \ref Mask Address Methods description of this argument. + /// This will default to eAddressMaskRangeLow which is the + /// only set of masks used normally. + void + SetAddressableBits(AddressMaskType type, uint32_t num_bits, + AddressMaskRange addr_range = lldb::eAddressMaskRangeLow); + + /// Clear the non-address bits of an \a addr value and return a + /// virtual address in memory. + /// + /// Bits that are not used in addressing may be used for other purposes; + /// pointer authentication, or metadata in the top byte, or the 0th bit + /// of armv7 code addresses to indicate arm/thumb are common examples. + /// + /// \param[in] addr + /// The address that should be cleared of non-address bits. + /// + /// \param[in] type + /// See \ref Mask Address Methods description of this argument. + /// eAddressMaskTypeAny is the default value, correct when it + /// is unknown if the address is a code or data address. + lldb::addr_t + FixAddress(lldb::addr_t addr, + lldb::AddressMaskType type = lldb::eAddressMaskTypeAny); + /// \} + /// Allocate memory within the process. /// /// This function will allocate memory in the process's address space. diff --git a/lldb/include/lldb/Target/ABI.h b/lldb/include/lldb/Target/ABI.h index f600e29c7c4b..7b646d743346 100644 --- a/lldb/include/lldb/Target/ABI.h +++ b/lldb/include/lldb/Target/ABI.h @@ -122,8 +122,8 @@ public: /// ARM uses bit zero to signify a code address is thumb, so any ARM ABI /// plug-ins would strip those bits. /// @{ - virtual lldb::addr_t FixCodeAddress(lldb::addr_t pc) { return pc; } - virtual lldb::addr_t FixDataAddress(lldb::addr_t pc) { return pc; } + virtual lldb::addr_t FixCodeAddress(lldb::addr_t pc); + virtual lldb::addr_t FixDataAddress(lldb::addr_t pc); /// @} /// Use this method when you do not know, or do not care what kind of address @@ -166,10 +166,6 @@ protected: lldb::ProcessWP m_process_wp; std::unique_ptr m_mc_register_info_up; - virtual lldb::addr_t FixCodeAddress(lldb::addr_t pc, lldb::addr_t mask) { - return pc; - } - private: ABI(const ABI &) = delete; const ABI &operator=(const ABI &) = delete; diff --git a/lldb/include/lldb/Target/Process.h b/lldb/include/lldb/Target/Process.h index 0ad626ffd361..e260e1b4b797 100644 --- a/lldb/include/lldb/Target/Process.h +++ b/lldb/include/lldb/Target/Process.h @@ -1422,9 +1422,23 @@ public: virtual void DidExit() {} + /// Get the current address mask in the Process + /// + /// This mask can used to set/clear non-address bits in an addr_t. + /// + /// \return + /// The current address mask. + /// Bits which are set to 1 are not used for addressing. + /// An address mask of 0 means all bits are used for addressing. + /// An address mask of LLDB_INVALID_ADDRESS_MASK (all 1's) means + /// that no mask has been set. lldb::addr_t GetCodeAddressMask(); lldb::addr_t GetDataAddressMask(); + /// The highmem masks are for targets where we may have different masks + /// for low memory versus high memory addresses, and they will be left + /// as LLDB_INVALID_ADDRESS_MASK normally, meaning the base masks + /// should be applied to all addresses. lldb::addr_t GetHighmemCodeAddressMask(); lldb::addr_t GetHighmemDataAddressMask(); @@ -3096,16 +3110,20 @@ protected: // if run while destructing. We use this flag to determine that. std::atomic m_destructing; - /// Mask for code an data addresses. The default value (0) means no mask is - /// set. The bits set to 1 indicate bits that are NOT significant for - /// addressing. - /// The highmem versions are for targets where we may have different masks - /// for low memory versus high memory addresses. + /// Mask for code an data addresses. + /// The default value LLDB_INVALID_ADDRESS_MASK means no mask has been set, + /// and addresses values should not be modified. + /// In these masks, the bits are set to 1 indicate bits that are not + /// significant for addressing. + /// The highmem masks are for targets where we may have different masks + /// for low memory versus high memory addresses, and they will be left + /// as LLDB_INVALID_ADDRESS_MASK normally, meaning the base masks + /// should be applied to all addresses. /// @{ - lldb::addr_t m_code_address_mask = 0; - lldb::addr_t m_data_address_mask = 0; - lldb::addr_t m_highmem_code_address_mask = 0; - lldb::addr_t m_highmem_data_address_mask = 0; + lldb::addr_t m_code_address_mask = LLDB_INVALID_ADDRESS_MASK; + lldb::addr_t m_data_address_mask = LLDB_INVALID_ADDRESS_MASK; + lldb::addr_t m_highmem_code_address_mask = LLDB_INVALID_ADDRESS_MASK; + lldb::addr_t m_highmem_data_address_mask = LLDB_INVALID_ADDRESS_MASK; /// @} bool m_clear_thread_plans_on_stop; diff --git a/lldb/include/lldb/Utility/AddressableBits.h b/lldb/include/lldb/Utility/AddressableBits.h index 13c21329a8c6..75752fcf840a 100644 --- a/lldb/include/lldb/Utility/AddressableBits.h +++ b/lldb/include/lldb/Utility/AddressableBits.h @@ -10,6 +10,7 @@ #define LLDB_UTILITY_ADDRESSABLEBITS_H #include "lldb/lldb-forward.h" +#include "lldb/lldb-public.h" namespace lldb_private { @@ -33,6 +34,8 @@ public: void SetHighmemAddressableBits(uint32_t highmem_addressing_bits); + static lldb::addr_t AddressableBitToMask(uint32_t addressable_bits); + void SetProcessMasks(lldb_private::Process &process); private: diff --git a/lldb/include/lldb/lldb-defines.h b/lldb/include/lldb/lldb-defines.h index 469be92eabec..c7bd019c5c90 100644 --- a/lldb/include/lldb/lldb-defines.h +++ b/lldb/include/lldb/lldb-defines.h @@ -127,6 +127,11 @@ #define MAX_PATH 260 #endif +/// Address Mask +/// Bits not used for addressing are set to 1 in the mask; +/// all mask bits set is an invalid value. +#define LLDB_INVALID_ADDRESS_MASK UINT64_MAX + // ignore GCC function attributes #if defined(_MSC_VER) && !defined(__clang__) #define __attribute__(X) diff --git a/lldb/include/lldb/lldb-enumerations.h b/lldb/include/lldb/lldb-enumerations.h index 85769071dae7..646f7bfda984 100644 --- a/lldb/include/lldb/lldb-enumerations.h +++ b/lldb/include/lldb/lldb-enumerations.h @@ -1323,6 +1323,22 @@ enum SymbolDownload { eSymbolDownloadForeground = 2, }; +/// Used in the SBProcess AddressMask/FixAddress methods. +enum AddressMaskType { + eAddressMaskTypeCode = 0, + eAddressMaskTypeData, + eAddressMaskTypeAny, + eAddressMaskTypeAll = eAddressMaskTypeAny +}; + +/// Used in the SBProcess AddressMask/FixAddress methods. +enum AddressMaskRange { + eAddressMaskRangeLow = 0, + eAddressMaskRangeHigh, + eAddressMaskRangeAny, + eAddressMaskRangeAll = eAddressMaskRangeAny, +}; + } // namespace lldb #endif // LLDB_LLDB_ENUMERATIONS_H diff --git a/lldb/source/API/SBProcess.cpp b/lldb/source/API/SBProcess.cpp index a9fe91532468..b80664882ebc 100644 --- a/lldb/source/API/SBProcess.cpp +++ b/lldb/source/API/SBProcess.cpp @@ -1255,6 +1255,98 @@ lldb::SBFileSpec SBProcess::GetCoreFile() { return SBFileSpec(core_file); } +addr_t SBProcess::GetAddressMask(AddressMaskType type, + AddressMaskRange addr_range) { + LLDB_INSTRUMENT_VA(this, type, addr_range); + + if (ProcessSP process_sp = GetSP()) { + switch (type) { + case eAddressMaskTypeCode: + if (addr_range == eAddressMaskRangeHigh) + return process_sp->GetHighmemCodeAddressMask(); + else + return process_sp->GetCodeAddressMask(); + case eAddressMaskTypeData: + if (addr_range == eAddressMaskRangeHigh) + return process_sp->GetHighmemDataAddressMask(); + else + return process_sp->GetDataAddressMask(); + case eAddressMaskTypeAny: + if (addr_range == eAddressMaskRangeHigh) + return process_sp->GetHighmemDataAddressMask(); + else + return process_sp->GetDataAddressMask(); + } + } + return LLDB_INVALID_ADDRESS_MASK; +} + +void SBProcess::SetAddressMask(AddressMaskType type, addr_t mask, + AddressMaskRange addr_range) { + LLDB_INSTRUMENT_VA(this, type, mask, addr_range); + + if (ProcessSP process_sp = GetSP()) { + switch (type) { + case eAddressMaskTypeCode: + if (addr_range == eAddressMaskRangeAll) { + process_sp->SetCodeAddressMask(mask); + process_sp->SetHighmemCodeAddressMask(mask); + } else if (addr_range == eAddressMaskRangeHigh) { + process_sp->SetHighmemCodeAddressMask(mask); + } else { + process_sp->SetCodeAddressMask(mask); + } + break; + case eAddressMaskTypeData: + if (addr_range == eAddressMaskRangeAll) { + process_sp->SetDataAddressMask(mask); + process_sp->SetHighmemDataAddressMask(mask); + } else if (addr_range == eAddressMaskRangeHigh) { + process_sp->SetHighmemDataAddressMask(mask); + } else { + process_sp->SetDataAddressMask(mask); + } + break; + case eAddressMaskTypeAll: + if (addr_range == eAddressMaskRangeAll) { + process_sp->SetCodeAddressMask(mask); + process_sp->SetDataAddressMask(mask); + process_sp->SetHighmemCodeAddressMask(mask); + process_sp->SetHighmemDataAddressMask(mask); + } else if (addr_range == eAddressMaskRangeHigh) { + process_sp->SetHighmemCodeAddressMask(mask); + process_sp->SetHighmemDataAddressMask(mask); + } else { + process_sp->SetCodeAddressMask(mask); + process_sp->SetDataAddressMask(mask); + } + break; + } + } +} + +void SBProcess::SetAddressableBits(AddressMaskType type, uint32_t num_bits, + AddressMaskRange addr_range) { + LLDB_INSTRUMENT_VA(this, type, num_bits, addr_range); + + SetAddressMask(type, AddressableBits::AddressableBitToMask(num_bits), + addr_range); +} + +addr_t SBProcess::FixAddress(addr_t addr, AddressMaskType type) { + LLDB_INSTRUMENT_VA(this, addr, type); + + if (ProcessSP process_sp = GetSP()) { + if (type == eAddressMaskTypeAny) + return process_sp->FixAnyAddress(addr); + else if (type == eAddressMaskTypeData) + return process_sp->FixDataAddress(addr); + else if (type == eAddressMaskTypeCode) + return process_sp->FixCodeAddress(addr); + } + return addr; +} + lldb::addr_t SBProcess::AllocateMemory(size_t size, uint32_t permissions, lldb::SBError &sb_error) { LLDB_INSTRUMENT_VA(this, size, permissions, sb_error); diff --git a/lldb/source/Commands/CommandObjectProcess.cpp b/lldb/source/Commands/CommandObjectProcess.cpp index 9ac97eb66b62..3587a8f529e4 100644 --- a/lldb/source/Commands/CommandObjectProcess.cpp +++ b/lldb/source/Commands/CommandObjectProcess.cpp @@ -1408,7 +1408,7 @@ protected: if (m_options.m_verbose) { addr_t code_mask = process->GetCodeAddressMask(); addr_t data_mask = process->GetDataAddressMask(); - if (code_mask != 0) { + if (code_mask != LLDB_INVALID_ADDRESS_MASK) { int bits = std::bitset<64>(~code_mask).count(); result.AppendMessageWithFormat( "Addressable code address mask: 0x%" PRIx64 "\n", code_mask); diff --git a/lldb/source/Plugins/ABI/AArch64/ABIAArch64.cpp b/lldb/source/Plugins/ABI/AArch64/ABIAArch64.cpp index ef53309f065f..256c1f828feb 100644 --- a/lldb/source/Plugins/ABI/AArch64/ABIAArch64.cpp +++ b/lldb/source/Plugins/ABI/AArch64/ABIAArch64.cpp @@ -6,6 +6,8 @@ // //===----------------------------------------------------------------------===// +#include "lldb/lldb-types.h" + #include "ABIAArch64.h" #include "ABIMacOSX_arm64.h" #include "ABISysV_arm64.h" @@ -16,6 +18,8 @@ #include #include +using namespace lldb; + LLDB_PLUGIN_DEFINE(ABIAArch64) void ABIAArch64::Initialize() { @@ -29,14 +33,35 @@ void ABIAArch64::Terminate() { } lldb::addr_t ABIAArch64::FixCodeAddress(lldb::addr_t pc) { - if (lldb::ProcessSP process_sp = GetProcessSP()) - return FixAddress(pc, process_sp->GetCodeAddressMask()); + if (lldb::ProcessSP process_sp = GetProcessSP()) { + // b55 is the highest bit outside TBI (if it's enabled), use + // it to determine if the high bits are set to 0 or 1. + const addr_t pac_sign_extension = 0x0080000000000000ULL; + addr_t mask = process_sp->GetCodeAddressMask(); + // Test if the high memory mask has been overriden separately + if (pc & pac_sign_extension && + process_sp->GetHighmemCodeAddressMask() != LLDB_INVALID_ADDRESS_MASK) + mask = process_sp->GetHighmemCodeAddressMask(); + + if (mask != LLDB_INVALID_ADDRESS_MASK) + return FixAddress(pc, mask); + } return pc; } lldb::addr_t ABIAArch64::FixDataAddress(lldb::addr_t pc) { - if (lldb::ProcessSP process_sp = GetProcessSP()) - return FixAddress(pc, process_sp->GetDataAddressMask()); + if (lldb::ProcessSP process_sp = GetProcessSP()) { + // b55 is the highest bit outside TBI (if it's enabled), use + // it to determine if the high bits are set to 0 or 1. + const addr_t pac_sign_extension = 0x0080000000000000ULL; + addr_t mask = process_sp->GetDataAddressMask(); + // Test if the high memory mask has been overriden separately + if (pc & pac_sign_extension && + process_sp->GetHighmemDataAddressMask() != LLDB_INVALID_ADDRESS_MASK) + mask = process_sp->GetHighmemDataAddressMask(); + if (mask != LLDB_INVALID_ADDRESS_MASK) + return FixAddress(pc, mask); + } return pc; } diff --git a/lldb/source/Plugins/ABI/AArch64/ABIMacOSX_arm64.cpp b/lldb/source/Plugins/ABI/AArch64/ABIMacOSX_arm64.cpp index f4ef9b4fc824..045d6a405e69 100644 --- a/lldb/source/Plugins/ABI/AArch64/ABIMacOSX_arm64.cpp +++ b/lldb/source/Plugins/ABI/AArch64/ABIMacOSX_arm64.cpp @@ -814,11 +814,11 @@ addr_t ABIMacOSX_arm64::FixCodeAddress(addr_t pc) { mask = process_sp->GetCodeAddressMask(); if (pc & pac_sign_extension) { addr_t highmem_mask = process_sp->GetHighmemCodeAddressMask(); - if (highmem_mask) + if (highmem_mask != LLDB_INVALID_ADDRESS_MASK) mask = highmem_mask; } } - if (mask == 0) + if (mask == LLDB_INVALID_ADDRESS_MASK) mask = tbi_mask; return (pc & pac_sign_extension) ? pc | mask : pc & (~mask); @@ -833,11 +833,11 @@ addr_t ABIMacOSX_arm64::FixDataAddress(addr_t pc) { mask = process_sp->GetDataAddressMask(); if (pc & pac_sign_extension) { addr_t highmem_mask = process_sp->GetHighmemDataAddressMask(); - if (highmem_mask) + if (highmem_mask != LLDB_INVALID_ADDRESS_MASK) mask = highmem_mask; } } - if (mask == 0) + if (mask == LLDB_INVALID_ADDRESS_MASK) mask = tbi_mask; return (pc & pac_sign_extension) ? pc | mask : pc & (~mask); diff --git a/lldb/source/Plugins/ABI/AArch64/ABISysV_arm64.cpp b/lldb/source/Plugins/ABI/AArch64/ABISysV_arm64.cpp index bf3c5ddd5889..cd7481f88efd 100644 --- a/lldb/source/Plugins/ABI/AArch64/ABISysV_arm64.cpp +++ b/lldb/source/Plugins/ABI/AArch64/ABISysV_arm64.cpp @@ -775,6 +775,8 @@ ValueObjectSP ABISysV_arm64::GetReturnValueObjectImpl( } lldb::addr_t ABISysV_arm64::FixAddress(addr_t pc, addr_t mask) { + if (mask == LLDB_INVALID_ADDRESS_MASK) + return pc; lldb::addr_t pac_sign_extension = 0x0080000000000000ULL; return (pc & pac_sign_extension) ? pc | mask : pc & (~mask); } @@ -782,12 +784,12 @@ lldb::addr_t ABISysV_arm64::FixAddress(addr_t pc, addr_t mask) { // Reads code or data address mask for the current Linux process. static lldb::addr_t ReadLinuxProcessAddressMask(lldb::ProcessSP process_sp, llvm::StringRef reg_name) { - // 0 means there isn't a mask or it has not been read yet. - // We do not return the top byte mask unless thread_sp is valid. - // This prevents calls to this function before the thread is setup locking - // in the value to just the top byte mask, in cases where pointer - // authentication might also be active. - uint64_t address_mask = 0; + // LLDB_INVALID_ADDRESS_MASK means there isn't a mask or it has not been read + // yet. We do not return the top byte mask unless thread_sp is valid. This + // prevents calls to this function before the thread is setup locking in the + // value to just the top byte mask, in cases where pointer authentication + // might also be active. + uint64_t address_mask = LLDB_INVALID_ADDRESS_MASK; lldb::ThreadSP thread_sp = process_sp->GetThreadList().GetSelectedThread(); if (thread_sp) { // Linux configures user-space virtual addresses with top byte ignored. @@ -814,11 +816,20 @@ static lldb::addr_t ReadLinuxProcessAddressMask(lldb::ProcessSP process_sp, lldb::addr_t ABISysV_arm64::FixCodeAddress(lldb::addr_t pc) { if (lldb::ProcessSP process_sp = GetProcessSP()) { if (process_sp->GetTarget().GetArchitecture().GetTriple().isOSLinux() && - !process_sp->GetCodeAddressMask()) + process_sp->GetCodeAddressMask() == LLDB_INVALID_ADDRESS_MASK) process_sp->SetCodeAddressMask( ReadLinuxProcessAddressMask(process_sp, "code_mask")); - return FixAddress(pc, process_sp->GetCodeAddressMask()); + // b55 is the highest bit outside TBI (if it's enabled), use + // it to determine if the high bits are set to 0 or 1. + const addr_t pac_sign_extension = 0x0080000000000000ULL; + addr_t mask = process_sp->GetCodeAddressMask(); + // Test if the high memory mask has been overriden separately + if (pc & pac_sign_extension && + process_sp->GetHighmemCodeAddressMask() != LLDB_INVALID_ADDRESS_MASK) + mask = process_sp->GetHighmemCodeAddressMask(); + + return FixAddress(pc, mask); } return pc; } @@ -826,11 +837,20 @@ lldb::addr_t ABISysV_arm64::FixCodeAddress(lldb::addr_t pc) { lldb::addr_t ABISysV_arm64::FixDataAddress(lldb::addr_t pc) { if (lldb::ProcessSP process_sp = GetProcessSP()) { if (process_sp->GetTarget().GetArchitecture().GetTriple().isOSLinux() && - !process_sp->GetDataAddressMask()) + process_sp->GetDataAddressMask() == LLDB_INVALID_ADDRESS_MASK) process_sp->SetDataAddressMask( ReadLinuxProcessAddressMask(process_sp, "data_mask")); - return FixAddress(pc, process_sp->GetDataAddressMask()); + // b55 is the highest bit outside TBI (if it's enabled), use + // it to determine if the high bits are set to 0 or 1. + const addr_t pac_sign_extension = 0x0080000000000000ULL; + addr_t mask = process_sp->GetDataAddressMask(); + // Test if the high memory mask has been overriden separately + if (pc & pac_sign_extension && + process_sp->GetHighmemDataAddressMask() != LLDB_INVALID_ADDRESS_MASK) + mask = process_sp->GetHighmemDataAddressMask(); + + return FixAddress(pc, mask); } return pc; } diff --git a/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp b/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp index 3bdbce5a3b7c..8d83937aab66 100644 --- a/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp +++ b/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp @@ -22,6 +22,7 @@ #include "lldb/Target/Target.h" #include "lldb/Target/Thread.h" #include "lldb/Target/ThreadPlanRunToAddress.h" +#include "lldb/Utility/AddressableBits.h" #include "lldb/Utility/DataBuffer.h" #include "lldb/Utility/DataBufferHeap.h" #include "lldb/Utility/LLDBLog.h" @@ -1109,7 +1110,7 @@ void DynamicLoaderDarwinKernel::LoadKernelModuleIfNeeded() { // T1Sz is 25, then 64-25 == 39, bits 0..38 are used for // addressing, bits 39..63 are used for PAC/TBI or whatever. uint32_t virt_addr_bits = 64 - sym_value; - addr_t mask = ~((1ULL << virt_addr_bits) - 1); + addr_t mask = AddressableBits::AddressableBitToMask(virt_addr_bits); m_process->SetCodeAddressMask(mask); m_process->SetDataAddressMask(mask); } else { diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp index 729fbd325671..bcf3a3274cf3 100644 --- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp +++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp @@ -6620,7 +6620,7 @@ bool ObjectFileMachO::SaveCore(const lldb::ProcessSP &process_sp, // Bits will be set to indicate which bits are NOT used in // addressing in this process or 0 for unknown. uint64_t address_mask = process_sp->GetCodeAddressMask(); - if (address_mask != 0) { + if (address_mask != LLDB_INVALID_ADDRESS_MASK) { // LC_NOTE "addrable bits" mach_header.ncmds++; mach_header.sizeofcmds += sizeof(llvm::MachO::note_command); @@ -6654,7 +6654,7 @@ bool ObjectFileMachO::SaveCore(const lldb::ProcessSP &process_sp, std::vector> lc_notes; // Add "addrable bits" LC_NOTE when an address mask is available - if (address_mask != 0) { + if (address_mask != LLDB_INVALID_ADDRESS_MASK) { std::unique_ptr addrable_bits_lcnote_up( new LCNoteEntry(addr_byte_size, byte_order)); addrable_bits_lcnote_up->name = "addrable bits"; diff --git a/lldb/source/Target/ABI.cpp b/lldb/source/Target/ABI.cpp index 86bf01180271..110b5c86fc42 100644 --- a/lldb/source/Target/ABI.cpp +++ b/lldb/source/Target/ABI.cpp @@ -147,6 +147,39 @@ ValueObjectSP ABI::GetReturnValueObject(Thread &thread, CompilerType &ast_type, return return_valobj_sp; } +addr_t ABI::FixCodeAddress(lldb::addr_t pc) { + ProcessSP process_sp(GetProcessSP()); + + addr_t mask = process_sp->GetCodeAddressMask(); + if (mask == LLDB_INVALID_ADDRESS_MASK) + return pc; + + // Assume the high bit is used for addressing, which + // may not be correct on all architectures e.g. AArch64 + // where Top Byte Ignore mode is often used to store + // metadata in the top byte, and b55 is the bit used for + // differentiating between low- and high-memory addresses. + // That target's ABIs need to override this method. + bool is_highmem = pc & (1ULL << 63); + return is_highmem ? pc | mask : pc & (~mask); +} + +addr_t ABI::FixDataAddress(lldb::addr_t pc) { + ProcessSP process_sp(GetProcessSP()); + addr_t mask = process_sp->GetDataAddressMask(); + if (mask == LLDB_INVALID_ADDRESS_MASK) + return pc; + + // Assume the high bit is used for addressing, which + // may not be correct on all architectures e.g. AArch64 + // where Top Byte Ignore mode is often used to store + // metadata in the top byte, and b55 is the bit used for + // differentiating between low- and high-memory addresses. + // That target's ABIs need to override this method. + bool is_highmem = pc & (1ULL << 63); + return is_highmem ? pc | mask : pc & (~mask); +} + ValueObjectSP ABI::GetReturnValueObject(Thread &thread, llvm::Type &ast_type, bool persistent) const { ValueObjectSP return_valobj_sp; diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp index 137795cb8cec..6d58873b54a3 100644 --- a/lldb/source/Target/Process.cpp +++ b/lldb/source/Target/Process.cpp @@ -5682,30 +5682,32 @@ void Process::Flush() { lldb::addr_t Process::GetCodeAddressMask() { if (uint32_t num_bits_setting = GetVirtualAddressableBits()) - return ~((1ULL << num_bits_setting) - 1); + return AddressableBits::AddressableBitToMask(num_bits_setting); return m_code_address_mask; } lldb::addr_t Process::GetDataAddressMask() { if (uint32_t num_bits_setting = GetVirtualAddressableBits()) - return ~((1ULL << num_bits_setting) - 1); + return AddressableBits::AddressableBitToMask(num_bits_setting); return m_data_address_mask; } lldb::addr_t Process::GetHighmemCodeAddressMask() { if (uint32_t num_bits_setting = GetHighmemVirtualAddressableBits()) - return ~((1ULL << num_bits_setting) - 1); - if (m_highmem_code_address_mask) + return AddressableBits::AddressableBitToMask(num_bits_setting); + + if (m_highmem_code_address_mask != LLDB_INVALID_ADDRESS_MASK) return m_highmem_code_address_mask; return GetCodeAddressMask(); } lldb::addr_t Process::GetHighmemDataAddressMask() { if (uint32_t num_bits_setting = GetHighmemVirtualAddressableBits()) - return ~((1ULL << num_bits_setting) - 1); - if (m_highmem_data_address_mask) + return AddressableBits::AddressableBitToMask(num_bits_setting); + + if (m_highmem_data_address_mask != LLDB_INVALID_ADDRESS_MASK) return m_highmem_data_address_mask; return GetDataAddressMask(); } diff --git a/lldb/source/Utility/AddressableBits.cpp b/lldb/source/Utility/AddressableBits.cpp index c6e25f608da7..7f9d7ec6c134 100644 --- a/lldb/source/Utility/AddressableBits.cpp +++ b/lldb/source/Utility/AddressableBits.cpp @@ -33,18 +33,26 @@ void AddressableBits::SetHighmemAddressableBits( m_high_memory_addr_bits = highmem_addressing_bits; } +addr_t AddressableBits::AddressableBitToMask(uint32_t addressable_bits) { + assert(addressable_bits <= sizeof(addr_t) * 8); + if (addressable_bits == 64) + return 0; // all bits used for addressing + else + return ~((1ULL << addressable_bits) - 1); +} + void AddressableBits::SetProcessMasks(Process &process) { if (m_low_memory_addr_bits == 0 && m_high_memory_addr_bits == 0) return; if (m_low_memory_addr_bits != 0) { - addr_t low_addr_mask = ~((1ULL << m_low_memory_addr_bits) - 1); + addr_t low_addr_mask = AddressableBitToMask(m_low_memory_addr_bits); process.SetCodeAddressMask(low_addr_mask); process.SetDataAddressMask(low_addr_mask); } if (m_high_memory_addr_bits != 0) { - addr_t hi_addr_mask = ~((1ULL << m_high_memory_addr_bits) - 1); + addr_t hi_addr_mask = AddressableBitToMask(m_high_memory_addr_bits); process.SetHighmemCodeAddressMask(hi_addr_mask); process.SetHighmemDataAddressMask(hi_addr_mask); } diff --git a/lldb/test/API/python_api/process/address-masks/Makefile b/lldb/test/API/python_api/process/address-masks/Makefile new file mode 100644 index 000000000000..10495940055b --- /dev/null +++ b/lldb/test/API/python_api/process/address-masks/Makefile @@ -0,0 +1,3 @@ +C_SOURCES := main.c + +include Makefile.rules diff --git a/lldb/test/API/python_api/process/address-masks/TestAddressMasks.py b/lldb/test/API/python_api/process/address-masks/TestAddressMasks.py new file mode 100644 index 000000000000..e0a570c15961 --- /dev/null +++ b/lldb/test/API/python_api/process/address-masks/TestAddressMasks.py @@ -0,0 +1,131 @@ +"""Test Python APIs for setting, getting, and using address masks.""" + +import os +import lldb +from lldbsuite.test.decorators import * +from lldbsuite.test.lldbtest import * +from lldbsuite.test import lldbutil + + +class AddressMasksTestCase(TestBase): + NO_DEBUG_INFO_TESTCASE = True + + def reset_all_masks(self, process): + process.SetAddressMask( + lldb.eAddressMaskTypeAll, + lldb.LLDB_INVALID_ADDRESS_MASK, + lldb.eAddressMaskRangeAll, + ) + self.runCmd("settings set target.process.virtual-addressable-bits 0") + self.runCmd("settings set target.process.highmem-virtual-addressable-bits 0") + + def test_address_masks(self): + self.build() + (target, process, t, bp) = lldbutil.run_to_source_breakpoint( + self, "break here", lldb.SBFileSpec("main.c") + ) + + process.SetAddressableBits(lldb.eAddressMaskTypeAll, 42) + self.assertEqual(0x0000029500003F94, process.FixAddress(0x00265E9500003F94)) + self.reset_all_masks(process) + + # ~((1ULL<<42)-1) == 0xfffffc0000000000 + process.SetAddressMask(lldb.eAddressMaskTypeAll, 0xFFFFFC0000000000) + self.assertEqual(0x0000029500003F94, process.FixAddress(0x00265E9500003F94)) + self.reset_all_masks(process) + + # Check that all bits can pass through unmodified + process.SetAddressableBits(lldb.eAddressMaskTypeAll, 64) + self.assertEqual(0x00265E9500003F94, process.FixAddress(0x00265E9500003F94)) + self.reset_all_masks(process) + + process.SetAddressableBits( + lldb.eAddressMaskTypeAll, 42, lldb.eAddressMaskRangeAll + ) + self.assertEqual(0x000002950001F694, process.FixAddress(0x00265E950001F694)) + self.assertEqual(0xFFFFFE950000F694, process.FixAddress(0xFFA65E950000F694)) + self.reset_all_masks(process) + + # Set a eAddressMaskTypeCode which has the low 3 bits marked as non-address + # bits, confirm that they're cleared by FixAddress. + process.SetAddressableBits( + lldb.eAddressMaskTypeAll, 42, lldb.eAddressMaskRangeAll + ) + mask = process.GetAddressMask(lldb.eAddressMaskTypeAny) + process.SetAddressMask(lldb.eAddressMaskTypeCode, mask | 0x3) + self.assertEqual(0x000002950001F697, process.FixAddress(0x00265E950001F697)) + self.assertEqual(0xFFFFFE950000F697, process.FixAddress(0xFFA65E950000F697)) + self.assertEqual( + 0x000002950001F697, + process.FixAddress(0x00265E950001F697, lldb.eAddressMaskTypeData), + ) + self.assertEqual( + 0x000002950001F694, + process.FixAddress(0x00265E950001F697, lldb.eAddressMaskTypeCode), + ) + self.reset_all_masks(process) + + # The user can override whatever settings the Process thinks should be used. + process.SetAddressableBits( + lldb.eAddressMaskTypeAll, 42, lldb.eAddressMaskRangeLow + ) + self.runCmd("settings set target.process.virtual-addressable-bits 15") + self.assertEqual(0x0000000000007694, process.FixAddress(0x00265E950001F694)) + self.assertEqual(0xFFFFFFFFFFFFF694, process.FixAddress(0xFFA65E950000F694)) + self.runCmd("settings set target.process.virtual-addressable-bits 0") + self.assertEqual(0x000002950001F694, process.FixAddress(0x00265E950001F694)) + self.reset_all_masks(process) + + # AArch64 can have different address masks for high and low memory, when different + # page tables are set up. + @skipIf(archs=no_match(["arm64", "arm64e", "aarch64"])) + def test_address_masks_target_supports_highmem_tests(self): + self.build() + (target, process, t, bp) = lldbutil.run_to_source_breakpoint( + self, "break here", lldb.SBFileSpec("main.c") + ) + + process.SetAddressableBits( + lldb.eAddressMaskTypeAll, 42, lldb.eAddressMaskRangeLow + ) + process.SetAddressableBits( + lldb.eAddressMaskTypeAll, 15, lldb.eAddressMaskRangeHigh + ) + self.assertEqual(0x000002950001F694, process.FixAddress(0x00265E950001F694)) + self.assertEqual(0xFFFFFFFFFFFFF694, process.FixAddress(0xFFA65E950000F694)) + self.reset_all_masks(process) + + # The user can override whatever settings the Process thinks should be used. + process.SetAddressableBits( + lldb.eAddressMaskTypeAll, 42, lldb.eAddressMaskRangeAll + ) + self.runCmd("settings set target.process.virtual-addressable-bits 15") + self.runCmd("settings set target.process.highmem-virtual-addressable-bits 15") + self.assertEqual(0x0000000000007694, process.FixAddress(0x00265E950001F694)) + self.assertEqual(0xFFFFFFFFFFFFF694, process.FixAddress(0xFFA65E950000F694)) + self.runCmd("settings set target.process.virtual-addressable-bits 0") + self.runCmd("settings set target.process.highmem-virtual-addressable-bits 0") + self.assertEqual(0x000002950001F694, process.FixAddress(0x00265E950001F694)) + self.reset_all_masks(process) + + # On most targets where we have a single mask for all address range, confirm + # that the high memory masks are ignored. + @skipIf(archs=["arm64", "arm64e", "aarch64"]) + def test_address_masks_target_no_highmem(self): + self.build() + (target, process, t, bp) = lldbutil.run_to_source_breakpoint( + self, "break here", lldb.SBFileSpec("main.c") + ) + + process.SetAddressableBits( + lldb.eAddressMaskTypeAll, 42, lldb.eAddressMaskRangeLow + ) + process.SetAddressableBits( + lldb.eAddressMaskTypeAll, 15, lldb.eAddressMaskRangeHigh + ) + self.assertEqual(0x000002950001F694, process.FixAddress(0x00265E950001F694)) + self.assertEqual(0xFFFFFE950000F694, process.FixAddress(0xFFA65E950000F694)) + self.runCmd("settings set target.process.virtual-addressable-bits 15") + self.runCmd("settings set target.process.highmem-virtual-addressable-bits 42") + self.assertEqual(0x0000000000007694, process.FixAddress(0x00265E950001F694)) + self.assertEqual(0xFFFFFFFFFFFFF694, process.FixAddress(0xFFA65E950000F694)) diff --git a/lldb/test/API/python_api/process/address-masks/main.c b/lldb/test/API/python_api/process/address-masks/main.c new file mode 100644 index 000000000000..f21a10a16d5a --- /dev/null +++ b/lldb/test/API/python_api/process/address-masks/main.c @@ -0,0 +1,5 @@ +#include + +int main(int argc, char const *argv[]) { + puts("Hello address masking world"); // break here +} -- GitLab From 064c2e7579228389c8a6b78ee40280c319935715 Mon Sep 17 00:00:00 2001 From: Krzysztof Parzyszek Date: Wed, 6 Mar 2024 12:07:07 -0600 Subject: [PATCH 333/929] Fix failing TableGen tests --- llvm/test/TableGen/directive1.td | 3 +-- llvm/test/TableGen/directive2.td | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/llvm/test/TableGen/directive1.td b/llvm/test/TableGen/directive1.td index aa47d7ee666d..3184f625ead9 100644 --- a/llvm/test/TableGen/directive1.td +++ b/llvm/test/TableGen/directive1.td @@ -370,9 +370,8 @@ def TDL_DirA : Directive<"dira"> { // IMPL-NEXT: switch (Dir) { // IMPL-NEXT: case llvm::tdl::Directive::TDLD_dira: // IMPL-NEXT: return llvm::tdl::Association::None; -// IMPL-NEXT: default: -// IMPL-NEXT: llvm_unreachable("Unexpected directive"); // IMPL-NEXT: } // switch(Dir) +// IMPL-NEXT: llvm_unreachable("Unexpected directive"); // IMPL-NEXT: } // IMPL-EMPTY: // IMPL-NEXT: #endif // GEN_DIRECTIVES_IMPL diff --git a/llvm/test/TableGen/directive2.td b/llvm/test/TableGen/directive2.td index 90f9c45b5e15..d6fa4835c8df 100644 --- a/llvm/test/TableGen/directive2.td +++ b/llvm/test/TableGen/directive2.td @@ -301,9 +301,8 @@ def TDL_DirA : Directive<"dira"> { // IMPL-NEXT: switch (Dir) { // IMPL-NEXT: case llvm::tdl::Directive::TDLD_dira: // IMPL-NEXT: return llvm::tdl::Association::Block; -// IMPL-NEXT: default: -// IMPL-NEXT: llvm_unreachable("Unexpected directive"); // IMPL-NEXT: } // switch(Dir) +// IMPL-NEXT: llvm_unreachable("Unexpected directive"); // IMPL-NEXT: } // IMPL-EMPTY: // IMPL-NEXT: #endif // GEN_DIRECTIVES_IMPL -- GitLab From 2542d34522784e27850a17a8c39a1ad4b43622ab Mon Sep 17 00:00:00 2001 From: Slava Zakharin Date: Wed, 6 Mar 2024 10:19:58 -0800 Subject: [PATCH 334/929] [mlir][inliner] Refactor MLIR inliner pass and utils. (#84059) This is just code refactoring done as a preparation for adding MLIR inliner cost model hook(s). Related discussion: https://discourse.llvm.org/t/inliner-cost-model/2992 The logic of SCC-based MLIR inliner is separated into the Inliner implementation. The MLIR inliner pass becomes, well, just a pass that invokes the SCC-based MLIR inliner. --- mlir/include/mlir/Transforms/Inliner.h | 120 +++++ mlir/include/mlir/Transforms/Passes.td | 4 +- mlir/lib/Transforms/CMakeLists.txt | 2 +- mlir/lib/Transforms/InlinerPass.cpp | 155 ++++++ mlir/lib/Transforms/Utils/CMakeLists.txt | 1 + mlir/lib/Transforms/{ => Utils}/Inliner.cpp | 533 ++++++++------------ 6 files changed, 500 insertions(+), 315 deletions(-) create mode 100644 mlir/include/mlir/Transforms/Inliner.h create mode 100644 mlir/lib/Transforms/InlinerPass.cpp rename mlir/lib/Transforms/{ => Utils}/Inliner.cpp (76%) diff --git a/mlir/include/mlir/Transforms/Inliner.h b/mlir/include/mlir/Transforms/Inliner.h new file mode 100644 index 000000000000..1fe61fb4bbe7 --- /dev/null +++ b/mlir/include/mlir/Transforms/Inliner.h @@ -0,0 +1,120 @@ +//===- Inliner.h - Inliner pass utilities -----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This header file declares utility structures for the inliner pass. +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_TRANSFORMS_INLINER_H +#define MLIR_TRANSFORMS_INLINER_H + +#include "mlir/Analysis/CallGraph.h" +#include "mlir/Interfaces/CallInterfaces.h" +#include "mlir/Pass/AnalysisManager.h" +#include "mlir/Pass/PassManager.h" +#include "mlir/Support/LogicalResult.h" +#include "llvm/ADT/StringMap.h" + +namespace mlir { +class OpPassManager; +class Operation; + +class InlinerConfig { +public: + using DefaultPipelineTy = std::function; + using OpPipelinesTy = llvm::StringMap; + + InlinerConfig() = default; + InlinerConfig(DefaultPipelineTy defaultPipeline, + unsigned maxInliningIterations) + : defaultPipeline(std::move(defaultPipeline)), + maxInliningIterations(maxInliningIterations) {} + + const DefaultPipelineTy &getDefaultPipeline() const { + return defaultPipeline; + } + const OpPipelinesTy &getOpPipelines() const { return opPipelines; } + unsigned getMaxInliningIterations() const { return maxInliningIterations; } + void setDefaultPipeline(DefaultPipelineTy pipeline) { + defaultPipeline = std::move(pipeline); + } + void setOpPipelines(OpPipelinesTy pipelines) { + opPipelines = std::move(pipelines); + } + void setMaxInliningIterations(unsigned max) { maxInliningIterations = max; } + +private: + /// An optional function that constructs an optimization pipeline for + /// a given operation. This optimization pipeline is applied + /// only to those callable operations that do not have dedicated + /// optimization pipeline in opPipelines (based on the operation name). + DefaultPipelineTy defaultPipeline; + /// A map of operation names to pass pipelines to use when optimizing + /// callable operations of these types. This provides a specialized pipeline + /// instead of the one produced by defaultPipeline. + OpPipelinesTy opPipelines; + /// For SCC-based inlining algorithms, specifies maximum number of iterations + /// when inlining within an SCC. + unsigned maxInliningIterations{0}; +}; + +/// This is an implementation of the inliner +/// that operates bottom up over the Strongly Connected Components(SCCs) +/// of the CallGraph. This enables a more incremental propagation +/// of inlining decisions from the leafs to the roots of the callgraph. +class Inliner { +public: + using RunPipelineHelperTy = std::function; + + Inliner(Operation *op, CallGraph &cg, Pass &pass, AnalysisManager am, + RunPipelineHelperTy runPipelineHelper, const InlinerConfig &config) + : op(op), cg(cg), pass(pass), am(am), + runPipelineHelper(std::move(runPipelineHelper)), config(config) {} + Inliner(Inliner &) = delete; + void operator=(const Inliner &) = delete; + + /// Perform inlining on a OpTrait::SymbolTable operation. + LogicalResult doInlining(); + + /// This struct represents a resolved call to a given callgraph node. Given + /// that the call does not actually contain a direct reference to the + /// Region(CallGraphNode) that it is dispatching to, we need to resolve them + /// explicitly. + struct ResolvedCall { + ResolvedCall(CallOpInterface call, CallGraphNode *sourceNode, + CallGraphNode *targetNode) + : call(call), sourceNode(sourceNode), targetNode(targetNode) {} + CallOpInterface call; + CallGraphNode *sourceNode, *targetNode; + }; + +protected: + /// An OpTrait::SymbolTable operation to run the inlining on. + Operation *op; + /// A CallGraph analysis for the given operation. + CallGraph &cg; + /// A reference to the pass using this inliner. + Pass &pass; + /// Analysis manager for the given operation instance. + AnalysisManager am; + /// A callback for running a nested pass pipeline on the operation + /// contained within the main operation. + const RunPipelineHelperTy runPipelineHelper; + /// The inliner configuration parameters. + const InlinerConfig &config; + +private: + /// Forward declaration of the class providing the actual implementation. + class Impl; + +public: +}; +} // namespace mlir + +#endif // MLIR_TRANSFORMS_INLINER_H diff --git a/mlir/include/mlir/Transforms/Passes.td b/mlir/include/mlir/Transforms/Passes.td index 2d2d54fb8fb5..b8fdf7a58047 100644 --- a/mlir/include/mlir/Transforms/Passes.td +++ b/mlir/include/mlir/Transforms/Passes.td @@ -269,7 +269,9 @@ def Inliner : Pass<"inline"> { let constructor = "mlir::createInlinerPass()"; let options = [ Option<"defaultPipelineStr", "default-pipeline", "std::string", - /*default=*/"\"canonicalize\"", "The default optimizer pipeline used for callables">, + /*default=*/"\"canonicalize\"", + "The optimizer pipeline used for callables that do not have " + "a dedicated optimizer pipeline in opPipelineList">, ListOption<"opPipelineList", "op-pipelines", "OpPassManager", "Callable operation specific optimizer pipelines (in the form " "of `dialect.op(pipeline)`)">, diff --git a/mlir/lib/Transforms/CMakeLists.txt b/mlir/lib/Transforms/CMakeLists.txt index af51a4ab1157..6c32ecf8a2a2 100644 --- a/mlir/lib/Transforms/CMakeLists.txt +++ b/mlir/lib/Transforms/CMakeLists.txt @@ -5,7 +5,7 @@ add_mlir_library(MLIRTransforms ControlFlowSink.cpp CSE.cpp GenerateRuntimeVerification.cpp - Inliner.cpp + InlinerPass.cpp LocationSnapshot.cpp LoopInvariantCodeMotion.cpp Mem2Reg.cpp diff --git a/mlir/lib/Transforms/InlinerPass.cpp b/mlir/lib/Transforms/InlinerPass.cpp new file mode 100644 index 000000000000..c058e8050cd1 --- /dev/null +++ b/mlir/lib/Transforms/InlinerPass.cpp @@ -0,0 +1,155 @@ +//===- InlinerPass.cpp - Pass to inline function calls --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements a basic inlining algorithm that operates bottom up over +// the Strongly Connect Components(SCCs) of the CallGraph. This enables a more +// incremental propagation of inlining decisions from the leafs to the roots of +// the callgraph. +// +//===----------------------------------------------------------------------===// + +#include "mlir/Transforms/Passes.h" + +#include "mlir/Analysis/CallGraph.h" +#include "mlir/Pass/PassManager.h" +#include "mlir/Transforms/Inliner.h" + +namespace mlir { +#define GEN_PASS_DEF_INLINER +#include "mlir/Transforms/Passes.h.inc" +} // namespace mlir + +using namespace mlir; + +/// This function implements the inliner optimization pipeline. +static void defaultInlinerOptPipeline(OpPassManager &pm) { + pm.addPass(createCanonicalizerPass()); +} + +//===----------------------------------------------------------------------===// +// InlinerPass +//===----------------------------------------------------------------------===// + +namespace { +class InlinerPass : public impl::InlinerBase { +public: + InlinerPass(); + InlinerPass(const InlinerPass &) = default; + InlinerPass(std::function defaultPipeline); + InlinerPass(std::function defaultPipeline, + llvm::StringMap opPipelines); + void runOnOperation() override; + + /// A callback provided to the inliner driver to execute + /// the specified pass pipeline on the given operation + /// within the context of the current inliner pass, + /// which is passed as the first argument. + /// runPipeline API is protected within the Pass class, + /// so this helper is required to call it from the foreign + /// inliner driver. + static LogicalResult runPipelineHelper(Pass &pass, OpPassManager &pipeline, + Operation *op) { + return mlir::cast(pass).runPipeline(pipeline, op); + } + +private: + /// Attempt to initialize the options of this pass from the given string. + /// Derived classes may override this method to hook into the point at which + /// options are initialized, but should generally always invoke this base + /// class variant. + LogicalResult initializeOptions(StringRef options) override; + + /// Inliner configuration parameters created from the pass options. + InlinerConfig config; +}; +} // namespace + +InlinerPass::InlinerPass() : InlinerPass(defaultInlinerOptPipeline) {} + +InlinerPass::InlinerPass( + std::function defaultPipelineArg) + : InlinerPass(std::move(defaultPipelineArg), + llvm::StringMap{}) {} + +InlinerPass::InlinerPass(std::function defaultPipeline, + llvm::StringMap opPipelines) + : config(std::move(defaultPipeline), maxInliningIterations) { + if (opPipelines.empty()) + return; + + // Update the option for the op specific optimization pipelines. + for (auto &it : opPipelines) + opPipelineList.addValue(it.second); + config.setOpPipelines(std::move(opPipelines)); +} + +void InlinerPass::runOnOperation() { + CallGraph &cg = getAnalysis(); + + // The inliner should only be run on operations that define a symbol table, + // as the callgraph will need to resolve references. + Operation *op = getOperation(); + if (!op->hasTrait()) { + op->emitOpError() << " was scheduled to run under the inliner, but does " + "not define a symbol table"; + return signalPassFailure(); + } + + // Get an instance of the inliner. + Inliner inliner(op, cg, *this, getAnalysisManager(), runPipelineHelper, + config); + + // Run the inlining. + if (failed(inliner.doInlining())) + signalPassFailure(); + return; +} + +LogicalResult InlinerPass::initializeOptions(StringRef options) { + if (failed(Pass::initializeOptions(options))) + return failure(); + + // Initialize the pipeline builder for operations without the dedicated + // optimization pipeline in opPipelineList to use the option string. + // TODO: Use a generic pass manager for the pre-inline pipeline, and remove + // this. + if (!defaultPipelineStr.empty()) { + std::string defaultPipelineCopy = defaultPipelineStr; + config.setDefaultPipeline([=](OpPassManager &pm) { + (void)parsePassPipeline(defaultPipelineCopy, pm); + }); + } else if (defaultPipelineStr.getNumOccurrences()) { + config.setDefaultPipeline(nullptr); + } + + // Initialize the op specific pass pipelines. + llvm::StringMap pipelines; + for (OpPassManager pipeline : opPipelineList) + if (!pipeline.empty()) + pipelines.try_emplace(pipeline.getOpAnchorName(), pipeline); + config.setOpPipelines(std::move(pipelines)); + + config.setMaxInliningIterations(maxInliningIterations); + + return success(); +} + +std::unique_ptr mlir::createInlinerPass() { + return std::make_unique(); +} +std::unique_ptr +mlir::createInlinerPass(llvm::StringMap opPipelines) { + return std::make_unique(defaultInlinerOptPipeline, + std::move(opPipelines)); +} +std::unique_ptr mlir::createInlinerPass( + llvm::StringMap opPipelines, + std::function defaultPipelineBuilder) { + return std::make_unique(std::move(defaultPipelineBuilder), + std::move(opPipelines)); +} diff --git a/mlir/lib/Transforms/Utils/CMakeLists.txt b/mlir/lib/Transforms/Utils/CMakeLists.txt index 1c608e0634a6..d6aac0e2da4f 100644 --- a/mlir/lib/Transforms/Utils/CMakeLists.txt +++ b/mlir/lib/Transforms/Utils/CMakeLists.txt @@ -5,6 +5,7 @@ add_mlir_library(MLIRTransformUtils DialectConversion.cpp FoldUtils.cpp GreedyPatternRewriteDriver.cpp + Inliner.cpp InliningUtils.cpp LoopInvariantCodeMotionUtils.cpp OneToNTypeConversion.cpp diff --git a/mlir/lib/Transforms/Inliner.cpp b/mlir/lib/Transforms/Utils/Inliner.cpp similarity index 76% rename from mlir/lib/Transforms/Inliner.cpp rename to mlir/lib/Transforms/Utils/Inliner.cpp index b32b0fc28c78..74776a73db9a 100644 --- a/mlir/lib/Transforms/Inliner.cpp +++ b/mlir/lib/Transforms/Utils/Inliner.cpp @@ -1,4 +1,4 @@ -//===- Inliner.cpp - Pass to inline function calls ------------------------===// +//===- Inliner.cpp ---- SCC-based inliner ---------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,38 +6,29 @@ // //===----------------------------------------------------------------------===// // -// This file implements a basic inlining algorithm that operates bottom up over -// the Strongly Connect Components(SCCs) of the CallGraph. This enables a more -// incremental propagation of inlining decisions from the leafs to the roots of -// the callgraph. +// This file implements Inliner that uses a basic inlining +// algorithm that operates bottom up over the Strongly Connect Components(SCCs) +// of the CallGraph. This enables a more incremental propagation of inlining +// decisions from the leafs to the roots of the callgraph. // //===----------------------------------------------------------------------===// -#include "mlir/Transforms/Passes.h" - -#include "mlir/Analysis/CallGraph.h" +#include "mlir/Transforms/Inliner.h" #include "mlir/IR/Threading.h" #include "mlir/Interfaces/CallInterfaces.h" #include "mlir/Interfaces/SideEffectInterfaces.h" -#include "mlir/Pass/PassManager.h" +#include "mlir/Pass/Pass.h" #include "mlir/Support/DebugStringHelper.h" #include "mlir/Transforms/InliningUtils.h" #include "llvm/ADT/SCCIterator.h" +#include "llvm/ADT/SmallPtrSet.h" #include "llvm/Support/Debug.h" -namespace mlir { -#define GEN_PASS_DEF_INLINER -#include "mlir/Transforms/Passes.h.inc" -} // namespace mlir - #define DEBUG_TYPE "inlining" using namespace mlir; -/// This function implements the default inliner optimization pipeline. -static void defaultInlinerOptPipeline(OpPassManager &pm) { - pm.addPass(createCanonicalizerPass()); -} +using ResolvedCall = Inliner::ResolvedCall; //===----------------------------------------------------------------------===// // Symbol Use Tracking @@ -306,20 +297,6 @@ static LogicalResult runTransformOnCGSCCs( return success(); } -namespace { -/// This struct represents a resolved call to a given callgraph node. Given that -/// the call does not actually contain a direct reference to the -/// Region(CallGraphNode) that it is dispatching to, we need to resolve them -/// explicitly. -struct ResolvedCall { - ResolvedCall(CallOpInterface call, CallGraphNode *sourceNode, - CallGraphNode *targetNode) - : call(call), sourceNode(sourceNode), targetNode(targetNode) {} - CallOpInterface call; - CallGraphNode *sourceNode, *targetNode; -}; -} // namespace - /// Collect all of the callable operations within the given range of blocks. If /// `traverseNestedCGNodes` is true, this will also collect call operations /// inside of nested callgraph nodes. @@ -368,7 +345,7 @@ static void collectCallOps(iterator_range blocks, } //===----------------------------------------------------------------------===// -// Inliner +// InlinerInterfaceImpl //===----------------------------------------------------------------------===// #ifndef NDEBUG @@ -397,9 +374,9 @@ static bool inlineHistoryIncludes( namespace { /// This class provides a specialization of the main inlining interface. -struct Inliner : public InlinerInterface { - Inliner(MLIRContext *context, CallGraph &cg, - SymbolTableCollection &symbolTable) +struct InlinerInterfaceImpl : public InlinerInterface { + InlinerInterfaceImpl(MLIRContext *context, CallGraph &cg, + SymbolTableCollection &symbolTable) : InlinerInterface(context), cg(cg), symbolTable(symbolTable) {} /// Process a set of blocks that have been inlined. This callback is invoked @@ -442,45 +419,172 @@ struct Inliner : public InlinerInterface { }; } // namespace -/// Returns true if the given call should be inlined. -static bool shouldInline(ResolvedCall &resolvedCall) { - // Don't allow inlining terminator calls. We currently don't support this - // case. - if (resolvedCall.call->hasTrait()) - return false; +namespace mlir { - // Don't allow inlining if the target is an ancestor of the call. This - // prevents inlining recursively. - Region *callableRegion = resolvedCall.targetNode->getCallableRegion(); - if (callableRegion->isAncestor(resolvedCall.call->getParentRegion())) - return false; +class Inliner::Impl { +public: + Impl(Inliner &inliner) : inliner(inliner) {} - // Don't allow inlining if the callee has multiple blocks (unstructured - // control flow) but we cannot be sure that the caller region supports that. - bool calleeHasMultipleBlocks = - llvm::hasNItemsOrMore(*callableRegion, /*N=*/2); - // If both parent ops have the same type, it is safe to inline. Otherwise, - // decide based on whether the op has the SingleBlock trait or not. - // Note: This check does currently not account for SizedRegion/MaxSizedRegion. - auto callerRegionSupportsMultipleBlocks = [&]() { - return callableRegion->getParentOp()->getName() == - resolvedCall.call->getParentOp()->getName() || - !resolvedCall.call->getParentOp() - ->mightHaveTrait(); - }; - if (calleeHasMultipleBlocks && !callerRegionSupportsMultipleBlocks()) - return false; + /// Attempt to inline calls within the given scc, and run simplifications, + /// until a fixed point is reached. This allows for the inlining of newly + /// devirtualized calls. Returns failure if there was a fatal error during + /// inlining. + LogicalResult inlineSCC(InlinerInterfaceImpl &inlinerIface, + CGUseList &useList, CallGraphSCC ¤tSCC, + MLIRContext *context); - // Otherwise, inline. - return true; +private: + /// Optimize the nodes within the given SCC with one of the held optimization + /// pass pipelines. Returns failure if an error occurred during the + /// optimization of the SCC, success otherwise. + LogicalResult optimizeSCC(CallGraph &cg, CGUseList &useList, + CallGraphSCC ¤tSCC, MLIRContext *context); + + /// Optimize the nodes within the given SCC in parallel. Returns failure if an + /// error occurred during the optimization of the SCC, success otherwise. + LogicalResult optimizeSCCAsync(MutableArrayRef nodesToVisit, + MLIRContext *context); + + /// Optimize the given callable node with one of the pass managers provided + /// with `pipelines`, or the generic pre-inline pipeline. Returns failure if + /// an error occurred during the optimization of the callable, success + /// otherwise. + LogicalResult optimizeCallable(CallGraphNode *node, + llvm::StringMap &pipelines); + + /// Attempt to inline calls within the given scc. This function returns + /// success if any calls were inlined, failure otherwise. + LogicalResult inlineCallsInSCC(InlinerInterfaceImpl &inlinerIface, + CGUseList &useList, CallGraphSCC ¤tSCC); + + /// Returns true if the given call should be inlined. + bool shouldInline(ResolvedCall &resolvedCall); + +private: + Inliner &inliner; + llvm::SmallVector> pipelines; +}; + +LogicalResult Inliner::Impl::inlineSCC(InlinerInterfaceImpl &inlinerIface, + CGUseList &useList, + CallGraphSCC ¤tSCC, + MLIRContext *context) { + // Continuously simplify and inline until we either reach a fixed point, or + // hit the maximum iteration count. Simplifying early helps to refine the cost + // model, and in future iterations may devirtualize new calls. + unsigned iterationCount = 0; + do { + if (failed(optimizeSCC(inlinerIface.cg, useList, currentSCC, context))) + return failure(); + if (failed(inlineCallsInSCC(inlinerIface, useList, currentSCC))) + break; + } while (++iterationCount < inliner.config.getMaxInliningIterations()); + return success(); +} + +LogicalResult Inliner::Impl::optimizeSCC(CallGraph &cg, CGUseList &useList, + CallGraphSCC ¤tSCC, + MLIRContext *context) { + // Collect the sets of nodes to simplify. + SmallVector nodesToVisit; + for (auto *node : currentSCC) { + if (node->isExternal()) + continue; + + // Don't simplify nodes with children. Nodes with children require special + // handling as we may remove the node during simplification. In the future, + // we should be able to handle this case with proper node deletion tracking. + if (node->hasChildren()) + continue; + + // We also won't apply simplifications to nodes that can't have passes + // scheduled on them. + auto *region = node->getCallableRegion(); + if (!region->getParentOp()->hasTrait()) + continue; + nodesToVisit.push_back(node); + } + if (nodesToVisit.empty()) + return success(); + + // Optimize each of the nodes within the SCC in parallel. + if (failed(optimizeSCCAsync(nodesToVisit, context))) + return failure(); + + // Recompute the uses held by each of the nodes. + for (CallGraphNode *node : nodesToVisit) + useList.recomputeUses(node, cg); + return success(); +} + +LogicalResult +Inliner::Impl::optimizeSCCAsync(MutableArrayRef nodesToVisit, + MLIRContext *ctx) { + // We must maintain a fixed pool of pass managers which is at least as large + // as the maximum parallelism of the failableParallelForEach below. + // Note: The number of pass managers here needs to remain constant + // to prevent issues with pass instrumentations that rely on having the same + // pass manager for the main thread. + size_t numThreads = ctx->getNumThreads(); + const auto &opPipelines = inliner.config.getOpPipelines(); + if (pipelines.size() < numThreads) { + pipelines.reserve(numThreads); + pipelines.resize(numThreads, opPipelines); + } + + // Ensure an analysis manager has been constructed for each of the nodes. + // This prevents thread races when running the nested pipelines. + for (CallGraphNode *node : nodesToVisit) + inliner.am.nest(node->getCallableRegion()->getParentOp()); + + // An atomic failure variable for the async executors. + std::vector> activePMs(pipelines.size()); + std::fill(activePMs.begin(), activePMs.end(), false); + return failableParallelForEach(ctx, nodesToVisit, [&](CallGraphNode *node) { + // Find a pass manager for this operation. + auto it = llvm::find_if(activePMs, [](std::atomic &isActive) { + bool expectedInactive = false; + return isActive.compare_exchange_strong(expectedInactive, true); + }); + assert(it != activePMs.end() && + "could not find inactive pass manager for thread"); + unsigned pmIndex = it - activePMs.begin(); + + // Optimize this callable node. + LogicalResult result = optimizeCallable(node, pipelines[pmIndex]); + + // Reset the active bit for this pass manager. + activePMs[pmIndex].store(false); + return result; + }); +} + +LogicalResult +Inliner::Impl::optimizeCallable(CallGraphNode *node, + llvm::StringMap &pipelines) { + Operation *callable = node->getCallableRegion()->getParentOp(); + StringRef opName = callable->getName().getStringRef(); + auto pipelineIt = pipelines.find(opName); + const auto &defaultPipeline = inliner.config.getDefaultPipeline(); + if (pipelineIt == pipelines.end()) { + // If a pipeline didn't exist, use the generic pipeline if possible. + if (!defaultPipeline) + return success(); + + OpPassManager defaultPM(opName); + defaultPipeline(defaultPM); + pipelineIt = pipelines.try_emplace(opName, std::move(defaultPM)).first; + } + return inliner.runPipelineHelper(inliner.pass, pipelineIt->second, callable); } /// Attempt to inline calls within the given scc. This function returns /// success if any calls were inlined, failure otherwise. -static LogicalResult inlineCallsInSCC(Inliner &inliner, CGUseList &useList, - CallGraphSCC ¤tSCC) { - CallGraph &cg = inliner.cg; - auto &calls = inliner.calls; +LogicalResult +Inliner::Impl::inlineCallsInSCC(InlinerInterfaceImpl &inlinerIface, + CGUseList &useList, CallGraphSCC ¤tSCC) { + CallGraph &cg = inlinerIface.cg; + auto &calls = inlinerIface.calls; // A set of dead nodes to remove after inlining. llvm::SmallSetVector deadNodes; @@ -496,8 +600,9 @@ static LogicalResult inlineCallsInSCC(Inliner &inliner, CGUseList &useList, if (useList.isDead(node)) { deadNodes.insert(node); } else { - collectCallOps(*node->getCallableRegion(), node, cg, inliner.symbolTable, - calls, /*traverseNestedCGNodes=*/false); + collectCallOps(*node->getCallableRegion(), node, cg, + inlinerIface.symbolTable, calls, + /*traverseNestedCGNodes=*/false); } } @@ -544,9 +649,10 @@ static LogicalResult inlineCallsInSCC(Inliner &inliner, CGUseList &useList, // then inline it in-place and delete the node if successful. bool inlineInPlace = useList.hasOneUseAndDiscardable(it.targetNode); - LogicalResult inlineResult = inlineCall( - inliner, call, cast(targetRegion->getParentOp()), - targetRegion, /*shouldCloneInlinedRegion=*/!inlineInPlace); + LogicalResult inlineResult = + inlineCall(inlinerIface, call, + cast(targetRegion->getParentOp()), + targetRegion, /*shouldCloneInlinedRegion=*/!inlineInPlace); if (failed(inlineResult)) { LLVM_DEBUG(llvm::dbgs() << "** Failed to inline\n"); continue; @@ -592,262 +698,63 @@ static LogicalResult inlineCallsInSCC(Inliner &inliner, CGUseList &useList, for (CallGraphNode *node : deadNodes) { currentSCC.remove(node); - inliner.markForDeletion(node); + inlinerIface.markForDeletion(node); } calls.clear(); return success(inlinedAnyCalls); } -//===----------------------------------------------------------------------===// -// InlinerPass -//===----------------------------------------------------------------------===// - -namespace { -class InlinerPass : public impl::InlinerBase { -public: - InlinerPass(); - InlinerPass(const InlinerPass &) = default; - InlinerPass(std::function defaultPipeline); - InlinerPass(std::function defaultPipeline, - llvm::StringMap opPipelines); - void runOnOperation() override; - -private: - /// Attempt to inline calls within the given scc, and run simplifications, - /// until a fixed point is reached. This allows for the inlining of newly - /// devirtualized calls. Returns failure if there was a fatal error during - /// inlining. - LogicalResult inlineSCC(Inliner &inliner, CGUseList &useList, - CallGraphSCC ¤tSCC, MLIRContext *context); - - /// Optimize the nodes within the given SCC with one of the held optimization - /// pass pipelines. Returns failure if an error occurred during the - /// optimization of the SCC, success otherwise. - LogicalResult optimizeSCC(CallGraph &cg, CGUseList &useList, - CallGraphSCC ¤tSCC, MLIRContext *context); - - /// Optimize the nodes within the given SCC in parallel. Returns failure if an - /// error occurred during the optimization of the SCC, success otherwise. - LogicalResult optimizeSCCAsync(MutableArrayRef nodesToVisit, - MLIRContext *context); - - /// Optimize the given callable node with one of the pass managers provided - /// with `pipelines`, or the default pipeline. Returns failure if an error - /// occurred during the optimization of the callable, success otherwise. - LogicalResult optimizeCallable(CallGraphNode *node, - llvm::StringMap &pipelines); - - /// Attempt to initialize the options of this pass from the given string. - /// Derived classes may override this method to hook into the point at which - /// options are initialized, but should generally always invoke this base - /// class variant. - LogicalResult initializeOptions(StringRef options) override; - - /// An optional function that constructs a default optimization pipeline for - /// a given operation. - std::function defaultPipeline; - /// A map of operation names to pass pipelines to use when optimizing - /// callable operations of these types. This provides a specialized pipeline - /// instead of the default. The vector size is the number of threads used - /// during optimization. - SmallVector, 8> opPipelines; -}; -} // namespace +/// Returns true if the given call should be inlined. +bool Inliner::Impl::shouldInline(ResolvedCall &resolvedCall) { + // Don't allow inlining terminator calls. We currently don't support this + // case. + if (resolvedCall.call->hasTrait()) + return false; -InlinerPass::InlinerPass() : InlinerPass(defaultInlinerOptPipeline) {} -InlinerPass::InlinerPass( - std::function defaultPipelineArg) - : defaultPipeline(std::move(defaultPipelineArg)) { - opPipelines.push_back({}); -} + // Don't allow inlining if the target is an ancestor of the call. This + // prevents inlining recursively. + Region *callableRegion = resolvedCall.targetNode->getCallableRegion(); + if (callableRegion->isAncestor(resolvedCall.call->getParentRegion())) + return false; -InlinerPass::InlinerPass(std::function defaultPipeline, - llvm::StringMap opPipelines) - : InlinerPass(std::move(defaultPipeline)) { - if (opPipelines.empty()) - return; + // Don't allow inlining if the callee has multiple blocks (unstructured + // control flow) but we cannot be sure that the caller region supports that. + bool calleeHasMultipleBlocks = + llvm::hasNItemsOrMore(*callableRegion, /*N=*/2); + // If both parent ops have the same type, it is safe to inline. Otherwise, + // decide based on whether the op has the SingleBlock trait or not. + // Note: This check does currently not account for SizedRegion/MaxSizedRegion. + auto callerRegionSupportsMultipleBlocks = [&]() { + return callableRegion->getParentOp()->getName() == + resolvedCall.call->getParentOp()->getName() || + !resolvedCall.call->getParentOp() + ->mightHaveTrait(); + }; + if (calleeHasMultipleBlocks && !callerRegionSupportsMultipleBlocks()) + return false; - // Update the option for the op specific optimization pipelines. - for (auto &it : opPipelines) - opPipelineList.addValue(it.second); - this->opPipelines.emplace_back(std::move(opPipelines)); + // Otherwise, inline. + return true; } -void InlinerPass::runOnOperation() { - CallGraph &cg = getAnalysis(); - auto *context = &getContext(); - - // The inliner should only be run on operations that define a symbol table, - // as the callgraph will need to resolve references. - Operation *op = getOperation(); - if (!op->hasTrait()) { - op->emitOpError() << " was scheduled to run under the inliner, but does " - "not define a symbol table"; - return signalPassFailure(); - } - +LogicalResult Inliner::doInlining() { + Impl impl(*this); + auto *context = op->getContext(); // Run the inline transform in post-order over the SCCs in the callgraph. SymbolTableCollection symbolTable; - Inliner inliner(context, cg, symbolTable); - CGUseList useList(getOperation(), cg, symbolTable); + // FIXME: some clean-up can be done for the arguments + // of the Impl's methods, if the inlinerIface and useList + // become the states of the Impl. + InlinerInterfaceImpl inlinerIface(context, cg, symbolTable); + CGUseList useList(op, cg, symbolTable); LogicalResult result = runTransformOnCGSCCs(cg, [&](CallGraphSCC &scc) { - return inlineSCC(inliner, useList, scc, context); + return impl.inlineSCC(inlinerIface, useList, scc, context); }); if (failed(result)) - return signalPassFailure(); - - // After inlining, make sure to erase any callables proven to be dead. - inliner.eraseDeadCallables(); -} - -LogicalResult InlinerPass::inlineSCC(Inliner &inliner, CGUseList &useList, - CallGraphSCC ¤tSCC, - MLIRContext *context) { - // Continuously simplify and inline until we either reach a fixed point, or - // hit the maximum iteration count. Simplifying early helps to refine the cost - // model, and in future iterations may devirtualize new calls. - unsigned iterationCount = 0; - do { - if (failed(optimizeSCC(inliner.cg, useList, currentSCC, context))) - return failure(); - if (failed(inlineCallsInSCC(inliner, useList, currentSCC))) - break; - } while (++iterationCount < maxInliningIterations); - return success(); -} - -LogicalResult InlinerPass::optimizeSCC(CallGraph &cg, CGUseList &useList, - CallGraphSCC ¤tSCC, - MLIRContext *context) { - // Collect the sets of nodes to simplify. - SmallVector nodesToVisit; - for (auto *node : currentSCC) { - if (node->isExternal()) - continue; - - // Don't simplify nodes with children. Nodes with children require special - // handling as we may remove the node during simplification. In the future, - // we should be able to handle this case with proper node deletion tracking. - if (node->hasChildren()) - continue; - - // We also won't apply simplifications to nodes that can't have passes - // scheduled on them. - auto *region = node->getCallableRegion(); - if (!region->getParentOp()->hasTrait()) - continue; - nodesToVisit.push_back(node); - } - if (nodesToVisit.empty()) - return success(); - - // Optimize each of the nodes within the SCC in parallel. - if (failed(optimizeSCCAsync(nodesToVisit, context))) - return failure(); - - // Recompute the uses held by each of the nodes. - for (CallGraphNode *node : nodesToVisit) - useList.recomputeUses(node, cg); - return success(); -} - -LogicalResult -InlinerPass::optimizeSCCAsync(MutableArrayRef nodesToVisit, - MLIRContext *ctx) { - // We must maintain a fixed pool of pass managers which is at least as large - // as the maximum parallelism of the failableParallelForEach below. - // Note: The number of pass managers here needs to remain constant - // to prevent issues with pass instrumentations that rely on having the same - // pass manager for the main thread. - size_t numThreads = ctx->getNumThreads(); - if (opPipelines.size() < numThreads) { - // Reserve before resizing so that we can use a reference to the first - // element. - opPipelines.reserve(numThreads); - opPipelines.resize(numThreads, opPipelines.front()); - } - - // Ensure an analysis manager has been constructed for each of the nodes. - // This prevents thread races when running the nested pipelines. - for (CallGraphNode *node : nodesToVisit) - getAnalysisManager().nest(node->getCallableRegion()->getParentOp()); - - // An atomic failure variable for the async executors. - std::vector> activePMs(opPipelines.size()); - std::fill(activePMs.begin(), activePMs.end(), false); - return failableParallelForEach(ctx, nodesToVisit, [&](CallGraphNode *node) { - // Find a pass manager for this operation. - auto it = llvm::find_if(activePMs, [](std::atomic &isActive) { - bool expectedInactive = false; - return isActive.compare_exchange_strong(expectedInactive, true); - }); - assert(it != activePMs.end() && - "could not find inactive pass manager for thread"); - unsigned pmIndex = it - activePMs.begin(); - - // Optimize this callable node. - LogicalResult result = optimizeCallable(node, opPipelines[pmIndex]); - - // Reset the active bit for this pass manager. - activePMs[pmIndex].store(false); return result; - }); -} - -LogicalResult -InlinerPass::optimizeCallable(CallGraphNode *node, - llvm::StringMap &pipelines) { - Operation *callable = node->getCallableRegion()->getParentOp(); - StringRef opName = callable->getName().getStringRef(); - auto pipelineIt = pipelines.find(opName); - if (pipelineIt == pipelines.end()) { - // If a pipeline didn't exist, use the default if possible. - if (!defaultPipeline) - return success(); - - OpPassManager defaultPM(opName); - defaultPipeline(defaultPM); - pipelineIt = pipelines.try_emplace(opName, std::move(defaultPM)).first; - } - return runPipeline(pipelineIt->second, callable); -} - -LogicalResult InlinerPass::initializeOptions(StringRef options) { - if (failed(Pass::initializeOptions(options))) - return failure(); - - // Initialize the default pipeline builder to use the option string. - // TODO: Use a generic pass manager for default pipelines, and remove this. - if (!defaultPipelineStr.empty()) { - std::string defaultPipelineCopy = defaultPipelineStr; - defaultPipeline = [=](OpPassManager &pm) { - (void)parsePassPipeline(defaultPipelineCopy, pm); - }; - } else if (defaultPipelineStr.getNumOccurrences()) { - defaultPipeline = nullptr; - } - - // Initialize the op specific pass pipelines. - llvm::StringMap pipelines; - for (OpPassManager pipeline : opPipelineList) - if (!pipeline.empty()) - pipelines.try_emplace(pipeline.getOpAnchorName(), pipeline); - opPipelines.assign({std::move(pipelines)}); + // After inlining, make sure to erase any callables proven to be dead. + inlinerIface.eraseDeadCallables(); return success(); } - -std::unique_ptr mlir::createInlinerPass() { - return std::make_unique(); -} -std::unique_ptr -mlir::createInlinerPass(llvm::StringMap opPipelines) { - return std::make_unique(defaultInlinerOptPipeline, - std::move(opPipelines)); -} -std::unique_ptr mlir::createInlinerPass( - llvm::StringMap opPipelines, - std::function defaultPipelineBuilder) { - return std::make_unique(std::move(defaultPipelineBuilder), - std::move(opPipelines)); -} +} // namespace mlir -- GitLab From 8b5b294ec2cf876bc5eb5bd5fcb56ef487e36d60 Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Wed, 6 Mar 2024 09:22:52 -0800 Subject: [PATCH 335/929] [SCEV] Print predicate backedge count only if new information available When printing the result of SCEV's analysis, we can avoid printing the predicated backedge taken count and the predicates if the predicates are empty and no new information is provided. This helps to reduce the verbosity of the output. --- llvm/lib/Analysis/ScalarEvolution.cpp | 23 ++--- .../ScalarEvolution/2008-11-18-Stride1.ll | 2 - .../ScalarEvolution/2008-11-18-Stride2.ll | 2 - .../test/Analysis/ScalarEvolution/ZeroStep.ll | 1 - .../add-expr-pointer-operand-sorting.ll | 1 - .../Analysis/ScalarEvolution/add-like-or.ll | 2 - ...drec-computed-during-addrec-calculation.ll | 4 - .../ScalarEvolution/addrec-sub-nsw.ll | 4 - llvm/test/Analysis/ScalarEvolution/alloca.ll | 2 - .../backedge-taken-count-guard-info.ll | 4 - .../ScalarEvolution/becount-invalidation.ll | 2 - .../Analysis/ScalarEvolution/cycled_phis.ll | 5 - .../ScalarEvolution/decrementing_addrecs.ll | 4 - .../exact-exit-count-more-precise.ll | 5 - .../ScalarEvolution/exhaustive-trip-counts.ll | 2 - .../ScalarEvolution/exit-count-select-safe.ll | 79 ---------------- .../ScalarEvolution/exit-count-select.ll | 14 --- .../ScalarEvolution/exponential-behavior.ll | 1 - .../ScalarEvolution/finite-trip-count.ll | 22 ----- .../flags-from-poison-noautogen.ll | 4 - .../ScalarEvolution/flags-from-poison.ll | 77 --------------- llvm/test/Analysis/ScalarEvolution/fold.ll | 2 - .../ScalarEvolution/huge-trip-multiple.ll | 10 -- .../ScalarEvolution/incorrect-exit-count.ll | 3 - .../increasing-or-decreasing-iv.ll | 16 ---- .../Analysis/ScalarEvolution/invalidation.ll | 1 - .../Analysis/ScalarEvolution/iv-poison.ll | 12 --- .../load-with-range-metadata.ll | 4 - llvm/test/Analysis/ScalarEvolution/load.ll | 4 - .../ScalarEvolution/logical-operations.ll | 2 - ...en-count-guard-info-rewrite-expressions.ll | 36 ------- .../max-backedge-taken-count-guard-info.ll | 93 ------------------- ...-backedge-taken-count-limit-by-wrapping.ll | 8 -- .../max-be-count-not-constant.ll | 2 - .../ScalarEvolution/max-expr-cache.ll | 6 -- .../max-trip-count-address-space.ll | 4 - .../ScalarEvolution/max-trip-count.ll | 32 ------- .../Analysis/ScalarEvolution/min-max-exprs.ll | 2 - .../Analysis/ScalarEvolution/ne-overflow.ll | 18 ---- .../ScalarEvolution/no-wrap-add-exprs.ll | 2 - .../no-wrap-symbolic-becount.ll | 6 -- .../ScalarEvolution/nsw-offset-assume.ll | 2 - .../Analysis/ScalarEvolution/nsw-offset.ll | 2 - llvm/test/Analysis/ScalarEvolution/nsw.ll | 21 ----- .../Analysis/ScalarEvolution/outer_phi.ll | 4 - .../overflow-intrinsics-trip-count.ll | 26 ------ llvm/test/Analysis/ScalarEvolution/pr25369.ll | 9 -- llvm/test/Analysis/ScalarEvolution/pr34538.ll | 4 - llvm/test/Analysis/ScalarEvolution/pr48225.ll | 2 - .../pr58402-large-number-of-zext-exprs.ll | 1 - llvm/test/Analysis/ScalarEvolution/pr76234.ll | 1 - .../ptrtoint-constantexpr-loop.ll | 21 ----- .../test/Analysis/ScalarEvolution/ptrtoint.ll | 14 --- .../ScalarEvolution/range-signedness.ll | 3 - .../Analysis/ScalarEvolution/range_nw_flag.ll | 12 --- llvm/test/Analysis/ScalarEvolution/ranges.ll | 22 ----- llvm/test/Analysis/ScalarEvolution/sdiv.ll | 2 - .../ScalarEvolution/sext-add-inreg-loop.ll | 2 - .../test/Analysis/ScalarEvolution/sext-mul.ll | 4 - .../Analysis/ScalarEvolution/sext-to-zext.ll | 2 - .../test/Analysis/ScalarEvolution/shift-op.ll | 10 -- .../ScalarEvolution/shift-recurrences.ll | 44 --------- llvm/test/Analysis/ScalarEvolution/sle.ll | 2 - .../ScalarEvolution/smax-br-phi-idioms.ll | 4 - .../ScalarEvolution/smin-smax-folds.ll | 6 -- .../ScalarEvolution/solve-quadratic-i1.ll | 5 - .../solve-quadratic-overflow.ll | 2 - llvm/test/Analysis/ScalarEvolution/srem.ll | 2 - .../symbolic_max_exit_count.ll | 12 --- .../trip-count-andor-selectform.ll | 28 ------ .../ScalarEvolution/trip-count-andor.ll | 28 ------ .../trip-count-implied-addrec.ll | 18 ---- .../ScalarEvolution/trip-count-minmax.ll | 14 --- .../trip-count-negative-stride.ll | 51 ---------- .../trip-count-non-unit-stride.ll | 6 -- .../ScalarEvolution/trip-count-pow2.ll | 10 -- .../ScalarEvolution/trip-count-switch.ll | 2 - .../trip-count-unknown-stride.ll | 14 --- .../Analysis/ScalarEvolution/trip-count.ll | 8 -- .../Analysis/ScalarEvolution/trip-count10.ll | 10 -- .../Analysis/ScalarEvolution/trip-count12.ll | 2 - .../Analysis/ScalarEvolution/trip-count13.ll | 10 -- .../Analysis/ScalarEvolution/trip-count14.ll | 16 ---- .../Analysis/ScalarEvolution/trip-count15.ll | 7 -- .../Analysis/ScalarEvolution/trip-count2.ll | 2 - .../Analysis/ScalarEvolution/trip-count3.ll | 2 - .../Analysis/ScalarEvolution/trip-count4.ll | 2 - .../Analysis/ScalarEvolution/trip-count6.ll | 1 - .../Analysis/ScalarEvolution/trip-count7.ll | 1 - .../Analysis/ScalarEvolution/trip-count8.ll | 2 - .../Analysis/ScalarEvolution/trip-count9.ll | 45 --------- .../trip-multiple-guard-info.ll | 33 ------- .../Analysis/ScalarEvolution/trip-multiple.ll | 16 ---- .../ScalarEvolution/umin-umax-folds.ll | 18 ---- .../Analysis/ScalarEvolution/unknown_phis.ll | 2 - .../ScalarEvolution/widenable-condition.ll | 1 - .../Transforms/LoopSimplify/preserve-scev.ll | 9 -- 97 files changed, 12 insertions(+), 1110 deletions(-) diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 096091db8019..57e3619ed84e 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -13484,8 +13484,9 @@ static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE, if (ExitingBlocks.size() != 1) OS << " "; - if (SE->hasLoopInvariantBackedgeTakenCount(L)) - OS << "backedge-taken count is " << *SE->getBackedgeTakenCount(L) << "\n"; + auto *BTC = SE->getBackedgeTakenCount(L); + if (!isa(BTC)) + OS << "backedge-taken count is " << *BTC << "\n"; else OS << "Unpredictable backedge-taken count.\n"; @@ -13531,19 +13532,19 @@ static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE, << "\n"; } - OS << "Loop "; - L->getHeader()->printAsOperand(OS, /*PrintType=*/false); - OS << ": "; - SmallVector Preds; - auto PBT = SE->getPredicatedBackedgeTakenCount(L, Preds); - if (!isa(PBT)) { - OS << "Predicated backedge-taken count is " << *PBT << "\n"; + auto *PBT = SE->getPredicatedBackedgeTakenCount(L, Preds); + if (PBT != BTC || !Preds.empty()) { + OS << "Loop "; + L->getHeader()->printAsOperand(OS, /*PrintType=*/false); + OS << ": "; + if (!isa(PBT)) + OS << "Predicated backedge-taken count is " << *PBT << "\n"; + else + OS << "Unpredictable predicated backedge-taken count.\n"; OS << " Predicates:\n"; for (const auto *P : Preds) P->print(OS, 4); - } else { - OS << "Unpredictable predicated backedge-taken count.\n"; } if (SE->hasLoopInvariantBackedgeTakenCount(L)) { diff --git a/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride1.ll b/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride1.ll index 104c3177db1f..5a263f59e653 100644 --- a/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride1.ll +++ b/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride1.ll @@ -13,8 +13,6 @@ define i32 @f(i32 %x) nounwind readnone { ; CHECK-NEXT: Loop %bb: backedge-taken count is ((-5 + %x) /u 3) ; CHECK-NEXT: Loop %bb: constant max backedge-taken count is i32 1431655764 ; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is ((-5 + %x) /u 3) -; CHECK-NEXT: Loop %bb: Predicated backedge-taken count is ((-5 + %x) /u 3) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %bb: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride2.ll b/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride2.ll index 6f5e60d9bc2e..11953110417b 100644 --- a/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride2.ll +++ b/llvm/test/Analysis/ScalarEvolution/2008-11-18-Stride2.ll @@ -11,8 +11,6 @@ define i32 @f(i32 %x) nounwind readnone { ; CHECK-NEXT: Loop %bb: backedge-taken count is (((-3 + (-1 * (1 umin (-3 + (-1 * %x) + (1000 umax (3 + %x))))) + (-1 * %x) + (1000 umax (3 + %x))) /u 3) + (1 umin (-3 + (-1 * %x) + (1000 umax (3 + %x))))) ; CHECK-NEXT: Loop %bb: constant max backedge-taken count is i32 334 ; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is (((-3 + (-1 * (1 umin (-3 + (-1 * %x) + (1000 umax (3 + %x))))) + (-1 * %x) + (1000 umax (3 + %x))) /u 3) + (1 umin (-3 + (-1 * %x) + (1000 umax (3 + %x))))) -; CHECK-NEXT: Loop %bb: Predicated backedge-taken count is (((-3 + (-1 * (1 umin (-3 + (-1 * %x) + (1000 umax (3 + %x))))) + (-1 * %x) + (1000 umax (3 + %x))) /u 3) + (1 umin (-3 + (-1 * %x) + (1000 umax (3 + %x))))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %bb: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/ZeroStep.ll b/llvm/test/Analysis/ScalarEvolution/ZeroStep.ll index 51b8dd7e2b33..d008e4483f8f 100644 --- a/llvm/test/Analysis/ScalarEvolution/ZeroStep.ll +++ b/llvm/test/Analysis/ScalarEvolution/ZeroStep.ll @@ -16,7 +16,6 @@ define void @foo() { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop diff --git a/llvm/test/Analysis/ScalarEvolution/add-expr-pointer-operand-sorting.ll b/llvm/test/Analysis/ScalarEvolution/add-expr-pointer-operand-sorting.ll index 3cd464c01eb6..1799d15e67e7 100644 --- a/llvm/test/Analysis/ScalarEvolution/add-expr-pointer-operand-sorting.ll +++ b/llvm/test/Analysis/ScalarEvolution/add-expr-pointer-operand-sorting.ll @@ -49,7 +49,6 @@ define i32 @d(i32 %base) { ; CHECK-NEXT: Loop %for.cond: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.cond: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.cond: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.cond: Unpredictable predicated backedge-taken count. ; entry: %e = alloca [1 x [1 x i8]], align 1 diff --git a/llvm/test/Analysis/ScalarEvolution/add-like-or.ll b/llvm/test/Analysis/ScalarEvolution/add-like-or.ll index a525f9f17f07..71feb8a9702b 100644 --- a/llvm/test/Analysis/ScalarEvolution/add-like-or.ll +++ b/llvm/test/Analysis/ScalarEvolution/add-like-or.ll @@ -68,8 +68,6 @@ define void @mask-high(i64 %arg, ptr dereferenceable(4) %arg1) { ; CHECK-NEXT: Loop %bb6: backedge-taken count is (-1 + (-16 * (%arg /u 16)) + ((sext i32 %i to i64) smax (1 + (16 * (%arg /u 16))))) ; CHECK-NEXT: Loop %bb6: constant max backedge-taken count is i64 -9223372034707292162 ; CHECK-NEXT: Loop %bb6: symbolic max backedge-taken count is (-1 + (-16 * (%arg /u 16)) + ((sext i32 %i to i64) smax (1 + (16 * (%arg /u 16))))) -; CHECK-NEXT: Loop %bb6: Predicated backedge-taken count is (-1 + (-16 * (%arg /u 16)) + ((sext i32 %i to i64) smax (1 + (16 * (%arg /u 16))))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %bb6: Trip multiple is 1 ; bb: diff --git a/llvm/test/Analysis/ScalarEvolution/addrec-computed-during-addrec-calculation.ll b/llvm/test/Analysis/ScalarEvolution/addrec-computed-during-addrec-calculation.ll index ceb293dfa6c0..411affc344c5 100644 --- a/llvm/test/Analysis/ScalarEvolution/addrec-computed-during-addrec-calculation.ll +++ b/llvm/test/Analysis/ScalarEvolution/addrec-computed-during-addrec-calculation.ll @@ -29,17 +29,13 @@ define void @test(ptr %p) { ; CHECK-NEXT: Loop %loop2: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop2: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop2: symbolic max backedge-taken count is -1 -; CHECK-NEXT: Loop %loop2: Unpredictable predicated backedge-taken count. ; CHECK-NEXT: Loop %loop3: backedge-taken count is false ; CHECK-NEXT: Loop %loop3: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop3: symbolic max backedge-taken count is false -; CHECK-NEXT: Loop %loop3: Predicated backedge-taken count is false -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop3: Trip multiple is 1 ; CHECK-NEXT: Loop %loop.header: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop.header: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop.header: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop.header: Unpredictable predicated backedge-taken count. ; entry: br label %loop.header diff --git a/llvm/test/Analysis/ScalarEvolution/addrec-sub-nsw.ll b/llvm/test/Analysis/ScalarEvolution/addrec-sub-nsw.ll index a021b1790bdb..13ebb0b97c26 100644 --- a/llvm/test/Analysis/ScalarEvolution/addrec-sub-nsw.ll +++ b/llvm/test/Analysis/ScalarEvolution/addrec-sub-nsw.ll @@ -14,8 +14,6 @@ define i32 @test_1_non_negative(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (1 smax %n)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (1 smax %n)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (1 smax %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -53,8 +51,6 @@ define i32 @test_2_non_positive(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (1 smax %n)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (1 smax %n)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (1 smax %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/alloca.ll b/llvm/test/Analysis/ScalarEvolution/alloca.ll index cf299104198e..39d43e7aaa4f 100644 --- a/llvm/test/Analysis/ScalarEvolution/alloca.ll +++ b/llvm/test/Analysis/ScalarEvolution/alloca.ll @@ -46,8 +46,6 @@ define void @alloca_icmp_null_exit_count() { ; CHECK-NEXT: Loop %loop: backedge-taken count is 2 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 2 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 2 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 3 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info.ll index 2ba00196464f..b78438b9bd67 100644 --- a/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info.ll +++ b/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info.ll @@ -16,8 +16,6 @@ define void @loop_guard_improves_exact_backedge_taken_count_1(i32 %conv) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 0 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 0 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 0 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -52,8 +50,6 @@ define void @loop_guard_improves_exact_backedge_taken_count_2(i32 %conv) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (zext i1 (trunc i32 %conv to i1) to i64) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i1 (trunc i32 %conv to i1) to i64) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (zext i1 (trunc i32 %conv to i1) to i64) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 2 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/becount-invalidation.ll b/llvm/test/Analysis/ScalarEvolution/becount-invalidation.ll index a87f07716fa6..b1cd95f1990c 100644 --- a/llvm/test/Analysis/ScalarEvolution/becount-invalidation.ll +++ b/llvm/test/Analysis/ScalarEvolution/becount-invalidation.ll @@ -34,11 +34,9 @@ define void @test(ptr %arg) { ; CHECK-NEXT: Loop %loop2.header: symbolic max backedge-taken count is false ; CHECK-NEXT: symbolic max exit count for loop2.header: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for loop2.latch: false -; CHECK-NEXT: Loop %loop2.header: Unpredictable predicated backedge-taken count. ; CHECK-NEXT: Loop %loop.header: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop.header: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop.header: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop.header: Unpredictable predicated backedge-taken count. ; entry: br label %loop.header diff --git a/llvm/test/Analysis/ScalarEvolution/cycled_phis.ll b/llvm/test/Analysis/ScalarEvolution/cycled_phis.ll index a04a93bd005c..ec244595e8fe 100644 --- a/llvm/test/Analysis/ScalarEvolution/cycled_phis.ll +++ b/llvm/test/Analysis/ScalarEvolution/cycled_phis.ll @@ -17,7 +17,6 @@ define void @test_01() { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -54,11 +53,9 @@ define void @test_02(ptr %p, ptr %q) { ; CHECK-NEXT: Loop %inner_loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %inner_loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %inner_loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %inner_loop: Unpredictable predicated backedge-taken count. ; CHECK-NEXT: Loop %outer_loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %outer_loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %outer_loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %outer_loop: Unpredictable predicated backedge-taken count. ; entry: %start = load i32, ptr %p, !range !0 @@ -107,11 +104,9 @@ define void @test_03(ptr %p, ptr %q) { ; CHECK-NEXT: Loop %inner_loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %inner_loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %inner_loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %inner_loop: Unpredictable predicated backedge-taken count. ; CHECK-NEXT: Loop %outer_loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %outer_loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %outer_loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %outer_loop: Unpredictable predicated backedge-taken count. ; entry: %start_1 = load i32, ptr %p, !range !0 diff --git a/llvm/test/Analysis/ScalarEvolution/decrementing_addrecs.ll b/llvm/test/Analysis/ScalarEvolution/decrementing_addrecs.ll index c681df485e27..c3a1943afd6b 100644 --- a/llvm/test/Analysis/ScalarEvolution/decrementing_addrecs.ll +++ b/llvm/test/Analysis/ScalarEvolution/decrementing_addrecs.ll @@ -49,8 +49,6 @@ define i32 @test_step_1_flags(i32 %n) { ; DEFAULT-NEXT: Loop %loop: backedge-taken count is (-1 + %n) ; DEFAULT-NEXT: Loop %loop: constant max backedge-taken count is i32 2147483646 ; DEFAULT-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %n) -; DEFAULT-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %n) -; DEFAULT-NEXT: Predicates: ; DEFAULT-NEXT: Loop %loop: Trip multiple is 1 ; ; EXPENSIVE_SHARPENING-LABEL: 'test_step_1_flags' @@ -75,8 +73,6 @@ define i32 @test_step_1_flags(i32 %n) { ; EXPENSIVE_SHARPENING-NEXT: Loop %loop: backedge-taken count is (-1 + %n) ; EXPENSIVE_SHARPENING-NEXT: Loop %loop: constant max backedge-taken count is i32 2147483646 ; EXPENSIVE_SHARPENING-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %n) -; EXPENSIVE_SHARPENING-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %n) -; EXPENSIVE_SHARPENING-NEXT: Predicates: ; EXPENSIVE_SHARPENING-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/exact-exit-count-more-precise.ll b/llvm/test/Analysis/ScalarEvolution/exact-exit-count-more-precise.ll index 27d30c524f21..45beb4f21e9e 100644 --- a/llvm/test/Analysis/ScalarEvolution/exact-exit-count-more-precise.ll +++ b/llvm/test/Analysis/ScalarEvolution/exact-exit-count-more-precise.ll @@ -16,13 +16,10 @@ define void @test_and(i16 %in) { ; CHECK-NEXT: Loop %bb1.i: backedge-taken count is (1 + (-1 * %in)) ; CHECK-NEXT: Loop %bb1.i: constant max backedge-taken count is i16 -1 ; CHECK-NEXT: Loop %bb1.i: symbolic max backedge-taken count is (1 + (-1 * %in)) -; CHECK-NEXT: Loop %bb1.i: Predicated backedge-taken count is (1 + (-1 * %in)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %bb1.i: Trip multiple is 1 ; CHECK-NEXT: Loop %bb2: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %bb2: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %bb2: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %bb2: Unpredictable predicated backedge-taken count. ; br label %bb2 @@ -58,8 +55,6 @@ define void @test_or() { ; CHECK-NEXT: Loop %BB: backedge-taken count is undef ; CHECK-NEXT: Loop %BB: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %BB: symbolic max backedge-taken count is undef -; CHECK-NEXT: Loop %BB: Predicated backedge-taken count is undef -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %BB: Trip multiple is 1 ; %C10 = icmp slt i1 undef, undef diff --git a/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll b/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll index 37f99cf9f24f..03c46f4826c6 100644 --- a/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll +++ b/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll @@ -11,8 +11,6 @@ define void @f_0() { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 5 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 5 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 5 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 5 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 6 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/exit-count-select-safe.ll b/llvm/test/Analysis/ScalarEvolution/exit-count-select-safe.ll index 6bf01fcde5f2..5b32b3ad246a 100644 --- a/llvm/test/Analysis/ScalarEvolution/exit-count-select-safe.ll +++ b/llvm/test/Analysis/ScalarEvolution/exit-count-select-safe.ll @@ -14,8 +14,6 @@ define i32 @logical_and_2ops(i32 %n, i32 %m) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin_seq %m) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin_seq %m) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin_seq %m) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -44,8 +42,6 @@ define i32 @logical_or_2ops(i32 %n, i32 %m) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin_seq %m) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin_seq %m) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin_seq %m) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -76,8 +72,6 @@ define i32 @logical_and_3ops(i32 %n, i32 %m, i32 %k) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin_seq %m umin_seq %k) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin_seq %m umin_seq %k) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin_seq %m umin_seq %k) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -110,8 +104,6 @@ define i32 @logical_or_3ops(i32 %n, i32 %m, i32 %k) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin_seq %m umin_seq %k) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin_seq %m umin_seq %k) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin_seq %m umin_seq %k) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -146,8 +138,6 @@ define i32 @logical_or_3ops_duplicate(i32 %n, i32 %m, i32 %k) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin_seq %m umin_seq %k) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin_seq %m umin_seq %k) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin_seq %m umin_seq %k) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -184,8 +174,6 @@ define i32 @logical_or_3ops_redundant_uminseq_operand(i32 %n, i32 %m, i32 %k) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((%n umin %m) umin_seq %k) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((%n umin %m) umin_seq %k) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((%n umin %m) umin_seq %k) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -221,8 +209,6 @@ define i32 @logical_or_3ops_redundant_umin_operand(i32 %n, i32 %m, i32 %k) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin_seq %k umin_seq %m) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin_seq %k umin_seq %m) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin_seq %k umin_seq %m) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -260,8 +246,6 @@ define i32 @logical_or_4ops_redundant_operand_across_umins(i32 %n, i32 %m, i32 % ; CHECK-NEXT: Loop %loop: backedge-taken count is ((%n umin %m) umin_seq %k umin_seq %q) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((%n umin %m) umin_seq %k umin_seq %q) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((%n umin %m) umin_seq %k umin_seq %q) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -300,8 +284,6 @@ define i32 @logical_or_3ops_operand_wise_redundant_umin(i32 %n, i32 %m, i32 %k) ; CHECK-NEXT: Loop %loop: backedge-taken count is ((%n umin %m) umin_seq %k) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((%n umin %m) umin_seq %k) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((%n umin %m) umin_seq %k) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -338,8 +320,6 @@ define i32 @logical_or_3ops_partially_redundant_umin(i32 %n, i32 %m, i32 %k) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin_seq (%m umin %k)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin_seq (%m umin %k)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin_seq (%m umin %k)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -384,14 +364,10 @@ define i32 @logical_or_5ops_redundant_opearand_of_inner_uminseq(i32 %a, i32 %b, ; CHECK-NEXT: Loop %loop: backedge-taken count is (%a umin_seq %b umin_seq ((%e umin_seq %d) umin %c)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%a umin_seq %b umin_seq ((%e umin_seq %d) umin %c)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%a umin_seq %b umin_seq ((%e umin_seq %d) umin %c)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; CHECK-NEXT: Loop %first.loop: backedge-taken count is (%e umin_seq %d umin_seq %a) ; CHECK-NEXT: Loop %first.loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %first.loop: symbolic max backedge-taken count is (%e umin_seq %d umin_seq %a) -; CHECK-NEXT: Loop %first.loop: Predicated backedge-taken count is (%e umin_seq %d umin_seq %a) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %first.loop: Trip multiple is 1 ; entry: @@ -437,8 +413,6 @@ define i32 @logical_and_2ops_and_constant(i32 %n, i32 %m, i32 %k) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (42 umin %n) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 42 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (42 umin %n) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (42 umin %n) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -476,19 +450,14 @@ define i32 @computeSCEVAtScope(i32 %d.0) { ; CHECK-NEXT: Loop %for.cond: backedge-taken count is (-1 * %d.0) ; CHECK-NEXT: Loop %for.cond: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %for.cond: symbolic max backedge-taken count is (-1 * %d.0) -; CHECK-NEXT: Loop %for.cond: Predicated backedge-taken count is (-1 * %d.0) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.cond: Trip multiple is 1 ; CHECK-NEXT: Loop %for.cond4: backedge-taken count is 0 ; CHECK-NEXT: Loop %for.cond4: constant max backedge-taken count is i32 0 ; CHECK-NEXT: Loop %for.cond4: symbolic max backedge-taken count is 0 -; CHECK-NEXT: Loop %for.cond4: Predicated backedge-taken count is 0 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.cond4: Trip multiple is 1 ; CHECK-NEXT: Loop %while.cond: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %while.cond: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %while.cond: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %while.cond: Unpredictable predicated backedge-taken count. ; entry: br label %while.cond @@ -545,8 +514,6 @@ define i64 @uminseq_vs_ptrtoint_complexity(i64 %n, i64 %m, ptr %ptr) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin_seq %m) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin_seq %m) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin_seq %m) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -579,8 +546,6 @@ define i32 @logical_and_implies_poison1(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((1 + %n) umin %n) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((1 + %n) umin %n) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((1 + %n) umin %n) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -612,8 +577,6 @@ define i32 @logical_and_implies_poison2(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((1 + %n) umin %n) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((1 + %n) umin %n) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((1 + %n) umin %n) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -645,8 +608,6 @@ define i32 @logical_and_implies_poison3(i32 %n, i32 %m) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((%n + %m) umin %n) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((%n + %m) umin %n) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((%n + %m) umin %n) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -678,8 +639,6 @@ define i32 @logical_and_implies_poison_wrong_direction(i32 %n, i32 %m) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin_seq (%n + %m)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin_seq (%n + %m)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin_seq (%n + %m)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -709,8 +668,6 @@ define i32 @logical_and_implies_poison_noundef(i32 %n, i32 noundef %m) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin %m) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin %m) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin %m) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -739,8 +696,6 @@ define i32 @logical_and_implies_poison_noundef_wrong_direction(i32 %n, i32 nound ; CHECK-NEXT: Loop %loop: backedge-taken count is (%m umin_seq %n) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%m umin_seq %n) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%m umin_seq %n) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -773,8 +728,6 @@ define i32 @logical_and_implies_poison_complex1(i32 %n, i32 %m) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((%n + %m) umin (1 + %n + %m)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((%n + %m) umin (1 + %n + %m)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((%n + %m) umin (1 + %n + %m)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -809,8 +762,6 @@ define i32 @logical_and_implies_poison_complex2(i32 %n, i32 %m, i32 %l) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((%n + %m) umin (%n + %m + %l)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((%n + %m) umin (%n + %m + %l)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((%n + %m) umin (%n + %m + %l)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -845,8 +796,6 @@ define i32 @logical_and_implies_poison_complex_wrong_direction(i32 %n, i32 %m, i ; CHECK-NEXT: Loop %loop: backedge-taken count is ((%n + %m) umin_seq (%n + %m + %l)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((%n + %m) umin_seq (%n + %m + %l)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((%n + %m) umin_seq (%n + %m + %l)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -881,8 +830,6 @@ define i32 @logical_and_implies_multiple_ops(i32 %n, i32 %m) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (((1 + %n) umin %n) umin_seq %m) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (((1 + %n) umin %n) umin_seq %m) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (((1 + %n) umin %n) umin_seq %m) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -918,8 +865,6 @@ define i32 @logical_and_implies_multiple_ops2(i32 %n, i32 %m) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin_seq ((1 + %n) umin %m)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin_seq ((1 + %n) umin %m)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin_seq ((1 + %n) umin %m)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -955,8 +900,6 @@ define i32 @logical_and_implies_multiple_ops3(i32 %n, i32 %m) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (%m umin_seq ((1 + %n) umin %n)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%m umin_seq ((1 + %n) umin %n)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%m umin_seq ((1 + %n) umin %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -992,8 +935,6 @@ define i32 @logical_and_not_zero(i16 %n, i32 %m) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((1 + (zext i16 %n to i32)) umin %m) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 65536 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((1 + (zext i16 %n to i32)) umin %m) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((1 + (zext i16 %n to i32)) umin %m) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1028,8 +969,6 @@ define i32 @logical_and_not_zero_wrong_order(i16 %n, i32 %m) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (%m umin_seq (1 + (zext i16 %n to i32))) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 65536 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%m umin_seq (1 + (zext i16 %n to i32))) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%m umin_seq (1 + (zext i16 %n to i32))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1060,8 +999,6 @@ define i32 @logical_and_not_zero_needs_context(i32 %n, i32 %m) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (%n umin_seq %m) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%n umin_seq %m) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%n umin_seq %m) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1099,8 +1036,6 @@ define i32 @logical_and_known_smaller(i16 %n, i16 %m) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (zext i16 %n to i32) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 65535 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i16 %n to i32) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (zext i16 %n to i32) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1138,8 +1073,6 @@ define i32 @logical_and_known_smaller_equal(i16 %n, i16 %m) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (zext i16 %n to i32) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 65535 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i16 %n to i32) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (zext i16 %n to i32) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1177,8 +1110,6 @@ define i32 @logical_and_not_known_smaller_equal(i16 %n, i16 %m) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((zext i16 %n to i32) umin_seq (65534 + (zext i16 %m to i32))) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 65535 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((zext i16 %n to i32) umin_seq (65534 + (zext i16 %m to i32))) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((zext i16 %n to i32) umin_seq (65534 + (zext i16 %m to i32))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1216,8 +1147,6 @@ define i32 @logical_and_known_greater(i16 %n, i16 %m) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (zext i16 %n to i32) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 65535 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i16 %n to i32) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (zext i16 %n to i32) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1255,8 +1184,6 @@ define i32 @logical_and_known_greater_equal(i16 %n, i16 %m) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (zext i16 %n to i32) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 65535 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i16 %n to i32) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (zext i16 %n to i32) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1294,8 +1221,6 @@ define i32 @logical_and_not_known_greater_equal(i16 %n, i16 %m) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((zext i16 %n to i32) umin (65534 + (zext i16 %m to i32))) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 65535 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((zext i16 %n to i32) umin (65534 + (zext i16 %m to i32))) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((zext i16 %n to i32) umin (65534 + (zext i16 %m to i32))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1327,8 +1252,6 @@ define i32 @logical_and_zero_arg1(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 0 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 0 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 0 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1357,8 +1280,6 @@ define i32 @logical_and_zero_arg2(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 0 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 0 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 0 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/exit-count-select.ll b/llvm/test/Analysis/ScalarEvolution/exit-count-select.ll index 060fb7967120..d9ca46ba6787 100644 --- a/llvm/test/Analysis/ScalarEvolution/exit-count-select.ll +++ b/llvm/test/Analysis/ScalarEvolution/exit-count-select.ll @@ -16,8 +16,6 @@ define void @logical_and_m_const(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (2 umin %n) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (2 umin %n) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (2 umin %n) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -48,8 +46,6 @@ define void @logical_and_nonzero(i32 %m) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (2 umin %m) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (2 umin %m) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (2 umin %m) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -81,8 +77,6 @@ define void @logical_and_zero(i32 %m) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 0 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 0 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 0 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -115,7 +109,6 @@ define void @logical_and_inversed(i32 %n, i32 %m) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -145,8 +138,6 @@ define void @logical_or_m_const(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (2 umin %n) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (2 umin %n) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (2 umin %n) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -177,8 +168,6 @@ define void @logical_or_nonzero(i32 %m) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (2 umin %m) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (2 umin %m) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (2 umin %m) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -210,8 +199,6 @@ define void @logical_or_zero(i32 %m) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 0 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 0 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 0 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -244,7 +231,6 @@ define void @logical_or_inversed(i32 %n, i32 %m) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop diff --git a/llvm/test/Analysis/ScalarEvolution/exponential-behavior.ll b/llvm/test/Analysis/ScalarEvolution/exponential-behavior.ll index 1de7aa5e870e..0b1854ecbd2e 100644 --- a/llvm/test/Analysis/ScalarEvolution/exponential-behavior.ll +++ b/llvm/test/Analysis/ScalarEvolution/exponential-behavior.ll @@ -12,7 +12,6 @@ define void @f(i32 %n, ptr %ptr) { ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 ; CHECK-NEXT: symbolic max exit count for loop: 0 ; CHECK-NEXT: symbolic max exit count for be: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop diff --git a/llvm/test/Analysis/ScalarEvolution/finite-trip-count.ll b/llvm/test/Analysis/ScalarEvolution/finite-trip-count.ll index 339c3ffa224e..b3e3b208bf90 100644 --- a/llvm/test/Analysis/ScalarEvolution/finite-trip-count.ll +++ b/llvm/test/Analysis/ScalarEvolution/finite-trip-count.ll @@ -12,8 +12,6 @@ define void @sle_pre_inc(i32 %len) willreturn { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (0 smax (1 + %len)) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483647 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (0 smax (1 + %len)) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (0 smax (1 + %len)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -36,8 +34,6 @@ define void @sle_post_inc(i32 %len) willreturn { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (1 smax (1 + %len))) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (1 smax (1 + %len))) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (1 smax (1 + %len))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -60,7 +56,6 @@ define void @sle_pre_inc_infinite(i32 %len) { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -82,8 +77,6 @@ define void @ule_pre_inc(i32 %len) willreturn { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (1 + %len) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (1 + %len) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (1 + %len) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -106,8 +99,6 @@ define void @ule_post_inc(i32 %len) willreturn { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (1 umax (1 + %len))) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (1 umax (1 + %len))) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (1 umax (1 + %len))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -130,7 +121,6 @@ define void @ule_pre_inc_infinite(i32 %len) { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -152,8 +142,6 @@ define void @sge_pre_inc(i32 %end) willreturn { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (100 + (-1 * (100 smin (-1 + %end)))) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2147483548 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (100 + (-1 * (100 smin (-1 + %end)))) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (100 + (-1 * (100 smin (-1 + %end)))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -176,8 +164,6 @@ define void @sge_post_inc(i32 %end) willreturn { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (100 + (-1 * (100 smin (-1 + %end)))) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2147483548 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (100 + (-1 * (100 smin (-1 + %end)))) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (100 + (-1 * (100 smin (-1 + %end)))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -200,7 +186,6 @@ define void @sge_pre_inc_infinite(i32 %end) { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -222,8 +207,6 @@ define void @use_pre_inc(i32 %end) willreturn { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (100 + (-1 * (100 umin (-1 + %end)))) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 100 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (100 + (-1 * (100 umin (-1 + %end)))) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (100 + (-1 * (100 umin (-1 + %end)))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -246,8 +229,6 @@ define void @use_post_inc(i32 %end) willreturn { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (99 + (-1 * (99 umin (-1 + %end)))) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 99 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (99 + (-1 * (99 umin (-1 + %end)))) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (99 + (-1 * (99 umin (-1 + %end)))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -270,7 +251,6 @@ define void @uge_pre_inc_infinite(i32 %end) { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -292,8 +272,6 @@ define void @pr54191(i64 %n) mustprogress { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-2 + %n) /u 2) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4611686018427387902 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-2 + %n) /u 2) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-2 + %n) /u 2) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/flags-from-poison-noautogen.ll b/llvm/test/Analysis/ScalarEvolution/flags-from-poison-noautogen.ll index eed81eec97ce..37ea771faf14 100644 --- a/llvm/test/Analysis/ScalarEvolution/flags-from-poison-noautogen.ll +++ b/llvm/test/Analysis/ScalarEvolution/flags-from-poison-noautogen.ll @@ -26,14 +26,10 @@ define void @subrecurrences(i32 %outer_l, i32 %inner_l, i32 %val) { ; CHECK-NEXT: Loop %inner: backedge-taken count is %inner_l ; CHECK-NEXT: Loop %inner: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %inner: symbolic max backedge-taken count is %inner_l -; CHECK-NEXT: Loop %inner: Predicated backedge-taken count is %inner_l -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %inner: Trip multiple is 1 ; CHECK-NEXT: Loop %outer: backedge-taken count is %outer_l ; CHECK-NEXT: Loop %outer: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %outer: symbolic max backedge-taken count is %outer_l -; CHECK-NEXT: Loop %outer: Predicated backedge-taken count is %outer_l -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %outer: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/flags-from-poison.ll b/llvm/test/Analysis/ScalarEvolution/flags-from-poison.ll index 16ffb54b01f6..a8531a8f5779 100644 --- a/llvm/test/Analysis/ScalarEvolution/flags-from-poison.ll +++ b/llvm/test/Analysis/ScalarEvolution/flags-from-poison.ll @@ -32,8 +32,6 @@ define void @test-add-nsw(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -71,8 +69,6 @@ define void @test-add-nuw(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -111,8 +107,6 @@ define void @test-add-scope-invariant(ptr %input, i32 %needle) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %offset) + %needle) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %offset) + %needle) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %offset) + %needle) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -152,7 +146,6 @@ define void @test-add-scope-bound(ptr %input, i32 %needle) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -186,7 +179,6 @@ define void @test-add-scope-bound-unkn-preheader(ptr %input, i32 %needle) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: %offset = load i32, ptr %input @@ -218,7 +210,6 @@ define void @test-add-scope-bound-unkn-preheader-neg1(ptr %input, i32 %needle) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: %offset = load i32, ptr %input @@ -251,7 +242,6 @@ define void @test-add-scope-bound-unkn-preheader-neg2(ptr %input, i32 %needle) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: %offset = load i32, ptr %input @@ -287,7 +277,6 @@ define void @test-add-scope-bound-unkn-header(ptr %input, i32 %needle) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -322,7 +311,6 @@ define void @test-add-scope-bound-unkn-header2(ptr %input, i32 %needle) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -358,7 +346,6 @@ define void @test-add-scope-bound-unkn-header-neg(ptr %input, i32 %needle) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -394,8 +381,6 @@ define void @test-add-nuw-from-icmp(ptr %input, i32 %offset, ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; i32 %numIterations) { @@ -434,8 +419,6 @@ define void @test-add-no-load(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -472,8 +455,6 @@ define void @test-add-not-header(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -511,8 +492,6 @@ define void @test-add-not-header2(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -556,7 +535,6 @@ define void @test-add-not-header3(ptr %input, i32 %offset, i32 %numIterations, ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: symbolic max exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for loop2: (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; ptr %cond_buf) { entry: @@ -595,8 +573,6 @@ define void @test-add-not-header4(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -637,7 +613,6 @@ define void @test-add-not-header5(ptr %input, i32 %offset) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -672,8 +647,6 @@ define void @test-add-not-header6(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -711,8 +684,6 @@ define void @test-add-not-header7(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -752,8 +723,6 @@ define void @test-add-not-header8(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -793,8 +762,6 @@ define void @test-add-call(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -831,8 +798,6 @@ define void @test-add-call2(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -868,8 +833,6 @@ define void @test-gep-propagates-poison(ptr %input, i32 %offset, i32 %numIterati ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -907,8 +870,6 @@ define void @test-add-mul-propagates(ptr %input, i32 %offset, i32 %numIterations ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -946,8 +907,6 @@ define void @test-mul-propagates-poison(ptr %input, i32 %offset, i32 %numIterati ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -984,8 +943,6 @@ define void @test-mul-propagates-poison-2(ptr %input, i32 %offset, i32 %numItera ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1021,8 +978,6 @@ define void @test-add-div(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1056,8 +1011,6 @@ define void @test-add-div2(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1091,8 +1044,6 @@ define void @test-add-store(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1134,8 +1085,6 @@ define void @test-add-twice(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1178,8 +1127,6 @@ define void @test-mul-nsw(ptr %input, i32 %stride, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1216,8 +1163,6 @@ define void @test-mul-nuw(ptr %input, i32 %stride, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1256,8 +1201,6 @@ define void @test-shl-nsw(ptr %input, i32 %start, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %start) + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %start) + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %start) + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1296,8 +1239,6 @@ define void @test-shl-nuw-edgecase(ptr %input, i32 %start, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %start) + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %start) + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %start) + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1336,8 +1277,6 @@ define void @test-shl-nuw-nsw(ptr %input, i32 %start, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %start) + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %start) + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %start) + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1376,8 +1315,6 @@ define void @test-shl-no-nsw(ptr %input, i32 %start, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %start) + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %start) + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %start) + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1416,8 +1353,6 @@ define void @test-shl-nsw-edgecase(ptr %input, i32 %start, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %start) + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %start) + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %start) + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1454,8 +1389,6 @@ define void @test-shl-nuw(ptr %input, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1495,8 +1428,6 @@ define void @test-sub-no-nsw(ptr %input, i32 %start, i32 %sub, i32 %numIteration ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %start) + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %start) + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %start) + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1537,8 +1468,6 @@ define void @test-sub-nsw(ptr %input, i32 %start, i32 %sub, i32 %numIterations) ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %start) + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %start) + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %start) + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1578,8 +1507,6 @@ define void @test-sub-nsw-lhs-non-negative(ptr %input, i32 %sub, i32 %numIterati ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1625,7 +1552,6 @@ define void @test-sext-sub(ptr %input, i32 %sub, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: symbolic max exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for cont: (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -1676,8 +1602,6 @@ define void @test-sub-with-add(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1744,7 +1668,6 @@ define i32 @pr28932() { ; CHECK-NEXT: Loop %for.cond: Unpredictable symbolic max backedge-taken count. ; CHECK-NEXT: symbolic max exit count for if.then: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for if.else: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %for.cond: Unpredictable predicated backedge-taken count. ; entry: %pre = load ptr, ptr @a, align 8 diff --git a/llvm/test/Analysis/ScalarEvolution/fold.ll b/llvm/test/Analysis/ScalarEvolution/fold.ll index 485458b8ec84..3b6df8e6d0a5 100644 --- a/llvm/test/Analysis/ScalarEvolution/fold.ll +++ b/llvm/test/Analysis/ScalarEvolution/fold.ll @@ -80,8 +80,6 @@ define void @test4(i32 %x, i32 %y) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 20 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 20 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 20 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 20 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 21 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/huge-trip-multiple.ll b/llvm/test/Analysis/ScalarEvolution/huge-trip-multiple.ll index 463f54fb549f..2825dc4d1215 100644 --- a/llvm/test/Analysis/ScalarEvolution/huge-trip-multiple.ll +++ b/llvm/test/Analysis/ScalarEvolution/huge-trip-multiple.ll @@ -17,8 +17,6 @@ define void @trip_count_4294967295() { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 4294967294 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 4294967294 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 4294967294 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 4294967294 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 4294967295 ; entry: @@ -46,8 +44,6 @@ define void @trip_count_4294967296() { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 4294967295 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 4294967295 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 4294967295 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 4294967295 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 2147483648 ; entry: @@ -75,8 +71,6 @@ define void @trip_count_8589935692() { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 8589934591 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 8589934591 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 8589934591 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 8589934591 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 2147483648 ; entry: @@ -104,8 +98,6 @@ define void @trip_count_9223372036854775808() { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 9223372036854775807 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 9223372036854775807 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 9223372036854775807 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 9223372036854775807 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 2147483648 ; entry: @@ -133,8 +125,6 @@ define void @trip_count_18446744073709551615() { ; CHECK-NEXT: Loop %for.body: backedge-taken count is -2 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is -2 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is -2 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/incorrect-exit-count.ll b/llvm/test/Analysis/ScalarEvolution/incorrect-exit-count.ll index cb8429d9ecd9..f7cec4996e8f 100644 --- a/llvm/test/Analysis/ScalarEvolution/incorrect-exit-count.ll +++ b/llvm/test/Analysis/ScalarEvolution/incorrect-exit-count.ll @@ -64,7 +64,6 @@ define dso_local i32 @f() { ; CHECK-NEXT: Loop %for.cond6: symbolic max backedge-taken count is 0 ; CHECK-NEXT: symbolic max exit count for for.cond6: 0 ; CHECK-NEXT: symbolic max exit count for for.end: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %for.cond6: Unpredictable predicated backedge-taken count. ; CHECK-NEXT: Loop %inner.loop: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for inner.loop: 0 ; CHECK-NEXT: exit count for for.end.3: ***COULDNOTCOMPUTE*** @@ -72,7 +71,6 @@ define dso_local i32 @f() { ; CHECK-NEXT: Loop %inner.loop: symbolic max backedge-taken count is 0 ; CHECK-NEXT: symbolic max exit count for inner.loop: 0 ; CHECK-NEXT: symbolic max exit count for for.end.3: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %inner.loop: Unpredictable predicated backedge-taken count. ; CHECK-NEXT: Loop %outer.loop: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for for.cond6: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for inner.loop: ***COULDNOTCOMPUTE*** @@ -82,7 +80,6 @@ define dso_local i32 @f() { ; CHECK-NEXT: symbolic max exit count for for.cond6: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for inner.loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for for.inc13.3: 2 -; CHECK-NEXT: Loop %outer.loop: Unpredictable predicated backedge-taken count. ; entry: store i32 3, ptr @a, align 4 diff --git a/llvm/test/Analysis/ScalarEvolution/increasing-or-decreasing-iv.ll b/llvm/test/Analysis/ScalarEvolution/increasing-or-decreasing-iv.ll index 9ec4f5eb6688..69bede4434b5 100644 --- a/llvm/test/Analysis/ScalarEvolution/increasing-or-decreasing-iv.ll +++ b/llvm/test/Analysis/ScalarEvolution/increasing-or-decreasing-iv.ll @@ -20,8 +20,6 @@ define void @f0(i1 %c) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 127 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 127 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 127 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 127 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 128 ; entry: @@ -88,8 +86,6 @@ define void @f1(i1 %c) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 15 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 15 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 15 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 15 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 16 ; entry: @@ -149,8 +145,6 @@ define void @f2(i1 %c) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 127 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 127 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 127 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 127 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 128 ; entry: @@ -192,8 +186,6 @@ define void @f3(i1 %c) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 127 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i16 127 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 127 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 127 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 128 ; entry: @@ -242,8 +234,6 @@ define void @f4(i1 %c) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 127 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 127 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 127 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 127 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 128 ; @@ -286,8 +276,6 @@ define void @f5(i1 %c) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 127 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i16 127 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 127 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 127 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 128 ; entry: @@ -332,8 +320,6 @@ define void @f6(i1 %c) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 127 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i16 127 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 127 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 127 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 128 ; entry: @@ -381,8 +367,6 @@ define void @f7(i1 %c) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 127 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i16 127 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 127 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 127 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 128 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/invalidation.ll b/llvm/test/Analysis/ScalarEvolution/invalidation.ll index f5d9c2b083d5..959720d2d6e4 100644 --- a/llvm/test/Analysis/ScalarEvolution/invalidation.ll +++ b/llvm/test/Analysis/ScalarEvolution/invalidation.ll @@ -41,7 +41,6 @@ define void @test(i32 %n) { ; CHECK-LABEL: Classifying expressions for: @test ; CHECK: Loop %loop: backedge-taken count is 14 ; CHECK: Loop %loop: constant max backedge-taken count is i32 14 -; CHECK: Loop %loop: Predicated backedge-taken count is 14 entry: br label %loop diff --git a/llvm/test/Analysis/ScalarEvolution/iv-poison.ll b/llvm/test/Analysis/ScalarEvolution/iv-poison.ll index 771a93f79cb1..6f0ff8ba997b 100644 --- a/llvm/test/Analysis/ScalarEvolution/iv-poison.ll +++ b/llvm/test/Analysis/ScalarEvolution/iv-poison.ll @@ -17,7 +17,6 @@ define i2 @iv_nsw_poison(i2 %arg) { ; CHECK-NEXT: Loop %bb1: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %bb1: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %bb1: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %bb1: Unpredictable predicated backedge-taken count. ; bb: br label %bb1 @@ -49,7 +48,6 @@ define i4 @iv_nsw_poison2(i4 %0, i4 %end, i4 %start) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -81,7 +79,6 @@ define i2 @iv_both_adds_nsw(i2 %arg) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; bb: br label %loop @@ -113,7 +110,6 @@ define i4 @iv_both_adds_nsw_extra_use(i4 %arg) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; bb: br label %loop @@ -147,7 +143,6 @@ define i4 @iv_both_adds_nsw_extra_use_incs_reordered(i4 %arg) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; bb: br label %loop @@ -181,7 +176,6 @@ define i4 @iv_nsw_poison_extra_use(i4 %0, i4 %end, i4 %start) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -216,7 +210,6 @@ define i2 @iv_nuw_poison(i2 %arg, i2 %start) { ; CHECK-NEXT: Loop %bb1: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %bb1: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %bb1: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %bb1: Unpredictable predicated backedge-taken count. ; bb: br label %bb1 @@ -248,7 +241,6 @@ define i4 @iv_nuw_poison2(i4 %0, i4 %end, i4 %start) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -280,7 +272,6 @@ define i2 @iv_both_adds_nuw(i2 %arg, i2 %start) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; bb: br label %loop @@ -312,7 +303,6 @@ define i4 @iv_both_adds_nuw_extra_use(i4 %arg, i4 %start) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; bb: br label %loop @@ -346,7 +336,6 @@ define i4 @iv_both_adds_nuw_extra_use_incs_reordered(i4 %arg, i4 %start) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; bb: br label %loop @@ -380,7 +369,6 @@ define i4 @iv_nuw_poison_extra_use(i4 %0, i4 %end, i4 %start) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop diff --git a/llvm/test/Analysis/ScalarEvolution/load-with-range-metadata.ll b/llvm/test/Analysis/ScalarEvolution/load-with-range-metadata.ll index 6d8416cc3e89..9e7eabba4d55 100644 --- a/llvm/test/Analysis/ScalarEvolution/load-with-range-metadata.ll +++ b/llvm/test/Analysis/ScalarEvolution/load-with-range-metadata.ll @@ -8,8 +8,6 @@ define i32 @slt_trip_count_with_range(ptr %ptr0, ptr %ptr1) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %limit) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 98 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %limit) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %limit) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -33,8 +31,6 @@ define i32 @ult_trip_count_with_range(ptr %ptr0, ptr %ptr1) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %limit) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 98 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %limit) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %limit) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/load.ll b/llvm/test/Analysis/ScalarEvolution/load.ll index f31dea4db446..6bcb09b3bb61 100644 --- a/llvm/test/Analysis/ScalarEvolution/load.ll +++ b/llvm/test/Analysis/ScalarEvolution/load.ll @@ -33,8 +33,6 @@ define i32 @test1() nounwind readnone { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 49 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 49 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 49 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 49 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 50 ; entry: @@ -87,8 +85,6 @@ define i32 @test2() nounwind uwtable readonly { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 4 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 4 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 4 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 4 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 5 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/logical-operations.ll b/llvm/test/Analysis/ScalarEvolution/logical-operations.ll index 7f4e6a0bd95f..b8ed560a49de 100644 --- a/llvm/test/Analysis/ScalarEvolution/logical-operations.ll +++ b/llvm/test/Analysis/ScalarEvolution/logical-operations.ll @@ -434,8 +434,6 @@ define ptr @tautological_select_like_phi(i32 %tc) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 100 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 100 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 100 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 100 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 101 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-rewrite-expressions.ll b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-rewrite-expressions.ll index bc01ade3a219..81b08694bd79 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-rewrite-expressions.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-rewrite-expressions.ll @@ -19,8 +19,6 @@ define void @rewrite_zext(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -63,8 +61,6 @@ define i32 @rewrite_zext_min_max(i32 %N, ptr %arr) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -111,8 +107,6 @@ define i32 @rewrite_min_max_zext(i32 %N, ptr %arr) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -159,8 +153,6 @@ define i32 @rewrite_sext_min_max(i32 %N, ptr %arr) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -207,8 +199,6 @@ define i32 @rewrite_min_max_sext(i32 %N, ptr %arr) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -257,8 +247,6 @@ define i32 @rewrite_zext_with_info_from_icmp_ne(i32 %N) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 0 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 0 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 0 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -307,8 +295,6 @@ define i32 @rewrite_zext_no_icmp_ne(i32 %N) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * ((4 + (zext i32 (-1 + (zext i2 (trunc i32 %N to i2) to i32)) to i64)) /u 4))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 1073741823 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * ((4 + (zext i32 (-1 + (zext i2 (trunc i32 %N to i2) to i32)) to i64)) /u 4))) /u 4) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * ((4 + (zext i32 (-1 + (zext i2 (trunc i32 %N to i2) to i32)) to i64)) /u 4))) /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -349,8 +335,6 @@ define void @rewrite_zext_and_base_1(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -394,8 +378,6 @@ define void @rewrite_zext_and_base_2(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -437,8 +419,6 @@ define void @guard_pessimizes_analysis_step2(i1 %c, i32 %N) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((14 + (-1 * %init)) /u 2) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 6 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((14 + (-1 * %init)) /u 2) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((14 + (-1 * %init)) /u 2) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -485,8 +465,6 @@ define i32 @rewrite_sext_slt_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -531,8 +509,6 @@ define i32 @rewrite_zext_ult_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -577,8 +553,6 @@ define i32 @rewrite_zext_ule_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -623,8 +597,6 @@ define i32 @rewrite_zext_sle_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -669,8 +641,6 @@ define i32 @rewrite_zext_uge_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -715,8 +685,6 @@ define i32 @rewrite_sext_sge_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -761,8 +729,6 @@ define i32 @rewrite_zext_ugt_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -807,8 +773,6 @@ define i32 @rewrite_sext_sgt_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll index 2f293147f0cc..845494460b23 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll @@ -16,8 +16,6 @@ define void @test_guard_less_than_16(ptr nocapture %a, i64 %i) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (15 + (-1 * %i)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 15 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (15 + (-1 * %i)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (15 + (-1 * %i)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -49,8 +47,6 @@ define void @test_guard_less_than_16_operands_swapped(ptr nocapture %a, i64 %i) ; CHECK-NEXT: Loop %loop: backedge-taken count is (15 + (-1 * %i)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 15 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (15 + (-1 * %i)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (15 + (-1 * %i)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -82,8 +78,6 @@ define void @test_guard_less_than_16_branches_flipped(ptr nocapture %a, i64 %i) ; CHECK-NEXT: Loop %loop: backedge-taken count is (15 + (-1 * %i)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (15 + (-1 * %i)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (15 + (-1 * %i)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -115,8 +109,6 @@ define void @test_guard_uge_16_branches_flipped(ptr nocapture %a, i64 %i) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (15 + (-1 * %i)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 15 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (15 + (-1 * %i)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (15 + (-1 * %i)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -148,8 +140,6 @@ define void @test_guard_eq_12(ptr nocapture %a, i64 %N) { ; CHECK-NEXT: Loop %loop: backedge-taken count is %N ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 12 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %N -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %N -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 13 ; entry: @@ -181,8 +171,6 @@ define void @test_guard_ule_12(ptr nocapture %a, i64 %N) { ; CHECK-NEXT: Loop %loop: backedge-taken count is %N ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 12 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %N -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %N -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -214,8 +202,6 @@ define void @test_guard_ule_12_step2(ptr nocapture %a, i64 %N) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (%N /u 2) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 6 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%N /u 2) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%N /u 2) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -247,8 +233,6 @@ define void @test_multiple_const_guards_order1(ptr nocapture %a, i64 %i) { ; CHECK-NEXT: Loop %loop: backedge-taken count is %i ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 9 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %i -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %i -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -284,8 +268,6 @@ define void @test_multiple_const_guards_order2(ptr nocapture %a, i64 %i) { ; CHECK-NEXT: Loop %loop: backedge-taken count is %i ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 9 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %i -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %i -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -321,8 +303,6 @@ define void @test_multiple_var_guards_order1(ptr nocapture %a, i64 %i, i64 %N) { ; CHECK-NEXT: Loop %loop: backedge-taken count is %i ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 10 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %i -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %i -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -358,8 +338,6 @@ define void @test_multiple_var_guards_order2(ptr nocapture %a, i64 %i, i64 %N) { ; CHECK-NEXT: Loop %loop: backedge-taken count is %i ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 10 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %i -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %i -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -396,8 +374,6 @@ define void @test_multiple_var_guards_cycle(ptr nocapture %a, i64 %i, i64 %N) { ; CHECK-NEXT: Loop %loop: backedge-taken count is %N ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %N -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %N -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -433,8 +409,6 @@ define void @test_guard_ult_ne(ptr nocapture readonly %data, i64 %count) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -470,8 +444,6 @@ define void @test_guard_ne_ult(ptr nocapture readonly %data, i64 %count) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -509,8 +481,6 @@ define void @test_guard_if_and_enter(ptr nocapture readonly %data, i64 %count) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -546,8 +516,6 @@ define void @test_guard_if_and_skip(ptr nocapture readonly %data, i64 %count) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -585,8 +553,6 @@ define void @test_guard_if_and_and(ptr nocapture readonly %data, i64 %count, i1 ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -625,8 +591,6 @@ define void @test_guard_if_and_or(ptr nocapture readonly %data, i64 %count, i1 % ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -663,8 +627,6 @@ define void @test_guard_if_or_skip(ptr nocapture readonly %data, i64 %count) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -700,8 +662,6 @@ define void @test_guard_if_or_enter(ptr nocapture readonly %data, i64 %count) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -739,8 +699,6 @@ define void @test_guard_if_or_or(ptr nocapture readonly %data, i64 %count, i1 %c ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -779,8 +737,6 @@ define void @test_guard_if_or_and(ptr nocapture readonly %data, i64 %count, i1 % ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -818,8 +774,6 @@ define void @test_guard_and_assume(ptr nocapture readonly %data, i64 %count) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -855,8 +809,6 @@ define void @test_guard_assume_and(ptr nocapture readonly %data, i64 %count) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %count) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %count) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %count) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -894,8 +846,6 @@ define void @guard_pessimizes_analysis_step1(i1 %c, i32 %N) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (9 + (-1 * %init)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 7 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (9 + (-1 * %init)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (9 + (-1 * %init)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -935,8 +885,6 @@ define void @guard_pessimizes_analysis_step2(i1 %c, i32 %N) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((8 + (-1 * %init)) /u 2) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 3 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((8 + (-1 * %init)) /u 2) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((8 + (-1 * %init)) /u 2) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -977,17 +925,13 @@ define void @crash(ptr %ptr) { ; CHECK-NEXT: Loop %while.body125: backedge-taken count is {(-2 + (-1 * (ptrtoint ptr %ptr to i64))),+,-1}<%while.cond111> ; CHECK-NEXT: Loop %while.body125: constant max backedge-taken count is i64 -2 ; CHECK-NEXT: Loop %while.body125: symbolic max backedge-taken count is {(-2 + (-1 * (ptrtoint ptr %ptr to i64))),+,-1}<%while.cond111> -; CHECK-NEXT: Loop %while.body125: Predicated backedge-taken count is {(-2 + (-1 * (ptrtoint ptr %ptr to i64))),+,-1}<%while.cond111> -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %while.body125: Trip multiple is 1 ; CHECK-NEXT: Loop %while.cond111: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %while.cond111: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %while.cond111: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %while.cond111: Unpredictable predicated backedge-taken count. ; CHECK-NEXT: Loop %while.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %while.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %while.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %while.body: Unpredictable predicated backedge-taken count. ; entry: br label %while.body @@ -1031,8 +975,6 @@ define void @test_guard_uge(i32 %blockSize) { ; CHECK-NEXT: Loop %while.body: backedge-taken count is (-1 + (%blockSize /u 4)) ; CHECK-NEXT: Loop %while.body: constant max backedge-taken count is i32 1073741822 ; CHECK-NEXT: Loop %while.body: symbolic max backedge-taken count is (-1 + (%blockSize /u 4)) -; CHECK-NEXT: Loop %while.body: Predicated backedge-taken count is (-1 + (%blockSize /u 4)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %while.body: Trip multiple is 1 ; %shr = lshr i32 %blockSize, 2 @@ -1068,8 +1010,6 @@ define void @test_guard_ugt(i32 %blockSize) { ; CHECK-NEXT: Loop %while.body: backedge-taken count is (-1 + (%blockSize /u 4)) ; CHECK-NEXT: Loop %while.body: constant max backedge-taken count is i32 1073741822 ; CHECK-NEXT: Loop %while.body: symbolic max backedge-taken count is (-1 + (%blockSize /u 4)) -; CHECK-NEXT: Loop %while.body: Predicated backedge-taken count is (-1 + (%blockSize /u 4)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %while.body: Trip multiple is 1 ; %shr = lshr i32 %blockSize, 2 @@ -1105,8 +1045,6 @@ define void @test_guard_uge_and_ule(i32 %blockSize) { ; CHECK-NEXT: Loop %while.body: backedge-taken count is (-1 + (%blockSize /u 4)) ; CHECK-NEXT: Loop %while.body: constant max backedge-taken count is i32 255 ; CHECK-NEXT: Loop %while.body: symbolic max backedge-taken count is (-1 + (%blockSize /u 4)) -; CHECK-NEXT: Loop %while.body: Predicated backedge-taken count is (-1 + (%blockSize /u 4)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %while.body: Trip multiple is 1 ; %shr = lshr i32 %blockSize, 2 @@ -1146,8 +1084,6 @@ define void @test_guard_ugt_and_ult(i32 %blockSize) { ; CHECK-NEXT: Loop %while.body: backedge-taken count is (-1 + (%blockSize /u 4)) ; CHECK-NEXT: Loop %while.body: constant max backedge-taken count is i32 255 ; CHECK-NEXT: Loop %while.body: symbolic max backedge-taken count is (-1 + (%blockSize /u 4)) -; CHECK-NEXT: Loop %while.body: Predicated backedge-taken count is (-1 + (%blockSize /u 4)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %while.body: Trip multiple is 1 ; %shr = lshr i32 %blockSize, 2 @@ -1189,8 +1125,6 @@ define void @test_guard_slt_sgt_1(ptr nocapture %a, i64 %N) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %N) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 10 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %N) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %N) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1226,8 +1160,6 @@ define void @test_guard_slt_sgt_2(ptr nocapture %a, i64 %i) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (17 + (-1 * %i)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 12 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (17 + (-1 * %i)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (17 + (-1 * %i)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1263,8 +1195,6 @@ define void @test_guard_sle_sge_1(ptr nocapture %a, i64 %N) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %N) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 11 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %N) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %N) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1300,8 +1230,6 @@ define void @test_guard_sle_sge_2(ptr nocapture %a, i64 %i) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (17 + (-1 * %i)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 13 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (17 + (-1 * %i)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (17 + (-1 * %i)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1340,8 +1268,6 @@ define void @optimized_range_check_unsigned(ptr %pred, i32 %N) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %N) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 6 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %N) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %N) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1377,8 +1303,6 @@ define void @optimized_range_check_unsigned_icmp_ops_swapped(ptr %pred, i32 %N) ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %N) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 6 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %N) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %N) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1416,8 +1340,6 @@ define void @optimized_range_check_unsigned2(ptr %pred, i32 %N) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %N) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 20 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %N) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %N) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1456,8 +1378,6 @@ define void @optimized_range_check_unsigned3(ptr %pred, i1 %c) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %N) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %N) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %N) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1495,8 +1415,6 @@ define void @not_optimized_range_check_unsigned1(ptr %pred, i32 %N) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %N) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %N) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %N) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1533,8 +1451,6 @@ define void @not_optimized_range_check_unsigned2(ptr %pred, i32 %N) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %N) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %N) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %N) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1565,8 +1481,6 @@ define i32 @sle_sgt_ult_umax_to_smax(i32 %num) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + %num) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 6 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + %num) /u 4) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + %num) /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; guard.1: @@ -1603,8 +1517,6 @@ define i32 @ult_sle_sgt_umax_to_smax(i32 %num) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + %num) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 1073741823 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + %num) /u 4) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + %num) /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; guard.1: @@ -1640,8 +1552,6 @@ define i32 @ptr_induction_ult_1(ptr %a, ptr %b) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 0 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 0 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 0 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1670,7 +1580,6 @@ define i32 @ptr_induction_ult_2(ptr %a, ptr %b) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: %cmp.6 = icmp ult ptr %a, %b @@ -1701,8 +1610,6 @@ define void @gep_addrec_nw(ptr %a) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 378 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 378 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 378 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 378 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 379 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-limit-by-wrapping.ll b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-limit-by-wrapping.ll index 45e6337a9b35..72c9457b773a 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-limit-by-wrapping.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-limit-by-wrapping.ll @@ -10,8 +10,6 @@ define void @max_backedge_taken_count_by_wrapping1_nsw_nuw(i8 %N, ptr %ptr) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (%N /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 63 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%N /u 4) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%N /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -35,8 +33,6 @@ define void @max_backedge_taken_count_by_wrapping1_nuw(i8 %N, ptr %ptr) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (%N /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 63 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%N /u 4) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%N /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -60,8 +56,6 @@ define void @max_backedge_taken_count_by_wrapping2_nsw_nuw(i8 %N, ptr %ptr) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-64 + %N) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 63 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-64 + %N) /u 4) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-64 + %N) /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -85,8 +79,6 @@ define void @max_backedge_taken_count_by_wrapping2_nuw(i8 %N, ptr %ptr) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-64 + %N) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 63 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-64 + %N) /u 4) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-64 + %N) /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/max-be-count-not-constant.ll b/llvm/test/Analysis/ScalarEvolution/max-be-count-not-constant.ll index 385864b98887..7fe767612271 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-be-count-not-constant.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-be-count-not-constant.ll @@ -23,8 +23,6 @@ define void @pluto(i32 %arg) { ; CHECK-NEXT: Loop %bb2: backedge-taken count is (1 /u (2 + %tmp)) ; CHECK-NEXT: Loop %bb2: constant max backedge-taken count is i32 1 ; CHECK-NEXT: Loop %bb2: symbolic max backedge-taken count is (1 /u (2 + %tmp)) -; CHECK-NEXT: Loop %bb2: Predicated backedge-taken count is (1 /u (2 + %tmp)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %bb2: Trip multiple is 1 ; bb: diff --git a/llvm/test/Analysis/ScalarEvolution/max-expr-cache.ll b/llvm/test/Analysis/ScalarEvolution/max-expr-cache.ll index 46e65a36dc65..e1c05c4b431f 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-expr-cache.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-expr-cache.ll @@ -90,13 +90,10 @@ define void @smax(i32 %tmp3) { ; CHECK-NEXT: Loop %bb53: backedge-taken count is (-1 + (zext i32 (0 smax %tmp49) to i64) + (-1 * undef)) ; CHECK-NEXT: Loop %bb53: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %bb53: symbolic max backedge-taken count is (-1 + (zext i32 (0 smax %tmp49) to i64) + (-1 * undef)) -; CHECK-NEXT: Loop %bb53: Predicated backedge-taken count is (-1 + (zext i32 (0 smax %tmp49) to i64) + (-1 * undef)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %bb53: Trip multiple is 1 ; CHECK-NEXT: Loop %bb4: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %bb4: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %bb4: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %bb4: Unpredictable predicated backedge-taken count. ; entry: br label %bb4 @@ -252,13 +249,10 @@ define void @umax(i32 %tmp3) { ; CHECK-NEXT: Loop %bb53: backedge-taken count is (-1 + (zext i32 %tmp49 to i64) + (-1 * undef)) ; CHECK-NEXT: Loop %bb53: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %bb53: symbolic max backedge-taken count is (-1 + (zext i32 %tmp49 to i64) + (-1 * undef)) -; CHECK-NEXT: Loop %bb53: Predicated backedge-taken count is (-1 + (zext i32 %tmp49 to i64) + (-1 * undef)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %bb53: Trip multiple is 1 ; CHECK-NEXT: Loop %bb4: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %bb4: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %bb4: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %bb4: Unpredictable predicated backedge-taken count. ; entry: br label %bb4 diff --git a/llvm/test/Analysis/ScalarEvolution/max-trip-count-address-space.ll b/llvm/test/Analysis/ScalarEvolution/max-trip-count-address-space.ll index 802b76b4072a..4838e89704c0 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-trip-count-address-space.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-trip-count-address-space.ll @@ -12,8 +12,6 @@ define void @foo(ptr addrspace(1) nocapture %d, i32 %n) nounwind { ; CHECK-NEXT: Loop %bb: backedge-taken count is (-1 + %n) ; CHECK-NEXT: Loop %bb: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is (-1 + %n) -; CHECK-NEXT: Loop %bb: Predicated backedge-taken count is (-1 + %n) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %bb: Trip multiple is 1 ; entry: @@ -52,8 +50,6 @@ define void @test(ptr addrspace(1) %a, i32 %n) nounwind { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (zext i32 %n to i64)) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (zext i32 %n to i64)) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (zext i32 %n to i64)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/max-trip-count.ll b/llvm/test/Analysis/ScalarEvolution/max-trip-count.ll index 7f08bec162a5..45552b13ed4f 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-trip-count.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-trip-count.ll @@ -10,8 +10,6 @@ define void @foo(ptr nocapture %d, i32 %n) nounwind { ; CHECK-NEXT: Loop %bb: backedge-taken count is (-1 + %n) ; CHECK-NEXT: Loop %bb: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is (-1 + %n) -; CHECK-NEXT: Loop %bb: Predicated backedge-taken count is (-1 + %n) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %bb: Trip multiple is 1 ; entry: @@ -61,7 +59,6 @@ define i32 @main() nounwind { ; CHECK-NEXT: Loop %for.cond: symbolic max backedge-taken count is 5 ; CHECK-NEXT: symbolic max exit count for for.cond: 5 ; CHECK-NEXT: symbolic max exit count for for.body: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %for.cond: Unpredictable predicated backedge-taken count. ; entry: br label %for.cond @@ -95,8 +92,6 @@ define void @test(ptr %a, i32 %n) nounwind { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (zext i32 %n to i64)) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (zext i32 %n to i64)) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (zext i32 %n to i64)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -136,7 +131,6 @@ define i32 @pr19799() { ; CHECK-NEXT: Loop %for.body.i: symbolic max backedge-taken count is 1 ; CHECK-NEXT: symbolic max exit count for for.body.i: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for for.cond.i: 1 -; CHECK-NEXT: Loop %for.body.i: Unpredictable predicated backedge-taken count. ; entry: store i32 -1, ptr @a, align 4 @@ -170,7 +164,6 @@ define i32 @pr18886() { ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 3 ; CHECK-NEXT: symbolic max exit count for for.body: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for for.cond: 3 -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: store i64 -21, ptr @aa, align 8 @@ -208,7 +201,6 @@ define i32 @cannot_compute_mustexit() { ; CHECK-NEXT: Loop %for.body.i: Unpredictable symbolic max backedge-taken count. ; CHECK-NEXT: symbolic max exit count for for.body.i: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for for.cond.i: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %for.body.i: Unpredictable predicated backedge-taken count. ; entry: store i32 -1, ptr @a, align 4 @@ -243,8 +235,6 @@ define i32 @two_mustexit() { ; CHECK-NEXT: Loop %for.body.i: symbolic max backedge-taken count is 1 ; CHECK-NEXT: symbolic max exit count for for.body.i: 1 ; CHECK-NEXT: symbolic max exit count for for.cond.i: 2 -; CHECK-NEXT: Loop %for.body.i: Predicated backedge-taken count is 1 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body.i: Trip multiple is 1 ; entry: @@ -272,8 +262,6 @@ define i32 @ne_max_trip_count_1(i32 %n) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (zext i3 (trunc i32 %n to i3) to i32) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 7 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (zext i3 (trunc i32 %n to i3) to i32) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (zext i3 (trunc i32 %n to i3) to i32) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -296,8 +284,6 @@ define i32 @ne_max_trip_count_2(i32 %n) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (zext i3 (trunc i32 %n to i3) to i32)) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (zext i3 (trunc i32 %n to i3) to i32)) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (zext i3 (trunc i32 %n to i3) to i32)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -320,8 +306,6 @@ define i32 @ne_max_trip_count_3(i32 %n) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (zext i3 (trunc i32 %n to i3) to i32)) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 6 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (zext i3 (trunc i32 %n to i3) to i32)) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (zext i3 (trunc i32 %n to i3) to i32)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -351,8 +335,6 @@ define i32 @ne_max_trip_count_4(i32 %n) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %n) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %n) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %n) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -383,7 +365,6 @@ define void @changing_end_bound(ptr %n_addr, ptr %addr) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 2147483646 -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -412,7 +393,6 @@ define void @changing_end_bound2(i32 %start, ptr %n_addr, ptr %addr) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is -1 -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -439,7 +419,6 @@ define void @changing_end_bound3(i32 %start, ptr %n_addr, ptr %addr) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 1073741823 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 1073741823 -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -467,7 +446,6 @@ define void @changing_end_bound4(i32 %start, ptr %n_addr, ptr %addr) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -495,7 +473,6 @@ define void @changing_end_bound5(i32 %stride, i32 %start, ptr %n_addr, ptr %addr ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -522,7 +499,6 @@ define void @changing_end_bound6(i32 %start, ptr %n_addr, ptr %addr) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -549,7 +525,6 @@ define void @changing_end_bound7(i32 %start, ptr %n_addr, ptr %addr) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -575,8 +550,6 @@ define void @max_overflow_se(i8 %n) mustprogress { ; CHECK-NEXT: Loop %loop: backedge-taken count is 0 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 0 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 0 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -604,7 +577,6 @@ define void @max_overflow_me(i8 %n) mustprogress { ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 1 ; CHECK-NEXT: symbolic max exit count for loop: 1 ; CHECK-NEXT: symbolic max exit count for latch: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -632,8 +604,6 @@ define void @bool_stride(i1 %s, i1 %n) mustprogress { ; CHECK-NEXT: Loop %loop: backedge-taken count is false ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is false -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -658,8 +628,6 @@ define void @ne_zero_max_btc(i32 %a) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 0 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 0 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 0 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 0 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/min-max-exprs.ll b/llvm/test/Analysis/ScalarEvolution/min-max-exprs.ll index 8616c98ef1ed..6ededf247771 100644 --- a/llvm/test/Analysis/ScalarEvolution/min-max-exprs.ll +++ b/llvm/test/Analysis/ScalarEvolution/min-max-exprs.ll @@ -49,8 +49,6 @@ define void @f(ptr %A, i32 %N) { ; CHECK-NEXT: Loop %bb1: backedge-taken count is (0 smax %N) ; CHECK-NEXT: Loop %bb1: constant max backedge-taken count is i32 2147483647 ; CHECK-NEXT: Loop %bb1: symbolic max backedge-taken count is (0 smax %N) -; CHECK-NEXT: Loop %bb1: Predicated backedge-taken count is (0 smax %N) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %bb1: Trip multiple is 1 ; bb: diff --git a/llvm/test/Analysis/ScalarEvolution/ne-overflow.ll b/llvm/test/Analysis/ScalarEvolution/ne-overflow.ll index 9f508f6ddafe..7b14807364a4 100644 --- a/llvm/test/Analysis/ScalarEvolution/ne-overflow.ll +++ b/llvm/test/Analysis/ScalarEvolution/ne-overflow.ll @@ -13,8 +13,6 @@ define void @test(i32 %N) mustprogress { ; CHECK-NEXT: Loop %for.body: backedge-taken count is ((-2 + %N) /u 2) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483647 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((-2 + %N) /u 2) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-2 + %N) /u 2) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -36,8 +34,6 @@ define void @test_preinc(i32 %N) mustprogress { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (%N /u 2) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483647 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (%N /u 2) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (%N /u 2) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -62,7 +58,6 @@ define void @test_well_defined_infinite_st(i32 %N) mustprogress { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -84,7 +79,6 @@ define void @test_well_defined_infinite_ld(i32 %N) mustprogress { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -106,7 +100,6 @@ define void @test_no_mustprogress(i32 %N) { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -129,8 +122,6 @@ define void @test_1024(i32 %N) mustprogress { ; CHECK-NEXT: Loop %for.body: backedge-taken count is ((-1024 + %N) /u 1024) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 4194303 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((-1024 + %N) /u 1024) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-1024 + %N) /u 1024) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -152,8 +143,6 @@ define void @test_uneven_divide(i32 %N) mustprogress { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (-1431655765 * %N)) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (-1431655765 * %N)) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (-1431655765 * %N)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -175,7 +164,6 @@ define void @test_non_invariant_rhs() mustprogress { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -199,7 +187,6 @@ define void @test_abnormal_exit(i32 %N) mustprogress { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -226,7 +213,6 @@ define void @test_other_exit(i32 %N) mustprogress { ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 9 ; CHECK-NEXT: symbolic max exit count for for.body: 9 ; CHECK-NEXT: symbolic max exit count for for.latch: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -275,7 +261,6 @@ define void @test_sext(i64 %N) mustprogress { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -297,7 +282,6 @@ define void @test_zext_of_sext(i64 %N) mustprogress { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -320,7 +304,6 @@ define void @test_zext_offset(i64 %N) mustprogress { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -343,7 +326,6 @@ define void @test_sext_offset(i64 %N) mustprogress { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body diff --git a/llvm/test/Analysis/ScalarEvolution/no-wrap-add-exprs.ll b/llvm/test/Analysis/ScalarEvolution/no-wrap-add-exprs.ll index 10f38fc1af37..bd2ffddf396f 100644 --- a/llvm/test/Analysis/ScalarEvolution/no-wrap-add-exprs.ll +++ b/llvm/test/Analysis/ScalarEvolution/no-wrap-add-exprs.ll @@ -306,7 +306,6 @@ define i1 @test2_a(i32 %a, i32 %b, i1 %will_overflow) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br i1 %will_overflow, label %exit1, label %loop @@ -343,7 +342,6 @@ define i1 @test2_b(i32 %a, i32 %b, i1 %will_overflow) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br i1 %will_overflow, label %exit1, label %loop diff --git a/llvm/test/Analysis/ScalarEvolution/no-wrap-symbolic-becount.ll b/llvm/test/Analysis/ScalarEvolution/no-wrap-symbolic-becount.ll index eec7f16148eb..e13a8976bf5a 100644 --- a/llvm/test/Analysis/ScalarEvolution/no-wrap-symbolic-becount.ll +++ b/llvm/test/Analysis/ScalarEvolution/no-wrap-symbolic-becount.ll @@ -30,7 +30,6 @@ define i32 @test_01(i32 %start, ptr %p, ptr %q) { ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i32 %start to i64) ; CHECK-NEXT: symbolic max exit count for loop: (zext i32 %start to i64) ; CHECK-NEXT: symbolic max exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: %0 = zext i32 %start to i64 @@ -76,8 +75,6 @@ define i32 @test_02(i32 %start, ptr %p, ptr %q) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (9223372036854775806 + (2147483648 * (zext i32 %start to i64))) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -2147483650 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (9223372036854775806 + (2147483648 * (zext i32 %start to i64))) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (9223372036854775806 + (2147483648 * (zext i32 %start to i64))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -108,7 +105,6 @@ define void @pointer_iv_nowrap(ptr %startptr, ptr %endptr) local_unnamed_addr { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: %init = getelementptr inbounds i8, ptr %startptr, i64 2000 @@ -137,8 +133,6 @@ define void @pointer_iv_nowrap_guard(ptr %startptr, ptr %endptr) local_unnamed_a ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-8001 + (-1 * (ptrtoint ptr %startptr to i64)) + ((8004 + (ptrtoint ptr %startptr to i64)) umax (ptrtoint ptr %endptr to i64))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4611686018427387903 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-8001 + (-1 * (ptrtoint ptr %startptr to i64)) + ((8004 + (ptrtoint ptr %startptr to i64)) umax (ptrtoint ptr %endptr to i64))) /u 4) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-8001 + (-1 * (ptrtoint ptr %startptr to i64)) + ((8004 + (ptrtoint ptr %startptr to i64)) umax (ptrtoint ptr %endptr to i64))) /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/nsw-offset-assume.ll b/llvm/test/Analysis/ScalarEvolution/nsw-offset-assume.ll index e8462fc7459e..305a2bfca2b6 100644 --- a/llvm/test/Analysis/ScalarEvolution/nsw-offset-assume.ll +++ b/llvm/test/Analysis/ScalarEvolution/nsw-offset-assume.ll @@ -47,8 +47,6 @@ define void @foo(i32 %no, ptr nocapture %d, ptr nocapture %q) nounwind { ; CHECK-NEXT: Loop %bb: backedge-taken count is ((-1 + (2 * (%no /u 2))) /u 2) ; CHECK-NEXT: Loop %bb: constant max backedge-taken count is i32 1073741822 ; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is ((-1 + (2 * (%no /u 2))) /u 2) -; CHECK-NEXT: Loop %bb: Predicated backedge-taken count is ((-1 + (2 * (%no /u 2))) /u 2) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %bb: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/nsw-offset.ll b/llvm/test/Analysis/ScalarEvolution/nsw-offset.ll index f27018ffe17c..d6e78eca398e 100644 --- a/llvm/test/Analysis/ScalarEvolution/nsw-offset.ll +++ b/llvm/test/Analysis/ScalarEvolution/nsw-offset.ll @@ -44,8 +44,6 @@ define void @foo(i32 %no, ptr nocapture %d, ptr nocapture %q) nounwind { ; CHECK-NEXT: Loop %bb: backedge-taken count is ((-1 + (2 * (%no /u 2))) /u 2) ; CHECK-NEXT: Loop %bb: constant max backedge-taken count is i32 1073741822 ; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is ((-1 + (2 * (%no /u 2))) /u 2) -; CHECK-NEXT: Loop %bb: Predicated backedge-taken count is ((-1 + (2 * (%no /u 2))) /u 2) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %bb: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/nsw.ll b/llvm/test/Analysis/ScalarEvolution/nsw.ll index 0240902a18fb..275d3f19ab13 100644 --- a/llvm/test/Analysis/ScalarEvolution/nsw.ll +++ b/llvm/test/Analysis/ScalarEvolution/nsw.ll @@ -30,7 +30,6 @@ define void @test1(ptr %p) nounwind { ; CHECK-NEXT: Loop %bb: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %bb: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %bb: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %bb: Unpredictable predicated backedge-taken count. ; entry: %tmp = load double, ptr %p, align 8 ; [#uses=1] @@ -79,8 +78,6 @@ define void @test2(ptr %begin, ptr %end) ssp { ; CHECK-NEXT: Loop %for.body.i.i: backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %begin to i64)) + (ptrtoint ptr %end to i64)) /u 4) ; CHECK-NEXT: Loop %for.body.i.i: constant max backedge-taken count is i64 4611686018427387903 ; CHECK-NEXT: Loop %for.body.i.i: symbolic max backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %begin to i64)) + (ptrtoint ptr %end to i64)) /u 4) -; CHECK-NEXT: Loop %for.body.i.i: Predicated backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %begin to i64)) + (ptrtoint ptr %end to i64)) /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body.i.i: Trip multiple is 1 ; entry: @@ -120,8 +117,6 @@ define void @test3(ptr %begin, ptr %end) nounwind ssp { ; CHECK-NEXT: Loop %for.body.i.i: backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %begin to i64)) + (ptrtoint ptr %end to i64)) /u 4) ; CHECK-NEXT: Loop %for.body.i.i: constant max backedge-taken count is i64 4611686018427387903 ; CHECK-NEXT: Loop %for.body.i.i: symbolic max backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %begin to i64)) + (ptrtoint ptr %end to i64)) /u 4) -; CHECK-NEXT: Loop %for.body.i.i: Predicated backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %begin to i64)) + (ptrtoint ptr %end to i64)) /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body.i.i: Trip multiple is 1 ; entry: @@ -183,8 +178,6 @@ define i32 @PR12375(ptr readnone %arg) { ; CHECK-NEXT: Loop %bb1: backedge-taken count is 1 ; CHECK-NEXT: Loop %bb1: constant max backedge-taken count is i64 1 ; CHECK-NEXT: Loop %bb1: symbolic max backedge-taken count is 1 -; CHECK-NEXT: Loop %bb1: Predicated backedge-taken count is 1 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %bb1: Trip multiple is 2 ; bb: @@ -214,8 +207,6 @@ define void @PR12376(ptr nocapture %arg, ptr nocapture %arg1) { ; CHECK-NEXT: Loop %bb2: backedge-taken count is ((-1 + (-1 * (ptrtoint ptr %arg to i64)) + ((4 + (ptrtoint ptr %arg to i64)) umax (ptrtoint ptr %arg1 to i64))) /u 4) ; CHECK-NEXT: Loop %bb2: constant max backedge-taken count is i64 4611686018427387902 ; CHECK-NEXT: Loop %bb2: symbolic max backedge-taken count is ((-1 + (-1 * (ptrtoint ptr %arg to i64)) + ((4 + (ptrtoint ptr %arg to i64)) umax (ptrtoint ptr %arg1 to i64))) /u 4) -; CHECK-NEXT: Loop %bb2: Predicated backedge-taken count is ((-1 + (-1 * (ptrtoint ptr %arg to i64)) + ((4 + (ptrtoint ptr %arg to i64)) umax (ptrtoint ptr %arg1 to i64))) /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %bb2: Trip multiple is 1 ; bb: @@ -250,8 +241,6 @@ define void @nswnowrap(i32 %v, ptr %buf) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is ((-1 * %v) + ((1 + %v) smax %v)) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 1, actual taken count either this or zero. ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((-1 * %v) + ((1 + %v) smax %v)), actual taken count either this or zero. -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-1 * %v) + ((1 + %v) smax %v)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -293,8 +282,6 @@ define void @test4(i32 %arg) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (-1 * %arg) + (10 smax (1 + %arg))) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2147483639 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (-1 * %arg) + (10 smax (1 + %arg))) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (-1 * %arg) + (10 smax (1 + %arg))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -327,8 +314,6 @@ define void @bad_postinc_nsw_a(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 7) + (1 umin %n)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 613566756 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 7) + (1 umin %n)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 7) + (1 umin %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -359,7 +344,6 @@ define void @postinc_poison_prop_through_and(i32 %n) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -388,8 +372,6 @@ define void @pr28012(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 + (7 umax %n)) /u 7) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 613566755 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 + (7 umax %n)) /u 7) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 + (7 umax %n)) /u 7) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -421,7 +403,6 @@ define void @select_cond_poison_propagation(ptr %p, i32 %x) nounwind { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -453,8 +434,6 @@ define void @pr66066() { ; CHECK-NEXT: Loop %loop: backedge-taken count is 1 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 1 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 1 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 2 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/outer_phi.ll b/llvm/test/Analysis/ScalarEvolution/outer_phi.ll index c1faa5a8ac59..e4a9753b2405 100644 --- a/llvm/test/Analysis/ScalarEvolution/outer_phi.ll +++ b/llvm/test/Analysis/ScalarEvolution/outer_phi.ll @@ -24,7 +24,6 @@ define i32 @test_01(i32 %a, i32 %b) { ; CHECK-NEXT: Loop %inner: symbolic max backedge-taken count is %b ; CHECK-NEXT: symbolic max exit count for inner: %b ; CHECK-NEXT: symbolic max exit count for inner.backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %inner: Unpredictable predicated backedge-taken count. ; CHECK-NEXT: Loop %outer: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for inner: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for outer.backedge: ***COULDNOTCOMPUTE*** @@ -32,7 +31,6 @@ define i32 @test_01(i32 %a, i32 %b) { ; CHECK-NEXT: Loop %outer: Unpredictable symbolic max backedge-taken count. ; CHECK-NEXT: symbolic max exit count for inner: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for outer.backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %outer: Unpredictable predicated backedge-taken count. ; entry: %b_is_non_negative = icmp sge i32 %b, 0 @@ -88,7 +86,6 @@ define i32 @test_02(i32 %a, i32 %b) { ; CHECK-NEXT: Loop %inner: symbolic max backedge-taken count is ((-1 * %outer.iv) + (%b smax %outer.iv)) ; CHECK-NEXT: symbolic max exit count for inner: ((-1 * %outer.iv) + (%b smax %outer.iv)) ; CHECK-NEXT: symbolic max exit count for inner.backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %inner: Unpredictable predicated backedge-taken count. ; CHECK-NEXT: Loop %outer: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for inner: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for outer.backedge: ***COULDNOTCOMPUTE*** @@ -96,7 +93,6 @@ define i32 @test_02(i32 %a, i32 %b) { ; CHECK-NEXT: Loop %outer: Unpredictable symbolic max backedge-taken count. ; CHECK-NEXT: symbolic max exit count for inner: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for outer.backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %outer: Unpredictable predicated backedge-taken count. ; entry: %b_is_non_negative = icmp sge i32 %b, 0 diff --git a/llvm/test/Analysis/ScalarEvolution/overflow-intrinsics-trip-count.ll b/llvm/test/Analysis/ScalarEvolution/overflow-intrinsics-trip-count.ll index 821572af55c2..5d1c5b5fb6f9 100644 --- a/llvm/test/Analysis/ScalarEvolution/overflow-intrinsics-trip-count.ll +++ b/llvm/test/Analysis/ScalarEvolution/overflow-intrinsics-trip-count.ll @@ -14,8 +14,6 @@ define void @uadd_exhaustive() { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 35 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 35 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 35 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 35 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 36 ; entry: @@ -41,8 +39,6 @@ define void @sadd_exhaustive() { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 67 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 67 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 67 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 67 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 68 ; entry: @@ -68,8 +64,6 @@ define void @usub_exhaustive() { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 50 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 50 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 50 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 50 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 51 ; entry: @@ -95,8 +89,6 @@ define void @ssub_exhaustive() { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 68 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 68 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 68 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 68 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 69 ; entry: @@ -122,8 +114,6 @@ define void @smul_exhaustive() { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 14 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 14 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 14 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 14 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 15 ; entry: @@ -149,8 +139,6 @@ define void @umul_exhaustive() { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 15 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 15 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 15 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 15 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 16 ; entry: @@ -176,8 +164,6 @@ define void @uadd_symbolic_start(i16 %start) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (-1 * %start)) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (-1 * %start)) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (-1 * %start)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -203,8 +189,6 @@ define void @sadd_symbolic_start(i16 %start) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (32767 + (-1 * %start)) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (32767 + (-1 * %start)) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (32767 + (-1 * %start)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -230,7 +214,6 @@ define void @sadd_symbolic_start2(i16 %start) { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br i1 undef, label %for.end, label %for.body.preheader @@ -256,7 +239,6 @@ define void @sadd_symbolic_swapped(i16 %start) { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br i1 undef, label %for.end, label %for.body.preheader @@ -281,8 +263,6 @@ define void @usub_symbolic_start(i16 %start) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is %start ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is %start -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is %start -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -308,8 +288,6 @@ define void @ssub_symbolic_start(i16 %start) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-32768 + %start) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-32768 + %start) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-32768 + %start) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -335,7 +313,6 @@ define void @smul_symbolic_start(i16 %start) { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br i1 undef, label %for.end, label %for.body.preheader @@ -360,7 +337,6 @@ define void @umul_symbolic_start(i16 %start) { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br i1 undef, label %for.end, label %for.body.preheader @@ -389,8 +365,6 @@ define void @sadd_symbolic_non_latch(i16 %start) { ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((230 + (-1 * %start)) umin (32767 + (-1 * %start))) ; CHECK-NEXT: symbolic max exit count for for.body: (32767 + (-1 * %start)) ; CHECK-NEXT: symbolic max exit count for for.latch: (230 + (-1 * %start)) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((230 + (-1 * %start)) umin (32767 + (-1 * %start))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/pr25369.ll b/llvm/test/Analysis/ScalarEvolution/pr25369.ll index 6b5fc6d0a596..2dc3a1272cc1 100644 --- a/llvm/test/Analysis/ScalarEvolution/pr25369.ll +++ b/llvm/test/Analysis/ScalarEvolution/pr25369.ll @@ -11,19 +11,14 @@ define void @hoge1() { ; CHECK-NEXT: Loop %bb13: backedge-taken count is (-2 + (2 * undef) + %tmp7 + %tmp6) ; CHECK-NEXT: Loop %bb13: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %bb13: symbolic max backedge-taken count is (-2 + (2 * undef) + %tmp7 + %tmp6) -; CHECK-NEXT: Loop %bb13: Predicated backedge-taken count is (-2 + (2 * undef) + %tmp7 + %tmp6) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %bb13: Trip multiple is 1 ; CHECK-NEXT: Loop %bb4: backedge-taken count is 20 ; CHECK-NEXT: Loop %bb4: constant max backedge-taken count is i64 20 ; CHECK-NEXT: Loop %bb4: symbolic max backedge-taken count is 20 -; CHECK-NEXT: Loop %bb4: Predicated backedge-taken count is 20 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %bb4: Trip multiple is 21 ; CHECK-NEXT: Loop %bb2: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %bb2: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %bb2: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %bb2: Unpredictable predicated backedge-taken count. ; bb: br i1 undef, label %bb4, label %bb2 @@ -65,17 +60,13 @@ define void @hoge2() { ; CHECK-NEXT: Loop %bb13: backedge-taken count is (-2 + (2 * undef) + %tmp7 + %tmp6) ; CHECK-NEXT: Loop %bb13: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %bb13: symbolic max backedge-taken count is (-2 + (2 * undef) + %tmp7 + %tmp6) -; CHECK-NEXT: Loop %bb13: Predicated backedge-taken count is (-2 + (2 * undef) + %tmp7 + %tmp6) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %bb13: Trip multiple is 1 ; CHECK-NEXT: Loop %bb4: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %bb4: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %bb4: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %bb4: Unpredictable predicated backedge-taken count. ; CHECK-NEXT: Loop %bb2: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %bb2: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %bb2: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %bb2: Unpredictable predicated backedge-taken count. ; bb: br i1 undef, label %bb4, label %bb2 diff --git a/llvm/test/Analysis/ScalarEvolution/pr34538.ll b/llvm/test/Analysis/ScalarEvolution/pr34538.ll index 9386c0eb5457..03fcbbcfd7ca 100644 --- a/llvm/test/Analysis/ScalarEvolution/pr34538.ll +++ b/llvm/test/Analysis/ScalarEvolution/pr34538.ll @@ -7,8 +7,6 @@ define i32 @pr34538() local_unnamed_addr #0 { ; CHECK-NEXT: Loop %do.body: backedge-taken count is 10000 ; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is i32 10000 ; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is 10000 -; CHECK-NEXT: Loop %do.body: Predicated backedge-taken count is 10000 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %do.body: Trip multiple is 10001 ; entry: @@ -32,8 +30,6 @@ define i32 @foo() { ; CHECK-NEXT: Loop %do.body: backedge-taken count is 5000 ; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is i32 5000 ; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is 5000 -; CHECK-NEXT: Loop %do.body: Predicated backedge-taken count is 5000 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %do.body: Trip multiple is 5001 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/pr48225.ll b/llvm/test/Analysis/ScalarEvolution/pr48225.ll index ca4b1bd42bae..71ca293e311a 100644 --- a/llvm/test/Analysis/ScalarEvolution/pr48225.ll +++ b/llvm/test/Analysis/ScalarEvolution/pr48225.ll @@ -28,7 +28,6 @@ define void @test_and(i1 %boolcond) { ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 2 ; CHECK-NEXT: symbolic max exit count for loop: 2 ; CHECK-NEXT: symbolic max exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: %conv = zext i1 %boolcond to i32 @@ -78,7 +77,6 @@ define void @test_or(i1 %boolcond) { ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 2 ; CHECK-NEXT: symbolic max exit count for loop: 2 ; CHECK-NEXT: symbolic max exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: %conv = zext i1 %boolcond to i32 diff --git a/llvm/test/Analysis/ScalarEvolution/pr58402-large-number-of-zext-exprs.ll b/llvm/test/Analysis/ScalarEvolution/pr58402-large-number-of-zext-exprs.ll index a6d155da4bb9..c79befac2fb1 100644 --- a/llvm/test/Analysis/ScalarEvolution/pr58402-large-number-of-zext-exprs.ll +++ b/llvm/test/Analysis/ScalarEvolution/pr58402-large-number-of-zext-exprs.ll @@ -82,7 +82,6 @@ define i32 @pr58402_large_number_of_zext(ptr %dst) { ; CHECK-NEXT: Loop %header: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %header: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %header: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %header: Unpredictable predicated backedge-taken count. ; entry: br label %header diff --git a/llvm/test/Analysis/ScalarEvolution/pr76234.ll b/llvm/test/Analysis/ScalarEvolution/pr76234.ll index 0d82f0ed1a81..bdc5a4ce3d58 100644 --- a/llvm/test/Analysis/ScalarEvolution/pr76234.ll +++ b/llvm/test/Analysis/ScalarEvolution/pr76234.ll @@ -13,7 +13,6 @@ define i32 @PR76234() { ; CHECK-NEXT: Loop %1: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %1: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %1: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %1: Unpredictable predicated backedge-taken count. ; %B9 = shl i896 0, -264147265567832623176169892458258303259423663018060761063980354513336951278362429737208627943828593947337197496628564339441173779751342768625269489231469788454193341999502542084365758838213220526512116454105594202074014146375780869419198449383518238244769290448868999168 br label %1 diff --git a/llvm/test/Analysis/ScalarEvolution/ptrtoint-constantexpr-loop.ll b/llvm/test/Analysis/ScalarEvolution/ptrtoint-constantexpr-loop.ll index 20037b58bd20..d92056b9ffdd 100644 --- a/llvm/test/Analysis/ScalarEvolution/ptrtoint-constantexpr-loop.ll +++ b/llvm/test/Analysis/ScalarEvolution/ptrtoint-constantexpr-loop.ll @@ -24,7 +24,6 @@ define hidden ptr @trunc_ptr_to_i64(ptr %arg, ptr %arg10) { ; PTR64_IDX64-NEXT: Loop %bb11: Unpredictable backedge-taken count. ; PTR64_IDX64-NEXT: Loop %bb11: Unpredictable constant max backedge-taken count. ; PTR64_IDX64-NEXT: Loop %bb11: Unpredictable symbolic max backedge-taken count. -; PTR64_IDX64-NEXT: Loop %bb11: Unpredictable predicated backedge-taken count. ; ; PTR64_IDX32-LABEL: 'trunc_ptr_to_i64' ; PTR64_IDX32-NEXT: Classifying expressions for: @trunc_ptr_to_i64 @@ -42,7 +41,6 @@ define hidden ptr @trunc_ptr_to_i64(ptr %arg, ptr %arg10) { ; PTR64_IDX32-NEXT: Loop %bb11: Unpredictable backedge-taken count. ; PTR64_IDX32-NEXT: Loop %bb11: Unpredictable constant max backedge-taken count. ; PTR64_IDX32-NEXT: Loop %bb11: Unpredictable symbolic max backedge-taken count. -; PTR64_IDX32-NEXT: Loop %bb11: Unpredictable predicated backedge-taken count. ; ; PTR16_IDX16-LABEL: 'trunc_ptr_to_i64' ; PTR16_IDX16-NEXT: Classifying expressions for: @trunc_ptr_to_i64 @@ -60,7 +58,6 @@ define hidden ptr @trunc_ptr_to_i64(ptr %arg, ptr %arg10) { ; PTR16_IDX16-NEXT: Loop %bb11: Unpredictable backedge-taken count. ; PTR16_IDX16-NEXT: Loop %bb11: Unpredictable constant max backedge-taken count. ; PTR16_IDX16-NEXT: Loop %bb11: Unpredictable symbolic max backedge-taken count. -; PTR16_IDX16-NEXT: Loop %bb11: Unpredictable predicated backedge-taken count. ; bb: br label %bb11 @@ -97,7 +94,6 @@ define hidden ptr @trunc_ptr_to_i32(ptr %arg, ptr %arg10) { ; PTR64_IDX64-NEXT: Loop %bb11: Unpredictable backedge-taken count. ; PTR64_IDX64-NEXT: Loop %bb11: Unpredictable constant max backedge-taken count. ; PTR64_IDX64-NEXT: Loop %bb11: Unpredictable symbolic max backedge-taken count. -; PTR64_IDX64-NEXT: Loop %bb11: Unpredictable predicated backedge-taken count. ; ; PTR64_IDX32-LABEL: 'trunc_ptr_to_i32' ; PTR64_IDX32-NEXT: Classifying expressions for: @trunc_ptr_to_i32 @@ -115,7 +111,6 @@ define hidden ptr @trunc_ptr_to_i32(ptr %arg, ptr %arg10) { ; PTR64_IDX32-NEXT: Loop %bb11: Unpredictable backedge-taken count. ; PTR64_IDX32-NEXT: Loop %bb11: Unpredictable constant max backedge-taken count. ; PTR64_IDX32-NEXT: Loop %bb11: Unpredictable symbolic max backedge-taken count. -; PTR64_IDX32-NEXT: Loop %bb11: Unpredictable predicated backedge-taken count. ; ; PTR16_IDX16-LABEL: 'trunc_ptr_to_i32' ; PTR16_IDX16-NEXT: Classifying expressions for: @trunc_ptr_to_i32 @@ -133,7 +128,6 @@ define hidden ptr @trunc_ptr_to_i32(ptr %arg, ptr %arg10) { ; PTR16_IDX16-NEXT: Loop %bb11: Unpredictable backedge-taken count. ; PTR16_IDX16-NEXT: Loop %bb11: Unpredictable constant max backedge-taken count. ; PTR16_IDX16-NEXT: Loop %bb11: Unpredictable symbolic max backedge-taken count. -; PTR16_IDX16-NEXT: Loop %bb11: Unpredictable predicated backedge-taken count. ; bb: br label %bb11 @@ -170,7 +164,6 @@ define hidden ptr @trunc_ptr_to_i128(ptr %arg, ptr %arg10) { ; PTR64_IDX64-NEXT: Loop %bb11: Unpredictable backedge-taken count. ; PTR64_IDX64-NEXT: Loop %bb11: Unpredictable constant max backedge-taken count. ; PTR64_IDX64-NEXT: Loop %bb11: Unpredictable symbolic max backedge-taken count. -; PTR64_IDX64-NEXT: Loop %bb11: Unpredictable predicated backedge-taken count. ; ; PTR64_IDX32-LABEL: 'trunc_ptr_to_i128' ; PTR64_IDX32-NEXT: Classifying expressions for: @trunc_ptr_to_i128 @@ -188,7 +181,6 @@ define hidden ptr @trunc_ptr_to_i128(ptr %arg, ptr %arg10) { ; PTR64_IDX32-NEXT: Loop %bb11: Unpredictable backedge-taken count. ; PTR64_IDX32-NEXT: Loop %bb11: Unpredictable constant max backedge-taken count. ; PTR64_IDX32-NEXT: Loop %bb11: Unpredictable symbolic max backedge-taken count. -; PTR64_IDX32-NEXT: Loop %bb11: Unpredictable predicated backedge-taken count. ; ; PTR16_IDX16-LABEL: 'trunc_ptr_to_i128' ; PTR16_IDX16-NEXT: Classifying expressions for: @trunc_ptr_to_i128 @@ -206,7 +198,6 @@ define hidden ptr @trunc_ptr_to_i128(ptr %arg, ptr %arg10) { ; PTR16_IDX16-NEXT: Loop %bb11: Unpredictable backedge-taken count. ; PTR16_IDX16-NEXT: Loop %bb11: Unpredictable constant max backedge-taken count. ; PTR16_IDX16-NEXT: Loop %bb11: Unpredictable symbolic max backedge-taken count. -; PTR16_IDX16-NEXT: Loop %bb11: Unpredictable predicated backedge-taken count. ; bb: br label %bb11 @@ -238,7 +229,6 @@ define void @zext_ptr_to_i32(i32 %arg, i32 %arg6) { ; PTR64_IDX64-NEXT: Loop %bb7: Unpredictable backedge-taken count. ; PTR64_IDX64-NEXT: Loop %bb7: Unpredictable constant max backedge-taken count. ; PTR64_IDX64-NEXT: Loop %bb7: Unpredictable symbolic max backedge-taken count. -; PTR64_IDX64-NEXT: Loop %bb7: Unpredictable predicated backedge-taken count. ; ; PTR64_IDX32-LABEL: 'zext_ptr_to_i32' ; PTR64_IDX32-NEXT: Classifying expressions for: @zext_ptr_to_i32 @@ -250,7 +240,6 @@ define void @zext_ptr_to_i32(i32 %arg, i32 %arg6) { ; PTR64_IDX32-NEXT: Loop %bb7: Unpredictable backedge-taken count. ; PTR64_IDX32-NEXT: Loop %bb7: Unpredictable constant max backedge-taken count. ; PTR64_IDX32-NEXT: Loop %bb7: Unpredictable symbolic max backedge-taken count. -; PTR64_IDX32-NEXT: Loop %bb7: Unpredictable predicated backedge-taken count. ; ; PTR16_IDX16-LABEL: 'zext_ptr_to_i32' ; PTR16_IDX16-NEXT: Classifying expressions for: @zext_ptr_to_i32 @@ -262,7 +251,6 @@ define void @zext_ptr_to_i32(i32 %arg, i32 %arg6) { ; PTR16_IDX16-NEXT: Loop %bb7: Unpredictable backedge-taken count. ; PTR16_IDX16-NEXT: Loop %bb7: Unpredictable constant max backedge-taken count. ; PTR16_IDX16-NEXT: Loop %bb7: Unpredictable symbolic max backedge-taken count. -; PTR16_IDX16-NEXT: Loop %bb7: Unpredictable predicated backedge-taken count. ; bb: br label %bb7 @@ -291,7 +279,6 @@ define void @sext_to_i32(i32 %arg, i32 %arg6) { ; PTR64_IDX64-NEXT: Loop %bb7: Unpredictable backedge-taken count. ; PTR64_IDX64-NEXT: Loop %bb7: Unpredictable constant max backedge-taken count. ; PTR64_IDX64-NEXT: Loop %bb7: Unpredictable symbolic max backedge-taken count. -; PTR64_IDX64-NEXT: Loop %bb7: Unpredictable predicated backedge-taken count. ; ; PTR64_IDX32-LABEL: 'sext_to_i32' ; PTR64_IDX32-NEXT: Classifying expressions for: @sext_to_i32 @@ -305,7 +292,6 @@ define void @sext_to_i32(i32 %arg, i32 %arg6) { ; PTR64_IDX32-NEXT: Loop %bb7: Unpredictable backedge-taken count. ; PTR64_IDX32-NEXT: Loop %bb7: Unpredictable constant max backedge-taken count. ; PTR64_IDX32-NEXT: Loop %bb7: Unpredictable symbolic max backedge-taken count. -; PTR64_IDX32-NEXT: Loop %bb7: Unpredictable predicated backedge-taken count. ; ; PTR16_IDX16-LABEL: 'sext_to_i32' ; PTR16_IDX16-NEXT: Classifying expressions for: @sext_to_i32 @@ -319,7 +305,6 @@ define void @sext_to_i32(i32 %arg, i32 %arg6) { ; PTR16_IDX16-NEXT: Loop %bb7: Unpredictable backedge-taken count. ; PTR16_IDX16-NEXT: Loop %bb7: Unpredictable constant max backedge-taken count. ; PTR16_IDX16-NEXT: Loop %bb7: Unpredictable symbolic max backedge-taken count. -; PTR16_IDX16-NEXT: Loop %bb7: Unpredictable predicated backedge-taken count. ; bb: br label %bb7 @@ -351,8 +336,6 @@ define i64 @sext_like_noop(i32 %n) { ; PTR64_IDX64-NEXT: Loop %for.body: backedge-taken count is (-2 + (trunc i64 (ptrtoint ptr @sext_like_noop to i64) to i32)) ; PTR64_IDX64-NEXT: Loop %for.body: constant max backedge-taken count is i32 -1 ; PTR64_IDX64-NEXT: Loop %for.body: symbolic max backedge-taken count is (-2 + (trunc i64 (ptrtoint ptr @sext_like_noop to i64) to i32)) -; PTR64_IDX64-NEXT: Loop %for.body: Predicated backedge-taken count is (-2 + (trunc i64 (ptrtoint ptr @sext_like_noop to i64) to i32)) -; PTR64_IDX64-NEXT: Predicates: ; PTR64_IDX64-NEXT: Loop %for.body: Trip multiple is 1 ; ; PTR64_IDX32-LABEL: 'sext_like_noop' @@ -369,8 +352,6 @@ define i64 @sext_like_noop(i32 %n) { ; PTR64_IDX32-NEXT: Loop %for.body: backedge-taken count is (-2 + ptrtoint (ptr @sext_like_noop to i32)) ; PTR64_IDX32-NEXT: Loop %for.body: constant max backedge-taken count is i32 -1 ; PTR64_IDX32-NEXT: Loop %for.body: symbolic max backedge-taken count is (-2 + ptrtoint (ptr @sext_like_noop to i32)) -; PTR64_IDX32-NEXT: Loop %for.body: Predicated backedge-taken count is (-2 + ptrtoint (ptr @sext_like_noop to i32)) -; PTR64_IDX32-NEXT: Predicates: ; PTR64_IDX32-NEXT: Loop %for.body: Trip multiple is 1 ; ; PTR16_IDX16-LABEL: 'sext_like_noop' @@ -387,8 +368,6 @@ define i64 @sext_like_noop(i32 %n) { ; PTR16_IDX16-NEXT: Loop %for.body: backedge-taken count is (-2 + (zext i16 (ptrtoint ptr @sext_like_noop to i16) to i32)) ; PTR16_IDX16-NEXT: Loop %for.body: constant max backedge-taken count is i32 -1 ; PTR16_IDX16-NEXT: Loop %for.body: symbolic max backedge-taken count is (-2 + (zext i16 (ptrtoint ptr @sext_like_noop to i16) to i32)) -; PTR16_IDX16-NEXT: Loop %for.body: Predicated backedge-taken count is (-2 + (zext i16 (ptrtoint ptr @sext_like_noop to i16) to i32)) -; PTR16_IDX16-NEXT: Predicates: ; PTR16_IDX16-NEXT: Loop %for.body: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/ptrtoint.ll b/llvm/test/Analysis/ScalarEvolution/ptrtoint.ll index f30b3752d7b3..e784d2538598 100644 --- a/llvm/test/Analysis/ScalarEvolution/ptrtoint.ll +++ b/llvm/test/Analysis/ScalarEvolution/ptrtoint.ll @@ -231,8 +231,6 @@ define void @ptrtoint_of_addrec(ptr %in, i32 %count) { ; X64-NEXT: Loop %loop: backedge-taken count is (-1 + (zext i32 %count to i64)) ; X64-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; X64-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (zext i32 %count to i64)) -; X64-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (zext i32 %count to i64)) -; X64-NEXT: Predicates: ; X64-NEXT: Loop %loop: Trip multiple is 1 ; ; X32-LABEL: 'ptrtoint_of_addrec' @@ -251,8 +249,6 @@ define void @ptrtoint_of_addrec(ptr %in, i32 %count) { ; X32-NEXT: Loop %loop: backedge-taken count is (-1 + (zext i32 %count to i64)) ; X32-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; X32-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (zext i32 %count to i64)) -; X32-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (zext i32 %count to i64)) -; X32-NEXT: Predicates: ; X32-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -399,8 +395,6 @@ define void @pr46786_c26_char(ptr %arg, ptr %arg1, ptr %arg2) { ; X64-NEXT: Loop %bb6: backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) ; X64-NEXT: Loop %bb6: constant max backedge-taken count is i64 -1 ; X64-NEXT: Loop %bb6: symbolic max backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) -; X64-NEXT: Loop %bb6: Predicated backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) -; X64-NEXT: Predicates: ; X64-NEXT: Loop %bb6: Trip multiple is 1 ; ; X32-LABEL: 'pr46786_c26_char' @@ -427,8 +421,6 @@ define void @pr46786_c26_char(ptr %arg, ptr %arg1, ptr %arg2) { ; X32-NEXT: Loop %bb6: backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) ; X32-NEXT: Loop %bb6: constant max backedge-taken count is i32 -1 ; X32-NEXT: Loop %bb6: symbolic max backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) -; X32-NEXT: Loop %bb6: Predicated backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) -; X32-NEXT: Predicates: ; X32-NEXT: Loop %bb6: Trip multiple is 1 ; %i = icmp eq ptr %arg, %arg1 @@ -488,8 +480,6 @@ define void @pr46786_c26_int(ptr %arg, ptr %arg1, ptr %arg2) { ; X64-NEXT: Loop %bb6: backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) /u 4) ; X64-NEXT: Loop %bb6: constant max backedge-taken count is i64 4611686018427387903 ; X64-NEXT: Loop %bb6: symbolic max backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) /u 4) -; X64-NEXT: Loop %bb6: Predicated backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) /u 4) -; X64-NEXT: Predicates: ; X64-NEXT: Loop %bb6: Trip multiple is 1 ; ; X32-LABEL: 'pr46786_c26_int' @@ -518,8 +508,6 @@ define void @pr46786_c26_int(ptr %arg, ptr %arg1, ptr %arg2) { ; X32-NEXT: Loop %bb6: backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) /u 4) ; X32-NEXT: Loop %bb6: constant max backedge-taken count is i32 1073741823 ; X32-NEXT: Loop %bb6: symbolic max backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) /u 4) -; X32-NEXT: Loop %bb6: Predicated backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) /u 4) -; X32-NEXT: Predicates: ; X32-NEXT: Loop %bb6: Trip multiple is 1 ; %i = icmp eq ptr %arg, %arg1 @@ -568,7 +556,6 @@ define void @ptrtoint_of_integer(ptr %arg, i64 %arg1, i1 %arg2) local_unnamed_ad ; X64-NEXT: Loop %bb8: symbolic max backedge-taken count is (-2 + (-1 * %arg1) + (ptrtoint ptr %arg to i64)) ; X64-NEXT: symbolic max exit count for bb8: ***COULDNOTCOMPUTE*** ; X64-NEXT: symbolic max exit count for bb10: (-2 + (-1 * %arg1) + (ptrtoint ptr %arg to i64)) -; X64-NEXT: Loop %bb8: Unpredictable predicated backedge-taken count. ; ; X32-LABEL: 'ptrtoint_of_integer' ; X32-NEXT: Classifying expressions for: @ptrtoint_of_integer @@ -588,7 +575,6 @@ define void @ptrtoint_of_integer(ptr %arg, i64 %arg1, i1 %arg2) local_unnamed_ad ; X32-NEXT: Loop %bb8: symbolic max backedge-taken count is (-2 + (zext i32 (ptrtoint ptr %arg to i32) to i64) + (-1 * %arg1)) ; X32-NEXT: symbolic max exit count for bb8: ***COULDNOTCOMPUTE*** ; X32-NEXT: symbolic max exit count for bb10: (-2 + (zext i32 (ptrtoint ptr %arg to i32) to i64) + (-1 * %arg1)) -; X32-NEXT: Loop %bb8: Unpredictable predicated backedge-taken count. ; bb: %i = icmp eq ptr %arg, null diff --git a/llvm/test/Analysis/ScalarEvolution/range-signedness.ll b/llvm/test/Analysis/ScalarEvolution/range-signedness.ll index 74b78b35513e..4484bfc1924b 100644 --- a/llvm/test/Analysis/ScalarEvolution/range-signedness.ll +++ b/llvm/test/Analysis/ScalarEvolution/range-signedness.ll @@ -14,7 +14,6 @@ define void @x(ptr %cond) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -42,8 +41,6 @@ define void @y(ptr %addr) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 10 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 10 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 10 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 10 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 11 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/range_nw_flag.ll b/llvm/test/Analysis/ScalarEvolution/range_nw_flag.ll index dd487be904d5..02168e9b8195 100644 --- a/llvm/test/Analysis/ScalarEvolution/range_nw_flag.ll +++ b/llvm/test/Analysis/ScalarEvolution/range_nw_flag.ll @@ -17,8 +17,6 @@ define void @test-add-nuw(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -53,8 +51,6 @@ define void @test-addrec-nuw(ptr %input, i32 %offset, i32 %numIterations) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -91,8 +87,6 @@ define void @test-addrec-nsw-start-neg-strip-neg(ptr %input, i32 %offset, i32 %n ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %numIterations)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %numIterations)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %numIterations)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -129,8 +123,6 @@ define void @test-addrec-nsw-start-pos-strip-neg(ptr %input, i32 %offset, i32 %n ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %numIterations)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %numIterations)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %numIterations)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -167,8 +159,6 @@ define void @test-addrec-nsw-start-pos-strip-pos(ptr %input, i32 %offset, i32 %n ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -205,8 +195,6 @@ define void @test-addrec-nsw-start-neg-strip-pos(ptr %input, i32 %offset, i32 %n ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %numIterations) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %numIterations) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/ranges.ll b/llvm/test/Analysis/ScalarEvolution/ranges.ll index e6e280c676b9..cf9d999a6a1f 100644 --- a/llvm/test/Analysis/ScalarEvolution/ranges.ll +++ b/llvm/test/Analysis/ScalarEvolution/ranges.ll @@ -116,7 +116,6 @@ define i32 @phi_div() { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -138,8 +137,6 @@ define void @add_6(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 6) + (1 umin %n)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 715827882 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 6) + (1 umin %n)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 6) + (1 umin %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -165,8 +162,6 @@ define void @add_7(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 7) + (1 umin %n)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 613566756 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 7) + (1 umin %n)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 7) + (1 umin %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -192,8 +187,6 @@ define void @add_8(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((7 + %n) /u 8) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 536870911 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((7 + %n) /u 8) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((7 + %n) /u 8) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -220,8 +213,6 @@ define void @add_9(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 9) + (1 umin %n)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 477218588 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 9) + (1 umin %n)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 9) + (1 umin %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -248,8 +239,6 @@ define void @add_10(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 10) + (1 umin %n)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 429496729 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 10) + (1 umin %n)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 10) + (1 umin %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -276,7 +265,6 @@ define void @add_8_wrap(i32 %n) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -302,7 +290,6 @@ define void @add_10_wrap(i32 %n) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -328,7 +315,6 @@ define void @mul_6(i32 %n) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -354,7 +340,6 @@ define void @mul_7(i32 %n) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -380,7 +365,6 @@ define void @mul_8(i32 %n) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -406,7 +390,6 @@ define void @mul_9(i32 %n) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -432,7 +415,6 @@ define void @mul_10(i32 %n) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -458,7 +440,6 @@ define void @mul_8_wrap(i32 %n) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -484,7 +465,6 @@ define void @mul_10_wrap(i32 %n) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -513,8 +493,6 @@ define void @truncate(i16 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 9) + (1 umin %n)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i16 7281 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 9) + (1 umin %n)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((((-1 * (1 umin %n)) + %n) /u 9) + (1 umin %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/sdiv.ll b/llvm/test/Analysis/ScalarEvolution/sdiv.ll index 48856d00b1c2..e01f84fb2226 100644 --- a/llvm/test/Analysis/ScalarEvolution/sdiv.ll +++ b/llvm/test/Analysis/ScalarEvolution/sdiv.ll @@ -33,8 +33,6 @@ define dso_local void @_Z4loopi(i32 %width) local_unnamed_addr #0 { ; CHECK-NEXT: Loop %for.cond: backedge-taken count is %width ; CHECK-NEXT: Loop %for.cond: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %for.cond: symbolic max backedge-taken count is %width -; CHECK-NEXT: Loop %for.cond: Predicated backedge-taken count is %width -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.cond: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/sext-add-inreg-loop.ll b/llvm/test/Analysis/ScalarEvolution/sext-add-inreg-loop.ll index 3de4dcfc4046..db0bac933086 100644 --- a/llvm/test/Analysis/ScalarEvolution/sext-add-inreg-loop.ll +++ b/llvm/test/Analysis/ScalarEvolution/sext-add-inreg-loop.ll @@ -26,8 +26,6 @@ define dso_local i32 @test_loop(ptr nocapture noundef readonly %x) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 8 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 8 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 8 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 8 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 9 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/sext-mul.ll b/llvm/test/Analysis/ScalarEvolution/sext-mul.ll index e375e7a066c9..f369bfe0b631 100644 --- a/llvm/test/Analysis/ScalarEvolution/sext-mul.ll +++ b/llvm/test/Analysis/ScalarEvolution/sext-mul.ll @@ -32,8 +32,6 @@ define void @foo(ptr nocapture %arg, i32 %arg1, i32 %arg2) { ; CHECK-NEXT: Loop %bb7: backedge-taken count is (-1 + (zext i32 %arg2 to i64)) ; CHECK-NEXT: Loop %bb7: constant max backedge-taken count is i64 2147483646 ; CHECK-NEXT: Loop %bb7: symbolic max backedge-taken count is (-1 + (zext i32 %arg2 to i64)) -; CHECK-NEXT: Loop %bb7: Predicated backedge-taken count is (-1 + (zext i32 %arg2 to i64)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %bb7: Trip multiple is 1 ; bb: @@ -99,8 +97,6 @@ define void @goo(ptr nocapture %arg3, i32 %arg4, i32 %arg5) { ; CHECK-NEXT: Loop %bb7: backedge-taken count is (-1 + (zext i32 %arg5 to i128)) ; CHECK-NEXT: Loop %bb7: constant max backedge-taken count is i128 2147483646 ; CHECK-NEXT: Loop %bb7: symbolic max backedge-taken count is (-1 + (zext i32 %arg5 to i128)) -; CHECK-NEXT: Loop %bb7: Predicated backedge-taken count is (-1 + (zext i32 %arg5 to i128)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %bb7: Trip multiple is 1 ; bb: diff --git a/llvm/test/Analysis/ScalarEvolution/sext-to-zext.ll b/llvm/test/Analysis/ScalarEvolution/sext-to-zext.ll index 7db59734e6be..f7d01974343d 100644 --- a/llvm/test/Analysis/ScalarEvolution/sext-to-zext.ll +++ b/llvm/test/Analysis/ScalarEvolution/sext-to-zext.ll @@ -22,8 +22,6 @@ define void @f(i1 %c) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 99 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 99 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 99 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 99 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 100 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/shift-op.ll b/llvm/test/Analysis/ScalarEvolution/shift-op.ll index 5a1e45431aaf..c3984673cfc7 100644 --- a/llvm/test/Analysis/ScalarEvolution/shift-op.ll +++ b/llvm/test/Analysis/ScalarEvolution/shift-op.ll @@ -8,7 +8,6 @@ define void @test0(i32 %init) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 32 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 32 -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -30,7 +29,6 @@ define void @test1(i32 %init) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 32 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 32 -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -54,7 +52,6 @@ define void @test2(i32 %init) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -76,7 +73,6 @@ define void @test3(ptr %init.ptr) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 32 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 32 -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: %init = load i32, ptr %init.ptr, !range !0 @@ -99,7 +95,6 @@ define void @test4(ptr %init.ptr) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 32 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 32 -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: %init = load i32, ptr %init.ptr, !range !1 @@ -123,7 +118,6 @@ define void @test5(ptr %init.ptr) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: %init = load i32, ptr %init.ptr, !range !1 @@ -147,7 +141,6 @@ define void @test6(i32 %init, i32 %shift.amt) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -169,7 +162,6 @@ define void @test7(i32 %init) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 32 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 32 -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -193,7 +185,6 @@ define void @test8(i32 %init) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -217,7 +208,6 @@ define void @test9() { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop diff --git a/llvm/test/Analysis/ScalarEvolution/shift-recurrences.ll b/llvm/test/Analysis/ScalarEvolution/shift-recurrences.ll index 0a2e121c0502..2362741581ef 100644 --- a/llvm/test/Analysis/ScalarEvolution/shift-recurrences.ll +++ b/llvm/test/Analysis/ScalarEvolution/shift-recurrences.ll @@ -12,7 +12,6 @@ define void @test_lshr() { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -36,7 +35,6 @@ define void @test_lshr2() { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -60,7 +58,6 @@ define void @test_ashr_zeros() { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -83,7 +80,6 @@ define void @test_ashr_ones() { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -107,7 +103,6 @@ define void @test_ashr_ones2() { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -132,7 +127,6 @@ define void @test_ashr_unknown(i64 %start) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -157,7 +151,6 @@ define void @test_ashr_wrong_op(i64 %start) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -181,7 +174,6 @@ define void @test_shl() { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -209,8 +201,6 @@ define void @test_shl2() { ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -244,8 +234,6 @@ define void @test_shl3(i1 %c) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -278,8 +266,6 @@ define void @test_shl4() { ; CHECK-NEXT: Loop %loop: backedge-taken count is 60 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 60 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 60 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 60 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 61 ; entry: @@ -311,8 +297,6 @@ define void @test_shl5() { ; CHECK-NEXT: Loop %loop: backedge-taken count is 61 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 61 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 61 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 61 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 62 ; entry: @@ -346,8 +330,6 @@ define void @test_shl6(i1 %c) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -380,8 +362,6 @@ define void @test_shl7(i1 %c, i64 %shiftamt) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -434,7 +414,6 @@ define void @unreachable_binop() { ; CHECK-NEXT: Loop %header: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %header: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %header: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %header: Unpredictable predicated backedge-taken count. ; entry: br label %header @@ -492,7 +471,6 @@ define void @nonloop_recurrence_2() { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; bb: br label %loop @@ -527,8 +505,6 @@ define void @test_ashr_tc_positive() { ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -559,8 +535,6 @@ define void @test_ashr_tc_negative() { ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -593,8 +567,6 @@ define void @test_ashr_tc_either(i1 %a) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 60 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 60 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 60 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 60 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 61 ; entry: @@ -626,8 +598,6 @@ define void @test_ashr_zero_shift() { ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -658,8 +628,6 @@ define void @test_lshr_tc_positive() { ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -690,8 +658,6 @@ define void @test_lshr_tc_negative() { ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -724,8 +690,6 @@ define void @test_lshr_tc_either(i1 %a) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -757,8 +721,6 @@ define void @test_lshr_zero_shift() { ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -790,8 +752,6 @@ define void @test_lshr_power_of_2_start() { ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -823,8 +783,6 @@ define void @test_lshr_arbitrary_start() { ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -855,8 +813,6 @@ define void @test_lshr_start_power_of_2_plus_one() { ; CHECK-NEXT: Loop %loop: backedge-taken count is 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 4 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/sle.ll b/llvm/test/Analysis/ScalarEvolution/sle.ll index 16264d463f61..9779adb3f041 100644 --- a/llvm/test/Analysis/ScalarEvolution/sle.ll +++ b/llvm/test/Analysis/ScalarEvolution/sle.ll @@ -10,8 +10,6 @@ define void @le(i64 %n, ptr nocapture %p) nounwind { ; CHECK-NEXT: Loop %for.body: backedge-taken count is %n ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 9223372036854775807 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is %n -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is %n -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/smax-br-phi-idioms.ll b/llvm/test/Analysis/ScalarEvolution/smax-br-phi-idioms.ll index 2f0e547b03e5..471c255805f3 100644 --- a/llvm/test/Analysis/ScalarEvolution/smax-br-phi-idioms.ll +++ b/llvm/test/Analysis/ScalarEvolution/smax-br-phi-idioms.ll @@ -82,7 +82,6 @@ define i32 @f3(i32 %x, i32 %init, i32 %lim) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -121,7 +120,6 @@ define i32 @f4(i32 %x, i32 %init, i32 %lim) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: %c = icmp sge i32 %init, 0 @@ -161,8 +159,6 @@ define i32 @f5(ptr %val) { ; CHECK-NEXT: Loop %for.end: symbolic max backedge-taken count is false ; CHECK-NEXT: symbolic max exit count for for.end: false ; CHECK-NEXT: symbolic max exit count for for.condt: false -; CHECK-NEXT: Loop %for.end: Predicated backedge-taken count is false -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.end: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/smin-smax-folds.ll b/llvm/test/Analysis/ScalarEvolution/smin-smax-folds.ll index 2de71ee40304..e6872168d5bc 100644 --- a/llvm/test/Analysis/ScalarEvolution/smin-smax-folds.ll +++ b/llvm/test/Analysis/ScalarEvolution/smin-smax-folds.ll @@ -36,8 +36,6 @@ define void @smin_simplify_with_guard(i32 %n) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is %n ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483647 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is %n -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is %n -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -73,8 +71,6 @@ define void @smin_to_smax(i32 %n) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is ((-1 * (0 smin %n)) + %n) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483647 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((-1 * (0 smin %n)) + %n) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-1 * (0 smin %n)) + %n) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -109,8 +105,6 @@ define void @smax_simplify_with_guard(i32 %start, i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 * %start) + %n) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 * %start) + %n) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 * %start) + %n) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/solve-quadratic-i1.ll b/llvm/test/Analysis/ScalarEvolution/solve-quadratic-i1.ll index 0c718fe4f880..5f758e860a10 100644 --- a/llvm/test/Analysis/ScalarEvolution/solve-quadratic-i1.ll +++ b/llvm/test/Analysis/ScalarEvolution/solve-quadratic-i1.ll @@ -20,8 +20,6 @@ define void @f0() { ; CHECK-NEXT: Loop %b1: backedge-taken count is 1 ; CHECK-NEXT: Loop %b1: constant max backedge-taken count is i6 1 ; CHECK-NEXT: Loop %b1: symbolic max backedge-taken count is 1 -; CHECK-NEXT: Loop %b1: Predicated backedge-taken count is 1 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %b1: Trip multiple is 2 ; b0: @@ -67,12 +65,9 @@ define void @f1() #0 { ; CHECK-NEXT: Loop %b3: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %b3: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %b3: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %b3: Unpredictable predicated backedge-taken count. ; CHECK-NEXT: Loop %b1: backedge-taken count is 2 ; CHECK-NEXT: Loop %b1: constant max backedge-taken count is i6 2 ; CHECK-NEXT: Loop %b1: symbolic max backedge-taken count is 2 -; CHECK-NEXT: Loop %b1: Predicated backedge-taken count is 2 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %b1: Trip multiple is 3 ; b0: diff --git a/llvm/test/Analysis/ScalarEvolution/solve-quadratic-overflow.ll b/llvm/test/Analysis/ScalarEvolution/solve-quadratic-overflow.ll index 76c5f13a7992..aac0d618a6a3 100644 --- a/llvm/test/Analysis/ScalarEvolution/solve-quadratic-overflow.ll +++ b/llvm/test/Analysis/ScalarEvolution/solve-quadratic-overflow.ll @@ -26,8 +26,6 @@ define signext i32 @f0() { ; CHECK-NEXT: Loop %b1: backedge-taken count is 255 ; CHECK-NEXT: Loop %b1: constant max backedge-taken count is i16 255 ; CHECK-NEXT: Loop %b1: symbolic max backedge-taken count is 255 -; CHECK-NEXT: Loop %b1: Predicated backedge-taken count is 255 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %b1: Trip multiple is 256 ; b0: diff --git a/llvm/test/Analysis/ScalarEvolution/srem.ll b/llvm/test/Analysis/ScalarEvolution/srem.ll index cc67865f66ce..ff898c963d0d 100644 --- a/llvm/test/Analysis/ScalarEvolution/srem.ll +++ b/llvm/test/Analysis/ScalarEvolution/srem.ll @@ -33,8 +33,6 @@ define dso_local void @_Z4loopi(i32 %width) local_unnamed_addr #0 { ; CHECK-NEXT: Loop %for.cond: backedge-taken count is %width ; CHECK-NEXT: Loop %for.cond: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %for.cond: symbolic max backedge-taken count is %width -; CHECK-NEXT: Loop %for.cond: Predicated backedge-taken count is %width -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.cond: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/symbolic_max_exit_count.ll b/llvm/test/Analysis/ScalarEvolution/symbolic_max_exit_count.ll index da3fd481dd79..1f88c9ec985f 100644 --- a/llvm/test/Analysis/ScalarEvolution/symbolic_max_exit_count.ll +++ b/llvm/test/Analysis/ScalarEvolution/symbolic_max_exit_count.ll @@ -24,7 +24,6 @@ define i32 @test_simple_case(i32 %start, i32 %len) { ; CHECK-NEXT: symbolic max exit count for loop: %start ; CHECK-NEXT: symbolic max exit count for range_check_block: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -83,7 +82,6 @@ define i32 @test_litter_conditions(i32 %start, i32 %len) { ; CHECK-NEXT: symbolic max exit count for loop: %start ; CHECK-NEXT: symbolic max exit count for range_check_block: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -146,7 +144,6 @@ define i32 @test_litter_conditions_bad_context(i32 %start, i32 %len) { ; CHECK-NEXT: symbolic max exit count for loop: %start ; CHECK-NEXT: symbolic max exit count for range_check_block: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -201,7 +198,6 @@ define i32 @test_and_conditions(i32 %start, i32 %len) { ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %start ; CHECK-NEXT: symbolic max exit count for loop: %start ; CHECK-NEXT: symbolic max exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -245,7 +241,6 @@ define i32 @test_mixup_constant_symbolic(i32 %end, i32 %len) { ; CHECK-NEXT: symbolic max exit count for loop: %end ; CHECK-NEXT: symbolic max exit count for range_check_block: 1000 ; CHECK-NEXT: symbolic max exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -293,7 +288,6 @@ define i32 @test_mixup_constant_symbolic_merged(i32 %end, i32 %len) { ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (1000 umin %end) ; CHECK-NEXT: symbolic max exit count for loop: (1000 umin %end) ; CHECK-NEXT: symbolic max exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -352,7 +346,6 @@ define i32 @test_two_phis(i32 %start_1, i32 %start_2, i32 %len) { ; CHECK-NEXT: symbolic max exit count for zero_check_block: %start_2 ; CHECK-NEXT: symbolic max exit count for range_check_block: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -411,8 +404,6 @@ define i32 @test_two_phis_simple(i32 %start_1, i32 %start_2, i32 %len) { ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (%start_1 umin_seq %start_2) ; CHECK-NEXT: symbolic max exit count for loop: %start_1 ; CHECK-NEXT: symbolic max exit count for backedge: %start_2 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (%start_1 umin_seq %start_2) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -469,7 +460,6 @@ define i32 @test_two_phis_arithmetic_and(i32 %start_1, i32 %start_2, i32 %len) { ; CHECK-NEXT: symbolic max exit count for loop: (%start_1 umin %start_2) ; CHECK-NEXT: symbolic max exit count for range_check_block: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -543,7 +533,6 @@ define i32 @test_two_phis_logical_or(i32 %start_1, i32 %start_2, i32 %len) { ; CHECK-NEXT: symbolic max exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for range_check_block: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -616,7 +605,6 @@ define i32 @test_two_phis_logical_and(i32 %start_1, i32 %start_2, i32 %len) { ; CHECK-NEXT: symbolic max exit count for loop: (%start_1 umin_seq %start_2) ; CHECK-NEXT: symbolic max exit count for range_check_block: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for backedge: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-andor-selectform.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-andor-selectform.ll index 4527211f3ab9..6b2251c92070 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-andor-selectform.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-andor-selectform.ll @@ -10,8 +10,6 @@ define void @unsimplified_and1(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is %n ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -34,8 +32,6 @@ define void @unsimplified_and2(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is %n ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -58,8 +54,6 @@ define void @unsimplified_and3(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is false ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is false -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -82,8 +76,6 @@ define void @unsimplified_and4(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is false ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is false -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -106,7 +98,6 @@ define void @unsimplified_or1(i32 %n) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -128,7 +119,6 @@ define void @unsimplified_or2(i32 %n) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -150,8 +140,6 @@ define void @unsimplified_or3(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is %n ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -174,8 +162,6 @@ define void @unsimplified_or4(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is %n ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -198,8 +184,6 @@ define void @reversed_and1(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is %n ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -222,8 +206,6 @@ define void @reversed_and2(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is %n ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -246,7 +228,6 @@ define void @reversed_and3(i32 %n) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -268,7 +249,6 @@ define void @reversed_and4(i32 %n) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -290,8 +270,6 @@ define void @reversed_or1(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is false ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is false -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -314,8 +292,6 @@ define void @reversed_or2(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is false ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is false -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -338,8 +314,6 @@ define void @reversed_or3(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is %n ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -362,8 +336,6 @@ define void @reversed_or4(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is %n ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-andor.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-andor.ll index aa1fad42b6fb..a7a2929b72d1 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-andor.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-andor.ll @@ -10,8 +10,6 @@ define void @unsimplified_and1(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is %n ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -34,8 +32,6 @@ define void @unsimplified_and2(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is %n ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -58,8 +54,6 @@ define void @unsimplified_and3(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is false ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is false -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -82,8 +76,6 @@ define void @unsimplified_and4(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is false ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is false -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -106,7 +98,6 @@ define void @unsimplified_or1(i32 %n) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -128,7 +119,6 @@ define void @unsimplified_or2(i32 %n) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -150,8 +140,6 @@ define void @unsimplified_or3(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is %n ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -174,8 +162,6 @@ define void @unsimplified_or4(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is %n ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -198,8 +184,6 @@ define void @reversed_and1(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is %n ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -222,8 +206,6 @@ define void @reversed_and2(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is %n ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -246,7 +228,6 @@ define void @reversed_and3(i32 %n) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -268,7 +249,6 @@ define void @reversed_and4(i32 %n) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop @@ -290,8 +270,6 @@ define void @reversed_or1(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is false ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is false -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -314,8 +292,6 @@ define void @reversed_or2(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is false ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is false -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -338,8 +314,6 @@ define void @reversed_or3(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is %n ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -362,8 +336,6 @@ define void @reversed_or4(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is %n ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is %n -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is %n -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-implied-addrec.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-implied-addrec.ll index e67c7dff14c2..114e7f3baa89 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-implied-addrec.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-implied-addrec.ll @@ -15,8 +15,6 @@ define void @nw_implies_nuw(i16 %n) mustprogress { ; CHECK-NEXT: Loop %for.body: backedge-taken count is %n ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is %n -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is %n -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -39,7 +37,6 @@ define void @neg_nw_nuw(i16 %n) mustprogress { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -85,7 +82,6 @@ define void @neg_nw_nsw(i16 %n) mustprogress { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -160,7 +156,6 @@ define void @rhs_mustexit_3(i16 %n.raw) mustprogress { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: %n.and = and i16 %n.raw, 255 @@ -186,7 +181,6 @@ define void @rhs_mustexit_nonzero_step(i16 %n.raw, i8 %step.raw) mustprogress { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: %n.and = and i16 %n.raw, 255 @@ -212,7 +206,6 @@ define void @neg_maybe_zero_step(i16 %n.raw, i8 %step) mustprogress { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: %n.and = and i16 %n.raw, 255 @@ -293,8 +286,6 @@ define void @rhs_narrow_range(i16 %n.raw) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (1 umax (2 * (zext i7 (trunc i16 (%n.raw /u 2) to i7) to i16)))) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 253 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (1 umax (2 * (zext i7 (trunc i16 (%n.raw /u 2) to i7) to i16)))) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (1 umax (2 * (zext i7 (trunc i16 (%n.raw /u 2) to i7) to i16)))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -320,7 +311,6 @@ define void @ugt_constant_rhs(i16 %n.raw, i8 %start) mustprogress { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -343,8 +333,6 @@ define void @ult_constant_rhs(i16 %n.raw, i8 %start) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (255 + (-1 * (zext i8 (1 + %start) to i16))) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 255 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (255 + (-1 * (zext i8 (1 + %start) to i16))) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (255 + (-1 * (zext i8 (1 + %start) to i16))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -368,8 +356,6 @@ define void @ult_constant_rhs_stride2(i16 %n.raw, i8 %start) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is ((1 + (-1 * (zext i8 (2 + %start) to i16)) + (254 umax (zext i8 (2 + %start) to i16))) /u 2) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 127 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((1 + (-1 * (zext i8 (2 + %start) to i16)) + (254 umax (zext i8 (2 + %start) to i16))) /u 2) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((1 + (-1 * (zext i8 (2 + %start) to i16)) + (254 umax (zext i8 (2 + %start) to i16))) /u 2) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -418,8 +404,6 @@ define void @ult_restricted_rhs(i16 %n.raw) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (1 umax (zext i8 (trunc i16 %n.raw to i8) to i16))) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 254 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (1 umax (zext i8 (trunc i16 %n.raw to i8) to i16))) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (1 umax (zext i8 (trunc i16 %n.raw to i8) to i16))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -443,8 +427,6 @@ define void @ult_guarded_rhs(i16 %n) {; ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (1 umax %n)) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (1 umax %n)) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (1 umax %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-minmax.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-minmax.ll index f9d490d5ee38..8d091a00ed4b 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-minmax.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-minmax.ll @@ -17,8 +17,6 @@ define void @nomulitply(i32 noundef %a, i32 noundef %b) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (%a umin %b)) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (%a umin %b)) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + (%a umin %b)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; ; No information about a or b. Trip multiple is 1. @@ -63,8 +61,6 @@ define void @umin(i32 noundef %a, i32 noundef %b) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + ((2 * %a) umin (4 * %b))) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + ((2 * %a) umin (4 * %b))) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + ((2 * %a) umin (4 * %b))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; ; void umin(unsigned a, unsigned b) { @@ -113,8 +109,6 @@ define void @umax(i32 noundef %a, i32 noundef %b) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + ((2 * %a) umax (4 * %b))) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -3 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + ((2 * %a) umax (4 * %b))) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + ((2 * %a) umax (4 * %b))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 2 ; @@ -163,8 +157,6 @@ define void @smin(i32 noundef %a, i32 noundef %b) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + ((2 * %a) smin (4 * %b))) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + ((2 * %a) smin (4 * %b))) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + ((2 * %a) smin (4 * %b))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; ; void smin(signed a, signed b) { @@ -212,8 +204,6 @@ define void @smax(i32 noundef %a, i32 noundef %b) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + ((2 * %a) smax (4 * %b))) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -3 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + ((2 * %a) smax (4 * %b))) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + ((2 * %a) smax (4 * %b))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 2 ; ; void smax(signed a, signed b) { @@ -261,8 +251,6 @@ define void @umin_seq2(i32 %n, i32 %m) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 + (1 umax (2 * %n))) umin_seq (-1 + (1 umax (16 * %m)))) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -17 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 + (1 umax (2 * %n))) umin_seq (-1 + (1 umax (16 * %m)))) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 + (1 umax (2 * %n))) umin_seq (-1 + (1 umax (16 * %m)))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; ; Can't find that trip multiple is 2 for this case of umin_seq @@ -299,8 +287,6 @@ define void @umin-3and6(i32 noundef %a, i32 noundef %b) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + ((3 * %a) umin (6 * %b))) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + ((3 * %a) umin (6 * %b))) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + ((3 * %a) umin (6 * %b))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; ; Trip multiple is 1 because we use GetMinTrailingZeros() to compute trip multiples. diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-negative-stride.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-negative-stride.ll index 3cd0904fc712..60120ed2f8eb 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-negative-stride.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-negative-stride.ll @@ -15,7 +15,6 @@ define void @ult_wrap() { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -38,7 +37,6 @@ define void @ult_infinite() { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -61,8 +59,6 @@ define void @ult_infinite_ub() mustprogress { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 1 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 1 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 1 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 2 ; entry: @@ -86,8 +82,6 @@ define void @ult_129_not_taken() { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 0 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 0 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 0 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 0 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -109,8 +103,6 @@ define void @ult_129_unknown_start(i8 %start) mustprogress { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 0 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 0 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 0 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 0 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -134,7 +126,6 @@ define void @ult_not_taken(i8 %step) { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: %assume = icmp ult i8 128, %step @@ -161,8 +152,6 @@ define void @ult_ub1() { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 2 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 2 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 2 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 3 ; entry: @@ -186,8 +175,6 @@ define void @ult_ub2() { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 0 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 0 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 0 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 0 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -211,8 +198,6 @@ define void @ult_129_preinc() { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 1 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 1 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 1 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 2 ; entry: @@ -234,8 +219,6 @@ define void @ult_preinc(i8 %step) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 1 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 1 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 1 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 2 ; entry: @@ -259,7 +242,6 @@ define void @ult_129_varying_rhs(ptr %n_p) { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -281,7 +263,6 @@ define void @ult_symbolic_varying_rhs(ptr %n_p, i8 %step) { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: %assume = icmp ult i8 128, %step @@ -311,8 +292,6 @@ define void @slt_wrap() { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 63 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 63 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 63 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 63 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 64 ; entry: @@ -336,7 +315,6 @@ define void @slt_infinite() { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -359,8 +337,6 @@ define void @slt_infinite_ub() mustprogress { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 0 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 0 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 0 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 0 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -384,8 +360,6 @@ define void @slt_129_not_taken() { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 0 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 0 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 0 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 0 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -408,7 +382,6 @@ define void @slt_not_taken(i8 %step) { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: %assume = icmp ult i8 128, %step @@ -431,8 +404,6 @@ define void @slt_129_unknown_start(i8 %start) mustprogress { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (((127 + (-1 * (1 umin (127 + (-1 * %start) + (0 smax (-127 + %start))))) + (-1 * %start) + (0 smax (-127 + %start))) /u -127) + (1 umin (127 + (-1 * %start) + (0 smax (-127 + %start))))) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (((127 + (-1 * (1 umin (127 + (-1 * %start) + (0 smax (-127 + %start))))) + (-1 * %start) + (0 smax (-127 + %start))) /u -127) + (1 umin (127 + (-1 * %start) + (0 smax (-127 + %start))))) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (((127 + (-1 * (1 umin (127 + (-1 * %start) + (0 smax (-127 + %start))))) + (-1 * %start) + (0 smax (-127 + %start))) /u -127) + (1 umin (127 + (-1 * %start) + (0 smax (-127 + %start))))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -457,8 +428,6 @@ define void @slt_ub1() { ; CHECK-NEXT: Loop %for.body: backedge-taken count is false ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is false -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is false -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -482,8 +451,6 @@ define void @slt_ub2() { ; CHECK-NEXT: Loop %for.body: backedge-taken count is false ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is false -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is false -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -507,8 +474,6 @@ define void @slt_129_preinc() { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 1 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 1 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 1 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 2 ; entry: @@ -530,8 +495,6 @@ define void @slt_preinc(i8 %step) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is 1 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 1 -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 1 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 2 ; entry: @@ -555,7 +518,6 @@ define void @slt_129_varying_rhs(ptr %n_p) { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -577,7 +539,6 @@ define void @slt_symbolic_varying_rhs(ptr %n_p, i8 %step) { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: %assume = icmp ult i8 128, %step @@ -604,14 +565,10 @@ define void @step_is_neg_addrec_slt_8(i64 %n) { ; CHECK-NEXT: Loop %inner: backedge-taken count is (7 /u {0,+,-1}<%outer.header>) ; CHECK-NEXT: Loop %inner: constant max backedge-taken count is i32 8 ; CHECK-NEXT: Loop %inner: symbolic max backedge-taken count is (7 /u {0,+,-1}<%outer.header>) -; CHECK-NEXT: Loop %inner: Predicated backedge-taken count is (7 /u {0,+,-1}<%outer.header>) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %inner: Trip multiple is 1 ; CHECK-NEXT: Loop %outer.header: backedge-taken count is 0 ; CHECK-NEXT: Loop %outer.header: constant max backedge-taken count is i64 0 ; CHECK-NEXT: Loop %outer.header: symbolic max backedge-taken count is 0 -; CHECK-NEXT: Loop %outer.header: Predicated backedge-taken count is 0 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %outer.header: Trip multiple is 1 ; entry: @@ -646,14 +603,10 @@ define void @step_is_neg_addrec_slt_var(i32 %n) { ; CHECK-NEXT: Loop %inner: backedge-taken count is ({0,+,1}<%outer.header> + ({0,+,-1}<%outer.header> smax %n)) ; CHECK-NEXT: Loop %inner: constant max backedge-taken count is i32 2147483647 ; CHECK-NEXT: Loop %inner: symbolic max backedge-taken count is ({0,+,1}<%outer.header> + ({0,+,-1}<%outer.header> smax %n)) -; CHECK-NEXT: Loop %inner: Predicated backedge-taken count is ({0,+,1}<%outer.header> + ({0,+,-1}<%outer.header> smax %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %inner: Trip multiple is 1 ; CHECK-NEXT: Loop %outer.header: backedge-taken count is 0 ; CHECK-NEXT: Loop %outer.header: constant max backedge-taken count is i64 0 ; CHECK-NEXT: Loop %outer.header: symbolic max backedge-taken count is 0 -; CHECK-NEXT: Loop %outer.header: Predicated backedge-taken count is 0 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %outer.header: Trip multiple is 1 ; entry: @@ -688,14 +641,10 @@ define void @step_is_neg_addrec_unknown_start(i32 %n) { ; CHECK-NEXT: Loop %inner: backedge-taken count is ({(-1 * %n),+,1}<%outer.header> + (8 smax {%n,+,-1}<%outer.header>)) ; CHECK-NEXT: Loop %inner: constant max backedge-taken count is i32 -2147483640 ; CHECK-NEXT: Loop %inner: symbolic max backedge-taken count is ({(-1 * %n),+,1}<%outer.header> + (8 smax {%n,+,-1}<%outer.header>)) -; CHECK-NEXT: Loop %inner: Predicated backedge-taken count is ({(-1 * %n),+,1}<%outer.header> + (8 smax {%n,+,-1}<%outer.header>)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %inner: Trip multiple is 1 ; CHECK-NEXT: Loop %outer.header: backedge-taken count is 0 ; CHECK-NEXT: Loop %outer.header: constant max backedge-taken count is i64 0 ; CHECK-NEXT: Loop %outer.header: symbolic max backedge-taken count is 0 -; CHECK-NEXT: Loop %outer.header: Predicated backedge-taken count is 0 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %outer.header: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-non-unit-stride.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-non-unit-stride.ll index 6c79d632965d..b52b4a984296 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-non-unit-stride.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-non-unit-stride.ll @@ -19,7 +19,6 @@ define void @test_preinc_ult(i64 %len) { ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. ; CHECK-NEXT: symbolic max exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for latch: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; start: br label %loop @@ -55,7 +54,6 @@ define void @test_postinc_ult(i64 %len) { ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. ; CHECK-NEXT: symbolic max exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for latch: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; start: br label %loop @@ -90,7 +88,6 @@ define void @test_preinc_slt(i64 %len) { ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. ; CHECK-NEXT: symbolic max exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for latch: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; start: br label %loop @@ -126,7 +123,6 @@ define void @test_postinc_slt(i64 %len) { ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. ; CHECK-NEXT: symbolic max exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for latch: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; start: br label %loop @@ -161,7 +157,6 @@ define void @test_preinc_sgt(i64 %lim) { ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. ; CHECK-NEXT: symbolic max exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for latch: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; start: br label %loop @@ -197,7 +192,6 @@ define void @test_postinc_sgt(i64 %lim) { ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. ; CHECK-NEXT: symbolic max exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for latch: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; start: br label %loop diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-pow2.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-pow2.ll index 8feb7a87b34b..fb225376d38b 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-pow2.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-pow2.ll @@ -7,8 +7,6 @@ define void @test1(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-32 + (96 * %n)) /u 32) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 134217727 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-32 + (96 * %n)) /u 32) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-32 + (96 * %n)) /u 32) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -31,8 +29,6 @@ define i32 @test2(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-32 + (32 * (%n /u 32))) /u 32) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 134217727 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-32 + (32 * (%n /u 32))) /u 32) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-32 + (32 * (%n /u 32))) /u 32) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -54,8 +50,6 @@ define void @test3(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-32 + (32 * %n)) /u 32) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 134217727 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-32 + (32 * %n)) /u 32) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-32 + (32 * %n)) /u 32) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -77,8 +71,6 @@ define void @test4(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (-1431655764 * %n)) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 1073741823 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (-1431655764 * %n)) /u 4) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (-1431655764 * %n)) /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -100,8 +92,6 @@ define void @test5(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * %n)) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 1073741823 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (4 * %n)) /u 4) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (4 * %n)) /u 4) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-switch.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-switch.ll index b315deb5a87c..df8bb0ae043a 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-switch.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-switch.ll @@ -10,8 +10,6 @@ define void @test1() nounwind { ; CHECK-NEXT: Loop %for.cond: backedge-taken count is 2 ; CHECK-NEXT: Loop %for.cond: constant max backedge-taken count is i32 2 ; CHECK-NEXT: Loop %for.cond: symbolic max backedge-taken count is 2 -; CHECK-NEXT: Loop %for.cond: Predicated backedge-taken count is 2 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.cond: Trip multiple is 3 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-unknown-stride.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-unknown-stride.ll index ef7084130335..eb55e6abc95e 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-unknown-stride.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-unknown-stride.ll @@ -17,8 +17,6 @@ define void @foo1(ptr nocapture %A, i32 %n, i32 %s) mustprogress { ; CHECK-NEXT: Loop %for.body: backedge-taken count is ((-1 + (%n smax %s)) /u %s) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((-1 + (%n smax %s)) /u %s) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-1 + (%n smax %s)) /u %s) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -52,8 +50,6 @@ define void @foo2(ptr nocapture %A, i32 %n, i32 %s) mustprogress { ; CHECK-NEXT: Loop %for.body: backedge-taken count is ((((-1 * (1 umin ((-1 * %s) + (%n smax %s)))) + (-1 * %s) + (%n smax %s)) /u (1 umax %s)) + (1 umin ((-1 * %s) + (%n smax %s)))) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((((-1 * (1 umin ((-1 * %s) + (%n smax %s)))) + (-1 * %s) + (%n smax %s)) /u (1 umax %s)) + (1 umin ((-1 * %s) + (%n smax %s)))) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((((-1 * (1 umin ((-1 * %s) + (%n smax %s)))) + (-1 * %s) + (%n smax %s)) /u (1 umax %s)) + (1 umin ((-1 * %s) + (%n smax %s)))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -83,7 +79,6 @@ define void @foo3(ptr nocapture %A, i32 %n, i32 %s) { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -111,8 +106,6 @@ define void @foo4(ptr nocapture %A, i32 %n, i32 %s) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is ((((-1 * (1 umin ((-1 * %s) + (%n smax %s)))) + (-1 * %s) + (%n smax %s)) /u (1 umax %s)) + (1 umin ((-1 * %s) + (%n smax %s)))) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((((-1 * (1 umin ((-1 * %s) + (%n smax %s)))) + (-1 * %s) + (%n smax %s)) /u (1 umax %s)) + (1 umin ((-1 * %s) + (%n smax %s)))) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((((-1 * (1 umin ((-1 * %s) + (%n smax %s)))) + (-1 * %s) + (%n smax %s)) /u (1 umax %s)) + (1 umin ((-1 * %s) + (%n smax %s)))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -144,8 +137,6 @@ define void @foo5(ptr nocapture %A, i32 %n, i32 %s, i32 %start) mustprogress { ; CHECK-NEXT: Loop %for.body: backedge-taken count is ((((-1 * (1 umin ((-1 * %start) + (%n smax %start)))) + (-1 * %start) + (%n smax %start)) /u (1 umax %s)) + (1 umin ((-1 * %start) + (%n smax %start)))) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((((-1 * (1 umin ((-1 * %start) + (%n smax %start)))) + (-1 * %start) + (%n smax %start)) /u (1 umax %s)) + (1 umin ((-1 * %start) + (%n smax %start)))) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((((-1 * (1 umin ((-1 * %start) + (%n smax %start)))) + (-1 * %start) + (%n smax %start)) /u (1 umax %s)) + (1 umin ((-1 * %start) + (%n smax %start)))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -175,7 +166,6 @@ define void @zero_stride(ptr nocapture %A, i32 %n) { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -203,7 +193,6 @@ define void @zero_stride_ub(ptr nocapture %A) { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body @@ -231,8 +220,6 @@ define void @zero_stride_symbolic(ptr nocapture %A, i32 %n, i32 %zero) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is ((((-1 * (1 umin ((-1 * %zero) + (%n smax %zero)))) + (-1 * %zero) + (%n smax %zero)) /u (1 umax %zero)) + (1 umin ((-1 * %zero) + (%n smax %zero)))) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((((-1 * (1 umin ((-1 * %zero) + (%n smax %zero)))) + (-1 * %zero) + (%n smax %zero)) /u (1 umax %zero)) + (1 umin ((-1 * %zero) + (%n smax %zero)))) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((((-1 * (1 umin ((-1 * %zero) + (%n smax %zero)))) + (-1 * %zero) + (%n smax %zero)) /u (1 umax %zero)) + (1 umin ((-1 * %zero) + (%n smax %zero)))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -261,7 +248,6 @@ define void @zero_stride_varying_rhs(ptr nocapture %A, ptr %n_p, i32 %zero) { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: br label %for.body diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count.ll b/llvm/test/Analysis/ScalarEvolution/trip-count.ll index 9f34c43b907c..8f3e82984431 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count.ll @@ -12,8 +12,6 @@ define void @PR1101(i32 %N) { ; CHECK-NEXT: Loop %bb3: backedge-taken count is 10000 ; CHECK-NEXT: Loop %bb3: constant max backedge-taken count is i32 10000 ; CHECK-NEXT: Loop %bb3: symbolic max backedge-taken count is 10000 -; CHECK-NEXT: Loop %bb3: Predicated backedge-taken count is 10000 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %bb3: Trip multiple is 10001 ; entry: @@ -43,8 +41,6 @@ define i32 @PR22795() { ; CHECK-NEXT: Loop %preheader: backedge-taken count is 7 ; CHECK-NEXT: Loop %preheader: constant max backedge-taken count is i64 7 ; CHECK-NEXT: Loop %preheader: symbolic max backedge-taken count is 7 -; CHECK-NEXT: Loop %preheader: Predicated backedge-taken count is 7 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %preheader: Trip multiple is 8 ; entry: @@ -106,8 +102,6 @@ define void @pr28012(i32 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is -1431655751 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1431655751 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is -1431655751 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is -1431655751 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 2863311546 ; entry: @@ -130,8 +124,6 @@ define void @non_zero_from_loop_guard(i16 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (%n /u 2)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i16 32766 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (%n /u 2)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (%n /u 2)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count10.ll b/llvm/test/Analysis/ScalarEvolution/trip-count10.ll index acce7a81e0ab..664ca3a1b3d3 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count10.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count10.ll @@ -9,7 +9,6 @@ define void @a(i64 %n) nounwind { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: %t0 = icmp sgt i64 %n, 0 @@ -30,8 +29,6 @@ define void @b(i64 %n) nounwind { ; CHECK-NEXT: Loop %loop: backedge-taken count is false ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is false -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -53,8 +50,6 @@ define void @c(i64 %n) nounwind { ; CHECK-NEXT: Loop %loop: backedge-taken count is false ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is false -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -76,7 +71,6 @@ define void @d(i64 %n) nounwind { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: %t0 = icmp sgt i64 %n, 0 @@ -105,7 +99,6 @@ define void @nonpolynomial() { ; CHECK-NEXT: Loop %loophead: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loophead: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loophead: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loophead: Unpredictable predicated backedge-taken count. ; entry: br label %loophead @@ -134,8 +127,6 @@ define void @constant_phi_operands() nounwind { ; CHECK-NEXT: Loop %loop: backedge-taken count is 1 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 1 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 1 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 2 ; entry: @@ -160,7 +151,6 @@ define void @exit_orcond_nsw(ptr %a) nounwind { ; CHECK-NEXT: Loop %for.body.i: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body.i: constant max backedge-taken count is i32 1 ; CHECK-NEXT: Loop %for.body.i: symbolic max backedge-taken count is 1 -; CHECK-NEXT: Loop %for.body.i: Unpredictable predicated backedge-taken count. ; entry: br label %for.body.i diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count12.ll b/llvm/test/Analysis/ScalarEvolution/trip-count12.ll index 0eb903d64b2a..a7ea2ca654a2 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count12.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count12.ll @@ -9,8 +9,6 @@ define zeroext i16 @test(ptr nocapture %p, i32 %len) nounwind readonly { ; CHECK-NEXT: Loop %for.body: backedge-taken count is ((-2 + %len) /u 2) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 1073741823 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is ((-2 + %len) /u 2) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is ((-2 + %len) /u 2) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count13.ll b/llvm/test/Analysis/ScalarEvolution/trip-count13.ll index ef8640e65968..720b2ae49c09 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count13.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count13.ll @@ -9,8 +9,6 @@ define void @u_0(i8 %rhs) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-100 + (-1 * %rhs) + ((100 + %rhs) umax %rhs)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 -100, actual taken count either this or zero. ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-100 + (-1 * %rhs) + ((100 + %rhs) umax %rhs)), actual taken count either this or zero. -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-100 + (-1 * %rhs) + ((100 + %rhs) umax %rhs)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -35,8 +33,6 @@ define void @u_1(i8 %start) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 * %start) + ((-100 + %start) umax %start)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 -100, actual taken count either this or zero. ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 * %start) + ((-100 + %start) umax %start)), actual taken count either this or zero. -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 * %start) + ((-100 + %start) umax %start)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -62,8 +58,6 @@ define void @s_0(i8 %rhs) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-100 + (-1 * %rhs) + ((100 + %rhs) smax %rhs)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 -100, actual taken count either this or zero. ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-100 + (-1 * %rhs) + ((100 + %rhs) smax %rhs)), actual taken count either this or zero. -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-100 + (-1 * %rhs) + ((100 + %rhs) smax %rhs)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -89,8 +83,6 @@ define void @s_1(i8 %start) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 * %start) + ((-100 + %start) smax %start)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 -100, actual taken count either this or zero. ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 * %start) + ((-100 + %start) smax %start)), actual taken count either this or zero. -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 * %start) + ((-100 + %start) smax %start)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -116,8 +108,6 @@ define void @s_2(i8 %start) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 * ((-100 + %start) smin %start)) + %start) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 * ((-100 + %start) smin %start)) + %start) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 * ((-100 + %start) smin %start)) + %start) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count14.ll b/llvm/test/Analysis/ScalarEvolution/trip-count14.ll index 6561e080bee1..8d87250a7477 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count14.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count14.ll @@ -7,8 +7,6 @@ define void @s32_max1(i32 %n, ptr %p) { ; CHECK-NEXT: Loop %do.body: backedge-taken count is ((-1 * %n) + ((1 + %n) smax %n)) ; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is i32 1, actual taken count either this or zero. ; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is ((-1 * %n) + ((1 + %n) smax %n)), actual taken count either this or zero. -; CHECK-NEXT: Loop %do.body: Predicated backedge-taken count is ((-1 * %n) + ((1 + %n) smax %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %do.body: Trip multiple is 1 ; entry: @@ -33,8 +31,6 @@ define void @s32_max2(i32 %n, ptr %p) { ; CHECK-NEXT: Loop %do.body: backedge-taken count is ((-1 * %n) + ((2 + %n) smax %n)) ; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is i32 2, actual taken count either this or zero. ; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is ((-1 * %n) + ((2 + %n) smax %n)), actual taken count either this or zero. -; CHECK-NEXT: Loop %do.body: Predicated backedge-taken count is ((-1 * %n) + ((2 + %n) smax %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %do.body: Trip multiple is 1 ; entry: @@ -59,8 +55,6 @@ define void @s32_maxx(i32 %n, i32 %x, ptr %p) { ; CHECK-NEXT: Loop %do.body: backedge-taken count is ((-1 * %n) + ((%n + %x) smax %n)) ; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is ((-1 * %n) + ((%n + %x) smax %n)) -; CHECK-NEXT: Loop %do.body: Predicated backedge-taken count is ((-1 * %n) + ((%n + %x) smax %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %do.body: Trip multiple is 1 ; entry: @@ -89,8 +83,6 @@ define void @s32_max2_unpredictable_exit(i32 %n, i32 %x, ptr %p) { ; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is (((-1 * %n) + ((2 + %n) smax %n)) umin ((-1 * %n) + %x)) ; CHECK-NEXT: symbolic max exit count for do.body: ((-1 * %n) + %x) ; CHECK-NEXT: symbolic max exit count for if.end: ((-1 * %n) + ((2 + %n) smax %n)) -; CHECK-NEXT: Loop %do.body: Predicated backedge-taken count is (((-1 * %n) + ((2 + %n) smax %n)) umin ((-1 * %n) + %x)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %do.body: Trip multiple is 1 ; entry: @@ -119,8 +111,6 @@ define void @u32_max1(i32 %n, ptr %p) { ; CHECK-NEXT: Loop %do.body: backedge-taken count is ((-1 * %n) + ((1 + %n) umax %n)) ; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is i32 1, actual taken count either this or zero. ; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is ((-1 * %n) + ((1 + %n) umax %n)), actual taken count either this or zero. -; CHECK-NEXT: Loop %do.body: Predicated backedge-taken count is ((-1 * %n) + ((1 + %n) umax %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %do.body: Trip multiple is 1 ; entry: @@ -145,8 +135,6 @@ define void @u32_max2(i32 %n, ptr %p) { ; CHECK-NEXT: Loop %do.body: backedge-taken count is ((-1 * %n) + ((2 + %n) umax %n)) ; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is i32 2, actual taken count either this or zero. ; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is ((-1 * %n) + ((2 + %n) umax %n)), actual taken count either this or zero. -; CHECK-NEXT: Loop %do.body: Predicated backedge-taken count is ((-1 * %n) + ((2 + %n) umax %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %do.body: Trip multiple is 1 ; entry: @@ -171,8 +159,6 @@ define void @u32_maxx(i32 %n, i32 %x, ptr %p) { ; CHECK-NEXT: Loop %do.body: backedge-taken count is ((-1 * %n) + ((%n + %x) umax %n)) ; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is ((-1 * %n) + ((%n + %x) umax %n)) -; CHECK-NEXT: Loop %do.body: Predicated backedge-taken count is ((-1 * %n) + ((%n + %x) umax %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %do.body: Trip multiple is 1 ; entry: @@ -201,8 +187,6 @@ define void @u32_max2_unpredictable_exit(i32 %n, i32 %x, ptr %p) { ; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is (((-1 * %n) + ((2 + %n) umax %n)) umin ((-1 * %n) + %x)) ; CHECK-NEXT: symbolic max exit count for do.body: ((-1 * %n) + %x) ; CHECK-NEXT: symbolic max exit count for if.end: ((-1 * %n) + ((2 + %n) umax %n)) -; CHECK-NEXT: Loop %do.body: Predicated backedge-taken count is (((-1 * %n) + ((2 + %n) umax %n)) umin ((-1 * %n) + %x)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %do.body: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count15.ll b/llvm/test/Analysis/ScalarEvolution/trip-count15.ll index 3b379510701f..ad96576b12f4 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count15.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count15.ll @@ -14,8 +14,6 @@ define void @umin_unsigned_check(i64 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (1 + (4096 umin %n)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4097 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (1 + (4096 umin %n)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (1 + (4096 umin %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -45,8 +43,6 @@ define void @umin_signed_check(i64 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (1 + (4096 umin %n)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4097 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (1 + (4096 umin %n)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (1 + (4096 umin %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -76,8 +72,6 @@ define void @smin_signed_check(i64 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (0 smax (1 + (4096 smin %n))) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4097 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (0 smax (1 + (4096 smin %n))) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (0 smax (1 + (4096 smin %n))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -107,7 +101,6 @@ define void @smin_unsigned_check(i64 %n) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: %min.cmp = icmp slt i64 4096, %n diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count2.ll b/llvm/test/Analysis/ScalarEvolution/trip-count2.ll index 247385dad6dc..ffbdcf7376b2 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count2.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count2.ll @@ -9,8 +9,6 @@ define void @PR1101(i32 %N) { ; CHECK-NEXT: Loop %bb3: backedge-taken count is 4 ; CHECK-NEXT: Loop %bb3: constant max backedge-taken count is i32 4 ; CHECK-NEXT: Loop %bb3: symbolic max backedge-taken count is 4 -; CHECK-NEXT: Loop %bb3: Predicated backedge-taken count is 4 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %bb3: Trip multiple is 5 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count3.ll b/llvm/test/Analysis/ScalarEvolution/trip-count3.ll index de5b9ecf9013..33ac837caefc 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count3.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count3.ll @@ -42,8 +42,6 @@ define void @sha_stream_bb3_2E_i(ptr %sha_info, ptr %data1, i32, ptr %buffer_add ; CHECK-NEXT: Loop %bb3.i: backedge-taken count is ((63 + (-1 * (63 smin %0)) + %0) /u 64) ; CHECK-NEXT: Loop %bb3.i: constant max backedge-taken count is i32 33554431 ; CHECK-NEXT: Loop %bb3.i: symbolic max backedge-taken count is ((63 + (-1 * (63 smin %0)) + %0) /u 64) -; CHECK-NEXT: Loop %bb3.i: Predicated backedge-taken count is ((63 + (-1 * (63 smin %0)) + %0) /u 64) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %bb3.i: Trip multiple is 1 ; newFuncRoot: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count4.ll b/llvm/test/Analysis/ScalarEvolution/trip-count4.ll index c289557d6fb8..0de2a3d9ff3e 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count4.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count4.ll @@ -9,8 +9,6 @@ define void @another_count_down_signed(ptr %d, i64 %n) nounwind { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-11 + %n) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-11 + %n) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-11 + %n) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count6.ll b/llvm/test/Analysis/ScalarEvolution/trip-count6.ll index a0bed8fdb0a1..e4ded1d14b65 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count6.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count6.ll @@ -13,7 +13,6 @@ define i8 @f() { ; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is 1 ; CHECK-NEXT: symbolic max exit count for bb: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for bb2: 1 -; CHECK-NEXT: Loop %bb: Unpredictable predicated backedge-taken count. ; entry: tail call i32 @fegetround( ) ; :0 [#uses=1] diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count7.ll b/llvm/test/Analysis/ScalarEvolution/trip-count7.ll index b143b252f27a..3dca3efa25b4 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count7.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count7.ll @@ -66,7 +66,6 @@ define void @Doit_bb7_2E_i(ptr %x1, ptr %c, ptr %b, ptr %a, ptr %q, ptr %x1.sub, ; CHECK-NEXT: Loop %bb7.i: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %bb7.i: constant max backedge-taken count is i32 8 ; CHECK-NEXT: Loop %bb7.i: symbolic max backedge-taken count is 8 -; CHECK-NEXT: Loop %bb7.i: Unpredictable predicated backedge-taken count. ; newFuncRoot: br label %bb7.i diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count8.ll b/llvm/test/Analysis/ScalarEvolution/trip-count8.ll index 69dec1c5d3c8..ca189d197724 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count8.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count8.ll @@ -10,8 +10,6 @@ define i32 @foo(i32 %ecx) nounwind { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %ecx) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %ecx) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %ecx) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count9.ll b/llvm/test/Analysis/ScalarEvolution/trip-count9.ll index 07fdb508be99..f471de330fc6 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count9.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count9.ll @@ -15,8 +15,6 @@ define void @foo(i4 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %n) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 6 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %n) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %n) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -37,7 +35,6 @@ define void @step2(i4 %n) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: %s = icmp sgt i4 %n, 0 @@ -57,8 +54,6 @@ define void @start1(i4 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-2 + (2 smax %n)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 5 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-2 + (2 smax %n)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-2 + (2 smax %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -79,7 +74,6 @@ define void @start1_step2(i4 %n) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: %s = icmp sgt i4 %n, 0 @@ -99,8 +93,6 @@ define void @startx(i4 %n, i4 %x) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax %n)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax %n)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -121,7 +113,6 @@ define void @startx_step2(i4 %n, i4 %x) { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: %s = icmp sgt i4 %n, 0 @@ -141,8 +132,6 @@ define void @nsw(i4 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + %n) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 6 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + %n) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + %n) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -171,8 +160,6 @@ define void @nsw_step2(i4 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 + %n) /u 2) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 + %n) /u 2) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 + %n) /u 2) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -193,8 +180,6 @@ define void @nsw_start1(i4 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-2 + (2 smax %n)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 5 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-2 + (2 smax %n)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-2 + (2 smax %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -215,8 +200,6 @@ define void @nsw_start1_step2(i4 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-2 + (3 smax %n)) /u 2) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-2 + (3 smax %n)) /u 2) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-2 + (3 smax %n)) /u 2) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -237,8 +220,6 @@ define void @nsw_startx(i4 %n, i4 %x) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax %n)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax %n)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -259,8 +240,6 @@ define void @nsw_startx_step2(i4 %n, i4 %x) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax %n)) /u 2) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 7 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax %n)) /u 2) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax %n)) /u 2) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -281,8 +260,6 @@ define void @even(i4 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (2 * %n)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 5 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (2 * %n)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (2 * %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 2 ; entry: @@ -304,8 +281,6 @@ define void @even_step2(i4 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 + (2 * %n)) /u 2) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 + (2 * %n)) /u 2) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 + (2 * %n)) /u 2) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -327,8 +302,6 @@ define void @even_start1(i4 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-2 + (2 * %n)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-2 + (2 * %n)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-2 + (2 * %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -350,8 +323,6 @@ define void @even_start1_step2(i4 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-2 + (2 * %n)) /u 2) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-2 + (2 * %n)) /u 2) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-2 + (2 * %n)) /u 2) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -373,8 +344,6 @@ define void @even_startx(i4 %n, i4 %x) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax (2 * %n))) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 -2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax (2 * %n))) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax (2 * %n))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -396,8 +365,6 @@ define void @even_startx_step2(i4 %n, i4 %x) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax (2 * %n))) /u 2) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 7 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax (2 * %n))) /u 2) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax (2 * %n))) /u 2) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -419,8 +386,6 @@ define void @even_nsw(i4 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (2 * %n)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 5 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (2 * %n)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (2 * %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 2 ; entry: @@ -442,8 +407,6 @@ define void @even_nsw_step2(i4 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 + (2 * %n)) /u 2) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 + (2 * %n)) /u 2) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 + (2 * %n)) /u 2) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -465,8 +428,6 @@ define void @even_nsw_start1(i4 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-2 + (2 * %n)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 4 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-2 + (2 * %n)) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-2 + (2 * %n)) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -488,8 +449,6 @@ define void @even_nsw_start1_step2(i4 %n) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-2 + (2 * %n)) /u 2) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-2 + (2 * %n)) /u 2) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-2 + (2 * %n)) /u 2) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -511,8 +470,6 @@ define void @even_nsw_startx(i4 %n, i4 %x) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax (2 * %n))) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 -2 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax (2 * %n))) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax (2 * %n))) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -534,8 +491,6 @@ define void @even_nsw_startx_step2(i4 %n, i4 %x) { ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax (2 * %n))) /u 2) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i4 7 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax (2 * %n))) /u 2) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax (2 * %n))) /u 2) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll index 7cb65d23764e..bf140c7fa216 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll @@ -16,8 +16,6 @@ define void @test_trip_multiple_4(i32 %num) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 4 ; entry: @@ -51,8 +49,6 @@ define void @test_trip_multiple_4_guard(i32 %num) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 4 ; entry: @@ -87,8 +83,6 @@ define void @test_trip_multiple_4_ugt_5(i32 %num) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 4 ; entry: @@ -123,8 +117,6 @@ define void @test_trip_multiple_4_ugt_5_order_swapped(i32 %num) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 4 ; entry: @@ -158,8 +150,6 @@ define void @test_trip_multiple_4_sgt_5(i32 %num) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 4 ; entry: @@ -194,8 +184,6 @@ define void @test_trip_multiple_4_sgt_5_order_swapped(i32 %num) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 4 ; entry: @@ -229,8 +217,6 @@ define void @test_trip_multiple_4_uge_5(i32 %num) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 4 ; entry: @@ -265,8 +251,6 @@ define void @test_trip_multiple_4_uge_5_order_swapped(i32 %num) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 4 ; entry: @@ -301,8 +285,6 @@ define void @test_trip_multiple_4_sge_5(i32 %num) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 4 ; entry: @@ -336,8 +318,6 @@ define void @test_trip_multiple_4_sge_5_order_swapped(i32 %num) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 4 ; entry: @@ -372,8 +352,6 @@ define void @test_trip_multiple_4_icmp_ops_swapped(i32 %num) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 4 ; entry: @@ -407,8 +385,6 @@ define void @test_trip_multiple_4_upper_lower_bounds(i32 %num) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 4 ; entry: @@ -444,8 +420,6 @@ define void @test_trip_multiple_4_upper_lower_bounds_swapped1(i32 %num) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 4 ; entry: @@ -481,8 +455,6 @@ define void @test_trip_multiple_4_upper_lower_bounds_swapped2(i32 %num) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 4 ; entry: @@ -518,8 +490,6 @@ define void @test_trip_multiple_5(i32 %num) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 5 ; entry: @@ -555,8 +525,6 @@ define void @test_trunc_operand_larger_than_urem_expr(i64 %N) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %N) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %N) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %N) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -589,7 +557,6 @@ define void @test_trip_multiple_4_vectorized_iv(i32 %num) { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Unpredictable predicated backedge-taken count. ; entry: %u = urem i32 %num, 4 diff --git a/llvm/test/Analysis/ScalarEvolution/trip-multiple.ll b/llvm/test/Analysis/ScalarEvolution/trip-multiple.ll index f0065b9c2d8b..bd0f603db942 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-multiple.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-multiple.ll @@ -27,8 +27,6 @@ define void @trip_multiple_3(i32 noundef %num) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 3 ; entry: @@ -63,8 +61,6 @@ define void @trip_multiple_4(i32 noundef %num) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -5 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 4 ; entry: @@ -100,8 +96,6 @@ define void @trip_multiple_5(i32 noundef %num) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 5 ; entry: @@ -137,8 +131,6 @@ define void @trip_multiple_6(i32 noundef %num) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -5 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 6 ; entry: @@ -174,8 +166,6 @@ define void @trip_multiple_7(i32 noundef %num) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -5 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 7 ; entry: @@ -211,8 +201,6 @@ define void @trip_multiple_8(i32 noundef %num) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -9 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 8 ; entry: @@ -247,8 +235,6 @@ define void @trip_multiple_9(i32 noundef %num) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -5 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 9 ; entry: @@ -283,8 +269,6 @@ define void @trip_multiple_10(i32 noundef %num) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + %num) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 -7 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + %num) -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is (-1 + %num) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %for.body: Trip multiple is 10 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/umin-umax-folds.ll b/llvm/test/Analysis/ScalarEvolution/umin-umax-folds.ll index 917503b23a74..fbdbefb875fb 100644 --- a/llvm/test/Analysis/ScalarEvolution/umin-umax-folds.ll +++ b/llvm/test/Analysis/ScalarEvolution/umin-umax-folds.ll @@ -18,8 +18,6 @@ define void @umin_sext_x_zext_x(i32 %len) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (zext i32 %len to i64) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4294967295 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i32 %len to i64) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (zext i32 %len to i64) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -54,8 +52,6 @@ define void @ule_sext_x_zext_x(i32 %len) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (zext i32 %len to i64) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4294967295 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i32 %len to i64) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (zext i32 %len to i64) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -90,8 +86,6 @@ define void @uge_sext_x_zext_x(i32 %len) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (sext i32 %len to i64) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (sext i32 %len to i64) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (sext i32 %len to i64) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -126,8 +120,6 @@ define void @ult_sext_x_zext_x(i32 %len) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (zext i32 %len to i64) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4294967295 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i32 %len to i64) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (zext i32 %len to i64) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -162,8 +154,6 @@ define void @ugt_sext_x_zext_x(i32 %len) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (sext i32 %len to i64) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (sext i32 %len to i64) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (sext i32 %len to i64) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -198,8 +188,6 @@ define void @sle_sext_x_zext_x(i32 %len) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (zext i32 %len to i64) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4294967295 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i32 %len to i64) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (zext i32 %len to i64) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -234,8 +222,6 @@ define void @sge_sext_x_zext_x(i32 %len) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (zext i32 %len to i64) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4294967295 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i32 %len to i64) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (zext i32 %len to i64) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -270,8 +256,6 @@ define void @slt_sext_x_zext_x(i32 %len) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (sext i32 %len to i64) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (sext i32 %len to i64) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (sext i32 %len to i64) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -306,8 +290,6 @@ define void @sgt_sext_x_zext_x(i32 %len) { ; CHECK-NEXT: Loop %loop: backedge-taken count is (zext i32 %len to i64) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4294967295 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i32 %len to i64) -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (zext i32 %len to i64) -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/unknown_phis.ll b/llvm/test/Analysis/ScalarEvolution/unknown_phis.ll index 685a3d182dc4..2a8d4de24b58 100644 --- a/llvm/test/Analysis/ScalarEvolution/unknown_phis.ll +++ b/llvm/test/Analysis/ScalarEvolution/unknown_phis.ll @@ -50,8 +50,6 @@ define void @merge_values_with_ranges_looped(ptr %a_len_ptr, ptr %b_len_ptr) { ; CHECK-NEXT: Loop %loop: backedge-taken count is 99 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 99 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 99 -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is 99 -; CHECK-NEXT: Predicates: ; CHECK-NEXT: Loop %loop: Trip multiple is 100 ; diff --git a/llvm/test/Analysis/ScalarEvolution/widenable-condition.ll b/llvm/test/Analysis/ScalarEvolution/widenable-condition.ll index 867b06bd24ca..f1bda1705dff 100644 --- a/llvm/test/Analysis/ScalarEvolution/widenable-condition.ll +++ b/llvm/test/Analysis/ScalarEvolution/widenable-condition.ll @@ -25,7 +25,6 @@ define i32 @wc_max() { ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 1999 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 1999 -; CHECK-NEXT: Loop %loop: Unpredictable predicated backedge-taken count. ; entry: br label %loop diff --git a/llvm/test/Transforms/LoopSimplify/preserve-scev.ll b/llvm/test/Transforms/LoopSimplify/preserve-scev.ll index 2763ef44b8d5..5bc81565b409 100644 --- a/llvm/test/Transforms/LoopSimplify/preserve-scev.ll +++ b/llvm/test/Transforms/LoopSimplify/preserve-scev.ll @@ -14,10 +14,8 @@ target datalayout = "n8:16:32:64" ; CHECK-LABEL: Determining loop execution counts for: @test ; CHECK: Loop %for.body18: Unpredictable backedge-taken count. ; CHECK: Loop %for.body18: constant max backedge-taken count is i32 2147483646 -; CHECK: Loop %for.body18: Unpredictable predicated backedge-taken count. ; CHECK: Loop %for.cond: Unpredictable backedge-taken count. ; CHECK: Loop %for.cond: Unpredictable constant max backedge-taken count. -; CHECK: Loop %for.cond: Unpredictable predicated backedge-taken count. ; ; Now simplify the loop, which should cause SCEV to re-compute more precise ; info here in addition to having preheader PHIs. Second SCEV print: @@ -26,13 +24,10 @@ target datalayout = "n8:16:32:64" ; CHECK-LABEL: Determining loop execution counts for: @test ; CHECK: Loop %for.body18: Unpredictable backedge-taken count. ; CHECK: Loop %for.body18: constant max backedge-taken count is i32 2147483646 -; CHECK: Loop %for.body18: Unpredictable predicated backedge-taken count. ; CHECK: Loop %for.cond: Unpredictable backedge-taken count. ; CHECK: Loop %for.cond: constant max backedge-taken count is i32 -2147483647 -; CHECK: Loop %for.cond: Unpredictable predicated backedge-taken count. ; CHECK: Loop %for.cond.outer: Unpredictable backedge-taken count. ; CHECK: Loop %for.cond.outer: Unpredictable constant max backedge-taken count. -; CHECK: Loop %for.cond.outer: Unpredictable predicated backedge-taken count. define i32 @test() nounwind { entry: br label %for.cond @@ -83,20 +78,16 @@ declare void @foo() nounwind ; CHECK-LABEL: Determining loop execution counts for: @mergeExit ; CHECK: Loop %while.cond191: Unpredictable backedge-taken count. ; CHECK: Loop %while.cond191: constant max backedge-taken count is i32 -1 -; CHECK: Loop %while.cond191: Unpredictable predicated backedge-taken count. ; CHECK: Loop %while.cond191.outer: Unpredictable backedge-taken count. ; CHECK: Loop %while.cond191.outer: Unpredictable constant max backedge-taken count. -; CHECK: Loop %while.cond191.outer: Unpredictable predicated backedge-taken count. ; ; After simplifying, the max backedge count is refined. ; Second SCEV print: ; CHECK-LABEL: Determining loop execution counts for: @mergeExit ; CHECK: Loop %while.cond191: backedge-taken count is 0 ; CHECK: Loop %while.cond191: constant max backedge-taken count is i32 0 -; CHECK: Loop %while.cond191: Predicated backedge-taken count is 0 ; CHECK: Loop %while.cond191.outer: Unpredictable backedge-taken count. ; CHECK: Loop %while.cond191.outer: constant max backedge-taken count is i1 false -; CHECK: Loop %while.cond191.outer: Unpredictable predicated backedge-taken count. define void @mergeExit(i32 %MapAttrCount) nounwind uwtable ssp { entry: br i1 undef, label %if.then124, label %if.end126 -- GitLab From 954f891af20d193f6a5f41d7ede6a9313a49cfc3 Mon Sep 17 00:00:00 2001 From: Krzysztof Parzyszek Date: Wed, 6 Mar 2024 12:42:44 -0600 Subject: [PATCH 336/929] [Unittests] Fix RISCV unit tests build /usr/bin/ld: CMakeFiles/RISCVTests.dir/RISCVInstrInfoTest.cpp.o: undefined reference to symbol '_ZNK4llvm12LocationSize5printERNS_11raw_ostreamE' /usr/bin/ld: /work/kparzysz/git/llvm.org/b/x86/lib/libLLVMAnalysis.so.19.0 git: error adding symbols: DSO missing from command line collect2: error: ld returned 1 exit status The undefined symbol is llvm::LocationSize::print(llvm::raw_ostream&) const --- llvm/unittests/Target/RISCV/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/unittests/Target/RISCV/CMakeLists.txt b/llvm/unittests/Target/RISCV/CMakeLists.txt index b58d605355ba..0a64bf273e31 100644 --- a/llvm/unittests/Target/RISCV/CMakeLists.txt +++ b/llvm/unittests/Target/RISCV/CMakeLists.txt @@ -7,6 +7,7 @@ set(LLVM_LINK_COMPONENTS RISCVCodeGen RISCVDesc RISCVInfo + Analysis CodeGen Core MC -- GitLab From 8bdddcf0bb5a40e6ce6cbf7fc6b7ce576e2b032d Mon Sep 17 00:00:00 2001 From: jeffreytan81 Date: Wed, 6 Mar 2024 10:50:32 -0800 Subject: [PATCH 337/929] Fix lldb crash while handling concurrent vfork() (#81564) We got user reporting lldb crash while the debuggee is calling vfork() concurrently from multiple threads. The crash happens because the current implementation can only handle single vfork, vforkdone protocol transaction. This diff fixes the crash by lldb-server storing forked debuggee's pair in jstopinfo which will be decoded by lldb client to create StopInfoVFork for follow parent/child policy. Each StopInfoVFork will later have a corresponding vforkdone packet. So the patch also changes the `m_vfork_in_progress` to be reference counting based. Two new test cases are added which crash/assert without the changes in this patch. --------- Co-authored-by: jeffreytan81 --- .../Process/Linux/NativeThreadLinux.cpp | 3 + .../Process/gdb-remote/ProcessGDBRemote.cpp | 28 +++-- .../Process/gdb-remote/ProcessGDBRemote.h | 3 +- .../fork/concurrent_vfork/Makefile | 4 + .../concurrent_vfork/TestConcurrentVFork.py | 112 ++++++++++++++++++ .../fork/concurrent_vfork/main.cpp | 105 ++++++++++++++++ 6 files changed, 243 insertions(+), 12 deletions(-) create mode 100644 lldb/test/API/functionalities/fork/concurrent_vfork/Makefile create mode 100644 lldb/test/API/functionalities/fork/concurrent_vfork/TestConcurrentVFork.py create mode 100644 lldb/test/API/functionalities/fork/concurrent_vfork/main.cpp diff --git a/lldb/source/Plugins/Process/Linux/NativeThreadLinux.cpp b/lldb/source/Plugins/Process/Linux/NativeThreadLinux.cpp index b62e9f643fa7..26cb26daabf5 100644 --- a/lldb/source/Plugins/Process/Linux/NativeThreadLinux.cpp +++ b/lldb/source/Plugins/Process/Linux/NativeThreadLinux.cpp @@ -456,6 +456,9 @@ void NativeThreadLinux::SetStoppedByFork(bool is_vfork, lldb::pid_t child_pid) { m_stop_info.signo = SIGTRAP; m_stop_info.details.fork.child_pid = child_pid; m_stop_info.details.fork.child_tid = child_pid; + m_stop_description = std::to_string(child_pid); + m_stop_description += " "; + m_stop_description += std::to_string(child_pid); } void NativeThreadLinux::SetStoppedByVForkDone() { diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp index 51ceb12f1a57..5b9a9d71802f 100644 --- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp +++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp @@ -263,10 +263,9 @@ ProcessGDBRemote::ProcessGDBRemote(lldb::TargetSP target_sp, m_continue_C_tids(), m_continue_s_tids(), m_continue_S_tids(), m_max_memory_size(0), m_remote_stub_max_memory_size(0), m_addr_to_mmap_size(), m_thread_create_bp_sp(), - m_waiting_for_attach(false), - m_command_sp(), m_breakpoint_pc_offset(0), + m_waiting_for_attach(false), m_command_sp(), m_breakpoint_pc_offset(0), m_initial_tid(LLDB_INVALID_THREAD_ID), m_allow_flash_writes(false), - m_erased_flash_ranges(), m_vfork_in_progress(false) { + m_erased_flash_ranges(), m_vfork_in_progress_count(0) { m_async_broadcaster.SetEventName(eBroadcastBitAsyncThreadShouldExit, "async thread should exit"); m_async_broadcaster.SetEventName(eBroadcastBitAsyncContinue, @@ -5293,8 +5292,10 @@ public: (ProcessGDBRemote *)m_interpreter.GetExecutionContext() .GetProcessPtr(); if (process) { - StreamSP output_stream_sp( - m_interpreter.GetDebugger().GetAsyncOutputStream()); + StreamSP output_stream_sp = result.GetImmediateOutputStream(); + if (!output_stream_sp) + output_stream_sp = + StreamSP(m_interpreter.GetDebugger().GetAsyncOutputStream()); result.SetImmediateOutputStream(output_stream_sp); const uint32_t num_packets = @@ -5634,8 +5635,11 @@ void ProcessGDBRemote::DidFork(lldb::pid_t child_pid, lldb::tid_t child_tid) { void ProcessGDBRemote::DidVFork(lldb::pid_t child_pid, lldb::tid_t child_tid) { Log *log = GetLog(GDBRLog::Process); - assert(!m_vfork_in_progress); - m_vfork_in_progress = true; + LLDB_LOG( + log, + "ProcessGDBRemote::DidFork() called for child_pid: {0}, child_tid {1}", + child_pid, child_tid); + ++m_vfork_in_progress_count; // Disable all software breakpoints for the duration of vfork. if (m_gdb_comm.SupportsGDBStoppointPacket(eBreakpointSoftware)) @@ -5689,8 +5693,8 @@ void ProcessGDBRemote::DidVFork(lldb::pid_t child_pid, lldb::tid_t child_tid) { } void ProcessGDBRemote::DidVForkDone() { - assert(m_vfork_in_progress); - m_vfork_in_progress = false; + assert(m_vfork_in_progress_count > 0); + --m_vfork_in_progress_count; // Reenable all software breakpoints that were enabled before vfork. if (m_gdb_comm.SupportsGDBStoppointPacket(eBreakpointSoftware)) @@ -5700,7 +5704,9 @@ void ProcessGDBRemote::DidVForkDone() { void ProcessGDBRemote::DidExec() { // If we are following children, vfork is finished by exec (rather than // vforkdone that is submitted for parent). - if (GetFollowForkMode() == eFollowChild) - m_vfork_in_progress = false; + if (GetFollowForkMode() == eFollowChild) { + if (m_vfork_in_progress_count > 0) + --m_vfork_in_progress_count; + } Process::DidExec(); } diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h index c1ea1cc79055..610a1ee0b34d 100644 --- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h +++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h @@ -301,7 +301,8 @@ protected: using FlashRange = FlashRangeVector::Entry; FlashRangeVector m_erased_flash_ranges; - bool m_vfork_in_progress; + // Number of vfork() operations being handled. + uint32_t m_vfork_in_progress_count; // Accessors bool IsRunning(lldb::StateType state) { diff --git a/lldb/test/API/functionalities/fork/concurrent_vfork/Makefile b/lldb/test/API/functionalities/fork/concurrent_vfork/Makefile new file mode 100644 index 000000000000..c46619c66234 --- /dev/null +++ b/lldb/test/API/functionalities/fork/concurrent_vfork/Makefile @@ -0,0 +1,4 @@ +CXX_SOURCES := main.cpp +ENABLE_THREADS := YES + +include Makefile.rules diff --git a/lldb/test/API/functionalities/fork/concurrent_vfork/TestConcurrentVFork.py b/lldb/test/API/functionalities/fork/concurrent_vfork/TestConcurrentVFork.py new file mode 100644 index 000000000000..2dcbb728549f --- /dev/null +++ b/lldb/test/API/functionalities/fork/concurrent_vfork/TestConcurrentVFork.py @@ -0,0 +1,112 @@ +""" +Make sure that the concurrent vfork() from multiple threads works correctly. +""" + +import lldb +import lldbsuite.test.lldbutil as lldbutil +from lldbsuite.test.lldbtest import * +from lldbsuite.test.decorators import * + + +class TestConcurrentVFork(TestBase): + NO_DEBUG_INFO_TESTCASE = True + + def build_run_to_breakpoint(self, use_fork, call_exec): + self.build() + + args = [] + if use_fork: + args.append("--fork") + if call_exec: + args.append("--exec") + launch_info = lldb.SBLaunchInfo(args) + launch_info.SetWorkingDirectory(self.getBuildDir()) + + return lldbutil.run_to_source_breakpoint( + self, "// break here", lldb.SBFileSpec("main.cpp") + ) + + def follow_parent_helper(self, use_fork, call_exec): + (target, process, thread, bkpt) = self.build_run_to_breakpoint( + use_fork, call_exec + ) + + parent_pid = target.FindFirstGlobalVariable("g_pid").GetValueAsUnsigned() + self.runCmd("settings set target.process.follow-fork-mode parent") + self.runCmd("settings set target.process.stop-on-exec False", check=False) + self.expect( + "continue", substrs=[f"Process {parent_pid} exited with status = 0"] + ) + + def follow_child_helper(self, use_fork, call_exec): + self.build_run_to_breakpoint(use_fork, call_exec) + + self.runCmd("settings set target.process.follow-fork-mode child") + self.runCmd("settings set target.process.stop-on-exec False", check=False) + # Child process exits with code "index + 10" since index is [0-4] + # so the exit code should be 1[0-4] + self.expect("continue", patterns=[r"exited with status = 1[0-4]"]) + + @skipUnlessPlatform(["linux"]) + def test_follow_parent_vfork_no_exec(self): + """ + Make sure that debugging concurrent vfork() from multiple threads won't crash lldb during follow-parent. + And follow-parent successfully detach all child processes and exit debugger without calling exec. + """ + self.follow_parent_helper(use_fork=False, call_exec=False) + + @skipUnlessPlatform(["linux"]) + def test_follow_parent_fork_no_exec(self): + """ + Make sure that debugging concurrent fork() from multiple threads won't crash lldb during follow-parent. + And follow-parent successfully detach all child processes and exit debugger without calling exec + """ + self.follow_parent_helper(use_fork=True, call_exec=False) + + @skipUnlessPlatform(["linux"]) + def test_follow_parent_vfork_call_exec(self): + """ + Make sure that debugging concurrent vfork() from multiple threads won't crash lldb during follow-parent. + And follow-parent successfully detach all child processes and exit debugger after calling exec. + """ + self.follow_parent_helper(use_fork=False, call_exec=True) + + @skipUnlessPlatform(["linux"]) + def test_follow_parent_fork_call_exec(self): + """ + Make sure that debugging concurrent vfork() from multiple threads won't crash lldb during follow-parent. + And follow-parent successfully detach all child processes and exit debugger after calling exec. + """ + self.follow_parent_helper(use_fork=True, call_exec=True) + + @skipUnlessPlatform(["linux"]) + def test_follow_child_vfork_no_exec(self): + """ + Make sure that debugging concurrent vfork() from multiple threads won't crash lldb during follow-child. + And follow-child successfully detach parent process and exit child process with correct exit code without calling exec. + """ + self.follow_child_helper(use_fork=False, call_exec=False) + + @skipUnlessPlatform(["linux"]) + def test_follow_child_fork_no_exec(self): + """ + Make sure that debugging concurrent fork() from multiple threads won't crash lldb during follow-child. + And follow-child successfully detach parent process and exit child process with correct exit code without calling exec. + """ + self.follow_child_helper(use_fork=True, call_exec=False) + + @skipUnlessPlatform(["linux"]) + def test_follow_child_vfork_call_exec(self): + """ + Make sure that debugging concurrent vfork() from multiple threads won't crash lldb during follow-child. + And follow-child successfully detach parent process and exit child process with correct exit code after calling exec. + """ + self.follow_child_helper(use_fork=False, call_exec=True) + + @skipUnlessPlatform(["linux"]) + def test_follow_child_fork_call_exec(self): + """ + Make sure that debugging concurrent fork() from multiple threads won't crash lldb during follow-child. + And follow-child successfully detach parent process and exit child process with correct exit code after calling exec. + """ + self.follow_child_helper(use_fork=True, call_exec=True) diff --git a/lldb/test/API/functionalities/fork/concurrent_vfork/main.cpp b/lldb/test/API/functionalities/fork/concurrent_vfork/main.cpp new file mode 100644 index 000000000000..b0a4446ba015 --- /dev/null +++ b/lldb/test/API/functionalities/fork/concurrent_vfork/main.cpp @@ -0,0 +1,105 @@ +#include +#include +#include +#include +#include +#include +#include + +pid_t g_pid = 0; +std::mutex g_child_pids_mutex; +std::vector g_child_pids; + +const char *g_program = nullptr; +bool g_use_vfork = true; // Use vfork by default. +bool g_call_exec = false; // Does not call exec by default. + +int call_vfork(int index) { + pid_t child_pid = 0; + if (g_use_vfork) { + child_pid = vfork(); + } else { + child_pid = fork(); + } + + if (child_pid == -1) { + // Error handling + perror("vfork"); + return 1; + } else if (child_pid == 0) { + // This code is executed by the child process + g_pid = getpid(); + printf("Child process: %d\n", g_pid); + + if (g_call_exec) { + std::string child_exit_code = std::to_string(index + 10); + execl(g_program, g_program, "--child", child_exit_code.c_str(), NULL); + } else { + _exit(index + 10); + } + } else { + // This code is executed by the parent process + printf("[Parent] Forked process id: %d\n", child_pid); + } + return 0; +} + +void wait_all_children_to_exit() { + std::lock_guard Lock(g_child_pids_mutex); + for (pid_t child_pid : g_child_pids) { + int child_status = 0; + pid_t pid = waitpid(child_pid, &child_status, 0); + if (child_status != 0) { + int exit_code = WEXITSTATUS(child_status); + if (exit_code > 15 || exit_code < 10) { + printf("Error: child process exits with unexpected code %d\n", + exit_code); + _exit(1); // This will let our program know that some child processes + // didn't exist with an expected exit status. + } + } + if (pid != child_pid) + _exit(2); // This will let our program know it didn't succeed + } +} + +void create_threads(int num_threads) { + std::vector threads; + for (int i = 0; i < num_threads; ++i) { + threads.emplace_back(std::thread(call_vfork, i)); + } + printf("Created %d threads, joining...\n", + num_threads); // end_of_create_threads + for (auto &thread : threads) { + thread.join(); + } + wait_all_children_to_exit(); +} + +// Can be called in various ways: +// 1. [program]: use vfork and not call exec +// 2. [program] --fork: use fork and not call exec +// 3. [program] --fork --exec: use fork and call exec +// 4. [program] --exec: use vfork and call exec +// 5. [program] --child [exit_code]: child process +int main(int argc, char *argv[]) { + g_pid = getpid(); + g_program = argv[0]; + + for (int i = 1; i < argc; ++i) { + if (strcmp(argv[i], "--child") == 0) { + assert(i + 1 < argc); + int child_exit_code = std::stoi(argv[i + 1]); + printf("Child process: %d, exiting with code %d\n", g_pid, + child_exit_code); + _exit(child_exit_code); + } else if (strcmp(argv[i], "--fork") == 0) + g_use_vfork = false; + else if (strcmp(argv[i], "--exec") == 0) + g_call_exec = true; + } + + int num_threads = 5; // break here + create_threads(num_threads); + return 0; +} -- GitLab From 04bbbba271ebe4c2421f34a4fbf34c328df9f111 Mon Sep 17 00:00:00 2001 From: Jason Molenda Date: Wed, 6 Mar 2024 10:53:37 -0800 Subject: [PATCH 338/929] Skip TestAddressMasks API tests on 32-bit arm TestAddressMasks failed on the lldb-arm-buntu bot with the Code address mask test, mask = process.GetAddressMask(lldb.eAddressMaskTypeAny) process.SetAddressMask(lldb.eAddressMaskTypeCode, mask | 0x3) self.assertEqual( 0x000002950001F694, process.FixAddress(0x00265E950001F697, lldb.eAddressMaskTypeCode), ) The API returned 0x000002950001f694 instead of the expected 0x00265e950001f696. The low bits differ because ABISysV_arm hardcodes the Code address mask to clear the 0th bit, it doesn't use the Process code mask. I didn't debug why some of the high bytes were dropped. The address mask APIs are only important on 64-bit targets, where many of the bits are not used for addressing and are used for metadata instead, so I'm going to skip these tests on 32-bit arm instead of debugging. --- .../API/python_api/process/address-masks/TestAddressMasks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lldb/test/API/python_api/process/address-masks/TestAddressMasks.py b/lldb/test/API/python_api/process/address-masks/TestAddressMasks.py index e0a570c15961..152776efc726 100644 --- a/lldb/test/API/python_api/process/address-masks/TestAddressMasks.py +++ b/lldb/test/API/python_api/process/address-masks/TestAddressMasks.py @@ -19,6 +19,7 @@ class AddressMasksTestCase(TestBase): self.runCmd("settings set target.process.virtual-addressable-bits 0") self.runCmd("settings set target.process.highmem-virtual-addressable-bits 0") + @skipIf(archs=["arm"]) # 32-bit arm ABI hardcodes Code mask, is 32-bit def test_address_masks(self): self.build() (target, process, t, bp) = lldbutil.run_to_source_breakpoint( @@ -79,6 +80,7 @@ class AddressMasksTestCase(TestBase): # AArch64 can have different address masks for high and low memory, when different # page tables are set up. @skipIf(archs=no_match(["arm64", "arm64e", "aarch64"])) + @skipIf(archs=["arm"]) # 32-bit arm ABI hardcodes Code mask, is 32-bit def test_address_masks_target_supports_highmem_tests(self): self.build() (target, process, t, bp) = lldbutil.run_to_source_breakpoint( @@ -111,6 +113,7 @@ class AddressMasksTestCase(TestBase): # On most targets where we have a single mask for all address range, confirm # that the high memory masks are ignored. @skipIf(archs=["arm64", "arm64e", "aarch64"]) + @skipIf(archs=["arm"]) # 32-bit arm ABI hardcodes Code mask, is 32-bit def test_address_masks_target_no_highmem(self): self.build() (target, process, t, bp) = lldbutil.run_to_source_breakpoint( -- GitLab From 6494f9bb8ac1e6f7526b72ee07f71527b8e66066 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Wed, 6 Mar 2024 20:23:09 +0100 Subject: [PATCH 339/929] [clang-repl] Expose setter for triple in IncrementalCompilerBuilder (#84174) With out-of-process execution the target triple can be different from the one on the host. We need an interface to configure it. --- clang/include/clang/Interpreter/Interpreter.h | 5 ++- clang/lib/Interpreter/Interpreter.cpp | 12 ++++--- clang/unittests/Interpreter/CMakeLists.txt | 1 + .../IncrementalCompilerBuilderTest.cpp | 35 +++++++++++++++++++ 4 files changed, 47 insertions(+), 6 deletions(-) create mode 100644 clang/unittests/Interpreter/IncrementalCompilerBuilderTest.cpp diff --git a/clang/include/clang/Interpreter/Interpreter.h b/clang/include/clang/Interpreter/Interpreter.h index 292fa566ae70..c8f932e95c47 100644 --- a/clang/include/clang/Interpreter/Interpreter.h +++ b/clang/include/clang/Interpreter/Interpreter.h @@ -48,6 +48,8 @@ public: UserArgs = Args; } + void SetTargetTriple(std::string TT) { TargetTriple = TT; } + // General C++ llvm::Expected> CreateCpp(); @@ -62,11 +64,12 @@ public: private: static llvm::Expected> - create(std::vector &ClangArgv); + create(std::string TT, std::vector &ClangArgv); llvm::Expected> createCuda(bool device); std::vector UserArgs; + std::optional TargetTriple; llvm::StringRef OffloadArch; llvm::StringRef CudaSDKPath; diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp index 9f97a3c6b0be..37696b289764 100644 --- a/clang/lib/Interpreter/Interpreter.cpp +++ b/clang/lib/Interpreter/Interpreter.cpp @@ -132,7 +132,8 @@ CreateCI(const llvm::opt::ArgStringList &Argv) { } // anonymous namespace llvm::Expected> -IncrementalCompilerBuilder::create(std::vector &ClangArgv) { +IncrementalCompilerBuilder::create(std::string TT, + std::vector &ClangArgv) { // If we don't know ClangArgv0 or the address of main() at this point, try // to guess it anyway (it's possible on some platforms). @@ -162,8 +163,7 @@ IncrementalCompilerBuilder::create(std::vector &ClangArgv) { TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer; DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer); - driver::Driver Driver(/*MainBinaryName=*/ClangArgv[0], - llvm::sys::getProcessTriple(), Diags); + driver::Driver Driver(/*MainBinaryName=*/ClangArgv[0], TT, Diags); Driver.setCheckInputsExist(false); // the input comes from mem buffers llvm::ArrayRef RF = llvm::ArrayRef(ClangArgv); std::unique_ptr Compilation(Driver.BuildCompilation(RF)); @@ -185,7 +185,8 @@ IncrementalCompilerBuilder::CreateCpp() { Argv.push_back("-xc++"); Argv.insert(Argv.end(), UserArgs.begin(), UserArgs.end()); - return IncrementalCompilerBuilder::create(Argv); + std::string TT = TargetTriple ? *TargetTriple : llvm::sys::getProcessTriple(); + return IncrementalCompilerBuilder::create(TT, Argv); } llvm::Expected> @@ -213,7 +214,8 @@ IncrementalCompilerBuilder::createCuda(bool device) { Argv.insert(Argv.end(), UserArgs.begin(), UserArgs.end()); - return IncrementalCompilerBuilder::create(Argv); + std::string TT = TargetTriple ? *TargetTriple : llvm::sys::getProcessTriple(); + return IncrementalCompilerBuilder::create(TT, Argv); } llvm::Expected> diff --git a/clang/unittests/Interpreter/CMakeLists.txt b/clang/unittests/Interpreter/CMakeLists.txt index 712641afb976..0ddedb283e07 100644 --- a/clang/unittests/Interpreter/CMakeLists.txt +++ b/clang/unittests/Interpreter/CMakeLists.txt @@ -7,6 +7,7 @@ set(LLVM_LINK_COMPONENTS ) add_clang_unittest(ClangReplInterpreterTests + IncrementalCompilerBuilderTest.cpp IncrementalProcessingTest.cpp InterpreterTest.cpp CodeCompletionTest.cpp diff --git a/clang/unittests/Interpreter/IncrementalCompilerBuilderTest.cpp b/clang/unittests/Interpreter/IncrementalCompilerBuilderTest.cpp new file mode 100644 index 000000000000..1cc0223465c8 --- /dev/null +++ b/clang/unittests/Interpreter/IncrementalCompilerBuilderTest.cpp @@ -0,0 +1,35 @@ +//=== unittests/Interpreter/IncrementalCompilerBuilderTest.cpp ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/Basic/TargetOptions.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Interpreter/Interpreter.h" +#include "llvm/Support/Error.h" +#include "gtest/gtest.h" + +using namespace llvm; +using namespace clang; + +namespace { + +TEST(IncrementalCompilerBuilder, SetCompilerArgs) { + std::vector ClangArgv = {"-Xclang", "-ast-dump-all"}; + auto CB = clang::IncrementalCompilerBuilder(); + CB.SetCompilerArgs(ClangArgv); + auto CI = cantFail(CB.CreateCpp()); + EXPECT_TRUE(CI->getFrontendOpts().ASTDumpAll); +} + +TEST(IncrementalCompilerBuilder, SetTargetTriple) { + auto CB = clang::IncrementalCompilerBuilder(); + CB.SetTargetTriple("armv6-none-eabi"); + auto CI = cantFail(CB.CreateCpp()); + EXPECT_EQ(CI->getTargetOpts().Triple, "armv6-none-unknown-eabi"); +} + +} // end anonymous namespace -- GitLab From a6c84079f69f13d745aaaa4595cf8804b119a6c4 Mon Sep 17 00:00:00 2001 From: Alexander Yermolovich <43973793+ayermolo@users.noreply.github.com> Date: Wed, 6 Mar 2024 11:24:44 -0800 Subject: [PATCH 340/929] [CLANG][DWARF] Do not emit -ggnu-pubnames for LLDB tuning, unless -ggnu-pubnames is specified. (#83331) When -gsplit-dwarf is passed in clang emmmits -ggnu-pubnames which results in .debug_gnu_pubnames/..debug_gnu_pubtypes being generated. This is used by GDB, but not by LLDB. Changed so that these sections are not emitted for LLDB tuning, unless flag is passed explicitly. --- clang/lib/Driver/ToolChains/Clang.cpp | 14 ++++++++++---- clang/test/Driver/split-debug.c | 10 ++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index 858d20fbfac0..deb2dac80afe 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -4478,14 +4478,20 @@ renderDebugOptions(const ToolChain &TC, const Driver &D, const llvm::Triple &T, Args.getLastArg(options::OPT_ggnu_pubnames, options::OPT_gno_gnu_pubnames, options::OPT_gpubnames, options::OPT_gno_pubnames); if (DwarfFission != DwarfFissionKind::None || - (PubnamesArg && checkDebugInfoOption(PubnamesArg, Args, D, TC))) - if (!PubnamesArg || - (!PubnamesArg->getOption().matches(options::OPT_gno_gnu_pubnames) && - !PubnamesArg->getOption().matches(options::OPT_gno_pubnames))) + (PubnamesArg && checkDebugInfoOption(PubnamesArg, Args, D, TC))) { + const bool OptionSet = + (PubnamesArg && + (PubnamesArg->getOption().matches(options::OPT_gpubnames) || + PubnamesArg->getOption().matches(options::OPT_ggnu_pubnames))); + if ((DebuggerTuning != llvm::DebuggerKind::LLDB || OptionSet) && + (!PubnamesArg || + (!PubnamesArg->getOption().matches(options::OPT_gno_gnu_pubnames) && + !PubnamesArg->getOption().matches(options::OPT_gno_pubnames)))) CmdArgs.push_back(PubnamesArg && PubnamesArg->getOption().matches( options::OPT_gpubnames) ? "-gpubnames" : "-ggnu-pubnames"); + } const auto *SimpleTemplateNamesArg = Args.getLastArg(options::OPT_gsimple_template_names, options::OPT_gno_simple_template_names); diff --git a/clang/test/Driver/split-debug.c b/clang/test/Driver/split-debug.c index 968f33b4cc03..57f3989ed7b5 100644 --- a/clang/test/Driver/split-debug.c +++ b/clang/test/Driver/split-debug.c @@ -124,3 +124,13 @@ // G1_NOSPLIT: "-debug-info-kind=line-tables-only" // G1_NOSPLIT-NOT: "-split-dwarf-file" // G1_NOSPLIT-NOT: "-split-dwarf-output" + +/// Do not generate -ggnu-pubnames for -glldb +// RUN: %clang -### -c -target x86_64 -gsplit-dwarf -g -glldb %s 2>&1 | FileCheck %s --check-prefixes=GLLDBSPLIT + +// GLLDBSPLIT-NOT: "-ggnu-pubnames" + +/// Generate -ggnu-pubnames for -glldb when it is explicitly enabled +// RUN: %clang -### -c -target x86_64 -gsplit-dwarf -g -glldb -ggnu-pubnames %s 2>&1 | FileCheck %s --check-prefixes=GLLDBSPLIT2 + +// GLLDBSPLIT2: "-ggnu-pubnames" -- GitLab From c5aacb0dbcb19b154441852763785f0356e5218b Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Fri, 1 Mar 2024 18:48:45 -0600 Subject: [PATCH 341/929] [ValueTracking] Add fast path to avoid second recursive call in `isKnownPositive`; NFC Just a simple compile time improvement. This function isn't used much, however, so its not particularly impactful. Closes #83638 --- llvm/lib/Analysis/ValueTracking.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index 53c6326b8ec2..52ae9f034e5d 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -294,9 +294,11 @@ bool llvm::isKnownPositive(const Value *V, const SimplifyQuery &SQ, if (auto *CI = dyn_cast(V)) return CI->getValue().isStrictlyPositive(); - // TODO: We'd doing two recursive queries here. We should factor this such - // that only a single query is needed. - return isKnownNonNegative(V, SQ, Depth) && ::isKnownNonZero(V, Depth, SQ); + // If `isKnownNonNegative` ever becomes more sophisticated, make sure to keep + // this updated. + KnownBits Known = computeKnownBits(V, Depth, SQ); + return Known.isNonNegative() && + (Known.isNonZero() || ::isKnownNonZero(V, Depth, SQ)); } bool llvm::isKnownNegative(const Value *V, const SimplifyQuery &SQ, -- GitLab From 862e742292f4a46c4d3879e9cc0d9902ef87f107 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Wed, 21 Feb 2024 16:05:20 -0600 Subject: [PATCH 342/929] [InstCombine] Add tests for folding `(fp_binop ({s|u}itofp x), ({s|u}itofp y))`; NFC --- .../Transforms/InstCombine/binop-itofp.ll | 1024 +++++++++++++++++ 1 file changed, 1024 insertions(+) create mode 100644 llvm/test/Transforms/InstCombine/binop-itofp.ll diff --git a/llvm/test/Transforms/InstCombine/binop-itofp.ll b/llvm/test/Transforms/InstCombine/binop-itofp.ll new file mode 100644 index 000000000000..6354be7d723b --- /dev/null +++ b/llvm/test/Transforms/InstCombine/binop-itofp.ll @@ -0,0 +1,1024 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -passes=instcombine -S | FileCheck %s + +define half @test_ui_ui_i8_add(i8 noundef %x_in, i8 noundef %y_in) { +; CHECK-LABEL: @test_ui_ui_i8_add( +; CHECK-NEXT: [[X:%.*]] = and i8 [[X_IN:%.*]], 127 +; CHECK-NEXT: [[Y:%.*]] = and i8 [[Y_IN:%.*]], 127 +; CHECK-NEXT: [[XF:%.*]] = uitofp i8 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = uitofp i8 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = and i8 %x_in, 127 + %y = and i8 %y_in, 127 + %xf = uitofp i8 %x to half + %yf = uitofp i8 %y to half + %r = fadd half %xf, %yf + ret half %r +} + +define half @test_ui_ui_i8_add_fail_overflow(i8 noundef %x_in, i8 noundef %y_in) { +; CHECK-LABEL: @test_ui_ui_i8_add_fail_overflow( +; CHECK-NEXT: [[X:%.*]] = and i8 [[X_IN:%.*]], 127 +; CHECK-NEXT: [[Y:%.*]] = and i8 [[Y_IN:%.*]], -127 +; CHECK-NEXT: [[XF:%.*]] = uitofp i8 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = uitofp i8 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = and i8 %x_in, 127 + %y = and i8 %y_in, 129 + %xf = uitofp i8 %x to half + %yf = uitofp i8 %y to half + %r = fadd half %xf, %yf + ret half %r +} + +define half @test_ui_ui_i8_add_C(i8 noundef %x_in) { +; CHECK-LABEL: @test_ui_ui_i8_add_C( +; CHECK-NEXT: [[X:%.*]] = and i8 [[X_IN:%.*]], 127 +; CHECK-NEXT: [[XF:%.*]] = uitofp i8 [[X]] to half +; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], 0xH5800 +; CHECK-NEXT: ret half [[R]] +; + %x = and i8 %x_in, 127 + %xf = uitofp i8 %x to half + %r = fadd half %xf, 128.0 + ret half %r +} + +define half @test_ui_ui_i8_add_C_fail_no_repr(i8 noundef %x_in) { +; CHECK-LABEL: @test_ui_ui_i8_add_C_fail_no_repr( +; CHECK-NEXT: [[X:%.*]] = and i8 [[X_IN:%.*]], 127 +; CHECK-NEXT: [[XF:%.*]] = uitofp i8 [[X]] to half +; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], 0xH57F8 +; CHECK-NEXT: ret half [[R]] +; + %x = and i8 %x_in, 127 + %xf = uitofp i8 %x to half + %r = fadd half %xf, 127.5 + ret half %r +} + +define half @test_ui_ui_i8_add_C_fail_overflow(i8 noundef %x_in) { +; CHECK-LABEL: @test_ui_ui_i8_add_C_fail_overflow( +; CHECK-NEXT: [[X:%.*]] = and i8 [[X_IN:%.*]], 127 +; CHECK-NEXT: [[XF:%.*]] = uitofp i8 [[X]] to half +; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], 0xH5808 +; CHECK-NEXT: ret half [[R]] +; + %x = and i8 %x_in, 127 + %xf = uitofp i8 %x to half + %r = fadd half %xf, 129.0 + ret half %r +} + +define half @test_si_si_i8_add(i8 noundef %x_in, i8 noundef %y_in) { +; CHECK-LABEL: @test_si_si_i8_add( +; CHECK-NEXT: [[X:%.*]] = or i8 [[X_IN:%.*]], -64 +; CHECK-NEXT: [[Y:%.*]] = or i8 [[Y_IN:%.*]], -64 +; CHECK-NEXT: [[ADDCONV:%.*]] = add nsw i8 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = sitofp i8 [[ADDCONV]] to half +; CHECK-NEXT: ret half [[R]] +; + %x = or i8 %x_in, -64 + %y = or i8 %y_in, -64 + %xf = sitofp i8 %x to half + %yf = sitofp i8 %y to half + %r = fadd half %xf, %yf + ret half %r +} + +define half @test_si_si_i8_add_fail_overflow(i8 noundef %x_in, i8 noundef %y_in) { +; CHECK-LABEL: @test_si_si_i8_add_fail_overflow( +; CHECK-NEXT: [[X:%.*]] = or i8 [[X_IN:%.*]], -64 +; CHECK-NEXT: [[Y:%.*]] = or i8 [[Y_IN:%.*]], -65 +; CHECK-NEXT: [[XF:%.*]] = sitofp i8 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = sitofp i8 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = or i8 %x_in, -64 + %y = or i8 %y_in, -65 + %xf = sitofp i8 %x to half + %yf = sitofp i8 %y to half + %r = fadd half %xf, %yf + ret half %r +} + +define half @test_ui_si_i8_add(i8 noundef %x_in, i8 noundef %y_in) { +; CHECK-LABEL: @test_ui_si_i8_add( +; CHECK-NEXT: [[X:%.*]] = and i8 [[X_IN:%.*]], 63 +; CHECK-NEXT: [[Y:%.*]] = and i8 [[Y_IN:%.*]], 63 +; CHECK-NEXT: [[XF:%.*]] = sitofp i8 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = uitofp i8 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = and i8 %x_in, 63 + %y = and i8 %y_in, 63 + %xf = sitofp i8 %x to half + %yf = uitofp i8 %y to half + %r = fadd half %xf, %yf + ret half %r +} + +define half @test_ui_si_i8_add_overflow(i8 noundef %x_in, i8 noundef %y_in) { +; CHECK-LABEL: @test_ui_si_i8_add_overflow( +; CHECK-NEXT: [[X:%.*]] = and i8 [[X_IN:%.*]], 63 +; CHECK-NEXT: [[Y:%.*]] = and i8 [[Y_IN:%.*]], 65 +; CHECK-NEXT: [[XF:%.*]] = sitofp i8 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = uitofp i8 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = and i8 %x_in, 63 + %y = and i8 %y_in, 65 + %xf = sitofp i8 %x to half + %yf = uitofp i8 %y to half + %r = fadd half %xf, %yf + ret half %r +} + +define half @test_ui_ui_i8_sub_C(i8 noundef %x_in) { +; CHECK-LABEL: @test_ui_ui_i8_sub_C( +; CHECK-NEXT: [[X:%.*]] = or i8 [[X_IN:%.*]], -128 +; CHECK-NEXT: [[XF:%.*]] = uitofp i8 [[X]] to half +; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], 0xHD800 +; CHECK-NEXT: ret half [[R]] +; + %x = or i8 %x_in, 128 + %xf = uitofp i8 %x to half + %r = fsub half %xf, 128.0 + ret half %r +} + +define half @test_ui_ui_i8_sub_C_fail_overflow(i8 noundef %x_in) { +; CHECK-LABEL: @test_ui_ui_i8_sub_C_fail_overflow( +; CHECK-NEXT: [[X:%.*]] = and i8 [[X_IN:%.*]], 127 +; CHECK-NEXT: [[XF:%.*]] = uitofp i8 [[X]] to half +; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], 0xHD800 +; CHECK-NEXT: ret half [[R]] +; + %x = and i8 %x_in, 127 + %xf = uitofp i8 %x to half + %r = fsub half %xf, 128.0 + ret half %r +} + +define half @test_si_si_i8_sub(i8 noundef %x_in, i8 noundef %y_in) { +; CHECK-LABEL: @test_si_si_i8_sub( +; CHECK-NEXT: [[X:%.*]] = and i8 [[X_IN:%.*]], 63 +; CHECK-NEXT: [[Y:%.*]] = or i8 [[Y_IN:%.*]], -64 +; CHECK-NEXT: [[XF:%.*]] = sitofp i8 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = sitofp i8 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fsub half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = and i8 %x_in, 63 + %y = or i8 %y_in, -64 + %xf = sitofp i8 %x to half + %yf = sitofp i8 %y to half + %r = fsub half %xf, %yf + ret half %r +} + +define half @test_si_si_i8_sub_fail_overflow(i8 noundef %x_in, i8 noundef %y_in) { +; CHECK-LABEL: @test_si_si_i8_sub_fail_overflow( +; CHECK-NEXT: [[X:%.*]] = and i8 [[X_IN:%.*]], 63 +; CHECK-NEXT: [[Y:%.*]] = or i8 [[Y_IN:%.*]], -65 +; CHECK-NEXT: [[XF:%.*]] = sitofp i8 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = sitofp i8 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fsub half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = and i8 %x_in, 63 + %y = or i8 %y_in, -65 + %xf = sitofp i8 %x to half + %yf = sitofp i8 %y to half + %r = fsub half %xf, %yf + ret half %r +} + +define half @test_si_si_i8_sub_C(i8 noundef %x_in) { +; CHECK-LABEL: @test_si_si_i8_sub_C( +; CHECK-NEXT: [[X:%.*]] = and i8 [[X_IN:%.*]], 63 +; CHECK-NEXT: [[ADDCONV:%.*]] = or disjoint i8 [[X]], 64 +; CHECK-NEXT: [[R:%.*]] = sitofp i8 [[ADDCONV]] to half +; CHECK-NEXT: ret half [[R]] +; + %x = and i8 %x_in, 63 + %xf = sitofp i8 %x to half + %r = fsub half %xf, -64.0 + ret half %r +} + +define half @test_si_si_i8_sub_C_fail_overflow(i8 noundef %x_in) { +; CHECK-LABEL: @test_si_si_i8_sub_C_fail_overflow( +; CHECK-NEXT: [[X:%.*]] = and i8 [[X_IN:%.*]], 65 +; CHECK-NEXT: [[XF:%.*]] = sitofp i8 [[X]] to half +; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], 0xH5400 +; CHECK-NEXT: ret half [[R]] +; + %x = and i8 %x_in, 65 + %xf = sitofp i8 %x to half + %r = fsub half %xf, -64.0 + ret half %r +} + +define half @test_ui_si_i8_sub(i8 noundef %x_in, i8 noundef %y_in) { +; CHECK-LABEL: @test_ui_si_i8_sub( +; CHECK-NEXT: [[X:%.*]] = or i8 [[X_IN:%.*]], 64 +; CHECK-NEXT: [[Y:%.*]] = and i8 [[Y_IN:%.*]], 63 +; CHECK-NEXT: [[XF:%.*]] = sitofp i8 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = uitofp i8 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fsub half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = or i8 %x_in, 64 + %y = and i8 %y_in, 63 + %xf = sitofp i8 %x to half + %yf = uitofp i8 %y to half + %r = fsub half %xf, %yf + ret half %r +} + +define half @test_ui_si_i8_sub_fail_maybe_sign(i8 noundef %x_in, i8 noundef %y_in) { +; CHECK-LABEL: @test_ui_si_i8_sub_fail_maybe_sign( +; CHECK-NEXT: [[X:%.*]] = or i8 [[X_IN:%.*]], 64 +; CHECK-NEXT: [[Y:%.*]] = and i8 [[Y_IN:%.*]], 63 +; CHECK-NEXT: [[XF:%.*]] = uitofp i8 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = sitofp i8 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fsub half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = or i8 %x_in, 64 + %y = and i8 %y_in, 63 + %xf = uitofp i8 %x to half + %yf = sitofp i8 %y to half + %r = fsub half %xf, %yf + ret half %r +} + +define half @test_ui_ui_i8_mul(i8 noundef %x_in, i8 noundef %y_in) { +; CHECK-LABEL: @test_ui_ui_i8_mul( +; CHECK-NEXT: [[X:%.*]] = and i8 [[X_IN:%.*]], 15 +; CHECK-NEXT: [[Y:%.*]] = and i8 [[Y_IN:%.*]], 15 +; CHECK-NEXT: [[XF:%.*]] = uitofp i8 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = uitofp i8 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = and i8 %x_in, 15 + %y = and i8 %y_in, 15 + %xf = uitofp i8 %x to half + %yf = uitofp i8 %y to half + %r = fmul half %xf, %yf + ret half %r +} + +define half @test_ui_ui_i8_mul_C(i8 noundef %x_in) { +; CHECK-LABEL: @test_ui_ui_i8_mul_C( +; CHECK-NEXT: [[X:%.*]] = and i8 [[X_IN:%.*]], 15 +; CHECK-NEXT: [[XF:%.*]] = uitofp i8 [[X]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], 0xH4C00 +; CHECK-NEXT: ret half [[R]] +; + %x = and i8 %x_in, 15 + %xf = uitofp i8 %x to half + %r = fmul half %xf, 16.0 + ret half %r +} + +define half @test_ui_ui_i8_mul_C_fail_overlow(i8 noundef %x_in) { +; CHECK-LABEL: @test_ui_ui_i8_mul_C_fail_overlow( +; CHECK-NEXT: [[X:%.*]] = and i8 [[X_IN:%.*]], 14 +; CHECK-NEXT: [[XF:%.*]] = uitofp i8 [[X]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], 0xH4CC0 +; CHECK-NEXT: ret half [[R]] +; + %x = and i8 %x_in, 14 + %xf = uitofp i8 %x to half + %r = fmul half %xf, 19.0 + ret half %r +} + +define half @test_si_si_i8_mul(i8 noundef %x_in, i8 noundef %y_in) { +; CHECK-LABEL: @test_si_si_i8_mul( +; CHECK-NEXT: [[XX:%.*]] = and i8 [[X_IN:%.*]], 6 +; CHECK-NEXT: [[X:%.*]] = or disjoint i8 [[XX]], 1 +; CHECK-NEXT: [[Y:%.*]] = or i8 [[Y_IN:%.*]], -8 +; CHECK-NEXT: [[XF:%.*]] = sitofp i8 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = sitofp i8 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %xx = and i8 %x_in, 6 + %x = add nsw nuw i8 %xx, 1 + %y = or i8 %y_in, -8 + %xf = sitofp i8 %x to half + %yf = sitofp i8 %y to half + %r = fmul half %xf, %yf + ret half %r +} + +define half @test_si_si_i8_mul_fail_maybe_zero(i8 noundef %x_in, i8 noundef %y_in) { +; CHECK-LABEL: @test_si_si_i8_mul_fail_maybe_zero( +; CHECK-NEXT: [[X:%.*]] = and i8 [[X_IN:%.*]], 7 +; CHECK-NEXT: [[Y:%.*]] = or i8 [[Y_IN:%.*]], -8 +; CHECK-NEXT: [[XF:%.*]] = sitofp i8 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = sitofp i8 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = and i8 %x_in, 7 + %y = or i8 %y_in, -8 + %xf = sitofp i8 %x to half + %yf = sitofp i8 %y to half + %r = fmul half %xf, %yf + ret half %r +} + +define half @test_si_si_i8_mul_C_fail_no_repr(i8 noundef %x_in) { +; CHECK-LABEL: @test_si_si_i8_mul_C_fail_no_repr( +; CHECK-NEXT: [[XX:%.*]] = and i8 [[X_IN:%.*]], 6 +; CHECK-NEXT: [[X:%.*]] = or disjoint i8 [[XX]], 1 +; CHECK-NEXT: [[XF:%.*]] = sitofp i8 [[X]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], 0xHC780 +; CHECK-NEXT: ret half [[R]] +; + %xx = and i8 %x_in, 6 + %x = add nsw nuw i8 %xx, 1 + %xf = sitofp i8 %x to half + %r = fmul half %xf, -7.5 + ret half %r +} + +define half @test_si_si_i8_mul_C_fail_overflow(i8 noundef %x_in) { +; CHECK-LABEL: @test_si_si_i8_mul_C_fail_overflow( +; CHECK-NEXT: [[XX:%.*]] = and i8 [[X_IN:%.*]], 6 +; CHECK-NEXT: [[X:%.*]] = or disjoint i8 [[XX]], 1 +; CHECK-NEXT: [[XF:%.*]] = sitofp i8 [[X]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], 0xHCCC0 +; CHECK-NEXT: ret half [[R]] +; + %xx = and i8 %x_in, 6 + %x = add nsw nuw i8 %xx, 1 + %xf = sitofp i8 %x to half + %r = fmul half %xf, -19.0 + ret half %r +} + +define half @test_ui_si_i8_mul(i8 noundef %x_in, i8 noundef %y_in) { +; CHECK-LABEL: @test_ui_si_i8_mul( +; CHECK-NEXT: [[XX:%.*]] = and i8 [[X_IN:%.*]], 6 +; CHECK-NEXT: [[X:%.*]] = or disjoint i8 [[XX]], 1 +; CHECK-NEXT: [[YY:%.*]] = and i8 [[Y_IN:%.*]], 7 +; CHECK-NEXT: [[Y:%.*]] = add nuw nsw i8 [[YY]], 1 +; CHECK-NEXT: [[XF:%.*]] = sitofp i8 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = uitofp i8 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %xx = and i8 %x_in, 6 + %x = add i8 %xx, 1 + %yy = and i8 %y_in, 7 + %y = add i8 %yy, 1 + %xf = sitofp i8 %x to half + %yf = uitofp i8 %y to half + %r = fmul half %xf, %yf + ret half %r +} + +define half @test_ui_si_i8_mul_fail_maybe_zero(i8 noundef %x_in, i8 noundef %y_in) { +; CHECK-LABEL: @test_ui_si_i8_mul_fail_maybe_zero( +; CHECK-NEXT: [[XX:%.*]] = and i8 [[X_IN:%.*]], 7 +; CHECK-NEXT: [[X:%.*]] = add nuw nsw i8 [[XX]], 1 +; CHECK-NEXT: [[Y:%.*]] = and i8 [[Y_IN:%.*]], 7 +; CHECK-NEXT: [[XF:%.*]] = sitofp i8 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = uitofp i8 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %xx = and i8 %x_in, 7 + %x = add i8 %xx, 1 + %y = and i8 %y_in, 7 + %xf = sitofp i8 %x to half + %yf = uitofp i8 %y to half + %r = fmul half %xf, %yf + ret half %r +} + +define half @test_ui_si_i8_mul_fail_signed(i8 noundef %x_in, i8 noundef %y_in) { +; CHECK-LABEL: @test_ui_si_i8_mul_fail_signed( +; CHECK-NEXT: [[XX:%.*]] = and i8 [[X_IN:%.*]], 7 +; CHECK-NEXT: [[X:%.*]] = add nuw nsw i8 [[XX]], 1 +; CHECK-NEXT: [[Y:%.*]] = or i8 [[Y_IN:%.*]], -4 +; CHECK-NEXT: [[XF:%.*]] = sitofp i8 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = uitofp i8 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %xx = and i8 %x_in, 7 + %x = add i8 %xx, 1 + %y = or i8 %y_in, -4 + %xf = sitofp i8 %x to half + %yf = uitofp i8 %y to half + %r = fmul half %xf, %yf + ret half %r +} + +define half @test_ui_ui_i16_add(i16 noundef %x_in, i16 noundef %y_in) { +; CHECK-LABEL: @test_ui_ui_i16_add( +; CHECK-NEXT: [[X:%.*]] = and i16 [[X_IN:%.*]], 2047 +; CHECK-NEXT: [[Y:%.*]] = and i16 [[Y_IN:%.*]], 2047 +; CHECK-NEXT: [[XF:%.*]] = uitofp i16 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = uitofp i16 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = and i16 %x_in, 2047 + %y = and i16 %y_in, 2047 + %xf = uitofp i16 %x to half + %yf = uitofp i16 %y to half + %r = fadd half %xf, %yf + ret half %r +} + +define half @test_ui_ui_i16_add_fail_not_promotable(i16 noundef %x_in, i16 noundef %y_in) { +; CHECK-LABEL: @test_ui_ui_i16_add_fail_not_promotable( +; CHECK-NEXT: [[X:%.*]] = and i16 [[X_IN:%.*]], 2049 +; CHECK-NEXT: [[Y:%.*]] = and i16 [[Y_IN:%.*]], 2047 +; CHECK-NEXT: [[XF:%.*]] = uitofp i16 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = uitofp i16 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = and i16 %x_in, 2049 + %y = and i16 %y_in, 2047 + %xf = uitofp i16 %x to half + %yf = uitofp i16 %y to half + %r = fadd half %xf, %yf + ret half %r +} + +define half @test_ui_ui_i16_add_C(i16 noundef %x_in) { +; CHECK-LABEL: @test_ui_ui_i16_add_C( +; CHECK-NEXT: [[X:%.*]] = and i16 [[X_IN:%.*]], 2047 +; CHECK-NEXT: [[XF:%.*]] = uitofp i16 [[X]] to half +; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], 0xH7BC0 +; CHECK-NEXT: ret half [[R]] +; + %x = and i16 %x_in, 2047 + %xf = uitofp i16 %x to half + %r = fadd half %xf, 63488.0 + ret half %r +} + +define half @test_ui_ui_i16_add_C_fail_overflow(i16 noundef %x_in) { +; CHECK-LABEL: @test_ui_ui_i16_add_C_fail_overflow( +; CHECK-NEXT: [[X:%.*]] = and i16 [[X_IN:%.*]], 2047 +; CHECK-NEXT: [[XF:%.*]] = uitofp i16 [[X]] to half +; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], 0xH7BD0 +; CHECK-NEXT: ret half [[R]] +; + %x = and i16 %x_in, 2047 + %xf = uitofp i16 %x to half + %r = fadd half %xf, 64000.0 + ret half %r +} + +define half @test_si_si_i16_add(i16 noundef %x_in, i16 noundef %y_in) { +; CHECK-LABEL: @test_si_si_i16_add( +; CHECK-NEXT: [[X:%.*]] = or i16 [[X_IN:%.*]], -2048 +; CHECK-NEXT: [[Y:%.*]] = or i16 [[Y_IN:%.*]], -2048 +; CHECK-NEXT: [[XF:%.*]] = sitofp i16 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = sitofp i16 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = or i16 %x_in, -2048 + %y = or i16 %y_in, -2048 + %xf = sitofp i16 %x to half + %yf = sitofp i16 %y to half + %r = fadd half %xf, %yf + ret half %r +} + +define half @test_si_si_i16_add_fail_no_promotion(i16 noundef %x_in, i16 noundef %y_in) { +; CHECK-LABEL: @test_si_si_i16_add_fail_no_promotion( +; CHECK-NEXT: [[XX:%.*]] = or i16 [[X_IN:%.*]], -2048 +; CHECK-NEXT: [[X:%.*]] = add nsw i16 [[XX]], -1 +; CHECK-NEXT: [[Y:%.*]] = or i16 [[Y_IN:%.*]], -2048 +; CHECK-NEXT: [[XF:%.*]] = sitofp i16 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = sitofp i16 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %xx = or i16 %x_in, -2048 + %x = sub i16 %xx, 1 + %y = or i16 %y_in, -2048 + %xf = sitofp i16 %x to half + %yf = sitofp i16 %y to half + %r = fadd half %xf, %yf + ret half %r +} + +define half @test_si_si_i16_add_C_overflow(i16 noundef %x_in) { +; CHECK-LABEL: @test_si_si_i16_add_C_overflow( +; CHECK-NEXT: [[X:%.*]] = or i16 [[X_IN:%.*]], -2048 +; CHECK-NEXT: [[XF:%.*]] = sitofp i16 [[X]] to half +; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], 0xH7840 +; CHECK-NEXT: ret half [[R]] +; + %x = or i16 %x_in, -2048 + %xf = sitofp i16 %x to half + %r = fadd half %xf, 0xH7840 + ret half %r +} + +define half @test_si_si_i16_sub(i16 noundef %x_in, i16 noundef %y_in) { +; CHECK-LABEL: @test_si_si_i16_sub( +; CHECK-NEXT: [[X:%.*]] = or i16 [[X_IN:%.*]], -2048 +; CHECK-NEXT: [[Y:%.*]] = and i16 [[Y_IN:%.*]], 2047 +; CHECK-NEXT: [[XF:%.*]] = sitofp i16 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = sitofp i16 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fsub half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = or i16 %x_in, -2048 + %y = and i16 %y_in, 2047 + %xf = sitofp i16 %x to half + %yf = sitofp i16 %y to half + %r = fsub half %xf, %yf + ret half %r +} + +define half @test_si_si_i16_sub_fail_no_promotion(i16 noundef %x_in, i16 noundef %y_in) { +; CHECK-LABEL: @test_si_si_i16_sub_fail_no_promotion( +; CHECK-NEXT: [[X:%.*]] = and i16 [[X_IN:%.*]], 2047 +; CHECK-NEXT: [[Y:%.*]] = or i16 [[Y_IN:%.*]], -2049 +; CHECK-NEXT: [[XF:%.*]] = sitofp i16 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = sitofp i16 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fsub half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = and i16 %x_in, 2047 + %y = or i16 %y_in, -2049 + %xf = sitofp i16 %x to half + %yf = sitofp i16 %y to half + %r = fsub half %xf, %yf + ret half %r +} + +define half @test_ui_si_i16_sub(i16 noundef %x_in, i16 noundef %y_in) { +; CHECK-LABEL: @test_ui_si_i16_sub( +; CHECK-NEXT: [[X:%.*]] = and i16 [[X_IN:%.*]], 2047 +; CHECK-NEXT: [[Y:%.*]] = and i16 [[Y_IN:%.*]], 2047 +; CHECK-NEXT: [[XF:%.*]] = uitofp i16 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = sitofp i16 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fsub half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = and i16 %x_in, 2047 + %y = and i16 %y_in, 2047 + %xf = uitofp i16 %x to half + %yf = sitofp i16 %y to half + %r = fsub half %xf, %yf + ret half %r +} + +define half @test_ui_si_i16_sub_fail_maybe_signed(i16 noundef %x_in, i16 noundef %y_in) { +; CHECK-LABEL: @test_ui_si_i16_sub_fail_maybe_signed( +; CHECK-NEXT: [[X:%.*]] = or i16 [[X_IN:%.*]], -2048 +; CHECK-NEXT: [[Y:%.*]] = and i16 [[Y_IN:%.*]], 2047 +; CHECK-NEXT: [[XF:%.*]] = uitofp i16 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = sitofp i16 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fsub half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = or i16 %x_in, -2048 + %y = and i16 %y_in, 2047 + %xf = uitofp i16 %x to half + %yf = sitofp i16 %y to half + %r = fsub half %xf, %yf + ret half %r +} + +define half @test_ui_ui_i16_mul(i16 noundef %x_in, i16 noundef %y_in) { +; CHECK-LABEL: @test_ui_ui_i16_mul( +; CHECK-NEXT: [[X:%.*]] = and i16 [[X_IN:%.*]], 255 +; CHECK-NEXT: [[Y:%.*]] = and i16 [[Y_IN:%.*]], 255 +; CHECK-NEXT: [[XF:%.*]] = uitofp i16 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = uitofp i16 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = and i16 %x_in, 255 + %y = and i16 %y_in, 255 + %xf = uitofp i16 %x to half + %yf = uitofp i16 %y to half + %r = fmul half %xf, %yf + ret half %r +} + +define half @test_ui_ui_i16_mul_fail_no_promotion(i16 noundef %x_in, i16 noundef %y_in) { +; CHECK-LABEL: @test_ui_ui_i16_mul_fail_no_promotion( +; CHECK-NEXT: [[X:%.*]] = and i16 [[X_IN:%.*]], 4095 +; CHECK-NEXT: [[Y:%.*]] = and i16 [[Y_IN:%.*]], 3 +; CHECK-NEXT: [[XF:%.*]] = uitofp i16 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = uitofp i16 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = and i16 %x_in, 4095 + %y = and i16 %y_in, 3 + %xf = uitofp i16 %x to half + %yf = uitofp i16 %y to half + %r = fmul half %xf, %yf + ret half %r +} + +define half @test_si_si_i16_mul(i16 noundef %x_in, i16 noundef %y_in) { +; CHECK-LABEL: @test_si_si_i16_mul( +; CHECK-NEXT: [[XX:%.*]] = and i16 [[X_IN:%.*]], 126 +; CHECK-NEXT: [[X:%.*]] = or disjoint i16 [[XX]], 1 +; CHECK-NEXT: [[Y:%.*]] = or i16 [[Y_IN:%.*]], -255 +; CHECK-NEXT: [[XF:%.*]] = sitofp i16 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = sitofp i16 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %xx = and i16 %x_in, 126 + %x = add nsw nuw i16 %xx, 1 + %y = or i16 %y_in, -255 + %xf = sitofp i16 %x to half + %yf = sitofp i16 %y to half + %r = fmul half %xf, %yf + ret half %r +} + +define half @test_si_si_i16_mul_fail_overflow(i16 noundef %x_in, i16 noundef %y_in) { +; CHECK-LABEL: @test_si_si_i16_mul_fail_overflow( +; CHECK-NEXT: [[XX:%.*]] = and i16 [[X_IN:%.*]], 126 +; CHECK-NEXT: [[X:%.*]] = or disjoint i16 [[XX]], 1 +; CHECK-NEXT: [[Y:%.*]] = or i16 [[Y_IN:%.*]], -257 +; CHECK-NEXT: [[XF:%.*]] = sitofp i16 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = sitofp i16 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %xx = and i16 %x_in, 126 + %x = add nsw nuw i16 %xx, 1 + %y = or i16 %y_in, -257 + %xf = sitofp i16 %x to half + %yf = sitofp i16 %y to half + %r = fmul half %xf, %yf + ret half %r +} + +define half @test_si_si_i16_mul_C_fail_overflow(i16 noundef %x_in) { +; CHECK-LABEL: @test_si_si_i16_mul_C_fail_overflow( +; CHECK-NEXT: [[X:%.*]] = or i16 [[X_IN:%.*]], -129 +; CHECK-NEXT: [[XF:%.*]] = sitofp i16 [[X]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], 0xH5800 +; CHECK-NEXT: ret half [[R]] +; + %x = or i16 %x_in, -129 + %xf = sitofp i16 %x to half + %r = fmul half %xf, 128.0 + ret half %r +} + +define half @test_si_si_i16_mul_C_fail_no_promotion(i16 noundef %x_in) { +; CHECK-LABEL: @test_si_si_i16_mul_C_fail_no_promotion( +; CHECK-NEXT: [[X:%.*]] = or i16 [[X_IN:%.*]], -4097 +; CHECK-NEXT: [[XF:%.*]] = sitofp i16 [[X]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], 0xH4500 +; CHECK-NEXT: ret half [[R]] +; + %x = or i16 %x_in, -4097 + %xf = sitofp i16 %x to half + %r = fmul half %xf, 5.0 + ret half %r +} + +define half @test_ui_si_i16_mul(i16 noundef %x_in, i16 noundef %y_in) { +; CHECK-LABEL: @test_ui_si_i16_mul( +; CHECK-NEXT: [[XX:%.*]] = and i16 [[X_IN:%.*]], 126 +; CHECK-NEXT: [[X:%.*]] = or disjoint i16 [[XX]], 1 +; CHECK-NEXT: [[YY:%.*]] = and i16 [[Y_IN:%.*]], 126 +; CHECK-NEXT: [[Y:%.*]] = or disjoint i16 [[YY]], 1 +; CHECK-NEXT: [[XF:%.*]] = sitofp i16 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = uitofp i16 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %xx = and i16 %x_in, 126 + %x = add i16 %xx, 1 + %yy = and i16 %y_in, 126 + %y = add i16 %yy, 1 + %xf = sitofp i16 %x to half + %yf = uitofp i16 %y to half + %r = fmul half %xf, %yf + ret half %r +} + +define half @test_ui_ui_i12_add(i12 noundef %x_in, i12 noundef %y_in) { +; CHECK-LABEL: @test_ui_ui_i12_add( +; CHECK-NEXT: [[X:%.*]] = and i12 [[X_IN:%.*]], 2047 +; CHECK-NEXT: [[Y:%.*]] = and i12 [[Y_IN:%.*]], 2047 +; CHECK-NEXT: [[XF:%.*]] = uitofp i12 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = uitofp i12 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = and i12 %x_in, 2047 + %y = and i12 %y_in, 2047 + %xf = uitofp i12 %x to half + %yf = uitofp i12 %y to half + %r = fadd half %xf, %yf + ret half %r +} + +define half @test_ui_ui_i12_add_fail_overflow(i12 noundef %x_in, i12 noundef %y_in) { +; CHECK-LABEL: @test_ui_ui_i12_add_fail_overflow( +; CHECK-NEXT: [[X:%.*]] = and i12 [[X_IN:%.*]], 2047 +; CHECK-NEXT: [[Y:%.*]] = and i12 [[Y_IN:%.*]], -2047 +; CHECK-NEXT: [[XF:%.*]] = uitofp i12 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = uitofp i12 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = and i12 %x_in, 2047 + %y = and i12 %y_in, 2049 + %xf = uitofp i12 %x to half + %yf = uitofp i12 %y to half + %r = fadd half %xf, %yf + ret half %r +} + + +define half @test_si_si_i12_add(i12 noundef %x_in, i12 noundef %y_in) { +; CHECK-LABEL: @test_si_si_i12_add( +; CHECK-NEXT: [[X:%.*]] = or i12 [[X_IN:%.*]], -1024 +; CHECK-NEXT: [[Y:%.*]] = or i12 [[Y_IN:%.*]], -1024 +; CHECK-NEXT: [[XF:%.*]] = sitofp i12 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = sitofp i12 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = or i12 %x_in, -1024 + %y = or i12 %y_in, -1024 + %xf = sitofp i12 %x to half + %yf = sitofp i12 %y to half + %r = fadd half %xf, %yf + ret half %r +} + +define half @test_si_si_i12_add_fail_overflow(i12 noundef %x_in, i12 noundef %y_in) { +; CHECK-LABEL: @test_si_si_i12_add_fail_overflow( +; CHECK-NEXT: [[X:%.*]] = or i12 [[X_IN:%.*]], -1025 +; CHECK-NEXT: [[Y:%.*]] = or i12 [[Y_IN:%.*]], -1025 +; CHECK-NEXT: [[XF:%.*]] = sitofp i12 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = sitofp i12 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = or i12 %x_in, -1025 + %y = or i12 %y_in, -1025 + %xf = sitofp i12 %x to half + %yf = sitofp i12 %y to half + %r = fadd half %xf, %yf + ret half %r +} + +define half @test_si_si_i12_add_C_fail_overflow(i12 noundef %x_in) { +; CHECK-LABEL: @test_si_si_i12_add_C_fail_overflow( +; CHECK-NEXT: [[X:%.*]] = or i12 [[X_IN:%.*]], -2048 +; CHECK-NEXT: [[XF:%.*]] = sitofp i12 [[X]] to half +; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], 0xHBC00 +; CHECK-NEXT: ret half [[R]] +; + %x = or i12 %x_in, -2048 + %xf = sitofp i12 %x to half + %r = fadd half %xf, -1.0 + ret half %r +} + +define half @test_ui_ui_i12_sub(i12 noundef %x_in, i12 noundef %y_in) { +; CHECK-LABEL: @test_ui_ui_i12_sub( +; CHECK-NEXT: [[X:%.*]] = and i12 [[X_IN:%.*]], 1023 +; CHECK-NEXT: [[Y:%.*]] = and i12 [[Y_IN:%.*]], 1023 +; CHECK-NEXT: [[XF:%.*]] = uitofp i12 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = uitofp i12 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fsub half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = and i12 %x_in, 1023 + %y = and i12 %y_in, 1023 + %xf = uitofp i12 %x to half + %yf = uitofp i12 %y to half + %r = fsub half %xf, %yf + ret half %r +} + +define half @test_ui_ui_i12_sub_fail_overflow(i12 noundef %x_in, i12 noundef %y_in) { +; CHECK-LABEL: @test_ui_ui_i12_sub_fail_overflow( +; CHECK-NEXT: [[X:%.*]] = and i12 [[X_IN:%.*]], 1023 +; CHECK-NEXT: [[Y:%.*]] = and i12 [[Y_IN:%.*]], 2047 +; CHECK-NEXT: [[XF:%.*]] = uitofp i12 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = uitofp i12 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fsub half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = and i12 %x_in, 1023 + %y = and i12 %y_in, 2047 + %xf = uitofp i12 %x to half + %yf = uitofp i12 %y to half + %r = fsub half %xf, %yf + ret half %r +} + + +define half @test_si_si_i12_sub(i12 noundef %x_in, i12 noundef %y_in) { +; CHECK-LABEL: @test_si_si_i12_sub( +; CHECK-NEXT: [[X:%.*]] = and i12 [[X_IN:%.*]], 1023 +; CHECK-NEXT: [[Y:%.*]] = or i12 [[Y_IN:%.*]], -1024 +; CHECK-NEXT: [[XF:%.*]] = sitofp i12 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = sitofp i12 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fsub half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = and i12 %x_in, 1023 + %y = or i12 %y_in, -1024 + %xf = sitofp i12 %x to half + %yf = sitofp i12 %y to half + %r = fsub half %xf, %yf + ret half %r +} + +define half @test_si_si_i12_sub_fail_overflow(i12 noundef %x, i12 noundef %y) { +; CHECK-LABEL: @test_si_si_i12_sub_fail_overflow( +; CHECK-NEXT: [[XF:%.*]] = sitofp i12 [[X:%.*]] to half +; CHECK-NEXT: [[YF:%.*]] = sitofp i12 [[Y:%.*]] to half +; CHECK-NEXT: [[R:%.*]] = fsub half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %xf = sitofp i12 %x to half + %yf = sitofp i12 %y to half + %r = fsub half %xf, %yf + ret half %r +} + +define half @test_ui_ui_i12_mul(i12 noundef %x_in, i12 noundef %y_in) { +; CHECK-LABEL: @test_ui_ui_i12_mul( +; CHECK-NEXT: [[X:%.*]] = and i12 [[X_IN:%.*]], 31 +; CHECK-NEXT: [[Y:%.*]] = and i12 [[Y_IN:%.*]], 63 +; CHECK-NEXT: [[XF:%.*]] = uitofp i12 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = uitofp i12 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = and i12 %x_in, 31 + %y = and i12 %y_in, 63 + %xf = uitofp i12 %x to half + %yf = uitofp i12 %y to half + %r = fmul half %xf, %yf + ret half %r +} + +define half @test_ui_ui_i12_mul_fail_overflow(i12 noundef %x_in, i12 noundef %y_in) { +; CHECK-LABEL: @test_ui_ui_i12_mul_fail_overflow( +; CHECK-NEXT: [[XX:%.*]] = and i12 [[X_IN:%.*]], 31 +; CHECK-NEXT: [[X:%.*]] = add nuw nsw i12 [[XX]], 1 +; CHECK-NEXT: [[Y:%.*]] = and i12 [[Y_IN:%.*]], 63 +; CHECK-NEXT: [[XF:%.*]] = uitofp i12 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = uitofp i12 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %xx = and i12 %x_in, 31 + %x = add i12 %xx, 1 + %y = and i12 %y_in, 63 + %xf = uitofp i12 %x to half + %yf = uitofp i12 %y to half + %r = fmul half %xf, %yf + ret half %r +} + +define half @test_ui_ui_i12_mul_C(i12 noundef %x_in) { +; CHECK-LABEL: @test_ui_ui_i12_mul_C( +; CHECK-NEXT: [[X:%.*]] = and i12 [[X_IN:%.*]], 31 +; CHECK-NEXT: [[XF:%.*]] = uitofp i12 [[X]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], 0xH5400 +; CHECK-NEXT: ret half [[R]] +; + %x = and i12 %x_in, 31 + %xf = uitofp i12 %x to half + %r = fmul half %xf, 64.0 + ret half %r +} + +define half @test_si_si_i12_mul(i12 noundef %x_in, i12 noundef %y_in) { +; CHECK-LABEL: @test_si_si_i12_mul( +; CHECK-NEXT: [[XX:%.*]] = and i12 [[X_IN:%.*]], 30 +; CHECK-NEXT: [[X:%.*]] = or disjoint i12 [[XX]], 1 +; CHECK-NEXT: [[Y:%.*]] = or i12 [[Y_IN:%.*]], -64 +; CHECK-NEXT: [[XF:%.*]] = sitofp i12 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = sitofp i12 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %xx = and i12 %x_in, 30 + %x = add nsw nuw i12 %xx, 1 + %y = or i12 %y_in, -64 + %xf = sitofp i12 %x to half + %yf = sitofp i12 %y to half + %r = fmul half %xf, %yf + ret half %r +} + +define half @test_si_si_i12_mul_fail_overflow(i12 noundef %x_in, i12 noundef %y_in) { +; CHECK-LABEL: @test_si_si_i12_mul_fail_overflow( +; CHECK-NEXT: [[XX:%.*]] = and i12 [[X_IN:%.*]], 30 +; CHECK-NEXT: [[X:%.*]] = or disjoint i12 [[XX]], 1 +; CHECK-NEXT: [[Y:%.*]] = or i12 [[Y_IN:%.*]], -128 +; CHECK-NEXT: [[XF:%.*]] = sitofp i12 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = sitofp i12 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %xx = and i12 %x_in, 30 + %x = add nsw nuw i12 %xx, 1 + %y = or i12 %y_in, -128 + %xf = sitofp i12 %x to half + %yf = sitofp i12 %y to half + %r = fmul half %xf, %yf + ret half %r +} + +define half @test_si_si_i12_mul_fail_maybe_non_zero(i12 noundef %x_in, i12 noundef %y_in) { +; CHECK-LABEL: @test_si_si_i12_mul_fail_maybe_non_zero( +; CHECK-NEXT: [[X:%.*]] = and i12 [[X_IN:%.*]], 30 +; CHECK-NEXT: [[Y:%.*]] = or i12 [[Y_IN:%.*]], -128 +; CHECK-NEXT: [[XF:%.*]] = sitofp i12 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = sitofp i12 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %x = and i12 %x_in, 30 + %y = or i12 %y_in, -128 + %xf = sitofp i12 %x to half + %yf = sitofp i12 %y to half + %r = fmul half %xf, %yf + ret half %r +} + +define half @test_si_si_i12_mul_C(i12 noundef %x_in) { +; CHECK-LABEL: @test_si_si_i12_mul_C( +; CHECK-NEXT: [[X:%.*]] = or i12 [[X_IN:%.*]], -64 +; CHECK-NEXT: [[XF:%.*]] = sitofp i12 [[X]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], 0xHCC00 +; CHECK-NEXT: ret half [[R]] +; + %x = or i12 %x_in, -64 + %xf = sitofp i12 %x to half + %r = fmul half %xf, -16.0 + ret half %r +} + +define half @test_si_si_i12_mul_C_fail_overflow(i12 noundef %x_in) { +; CHECK-LABEL: @test_si_si_i12_mul_C_fail_overflow( +; CHECK-NEXT: [[X:%.*]] = or i12 [[X_IN:%.*]], -64 +; CHECK-NEXT: [[XF:%.*]] = sitofp i12 [[X]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], 0xHD400 +; CHECK-NEXT: ret half [[R]] +; + %x = or i12 %x_in, -64 + %xf = sitofp i12 %x to half + %r = fmul half %xf, -64.0 + ret half %r +} + +define half @test_ui_si_i12_mul_nsw(i12 noundef %x_in, i12 noundef %y_in) { +; CHECK-LABEL: @test_ui_si_i12_mul_nsw( +; CHECK-NEXT: [[XX:%.*]] = and i12 [[X_IN:%.*]], 31 +; CHECK-NEXT: [[X:%.*]] = add nuw nsw i12 [[XX]], 1 +; CHECK-NEXT: [[YY:%.*]] = and i12 [[Y_IN:%.*]], 30 +; CHECK-NEXT: [[Y:%.*]] = or disjoint i12 [[YY]], 1 +; CHECK-NEXT: [[XF:%.*]] = uitofp i12 [[X]] to half +; CHECK-NEXT: [[YF:%.*]] = sitofp i12 [[Y]] to half +; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: ret half [[R]] +; + %xx = and i12 %x_in, 31 + %x = add i12 %xx, 1 + %yy = and i12 %y_in, 30 + %y = add i12 %yy, 1 + %xf = uitofp i12 %x to half + %yf = sitofp i12 %y to half + %r = fmul half %xf, %yf + ret half %r +} -- GitLab From 0f5849eeeebc410e420ee11b7e59b4dd28c65318 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Wed, 21 Feb 2024 16:13:01 -0600 Subject: [PATCH 343/929] [InstCombine] Move folding `(add (sitofp x), (sitofp y))` impl to InstructionCombiner; NFC --- .../InstCombine/InstCombineAddSub.cpp | 60 +--------------- .../InstCombine/InstCombineInternal.h | 1 + .../InstCombine/InstructionCombining.cpp | 68 +++++++++++++++++++ .../test/Transforms/InstCombine/add-sitofp.ll | 16 ++--- .../Transforms/InstCombine/binop-itofp.ll | 8 +-- 5 files changed, 84 insertions(+), 69 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp index 36a5faa5f674..770df1093df0 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp @@ -1867,64 +1867,10 @@ Instruction *InstCombinerImpl::visitFAdd(BinaryOperator &I) { // Check for (fadd double (sitofp x), y), see if we can merge this into an // integer add followed by a promotion. - Value *LHS = I.getOperand(0), *RHS = I.getOperand(1); - if (SIToFPInst *LHSConv = dyn_cast(LHS)) { - Value *LHSIntVal = LHSConv->getOperand(0); - Type *FPType = LHSConv->getType(); - - // TODO: This check is overly conservative. In many cases known bits - // analysis can tell us that the result of the addition has less significant - // bits than the integer type can hold. - auto IsValidPromotion = [](Type *FTy, Type *ITy) { - Type *FScalarTy = FTy->getScalarType(); - Type *IScalarTy = ITy->getScalarType(); - - // Do we have enough bits in the significand to represent the result of - // the integer addition? - unsigned MaxRepresentableBits = - APFloat::semanticsPrecision(FScalarTy->getFltSemantics()); - return IScalarTy->getIntegerBitWidth() <= MaxRepresentableBits; - }; - - // (fadd double (sitofp x), fpcst) --> (sitofp (add int x, intcst)) - // ... if the constant fits in the integer value. This is useful for things - // like (double)(x & 1234) + 4.0 -> (double)((X & 1234)+4) which no longer - // requires a constant pool load, and generally allows the add to be better - // instcombined. - if (ConstantFP *CFP = dyn_cast(RHS)) - if (IsValidPromotion(FPType, LHSIntVal->getType())) { - Constant *CI = ConstantFoldCastOperand(Instruction::FPToSI, CFP, - LHSIntVal->getType(), DL); - if (LHSConv->hasOneUse() && - ConstantFoldCastOperand(Instruction::SIToFP, CI, I.getType(), DL) == - CFP && - willNotOverflowSignedAdd(LHSIntVal, CI, I)) { - // Insert the new integer add. - Value *NewAdd = Builder.CreateNSWAdd(LHSIntVal, CI, "addconv"); - return new SIToFPInst(NewAdd, I.getType()); - } - } - - // (fadd double (sitofp x), (sitofp y)) --> (sitofp (add int x, y)) - if (SIToFPInst *RHSConv = dyn_cast(RHS)) { - Value *RHSIntVal = RHSConv->getOperand(0); - // It's enough to check LHS types only because we require int types to - // be the same for this transform. - if (IsValidPromotion(FPType, LHSIntVal->getType())) { - // Only do this if x/y have the same type, if at least one of them has a - // single use (so we don't increase the number of int->fp conversions), - // and if the integer add will not overflow. - if (LHSIntVal->getType() == RHSIntVal->getType() && - (LHSConv->hasOneUse() || RHSConv->hasOneUse()) && - willNotOverflowSignedAdd(LHSIntVal, RHSIntVal, I)) { - // Insert the new integer add. - Value *NewAdd = Builder.CreateNSWAdd(LHSIntVal, RHSIntVal, "addconv"); - return new SIToFPInst(NewAdd, I.getType()); - } - } - } - } + if (Instruction *R = foldFBinOpOfIntCasts(I)) + return R; + Value *LHS = I.getOperand(0), *RHS = I.getOperand(1); // Handle specials cases for FAdd with selects feeding the operation if (Value *V = SimplifySelectsFeedingBinaryOp(I, LHS, RHS)) return replaceInstUsesWith(I, V); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h index 0b4283bc3765..57148d719d9b 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h +++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h @@ -379,6 +379,7 @@ private: Instruction *scalarizePHI(ExtractElementInst &EI, PHINode *PN); Instruction *foldBitcastExtElt(ExtractElementInst &ExtElt); Instruction *foldCastedBitwiseLogic(BinaryOperator &I); + Instruction *foldFBinOpOfIntCasts(BinaryOperator &I); Instruction *foldBinopOfSextBoolToSelect(BinaryOperator &I); Instruction *narrowBinOp(TruncInst &Trunc); Instruction *narrowMaskedBinOp(BinaryOperator &And); diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index fec33c5ea53b..a22f87e2ba30 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -1401,6 +1401,74 @@ Value *InstCombinerImpl::dyn_castNegVal(Value *V) const { return nullptr; } +// Try to fold: +// 1) (add (sitofp x), (sitofp y)) +// -> (sitofp (add x, y)) +// 2) (add (sitofp x), FpC) +// -> (sitofp (add x, (fptosi FpC))) +Instruction *InstCombinerImpl::foldFBinOpOfIntCasts(BinaryOperator &BO) { + // Check for (fadd double (sitofp x), y), see if we can merge this into an + // integer add followed by a promotion. + Value *LHS = BO.getOperand(0), *RHS = BO.getOperand(1); + if (SIToFPInst *LHSConv = dyn_cast(LHS)) { + Value *LHSIntVal = LHSConv->getOperand(0); + Type *FPType = LHSConv->getType(); + + // TODO: This check is overly conservative. In many cases known bits + // analysis can tell us that the result of the addition has less significant + // bits than the integer type can hold. + auto IsValidPromotion = [](Type *FTy, Type *ITy) { + Type *FScalarTy = FTy->getScalarType(); + Type *IScalarTy = ITy->getScalarType(); + + // Do we have enough bits in the significand to represent the result of + // the integer addition? + unsigned MaxRepresentableBits = + APFloat::semanticsPrecision(FScalarTy->getFltSemantics()); + return IScalarTy->getIntegerBitWidth() <= MaxRepresentableBits; + }; + + // (fadd double (sitofp x), fpcst) --> (sitofp (add int x, intcst)) + // ... if the constant fits in the integer value. This is useful for things + // like (double)(x & 1234) + 4.0 -> (double)((X & 1234)+4) which no longer + // requires a constant pool load, and generally allows the add to be better + // instcombined. + if (ConstantFP *CFP = dyn_cast(RHS)) + if (IsValidPromotion(FPType, LHSIntVal->getType())) { + Constant *CI = ConstantFoldCastOperand(Instruction::FPToSI, CFP, + LHSIntVal->getType(), DL); + if (LHSConv->hasOneUse() && + ConstantFoldCastOperand(Instruction::SIToFP, CI, BO.getType(), + DL) == CFP && + willNotOverflowSignedAdd(LHSIntVal, CI, BO)) { + // Insert the new integer add. + Value *NewAdd = Builder.CreateNSWAdd(LHSIntVal, CI); + return new SIToFPInst(NewAdd, BO.getType()); + } + } + + // (fadd double (sitofp x), (sitofp y)) --> (sitofp (add int x, y)) + if (SIToFPInst *RHSConv = dyn_cast(RHS)) { + Value *RHSIntVal = RHSConv->getOperand(0); + // It's enough to check LHS types only because we require int types to + // be the same for this transform. + if (IsValidPromotion(FPType, LHSIntVal->getType())) { + // Only do this if x/y have the same type, if at least one of them has a + // single use (so we don't increase the number of int->fp conversions), + // and if the integer add will not overflow. + if (LHSIntVal->getType() == RHSIntVal->getType() && + (LHSConv->hasOneUse() || RHSConv->hasOneUse()) && + willNotOverflowSignedAdd(LHSIntVal, RHSIntVal, BO)) { + // Insert the new integer add. + Value *NewAdd = Builder.CreateNSWAdd(LHSIntVal, RHSIntVal); + return new SIToFPInst(NewAdd, BO.getType()); + } + } + } + } + return nullptr; +} + /// A binop with a constant operand and a sign-extended boolean operand may be /// converted into a select of constants by applying the binary operation to /// the constant with the two possible values of the extended boolean (0 or -1). diff --git a/llvm/test/Transforms/InstCombine/add-sitofp.ll b/llvm/test/Transforms/InstCombine/add-sitofp.ll index db44b806593b..206c0a7ebd26 100644 --- a/llvm/test/Transforms/InstCombine/add-sitofp.ll +++ b/llvm/test/Transforms/InstCombine/add-sitofp.ll @@ -5,8 +5,8 @@ define double @x(i32 %a, i32 %b) { ; CHECK-LABEL: @x( ; CHECK-NEXT: [[M:%.*]] = lshr i32 [[A:%.*]], 24 ; CHECK-NEXT: [[N:%.*]] = and i32 [[M]], [[B:%.*]] -; CHECK-NEXT: [[ADDCONV:%.*]] = add nuw nsw i32 [[N]], 1 -; CHECK-NEXT: [[P:%.*]] = sitofp i32 [[ADDCONV]] to double +; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i32 [[N]], 1 +; CHECK-NEXT: [[P:%.*]] = sitofp i32 [[TMP1]] to double ; CHECK-NEXT: ret double [[P]] ; %m = lshr i32 %a, 24 @@ -19,8 +19,8 @@ define double @x(i32 %a, i32 %b) { define double @test(i32 %a) { ; CHECK-LABEL: @test( ; CHECK-NEXT: [[A_AND:%.*]] = and i32 [[A:%.*]], 1073741823 -; CHECK-NEXT: [[ADDCONV:%.*]] = add nuw nsw i32 [[A_AND]], 1 -; CHECK-NEXT: [[RES:%.*]] = sitofp i32 [[ADDCONV]] to double +; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i32 [[A_AND]], 1 +; CHECK-NEXT: [[RES:%.*]] = sitofp i32 [[TMP1]] to double ; CHECK-NEXT: ret double [[RES]] ; ; Drop two highest bits to guarantee that %a + 1 doesn't overflow @@ -48,8 +48,8 @@ define double @test_2(i32 %a, i32 %b) { ; CHECK-LABEL: @test_2( ; CHECK-NEXT: [[A_AND:%.*]] = and i32 [[A:%.*]], 1073741823 ; CHECK-NEXT: [[B_AND:%.*]] = and i32 [[B:%.*]], 1073741823 -; CHECK-NEXT: [[ADDCONV:%.*]] = add nuw nsw i32 [[A_AND]], [[B_AND]] -; CHECK-NEXT: [[RES:%.*]] = sitofp i32 [[ADDCONV]] to double +; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i32 [[A_AND]], [[B_AND]] +; CHECK-NEXT: [[RES:%.*]] = sitofp i32 [[TMP1]] to double ; CHECK-NEXT: ret double [[RES]] ; ; Drop two highest bits to guarantee that %a + %b doesn't overflow @@ -105,8 +105,8 @@ define <4 x double> @test_4(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: @test_4( ; CHECK-NEXT: [[A_AND:%.*]] = and <4 x i32> [[A:%.*]], ; CHECK-NEXT: [[B_AND:%.*]] = and <4 x i32> [[B:%.*]], -; CHECK-NEXT: [[ADDCONV:%.*]] = add nuw nsw <4 x i32> [[A_AND]], [[B_AND]] -; CHECK-NEXT: [[RES:%.*]] = sitofp <4 x i32> [[ADDCONV]] to <4 x double> +; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw <4 x i32> [[A_AND]], [[B_AND]] +; CHECK-NEXT: [[RES:%.*]] = sitofp <4 x i32> [[TMP1]] to <4 x double> ; CHECK-NEXT: ret <4 x double> [[RES]] ; ; Drop two highest bits to guarantee that %a + %b doesn't overflow diff --git a/llvm/test/Transforms/InstCombine/binop-itofp.ll b/llvm/test/Transforms/InstCombine/binop-itofp.ll index 6354be7d723b..ea1ad9f8d881 100644 --- a/llvm/test/Transforms/InstCombine/binop-itofp.ll +++ b/llvm/test/Transforms/InstCombine/binop-itofp.ll @@ -78,8 +78,8 @@ define half @test_si_si_i8_add(i8 noundef %x_in, i8 noundef %y_in) { ; CHECK-LABEL: @test_si_si_i8_add( ; CHECK-NEXT: [[X:%.*]] = or i8 [[X_IN:%.*]], -64 ; CHECK-NEXT: [[Y:%.*]] = or i8 [[Y_IN:%.*]], -64 -; CHECK-NEXT: [[ADDCONV:%.*]] = add nsw i8 [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = sitofp i8 [[ADDCONV]] to half +; CHECK-NEXT: [[TMP1:%.*]] = add nsw i8 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = sitofp i8 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %x = or i8 %x_in, -64 @@ -204,8 +204,8 @@ define half @test_si_si_i8_sub_fail_overflow(i8 noundef %x_in, i8 noundef %y_in) define half @test_si_si_i8_sub_C(i8 noundef %x_in) { ; CHECK-LABEL: @test_si_si_i8_sub_C( ; CHECK-NEXT: [[X:%.*]] = and i8 [[X_IN:%.*]], 63 -; CHECK-NEXT: [[ADDCONV:%.*]] = or disjoint i8 [[X]], 64 -; CHECK-NEXT: [[R:%.*]] = sitofp i8 [[ADDCONV]] to half +; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i8 [[X]], 64 +; CHECK-NEXT: [[R:%.*]] = sitofp i8 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %x = and i8 %x_in, 63 -- GitLab From 946ea4e3ca4c908bfa7c196b982795f5c390b923 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Wed, 21 Feb 2024 16:05:30 -0600 Subject: [PATCH 344/929] [InstCombine] Add folds for `(fp_binop ({s|u}itofp x), ({s|u}itofp y))` The full fold is one of the following: 1) `(fp_binop ({s|u}itofp x), ({s|u}itofp y))` -> `({s|u}itofp (int_binop x, y))` 2) `(fp_binop ({s|u}itofp x), FpC)` -> `({s|u}itofp (int_binop x, (fpto{s|u}i FpC)))` And support the following binops: `fmul` -> `mul` `fadd` -> `add` `fsub` -> `sub` Proofs: https://alive2.llvm.org/ce/z/zuacA8 The proofs timeout, so they must be reproduced locally. Closes #82555 --- .../InstCombine/InstCombineAddSub.cpp | 3 + .../InstCombine/InstCombineMulDivRem.cpp | 3 + .../InstCombine/InstructionCombining.cpp | 223 +++++++++++++----- .../test/Transforms/InstCombine/add-sitofp.ll | 8 +- .../Transforms/InstCombine/binop-itofp.ll | 140 +++++------ llvm/test/Transforms/InstCombine/pr33453.ll | 4 +- 6 files changed, 231 insertions(+), 150 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp index 770df1093df0..aaf7184a5562 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp @@ -2793,6 +2793,9 @@ Instruction *InstCombinerImpl::visitFSub(BinaryOperator &I) { if (Instruction *X = foldFNegIntoConstant(I, DL)) return X; + if (Instruction *R = foldFBinOpOfIntCasts(I)) + return R; + Value *X, *Y; Constant *C; diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp index 0bd4b6d1a835..3ebf6b3d9bf7 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp @@ -769,6 +769,9 @@ Instruction *InstCombinerImpl::visitFMul(BinaryOperator &I) { if (Instruction *R = foldFPSignBitOps(I)) return R; + if (Instruction *R = foldFBinOpOfIntCasts(I)) + return R; + // X * -1.0 --> -X Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); if (match(Op1, m_SpecificFP(-1.0))) diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index a22f87e2ba30..f3a740c1b161 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -1402,71 +1402,176 @@ Value *InstCombinerImpl::dyn_castNegVal(Value *V) const { } // Try to fold: -// 1) (add (sitofp x), (sitofp y)) -// -> (sitofp (add x, y)) -// 2) (add (sitofp x), FpC) -// -> (sitofp (add x, (fptosi FpC))) +// 1) (fp_binop ({s|u}itofp x), ({s|u}itofp y)) +// -> ({s|u}itofp (int_binop x, y)) +// 2) (fp_binop ({s|u}itofp x), FpC) +// -> ({s|u}itofp (int_binop x, (fpto{s|u}i FpC))) Instruction *InstCombinerImpl::foldFBinOpOfIntCasts(BinaryOperator &BO) { - // Check for (fadd double (sitofp x), y), see if we can merge this into an - // integer add followed by a promotion. - Value *LHS = BO.getOperand(0), *RHS = BO.getOperand(1); - if (SIToFPInst *LHSConv = dyn_cast(LHS)) { - Value *LHSIntVal = LHSConv->getOperand(0); - Type *FPType = LHSConv->getType(); - - // TODO: This check is overly conservative. In many cases known bits - // analysis can tell us that the result of the addition has less significant - // bits than the integer type can hold. - auto IsValidPromotion = [](Type *FTy, Type *ITy) { - Type *FScalarTy = FTy->getScalarType(); - Type *IScalarTy = ITy->getScalarType(); - - // Do we have enough bits in the significand to represent the result of - // the integer addition? - unsigned MaxRepresentableBits = - APFloat::semanticsPrecision(FScalarTy->getFltSemantics()); - return IScalarTy->getIntegerBitWidth() <= MaxRepresentableBits; - }; + Value *IntOps[2] = {nullptr, nullptr}; + Constant *Op1FpC = nullptr; + + // Check for: + // 1) (binop ({s|u}itofp x), ({s|u}itofp y)) + // 2) (binop ({s|u}itofp x), FpC) + if (!match(BO.getOperand(0), m_SIToFP(m_Value(IntOps[0]))) && + !match(BO.getOperand(0), m_UIToFP(m_Value(IntOps[0])))) + return nullptr; - // (fadd double (sitofp x), fpcst) --> (sitofp (add int x, intcst)) - // ... if the constant fits in the integer value. This is useful for things - // like (double)(x & 1234) + 4.0 -> (double)((X & 1234)+4) which no longer - // requires a constant pool load, and generally allows the add to be better - // instcombined. - if (ConstantFP *CFP = dyn_cast(RHS)) - if (IsValidPromotion(FPType, LHSIntVal->getType())) { - Constant *CI = ConstantFoldCastOperand(Instruction::FPToSI, CFP, - LHSIntVal->getType(), DL); - if (LHSConv->hasOneUse() && - ConstantFoldCastOperand(Instruction::SIToFP, CI, BO.getType(), - DL) == CFP && - willNotOverflowSignedAdd(LHSIntVal, CI, BO)) { - // Insert the new integer add. - Value *NewAdd = Builder.CreateNSWAdd(LHSIntVal, CI); - return new SIToFPInst(NewAdd, BO.getType()); - } - } + if (!match(BO.getOperand(1), m_Constant(Op1FpC)) && + !match(BO.getOperand(1), m_SIToFP(m_Value(IntOps[1]))) && + !match(BO.getOperand(1), m_UIToFP(m_Value(IntOps[1])))) + return nullptr; - // (fadd double (sitofp x), (sitofp y)) --> (sitofp (add int x, y)) - if (SIToFPInst *RHSConv = dyn_cast(RHS)) { - Value *RHSIntVal = RHSConv->getOperand(0); - // It's enough to check LHS types only because we require int types to - // be the same for this transform. - if (IsValidPromotion(FPType, LHSIntVal->getType())) { - // Only do this if x/y have the same type, if at least one of them has a - // single use (so we don't increase the number of int->fp conversions), - // and if the integer add will not overflow. - if (LHSIntVal->getType() == RHSIntVal->getType() && - (LHSConv->hasOneUse() || RHSConv->hasOneUse()) && - willNotOverflowSignedAdd(LHSIntVal, RHSIntVal, BO)) { - // Insert the new integer add. - Value *NewAdd = Builder.CreateNSWAdd(LHSIntVal, RHSIntVal); - return new SIToFPInst(NewAdd, BO.getType()); - } + Type *FPTy = BO.getType(); + Type *IntTy = IntOps[0]->getType(); + + // Do we have signed casts? + bool OpsFromSigned = isa(BO.getOperand(0)); + + unsigned IntSz = IntTy->getScalarSizeInBits(); + // This is the maximum number of inuse bits by the integer where the int -> fp + // casts are exact. + unsigned MaxRepresentableBits = + APFloat::semanticsPrecision(FPTy->getScalarType()->getFltSemantics()); + + // Cache KnownBits a bit to potentially save some analysis. + WithCache OpsKnown[2] = {IntOps[0], IntOps[1]}; + + // Preserve known number of leading bits. This can allow us to trivial nsw/nuw + // checks later on. + unsigned NumUsedLeadingBits[2] = {IntSz, IntSz}; + + auto IsNonZero = [&](unsigned OpNo) -> bool { + if (OpsKnown[OpNo].hasKnownBits() && + OpsKnown[OpNo].getKnownBits(SQ).isNonZero()) + return true; + return isKnownNonZero(IntOps[OpNo], SQ.DL); + }; + + auto IsNonNeg = [&](unsigned OpNo) -> bool { + if (OpsKnown[OpNo].hasKnownBits() && + OpsKnown[OpNo].getKnownBits(SQ).isNonNegative()) + return true; + return isKnownNonNegative(IntOps[OpNo], SQ); + }; + + // Check if we know for certain that ({s|u}itofp op) is exact. + auto IsValidPromotion = [&](unsigned OpNo) -> bool { + // If fp precision >= bitwidth(op) then its exact. + // NB: This is slightly conservative for `sitofp`. For signed conversion, we + // can handle `MaxRepresentableBits == IntSz - 1` as the sign bit will be + // handled specially. We can't, however, increase the bound arbitrarily for + // `sitofp` as for larger sizes, it won't sign extend. + if (MaxRepresentableBits < IntSz) { + // Otherwise if its signed cast check that fp precisions >= bitwidth(op) - + // numSignBits(op). + // TODO: If we add support for `WithCache` in `ComputeNumSignBits`, change + // `IntOps[OpNo]` arguments to `KnownOps[OpNo]`. + if (OpsFromSigned) + NumUsedLeadingBits[OpNo] = IntSz - ComputeNumSignBits(IntOps[OpNo]); + // Finally for unsigned check that fp precision >= bitwidth(op) - + // numLeadingZeros(op). + else { + NumUsedLeadingBits[OpNo] = + IntSz - OpsKnown[OpNo].getKnownBits(SQ).countMinLeadingZeros(); } } + // NB: We could also check if op is known to be a power of 2 or zero (which + // will always be representable). Its unlikely, however, that is we are + // unable to bound op in any way we will be able to pass the overflow checks + // later on. + + if (MaxRepresentableBits < NumUsedLeadingBits[OpNo]) + return false; + // Signed + Mul also requires that op is non-zero to avoid -0 cases. + return !OpsFromSigned || BO.getOpcode() != Instruction::FMul || + IsNonZero(OpNo); + }; + + // If we have a constant rhs, see if we can losslessly convert it to an int. + if (Op1FpC != nullptr) { + Constant *Op1IntC = ConstantFoldCastOperand( + OpsFromSigned ? Instruction::FPToSI : Instruction::FPToUI, Op1FpC, + IntTy, DL); + if (Op1IntC == nullptr) + return nullptr; + if (ConstantFoldCastOperand(OpsFromSigned ? Instruction::SIToFP + : Instruction::UIToFP, + Op1IntC, FPTy, DL) != Op1FpC) + return nullptr; + + // First try to keep sign of cast the same. + IntOps[1] = Op1IntC; } - return nullptr; + + // Ensure lhs/rhs integer types match. + if (IntTy != IntOps[1]->getType()) + return nullptr; + + if (Op1FpC == nullptr) { + if (OpsFromSigned != isa(BO.getOperand(1))) { + // If we have a signed + unsigned, see if we can treat both as signed + // (uitofp nneg x) == (sitofp nneg x). + if (OpsFromSigned ? !IsNonNeg(1) : !IsNonNeg(0)) + return nullptr; + OpsFromSigned = true; + } + if (!IsValidPromotion(1)) + return nullptr; + } + if (!IsValidPromotion(0)) + return nullptr; + + // Final we check if the integer version of the binop will not overflow. + BinaryOperator::BinaryOps IntOpc; + // Because of the precision check, we can often rule out overflows. + bool NeedsOverflowCheck = true; + // Try to conservatively rule out overflow based on the already done precision + // checks. + unsigned OverflowMaxOutputBits = OpsFromSigned ? 2 : 1; + unsigned OverflowMaxCurBits = + std::max(NumUsedLeadingBits[0], NumUsedLeadingBits[1]); + bool OutputSigned = OpsFromSigned; + switch (BO.getOpcode()) { + case Instruction::FAdd: + IntOpc = Instruction::Add; + OverflowMaxOutputBits += OverflowMaxCurBits; + break; + case Instruction::FSub: + IntOpc = Instruction::Sub; + OverflowMaxOutputBits += OverflowMaxCurBits; + break; + case Instruction::FMul: + IntOpc = Instruction::Mul; + OverflowMaxOutputBits += OverflowMaxCurBits * 2; + break; + default: + llvm_unreachable("Unsupported binop"); + } + // The precision check may have already ruled out overflow. + if (OverflowMaxOutputBits < IntSz) { + NeedsOverflowCheck = false; + // We can bound unsigned overflow from sub to in range signed value (this is + // what allows us to avoid the overflow check for sub). + if (IntOpc == Instruction::Sub) + OutputSigned = true; + } + + // Precision check did not rule out overflow, so need to check. + // TODO: If we add support for `WithCache` in `willNotOverflow`, change + // `IntOps[...]` arguments to `KnownOps[...]`. + if (NeedsOverflowCheck && + !willNotOverflow(IntOpc, IntOps[0], IntOps[1], BO, OutputSigned)) + return nullptr; + + Value *IntBinOp = Builder.CreateBinOp(IntOpc, IntOps[0], IntOps[1]); + if (auto *IntBO = dyn_cast(IntBinOp)) { + IntBO->setHasNoSignedWrap(OutputSigned); + IntBO->setHasNoUnsignedWrap(!OutputSigned); + } + if (OutputSigned) + return new SIToFPInst(IntBinOp, FPTy); + return new UIToFPInst(IntBinOp, FPTy); } /// A binop with a constant operand and a sign-extended boolean operand may be diff --git a/llvm/test/Transforms/InstCombine/add-sitofp.ll b/llvm/test/Transforms/InstCombine/add-sitofp.ll index 206c0a7ebd26..049db8c84a52 100644 --- a/llvm/test/Transforms/InstCombine/add-sitofp.ll +++ b/llvm/test/Transforms/InstCombine/add-sitofp.ll @@ -83,15 +83,13 @@ define float @test_2_neg(i32 %a, i32 %b) { ret float %res } -; This test demonstrates overly conservative legality check. The float addition -; can be replaced with the integer addition because the result of the operation -; can be represented in float, but we don't do that now. +; can be represented in float. define float @test_3(i32 %a, i32 %b) { ; CHECK-LABEL: @test_3( ; CHECK-NEXT: [[M:%.*]] = lshr i32 [[A:%.*]], 24 ; CHECK-NEXT: [[N:%.*]] = and i32 [[M]], [[B:%.*]] -; CHECK-NEXT: [[O:%.*]] = sitofp i32 [[N]] to float -; CHECK-NEXT: [[P:%.*]] = fadd float [[O]], 1.000000e+00 +; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i32 [[N]], 1 +; CHECK-NEXT: [[P:%.*]] = sitofp i32 [[TMP1]] to float ; CHECK-NEXT: ret float [[P]] ; %m = lshr i32 %a, 24 diff --git a/llvm/test/Transforms/InstCombine/binop-itofp.ll b/llvm/test/Transforms/InstCombine/binop-itofp.ll index ea1ad9f8d881..ffa893745791 100644 --- a/llvm/test/Transforms/InstCombine/binop-itofp.ll +++ b/llvm/test/Transforms/InstCombine/binop-itofp.ll @@ -5,9 +5,8 @@ define half @test_ui_ui_i8_add(i8 noundef %x_in, i8 noundef %y_in) { ; CHECK-LABEL: @test_ui_ui_i8_add( ; CHECK-NEXT: [[X:%.*]] = and i8 [[X_IN:%.*]], 127 ; CHECK-NEXT: [[Y:%.*]] = and i8 [[Y_IN:%.*]], 127 -; CHECK-NEXT: [[XF:%.*]] = uitofp i8 [[X]] to half -; CHECK-NEXT: [[YF:%.*]] = uitofp i8 [[Y]] to half -; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], [[YF]] +; CHECK-NEXT: [[TMP1:%.*]] = add nuw i8 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = uitofp i8 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %x = and i8 %x_in, 127 @@ -37,9 +36,8 @@ define half @test_ui_ui_i8_add_fail_overflow(i8 noundef %x_in, i8 noundef %y_in) define half @test_ui_ui_i8_add_C(i8 noundef %x_in) { ; CHECK-LABEL: @test_ui_ui_i8_add_C( -; CHECK-NEXT: [[X:%.*]] = and i8 [[X_IN:%.*]], 127 -; CHECK-NEXT: [[XF:%.*]] = uitofp i8 [[X]] to half -; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], 0xH5800 +; CHECK-NEXT: [[TMP1:%.*]] = or i8 [[X_IN:%.*]], -128 +; CHECK-NEXT: [[R:%.*]] = uitofp i8 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %x = and i8 %x_in, 127 @@ -111,9 +109,8 @@ define half @test_ui_si_i8_add(i8 noundef %x_in, i8 noundef %y_in) { ; CHECK-LABEL: @test_ui_si_i8_add( ; CHECK-NEXT: [[X:%.*]] = and i8 [[X_IN:%.*]], 63 ; CHECK-NEXT: [[Y:%.*]] = and i8 [[Y_IN:%.*]], 63 -; CHECK-NEXT: [[XF:%.*]] = sitofp i8 [[X]] to half -; CHECK-NEXT: [[YF:%.*]] = uitofp i8 [[Y]] to half -; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], [[YF]] +; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i8 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = sitofp i8 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %x = and i8 %x_in, 63 @@ -143,9 +140,8 @@ define half @test_ui_si_i8_add_overflow(i8 noundef %x_in, i8 noundef %y_in) { define half @test_ui_ui_i8_sub_C(i8 noundef %x_in) { ; CHECK-LABEL: @test_ui_ui_i8_sub_C( -; CHECK-NEXT: [[X:%.*]] = or i8 [[X_IN:%.*]], -128 -; CHECK-NEXT: [[XF:%.*]] = uitofp i8 [[X]] to half -; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], 0xHD800 +; CHECK-NEXT: [[TMP1:%.*]] = and i8 [[X_IN:%.*]], 127 +; CHECK-NEXT: [[R:%.*]] = uitofp i8 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %x = or i8 %x_in, 128 @@ -171,9 +167,8 @@ define half @test_si_si_i8_sub(i8 noundef %x_in, i8 noundef %y_in) { ; CHECK-LABEL: @test_si_si_i8_sub( ; CHECK-NEXT: [[X:%.*]] = and i8 [[X_IN:%.*]], 63 ; CHECK-NEXT: [[Y:%.*]] = or i8 [[Y_IN:%.*]], -64 -; CHECK-NEXT: [[XF:%.*]] = sitofp i8 [[X]] to half -; CHECK-NEXT: [[YF:%.*]] = sitofp i8 [[Y]] to half -; CHECK-NEXT: [[R:%.*]] = fsub half [[XF]], [[YF]] +; CHECK-NEXT: [[TMP1:%.*]] = sub nsw i8 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = sitofp i8 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %x = and i8 %x_in, 63 @@ -231,9 +226,8 @@ define half @test_ui_si_i8_sub(i8 noundef %x_in, i8 noundef %y_in) { ; CHECK-LABEL: @test_ui_si_i8_sub( ; CHECK-NEXT: [[X:%.*]] = or i8 [[X_IN:%.*]], 64 ; CHECK-NEXT: [[Y:%.*]] = and i8 [[Y_IN:%.*]], 63 -; CHECK-NEXT: [[XF:%.*]] = sitofp i8 [[X]] to half -; CHECK-NEXT: [[YF:%.*]] = uitofp i8 [[Y]] to half -; CHECK-NEXT: [[R:%.*]] = fsub half [[XF]], [[YF]] +; CHECK-NEXT: [[TMP1:%.*]] = sub nuw nsw i8 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = sitofp i8 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %x = or i8 %x_in, 64 @@ -265,9 +259,8 @@ define half @test_ui_ui_i8_mul(i8 noundef %x_in, i8 noundef %y_in) { ; CHECK-LABEL: @test_ui_ui_i8_mul( ; CHECK-NEXT: [[X:%.*]] = and i8 [[X_IN:%.*]], 15 ; CHECK-NEXT: [[Y:%.*]] = and i8 [[Y_IN:%.*]], 15 -; CHECK-NEXT: [[XF:%.*]] = uitofp i8 [[X]] to half -; CHECK-NEXT: [[YF:%.*]] = uitofp i8 [[Y]] to half -; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i8 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = uitofp i8 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %x = and i8 %x_in, 15 @@ -280,9 +273,8 @@ define half @test_ui_ui_i8_mul(i8 noundef %x_in, i8 noundef %y_in) { define half @test_ui_ui_i8_mul_C(i8 noundef %x_in) { ; CHECK-LABEL: @test_ui_ui_i8_mul_C( -; CHECK-NEXT: [[X:%.*]] = and i8 [[X_IN:%.*]], 15 -; CHECK-NEXT: [[XF:%.*]] = uitofp i8 [[X]] to half -; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], 0xH4C00 +; CHECK-NEXT: [[TMP1:%.*]] = shl i8 [[X_IN:%.*]], 4 +; CHECK-NEXT: [[R:%.*]] = uitofp i8 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %x = and i8 %x_in, 15 @@ -309,9 +301,8 @@ define half @test_si_si_i8_mul(i8 noundef %x_in, i8 noundef %y_in) { ; CHECK-NEXT: [[XX:%.*]] = and i8 [[X_IN:%.*]], 6 ; CHECK-NEXT: [[X:%.*]] = or disjoint i8 [[XX]], 1 ; CHECK-NEXT: [[Y:%.*]] = or i8 [[Y_IN:%.*]], -8 -; CHECK-NEXT: [[XF:%.*]] = sitofp i8 [[X]] to half -; CHECK-NEXT: [[YF:%.*]] = sitofp i8 [[Y]] to half -; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: [[TMP1:%.*]] = mul nsw i8 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = sitofp i8 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %xx = and i8 %x_in, 6 @@ -376,9 +367,8 @@ define half @test_ui_si_i8_mul(i8 noundef %x_in, i8 noundef %y_in) { ; CHECK-NEXT: [[X:%.*]] = or disjoint i8 [[XX]], 1 ; CHECK-NEXT: [[YY:%.*]] = and i8 [[Y_IN:%.*]], 7 ; CHECK-NEXT: [[Y:%.*]] = add nuw nsw i8 [[YY]], 1 -; CHECK-NEXT: [[XF:%.*]] = sitofp i8 [[X]] to half -; CHECK-NEXT: [[YF:%.*]] = uitofp i8 [[Y]] to half -; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: [[TMP1:%.*]] = mul nuw nsw i8 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = sitofp i8 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %xx = and i8 %x_in, 6 @@ -433,9 +423,8 @@ define half @test_ui_ui_i16_add(i16 noundef %x_in, i16 noundef %y_in) { ; CHECK-LABEL: @test_ui_ui_i16_add( ; CHECK-NEXT: [[X:%.*]] = and i16 [[X_IN:%.*]], 2047 ; CHECK-NEXT: [[Y:%.*]] = and i16 [[Y_IN:%.*]], 2047 -; CHECK-NEXT: [[XF:%.*]] = uitofp i16 [[X]] to half -; CHECK-NEXT: [[YF:%.*]] = uitofp i16 [[Y]] to half -; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], [[YF]] +; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i16 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = uitofp i16 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %x = and i16 %x_in, 2047 @@ -465,9 +454,8 @@ define half @test_ui_ui_i16_add_fail_not_promotable(i16 noundef %x_in, i16 nound define half @test_ui_ui_i16_add_C(i16 noundef %x_in) { ; CHECK-LABEL: @test_ui_ui_i16_add_C( -; CHECK-NEXT: [[X:%.*]] = and i16 [[X_IN:%.*]], 2047 -; CHECK-NEXT: [[XF:%.*]] = uitofp i16 [[X]] to half -; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], 0xH7BC0 +; CHECK-NEXT: [[TMP1:%.*]] = or i16 [[X_IN:%.*]], -2048 +; CHECK-NEXT: [[R:%.*]] = uitofp i16 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %x = and i16 %x_in, 2047 @@ -493,9 +481,8 @@ define half @test_si_si_i16_add(i16 noundef %x_in, i16 noundef %y_in) { ; CHECK-LABEL: @test_si_si_i16_add( ; CHECK-NEXT: [[X:%.*]] = or i16 [[X_IN:%.*]], -2048 ; CHECK-NEXT: [[Y:%.*]] = or i16 [[Y_IN:%.*]], -2048 -; CHECK-NEXT: [[XF:%.*]] = sitofp i16 [[X]] to half -; CHECK-NEXT: [[YF:%.*]] = sitofp i16 [[Y]] to half -; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], [[YF]] +; CHECK-NEXT: [[TMP1:%.*]] = add nsw i16 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = sitofp i16 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %x = or i16 %x_in, -2048 @@ -542,9 +529,8 @@ define half @test_si_si_i16_sub(i16 noundef %x_in, i16 noundef %y_in) { ; CHECK-LABEL: @test_si_si_i16_sub( ; CHECK-NEXT: [[X:%.*]] = or i16 [[X_IN:%.*]], -2048 ; CHECK-NEXT: [[Y:%.*]] = and i16 [[Y_IN:%.*]], 2047 -; CHECK-NEXT: [[XF:%.*]] = sitofp i16 [[X]] to half -; CHECK-NEXT: [[YF:%.*]] = sitofp i16 [[Y]] to half -; CHECK-NEXT: [[R:%.*]] = fsub half [[XF]], [[YF]] +; CHECK-NEXT: [[TMP1:%.*]] = sub nuw nsw i16 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = sitofp i16 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %x = or i16 %x_in, -2048 @@ -576,9 +562,8 @@ define half @test_ui_si_i16_sub(i16 noundef %x_in, i16 noundef %y_in) { ; CHECK-LABEL: @test_ui_si_i16_sub( ; CHECK-NEXT: [[X:%.*]] = and i16 [[X_IN:%.*]], 2047 ; CHECK-NEXT: [[Y:%.*]] = and i16 [[Y_IN:%.*]], 2047 -; CHECK-NEXT: [[XF:%.*]] = uitofp i16 [[X]] to half -; CHECK-NEXT: [[YF:%.*]] = sitofp i16 [[Y]] to half -; CHECK-NEXT: [[R:%.*]] = fsub half [[XF]], [[YF]] +; CHECK-NEXT: [[TMP1:%.*]] = sub nsw i16 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = sitofp i16 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %x = and i16 %x_in, 2047 @@ -610,9 +595,8 @@ define half @test_ui_ui_i16_mul(i16 noundef %x_in, i16 noundef %y_in) { ; CHECK-LABEL: @test_ui_ui_i16_mul( ; CHECK-NEXT: [[X:%.*]] = and i16 [[X_IN:%.*]], 255 ; CHECK-NEXT: [[Y:%.*]] = and i16 [[Y_IN:%.*]], 255 -; CHECK-NEXT: [[XF:%.*]] = uitofp i16 [[X]] to half -; CHECK-NEXT: [[YF:%.*]] = uitofp i16 [[Y]] to half -; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i16 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = uitofp i16 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %x = and i16 %x_in, 255 @@ -645,9 +629,8 @@ define half @test_si_si_i16_mul(i16 noundef %x_in, i16 noundef %y_in) { ; CHECK-NEXT: [[XX:%.*]] = and i16 [[X_IN:%.*]], 126 ; CHECK-NEXT: [[X:%.*]] = or disjoint i16 [[XX]], 1 ; CHECK-NEXT: [[Y:%.*]] = or i16 [[Y_IN:%.*]], -255 -; CHECK-NEXT: [[XF:%.*]] = sitofp i16 [[X]] to half -; CHECK-NEXT: [[YF:%.*]] = sitofp i16 [[Y]] to half -; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: [[TMP1:%.*]] = mul nsw i16 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = sitofp i16 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %xx = and i16 %x_in, 126 @@ -710,9 +693,8 @@ define half @test_ui_si_i16_mul(i16 noundef %x_in, i16 noundef %y_in) { ; CHECK-NEXT: [[X:%.*]] = or disjoint i16 [[XX]], 1 ; CHECK-NEXT: [[YY:%.*]] = and i16 [[Y_IN:%.*]], 126 ; CHECK-NEXT: [[Y:%.*]] = or disjoint i16 [[YY]], 1 -; CHECK-NEXT: [[XF:%.*]] = sitofp i16 [[X]] to half -; CHECK-NEXT: [[YF:%.*]] = uitofp i16 [[Y]] to half -; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: [[TMP1:%.*]] = mul nuw nsw i16 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = sitofp i16 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %xx = and i16 %x_in, 126 @@ -729,9 +711,8 @@ define half @test_ui_ui_i12_add(i12 noundef %x_in, i12 noundef %y_in) { ; CHECK-LABEL: @test_ui_ui_i12_add( ; CHECK-NEXT: [[X:%.*]] = and i12 [[X_IN:%.*]], 2047 ; CHECK-NEXT: [[Y:%.*]] = and i12 [[Y_IN:%.*]], 2047 -; CHECK-NEXT: [[XF:%.*]] = uitofp i12 [[X]] to half -; CHECK-NEXT: [[YF:%.*]] = uitofp i12 [[Y]] to half -; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], [[YF]] +; CHECK-NEXT: [[TMP1:%.*]] = add nuw i12 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = uitofp i12 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %x = and i12 %x_in, 2047 @@ -764,9 +745,8 @@ define half @test_si_si_i12_add(i12 noundef %x_in, i12 noundef %y_in) { ; CHECK-LABEL: @test_si_si_i12_add( ; CHECK-NEXT: [[X:%.*]] = or i12 [[X_IN:%.*]], -1024 ; CHECK-NEXT: [[Y:%.*]] = or i12 [[Y_IN:%.*]], -1024 -; CHECK-NEXT: [[XF:%.*]] = sitofp i12 [[X]] to half -; CHECK-NEXT: [[YF:%.*]] = sitofp i12 [[Y]] to half -; CHECK-NEXT: [[R:%.*]] = fadd half [[XF]], [[YF]] +; CHECK-NEXT: [[TMP1:%.*]] = add nsw i12 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = sitofp i12 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %x = or i12 %x_in, -1024 @@ -811,9 +791,8 @@ define half @test_ui_ui_i12_sub(i12 noundef %x_in, i12 noundef %y_in) { ; CHECK-LABEL: @test_ui_ui_i12_sub( ; CHECK-NEXT: [[X:%.*]] = and i12 [[X_IN:%.*]], 1023 ; CHECK-NEXT: [[Y:%.*]] = and i12 [[Y_IN:%.*]], 1023 -; CHECK-NEXT: [[XF:%.*]] = uitofp i12 [[X]] to half -; CHECK-NEXT: [[YF:%.*]] = uitofp i12 [[Y]] to half -; CHECK-NEXT: [[R:%.*]] = fsub half [[XF]], [[YF]] +; CHECK-NEXT: [[TMP1:%.*]] = sub nsw i12 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = sitofp i12 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %x = and i12 %x_in, 1023 @@ -846,9 +825,8 @@ define half @test_si_si_i12_sub(i12 noundef %x_in, i12 noundef %y_in) { ; CHECK-LABEL: @test_si_si_i12_sub( ; CHECK-NEXT: [[X:%.*]] = and i12 [[X_IN:%.*]], 1023 ; CHECK-NEXT: [[Y:%.*]] = or i12 [[Y_IN:%.*]], -1024 -; CHECK-NEXT: [[XF:%.*]] = sitofp i12 [[X]] to half -; CHECK-NEXT: [[YF:%.*]] = sitofp i12 [[Y]] to half -; CHECK-NEXT: [[R:%.*]] = fsub half [[XF]], [[YF]] +; CHECK-NEXT: [[TMP1:%.*]] = sub nsw i12 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = sitofp i12 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %x = and i12 %x_in, 1023 @@ -876,9 +854,8 @@ define half @test_ui_ui_i12_mul(i12 noundef %x_in, i12 noundef %y_in) { ; CHECK-LABEL: @test_ui_ui_i12_mul( ; CHECK-NEXT: [[X:%.*]] = and i12 [[X_IN:%.*]], 31 ; CHECK-NEXT: [[Y:%.*]] = and i12 [[Y_IN:%.*]], 63 -; CHECK-NEXT: [[XF:%.*]] = uitofp i12 [[X]] to half -; CHECK-NEXT: [[YF:%.*]] = uitofp i12 [[Y]] to half -; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: [[TMP1:%.*]] = mul nuw nsw i12 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = uitofp i12 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %x = and i12 %x_in, 31 @@ -894,9 +871,8 @@ define half @test_ui_ui_i12_mul_fail_overflow(i12 noundef %x_in, i12 noundef %y_ ; CHECK-NEXT: [[XX:%.*]] = and i12 [[X_IN:%.*]], 31 ; CHECK-NEXT: [[X:%.*]] = add nuw nsw i12 [[XX]], 1 ; CHECK-NEXT: [[Y:%.*]] = and i12 [[Y_IN:%.*]], 63 -; CHECK-NEXT: [[XF:%.*]] = uitofp i12 [[X]] to half -; CHECK-NEXT: [[YF:%.*]] = uitofp i12 [[Y]] to half -; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i12 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = uitofp i12 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %xx = and i12 %x_in, 31 @@ -910,9 +886,9 @@ define half @test_ui_ui_i12_mul_fail_overflow(i12 noundef %x_in, i12 noundef %y_ define half @test_ui_ui_i12_mul_C(i12 noundef %x_in) { ; CHECK-LABEL: @test_ui_ui_i12_mul_C( -; CHECK-NEXT: [[X:%.*]] = and i12 [[X_IN:%.*]], 31 -; CHECK-NEXT: [[XF:%.*]] = uitofp i12 [[X]] to half -; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], 0xH5400 +; CHECK-NEXT: [[X:%.*]] = shl i12 [[X_IN:%.*]], 6 +; CHECK-NEXT: [[TMP1:%.*]] = and i12 [[X]], 1984 +; CHECK-NEXT: [[R:%.*]] = uitofp i12 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %x = and i12 %x_in, 31 @@ -926,9 +902,8 @@ define half @test_si_si_i12_mul(i12 noundef %x_in, i12 noundef %y_in) { ; CHECK-NEXT: [[XX:%.*]] = and i12 [[X_IN:%.*]], 30 ; CHECK-NEXT: [[X:%.*]] = or disjoint i12 [[XX]], 1 ; CHECK-NEXT: [[Y:%.*]] = or i12 [[Y_IN:%.*]], -64 -; CHECK-NEXT: [[XF:%.*]] = sitofp i12 [[X]] to half -; CHECK-NEXT: [[YF:%.*]] = sitofp i12 [[Y]] to half -; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: [[TMP1:%.*]] = mul nsw i12 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = sitofp i12 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %xx = and i12 %x_in, 30 @@ -979,8 +954,8 @@ define half @test_si_si_i12_mul_fail_maybe_non_zero(i12 noundef %x_in, i12 nound define half @test_si_si_i12_mul_C(i12 noundef %x_in) { ; CHECK-LABEL: @test_si_si_i12_mul_C( ; CHECK-NEXT: [[X:%.*]] = or i12 [[X_IN:%.*]], -64 -; CHECK-NEXT: [[XF:%.*]] = sitofp i12 [[X]] to half -; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], 0xHCC00 +; CHECK-NEXT: [[TMP1:%.*]] = mul nsw i12 [[X]], -16 +; CHECK-NEXT: [[R:%.*]] = sitofp i12 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %x = or i12 %x_in, -64 @@ -1008,9 +983,8 @@ define half @test_ui_si_i12_mul_nsw(i12 noundef %x_in, i12 noundef %y_in) { ; CHECK-NEXT: [[X:%.*]] = add nuw nsw i12 [[XX]], 1 ; CHECK-NEXT: [[YY:%.*]] = and i12 [[Y_IN:%.*]], 30 ; CHECK-NEXT: [[Y:%.*]] = or disjoint i12 [[YY]], 1 -; CHECK-NEXT: [[XF:%.*]] = uitofp i12 [[X]] to half -; CHECK-NEXT: [[YF:%.*]] = sitofp i12 [[Y]] to half -; CHECK-NEXT: [[R:%.*]] = fmul half [[XF]], [[YF]] +; CHECK-NEXT: [[TMP1:%.*]] = mul nuw nsw i12 [[X]], [[Y]] +; CHECK-NEXT: [[R:%.*]] = sitofp i12 [[TMP1]] to half ; CHECK-NEXT: ret half [[R]] ; %xx = and i12 %x_in, 31 diff --git a/llvm/test/Transforms/InstCombine/pr33453.ll b/llvm/test/Transforms/InstCombine/pr33453.ll index 09f1569bc95e..45f87b753006 100644 --- a/llvm/test/Transforms/InstCombine/pr33453.ll +++ b/llvm/test/Transforms/InstCombine/pr33453.ll @@ -6,9 +6,7 @@ define float @patatino() { ; CHECK-LABEL: @patatino( -; CHECK-NEXT: [[UITOFP1:%.*]] = uitofp i1 icmp eq (ptr getelementptr inbounds (i16, ptr @g2, i64 1), ptr @g1) to float -; CHECK-NEXT: [[UITOFP2:%.*]] = uitofp i1 icmp eq (ptr getelementptr inbounds (i16, ptr @g2, i64 1), ptr @g1) to float -; CHECK-NEXT: [[FMUL:%.*]] = fmul float [[UITOFP1]], [[UITOFP2]] +; CHECK-NEXT: [[FMUL:%.*]] = uitofp i1 mul (i1 icmp eq (ptr getelementptr inbounds (i16, ptr @g2, i64 1), ptr @g1), i1 icmp eq (ptr getelementptr inbounds (i16, ptr @g2, i64 1), ptr @g1)) to float ; CHECK-NEXT: ret float [[FMUL]] ; %uitofp1 = uitofp i1 icmp eq (ptr getelementptr inbounds (i16, ptr @g2, i64 1), ptr @g1) to float -- GitLab From f95a76391c527ba23574cb1d8d8803c9a1d0b7c4 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Wed, 6 Mar 2024 19:29:40 +0000 Subject: [PATCH 345/929] [gn build] Port 6494f9bb8ac1 --- llvm/utils/gn/secondary/clang/unittests/Interpreter/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/clang/unittests/Interpreter/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/Interpreter/BUILD.gn index 441d57187cd2..a20066436a3b 100644 --- a/llvm/utils/gn/secondary/clang/unittests/Interpreter/BUILD.gn +++ b/llvm/utils/gn/secondary/clang/unittests/Interpreter/BUILD.gn @@ -12,6 +12,7 @@ unittest("ClangReplInterpreterTests") { ] sources = [ "CodeCompletionTest.cpp", + "IncrementalCompilerBuilderTest.cpp", "IncrementalProcessingTest.cpp", "InterpreterTest.cpp", ] -- GitLab From 15b7b3182cc28f4f0b950bd73d931caa27b833ec Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Wed, 6 Mar 2024 20:49:12 +0100 Subject: [PATCH 346/929] [libomptarget] Support BE ELF files in plugins-nextgen (#83976) Code in plugins-nextgen reading ELF files is currently hard-coded to assume a 64-bit little-endian ELF format. Unfortunately, this assumption is even embedded in the interface between GlobalHandler and Utils/ELF routines, which use ELF64LE types. To fix this, I've refactored the interface to push all ELF specific types into Utils/ELF. Specifically, this patch removes both the getSymbol and getSymbolAddress routines and replaces them with a single findSymbolInImage, which gets a StringRef identifying the raw object file image as input, and returns a StringRef covering the data addressed by the symbol (address and size) if found, or std::nullopt otherwise. This allows properly templating over multiple ELF format variants inside Utils/ELF; specifically, this patch adds support for 64-bit big-endian ELF files in addition to 64-bit little-endian files. --- .../common/include/GlobalHandler.h | 3 - .../common/include/Utils/ELF.h | 16 ++-- .../common/src/GlobalHandler.cpp | 46 ++-------- .../plugins-nextgen/common/src/Utils/ELF.cpp | 91 ++++++++++++++----- 4 files changed, 85 insertions(+), 71 deletions(-) diff --git a/openmp/libomptarget/plugins-nextgen/common/include/GlobalHandler.h b/openmp/libomptarget/plugins-nextgen/common/include/GlobalHandler.h index 5c767995126b..57d93b88afca 100644 --- a/openmp/libomptarget/plugins-nextgen/common/include/GlobalHandler.h +++ b/openmp/libomptarget/plugins-nextgen/common/include/GlobalHandler.h @@ -103,9 +103,6 @@ class GenericGlobalHandlerTy { public: virtual ~GenericGlobalHandlerTy() {} - /// Helper function for getting an ELF from a device image. - Expected getELFObjectFile(DeviceImageTy &Image); - /// Returns whether the symbol named \p SymName is present in the given \p /// Image. bool isSymbolInImage(GenericDeviceTy &Device, DeviceImageTy &Image, diff --git a/openmp/libomptarget/plugins-nextgen/common/include/Utils/ELF.h b/openmp/libomptarget/plugins-nextgen/common/include/Utils/ELF.h index 140a6b6b84aa..2268e4b6c4af 100644 --- a/openmp/libomptarget/plugins-nextgen/common/include/Utils/ELF.h +++ b/openmp/libomptarget/plugins-nextgen/common/include/Utils/ELF.h @@ -27,18 +27,14 @@ bool isELF(llvm::StringRef Buffer); /// Checks if the given \p Object is a valid ELF matching the e_machine value. llvm::Expected checkMachine(llvm::StringRef Object, uint16_t EMachine); -/// Returns a pointer to the given \p Symbol inside of an ELF object. -llvm::Expected getSymbolAddress( - const llvm::object::ELFObjectFile &ELFObj, - const llvm::object::ELF64LE::Sym &Symbol); - -/// Returns the symbol associated with the \p Name in the \p ELFObj. It will +/// Returns the symbol associated with the \p Name in the \p Obj. It will /// first search for the hash sections to identify symbols from the hash table. /// If that fails it will fall back to a linear search in the case of an -/// executable file without a hash table. -llvm::Expected -getSymbol(const llvm::object::ELFObjectFile &ELFObj, - llvm::StringRef Name); +/// executable file without a hash table. If the symbol is found, it returns +/// a StringRef covering the symbol's data in the Obj buffer, based on its +/// address and size; otherwise, it returns std::nullopt. +llvm::Expected> +findSymbolInImage(const llvm::StringRef Obj, llvm::StringRef Name); } // namespace elf } // namespace utils diff --git a/openmp/libomptarget/plugins-nextgen/common/src/GlobalHandler.cpp b/openmp/libomptarget/plugins-nextgen/common/src/GlobalHandler.cpp index d398f60c55bd..2269f27e4602 100644 --- a/openmp/libomptarget/plugins-nextgen/common/src/GlobalHandler.cpp +++ b/openmp/libomptarget/plugins-nextgen/common/src/GlobalHandler.cpp @@ -25,16 +25,6 @@ using namespace omp; using namespace target; using namespace plugin; -Expected -GenericGlobalHandlerTy::getELFObjectFile(DeviceImageTy &Image) { - assert(utils::elf::isELF(Image.getMemoryBuffer().getBuffer()) && - "Input is not an ELF file"); - - Expected ElfOrErr = - ELF64LEObjectFile::create(Image.getMemoryBuffer()); - return ElfOrErr; -} - Error GenericGlobalHandlerTy::moveGlobalBetweenDeviceAndHost( GenericDeviceTy &Device, DeviceImageTy &Image, const GlobalTy &HostGlobal, bool Device2Host) { @@ -81,55 +71,37 @@ Error GenericGlobalHandlerTy::moveGlobalBetweenDeviceAndHost( bool GenericGlobalHandlerTy::isSymbolInImage(GenericDeviceTy &Device, DeviceImageTy &Image, StringRef SymName) { - // Get the ELF object file for the image. Notice the ELF object may already - // be created in previous calls, so we can reuse it. If this is unsuccessful - // just return false as we couldn't find it. - auto ELFObjOrErr = getELFObjectFile(Image); - if (!ELFObjOrErr) { - consumeError(ELFObjOrErr.takeError()); - return false; - } // Search the ELF symbol using the symbol name. - auto SymOrErr = utils::elf::getSymbol(*ELFObjOrErr, SymName); + auto SymOrErr = utils::elf::findSymbolInImage( + Image.getMemoryBuffer().getBuffer(), SymName); if (!SymOrErr) { consumeError(SymOrErr.takeError()); return false; } - return *SymOrErr; + return SymOrErr->has_value(); } Error GenericGlobalHandlerTy::getGlobalMetadataFromImage( GenericDeviceTy &Device, DeviceImageTy &Image, GlobalTy &ImageGlobal) { - // Get the ELF object file for the image. Notice the ELF object may already - // be created in previous calls, so we can reuse it. - auto ELFObj = getELFObjectFile(Image); - if (!ELFObj) - return ELFObj.takeError(); - // Search the ELF symbol using the symbol name. - auto SymOrErr = utils::elf::getSymbol(*ELFObj, ImageGlobal.getName()); + auto SymOrErr = utils::elf::findSymbolInImage( + Image.getMemoryBuffer().getBuffer(), ImageGlobal.getName()); if (!SymOrErr) return Plugin::error("Failed ELF lookup of global '%s': %s", ImageGlobal.getName().data(), toString(SymOrErr.takeError()).data()); - if (!*SymOrErr) + if (!SymOrErr->has_value()) return Plugin::error("Failed to find global symbol '%s' in the ELF image", ImageGlobal.getName().data()); - auto AddrOrErr = utils::elf::getSymbolAddress(*ELFObj, **SymOrErr); - // Get the section to which the symbol belongs. - if (!AddrOrErr) - return Plugin::error("Failed to get ELF symbol from global '%s': %s", - ImageGlobal.getName().data(), - toString(AddrOrErr.takeError()).data()); - // Setup the global symbol's address and size. - ImageGlobal.setPtr(const_cast(*AddrOrErr)); - ImageGlobal.setSize((*SymOrErr)->st_size); + auto Symbol = **SymOrErr; + ImageGlobal.setPtr(static_cast(const_cast(Symbol.data()))); + ImageGlobal.setSize(Symbol.size()); return Plugin::success(); } diff --git a/openmp/libomptarget/plugins-nextgen/common/src/Utils/ELF.cpp b/openmp/libomptarget/plugins-nextgen/common/src/Utils/ELF.cpp index c84c3bad5def..88bb097165ef 100644 --- a/openmp/libomptarget/plugins-nextgen/common/src/Utils/ELF.cpp +++ b/openmp/libomptarget/plugins-nextgen/common/src/Utils/ELF.cpp @@ -36,18 +36,10 @@ bool utils::elf::isELF(StringRef Buffer) { } } -Expected utils::elf::checkMachine(StringRef Object, uint16_t EMachine) { - assert(isELF(Object) && "Input is not an ELF!"); - - Expected ElfOrErr = - ELF64LEObjectFile::create(MemoryBufferRef(Object, /*Identifier=*/""), - /*InitContent=*/false); - if (!ElfOrErr) - return ElfOrErr.takeError(); - - const auto Header = ElfOrErr->getELFFile().getHeader(); - if (Header.e_ident[EI_CLASS] != ELFCLASS64) - return createError("Only 64-bit ELF files are supported"); +template +static Expected +checkMachineImpl(const object::ELFObjectFile &ELFObj, uint16_t EMachine) { + const auto Header = ELFObj.getELFFile().getHeader(); if (Header.e_type != ET_EXEC && Header.e_type != ET_DYN) return createError("Only executable ELF files are supported"); @@ -71,6 +63,25 @@ Expected utils::elf::checkMachine(StringRef Object, uint16_t EMachine) { return Header.e_machine == EMachine; } +Expected utils::elf::checkMachine(StringRef Object, uint16_t EMachine) { + assert(isELF(Object) && "Input is not an ELF!"); + + Expected> ElfOrErr = + ObjectFile::createELFObjectFile( + MemoryBufferRef(Object, /*Identifier=*/""), + /*InitContent=*/false); + if (!ElfOrErr) + return ElfOrErr.takeError(); + + if (const ELF64LEObjectFile *ELFObj = + dyn_cast(&**ElfOrErr)) + return checkMachineImpl(*ELFObj, EMachine); + if (const ELF64BEObjectFile *ELFObj = + dyn_cast(&**ElfOrErr)) + return checkMachineImpl(*ELFObj, EMachine); + return createError("Only 64-bit ELF files are supported"); +} + template static Expected getSymbolFromGnuHashTable(StringRef Name, const typename ELFT::GnuHash &HashTab, @@ -231,8 +242,9 @@ getSymTableSymbol(const ELFFile &Elf, const typename ELFT::Shdr &Sec, return nullptr; } -Expected -utils::elf::getSymbol(const ELFObjectFile &ELFObj, StringRef Name) { +template +static Expected +getSymbol(const ELFObjectFile &ELFObj, StringRef Name) { // First try to look up the symbol via the hash table. for (ELFSectionRef Sec : ELFObj.sections()) { if (Sec.getType() != SHT_HASH && Sec.getType() != SHT_GNU_HASH) @@ -241,8 +253,7 @@ utils::elf::getSymbol(const ELFObjectFile &ELFObj, StringRef Name) { auto HashTabOrErr = ELFObj.getELFFile().getSection(Sec.getIndex()); if (!HashTabOrErr) return HashTabOrErr.takeError(); - return getHashTableSymbol(ELFObj.getELFFile(), **HashTabOrErr, - Name); + return getHashTableSymbol(ELFObj.getELFFile(), **HashTabOrErr, Name); } // If this is an executable file check the entire standard symbol table. @@ -253,16 +264,17 @@ utils::elf::getSymbol(const ELFObjectFile &ELFObj, StringRef Name) { auto SymTabOrErr = ELFObj.getELFFile().getSection(Sec.getIndex()); if (!SymTabOrErr) return SymTabOrErr.takeError(); - return getSymTableSymbol(ELFObj.getELFFile(), **SymTabOrErr, Name); + return getSymTableSymbol(ELFObj.getELFFile(), **SymTabOrErr, Name); } return nullptr; } -Expected utils::elf::getSymbolAddress( - const object::ELFObjectFile &ELFObj, - const object::ELF64LE::Sym &Symbol) { - const ELFFile &ELFFile = ELFObj.getELFFile(); +template +static Expected +getSymbolAddress(const object::ELFObjectFile &ELFObj, + const typename ELFT::Sym &Symbol) { + const ELFFile &ELFFile = ELFObj.getELFFile(); auto SecOrErr = ELFFile.getSection(Symbol.st_shndx); if (!SecOrErr) @@ -283,3 +295,40 @@ Expected utils::elf::getSymbolAddress( return ELFFile.base() + Offset; } + +template +static Expected> +findSymbolInImageImpl(const object::ELFObjectFile &ELFObj, + StringRef Name) { + auto SymOrErr = getSymbol(ELFObj, Name); + if (!SymOrErr) + return SymOrErr.takeError(); + if (!*SymOrErr) + return std::nullopt; + + // If the symbol was found, return a StringRef covering the associated data, + // based on the symbol's address and size. + auto AddrOrErr = getSymbolAddress(ELFObj, **SymOrErr); + if (!AddrOrErr) + return AddrOrErr.takeError(); + return StringRef(static_cast(*AddrOrErr), (*SymOrErr)->st_size); +} + +Expected> +utils::elf::findSymbolInImage(StringRef Obj, StringRef Name) { + assert(isELF(Obj) && "Input is not an ELF!"); + + Expected> ElfOrErr = + ObjectFile::createELFObjectFile(MemoryBufferRef(Obj, /*Identifier=*/""), + /*InitContent=*/false); + if (!ElfOrErr) + return ElfOrErr.takeError(); + + if (const ELF64LEObjectFile *ELFObj = + dyn_cast(&**ElfOrErr)) + return findSymbolInImageImpl(*ELFObj, Name); + if (const ELF64BEObjectFile *ELFObj = + dyn_cast(&**ElfOrErr)) + return findSymbolInImageImpl(*ELFObj, Name); + return createError("Only 64-bit ELF files are supported"); +} -- GitLab From 3ecd38c8e1d34b1e4639a1de9f0cb56c7957cbd2 Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Wed, 6 Mar 2024 20:50:01 +0100 Subject: [PATCH 347/929] [libomptarget] Build plugins-nextgen for SystemZ (#83978) The plugin was not getting built as the build_generic_elf64 macro assumes the LLVM triple processor name matches the CMake processor name, which is unfortunately not the case for SystemZ. Fix this by providing two separate arguments instead. Actually building the plugin exposed a number of other issues causing various test failures. Specifically, I've had to add the SystemZ target to - CompilerInvocation::ParseLangArgs - linkDevice in ClangLinuxWrapper.cpp - OMPContext::OMPContext (to set the device_kind_cpu trait) - LIBOMPTARGET_ALL_TARGETS in libomptarget/CMakeLists.txt - a check_plugin_target call in libomptarget/src/CMakeLists.txt Finally, I've had to set a number of test cases to UNSUPPORTED on s390x-ibm-linux-gnu; all these tests were already marked as UNSUPPORTED for x86_64-pc-linux-gnu and aarch64-unknown-linux-gnu and are failing on s390x for what seem to be the same reason. In addition, this also requires support for BE ELF files in plugins-nextgen: https://github.com/llvm/llvm-project/pull/83976 --- clang/lib/Frontend/CompilerInvocation.cpp | 1 + clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp | 1 + llvm/lib/Frontend/OpenMP/OMPContext.cpp | 1 + openmp/libomptarget/CMakeLists.txt | 2 ++ openmp/libomptarget/plugins-nextgen/CMakeLists.txt | 9 ++++++--- .../libomptarget/plugins-nextgen/aarch64/CMakeLists.txt | 2 +- openmp/libomptarget/plugins-nextgen/ppc64/CMakeLists.txt | 2 +- .../libomptarget/plugins-nextgen/ppc64le/CMakeLists.txt | 2 +- openmp/libomptarget/plugins-nextgen/s390x/CMakeLists.txt | 2 +- .../libomptarget/plugins-nextgen/x86_64/CMakeLists.txt | 2 +- openmp/libomptarget/src/CMakeLists.txt | 1 + openmp/libomptarget/test/api/omp_dynamic_shared_memory.c | 2 ++ openmp/libomptarget/test/jit/empty_kernel_lvl1.c | 2 ++ openmp/libomptarget/test/jit/empty_kernel_lvl2.c | 2 ++ openmp/libomptarget/test/jit/type_punning.c | 2 ++ openmp/libomptarget/test/mapping/auto_zero_copy.cpp | 2 ++ .../libomptarget/test/mapping/auto_zero_copy_globals.cpp | 2 ++ openmp/libomptarget/test/offloading/barrier_fence.c | 2 ++ openmp/libomptarget/test/offloading/bug49334.cpp | 2 ++ .../libomptarget/test/offloading/default_thread_limit.c | 2 ++ openmp/libomptarget/test/offloading/ompx_bare.c | 5 ++++- openmp/libomptarget/test/offloading/ompx_coords.c | 2 ++ openmp/libomptarget/test/offloading/ompx_saxpy_mixed.c | 2 ++ .../test/offloading/parallel_target_teams_reduction.cpp | 2 ++ openmp/libomptarget/test/offloading/small_trip_count.c | 2 ++ .../test/offloading/small_trip_count_thread_limit.cpp | 2 ++ openmp/libomptarget/test/offloading/spmdization.c | 2 ++ .../test/offloading/target_critical_region.cpp | 2 ++ openmp/libomptarget/test/offloading/thread_limit.c | 2 ++ openmp/libomptarget/test/ompt/target_memcpy.c | 2 ++ openmp/libomptarget/test/ompt/target_memcpy_emi.c | 2 ++ openmp/libomptarget/test/ompt/veccopy.c | 2 ++ openmp/libomptarget/test/ompt/veccopy_data.c | 2 ++ openmp/libomptarget/test/ompt/veccopy_disallow_both.c | 2 ++ openmp/libomptarget/test/ompt/veccopy_emi.c | 2 ++ openmp/libomptarget/test/ompt/veccopy_emi_map.c | 2 ++ openmp/libomptarget/test/ompt/veccopy_map.c | 2 ++ 37 files changed, 71 insertions(+), 9 deletions(-) diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index 691f3b989b81..7e17c3e49aca 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -4012,6 +4012,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args, if (TT.getArch() == llvm::Triple::UnknownArch || !(TT.getArch() == llvm::Triple::aarch64 || TT.isPPC() || + TT.getArch() == llvm::Triple::systemz || TT.getArch() == llvm::Triple::nvptx || TT.getArch() == llvm::Triple::nvptx64 || TT.getArch() == llvm::Triple::amdgcn || diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp index 576e8f2cd7f8..b3efa56b3302 100644 --- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp +++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp @@ -552,6 +552,7 @@ Expected linkDevice(ArrayRef InputFiles, case Triple::aarch64_be: case Triple::ppc64: case Triple::ppc64le: + case Triple::systemz: return generic::clang(InputFiles, Args); default: return createStringError(inconvertibleErrorCode(), diff --git a/llvm/lib/Frontend/OpenMP/OMPContext.cpp b/llvm/lib/Frontend/OpenMP/OMPContext.cpp index e870c5aa2ba6..37936d6000c8 100644 --- a/llvm/lib/Frontend/OpenMP/OMPContext.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPContext.cpp @@ -44,6 +44,7 @@ OMPContext::OMPContext(bool IsDeviceCompilation, Triple TargetTriple) { case Triple::ppcle: case Triple::ppc64: case Triple::ppc64le: + case Triple::systemz: case Triple::x86: case Triple::x86_64: ActiveTraits.set(unsigned(TraitProperty::device_kind_cpu)); diff --git a/openmp/libomptarget/CMakeLists.txt b/openmp/libomptarget/CMakeLists.txt index a74eff0c0beb..b382137b70ee 100644 --- a/openmp/libomptarget/CMakeLists.txt +++ b/openmp/libomptarget/CMakeLists.txt @@ -56,6 +56,8 @@ set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} x86_64-pc-linux-gnu-L set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} nvptx64-nvidia-cuda") set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} nvptx64-nvidia-cuda-LTO") set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} nvptx64-nvidia-cuda-JIT-LTO") +set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} s390x-ibm-linux-gnu") +set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} s390x-ibm-linux-gnu-LTO") # Once the plugins for the different targets are validated, they will be added to # the list of supported targets in the current system. diff --git a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt index 3ca02368253e..b6fc136e8a17 100644 --- a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt @@ -12,12 +12,15 @@ add_subdirectory(common) -# void build_generic_elf64(string tmachine, string tmachine_name, string tmachine_libname, string elf_machine_id); +# void build_generic_elf64(string tmachine, string tmachine_name, string tmachine_libname, +# string tmachine_llvm, string tmachine_triple, string elf_machine_id); # - build a plugin for an ELF based generic 64-bit target based on libffi. # - tmachine: name of the machine processor as used in the cmake build system. # - tmachine_name: name of the machine to be printed with the debug messages. # - tmachine_libname: machine name to be appended to the plugin library name. -macro(build_generic_elf64 tmachine tmachine_name tmachine_libname tmachine_triple elf_machine_id) +# - tmachine_llvm: LLVM triple for the processor +# - tmachine_triple: GNU target triple +macro(build_generic_elf64 tmachine tmachine_name tmachine_libname tmachine_llvm tmachine_triple elf_machine_id) if(CMAKE_SYSTEM_PROCESSOR MATCHES "${tmachine}$") # Define macro to be used as prefix of the runtime messages for this target. add_definitions("-DTARGET_NAME=${tmachine_name}") @@ -30,7 +33,7 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "${tmachine}$") add_definitions("-DTARGET_ELF_ID=${elf_machine_id}") # Define target triple - add_definitions("-DLIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE=${tmachine}") + add_definitions("-DLIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE=${tmachine_llvm}") add_llvm_library("omptarget.rtl.${tmachine_libname}" SHARED diff --git a/openmp/libomptarget/plugins-nextgen/aarch64/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/aarch64/CMakeLists.txt index 2c2b753590e2..663ab4d60ff9 100644 --- a/openmp/libomptarget/plugins-nextgen/aarch64/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/aarch64/CMakeLists.txt @@ -11,7 +11,7 @@ ##===----------------------------------------------------------------------===## if(CMAKE_SYSTEM_NAME MATCHES "Linux") - build_generic_elf64("aarch64" "aarch64" "aarch64" "aarch64-unknown-linux-gnu" "183") + build_generic_elf64("aarch64" "aarch64" "aarch64" "aarch64" "aarch64-unknown-linux-gnu" "183") else() libomptarget_say("Not building aarch64 NextGen offloading plugin: machine not found in the system.") endif() diff --git a/openmp/libomptarget/plugins-nextgen/ppc64/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/ppc64/CMakeLists.txt index 0cccc9cb82e4..77466c111ee0 100644 --- a/openmp/libomptarget/plugins-nextgen/ppc64/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/ppc64/CMakeLists.txt @@ -11,7 +11,7 @@ ##===----------------------------------------------------------------------===## if(CMAKE_SYSTEM_NAME MATCHES "Linux") - build_generic_elf64("ppc64" "PPC64" "ppc64" "powerpc64-ibm-linux-gnu" "21") + build_generic_elf64("ppc64" "PPC64" "ppc64" "ppc64" "powerpc64-ibm-linux-gnu" "21") else() libomptarget_say("Not building ppc64 NextGen offloading plugin: machine not found in the system.") endif() diff --git a/openmp/libomptarget/plugins-nextgen/ppc64le/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/ppc64le/CMakeLists.txt index 9461d79d145d..91d21627a327 100644 --- a/openmp/libomptarget/plugins-nextgen/ppc64le/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/ppc64le/CMakeLists.txt @@ -11,7 +11,7 @@ ##===----------------------------------------------------------------------===## if(CMAKE_SYSTEM_NAME MATCHES "Linux") - build_generic_elf64("ppc64le" "PPC64le" "ppc64" "powerpc64le-ibm-linux-gnu" "21") + build_generic_elf64("ppc64le" "PPC64le" "ppc64" "ppc64le" "powerpc64le-ibm-linux-gnu" "21") else() libomptarget_say("Not building ppc64le NextGen offloading plugin: machine not found in the system.") endif() diff --git a/openmp/libomptarget/plugins-nextgen/s390x/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/s390x/CMakeLists.txt index 1b12a2928999..0388a235d289 100644 --- a/openmp/libomptarget/plugins-nextgen/s390x/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/s390x/CMakeLists.txt @@ -11,7 +11,7 @@ ##===----------------------------------------------------------------------===## if(CMAKE_SYSTEM_NAME MATCHES "Linux") - build_generic_elf64("SystemZ" "S390X" "s390x" "s390x-ibm-linux-gnu" "22") + build_generic_elf64("s390x" "S390X" "s390x" "systemz" "s390x-ibm-linux-gnu" "22") else() libomptarget_say("Not building s390x NextGen offloading plugin: machine not found in the system.") endif() diff --git a/openmp/libomptarget/plugins-nextgen/x86_64/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/x86_64/CMakeLists.txt index 129d526a2ae7..27cf3e069a37 100644 --- a/openmp/libomptarget/plugins-nextgen/x86_64/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/x86_64/CMakeLists.txt @@ -11,7 +11,7 @@ ##===----------------------------------------------------------------------===## if(CMAKE_SYSTEM_NAME MATCHES "Linux") - build_generic_elf64("x86_64" "x86_64" "x86_64" "x86_64-pc-linux-gnu" "62") + build_generic_elf64("x86_64" "x86_64" "x86_64" "x86_64" "x86_64-pc-linux-gnu" "62") else() libomptarget_say("Not building x86_64 NextGen offloading plugin: machine not found in the system.") endif() diff --git a/openmp/libomptarget/src/CMakeLists.txt b/openmp/libomptarget/src/CMakeLists.txt index 9bc3f3339583..d0971bd4ef07 100644 --- a/openmp/libomptarget/src/CMakeLists.txt +++ b/openmp/libomptarget/src/CMakeLists.txt @@ -73,6 +73,7 @@ if (NOT LIBOMPTARGET_PLUGINS_TO_LOAD) check_plugin_target(cuda) check_plugin_target(aarch64) check_plugin_target(amdgpu) + check_plugin_target(s390x) endif() list(TRANSFORM LIBOMPTARGET_PLUGINS_TO_LOAD PREPEND "\"libomptarget.rtl.") diff --git a/openmp/libomptarget/test/api/omp_dynamic_shared_memory.c b/openmp/libomptarget/test/api/omp_dynamic_shared_memory.c index 5095a69a375c..3fe75f24db3e 100644 --- a/openmp/libomptarget/test/api/omp_dynamic_shared_memory.c +++ b/openmp/libomptarget/test/api/omp_dynamic_shared_memory.c @@ -10,6 +10,8 @@ // UNSUPPORTED: x86_64-pc-linux-gnu-LTO // UNSUPPORTED: aarch64-unknown-linux-gnu // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO #include #include diff --git a/openmp/libomptarget/test/jit/empty_kernel_lvl1.c b/openmp/libomptarget/test/jit/empty_kernel_lvl1.c index c908c8ba0650..a0b8cd448837 100644 --- a/openmp/libomptarget/test/jit/empty_kernel_lvl1.c +++ b/openmp/libomptarget/test/jit/empty_kernel_lvl1.c @@ -32,5 +32,7 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO #include "empty_kernel.inc" diff --git a/openmp/libomptarget/test/jit/empty_kernel_lvl2.c b/openmp/libomptarget/test/jit/empty_kernel_lvl2.c index 0b88d33292c5..81a04f55ce43 100644 --- a/openmp/libomptarget/test/jit/empty_kernel_lvl2.c +++ b/openmp/libomptarget/test/jit/empty_kernel_lvl2.c @@ -92,5 +92,7 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO #include "empty_kernel.inc" diff --git a/openmp/libomptarget/test/jit/type_punning.c b/openmp/libomptarget/test/jit/type_punning.c index 23aa69bba7b9..10e3d2cef718 100644 --- a/openmp/libomptarget/test/jit/type_punning.c +++ b/openmp/libomptarget/test/jit/type_punning.c @@ -12,6 +12,8 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO // Ensure that there is only the kernel function left, not any outlined // parallel regions. diff --git a/openmp/libomptarget/test/mapping/auto_zero_copy.cpp b/openmp/libomptarget/test/mapping/auto_zero_copy.cpp index 6f9d8c2b128c..46641200bb56 100644 --- a/openmp/libomptarget/test/mapping/auto_zero_copy.cpp +++ b/openmp/libomptarget/test/mapping/auto_zero_copy.cpp @@ -13,6 +13,8 @@ // UNSUPPORTED: nvptx64-nvidia-cuda-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO // REQUIRES: unified_shared_memory diff --git a/openmp/libomptarget/test/mapping/auto_zero_copy_globals.cpp b/openmp/libomptarget/test/mapping/auto_zero_copy_globals.cpp index 4a13d270aeeb..55dfb2807ebc 100644 --- a/openmp/libomptarget/test/mapping/auto_zero_copy_globals.cpp +++ b/openmp/libomptarget/test/mapping/auto_zero_copy_globals.cpp @@ -9,6 +9,8 @@ // UNSUPPORTED: nvptx64-nvidia-cuda-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO // REQUIRES: unified_shared_memory diff --git a/openmp/libomptarget/test/offloading/barrier_fence.c b/openmp/libomptarget/test/offloading/barrier_fence.c index 5d1096478ed9..b9a8ca27965a 100644 --- a/openmp/libomptarget/test/offloading/barrier_fence.c +++ b/openmp/libomptarget/test/offloading/barrier_fence.c @@ -7,6 +7,8 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO #include #include diff --git a/openmp/libomptarget/test/offloading/bug49334.cpp b/openmp/libomptarget/test/offloading/bug49334.cpp index a22d3fe9f665..1f19dab37881 100644 --- a/openmp/libomptarget/test/offloading/bug49334.cpp +++ b/openmp/libomptarget/test/offloading/bug49334.cpp @@ -9,6 +9,8 @@ // UNSUPPORTED: x86_64-pc-linux-gnu-LTO // UNSUPPORTED: aarch64-unknown-linux-gnu // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO // UNSUPPORTED: amdgcn-amd-amdhsa // UNSUPPORTED: nvptx64-nvidia-cuda // UNSUPPORTED: nvptx64-nvidia-cuda-LTO diff --git a/openmp/libomptarget/test/offloading/default_thread_limit.c b/openmp/libomptarget/test/offloading/default_thread_limit.c index d32e7df418cb..4da02bbb152e 100644 --- a/openmp/libomptarget/test/offloading/default_thread_limit.c +++ b/openmp/libomptarget/test/offloading/default_thread_limit.c @@ -9,6 +9,8 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO __attribute__((optnone)) int optnone() { return 1; } diff --git a/openmp/libomptarget/test/offloading/ompx_bare.c b/openmp/libomptarget/test/offloading/ompx_bare.c index fb3810bd1df1..3dabdcd15e0d 100644 --- a/openmp/libomptarget/test/offloading/ompx_bare.c +++ b/openmp/libomptarget/test/offloading/ompx_bare.c @@ -1,10 +1,13 @@ // RUN: %libomptarget-compile-generic -// RUN: env LIBOMPTARGET_INFO=63 %libomptarget-run-generic 2>&1 | %fcheck-generic +// RUN: env LIBOMPTARGET_INFO=63 %libomptarget-run-generic 2>&1 | \ +// RUN: %fcheck-generic // // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO // UNSUPPORTED: aarch64-unknown-linux-gnu // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO #include #include diff --git a/openmp/libomptarget/test/offloading/ompx_coords.c b/openmp/libomptarget/test/offloading/ompx_coords.c index 61dad61f4640..5e4e14b4c6da 100644 --- a/openmp/libomptarget/test/offloading/ompx_coords.c +++ b/openmp/libomptarget/test/offloading/ompx_coords.c @@ -4,6 +4,8 @@ // UNSUPPORTED: x86_64-pc-linux-gnu-LTO // UNSUPPORTED: aarch64-unknown-linux-gnu // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO #include #include diff --git a/openmp/libomptarget/test/offloading/ompx_saxpy_mixed.c b/openmp/libomptarget/test/offloading/ompx_saxpy_mixed.c index 440b694e3ac7..f479be8a484f 100644 --- a/openmp/libomptarget/test/offloading/ompx_saxpy_mixed.c +++ b/openmp/libomptarget/test/offloading/ompx_saxpy_mixed.c @@ -4,6 +4,8 @@ // UNSUPPORTED: x86_64-pc-linux-gnu-LTO // UNSUPPORTED: aarch64-unknown-linux-gnu // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO #include #include diff --git a/openmp/libomptarget/test/offloading/parallel_target_teams_reduction.cpp b/openmp/libomptarget/test/offloading/parallel_target_teams_reduction.cpp index 5303a9463f15..10e1b33f1ce4 100644 --- a/openmp/libomptarget/test/offloading/parallel_target_teams_reduction.cpp +++ b/openmp/libomptarget/test/offloading/parallel_target_teams_reduction.cpp @@ -6,6 +6,8 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO #include #include diff --git a/openmp/libomptarget/test/offloading/small_trip_count.c b/openmp/libomptarget/test/offloading/small_trip_count.c index d8bef667607a..65f094f15746 100644 --- a/openmp/libomptarget/test/offloading/small_trip_count.c +++ b/openmp/libomptarget/test/offloading/small_trip_count.c @@ -9,6 +9,8 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO #define N 128 diff --git a/openmp/libomptarget/test/offloading/small_trip_count_thread_limit.cpp b/openmp/libomptarget/test/offloading/small_trip_count_thread_limit.cpp index 9796c2dc1166..b7ae52a62c83 100644 --- a/openmp/libomptarget/test/offloading/small_trip_count_thread_limit.cpp +++ b/openmp/libomptarget/test/offloading/small_trip_count_thread_limit.cpp @@ -7,6 +7,8 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO int main(int argc, char *argv[]) { constexpr const int block_size = 256; diff --git a/openmp/libomptarget/test/offloading/spmdization.c b/openmp/libomptarget/test/offloading/spmdization.c index 2cf30ff593b9..77913bec8342 100644 --- a/openmp/libomptarget/test/offloading/spmdization.c +++ b/openmp/libomptarget/test/offloading/spmdization.c @@ -11,6 +11,8 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO #include #include diff --git a/openmp/libomptarget/test/offloading/target_critical_region.cpp b/openmp/libomptarget/test/offloading/target_critical_region.cpp index 9a741bef6c59..495632bf76e1 100644 --- a/openmp/libomptarget/test/offloading/target_critical_region.cpp +++ b/openmp/libomptarget/test/offloading/target_critical_region.cpp @@ -6,6 +6,8 @@ // UNSUPPORTED: nvptx64-nvidia-cuda-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO // UNSUPPORTED: amdgcn-amd-amdhsa #include diff --git a/openmp/libomptarget/test/offloading/thread_limit.c b/openmp/libomptarget/test/offloading/thread_limit.c index 65275a8f2795..a8cc51b651dc 100644 --- a/openmp/libomptarget/test/offloading/thread_limit.c +++ b/openmp/libomptarget/test/offloading/thread_limit.c @@ -9,6 +9,8 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO int main() { int n = 1 << 20; diff --git a/openmp/libomptarget/test/ompt/target_memcpy.c b/openmp/libomptarget/test/ompt/target_memcpy.c index 80a8d6a4b32e..3224c5ac4c91 100644 --- a/openmp/libomptarget/test/ompt/target_memcpy.c +++ b/openmp/libomptarget/test/ompt/target_memcpy.c @@ -4,6 +4,8 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO /* * Verify that for the target OpenMP APIs, the return address is non-null and diff --git a/openmp/libomptarget/test/ompt/target_memcpy_emi.c b/openmp/libomptarget/test/ompt/target_memcpy_emi.c index 5347f38b87b6..fd7da13cb05d 100644 --- a/openmp/libomptarget/test/ompt/target_memcpy_emi.c +++ b/openmp/libomptarget/test/ompt/target_memcpy_emi.c @@ -4,6 +4,8 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO /* * Verify all three data transfer directions: H2D, D2D and D2H diff --git a/openmp/libomptarget/test/ompt/veccopy.c b/openmp/libomptarget/test/ompt/veccopy.c index 80e71fd8a48c..bc70f5a0b5b6 100644 --- a/openmp/libomptarget/test/ompt/veccopy.c +++ b/openmp/libomptarget/test/ompt/veccopy.c @@ -4,6 +4,8 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO /* * Example OpenMP program that registers non-EMI callbacks diff --git a/openmp/libomptarget/test/ompt/veccopy_data.c b/openmp/libomptarget/test/ompt/veccopy_data.c index cef1de316a7a..264b484b881f 100644 --- a/openmp/libomptarget/test/ompt/veccopy_data.c +++ b/openmp/libomptarget/test/ompt/veccopy_data.c @@ -4,6 +4,8 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO /* * Example OpenMP program that registers EMI callbacks. diff --git a/openmp/libomptarget/test/ompt/veccopy_disallow_both.c b/openmp/libomptarget/test/ompt/veccopy_disallow_both.c index 06293e413ba4..a011c21955bb 100644 --- a/openmp/libomptarget/test/ompt/veccopy_disallow_both.c +++ b/openmp/libomptarget/test/ompt/veccopy_disallow_both.c @@ -4,6 +4,8 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO /* * Example OpenMP program that shows that both EMI and non-EMI diff --git a/openmp/libomptarget/test/ompt/veccopy_emi.c b/openmp/libomptarget/test/ompt/veccopy_emi.c index b597d7be6aff..8718a39b4af8 100644 --- a/openmp/libomptarget/test/ompt/veccopy_emi.c +++ b/openmp/libomptarget/test/ompt/veccopy_emi.c @@ -4,6 +4,8 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO /* * Example OpenMP program that registers EMI callbacks diff --git a/openmp/libomptarget/test/ompt/veccopy_emi_map.c b/openmp/libomptarget/test/ompt/veccopy_emi_map.c index ce6f6e30d581..2accba34034e 100644 --- a/openmp/libomptarget/test/ompt/veccopy_emi_map.c +++ b/openmp/libomptarget/test/ompt/veccopy_emi_map.c @@ -4,6 +4,8 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO /* * Example OpenMP program that shows that map-EMI callbacks are not supported. diff --git a/openmp/libomptarget/test/ompt/veccopy_map.c b/openmp/libomptarget/test/ompt/veccopy_map.c index 83f63a6e6049..56a0dd48f536 100644 --- a/openmp/libomptarget/test/ompt/veccopy_map.c +++ b/openmp/libomptarget/test/ompt/veccopy_map.c @@ -4,6 +4,8 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO +// UNSUPPORTED: s390x-ibm-linux-gnu +// UNSUPPORTED: s390x-ibm-linux-gnu-LTO /* * Example OpenMP program that shows that map callbacks are not supported. -- GitLab From b64482e23eefaef7738fde35d0b7c4174aaa6597 Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Wed, 6 Mar 2024 21:00:25 +0100 Subject: [PATCH 348/929] [libomptarget] Fix CUDA plugin build regression After 3ecd38c8e, the Handler.getELFObjectFile routine is no longer available. Call ELF64LEObjectFile::create directly, which should always be suitable for CUDA images. --- openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp b/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp index f85a00cd1cd5..e9742c9a13de 100644 --- a/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp +++ b/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp @@ -1160,7 +1160,7 @@ private: // automatically so we must create it ourselves. The backend will emit // several globals that contain function pointers we can call. These are // prefixed with a known name due to Nvidia's lack of section support. - auto ELFObjOrErr = Handler.getELFObjectFile(Image); + auto ELFObjOrErr = ELF64LEObjectFile::create(Image.getMemoryBuffer()); if (!ELFObjOrErr) return ELFObjOrErr.takeError(); -- GitLab From eb8f379567e8d014194faefe02ce92813e237afc Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 6 Mar 2024 20:08:00 +0000 Subject: [PATCH 349/929] [DSE] Remove malloc from EarliestEscapeInfo before removing. (#84157) Not removing the malloc from earliest escape info leaves stale entries in the cache. Fixes https://github.com/llvm/llvm-project/issues/84051. PR: https://github.com/llvm/llvm-project/pull/84157 --- .../Scalar/DeadStoreElimination.cpp | 4 +- ...alloc-earliest-escape-info-invalidation.ll | 302 ++++++++++++++++++ 2 files changed, 304 insertions(+), 2 deletions(-) create mode 100644 llvm/test/Transforms/DeadStoreElimination/malloc-earliest-escape-info-invalidation.ll diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp index 3d1dac5ea17e..392e6ad5a66b 100644 --- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp +++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp @@ -1908,15 +1908,15 @@ struct DSEState { Malloc->getArgOperand(0), IRB, TLI); if (!Calloc) return false; + MemorySSAUpdater Updater(&MSSA); auto *NewAccess = Updater.createMemoryAccessAfter(cast(Calloc), nullptr, MallocDef); auto *NewAccessMD = cast(NewAccess); Updater.insertDef(NewAccessMD, /*RenameUses=*/true); - Updater.removeMemoryAccess(Malloc); Malloc->replaceAllUsesWith(Calloc); - Malloc->eraseFromParent(); + deleteDeadInstruction(Malloc); return true; } diff --git a/llvm/test/Transforms/DeadStoreElimination/malloc-earliest-escape-info-invalidation.ll b/llvm/test/Transforms/DeadStoreElimination/malloc-earliest-escape-info-invalidation.ll new file mode 100644 index 000000000000..60a010cd49ce --- /dev/null +++ b/llvm/test/Transforms/DeadStoreElimination/malloc-earliest-escape-info-invalidation.ll @@ -0,0 +1,302 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -p dse -S %s | FileCheck %s + +target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64" + +define void @widget(ptr %a) { +; CHECK-LABEL: define void @widget( +; CHECK-SAME: ptr [[A:%.*]]) { +; CHECK-NEXT: bb: +; CHECK-NEXT: [[CALL1:%.*]] = tail call noalias ptr @malloc(i64 0) +; CHECK-NEXT: store ptr [[CALL1]], ptr [[A]], align 8 +; CHECK-NEXT: [[LOAD:%.*]] = load ptr, ptr [[A]], align 8 +; CHECK-NEXT: [[LOAD2:%.*]] = load i32, ptr [[LOAD]], align 8 +; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr i8, ptr [[CALL1]], i64 0 +; CHECK-NEXT: [[GETELEMENTPTR3:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR]], i64 1 +; CHECK-NEXT: [[GETELEMENTPTR4:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR]], i64 8 +; CHECK-NEXT: store i16 0, ptr [[GETELEMENTPTR4]], align 4 +; CHECK-NEXT: [[GETELEMENTPTR5:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR]], i64 12 +; CHECK-NEXT: store i32 0, ptr [[CALL1]], align 4 +; CHECK-NEXT: [[LOAD6:%.*]] = load i32, ptr inttoptr (i64 4 to ptr), align 4 +; CHECK-NEXT: br label [[BB48:%.*]] +; CHECK: bb7: +; CHECK-NEXT: br label [[BB9:%.*]] +; CHECK: bb8: +; CHECK-NEXT: br label [[BB53:%.*]] +; CHECK: bb9: +; CHECK-NEXT: [[PHI:%.*]] = phi ptr [ [[CALL1]], [[BB7:%.*]] ], [ [[A]], [[BB43:%.*]] ] +; CHECK-NEXT: [[GETELEMENTPTR10:%.*]] = getelementptr i8, ptr [[PHI]], i64 0 +; CHECK-NEXT: [[GETELEMENTPTR11:%.*]] = getelementptr i8, ptr [[PHI]], i64 0 +; CHECK-NEXT: [[GETELEMENTPTR12:%.*]] = getelementptr i8, ptr [[PHI]], i64 0 +; CHECK-NEXT: [[GETELEMENTPTR13:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR12]], i64 1 +; CHECK-NEXT: store i8 0, ptr [[CALL1]], align 1 +; CHECK-NEXT: br label [[BB29:%.*]] +; CHECK: bb14: +; CHECK-NEXT: [[GETELEMENTPTR15:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR10]], i64 8 +; CHECK-NEXT: [[LOAD16:%.*]] = load i16, ptr [[CALL1]], align 4 +; CHECK-NEXT: br i1 false, label [[BB22:%.*]], label [[BB17:%.*]] +; CHECK: bb17: +; CHECK-NEXT: [[GETELEMENTPTR18:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR11]], i64 8 +; CHECK-NEXT: [[LOAD19:%.*]] = load i16, ptr [[CALL1]], align 4 +; CHECK-NEXT: [[GETELEMENTPTR20:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR12]], i64 8 +; CHECK-NEXT: store i16 0, ptr [[CALL1]], align 4 +; CHECK-NEXT: [[GETELEMENTPTR21:%.*]] = getelementptr i8, ptr [[PHI]], i64 0 +; CHECK-NEXT: br label [[BB25:%.*]] +; CHECK: bb22: +; CHECK-NEXT: [[GETELEMENTPTR23:%.*]] = getelementptr i8, ptr [[PHI]], i64 0 +; CHECK-NEXT: [[GETELEMENTPTR24:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR23]], i64 12 +; CHECK-NEXT: br label [[BB25]] +; CHECK: bb25: +; CHECK-NEXT: [[PHI26:%.*]] = phi ptr [ [[A]], [[BB17]] ], [ [[CALL1]], [[BB22]] ] +; CHECK-NEXT: [[PHI27:%.*]] = phi ptr [ [[CALL1]], [[BB17]] ], [ [[CALL1]], [[BB22]] ] +; CHECK-NEXT: [[PHI28:%.*]] = phi ptr [ [[CALL1]], [[BB17]] ], [ [[CALL1]], [[BB22]] ] +; CHECK-NEXT: store i32 0, ptr [[CALL1]], align 4 +; CHECK-NEXT: br label [[BB29]] +; CHECK: bb29: +; CHECK-NEXT: [[PHI30:%.*]] = phi ptr [ [[CALL1]], [[BB9]] ], [ [[CALL1]], [[BB25]] ] +; CHECK-NEXT: [[PHI31:%.*]] = phi ptr [ [[CALL1]], [[BB9]] ], [ [[CALL1]], [[BB25]] ] +; CHECK-NEXT: [[LOAD32:%.*]] = load i8, ptr [[CALL1]], align 4 +; CHECK-NEXT: [[LOAD33:%.*]] = load i8, ptr [[CALL1]], align 4 +; CHECK-NEXT: [[GETELEMENTPTR34:%.*]] = getelementptr i8, ptr [[PHI31]], i64 12 +; CHECK-NEXT: [[GETELEMENTPTR35:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR12]], i64 12 +; CHECK-NEXT: br label [[BB86:%.*]] +; CHECK: bb36: +; CHECK-NEXT: [[GETELEMENTPTR37:%.*]] = getelementptr i8, ptr [[PHI30]], i64 12 +; CHECK-NEXT: br label [[BB38:%.*]] +; CHECK: bb38: +; CHECK-NEXT: [[GETELEMENTPTR39:%.*]] = getelementptr [0 x i32], ptr [[GETELEMENTPTR34]], i64 0, i64 0 +; CHECK-NEXT: [[LOAD40:%.*]] = load i32, ptr [[CALL1]], align 4 +; CHECK-NEXT: [[GETELEMENTPTR41:%.*]] = getelementptr [0 x i32], ptr [[GETELEMENTPTR37]], i64 0, i64 0 +; CHECK-NEXT: [[LOAD42:%.*]] = load i32, ptr [[CALL1]], align 4 +; CHECK-NEXT: br label [[BB38]] +; CHECK: bb43: +; CHECK-NEXT: [[GETELEMENTPTR44:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR12]], i64 8 +; CHECK-NEXT: [[LOAD45:%.*]] = load i16, ptr [[CALL1]], align 4 +; CHECK-NEXT: store i16 0, ptr [[CALL1]], align 4 +; CHECK-NEXT: store i8 0, ptr [[CALL1]], align 4 +; CHECK-NEXT: [[GETELEMENTPTR46:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR12]], i64 12 +; CHECK-NEXT: store i32 0, ptr [[CALL1]], align 4 +; CHECK-NEXT: [[GETELEMENTPTR47:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR12]], i64 16 +; CHECK-NEXT: store i32 0, ptr [[CALL1]], align 4 +; CHECK-NEXT: br label [[BB9]] +; CHECK: bb48: +; CHECK-NEXT: [[GETELEMENTPTR49:%.*]] = getelementptr i8, ptr [[CALL1]], i64 0 +; CHECK-NEXT: [[GETELEMENTPTR50:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR49]], i64 1 +; CHECK-NEXT: [[GETELEMENTPTR51:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR49]], i64 8 +; CHECK-NEXT: [[GETELEMENTPTR52:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR49]], i64 12 +; CHECK-NEXT: store i32 0, ptr [[CALL1]], align 4 +; CHECK-NEXT: br label [[BB48]] +; CHECK: bb53: +; CHECK-NEXT: [[PHI54:%.*]] = phi ptr [ [[CALL1]], [[BB8:%.*]] ], [ [[A]], [[BB71:%.*]] ] +; CHECK-NEXT: [[GETELEMENTPTR55:%.*]] = getelementptr i8, ptr [[PHI54]], i64 0 +; CHECK-NEXT: [[GETELEMENTPTR56:%.*]] = getelementptr i8, ptr [[PHI54]], i64 0 +; CHECK-NEXT: [[GETELEMENTPTR57:%.*]] = getelementptr i8, ptr [[PHI54]], i64 0 +; CHECK-NEXT: [[GETELEMENTPTR58:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR57]], i64 1 +; CHECK-NEXT: br label [[BB71]] +; CHECK: bb59: +; CHECK-NEXT: [[GETELEMENTPTR60:%.*]] = getelementptr i8, ptr [[PHI54]], i64 0 +; CHECK-NEXT: [[GETELEMENTPTR61:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR60]], i64 12 +; CHECK-NEXT: br label [[BB67:%.*]] +; CHECK: bb62: +; CHECK-NEXT: [[GETELEMENTPTR63:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR56]], i64 8 +; CHECK-NEXT: [[LOAD64:%.*]] = load i16, ptr [[CALL1]], align 4 +; CHECK-NEXT: [[GETELEMENTPTR65:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR57]], i64 8 +; CHECK-NEXT: store i16 0, ptr [[CALL1]], align 4 +; CHECK-NEXT: [[GETELEMENTPTR66:%.*]] = getelementptr i8, ptr [[PHI54]], i64 0 +; CHECK-NEXT: br label [[BB67]] +; CHECK: bb67: +; CHECK-NEXT: [[PHI68:%.*]] = phi ptr [ [[A]], [[BB62:%.*]] ], [ [[CALL1]], [[BB59:%.*]] ] +; CHECK-NEXT: [[PHI69:%.*]] = phi ptr [ [[CALL1]], [[BB62]] ], [ [[CALL1]], [[BB59]] ] +; CHECK-NEXT: [[PHI70:%.*]] = phi ptr [ [[CALL1]], [[BB62]] ], [ [[CALL1]], [[BB59]] ] +; CHECK-NEXT: store i32 0, ptr [[CALL1]], align 4 +; CHECK-NEXT: br label [[BB71]] +; CHECK: bb71: +; CHECK-NEXT: [[PHI72:%.*]] = phi ptr [ [[CALL1]], [[BB53]] ], [ [[CALL1]], [[BB67]] ] +; CHECK-NEXT: [[PHI73:%.*]] = phi ptr [ [[CALL1]], [[BB53]] ], [ [[CALL1]], [[BB67]] ] +; CHECK-NEXT: [[LOAD74:%.*]] = load i8, ptr [[CALL1]], align 4 +; CHECK-NEXT: [[LOAD75:%.*]] = load i8, ptr [[CALL1]], align 4 +; CHECK-NEXT: [[GETELEMENTPTR76:%.*]] = getelementptr i8, ptr [[PHI72]], i64 12 +; CHECK-NEXT: [[GETELEMENTPTR77:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR57]], i64 12 +; CHECK-NEXT: [[GETELEMENTPTR78:%.*]] = getelementptr [0 x i32], ptr [[GETELEMENTPTR76]], i64 0, i64 0 +; CHECK-NEXT: [[LOAD79:%.*]] = load i32, ptr [[CALL1]], align 4 +; CHECK-NEXT: [[GETELEMENTPTR80:%.*]] = getelementptr [0 x i32], ptr [[GETELEMENTPTR77]], i64 0, i64 0 +; CHECK-NEXT: store i32 0, ptr [[CALL1]], align 4 +; CHECK-NEXT: [[LOAD81:%.*]] = load i8, ptr [[CALL1]], align 4 +; CHECK-NEXT: [[GETELEMENTPTR82:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR57]], i64 8 +; CHECK-NEXT: [[LOAD83:%.*]] = load i16, ptr [[CALL1]], align 4 +; CHECK-NEXT: store i16 0, ptr [[CALL1]], align 4 +; CHECK-NEXT: store i8 0, ptr [[CALL1]], align 4 +; CHECK-NEXT: [[GETELEMENTPTR84:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR57]], i64 12 +; CHECK-NEXT: store i32 0, ptr [[CALL1]], align 4 +; CHECK-NEXT: [[GETELEMENTPTR85:%.*]] = getelementptr i8, ptr [[GETELEMENTPTR57]], i64 16 +; CHECK-NEXT: store i32 0, ptr [[CALL1]], align 4 +; CHECK-NEXT: br label [[BB53]] +; CHECK: bb86: +; CHECK-NEXT: [[GETELEMENTPTR87:%.*]] = getelementptr [0 x i32], ptr [[GETELEMENTPTR34]], i64 0, i64 0 +; CHECK-NEXT: [[LOAD88:%.*]] = load i32, ptr [[CALL1]], align 4 +; CHECK-NEXT: [[GETELEMENTPTR89:%.*]] = getelementptr [0 x i32], ptr [[GETELEMENTPTR35]], i64 0, i64 0 +; CHECK-NEXT: store i32 0, ptr [[CALL1]], align 4 +; CHECK-NEXT: br label [[BB86]] +; +bb: + %call = tail call ptr @malloc(i64 1) + tail call void @llvm.memset.p0.i64(ptr %call, i8 0, i64 1, i1 false) + %call1 = tail call noalias ptr @malloc(i64 0) + store ptr %call1, ptr %a, align 8 + %load = load ptr, ptr %a, align 8 + %load2 = load i32, ptr %load, align 8 + %getelementptr = getelementptr i8, ptr %call1, i64 0 + %getelementptr3 = getelementptr i8, ptr %getelementptr, i64 1 + store i8 0, ptr %call1, align 1 + %getelementptr4 = getelementptr i8, ptr %getelementptr, i64 8 + store i16 0, ptr %getelementptr4, align 4 + %getelementptr5 = getelementptr i8, ptr %getelementptr, i64 12 + store i32 0, ptr %call1, align 4 + %load6 = load i32, ptr inttoptr (i64 4 to ptr), align 4 + br label %bb48 + +bb7: ; No predecessors! + br label %bb9 + +bb8: ; No predecessors! + br label %bb53 + +bb9: ; preds = %bb43, %bb7 + %phi = phi ptr [ %call1, %bb7 ], [ %a, %bb43 ] + %getelementptr10 = getelementptr i8, ptr %phi, i64 0 + %getelementptr11 = getelementptr i8, ptr %phi, i64 0 + %getelementptr12 = getelementptr i8, ptr %phi, i64 0 + %getelementptr13 = getelementptr i8, ptr %getelementptr12, i64 1 + store i8 0, ptr %call1, align 1 + br label %bb29 + +bb14: ; No predecessors! + %getelementptr15 = getelementptr i8, ptr %getelementptr10, i64 8 + %load16 = load i16, ptr %call1, align 4 + br i1 false, label %bb22, label %bb17 + +bb17: ; preds = %bb14 + %getelementptr18 = getelementptr i8, ptr %getelementptr11, i64 8 + %load19 = load i16, ptr %call1, align 4 + %getelementptr20 = getelementptr i8, ptr %getelementptr12, i64 8 + store i16 0, ptr %call1, align 4 + %getelementptr21 = getelementptr i8, ptr %phi, i64 0 + br label %bb25 + +bb22: ; preds = %bb14 + %getelementptr23 = getelementptr i8, ptr %phi, i64 0 + %getelementptr24 = getelementptr i8, ptr %getelementptr23, i64 12 + br label %bb25 + +bb25: ; preds = %bb22, %bb17 + %phi26 = phi ptr [ %a, %bb17 ], [ %call1, %bb22 ] + %phi27 = phi ptr [ %call1, %bb17 ], [ %call1, %bb22 ] + %phi28 = phi ptr [ %call1, %bb17 ], [ %call1, %bb22 ] + store i32 0, ptr %call1, align 4 + br label %bb29 + +bb29: ; preds = %bb25, %bb9 + %phi30 = phi ptr [ %call1, %bb9 ], [ %call1, %bb25 ] + %phi31 = phi ptr [ %call1, %bb9 ], [ %call1, %bb25 ] + %load32 = load i8, ptr %call1, align 4 + %load33 = load i8, ptr %call1, align 4 + %getelementptr34 = getelementptr i8, ptr %phi31, i64 12 + %getelementptr35 = getelementptr i8, ptr %getelementptr12, i64 12 + br label %bb86 + +bb36: ; No predecessors! + %getelementptr37 = getelementptr i8, ptr %phi30, i64 12 + br label %bb38 + +bb38: ; preds = %bb38, %bb36 + %getelementptr39 = getelementptr [0 x i32], ptr %getelementptr34, i64 0, i64 0 + %load40 = load i32, ptr %call1, align 4 + %getelementptr41 = getelementptr [0 x i32], ptr %getelementptr37, i64 0, i64 0 + %load42 = load i32, ptr %call1, align 4 + br label %bb38 + +bb43: ; No predecessors! + %getelementptr44 = getelementptr i8, ptr %getelementptr12, i64 8 + %load45 = load i16, ptr %call1, align 4 + store i16 0, ptr %call1, align 4 + store i8 0, ptr %call1, align 4 + %getelementptr46 = getelementptr i8, ptr %getelementptr12, i64 12 + store i32 0, ptr %call1, align 4 + %getelementptr47 = getelementptr i8, ptr %getelementptr12, i64 16 + store i32 0, ptr %call1, align 4 + br label %bb9 + +bb48: ; preds = %bb48, %bb + %getelementptr49 = getelementptr i8, ptr %call1, i64 0 + %getelementptr50 = getelementptr i8, ptr %getelementptr49, i64 1 + store i8 0, ptr %call1, align 1 + %getelementptr51 = getelementptr i8, ptr %getelementptr49, i64 8 + store i16 0, ptr %call1, align 4 + %getelementptr52 = getelementptr i8, ptr %getelementptr49, i64 12 + store i32 0, ptr %call1, align 4 + br label %bb48 + +bb53: ; preds = %bb71, %bb8 + %phi54 = phi ptr [ %call1, %bb8 ], [ %a, %bb71 ] + %getelementptr55 = getelementptr i8, ptr %phi54, i64 0 + %getelementptr56 = getelementptr i8, ptr %phi54, i64 0 + %getelementptr57 = getelementptr i8, ptr %phi54, i64 0 + %getelementptr58 = getelementptr i8, ptr %getelementptr57, i64 1 + br label %bb71 + +bb59: ; No predecessors! + %getelementptr60 = getelementptr i8, ptr %phi54, i64 0 + %getelementptr61 = getelementptr i8, ptr %getelementptr60, i64 12 + br label %bb67 + +bb62: ; No predecessors! + %getelementptr63 = getelementptr i8, ptr %getelementptr56, i64 8 + %load64 = load i16, ptr %call1, align 4 + %getelementptr65 = getelementptr i8, ptr %getelementptr57, i64 8 + store i16 0, ptr %call1, align 4 + %getelementptr66 = getelementptr i8, ptr %phi54, i64 0 + br label %bb67 + +bb67: ; preds = %bb62, %bb59 + %phi68 = phi ptr [ %a, %bb62 ], [ %call1, %bb59 ] + %phi69 = phi ptr [ %call1, %bb62 ], [ %call1, %bb59 ] + %phi70 = phi ptr [ %call1, %bb62 ], [ %call1, %bb59 ] + store i32 0, ptr %call1, align 4 + br label %bb71 + +bb71: ; preds = %bb67, %bb53 + %phi72 = phi ptr [ %call1, %bb53 ], [ %call1, %bb67 ] + %phi73 = phi ptr [ %call1, %bb53 ], [ %call1, %bb67 ] + %load74 = load i8, ptr %call1, align 4 + %load75 = load i8, ptr %call1, align 4 + %getelementptr76 = getelementptr i8, ptr %phi72, i64 12 + %getelementptr77 = getelementptr i8, ptr %getelementptr57, i64 12 + %getelementptr78 = getelementptr [0 x i32], ptr %getelementptr76, i64 0, i64 0 + %load79 = load i32, ptr %call1, align 4 + %getelementptr80 = getelementptr [0 x i32], ptr %getelementptr77, i64 0, i64 0 + store i32 0, ptr %call1, align 4 + %load81 = load i8, ptr %call1, align 4 + %getelementptr82 = getelementptr i8, ptr %getelementptr57, i64 8 + %load83 = load i16, ptr %call1, align 4 + store i16 0, ptr %call1, align 4 + store i8 0, ptr %call1, align 4 + %getelementptr84 = getelementptr i8, ptr %getelementptr57, i64 12 + store i32 0, ptr %call1, align 4 + %getelementptr85 = getelementptr i8, ptr %getelementptr57, i64 16 + store i32 0, ptr %call1, align 4 + br label %bb53 + +bb86: ; preds = %bb86, %bb29 + %getelementptr87 = getelementptr [0 x i32], ptr %getelementptr34, i64 0, i64 0 + %load88 = load i32, ptr %call1, align 4 + %getelementptr89 = getelementptr [0 x i32], ptr %getelementptr35, i64 0, i64 0 + store i32 0, ptr %call1, align 4 + br label %bb86 +} + +declare ptr @malloc(i64) + +; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: write) +declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) #0 + +attributes #0 = { nocallback nofree nounwind willreturn memory(argmem: write) } -- GitLab From 2b184c8f64b577a4cef2c2abde2dceaeece4874d Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Wed, 6 Mar 2024 21:10:39 +0100 Subject: [PATCH 350/929] [bazel] Add missing dependencies for 2542d34522784e27850a17a8c39a1ad4b43622ab --- utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 04b1a210b2e2..d2bacbb6ce47 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -7408,11 +7408,13 @@ cc_library( ), includes = ["include"], deps = [ + ":Analysis", ":ControlFlowInterfaces", ":FunctionInterfaces", ":IR", ":LoopLikeInterface", ":MemorySlotInterfaces", + ":Pass", ":Rewrite", ":SideEffectInterfaces", ":SubsetOpInterface", -- GitLab From 5ab86ef7c1bf66e7ae3b6fed97084715484c34e3 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 6 Mar 2024 20:10:22 +0000 Subject: [PATCH 351/929] [VPlan] Remove unused OverrideAllowed arg from getVPValue (NFCI). --- llvm/lib/Transforms/Vectorize/VPlan.h | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index bc499400bced..af6d0081bffe 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2992,13 +2992,11 @@ public: Value2VPValue[V] = VPV; } - /// Returns the VPValue for \p V. \p OverrideAllowed can be used to disable - /// /// checking whether it is safe to query VPValues using IR Values. - VPValue *getVPValue(Value *V, bool OverrideAllowed = false) { + /// Returns the VPValue for \p V. + VPValue *getVPValue(Value *V) { assert(V && "Trying to get the VPValue of a null Value"); assert(Value2VPValue.count(V) && "Value does not exist in VPlan"); - assert((Value2VPValueEnabled || OverrideAllowed || - Value2VPValue[V]->isLiveIn()) && + assert((Value2VPValueEnabled || Value2VPValue[V]->isLiveIn()) && "Value2VPValue mapping may be out of date!"); return Value2VPValue[V]; } -- GitLab From d4f4f802363d4227c61553872bdd61abcdeb8c08 Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Wed, 6 Mar 2024 21:37:35 +0100 Subject: [PATCH 352/929] Revert "[libomptarget] Fix CUDA plugin build regression" This reverts commit b64482e23eefaef7738fde35d0b7c4174aaa6597. --- openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp b/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp index e9742c9a13de..f85a00cd1cd5 100644 --- a/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp +++ b/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp @@ -1160,7 +1160,7 @@ private: // automatically so we must create it ourselves. The backend will emit // several globals that contain function pointers we can call. These are // prefixed with a known name due to Nvidia's lack of section support. - auto ELFObjOrErr = ELF64LEObjectFile::create(Image.getMemoryBuffer()); + auto ELFObjOrErr = Handler.getELFObjectFile(Image); if (!ELFObjOrErr) return ELFObjOrErr.takeError(); -- GitLab From 70677c81debbeacfc2a2fe968e632522ea1b53e1 Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Wed, 6 Mar 2024 21:37:43 +0100 Subject: [PATCH 353/929] Revert "[libomptarget] Build plugins-nextgen for SystemZ (#83978)" This reverts commit 3ecd38c8e1d34b1e4639a1de9f0cb56c7957cbd2. --- clang/lib/Frontend/CompilerInvocation.cpp | 1 - clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp | 1 - llvm/lib/Frontend/OpenMP/OMPContext.cpp | 1 - openmp/libomptarget/CMakeLists.txt | 2 -- openmp/libomptarget/plugins-nextgen/CMakeLists.txt | 9 +++------ .../libomptarget/plugins-nextgen/aarch64/CMakeLists.txt | 2 +- openmp/libomptarget/plugins-nextgen/ppc64/CMakeLists.txt | 2 +- .../libomptarget/plugins-nextgen/ppc64le/CMakeLists.txt | 2 +- openmp/libomptarget/plugins-nextgen/s390x/CMakeLists.txt | 2 +- .../libomptarget/plugins-nextgen/x86_64/CMakeLists.txt | 2 +- openmp/libomptarget/src/CMakeLists.txt | 1 - openmp/libomptarget/test/api/omp_dynamic_shared_memory.c | 2 -- openmp/libomptarget/test/jit/empty_kernel_lvl1.c | 2 -- openmp/libomptarget/test/jit/empty_kernel_lvl2.c | 2 -- openmp/libomptarget/test/jit/type_punning.c | 2 -- openmp/libomptarget/test/mapping/auto_zero_copy.cpp | 2 -- .../libomptarget/test/mapping/auto_zero_copy_globals.cpp | 2 -- openmp/libomptarget/test/offloading/barrier_fence.c | 2 -- openmp/libomptarget/test/offloading/bug49334.cpp | 2 -- .../libomptarget/test/offloading/default_thread_limit.c | 2 -- openmp/libomptarget/test/offloading/ompx_bare.c | 5 +---- openmp/libomptarget/test/offloading/ompx_coords.c | 2 -- openmp/libomptarget/test/offloading/ompx_saxpy_mixed.c | 2 -- .../test/offloading/parallel_target_teams_reduction.cpp | 2 -- openmp/libomptarget/test/offloading/small_trip_count.c | 2 -- .../test/offloading/small_trip_count_thread_limit.cpp | 2 -- openmp/libomptarget/test/offloading/spmdization.c | 2 -- .../test/offloading/target_critical_region.cpp | 2 -- openmp/libomptarget/test/offloading/thread_limit.c | 2 -- openmp/libomptarget/test/ompt/target_memcpy.c | 2 -- openmp/libomptarget/test/ompt/target_memcpy_emi.c | 2 -- openmp/libomptarget/test/ompt/veccopy.c | 2 -- openmp/libomptarget/test/ompt/veccopy_data.c | 2 -- openmp/libomptarget/test/ompt/veccopy_disallow_both.c | 2 -- openmp/libomptarget/test/ompt/veccopy_emi.c | 2 -- openmp/libomptarget/test/ompt/veccopy_emi_map.c | 2 -- openmp/libomptarget/test/ompt/veccopy_map.c | 2 -- 37 files changed, 9 insertions(+), 71 deletions(-) diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index 7e17c3e49aca..691f3b989b81 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -4012,7 +4012,6 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args, if (TT.getArch() == llvm::Triple::UnknownArch || !(TT.getArch() == llvm::Triple::aarch64 || TT.isPPC() || - TT.getArch() == llvm::Triple::systemz || TT.getArch() == llvm::Triple::nvptx || TT.getArch() == llvm::Triple::nvptx64 || TT.getArch() == llvm::Triple::amdgcn || diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp index b3efa56b3302..576e8f2cd7f8 100644 --- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp +++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp @@ -552,7 +552,6 @@ Expected linkDevice(ArrayRef InputFiles, case Triple::aarch64_be: case Triple::ppc64: case Triple::ppc64le: - case Triple::systemz: return generic::clang(InputFiles, Args); default: return createStringError(inconvertibleErrorCode(), diff --git a/llvm/lib/Frontend/OpenMP/OMPContext.cpp b/llvm/lib/Frontend/OpenMP/OMPContext.cpp index 37936d6000c8..e870c5aa2ba6 100644 --- a/llvm/lib/Frontend/OpenMP/OMPContext.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPContext.cpp @@ -44,7 +44,6 @@ OMPContext::OMPContext(bool IsDeviceCompilation, Triple TargetTriple) { case Triple::ppcle: case Triple::ppc64: case Triple::ppc64le: - case Triple::systemz: case Triple::x86: case Triple::x86_64: ActiveTraits.set(unsigned(TraitProperty::device_kind_cpu)); diff --git a/openmp/libomptarget/CMakeLists.txt b/openmp/libomptarget/CMakeLists.txt index b382137b70ee..a74eff0c0beb 100644 --- a/openmp/libomptarget/CMakeLists.txt +++ b/openmp/libomptarget/CMakeLists.txt @@ -56,8 +56,6 @@ set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} x86_64-pc-linux-gnu-L set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} nvptx64-nvidia-cuda") set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} nvptx64-nvidia-cuda-LTO") set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} nvptx64-nvidia-cuda-JIT-LTO") -set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} s390x-ibm-linux-gnu") -set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} s390x-ibm-linux-gnu-LTO") # Once the plugins for the different targets are validated, they will be added to # the list of supported targets in the current system. diff --git a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt index b6fc136e8a17..3ca02368253e 100644 --- a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt @@ -12,15 +12,12 @@ add_subdirectory(common) -# void build_generic_elf64(string tmachine, string tmachine_name, string tmachine_libname, -# string tmachine_llvm, string tmachine_triple, string elf_machine_id); +# void build_generic_elf64(string tmachine, string tmachine_name, string tmachine_libname, string elf_machine_id); # - build a plugin for an ELF based generic 64-bit target based on libffi. # - tmachine: name of the machine processor as used in the cmake build system. # - tmachine_name: name of the machine to be printed with the debug messages. # - tmachine_libname: machine name to be appended to the plugin library name. -# - tmachine_llvm: LLVM triple for the processor -# - tmachine_triple: GNU target triple -macro(build_generic_elf64 tmachine tmachine_name tmachine_libname tmachine_llvm tmachine_triple elf_machine_id) +macro(build_generic_elf64 tmachine tmachine_name tmachine_libname tmachine_triple elf_machine_id) if(CMAKE_SYSTEM_PROCESSOR MATCHES "${tmachine}$") # Define macro to be used as prefix of the runtime messages for this target. add_definitions("-DTARGET_NAME=${tmachine_name}") @@ -33,7 +30,7 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "${tmachine}$") add_definitions("-DTARGET_ELF_ID=${elf_machine_id}") # Define target triple - add_definitions("-DLIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE=${tmachine_llvm}") + add_definitions("-DLIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE=${tmachine}") add_llvm_library("omptarget.rtl.${tmachine_libname}" SHARED diff --git a/openmp/libomptarget/plugins-nextgen/aarch64/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/aarch64/CMakeLists.txt index 663ab4d60ff9..2c2b753590e2 100644 --- a/openmp/libomptarget/plugins-nextgen/aarch64/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/aarch64/CMakeLists.txt @@ -11,7 +11,7 @@ ##===----------------------------------------------------------------------===## if(CMAKE_SYSTEM_NAME MATCHES "Linux") - build_generic_elf64("aarch64" "aarch64" "aarch64" "aarch64" "aarch64-unknown-linux-gnu" "183") + build_generic_elf64("aarch64" "aarch64" "aarch64" "aarch64-unknown-linux-gnu" "183") else() libomptarget_say("Not building aarch64 NextGen offloading plugin: machine not found in the system.") endif() diff --git a/openmp/libomptarget/plugins-nextgen/ppc64/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/ppc64/CMakeLists.txt index 77466c111ee0..0cccc9cb82e4 100644 --- a/openmp/libomptarget/plugins-nextgen/ppc64/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/ppc64/CMakeLists.txt @@ -11,7 +11,7 @@ ##===----------------------------------------------------------------------===## if(CMAKE_SYSTEM_NAME MATCHES "Linux") - build_generic_elf64("ppc64" "PPC64" "ppc64" "ppc64" "powerpc64-ibm-linux-gnu" "21") + build_generic_elf64("ppc64" "PPC64" "ppc64" "powerpc64-ibm-linux-gnu" "21") else() libomptarget_say("Not building ppc64 NextGen offloading plugin: machine not found in the system.") endif() diff --git a/openmp/libomptarget/plugins-nextgen/ppc64le/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/ppc64le/CMakeLists.txt index 91d21627a327..9461d79d145d 100644 --- a/openmp/libomptarget/plugins-nextgen/ppc64le/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/ppc64le/CMakeLists.txt @@ -11,7 +11,7 @@ ##===----------------------------------------------------------------------===## if(CMAKE_SYSTEM_NAME MATCHES "Linux") - build_generic_elf64("ppc64le" "PPC64le" "ppc64" "ppc64le" "powerpc64le-ibm-linux-gnu" "21") + build_generic_elf64("ppc64le" "PPC64le" "ppc64" "powerpc64le-ibm-linux-gnu" "21") else() libomptarget_say("Not building ppc64le NextGen offloading plugin: machine not found in the system.") endif() diff --git a/openmp/libomptarget/plugins-nextgen/s390x/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/s390x/CMakeLists.txt index 0388a235d289..1b12a2928999 100644 --- a/openmp/libomptarget/plugins-nextgen/s390x/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/s390x/CMakeLists.txt @@ -11,7 +11,7 @@ ##===----------------------------------------------------------------------===## if(CMAKE_SYSTEM_NAME MATCHES "Linux") - build_generic_elf64("s390x" "S390X" "s390x" "systemz" "s390x-ibm-linux-gnu" "22") + build_generic_elf64("SystemZ" "S390X" "s390x" "s390x-ibm-linux-gnu" "22") else() libomptarget_say("Not building s390x NextGen offloading plugin: machine not found in the system.") endif() diff --git a/openmp/libomptarget/plugins-nextgen/x86_64/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/x86_64/CMakeLists.txt index 27cf3e069a37..129d526a2ae7 100644 --- a/openmp/libomptarget/plugins-nextgen/x86_64/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/x86_64/CMakeLists.txt @@ -11,7 +11,7 @@ ##===----------------------------------------------------------------------===## if(CMAKE_SYSTEM_NAME MATCHES "Linux") - build_generic_elf64("x86_64" "x86_64" "x86_64" "x86_64" "x86_64-pc-linux-gnu" "62") + build_generic_elf64("x86_64" "x86_64" "x86_64" "x86_64-pc-linux-gnu" "62") else() libomptarget_say("Not building x86_64 NextGen offloading plugin: machine not found in the system.") endif() diff --git a/openmp/libomptarget/src/CMakeLists.txt b/openmp/libomptarget/src/CMakeLists.txt index d0971bd4ef07..9bc3f3339583 100644 --- a/openmp/libomptarget/src/CMakeLists.txt +++ b/openmp/libomptarget/src/CMakeLists.txt @@ -73,7 +73,6 @@ if (NOT LIBOMPTARGET_PLUGINS_TO_LOAD) check_plugin_target(cuda) check_plugin_target(aarch64) check_plugin_target(amdgpu) - check_plugin_target(s390x) endif() list(TRANSFORM LIBOMPTARGET_PLUGINS_TO_LOAD PREPEND "\"libomptarget.rtl.") diff --git a/openmp/libomptarget/test/api/omp_dynamic_shared_memory.c b/openmp/libomptarget/test/api/omp_dynamic_shared_memory.c index 3fe75f24db3e..5095a69a375c 100644 --- a/openmp/libomptarget/test/api/omp_dynamic_shared_memory.c +++ b/openmp/libomptarget/test/api/omp_dynamic_shared_memory.c @@ -10,8 +10,6 @@ // UNSUPPORTED: x86_64-pc-linux-gnu-LTO // UNSUPPORTED: aarch64-unknown-linux-gnu // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO #include #include diff --git a/openmp/libomptarget/test/jit/empty_kernel_lvl1.c b/openmp/libomptarget/test/jit/empty_kernel_lvl1.c index a0b8cd448837..c908c8ba0650 100644 --- a/openmp/libomptarget/test/jit/empty_kernel_lvl1.c +++ b/openmp/libomptarget/test/jit/empty_kernel_lvl1.c @@ -32,7 +32,5 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO #include "empty_kernel.inc" diff --git a/openmp/libomptarget/test/jit/empty_kernel_lvl2.c b/openmp/libomptarget/test/jit/empty_kernel_lvl2.c index 81a04f55ce43..0b88d33292c5 100644 --- a/openmp/libomptarget/test/jit/empty_kernel_lvl2.c +++ b/openmp/libomptarget/test/jit/empty_kernel_lvl2.c @@ -92,7 +92,5 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO #include "empty_kernel.inc" diff --git a/openmp/libomptarget/test/jit/type_punning.c b/openmp/libomptarget/test/jit/type_punning.c index 10e3d2cef718..23aa69bba7b9 100644 --- a/openmp/libomptarget/test/jit/type_punning.c +++ b/openmp/libomptarget/test/jit/type_punning.c @@ -12,8 +12,6 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO // Ensure that there is only the kernel function left, not any outlined // parallel regions. diff --git a/openmp/libomptarget/test/mapping/auto_zero_copy.cpp b/openmp/libomptarget/test/mapping/auto_zero_copy.cpp index 46641200bb56..6f9d8c2b128c 100644 --- a/openmp/libomptarget/test/mapping/auto_zero_copy.cpp +++ b/openmp/libomptarget/test/mapping/auto_zero_copy.cpp @@ -13,8 +13,6 @@ // UNSUPPORTED: nvptx64-nvidia-cuda-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO // REQUIRES: unified_shared_memory diff --git a/openmp/libomptarget/test/mapping/auto_zero_copy_globals.cpp b/openmp/libomptarget/test/mapping/auto_zero_copy_globals.cpp index 55dfb2807ebc..4a13d270aeeb 100644 --- a/openmp/libomptarget/test/mapping/auto_zero_copy_globals.cpp +++ b/openmp/libomptarget/test/mapping/auto_zero_copy_globals.cpp @@ -9,8 +9,6 @@ // UNSUPPORTED: nvptx64-nvidia-cuda-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO // REQUIRES: unified_shared_memory diff --git a/openmp/libomptarget/test/offloading/barrier_fence.c b/openmp/libomptarget/test/offloading/barrier_fence.c index b9a8ca27965a..5d1096478ed9 100644 --- a/openmp/libomptarget/test/offloading/barrier_fence.c +++ b/openmp/libomptarget/test/offloading/barrier_fence.c @@ -7,8 +7,6 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO #include #include diff --git a/openmp/libomptarget/test/offloading/bug49334.cpp b/openmp/libomptarget/test/offloading/bug49334.cpp index 1f19dab37881..a22d3fe9f665 100644 --- a/openmp/libomptarget/test/offloading/bug49334.cpp +++ b/openmp/libomptarget/test/offloading/bug49334.cpp @@ -9,8 +9,6 @@ // UNSUPPORTED: x86_64-pc-linux-gnu-LTO // UNSUPPORTED: aarch64-unknown-linux-gnu // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO // UNSUPPORTED: amdgcn-amd-amdhsa // UNSUPPORTED: nvptx64-nvidia-cuda // UNSUPPORTED: nvptx64-nvidia-cuda-LTO diff --git a/openmp/libomptarget/test/offloading/default_thread_limit.c b/openmp/libomptarget/test/offloading/default_thread_limit.c index 4da02bbb152e..d32e7df418cb 100644 --- a/openmp/libomptarget/test/offloading/default_thread_limit.c +++ b/openmp/libomptarget/test/offloading/default_thread_limit.c @@ -9,8 +9,6 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO __attribute__((optnone)) int optnone() { return 1; } diff --git a/openmp/libomptarget/test/offloading/ompx_bare.c b/openmp/libomptarget/test/offloading/ompx_bare.c index 3dabdcd15e0d..fb3810bd1df1 100644 --- a/openmp/libomptarget/test/offloading/ompx_bare.c +++ b/openmp/libomptarget/test/offloading/ompx_bare.c @@ -1,13 +1,10 @@ // RUN: %libomptarget-compile-generic -// RUN: env LIBOMPTARGET_INFO=63 %libomptarget-run-generic 2>&1 | \ -// RUN: %fcheck-generic +// RUN: env LIBOMPTARGET_INFO=63 %libomptarget-run-generic 2>&1 | %fcheck-generic // // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO // UNSUPPORTED: aarch64-unknown-linux-gnu // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO #include #include diff --git a/openmp/libomptarget/test/offloading/ompx_coords.c b/openmp/libomptarget/test/offloading/ompx_coords.c index 5e4e14b4c6da..61dad61f4640 100644 --- a/openmp/libomptarget/test/offloading/ompx_coords.c +++ b/openmp/libomptarget/test/offloading/ompx_coords.c @@ -4,8 +4,6 @@ // UNSUPPORTED: x86_64-pc-linux-gnu-LTO // UNSUPPORTED: aarch64-unknown-linux-gnu // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO #include #include diff --git a/openmp/libomptarget/test/offloading/ompx_saxpy_mixed.c b/openmp/libomptarget/test/offloading/ompx_saxpy_mixed.c index f479be8a484f..440b694e3ac7 100644 --- a/openmp/libomptarget/test/offloading/ompx_saxpy_mixed.c +++ b/openmp/libomptarget/test/offloading/ompx_saxpy_mixed.c @@ -4,8 +4,6 @@ // UNSUPPORTED: x86_64-pc-linux-gnu-LTO // UNSUPPORTED: aarch64-unknown-linux-gnu // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO #include #include diff --git a/openmp/libomptarget/test/offloading/parallel_target_teams_reduction.cpp b/openmp/libomptarget/test/offloading/parallel_target_teams_reduction.cpp index 10e1b33f1ce4..5303a9463f15 100644 --- a/openmp/libomptarget/test/offloading/parallel_target_teams_reduction.cpp +++ b/openmp/libomptarget/test/offloading/parallel_target_teams_reduction.cpp @@ -6,8 +6,6 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO #include #include diff --git a/openmp/libomptarget/test/offloading/small_trip_count.c b/openmp/libomptarget/test/offloading/small_trip_count.c index 65f094f15746..d8bef667607a 100644 --- a/openmp/libomptarget/test/offloading/small_trip_count.c +++ b/openmp/libomptarget/test/offloading/small_trip_count.c @@ -9,8 +9,6 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO #define N 128 diff --git a/openmp/libomptarget/test/offloading/small_trip_count_thread_limit.cpp b/openmp/libomptarget/test/offloading/small_trip_count_thread_limit.cpp index b7ae52a62c83..9796c2dc1166 100644 --- a/openmp/libomptarget/test/offloading/small_trip_count_thread_limit.cpp +++ b/openmp/libomptarget/test/offloading/small_trip_count_thread_limit.cpp @@ -7,8 +7,6 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO int main(int argc, char *argv[]) { constexpr const int block_size = 256; diff --git a/openmp/libomptarget/test/offloading/spmdization.c b/openmp/libomptarget/test/offloading/spmdization.c index 77913bec8342..2cf30ff593b9 100644 --- a/openmp/libomptarget/test/offloading/spmdization.c +++ b/openmp/libomptarget/test/offloading/spmdization.c @@ -11,8 +11,6 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO #include #include diff --git a/openmp/libomptarget/test/offloading/target_critical_region.cpp b/openmp/libomptarget/test/offloading/target_critical_region.cpp index 495632bf76e1..9a741bef6c59 100644 --- a/openmp/libomptarget/test/offloading/target_critical_region.cpp +++ b/openmp/libomptarget/test/offloading/target_critical_region.cpp @@ -6,8 +6,6 @@ // UNSUPPORTED: nvptx64-nvidia-cuda-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO // UNSUPPORTED: amdgcn-amd-amdhsa #include diff --git a/openmp/libomptarget/test/offloading/thread_limit.c b/openmp/libomptarget/test/offloading/thread_limit.c index a8cc51b651dc..65275a8f2795 100644 --- a/openmp/libomptarget/test/offloading/thread_limit.c +++ b/openmp/libomptarget/test/offloading/thread_limit.c @@ -9,8 +9,6 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO int main() { int n = 1 << 20; diff --git a/openmp/libomptarget/test/ompt/target_memcpy.c b/openmp/libomptarget/test/ompt/target_memcpy.c index 3224c5ac4c91..80a8d6a4b32e 100644 --- a/openmp/libomptarget/test/ompt/target_memcpy.c +++ b/openmp/libomptarget/test/ompt/target_memcpy.c @@ -4,8 +4,6 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO /* * Verify that for the target OpenMP APIs, the return address is non-null and diff --git a/openmp/libomptarget/test/ompt/target_memcpy_emi.c b/openmp/libomptarget/test/ompt/target_memcpy_emi.c index fd7da13cb05d..5347f38b87b6 100644 --- a/openmp/libomptarget/test/ompt/target_memcpy_emi.c +++ b/openmp/libomptarget/test/ompt/target_memcpy_emi.c @@ -4,8 +4,6 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO /* * Verify all three data transfer directions: H2D, D2D and D2H diff --git a/openmp/libomptarget/test/ompt/veccopy.c b/openmp/libomptarget/test/ompt/veccopy.c index bc70f5a0b5b6..80e71fd8a48c 100644 --- a/openmp/libomptarget/test/ompt/veccopy.c +++ b/openmp/libomptarget/test/ompt/veccopy.c @@ -4,8 +4,6 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO /* * Example OpenMP program that registers non-EMI callbacks diff --git a/openmp/libomptarget/test/ompt/veccopy_data.c b/openmp/libomptarget/test/ompt/veccopy_data.c index 264b484b881f..cef1de316a7a 100644 --- a/openmp/libomptarget/test/ompt/veccopy_data.c +++ b/openmp/libomptarget/test/ompt/veccopy_data.c @@ -4,8 +4,6 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO /* * Example OpenMP program that registers EMI callbacks. diff --git a/openmp/libomptarget/test/ompt/veccopy_disallow_both.c b/openmp/libomptarget/test/ompt/veccopy_disallow_both.c index a011c21955bb..06293e413ba4 100644 --- a/openmp/libomptarget/test/ompt/veccopy_disallow_both.c +++ b/openmp/libomptarget/test/ompt/veccopy_disallow_both.c @@ -4,8 +4,6 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO /* * Example OpenMP program that shows that both EMI and non-EMI diff --git a/openmp/libomptarget/test/ompt/veccopy_emi.c b/openmp/libomptarget/test/ompt/veccopy_emi.c index 8718a39b4af8..b597d7be6aff 100644 --- a/openmp/libomptarget/test/ompt/veccopy_emi.c +++ b/openmp/libomptarget/test/ompt/veccopy_emi.c @@ -4,8 +4,6 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO /* * Example OpenMP program that registers EMI callbacks diff --git a/openmp/libomptarget/test/ompt/veccopy_emi_map.c b/openmp/libomptarget/test/ompt/veccopy_emi_map.c index 2accba34034e..ce6f6e30d581 100644 --- a/openmp/libomptarget/test/ompt/veccopy_emi_map.c +++ b/openmp/libomptarget/test/ompt/veccopy_emi_map.c @@ -4,8 +4,6 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO /* * Example OpenMP program that shows that map-EMI callbacks are not supported. diff --git a/openmp/libomptarget/test/ompt/veccopy_map.c b/openmp/libomptarget/test/ompt/veccopy_map.c index 56a0dd48f536..83f63a6e6049 100644 --- a/openmp/libomptarget/test/ompt/veccopy_map.c +++ b/openmp/libomptarget/test/ompt/veccopy_map.c @@ -4,8 +4,6 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO -// UNSUPPORTED: s390x-ibm-linux-gnu -// UNSUPPORTED: s390x-ibm-linux-gnu-LTO /* * Example OpenMP program that shows that map callbacks are not supported. -- GitLab From fb7cc73975f3ac262f2c80af2f399caead005259 Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Wed, 6 Mar 2024 21:37:45 +0100 Subject: [PATCH 354/929] Revert "[libomptarget] Support BE ELF files in plugins-nextgen (#83976)" This reverts commit 15b7b3182cc28f4f0b950bd73d931caa27b833ec. --- .../common/include/GlobalHandler.h | 3 + .../common/include/Utils/ELF.h | 16 ++-- .../common/src/GlobalHandler.cpp | 46 ++++++++-- .../plugins-nextgen/common/src/Utils/ELF.cpp | 91 +++++-------------- 4 files changed, 71 insertions(+), 85 deletions(-) diff --git a/openmp/libomptarget/plugins-nextgen/common/include/GlobalHandler.h b/openmp/libomptarget/plugins-nextgen/common/include/GlobalHandler.h index 57d93b88afca..5c767995126b 100644 --- a/openmp/libomptarget/plugins-nextgen/common/include/GlobalHandler.h +++ b/openmp/libomptarget/plugins-nextgen/common/include/GlobalHandler.h @@ -103,6 +103,9 @@ class GenericGlobalHandlerTy { public: virtual ~GenericGlobalHandlerTy() {} + /// Helper function for getting an ELF from a device image. + Expected getELFObjectFile(DeviceImageTy &Image); + /// Returns whether the symbol named \p SymName is present in the given \p /// Image. bool isSymbolInImage(GenericDeviceTy &Device, DeviceImageTy &Image, diff --git a/openmp/libomptarget/plugins-nextgen/common/include/Utils/ELF.h b/openmp/libomptarget/plugins-nextgen/common/include/Utils/ELF.h index 2268e4b6c4af..140a6b6b84aa 100644 --- a/openmp/libomptarget/plugins-nextgen/common/include/Utils/ELF.h +++ b/openmp/libomptarget/plugins-nextgen/common/include/Utils/ELF.h @@ -27,14 +27,18 @@ bool isELF(llvm::StringRef Buffer); /// Checks if the given \p Object is a valid ELF matching the e_machine value. llvm::Expected checkMachine(llvm::StringRef Object, uint16_t EMachine); -/// Returns the symbol associated with the \p Name in the \p Obj. It will +/// Returns a pointer to the given \p Symbol inside of an ELF object. +llvm::Expected getSymbolAddress( + const llvm::object::ELFObjectFile &ELFObj, + const llvm::object::ELF64LE::Sym &Symbol); + +/// Returns the symbol associated with the \p Name in the \p ELFObj. It will /// first search for the hash sections to identify symbols from the hash table. /// If that fails it will fall back to a linear search in the case of an -/// executable file without a hash table. If the symbol is found, it returns -/// a StringRef covering the symbol's data in the Obj buffer, based on its -/// address and size; otherwise, it returns std::nullopt. -llvm::Expected> -findSymbolInImage(const llvm::StringRef Obj, llvm::StringRef Name); +/// executable file without a hash table. +llvm::Expected +getSymbol(const llvm::object::ELFObjectFile &ELFObj, + llvm::StringRef Name); } // namespace elf } // namespace utils diff --git a/openmp/libomptarget/plugins-nextgen/common/src/GlobalHandler.cpp b/openmp/libomptarget/plugins-nextgen/common/src/GlobalHandler.cpp index 2269f27e4602..d398f60c55bd 100644 --- a/openmp/libomptarget/plugins-nextgen/common/src/GlobalHandler.cpp +++ b/openmp/libomptarget/plugins-nextgen/common/src/GlobalHandler.cpp @@ -25,6 +25,16 @@ using namespace omp; using namespace target; using namespace plugin; +Expected +GenericGlobalHandlerTy::getELFObjectFile(DeviceImageTy &Image) { + assert(utils::elf::isELF(Image.getMemoryBuffer().getBuffer()) && + "Input is not an ELF file"); + + Expected ElfOrErr = + ELF64LEObjectFile::create(Image.getMemoryBuffer()); + return ElfOrErr; +} + Error GenericGlobalHandlerTy::moveGlobalBetweenDeviceAndHost( GenericDeviceTy &Device, DeviceImageTy &Image, const GlobalTy &HostGlobal, bool Device2Host) { @@ -71,37 +81,55 @@ Error GenericGlobalHandlerTy::moveGlobalBetweenDeviceAndHost( bool GenericGlobalHandlerTy::isSymbolInImage(GenericDeviceTy &Device, DeviceImageTy &Image, StringRef SymName) { + // Get the ELF object file for the image. Notice the ELF object may already + // be created in previous calls, so we can reuse it. If this is unsuccessful + // just return false as we couldn't find it. + auto ELFObjOrErr = getELFObjectFile(Image); + if (!ELFObjOrErr) { + consumeError(ELFObjOrErr.takeError()); + return false; + } // Search the ELF symbol using the symbol name. - auto SymOrErr = utils::elf::findSymbolInImage( - Image.getMemoryBuffer().getBuffer(), SymName); + auto SymOrErr = utils::elf::getSymbol(*ELFObjOrErr, SymName); if (!SymOrErr) { consumeError(SymOrErr.takeError()); return false; } - return SymOrErr->has_value(); + return *SymOrErr; } Error GenericGlobalHandlerTy::getGlobalMetadataFromImage( GenericDeviceTy &Device, DeviceImageTy &Image, GlobalTy &ImageGlobal) { + // Get the ELF object file for the image. Notice the ELF object may already + // be created in previous calls, so we can reuse it. + auto ELFObj = getELFObjectFile(Image); + if (!ELFObj) + return ELFObj.takeError(); + // Search the ELF symbol using the symbol name. - auto SymOrErr = utils::elf::findSymbolInImage( - Image.getMemoryBuffer().getBuffer(), ImageGlobal.getName()); + auto SymOrErr = utils::elf::getSymbol(*ELFObj, ImageGlobal.getName()); if (!SymOrErr) return Plugin::error("Failed ELF lookup of global '%s': %s", ImageGlobal.getName().data(), toString(SymOrErr.takeError()).data()); - if (!SymOrErr->has_value()) + if (!*SymOrErr) return Plugin::error("Failed to find global symbol '%s' in the ELF image", ImageGlobal.getName().data()); + auto AddrOrErr = utils::elf::getSymbolAddress(*ELFObj, **SymOrErr); + // Get the section to which the symbol belongs. + if (!AddrOrErr) + return Plugin::error("Failed to get ELF symbol from global '%s': %s", + ImageGlobal.getName().data(), + toString(AddrOrErr.takeError()).data()); + // Setup the global symbol's address and size. - auto Symbol = **SymOrErr; - ImageGlobal.setPtr(static_cast(const_cast(Symbol.data()))); - ImageGlobal.setSize(Symbol.size()); + ImageGlobal.setPtr(const_cast(*AddrOrErr)); + ImageGlobal.setSize((*SymOrErr)->st_size); return Plugin::success(); } diff --git a/openmp/libomptarget/plugins-nextgen/common/src/Utils/ELF.cpp b/openmp/libomptarget/plugins-nextgen/common/src/Utils/ELF.cpp index 88bb097165ef..c84c3bad5def 100644 --- a/openmp/libomptarget/plugins-nextgen/common/src/Utils/ELF.cpp +++ b/openmp/libomptarget/plugins-nextgen/common/src/Utils/ELF.cpp @@ -36,10 +36,18 @@ bool utils::elf::isELF(StringRef Buffer) { } } -template -static Expected -checkMachineImpl(const object::ELFObjectFile &ELFObj, uint16_t EMachine) { - const auto Header = ELFObj.getELFFile().getHeader(); +Expected utils::elf::checkMachine(StringRef Object, uint16_t EMachine) { + assert(isELF(Object) && "Input is not an ELF!"); + + Expected ElfOrErr = + ELF64LEObjectFile::create(MemoryBufferRef(Object, /*Identifier=*/""), + /*InitContent=*/false); + if (!ElfOrErr) + return ElfOrErr.takeError(); + + const auto Header = ElfOrErr->getELFFile().getHeader(); + if (Header.e_ident[EI_CLASS] != ELFCLASS64) + return createError("Only 64-bit ELF files are supported"); if (Header.e_type != ET_EXEC && Header.e_type != ET_DYN) return createError("Only executable ELF files are supported"); @@ -63,25 +71,6 @@ checkMachineImpl(const object::ELFObjectFile &ELFObj, uint16_t EMachine) { return Header.e_machine == EMachine; } -Expected utils::elf::checkMachine(StringRef Object, uint16_t EMachine) { - assert(isELF(Object) && "Input is not an ELF!"); - - Expected> ElfOrErr = - ObjectFile::createELFObjectFile( - MemoryBufferRef(Object, /*Identifier=*/""), - /*InitContent=*/false); - if (!ElfOrErr) - return ElfOrErr.takeError(); - - if (const ELF64LEObjectFile *ELFObj = - dyn_cast(&**ElfOrErr)) - return checkMachineImpl(*ELFObj, EMachine); - if (const ELF64BEObjectFile *ELFObj = - dyn_cast(&**ElfOrErr)) - return checkMachineImpl(*ELFObj, EMachine); - return createError("Only 64-bit ELF files are supported"); -} - template static Expected getSymbolFromGnuHashTable(StringRef Name, const typename ELFT::GnuHash &HashTab, @@ -242,9 +231,8 @@ getSymTableSymbol(const ELFFile &Elf, const typename ELFT::Shdr &Sec, return nullptr; } -template -static Expected -getSymbol(const ELFObjectFile &ELFObj, StringRef Name) { +Expected +utils::elf::getSymbol(const ELFObjectFile &ELFObj, StringRef Name) { // First try to look up the symbol via the hash table. for (ELFSectionRef Sec : ELFObj.sections()) { if (Sec.getType() != SHT_HASH && Sec.getType() != SHT_GNU_HASH) @@ -253,7 +241,8 @@ getSymbol(const ELFObjectFile &ELFObj, StringRef Name) { auto HashTabOrErr = ELFObj.getELFFile().getSection(Sec.getIndex()); if (!HashTabOrErr) return HashTabOrErr.takeError(); - return getHashTableSymbol(ELFObj.getELFFile(), **HashTabOrErr, Name); + return getHashTableSymbol(ELFObj.getELFFile(), **HashTabOrErr, + Name); } // If this is an executable file check the entire standard symbol table. @@ -264,17 +253,16 @@ getSymbol(const ELFObjectFile &ELFObj, StringRef Name) { auto SymTabOrErr = ELFObj.getELFFile().getSection(Sec.getIndex()); if (!SymTabOrErr) return SymTabOrErr.takeError(); - return getSymTableSymbol(ELFObj.getELFFile(), **SymTabOrErr, Name); + return getSymTableSymbol(ELFObj.getELFFile(), **SymTabOrErr, Name); } return nullptr; } -template -static Expected -getSymbolAddress(const object::ELFObjectFile &ELFObj, - const typename ELFT::Sym &Symbol) { - const ELFFile &ELFFile = ELFObj.getELFFile(); +Expected utils::elf::getSymbolAddress( + const object::ELFObjectFile &ELFObj, + const object::ELF64LE::Sym &Symbol) { + const ELFFile &ELFFile = ELFObj.getELFFile(); auto SecOrErr = ELFFile.getSection(Symbol.st_shndx); if (!SecOrErr) @@ -295,40 +283,3 @@ getSymbolAddress(const object::ELFObjectFile &ELFObj, return ELFFile.base() + Offset; } - -template -static Expected> -findSymbolInImageImpl(const object::ELFObjectFile &ELFObj, - StringRef Name) { - auto SymOrErr = getSymbol(ELFObj, Name); - if (!SymOrErr) - return SymOrErr.takeError(); - if (!*SymOrErr) - return std::nullopt; - - // If the symbol was found, return a StringRef covering the associated data, - // based on the symbol's address and size. - auto AddrOrErr = getSymbolAddress(ELFObj, **SymOrErr); - if (!AddrOrErr) - return AddrOrErr.takeError(); - return StringRef(static_cast(*AddrOrErr), (*SymOrErr)->st_size); -} - -Expected> -utils::elf::findSymbolInImage(StringRef Obj, StringRef Name) { - assert(isELF(Obj) && "Input is not an ELF!"); - - Expected> ElfOrErr = - ObjectFile::createELFObjectFile(MemoryBufferRef(Obj, /*Identifier=*/""), - /*InitContent=*/false); - if (!ElfOrErr) - return ElfOrErr.takeError(); - - if (const ELF64LEObjectFile *ELFObj = - dyn_cast(&**ElfOrErr)) - return findSymbolInImageImpl(*ELFObj, Name); - if (const ELF64BEObjectFile *ELFObj = - dyn_cast(&**ElfOrErr)) - return findSymbolInImageImpl(*ELFObj, Name); - return createError("Only 64-bit ELF files are supported"); -} -- GitLab From e946b5a87b2db307da076093d0a9a72ecb4ec089 Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Wed, 6 Mar 2024 12:38:40 -0800 Subject: [PATCH 355/929] [SCEV] Autogenerate more scev analysis check tests --- .../2007-07-15-NegativeStride.ll | 11 +- .../2007-09-27-LargeStepping.ll | 11 +- .../2008-05-25-NegativeStepToZero.ll | 11 +- .../ScalarEvolution/2008-07-19-WrappingIV.ll | 11 +- .../ScalarEvolution/2008-12-08-FiniteSGE.ll | 11 +- .../2009-05-09-PointerEdgeCount.ll | 11 +- .../ScalarEvolution/SolveQuadraticEquation.ll | 38 +++++-- .../ScalarEvolution/exact_iter_count.ll | 29 +++-- .../PhaseOrdering/scev-custom-dl.ll | 104 ++++++++++++++++-- 9 files changed, 204 insertions(+), 33 deletions(-) diff --git a/llvm/test/Analysis/ScalarEvolution/2007-07-15-NegativeStride.ll b/llvm/test/Analysis/ScalarEvolution/2007-07-15-NegativeStride.ll index 6af105e3c28d..aade3499dba1 100644 --- a/llvm/test/Analysis/ScalarEvolution/2007-07-15-NegativeStride.ll +++ b/llvm/test/Analysis/ScalarEvolution/2007-07-15-NegativeStride.ll @@ -1,11 +1,18 @@ -; RUN: opt < %s -disable-output "-passes=print" -scalar-evolution-max-iterations=0 2>&1 | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -disable-output "-passes=print" -scalar-evolution-max-iterations=0 -scalar-evolution-classify-expressions=0 2>&1 | FileCheck %s ; PR1533 @array = weak global [101 x i32] zeroinitializer, align 32 ; [#uses=1] -; CHECK: Loop %bb: backedge-taken count is 100 define void @loop(i32 %x) { +; CHECK-LABEL: 'loop' +; CHECK-NEXT: Determining loop execution counts for: @loop +; CHECK-NEXT: Loop %bb: backedge-taken count is 100 +; CHECK-NEXT: Loop %bb: constant max backedge-taken count is i32 100 +; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is 100 +; CHECK-NEXT: Loop %bb: Trip multiple is 101 +; entry: br label %bb diff --git a/llvm/test/Analysis/ScalarEvolution/2007-09-27-LargeStepping.ll b/llvm/test/Analysis/ScalarEvolution/2007-09-27-LargeStepping.ll index 321b29efb5f6..f79d5daee741 100644 --- a/llvm/test/Analysis/ScalarEvolution/2007-09-27-LargeStepping.ll +++ b/llvm/test/Analysis/ScalarEvolution/2007-09-27-LargeStepping.ll @@ -1,9 +1,16 @@ -; RUN: opt < %s -disable-output "-passes=print" -scalar-evolution-max-iterations=0 2>&1 | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -disable-output "-passes=print" -scalar-evolution-max-iterations=0 -scalar-evolution-classify-expressions=0 2>&1 | FileCheck %s ; PR1706 -; CHECK: backedge-taken count is 13 define i32 @f() { +; CHECK-LABEL: 'f' +; CHECK-NEXT: Determining loop execution counts for: @f +; CHECK-NEXT: Loop %bb5: backedge-taken count is 13 +; CHECK-NEXT: Loop %bb5: constant max backedge-taken count is i32 13 +; CHECK-NEXT: Loop %bb5: symbolic max backedge-taken count is 13 +; CHECK-NEXT: Loop %bb5: Trip multiple is 14 +; entry: br label %bb5 diff --git a/llvm/test/Analysis/ScalarEvolution/2008-05-25-NegativeStepToZero.ll b/llvm/test/Analysis/ScalarEvolution/2008-05-25-NegativeStepToZero.ll index 6ed38ffb95fe..a9545f436b89 100644 --- a/llvm/test/Analysis/ScalarEvolution/2008-05-25-NegativeStepToZero.ll +++ b/llvm/test/Analysis/ScalarEvolution/2008-05-25-NegativeStepToZero.ll @@ -1,9 +1,16 @@ -; RUN: opt < %s -disable-output "-passes=print" -scalar-evolution-max-iterations=0 2>&1 | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -disable-output "-passes=print" -scalar-evolution-max-iterations=0 -scalar-evolution-classify-expressions=0 2>&1 | FileCheck %s ; PR2364 -; CHECK: backedge-taken count is 61 define i32 @func_6() nounwind { +; CHECK-LABEL: 'func_6' +; CHECK-NEXT: Determining loop execution counts for: @func_6 +; CHECK-NEXT: Loop %bb5: backedge-taken count is 61 +; CHECK-NEXT: Loop %bb5: constant max backedge-taken count is i8 61 +; CHECK-NEXT: Loop %bb5: symbolic max backedge-taken count is 61 +; CHECK-NEXT: Loop %bb5: Trip multiple is 62 +; entry: br label %bb5 diff --git a/llvm/test/Analysis/ScalarEvolution/2008-07-19-WrappingIV.ll b/llvm/test/Analysis/ScalarEvolution/2008-07-19-WrappingIV.ll index b3361844c468..45fe08a340eb 100644 --- a/llvm/test/Analysis/ScalarEvolution/2008-07-19-WrappingIV.ll +++ b/llvm/test/Analysis/ScalarEvolution/2008-07-19-WrappingIV.ll @@ -1,9 +1,16 @@ -; RUN: opt < %s -disable-output "-passes=print" -scalar-evolution-max-iterations=0 2>&1 | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -disable-output "-passes=print" -scalar-evolution-max-iterations=0 -scalar-evolution-classify-expressions=0 2>&1 | FileCheck %s ; PR2088 -; CHECK: backedge-taken count is 113 define void @fun() { +; CHECK-LABEL: 'fun' +; CHECK-NEXT: Determining loop execution counts for: @fun +; CHECK-NEXT: Loop %loop: backedge-taken count is 113 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 113 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 113 +; CHECK-NEXT: Loop %loop: Trip multiple is 114 +; entry: br label %loop loop: diff --git a/llvm/test/Analysis/ScalarEvolution/2008-12-08-FiniteSGE.ll b/llvm/test/Analysis/ScalarEvolution/2008-12-08-FiniteSGE.ll index fabd37a26935..fab2e5495653 100644 --- a/llvm/test/Analysis/ScalarEvolution/2008-12-08-FiniteSGE.ll +++ b/llvm/test/Analysis/ScalarEvolution/2008-12-08-FiniteSGE.ll @@ -1,8 +1,15 @@ -; RUN: opt < %s -disable-output "-passes=print" 2>&1 | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -disable-output "-passes=print" -scalar-evolution-classify-expressions=0 2>&1 | FileCheck %s -; CHECK: backedge-taken count is 255 define i32 @foo(i32 %x, i32 %y, ptr %lam, ptr %alp) nounwind { +; CHECK-LABEL: 'foo' +; CHECK-NEXT: Determining loop execution counts for: @foo +; CHECK-NEXT: Loop %bb1: backedge-taken count is 255 +; CHECK-NEXT: Loop %bb1: constant max backedge-taken count is i32 255 +; CHECK-NEXT: Loop %bb1: symbolic max backedge-taken count is 255 +; CHECK-NEXT: Loop %bb1: Trip multiple is 256 +; bb1.thread: br label %bb1 diff --git a/llvm/test/Analysis/ScalarEvolution/2009-05-09-PointerEdgeCount.ll b/llvm/test/Analysis/ScalarEvolution/2009-05-09-PointerEdgeCount.ll index 452ba1bd2edb..c683fbaa9aef 100644 --- a/llvm/test/Analysis/ScalarEvolution/2009-05-09-PointerEdgeCount.ll +++ b/llvm/test/Analysis/ScalarEvolution/2009-05-09-PointerEdgeCount.ll @@ -1,7 +1,7 @@ -; RUN: opt < %s -disable-output "-passes=print" 2>&1 | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -disable-output "-passes=print" -scalar-evolution-classify-expressions=0 2>&1 | FileCheck %s ; PR3171 -; CHECK: count is 2 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128" @@ -9,6 +9,13 @@ target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:1 %struct.NonPod = type { [2 x %struct.Foo] } define void @_Z3foov() nounwind { +; CHECK-LABEL: '_Z3foov' +; CHECK-NEXT: Determining loop execution counts for: @_Z3foov +; CHECK-NEXT: Loop %bb1.i: backedge-taken count is 2 +; CHECK-NEXT: Loop %bb1.i: constant max backedge-taken count is i64 2 +; CHECK-NEXT: Loop %bb1.i: symbolic max backedge-taken count is 2 +; CHECK-NEXT: Loop %bb1.i: Trip multiple is 3 +; entry: %x = alloca %struct.NonPod, align 8 ; [#uses=2] %0 = getelementptr %struct.NonPod, ptr %x, i32 0, i32 0 ; [#uses=1] diff --git a/llvm/test/Analysis/ScalarEvolution/SolveQuadraticEquation.ll b/llvm/test/Analysis/ScalarEvolution/SolveQuadraticEquation.ll index e09ebea43083..d87c2cf466a6 100644 --- a/llvm/test/Analysis/ScalarEvolution/SolveQuadraticEquation.ll +++ b/llvm/test/Analysis/ScalarEvolution/SolveQuadraticEquation.ll @@ -1,10 +1,18 @@ -; RUN: opt < %s -disable-output "-passes=print" -scalar-evolution-max-iterations=0 2>&1 | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -disable-output "-passes=print" -scalar-evolution-max-iterations=0 -scalar-evolution-classify-expressions=0 2>&1 | FileCheck %s ; PR1101 -@A = weak global [1000 x i32] zeroinitializer, align 32 +@A = weak global [1000 x i32] zeroinitializer, align 32 define void @test1(i32 %N) { +; CHECK-LABEL: 'test1' +; CHECK-NEXT: Determining loop execution counts for: @test1 +; CHECK-NEXT: Loop %bb3: backedge-taken count is 100 +; CHECK-NEXT: Loop %bb3: constant max backedge-taken count is i32 100 +; CHECK-NEXT: Loop %bb3: symbolic max backedge-taken count is 100 +; CHECK-NEXT: Loop %bb3: Trip multiple is 101 +; entry: %"alloca point" = bitcast i32 0 to i32 ; [#uses=0] br label %bb3 @@ -29,14 +37,21 @@ bb5: ; preds = %bb3 return: ; preds = %bb5 ret void } -; CHECK: Determining loop execution counts for: @test1 -; CHECK-NEXT: backedge-taken count is 100 ; PR10383 ; These next two used to crash. define void @test2(i1 %cmp, i64 %n) { +; CHECK-LABEL: 'test2' +; CHECK-NEXT: Determining loop execution counts for: @test2 +; CHECK-NEXT: Loop %for.body2: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %for.body2: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %for.body2: Unpredictable symbolic max backedge-taken count. +; CHECK-NEXT: Loop %for.body1: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %for.body1: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %for.body1: Unpredictable symbolic max backedge-taken count. +; entry: br label %for.body1 @@ -59,9 +74,14 @@ for.body2: end: ret void } -; CHECK: Determining loop execution counts for: @test2 define i32 @test3() { +; CHECK-LABEL: 'test3' +; CHECK-NEXT: Determining loop execution counts for: @test3 +; CHECK-NEXT: Loop %for.inc479: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %for.inc479: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %for.inc479: Unpredictable symbolic max backedge-taken count. +; if.then466: br i1 undef, label %for.cond539.preheader, label %for.inc479 @@ -78,12 +98,17 @@ for.inc479: for.cond539.preheader: unreachable } -; CHECK: Determining loop execution counts for: @test3 ; PR13489 ; We used to crash on this too. define void @test4() { +; CHECK-LABEL: 'test4' +; CHECK-NEXT: Determining loop execution counts for: @test4 +; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. +; entry: br label %for.body @@ -99,4 +124,3 @@ for.end: ; preds = %for.body ret void } -; CHECK: Determining loop execution counts for: @test4 diff --git a/llvm/test/Analysis/ScalarEvolution/exact_iter_count.ll b/llvm/test/Analysis/ScalarEvolution/exact_iter_count.ll index 86848c0af839..9e6c6aa651b0 100644 --- a/llvm/test/Analysis/ScalarEvolution/exact_iter_count.ll +++ b/llvm/test/Analysis/ScalarEvolution/exact_iter_count.ll @@ -1,11 +1,19 @@ -; RUN: opt < %s "-passes=print" -disable-output 2>&1 | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s "-passes=print" -scalar-evolution-classify-expressions=0 -disable-output 2>&1 | FileCheck %s ; One side exit dominating the latch, exact backedge taken count is known. define void @test_01() { - -; CHECK-LABEL: Determining loop execution counts for: @test_01 +; CHECK-LABEL: 'test_01' +; CHECK-NEXT: Determining loop execution counts for: @test_01 ; CHECK-NEXT: Loop %loop: backedge-taken count is 50 - +; CHECK-NEXT: exit count for loop: 50 +; CHECK-NEXT: exit count for backedge: 100 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 50 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 50 +; CHECK-NEXT: symbolic max exit count for loop: 50 +; CHECK-NEXT: symbolic max exit count for backedge: 100 +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; entry: br label %loop @@ -27,10 +35,17 @@ side.exit: } define void @test_02(i1 %c) { - -; CHECK-LABEL: Determining loop execution counts for: @test_02 +; CHECK-LABEL: 'test_02' +; CHECK-NEXT: Determining loop execution counts for: @test_02 ; CHECK-NEXT: Loop %loop: backedge-taken count is 50 - +; CHECK-NEXT: exit count for merge: 50 +; CHECK-NEXT: exit count for backedge: 100 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 50 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 50 +; CHECK-NEXT: symbolic max exit count for merge: 50 +; CHECK-NEXT: symbolic max exit count for backedge: 100 +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; entry: br label %loop diff --git a/llvm/test/Transforms/PhaseOrdering/scev-custom-dl.ll b/llvm/test/Transforms/PhaseOrdering/scev-custom-dl.ll index 05f0651857f2..467f1e0210ff 100644 --- a/llvm/test/Transforms/PhaseOrdering/scev-custom-dl.ll +++ b/llvm/test/Transforms/PhaseOrdering/scev-custom-dl.ll @@ -1,4 +1,5 @@ -; RUN: opt -passes='default,print' -S < %s 2>&1 | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -passes='default,print' -disable-output -S < %s 2>&1 | FileCheck %s target datalayout = "e-m:m-p:40:64:64:32-i32:32-i16:16-i8:8-n32" @@ -7,11 +8,29 @@ target datalayout = "e-m:m-p:40:64:64:32-i32:32-i16:16-i8:8-n32" ; Test that the standard passes don't obfuscate the IR so scalar evolution can't ; recognize expressions. -; CHECK: test1 ; The loop body contains two increments by %div. ; Make sure that 2*%div is recognizable, and not expressed as a bit mask of %d. -; CHECK: --> {%p,+,(8 * (%d /u 4))} define void @test1(i32 %d, ptr %p) nounwind uwtable ssp { +; CHECK-LABEL: 'test1' +; CHECK-NEXT: Classifying expressions for: @test1 +; CHECK-NEXT: %div1 = lshr i32 %d, 2 +; CHECK-NEXT: --> (%d /u 4) U: [0,1073741824) S: [0,1073741824) +; CHECK-NEXT: %i.03 = phi i32 [ 0, %entry ], [ %inc, %for.body ] +; CHECK-NEXT: --> {0,+,1}<%for.body> U: [0,64) S: [0,64) Exits: 63 LoopDispositions: { %for.body: Computable } +; CHECK-NEXT: %p.addr.02 = phi ptr [ %p, %entry ], [ %add.ptr1, %for.body ] +; CHECK-NEXT: --> {%p,+,(8 * (%d /u 4))}<%for.body> U: full-set S: full-set Exits: ((504 * (%d /u 4)) + %p) LoopDispositions: { %for.body: Computable } +; CHECK-NEXT: %add.ptr = getelementptr inbounds i32, ptr %p.addr.02, i32 %div1 +; CHECK-NEXT: --> {((4 * (%d /u 4)) + %p),+,(8 * (%d /u 4))}<%for.body> U: full-set S: full-set Exits: ((508 * (%d /u 4)) + %p) LoopDispositions: { %for.body: Computable } +; CHECK-NEXT: %add.ptr1 = getelementptr inbounds i32, ptr %add.ptr, i32 %div1 +; CHECK-NEXT: --> {((8 * (%d /u 4)) + %p),+,(8 * (%d /u 4))}<%for.body> U: full-set S: full-set Exits: ((512 * (%d /u 4)) + %p) LoopDispositions: { %for.body: Computable } +; CHECK-NEXT: %inc = add nuw nsw i32 %i.03, 1 +; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,65) S: [1,65) Exits: 64 LoopDispositions: { %for.body: Computable } +; CHECK-NEXT: Determining loop execution counts for: @test1 +; CHECK-NEXT: Loop %for.body: backedge-taken count is 63 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 63 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 63 +; CHECK-NEXT: Loop %for.body: Trip multiple is 64 +; entry: %div = udiv i32 %d, 4 br label %for.cond @@ -37,10 +56,28 @@ for.end: ; preds = %for.cond ret void } -; CHECK: test1a ; Same thing as test1, but it is even more tempting to fold 2 * (%d /u 2) -; CHECK: --> {%p,+,(8 * (%d /u 2))} define void @test1a(i32 %d, ptr %p) nounwind uwtable ssp { +; CHECK-LABEL: 'test1a' +; CHECK-NEXT: Classifying expressions for: @test1a +; CHECK-NEXT: %div1 = lshr i32 %d, 1 +; CHECK-NEXT: --> (%d /u 2) U: [0,-2147483648) S: [0,-2147483648) +; CHECK-NEXT: %i.03 = phi i32 [ 0, %entry ], [ %inc, %for.body ] +; CHECK-NEXT: --> {0,+,1}<%for.body> U: [0,64) S: [0,64) Exits: 63 LoopDispositions: { %for.body: Computable } +; CHECK-NEXT: %p.addr.02 = phi ptr [ %p, %entry ], [ %add.ptr1, %for.body ] +; CHECK-NEXT: --> {%p,+,(8 * (%d /u 2))}<%for.body> U: full-set S: full-set Exits: ((504 * (%d /u 2)) + %p) LoopDispositions: { %for.body: Computable } +; CHECK-NEXT: %add.ptr = getelementptr inbounds i32, ptr %p.addr.02, i32 %div1 +; CHECK-NEXT: --> {((4 * (%d /u 2)) + %p),+,(8 * (%d /u 2))}<%for.body> U: full-set S: full-set Exits: ((508 * (%d /u 2)) + %p) LoopDispositions: { %for.body: Computable } +; CHECK-NEXT: %add.ptr1 = getelementptr inbounds i32, ptr %add.ptr, i32 %div1 +; CHECK-NEXT: --> {((8 * (%d /u 2)) + %p),+,(8 * (%d /u 2))}<%for.body> U: full-set S: full-set Exits: ((512 * (%d /u 2)) + %p) LoopDispositions: { %for.body: Computable } +; CHECK-NEXT: %inc = add nuw nsw i32 %i.03, 1 +; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,65) S: [1,65) Exits: 64 LoopDispositions: { %for.body: Computable } +; CHECK-NEXT: Determining loop execution counts for: @test1a +; CHECK-NEXT: Loop %for.body: backedge-taken count is 63 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 63 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 63 +; CHECK-NEXT: Loop %for.body: Trip multiple is 64 +; entry: %div = udiv i32 %d, 2 br label %for.cond @@ -68,9 +105,22 @@ for.end: ; preds = %for.cond @array = weak global [101 x i32] zeroinitializer, align 32 ; [#uses=1] -; CHECK: Loop %bb: backedge-taken count is 100 define void @test_range_ref1a(i32 %x) { +; CHECK-LABEL: 'test_range_ref1a' +; CHECK-NEXT: Classifying expressions for: @test_range_ref1a +; CHECK-NEXT: %i.01.0 = phi i32 [ 100, %entry ], [ %tmp4, %bb ] +; CHECK-NEXT: --> {100,+,-1}<%bb> U: [0,101) S: [0,101) Exits: 0 LoopDispositions: { %bb: Computable } +; CHECK-NEXT: %tmp1 = getelementptr [101 x i32], ptr @array, i32 0, i32 %i.01.0 +; CHECK-NEXT: --> {(400 + @array),+,-4}<%bb> U: [0,-3) S: [-2147483648,2147483645) Exits: @array LoopDispositions: { %bb: Computable } +; CHECK-NEXT: %tmp4 = add nsw i32 %i.01.0, -1 +; CHECK-NEXT: --> {99,+,-1}<%bb> U: [-1,100) S: [-1,100) Exits: -1 LoopDispositions: { %bb: Computable } +; CHECK-NEXT: Determining loop execution counts for: @test_range_ref1a +; CHECK-NEXT: Loop %bb: backedge-taken count is 100 +; CHECK-NEXT: Loop %bb: constant max backedge-taken count is i32 100 +; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is 100 +; CHECK-NEXT: Loop %bb: Trip multiple is 101 +; entry: br label %bb @@ -87,6 +137,32 @@ return: ; preds = %bb } define i32 @test_loop_idiom_recogize(i32 %x, i32 %y, ptr %lam, ptr %alp) nounwind { +; CHECK-LABEL: 'test_loop_idiom_recogize' +; CHECK-NEXT: Classifying expressions for: @test_loop_idiom_recogize +; CHECK-NEXT: %indvar = phi i32 [ 0, %bb1.thread ], [ %indvar.next, %bb1 ] +; CHECK-NEXT: --> {0,+,1}<%bb1> U: [0,256) S: [0,256) Exits: 255 LoopDispositions: { %bb1: Computable } +; CHECK-NEXT: %i.0.reg2mem.0 = sub nuw nsw i32 255, %indvar +; CHECK-NEXT: --> {255,+,-1}<%bb1> U: [0,256) S: [0,256) Exits: 0 LoopDispositions: { %bb1: Computable } +; CHECK-NEXT: %0 = getelementptr i32, ptr %alp, i32 %i.0.reg2mem.0 +; CHECK-NEXT: --> {(1020 + %alp),+,-4}<%bb1> U: full-set S: full-set Exits: %alp LoopDispositions: { %bb1: Computable } +; CHECK-NEXT: %1 = load i32, ptr %0, align 4 +; CHECK-NEXT: --> %1 U: full-set S: full-set Exits: <> LoopDispositions: { %bb1: Variant } +; CHECK-NEXT: %2 = getelementptr i32, ptr %lam, i32 %i.0.reg2mem.0 +; CHECK-NEXT: --> {(1020 + %lam),+,-4}<%bb1> U: full-set S: full-set Exits: %lam LoopDispositions: { %bb1: Computable } +; CHECK-NEXT: %indvar.next = add nuw nsw i32 %indvar, 1 +; CHECK-NEXT: --> {1,+,1}<%bb1> U: [1,257) S: [1,257) Exits: 256 LoopDispositions: { %bb1: Computable } +; CHECK-NEXT: %tmp10 = mul i32 %x, 255 +; CHECK-NEXT: --> (255 * %x) U: full-set S: full-set +; CHECK-NEXT: %z.0.reg2mem.0 = add i32 %y, %x +; CHECK-NEXT: --> (%x + %y) U: full-set S: full-set +; CHECK-NEXT: %3 = add i32 %z.0.reg2mem.0, %tmp10 +; CHECK-NEXT: --> ((256 * %x) + %y) U: full-set S: full-set +; CHECK-NEXT: Determining loop execution counts for: @test_loop_idiom_recogize +; CHECK-NEXT: Loop %bb1: backedge-taken count is 255 +; CHECK-NEXT: Loop %bb1: constant max backedge-taken count is i32 255 +; CHECK-NEXT: Loop %bb1: symbolic max backedge-taken count is 255 +; CHECK-NEXT: Loop %bb1: Trip multiple is 256 +; bb1.thread: br label %bb1 @@ -114,8 +190,22 @@ declare void @use(i1) declare void @llvm.experimental.guard(i1, ...) ; This tests getRangeRef acts as intended with different idx size. -; CHECK: Loop %loop: Unpredictable constant max backedge-taken count. define void @test_range_ref1(i8 %t) { +; CHECK-LABEL: 'test_range_ref1' +; CHECK-NEXT: Classifying expressions for: @test_range_ref1 +; CHECK-NEXT: %0 = zext i8 %t to i40 +; CHECK-NEXT: --> (zext i8 %t to i40) U: [0,256) S: [0,256) +; CHECK-NEXT: %t.ptr = inttoptr i40 %0 to ptr +; CHECK-NEXT: --> %t.ptr U: [0,256) S: [0,256) +; CHECK-NEXT: %idx = phi ptr [ %t.ptr, %entry ], [ %snext, %loop ] +; CHECK-NEXT: --> {%t.ptr,+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %snext = getelementptr inbounds i8, ptr %idx, i32 1 +; CHECK-NEXT: --> {(1 + %t.ptr),+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: Determining loop execution counts for: @test_range_ref1 +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. +; entry: %t.ptr = inttoptr i8 %t to ptr %p.42 = inttoptr i8 42 to ptr -- GitLab From 012d217174e28dc000ba1f437827cd23ec5ef606 Mon Sep 17 00:00:00 2001 From: Cameron McInally Date: Wed, 6 Mar 2024 15:59:35 -0500 Subject: [PATCH 356/929] [LV] Use scalar CMP for active-lane-mask with scalar VF (#83902) Instead of generating a <1 x i1> active lane mask intrinsic, generate the equivalent scalar ICMP instead. This allows us to avoid unnecessarily extracting the scalar part from the vector mask. Fixes llvm#73894. --- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 6 + .../AArch64/uniform-args-call-variants.ll | 45 ++++--- llvm/test/Transforms/LoopVectorize/pr73894.ll | 111 ++++++++++++++++++ .../strict-fadd-interleave-only.ll | 66 +++++------ 4 files changed, 168 insertions(+), 60 deletions(-) create mode 100644 llvm/test/Transforms/LoopVectorize/pr73894.ll diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 27b72575ddd5..40ebec7305b4 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -314,6 +314,12 @@ Value *VPInstruction::generateInstruction(VPTransformState &State, // Get the original loop tripcount. Value *ScalarTC = State.get(getOperand(1), VPIteration(Part, 0)); + // If this part of the active lane mask is scalar, generate the CMP directly + // to avoid unnecessary extracts. + if (State.VF.isScalar()) + return Builder.CreateCmp(CmpInst::Predicate::ICMP_ULT, VIVElem0, ScalarTC, + Name); + auto *Int1Ty = Type::getInt1Ty(Builder.getContext()); auto *PredTy = VectorType::get(Int1Ty, State.VF); return Builder.CreateIntrinsic(Intrinsic::get_active_lane_mask, diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll index 17edfe513dd0..4957bbeda671 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll @@ -208,40 +208,37 @@ define void @test_uniform_not_invariant(ptr noalias %dst, ptr readonly %src, i64 ; INTERLEAVE-NEXT: entry: ; INTERLEAVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 2) ; INTERLEAVE-NEXT: [[TMP1:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 2) -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <1 x i1> @llvm.get.active.lane.mask.v1i1.i64(i64 0, i64 [[N]]) -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <1 x i1> @llvm.get.active.lane.mask.v1i1.i64(i64 1, i64 [[N]]) +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = icmp ne i64 [[N]], 0 +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ugt i64 [[N]], 1 ; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] ; INTERLEAVE: vector.body: ; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE4:%.*]] ] -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <1 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_STORE_CONTINUE4]] ] -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <1 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], [[PRED_STORE_CONTINUE4]] ] -; INTERLEAVE-NEXT: [[TMP2:%.*]] = extractelement <1 x i1> [[ACTIVE_LANE_MASK]], i64 0 -; INTERLEAVE-NEXT: br i1 [[TMP2]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_STORE_CONTINUE4]] ] +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY1]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], [[PRED_STORE_CONTINUE4]] ] +; INTERLEAVE-NEXT: br i1 [[ACTIVE_LANE_MASK]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] ; INTERLEAVE: pred.store.if: -; INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]] -; INTERLEAVE-NEXT: [[TMP4:%.*]] = load double, ptr [[TMP3]], align 8 -; INTERLEAVE-NEXT: [[TMP5:%.*]] = call double @foo(double [[TMP4]], i64 [[INDEX]]) #[[ATTR5:[0-9]+]] -; INTERLEAVE-NEXT: [[TMP6:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]] -; INTERLEAVE-NEXT: store double [[TMP5]], ptr [[TMP6]], align 8 +; INTERLEAVE-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]] +; INTERLEAVE-NEXT: [[TMP3:%.*]] = load double, ptr [[TMP2]], align 8 +; INTERLEAVE-NEXT: [[TMP4:%.*]] = call double @foo(double [[TMP3]], i64 [[INDEX]]) #[[ATTR5:[0-9]+]] +; INTERLEAVE-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]] +; INTERLEAVE-NEXT: store double [[TMP4]], ptr [[TMP5]], align 8 ; INTERLEAVE-NEXT: br label [[PRED_STORE_CONTINUE]] ; INTERLEAVE: pred.store.continue: -; INTERLEAVE-NEXT: [[TMP7:%.*]] = extractelement <1 x i1> [[ACTIVE_LANE_MASK2]], i64 0 -; INTERLEAVE-NEXT: br i1 [[TMP7]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4]] +; INTERLEAVE-NEXT: br i1 [[ACTIVE_LANE_MASK2]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4]] ; INTERLEAVE: pred.store.if3: -; INTERLEAVE-NEXT: [[TMP8:%.*]] = or disjoint i64 [[INDEX]], 1 -; INTERLEAVE-NEXT: [[TMP9:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP8]] -; INTERLEAVE-NEXT: [[TMP10:%.*]] = load double, ptr [[TMP9]], align 8 -; INTERLEAVE-NEXT: [[TMP11:%.*]] = call double @foo(double [[TMP10]], i64 [[TMP8]]) #[[ATTR5]] -; INTERLEAVE-NEXT: [[TMP12:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[TMP8]] -; INTERLEAVE-NEXT: store double [[TMP11]], ptr [[TMP12]], align 8 +; INTERLEAVE-NEXT: [[TMP6:%.*]] = or disjoint i64 [[INDEX]], 1 +; INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP6]] +; INTERLEAVE-NEXT: [[TMP8:%.*]] = load double, ptr [[TMP7]], align 8 +; INTERLEAVE-NEXT: [[TMP9:%.*]] = call double @foo(double [[TMP8]], i64 [[TMP6]]) #[[ATTR5]] +; INTERLEAVE-NEXT: [[TMP10:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[TMP6]] +; INTERLEAVE-NEXT: store double [[TMP9]], ptr [[TMP10]], align 8 ; INTERLEAVE-NEXT: br label [[PRED_STORE_CONTINUE4]] ; INTERLEAVE: pred.store.continue4: ; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2 -; INTERLEAVE-NEXT: [[TMP13:%.*]] = or disjoint i64 [[INDEX]], 1 -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <1 x i1> @llvm.get.active.lane.mask.v1i1.i64(i64 [[INDEX]], i64 [[TMP0]]) -; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = call <1 x i1> @llvm.get.active.lane.mask.v1i1.i64(i64 [[TMP13]], i64 [[TMP1]]) -; INTERLEAVE-NEXT: [[TMP14:%.*]] = extractelement <1 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0 -; INTERLEAVE-NEXT: br i1 [[TMP14]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP:%.*]], !llvm.loop [[LOOP4:![0-9]+]] +; INTERLEAVE-NEXT: [[TMP11:%.*]] = or disjoint i64 [[INDEX]], 1 +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX]], [[TMP0]] +; INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT5]] = icmp ult i64 [[TMP11]], [[TMP1]] +; INTERLEAVE-NEXT: br i1 [[ACTIVE_LANE_MASK_NEXT]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP:%.*]], !llvm.loop [[LOOP4:![0-9]+]] ; INTERLEAVE: for.cond.cleanup: ; INTERLEAVE-NEXT: ret void ; diff --git a/llvm/test/Transforms/LoopVectorize/pr73894.ll b/llvm/test/Transforms/LoopVectorize/pr73894.ll new file mode 100644 index 000000000000..4203a8eb8df6 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/pr73894.ll @@ -0,0 +1,111 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -passes=loop-vectorize -mcpu=neoverse-v1 -force-vector-interleave=2 -force-vector-width=1 -S %s | FileCheck %s + +target triple = "arm64-linux" + +define i32 @pr70988() { +; CHECK-LABEL: define i32 @pr70988( +; CHECK-SAME: ) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr null, align 4 +; CHECK-NEXT: [[TMP1:%.*]] = and i32 [[TMP0]], 15 +; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP1]], i32 1) +; CHECK-NEXT: [[UMAX:%.*]] = zext i32 [[TMP2]] to i64 +; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], 1 +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 2 +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = icmp ult i64 0, [[UMAX]] +; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ult i64 1, [[UMAX]] +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT6:%.*]], [[PRED_LOAD_CONTINUE5:%.*]] ] +; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_LOAD_CONTINUE5]] ] +; CHECK-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY1]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT7:%.*]], [[PRED_LOAD_CONTINUE5]] ] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[PRED_LOAD_CONTINUE5]] ] +; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[PRED_LOAD_CONTINUE5]] ] +; CHECK-NEXT: br i1 [[ACTIVE_LANE_MASK]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; CHECK: pred.load.if: +; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr null, i64 [[TMP3]] +; CHECK-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP4]], align 8 +; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]] +; CHECK: pred.load.continue: +; CHECK-NEXT: [[TMP7:%.*]] = phi ptr [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP8:%.*]] = phi i32 [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ] +; CHECK-NEXT: br i1 [[ACTIVE_LANE_MASK2]], label [[PRED_LOAD_IF4:%.*]], label [[PRED_LOAD_CONTINUE5]] +; CHECK: pred.load.if4: +; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr null, i64 [[TMP9]] +; CHECK-NEXT: [[TMP11:%.*]] = load ptr, ptr [[TMP10]], align 8 +; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE5]] +; CHECK: pred.load.continue5: +; CHECK-NEXT: [[TMP13:%.*]] = phi ptr [ poison, [[PRED_LOAD_CONTINUE]] ], [ [[TMP11]], [[PRED_LOAD_IF4]] ] +; CHECK-NEXT: [[TMP14:%.*]] = phi i32 [ poison, [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF4]] ] +; CHECK-NEXT: [[TMP15:%.*]] = tail call i32 @llvm.smax.i32(i32 [[TMP8]], i32 [[VEC_PHI]]) +; CHECK-NEXT: [[TMP16:%.*]] = tail call i32 @llvm.smax.i32(i32 [[TMP14]], i32 [[VEC_PHI3]]) +; CHECK-NEXT: [[TMP17]] = select i1 [[ACTIVE_LANE_MASK]], i32 [[TMP15]], i32 [[VEC_PHI]] +; CHECK-NEXT: [[TMP18]] = select i1 [[ACTIVE_LANE_MASK2]], i32 [[TMP16]], i32 [[VEC_PHI3]] +; CHECK-NEXT: [[INDEX_NEXT:%.*]] = add i64 [[INDEX]], 2 +; CHECK-NEXT: [[INDEX_NEXT6]] = add i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[INDEX_NEXT]], 1 +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX_NEXT]], [[UMAX]] +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT7]] = icmp ult i64 [[TMP19]], [[UMAX]] +; CHECK-NEXT: [[TMP20:%.*]] = xor i1 [[ACTIVE_LANE_MASK_NEXT]], true +; CHECK-NEXT: [[TMP21:%.*]] = xor i1 [[ACTIVE_LANE_MASK_NEXT7]], true +; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK: middle.block: +; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP17]], i32 [[TMP18]]) +; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[RDX_MINMAX]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[INDUC:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDUC_NEXT:%.*]], [[LOOP]] ] +; CHECK-NEXT: [[MAX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[TMP24:%.*]], [[LOOP]] ] +; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr null, i64 [[INDUC]] +; CHECK-NEXT: [[TMP22:%.*]] = load ptr, ptr [[GEP]], align 8 +; CHECK-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4 +; CHECK-NEXT: [[TMP24]] = tail call i32 @llvm.smax.i32(i32 [[TMP23]], i32 [[MAX]]) +; CHECK-NEXT: [[INDUC_NEXT]] = add nuw nsw i64 [[INDUC]], 1 +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDUC_NEXT]], [[UMAX]] +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK: exit: +; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[TMP24]], [[LOOP]] ], [ [[RDX_MINMAX]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: ret i32 [[RES]] +; +entry: + %0 = load i32, ptr null + %1 = and i32 %0, 15 + %2 = call i32 @llvm.umax.i32(i32 %1, i32 1) + %umax = zext i32 %2 to i64 + br label %loop + +loop: + %induc = phi i64 [ 0, %entry ], [ %induc.next, %loop ] + %max = phi i32 [ 0, %entry ], [ %5, %loop ] + %gep = getelementptr i32, ptr null, i64 %induc + %3 = load ptr, ptr %gep + %4 = load i32, ptr %3 + %5 = tail call i32 @llvm.smax.i32(i32 %4, i32 %max) + %induc.next = add nuw nsw i64 %induc, 1 + %exitcond.not = icmp eq i64 %induc.next, %umax + br i1 %exitcond.not, label %exit, label %loop + +exit: + %res = phi i32 [ %5, %loop ] + ret i32 %res +} + +declare i32 @llvm.smax.i32(i32, i32) +declare i32 @llvm.umax.i32(i32, i32) +;. +; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} +; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1} +; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"} +; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]} +;. diff --git a/llvm/test/Transforms/LoopVectorize/strict-fadd-interleave-only.ll b/llvm/test/Transforms/LoopVectorize/strict-fadd-interleave-only.ll index 452fa9fa8c8e..97c84f251fef 100644 --- a/llvm/test/Transforms/LoopVectorize/strict-fadd-interleave-only.ll +++ b/llvm/test/Transforms/LoopVectorize/strict-fadd-interleave-only.ll @@ -46,25 +46,23 @@ define float @pr70988() { ; CHECK-ALM-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-ALM: vector.body: ; CHECK-ALM-NEXT: [[INDEX1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ] -; CHECK-ALM-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ] +; CHECK-ALM-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ] ; CHECK-ALM-NEXT: [[TMP0:%.*]] = add i32 [[INDEX1]], 0 ; CHECK-ALM-NEXT: [[TMP1:%.*]] = add i32 [[INDEX1]], 1 -; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <1 x i1> @llvm.get.active.lane.mask.v1i1.i32(i32 [[TMP0]], i32 1021) -; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = call <1 x i1> @llvm.get.active.lane.mask.v1i1.i32(i32 [[TMP1]], i32 1021) -; CHECK-ALM-NEXT: [[TMP2:%.*]] = extractelement <1 x i1> [[ACTIVE_LANE_MASK]], i32 0 -; CHECK-ALM-NEXT: [[TMP3:%.*]] = select contract i1 [[TMP2]], float 1.000000e+00, float -0.000000e+00 -; CHECK-ALM-NEXT: [[TMP4:%.*]] = fadd contract float [[VEC_PHI]], [[TMP3]] -; CHECK-ALM-NEXT: [[TMP5:%.*]] = extractelement <1 x i1> [[ACTIVE_LANE_MASK2]], i32 0 -; CHECK-ALM-NEXT: [[TMP6:%.*]] = select contract i1 [[TMP5]], float 1.000000e+00, float -0.000000e+00 -; CHECK-ALM-NEXT: [[TMP7]] = fadd contract float [[TMP4]], [[TMP6]] +; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK:%.*]] = icmp ult i32 [[TMP0]], 1021 +; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = icmp ult i32 [[TMP1]], 1021 +; CHECK-ALM-NEXT: [[TMP2:%.*]] = select contract i1 [[ACTIVE_LANE_MASK]], float 1.000000e+00, float -0.000000e+00 +; CHECK-ALM-NEXT: [[TMP3:%.*]] = fadd contract float [[VEC_PHI]], [[TMP2]] +; CHECK-ALM-NEXT: [[TMP4:%.*]] = select contract i1 [[ACTIVE_LANE_MASK2]], float 1.000000e+00, float -0.000000e+00 +; CHECK-ALM-NEXT: [[TMP5]] = fadd contract float [[TMP3]], [[TMP4]] ; CHECK-ALM-NEXT: [[INDEX_NEXT3]] = add i32 [[INDEX1]], 2 -; CHECK-ALM-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT3]], 1022 -; CHECK-ALM-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-ALM-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT3]], 1022 +; CHECK-ALM-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK-ALM: middle.block: ; CHECK-ALM-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] ; CHECK-ALM: scalar.ph: ; CHECK-ALM-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 1022, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] -; CHECK-ALM-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ] +; CHECK-ALM-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ] ; CHECK-ALM-NEXT: br label [[LOOP:%.*]] ; CHECK-ALM: loop: ; CHECK-ALM-NEXT: [[INDEX:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDEX_NEXT:%.*]], [[LOOP]] ] @@ -74,7 +72,7 @@ define float @pr70988() { ; CHECK-ALM-NEXT: [[COND:%.*]] = icmp ult i32 [[INDEX_NEXT]], 1021 ; CHECK-ALM-NEXT: br i1 [[COND]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP3:![0-9]+]] ; CHECK-ALM: exit: -; CHECK-ALM-NEXT: [[DOTLCSSA:%.*]] = phi float [ [[RDX_NEXT]], [[LOOP]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ] +; CHECK-ALM-NEXT: [[DOTLCSSA:%.*]] = phi float [ [[RDX_NEXT]], [[LOOP]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ] ; CHECK-ALM-NEXT: ret float [[DOTLCSSA]] ; entry: @@ -157,41 +155,37 @@ define float @pr72720reduction_using_active_lane_mask(ptr %src) { ; CHECK-ALM-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-ALM: vector.body: ; CHECK-ALM-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE3:%.*]] ] -; CHECK-ALM-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[PRED_LOAD_CONTINUE3]] ] +; CHECK-ALM-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[PRED_LOAD_CONTINUE3]] ] ; CHECK-ALM-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0 ; CHECK-ALM-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1 -; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <1 x i1> @llvm.get.active.lane.mask.v1i1.i32(i32 [[TMP0]], i32 15) -; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = call <1 x i1> @llvm.get.active.lane.mask.v1i1.i32(i32 [[TMP1]], i32 15) -; CHECK-ALM-NEXT: [[TMP2:%.*]] = extractelement <1 x i1> [[ACTIVE_LANE_MASK]], i32 0 -; CHECK-ALM-NEXT: br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK:%.*]] = icmp ult i32 [[TMP0]], 15 +; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = icmp ult i32 [[TMP1]], 15 +; CHECK-ALM-NEXT: br i1 [[ACTIVE_LANE_MASK]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] ; CHECK-ALM: pred.load.if: -; CHECK-ALM-NEXT: [[TMP3:%.*]] = getelementptr float, ptr [[SRC]], i32 [[TMP0]] -; CHECK-ALM-NEXT: [[TMP4:%.*]] = load float, ptr [[TMP3]], align 4 +; CHECK-ALM-NEXT: [[TMP2:%.*]] = getelementptr float, ptr [[SRC]], i32 [[TMP0]] +; CHECK-ALM-NEXT: [[TMP3:%.*]] = load float, ptr [[TMP2]], align 4 ; CHECK-ALM-NEXT: br label [[PRED_LOAD_CONTINUE]] ; CHECK-ALM: pred.load.continue: -; CHECK-ALM-NEXT: [[TMP5:%.*]] = phi float [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[PRED_LOAD_IF]] ] -; CHECK-ALM-NEXT: [[TMP6:%.*]] = extractelement <1 x i1> [[ACTIVE_LANE_MASK1]], i32 0 -; CHECK-ALM-NEXT: br i1 [[TMP6]], label [[PRED_LOAD_IF2:%.*]], label [[PRED_LOAD_CONTINUE3]] +; CHECK-ALM-NEXT: [[TMP4:%.*]] = phi float [ poison, [[VECTOR_BODY]] ], [ [[TMP3]], [[PRED_LOAD_IF]] ] +; CHECK-ALM-NEXT: br i1 [[ACTIVE_LANE_MASK1]], label [[PRED_LOAD_IF2:%.*]], label [[PRED_LOAD_CONTINUE3]] ; CHECK-ALM: pred.load.if2: -; CHECK-ALM-NEXT: [[TMP7:%.*]] = getelementptr float, ptr [[SRC]], i32 [[TMP1]] -; CHECK-ALM-NEXT: [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4 +; CHECK-ALM-NEXT: [[TMP5:%.*]] = getelementptr float, ptr [[SRC]], i32 [[TMP1]] +; CHECK-ALM-NEXT: [[TMP6:%.*]] = load float, ptr [[TMP5]], align 4 ; CHECK-ALM-NEXT: br label [[PRED_LOAD_CONTINUE3]] ; CHECK-ALM: pred.load.continue3: -; CHECK-ALM-NEXT: [[TMP9:%.*]] = phi float [ poison, [[PRED_LOAD_CONTINUE]] ], [ [[TMP8]], [[PRED_LOAD_IF2]] ] -; CHECK-ALM-NEXT: [[TMP10:%.*]] = extractelement <1 x i1> [[ACTIVE_LANE_MASK]], i32 0 -; CHECK-ALM-NEXT: [[TMP11:%.*]] = select contract i1 [[TMP10]], float [[TMP5]], float -0.000000e+00 -; CHECK-ALM-NEXT: [[TMP12:%.*]] = fadd contract float [[VEC_PHI]], [[TMP11]] -; CHECK-ALM-NEXT: [[TMP13:%.*]] = extractelement <1 x i1> [[ACTIVE_LANE_MASK1]], i32 0 -; CHECK-ALM-NEXT: [[TMP14:%.*]] = select contract i1 [[TMP13]], float [[TMP9]], float -0.000000e+00 -; CHECK-ALM-NEXT: [[TMP15]] = fadd contract float [[TMP12]], [[TMP14]] +; CHECK-ALM-NEXT: [[TMP7:%.*]] = phi float [ poison, [[PRED_LOAD_CONTINUE]] ], [ [[TMP6]], [[PRED_LOAD_IF2]] ] +; CHECK-ALM-NEXT: [[TMP8:%.*]] = select contract i1 [[ACTIVE_LANE_MASK]], float [[TMP4]], float -0.000000e+00 +; CHECK-ALM-NEXT: [[TMP9:%.*]] = fadd contract float [[VEC_PHI]], [[TMP8]] +; CHECK-ALM-NEXT: [[TMP10:%.*]] = select contract i1 [[ACTIVE_LANE_MASK1]], float [[TMP7]], float -0.000000e+00 +; CHECK-ALM-NEXT: [[TMP11]] = fadd contract float [[TMP9]], [[TMP10]] ; CHECK-ALM-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2 -; CHECK-ALM-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16 -; CHECK-ALM-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; CHECK-ALM-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16 +; CHECK-ALM-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK-ALM: middle.block: ; CHECK-ALM-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] ; CHECK-ALM: scalar.ph: ; CHECK-ALM-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 16, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] -; CHECK-ALM-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[TMP15]], [[MIDDLE_BLOCK]] ] +; CHECK-ALM-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ] ; CHECK-ALM-NEXT: br label [[LOOP:%.*]] ; CHECK-ALM: loop: ; CHECK-ALM-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NARROW:%.*]], [[LOOP]] ] @@ -203,7 +197,7 @@ define float @pr72720reduction_using_active_lane_mask(ptr %src) { ; CHECK-ALM-NEXT: [[EC:%.*]] = icmp ult i32 [[NARROW]], 15 ; CHECK-ALM-NEXT: br i1 [[EC]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK-ALM: exit: -; CHECK-ALM-NEXT: [[DOTLCSSA:%.*]] = phi float [ [[RDX_NEXT]], [[LOOP]] ], [ [[TMP15]], [[MIDDLE_BLOCK]] ] +; CHECK-ALM-NEXT: [[DOTLCSSA:%.*]] = phi float [ [[RDX_NEXT]], [[LOOP]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ] ; CHECK-ALM-NEXT: ret float [[DOTLCSSA]] ; entry: -- GitLab From bec2d105c7b8b39c6d090146d076201d1cd61991 Mon Sep 17 00:00:00 2001 From: Janek van Oirschot <5994977+JanekvO@users.noreply.github.com> Date: Wed, 6 Mar 2024 21:01:54 +0000 Subject: [PATCH 357/929] [AMDGPU] Add AMDGPU specific variadic operation MCExprs (#82022) Adds AMDGPU specific variadic MCExpr operations 'max' and 'or'. --- llvm/docs/AMDGPUUsage.rst | 19 +++ .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp | 55 ++++++++ .../AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp | 94 +++++++++++++ .../Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h | 74 ++++++++++ .../Target/AMDGPU/MCTargetDesc/CMakeLists.txt | 1 + llvm/test/MC/AMDGPU/mcexpr_amd.s | 130 ++++++++++++++++++ llvm/test/MC/AMDGPU/mcexpr_amd_err.s | 53 +++++++ 7 files changed, 426 insertions(+) create mode 100644 llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp create mode 100644 llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h create mode 100644 llvm/test/MC/AMDGPU/mcexpr_amd.s create mode 100644 llvm/test/MC/AMDGPU/mcexpr_amd_err.s diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst index 7f39f69cae60..c7cb06ffbef1 100644 --- a/llvm/docs/AMDGPUUsage.rst +++ b/llvm/docs/AMDGPUUsage.rst @@ -1534,6 +1534,25 @@ The AMDGPU backend supports the following calling conventions: =============================== ========================================================== +AMDGPU MCExpr +------------- + +As part of the AMDGPU MC layer, AMDGPU provides the following target specific +``MCExpr``\s. + + .. table:: AMDGPU MCExpr types: + :name: amdgpu-mcexpr-table + + =================== ================= ======================================================== + MCExpr Operands Return value + =================== ================= ======================================================== + ``max(arg, ...)`` 1 or more Variadic signed operation that returns the maximum + value of all its arguments. + + ``or(arg, ...)`` 1 or more Variadic signed operation that returns the bitwise-or + result of all its arguments. + + =================== ================= ======================================================== .. _amdgpu-elf-code-object: diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp index cb4eddfe5320..16a5d6879ce7 100644 --- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp +++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "AMDKernelCodeT.h" +#include "MCTargetDesc/AMDGPUMCExpr.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "MCTargetDesc/AMDGPUTargetStreamer.h" #include "SIDefines.h" @@ -1816,6 +1817,7 @@ private: public: void onBeginOfFile() override; + bool parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) override; ParseStatus parseCustomOperand(OperandVector &Operands, unsigned MCK); @@ -8277,6 +8279,59 @@ void AMDGPUAsmParser::onBeginOfFile() { getTargetStreamer().EmitDirectiveAMDGCNTarget(); } +/// Parse AMDGPU specific expressions. +/// +/// expr ::= or(expr, ...) | +/// max(expr, ...) +/// +bool AMDGPUAsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) { + using AGVK = AMDGPUVariadicMCExpr::VariadicKind; + + if (isToken(AsmToken::Identifier)) { + StringRef TokenId = getTokenStr(); + AGVK VK = StringSwitch(TokenId) + .Case("max", AGVK::AGVK_Max) + .Case("or", AGVK::AGVK_Or) + .Default(AGVK::AGVK_None); + + if (VK != AGVK::AGVK_None && peekToken().is(AsmToken::LParen)) { + SmallVector Exprs; + uint64_t CommaCount = 0; + lex(); // Eat 'max'/'or' + lex(); // Eat '(' + while (true) { + if (trySkipToken(AsmToken::RParen)) { + if (Exprs.empty()) { + Error(getToken().getLoc(), + "empty " + Twine(TokenId) + " expression"); + return true; + } + if (CommaCount + 1 != Exprs.size()) { + Error(getToken().getLoc(), + "mismatch of commas in " + Twine(TokenId) + " expression"); + return true; + } + Res = AMDGPUVariadicMCExpr::create(VK, Exprs, getContext()); + return false; + } + const MCExpr *Expr; + if (getParser().parseExpression(Expr, EndLoc)) + return true; + Exprs.push_back(Expr); + bool LastTokenWasComma = trySkipToken(AsmToken::Comma); + if (LastTokenWasComma) + CommaCount++; + if (!LastTokenWasComma && !isToken(AsmToken::RParen)) { + Error(getToken().getLoc(), + "unexpected token in " + Twine(TokenId) + " expression"); + return true; + } + } + } + } + return getParser().parsePrimaryExpr(Res, EndLoc, nullptr); +} + ParseStatus AMDGPUAsmParser::parseOModSI(OperandVector &Operands) { StringRef Name = getTokenStr(); if (Name == "mul") { diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp new file mode 100644 index 000000000000..0659937e06a2 --- /dev/null +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp @@ -0,0 +1,94 @@ +//===- AMDGPUMCExpr.cpp - AMDGPU specific MC expression classes -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "AMDGPUMCExpr.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCStreamer.h" +#include "llvm/MC/MCSymbol.h" +#include "llvm/MC/MCValue.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/raw_ostream.h" +#include + +using namespace llvm; + +const AMDGPUVariadicMCExpr * +AMDGPUVariadicMCExpr::create(VariadicKind Kind, ArrayRef Args, + MCContext &Ctx) { + return new (Ctx) AMDGPUVariadicMCExpr(Kind, Args); +} + +const MCExpr *AMDGPUVariadicMCExpr::getSubExpr(size_t Index) const { + assert(Index < Args.size() && + "Indexing out of bounds AMDGPUVariadicMCExpr sub-expr"); + return Args[Index]; +} + +void AMDGPUVariadicMCExpr::printImpl(raw_ostream &OS, + const MCAsmInfo *MAI) const { + switch (Kind) { + default: + llvm_unreachable("Unknown AMDGPUVariadicMCExpr kind."); + case AGVK_Or: + OS << "or("; + break; + case AGVK_Max: + OS << "max("; + break; + } + for (auto It = Args.begin(); It != Args.end(); ++It) { + (*It)->print(OS, MAI, /*InParens=*/false); + if ((It + 1) != Args.end()) + OS << ", "; + } + OS << ')'; +} + +static int64_t op(AMDGPUVariadicMCExpr::VariadicKind Kind, int64_t Arg1, + int64_t Arg2) { + switch (Kind) { + default: + llvm_unreachable("Unknown AMDGPUVariadicMCExpr kind."); + case AMDGPUVariadicMCExpr::AGVK_Max: + return std::max(Arg1, Arg2); + case AMDGPUVariadicMCExpr::AGVK_Or: + return Arg1 | Arg2; + } +} + +bool AMDGPUVariadicMCExpr::evaluateAsRelocatableImpl( + MCValue &Res, const MCAsmLayout *Layout, const MCFixup *Fixup) const { + std::optional Total; + + for (const MCExpr *Arg : Args) { + MCValue ArgRes; + if (!Arg->evaluateAsRelocatable(ArgRes, Layout, Fixup) || + !ArgRes.isAbsolute()) + return false; + + if (!Total.has_value()) + Total = ArgRes.getConstant(); + Total = op(Kind, *Total, ArgRes.getConstant()); + } + + Res = MCValue::get(*Total); + return true; +} + +void AMDGPUVariadicMCExpr::visitUsedExpr(MCStreamer &Streamer) const { + for (const MCExpr *Arg : Args) + Streamer.visitUsedExpr(*Arg); +} + +MCFragment *AMDGPUVariadicMCExpr::findAssociatedFragment() const { + for (const MCExpr *Arg : Args) { + if (Arg->findAssociatedFragment()) + return Arg->findAssociatedFragment(); + } + return nullptr; +} diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h new file mode 100644 index 000000000000..9e8452f2c624 --- /dev/null +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h @@ -0,0 +1,74 @@ +//===- AMDGPUMCExpr.h - AMDGPU specific MC expression classes ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUMCEXPR_H +#define LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUMCEXPR_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/MC/MCExpr.h" + +namespace llvm { + +/// AMDGPU target specific variadic MCExpr operations. +/// +/// Takes in a minimum of 1 argument to be used with an operation. The supported +/// operations are: +/// - (bitwise) or +/// - max +/// +/// \note If the 'or'/'max' operations are provided only a single argument, the +/// operation will act as a no-op and simply resolve as the provided argument. +/// +class AMDGPUVariadicMCExpr : public MCTargetExpr { +public: + enum VariadicKind { AGVK_None, AGVK_Or, AGVK_Max }; + +private: + VariadicKind Kind; + SmallVector Args; + + AMDGPUVariadicMCExpr(VariadicKind Kind, ArrayRef Args) + : Kind(Kind), Args(Args) { + assert(Args.size() >= 1 && "Needs a minimum of one expression."); + assert(Kind != AGVK_None && + "Cannot construct AMDGPUVariadicMCExpr of kind none."); + } + +public: + static const AMDGPUVariadicMCExpr * + create(VariadicKind Kind, ArrayRef Args, MCContext &Ctx); + + static const AMDGPUVariadicMCExpr *createOr(ArrayRef Args, + MCContext &Ctx) { + return create(VariadicKind::AGVK_Or, Args, Ctx); + } + + static const AMDGPUVariadicMCExpr *createMax(ArrayRef Args, + MCContext &Ctx) { + return create(VariadicKind::AGVK_Max, Args, Ctx); + } + + VariadicKind getKind() const { return Kind; } + const MCExpr *getSubExpr(size_t Index) const; + + void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override; + bool evaluateAsRelocatableImpl(MCValue &Res, const MCAsmLayout *Layout, + const MCFixup *Fixup) const override; + void visitUsedExpr(MCStreamer &Streamer) const override; + MCFragment *findAssociatedFragment() const override; + void fixELFSymbolsInTLSFixups(MCAssembler &) const override{}; + + static bool classof(const MCExpr *E) { + return E->getKind() == MCExpr::Target; + } +}; + +} // end namespace llvm + +#endif // LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUMCEXPR_H diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt index 5dc76071b059..0842a58f794b 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt @@ -5,6 +5,7 @@ add_llvm_component_library(LLVMAMDGPUDesc AMDGPUInstPrinter.cpp AMDGPUMCAsmInfo.cpp AMDGPUMCCodeEmitter.cpp + AMDGPUMCExpr.cpp AMDGPUMCTargetDesc.cpp AMDGPUTargetStreamer.cpp R600InstPrinter.cpp diff --git a/llvm/test/MC/AMDGPU/mcexpr_amd.s b/llvm/test/MC/AMDGPU/mcexpr_amd.s new file mode 100644 index 000000000000..a9639c3acc30 --- /dev/null +++ b/llvm/test/MC/AMDGPU/mcexpr_amd.s @@ -0,0 +1,130 @@ +// RUN: llvm-mc -triple amdgcn-amd-amdhsa < %s | FileCheck --check-prefix=ASM %s +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -filetype=obj < %s > %t +// RUN: llvm-objdump --syms %t | FileCheck --check-prefix=OBJDUMP %s + +// OBJDUMP: SYMBOL TABLE: +// OBJDUMP-NEXT: 0000000000000000 l *ABS* 0000000000000000 zero +// OBJDUMP-NEXT: 0000000000000001 l *ABS* 0000000000000000 one +// OBJDUMP-NEXT: 0000000000000002 l *ABS* 0000000000000000 two +// OBJDUMP-NEXT: 0000000000000003 l *ABS* 0000000000000000 three +// OBJDUMP-NEXT: 7fffffffffffffff l *ABS* 0000000000000000 i64_max +// OBJDUMP-NEXT: 8000000000000000 l *ABS* 0000000000000000 i64_min +// OBJDUMP-NEXT: 0000000000000005 l *ABS* 0000000000000000 max_expression_all +// OBJDUMP-NEXT: 0000000000000005 l *ABS* 0000000000000000 five +// OBJDUMP-NEXT: 0000000000000004 l *ABS* 0000000000000000 four +// OBJDUMP-NEXT: 0000000000000002 l *ABS* 0000000000000000 max_expression_two +// OBJDUMP-NEXT: 0000000000000001 l *ABS* 0000000000000000 max_expression_one +// OBJDUMP-NEXT: 000000000000000a l *ABS* 0000000000000000 max_literals +// OBJDUMP-NEXT: 000000000000000f l *ABS* 0000000000000000 max_with_max_sym +// OBJDUMP-NEXT: 000000000000000f l *ABS* 0000000000000000 max +// OBJDUMP-NEXT: ffffffffffffffff l *ABS* 0000000000000000 neg_one +// OBJDUMP-NEXT: ffffffffffffffff l *ABS* 0000000000000000 max_neg_numbers +// OBJDUMP-NEXT: ffffffffffffffff l *ABS* 0000000000000000 max_neg_number +// OBJDUMP-NEXT: 0000000000000003 l *ABS* 0000000000000000 max_with_subexpr +// OBJDUMP-NEXT: 0000000000000006 l *ABS* 0000000000000000 max_as_subexpr +// OBJDUMP-NEXT: 0000000000000005 l *ABS* 0000000000000000 max_recursive_subexpr +// OBJDUMP-NEXT: 7fffffffffffffff l *ABS* 0000000000000000 max_expr_one_max +// OBJDUMP-NEXT: 7fffffffffffffff l *ABS* 0000000000000000 max_expr_two_max +// OBJDUMP-NEXT: 7fffffffffffffff l *ABS* 0000000000000000 max_expr_three_max +// OBJDUMP-NEXT: 8000000000000000 l *ABS* 0000000000000000 max_expr_one_min +// OBJDUMP-NEXT: 0000000000000003 l *ABS* 0000000000000000 max_expr_two_min +// OBJDUMP-NEXT: 0000000000989680 l *ABS* 0000000000000000 max_expr_three_min +// OBJDUMP-NEXT: 0000000000000007 l *ABS* 0000000000000000 or_expression_all +// OBJDUMP-NEXT: 0000000000000003 l *ABS* 0000000000000000 or_expression_two +// OBJDUMP-NEXT: 0000000000000001 l *ABS* 0000000000000000 or_expression_one +// OBJDUMP-NEXT: 000000000000000f l *ABS* 0000000000000000 or_literals +// OBJDUMP-NEXT: 0000000000000000 l *ABS* 0000000000000000 or_false +// OBJDUMP-NEXT: 00000000000000ff l *ABS* 0000000000000000 or_with_or_sym +// OBJDUMP-NEXT: 00000000000000ff l *ABS* 0000000000000000 or +// OBJDUMP-NEXT: 0000000000000003 l *ABS* 0000000000000000 or_with_subexpr +// OBJDUMP-NEXT: 0000000000000008 l *ABS* 0000000000000000 or_as_subexpr +// OBJDUMP-NEXT: 0000000000000007 l *ABS* 0000000000000000 or_recursive_subexpr + +// ASM: .set zero, 0 +// ASM: .set one, 1 +// ASM: .set two, 2 +// ASM: .set three, 3 +// ASM: .set i64_max, 9223372036854775807 +// ASM: .set i64_min, -9223372036854775808 + +.set zero, 0 +.set one, 1 +.set two, 2 +.set three, 3 +.set i64_max, 0x7FFFFFFFFFFFFFFF +.set i64_min, 0x8000000000000000 + +// ASM: .set max_expression_all, max(1, 2, five, 3, four) +// ASM: .set max_expression_two, 2 +// ASM: .set max_expression_one, 1 +// ASM: .set max_literals, 10 +// ASM: .set max_with_max_sym, max(max, 4, 3, 1, 2) + +.set max_expression_all, max(one, two, five, three, four) +.set max_expression_two, max(one, two) +.set max_expression_one, max(one) +.set max_literals, max(1,2,3,4,5,6,7,8,9,10) +.set max_with_max_sym, max(max, 4, 3, one, two) + +// ASM: .set max_neg_numbers, -1 +// ASM: .set max_neg_number, -1 + +.set neg_one, -1 +.set max_neg_numbers, max(-5, -4, -3, -2, neg_one) +.set max_neg_number, max(neg_one) + +// ASM: .set max_with_subexpr, 3 +// ASM: .set max_as_subexpr, 1+(max(4, 3, five)) +// ASM: .set max_recursive_subexpr, max(max(1, four), 3, max_expression_all) + +.set max_with_subexpr, max(((one | 3) << 3) / 8) +.set max_as_subexpr, 1 + max(4, 3, five) +.set max_recursive_subexpr, max(max(one, four), three, max_expression_all) + +// ASM: .set max_expr_one_max, 9223372036854775807 +// ASM: .set max_expr_two_max, max(9223372036854775807, five) +// ASM: .set max_expr_three_max, max(9223372036854775807, five, 10000000) + +.set max_expr_one_max, max(i64_max) +.set max_expr_two_max, max(i64_max, five) +.set max_expr_three_max, max(i64_max, five, 10000000) + +// ASM: .set max_expr_one_min, -9223372036854775808 +// ASM: .set max_expr_two_min, 3 +// ASM: .set max_expr_three_min, 10000000 + +.set max_expr_one_min, max(i64_min) +.set max_expr_two_min, max(i64_min, three) +.set max_expr_three_min, max(i64_min, three, 10000000) + +// ASM: .set or_expression_all, or(1, 2, five, 3, four) +// ASM: .set or_expression_two, 3 +// ASM: .set or_expression_one, 1 +// ASM: .set or_literals, 15 +// ASM: .set or_false, 0 +// ASM: .set or_with_or_sym, or(or, 4, 3, 1, 2) + +.set or_expression_all, or(one, two, five, three, four) +.set or_expression_two, or(one, two) +.set or_expression_one, or(one) +.set or_literals, or(1,2,3,4,5,6,7,8,9,10) +.set or_false, or(zero, 0, (2-2), 5 > 6) +.set or_with_or_sym, or(or, 4, 3, one, two) + +// ASM: .set or_with_subexpr, 3 +// ASM: .set or_as_subexpr, 1+(or(4, 3, five)) +// ASM: .set or_recursive_subexpr, or(or(1, four), 3, or_expression_all) + +.set or_with_subexpr, or(((one | 3) << 3) / 8) +.set or_as_subexpr, 1 + or(4, 3, five) +.set or_recursive_subexpr, or(or(one, four), three, or_expression_all) + +// ASM: .set four, 4 +// ASM: .set five, 5 +// ASM: .set max, 15 +// ASM: .set or, 255 + +.set four, 4 +.set five, 5 +.set max, 0xF +.set or, 0xFF diff --git a/llvm/test/MC/AMDGPU/mcexpr_amd_err.s b/llvm/test/MC/AMDGPU/mcexpr_amd_err.s new file mode 100644 index 000000000000..ea02e0136272 --- /dev/null +++ b/llvm/test/MC/AMDGPU/mcexpr_amd_err.s @@ -0,0 +1,53 @@ +// RUN: not llvm-mc -triple amdgcn-amd-amdhsa %s 2>&1 | FileCheck --check-prefix=ASM %s + +.set one, 1 +.set two, 2 +.set three, 3 + +.set max_empty, max() +// ASM: :[[@LINE-1]]:{{[0-9]+}}: error: empty max expression +// ASM: :[[@LINE-2]]:{{[0-9]+}}: error: missing expression + +.set or_empty, or() +// ASM: :[[@LINE-1]]:{{[0-9]+}}: error: empty or expression +// ASM: :[[@LINE-2]]:{{[0-9]+}}: error: missing expression + +.set max_post_aux_comma, max(one,) +// ASM: :[[@LINE-1]]:{{[0-9]+}}: error: mismatch of commas in max expression +// ASM: :[[@LINE-2]]:{{[0-9]+}}: error: missing expression + +.set max_pre_aux_comma, max(,one) +// asm: :[[@line-1]]:{{[0-9]+}}: error: unknown token in expression +// ASM: :[[@LINE-2]]:{{[0-9]+}}: error: missing expression + +.set max_double_comma, max(one,, two) +// ASM: :[[@LINE-1]]:{{[0-9]+}}: error: unknown token in expression +// ASM: :[[@LINE-2]]:{{[0-9]+}}: error: missing expression + +.set max_no_comma, max(one two) +// ASM: :[[@LINE-1]]:{{[0-9]+}}: error: unexpected token in max expression +// ASM: :[[@LINE-2]]:{{[0-9]+}}: error: missing expression + +.set max_missing_paren, max(two +// ASM: :[[@LINE-1]]:{{[0-9]+}}: error: unexpected token in max expression +// ASM: :[[@LINE-2]]:{{[0-9]+}}: error: missing expression + +.set max_expression_one, max(three, four, +// ASM: :[[@LINE-1]]:{{[0-9]+}}: error: unknown token in expression +// ASM: :[[@LINE-2]]:{{[0-9]+}}: error: missing expression + +.set or_expression_one, or(four, five +// ASM: :[[@LINE-1]]:{{[0-9]+}}: error: unexpected token in or expression +// ASM: :[[@LINE-2]]:{{[0-9]+}}: error: missing expression + +.set max_no_lparen, max four, five) +// ASM: :[[@LINE-1]]:{{[0-9]+}}: error: expected newline + +.set max_no_paren, max one, two, three +// ASM: :[[@LINE-1]]:{{[0-9]+}}: error: expected newline + +.set max_rparen_only, max) +// ASM: :[[@LINE-1]]:{{[0-9]+}}: error: expected newline + +.set four, 4 +.set five, 5 -- GitLab From 3b1512c477694c5ee4fb20fae416e834bf42518e Mon Sep 17 00:00:00 2001 From: Joe Nash Date: Wed, 6 Mar 2024 10:45:48 -0500 Subject: [PATCH 358/929] [AMDGPU] Make gfx11 vop2 disassembler tests use strict-whitespace NFC. Adds -strict-whitespace to RUN lines and adjusts CHECK line space padding accordingly. See also (#84078) --- .../Disassembler/AMDGPU/gfx11_dasm_vop2.txt | 460 +++++++++--------- .../AMDGPU/gfx11_dasm_vop3_from_vop2.txt | 36 +- 2 files changed, 248 insertions(+), 248 deletions(-) diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2.txt index d12125ed9f46..8cb47b0a596d 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop2.txt @@ -1,6 +1,6 @@ -# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W32,GFX11-FAKE16 %s -# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W32,GFX11-REAL16 %s -# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-WavefrontSize32,+WavefrontSize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W64 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefixes=GFX11,W32,GFX11-FAKE16 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefixes=GFX11,W32,GFX11-REAL16 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-WavefrontSize32,+WavefrontSize64 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefixes=GFX11,W64 %s # W32: v_add_co_ci_u32_e32 v5, vcc_lo, v1, v2, vcc_lo ; encoding: [0x01,0x05,0x0a,0x40] # W64: v_add_co_ci_u32_e32 v5, vcc, v1, v2, vcc ; encoding: [0x01,0x05,0x0a,0x40] @@ -62,79 +62,79 @@ # W64: v_add_co_ci_u32_e32 v255, vcc, 0xaf123456, v255, vcc ; encoding: [0xff,0xfe,0xff,0x41,0x56,0x34,0x12,0xaf] 0xff,0xfe,0xff,0x41,0x56,0x34,0x12,0xaf -# GFX11-REAL16: v_add_f16_e32 v5.l, v1.l, v2.l ; encoding: [0x01,0x05,0x0a,0x64] -# GFX11-FAKE16: v_add_f16_e32 v5, v1, v2 ; encoding: [0x01,0x05,0x0a,0x64] +# GFX11-REAL16: v_add_f16_e32 v5.l, v1.l, v2.l ; encoding: [0x01,0x05,0x0a,0x64] +# GFX11-FAKE16: v_add_f16_e32 v5, v1, v2 ; encoding: [0x01,0x05,0x0a,0x64] 0x01,0x05,0x0a,0x64 -# GFX11-REAL16: v_add_f16_e32 v5.l, v1.h, v2.l ; encoding: [0x81,0x05,0x0a,0x64] +# GFX11-REAL16: v_add_f16_e32 v5.l, v1.h, v2.l ; encoding: [0x81,0x05,0x0a,0x64] # GFX11-FAKE16: v_add_f16_e32 v5, v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x0a,0x64] 0x81,0x05,0x0a,0x64 -# GFX11-REAL16: v_add_f16_e32 v5.l, v127.l, v2.l ; encoding: [0x7f,0x05,0x0a,0x64] -# GFX11-FAKE16: v_add_f16_e32 v5, v127, v2 ; encoding: [0x7f,0x05,0x0a,0x64] +# GFX11-REAL16: v_add_f16_e32 v5.l, v127.l, v2.l ; encoding: [0x7f,0x05,0x0a,0x64] +# GFX11-FAKE16: v_add_f16_e32 v5, v127, v2 ; encoding: [0x7f,0x05,0x0a,0x64] 0x7f,0x05,0x0a,0x64 -# GFX11-REAL16: v_add_f16_e32 v5.l, v127.h, v2.l ; encoding: [0xff,0x05,0x0a,0x64] +# GFX11-REAL16: v_add_f16_e32 v5.l, v127.h, v2.l ; encoding: [0xff,0x05,0x0a,0x64] # GFX11-FAKE16: v_add_f16_e32 v5, v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x0a,0x64] 0xff,0x05,0x0a,0x64 -# GFX11-REAL16: v_add_f16_e32 v5.l, s1, v2.l ; encoding: [0x01,0x04,0x0a,0x64] -# GFX11-FAKE16: v_add_f16_e32 v5, s1, v2 ; encoding: [0x01,0x04,0x0a,0x64] +# GFX11-REAL16: v_add_f16_e32 v5.l, s1, v2.l ; encoding: [0x01,0x04,0x0a,0x64] +# GFX11-FAKE16: v_add_f16_e32 v5, s1, v2 ; encoding: [0x01,0x04,0x0a,0x64] 0x01,0x04,0x0a,0x64 -# GFX11-REAL16: v_add_f16_e32 v5.l, s105, v2.l ; encoding: [0x69,0x04,0x0a,0x64] -# GFX11-FAKE16: v_add_f16_e32 v5, s105, v2 ; encoding: [0x69,0x04,0x0a,0x64] +# GFX11-REAL16: v_add_f16_e32 v5.l, s105, v2.l ; encoding: [0x69,0x04,0x0a,0x64] +# GFX11-FAKE16: v_add_f16_e32 v5, s105, v2 ; encoding: [0x69,0x04,0x0a,0x64] 0x69,0x04,0x0a,0x64 -# GFX11-REAL16: v_add_f16_e32 v5.l, vcc_lo, v2.l ; encoding: [0x6a,0x04,0x0a,0x64] -# GFX11-FAKE16: v_add_f16_e32 v5, vcc_lo, v2 ; encoding: [0x6a,0x04,0x0a,0x64] +# GFX11-REAL16: v_add_f16_e32 v5.l, vcc_lo, v2.l ; encoding: [0x6a,0x04,0x0a,0x64] +# GFX11-FAKE16: v_add_f16_e32 v5, vcc_lo, v2 ; encoding: [0x6a,0x04,0x0a,0x64] 0x6a,0x04,0x0a,0x64 -# GFX11-REAL16: v_add_f16_e32 v5.l, vcc_hi, v2.l ; encoding: [0x6b,0x04,0x0a,0x64] -# GFX11-FAKE16: v_add_f16_e32 v5, vcc_hi, v2 ; encoding: [0x6b,0x04,0x0a,0x64] +# GFX11-REAL16: v_add_f16_e32 v5.l, vcc_hi, v2.l ; encoding: [0x6b,0x04,0x0a,0x64] +# GFX11-FAKE16: v_add_f16_e32 v5, vcc_hi, v2 ; encoding: [0x6b,0x04,0x0a,0x64] 0x6b,0x04,0x0a,0x64 -# GFX11-REAL16: v_add_f16_e32 v5.l, ttmp15, v2.l ; encoding: [0x7b,0x04,0x0a,0x64] -# GFX11-FAKE16: v_add_f16_e32 v5, ttmp15, v2 ; encoding: [0x7b,0x04,0x0a,0x64] +# GFX11-REAL16: v_add_f16_e32 v5.l, ttmp15, v2.l ; encoding: [0x7b,0x04,0x0a,0x64] +# GFX11-FAKE16: v_add_f16_e32 v5, ttmp15, v2 ; encoding: [0x7b,0x04,0x0a,0x64] 0x7b,0x04,0x0a,0x64 -# GFX11-REAL16: v_add_f16_e32 v5.l, m0, v2.l ; encoding: [0x7d,0x04,0x0a,0x64] -# GFX11-FAKE16: v_add_f16_e32 v5, m0, v2 ; encoding: [0x7d,0x04,0x0a,0x64] +# GFX11-REAL16: v_add_f16_e32 v5.l, m0, v2.l ; encoding: [0x7d,0x04,0x0a,0x64] +# GFX11-FAKE16: v_add_f16_e32 v5, m0, v2 ; encoding: [0x7d,0x04,0x0a,0x64] 0x7d,0x04,0x0a,0x64 -# GFX11-REAL16: v_add_f16_e32 v5.l, exec_lo, v2.l ; encoding: [0x7e,0x04,0x0a,0x64] -# GFX11-FAKE16: v_add_f16_e32 v5, exec_lo, v2 ; encoding: [0x7e,0x04,0x0a,0x64] +# GFX11-REAL16: v_add_f16_e32 v5.l, exec_lo, v2.l ; encoding: [0x7e,0x04,0x0a,0x64] +# GFX11-FAKE16: v_add_f16_e32 v5, exec_lo, v2 ; encoding: [0x7e,0x04,0x0a,0x64] 0x7e,0x04,0x0a,0x64 -# GFX11-REAL16: v_add_f16_e32 v5.l, exec_hi, v2.l ; encoding: [0x7f,0x04,0x0a,0x64] -# GFX11-FAKE16: v_add_f16_e32 v5, exec_hi, v2 ; encoding: [0x7f,0x04,0x0a,0x64] +# GFX11-REAL16: v_add_f16_e32 v5.l, exec_hi, v2.l ; encoding: [0x7f,0x04,0x0a,0x64] +# GFX11-FAKE16: v_add_f16_e32 v5, exec_hi, v2 ; encoding: [0x7f,0x04,0x0a,0x64] 0x7f,0x04,0x0a,0x64 -# GFX11-REAL16: v_add_f16_e32 v5.l, null, v2.l ; encoding: [0x7c,0x04,0x0a,0x64] -# GFX11-FAKE16: v_add_f16_e32 v5, null, v2 ; encoding: [0x7c,0x04,0x0a,0x64] +# GFX11-REAL16: v_add_f16_e32 v5.l, null, v2.l ; encoding: [0x7c,0x04,0x0a,0x64] +# GFX11-FAKE16: v_add_f16_e32 v5, null, v2 ; encoding: [0x7c,0x04,0x0a,0x64] 0x7c,0x04,0x0a,0x64 -# GFX11-REAL16: v_add_f16_e32 v5.l, -1, v2.l ; encoding: [0xc1,0x04,0x0a,0x64] -# GFX11-FAKE16: v_add_f16_e32 v5, -1, v2 ; encoding: [0xc1,0x04,0x0a,0x64] +# GFX11-REAL16: v_add_f16_e32 v5.l, -1, v2.l ; encoding: [0xc1,0x04,0x0a,0x64] +# GFX11-FAKE16: v_add_f16_e32 v5, -1, v2 ; encoding: [0xc1,0x04,0x0a,0x64] 0xc1,0x04,0x0a,0x64 -# GFX11-REAL16: v_add_f16_e32 v5.l, 0.5, v2.l ; encoding: [0xf0,0x04,0x0a,0x64] -# GFX11-FAKE16: v_add_f16_e32 v5, 0.5, v2 ; encoding: [0xf0,0x04,0x0a,0x64] +# GFX11-REAL16: v_add_f16_e32 v5.l, 0.5, v2.l ; encoding: [0xf0,0x04,0x0a,0x64] +# GFX11-FAKE16: v_add_f16_e32 v5, 0.5, v2 ; encoding: [0xf0,0x04,0x0a,0x64] 0xf0,0x04,0x0a,0x64 -# GFX11-REAL16: v_add_f16_e32 v5.l, src_scc, v2.l ; encoding: [0xfd,0x04,0x0a,0x64] -# GFX11-FAKE16: v_add_f16_e32 v5, src_scc, v2 ; encoding: [0xfd,0x04,0x0a,0x64] +# GFX11-REAL16: v_add_f16_e32 v5.l, src_scc, v2.l ; encoding: [0xfd,0x04,0x0a,0x64] +# GFX11-FAKE16: v_add_f16_e32 v5, src_scc, v2 ; encoding: [0xfd,0x04,0x0a,0x64] 0xfd,0x04,0x0a,0x64 -# GFX11-REAL16: v_add_f16_e32 v5.h, src_scc, v2.h ; encoding: [0xfd,0x04,0x0b,0x65] +# GFX11-REAL16: v_add_f16_e32 v5.h, src_scc, v2.h ; encoding: [0xfd,0x04,0x0b,0x65] # COM: TODO: GFX11-FAKE16: warning: invalid instruction encoding 0xfd,0x04,0x0b,0x65 0xfd,0x04,0x0b,0x65 -# GFX11-REAL16: v_add_f16_e32 v127.l, 0xfe0b, v127.l ; encoding: [0xff,0xfe,0xfe,0x64,0x0b,0xfe,0x00,0x00] -# GFX11-FAKE16: v_add_f16_e32 v127, 0xfe0b, v127 ; encoding: [0xff,0xfe,0xfe,0x64,0x0b,0xfe,0x00,0x00] +# GFX11-REAL16: v_add_f16_e32 v127.l, 0xfe0b, v127.l ; encoding: [0xff,0xfe,0xfe,0x64,0x0b,0xfe,0x00,0x00] +# GFX11-FAKE16: v_add_f16_e32 v127, 0xfe0b, v127 ; encoding: [0xff,0xfe,0xfe,0x64,0x0b,0xfe,0x00,0x00] 0xff,0xfe,0xfe,0x64,0x0b,0xfe,0x00,0x00 -# GFX11-REAL16: v_add_f16_e32 v127.h, 0xfe0b, v127.h ; encoding: [0xff,0xfe,0xff,0x65,0x0b,0xfe,0x00,0x00] +# GFX11-REAL16: v_add_f16_e32 v127.h, 0xfe0b, v127.h ; encoding: [0xff,0xfe,0xff,0x65,0x0b,0xfe,0x00,0x00] # COM: TODO: GFX11-FAKE16: warning: invalid instruction encoding 0xff,0xfe,0xff,0x65,0x0b,0xfe,0x00,0x00 0xff,0xfe,0xff,0x65,0x0b,0xfe,0x00,0x00 @@ -318,60 +318,60 @@ # GFX11: v_ashrrev_i32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x35,0x56,0x34,0x12,0xaf] 0xff,0xfe,0xff,0x35,0x56,0x34,0x12,0xaf -# W32: v_cndmask_b32_e32 v5, v1, v2, vcc_lo ; encoding: [0x01,0x05,0x0a,0x02] -# W64: v_cndmask_b32_e32 v5, v1, v2, vcc ; encoding: [0x01,0x05,0x0a,0x02] +# W32: v_cndmask_b32_e32 v5, v1, v2, vcc_lo ; encoding: [0x01,0x05,0x0a,0x02] +# W64: v_cndmask_b32_e32 v5, v1, v2, vcc ; encoding: [0x01,0x05,0x0a,0x02] 0x01,0x05,0x0a,0x02 -# W32: v_cndmask_b32_e32 v5, v255, v2, vcc_lo ; encoding: [0xff,0x05,0x0a,0x02] -# W64: v_cndmask_b32_e32 v5, v255, v2, vcc ; encoding: [0xff,0x05,0x0a,0x02] +# W32: v_cndmask_b32_e32 v5, v255, v2, vcc_lo ; encoding: [0xff,0x05,0x0a,0x02] +# W64: v_cndmask_b32_e32 v5, v255, v2, vcc ; encoding: [0xff,0x05,0x0a,0x02] 0xff,0x05,0x0a,0x02 -# W32: v_cndmask_b32_e32 v5, s1, v2, vcc_lo ; encoding: [0x01,0x04,0x0a,0x02] -# W64: v_cndmask_b32_e32 v5, s1, v2, vcc ; encoding: [0x01,0x04,0x0a,0x02] +# W32: v_cndmask_b32_e32 v5, s1, v2, vcc_lo ; encoding: [0x01,0x04,0x0a,0x02] +# W64: v_cndmask_b32_e32 v5, s1, v2, vcc ; encoding: [0x01,0x04,0x0a,0x02] 0x01,0x04,0x0a,0x02 -# W32: v_cndmask_b32_e32 v5, s105, v2, vcc_lo ; encoding: [0x69,0x04,0x0a,0x02] -# W64: v_cndmask_b32_e32 v5, s105, v2, vcc ; encoding: [0x69,0x04,0x0a,0x02] +# W32: v_cndmask_b32_e32 v5, s105, v2, vcc_lo ; encoding: [0x69,0x04,0x0a,0x02] +# W64: v_cndmask_b32_e32 v5, s105, v2, vcc ; encoding: [0x69,0x04,0x0a,0x02] 0x69,0x04,0x0a,0x02 -# W32: v_cndmask_b32_e32 v5, vcc_lo, v2, vcc_lo ; encoding: [0x6a,0x04,0x0a,0x02] -# W64: v_cndmask_b32_e32 v5, vcc_lo, v2, vcc ; encoding: [0x6a,0x04,0x0a,0x02] +# W32: v_cndmask_b32_e32 v5, vcc_lo, v2, vcc_lo ; encoding: [0x6a,0x04,0x0a,0x02] +# W64: v_cndmask_b32_e32 v5, vcc_lo, v2, vcc ; encoding: [0x6a,0x04,0x0a,0x02] 0x6a,0x04,0x0a,0x02 -# W32: v_cndmask_b32_e32 v5, vcc_hi, v2, vcc_lo ; encoding: [0x6b,0x04,0x0a,0x02] -# W64: v_cndmask_b32_e32 v5, vcc_hi, v2, vcc ; encoding: [0x6b,0x04,0x0a,0x02] +# W32: v_cndmask_b32_e32 v5, vcc_hi, v2, vcc_lo ; encoding: [0x6b,0x04,0x0a,0x02] +# W64: v_cndmask_b32_e32 v5, vcc_hi, v2, vcc ; encoding: [0x6b,0x04,0x0a,0x02] 0x6b,0x04,0x0a,0x02 -# W32: v_cndmask_b32_e32 v5, ttmp15, v2, vcc_lo ; encoding: [0x7b,0x04,0x0a,0x02] -# W64: v_cndmask_b32_e32 v5, ttmp15, v2, vcc ; encoding: [0x7b,0x04,0x0a,0x02] +# W32: v_cndmask_b32_e32 v5, ttmp15, v2, vcc_lo ; encoding: [0x7b,0x04,0x0a,0x02] +# W64: v_cndmask_b32_e32 v5, ttmp15, v2, vcc ; encoding: [0x7b,0x04,0x0a,0x02] 0x7b,0x04,0x0a,0x02 -# W32: v_cndmask_b32_e32 v5, m0, v2, vcc_lo ; encoding: [0x7d,0x04,0x0a,0x02] -# W64: v_cndmask_b32_e32 v5, m0, v2, vcc ; encoding: [0x7d,0x04,0x0a,0x02] +# W32: v_cndmask_b32_e32 v5, m0, v2, vcc_lo ; encoding: [0x7d,0x04,0x0a,0x02] +# W64: v_cndmask_b32_e32 v5, m0, v2, vcc ; encoding: [0x7d,0x04,0x0a,0x02] 0x7d,0x04,0x0a,0x02 # W32: v_cndmask_b32_e32 v5, exec_lo, v2, vcc_lo ; encoding: [0x7e,0x04,0x0a,0x02] -# W64: v_cndmask_b32_e32 v5, exec_lo, v2, vcc ; encoding: [0x7e,0x04,0x0a,0x02] +# W64: v_cndmask_b32_e32 v5, exec_lo, v2, vcc ; encoding: [0x7e,0x04,0x0a,0x02] 0x7e,0x04,0x0a,0x02 # W32: v_cndmask_b32_e32 v5, exec_hi, v2, vcc_lo ; encoding: [0x7f,0x04,0x0a,0x02] -# W64: v_cndmask_b32_e32 v5, exec_hi, v2, vcc ; encoding: [0x7f,0x04,0x0a,0x02] +# W64: v_cndmask_b32_e32 v5, exec_hi, v2, vcc ; encoding: [0x7f,0x04,0x0a,0x02] 0x7f,0x04,0x0a,0x02 -# W32: v_cndmask_b32_e32 v5, null, v2, vcc_lo ; encoding: [0x7c,0x04,0x0a,0x02] -# W64: v_cndmask_b32_e32 v5, null, v2, vcc ; encoding: [0x7c,0x04,0x0a,0x02] +# W32: v_cndmask_b32_e32 v5, null, v2, vcc_lo ; encoding: [0x7c,0x04,0x0a,0x02] +# W64: v_cndmask_b32_e32 v5, null, v2, vcc ; encoding: [0x7c,0x04,0x0a,0x02] 0x7c,0x04,0x0a,0x02 -# W32: v_cndmask_b32_e32 v5, -1, v2, vcc_lo ; encoding: [0xc1,0x04,0x0a,0x02] -# W64: v_cndmask_b32_e32 v5, -1, v2, vcc ; encoding: [0xc1,0x04,0x0a,0x02] +# W32: v_cndmask_b32_e32 v5, -1, v2, vcc_lo ; encoding: [0xc1,0x04,0x0a,0x02] +# W64: v_cndmask_b32_e32 v5, -1, v2, vcc ; encoding: [0xc1,0x04,0x0a,0x02] 0xc1,0x04,0x0a,0x02 -# W32: v_cndmask_b32_e32 v5, 0.5, v2, vcc_lo ; encoding: [0xf0,0x04,0x0a,0x02] -# W64: v_cndmask_b32_e32 v5, 0.5, v2, vcc ; encoding: [0xf0,0x04,0x0a,0x02] +# W32: v_cndmask_b32_e32 v5, 0.5, v2, vcc_lo ; encoding: [0xf0,0x04,0x0a,0x02] +# W64: v_cndmask_b32_e32 v5, 0.5, v2, vcc ; encoding: [0xf0,0x04,0x0a,0x02] 0xf0,0x04,0x0a,0x02 # W32: v_cndmask_b32_e32 v5, src_scc, v2, vcc_lo ; encoding: [0xfd,0x04,0x0a,0x02] -# W64: v_cndmask_b32_e32 v5, src_scc, v2, vcc ; encoding: [0xfd,0x04,0x0a,0x02] +# W64: v_cndmask_b32_e32 v5, src_scc, v2, vcc ; encoding: [0xfd,0x04,0x0a,0x02] 0xfd,0x04,0x0a,0x02 # W32: v_cndmask_b32_e32 v255, 0xaf123456, v255, vcc_lo ; encoding: [0xff,0xfe,0xff,0x03,0x56,0x34,0x12,0xaf] @@ -918,79 +918,79 @@ # GFX11: v_lshrrev_b32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x33,0x56,0x34,0x12,0xaf] 0xff,0xfe,0xff,0x33,0x56,0x34,0x12,0xaf -# GFX11-REAL16: v_max_f16_e32 v5.l, v1.l, v2.l ; encoding: [0x01,0x05,0x0a,0x72] -# GFX11-FAKE16: v_max_f16_e32 v5, v1, v2 ; encoding: [0x01,0x05,0x0a,0x72] +# GFX11-REAL16: v_max_f16_e32 v5.l, v1.l, v2.l ; encoding: [0x01,0x05,0x0a,0x72] +# GFX11-FAKE16: v_max_f16_e32 v5, v1, v2 ; encoding: [0x01,0x05,0x0a,0x72] 0x01,0x05,0x0a,0x72 -# GFX11-REAL16: v_max_f16_e32 v5.l, v1.h, v2.l ; encoding: [0x81,0x05,0x0a,0x72] +# GFX11-REAL16: v_max_f16_e32 v5.l, v1.h, v2.l ; encoding: [0x81,0x05,0x0a,0x72] # GFX11-FAKE16: v_max_f16_e32 v5, v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x0a,0x72] 0x81,0x05,0x0a,0x72 -# GFX11-REAL16: v_max_f16_e32 v5.l, v127.l, v2.l ; encoding: [0x7f,0x05,0x0a,0x72] -# GFX11-FAKE16: v_max_f16_e32 v5, v127, v2 ; encoding: [0x7f,0x05,0x0a,0x72] +# GFX11-REAL16: v_max_f16_e32 v5.l, v127.l, v2.l ; encoding: [0x7f,0x05,0x0a,0x72] +# GFX11-FAKE16: v_max_f16_e32 v5, v127, v2 ; encoding: [0x7f,0x05,0x0a,0x72] 0x7f,0x05,0x0a,0x72 -# GFX11-REAL16: v_max_f16_e32 v5.l, v127.h, v2.l ; encoding: [0xff,0x05,0x0a,0x72] +# GFX11-REAL16: v_max_f16_e32 v5.l, v127.h, v2.l ; encoding: [0xff,0x05,0x0a,0x72] # GFX11-FAKE16: v_max_f16_e32 v5, v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x0a,0x72] 0xff,0x05,0x0a,0x72 -# GFX11-REAL16: v_max_f16_e32 v5.l, s1, v2.l ; encoding: [0x01,0x04,0x0a,0x72] -# GFX11-FAKE16: v_max_f16_e32 v5, s1, v2 ; encoding: [0x01,0x04,0x0a,0x72] +# GFX11-REAL16: v_max_f16_e32 v5.l, s1, v2.l ; encoding: [0x01,0x04,0x0a,0x72] +# GFX11-FAKE16: v_max_f16_e32 v5, s1, v2 ; encoding: [0x01,0x04,0x0a,0x72] 0x01,0x04,0x0a,0x72 -# GFX11-REAL16: v_max_f16_e32 v5.l, s105, v2.l ; encoding: [0x69,0x04,0x0a,0x72] -# GFX11-FAKE16: v_max_f16_e32 v5, s105, v2 ; encoding: [0x69,0x04,0x0a,0x72] +# GFX11-REAL16: v_max_f16_e32 v5.l, s105, v2.l ; encoding: [0x69,0x04,0x0a,0x72] +# GFX11-FAKE16: v_max_f16_e32 v5, s105, v2 ; encoding: [0x69,0x04,0x0a,0x72] 0x69,0x04,0x0a,0x72 -# GFX11-REAL16: v_max_f16_e32 v5.l, vcc_lo, v2.l ; encoding: [0x6a,0x04,0x0a,0x72] -# GFX11-FAKE16: v_max_f16_e32 v5, vcc_lo, v2 ; encoding: [0x6a,0x04,0x0a,0x72] +# GFX11-REAL16: v_max_f16_e32 v5.l, vcc_lo, v2.l ; encoding: [0x6a,0x04,0x0a,0x72] +# GFX11-FAKE16: v_max_f16_e32 v5, vcc_lo, v2 ; encoding: [0x6a,0x04,0x0a,0x72] 0x6a,0x04,0x0a,0x72 -# GFX11-REAL16: v_max_f16_e32 v5.l, vcc_hi, v2.l ; encoding: [0x6b,0x04,0x0a,0x72] -# GFX11-FAKE16: v_max_f16_e32 v5, vcc_hi, v2 ; encoding: [0x6b,0x04,0x0a,0x72] +# GFX11-REAL16: v_max_f16_e32 v5.l, vcc_hi, v2.l ; encoding: [0x6b,0x04,0x0a,0x72] +# GFX11-FAKE16: v_max_f16_e32 v5, vcc_hi, v2 ; encoding: [0x6b,0x04,0x0a,0x72] 0x6b,0x04,0x0a,0x72 -# GFX11-REAL16: v_max_f16_e32 v5.l, ttmp15, v2.l ; encoding: [0x7b,0x04,0x0a,0x72] -# GFX11-FAKE16: v_max_f16_e32 v5, ttmp15, v2 ; encoding: [0x7b,0x04,0x0a,0x72] +# GFX11-REAL16: v_max_f16_e32 v5.l, ttmp15, v2.l ; encoding: [0x7b,0x04,0x0a,0x72] +# GFX11-FAKE16: v_max_f16_e32 v5, ttmp15, v2 ; encoding: [0x7b,0x04,0x0a,0x72] 0x7b,0x04,0x0a,0x72 -# GFX11-REAL16: v_max_f16_e32 v5.l, m0, v2.l ; encoding: [0x7d,0x04,0x0a,0x72] -# GFX11-FAKE16: v_max_f16_e32 v5, m0, v2 ; encoding: [0x7d,0x04,0x0a,0x72] +# GFX11-REAL16: v_max_f16_e32 v5.l, m0, v2.l ; encoding: [0x7d,0x04,0x0a,0x72] +# GFX11-FAKE16: v_max_f16_e32 v5, m0, v2 ; encoding: [0x7d,0x04,0x0a,0x72] 0x7d,0x04,0x0a,0x72 -# GFX11-REAL16: v_max_f16_e32 v5.l, exec_lo, v2.l ; encoding: [0x7e,0x04,0x0a,0x72] -# GFX11-FAKE16: v_max_f16_e32 v5, exec_lo, v2 ; encoding: [0x7e,0x04,0x0a,0x72] +# GFX11-REAL16: v_max_f16_e32 v5.l, exec_lo, v2.l ; encoding: [0x7e,0x04,0x0a,0x72] +# GFX11-FAKE16: v_max_f16_e32 v5, exec_lo, v2 ; encoding: [0x7e,0x04,0x0a,0x72] 0x7e,0x04,0x0a,0x72 -# GFX11-REAL16: v_max_f16_e32 v5.l, exec_hi, v2.l ; encoding: [0x7f,0x04,0x0a,0x72] -# GFX11-FAKE16: v_max_f16_e32 v5, exec_hi, v2 ; encoding: [0x7f,0x04,0x0a,0x72] +# GFX11-REAL16: v_max_f16_e32 v5.l, exec_hi, v2.l ; encoding: [0x7f,0x04,0x0a,0x72] +# GFX11-FAKE16: v_max_f16_e32 v5, exec_hi, v2 ; encoding: [0x7f,0x04,0x0a,0x72] 0x7f,0x04,0x0a,0x72 -# GFX11-REAL16: v_max_f16_e32 v5.l, null, v2.l ; encoding: [0x7c,0x04,0x0a,0x72] -# GFX11-FAKE16: v_max_f16_e32 v5, null, v2 ; encoding: [0x7c,0x04,0x0a,0x72] +# GFX11-REAL16: v_max_f16_e32 v5.l, null, v2.l ; encoding: [0x7c,0x04,0x0a,0x72] +# GFX11-FAKE16: v_max_f16_e32 v5, null, v2 ; encoding: [0x7c,0x04,0x0a,0x72] 0x7c,0x04,0x0a,0x72 -# GFX11-REAL16: v_max_f16_e32 v5.l, -1, v2.l ; encoding: [0xc1,0x04,0x0a,0x72] -# GFX11-FAKE16: v_max_f16_e32 v5, -1, v2 ; encoding: [0xc1,0x04,0x0a,0x72] +# GFX11-REAL16: v_max_f16_e32 v5.l, -1, v2.l ; encoding: [0xc1,0x04,0x0a,0x72] +# GFX11-FAKE16: v_max_f16_e32 v5, -1, v2 ; encoding: [0xc1,0x04,0x0a,0x72] 0xc1,0x04,0x0a,0x72 -# GFX11-REAL16: v_max_f16_e32 v5.l, 0.5, v2.l ; encoding: [0xf0,0x04,0x0a,0x72] -# GFX11-FAKE16: v_max_f16_e32 v5, 0.5, v2 ; encoding: [0xf0,0x04,0x0a,0x72] +# GFX11-REAL16: v_max_f16_e32 v5.l, 0.5, v2.l ; encoding: [0xf0,0x04,0x0a,0x72] +# GFX11-FAKE16: v_max_f16_e32 v5, 0.5, v2 ; encoding: [0xf0,0x04,0x0a,0x72] 0xf0,0x04,0x0a,0x72 -# GFX11-REAL16: v_max_f16_e32 v5.l, src_scc, v2.l ; encoding: [0xfd,0x04,0x0a,0x72] -# GFX11-FAKE16: v_max_f16_e32 v5, src_scc, v2 ; encoding: [0xfd,0x04,0x0a,0x72] +# GFX11-REAL16: v_max_f16_e32 v5.l, src_scc, v2.l ; encoding: [0xfd,0x04,0x0a,0x72] +# GFX11-FAKE16: v_max_f16_e32 v5, src_scc, v2 ; encoding: [0xfd,0x04,0x0a,0x72] 0xfd,0x04,0x0a,0x72 -# GFX11-REAL16: v_max_f16_e32 v5.h, src_scc, v2.h ; encoding: [0xfd,0x04,0x0b,0x73] +# GFX11-REAL16: v_max_f16_e32 v5.h, src_scc, v2.h ; encoding: [0xfd,0x04,0x0b,0x73] # COM: TODO: GFX11-FAKE16: warning: invalid instruction encoding 0xfd,0x04,0x0b,0x73 0xfd,0x04,0x0b,0x73 -# GFX11-REAL16: v_max_f16_e32 v127.l, 0xfe0b, v127.l ; encoding: [0xff,0xfe,0xfe,0x72,0x0b,0xfe,0x00,0x00] -# GFX11-FAKE16: v_max_f16_e32 v127, 0xfe0b, v127 ; encoding: [0xff,0xfe,0xfe,0x72,0x0b,0xfe,0x00,0x00] +# GFX11-REAL16: v_max_f16_e32 v127.l, 0xfe0b, v127.l ; encoding: [0xff,0xfe,0xfe,0x72,0x0b,0xfe,0x00,0x00] +# GFX11-FAKE16: v_max_f16_e32 v127, 0xfe0b, v127 ; encoding: [0xff,0xfe,0xfe,0x72,0x0b,0xfe,0x00,0x00] 0xff,0xfe,0xfe,0x72,0x0b,0xfe,0x00,0x00 -# GFX11-REAL16: v_max_f16_e32 v127.h, 0xfe0b, v127.h ; encoding: [0xff,0xfe,0xff,0x73,0x0b,0xfe,0x00,0x00] +# GFX11-REAL16: v_max_f16_e32 v127.h, 0xfe0b, v127.h ; encoding: [0xff,0xfe,0xff,0x73,0x0b,0xfe,0x00,0x00] # COM: TODO: GFX11-FAKE16: warning: invalid instruction encoding 0xff,0xfe,0xff,0x73,0x0b,0xfe,0x00,0x00 0xff,0xfe,0xff,0x73,0x0b,0xfe,0x00,0x00 @@ -1129,79 +1129,79 @@ # GFX11: v_max_u32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x29,0x56,0x34,0x12,0xaf] 0xff,0xfe,0xff,0x29,0x56,0x34,0x12,0xaf -# GFX11-REAL16: v_min_f16_e32 v5.l, v1.l, v2.l ; encoding: [0x01,0x05,0x0a,0x74] -# GFX11-FAKE16: v_min_f16_e32 v5, v1, v2 ; encoding: [0x01,0x05,0x0a,0x74] +# GFX11-REAL16: v_min_f16_e32 v5.l, v1.l, v2.l ; encoding: [0x01,0x05,0x0a,0x74] +# GFX11-FAKE16: v_min_f16_e32 v5, v1, v2 ; encoding: [0x01,0x05,0x0a,0x74] 0x01,0x05,0x0a,0x74 -# GFX11-REAL16: v_min_f16_e32 v5.l, v1.h, v2.l ; encoding: [0x81,0x05,0x0a,0x74] +# GFX11-REAL16: v_min_f16_e32 v5.l, v1.h, v2.l ; encoding: [0x81,0x05,0x0a,0x74] # GFX11-FAKE16: v_min_f16_e32 v5, v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x0a,0x74] 0x81,0x05,0x0a,0x74 -# GFX11-REAL16: v_min_f16_e32 v5.l, v127.l, v2.l ; encoding: [0x7f,0x05,0x0a,0x74] -# GFX11-FAKE16: v_min_f16_e32 v5, v127, v2 ; encoding: [0x7f,0x05,0x0a,0x74] +# GFX11-REAL16: v_min_f16_e32 v5.l, v127.l, v2.l ; encoding: [0x7f,0x05,0x0a,0x74] +# GFX11-FAKE16: v_min_f16_e32 v5, v127, v2 ; encoding: [0x7f,0x05,0x0a,0x74] 0x7f,0x05,0x0a,0x74 -# GFX11-REAL16: v_min_f16_e32 v5.l, v127.h, v2.l ; encoding: [0xff,0x05,0x0a,0x74] +# GFX11-REAL16: v_min_f16_e32 v5.l, v127.h, v2.l ; encoding: [0xff,0x05,0x0a,0x74] # GFX11-FAKE16: v_min_f16_e32 v5, v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x0a,0x74] 0xff,0x05,0x0a,0x74 -# GFX11-REAL16: v_min_f16_e32 v5.l, s1, v2.l ; encoding: [0x01,0x04,0x0a,0x74] -# GFX11-FAKE16: v_min_f16_e32 v5, s1, v2 ; encoding: [0x01,0x04,0x0a,0x74] +# GFX11-REAL16: v_min_f16_e32 v5.l, s1, v2.l ; encoding: [0x01,0x04,0x0a,0x74] +# GFX11-FAKE16: v_min_f16_e32 v5, s1, v2 ; encoding: [0x01,0x04,0x0a,0x74] 0x01,0x04,0x0a,0x74 -# GFX11-REAL16: v_min_f16_e32 v5.l, s105, v2.l ; encoding: [0x69,0x04,0x0a,0x74] -# GFX11-FAKE16: v_min_f16_e32 v5, s105, v2 ; encoding: [0x69,0x04,0x0a,0x74] +# GFX11-REAL16: v_min_f16_e32 v5.l, s105, v2.l ; encoding: [0x69,0x04,0x0a,0x74] +# GFX11-FAKE16: v_min_f16_e32 v5, s105, v2 ; encoding: [0x69,0x04,0x0a,0x74] 0x69,0x04,0x0a,0x74 -# GFX11-REAL16: v_min_f16_e32 v5.l, vcc_lo, v2.l ; encoding: [0x6a,0x04,0x0a,0x74] -# GFX11-FAKE16: v_min_f16_e32 v5, vcc_lo, v2 ; encoding: [0x6a,0x04,0x0a,0x74] +# GFX11-REAL16: v_min_f16_e32 v5.l, vcc_lo, v2.l ; encoding: [0x6a,0x04,0x0a,0x74] +# GFX11-FAKE16: v_min_f16_e32 v5, vcc_lo, v2 ; encoding: [0x6a,0x04,0x0a,0x74] 0x6a,0x04,0x0a,0x74 -# GFX11-REAL16: v_min_f16_e32 v5.l, vcc_hi, v2.l ; encoding: [0x6b,0x04,0x0a,0x74] -# GFX11-FAKE16: v_min_f16_e32 v5, vcc_hi, v2 ; encoding: [0x6b,0x04,0x0a,0x74] +# GFX11-REAL16: v_min_f16_e32 v5.l, vcc_hi, v2.l ; encoding: [0x6b,0x04,0x0a,0x74] +# GFX11-FAKE16: v_min_f16_e32 v5, vcc_hi, v2 ; encoding: [0x6b,0x04,0x0a,0x74] 0x6b,0x04,0x0a,0x74 -# GFX11-REAL16: v_min_f16_e32 v5.l, ttmp15, v2.l ; encoding: [0x7b,0x04,0x0a,0x74] -# GFX11-FAKE16: v_min_f16_e32 v5, ttmp15, v2 ; encoding: [0x7b,0x04,0x0a,0x74] +# GFX11-REAL16: v_min_f16_e32 v5.l, ttmp15, v2.l ; encoding: [0x7b,0x04,0x0a,0x74] +# GFX11-FAKE16: v_min_f16_e32 v5, ttmp15, v2 ; encoding: [0x7b,0x04,0x0a,0x74] 0x7b,0x04,0x0a,0x74 -# GFX11-REAL16: v_min_f16_e32 v5.l, m0, v2.l ; encoding: [0x7d,0x04,0x0a,0x74] -# GFX11-FAKE16: v_min_f16_e32 v5, m0, v2 ; encoding: [0x7d,0x04,0x0a,0x74] +# GFX11-REAL16: v_min_f16_e32 v5.l, m0, v2.l ; encoding: [0x7d,0x04,0x0a,0x74] +# GFX11-FAKE16: v_min_f16_e32 v5, m0, v2 ; encoding: [0x7d,0x04,0x0a,0x74] 0x7d,0x04,0x0a,0x74 -# GFX11-REAL16: v_min_f16_e32 v5.l, exec_lo, v2.l ; encoding: [0x7e,0x04,0x0a,0x74] -# GFX11-FAKE16: v_min_f16_e32 v5, exec_lo, v2 ; encoding: [0x7e,0x04,0x0a,0x74] +# GFX11-REAL16: v_min_f16_e32 v5.l, exec_lo, v2.l ; encoding: [0x7e,0x04,0x0a,0x74] +# GFX11-FAKE16: v_min_f16_e32 v5, exec_lo, v2 ; encoding: [0x7e,0x04,0x0a,0x74] 0x7e,0x04,0x0a,0x74 -# GFX11-REAL16: v_min_f16_e32 v5.l, exec_hi, v2.l ; encoding: [0x7f,0x04,0x0a,0x74] -# GFX11-FAKE16: v_min_f16_e32 v5, exec_hi, v2 ; encoding: [0x7f,0x04,0x0a,0x74] +# GFX11-REAL16: v_min_f16_e32 v5.l, exec_hi, v2.l ; encoding: [0x7f,0x04,0x0a,0x74] +# GFX11-FAKE16: v_min_f16_e32 v5, exec_hi, v2 ; encoding: [0x7f,0x04,0x0a,0x74] 0x7f,0x04,0x0a,0x74 -# GFX11-REAL16: v_min_f16_e32 v5.l, null, v2.l ; encoding: [0x7c,0x04,0x0a,0x74] -# GFX11-FAKE16: v_min_f16_e32 v5, null, v2 ; encoding: [0x7c,0x04,0x0a,0x74] +# GFX11-REAL16: v_min_f16_e32 v5.l, null, v2.l ; encoding: [0x7c,0x04,0x0a,0x74] +# GFX11-FAKE16: v_min_f16_e32 v5, null, v2 ; encoding: [0x7c,0x04,0x0a,0x74] 0x7c,0x04,0x0a,0x74 -# GFX11-REAL16: v_min_f16_e32 v5.l, -1, v2.l ; encoding: [0xc1,0x04,0x0a,0x74] -# GFX11-FAKE16: v_min_f16_e32 v5, -1, v2 ; encoding: [0xc1,0x04,0x0a,0x74] +# GFX11-REAL16: v_min_f16_e32 v5.l, -1, v2.l ; encoding: [0xc1,0x04,0x0a,0x74] +# GFX11-FAKE16: v_min_f16_e32 v5, -1, v2 ; encoding: [0xc1,0x04,0x0a,0x74] 0xc1,0x04,0x0a,0x74 -# GFX11-REAL16: v_min_f16_e32 v5.l, 0.5, v2.l ; encoding: [0xf0,0x04,0x0a,0x74] -# GFX11-FAKE16: v_min_f16_e32 v5, 0.5, v2 ; encoding: [0xf0,0x04,0x0a,0x74] +# GFX11-REAL16: v_min_f16_e32 v5.l, 0.5, v2.l ; encoding: [0xf0,0x04,0x0a,0x74] +# GFX11-FAKE16: v_min_f16_e32 v5, 0.5, v2 ; encoding: [0xf0,0x04,0x0a,0x74] 0xf0,0x04,0x0a,0x74 -# GFX11-REAL16: v_min_f16_e32 v5.l, src_scc, v2.l ; encoding: [0xfd,0x04,0x0a,0x74] -# GFX11-FAKE16: v_min_f16_e32 v5, src_scc, v2 ; encoding: [0xfd,0x04,0x0a,0x74] +# GFX11-REAL16: v_min_f16_e32 v5.l, src_scc, v2.l ; encoding: [0xfd,0x04,0x0a,0x74] +# GFX11-FAKE16: v_min_f16_e32 v5, src_scc, v2 ; encoding: [0xfd,0x04,0x0a,0x74] 0xfd,0x04,0x0a,0x74 -# GFX11-REAL16: v_min_f16_e32 v5.h, src_scc, v2.h ; encoding: [0xfd,0x04,0x0b,0x75] +# GFX11-REAL16: v_min_f16_e32 v5.h, src_scc, v2.h ; encoding: [0xfd,0x04,0x0b,0x75] # COM: TODO: GFX11-FAKE16: warning: invalid instruction encoding 0xfd,0x04,0x0b,0x75 0xfd,0x04,0x0b,0x75 -# GFX11-REAL16: v_min_f16_e32 v127.l, 0xfe0b, v127.l ; encoding: [0xff,0xfe,0xfe,0x74,0x0b,0xfe,0x00,0x00] -# GFX11-FAKE16: v_min_f16_e32 v127, 0xfe0b, v127 ; encoding: [0xff,0xfe,0xfe,0x74,0x0b,0xfe,0x00,0x00] +# GFX11-REAL16: v_min_f16_e32 v127.l, 0xfe0b, v127.l ; encoding: [0xff,0xfe,0xfe,0x74,0x0b,0xfe,0x00,0x00] +# GFX11-FAKE16: v_min_f16_e32 v127, 0xfe0b, v127 ; encoding: [0xff,0xfe,0xfe,0x74,0x0b,0xfe,0x00,0x00] 0xff,0xfe,0xfe,0x74,0x0b,0xfe,0x00,0x00 -# GFX11-REAL16: v_min_f16_e32 v127.h, 0xfe0b, v127.h ; encoding: [0xff,0xfe,0xff,0x75,0x0b,0xfe,0x00,0x00] +# GFX11-REAL16: v_min_f16_e32 v127.h, 0xfe0b, v127.h ; encoding: [0xff,0xfe,0xff,0x75,0x0b,0xfe,0x00,0x00] # COM: TODO: GFX11-FAKE16: warning: invalid instruction encoding 0xff,0xfe,0xff,0x75,0x0b,0xfe,0x00,0x00 0xff,0xfe,0xff,0x75,0x0b,0xfe,0x00,0x00 @@ -1385,79 +1385,79 @@ # GFX11: v_mul_dx9_zero_f32_e32 v255, 0xaf123456, v255 ; encoding: [0xff,0xfe,0xff,0x0f,0x56,0x34,0x12,0xaf] 0xff,0xfe,0xff,0x0f,0x56,0x34,0x12,0xaf -# GFX11-REAL16: v_mul_f16_e32 v5.l, v1.l, v2.l ; encoding: [0x01,0x05,0x0a,0x6a] -# GFX11-FAKE16: v_mul_f16_e32 v5, v1, v2 ; encoding: [0x01,0x05,0x0a,0x6a] +# GFX11-REAL16: v_mul_f16_e32 v5.l, v1.l, v2.l ; encoding: [0x01,0x05,0x0a,0x6a] +# GFX11-FAKE16: v_mul_f16_e32 v5, v1, v2 ; encoding: [0x01,0x05,0x0a,0x6a] 0x01,0x05,0x0a,0x6a -# GFX11-REAL16: v_mul_f16_e32 v5.l, v1.h, v2.l ; encoding: [0x81,0x05,0x0a,0x6a] +# GFX11-REAL16: v_mul_f16_e32 v5.l, v1.h, v2.l ; encoding: [0x81,0x05,0x0a,0x6a] # GFX11-FAKE16: v_mul_f16_e32 v5, v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x0a,0x6a 0x81,0x05,0x0a,0x6a -# GFX11-REAL16: v_mul_f16_e32 v5.l, v127.l, v2.l ; encoding: [0x7f,0x05,0x0a,0x6a] -# GFX11-FAKE16: v_mul_f16_e32 v5, v127, v2 ; encoding: [0x7f,0x05,0x0a,0x6a] +# GFX11-REAL16: v_mul_f16_e32 v5.l, v127.l, v2.l ; encoding: [0x7f,0x05,0x0a,0x6a] +# GFX11-FAKE16: v_mul_f16_e32 v5, v127, v2 ; encoding: [0x7f,0x05,0x0a,0x6a] 0x7f,0x05,0x0a,0x6a -# GFX11-REAL16: v_mul_f16_e32 v5.l, v127.h, v2.l ; encoding: [0xff,0x05,0x0a,0x6a] +# GFX11-REAL16: v_mul_f16_e32 v5.l, v127.h, v2.l ; encoding: [0xff,0x05,0x0a,0x6a] # GFX11-FAKE16: v_mul_f16_e32 v5, v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x0a,0x6a] 0xff,0x05,0x0a,0x6a -# GFX11-REAL16: v_mul_f16_e32 v5.l, s1, v2.l ; encoding: [0x01,0x04,0x0a,0x6a] -# GFX11-FAKE16: v_mul_f16_e32 v5, s1, v2 ; encoding: [0x01,0x04,0x0a,0x6a] +# GFX11-REAL16: v_mul_f16_e32 v5.l, s1, v2.l ; encoding: [0x01,0x04,0x0a,0x6a] +# GFX11-FAKE16: v_mul_f16_e32 v5, s1, v2 ; encoding: [0x01,0x04,0x0a,0x6a] 0x01,0x04,0x0a,0x6a -# GFX11-REAL16: v_mul_f16_e32 v5.l, s105, v2.l ; encoding: [0x69,0x04,0x0a,0x6a] -# GFX11-FAKE16: v_mul_f16_e32 v5, s105, v2 ; encoding: [0x69,0x04,0x0a,0x6a] +# GFX11-REAL16: v_mul_f16_e32 v5.l, s105, v2.l ; encoding: [0x69,0x04,0x0a,0x6a] +# GFX11-FAKE16: v_mul_f16_e32 v5, s105, v2 ; encoding: [0x69,0x04,0x0a,0x6a] 0x69,0x04,0x0a,0x6a -# GFX11-REAL16: v_mul_f16_e32 v5.l, vcc_lo, v2.l ; encoding: [0x6a,0x04,0x0a,0x6a] -# GFX11-FAKE16: v_mul_f16_e32 v5, vcc_lo, v2 ; encoding: [0x6a,0x04,0x0a,0x6a] +# GFX11-REAL16: v_mul_f16_e32 v5.l, vcc_lo, v2.l ; encoding: [0x6a,0x04,0x0a,0x6a] +# GFX11-FAKE16: v_mul_f16_e32 v5, vcc_lo, v2 ; encoding: [0x6a,0x04,0x0a,0x6a] 0x6a,0x04,0x0a,0x6a -# GFX11-REAL16: v_mul_f16_e32 v5.l, vcc_hi, v2.l ; encoding: [0x6b,0x04,0x0a,0x6a] -# GFX11-FAKE16: v_mul_f16_e32 v5, vcc_hi, v2 ; encoding: [0x6b,0x04,0x0a,0x6a] +# GFX11-REAL16: v_mul_f16_e32 v5.l, vcc_hi, v2.l ; encoding: [0x6b,0x04,0x0a,0x6a] +# GFX11-FAKE16: v_mul_f16_e32 v5, vcc_hi, v2 ; encoding: [0x6b,0x04,0x0a,0x6a] 0x6b,0x04,0x0a,0x6a -# GFX11-REAL16: v_mul_f16_e32 v5.l, ttmp15, v2.l ; encoding: [0x7b,0x04,0x0a,0x6a] -# GFX11-FAKE16: v_mul_f16_e32 v5, ttmp15, v2 ; encoding: [0x7b,0x04,0x0a,0x6a] +# GFX11-REAL16: v_mul_f16_e32 v5.l, ttmp15, v2.l ; encoding: [0x7b,0x04,0x0a,0x6a] +# GFX11-FAKE16: v_mul_f16_e32 v5, ttmp15, v2 ; encoding: [0x7b,0x04,0x0a,0x6a] 0x7b,0x04,0x0a,0x6a -# GFX11-REAL16: v_mul_f16_e32 v5.l, m0, v2.l ; encoding: [0x7d,0x04,0x0a,0x6a] -# GFX11-FAKE16: v_mul_f16_e32 v5, m0, v2 ; encoding: [0x7d,0x04,0x0a,0x6a] +# GFX11-REAL16: v_mul_f16_e32 v5.l, m0, v2.l ; encoding: [0x7d,0x04,0x0a,0x6a] +# GFX11-FAKE16: v_mul_f16_e32 v5, m0, v2 ; encoding: [0x7d,0x04,0x0a,0x6a] 0x7d,0x04,0x0a,0x6a -# GFX11-REAL16: v_mul_f16_e32 v5.l, exec_lo, v2.l ; encoding: [0x7e,0x04,0x0a,0x6a] -# GFX11-FAKE16: v_mul_f16_e32 v5, exec_lo, v2 ; encoding: [0x7e,0x04,0x0a,0x6a] +# GFX11-REAL16: v_mul_f16_e32 v5.l, exec_lo, v2.l ; encoding: [0x7e,0x04,0x0a,0x6a] +# GFX11-FAKE16: v_mul_f16_e32 v5, exec_lo, v2 ; encoding: [0x7e,0x04,0x0a,0x6a] 0x7e,0x04,0x0a,0x6a -# GFX11-REAL16: v_mul_f16_e32 v5.l, exec_hi, v2.l ; encoding: [0x7f,0x04,0x0a,0x6a] -# GFX11-FAKE16: v_mul_f16_e32 v5, exec_hi, v2 ; encoding: [0x7f,0x04,0x0a,0x6a] +# GFX11-REAL16: v_mul_f16_e32 v5.l, exec_hi, v2.l ; encoding: [0x7f,0x04,0x0a,0x6a] +# GFX11-FAKE16: v_mul_f16_e32 v5, exec_hi, v2 ; encoding: [0x7f,0x04,0x0a,0x6a] 0x7f,0x04,0x0a,0x6a -# GFX11-REAL16: v_mul_f16_e32 v5.l, null, v2.l ; encoding: [0x7c,0x04,0x0a,0x6a] -# GFX11-FAKE16: v_mul_f16_e32 v5, null, v2 ; encoding: [0x7c,0x04,0x0a,0x6a] +# GFX11-REAL16: v_mul_f16_e32 v5.l, null, v2.l ; encoding: [0x7c,0x04,0x0a,0x6a] +# GFX11-FAKE16: v_mul_f16_e32 v5, null, v2 ; encoding: [0x7c,0x04,0x0a,0x6a] 0x7c,0x04,0x0a,0x6a -# GFX11-REAL16: v_mul_f16_e32 v5.l, -1, v2.l ; encoding: [0xc1,0x04,0x0a,0x6a] -# GFX11-FAKE16: v_mul_f16_e32 v5, -1, v2 ; encoding: [0xc1,0x04,0x0a,0x6a] +# GFX11-REAL16: v_mul_f16_e32 v5.l, -1, v2.l ; encoding: [0xc1,0x04,0x0a,0x6a] +# GFX11-FAKE16: v_mul_f16_e32 v5, -1, v2 ; encoding: [0xc1,0x04,0x0a,0x6a] 0xc1,0x04,0x0a,0x6a -# GFX11-REAL16: v_mul_f16_e32 v5.l, 0.5, v2.l ; encoding: [0xf0,0x04,0x0a,0x6a] -# GFX11-FAKE16: v_mul_f16_e32 v5, 0.5, v2 ; encoding: [0xf0,0x04,0x0a,0x6a] +# GFX11-REAL16: v_mul_f16_e32 v5.l, 0.5, v2.l ; encoding: [0xf0,0x04,0x0a,0x6a] +# GFX11-FAKE16: v_mul_f16_e32 v5, 0.5, v2 ; encoding: [0xf0,0x04,0x0a,0x6a] 0xf0,0x04,0x0a,0x6a -# GFX11-REAL16: v_mul_f16_e32 v5.l, src_scc, v2.l ; encoding: [0xfd,0x04,0x0a,0x6a] -# GFX11-FAKE16: v_mul_f16_e32 v5, src_scc, v2 ; encoding: [0xfd,0x04,0x0a,0x6a] +# GFX11-REAL16: v_mul_f16_e32 v5.l, src_scc, v2.l ; encoding: [0xfd,0x04,0x0a,0x6a] +# GFX11-FAKE16: v_mul_f16_e32 v5, src_scc, v2 ; encoding: [0xfd,0x04,0x0a,0x6a] 0xfd,0x04,0x0a,0x6a -# GFX11-REAL16: v_mul_f16_e32 v5.h, src_scc, v2.h ; encoding: [0xfd,0x04,0x0b,0x6b] +# GFX11-REAL16: v_mul_f16_e32 v5.h, src_scc, v2.h ; encoding: [0xfd,0x04,0x0b,0x6b] # COM: TODO: GFX11-FAKE16: warning: invalid instruction encoding 0xfd,0x04,0x0b,0x6b 0xfd,0x04,0x0b,0x6b -# GFX11-REAL16: v_mul_f16_e32 v127.l, 0xfe0b, v127.l ; encoding: [0xff,0xfe,0xfe,0x6a,0x0b,0xfe,0x00,0x00] -# GFX11-FAKE16: v_mul_f16_e32 v127, 0xfe0b, v127 ; encoding: [0xff,0xfe,0xfe,0x6a,0x0b,0xfe,0x00,0x00] +# GFX11-REAL16: v_mul_f16_e32 v127.l, 0xfe0b, v127.l ; encoding: [0xff,0xfe,0xfe,0x6a,0x0b,0xfe,0x00,0x00] +# GFX11-FAKE16: v_mul_f16_e32 v127, 0xfe0b, v127 ; encoding: [0xff,0xfe,0xfe,0x6a,0x0b,0xfe,0x00,0x00] 0xff,0xfe,0xfe,0x6a,0x0b,0xfe,0x00,0x00 -# GFX11-REAL16: v_mul_f16_e32 v127.h, 0xfe0b, v127.h ; encoding: [0xff,0xfe,0xff,0x6b,0x0b,0xfe,0x00,0x00] +# GFX11-REAL16: v_mul_f16_e32 v127.h, 0xfe0b, v127.h ; encoding: [0xff,0xfe,0xff,0x6b,0x0b,0xfe,0x00,0x00] # COM: TODO: GFX11-FAKE16: warning: invalid instruction encoding 0xff,0xfe,0xff,0x6b,0x0b,0xfe,0x00,0x00 0xff,0xfe,0xff,0x6b,0x0b,0xfe,0x00,0x00 @@ -1836,79 +1836,79 @@ # W64: v_sub_co_ci_u32_e32 v255, vcc, 0xaf123456, v255, vcc ; encoding: [0xff,0xfe,0xff,0x43,0x56,0x34,0x12,0xaf] 0xff,0xfe,0xff,0x43,0x56,0x34,0x12,0xaf -# GFX11-REAL16: v_sub_f16_e32 v5.l, v1.l, v2.l ; encoding: [0x01,0x05,0x0a,0x66] -# GFX11-FAKE16: v_sub_f16_e32 v5, v1, v2 ; encoding: [0x01,0x05,0x0a,0x66] +# GFX11-REAL16: v_sub_f16_e32 v5.l, v1.l, v2.l ; encoding: [0x01,0x05,0x0a,0x66] +# GFX11-FAKE16: v_sub_f16_e32 v5, v1, v2 ; encoding: [0x01,0x05,0x0a,0x66] 0x01,0x05,0x0a,0x66 -# GFX11-REAL16: v_sub_f16_e32 v5.l, v1.h, v2.l ; encoding: [0x81,0x05,0x0a,0x66] +# GFX11-REAL16: v_sub_f16_e32 v5.l, v1.h, v2.l ; encoding: [0x81,0x05,0x0a,0x66] # GFX11-FAKE16: v_sub_f16_e32 v5, v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x0a,0x66] 0x81,0x05,0x0a,0x66 -# GFX11-REAL16: v_sub_f16_e32 v5.l, v127.l, v2.l ; encoding: [0x7f,0x05,0x0a,0x66] -# GFX11-FAKE16: v_sub_f16_e32 v5, v127, v2 ; encoding: [0x7f,0x05,0x0a,0x66] +# GFX11-REAL16: v_sub_f16_e32 v5.l, v127.l, v2.l ; encoding: [0x7f,0x05,0x0a,0x66] +# GFX11-FAKE16: v_sub_f16_e32 v5, v127, v2 ; encoding: [0x7f,0x05,0x0a,0x66] 0x7f,0x05,0x0a,0x66 -# GFX11-REAL16: v_sub_f16_e32 v5.l, v127.h, v2.l ; encoding: [0xff,0x05,0x0a,0x66] +# GFX11-REAL16: v_sub_f16_e32 v5.l, v127.h, v2.l ; encoding: [0xff,0x05,0x0a,0x66] # GFX11-FAKE16: v_sub_f16_e32 v5, v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x0a,0x66] 0xff,0x05,0x0a,0x66 -# GFX11-REAL16: v_sub_f16_e32 v5.l, s1, v2.l ; encoding: [0x01,0x04,0x0a,0x66] -# GFX11-FAKE16: v_sub_f16_e32 v5, s1, v2 ; encoding: [0x01,0x04,0x0a,0x66] +# GFX11-REAL16: v_sub_f16_e32 v5.l, s1, v2.l ; encoding: [0x01,0x04,0x0a,0x66] +# GFX11-FAKE16: v_sub_f16_e32 v5, s1, v2 ; encoding: [0x01,0x04,0x0a,0x66] 0x01,0x04,0x0a,0x66 -# GFX11-REAL16: v_sub_f16_e32 v5.l, s105, v2.l ; encoding: [0x69,0x04,0x0a,0x66] -# GFX11-FAKE16: v_sub_f16_e32 v5, s105, v2 ; encoding: [0x69,0x04,0x0a,0x66] +# GFX11-REAL16: v_sub_f16_e32 v5.l, s105, v2.l ; encoding: [0x69,0x04,0x0a,0x66] +# GFX11-FAKE16: v_sub_f16_e32 v5, s105, v2 ; encoding: [0x69,0x04,0x0a,0x66] 0x69,0x04,0x0a,0x66 -# GFX11-REAL16: v_sub_f16_e32 v5.l, vcc_lo, v2.l ; encoding: [0x6a,0x04,0x0a,0x66] -# GFX11-FAKE16: v_sub_f16_e32 v5, vcc_lo, v2 ; encoding: [0x6a,0x04,0x0a,0x66] +# GFX11-REAL16: v_sub_f16_e32 v5.l, vcc_lo, v2.l ; encoding: [0x6a,0x04,0x0a,0x66] +# GFX11-FAKE16: v_sub_f16_e32 v5, vcc_lo, v2 ; encoding: [0x6a,0x04,0x0a,0x66] 0x6a,0x04,0x0a,0x66 -# GFX11-REAL16: v_sub_f16_e32 v5.l, vcc_hi, v2.l ; encoding: [0x6b,0x04,0x0a,0x66] -# GFX11-FAKE16: v_sub_f16_e32 v5, vcc_hi, v2 ; encoding: [0x6b,0x04,0x0a,0x66] +# GFX11-REAL16: v_sub_f16_e32 v5.l, vcc_hi, v2.l ; encoding: [0x6b,0x04,0x0a,0x66] +# GFX11-FAKE16: v_sub_f16_e32 v5, vcc_hi, v2 ; encoding: [0x6b,0x04,0x0a,0x66] 0x6b,0x04,0x0a,0x66 -# GFX11-REAL16: v_sub_f16_e32 v5.l, ttmp15, v2.l ; encoding: [0x7b,0x04,0x0a,0x66] -# GFX11-FAKE16: v_sub_f16_e32 v5, ttmp15, v2 ; encoding: [0x7b,0x04,0x0a,0x66] +# GFX11-REAL16: v_sub_f16_e32 v5.l, ttmp15, v2.l ; encoding: [0x7b,0x04,0x0a,0x66] +# GFX11-FAKE16: v_sub_f16_e32 v5, ttmp15, v2 ; encoding: [0x7b,0x04,0x0a,0x66] 0x7b,0x04,0x0a,0x66 -# GFX11-REAL16: v_sub_f16_e32 v5.l, m0, v2.l ; encoding: [0x7d,0x04,0x0a,0x66] -# GFX11-FAKE16: v_sub_f16_e32 v5, m0, v2 ; encoding: [0x7d,0x04,0x0a,0x66] +# GFX11-REAL16: v_sub_f16_e32 v5.l, m0, v2.l ; encoding: [0x7d,0x04,0x0a,0x66] +# GFX11-FAKE16: v_sub_f16_e32 v5, m0, v2 ; encoding: [0x7d,0x04,0x0a,0x66] 0x7d,0x04,0x0a,0x66 -# GFX11-REAL16: v_sub_f16_e32 v5.l, exec_lo, v2.l ; encoding: [0x7e,0x04,0x0a,0x66] -# GFX11-FAKE16: v_sub_f16_e32 v5, exec_lo, v2 ; encoding: [0x7e,0x04,0x0a,0x66] +# GFX11-REAL16: v_sub_f16_e32 v5.l, exec_lo, v2.l ; encoding: [0x7e,0x04,0x0a,0x66] +# GFX11-FAKE16: v_sub_f16_e32 v5, exec_lo, v2 ; encoding: [0x7e,0x04,0x0a,0x66] 0x7e,0x04,0x0a,0x66 -# GFX11-REAL16: v_sub_f16_e32 v5.l, exec_hi, v2.l ; encoding: [0x7f,0x04,0x0a,0x66] -# GFX11-FAKE16: v_sub_f16_e32 v5, exec_hi, v2 ; encoding: [0x7f,0x04,0x0a,0x66] +# GFX11-REAL16: v_sub_f16_e32 v5.l, exec_hi, v2.l ; encoding: [0x7f,0x04,0x0a,0x66] +# GFX11-FAKE16: v_sub_f16_e32 v5, exec_hi, v2 ; encoding: [0x7f,0x04,0x0a,0x66] 0x7f,0x04,0x0a,0x66 -# GFX11-REAL16: v_sub_f16_e32 v5.l, null, v2.l ; encoding: [0x7c,0x04,0x0a,0x66] -# GFX11-FAKE16: v_sub_f16_e32 v5, null, v2 ; encoding: [0x7c,0x04,0x0a,0x66] +# GFX11-REAL16: v_sub_f16_e32 v5.l, null, v2.l ; encoding: [0x7c,0x04,0x0a,0x66] +# GFX11-FAKE16: v_sub_f16_e32 v5, null, v2 ; encoding: [0x7c,0x04,0x0a,0x66] 0x7c,0x04,0x0a,0x66 -# GFX11-REAL16: v_sub_f16_e32 v5.l, -1, v2.l ; encoding: [0xc1,0x04,0x0a,0x66] -# GFX11-FAKE16: v_sub_f16_e32 v5, -1, v2 ; encoding: [0xc1,0x04,0x0a,0x66] +# GFX11-REAL16: v_sub_f16_e32 v5.l, -1, v2.l ; encoding: [0xc1,0x04,0x0a,0x66] +# GFX11-FAKE16: v_sub_f16_e32 v5, -1, v2 ; encoding: [0xc1,0x04,0x0a,0x66] 0xc1,0x04,0x0a,0x66 -# GFX11-REAL16: v_sub_f16_e32 v5.l, 0.5, v2.l ; encoding: [0xf0,0x04,0x0a,0x66] -# GFX11-FAKE16: v_sub_f16_e32 v5, 0.5, v2 ; encoding: [0xf0,0x04,0x0a,0x66] +# GFX11-REAL16: v_sub_f16_e32 v5.l, 0.5, v2.l ; encoding: [0xf0,0x04,0x0a,0x66] +# GFX11-FAKE16: v_sub_f16_e32 v5, 0.5, v2 ; encoding: [0xf0,0x04,0x0a,0x66] 0xf0,0x04,0x0a,0x66 -# GFX11-REAL16: v_sub_f16_e32 v5.l, src_scc, v2.l ; encoding: [0xfd,0x04,0x0a,0x66] -# GFX11-FAKE16: v_sub_f16_e32 v5, src_scc, v2 ; encoding: [0xfd,0x04,0x0a,0x66] +# GFX11-REAL16: v_sub_f16_e32 v5.l, src_scc, v2.l ; encoding: [0xfd,0x04,0x0a,0x66] +# GFX11-FAKE16: v_sub_f16_e32 v5, src_scc, v2 ; encoding: [0xfd,0x04,0x0a,0x66] 0xfd,0x04,0x0a,0x66 -# GFX11-REAL16: v_sub_f16_e32 v5.h, src_scc, v2.h ; encoding: [0xfd,0x04,0x0b,0x67] +# GFX11-REAL16: v_sub_f16_e32 v5.h, src_scc, v2.h ; encoding: [0xfd,0x04,0x0b,0x67] # COM: TODO: GFX11-FAKE16: warning: invalid instruction encoding 0xfd,0x04,0x0b,0x67 0xfd,0x04,0x0b,0x67 -# GFX11-REAL16: v_sub_f16_e32 v127.l, 0xfe0b, v127.l ; encoding: [0xff,0xfe,0xfe,0x66,0x0b,0xfe,0x00,0x00] -# GFX11-FAKE16: v_sub_f16_e32 v127, 0xfe0b, v127 ; encoding: [0xff,0xfe,0xfe,0x66,0x0b,0xfe,0x00,0x00] +# GFX11-REAL16: v_sub_f16_e32 v127.l, 0xfe0b, v127.l ; encoding: [0xff,0xfe,0xfe,0x66,0x0b,0xfe,0x00,0x00] +# GFX11-FAKE16: v_sub_f16_e32 v127, 0xfe0b, v127 ; encoding: [0xff,0xfe,0xfe,0x66,0x0b,0xfe,0x00,0x00] 0xff,0xfe,0xfe,0x66,0x0b,0xfe,0x00,0x00 -# GFX11-REAL16: v_sub_f16_e32 v127.h, 0xfe0b, v127.h ; encoding: [0xff,0xfe,0xff,0x67,0x0b,0xfe,0x00,0x00] +# GFX11-REAL16: v_sub_f16_e32 v127.h, 0xfe0b, v127.h ; encoding: [0xff,0xfe,0xff,0x67,0x0b,0xfe,0x00,0x00] # COM: TODO: GFX11-FAKE16: warning: invalid instruction encoding 0xff,0xfe,0xff,0x67,0x0b,0xfe,0x00,0x00 0xff,0xfe,0xff,0x67,0x0b,0xfe,0x00,0x00 @@ -2062,76 +2062,76 @@ # W64: v_subrev_co_ci_u32_e32 v255, vcc, 0xaf123456, v255, vcc ; encoding: [0xff,0xfe,0xff,0x45,0x56,0x34,0x12,0xaf] 0xff,0xfe,0xff,0x45,0x56,0x34,0x12,0xaf -# GFX11-REAL16: v_subrev_f16_e32 v5.l, v1.l, v2.l ; encoding: [0x01,0x05,0x0a,0x68] -# GFX11-FAKE16: v_subrev_f16_e32 v5, v1, v2 ; encoding: [0x01,0x05,0x0a,0x68] +# GFX11-REAL16: v_subrev_f16_e32 v5.l, v1.l, v2.l ; encoding: [0x01,0x05,0x0a,0x68] +# GFX11-FAKE16: v_subrev_f16_e32 v5, v1, v2 ; encoding: [0x01,0x05,0x0a,0x68] 0x01,0x05,0x0a,0x68 -# GFX11-REAL16: v_subrev_f16_e32 v5.l, v1.h, v2.l ; encoding: [0x81,0x05,0x0a,0x68] +# GFX11-REAL16: v_subrev_f16_e32 v5.l, v1.h, v2.l ; encoding: [0x81,0x05,0x0a,0x68] # GFX11-FAKE16: v_subrev_f16_e32 v5, v129/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0x81,0x05,0x0a,0x68] 0x81,0x05,0x0a,0x68 -# GFX11-REAL16: v_subrev_f16_e32 v5.l, v127.l, v2.l ; encoding: [0x7f,0x05,0x0a,0x68] -# GFX11-FAKE16: v_subrev_f16_e32 v5, v127, v2 ; encoding: [0x7f,0x05,0x0a,0x68] +# GFX11-REAL16: v_subrev_f16_e32 v5.l, v127.l, v2.l ; encoding: [0x7f,0x05,0x0a,0x68] +# GFX11-FAKE16: v_subrev_f16_e32 v5, v127, v2 ; encoding: [0x7f,0x05,0x0a,0x68] 0x7f,0x05,0x0a,0x68 -# GFX11-REAL16: v_subrev_f16_e32 v5.l, v127.h, v2.l ; encoding: [0xff,0x05,0x0a,0x68] +# GFX11-REAL16: v_subrev_f16_e32 v5.l, v127.h, v2.l ; encoding: [0xff,0x05,0x0a,0x68] # GFX11-FAKE16: v_subrev_f16_e32 v5, v255/*Invalid register, operand has 'VS_32_Lo128' register class*/, v2 ; encoding: [0xff,0x05,0x0a,0x68] 0xff,0x05,0x0a,0x68 -# GFX11-REAL16: v_subrev_f16_e32 v5.l, s1, v2.l ; encoding: [0x01,0x04,0x0a,0x68] -# GFX11-FAKE16: v_subrev_f16_e32 v5, s1, v2 ; encoding: [0x01,0x04,0x0a,0x68] +# GFX11-REAL16: v_subrev_f16_e32 v5.l, s1, v2.l ; encoding: [0x01,0x04,0x0a,0x68] +# GFX11-FAKE16: v_subrev_f16_e32 v5, s1, v2 ; encoding: [0x01,0x04,0x0a,0x68] 0x01,0x04,0x0a,0x68 -# GFX11-REAL16: v_subrev_f16_e32 v5.l, s105, v2.l ; encoding: [0x69,0x04,0x0a,0x68] -# GFX11-FAKE16: v_subrev_f16_e32 v5, s105, v2 ; encoding: [0x69,0x04,0x0a,0x68] +# GFX11-REAL16: v_subrev_f16_e32 v5.l, s105, v2.l ; encoding: [0x69,0x04,0x0a,0x68] +# GFX11-FAKE16: v_subrev_f16_e32 v5, s105, v2 ; encoding: [0x69,0x04,0x0a,0x68] 0x69,0x04,0x0a,0x68 -# GFX11-REAL16: v_subrev_f16_e32 v5.l, vcc_lo, v2.l ; encoding: [0x6a,0x04,0x0a,0x68] -# GFX11-FAKE16: v_subrev_f16_e32 v5, vcc_lo, v2 ; encoding: [0x6a,0x04,0x0a,0x68] +# GFX11-REAL16: v_subrev_f16_e32 v5.l, vcc_lo, v2.l ; encoding: [0x6a,0x04,0x0a,0x68] +# GFX11-FAKE16: v_subrev_f16_e32 v5, vcc_lo, v2 ; encoding: [0x6a,0x04,0x0a,0x68] 0x6a,0x04,0x0a,0x68 -# GFX11-REAL16: v_subrev_f16_e32 v5.l, vcc_hi, v2.l ; encoding: [0x6b,0x04,0x0a,0x68] -# GFX11-FAKE16: v_subrev_f16_e32 v5, vcc_hi, v2 ; encoding: [0x6b,0x04,0x0a,0x68] +# GFX11-REAL16: v_subrev_f16_e32 v5.l, vcc_hi, v2.l ; encoding: [0x6b,0x04,0x0a,0x68] +# GFX11-FAKE16: v_subrev_f16_e32 v5, vcc_hi, v2 ; encoding: [0x6b,0x04,0x0a,0x68] 0x6b,0x04,0x0a,0x68 -# GFX11-REAL16: v_subrev_f16_e32 v5.l, ttmp15, v2.l ; encoding: [0x7b,0x04,0x0a,0x68] -# GFX11-FAKE16: v_subrev_f16_e32 v5, ttmp15, v2 ; encoding: [0x7b,0x04,0x0a,0x68] +# GFX11-REAL16: v_subrev_f16_e32 v5.l, ttmp15, v2.l ; encoding: [0x7b,0x04,0x0a,0x68] +# GFX11-FAKE16: v_subrev_f16_e32 v5, ttmp15, v2 ; encoding: [0x7b,0x04,0x0a,0x68] 0x7b,0x04,0x0a,0x68 -# GFX11-REAL16: v_subrev_f16_e32 v5.l, m0, v2.l ; encoding: [0x7d,0x04,0x0a,0x68] -# GFX11-FAKE16: v_subrev_f16_e32 v5, m0, v2 ; encoding: [0x7d,0x04,0x0a,0x68] +# GFX11-REAL16: v_subrev_f16_e32 v5.l, m0, v2.l ; encoding: [0x7d,0x04,0x0a,0x68] +# GFX11-FAKE16: v_subrev_f16_e32 v5, m0, v2 ; encoding: [0x7d,0x04,0x0a,0x68] 0x7d,0x04,0x0a,0x68 -# GFX11-REAL16: v_subrev_f16_e32 v5.l, exec_lo, v2.l ; encoding: [0x7e,0x04,0x0a,0x68] -# GFX11-FAKE16: v_subrev_f16_e32 v5, exec_lo, v2 ; encoding: [0x7e,0x04,0x0a,0x68] +# GFX11-REAL16: v_subrev_f16_e32 v5.l, exec_lo, v2.l ; encoding: [0x7e,0x04,0x0a,0x68] +# GFX11-FAKE16: v_subrev_f16_e32 v5, exec_lo, v2 ; encoding: [0x7e,0x04,0x0a,0x68] 0x7e,0x04,0x0a,0x68 -# GFX11-REAL16: v_subrev_f16_e32 v5.l, exec_hi, v2.l ; encoding: [0x7f,0x04,0x0a,0x68] -# GFX11-FAKE16: v_subrev_f16_e32 v5, exec_hi, v2 ; encoding: [0x7f,0x04,0x0a,0x68] +# GFX11-REAL16: v_subrev_f16_e32 v5.l, exec_hi, v2.l ; encoding: [0x7f,0x04,0x0a,0x68] +# GFX11-FAKE16: v_subrev_f16_e32 v5, exec_hi, v2 ; encoding: [0x7f,0x04,0x0a,0x68] 0x7f,0x04,0x0a,0x68 -# GFX11-REAL16: v_subrev_f16_e32 v5.l, null, v2.l ; encoding: [0x7c,0x04,0x0a,0x68] -# GFX11-FAKE16: v_subrev_f16_e32 v5, null, v2 ; encoding: [0x7c,0x04,0x0a,0x68] +# GFX11-REAL16: v_subrev_f16_e32 v5.l, null, v2.l ; encoding: [0x7c,0x04,0x0a,0x68] +# GFX11-FAKE16: v_subrev_f16_e32 v5, null, v2 ; encoding: [0x7c,0x04,0x0a,0x68] 0x7c,0x04,0x0a,0x68 -# GFX11-REAL16: v_subrev_f16_e32 v5.l, -1, v2.l ; encoding: [0xc1,0x04,0x0a,0x68] -# GFX11-FAKE16: v_subrev_f16_e32 v5, -1, v2 ; encoding: [0xc1,0x04,0x0a,0x68] +# GFX11-REAL16: v_subrev_f16_e32 v5.l, -1, v2.l ; encoding: [0xc1,0x04,0x0a,0x68] +# GFX11-FAKE16: v_subrev_f16_e32 v5, -1, v2 ; encoding: [0xc1,0x04,0x0a,0x68] 0xc1,0x04,0x0a,0x68 -# GFX11-REAL16: v_subrev_f16_e32 v5.l, 0.5, v2.l ; encoding: [0xf0,0x04,0x0a,0x68] -# GFX11-FAKE16: v_subrev_f16_e32 v5, 0.5, v2 ; encoding: [0xf0,0x04,0x0a,0x68] +# GFX11-REAL16: v_subrev_f16_e32 v5.l, 0.5, v2.l ; encoding: [0xf0,0x04,0x0a,0x68] +# GFX11-FAKE16: v_subrev_f16_e32 v5, 0.5, v2 ; encoding: [0xf0,0x04,0x0a,0x68] 0xf0,0x04,0x0a,0x68 -# GFX11-REAL16: v_subrev_f16_e32 v5.l, src_scc, v2.l ; encoding: [0xfd,0x04,0x0a,0x68] -# GFX11-FAKE16: v_subrev_f16_e32 v5, src_scc, v2 ; encoding: [0xfd,0x04,0x0a,0x68] +# GFX11-REAL16: v_subrev_f16_e32 v5.l, src_scc, v2.l ; encoding: [0xfd,0x04,0x0a,0x68] +# GFX11-FAKE16: v_subrev_f16_e32 v5, src_scc, v2 ; encoding: [0xfd,0x04,0x0a,0x68] 0xfd,0x04,0x0a,0x68 -# GFX11-REAL16: v_subrev_f16_e32 v5.h, src_scc, v2.h ; encoding: [0xfd,0x04,0x0b,0x69] +# GFX11-REAL16: v_subrev_f16_e32 v5.h, src_scc, v2.h ; encoding: [0xfd,0x04,0x0b,0x69] # COM: TODO: GFX11-FAKE16: warning: invalid instruction encoding 0xfd,0x04,0x0b,0x69 0xfd,0x04,0x0b,0x69 # GFX11-REAL16: v_subrev_f16_e32 v127.l, 0xfe0b, v127.l ; encoding: [0xff,0xfe,0xfe,0x68,0x0b,0xfe,0x00,0x00] -# GFX11-FAKE16: v_subrev_f16_e32 v127, 0xfe0b, v127 ; encoding: [0xff,0xfe,0xfe,0x68,0x0b,0xfe,0x00,0x00] +# GFX11-FAKE16: v_subrev_f16_e32 v127, 0xfe0b, v127 ; encoding: [0xff,0xfe,0xfe,0x68,0x0b,0xfe,0x00,0x00] 0xff,0xfe,0xfe,0x68,0x0b,0xfe,0x00,0x00 # GFX11-REAL16: v_subrev_f16_e32 v127.h, 0xfe0b, v127.h ; encoding: [0xff,0xfe,0xff,0x69,0x0b,0xfe,0x00,0x00] diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_from_vop2.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_from_vop2.txt index 503d644b61b5..bd57ebfe3b5e 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_from_vop2.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_from_vop2.txt @@ -1,5 +1,5 @@ -# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W32 %s -# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX11,W64 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefixes=GFX11,W32 %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-wavefrontsize32,+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefixes=GFX11,W64 %s # W32: v_add_co_ci_u32_e64 v5, s12, v1, 0xaf123456, s6 ; encoding: [0x05,0x0c,0x20,0xd5,0x01,0xff,0x19,0x00,0x56,0x34,0x12,0xaf] # W64: v_add_co_ci_u32_e64 v5, s[12:13], v1, 0xaf123456, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0x01,0xff,0x19,0x00,0x56,0x34,0x12,0xaf] @@ -25,7 +25,7 @@ # W64: v_add_co_ci_u32_e64 v5, s[12:13], ttmp15, ttmp15, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0x7b,0xf6,0x18,0x00] 0x05,0x0c,0x20,0xd5,0x7b,0xf6,0x18,0x00 -# W32: v_add_co_ci_u32_e64 v5, s12, m0, 0.5, s6 ; encoding: [0x05,0x0c,0x20,0xd5,0x7d,0xe0,0x19,0x00] +# W32: v_add_co_ci_u32_e64 v5, s12, m0, 0.5, s6 ; encoding: [0x05,0x0c,0x20,0xd5,0x7d,0xe0,0x19,0x00] # W64: v_add_co_ci_u32_e64 v5, s[12:13], m0, 0.5, s[6:7] ; encoding: [0x05,0x0c,0x20,0xd5,0x7d,0xe0,0x19,0x00] 0x05,0x0c,0x20,0xd5,0x7d,0xe0,0x19,0x00 @@ -281,48 +281,48 @@ # GFX11: v_ashrrev_i32_e64 v255, 0xaf123456, vcc_hi ; encoding: [0xff,0x00,0x1a,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf] 0xff,0x00,0x1a,0xd5,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf -# W32: v_cndmask_b32_e64 v5, v1, 0xaf123456, s6 ; encoding: [0x05,0x00,0x01,0xd5,0x01,0xff,0x19,0x00,0x56,0x34,0x12,0xaf] +# W32: v_cndmask_b32_e64 v5, v1, 0xaf123456, s6 ; encoding: [0x05,0x00,0x01,0xd5,0x01,0xff,0x19,0x00,0x56,0x34,0x12,0xaf] # W64: v_cndmask_b32_e64 v5, v1, 0xaf123456, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x01,0xff,0x19,0x00,0x56,0x34,0x12,0xaf] 0x05,0x00,0x01,0xd5,0x01,0xff,0x19,0x00,0x56,0x34,0x12,0xaf -# W32: v_cndmask_b32_e64 v5, v255, src_scc, s6 ; encoding: [0x05,0x00,0x01,0xd5,0xff,0xfb,0x19,0x00] +# W32: v_cndmask_b32_e64 v5, v255, src_scc, s6 ; encoding: [0x05,0x00,0x01,0xd5,0xff,0xfb,0x19,0x00] # W64: v_cndmask_b32_e64 v5, v255, src_scc, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0xff,0xfb,0x19,0x00] 0x05,0x00,0x01,0xd5,0xff,0xfb,0x19,0x00 -# W32: v_cndmask_b32_e64 v5, s105, s105, s6 ; encoding: [0x05,0x00,0x01,0xd5,0x69,0xd2,0x18,0x00] -# W64: v_cndmask_b32_e64 v5, s105, s105, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x69,0xd2,0x18,0x00] +# W32: v_cndmask_b32_e64 v5, s105, s105, s6 ; encoding: [0x05,0x00,0x01,0xd5,0x69,0xd2,0x18,0x00] +# W64: v_cndmask_b32_e64 v5, s105, s105, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x69,0xd2,0x18,0x00] 0x05,0x00,0x01,0xd5,0x69,0xd2,0x18,0x00 -# W32: v_cndmask_b32_e64 v5, vcc_lo, v2, s6 ; encoding: [0x05,0x00,0x01,0xd5,0x6a,0x04,0x1a,0x00] -# W64: v_cndmask_b32_e64 v5, vcc_lo, v2, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x6a,0x04,0x1a,0x00] +# W32: v_cndmask_b32_e64 v5, vcc_lo, v2, s6 ; encoding: [0x05,0x00,0x01,0xd5,0x6a,0x04,0x1a,0x00] +# W64: v_cndmask_b32_e64 v5, vcc_lo, v2, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x6a,0x04,0x1a,0x00] 0x05,0x00,0x01,0xd5,0x6a,0x04,0x1a,0x00 -# W32: v_cndmask_b32_e64 v5, vcc_hi, v255, s6 ; encoding: [0x05,0x00,0x01,0xd5,0x6b,0xfe,0x1b,0x00] +# W32: v_cndmask_b32_e64 v5, vcc_hi, v255, s6 ; encoding: [0x05,0x00,0x01,0xd5,0x6b,0xfe,0x1b,0x00] # W64: v_cndmask_b32_e64 v5, vcc_hi, v255, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x6b,0xfe,0x1b,0x00] 0x05,0x00,0x01,0xd5,0x6b,0xfe,0x1b,0x00 -# W32: v_cndmask_b32_e64 v5, ttmp15, ttmp15, s6 ; encoding: [0x05,0x00,0x01,0xd5,0x7b,0xf6,0x18,0x00] +# W32: v_cndmask_b32_e64 v5, ttmp15, ttmp15, s6 ; encoding: [0x05,0x00,0x01,0xd5,0x7b,0xf6,0x18,0x00] # W64: v_cndmask_b32_e64 v5, ttmp15, ttmp15, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x7b,0xf6,0x18,0x00] 0x05,0x00,0x01,0xd5,0x7b,0xf6,0x18,0x00 -# W32: v_cndmask_b32_e64 v5, m0, 0.5, s6 ; encoding: [0x05,0x00,0x01,0xd5,0x7d,0xe0,0x19,0x00] -# W64: v_cndmask_b32_e64 v5, m0, 0.5, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x7d,0xe0,0x19,0x00] +# W32: v_cndmask_b32_e64 v5, m0, 0.5, s6 ; encoding: [0x05,0x00,0x01,0xd5,0x7d,0xe0,0x19,0x00] +# W64: v_cndmask_b32_e64 v5, m0, 0.5, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x7d,0xe0,0x19,0x00] 0x05,0x00,0x01,0xd5,0x7d,0xe0,0x19,0x00 # W32: v_cndmask_b32_e64 v5, exec_lo, exec_lo, s6 ; encoding: [0x05,0x00,0x01,0xd5,0x7e,0xfc,0x18,0x00] # W64: v_cndmask_b32_e64 v5, exec_lo, exec_lo, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x7e,0xfc,0x18,0x00] 0x05,0x00,0x01,0xd5,0x7e,0xfc,0x18,0x00 -# W32: v_cndmask_b32_e64 v5, exec_hi, -1, s6 ; encoding: [0x05,0x00,0x01,0xd5,0x7f,0x82,0x19,0x00] +# W32: v_cndmask_b32_e64 v5, exec_hi, -1, s6 ; encoding: [0x05,0x00,0x01,0xd5,0x7f,0x82,0x19,0x00] # W64: v_cndmask_b32_e64 v5, exec_hi, -1, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x7f,0x82,0x19,0x00] 0x05,0x00,0x01,0xd5,0x7f,0x82,0x19,0x00 -# W32: v_cndmask_b32_e64 v5, null, exec_hi, s6 ; encoding: [0x05,0x00,0x01,0xd5,0x7c,0xfe,0x18,0x00] +# W32: v_cndmask_b32_e64 v5, null, exec_hi, s6 ; encoding: [0x05,0x00,0x01,0xd5,0x7c,0xfe,0x18,0x00] # W64: v_cndmask_b32_e64 v5, null, exec_hi, s[6:7] ; encoding: [0x05,0x00,0x01,0xd5,0x7c,0xfe,0x18,0x00] 0x05,0x00,0x01,0xd5,0x7c,0xfe,0x18,0x00 -# W32: v_cndmask_b32_e64 v5, -1, m0, s104 ; encoding: [0x05,0x00,0x01,0xd5,0xc1,0xfa,0xa0,0x01] -# W64: v_cndmask_b32_e64 v5, -1, m0, s[104:105] ; encoding: [0x05,0x00,0x01,0xd5,0xc1,0xfa,0xa0,0x01] +# W32: v_cndmask_b32_e64 v5, -1, m0, s104 ; encoding: [0x05,0x00,0x01,0xd5,0xc1,0xfa,0xa0,0x01] +# W64: v_cndmask_b32_e64 v5, -1, m0, s[104:105] ; encoding: [0x05,0x00,0x01,0xd5,0xc1,0xfa,0xa0,0x01] 0x05,0x00,0x01,0xd5,0xc1,0xfa,0xa0,0x01 # W32: v_cndmask_b32_e64 v5, 0.5, -|vcc_lo|, vcc_lo ; encoding: [0x05,0x02,0x01,0xd5,0xf0,0xd4,0xa8,0x41] @@ -1401,7 +1401,7 @@ # W64: v_sub_co_ci_u32_e64 v5, s[12:13], ttmp15, ttmp15, s[6:7] ; encoding: [0x05,0x0c,0x21,0xd5,0x7b,0xf6,0x18,0x00] 0x05,0x0c,0x21,0xd5,0x7b,0xf6,0x18,0x00 -# W32: v_sub_co_ci_u32_e64 v5, s12, m0, 0.5, s6 ; encoding: [0x05,0x0c,0x21,0xd5,0x7d,0xe0,0x19,0x00] +# W32: v_sub_co_ci_u32_e64 v5, s12, m0, 0.5, s6 ; encoding: [0x05,0x0c,0x21,0xd5,0x7d,0xe0,0x19,0x00] # W64: v_sub_co_ci_u32_e64 v5, s[12:13], m0, 0.5, s[6:7] ; encoding: [0x05,0x0c,0x21,0xd5,0x7d,0xe0,0x19,0x00] 0x05,0x0c,0x21,0xd5,0x7d,0xe0,0x19,0x00 -- GitLab From 0d38f21e4ab7fe7cebe76a9d7c218ec54dba1e98 Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Wed, 6 Mar 2024 10:36:45 -0800 Subject: [PATCH 359/929] [SCEV] Extend type hint in analysis output to all backedge kinds This extends the work from 7755c26 to all of the different backend taken count kinds that we print for the scev analysis printer. As before, the goal is to cut down on confusion as i4 -1 is a very different (unsigned) value from i32 -1. --- llvm/lib/Analysis/ScalarEvolution.cpp | 50 +++++++++----- .../2007-07-15-NegativeStride.ll | 4 +- .../2007-09-27-LargeStepping.ll | 4 +- .../2008-05-25-NegativeStepToZero.ll | 4 +- .../ScalarEvolution/2008-07-19-WrappingIV.ll | 4 +- .../ScalarEvolution/2008-12-08-FiniteSGE.ll | 4 +- .../2009-05-09-PointerEdgeCount.ll | 4 +- .../ScalarEvolution/SolveQuadraticEquation.ll | 4 +- ...drec-computed-during-addrec-calculation.ll | 6 +- llvm/test/Analysis/ScalarEvolution/alloca.ll | 4 +- .../backedge-taken-count-guard-info.ll | 4 +- .../ScalarEvolution/becount-invalidation.ll | 6 +- .../ScalarEvolution/exact_iter_count.ll | 24 +++---- .../ScalarEvolution/exhaustive-trip-counts.ll | 4 +- .../ScalarEvolution/exit-count-select-safe.ll | 12 ++-- .../ScalarEvolution/exit-count-select.ll | 8 +-- .../ScalarEvolution/exponential-behavior.ll | 6 +- llvm/test/Analysis/ScalarEvolution/fold.ll | 4 +- .../ScalarEvolution/huge-trip-multiple.ll | 20 +++--- .../ScalarEvolution/incorrect-exit-count.ll | 18 ++--- .../increasing-or-decreasing-iv.ll | 32 ++++----- .../Analysis/ScalarEvolution/invalidation.ll | 2 +- llvm/test/Analysis/ScalarEvolution/load.ll | 8 +-- .../ScalarEvolution/logical-operations.ll | 4 +- ...en-count-guard-info-rewrite-expressions.ll | 4 +- .../max-backedge-taken-count-guard-info.ll | 8 +-- .../ScalarEvolution/max-trip-count.ll | 54 +++++++-------- .../Analysis/ScalarEvolution/ne-overflow.ll | 6 +- llvm/test/Analysis/ScalarEvolution/nsw.ll | 8 +-- .../overflow-intrinsics-trip-count.ll | 24 +++---- llvm/test/Analysis/ScalarEvolution/pr25369.ll | 4 +- llvm/test/Analysis/ScalarEvolution/pr34538.ll | 8 +-- llvm/test/Analysis/ScalarEvolution/pr48225.ll | 12 ++-- .../ScalarEvolution/range-signedness.ll | 4 +- .../ScalarEvolution/sext-add-inreg-loop.ll | 4 +- .../Analysis/ScalarEvolution/sext-to-zext.ll | 4 +- .../test/Analysis/ScalarEvolution/shift-op.ll | 10 +-- .../ScalarEvolution/shift-recurrences.ll | 68 +++++++++---------- .../ScalarEvolution/smax-br-phi-idioms.ll | 12 ++-- .../ScalarEvolution/solve-quadratic-i1.ll | 8 +-- .../solve-quadratic-overflow.ll | 4 +- .../ScalarEvolution/solve-quadratic.ll | 8 +-- .../symbolic_max_exit_count.ll | 4 +- .../trip-count-andor-selectform.ll | 16 ++--- .../ScalarEvolution/trip-count-andor.ll | 16 ++--- .../trip-count-implied-addrec.ll | 2 +- .../trip-count-negative-stride.ll | 68 +++++++++---------- .../ScalarEvolution/trip-count-switch.ll | 4 +- .../Analysis/ScalarEvolution/trip-count.ll | 12 ++-- .../Analysis/ScalarEvolution/trip-count10.ll | 14 ++-- .../Analysis/ScalarEvolution/trip-count2.ll | 4 +- .../Analysis/ScalarEvolution/trip-count6.ll | 6 +- .../Analysis/ScalarEvolution/trip-count7.ll | 2 +- .../Analysis/ScalarEvolution/unknown_phis.ll | 4 +- .../ScalarEvolution/widenable-condition.ll | 2 +- .../Transforms/LoopSimplify/preserve-scev.ll | 2 +- .../PhaseOrdering/scev-custom-dl.ll | 16 ++--- .../SimpleLoopUnswitch/update-scev-2.ll | 2 +- 58 files changed, 340 insertions(+), 324 deletions(-) diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 57e3619ed84e..15c2965aede1 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -13469,6 +13469,14 @@ bool ScalarEvolution::hasLoopInvariantBackedgeTakenCount(const Loop *L) { return !isa(getBackedgeTakenCount(L)); } +/// When printing a top-level SCEV for trip counts, it's helpful to include +/// a type for constants which are otherwise hard to disambiguate. +static void PrintSCEVWithTypeHint(raw_ostream &OS, const SCEV* S) { + if (isa(S)) + OS << *S->getType() << " "; + OS << *S; +} + static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE, const Loop *L) { // Print all inner loops first @@ -13485,15 +13493,18 @@ static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE, OS << " "; auto *BTC = SE->getBackedgeTakenCount(L); - if (!isa(BTC)) - OS << "backedge-taken count is " << *BTC << "\n"; - else - OS << "Unpredictable backedge-taken count.\n"; + if (!isa(BTC)) { + OS << "backedge-taken count is "; + PrintSCEVWithTypeHint(OS, BTC); + } else + OS << "Unpredictable backedge-taken count."; + OS << "\n"; if (ExitingBlocks.size() > 1) for (BasicBlock *ExitingBlock : ExitingBlocks) { - OS << " exit count for " << ExitingBlock->getName() << ": " - << *SE->getExitCount(L, ExitingBlock) << "\n"; + OS << " exit count for " << ExitingBlock->getName() << ": "; + PrintSCEVWithTypeHint(OS, SE->getExitCount(L, ExitingBlock)); + OS << "\n"; } OS << "Loop "; @@ -13502,8 +13513,8 @@ static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE, auto *ConstantBTC = SE->getConstantMaxBackedgeTakenCount(L); if (!isa(ConstantBTC)) { - OS << "constant max backedge-taken count is " - << *ConstantBTC->getType() << " " << *ConstantBTC; + OS << "constant max backedge-taken count is "; + PrintSCEVWithTypeHint(OS, ConstantBTC); if (SE->isBackedgeTakenCountMaxOrZero(L)) OS << ", actual taken count either this or zero."; } else { @@ -13517,19 +13528,22 @@ static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE, auto *SymbolicBTC = SE->getSymbolicMaxBackedgeTakenCount(L); if (!isa(SymbolicBTC)) { - OS << "symbolic max backedge-taken count is " << *SymbolicBTC; + OS << "symbolic max backedge-taken count is "; + PrintSCEVWithTypeHint(OS, SymbolicBTC); if (SE->isBackedgeTakenCountMaxOrZero(L)) OS << ", actual taken count either this or zero."; } else { OS << "Unpredictable symbolic max backedge-taken count. "; } - OS << "\n"; + if (ExitingBlocks.size() > 1) for (BasicBlock *ExitingBlock : ExitingBlocks) { - OS << " symbolic max exit count for " << ExitingBlock->getName() << ": " - << *SE->getExitCount(L, ExitingBlock, ScalarEvolution::SymbolicMaximum) - << "\n"; + OS << " symbolic max exit count for " << ExitingBlock->getName() << ": "; + auto *ExitBTC = SE->getExitCount(L, ExitingBlock, + ScalarEvolution::SymbolicMaximum); + PrintSCEVWithTypeHint(OS, ExitBTC); + OS << "\n"; } SmallVector Preds; @@ -13538,10 +13552,12 @@ static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE, OS << "Loop "; L->getHeader()->printAsOperand(OS, /*PrintType=*/false); OS << ": "; - if (!isa(PBT)) - OS << "Predicated backedge-taken count is " << *PBT << "\n"; - else - OS << "Unpredictable predicated backedge-taken count.\n"; + if (!isa(PBT)) { + OS << "Predicated backedge-taken count is "; + PrintSCEVWithTypeHint(OS, PBT); + } else + OS << "Unpredictable predicated backedge-taken count."; + OS << "\n"; OS << " Predicates:\n"; for (const auto *P : Preds) P->print(OS, 4); diff --git a/llvm/test/Analysis/ScalarEvolution/2007-07-15-NegativeStride.ll b/llvm/test/Analysis/ScalarEvolution/2007-07-15-NegativeStride.ll index aade3499dba1..bee5d9df98d5 100644 --- a/llvm/test/Analysis/ScalarEvolution/2007-07-15-NegativeStride.ll +++ b/llvm/test/Analysis/ScalarEvolution/2007-07-15-NegativeStride.ll @@ -8,9 +8,9 @@ define void @loop(i32 %x) { ; CHECK-LABEL: 'loop' ; CHECK-NEXT: Determining loop execution counts for: @loop -; CHECK-NEXT: Loop %bb: backedge-taken count is 100 +; CHECK-NEXT: Loop %bb: backedge-taken count is i32 100 ; CHECK-NEXT: Loop %bb: constant max backedge-taken count is i32 100 -; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is 100 +; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is i32 100 ; CHECK-NEXT: Loop %bb: Trip multiple is 101 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/2007-09-27-LargeStepping.ll b/llvm/test/Analysis/ScalarEvolution/2007-09-27-LargeStepping.ll index f79d5daee741..8266b080af24 100644 --- a/llvm/test/Analysis/ScalarEvolution/2007-09-27-LargeStepping.ll +++ b/llvm/test/Analysis/ScalarEvolution/2007-09-27-LargeStepping.ll @@ -6,9 +6,9 @@ define i32 @f() { ; CHECK-LABEL: 'f' ; CHECK-NEXT: Determining loop execution counts for: @f -; CHECK-NEXT: Loop %bb5: backedge-taken count is 13 +; CHECK-NEXT: Loop %bb5: backedge-taken count is i32 13 ; CHECK-NEXT: Loop %bb5: constant max backedge-taken count is i32 13 -; CHECK-NEXT: Loop %bb5: symbolic max backedge-taken count is 13 +; CHECK-NEXT: Loop %bb5: symbolic max backedge-taken count is i32 13 ; CHECK-NEXT: Loop %bb5: Trip multiple is 14 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/2008-05-25-NegativeStepToZero.ll b/llvm/test/Analysis/ScalarEvolution/2008-05-25-NegativeStepToZero.ll index a9545f436b89..c8e876a61a84 100644 --- a/llvm/test/Analysis/ScalarEvolution/2008-05-25-NegativeStepToZero.ll +++ b/llvm/test/Analysis/ScalarEvolution/2008-05-25-NegativeStepToZero.ll @@ -6,9 +6,9 @@ define i32 @func_6() nounwind { ; CHECK-LABEL: 'func_6' ; CHECK-NEXT: Determining loop execution counts for: @func_6 -; CHECK-NEXT: Loop %bb5: backedge-taken count is 61 +; CHECK-NEXT: Loop %bb5: backedge-taken count is i8 61 ; CHECK-NEXT: Loop %bb5: constant max backedge-taken count is i8 61 -; CHECK-NEXT: Loop %bb5: symbolic max backedge-taken count is 61 +; CHECK-NEXT: Loop %bb5: symbolic max backedge-taken count is i8 61 ; CHECK-NEXT: Loop %bb5: Trip multiple is 62 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/2008-07-19-WrappingIV.ll b/llvm/test/Analysis/ScalarEvolution/2008-07-19-WrappingIV.ll index 45fe08a340eb..062611c78ebf 100644 --- a/llvm/test/Analysis/ScalarEvolution/2008-07-19-WrappingIV.ll +++ b/llvm/test/Analysis/ScalarEvolution/2008-07-19-WrappingIV.ll @@ -6,9 +6,9 @@ define void @fun() { ; CHECK-LABEL: 'fun' ; CHECK-NEXT: Determining loop execution counts for: @fun -; CHECK-NEXT: Loop %loop: backedge-taken count is 113 +; CHECK-NEXT: Loop %loop: backedge-taken count is i8 113 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 113 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 113 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i8 113 ; CHECK-NEXT: Loop %loop: Trip multiple is 114 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/2008-12-08-FiniteSGE.ll b/llvm/test/Analysis/ScalarEvolution/2008-12-08-FiniteSGE.ll index fab2e5495653..cdf62db22151 100644 --- a/llvm/test/Analysis/ScalarEvolution/2008-12-08-FiniteSGE.ll +++ b/llvm/test/Analysis/ScalarEvolution/2008-12-08-FiniteSGE.ll @@ -5,9 +5,9 @@ define i32 @foo(i32 %x, i32 %y, ptr %lam, ptr %alp) nounwind { ; CHECK-LABEL: 'foo' ; CHECK-NEXT: Determining loop execution counts for: @foo -; CHECK-NEXT: Loop %bb1: backedge-taken count is 255 +; CHECK-NEXT: Loop %bb1: backedge-taken count is i32 255 ; CHECK-NEXT: Loop %bb1: constant max backedge-taken count is i32 255 -; CHECK-NEXT: Loop %bb1: symbolic max backedge-taken count is 255 +; CHECK-NEXT: Loop %bb1: symbolic max backedge-taken count is i32 255 ; CHECK-NEXT: Loop %bb1: Trip multiple is 256 ; bb1.thread: diff --git a/llvm/test/Analysis/ScalarEvolution/2009-05-09-PointerEdgeCount.ll b/llvm/test/Analysis/ScalarEvolution/2009-05-09-PointerEdgeCount.ll index c683fbaa9aef..541513186212 100644 --- a/llvm/test/Analysis/ScalarEvolution/2009-05-09-PointerEdgeCount.ll +++ b/llvm/test/Analysis/ScalarEvolution/2009-05-09-PointerEdgeCount.ll @@ -11,9 +11,9 @@ target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:1 define void @_Z3foov() nounwind { ; CHECK-LABEL: '_Z3foov' ; CHECK-NEXT: Determining loop execution counts for: @_Z3foov -; CHECK-NEXT: Loop %bb1.i: backedge-taken count is 2 +; CHECK-NEXT: Loop %bb1.i: backedge-taken count is i64 2 ; CHECK-NEXT: Loop %bb1.i: constant max backedge-taken count is i64 2 -; CHECK-NEXT: Loop %bb1.i: symbolic max backedge-taken count is 2 +; CHECK-NEXT: Loop %bb1.i: symbolic max backedge-taken count is i64 2 ; CHECK-NEXT: Loop %bb1.i: Trip multiple is 3 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/SolveQuadraticEquation.ll b/llvm/test/Analysis/ScalarEvolution/SolveQuadraticEquation.ll index d87c2cf466a6..422a0b8cecf3 100644 --- a/llvm/test/Analysis/ScalarEvolution/SolveQuadraticEquation.ll +++ b/llvm/test/Analysis/ScalarEvolution/SolveQuadraticEquation.ll @@ -8,9 +8,9 @@ define void @test1(i32 %N) { ; CHECK-LABEL: 'test1' ; CHECK-NEXT: Determining loop execution counts for: @test1 -; CHECK-NEXT: Loop %bb3: backedge-taken count is 100 +; CHECK-NEXT: Loop %bb3: backedge-taken count is i32 100 ; CHECK-NEXT: Loop %bb3: constant max backedge-taken count is i32 100 -; CHECK-NEXT: Loop %bb3: symbolic max backedge-taken count is 100 +; CHECK-NEXT: Loop %bb3: symbolic max backedge-taken count is i32 100 ; CHECK-NEXT: Loop %bb3: Trip multiple is 101 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/addrec-computed-during-addrec-calculation.ll b/llvm/test/Analysis/ScalarEvolution/addrec-computed-during-addrec-calculation.ll index 411affc344c5..aab2c49e2973 100644 --- a/llvm/test/Analysis/ScalarEvolution/addrec-computed-during-addrec-calculation.ll +++ b/llvm/test/Analysis/ScalarEvolution/addrec-computed-during-addrec-calculation.ll @@ -28,10 +28,10 @@ define void @test(ptr %p) { ; CHECK-NEXT: Determining loop execution counts for: @test ; CHECK-NEXT: Loop %loop2: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop2: constant max backedge-taken count is i32 -1 -; CHECK-NEXT: Loop %loop2: symbolic max backedge-taken count is -1 -; CHECK-NEXT: Loop %loop3: backedge-taken count is false +; CHECK-NEXT: Loop %loop2: symbolic max backedge-taken count is i32 -1 +; CHECK-NEXT: Loop %loop3: backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop3: constant max backedge-taken count is i1 false -; CHECK-NEXT: Loop %loop3: symbolic max backedge-taken count is false +; CHECK-NEXT: Loop %loop3: symbolic max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop3: Trip multiple is 1 ; CHECK-NEXT: Loop %loop.header: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop.header: Unpredictable constant max backedge-taken count. diff --git a/llvm/test/Analysis/ScalarEvolution/alloca.ll b/llvm/test/Analysis/ScalarEvolution/alloca.ll index 39d43e7aaa4f..d4e00c69dcf6 100644 --- a/llvm/test/Analysis/ScalarEvolution/alloca.ll +++ b/llvm/test/Analysis/ScalarEvolution/alloca.ll @@ -43,9 +43,9 @@ define void @alloca_icmp_null_exit_count() { ; CHECK-NEXT: %and = and i1 %cmp1, %cmp2 ; CHECK-NEXT: --> (%cmp2 umin %cmp1) U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @alloca_icmp_null_exit_count -; CHECK-NEXT: Loop %loop: backedge-taken count is 2 +; CHECK-NEXT: Loop %loop: backedge-taken count is i64 2 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 2 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 2 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i64 2 ; CHECK-NEXT: Loop %loop: Trip multiple is 3 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info.ll index b78438b9bd67..1f475e80e562 100644 --- a/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info.ll +++ b/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info.ll @@ -13,9 +13,9 @@ define void @loop_guard_improves_exact_backedge_taken_count_1(i32 %conv) { ; CHECK-NEXT: %iv.next = add i64 %iv, 1 ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,2) S: [1,2) Exits: 1 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @loop_guard_improves_exact_backedge_taken_count_1 -; CHECK-NEXT: Loop %loop: backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: backedge-taken count is i64 0 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 0 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i64 0 ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/becount-invalidation.ll b/llvm/test/Analysis/ScalarEvolution/becount-invalidation.ll index b1cd95f1990c..0e37cf5efe3e 100644 --- a/llvm/test/Analysis/ScalarEvolution/becount-invalidation.ll +++ b/llvm/test/Analysis/ScalarEvolution/becount-invalidation.ll @@ -29,11 +29,11 @@ define void @test(ptr %arg) { ; CHECK-NEXT: Determining loop execution counts for: @test ; CHECK-NEXT: Loop %loop2.header: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for loop2.header: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: exit count for loop2.latch: false +; CHECK-NEXT: exit count for loop2.latch: i1 false ; CHECK-NEXT: Loop %loop2.header: constant max backedge-taken count is i1 false -; CHECK-NEXT: Loop %loop2.header: symbolic max backedge-taken count is false +; CHECK-NEXT: Loop %loop2.header: symbolic max backedge-taken count is i1 false ; CHECK-NEXT: symbolic max exit count for loop2.header: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: symbolic max exit count for loop2.latch: false +; CHECK-NEXT: symbolic max exit count for loop2.latch: i1 false ; CHECK-NEXT: Loop %loop.header: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop.header: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop.header: Unpredictable symbolic max backedge-taken count. diff --git a/llvm/test/Analysis/ScalarEvolution/exact_iter_count.ll b/llvm/test/Analysis/ScalarEvolution/exact_iter_count.ll index 9e6c6aa651b0..ff49ab280e89 100644 --- a/llvm/test/Analysis/ScalarEvolution/exact_iter_count.ll +++ b/llvm/test/Analysis/ScalarEvolution/exact_iter_count.ll @@ -5,13 +5,13 @@ define void @test_01() { ; CHECK-LABEL: 'test_01' ; CHECK-NEXT: Determining loop execution counts for: @test_01 -; CHECK-NEXT: Loop %loop: backedge-taken count is 50 -; CHECK-NEXT: exit count for loop: 50 -; CHECK-NEXT: exit count for backedge: 100 +; CHECK-NEXT: Loop %loop: backedge-taken count is i32 50 +; CHECK-NEXT: exit count for loop: i32 50 +; CHECK-NEXT: exit count for backedge: i32 100 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 50 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 50 -; CHECK-NEXT: symbolic max exit count for loop: 50 -; CHECK-NEXT: symbolic max exit count for backedge: 100 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 50 +; CHECK-NEXT: symbolic max exit count for loop: i32 50 +; CHECK-NEXT: symbolic max exit count for backedge: i32 100 ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -37,13 +37,13 @@ side.exit: define void @test_02(i1 %c) { ; CHECK-LABEL: 'test_02' ; CHECK-NEXT: Determining loop execution counts for: @test_02 -; CHECK-NEXT: Loop %loop: backedge-taken count is 50 -; CHECK-NEXT: exit count for merge: 50 -; CHECK-NEXT: exit count for backedge: 100 +; CHECK-NEXT: Loop %loop: backedge-taken count is i32 50 +; CHECK-NEXT: exit count for merge: i32 50 +; CHECK-NEXT: exit count for backedge: i32 100 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 50 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 50 -; CHECK-NEXT: symbolic max exit count for merge: 50 -; CHECK-NEXT: symbolic max exit count for backedge: 100 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 50 +; CHECK-NEXT: symbolic max exit count for merge: i32 50 +; CHECK-NEXT: symbolic max exit count for backedge: i32 100 ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll b/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll index 03c46f4826c6..21237f426693 100644 --- a/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll +++ b/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll @@ -8,9 +8,9 @@ define void @f_0() { ; ; CHECK-LABEL: 'f_0' ; CHECK-NEXT: Determining loop execution counts for: @f_0 -; CHECK-NEXT: Loop %for.body: backedge-taken count is 5 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i32 5 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 5 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 5 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i32 5 ; CHECK-NEXT: Loop %for.body: Trip multiple is 6 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/exit-count-select-safe.ll b/llvm/test/Analysis/ScalarEvolution/exit-count-select-safe.ll index 5b32b3ad246a..1043b2f6f56f 100644 --- a/llvm/test/Analysis/ScalarEvolution/exit-count-select-safe.ll +++ b/llvm/test/Analysis/ScalarEvolution/exit-count-select-safe.ll @@ -451,9 +451,9 @@ define i32 @computeSCEVAtScope(i32 %d.0) { ; CHECK-NEXT: Loop %for.cond: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %for.cond: symbolic max backedge-taken count is (-1 * %d.0) ; CHECK-NEXT: Loop %for.cond: Trip multiple is 1 -; CHECK-NEXT: Loop %for.cond4: backedge-taken count is 0 +; CHECK-NEXT: Loop %for.cond4: backedge-taken count is i32 0 ; CHECK-NEXT: Loop %for.cond4: constant max backedge-taken count is i32 0 -; CHECK-NEXT: Loop %for.cond4: symbolic max backedge-taken count is 0 +; CHECK-NEXT: Loop %for.cond4: symbolic max backedge-taken count is i32 0 ; CHECK-NEXT: Loop %for.cond4: Trip multiple is 1 ; CHECK-NEXT: Loop %while.cond: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %while.cond: Unpredictable constant max backedge-taken count. @@ -1249,9 +1249,9 @@ define i32 @logical_and_zero_arg1(i32 %n) { ; CHECK-NEXT: %cond = select i1 %cond_p0, i1 %cond_p1, i1 false ; CHECK-NEXT: --> (%cond_p0 umin_seq %cond_p1) U: full-set S: full-set Exits: false LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_zero_arg1 -; CHECK-NEXT: Loop %loop: backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: backedge-taken count is i32 0 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 0 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 0 ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1277,9 +1277,9 @@ define i32 @logical_and_zero_arg2(i32 %n) { ; CHECK-NEXT: %cond = select i1 %cond_p0, i1 %cond_p1, i1 false ; CHECK-NEXT: --> (%cond_p1 umin %cond_p0) U: full-set S: full-set Exits: false LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_zero_arg2 -; CHECK-NEXT: Loop %loop: backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: backedge-taken count is i32 0 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 0 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 0 ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/exit-count-select.ll b/llvm/test/Analysis/ScalarEvolution/exit-count-select.ll index d9ca46ba6787..6922c90158d8 100644 --- a/llvm/test/Analysis/ScalarEvolution/exit-count-select.ll +++ b/llvm/test/Analysis/ScalarEvolution/exit-count-select.ll @@ -74,9 +74,9 @@ define void @logical_and_zero(i32 %m) { ; CHECK-NEXT: %cond = select i1 %cond_i, i1 %cond_i2, i1 false ; CHECK-NEXT: --> (%cond_i umin_seq %cond_i2) U: full-set S: full-set Exits: false LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_and_zero -; CHECK-NEXT: Loop %loop: backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: backedge-taken count is i32 0 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 0 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 0 ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -196,9 +196,9 @@ define void @logical_or_zero(i32 %m) { ; CHECK-NEXT: %cond = select i1 %cond_i, i1 true, i1 %cond_i2 ; CHECK-NEXT: --> (true + ((true + %cond_i) umin_seq (true + %cond_i2))) U: full-set S: full-set Exits: true LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @logical_or_zero -; CHECK-NEXT: Loop %loop: backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: backedge-taken count is i32 0 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 0 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 0 ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/exponential-behavior.ll b/llvm/test/Analysis/ScalarEvolution/exponential-behavior.ll index 0b1854ecbd2e..397e1d5efcc7 100644 --- a/llvm/test/Analysis/ScalarEvolution/exponential-behavior.ll +++ b/llvm/test/Analysis/ScalarEvolution/exponential-behavior.ll @@ -6,11 +6,11 @@ define void @f(i32 %n, ptr %ptr) { ; CHECK-LABEL: 'f' ; CHECK-NEXT: Determining loop execution counts for: @f ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. -; CHECK-NEXT: exit count for loop: 0 +; CHECK-NEXT: exit count for loop: i32 0 ; CHECK-NEXT: exit count for be: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 0 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 -; CHECK-NEXT: symbolic max exit count for loop: 0 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 0 +; CHECK-NEXT: symbolic max exit count for loop: i32 0 ; CHECK-NEXT: symbolic max exit count for be: ***COULDNOTCOMPUTE*** ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/fold.ll b/llvm/test/Analysis/ScalarEvolution/fold.ll index 3b6df8e6d0a5..670523ca1bb5 100644 --- a/llvm/test/Analysis/ScalarEvolution/fold.ll +++ b/llvm/test/Analysis/ScalarEvolution/fold.ll @@ -77,9 +77,9 @@ define void @test4(i32 %x, i32 %y) { ; CHECK-NEXT: %I = add i32 %A, 1 ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,22) S: [1,22) Exits: 21 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test4 -; CHECK-NEXT: Loop %loop: backedge-taken count is 20 +; CHECK-NEXT: Loop %loop: backedge-taken count is i32 20 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 20 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 20 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 20 ; CHECK-NEXT: Loop %loop: Trip multiple is 21 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/huge-trip-multiple.ll b/llvm/test/Analysis/ScalarEvolution/huge-trip-multiple.ll index 2825dc4d1215..32b4d035b8be 100644 --- a/llvm/test/Analysis/ScalarEvolution/huge-trip-multiple.ll +++ b/llvm/test/Analysis/ScalarEvolution/huge-trip-multiple.ll @@ -14,9 +14,9 @@ define void @trip_count_4294967295() { ; CHECK-NEXT: %add = add nuw nsw i64 %i.02, 1 ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,4294967296) S: [1,4294967296) Exits: 4294967295 LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @trip_count_4294967295 -; CHECK-NEXT: Loop %for.body: backedge-taken count is 4294967294 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i64 4294967294 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 4294967294 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 4294967294 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i64 4294967294 ; CHECK-NEXT: Loop %for.body: Trip multiple is 4294967295 ; entry: @@ -41,9 +41,9 @@ define void @trip_count_4294967296() { ; CHECK-NEXT: %add = add nuw nsw i64 %i.02, 1 ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,4294967297) S: [1,4294967297) Exits: 4294967296 LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @trip_count_4294967296 -; CHECK-NEXT: Loop %for.body: backedge-taken count is 4294967295 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i64 4294967295 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 4294967295 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 4294967295 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i64 4294967295 ; CHECK-NEXT: Loop %for.body: Trip multiple is 2147483648 ; entry: @@ -68,9 +68,9 @@ define void @trip_count_8589935692() { ; CHECK-NEXT: %add = add nuw nsw i64 %i.02, 1 ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,8589934593) S: [1,8589934593) Exits: 8589934592 LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @trip_count_8589935692 -; CHECK-NEXT: Loop %for.body: backedge-taken count is 8589934591 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i64 8589934591 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 8589934591 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 8589934591 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i64 8589934591 ; CHECK-NEXT: Loop %for.body: Trip multiple is 2147483648 ; entry: @@ -95,9 +95,9 @@ define void @trip_count_9223372036854775808() { ; CHECK-NEXT: %add = add nuw nsw i64 %i.02, 1 ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,-9223372036854775807) S: [1,-9223372036854775807) Exits: -9223372036854775808 LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @trip_count_9223372036854775808 -; CHECK-NEXT: Loop %for.body: backedge-taken count is 9223372036854775807 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i64 9223372036854775807 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 9223372036854775807 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 9223372036854775807 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i64 9223372036854775807 ; CHECK-NEXT: Loop %for.body: Trip multiple is 2147483648 ; entry: @@ -122,9 +122,9 @@ define void @trip_count_18446744073709551615() { ; CHECK-NEXT: %add = add nuw nsw i64 %i.02, 1 ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,0) S: [1,0) Exits: -1 LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @trip_count_18446744073709551615 -; CHECK-NEXT: Loop %for.body: backedge-taken count is -2 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i64 -2 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 -2 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is -2 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i64 -2 ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/incorrect-exit-count.ll b/llvm/test/Analysis/ScalarEvolution/incorrect-exit-count.ll index f7cec4996e8f..b456f11bade6 100644 --- a/llvm/test/Analysis/ScalarEvolution/incorrect-exit-count.ll +++ b/llvm/test/Analysis/ScalarEvolution/incorrect-exit-count.ll @@ -58,28 +58,28 @@ define dso_local i32 @f() { ; CHECK-NEXT: --> {2,+,-1}<%outer.loop> U: [0,3) S: [0,3) Exits: <> LoopDispositions: { %outer.loop: Computable, %for.cond6: Invariant, %inner.loop: Invariant } ; CHECK-NEXT: Determining loop execution counts for: @f ; CHECK-NEXT: Loop %for.cond6: Unpredictable backedge-taken count. -; CHECK-NEXT: exit count for for.cond6: 0 +; CHECK-NEXT: exit count for for.cond6: i32 0 ; CHECK-NEXT: exit count for for.end: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: Loop %for.cond6: constant max backedge-taken count is i32 0 -; CHECK-NEXT: Loop %for.cond6: symbolic max backedge-taken count is 0 -; CHECK-NEXT: symbolic max exit count for for.cond6: 0 +; CHECK-NEXT: Loop %for.cond6: symbolic max backedge-taken count is i32 0 +; CHECK-NEXT: symbolic max exit count for for.cond6: i32 0 ; CHECK-NEXT: symbolic max exit count for for.end: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: Loop %inner.loop: Unpredictable backedge-taken count. -; CHECK-NEXT: exit count for inner.loop: 0 +; CHECK-NEXT: exit count for inner.loop: i32 0 ; CHECK-NEXT: exit count for for.end.3: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: Loop %inner.loop: constant max backedge-taken count is i32 0 -; CHECK-NEXT: Loop %inner.loop: symbolic max backedge-taken count is 0 -; CHECK-NEXT: symbolic max exit count for inner.loop: 0 +; CHECK-NEXT: Loop %inner.loop: symbolic max backedge-taken count is i32 0 +; CHECK-NEXT: symbolic max exit count for inner.loop: i32 0 ; CHECK-NEXT: symbolic max exit count for for.end.3: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: Loop %outer.loop: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for for.cond6: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for inner.loop: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: exit count for for.inc13.3: 2 +; CHECK-NEXT: exit count for for.inc13.3: i32 2 ; CHECK-NEXT: Loop %outer.loop: constant max backedge-taken count is i32 2 -; CHECK-NEXT: Loop %outer.loop: symbolic max backedge-taken count is 2 +; CHECK-NEXT: Loop %outer.loop: symbolic max backedge-taken count is i32 2 ; CHECK-NEXT: symbolic max exit count for for.cond6: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for inner.loop: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: symbolic max exit count for for.inc13.3: 2 +; CHECK-NEXT: symbolic max exit count for for.inc13.3: i32 2 ; entry: store i32 3, ptr @a, align 4 diff --git a/llvm/test/Analysis/ScalarEvolution/increasing-or-decreasing-iv.ll b/llvm/test/Analysis/ScalarEvolution/increasing-or-decreasing-iv.ll index 69bede4434b5..2264bfe4fce6 100644 --- a/llvm/test/Analysis/ScalarEvolution/increasing-or-decreasing-iv.ll +++ b/llvm/test/Analysis/ScalarEvolution/increasing-or-decreasing-iv.ll @@ -17,9 +17,9 @@ define void @f0(i1 %c) { ; CHECK-NEXT: %loop.iv.inc = add i32 %loop.iv, 1 ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,129) S: [1,129) Exits: 128 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @f0 -; CHECK-NEXT: Loop %loop: backedge-taken count is 127 +; CHECK-NEXT: Loop %loop: backedge-taken count is i32 127 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 127 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 127 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 127 ; CHECK-NEXT: Loop %loop: Trip multiple is 128 ; entry: @@ -83,9 +83,9 @@ define void @f1(i1 %c) { ; CHECK-NEXT: %loop.iv.inc = add i32 %loop.iv, 1 ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,17) S: [1,17) Exits: 16 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @f1 -; CHECK-NEXT: Loop %loop: backedge-taken count is 15 +; CHECK-NEXT: Loop %loop: backedge-taken count is i32 15 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 15 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 15 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 15 ; CHECK-NEXT: Loop %loop: Trip multiple is 16 ; entry: @@ -142,9 +142,9 @@ define void @f2(i1 %c) { ; CHECK-NEXT: %loop.iv.inc = add i32 %loop.iv, 1 ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,129) S: [1,129) Exits: 128 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @f2 -; CHECK-NEXT: Loop %loop: backedge-taken count is 127 +; CHECK-NEXT: Loop %loop: backedge-taken count is i32 127 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 127 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 127 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 127 ; CHECK-NEXT: Loop %loop: Trip multiple is 128 ; entry: @@ -183,9 +183,9 @@ define void @f3(i1 %c) { ; CHECK-NEXT: %loop.iv.inc = add i16 %loop.iv, 1 ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,129) S: [1,129) Exits: 128 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @f3 -; CHECK-NEXT: Loop %loop: backedge-taken count is 127 +; CHECK-NEXT: Loop %loop: backedge-taken count is i16 127 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i16 127 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 127 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i16 127 ; CHECK-NEXT: Loop %loop: Trip multiple is 128 ; entry: @@ -231,9 +231,9 @@ define void @f4(i1 %c) { ; CHECK-NEXT: %loop.iv.inc = add i32 %loop.iv, 1 ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,129) S: [1,129) Exits: 128 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @f4 -; CHECK-NEXT: Loop %loop: backedge-taken count is 127 +; CHECK-NEXT: Loop %loop: backedge-taken count is i32 127 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 127 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 127 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 127 ; CHECK-NEXT: Loop %loop: Trip multiple is 128 ; @@ -273,9 +273,9 @@ define void @f5(i1 %c) { ; CHECK-NEXT: %loop.iv.inc = add i16 %loop.iv, 1 ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,129) S: [1,129) Exits: 128 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @f5 -; CHECK-NEXT: Loop %loop: backedge-taken count is 127 +; CHECK-NEXT: Loop %loop: backedge-taken count is i16 127 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i16 127 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 127 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i16 127 ; CHECK-NEXT: Loop %loop: Trip multiple is 128 ; entry: @@ -317,9 +317,9 @@ define void @f6(i1 %c) { ; CHECK-NEXT: %loop.iv.inc = add i16 %loop.iv, 1 ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,129) S: [1,129) Exits: 128 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @f6 -; CHECK-NEXT: Loop %loop: backedge-taken count is 127 +; CHECK-NEXT: Loop %loop: backedge-taken count is i16 127 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i16 127 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 127 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i16 127 ; CHECK-NEXT: Loop %loop: Trip multiple is 128 ; entry: @@ -364,9 +364,9 @@ define void @f7(i1 %c) { ; CHECK-NEXT: %loop.iv.inc = add i16 %loop.iv, 1 ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,129) S: [1,129) Exits: 128 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @f7 -; CHECK-NEXT: Loop %loop: backedge-taken count is 127 +; CHECK-NEXT: Loop %loop: backedge-taken count is i16 127 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i16 127 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 127 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i16 127 ; CHECK-NEXT: Loop %loop: Trip multiple is 128 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/invalidation.ll b/llvm/test/Analysis/ScalarEvolution/invalidation.ll index 959720d2d6e4..6f5f977bc26e 100644 --- a/llvm/test/Analysis/ScalarEvolution/invalidation.ll +++ b/llvm/test/Analysis/ScalarEvolution/invalidation.ll @@ -39,7 +39,7 @@ target triple = "x86_64-unknown-linux-gnu" ; debug pass printing continuing to match, ASan and other tools can catch it. define void @test(i32 %n) { ; CHECK-LABEL: Classifying expressions for: @test -; CHECK: Loop %loop: backedge-taken count is 14 +; CHECK: Loop %loop: backedge-taken count is i32 14 ; CHECK: Loop %loop: constant max backedge-taken count is i32 14 entry: diff --git a/llvm/test/Analysis/ScalarEvolution/load.ll b/llvm/test/Analysis/ScalarEvolution/load.ll index 6bcb09b3bb61..0f0bc1ef1bf2 100644 --- a/llvm/test/Analysis/ScalarEvolution/load.ll +++ b/llvm/test/Analysis/ScalarEvolution/load.ll @@ -30,9 +30,9 @@ define i32 @test1() nounwind readnone { ; CHECK-NEXT: %inc = add nsw i32 %i.03, 1 ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,51) S: [1,51) Exits: 50 LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test1 -; CHECK-NEXT: Loop %for.body: backedge-taken count is 49 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i32 49 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 49 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 49 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i32 49 ; CHECK-NEXT: Loop %for.body: Trip multiple is 50 ; entry: @@ -82,9 +82,9 @@ define i32 @test2() nounwind uwtable readonly { ; CHECK-NEXT: %1 = load ptr, ptr %next, align 8 ; CHECK-NEXT: --> %1 U: full-set S: full-set Exits: null LoopDispositions: { %for.body: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test2 -; CHECK-NEXT: Loop %for.body: backedge-taken count is 4 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i32 4 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 4 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 4 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i32 4 ; CHECK-NEXT: Loop %for.body: Trip multiple is 5 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/logical-operations.ll b/llvm/test/Analysis/ScalarEvolution/logical-operations.ll index b8ed560a49de..8f836384d7fc 100644 --- a/llvm/test/Analysis/ScalarEvolution/logical-operations.ll +++ b/llvm/test/Analysis/ScalarEvolution/logical-operations.ll @@ -431,9 +431,9 @@ define ptr @tautological_select_like_phi(i32 %tc) { ; CHECK-NEXT: %iv.next = add i32 %iv, 1 ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,102) S: [1,102) Exits: 101 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @tautological_select_like_phi -; CHECK-NEXT: Loop %loop: backedge-taken count is 100 +; CHECK-NEXT: Loop %loop: backedge-taken count is i32 100 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 100 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 100 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 100 ; CHECK-NEXT: Loop %loop: Trip multiple is 101 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-rewrite-expressions.ll b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-rewrite-expressions.ll index 81b08694bd79..58044915ae87 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-rewrite-expressions.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-rewrite-expressions.ll @@ -244,9 +244,9 @@ define i32 @rewrite_zext_with_info_from_icmp_ne(i32 %N) { ; CHECK-NEXT: %iv.next = add i64 %iv, 4 ; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,5) S: [4,5) Exits: 4 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_zext_with_info_from_icmp_ne -; CHECK-NEXT: Loop %loop: backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: backedge-taken count is i64 0 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 0 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i64 0 ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll index 845494460b23..413bd21554c9 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll @@ -1549,9 +1549,9 @@ define i32 @ptr_induction_ult_1(ptr %a, ptr %b) { ; CHECK-NEXT: %ptr.iv.next = getelementptr i32, ptr %ptr.iv, i64 1 ; CHECK-NEXT: --> {(4 + %a),+,4}<%loop> U: full-set S: full-set Exits: (4 + %a) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @ptr_induction_ult_1 -; CHECK-NEXT: Loop %loop: backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: backedge-taken count is i64 0 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 0 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i64 0 ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -1607,9 +1607,9 @@ define void @gep_addrec_nw(ptr %a) { ; CHECK-NEXT: %uglygep2 = getelementptr i8, ptr %lsr.iv1, i64 4 ; CHECK-NEXT: --> {(4 + %a),+,4}<%for.body> U: full-set S: full-set Exits: (1516 + %a) LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @gep_addrec_nw -; CHECK-NEXT: Loop %for.body: backedge-taken count is 378 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i64 378 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 378 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 378 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i64 378 ; CHECK-NEXT: Loop %for.body: Trip multiple is 379 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/max-trip-count.ll b/llvm/test/Analysis/ScalarEvolution/max-trip-count.ll index 45552b13ed4f..501aa963f929 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-trip-count.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-trip-count.ll @@ -53,11 +53,11 @@ define i32 @main() nounwind { ; CHECK-LABEL: 'main' ; CHECK-NEXT: Determining loop execution counts for: @main ; CHECK-NEXT: Loop %for.cond: Unpredictable backedge-taken count. -; CHECK-NEXT: exit count for for.cond: 5 +; CHECK-NEXT: exit count for for.cond: i32 5 ; CHECK-NEXT: exit count for for.body: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: Loop %for.cond: constant max backedge-taken count is i32 5 -; CHECK-NEXT: Loop %for.cond: symbolic max backedge-taken count is 5 -; CHECK-NEXT: symbolic max exit count for for.cond: 5 +; CHECK-NEXT: Loop %for.cond: symbolic max backedge-taken count is i32 5 +; CHECK-NEXT: symbolic max exit count for for.cond: i32 5 ; CHECK-NEXT: symbolic max exit count for for.body: ***COULDNOTCOMPUTE*** ; entry: @@ -126,11 +126,11 @@ define i32 @pr19799() { ; CHECK-NEXT: Determining loop execution counts for: @pr19799 ; CHECK-NEXT: Loop %for.body.i: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for for.body.i: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: exit count for for.cond.i: 1 +; CHECK-NEXT: exit count for for.cond.i: i32 1 ; CHECK-NEXT: Loop %for.body.i: constant max backedge-taken count is i32 1 -; CHECK-NEXT: Loop %for.body.i: symbolic max backedge-taken count is 1 +; CHECK-NEXT: Loop %for.body.i: symbolic max backedge-taken count is i32 1 ; CHECK-NEXT: symbolic max exit count for for.body.i: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: symbolic max exit count for for.cond.i: 1 +; CHECK-NEXT: symbolic max exit count for for.cond.i: i32 1 ; entry: store i32 -1, ptr @a, align 4 @@ -159,11 +159,11 @@ define i32 @pr18886() { ; CHECK-NEXT: Determining loop execution counts for: @pr18886 ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for for.body: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: exit count for for.cond: 3 +; CHECK-NEXT: exit count for for.cond: i64 3 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 3 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 3 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i64 3 ; CHECK-NEXT: symbolic max exit count for for.body: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: symbolic max exit count for for.cond: 3 +; CHECK-NEXT: symbolic max exit count for for.cond: i64 3 ; entry: store i64 -21, ptr @aa, align 8 @@ -228,13 +228,13 @@ bar.exit: ; preds = %for.cond.i, %for.bo define i32 @two_mustexit() { ; CHECK-LABEL: 'two_mustexit' ; CHECK-NEXT: Determining loop execution counts for: @two_mustexit -; CHECK-NEXT: Loop %for.body.i: backedge-taken count is 1 -; CHECK-NEXT: exit count for for.body.i: 1 -; CHECK-NEXT: exit count for for.cond.i: 2 +; CHECK-NEXT: Loop %for.body.i: backedge-taken count is i32 1 +; CHECK-NEXT: exit count for for.body.i: i32 1 +; CHECK-NEXT: exit count for for.cond.i: i32 2 ; CHECK-NEXT: Loop %for.body.i: constant max backedge-taken count is i32 1 -; CHECK-NEXT: Loop %for.body.i: symbolic max backedge-taken count is 1 -; CHECK-NEXT: symbolic max exit count for for.body.i: 1 -; CHECK-NEXT: symbolic max exit count for for.cond.i: 2 +; CHECK-NEXT: Loop %for.body.i: symbolic max backedge-taken count is i32 1 +; CHECK-NEXT: symbolic max exit count for for.body.i: i32 1 +; CHECK-NEXT: symbolic max exit count for for.cond.i: i32 2 ; CHECK-NEXT: Loop %for.body.i: Trip multiple is 1 ; entry: @@ -364,7 +364,7 @@ define void @changing_end_bound(ptr %n_addr, ptr %addr) { ; CHECK-NEXT: Determining loop execution counts for: @changing_end_bound ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 2147483646 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 2147483646 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 2147483646 ; entry: br label %loop @@ -392,7 +392,7 @@ define void @changing_end_bound2(i32 %start, ptr %n_addr, ptr %addr) { ; CHECK-NEXT: Determining loop execution counts for: @changing_end_bound2 ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is -1 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 -1 ; entry: br label %loop @@ -418,7 +418,7 @@ define void @changing_end_bound3(i32 %start, ptr %n_addr, ptr %addr) { ; CHECK-NEXT: Determining loop execution counts for: @changing_end_bound3 ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 1073741823 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 1073741823 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 1073741823 ; entry: br label %loop @@ -547,9 +547,9 @@ loop.exit: define void @max_overflow_se(i8 %n) mustprogress { ; CHECK-LABEL: 'max_overflow_se' ; CHECK-NEXT: Determining loop execution counts for: @max_overflow_se -; CHECK-NEXT: Loop %loop: backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: backedge-taken count is i8 0 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 0 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 0 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i8 0 ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -571,11 +571,11 @@ define void @max_overflow_me(i8 %n) mustprogress { ; CHECK-LABEL: 'max_overflow_me' ; CHECK-NEXT: Determining loop execution counts for: @max_overflow_me ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. -; CHECK-NEXT: exit count for loop: 1 +; CHECK-NEXT: exit count for loop: i8 1 ; CHECK-NEXT: exit count for latch: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 1 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 1 -; CHECK-NEXT: symbolic max exit count for loop: 1 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i8 1 +; CHECK-NEXT: symbolic max exit count for loop: i8 1 ; CHECK-NEXT: symbolic max exit count for latch: ***COULDNOTCOMPUTE*** ; entry: @@ -601,9 +601,9 @@ exit: define void @bool_stride(i1 %s, i1 %n) mustprogress { ; CHECK-LABEL: 'bool_stride' ; CHECK-NEXT: Determining loop execution counts for: @bool_stride -; CHECK-NEXT: Loop %loop: backedge-taken count is false +; CHECK-NEXT: Loop %loop: backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -625,9 +625,9 @@ exit: define void @ne_zero_max_btc(i32 %a) { ; CHECK-LABEL: 'ne_zero_max_btc' ; CHECK-NEXT: Determining loop execution counts for: @ne_zero_max_btc -; CHECK-NEXT: Loop %for.body: backedge-taken count is 0 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i64 0 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 0 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 0 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i64 0 ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/ne-overflow.ll b/llvm/test/Analysis/ScalarEvolution/ne-overflow.ll index 7b14807364a4..82b4d0e4fb48 100644 --- a/llvm/test/Analysis/ScalarEvolution/ne-overflow.ll +++ b/llvm/test/Analysis/ScalarEvolution/ne-overflow.ll @@ -207,11 +207,11 @@ define void @test_other_exit(i32 %N) mustprogress { ; CHECK-LABEL: 'test_other_exit' ; CHECK-NEXT: Determining loop execution counts for: @test_other_exit ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. -; CHECK-NEXT: exit count for for.body: 9 +; CHECK-NEXT: exit count for for.body: i32 9 ; CHECK-NEXT: exit count for for.latch: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 9 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 9 -; CHECK-NEXT: symbolic max exit count for for.body: 9 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i32 9 +; CHECK-NEXT: symbolic max exit count for for.body: i32 9 ; CHECK-NEXT: symbolic max exit count for for.latch: ***COULDNOTCOMPUTE*** ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/nsw.ll b/llvm/test/Analysis/ScalarEvolution/nsw.ll index 275d3f19ab13..4d668d1ffef1 100644 --- a/llvm/test/Analysis/ScalarEvolution/nsw.ll +++ b/llvm/test/Analysis/ScalarEvolution/nsw.ll @@ -175,9 +175,9 @@ define i32 @PR12375(ptr readnone %arg) { ; CHECK-NEXT: %tmp5 = getelementptr inbounds i32, ptr %tmp2, i64 1 ; CHECK-NEXT: --> {(4 + %arg),+,4}<%bb1> U: [4,0) S: [4,0) Exits: (8 + %arg) LoopDispositions: { %bb1: Computable } ; CHECK-NEXT: Determining loop execution counts for: @PR12375 -; CHECK-NEXT: Loop %bb1: backedge-taken count is 1 +; CHECK-NEXT: Loop %bb1: backedge-taken count is i64 1 ; CHECK-NEXT: Loop %bb1: constant max backedge-taken count is i64 1 -; CHECK-NEXT: Loop %bb1: symbolic max backedge-taken count is 1 +; CHECK-NEXT: Loop %bb1: symbolic max backedge-taken count is i64 1 ; CHECK-NEXT: Loop %bb1: Trip multiple is 2 ; bb: @@ -431,9 +431,9 @@ define void @pr66066() { ; CHECK-NEXT: %shl = shl i8 %iv, 7 ; CHECK-NEXT: --> {-128,+,-128}<%loop> U: [0,-127) S: [-128,1) Exits: 0 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @pr66066 -; CHECK-NEXT: Loop %loop: backedge-taken count is 1 +; CHECK-NEXT: Loop %loop: backedge-taken count is i8 1 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 1 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 1 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i8 1 ; CHECK-NEXT: Loop %loop: Trip multiple is 2 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/overflow-intrinsics-trip-count.ll b/llvm/test/Analysis/ScalarEvolution/overflow-intrinsics-trip-count.ll index 5d1c5b5fb6f9..15ba27682a12 100644 --- a/llvm/test/Analysis/ScalarEvolution/overflow-intrinsics-trip-count.ll +++ b/llvm/test/Analysis/ScalarEvolution/overflow-intrinsics-trip-count.ll @@ -11,9 +11,9 @@ declare { i16, i1 } @llvm.umul.with.overflow.i16(i16, i16) nounwind readnone define void @uadd_exhaustive() { ; CHECK-LABEL: 'uadd_exhaustive' ; CHECK-NEXT: Determining loop execution counts for: @uadd_exhaustive -; CHECK-NEXT: Loop %for.body: backedge-taken count is 35 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i16 35 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 35 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 35 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i16 35 ; CHECK-NEXT: Loop %for.body: Trip multiple is 36 ; entry: @@ -36,9 +36,9 @@ for.end: ; preds = %for.body, %entry define void @sadd_exhaustive() { ; CHECK-LABEL: 'sadd_exhaustive' ; CHECK-NEXT: Determining loop execution counts for: @sadd_exhaustive -; CHECK-NEXT: Loop %for.body: backedge-taken count is 67 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i16 67 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 67 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 67 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i16 67 ; CHECK-NEXT: Loop %for.body: Trip multiple is 68 ; entry: @@ -61,9 +61,9 @@ for.end: ; preds = %for.body, %entry define void @usub_exhaustive() { ; CHECK-LABEL: 'usub_exhaustive' ; CHECK-NEXT: Determining loop execution counts for: @usub_exhaustive -; CHECK-NEXT: Loop %for.body: backedge-taken count is 50 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i16 50 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 50 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 50 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i16 50 ; CHECK-NEXT: Loop %for.body: Trip multiple is 51 ; entry: @@ -86,9 +86,9 @@ for.end: ; preds = %for.body, %entry define void @ssub_exhaustive() { ; CHECK-LABEL: 'ssub_exhaustive' ; CHECK-NEXT: Determining loop execution counts for: @ssub_exhaustive -; CHECK-NEXT: Loop %for.body: backedge-taken count is 68 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i16 68 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i16 68 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 68 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i16 68 ; CHECK-NEXT: Loop %for.body: Trip multiple is 69 ; entry: @@ -111,9 +111,9 @@ for.end: ; preds = %for.body, %entry define void @smul_exhaustive() { ; CHECK-LABEL: 'smul_exhaustive' ; CHECK-NEXT: Determining loop execution counts for: @smul_exhaustive -; CHECK-NEXT: Loop %for.body: backedge-taken count is 14 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i32 14 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 14 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 14 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i32 14 ; CHECK-NEXT: Loop %for.body: Trip multiple is 15 ; entry: @@ -136,9 +136,9 @@ for.end: ; preds = %for.body, %entry define void @umul_exhaustive() { ; CHECK-LABEL: 'umul_exhaustive' ; CHECK-NEXT: Determining loop execution counts for: @umul_exhaustive -; CHECK-NEXT: Loop %for.body: backedge-taken count is 15 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i32 15 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 15 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 15 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i32 15 ; CHECK-NEXT: Loop %for.body: Trip multiple is 16 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/pr25369.ll b/llvm/test/Analysis/ScalarEvolution/pr25369.ll index 2dc3a1272cc1..5783ef24b8a0 100644 --- a/llvm/test/Analysis/ScalarEvolution/pr25369.ll +++ b/llvm/test/Analysis/ScalarEvolution/pr25369.ll @@ -12,9 +12,9 @@ define void @hoge1() { ; CHECK-NEXT: Loop %bb13: constant max backedge-taken count is i32 -1 ; CHECK-NEXT: Loop %bb13: symbolic max backedge-taken count is (-2 + (2 * undef) + %tmp7 + %tmp6) ; CHECK-NEXT: Loop %bb13: Trip multiple is 1 -; CHECK-NEXT: Loop %bb4: backedge-taken count is 20 +; CHECK-NEXT: Loop %bb4: backedge-taken count is i64 20 ; CHECK-NEXT: Loop %bb4: constant max backedge-taken count is i64 20 -; CHECK-NEXT: Loop %bb4: symbolic max backedge-taken count is 20 +; CHECK-NEXT: Loop %bb4: symbolic max backedge-taken count is i64 20 ; CHECK-NEXT: Loop %bb4: Trip multiple is 21 ; CHECK-NEXT: Loop %bb2: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %bb2: Unpredictable constant max backedge-taken count. diff --git a/llvm/test/Analysis/ScalarEvolution/pr34538.ll b/llvm/test/Analysis/ScalarEvolution/pr34538.ll index 03fcbbcfd7ca..fe482a82d699 100644 --- a/llvm/test/Analysis/ScalarEvolution/pr34538.ll +++ b/llvm/test/Analysis/ScalarEvolution/pr34538.ll @@ -4,9 +4,9 @@ define i32 @pr34538() local_unnamed_addr #0 { ; CHECK-LABEL: 'pr34538' ; CHECK-NEXT: Determining loop execution counts for: @pr34538 -; CHECK-NEXT: Loop %do.body: backedge-taken count is 10000 +; CHECK-NEXT: Loop %do.body: backedge-taken count is i32 10000 ; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is i32 10000 -; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is 10000 +; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is i32 10000 ; CHECK-NEXT: Loop %do.body: Trip multiple is 10001 ; entry: @@ -27,9 +27,9 @@ do.end: ; preds = %do.body define i32 @foo() { ; CHECK-LABEL: 'foo' ; CHECK-NEXT: Determining loop execution counts for: @foo -; CHECK-NEXT: Loop %do.body: backedge-taken count is 5000 +; CHECK-NEXT: Loop %do.body: backedge-taken count is i32 5000 ; CHECK-NEXT: Loop %do.body: constant max backedge-taken count is i32 5000 -; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is 5000 +; CHECK-NEXT: Loop %do.body: symbolic max backedge-taken count is i32 5000 ; CHECK-NEXT: Loop %do.body: Trip multiple is 5001 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/pr48225.ll b/llvm/test/Analysis/ScalarEvolution/pr48225.ll index 71ca293e311a..157e98f99aa9 100644 --- a/llvm/test/Analysis/ScalarEvolution/pr48225.ll +++ b/llvm/test/Analysis/ScalarEvolution/pr48225.ll @@ -22,11 +22,11 @@ define void @test_and(i1 %boolcond) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,4) S: [1,4) Exits: <> LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_and ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. -; CHECK-NEXT: exit count for loop: 2 +; CHECK-NEXT: exit count for loop: i32 2 ; CHECK-NEXT: exit count for backedge: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 2 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 2 -; CHECK-NEXT: symbolic max exit count for loop: 2 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 2 +; CHECK-NEXT: symbolic max exit count for loop: i32 2 ; CHECK-NEXT: symbolic max exit count for backedge: ***COULDNOTCOMPUTE*** ; entry: @@ -71,11 +71,11 @@ define void @test_or(i1 %boolcond) { ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,4) S: [1,4) Exits: <> LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_or ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. -; CHECK-NEXT: exit count for loop: 2 +; CHECK-NEXT: exit count for loop: i32 2 ; CHECK-NEXT: exit count for backedge: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 2 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 2 -; CHECK-NEXT: symbolic max exit count for loop: 2 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 2 +; CHECK-NEXT: symbolic max exit count for loop: i32 2 ; CHECK-NEXT: symbolic max exit count for backedge: ***COULDNOTCOMPUTE*** ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/range-signedness.ll b/llvm/test/Analysis/ScalarEvolution/range-signedness.ll index 4484bfc1924b..2b91ac1109ed 100644 --- a/llvm/test/Analysis/ScalarEvolution/range-signedness.ll +++ b/llvm/test/Analysis/ScalarEvolution/range-signedness.ll @@ -38,9 +38,9 @@ define void @y(ptr %addr) { ; CHECK-NEXT: %idx.inc = add i8 %idx, 1 ; CHECK-NEXT: --> {-4,+,1}<%loop> U: [-4,7) S: [-4,7) Exits: 6 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @y -; CHECK-NEXT: Loop %loop: backedge-taken count is 10 +; CHECK-NEXT: Loop %loop: backedge-taken count is i8 10 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i8 10 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 10 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i8 10 ; CHECK-NEXT: Loop %loop: Trip multiple is 11 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/sext-add-inreg-loop.ll b/llvm/test/Analysis/ScalarEvolution/sext-add-inreg-loop.ll index db0bac933086..1971f668a5e2 100644 --- a/llvm/test/Analysis/ScalarEvolution/sext-add-inreg-loop.ll +++ b/llvm/test/Analysis/ScalarEvolution/sext-add-inreg-loop.ll @@ -23,9 +23,9 @@ define dso_local i32 @test_loop(ptr nocapture noundef readonly %x) { ; CHECK-NEXT: %inc = add nuw nsw i64 %i.03, 1 ; CHECK-NEXT: --> {2,+,1}<%for.body> U: [2,11) S: [2,11) Exits: 10 LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_loop -; CHECK-NEXT: Loop %for.body: backedge-taken count is 8 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i64 8 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 8 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 8 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i64 8 ; CHECK-NEXT: Loop %for.body: Trip multiple is 9 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/sext-to-zext.ll b/llvm/test/Analysis/ScalarEvolution/sext-to-zext.ll index f7d01974343d..3725bd9faa8e 100644 --- a/llvm/test/Analysis/ScalarEvolution/sext-to-zext.ll +++ b/llvm/test/Analysis/ScalarEvolution/sext-to-zext.ll @@ -19,9 +19,9 @@ define void @f(i1 %c) { ; CHECK-NEXT: %iv.sext = sext i32 %iv to i64 ; CHECK-NEXT: --> {(zext i32 %start to i64),+,(sext i32 %step to i64)}<%loop> U: [0,101) S: [0,101) Exits: ((zext i32 %start to i64) + (99 * (sext i32 %step to i64))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @f -; CHECK-NEXT: Loop %loop: backedge-taken count is 99 +; CHECK-NEXT: Loop %loop: backedge-taken count is i32 99 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 99 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 99 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 99 ; CHECK-NEXT: Loop %loop: Trip multiple is 100 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/shift-op.ll b/llvm/test/Analysis/ScalarEvolution/shift-op.ll index c3984673cfc7..150e4c996716 100644 --- a/llvm/test/Analysis/ScalarEvolution/shift-op.ll +++ b/llvm/test/Analysis/ScalarEvolution/shift-op.ll @@ -7,7 +7,7 @@ define void @test0(i32 %init) { ; CHECK-NEXT: Determining loop execution counts for: @test0 ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 32 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 32 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 32 ; entry: br label %loop @@ -28,7 +28,7 @@ define void @test1(i32 %init) { ; CHECK-NEXT: Determining loop execution counts for: @test1 ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 32 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 32 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 32 ; entry: br label %loop @@ -72,7 +72,7 @@ define void @test3(ptr %init.ptr) { ; CHECK-NEXT: Determining loop execution counts for: @test3 ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 32 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 32 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 32 ; entry: %init = load i32, ptr %init.ptr, !range !0 @@ -94,7 +94,7 @@ define void @test4(ptr %init.ptr) { ; CHECK-NEXT: Determining loop execution counts for: @test4 ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 32 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 32 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 32 ; entry: %init = load i32, ptr %init.ptr, !range !1 @@ -161,7 +161,7 @@ define void @test7(i32 %init) { ; CHECK-NEXT: Determining loop execution counts for: @test7 ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 32 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 32 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 32 ; entry: br label %loop diff --git a/llvm/test/Analysis/ScalarEvolution/shift-recurrences.ll b/llvm/test/Analysis/ScalarEvolution/shift-recurrences.ll index 2362741581ef..9806fd3d57f6 100644 --- a/llvm/test/Analysis/ScalarEvolution/shift-recurrences.ll +++ b/llvm/test/Analysis/ScalarEvolution/shift-recurrences.ll @@ -198,9 +198,9 @@ define void @test_shl2() { ; CHECK-NEXT: %iv.shl.next = shl i64 %iv.shl, 1 ; CHECK-NEXT: --> (2 * %iv.shl) U: [8,129) S: [8,129) Exits: 128 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_shl2 -; CHECK-NEXT: Loop %loop: backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -231,9 +231,9 @@ define void @test_shl3(i1 %c) { ; CHECK-NEXT: %iv.shl.next = shl i64 %iv.shl, %shiftamt ; CHECK-NEXT: --> %iv.shl.next U: [0,-3) S: [-9223372036854775808,9223372036854775805) Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_shl3 -; CHECK-NEXT: Loop %loop: backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -263,9 +263,9 @@ define void @test_shl4() { ; CHECK-NEXT: %iv.shl.next = shl i64 %iv.shl, 1 ; CHECK-NEXT: --> (2 * %iv.shl) U: [8,-9223372036854775807) S: [-9223372036854775808,9223372036854775801) Exits: -9223372036854775808 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_shl4 -; CHECK-NEXT: Loop %loop: backedge-taken count is 60 +; CHECK-NEXT: Loop %loop: backedge-taken count is i64 60 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 60 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 60 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i64 60 ; CHECK-NEXT: Loop %loop: Trip multiple is 61 ; entry: @@ -294,9 +294,9 @@ define void @test_shl5() { ; CHECK-NEXT: %iv.shl.next = shl i64 %iv.shl, 1 ; CHECK-NEXT: --> (2 * %iv.shl) U: [0,-7) S: [-9223372036854775808,9223372036854775801) Exits: 0 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_shl5 -; CHECK-NEXT: Loop %loop: backedge-taken count is 61 +; CHECK-NEXT: Loop %loop: backedge-taken count is i64 61 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 61 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 61 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i64 61 ; CHECK-NEXT: Loop %loop: Trip multiple is 62 ; entry: @@ -327,9 +327,9 @@ define void @test_shl6(i1 %c) { ; CHECK-NEXT: %iv.shl.next = shl i64 %iv.shl, %shiftamt ; CHECK-NEXT: --> %iv.shl.next U: [0,-3) S: [-9223372036854775808,9223372036854775805) Exits: 16 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_shl6 -; CHECK-NEXT: Loop %loop: backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -359,9 +359,9 @@ define void @test_shl7(i1 %c, i64 %shiftamt) { ; CHECK-NEXT: %iv.shl.next = shl i64 %iv.shl, %shiftamt ; CHECK-NEXT: --> %iv.shl.next U: [0,-3) S: [-9223372036854775808,9223372036854775805) Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_shl7 -; CHECK-NEXT: Loop %loop: backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -502,9 +502,9 @@ define void @test_ashr_tc_positive() { ; CHECK-NEXT: %iv.ashr.next = ashr i64 %iv.ashr, 1 ; CHECK-NEXT: --> %iv.ashr.next U: [0,512) S: [0,512) Exits: 31 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_ashr_tc_positive -; CHECK-NEXT: Loop %loop: backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -532,9 +532,9 @@ define void @test_ashr_tc_negative() { ; CHECK-NEXT: %iv.ashr.next = ashr i8 %iv.ashr, 1 ; CHECK-NEXT: --> %iv.ashr.next U: [-64,0) S: [-64,0) Exits: -4 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_ashr_tc_negative -; CHECK-NEXT: Loop %loop: backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -564,9 +564,9 @@ define void @test_ashr_tc_either(i1 %a) { ; CHECK-NEXT: %iv.ashr.next = ashr i8 %iv.ashr, 1 ; CHECK-NEXT: --> %iv.ashr.next U: [-16,16) S: [-16,16) Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_ashr_tc_either -; CHECK-NEXT: Loop %loop: backedge-taken count is 60 +; CHECK-NEXT: Loop %loop: backedge-taken count is i64 60 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 60 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 60 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i64 60 ; CHECK-NEXT: Loop %loop: Trip multiple is 61 ; entry: @@ -595,9 +595,9 @@ define void @test_ashr_zero_shift() { ; CHECK-NEXT: %iv.ashr.next = ashr i64 %iv.ashr, 0 ; CHECK-NEXT: --> %iv.ashr U: [1023,1024) S: [1023,1024) Exits: 1023 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_ashr_zero_shift -; CHECK-NEXT: Loop %loop: backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -625,9 +625,9 @@ define void @test_lshr_tc_positive() { ; CHECK-NEXT: %iv.lshr.next = lshr i64 %iv.lshr, 1 ; CHECK-NEXT: --> (%iv.lshr /u 2) U: [31,512) S: [31,512) Exits: 31 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_lshr_tc_positive -; CHECK-NEXT: Loop %loop: backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -655,9 +655,9 @@ define void @test_lshr_tc_negative() { ; CHECK-NEXT: %iv.lshr.next = lshr i8 %iv.lshr, 1 ; CHECK-NEXT: --> (%iv.lshr /u 2) U: [7,-128) S: [7,-128) Exits: 7 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_lshr_tc_negative -; CHECK-NEXT: Loop %loop: backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -687,9 +687,9 @@ define void @test_lshr_tc_either(i1 %a) { ; CHECK-NEXT: %iv.lshr.next = lshr i8 %iv.lshr, 1 ; CHECK-NEXT: --> (%iv.lshr /u 2) U: [0,-128) S: [0,-128) Exits: <> LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_lshr_tc_either -; CHECK-NEXT: Loop %loop: backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -718,9 +718,9 @@ define void @test_lshr_zero_shift() { ; CHECK-NEXT: %iv.lshr.next = lshr i64 %iv.lshr, 0 ; CHECK-NEXT: --> %iv.lshr U: [1023,1024) S: [1023,1024) Exits: 1023 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_lshr_zero_shift -; CHECK-NEXT: Loop %loop: backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -749,9 +749,9 @@ define void @test_lshr_power_of_2_start() { ; CHECK-NEXT: %iv.lshr.next = lshr i64 %iv.lshr, 2 ; CHECK-NEXT: --> (%iv.lshr /u 4) U: [1,257) S: [1,257) Exits: 1 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_lshr_power_of_2_start -; CHECK-NEXT: Loop %loop: backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -780,9 +780,9 @@ define void @test_lshr_arbitrary_start() { ; CHECK-NEXT: %iv.lshr.next = lshr i64 %iv.lshr, 2 ; CHECK-NEXT: --> (%iv.lshr /u 4) U: [0,240) S: [0,240) Exits: 0 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_lshr_arbitrary_start -; CHECK-NEXT: Loop %loop: backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: @@ -810,9 +810,9 @@ define void @test_lshr_start_power_of_2_plus_one() { ; CHECK-NEXT: %iv.lshr.next = lshr i64 %iv.lshr, 2 ; CHECK-NEXT: --> (%iv.lshr /u 4) U: [1,257) S: [1,257) Exits: 1 LoopDispositions: { %loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_lshr_start_power_of_2_plus_one -; CHECK-NEXT: Loop %loop: backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 4 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i64 4 ; CHECK-NEXT: Loop %loop: Trip multiple is 5 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/smax-br-phi-idioms.ll b/llvm/test/Analysis/ScalarEvolution/smax-br-phi-idioms.ll index 471c255805f3..ea4b25c62a7f 100644 --- a/llvm/test/Analysis/ScalarEvolution/smax-br-phi-idioms.ll +++ b/llvm/test/Analysis/ScalarEvolution/smax-br-phi-idioms.ll @@ -152,13 +152,13 @@ define i32 @f5(ptr %val) { ; CHECK-NEXT: %init = phi i32 [ 0, %for.condt ], [ %inc, %for.end ] ; CHECK-NEXT: --> %inc U: full-set S: full-set ; CHECK-NEXT: Determining loop execution counts for: @f5 -; CHECK-NEXT: Loop %for.end: backedge-taken count is false -; CHECK-NEXT: exit count for for.end: false -; CHECK-NEXT: exit count for for.condt: false +; CHECK-NEXT: Loop %for.end: backedge-taken count is i1 false +; CHECK-NEXT: exit count for for.end: i1 false +; CHECK-NEXT: exit count for for.condt: i1 false ; CHECK-NEXT: Loop %for.end: constant max backedge-taken count is i1 false -; CHECK-NEXT: Loop %for.end: symbolic max backedge-taken count is false -; CHECK-NEXT: symbolic max exit count for for.end: false -; CHECK-NEXT: symbolic max exit count for for.condt: false +; CHECK-NEXT: Loop %for.end: symbolic max backedge-taken count is i1 false +; CHECK-NEXT: symbolic max exit count for for.end: i1 false +; CHECK-NEXT: symbolic max exit count for for.condt: i1 false ; CHECK-NEXT: Loop %for.end: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/solve-quadratic-i1.ll b/llvm/test/Analysis/ScalarEvolution/solve-quadratic-i1.ll index 5f758e860a10..fa4e5fb0ac43 100644 --- a/llvm/test/Analysis/ScalarEvolution/solve-quadratic-i1.ll +++ b/llvm/test/Analysis/ScalarEvolution/solve-quadratic-i1.ll @@ -17,9 +17,9 @@ define void @f0() { ; CHECK-NEXT: %v4 = and i16 %v3, 1 ; CHECK-NEXT: --> (zext i1 {true,+,true,+,true}<%b1> to i16) U: [0,2) S: [0,2) Exits: 0 LoopDispositions: { %b1: Computable } ; CHECK-NEXT: Determining loop execution counts for: @f0 -; CHECK-NEXT: Loop %b1: backedge-taken count is 1 +; CHECK-NEXT: Loop %b1: backedge-taken count is i6 1 ; CHECK-NEXT: Loop %b1: constant max backedge-taken count is i6 1 -; CHECK-NEXT: Loop %b1: symbolic max backedge-taken count is 1 +; CHECK-NEXT: Loop %b1: symbolic max backedge-taken count is i6 1 ; CHECK-NEXT: Loop %b1: Trip multiple is 2 ; b0: @@ -65,9 +65,9 @@ define void @f1() #0 { ; CHECK-NEXT: Loop %b3: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %b3: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %b3: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %b1: backedge-taken count is 2 +; CHECK-NEXT: Loop %b1: backedge-taken count is i6 2 ; CHECK-NEXT: Loop %b1: constant max backedge-taken count is i6 2 -; CHECK-NEXT: Loop %b1: symbolic max backedge-taken count is 2 +; CHECK-NEXT: Loop %b1: symbolic max backedge-taken count is i6 2 ; CHECK-NEXT: Loop %b1: Trip multiple is 3 ; b0: diff --git a/llvm/test/Analysis/ScalarEvolution/solve-quadratic-overflow.ll b/llvm/test/Analysis/ScalarEvolution/solve-quadratic-overflow.ll index aac0d618a6a3..2e8bdcbaae0a 100644 --- a/llvm/test/Analysis/ScalarEvolution/solve-quadratic-overflow.ll +++ b/llvm/test/Analysis/ScalarEvolution/solve-quadratic-overflow.ll @@ -23,9 +23,9 @@ define signext i32 @f0() { ; CHECK-NEXT: %v7 = sext i16 %v5 to i32 ; CHECK-NEXT: --> {-1,+,-1}<%b1> U: [-256,0) S: [-256,0) --> -256 U: [-256,-255) S: [-256,-255) ; CHECK-NEXT: Determining loop execution counts for: @f0 -; CHECK-NEXT: Loop %b1: backedge-taken count is 255 +; CHECK-NEXT: Loop %b1: backedge-taken count is i16 255 ; CHECK-NEXT: Loop %b1: constant max backedge-taken count is i16 255 -; CHECK-NEXT: Loop %b1: symbolic max backedge-taken count is 255 +; CHECK-NEXT: Loop %b1: symbolic max backedge-taken count is i16 255 ; CHECK-NEXT: Loop %b1: Trip multiple is 256 ; b0: diff --git a/llvm/test/Analysis/ScalarEvolution/solve-quadratic.ll b/llvm/test/Analysis/ScalarEvolution/solve-quadratic.ll index 984d83895fc1..fd02ef672a96 100644 --- a/llvm/test/Analysis/ScalarEvolution/solve-quadratic.ll +++ b/llvm/test/Analysis/ScalarEvolution/solve-quadratic.ll @@ -92,7 +92,7 @@ exit: ; CHECK: {{.*}}SolveQuadraticEquationWrap{{.*}}: solving 1x^2 + -73x + -146, rw:33 ; CHECK: {{.*}}SolveQuadraticEquationWrap{{.*}}: updated coefficients 1x^2 + -73x + -146, rw:33 ; CHECK: {{.*}}SolveQuadraticEquationWrap{{.*}}: solution (wrap): 75 -; CHECK: Loop %loop: backedge-taken count is 75 +; CHECK: Loop %loop: backedge-taken count is i32 75 define signext i32 @test02() { entry: br label %loop @@ -124,7 +124,7 @@ exit: ; CHECK: {{.*}}SolveQuadraticEquationWrap{{.*}}: solving 2x^2 + -4x + 2, rw:5 ; CHECK: {{.*}}SolveQuadraticEquationWrap{{.*}}: updated coefficients 2x^2 + -4x + 2, rw:5 ; CHECK: {{.*}}SolveQuadraticEquationWrap{{.*}}: solution (root): 1 -; CHECK: Loop %loop: backedge-taken count is 1 +; CHECK: Loop %loop: backedge-taken count is i4 1 define signext i32 @test03() { entry: br label %loop @@ -243,7 +243,7 @@ exit: ; CHECK: {{.*}}SolveQuadraticEquationWrap{{.*}}: solving -1x^2 + -1x + -2, rw:33 ; CHECK: {{.*}}SolveQuadraticEquationWrap{{.*}}: updated coefficients 1x^2 + 1x + -8589934590, rw:33 ; CHECK: {{.*}}SolveQuadraticEquationWrap{{.*}}: solution (wrap): 92682 -; CHECK: Loop %loop: backedge-taken count is 2 +; CHECK: Loop %loop: backedge-taken count is i32 2 define signext i32 @test05() { entry: @@ -287,7 +287,7 @@ exit: ; CHECK: {{.*}}SolveQuadraticEquationWrap{{.*}}: solving 1x^2 + -199999x + -12, rw:33 ; CHECK: {{.*}}SolveQuadraticEquationWrap{{.*}}: updated coefficients 1x^2 + -199999x + 8589934580, rw:33 ; CHECK: {{.*}}SolveQuadraticEquationWrap{{.*}}: solution (wrap): 62450 -; CHECK: Loop %loop: backedge-taken count is 24469 +; CHECK: Loop %loop: backedge-taken count is i32 24469 define signext i32 @test06() { entry: br label %loop diff --git a/llvm/test/Analysis/ScalarEvolution/symbolic_max_exit_count.ll b/llvm/test/Analysis/ScalarEvolution/symbolic_max_exit_count.ll index 1f88c9ec985f..ab43182d2e82 100644 --- a/llvm/test/Analysis/ScalarEvolution/symbolic_max_exit_count.ll +++ b/llvm/test/Analysis/ScalarEvolution/symbolic_max_exit_count.ll @@ -234,12 +234,12 @@ define i32 @test_mixup_constant_symbolic(i32 %end, i32 %len) { ; CHECK-NEXT: Determining loop execution counts for: @test_mixup_constant_symbolic ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for loop: %end -; CHECK-NEXT: exit count for range_check_block: 1000 +; CHECK-NEXT: exit count for range_check_block: i32 1000 ; CHECK-NEXT: exit count for backedge: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 1000 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (1000 umin %end) ; CHECK-NEXT: symbolic max exit count for loop: %end -; CHECK-NEXT: symbolic max exit count for range_check_block: 1000 +; CHECK-NEXT: symbolic max exit count for range_check_block: i32 1000 ; CHECK-NEXT: symbolic max exit count for backedge: ***COULDNOTCOMPUTE*** ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-andor-selectform.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-andor-selectform.ll index 6b2251c92070..ebdf7d309ca4 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-andor-selectform.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-andor-selectform.ll @@ -51,9 +51,9 @@ leave: define void @unsimplified_and3(i32 %n) { ; CHECK-LABEL: 'unsimplified_and3' ; CHECK-NEXT: Determining loop execution counts for: @unsimplified_and3 -; CHECK-NEXT: Loop %loop: backedge-taken count is false +; CHECK-NEXT: Loop %loop: backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -73,9 +73,9 @@ leave: define void @unsimplified_and4(i32 %n) { ; CHECK-LABEL: 'unsimplified_and4' ; CHECK-NEXT: Determining loop execution counts for: @unsimplified_and4 -; CHECK-NEXT: Loop %loop: backedge-taken count is false +; CHECK-NEXT: Loop %loop: backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -267,9 +267,9 @@ leave: define void @reversed_or1(i32 %n) { ; CHECK-LABEL: 'reversed_or1' ; CHECK-NEXT: Determining loop execution counts for: @reversed_or1 -; CHECK-NEXT: Loop %loop: backedge-taken count is false +; CHECK-NEXT: Loop %loop: backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -289,9 +289,9 @@ leave: define void @reversed_or2(i32 %n) { ; CHECK-LABEL: 'reversed_or2' ; CHECK-NEXT: Determining loop execution counts for: @reversed_or2 -; CHECK-NEXT: Loop %loop: backedge-taken count is false +; CHECK-NEXT: Loop %loop: backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-andor.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-andor.ll index a7a2929b72d1..9917a72c47e1 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-andor.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-andor.ll @@ -51,9 +51,9 @@ leave: define void @unsimplified_and3(i32 %n) { ; CHECK-LABEL: 'unsimplified_and3' ; CHECK-NEXT: Determining loop execution counts for: @unsimplified_and3 -; CHECK-NEXT: Loop %loop: backedge-taken count is false +; CHECK-NEXT: Loop %loop: backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -73,9 +73,9 @@ leave: define void @unsimplified_and4(i32 %n) { ; CHECK-LABEL: 'unsimplified_and4' ; CHECK-NEXT: Determining loop execution counts for: @unsimplified_and4 -; CHECK-NEXT: Loop %loop: backedge-taken count is false +; CHECK-NEXT: Loop %loop: backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -267,9 +267,9 @@ leave: define void @reversed_or1(i32 %n) { ; CHECK-LABEL: 'reversed_or1' ; CHECK-NEXT: Determining loop execution counts for: @reversed_or1 -; CHECK-NEXT: Loop %loop: backedge-taken count is false +; CHECK-NEXT: Loop %loop: backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -289,9 +289,9 @@ leave: define void @reversed_or2(i32 %n) { ; CHECK-LABEL: 'reversed_or2' ; CHECK-NEXT: Determining loop execution counts for: @reversed_or2 -; CHECK-NEXT: Loop %loop: backedge-taken count is false +; CHECK-NEXT: Loop %loop: backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-implied-addrec.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-implied-addrec.ll index 114e7f3baa89..e9c13f551b4d 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-implied-addrec.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-implied-addrec.ll @@ -104,7 +104,7 @@ define void @actually_infinite() { ; CHECK-NEXT: Loop %for.body: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %for.body: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is 257 +; CHECK-NEXT: Loop %for.body: Predicated backedge-taken count is i16 257 ; CHECK-NEXT: Predicates: ; CHECK-NEXT: {0,+,1}<%for.body> Added Flags: ; diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-negative-stride.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-negative-stride.ll index 60120ed2f8eb..5a3517961e1a 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-negative-stride.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-negative-stride.ll @@ -56,9 +56,9 @@ for.end: ; preds = %for.body, %entry define void @ult_infinite_ub() mustprogress { ; CHECK-LABEL: 'ult_infinite_ub' ; CHECK-NEXT: Determining loop execution counts for: @ult_infinite_ub -; CHECK-NEXT: Loop %for.body: backedge-taken count is 1 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i8 1 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 1 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 1 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i8 1 ; CHECK-NEXT: Loop %for.body: Trip multiple is 2 ; entry: @@ -79,9 +79,9 @@ for.end: ; preds = %for.body, %entry define void @ult_129_not_taken() { ; CHECK-LABEL: 'ult_129_not_taken' ; CHECK-NEXT: Determining loop execution counts for: @ult_129_not_taken -; CHECK-NEXT: Loop %for.body: backedge-taken count is 0 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i8 0 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 0 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 0 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i8 0 ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -100,9 +100,9 @@ for.end: ; preds = %for.body, %entry define void @ult_129_unknown_start(i8 %start) mustprogress { ; CHECK-LABEL: 'ult_129_unknown_start' ; CHECK-NEXT: Determining loop execution counts for: @ult_129_unknown_start -; CHECK-NEXT: Loop %for.body: backedge-taken count is 0 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i8 0 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 0 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 0 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i8 0 ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -149,9 +149,9 @@ for.end: ; preds = %for.body, %entry define void @ult_ub1() { ; CHECK-LABEL: 'ult_ub1' ; CHECK-NEXT: Determining loop execution counts for: @ult_ub1 -; CHECK-NEXT: Loop %for.body: backedge-taken count is 2 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i32 2 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 2 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i32 2 ; CHECK-NEXT: Loop %for.body: Trip multiple is 3 ; entry: @@ -172,9 +172,9 @@ for.end: ; preds = %for.body, %entry define void @ult_ub2() { ; CHECK-LABEL: 'ult_ub2' ; CHECK-NEXT: Determining loop execution counts for: @ult_ub2 -; CHECK-NEXT: Loop %for.body: backedge-taken count is 0 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i8 0 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 0 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 0 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i8 0 ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -195,9 +195,9 @@ for.end: ; preds = %for.body, %entry define void @ult_129_preinc() { ; CHECK-LABEL: 'ult_129_preinc' ; CHECK-NEXT: Determining loop execution counts for: @ult_129_preinc -; CHECK-NEXT: Loop %for.body: backedge-taken count is 1 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i8 1 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 1 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 1 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i8 1 ; CHECK-NEXT: Loop %for.body: Trip multiple is 2 ; entry: @@ -216,9 +216,9 @@ for.end: ; preds = %for.body, %entry define void @ult_preinc(i8 %step) { ; CHECK-LABEL: 'ult_preinc' ; CHECK-NEXT: Determining loop execution counts for: @ult_preinc -; CHECK-NEXT: Loop %for.body: backedge-taken count is 1 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i8 1 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 1 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 1 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i8 1 ; CHECK-NEXT: Loop %for.body: Trip multiple is 2 ; entry: @@ -289,9 +289,9 @@ for.end: ; preds = %for.body, %entry define void @slt_wrap() { ; CHECK-LABEL: 'slt_wrap' ; CHECK-NEXT: Determining loop execution counts for: @slt_wrap -; CHECK-NEXT: Loop %for.body: backedge-taken count is 63 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i8 63 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 63 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 63 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i8 63 ; CHECK-NEXT: Loop %for.body: Trip multiple is 64 ; entry: @@ -334,9 +334,9 @@ for.end: ; preds = %for.body, %entry define void @slt_infinite_ub() mustprogress { ; CHECK-LABEL: 'slt_infinite_ub' ; CHECK-NEXT: Determining loop execution counts for: @slt_infinite_ub -; CHECK-NEXT: Loop %for.body: backedge-taken count is 0 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i8 0 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 0 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 0 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i8 0 ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -357,9 +357,9 @@ for.end: ; preds = %for.body, %entry define void @slt_129_not_taken() { ; CHECK-LABEL: 'slt_129_not_taken' ; CHECK-NEXT: Determining loop execution counts for: @slt_129_not_taken -; CHECK-NEXT: Loop %for.body: backedge-taken count is 0 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i8 0 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 0 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 0 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i8 0 ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -425,9 +425,9 @@ for.end: ; preds = %for.body, %entry define void @slt_ub1() { ; CHECK-LABEL: 'slt_ub1' ; CHECK-NEXT: Determining loop execution counts for: @slt_ub1 -; CHECK-NEXT: Loop %for.body: backedge-taken count is false +; CHECK-NEXT: Loop %for.body: backedge-taken count is i1 false ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i1 false -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is false +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i1 false ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -448,9 +448,9 @@ for.end: ; preds = %for.body, %entry define void @slt_ub2() { ; CHECK-LABEL: 'slt_ub2' ; CHECK-NEXT: Determining loop execution counts for: @slt_ub2 -; CHECK-NEXT: Loop %for.body: backedge-taken count is false +; CHECK-NEXT: Loop %for.body: backedge-taken count is i1 false ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i1 false -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is false +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i1 false ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; entry: @@ -471,9 +471,9 @@ for.end: ; preds = %for.body, %entry define void @slt_129_preinc() { ; CHECK-LABEL: 'slt_129_preinc' ; CHECK-NEXT: Determining loop execution counts for: @slt_129_preinc -; CHECK-NEXT: Loop %for.body: backedge-taken count is 1 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i8 1 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 1 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 1 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i8 1 ; CHECK-NEXT: Loop %for.body: Trip multiple is 2 ; entry: @@ -492,9 +492,9 @@ for.end: ; preds = %for.body, %entry define void @slt_preinc(i8 %step) { ; CHECK-LABEL: 'slt_preinc' ; CHECK-NEXT: Determining loop execution counts for: @slt_preinc -; CHECK-NEXT: Loop %for.body: backedge-taken count is 1 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i8 1 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i8 1 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 1 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i8 1 ; CHECK-NEXT: Loop %for.body: Trip multiple is 2 ; entry: @@ -566,9 +566,9 @@ define void @step_is_neg_addrec_slt_8(i64 %n) { ; CHECK-NEXT: Loop %inner: constant max backedge-taken count is i32 8 ; CHECK-NEXT: Loop %inner: symbolic max backedge-taken count is (7 /u {0,+,-1}<%outer.header>) ; CHECK-NEXT: Loop %inner: Trip multiple is 1 -; CHECK-NEXT: Loop %outer.header: backedge-taken count is 0 +; CHECK-NEXT: Loop %outer.header: backedge-taken count is i64 0 ; CHECK-NEXT: Loop %outer.header: constant max backedge-taken count is i64 0 -; CHECK-NEXT: Loop %outer.header: symbolic max backedge-taken count is 0 +; CHECK-NEXT: Loop %outer.header: symbolic max backedge-taken count is i64 0 ; CHECK-NEXT: Loop %outer.header: Trip multiple is 1 ; entry: @@ -604,9 +604,9 @@ define void @step_is_neg_addrec_slt_var(i32 %n) { ; CHECK-NEXT: Loop %inner: constant max backedge-taken count is i32 2147483647 ; CHECK-NEXT: Loop %inner: symbolic max backedge-taken count is ({0,+,1}<%outer.header> + ({0,+,-1}<%outer.header> smax %n)) ; CHECK-NEXT: Loop %inner: Trip multiple is 1 -; CHECK-NEXT: Loop %outer.header: backedge-taken count is 0 +; CHECK-NEXT: Loop %outer.header: backedge-taken count is i64 0 ; CHECK-NEXT: Loop %outer.header: constant max backedge-taken count is i64 0 -; CHECK-NEXT: Loop %outer.header: symbolic max backedge-taken count is 0 +; CHECK-NEXT: Loop %outer.header: symbolic max backedge-taken count is i64 0 ; CHECK-NEXT: Loop %outer.header: Trip multiple is 1 ; entry: @@ -642,9 +642,9 @@ define void @step_is_neg_addrec_unknown_start(i32 %n) { ; CHECK-NEXT: Loop %inner: constant max backedge-taken count is i32 -2147483640 ; CHECK-NEXT: Loop %inner: symbolic max backedge-taken count is ({(-1 * %n),+,1}<%outer.header> + (8 smax {%n,+,-1}<%outer.header>)) ; CHECK-NEXT: Loop %inner: Trip multiple is 1 -; CHECK-NEXT: Loop %outer.header: backedge-taken count is 0 +; CHECK-NEXT: Loop %outer.header: backedge-taken count is i64 0 ; CHECK-NEXT: Loop %outer.header: constant max backedge-taken count is i64 0 -; CHECK-NEXT: Loop %outer.header: symbolic max backedge-taken count is 0 +; CHECK-NEXT: Loop %outer.header: symbolic max backedge-taken count is i64 0 ; CHECK-NEXT: Loop %outer.header: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-switch.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-switch.ll index df8bb0ae043a..58ed6aea2bbc 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-switch.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-switch.ll @@ -7,9 +7,9 @@ define void @test1() nounwind { ; ; CHECK-LABEL: 'test1' ; CHECK-NEXT: Determining loop execution counts for: @test1 -; CHECK-NEXT: Loop %for.cond: backedge-taken count is 2 +; CHECK-NEXT: Loop %for.cond: backedge-taken count is i32 2 ; CHECK-NEXT: Loop %for.cond: constant max backedge-taken count is i32 2 -; CHECK-NEXT: Loop %for.cond: symbolic max backedge-taken count is 2 +; CHECK-NEXT: Loop %for.cond: symbolic max backedge-taken count is i32 2 ; CHECK-NEXT: Loop %for.cond: Trip multiple is 3 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count.ll b/llvm/test/Analysis/ScalarEvolution/trip-count.ll index 8f3e82984431..9dd9e3a5ee99 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count.ll @@ -9,9 +9,9 @@ target triple = "x86_64-unknown-linux-gnu" define void @PR1101(i32 %N) { ; CHECK-LABEL: 'PR1101' ; CHECK-NEXT: Determining loop execution counts for: @PR1101 -; CHECK-NEXT: Loop %bb3: backedge-taken count is 10000 +; CHECK-NEXT: Loop %bb3: backedge-taken count is i32 10000 ; CHECK-NEXT: Loop %bb3: constant max backedge-taken count is i32 10000 -; CHECK-NEXT: Loop %bb3: symbolic max backedge-taken count is 10000 +; CHECK-NEXT: Loop %bb3: symbolic max backedge-taken count is i32 10000 ; CHECK-NEXT: Loop %bb3: Trip multiple is 10001 ; entry: @@ -38,9 +38,9 @@ return: ; preds = %bb5 define i32 @PR22795() { ; CHECK-LABEL: 'PR22795' ; CHECK-NEXT: Determining loop execution counts for: @PR22795 -; CHECK-NEXT: Loop %preheader: backedge-taken count is 7 +; CHECK-NEXT: Loop %preheader: backedge-taken count is i64 7 ; CHECK-NEXT: Loop %preheader: constant max backedge-taken count is i64 7 -; CHECK-NEXT: Loop %preheader: symbolic max backedge-taken count is 7 +; CHECK-NEXT: Loop %preheader: symbolic max backedge-taken count is i64 7 ; CHECK-NEXT: Loop %preheader: Trip multiple is 8 ; entry: @@ -99,9 +99,9 @@ declare void @may_exit() nounwind define void @pr28012(i32 %n) { ; CHECK-LABEL: 'pr28012' ; CHECK-NEXT: Determining loop execution counts for: @pr28012 -; CHECK-NEXT: Loop %loop: backedge-taken count is -1431655751 +; CHECK-NEXT: Loop %loop: backedge-taken count is i32 -1431655751 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 -1431655751 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is -1431655751 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 -1431655751 ; CHECK-NEXT: Loop %loop: Trip multiple is 2863311546 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count10.ll b/llvm/test/Analysis/ScalarEvolution/trip-count10.ll index 664ca3a1b3d3..306ac326bdf0 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count10.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count10.ll @@ -26,9 +26,9 @@ return: define void @b(i64 %n) nounwind { ; CHECK-LABEL: 'b' ; CHECK-NEXT: Determining loop execution counts for: @b -; CHECK-NEXT: Loop %loop: backedge-taken count is false +; CHECK-NEXT: Loop %loop: backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -47,9 +47,9 @@ return: define void @c(i64 %n) nounwind { ; CHECK-LABEL: 'c' ; CHECK-NEXT: Determining loop execution counts for: @c -; CHECK-NEXT: Loop %loop: backedge-taken count is false +; CHECK-NEXT: Loop %loop: backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i1 false -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is false +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i1 false ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: @@ -124,9 +124,9 @@ retbb: define void @constant_phi_operands() nounwind { ; CHECK-LABEL: 'constant_phi_operands' ; CHECK-NEXT: Determining loop execution counts for: @constant_phi_operands -; CHECK-NEXT: Loop %loop: backedge-taken count is 1 +; CHECK-NEXT: Loop %loop: backedge-taken count is i32 1 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 1 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 1 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 1 ; CHECK-NEXT: Loop %loop: Trip multiple is 2 ; entry: @@ -150,7 +150,7 @@ define void @exit_orcond_nsw(ptr %a) nounwind { ; CHECK-NEXT: Determining loop execution counts for: @exit_orcond_nsw ; CHECK-NEXT: Loop %for.body.i: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %for.body.i: constant max backedge-taken count is i32 1 -; CHECK-NEXT: Loop %for.body.i: symbolic max backedge-taken count is 1 +; CHECK-NEXT: Loop %for.body.i: symbolic max backedge-taken count is i32 1 ; entry: br label %for.body.i diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count2.ll b/llvm/test/Analysis/ScalarEvolution/trip-count2.ll index ffbdcf7376b2..340fbb635afd 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count2.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count2.ll @@ -6,9 +6,9 @@ define void @PR1101(i32 %N) { ; CHECK-LABEL: 'PR1101' ; CHECK-NEXT: Determining loop execution counts for: @PR1101 -; CHECK-NEXT: Loop %bb3: backedge-taken count is 4 +; CHECK-NEXT: Loop %bb3: backedge-taken count is i32 4 ; CHECK-NEXT: Loop %bb3: constant max backedge-taken count is i32 4 -; CHECK-NEXT: Loop %bb3: symbolic max backedge-taken count is 4 +; CHECK-NEXT: Loop %bb3: symbolic max backedge-taken count is i32 4 ; CHECK-NEXT: Loop %bb3: Trip multiple is 5 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count6.ll b/llvm/test/Analysis/ScalarEvolution/trip-count6.ll index e4ded1d14b65..ed256323aedb 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count6.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count6.ll @@ -8,11 +8,11 @@ define i8 @f() { ; CHECK-NEXT: Determining loop execution counts for: @f ; CHECK-NEXT: Loop %bb: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for bb: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: exit count for bb2: 1 +; CHECK-NEXT: exit count for bb2: i8 1 ; CHECK-NEXT: Loop %bb: constant max backedge-taken count is i8 1 -; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is 1 +; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is i8 1 ; CHECK-NEXT: symbolic max exit count for bb: ***COULDNOTCOMPUTE*** -; CHECK-NEXT: symbolic max exit count for bb2: 1 +; CHECK-NEXT: symbolic max exit count for bb2: i8 1 ; entry: tail call i32 @fegetround( ) ; :0 [#uses=1] diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count7.ll b/llvm/test/Analysis/ScalarEvolution/trip-count7.ll index 3dca3efa25b4..fedd39f7c904 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count7.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count7.ll @@ -65,7 +65,7 @@ define void @Doit_bb7_2E_i(ptr %x1, ptr %c, ptr %b, ptr %a, ptr %q, ptr %x1.sub, ; CHECK-NEXT: Determining loop execution counts for: @Doit_bb7_2E_i ; CHECK-NEXT: Loop %bb7.i: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %bb7.i: constant max backedge-taken count is i32 8 -; CHECK-NEXT: Loop %bb7.i: symbolic max backedge-taken count is 8 +; CHECK-NEXT: Loop %bb7.i: symbolic max backedge-taken count is i32 8 ; newFuncRoot: br label %bb7.i diff --git a/llvm/test/Analysis/ScalarEvolution/unknown_phis.ll b/llvm/test/Analysis/ScalarEvolution/unknown_phis.ll index 2a8d4de24b58..bdfe38f67de0 100644 --- a/llvm/test/Analysis/ScalarEvolution/unknown_phis.ll +++ b/llvm/test/Analysis/ScalarEvolution/unknown_phis.ll @@ -47,9 +47,9 @@ define void @merge_values_with_ranges_looped(ptr %a_len_ptr, ptr %b_len_ptr) { ; CHECK-NEXT: %iv.next = add i32 %iv, 1 ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,101) S: [1,101) Exits: 100 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @merge_values_with_ranges_looped -; CHECK-NEXT: Loop %loop: backedge-taken count is 99 +; CHECK-NEXT: Loop %loop: backedge-taken count is i32 99 ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 99 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 99 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 99 ; CHECK-NEXT: Loop %loop: Trip multiple is 100 ; diff --git a/llvm/test/Analysis/ScalarEvolution/widenable-condition.ll b/llvm/test/Analysis/ScalarEvolution/widenable-condition.ll index f1bda1705dff..41d425c30925 100644 --- a/llvm/test/Analysis/ScalarEvolution/widenable-condition.ll +++ b/llvm/test/Analysis/ScalarEvolution/widenable-condition.ll @@ -24,7 +24,7 @@ define i32 @wc_max() { ; CHECK-NEXT: Determining loop execution counts for: @wc_max ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 1999 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is 1999 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 1999 ; entry: br label %loop diff --git a/llvm/test/Transforms/LoopSimplify/preserve-scev.ll b/llvm/test/Transforms/LoopSimplify/preserve-scev.ll index 5bc81565b409..4f5b7ac31e8b 100644 --- a/llvm/test/Transforms/LoopSimplify/preserve-scev.ll +++ b/llvm/test/Transforms/LoopSimplify/preserve-scev.ll @@ -84,7 +84,7 @@ declare void @foo() nounwind ; After simplifying, the max backedge count is refined. ; Second SCEV print: ; CHECK-LABEL: Determining loop execution counts for: @mergeExit -; CHECK: Loop %while.cond191: backedge-taken count is 0 +; CHECK: Loop %while.cond191: backedge-taken count is i32 0 ; CHECK: Loop %while.cond191: constant max backedge-taken count is i32 0 ; CHECK: Loop %while.cond191.outer: Unpredictable backedge-taken count. ; CHECK: Loop %while.cond191.outer: constant max backedge-taken count is i1 false diff --git a/llvm/test/Transforms/PhaseOrdering/scev-custom-dl.ll b/llvm/test/Transforms/PhaseOrdering/scev-custom-dl.ll index 467f1e0210ff..aaea1a453664 100644 --- a/llvm/test/Transforms/PhaseOrdering/scev-custom-dl.ll +++ b/llvm/test/Transforms/PhaseOrdering/scev-custom-dl.ll @@ -26,9 +26,9 @@ define void @test1(i32 %d, ptr %p) nounwind uwtable ssp { ; CHECK-NEXT: %inc = add nuw nsw i32 %i.03, 1 ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,65) S: [1,65) Exits: 64 LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test1 -; CHECK-NEXT: Loop %for.body: backedge-taken count is 63 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i32 63 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 63 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 63 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i32 63 ; CHECK-NEXT: Loop %for.body: Trip multiple is 64 ; entry: @@ -73,9 +73,9 @@ define void @test1a(i32 %d, ptr %p) nounwind uwtable ssp { ; CHECK-NEXT: %inc = add nuw nsw i32 %i.03, 1 ; CHECK-NEXT: --> {1,+,1}<%for.body> U: [1,65) S: [1,65) Exits: 64 LoopDispositions: { %for.body: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test1a -; CHECK-NEXT: Loop %for.body: backedge-taken count is 63 +; CHECK-NEXT: Loop %for.body: backedge-taken count is i32 63 ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 63 -; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is 63 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is i32 63 ; CHECK-NEXT: Loop %for.body: Trip multiple is 64 ; entry: @@ -116,9 +116,9 @@ define void @test_range_ref1a(i32 %x) { ; CHECK-NEXT: %tmp4 = add nsw i32 %i.01.0, -1 ; CHECK-NEXT: --> {99,+,-1}<%bb> U: [-1,100) S: [-1,100) Exits: -1 LoopDispositions: { %bb: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_range_ref1a -; CHECK-NEXT: Loop %bb: backedge-taken count is 100 +; CHECK-NEXT: Loop %bb: backedge-taken count is i32 100 ; CHECK-NEXT: Loop %bb: constant max backedge-taken count is i32 100 -; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is 100 +; CHECK-NEXT: Loop %bb: symbolic max backedge-taken count is i32 100 ; CHECK-NEXT: Loop %bb: Trip multiple is 101 ; entry: @@ -158,9 +158,9 @@ define i32 @test_loop_idiom_recogize(i32 %x, i32 %y, ptr %lam, ptr %alp) nounwin ; CHECK-NEXT: %3 = add i32 %z.0.reg2mem.0, %tmp10 ; CHECK-NEXT: --> ((256 * %x) + %y) U: full-set S: full-set ; CHECK-NEXT: Determining loop execution counts for: @test_loop_idiom_recogize -; CHECK-NEXT: Loop %bb1: backedge-taken count is 255 +; CHECK-NEXT: Loop %bb1: backedge-taken count is i32 255 ; CHECK-NEXT: Loop %bb1: constant max backedge-taken count is i32 255 -; CHECK-NEXT: Loop %bb1: symbolic max backedge-taken count is 255 +; CHECK-NEXT: Loop %bb1: symbolic max backedge-taken count is i32 255 ; CHECK-NEXT: Loop %bb1: Trip multiple is 256 ; bb1.thread: diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/update-scev-2.ll b/llvm/test/Transforms/SimpleLoopUnswitch/update-scev-2.ll index f38fd962acbf..6ca8b4049b1a 100644 --- a/llvm/test/Transforms/SimpleLoopUnswitch/update-scev-2.ll +++ b/llvm/test/Transforms/SimpleLoopUnswitch/update-scev-2.ll @@ -17,7 +17,7 @@ ; CHECK-SCEV-NEXT: %0 = add i16 %j.0, 1 ; CHECK-SCEV-NEXT: --> {1,+,1}<%lbl1> U: [1,4) S: [1,4) Exits: 3 LoopDispositions: { %lbl1: Computable, %lbl2: Invariant } ; CHECK-SCEV-DAG: Loop %lbl2: Unpredictable backedge-taken count. -; CHECK-SCEV-DAG: Loop %lbl1: backedge-taken count is 2 +; CHECK-SCEV-DAG: Loop %lbl1: backedge-taken count is i16 2 ; ; After the unswitch: ; -- GitLab From c3acbf6bb06f9039f9850e18e0ae2f2adef63905 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Wed, 6 Mar 2024 13:17:43 -0800 Subject: [PATCH 360/929] [test] Make two sanitize-coverage tests pass with glibc 2.39+ glibc 2.39 added `nonnull` attribute to most libio functions accepting a `FILE*` parameter, including fprintf[1]. The -fsanitize=undefined mode checks the argument to fprintf and has extra counters, not expected by two tests. Specify -fno-sanitize=nonnull-attribute to make the two tests pass. Fix #82883 [1]: https://sourceware.org/git/?p=glibc.git;a=commit;h=64b1a44183a3094672ed304532bedb9acc707554 Pull Request: https://github.com/llvm/llvm-project/pull/84231 --- .../sanitizer_coverage_inline8bit_counter_default_impl.cpp | 4 +++- .../TestCases/sanitizer_coverage_symbolize.cpp | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/compiler-rt/test/sanitizer_common/TestCases/sanitizer_coverage_inline8bit_counter_default_impl.cpp b/compiler-rt/test/sanitizer_common/TestCases/sanitizer_coverage_inline8bit_counter_default_impl.cpp index 1ac04b53491e..1d1fbf7299e8 100644 --- a/compiler-rt/test/sanitizer_common/TestCases/sanitizer_coverage_inline8bit_counter_default_impl.cpp +++ b/compiler-rt/test/sanitizer_common/TestCases/sanitizer_coverage_inline8bit_counter_default_impl.cpp @@ -3,7 +3,9 @@ // REQUIRES: has_sancovcc,stable-runtime,linux,x86_64-target-arch -// RUN: %clangxx -O0 %s -fsanitize-coverage=inline-8bit-counters,pc-table -o %t +/// In glibc 2.39+, fprintf has a nonnull attribute. Disable nonnull-attribute, +/// which would increase counters for ubsan. +// RUN: %clangxx -O0 %s -fsanitize-coverage=inline-8bit-counters,pc-table -fno-sanitize=nonnull-attribute -o %t // RUN: rm -f %t-counters %t-pcs // RUN: env %tool_options="cov_8bit_counters_out=%t-counters cov_pcs_out=%t-pcs verbosity=1" %run %t 2>&1 | FileCheck %s diff --git a/compiler-rt/test/sanitizer_common/TestCases/sanitizer_coverage_symbolize.cpp b/compiler-rt/test/sanitizer_common/TestCases/sanitizer_coverage_symbolize.cpp index daa994c81162..b168954a1c92 100644 --- a/compiler-rt/test/sanitizer_common/TestCases/sanitizer_coverage_symbolize.cpp +++ b/compiler-rt/test/sanitizer_common/TestCases/sanitizer_coverage_symbolize.cpp @@ -7,7 +7,9 @@ // RUN: rm -rf $DIR // RUN: mkdir -p $DIR // RUN: cd $DIR -// RUN: %clangxx -O0 -fsanitize-coverage=trace-pc-guard %s -o %t +/// In glibc 2.39+, fprintf has a nonnull attribute. Disable nonnull-attribute, +/// which would increase counters for ubsan. +// RUN: %clangxx -O0 -fsanitize-coverage=trace-pc-guard -fno-sanitize=nonnull-attribute %s -o %t // RUN: %env_tool_opts=coverage=1 %t 2>&1 | FileCheck %s // RUN: rm -rf $DIR -- GitLab From 1ed15cd5de69bc9347b0750281f5f711018c6506 Mon Sep 17 00:00:00 2001 From: mmilanifard Date: Wed, 6 Mar 2024 13:18:13 -0800 Subject: [PATCH 361/929] Adding missing dependencies to BUILD.bazel (#84235) Build fix after (#84007). --- utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index d2bacbb6ce47..89ed837fc10f 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -6187,6 +6187,9 @@ cc_library( ":NVVMToLLVMIRTranslation", ":TargetLLVM", ":ToLLVMIRTranslation", + "@cuda//:cuda_headers", + "@cuda//:cusparse_static", + "@cuda//:libcuda", ":config", "//llvm:NVPTXCodeGen", "//llvm:Support", -- GitLab From 351a82b8369732dbd95a85ceb499c92f22103c61 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Wed, 6 Mar 2024 21:19:52 +0000 Subject: [PATCH 362/929] [gn build] Port bec2d105c7b8 --- .../gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn index 3d8e3e66d26d..12d875cf40c9 100644 --- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn @@ -103,6 +103,7 @@ static_library("MCTargetDesc") { "AMDGPUInstPrinter.cpp", "AMDGPUMCAsmInfo.cpp", "AMDGPUMCCodeEmitter.cpp", + "AMDGPUMCExpr.cpp", "AMDGPUMCTargetDesc.cpp", "AMDGPUTargetStreamer.cpp", "R600InstPrinter.cpp", -- GitLab From f0eb0c5689b25153067e66647590a8300b997740 Mon Sep 17 00:00:00 2001 From: Greg Clayton Date: Wed, 6 Mar 2024 13:49:26 -0800 Subject: [PATCH 363/929] Don't require a UUID in a .dwp file. (#83935) DWP files don't usually have a GNU build ID built into them. When searching for a .dwp file, don't require a UUID to be in the .dwp file. The debug info search information was checking for a UUID in the .dwp file when debug info search paths were being used. This is now fixed by not specifying the UUID in the ModuleSpec being used for the .dwp file search. --- .../Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp | 1 - .../DWARF/x86/dwp-separate-debug-file.cpp | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp index 84ff4c2565a0..5f67658f86ea 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp @@ -4377,7 +4377,6 @@ const std::shared_ptr &SymbolFileDWARF::GetDwpSymbolFile() { FileSpecList search_paths = Target::GetDefaultDebugFileSearchPaths(); ModuleSpec module_spec; module_spec.GetFileSpec() = m_objfile_sp->GetFileSpec(); - module_spec.GetUUID() = m_objfile_sp->GetUUID(); for (const auto &symfile : symfiles.files()) { module_spec.GetSymbolFileSpec() = FileSpec(symfile.GetPath() + ".dwp", symfile.GetPathStyle()); diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/dwp-separate-debug-file.cpp b/lldb/test/Shell/SymbolFile/DWARF/x86/dwp-separate-debug-file.cpp index 9a8149065b6e..1d636ede41b5 100644 --- a/lldb/test/Shell/SymbolFile/DWARF/x86/dwp-separate-debug-file.cpp +++ b/lldb/test/Shell/SymbolFile/DWARF/x86/dwp-separate-debug-file.cpp @@ -139,6 +139,20 @@ // RUN: -o "target variable a" \ // RUN: -b %t | FileCheck %s +// Now move the .debug and .dwp file into another directory so that we can use +// the target.debug-file-search-paths setting to search for the files. +// RUN: mkdir -p %t-debug-info-dir +// RUN: mv %t.dwp %t-debug-info-dir +// RUN: mv %t.debug %t-debug-info-dir +// RUN: %lldb \ +// RUN: -O "log enable dwarf split" \ +// RUN: -O "setting set target.debug-file-search-paths '%t-debug-info-dir'" \ +// RUN: -o "target variable a" \ +// RUN: -b %t | FileCheck %s +// RUN: + +// Now move the .debug and .dwp file into another directory so that we can use +// the target.debug-file-search-paths setting to search for the files. // CHECK: Searching for DWP using: // CHECK: Found DWP file: // CHECK: (A) a = (x = 47) -- GitLab From 47bc565ca7990a2de20af4030baf08ac62739aca Mon Sep 17 00:00:00 2001 From: lhunloh <8047408+lhunloh@users.noreply.github.com> Date: Wed, 6 Mar 2024 22:07:30 +0000 Subject: [PATCH 364/929] [MLIR] [Transforms] Let `transform.structured.convert_to_loops` return handles to loops (#83984) This lets `transform.structured.convert_to_loops` return handles to the generated loops, making this transformation more useful to use for (transformation-)nesting purposes. This is modelled after SCFs `transform.loop.forall_to_for` which returns handles to loops. Introduced in commit aa2a96a24ae3a8cc04635ab6ede474c5f2665053, with a note that they might move out of the `Linalg`-Dialect, but no reason given for the non-return of handles. As far as I can see, this transform always returns loops. --- .../Linalg/TransformOps/LinalgTransformOps.td | 22 +++--- .../TransformOps/LinalgTransformOps.cpp | 35 ++++++--- .../lower-to-loops-using-interface.mlir | 75 +++++++++++++++++-- 3 files changed, 101 insertions(+), 31 deletions(-) diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td index 53ed31877c6f..bdeab55091b9 100644 --- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td +++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td @@ -1274,33 +1274,29 @@ def ScalarizeOp : Op, ReportTrackingListenerFailuresOpTrait]> { let description = [{ For operations that implement the `TilingInterface`, and implement the `generateScalarImplementation` method, lowers the operation to - loops. This operation does not return any handles. + loops. The return handle points to all generated loops. + Fails if the payload ops cannot be lowered to loops. }]; let arguments = (ins TransformHandleTypeInterface:$target); - let results = (outs); + let results = (outs TransformHandleTypeInterface:$result); let assemblyFormat = [{ - $target attr-dict `:` type($target) - }]; - - let extraClassDeclaration = [{ - ::mlir::DiagnosedSilenceableFailure applyToOne( - ::mlir::transform::TransformRewriter &rewriter, - ::mlir::TilingInterface target, - ::mlir::transform::ApplyToEachResultList &results, - ::mlir::transform::TransformState &state); + $target attr-dict `:` functional-type(operands, results) }]; } - //===----------------------------------------------------------------------===// // DecomposeInterfaceOp //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp index 0ac0a89dcc76..ae28049f02e3 100644 --- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp +++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp @@ -2112,16 +2112,31 @@ transform::ScalarizeOp::applyToOne(transform::TransformRewriter &rewriter, // ConvertToLoopsOp //===----------------------------------------------------------------------===// -DiagnosedSilenceableFailure transform::ConvertToLoopsOp::applyToOne( - transform::TransformRewriter &rewriter, TilingInterface target, - transform::ApplyToEachResultList &results, - transform::TransformState &state) { - rewriter.setInsertionPoint(target); - FailureOr> loops = - scf::lowerToLoopsUsingSCFForOp(rewriter, target); - if (failed(loops)) - return emitDefaultDefiniteFailure(target); - rewriter.eraseOp(target); +DiagnosedSilenceableFailure +transform::ConvertToLoopsOp::apply(transform::TransformRewriter &rewriter, + transform::TransformResults &results, + transform::TransformState &state) { + SmallVector loops; + for (Operation *target : state.getPayloadOps(getTarget())) { + auto tilingOp = dyn_cast(*target); + if (!target) { + DiagnosedSilenceableFailure diag = + emitSilenceableError() + << "expected the payload to implement TilingInterface"; + diag.attachNote(target->getLoc()) << "payload op"; + return diag; + } + rewriter.setInsertionPoint(target); + FailureOr> generatedLoops = + scf::lowerToLoopsUsingSCFForOp(rewriter, tilingOp); + if (failed(generatedLoops)) + return emitDefaultDefiniteFailure(target); + for (scf::ForOp &loop : *generatedLoops) { + loops.push_back(loop.getOperation()); + } + rewriter.eraseOp(target); + } + results.set(cast(getResult()), loops); return DiagnosedSilenceableFailure::success(); } diff --git a/mlir/test/Interfaces/TilingInterface/lower-to-loops-using-interface.mlir b/mlir/test/Interfaces/TilingInterface/lower-to-loops-using-interface.mlir index 1b2c553b25de..8cbee3cbb758 100644 --- a/mlir/test/Interfaces/TilingInterface/lower-to-loops-using-interface.mlir +++ b/mlir/test/Interfaces/TilingInterface/lower-to-loops-using-interface.mlir @@ -11,7 +11,8 @@ module attributes {transform.with_named_sequence} { transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) { %matmul = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op - transform.structured.convert_to_loops %matmul : !transform.any_op + %0 = transform.structured.convert_to_loops %matmul + : (!transform.any_op) -> (!transform.any_op) transform.yield } } @@ -37,6 +38,57 @@ module attributes {transform.with_named_sequence} { // ----- +func.func @gemm(%arg0 : memref, %arg1 : memref, + %arg2 : memref, %arg3 : memref, %arg4 : memref) { + linalg.matmul ins(%arg0, %arg1 : memref, memref) + outs(%arg2 : memref) + linalg.matvec ins(%arg0, %arg3 : memref, memref) + outs(%arg4 : memref) + return +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) { + %linalg_ops = transform.structured.match interface{TilingInterface} in %arg1 + : (!transform.any_op) -> !transform.any_op + %0 = transform.structured.convert_to_loops %linalg_ops + : (!transform.any_op) -> (!transform.any_op) + %1:5 = transform.split_handle %0 + : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op) + transform.yield + } +} +// CHECK-LABEL: func @gemm +// CHECK-SAME: %[[ARG0:[a-zA-Z0-9]+]]: memref +// CHECK-SAME: %[[ARG1:[a-zA-Z0-9]+]]: memref +// CHECK-SAME: %[[ARG2:[a-zA-Z0-9]+]]: memref +// CHECK-SAME: %[[ARG3:[a-zA-Z0-9]+]]: memref +// CHECK-SAME: %[[ARG4:[a-zA-Z0-9]+]]: memref +// CHECK-DAG: %[[C0:.+]] = arith.constant 0 : index +// CHECK-DAG: %[[M:.+]] = memref.dim %[[ARG0]], %[[C0]] +// CHECK-DAG: %[[C1:.+]] = arith.constant 1 : index +// CHECK-DAG: %[[K:.+]] = memref.dim %[[ARG0]], %[[C1]] +// CHECK-DAG: %[[N:.+]] = memref.dim %[[ARG1]], %[[C1]] +// CHECK: scf.for %[[IV0:[a-zA-Z0-9]+]] = %[[C0]] to %[[M]] step %[[C1]] +// CHECK: scf.for %[[IV1:[a-zA-Z0-9]+]] = %[[C0]] to %[[N]] step %[[C1]] +// CHECK: scf.for %[[IV2:[a-zA-Z0-9]+]] = %[[C0]] to %[[K]] step %[[C1]] +// CHECK-DAG: %[[LHS:.+]] = memref.load %[[ARG0]][%[[IV0]], %[[IV2]]] +// CHECK-DAG: %[[RHS:.+]] = memref.load %[[ARG1]][%[[IV2]], %[[IV1]]] +// CHECK-DAG: %[[OUT:.+]] = memref.load %[[ARG2]][%[[IV0]], %[[IV1]]] +// CHECK: %[[MULF:.+]] = arith.mulf %[[LHS]], %[[RHS]] +// CHECK: %[[ADDF:.+]] = arith.addf %[[OUT]], %[[MULF]] +// CHECK: memref.store %[[ADDF]], %[[ARG2]][%[[IV0]], %[[IV1]]] +// CHECK: scf.for %[[IV3:[a-zA-Z0-9]+]] = %[[C0]] to %[[M]] step %[[C1]] +// CHECK: scf.for %[[IV4:[a-zA-Z0-9]+]] = %[[C0]] to %[[K]] step %[[C1]] +// CHECK-DAG: %[[LHS:.+]] = memref.load %[[ARG0]][%[[IV3]], %[[IV4]]] +// CHECK-DAG: %[[RHS:.+]] = memref.load %[[ARG3]][%[[IV4]]] +// CHECK-DAG: %[[OUT:.+]] = memref.load %[[ARG4]][%[[IV3]]] +// CHECK: %[[MULF:.+]] = arith.mulf %[[LHS]], %[[RHS]] +// CHECK: %[[ADDF:.+]] = arith.addf %[[OUT]], %[[MULF]] +// CHECK: memref.store %[[ADDF]], %[[ARG4]][%[[IV3]]] + +// ----- + func.func @indexed_generic(%arg0 : memref<200x300xi32>, %arg1 : memref<300xi16>, %arg2 : memref<200xi8>, %arg3 : memref<300x200xi64>) { linalg.generic { @@ -66,7 +118,8 @@ module attributes {transform.with_named_sequence} { transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) { %generic = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op - transform.structured.convert_to_loops %generic : !transform.any_op + %0 = transform.structured.convert_to_loops %generic + : (!transform.any_op) -> (!transform.any_op) transform.yield } } @@ -111,7 +164,8 @@ module attributes {transform.with_named_sequence} { transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) { %conv = transform.structured.match ops{["linalg.conv_2d_nhwc_hwcf"]} in %arg1 : (!transform.any_op) -> !transform.any_op - transform.structured.convert_to_loops %conv : !transform.any_op + %0 = transform.structured.convert_to_loops %conv + : (!transform.any_op) -> (!transform.any_op) transform.yield } } @@ -165,7 +219,8 @@ module attributes {transform.with_named_sequence} { transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) { %pool = transform.structured.match ops{["linalg.pooling_nhwc_max"]} in %arg1 : (!transform.any_op) -> !transform.any_op - transform.structured.convert_to_loops %pool : !transform.any_op + %0 = transform.structured.convert_to_loops %pool + : (!transform.any_op) -> (!transform.any_op) transform.yield } } @@ -216,7 +271,8 @@ module attributes {transform.with_named_sequence} { transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) { %map = transform.structured.match ops{["linalg.map"]} in %arg1 : (!transform.any_op) -> !transform.any_op - transform.structured.convert_to_loops %map : !transform.any_op + %0 = transform.structured.convert_to_loops %map + : (!transform.any_op) -> (!transform.any_op) transform.yield } } @@ -248,7 +304,8 @@ module attributes {transform.with_named_sequence} { transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) { %transpose = transform.structured.match ops{["linalg.transpose"]} in %arg1 : (!transform.any_op) -> !transform.any_op - transform.structured.convert_to_loops %transpose : !transform.any_op + %0 = transform.structured.convert_to_loops %transpose + : (!transform.any_op) -> (!transform.any_op) transform.yield } } @@ -285,7 +342,8 @@ module attributes {transform.with_named_sequence} { transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) { %reduce = transform.structured.match ops{["linalg.reduce"]} in %arg1 : (!transform.any_op) -> !transform.any_op - transform.structured.convert_to_loops %reduce : !transform.any_op + %0 = transform.structured.convert_to_loops %reduce + : (!transform.any_op) -> (!transform.any_op) transform.yield } } @@ -322,7 +380,8 @@ module attributes {transform.with_named_sequence} { transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) { %broadcast = transform.structured.match ops{["linalg.broadcast"]} in %arg1 : (!transform.any_op) -> !transform.any_op - transform.structured.convert_to_loops %broadcast : !transform.any_op + %0 = transform.structured.convert_to_loops %broadcast + : (!transform.any_op) -> (!transform.any_op) transform.yield } } -- GitLab From 5cd45e442e99f8e01127f37548b124fa0e70f96a Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Wed, 6 Mar 2024 14:03:04 -0800 Subject: [PATCH 365/929] [SCEV] Precommit test for widened signed induction variables These tests highlight that we have missed oppurtunities proving trip count bounds when our start/end values are sign extended from smaller types and we have either a loop guard to relate our start vs end, or a nsw/nuw fact to bound end. --- .../Analysis/ScalarEvolution/trip-count.ll | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count.ll b/llvm/test/Analysis/ScalarEvolution/trip-count.ll index 9dd9e3a5ee99..cbe07effdeb2 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count.ll @@ -140,3 +140,74 @@ loop: exit: ret void } + +define void @dual_sext_ne_with_slt_guard(i8 %s, i8 %n) { +; CHECK-LABEL: 'dual_sext_ne_with_slt_guard' +; CHECK-NEXT: Determining loop execution counts for: @dual_sext_ne_with_slt_guard +; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (sext i8 %n to i64) + (-1 * (sext i8 %s to i64))) +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 -1 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (sext i8 %n to i64) + (-1 * (sext i8 %s to i64))) +; CHECK-NEXT: Loop %for.body: Trip multiple is 1 +; +entry: + %cmp4 = icmp slt i8 %s, %n + br i1 %cmp4, label %for.body.preheader, label %exit + +for.body.preheader: + %0 = sext i8 %s to i64 + %wide.trip.count = sext i8 %n to i64 + br label %for.body + +for.body: + %iv = phi i64 [ %0, %for.body.preheader ], [ %iv.next, %for.body ] + %iv.next = add nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %wide.trip.count + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret void +} + +define void @dual_sext_ne_with_nsw_inc(i8 %s, i64 %n) { +; CHECK-LABEL: 'dual_sext_ne_with_nsw_inc' +; CHECK-NEXT: Determining loop execution counts for: @dual_sext_ne_with_nsw_inc +; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (-1 * (sext i8 %s to i64)) + %n) +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 -1 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (-1 * (sext i8 %s to i64)) + %n) +; CHECK-NEXT: Loop %for.body: Trip multiple is 1 +; +entry: + %0 = sext i8 %s to i64 + br label %for.body + +for.body: + %iv = phi i64 [ %0, %entry ], [ %iv.next, %for.body ] + %iv.next = add nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %n + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret void +} + +define void @dual_sext_ne_with_nuw_inc(i8 %s, i64 %n) { +; CHECK-LABEL: 'dual_sext_ne_with_nuw_inc' +; CHECK-NEXT: Determining loop execution counts for: @dual_sext_ne_with_nuw_inc +; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (-1 * (sext i8 %s to i64)) + %n) +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 -1 +; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (-1 * (sext i8 %s to i64)) + %n) +; CHECK-NEXT: Loop %for.body: Trip multiple is 1 +; +entry: + %0 = sext i8 %s to i64 + br label %for.body + +for.body: + %iv = phi i64 [ %0, %entry ], [ %iv.next, %for.body ] + %iv.next = add nuw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %n + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret void +} -- GitLab From f8c5a68365fb82d9ffc64511f4b50b9c1e68144f Mon Sep 17 00:00:00 2001 From: Duo Wang Date: Wed, 6 Mar 2024 14:19:27 -0800 Subject: [PATCH 366/929] [clang][test] Add test for incompatible cv-qualified reference types in conversion function template (#81950) We currently lack test coverage for [SemaTemplateDeduction.cpp#L1619-L1620](https://github.com/llvm/llvm-project/blob/fe20a75/clang/lib/Sema/SemaTemplateDeduction.cpp#L1619-L1620), which handles the case where both the P type and the A type are reference types but A is more cv-qualified than P. This is deemed non-deduced unless both A and P are possibly cv-qualified forms of the template parameter T. This PR adds tests for that logic. --- .../temp.fct.spec/temp.deduct/temp.deduct.conv/p4.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/clang/test/CXX/temp/temp.fct.spec/temp.deduct/temp.deduct.conv/p4.cpp b/clang/test/CXX/temp/temp.fct.spec/temp.deduct/temp.deduct.conv/p4.cpp index 085976b08133..974240c51484 100644 --- a/clang/test/CXX/temp/temp.fct.spec/temp.deduct/temp.deduct.conv/p4.cpp +++ b/clang/test/CXX/temp/temp.fct.spec/temp.deduct/temp.deduct.conv/p4.cpp @@ -130,6 +130,15 @@ namespace non_ptr_ref_cv_qual { int (&test_conv_to_arr_1)[3] = ConvToArr(); // ok const int (&test_conv_to_arr_2)[3] = ConvToArr(); // ok, with qualification conversion + struct ConvToConstArr { + template + operator const Arr &() { // expected-note {{candidate}} + static_assert(N == 3, ""); + } + }; + Arr &test_conv_to_const_arr_1 = ConvToConstArr(); // expected-error {{no viable}} + const Arr &test_conv_to_const_arr_2 = ConvToConstArr(); // ok + #if __cplusplus >= 201702L template using Function = T(U...) noexcept(Noexcept); template struct ConvToFunction { -- GitLab From d34b3c9c5ab75c2ed0d7023889a1dcb4ce0a0f99 Mon Sep 17 00:00:00 2001 From: Michael Jones Date: Wed, 6 Mar 2024 14:30:05 -0800 Subject: [PATCH 367/929] [libc] Add max length argument to decimal to float (#84091) The implementation for from_chars in libcxx is possibly going to use our decimal to float utilities, but to do that we need to support limiting the length of the string to be parsed. This patch adds support for that length limiting to decimal_exp_to_float, as well as the functions it calls (high precision decimal, str to integer). --- libc/src/__support/high_precision_decimal.h | 116 +++++---- libc/src/__support/str_to_float.h | 40 +-- libc/src/__support/str_to_integer.h | 85 ++++--- libc/test/src/CMakeLists.txt | 4 +- libc/test/src/__support/CMakeLists.txt | 13 + .../__support/high_precision_decimal_test.cpp | 28 ++ .../src/__support/str_to_integer_test.cpp | 240 ++++++++++++++++++ 7 files changed, 418 insertions(+), 108 deletions(-) create mode 100644 libc/test/src/__support/str_to_integer_test.cpp diff --git a/libc/src/__support/high_precision_decimal.h b/libc/src/__support/high_precision_decimal.h index d29f8c4cd932..2c5a349e4495 100644 --- a/libc/src/__support/high_precision_decimal.h +++ b/libc/src/__support/high_precision_decimal.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_SRC___SUPPORT_HIGH_PRECISION_DECIMAL_H #define LLVM_LIBC_SRC___SUPPORT_HIGH_PRECISION_DECIMAL_H +#include "src/__support/CPP/limits.h" #include "src/__support/ctype_utils.h" #include "src/__support/str_to_integer.h" #include @@ -115,9 +116,10 @@ class HighPrecisionDecimal { uint8_t digits[MAX_NUM_DIGITS]; private: - bool should_round_up(int32_t roundToDigit, RoundDirection round) { - if (roundToDigit < 0 || - static_cast(roundToDigit) >= this->num_digits) { + LIBC_INLINE bool should_round_up(int32_t round_to_digit, + RoundDirection round) { + if (round_to_digit < 0 || + static_cast(round_to_digit) >= this->num_digits) { return false; } @@ -133,8 +135,8 @@ private: // Else round to nearest. // If we're right in the middle and there are no extra digits - if (this->digits[roundToDigit] == 5 && - static_cast(roundToDigit + 1) == this->num_digits) { + if (this->digits[round_to_digit] == 5 && + static_cast(round_to_digit + 1) == this->num_digits) { // Round up if we've truncated (since that means the result is slightly // higher than what's represented.) @@ -143,22 +145,22 @@ private: } // If this exactly halfway, round to even. - if (roundToDigit == 0) + if (round_to_digit == 0) // When the input is ".5". return false; - return this->digits[roundToDigit - 1] % 2 != 0; + return this->digits[round_to_digit - 1] % 2 != 0; } - // If there are digits after roundToDigit, they must be non-zero since we + // If there are digits after round_to_digit, they must be non-zero since we // trim trailing zeroes after all operations that change digits. - return this->digits[roundToDigit] >= 5; + return this->digits[round_to_digit] >= 5; } // Takes an amount to left shift and returns the number of new digits needed // to store the result based on LEFT_SHIFT_DIGIT_TABLE. - uint32_t get_num_new_digits(uint32_t lShiftAmount) { + LIBC_INLINE uint32_t get_num_new_digits(uint32_t lshift_amount) { const char *power_of_five = - LEFT_SHIFT_DIGIT_TABLE[lShiftAmount].power_of_five; - uint32_t new_digits = LEFT_SHIFT_DIGIT_TABLE[lShiftAmount].new_digits; + LEFT_SHIFT_DIGIT_TABLE[lshift_amount].power_of_five; + uint32_t new_digits = LEFT_SHIFT_DIGIT_TABLE[lshift_amount].new_digits; uint32_t digit_index = 0; while (power_of_five[digit_index] != 0) { if (digit_index >= this->num_digits) { @@ -176,7 +178,7 @@ private: } // Trim all trailing 0s - void trim_trailing_zeroes() { + LIBC_INLINE void trim_trailing_zeroes() { while (this->num_digits > 0 && this->digits[this->num_digits - 1] == 0) { --this->num_digits; } @@ -186,19 +188,19 @@ private: } // Perform a digitwise binary non-rounding right shift on this value by - // shiftAmount. The shiftAmount can't be more than MAX_SHIFT_AMOUNT to prevent - // overflow. - void right_shift(uint32_t shiftAmount) { + // shift_amount. The shift_amount can't be more than MAX_SHIFT_AMOUNT to + // prevent overflow. + LIBC_INLINE void right_shift(uint32_t shift_amount) { uint32_t read_index = 0; uint32_t write_index = 0; uint64_t accumulator = 0; - const uint64_t shift_mask = (uint64_t(1) << shiftAmount) - 1; + const uint64_t shift_mask = (uint64_t(1) << shift_amount) - 1; // Warm Up phase: we don't have enough digits to start writing, so just // read them into the accumulator. - while (accumulator >> shiftAmount == 0) { + while (accumulator >> shift_amount == 0) { uint64_t read_digit = 0; // If there are still digits to read, read the next one, else the digit is // assumed to be 0. @@ -217,7 +219,7 @@ private: // read. Keep reading until we run out of digits. while (read_index < this->num_digits) { uint64_t read_digit = this->digits[read_index]; - uint64_t write_digit = accumulator >> shiftAmount; + uint64_t write_digit = accumulator >> shift_amount; accumulator &= shift_mask; this->digits[write_index] = static_cast(write_digit); accumulator = accumulator * 10 + read_digit; @@ -228,7 +230,7 @@ private: // Cool Down phase: All of the readable digits have been read, so just write // the remainder, while treating any more digits as 0. while (accumulator > 0) { - uint64_t write_digit = accumulator >> shiftAmount; + uint64_t write_digit = accumulator >> shift_amount; accumulator &= shift_mask; if (write_index < MAX_NUM_DIGITS) { this->digits[write_index] = static_cast(write_digit); @@ -243,10 +245,10 @@ private: } // Perform a digitwise binary non-rounding left shift on this value by - // shiftAmount. The shiftAmount can't be more than MAX_SHIFT_AMOUNT to prevent - // overflow. - void left_shift(uint32_t shiftAmount) { - uint32_t new_digits = this->get_num_new_digits(shiftAmount); + // shift_amount. The shift_amount can't be more than MAX_SHIFT_AMOUNT to + // prevent overflow. + LIBC_INLINE void left_shift(uint32_t shift_amount) { + uint32_t new_digits = this->get_num_new_digits(shift_amount); int32_t read_index = this->num_digits - 1; uint32_t write_index = this->num_digits + new_digits; @@ -260,7 +262,7 @@ private: // writing. while (read_index >= 0) { accumulator += static_cast(this->digits[read_index]) - << shiftAmount; + << shift_amount; uint64_t next_accumulator = accumulator / 10; uint64_t write_digit = accumulator - (10 * next_accumulator); --write_index; @@ -296,45 +298,52 @@ private: } public: - // numString is assumed to be a string of numeric characters. It doesn't + // num_string is assumed to be a string of numeric characters. It doesn't // handle leading spaces. - HighPrecisionDecimal(const char *__restrict numString) { + LIBC_INLINE + HighPrecisionDecimal( + const char *__restrict num_string, + const size_t num_len = cpp::numeric_limits::max()) { bool saw_dot = false; + size_t num_cur = 0; // This counts the digits in the number, even if there isn't space to store // them all. uint32_t total_digits = 0; - while (isdigit(*numString) || *numString == '.') { - if (*numString == '.') { + while (num_cur < num_len && + (isdigit(num_string[num_cur]) || num_string[num_cur] == '.')) { + if (num_string[num_cur] == '.') { if (saw_dot) { break; } this->decimal_point = total_digits; saw_dot = true; } else { - if (*numString == '0' && this->num_digits == 0) { + if (num_string[num_cur] == '0' && this->num_digits == 0) { --this->decimal_point; - ++numString; + ++num_cur; continue; } ++total_digits; if (this->num_digits < MAX_NUM_DIGITS) { this->digits[this->num_digits] = - static_cast(*numString - '0'); + static_cast(num_string[num_cur] - '0'); ++this->num_digits; - } else if (*numString != '0') { + } else if (num_string[num_cur] != '0') { this->truncated = true; } } - ++numString; + ++num_cur; } if (!saw_dot) this->decimal_point = total_digits; - if ((*numString | 32) == 'e') { - ++numString; - if (isdigit(*numString) || *numString == '+' || *numString == '-') { - auto result = strtointeger(numString, 10); + if (num_cur < num_len && ((num_string[num_cur] | 32) == 'e')) { + ++num_cur; + if (isdigit(num_string[num_cur]) || num_string[num_cur] == '+' || + num_string[num_cur] == '-') { + auto result = + strtointeger(num_string + num_cur, 10, num_len - num_cur); if (result.has_error()) { // TODO: handle error } @@ -358,33 +367,34 @@ public: this->trim_trailing_zeroes(); } - // Binary shift left (shiftAmount > 0) or right (shiftAmount < 0) - void shift(int shiftAmount) { - if (shiftAmount == 0) { + // Binary shift left (shift_amount > 0) or right (shift_amount < 0) + LIBC_INLINE void shift(int shift_amount) { + if (shift_amount == 0) { return; } // Left - else if (shiftAmount > 0) { - while (static_cast(shiftAmount) > MAX_SHIFT_AMOUNT) { + else if (shift_amount > 0) { + while (static_cast(shift_amount) > MAX_SHIFT_AMOUNT) { this->left_shift(MAX_SHIFT_AMOUNT); - shiftAmount -= MAX_SHIFT_AMOUNT; + shift_amount -= MAX_SHIFT_AMOUNT; } - this->left_shift(shiftAmount); + this->left_shift(shift_amount); } // Right else { - while (static_cast(shiftAmount) < -MAX_SHIFT_AMOUNT) { + while (static_cast(shift_amount) < -MAX_SHIFT_AMOUNT) { this->right_shift(MAX_SHIFT_AMOUNT); - shiftAmount += MAX_SHIFT_AMOUNT; + shift_amount += MAX_SHIFT_AMOUNT; } - this->right_shift(-shiftAmount); + this->right_shift(-shift_amount); } } // Round the number represented to the closest value of unsigned int type T. // This is done ignoring overflow. template - T round_to_integer_type(RoundDirection round = RoundDirection::Nearest) { + LIBC_INLINE T + round_to_integer_type(RoundDirection round = RoundDirection::Nearest) { T result = 0; uint32_t cur_digit = 0; @@ -404,10 +414,10 @@ public: // Extra functions for testing. - uint8_t *get_digits() { return this->digits; } - uint32_t get_num_digits() { return this->num_digits; } - int32_t get_decimal_point() { return this->decimal_point; } - void set_truncated(bool trunc) { this->truncated = trunc; } + LIBC_INLINE uint8_t *get_digits() { return this->digits; } + LIBC_INLINE uint32_t get_num_digits() { return this->num_digits; } + LIBC_INLINE int32_t get_decimal_point() { return this->decimal_point; } + LIBC_INLINE void set_truncated(bool trunc) { this->truncated = trunc; } }; } // namespace internal diff --git a/libc/src/__support/str_to_float.h b/libc/src/__support/str_to_float.h index d2bf3f85b270..6caf8e62a454 100644 --- a/libc/src/__support/str_to_float.h +++ b/libc/src/__support/str_to_float.h @@ -313,14 +313,15 @@ constexpr int32_t NUM_POWERS_OF_TWO = // on the Simple Decimal Conversion algorithm by Nigel Tao, described at this // link: https://nigeltao.github.io/blog/2020/parse-number-f64-simple.html template -LIBC_INLINE FloatConvertReturn -simple_decimal_conversion(const char *__restrict numStart, - RoundDirection round = RoundDirection::Nearest) { +LIBC_INLINE FloatConvertReturn simple_decimal_conversion( + const char *__restrict numStart, + const size_t num_len = cpp::numeric_limits::max(), + RoundDirection round = RoundDirection::Nearest) { using FPBits = typename fputil::FPBits; using StorageType = typename FPBits::StorageType; int32_t exp2 = 0; - HighPrecisionDecimal hpd = HighPrecisionDecimal(numStart); + HighPrecisionDecimal hpd = HighPrecisionDecimal(numStart, num_len); FloatConvertReturn output; @@ -600,13 +601,17 @@ clinger_fast_path(ExpandedFloat init_num, // non-inf result for this size of float. The value is // log10(2^(exponent bias)). // The generic approximation uses the fact that log10(2^x) ~= x/3 -template constexpr int32_t get_upper_bound() { +template LIBC_INLINE constexpr int32_t get_upper_bound() { return fputil::FPBits::EXP_BIAS / 3; } -template <> constexpr int32_t get_upper_bound() { return 39; } +template <> LIBC_INLINE constexpr int32_t get_upper_bound() { + return 39; +} -template <> constexpr int32_t get_upper_bound() { return 309; } +template <> LIBC_INLINE constexpr int32_t get_upper_bound() { + return 309; +} // The lower bound is the largest negative base-10 exponent that could possibly // give a non-zero result for this size of float. The value is @@ -616,18 +621,18 @@ template <> constexpr int32_t get_upper_bound() { return 309; } // low base 10 exponent with a very high intermediate mantissa can cancel each // other out, and subnormal numbers allow for the result to be at the very low // end of the final mantissa. -template constexpr int32_t get_lower_bound() { +template LIBC_INLINE constexpr int32_t get_lower_bound() { using FPBits = typename fputil::FPBits; return -((FPBits::EXP_BIAS + static_cast(FPBits::FRACTION_LEN + FPBits::STORAGE_LEN)) / 3); } -template <> constexpr int32_t get_lower_bound() { +template <> LIBC_INLINE constexpr int32_t get_lower_bound() { return -(39 + 6 + 10); } -template <> constexpr int32_t get_lower_bound() { +template <> LIBC_INLINE constexpr int32_t get_lower_bound() { return -(309 + 15 + 20); } @@ -637,9 +642,10 @@ template <> constexpr int32_t get_lower_bound() { // accuracy. The resulting mantissa and exponent are placed in outputMantissa // and outputExp2. template -LIBC_INLINE FloatConvertReturn -decimal_exp_to_float(ExpandedFloat init_num, const char *__restrict numStart, - bool truncated, RoundDirection round) { +LIBC_INLINE FloatConvertReturn decimal_exp_to_float( + ExpandedFloat init_num, bool truncated, RoundDirection round, + const char *__restrict numStart, + const size_t num_len = cpp::numeric_limits::max()) { using FPBits = typename fputil::FPBits; using StorageType = typename FPBits::StorageType; @@ -701,7 +707,7 @@ decimal_exp_to_float(ExpandedFloat init_num, const char *__restrict numStart, #endif // LIBC_COPT_STRTOFLOAT_DISABLE_EISEL_LEMIRE #ifndef LIBC_COPT_STRTOFLOAT_DISABLE_SIMPLE_DECIMAL_CONVERSION - output = simple_decimal_conversion(numStart, round); + output = simple_decimal_conversion(numStart, num_len, round); #else #warning "Simple decimal conversion is disabled, result may not be correct." #endif // LIBC_COPT_STRTOFLOAT_DISABLE_SIMPLE_DECIMAL_CONVERSION @@ -894,6 +900,8 @@ decimal_string_to_float(const char *__restrict src, const char DECIMAL_POINT, if (!seen_digit) return output; + // TODO: When adding max length argument, handle the case of a trailing + // EXPONENT MARKER, see scanf for more details. if (tolower(src[index]) == EXPONENT_MARKER) { bool has_sign = false; if (src[index + 1] == '+' || src[index + 1] == '-') { @@ -928,7 +936,7 @@ decimal_string_to_float(const char *__restrict src, const char DECIMAL_POINT, output.value = {0, 0}; } else { auto temp = - decimal_exp_to_float({mantissa, exponent}, src, truncated, round); + decimal_exp_to_float({mantissa, exponent}, truncated, round, src); output.value = temp.num; output.error = temp.error; } @@ -1071,6 +1079,8 @@ nan_mantissa_from_ncharseq(const cpp::string_view ncharseq) { // Takes a pointer to a string and a pointer to a string pointer. This function // is used as the backend for all of the string to float functions. +// TODO: Add src_len member to match strtointeger. +// TODO: Next, move from char* and length to string_view template LIBC_INLINE StrToNumResult strtofloatingpoint(const char *__restrict src) { using FPBits = typename fputil::FPBits; diff --git a/libc/src/__support/str_to_integer.h b/libc/src/__support/str_to_integer.h index e83a508e086b..b87808993fee 100644 --- a/libc/src/__support/str_to_integer.h +++ b/libc/src/__support/str_to_integer.h @@ -21,11 +21,15 @@ namespace internal { // Returns a pointer to the first character in src that is not a whitespace // character (as determined by isspace()) -LIBC_INLINE const char *first_non_whitespace(const char *__restrict src) { - while (internal::isspace(*src)) { - ++src; +// TODO: Change from returning a pointer to returning a length. +LIBC_INLINE const char * +first_non_whitespace(const char *__restrict src, + size_t src_len = cpp::numeric_limits::max()) { + size_t src_cur = 0; + while (src_cur < src_len && internal::isspace(src[src_cur])) { + ++src_cur; } - return src; + return src + src_cur; } LIBC_INLINE int b36_char_to_int(char input) { @@ -38,61 +42,64 @@ LIBC_INLINE int b36_char_to_int(char input) { // checks if the next 3 characters of the string pointer are the start of a // hexadecimal number. Does not advance the string pointer. -LIBC_INLINE bool is_hex_start(const char *__restrict src) { +LIBC_INLINE bool +is_hex_start(const char *__restrict src, + size_t src_len = cpp::numeric_limits::max()) { + if (src_len < 3) + return false; return *src == '0' && (*(src + 1) | 32) == 'x' && isalnum(*(src + 2)) && b36_char_to_int(*(src + 2)) < 16; } // Takes the address of the string pointer and parses the base from the start of -// it. This function will advance |src| to the first valid digit in the inferred -// base. -LIBC_INLINE int infer_base(const char *__restrict *__restrict src) { +// it. +LIBC_INLINE int infer_base(const char *__restrict src, size_t src_len) { // A hexadecimal number is defined as "the prefix 0x or 0X followed by a // sequence of the decimal digits and the letters a (or A) through f (or F) // with values 10 through 15 respectively." (C standard 6.4.4.1) - if (is_hex_start(*src)) { - (*src) += 2; + if (is_hex_start(src, src_len)) return 16; - } // An octal number is defined as "the prefix 0 optionally followed by a - // sequence of the digits 0 through 7 only" (C standard 6.4.4.1) and so any - // number that starts with 0, including just 0, is an octal number. - else if (**src == '0') { + // An octal number is defined as "the prefix 0 optionally followed by a + // sequence of the digits 0 through 7 only" (C standard 6.4.4.1) and so any + // number that starts with 0, including just 0, is an octal number. + if (src_len > 0 && src[0] == '0') return 8; - } // A decimal number is defined as beginning "with a nonzero digit and - // consist[ing] of a sequence of decimal digits." (C standard 6.4.4.1) - else { - return 10; - } + // A decimal number is defined as beginning "with a nonzero digit and + // consist[ing] of a sequence of decimal digits." (C standard 6.4.4.1) + return 10; } // Takes a pointer to a string and the base to convert to. This function is used // as the backend for all of the string to int functions. template -LIBC_INLINE StrToNumResult strtointeger(const char *__restrict src, - int base) { +LIBC_INLINE StrToNumResult +strtointeger(const char *__restrict src, int base, + const size_t src_len = cpp::numeric_limits::max()) { + // TODO: Rewrite to support numbers longer than long long unsigned long long result = 0; bool is_number = false; - const char *original_src = src; + size_t src_cur = 0; int error_val = 0; - if (base < 0 || base == 1 || base > 36) { - error_val = EINVAL; - return {0, 0, error_val}; - } + if (src_len == 0) + return {0, 0, 0}; + + if (base < 0 || base == 1 || base > 36) + return {0, 0, EINVAL}; - src = first_non_whitespace(src); + src_cur = first_non_whitespace(src, src_len) - src; char result_sign = '+'; - if (*src == '+' || *src == '-') { - result_sign = *src; - ++src; + if (src[src_cur] == '+' || src[src_cur] == '-') { + result_sign = src[src_cur]; + ++src_cur; } - if (base == 0) { - base = infer_base(&src); - } else if (base == 16 && is_hex_start(src)) { - src = src + 2; - } + if (base == 0) + base = infer_base(src + src_cur, src_len - src_cur); + + if (base == 16 && is_hex_start(src + src_cur, src_len - src_cur)) + src_cur = src_cur + 2; constexpr bool IS_UNSIGNED = (cpp::numeric_limits::min() == 0); const bool is_positive = (result_sign == '+'); @@ -103,13 +110,13 @@ LIBC_INLINE StrToNumResult strtointeger(const char *__restrict src, unsigned long long const abs_max = (is_positive ? cpp::numeric_limits::max() : NEGATIVE_MAX); unsigned long long const abs_max_div_by_base = abs_max / base; - while (isalnum(*src)) { - int cur_digit = b36_char_to_int(*src); + while (src_cur < src_len && isalnum(src[src_cur])) { + int cur_digit = b36_char_to_int(src[src_cur]); if (cur_digit >= base) break; is_number = true; - ++src; + ++src_cur; // If the number has already hit the maximum value for the current type then // the result cannot change, but we still need to advance src to the end of @@ -133,7 +140,7 @@ LIBC_INLINE StrToNumResult strtointeger(const char *__restrict src, } } - ptrdiff_t str_len = is_number ? (src - original_src) : 0; + ptrdiff_t str_len = is_number ? (src_cur) : 0; if (error_val == ERANGE) { if (is_positive || IS_UNSIGNED) diff --git a/libc/test/src/CMakeLists.txt b/libc/test/src/CMakeLists.txt index 9ad868551f07..f70ffda3f700 100644 --- a/libc/test/src/CMakeLists.txt +++ b/libc/test/src/CMakeLists.txt @@ -40,7 +40,6 @@ add_subdirectory(__support) add_subdirectory(ctype) add_subdirectory(errno) add_subdirectory(fenv) -add_subdirectory(inttypes) add_subdirectory(math) add_subdirectory(search) add_subdirectory(stdbit) @@ -50,6 +49,9 @@ add_subdirectory(stdlib) add_subdirectory(string) add_subdirectory(wchar) +# Depends on utilities in stdlib +add_subdirectory(inttypes) + if(${LIBC_TARGET_OS} STREQUAL "linux") add_subdirectory(fcntl) add_subdirectory(sched) diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt index 7200ac276fe5..8c861b576f9b 100644 --- a/libc/test/src/__support/CMakeLists.txt +++ b/libc/test/src/__support/CMakeLists.txt @@ -56,6 +56,19 @@ add_libc_test( libc.src.errno.errno ) + +add_libc_test( + str_to_integer_test + SUITE + libc-support-tests + SRCS + str_to_integer_test.cpp + DEPENDS + libc.src.__support.integer_literals + libc.src.__support.str_to_integer + libc.src.errno.errno +) + add_libc_test( integer_to_string_test SUITE diff --git a/libc/test/src/__support/high_precision_decimal_test.cpp b/libc/test/src/__support/high_precision_decimal_test.cpp index a9c039e45774..2bb28bcdab02 100644 --- a/libc/test/src/__support/high_precision_decimal_test.cpp +++ b/libc/test/src/__support/high_precision_decimal_test.cpp @@ -406,3 +406,31 @@ TEST(LlvmLibcHighPrecisionDecimalTest, BigExpTest) { // Same, but since the number is negative the net result is -123456788 EXPECT_EQ(big_negative_hpd.get_decimal_point(), -123456789 + 1); } + +TEST(LlvmLibcHighPrecisionDecimalTest, NumLenExpTest) { + LIBC_NAMESPACE::internal::HighPrecisionDecimal hpd = + LIBC_NAMESPACE::internal::HighPrecisionDecimal("1e123456789", 5); + + // The length of 5 includes things like the "e" so it only gets 3 digits of + // exponent. + EXPECT_EQ(hpd.get_decimal_point(), 123 + 1); + + LIBC_NAMESPACE::internal::HighPrecisionDecimal negative_hpd = + LIBC_NAMESPACE::internal::HighPrecisionDecimal("1e-123456789", 5); + + // The negative sign also counts as a character. + EXPECT_EQ(negative_hpd.get_decimal_point(), -12 + 1); +} + +TEST(LlvmLibcHighPrecisionDecimalTest, NumLenDigitsTest) { + LIBC_NAMESPACE::internal::HighPrecisionDecimal hpd = + LIBC_NAMESPACE::internal::HighPrecisionDecimal("123456789e1", 5); + + EXPECT_EQ(hpd.round_to_integer_type(), uint64_t(12345)); + + LIBC_NAMESPACE::internal::HighPrecisionDecimal longer_hpd = + LIBC_NAMESPACE::internal::HighPrecisionDecimal("123456789e1", 10); + + // With 10 characters it should see the e, but not actually act on it. + EXPECT_EQ(longer_hpd.round_to_integer_type(), uint64_t(123456789)); +} diff --git a/libc/test/src/__support/str_to_integer_test.cpp b/libc/test/src/__support/str_to_integer_test.cpp new file mode 100644 index 000000000000..34b645b4b38c --- /dev/null +++ b/libc/test/src/__support/str_to_integer_test.cpp @@ -0,0 +1,240 @@ +//===-- Unittests for str_to_integer --------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/str_to_integer.h" +#include "src/errno/libc_errno.h" +#include + +#include "test/UnitTest/Test.h" + +// This file is for testing the src_len argument and other internal interface +// features. Primary testing is done in stdlib/StrolTest.cpp through the public +// interface. + +TEST(LlvmLibcStrToIntegerTest, SimpleLength) { + auto result = LIBC_NAMESPACE::internal::strtointeger("12345", 10, 10); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(5)); + ASSERT_EQ(result.value, 12345); + + result = LIBC_NAMESPACE::internal::strtointeger("12345", 10, 2); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(2)); + ASSERT_EQ(result.value, 12); + + result = LIBC_NAMESPACE::internal::strtointeger("12345", 10, 0); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(0)); + ASSERT_EQ(result.value, 0); +} + +TEST(LlvmLibcStrToIntegerTest, LeadingSpaces) { + auto result = + LIBC_NAMESPACE::internal::strtointeger(" 12345", 10, 15); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(10)); + ASSERT_EQ(result.value, 12345); + + result = LIBC_NAMESPACE::internal::strtointeger(" 12345", 10, 10); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(10)); + ASSERT_EQ(result.value, 12345); + + result = LIBC_NAMESPACE::internal::strtointeger(" 12345", 10, 7); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(7)); + ASSERT_EQ(result.value, 12); + + result = LIBC_NAMESPACE::internal::strtointeger(" 12345", 10, 5); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(0)); + ASSERT_EQ(result.value, 0); + + result = LIBC_NAMESPACE::internal::strtointeger(" 12345", 10, 0); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(0)); + ASSERT_EQ(result.value, 0); +} + +TEST(LlvmLibcStrToIntegerTest, LeadingSign) { + auto result = LIBC_NAMESPACE::internal::strtointeger("+12345", 10, 10); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(6)); + ASSERT_EQ(result.value, 12345); + + result = LIBC_NAMESPACE::internal::strtointeger("-12345", 10, 10); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(6)); + ASSERT_EQ(result.value, -12345); + + result = LIBC_NAMESPACE::internal::strtointeger("+12345", 10, 6); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(6)); + ASSERT_EQ(result.value, 12345); + + result = LIBC_NAMESPACE::internal::strtointeger("-12345", 10, 6); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(6)); + ASSERT_EQ(result.value, -12345); + + result = LIBC_NAMESPACE::internal::strtointeger("+12345", 10, 3); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(3)); + ASSERT_EQ(result.value, 12); + + result = LIBC_NAMESPACE::internal::strtointeger("-12345", 10, 3); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(3)); + ASSERT_EQ(result.value, -12); + + result = LIBC_NAMESPACE::internal::strtointeger("+12345", 10, 1); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(0)); + ASSERT_EQ(result.value, 0); + + result = LIBC_NAMESPACE::internal::strtointeger("-12345", 10, 1); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(0)); + ASSERT_EQ(result.value, 0); + + result = LIBC_NAMESPACE::internal::strtointeger("+12345", 10, 0); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(0)); + ASSERT_EQ(result.value, 0); + + result = LIBC_NAMESPACE::internal::strtointeger("-12345", 10, 0); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(0)); + ASSERT_EQ(result.value, 0); +} + +TEST(LlvmLibcStrToIntegerTest, Base16PrefixAutoSelect) { + auto result = LIBC_NAMESPACE::internal::strtointeger("0x12345", 0, 10); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(7)); + ASSERT_EQ(result.value, 0x12345); + + result = LIBC_NAMESPACE::internal::strtointeger("0x12345", 0, 7); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(7)); + ASSERT_EQ(result.value, 0x12345); + + result = LIBC_NAMESPACE::internal::strtointeger("0x12345", 0, 5); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(5)); + ASSERT_EQ(result.value, 0x123); + + result = LIBC_NAMESPACE::internal::strtointeger("0x12345", 0, 2); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(1)); + ASSERT_EQ(result.value, 0); + + result = LIBC_NAMESPACE::internal::strtointeger("0x12345", 0, 0); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(0)); + ASSERT_EQ(result.value, 0); +} + +TEST(LlvmLibcStrToIntegerTest, Base16PrefixManualSelect) { + auto result = LIBC_NAMESPACE::internal::strtointeger("0x12345", 16, 10); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(7)); + ASSERT_EQ(result.value, 0x12345); + + result = LIBC_NAMESPACE::internal::strtointeger("0x12345", 16, 7); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(7)); + ASSERT_EQ(result.value, 0x12345); + + result = LIBC_NAMESPACE::internal::strtointeger("0x12345", 16, 5); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(5)); + ASSERT_EQ(result.value, 0x123); + + result = LIBC_NAMESPACE::internal::strtointeger("0x12345", 16, 2); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(1)); + ASSERT_EQ(result.value, 0); + + result = LIBC_NAMESPACE::internal::strtointeger("0x12345", 16, 0); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(0)); + ASSERT_EQ(result.value, 0); +} + +TEST(LlvmLibcStrToIntegerTest, Base8PrefixAutoSelect) { + auto result = LIBC_NAMESPACE::internal::strtointeger("012345", 0, 10); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(6)); + ASSERT_EQ(result.value, 012345); + + result = LIBC_NAMESPACE::internal::strtointeger("012345", 0, 6); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(6)); + ASSERT_EQ(result.value, 012345); + + result = LIBC_NAMESPACE::internal::strtointeger("012345", 0, 4); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(4)); + ASSERT_EQ(result.value, 0123); + + result = LIBC_NAMESPACE::internal::strtointeger("012345", 0, 1); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(1)); + ASSERT_EQ(result.value, 0); + + result = LIBC_NAMESPACE::internal::strtointeger("012345", 0, 0); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(0)); + ASSERT_EQ(result.value, 0); +} + +TEST(LlvmLibcStrToIntegerTest, Base8PrefixManualSelect) { + auto result = LIBC_NAMESPACE::internal::strtointeger("012345", 8, 10); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(6)); + ASSERT_EQ(result.value, 012345); + + result = LIBC_NAMESPACE::internal::strtointeger("012345", 8, 6); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(6)); + ASSERT_EQ(result.value, 012345); + + result = LIBC_NAMESPACE::internal::strtointeger("012345", 8, 4); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(4)); + ASSERT_EQ(result.value, 0123); + + result = LIBC_NAMESPACE::internal::strtointeger("012345", 8, 1); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(1)); + ASSERT_EQ(result.value, 0); + + result = LIBC_NAMESPACE::internal::strtointeger("012345", 8, 0); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(0)); + ASSERT_EQ(result.value, 0); +} + +TEST(LlvmLibcStrToIntegerTest, CombinedTests) { + auto result = + LIBC_NAMESPACE::internal::strtointeger(" -0x123", 0, 10); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(10)); + ASSERT_EQ(result.value, -0x123); + + result = LIBC_NAMESPACE::internal::strtointeger(" -0x123", 0, 8); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(8)); + ASSERT_EQ(result.value, -0x1); + + result = LIBC_NAMESPACE::internal::strtointeger(" -0x123", 0, 7); + EXPECT_FALSE(result.has_error()); + EXPECT_EQ(result.parsed_len, ptrdiff_t(6)); + ASSERT_EQ(result.value, 0); +} -- GitLab From 395bc7909e9c65cc40680c6a5b8f17f54b5f61f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingo=20M=C3=BCller?= Date: Wed, 6 Mar 2024 23:36:21 +0100 Subject: [PATCH 368/929] [mlir][bazel] Don't set MLIR_ENABLE_NVPTXCOMPILER on if_cuda_available. (#84238) was available. However, it turns out that the NVPTX compiler is not part of every CUDA distribution, so `if_cuda_available` may evaluate to true without that compiler being present, which breaks the build. This PR thus sets the macro to 0 always (which was the behavior before #84007). --- utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 89ed837fc10f..0c49ced55154 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -35,17 +35,12 @@ expand_template( "#cmakedefine01 MLIR_DEPRECATED_GPU_SERIALIZATION_ENABLE": "#define MLIR_DEPRECATED_GPU_SERIALIZATION_ENABLE 0", "#cmakedefine01 MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS": "#define MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS 0", "#cmakedefine MLIR_GREEDY_REWRITE_RANDOMIZER_SEED ${MLIR_GREEDY_REWRITE_RANDOMIZER_SEED}": "/* #undef MLIR_GREEDY_REWRITE_RANDOMIZER_SEED */", + "#cmakedefine01 MLIR_ENABLE_NVPTXCOMPILER": "#define MLIR_ENABLE_NVPTXCOMPILER 0", "#cmakedefine01 MLIR_ENABLE_PDL_IN_PATTERNMATCH": "#define MLIR_ENABLE_PDL_IN_PATTERNMATCH 1", "#cmakedefine01 MLIR_ENABLE_ROCM_CONVERSIONS": "#define MLIR_ENABLE_ROCM_CONVERSIONS 0", } | if_cuda_available( - { - "#cmakedefine01 MLIR_ENABLE_CUDA_CONVERSIONS": "#define MLIR_ENABLE_CUDA_CONVERSIONS 1", - "#cmakedefine01 MLIR_ENABLE_NVPTXCOMPILER": "#define MLIR_ENABLE_NVPTXCOMPILER 1", - }, - { - "#cmakedefine01 MLIR_ENABLE_CUDA_CONVERSIONS": "#define MLIR_ENABLE_CUDA_CONVERSIONS 0", - "#cmakedefine01 MLIR_ENABLE_NVPTXCOMPILER": "#define MLIR_ENABLE_NVPTXCOMPILER 0", - }, + {"#cmakedefine01 MLIR_ENABLE_CUDA_CONVERSIONS": "#define MLIR_ENABLE_CUDA_CONVERSIONS 1"}, + {"#cmakedefine01 MLIR_ENABLE_CUDA_CONVERSIONS": "#define MLIR_ENABLE_CUDA_CONVERSIONS 0"}, ), template = "include/mlir/Config/mlir-config.h.cmake", ) -- GitLab From 96813de52d09d983d14a30ef5e2b4b8658f8e952 Mon Sep 17 00:00:00 2001 From: Changpeng Fang Date: Wed, 6 Mar 2024 14:37:03 -0800 Subject: [PATCH 369/929] AMDGPU: Define a feature for v_dot4_f32_* instructions (#84248) FeatureDot11Insts (dot11-insts) for: v_dot4_f32_fp8_fp8, v_dot4_f32_fp8_bf8, v_dot4_f32_bf8_fp8, v_dot4_f32_bf8_bf8 --- clang/include/clang/Basic/BuiltinsAMDGPU.def | 8 ++++---- clang/test/CodeGenOpenCL/amdgpu-features.cl | 4 ++-- .../test/CodeGenOpenCL/builtins-amdgcn-dl-insts-err.cl | 8 ++++---- llvm/lib/Target/AMDGPU/AMDGPU.td | 10 ++++++++++ llvm/lib/Target/AMDGPU/GCNSubtarget.h | 5 +++++ llvm/lib/Target/AMDGPU/VOP3PInstructions.td | 2 ++ llvm/lib/TargetParser/TargetParser.cpp | 1 + 7 files changed, 28 insertions(+), 10 deletions(-) diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def index 6628e8f265fe..61ec8b79bf05 100644 --- a/clang/include/clang/Basic/BuiltinsAMDGPU.def +++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def @@ -256,10 +256,10 @@ TARGET_BUILTIN(__builtin_amdgcn_sudot4, "iIbiIbiiIb", "nc", "dot8-insts") TARGET_BUILTIN(__builtin_amdgcn_sdot8, "SiSiSiSiIb", "nc", "dot1-insts") TARGET_BUILTIN(__builtin_amdgcn_udot8, "UiUiUiUiIb", "nc", "dot7-insts") TARGET_BUILTIN(__builtin_amdgcn_sudot8, "iIbiIbiiIb", "nc", "dot8-insts") -TARGET_BUILTIN(__builtin_amdgcn_dot4_f32_fp8_bf8, "fUiUif", "nc", "gfx12-insts") -TARGET_BUILTIN(__builtin_amdgcn_dot4_f32_bf8_fp8, "fUiUif", "nc", "gfx12-insts") -TARGET_BUILTIN(__builtin_amdgcn_dot4_f32_fp8_fp8, "fUiUif", "nc", "gfx12-insts") -TARGET_BUILTIN(__builtin_amdgcn_dot4_f32_bf8_bf8, "fUiUif", "nc", "gfx12-insts") +TARGET_BUILTIN(__builtin_amdgcn_dot4_f32_fp8_bf8, "fUiUif", "nc", "dot11-insts") +TARGET_BUILTIN(__builtin_amdgcn_dot4_f32_bf8_fp8, "fUiUif", "nc", "dot11-insts") +TARGET_BUILTIN(__builtin_amdgcn_dot4_f32_fp8_fp8, "fUiUif", "nc", "dot11-insts") +TARGET_BUILTIN(__builtin_amdgcn_dot4_f32_bf8_bf8, "fUiUif", "nc", "dot11-insts") //===----------------------------------------------------------------------===// // GFX10+ only builtins. diff --git a/clang/test/CodeGenOpenCL/amdgpu-features.cl b/clang/test/CodeGenOpenCL/amdgpu-features.cl index 9c8ca0bb96f6..7387f9a22f0d 100644 --- a/clang/test/CodeGenOpenCL/amdgpu-features.cl +++ b/clang/test/CodeGenOpenCL/amdgpu-features.cl @@ -100,8 +100,8 @@ // GFX1103: "target-features"="+16-bit-insts,+atomic-fadd-rtn-insts,+ci-insts,+dl-insts,+dot10-insts,+dot5-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32" // GFX1150: "target-features"="+16-bit-insts,+atomic-fadd-rtn-insts,+ci-insts,+dl-insts,+dot10-insts,+dot5-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32" // GFX1151: "target-features"="+16-bit-insts,+atomic-fadd-rtn-insts,+ci-insts,+dl-insts,+dot10-insts,+dot5-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32" -// GFX1200: "target-features"="+16-bit-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-global-pk-add-bf16-inst,+ci-insts,+dl-insts,+dot10-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+fp8-conversion-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32" -// GFX1201: "target-features"="+16-bit-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-global-pk-add-bf16-inst,+ci-insts,+dl-insts,+dot10-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+fp8-conversion-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32" +// GFX1200: "target-features"="+16-bit-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-global-pk-add-bf16-inst,+ci-insts,+dl-insts,+dot10-insts,+dot11-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+fp8-conversion-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32" +// GFX1201: "target-features"="+16-bit-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-global-pk-add-bf16-inst,+ci-insts,+dl-insts,+dot10-insts,+dot11-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+fp8-conversion-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32" // GFX1103-W64: "target-features"="+16-bit-insts,+atomic-fadd-rtn-insts,+ci-insts,+dl-insts,+dot10-insts,+dot5-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize64" diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-dl-insts-err.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-dl-insts-err.cl index f5317683d0ff..ce36a807a6c0 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn-dl-insts-err.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-dl-insts-err.cl @@ -50,8 +50,8 @@ kernel void builtins_amdgcn_dl_insts_err( iOut[3] = __builtin_amdgcn_sudot8(false, A, true, B, C, false); // expected-error {{'__builtin_amdgcn_sudot8' needs target feature dot8-insts}} iOut[4] = __builtin_amdgcn_sudot8(true, A, false, B, C, true); // expected-error {{'__builtin_amdgcn_sudot8' needs target feature dot8-insts}} - fOut[5] = __builtin_amdgcn_dot4_f32_fp8_bf8(uiA, uiB, fC); // expected-error {{'__builtin_amdgcn_dot4_f32_fp8_bf8' needs target feature gfx12-insts}} - fOut[6] = __builtin_amdgcn_dot4_f32_bf8_fp8(uiA, uiB, fC); // expected-error {{'__builtin_amdgcn_dot4_f32_bf8_fp8' needs target feature gfx12-insts}} - fOut[7] = __builtin_amdgcn_dot4_f32_fp8_fp8(uiA, uiB, fC); // expected-error {{'__builtin_amdgcn_dot4_f32_fp8_fp8' needs target feature gfx12-insts}} - fOut[8] = __builtin_amdgcn_dot4_f32_bf8_bf8(uiA, uiB, fC); // expected-error {{'__builtin_amdgcn_dot4_f32_bf8_bf8' needs target feature gfx12-insts}} + fOut[5] = __builtin_amdgcn_dot4_f32_fp8_bf8(uiA, uiB, fC); // expected-error {{'__builtin_amdgcn_dot4_f32_fp8_bf8' needs target feature dot11-insts}} + fOut[6] = __builtin_amdgcn_dot4_f32_bf8_fp8(uiA, uiB, fC); // expected-error {{'__builtin_amdgcn_dot4_f32_bf8_fp8' needs target feature dot11-insts}} + fOut[7] = __builtin_amdgcn_dot4_f32_fp8_fp8(uiA, uiB, fC); // expected-error {{'__builtin_amdgcn_dot4_f32_fp8_fp8' needs target feature dot11-insts}} + fOut[8] = __builtin_amdgcn_dot4_f32_bf8_bf8(uiA, uiB, fC); // expected-error {{'__builtin_amdgcn_dot4_f32_bf8_bf8' needs target feature dot11-insts}} } diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td index 8906c46f279e..394235476769 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.td +++ b/llvm/lib/Target/AMDGPU/AMDGPU.td @@ -650,6 +650,12 @@ def FeatureDot10Insts : SubtargetFeature<"dot10-insts", "Has v_dot2_f32_f16 instruction" >; +def FeatureDot11Insts : SubtargetFeature<"dot11-insts", + "HasDot11Insts", + "true", + "Has v_dot4_f32_fp8_fp8, v_dot4_f32_fp8_bf8, v_dot4_f32_bf8_fp8, v_dot4_f32_bf8_bf8 instructions" +>; + def FeatureMAIInsts : SubtargetFeature<"mai-insts", "HasMAIInsts", "true", @@ -1521,6 +1527,7 @@ def FeatureISAVersion12 : FeatureSet< FeatureDot8Insts, FeatureDot9Insts, FeatureDot10Insts, + FeatureDot11Insts, FeatureNSAEncoding, FeaturePartialNSAEncoding, FeatureWavefrontSize32, @@ -2029,6 +2036,9 @@ def HasDot9Insts : Predicate<"Subtarget->hasDot9Insts()">, def HasDot10Insts : Predicate<"Subtarget->hasDot10Insts()">, AssemblerPredicate<(all_of FeatureDot10Insts)>; +def HasDot11Insts : Predicate<"Subtarget->hasDot11Insts()">, + AssemblerPredicate<(all_of FeatureDot11Insts)>; + def HasGetWaveIdInst : Predicate<"Subtarget->hasGetWaveIdInst()">, AssemblerPredicate<(all_of FeatureGetWaveIdInst)>; diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h index 3283ac72aa4d..b6c01da7d98e 100644 --- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h +++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h @@ -153,6 +153,7 @@ protected: bool HasDot8Insts = false; bool HasDot9Insts = false; bool HasDot10Insts = false; + bool HasDot11Insts = false; bool HasMAIInsts = false; bool HasFP8Insts = false; bool HasFP8ConversionInsts = false; @@ -793,6 +794,10 @@ public: return HasDot10Insts; } + bool hasDot11Insts() const { + return HasDot11Insts; + } + bool hasMAIInsts() const { return HasMAIInsts; } diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td index ac3c8f95306b..e1131bbb78d3 100644 --- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td +++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td @@ -480,10 +480,12 @@ multiclass VOP3PDOTF8Inst { i32:$src2_modifiers, f32:$src2)>; } +let OtherPredicates = [HasDot11Insts] in { defm V_DOT4_F32_FP8_BF8 : VOP3PDOTF8Inst<"v_dot4_f32_fp8_bf8", int_amdgcn_dot4_f32_fp8_bf8>; defm V_DOT4_F32_BF8_FP8 : VOP3PDOTF8Inst<"v_dot4_f32_bf8_fp8", int_amdgcn_dot4_f32_bf8_fp8>; defm V_DOT4_F32_FP8_FP8 : VOP3PDOTF8Inst<"v_dot4_f32_fp8_fp8", int_amdgcn_dot4_f32_fp8_fp8>; defm V_DOT4_F32_BF8_BF8 : VOP3PDOTF8Inst<"v_dot4_f32_bf8_bf8", int_amdgcn_dot4_f32_bf8_bf8>; +} def : UDot2Pat; def : SDot2Pat; diff --git a/llvm/lib/TargetParser/TargetParser.cpp b/llvm/lib/TargetParser/TargetParser.cpp index a31027c59ee9..0d784a79e5ba 100644 --- a/llvm/lib/TargetParser/TargetParser.cpp +++ b/llvm/lib/TargetParser/TargetParser.cpp @@ -318,6 +318,7 @@ void AMDGPU::fillAMDGPUFeatureMap(StringRef GPU, const Triple &T, Features["dot8-insts"] = true; Features["dot9-insts"] = true; Features["dot10-insts"] = true; + Features["dot11-insts"] = true; Features["dl-insts"] = true; Features["atomic-ds-pk-add-16-insts"] = true; Features["atomic-flat-pk-add-16-insts"] = true; -- GitLab From ee1bcf74ea68d76b01ae2b58050f8f33bb684cac Mon Sep 17 00:00:00 2001 From: Changpeng Fang Date: Wed, 6 Mar 2024 14:59:28 -0800 Subject: [PATCH 370/929] AMDGPI: Rename HasExpOrExportInsts to HasExportInsts. NFC (#84252) --- llvm/lib/Target/AMDGPU/AMDGPU.td | 2 +- llvm/lib/Target/AMDGPU/EXPInstructions.td | 6 +++--- llvm/lib/Target/AMDGPU/GCNSubtarget.h | 2 +- llvm/lib/Target/AMDGPU/SOPInstructions.td | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td index 394235476769..5d905f5cadc0 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.td +++ b/llvm/lib/Target/AMDGPU/AMDGPU.td @@ -1881,7 +1881,7 @@ def D16PreservesUnusedBits : def LDSRequiresM0Init : Predicate<"Subtarget->ldsRequiresM0Init()">; def NotLDSRequiresM0Init : Predicate<"!Subtarget->ldsRequiresM0Init()">; -def HasExpOrExportInsts : Predicate<"Subtarget->hasExpOrExportInsts()">, +def HasExportInsts : Predicate<"Subtarget->hasExportInsts()">, AssemblerPredicate<(all_of (not FeatureGFX90AInsts))>; def HasInterpInsts : Predicate<"Subtarget->hasInterpInsts()">, diff --git a/llvm/lib/Target/AMDGPU/EXPInstructions.td b/llvm/lib/Target/AMDGPU/EXPInstructions.td index 5e3f555e4411..cce8734b72d4 100644 --- a/llvm/lib/Target/AMDGPU/EXPInstructions.td +++ b/llvm/lib/Target/AMDGPU/EXPInstructions.td @@ -58,12 +58,12 @@ class EXP_Real_Row; def EXP_DONE : EXP_Pseudo<0, 1>; def EXP_ROW : EXP_Pseudo<1, 0>; def EXP_ROW_DONE : EXP_Pseudo<1, 1>; -} // let SubtargetPredicate = HasExpOrExportInsts +} // let SubtargetPredicate = HasExportInsts //===----------------------------------------------------------------------===// // SI, VI, GFX10. @@ -117,7 +117,7 @@ multiclass EXP_Real_gfx11 { multiclass VEXPORT_Real_gfx12 { defvar ps = !cast(NAME); def _gfx12 : EXP_Real_Row, - EXPe_Row, MnemonicAlias<"exp", "export">, Requires<[isGFX12Plus, HasExpOrExportInsts]> { + EXPe_Row, MnemonicAlias<"exp", "export">, Requires<[isGFX12Plus, HasExportInsts]> { let AssemblerPredicate = isGFX12Only; let DecoderNamespace = "GFX12"; let row = ps.row; diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h index b6c01da7d98e..afe3af07dc37 100644 --- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h +++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h @@ -648,7 +648,7 @@ public: // BUFFER/FLAT/GLOBAL_ATOMIC_ADD/MIN/MAX_F64 bool hasBufferFlatGlobalAtomicsF64() const { return hasGFX90AInsts(); } - bool hasExpOrExportInsts() const { + bool hasExportInsts() const { return !hasGFX940Insts(); } diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td index e14f7f95d467..bccdf4b5ca63 100644 --- a/llvm/lib/Target/AMDGPU/SOPInstructions.td +++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td @@ -1705,7 +1705,7 @@ let SubtargetPredicate = isGFX10Plus in { } // End SubtargetPredicate = isGFX10Plus let SubtargetPredicate = isGFX11Plus in { -let OtherPredicates = [HasExpOrExportInsts] in +let OtherPredicates = [HasExportInsts] in def S_WAIT_EVENT : SOPP_Pseudo<"s_wait_event", (ins s16imm:$simm16), "$simm16"> { let hasSideEffects = 1; @@ -1738,7 +1738,7 @@ let OtherPredicates = [HasImageInsts] in { SOPP_Pseudo<"s_wait_bvhcnt", (ins s16imm:$simm16), "$simm16", [(int_amdgcn_s_wait_bvhcnt timm:$simm16)]>; } // End OtherPredicates = [HasImageInsts]. -let OtherPredicates = [HasExpOrExportInsts] in +let OtherPredicates = [HasExportInsts] in def S_WAIT_EXPCNT : SOPP_Pseudo<"s_wait_expcnt", (ins s16imm:$simm16), "$simm16", [(int_amdgcn_s_wait_expcnt timm:$simm16)]>; -- GitLab From ad33fe12812a07c80d5b988e99bea11d5f73414b Mon Sep 17 00:00:00 2001 From: lntue <35648136+lntue@users.noreply.github.com> Date: Wed, 6 Mar 2024 18:35:44 -0500 Subject: [PATCH 371/929] [libc][stdfix] Add integer square root with fixed point output functions. (#83959) Fix https://github.com/llvm/llvm-project/issues/83924. --- libc/config/baremetal/api.td | 1 + libc/config/baremetal/arm/entrypoints.txt | 2 + libc/config/baremetal/riscv/entrypoints.txt | 2 + libc/config/linux/api.td | 2 +- libc/config/linux/x86_64/entrypoints.txt | 2 + libc/docs/math/stdfix.rst | 21 +- libc/spec/llvm_libc_ext.td | 24 +- libc/spec/stdc_ext.td | 8 - libc/src/__support/fixed_point/CMakeLists.txt | 1 + libc/src/__support/fixed_point/fx_rep.h | 24 ++ libc/src/__support/fixed_point/sqrt.h | 207 ++++++++++++++---- libc/src/stdfix/CMakeLists.txt | 27 ++- libc/src/stdfix/uhksqrtus.cpp | 23 ++ libc/src/stdfix/uhksqrtus.h | 20 ++ libc/src/stdfix/uksqrtui.cpp | 23 ++ libc/src/stdfix/uksqrtui.h | 20 ++ libc/test/src/stdfix/CMakeLists.txt | 41 +++- libc/test/src/stdfix/ISqrtTest.h | 63 ++++++ libc/test/src/stdfix/uhksqrtus_test.cpp | 20 ++ libc/test/src/stdfix/uksqrtui_test.cpp | 20 ++ 20 files changed, 491 insertions(+), 60 deletions(-) create mode 100644 libc/src/stdfix/uhksqrtus.cpp create mode 100644 libc/src/stdfix/uhksqrtus.h create mode 100644 libc/src/stdfix/uksqrtui.cpp create mode 100644 libc/src/stdfix/uksqrtui.h create mode 100644 libc/test/src/stdfix/ISqrtTest.h create mode 100644 libc/test/src/stdfix/uhksqrtus_test.cpp create mode 100644 libc/test/src/stdfix/uksqrtui_test.cpp diff --git a/libc/config/baremetal/api.td b/libc/config/baremetal/api.td index a132d8308122..008eb45386f2 100644 --- a/libc/config/baremetal/api.td +++ b/libc/config/baremetal/api.td @@ -2,6 +2,7 @@ include "config/public_api.td" include "spec/stdc.td" include "spec/stdc_ext.td" +include "spec/llvm_libc_ext.td" def AssertMacro : MacroDef<"assert"> { let Defn = [{ diff --git a/libc/config/baremetal/arm/entrypoints.txt b/libc/config/baremetal/arm/entrypoints.txt index c9887b6e855a..99796ad5edf5 100644 --- a/libc/config/baremetal/arm/entrypoints.txt +++ b/libc/config/baremetal/arm/entrypoints.txt @@ -306,6 +306,8 @@ if(LIBC_COMPILER_HAS_FIXED_POINT) libc.src.stdfix.sqrtur # libc.src.stdfix.sqrtulk libc.src.stdfix.sqrtulr + libc.src.stdfix.uhksqrtus + libc.src.stdfix.uksqrtui ) endif() diff --git a/libc/config/baremetal/riscv/entrypoints.txt b/libc/config/baremetal/riscv/entrypoints.txt index c9887b6e855a..99796ad5edf5 100644 --- a/libc/config/baremetal/riscv/entrypoints.txt +++ b/libc/config/baremetal/riscv/entrypoints.txt @@ -306,6 +306,8 @@ if(LIBC_COMPILER_HAS_FIXED_POINT) libc.src.stdfix.sqrtur # libc.src.stdfix.sqrtulk libc.src.stdfix.sqrtulr + libc.src.stdfix.uhksqrtus + libc.src.stdfix.uksqrtui ) endif() diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td index 5a1d7642f1ae..526fd03f94f6 100644 --- a/libc/config/linux/api.td +++ b/libc/config/linux/api.td @@ -5,8 +5,8 @@ include "spec/posix.td" include "spec/linux.td" include "spec/gnu_ext.td" include "spec/bsd_ext.td" -include "spec/llvm_libc_ext.td" include "spec/stdc_ext.td" +include "spec/llvm_libc_ext.td" def AssertMacro : MacroDef<"assert"> { let Defn = [{ diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index a6c3041773df..705ec10960c4 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -495,6 +495,8 @@ if(LIBC_COMPILER_HAS_FIXED_POINT) libc.src.stdfix.sqrtur # libc.src.stdfix.sqrtulk libc.src.stdfix.sqrtulr + libc.src.stdfix.uhksqrtus + libc.src.stdfix.uksqrtui ) endif() diff --git a/libc/docs/math/stdfix.rst b/libc/docs/math/stdfix.rst index 79f499e61f12..5e39d5c01d1e 100644 --- a/libc/docs/math/stdfix.rst +++ b/libc/docs/math/stdfix.rst @@ -4,14 +4,19 @@ StdFix Functions .. include:: ../check.rst -Standards ---------- +Standards and Goals +------------------- - stdfix.h is specified in the `ISO/IEC TR 18037:2008 `_, C extensions to support embedded processors . - Its `specifications `_. +- Our goal is to implement a complete set of math functions for fixed point + types, most of them are currently not included in the ISO/IEC TR + 18037:2008 standard. Our math functions for fixed point types are modeled + after the C99/C23 math functions for floating point types. + --------------- Source location --------------- @@ -53,6 +58,8 @@ Predefined Macros Fixed-point Arithmetics ======================= +The following functions are included in the ISO/IEC TR 18037:2008 standard. + +---------------+------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+ | Function Name | _Fract (r) | _Accum (k) | | +------------------------------+----------------------------+------------------------------+------------------------------+----------------------------+------------------------------+ @@ -78,8 +85,6 @@ Fixed-point Arithmetics +---------------+----------------+-------------+---------------+------------+----------------+-------------+----------------+-------------+---------------+------------+----------------+-------------+ | round | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | |check| | +---------------+----------------+-------------+---------------+------------+----------------+-------------+----------------+-------------+---------------+------------+----------------+-------------+ -| sqrt | |check| | | |check| | | |check| | | |check| | | |check| | | | | -+---------------+----------------+-------------+---------------+------------+----------------+-------------+----------------+-------------+---------------+------------+----------------+-------------+ ================== ========= Type Generic Macro Available @@ -93,6 +98,9 @@ roundfx Higher math functions ===================== +The following math functions are modeled after C99/C23 math functions for +floating point types, but are not part of the ISO/IEC TR 18037:2008 spec. + +---------------+------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+ | Function Name | _Fract (r) | _Accum (k) | | +------------------------------+----------------------------+------------------------------+------------------------------+----------------------------+------------------------------+ @@ -108,6 +116,8 @@ Higher math functions +---------------+----------------+-------------+---------------+------------+----------------+-------------+----------------+-------------+---------------+------------+----------------+-------------+ | sin | | | | | | | | | | | | | +---------------+----------------+-------------+---------------+------------+----------------+-------------+----------------+-------------+---------------+------------+----------------+-------------+ +| sqrt | |check| | | |check| | | |check| | | |check| | | |check| | | | | ++---------------+----------------+-------------+---------------+------------+----------------+-------------+----------------+-------------+---------------+------------+----------------+-------------+ | tan | | | | | | | | | | | | | +---------------+----------------+-------------+---------------+------------+----------------+-------------+----------------+-------------+---------------+------------+----------------+-------------+ @@ -115,6 +125,9 @@ Higher math functions Conversion Functions ==================== +The following conversion functions are included in the ISO/IEC TR 18037:2008 +standard. + +---------------+------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+ | Function Name | _Fract (r) | _Accum (k) | | +------------------------------+----------------------------+------------------------------+------------------------------+----------------------------+------------------------------+ diff --git a/libc/spec/llvm_libc_ext.td b/libc/spec/llvm_libc_ext.td index 2fd0c5f78fb8..274284ed5705 100644 --- a/libc/spec/llvm_libc_ext.td +++ b/libc/spec/llvm_libc_ext.td @@ -51,9 +51,29 @@ def LLVMLibcExt : StandardSpec<"llvm_libc_ext"> { ] >; + HeaderSpec StdFix = HeaderSpec< + "stdfix.h", + [], // macros + [], // types + [], // enums + [ // functions + GuardedFunctionSpec<"sqrtuhr", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, + GuardedFunctionSpec<"sqrtur", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, + GuardedFunctionSpec<"sqrtulr", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, + + GuardedFunctionSpec<"sqrtuhk", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, + GuardedFunctionSpec<"sqrtuk", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, + GuardedFunctionSpec<"sqrtulk", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, + + GuardedFunctionSpec<"uhksqrtus", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, + GuardedFunctionSpec<"uksqrtui", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, + ] + >; + let Headers = [ - Strings, - Sched, Assert, + Sched, + Stdfix, + Strings, ]; } diff --git a/libc/spec/stdc_ext.td b/libc/spec/stdc_ext.td index be1e6d4ba2fc..6620142146c4 100644 --- a/libc/spec/stdc_ext.td +++ b/libc/spec/stdc_ext.td @@ -47,14 +47,6 @@ def StdcExt : StandardSpec<"stdc_ext"> { GuardedFunctionSpec<"rounduhk", RetValSpec, [ArgSpec, ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, GuardedFunctionSpec<"rounduk", RetValSpec, [ArgSpec, ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, GuardedFunctionSpec<"roundulk", RetValSpec, [ArgSpec, ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, - - GuardedFunctionSpec<"sqrtuhr", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, - GuardedFunctionSpec<"sqrtur", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, - GuardedFunctionSpec<"sqrtulr", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, - - GuardedFunctionSpec<"sqrtuhk", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, - GuardedFunctionSpec<"sqrtuk", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, - GuardedFunctionSpec<"sqrtulk", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, ] >; diff --git a/libc/src/__support/fixed_point/CMakeLists.txt b/libc/src/__support/fixed_point/CMakeLists.txt index 0ed118f24088..3b744081765e 100644 --- a/libc/src/__support/fixed_point/CMakeLists.txt +++ b/libc/src/__support/fixed_point/CMakeLists.txt @@ -32,5 +32,6 @@ add_header_library( libc.src.__support.macros.attributes libc.src.__support.macros.optimization libc.src.__support.CPP.bit + libc.src.__support.CPP.limits libc.src.__support.CPP.type_traits ) diff --git a/libc/src/__support/fixed_point/fx_rep.h b/libc/src/__support/fixed_point/fx_rep.h index f8593a93684c..042cd2b20714 100644 --- a/libc/src/__support/fixed_point/fx_rep.h +++ b/libc/src/__support/fixed_point/fx_rep.h @@ -48,6 +48,8 @@ template <> struct FXRep { LIBC_INLINE static constexpr Type MAX() { return SFRACT_MIN; } LIBC_INLINE static constexpr Type ZERO() { return 0.0HR; } LIBC_INLINE static constexpr Type EPS() { return SFRACT_EPSILON; } + LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5HR; } + LIBC_INLINE static constexpr Type ONE_FOURTH() { return 0.25HR; } using StorageType = typename internal::Storage::Type; using CompType = cpp::make_signed_t; @@ -66,6 +68,8 @@ template <> struct FXRep { LIBC_INLINE static constexpr Type MAX() { return USFRACT_MIN; } LIBC_INLINE static constexpr Type ZERO() { return 0.0UHR; } LIBC_INLINE static constexpr Type EPS() { return USFRACT_EPSILON; } + LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5UHR; } + LIBC_INLINE static constexpr Type ONE_FOURTH() { return 0.25UHR; } using StorageType = typename internal::Storage::Type; using CompType = cpp::make_unsigned_t; @@ -84,6 +88,8 @@ template <> struct FXRep { LIBC_INLINE static constexpr Type MAX() { return FRACT_MIN; } LIBC_INLINE static constexpr Type ZERO() { return 0.0R; } LIBC_INLINE static constexpr Type EPS() { return FRACT_EPSILON; } + LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5R; } + LIBC_INLINE static constexpr Type ONE_FOURTH() { return 0.25R; } using StorageType = typename internal::Storage::Type; using CompType = cpp::make_signed_t; @@ -102,6 +108,8 @@ template <> struct FXRep { LIBC_INLINE static constexpr Type MAX() { return UFRACT_MIN; } LIBC_INLINE static constexpr Type ZERO() { return 0.0UR; } LIBC_INLINE static constexpr Type EPS() { return UFRACT_EPSILON; } + LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5UR; } + LIBC_INLINE static constexpr Type ONE_FOURTH() { return 0.25UR; } using StorageType = typename internal::Storage::Type; using CompType = cpp::make_unsigned_t; @@ -120,6 +128,8 @@ template <> struct FXRep { LIBC_INLINE static constexpr Type MAX() { return LFRACT_MIN; } LIBC_INLINE static constexpr Type ZERO() { return 0.0LR; } LIBC_INLINE static constexpr Type EPS() { return LFRACT_EPSILON; } + LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5LR; } + LIBC_INLINE static constexpr Type ONE_FOURTH() { return 0.25LR; } using StorageType = typename internal::Storage::Type; using CompType = cpp::make_signed_t; @@ -138,6 +148,8 @@ template <> struct FXRep { LIBC_INLINE static constexpr Type MAX() { return ULFRACT_MIN; } LIBC_INLINE static constexpr Type ZERO() { return 0.0ULR; } LIBC_INLINE static constexpr Type EPS() { return ULFRACT_EPSILON; } + LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5ULR; } + LIBC_INLINE static constexpr Type ONE_FOURTH() { return 0.25ULR; } using StorageType = typename internal::Storage::Type; using CompType = cpp::make_unsigned_t; @@ -156,6 +168,8 @@ template <> struct FXRep { LIBC_INLINE static constexpr Type MAX() { return SACCUM_MIN; } LIBC_INLINE static constexpr Type ZERO() { return 0.0HK; } LIBC_INLINE static constexpr Type EPS() { return SACCUM_EPSILON; } + LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5HK; } + LIBC_INLINE static constexpr Type ONE_FOURTH() { return 0.25HK; } using StorageType = typename internal::Storage::Type; using CompType = cpp::make_signed_t; @@ -174,6 +188,8 @@ template <> struct FXRep { LIBC_INLINE static constexpr Type MAX() { return USACCUM_MIN; } LIBC_INLINE static constexpr Type ZERO() { return 0.0UHK; } LIBC_INLINE static constexpr Type EPS() { return USACCUM_EPSILON; } + LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5UHK; } + LIBC_INLINE static constexpr Type ONE_FOURTH() { return 0.25UHK; } using StorageType = typename internal::Storage::Type; using CompType = cpp::make_unsigned_t; @@ -192,6 +208,8 @@ template <> struct FXRep { LIBC_INLINE static constexpr Type MAX() { return ACCUM_MIN; } LIBC_INLINE static constexpr Type ZERO() { return 0.0K; } LIBC_INLINE static constexpr Type EPS() { return ACCUM_EPSILON; } + LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5K; } + LIBC_INLINE static constexpr Type ONE_FOURTH() { return 0.25K; } using StorageType = typename internal::Storage::Type; using CompType = cpp::make_signed_t; @@ -210,6 +228,8 @@ template <> struct FXRep { LIBC_INLINE static constexpr Type MAX() { return UACCUM_MIN; } LIBC_INLINE static constexpr Type ZERO() { return 0.0UK; } LIBC_INLINE static constexpr Type EPS() { return UACCUM_EPSILON; } + LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5UK; } + LIBC_INLINE static constexpr Type ONE_FOURTH() { return 0.25UK; } using StorageType = typename internal::Storage::Type; using CompType = cpp::make_unsigned_t; @@ -228,6 +248,8 @@ template <> struct FXRep { LIBC_INLINE static constexpr Type MAX() { return LACCUM_MIN; } LIBC_INLINE static constexpr Type ZERO() { return 0.0LK; } LIBC_INLINE static constexpr Type EPS() { return LACCUM_EPSILON; } + LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5LK; } + LIBC_INLINE static constexpr Type ONE_FOURTH() { return 0.25LK; } using StorageType = typename internal::Storage::Type; using CompType = cpp::make_signed_t; @@ -246,6 +268,8 @@ template <> struct FXRep { LIBC_INLINE static constexpr Type MAX() { return ULACCUM_MIN; } LIBC_INLINE static constexpr Type ZERO() { return 0.0ULK; } LIBC_INLINE static constexpr Type EPS() { return ULACCUM_EPSILON; } + LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5ULK; } + LIBC_INLINE static constexpr Type ONE_FOURTH() { return 0.25ULK; } using StorageType = typename internal::Storage::Type; using CompType = cpp::make_unsigned_t; diff --git a/libc/src/__support/fixed_point/sqrt.h b/libc/src/__support/fixed_point/sqrt.h index d8df294b18a1..4ec016ceab00 100644 --- a/libc/src/__support/fixed_point/sqrt.h +++ b/libc/src/__support/fixed_point/sqrt.h @@ -11,6 +11,7 @@ #include "include/llvm-libc-macros/stdfix-macros.h" #include "src/__support/CPP/bit.h" +#include "src/__support/CPP/limits.h" // CHAR_BIT #include "src/__support/CPP/type_traits.h" #include "src/__support/macros/attributes.h" // LIBC_INLINE #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY @@ -28,16 +29,73 @@ template struct SqrtConfig; template <> struct SqrtConfig { using Type = unsigned short fract; static constexpr int EXTRA_STEPS = 0; + + // Linear approximation for the initial values, with errors bounded by: + // max(1.5 * 2^-11, eps) + // Generated with Sollya: + // > for i from 4 to 15 do { + // P = fpminimax(sqrt(x), 1, [|8, 8|], [i * 2^-4, (i + 1)*2^-4], + // fixed, absolute); + // print("{", coeff(P, 1), "uhr,", coeff(P, 0), "uhr},"); + // }; + static constexpr Type FIRST_APPROX[12][2] = { + {0x1.e8p-1uhr, 0x1.0cp-2uhr}, {0x1.bap-1uhr, 0x1.28p-2uhr}, + {0x1.94p-1uhr, 0x1.44p-2uhr}, {0x1.74p-1uhr, 0x1.6p-2uhr}, + {0x1.6p-1uhr, 0x1.74p-2uhr}, {0x1.4ep-1uhr, 0x1.88p-2uhr}, + {0x1.3ep-1uhr, 0x1.9cp-2uhr}, {0x1.32p-1uhr, 0x1.acp-2uhr}, + {0x1.22p-1uhr, 0x1.c4p-2uhr}, {0x1.18p-1uhr, 0x1.d4p-2uhr}, + {0x1.08p-1uhr, 0x1.fp-2uhr}, {0x1.04p-1uhr, 0x1.f8p-2uhr}, + }; }; template <> struct SqrtConfig { using Type = unsigned fract; static constexpr int EXTRA_STEPS = 1; + + // Linear approximation for the initial values, with errors bounded by: + // max(1.5 * 2^-11, eps) + // Generated with Sollya: + // > for i from 4 to 15 do { + // P = fpminimax(sqrt(x), 1, [|16, 16|], [i * 2^-4, (i + 1)*2^-4], + // fixed, absolute); + // print("{", coeff(P, 1), "ur,", coeff(P, 0), "ur},"); + // }; + static constexpr Type FIRST_APPROX[12][2] = { + {0x1.e378p-1ur, 0x1.0ebp-2ur}, {0x1.b512p-1ur, 0x1.2b94p-2ur}, + {0x1.91fp-1ur, 0x1.45dcp-2ur}, {0x1.7622p-1ur, 0x1.5e24p-2ur}, + {0x1.5f5ap-1ur, 0x1.74e4p-2ur}, {0x1.4c58p-1ur, 0x1.8a4p-2ur}, + {0x1.3c1ep-1ur, 0x1.9e84p-2ur}, {0x1.2e0cp-1ur, 0x1.b1d8p-2ur}, + {0x1.21aap-1ur, 0x1.c468p-2ur}, {0x1.16bap-1ur, 0x1.d62cp-2ur}, + {0x1.0cfp-1ur, 0x1.e74cp-2ur}, {0x1.0418p-1ur, 0x1.f7ep-2ur}, + }; }; template <> struct SqrtConfig { using Type = unsigned long fract; static constexpr int EXTRA_STEPS = 2; + + // Linear approximation for the initial values, with errors bounded by: + // max(1.5 * 2^-11, eps) + // Generated with Sollya: + // > for i from 4 to 15 do { + // P = fpminimax(sqrt(x), 1, [|32, 32|], [i * 2^-4, (i + 1)*2^-4], + // fixed, absolute); + // print("{", coeff(P, 1), "ulr,", coeff(P, 0), "ulr},"); + // }; + static constexpr Type FIRST_APPROX[12][2] = { + {0x1.e3779b98p-1ulr, 0x1.0eaff788p-2ulr}, + {0x1.b5167872p-1ulr, 0x1.2b908ad4p-2ulr}, + {0x1.91f195cap-1ulr, 0x1.45da800cp-2ulr}, + {0x1.761ebcb4p-1ulr, 0x1.5e27004cp-2ulr}, + {0x1.5f619986p-1ulr, 0x1.74db933cp-2ulr}, + {0x1.4c583adep-1ulr, 0x1.8a3fbfccp-2ulr}, + {0x1.3c1a591cp-1ulr, 0x1.9e88373cp-2ulr}, + {0x1.2e08545ap-1ulr, 0x1.b1dd2534p-2ulr}, + {0x1.21b05c0ap-1ulr, 0x1.c45e023p-2ulr}, + {0x1.16becd02p-1ulr, 0x1.d624031p-2ulr}, + {0x1.0cf49fep-1ulr, 0x1.e743b844p-2ulr}, + {0x1.04214e9cp-1ulr, 0x1.f7ce2c3cp-2ulr}, + }; }; template <> @@ -46,46 +104,38 @@ struct SqrtConfig : SqrtConfig {}; template <> struct SqrtConfig : SqrtConfig {}; -// TODO: unsigned long accum type is 64-bit, and will need 64-bit fract type. -// Probably we will use DyadicFloat<64> for intermediate computations instead. - -// Linear approximation for the initial values, with errors bounded by: -// max(1.5 * 2^-11, eps) -// Generated with Sollya: -// > for i from 4 to 15 do { -// P = fpminimax(sqrt(x), 1, [|8, 8|], [i * 2^-4, (i + 1)*2^-4], -// fixed, absolute); -// print("{", coeff(P, 1), "uhr,", coeff(P, 0), "uhr},"); -// }; -static constexpr unsigned short fract SQRT_FIRST_APPROX[12][2] = { - {0x1.e8p-1uhr, 0x1.0cp-2uhr}, {0x1.bap-1uhr, 0x1.28p-2uhr}, - {0x1.94p-1uhr, 0x1.44p-2uhr}, {0x1.74p-1uhr, 0x1.6p-2uhr}, - {0x1.6p-1uhr, 0x1.74p-2uhr}, {0x1.4ep-1uhr, 0x1.88p-2uhr}, - {0x1.3ep-1uhr, 0x1.9cp-2uhr}, {0x1.32p-1uhr, 0x1.acp-2uhr}, - {0x1.22p-1uhr, 0x1.c4p-2uhr}, {0x1.18p-1uhr, 0x1.d4p-2uhr}, - {0x1.08p-1uhr, 0x1.fp-2uhr}, {0x1.04p-1uhr, 0x1.f8p-2uhr}, +// Integer square root +template <> struct SqrtConfig { + using OutType = unsigned short accum; + using FracType = unsigned fract; + // For fast-but-less-accurate version + using FastFracType = unsigned short fract; + using HalfType = unsigned char; }; -} // namespace internal +template <> struct SqrtConfig { + using OutType = unsigned accum; + using FracType = unsigned long fract; + // For fast-but-less-accurate version + using FastFracType = unsigned fract; + using HalfType = unsigned short; +}; -template -LIBC_INLINE constexpr cpp::enable_if_t, T> sqrt(T x) { - using BitType = typename FXRep::StorageType; - BitType x_bit = cpp::bit_cast(x); +// TODO: unsigned long accum type is 64-bit, and will need 64-bit fract type. +// Probably we will use DyadicFloat<64> for intermediate computations instead. - if (LIBC_UNLIKELY(x_bit == 0)) - return FXRep::ZERO(); +} // namespace internal - int leading_zeros = cpp::countl_zero(x_bit); - constexpr int STORAGE_LENGTH = sizeof(BitType) * CHAR_BIT; - constexpr int EXP_ADJUSTMENT = STORAGE_LENGTH - FXRep::FRACTION_LEN - 1; - // x_exp is the real exponent of the leading bit of x. - int x_exp = EXP_ADJUSTMENT - leading_zeros; - int shift = EXP_ADJUSTMENT - 1 - (x_exp & (~1)); - // Normalize. - x_bit <<= shift; - using FracType = typename internal::SqrtConfig::Type; - FracType x_frac = cpp::bit_cast(x_bit); +// Core computation for sqrt with normalized inputs (0.25 <= x < 1). +template +LIBC_INLINE constexpr typename Config::Type +sqrt_core(typename Config::Type x_frac) { + using FracType = typename Config::Type; + using FXRep = FXRep; + using StorageType = typename FXRep::StorageType; + // Exact case: + if (x_frac == FXRep::ONE_FOURTH()) + return FXRep::ONE_HALF(); // Use use Newton method to approximate sqrt(a): // x_{n + 1} = 1/2 (x_n + a / x_n) @@ -96,9 +146,10 @@ LIBC_INLINE constexpr cpp::enable_if_t, T> sqrt(T x) { // are between 0b0100 and 0b1111. Hence the lookup table only needs 12 // entries, and we can get the index by subtracting the leading 4 bits of // x_frac by 4 = 0b0100. - int index = (x_bit >> (STORAGE_LENGTH - 4)) - 4; - FracType a = static_cast(internal::SQRT_FIRST_APPROX[index][0]); - FracType b = static_cast(internal::SQRT_FIRST_APPROX[index][1]); + StorageType x_bit = cpp::bit_cast(x_frac); + int index = (static_cast(x_bit >> (FXRep::TOTAL_LEN - 4))) - 4; + FracType a = Config::FIRST_APPROX[index][0]; + FracType b = Config::FIRST_APPROX[index][1]; // Initial approximation step. // Estimated error bounds: | r - sqrt(x_frac) | < max(1.5 * 2^-11, eps). @@ -112,9 +163,34 @@ LIBC_INLINE constexpr cpp::enable_if_t, T> sqrt(T x) { // Blanchard, J. D. and Chamberland, M., "Newton's Method Without Division", // The American Mathematical Monthly (2023). // https://chamberland.math.grinnell.edu/papers/newton.pdf - for (int i = 0; i < internal::SqrtConfig::EXTRA_STEPS; ++i) + for (int i = 0; i < Config::EXTRA_STEPS; ++i) r = (r >> 1) + (x_frac >> 1) / r; + return r; +} + +template +LIBC_INLINE constexpr cpp::enable_if_t, T> sqrt(T x) { + using BitType = typename FXRep::StorageType; + BitType x_bit = cpp::bit_cast(x); + + if (LIBC_UNLIKELY(x_bit == 0)) + return FXRep::ZERO(); + + int leading_zeros = cpp::countl_zero(x_bit); + constexpr int STORAGE_LENGTH = sizeof(BitType) * CHAR_BIT; + constexpr int EXP_ADJUSTMENT = STORAGE_LENGTH - FXRep::FRACTION_LEN - 1; + // x_exp is the real exponent of the leading bit of x. + int x_exp = EXP_ADJUSTMENT - leading_zeros; + int shift = EXP_ADJUSTMENT - 1 - (x_exp & (~1)); + // Normalize. + x_bit <<= shift; + using FracType = typename internal::SqrtConfig::Type; + FracType x_frac = cpp::bit_cast(x_bit); + + // Compute sqrt(x_frac) using Newton-method. + FracType r = sqrt_core>(x_frac); + // Re-scaling r >>= EXP_ADJUSTMENT - (x_exp >> 1); @@ -122,6 +198,59 @@ LIBC_INLINE constexpr cpp::enable_if_t, T> sqrt(T x) { return cpp::bit_cast(r); } +// Integer square root - Accurate version: +// Absolute errors < 2^(-fraction length). +template +LIBC_INLINE constexpr typename internal::SqrtConfig::OutType isqrt(T x) { + using OutType = typename internal::SqrtConfig::OutType; + using FracType = typename internal::SqrtConfig::FracType; + + if (x == 0) + return FXRep::ZERO(); + + // Normalize the leading bits to the first two bits. + // Shift and then Bit cast x to x_frac gives us: + // x = 2^(FRACTION_LEN + 1 - shift) * x_frac; + int leading_zeros = cpp::countl_zero(x); + int shift = ((leading_zeros >> 1) << 1); + x <<= shift; + // Convert to frac type and compute square root. + FracType x_frac = cpp::bit_cast(x); + FracType r = sqrt_core>(x_frac); + // To rescale back to the OutType (Accum) + r >>= (shift >> 1); + + return cpp::bit_cast(r); +} + +// Integer square root - Fast but less accurate version: +// Relative errors < 2^(-fraction length). +template +LIBC_INLINE constexpr typename internal::SqrtConfig::OutType +isqrt_fast(T x) { + using OutType = typename internal::SqrtConfig::OutType; + using FracType = typename internal::SqrtConfig::FastFracType; + using StorageType = typename FXRep::StorageType; + + if (x == 0) + return FXRep::ZERO(); + + // Normalize the leading bits to the first two bits. + // Shift and then Bit cast x to x_frac gives us: + // x = 2^(FRACTION_LEN + 1 - shift) * x_frac; + int leading_zeros = cpp::countl_zero(x); + int shift = (leading_zeros & (~1)); + x <<= shift; + // Convert to frac type and compute square root. + FracType x_frac = cpp::bit_cast( + static_cast(x >> FXRep::FRACTION_LEN)); + OutType r = + static_cast(sqrt_core>(x_frac)); + // To rescale back to the OutType (Accum) + r <<= (FXRep::INTEGRAL_LEN - (shift >> 1)); + return cpp::bit_cast(r); +} + } // namespace LIBC_NAMESPACE::fixed_point #endif // LIBC_COMPILER_HAS_FIXED_POINT diff --git a/libc/src/stdfix/CMakeLists.txt b/libc/src/stdfix/CMakeLists.txt index cb2134fe33cf..3a1cb66b7abc 100644 --- a/libc/src/stdfix/CMakeLists.txt +++ b/libc/src/stdfix/CMakeLists.txt @@ -11,7 +11,6 @@ foreach(suffix IN ITEMS hr r lr hk k lk) abs${suffix}.cpp COMPILE_OPTIONS -O3 - -ffixed-point DEPENDS libc.src.__support.fixed_point.fx_bits ) @@ -26,7 +25,6 @@ foreach(suffix IN ITEMS uhr ur ulr uhk uk) sqrt${suffix}.cpp COMPILE_OPTIONS -O3 - -ffixed-point DEPENDS libc.src.__support.fixed_point.sqrt ) @@ -41,8 +39,31 @@ foreach(suffix IN ITEMS hr r lr hk k lk uhr ur ulr uhk uk ulk) round${suffix}.cpp COMPILE_OPTIONS -O3 - -ffixed-point DEPENDS libc.src.__support.fixed_point.fx_bits ) endforeach() + +add_entrypoint_object( + uhksqrtus + HDRS + uhksqrtus.h + SRCS + uhksqrtus.cpp + COMPILE_OPTIONS + -O3 + DEPENDS + libc.src.__support.fixed_point.sqrt +) + +add_entrypoint_object( + uksqrtui + HDRS + uksqrtui.h + SRCS + uksqrtui.cpp + COMPILE_OPTIONS + -O3 + DEPENDS + libc.src.__support.fixed_point.sqrt +) diff --git a/libc/src/stdfix/uhksqrtus.cpp b/libc/src/stdfix/uhksqrtus.cpp new file mode 100644 index 000000000000..335750ae902b --- /dev/null +++ b/libc/src/stdfix/uhksqrtus.cpp @@ -0,0 +1,23 @@ +//===-- Implementation of uhksqrtus function ------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "uhksqrtus.h" +#include "src/__support/common.h" +#include "src/__support/fixed_point/sqrt.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(unsigned short accum, uhksqrtus, (unsigned short x)) { +#ifdef LIBC_FAST_MATH + return fixed_point::isqrt_fast(x); +#else + return fixed_point::isqrt(x); +#endif +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdfix/uhksqrtus.h b/libc/src/stdfix/uhksqrtus.h new file mode 100644 index 000000000000..c24846a80030 --- /dev/null +++ b/libc/src/stdfix/uhksqrtus.h @@ -0,0 +1,20 @@ +//===-- Implementation header for uhksqrtus ---------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDFIX_UHKSQRTUS_H +#define LLVM_LIBC_SRC_STDFIX_UHKSQRTUS_H + +#include "include/llvm-libc-macros/stdfix-macros.h" + +namespace LIBC_NAMESPACE { + +unsigned short accum uhksqrtus(unsigned short x); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDFIX_UHKSQRTUS_H diff --git a/libc/src/stdfix/uksqrtui.cpp b/libc/src/stdfix/uksqrtui.cpp new file mode 100644 index 000000000000..ee1ae1335027 --- /dev/null +++ b/libc/src/stdfix/uksqrtui.cpp @@ -0,0 +1,23 @@ +//===-- Implementation of uksqrtui function -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "uksqrtui.h" +#include "src/__support/common.h" +#include "src/__support/fixed_point/sqrt.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(unsigned accum, uksqrtui, (unsigned int x)) { +#ifdef LIBC_FAST_MATH + return fixed_point::isqrt_fast(x); +#else + return fixed_point::isqrt(x); +#endif +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdfix/uksqrtui.h b/libc/src/stdfix/uksqrtui.h new file mode 100644 index 000000000000..cd4ff41ea100 --- /dev/null +++ b/libc/src/stdfix/uksqrtui.h @@ -0,0 +1,20 @@ +//===-- Implementation header for uksqrtui ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDFIX_UKSQRTUI_H +#define LLVM_LIBC_SRC_STDFIX_UKSQRTUI_H + +#include "include/llvm-libc-macros/stdfix-macros.h" + +namespace LIBC_NAMESPACE { + +unsigned accum uksqrtui(unsigned int x); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDFIX_UKSQRTUI_H diff --git a/libc/test/src/stdfix/CMakeLists.txt b/libc/test/src/stdfix/CMakeLists.txt index 4140b5b29f3b..d3e122884eb4 100644 --- a/libc/test/src/stdfix/CMakeLists.txt +++ b/libc/test/src/stdfix/CMakeLists.txt @@ -15,7 +15,6 @@ foreach(suffix IN ITEMS hr r lr hk k lk) abs${suffix}_test.cpp COMPILE_OPTIONS -O3 - -ffixed-point DEPENDS libc.src.stdfix.abs${suffix} libc.src.__support.fixed_point.fx_bits @@ -33,7 +32,6 @@ foreach(suffix IN ITEMS uhr ur ulr uhk uk) sqrt${suffix}_test.cpp COMPILE_OPTIONS -O3 - -ffixed-point DEPENDS libc.src.stdfix.sqrt${suffix} libc.src.__support.CPP.bit @@ -55,9 +53,46 @@ foreach(suffix IN ITEMS hr r lr hk k lk uhr ur ulr uhk uk ulk) round${suffix}_test.cpp COMPILE_OPTIONS -O3 - -ffixed-point DEPENDS libc.src.stdfix.round${suffix} libc.src.__support.fixed_point.fx_bits ) endforeach() + +add_libc_test( + uhksqrtus_test + SUITE + libc-stdfix-tests + HDRS + ISqrtTest.h + SRCS + uhksqrtus_test.cpp + COMPILE_OPTIONS + -O3 + DEPENDS + libc.src.stdfix.uhksqrtus + libc.src.__support.CPP.bit + libc.src.__support.fixed_point.fx_rep + libc.src.__support.fixed_point.sqrt + libc.src.__support.FPUtil.basic_operations + libc.src.__support.FPUtil.sqrt +) + +add_libc_test( + uksqrtui_test + SUITE + libc-stdfix-tests + HDRS + ISqrtTest.h + SRCS + uksqrtui_test.cpp + COMPILE_OPTIONS + -O3 + DEPENDS + libc.src.stdfix.uksqrtui + libc.src.__support.CPP.bit + libc.src.__support.fixed_point.fx_rep + libc.src.__support.fixed_point.sqrt + libc.src.__support.FPUtil.basic_operations + libc.src.__support.FPUtil.sqrt +) diff --git a/libc/test/src/stdfix/ISqrtTest.h b/libc/test/src/stdfix/ISqrtTest.h new file mode 100644 index 000000000000..405162b706a9 --- /dev/null +++ b/libc/test/src/stdfix/ISqrtTest.h @@ -0,0 +1,63 @@ +//===-- Utility class to test integer sqrt ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "test/UnitTest/FPMatcher.h" +#include "test/UnitTest/Test.h" + +#include "src/__support/CPP/bit.h" +#include "src/__support/FPUtil/BasicOperations.h" +#include "src/__support/FPUtil/sqrt.h" +#include "src/__support/fixed_point/fx_rep.h" +#include "src/__support/fixed_point/sqrt.h" + +template class ISqrtTest : public LIBC_NAMESPACE::testing::Test { + + using OutType = + typename LIBC_NAMESPACE::fixed_point::internal::SqrtConfig::OutType; + using FXRep = LIBC_NAMESPACE::fixed_point::FXRep; + static constexpr OutType zero = FXRep::ZERO(); + static constexpr OutType one = static_cast(1); + static constexpr OutType eps = FXRep::EPS(); + +public: + typedef OutType (*SqrtFunc)(T); + + void testSpecialNumbers(SqrtFunc func) { + EXPECT_EQ(zero, func(T(0))); + + EXPECT_EQ(one, func(T(1))); + EXPECT_EQ(static_cast(2.0), func(T(4))); + EXPECT_EQ(static_cast(4.0), func(T(16))); + EXPECT_EQ(static_cast(16.0), func(T(256))); + + constexpr int COUNT = 255; + constexpr double ERR = 3.0 * static_cast(eps); + double x_d = 0.0; + T x = 0; + for (int i = 0; i < COUNT; ++i) { + x_d += 1.0; + ++x; + double y_d = static_cast(func(x)); + double result = LIBC_NAMESPACE::fputil::sqrt(x_d); + double errors = LIBC_NAMESPACE::fputil::abs((y_d / result) - 1.0); + if (errors > ERR) { + // Print out the failure input and output. + EXPECT_EQ(x, T(0)); + EXPECT_EQ(func(x), zero); + } + ASSERT_TRUE(errors <= ERR); + } + } +}; + +#define LIST_ISQRT_TESTS(Name, T, func) \ + using LlvmLibcISqrt##Name##Test = ISqrtTest; \ + TEST_F(LlvmLibcISqrt##Name##Test, SpecialNumbers) { \ + testSpecialNumbers(&func); \ + } \ + static_assert(true, "Require semicolon.") diff --git a/libc/test/src/stdfix/uhksqrtus_test.cpp b/libc/test/src/stdfix/uhksqrtus_test.cpp new file mode 100644 index 000000000000..a33297413980 --- /dev/null +++ b/libc/test/src/stdfix/uhksqrtus_test.cpp @@ -0,0 +1,20 @@ +//===-- Unittests for uhksqrtus -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "ISqrtTest.h" + +#include "src/__support/fixed_point/sqrt.h" +#include "src/stdfix/uhksqrtus.h" + +unsigned short accum uhksqrtus_fast(unsigned short x) { + return LIBC_NAMESPACE::fixed_point::isqrt_fast(x); +} + +LIST_ISQRT_TESTS(US, unsigned short, LIBC_NAMESPACE::uhksqrtus); + +LIST_ISQRT_TESTS(USFast, unsigned short, uhksqrtus_fast); diff --git a/libc/test/src/stdfix/uksqrtui_test.cpp b/libc/test/src/stdfix/uksqrtui_test.cpp new file mode 100644 index 000000000000..0f4c057099da --- /dev/null +++ b/libc/test/src/stdfix/uksqrtui_test.cpp @@ -0,0 +1,20 @@ +//===-- Unittests for uksqrtui --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "ISqrtTest.h" + +#include "src/__support/fixed_point/sqrt.h" +#include "src/stdfix/uksqrtui.h" + +unsigned accum uksqrtui_fast(unsigned int x) { + return LIBC_NAMESPACE::fixed_point::isqrt_fast(x); +} + +LIST_ISQRT_TESTS(UI, unsigned int, LIBC_NAMESPACE::uksqrtui); + +LIST_ISQRT_TESTS(UIFast, unsigned int, uksqrtui_fast); -- GitLab From 7daa36ae7fe66c7cb0b6cb4311dcb50f120f24f6 Mon Sep 17 00:00:00 2001 From: Changpeng Fang Date: Wed, 6 Mar 2024 15:40:23 -0800 Subject: [PATCH 372/929] [AMDGPU] Make ds_bvh_stack_rtn_b32 depend on HasImageInsts, NFC (#84256) For GFX11. --- llvm/lib/Target/AMDGPU/DSInstructions.td | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td index 219ff37b0a15..5b94102770cd 100644 --- a/llvm/lib/Target/AMDGPU/DSInstructions.td +++ b/llvm/lib/Target/AMDGPU/DSInstructions.td @@ -714,6 +714,7 @@ def DS_SUB_GS_REG_RTN : DS_0A1D_RET_GDS<"ds_sub_gs_reg_rtn", VReg_64, VGPR_32>; let SubtargetPredicate = isGFX11Plus in { +let OtherPredicates = [HasImageInsts] in def DS_BVH_STACK_RTN_B32 : DS_BVH_STACK<"ds_bvh_stack_rtn_b32">; } // let SubtargetPredicate = isGFX11Plus -- GitLab From e3faae96e76c31c163973854cfe6689f28acc4ff Mon Sep 17 00:00:00 2001 From: AtariDreams <83477269+AtariDreams@users.noreply.github.com> Date: Wed, 6 Mar 2024 18:41:15 -0500 Subject: [PATCH 373/929] [Clang] [AST] Resolve FIXME: Assign E to nullptr (#84229) We do not support GCC 4.8 anymore. --- clang/include/clang/AST/Expr.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h index f7857d63bdef..446bec4081e8 100644 --- a/clang/include/clang/AST/Expr.h +++ b/clang/include/clang/AST/Expr.h @@ -5843,7 +5843,7 @@ class GenericSelectionExpr final std::conditional_t; using TSIPtrPtrTy = std::conditional_t; - StmtPtrPtrTy E; // = nullptr; FIXME: Once support for gcc 4.8 is dropped. + StmtPtrPtrTy E = nullptr; TSIPtrPtrTy TSI; // Kept in sync with E. unsigned Offset = 0, SelectedOffset = 0; AssociationIteratorTy(StmtPtrPtrTy E, TSIPtrPtrTy TSI, unsigned Offset, -- GitLab From 8acef12030617e7e160eee38d92bc91007c7fd94 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Wed, 6 Mar 2024 15:45:49 -0800 Subject: [PATCH 374/929] [NFC] [scudo] remove DCHECK (#84255) this gets checked in StackDepot::init anyway --- compiler-rt/lib/scudo/standalone/combined.h | 1 - 1 file changed, 1 deletion(-) diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h index fa6077384d98..069b5f64475d 100644 --- a/compiler-rt/lib/scudo/standalone/combined.h +++ b/compiler-rt/lib/scudo/standalone/combined.h @@ -1570,7 +1570,6 @@ private: if (TabSize > UINT32_MAX / kFramesPerStack) return; u32 RingSize = static_cast(TabSize * kFramesPerStack); - DCHECK(isPowerOfTwo(RingSize)); uptr StackDepotSize = sizeof(StackDepot) + sizeof(atomic_u64) * RingSize + sizeof(atomic_u32) * TabSize; -- GitLab From 6fc5dc3ebb724965662e1242b1a5e2be840ced13 Mon Sep 17 00:00:00 2001 From: lntue <35648136+lntue@users.noreply.github.com> Date: Wed, 6 Mar 2024 18:51:47 -0500 Subject: [PATCH 375/929] [libc] Fix a typo in llvm_libc_ext.td. (#84259) --- libc/spec/llvm_libc_ext.td | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libc/spec/llvm_libc_ext.td b/libc/spec/llvm_libc_ext.td index 274284ed5705..3241ec055037 100644 --- a/libc/spec/llvm_libc_ext.td +++ b/libc/spec/llvm_libc_ext.td @@ -73,7 +73,7 @@ def LLVMLibcExt : StandardSpec<"llvm_libc_ext"> { let Headers = [ Assert, Sched, - Stdfix, + StdFix, Strings, ]; } -- GitLab From 1a37147af5a91635e3982a9cbbdfe1d7492af84c Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Wed, 6 Mar 2024 15:57:34 -0800 Subject: [PATCH 376/929] [SCEV] Match both (-1)b + a and a + (-1)b as a - b (#84247) In our analysis of guarding conditions, we were converting a-b == 0 into a == b alternate form, but we were only checking for one of the two forms for the sub. There's no requirement that the multiply only be on the LHS of the add. --- llvm/lib/Analysis/ScalarEvolution.cpp | 32 ++++++++++++------- .../Analysis/ScalarEvolution/trip-count.ll | 2 +- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 15c2965aede1..acc0aa23107b 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -10577,6 +10577,25 @@ static bool HasSameValue(const SCEV *A, const SCEV *B) { return false; } +static bool MatchBinarySub(const SCEV *S, const SCEV *&LHS, const SCEV *&RHS) { + const SCEVAddExpr *Add = dyn_cast(S); + if (!Add || Add->getNumOperands() != 2) + return false; + if (auto *ME = dyn_cast(Add->getOperand(0)); + ME && ME->getNumOperands() == 2 && ME->getOperand(0)->isAllOnesValue()) { + LHS = Add->getOperand(1); + RHS = ME->getOperand(1); + return true; + } + if (auto *ME = dyn_cast(Add->getOperand(1)); + ME && ME->getNumOperands() == 2 && ME->getOperand(0)->isAllOnesValue()) { + LHS = Add->getOperand(0); + RHS = ME->getOperand(1); + return true; + } + return false; +} + bool ScalarEvolution::SimplifyICmpOperands(ICmpInst::Predicate &Pred, const SCEV *&LHS, const SCEV *&RHS, unsigned Depth) { @@ -10652,19 +10671,10 @@ bool ScalarEvolution::SimplifyICmpOperands(ICmpInst::Predicate &Pred, case ICmpInst::ICMP_EQ: case ICmpInst::ICMP_NE: // Fold ((-1) * %a) + %b == 0 (equivalent to %b-%a == 0) into %a == %b. - if (!RA) - if (const SCEVAddExpr *AE = dyn_cast(LHS)) - if (const SCEVMulExpr *ME = - dyn_cast(AE->getOperand(0))) - if (AE->getNumOperands() == 2 && ME->getNumOperands() == 2 && - ME->getOperand(0)->isAllOnesValue()) { - RHS = AE->getOperand(1); - LHS = ME->getOperand(1); - Changed = true; - } + if (RA.isZero() && MatchBinarySub(LHS, LHS, RHS)) + Changed = true; break; - // The "Should have been caught earlier!" messages refer to the fact // that the ExactCR.isFullSet() or ExactCR.isEmptySet() check above // should have fired on the corresponding cases, and canonicalized the diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count.ll b/llvm/test/Analysis/ScalarEvolution/trip-count.ll index cbe07effdeb2..8fc5b9b40961 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count.ll @@ -145,7 +145,7 @@ define void @dual_sext_ne_with_slt_guard(i8 %s, i8 %n) { ; CHECK-LABEL: 'dual_sext_ne_with_slt_guard' ; CHECK-NEXT: Determining loop execution counts for: @dual_sext_ne_with_slt_guard ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + (sext i8 %n to i64) + (-1 * (sext i8 %s to i64))) -; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 -1 +; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i64 -2 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + (sext i8 %n to i64) + (-1 * (sext i8 %s to i64))) ; CHECK-NEXT: Loop %for.body: Trip multiple is 1 ; -- GitLab From 6a39a714423ba5675b9e211a80c1dae1ff0e6eac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Thu, 7 Mar 2024 01:01:07 +0100 Subject: [PATCH 377/929] Revert "[clang-repl] Pass triple to IncrementalCompilerBuilder as explicit argument" (#84261) Reverts llvm/llvm-project#84174 due too sanitizer memory leak detection --- clang/include/clang/Interpreter/Interpreter.h | 5 +-- clang/lib/Interpreter/Interpreter.cpp | 12 +++---- clang/unittests/Interpreter/CMakeLists.txt | 1 - .../IncrementalCompilerBuilderTest.cpp | 35 ------------------- 4 files changed, 6 insertions(+), 47 deletions(-) delete mode 100644 clang/unittests/Interpreter/IncrementalCompilerBuilderTest.cpp diff --git a/clang/include/clang/Interpreter/Interpreter.h b/clang/include/clang/Interpreter/Interpreter.h index c8f932e95c47..292fa566ae70 100644 --- a/clang/include/clang/Interpreter/Interpreter.h +++ b/clang/include/clang/Interpreter/Interpreter.h @@ -48,8 +48,6 @@ public: UserArgs = Args; } - void SetTargetTriple(std::string TT) { TargetTriple = TT; } - // General C++ llvm::Expected> CreateCpp(); @@ -64,12 +62,11 @@ public: private: static llvm::Expected> - create(std::string TT, std::vector &ClangArgv); + create(std::vector &ClangArgv); llvm::Expected> createCuda(bool device); std::vector UserArgs; - std::optional TargetTriple; llvm::StringRef OffloadArch; llvm::StringRef CudaSDKPath; diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp index 37696b289764..9f97a3c6b0be 100644 --- a/clang/lib/Interpreter/Interpreter.cpp +++ b/clang/lib/Interpreter/Interpreter.cpp @@ -132,8 +132,7 @@ CreateCI(const llvm::opt::ArgStringList &Argv) { } // anonymous namespace llvm::Expected> -IncrementalCompilerBuilder::create(std::string TT, - std::vector &ClangArgv) { +IncrementalCompilerBuilder::create(std::vector &ClangArgv) { // If we don't know ClangArgv0 or the address of main() at this point, try // to guess it anyway (it's possible on some platforms). @@ -163,7 +162,8 @@ IncrementalCompilerBuilder::create(std::string TT, TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer; DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer); - driver::Driver Driver(/*MainBinaryName=*/ClangArgv[0], TT, Diags); + driver::Driver Driver(/*MainBinaryName=*/ClangArgv[0], + llvm::sys::getProcessTriple(), Diags); Driver.setCheckInputsExist(false); // the input comes from mem buffers llvm::ArrayRef RF = llvm::ArrayRef(ClangArgv); std::unique_ptr Compilation(Driver.BuildCompilation(RF)); @@ -185,8 +185,7 @@ IncrementalCompilerBuilder::CreateCpp() { Argv.push_back("-xc++"); Argv.insert(Argv.end(), UserArgs.begin(), UserArgs.end()); - std::string TT = TargetTriple ? *TargetTriple : llvm::sys::getProcessTriple(); - return IncrementalCompilerBuilder::create(TT, Argv); + return IncrementalCompilerBuilder::create(Argv); } llvm::Expected> @@ -214,8 +213,7 @@ IncrementalCompilerBuilder::createCuda(bool device) { Argv.insert(Argv.end(), UserArgs.begin(), UserArgs.end()); - std::string TT = TargetTriple ? *TargetTriple : llvm::sys::getProcessTriple(); - return IncrementalCompilerBuilder::create(TT, Argv); + return IncrementalCompilerBuilder::create(Argv); } llvm::Expected> diff --git a/clang/unittests/Interpreter/CMakeLists.txt b/clang/unittests/Interpreter/CMakeLists.txt index 0ddedb283e07..712641afb976 100644 --- a/clang/unittests/Interpreter/CMakeLists.txt +++ b/clang/unittests/Interpreter/CMakeLists.txt @@ -7,7 +7,6 @@ set(LLVM_LINK_COMPONENTS ) add_clang_unittest(ClangReplInterpreterTests - IncrementalCompilerBuilderTest.cpp IncrementalProcessingTest.cpp InterpreterTest.cpp CodeCompletionTest.cpp diff --git a/clang/unittests/Interpreter/IncrementalCompilerBuilderTest.cpp b/clang/unittests/Interpreter/IncrementalCompilerBuilderTest.cpp deleted file mode 100644 index 1cc0223465c8..000000000000 --- a/clang/unittests/Interpreter/IncrementalCompilerBuilderTest.cpp +++ /dev/null @@ -1,35 +0,0 @@ -//=== unittests/Interpreter/IncrementalCompilerBuilderTest.cpp ------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "clang/Basic/TargetOptions.h" -#include "clang/Frontend/CompilerInstance.h" -#include "clang/Interpreter/Interpreter.h" -#include "llvm/Support/Error.h" -#include "gtest/gtest.h" - -using namespace llvm; -using namespace clang; - -namespace { - -TEST(IncrementalCompilerBuilder, SetCompilerArgs) { - std::vector ClangArgv = {"-Xclang", "-ast-dump-all"}; - auto CB = clang::IncrementalCompilerBuilder(); - CB.SetCompilerArgs(ClangArgv); - auto CI = cantFail(CB.CreateCpp()); - EXPECT_TRUE(CI->getFrontendOpts().ASTDumpAll); -} - -TEST(IncrementalCompilerBuilder, SetTargetTriple) { - auto CB = clang::IncrementalCompilerBuilder(); - CB.SetTargetTriple("armv6-none-eabi"); - auto CI = cantFail(CB.CreateCpp()); - EXPECT_EQ(CI->getTargetOpts().Triple, "armv6-none-unknown-eabi"); -} - -} // end anonymous namespace -- GitLab From b84f3ea9963058cdd910f5106ca2a70c1bb9e460 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Thu, 7 Mar 2024 00:03:07 +0000 Subject: [PATCH 378/929] [gn build] Port 6a39a714423b --- llvm/utils/gn/secondary/clang/unittests/Interpreter/BUILD.gn | 1 - 1 file changed, 1 deletion(-) diff --git a/llvm/utils/gn/secondary/clang/unittests/Interpreter/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/Interpreter/BUILD.gn index a20066436a3b..441d57187cd2 100644 --- a/llvm/utils/gn/secondary/clang/unittests/Interpreter/BUILD.gn +++ b/llvm/utils/gn/secondary/clang/unittests/Interpreter/BUILD.gn @@ -12,7 +12,6 @@ unittest("ClangReplInterpreterTests") { ] sources = [ "CodeCompletionTest.cpp", - "IncrementalCompilerBuilderTest.cpp", "IncrementalProcessingTest.cpp", "InterpreterTest.cpp", ] -- GitLab From 4258b0e13ff8eced081f3344677f02373ea88127 Mon Sep 17 00:00:00 2001 From: Kirill Stoimenov <87100199+kstoimenov@users.noreply.github.com> Date: Wed, 6 Mar 2024 16:16:59 -0800 Subject: [PATCH 379/929] [HWASAN] Follow up for #83503 implement selective instrumentation (#83942) 1. Change tests to use IR instead of -stats to avoid depending on Debug mode 2. Add SkipInstrumentationRandomRate 3. Remove HWASAN from stat strings --- .../Instrumentation/HWAddressSanitizer.cpp | 57 ++++++++++++------- .../HWAddressSanitizer/pgo-opt-out-no-ps.ll | 36 ++++++++---- .../HWAddressSanitizer/pgo-opt-out.ll | 33 ++++++++--- 3 files changed, 86 insertions(+), 40 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp index 4404382a85b7..236ee8910d46 100644 --- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -52,6 +52,7 @@ #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/RandomNumberGenerator.h" #include "llvm/Support/raw_ostream.h" #include "llvm/TargetParser/Triple.h" #include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h" @@ -61,6 +62,7 @@ #include "llvm/Transforms/Utils/ModuleUtils.h" #include "llvm/Transforms/Utils/PromoteMemToReg.h" #include +#include using namespace llvm; @@ -181,16 +183,23 @@ static cl::opt ClWithTls( cl::Hidden, cl::init(true)); static cl::opt - CSkipHotCode("hwasan-skip-hot-code", - cl::desc("Do not instument hot functions based on FDO."), - cl::Hidden, cl::init(false)); + CSelectiveInstrumentation("hwasan-selective-instrumentation", + cl::desc("Use selective instrumentation"), + cl::Hidden, cl::init(false)); -static cl::opt HotPercentileCutoff("hwasan-percentile-cutoff-hot", - cl::init(0)); +static cl::opt HotPercentileCutoff( + "hwasan-percentile-cutoff-hot", cl::init(0), + cl::desc("Alternative hot percentile cuttoff." + "By default `-profile-summary-cutoff-hot` is used.")); -STATISTIC(NumTotalFuncs, "Number of total funcs HWASAN"); -STATISTIC(NumInstrumentedFuncs, "Number of HWASAN instrumented funcs"); -STATISTIC(NumNoProfileSummaryFuncs, "Number of HWASAN funcs without PS"); +static cl::opt + RandomSkipRate("hwasan-random-skip-rate", cl::init(0), + cl::desc("Probability value in the range [0.0, 1.0] " + "to skip instrumentation of a function.")); + +STATISTIC(NumTotalFuncs, "Number of total funcs"); +STATISTIC(NumInstrumentedFuncs, "Number of instrumented funcs"); +STATISTIC(NumNoProfileSummaryFuncs, "Number of funcs without PS"); // Mode for selecting how to insert frame record info into the stack ring // buffer. @@ -291,6 +300,8 @@ public: this->CompileKernel = ClEnableKhwasan.getNumOccurrences() > 0 ? ClEnableKhwasan : CompileKernel; + this->Rng = + RandomSkipRate.getNumOccurrences() ? M.createRNG("hwasan") : nullptr; initializeModule(); } @@ -372,6 +383,7 @@ private: Module &M; const StackSafetyGlobalInfo *SSI; Triple TargetTriple; + std::unique_ptr Rng; /// This struct defines the shadow mapping using the rule: /// shadow = (mem >> Scale) + Offset. @@ -1526,19 +1538,26 @@ void HWAddressSanitizer::sanitizeFunction(Function &F, return; NumTotalFuncs++; - if (CSkipHotCode) { - auto &MAMProxy = FAM.getResult(F); - ProfileSummaryInfo *PSI = - MAMProxy.getCachedResult(*F.getParent()); - if (PSI && PSI->hasProfileSummary()) { - auto &BFI = FAM.getResult(F); - if ((HotPercentileCutoff.getNumOccurrences() && HotPercentileCutoff >= 0) - ? PSI->isFunctionHotInCallGraphNthPercentile(HotPercentileCutoff, - &F, BFI) - : PSI->isFunctionHotInCallGraph(&F, BFI)) + if (CSelectiveInstrumentation) { + if (RandomSkipRate.getNumOccurrences()) { + std::bernoulli_distribution D(RandomSkipRate); + if (D(*Rng)) return; } else { - ++NumNoProfileSummaryFuncs; + auto &MAMProxy = FAM.getResult(F); + ProfileSummaryInfo *PSI = + MAMProxy.getCachedResult(*F.getParent()); + if (PSI && PSI->hasProfileSummary()) { + auto &BFI = FAM.getResult(F); + if ((HotPercentileCutoff.getNumOccurrences() && + HotPercentileCutoff >= 0) + ? PSI->isFunctionHotInCallGraphNthPercentile( + HotPercentileCutoff, &F, BFI) + : PSI->isFunctionHotInCallGraph(&F, BFI)) + return; + } else { + ++NumNoProfileSummaryFuncs; + } } } NumInstrumentedFuncs++; diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/pgo-opt-out-no-ps.ll b/llvm/test/Instrumentation/HWAddressSanitizer/pgo-opt-out-no-ps.ll index 2aa218fa1522..8d96ab021288 100644 --- a/llvm/test/Instrumentation/HWAddressSanitizer/pgo-opt-out-no-ps.ll +++ b/llvm/test/Instrumentation/HWAddressSanitizer/pgo-opt-out-no-ps.ll @@ -1,16 +1,28 @@ -; RUN: opt < %s -passes='require,hwasan' -S -stats 2>&1 \ -; RUN: -hwasan-skip-hot-code=0 | FileCheck %s --check-prefix=FULL -; RUN: opt < %s -passes='require,hwasan' -S -stats 2>&1 \ -; RUN: -hwasan-skip-hot-code=1 | FileCheck %s --check-prefix=SELSAN +; RUN: opt < %s -passes='require,hwasan' -S \ +; RUN: -hwasan-selective-instrumentation=0 | FileCheck %s --check-prefix=FULL +; RUN: opt < %s -passes='require,hwasan' -S \ +; RUN: -hwasan-selective-instrumentation=1 | FileCheck %s --check-prefix=SELSAN -; REQUIRES: asserts +; FULL: @not_sanitized +; FULL-NEXT: %x = alloca i8, i64 4 +; FULL: @sanitized_no_ps +; FULL-NEXT: @__hwasan_tls -; FULL: 1 hwasan - Number of HWASAN instrumented funcs -; FULL: 1 hwasan - Number of total funcs HWASAN +; SELSAN: @not_sanitized +; SELSAN-NEXT: %x = alloca i8, i64 4 +; SELSAN: @sanitized_no_ps +; SELSAN-NEXT: @__hwasan_tls -; SELSAN: 1 hwasan - Number of HWASAN instrumented funcs -; SELSAN: 1 hwasan - Number of HWASAN funcs without PS -; SELSAN: 1 hwasan - Number of total funcs HWASAN +declare void @use(ptr) -define void @not_sanitized() { ret void } -define void @sanitized_no_ps() sanitize_hwaddress { ret void } +define void @not_sanitized() { + %x = alloca i8, i64 4 + call void @use(ptr %x) + ret void + } + +define void @sanitized_no_ps() sanitize_hwaddress { + %x = alloca i8, i64 4 + call void @use(ptr %x) + ret void + } diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/pgo-opt-out.ll b/llvm/test/Instrumentation/HWAddressSanitizer/pgo-opt-out.ll index 65a5f8c96896..28e43a99883e 100644 --- a/llvm/test/Instrumentation/HWAddressSanitizer/pgo-opt-out.ll +++ b/llvm/test/Instrumentation/HWAddressSanitizer/pgo-opt-out.ll @@ -1,16 +1,31 @@ -; RUN: opt < %s -passes='require,hwasan' -S -stats 2>&1 \ -; RUN: -hwasan-skip-hot-code=1 | FileCheck %s --check-prefix=DEFAULT -; RUN: opt < %s -passes='require,hwasan' -S -stats 2>&1 \ -; RUN: -hwasan-skip-hot-code=1 -hwasan-percentile-cutoff-hot=700000 | FileCheck %s --check-prefix=PERCENT +; RUN: opt < %s -passes='require,hwasan' -S -hwasan-selective-instrumentation=1 \ +; RUN: | FileCheck %s --check-prefix=DEFAULT +; RUN: opt < %s -passes='require,hwasan' -S -hwasan-selective-instrumentation=1 \ +; RUN: -hwasan-percentile-cutoff-hot=700000 | FileCheck %s --check-prefix=HOT_RATE +; RUN: opt < %s -passes='require,hwasan' -S -hwasan-selective-instrumentation=1 \ +; RUN: -hwasan-random-skip-rate=0.0 | FileCheck %s --check-prefix=RANDOM_RATE_0 +; RUN: opt < %s -passes='require,hwasan' -S -hwasan-selective-instrumentation=1 \ +; RUN: -hwasan-random-skip-rate=1.0 | FileCheck %s --check-prefix=RANDOM_RATE_1 -; REQUIRES: asserts +; DEFAULT: @sanitized +; DEFAULT-NEXT: %x = alloca i8, i64 4 -; DEFAULT: 1 hwasan - Number of total funcs HWASAN +; HOT_RATE: @sanitized +; HOT_RATE-NEXT: @__hwasan_tls -; PERCENT: 1 hwasan - Number of HWASAN instrumented funcs -; PERCENT: 1 hwasan - Number of total funcs HWASAN +; RANDOM_RATE_0: @sanitized +; RANDOM_RATE_0-NEXT: @__hwasan_tls -define void @sanitized() sanitize_hwaddress !prof !36 { ret void } +; RANDOM_RATE_1: @sanitized +; RANDOM_RATE_1-NEXT: %x = alloca i8, i64 4 + +declare void @use(ptr) + +define void @sanitized(i32 noundef %0) sanitize_hwaddress !prof !36 { + %x = alloca i8, i64 4 + call void @use(ptr %x) + ret void +} !llvm.module.flags = !{!6} !6 = !{i32 1, !"ProfileSummary", !7} -- GitLab From 0497c77e9e02f0dcccd42a6afd65d0356a5dbb57 Mon Sep 17 00:00:00 2001 From: Felipe de Azevedo Piovezan Date: Wed, 6 Mar 2024 16:25:22 -0800 Subject: [PATCH 380/929] [lldb] Print mangled names with verbose break list (#84071) When debugging LLDB itself, it can often be useful to know the mangled name of the function where a breakpoint is set. Since the `--verbose` setting of `break --list` is aimed at debugging LLDB, this patch makes it so that the mangled name is also printed in that mode. Note about testing: since mangling is not the same on Windows and Linux, the test refrains from hardcoding mangled names. --- lldb/source/Breakpoint/BreakpointLocation.cpp | 6 ++++++ .../breakpoint_options/TestBreakpointOptions.py | 13 ++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/lldb/source/Breakpoint/BreakpointLocation.cpp b/lldb/source/Breakpoint/BreakpointLocation.cpp index f7b8ca1f5506..b48ec1398d63 100644 --- a/lldb/source/Breakpoint/BreakpointLocation.cpp +++ b/lldb/source/Breakpoint/BreakpointLocation.cpp @@ -524,6 +524,12 @@ void BreakpointLocation::GetDescription(Stream *s, s->EOL(); s->Indent("function = "); s->PutCString(sc.function->GetName().AsCString("")); + if (ConstString mangled_name = + sc.function->GetMangled().GetMangledName()) { + s->EOL(); + s->Indent("mangled function = "); + s->PutCString(mangled_name.AsCString()); + } } if (sc.line_entry.line > 0) { diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_options/TestBreakpointOptions.py b/lldb/test/API/functionalities/breakpoint/breakpoint_options/TestBreakpointOptions.py index 129290909029..5179ffe730b9 100644 --- a/lldb/test/API/functionalities/breakpoint/breakpoint_options/TestBreakpointOptions.py +++ b/lldb/test/API/functionalities/breakpoint/breakpoint_options/TestBreakpointOptions.py @@ -82,7 +82,7 @@ class BreakpointOptionsTestCase(TestBase): self.runCmd("file " + exe, CURRENT_EXECUTABLE_SET) # This should create a breakpoint with 1 locations. - lldbutil.run_break_set_by_symbol( + bp_id = lldbutil.run_break_set_by_symbol( self, "ns::func", sym_exact=False, @@ -90,6 +90,17 @@ class BreakpointOptionsTestCase(TestBase): num_expected_locations=1, ) + location = self.target().FindBreakpointByID(bp_id).GetLocationAtIndex(0) + function = location.GetAddress().GetFunction() + self.expect( + "breakpoint list -v", + "Verbose breakpoint list contains mangled names", + substrs=[ + "function = ns::func", + f"mangled function = {function.GetMangledName()}", + ], + ) + # This should create a breakpoint with 0 locations. lldbutil.run_break_set_by_symbol( self, -- GitLab From 318bff6811e7a7e0d3295ed85aa3ad01d475cc8c Mon Sep 17 00:00:00 2001 From: Pranav Kant Date: Wed, 6 Mar 2024 16:40:23 -0800 Subject: [PATCH 381/929] [clang][CUDA] Disable float128 diagnostics for device compilation (#83918) --- clang/lib/Sema/SemaDeclAttr.cpp | 4 +++- clang/lib/Sema/SemaType.cpp | 2 +- clang/test/SemaCUDA/float128.cu | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 clang/test/SemaCUDA/float128.cu diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp index 397b5db0dc06..e6943efb345c 100644 --- a/clang/lib/Sema/SemaDeclAttr.cpp +++ b/clang/lib/Sema/SemaDeclAttr.cpp @@ -4877,7 +4877,9 @@ void Sema::AddModeAttr(Decl *D, const AttributeCommonInfo &CI, NewElemTy = Context.getRealTypeForBitwidth(DestWidth, ExplicitType); if (NewElemTy.isNull()) { - Diag(AttrLoc, diag::err_machine_mode) << 1 /*Unsupported*/ << Name; + // Only emit diagnostic on host for 128-bit mode attribute + if (!(DestWidth == 128 && getLangOpts().CUDAIsDevice)) + Diag(AttrLoc, diag::err_machine_mode) << 1 /*Unsupported*/ << Name; return; } diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp index 1e43e36016a6..3148299f6467 100644 --- a/clang/lib/Sema/SemaType.cpp +++ b/clang/lib/Sema/SemaType.cpp @@ -1561,7 +1561,7 @@ static QualType ConvertDeclSpecToType(TypeProcessingState &state) { break; case DeclSpec::TST_float128: if (!S.Context.getTargetInfo().hasFloat128Type() && - !S.getLangOpts().SYCLIsDevice && + !S.getLangOpts().SYCLIsDevice && !S.getLangOpts().CUDAIsDevice && !(S.getLangOpts().OpenMP && S.getLangOpts().OpenMPIsTargetDevice)) S.Diag(DS.getTypeSpecTypeLoc(), diag::err_type_unsupported) << "__float128"; diff --git a/clang/test/SemaCUDA/float128.cu b/clang/test/SemaCUDA/float128.cu new file mode 100644 index 000000000000..f8f20cb1588d --- /dev/null +++ b/clang/test/SemaCUDA/float128.cu @@ -0,0 +1,18 @@ +// CPU-side compilation on x86 (no errors expected). +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -aux-triple nvptx64 -x cuda -fsyntax-only -verify=cpu %s + +// GPU-side compilation on x86 (no errors expected) +// RUN: %clang_cc1 -triple nvptx64 -aux-triple x86_64-unknown-linux-gnu -fcuda-is-device -x cuda -fsyntax-only -verify=gpu %s + +// cpu-no-diagnostics +typedef _Complex float __cfloat128 __attribute__ ((__mode__ (__TC__))); +typedef __float128 _Float128; + +// gpu-note@+1 {{'a' defined here}} +__attribute__((device)) __float128 f(__float128 a, float b) { + // gpu-note@+1 {{'c' defined here}} + __float128 c = b + 1.0; + // gpu-error@+2 {{'a' requires 128 bit size '__float128' type support, but target 'nvptx64' does not support it}} + // gpu-error@+1 {{'c' requires 128 bit size '__float128' type support, but target 'nvptx64' does not support it}} + return a + c; +} \ No newline at end of file -- GitLab From 551e20d190868fe47f8efbfd1dbf38191cc2c95d Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Wed, 6 Mar 2024 17:19:59 -0800 Subject: [PATCH 382/929] [ELF] Reject error-prone meta characters in input section description The lexer is overly permissive. When parsing file patterns in an input section description and there is a missing `)`, we would accept many non-sensible tokens (e.g. `}`) as patterns, leading to confusion, e.g. `*(SORT_BY_ALIGNMENT(SORT_BY_NAME(.text*)) } PROVIDE_HIDDEN(__code_end = .)` (#81804). Ideally, the lexer should be stateful to report more errors like GNU ld and get rid of hacks like `ScriptLexer::maybeSplitExpr`, but that would require a large rewrite of the lexer. For now, just reject certain non-wildcard meta characters to detect common mistakes. Pull Request: https://github.com/llvm/llvm-project/pull/84130 --- lld/ELF/ScriptParser.cpp | 14 ++++++++++-- lld/test/ELF/linkerscript/wildcards.s | 32 ++++++++++++++++++++------- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/lld/ELF/ScriptParser.cpp b/lld/ELF/ScriptParser.cpp index f0ede1f43bbd..3bb1de99480f 100644 --- a/lld/ELF/ScriptParser.cpp +++ b/lld/ELF/ScriptParser.cpp @@ -717,9 +717,19 @@ SmallVector ScriptParser::readInputSectionsList() { StringMatcher SectionMatcher; // Break if the next token is ), EXCLUDE_FILE, or SORT*. - while (!errorCount() && peek() != ")" && peek() != "EXCLUDE_FILE" && - peekSortKind() == SortSectionPolicy::Default) + while (!errorCount() && peekSortKind() == SortSectionPolicy::Default) { + StringRef s = peek(); + if (s == ")" || s == "EXCLUDE_FILE") + break; + // Detect common mistakes when certain non-wildcard meta characters are + // used without a closing ')'. + if (!s.empty() && strchr("(){}", s[0])) { + skip(); + setError("section pattern is expected"); + break; + } SectionMatcher.addPattern(unquote(next())); + } if (!SectionMatcher.empty()) ret.push_back({std::move(excludeFilePat), std::move(SectionMatcher)}); diff --git a/lld/test/ELF/linkerscript/wildcards.s b/lld/test/ELF/linkerscript/wildcards.s index 1eea27891dfc..e9ae722e4489 100644 --- a/lld/test/ELF/linkerscript/wildcards.s +++ b/lld/test/ELF/linkerscript/wildcards.s @@ -91,26 +91,42 @@ SECTIONS { .text : { *([.]abc .ab[v-y] ) } } -## Test a few non-wildcard meta characters rejected by GNU ld. +## Test a few non-wildcard characters rejected by GNU ld. #--- lbrace.lds -# RUN: ld.lld -T lbrace.lds a.o -o out +# RUN: not ld.lld -T lbrace.lds a.o 2>&1 | FileCheck %s --check-prefix=ERR-LBRACE --match-full-lines --strict-whitespace +# ERR-LBRACE:{{.*}}: section pattern is expected +# ERR-LBRACE-NEXT:>>> .text : { *(.a* { ) } +# ERR-LBRACE-NEXT:>>> ^ SECTIONS { .text : { *(.a* { ) } } +#--- lbrace2.lds +# RUN: not ld.lld -T lbrace2.lds a.o 2>&1 | FileCheck %s --check-prefix=ERR-LBRACE2 --match-full-lines --strict-whitespace +# ERR-LBRACE2:{{.*}}: section pattern is expected +# ERR-LBRACE2-NEXT:>>> .text : { *(.a*{) } +# ERR-LBRACE2-NEXT:>>> ^ +SECTIONS { + .text : { *(.a*{) } +} + #--- lparen.lds -## ( is recognized as a section name pattern. Note, ( is rejected by GNU ld. -# RUN: ld.lld -T lparen.lds a.o -o out -# RUN: llvm-objdump --section-headers out | FileCheck --check-prefix=SEC-NO %s +# RUN: not ld.lld -T lparen.lds a.o 2>&1 | FileCheck %s --check-prefix=ERR-LPAREN --match-full-lines --strict-whitespace +# ERR-LPAREN:{{.*}}: section pattern is expected +# ERR-LPAREN-NEXT:>>> .text : { *(.a* ( ) } +# ERR-LPAREN-NEXT:>>> ^ SECTIONS { - .text : { *(.a* ( ) } + .text : { *(.a* ( ) } } #--- rbrace.lds -# RUN: ld.lld -T rbrace.lds a.o -o out +# RUN: not ld.lld -T rbrace.lds a.o 2>&1 | FileCheck %s --check-prefix=ERR-RBRACE --match-full-lines --strict-whitespace +# ERR-RBRACE:{{.*}}: section pattern is expected +# ERR-RBRACE-NEXT:>>> .text : { *(.a* x = 3; } ) } +# ERR-RBRACE-NEXT:>>> ^ SECTIONS { - .text : { *(.a* } ) } + .text : { *(.a* x = 3; } ) } } #--- rparen.lds -- GitLab From 59a92019fbc0a67ec82a903a4f6167ad45545a7f Mon Sep 17 00:00:00 2001 From: Matthias Springer Date: Thu, 7 Mar 2024 10:26:22 +0900 Subject: [PATCH 383/929] [mlir][IR] Make `replaceOp` / `replaceAllUsesWith` API consistent (#82629) * `replaceOp` replaces all uses of the original op and erases the old op. * `replaceAllUsesWith` replaces all uses of the original op/value/block. It does not erase any IR. This commit renames `replaceOpWithIf` to `replaceUsesWithIf`. `replaceOpWithIf` was a misnomer because the function never erases the original op. Similarly, `replaceOpWithinBlock` is renamed to `replaceUsesWithinBlock`. (No "operation replaced" is sent because the op is not erased.) Also improve comments. --- mlir/include/mlir/IR/PatternMatch.h | 87 +++++++++---------- .../mlir/Transforms/DialectConversion.h | 6 -- .../Linalg/Transforms/DecomposeLinalgOps.cpp | 4 +- mlir/lib/Dialect/Linalg/Utils/Utils.cpp | 2 +- mlir/lib/IR/PatternMatch.cpp | 73 ++++++---------- .../Transforms/Utils/DialectConversion.cpp | 13 --- mlir/lib/Transforms/Utils/RegionUtils.cpp | 2 +- mlir/test/lib/Dialect/Test/TestPatterns.cpp | 2 +- 8 files changed, 74 insertions(+), 115 deletions(-) diff --git a/mlir/include/mlir/IR/PatternMatch.h b/mlir/include/mlir/IR/PatternMatch.h index f8d22cfb22af..e3500b3f9446 100644 --- a/mlir/include/mlir/IR/PatternMatch.h +++ b/mlir/include/mlir/IR/PatternMatch.h @@ -497,42 +497,19 @@ public: Region::iterator before); void inlineRegionBefore(Region ®ion, Block *before); - /// This method replaces the uses of the results of `op` with the values in - /// `newValues` when the provided `functor` returns true for a specific use. - /// The number of values in `newValues` is required to match the number of - /// results of `op`. `allUsesReplaced`, if non-null, is set to true if all of - /// the uses of `op` were replaced. Note that in some rewriters, the given - /// 'functor' may be stored beyond the lifetime of the rewrite being applied. - /// As such, the function should not capture by reference and instead use - /// value capture as necessary. - virtual void - replaceOpWithIf(Operation *op, ValueRange newValues, bool *allUsesReplaced, - llvm::unique_function functor); - void replaceOpWithIf(Operation *op, ValueRange newValues, - llvm::unique_function functor) { - replaceOpWithIf(op, newValues, /*allUsesReplaced=*/nullptr, - std::move(functor)); - } - - /// This method replaces the uses of the results of `op` with the values in - /// `newValues` when a use is nested within the given `block`. The number of - /// values in `newValues` is required to match the number of results of `op`. - /// If all uses of this operation are replaced, the operation is erased. - void replaceOpWithinBlock(Operation *op, ValueRange newValues, Block *block, - bool *allUsesReplaced = nullptr); - - /// This method replaces the results of the operation with the specified list - /// of values. The number of provided values must match the number of results - /// of the operation. The replaced op is erased. + /// Replace the results of the given (original) operation with the specified + /// list of values (replacements). The result types of the given op and the + /// replacements must match. The original op is erased. virtual void replaceOp(Operation *op, ValueRange newValues); - /// This method replaces the results of the operation with the specified - /// new op (replacement). The number of results of the two operations must - /// match. The replaced op is erased. + /// Replace the results of the given (original) operation with the specified + /// new op (replacement). The result types of the two ops must match. The + /// original op is erased. virtual void replaceOp(Operation *op, Operation *newOp); - /// Replaces the result op with a new op that is created without verification. - /// The result values of the two ops must be the same types. + /// Replace the results of the given (original) op with a new op that is + /// created without verification (replacement). The result values of the two + /// ops must match. The original op is erased. template OpTy replaceOpWithNewOp(Operation *op, Args &&...args) { auto newOp = create(op->getLoc(), std::forward(args)...); @@ -634,9 +611,8 @@ public: finalizeOpModification(root); } - /// Find uses of `from` and replace them with `to`. It also marks every - /// modified uses and notifies the rewriter that an in-place operation - /// modification is about to happen. + /// Find uses of `from` and replace them with `to`. Also notify the listener + /// about every in-place op modification (for every use that was replaced). void replaceAllUsesWith(Value from, Value to) { return replaceAllUsesWith(from.getImpl(), to); } @@ -652,22 +628,43 @@ public: for (auto it : llvm::zip(from, to)) replaceAllUsesWith(std::get<0>(it), std::get<1>(it)); } + void replaceAllUsesWith(Operation *from, ValueRange to) { + replaceAllUsesWith(from->getResults(), to); + } /// Find uses of `from` and replace them with `to` if the `functor` returns - /// true. It also marks every modified uses and notifies the rewriter that an - /// in-place operation modification is about to happen. + /// true. Also notify the listener about every in-place op modification (for + /// every use that was replaced). The optional `allUsesReplaced` flag is set + /// to "true" if all uses were replaced. void replaceUsesWithIf(Value from, Value to, - function_ref functor); + function_ref functor, + bool *allUsesReplaced = nullptr); void replaceUsesWithIf(ValueRange from, ValueRange to, - function_ref functor) { - assert(from.size() == to.size() && "incorrect number of replacements"); - for (auto it : llvm::zip(from, to)) - replaceUsesWithIf(std::get<0>(it), std::get<1>(it), functor); + function_ref functor, + bool *allUsesReplaced = nullptr); + void replaceUsesWithIf(Operation *from, ValueRange to, + function_ref functor, + bool *allUsesReplaced = nullptr) { + replaceUsesWithIf(from->getResults(), to, functor, allUsesReplaced); + } + + /// Find uses of `from` within `block` and replace them with `to`. Also notify + /// the listener about every in-place op modification (for every use that was + /// replaced). The optional `allUsesReplaced` flag is set to "true" if all + /// uses were replaced. + void replaceUsesWithinBlock(Operation *op, ValueRange newValues, Block *block, + bool *allUsesReplaced = nullptr) { + replaceUsesWithIf( + op, newValues, + [block](OpOperand &use) { + return block->getParentOp()->isProperAncestor(use.getOwner()); + }, + allUsesReplaced); } /// Find uses of `from` and replace them with `to` except if the user is - /// `exceptedUser`. It also marks every modified uses and notifies the - /// rewriter that an in-place operation modification is about to happen. + /// `exceptedUser`. Also notify the listener about every in-place op + /// modification (for every use that was replaced). void replaceAllUsesExcept(Value from, Value to, Operation *exceptedUser) { return replaceUsesWithIf(from, to, [&](OpOperand &use) { Operation *user = use.getOwner(); @@ -675,7 +672,7 @@ public: }); } - /// Used to notify the rewriter that the IR failed to be rewritten because of + /// Used to notify the listener that the IR failed to be rewritten because of /// a match failure, and provide a callback to populate a diagnostic with the /// reason why the failure occurred. This method allows for derived rewriters /// to optionally hook into the reason why a rewrite failed, and display it to diff --git a/mlir/include/mlir/Transforms/DialectConversion.h b/mlir/include/mlir/Transforms/DialectConversion.h index 84396529eb7c..01fde101ef3c 100644 --- a/mlir/include/mlir/Transforms/DialectConversion.h +++ b/mlir/include/mlir/Transforms/DialectConversion.h @@ -720,12 +720,6 @@ public: /// patterns even if a failure is encountered during the rewrite step. bool canRecoverFromRewriteFailure() const override { return true; } - /// PatternRewriter hook for replacing an operation when the given functor - /// returns "true". - void replaceOpWithIf( - Operation *op, ValueRange newValues, bool *allUsesReplaced, - llvm::unique_function functor) override; - /// PatternRewriter hook for replacing an operation. void replaceOp(Operation *op, ValueRange newValues) override; diff --git a/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp b/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp index 5cd6d4597aff..1658ea67a460 100644 --- a/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp +++ b/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp @@ -370,8 +370,8 @@ DecomposeLinalgOp::matchAndRewrite(GenericOp genericOp, scalarReplacements.push_back( residualGenericOpBody->getArgument(num + origNumInputs)); bool allUsesReplaced = false; - rewriter.replaceOpWithinBlock(peeledScalarOperation, scalarReplacements, - residualGenericOpBody, &allUsesReplaced); + rewriter.replaceUsesWithinBlock(peeledScalarOperation, scalarReplacements, + residualGenericOpBody, &allUsesReplaced); assert(!allUsesReplaced && "peeled scalar operation is erased when it wasnt expected to be"); } diff --git a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp index 43c408a97687..74f6d97aeea5 100644 --- a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp +++ b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp @@ -870,7 +870,7 @@ void offsetIndices(RewriterBase &b, LinalgOp linalgOp, {getAsOpFoldResult(indexOp.getResult()), offsets[indexOp.getDim()]}); Value materialized = getValueOrCreateConstantIndexOp(b, indexOp.getLoc(), applied); - b.replaceOpWithIf(indexOp, materialized, [&](OpOperand &use) { + b.replaceUsesWithIf(indexOp, materialized, [&](OpOperand &use) { return use.getOwner() != materialized.getDefiningOp(); }); } diff --git a/mlir/lib/IR/PatternMatch.cpp b/mlir/lib/IR/PatternMatch.cpp index 8796289d7257..0a88e40f73ec 100644 --- a/mlir/lib/IR/PatternMatch.cpp +++ b/mlir/lib/IR/PatternMatch.cpp @@ -110,41 +110,6 @@ RewriterBase::~RewriterBase() { // Out of line to provide a vtable anchor for the class. } -/// This method replaces the uses of the results of `op` with the values in -/// `newValues` when the provided `functor` returns true for a specific use. -/// The number of values in `newValues` is required to match the number of -/// results of `op`. -void RewriterBase::replaceOpWithIf( - Operation *op, ValueRange newValues, bool *allUsesReplaced, - llvm::unique_function functor) { - assert(op->getNumResults() == newValues.size() && - "incorrect number of values to replace operation"); - - // Notify the listener that we're about to replace this op. - if (auto *rewriteListener = dyn_cast_if_present(listener)) - rewriteListener->notifyOperationReplaced(op, newValues); - - // Replace each use of the results when the functor is true. - bool replacedAllUses = true; - for (auto it : llvm::zip(op->getResults(), newValues)) { - replaceUsesWithIf(std::get<0>(it), std::get<1>(it), functor); - replacedAllUses &= std::get<0>(it).use_empty(); - } - if (allUsesReplaced) - *allUsesReplaced = replacedAllUses; -} - -/// This method replaces the uses of the results of `op` with the values in -/// `newValues` when a use is nested within the given `block`. The number of -/// values in `newValues` is required to match the number of results of `op`. -/// If all uses of this operation are replaced, the operation is erased. -void RewriterBase::replaceOpWithinBlock(Operation *op, ValueRange newValues, - Block *block, bool *allUsesReplaced) { - replaceOpWithIf(op, newValues, allUsesReplaced, [block](OpOperand &use) { - return block->getParentOp()->isProperAncestor(use.getOwner()); - }); -} - /// This method replaces the results of the operation with the specified list of /// values. The number of provided values must match the number of results of /// the operation. The replaced op is erased. @@ -156,9 +121,8 @@ void RewriterBase::replaceOp(Operation *op, ValueRange newValues) { if (auto *rewriteListener = dyn_cast_if_present(listener)) rewriteListener->notifyOperationReplaced(op, newValues); - // Replace results one-by-one. Also notifies the listener of modifications. - for (auto it : llvm::zip(op->getResults(), newValues)) - replaceAllUsesWith(std::get<0>(it), std::get<1>(it)); + // Replace all result uses. Also notifies the listener of modifications. + replaceAllUsesWith(op, newValues); // Erase op and notify listener. eraseOp(op); @@ -176,9 +140,8 @@ void RewriterBase::replaceOp(Operation *op, Operation *newOp) { if (auto *rewriteListener = dyn_cast_if_present(listener)) rewriteListener->notifyOperationReplaced(op, newOp); - // Replace results one-by-one. Also notifies the listener of modifications. - for (auto it : llvm::zip(op->getResults(), newOp->getResults())) - replaceAllUsesWith(std::get<0>(it), std::get<1>(it)); + // Replace all result uses. Also notifies the listener of modifications. + replaceAllUsesWith(op, newOp->getResults()); // Erase op and notify listener. eraseOp(op); @@ -279,15 +242,33 @@ void RewriterBase::finalizeOpModification(Operation *op) { rewriteListener->notifyOperationModified(op); } -/// Find uses of `from` and replace them with `to` if the `functor` returns -/// true. It also marks every modified uses and notifies the rewriter that an -/// in-place operation modification is about to happen. void RewriterBase::replaceUsesWithIf(Value from, Value to, - function_ref functor) { + function_ref functor, + bool *allUsesReplaced) { + bool allReplaced = true; for (OpOperand &operand : llvm::make_early_inc_range(from.getUses())) { - if (functor(operand)) + bool replace = functor(operand); + if (replace) modifyOpInPlace(operand.getOwner(), [&]() { operand.set(to); }); + allReplaced &= replace; } + if (allUsesReplaced) + *allUsesReplaced = allReplaced; +} + +void RewriterBase::replaceUsesWithIf(ValueRange from, ValueRange to, + function_ref functor, + bool *allUsesReplaced) { + assert(from.size() == to.size() && "incorrect number of replacements"); + bool allReplaced = true; + for (auto it : llvm::zip_equal(from, to)) { + bool r; + replaceUsesWithIf(std::get<0>(it), std::get<1>(it), functor, + /*allUsesReplaced=*/&r); + allReplaced &= r; + } + if (allUsesReplaced) + *allUsesReplaced = allReplaced; } void RewriterBase::inlineBlockBefore(Block *source, Block *dest, diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp index 4741110bc606..d7dc902a9a5e 100644 --- a/mlir/lib/Transforms/Utils/DialectConversion.cpp +++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp @@ -1528,19 +1528,6 @@ ConversionPatternRewriter::ConversionPatternRewriter( ConversionPatternRewriter::~ConversionPatternRewriter() = default; -void ConversionPatternRewriter::replaceOpWithIf( - Operation *op, ValueRange newValues, bool *allUsesReplaced, - llvm::unique_function functor) { - // TODO: To support this we will need to rework a bit of how replacements are - // tracked, given that this isn't guranteed to replace all of the uses of an - // operation. The main change is that now an operation can be replaced - // multiple times, in parts. The current "set" based tracking is mainly useful - // for tracking if a replaced operation should be ignored, i.e. if all of the - // uses will be replaced. - llvm_unreachable( - "replaceOpWithIf is currently not supported by DialectConversion"); -} - void ConversionPatternRewriter::replaceOp(Operation *op, Operation *newOp) { assert(op && newOp && "expected non-null op"); replaceOp(op, newOp->getResults()); diff --git a/mlir/lib/Transforms/Utils/RegionUtils.cpp b/mlir/lib/Transforms/Utils/RegionUtils.cpp index e8b07143fc60..eff8acdfb33d 100644 --- a/mlir/lib/Transforms/Utils/RegionUtils.cpp +++ b/mlir/lib/Transforms/Utils/RegionUtils.cpp @@ -161,7 +161,7 @@ SmallVector mlir::makeRegionIsolatedFromAbove( rewriter.setInsertionPointToStart(newEntryBlock); for (auto *clonedOp : clonedOperations) { Operation *newOp = rewriter.clone(*clonedOp, map); - rewriter.replaceOpWithIf(clonedOp, newOp->getResults(), replaceIfFn); + rewriter.replaceUsesWithIf(clonedOp, newOp->getResults(), replaceIfFn); } rewriter.mergeBlocks( entryBlock, newEntryBlock, diff --git a/mlir/test/lib/Dialect/Test/TestPatterns.cpp b/mlir/test/lib/Dialect/Test/TestPatterns.cpp index abc0e43c7b7f..27eae2ffd694 100644 --- a/mlir/test/lib/Dialect/Test/TestPatterns.cpp +++ b/mlir/test/lib/Dialect/Test/TestPatterns.cpp @@ -1836,7 +1836,7 @@ struct TestSelectiveOpReplacementPattern : public OpRewritePattern { OperandRange operands = op.getOperands(); // Replace non-terminator uses with the first operand. - rewriter.replaceOpWithIf(op, operands[0], [](OpOperand &operand) { + rewriter.replaceUsesWithIf(op, operands[0], [](OpOperand &operand) { return operand.getOwner()->hasTrait(); }); // Replace everything else with the second operand if the operation isn't -- GitLab From 84842f4b3ba175d744c03d40baa8e371522615e8 Mon Sep 17 00:00:00 2001 From: Congcong Cai Date: Thu, 7 Mar 2024 09:39:29 +0800 Subject: [PATCH 384/929] [clang-tidy] bugprone-assert-side-effect can detect side effect from non-const reference parameters (#84095) Fixes: #84092 --- .../bugprone/AssertSideEffectCheck.cpp | 20 ++++++++++++---- clang-tools-extra/docs/ReleaseNotes.rst | 4 ++++ .../checkers/bugprone/assert-side-effect.cpp | 24 +++++++++++++++++++ 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/clang-tools-extra/clang-tidy/bugprone/AssertSideEffectCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/AssertSideEffectCheck.cpp index 43bedd4f73ef..c650aae4fa03 100644 --- a/clang-tools-extra/clang-tidy/bugprone/AssertSideEffectCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/AssertSideEffectCheck.cpp @@ -60,16 +60,26 @@ AST_MATCHER_P2(Expr, hasSideEffect, bool, CheckFunctionCalls, } if (const auto *CExpr = dyn_cast(E)) { - bool Result = CheckFunctionCalls; + if (!CheckFunctionCalls) + return false; if (const auto *FuncDecl = CExpr->getDirectCallee()) { if (FuncDecl->getDeclName().isIdentifier() && IgnoredFunctionsMatcher.matches(*FuncDecl, Finder, Builder)) // exceptions come here - Result = false; - else if (const auto *MethodDecl = dyn_cast(FuncDecl)) - Result &= !MethodDecl->isConst(); + return false; + for (size_t I = 0; I < FuncDecl->getNumParams(); I++) { + const ParmVarDecl *P = FuncDecl->getParamDecl(I); + const Expr *ArgExpr = + I < CExpr->getNumArgs() ? CExpr->getArg(I) : nullptr; + const QualType PT = P->getType().getCanonicalType(); + if (ArgExpr && !ArgExpr->isXValue() && PT->isReferenceType() && + !PT.getNonReferenceType().isConstQualified()) + return true; + } + if (const auto *MethodDecl = dyn_cast(FuncDecl)) + return !MethodDecl->isConst(); } - return Result; + return true; } return isa(E) || isa(E) || isa(E); diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst index 1b839a35c3ed..d98c4ff9a750 100644 --- a/clang-tools-extra/docs/ReleaseNotes.rst +++ b/clang-tools-extra/docs/ReleaseNotes.rst @@ -128,6 +128,10 @@ New check aliases Changes in existing checks ^^^^^^^^^^^^^^^^^^^^^^^^^^ +- Improved :doc:`bugprone-assert-side-effect + ` check by detecting side + effect from calling a method with non-const reference parameters. + - Improved :doc:`bugprone-non-zero-enum-to-bool-conversion ` check by eliminating false positives resulting from direct usage of bitwise operators diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/assert-side-effect.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/assert-side-effect.cpp index c11638aa823a..5cdc1afb3d90 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/assert-side-effect.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/assert-side-effect.cpp @@ -108,3 +108,27 @@ int main() { return 0; } + +namespace parameter_anaylysis { + +struct S { + bool value(int) const; + bool leftValueRef(int &) const; + bool constRef(int const &) const; + bool rightValueRef(int &&) const; +}; + +void foo() { + S s{}; + int i = 0; + assert(s.value(0)); + assert(s.value(i)); + assert(s.leftValueRef(i)); + // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: side effect in assert() condition discarded in release builds + assert(s.constRef(0)); + assert(s.constRef(i)); + assert(s.rightValueRef(0)); + assert(s.rightValueRef(static_cast(i))); +} + +} // namespace parameter_anaylysis -- GitLab From 9fe5aa31eccff49632e40bcdad6e64fc00190889 Mon Sep 17 00:00:00 2001 From: Younan Zhang Date: Thu, 7 Mar 2024 09:49:02 +0800 Subject: [PATCH 385/929] [clang][Sema] Skip the RequiresExprBodyDecls for lambda dependencies (#83997) The dependency of a lambda inside of a `RequiresExprBodyDecl` was previously affected by its parent, e.g., `ClassTemplateSpecializationDecl`. This made the lambda always dependent regardless of the template arguments we had, which caused some crashes on the constraint evaluation later. This fixes https://github.com/llvm/llvm-project/issues/56556, fixes https://github.com/llvm/llvm-project/issues/82849 and a case demonstrated by https://github.com/llvm/llvm-project/issues/49570#issuecomment-1664966972. --- clang/docs/ReleaseNotes.rst | 2 + clang/lib/Sema/TreeTransform.h | 23 +++++++- clang/test/SemaTemplate/concepts-lambda.cpp | 58 +++++++++++++++++++++ 3 files changed, 81 insertions(+), 2 deletions(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 942820a52685..8300a8484585 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -319,6 +319,8 @@ Bug Fixes to C++ Support Fixes (#GH80630) - Fix a crash when an explicit template argument list is used with a name for which lookup finds a non-template function and a dependent using declarator. +- Fixed an issue where the ``RequiresExprBody`` was involved in the lambda dependency + calculation. (#GH56556), (#GH82849). - Fix a bug where overload resolution falsely reported an ambiguity when it was comparing a member-function against a non member function or a member-function with an explicit object parameter against a member function with no explicit object parameter diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h index 7389a48fe56f..409aee73d960 100644 --- a/clang/lib/Sema/TreeTransform.h +++ b/clang/lib/Sema/TreeTransform.h @@ -13649,10 +13649,29 @@ TreeTransform::TransformLambdaExpr(LambdaExpr *E) { // use evaluation contexts to distinguish the function parameter case. CXXRecordDecl::LambdaDependencyKind DependencyKind = CXXRecordDecl::LDK_Unknown; + DeclContext *DC = getSema().CurContext; + // A RequiresExprBodyDecl is not interesting for dependencies. + // For the following case, + // + // template + // concept C = requires { [] {}; }; + // + // template + // struct Widget; + // + // template + // struct Widget {}; + // + // While we are substituting Widget, the parent of DC would be + // the template specialization itself. Thus, the lambda expression + // will be deemed as dependent even if there are no dependent template + // arguments. + // (A ClassTemplateSpecializationDecl is always a dependent context.) + while (DC->getDeclKind() == Decl::Kind::RequiresExprBody) + DC = DC->getParent(); if ((getSema().isUnevaluatedContext() || getSema().isConstantEvaluatedContext()) && - (getSema().CurContext->isFileContext() || - !getSema().CurContext->getParent()->isDependentContext())) + (DC->isFileContext() || !DC->getParent()->isDependentContext())) DependencyKind = CXXRecordDecl::LDK_NeverDependent; CXXRecordDecl *OldClass = E->getLambdaClass(); diff --git a/clang/test/SemaTemplate/concepts-lambda.cpp b/clang/test/SemaTemplate/concepts-lambda.cpp index 0b7580f91043..fac790d09f9c 100644 --- a/clang/test/SemaTemplate/concepts-lambda.cpp +++ b/clang/test/SemaTemplate/concepts-lambda.cpp @@ -90,6 +90,64 @@ struct Foo { static_assert(ConstructibleWithN); +namespace GH56556 { + +template +inline constexpr It declare (); + +template typename Template> +concept D = requires { + { [] (Template &) {}(declare()) }; +}; + +template +struct B {}; + +template +struct Adapter; + +template T> +struct Adapter {}; + +template struct Adapter>; + +} // namespace GH56556 + +namespace GH82849 { + +template +concept C = requires(T t) { + requires requires (T u) { + [](V) { + return requires(V v) { + [](V w) {}(v); + }; + }(t); + }; +}; + +template +struct Widget; + +template +struct Widget { + static F create(F from) { + return from; + } +}; + +template +bool foo() { + return C; +} + +void bar() { + // https://github.com/llvm/llvm-project/issues/49570#issuecomment-1664966972 + Widget::create(0); +} + +} // namespace GH82849 + } // GH60642 reported an assert being hit, make sure we don't assert. -- GitLab From 4f1258fd913a65338c8fbf5cf38163b4e026964a Mon Sep 17 00:00:00 2001 From: Phoebe Wang Date: Thu, 7 Mar 2024 09:52:40 +0800 Subject: [PATCH 386/929] [X86] Change tuning of Sierraforest and Grandridge to more latest target (#84179) --- llvm/lib/Target/X86/X86.td | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td index e89ddcc570c9..a2a65ce75d6b 100644 --- a/llvm/lib/Target/X86/X86.td +++ b/llvm/lib/Target/X86/X86.td @@ -1701,10 +1701,6 @@ foreach P = ["goldmont_plus", "goldmont-plus"] in { } def : ProcModel<"tremont", SLMModel, ProcessorFeatures.TRMFeatures, ProcessorFeatures.TRMTuning>; -foreach P = ["sierraforest", "grandridge"] in { - def : ProcModel; -} // "Arrandale" along with corei3 and corei5 foreach P = ["nehalem", "corei7", "core_i7_sse4_2"] in { @@ -1779,6 +1775,10 @@ def : ProcModel<"alderlake", AlderlakePModel, // FIXME: Use Gracemont Schedule Model when it is ready. def : ProcModel<"gracemont", AlderlakePModel, ProcessorFeatures.ADLFeatures, ProcessorFeatures.GRTTuning>; +foreach P = ["sierraforest", "grandridge"] in { + def : ProcModel; +} def : ProcModel<"raptorlake", AlderlakePModel, ProcessorFeatures.ADLFeatures, ProcessorFeatures.ADLTuning>; def : ProcModel<"meteorlake", AlderlakePModel, -- GitLab From 416debf79b90256b794efb583696dbc085f3f050 Mon Sep 17 00:00:00 2001 From: Cameron McInally Date: Wed, 6 Mar 2024 21:25:28 -0500 Subject: [PATCH 387/929] [test] Move pr73894.ll to AArch64 directory and update the target triple (#84269) pr73894.ll is failing on a number of non-AArch64 buildbots. I'm not certain that this is a proper fix, but I think it's best to move the test to the test/Transforms/LoopVectorize/AArch64/ directory and replace the triple with one commonly used in that directory. llvm#73894 --- llvm/test/Transforms/LoopVectorize/{ => AArch64}/pr73894.ll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename llvm/test/Transforms/LoopVectorize/{ => AArch64}/pr73894.ll (99%) diff --git a/llvm/test/Transforms/LoopVectorize/pr73894.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll similarity index 99% rename from llvm/test/Transforms/LoopVectorize/pr73894.ll rename to llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll index 4203a8eb8df6..1970ac966535 100644 --- a/llvm/test/Transforms/LoopVectorize/pr73894.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 ; RUN: opt -passes=loop-vectorize -mcpu=neoverse-v1 -force-vector-interleave=2 -force-vector-width=1 -S %s | FileCheck %s -target triple = "arm64-linux" +target triple = "aarch64-unknown-linux-gnu" define i32 @pr70988() { ; CHECK-LABEL: define i32 @pr70988( -- GitLab From ec72909b62ce568ed27c6ad0581e50c0fdd70df4 Mon Sep 17 00:00:00 2001 From: Jordan Rupprecht Date: Wed, 6 Mar 2024 18:41:18 -0800 Subject: [PATCH 388/929] [lldb][test] iwyu for vfork test --- lldb/test/API/functionalities/fork/concurrent_vfork/main.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/lldb/test/API/functionalities/fork/concurrent_vfork/main.cpp b/lldb/test/API/functionalities/fork/concurrent_vfork/main.cpp index b0a4446ba015..2f3a95dc5c6e 100644 --- a/lldb/test/API/functionalities/fork/concurrent_vfork/main.cpp +++ b/lldb/test/API/functionalities/fork/concurrent_vfork/main.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include -- GitLab From 51e23545e79957af43be9be0e6c31b77a4665279 Mon Sep 17 00:00:00 2001 From: mmilanifard Date: Wed, 6 Mar 2024 18:43:22 -0800 Subject: [PATCH 389/929] Revert "Adding missing dependencies to BUILD.bazel" (#84251) Reverts llvm/llvm-project#84235 With changes in #84238 these dependencies are not required. --- utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 3 --- 1 file changed, 3 deletions(-) diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 0c49ced55154..86b38ebd2217 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -6182,9 +6182,6 @@ cc_library( ":NVVMToLLVMIRTranslation", ":TargetLLVM", ":ToLLVMIRTranslation", - "@cuda//:cuda_headers", - "@cuda//:cusparse_static", - "@cuda//:libcuda", ":config", "//llvm:NVPTXCodeGen", "//llvm:Support", -- GitLab From 0083c3eb83bb984fcc9bfbf1b670df8ea3f1fe97 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Wed, 6 Mar 2024 19:37:49 -0800 Subject: [PATCH 390/929] Revert "[AMDGPU] Add AMDGPU specific variadic operation MCExprs" (#84273) Reverts llvm/llvm-project#82022 Fails on hwasan build bot: https://lab.llvm.org/buildbot/#/builders/236/builds/9874/steps/10/logs/stdio --- llvm/docs/AMDGPUUsage.rst | 19 --- .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp | 55 -------- .../AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp | 94 ------------- .../Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h | 74 ---------- .../Target/AMDGPU/MCTargetDesc/CMakeLists.txt | 1 - llvm/test/MC/AMDGPU/mcexpr_amd.s | 130 ------------------ llvm/test/MC/AMDGPU/mcexpr_amd_err.s | 53 ------- 7 files changed, 426 deletions(-) delete mode 100644 llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp delete mode 100644 llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h delete mode 100644 llvm/test/MC/AMDGPU/mcexpr_amd.s delete mode 100644 llvm/test/MC/AMDGPU/mcexpr_amd_err.s diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst index c7cb06ffbef1..7f39f69cae60 100644 --- a/llvm/docs/AMDGPUUsage.rst +++ b/llvm/docs/AMDGPUUsage.rst @@ -1534,25 +1534,6 @@ The AMDGPU backend supports the following calling conventions: =============================== ========================================================== -AMDGPU MCExpr -------------- - -As part of the AMDGPU MC layer, AMDGPU provides the following target specific -``MCExpr``\s. - - .. table:: AMDGPU MCExpr types: - :name: amdgpu-mcexpr-table - - =================== ================= ======================================================== - MCExpr Operands Return value - =================== ================= ======================================================== - ``max(arg, ...)`` 1 or more Variadic signed operation that returns the maximum - value of all its arguments. - - ``or(arg, ...)`` 1 or more Variadic signed operation that returns the bitwise-or - result of all its arguments. - - =================== ================= ======================================================== .. _amdgpu-elf-code-object: diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp index 16a5d6879ce7..cb4eddfe5320 100644 --- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp +++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "AMDKernelCodeT.h" -#include "MCTargetDesc/AMDGPUMCExpr.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "MCTargetDesc/AMDGPUTargetStreamer.h" #include "SIDefines.h" @@ -1817,7 +1816,6 @@ private: public: void onBeginOfFile() override; - bool parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) override; ParseStatus parseCustomOperand(OperandVector &Operands, unsigned MCK); @@ -8279,59 +8277,6 @@ void AMDGPUAsmParser::onBeginOfFile() { getTargetStreamer().EmitDirectiveAMDGCNTarget(); } -/// Parse AMDGPU specific expressions. -/// -/// expr ::= or(expr, ...) | -/// max(expr, ...) -/// -bool AMDGPUAsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) { - using AGVK = AMDGPUVariadicMCExpr::VariadicKind; - - if (isToken(AsmToken::Identifier)) { - StringRef TokenId = getTokenStr(); - AGVK VK = StringSwitch(TokenId) - .Case("max", AGVK::AGVK_Max) - .Case("or", AGVK::AGVK_Or) - .Default(AGVK::AGVK_None); - - if (VK != AGVK::AGVK_None && peekToken().is(AsmToken::LParen)) { - SmallVector Exprs; - uint64_t CommaCount = 0; - lex(); // Eat 'max'/'or' - lex(); // Eat '(' - while (true) { - if (trySkipToken(AsmToken::RParen)) { - if (Exprs.empty()) { - Error(getToken().getLoc(), - "empty " + Twine(TokenId) + " expression"); - return true; - } - if (CommaCount + 1 != Exprs.size()) { - Error(getToken().getLoc(), - "mismatch of commas in " + Twine(TokenId) + " expression"); - return true; - } - Res = AMDGPUVariadicMCExpr::create(VK, Exprs, getContext()); - return false; - } - const MCExpr *Expr; - if (getParser().parseExpression(Expr, EndLoc)) - return true; - Exprs.push_back(Expr); - bool LastTokenWasComma = trySkipToken(AsmToken::Comma); - if (LastTokenWasComma) - CommaCount++; - if (!LastTokenWasComma && !isToken(AsmToken::RParen)) { - Error(getToken().getLoc(), - "unexpected token in " + Twine(TokenId) + " expression"); - return true; - } - } - } - } - return getParser().parsePrimaryExpr(Res, EndLoc, nullptr); -} - ParseStatus AMDGPUAsmParser::parseOModSI(OperandVector &Operands) { StringRef Name = getTokenStr(); if (Name == "mul") { diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp deleted file mode 100644 index 0659937e06a2..000000000000 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp +++ /dev/null @@ -1,94 +0,0 @@ -//===- AMDGPUMCExpr.cpp - AMDGPU specific MC expression classes -----------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "AMDGPUMCExpr.h" -#include "llvm/MC/MCContext.h" -#include "llvm/MC/MCStreamer.h" -#include "llvm/MC/MCSymbol.h" -#include "llvm/MC/MCValue.h" -#include "llvm/Support/Allocator.h" -#include "llvm/Support/raw_ostream.h" -#include - -using namespace llvm; - -const AMDGPUVariadicMCExpr * -AMDGPUVariadicMCExpr::create(VariadicKind Kind, ArrayRef Args, - MCContext &Ctx) { - return new (Ctx) AMDGPUVariadicMCExpr(Kind, Args); -} - -const MCExpr *AMDGPUVariadicMCExpr::getSubExpr(size_t Index) const { - assert(Index < Args.size() && - "Indexing out of bounds AMDGPUVariadicMCExpr sub-expr"); - return Args[Index]; -} - -void AMDGPUVariadicMCExpr::printImpl(raw_ostream &OS, - const MCAsmInfo *MAI) const { - switch (Kind) { - default: - llvm_unreachable("Unknown AMDGPUVariadicMCExpr kind."); - case AGVK_Or: - OS << "or("; - break; - case AGVK_Max: - OS << "max("; - break; - } - for (auto It = Args.begin(); It != Args.end(); ++It) { - (*It)->print(OS, MAI, /*InParens=*/false); - if ((It + 1) != Args.end()) - OS << ", "; - } - OS << ')'; -} - -static int64_t op(AMDGPUVariadicMCExpr::VariadicKind Kind, int64_t Arg1, - int64_t Arg2) { - switch (Kind) { - default: - llvm_unreachable("Unknown AMDGPUVariadicMCExpr kind."); - case AMDGPUVariadicMCExpr::AGVK_Max: - return std::max(Arg1, Arg2); - case AMDGPUVariadicMCExpr::AGVK_Or: - return Arg1 | Arg2; - } -} - -bool AMDGPUVariadicMCExpr::evaluateAsRelocatableImpl( - MCValue &Res, const MCAsmLayout *Layout, const MCFixup *Fixup) const { - std::optional Total; - - for (const MCExpr *Arg : Args) { - MCValue ArgRes; - if (!Arg->evaluateAsRelocatable(ArgRes, Layout, Fixup) || - !ArgRes.isAbsolute()) - return false; - - if (!Total.has_value()) - Total = ArgRes.getConstant(); - Total = op(Kind, *Total, ArgRes.getConstant()); - } - - Res = MCValue::get(*Total); - return true; -} - -void AMDGPUVariadicMCExpr::visitUsedExpr(MCStreamer &Streamer) const { - for (const MCExpr *Arg : Args) - Streamer.visitUsedExpr(*Arg); -} - -MCFragment *AMDGPUVariadicMCExpr::findAssociatedFragment() const { - for (const MCExpr *Arg : Args) { - if (Arg->findAssociatedFragment()) - return Arg->findAssociatedFragment(); - } - return nullptr; -} diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h deleted file mode 100644 index 9e8452f2c624..000000000000 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h +++ /dev/null @@ -1,74 +0,0 @@ -//===- AMDGPUMCExpr.h - AMDGPU specific MC expression classes ---*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUMCEXPR_H -#define LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUMCEXPR_H - -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/MC/MCExpr.h" - -namespace llvm { - -/// AMDGPU target specific variadic MCExpr operations. -/// -/// Takes in a minimum of 1 argument to be used with an operation. The supported -/// operations are: -/// - (bitwise) or -/// - max -/// -/// \note If the 'or'/'max' operations are provided only a single argument, the -/// operation will act as a no-op and simply resolve as the provided argument. -/// -class AMDGPUVariadicMCExpr : public MCTargetExpr { -public: - enum VariadicKind { AGVK_None, AGVK_Or, AGVK_Max }; - -private: - VariadicKind Kind; - SmallVector Args; - - AMDGPUVariadicMCExpr(VariadicKind Kind, ArrayRef Args) - : Kind(Kind), Args(Args) { - assert(Args.size() >= 1 && "Needs a minimum of one expression."); - assert(Kind != AGVK_None && - "Cannot construct AMDGPUVariadicMCExpr of kind none."); - } - -public: - static const AMDGPUVariadicMCExpr * - create(VariadicKind Kind, ArrayRef Args, MCContext &Ctx); - - static const AMDGPUVariadicMCExpr *createOr(ArrayRef Args, - MCContext &Ctx) { - return create(VariadicKind::AGVK_Or, Args, Ctx); - } - - static const AMDGPUVariadicMCExpr *createMax(ArrayRef Args, - MCContext &Ctx) { - return create(VariadicKind::AGVK_Max, Args, Ctx); - } - - VariadicKind getKind() const { return Kind; } - const MCExpr *getSubExpr(size_t Index) const; - - void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override; - bool evaluateAsRelocatableImpl(MCValue &Res, const MCAsmLayout *Layout, - const MCFixup *Fixup) const override; - void visitUsedExpr(MCStreamer &Streamer) const override; - MCFragment *findAssociatedFragment() const override; - void fixELFSymbolsInTLSFixups(MCAssembler &) const override{}; - - static bool classof(const MCExpr *E) { - return E->getKind() == MCExpr::Target; - } -}; - -} // end namespace llvm - -#endif // LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUMCEXPR_H diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt index 0842a58f794b..5dc76071b059 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt @@ -5,7 +5,6 @@ add_llvm_component_library(LLVMAMDGPUDesc AMDGPUInstPrinter.cpp AMDGPUMCAsmInfo.cpp AMDGPUMCCodeEmitter.cpp - AMDGPUMCExpr.cpp AMDGPUMCTargetDesc.cpp AMDGPUTargetStreamer.cpp R600InstPrinter.cpp diff --git a/llvm/test/MC/AMDGPU/mcexpr_amd.s b/llvm/test/MC/AMDGPU/mcexpr_amd.s deleted file mode 100644 index a9639c3acc30..000000000000 --- a/llvm/test/MC/AMDGPU/mcexpr_amd.s +++ /dev/null @@ -1,130 +0,0 @@ -// RUN: llvm-mc -triple amdgcn-amd-amdhsa < %s | FileCheck --check-prefix=ASM %s -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -filetype=obj < %s > %t -// RUN: llvm-objdump --syms %t | FileCheck --check-prefix=OBJDUMP %s - -// OBJDUMP: SYMBOL TABLE: -// OBJDUMP-NEXT: 0000000000000000 l *ABS* 0000000000000000 zero -// OBJDUMP-NEXT: 0000000000000001 l *ABS* 0000000000000000 one -// OBJDUMP-NEXT: 0000000000000002 l *ABS* 0000000000000000 two -// OBJDUMP-NEXT: 0000000000000003 l *ABS* 0000000000000000 three -// OBJDUMP-NEXT: 7fffffffffffffff l *ABS* 0000000000000000 i64_max -// OBJDUMP-NEXT: 8000000000000000 l *ABS* 0000000000000000 i64_min -// OBJDUMP-NEXT: 0000000000000005 l *ABS* 0000000000000000 max_expression_all -// OBJDUMP-NEXT: 0000000000000005 l *ABS* 0000000000000000 five -// OBJDUMP-NEXT: 0000000000000004 l *ABS* 0000000000000000 four -// OBJDUMP-NEXT: 0000000000000002 l *ABS* 0000000000000000 max_expression_two -// OBJDUMP-NEXT: 0000000000000001 l *ABS* 0000000000000000 max_expression_one -// OBJDUMP-NEXT: 000000000000000a l *ABS* 0000000000000000 max_literals -// OBJDUMP-NEXT: 000000000000000f l *ABS* 0000000000000000 max_with_max_sym -// OBJDUMP-NEXT: 000000000000000f l *ABS* 0000000000000000 max -// OBJDUMP-NEXT: ffffffffffffffff l *ABS* 0000000000000000 neg_one -// OBJDUMP-NEXT: ffffffffffffffff l *ABS* 0000000000000000 max_neg_numbers -// OBJDUMP-NEXT: ffffffffffffffff l *ABS* 0000000000000000 max_neg_number -// OBJDUMP-NEXT: 0000000000000003 l *ABS* 0000000000000000 max_with_subexpr -// OBJDUMP-NEXT: 0000000000000006 l *ABS* 0000000000000000 max_as_subexpr -// OBJDUMP-NEXT: 0000000000000005 l *ABS* 0000000000000000 max_recursive_subexpr -// OBJDUMP-NEXT: 7fffffffffffffff l *ABS* 0000000000000000 max_expr_one_max -// OBJDUMP-NEXT: 7fffffffffffffff l *ABS* 0000000000000000 max_expr_two_max -// OBJDUMP-NEXT: 7fffffffffffffff l *ABS* 0000000000000000 max_expr_three_max -// OBJDUMP-NEXT: 8000000000000000 l *ABS* 0000000000000000 max_expr_one_min -// OBJDUMP-NEXT: 0000000000000003 l *ABS* 0000000000000000 max_expr_two_min -// OBJDUMP-NEXT: 0000000000989680 l *ABS* 0000000000000000 max_expr_three_min -// OBJDUMP-NEXT: 0000000000000007 l *ABS* 0000000000000000 or_expression_all -// OBJDUMP-NEXT: 0000000000000003 l *ABS* 0000000000000000 or_expression_two -// OBJDUMP-NEXT: 0000000000000001 l *ABS* 0000000000000000 or_expression_one -// OBJDUMP-NEXT: 000000000000000f l *ABS* 0000000000000000 or_literals -// OBJDUMP-NEXT: 0000000000000000 l *ABS* 0000000000000000 or_false -// OBJDUMP-NEXT: 00000000000000ff l *ABS* 0000000000000000 or_with_or_sym -// OBJDUMP-NEXT: 00000000000000ff l *ABS* 0000000000000000 or -// OBJDUMP-NEXT: 0000000000000003 l *ABS* 0000000000000000 or_with_subexpr -// OBJDUMP-NEXT: 0000000000000008 l *ABS* 0000000000000000 or_as_subexpr -// OBJDUMP-NEXT: 0000000000000007 l *ABS* 0000000000000000 or_recursive_subexpr - -// ASM: .set zero, 0 -// ASM: .set one, 1 -// ASM: .set two, 2 -// ASM: .set three, 3 -// ASM: .set i64_max, 9223372036854775807 -// ASM: .set i64_min, -9223372036854775808 - -.set zero, 0 -.set one, 1 -.set two, 2 -.set three, 3 -.set i64_max, 0x7FFFFFFFFFFFFFFF -.set i64_min, 0x8000000000000000 - -// ASM: .set max_expression_all, max(1, 2, five, 3, four) -// ASM: .set max_expression_two, 2 -// ASM: .set max_expression_one, 1 -// ASM: .set max_literals, 10 -// ASM: .set max_with_max_sym, max(max, 4, 3, 1, 2) - -.set max_expression_all, max(one, two, five, three, four) -.set max_expression_two, max(one, two) -.set max_expression_one, max(one) -.set max_literals, max(1,2,3,4,5,6,7,8,9,10) -.set max_with_max_sym, max(max, 4, 3, one, two) - -// ASM: .set max_neg_numbers, -1 -// ASM: .set max_neg_number, -1 - -.set neg_one, -1 -.set max_neg_numbers, max(-5, -4, -3, -2, neg_one) -.set max_neg_number, max(neg_one) - -// ASM: .set max_with_subexpr, 3 -// ASM: .set max_as_subexpr, 1+(max(4, 3, five)) -// ASM: .set max_recursive_subexpr, max(max(1, four), 3, max_expression_all) - -.set max_with_subexpr, max(((one | 3) << 3) / 8) -.set max_as_subexpr, 1 + max(4, 3, five) -.set max_recursive_subexpr, max(max(one, four), three, max_expression_all) - -// ASM: .set max_expr_one_max, 9223372036854775807 -// ASM: .set max_expr_two_max, max(9223372036854775807, five) -// ASM: .set max_expr_three_max, max(9223372036854775807, five, 10000000) - -.set max_expr_one_max, max(i64_max) -.set max_expr_two_max, max(i64_max, five) -.set max_expr_three_max, max(i64_max, five, 10000000) - -// ASM: .set max_expr_one_min, -9223372036854775808 -// ASM: .set max_expr_two_min, 3 -// ASM: .set max_expr_three_min, 10000000 - -.set max_expr_one_min, max(i64_min) -.set max_expr_two_min, max(i64_min, three) -.set max_expr_three_min, max(i64_min, three, 10000000) - -// ASM: .set or_expression_all, or(1, 2, five, 3, four) -// ASM: .set or_expression_two, 3 -// ASM: .set or_expression_one, 1 -// ASM: .set or_literals, 15 -// ASM: .set or_false, 0 -// ASM: .set or_with_or_sym, or(or, 4, 3, 1, 2) - -.set or_expression_all, or(one, two, five, three, four) -.set or_expression_two, or(one, two) -.set or_expression_one, or(one) -.set or_literals, or(1,2,3,4,5,6,7,8,9,10) -.set or_false, or(zero, 0, (2-2), 5 > 6) -.set or_with_or_sym, or(or, 4, 3, one, two) - -// ASM: .set or_with_subexpr, 3 -// ASM: .set or_as_subexpr, 1+(or(4, 3, five)) -// ASM: .set or_recursive_subexpr, or(or(1, four), 3, or_expression_all) - -.set or_with_subexpr, or(((one | 3) << 3) / 8) -.set or_as_subexpr, 1 + or(4, 3, five) -.set or_recursive_subexpr, or(or(one, four), three, or_expression_all) - -// ASM: .set four, 4 -// ASM: .set five, 5 -// ASM: .set max, 15 -// ASM: .set or, 255 - -.set four, 4 -.set five, 5 -.set max, 0xF -.set or, 0xFF diff --git a/llvm/test/MC/AMDGPU/mcexpr_amd_err.s b/llvm/test/MC/AMDGPU/mcexpr_amd_err.s deleted file mode 100644 index ea02e0136272..000000000000 --- a/llvm/test/MC/AMDGPU/mcexpr_amd_err.s +++ /dev/null @@ -1,53 +0,0 @@ -// RUN: not llvm-mc -triple amdgcn-amd-amdhsa %s 2>&1 | FileCheck --check-prefix=ASM %s - -.set one, 1 -.set two, 2 -.set three, 3 - -.set max_empty, max() -// ASM: :[[@LINE-1]]:{{[0-9]+}}: error: empty max expression -// ASM: :[[@LINE-2]]:{{[0-9]+}}: error: missing expression - -.set or_empty, or() -// ASM: :[[@LINE-1]]:{{[0-9]+}}: error: empty or expression -// ASM: :[[@LINE-2]]:{{[0-9]+}}: error: missing expression - -.set max_post_aux_comma, max(one,) -// ASM: :[[@LINE-1]]:{{[0-9]+}}: error: mismatch of commas in max expression -// ASM: :[[@LINE-2]]:{{[0-9]+}}: error: missing expression - -.set max_pre_aux_comma, max(,one) -// asm: :[[@line-1]]:{{[0-9]+}}: error: unknown token in expression -// ASM: :[[@LINE-2]]:{{[0-9]+}}: error: missing expression - -.set max_double_comma, max(one,, two) -// ASM: :[[@LINE-1]]:{{[0-9]+}}: error: unknown token in expression -// ASM: :[[@LINE-2]]:{{[0-9]+}}: error: missing expression - -.set max_no_comma, max(one two) -// ASM: :[[@LINE-1]]:{{[0-9]+}}: error: unexpected token in max expression -// ASM: :[[@LINE-2]]:{{[0-9]+}}: error: missing expression - -.set max_missing_paren, max(two -// ASM: :[[@LINE-1]]:{{[0-9]+}}: error: unexpected token in max expression -// ASM: :[[@LINE-2]]:{{[0-9]+}}: error: missing expression - -.set max_expression_one, max(three, four, -// ASM: :[[@LINE-1]]:{{[0-9]+}}: error: unknown token in expression -// ASM: :[[@LINE-2]]:{{[0-9]+}}: error: missing expression - -.set or_expression_one, or(four, five -// ASM: :[[@LINE-1]]:{{[0-9]+}}: error: unexpected token in or expression -// ASM: :[[@LINE-2]]:{{[0-9]+}}: error: missing expression - -.set max_no_lparen, max four, five) -// ASM: :[[@LINE-1]]:{{[0-9]+}}: error: expected newline - -.set max_no_paren, max one, two, three -// ASM: :[[@LINE-1]]:{{[0-9]+}}: error: expected newline - -.set max_rparen_only, max) -// ASM: :[[@LINE-1]]:{{[0-9]+}}: error: expected newline - -.set four, 4 -.set five, 5 -- GitLab From 38a73fb1575f7d71f481177595ef66723d1ca0b3 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Thu, 7 Mar 2024 03:38:16 +0000 Subject: [PATCH 391/929] [gn build] Port 0083c3eb83bb --- .../gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn | 1 - 1 file changed, 1 deletion(-) diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn index 12d875cf40c9..3d8e3e66d26d 100644 --- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn @@ -103,7 +103,6 @@ static_library("MCTargetDesc") { "AMDGPUInstPrinter.cpp", "AMDGPUMCAsmInfo.cpp", "AMDGPUMCCodeEmitter.cpp", - "AMDGPUMCExpr.cpp", "AMDGPUMCTargetDesc.cpp", "AMDGPUTargetStreamer.cpp", "R600InstPrinter.cpp", -- GitLab From c7fbbec86c483019e9340ee4573d6d8b4f878f72 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Wed, 6 Mar 2024 20:40:36 -0800 Subject: [PATCH 392/929] [lldb] Remove unused #includes in ClangModulesDeclVendor.cpp (#84262) --- .../ExpressionParser/Clang/ClangModulesDeclVendor.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp index 5ce0d3537823..cc5250ca0f78 100644 --- a/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp +++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp @@ -8,7 +8,6 @@ #include "clang/Basic/Diagnostic.h" #include "clang/Basic/DiagnosticFrontend.h" -#include "clang/Basic/DiagnosticSerialization.h" #include "clang/Basic/TargetInfo.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/FrontendActions.h" @@ -19,20 +18,15 @@ #include "clang/Sema/Lookup.h" #include "clang/Serialization/ASTReader.h" #include "llvm/ADT/StringRef.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/FormatVariadic.h" #include "llvm/Support/Path.h" #include "llvm/Support/Threading.h" #include "ClangHost.h" #include "ClangModulesDeclVendor.h" -#include "ModuleDependencyCollector.h" #include "Plugins/TypeSystem/Clang/TypeSystemClang.h" #include "lldb/Core/ModuleList.h" #include "lldb/Core/Progress.h" -#include "lldb/Host/Host.h" -#include "lldb/Host/HostInfo.h" #include "lldb/Symbol/CompileUnit.h" #include "lldb/Symbol/SourceModule.h" #include "lldb/Target/Target.h" @@ -40,10 +34,8 @@ #include "lldb/Utility/LLDBAssert.h" #include "lldb/Utility/LLDBLog.h" #include "lldb/Utility/Log.h" -#include "lldb/Utility/StreamString.h" #include -#include using namespace lldb_private; -- GitLab From ca7492fae42c5a21db96eccaca7a5e827ab1bf65 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Wed, 6 Mar 2024 20:41:46 -0800 Subject: [PATCH 393/929] [lldb] Extract getter function for experimental target properties (NFC) (#83504) In Swift's downstream lldb, there are a number of experimental properties. This change extracts a getter function containing the common logic for getting a boolean valued experimental property. This also deletes `SetInjectLocalVariables` which isn't used anywhere. --- lldb/include/lldb/Target/Target.h | 6 ++++-- lldb/source/Target/Target.cpp | 24 +++++++++--------------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/lldb/include/lldb/Target/Target.h b/lldb/include/lldb/Target/Target.h index 8f57358981d4..2c2e6b2831cc 100644 --- a/lldb/include/lldb/Target/Target.h +++ b/lldb/include/lldb/Target/Target.h @@ -244,8 +244,6 @@ public: bool GetInjectLocalVariables(ExecutionContext *exe_ctx) const; - void SetInjectLocalVariables(ExecutionContext *exe_ctx, bool b); - void SetRequireHardwareBreakpoints(bool b); bool GetRequireHardwareBreakpoints() const; @@ -259,6 +257,10 @@ public: bool GetDebugUtilityExpression() const; private: + std::optional + GetExperimentalPropertyValue(size_t prop_idx, + ExecutionContext *exe_ctx = nullptr) const; + // Callbacks for m_launch_info. void Arg0ValueChangedCallback(); void RunArgsValueChangedCallback(); diff --git a/lldb/source/Target/Target.cpp b/lldb/source/Target/Target.cpp index e982a30a3ae4..09b0ac42631d 100644 --- a/lldb/source/Target/Target.cpp +++ b/lldb/source/Target/Target.cpp @@ -43,6 +43,7 @@ #include "lldb/Symbol/ObjectFile.h" #include "lldb/Symbol/Symbol.h" #include "lldb/Target/ABI.h" +#include "lldb/Target/ExecutionContext.h" #include "lldb/Target/Language.h" #include "lldb/Target/LanguageRuntime.h" #include "lldb/Target/Process.h" @@ -4227,28 +4228,21 @@ void TargetProperties::UpdateLaunchInfoFromProperties() { DisableSTDIOValueChangedCallback(); } -bool TargetProperties::GetInjectLocalVariables( - ExecutionContext *exe_ctx) const { +std::optional TargetProperties::GetExperimentalPropertyValue( + size_t prop_idx, ExecutionContext *exe_ctx) const { const Property *exp_property = m_collection_sp->GetPropertyAtIndex(ePropertyExperimental, exe_ctx); OptionValueProperties *exp_values = exp_property->GetValue()->GetAsProperties(); if (exp_values) - return exp_values - ->GetPropertyAtIndexAs(ePropertyInjectLocalVars, exe_ctx) - .value_or(true); - else - return true; + return exp_values->GetPropertyAtIndexAs(prop_idx, exe_ctx); + return std::nullopt; } -void TargetProperties::SetInjectLocalVariables(ExecutionContext *exe_ctx, - bool b) { - const Property *exp_property = - m_collection_sp->GetPropertyAtIndex(ePropertyExperimental, exe_ctx); - OptionValueProperties *exp_values = - exp_property->GetValue()->GetAsProperties(); - if (exp_values) - exp_values->SetPropertyAtIndex(ePropertyInjectLocalVars, true, exe_ctx); +bool TargetProperties::GetInjectLocalVariables( + ExecutionContext *exe_ctx) const { + return GetExperimentalPropertyValue(ePropertyInjectLocalVars, exe_ctx) + .value_or(true); } ArchSpec TargetProperties::GetDefaultArchitecture() const { -- GitLab From 2fcf248ec2614dd966f31d471a1a97989a76fd9d Mon Sep 17 00:00:00 2001 From: Connor Sughrue <55301806+cpsughrue@users.noreply.github.com> Date: Wed, 6 Mar 2024 20:45:11 -0800 Subject: [PATCH 394/929] Reland "[llvm][Support] Add support for executing a detached process (#81708)" (#83367) Relands #81708, which was reverted by f410f74cd5b26319b5796e0404c6a0f3b5cc00a5, now with a corrected unit test. Origionally the test failed on Windows when run with lit as `GetConsoleWindow` could not retrieve a window handle regardless of whether `DetachProcess` was `true` or `false`. The test now uses `GetStdHandle(STD_OUTPUT_HANDLE)` which does not rely on a console window existing. Original commit message below. Adds a new parameter, `bool DetachProcess` with a default option of `false`, to `llvm::sys::ExecuteNoWait`, which, when set to `true`, executes the specified program without a controlling terminal. Functionality added so that the module build daemon can be run without a controlling terminal. --- llvm/include/llvm/Support/Program.h | 25 +++---- llvm/lib/Support/Program.cpp | 9 +-- llvm/lib/Support/Unix/Program.inc | 18 ++++-- llvm/lib/Support/Windows/Program.inc | 7 +- llvm/unittests/Support/ProgramTest.cpp | 90 ++++++++++++++++++++++++++ 5 files changed, 128 insertions(+), 21 deletions(-) diff --git a/llvm/include/llvm/Support/Program.h b/llvm/include/llvm/Support/Program.h index 4c1133e44a21..9df94eb604c7 100644 --- a/llvm/include/llvm/Support/Program.h +++ b/llvm/include/llvm/Support/Program.h @@ -141,18 +141,21 @@ namespace sys { /// program shall run on. ); - /// Similar to ExecuteAndWait, but returns immediately. - /// @returns The \see ProcessInfo of the newly launched process. + /// Similar to \ref ExecuteAndWait, but returns immediately. + /// \returns The \ref ProcessInfo of the newly launched process. /// \note On Microsoft Windows systems, users will need to either call - /// \see Wait until the process finished execution or win32 CloseHandle() API - /// on ProcessInfo.ProcessHandle to avoid memory leaks. - ProcessInfo ExecuteNoWait(StringRef Program, ArrayRef Args, - std::optional> Env, - ArrayRef> Redirects = {}, - unsigned MemoryLimit = 0, - std::string *ErrMsg = nullptr, - bool *ExecutionFailed = nullptr, - BitVector *AffinityMask = nullptr); + /// \ref Wait until the process has finished executing or win32's CloseHandle + /// API on ProcessInfo.ProcessHandle to avoid memory leaks. + ProcessInfo ExecuteNoWait( + StringRef Program, ArrayRef Args, + std::optional> Env, + ArrayRef> Redirects = {}, + unsigned MemoryLimit = 0, std::string *ErrMsg = nullptr, + bool *ExecutionFailed = nullptr, BitVector *AffinityMask = nullptr, + /// If true the executed program detatches from the controlling + /// terminal. I/O streams such as llvm::outs, llvm::errs, and stdin will + /// be closed until redirected to another output location + bool DetachProcess = false); /// Return true if the given arguments fit within system-specific /// argument length limits. diff --git a/llvm/lib/Support/Program.cpp b/llvm/lib/Support/Program.cpp index 1dcd45e2d69e..181f68cfbb8c 100644 --- a/llvm/lib/Support/Program.cpp +++ b/llvm/lib/Support/Program.cpp @@ -27,7 +27,7 @@ static bool Execute(ProcessInfo &PI, StringRef Program, std::optional> Env, ArrayRef> Redirects, unsigned MemoryLimit, std::string *ErrMsg, - BitVector *AffinityMask); + BitVector *AffinityMask, bool DetachProcess); int sys::ExecuteAndWait(StringRef Program, ArrayRef Args, std::optional> Env, @@ -39,7 +39,7 @@ int sys::ExecuteAndWait(StringRef Program, ArrayRef Args, assert(Redirects.empty() || Redirects.size() == 3); ProcessInfo PI; if (Execute(PI, Program, Args, Env, Redirects, MemoryLimit, ErrMsg, - AffinityMask)) { + AffinityMask, /*DetachProcess=*/false)) { if (ExecutionFailed) *ExecutionFailed = false; ProcessInfo Result = Wait( @@ -58,13 +58,14 @@ ProcessInfo sys::ExecuteNoWait(StringRef Program, ArrayRef Args, std::optional> Env, ArrayRef> Redirects, unsigned MemoryLimit, std::string *ErrMsg, - bool *ExecutionFailed, BitVector *AffinityMask) { + bool *ExecutionFailed, BitVector *AffinityMask, + bool DetachProcess) { assert(Redirects.empty() || Redirects.size() == 3); ProcessInfo PI; if (ExecutionFailed) *ExecutionFailed = false; if (!Execute(PI, Program, Args, Env, Redirects, MemoryLimit, ErrMsg, - AffinityMask)) + AffinityMask, DetachProcess)) if (ExecutionFailed) *ExecutionFailed = true; diff --git a/llvm/lib/Support/Unix/Program.inc b/llvm/lib/Support/Unix/Program.inc index 5d9757bcc51b..2742734bb11e 100644 --- a/llvm/lib/Support/Unix/Program.inc +++ b/llvm/lib/Support/Unix/Program.inc @@ -173,10 +173,11 @@ toNullTerminatedCStringArray(ArrayRef Strings, StringSaver &Saver) { } static bool Execute(ProcessInfo &PI, StringRef Program, - ArrayRef Args, std::optional> Env, + ArrayRef Args, + std::optional> Env, ArrayRef> Redirects, unsigned MemoryLimit, std::string *ErrMsg, - BitVector *AffinityMask) { + BitVector *AffinityMask, bool DetachProcess) { if (!llvm::sys::fs::exists(Program)) { if (ErrMsg) *ErrMsg = std::string("Executable \"") + Program.str() + @@ -202,7 +203,8 @@ static bool Execute(ProcessInfo &PI, StringRef Program, // If this OS has posix_spawn and there is no memory limit being implied, use // posix_spawn. It is more efficient than fork/exec. #ifdef HAVE_POSIX_SPAWN - if (MemoryLimit == 0) { + // Cannot use posix_spawn if you would like to detach the process + if (MemoryLimit == 0 && !DetachProcess) { posix_spawn_file_actions_t FileActionsStore; posix_spawn_file_actions_t *FileActions = nullptr; @@ -270,7 +272,7 @@ static bool Execute(ProcessInfo &PI, StringRef Program, return true; } -#endif +#endif // HAVE_POSIX_SPAWN // Create a child process. int child = fork(); @@ -307,6 +309,14 @@ static bool Execute(ProcessInfo &PI, StringRef Program, } } + if (DetachProcess) { + // Detach from controlling terminal + if (::setsid() == -1) { + MakeErrMsg(ErrMsg, "Could not detach process, ::setsid failed"); + return false; + } + } + // Set memory limits if (MemoryLimit != 0) { SetMemoryLimits(MemoryLimit); diff --git a/llvm/lib/Support/Windows/Program.inc b/llvm/lib/Support/Windows/Program.inc index 0de9d3f75644..d98d55f317a3 100644 --- a/llvm/lib/Support/Windows/Program.inc +++ b/llvm/lib/Support/Windows/Program.inc @@ -172,10 +172,11 @@ static HANDLE RedirectIO(std::optional Path, int fd, } // namespace llvm static bool Execute(ProcessInfo &PI, StringRef Program, - ArrayRef Args, std::optional> Env, + ArrayRef Args, + std::optional> Env, ArrayRef> Redirects, unsigned MemoryLimit, std::string *ErrMsg, - BitVector *AffinityMask) { + BitVector *AffinityMask, bool DetachProcess) { if (!sys::fs::can_execute(Program)) { if (ErrMsg) *ErrMsg = "program not executable"; @@ -284,6 +285,8 @@ static bool Execute(ProcessInfo &PI, StringRef Program, unsigned CreateFlags = CREATE_UNICODE_ENVIRONMENT; if (AffinityMask) CreateFlags |= CREATE_SUSPENDED; + if (DetachProcess) + CreateFlags |= DETACHED_PROCESS; std::vector CommandUtf16(Command.size() + 1, 0); std::copy(Command.begin(), Command.end(), CommandUtf16.begin()); diff --git a/llvm/unittests/Support/ProgramTest.cpp b/llvm/unittests/Support/ProgramTest.cpp index 2e2b1958b9ac..b1b35eacd1f6 100644 --- a/llvm/unittests/Support/ProgramTest.cpp +++ b/llvm/unittests/Support/ProgramTest.cpp @@ -260,6 +260,96 @@ TEST_F(ProgramEnvTest, TestExecuteNoWait) { ASSERT_GT(LoopCount, 1u) << "LoopCount should be >1"; } +TEST_F(ProgramEnvTest, TestExecuteNoWaitDetached) { + using namespace llvm::sys; + + if (getenv("LLVM_PROGRAM_TEST_EXECUTE_NO_WAIT_DETACHED")) { + sleep_for(/*seconds=*/5); + char *Detached = getenv("LLVM_PROGRAM_TEST_EXECUTE_NO_WAIT_DETACHED_TRUE"); +#if _WIN32 + HANDLE StdHandle = GetStdHandle(STD_OUTPUT_HANDLE); + + if (Detached && (StdHandle == INVALID_HANDLE_VALUE || StdHandle == NULL)) + exit(100); + if (!Detached && (StdHandle != INVALID_HANDLE_VALUE && StdHandle != NULL)) + exit(200); +#else + int ParentSID = std::stoi( + std::string(getenv("LLVM_PROGRAM_TEST_EXECUTE_NO_WAIT_DETACHED_SID"))); + + pid_t ChildSID = ::getsid(0); + if (ChildSID == -1) { + llvm::errs() << "Could not get process SID: " << strerror(errno) << '\n'; + exit(1); + } + + if (Detached && (ChildSID != ParentSID)) + exit(100); + if (!Detached && (ChildSID == ParentSID)) + exit(200); +#endif + exit(0); + } + + std::string Executable = + sys::fs::getMainExecutable(TestMainArgv0, &ProgramTestStringArg1); + StringRef argv[] = { + Executable, "--gtest_filter=ProgramEnvTest.TestExecuteNoWaitDetached"}; + addEnvVar("LLVM_PROGRAM_TEST_EXECUTE_NO_WAIT_DETACHED=1"); + +#if _WIN32 + // Depending on how the test is run it may already be detached from a + // console. Temporarily allocate a new console. If a console already + // exists AllocConsole will harmlessly fail and return false + BOOL AllocConsoleSuccess = AllocConsole(); + + // Confirm existence of console + HANDLE StdHandle = GetStdHandle(STD_OUTPUT_HANDLE); + ASSERT_TRUE(StdHandle != INVALID_HANDLE_VALUE && StdHandle != NULL); +#else + pid_t SID = ::getsid(0); + ASSERT_NE(SID, -1); + std::string SIDEnvVar = + "LLVM_PROGRAM_TEST_EXECUTE_NO_WAIT_DETACHED_SID=" + std::to_string(SID); + addEnvVar(SIDEnvVar); +#endif + + // DetachProcess = true + { + std::string Error; + bool ExecutionFailed; + std::vector Env = getEnviron(); + Env.emplace_back("LLVM_PROGRAM_TEST_EXECUTE_NO_WAIT_DETACHED_TRUE=1"); + ProcessInfo PI1 = + ExecuteNoWait(Executable, argv, Env, {}, 0, &Error, &ExecutionFailed, + nullptr, /*DetachProcess=*/true); + ASSERT_FALSE(ExecutionFailed) << Error; + ASSERT_NE(PI1.Pid, ProcessInfo::InvalidPid) << "Invalid process id"; + ProcessInfo WaitResult = Wait(PI1, std::nullopt, &Error); + ASSERT_EQ(WaitResult.ReturnCode, 100); + } + + // DetachProcess = false + { + std::string Error; + bool ExecutionFailed; + ProcessInfo PI2 = + ExecuteNoWait(Executable, argv, getEnviron(), {}, 0, &Error, + &ExecutionFailed, nullptr, /*DetachProcess=*/false); + ASSERT_FALSE(ExecutionFailed) << Error; + ASSERT_NE(PI2.Pid, ProcessInfo::InvalidPid) << "Invalid process id"; + ProcessInfo WaitResult = Wait(PI2, std::nullopt, &Error); + ASSERT_EQ(WaitResult.ReturnCode, 200); + } +#if _WIN32 + // If console was allocated then free the console + if (AllocConsoleSuccess) { + BOOL FreeConsoleSuccess = FreeConsole(); + ASSERT_NE(FreeConsoleSuccess, 0); + } +#endif +} + TEST_F(ProgramEnvTest, TestExecuteAndWaitTimeout) { using namespace llvm::sys; -- GitLab From 248468097faa9739ae66ff6fe1836e529d58455a Mon Sep 17 00:00:00 2001 From: Martin Wehking Date: Thu, 7 Mar 2024 05:01:34 +0000 Subject: [PATCH 395/929] Add non-null check before accessing pointer (#83459) Add a check if RC is not null to ensure that a consecutive access is safe. A static analyzer flagged this issue since hasVectorRegisters potentially dereferences RC. --- llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp index 634b4aeb30a7..cb448aaafa4c 100644 --- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp +++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp @@ -1054,6 +1054,7 @@ void SIFoldOperands::foldOperand( // Don't fold if OpToFold doesn't hold an aligned register. const TargetRegisterClass *RC = TRI->getRegClassForReg(*MRI, OpToFold.getReg()); + assert(RC); if (TRI->hasVectorRegisters(RC) && OpToFold.getSubReg()) { unsigned SubReg = OpToFold.getSubReg(); if (const TargetRegisterClass *SubRC = -- GitLab From 40671156757cc3f2304085a6642a24efc8165df7 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Wed, 6 Mar 2024 21:12:10 -0800 Subject: [PATCH 396/929] [lldb] Log module build remarks to types log too (#84260) --- .../Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp index cc5250ca0f78..2d778e410b0e 100644 --- a/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp +++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp @@ -183,7 +183,7 @@ void StoringDiagnosticConsumer::EndSourceFile() { bool StoringDiagnosticConsumer::HandleModuleRemark( const clang::Diagnostic &info) { - Log *log = GetLog(LLDBLog::Expressions); + Log *log = GetLog(LLDBLog::Types | LLDBLog::Expressions); switch (info.getID()) { case clang::diag::remark_module_build: { const auto &module_name = info.getArgStdStr(0); -- GitLab From 95bde4b9d6fe258ff7fea6da5024b2a8813c1e0c Mon Sep 17 00:00:00 2001 From: Alexander Richardson Date: Wed, 6 Mar 2024 22:14:15 -0800 Subject: [PATCH 397/929] [MSan] Pass -fsanitize-ignorelist to the instrumented libcxxabi This ensures that the MSan unit tests are able to pass with an uninstrumented libunwind. We need to avoid instrumentation for __gxx_personality_v0, which is part of the default msan_ignorelist.txt that is installed into the resource directory. However, if we are trying to test the just-built libraries, this global ignore list may not be present yet, so we still instrument the function. Arguably this function should not be on the default ignore list since it is only a problem when building libcxxabi with MSan instrumentation and without an instrumented libunwind, so maybe the logic should really be part of the libcxxabi build. However, that could be done as a follow-up. See 2f856a36e0b270b184051d10a18d4b4238b4c033 for more context. Reviewed By: vitalybuka Pull Request: https://github.com/llvm/llvm-project/pull/83652 --- compiler-rt/lib/msan/tests/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler-rt/lib/msan/tests/CMakeLists.txt b/compiler-rt/lib/msan/tests/CMakeLists.txt index 6ef63ff82166..412a0f6b3de7 100644 --- a/compiler-rt/lib/msan/tests/CMakeLists.txt +++ b/compiler-rt/lib/msan/tests/CMakeLists.txt @@ -10,7 +10,8 @@ set(MSAN_LIBCXX_CFLAGS -fsanitize-memory-track-origins -fno-sanitize-memory-param-retval # unittests test mostly this mode. -Wno-pedantic - -Xclang -fdepfile-entry=${COMPILER_RT_OUTPUT_DIR}/share/msan_ignorelist.txt + # Do not instrument __gxx_personality_v0 (part of libcxxabi) + -fsanitize-ignorelist=${COMPILER_RT_OUTPUT_DIR}/share/msan_ignorelist.txt ) # Unittest sources and build flags. -- GitLab From 2e25926b50183cb3d819e85ef7bdf00001027859 Mon Sep 17 00:00:00 2001 From: Alexander Richardson Date: Wed, 6 Mar 2024 22:16:30 -0800 Subject: [PATCH 398/929] [compiler-rt] Only query llvm-config in the orc tests This check for assertions is only used inside the test/orc directory, but doing it in the top level lit config means all testsuites depend on llvm-config being present. This is not necessarily needed e.g. when testing just the builtins. While touching this code, simplify it a bit by using subprocess.check_output() instead of Popen() and use a string comparison instead of a regex match. Reviewed By: lhames Pull Request: https://github.com/llvm/llvm-project/pull/83705 --- compiler-rt/test/lit.common.cfg.py | 16 ---------------- compiler-rt/test/orc/lit.cfg.py | 12 ++++++++++++ 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/compiler-rt/test/lit.common.cfg.py b/compiler-rt/test/lit.common.cfg.py index 113777b0ea8a..ae28681915af 100644 --- a/compiler-rt/test/lit.common.cfg.py +++ b/compiler-rt/test/lit.common.cfg.py @@ -738,22 +738,6 @@ if config.lto_supported: if config.have_rpc_xdr_h: config.available_features.add("sunrpc") -# Ask llvm-config about assertion mode. -try: - llvm_config_cmd = subprocess.Popen( - [os.path.join(config.llvm_tools_dir, "llvm-config"), "--assertion-mode"], - stdout=subprocess.PIPE, - env=config.environment, - ) -except OSError as e: - print("Could not launch llvm-config in " + config.llvm_tools_dir) - print(" Failed with error #{0}: {1}".format(e.errno, e.strerror)) - exit(42) - -if re.search(r"ON", llvm_config_cmd.stdout.read().decode("ascii")): - config.available_features.add("asserts") -llvm_config_cmd.wait() - # Sanitizer tests tend to be flaky on Windows due to PR24554, so add some # retries. We don't do this on otther platforms because it's slower. if platform.system() == "Windows": diff --git a/compiler-rt/test/orc/lit.cfg.py b/compiler-rt/test/orc/lit.cfg.py index bd031d79826d..897cefb3d193 100644 --- a/compiler-rt/test/orc/lit.cfg.py +++ b/compiler-rt/test/orc/lit.cfg.py @@ -1,6 +1,7 @@ # -*- Python -*- import os +import subprocess # Setup config name. config.name = "ORC" + config.name_suffix @@ -79,3 +80,14 @@ config.excludes = ["Inputs"] if config.host_os not in ["Darwin", "FreeBSD", "Linux", "Windows"]: config.unsupported = True + +# Ask llvm-config about assertion mode. +try: + llvm_config_result = subprocess.check_output( + [os.path.join(config.llvm_tools_dir, "llvm-config"), "--assertion-mode"], + env=config.environment, + ) + if llvm_config_result.startswith(b"ON"): + config.available_features.add("asserts") +except OSError as e: + lit_config.warning(f"Could not determine if LLVM was built with assertions: {e}") -- GitLab From 68d07bf34f6b044193c429d9f8cf07500a4775ea Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Thu, 7 Mar 2024 14:18:37 +0800 Subject: [PATCH 399/929] [RISCV][NFC] Add helpers for RVV register classes There are two places in tree that use these helpers and there will be more future usages. Reviewers: asb, BeMg, lukel97 Reviewed By: BeMg, lukel97 Pull Request: https://github.com/llvm/llvm-project/pull/84144 --- llvm/lib/Target/RISCV/RISCVRegisterInfo.h | 22 ++++++++++++++++++++ llvm/lib/Target/RISCV/RISCVTargetMachine.cpp | 16 +------------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.h b/llvm/lib/Target/RISCV/RISCVRegisterInfo.h index e46fe8ecb900..943c4f2627cf 100644 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.h +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.h @@ -112,11 +112,33 @@ struct RISCVRegisterInfo : public RISCVGenRegisterInfo { bool doesRegClassHavePseudoInitUndef( const TargetRegisterClass *RC) const override { + return isVRRegClass(RC); + } + + static bool isVRRegClass(const TargetRegisterClass *RC) { return RISCV::VRRegClass.hasSubClassEq(RC) || RISCV::VRM2RegClass.hasSubClassEq(RC) || RISCV::VRM4RegClass.hasSubClassEq(RC) || RISCV::VRM8RegClass.hasSubClassEq(RC); } + + static bool isVRNRegClass(const TargetRegisterClass *RC) { + return RISCV::VRN2M1RegClass.hasSubClassEq(RC) || + RISCV::VRN2M2RegClass.hasSubClassEq(RC) || + RISCV::VRN2M4RegClass.hasSubClassEq(RC) || + RISCV::VRN3M1RegClass.hasSubClassEq(RC) || + RISCV::VRN3M2RegClass.hasSubClassEq(RC) || + RISCV::VRN4M1RegClass.hasSubClassEq(RC) || + RISCV::VRN4M2RegClass.hasSubClassEq(RC) || + RISCV::VRN5M1RegClass.hasSubClassEq(RC) || + RISCV::VRN6M1RegClass.hasSubClassEq(RC) || + RISCV::VRN7M1RegClass.hasSubClassEq(RC) || + RISCV::VRN8M1RegClass.hasSubClassEq(RC); + } + + static bool isRVVRegClass(const TargetRegisterClass *RC) { + return isVRRegClass(RC) || isVRNRegClass(RC); + } }; } diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp index 6fe0abaccb9d..ae1a6f179a49 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp @@ -278,21 +278,7 @@ public: static bool onlyAllocateRVVReg(const TargetRegisterInfo &TRI, const TargetRegisterClass &RC) { - return RISCV::VRRegClass.hasSubClassEq(&RC) || - RISCV::VRM2RegClass.hasSubClassEq(&RC) || - RISCV::VRM4RegClass.hasSubClassEq(&RC) || - RISCV::VRM8RegClass.hasSubClassEq(&RC) || - RISCV::VRN2M1RegClass.hasSubClassEq(&RC) || - RISCV::VRN2M2RegClass.hasSubClassEq(&RC) || - RISCV::VRN2M4RegClass.hasSubClassEq(&RC) || - RISCV::VRN3M1RegClass.hasSubClassEq(&RC) || - RISCV::VRN3M2RegClass.hasSubClassEq(&RC) || - RISCV::VRN4M1RegClass.hasSubClassEq(&RC) || - RISCV::VRN4M2RegClass.hasSubClassEq(&RC) || - RISCV::VRN5M1RegClass.hasSubClassEq(&RC) || - RISCV::VRN6M1RegClass.hasSubClassEq(&RC) || - RISCV::VRN7M1RegClass.hasSubClassEq(&RC) || - RISCV::VRN8M1RegClass.hasSubClassEq(&RC); + return RISCVRegisterInfo::isRVVRegClass(&RC); } static FunctionPass *useDefaultRegisterAllocator() { return nullptr; } -- GitLab From 50bdc6f3ec3418073805d2b535f3f72a7d677577 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Wed, 6 Mar 2024 22:37:18 -0800 Subject: [PATCH 400/929] [BOLT,test] Remove -relax-relocations The option is always true (see 2aedfdd9b82e6c72a28576d0e8ea854f1300ff4e). --- bolt/test/X86/gotpcrelx.s | 3 +-- bolt/test/X86/pt_gnu_relro.s | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/bolt/test/X86/gotpcrelx.s b/bolt/test/X86/gotpcrelx.s index ddac5ebda0ca..6dec125c6a72 100644 --- a/bolt/test/X86/gotpcrelx.s +++ b/bolt/test/X86/gotpcrelx.s @@ -5,8 +5,7 @@ ## kinds of handling of the relocation by the linker (no relaxation, pic, and ## non-pic). -# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-linux \ -# RUN: -relax-relocations %s -o %t.o +# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-linux %s -o %t.o # RUN: ld.lld %t.o -o %t.exe -q # RUN: ld.lld %t.o -o %t.pie.exe -q -pie # RUN: ld.lld %t.o -o %t.no-relax.exe -q --no-relax diff --git a/bolt/test/X86/pt_gnu_relro.s b/bolt/test/X86/pt_gnu_relro.s index ad8d475c8372..fa4af8287494 100644 --- a/bolt/test/X86/pt_gnu_relro.s +++ b/bolt/test/X86/pt_gnu_relro.s @@ -3,7 +3,7 @@ # Check that BOLT recognizes PT_GNU_RELRO segment and marks respective sections # accordingly. -# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-linux %s -o %t.o -relax-relocations +# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-linux %s -o %t.o # RUN: ld.lld %t.o -o %t.exe -q --no-relax # RUN: llvm-readelf -We %t.exe | FileCheck --check-prefix=READELF %s # Unfortunately there's no direct way to extract a segment to section mapping -- GitLab From 00efb343529cce025055ca8284a61b22b48924c9 Mon Sep 17 00:00:00 2001 From: Amara Emerson Date: Wed, 6 Mar 2024 22:41:09 -0800 Subject: [PATCH 401/929] [AArch64][GlobalISel] Fix crash during G_SHUFFLE_VECTOR legalization. A new widening rule was running before the shuffle was canonicalized into a homogenous form. Moving the rules around to ensure it's done before the widening fixes the crash, although this particular test still falls back. --- .../AArch64/GISel/AArch64LegalizerInfo.cpp | 12 +++---- .../legalize-shuffle-vector-widen-crash.ll | 32 +++++++++++++++++++ 2 files changed, 38 insertions(+), 6 deletions(-) create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/legalize-shuffle-vector-widen-crash.ll diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp index 308364aadaf7..ad389cfc75aa 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp @@ -956,18 +956,18 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) }, changeTo(1, 0)) .moreElementsToNextPow2(0) - .widenScalarOrEltToNextPow2OrMinSize(0, 8) - .clampNumElements(0, v8s8, v16s8) - .clampNumElements(0, v4s16, v8s16) - .clampNumElements(0, v4s32, v4s32) - .clampNumElements(0, v2s64, v2s64) .moreElementsIf( [](const LegalityQuery &Query) { return Query.Types[0].isVector() && Query.Types[1].isVector() && Query.Types[0].getNumElements() < Query.Types[1].getNumElements(); }, - changeTo(0, 1)); + changeTo(0, 1)) + .widenScalarOrEltToNextPow2OrMinSize(0, 8) + .clampNumElements(0, v8s8, v16s8) + .clampNumElements(0, v4s16, v8s16) + .clampNumElements(0, v4s32, v4s32) + .clampNumElements(0, v2s64, v2s64); getActionDefinitionsBuilder(G_CONCAT_VECTORS) .legalFor({{v4s32, v2s32}, {v8s16, v4s16}, {v16s8, v8s8}}); diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-shuffle-vector-widen-crash.ll b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-shuffle-vector-widen-crash.ll new file mode 100644 index 000000000000..42a8f51002f2 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-shuffle-vector-widen-crash.ll @@ -0,0 +1,32 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -global-isel-abort=2 -global-isel -o - %s | FileCheck %s +target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" +target triple = "arm64-apple-macosx11.0.0" + +declare i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32>) #0 + +; This test currently falls back but ensures we don't crash. + +define i32 @bar() { +; CHECK-LABEL: bar: +; CHECK: ; %bb.0: ; %bb +; CHECK-NEXT: movi.2d v0, #0000000000000000 +; CHECK-NEXT: addv.4s s0, v0 +; CHECK-NEXT: fmov w0, s0 +; CHECK-NEXT: ret +bb: + %shufflevector = shufflevector <8 x i1> zeroinitializer, <8 x i1> zeroinitializer, <4 x i32> + %zext = zext <4 x i1> %shufflevector to <4 x i32> + %call = call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> %zext) + %icmp = icmp eq i32 %call, 0 + br i1 %icmp, label %bb1, label %bb2 + +bb1: ; preds = %bb2, %bb + ret i32 %call + +bb2: ; preds = %bb + %sext = sext i32 0 to i64 + br label %bb1 +} + +attributes #0 = { nocallback nofree nosync nounwind willreturn memory(none) } -- GitLab From 2f1d79ee702148f452584367ddd0e3c81ecf3c64 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Wed, 6 Mar 2024 22:50:37 -0800 Subject: [PATCH 402/929] [ELF,test] Remove unneeded -relax-relocations from llvm-mc This is option is x86-64 specific and should not be used by AMDGPU/SystemZ. On x86-64, the option defaults to true and is unnecessary. --- lld/test/ELF/amdgpu-duplicate-sym.s | 2 +- lld/test/ELF/i386-relax-reloc.s | 2 +- lld/test/ELF/systemz-gotent-relax-align.s | 2 +- lld/test/ELF/systemz-gotent-relax-und-dso.s | 4 ++-- lld/test/ELF/systemz-gotent-relax.s | 2 +- lld/test/ELF/x86-64-gotpc-relax-nopic.s | 2 +- lld/test/ELF/x86-64-gotpc-relax-und-dso.s | 4 ++-- lld/test/ELF/x86-64-gotpc-relax.s | 2 +- lld/test/ELF/x86-64-relax-got-abs.s | 2 +- lld/test/ELF/x86-64-relax-offset.s | 5 ++--- 10 files changed, 13 insertions(+), 14 deletions(-) diff --git a/lld/test/ELF/amdgpu-duplicate-sym.s b/lld/test/ELF/amdgpu-duplicate-sym.s index db92dad01f65..9423a2cee1d4 100644 --- a/lld/test/ELF/amdgpu-duplicate-sym.s +++ b/lld/test/ELF/amdgpu-duplicate-sym.s @@ -1,5 +1,5 @@ # REQUIRES: amdgpu -# RUN: llvm-mc -filetype=obj -triple amdgcn-amd-amdhsa -mcpu=gfx1031 --position-independent --relax-relocations %s -o %t.o +# RUN: llvm-mc -filetype=obj -triple amdgcn-amd-amdhsa -mcpu=gfx1031 --position-independent %s -o %t.o # We use lld-link on purpose to exercise -flavor. # RUN: lld-link -flavor gnu -shared %t.o -o /dev/null diff --git a/lld/test/ELF/i386-relax-reloc.s b/lld/test/ELF/i386-relax-reloc.s index 7376af7c8caa..1db493981370 100644 --- a/lld/test/ELF/i386-relax-reloc.s +++ b/lld/test/ELF/i386-relax-reloc.s @@ -1,5 +1,5 @@ // REQUIRES: x86 -// RUN: llvm-mc -filetype=obj -triple=i686-pc-linux %s -o %t.o -relax-relocations +// RUN: llvm-mc -filetype=obj -triple=i686-pc-linux %s -o %t.o // RUN: ld.lld -shared %t.o -o %t.so // RUN: llvm-readelf -S %t.so | FileCheck --check-prefix=SEC %s // RUN: llvm-objdump --no-print-imm-hex -d --no-show-raw-insn %t.so | FileCheck %s diff --git a/lld/test/ELF/systemz-gotent-relax-align.s b/lld/test/ELF/systemz-gotent-relax-align.s index c6326086f56d..125d74ba4a21 100644 --- a/lld/test/ELF/systemz-gotent-relax-align.s +++ b/lld/test/ELF/systemz-gotent-relax-align.s @@ -1,7 +1,7 @@ # REQUIRES: systemz ## Verify that R_390_GOTENT optimization is not performed on misaligned symbols. -# RUN: llvm-mc -filetype=obj -relax-relocations -triple=s390x-unknown-linux %s -o %t.o +# RUN: llvm-mc -filetype=obj -triple=s390x-unknown-linux %s -o %t.o # RUN: ld.lld %t.o -o %t1 # RUN: llvm-readelf -S -r -x .got -x .got.plt %t1 | FileCheck --check-prefixes=CHECK %s # RUN: llvm-objdump --no-print-imm-hex -d %t1 | FileCheck --check-prefix=DISASM %s diff --git a/lld/test/ELF/systemz-gotent-relax-und-dso.s b/lld/test/ELF/systemz-gotent-relax-und-dso.s index 57369a417fd4..fdbda701dad1 100644 --- a/lld/test/ELF/systemz-gotent-relax-und-dso.s +++ b/lld/test/ELF/systemz-gotent-relax-und-dso.s @@ -1,6 +1,6 @@ # REQUIRES: systemz -# RUN: llvm-mc -filetype=obj -relax-relocations -triple=s390x-unknown-linux %s -o %t.o -# RUN: llvm-mc -filetype=obj -relax-relocations -triple=s390x-unknown-linux %S/Inputs/gotpc-relax-und-dso.s -o %tdso.o +# RUN: llvm-mc -filetype=obj -triple=s390x-unknown-linux %s -o %t.o +# RUN: llvm-mc -filetype=obj -triple=s390x-unknown-linux %S/Inputs/gotpc-relax-und-dso.s -o %tdso.o # RUN: ld.lld -shared %tdso.o -soname=t.so -o %t.so # RUN: ld.lld --hash-style=sysv -shared %t.o %t.so -o %t # RUN: llvm-readelf -r %t | FileCheck --check-prefix=RELOC %s diff --git a/lld/test/ELF/systemz-gotent-relax.s b/lld/test/ELF/systemz-gotent-relax.s index f665e1af9e53..7ff82b9a1900 100644 --- a/lld/test/ELF/systemz-gotent-relax.s +++ b/lld/test/ELF/systemz-gotent-relax.s @@ -1,7 +1,7 @@ # REQUIRES: systemz ## Test R_390_GOTENT optimization. -# RUN: llvm-mc -filetype=obj -relax-relocations -triple=s390x-unknown-linux %s -o %t.o +# RUN: llvm-mc -filetype=obj -triple=s390x-unknown-linux %s -o %t.o # RUN: ld.lld %t.o -o %t1 --no-apply-dynamic-relocs # RUN: llvm-readelf -S -r -x .got.plt %t1 | FileCheck --check-prefixes=CHECK,NOAPPLY %s # RUN: ld.lld %t.o -o %t1 --apply-dynamic-relocs diff --git a/lld/test/ELF/x86-64-gotpc-relax-nopic.s b/lld/test/ELF/x86-64-gotpc-relax-nopic.s index cd12be86c740..7481904d16f1 100644 --- a/lld/test/ELF/x86-64-gotpc-relax-nopic.s +++ b/lld/test/ELF/x86-64-gotpc-relax-nopic.s @@ -1,5 +1,5 @@ # REQUIRES: x86 -# RUN: llvm-mc -filetype=obj -relax-relocations -triple=x86_64-unknown-linux %s -o %t.o +# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.o # RUN: ld.lld %t.o -o %t1 # RUN: llvm-readobj --symbols -r %t1 | FileCheck --check-prefix=SYMRELOC %s # RUN: llvm-objdump -d --no-show-raw-insn --print-imm-hex %t1 | FileCheck --check-prefix=DISASM %s diff --git a/lld/test/ELF/x86-64-gotpc-relax-und-dso.s b/lld/test/ELF/x86-64-gotpc-relax-und-dso.s index 0704d2c2c4ae..329b1db80963 100644 --- a/lld/test/ELF/x86-64-gotpc-relax-und-dso.s +++ b/lld/test/ELF/x86-64-gotpc-relax-und-dso.s @@ -1,6 +1,6 @@ # REQUIRES: x86 -# RUN: llvm-mc -filetype=obj -relax-relocations -triple=x86_64-unknown-linux %s -o %t.o -# RUN: llvm-mc -filetype=obj -relax-relocations -triple=x86_64-pc-linux %S/Inputs/gotpc-relax-und-dso.s -o %tdso.o +# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.o +# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %S/Inputs/gotpc-relax-und-dso.s -o %tdso.o # RUN: ld.lld -shared %tdso.o -soname=t.so -o %t.so # RUN: ld.lld --hash-style=sysv -shared %t.o %t.so -o %t # RUN: llvm-readobj -r -S %t | FileCheck --check-prefix=RELOC %s diff --git a/lld/test/ELF/x86-64-gotpc-relax.s b/lld/test/ELF/x86-64-gotpc-relax.s index de858bffc07e..5945bfc04a02 100644 --- a/lld/test/ELF/x86-64-gotpc-relax.s +++ b/lld/test/ELF/x86-64-gotpc-relax.s @@ -1,7 +1,7 @@ # REQUIRES: x86 ## Test R_X86_64_GOTPCRELX and R_X86_64_REX_GOTPCRELX GOT optimization. -# RUN: llvm-mc -filetype=obj -relax-relocations -triple=x86_64-unknown-linux %s -o %t.o +# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.o # RUN: ld.lld %t.o -o %t1 --no-apply-dynamic-relocs # RUN: llvm-readelf -S -r -x .got.plt %t1 | FileCheck --check-prefixes=CHECK,NOAPPLY %s # RUN: ld.lld %t.o -o %t1 --apply-dynamic-relocs diff --git a/lld/test/ELF/x86-64-relax-got-abs.s b/lld/test/ELF/x86-64-relax-got-abs.s index 9af63ac33a4b..b88e3ccd8a42 100644 --- a/lld/test/ELF/x86-64-relax-got-abs.s +++ b/lld/test/ELF/x86-64-relax-got-abs.s @@ -3,7 +3,7 @@ ## Don't relax R_X86_64_GOTPCRELX to an absolute symbol. ## In -no-pie mode, it can be relaxed, but it may not worth it. -# RUN: llvm-mc -filetype=obj -relax-relocations -triple=x86_64 %s -o %t.o +# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o # RUN: ld.lld %t.o -o %t.so -shared # RUN: llvm-objdump --no-print-imm-hex -d %t.so | FileCheck %s diff --git a/lld/test/ELF/x86-64-relax-offset.s b/lld/test/ELF/x86-64-relax-offset.s index 3858e153fd86..09f6b47b6931 100644 --- a/lld/test/ELF/x86-64-relax-offset.s +++ b/lld/test/ELF/x86-64-relax-offset.s @@ -3,9 +3,8 @@ /// Test we use input r_offset when deciding if R_X86_64_GOTPCRELX /// relaxation is applicable. -// RUN: llvm-mc -filetype=obj -relax-relocations -triple=x86_64-pc-linux %s \ -// RUN: -o %t.o -// RUN: llvm-mc -filetype=obj -relax-relocations -triple=x86_64-pc-linux \ +// RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o +// RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux \ // RUN: %p/Inputs/x86-64-relax-offset.s -o %t2.o // RUN: ld.lld %t2.o %t.o -o %t.so -shared // RUN: llvm-objdump --no-print-imm-hex -d %t.so | FileCheck %s -- GitLab From e63ea9d6f7b1d15f4819e6af8ee9452046e1548c Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Wed, 6 Mar 2024 23:03:11 -0800 Subject: [PATCH 403/929] [CommandFlags] Rename option -relax-elf-relocations to -x86-relax-relocations relax-elf-relocations is misleading and there were AMDGPU/SystemZ tests misusing this x86-specific option. --- llvm/lib/CodeGen/CommandFlags.cpp | 2 +- .../test/CodeGen/X86/tailjmp_gotpcrel_relax_relocation.ll | 2 +- llvm/test/CodeGen/X86/tls-no-plt.ll | 8 ++++---- llvm/test/ThinLTO/X86/cache-config.ll | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/llvm/lib/CodeGen/CommandFlags.cpp b/llvm/lib/CodeGen/CommandFlags.cpp index d82b2a85016c..d61b7082cb22 100644 --- a/llvm/lib/CodeGen/CommandFlags.cpp +++ b/llvm/lib/CodeGen/CommandFlags.cpp @@ -363,7 +363,7 @@ codegen::RegisterCodeGenFlags::RegisterCodeGenFlags() { CGBINDOPT(UseCtors); static cl::opt RelaxELFRelocations( - "relax-elf-relocations", + "x86-relax-relocations", cl::desc( "Emit GOTPCRELX/REX_GOTPCRELX instead of GOTPCREL on x86-64 ELF"), cl::init(true)); diff --git a/llvm/test/CodeGen/X86/tailjmp_gotpcrel_relax_relocation.ll b/llvm/test/CodeGen/X86/tailjmp_gotpcrel_relax_relocation.ll index 3fb161cc2cba..cc8fd07a488c 100644 --- a/llvm/test/CodeGen/X86/tailjmp_gotpcrel_relax_relocation.ll +++ b/llvm/test/CodeGen/X86/tailjmp_gotpcrel_relax_relocation.ll @@ -1,4 +1,4 @@ -; RUN: llc -filetype=obj -relax-elf-relocations=true -mtriple=x86_64-linux-gnu -o - %s | llvm-objdump - -d -r | FileCheck %s +; RUN: llc -filetype=obj -x86-relax-relocations=true -mtriple=x86_64-linux-gnu -o - %s | llvm-objdump - -d -r | FileCheck %s ; CHECK: jmpq *(%rip) ; CHECK-NEXT: R_X86_64_GOTPCRELX diff --git a/llvm/test/CodeGen/X86/tls-no-plt.ll b/llvm/test/CodeGen/X86/tls-no-plt.ll index b076a77e8adb..c555d7f0e4cc 100644 --- a/llvm/test/CodeGen/X86/tls-no-plt.ll +++ b/llvm/test/CodeGen/X86/tls-no-plt.ll @@ -1,10 +1,10 @@ -; RUN: llc < %s -mtriple=i386-linux-musl -relocation-model=pic -relax-elf-relocations=true | FileCheck --check-prefixes=CHECK,X86 %s -; RUN: llc < %s -mtriple=x86_64-linux-musl -relocation-model=pic -relax-elf-relocations=true | FileCheck --check-prefixes=CHECK,X64 %s +; RUN: llc < %s -mtriple=i386-linux-musl -relocation-model=pic -x86-relax-relocations=true | FileCheck --check-prefixes=CHECK,X86 %s +; RUN: llc < %s -mtriple=x86_64-linux-musl -relocation-model=pic -x86-relax-relocations=true | FileCheck --check-prefixes=CHECK,X64 %s ;; If GOTPCRELX is disabled, don't use GOT for __tls_get_addr to work around ;; a ld.bfd bug (binutils PR24784). -; RUN: llc < %s -mtriple=i386-linux-musl -relocation-model=pic -relax-elf-relocations=false | FileCheck --check-prefixes=CHECK,X86-PLT %s -; RUN: llc < %s -mtriple=x86_64-linux-musl -relocation-model=pic -relax-elf-relocations=false | FileCheck --check-prefixes=CHECK,X64-PLT %s +; RUN: llc < %s -mtriple=i386-linux-musl -relocation-model=pic -x86-relax-relocations=false | FileCheck --check-prefixes=CHECK,X86-PLT %s +; RUN: llc < %s -mtriple=x86_64-linux-musl -relocation-model=pic -x86-relax-relocations=false | FileCheck --check-prefixes=CHECK,X64-PLT %s @gd = thread_local global i32 0 @ld = internal thread_local global i32 0 diff --git a/llvm/test/ThinLTO/X86/cache-config.ll b/llvm/test/ThinLTO/X86/cache-config.ll index b261ead077b0..cef9338aa3ff 100644 --- a/llvm/test/ThinLTO/X86/cache-config.ll +++ b/llvm/test/ThinLTO/X86/cache-config.ll @@ -3,7 +3,7 @@ ; RUN: llvm-lto2 run -o %t.o %t.bc -cache-dir %t.cache -r=%t.bc,globalfunc,plx ; RUN: llvm-lto2 run -o %t.o %t.bc -cache-dir %t.cache -r=%t.bc,globalfunc,plx -mcpu=core2 -; RUN: llvm-lto2 run -o %t.o %t.bc -cache-dir %t.cache -r=%t.bc,globalfunc,plx -relax-elf-relocations=0 +; RUN: llvm-lto2 run -o %t.o %t.bc -cache-dir %t.cache -r=%t.bc,globalfunc,plx -x86-relax-relocations=0 ; RUN: llvm-lto2 run -o %t.o %t.bc -cache-dir %t.cache -r=%t.bc,globalfunc,plx -function-sections ; RUN: llvm-lto2 run -o %t.o %t.bc -cache-dir %t.cache -r=%t.bc,globalfunc,plx -data-sections ; RUN: llvm-lto2 run -o %t.o %t.bc -cache-dir %t.cache -r=%t.bc,globalfunc,plx -debugger-tune=sce -- GitLab From 886ecb3078e5f3a5cffc70408a637242c223c363 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Wed, 6 Mar 2024 23:12:40 -0800 Subject: [PATCH 404/929] [mlir] Remove setRelaxELFRelocations. NFC The option is always true (see 2aedfdd9b82e6c72a28576d0e8ea854f1300ff4e) and the MCAsmInfo option is going away in favor of MCTargetOptions. --- mlir/lib/Dialect/GPU/Transforms/SerializeToHsaco.cpp | 1 - mlir/lib/Target/LLVM/ROCDL/Target.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/mlir/lib/Dialect/GPU/Transforms/SerializeToHsaco.cpp b/mlir/lib/Dialect/GPU/Transforms/SerializeToHsaco.cpp index 2af898b56774..b07addc77b56 100644 --- a/mlir/lib/Dialect/GPU/Transforms/SerializeToHsaco.cpp +++ b/mlir/lib/Dialect/GPU/Transforms/SerializeToHsaco.cpp @@ -340,7 +340,6 @@ LogicalResult SerializeToHsacoPass::assembleIsa(const std::string &isa, target->createMCRegInfo(this->triple)); std::unique_ptr mai( target->createMCAsmInfo(*mri, this->triple, mcOptions)); - mai->setRelaxELFRelocations(true); std::unique_ptr sti( target->createMCSubtargetInfo(this->triple, this->chip, this->features)); diff --git a/mlir/lib/Target/LLVM/ROCDL/Target.cpp b/mlir/lib/Target/LLVM/ROCDL/Target.cpp index 4f01c6a66781..66593fd8a55f 100644 --- a/mlir/lib/Target/LLVM/ROCDL/Target.cpp +++ b/mlir/lib/Target/LLVM/ROCDL/Target.cpp @@ -276,7 +276,6 @@ SerializeGPUModuleBase::assembleIsa(StringRef isa) { target->createMCRegInfo(targetTriple)); std::unique_ptr mai( target->createMCAsmInfo(*mri, targetTriple, mcOptions)); - mai->setRelaxELFRelocations(true); std::unique_ptr sti( target->createMCSubtargetInfo(targetTriple, chip, features)); -- GitLab From a3319371970b599ef65ef1567c440fbdc3a330f4 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Wed, 6 Mar 2024 23:19:59 -0800 Subject: [PATCH 405/929] [MC] Move CompressDebugSections/RelaxELFRelocations from TargetOptions/MCAsmInfo to MCTargetOptions The convention is for such MC-specific options to reside in MCTargetOptions. However, CompressDebugSections/RelaxELFRelocations do not follow the convention: `CompressDebugSections` is defined in both TargetOptions and MCAsmInfo and there is forwarding complexity. Move the option to MCTargetOptions and hereby simplify the code. Rename the misleading RelaxELFRelocations to X86RelaxRelocations. llvm-mc -relax-relocations and llc -x86-relax-relocations can now be unified. --- clang/lib/CodeGen/BackendUtil.cpp | 5 ++-- clang/tools/driver/cc1as_main.cpp | 4 +-- lld/test/ELF/weak-undef-got-pie.s | 2 +- llvm/include/llvm/CodeGen/CommandFlags.h | 2 -- llvm/include/llvm/MC/MCAsmInfo.h | 16 ---------- llvm/include/llvm/MC/MCContext.h | 2 ++ llvm/include/llvm/MC/MCTargetOptions.h | 7 +++++ .../llvm/MC/MCTargetOptionsCommandFlags.h | 2 ++ llvm/include/llvm/Target/TargetOptions.h | 29 ++++++++----------- llvm/lib/CodeGen/CommandFlags.cpp | 9 ------ llvm/lib/CodeGen/LLVMTargetMachine.cpp | 4 --- llvm/lib/LTO/LTO.cpp | 4 +-- llvm/lib/MC/ELFObjectWriter.cpp | 9 +++--- llvm/lib/MC/MCTargetOptionsCommandFlags.cpp | 9 ++++++ .../X86/MCTargetDesc/X86ELFObjectWriter.cpp | 4 +-- llvm/lib/Target/X86/X86MCInstLower.cpp | 6 ++-- .../JITLink/AArch64/ELF_minimal.s | 2 +- .../JITLink/AArch64/ELF_relocations.s | 2 +- .../x86-64/COFF_small_pic_relocations.s | 2 +- .../JITLink/x86-64/ELF_common_var.s | 2 +- llvm/test/MC/ELF/got-relaxed-i386.s | 2 +- llvm/test/MC/ELF/relocation-386.s | 4 +-- llvm/test/MC/X86/gotpcrelx.s | 2 +- llvm/tools/gold/gold-plugin.cpp | 2 +- llvm/tools/llvm-mc/llvm-mc.cpp | 12 ++------ 25 files changed, 61 insertions(+), 83 deletions(-) diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp index 056f790d4185..7310e3817c79 100644 --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -356,8 +356,6 @@ static bool initTargetOptions(DiagnosticsEngine &Diags, llvm::TargetMachine::parseBinutilsVersion(CodeGenOpts.BinutilsVersion); Options.UseInitArray = CodeGenOpts.UseInitArray; Options.DisableIntegratedAS = CodeGenOpts.DisableIntegratedAS; - Options.CompressDebugSections = CodeGenOpts.getCompressDebugSections(); - Options.RelaxELFRelocations = CodeGenOpts.RelaxELFRelocations; // Set EABI version. Options.EABIVersion = TargetOpts.EABIVersion; @@ -460,6 +458,9 @@ static bool initTargetOptions(DiagnosticsEngine &Diags, Options.MCOptions.AsmVerbose = CodeGenOpts.AsmVerbose; Options.MCOptions.Dwarf64 = CodeGenOpts.Dwarf64; Options.MCOptions.PreserveAsmComments = CodeGenOpts.PreserveAsmComments; + Options.MCOptions.X86RelaxRelocations = CodeGenOpts.RelaxELFRelocations; + Options.MCOptions.CompressDebugSections = + CodeGenOpts.getCompressDebugSections(); Options.MCOptions.ABIName = TargetOpts.ABI; for (const auto &Entry : HSOpts.UserEntries) if (!Entry.IsFramework && diff --git a/clang/tools/driver/cc1as_main.cpp b/clang/tools/driver/cc1as_main.cpp index a55e06500d9d..5498c3f9d4a2 100644 --- a/clang/tools/driver/cc1as_main.cpp +++ b/clang/tools/driver/cc1as_main.cpp @@ -428,6 +428,8 @@ static bool ExecuteAssemblerImpl(AssemblerInvocation &Opts, MCTargetOptions MCOptions; MCOptions.EmitDwarfUnwind = Opts.EmitDwarfUnwind; MCOptions.EmitCompactUnwindNonCanonical = Opts.EmitCompactUnwindNonCanonical; + MCOptions.X86RelaxRelocations = Opts.RelaxELFRelocations; + MCOptions.CompressDebugSections = Opts.CompressDebugSections; MCOptions.AsSecureLogFile = Opts.AsSecureLogFile; std::unique_ptr MAI( @@ -436,9 +438,7 @@ static bool ExecuteAssemblerImpl(AssemblerInvocation &Opts, // Ensure MCAsmInfo initialization occurs before any use, otherwise sections // may be created with a combination of default and explicit settings. - MAI->setCompressDebugSections(Opts.CompressDebugSections); - MAI->setRelaxELFRelocations(Opts.RelaxELFRelocations); bool IsBinary = Opts.OutputType == AssemblerInvocation::FT_Obj; if (Opts.OutputPath.empty()) diff --git a/lld/test/ELF/weak-undef-got-pie.s b/lld/test/ELF/weak-undef-got-pie.s index c695ecdbafb1..2301400f4e0b 100644 --- a/lld/test/ELF/weak-undef-got-pie.s +++ b/lld/test/ELF/weak-undef-got-pie.s @@ -1,7 +1,7 @@ # REQUIRES: x86 # RUN: llvm-mc -filetype=obj -triple=x86_64 %p/Inputs/dummy-shared.s -o %t1.o # RUN: ld.lld %t1.o -shared -o %t1.so -# RUN: llvm-mc -filetype=obj -relax-relocations=false -triple=x86_64 %s -o %t.o +# RUN: llvm-mc -filetype=obj -x86-relax-relocations=false -triple=x86_64 %s -o %t.o # RUN: ld.lld -pie %t.o %t1.so -o %t # RUN: llvm-readobj -r %t | FileCheck --check-prefix=RELOCS %s diff --git a/llvm/include/llvm/CodeGen/CommandFlags.h b/llvm/include/llvm/CodeGen/CommandFlags.h index d74d72450060..244dabd38cf6 100644 --- a/llvm/include/llvm/CodeGen/CommandFlags.h +++ b/llvm/include/llvm/CodeGen/CommandFlags.h @@ -98,8 +98,6 @@ bool getUseCtors(); bool getDisableIntegratedAS(); -bool getRelaxELFRelocations(); - bool getDataSections(); std::optional getExplicitDataSections(); diff --git a/llvm/include/llvm/MC/MCAsmInfo.h b/llvm/include/llvm/MC/MCAsmInfo.h index 56492368bd98..f9dd6012d5e9 100644 --- a/llvm/include/llvm/MC/MCAsmInfo.h +++ b/llvm/include/llvm/MC/MCAsmInfo.h @@ -524,17 +524,10 @@ protected: /// Preserve Comments in assembly bool PreserveAsmComments; - /// Compress DWARF debug sections. Defaults to no compression. - DebugCompressionType CompressDebugSections = DebugCompressionType::None; - /// True if the integrated assembler should interpret 'a >> b' constant /// expressions as logical rather than arithmetic. bool UseLogicalShr = true; - // If true, emit GOTPCRELX/REX_GOTPCRELX instead of GOTPCREL, on - // X86_64 ELF. - bool RelaxELFRelocations = true; - // If true, then the lexer and expression parser will support %neg(), // %hi(), and similar unary operators. bool HasMipsExpressions = false; @@ -875,18 +868,9 @@ public: PreserveAsmComments = Value; } - DebugCompressionType compressDebugSections() const { - return CompressDebugSections; - } - - void setCompressDebugSections(DebugCompressionType CompressDebugSections) { - this->CompressDebugSections = CompressDebugSections; - } bool shouldUseLogicalShr() const { return UseLogicalShr; } - bool canRelaxRelocations() const { return RelaxELFRelocations; } - void setRelaxELFRelocations(bool V) { RelaxELFRelocations = V; } bool hasMipsExpressions() const { return HasMipsExpressions; } bool needsFunctionDescriptors() const { return NeedsFunctionDescriptors; } bool shouldUseMotorolaIntegers() const { return UseMotorolaIntegers; } diff --git a/llvm/include/llvm/MC/MCContext.h b/llvm/include/llvm/MC/MCContext.h index 68d6f3e59d2d..3f585d4d2efa 100644 --- a/llvm/include/llvm/MC/MCContext.h +++ b/llvm/include/llvm/MC/MCContext.h @@ -451,6 +451,8 @@ public: const MCSubtargetInfo *getSubtargetInfo() const { return MSTI; } + const MCTargetOptions *getTargetOptions() const { return TargetOptions; } + CodeViewContext &getCVContext(); void setAllowTemporaryLabels(bool Value) { AllowTemporaryLabels = Value; } diff --git a/llvm/include/llvm/MC/MCTargetOptions.h b/llvm/include/llvm/MC/MCTargetOptions.h index a7295879e15f..0cf2806bd480 100644 --- a/llvm/include/llvm/MC/MCTargetOptions.h +++ b/llvm/include/llvm/MC/MCTargetOptions.h @@ -61,6 +61,10 @@ public: bool Dwarf64 : 1; + // If true, prefer R_X86_64_[REX_]GOTPCRELX to R_X86_64_GOTPCREL on x86-64 + // ELF. + bool X86RelaxRelocations = true; + EmitDwarfUnwindType EmitDwarfUnwind; int DwarfVersion = 0; @@ -76,6 +80,9 @@ public: }; DwarfDirectory MCUseDwarfDirectory; + // Whether to compress DWARF debug sections. + DebugCompressionType CompressDebugSections = DebugCompressionType::None; + std::string ABIName; std::string AssemblyLanguage; std::string SplitDwarfFile; diff --git a/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.h b/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.h index ba3784cab5b1..fc35eea09c4b 100644 --- a/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.h +++ b/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.h @@ -49,6 +49,8 @@ bool getNoDeprecatedWarn(); bool getNoTypeCheck(); +bool getX86RelaxRelocations(); + std::string getABIName(); std::string getAsSecureLogFile(); diff --git a/llvm/include/llvm/Target/TargetOptions.h b/llvm/include/llvm/Target/TargetOptions.h index f64cb06b2d77..d37e9d9576ba 100644 --- a/llvm/include/llvm/Target/TargetOptions.h +++ b/llvm/include/llvm/Target/TargetOptions.h @@ -141,18 +141,18 @@ namespace llvm { HonorSignDependentRoundingFPMathOption(false), NoZerosInBSS(false), GuaranteedTailCallOpt(false), StackSymbolOrdering(true), EnableFastISel(false), EnableGlobalISel(false), UseInitArray(false), - DisableIntegratedAS(false), RelaxELFRelocations(true), - FunctionSections(false), DataSections(false), - IgnoreXCOFFVisibility(false), XCOFFTracebackTable(true), - UniqueSectionNames(true), UniqueBasicBlockSectionNames(false), - TrapUnreachable(false), NoTrapAfterNoreturn(false), TLSSize(0), - EmulatedTLS(false), EnableTLSDESC(false), EnableIPRA(false), - EmitStackSizeSection(false), EnableMachineOutliner(false), - EnableMachineFunctionSplitter(false), SupportsDefaultOutlining(false), - EmitAddrsig(false), BBAddrMap(false), EmitCallSiteInfo(false), - SupportsDebugEntryValues(false), EnableDebugEntryValues(false), - ValueTrackingVariableLocations(false), ForceDwarfFrameSection(false), - XRayFunctionIndex(true), DebugStrictDwarf(false), Hotpatch(false), + DisableIntegratedAS(false), FunctionSections(false), + DataSections(false), IgnoreXCOFFVisibility(false), + XCOFFTracebackTable(true), UniqueSectionNames(true), + UniqueBasicBlockSectionNames(false), TrapUnreachable(false), + NoTrapAfterNoreturn(false), TLSSize(0), EmulatedTLS(false), + EnableTLSDESC(false), EnableIPRA(false), EmitStackSizeSection(false), + EnableMachineOutliner(false), EnableMachineFunctionSplitter(false), + SupportsDefaultOutlining(false), EmitAddrsig(false), BBAddrMap(false), + EmitCallSiteInfo(false), SupportsDebugEntryValues(false), + EnableDebugEntryValues(false), ValueTrackingVariableLocations(false), + ForceDwarfFrameSection(false), XRayFunctionIndex(true), + DebugStrictDwarf(false), Hotpatch(false), PPCGenScalarMASSEntries(false), JMCInstrument(false), EnableCFIFixup(false), MisExpect(false), XCOFFReadOnlyPointers(false), FPDenormalMode(DenormalMode::IEEE, DenormalMode::IEEE) {} @@ -260,11 +260,6 @@ namespace llvm { /// Disable the integrated assembler. unsigned DisableIntegratedAS : 1; - /// Compress DWARF debug sections. - DebugCompressionType CompressDebugSections = DebugCompressionType::None; - - unsigned RelaxELFRelocations : 1; - /// Emit functions into separate sections. unsigned FunctionSections : 1; diff --git a/llvm/lib/CodeGen/CommandFlags.cpp b/llvm/lib/CodeGen/CommandFlags.cpp index d61b7082cb22..14ac4b2102c2 100644 --- a/llvm/lib/CodeGen/CommandFlags.cpp +++ b/llvm/lib/CodeGen/CommandFlags.cpp @@ -85,7 +85,6 @@ CGOPT(bool, StackRealign) CGOPT(std::string, TrapFuncName) CGOPT(bool, UseCtors) CGOPT(bool, DisableIntegratedAS) -CGOPT(bool, RelaxELFRelocations) CGOPT_EXP(bool, DataSections) CGOPT_EXP(bool, FunctionSections) CGOPT(bool, IgnoreXCOFFVisibility) @@ -362,13 +361,6 @@ codegen::RegisterCodeGenFlags::RegisterCodeGenFlags() { cl::init(false)); CGBINDOPT(UseCtors); - static cl::opt RelaxELFRelocations( - "x86-relax-relocations", - cl::desc( - "Emit GOTPCRELX/REX_GOTPCRELX instead of GOTPCREL on x86-64 ELF"), - cl::init(true)); - CGBINDOPT(RelaxELFRelocations); - static cl::opt DataSections( "data-sections", cl::desc("Emit data into separate sections"), cl::init(false)); @@ -568,7 +560,6 @@ codegen::InitTargetOptionsFromCodeGenFlags(const Triple &TheTriple) { Options.StackSymbolOrdering = getStackSymbolOrdering(); Options.UseInitArray = !getUseCtors(); Options.DisableIntegratedAS = getDisableIntegratedAS(); - Options.RelaxELFRelocations = getRelaxELFRelocations(); Options.DataSections = getExplicitDataSections().value_or(TheTriple.hasDefaultDataSections()); Options.FunctionSections = getFunctionSections(); diff --git a/llvm/lib/CodeGen/LLVMTargetMachine.cpp b/llvm/lib/CodeGen/LLVMTargetMachine.cpp index 42cabb58e518..94ab8ed442eb 100644 --- a/llvm/lib/CodeGen/LLVMTargetMachine.cpp +++ b/llvm/lib/CodeGen/LLVMTargetMachine.cpp @@ -77,10 +77,6 @@ void LLVMTargetMachine::initAsmInfo() { TmpAsmInfo->setPreserveAsmComments(Options.MCOptions.PreserveAsmComments); - TmpAsmInfo->setCompressDebugSections(Options.CompressDebugSections); - - TmpAsmInfo->setRelaxELFRelocations(Options.RelaxELFRelocations); - TmpAsmInfo->setFullRegisterNames(Options.MCOptions.PPCUseFullRegisterNames); if (Options.ExceptionModel != ExceptionHandling::None) diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index 9c93ec70da77..b58418c64a11 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -124,9 +124,9 @@ void llvm::computeLTOCacheKey( AddString(Conf.CPU); // FIXME: Hash more of Options. For now all clients initialize Options from // command-line flags (which is unsupported in production), but may set - // RelaxELFRelocations. The clang driver can also pass FunctionSections, + // X86RelaxRelocations. The clang driver can also pass FunctionSections, // DataSections and DebuggerTuning via command line flags. - AddUnsigned(Conf.Options.RelaxELFRelocations); + AddUnsigned(Conf.Options.MCOptions.X86RelaxRelocations); AddUnsigned(Conf.Options.FunctionSections); AddUnsigned(Conf.Options.DataSections); AddUnsigned((unsigned)Conf.Options.DebuggerTuning); diff --git a/llvm/lib/MC/ELFObjectWriter.cpp b/llvm/lib/MC/ELFObjectWriter.cpp index 531d29954c38..3c4d3ab9a508 100644 --- a/llvm/lib/MC/ELFObjectWriter.cpp +++ b/llvm/lib/MC/ELFObjectWriter.cpp @@ -875,11 +875,10 @@ void ELFWriter::writeSectionData(const MCAssembler &Asm, MCSection &Sec, const MCAsmLayout &Layout) { MCSectionELF &Section = static_cast(Sec); StringRef SectionName = Section.getName(); - - auto &MC = Asm.getContext(); - const auto &MAI = MC.getAsmInfo(); - - const DebugCompressionType CompressionType = MAI->compressDebugSections(); + auto &Ctx = Asm.getContext(); + const DebugCompressionType CompressionType = + Ctx.getTargetOptions() ? Ctx.getTargetOptions()->CompressDebugSections + : DebugCompressionType::None; if (CompressionType == DebugCompressionType::None || !SectionName.starts_with(".debug_")) { Asm.writeSectionData(W.OS, &Section, Layout); diff --git a/llvm/lib/MC/MCTargetOptionsCommandFlags.cpp b/llvm/lib/MC/MCTargetOptionsCommandFlags.cpp index fb8334d626cb..31bfcdc3e4e7 100644 --- a/llvm/lib/MC/MCTargetOptionsCommandFlags.cpp +++ b/llvm/lib/MC/MCTargetOptionsCommandFlags.cpp @@ -46,6 +46,7 @@ MCOPT(bool, FatalWarnings) MCOPT(bool, NoWarn) MCOPT(bool, NoDeprecatedWarn) MCOPT(bool, NoTypeCheck) +MCOPT(bool, X86RelaxRelocations) MCOPT(std::string, ABIName) MCOPT(std::string, AsSecureLogFile) @@ -122,6 +123,13 @@ llvm::mc::RegisterMCTargetOptionsFlags::RegisterMCTargetOptionsFlags() { "no-type-check", cl::desc("Suppress type errors (Wasm)")); MCBINDOPT(NoTypeCheck); + static cl::opt X86RelaxRelocations( + "x86-relax-relocations", + cl::desc( + "Emit GOTPCRELX/REX_GOTPCRELX instead of GOTPCREL on x86-64 ELF"), + cl::init(true)); + MCBINDOPT(X86RelaxRelocations); + static cl::opt ABIName( "target-abi", cl::Hidden, cl::desc("The name of the ABI to be targeted from the backend."), @@ -148,6 +156,7 @@ MCTargetOptions llvm::mc::InitMCTargetOptionsFromFlags() { Options.MCNoWarn = getNoWarn(); Options.MCNoDeprecatedWarn = getNoDeprecatedWarn(); Options.MCNoTypeCheck = getNoTypeCheck(); + Options.X86RelaxRelocations = getX86RelaxRelocations(); Options.EmitDwarfUnwind = getEmitDwarfUnwind(); Options.EmitCompactUnwindNonCanonical = getEmitCompactUnwindNonCanonical(); Options.AsSecureLogFile = getAsSecureLogFile(); diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp index 373e29bf6a83..0b2efdfc16cc 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp @@ -207,7 +207,7 @@ static unsigned getRelocType64(MCContext &Ctx, SMLoc Loc, // Older versions of ld.bfd/ld.gold/lld // do not support GOTPCRELX/REX_GOTPCRELX, // and we want to keep back-compatibility. - if (!Ctx.getAsmInfo()->canRelaxRelocations()) + if (!Ctx.getTargetOptions()->X86RelaxRelocations) return ELF::R_X86_64_GOTPCREL; switch (unsigned(Kind)) { default: @@ -259,7 +259,7 @@ static unsigned getRelocType32(MCContext &Ctx, SMLoc Loc, return ELF::R_386_GOTPC; // Older versions of ld.bfd/ld.gold/lld do not support R_386_GOT32X and we // want to maintain compatibility. - if (!Ctx.getAsmInfo()->canRelaxRelocations()) + if (!Ctx.getTargetOptions()->X86RelaxRelocations) return ELF::R_386_GOT32; return Kind == MCFixupKind(X86::reloc_signed_4byte_relax) diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp index d3b7d97a83ca..64d4d411e7b4 100644 --- a/llvm/lib/Target/X86/X86MCInstLower.cpp +++ b/llvm/lib/Target/X86/X86MCInstLower.cpp @@ -546,13 +546,13 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering, const MCSymbolRefExpr *Sym = MCSymbolRefExpr::create( MCInstLowering.GetSymbolFromOperand(MI.getOperand(3)), SRVK, Ctx); - // As of binutils 2.32, ld has a bogus TLS relaxation error when the GD/LD + // Before binutils 2.41, ld has a bogus TLS relaxation error when the GD/LD // code sequence using R_X86_64_GOTPCREL (instead of R_X86_64_GOTPCRELX) is // attempted to be relaxed to IE/LE (binutils PR24784). Work around the bug by // only using GOT when GOTPCRELX is enabled. - // TODO Delete the workaround when GOTPCRELX becomes commonplace. + // TODO Delete the workaround when rustc no longer relies on the hack bool UseGot = MMI->getModule()->getRtLibUseGOT() && - Ctx.getAsmInfo()->canRelaxRelocations(); + Ctx.getTargetOptions()->X86RelaxRelocations; if (Is64Bits) { bool NeedsPadding = SRVK == MCSymbolRefExpr::VK_TLSGD; diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_minimal.s b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_minimal.s index 2e020b74361a..ae582c9cdc56 100644 --- a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_minimal.s +++ b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_minimal.s @@ -1,5 +1,5 @@ # RUN: rm -rf %t && mkdir -p %t -# RUN: llvm-mc -triple=aarch64-unknown-linux-gnu -relax-relocations=false \ +# RUN: llvm-mc -triple=aarch64-unknown-linux-gnu -x86-relax-relocations=false \ # RUN: -position-independent -filetype=obj -o %t/elf_minimal.o %s # RUN: llvm-jitlink -noexec %t/elf_minimal.o diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_relocations.s b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_relocations.s index b260961f741d..fccef479c2c3 100644 --- a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_relocations.s +++ b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_relocations.s @@ -1,5 +1,5 @@ # RUN: rm -rf %t && mkdir -p %t -# RUN: llvm-mc -triple=aarch64-unknown-linux-gnu -relax-relocations=false \ +# RUN: llvm-mc -triple=aarch64-unknown-linux-gnu -x86-relax-relocations=false \ # RUN: -position-independent -filetype=obj -o %t/elf_reloc.o %s # RUN: llvm-jitlink -noexec \ # RUN: -abs external_data=0xdeadbeef \ diff --git a/llvm/test/ExecutionEngine/JITLink/x86-64/COFF_small_pic_relocations.s b/llvm/test/ExecutionEngine/JITLink/x86-64/COFF_small_pic_relocations.s index 38f4fb11ba01..b429b94bb3e6 100644 --- a/llvm/test/ExecutionEngine/JITLink/x86-64/COFF_small_pic_relocations.s +++ b/llvm/test/ExecutionEngine/JITLink/x86-64/COFF_small_pic_relocations.s @@ -1,5 +1,5 @@ # RUN: rm -rf %t && mkdir -p %t -# RUN: llvm-mc -triple=x86_64-windows-msvc -relax-relocations=false \ +# RUN: llvm-mc -triple=x86_64-windows-msvc -x86-relax-relocations=false \ # RUN: -position-independent -filetype=obj -o %t/coff_sm_reloc.o %s # RUN: llvm-jitlink -noexec \ # RUN: -slab-allocate 100Kb -slab-address 0xfff00000 -slab-page-size 4096 \ diff --git a/llvm/test/ExecutionEngine/JITLink/x86-64/ELF_common_var.s b/llvm/test/ExecutionEngine/JITLink/x86-64/ELF_common_var.s index faa963f3b531..8bf384576c57 100644 --- a/llvm/test/ExecutionEngine/JITLink/x86-64/ELF_common_var.s +++ b/llvm/test/ExecutionEngine/JITLink/x86-64/ELF_common_var.s @@ -1,5 +1,5 @@ # RUN: rm -rf %t && mkdir -p %t -# RUN: llvm-mc -triple=x86_64-unknown-linux -relax-relocations=false -position-independent -filetype=obj -o %t/elf_common.o %s +# RUN: llvm-mc -triple=x86_64-unknown-linux -x86-relax-relocations=false -position-independent -filetype=obj -o %t/elf_common.o %s # RUN: llvm-jitlink -entry=load_common -noexec -check %s %t/elf_common.o # # Check that common variable GOT entry is synthesized correctly. diff --git a/llvm/test/MC/ELF/got-relaxed-i386.s b/llvm/test/MC/ELF/got-relaxed-i386.s index 465b6fa07f6f..260390067b7b 100644 --- a/llvm/test/MC/ELF/got-relaxed-i386.s +++ b/llvm/test/MC/ELF/got-relaxed-i386.s @@ -1,5 +1,5 @@ // RUN: llvm-mc -filetype=obj -triple i386-pc-linux %s -o - | llvm-readobj -r - | FileCheck %s -// RUN: llvm-mc -filetype=obj -relax-relocations=false -triple i386-pc-linux %s -o - | llvm-readobj -r - | FileCheck --check-prefix=OLD %s +// RUN: llvm-mc -filetype=obj -x86-relax-relocations=false -triple i386-pc-linux %s -o - | llvm-readobj -r - | FileCheck --check-prefix=OLD %s movl mov@GOT(%ebx), %eax mull mul@GOT(%ebx) diff --git a/llvm/test/MC/ELF/relocation-386.s b/llvm/test/MC/ELF/relocation-386.s index dd252f5ff74c..524f981ed8a2 100644 --- a/llvm/test/MC/ELF/relocation-386.s +++ b/llvm/test/MC/ELF/relocation-386.s @@ -1,5 +1,5 @@ -// RUN: llvm-mc -filetype=obj -triple i386-pc-linux-gnu %s -relax-relocations=false -o - | llvm-readobj -r - | FileCheck %s --check-prefix=CHECK --check-prefix=I386 -// RUN: llvm-mc -filetype=obj -triple i386-pc-elfiamcu %s -relax-relocations=false -o - | llvm-readobj -r - | FileCheck %s --check-prefix=CHECK --check-prefix=IAMCU +// RUN: llvm-mc -filetype=obj -triple i386-pc-linux-gnu %s -x86-relax-relocations=false -o - | llvm-readobj -r - | FileCheck %s --check-prefix=CHECK --check-prefix=I386 +// RUN: llvm-mc -filetype=obj -triple i386-pc-elfiamcu %s -x86-relax-relocations=false -o - | llvm-readobj -r - | FileCheck %s --check-prefix=CHECK --check-prefix=IAMCU // RUN: not llvm-mc -filetype=obj -triple=i686 --defsym ERR=1 %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR --implicit-check-not=error: // Test that we produce the correct relocation types and that the relocations diff --git a/llvm/test/MC/X86/gotpcrelx.s b/llvm/test/MC/X86/gotpcrelx.s index 91f20c6c567a..e63e3e9a946f 100644 --- a/llvm/test/MC/X86/gotpcrelx.s +++ b/llvm/test/MC/X86/gotpcrelx.s @@ -1,6 +1,6 @@ # RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o # RUN: llvm-readobj -r %t.o | FileCheck %s --check-prefixes=CHECK,COMMON -# RUN: llvm-mc -filetype=obj -triple=x86_64 -relax-relocations=false %s -o %t1.o +# RUN: llvm-mc -filetype=obj -triple=x86_64 -x86-relax-relocations=false %s -o %t1.o # RUN: llvm-readobj -r %t1.o | FileCheck %s --check-prefixes=NORELAX,COMMON # COMMON: Relocations [ diff --git a/llvm/tools/gold/gold-plugin.cpp b/llvm/tools/gold/gold-plugin.cpp index 257832ab7671..b8a33f74bd57 100644 --- a/llvm/tools/gold/gold-plugin.cpp +++ b/llvm/tools/gold/gold-plugin.cpp @@ -857,7 +857,7 @@ static std::unique_ptr createLTO(IndexWriteCallback OnIndexWrite, // Disable the new X86 relax relocations since gold might not support them. // FIXME: Check the gold version or add a new option to enable them. - Conf.Options.RelaxELFRelocations = false; + Conf.Options.MCOptions.X86RelaxRelocations = false; // Toggle function/data sections. if (!codegen::getExplicitFunctionSections()) diff --git a/llvm/tools/llvm-mc/llvm-mc.cpp b/llvm/tools/llvm-mc/llvm-mc.cpp index cf6aaa1f0698..8eb53e440459 100644 --- a/llvm/tools/llvm-mc/llvm-mc.cpp +++ b/llvm/tools/llvm-mc/llvm-mc.cpp @@ -66,11 +66,6 @@ static cl::opt ShowEncoding("show-encoding", cl::desc("Show instruction encodings"), cl::cat(MCCategory)); -static cl::opt RelaxELFRel( - "relax-relocations", cl::init(true), - cl::desc("Emit R_X86_64_GOTPCRELX instead of R_X86_64_GOTPCREL"), - cl::cat(MCCategory)); - static cl::opt CompressDebugSections( "compress-debug-sections", cl::ValueOptional, cl::init(DebugCompressionType::None), @@ -363,9 +358,10 @@ int main(int argc, char **argv) { cl::HideUnrelatedOptions({&MCCategory, &getColorCategory()}); cl::ParseCommandLineOptions(argc, argv, "llvm machine code playground\n"); - const MCTargetOptions MCOptions = mc::InitMCTargetOptionsFromFlags(); - setDwarfDebugFlags(argc, argv); + MCTargetOptions MCOptions = mc::InitMCTargetOptionsFromFlags(); + MCOptions.CompressDebugSections = CompressDebugSections.getValue(); + setDwarfDebugFlags(argc, argv); setDwarfDebugProducer(); const char *ProgName = argv[0]; @@ -401,7 +397,6 @@ int main(int argc, char **argv) { TheTarget->createMCAsmInfo(*MRI, TripleName, MCOptions)); assert(MAI && "Unable to create target asm info!"); - MAI->setRelaxELFRelocations(RelaxELFRel); if (CompressDebugSections != DebugCompressionType::None) { if (const char *Reason = compression::getReasonIfUnsupported( compression::formatFor(CompressDebugSections))) { @@ -410,7 +405,6 @@ int main(int argc, char **argv) { return 1; } } - MAI->setCompressDebugSections(CompressDebugSections); MAI->setPreserveAsmComments(PreserveComments); // Package up features to be passed to target/subtarget -- GitLab From eae4f56cb40c5db0ec2c4e967107cba2473246f1 Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Thu, 7 Mar 2024 07:22:33 +0000 Subject: [PATCH 406/929] [SROA] Fix phi gep unfolding with an alloca not in entry block Fixes a crash reported in #83494. --- llvm/lib/Transforms/Scalar/SROA.cpp | 8 +++++-- llvm/test/Transforms/SROA/phi-gep.ll | 35 ++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp index a575dc74d8db..e11b984f13bb 100644 --- a/llvm/lib/Transforms/Scalar/SROA.cpp +++ b/llvm/lib/Transforms/Scalar/SROA.cpp @@ -4062,10 +4062,14 @@ private: bool unfoldGEPPhi(GetElementPtrInst &GEPI) { // To prevent infinitely expanding recursive phis, bail if the GEP pointer // operand (looking through the phi if it is the phi we want to unfold) is - // an instruction besides an alloca. + // an instruction besides a static alloca. PHINode *Phi = dyn_cast(GEPI.getPointerOperand()); auto IsInvalidPointerOperand = [](Value *V) { - return isa(V) && !isa(V); + if (!isa(V)) + return false; + if (auto *AI = dyn_cast(V)) + return !AI->isStaticAlloca(); + return true; }; if (Phi) { if (any_of(Phi->operands(), IsInvalidPointerOperand)) diff --git a/llvm/test/Transforms/SROA/phi-gep.ll b/llvm/test/Transforms/SROA/phi-gep.ll index 33f42af69608..c2dfa7578eb4 100644 --- a/llvm/test/Transforms/SROA/phi-gep.ll +++ b/llvm/test/Transforms/SROA/phi-gep.ll @@ -638,6 +638,41 @@ bb3: ret i1 %icmp } +define i32 @test_phi_mem2reg_alloca_not_in_entry_block(i1 %arg) { +; CHECK-LABEL: @test_phi_mem2reg_alloca_not_in_entry_block( +; CHECK-NEXT: bb: +; CHECK-NEXT: [[ALLOCA:%.*]] = alloca i64, align 8 +; CHECK-NEXT: store i64 123, ptr [[ALLOCA]], align 4 +; CHECK-NEXT: br label [[BB2:%.*]] +; CHECK: bb2: +; CHECK-NEXT: [[ALLOCA2:%.*]] = alloca i64, align 8 +; CHECK-NEXT: store i64 124, ptr [[ALLOCA]], align 4 +; CHECK-NEXT: br i1 [[ARG:%.*]], label [[BB3:%.*]], label [[BB4:%.*]] +; CHECK: bb3: +; CHECK-NEXT: br label [[BB4]] +; CHECK: bb4: +; CHECK-NEXT: [[PHI:%.*]] = phi ptr [ [[ALLOCA]], [[BB2]] ], [ [[ALLOCA2]], [[BB3]] ] +; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[PHI]], i64 1 +; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr [[GEP]], align 4 +; CHECK-NEXT: ret i32 [[LOAD]] +; +bb: + %alloca = alloca i64 + store i64 123, ptr %alloca + br label %bb2 +bb2: + %alloca2 = alloca i64 + store i64 124, ptr %alloca + br i1 %arg, label %bb3, label %bb4 +bb3: + br label %bb4 +bb4: + %phi = phi ptr [ %alloca, %bb2 ], [ %alloca2, %bb3 ] + %gep = getelementptr i32, ptr %phi, i64 1 + %load = load i32, ptr %gep + ret i32 %load +} + define i64 @test_unfold_phi_duplicate_phi_entry(ptr %arg, i8 %arg1, i1 %arg2) { ; CHECK-LABEL: @test_unfold_phi_duplicate_phi_entry( ; CHECK-NEXT: bb: -- GitLab From 02629793a4581e79cda342eaae007704b4daa8ce Mon Sep 17 00:00:00 2001 From: Maksim Panchenko Date: Wed, 6 Mar 2024 23:34:03 -0800 Subject: [PATCH 407/929] [BOLT] Add reading support for Linux kernel __bug_table section (#84082) Read __bug_table section and annotate ud2 instructions with a corresponding bug entry ID. --- bolt/lib/Rewrite/LinuxKernelRewriter.cpp | 105 +++++++++++++++-------- bolt/test/X86/linux-bug-table.s | 46 ++++++++++ 2 files changed, 116 insertions(+), 35 deletions(-) create mode 100644 bolt/test/X86/linux-bug-table.s diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp index a78397199972..964a47346592 100644 --- a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp +++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp @@ -157,6 +157,12 @@ class LinuxKernelRewriter final : public MetadataRewriter { /// Alignment of paravirtual patch structures. static constexpr size_t PARA_PATCH_ALIGN = 8; + /// Section containing Linux bug table. + ErrorOr BugTableSection = std::errc::bad_address; + + /// Size of bug_entry struct. + static constexpr size_t BUG_TABLE_ENTRY_SIZE = 12; + /// Insert an LKMarker for a given code pointer \p PC from a non-code section /// \p SectionName. void insertLKMarker(uint64_t PC, uint64_t SectionOffset, @@ -172,9 +178,6 @@ class LinuxKernelRewriter final : public MetadataRewriter { /// Process __ksymtab and __ksymtab_gpl. void processLKKSymtab(bool IsGPL = false); - /// Process special linux kernel section, __bug_table. - void processLKBugTable(); - /// Process special linux kernel section, .smp_locks. void processLKSMPLocks(); @@ -200,6 +203,8 @@ class LinuxKernelRewriter final : public MetadataRewriter { /// Paravirtual instruction patch sites. Error readParaInstructions(); + Error readBugTable(); + /// Mark instructions referenced by kernel metadata. Error markInstructions(); @@ -224,6 +229,9 @@ public: if (Error E = readParaInstructions()) return E; + if (Error E = readBugTable()) + return E; + return Error::success(); } @@ -289,7 +297,6 @@ void LinuxKernelRewriter::processLKSections() { processLKPCIFixup(); processLKKSymtab(); processLKKSymtab(true); - processLKBugTable(); processLKSMPLocks(); } @@ -356,37 +363,6 @@ void LinuxKernelRewriter::processLKKSymtab(bool IsGPL) { } } -/// Process __bug_table section. -/// This section contains information useful for kernel debugging. -/// Each entry in the section is a struct bug_entry that contains a pointer to -/// the ud2 instruction corresponding to the bug, corresponding file name (both -/// pointers use PC relative offset addressing), line number, and flags. -/// The definition of the struct bug_entry can be found in -/// `include/asm-generic/bug.h` -void LinuxKernelRewriter::processLKBugTable() { - ErrorOr SectionOrError = - BC.getUniqueSectionByName("__bug_table"); - if (!SectionOrError) - return; - - const uint64_t SectionSize = SectionOrError->getSize(); - const uint64_t SectionAddress = SectionOrError->getAddress(); - assert((SectionSize % 12) == 0 && - "The size of the __bug_table section should be a multiple of 12"); - for (uint64_t I = 0; I < SectionSize; I += 12) { - const uint64_t EntryAddress = SectionAddress + I; - ErrorOr Offset = BC.getSignedValueAtAddress(EntryAddress, 4); - assert(Offset && - "Reading valid PC-relative offset for a __bug_table entry"); - const int32_t SignedOffset = *Offset; - const uint64_t RefAddress = EntryAddress + SignedOffset; - assert(BC.getBinaryFunctionContainingAddress(RefAddress) && - "__bug_table entries should point to a function"); - - insertLKMarker(RefAddress, I, SignedOffset, true, "__bug_table"); - } -} - /// .smp_locks section contains PC-relative references to instructions with LOCK /// prefix. The prefix can be converted to NOP at boot time on non-SMP systems. void LinuxKernelRewriter::processLKSMPLocks() { @@ -1097,6 +1073,65 @@ Error LinuxKernelRewriter::readParaInstructions() { return Error::success(); } +/// Process __bug_table section. +/// This section contains information useful for kernel debugging. +/// Each entry in the section is a struct bug_entry that contains a pointer to +/// the ud2 instruction corresponding to the bug, corresponding file name (both +/// pointers use PC relative offset addressing), line number, and flags. +/// The definition of the struct bug_entry can be found in +/// `include/asm-generic/bug.h` +/// +/// NB: find_bug() uses linear search to match an address to an entry in the bug +/// table. Hence there is no need to sort entries when rewriting the table. +Error LinuxKernelRewriter::readBugTable() { + BugTableSection = BC.getUniqueSectionByName("__bug_table"); + if (!BugTableSection) + return Error::success(); + + if (BugTableSection->getSize() % BUG_TABLE_ENTRY_SIZE) + return createStringError(errc::executable_format_error, + "bug table size error"); + + const uint64_t SectionAddress = BugTableSection->getAddress(); + DataExtractor DE(BugTableSection->getContents(), BC.AsmInfo->isLittleEndian(), + BC.AsmInfo->getCodePointerSize()); + DataExtractor::Cursor Cursor(0); + uint32_t EntryID = 0; + while (Cursor && Cursor.tell() < BugTableSection->getSize()) { + const uint64_t Pos = Cursor.tell(); + const uint64_t InstAddress = + SectionAddress + Pos + (int32_t)DE.getU32(Cursor); + Cursor.seek(Pos + BUG_TABLE_ENTRY_SIZE); + + if (!Cursor) + return createStringError(errc::executable_format_error, + "out of bounds while reading __bug_table"); + + ++EntryID; + + BinaryFunction *BF = BC.getBinaryFunctionContainingAddress(InstAddress); + if (!BF && opts::Verbosity) { + BC.outs() << "BOLT-INFO: no function matches address 0x" + << Twine::utohexstr(InstAddress) + << " referenced by bug table\n"; + } + + if (BF && BC.shouldEmit(*BF)) { + MCInst *Inst = BF->getInstructionAtOffset(InstAddress - BF->getAddress()); + if (!Inst) + return createStringError(errc::executable_format_error, + "no instruction at address 0x%" PRIx64 + " referenced by bug table entry %d", + InstAddress, EntryID); + BC.MIB->addAnnotation(*Inst, "BugEntry", EntryID); + } + } + + BC.outs() << "BOLT-INFO: parsed " << EntryID << " bug table entries\n"; + + return Error::success(); +} + } // namespace std::unique_ptr diff --git a/bolt/test/X86/linux-bug-table.s b/bolt/test/X86/linux-bug-table.s new file mode 100644 index 000000000000..e8de2fb6cba7 --- /dev/null +++ b/bolt/test/X86/linux-bug-table.s @@ -0,0 +1,46 @@ +# REQUIRES: system-linux + +## Check that BOLT correctly parses the Linux kernel __bug_table section. + +# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o +# RUN: %clang %cflags -nostdlib %t.o -o %t.exe \ +# RUN: -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr,--no-pie + +## Verify bug entry bindings to instructions. + +# RUN: llvm-bolt %t.exe --print-normalized -o %t.out | FileCheck %s + +# CHECK: BOLT-INFO: Linux kernel binary detected +# CHECK: BOLT-INFO: parsed 2 bug table entries + + .text + .globl _start + .type _start, %function +_start: +# CHECK: Binary Function "_start" + nop +.L0: + ud2 +# CHECK: ud2 +# CHECK-SAME: BugEntry: 1 + nop +.L1: + ud2 +# CHECK: ud2 +# CHECK-SAME: BugEntry: 2 + ret + .size _start, .-_start + + +## Bug table. + .section __bug_table,"a",@progbits +1: + .long .L0 - . # instruction + .org 1b + 12 +2: + .long .L1 - . # instruction + .org 2b + 12 + +## Fake Linux Kernel sections. + .section __ksymtab,"a",@progbits + .section __ksymtab_gpl,"a",@progbits -- GitLab From 7c63431cc22c68742a6a42d3304fdb68431247c3 Mon Sep 17 00:00:00 2001 From: Marius Brehler Date: Thu, 7 Mar 2024 08:37:47 +0100 Subject: [PATCH 408/929] [mlir][EmitC] Introduce a `CExpression` trait (#84177) This adds a `CExpression` trait and replaces the `isCExpression()` function. --- mlir/include/mlir/Dialect/EmitC/IR/EmitC.h | 1 + mlir/include/mlir/Dialect/EmitC/IR/EmitC.td | 39 +++++++++---------- .../mlir/Dialect/EmitC/IR/EmitCTraits.h | 30 ++++++++++++++ mlir/lib/Dialect/EmitC/IR/EmitC.cpp | 3 +- .../EmitC/Transforms/FormExpressions.cpp | 2 +- .../Dialect/EmitC/Transforms/Transforms.cpp | 3 +- 6 files changed, 54 insertions(+), 24 deletions(-) create mode 100644 mlir/include/mlir/Dialect/EmitC/IR/EmitCTraits.h diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.h b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.h index 3d38744527d5..1f0df3cb336b 100644 --- a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.h +++ b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.h @@ -14,6 +14,7 @@ #define MLIR_DIALECT_EMITC_IR_EMITC_H #include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/Dialect/EmitC/IR/EmitCTraits.h" #include "mlir/IR/Builders.h" #include "mlir/IR/BuiltinOps.h" #include "mlir/IR/BuiltinTypes.h" diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td index 6bef395e94eb..02ab73fa2ca5 100644 --- a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td +++ b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td @@ -47,11 +47,14 @@ class EmitC_BinaryOp traits = []> : let assemblyFormat = "operands attr-dict `:` functional-type(operands, results)"; } +// EmitC OpTrait +def CExpression : NativeOpTrait<"emitc::CExpression">; + // Types only used in binary arithmetic operations. def IntegerIndexOrOpaqueType : AnyTypeOf<[AnyInteger, Index, EmitC_OpaqueType]>; def FloatIntegerIndexOrOpaqueType : AnyTypeOf<[AnyFloat, IntegerIndexOrOpaqueType]>; -def EmitC_AddOp : EmitC_BinaryOp<"add", []> { +def EmitC_AddOp : EmitC_BinaryOp<"add", [CExpression]> { let summary = "Addition operation"; let description = [{ With the `add` operation the arithmetic operator + (addition) can @@ -74,7 +77,7 @@ def EmitC_AddOp : EmitC_BinaryOp<"add", []> { let hasVerifier = 1; } -def EmitC_ApplyOp : EmitC_Op<"apply", []> { +def EmitC_ApplyOp : EmitC_Op<"apply", [CExpression]> { let summary = "Apply operation"; let description = [{ With the `apply` operation the operators & (address of) and * (contents of) @@ -211,7 +214,7 @@ def EmitC_BitwiseXorOp : EmitC_BinaryOp<"bitwise_xor", []> { }]; } -def EmitC_CallOpaqueOp : EmitC_Op<"call_opaque", []> { +def EmitC_CallOpaqueOp : EmitC_Op<"call_opaque", [CExpression]> { let summary = "Opaque call operation"; let description = [{ The `call_opaque` operation represents a C++ function call. The callee @@ -257,10 +260,10 @@ def EmitC_CallOpaqueOp : EmitC_Op<"call_opaque", []> { let hasVerifier = 1; } -def EmitC_CastOp : EmitC_Op<"cast", [ - DeclareOpInterfaceMethods, - SameOperandsAndResultShape - ]> { +def EmitC_CastOp : EmitC_Op<"cast", + [CExpression, + DeclareOpInterfaceMethods, + SameOperandsAndResultShape]> { let summary = "Cast operation"; let description = [{ The `cast` operation performs an explicit type conversion and is emitted @@ -284,7 +287,7 @@ def EmitC_CastOp : EmitC_Op<"cast", [ let assemblyFormat = "$source attr-dict `:` type($source) `to` type($dest)"; } -def EmitC_CmpOp : EmitC_BinaryOp<"cmp", []> { +def EmitC_CmpOp : EmitC_BinaryOp<"cmp", [CExpression]> { let summary = "Comparison operation"; let description = [{ With the `cmp` operation the comparison operators ==, !=, <, <=, >, >=, <=> @@ -355,7 +358,7 @@ def EmitC_ConstantOp : EmitC_Op<"constant", [ConstantLike]> { let hasVerifier = 1; } -def EmitC_DivOp : EmitC_BinaryOp<"div", []> { +def EmitC_DivOp : EmitC_BinaryOp<"div", [CExpression]> { let summary = "Division operation"; let description = [{ With the `div` operation the arithmetic operator / (division) can @@ -409,9 +412,8 @@ def EmitC_ExpressionOp : EmitC_Op<"expression", int32_t v7 = foo(v1 + v2) * (v3 + v4); ``` - The operations allowed within expression body are `emitc.add`, - `emitc.apply`, `emitc.call_opaque`, `emitc.cast`, `emitc.cmp`, `emitc.div`, - `emitc.mul`, `emitc.rem`, and `emitc.sub`. + The operations allowed within expression body are EmitC operations with the + CExpression trait. When specified, the optional `do_not_inline` indicates that the expression is to be emitted as seen above, i.e. as the rhs of an EmitC SSA value @@ -427,14 +429,9 @@ def EmitC_ExpressionOp : EmitC_Op<"expression", let assemblyFormat = "attr-dict (`noinline` $do_not_inline^)? `:` type($result) $region"; let extraClassDeclaration = [{ - static bool isCExpression(Operation &op) { - return isa(op); - } bool hasSideEffects() { auto predicate = [](Operation &op) { - assert(isCExpression(op) && "Expected a C expression"); + assert(op.hasTrait() && "Expected a C expression"); // Conservatively assume calls to read and write memory. if (isa(op)) return true; @@ -837,7 +834,7 @@ def EmitC_LogicalOrOp : EmitC_BinaryOp<"logical_or", []> { let assemblyFormat = "operands attr-dict `:` type(operands)"; } -def EmitC_MulOp : EmitC_BinaryOp<"mul", []> { +def EmitC_MulOp : EmitC_BinaryOp<"mul", [CExpression]> { let summary = "Multiplication operation"; let description = [{ With the `mul` operation the arithmetic operator * (multiplication) can @@ -861,7 +858,7 @@ def EmitC_MulOp : EmitC_BinaryOp<"mul", []> { let results = (outs FloatIntegerIndexOrOpaqueType); } -def EmitC_RemOp : EmitC_BinaryOp<"rem", []> { +def EmitC_RemOp : EmitC_BinaryOp<"rem", [CExpression]> { let summary = "Remainder operation"; let description = [{ With the `rem` operation the arithmetic operator % (remainder) can @@ -883,7 +880,7 @@ def EmitC_RemOp : EmitC_BinaryOp<"rem", []> { let results = (outs IntegerIndexOrOpaqueType); } -def EmitC_SubOp : EmitC_BinaryOp<"sub", []> { +def EmitC_SubOp : EmitC_BinaryOp<"sub", [CExpression]> { let summary = "Subtraction operation"; let description = [{ With the `sub` operation the arithmetic operator - (subtraction) can diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitCTraits.h b/mlir/include/mlir/Dialect/EmitC/IR/EmitCTraits.h new file mode 100644 index 000000000000..c1602dfce4b4 --- /dev/null +++ b/mlir/include/mlir/Dialect/EmitC/IR/EmitCTraits.h @@ -0,0 +1,30 @@ +//===- EmitCTraits.h - EmitC trait definitions ------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file declares C++ classes for some of the traits used in the EmitC +// dialect. +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_EMITC_IR_EMITCTRAITS_H +#define MLIR_DIALECT_EMITC_IR_EMITCTRAITS_H + +#include "mlir/IR/OpDefinition.h" + +namespace mlir { +namespace OpTrait { +namespace emitc { + +template +class CExpression : public TraitBase {}; + +} // namespace emitc +} // namespace OpTrait +} // namespace mlir + +#endif // MLIR_DIALECT_EMITC_IR_EMITCTRAITS_H diff --git a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp index 4df8149b94c9..07ee1d394287 100644 --- a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp +++ b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "mlir/Dialect/EmitC/IR/EmitC.h" +#include "mlir/Dialect/EmitC/IR/EmitCTraits.h" #include "mlir/IR/Builders.h" #include "mlir/IR/BuiltinAttributes.h" #include "mlir/IR/DialectImplementation.h" @@ -244,7 +245,7 @@ LogicalResult ExpressionOp::verify() { return emitOpError("requires yielded type to match return type"); for (Operation &op : region.front().without_terminator()) { - if (!isCExpression(op)) + if (!op.hasTrait()) return emitOpError("contains an unsupported operation"); if (op.getNumResults() != 1) return emitOpError("requires exactly one result for each operation"); diff --git a/mlir/lib/Dialect/EmitC/Transforms/FormExpressions.cpp b/mlir/lib/Dialect/EmitC/Transforms/FormExpressions.cpp index 21212155ffb2..5b03f81b305f 100644 --- a/mlir/lib/Dialect/EmitC/Transforms/FormExpressions.cpp +++ b/mlir/lib/Dialect/EmitC/Transforms/FormExpressions.cpp @@ -36,7 +36,7 @@ struct FormExpressionsPass // Wrap each C operator op with an expression op. OpBuilder builder(context); auto matchFun = [&](Operation *op) { - if (emitc::ExpressionOp::isCExpression(*op)) + if (op->hasTrait()) createExpression(op, builder); }; rootOp->walk(matchFun); diff --git a/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp b/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp index 88b691b50f32..87350ecdceaa 100644 --- a/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp +++ b/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp @@ -16,7 +16,8 @@ namespace mlir { namespace emitc { ExpressionOp createExpression(Operation *op, OpBuilder &builder) { - assert(ExpressionOp::isCExpression(*op) && "Expected a C expression"); + assert(op->hasTrait() && + "Expected a C expression"); // Create an expression yielding the value returned by op. assert(op->getNumResults() == 1 && "Expected exactly one result"); -- GitLab From 7524ad9aa7b1b5003fe554a6ac8e434d50027dfb Mon Sep 17 00:00:00 2001 From: Amara Emerson Date: Thu, 7 Mar 2024 00:10:14 -0800 Subject: [PATCH 409/929] [AArch64][GlobalISel] Fix incorrect selection of monotonic s32->s64 anyext load. This load isn't selected by tablegen due to the anyext, but wasn't generating a subreg_to_reg. Maybe it shouldn't be formed at all during the combiner but to stop crashes later in codegen select it manually for now. --- .../GISel/AArch64InstructionSelector.cpp | 9 ++--- .../GlobalISel/select-atomic-load-store.mir | 33 ++++++++++++++++--- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp index 665288379239..48b73dced09b 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp @@ -2997,13 +2997,14 @@ bool AArch64InstructionSelector::select(MachineInstr &I) { } } - if (IsZExtLoad) { - // The zextload from a smaller type to i32 should be handled by the + if (IsZExtLoad || + (isa(LdSt) && ValTy == LLT::scalar(64) && MemSizeInBits == 32)) { + // The any/zextload from a smaller type to i32 should be handled by the // importer. if (MRI.getType(LoadStore->getOperand(0).getReg()).getSizeInBits() != 64) return false; - // If we have a ZEXTLOAD then change the load's type to be a narrower reg - // and zero_extend with SUBREG_TO_REG. + // If we have an extending load then change the load's type to be a + // narrower reg and zero_extend with SUBREG_TO_REG. Register LdReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass); Register DstReg = LoadStore->getOperand(0).getReg(); LoadStore->getOperand(0).setReg(LdReg); diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-atomic-load-store.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-atomic-load-store.mir index 5787f914b965..6b4bbb85b2ec 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/select-atomic-load-store.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-atomic-load-store.mir @@ -9,6 +9,11 @@ ret i8 %v } + define i32 @anyext_load_monotonic_i32() { + %v = load atomic i32, ptr null monotonic, align 4 + ret i32 %v + } + ... --- name: load_acq_i8 @@ -25,13 +30,33 @@ body: | ; CHECK-LABEL: name: load_acq_i8 ; CHECK: liveins: $x0 - ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0 - ; CHECK: [[LDARB:%[0-9]+]]:gpr32 = LDARB [[COPY]] :: (load acquire (s8) from %ir.ptr, align 8) - ; CHECK: $w0 = COPY [[LDARB]] - ; CHECK: RET_ReallyLR implicit $w0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0 + ; CHECK-NEXT: [[LDARB:%[0-9]+]]:gpr32 = LDARB [[COPY]] :: (load acquire (s8) from %ir.ptr, align 8) + ; CHECK-NEXT: $w0 = COPY [[LDARB]] + ; CHECK-NEXT: RET_ReallyLR implicit $w0 %0:gpr(p0) = COPY $x0 %2:gpr(s32) = G_LOAD %0(p0) :: (load acquire (s8) from %ir.ptr, align 8) $w0 = COPY %2(s32) RET_ReallyLR implicit $w0 ... +--- +name: anyext_load_monotonic_i32 +legalized: true +regBankSelected: true +tracksRegLiveness: true +body: | + bb.1: + ; CHECK-LABEL: name: anyext_load_monotonic_i32 + ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $xzr + ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load monotonic (s32) from `ptr null`) + ; CHECK-NEXT: %ld:gpr64all = SUBREG_TO_REG 0, [[LDRWui]], %subreg.sub_32 + ; CHECK-NEXT: $x0 = COPY %ld + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %1:gpr(p0) = G_CONSTANT i64 0 + %ld:gpr(s64) = G_LOAD %1(p0) :: (load monotonic (s32) from `ptr null`) + $x0 = COPY %ld(s64) + RET_ReallyLR implicit $x0 + +... -- GitLab From 7a0e222a17058a311b69153d0b6f1b4459414778 Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Thu, 7 Mar 2024 08:16:52 +0000 Subject: [PATCH 410/929] Revert "Convert many LivePhysRegs uses to LiveRegUnits (#83905)" This reverts commit 2a13422b8bcee449405e3ebff957b4020805f91c. It was causing test failures on the expensive check builders. --- llvm/lib/CodeGen/ReachingDefAnalysis.cpp | 24 +++++----- .../Target/AArch64/AArch64FrameLowering.cpp | 28 ++++++----- .../Target/AMDGPU/SIOptimizeExecMasking.cpp | 8 ++-- llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp | 6 +-- llvm/lib/Target/ARM/Thumb1FrameLowering.cpp | 6 +-- .../lib/Target/SystemZ/SystemZElimCompare.cpp | 6 +-- llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp | 6 +-- .../lib/Target/SystemZ/SystemZShortenInst.cpp | 8 ++-- llvm/lib/Target/X86/X86FloatingPoint.cpp | 8 ++-- .../CodeGen/AArch64/arm64-shrink-wrapping.ll | 48 ++++++++++--------- .../AArch64/stack-probing-no-scratch-reg.mir | 38 +++++++-------- llvm/test/CodeGen/Thumb/PR35481.ll | 14 +++--- 12 files changed, 104 insertions(+), 96 deletions(-) diff --git a/llvm/lib/CodeGen/ReachingDefAnalysis.cpp b/llvm/lib/CodeGen/ReachingDefAnalysis.cpp index 07fa92889d88..61a668907be7 100644 --- a/llvm/lib/CodeGen/ReachingDefAnalysis.cpp +++ b/llvm/lib/CodeGen/ReachingDefAnalysis.cpp @@ -6,10 +6,10 @@ // //===----------------------------------------------------------------------===// -#include "llvm/CodeGen/ReachingDefAnalysis.h" -#include "llvm/ADT/SetOperations.h" #include "llvm/ADT/SmallSet.h" -#include "llvm/CodeGen/LiveRegUnits.h" +#include "llvm/ADT/SetOperations.h" +#include "llvm/CodeGen/LivePhysRegs.h" +#include "llvm/CodeGen/ReachingDefAnalysis.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/Support/Debug.h" @@ -421,9 +421,9 @@ void ReachingDefAnalysis::getLiveOuts(MachineBasicBlock *MBB, return; VisitedBBs.insert(MBB); - LiveRegUnits LiveRegs(*TRI); + LivePhysRegs LiveRegs(*TRI); LiveRegs.addLiveOuts(*MBB); - if (LiveRegs.available(PhysReg)) + if (LiveRegs.available(MBB->getParent()->getRegInfo(), PhysReg)) return; if (auto *Def = getLocalLiveOutMIDef(MBB, PhysReg)) @@ -469,11 +469,11 @@ MachineInstr *ReachingDefAnalysis::getMIOperand(MachineInstr *MI, bool ReachingDefAnalysis::isRegUsedAfter(MachineInstr *MI, MCRegister PhysReg) const { MachineBasicBlock *MBB = MI->getParent(); - LiveRegUnits LiveRegs(*TRI); + LivePhysRegs LiveRegs(*TRI); LiveRegs.addLiveOuts(*MBB); // Yes if the register is live out of the basic block. - if (!LiveRegs.available(PhysReg)) + if (!LiveRegs.available(MBB->getParent()->getRegInfo(), PhysReg)) return true; // Walk backwards through the block to see if the register is live at some @@ -481,7 +481,7 @@ bool ReachingDefAnalysis::isRegUsedAfter(MachineInstr *MI, for (MachineInstr &Last : instructionsWithoutDebug(MBB->instr_rbegin(), MBB->instr_rend())) { LiveRegs.stepBackward(Last); - if (!LiveRegs.available(PhysReg)) + if (!LiveRegs.available(MBB->getParent()->getRegInfo(), PhysReg)) return InstIds.lookup(&Last) > InstIds.lookup(MI); } return false; @@ -504,9 +504,9 @@ bool ReachingDefAnalysis::isRegDefinedAfter(MachineInstr *MI, bool ReachingDefAnalysis::isReachingDefLiveOut(MachineInstr *MI, MCRegister PhysReg) const { MachineBasicBlock *MBB = MI->getParent(); - LiveRegUnits LiveRegs(*TRI); + LivePhysRegs LiveRegs(*TRI); LiveRegs.addLiveOuts(*MBB); - if (LiveRegs.available(PhysReg)) + if (LiveRegs.available(MBB->getParent()->getRegInfo(), PhysReg)) return false; auto Last = MBB->getLastNonDebugInstr(); @@ -525,9 +525,9 @@ bool ReachingDefAnalysis::isReachingDefLiveOut(MachineInstr *MI, MachineInstr * ReachingDefAnalysis::getLocalLiveOutMIDef(MachineBasicBlock *MBB, MCRegister PhysReg) const { - LiveRegUnits LiveRegs(*TRI); + LivePhysRegs LiveRegs(*TRI); LiveRegs.addLiveOuts(*MBB); - if (LiveRegs.available(PhysReg)) + if (LiveRegs.available(MBB->getParent()->getRegInfo(), PhysReg)) return nullptr; auto Last = MBB->getLastNonDebugInstr(); diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp index 11cf42bbc80e..5cc612e89162 100644 --- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp @@ -197,7 +197,6 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/LivePhysRegs.h" -#include "llvm/CodeGen/LiveRegUnits.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" @@ -989,7 +988,7 @@ void AArch64FrameLowering::emitZeroCallUsedRegs(BitVector RegsToZero, } } -static void getLiveRegsForEntryMBB(LiveRegUnits &LiveRegs, +static void getLiveRegsForEntryMBB(LivePhysRegs &LiveRegs, const MachineBasicBlock &MBB) { const MachineFunction *MF = MBB.getParent(); LiveRegs.addLiveIns(MBB); @@ -1019,15 +1018,16 @@ static Register findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB) { const AArch64Subtarget &Subtarget = MF->getSubtarget(); const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo(); - LiveRegUnits LiveRegs(TRI); + LivePhysRegs LiveRegs(TRI); getLiveRegsForEntryMBB(LiveRegs, *MBB); // Prefer X9 since it was historically used for the prologue scratch reg. - if (LiveRegs.available(AArch64::X9)) + const MachineRegisterInfo &MRI = MF->getRegInfo(); + if (LiveRegs.available(MRI, AArch64::X9)) return AArch64::X9; - for (Register Reg : AArch64::GPR64RegClass) { - if (LiveRegs.available(Reg)) + for (unsigned Reg : AArch64::GPR64RegClass) { + if (LiveRegs.available(MRI, Reg)) return Reg; } return AArch64::NoRegister; @@ -1044,11 +1044,13 @@ bool AArch64FrameLowering::canUseAsPrologue( if (AFI->hasSwiftAsyncContext()) { const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo(); - LiveRegUnits LiveRegs(TRI); + const MachineRegisterInfo &MRI = MF->getRegInfo(); + LivePhysRegs LiveRegs(TRI); getLiveRegsForEntryMBB(LiveRegs, MBB); // The StoreSwiftAsyncContext clobbers X16 and X17. Make sure they are // available. - if (!LiveRegs.available(AArch64::X16) || !LiveRegs.available(AArch64::X17)) + if (!LiveRegs.available(MRI, AArch64::X16) || + !LiveRegs.available(MRI, AArch64::X17)) return false; } @@ -1601,7 +1603,7 @@ static void emitDefineCFAWithFP(MachineFunction &MF, MachineBasicBlock &MBB, /// Collect live registers from the end of \p MI's parent up to (including) \p /// MI in \p LiveRegs. static void getLivePhysRegsUpTo(MachineInstr &MI, const TargetRegisterInfo &TRI, - LiveRegUnits &LiveRegs) { + LivePhysRegs &LiveRegs) { MachineBasicBlock &MBB = *MI.getParent(); LiveRegs.addLiveOuts(MBB); @@ -1639,7 +1641,7 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, NonFrameStart->getFlag(MachineInstr::FrameSetup)) ++NonFrameStart; - LiveRegUnits LiveRegs(*TRI); + LivePhysRegs LiveRegs(*TRI); if (NonFrameStart != MBB.end()) { getLivePhysRegsUpTo(*NonFrameStart, *TRI, LiveRegs); // Ignore registers used for stack management for now. @@ -1657,7 +1659,7 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, make_range(MBB.instr_begin(), NonFrameStart->getIterator())) { for (auto &Op : MI.operands()) if (Op.isReg() && Op.isDef()) - assert(LiveRegs.available(Op.getReg()) && + assert(!LiveRegs.contains(Op.getReg()) && "live register clobbered by inserted prologue instructions"); } }); @@ -4012,7 +4014,7 @@ MachineBasicBlock::iterator tryMergeAdjacentSTG(MachineBasicBlock::iterator II, // FIXME : This approach of bailing out from merge is conservative in // some ways like even if stg loops are not present after merge the // insert list, this liveness check is done (which is not needed). - LiveRegUnits LiveRegs(*(MBB->getParent()->getSubtarget().getRegisterInfo())); + LivePhysRegs LiveRegs(*(MBB->getParent()->getSubtarget().getRegisterInfo())); LiveRegs.addLiveOuts(*MBB); for (auto I = MBB->rbegin();; ++I) { MachineInstr &MI = *I; @@ -4021,7 +4023,7 @@ MachineBasicBlock::iterator tryMergeAdjacentSTG(MachineBasicBlock::iterator II, LiveRegs.stepBackward(*I); } InsertI++; - if (!LiveRegs.available(AArch64::NZCV)) + if (LiveRegs.contains(AArch64::NZCV)) return InsertI; llvm::stable_sort(Instrs, diff --git a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp index d510e7295125..e3f54d01eb22 100644 --- a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp +++ b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp @@ -11,7 +11,7 @@ #include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "SIRegisterInfo.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/CodeGen/LiveRegUnits.h" +#include "llvm/CodeGen/LivePhysRegs.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/TargetRegisterInfo.h" @@ -313,7 +313,7 @@ MachineBasicBlock::reverse_iterator SIOptimizeExecMasking::findExecCopy( return E; } -// XXX - Seems LiveRegUnits doesn't work correctly since it will incorrectly +// XXX - Seems LivePhysRegs doesn't work correctly since it will incorrectly // report the register as unavailable because a super-register with a lane mask // is unavailable. static bool isLiveOut(const MachineBasicBlock &MBB, unsigned Reg) { @@ -383,7 +383,7 @@ bool SIOptimizeExecMasking::isRegisterInUseBetween(MachineInstr &Stop, MCRegister Reg, bool UseLiveOuts, bool IgnoreStart) const { - LiveRegUnits LR(*TRI); + LivePhysRegs LR(*TRI); if (UseLiveOuts) LR.addLiveOuts(*Stop.getParent()); @@ -396,7 +396,7 @@ bool SIOptimizeExecMasking::isRegisterInUseBetween(MachineInstr &Stop, LR.stepBackward(*A); } - return !LR.available(Reg); + return !LR.available(*MRI, Reg); } // Determine if a register Reg is not re-defined and still in use diff --git a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp index 9bcf00079744..6121055eb021 100644 --- a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp +++ b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp @@ -31,7 +31,7 @@ #include "llvm/ADT/Statistic.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/CodeGen/LiveRegUnits.h" +#include "llvm/CodeGen/LivePhysRegs.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFrameInfo.h" @@ -109,7 +109,7 @@ namespace { const ARMSubtarget *STI; const TargetLowering *TL; ARMFunctionInfo *AFI; - LiveRegUnits LiveRegs; + LivePhysRegs LiveRegs; RegisterClassInfo RegClassInfo; MachineBasicBlock::const_iterator LiveRegPos; bool LiveRegsValid; @@ -589,7 +589,7 @@ unsigned ARMLoadStoreOpt::findFreeReg(const TargetRegisterClass &RegClass) { } for (unsigned Reg : RegClassInfo.getOrder(&RegClass)) - if (LiveRegs.available(Reg)) + if (LiveRegs.available(MF->getRegInfo(), Reg)) return Reg; return 0; } diff --git a/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp b/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp index a8cf036f363c..0f4ece64bff5 100644 --- a/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp +++ b/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp @@ -612,11 +612,11 @@ bool Thumb1FrameLowering::needPopSpecialFixUp(const MachineFunction &MF) const { static void findTemporariesForLR(const BitVector &GPRsNoLRSP, const BitVector &PopFriendly, - const LiveRegUnits &UsedRegs, unsigned &PopReg, + const LivePhysRegs &UsedRegs, unsigned &PopReg, unsigned &TmpReg, MachineRegisterInfo &MRI) { PopReg = TmpReg = 0; for (auto Reg : GPRsNoLRSP.set_bits()) { - if (UsedRegs.available(Reg)) { + if (UsedRegs.available(MRI, Reg)) { // Remember the first pop-friendly register and exit. if (PopFriendly.test(Reg)) { PopReg = Reg; @@ -684,7 +684,7 @@ bool Thumb1FrameLowering::emitPopSpecialFixUp(MachineBasicBlock &MBB, // Look for a temporary register to use. // First, compute the liveness information. const TargetRegisterInfo &TRI = *STI.getRegisterInfo(); - LiveRegUnits UsedRegs(TRI); + LivePhysRegs UsedRegs(TRI); UsedRegs.addLiveOuts(MBB); // The semantic of pristines changed recently and now, // the callee-saved registers that are touched in the function diff --git a/llvm/lib/Target/SystemZ/SystemZElimCompare.cpp b/llvm/lib/Target/SystemZ/SystemZElimCompare.cpp index e58f50e471fc..7423ed429ffb 100644 --- a/llvm/lib/Target/SystemZ/SystemZElimCompare.cpp +++ b/llvm/lib/Target/SystemZ/SystemZElimCompare.cpp @@ -18,7 +18,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringRef.h" -#include "llvm/CodeGen/LiveRegUnits.h" +#include "llvm/CodeGen/LivePhysRegs.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" @@ -690,9 +690,9 @@ bool SystemZElimCompare::processBlock(MachineBasicBlock &MBB) { // Walk backwards through the block looking for comparisons, recording // all CC users as we go. The subroutines can delete Compare and // instructions before it. - LiveRegUnits LiveRegs(*TRI); + LivePhysRegs LiveRegs(*TRI); LiveRegs.addLiveOuts(MBB); - bool CompleteCCUsers = LiveRegs.available(SystemZ::CC); + bool CompleteCCUsers = !LiveRegs.contains(SystemZ::CC); SmallVector CCUsers; MachineBasicBlock::iterator MBBI = MBB.end(); while (MBBI != MBB.begin()) { diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp index 53e9bf9a9d1b..046a12208467 100644 --- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp +++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp @@ -18,7 +18,7 @@ #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/LiveInterval.h" #include "llvm/CodeGen/LiveIntervals.h" -#include "llvm/CodeGen/LiveRegUnits.h" +#include "llvm/CodeGen/LivePhysRegs.h" #include "llvm/CodeGen/LiveVariables.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFrameInfo.h" @@ -1874,9 +1874,9 @@ prepareCompareSwapOperands(MachineBasicBlock::iterator const MBBI) const { } } if (CCLive) { - LiveRegUnits LiveRegs(*MBB->getParent()->getSubtarget().getRegisterInfo()); + LivePhysRegs LiveRegs(*MBB->getParent()->getSubtarget().getRegisterInfo()); LiveRegs.addLiveOuts(*MBB); - if (!LiveRegs.available(SystemZ::CC)) + if (LiveRegs.contains(SystemZ::CC)) return false; } diff --git a/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp b/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp index c0adfdbf120b..30b22fa1ce92 100644 --- a/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp +++ b/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp @@ -13,7 +13,7 @@ //===----------------------------------------------------------------------===// #include "SystemZTargetMachine.h" -#include "llvm/CodeGen/LiveRegUnits.h" +#include "llvm/CodeGen/LivePhysRegs.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/TargetRegisterInfo.h" @@ -46,7 +46,7 @@ private: const SystemZInstrInfo *TII; const TargetRegisterInfo *TRI; - LiveRegUnits LiveRegs; + LivePhysRegs LiveRegs; }; char SystemZShortenInst::ID = 0; @@ -88,7 +88,7 @@ bool SystemZShortenInst::shortenIIF(MachineInstr &MI, unsigned LLIxL, unsigned GR64BitReg = TRI->getMatchingSuperReg(Reg, thisSubRegIdx, &SystemZ::GR64BitRegClass); Register OtherReg = TRI->getSubReg(GR64BitReg, otherSubRegIdx); - if (!LiveRegs.available(OtherReg)) + if (LiveRegs.contains(OtherReg)) return false; uint64_t Imm = MI.getOperand(1).getImm(); @@ -143,7 +143,7 @@ bool SystemZShortenInst::shortenOn001(MachineInstr &MI, unsigned Opcode) { // Calls shortenOn001 if CCLive is false. CC def operand is added in // case of success. bool SystemZShortenInst::shortenOn001AddCC(MachineInstr &MI, unsigned Opcode) { - if (LiveRegs.available(SystemZ::CC) && shortenOn001(MI, Opcode)) { + if (!LiveRegs.contains(SystemZ::CC) && shortenOn001(MI, Opcode)) { MachineInstrBuilder(*MI.getParent()->getParent(), &MI) .addReg(SystemZ::CC, RegState::ImplicitDefine | RegState::Dead); return true; diff --git a/llvm/lib/Target/X86/X86FloatingPoint.cpp b/llvm/lib/Target/X86/X86FloatingPoint.cpp index 260879ffaa4f..ca4d03913d09 100644 --- a/llvm/lib/Target/X86/X86FloatingPoint.cpp +++ b/llvm/lib/Target/X86/X86FloatingPoint.cpp @@ -30,7 +30,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/EdgeBundles.h" -#include "llvm/CodeGen/LiveRegUnits.h" +#include "llvm/CodeGen/LivePhysRegs.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" @@ -1751,7 +1751,7 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &Inst) { void FPS::setKillFlags(MachineBasicBlock &MBB) const { const TargetRegisterInfo &TRI = *MBB.getParent()->getSubtarget().getRegisterInfo(); - LiveRegUnits LPR(TRI); + LivePhysRegs LPR(TRI); LPR.addLiveOuts(MBB); @@ -1773,14 +1773,14 @@ void FPS::setKillFlags(MachineBasicBlock &MBB) const { if (MO.isDef()) { Defs.set(Reg); - if (LPR.available(MO.getReg())) + if (!LPR.contains(MO.getReg())) MO.setIsDead(); } else Uses.push_back(&MO); } for (auto *MO : Uses) - if (Defs.test(getFPReg(*MO)) || LPR.available(MO->getReg())) + if (Defs.test(getFPReg(*MO)) || !LPR.contains(MO->getReg())) MO->setIsKill(); LPR.stepBackward(MI); diff --git a/llvm/test/CodeGen/AArch64/arm64-shrink-wrapping.ll b/llvm/test/CodeGen/AArch64/arm64-shrink-wrapping.ll index a5fcbf764b64..5806bcf0dacf 100644 --- a/llvm/test/CodeGen/AArch64/arm64-shrink-wrapping.ll +++ b/llvm/test/CodeGen/AArch64/arm64-shrink-wrapping.ll @@ -1028,22 +1028,14 @@ false: ret i32 %tmp.0 } -; Re-aligned stack pointer with all caller-save regs live. +; Re-aligned stack pointer with all caller-save regs live. See bug +; 26642. In this case we currently avoid shrink wrapping because +; ensuring we have a scratch register to re-align the stack pointer is +; too complicated. Output should be the same for both enabled and +; disabled shrink wrapping. define void @stack_realign2(i32 %a, i32 %b, ptr %ptr1, ptr %ptr2, ptr %ptr3, ptr %ptr4, ptr %ptr5, ptr %ptr6) { ; ENABLE-LABEL: stack_realign2: ; ENABLE: ; %bb.0: -; ENABLE-NEXT: lsl w8, w1, w0 -; ENABLE-NEXT: lsr w9, w0, w1 -; ENABLE-NEXT: lsl w14, w0, w1 -; ENABLE-NEXT: lsr w11, w1, w0 -; ENABLE-NEXT: add w15, w1, w0 -; ENABLE-NEXT: sub w10, w8, w9 -; ENABLE-NEXT: subs w17, w1, w0 -; ENABLE-NEXT: add w16, w14, w8 -; ENABLE-NEXT: add w12, w9, w11 -; ENABLE-NEXT: add w13, w11, w15 -; ENABLE-NEXT: b.le LBB14_2 -; ENABLE-NEXT: ; %bb.1: ; %true ; ENABLE-NEXT: stp x28, x27, [sp, #-96]! ; 16-byte Folded Spill ; ENABLE-NEXT: stp x26, x25, [sp, #16] ; 16-byte Folded Spill ; ENABLE-NEXT: stp x24, x23, [sp, #32] ; 16-byte Folded Spill @@ -1051,8 +1043,8 @@ define void @stack_realign2(i32 %a, i32 %b, ptr %ptr1, ptr %ptr2, ptr %ptr3, ptr ; ENABLE-NEXT: stp x20, x19, [sp, #64] ; 16-byte Folded Spill ; ENABLE-NEXT: stp x29, x30, [sp, #80] ; 16-byte Folded Spill ; ENABLE-NEXT: add x29, sp, #80 -; ENABLE-NEXT: sub x18, sp, #32 -; ENABLE-NEXT: and sp, x18, #0xffffffffffffffe0 +; ENABLE-NEXT: sub x9, sp, #32 +; ENABLE-NEXT: and sp, x9, #0xffffffffffffffe0 ; ENABLE-NEXT: .cfi_def_cfa w29, 16 ; ENABLE-NEXT: .cfi_offset w30, -8 ; ENABLE-NEXT: .cfi_offset w29, -16 @@ -1066,17 +1058,22 @@ define void @stack_realign2(i32 %a, i32 %b, ptr %ptr1, ptr %ptr2, ptr %ptr3, ptr ; ENABLE-NEXT: .cfi_offset w26, -80 ; ENABLE-NEXT: .cfi_offset w27, -88 ; ENABLE-NEXT: .cfi_offset w28, -96 +; ENABLE-NEXT: lsl w8, w1, w0 +; ENABLE-NEXT: lsr w9, w0, w1 +; ENABLE-NEXT: lsl w14, w0, w1 +; ENABLE-NEXT: lsr w11, w1, w0 +; ENABLE-NEXT: add w15, w1, w0 +; ENABLE-NEXT: sub w10, w8, w9 +; ENABLE-NEXT: subs w17, w1, w0 +; ENABLE-NEXT: add w16, w14, w8 +; ENABLE-NEXT: add w12, w9, w11 +; ENABLE-NEXT: add w13, w11, w15 +; ENABLE-NEXT: b.le LBB14_2 +; ENABLE-NEXT: ; %bb.1: ; %true ; ENABLE-NEXT: str w0, [sp] ; ENABLE-NEXT: ; InlineAsm Start ; ENABLE-NEXT: nop ; ENABLE-NEXT: ; InlineAsm End -; ENABLE-NEXT: sub sp, x29, #80 -; ENABLE-NEXT: ldp x29, x30, [sp, #80] ; 16-byte Folded Reload -; ENABLE-NEXT: ldp x20, x19, [sp, #64] ; 16-byte Folded Reload -; ENABLE-NEXT: ldp x22, x21, [sp, #48] ; 16-byte Folded Reload -; ENABLE-NEXT: ldp x24, x23, [sp, #32] ; 16-byte Folded Reload -; ENABLE-NEXT: ldp x26, x25, [sp, #16] ; 16-byte Folded Reload -; ENABLE-NEXT: ldp x28, x27, [sp], #96 ; 16-byte Folded Reload ; ENABLE-NEXT: LBB14_2: ; %false ; ENABLE-NEXT: str w14, [x2] ; ENABLE-NEXT: str w8, [x3] @@ -1087,6 +1084,13 @@ define void @stack_realign2(i32 %a, i32 %b, ptr %ptr1, ptr %ptr2, ptr %ptr3, ptr ; ENABLE-NEXT: stp w0, w1, [x2, #4] ; ENABLE-NEXT: stp w16, w10, [x2, #12] ; ENABLE-NEXT: stp w12, w13, [x2, #20] +; ENABLE-NEXT: sub sp, x29, #80 +; ENABLE-NEXT: ldp x29, x30, [sp, #80] ; 16-byte Folded Reload +; ENABLE-NEXT: ldp x20, x19, [sp, #64] ; 16-byte Folded Reload +; ENABLE-NEXT: ldp x22, x21, [sp, #48] ; 16-byte Folded Reload +; ENABLE-NEXT: ldp x24, x23, [sp, #32] ; 16-byte Folded Reload +; ENABLE-NEXT: ldp x26, x25, [sp, #16] ; 16-byte Folded Reload +; ENABLE-NEXT: ldp x28, x27, [sp], #96 ; 16-byte Folded Reload ; ENABLE-NEXT: ret ; ; DISABLE-LABEL: stack_realign2: diff --git a/llvm/test/CodeGen/AArch64/stack-probing-no-scratch-reg.mir b/llvm/test/CodeGen/AArch64/stack-probing-no-scratch-reg.mir index 078d8a5bf6b6..f50bd9ab4b8a 100644 --- a/llvm/test/CodeGen/AArch64/stack-probing-no-scratch-reg.mir +++ b/llvm/test/CodeGen/AArch64/stack-probing-no-scratch-reg.mir @@ -43,43 +43,43 @@ machineFunctionInfo: {} body: | ; CHECK-LABEL: name: f ; CHECK: bb.0.entry: - ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) - ; CHECK-NEXT: liveins: $w0, $lr - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: $x9 = IMPLICIT_DEF - ; CHECK-NEXT: dead $wzr = SUBSWri killed renamable $w0, 1, 0, implicit-def $nzcv - ; CHECK-NEXT: Bcc 12, %bb.2, implicit $nzcv - ; CHECK-NEXT: B %bb.1 - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: bb.1.if.then1: ; CHECK-NEXT: successors: %bb.3(0x80000000) - ; CHECK-NEXT: liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8, $x9, $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17, $x18, $x19, $x20, $x21, $x22, $x23, $x23, $x25, $x25, $x27, $x28, $lr + ; CHECK-NEXT: liveins: $w0, $lr ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: early-clobber $sp = frame-setup STPXpre killed $fp, killed $lr, $sp, -2 :: (store (s64) into %stack.2), (store (s64) into %stack.1) - ; CHECK-NEXT: $xzr = frame-setup SUBXri $sp, 36, 12 + ; CHECK-NEXT: $x9 = frame-setup SUBXri $sp, 36, 12 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: bb.3.if.then1: + ; CHECK-NEXT: bb.3.entry: ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.3(0x40000000) + ; CHECK-NEXT: liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8, $x9, $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17, $x18, $x19, $x20, $x21, $x22, $x23, $x25, $x27, $x28 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: $sp = frame-setup SUBXri $sp, 1, 12 ; CHECK-NEXT: frame-setup STRXui $xzr, $sp, 0 - ; CHECK-NEXT: $xzr = frame-setup SUBSXrx64 $sp, $xzr, 24, implicit-def $nzcv + ; CHECK-NEXT: $xzr = frame-setup SUBSXrx64 $sp, $x9, 24, implicit-def $nzcv ; CHECK-NEXT: frame-setup Bcc 1, %bb.3, implicit $nzcv ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: bb.4.if.then1: - ; CHECK-NEXT: successors: %bb.2(0x80000000) + ; CHECK-NEXT: bb.4.entry: + ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; CHECK-NEXT: liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8, $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17, $x18, $x19, $x20, $x21, $x22, $x23, $x25, $x27, $x28 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: $sp = frame-setup SUBXri $sp, 2544, 0 ; CHECK-NEXT: frame-setup STRXui $xzr, $sp, 0 + ; CHECK-NEXT: $x9 = IMPLICIT_DEF + ; CHECK-NEXT: dead $wzr = SUBSWri killed renamable $w0, 1, 0, implicit-def $nzcv + ; CHECK-NEXT: Bcc 12, %bb.2, implicit $nzcv + ; CHECK-NEXT: B %bb.1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1.if.then1: + ; CHECK-NEXT: successors: %bb.2(0x80000000) + ; CHECK-NEXT: liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8, $x9, $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17, $x18, $x19, $x20, $x21, $x22, $x23, $x23, $x25, $x25, $x27, $x28 + ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: $x0 = ADDXri $sp, 0, 0 ; CHECK-NEXT: BL @g, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $x0, implicit-def $sp + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2.exit: ; CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 36, 12 ; CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 2544, 0 ; CHECK-NEXT: early-clobber $sp, $fp, $lr = frame-destroy LDPXpost $sp, 2 :: (load (s64) from %stack.2), (load (s64) from %stack.1) - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: bb.2.exit: - ; CHECK-NEXT: liveins: $lr - ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: RET_ReallyLR bb.0.entry: successors: %bb.1(0x40000000), %bb.2(0x40000000) diff --git a/llvm/test/CodeGen/Thumb/PR35481.ll b/llvm/test/CodeGen/Thumb/PR35481.ll index e48d1547782c..ad3215ecb949 100644 --- a/llvm/test/CodeGen/Thumb/PR35481.ll +++ b/llvm/test/CodeGen/Thumb/PR35481.ll @@ -18,10 +18,11 @@ define <4 x i32> @f() local_unnamed_addr #0 { ; CHECK-V4T-NEXT: movs r2, #3 ; CHECK-V4T-NEXT: movs r3, #4 ; CHECK-V4T-NEXT: bl g -; CHECK-V4T-NEXT: ldr r7, [sp, #4] -; CHECK-V4T-NEXT: mov lr, r7 ; CHECK-V4T-NEXT: pop {r7} -; CHECK-V4T-NEXT: add sp, #4 +; CHECK-V4T-NEXT: mov r12, r0 +; CHECK-V4T-NEXT: pop {r0} +; CHECK-V4T-NEXT: mov lr, r0 +; CHECK-V4T-NEXT: mov r0, r12 ; CHECK-V4T-NEXT: bx lr ; ; CHECK-V8M-LABEL: f: @@ -35,10 +36,11 @@ define <4 x i32> @f() local_unnamed_addr #0 { ; CHECK-V8M-NEXT: movs r1, #2 ; CHECK-V8M-NEXT: movs r2, #3 ; CHECK-V8M-NEXT: movs r3, #4 -; CHECK-V8M-NEXT: ldr r7, [sp, #4] -; CHECK-V8M-NEXT: mov lr, r7 ; CHECK-V8M-NEXT: pop {r7} -; CHECK-V8M-NEXT: add sp, #4 +; CHECK-V8M-NEXT: mov r12, r0 +; CHECK-V8M-NEXT: pop {r0} +; CHECK-V8M-NEXT: mov lr, r0 +; CHECK-V8M-NEXT: mov r0, r12 ; CHECK-V8M-NEXT: b g entry: %call = tail call i32 @h(i32 1) -- GitLab From 91808c845fd6f0624525b6d6348b2c284628ce58 Mon Sep 17 00:00:00 2001 From: Michal Paszkowski Date: Thu, 7 Mar 2024 00:22:13 -0800 Subject: [PATCH 411/929] [docs] Change SPIR-V backend meeting day (#84286) Changing SPIR-V backend meeting day and removing my office hours --- llvm/docs/GettingInvolved.rst | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/llvm/docs/GettingInvolved.rst b/llvm/docs/GettingInvolved.rst index f89483904ab7..763aeb87c688 100644 --- a/llvm/docs/GettingInvolved.rst +++ b/llvm/docs/GettingInvolved.rst @@ -215,7 +215,7 @@ what to add to your calendar invite. - `gcal `__ - `Minutes/docs `__ * - LLVM SPIR-V Backend Working Group - - Every week on Thursday + - Every week on Monday - - `Meeting details/agenda `__ * - SYCL Upstream Working Group @@ -305,11 +305,6 @@ The :doc:`CodeOfConduct` applies to all office hours. - Monthly, 2nd Wednesday of the month at 11:00am PT, for 30 minutes. - `Zoom `__ - English, Russian - * - Michal Paszkowski - - SPIR-V backend, IGC, OpenCL, and IR transformations - - Monthly, 3rd Thursday of the month at 21:00 Warsaw/Poland time, 1 hour slot. - - `MS Teams `__ - - English, Polish * - Quentin Colombet (he/him) - LLVM/MLIR; Codegen (Instruction selection (GlobalISel/SDISel), Machine IR, Register allocation, etc.); Optimizations; MCA -- GitLab From 99500e8c08a4d941acb8a7eb00523296fb2acf7a Mon Sep 17 00:00:00 2001 From: Mariya Podchishchaeva Date: Thu, 7 Mar 2024 11:36:50 +0300 Subject: [PATCH 412/929] [Clang][C++23] Implement P2448R2: Relaxing some constexpr restrictions (#77753) Per https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p2448r2.html function/constructor/destructor can be marked `constexpr` even though it never produces a constant expression. Non-literal types as return types and parameter types of functions marked `constexpr` are also allowed. Since this is not a DR, the diagnostic messages are still preserved for C++ standards older than C++23. --- clang/docs/ReleaseNotes.rst | 2 + .../clang/Basic/DiagnosticSemaKinds.td | 26 +-- clang/lib/AST/DeclCXX.cpp | 13 +- clang/lib/Sema/SemaDeclCXX.cpp | 93 +++++----- clang/test/AST/Interp/cxx23.cpp | 59 ++----- .../class.compare.default/p3.cpp | 40 ++--- .../class.compare.default/p4.cpp | 20 +-- .../dcl.dcl/dcl.spec/dcl.constexpr/dtor.cpp | 8 +- .../dcl.dcl/dcl.spec/dcl.constexpr/p3-2b.cpp | 10 +- .../CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3.cpp | 18 +- .../CXX/dcl.dcl/dcl.spec/dcl.constexpr/p4.cpp | 8 +- .../dcl.fct.def/dcl.fct.def.default/p2.cpp | 6 +- clang/test/CXX/drs/dr13xx.cpp | 22 +-- clang/test/CXX/drs/dr14xx.cpp | 6 +- clang/test/CXX/drs/dr15xx.cpp | 21 ++- clang/test/CXX/drs/dr16xx.cpp | 20 +-- clang/test/CXX/drs/dr6xx.cpp | 24 +-- clang/test/CXX/expr/expr.const/p5-26.cpp | 4 +- clang/test/CXX/special/class.copy/p13-0x.cpp | 2 +- .../SemaCXX/constant-expression-cxx11.cpp | 38 +++-- .../SemaCXX/constant-expression-cxx14.cpp | 33 ++-- .../SemaCXX/constant-expression-cxx2b.cpp | 24 +-- .../test/SemaCXX/cxx23-invalid-constexpr.cpp | 159 ++++++++++++++++++ clang/test/SemaCXX/cxx2a-consteval.cpp | 2 +- .../SemaCXX/deduced-return-type-cxx14.cpp | 8 +- .../addrspace-constructors.clcpp | 2 +- clang/www/cxx_status.html | 9 +- 27 files changed, 408 insertions(+), 269 deletions(-) create mode 100644 clang/test/SemaCXX/cxx23-invalid-constexpr.cpp diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 8300a8484585..1b901a27fd19 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -97,6 +97,8 @@ C++23 Feature Support - Implemented `P2718R0: Lifetime extension in range-based for loops `_. Also materialize temporary object which is a prvalue in discarded-value expression. +- Implemented `P2448R2: Relaxing some constexpr restrictions `_. + C++2c Feature Support ^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index 5a90e631a894..c8dfdc08f5ea 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -9607,13 +9607,10 @@ def err_defaulted_copy_assign_not_ref : Error< "the parameter for an explicitly-defaulted copy assignment operator must be an " "lvalue reference type">; def err_incorrect_defaulted_constexpr : Error< - "defaulted definition of %sub{select_special_member_kind}0 " - "is not constexpr">; + "defaulted definition of %sub{select_special_member_kind}0 cannot be marked %select{constexpr|consteval}1 " + "before C++23">; def err_incorrect_defaulted_constexpr_with_vb: Error< "%sub{select_special_member_kind}0 cannot be 'constexpr' in a class with virtual base class">; -def err_incorrect_defaulted_consteval : Error< - "defaulted declaration of %sub{select_special_member_kind}0 " - "cannot be consteval because implicit definition is not constexpr">; def warn_defaulted_method_deleted : Warning< "explicitly defaulted %sub{select_special_member_kind}0 is implicitly " "deleted">, InGroup; @@ -9724,21 +9721,12 @@ def note_defaulted_comparison_cannot_deduce_undeduced_auto : Note< "%select{|member|base class}0 %1 declared here">; def note_defaulted_comparison_cannot_deduce_callee : Note< "selected 'operator<=>' for %select{|member|base class}0 %1 declared here">; -def ext_defaulted_comparison_constexpr_mismatch : Extension< +def err_defaulted_comparison_constexpr_mismatch : Error< "defaulted definition of %select{%sub{select_defaulted_comparison_kind}1|" - "three-way comparison operator}0 that is " - "declared %select{constexpr|consteval}2 but" - "%select{|for which the corresponding implicit 'operator==' }0 " - "invokes a non-constexpr comparison function is a C++23 extension">, - InGroup>; -def warn_cxx23_compat_defaulted_comparison_constexpr_mismatch : Warning< - "defaulted definition of %select{%sub{select_defaulted_comparison_kind}1|" - "three-way comparison operator}0 that is " - "declared %select{constexpr|consteval}2 but" - "%select{|for which the corresponding implicit 'operator==' }0 " - "invokes a non-constexpr comparison function is incompatible with C++ " - "standards before C++23">, - InGroup, DefaultIgnore; + "three-way comparison operator}0 cannot be " + "declared %select{constexpr|consteval}2 because " + "%select{it|for which the corresponding implicit 'operator==' }0 " + "invokes a non-constexpr comparison function ">; def note_defaulted_comparison_not_constexpr : Note< "non-constexpr comparison function would be used to compare " "%select{|member %1|base class %1}0">; diff --git a/clang/lib/AST/DeclCXX.cpp b/clang/lib/AST/DeclCXX.cpp index b4f2327d9c56..1c3dcf63465c 100644 --- a/clang/lib/AST/DeclCXX.cpp +++ b/clang/lib/AST/DeclCXX.cpp @@ -400,10 +400,11 @@ CXXRecordDecl::setBases(CXXBaseSpecifier const * const *Bases, // C++11 [class.ctor]p6: // If that user-written default constructor would satisfy the - // requirements of a constexpr constructor, the implicitly-defined - // default constructor is constexpr. + // requirements of a constexpr constructor/function(C++23), the + // implicitly-defined default constructor is constexpr. if (!BaseClassDecl->hasConstexprDefaultConstructor()) - data().DefaultedDefaultConstructorIsConstexpr = false; + data().DefaultedDefaultConstructorIsConstexpr = + C.getLangOpts().CPlusPlus23; // C++1z [class.copy]p8: // The implicitly-declared copy constructor for a class X will have @@ -548,7 +549,8 @@ void CXXRecordDecl::addedClassSubobject(CXXRecordDecl *Subobj) { // -- for every subobject of class type or (possibly multi-dimensional) // array thereof, that class type shall have a constexpr destructor if (!Subobj->hasConstexprDestructor()) - data().DefaultedDestructorIsConstexpr = false; + data().DefaultedDestructorIsConstexpr = + getASTContext().getLangOpts().CPlusPlus23; // C++20 [temp.param]p7: // A structural type is [...] a literal class type [for which] the types @@ -1297,7 +1299,8 @@ void CXXRecordDecl::addedMember(Decl *D) { !FieldRec->hasConstexprDefaultConstructor() && !isUnion()) // The standard requires any in-class initializer to be a constant // expression. We consider this to be a defect. - data().DefaultedDefaultConstructorIsConstexpr = false; + data().DefaultedDefaultConstructorIsConstexpr = + Context.getLangOpts().CPlusPlus23; // C++11 [class.copy]p8: // The implicitly-declared copy constructor for a class X will have diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp index 199f2523cfb5..e258a4f7c894 100644 --- a/clang/lib/Sema/SemaDeclCXX.cpp +++ b/clang/lib/Sema/SemaDeclCXX.cpp @@ -1715,6 +1715,8 @@ static bool CheckLiteralType(Sema &SemaRef, Sema::CheckConstexprKind Kind, static bool CheckConstexprDestructorSubobjects(Sema &SemaRef, const CXXDestructorDecl *DD, Sema::CheckConstexprKind Kind) { + assert(!SemaRef.getLangOpts().CPlusPlus23 && + "this check is obsolete for C++23"); auto Check = [&](SourceLocation Loc, QualType T, const FieldDecl *FD) { const CXXRecordDecl *RD = T->getBaseElementTypeUnsafe()->getAsCXXRecordDecl(); @@ -1746,6 +1748,8 @@ static bool CheckConstexprDestructorSubobjects(Sema &SemaRef, static bool CheckConstexprParameterTypes(Sema &SemaRef, const FunctionDecl *FD, Sema::CheckConstexprKind Kind) { + assert(!SemaRef.getLangOpts().CPlusPlus23 && + "this check is obsolete for C++23"); unsigned ArgIndex = 0; const auto *FT = FD->getType()->castAs(); for (FunctionProtoType::param_type_iterator i = FT->param_type_begin(), @@ -1767,6 +1771,8 @@ static bool CheckConstexprParameterTypes(Sema &SemaRef, /// true. If not, produce a suitable diagnostic and return false. static bool CheckConstexprReturnType(Sema &SemaRef, const FunctionDecl *FD, Sema::CheckConstexprKind Kind) { + assert(!SemaRef.getLangOpts().CPlusPlus23 && + "this check is obsolete for C++23"); if (CheckLiteralType(SemaRef, Kind, FD->getLocation(), FD->getReturnType(), diag::err_constexpr_non_literal_return, FD->isConsteval())) @@ -1856,16 +1862,18 @@ bool Sema::CheckConstexprFunctionDefinition(const FunctionDecl *NewFD, } } - // - its return type shall be a literal type; - if (!CheckConstexprReturnType(*this, NewFD, Kind)) + // - its return type shall be a literal type; (removed in C++23) + if (!getLangOpts().CPlusPlus23 && + !CheckConstexprReturnType(*this, NewFD, Kind)) return false; } if (auto *Dtor = dyn_cast(NewFD)) { // A destructor can be constexpr only if the defaulted destructor could be; // we don't need to check the members and bases if we already know they all - // have constexpr destructors. - if (!Dtor->getParent()->defaultedDestructorIsConstexpr()) { + // have constexpr destructors. (removed in C++23) + if (!getLangOpts().CPlusPlus23 && + !Dtor->getParent()->defaultedDestructorIsConstexpr()) { if (Kind == CheckConstexprKind::CheckValid) return false; if (!CheckConstexprDestructorSubobjects(*this, Dtor, Kind)) @@ -1873,8 +1881,9 @@ bool Sema::CheckConstexprFunctionDefinition(const FunctionDecl *NewFD, } } - // - each of its parameter types shall be a literal type; - if (!CheckConstexprParameterTypes(*this, NewFD, Kind)) + // - each of its parameter types shall be a literal type; (removed in C++23) + if (!getLangOpts().CPlusPlus23 && + !CheckConstexprParameterTypes(*this, NewFD, Kind)) return false; Stmt *Body = NewFD->getBody(); @@ -2457,7 +2466,8 @@ static bool CheckConstexprFunctionBody(Sema &SemaRef, const FunctionDecl *Dcl, // function", so is not checked in CheckValid mode. SmallVector Diags; if (Kind == Sema::CheckConstexprKind::Diagnose && - !Expr::isPotentialConstantExpr(Dcl, Diags)) { + !Expr::isPotentialConstantExpr(Dcl, Diags) && + !SemaRef.getLangOpts().CPlusPlus23) { SemaRef.Diag(Dcl->getLocation(), diag::ext_constexpr_function_never_constant_expr) << isa(Dcl) << Dcl->isConsteval() @@ -7535,21 +7545,23 @@ static bool defaultedSpecialMemberIsConstexpr( // C++1y [class.copy]p26: // -- [the class] is a literal type, and - if (!Ctor && !ClassDecl->isLiteral()) + if (!Ctor && !ClassDecl->isLiteral() && !S.getLangOpts().CPlusPlus23) return false; // -- every constructor involved in initializing [...] base class // sub-objects shall be a constexpr constructor; // -- the assignment operator selected to copy/move each direct base // class is a constexpr function, and - for (const auto &B : ClassDecl->bases()) { - const RecordType *BaseType = B.getType()->getAs(); - if (!BaseType) - continue; - CXXRecordDecl *BaseClassDecl = cast(BaseType->getDecl()); - if (!specialMemberIsConstexpr(S, BaseClassDecl, CSM, 0, ConstArg, - InheritedCtor, Inherited)) - return false; + if (!S.getLangOpts().CPlusPlus23) { + for (const auto &B : ClassDecl->bases()) { + const RecordType *BaseType = B.getType()->getAs(); + if (!BaseType) + continue; + CXXRecordDecl *BaseClassDecl = cast(BaseType->getDecl()); + if (!specialMemberIsConstexpr(S, BaseClassDecl, CSM, 0, ConstArg, + InheritedCtor, Inherited)) + return false; + } } // -- every constructor involved in initializing non-static data members @@ -7559,20 +7571,22 @@ static bool defaultedSpecialMemberIsConstexpr( // -- for each non-static data member of X that is of class type (or array // thereof), the assignment operator selected to copy/move that member is // a constexpr function - for (const auto *F : ClassDecl->fields()) { - if (F->isInvalidDecl()) - continue; - if (CSM == Sema::CXXDefaultConstructor && F->hasInClassInitializer()) - continue; - QualType BaseType = S.Context.getBaseElementType(F->getType()); - if (const RecordType *RecordTy = BaseType->getAs()) { - CXXRecordDecl *FieldRecDecl = cast(RecordTy->getDecl()); - if (!specialMemberIsConstexpr(S, FieldRecDecl, CSM, - BaseType.getCVRQualifiers(), - ConstArg && !F->isMutable())) + if (!S.getLangOpts().CPlusPlus23) { + for (const auto *F : ClassDecl->fields()) { + if (F->isInvalidDecl()) + continue; + if (CSM == Sema::CXXDefaultConstructor && F->hasInClassInitializer()) + continue; + QualType BaseType = S.Context.getBaseElementType(F->getType()); + if (const RecordType *RecordTy = BaseType->getAs()) { + CXXRecordDecl *FieldRecDecl = cast(RecordTy->getDecl()); + if (!specialMemberIsConstexpr(S, FieldRecDecl, CSM, + BaseType.getCVRQualifiers(), + ConstArg && !F->isMutable())) + return false; + } else if (CSM == Sema::CXXDefaultConstructor) { return false; - } else if (CSM == Sema::CXXDefaultConstructor) { - return false; + } } } @@ -7858,18 +7872,17 @@ bool Sema::CheckExplicitlyDefaultedSpecialMember(CXXMethodDecl *MD, MD->isConstexpr() && !Constexpr && MD->getTemplatedKind() == FunctionDecl::TK_NonTemplate) { if (!MD->isConsteval() && RD->getNumVBases()) { - Diag(MD->getBeginLoc(), diag::err_incorrect_defaulted_constexpr_with_vb) + Diag(MD->getBeginLoc(), + diag::err_incorrect_defaulted_constexpr_with_vb) << CSM; for (const auto &I : RD->vbases()) Diag(I.getBeginLoc(), diag::note_constexpr_virtual_base_here); } else { - Diag(MD->getBeginLoc(), MD->isConsteval() - ? diag::err_incorrect_defaulted_consteval - : diag::err_incorrect_defaulted_constexpr) - << CSM; + Diag(MD->getBeginLoc(), diag::err_incorrect_defaulted_constexpr) + << CSM << MD->isConsteval(); } - // FIXME: Explain why the special member can't be constexpr. - HadError = true; + HadError = true; + // FIXME: Explain why the special member can't be constexpr. } if (First) { @@ -9101,13 +9114,11 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD, // - if the function is a constructor or destructor, its class does not // have any virtual base classes. if (FD->isConstexpr()) { - if (CheckConstexprReturnType(*this, FD, CheckConstexprKind::Diagnose) && + if (!getLangOpts().CPlusPlus23 && + CheckConstexprReturnType(*this, FD, CheckConstexprKind::Diagnose) && CheckConstexprParameterTypes(*this, FD, CheckConstexprKind::Diagnose) && !Info.Constexpr) { - Diag(FD->getBeginLoc(), - getLangOpts().CPlusPlus23 - ? diag::warn_cxx23_compat_defaulted_comparison_constexpr_mismatch - : diag::ext_defaulted_comparison_constexpr_mismatch) + Diag(FD->getBeginLoc(), diag::err_defaulted_comparison_constexpr_mismatch) << FD->isImplicit() << (int)DCK << FD->isConsteval(); DefaultedComparisonAnalyzer(*this, RD, FD, DCK, DefaultedComparisonAnalyzer::ExplainConstexpr) diff --git a/clang/test/AST/Interp/cxx23.cpp b/clang/test/AST/Interp/cxx23.cpp index f1df936a5abe..127b58915127 100644 --- a/clang/test/AST/Interp/cxx23.cpp +++ b/clang/test/AST/Interp/cxx23.cpp @@ -1,82 +1,58 @@ -// RUN: %clang_cc1 -std=c++20 -fsyntax-only -fcxx-exceptions -verify=ref20,all %s +// RUN: %clang_cc1 -std=c++20 -fsyntax-only -fcxx-exceptions -verify=ref20,all,all-20 %s // RUN: %clang_cc1 -std=c++23 -fsyntax-only -fcxx-exceptions -verify=ref23,all %s -// RUN: %clang_cc1 -std=c++20 -fsyntax-only -fcxx-exceptions -verify=expected20,all %s -fexperimental-new-constant-interpreter +// RUN: %clang_cc1 -std=c++20 -fsyntax-only -fcxx-exceptions -verify=expected20,all,all-20 %s -fexperimental-new-constant-interpreter // RUN: %clang_cc1 -std=c++23 -fsyntax-only -fcxx-exceptions -verify=expected23,all %s -fexperimental-new-constant-interpreter /// FIXME: The new interpreter is missing all the 'control flows through...' diagnostics. constexpr int f(int n) { // ref20-error {{constexpr function never produces a constant expression}} \ - // ref23-error {{constexpr function never produces a constant expression}} \ - // expected20-error {{constexpr function never produces a constant expression}} \ - // expected23-error {{constexpr function never produces a constant expression}} + // expected20-error {{constexpr function never produces a constant expression}} static const int m = n; // ref20-note {{control flows through the definition of a static variable}} \ // ref20-warning {{is a C++23 extension}} \ - // ref23-note {{control flows through the definition of a static variable}} \ // expected20-warning {{is a C++23 extension}} \ // expected20-note {{declared here}} \ - // expected23-note {{declared here}} - return m; // expected20-note {{initializer of 'm' is not a constant expression}} \ - // expected23-note {{initializer of 'm' is not a constant expression}} + return m; // expected20-note {{initializer of 'm' is not a constant expression}} } constexpr int g(int n) { // ref20-error {{constexpr function never produces a constant expression}} \ - // ref23-error {{constexpr function never produces a constant expression}} \ - // expected20-error {{constexpr function never produces a constant expression}} \ - // expected23-error {{constexpr function never produces a constant expression}} + // expected20-error {{constexpr function never produces a constant expression}} thread_local const int m = n; // ref20-note {{control flows through the definition of a thread_local variable}} \ // ref20-warning {{is a C++23 extension}} \ - // ref23-note {{control flows through the definition of a thread_local variable}} \ // expected20-warning {{is a C++23 extension}} \ - // expected20-note {{declared here}} \ - // expected23-note {{declared here}} - return m; // expected20-note {{initializer of 'm' is not a constant expression}} \ - // expected23-note {{initializer of 'm' is not a constant expression}} + // expected20-note {{declared here}} + return m; // expected20-note {{initializer of 'm' is not a constant expression}} } constexpr int c_thread_local(int n) { // ref20-error {{constexpr function never produces a constant expression}} \ - // ref23-error {{constexpr function never produces a constant expression}} \ - // expected20-error {{constexpr function never produces a constant expression}} \ - // expected23-error {{constexpr function never produces a constant expression}} + // expected20-error {{constexpr function never produces a constant expression}} static _Thread_local int m = 0; // ref20-note {{control flows through the definition of a thread_local variable}} \ // ref20-warning {{is a C++23 extension}} \ - // ref23-note {{control flows through the definition of a thread_local variable}} \ // expected20-warning {{is a C++23 extension}} \ - // expected20-note {{declared here}} \ - // expected23-note {{declared here}} - return m; // expected20-note {{read of non-const variable}} \ - // expected23-note {{read of non-const variable}} + // expected20-note {{declared here}} + return m; // expected20-note {{read of non-const variable}} } constexpr int gnu_thread_local(int n) { // ref20-error {{constexpr function never produces a constant expression}} \ - // ref23-error {{constexpr function never produces a constant expression}} \ - // expected20-error {{constexpr function never produces a constant expression}} \ - // expected23-error {{constexpr function never produces a constant expression}} + // expected20-error {{constexpr function never produces a constant expression}} static __thread int m = 0; // ref20-note {{control flows through the definition of a thread_local variable}} \ // ref20-warning {{is a C++23 extension}} \ - // ref23-note {{control flows through the definition of a thread_local variable}} \ // expected20-warning {{is a C++23 extension}} \ - // expected20-note {{declared here}} \ - // expected23-note {{declared here}} - return m; // expected20-note {{read of non-const variable}} \ - // expected23-note {{read of non-const variable}} + // expected20-note {{declared here}} + return m; // expected20-note {{read of non-const variable}} } -constexpr int h(int n) { // ref20-error {{constexpr function never produces a constant expression}} \ - // ref23-error {{constexpr function never produces a constant expression}} +constexpr int h(int n) { // ref20-error {{constexpr function never produces a constant expression}} static const int m = n; // ref20-note {{control flows through the definition of a static variable}} \ // ref20-warning {{is a C++23 extension}} \ - // ref23-note {{control flows through the definition of a static variable}} \ // expected20-warning {{is a C++23 extension}} return &m - &m; } -constexpr int i(int n) { // ref20-error {{constexpr function never produces a constant expression}} \ - // ref23-error {{constexpr function never produces a constant expression}} +constexpr int i(int n) { // ref20-error {{constexpr function never produces a constant expression}} thread_local const int m = n; // ref20-note {{control flows through the definition of a thread_local variable}} \ // ref20-warning {{is a C++23 extension}} \ - // ref23-note {{control flows through the definition of a thread_local variable}} \ // expected20-warning {{is a C++23 extension}} return &m - &m; } @@ -132,8 +108,9 @@ namespace StaticOperators { static_assert(f2() == 3); struct S1 { - constexpr S1() { // all-error {{never produces a constant expression}} - throw; // all-note 2{{not valid in a constant expression}} + constexpr S1() { // all-20-error {{never produces a constant expression}} + throw; // all-note {{not valid in a constant expression}} \ + // all-20-note {{not valid in a constant expression}} } static constexpr int operator()() { return 3; } // ref20-warning {{C++23 extension}} \ // expected20-warning {{C++23 extension}} diff --git a/clang/test/CXX/class/class.compare/class.compare.default/p3.cpp b/clang/test/CXX/class/class.compare/class.compare.default/p3.cpp index 166bd97e2731..c73eb0dee995 100644 --- a/clang/test/CXX/class/class.compare/class.compare.default/p3.cpp +++ b/clang/test/CXX/class/class.compare/class.compare.default/p3.cpp @@ -1,8 +1,8 @@ // This test is for the [class.compare.default]p3 added by P2002R0 -// Also covers modifications made by P2448R2 and extension warnings +// Also covers modifications made by P2448R2 -// RUN: %clang_cc1 -std=c++2a -verify %s -// RUN: %clang_cc1 -std=c++2a -Wc++23-default-comp-relaxed-constexpr -verify=expected,extension %s +// RUN: %clang_cc1 -std=c++2a -verify=expected,cxx2a %s +// RUN: %clang_cc1 -std=c++23 -verify=expected %s namespace std { struct strong_ordering { @@ -82,10 +82,12 @@ struct TestB { }; struct C { - friend bool operator==(const C&, const C&); // expected-note {{previous}} extension-note 2{{non-constexpr comparison function declared here}} + friend bool operator==(const C&, const C&); // expected-note {{previous}} \ + // cxx2a-note 2{{declared here}} friend bool operator!=(const C&, const C&) = default; // expected-note {{previous}} - friend std::strong_ordering operator<=>(const C&, const C&); // expected-note {{previous}} extension-note 2{{non-constexpr comparison function declared here}} + friend std::strong_ordering operator<=>(const C&, const C&); // expected-note {{previous}} \ + // cxx2a-note 2{{declared here}} friend bool operator<(const C&, const C&) = default; // expected-note {{previous}} friend bool operator<=(const C&, const C&) = default; // expected-note {{previous}} friend bool operator>(const C&, const C&) = default; // expected-note {{previous}} @@ -129,23 +131,23 @@ struct TestD { struct E { A a; - C c; // extension-note 2{{non-constexpr comparison function would be used to compare member 'c'}} + C c; // cxx2a-note 2{{non-constexpr comparison function would be used to compare member 'c'}} A b; - friend constexpr bool operator==(const E&, const E&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}} + friend constexpr bool operator==(const E&, const E&) = default; // cxx2a-error {{cannot be declared constexpr}} friend constexpr bool operator!=(const E&, const E&) = default; - friend constexpr std::strong_ordering operator<=>(const E&, const E&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}} + friend constexpr std::strong_ordering operator<=>(const E&, const E&) = default; // cxx2a-error {{cannot be declared constexpr}} friend constexpr bool operator<(const E&, const E&) = default; friend constexpr bool operator<=(const E&, const E&) = default; friend constexpr bool operator>(const E&, const E&) = default; friend constexpr bool operator>=(const E&, const E&) = default; }; -struct E2 : A, C { // extension-note 2{{non-constexpr comparison function would be used to compare base class 'C'}} - friend constexpr bool operator==(const E2&, const E2&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}} +struct E2 : A, C { // cxx2a-note 2{{non-constexpr comparison function would be used to compare base class 'C'}} + friend constexpr bool operator==(const E2&, const E2&) = default; // cxx2a-error {{cannot be declared constexpr}} friend constexpr bool operator!=(const E2&, const E2&) = default; - friend constexpr std::strong_ordering operator<=>(const E2&, const E2&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}} + friend constexpr std::strong_ordering operator<=>(const E2&, const E2&) = default; // cxx2a-error {{cannot be declared constexpr}} friend constexpr bool operator<(const E2&, const E2&) = default; friend constexpr bool operator<=(const E2&, const E2&) = default; friend constexpr bool operator>(const E2&, const E2&) = default; @@ -153,14 +155,14 @@ struct E2 : A, C { // extension-note 2{{non-constexpr comparison function would }; struct F { - friend bool operator==(const F&, const F&); // extension-note {{non-constexpr comparison function declared here}} - friend constexpr bool operator!=(const F&, const F&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}} - - friend std::strong_ordering operator<=>(const F&, const F&); // extension-note 4{{non-constexpr comparison function declared here}} - friend constexpr bool operator<(const F&, const F&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}} - friend constexpr bool operator<=(const F&, const F&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}} - friend constexpr bool operator>(const F&, const F&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}} - friend constexpr bool operator>=(const F&, const F&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}} + friend bool operator==(const F&, const F&); // cxx2a-note {{declared here}} + friend constexpr bool operator!=(const F&, const F&) = default; // cxx2a-error {{cannot be declared constexpr}} + + friend std::strong_ordering operator<=>(const F&, const F&); // cxx2a-note 4{{non-constexpr comparison function declared here}} + friend constexpr bool operator<(const F&, const F&) = default; // cxx2a-error {{cannot be declared constexpr}} + friend constexpr bool operator<=(const F&, const F&) = default; // cxx2a-error {{cannot be declared constexpr}} + friend constexpr bool operator>(const F&, const F&) = default; // cxx2a-error {{cannot be declared constexpr}} + friend constexpr bool operator>=(const F&, const F&) = default; // cxx2a-error {{cannot be declared constexpr}} }; // No implicit 'constexpr' if it's not the first declaration. diff --git a/clang/test/CXX/class/class.compare/class.compare.default/p4.cpp b/clang/test/CXX/class/class.compare/class.compare.default/p4.cpp index 02cdd7f85aeb..534c3b34d883 100644 --- a/clang/test/CXX/class/class.compare/class.compare.default/p4.cpp +++ b/clang/test/CXX/class/class.compare/class.compare.default/p4.cpp @@ -1,9 +1,9 @@ -// RUN: %clang_cc1 -std=c++2a -verify %s -// RUN: %clang_cc1 -std=c++2a -Wc++23-default-comp-relaxed-constexpr -verify=expected,extension %s +// RUN: %clang_cc1 -std=c++2a -verify=expected,cxx2a %s +// RUN: %clang_cc1 -std=c++23 -verify=expected %s // This test is for [class.compare.default]p3 as modified and renumbered to p4 // by P2002R0. -// Also covers modifications made by P2448R2 and extension warnings +// Also covers modifications made by P2448R2 namespace std { struct strong_ordering { @@ -78,13 +78,13 @@ void use_g(G g) { } struct H { - bool operator==(const H&) const; // extension-note {{non-constexpr comparison function declared here}} + bool operator==(const H&) const; // cxx2a-note {{non-constexpr comparison function declared here}} constexpr std::strong_ordering operator<=>(const H&) const { return std::strong_ordering::equal; } }; struct I { - H h; // extension-note {{non-constexpr comparison function would be used to compare member 'h'}} - constexpr std::strong_ordering operator<=>(const I&) const = default; // extension-warning {{implicit 'operator==' invokes a non-constexpr comparison function is a C++23 extension}} + H h; // cxx2a-note {{non-constexpr comparison function would be used to compare member 'h'}} + constexpr std::strong_ordering operator<=>(const I&) const = default; // cxx2a-error {{cannot be declared constexpr}} }; struct J { @@ -148,16 +148,16 @@ namespace NoInjectionIfOperatorEqualsDeclared { namespace GH61238 { template struct my_struct { - A value; // extension-note {{non-constexpr comparison function would be used to compare member 'value'}} + A value; // cxx2a-note {{non-constexpr comparison function would be used to compare member 'value'}} - constexpr friend bool operator==(const my_struct &, const my_struct &) noexcept = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}} + constexpr friend bool operator==(const my_struct &, const my_struct &) noexcept = default; // cxx2a-error {{cannot be declared constexpr}} }; struct non_constexpr_type { - friend bool operator==(non_constexpr_type, non_constexpr_type) noexcept { // extension-note {{non-constexpr comparison function declared here}} + friend bool operator==(non_constexpr_type, non_constexpr_type) noexcept { // cxx2a-note {{non-constexpr comparison function declared here}} return false; } }; -my_struct obj; // extension-note {{in instantiation of template class 'GH61238::my_struct' requested here}} +my_struct obj; // cxx2a-note {{in instantiation of template class 'GH61238::my_struct' requested here}} } diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/dtor.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/dtor.cpp index 7ad2e582a812..48bc8fb426bc 100644 --- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/dtor.cpp +++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/dtor.cpp @@ -58,12 +58,12 @@ namespace subobject { struct A { ~A(); }; - struct B : A { // expected-note {{here}} - constexpr ~B() {} // expected-error {{destructor cannot be declared constexpr because base class 'A' does not have a constexpr destructor}} + struct B : A { // cxx2a-note {{here}} + constexpr ~B() {} // cxx2a-error {{destructor cannot be declared constexpr because base class 'A' does not have a constexpr destructor}} }; struct C { - A a; // expected-note {{here}} - constexpr ~C() {} // expected-error {{destructor cannot be declared constexpr because data member 'a' does not have a constexpr destructor}} + A a; // cxx2a-note {{here}} + constexpr ~C() {} // cxx2a-error {{destructor cannot be declared constexpr because data member 'a' does not have a constexpr destructor}} }; struct D : A { A a; diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3-2b.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3-2b.cpp index c07502c0555b..8cb37ae6d1cd 100644 --- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3-2b.cpp +++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3-2b.cpp @@ -14,9 +14,8 @@ constexpr int i(int n) { return m; } -constexpr int g() { // expected-error {{constexpr function never produces a constant expression}} - goto test; // expected-note {{subexpression not valid in a constant expression}} \ - // expected-warning {{use of this statement in a constexpr function is incompatible with C++ standards before C++23}} +constexpr int g() { + goto test; // expected-warning {{use of this statement in a constexpr function is incompatible with C++ standards before C++23}} test: return 0; } @@ -29,9 +28,8 @@ struct NonLiteral { // expected-note 2 {{'NonLiteral' is not literal}} NonLiteral() {} }; -constexpr void non_literal() { // expected-error {{constexpr function never produces a constant expression}} - NonLiteral n; // expected-note {{non-literal type 'NonLiteral' cannot be used in a constant expression}} \ - // expected-warning {{definition of a variable of non-literal type in a constexpr function is incompatible with C++ standards before C++23}} +constexpr void non_literal() { + NonLiteral n; // expected-warning {{definition of a variable of non-literal type in a constexpr function is incompatible with C++ standards before C++23}} } constexpr void non_literal2(bool b) { diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3.cpp index 6214ff8006d6..4416c8252264 100644 --- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3.cpp +++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3.cpp @@ -1,6 +1,6 @@ // RUN: %clang_cc1 -fcxx-exceptions -verify=expected,beforecxx14,beforecxx20,beforecxx23 -std=c++11 %s -// RUN: %clang_cc1 -fcxx-exceptions -verify=expected,aftercxx14,beforecxx20,beforecxx23 -std=c++14 %s -// RUN: %clang_cc1 -fcxx-exceptions -verify=expected,aftercxx14,aftercxx20,beforecxx23 -std=c++20 %s +// RUN: %clang_cc1 -fcxx-exceptions -verify=expected,aftercxx14,beforecxx20,beforecxx23,cxx14_20 -std=c++14 %s +// RUN: %clang_cc1 -fcxx-exceptions -verify=expected,aftercxx14,aftercxx20,beforecxx23,cxx14_20 -std=c++20 %s // RUN: %clang_cc1 -fcxx-exceptions -verify=expected,aftercxx14,aftercxx20 -std=c++23 %s namespace N { @@ -11,7 +11,7 @@ namespace M { typedef double D; } -struct NonLiteral { // expected-note 2{{no constexpr constructors}} +struct NonLiteral { // beforecxx23-note 2{{no constexpr constructors}} NonLiteral() {} NonLiteral(int) {} }; @@ -43,7 +43,7 @@ struct T : SS, NonLiteral { // - its return type shall be a literal type; // Once we support P2448R2 constexpr functions will be allowd to return non-literal types // The destructor will also be allowed - constexpr NonLiteral NonLiteralReturn() const { return {}; } // expected-error {{constexpr function's return type 'NonLiteral' is not a literal type}} + constexpr NonLiteral NonLiteralReturn() const { return {}; } // beforecxx23-error {{constexpr function's return type 'NonLiteral' is not a literal type}} constexpr void VoidReturn() const { return; } // beforecxx14-error {{constexpr function's return type 'void' is not a literal type}} constexpr ~T(); // beforecxx20-error {{destructor cannot be declared constexpr}} @@ -52,7 +52,7 @@ struct T : SS, NonLiteral { // - each of its parameter types shall be a literal type; // Once we support P2448R2 constexpr functions will be allowd to have parameters of non-literal types - constexpr int NonLiteralParam(NonLiteral) const { return 0; } // expected-error {{constexpr function's 1st parameter type 'NonLiteral' is not a literal type}} + constexpr int NonLiteralParam(NonLiteral) const { return 0; } // beforecxx23-error {{constexpr function's 1st parameter type 'NonLiteral' is not a literal type}} typedef int G(NonLiteral) const; constexpr G NonLiteralParam2; // ok until definition @@ -66,7 +66,7 @@ struct T : SS, NonLiteral { // constexpr since they can't be const. constexpr T &operator=(const T &) = default; // beforecxx14-error {{an explicitly-defaulted copy assignment operator may not have 'const', 'constexpr' or 'volatile' qualifiers}} \ // beforecxx14-warning {{C++14}} \ - // aftercxx14-error{{defaulted definition of copy assignment operator is not constexpr}} + // cxx14_20-error{{defaulted definition of copy assignment operator cannot be marked constexpr}} }; constexpr int T::OutOfLineVirtual() const { return 0; } @@ -229,9 +229,9 @@ namespace DR1364 { return k; // ok, even though lvalue-to-rvalue conversion of a function // parameter is not allowed in a constant expression. } - int kGlobal; // expected-note {{here}} - constexpr int f() { // expected-error {{constexpr function never produces a constant expression}} - return kGlobal; // expected-note {{read of non-const}} + int kGlobal; // beforecxx23-note {{here}} + constexpr int f() { // beforecxx23-error {{constexpr function never produces a constant expression}} + return kGlobal; // beforecxx23-note {{read of non-const}} } } diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p4.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p4.cpp index f1f677ebfcd3..92698ec1c738 100644 --- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p4.cpp +++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p4.cpp @@ -272,7 +272,7 @@ struct X { union XU1 { int a; constexpr XU1() = default; }; #ifndef CXX2A -// expected-error@-2{{not constexpr}} +// expected-error@-2{{cannot be marked constexpr}} #endif union XU2 { int a = 1; constexpr XU2() = default; }; @@ -282,7 +282,7 @@ struct XU3 { }; constexpr XU3() = default; #ifndef CXX2A - // expected-error@-2{{not constexpr}} + // expected-error@-2{{cannot be marked constexpr}} #endif }; struct XU4 { @@ -333,7 +333,7 @@ namespace CtorLookup { constexpr B(B&); }; constexpr B::B(const B&) = default; - constexpr B::B(B&) = default; // expected-error {{not constexpr}} + constexpr B::B(B&) = default; // expected-error {{cannot be marked constexpr}} struct C { A a; @@ -342,7 +342,7 @@ namespace CtorLookup { constexpr C(C&); }; constexpr C::C(const C&) = default; - constexpr C::C(C&) = default; // expected-error {{not constexpr}} + constexpr C::C(C&) = default; // expected-error {{cannot be marked constexpr}} } namespace PR14503 { diff --git a/clang/test/CXX/dcl.decl/dcl.fct.def/dcl.fct.def.default/p2.cpp b/clang/test/CXX/dcl.decl/dcl.fct.def/dcl.fct.def.default/p2.cpp index 5b525fc91aba..849594307390 100644 --- a/clang/test/CXX/dcl.decl/dcl.fct.def/dcl.fct.def.default/p2.cpp +++ b/clang/test/CXX/dcl.decl/dcl.fct.def/dcl.fct.def.default/p2.cpp @@ -3,7 +3,7 @@ // An explicitly-defaulted function may be declared constexpr only if it would // have been implicitly declared as constexpr. struct S1 { - constexpr S1() = default; // expected-error {{defaulted definition of default constructor is not constexpr}} + constexpr S1() = default; // expected-error {{defaulted definition of default constructor cannot be marked constexpr}} constexpr S1(const S1&) = default; constexpr S1(S1&&) = default; constexpr S1 &operator=(const S1&) const = default; // expected-error {{explicitly-defaulted copy assignment operator may not have}} @@ -18,8 +18,8 @@ struct NoCopyMove { }; struct S2 { constexpr S2() = default; - constexpr S2(const S2&) = default; // expected-error {{defaulted definition of copy constructor is not constexpr}} - constexpr S2(S2&&) = default; // expected-error {{defaulted definition of move constructor is not constexpr}} + constexpr S2(const S2&) = default; // expected-error {{defaulted definition of copy constructor cannot be marked constexpr}} + constexpr S2(S2&&) = default; // expected-error {{defaulted definition of move constructor cannot be marked}} NoCopyMove ncm; }; diff --git a/clang/test/CXX/drs/dr13xx.cpp b/clang/test/CXX/drs/dr13xx.cpp index effdc53040d0..d8e3b5d87bd1 100644 --- a/clang/test/CXX/drs/dr13xx.cpp +++ b/clang/test/CXX/drs/dr13xx.cpp @@ -1,8 +1,8 @@ // RUN: %clang_cc1 -std=c++98 %s -verify=expected,cxx98-14,cxx98 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 %s -verify=expected,cxx11-17,cxx11-14,cxx98-14,since-cxx11,cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 %s -verify=expected,cxx11-17,cxx11-14,since-cxx14,cxx98-14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 %s -verify=expected,cxx11-17,since-cxx14,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 %s -verify=expected,cxx11-20,cxx11-17,cxx11-14,cxx98-14,since-cxx11,cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 %s -verify=expected,cxx11-20,cxx11-17,cxx11-14,since-cxx14,cxx98-14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 %s -verify=expected,cxx11-20,cxx11-17,since-cxx14,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 %s -verify=expected,cxx11-20,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors // RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors // RUN: %clang_cc1 -std=c++2c %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors @@ -485,11 +485,11 @@ namespace dr1358 { // dr1358: 3.1 struct B : Virt { int member; constexpr B(NonLit u) : member(u) {} - // since-cxx11-error@-1 {{constexpr constructor's 1st parameter type 'NonLit' is not a literal type}} - // since-cxx11-note@#dr1358-NonLit {{'NonLit' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}} + // cxx11-20-error@-1 {{constexpr constructor's 1st parameter type 'NonLit' is not a literal type}} + // cxx11-20-note@#dr1358-NonLit {{'NonLit' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}} constexpr NonLit f(NonLit u) const { return NonLit(); } - // since-cxx11-error@-1 {{constexpr function's return type 'NonLit' is not a literal type}} - // since-cxx11-note@#dr1358-NonLit {{'NonLit' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}} + // cxx11-20-error@-1 {{constexpr function's return type 'NonLit' is not a literal type}} + // cxx11-20-note@#dr1358-NonLit {{'NonLit' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}} }; #endif } @@ -498,13 +498,13 @@ namespace dr1359 { // dr1359: 3.5 #if __cplusplus >= 201103L union A { constexpr A() = default; }; union B { constexpr B() = default; int a; }; // #dr1359-B - // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} + // cxx11-17-error@-1 {{defaulted definition of default constructor cannot be marked constexpr before C++23}} union C { constexpr C() = default; int a, b; }; // #dr1359-C - // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} + // cxx11-17-error@-1 {{defaulted definition of default constructor cannot be marked constexpr}} struct X { constexpr X() = default; union {}; }; // since-cxx11-error@-1 {{declaration does not declare anything}} struct Y { constexpr Y() = default; union { int a; }; }; // #dr1359-Y - // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} + // cxx11-17-error@-1 {{defaulted definition of default constructor cannot be marked constexpr}} constexpr A a = A(); constexpr B b = B(); diff --git a/clang/test/CXX/drs/dr14xx.cpp b/clang/test/CXX/drs/dr14xx.cpp index 58a2b3a0d027..ed6dda731fd5 100644 --- a/clang/test/CXX/drs/dr14xx.cpp +++ b/clang/test/CXX/drs/dr14xx.cpp @@ -153,16 +153,16 @@ namespace dr1460 { // dr1460: 3.5 namespace Defaulted { union A { constexpr A() = default; }; union B { int n; constexpr B() = default; }; - // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} + // cxx11-17-error@-1 {{defaulted definition of default constructor cannot be marked constexpr}} union C { int n = 0; constexpr C() = default; }; struct D { union {}; constexpr D() = default; }; // expected-error@-1 {{declaration does not declare anything}} struct E { union { int n; }; constexpr E() = default; }; - // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} + // cxx11-17-error@-1 {{defaulted definition of default constructor cannot be marked constexpr}} struct F { union { int n = 0; }; constexpr F() = default; }; struct G { union { int n = 0; }; union { int m; }; constexpr G() = default; }; - // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} + // cxx11-17-error@-1 {{defaulted definition of default constructor cannot be marked constexpr}} struct H { union { int n = 0; diff --git a/clang/test/CXX/drs/dr15xx.cpp b/clang/test/CXX/drs/dr15xx.cpp index ac503db625ba..195c0fa610d5 100644 --- a/clang/test/CXX/drs/dr15xx.cpp +++ b/clang/test/CXX/drs/dr15xx.cpp @@ -1,10 +1,10 @@ // RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,cxx11-14 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,cxx11-14,cxx14-17 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx20,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx20,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx20,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-20,since-cxx11,cxx11-14 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-20,since-cxx11,cxx11-14,cxx14-17 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-20,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-20,since-cxx20,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx23,since-cxx20,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx23,since-cxx20,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors namespace dr1512 { // dr1512: 4 void f(char *p) { @@ -407,7 +407,7 @@ namespace dr1573 { // dr1573: 3.9 B b(1, 'x', 4.0, "hello"); // ok // inherited constructor is effectively constexpr if the user-written constructor would be - struct C { C(); constexpr C(int) {} }; + struct C { C(); constexpr C(int) {} }; // #dr1573-C struct D : C { using C::C; }; constexpr D d = D(0); // ok struct E : C { using C::C; A a; }; // #dr1573-E @@ -420,8 +420,11 @@ namespace dr1573 { // dr1573: 3.9 struct F : C { using C::C; C c; }; // #dr1573-F constexpr F f = F(0); // since-cxx11-error@-1 {{constexpr variable 'f' must be initialized by a constant expression}} - // since-cxx11-note@-2 {{constructor inherited from base class 'C' cannot be used in a constant expression; derived class cannot be implicitly initialized}} - // since-cxx11-note@#dr1573-F {{declared here}} + // cxx11-20-note@-2 {{constructor inherited from base class 'C' cannot be used in a constant expression; derived class cannot be implicitly initialized}} + // since-cxx23-note@-3 {{in implicit initialization for inherited constructor of 'F'}} + // since-cxx23-note@#dr1573-F {{non-constexpr constructor 'C' cannot be used in a constant expression}} + // cxx11-20-note@#dr1573-F {{declared here}} + // since-cxx23-note@#dr1573-C {{declared here}} // inherited constructor is effectively deleted if the user-written constructor would be struct G { G(int); }; diff --git a/clang/test/CXX/drs/dr16xx.cpp b/clang/test/CXX/drs/dr16xx.cpp index 2dd7d1502e59..766c90d3bc7b 100644 --- a/clang/test/CXX/drs/dr16xx.cpp +++ b/clang/test/CXX/drs/dr16xx.cpp @@ -1,10 +1,10 @@ // RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify=expected,cxx98-14,cxx98 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,cxx98-14,since-cxx11,cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx14,cxx98-14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx14,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-20,cxx98-14,since-cxx11,cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-20,since-cxx14,cxx98-14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-20,since-cxx14,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-20,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx23,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx23,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors #if __cplusplus == 199711L #define static_assert(...) __extension__ _Static_assert(__VA_ARGS__) @@ -256,12 +256,12 @@ namespace dr1658 { // dr1658: 5 struct A { A(A&); }; struct B : virtual A { virtual void f() = 0; }; struct C : virtual A { virtual void f(); }; - struct D : A { virtual void f() = 0; }; + struct D : A { virtual void f() = 0; }; // since-cxx23-note {{previous declaration is here}} struct X { friend B::B(const B&) throw(); friend C::C(C&); - friend D::D(D&); + friend D::D(D&); // since-cxx23-error {{non-constexpr declaration of 'D' follows constexpr declaration}} }; } @@ -350,8 +350,8 @@ namespace dr1684 { // dr1684: 3.6 }; constexpr int f(NonLiteral &) { return 0; } constexpr int f(NonLiteral) { return 0; } - // since-cxx11-error@-1 {{constexpr function's 1st parameter type 'NonLiteral' is not a literal type}} - // since-cxx11-note@#dr1684-struct {{'NonLiteral' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}} + // cxx11-20-error@-1 {{constexpr function's 1st parameter type 'NonLiteral' is not a literal type}} + // cxx11-20-note@#dr1684-struct {{'NonLiteral' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}} #endif } diff --git a/clang/test/CXX/drs/dr6xx.cpp b/clang/test/CXX/drs/dr6xx.cpp index b35d3051ab55..190e05784f32 100644 --- a/clang/test/CXX/drs/dr6xx.cpp +++ b/clang/test/CXX/drs/dr6xx.cpp @@ -1,8 +1,8 @@ // RUN: %clang_cc1 -std=c++98 %s -verify=expected,cxx98-17,cxx98-14,cxx98 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking -// RUN: %clang_cc1 -std=c++11 %s -verify=expected,cxx98-17,cxx11-17,cxx98-14,since-cxx11,cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking -// RUN: %clang_cc1 -std=c++14 %s -verify=expected,cxx98-17,cxx11-17,cxx98-14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking -// RUN: %clang_cc1 -std=c++17 %s -verify=expected,cxx98-17,cxx11-17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking -// RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking +// RUN: %clang_cc1 -std=c++11 %s -verify=expected,cxx11-20,cxx98-17,cxx11-17,cxx98-14,since-cxx11,cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking +// RUN: %clang_cc1 -std=c++14 %s -verify=expected,cxx11-20,cxx98-17,cxx11-17,cxx98-14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking +// RUN: %clang_cc1 -std=c++17 %s -verify=expected,cxx11-20,cxx98-17,cxx11-17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking +// RUN: %clang_cc1 -std=c++20 %s -verify=expected,cxx11-20,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking // RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking namespace dr600 { // dr600: 2.8 @@ -584,8 +584,8 @@ namespace dr647 { // dr647: 3.1 struct C { constexpr C(NonLiteral); constexpr C(NonLiteral, int) {} - // since-cxx11-error@-1 {{constexpr constructor's 1st parameter type 'NonLiteral' is not a literal type}} - // since-cxx11-note@#dr647-NonLiteral {{'NonLiteral' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}} + // cxx11-20-error@-1 {{constexpr constructor's 1st parameter type 'NonLiteral' is not a literal type}} + // cxx11-20-note@#dr647-NonLiteral {{'NonLiteral' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}} constexpr C() try {} catch (...) {} // cxx11-17-error@-1 {{function try block in constexpr constructor is a C++20 extension}} // cxx11-error@-2 {{use of this statement in a constexpr constructor is a C++14 extension}} @@ -609,15 +609,15 @@ namespace dr647 { // dr647: 3.1 d(0) {} constexpr E(int) - // since-cxx11-error@-1 {{constexpr constructor never produces a constant expression}} - // since-cxx11-note@#dr647-int-d {{non-constexpr constructor 'D' cannot be used in a constant expression}} - // since-cxx11-note@#dr647-D-float-ctor {{declared here}} + // cxx11-20-error@-1 {{constexpr constructor never produces a constant expression}} + // cxx11-20-note@#dr647-int-d {{non-constexpr constructor 'D' cannot be used in a constant expression}} + // cxx11-20-note@#dr647-D-float-ctor {{declared here}} : n(0), d(0.0f) {} // #dr647-int-d constexpr E(float f) - // since-cxx11-error@-1 {{never produces a constant expression}} - // since-cxx11-note@#dr647-float-d {{non-constexpr constructor}} - // since-cxx11-note@#dr647-D-float-ctor {{declared here}} + // cxx11-20-error@-1 {{never produces a constant expression}} + // cxx11-20-note@#dr647-float-d {{non-constexpr constructor}} + // cxx11-20-note@#dr647-D-float-ctor {{declared here}} : n(get()), d(D(0) + f) {} // #dr647-float-d }; diff --git a/clang/test/CXX/expr/expr.const/p5-26.cpp b/clang/test/CXX/expr/expr.const/p5-26.cpp index de2afa71b426..3624b1e5a3e3 100644 --- a/clang/test/CXX/expr/expr.const/p5-26.cpp +++ b/clang/test/CXX/expr/expr.const/p5-26.cpp @@ -5,11 +5,11 @@ struct S {}; struct T : S {} t; -consteval void test() { // cxx23-error{{consteval function never produces a constant expression}} +consteval void test() { void* a = &t; const void* b = &t; volatile void* c = &t; - (void)static_cast(a); //cxx23-note {{cast from 'void *' is not allowed in a constant expression in C++ standards before C++2c}} + (void)static_cast(a); (void)static_cast(a); (void)static_cast(a); diff --git a/clang/test/CXX/special/class.copy/p13-0x.cpp b/clang/test/CXX/special/class.copy/p13-0x.cpp index 16c8a4029cba..013d5b565823 100644 --- a/clang/test/CXX/special/class.copy/p13-0x.cpp +++ b/clang/test/CXX/special/class.copy/p13-0x.cpp @@ -125,7 +125,7 @@ namespace Mutable { mutable A a; }; struct C { - constexpr C(const C &) = default; // expected-error {{not constexpr}} + constexpr C(const C &) = default; // expected-error {{cannot be marked constexpr}} A a; }; } diff --git a/clang/test/SemaCXX/constant-expression-cxx11.cpp b/clang/test/SemaCXX/constant-expression-cxx11.cpp index 9e2ae07cbe4c..efb391ba0922 100644 --- a/clang/test/SemaCXX/constant-expression-cxx11.cpp +++ b/clang/test/SemaCXX/constant-expression-cxx11.cpp @@ -1273,8 +1273,8 @@ namespace PR11595 { struct B { B(); A& x; }; static_assert(B().x == 3, ""); // expected-error {{constant expression}} expected-note {{non-literal type 'B' cannot be used in a constant expression}} - constexpr bool f(int k) { // expected-error {{constexpr function never produces a constant expression}} - return B().x == k; // expected-note {{non-literal type 'B' cannot be used in a constant expression}} + constexpr bool f(int k) { // cxx11_20-error {{constexpr function never produces a constant expression}} + return B().x == k; // cxx11_20-note {{non-literal type 'B' cannot be used in a constant expression}} } } @@ -1326,8 +1326,8 @@ namespace ExternConstexpr { constexpr int g() { return q; } // expected-note {{outside its lifetime}} constexpr int q = g(); // expected-error {{constant expression}} expected-note {{in call}} - extern int r; // expected-note {{here}} - constexpr int h() { return r; } // expected-error {{never produces a constant}} expected-note {{read of non-const}} + extern int r; // cxx11_20-note {{here}} + constexpr int h() { return r; } // cxx11_20-error {{never produces a constant}} cxx11_20-note {{read of non-const}} struct S { int n; }; extern const S s; @@ -1678,7 +1678,7 @@ namespace ImplicitConstexpr { struct R { constexpr R() noexcept; constexpr R(const R&) noexcept; constexpr R(R&&) noexcept; ~R() noexcept; }; struct S { R r; }; // expected-note 3{{here}} struct T { T(const T&) noexcept; T(T &&) noexcept; ~T() noexcept; }; - struct U { T t; }; // expected-note 3{{here}} + struct U { T t; }; // cxx11_20-note 3{{here}} static_assert(!__is_literal_type(Q), ""); static_assert(!__is_literal_type(R), ""); static_assert(!__is_literal_type(S), ""); @@ -1691,9 +1691,9 @@ namespace ImplicitConstexpr { friend S::S() noexcept; // expected-error {{follows constexpr}} friend S::S(S&&) noexcept; // expected-error {{follows constexpr}} friend S::S(const S&) noexcept; // expected-error {{follows constexpr}} - friend constexpr U::U() noexcept; // expected-error {{follows non-constexpr}} - friend constexpr U::U(U&&) noexcept; // expected-error {{follows non-constexpr}} - friend constexpr U::U(const U&) noexcept; // expected-error {{follows non-constexpr}} + friend constexpr U::U() noexcept; // cxx11_20-error {{follows non-constexpr}} + friend constexpr U::U(U&&) noexcept; // cxx11_20-error {{follows non-constexpr}} + friend constexpr U::U(const U&) noexcept; // cxx11_20-error {{follows non-constexpr}} }; } @@ -1906,9 +1906,9 @@ namespace StmtExpr { }); } static_assert(g(123) == 15129, ""); - constexpr int h() { // expected-error {{never produces a constant}} + constexpr int h() { // cxx11_20-error {{never produces a constant}} return ({ // expected-warning {{extension}} - return 0; // expected-note {{not supported}} + return 0; // cxx11_20-note {{not supported}} 1; }); } @@ -2093,8 +2093,8 @@ namespace ZeroSizeTypes { // expected-note@-2 {{subtraction of pointers to type 'int[0]' of zero size}} int arr[5][0]; - constexpr int f() { // expected-error {{never produces a constant expression}} - return &arr[3] - &arr[0]; // expected-note {{subtraction of pointers to type 'int[0]' of zero size}} + constexpr int f() { // cxx11_20-error {{never produces a constant expression}} + return &arr[3] - &arr[0]; // cxx11_20-note {{subtraction of pointers to type 'int[0]' of zero size}} } } @@ -2118,8 +2118,8 @@ namespace NeverConstantTwoWays { // If we see something non-constant but foldable followed by something // non-constant and not foldable, we want the first diagnostic, not the // second. - constexpr int f(int n) { // expected-error {{never produces a constant expression}} - return (int *)(long)&n == &n ? // expected-note {{reinterpret_cast}} + constexpr int f(int n) { // cxx11_20-error {{never produces a constant expression}} + return (int *)(long)&n == &n ? // cxx11_20-note {{reinterpret_cast}} 1 / 0 : // expected-warning {{division by zero}} 0; } @@ -2277,7 +2277,8 @@ namespace InheritedCtor { struct A { constexpr A(int) {} }; struct B : A { int n; using A::A; }; // expected-note {{here}} - constexpr B b(0); // expected-error {{constant expression}} expected-note {{derived class}} + constexpr B b(0); // expected-error {{constant expression}} cxx11_20-note {{derived class}}\ + // cxx23-note {{not initialized}} struct C : A { using A::A; struct { union { int n, m = 0; }; union { int a = 0; }; int k = 0; }; struct {}; union {}; }; // expected-warning 6{{}} constexpr C c(0); @@ -2316,10 +2317,11 @@ namespace InheritedCtor { namespace PR28366 { namespace ns1 { -void f(char c) { //expected-note2{{declared here}} +void f(char c) { //expected-note{{declared here}} + //cxx11_20-note@-1{{declared here}} struct X { - static constexpr char f() { //expected-error{{never produces a constant expression}} - return c; //expected-error{{reference to local}} expected-note{{function parameter}} + static constexpr char f() { // cxx11_20-error {{never produces a constant expression}} + return c; //expected-error{{reference to local}} cxx11_20-note{{function parameter}} } }; int I = X::f(); diff --git a/clang/test/SemaCXX/constant-expression-cxx14.cpp b/clang/test/SemaCXX/constant-expression-cxx14.cpp index 273d7ff3a208..80a7a2dd3153 100644 --- a/clang/test/SemaCXX/constant-expression-cxx14.cpp +++ b/clang/test/SemaCXX/constant-expression-cxx14.cpp @@ -44,13 +44,13 @@ constexpr int g(int k) { return 3 * k3 + 5 * k2 + n * k - 20; } static_assert(g(2) == 42, ""); -constexpr int h(int n) { // expected-error {{constexpr function never produces a constant expression}} - static const int m = n; // expected-note {{control flows through the definition of a static variable}} \ +constexpr int h(int n) { // cxx14_20-error {{constexpr function never produces a constant expression}} + static const int m = n; // cxx14_20-note {{control flows through the definition of a static variable}} \ // cxx14_20-warning {{definition of a static variable in a constexpr function is a C++23 extension}} return m; } -constexpr int i(int n) { // expected-error {{constexpr function never produces a constant expression}} - thread_local const int m = n; // expected-note {{control flows through the definition of a thread_local variable}} \ +constexpr int i(int n) { // cxx14_20-error {{constexpr function never produces a constant expression}} + thread_local const int m = n; // cxx14_20-note {{control flows through the definition of a thread_local variable}} \ // cxx14_20-warning {{definition of a thread_local variable in a constexpr function is a C++23 extension}} return m; } @@ -68,6 +68,7 @@ constexpr int j(int k) { } } } // expected-note 2{{control reached end of constexpr function}} + // cxx23-warning@-1 {{does not return a value in all control paths}} static_assert(j(0) == -3, ""); static_assert(j(1) == 5, ""); static_assert(j(2), ""); // expected-error {{constant expression}} expected-note {{in call to 'j(2)'}} @@ -104,10 +105,10 @@ static_assert(l(false) == 5, ""); static_assert(l(true), ""); // expected-error {{constant expression}} expected-note {{in call to 'l(true)'}} // Potential constant expression checking is still applied where possible. -constexpr int htonl(int x) { // expected-error {{never produces a constant expression}} +constexpr int htonl(int x) { // cxx14_20-error {{never produces a constant expression}} typedef unsigned char uchar; uchar arr[4] = { uchar(x >> 24), uchar(x >> 16), uchar(x >> 8), uchar(x) }; - return *reinterpret_cast(arr); // expected-note {{reinterpret_cast is not allowed in a constant expression}} + return *reinterpret_cast(arr); // cxx14_20-note {{reinterpret_cast is not allowed in a constant expression}} } constexpr int maybe_htonl(bool isBigEndian, int x) { @@ -183,7 +184,7 @@ namespace string_assign { static_assert(!test1(100), ""); static_assert(!test1(101), ""); // expected-error {{constant expression}} expected-note {{in call to 'test1(101)'}} - constexpr void f() { // expected-error{{constexpr function never produces a constant expression}} expected-note@+2{{assignment to dereferenced one-past-the-end pointer is not allowed in a constant expression}} + constexpr void f() { // cxx14_20-error{{constexpr function never produces a constant expression}} cxx14_20-note@+2{{assignment to dereferenced one-past-the-end pointer is not allowed in a constant expression}} char foo[10] = { "z" }; // expected-note {{here}} foo[10] = 'x'; // expected-warning {{past the end}} } @@ -207,14 +208,14 @@ namespace array_resize { namespace potential_const_expr { constexpr void set(int &n) { n = 1; } constexpr int div_zero_1() { int z = 0; set(z); return 100 / z; } // no error - constexpr int div_zero_2() { // expected-error {{never produces a constant expression}} + constexpr int div_zero_2() { // cxx14_20-error {{never produces a constant expression}} int z = 0; - return 100 / (set(z), 0); // expected-note {{division by zero}} + return 100 / (set(z), 0); // cxx14_20-note {{division by zero}} } - int n; // expected-note {{declared here}} - constexpr int ref() { // expected-error {{never produces a constant expression}} + int n; // cxx14_20-note {{declared here}} + constexpr int ref() { // cxx14_20-error {{never produces a constant expression}} int &r = n; - return r; // expected-note {{read of non-const variable 'n'}} + return r; // cxx14_20-note {{read of non-const variable 'n'}} } } @@ -846,8 +847,8 @@ namespace StmtExpr { static_assert(g() == 0, ""); // expected-error {{constant expression}} expected-note {{in call}} // FIXME: We should handle the void statement expression case. - constexpr int h() { // expected-error {{never produces a constant}} - ({ if (true) {} }); // expected-note {{not supported}} + constexpr int h() { // cxx14_20-error {{never produces a constant}} + ({ if (true) {} }); // cxx14_20-note {{not supported}} return 0; } } @@ -1043,9 +1044,9 @@ static_assert(sum(Cs) == 'a' + 'b', ""); // expected-error{{not an integral cons constexpr int S = sum(Cs); // expected-error{{must be initialized by a constant expression}} expected-note{{in call}} } -constexpr void PR28739(int n) { // expected-error {{never produces a constant}} +constexpr void PR28739(int n) { // cxx14_20-error {{never produces a constant}} int *p = &n; // expected-note {{array 'p' declared here}} - p += (__int128)(unsigned long)-1; // expected-note {{cannot refer to element 18446744073709551615 of non-array object in a constant expression}} + p += (__int128)(unsigned long)-1; // cxx14_20-note {{cannot refer to element 18446744073709551615 of non-array object in a constant expression}} // expected-warning@-1 {{the pointer incremented by 18446744073709551615 refers past the last possible element for an array in 64-bit address space containing 32-bit (4-byte) elements (max possible 4611686018427387904 elements)}} } diff --git a/clang/test/SemaCXX/constant-expression-cxx2b.cpp b/clang/test/SemaCXX/constant-expression-cxx2b.cpp index 2ee1d48d1cd6..2519839b7ac5 100644 --- a/clang/test/SemaCXX/constant-expression-cxx2b.cpp +++ b/clang/test/SemaCXX/constant-expression-cxx2b.cpp @@ -10,36 +10,36 @@ struct Constexpr{}; #if __cplusplus > 202002L -constexpr int f(int n) { // expected-error {{constexpr function never produces a constant expression}} - static const int m = n; // expected-note {{control flows through the definition of a static variable}} \ +constexpr int f(int n) { // cxx2a-error {{constexpr function never produces a constant expression}} + static const int m = n; // cxx2a-note {{control flows through the definition of a static variable}} \ // cxx23-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++23}} return m; } -constexpr int g(int n) { // expected-error {{constexpr function never produces a constant expression}} - thread_local const int m = n; // expected-note {{control flows through the definition of a thread_local variable}} \ +constexpr int g(int n) { // cxx2a-error {{constexpr function never produces a constant expression}} + thread_local const int m = n; // cxx2a-note {{control flows through the definition of a thread_local variable}} \ // cxx23-warning {{definition of a thread_local variable in a constexpr function is incompatible with C++ standards before C++23}} return m; } -constexpr int c_thread_local(int n) { // expected-error {{constexpr function never produces a constant expression}} - static _Thread_local int m = 0; // expected-note {{control flows through the definition of a thread_local variable}} \ +constexpr int c_thread_local(int n) { // cxx2a-error {{constexpr function never produces a constant expression}} + static _Thread_local int m = 0; // cxx2a-note {{control flows through the definition of a thread_local variable}} \ // cxx23-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++23}} return m; } -constexpr int gnu_thread_local(int n) { // expected-error {{constexpr function never produces a constant expression}} - static __thread int m = 0; // expected-note {{control flows through the definition of a thread_local variable}} \ +constexpr int gnu_thread_local(int n) { // cxx2a-error {{constexpr function never produces a constant expression}} + static __thread int m = 0; // cxx2a-note {{control flows through the definition of a thread_local variable}} \ // cxx23-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++23}} return m; } -constexpr int h(int n) { // expected-error {{constexpr function never produces a constant expression}} - static const int m = n; // expected-note {{control flows through the definition of a static variable}} \ +constexpr int h(int n) { // cxx2a-error {{constexpr function never produces a constant expression}} + static const int m = n; // cxx2a-note {{control flows through the definition of a static variable}} \ // cxx23-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++23}} return &m - &m; } -constexpr int i(int n) { // expected-error {{constexpr function never produces a constant expression}} - thread_local const int m = n; // expected-note {{control flows through the definition of a thread_local variable}} \ +constexpr int i(int n) { // cxx2a-error {{constexpr function never produces a constant expression}} + thread_local const int m = n; // cxx2a-note {{control flows through the definition of a thread_local variable}} \ // cxx23-warning {{definition of a thread_local variable in a constexpr function is incompatible with C++ standards before C++23}} return &m - &m; } diff --git a/clang/test/SemaCXX/cxx23-invalid-constexpr.cpp b/clang/test/SemaCXX/cxx23-invalid-constexpr.cpp new file mode 100644 index 000000000000..4dc16c59d805 --- /dev/null +++ b/clang/test/SemaCXX/cxx23-invalid-constexpr.cpp @@ -0,0 +1,159 @@ +// RUN: %clang_cc1 -fsyntax-only -verify=expected -std=c++23 %s + +// This test covers modifications made by P2448R2. + +// Check that there is no error when a constexpr function that never produces a +// constant expression, but still an error if such function is called from +// constexpr context. +constexpr int F(int N) { + double D = 2.0 / 0.0; // expected-note {{division by zero}} + return 1; +} + +constexpr int F0(int N) { + if (N == 0) + double d2 = 2.0 / 0.0; // expected-note {{division by zero}} + return 1; +} + +template +constexpr int FT(T N) { + double D = 2.0 / 0.0; // expected-note {{division by zero}} + return 1; +} + +class NonLiteral { // expected-note {{'NonLiteral' is not literal because it is not an aggregate and has no constexpr constructors}} +public: + NonLiteral() {} + ~NonLiteral() {} +}; + +constexpr NonLiteral F1() { + return NonLiteral{}; +} + +constexpr int F2(NonLiteral N) { + return 8; +} + +class Derived : public NonLiteral { + constexpr ~Derived() {}; +}; + +class Derived1 : public NonLiteral { + constexpr Derived1() : NonLiteral () {} +}; + + +struct X { + X(); + X(const X&); + X(X&&); + X& operator=(X&); + X& operator=(X&& other); + bool operator==(X const&) const; +}; + +template +struct Wrapper { + constexpr Wrapper() = default; + constexpr Wrapper(Wrapper const&) = default; + constexpr Wrapper(T const& t) : t(t) { } + constexpr Wrapper(Wrapper &&) = default; + constexpr X get() const { return t; } + constexpr bool operator==(Wrapper const&) const = default; + private: + T t; +}; + +struct WrapperNonT { + constexpr WrapperNonT() = default; + constexpr WrapperNonT(WrapperNonT const&) = default; + constexpr WrapperNonT(X const& t) : t(t) { } + constexpr WrapperNonT(WrapperNonT &&) = default; + constexpr WrapperNonT& operator=(WrapperNonT &) = default; + constexpr WrapperNonT& operator=(WrapperNonT&& other) = default; + constexpr X get() const { return t; } + constexpr bool operator==(WrapperNonT const&) const = default; + private: + X t; +}; + +struct NonDefaultMembers { + constexpr NonDefaultMembers() {}; // expected-note {{non-literal type 'X' cannot be used in a constant expression}} + constexpr NonDefaultMembers(NonDefaultMembers const&) {}; + constexpr NonDefaultMembers(NonDefaultMembers &&) {}; + constexpr NonDefaultMembers& operator=(NonDefaultMembers &other) {this->t = other.t; return *this;} + constexpr NonDefaultMembers& operator=(NonDefaultMembers&& other) {this->t = other.t; return *this;} + constexpr bool operator==(NonDefaultMembers const& other) const {return this->t == other.t;} + X t; +}; + +int Glob = 0; +class C1 { +public: + constexpr C1() : D(Glob) {}; +private: + int D; +}; + +void test() { + + constexpr int A = F(3); // expected-error {{constexpr variable 'A' must be initialized by a constant expression}} + // expected-note@-1 {{in call}} + F(3); + constexpr int B = F0(0); // expected-error {{constexpr variable 'B' must be initialized by a constant expression}} + // expected-note@-1 {{in call}} + F0(0); + constexpr auto C = F1(); // expected-error {{constexpr variable cannot have non-literal type 'const NonLiteral'}} + F1(); + NonLiteral L; + constexpr auto D = F2(L); // expected-error {{constexpr variable 'D' must be initialized by a constant expression}} + // expected-note@-1 {{non-literal type 'NonLiteral' cannot be used in a constant expression}} + + constexpr auto E = FT(1); // expected-error {{constexpr variable 'E' must be initialized by a constant expression}} + // expected-note@-1 {{in call}} + F2(L); + + Wrapper x; + WrapperNonT x1; + NonDefaultMembers x2; + + // TODO these produce notes with an invalid source location. + // static_assert((Wrapper(), true)); + // static_assert((WrapperNonT(), true),""); + + static_assert((NonDefaultMembers(), true),""); // expected-error{{expression is not an integral constant expression}} \ + // expected-note {{in call to}} + constexpr bool FFF = (NonDefaultMembers() == NonDefaultMembers()); // expected-error{{must be initialized by a constant expression}} \ + // expected-note{{non-literal}} +} + +struct A { + A (); + ~A(); +}; + +template +struct opt +{ + union { + char c; + T data; + }; + + constexpr opt() {} + + constexpr ~opt() { + if (engaged) + data.~T(); + } + + bool engaged = false; +}; + +consteval void foo() { + opt a; +} + +void bar() { foo(); } diff --git a/clang/test/SemaCXX/cxx2a-consteval.cpp b/clang/test/SemaCXX/cxx2a-consteval.cpp index d8482ec53f0e..192621225a54 100644 --- a/clang/test/SemaCXX/cxx2a-consteval.cpp +++ b/clang/test/SemaCXX/cxx2a-consteval.cpp @@ -54,7 +54,7 @@ struct C { struct D { C c; - consteval D() = default; // expected-error {{cannot be consteval}} + consteval D() = default; // expected-error {{cannot be marked consteval}} consteval ~D() = default; // expected-error {{destructor cannot be declared consteval}} }; diff --git a/clang/test/SemaCXX/deduced-return-type-cxx14.cpp b/clang/test/SemaCXX/deduced-return-type-cxx14.cpp index 415bbbf1a0bc..431d77ca785b 100644 --- a/clang/test/SemaCXX/deduced-return-type-cxx14.cpp +++ b/clang/test/SemaCXX/deduced-return-type-cxx14.cpp @@ -1,8 +1,8 @@ // RUN: %clang_cc1 -std=c++23 -fsyntax-only -verify=expected,since-cxx20,since-cxx14,cxx20_23,cxx23 %s // RUN: %clang_cc1 -std=c++23 -fsyntax-only -verify=expected,since-cxx20,since-cxx14,cxx20_23,cxx23 %s -fdelayed-template-parsing -DDELAYED_TEMPLATE_PARSING -// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,since-cxx20,since-cxx14,cxx14_20,cxx20_23 %s -// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,since-cxx20,since-cxx14,cxx14_20,cxx20_23 %s -fdelayed-template-parsing -DDELAYED_TEMPLATE_PARSING +// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,cxx20,since-cxx20,since-cxx14,cxx14_20,cxx20_23 %s +// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,cxx20,since-cxx20,since-cxx14,cxx14_20,cxx20_23 %s -fdelayed-template-parsing -DDELAYED_TEMPLATE_PARSING // RUN: %clang_cc1 -std=c++14 -fsyntax-only -verify=expected,since-cxx14,cxx14_20,cxx14 %s // RUN: %clang_cc1 -std=c++14 -fsyntax-only -verify=expected,since-cxx14,cxx14_20,cxx14 %s -fdelayed-template-parsing -DDELAYED_TEMPLATE_PARSING @@ -299,8 +299,8 @@ namespace Constexpr { constexpr int q = Y().f(); // expected-error {{must be initialized by a constant expression}} expected-note {{in call to 'Y().f()'}} } struct NonLiteral { ~NonLiteral(); } nl; // cxx14-note {{user-provided destructor}} - // cxx20_23-note@-1 {{'NonLiteral' is not literal because its destructor is not constexpr}} - constexpr auto f2(int n) { return nl; } // expected-error {{return type 'struct NonLiteral' is not a literal type}} + // cxx20-note@-1 {{'NonLiteral' is not literal because its destructor is not constexpr}} + constexpr auto f2(int n) { return nl; } // cxx14_20-error {{constexpr function's return type 'struct NonLiteral' is not a literal type}} } // It's not really clear whether these are valid, but this matches g++. diff --git a/clang/test/SemaOpenCLCXX/addrspace-constructors.clcpp b/clang/test/SemaOpenCLCXX/addrspace-constructors.clcpp index 1b97484767b1..067a404c489a 100644 --- a/clang/test/SemaOpenCLCXX/addrspace-constructors.clcpp +++ b/clang/test/SemaOpenCLCXX/addrspace-constructors.clcpp @@ -54,5 +54,5 @@ struct Z { struct W { int w; - constexpr W() __constant = default; // expected-error {{defaulted definition of default constructor is not constexpr}} + constexpr W() __constant = default; // expected-error {{defaulted definition of default constructor cannot be marked constexpr}} }; diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html index 421b3426b006..5ed27cdd43b3 100755 --- a/clang/www/cxx_status.html +++ b/clang/www/cxx_status.html @@ -356,14 +356,7 @@ C++23, informally referred to as C++26.

Relaxing some constexpr restrictions P2448R2 - -
Clang 17 (Partial) - We do not support outside of defaulted special memeber functions the change that constexpr functions no - longer have to be constexpr compatible but rather support a less restricted requirements for constexpr - functions. Which include allowing non-literal types as return values and parameters, allow calling of - non-constexpr functions and constructors. -
- + Clang 19 Using unknown pointers and references in constant expressions -- GitLab From c59129a7c79448837d665de8f2743ad4b14666f6 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Thu, 7 Mar 2024 16:50:26 +0800 Subject: [PATCH 413/929] [RISCV] Recursively split concat_vector into smaller LMULs (#83035) This is the concat_vector equivalent of #81312, in that we recursively split concat_vectors with more than two operands into smaller concat_vectors. This allows us to break up the chain of vslideups, as well as perform the vslideups at a smaller LMUL, which in turn reduces register pressure as the previous lowering performed N vslideups at the highest result LMUL. For now, it stops splitting past MF2. This is done as a DAG combine so that any undef operands are combined away: If we do this during lowering then we end up with unnecessary vslideups of undefs. --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 60 +- .../CodeGen/RISCV/rvv/active_lane_mask.ll | 91 +- .../RISCV/rvv/combine-store-extract-crash.ll | 4 +- .../CodeGen/RISCV/rvv/extract-subvector.ll | 3 +- .../RISCV/rvv/fixed-vectors-shuffle-concat.ll | 307 ++--- .../rvv/fixed-vectors-strided-load-combine.ll | 126 +- .../CodeGen/RISCV/rvv/fpclamptosat_vec.ll | 1066 ++++++++++------- llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll | 13 +- llvm/test/CodeGen/RISCV/rvv/pr63596.ll | 37 +- 9 files changed, 926 insertions(+), 781 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 4c3dc63afd87..750d70c03eab 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -15283,13 +15283,62 @@ static SDValue performINSERT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG, return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); } +// Recursively split up concat_vectors with more than 2 operands: +// +// concat_vector op1, op2, op3, op4 +// -> +// concat_vector (concat_vector op1, op2), (concat_vector op3, op4) +// +// This reduces the length of the chain of vslideups and allows us to perform +// the vslideups at a smaller LMUL, limited to MF2. +// +// We do this as a DAG combine rather than during lowering so that any undef +// operands can get combined away. +static SDValue +performCONCAT_VECTORSSplitCombine(SDNode *N, SelectionDAG &DAG, + const RISCVTargetLowering &TLI) { + SDLoc DL(N); + + if (N->getNumOperands() <= 2) + return SDValue(); + + if (!TLI.isTypeLegal(N->getValueType(0))) + return SDValue(); + MVT VT = N->getSimpleValueType(0); + + // Don't split any further than MF2. + MVT ContainerVT = VT; + if (VT.isFixedLengthVector()) + ContainerVT = getContainerForFixedLengthVector(DAG, VT, TLI.getSubtarget()); + if (ContainerVT.bitsLT(getLMUL1VT(ContainerVT))) + return SDValue(); + + MVT HalfVT = VT.getHalfNumVectorElementsVT(); + assert(isPowerOf2_32(N->getNumOperands())); + size_t HalfNumOps = N->getNumOperands() / 2; + SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT, + N->ops().take_front(HalfNumOps)); + SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT, + N->ops().drop_front(HalfNumOps)); + + // Lower to an insert_subvector directly so the concat_vectors don't get + // recombined. + SDValue Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Lo, + DAG.getVectorIdxConstant(0, DL)); + Vec = DAG.getNode( + ISD::INSERT_SUBVECTOR, DL, VT, Vec, Hi, + DAG.getVectorIdxConstant(HalfVT.getVectorMinNumElements(), DL)); + return Vec; +} + // If we're concatenating a series of vector loads like // concat_vectors (load v4i8, p+0), (load v4i8, p+n), (load v4i8, p+n*2) ... // Then we can turn this into a strided load by widening the vector elements // vlse32 p, stride=n -static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG, - const RISCVSubtarget &Subtarget, - const RISCVTargetLowering &TLI) { +static SDValue +performCONCAT_VECTORSStridedLoadCombine(SDNode *N, SelectionDAG &DAG, + const RISCVSubtarget &Subtarget, + const RISCVTargetLowering &TLI) { SDLoc DL(N); EVT VT = N->getValueType(0); @@ -16394,7 +16443,10 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N, return V; break; case ISD::CONCAT_VECTORS: - if (SDValue V = performCONCAT_VECTORSCombine(N, DAG, Subtarget, *this)) + if (SDValue V = + performCONCAT_VECTORSStridedLoadCombine(N, DAG, Subtarget, *this)) + return V; + if (SDValue V = performCONCAT_VECTORSSplitCombine(N, DAG, *this)) return V; break; case ISD::INSERT_VECTOR_ELT: diff --git a/llvm/test/CodeGen/RISCV/rvv/active_lane_mask.ll b/llvm/test/CodeGen/RISCV/rvv/active_lane_mask.ll index 87d95d7596d4..139579b3d2a3 100644 --- a/llvm/test/CodeGen/RISCV/rvv/active_lane_mask.ll +++ b/llvm/test/CodeGen/RISCV/rvv/active_lane_mask.ll @@ -161,72 +161,71 @@ define <64 x i1> @fv64(ptr %p, i64 %index, i64 %tc) { define <128 x i1> @fv128(ptr %p, i64 %index, i64 %tc) { ; CHECK-LABEL: fv128: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma ; CHECK-NEXT: lui a0, %hi(.LCPI10_0) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_0) +; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma ; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vid.v v16 -; CHECK-NEXT: vsaddu.vx v16, v16, a1 -; CHECK-NEXT: vmsltu.vx v0, v16, a2 -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 4, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 2 ; CHECK-NEXT: lui a0, %hi(.LCPI10_1) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_1) -; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) +; CHECK-NEXT: vle8.v v9, (a0) ; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 6, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 4 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v10, v16, a2 +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v8, v16, a2 +; CHECK-NEXT: vsetivli zero, 4, e8, mf2, tu, ma +; CHECK-NEXT: vslideup.vi v8, v10, 2 ; CHECK-NEXT: lui a0, %hi(.LCPI10_2) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_2) ; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 8, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 6 +; CHECK-NEXT: vle8.v v9, (a0) +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v9, v16, a2 +; CHECK-NEXT: vsetivli zero, 6, e8, mf2, tu, ma +; CHECK-NEXT: vslideup.vi v8, v9, 4 ; CHECK-NEXT: lui a0, %hi(.LCPI10_3) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_3) ; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 10, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 8 +; CHECK-NEXT: vle8.v v9, (a0) +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v9, v16, a2 +; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v8, v9, 6 +; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma ; CHECK-NEXT: lui a0, %hi(.LCPI10_4) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_4) -; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 12, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 10 +; CHECK-NEXT: vle8.v v9, (a0) +; CHECK-NEXT: vid.v v16 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v0, v16, a2 +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v9, v16, a2 +; CHECK-NEXT: vsetivli zero, 4, e8, mf2, tu, ma +; CHECK-NEXT: vslideup.vi v0, v9, 2 ; CHECK-NEXT: lui a0, %hi(.LCPI10_5) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_5) ; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 14, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 12 +; CHECK-NEXT: vle8.v v9, (a0) +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v9, v16, a2 +; CHECK-NEXT: vsetivli zero, 6, e8, mf2, tu, ma +; CHECK-NEXT: vslideup.vi v0, v9, 4 ; CHECK-NEXT: lui a0, %hi(.LCPI10_6) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_6) ; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma -; CHECK-NEXT: vslideup.vi v0, v16, 14 +; CHECK-NEXT: vle8.v v9, (a0) +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v9, v16, a2 +; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v0, v9, 6 +; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma +; CHECK-NEXT: vslideup.vi v0, v8, 8 ; CHECK-NEXT: ret %mask = call <128 x i1> @llvm.get.active.lane.mask.v128i1.i64(i64 %index, i64 %tc) ret <128 x i1> %mask diff --git a/llvm/test/CodeGen/RISCV/rvv/combine-store-extract-crash.ll b/llvm/test/CodeGen/RISCV/rvv/combine-store-extract-crash.ll index c64216180c2a..ed434deea1a8 100644 --- a/llvm/test/CodeGen/RISCV/rvv/combine-store-extract-crash.ll +++ b/llvm/test/CodeGen/RISCV/rvv/combine-store-extract-crash.ll @@ -19,7 +19,7 @@ define void @test(ptr %ref_array, ptr %sad_array) { ; RV32-NEXT: th.swia a0, (a1), 4, 0 ; RV32-NEXT: vsetivli zero, 4, e8, mf4, ta, ma ; RV32-NEXT: vle8.v v10, (a3) -; RV32-NEXT: vsetivli zero, 8, e8, m1, tu, ma +; RV32-NEXT: vsetivli zero, 8, e8, mf2, ta, ma ; RV32-NEXT: vslideup.vi v10, v9, 4 ; RV32-NEXT: vsetivli zero, 16, e32, m4, ta, ma ; RV32-NEXT: vzext.vf4 v12, v10 @@ -42,7 +42,7 @@ define void @test(ptr %ref_array, ptr %sad_array) { ; RV64-NEXT: th.swia a0, (a1), 4, 0 ; RV64-NEXT: vsetivli zero, 4, e8, mf4, ta, ma ; RV64-NEXT: vle8.v v10, (a3) -; RV64-NEXT: vsetivli zero, 8, e8, m1, tu, ma +; RV64-NEXT: vsetivli zero, 8, e8, mf2, ta, ma ; RV64-NEXT: vslideup.vi v10, v9, 4 ; RV64-NEXT: vsetivli zero, 16, e32, m4, ta, ma ; RV64-NEXT: vzext.vf4 v12, v10 diff --git a/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll b/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll index 76aa2b913c65..e15e6452163b 100644 --- a/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll +++ b/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll @@ -469,9 +469,8 @@ define @extract_nxv6f16_nxv12f16_6( %in) ; CHECK: # %bb.0: ; CHECK-NEXT: csrr a0, vlenb ; CHECK-NEXT: srli a0, a0, 2 -; CHECK-NEXT: vsetvli zero, a0, e16, m1, ta, ma -; CHECK-NEXT: vslidedown.vx v13, v10, a0 ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma +; CHECK-NEXT: vslidedown.vx v13, v10, a0 ; CHECK-NEXT: vslidedown.vx v12, v9, a0 ; CHECK-NEXT: add a1, a0, a0 ; CHECK-NEXT: vsetvli zero, a1, e16, m1, ta, ma diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll index e5bef20fd9e2..8474f95edd81 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll @@ -5,6 +5,59 @@ ; RUN: llc < %s -mtriple=riscv32 -mattr=+v -riscv-v-vector-bits-max=128 -verify-machineinstrs | FileCheck -check-prefixes=CHECK,VLS %s ; RUN: llc < %s -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -verify-machineinstrs | FileCheck -check-prefixes=CHECK,VLS %s +define <8 x i16> @concat_2xv4i16(<4 x i16> %a, <4 x i16> %b) { +; CHECK-LABEL: concat_2xv4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; CHECK-NEXT: vslideup.vi v8, v9, 4 +; CHECK-NEXT: ret + %ab = shufflevector <4 x i16> %a, <4 x i16> %b, <8 x i32> + ret <8 x i16> %ab +} + +define <8 x i16> @concat_4xv2i16(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c, <2 x i16> %d) { +; CHECK-LABEL: concat_4xv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v10, v11, 2 +; CHECK-NEXT: vslideup.vi v8, v9, 2 +; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; CHECK-NEXT: vslideup.vi v8, v10, 4 +; CHECK-NEXT: ret + %ab = shufflevector <2 x i16> %a, <2 x i16> %b, <4 x i32> + %cd = shufflevector <2 x i16> %c, <2 x i16> %d, <4 x i32> + %abcd = shufflevector <4 x i16> %ab, <4 x i16> %cd, <8 x i32> + ret <8 x i16> %abcd +} + +define <8 x i16> @concat_8xv1i16(<1 x i16> %a, <1 x i16> %b, <1 x i16> %c, <1 x i16> %d, <1 x i16> %e, <1 x i16> %f, <1 x i16> %g, <1 x i16> %h) { +; CHECK-LABEL: concat_8xv1i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 2, e16, mf2, tu, ma +; CHECK-NEXT: vslideup.vi v12, v13, 1 +; CHECK-NEXT: vsetivli zero, 3, e16, mf2, tu, ma +; CHECK-NEXT: vslideup.vi v12, v14, 2 +; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v12, v15, 3 +; CHECK-NEXT: vsetivli zero, 2, e16, mf2, tu, ma +; CHECK-NEXT: vslideup.vi v8, v9, 1 +; CHECK-NEXT: vsetivli zero, 3, e16, mf2, tu, ma +; CHECK-NEXT: vslideup.vi v8, v10, 2 +; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v8, v11, 3 +; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; CHECK-NEXT: vslideup.vi v8, v12, 4 +; CHECK-NEXT: ret + %ab = shufflevector <1 x i16> %a, <1 x i16> %b, <2 x i32> + %cd = shufflevector <1 x i16> %c, <1 x i16> %d, <2 x i32> + %abcd = shufflevector <2 x i16> %ab, <2 x i16> %cd, <4 x i32> + %ef = shufflevector <1 x i16> %e, <1 x i16> %f, <2 x i32> + %gh = shufflevector <1 x i16> %g, <1 x i16> %h, <2 x i32> + %efgh = shufflevector <2 x i16> %ef, <2 x i16> %gh, <4 x i32> + %abcdefgh = shufflevector <4 x i16> %abcd, <4 x i16> %efgh, <8 x i32> + ret <8 x i16> %abcdefgh +} + define <8 x i32> @concat_2xv4i32(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: concat_2xv4i32: ; CHECK: # %bb.0: @@ -19,14 +72,11 @@ define <8 x i32> @concat_2xv4i32(<4 x i32> %a, <4 x i32> %b) { define <8 x i32> @concat_4xv2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c, <2 x i32> %d) { ; CHECK-LABEL: concat_4xv2i32: ; CHECK: # %bb.0: -; CHECK-NEXT: vmv1r.v v12, v11 -; CHECK-NEXT: vmv1r.v v14, v9 -; CHECK-NEXT: vsetivli zero, 4, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v14, 2 -; CHECK-NEXT: vsetivli zero, 6, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v10, 4 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vi v10, v11, 2 +; CHECK-NEXT: vslideup.vi v8, v9, 2 ; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-NEXT: vslideup.vi v8, v12, 6 +; CHECK-NEXT: vslideup.vi v8, v10, 4 ; CHECK-NEXT: ret %ab = shufflevector <2 x i32> %a, <2 x i32> %b, <4 x i32> %cd = shufflevector <2 x i32> %c, <2 x i32> %d, <4 x i32> @@ -37,24 +87,18 @@ define <8 x i32> @concat_4xv2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c, <2 x define <8 x i32> @concat_8xv1i32(<1 x i32> %a, <1 x i32> %b, <1 x i32> %c, <1 x i32> %d, <1 x i32> %e, <1 x i32> %f, <1 x i32> %g, <1 x i32> %h) { ; CHECK-LABEL: concat_8xv1i32: ; CHECK: # %bb.0: -; CHECK-NEXT: vmv1r.v v16, v15 -; CHECK-NEXT: vmv1r.v v18, v13 -; CHECK-NEXT: vmv1r.v v20, v11 -; CHECK-NEXT: vmv1r.v v22, v9 -; CHECK-NEXT: vsetivli zero, 2, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v22, 1 -; CHECK-NEXT: vsetivli zero, 3, e32, m2, tu, ma +; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v14, v15, 1 +; CHECK-NEXT: vslideup.vi v12, v13, 1 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vi v12, v14, 2 +; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v10, v11, 1 +; CHECK-NEXT: vslideup.vi v8, v9, 1 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vslideup.vi v8, v10, 2 -; CHECK-NEXT: vsetivli zero, 4, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v20, 3 -; CHECK-NEXT: vsetivli zero, 5, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v12, 4 -; CHECK-NEXT: vsetivli zero, 6, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v18, 5 -; CHECK-NEXT: vsetivli zero, 7, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v14, 6 ; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-NEXT: vslideup.vi v8, v16, 7 +; CHECK-NEXT: vslideup.vi v8, v12, 4 ; CHECK-NEXT: ret %ab = shufflevector <1 x i32> %a, <1 x i32> %b, <2 x i32> %cd = shufflevector <1 x i32> %c, <1 x i32> %d, <2 x i32> @@ -80,15 +124,14 @@ define <16 x i32> @concat_2xv8i32(<8 x i32> %a, <8 x i32> %b) { define <16 x i32> @concat_4xv4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32> %d) { ; CHECK-LABEL: concat_4xv4i32: ; CHECK: # %bb.0: -; CHECK-NEXT: vmv1r.v v12, v11 -; CHECK-NEXT: vmv1r.v v16, v10 -; CHECK-NEXT: vmv1r.v v20, v9 -; CHECK-NEXT: vsetivli zero, 8, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v20, 4 -; CHECK-NEXT: vsetivli zero, 12, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v16, 8 +; CHECK-NEXT: vmv1r.v v14, v11 +; CHECK-NEXT: vmv1r.v v12, v10 +; CHECK-NEXT: vmv1r.v v10, v9 +; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-NEXT: vslideup.vi v12, v14, 4 +; CHECK-NEXT: vslideup.vi v8, v10, 4 ; CHECK-NEXT: vsetivli zero, 16, e32, m4, ta, ma -; CHECK-NEXT: vslideup.vi v8, v12, 12 +; CHECK-NEXT: vslideup.vi v8, v12, 8 ; CHECK-NEXT: ret %ab = shufflevector <4 x i32> %a, <4 x i32> %b, <8 x i32> %cd = shufflevector <4 x i32> %c, <4 x i32> %d, <8 x i32> @@ -99,26 +142,18 @@ define <16 x i32> @concat_4xv4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x define <16 x i32> @concat_8xv2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c, <2 x i32> %d, <2 x i32> %e, <2 x i32> %f, <2 x i32> %g, <2 x i32> %h) { ; CHECK-LABEL: concat_8xv2i32: ; CHECK: # %bb.0: -; CHECK-NEXT: vmv1r.v v16, v15 -; CHECK-NEXT: vmv1r.v v20, v14 -; CHECK-NEXT: vmv1r.v v24, v13 -; CHECK-NEXT: vmv1r.v v28, v11 -; CHECK-NEXT: vmv1r.v v4, v10 -; CHECK-NEXT: vmv1r.v v0, v9 -; CHECK-NEXT: vsetivli zero, 4, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v0, 2 -; CHECK-NEXT: vsetivli zero, 6, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v4, 4 -; CHECK-NEXT: vsetivli zero, 8, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v28, 6 -; CHECK-NEXT: vsetivli zero, 10, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v12, 8 -; CHECK-NEXT: vsetivli zero, 12, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v24, 10 -; CHECK-NEXT: vsetivli zero, 14, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v20, 12 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vi v14, v15, 2 +; CHECK-NEXT: vslideup.vi v12, v13, 2 +; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-NEXT: vslideup.vi v12, v14, 4 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vi v10, v11, 2 +; CHECK-NEXT: vslideup.vi v8, v9, 2 +; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-NEXT: vslideup.vi v8, v10, 4 ; CHECK-NEXT: vsetivli zero, 16, e32, m4, ta, ma -; CHECK-NEXT: vslideup.vi v8, v16, 14 +; CHECK-NEXT: vslideup.vi v8, v12, 8 ; CHECK-NEXT: ret %ab = shufflevector <2 x i32> %a, <2 x i32> %b, <4 x i32> %cd = shufflevector <2 x i32> %c, <2 x i32> %d, <4 x i32> @@ -152,29 +187,27 @@ define <32 x i32> @concat_2xv16i32(<16 x i32> %a, <16 x i32> %b) { define <32 x i32> @concat_4xv8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d) { ; VLA-LABEL: concat_4xv8i32: ; VLA: # %bb.0: -; VLA-NEXT: vmv2r.v v16, v14 -; VLA-NEXT: vmv2r.v v24, v12 -; VLA-NEXT: vmv2r.v v0, v10 -; VLA-NEXT: vsetivli zero, 16, e32, m8, tu, ma -; VLA-NEXT: vslideup.vi v8, v0, 8 -; VLA-NEXT: vsetivli zero, 24, e32, m8, tu, ma -; VLA-NEXT: vslideup.vi v8, v24, 16 +; VLA-NEXT: vmv2r.v v20, v14 +; VLA-NEXT: vmv2r.v v16, v12 +; VLA-NEXT: vmv2r.v v12, v10 +; VLA-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLA-NEXT: vslideup.vi v16, v20, 8 +; VLA-NEXT: vslideup.vi v8, v12, 8 ; VLA-NEXT: li a0, 32 ; VLA-NEXT: vsetvli zero, a0, e32, m8, ta, ma -; VLA-NEXT: vslideup.vi v8, v16, 24 +; VLA-NEXT: vslideup.vi v8, v16, 16 ; VLA-NEXT: ret ; ; VLS-LABEL: concat_4xv8i32: ; VLS: # %bb.0: -; VLS-NEXT: vmv2r.v v16, v14 -; VLS-NEXT: vmv2r.v v24, v12 -; VLS-NEXT: vmv2r.v v0, v10 -; VLS-NEXT: vsetivli zero, 16, e32, m8, tu, ma -; VLS-NEXT: vslideup.vi v8, v0, 8 -; VLS-NEXT: vsetivli zero, 24, e32, m8, tu, ma -; VLS-NEXT: vslideup.vi v8, v24, 16 +; VLS-NEXT: vmv2r.v v20, v14 +; VLS-NEXT: vmv2r.v v16, v12 +; VLS-NEXT: vmv2r.v v12, v10 +; VLS-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLS-NEXT: vslideup.vi v16, v20, 8 +; VLS-NEXT: vslideup.vi v8, v12, 8 ; VLS-NEXT: vsetvli a0, zero, e32, m8, ta, ma -; VLS-NEXT: vslideup.vi v8, v16, 24 +; VLS-NEXT: vslideup.vi v8, v16, 16 ; VLS-NEXT: ret %ab = shufflevector <8 x i32> %a, <8 x i32> %b, <16 x i32> %cd = shufflevector <8 x i32> %c, <8 x i32> %d, <16 x i32> @@ -185,123 +218,49 @@ define <32 x i32> @concat_4xv8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x define <32 x i32> @concat_8xv4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32> %d, <4 x i32> %e, <4 x i32> %f, <4 x i32> %g, <4 x i32> %h) { ; VLA-LABEL: concat_8xv4i32: ; VLA: # %bb.0: -; VLA-NEXT: addi sp, sp, -16 -; VLA-NEXT: .cfi_def_cfa_offset 16 -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 5 -; VLA-NEXT: sub sp, sp, a0 -; VLA-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb -; VLA-NEXT: vmv1r.v v16, v15 -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 3 -; VLA-NEXT: mv a1, a0 -; VLA-NEXT: slli a0, a0, 1 -; VLA-NEXT: add a0, a0, a1 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLA-NEXT: vmv1r.v v16, v14 -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 4 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLA-NEXT: vmv1r.v v16, v13 -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 3 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill +; VLA-NEXT: vmv1r.v v18, v15 +; VLA-NEXT: vmv1r.v v20, v14 +; VLA-NEXT: vmv1r.v v22, v13 ; VLA-NEXT: vmv1r.v v16, v12 -; VLA-NEXT: addi a0, sp, 16 -; VLA-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLA-NEXT: vmv1r.v v0, v11 -; VLA-NEXT: vmv1r.v v24, v10 -; VLA-NEXT: vmv1r.v v16, v9 -; VLA-NEXT: vsetivli zero, 8, e32, m8, tu, ma -; VLA-NEXT: vslideup.vi v8, v16, 4 -; VLA-NEXT: vsetivli zero, 12, e32, m8, tu, ma -; VLA-NEXT: vslideup.vi v8, v24, 8 -; VLA-NEXT: vsetivli zero, 16, e32, m8, tu, ma -; VLA-NEXT: vslideup.vi v8, v0, 12 -; VLA-NEXT: vsetivli zero, 20, e32, m8, tu, ma -; VLA-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLA-NEXT: vslideup.vi v8, v16, 16 -; VLA-NEXT: vsetivli zero, 24, e32, m8, tu, ma -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 3 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLA-NEXT: vslideup.vi v8, v16, 20 -; VLA-NEXT: vsetivli zero, 28, e32, m8, tu, ma -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 4 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLA-NEXT: vslideup.vi v8, v16, 24 +; VLA-NEXT: vmv1r.v v14, v11 +; VLA-NEXT: vmv1r.v v12, v10 +; VLA-NEXT: vmv1r.v v10, v9 +; VLA-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; VLA-NEXT: vslideup.vi v20, v18, 4 +; VLA-NEXT: vslideup.vi v16, v22, 4 +; VLA-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLA-NEXT: vslideup.vi v16, v20, 8 +; VLA-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; VLA-NEXT: vslideup.vi v12, v14, 4 +; VLA-NEXT: vslideup.vi v8, v10, 4 +; VLA-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLA-NEXT: vslideup.vi v8, v12, 8 ; VLA-NEXT: li a0, 32 ; VLA-NEXT: vsetvli zero, a0, e32, m8, ta, ma -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 3 -; VLA-NEXT: mv a1, a0 -; VLA-NEXT: slli a0, a0, 1 -; VLA-NEXT: add a0, a0, a1 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLA-NEXT: vslideup.vi v8, v16, 28 -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 5 -; VLA-NEXT: add sp, sp, a0 -; VLA-NEXT: addi sp, sp, 16 +; VLA-NEXT: vslideup.vi v8, v16, 16 ; VLA-NEXT: ret ; ; VLS-LABEL: concat_8xv4i32: ; VLS: # %bb.0: -; VLS-NEXT: addi sp, sp, -16 -; VLS-NEXT: .cfi_def_cfa_offset 16 -; VLS-NEXT: addi sp, sp, -512 -; VLS-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb -; VLS-NEXT: vmv1r.v v16, v15 -; VLS-NEXT: addi a0, sp, 400 -; VLS-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLS-NEXT: vmv1r.v v16, v14 -; VLS-NEXT: addi a0, sp, 272 -; VLS-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLS-NEXT: vmv1r.v v16, v13 -; VLS-NEXT: addi a0, sp, 144 -; VLS-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill +; VLS-NEXT: vmv1r.v v18, v15 +; VLS-NEXT: vmv1r.v v20, v14 +; VLS-NEXT: vmv1r.v v22, v13 ; VLS-NEXT: vmv1r.v v16, v12 -; VLS-NEXT: addi a0, sp, 16 -; VLS-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLS-NEXT: vmv1r.v v0, v11 -; VLS-NEXT: vmv1r.v v24, v10 -; VLS-NEXT: vmv1r.v v16, v9 -; VLS-NEXT: vsetivli zero, 8, e32, m8, tu, ma -; VLS-NEXT: vslideup.vi v8, v16, 4 -; VLS-NEXT: vsetivli zero, 12, e32, m8, tu, ma -; VLS-NEXT: vslideup.vi v8, v24, 8 -; VLS-NEXT: vsetivli zero, 16, e32, m8, tu, ma -; VLS-NEXT: vslideup.vi v8, v0, 12 -; VLS-NEXT: vsetivli zero, 20, e32, m8, tu, ma -; VLS-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLS-NEXT: vslideup.vi v8, v16, 16 -; VLS-NEXT: vsetivli zero, 24, e32, m8, tu, ma -; VLS-NEXT: addi a0, sp, 144 -; VLS-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLS-NEXT: vslideup.vi v8, v16, 20 -; VLS-NEXT: vsetivli zero, 28, e32, m8, tu, ma -; VLS-NEXT: addi a0, sp, 272 -; VLS-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLS-NEXT: vslideup.vi v8, v16, 24 +; VLS-NEXT: vmv1r.v v14, v11 +; VLS-NEXT: vmv1r.v v12, v10 +; VLS-NEXT: vmv1r.v v10, v9 +; VLS-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; VLS-NEXT: vslideup.vi v20, v18, 4 +; VLS-NEXT: vslideup.vi v16, v22, 4 +; VLS-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLS-NEXT: vslideup.vi v16, v20, 8 +; VLS-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; VLS-NEXT: vslideup.vi v12, v14, 4 +; VLS-NEXT: vslideup.vi v8, v10, 4 +; VLS-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLS-NEXT: vslideup.vi v8, v12, 8 ; VLS-NEXT: vsetvli a0, zero, e32, m8, ta, ma -; VLS-NEXT: addi a0, sp, 400 -; VLS-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLS-NEXT: vslideup.vi v8, v16, 28 -; VLS-NEXT: addi sp, sp, 512 -; VLS-NEXT: addi sp, sp, 16 +; VLS-NEXT: vslideup.vi v8, v16, 16 ; VLS-NEXT: ret %ab = shufflevector <4 x i32> %a, <4 x i32> %b, <8 x i32> %cd = shufflevector <4 x i32> %c, <4 x i32> %d, <8 x i32> diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll index ba5db552b854..37902aa18732 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll @@ -24,17 +24,15 @@ define void @widen_2xv4i16(ptr %x, ptr %z) { define void @widen_3xv4i16(ptr %x, ptr %z) { ; CHECK-LABEL: widen_3xv4i16: ; CHECK: # %bb.0: +; CHECK-NEXT: addi a2, a0, 16 ; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma -; CHECK-NEXT: vle16.v v8, (a0) -; CHECK-NEXT: addi a2, a0, 8 -; CHECK-NEXT: vle16.v v10, (a2) -; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: vle16.v v12, (a0) -; CHECK-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v10, 4 -; CHECK-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v12, 8 -; CHECK-NEXT: vse16.v v8, (a1) +; CHECK-NEXT: vle16.v v8, (a2) +; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; CHECK-NEXT: vle16.v v10, (a0) +; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma +; CHECK-NEXT: vslideup.vi v10, v8, 8 +; CHECK-NEXT: vsetivli zero, 12, e16, m2, ta, ma +; CHECK-NEXT: vse16.v v10, (a1) ; CHECK-NEXT: ret %a = load <4 x i16>, ptr %x %b.gep = getelementptr i8, ptr %x, i64 8 @@ -72,20 +70,18 @@ define void @widen_4xv4i16(ptr %x, ptr %z) { define void @widen_4xv4i16_unaligned(ptr %x, ptr %z) { ; CHECK-NO-MISALIGN-LABEL: widen_4xv4i16_unaligned: ; CHECK-NO-MISALIGN: # %bb.0: -; CHECK-NO-MISALIGN-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-NO-MISALIGN-NEXT: vsetivli zero, 8, e16, m1, ta, ma ; CHECK-NO-MISALIGN-NEXT: vle8.v v8, (a0) -; CHECK-NO-MISALIGN-NEXT: addi a2, a0, 8 -; CHECK-NO-MISALIGN-NEXT: vle8.v v10, (a2) ; CHECK-NO-MISALIGN-NEXT: addi a2, a0, 16 -; CHECK-NO-MISALIGN-NEXT: vle8.v v12, (a2) +; CHECK-NO-MISALIGN-NEXT: vle8.v v10, (a2) +; CHECK-NO-MISALIGN-NEXT: addi a2, a0, 8 ; CHECK-NO-MISALIGN-NEXT: addi a0, a0, 24 -; CHECK-NO-MISALIGN-NEXT: vle8.v v14, (a0) -; CHECK-NO-MISALIGN-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; CHECK-NO-MISALIGN-NEXT: vslideup.vi v8, v10, 4 -; CHECK-NO-MISALIGN-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; CHECK-NO-MISALIGN-NEXT: vslideup.vi v8, v12, 8 +; CHECK-NO-MISALIGN-NEXT: vle8.v v9, (a0) +; CHECK-NO-MISALIGN-NEXT: vle8.v v11, (a2) +; CHECK-NO-MISALIGN-NEXT: vslideup.vi v10, v9, 4 +; CHECK-NO-MISALIGN-NEXT: vslideup.vi v8, v11, 4 ; CHECK-NO-MISALIGN-NEXT: vsetivli zero, 16, e16, m2, ta, ma -; CHECK-NO-MISALIGN-NEXT: vslideup.vi v8, v14, 12 +; CHECK-NO-MISALIGN-NEXT: vslideup.vi v8, v10, 8 ; CHECK-NO-MISALIGN-NEXT: vse16.v v8, (a1) ; CHECK-NO-MISALIGN-NEXT: ret ; @@ -185,21 +181,14 @@ define void @strided_constant_0(ptr %x, ptr %z) { define void @strided_constant_mismatch_4xv4i16(ptr %x, ptr %z) { ; CHECK-LABEL: strided_constant_mismatch_4xv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma -; CHECK-NEXT: vle16.v v8, (a0) -; CHECK-NEXT: addi a2, a0, 2 -; CHECK-NEXT: vle16.v v10, (a2) ; CHECK-NEXT: addi a2, a0, 6 -; CHECK-NEXT: vle16.v v12, (a2) -; CHECK-NEXT: addi a0, a0, 8 -; CHECK-NEXT: vle16.v v14, (a0) -; CHECK-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v10, 4 -; CHECK-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v12, 8 -; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma -; CHECK-NEXT: vslideup.vi v8, v14, 12 -; CHECK-NEXT: vse16.v v8, (a1) +; CHECK-NEXT: li a3, 2 +; CHECK-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; CHECK-NEXT: vlse64.v v8, (a0), a3 +; CHECK-NEXT: vlse64.v v10, (a2), a3 +; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-NEXT: vslideup.vi v8, v10, 2 +; CHECK-NEXT: vse64.v v8, (a1) ; CHECK-NEXT: ret %a = load <4 x i16>, ptr %x %b.gep = getelementptr i8, ptr %x, i64 2 @@ -255,59 +244,38 @@ define void @strided_runtime_4xv4i16(ptr %x, ptr %z, i64 %s) { define void @strided_runtime_mismatch_4xv4i16(ptr %x, ptr %z, i64 %s, i64 %t) { ; RV32-LABEL: strided_runtime_mismatch_4xv4i16: ; RV32: # %bb.0: -; RV32-NEXT: vsetivli zero, 4, e16, mf2, ta, ma -; RV32-NEXT: vle16.v v8, (a0) -; RV32-NEXT: add a0, a0, a2 -; RV32-NEXT: vle16.v v10, (a0) -; RV32-NEXT: add a0, a0, a4 -; RV32-NEXT: vle16.v v12, (a0) -; RV32-NEXT: add a0, a0, a2 -; RV32-NEXT: vle16.v v14, (a0) -; RV32-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; RV32-NEXT: vslideup.vi v8, v10, 4 -; RV32-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; RV32-NEXT: vslideup.vi v8, v12, 8 -; RV32-NEXT: vsetivli zero, 16, e16, m2, ta, ma -; RV32-NEXT: vslideup.vi v8, v14, 12 -; RV32-NEXT: vse16.v v8, (a1) +; RV32-NEXT: add a3, a0, a2 +; RV32-NEXT: add a3, a3, a4 +; RV32-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; RV32-NEXT: vlse64.v v8, (a0), a2 +; RV32-NEXT: vlse64.v v10, (a3), a2 +; RV32-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; RV32-NEXT: vslideup.vi v8, v10, 2 +; RV32-NEXT: vse64.v v8, (a1) ; RV32-NEXT: ret ; ; RV64-LABEL: strided_runtime_mismatch_4xv4i16: ; RV64: # %bb.0: -; RV64-NEXT: vsetivli zero, 4, e16, mf2, ta, ma -; RV64-NEXT: vle16.v v8, (a0) -; RV64-NEXT: add a0, a0, a2 -; RV64-NEXT: vle16.v v10, (a0) -; RV64-NEXT: add a0, a0, a3 -; RV64-NEXT: vle16.v v12, (a0) -; RV64-NEXT: add a0, a0, a2 -; RV64-NEXT: vle16.v v14, (a0) -; RV64-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; RV64-NEXT: vslideup.vi v8, v10, 4 -; RV64-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; RV64-NEXT: vslideup.vi v8, v12, 8 -; RV64-NEXT: vsetivli zero, 16, e16, m2, ta, ma -; RV64-NEXT: vslideup.vi v8, v14, 12 -; RV64-NEXT: vse16.v v8, (a1) +; RV64-NEXT: add a4, a0, a2 +; RV64-NEXT: add a3, a4, a3 +; RV64-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; RV64-NEXT: vlse64.v v8, (a0), a2 +; RV64-NEXT: vlse64.v v10, (a3), a2 +; RV64-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; RV64-NEXT: vslideup.vi v8, v10, 2 +; RV64-NEXT: vse64.v v8, (a1) ; RV64-NEXT: ret ; ; ZVE64F-LABEL: strided_runtime_mismatch_4xv4i16: ; ZVE64F: # %bb.0: -; ZVE64F-NEXT: vsetivli zero, 4, e16, mf2, ta, ma -; ZVE64F-NEXT: vle16.v v8, (a0) -; ZVE64F-NEXT: add a0, a0, a2 -; ZVE64F-NEXT: vle16.v v10, (a0) -; ZVE64F-NEXT: add a0, a0, a3 -; ZVE64F-NEXT: vle16.v v12, (a0) -; ZVE64F-NEXT: add a0, a0, a2 -; ZVE64F-NEXT: vle16.v v14, (a0) -; ZVE64F-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; ZVE64F-NEXT: vslideup.vi v8, v10, 4 -; ZVE64F-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; ZVE64F-NEXT: vslideup.vi v8, v12, 8 -; ZVE64F-NEXT: vsetivli zero, 16, e16, m2, ta, ma -; ZVE64F-NEXT: vslideup.vi v8, v14, 12 -; ZVE64F-NEXT: vse16.v v8, (a1) +; ZVE64F-NEXT: add a4, a0, a2 +; ZVE64F-NEXT: add a3, a4, a3 +; ZVE64F-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; ZVE64F-NEXT: vlse64.v v8, (a0), a2 +; ZVE64F-NEXT: vlse64.v v10, (a3), a2 +; ZVE64F-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; ZVE64F-NEXT: vslideup.vi v8, v10, 2 +; ZVE64F-NEXT: vse64.v v8, (a1) ; ZVE64F-NEXT: ret %a = load <4 x i16>, ptr %x %b.gep = getelementptr i8, ptr %x, i64 %s diff --git a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll index 48ce7d623475..cbdabab65cc6 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll @@ -441,57 +441,50 @@ define <4 x i32> @stest_f16i32(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 ; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnclip.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb @@ -609,57 +602,50 @@ define <4 x i32> @utesth_f16i32(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 ; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnclipu.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb @@ -787,60 +773,53 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vslideup.vi v8, v10, 2 ; CHECK-V-NEXT: li a0, -1 ; CHECK-V-NEXT: srli a0, a0, 32 -; CHECK-V-NEXT: vmin.vx v8, v10, a0 +; CHECK-V-NEXT: vmin.vx v8, v8, a0 ; CHECK-V-NEXT: vmax.vx v10, v8, zero ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnsrl.wi v8, v10, 0 @@ -1404,90 +1383,125 @@ define <8 x i16> @stest_f16i16(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 4 ; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; CHECK-V-NEXT: vnclip.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload @@ -1682,90 +1696,125 @@ define <8 x i16> @utesth_f16i16(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 -; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma -; CHECK-V-NEXT: vnclipu.wi v8, v10, 0 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma ; CHECK-V-NEXT: csrr a0, vlenb ; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 4 +; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; CHECK-V-NEXT: vnclipu.wi v8, v10, 0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload @@ -1982,94 +2031,129 @@ define <8 x i16> @ustest_f16i16(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 +; CHECK-V-NEXT: vslideup.vi v8, v10, 4 ; CHECK-V-NEXT: lui a0, 16 ; CHECK-V-NEXT: addi a0, a0, -1 -; CHECK-V-NEXT: vmin.vx v8, v10, a0 +; CHECK-V-NEXT: vmin.vx v8, v8, a0 ; CHECK-V-NEXT: vmax.vx v10, v8, zero ; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; CHECK-V-NEXT: vnsrl.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload @@ -3728,57 +3812,50 @@ define <4 x i32> @stest_f16i32_mm(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 ; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnclip.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb @@ -3894,57 +3971,50 @@ define <4 x i32> @utesth_f16i32_mm(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 ; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnclipu.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb @@ -4071,60 +4141,53 @@ define <4 x i32> @ustest_f16i32_mm(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vslideup.vi v8, v10, 2 ; CHECK-V-NEXT: li a0, -1 ; CHECK-V-NEXT: srli a0, a0, 32 -; CHECK-V-NEXT: vmin.vx v8, v10, a0 +; CHECK-V-NEXT: vmin.vx v8, v8, a0 ; CHECK-V-NEXT: vmax.vx v10, v8, zero ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnsrl.wi v8, v10, 0 @@ -4676,90 +4739,125 @@ define <8 x i16> @stest_f16i16_mm(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 4 ; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; CHECK-V-NEXT: vnclip.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload @@ -4952,90 +5050,125 @@ define <8 x i16> @utesth_f16i16_mm(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 4 ; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; CHECK-V-NEXT: vnclipu.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload @@ -5251,94 +5384,129 @@ define <8 x i16> @ustest_f16i16_mm(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 +; CHECK-V-NEXT: vslideup.vi v8, v10, 4 ; CHECK-V-NEXT: lui a0, 16 ; CHECK-V-NEXT: addi a0, a0, -1 -; CHECK-V-NEXT: vmin.vx v8, v10, a0 +; CHECK-V-NEXT: vmin.vx v8, v8, a0 ; CHECK-V-NEXT: vmax.vx v10, v8, zero ; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; CHECK-V-NEXT: vnsrl.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload diff --git a/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll index f3ae03af7c78..0b236f6d3ff3 100644 --- a/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll @@ -2136,17 +2136,18 @@ define @mgather_baseidx_nxv32i8(ptr %base, ; RV64-NEXT: vluxei64.v v13, (a0), v24, v0.t ; RV64-NEXT: srli a1, a1, 2 ; RV64-NEXT: vsetvli a3, zero, e8, mf2, ta, ma -; RV64-NEXT: vslidedown.vx v0, v16, a1 -; RV64-NEXT: vsetvli a1, zero, e64, m8, ta, ma -; RV64-NEXT: vsext.vf8 v16, v10 -; RV64-NEXT: vsetvli zero, zero, e8, m1, ta, mu -; RV64-NEXT: vluxei64.v v14, (a0), v16, v0.t +; RV64-NEXT: vslidedown.vx v8, v16, a1 ; RV64-NEXT: vsetvli a1, zero, e8, mf4, ta, ma -; RV64-NEXT: vslidedown.vx v0, v0, a2 +; RV64-NEXT: vslidedown.vx v0, v8, a2 ; RV64-NEXT: vsetvli a1, zero, e64, m8, ta, ma ; RV64-NEXT: vsext.vf8 v16, v11 ; RV64-NEXT: vsetvli zero, zero, e8, m1, ta, mu ; RV64-NEXT: vluxei64.v v15, (a0), v16, v0.t +; RV64-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; RV64-NEXT: vsext.vf8 v16, v10 +; RV64-NEXT: vsetvli zero, zero, e8, m1, ta, mu +; RV64-NEXT: vmv1r.v v0, v8 +; RV64-NEXT: vluxei64.v v14, (a0), v16, v0.t ; RV64-NEXT: vmv4r.v v8, v12 ; RV64-NEXT: ret %ptrs = getelementptr inbounds i8, ptr %base, %idxs diff --git a/llvm/test/CodeGen/RISCV/rvv/pr63596.ll b/llvm/test/CodeGen/RISCV/rvv/pr63596.ll index c27488b18a01..d13d67fd0a88 100644 --- a/llvm/test/CodeGen/RISCV/rvv/pr63596.ll +++ b/llvm/test/CodeGen/RISCV/rvv/pr63596.ll @@ -9,39 +9,38 @@ define <4 x float> @foo(ptr %0) nounwind { ; CHECK-NEXT: sd s0, 32(sp) # 8-byte Folded Spill ; CHECK-NEXT: sd s1, 24(sp) # 8-byte Folded Spill ; CHECK-NEXT: sd s2, 16(sp) # 8-byte Folded Spill -; CHECK-NEXT: lhu s0, 6(a0) -; CHECK-NEXT: lhu s1, 4(a0) -; CHECK-NEXT: lhu s2, 0(a0) -; CHECK-NEXT: lhu a0, 2(a0) +; CHECK-NEXT: lhu s0, 0(a0) +; CHECK-NEXT: lhu s1, 2(a0) +; CHECK-NEXT: lhu s2, 4(a0) +; CHECK-NEXT: lhu a0, 6(a0) ; CHECK-NEXT: fmv.w.x fa0, a0 ; CHECK-NEXT: call __extendhfsf2 -; CHECK-NEXT: fsw fa0, 8(sp) +; CHECK-NEXT: fsw fa0, 4(sp) ; CHECK-NEXT: fmv.w.x fa0, s2 ; CHECK-NEXT: call __extendhfsf2 -; CHECK-NEXT: fsw fa0, 0(sp) +; CHECK-NEXT: fsw fa0, 12(sp) ; CHECK-NEXT: fmv.w.x fa0, s1 ; CHECK-NEXT: call __extendhfsf2 -; CHECK-NEXT: fsw fa0, 12(sp) +; CHECK-NEXT: fsw fa0, 8(sp) ; CHECK-NEXT: fmv.w.x fa0, s0 ; CHECK-NEXT: call __extendhfsf2 -; CHECK-NEXT: fsw fa0, 4(sp) -; CHECK-NEXT: addi a0, sp, 8 +; CHECK-NEXT: fsw fa0, 0(sp) +; CHECK-NEXT: addi a0, sp, 4 ; CHECK-NEXT: vsetivli zero, 1, e32, mf2, ta, ma -; CHECK-NEXT: vle32.v v9, (a0) -; CHECK-NEXT: mv a0, sp ; CHECK-NEXT: vle32.v v8, (a0) -; CHECK-NEXT: vsetivli zero, 2, e32, m1, tu, ma -; CHECK-NEXT: vslideup.vi v8, v9, 1 ; CHECK-NEXT: addi a0, sp, 12 -; CHECK-NEXT: vsetivli zero, 1, e32, mf2, ta, ma ; CHECK-NEXT: vle32.v v9, (a0) -; CHECK-NEXT: vsetivli zero, 3, e32, m1, tu, ma -; CHECK-NEXT: vslideup.vi v8, v9, 2 -; CHECK-NEXT: addi a0, sp, 4 +; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v9, v8, 1 +; CHECK-NEXT: addi a0, sp, 8 ; CHECK-NEXT: vsetivli zero, 1, e32, mf2, ta, ma -; CHECK-NEXT: vle32.v v9, (a0) +; CHECK-NEXT: vle32.v v10, (a0) +; CHECK-NEXT: mv a0, sp +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v8, v10, 1 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: vslideup.vi v8, v9, 3 +; CHECK-NEXT: vslideup.vi v8, v9, 2 ; CHECK-NEXT: ld ra, 40(sp) # 8-byte Folded Reload ; CHECK-NEXT: ld s0, 32(sp) # 8-byte Folded Reload ; CHECK-NEXT: ld s1, 24(sp) # 8-byte Folded Reload -- GitLab From 7ce1cfed9a11735f0f4ee8a3a8bebfa87ee76d07 Mon Sep 17 00:00:00 2001 From: Ryosuke Niwa Date: Thu, 7 Mar 2024 01:06:20 -0800 Subject: [PATCH 414/929] [alpha.webkit.UncountedLocalVarsChecker] Allow uncounted object references within trivial statements (#82229) This PR makes alpha.webkit.UncountedLocalVarsChecker ignore raw references and pointers to a ref counted type which appears within "trival" statements. To do this, this PR extends TrivialFunctionAnalysis so that it can also analyze "triviality" of statements as well as that of functions Each Visit* function is now augmented with withCachedResult, which is responsible for looking up and updating the cache for each Visit* functions. As this PR dramatically improves the false positive rate of the checker, it also deletes the code to ignore raw pointers and references within if and for statements. --- .../Checkers/WebKit/PtrTypesSemantics.cpp | 76 +++++++++---- .../Checkers/WebKit/PtrTypesSemantics.h | 7 +- .../WebKit/UncountedLocalVarsChecker.cpp | 82 +++++++------- .../Analysis/Checkers/WebKit/mock-types.h | 2 + .../Checkers/WebKit/uncounted-local-vars.cpp | 101 ++++++++++++++++-- 5 files changed, 203 insertions(+), 65 deletions(-) diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp index 01b191ab0eea..287f6a528700 100644 --- a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp @@ -253,6 +253,19 @@ class TrivialFunctionAnalysisVisitor return true; } + template + bool WithCachedResult(const Stmt *S, CheckFunction Function) { + // If the statement isn't in the cache, conservatively assume that + // it's not trivial until analysis completes. Insert false to the cache + // first to avoid infinite recursion. + auto [It, IsNew] = Cache.insert(std::make_pair(S, false)); + if (!IsNew) + return It->second; + bool Result = Function(); + Cache[S] = Result; + return Result; + } + public: using CacheTy = TrivialFunctionAnalysis::CacheTy; @@ -267,7 +280,7 @@ public: bool VisitCompoundStmt(const CompoundStmt *CS) { // A compound statement is allowed as long each individual sub-statement // is trivial. - return VisitChildren(CS); + return WithCachedResult(CS, [&]() { return VisitChildren(CS); }); } bool VisitReturnStmt(const ReturnStmt *RS) { @@ -279,17 +292,36 @@ public: bool VisitDeclStmt(const DeclStmt *DS) { return VisitChildren(DS); } bool VisitDoStmt(const DoStmt *DS) { return VisitChildren(DS); } - bool VisitIfStmt(const IfStmt *IS) { return VisitChildren(IS); } + bool VisitIfStmt(const IfStmt *IS) { + return WithCachedResult(IS, [&]() { return VisitChildren(IS); }); + } + bool VisitForStmt(const ForStmt *FS) { + return WithCachedResult(FS, [&]() { return VisitChildren(FS); }); + } + bool VisitCXXForRangeStmt(const CXXForRangeStmt *FS) { + return WithCachedResult(FS, [&]() { return VisitChildren(FS); }); + } + bool VisitWhileStmt(const WhileStmt *WS) { + return WithCachedResult(WS, [&]() { return VisitChildren(WS); }); + } bool VisitSwitchStmt(const SwitchStmt *SS) { return VisitChildren(SS); } bool VisitCaseStmt(const CaseStmt *CS) { return VisitChildren(CS); } bool VisitDefaultStmt(const DefaultStmt *DS) { return VisitChildren(DS); } bool VisitUnaryOperator(const UnaryOperator *UO) { // Operator '*' and '!' are allowed as long as the operand is trivial. - if (UO->getOpcode() == UO_Deref || UO->getOpcode() == UO_AddrOf || - UO->getOpcode() == UO_LNot) + auto op = UO->getOpcode(); + if (op == UO_Deref || op == UO_AddrOf || op == UO_LNot) return Visit(UO->getSubExpr()); + if (UO->isIncrementOp() || UO->isDecrementOp()) { + // Allow increment or decrement of a POD type. + if (auto *RefExpr = dyn_cast(UO->getSubExpr())) { + if (auto *Decl = dyn_cast(RefExpr->getDecl())) + return Decl->isLocalVarDeclOrParm() && + Decl->getType().isPODType(Decl->getASTContext()); + } + } // Other operators are non-trivial. return false; } @@ -304,22 +336,6 @@ public: return VisitChildren(CO); } - bool VisitDeclRefExpr(const DeclRefExpr *DRE) { - if (auto *decl = DRE->getDecl()) { - if (isa(decl)) - return true; - if (isa(decl)) - return true; - if (auto *VD = dyn_cast(decl)) { - if (VD->hasConstantInitialization() && VD->getEvaluatedValue()) - return true; - auto *Init = VD->getInit(); - return !Init || Visit(Init); - } - } - return false; - } - bool VisitAtomicExpr(const AtomicExpr *E) { return VisitChildren(E); } bool VisitStaticAssertDecl(const StaticAssertDecl *SAD) { @@ -436,6 +452,11 @@ public: return true; } + bool VisitDeclRefExpr(const DeclRefExpr *DRE) { + // The use of a variable is trivial. + return true; + } + // Constant literal expressions are always trivial bool VisitIntegerLiteral(const IntegerLiteral *E) { return true; } bool VisitFloatingLiteral(const FloatingLiteral *E) { return true; } @@ -449,7 +470,7 @@ public: } private: - CacheTy Cache; + CacheTy &Cache; }; bool TrivialFunctionAnalysis::isTrivialImpl( @@ -474,4 +495,17 @@ bool TrivialFunctionAnalysis::isTrivialImpl( return Result; } +bool TrivialFunctionAnalysis::isTrivialImpl( + const Stmt *S, TrivialFunctionAnalysis::CacheTy &Cache) { + // If the statement isn't in the cache, conservatively assume that + // it's not trivial until analysis completes. Unlike a function case, + // we don't insert an entry into the cache until Visit returns + // since Visit* functions themselves make use of the cache. + + TrivialFunctionAnalysisVisitor V(Cache); + bool Result = V.Visit(S); + assert(Cache.contains(S) && "Top-level statement not properly cached!"); + return Result; +} + } // namespace clang diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h index e07cd3139574..9ed8e7cab6ab 100644 --- a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h +++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h @@ -11,6 +11,7 @@ #include "llvm/ADT/APInt.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/PointerUnion.h" #include namespace clang { @@ -19,6 +20,7 @@ class CXXMethodDecl; class CXXRecordDecl; class Decl; class FunctionDecl; +class Stmt; class Type; // Ref-countability of a type is implicitly defined by Ref and RefPtr @@ -71,14 +73,17 @@ class TrivialFunctionAnalysis { public: /// \returns true if \p D is a "trivial" function. bool isTrivial(const Decl *D) const { return isTrivialImpl(D, TheCache); } + bool isTrivial(const Stmt *S) const { return isTrivialImpl(S, TheCache); } private: friend class TrivialFunctionAnalysisVisitor; - using CacheTy = llvm::DenseMap; + using CacheTy = + llvm::DenseMap, bool>; mutable CacheTy TheCache{}; static bool isTrivialImpl(const Decl *D, CacheTy &Cache); + static bool isTrivialImpl(const Stmt *S, CacheTy &Cache); }; } // namespace clang diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp index 5a72f53b12ed..6036ad58cf25 100644 --- a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp @@ -26,28 +26,6 @@ using namespace ento; namespace { -// for ( int a = ...) ... true -// for ( int a : ...) ... true -// if ( int* a = ) ... true -// anything else ... false -bool isDeclaredInForOrIf(const VarDecl *Var) { - assert(Var); - auto &ASTCtx = Var->getASTContext(); - auto parent = ASTCtx.getParents(*Var); - - if (parent.size() == 1) { - if (auto *DS = parent.begin()->get()) { - DynTypedNodeList grandParent = ASTCtx.getParents(*DS); - if (grandParent.size() == 1) { - return grandParent.begin()->get() || - grandParent.begin()->get() || - grandParent.begin()->get(); - } - } - } - return false; -} - // FIXME: should be defined by anotations in the future bool isRefcountedStringsHack(const VarDecl *V) { assert(V); @@ -143,6 +121,11 @@ public: // want to visit those, so we make our own RecursiveASTVisitor. struct LocalVisitor : public RecursiveASTVisitor { const UncountedLocalVarsChecker *Checker; + + TrivialFunctionAnalysis TFA; + + using Base = RecursiveASTVisitor; + explicit LocalVisitor(const UncountedLocalVarsChecker *Checker) : Checker(Checker) { assert(Checker); @@ -155,6 +138,36 @@ public: Checker->visitVarDecl(V); return true; } + + bool TraverseIfStmt(IfStmt *IS) { + if (!TFA.isTrivial(IS)) + return Base::TraverseIfStmt(IS); + return true; + } + + bool TraverseForStmt(ForStmt *FS) { + if (!TFA.isTrivial(FS)) + return Base::TraverseForStmt(FS); + return true; + } + + bool TraverseCXXForRangeStmt(CXXForRangeStmt *FRS) { + if (!TFA.isTrivial(FRS)) + return Base::TraverseCXXForRangeStmt(FRS); + return true; + } + + bool TraverseWhileStmt(WhileStmt *WS) { + if (!TFA.isTrivial(WS)) + return Base::TraverseWhileStmt(WS); + return true; + } + + bool TraverseCompoundStmt(CompoundStmt *CS) { + if (!TFA.isTrivial(CS)) + return Base::TraverseCompoundStmt(CS); + return true; + } }; LocalVisitor visitor(this); @@ -189,18 +202,16 @@ public: dyn_cast_or_null(Ref->getFoundDecl())) { const auto *MaybeGuardianArgType = MaybeGuardian->getType().getTypePtr(); - if (!MaybeGuardianArgType) - return; - const CXXRecordDecl *const MaybeGuardianArgCXXRecord = - MaybeGuardianArgType->getAsCXXRecordDecl(); - if (!MaybeGuardianArgCXXRecord) - return; - - if (MaybeGuardian->isLocalVarDecl() && - (isRefCounted(MaybeGuardianArgCXXRecord) || - isRefcountedStringsHack(MaybeGuardian)) && - isGuardedScopeEmbeddedInGuardianScope(V, MaybeGuardian)) { - return; + if (MaybeGuardianArgType) { + const CXXRecordDecl *const MaybeGuardianArgCXXRecord = + MaybeGuardianArgType->getAsCXXRecordDecl(); + if (MaybeGuardianArgCXXRecord) { + if (MaybeGuardian->isLocalVarDecl() && + (isRefCounted(MaybeGuardianArgCXXRecord) || + isRefcountedStringsHack(MaybeGuardian)) && + isGuardedScopeEmbeddedInGuardianScope(V, MaybeGuardian)) + return; + } } // Parameters are guaranteed to be safe for the duration of the call @@ -219,9 +230,6 @@ public: if (!V->isLocalVarDecl()) return true; - if (isDeclaredInForOrIf(V)) - return true; - return false; } diff --git a/clang/test/Analysis/Checkers/WebKit/mock-types.h b/clang/test/Analysis/Checkers/WebKit/mock-types.h index e2b3401d4073..aab99197dfa4 100644 --- a/clang/test/Analysis/Checkers/WebKit/mock-types.h +++ b/clang/test/Analysis/Checkers/WebKit/mock-types.h @@ -62,6 +62,8 @@ struct RefCountable { static Ref create(); void ref() {} void deref() {} + void method(); + int trivial() { return 123; } }; template T *downcast(T *t) { return t; } diff --git a/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp b/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp index 0fcd3b21376c..00673e91f471 100644 --- a/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp +++ b/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp @@ -2,6 +2,8 @@ #include "mock-types.h" +void someFunction(); + namespace raw_ptr { void foo() { RefCountable *bar; @@ -16,6 +18,13 @@ void foo_ref() { RefCountable automatic; RefCountable &bar = automatic; // expected-warning@-1{{Local variable 'bar' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}} + someFunction(); + bar.method(); +} + +void foo_ref_trivial() { + RefCountable automatic; + RefCountable &bar = automatic; } void bar_ref(RefCountable &) {} @@ -32,6 +41,8 @@ void foo2() { // missing embedded scope here RefCountable *bar = foo.get(); // expected-warning@-1{{Local variable 'bar' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}} + someFunction(); + bar->method(); } void foo3() { @@ -47,11 +58,35 @@ void foo4() { { RefCountable *bar = foo.get(); } } } + +void foo5() { + RefPtr foo; + auto* bar = foo.get(); + bar->trivial(); +} + +void foo6() { + RefPtr foo; + auto* bar = foo.get(); + // expected-warning@-1{{Local variable 'bar' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}} + bar->method(); +} + +struct SelfReferencingStruct { + SelfReferencingStruct* ptr; + RefCountable* obj { nullptr }; +}; + +void foo7(RefCountable* obj) { + SelfReferencingStruct bar = { &bar, obj }; + bar.obj->method(); +} + } // namespace guardian_scopes namespace auto_keyword { class Foo { - RefCountable *provide_ref_ctnbl() { return nullptr; } + RefCountable *provide_ref_ctnbl(); void evil_func() { RefCountable *bar = provide_ref_ctnbl(); @@ -62,13 +97,24 @@ class Foo { // expected-warning@-1{{Local variable 'baz2' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}} [[clang::suppress]] auto *baz_suppressed = provide_ref_ctnbl(); // no-warning } + + void func() { + RefCountable *bar = provide_ref_ctnbl(); + // expected-warning@-1{{Local variable 'bar' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}} + if (bar) + bar->method(); + } }; } // namespace auto_keyword namespace guardian_casts { void foo1() { RefPtr foo; - { RefCountable *bar = downcast(foo.get()); } + { + RefCountable *bar = downcast(foo.get()); + bar->method(); + } + foo->method(); } void foo2() { @@ -76,6 +122,7 @@ void foo2() { { RefCountable *bar = static_cast(downcast(foo.get())); + someFunction(); } } } // namespace guardian_casts @@ -83,7 +130,11 @@ void foo2() { namespace guardian_ref_conversion_operator { void foo() { Ref rc; - { RefCountable &rr = rc; } + { + RefCountable &rr = rc; + rr.method(); + someFunction(); + } } } // namespace guardian_ref_conversion_operator @@ -92,9 +143,47 @@ RefCountable *provide_ref_ctnbl() { return nullptr; } void foo() { // no warnings - if (RefCountable *a = provide_ref_ctnbl()) { } - for (RefCountable *a = provide_ref_ctnbl(); a != nullptr;) { } + if (RefCountable *a = provide_ref_ctnbl()) + a->trivial(); + for (RefCountable *b = provide_ref_ctnbl(); b != nullptr;) + b->trivial(); RefCountable *array[1]; - for (RefCountable *a : array) { } + for (RefCountable *c : array) + c->trivial(); + while (RefCountable *d = provide_ref_ctnbl()) + d->trivial(); + do { + RefCountable *e = provide_ref_ctnbl(); + e->trivial(); + } while (1); + someFunction(); } + +void bar() { + if (RefCountable *a = provide_ref_ctnbl()) { + // expected-warning@-1{{Local variable 'a' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}} + a->method(); + } + for (RefCountable *b = provide_ref_ctnbl(); b != nullptr;) { + // expected-warning@-1{{Local variable 'b' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}} + b->method(); + } + RefCountable *array[1]; + for (RefCountable *c : array) { + // expected-warning@-1{{Local variable 'c' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}} + c->method(); + } + + while (RefCountable *d = provide_ref_ctnbl()) { + // expected-warning@-1{{Local variable 'd' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}} + d->method(); + } + do { + RefCountable *e = provide_ref_ctnbl(); + // expected-warning@-1{{Local variable 'e' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}} + e->method(); + } while (1); + someFunction(); +} + } // namespace ignore_for_if -- GitLab From a6382de3999280ef7bf8bb63750686cdad889cd5 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Thu, 7 Mar 2024 14:39:59 +0530 Subject: [PATCH 415/929] AMDGPU: Refactor mfma hazard handling [NFC] (#84276) Try to make this editable by using functions for the number of wait states as a function of the number of passes. I'm assuming the current hazard test coverage is comprehensive. This could probably use another round to further simplify it. Alternatively, I believe this could all be expressed in a constant table indexed by an instruction classify function and number of passes. --- .../lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 334 ++++++++++-------- 1 file changed, 179 insertions(+), 155 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp index 7bed0d8ef0d6..e515b729e7d7 100644 --- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp +++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp @@ -2136,6 +2136,41 @@ int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) { return WaitStatesNeeded; } +static int +GFX940_XDL_N_PassWritesVGPROverlappedSMFMASrcCWaitStates(int NumPasses) { + // 2 pass -> 3 + // 4 pass -> 5 + // 8 pass -> 9 + // 16 pass -> 17 + return NumPasses + 1; +} + +static int +GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates(int NumPasses) { + // 2 pass -> 2 + // 4 pass -> 4 + // 8 pass -> 8 + // 16 pass -> 16 + return NumPasses; +} + +static int +GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses) { + // 2 pass -> 4 + // 4 pass -> 6 + // 8 pass -> 10 + // 16 pass -> 18 + return NumPasses + 2; +} + +static int GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses) { + // 2 pass -> 5 + // 4 pass -> 7 + // 8 pass -> 11 + // 16 pass -> 19 + return NumPasses + 3; +} + int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { int WaitStatesNeeded = 0; unsigned Opc = MI->getOpcode(); @@ -2164,13 +2199,6 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { for (const MachineOperand &Use : MI->explicit_uses()) { const int LegacyVALUNotDotWritesVGPRWaitStates = 2; const int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2; - const int GFX940_XDL2PassWritesVGPROverlappedSMFMASrcCWaitStates = 3; - const int GFX940_XDL4PassWritesVGPROverlappedSMFMASrcCWaitStates = 5; - const int GFX940_SMFMA4PassWritesVGPROverlappedSMFMASrcCWaitStates = 4; - const int GFX940_XDL8PassWritesVGPROverlappedSMFMASrcCWaitStates = 9; - const int GFX940_SMFMA8PassWritesVGPROverlappedSMFMASrcCWaitStates = 8; - const int GFX940_XDL16PassWritesVGPROverlappedSMFMASrcCWaitStates = 17; - const int GFX940_SMFMA16PassWritesVGPROverlappedSMFMASrcCWaitStates = 16; const int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8; const int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16; const int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3; @@ -2181,14 +2209,6 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5; const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11; const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19; - const int GFX940_SMFMA2PassWritesVGPROverlappedSrcABWaitStates = 4; - const int GFX940_SMFMA4PassWritesVGPROverlappedSrcABWaitStates = 6; - const int GFX940_SMFMA8PassWritesVGPROverlappedSrcABWaitStates = 10; - const int GFX940_SMFMA16PassWritesVGPROverlappedSrcABWaitStates = 18; - const int GFX940_XDL2PassWritesVGPROverlappedSrcABWaitStates = 5; - const int GFX940_XDL4PassWritesVGPROverlappedSrcABWaitStates = 7; - const int GFX940_XDL8PassWritesVGPROverlappedSrcABWaitStates = 11; - const int GFX940_XDL16PassWritesVGPROverlappedSrcABWaitStates = 19; const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6; const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11; const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4; @@ -2250,42 +2270,40 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates; break; default: - if (ST.hasGFX940Insts() && isXDL(ST, *MI) && !isXDL(ST, *MI1)) + int NumPasses = TSchedModel.computeInstrLatency(MI1); + if (ST.hasGFX940Insts()) { + if (isXDL(ST, *MI) && !isXDL(ST, *MI1)) + break; + + NeedWaitStates = + isXDL(ST, *MI1) + ? GFX940_XDL_N_PassWritesVGPROverlappedSMFMASrcCWaitStates( + NumPasses) + : GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates( + NumPasses); break; - switch (TSchedModel.computeInstrLatency(MI1)) { + } + + switch (NumPasses) { case 2: - NeedWaitStates = ST.hasGFX940Insts() - ? isXDL(ST, *MI1) - ? GFX940_XDL2PassWritesVGPROverlappedSMFMASrcCWaitStates - : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates - : isDGEMM(Opc) - ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates - : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates; - break; - case 4: - assert(ST.hasGFX940Insts()); - NeedWaitStates = isXDL(ST, *MI1) - ? GFX940_XDL4PassWritesVGPROverlappedSMFMASrcCWaitStates - : GFX940_SMFMA4PassWritesVGPROverlappedSMFMASrcCWaitStates; + NeedWaitStates = + isDGEMM(Opc) ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates + : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates; break; case 8: - NeedWaitStates = ST.hasGFX940Insts() - ? isXDL(ST, *MI1) - ? GFX940_XDL8PassWritesVGPROverlappedSMFMASrcCWaitStates - : GFX940_SMFMA8PassWritesVGPROverlappedSMFMASrcCWaitStates - : isDGEMM(Opc) - ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates - : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates; + NeedWaitStates = + isDGEMM(Opc) + ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates + : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates; + break; + case 16: + NeedWaitStates = + isDGEMM(Opc) + ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates + : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates; break; - case 16: [[fallthrough]]; default: - NeedWaitStates = ST.hasGFX940Insts() - ? isXDL(ST, *MI1) - ? GFX940_XDL16PassWritesVGPROverlappedSMFMASrcCWaitStates - : GFX940_SMFMA16PassWritesVGPROverlappedSMFMASrcCWaitStates - : isDGEMM(Opc) - ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates - : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates; + llvm_unreachable("unexpected number of passes"); } } } @@ -2302,34 +2320,30 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates; break; default: - switch (TSchedModel.computeInstrLatency(MI1)) { + int NumPasses = TSchedModel.computeInstrLatency(MI1); + + if (ST.hasGFX940Insts()) { + NeedWaitStates = + isXDL(ST, *MI1) + ? GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates( + NumPasses) + : GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates( + NumPasses); + break; + } + + switch (NumPasses) { case 2: - NeedWaitStates = ST.hasGFX940Insts() - ? isXDL(ST, *MI1) - ? GFX940_XDL2PassWritesVGPROverlappedSrcABWaitStates - : GFX940_SMFMA2PassWritesVGPROverlappedSrcABWaitStates - : SMFMA4x4WritesVGPROverlappedSrcABWaitStates; + NeedWaitStates = SMFMA4x4WritesVGPROverlappedSrcABWaitStates; break; case 4: - assert(ST.hasGFX940Insts()); - NeedWaitStates = isXDL(ST, *MI1) - ? GFX940_XDL4PassWritesVGPROverlappedSrcABWaitStates - : GFX940_SMFMA4PassWritesVGPROverlappedSrcABWaitStates; - break; + llvm_unreachable("unexpected number of passes for mfma"); case 8: - NeedWaitStates = ST.hasGFX940Insts() - ? isXDL(ST, *MI1) - ? GFX940_XDL8PassWritesVGPROverlappedSrcABWaitStates - : GFX940_SMFMA8PassWritesVGPROverlappedSrcABWaitStates - : SMFMA16x16WritesVGPROverlappedSrcABWaitStates; + NeedWaitStates = SMFMA16x16WritesVGPROverlappedSrcABWaitStates; break; case 16: [[fallthrough]]; default: - NeedWaitStates = ST.hasGFX940Insts() - ? isXDL(ST, *MI1) - ? GFX940_XDL16PassWritesVGPROverlappedSrcABWaitStates - : GFX940_SMFMA16PassWritesVGPROverlappedSrcABWaitStates - : SMFMA32x32WritesVGPROverlappedSrcABWaitStates; + NeedWaitStates = SMFMA32x32WritesVGPROverlappedSrcABWaitStates; } } } @@ -2393,6 +2407,38 @@ int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) { return WaitStatesNeeded; } +static int GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(int NumPasses) { + // 2 pass -> 4 + // 4 pass -> 6 + // 8 pass -> 10 + // 16 pass -> 18 + return NumPasses + 2; +} + +static int GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(int NumPasses) { + // 2 pass -> 5 + // 4 pass -> 7 + // 8 pass -> 11 + // 16 pass -> 19 + return NumPasses + 3; +} + +static int GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses) { + // 2 pass -> 5 + // 4 pass -> 7 + // 8 pass -> 11 + // 16 pass -> 19 + return NumPasses + 3; +} + +static int GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses) { + // 2 pass -> 4 + // 4 pass -> 6 + // 8 pass -> 10 + // 16 pass -> 18 + return NumPasses + 2; +} + int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { if (!ST.hasGFX90AInsts()) return 0; @@ -2455,14 +2501,6 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5; const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11; const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19; - const int GFX940_SMFMA2PassWriteVgprVALUMemExpReadWaitStates = 4; - const int GFX940_SMFMA4PassWriteVgprVALUMemExpReadWaitStates = 6; - const int GFX940_SMFMA8PassWriteVgprVALUMemExpReadWaitStates = 10; - const int GFX940_SMFMA16PassWriteVgprVALUMemExpReadWaitStates = 18; - const int GFX940_XDL2PassWriteVgprVALUMemExpReadWaitStates = 5; - const int GFX940_XDL4PassWriteVgprVALUMemExpReadWaitStates = 7; - const int GFX940_XDL8PassWriteVgprVALUMemExpReadWaitStates = 11; - const int GFX940_XDL16PassWriteVgprVALUMemExpReadWaitStates = 19; const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9; const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18; const int DMFMA4x4WriteVgprVALUReadWaitStates = 6; @@ -2516,47 +2554,44 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { continue; unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA); + int NumPasses = HazardDefLatency; int NeedWaitStates = MaxWaitStates; - switch (HazardDefLatency) { - case 2: - NeedWaitStates = - ST.hasGFX940Insts() - ? isXDL(ST, *MFMA) - ? GFX940_XDL2PassWriteVgprVALUMemExpReadWaitStates - : GFX940_SMFMA2PassWriteVgprVALUMemExpReadWaitStates - : SMFMA4x4WriteVgprVALUMemExpReadWaitStates; - break; - case 4: - assert(isDGEMM(MFMA->getOpcode()) || ST.hasGFX940Insts()); - NeedWaitStates = - isDGEMM(MFMA->getOpcode()) - ? IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates - : DMFMA4x4WriteVgprVALUReadWaitStates - : isXDL(ST, *MFMA) - ? GFX940_XDL4PassWriteVgprVALUMemExpReadWaitStates - : GFX940_SMFMA4PassWriteVgprVALUMemExpReadWaitStates; - break; - case 8: - NeedWaitStates = - isDGEMM(MFMA->getOpcode()) - ? IsMemOrExport ? DMFMA16x16WriteVgprMemExpReadWaitStates - : DMFMA16x16WriteVgprVALUReadWaitStates - : ST.hasGFX940Insts() - ? isXDL(ST, *MFMA) - ? GFX940_XDL8PassWriteVgprVALUMemExpReadWaitStates - : GFX940_SMFMA8PassWriteVgprVALUMemExpReadWaitStates - : SMFMA16x16WriteVgprVALUMemExpReadWaitStates; - break; - case 16: [[fallthrough]]; - default: - assert(!isDGEMM(MFMA->getOpcode())); + + if (isDGEMM(MFMA->getOpcode())) { + switch (HazardDefLatency) { + case 4: + NeedWaitStates = IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates + : DMFMA4x4WriteVgprVALUReadWaitStates; + break; + case 8: + case 16: + NeedWaitStates = IsMemOrExport + ? DMFMA16x16WriteVgprMemExpReadWaitStates + : DMFMA16x16WriteVgprVALUReadWaitStates; + break; + default: + llvm_unreachable("unexpected dgemm"); + } + } else if (ST.hasGFX940Insts()) { NeedWaitStates = - ST.hasGFX940Insts() - ? isXDL(ST, *MFMA) - ? GFX940_XDL16PassWriteVgprVALUMemExpReadWaitStates - : GFX940_SMFMA16PassWriteVgprVALUMemExpReadWaitStates - : SMFMA32x32WriteVgprVALUMemExpReadWaitStates; - break; + isXDL(ST, *MFMA) + ? GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(NumPasses) + : GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates( + NumPasses); + } else { + switch (HazardDefLatency) { + case 2: + NeedWaitStates = SMFMA4x4WriteVgprVALUMemExpReadWaitStates; + break; + case 8: + NeedWaitStates = SMFMA16x16WriteVgprVALUMemExpReadWaitStates; + break; + case 16: + NeedWaitStates = SMFMA32x32WriteVgprVALUMemExpReadWaitStates; + break; + default: + llvm_unreachable("unexpected number of passes for mfma"); + } } int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; @@ -2585,14 +2620,6 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { const int SMFMA4x4WriteVgprVALUWawWaitStates = 5; const int SMFMA16x16WriteVgprVALUWawWaitStates = 11; const int SMFMA32x32WriteVgprVALUWawWaitStates = 19; - const int GFX940_SMFMA2PassWriteVgprVALUWawWaitStates = 4; - const int GFX940_SMFMA4PassWriteVgprVALUWawWaitStates = 6; - const int GFX940_SMFMA8PassWriteVgprVALUWawWaitStates = 10; - const int GFX940_SMFMA16PassWriteVgprVALUWawWaitStates = 18; - const int GFX940_XDL2PassWriteVgprVALUWawWaitStates = 5; - const int GFX940_XDL4PassWriteVgprVALUWawWaitStates = 7; - const int GFX940_XDL8PassWriteVgprVALUWawWaitStates = 11; - const int GFX940_XDL16PassWriteVgprVALUWawWaitStates = 19; const int SMFMA4x4ReadVgprVALUWarWaitStates = 1; const int GFX940_XDL4PassReadVgprVALUWarWaitStates = 3; const int SMFMA16x16ReadVgprVALUWarWaitStates = 7; @@ -2617,42 +2644,39 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates); if (MFMA) { int NeedWaitStates = MaxWaitStates; - switch (TSchedModel.computeInstrLatency(MFMA)) { - case 2: - NeedWaitStates = ST.hasGFX940Insts() - ? isXDL(ST, *MFMA) - ? GFX940_XDL2PassWriteVgprVALUWawWaitStates - : GFX940_SMFMA2PassWriteVgprVALUWawWaitStates - : SMFMA4x4WriteVgprVALUWawWaitStates; - break; - case 4: - assert(isDGEMM(MFMA->getOpcode()) || ST.hasGFX940Insts()); - NeedWaitStates = isDGEMM(MFMA->getOpcode()) - ? DMFMA4x4WriteVgprVALUWriteWaitStates - : isXDL(ST, *MFMA) - ? GFX940_XDL4PassWriteVgprVALUWawWaitStates - : GFX940_SMFMA4PassWriteVgprVALUWawWaitStates; - break; - case 8: - NeedWaitStates = - isDGEMM(MFMA->getOpcode()) ? DMFMA16x16WriteVgprVALUWriteWaitStates - : + int NumPasses = TSchedModel.computeInstrLatency(MFMA); - ST.hasGFX940Insts() - ? isXDL(ST, *MFMA) ? GFX940_XDL8PassWriteVgprVALUWawWaitStates - : GFX940_SMFMA8PassWriteVgprVALUWawWaitStates - : SMFMA16x16WriteVgprVALUWawWaitStates; - break; - case 16: [[fallthrough]]; - default: - assert(!isDGEMM(MFMA->getOpcode())); + if (isDGEMM(MFMA->getOpcode())) { + switch (NumPasses) { + case 4: + NeedWaitStates = DMFMA4x4WriteVgprVALUWriteWaitStates; + break; + case 8: + case 16: + NeedWaitStates = DMFMA16x16WriteVgprVALUWriteWaitStates; + break; + default: + llvm_unreachable("unexpected number of cycles for dgemm"); + } + } else if (ST.hasGFX940Insts()) { NeedWaitStates = - ST.hasGFX940Insts() - ? isXDL(ST, *MFMA) - ? GFX940_XDL16PassWriteVgprVALUWawWaitStates - : GFX940_SMFMA16PassWriteVgprVALUWawWaitStates - : SMFMA32x32WriteVgprVALUWawWaitStates; - break; + isXDL(ST, *MFMA) + ? GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(NumPasses) + : GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(NumPasses); + } else { + switch (NumPasses) { + case 2: + NeedWaitStates = SMFMA4x4WriteVgprVALUWawWaitStates; + break; + case 8: + NeedWaitStates = SMFMA16x16WriteVgprVALUWawWaitStates; + break; + case 16: + NeedWaitStates = SMFMA32x32WriteVgprVALUWawWaitStates; + break; + default: + llvm_unreachable("Unexpected number of passes for mfma"); + } } int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; -- GitLab From 812c22b2ef5f3f194b8d452fc9f95714dce572f2 Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Thu, 7 Mar 2024 09:28:20 +0000 Subject: [PATCH 416/929] [RemoveDIs][wasm] Apply current debug mode to new function protos (#84292) This trips the verifier changes added in #83251 Stimulated by llvm/test/MC/WebAssembly/extern-functype-intrinsic.ll --- llvm/lib/Target/WebAssembly/WebAssemblyAddMissingPrototypes.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAddMissingPrototypes.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAddMissingPrototypes.cpp index 90e819912847..abcb1d0f1628 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAddMissingPrototypes.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAddMissingPrototypes.cpp @@ -136,6 +136,7 @@ bool WebAssemblyAddMissingPrototypes::runOnModule(Module &M) { Function::Create(NewType, F.getLinkage(), F.getName() + ".fixed_sig"); NewF->setAttributes(F.getAttributes()); NewF->removeFnAttr("no-prototype"); + NewF->IsNewDbgInfoFormat = F.IsNewDbgInfoFormat; Replacements.emplace_back(&F, NewF); } -- GitLab From 503c55e17037436dcd45ac69dea8967e67e3f5e8 Mon Sep 17 00:00:00 2001 From: ostannard Date: Thu, 7 Mar 2024 09:28:49 +0000 Subject: [PATCH 417/929] [AArch64] Move SLS later in pass pipeline (#84210) Currently, the SLS hardening pass is run before the machine outliner, which means that the outliner creates new functions and calls which do not have the SLS hardening applied. The fix for this is to move the SLS passes to after the outliner, as has recently been done for the return address signing pass. This also avoids a bug where the SLS outliner emits code with instructions after a return, which the outliner doesn't correctly handle. --- .../Target/AArch64/AArch64SLSHardening.cpp | 15 +++++++++++- .../Target/AArch64/AArch64TargetMachine.cpp | 5 ++-- llvm/test/CodeGen/AArch64/O0-pipeline.ll | 4 ++-- llvm/test/CodeGen/AArch64/O3-pipeline.ll | 4 ++-- .../AArch64/arm64-opt-remarks-lazy-bfi.ll | 24 +++++++++++++++---- llvm/test/CodeGen/AArch64/sls-crash.ll | 6 +++++ .../AArch64/sls-stackprotector-outliner.ll | 12 ++++++---- 7 files changed, 54 insertions(+), 16 deletions(-) create mode 100644 llvm/test/CodeGen/AArch64/sls-crash.ll diff --git a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp index ce3bc0b18375..41bbc003fd9b 100644 --- a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp +++ b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp @@ -220,7 +220,20 @@ void SLSBLRThunkInserter::populateThunk(MachineFunction &MF) { const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); - assert (MF.size() == 1); + + // Depending on whether this pass is in the same FunctionPassManager as the + // IR->MIR conversion, the thunk may be completely empty, or contain a single + // basic block with a single return instruction. Normalise it to contain a + // single empty basic block. + if (MF.size() == 1) { + assert(MF.front().size() == 1); + assert(MF.front().front().getOpcode() == AArch64::RET); + MF.front().erase(MF.front().begin()); + } else { + assert(MF.size() == 0); + MF.push_back(MF.CreateMachineBasicBlock()); + } + MachineBasicBlock *Entry = &MF.front(); Entry->clear(); diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp index 64c4ecd1fd6d..e5e60459e814 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp @@ -820,9 +820,6 @@ void AArch64PassConfig::addPreSched2() { // info. addPass(createAArch64SpeculationHardeningPass()); - addPass(createAArch64IndirectThunks()); - addPass(createAArch64SLSHardeningPass()); - if (TM->getOptLevel() != CodeGenOptLevel::None) { if (EnableFalkorHWPFFix) addPass(createFalkorHWPFFixPass()); @@ -855,6 +852,8 @@ void AArch64PassConfig::addPreEmitPass() { } void AArch64PassConfig::addPostBBSections() { + addPass(createAArch64IndirectThunks()); + addPass(createAArch64SLSHardeningPass()); addPass(createAArch64PointerAuthPass()); if (EnableBranchTargets) addPass(createAArch64BranchTargetsPass()); diff --git a/llvm/test/CodeGen/AArch64/O0-pipeline.ll b/llvm/test/CodeGen/AArch64/O0-pipeline.ll index 4f87bb2a3ee8..d1e38b85fa9c 100644 --- a/llvm/test/CodeGen/AArch64/O0-pipeline.ll +++ b/llvm/test/CodeGen/AArch64/O0-pipeline.ll @@ -64,8 +64,6 @@ ; CHECK-NEXT: AArch64 pseudo instruction expansion pass ; CHECK-NEXT: Insert KCFI indirect call checks ; CHECK-NEXT: AArch64 speculation hardening pass -; CHECK-NEXT: AArch64 Indirect Thunks -; CHECK-NEXT: AArch64 sls hardening pass ; CHECK-NEXT: Analyze Machine Code For Garbage Collection ; CHECK-NEXT: Insert fentry calls ; CHECK-NEXT: Insert XRay ops @@ -75,6 +73,8 @@ ; CHECK-NEXT: StackMap Liveness Analysis ; CHECK-NEXT: Live DEBUG_VALUE analysis ; CHECK-NEXT: Machine Sanitizer Binary Metadata +; CHECK-NEXT: AArch64 Indirect Thunks +; CHECK-NEXT: AArch64 sls hardening pass ; CHECK-NEXT: AArch64 Pointer Authentication ; CHECK-NEXT: AArch64 Branch Targets ; CHECK-NEXT: Branch relaxation pass diff --git a/llvm/test/CodeGen/AArch64/O3-pipeline.ll b/llvm/test/CodeGen/AArch64/O3-pipeline.ll index ae0dbed09979..eee9a27c90c1 100644 --- a/llvm/test/CodeGen/AArch64/O3-pipeline.ll +++ b/llvm/test/CodeGen/AArch64/O3-pipeline.ll @@ -205,8 +205,6 @@ ; CHECK-NEXT: AArch64 load / store optimization pass ; CHECK-NEXT: Insert KCFI indirect call checks ; CHECK-NEXT: AArch64 speculation hardening pass -; CHECK-NEXT: AArch64 Indirect Thunks -; CHECK-NEXT: AArch64 sls hardening pass ; CHECK-NEXT: MachineDominator Tree Construction ; CHECK-NEXT: Machine Natural Loop Construction ; CHECK-NEXT: Falkor HW Prefetch Fix Late Phase @@ -227,6 +225,8 @@ ; CHECK-NEXT: Machine Sanitizer Binary Metadata ; CHECK-NEXT: Machine Outliner ; CHECK-NEXT: FunctionPass Manager +; CHECK-NEXT: AArch64 Indirect Thunks +; CHECK-NEXT: AArch64 sls hardening pass ; CHECK-NEXT: AArch64 Pointer Authentication ; CHECK-NEXT: AArch64 Branch Targets ; CHECK-NEXT: Branch relaxation pass diff --git a/llvm/test/CodeGen/AArch64/arm64-opt-remarks-lazy-bfi.ll b/llvm/test/CodeGen/AArch64/arm64-opt-remarks-lazy-bfi.ll index 580886520789..3ffaf962425b 100644 --- a/llvm/test/CodeGen/AArch64/arm64-opt-remarks-lazy-bfi.ll +++ b/llvm/test/CodeGen/AArch64/arm64-opt-remarks-lazy-bfi.ll @@ -32,8 +32,16 @@ ; HOTNESS: Freeing Pass 'Machine Outliner' ; HOTNESS-NEXT: Executing Pass 'Function Pass Manager' -; HOTNESS-NEXT: Executing Pass 'Verify generated machine code' -; HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' +; HOTNESS-NEXT: Executing Pass 'Verify generated machine code' on Function 'empty_func'... +; HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' on Function 'empty_func'... +; HOTNESS-NEXT: Executing Pass 'AArch64 Indirect Thunks' on Function 'empty_func'... +; HOTNESS-NEXT: Freeing Pass 'AArch64 Indirect Thunks' on Function 'empty_func'... +; HOTNESS-NEXT: Executing Pass 'Verify generated machine code' on Function 'empty_func'... +; HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' on Function 'empty_func'... +; HOTNESS-NEXT: Executing Pass 'AArch64 sls hardening pass' on Function 'empty_func'... +; HOTNESS-NEXT: Freeing Pass 'AArch64 sls hardening pass' on Function 'empty_func'... +; HOTNESS-NEXT: Executing Pass 'Verify generated machine code' on Function 'empty_func'... +; HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' on Function 'empty_func'... ; HOTNESS-NEXT: Executing Pass 'AArch64 Pointer Authentication' on Function 'empty_func'... ; HOTNESS-NEXT: Freeing Pass 'AArch64 Pointer Authentication' on Function 'empty_func'... ; HOTNESS-NEXT: Executing Pass 'Verify generated machine code' on Function 'empty_func'... @@ -73,8 +81,16 @@ ; NO_HOTNESS: Freeing Pass 'Machine Outliner' ; NO_HOTNESS-NEXT: Executing Pass 'Function Pass Manager' -; NO_HOTNESS-NEXT: Executing Pass 'Verify generated machine code' -; NO_HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' +; NO_HOTNESS-NEXT: Executing Pass 'Verify generated machine code' on Function 'empty_func'... +; NO_HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' on Function 'empty_func'... +; NO_HOTNESS-NEXT: Executing Pass 'AArch64 Indirect Thunks' on Function 'empty_func'... +; NO_HOTNESS-NEXT: Freeing Pass 'AArch64 Indirect Thunks' on Function 'empty_func'... +; NO_HOTNESS-NEXT: Executing Pass 'Verify generated machine code' on Function 'empty_func'... +; NO_HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' on Function 'empty_func'... +; NO_HOTNESS-NEXT: Executing Pass 'AArch64 sls hardening pass' on Function 'empty_func'... +; NO_HOTNESS-NEXT: Freeing Pass 'AArch64 sls hardening pass' on Function 'empty_func'... +; NO_HOTNESS-NEXT: Executing Pass 'Verify generated machine code' on Function 'empty_func'... +; NO_HOTNESS-NEXT: Freeing Pass 'Verify generated machine code' on Function 'empty_func'... ; NO_HOTNESS-NEXT: Executing Pass 'AArch64 Pointer Authentication' on Function 'empty_func'... ; NO_HOTNESS-NEXT: Freeing Pass 'AArch64 Pointer Authentication' on Function 'empty_func'... ; NO_HOTNESS-NEXT: Executing Pass 'Verify generated machine code' on Function 'empty_func'... diff --git a/llvm/test/CodeGen/AArch64/sls-crash.ll b/llvm/test/CodeGen/AArch64/sls-crash.ll new file mode 100644 index 000000000000..5dfc3c7824a8 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sls-crash.ll @@ -0,0 +1,6 @@ +; RUN: llc -mtriple aarch64 -O0 < %s + +define hidden void @foo() "target-features"="+harden-sls-blr" { +entry: + ret void +} diff --git a/llvm/test/CodeGen/AArch64/sls-stackprotector-outliner.ll b/llvm/test/CodeGen/AArch64/sls-stackprotector-outliner.ll index 5f3b1503b46b..b281204a66e4 100644 --- a/llvm/test/CodeGen/AArch64/sls-stackprotector-outliner.ll +++ b/llvm/test/CodeGen/AArch64/sls-stackprotector-outliner.ll @@ -18,7 +18,8 @@ define hidden void @_ZTv0_n24_N2C6D1Ev(ptr %this) minsize sspreq "target-feature ; CHECK-NEXT: b.ne .LBB0_2 ; CHECK-NEXT: // %bb.1: // %entry ; CHECK-NEXT: ldr x30, [sp, #16] // 8-byte Folded Reload -; CHECK-NEXT: bl OUTLINED_FUNCTION_1 +; CHECK-NEXT: add x0, x0, x8 +; CHECK-NEXT: add sp, sp, #32 ; CHECK-NEXT: b _ZN2C6D1Ev ; CHECK-NEXT: dsb sy ; CHECK-NEXT: isb @@ -45,7 +46,8 @@ define hidden void @_ZTv0_n24_N2C6D0Ev(ptr %this) minsize sspreq "target-feature ; CHECK-NEXT: b.ne .LBB1_2 ; CHECK-NEXT: // %bb.1: // %entry ; CHECK-NEXT: ldr x30, [sp, #16] // 8-byte Folded Reload -; CHECK-NEXT: bl OUTLINED_FUNCTION_1 +; CHECK-NEXT: add x0, x0, x8 +; CHECK-NEXT: add sp, sp, #32 ; CHECK-NEXT: b _ZN2C6D0Ev ; CHECK-NEXT: dsb sy ; CHECK-NEXT: isb @@ -71,7 +73,8 @@ define hidden void @_ZTv0_n24_N3C10D1Ev(ptr %this) minsize sspreq "target-featur ; CHECK-NEXT: b.ne .LBB2_2 ; CHECK-NEXT: // %bb.1: // %entry ; CHECK-NEXT: ldr x30, [sp, #16] // 8-byte Folded Reload -; CHECK-NEXT: bl OUTLINED_FUNCTION_1 +; CHECK-NEXT: add x0, x0, x8 +; CHECK-NEXT: add sp, sp, #32 ; CHECK-NEXT: b _ZN3C10D1Ev ; CHECK-NEXT: dsb sy ; CHECK-NEXT: isb @@ -97,7 +100,8 @@ define hidden void @_ZTv0_n24_N3C10D0Ev(ptr %this) minsize sspreq "target-featur ; CHECK-NEXT: b.ne .LBB3_2 ; CHECK-NEXT: // %bb.1: // %entry ; CHECK-NEXT: ldr x30, [sp, #16] // 8-byte Folded Reload -; CHECK-NEXT: bl OUTLINED_FUNCTION_1 +; CHECK-NEXT: add x0, x0, x8 +; CHECK-NEXT: add sp, sp, #32 ; CHECK-NEXT: b _ZN3C10D0Ev ; CHECK-NEXT: dsb sy ; CHECK-NEXT: isb -- GitLab From 6e79f77adbbd338848ea770f2f2b110bc57a3990 Mon Sep 17 00:00:00 2001 From: Vincent Lee Date: Thu, 7 Mar 2024 01:46:36 -0800 Subject: [PATCH 418/929] [dataflow][nfc] Fix u8 string usage with c++20 (#84291) Clang returns an error when compiling this file with c++20 ``` error: ISO C++20 does not permit initialization of char array with UTF-8 string literal ``` It seems like c++20 treats u8strings differently than strings (probably needs char8_t). Make this a string to fix the error. --- clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp b/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp index ff4e18de2c70..d9f40d28859f 100644 --- a/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp +++ b/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp @@ -500,7 +500,7 @@ private: for (unsigned I = 0; I < CFG.getNumBlockIDs(); ++I) { std::string Name = blockID(I); // Rightwards arrow, vertical line - char ConvergenceMarker[] = u8"\\n\u2192\u007c"; + char ConvergenceMarker[] = "\\n\u2192\u007c"; if (BlockConverged[I]) Name += ConvergenceMarker; GraphS << " " << blockID(I) << " [id=" << blockID(I) << " label=\"" -- GitLab From 84f483dbeeba5ecadbf3e4a75bfb71525a3fa332 Mon Sep 17 00:00:00 2001 From: Guillaume Chatelet Date: Thu, 7 Mar 2024 11:01:09 +0100 Subject: [PATCH 419/929] [libc] Remove UB specializations of type traits for `BigInt` (#84035) The standard specifies that it it UB to specialize the following traits: - `std::is_integral` - `std::is_unsigned` - `std::make_unsigned` - `std::make_signed` This patch: - Removes specializations for `BigInt` - Transforms SFINAE for `bit.h` functions from template parameter to return type (This makes specialization easier). - Adds `BigInt` specialization for `bit.h` functions. - Fixes code depending on previous specializations. --- libc/src/__support/CMakeLists.txt | 1 + libc/src/__support/CPP/bit.h | 112 ++++++---- libc/src/__support/UInt.h | 192 ++++++++++++------ libc/src/__support/float_to_string.h | 2 +- libc/src/__support/integer_to_string.h | 19 +- libc/test/UnitTest/CMakeLists.txt | 1 + libc/test/UnitTest/LibcTest.cpp | 10 +- libc/test/UnitTest/LibcTest.h | 1 + libc/test/UnitTest/TestLogger.cpp | 8 +- libc/test/src/__support/CPP/bit_test.cpp | 49 +++-- .../llvm-project-overlay/libc/BUILD.bazel | 1 + .../libc/test/UnitTest/BUILD.bazel | 1 + 12 files changed, 272 insertions(+), 125 deletions(-) diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt index 1a4b3e9a2145..17c04aa57e6f 100644 --- a/libc/src/__support/CMakeLists.txt +++ b/libc/src/__support/CMakeLists.txt @@ -95,6 +95,7 @@ add_header_library( HDRS integer_to_string.h DEPENDS + .uint libc.src.__support.common libc.src.__support.CPP.algorithm libc.src.__support.CPP.limits diff --git a/libc/src/__support/CPP/bit.h b/libc/src/__support/CPP/bit.h index 7d11e7d5c497..bc2f595845a9 100644 --- a/libc/src/__support/CPP/bit.h +++ b/libc/src/__support/CPP/bit.h @@ -27,13 +27,14 @@ namespace LIBC_NAMESPACE::cpp { // This implementation of bit_cast requires trivially-constructible To, to avoid // UB in the implementation. -template < - typename To, typename From, - typename = cpp::enable_if_t::value && - cpp::is_trivially_copyable::value && - cpp::is_trivially_copyable::value>> -LIBC_INLINE constexpr To bit_cast(const From &from) { +template +LIBC_INLINE constexpr cpp::enable_if_t< + (sizeof(To) == sizeof(From)) && + cpp::is_trivially_constructible::value && + cpp::is_trivially_copyable::value && + cpp::is_trivially_copyable::value, + To> +bit_cast(const From &from) { MSAN_UNPOISON(&from, sizeof(From)); #if LIBC_HAS_BUILTIN(__builtin_bit_cast) return __builtin_bit_cast(To, from); @@ -51,8 +52,10 @@ LIBC_INLINE constexpr To bit_cast(const From &from) { #endif // LIBC_HAS_BUILTIN(__builtin_bit_cast) } -template >> -[[nodiscard]] LIBC_INLINE constexpr bool has_single_bit(T value) { +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, + bool> +has_single_bit(T value) { return (value != 0) && ((value & (value - 1)) == 0); } @@ -70,8 +73,9 @@ template >> /// Only unsigned integral types are allowed. /// /// Returns cpp::numeric_limits::digits on an input of 0. -template >> -[[nodiscard]] LIBC_INLINE constexpr int countr_zero(T value) { +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +countr_zero(T value) { if (!value) return cpp::numeric_limits::digits; if (value & 0x1) @@ -103,8 +107,9 @@ ADD_SPECIALIZATION(countr_zero, unsigned long long, __builtin_ctzll) /// Only unsigned integral types are allowed. /// /// Returns cpp::numeric_limits::digits on an input of 0. -template >> -[[nodiscard]] LIBC_INLINE constexpr int countl_zero(T value) { +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +countl_zero(T value) { if (!value) return cpp::numeric_limits::digits; // Bisection method. @@ -135,8 +140,9 @@ ADD_SPECIALIZATION(countl_zero, unsigned long long, __builtin_clzll) /// Only unsigned integral types are allowed. /// /// Returns cpp::numeric_limits::digits on an input of all ones. -template >> -[[nodiscard]] LIBC_INLINE constexpr int countl_one(T value) { +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +countl_one(T value) { return cpp::countl_zero(~value); } @@ -147,8 +153,9 @@ template >> /// Only unsigned integral types are allowed. /// /// Returns cpp::numeric_limits::digits on an input of all ones. -template >> -[[nodiscard]] LIBC_INLINE constexpr int countr_one(T value) { +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +countr_one(T value) { return cpp::countr_zero(~value); } @@ -156,8 +163,9 @@ template >> /// Returns 0 otherwise. /// /// Ex. bit_width(5) == 3. -template >> -[[nodiscard]] LIBC_INLINE constexpr int bit_width(T value) { +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +bit_width(T value) { return cpp::numeric_limits::digits - cpp::countl_zero(value); } @@ -165,8 +173,9 @@ template >> /// nonzero. Returns 0 otherwise. /// /// Ex. bit_floor(5) == 4. -template >> -[[nodiscard]] LIBC_INLINE constexpr T bit_floor(T value) { +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> +bit_floor(T value) { if (!value) return 0; return T(1) << (cpp::bit_width(value) - 1); @@ -179,8 +188,9 @@ template >> /// /// The return value is undefined if the input is larger than the largest power /// of two representable in T. -template >> -[[nodiscard]] LIBC_INLINE constexpr T bit_ceil(T value) { +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> +bit_ceil(T value) { if (value < 2) return 1; return T(1) << cpp::bit_width(value - 1u); @@ -190,28 +200,31 @@ template >> // from https://blog.regehr.org/archives/1063. // Forward-declare rotr so that rotl can use it. -template >> -[[nodiscard]] LIBC_INLINE constexpr T rotr(T value, int rotate); +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> +rotr(T value, int rotate); -template >> -[[nodiscard]] LIBC_INLINE constexpr T rotl(T value, int rotate) { +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> +rotl(T value, int rotate) { constexpr unsigned N = cpp::numeric_limits::digits; rotate = rotate % N; if (!rotate) return value; if (rotate < 0) - return cpp::rotr(value, -rotate); + return cpp::rotr(value, -rotate); return (value << rotate) | (value >> (N - rotate)); } -template -[[nodiscard]] LIBC_INLINE constexpr T rotr(T value, int rotate) { +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> +rotr(T value, int rotate) { constexpr unsigned N = cpp::numeric_limits::digits; rotate = rotate % N; if (!rotate) return value; if (rotate < 0) - return cpp::rotl(value, -rotate); + return cpp::rotl(value, -rotate); return (value >> rotate) | (value << (N - rotate)); } @@ -226,33 +239,44 @@ LIBC_INLINE constexpr To bit_or_static_cast(const From &from) { } } -template >> -[[nodiscard]] LIBC_INLINE constexpr int first_leading_zero(T value) { +// TODO: remove from 'bit.h' as it is not a standard function. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +first_leading_zero(T value) { return value == cpp::numeric_limits::max() ? 0 : countl_one(value) + 1; } -template >> -[[nodiscard]] LIBC_INLINE constexpr int first_leading_one(T value) { +// TODO: remove from 'bit.h' as it is not a standard function. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +first_leading_one(T value) { return first_leading_zero(static_cast(~value)); } -template >> -[[nodiscard]] LIBC_INLINE constexpr int first_trailing_zero(T value) { +// TODO: remove from 'bit.h' as it is not a standard function. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +first_trailing_zero(T value) { return value == cpp::numeric_limits::max() ? 0 : countr_zero(static_cast(~value)) + 1; } -template >> -[[nodiscard]] LIBC_INLINE constexpr int first_trailing_one(T value) { +// TODO: remove from 'bit.h' as it is not a standard function. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +first_trailing_one(T value) { return value == cpp::numeric_limits::max() ? 0 : countr_zero(value) + 1; } /// Count number of 1's aka population count or hamming weight. /// /// Only unsigned integral types are allowed. -template >> -[[nodiscard]] LIBC_INLINE constexpr int count_ones(T value) { +// TODO: rename as 'popcount' to follow the standard +// https://en.cppreference.com/w/cpp/numeric/popcount +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +count_ones(T value) { int count = 0; for (int i = 0; i != cpp::numeric_limits::digits; ++i) if ((value >> i) & 0x1) @@ -272,8 +296,10 @@ ADD_SPECIALIZATION(unsigned long long, __builtin_popcountll) // TODO: 128b specializations? #undef ADD_SPECIALIZATION -template >> -[[nodiscard]] LIBC_INLINE constexpr int count_zeros(T value) { +// TODO: remove from 'bit.h' as it is not a standard function. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +count_zeros(T value) { return count_ones(static_cast(~value)); } diff --git a/libc/src/__support/UInt.h b/libc/src/__support/UInt.h index 5973e6fab1d7..b3d8f00b9a01 100644 --- a/libc/src/__support/UInt.h +++ b/libc/src/__support/UInt.h @@ -43,6 +43,9 @@ struct BigInt { static_assert(is_integral_v && is_unsigned_v, "WordType must be unsigned integer."); + using word_type = WordType; + LIBC_INLINE_VAR static constexpr bool SIGNED = Signed; + LIBC_INLINE_VAR static constexpr size_t BITS = Bits; LIBC_INLINE_VAR static constexpr size_t WORD_SIZE = sizeof(WordType) * CHAR_BIT; @@ -50,6 +53,10 @@ struct BigInt { "Number of bits in BigInt should be a multiple of WORD_SIZE."); LIBC_INLINE_VAR static constexpr size_t WORD_COUNT = Bits / WORD_SIZE; + + using unsigned_type = BigInt; + using signed_type = BigInt; + cpp::array val{}; LIBC_INLINE constexpr BigInt() = default; @@ -579,19 +586,33 @@ struct BigInt { return *this; } - LIBC_INLINE constexpr uint64_t clz() { - uint64_t leading_zeroes = 0; - for (size_t i = WORD_COUNT; i > 0; --i) { - if (val[i - 1] == 0) { - leading_zeroes += WORD_SIZE; - } else { - leading_zeroes += countl_zero(val[i - 1]); + // TODO: remove and use cpp::countl_zero below. + [[nodiscard]] LIBC_INLINE constexpr int clz() const { + constexpr int word_digits = cpp::numeric_limits::digits; + int leading_zeroes = 0; + for (auto i = val.size(); i > 0;) { + --i; + const int zeroes = countl_zero(val[i]); + leading_zeroes += zeroes; + if (zeroes != word_digits) break; - } } return leading_zeroes; } + // TODO: remove and use cpp::countr_zero below. + [[nodiscard]] LIBC_INLINE constexpr int ctz() const { + constexpr int word_digits = cpp::numeric_limits::digits; + int trailing_zeroes = 0; + for (auto word : val) { + const int zeroes = countr_zero(word); + trailing_zeroes += zeroes; + if (zeroes != word_digits) + break; + } + return trailing_zeroes; + } + LIBC_INLINE constexpr void shift_left(size_t s) { if constexpr (Bits == WORD_SIZE) { // Use native types if possible. @@ -916,66 +937,123 @@ public: LIBC_INLINE_VAR static constexpr int digits = 128; }; -// Provides is_integral of U/Int<128>, U/Int<192>, U/Int<256>. -template -struct is_integral> : cpp::true_type {}; +// type traits to determine whether a T is a cpp::BigInt. +template struct is_big_int : cpp::false_type {}; -// Provides is_unsigned of UInt<128>, UInt<192>, UInt<256>. template -struct is_unsigned> : cpp::bool_constant {}; - -template -struct make_unsigned> - : type_identity> {}; - -template -struct make_signed> - : type_identity> {}; - -namespace internal { -template struct is_custom_uint : cpp::false_type {}; - -template -struct is_custom_uint> : cpp::true_type {}; -} // namespace internal - -// bit_cast to UInt -// Note: The standard scheme for SFINAE selection is to have exactly one -// function instanciation valid at a time. This is usually done by having a -// predicate in one function and the negated predicate in the other one. -// e.g. -// template::value == true> ... -// template::value == false> ... -// -// Unfortunately this would make the default 'cpp::bit_cast' aware of -// 'is_custom_uint' (or any other customization). To prevent exposing all -// customizations in the original function, we create a different function with -// four 'typename's instead of three - otherwise it would be considered as a -// redeclaration of the same function leading to "error: template parameter -// redefines default argument". -template ::value && - cpp::is_trivially_copyable::value>, - typename = cpp::enable_if_t::value>> -LIBC_INLINE constexpr To bit_cast(const From &from) { +struct is_big_int> : cpp::true_type {}; + +template +LIBC_INLINE_VAR constexpr bool is_big_int_v = is_big_int::value; + +// Specialization of cpp::bit_cast ('bit.h') from T to BigInt. +template +LIBC_INLINE constexpr cpp::enable_if_t< + (sizeof(To) == sizeof(From)) && cpp::is_trivially_copyable::value && + cpp::is_trivially_copyable::value && is_big_int::value, + To> +bit_cast(const From &from) { To out; using Storage = decltype(out.val); out.val = cpp::bit_cast(from); return out; } -// bit_cast from UInt -template < - typename To, size_t Bits, - typename = cpp::enable_if_t) && - cpp::is_trivially_constructible::value && - cpp::is_trivially_copyable::value && - cpp::is_trivially_copyable>::value>> -LIBC_INLINE constexpr To bit_cast(const UInt &from) { +// Specialization of cpp::bit_cast ('bit.h') from BigInt to T. +template +LIBC_INLINE constexpr cpp::enable_if_t< + sizeof(To) == sizeof(UInt) && + cpp::is_trivially_constructible::value && + cpp::is_trivially_copyable::value && + cpp::is_trivially_copyable>::value, + To> +bit_cast(const UInt &from) { return cpp::bit_cast(from.val); } +// Specialization of cpp::has_single_bit ('bit.h') for BigInt. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, bool> +has_single_bit(T value) { + int bits = 0; + for (auto word : value.val) { + if (word == 0) + continue; + bits += count_ones(word); + if (bits > 1) + return false; + } + return bits == 1; +} + +// Specialization of cpp::countr_zero ('bit.h') for BigInt. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +countr_zero(const T &value) { + return value.ctz(); +} + +// Specialization of cpp::countl_zero ('bit.h') for BigInt. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +countl_zero(const T &value) { + return value.clz(); +} + +// Specialization of cpp::countl_one ('bit.h') for BigInt. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +countl_one(T value) { + // TODO : Implement a faster version not involving operator~. + return cpp::countl_zero(~value); +} + +// Specialization of cpp::countr_one ('bit.h') for BigInt. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +countr_one(T value) { + // TODO : Implement a faster version not involving operator~. + return cpp::countr_zero(~value); +} + +// Specialization of cpp::bit_width ('bit.h') for BigInt. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +bit_width(T value) { + return cpp::numeric_limits::digits - cpp::countl_zero(value); +} + +// Forward-declare rotr so that rotl can use it. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> +rotr(T value, int rotate); + +// Specialization of cpp::rotl ('bit.h') for BigInt. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> +rotl(T value, int rotate) { + constexpr unsigned N = cpp::numeric_limits::digits; + rotate = rotate % N; + if (!rotate) + return value; + if (rotate < 0) + return cpp::rotr(value, -rotate); + return (value << rotate) | (value >> (N - rotate)); +} + +// Specialization of cpp::rotr ('bit.h') for BigInt. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> +rotr(T value, int rotate) { + constexpr unsigned N = cpp::numeric_limits::digits; + rotate = rotate % N; + if (!rotate) + return value; + if (rotate < 0) + return cpp::rotl(value, -rotate); + return (value >> rotate) | (value << (N - rotate)); +} + } // namespace LIBC_NAMESPACE::cpp #endif // LLVM_LIBC_SRC___SUPPORT_UINT_H diff --git a/libc/src/__support/float_to_string.h b/libc/src/__support/float_to_string.h index 744842ced8d7..27476433a945 100644 --- a/libc/src/__support/float_to_string.h +++ b/libc/src/__support/float_to_string.h @@ -713,7 +713,7 @@ template <> class FloatToString { float_as_fixed.shift_left(SHIFT_AMOUNT); // If there are still digits above the decimal point, handle those. - if (float_as_fixed.clz() < EXTRA_INT_WIDTH) { + if (float_as_fixed.clz() < static_cast(EXTRA_INT_WIDTH)) { cpp::UInt above_decimal_point = float_as_fixed >> FLOAT_AS_INT_WIDTH; diff --git a/libc/src/__support/integer_to_string.h b/libc/src/__support/integer_to_string.h index 81ed21ccfca1..a5872dce6520 100644 --- a/libc/src/__support/integer_to_string.h +++ b/libc/src/__support/integer_to_string.h @@ -67,6 +67,7 @@ #include "src/__support/CPP/span.h" #include "src/__support/CPP/string_view.h" #include "src/__support/CPP/type_traits.h" +#include "src/__support/UInt.h" // is_big_int #include "src/__support/common.h" namespace LIBC_NAMESPACE { @@ -149,6 +150,18 @@ public: using StringBufferWriter = StringBufferWriterImpl; using BackwardStringBufferWriter = StringBufferWriterImpl; +template struct IntegerWriterUnsigned {}; + +template +struct IntegerWriterUnsigned>> { + using type = cpp::make_unsigned_t; +}; + +template +struct IntegerWriterUnsigned>> { + using type = typename T::unsigned_type; +}; + } // namespace details namespace radix { @@ -163,7 +176,7 @@ template using Custom = details::Fmt; // See file header for documentation. template class IntegerToString { - static_assert(cpp::is_integral_v); + static_assert(cpp::is_integral_v || cpp::is_big_int_v); LIBC_INLINE static constexpr size_t compute_buffer_size() { constexpr auto MAX_DIGITS = []() -> size_t { @@ -208,8 +221,8 @@ template class IntegerToString { // An internal stateless structure that handles the number formatting logic. struct IntegerWriter { - static_assert(cpp::is_integral_v); - using UNSIGNED_T = cpp::make_unsigned_t; + static_assert(cpp::is_integral_v || cpp::is_big_int_v); + using UNSIGNED_T = typename details::IntegerWriterUnsigned::type; LIBC_INLINE static char digit_char(uint8_t digit) { if (digit < 10) diff --git a/libc/test/UnitTest/CMakeLists.txt b/libc/test/UnitTest/CMakeLists.txt index 4668f0061975..36837c553efc 100644 --- a/libc/test/UnitTest/CMakeLists.txt +++ b/libc/test/UnitTest/CMakeLists.txt @@ -74,6 +74,7 @@ add_unittest_framework_library( libc.src.__support.CPP.type_traits libc.src.__support.fixed_point.fx_rep libc.src.__support.OSUtil.osutil + libc.src.__support.uint libc.src.__support.uint128 ) diff --git a/libc/test/UnitTest/LibcTest.cpp b/libc/test/UnitTest/LibcTest.cpp index 7b0e4fca8368..0340f7ed3710 100644 --- a/libc/test/UnitTest/LibcTest.cpp +++ b/libc/test/UnitTest/LibcTest.cpp @@ -38,7 +38,8 @@ TestLogger &operator<<(TestLogger &logger, Location Loc) { // When the value is UInt128, __uint128_t or wider, show its hexadecimal // digits. template -cpp::enable_if_t && (sizeof(T) > sizeof(uint64_t)), +cpp::enable_if_t<(cpp::is_integral_v && (sizeof(T) > sizeof(uint64_t))) || + cpp::is_big_int_v, cpp::string> describeValue(T Value) { static_assert(sizeof(T) % 8 == 0, "Unsupported size of UInt"); @@ -47,11 +48,10 @@ describeValue(T Value) { } // When the value is of a standard integral type, just display it as normal. -template -cpp::enable_if_t && - sizeof(ValType) <= sizeof(uint64_t), +template +cpp::enable_if_t && (sizeof(T) <= sizeof(uint64_t)), cpp::string> -describeValue(ValType Value) { +describeValue(T Value) { return cpp::to_string(Value); } diff --git a/libc/test/UnitTest/LibcTest.h b/libc/test/UnitTest/LibcTest.h index 639f60058325..d26d6490bcb5 100644 --- a/libc/test/UnitTest/LibcTest.h +++ b/libc/test/UnitTest/LibcTest.h @@ -127,6 +127,7 @@ protected: // of type promotion. template || + cpp::is_big_int_v || cpp::is_fixed_point_v, int> = 0> bool test(TestCond Cond, ValType LHS, ValType RHS, const char *LHSStr, diff --git a/libc/test/UnitTest/TestLogger.cpp b/libc/test/UnitTest/TestLogger.cpp index 6bb0e17dc388..469b3a11d57d 100644 --- a/libc/test/UnitTest/TestLogger.cpp +++ b/libc/test/UnitTest/TestLogger.cpp @@ -2,6 +2,7 @@ #include "src/__support/CPP/string.h" #include "src/__support/CPP/string_view.h" #include "src/__support/OSUtil/io.h" // write_to_stderr +#include "src/__support/UInt.h" // is_big_int #include "src/__support/UInt128.h" #include @@ -47,8 +48,9 @@ template <> TestLogger &TestLogger::operator<<(void *addr) { } template TestLogger &TestLogger::operator<<(T t) { - if constexpr (cpp::is_integral_v && cpp::is_unsigned_v && - sizeof(T) > sizeof(uint64_t)) { + if constexpr (cpp::is_big_int_v || + (cpp::is_integral_v && cpp::is_unsigned_v && + (sizeof(T) > sizeof(uint64_t)))) { static_assert(sizeof(T) % 8 == 0, "Unsupported size of UInt"); const IntegerToString buffer(t); return *this << buffer.view(); @@ -68,7 +70,7 @@ template TestLogger &TestLogger::operator<< (unsigned short); template TestLogger &TestLogger::operator<< (unsigned int); template TestLogger &TestLogger::operator<< (unsigned long); template TestLogger & -TestLogger::operator<< (unsigned long long); + TestLogger::operator<< (unsigned long long); #ifdef __SIZEOF_INT128__ template TestLogger &TestLogger::operator<< <__uint128_t>(__uint128_t); diff --git a/libc/test/src/__support/CPP/bit_test.cpp b/libc/test/src/__support/CPP/bit_test.cpp index 115a5d505c4b..1e3d895e6453 100644 --- a/libc/test/src/__support/CPP/bit_test.cpp +++ b/libc/test/src/__support/CPP/bit_test.cpp @@ -12,21 +12,44 @@ #include +#include + namespace LIBC_NAMESPACE::cpp { -using UnsignedTypes = - testing::TypeList; + +using UnsignedTypes = testing::TypeList< #if defined(__SIZEOF_INT128__) - __uint128_t, + __uint128_t, #endif - cpp::UInt<128>>; + unsigned char, unsigned short, unsigned int, unsigned long, + unsigned long long, cpp::UInt<128>>; TYPED_TEST(LlvmLibcBitTest, HasSingleBit, UnsignedTypes) { - EXPECT_FALSE(has_single_bit(T(0))); - EXPECT_FALSE(has_single_bit(~T(0))); + constexpr auto ZERO = T(0); + constexpr auto ALL_ONES = T(~ZERO); + EXPECT_FALSE(has_single_bit(ZERO)); + EXPECT_FALSE(has_single_bit(ALL_ONES)); + for (T value = 1; value; value <<= 1) EXPECT_TRUE(has_single_bit(value)); + + // We test that if two bits are set has_single_bit returns false. + // We do this by setting the highest or lowest bit depending or where the + // current bit is. This is a bit convoluted but it helps catch a bug on BigInt + // where we have to work on an element-by-element basis. + constexpr auto MIDPOINT = T(ALL_ONES / 2); + constexpr auto LSB = T(1); + constexpr auto MSB = T(~(ALL_ONES >> 1)); + for (T value = 1; value; value <<= 1) { + auto two_bits_value = value | ((value <= MIDPOINT) ? MSB : LSB); + EXPECT_FALSE(has_single_bit(two_bits_value)); + } } TYPED_TEST(LlvmLibcBitTest, CountLZero, UnsignedTypes) { @@ -206,39 +229,39 @@ TEST(LlvmLibcBitTest, Rotr) { rotr(0x12345678deadbeefULL, -19)); } -TYPED_TEST(LlvmLibcBitTest, FirstLeadingZero, UnsignedTypes) { +TYPED_TEST(LlvmLibcBitTest, FirstLeadingZero, UnsignedTypesNoBigInt) { EXPECT_EQ(first_leading_zero(cpp::numeric_limits::max()), 0); for (int i = 0U; i != cpp::numeric_limits::digits; ++i) EXPECT_EQ(first_leading_zero(~(T(1) << i)), cpp::numeric_limits::digits - i); } -TYPED_TEST(LlvmLibcBitTest, FirstLeadingOne, UnsignedTypes) { +TYPED_TEST(LlvmLibcBitTest, FirstLeadingOne, UnsignedTypesNoBigInt) { EXPECT_EQ(first_leading_one(static_cast(0)), 0); for (int i = 0U; i != cpp::numeric_limits::digits; ++i) EXPECT_EQ(first_leading_one(T(1) << i), cpp::numeric_limits::digits - i); } -TYPED_TEST(LlvmLibcBitTest, FirstTrailingZero, UnsignedTypes) { +TYPED_TEST(LlvmLibcBitTest, FirstTrailingZero, UnsignedTypesNoBigInt) { EXPECT_EQ(first_trailing_zero(cpp::numeric_limits::max()), 0); for (int i = 0U; i != cpp::numeric_limits::digits; ++i) EXPECT_EQ(first_trailing_zero(~(T(1) << i)), i + 1); } -TYPED_TEST(LlvmLibcBitTest, FirstTrailingOne, UnsignedTypes) { +TYPED_TEST(LlvmLibcBitTest, FirstTrailingOne, UnsignedTypesNoBigInt) { EXPECT_EQ(first_trailing_one(cpp::numeric_limits::max()), 0); for (int i = 0U; i != cpp::numeric_limits::digits; ++i) EXPECT_EQ(first_trailing_one(T(1) << i), i + 1); } -TYPED_TEST(LlvmLibcBitTest, CountZeros, UnsignedTypes) { +TYPED_TEST(LlvmLibcBitTest, CountZeros, UnsignedTypesNoBigInt) { EXPECT_EQ(count_zeros(T(0)), cpp::numeric_limits::digits); for (int i = 0; i != cpp::numeric_limits::digits; ++i) EXPECT_EQ(count_zeros(cpp::numeric_limits::max() >> i), i); } -TYPED_TEST(LlvmLibcBitTest, CountOnes, UnsignedTypes) { +TYPED_TEST(LlvmLibcBitTest, CountOnes, UnsignedTypesNoBigInt) { EXPECT_EQ(count_ones(T(0)), 0); for (int i = 0; i != cpp::numeric_limits::digits; ++i) EXPECT_EQ(count_ones(cpp::numeric_limits::max() >> i), diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index 49a454379e1c..5c6cf761ebe7 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -507,6 +507,7 @@ libc_support_library( ":__support_cpp_span", ":__support_cpp_string_view", ":__support_cpp_type_traits", + ":__support_uint", ], ) diff --git a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel index a5c18fbb68b3..44692947af7c 100644 --- a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel @@ -18,6 +18,7 @@ libc_support_library( "//libc:__support_cpp_string", "//libc:__support_cpp_string_view", "//libc:__support_osutil_io", + "//libc:__support_uint", "//libc:__support_uint128", ], ) -- GitLab From 27844cb2fa2a66fe90f12240809b260709fb2cc9 Mon Sep 17 00:00:00 2001 From: Guillaume Chatelet Date: Thu, 7 Mar 2024 11:06:50 +0100 Subject: [PATCH 420/929] Revert "[libc] Remove UB specializations of type traits for `BigInt`" (#84297) Reverts llvm/llvm-project#84035 Several bots are failing: - https://lab.llvm.org/buildbot/#/builders/223/builds/37522 - https://lab.llvm.org/buildbot/#/builders/162/builds/51978 - https://lab.llvm.org/buildbot/#/builders/163/builds/52560 - https://lab.llvm.org/buildbot/#/builders/250/builds/19619 --- libc/src/__support/CMakeLists.txt | 1 - libc/src/__support/CPP/bit.h | 112 ++++------ libc/src/__support/UInt.h | 192 ++++++------------ libc/src/__support/float_to_string.h | 2 +- libc/src/__support/integer_to_string.h | 19 +- libc/test/UnitTest/CMakeLists.txt | 1 - libc/test/UnitTest/LibcTest.cpp | 10 +- libc/test/UnitTest/LibcTest.h | 1 - libc/test/UnitTest/TestLogger.cpp | 8 +- libc/test/src/__support/CPP/bit_test.cpp | 49 ++--- .../llvm-project-overlay/libc/BUILD.bazel | 1 - .../libc/test/UnitTest/BUILD.bazel | 1 - 12 files changed, 125 insertions(+), 272 deletions(-) diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt index 17c04aa57e6f..1a4b3e9a2145 100644 --- a/libc/src/__support/CMakeLists.txt +++ b/libc/src/__support/CMakeLists.txt @@ -95,7 +95,6 @@ add_header_library( HDRS integer_to_string.h DEPENDS - .uint libc.src.__support.common libc.src.__support.CPP.algorithm libc.src.__support.CPP.limits diff --git a/libc/src/__support/CPP/bit.h b/libc/src/__support/CPP/bit.h index bc2f595845a9..7d11e7d5c497 100644 --- a/libc/src/__support/CPP/bit.h +++ b/libc/src/__support/CPP/bit.h @@ -27,14 +27,13 @@ namespace LIBC_NAMESPACE::cpp { // This implementation of bit_cast requires trivially-constructible To, to avoid // UB in the implementation. -template -LIBC_INLINE constexpr cpp::enable_if_t< - (sizeof(To) == sizeof(From)) && - cpp::is_trivially_constructible::value && - cpp::is_trivially_copyable::value && - cpp::is_trivially_copyable::value, - To> -bit_cast(const From &from) { +template < + typename To, typename From, + typename = cpp::enable_if_t::value && + cpp::is_trivially_copyable::value && + cpp::is_trivially_copyable::value>> +LIBC_INLINE constexpr To bit_cast(const From &from) { MSAN_UNPOISON(&from, sizeof(From)); #if LIBC_HAS_BUILTIN(__builtin_bit_cast) return __builtin_bit_cast(To, from); @@ -52,10 +51,8 @@ bit_cast(const From &from) { #endif // LIBC_HAS_BUILTIN(__builtin_bit_cast) } -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, - bool> -has_single_bit(T value) { +template >> +[[nodiscard]] LIBC_INLINE constexpr bool has_single_bit(T value) { return (value != 0) && ((value & (value - 1)) == 0); } @@ -73,9 +70,8 @@ has_single_bit(T value) { /// Only unsigned integral types are allowed. /// /// Returns cpp::numeric_limits::digits on an input of 0. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> -countr_zero(T value) { +template >> +[[nodiscard]] LIBC_INLINE constexpr int countr_zero(T value) { if (!value) return cpp::numeric_limits::digits; if (value & 0x1) @@ -107,9 +103,8 @@ ADD_SPECIALIZATION(countr_zero, unsigned long long, __builtin_ctzll) /// Only unsigned integral types are allowed. /// /// Returns cpp::numeric_limits::digits on an input of 0. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> -countl_zero(T value) { +template >> +[[nodiscard]] LIBC_INLINE constexpr int countl_zero(T value) { if (!value) return cpp::numeric_limits::digits; // Bisection method. @@ -140,9 +135,8 @@ ADD_SPECIALIZATION(countl_zero, unsigned long long, __builtin_clzll) /// Only unsigned integral types are allowed. /// /// Returns cpp::numeric_limits::digits on an input of all ones. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> -countl_one(T value) { +template >> +[[nodiscard]] LIBC_INLINE constexpr int countl_one(T value) { return cpp::countl_zero(~value); } @@ -153,9 +147,8 @@ countl_one(T value) { /// Only unsigned integral types are allowed. /// /// Returns cpp::numeric_limits::digits on an input of all ones. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> -countr_one(T value) { +template >> +[[nodiscard]] LIBC_INLINE constexpr int countr_one(T value) { return cpp::countr_zero(~value); } @@ -163,9 +156,8 @@ countr_one(T value) { /// Returns 0 otherwise. /// /// Ex. bit_width(5) == 3. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> -bit_width(T value) { +template >> +[[nodiscard]] LIBC_INLINE constexpr int bit_width(T value) { return cpp::numeric_limits::digits - cpp::countl_zero(value); } @@ -173,9 +165,8 @@ bit_width(T value) { /// nonzero. Returns 0 otherwise. /// /// Ex. bit_floor(5) == 4. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> -bit_floor(T value) { +template >> +[[nodiscard]] LIBC_INLINE constexpr T bit_floor(T value) { if (!value) return 0; return T(1) << (cpp::bit_width(value) - 1); @@ -188,9 +179,8 @@ bit_floor(T value) { /// /// The return value is undefined if the input is larger than the largest power /// of two representable in T. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> -bit_ceil(T value) { +template >> +[[nodiscard]] LIBC_INLINE constexpr T bit_ceil(T value) { if (value < 2) return 1; return T(1) << cpp::bit_width(value - 1u); @@ -200,31 +190,28 @@ bit_ceil(T value) { // from https://blog.regehr.org/archives/1063. // Forward-declare rotr so that rotl can use it. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> -rotr(T value, int rotate); +template >> +[[nodiscard]] LIBC_INLINE constexpr T rotr(T value, int rotate); -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> -rotl(T value, int rotate) { +template >> +[[nodiscard]] LIBC_INLINE constexpr T rotl(T value, int rotate) { constexpr unsigned N = cpp::numeric_limits::digits; rotate = rotate % N; if (!rotate) return value; if (rotate < 0) - return cpp::rotr(value, -rotate); + return cpp::rotr(value, -rotate); return (value << rotate) | (value >> (N - rotate)); } -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> -rotr(T value, int rotate) { +template +[[nodiscard]] LIBC_INLINE constexpr T rotr(T value, int rotate) { constexpr unsigned N = cpp::numeric_limits::digits; rotate = rotate % N; if (!rotate) return value; if (rotate < 0) - return cpp::rotl(value, -rotate); + return cpp::rotl(value, -rotate); return (value >> rotate) | (value << (N - rotate)); } @@ -239,44 +226,33 @@ LIBC_INLINE constexpr To bit_or_static_cast(const From &from) { } } -// TODO: remove from 'bit.h' as it is not a standard function. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> -first_leading_zero(T value) { +template >> +[[nodiscard]] LIBC_INLINE constexpr int first_leading_zero(T value) { return value == cpp::numeric_limits::max() ? 0 : countl_one(value) + 1; } -// TODO: remove from 'bit.h' as it is not a standard function. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> -first_leading_one(T value) { +template >> +[[nodiscard]] LIBC_INLINE constexpr int first_leading_one(T value) { return first_leading_zero(static_cast(~value)); } -// TODO: remove from 'bit.h' as it is not a standard function. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> -first_trailing_zero(T value) { +template >> +[[nodiscard]] LIBC_INLINE constexpr int first_trailing_zero(T value) { return value == cpp::numeric_limits::max() ? 0 : countr_zero(static_cast(~value)) + 1; } -// TODO: remove from 'bit.h' as it is not a standard function. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> -first_trailing_one(T value) { +template >> +[[nodiscard]] LIBC_INLINE constexpr int first_trailing_one(T value) { return value == cpp::numeric_limits::max() ? 0 : countr_zero(value) + 1; } /// Count number of 1's aka population count or hamming weight. /// /// Only unsigned integral types are allowed. -// TODO: rename as 'popcount' to follow the standard -// https://en.cppreference.com/w/cpp/numeric/popcount -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> -count_ones(T value) { +template >> +[[nodiscard]] LIBC_INLINE constexpr int count_ones(T value) { int count = 0; for (int i = 0; i != cpp::numeric_limits::digits; ++i) if ((value >> i) & 0x1) @@ -296,10 +272,8 @@ ADD_SPECIALIZATION(unsigned long long, __builtin_popcountll) // TODO: 128b specializations? #undef ADD_SPECIALIZATION -// TODO: remove from 'bit.h' as it is not a standard function. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> -count_zeros(T value) { +template >> +[[nodiscard]] LIBC_INLINE constexpr int count_zeros(T value) { return count_ones(static_cast(~value)); } diff --git a/libc/src/__support/UInt.h b/libc/src/__support/UInt.h index b3d8f00b9a01..5973e6fab1d7 100644 --- a/libc/src/__support/UInt.h +++ b/libc/src/__support/UInt.h @@ -43,9 +43,6 @@ struct BigInt { static_assert(is_integral_v && is_unsigned_v, "WordType must be unsigned integer."); - using word_type = WordType; - LIBC_INLINE_VAR static constexpr bool SIGNED = Signed; - LIBC_INLINE_VAR static constexpr size_t BITS = Bits; LIBC_INLINE_VAR static constexpr size_t WORD_SIZE = sizeof(WordType) * CHAR_BIT; @@ -53,10 +50,6 @@ struct BigInt { "Number of bits in BigInt should be a multiple of WORD_SIZE."); LIBC_INLINE_VAR static constexpr size_t WORD_COUNT = Bits / WORD_SIZE; - - using unsigned_type = BigInt; - using signed_type = BigInt; - cpp::array val{}; LIBC_INLINE constexpr BigInt() = default; @@ -586,33 +579,19 @@ struct BigInt { return *this; } - // TODO: remove and use cpp::countl_zero below. - [[nodiscard]] LIBC_INLINE constexpr int clz() const { - constexpr int word_digits = cpp::numeric_limits::digits; - int leading_zeroes = 0; - for (auto i = val.size(); i > 0;) { - --i; - const int zeroes = countl_zero(val[i]); - leading_zeroes += zeroes; - if (zeroes != word_digits) + LIBC_INLINE constexpr uint64_t clz() { + uint64_t leading_zeroes = 0; + for (size_t i = WORD_COUNT; i > 0; --i) { + if (val[i - 1] == 0) { + leading_zeroes += WORD_SIZE; + } else { + leading_zeroes += countl_zero(val[i - 1]); break; + } } return leading_zeroes; } - // TODO: remove and use cpp::countr_zero below. - [[nodiscard]] LIBC_INLINE constexpr int ctz() const { - constexpr int word_digits = cpp::numeric_limits::digits; - int trailing_zeroes = 0; - for (auto word : val) { - const int zeroes = countr_zero(word); - trailing_zeroes += zeroes; - if (zeroes != word_digits) - break; - } - return trailing_zeroes; - } - LIBC_INLINE constexpr void shift_left(size_t s) { if constexpr (Bits == WORD_SIZE) { // Use native types if possible. @@ -937,121 +916,64 @@ public: LIBC_INLINE_VAR static constexpr int digits = 128; }; -// type traits to determine whether a T is a cpp::BigInt. -template struct is_big_int : cpp::false_type {}; - +// Provides is_integral of U/Int<128>, U/Int<192>, U/Int<256>. template -struct is_big_int> : cpp::true_type {}; - -template -LIBC_INLINE_VAR constexpr bool is_big_int_v = is_big_int::value; - -// Specialization of cpp::bit_cast ('bit.h') from T to BigInt. -template -LIBC_INLINE constexpr cpp::enable_if_t< - (sizeof(To) == sizeof(From)) && cpp::is_trivially_copyable::value && - cpp::is_trivially_copyable::value && is_big_int::value, - To> -bit_cast(const From &from) { - To out; - using Storage = decltype(out.val); - out.val = cpp::bit_cast(from); - return out; -} +struct is_integral> : cpp::true_type {}; -// Specialization of cpp::bit_cast ('bit.h') from BigInt to T. -template -LIBC_INLINE constexpr cpp::enable_if_t< - sizeof(To) == sizeof(UInt) && - cpp::is_trivially_constructible::value && - cpp::is_trivially_copyable::value && - cpp::is_trivially_copyable>::value, - To> -bit_cast(const UInt &from) { - return cpp::bit_cast(from.val); -} - -// Specialization of cpp::has_single_bit ('bit.h') for BigInt. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, bool> -has_single_bit(T value) { - int bits = 0; - for (auto word : value.val) { - if (word == 0) - continue; - bits += count_ones(word); - if (bits > 1) - return false; - } - return bits == 1; -} - -// Specialization of cpp::countr_zero ('bit.h') for BigInt. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> -countr_zero(const T &value) { - return value.ctz(); -} +// Provides is_unsigned of UInt<128>, UInt<192>, UInt<256>. +template +struct is_unsigned> : cpp::bool_constant {}; -// Specialization of cpp::countl_zero ('bit.h') for BigInt. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> -countl_zero(const T &value) { - return value.clz(); -} +template +struct make_unsigned> + : type_identity> {}; -// Specialization of cpp::countl_one ('bit.h') for BigInt. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> -countl_one(T value) { - // TODO : Implement a faster version not involving operator~. - return cpp::countl_zero(~value); -} +template +struct make_signed> + : type_identity> {}; -// Specialization of cpp::countr_one ('bit.h') for BigInt. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> -countr_one(T value) { - // TODO : Implement a faster version not involving operator~. - return cpp::countr_zero(~value); -} +namespace internal { +template struct is_custom_uint : cpp::false_type {}; -// Specialization of cpp::bit_width ('bit.h') for BigInt. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> -bit_width(T value) { - return cpp::numeric_limits::digits - cpp::countl_zero(value); -} +template +struct is_custom_uint> : cpp::true_type {}; +} // namespace internal -// Forward-declare rotr so that rotl can use it. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> -rotr(T value, int rotate); - -// Specialization of cpp::rotl ('bit.h') for BigInt. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> -rotl(T value, int rotate) { - constexpr unsigned N = cpp::numeric_limits::digits; - rotate = rotate % N; - if (!rotate) - return value; - if (rotate < 0) - return cpp::rotr(value, -rotate); - return (value << rotate) | (value >> (N - rotate)); +// bit_cast to UInt +// Note: The standard scheme for SFINAE selection is to have exactly one +// function instanciation valid at a time. This is usually done by having a +// predicate in one function and the negated predicate in the other one. +// e.g. +// template::value == true> ... +// template::value == false> ... +// +// Unfortunately this would make the default 'cpp::bit_cast' aware of +// 'is_custom_uint' (or any other customization). To prevent exposing all +// customizations in the original function, we create a different function with +// four 'typename's instead of three - otherwise it would be considered as a +// redeclaration of the same function leading to "error: template parameter +// redefines default argument". +template ::value && + cpp::is_trivially_copyable::value>, + typename = cpp::enable_if_t::value>> +LIBC_INLINE constexpr To bit_cast(const From &from) { + To out; + using Storage = decltype(out.val); + out.val = cpp::bit_cast(from); + return out; } -// Specialization of cpp::rotr ('bit.h') for BigInt. -template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> -rotr(T value, int rotate) { - constexpr unsigned N = cpp::numeric_limits::digits; - rotate = rotate % N; - if (!rotate) - return value; - if (rotate < 0) - return cpp::rotl(value, -rotate); - return (value >> rotate) | (value << (N - rotate)); +// bit_cast from UInt +template < + typename To, size_t Bits, + typename = cpp::enable_if_t) && + cpp::is_trivially_constructible::value && + cpp::is_trivially_copyable::value && + cpp::is_trivially_copyable>::value>> +LIBC_INLINE constexpr To bit_cast(const UInt &from) { + return cpp::bit_cast(from.val); } } // namespace LIBC_NAMESPACE::cpp diff --git a/libc/src/__support/float_to_string.h b/libc/src/__support/float_to_string.h index 27476433a945..744842ced8d7 100644 --- a/libc/src/__support/float_to_string.h +++ b/libc/src/__support/float_to_string.h @@ -713,7 +713,7 @@ template <> class FloatToString { float_as_fixed.shift_left(SHIFT_AMOUNT); // If there are still digits above the decimal point, handle those. - if (float_as_fixed.clz() < static_cast(EXTRA_INT_WIDTH)) { + if (float_as_fixed.clz() < EXTRA_INT_WIDTH) { cpp::UInt above_decimal_point = float_as_fixed >> FLOAT_AS_INT_WIDTH; diff --git a/libc/src/__support/integer_to_string.h b/libc/src/__support/integer_to_string.h index a5872dce6520..81ed21ccfca1 100644 --- a/libc/src/__support/integer_to_string.h +++ b/libc/src/__support/integer_to_string.h @@ -67,7 +67,6 @@ #include "src/__support/CPP/span.h" #include "src/__support/CPP/string_view.h" #include "src/__support/CPP/type_traits.h" -#include "src/__support/UInt.h" // is_big_int #include "src/__support/common.h" namespace LIBC_NAMESPACE { @@ -150,18 +149,6 @@ public: using StringBufferWriter = StringBufferWriterImpl; using BackwardStringBufferWriter = StringBufferWriterImpl; -template struct IntegerWriterUnsigned {}; - -template -struct IntegerWriterUnsigned>> { - using type = cpp::make_unsigned_t; -}; - -template -struct IntegerWriterUnsigned>> { - using type = typename T::unsigned_type; -}; - } // namespace details namespace radix { @@ -176,7 +163,7 @@ template using Custom = details::Fmt; // See file header for documentation. template class IntegerToString { - static_assert(cpp::is_integral_v || cpp::is_big_int_v); + static_assert(cpp::is_integral_v); LIBC_INLINE static constexpr size_t compute_buffer_size() { constexpr auto MAX_DIGITS = []() -> size_t { @@ -221,8 +208,8 @@ template class IntegerToString { // An internal stateless structure that handles the number formatting logic. struct IntegerWriter { - static_assert(cpp::is_integral_v || cpp::is_big_int_v); - using UNSIGNED_T = typename details::IntegerWriterUnsigned::type; + static_assert(cpp::is_integral_v); + using UNSIGNED_T = cpp::make_unsigned_t; LIBC_INLINE static char digit_char(uint8_t digit) { if (digit < 10) diff --git a/libc/test/UnitTest/CMakeLists.txt b/libc/test/UnitTest/CMakeLists.txt index 36837c553efc..4668f0061975 100644 --- a/libc/test/UnitTest/CMakeLists.txt +++ b/libc/test/UnitTest/CMakeLists.txt @@ -74,7 +74,6 @@ add_unittest_framework_library( libc.src.__support.CPP.type_traits libc.src.__support.fixed_point.fx_rep libc.src.__support.OSUtil.osutil - libc.src.__support.uint libc.src.__support.uint128 ) diff --git a/libc/test/UnitTest/LibcTest.cpp b/libc/test/UnitTest/LibcTest.cpp index 0340f7ed3710..7b0e4fca8368 100644 --- a/libc/test/UnitTest/LibcTest.cpp +++ b/libc/test/UnitTest/LibcTest.cpp @@ -38,8 +38,7 @@ TestLogger &operator<<(TestLogger &logger, Location Loc) { // When the value is UInt128, __uint128_t or wider, show its hexadecimal // digits. template -cpp::enable_if_t<(cpp::is_integral_v && (sizeof(T) > sizeof(uint64_t))) || - cpp::is_big_int_v, +cpp::enable_if_t && (sizeof(T) > sizeof(uint64_t)), cpp::string> describeValue(T Value) { static_assert(sizeof(T) % 8 == 0, "Unsupported size of UInt"); @@ -48,10 +47,11 @@ describeValue(T Value) { } // When the value is of a standard integral type, just display it as normal. -template -cpp::enable_if_t && (sizeof(T) <= sizeof(uint64_t)), +template +cpp::enable_if_t && + sizeof(ValType) <= sizeof(uint64_t), cpp::string> -describeValue(T Value) { +describeValue(ValType Value) { return cpp::to_string(Value); } diff --git a/libc/test/UnitTest/LibcTest.h b/libc/test/UnitTest/LibcTest.h index d26d6490bcb5..639f60058325 100644 --- a/libc/test/UnitTest/LibcTest.h +++ b/libc/test/UnitTest/LibcTest.h @@ -127,7 +127,6 @@ protected: // of type promotion. template || - cpp::is_big_int_v || cpp::is_fixed_point_v, int> = 0> bool test(TestCond Cond, ValType LHS, ValType RHS, const char *LHSStr, diff --git a/libc/test/UnitTest/TestLogger.cpp b/libc/test/UnitTest/TestLogger.cpp index 469b3a11d57d..6bb0e17dc388 100644 --- a/libc/test/UnitTest/TestLogger.cpp +++ b/libc/test/UnitTest/TestLogger.cpp @@ -2,7 +2,6 @@ #include "src/__support/CPP/string.h" #include "src/__support/CPP/string_view.h" #include "src/__support/OSUtil/io.h" // write_to_stderr -#include "src/__support/UInt.h" // is_big_int #include "src/__support/UInt128.h" #include @@ -48,9 +47,8 @@ template <> TestLogger &TestLogger::operator<<(void *addr) { } template TestLogger &TestLogger::operator<<(T t) { - if constexpr (cpp::is_big_int_v || - (cpp::is_integral_v && cpp::is_unsigned_v && - (sizeof(T) > sizeof(uint64_t)))) { + if constexpr (cpp::is_integral_v && cpp::is_unsigned_v && + sizeof(T) > sizeof(uint64_t)) { static_assert(sizeof(T) % 8 == 0, "Unsupported size of UInt"); const IntegerToString buffer(t); return *this << buffer.view(); @@ -70,7 +68,7 @@ template TestLogger &TestLogger::operator<< (unsigned short); template TestLogger &TestLogger::operator<< (unsigned int); template TestLogger &TestLogger::operator<< (unsigned long); template TestLogger & - TestLogger::operator<< (unsigned long long); +TestLogger::operator<< (unsigned long long); #ifdef __SIZEOF_INT128__ template TestLogger &TestLogger::operator<< <__uint128_t>(__uint128_t); diff --git a/libc/test/src/__support/CPP/bit_test.cpp b/libc/test/src/__support/CPP/bit_test.cpp index 1e3d895e6453..115a5d505c4b 100644 --- a/libc/test/src/__support/CPP/bit_test.cpp +++ b/libc/test/src/__support/CPP/bit_test.cpp @@ -12,44 +12,21 @@ #include -#include - namespace LIBC_NAMESPACE::cpp { -using UnsignedTypesNoBigInt = testing::TypeList< -#if defined(__SIZEOF_INT128__) - __uint128_t, -#endif - unsigned char, unsigned short, unsigned int, unsigned long, - unsigned long long>; - -using UnsignedTypes = testing::TypeList< +using UnsignedTypes = + testing::TypeList>; + cpp::UInt<128>>; TYPED_TEST(LlvmLibcBitTest, HasSingleBit, UnsignedTypes) { - constexpr auto ZERO = T(0); - constexpr auto ALL_ONES = T(~ZERO); - EXPECT_FALSE(has_single_bit(ZERO)); - EXPECT_FALSE(has_single_bit(ALL_ONES)); - + EXPECT_FALSE(has_single_bit(T(0))); + EXPECT_FALSE(has_single_bit(~T(0))); for (T value = 1; value; value <<= 1) EXPECT_TRUE(has_single_bit(value)); - - // We test that if two bits are set has_single_bit returns false. - // We do this by setting the highest or lowest bit depending or where the - // current bit is. This is a bit convoluted but it helps catch a bug on BigInt - // where we have to work on an element-by-element basis. - constexpr auto MIDPOINT = T(ALL_ONES / 2); - constexpr auto LSB = T(1); - constexpr auto MSB = T(~(ALL_ONES >> 1)); - for (T value = 1; value; value <<= 1) { - auto two_bits_value = value | ((value <= MIDPOINT) ? MSB : LSB); - EXPECT_FALSE(has_single_bit(two_bits_value)); - } } TYPED_TEST(LlvmLibcBitTest, CountLZero, UnsignedTypes) { @@ -229,39 +206,39 @@ TEST(LlvmLibcBitTest, Rotr) { rotr(0x12345678deadbeefULL, -19)); } -TYPED_TEST(LlvmLibcBitTest, FirstLeadingZero, UnsignedTypesNoBigInt) { +TYPED_TEST(LlvmLibcBitTest, FirstLeadingZero, UnsignedTypes) { EXPECT_EQ(first_leading_zero(cpp::numeric_limits::max()), 0); for (int i = 0U; i != cpp::numeric_limits::digits; ++i) EXPECT_EQ(first_leading_zero(~(T(1) << i)), cpp::numeric_limits::digits - i); } -TYPED_TEST(LlvmLibcBitTest, FirstLeadingOne, UnsignedTypesNoBigInt) { +TYPED_TEST(LlvmLibcBitTest, FirstLeadingOne, UnsignedTypes) { EXPECT_EQ(first_leading_one(static_cast(0)), 0); for (int i = 0U; i != cpp::numeric_limits::digits; ++i) EXPECT_EQ(first_leading_one(T(1) << i), cpp::numeric_limits::digits - i); } -TYPED_TEST(LlvmLibcBitTest, FirstTrailingZero, UnsignedTypesNoBigInt) { +TYPED_TEST(LlvmLibcBitTest, FirstTrailingZero, UnsignedTypes) { EXPECT_EQ(first_trailing_zero(cpp::numeric_limits::max()), 0); for (int i = 0U; i != cpp::numeric_limits::digits; ++i) EXPECT_EQ(first_trailing_zero(~(T(1) << i)), i + 1); } -TYPED_TEST(LlvmLibcBitTest, FirstTrailingOne, UnsignedTypesNoBigInt) { +TYPED_TEST(LlvmLibcBitTest, FirstTrailingOne, UnsignedTypes) { EXPECT_EQ(first_trailing_one(cpp::numeric_limits::max()), 0); for (int i = 0U; i != cpp::numeric_limits::digits; ++i) EXPECT_EQ(first_trailing_one(T(1) << i), i + 1); } -TYPED_TEST(LlvmLibcBitTest, CountZeros, UnsignedTypesNoBigInt) { +TYPED_TEST(LlvmLibcBitTest, CountZeros, UnsignedTypes) { EXPECT_EQ(count_zeros(T(0)), cpp::numeric_limits::digits); for (int i = 0; i != cpp::numeric_limits::digits; ++i) EXPECT_EQ(count_zeros(cpp::numeric_limits::max() >> i), i); } -TYPED_TEST(LlvmLibcBitTest, CountOnes, UnsignedTypesNoBigInt) { +TYPED_TEST(LlvmLibcBitTest, CountOnes, UnsignedTypes) { EXPECT_EQ(count_ones(T(0)), 0); for (int i = 0; i != cpp::numeric_limits::digits; ++i) EXPECT_EQ(count_ones(cpp::numeric_limits::max() >> i), diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index 5c6cf761ebe7..49a454379e1c 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -507,7 +507,6 @@ libc_support_library( ":__support_cpp_span", ":__support_cpp_string_view", ":__support_cpp_type_traits", - ":__support_uint", ], ) diff --git a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel index 44692947af7c..a5c18fbb68b3 100644 --- a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel @@ -18,7 +18,6 @@ libc_support_library( "//libc:__support_cpp_string", "//libc:__support_cpp_string_view", "//libc:__support_osutil_io", - "//libc:__support_uint", "//libc:__support_uint128", ], ) -- GitLab From bf7f62ab92241298ccd7af008b3b26daac9c220b Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Thu, 7 Mar 2024 10:03:49 +0000 Subject: [PATCH 421/929] [AMDGPU] Make use of Mnem_gfx11_gfx12. NFC. --- llvm/lib/Target/AMDGPU/BUFInstructions.td | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index b984126d8447..9c6934865bfa 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -2902,7 +2902,7 @@ multiclass MTBUF_Real_AllAddr_gfx11_gfx12 op> multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Renamed op, string real_name> : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl { defvar ps = get_BUF_ps; - def : MnemonicAlias, Requires<[isGFX11Plus]>; + def : Mnem_gfx11_gfx12; } defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x008, "tbuffer_load_d16_format_x">; -- GitLab From 469c5e3da46115b9625e2d4771bd19d4968e3fa9 Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Thu, 7 Mar 2024 10:21:07 +0000 Subject: [PATCH 422/929] [AMDGPU] Simplify definition of renamed DS instructions. NFC. Following the pattern used for SOP instructions, we can use the same multiclass with a default argument to define renamed and non-renamed instructions. --- llvm/lib/Target/AMDGPU/DSInstructions.td | 149 ++++++++++------------- 1 file changed, 66 insertions(+), 83 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td index 5b94102770cd..a84227ebf506 100644 --- a/llvm/lib/Target/AMDGPU/DSInstructions.td +++ b/llvm/lib/Target/AMDGPU/DSInstructions.td @@ -1210,33 +1210,24 @@ class Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12 op, DS_Pseudo ps, int ef, // GFX12. //===----------------------------------------------------------------------===// -let AssemblerPredicate = isGFX12Plus, DecoderNamespace = "GFX12" in { - multiclass DS_Real_gfx12 op> { - defvar ps = !cast(NAME); +multiclass DS_Real_gfx12 op, string name = !tolower(NAME)> { + defvar ps = !cast(NAME); + let AssemblerPredicate = isGFX12Plus, DecoderNamespace = "GFX12" in def _gfx12 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12; - } - - multiclass DS_Real_Renamed_gfx12 op, string name> { - defvar ps = !cast(NAME); - def _gfx12 : - Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12, - MnemonicAlias, - Requires<[isGFX12Plus]>; - } -} // End AssemblerPredicate = isGFX12Plus, DecoderNamespace = "GFX12" - -defm DS_MIN_F32 : DS_Real_Renamed_gfx12<0x012, "ds_min_num_f32">; -defm DS_MAX_F32 : DS_Real_Renamed_gfx12<0x013, "ds_max_num_f32">; -defm DS_MIN_RTN_F32 : DS_Real_Renamed_gfx12<0x032, "ds_min_num_rtn_f32">; -defm DS_MAX_RTN_F32 : DS_Real_Renamed_gfx12<0x033, "ds_max_num_rtn_f32">; -defm DS_MIN_F64 : DS_Real_Renamed_gfx12<0x052, "ds_min_num_f64">; -defm DS_MAX_F64 : DS_Real_Renamed_gfx12<0x053, "ds_max_num_f64">; -defm DS_MIN_RTN_F64 : DS_Real_Renamed_gfx12<0x072, "ds_min_num_rtn_f64">; -defm DS_MAX_RTN_F64 : DS_Real_Renamed_gfx12<0x073, "ds_max_num_rtn_f64">; + name, /*hasGDS=*/false>; + if !ne(ps.Mnemonic, name) then + def : MnemonicAlias, Requires<[isGFX12Plus]>; +} + +defm DS_MIN_F32 : DS_Real_gfx12<0x012, "ds_min_num_f32">; +defm DS_MAX_F32 : DS_Real_gfx12<0x013, "ds_max_num_f32">; +defm DS_MIN_RTN_F32 : DS_Real_gfx12<0x032, "ds_min_num_rtn_f32">; +defm DS_MAX_RTN_F32 : DS_Real_gfx12<0x033, "ds_max_num_rtn_f32">; +defm DS_MIN_F64 : DS_Real_gfx12<0x052, "ds_min_num_f64">; +defm DS_MAX_F64 : DS_Real_gfx12<0x053, "ds_max_num_f64">; +defm DS_MIN_RTN_F64 : DS_Real_gfx12<0x072, "ds_min_num_rtn_f64">; +defm DS_MAX_RTN_F64 : DS_Real_gfx12<0x073, "ds_max_num_rtn_f64">; defm DS_COND_SUB_U32 : DS_Real_gfx12<0x098>; defm DS_SUB_CLAMP_U32 : DS_Real_gfx12<0x099>; defm DS_COND_SUB_RTN_U32 : DS_Real_gfx12<0x0a8>; @@ -1256,65 +1247,57 @@ def : MnemonicAlias<"ds_subrev_rtn_u64", "ds_rsub_rtn_u64">, Requires<[isGFX12Pl // GFX11. //===----------------------------------------------------------------------===// -let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in { - multiclass DS_Real_gfx11 op> { +multiclass DS_Real_gfx11 op, string name = !tolower(NAME)> { + defvar ps = !cast(NAME); + let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in def _gfx11 : - Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12(NAME), - SIEncodingFamily.GFX11>; - } - - multiclass DS_Real_Renamed_gfx11 op, string name> { - defvar ps = !cast(NAME); - def _gfx11 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12, - MnemonicAlias, Requires<[isGFX11Only]>; - } -} // End AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" - -multiclass DS_Real_gfx11_gfx12 op> - : DS_Real_gfx11, DS_Real_gfx12; - -multiclass DS_Real_Renamed_gfx11_gfx12 op, string name> - : DS_Real_Renamed_gfx11, - DS_Real_Renamed_gfx12; - -defm DS_WRITE_B32 : DS_Real_Renamed_gfx11_gfx12<0x00d, "ds_store_b32">; -defm DS_WRITE2_B32 : DS_Real_Renamed_gfx11_gfx12<0x00e, "ds_store_2addr_b32">; -defm DS_WRITE2ST64_B32 : DS_Real_Renamed_gfx11_gfx12<0x00f, "ds_store_2addr_stride64_b32">; -defm DS_WRITE_B8 : DS_Real_Renamed_gfx11_gfx12<0x01e, "ds_store_b8">; -defm DS_WRITE_B16 : DS_Real_Renamed_gfx11_gfx12<0x01f, "ds_store_b16">; -defm DS_WRXCHG_RTN_B32 : DS_Real_Renamed_gfx11_gfx12<0x02d, "ds_storexchg_rtn_b32">; -defm DS_WRXCHG2_RTN_B32 : DS_Real_Renamed_gfx11_gfx12<0x02e, "ds_storexchg_2addr_rtn_b32">; -defm DS_WRXCHG2ST64_RTN_B32 : DS_Real_Renamed_gfx11_gfx12<0x02f, "ds_storexchg_2addr_stride64_rtn_b32">; -defm DS_READ_B32 : DS_Real_Renamed_gfx11_gfx12<0x036, "ds_load_b32">; -defm DS_READ2_B32 : DS_Real_Renamed_gfx11_gfx12<0x037, "ds_load_2addr_b32">; -defm DS_READ2ST64_B32 : DS_Real_Renamed_gfx11_gfx12<0x038, "ds_load_2addr_stride64_b32">; -defm DS_READ_I8 : DS_Real_Renamed_gfx11_gfx12<0x039, "ds_load_i8">; -defm DS_READ_U8 : DS_Real_Renamed_gfx11_gfx12<0x03a, "ds_load_u8">; -defm DS_READ_I16 : DS_Real_Renamed_gfx11_gfx12<0x03b, "ds_load_i16">; -defm DS_READ_U16 : DS_Real_Renamed_gfx11_gfx12<0x03c, "ds_load_u16">; -defm DS_WRITE_B64 : DS_Real_Renamed_gfx11_gfx12<0x04d, "ds_store_b64">; -defm DS_WRITE2_B64 : DS_Real_Renamed_gfx11_gfx12<0x04e, "ds_store_2addr_b64">; -defm DS_WRITE2ST64_B64 : DS_Real_Renamed_gfx11_gfx12<0x04f, "ds_store_2addr_stride64_b64">; -defm DS_WRXCHG_RTN_B64 : DS_Real_Renamed_gfx11_gfx12<0x06d, "ds_storexchg_rtn_b64">; -defm DS_WRXCHG2_RTN_B64 : DS_Real_Renamed_gfx11_gfx12<0x06e, "ds_storexchg_2addr_rtn_b64">; -defm DS_WRXCHG2ST64_RTN_B64 : DS_Real_Renamed_gfx11_gfx12<0x06f, "ds_storexchg_2addr_stride64_rtn_b64">; -defm DS_READ_B64 : DS_Real_Renamed_gfx11_gfx12<0x076, "ds_load_b64">; -defm DS_READ2_B64 : DS_Real_Renamed_gfx11_gfx12<0x077, "ds_load_2addr_b64">; -defm DS_READ2ST64_B64 : DS_Real_Renamed_gfx11_gfx12<0x078, "ds_load_2addr_stride64_b64">; -defm DS_WRITE_B8_D16_HI : DS_Real_Renamed_gfx11_gfx12<0x0a0, "ds_store_b8_d16_hi">; -defm DS_WRITE_B16_D16_HI : DS_Real_Renamed_gfx11_gfx12<0x0a1, "ds_store_b16_d16_hi">; -defm DS_READ_U8_D16 : DS_Real_Renamed_gfx11_gfx12<0x0a2, "ds_load_u8_d16">; -defm DS_READ_U8_D16_HI : DS_Real_Renamed_gfx11_gfx12<0x0a3, "ds_load_u8_d16_hi">; -defm DS_READ_I8_D16 : DS_Real_Renamed_gfx11_gfx12<0x0a4, "ds_load_i8_d16">; -defm DS_READ_I8_D16_HI : DS_Real_Renamed_gfx11_gfx12<0x0a5, "ds_load_i8_d16_hi">; -defm DS_READ_U16_D16 : DS_Real_Renamed_gfx11_gfx12<0x0a6, "ds_load_u16_d16">; -defm DS_READ_U16_D16_HI : DS_Real_Renamed_gfx11_gfx12<0x0a7, "ds_load_u16_d16_hi">; -defm DS_WRITE_ADDTID_B32 : DS_Real_Renamed_gfx11_gfx12<0x0b0, "ds_store_addtid_b32">; -defm DS_READ_ADDTID_B32 : DS_Real_Renamed_gfx11_gfx12<0x0b1, "ds_load_addtid_b32">; -defm DS_WRITE_B96 : DS_Real_Renamed_gfx11_gfx12<0x0de, "ds_store_b96">; -defm DS_WRITE_B128 : DS_Real_Renamed_gfx11_gfx12<0x0df, "ds_store_b128">; -defm DS_READ_B96 : DS_Real_Renamed_gfx11_gfx12<0x0fe, "ds_load_b96">; -defm DS_READ_B128 : DS_Real_Renamed_gfx11_gfx12<0x0ff, "ds_load_b128">; + Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12; + if !ne(ps.Mnemonic, name) then + def : MnemonicAlias, Requires<[isGFX11Only]>; +} + +multiclass DS_Real_gfx11_gfx12 op, string name = !tolower(NAME)> + : DS_Real_gfx11, DS_Real_gfx12; + +defm DS_WRITE_B32 : DS_Real_gfx11_gfx12<0x00d, "ds_store_b32">; +defm DS_WRITE2_B32 : DS_Real_gfx11_gfx12<0x00e, "ds_store_2addr_b32">; +defm DS_WRITE2ST64_B32 : DS_Real_gfx11_gfx12<0x00f, "ds_store_2addr_stride64_b32">; +defm DS_WRITE_B8 : DS_Real_gfx11_gfx12<0x01e, "ds_store_b8">; +defm DS_WRITE_B16 : DS_Real_gfx11_gfx12<0x01f, "ds_store_b16">; +defm DS_WRXCHG_RTN_B32 : DS_Real_gfx11_gfx12<0x02d, "ds_storexchg_rtn_b32">; +defm DS_WRXCHG2_RTN_B32 : DS_Real_gfx11_gfx12<0x02e, "ds_storexchg_2addr_rtn_b32">; +defm DS_WRXCHG2ST64_RTN_B32 : DS_Real_gfx11_gfx12<0x02f, "ds_storexchg_2addr_stride64_rtn_b32">; +defm DS_READ_B32 : DS_Real_gfx11_gfx12<0x036, "ds_load_b32">; +defm DS_READ2_B32 : DS_Real_gfx11_gfx12<0x037, "ds_load_2addr_b32">; +defm DS_READ2ST64_B32 : DS_Real_gfx11_gfx12<0x038, "ds_load_2addr_stride64_b32">; +defm DS_READ_I8 : DS_Real_gfx11_gfx12<0x039, "ds_load_i8">; +defm DS_READ_U8 : DS_Real_gfx11_gfx12<0x03a, "ds_load_u8">; +defm DS_READ_I16 : DS_Real_gfx11_gfx12<0x03b, "ds_load_i16">; +defm DS_READ_U16 : DS_Real_gfx11_gfx12<0x03c, "ds_load_u16">; +defm DS_WRITE_B64 : DS_Real_gfx11_gfx12<0x04d, "ds_store_b64">; +defm DS_WRITE2_B64 : DS_Real_gfx11_gfx12<0x04e, "ds_store_2addr_b64">; +defm DS_WRITE2ST64_B64 : DS_Real_gfx11_gfx12<0x04f, "ds_store_2addr_stride64_b64">; +defm DS_WRXCHG_RTN_B64 : DS_Real_gfx11_gfx12<0x06d, "ds_storexchg_rtn_b64">; +defm DS_WRXCHG2_RTN_B64 : DS_Real_gfx11_gfx12<0x06e, "ds_storexchg_2addr_rtn_b64">; +defm DS_WRXCHG2ST64_RTN_B64 : DS_Real_gfx11_gfx12<0x06f, "ds_storexchg_2addr_stride64_rtn_b64">; +defm DS_READ_B64 : DS_Real_gfx11_gfx12<0x076, "ds_load_b64">; +defm DS_READ2_B64 : DS_Real_gfx11_gfx12<0x077, "ds_load_2addr_b64">; +defm DS_READ2ST64_B64 : DS_Real_gfx11_gfx12<0x078, "ds_load_2addr_stride64_b64">; +defm DS_WRITE_B8_D16_HI : DS_Real_gfx11_gfx12<0x0a0, "ds_store_b8_d16_hi">; +defm DS_WRITE_B16_D16_HI : DS_Real_gfx11_gfx12<0x0a1, "ds_store_b16_d16_hi">; +defm DS_READ_U8_D16 : DS_Real_gfx11_gfx12<0x0a2, "ds_load_u8_d16">; +defm DS_READ_U8_D16_HI : DS_Real_gfx11_gfx12<0x0a3, "ds_load_u8_d16_hi">; +defm DS_READ_I8_D16 : DS_Real_gfx11_gfx12<0x0a4, "ds_load_i8_d16">; +defm DS_READ_I8_D16_HI : DS_Real_gfx11_gfx12<0x0a5, "ds_load_i8_d16_hi">; +defm DS_READ_U16_D16 : DS_Real_gfx11_gfx12<0x0a6, "ds_load_u16_d16">; +defm DS_READ_U16_D16_HI : DS_Real_gfx11_gfx12<0x0a7, "ds_load_u16_d16_hi">; +defm DS_WRITE_ADDTID_B32 : DS_Real_gfx11_gfx12<0x0b0, "ds_store_addtid_b32">; +defm DS_READ_ADDTID_B32 : DS_Real_gfx11_gfx12<0x0b1, "ds_load_addtid_b32">; +defm DS_WRITE_B96 : DS_Real_gfx11_gfx12<0x0de, "ds_store_b96">; +defm DS_WRITE_B128 : DS_Real_gfx11_gfx12<0x0df, "ds_store_b128">; +defm DS_READ_B96 : DS_Real_gfx11_gfx12<0x0fe, "ds_load_b96">; +defm DS_READ_B128 : DS_Real_gfx11_gfx12<0x0ff, "ds_load_b128">; // DS_CMPST_* are renamed to DS_CMPSTORE_* in GFX11, but also the data operands (src and cmp) are swapped // comparing to pre-GFX11. -- GitLab From afac64cef40c77320cc49808be30f3e5ef7f7357 Mon Sep 17 00:00:00 2001 From: Matthias Gehre <93204396+mgehre-amd@users.noreply.github.com> Date: Thu, 29 Feb 2024 07:32:30 +0100 Subject: [PATCH 423/929] [MLIR] BufferResultsToOutParams: Allow to configure memCpyFn This allows us to configure the pass to emit linalg.copy instead of memref.copy. This is consistent with one-shot-bufferize, which also allows to configure the `memCpyFn`, see https://discord.com/channels/636084430946959380/642426447167881246/1211698722438783087 --- .../Dialect/Bufferization/Transforms/Passes.h | 8 ++++++ .../Transforms/BufferResultsToOutParams.cpp | 28 ++++++++++++++----- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.h b/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.h index bb4b52219816..809f03407258 100644 --- a/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.h +++ b/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.h @@ -149,11 +149,19 @@ std::unique_ptr createBufferLoopHoistingPass(); // Options struct for BufferResultsToOutParams pass. // Note: defined only here, not in tablegen. struct BufferResultsToOutParamsOptions { + /// Memcpy function: Generate a memcpy between two memrefs. + using MemCpyFn = + std::function; + // Filter function; returns true if the function should be converted. // Defaults to true, i.e. all functions are converted. llvm::function_ref filterFn = [](func::FuncOp *func) { return true; }; + + /// Memcpy function; used to create a copy between two memrefs. + /// If this is empty, memref.copy is used. + std::optional memCpyFn; }; /// Creates a pass that converts memref function results to out-params. diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp index dd359c2dcca5..930f035339c1 100644 --- a/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp +++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferResultsToOutParams.cpp @@ -21,6 +21,7 @@ namespace bufferization { } // namespace mlir using namespace mlir; +using MemCpyFn = bufferization::BufferResultsToOutParamsOptions::MemCpyFn; /// Return `true` if the given MemRef type has a fully dynamic layout. static bool hasFullyDynamicLayoutMap(MemRefType type) { @@ -97,9 +98,10 @@ updateFuncOp(func::FuncOp func, // Updates all ReturnOps in the scope of the given func::FuncOp by either // keeping them as return values or copying the associated buffer contents into // the given out-params. -static void updateReturnOps(func::FuncOp func, - ArrayRef appendedEntryArgs) { - func.walk([&](func::ReturnOp op) { +static LogicalResult updateReturnOps(func::FuncOp func, + ArrayRef appendedEntryArgs, + MemCpyFn memCpyFn) { + auto res = func.walk([&](func::ReturnOp op) { SmallVector copyIntoOutParams; SmallVector keepAsReturnOperands; for (Value operand : op.getOperands()) { @@ -109,12 +111,16 @@ static void updateReturnOps(func::FuncOp func, keepAsReturnOperands.push_back(operand); } OpBuilder builder(op); - for (auto t : llvm::zip(copyIntoOutParams, appendedEntryArgs)) - builder.create(op.getLoc(), std::get<0>(t), - std::get<1>(t)); + for (auto t : llvm::zip(copyIntoOutParams, appendedEntryArgs)) { + if (failed( + memCpyFn(builder, op.getLoc(), std::get<0>(t), std::get<1>(t)))) + return WalkResult::interrupt(); + } builder.create(op.getLoc(), keepAsReturnOperands); op.erase(); + return WalkResult::advance(); }); + return failure(res.wasInterrupted()); } // Updates all CallOps in the scope of the given ModuleOp by allocating @@ -192,7 +198,15 @@ LogicalResult mlir::bufferization::promoteBufferResultsToOutParams( return failure(); if (func.isExternal()) continue; - updateReturnOps(func, appendedEntryArgs); + auto defaultMemCpyFn = [](OpBuilder &builder, Location loc, Value from, + Value to) { + builder.create(loc, from, to); + return success(); + }; + if (failed(updateReturnOps(func, appendedEntryArgs, + options.memCpyFn.value_or(defaultMemCpyFn)))) { + return failure(); + } } if (failed(updateCalls(module, options))) return failure(); -- GitLab From 6f54a54c6f5f644b4f4c79882154fd9737568c8e Mon Sep 17 00:00:00 2001 From: Alexandros Lamprineas Date: Thu, 7 Mar 2024 10:33:26 +0000 Subject: [PATCH 424/929] [FMV] Remove duplicate features from mangled name. (#84165) ACLE suggests: https://github.com/ARM-software/acle/pull/308. GCC emits diagnostics for attribute strings which contain duplicate features, but for now let's follow the SPEC in regards to mangling rules and we can change the semantic behavior of the compiler later if there's value to it. --- clang/lib/CodeGen/Targets/AArch64.cpp | 4 +++- clang/test/CodeGen/attr-target-version.c | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/clang/lib/CodeGen/Targets/AArch64.cpp b/clang/lib/CodeGen/Targets/AArch64.cpp index 725e8a70fddf..85117366de0e 100644 --- a/clang/lib/CodeGen/Targets/AArch64.cpp +++ b/clang/lib/CodeGen/Targets/AArch64.cpp @@ -886,9 +886,11 @@ void AArch64ABIInfo::appendAttributeMangling(StringRef AttrStr, return LHS.compare(RHS) < 0; }); + llvm::SmallDenseSet UniqueFeats; for (auto &Feat : Features) if (auto Ext = llvm::AArch64::parseArchExtension(Feat)) - Out << 'M' << Ext->Name; + if (UniqueFeats.insert(Ext->Name).second) + Out << 'M' << Ext->Name; } std::unique_ptr diff --git a/clang/test/CodeGen/attr-target-version.c b/clang/test/CodeGen/attr-target-version.c index ae1a8772f6cc..b7112c783da9 100644 --- a/clang/test/CodeGen/attr-target-version.c +++ b/clang/test/CodeGen/attr-target-version.c @@ -273,7 +273,7 @@ int hoo(void) { // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: -// CHECK-NEXT: ret ptr @fmv_inline._MfcmaMfp16Mfp16MrdmMsme +// CHECK-NEXT: ret ptr @fmv_inline._MfcmaMfp16MrdmMsme // CHECK: resolver_else: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 // CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 864726312827224064 @@ -582,7 +582,7 @@ int hoo(void) { // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfcmaMfp16Mfp16MrdmMsme +// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfcmaMfp16MrdmMsme // CHECK-SAME: () #[[ATTR13:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 -- GitLab From c40146c214a705a232848144d9412c8a7c73f0fe Mon Sep 17 00:00:00 2001 From: Marius Brehler Date: Thu, 7 Mar 2024 11:34:11 +0100 Subject: [PATCH 425/929] [mlir][EmitC] Add Arith to EmitC conversions (#84151) This adds patterns and a pass to convert the Arith dialect to EmitC. For now, this covers arithemtic binary ops operating on floating point types. It is not checked within the patterns whether the types, such as the Tensor type, are supported in the respective EmitC operations. If unsupported types should be converted, the conversion will fail anyway because no legal EmitC operation can be created. This can clearly be improved in a follow up, also resulting in better error messages. Functions for such checks should not solely be used in the conversions and should also be (re)used in the verifier. --- .../Conversion/ArithToEmitC/ArithToEmitC.h | 20 +++++++ .../ArithToEmitC/ArithToEmitCPass.h | 21 +++++++ mlir/include/mlir/Conversion/Passes.h | 1 + mlir/include/mlir/Conversion/Passes.td | 9 +++ .../Conversion/ArithToEmitC/ArithToEmitC.cpp | 60 +++++++++++++++++++ .../ArithToEmitC/ArithToEmitCPass.cpp | 53 ++++++++++++++++ .../Conversion/ArithToEmitC/CMakeLists.txt | 16 +++++ mlir/lib/Conversion/CMakeLists.txt | 1 + .../ArithToEmitC/arith-to-emitc.mlir | 14 +++++ .../llvm-project-overlay/mlir/BUILD.bazel | 27 +++++++++ 10 files changed, 222 insertions(+) create mode 100644 mlir/include/mlir/Conversion/ArithToEmitC/ArithToEmitC.h create mode 100644 mlir/include/mlir/Conversion/ArithToEmitC/ArithToEmitCPass.h create mode 100644 mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp create mode 100644 mlir/lib/Conversion/ArithToEmitC/ArithToEmitCPass.cpp create mode 100644 mlir/lib/Conversion/ArithToEmitC/CMakeLists.txt create mode 100644 mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir diff --git a/mlir/include/mlir/Conversion/ArithToEmitC/ArithToEmitC.h b/mlir/include/mlir/Conversion/ArithToEmitC/ArithToEmitC.h new file mode 100644 index 000000000000..9cb43689d1ce --- /dev/null +++ b/mlir/include/mlir/Conversion/ArithToEmitC/ArithToEmitC.h @@ -0,0 +1,20 @@ +//===- ArithToEmitC.h - Arith to EmitC Patterns -----------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_CONVERSION_ARITHTOEMITC_ARITHTOEMITC_H +#define MLIR_CONVERSION_ARITHTOEMITC_ARITHTOEMITC_H + +namespace mlir { +class RewritePatternSet; +class TypeConverter; + +void populateArithToEmitCPatterns(TypeConverter &typeConverter, + RewritePatternSet &patterns); +} // namespace mlir + +#endif // MLIR_CONVERSION_ARITHTOEMITC_ARITHTOEMITC_H diff --git a/mlir/include/mlir/Conversion/ArithToEmitC/ArithToEmitCPass.h b/mlir/include/mlir/Conversion/ArithToEmitC/ArithToEmitCPass.h new file mode 100644 index 000000000000..6b98fed7185e --- /dev/null +++ b/mlir/include/mlir/Conversion/ArithToEmitC/ArithToEmitCPass.h @@ -0,0 +1,21 @@ +//===- ArithToEmitCPass.h - Arith to EmitC Pass -----------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_CONVERSION_ARITHTOEMITC_ARITHTOEMITCPASS_H +#define MLIR_CONVERSION_ARITHTOEMITC_ARITHTOEMITCPASS_H + +#include + +namespace mlir { +class Pass; + +#define GEN_PASS_DECL_CONVERTARITHTOEMITC +#include "mlir/Conversion/Passes.h.inc" +} // namespace mlir + +#endif // MLIR_CONVERSION_ARITHTOEMITC_ARITHTOEMITCPASS_H diff --git a/mlir/include/mlir/Conversion/Passes.h b/mlir/include/mlir/Conversion/Passes.h index 81f69210fade..f2aa4fb53540 100644 --- a/mlir/include/mlir/Conversion/Passes.h +++ b/mlir/include/mlir/Conversion/Passes.h @@ -13,6 +13,7 @@ #include "mlir/Conversion/AffineToStandard/AffineToStandard.h" #include "mlir/Conversion/ArithToAMDGPU/ArithToAMDGPU.h" #include "mlir/Conversion/ArithToArmSME/ArithToArmSME.h" +#include "mlir/Conversion/ArithToEmitC/ArithToEmitCPass.h" #include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h" #include "mlir/Conversion/ArithToSPIRV/ArithToSPIRV.h" #include "mlir/Conversion/ArmNeon2dToIntr/ArmNeon2dToIntr.h" diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td index 94fc7a7d2194..bd81cc6d5323 100644 --- a/mlir/include/mlir/Conversion/Passes.td +++ b/mlir/include/mlir/Conversion/Passes.td @@ -133,6 +133,15 @@ def ArithToAMDGPUConversionPass : Pass<"convert-arith-to-amdgpu"> { ]; } +//===----------------------------------------------------------------------===// +// ArithToEmitC +//===----------------------------------------------------------------------===// + +def ConvertArithToEmitC : Pass<"convert-arith-to-emitc"> { + let summary = "Convert Arith dialect to EmitC dialect"; + let dependentDialects = ["emitc::EmitCDialect"]; +} + //===----------------------------------------------------------------------===// // ArithToLLVM //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp new file mode 100644 index 000000000000..6909534d4790 --- /dev/null +++ b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp @@ -0,0 +1,60 @@ +//===- ArithToEmitC.cpp - Arith to EmitC Patterns ---------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements patterns to convert the Arith dialect to the EmitC +// dialect. +// +//===----------------------------------------------------------------------===// + +#include "mlir/Conversion/ArithToEmitC/ArithToEmitC.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/EmitC/IR/EmitC.h" +#include "mlir/Transforms/DialectConversion.h" + +using namespace mlir; + +//===----------------------------------------------------------------------===// +// Conversion Patterns +//===----------------------------------------------------------------------===// + +namespace { +template +class ArithOpConversion final : public OpConversionPattern { +public: + using OpConversionPattern::OpConversionPattern; + + LogicalResult + matchAndRewrite(ArithOp arithOp, typename ArithOp::Adaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + rewriter.template replaceOpWithNewOp(arithOp, arithOp.getType(), + adaptor.getOperands()); + + return success(); + } +}; +} // namespace + +//===----------------------------------------------------------------------===// +// Pattern population +//===----------------------------------------------------------------------===// + +void mlir::populateArithToEmitCPatterns(TypeConverter &typeConverter, + RewritePatternSet &patterns) { + MLIRContext *ctx = patterns.getContext(); + + // clang-format off + patterns.add< + ArithOpConversion, + ArithOpConversion, + ArithOpConversion, + ArithOpConversion + >(typeConverter, ctx); + // clang-format on +} diff --git a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitCPass.cpp b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitCPass.cpp new file mode 100644 index 000000000000..b377c063a7aa --- /dev/null +++ b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitCPass.cpp @@ -0,0 +1,53 @@ +//===- ArithToEmitCPass.cpp - Arith to EmitC Pass ---------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements a pass to convert the Arith dialect to the EmitC +// dialect. +// +//===----------------------------------------------------------------------===// + +#include "mlir/Conversion/ArithToEmitC/ArithToEmitCPass.h" + +#include "mlir/Conversion/ArithToEmitC/ArithToEmitC.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/EmitC/IR/EmitC.h" +#include "mlir/Pass/Pass.h" +#include "mlir/Transforms/DialectConversion.h" + +namespace mlir { +#define GEN_PASS_DEF_CONVERTARITHTOEMITC +#include "mlir/Conversion/Passes.h.inc" +} // namespace mlir + +using namespace mlir; + +namespace { +struct ConvertArithToEmitC + : public impl::ConvertArithToEmitCBase { + void runOnOperation() override; +}; +} // namespace + +void ConvertArithToEmitC::runOnOperation() { + ConversionTarget target(getContext()); + + target.addLegalDialect(); + target.addIllegalDialect(); + target.addLegalOp(); + + RewritePatternSet patterns(&getContext()); + + TypeConverter typeConverter; + typeConverter.addConversion([](Type type) { return type; }); + + populateArithToEmitCPatterns(typeConverter, patterns); + + if (failed( + applyPartialConversion(getOperation(), target, std::move(patterns)))) + signalPassFailure(); +} diff --git a/mlir/lib/Conversion/ArithToEmitC/CMakeLists.txt b/mlir/lib/Conversion/ArithToEmitC/CMakeLists.txt new file mode 100644 index 000000000000..a3784f47c3bc --- /dev/null +++ b/mlir/lib/Conversion/ArithToEmitC/CMakeLists.txt @@ -0,0 +1,16 @@ +add_mlir_conversion_library(MLIRArithToEmitC + ArithToEmitC.cpp + ArithToEmitCPass.cpp + + ADDITIONAL_HEADER_DIRS + ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/ArithToEmitC + + DEPENDS + MLIRConversionPassIncGen + + LINK_LIBS PUBLIC + MLIRArithDialect + MLIREmitCDialect + MLIRPass + MLIRTransformUtils + ) diff --git a/mlir/lib/Conversion/CMakeLists.txt b/mlir/lib/Conversion/CMakeLists.txt index 9e421f7c49db..8219cf98575f 100644 --- a/mlir/lib/Conversion/CMakeLists.txt +++ b/mlir/lib/Conversion/CMakeLists.txt @@ -3,6 +3,7 @@ add_subdirectory(AMDGPUToROCDL) add_subdirectory(ArithCommon) add_subdirectory(ArithToAMDGPU) add_subdirectory(ArithToArmSME) +add_subdirectory(ArithToEmitC) add_subdirectory(ArithToLLVM) add_subdirectory(ArithToSPIRV) add_subdirectory(ArmNeon2dToIntr) diff --git a/mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir b/mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir new file mode 100644 index 000000000000..6a56474a5c48 --- /dev/null +++ b/mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir @@ -0,0 +1,14 @@ +// RUN: mlir-opt -convert-arith-to-emitc %s | FileCheck %s + +func.func @arith_ops(%arg0: f32, %arg1: f32) { + // CHECK: [[V0:[^ ]*]] = emitc.add %arg0, %arg1 : (f32, f32) -> f32 + %0 = arith.addf %arg0, %arg1 : f32 + // CHECK: [[V1:[^ ]*]] = emitc.div %arg0, %arg1 : (f32, f32) -> f32 + %1 = arith.divf %arg0, %arg1 : f32 + // CHECK: [[V2:[^ ]*]] = emitc.mul %arg0, %arg1 : (f32, f32) -> f32 + %2 = arith.mulf %arg0, %arg1 : f32 + // CHECK: [[V3:[^ ]*]] = emitc.sub %arg0, %arg1 : (f32, f32) -> f32 + %3 = arith.subf %arg0, %arg1 : f32 + + return +} diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 86b38ebd2217..9d6ca4ed932f 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -4014,6 +4014,7 @@ cc_library( ":AffineToStandard", ":ArithToAMDGPU", ":ArithToArmSME", + ":ArithToEmitC", ":ArithToLLVM", ":ArithToSPIRV", ":ArmNeon2dToIntr", @@ -8162,6 +8163,32 @@ cc_library( ], ) +cc_library( + name = "ArithToEmitC", + srcs = glob([ + "lib/Conversion/ArithToEmitC/*.cpp", + "lib/Conversion/ArithToEmitC/*.h", + ]), + hdrs = glob([ + "include/mlir/Conversion/ArithToEmitC/*.h", + ]), + includes = [ + "include", + "lib/Conversion/ArithToEmitC", + ], + deps = [ + ":ArithDialect", + ":ConversionPassIncGen", + ":EmitCDialect", + ":IR", + ":Pass", + ":Support", + ":TransformUtils", + ":Transforms", + "//llvm:Support", + ], +) + cc_library( name = "ArithToLLVM", srcs = glob(["lib/Conversion/ArithToLLVM/*.cpp"]), -- GitLab From 245d669f1d1c3f66d0d3d8aa7cffa5ef0d7747ff Mon Sep 17 00:00:00 2001 From: Guillaume Chatelet Date: Thu, 7 Mar 2024 11:41:35 +0100 Subject: [PATCH 426/929] [reland][libc] Remove UB specializations of type traits for `BigInt` (#84299) Note: This is a reland of #84035. The standard specifies that it it UB to specialize the following traits: - `std::is_integral` - `std::is_unsigned` - `std::make_unsigned` - `std::make_signed` This patch: - Removes specializations for `BigInt` - Transforms SFINAE for `bit.h` functions from template parameter to return type (This makes specialization easier). - Adds `BigInt` specialization for `bit.h` functions. - Fixes code depending on previous specializations. --- libc/src/__support/CMakeLists.txt | 1 + libc/src/__support/CPP/bit.h | 112 ++++++---- libc/src/__support/UInt.h | 192 ++++++++++++------ libc/src/__support/float_to_string.h | 2 +- libc/src/__support/integer_to_string.h | 19 +- libc/test/UnitTest/CMakeLists.txt | 1 + libc/test/UnitTest/LibcTest.cpp | 10 +- libc/test/UnitTest/LibcTest.h | 1 + libc/test/UnitTest/TestLogger.cpp | 8 +- libc/test/src/__support/CPP/bit_test.cpp | 47 +++-- .../test/src/__support/FPUtil/fpbits_test.cpp | 2 + .../llvm-project-overlay/libc/BUILD.bazel | 1 + .../libc/test/UnitTest/BUILD.bazel | 1 + 13 files changed, 272 insertions(+), 125 deletions(-) diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt index 1a4b3e9a2145..17c04aa57e6f 100644 --- a/libc/src/__support/CMakeLists.txt +++ b/libc/src/__support/CMakeLists.txt @@ -95,6 +95,7 @@ add_header_library( HDRS integer_to_string.h DEPENDS + .uint libc.src.__support.common libc.src.__support.CPP.algorithm libc.src.__support.CPP.limits diff --git a/libc/src/__support/CPP/bit.h b/libc/src/__support/CPP/bit.h index 7d11e7d5c497..bc2f595845a9 100644 --- a/libc/src/__support/CPP/bit.h +++ b/libc/src/__support/CPP/bit.h @@ -27,13 +27,14 @@ namespace LIBC_NAMESPACE::cpp { // This implementation of bit_cast requires trivially-constructible To, to avoid // UB in the implementation. -template < - typename To, typename From, - typename = cpp::enable_if_t::value && - cpp::is_trivially_copyable::value && - cpp::is_trivially_copyable::value>> -LIBC_INLINE constexpr To bit_cast(const From &from) { +template +LIBC_INLINE constexpr cpp::enable_if_t< + (sizeof(To) == sizeof(From)) && + cpp::is_trivially_constructible::value && + cpp::is_trivially_copyable::value && + cpp::is_trivially_copyable::value, + To> +bit_cast(const From &from) { MSAN_UNPOISON(&from, sizeof(From)); #if LIBC_HAS_BUILTIN(__builtin_bit_cast) return __builtin_bit_cast(To, from); @@ -51,8 +52,10 @@ LIBC_INLINE constexpr To bit_cast(const From &from) { #endif // LIBC_HAS_BUILTIN(__builtin_bit_cast) } -template >> -[[nodiscard]] LIBC_INLINE constexpr bool has_single_bit(T value) { +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, + bool> +has_single_bit(T value) { return (value != 0) && ((value & (value - 1)) == 0); } @@ -70,8 +73,9 @@ template >> /// Only unsigned integral types are allowed. /// /// Returns cpp::numeric_limits::digits on an input of 0. -template >> -[[nodiscard]] LIBC_INLINE constexpr int countr_zero(T value) { +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +countr_zero(T value) { if (!value) return cpp::numeric_limits::digits; if (value & 0x1) @@ -103,8 +107,9 @@ ADD_SPECIALIZATION(countr_zero, unsigned long long, __builtin_ctzll) /// Only unsigned integral types are allowed. /// /// Returns cpp::numeric_limits::digits on an input of 0. -template >> -[[nodiscard]] LIBC_INLINE constexpr int countl_zero(T value) { +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +countl_zero(T value) { if (!value) return cpp::numeric_limits::digits; // Bisection method. @@ -135,8 +140,9 @@ ADD_SPECIALIZATION(countl_zero, unsigned long long, __builtin_clzll) /// Only unsigned integral types are allowed. /// /// Returns cpp::numeric_limits::digits on an input of all ones. -template >> -[[nodiscard]] LIBC_INLINE constexpr int countl_one(T value) { +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +countl_one(T value) { return cpp::countl_zero(~value); } @@ -147,8 +153,9 @@ template >> /// Only unsigned integral types are allowed. /// /// Returns cpp::numeric_limits::digits on an input of all ones. -template >> -[[nodiscard]] LIBC_INLINE constexpr int countr_one(T value) { +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +countr_one(T value) { return cpp::countr_zero(~value); } @@ -156,8 +163,9 @@ template >> /// Returns 0 otherwise. /// /// Ex. bit_width(5) == 3. -template >> -[[nodiscard]] LIBC_INLINE constexpr int bit_width(T value) { +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +bit_width(T value) { return cpp::numeric_limits::digits - cpp::countl_zero(value); } @@ -165,8 +173,9 @@ template >> /// nonzero. Returns 0 otherwise. /// /// Ex. bit_floor(5) == 4. -template >> -[[nodiscard]] LIBC_INLINE constexpr T bit_floor(T value) { +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> +bit_floor(T value) { if (!value) return 0; return T(1) << (cpp::bit_width(value) - 1); @@ -179,8 +188,9 @@ template >> /// /// The return value is undefined if the input is larger than the largest power /// of two representable in T. -template >> -[[nodiscard]] LIBC_INLINE constexpr T bit_ceil(T value) { +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> +bit_ceil(T value) { if (value < 2) return 1; return T(1) << cpp::bit_width(value - 1u); @@ -190,28 +200,31 @@ template >> // from https://blog.regehr.org/archives/1063. // Forward-declare rotr so that rotl can use it. -template >> -[[nodiscard]] LIBC_INLINE constexpr T rotr(T value, int rotate); +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> +rotr(T value, int rotate); -template >> -[[nodiscard]] LIBC_INLINE constexpr T rotl(T value, int rotate) { +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> +rotl(T value, int rotate) { constexpr unsigned N = cpp::numeric_limits::digits; rotate = rotate % N; if (!rotate) return value; if (rotate < 0) - return cpp::rotr(value, -rotate); + return cpp::rotr(value, -rotate); return (value << rotate) | (value >> (N - rotate)); } -template -[[nodiscard]] LIBC_INLINE constexpr T rotr(T value, int rotate) { +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> +rotr(T value, int rotate) { constexpr unsigned N = cpp::numeric_limits::digits; rotate = rotate % N; if (!rotate) return value; if (rotate < 0) - return cpp::rotl(value, -rotate); + return cpp::rotl(value, -rotate); return (value >> rotate) | (value << (N - rotate)); } @@ -226,33 +239,44 @@ LIBC_INLINE constexpr To bit_or_static_cast(const From &from) { } } -template >> -[[nodiscard]] LIBC_INLINE constexpr int first_leading_zero(T value) { +// TODO: remove from 'bit.h' as it is not a standard function. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +first_leading_zero(T value) { return value == cpp::numeric_limits::max() ? 0 : countl_one(value) + 1; } -template >> -[[nodiscard]] LIBC_INLINE constexpr int first_leading_one(T value) { +// TODO: remove from 'bit.h' as it is not a standard function. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +first_leading_one(T value) { return first_leading_zero(static_cast(~value)); } -template >> -[[nodiscard]] LIBC_INLINE constexpr int first_trailing_zero(T value) { +// TODO: remove from 'bit.h' as it is not a standard function. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +first_trailing_zero(T value) { return value == cpp::numeric_limits::max() ? 0 : countr_zero(static_cast(~value)) + 1; } -template >> -[[nodiscard]] LIBC_INLINE constexpr int first_trailing_one(T value) { +// TODO: remove from 'bit.h' as it is not a standard function. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +first_trailing_one(T value) { return value == cpp::numeric_limits::max() ? 0 : countr_zero(value) + 1; } /// Count number of 1's aka population count or hamming weight. /// /// Only unsigned integral types are allowed. -template >> -[[nodiscard]] LIBC_INLINE constexpr int count_ones(T value) { +// TODO: rename as 'popcount' to follow the standard +// https://en.cppreference.com/w/cpp/numeric/popcount +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +count_ones(T value) { int count = 0; for (int i = 0; i != cpp::numeric_limits::digits; ++i) if ((value >> i) & 0x1) @@ -272,8 +296,10 @@ ADD_SPECIALIZATION(unsigned long long, __builtin_popcountll) // TODO: 128b specializations? #undef ADD_SPECIALIZATION -template >> -[[nodiscard]] LIBC_INLINE constexpr int count_zeros(T value) { +// TODO: remove from 'bit.h' as it is not a standard function. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +count_zeros(T value) { return count_ones(static_cast(~value)); } diff --git a/libc/src/__support/UInt.h b/libc/src/__support/UInt.h index 5973e6fab1d7..b3d8f00b9a01 100644 --- a/libc/src/__support/UInt.h +++ b/libc/src/__support/UInt.h @@ -43,6 +43,9 @@ struct BigInt { static_assert(is_integral_v && is_unsigned_v, "WordType must be unsigned integer."); + using word_type = WordType; + LIBC_INLINE_VAR static constexpr bool SIGNED = Signed; + LIBC_INLINE_VAR static constexpr size_t BITS = Bits; LIBC_INLINE_VAR static constexpr size_t WORD_SIZE = sizeof(WordType) * CHAR_BIT; @@ -50,6 +53,10 @@ struct BigInt { "Number of bits in BigInt should be a multiple of WORD_SIZE."); LIBC_INLINE_VAR static constexpr size_t WORD_COUNT = Bits / WORD_SIZE; + + using unsigned_type = BigInt; + using signed_type = BigInt; + cpp::array val{}; LIBC_INLINE constexpr BigInt() = default; @@ -579,19 +586,33 @@ struct BigInt { return *this; } - LIBC_INLINE constexpr uint64_t clz() { - uint64_t leading_zeroes = 0; - for (size_t i = WORD_COUNT; i > 0; --i) { - if (val[i - 1] == 0) { - leading_zeroes += WORD_SIZE; - } else { - leading_zeroes += countl_zero(val[i - 1]); + // TODO: remove and use cpp::countl_zero below. + [[nodiscard]] LIBC_INLINE constexpr int clz() const { + constexpr int word_digits = cpp::numeric_limits::digits; + int leading_zeroes = 0; + for (auto i = val.size(); i > 0;) { + --i; + const int zeroes = countl_zero(val[i]); + leading_zeroes += zeroes; + if (zeroes != word_digits) break; - } } return leading_zeroes; } + // TODO: remove and use cpp::countr_zero below. + [[nodiscard]] LIBC_INLINE constexpr int ctz() const { + constexpr int word_digits = cpp::numeric_limits::digits; + int trailing_zeroes = 0; + for (auto word : val) { + const int zeroes = countr_zero(word); + trailing_zeroes += zeroes; + if (zeroes != word_digits) + break; + } + return trailing_zeroes; + } + LIBC_INLINE constexpr void shift_left(size_t s) { if constexpr (Bits == WORD_SIZE) { // Use native types if possible. @@ -916,66 +937,123 @@ public: LIBC_INLINE_VAR static constexpr int digits = 128; }; -// Provides is_integral of U/Int<128>, U/Int<192>, U/Int<256>. -template -struct is_integral> : cpp::true_type {}; +// type traits to determine whether a T is a cpp::BigInt. +template struct is_big_int : cpp::false_type {}; -// Provides is_unsigned of UInt<128>, UInt<192>, UInt<256>. template -struct is_unsigned> : cpp::bool_constant {}; - -template -struct make_unsigned> - : type_identity> {}; - -template -struct make_signed> - : type_identity> {}; - -namespace internal { -template struct is_custom_uint : cpp::false_type {}; - -template -struct is_custom_uint> : cpp::true_type {}; -} // namespace internal - -// bit_cast to UInt -// Note: The standard scheme for SFINAE selection is to have exactly one -// function instanciation valid at a time. This is usually done by having a -// predicate in one function and the negated predicate in the other one. -// e.g. -// template::value == true> ... -// template::value == false> ... -// -// Unfortunately this would make the default 'cpp::bit_cast' aware of -// 'is_custom_uint' (or any other customization). To prevent exposing all -// customizations in the original function, we create a different function with -// four 'typename's instead of three - otherwise it would be considered as a -// redeclaration of the same function leading to "error: template parameter -// redefines default argument". -template ::value && - cpp::is_trivially_copyable::value>, - typename = cpp::enable_if_t::value>> -LIBC_INLINE constexpr To bit_cast(const From &from) { +struct is_big_int> : cpp::true_type {}; + +template +LIBC_INLINE_VAR constexpr bool is_big_int_v = is_big_int::value; + +// Specialization of cpp::bit_cast ('bit.h') from T to BigInt. +template +LIBC_INLINE constexpr cpp::enable_if_t< + (sizeof(To) == sizeof(From)) && cpp::is_trivially_copyable::value && + cpp::is_trivially_copyable::value && is_big_int::value, + To> +bit_cast(const From &from) { To out; using Storage = decltype(out.val); out.val = cpp::bit_cast(from); return out; } -// bit_cast from UInt -template < - typename To, size_t Bits, - typename = cpp::enable_if_t) && - cpp::is_trivially_constructible::value && - cpp::is_trivially_copyable::value && - cpp::is_trivially_copyable>::value>> -LIBC_INLINE constexpr To bit_cast(const UInt &from) { +// Specialization of cpp::bit_cast ('bit.h') from BigInt to T. +template +LIBC_INLINE constexpr cpp::enable_if_t< + sizeof(To) == sizeof(UInt) && + cpp::is_trivially_constructible::value && + cpp::is_trivially_copyable::value && + cpp::is_trivially_copyable>::value, + To> +bit_cast(const UInt &from) { return cpp::bit_cast(from.val); } +// Specialization of cpp::has_single_bit ('bit.h') for BigInt. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, bool> +has_single_bit(T value) { + int bits = 0; + for (auto word : value.val) { + if (word == 0) + continue; + bits += count_ones(word); + if (bits > 1) + return false; + } + return bits == 1; +} + +// Specialization of cpp::countr_zero ('bit.h') for BigInt. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +countr_zero(const T &value) { + return value.ctz(); +} + +// Specialization of cpp::countl_zero ('bit.h') for BigInt. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +countl_zero(const T &value) { + return value.clz(); +} + +// Specialization of cpp::countl_one ('bit.h') for BigInt. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +countl_one(T value) { + // TODO : Implement a faster version not involving operator~. + return cpp::countl_zero(~value); +} + +// Specialization of cpp::countr_one ('bit.h') for BigInt. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +countr_one(T value) { + // TODO : Implement a faster version not involving operator~. + return cpp::countr_zero(~value); +} + +// Specialization of cpp::bit_width ('bit.h') for BigInt. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +bit_width(T value) { + return cpp::numeric_limits::digits - cpp::countl_zero(value); +} + +// Forward-declare rotr so that rotl can use it. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> +rotr(T value, int rotate); + +// Specialization of cpp::rotl ('bit.h') for BigInt. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> +rotl(T value, int rotate) { + constexpr unsigned N = cpp::numeric_limits::digits; + rotate = rotate % N; + if (!rotate) + return value; + if (rotate < 0) + return cpp::rotr(value, -rotate); + return (value << rotate) | (value >> (N - rotate)); +} + +// Specialization of cpp::rotr ('bit.h') for BigInt. +template +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> +rotr(T value, int rotate) { + constexpr unsigned N = cpp::numeric_limits::digits; + rotate = rotate % N; + if (!rotate) + return value; + if (rotate < 0) + return cpp::rotl(value, -rotate); + return (value >> rotate) | (value << (N - rotate)); +} + } // namespace LIBC_NAMESPACE::cpp #endif // LLVM_LIBC_SRC___SUPPORT_UINT_H diff --git a/libc/src/__support/float_to_string.h b/libc/src/__support/float_to_string.h index 744842ced8d7..27476433a945 100644 --- a/libc/src/__support/float_to_string.h +++ b/libc/src/__support/float_to_string.h @@ -713,7 +713,7 @@ template <> class FloatToString { float_as_fixed.shift_left(SHIFT_AMOUNT); // If there are still digits above the decimal point, handle those. - if (float_as_fixed.clz() < EXTRA_INT_WIDTH) { + if (float_as_fixed.clz() < static_cast(EXTRA_INT_WIDTH)) { cpp::UInt above_decimal_point = float_as_fixed >> FLOAT_AS_INT_WIDTH; diff --git a/libc/src/__support/integer_to_string.h b/libc/src/__support/integer_to_string.h index 81ed21ccfca1..a5872dce6520 100644 --- a/libc/src/__support/integer_to_string.h +++ b/libc/src/__support/integer_to_string.h @@ -67,6 +67,7 @@ #include "src/__support/CPP/span.h" #include "src/__support/CPP/string_view.h" #include "src/__support/CPP/type_traits.h" +#include "src/__support/UInt.h" // is_big_int #include "src/__support/common.h" namespace LIBC_NAMESPACE { @@ -149,6 +150,18 @@ public: using StringBufferWriter = StringBufferWriterImpl; using BackwardStringBufferWriter = StringBufferWriterImpl; +template struct IntegerWriterUnsigned {}; + +template +struct IntegerWriterUnsigned>> { + using type = cpp::make_unsigned_t; +}; + +template +struct IntegerWriterUnsigned>> { + using type = typename T::unsigned_type; +}; + } // namespace details namespace radix { @@ -163,7 +176,7 @@ template using Custom = details::Fmt; // See file header for documentation. template class IntegerToString { - static_assert(cpp::is_integral_v); + static_assert(cpp::is_integral_v || cpp::is_big_int_v); LIBC_INLINE static constexpr size_t compute_buffer_size() { constexpr auto MAX_DIGITS = []() -> size_t { @@ -208,8 +221,8 @@ template class IntegerToString { // An internal stateless structure that handles the number formatting logic. struct IntegerWriter { - static_assert(cpp::is_integral_v); - using UNSIGNED_T = cpp::make_unsigned_t; + static_assert(cpp::is_integral_v || cpp::is_big_int_v); + using UNSIGNED_T = typename details::IntegerWriterUnsigned::type; LIBC_INLINE static char digit_char(uint8_t digit) { if (digit < 10) diff --git a/libc/test/UnitTest/CMakeLists.txt b/libc/test/UnitTest/CMakeLists.txt index 4668f0061975..36837c553efc 100644 --- a/libc/test/UnitTest/CMakeLists.txt +++ b/libc/test/UnitTest/CMakeLists.txt @@ -74,6 +74,7 @@ add_unittest_framework_library( libc.src.__support.CPP.type_traits libc.src.__support.fixed_point.fx_rep libc.src.__support.OSUtil.osutil + libc.src.__support.uint libc.src.__support.uint128 ) diff --git a/libc/test/UnitTest/LibcTest.cpp b/libc/test/UnitTest/LibcTest.cpp index 7b0e4fca8368..0340f7ed3710 100644 --- a/libc/test/UnitTest/LibcTest.cpp +++ b/libc/test/UnitTest/LibcTest.cpp @@ -38,7 +38,8 @@ TestLogger &operator<<(TestLogger &logger, Location Loc) { // When the value is UInt128, __uint128_t or wider, show its hexadecimal // digits. template -cpp::enable_if_t && (sizeof(T) > sizeof(uint64_t)), +cpp::enable_if_t<(cpp::is_integral_v && (sizeof(T) > sizeof(uint64_t))) || + cpp::is_big_int_v, cpp::string> describeValue(T Value) { static_assert(sizeof(T) % 8 == 0, "Unsupported size of UInt"); @@ -47,11 +48,10 @@ describeValue(T Value) { } // When the value is of a standard integral type, just display it as normal. -template -cpp::enable_if_t && - sizeof(ValType) <= sizeof(uint64_t), +template +cpp::enable_if_t && (sizeof(T) <= sizeof(uint64_t)), cpp::string> -describeValue(ValType Value) { +describeValue(T Value) { return cpp::to_string(Value); } diff --git a/libc/test/UnitTest/LibcTest.h b/libc/test/UnitTest/LibcTest.h index 639f60058325..d26d6490bcb5 100644 --- a/libc/test/UnitTest/LibcTest.h +++ b/libc/test/UnitTest/LibcTest.h @@ -127,6 +127,7 @@ protected: // of type promotion. template || + cpp::is_big_int_v || cpp::is_fixed_point_v, int> = 0> bool test(TestCond Cond, ValType LHS, ValType RHS, const char *LHSStr, diff --git a/libc/test/UnitTest/TestLogger.cpp b/libc/test/UnitTest/TestLogger.cpp index 6bb0e17dc388..469b3a11d57d 100644 --- a/libc/test/UnitTest/TestLogger.cpp +++ b/libc/test/UnitTest/TestLogger.cpp @@ -2,6 +2,7 @@ #include "src/__support/CPP/string.h" #include "src/__support/CPP/string_view.h" #include "src/__support/OSUtil/io.h" // write_to_stderr +#include "src/__support/UInt.h" // is_big_int #include "src/__support/UInt128.h" #include @@ -47,8 +48,9 @@ template <> TestLogger &TestLogger::operator<<(void *addr) { } template TestLogger &TestLogger::operator<<(T t) { - if constexpr (cpp::is_integral_v && cpp::is_unsigned_v && - sizeof(T) > sizeof(uint64_t)) { + if constexpr (cpp::is_big_int_v || + (cpp::is_integral_v && cpp::is_unsigned_v && + (sizeof(T) > sizeof(uint64_t)))) { static_assert(sizeof(T) % 8 == 0, "Unsupported size of UInt"); const IntegerToString buffer(t); return *this << buffer.view(); @@ -68,7 +70,7 @@ template TestLogger &TestLogger::operator<< (unsigned short); template TestLogger &TestLogger::operator<< (unsigned int); template TestLogger &TestLogger::operator<< (unsigned long); template TestLogger & -TestLogger::operator<< (unsigned long long); + TestLogger::operator<< (unsigned long long); #ifdef __SIZEOF_INT128__ template TestLogger &TestLogger::operator<< <__uint128_t>(__uint128_t); diff --git a/libc/test/src/__support/CPP/bit_test.cpp b/libc/test/src/__support/CPP/bit_test.cpp index 115a5d505c4b..25a80ca9209c 100644 --- a/libc/test/src/__support/CPP/bit_test.cpp +++ b/libc/test/src/__support/CPP/bit_test.cpp @@ -14,19 +14,40 @@ namespace LIBC_NAMESPACE::cpp { -using UnsignedTypes = - testing::TypeList>; + unsigned char, unsigned short, unsigned int, unsigned long, + unsigned long long>; + +using UnsignedTypes = testing::TypeList< +#if defined(__SIZEOF_INT128__) + __uint128_t, +#endif + unsigned char, unsigned short, unsigned int, unsigned long, + unsigned long long, cpp::UInt<128>>; TYPED_TEST(LlvmLibcBitTest, HasSingleBit, UnsignedTypes) { - EXPECT_FALSE(has_single_bit(T(0))); - EXPECT_FALSE(has_single_bit(~T(0))); + constexpr auto ZERO = T(0); + constexpr auto ALL_ONES = T(~ZERO); + EXPECT_FALSE(has_single_bit(ZERO)); + EXPECT_FALSE(has_single_bit(ALL_ONES)); + for (T value = 1; value; value <<= 1) EXPECT_TRUE(has_single_bit(value)); + + // We test that if two bits are set has_single_bit returns false. + // We do this by setting the highest or lowest bit depending or where the + // current bit is. This is a bit convoluted but it helps catch a bug on BigInt + // where we have to work on an element-by-element basis. + constexpr auto MIDPOINT = T(ALL_ONES / 2); + constexpr auto LSB = T(1); + constexpr auto MSB = T(~(ALL_ONES >> 1)); + for (T value = 1; value; value <<= 1) { + auto two_bits_value = value | ((value <= MIDPOINT) ? MSB : LSB); + EXPECT_FALSE(has_single_bit(two_bits_value)); + } } TYPED_TEST(LlvmLibcBitTest, CountLZero, UnsignedTypes) { @@ -206,39 +227,39 @@ TEST(LlvmLibcBitTest, Rotr) { rotr(0x12345678deadbeefULL, -19)); } -TYPED_TEST(LlvmLibcBitTest, FirstLeadingZero, UnsignedTypes) { +TYPED_TEST(LlvmLibcBitTest, FirstLeadingZero, UnsignedTypesNoBigInt) { EXPECT_EQ(first_leading_zero(cpp::numeric_limits::max()), 0); for (int i = 0U; i != cpp::numeric_limits::digits; ++i) EXPECT_EQ(first_leading_zero(~(T(1) << i)), cpp::numeric_limits::digits - i); } -TYPED_TEST(LlvmLibcBitTest, FirstLeadingOne, UnsignedTypes) { +TYPED_TEST(LlvmLibcBitTest, FirstLeadingOne, UnsignedTypesNoBigInt) { EXPECT_EQ(first_leading_one(static_cast(0)), 0); for (int i = 0U; i != cpp::numeric_limits::digits; ++i) EXPECT_EQ(first_leading_one(T(1) << i), cpp::numeric_limits::digits - i); } -TYPED_TEST(LlvmLibcBitTest, FirstTrailingZero, UnsignedTypes) { +TYPED_TEST(LlvmLibcBitTest, FirstTrailingZero, UnsignedTypesNoBigInt) { EXPECT_EQ(first_trailing_zero(cpp::numeric_limits::max()), 0); for (int i = 0U; i != cpp::numeric_limits::digits; ++i) EXPECT_EQ(first_trailing_zero(~(T(1) << i)), i + 1); } -TYPED_TEST(LlvmLibcBitTest, FirstTrailingOne, UnsignedTypes) { +TYPED_TEST(LlvmLibcBitTest, FirstTrailingOne, UnsignedTypesNoBigInt) { EXPECT_EQ(first_trailing_one(cpp::numeric_limits::max()), 0); for (int i = 0U; i != cpp::numeric_limits::digits; ++i) EXPECT_EQ(first_trailing_one(T(1) << i), i + 1); } -TYPED_TEST(LlvmLibcBitTest, CountZeros, UnsignedTypes) { +TYPED_TEST(LlvmLibcBitTest, CountZeros, UnsignedTypesNoBigInt) { EXPECT_EQ(count_zeros(T(0)), cpp::numeric_limits::digits); for (int i = 0; i != cpp::numeric_limits::digits; ++i) EXPECT_EQ(count_zeros(cpp::numeric_limits::max() >> i), i); } -TYPED_TEST(LlvmLibcBitTest, CountOnes, UnsignedTypes) { +TYPED_TEST(LlvmLibcBitTest, CountOnes, UnsignedTypesNoBigInt) { EXPECT_EQ(count_ones(T(0)), 0); for (int i = 0; i != cpp::numeric_limits::digits; ++i) EXPECT_EQ(count_ones(cpp::numeric_limits::max() >> i), diff --git a/libc/test/src/__support/FPUtil/fpbits_test.cpp b/libc/test/src/__support/FPUtil/fpbits_test.cpp index f5c27d4fc030..760031569c81 100644 --- a/libc/test/src/__support/FPUtil/fpbits_test.cpp +++ b/libc/test/src/__support/FPUtil/fpbits_test.cpp @@ -237,6 +237,8 @@ template constexpr auto make(Sign sign, FP fp) { return T::signaling_nan(sign); case FP::QUIET_NAN: return T::quiet_nan(sign); + default: + __builtin_unreachable(); } } diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index 49a454379e1c..5c6cf761ebe7 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -507,6 +507,7 @@ libc_support_library( ":__support_cpp_span", ":__support_cpp_string_view", ":__support_cpp_type_traits", + ":__support_uint", ], ) diff --git a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel index a5c18fbb68b3..44692947af7c 100644 --- a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel @@ -18,6 +18,7 @@ libc_support_library( "//libc:__support_cpp_string", "//libc:__support_cpp_string_view", "//libc:__support_osutil_io", + "//libc:__support_uint", "//libc:__support_uint128", ], ) -- GitLab From 55304d0d907fb26c298b84447a85e3a987d0adbc Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 7 Mar 2024 10:46:27 +0000 Subject: [PATCH 427/929] [CostModel] getInstructionCost - improve estimation of costs for length changing shuffles (#84156) Fix gap in the cost estimation for length changing shuffles, by adjusting the shuffle mask and either widening the shuffle inputs or extracting the lower elements of the result. A small step towards moving some of this implementation inside improveShuffleKindFromMask and/or target getShuffleCost handlers (and reduce the diffs in cost estimation depending on whether coming from a ShuffleVectorInst or the raw operands / mask components) --- .../llvm/Analysis/TargetTransformInfoImpl.h | 31 +- .../CostModel/AMDGPU/shufflevector.ll | 368 +++++++++--------- .../RISCV/shuffle-extract_subvector.ll | 4 +- .../CostModel/RISCV/shuffle-interleave.ll | 4 +- .../X86/shuffle-concat_subvector-codesize.ll | 170 +++++--- .../X86/shuffle-concat_subvector-latency.ll | 178 +++++---- .../shuffle-concat_subvector-sizelatency.ll | 170 +++++--- .../CostModel/X86/shuffle-concat_subvector.ll | 178 +++++---- .../X86/shuffle-extract_subvector-codesize.ll | 39 +- .../X86/shuffle-extract_subvector-latency.ll | 41 +- .../shuffle-extract_subvector-sizelatency.ll | 39 +- .../X86/shuffle-extract_subvector.ll | 41 +- 12 files changed, 756 insertions(+), 507 deletions(-) diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h index 71573b6b3b8f..095c2ff1e58b 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -1327,6 +1327,7 @@ public: ArrayRef Mask = Shuffle->getShuffleMask(); int NumSubElts, SubIndex; + // TODO: move more of this inside improveShuffleKindFromMask. if (Shuffle->changesLength()) { // Treat a 'subvector widening' as a free shuffle. if (Shuffle->increasesLength() && Shuffle->isIdentityWithPadding()) @@ -1355,7 +1356,35 @@ public: DemandedDstElts, CostKind); } - return CostKind == TTI::TCK_RecipThroughput ? -1 : 1; + bool IsUnary = isa(Operands[1]); + NumSubElts = VecSrcTy->getElementCount().getKnownMinValue(); + SmallVector AdjustMask(Mask.begin(), Mask.end()); + + // Widening shuffle - widening the source(s) to the new length + // (treated as free - see above), and then perform the adjusted + // shuffle at that width. + if (Shuffle->increasesLength()) { + for (int &M : AdjustMask) + M = M >= NumSubElts ? (M + (Mask.size() - NumSubElts)) : M; + + return TargetTTI->getShuffleCost( + IsUnary ? TTI::SK_PermuteSingleSrc : TTI::SK_PermuteTwoSrc, VecTy, + AdjustMask, CostKind, 0, nullptr); + } + + // Narrowing shuffle - perform shuffle at original wider width and + // then extract the lower elements. + AdjustMask.append(NumSubElts - Mask.size(), PoisonMaskElem); + + InstructionCost ShuffleCost = TargetTTI->getShuffleCost( + IsUnary ? TTI::SK_PermuteSingleSrc : TTI::SK_PermuteTwoSrc, + VecSrcTy, AdjustMask, CostKind, 0, nullptr); + + SmallVector ExtractMask(Mask.size()); + std::iota(ExtractMask.begin(), ExtractMask.end(), 0); + return ShuffleCost + TargetTTI->getShuffleCost( + TTI::SK_ExtractSubvector, VecTy, ExtractMask, + CostKind, 0, VecSrcTy, Operands); } if (Shuffle->isIdentity()) diff --git a/llvm/test/Analysis/CostModel/AMDGPU/shufflevector.ll b/llvm/test/Analysis/CostModel/AMDGPU/shufflevector.ll index 58f9dd3633e2..be5cca0765ed 100644 --- a/llvm/test/Analysis/CostModel/AMDGPU/shufflevector.ll +++ b/llvm/test/Analysis/CostModel/AMDGPU/shufflevector.ll @@ -25,27 +25,27 @@ define amdgpu_kernel void @shufflevector_i16() { ; GFX9-10-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shuf31 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> ; GFX9-10-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf23 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> ; GFX9-10-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf32 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf000 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> zeroinitializer -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf001 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf010 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf011 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf100 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf101 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf110 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf111 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf002 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf020 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf022 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf200 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf202 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf220 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf222 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf112 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf121 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf122 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf211 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf212 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf221 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shuf000 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> zeroinitializer +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf001 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf010 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf011 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf100 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf101 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf110 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf111 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf002 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf020 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf022 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf200 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf202 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf220 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shuf222 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf112 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf121 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf122 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf211 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf212 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf221 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> ; GFX9-10-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; ; VI-LABEL: 'shufflevector_i16' @@ -65,27 +65,27 @@ define amdgpu_kernel void @shufflevector_i16() { ; VI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shuf31 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> ; VI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf23 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> ; VI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shuf32 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf000 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> zeroinitializer -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf001 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf010 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf011 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf100 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf101 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf110 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf111 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf002 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf020 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf022 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf200 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf202 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf220 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf222 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf112 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf121 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf122 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf211 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf212 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf221 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shuf000 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> zeroinitializer +; VI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf001 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf010 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf011 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf100 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf101 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf110 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf111 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf002 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf020 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf022 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf200 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf202 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf220 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shuf222 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf112 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf121 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf122 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf211 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf212 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf221 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> ; VI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; ; GFX9-10-SIZE-LABEL: 'shufflevector_i16' @@ -105,27 +105,27 @@ define amdgpu_kernel void @shufflevector_i16() { ; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shuf31 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> ; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf23 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> ; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf32 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf000 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> zeroinitializer -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf001 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf010 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf011 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf100 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf101 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf110 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf111 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf002 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf020 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf022 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf200 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf202 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf220 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf222 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf112 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf121 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf122 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf211 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf212 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf221 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shuf000 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> zeroinitializer +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf001 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf010 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf011 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf100 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf101 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf110 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf111 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf002 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf020 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf022 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf200 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf202 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf220 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shuf222 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf112 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf121 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf122 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf211 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf212 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf221 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> ; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; VI-SIZE-LABEL: 'shufflevector_i16' @@ -145,27 +145,27 @@ define amdgpu_kernel void @shufflevector_i16() { ; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shuf31 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> ; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf23 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> ; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shuf32 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf000 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> zeroinitializer -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf001 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf010 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf011 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf100 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf101 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf110 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf111 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf002 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf020 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf022 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf200 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf202 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf220 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf222 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf112 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf121 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf122 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf211 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf212 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf221 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shuf000 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> zeroinitializer +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf001 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf010 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf011 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf100 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf101 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf110 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf111 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf002 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf020 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf022 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf200 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf202 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf220 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shuf222 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf112 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf121 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf122 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf211 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf212 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf221 = shufflevector <2 x i16> undef, <2 x i16> undef, <3 x i32> ; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %shuf00 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> zeroinitializer @@ -227,27 +227,27 @@ define amdgpu_kernel void @shufflevector_i8() { ; ALL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf31 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> ; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf23 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> ; ALL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf32 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf000 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> zeroinitializer -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf001 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf010 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf011 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf100 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf101 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf110 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf111 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf002 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf020 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf022 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf200 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf202 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf220 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf222 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf112 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf121 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf122 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf211 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf212 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf221 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf000 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> zeroinitializer +; ALL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf001 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf010 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf011 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf100 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf101 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf110 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf111 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf002 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf020 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf022 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf200 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf202 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf220 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf222 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf112 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf121 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf122 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf211 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf212 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf221 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> ; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; ; ALL-SIZE-LABEL: 'shufflevector_i8' @@ -267,27 +267,27 @@ define amdgpu_kernel void @shufflevector_i8() { ; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf31 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> ; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf23 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> ; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf32 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf000 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> zeroinitializer -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf001 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf010 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf011 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf100 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf101 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf110 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf111 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf002 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf020 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf022 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf200 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf202 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf220 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf222 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf112 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf121 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf122 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf211 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf212 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf221 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf000 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> zeroinitializer +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf001 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf010 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf011 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf100 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf101 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf110 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf111 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf002 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf020 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf022 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf200 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf202 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf220 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shuf222 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf112 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf121 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf122 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf211 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf212 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shuf221 = shufflevector <2 x i8> undef, <2 x i8> undef, <3 x i32> ; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %shuf00 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> zeroinitializer @@ -348,27 +348,27 @@ define amdgpu_kernel void @shufflevector_i32() { ; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf31 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> ; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf23 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> ; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf32 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf000 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> zeroinitializer -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf001 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf010 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf011 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf100 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf101 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf110 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf111 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf002 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf020 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf022 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf200 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf202 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf220 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf222 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf112 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf121 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf122 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf211 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf212 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %shuf221 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf000 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> zeroinitializer +; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf001 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf010 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf011 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf100 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf101 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf110 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf111 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf002 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf020 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf022 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf200 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf202 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf220 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf222 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf112 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf121 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf122 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf211 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf212 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf221 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> ; ALL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void ; ; ALL-SIZE-LABEL: 'shufflevector_i32' @@ -388,27 +388,27 @@ define amdgpu_kernel void @shufflevector_i32() { ; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf31 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> ; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf23 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> ; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf32 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf000 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> zeroinitializer -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf001 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf010 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf011 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf100 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf101 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf110 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf111 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf002 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf020 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf022 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf200 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf202 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf220 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf222 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf112 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf121 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf122 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf211 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf212 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> -; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shuf221 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf000 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> zeroinitializer +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf001 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf010 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf011 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf100 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf101 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf110 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf111 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf002 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf020 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf022 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf200 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf202 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf220 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf222 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf112 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf121 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf122 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf211 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf212 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> +; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %shuf221 = shufflevector <2 x i32> undef, <2 x i32> undef, <3 x i32> ; ALL-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %shuf00 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> zeroinitializer @@ -455,11 +455,11 @@ define amdgpu_kernel void @shufflevector_i32() { define void @shuffle() { ; GFX9-10-LABEL: 'shuffle' ; GFX9-10-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2i8_2 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %v2i8_4 = shufflevector <2 x i8> undef, <2 x i8> undef, <4 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_4 = shufflevector <2 x i8> undef, <2 x i8> undef, <4 x i32> ; GFX9-10-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_4 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %v2i8_8 = shufflevector <2 x i8> undef, <2 x i8> undef, <8 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %v4i8_8 = shufflevector <4 x i8> undef, <4 x i8> undef, <8 x i32> -; GFX9-10-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %v6i8_8 = shufflevector <6 x i8> undef, <6 x i8> undef, <8 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v2i8_8 = shufflevector <2 x i8> undef, <2 x i8> undef, <8 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v4i8_8 = shufflevector <4 x i8> undef, <4 x i8> undef, <8 x i32> +; GFX9-10-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v6i8_8 = shufflevector <6 x i8> undef, <6 x i8> undef, <8 x i32> ; GFX9-10-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v8i8_8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> ; GFX9-10-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v16i8_16 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> ; GFX9-10-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2i16_2 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> @@ -475,11 +475,11 @@ define void @shuffle() { ; ; VI-LABEL: 'shuffle' ; VI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2i8_2 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %v2i8_4 = shufflevector <2 x i8> undef, <2 x i8> undef, <4 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_4 = shufflevector <2 x i8> undef, <2 x i8> undef, <4 x i32> ; VI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_4 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %v2i8_8 = shufflevector <2 x i8> undef, <2 x i8> undef, <8 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %v4i8_8 = shufflevector <4 x i8> undef, <4 x i8> undef, <8 x i32> -; VI-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %v6i8_8 = shufflevector <6 x i8> undef, <6 x i8> undef, <8 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v2i8_8 = shufflevector <2 x i8> undef, <2 x i8> undef, <8 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v4i8_8 = shufflevector <4 x i8> undef, <4 x i8> undef, <8 x i32> +; VI-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v6i8_8 = shufflevector <6 x i8> undef, <6 x i8> undef, <8 x i32> ; VI-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v8i8_8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> ; VI-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v16i8_16 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> ; VI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i16_2 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> @@ -495,11 +495,11 @@ define void @shuffle() { ; ; GFX9-10-SIZE-LABEL: 'shuffle' ; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2i8_2 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i8_4 = shufflevector <2 x i8> undef, <2 x i8> undef, <4 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_4 = shufflevector <2 x i8> undef, <2 x i8> undef, <4 x i32> ; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_4 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i8_8 = shufflevector <2 x i8> undef, <2 x i8> undef, <8 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8_8 = shufflevector <4 x i8> undef, <4 x i8> undef, <8 x i32> -; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v6i8_8 = shufflevector <6 x i8> undef, <6 x i8> undef, <8 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v2i8_8 = shufflevector <2 x i8> undef, <2 x i8> undef, <8 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v4i8_8 = shufflevector <4 x i8> undef, <4 x i8> undef, <8 x i32> +; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v6i8_8 = shufflevector <6 x i8> undef, <6 x i8> undef, <8 x i32> ; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v8i8_8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> ; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v16i8_16 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> ; GFX9-10-SIZE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2i16_2 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> @@ -515,11 +515,11 @@ define void @shuffle() { ; ; VI-SIZE-LABEL: 'shuffle' ; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2i8_2 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i8_4 = shufflevector <2 x i8> undef, <2 x i8> undef, <4 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_4 = shufflevector <2 x i8> undef, <2 x i8> undef, <4 x i32> ; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_4 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i8_8 = shufflevector <2 x i8> undef, <2 x i8> undef, <8 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8_8 = shufflevector <4 x i8> undef, <4 x i8> undef, <8 x i32> -; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v6i8_8 = shufflevector <6 x i8> undef, <6 x i8> undef, <8 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v2i8_8 = shufflevector <2 x i8> undef, <2 x i8> undef, <8 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v4i8_8 = shufflevector <4 x i8> undef, <4 x i8> undef, <8 x i32> +; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v6i8_8 = shufflevector <6 x i8> undef, <6 x i8> undef, <8 x i32> ; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v8i8_8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> ; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v16i8_16 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> ; VI-SIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i16_2 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> diff --git a/llvm/test/Analysis/CostModel/RISCV/shuffle-extract_subvector.ll b/llvm/test/Analysis/CostModel/RISCV/shuffle-extract_subvector.ll index 3ac2b7e26650..b84f22907cc7 100644 --- a/llvm/test/Analysis/CostModel/RISCV/shuffle-extract_subvector.ll +++ b/llvm/test/Analysis/CostModel/RISCV/shuffle-extract_subvector.ll @@ -19,7 +19,7 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) { ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; CHECK-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; ; VLEN128-LABEL: 'test_vXf64' @@ -32,7 +32,7 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) { ; VLEN128-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; VLEN128-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; VLEN128-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; VLEN128-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; VLEN128-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; VLEN128-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> diff --git a/llvm/test/Analysis/CostModel/RISCV/shuffle-interleave.ll b/llvm/test/Analysis/CostModel/RISCV/shuffle-interleave.ll index afcf600e13ef..dd67772042cb 100644 --- a/llvm/test/Analysis/CostModel/RISCV/shuffle-interleave.ll +++ b/llvm/test/Analysis/CostModel/RISCV/shuffle-interleave.ll @@ -56,8 +56,8 @@ define <8 x i64> @interleave2_v8i64(<4 x i64> %v0, <4 x i64> %v1) { ; TODO: getInstructionCost doesn't call getShuffleCost here because the shuffle changes length define {<4 x i8>, <4 x i8>} @deinterleave_2(<8 x i8> %v) { ; CHECK-LABEL: 'deinterleave_2' -; CHECK-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %v0 = shufflevector <8 x i8> %v, <8 x i8> poison, <4 x i32> -; CHECK-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %v1 = shufflevector <8 x i8> %v, <8 x i8> poison, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v0 = shufflevector <8 x i8> %v, <8 x i8> poison, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v1 = shufflevector <8 x i8> %v, <8 x i8> poison, <4 x i32> ; CHECK-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %res0 = insertvalue { <4 x i8>, <4 x i8> } poison, <4 x i8> %v0, 0 ; CHECK-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %res1 = insertvalue { <4 x i8>, <4 x i8> } %res0, <4 x i8> %v1, 1 ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret { <4 x i8>, <4 x i8> } %res1 diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-codesize.ll b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-codesize.ll index 0c1c085f5afc..61d99c20fa96 100644 --- a/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-codesize.ll +++ b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-codesize.ll @@ -2,15 +2,15 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mattr=+sse2 | FileCheck %s -check-prefixes=SSE ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mattr=+ssse3 | FileCheck %s -check-prefixes=SSE ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mattr=+sse4.2 | FileCheck %s -check-prefixes=SSE -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mattr=+avx | FileCheck %s -check-prefixes=AVX -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mattr=+avx2 | FileCheck %s -check-prefixes=AVX -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512 -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512 -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mattr=+avx | FileCheck %s -check-prefixes=AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mattr=+avx2 | FileCheck %s -check-prefixes=AVX,AVX2 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512,AVX512VBMI ; ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=slm | FileCheck %s --check-prefixes=SSE ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=goldmont | FileCheck %s --check-prefixes=SSE -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=btver2 | FileCheck %s --check-prefixes=AVX +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=btver2 | FileCheck %s --check-prefixes=AVX,AVX1 ; ; Verify the cost model for concat_subvector style shuffles. @@ -19,14 +19,14 @@ define void @test_vXf64(<2 x double> %a128, <4 x double> %a256, <8 x double> %a512, <2 x double> %b128, <4 x double> %b256, <8 x double> %b512) { ; SSE-LABEL: 'test_vXf64' ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX-LABEL: 'test_vXf64' ; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void @@ -48,14 +48,14 @@ define void @test_vXf64(<2 x double> %a128, <4 x double> %a256, <8 x double> %a5 define void @test_vXi64(<2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, <2 x i64> %b128, <4 x i64> %b256, <8 x i64> %b512) { ; SSE-LABEL: 'test_vXi64' ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX-LABEL: 'test_vXi64' ; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void @@ -77,17 +77,24 @@ define void @test_vXi64(<2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, <2 x define void @test_vXf32(<4 x float> %a128, <8 x float> %a256, <16 x float> %a512, <4 x float> %b128, <8 x float> %b256, <16 x float> %b512) { ; SSE-LABEL: 'test_vXf32' ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; -; AVX-LABEL: 'test_vXf32' -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; AVX1-LABEL: 'test_vXf32' +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX2-LABEL: 'test_vXf32' +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512-LABEL: 'test_vXf32' ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> @@ -106,17 +113,24 @@ define void @test_vXf32(<4 x float> %a128, <8 x float> %a256, <16 x float> %a512 define void @test_vXi32(<4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, <4 x i32> %b128, <8 x i32> %b256, <16 x i32> %b512) { ; SSE-LABEL: 'test_vXi32' ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; -; AVX-LABEL: 'test_vXi32' -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; AVX1-LABEL: 'test_vXi32' +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX2-LABEL: 'test_vXi32' +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512-LABEL: 'test_vXi32' ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> @@ -133,26 +147,40 @@ define void @test_vXi32(<4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, <4 x } define void @test_vXi16(<8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, <8 x i16> %b128, <16 x i16> %b256, <32 x i16> %b512) { -; SSE-LABEL: 'test_vXi16' -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void -; -; AVX-LABEL: 'test_vXi16' -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void -; -; AVX512-LABEL: 'test_vXi16' -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; AVX1-LABEL: 'test_vXi16' +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 90 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX2-LABEL: 'test_vXi16' +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512F-LABEL: 'test_vXi16' +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512BW-LABEL: 'test_vXi16' +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512VBMI-LABEL: 'test_vXi16' +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> @@ -162,26 +190,40 @@ define void @test_vXi16(<8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, <8 } define void @test_vXi8(<16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512, <16 x i8> %b128, <32 x i8> %b256, <64 x i8> %b512) { -; SSE-LABEL: 'test_vXi8' -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void -; -; AVX-LABEL: 'test_vXi8' -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void -; -; AVX512-LABEL: 'test_vXi8' -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; AVX1-LABEL: 'test_vXi8' +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 90 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX2-LABEL: 'test_vXi8' +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512F-LABEL: 'test_vXi8' +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512BW-LABEL: 'test_vXi8' +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512VBMI-LABEL: 'test_vXi8' +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-latency.ll b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-latency.ll index 7244b7b4f00f..0f8503cd1c3c 100644 --- a/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-latency.ll +++ b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-latency.ll @@ -2,15 +2,15 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mattr=+sse2 | FileCheck %s -check-prefixes=SSE ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mattr=+ssse3 | FileCheck %s -check-prefixes=SSE ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mattr=+sse4.2 | FileCheck %s -check-prefixes=SSE -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mattr=+avx | FileCheck %s -check-prefixes=AVX -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mattr=+avx2 | FileCheck %s -check-prefixes=AVX -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512 -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512 -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mattr=+avx | FileCheck %s -check-prefixes=AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mattr=+avx2 | FileCheck %s -check-prefixes=AVX,AVX2 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512,AVX512VBMI ; ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=slm | FileCheck %s --check-prefixes=SSE ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=goldmont | FileCheck %s --check-prefixes=SSE -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=btver2 | FileCheck %s --check-prefixes=AVX +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=btver2 | FileCheck %s --check-prefixes=AVX,AVX1 ; ; Verify the cost model for concat_subvector style shuffles. @@ -19,21 +19,21 @@ define void @test_vXf64(<2 x double> %a128, <4 x double> %a256, <8 x double> %a512, <2 x double> %b128, <4 x double> %b256, <8 x double> %b512) { ; SSE-LABEL: 'test_vXf64' ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX-LABEL: 'test_vXf64' ; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512-LABEL: 'test_vXf64' ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void @@ -48,21 +48,21 @@ define void @test_vXf64(<2 x double> %a128, <4 x double> %a256, <8 x double> %a5 define void @test_vXi64(<2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, <2 x i64> %b128, <4 x i64> %b256, <8 x i64> %b512) { ; SSE-LABEL: 'test_vXi64' ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX-LABEL: 'test_vXi64' ; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512-LABEL: 'test_vXi64' ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void @@ -77,21 +77,28 @@ define void @test_vXi64(<2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, <2 x define void @test_vXf32(<4 x float> %a128, <8 x float> %a256, <16 x float> %a512, <4 x float> %b128, <8 x float> %b256, <16 x float> %b512) { ; SSE-LABEL: 'test_vXf32' ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; -; AVX-LABEL: 'test_vXf32' -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; AVX1-LABEL: 'test_vXf32' +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX2-LABEL: 'test_vXf32' +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512-LABEL: 'test_vXf32' ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void @@ -106,21 +113,28 @@ define void @test_vXf32(<4 x float> %a128, <8 x float> %a256, <16 x float> %a512 define void @test_vXi32(<4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, <4 x i32> %b128, <8 x i32> %b256, <16 x i32> %b512) { ; SSE-LABEL: 'test_vXi32' ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; -; AVX-LABEL: 'test_vXi32' -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; AVX1-LABEL: 'test_vXi32' +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX2-LABEL: 'test_vXi32' +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512-LABEL: 'test_vXi32' ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void @@ -133,26 +147,40 @@ define void @test_vXi32(<4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, <4 x } define void @test_vXi16(<8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, <8 x i16> %b128, <16 x i16> %b256, <32 x i16> %b512) { -; SSE-LABEL: 'test_vXi16' -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void -; -; AVX-LABEL: 'test_vXi16' -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void -; -; AVX512-LABEL: 'test_vXi16' -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; AVX1-LABEL: 'test_vXi16' +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 90 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX2-LABEL: 'test_vXi16' +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512F-LABEL: 'test_vXi16' +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512BW-LABEL: 'test_vXi16' +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512VBMI-LABEL: 'test_vXi16' +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> @@ -162,26 +190,40 @@ define void @test_vXi16(<8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, <8 } define void @test_vXi8(<16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512, <16 x i8> %b128, <32 x i8> %b256, <64 x i8> %b512) { -; SSE-LABEL: 'test_vXi8' -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void -; -; AVX-LABEL: 'test_vXi8' -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void -; -; AVX512-LABEL: 'test_vXi8' -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; AVX1-LABEL: 'test_vXi8' +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 90 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX2-LABEL: 'test_vXi8' +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512F-LABEL: 'test_vXi8' +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512BW-LABEL: 'test_vXi8' +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512VBMI-LABEL: 'test_vXi8' +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-sizelatency.ll index b1fa00b5a713..8c4f55eb8adc 100644 --- a/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-sizelatency.ll +++ b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector-sizelatency.ll @@ -2,15 +2,15 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mattr=+sse2 | FileCheck %s -check-prefixes=SSE ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mattr=+ssse3 | FileCheck %s -check-prefixes=SSE ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mattr=+sse4.2 | FileCheck %s -check-prefixes=SSE -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mattr=+avx | FileCheck %s -check-prefixes=AVX -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mattr=+avx2 | FileCheck %s -check-prefixes=AVX -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512 -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512 -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mattr=+avx | FileCheck %s -check-prefixes=AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mattr=+avx2 | FileCheck %s -check-prefixes=AVX,AVX2 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512,AVX512VBMI ; ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=slm | FileCheck %s --check-prefixes=SSE ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=goldmont | FileCheck %s --check-prefixes=SSE -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=btver2 | FileCheck %s --check-prefixes=AVX +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=btver2 | FileCheck %s --check-prefixes=AVX,AVX1 ; ; Verify the cost model for concat_subvector style shuffles. @@ -19,14 +19,14 @@ define void @test_vXf64(<2 x double> %a128, <4 x double> %a256, <8 x double> %a512, <2 x double> %b128, <4 x double> %b256, <8 x double> %b512) { ; SSE-LABEL: 'test_vXf64' ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX-LABEL: 'test_vXf64' ; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void @@ -48,14 +48,14 @@ define void @test_vXf64(<2 x double> %a128, <4 x double> %a256, <8 x double> %a5 define void @test_vXi64(<2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, <2 x i64> %b128, <4 x i64> %b256, <8 x i64> %b512) { ; SSE-LABEL: 'test_vXi64' ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX-LABEL: 'test_vXi64' ; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void @@ -77,17 +77,24 @@ define void @test_vXi64(<2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, <2 x define void @test_vXf32(<4 x float> %a128, <8 x float> %a256, <16 x float> %a512, <4 x float> %b128, <8 x float> %b256, <16 x float> %b512) { ; SSE-LABEL: 'test_vXf32' ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; -; AVX-LABEL: 'test_vXf32' -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; AVX1-LABEL: 'test_vXf32' +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX2-LABEL: 'test_vXf32' +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512-LABEL: 'test_vXf32' ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> @@ -106,17 +113,24 @@ define void @test_vXf32(<4 x float> %a128, <8 x float> %a256, <16 x float> %a512 define void @test_vXi32(<4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, <4 x i32> %b128, <8 x i32> %b256, <16 x i32> %b512) { ; SSE-LABEL: 'test_vXi32' ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; -; AVX-LABEL: 'test_vXi32' -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; AVX1-LABEL: 'test_vXi32' +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX2-LABEL: 'test_vXi32' +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512-LABEL: 'test_vXi32' ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> @@ -133,26 +147,40 @@ define void @test_vXi32(<4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, <4 x } define void @test_vXi16(<8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, <8 x i16> %b128, <16 x i16> %b256, <32 x i16> %b512) { -; SSE-LABEL: 'test_vXi16' -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void -; -; AVX-LABEL: 'test_vXi16' -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void -; -; AVX512-LABEL: 'test_vXi16' -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; AVX1-LABEL: 'test_vXi16' +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 90 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX2-LABEL: 'test_vXi16' +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512F-LABEL: 'test_vXi16' +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512BW-LABEL: 'test_vXi16' +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512VBMI-LABEL: 'test_vXi16' +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> @@ -162,26 +190,40 @@ define void @test_vXi16(<8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, <8 } define void @test_vXi8(<16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512, <16 x i8> %b128, <32 x i8> %b256, <64 x i8> %b512) { -; SSE-LABEL: 'test_vXi8' -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void -; -; AVX-LABEL: 'test_vXi8' -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void -; -; AVX512-LABEL: 'test_vXi8' -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; AVX1-LABEL: 'test_vXi8' +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 90 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX2-LABEL: 'test_vXi8' +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512F-LABEL: 'test_vXi8' +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512BW-LABEL: 'test_vXi8' +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512VBMI-LABEL: 'test_vXi8' +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector.ll b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector.ll index 60cb8cffd1a5..ffc470d5f344 100644 --- a/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector.ll +++ b/llvm/test/Analysis/CostModel/X86/shuffle-concat_subvector.ll @@ -2,15 +2,15 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mattr=+sse2 | FileCheck %s -check-prefixes=SSE ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mattr=+ssse3 | FileCheck %s -check-prefixes=SSE ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mattr=+sse4.2 | FileCheck %s -check-prefixes=SSE -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mattr=+avx | FileCheck %s -check-prefixes=AVX -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mattr=+avx2 | FileCheck %s -check-prefixes=AVX -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512 -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512 -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mattr=+avx | FileCheck %s -check-prefixes=AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mattr=+avx2 | FileCheck %s -check-prefixes=AVX,AVX2 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512,AVX512VBMI ; ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=slm | FileCheck %s --check-prefixes=SSE ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=goldmont | FileCheck %s --check-prefixes=SSE -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=btver2 | FileCheck %s --check-prefixes=AVX +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=btver2 | FileCheck %s --check-prefixes=AVX,AVX1 ; ; Verify the cost model for concat_subvector style shuffles. @@ -19,21 +19,21 @@ define void @test_vXf64(<2 x double> %a128, <4 x double> %a256, <8 x double> %a512, <2 x double> %b128, <4 x double> %b256, <8 x double> %b512) { ; SSE-LABEL: 'test_vXf64' ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; ; AVX-LABEL: 'test_vXf64' ; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; ; AVX512-LABEL: 'test_vXf64' ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <4 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x double> %a128, <2 x double> %b128, <8 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x double> %a256, <4 x double> %b256, <8 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <8 x double> %a512, <8 x double> %b512, <16 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void @@ -48,21 +48,21 @@ define void @test_vXf64(<2 x double> %a128, <4 x double> %a256, <8 x double> %a5 define void @test_vXi64(<2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, <2 x i64> %b128, <4 x i64> %b256, <8 x i64> %b512) { ; SSE-LABEL: 'test_vXi64' ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; ; AVX-LABEL: 'test_vXi64' ; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> +; AVX-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> ; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; ; AVX512-LABEL: 'test_vXi64' ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <4 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <2 x i64> %a128, <2 x i64> %b128, <8 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <4 x i64> %a256, <4 x i64> %b256, <8 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <8 x i64> %a512, <8 x i64> %b512, <16 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void @@ -77,21 +77,28 @@ define void @test_vXi64(<2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, <2 x define void @test_vXf32(<4 x float> %a128, <8 x float> %a256, <16 x float> %a512, <4 x float> %b128, <8 x float> %b256, <16 x float> %b512) { ; SSE-LABEL: 'test_vXf32' ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; -; AVX-LABEL: 'test_vXf32' -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; AVX1-LABEL: 'test_vXf32' +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX2-LABEL: 'test_vXf32' +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; ; AVX512-LABEL: 'test_vXf32' ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <8 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x float> %a128, <4 x float> %b128, <16 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x float> %a256, <8 x float> %b256, <16 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <16 x float> %a512, <16 x float> %b512, <32 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void @@ -106,21 +113,28 @@ define void @test_vXf32(<4 x float> %a128, <8 x float> %a256, <16 x float> %a512 define void @test_vXi32(<4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, <4 x i32> %b128, <8 x i32> %b256, <16 x i32> %b512) { ; SSE-LABEL: 'test_vXi32' ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; -; AVX-LABEL: 'test_vXi32' -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; AVX1-LABEL: 'test_vXi32' +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX2-LABEL: 'test_vXi32' +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; ; AVX512-LABEL: 'test_vXi32' ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <8 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_128 = shufflevector <4 x i32> %a128, <4 x i32> %b128, <16 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <8 x i32> %a256, <8 x i32> %b256, <16 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <16 x i32> %a512, <16 x i32> %b512, <32 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void @@ -133,26 +147,40 @@ define void @test_vXi32(<4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, <4 x } define void @test_vXi16(<8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, <8 x i16> %b128, <16 x i16> %b256, <32 x i16> %b512) { -; SSE-LABEL: 'test_vXi16' -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void -; -; AVX-LABEL: 'test_vXi16' -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void -; -; AVX512-LABEL: 'test_vXi16' -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; AVX1-LABEL: 'test_vXi16' +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 90 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX2-LABEL: 'test_vXi16' +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512F-LABEL: 'test_vXi16' +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512BW-LABEL: 'test_vXi16' +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512VBMI-LABEL: 'test_vXi16' +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <16 x i16> %a256, <16 x i16> %b256, <32 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <32 x i16> %a512, <32 x i16> %b512, <64 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %V256_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <16 x i32> %V512_128 = shufflevector <8 x i16> %a128, <8 x i16> %b128, <32 x i32> @@ -162,26 +190,40 @@ define void @test_vXi16(<8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, <8 } define void @test_vXi8(<16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512, <16 x i8> %b128, <32 x i8> %b256, <64 x i8> %b512) { -; SSE-LABEL: 'test_vXi8' -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void -; -; AVX-LABEL: 'test_vXi8' -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void -; -; AVX512-LABEL: 'test_vXi8' -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; AVX1-LABEL: 'test_vXi8' +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 90 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX2-LABEL: 'test_vXi8' +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512F-LABEL: 'test_vXi8' +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512BW-LABEL: 'test_vXi8' +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512VBMI-LABEL: 'test_vXi8' +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_256 = shufflevector <32 x i8> %a256, <32 x i8> %b256, <64 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V1024_512 = shufflevector <64 x i8> %a512, <64 x i8> %b512, <128 x i32> +; AVX512VBMI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %V256_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <32 x i32> %V512_128 = shufflevector <16 x i8> %a128, <16 x i8> %b128, <64 x i32> diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-codesize.ll b/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-codesize.ll index 9a86fbe105ec..91314d396905 100644 --- a/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-codesize.ll +++ b/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-codesize.ll @@ -27,21 +27,34 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) { ; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; -; AVX-LABEL: 'test_vXf64' -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; AVX1-LABEL: 'test_vXf64' +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX2-LABEL: 'test_vXf64' +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512-LABEL: 'test_vXf64' ; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-latency.ll b/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-latency.ll index 393dec82428b..33431083c68b 100644 --- a/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-latency.ll +++ b/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-latency.ll @@ -27,21 +27,34 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) { ; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; -; AVX-LABEL: 'test_vXf64' -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; AVX1-LABEL: 'test_vXf64' +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX2-LABEL: 'test_vXf64' +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512-LABEL: 'test_vXf64' ; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> @@ -53,7 +66,7 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) { ; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-sizelatency.ll index 63bb07bf4fd8..039758e0b6e3 100644 --- a/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-sizelatency.ll +++ b/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-sizelatency.ll @@ -27,21 +27,34 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) { ; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; -; AVX-LABEL: 'test_vXf64' -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; AVX1-LABEL: 'test_vXf64' +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX2-LABEL: 'test_vXf64' +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512-LABEL: 'test_vXf64' ; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector.ll b/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector.ll index b521a7594846..6a82a4a7432e 100644 --- a/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector.ll +++ b/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector.ll @@ -27,21 +27,34 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) { ; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; SSE-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; SSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; SSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; -; AVX-LABEL: 'test_vXf64' -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; AVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; AVX1-LABEL: 'test_vXf64' +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX2-LABEL: 'test_vXf64' +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> undef, <2 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; ; AVX512-LABEL: 'test_vXf64' ; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> @@ -53,7 +66,7 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) { ; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> -; AVX512-NEXT: Cost Model: Found an estimated cost of -1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> +; AVX512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> ; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> -- GitLab From 5830d1a2dff24d752459f215a0c8fc366f393596 Mon Sep 17 00:00:00 2001 From: martinboehme Date: Thu, 7 Mar 2024 11:48:51 +0100 Subject: [PATCH 428/929] Revert "[dataflow][nfc] Fix u8 string usage with c++20" (#84301) Reverts llvm/llvm-project#84291 The patch broke Windows builds. --- clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp b/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp index d9f40d28859f..ff4e18de2c70 100644 --- a/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp +++ b/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp @@ -500,7 +500,7 @@ private: for (unsigned I = 0; I < CFG.getNumBlockIDs(); ++I) { std::string Name = blockID(I); // Rightwards arrow, vertical line - char ConvergenceMarker[] = "\\n\u2192\u007c"; + char ConvergenceMarker[] = u8"\\n\u2192\u007c"; if (BlockConverged[I]) Name += ConvergenceMarker; GraphS << " " << blockID(I) << " [id=" << blockID(I) << " label=\"" -- GitLab From 4119042d76c79667b374ad85b3b92ef56cfd96e8 Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Thu, 7 Mar 2024 10:58:33 +0000 Subject: [PATCH 429/929] [AMDGPU] Simplify EXP Real instruction definitions. NFC. Pass the Pseudo (instead of its name) into EXP_Real_Row and EXP_Real_ComprVM since it is already available in all subclasses. --- llvm/lib/Target/AMDGPU/EXPInstructions.td | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/EXPInstructions.td b/llvm/lib/Target/AMDGPU/EXPInstructions.td index cce8734b72d4..b73b83031af0 100644 --- a/llvm/lib/Target/AMDGPU/EXPInstructions.td +++ b/llvm/lib/Target/AMDGPU/EXPInstructions.td @@ -37,18 +37,18 @@ class EXP_Pseudo } // Real instruction with optional asm operands "compr" and "vm". -class EXP_Real_ComprVM(pseudo)> +class EXP_Real_ComprVM : EXPCommon<0, ps.done, "exp$tgt $src0, $src1, $src2, $src3" #!if(ps.done, " done", "")#"$compr$vm">, - SIMCInstr { + SIMCInstr { let AsmMatchConverter = "cvtExp"; } // Real instruction with optional asm operand "row_en". -class EXP_Real_Row(pseudo)> +class EXP_Real_Row : EXPCommon, - SIMCInstr { + SIMCInstr { let AsmMatchConverter = "cvtExp"; } @@ -71,7 +71,7 @@ def EXP_ROW_DONE : EXP_Pseudo<1, 1>; multiclass EXP_Real_si { defvar ps = !cast(NAME); - def _si : EXP_Real_ComprVM, EXPe_ComprVM { + def _si : EXP_Real_ComprVM, EXPe_ComprVM { let AssemblerPredicate = isGFX6GFX7; let DecoderNamespace = "GFX6GFX7"; let done = ps.done; @@ -80,7 +80,7 @@ multiclass EXP_Real_si { multiclass EXP_Real_vi { defvar ps = !cast(NAME); - def _vi : EXP_Real_ComprVM, EXPe_vi { + def _vi : EXP_Real_ComprVM, EXPe_vi { let AssemblerPredicate = isGFX8GFX9; let SubtargetPredicate = isNotGFX90APlus; let DecoderNamespace = "GFX8"; @@ -90,7 +90,7 @@ multiclass EXP_Real_vi { multiclass EXP_Real_gfx10 { defvar ps = !cast(NAME); - def _gfx10 : EXP_Real_ComprVM, EXPe_ComprVM { + def _gfx10 : EXP_Real_ComprVM, EXPe_ComprVM { let AssemblerPredicate = isGFX10Only; let DecoderNamespace = "GFX10"; let done = ps.done; @@ -106,7 +106,7 @@ defm EXP_DONE : EXP_Real_si, EXP_Real_vi, EXP_Real_gfx10; multiclass EXP_Real_gfx11 { defvar ps = !cast(NAME); - def _gfx11 : EXP_Real_Row, EXPe_Row { + def _gfx11 : EXP_Real_Row, EXPe_Row { let AssemblerPredicate = isGFX11Only; let DecoderNamespace = "GFX11"; let row = ps.row; @@ -116,7 +116,7 @@ multiclass EXP_Real_gfx11 { multiclass VEXPORT_Real_gfx12 { defvar ps = !cast(NAME); - def _gfx12 : EXP_Real_Row, + def _gfx12 : EXP_Real_Row, EXPe_Row, MnemonicAlias<"exp", "export">, Requires<[isGFX12Plus, HasExportInsts]> { let AssemblerPredicate = isGFX12Only; let DecoderNamespace = "GFX12"; -- GitLab From 937a5396cf3e524ae40106a943a5c1f2c565fa00 Mon Sep 17 00:00:00 2001 From: Nikolas Klauser Date: Thu, 7 Mar 2024 12:32:25 +0100 Subject: [PATCH 430/929] [libc++] Remove unused includes from __type_traits/is_convertible.h (#83747) --- libcxx/include/__type_traits/is_convertible.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/libcxx/include/__type_traits/is_convertible.h b/libcxx/include/__type_traits/is_convertible.h index bc91d8b23430..414c2a6d6a0d 100644 --- a/libcxx/include/__type_traits/is_convertible.h +++ b/libcxx/include/__type_traits/is_convertible.h @@ -11,12 +11,6 @@ #include <__config> #include <__type_traits/integral_constant.h> -#include <__type_traits/is_array.h> -#include <__type_traits/is_function.h> -#include <__type_traits/is_void.h> -#include <__type_traits/remove_reference.h> -#include <__utility/declval.h> -#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) # pragma GCC system_header -- GitLab From 0086cc95b3b3ac4088d3d782cd490d0c08108b59 Mon Sep 17 00:00:00 2001 From: Diana Picus Date: Thu, 7 Mar 2024 12:46:42 +0100 Subject: [PATCH 431/929] [AMDGPU] Rename getNumVGPRBlocks. NFC (#84161) Rename getNumVGPRBlocks to getEncodedNumVGPRBlocks, to clarify that it's using the encoding granule. This is used to program the hardware. In practice, the hardware will use the alloc granule instead, so this patch also adds a new helper, getAllocatedNumVGPRBlocks, which can be useful when driving heuristics. --- llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp | 4 +-- .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp | 4 +-- .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 26 +++++++++++++------ llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 15 ++++++++--- 4 files changed, 33 insertions(+), 16 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp index 37a36b26b947..d9970a200804 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp @@ -868,8 +868,8 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo, ProgInfo.SGPRBlocks = IsaInfo::getNumSGPRBlocks( &STM, ProgInfo.NumSGPRsForWavesPerEU); - ProgInfo.VGPRBlocks = IsaInfo::getNumVGPRBlocks( - &STM, ProgInfo.NumVGPRsForWavesPerEU); + ProgInfo.VGPRBlocks = + IsaInfo::getEncodedNumVGPRBlocks(&STM, ProgInfo.NumVGPRsForWavesPerEU); const SIModeRegisterDefaults Mode = MFI->getMode(); diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp index cb4eddfe5320..d5efd4415562 100644 --- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp +++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp @@ -5344,8 +5344,8 @@ bool AMDGPUAsmParser::calculateGPRBlocks( NumSGPRs = IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG; } - VGPRBlocks = - IsaInfo::getNumVGPRBlocks(&getSTI(), NumVGPRs, EnableWavefrontSize32); + VGPRBlocks = IsaInfo::getEncodedNumVGPRBlocks(&getSTI(), NumVGPRs, + EnableWavefrontSize32); SGPRBlocks = IsaInfo::getNumSGPRBlocks(&getSTI(), NumSGPRs); return false; diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index 0eab7acc9ebc..62903a244dc8 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -1060,10 +1060,15 @@ unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed, STI->getFeatureBits().test(AMDGPU::FeatureXNACK)); } +static unsigned getGranulatedNumRegisterBlocks(unsigned NumRegs, + unsigned Granule) { + return divideCeil(std::max(1u, NumRegs), Granule); +} + unsigned getNumSGPRBlocks(const MCSubtargetInfo *STI, unsigned NumSGPRs) { - NumSGPRs = alignTo(std::max(1u, NumSGPRs), getSGPREncodingGranule(STI)); // SGPRBlocks is actual number of SGPR blocks minus 1. - return NumSGPRs / getSGPREncodingGranule(STI) - 1; + return getGranulatedNumRegisterBlocks(NumSGPRs, getSGPREncodingGranule(STI)) - + 1; } unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI, @@ -1158,14 +1163,19 @@ unsigned getMaxNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { return std::min(MaxNumVGPRs, AddressableNumVGPRs); } -unsigned getNumVGPRBlocks(const MCSubtargetInfo *STI, unsigned NumVGPRs, - std::optional EnableWavefrontSize32) { - NumVGPRs = alignTo(std::max(1u, NumVGPRs), - getVGPREncodingGranule(STI, EnableWavefrontSize32)); - // VGPRBlocks is actual number of VGPR blocks minus 1. - return NumVGPRs / getVGPREncodingGranule(STI, EnableWavefrontSize32) - 1; +unsigned getEncodedNumVGPRBlocks(const MCSubtargetInfo *STI, unsigned NumVGPRs, + std::optional EnableWavefrontSize32) { + return getGranulatedNumRegisterBlocks( + NumVGPRs, getVGPREncodingGranule(STI, EnableWavefrontSize32)) - + 1; } +unsigned getAllocatedNumVGPRBlocks(const MCSubtargetInfo *STI, + unsigned NumVGPRs, + std::optional EnableWavefrontSize32) { + return getGranulatedNumRegisterBlocks( + NumVGPRs, getVGPRAllocGranule(STI, EnableWavefrontSize32)); +} } // end namespace IsaInfo void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header, diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h index 6edf01d1217f..bb307cb67c9b 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h @@ -316,13 +316,20 @@ unsigned getNumWavesPerEUWithNumVGPRs(const MCSubtargetInfo *STI, unsigned NumVGPRs); /// \returns Number of VGPR blocks needed for given subtarget \p STI when -/// \p NumVGPRs are used. +/// \p NumVGPRs are used. We actually return the number of blocks -1, since +/// that's what we encode. /// /// For subtargets which support it, \p EnableWavefrontSize32 should match the /// ENABLE_WAVEFRONT_SIZE32 kernel descriptor field. -unsigned -getNumVGPRBlocks(const MCSubtargetInfo *STI, unsigned NumSGPRs, - std::optional EnableWavefrontSize32 = std::nullopt); +unsigned getEncodedNumVGPRBlocks( + const MCSubtargetInfo *STI, unsigned NumVGPRs, + std::optional EnableWavefrontSize32 = std::nullopt); + +/// \returns Number of VGPR blocks that need to be allocated for the given +/// subtarget \p STI when \p NumVGPRs are used. +unsigned getAllocatedNumVGPRBlocks( + const MCSubtargetInfo *STI, unsigned NumVGPRs, + std::optional EnableWavefrontSize32 = std::nullopt); } // end namespace IsaInfo -- GitLab From a11ab139e4de9cdad41c299f198515c09be6f05d Mon Sep 17 00:00:00 2001 From: martinboehme Date: Thu, 7 Mar 2024 12:53:26 +0100 Subject: [PATCH 432/929] [clang][dataflow] Fix u8 string error with C++20. (#84302) See also discussion on https://github.com/llvm/llvm-project/pull/84291. --- clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp b/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp index ff4e18de2c70..6afd66d9dc6a 100644 --- a/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp +++ b/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp @@ -500,7 +500,7 @@ private: for (unsigned I = 0; I < CFG.getNumBlockIDs(); ++I) { std::string Name = blockID(I); // Rightwards arrow, vertical line - char ConvergenceMarker[] = u8"\\n\u2192\u007c"; + const char *ConvergenceMarker = (const char *)u8"\\n\u2192\u007c"; if (BlockConverged[I]) Name += ConvergenceMarker; GraphS << " " << blockID(I) << " [id=" << blockID(I) << " label=\"" -- GitLab From d5aecf0c19fc8850d7d34ac8c339bcc7e133b5fb Mon Sep 17 00:00:00 2001 From: martinboehme Date: Thu, 7 Mar 2024 13:31:23 +0100 Subject: [PATCH 433/929] [clang][nullability] Don't discard expression state before end of full-expression. (#82611) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In https://github.com/llvm/llvm-project/pull/72985, I made a change to discard expression state (`ExprToLoc` and `ExprToVal`) at the beginning of each basic block. I did so with the claim that "we never need to access entries from these maps outside of the current basic block", noting that there are exceptions to this claim when control flow happens inside a full-expression (the operands of `&&`, `||`, and the conditional operator live in different basic blocks than the operator itself) but that we already have a mechanism for retrieving the values of these operands from the environment for the block they are computed in. It turns out, however, that the operands of these operators aren't the only expressions whose values can be accessed from a different basic block; when control flow happens within a full-expression, that control flow can be "interposed" between an expression and its parent. Here is an example: ```cxx void f(int*, int); bool cond(); void target() { int i = 0; f(&i, cond() ? 1 : 0); } ``` ([godbolt](https://godbolt.org/z/hrbj1Mj3o)) In the CFG[^1] , note how the expression for `&i` is computed in block B4, but the parent of this expression (the `CallExpr`) is located in block B1. The the argument expression `&i` and the `CallExpr` are essentially "torn apart" into different basic blocks by the conditional operator in the second argument. In other words, the edge between the `CallExpr` and its argument `&i` straddles the boundary between two blocks. I used to think that this scenario -- where an edge between an expression and one of its children straddles a block boundary -- could only happen between the expression that triggers the control flow (`&&`, `||`, or the conditional operator) and its children, but the example above shows that other expressions can be affected as well; the control flow is still triggered by `&&`, `||` or the conditional operator, but the expressions affected lie outside these operators. Discarding expression state too soon is harmful. For example, an analysis that checks the arguments of the `CallExpr` above would not be able to retrieve a value for the `&i` argument. This patch therefore ensures that we don't discard expression state before the end of a full-expression. In other cases -- when the evaluation of a full-expression is complete -- we still want to discard expression state for the reasons explained in https://github.com/llvm/llvm-project/pull/72985 (avoid performing joins on boolean values that are no longer needed, which unnecessarily extends the flow condition; improve debuggability by removing clutter from the expression state). The impact on performance from this change is about a 1% slowdown in the Crubit nullability check benchmarks: ``` name old cpu/op new cpu/op delta BM_PointerAnalysisCopyPointer 71.9µs ± 1% 71.9µs ± 2% ~ (p=0.987 n=15+20) BM_PointerAnalysisIntLoop 190µs ± 1% 192µs ± 2% +1.06% (p=0.000 n=14+16) BM_PointerAnalysisPointerLoop 325µs ± 5% 324µs ± 4% ~ (p=0.496 n=18+20) BM_PointerAnalysisBranch 193µs ± 0% 192µs ± 4% ~ (p=0.488 n=14+18) BM_PointerAnalysisLoopAndBranch 521µs ± 1% 525µs ± 3% +0.94% (p=0.017 n=18+19) BM_PointerAnalysisTwoLoops 337µs ± 1% 341µs ± 3% +1.19% (p=0.004 n=17+19) BM_PointerAnalysisJoinFilePath 1.62ms ± 2% 1.64ms ± 3% +0.92% (p=0.021 n=20+20) BM_PointerAnalysisCallInLoop 1.14ms ± 1% 1.15ms ± 4% ~ (p=0.135 n=16+18) ``` [^1]: ``` [B5 (ENTRY)] Succs (1): B4 [B1] 1: [B4.9] ? [B2.1] : [B3.1] 2: [B4.4]([B4.6], [B1.1]) Preds (2): B2 B3 Succs (1): B0 [B2] 1: 1 Preds (1): B4 Succs (1): B1 [B3] 1: 0 Preds (1): B4 Succs (1): B1 [B4] 1: 0 2: int i = 0; 3: f 4: [B4.3] (ImplicitCastExpr, FunctionToPointerDecay, void (*)(int *, int)) 5: i 6: &[B4.5] 7: cond 8: [B4.7] (ImplicitCastExpr, FunctionToPointerDecay, _Bool (*)(void)) 9: [B4.8]() T: [B4.9] ? ... : ... Preds (1): B5 Succs (2): B2 B3 [B0 (EXIT)] Preds (1): B1 ``` --- .../FlowSensitive/ControlFlowContext.h | 25 ++++- .../FlowSensitive/DataflowEnvironment.h | 11 +- .../FlowSensitive/ControlFlowContext.cpp | 38 ++++++- .../FlowSensitive/DataflowEnvironment.cpp | 28 ++++- .../TypeErasedDataflowAnalysis.cpp | 32 ++++-- .../FlowSensitive/DataflowEnvironmentTest.cpp | 3 +- .../TypeErasedDataflowAnalysisTest.cpp | 104 ++++++++++++------ 7 files changed, 189 insertions(+), 52 deletions(-) diff --git a/clang/include/clang/Analysis/FlowSensitive/ControlFlowContext.h b/clang/include/clang/Analysis/FlowSensitive/ControlFlowContext.h index 405e93287a05..9a0a00f3c013 100644 --- a/clang/include/clang/Analysis/FlowSensitive/ControlFlowContext.h +++ b/clang/include/clang/Analysis/FlowSensitive/ControlFlowContext.h @@ -58,19 +58,36 @@ public: return BlockReachable[B.getBlockID()]; } + /// Returns whether `B` contains an expression that is consumed in a + /// different block than `B` (i.e. the parent of the expression is in a + /// different block). + /// This happens if there is control flow within a full-expression (triggered + /// by `&&`, `||`, or the conditional operator). Note that the operands of + /// these operators are not the only expressions that can be consumed in a + /// different block. For example, in the function call + /// `f(&i, cond() ? 1 : 0)`, `&i` is in a different block than the `CallExpr`. + bool containsExprConsumedInDifferentBlock(const CFGBlock &B) const { + return ContainsExprConsumedInDifferentBlock.contains(&B); + } + private: - ControlFlowContext(const Decl &D, std::unique_ptr Cfg, - llvm::DenseMap StmtToBlock, - llvm::BitVector BlockReachable) + ControlFlowContext( + const Decl &D, std::unique_ptr Cfg, + llvm::DenseMap StmtToBlock, + llvm::BitVector BlockReachable, + llvm::DenseSet ContainsExprConsumedInDifferentBlock) : ContainingDecl(D), Cfg(std::move(Cfg)), StmtToBlock(std::move(StmtToBlock)), - BlockReachable(std::move(BlockReachable)) {} + BlockReachable(std::move(BlockReachable)), + ContainsExprConsumedInDifferentBlock( + std::move(ContainsExprConsumedInDifferentBlock)) {} /// The `Decl` containing the statement used to construct the CFG. const Decl &ContainingDecl; std::unique_ptr Cfg; llvm::DenseMap StmtToBlock; llvm::BitVector BlockReachable; + llvm::DenseSet ContainsExprConsumedInDifferentBlock; }; } // namespace dataflow diff --git a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h index 62e7af7ac219..e8f009ef6c79 100644 --- a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h +++ b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h @@ -210,6 +210,14 @@ public: bool equivalentTo(const Environment &Other, Environment::ValueModel &Model) const; + /// How to treat expression state (`ExprToLoc` and `ExprToVal`) in a join. + /// If the join happens within a full expression, expression state should be + /// kept; otherwise, we can discard it. + enum ExprJoinBehavior { + DiscardExprState, + KeepExprState, + }; + /// Joins two environments by taking the intersection of storage locations and /// values that are stored in them. Distinct values that are assigned to the /// same storage locations in `EnvA` and `EnvB` are merged using `Model`. @@ -218,7 +226,8 @@ public: /// /// `EnvA` and `EnvB` must use the same `DataflowAnalysisContext`. static Environment join(const Environment &EnvA, const Environment &EnvB, - Environment::ValueModel &Model); + Environment::ValueModel &Model, + ExprJoinBehavior ExprBehavior); /// Widens the environment point-wise, using `PrevEnv` as needed to inform the /// approximation. diff --git a/clang/lib/Analysis/FlowSensitive/ControlFlowContext.cpp b/clang/lib/Analysis/FlowSensitive/ControlFlowContext.cpp index 8aed19544be6..7c9f8fbb0a70 100644 --- a/clang/lib/Analysis/FlowSensitive/ControlFlowContext.cpp +++ b/clang/lib/Analysis/FlowSensitive/ControlFlowContext.cpp @@ -94,6 +94,38 @@ static llvm::BitVector findReachableBlocks(const CFG &Cfg) { return BlockReachable; } +static llvm::DenseSet +buildContainsExprConsumedInDifferentBlock( + const CFG &Cfg, + const llvm::DenseMap &StmtToBlock) { + llvm::DenseSet Result; + + auto CheckChildExprs = [&Result, &StmtToBlock](const Stmt *S, + const CFGBlock *Block) { + for (const Stmt *Child : S->children()) { + if (!isa(Child)) + continue; + const CFGBlock *ChildBlock = StmtToBlock.lookup(Child); + if (ChildBlock != Block) + Result.insert(ChildBlock); + } + }; + + for (const CFGBlock *Block : Cfg) { + if (Block == nullptr) + continue; + + for (const CFGElement &Element : *Block) + if (auto S = Element.getAs()) + CheckChildExprs(S->getStmt(), Block); + + if (const Stmt *TerminatorCond = Block->getTerminatorCondition()) + CheckChildExprs(TerminatorCond, Block); + } + + return Result; +} + llvm::Expected ControlFlowContext::build(const FunctionDecl &Func) { if (!Func.doesThisDeclarationHaveABody()) @@ -140,8 +172,12 @@ ControlFlowContext::build(const Decl &D, Stmt &S, ASTContext &C) { llvm::BitVector BlockReachable = findReachableBlocks(*Cfg); + llvm::DenseSet ContainsExprConsumedInDifferentBlock = + buildContainsExprConsumedInDifferentBlock(*Cfg, StmtToBlock); + return ControlFlowContext(D, std::move(Cfg), std::move(StmtToBlock), - std::move(BlockReachable)); + std::move(BlockReachable), + std::move(ContainsExprConsumedInDifferentBlock)); } } // namespace dataflow diff --git a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp index fd7b06efcc78..62332a18c44a 100644 --- a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp +++ b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp @@ -48,6 +48,24 @@ static llvm::DenseMap intersectDeclToLoc( return Result; } +// Performs a join on either `ExprToLoc` or `ExprToVal`. +// The maps must be consistent in the sense that any entries for the same +// expression must map to the same location / value. This is the case if we are +// performing a join for control flow within a full-expression (which is the +// only case when this function should be used). +template MapT joinExprMaps(const MapT &Map1, const MapT &Map2) { + MapT Result = Map1; + + for (const auto &Entry : Map2) { + [[maybe_unused]] auto [It, Inserted] = Result.insert(Entry); + // If there was an existing entry, its value should be the same as for the + // entry we were trying to insert. + assert(It->second == Entry.second); + } + + return Result; +} + // Whether to consider equivalent two values with an unknown relation. // // FIXME: this function is a hack enabling unsoundness to support @@ -627,7 +645,8 @@ LatticeJoinEffect Environment::widen(const Environment &PrevEnv, } Environment Environment::join(const Environment &EnvA, const Environment &EnvB, - Environment::ValueModel &Model) { + Environment::ValueModel &Model, + ExprJoinBehavior ExprBehavior) { assert(EnvA.DACtx == EnvB.DACtx); assert(EnvA.ThisPointeeLoc == EnvB.ThisPointeeLoc); assert(EnvA.CallStack == EnvB.CallStack); @@ -675,9 +694,10 @@ Environment Environment::join(const Environment &EnvA, const Environment &EnvB, JoinedEnv.LocToVal = joinLocToVal(EnvA.LocToVal, EnvB.LocToVal, EnvA, EnvB, JoinedEnv, Model); - // We intentionally leave `JoinedEnv.ExprToLoc` and `JoinedEnv.ExprToVal` - // empty, as we never need to access entries in these maps outside of the - // basic block that sets them. + if (ExprBehavior == KeepExprState) { + JoinedEnv.ExprToVal = joinExprMaps(EnvA.ExprToVal, EnvB.ExprToVal); + JoinedEnv.ExprToLoc = joinExprMaps(EnvA.ExprToLoc, EnvB.ExprToLoc); + } return JoinedEnv; } diff --git a/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp b/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp index 4c88c46142d6..a9f39e153d0c 100644 --- a/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp +++ b/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp @@ -221,6 +221,7 @@ private: // Avoids unneccesary copies of the environment. class JoinedStateBuilder { AnalysisContext &AC; + Environment::ExprJoinBehavior JoinBehavior; std::vector All; std::deque Owned; @@ -228,11 +229,13 @@ class JoinedStateBuilder { join(const TypeErasedDataflowAnalysisState &L, const TypeErasedDataflowAnalysisState &R) { return {AC.Analysis.joinTypeErased(L.Lattice, R.Lattice), - Environment::join(L.Env, R.Env, AC.Analysis)}; + Environment::join(L.Env, R.Env, AC.Analysis, JoinBehavior)}; } public: - JoinedStateBuilder(AnalysisContext &AC) : AC(AC) {} + JoinedStateBuilder(AnalysisContext &AC, + Environment::ExprJoinBehavior JoinBehavior) + : AC(AC), JoinBehavior(JoinBehavior) {} void addOwned(TypeErasedDataflowAnalysisState State) { Owned.push_back(std::move(State)); @@ -248,12 +251,12 @@ public: // initialize the state of each basic block differently. return {AC.Analysis.typeErasedInitialElement(), AC.InitEnv.fork()}; if (All.size() == 1) - // Join the environment with itself so that we discard the entries from - // `ExprToLoc` and `ExprToVal`. + // Join the environment with itself so that we discard expression state if + // desired. // FIXME: We could consider writing special-case code for this that only // does the discarding, but it's not clear if this is worth it. - return {All[0]->Lattice, - Environment::join(All[0]->Env, All[0]->Env, AC.Analysis)}; + return {All[0]->Lattice, Environment::join(All[0]->Env, All[0]->Env, + AC.Analysis, JoinBehavior)}; auto Result = join(*All[0], *All[1]); for (unsigned I = 2; I < All.size(); ++I) @@ -307,7 +310,22 @@ computeBlockInputState(const CFGBlock &Block, AnalysisContext &AC) { } } - JoinedStateBuilder Builder(AC); + // If any of the predecessor blocks contains an expression consumed in a + // different block, we need to keep expression state. + // Note that in this case, we keep expression state for all predecessors, + // rather than only those predecessors that actually contain an expression + // consumed in a different block. While this is potentially suboptimal, it's + // actually likely, if we have control flow within a full expression, that + // all predecessors have expression state consumed in a different block. + Environment::ExprJoinBehavior JoinBehavior = Environment::DiscardExprState; + for (const CFGBlock *Pred : Preds) { + if (Pred && AC.CFCtx.containsExprConsumedInDifferentBlock(*Pred)) { + JoinBehavior = Environment::KeepExprState; + break; + } + } + + JoinedStateBuilder Builder(AC, JoinBehavior); for (const CFGBlock *Pred : Preds) { // Skip if the `Block` is unreachable or control flow cannot get past it. if (!Pred || Pred->hasNoReturnElement()) diff --git a/clang/unittests/Analysis/FlowSensitive/DataflowEnvironmentTest.cpp b/clang/unittests/Analysis/FlowSensitive/DataflowEnvironmentTest.cpp index 8799d03dfd3c..465a8e21690c 100644 --- a/clang/unittests/Analysis/FlowSensitive/DataflowEnvironmentTest.cpp +++ b/clang/unittests/Analysis/FlowSensitive/DataflowEnvironmentTest.cpp @@ -190,7 +190,8 @@ TEST_F(EnvironmentTest, JoinRecords) { Env2.setValue(Loc, Val2); Environment::ValueModel Model; - Environment EnvJoined = Environment::join(Env1, Env2, Model); + Environment EnvJoined = + Environment::join(Env1, Env2, Model, Environment::DiscardExprState); auto *JoinedVal = cast(EnvJoined.getValue(Loc)); EXPECT_NE(JoinedVal, &Val1); EXPECT_NE(JoinedVal, &Val2); diff --git a/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp b/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp index 34f9b0b23719..9d05a0d6ca40 100644 --- a/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp +++ b/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp @@ -244,15 +244,17 @@ TEST_F(DiscardExprStateTest, WhileStatement) { EXPECT_NE(NotEqOpState.Env.getValue(NotEqOp), nullptr); // In the block that calls `foo(p)`, the value for `p != nullptr` is discarded - // because it is not consumed by this block. + // because it is not consumed outside the block it is in. const auto &CallFooState = blockStateForStmt(BlockStates, CallFoo); EXPECT_EQ(CallFooState.Env.getValue(NotEqOp), nullptr); } TEST_F(DiscardExprStateTest, BooleanOperator) { std::string Code = R"( - bool target(bool b1, bool b2) { - return b1 && b2; + void f(); + void target(bool b1, bool b2) { + if (b1 && b2) + f(); } )"; auto BlockStates = llvm::cantFail(runAnalysis( @@ -260,46 +262,80 @@ TEST_F(DiscardExprStateTest, BooleanOperator) { const auto &AndOp = matchNode(binaryOperator(hasOperatorName("&&"))); - const auto &Return = matchNode(returnStmt()); + const auto &CallF = + matchNode(callExpr(callee(functionDecl(hasName("f"))))); // In the block that evaluates the LHS of the `&&` operator, the LHS is // associated with a value, while the right-hand side is not (unsurprisingly, // as it hasn't been evaluated yet). const auto &LHSState = blockStateForStmt(BlockStates, *AndOp.getLHS()); auto *LHSValue = cast(LHSState.Env.getValue(*AndOp.getLHS())); - ASSERT_NE(LHSValue, nullptr); + EXPECT_NE(LHSValue, nullptr); EXPECT_EQ(LHSState.Env.getValue(*AndOp.getRHS()), nullptr); - // In the block that evaluates the RHS, the RHS is associated with a - // value. The value for the LHS has been discarded as it is not consumed by - // this block. + // In the block that evaluates the RHS, both the LHS and RHS are associated + // with values, as they are both subexpressions of the `&&` operator, which + // is evaluated in a later block. const auto &RHSState = blockStateForStmt(BlockStates, *AndOp.getRHS()); - EXPECT_EQ(RHSState.Env.getValue(*AndOp.getLHS()), nullptr); - auto *RHSValue = cast(RHSState.Env.getValue(*AndOp.getRHS())); - ASSERT_NE(RHSValue, nullptr); - - // In the block that evaluates the return statement, the expression `b1 && b2` - // is associated with a value (and check that it's the right one). - // The expressions `b1` and `b2` are _not_ associated with a value in this - // block, even though they are consumed by the block, because: - // * This block has two prececessor blocks (the one that evaluates `b1` and - // the one that evaluates `b2`). - // * `b1` is only associated with a value in the block that evaluates `b1` but - // not the block that evalutes `b2`, so the join operation discards the - // value for `b1`. - // * `b2` is only associated with a value in the block that evaluates `b2` but - // not the block that evaluates `b1`, the the join operation discards the - // value for `b2`. - // Nevertheless, the analysis generates the correct formula for `b1 && b2` - // because the transfer function for the `&&` operator retrieves the values - // for its operands from the environments for the blocks that compute the - // operands, rather than from the environment for the block that contains the - // `&&`. - const auto &ReturnState = blockStateForStmt(BlockStates, Return); - EXPECT_EQ(ReturnState.Env.getValue(*AndOp.getLHS()), nullptr); - EXPECT_EQ(ReturnState.Env.getValue(*AndOp.getRHS()), nullptr); - EXPECT_EQ(ReturnState.Env.getValue(AndOp), - &ReturnState.Env.makeAnd(*LHSValue, *RHSValue)); + EXPECT_EQ(RHSState.Env.getValue(*AndOp.getLHS()), LHSValue); + auto *RHSValue = RHSState.Env.get(*AndOp.getRHS()); + EXPECT_NE(RHSValue, nullptr); + + // In the block that evaluates `b1 && b2`, the `&&` as well as its operands + // are associated with values. + const auto &AndOpState = blockStateForStmt(BlockStates, AndOp); + EXPECT_EQ(AndOpState.Env.getValue(*AndOp.getLHS()), LHSValue); + EXPECT_EQ(AndOpState.Env.getValue(*AndOp.getRHS()), RHSValue); + EXPECT_EQ(AndOpState.Env.getValue(AndOp), + &AndOpState.Env.makeAnd(*LHSValue, *RHSValue)); + + // In the block that calls `f()`, none of `b1`, `b2`, or `b1 && b2` should be + // associated with values. + const auto &CallFState = blockStateForStmt(BlockStates, CallF); + EXPECT_EQ(CallFState.Env.getValue(*AndOp.getLHS()), nullptr); + EXPECT_EQ(CallFState.Env.getValue(*AndOp.getRHS()), nullptr); + EXPECT_EQ(CallFState.Env.getValue(AndOp), nullptr); +} + +TEST_F(DiscardExprStateTest, ConditionalOperator) { + std::string Code = R"( + void f(int*, int); + void g(); + bool cond(); + + void target() { + int i = 0; + if (cond()) + f(&i, cond() ? 1 : 0); + g(); + } + )"; + auto BlockStates = llvm::cantFail(runAnalysis( + Code, [](ASTContext &C) { return NoopAnalysis(C); })); + + const auto &AddrOfI = + matchNode(unaryOperator(hasOperatorName("&"))); + const auto &CallF = + matchNode(callExpr(callee(functionDecl(hasName("f"))))); + const auto &CallG = + matchNode(callExpr(callee(functionDecl(hasName("g"))))); + + // In the block that evaluates `&i`, it should obviously have a value. + const auto &AddrOfIState = blockStateForStmt(BlockStates, AddrOfI); + auto *AddrOfIVal = AddrOfIState.Env.get(AddrOfI); + EXPECT_NE(AddrOfIVal, nullptr); + + // Because of the conditional operator, the `f(...)` call is evaluated in a + // different block than `&i`, but `&i` still needs to have a value here + // because it's a subexpression of the call. + const auto &CallFState = blockStateForStmt(BlockStates, CallF); + EXPECT_NE(&CallFState, &AddrOfIState); + EXPECT_EQ(CallFState.Env.get(AddrOfI), AddrOfIVal); + + // In the block that calls `g()`, `&i` should no longer be associated with a + // value. + const auto &CallGState = blockStateForStmt(BlockStates, CallG); + EXPECT_EQ(CallGState.Env.get(AddrOfI), nullptr); } struct NonConvergingLattice { -- GitLab From 59e405b39416c8a5e2af93b2bfaa97a8c9d67f06 Mon Sep 17 00:00:00 2001 From: Kareem Ergawy Date: Thu, 7 Mar 2024 13:46:31 +0100 Subject: [PATCH 434/929] [flang][OpenMP] Add `%flang_fc1` `RUN` to delayed privatization tests (#84296) I did not know how `-mmlir` flag works and was deferring the addition of `--openm-enabled-delayed-privatization` until later because I thought some work needs to be done to do that. This commit just adds some extra `RUN` lines to delayed privatization tests to run them from `flang` as well. --- .../Lower/OpenMP/FIR/delayed-privatization-firstprivate.f90 | 5 ++++- .../test/Lower/OpenMP/FIR/delayed-privatization-private.f90 | 5 ++++- .../test/Lower/OpenMP/delayed-privatization-firstprivate.f90 | 5 ++++- .../OpenMP/delayed-privatization-private-firstprivate.f90 | 5 ++++- flang/test/Lower/OpenMP/delayed-privatization-private.f90 | 5 ++++- flang/test/Lower/OpenMP/delayed-privatization-reduction.f90 | 5 ++++- 6 files changed, 24 insertions(+), 6 deletions(-) diff --git a/flang/test/Lower/OpenMP/FIR/delayed-privatization-firstprivate.f90 b/flang/test/Lower/OpenMP/FIR/delayed-privatization-firstprivate.f90 index 122542345f10..50938342dee7 100644 --- a/flang/test/Lower/OpenMP/FIR/delayed-privatization-firstprivate.f90 +++ b/flang/test/Lower/OpenMP/FIR/delayed-privatization-firstprivate.f90 @@ -1,6 +1,9 @@ ! Test delayed privatization for the `private` clause. -! RUN: bbc -emit-fir -hlfir=false -fopenmp --openmp-enable-delayed-privatization -o - %s 2>&1 | FileCheck %s +! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -fopenmp -mmlir \ +! RUN: --openmp-enable-delayed-privatization -o - %s 2>&1 | FileCheck %s +! RUN: bbc -emit-fir -hlfir=false -fopenmp --openmp-enable-delayed-privatization \ +! RUN: -o - %s 2>&1 | FileCheck %s subroutine delayed_privatization_firstprivate implicit none diff --git a/flang/test/Lower/OpenMP/FIR/delayed-privatization-private.f90 b/flang/test/Lower/OpenMP/FIR/delayed-privatization-private.f90 index 2e9995ea1fd4..b13687faa3f2 100644 --- a/flang/test/Lower/OpenMP/FIR/delayed-privatization-private.f90 +++ b/flang/test/Lower/OpenMP/FIR/delayed-privatization-private.f90 @@ -1,6 +1,9 @@ ! Test delayed privatization for the `private` clause. -! RUN: bbc -emit-fir -hlfir=false -fopenmp --openmp-enable-delayed-privatization -o - %s 2>&1 | FileCheck %s +! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -fopenmp -mmlir \ +! RUN: --openmp-enable-delayed-privatization -o - %s 2>&1 | FileCheck %s +! RUN: bbc -emit-fir -hlfir=false -fopenmp --openmp-enable-delayed-privatization \ +! RUN: -o - %s 2>&1 | FileCheck %s subroutine delayed_privatization_private implicit none diff --git a/flang/test/Lower/OpenMP/delayed-privatization-firstprivate.f90 b/flang/test/Lower/OpenMP/delayed-privatization-firstprivate.f90 index e3d2a5a8af26..0fb81d68016a 100644 --- a/flang/test/Lower/OpenMP/delayed-privatization-firstprivate.f90 +++ b/flang/test/Lower/OpenMP/delayed-privatization-firstprivate.f90 @@ -1,6 +1,9 @@ ! Test delayed privatization for the `firstprivate` clause. -! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization -o - %s 2>&1 | FileCheck %s +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --openmp-enable-delayed-privatization \ +! RUN: -o - %s 2>&1 | FileCheck %s +! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization -o - %s 2>&1 \ +! RUN: | FileCheck %s subroutine delayed_privatization_firstprivate implicit none diff --git a/flang/test/Lower/OpenMP/delayed-privatization-private-firstprivate.f90 b/flang/test/Lower/OpenMP/delayed-privatization-private-firstprivate.f90 index 46eef6eb3bcf..337e7d5ec885 100644 --- a/flang/test/Lower/OpenMP/delayed-privatization-private-firstprivate.f90 +++ b/flang/test/Lower/OpenMP/delayed-privatization-private-firstprivate.f90 @@ -1,6 +1,9 @@ ! Test delayed privatization for both `private` and `firstprivate` clauses. -! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization -o - %s 2>&1 | FileCheck %s +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --openmp-enable-delayed-privatization \ +! RUN: -o - %s 2>&1 | FileCheck %s +! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization -o - %s 2>&1 \ +! RUN: | FileCheck %s subroutine delayed_privatization_private_firstprivate implicit none diff --git a/flang/test/Lower/OpenMP/delayed-privatization-private.f90 b/flang/test/Lower/OpenMP/delayed-privatization-private.f90 index 240e0e71bfcd..7208521bcd77 100644 --- a/flang/test/Lower/OpenMP/delayed-privatization-private.f90 +++ b/flang/test/Lower/OpenMP/delayed-privatization-private.f90 @@ -1,6 +1,9 @@ ! Test delayed privatization for the `private` clause. -! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization -o - %s 2>&1 | FileCheck %s +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --openmp-enable-delayed-privatization \ +! RUN: -o - %s 2>&1 | FileCheck %s +! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization -o - %s 2>&1 \ +! RUN: | FileCheck %s subroutine delayed_privatization_private implicit none diff --git a/flang/test/Lower/OpenMP/delayed-privatization-reduction.f90 b/flang/test/Lower/OpenMP/delayed-privatization-reduction.f90 index c61f352b9b05..a7eeb1facead 100644 --- a/flang/test/Lower/OpenMP/delayed-privatization-reduction.f90 +++ b/flang/test/Lower/OpenMP/delayed-privatization-reduction.f90 @@ -3,7 +3,10 @@ ! that the block arguments are added in the proper order (reductions first and ! then delayed privatization. -! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization -o - %s 2>&1 | FileCheck %s +! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --openmp-enable-delayed-privatization \ +! RUN: -o - %s 2>&1 | FileCheck %s +! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization -o - %s 2>&1 \ +! RUN: | FileCheck %s subroutine red_and_delayed_private integer :: red -- GitLab From 464d9d96b3565ead06396ffb8d02b4dcf9cb9556 Mon Sep 17 00:00:00 2001 From: Stephen Tozer Date: Thu, 7 Mar 2024 13:05:04 +0000 Subject: [PATCH 435/929] [RemoveDIs][DebugInfo][IR] Add parsing for non-intrinsic debug values (#79818) This patch adds support for parsing the proposed non-instruction debug info ("RemoveDIs") from textual IR, and adds a test for the parser as well as a set of verifier tests that are dependent on parsing to fire. An important detail of this patch is the fact that although we can now parse in the RemoveDIs (new) and Intrinsic (old) debug info formats, we will always convert back to the old format at the end of parsing - this is done for two reasons: firstly to ensure that every tool is able to process IR printed in the new format, regardless of whether that tool has had RemoveDIs support added, and secondly to maintain the effect of the existing flags: for the tools where support for the new format has been added, we will run LLVM passes in the new format iff `--try-experimental-debuginfo-iterators=true`, and we will print in the new format iff `--write-experimental-debuginfo-iterators=true`; the format of the textual IR input should have no effect on either of these features. --- llvm/include/llvm/AsmParser/LLParser.h | 4 + llvm/include/llvm/AsmParser/LLToken.h | 2 + .../include/llvm/IR/DebugProgramInstruction.h | 33 ++++ llvm/lib/AsmParser/LLLexer.cpp | 20 ++- llvm/lib/AsmParser/LLParser.cpp | 168 +++++++++++++++++- llvm/lib/IR/DebugProgramInstruction.cpp | 27 +++ llvm/lib/IR/Verifier.cpp | 2 +- llvm/test/Assembler/dbg-record-invalid-0.ll | 38 ++++ llvm/test/Assembler/dbg-record-invalid-1.ll | 39 ++++ llvm/test/Assembler/dbg-record-invalid-2.ll | 36 ++++ llvm/test/Assembler/dbg-record-invalid-3.ll | 39 ++++ llvm/test/Assembler/dbg-record-invalid-4.ll | 36 ++++ llvm/test/Assembler/dbg-record-invalid-5.ll | 35 ++++ llvm/test/Assembler/dbg-record-invalid-6.ll | 36 ++++ llvm/test/Assembler/dbg-record-invalid-7.ll | 36 ++++ llvm/test/Assembler/dbg-record-invalid-8.ll | 36 ++++ .../roundtrip-non-instruction-debug-info.ll | 94 ++++++++++ llvm/test/Verifier/RemoveDI/blockbyref.ll | 18 ++ .../Verifier/RemoveDI/dbg-invalid-vector.ll | 35 ++++ .../RemoveDI/di-subroutine-localvar.ll | 41 +++++ .../diexpression-entry-value-llvm-ir.ll | 34 ++++ .../test/Verifier/RemoveDI/fnarg-debuginfo.ll | 26 +++ llvm/test/Verifier/RemoveDI/fnarg-nodebug.ll | 58 ++++++ .../RemoveDI/invalid-disubrange-count-node.ll | 36 ++++ .../RemoveDI/llvm.dbg.declare-address.ll | 16 ++ .../RemoveDI/llvm.dbg.declare-expression.ll | 16 ++ .../RemoveDI/llvm.dbg.declare-variable.ll | 17 ++ .../llvm.dbg.intrinsic-dbg-attachment.ll | 55 ++++++ .../RemoveDI/llvm.dbg.value-expression.ll | 16 ++ .../Verifier/RemoveDI/llvm.dbg.value-value.ll | 17 ++ .../RemoveDI/llvm.dbg.value-variable.ll | 17 ++ llvm/test/Verifier/RemoveDI/set1.ll | 62 +++++++ 32 files changed, 1141 insertions(+), 4 deletions(-) create mode 100644 llvm/test/Assembler/dbg-record-invalid-0.ll create mode 100644 llvm/test/Assembler/dbg-record-invalid-1.ll create mode 100644 llvm/test/Assembler/dbg-record-invalid-2.ll create mode 100644 llvm/test/Assembler/dbg-record-invalid-3.ll create mode 100644 llvm/test/Assembler/dbg-record-invalid-4.ll create mode 100644 llvm/test/Assembler/dbg-record-invalid-5.ll create mode 100644 llvm/test/Assembler/dbg-record-invalid-6.ll create mode 100644 llvm/test/Assembler/dbg-record-invalid-7.ll create mode 100644 llvm/test/Assembler/dbg-record-invalid-8.ll create mode 100644 llvm/test/DebugInfo/roundtrip-non-instruction-debug-info.ll create mode 100644 llvm/test/Verifier/RemoveDI/blockbyref.ll create mode 100644 llvm/test/Verifier/RemoveDI/dbg-invalid-vector.ll create mode 100644 llvm/test/Verifier/RemoveDI/di-subroutine-localvar.ll create mode 100644 llvm/test/Verifier/RemoveDI/diexpression-entry-value-llvm-ir.ll create mode 100644 llvm/test/Verifier/RemoveDI/fnarg-debuginfo.ll create mode 100644 llvm/test/Verifier/RemoveDI/fnarg-nodebug.ll create mode 100644 llvm/test/Verifier/RemoveDI/invalid-disubrange-count-node.ll create mode 100644 llvm/test/Verifier/RemoveDI/llvm.dbg.declare-address.ll create mode 100644 llvm/test/Verifier/RemoveDI/llvm.dbg.declare-expression.ll create mode 100644 llvm/test/Verifier/RemoveDI/llvm.dbg.declare-variable.ll create mode 100644 llvm/test/Verifier/RemoveDI/llvm.dbg.intrinsic-dbg-attachment.ll create mode 100644 llvm/test/Verifier/RemoveDI/llvm.dbg.value-expression.ll create mode 100644 llvm/test/Verifier/RemoveDI/llvm.dbg.value-value.ll create mode 100644 llvm/test/Verifier/RemoveDI/llvm.dbg.value-variable.ll create mode 100644 llvm/test/Verifier/RemoveDI/set1.ll diff --git a/llvm/include/llvm/AsmParser/LLParser.h b/llvm/include/llvm/AsmParser/LLParser.h index f07f4c61f9d6..e5e1ade8b38b 100644 --- a/llvm/include/llvm/AsmParser/LLParser.h +++ b/llvm/include/llvm/AsmParser/LLParser.h @@ -178,6 +178,9 @@ namespace llvm { /// UpgradeDebuginfo so it can generate broken bitcode. bool UpgradeDebugInfo; + bool SeenNewDbgInfoFormat = false; + bool SeenOldDbgInfoFormat = false; + std::string SourceFileName; public: @@ -573,6 +576,7 @@ namespace llvm { bool parseMDNodeTail(MDNode *&N); bool parseMDNodeVector(SmallVectorImpl &Elts); bool parseMetadataAttachment(unsigned &Kind, MDNode *&MD); + bool parseDebugRecord(DbgRecord *&DR, PerFunctionState &PFS); bool parseInstructionMetadata(Instruction &Inst); bool parseGlobalObjectMetadataAttachment(GlobalObject &GO); bool parseOptionalFunctionMetadata(Function &F); diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h index 3c34706ee03e..5863a8d6e8ee 100644 --- a/llvm/include/llvm/AsmParser/LLToken.h +++ b/llvm/include/llvm/AsmParser/LLToken.h @@ -36,6 +36,7 @@ enum Kind { exclaim, // ! bar, // | colon, // : + hash, // # kw_vscale, kw_x, @@ -479,6 +480,7 @@ enum Kind { DISPFlag, // DISPFlagFoo DwarfMacinfo, // DW_MACINFO_foo ChecksumKind, // CSK_foo + DbgRecordType, // dbg_foo // Type valued tokens (TyVal). Type, diff --git a/llvm/include/llvm/IR/DebugProgramInstruction.h b/llvm/include/llvm/IR/DebugProgramInstruction.h index cf30b4d0b0aa..a8faf415a3ea 100644 --- a/llvm/include/llvm/IR/DebugProgramInstruction.h +++ b/llvm/include/llvm/IR/DebugProgramInstruction.h @@ -223,9 +223,19 @@ inline raw_ostream &operator<<(raw_ostream &OS, const DbgRecord &R) { class DPLabel : public DbgRecord { DbgRecordParamRef Label; + /// This constructor intentionally left private, so that it is only called via + /// "createUnresolvedDPLabel", which clearly expresses that it is for parsing + /// only. + DPLabel(MDNode *Label, MDNode *DL); + public: DPLabel(DILabel *Label, DebugLoc DL); + /// For use during parsing; creates a DPLabel from as-of-yet unresolved + /// MDNodes. Trying to access the resulting DPLabel's fields before they are + /// resolved, or if they resolve to the wrong type, will result in a crash. + static DPLabel *createUnresolvedDPLabel(MDNode *Label, MDNode *DL); + DPLabel *clone() const; void print(raw_ostream &O, bool IsForDebug = false) const; void print(raw_ostream &ROS, ModuleSlotTracker &MST, bool IsForDebug) const; @@ -286,6 +296,29 @@ public: DIAssignID *AssignID, Metadata *Address, DIExpression *AddressExpression, const DILocation *DI); +private: + /// Private constructor for creating new instances during parsing only. Only + /// called through `createUnresolvedDPValue` below, which makes clear that + /// this is used for parsing only, and will later return a subclass depending + /// on which Type is passed. + DPValue(LocationType Type, Metadata *Val, MDNode *Variable, + MDNode *Expression, MDNode *AssignID, Metadata *Address, + MDNode *AddressExpression, MDNode *DI); + +public: + /// Used to create DPValues during parsing, where some metadata references may + /// still be unresolved. Although for some fields a generic `Metadata*` + /// argument is accepted for forward type-references, the verifier and + /// accessors will reject incorrect types later on. The function is used for + /// all types of DPValues for simplicity while parsing, but asserts if any + /// necessary fields are empty or unused fields are not empty, i.e. if the + /// #dbg_assign fields are used for a non-dbg-assign type. + static DPValue *createUnresolvedDPValue(LocationType Type, Metadata *Val, + MDNode *Variable, MDNode *Expression, + MDNode *AssignID, Metadata *Address, + MDNode *AddressExpression, + MDNode *DI); + static DPValue *createDPVAssign(Value *Val, DILocalVariable *Variable, DIExpression *Expression, DIAssignID *AssignID, Value *Address, diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp index 5d8a50eee130..02f64fcfac4f 100644 --- a/llvm/lib/AsmParser/LLLexer.cpp +++ b/llvm/lib/AsmParser/LLLexer.cpp @@ -438,9 +438,12 @@ lltok::Kind LLLexer::LexCaret() { /// Lex all tokens that start with a # character. /// AttrGrpID ::= #[0-9]+ +/// Hash ::= # lltok::Kind LLLexer::LexHash() { // Handle AttrGrpID: #[0-9]+ - return LexUIntID(lltok::AttrGrpID); + if (isdigit(static_cast(CurPtr[0]))) + return LexUIntID(lltok::AttrGrpID); + return lltok::hash; } /// Lex a label, integer type, keyword, or hexadecimal integer constant. @@ -923,6 +926,21 @@ lltok::Kind LLLexer::LexIdentifier() { #undef DWKEYWORD +// Keywords for debug record types. +#define DBGRECORDTYPEKEYWORD(STR) \ + do { \ + if (Keyword == "dbg_" #STR) { \ + StrVal = #STR; \ + return lltok::DbgRecordType; \ + } \ + } while (false) + + DBGRECORDTYPEKEYWORD(value); + DBGRECORDTYPEKEYWORD(declare); + DBGRECORDTYPEKEYWORD(assign); + DBGRECORDTYPEKEYWORD(label); +#undef DBGRECORDTYPEKEYWORD + if (Keyword.starts_with("DIFlag")) { StrVal.assign(Keyword.begin(), Keyword.end()); return lltok::DIFlag; diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp index a91e2f690999..e140c9419520 100644 --- a/llvm/lib/AsmParser/LLParser.cpp +++ b/llvm/lib/AsmParser/LLParser.cpp @@ -62,6 +62,8 @@ static cl::opt AllowIncompleteIR( "Allow incomplete IR on a best effort basis (references to unknown " "metadata will be dropped)")); +extern llvm::cl::opt UseNewDbgInfoFormat; + static std::string getTypeString(Type *T) { std::string Result; raw_string_ostream Tmp(Result); @@ -69,6 +71,15 @@ static std::string getTypeString(Type *T) { return Tmp.str(); } +// Currently, we should always process modules in the old debug info format by +// default regardless of the module's format in IR; convert it to the old format +// here. +bool finalizeDebugInfoFormat(Module *M) { + if (M) + M->setIsNewDbgInfoFormat(false); + return false; +} + /// Run: module ::= toplevelentity* bool LLParser::Run(bool UpgradeDebugInfo, DataLayoutCallbackTy DataLayoutCallback) { @@ -86,7 +97,7 @@ bool LLParser::Run(bool UpgradeDebugInfo, } return parseTopLevelEntities() || validateEndOfModule(UpgradeDebugInfo) || - validateEndOfIndex(); + validateEndOfIndex() || finalizeDebugInfoFormat(M); } bool LLParser::parseStandaloneConstantValue(Constant *&C, @@ -6041,6 +6052,17 @@ bool LLParser::parseTypeAndBasicBlock(BasicBlock *&BB, LocTy &Loc, return false; } +bool isOldDbgFormatIntrinsic(StringRef Name) { + // Exit early for the common (non-debug-intrinsic) case. + // We can make this the only check when we begin supporting all "llvm.dbg" + // intrinsics in the new debug info format. + if (!Name.starts_with("llvm.dbg.")) + return false; + Intrinsic::ID FnID = Function::lookupIntrinsicID(Name); + return FnID == Intrinsic::dbg_declare || FnID == Intrinsic::dbg_value || + FnID == Intrinsic::dbg_assign; +} + /// FunctionHeader /// ::= OptionalLinkage OptionalPreemptionSpecifier OptionalVisibility /// OptionalCallingConv OptRetAttrs OptUnnamedAddr Type GlobalName @@ -6390,9 +6412,31 @@ bool LLParser::parseBasicBlock(PerFunctionState &PFS) { std::string NameStr; - // parse the instructions in this block until we get a terminator. + // Parse the instructions and debug values in this block until we get a + // terminator. Instruction *Inst; + auto DeleteDbgRecord = [](DbgRecord *DR) { DR->deleteRecord(); }; + using DbgRecordPtr = std::unique_ptr; + SmallVector TrailingDbgRecord; do { + // Handle debug records first - there should always be an instruction + // following the debug records, i.e. they cannot appear after the block + // terminator. + while (Lex.getKind() == lltok::hash) { + if (SeenOldDbgInfoFormat) + return error(Lex.getLoc(), "debug record should not appear in a module " + "containing debug info intrinsics"); + SeenNewDbgInfoFormat = true; + Lex.Lex(); + if (!M->IsNewDbgInfoFormat) + M->convertToNewDbgValues(); + + DbgRecord *DR; + if (parseDebugRecord(DR, PFS)) + return true; + TrailingDbgRecord.emplace_back(DR, DeleteDbgRecord); + } + // This instruction may have three possibilities for a name: a) none // specified, b) name specified "%foo =", c) number specified: "%4 =". LocTy NameLoc = Lex.getLoc(); @@ -6437,11 +6481,121 @@ bool LLParser::parseBasicBlock(PerFunctionState &PFS) { // Set the name on the instruction. if (PFS.setInstName(NameID, NameStr, NameLoc, Inst)) return true; + + // Attach any preceding debug values to this instruction. + for (DbgRecordPtr &DR : TrailingDbgRecord) + BB->insertDPValueBefore(DR.release(), Inst->getIterator()); + TrailingDbgRecord.clear(); } while (!Inst->isTerminator()); + assert(TrailingDbgRecord.empty() && + "All debug values should have been attached to an instruction."); + return false; } +/// parseDebugRecord +/// ::= #dbg_label '(' MDNode ')' +/// ::= #dbg_type '(' Metadata ',' MDNode ',' Metadata ',' +/// (MDNode ',' Metadata ',' Metadata ',')? MDNode ')' +bool LLParser::parseDebugRecord(DbgRecord *&DR, PerFunctionState &PFS) { + using RecordKind = DbgRecord::Kind; + using LocType = DPValue::LocationType; + LocTy DPVLoc = Lex.getLoc(); + if (Lex.getKind() != lltok::DbgRecordType) + return error(DPVLoc, "expected debug record type here"); + RecordKind RecordType = StringSwitch(Lex.getStrVal()) + .Case("declare", RecordKind::ValueKind) + .Case("value", RecordKind::ValueKind) + .Case("assign", RecordKind::ValueKind) + .Case("label", RecordKind::LabelKind); + + // Parsing labels is trivial; parse here and early exit, otherwise go into the + // full DPValue processing stage. + if (RecordType == RecordKind::LabelKind) { + Lex.Lex(); + if (parseToken(lltok::lparen, "Expected '(' here")) + return true; + MDNode *Label; + if (parseMDNode(Label)) + return true; + if (parseToken(lltok::comma, "Expected ',' here")) + return true; + MDNode *DbgLoc; + if (parseMDNode(DbgLoc)) + return true; + if (parseToken(lltok::rparen, "Expected ')' here")) + return true; + DR = DPLabel::createUnresolvedDPLabel(Label, DbgLoc); + return false; + } + + LocType ValueType = StringSwitch(Lex.getStrVal()) + .Case("declare", LocType::Declare) + .Case("value", LocType::Value) + .Case("assign", LocType::Assign); + + Lex.Lex(); + if (parseToken(lltok::lparen, "Expected '(' here")) + return true; + + // Parse Value field. + Metadata *ValLocMD; + if (parseMetadata(ValLocMD, &PFS)) + return true; + if (parseToken(lltok::comma, "Expected ',' here")) + return true; + + // Parse Variable field. + MDNode *Variable; + if (parseMDNode(Variable)) + return true; + if (parseToken(lltok::comma, "Expected ',' here")) + return true; + + // Parse Expression field. + MDNode *Expression; + if (parseMDNode(Expression)) + return true; + if (parseToken(lltok::comma, "Expected ',' here")) + return true; + + // Parse additional fields for #dbg_assign. + MDNode *AssignID = nullptr; + Metadata *AddressLocation = nullptr; + MDNode *AddressExpression = nullptr; + if (ValueType == LocType::Assign) { + // Parse DIAssignID. + if (parseMDNode(AssignID)) + return true; + if (parseToken(lltok::comma, "Expected ',' here")) + return true; + + // Parse address ValueAsMetadata. + if (parseMetadata(AddressLocation, &PFS)) + return true; + if (parseToken(lltok::comma, "Expected ',' here")) + return true; + + // Parse address DIExpression. + if (parseMDNode(AddressExpression)) + return true; + if (parseToken(lltok::comma, "Expected ',' here")) + return true; + } + + /// Parse DILocation. + MDNode *DebugLoc; + if (parseMDNode(DebugLoc)) + return true; + + if (parseToken(lltok::rparen, "Expected ')' here")) + return true; + DR = DPValue::createUnresolvedDPValue(ValueType, ValLocMD, Variable, + Expression, AssignID, AddressLocation, + AddressExpression, DebugLoc); + return false; +} //===----------------------------------------------------------------------===// // Instruction Parsing. //===----------------------------------------------------------------------===// @@ -7669,6 +7823,16 @@ bool LLParser::parseCall(Instruction *&Inst, PerFunctionState &PFS, } CI->setFastMathFlags(FMF); } + + if (CalleeID.Kind == ValID::t_GlobalName && + isOldDbgFormatIntrinsic(CalleeID.StrVal)) { + if (SeenNewDbgInfoFormat) { + CI->deleteValue(); + return error(CallLoc, "llvm.dbg intrinsic should not appear in a module " + "using non-intrinsic debug info"); + } + SeenOldDbgInfoFormat = true; + } CI->setAttributes(PAL); ForwardRefAttrGroups[CI] = FwdRefAttrGrps; Inst = CI; diff --git a/llvm/lib/IR/DebugProgramInstruction.cpp b/llvm/lib/IR/DebugProgramInstruction.cpp index a8d64024e179..5ff1e8c19db6 100644 --- a/llvm/lib/IR/DebugProgramInstruction.cpp +++ b/llvm/lib/IR/DebugProgramInstruction.cpp @@ -138,11 +138,38 @@ DbgRecord::createDebugIntrinsic(Module *M, Instruction *InsertBefore) const { llvm_unreachable("unsupported DbgRecord kind"); } +DPLabel::DPLabel(MDNode *Label, MDNode *DL) + : DbgRecord(LabelKind, DebugLoc(DL)), Label(Label) { + assert(Label && "Unexpected nullptr"); + assert((isa(Label) || Label->isTemporary()) && + "Label type must be or resolve to a DILabel"); +} DPLabel::DPLabel(DILabel *Label, DebugLoc DL) : DbgRecord(LabelKind, DL), Label(Label) { assert(Label && "Unexpected nullptr"); } +DPLabel *DPLabel::createUnresolvedDPLabel(MDNode *Label, MDNode *DL) { + return new DPLabel(Label, DL); +} + +DPValue::DPValue(DPValue::LocationType Type, Metadata *Val, MDNode *Variable, + MDNode *Expression, MDNode *AssignID, Metadata *Address, + MDNode *AddressExpression, MDNode *DI) + : DbgRecord(ValueKind, DebugLoc(DI)), + DebugValueUser({Val, Address, AssignID}), Type(Type), Variable(Variable), + Expression(Expression), AddressExpression(AddressExpression) {} + +DPValue *DPValue::createUnresolvedDPValue(DPValue::LocationType Type, + Metadata *Val, MDNode *Variable, + MDNode *Expression, MDNode *AssignID, + Metadata *Address, + MDNode *AddressExpression, + MDNode *DI) { + return new DPValue(Type, Val, Variable, Expression, AssignID, Address, + AddressExpression, DI); +} + DPValue *DPValue::createDPValue(Value *Location, DILocalVariable *DV, DIExpression *Expr, const DILocation *DI) { return new DPValue(ValueAsMetadata::get(Location), DV, Expr, DI, diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 74c7354e7bf1..fd5f7d57c258 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -6291,7 +6291,7 @@ void Verifier::visit(DPValue &DPV) { Var->getRawType()); auto *DLNode = DPV.getDebugLoc().getAsMDNode(); - CheckDI(isa_and_nonnull(DLNode), "invalid #dbg record location", + CheckDI(isa_and_nonnull(DLNode), "invalid #dbg record DILocation", &DPV, DLNode); DILocation *Loc = DPV.getDebugLoc(); diff --git a/llvm/test/Assembler/dbg-record-invalid-0.ll b/llvm/test/Assembler/dbg-record-invalid-0.ll new file mode 100644 index 000000000000..feb513a405f9 --- /dev/null +++ b/llvm/test/Assembler/dbg-record-invalid-0.ll @@ -0,0 +1,38 @@ +;; Test that we get a parser error when a debug record appears post-terminator. +;; Note: From the parser's perspective, the error is that the debug record is +;; appearing at the start of a new unnamed basic block which contains no actual +;; instructions. +; RUN: not llvm-as < %s 2>&1 | FileCheck %s +; ModuleID = '' +source_filename = "" + +define dso_local i32 @f(i32 %a) !dbg !7 { +entry: + ret i32 %a, !dbg !18 + #dbg_value(!DIArgList(i32 %a), !12, !DIExpression(), !14) +; CHECK: :[[@LINE+1]]:1: error: expected instruction opcode +} + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4, !5} +!llvm.ident = !{!6} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 18.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None) +!1 = !DIFile(filename: "print.c", directory: "/tmp") +!2 = !{} +!3 = !{i32 2, !"Dwarf Version", i32 5} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = !{i32 1, !"wchar_size", i32 4} +!6 = !{!"clang version 18.0.0"} +!7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 3, type: !8, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11) +!8 = !DISubroutineType(types: !9) +!9 = !{!10, !10} +!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!11 = !{!12, !13} +!12 = !DILocalVariable(name: "a", arg: 1, scope: !7, file: !1, line: 3, type: !10) +!13 = !DILocalVariable(name: "b", scope: !7, file: !1, line: 3, type: !10) +!14 = !DILocation(line: 3, column: 15, scope: !7) +!15 = distinct !DIAssignID() +!16 = !DILocation(line: 3, column: 20, scope: !7) +!17 = !DILocation(line: 3, column: 25, scope: !7) +!18 = !DILocation(line: 3, column: 30, scope: !7) diff --git a/llvm/test/Assembler/dbg-record-invalid-1.ll b/llvm/test/Assembler/dbg-record-invalid-1.ll new file mode 100644 index 000000000000..7ab5751777e8 --- /dev/null +++ b/llvm/test/Assembler/dbg-record-invalid-1.ll @@ -0,0 +1,39 @@ +;; Test that we get a parser error when a debug intrinsic appears in the same +;; module as a debug record. +; RUN: not llvm-as < %s 2>&1 | FileCheck %s +; ModuleID = '' +source_filename = "" + +define dso_local i32 @f(i32 %a) !dbg !7 { +entry: + #dbg_value(!DIArgList(i32 %a), !12, !DIExpression(), !14) +; CHECK: :[[@LINE+1]]:8: error: llvm.dbg intrinsic should not appear in a module using non-intrinsic debug info + call void @llvm.dbg.value(metadata i32 %a, metadata !12, metadata !DIExpression()), !dbg !14 + ret i32 %a, !dbg !18 +} + +declare void @llvm.dbg.value(metadata, metadata, metadata) + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4, !5} +!llvm.ident = !{!6} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 18.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None) +!1 = !DIFile(filename: "print.c", directory: "/tmp") +!2 = !{} +!3 = !{i32 2, !"Dwarf Version", i32 5} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = !{i32 1, !"wchar_size", i32 4} +!6 = !{!"clang version 18.0.0"} +!7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 3, type: !8, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11) +!8 = !DISubroutineType(types: !9) +!9 = !{!10, !10} +!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!11 = !{!12, !13} +!12 = !DILocalVariable(name: "a", arg: 1, scope: !7, file: !1, line: 3, type: !10) +!13 = !DILocalVariable(name: "b", scope: !7, file: !1, line: 3, type: !10) +!14 = !DILocation(line: 3, column: 15, scope: !7) +!15 = distinct !DIAssignID() +!16 = !DILocation(line: 3, column: 20, scope: !7) +!17 = !DILocation(line: 3, column: 25, scope: !7) +!18 = !DILocation(line: 3, column: 30, scope: !7) diff --git a/llvm/test/Assembler/dbg-record-invalid-2.ll b/llvm/test/Assembler/dbg-record-invalid-2.ll new file mode 100644 index 000000000000..a019f73feab9 --- /dev/null +++ b/llvm/test/Assembler/dbg-record-invalid-2.ll @@ -0,0 +1,36 @@ +;; Test that we get a parser error when we have a debug record with an +;; incorrect number of arguments. +; RUN: not llvm-as < %s 2>&1 | FileCheck %s +; ModuleID = '' +source_filename = "" + +define dso_local i32 @f(i32 %a) !dbg !7 { +entry: +; CHECK: :[[@LINE+1]]:24: error: expected '!' here + #dbg_value(i32 %a, i32 0, !DIExpression(), !14) + ret i32 %a, !dbg !18 +} + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4, !5} +!llvm.ident = !{!6} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 18.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None) +!1 = !DIFile(filename: "print.c", directory: "/tmp") +!2 = !{} +!3 = !{i32 2, !"Dwarf Version", i32 5} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = !{i32 1, !"wchar_size", i32 4} +!6 = !{!"clang version 18.0.0"} +!7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 3, type: !8, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11) +!8 = !DISubroutineType(types: !9) +!9 = !{!10, !10} +!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!11 = !{!12, !13} +!12 = !DILocalVariable(name: "a", arg: 1, scope: !7, file: !1, line: 3, type: !10) +!13 = !DILocalVariable(name: "b", scope: !7, file: !1, line: 3, type: !10) +!14 = !DILocation(line: 3, column: 15, scope: !7) +!15 = distinct !DIAssignID() +!16 = !DILocation(line: 3, column: 20, scope: !7) +!17 = !DILocation(line: 3, column: 25, scope: !7) +!18 = !DILocation(line: 3, column: 30, scope: !7) diff --git a/llvm/test/Assembler/dbg-record-invalid-3.ll b/llvm/test/Assembler/dbg-record-invalid-3.ll new file mode 100644 index 000000000000..e6f072373f54 --- /dev/null +++ b/llvm/test/Assembler/dbg-record-invalid-3.ll @@ -0,0 +1,39 @@ +;; Test that we get a parser error when a debug record appears in the same +;; module as a debug intrinsic. +; RUN: not llvm-as < %s 2>&1 | FileCheck %s +; ModuleID = '' +source_filename = "" + +define dso_local i32 @f(i32 %a) !dbg !7 { +entry: + call void @llvm.dbg.value(metadata i32 %a, metadata !12, metadata !DIExpression()), !dbg !14 +; CHECK: :[[@LINE+1]]:5: error: debug record should not appear in a module containing debug info intrinsics + #dbg_value(!DIArgList(i32 %a), !12, !DIExpression(), !14) + ret i32 %a, !dbg !18 +} + +declare void @llvm.dbg.value(metadata, metadata, metadata) + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4, !5} +!llvm.ident = !{!6} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 18.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None) +!1 = !DIFile(filename: "print.c", directory: "/tmp") +!2 = !{} +!3 = !{i32 2, !"Dwarf Version", i32 5} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = !{i32 1, !"wchar_size", i32 4} +!6 = !{!"clang version 18.0.0"} +!7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 3, type: !8, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11) +!8 = !DISubroutineType(types: !9) +!9 = !{!10, !10} +!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!11 = !{!12, !13} +!12 = !DILocalVariable(name: "a", arg: 1, scope: !7, file: !1, line: 3, type: !10) +!13 = !DILocalVariable(name: "b", scope: !7, file: !1, line: 3, type: !10) +!14 = !DILocation(line: 3, column: 15, scope: !7) +!15 = distinct !DIAssignID() +!16 = !DILocation(line: 3, column: 20, scope: !7) +!17 = !DILocation(line: 3, column: 25, scope: !7) +!18 = !DILocation(line: 3, column: 30, scope: !7) diff --git a/llvm/test/Assembler/dbg-record-invalid-4.ll b/llvm/test/Assembler/dbg-record-invalid-4.ll new file mode 100644 index 000000000000..f898477603c8 --- /dev/null +++ b/llvm/test/Assembler/dbg-record-invalid-4.ll @@ -0,0 +1,36 @@ +;; Test that we get a parser error when we have a debug record with an invalid +;; type. +; RUN: not llvm-as < %s 2>&1 | FileCheck %s +; ModuleID = '' +source_filename = "" + +define dso_local i32 @f(i32 %a) !dbg !7 { +entry: +; CHECK: :[[@LINE+1]]:6: error: expected debug record type here + #dbg_invalid(!DIArgList(i32 %a), !12, !DIExpression(), !14) + ret i32 %a, !dbg !18 +} + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4, !5} +!llvm.ident = !{!6} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 18.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None) +!1 = !DIFile(filename: "print.c", directory: "/tmp") +!2 = !{} +!3 = !{i32 2, !"Dwarf Version", i32 5} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = !{i32 1, !"wchar_size", i32 4} +!6 = !{!"clang version 18.0.0"} +!7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 3, type: !8, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11) +!8 = !DISubroutineType(types: !9) +!9 = !{!10, !10} +!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!11 = !{!12, !13} +!12 = !DILocalVariable(name: "a", arg: 1, scope: !7, file: !1, line: 3, type: !10) +!13 = !DILocalVariable(name: "b", scope: !7, file: !1, line: 3, type: !10) +!14 = !DILocation(line: 3, column: 15, scope: !7) +!15 = distinct !DIAssignID() +!16 = !DILocation(line: 3, column: 20, scope: !7) +!17 = !DILocation(line: 3, column: 25, scope: !7) +!18 = !DILocation(line: 3, column: 30, scope: !7) diff --git a/llvm/test/Assembler/dbg-record-invalid-5.ll b/llvm/test/Assembler/dbg-record-invalid-5.ll new file mode 100644 index 000000000000..5ea588b87668 --- /dev/null +++ b/llvm/test/Assembler/dbg-record-invalid-5.ll @@ -0,0 +1,35 @@ +;; Test that we get a parser error when a basic block contains only a debug +;; record. +; RUN: not llvm-as < %s 2>&1 | FileCheck %s +; ModuleID = '' +source_filename = "" + +define dso_local i32 @f(i32 %a) !dbg !7 { +entry: + #dbg_value(!DIArgList(i32 %a), !12, !DIExpression(), !14) +; CHECK: :[[@LINE+1]]:1: error: expected instruction opcode +} + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4, !5} +!llvm.ident = !{!6} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 18.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None) +!1 = !DIFile(filename: "print.c", directory: "/tmp") +!2 = !{} +!3 = !{i32 2, !"Dwarf Version", i32 5} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = !{i32 1, !"wchar_size", i32 4} +!6 = !{!"clang version 18.0.0"} +!7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 3, type: !8, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11) +!8 = !DISubroutineType(types: !9) +!9 = !{!10, !10} +!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!11 = !{!12, !13} +!12 = !DILocalVariable(name: "a", arg: 1, scope: !7, file: !1, line: 3, type: !10) +!13 = !DILocalVariable(name: "b", scope: !7, file: !1, line: 3, type: !10) +!14 = !DILocation(line: 3, column: 15, scope: !7) +!15 = distinct !DIAssignID() +!16 = !DILocation(line: 3, column: 20, scope: !7) +!17 = !DILocation(line: 3, column: 25, scope: !7) +!18 = !DILocation(line: 3, column: 30, scope: !7) diff --git a/llvm/test/Assembler/dbg-record-invalid-6.ll b/llvm/test/Assembler/dbg-record-invalid-6.ll new file mode 100644 index 000000000000..72dafcdb97fc --- /dev/null +++ b/llvm/test/Assembler/dbg-record-invalid-6.ll @@ -0,0 +1,36 @@ +;; Test that we get a parser error when we have a debug record with an +;; incorrect number of arguments. +; RUN: not llvm-as < %s 2>&1 | FileCheck %s +; ModuleID = '' +source_filename = "" + +define dso_local i32 @f(i32 %a) !dbg !7 { +entry: +; CHECK: :[[@LINE+1]]:46: error: expected '!' here + #dbg_value(i32 %a, !12, !DIExpression(), i32 0) + ret i32 %a, !dbg !18 +} + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4, !5} +!llvm.ident = !{!6} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 18.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None) +!1 = !DIFile(filename: "print.c", directory: "/tmp") +!2 = !{} +!3 = !{i32 2, !"Dwarf Version", i32 5} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = !{i32 1, !"wchar_size", i32 4} +!6 = !{!"clang version 18.0.0"} +!7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 3, type: !8, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11) +!8 = !DISubroutineType(types: !9) +!9 = !{!10, !10} +!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!11 = !{!12, !13} +!12 = !DILocalVariable(name: "a", arg: 1, scope: !7, file: !1, line: 3, type: !10) +!13 = !DILocalVariable(name: "b", scope: !7, file: !1, line: 3, type: !10) +!14 = !DILocation(line: 3, column: 15, scope: !7) +!15 = distinct !DIAssignID() +!16 = !DILocation(line: 3, column: 20, scope: !7) +!17 = !DILocation(line: 3, column: 25, scope: !7) +!18 = !DILocation(line: 3, column: 30, scope: !7) diff --git a/llvm/test/Assembler/dbg-record-invalid-7.ll b/llvm/test/Assembler/dbg-record-invalid-7.ll new file mode 100644 index 000000000000..036a85a2977f --- /dev/null +++ b/llvm/test/Assembler/dbg-record-invalid-7.ll @@ -0,0 +1,36 @@ +;; Test that we get a parser error when we have a debug record with an incorrect +;; number of arguments. +; RUN: not llvm-as < %s 2>&1 | FileCheck %s +; ModuleID = '' +source_filename = "" + +define dso_local i32 @f(i32 %a) !dbg !7 { +entry: +; CHECK: :[[@LINE+1]]:44: error: Expected ',' here + #dbg_value(i32 %a, !12, !DIExpression()) + ret i32 %a, !dbg !18 +} + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4, !5} +!llvm.ident = !{!6} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 18.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None) +!1 = !DIFile(filename: "print.c", directory: "/tmp") +!2 = !{} +!3 = !{i32 2, !"Dwarf Version", i32 5} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = !{i32 1, !"wchar_size", i32 4} +!6 = !{!"clang version 18.0.0"} +!7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 3, type: !8, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11) +!8 = !DISubroutineType(types: !9) +!9 = !{!10, !10} +!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!11 = !{!12, !13} +!12 = !DILocalVariable(name: "a", arg: 1, scope: !7, file: !1, line: 3, type: !10) +!13 = !DILocalVariable(name: "b", scope: !7, file: !1, line: 3, type: !10) +!14 = !DILocation(line: 3, column: 15, scope: !7) +!15 = distinct !DIAssignID() +!16 = !DILocation(line: 3, column: 20, scope: !7) +!17 = !DILocation(line: 3, column: 25, scope: !7) +!18 = !DILocation(line: 3, column: 30, scope: !7) diff --git a/llvm/test/Assembler/dbg-record-invalid-8.ll b/llvm/test/Assembler/dbg-record-invalid-8.ll new file mode 100644 index 000000000000..d0b8f36d7895 --- /dev/null +++ b/llvm/test/Assembler/dbg-record-invalid-8.ll @@ -0,0 +1,36 @@ +;; Test that we get a parser error when we have a debug assign record with an +;; incorrect number of arguments. +; RUN: not llvm-as < %s 2>&1 | FileCheck %s +; ModuleID = '' +source_filename = "" + +define dso_local i32 @f(i32 %a) !dbg !7 { +entry: +; CHECK: :[[@LINE+1]]:50: error: Expected ',' here + #dbg_assign(i32 %a, !12, !DIExpression(), !14) + ret i32 %a, !dbg !18 +} + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4, !5} +!llvm.ident = !{!6} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 18.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None) +!1 = !DIFile(filename: "print.c", directory: "/tmp") +!2 = !{} +!3 = !{i32 2, !"Dwarf Version", i32 5} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = !{i32 1, !"wchar_size", i32 4} +!6 = !{!"clang version 18.0.0"} +!7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 3, type: !8, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11) +!8 = !DISubroutineType(types: !9) +!9 = !{!10, !10} +!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!11 = !{!12, !13} +!12 = !DILocalVariable(name: "a", arg: 1, scope: !7, file: !1, line: 3, type: !10) +!13 = !DILocalVariable(name: "b", scope: !7, file: !1, line: 3, type: !10) +!14 = !DILocation(line: 3, column: 15, scope: !7) +!15 = distinct !DIAssignID() +!16 = !DILocation(line: 3, column: 20, scope: !7) +!17 = !DILocation(line: 3, column: 25, scope: !7) +!18 = !DILocation(line: 3, column: 30, scope: !7) diff --git a/llvm/test/DebugInfo/roundtrip-non-instruction-debug-info.ll b/llvm/test/DebugInfo/roundtrip-non-instruction-debug-info.ll new file mode 100644 index 000000000000..b15b76d1690c --- /dev/null +++ b/llvm/test/DebugInfo/roundtrip-non-instruction-debug-info.ll @@ -0,0 +1,94 @@ +;; Test that we can write in the old debug info format. +; RUN: opt --passes=verify -S --write-experimental-debuginfo=false < %s \ +; RUN: | FileCheck %s --check-prefixes=CHECK,OLDDBG --implicit-check-not=llvm.dbg --implicit-check-not=#dbg + +;; Test that we can write in the new debug info format... +; RUN: opt --passes=verify -S --write-experimental-debuginfo=true < %s \ +; RUN: | FileCheck %s --check-prefixes=CHECK,NEWDBG --implicit-check-not=llvm.dbg --implicit-check-not=#dbg + +;; ...and then read the new format and write the old format. +; RUN: opt --passes=verify -S --write-experimental-debuginfo=true < %s \ +; RUN: | opt --passes=verify -S --write-experimental-debuginfo=false \ +; RUN: | FileCheck %s --check-prefixes=CHECK,OLDDBG --implicit-check-not=llvm.dbg --implicit-check-not=#dbg + +;; Test also that the new flag is independent of the flag that enables use of +;; these non-instruction debug info during LLVM passes. +; RUN: opt --passes=verify -S --try-experimental-debuginfo-iterators --write-experimental-debuginfo=false < %s \ +; RUN: | FileCheck %s --check-prefixes=CHECK,OLDDBG --implicit-check-not=llvm.dbg --implicit-check-not=#dbg +; RUN: opt --passes=verify -S --try-experimental-debuginfo-iterators --write-experimental-debuginfo=true < %s \ +; RUN: | FileCheck %s --check-prefixes=CHECK,NEWDBG --implicit-check-not=llvm.dbg --implicit-check-not=#dbg + +; CHECK: @f(i32 %[[VAL_A:[0-9a-zA-Z]+]]) +; CHECK-NEXT: entry: +; OLDDBG-NEXT: call void @llvm.dbg.value(metadata i32 %[[VAL_A]], metadata ![[VAR_A:[0-9]+]], metadata !DIExpression()), !dbg ![[LOC_1:[0-9]+]] +; NEWDBG-NEXT: {{^}} #dbg_value(i32 %[[VAL_A]], ![[VAR_A:[0-9]+]], !DIExpression(), ![[LOC_1:[0-9]+]]) +; CHECK-NEXT: {{^}} %[[VAL_B:[0-9a-zA-Z]+]] = alloca +; OLDDBG-NEXT: call void @llvm.dbg.declare(metadata ptr %[[VAL_B]], metadata ![[VAR_B:[0-9]+]], metadata !DIExpression()), !dbg ![[LOC_2:[0-9]+]] +; NEWDBG-NEXT: {{^}} #dbg_declare(ptr %[[VAL_B]], ![[VAR_B:[0-9]+]], !DIExpression(), ![[LOC_2:[0-9]+]]) +; CHECK-NEXT: {{^}} %[[VAL_ADD:[0-9a-zA-Z]+]] = add i32 %[[VAL_A]], 5 +; OLDDBG-NEXT: call void @llvm.dbg.value(metadata !DIArgList(i32 %[[VAL_A]], i32 %[[VAL_ADD]]), metadata ![[VAR_A]], metadata !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_arg, 1, DW_OP_plus)), !dbg ![[LOC_3:[0-9]+]] +; NEWDBG-NEXT: {{^}} #dbg_value(!DIArgList(i32 %[[VAL_A]], i32 %[[VAL_ADD]]), ![[VAR_A]], !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_arg, 1, DW_OP_plus), ![[LOC_3:[0-9]+]]) +; OLDDBG-NEXT: call void @llvm.dbg.label(metadata ![[LABEL_ID:[0-9]+]]), !dbg ![[LOC_3]] +; NEWDBG-NEXT: {{^}} #dbg_label(![[LABEL_ID:[0-9]+]], ![[LOC_3]]) +; CHECK-NEXT: {{^}} store i32 %[[VAL_ADD]]{{.+}}, !DIAssignID ![[ASSIGNID:[0-9]+]] +; OLDDBG-NEXT: call void @llvm.dbg.assign(metadata i32 %[[VAL_ADD]], metadata ![[VAR_B]], metadata !DIExpression(), metadata ![[ASSIGNID]], metadata ptr %[[VAL_B]], metadata !DIExpression()), !dbg ![[LOC_4:[0-9]+]] +; NEWDBG-NEXT: {{^}} #dbg_assign(i32 %[[VAL_ADD]], ![[VAR_B]], !DIExpression(), ![[ASSIGNID]], ptr %[[VAL_B]], !DIExpression(), ![[LOC_4:[0-9]+]]) +; CHECK-NEXT: {{^}} ret i32 + +; OLDDBG-DAG: declare void @llvm.dbg.value +; OLDDBG-DAG: declare void @llvm.dbg.declare +; OLDDBG-DAG: declare void @llvm.dbg.assign +; OLDDBG-DAG: declare void @llvm.dbg.label + +; CHECK-DAG: llvm.dbg.cu +; CHECK-DAG: ![[VAR_A]] = !DILocalVariable(name: "a" +; CHECK-DAG: ![[VAR_B]] = !DILocalVariable(name: "b" +; CHECK-DAG: ![[LOC_1]] = !DILocation(line: 3, column: 15 +; CHECK-DAG: ![[LOC_2]] = !DILocation(line: 3, column: 20 +; CHECK-DAG: ![[LOC_3]] = !DILocation(line: 3, column: 25 +; CHECK-DAG: ![[LOC_4]] = !DILocation(line: 3, column: 30 +; CHECK-DAG: ![[LABEL_ID]] = !DILabel( + +define dso_local i32 @f(i32 %a) !dbg !7 { +entry: + call void @llvm.dbg.value(metadata i32 %a, metadata !20, metadata !DIExpression()), !dbg !30 + %b = alloca i32, !dbg !30, !DIAssignID !40 + call void @llvm.dbg.declare(metadata ptr %b, metadata !21, metadata !DIExpression()), !dbg !31 + %add = add i32 %a, 5, !dbg !31 + call void @llvm.dbg.value(metadata !DIArgList(i32 %a, i32 %add), metadata !20, metadata !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_arg, 1, DW_OP_plus)), !dbg !32 + call void @llvm.dbg.label(metadata !50), !dbg !32 + store i32 %add, ptr %b, !dbg !32, !DIAssignID !40 + call void @llvm.dbg.assign(metadata i32 %add, metadata !21, metadata !DIExpression(), metadata !40, metadata ptr %b, metadata !DIExpression()), !dbg !33 + ret i32 %add, !dbg !33 + +} + +declare void @llvm.dbg.value(metadata, metadata, metadata) +declare void @llvm.dbg.declare(metadata, metadata, metadata) +declare void @llvm.dbg.assign(metadata, metadata, metadata, metadata, metadata, metadata) +declare void @llvm.dbg.label(metadata) + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4, !5} +!llvm.ident = !{!6} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 18.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None) +!1 = !DIFile(filename: "print.c", directory: "/tmp") +!2 = !{} +!3 = !{i32 2, !"Dwarf Version", i32 5} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = !{i32 1, !"wchar_size", i32 4} +!6 = !{!"clang version 18.0.0"} +!7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 3, type: !8, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !13) +!8 = !DISubroutineType(types: !9) +!9 = !{!12, !12} +!12 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!13 = !{!20, !21} +!20 = !DILocalVariable(name: "a", arg: 1, scope: !7, file: !1, line: 3, type: !12) +!21 = !DILocalVariable(name: "b", scope: !7, file: !1, line: 3, type: !12) +!30 = !DILocation(line: 3, column: 15, scope: !7) +!31 = !DILocation(line: 3, column: 20, scope: !7) +!32 = !DILocation(line: 3, column: 25, scope: !7) +!33 = !DILocation(line: 3, column: 30, scope: !7) +!40 = distinct !DIAssignID() +!50 = !DILabel(scope: !7, name: "label", file: !1, line: 3) \ No newline at end of file diff --git a/llvm/test/Verifier/RemoveDI/blockbyref.ll b/llvm/test/Verifier/RemoveDI/blockbyref.ll new file mode 100644 index 000000000000..86321a6ae78e --- /dev/null +++ b/llvm/test/Verifier/RemoveDI/blockbyref.ll @@ -0,0 +1,18 @@ +; RUN: llvm-as -disable-output <%s 2>&1| FileCheck %s + +; CHECK: DIBlockByRefStruct on DICompositeType is no longer supported +; CHECK: warning: ignoring invalid debug info + +define void @foo() { +entry: + %s = alloca i32 + #dbg_declare(ptr %s, !2, !DIExpression(), !DILocation(scope: !1)) + ret void +} + + +!llvm.module.flags = !{!0} +!0 = !{i32 2, !"Debug Info Version", i32 3} +!1 = distinct !DISubprogram() +!2 = !DILocalVariable(scope: !1, type: !3) +!3 = !DICompositeType(tag: DW_TAG_structure_type, flags: DIFlagReservedBit4) diff --git a/llvm/test/Verifier/RemoveDI/dbg-invalid-vector.ll b/llvm/test/Verifier/RemoveDI/dbg-invalid-vector.ll new file mode 100644 index 000000000000..0832c361c308 --- /dev/null +++ b/llvm/test/Verifier/RemoveDI/dbg-invalid-vector.ll @@ -0,0 +1,35 @@ +; RUN: opt -passes=verify -disable-output <%s 2>&1 | FileCheck %s +; +; This test creates an invalid vector by defining multiple elements for the +; vector's DICompositeType definition. A vector should only have one element +; in its DICompositeType 'elements' array. +; +; CHECK: invalid vector + +@f.foo = private unnamed_addr constant <6 x float> zeroinitializer, align 32 + +define void @f() { + %1 = alloca <6 x float>, align 32 + #dbg_declare(ptr %1, !10, !DIExpression(), !18) + ret void +} + + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2) +!1 = !DIFile(filename: "test.c", directory: "/dbg/info") +!2 = !{} +!3 = !{i32 2, !"Dwarf Version", i32 4} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 3, type: !8, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2) +!8 = !DISubroutineType(types: !9) +!9 = !{null} +!10 = !DILocalVariable(name: "foo", scope: !7, file: !1, line: 4, type: !12) +!12 = !DICompositeType(tag: DW_TAG_array_type, baseType: !13, size: 256, flags: DIFlagVector, elements: !14) +!13 = !DIBasicType(name: "float", size: 32, encoding: DW_ATE_float) +!14 = !{!15, !19} +!15 = !DISubrange(count: 6) +!18 = !DILocation(line: 4, column: 48, scope: !7) +!19 = !DISubrange(count: 42) diff --git a/llvm/test/Verifier/RemoveDI/di-subroutine-localvar.ll b/llvm/test/Verifier/RemoveDI/di-subroutine-localvar.ll new file mode 100644 index 000000000000..14e588839899 --- /dev/null +++ b/llvm/test/Verifier/RemoveDI/di-subroutine-localvar.ll @@ -0,0 +1,41 @@ +; RUN: opt %s -passes=verify 2>&1 | FileCheck %s +; CHECK: invalid type +; CHECK: !20 = !DILocalVariable(name: "f", scope: !21, file: !13, line: 970, type: !14) +; CHECK: !14 = !DISubroutineType(types: !15) + + +%timespec.0.1.2.3.0.1.2 = type { i64, i64 } +define internal i64 @init_vdso_clock_gettime(i32, ptr nonnull) unnamed_addr !dbg !142 { + #dbg_value(ptr null, !162, !DIExpression(), !167) + ret i64 -38, !dbg !168 +} +!llvm.module.flags = !{!0} +!llvm.dbg.cu = !{!1} +!0 = !{i32 2, !"Debug Info Version", i32 3} +!1 = distinct !DICompileUnit(language: DW_LANG_C99, file: !2, producer: "zig 0.3.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !3, globals: !76) +!2 = !DIFile(filename: "test", directory: ".") +!3 = !{!4} +!4 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Arch", scope: !5, file: !5, line: 44, baseType: !6, size: 8, align: 8, elements: !7) +!5 = !DIFile(filename: "builtin.zig", directory: "/home/andy/.local/share/zig/stage1/builtin/ugMGxVES9OkDAffv3xhJS3KQVy0Wm1xPM3Bc6x4MBuup5aetdi5pVTrGRG2aDAn0") +!6 = !DIBasicType(name: "u7", size: 8, encoding: DW_ATE_unsigned) +!7 = !{!8} +!8 = !DIEnumerator(name: "armv8_5a", value: 0) +!76 = !{!77} +!77 = !DIGlobalVariableExpression(var: !78, expr: !DIExpression()) +!78 = distinct !DIGlobalVariable(name: "arch", linkageName: "arch", scope: !5, file: !5, line: 437, type: !4, isLocal: true, isDefinition: true) +!81 = !DIFile(filename: "index.zig", directory: "/store/dev/zig/build-llvm8-debug/lib/zig/std/os/linux") +!142 = distinct !DISubprogram(name: "init_vdso_clock_gettime", scope: !81, file: !81, line: 968, type: !143, scopeLine: 968, flags: DIFlagStaticMember, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition | DISPFlagOptimized, unit: !1, retainedNodes: !153) +!143 = !DISubroutineType(types: !144) +!144 = !{!145} +!145 = !DIBasicType(name: "usize", size: 64, encoding: DW_ATE_unsigned) +!146 = !DIBasicType(name: "i32", size: 32, encoding: DW_ATE_signed) +!153 = !{!154} +!154 = !DILocalVariable(name: "clk", arg: 1, scope: !142, file: !81, line: 968, type: !146) +!162 = !DILocalVariable(name: "f", scope: !163, file: !81, line: 970, type: !143) +!163 = distinct !DILexicalBlock(scope: !164, file: !81, line: 969, column: 5) +!164 = distinct !DILexicalBlock(scope: !165, file: !81, line: 968, column: 66) +!165 = distinct !DILexicalBlock(scope: !166, file: !81, line: 968, column: 45) +!166 = distinct !DILexicalBlock(scope: !142, file: !81, line: 968, column: 35) +!167 = !DILocation(line: 970, column: 5, scope: !163) +!168 = !DILocation(line: 972, column: 28, scope: !169) +!169 = distinct !DILexicalBlock(scope: !163, file: !81, line: 970, column: 5) diff --git a/llvm/test/Verifier/RemoveDI/diexpression-entry-value-llvm-ir.ll b/llvm/test/Verifier/RemoveDI/diexpression-entry-value-llvm-ir.ll new file mode 100644 index 000000000000..881ec4a86fb6 --- /dev/null +++ b/llvm/test/Verifier/RemoveDI/diexpression-entry-value-llvm-ir.ll @@ -0,0 +1,34 @@ +; RUN: llvm-as -disable-output <%s 2>&1| FileCheck %s + +; CHECK-NOT: #dbg_value +; CHECK: Entry values are only allowed in MIR unless they target a swiftasync Argument +; CHECK: #dbg_value(i32 %param, !{{.*}}, !DIExpression(DW_OP_LLVM_entry_value, 1) +; CHECK-NOT: #dbg_value +; CHECK-NOT: Entry values are only allowed +; CHECK: warning: ignoring invalid debug info + +define void @foo(i32 %param, ptr swiftasync %ok_param) !dbg !4 { +entry: + #dbg_value(i32 %param, !8, !DIExpression(DW_OP_LLVM_entry_value, 1), !9) + #dbg_value(ptr %ok_param, !8, !DIExpression(DW_OP_LLVM_entry_value, 1), !9) + #dbg_value(ptr poison, !8, !DIExpression(DW_OP_LLVM_entry_value, 1), !9) + #dbg_value(ptr undef, !8, !DIExpression(DW_OP_LLVM_entry_value, 1), !9) + ret void +} + + +attributes #0 = { nounwind readnone speculatable willreturn } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!2, !3} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, emissionKind: FullDebug) +!1 = !DIFile(filename: "a.c", directory: "/") +!2 = !{i32 2, !"Dwarf Version", i32 4} +!3 = !{i32 2, !"Debug Info Version", i32 3} +!4 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, type: !5, unit: !0) +!5 = !DISubroutineType(types: !6) +!6 = !{null, !7} +!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!8 = !DILocalVariable(name: "param", arg: 1, scope: !4, file: !1, type: !7) +!9 = !DILocation(line: 0, scope: !4) diff --git a/llvm/test/Verifier/RemoveDI/fnarg-debuginfo.ll b/llvm/test/Verifier/RemoveDI/fnarg-debuginfo.ll new file mode 100644 index 000000000000..db1a9a8ba189 --- /dev/null +++ b/llvm/test/Verifier/RemoveDI/fnarg-debuginfo.ll @@ -0,0 +1,26 @@ +; RUN: llvm-as -disable-output < %s -o /dev/null 2>&1 | FileCheck %s + + +define void @foo() !dbg !2 { +entry: + %a = alloca i32 + ; CHECK: conflicting debug info for argument + #dbg_value(i32 0, !3, !DIExpression(), !6) + #dbg_declare(ptr %a, !4, !DIExpression(), !6) + ret void, !dbg !6 +} + +; CHECK: warning: ignoring invalid debug info + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!7, !8} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", emissionKind: FullDebug) +!1 = !DIFile(filename: "x.c", directory: "/") +!2 = distinct !DISubprogram(name: "foo", scope: !0, isDefinition: true, unit: !0) +!3 = !DILocalVariable(name: "a", arg: 1, scope: !2, file: !1, line: 1, type: !5) +!4 = !DILocalVariable(name: "b", arg: 1, scope: !2, file: !1, line: 1, type: !5) +!5 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!6 = !DILocation(line: 1, scope: !2) +!7 = !{i32 2, !"Dwarf Version", i32 4} +!8 = !{i32 1, !"Debug Info Version", i32 3} diff --git a/llvm/test/Verifier/RemoveDI/fnarg-nodebug.ll b/llvm/test/Verifier/RemoveDI/fnarg-nodebug.ll new file mode 100644 index 000000000000..f5526030278e --- /dev/null +++ b/llvm/test/Verifier/RemoveDI/fnarg-nodebug.ll @@ -0,0 +1,58 @@ +; RUN: llvm-as < %s -o %t +; RUN: llvm-dis < %t -o - | FileCheck %s +; Created at -O1 from: +; int sink(int); +; __attribute__((always_inline)) int f(int i) { return sink(i); } +; __attribute__((always_inline)) int g(int j) { return sink(j); } +; __attribute__((nodebug)) int nodebug(int k) { return f(k)+g(k); } +source_filename = "t.c" +target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx10.12.0" + +declare i32 @sink(i32) local_unnamed_addr + +define i32 @nodebug(i32 %k) local_unnamed_addr #2 { +entry: +; This should not set off the FnArg Verifier. The two variables are in differrent scopes. + #dbg_value(i32 %k, !12, !13, !14) + %call.k = tail call i32 @sink(i32 %k) #4, !dbg !15 + #dbg_value(i32 %k, !19, !13, !20) + %call.k3 = tail call i32 @sink(i32 %k) #4, !dbg !21 + %add = add nsw i32 %call.k3, %call.k + ret i32 %add +} + +; Function Attrs: nounwind readnone + +attributes #2 = { nounwind ssp uwtable } +attributes #3 = { nounwind readnone } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4, !5} +!llvm.ident = !{!6} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 5.0.0 (trunk 297153) (llvm/trunk 297155)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2) +!1 = !DIFile(filename: "t.c", directory: "/tmp") +!2 = !{} +!3 = !{i32 2, !"Dwarf Version", i32 4} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = !{i32 1, !"PIC Level", i32 2} +!6 = !{!"clang version 5.0.0 (trunk 297153) (llvm/trunk 297155)"} +!7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 2, type: !8, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !11) +!8 = !DISubroutineType(types: !9) +!9 = !{!10, !10} +!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!11 = !{!12} +; CHECK: !DILocalVariable(name: "i", arg: 1 +!12 = !DILocalVariable(name: "i", arg: 1, scope: !7, file: !1, line: 2, type: !10) +!13 = !DIExpression() +!14 = !DILocation(line: 2, column: 42, scope: !7) +!15 = !DILocation(line: 2, column: 54, scope: !7) +!16 = !DILocation(line: 2, column: 47, scope: !7) +!17 = distinct !DISubprogram(name: "g", scope: !1, file: !1, line: 3, type: !8, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !18) +!18 = !{!19} +; CHECK: !DILocalVariable(name: "j", arg: 1 +!19 = !DILocalVariable(name: "j", arg: 1, scope: !17, file: !1, line: 3, type: !10) +!20 = !DILocation(line: 3, column: 42, scope: !17) +!21 = !DILocation(line: 3, column: 54, scope: !17) +!22 = !DILocation(line: 3, column: 47, scope: !17) diff --git a/llvm/test/Verifier/RemoveDI/invalid-disubrange-count-node.ll b/llvm/test/Verifier/RemoveDI/invalid-disubrange-count-node.ll new file mode 100644 index 000000000000..f36cee5946e4 --- /dev/null +++ b/llvm/test/Verifier/RemoveDI/invalid-disubrange-count-node.ll @@ -0,0 +1,36 @@ +; RUN: llvm-as < %s -disable-output 2>&1 | FileCheck %s + +define void @foo(i32 %n) { +entry: + %0 = zext i32 %n to i64 + %vla = alloca i32, i64 %0, align 16 + #dbg_declare(ptr %vla, !19, !DIExpression(), !18) + ret void +} + + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4, !5} +!llvm.ident = !{!6} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 5.0.1", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2) +!1 = !DIFile(filename: "vla.c", directory: "/path/to") +!2 = !{} +!3 = !{i32 2, !"Dwarf Version", i32 4} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = !{i32 1, !"wchar_size", i32 4} +!6 = !{!"clang version 5.0.1"} +!7 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 20, type: !8, isLocal: false, isDefinition: true, scopeLine: 20, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !11) +!8 = !DISubroutineType(types: !9) +!9 = !{null, !10} +!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!11 = !{!16, !19} +!12 = !DIExpression() +!16 = !DILocalVariable(name: "vla_expr", scope: !7, file: !1, line: 21, type: !17) +!17 = !DIBasicType(name: "long unsigned int", size: 64, encoding: DW_ATE_unsigned) +!18 = !DILocation(line: 21, column: 7, scope: !7) +!19 = !DILocalVariable(name: "vla", scope: !7, file: !1, line: 21, type: !20) +!20 = !DICompositeType(tag: DW_TAG_array_type, baseType: !10, align: 32, elements: !21) +!21 = !{!22} +; CHECK: Count must be signed constant or DIVariable or DIExpression +!22 = !DISubrange(count: !17) diff --git a/llvm/test/Verifier/RemoveDI/llvm.dbg.declare-address.ll b/llvm/test/Verifier/RemoveDI/llvm.dbg.declare-address.ll new file mode 100644 index 000000000000..9d400b892ce8 --- /dev/null +++ b/llvm/test/Verifier/RemoveDI/llvm.dbg.declare-address.ll @@ -0,0 +1,16 @@ +; RUN: llvm-as -disable-output <%s 2>&1 | FileCheck %s +; CHECK: invalid #dbg record address/value +; CHECK-NEXT: #dbg_declare({{.*}}) +; CHECK-NEXT: !"" +; CHECK: warning: ignoring invalid debug info + +define void @foo(i32 %a) { +entry: + %s = alloca i32 + #dbg_declare(!"", !DILocalVariable(scope: !1), !DIExpression(), !DILocation(scope: !1)) + ret void +} + +!llvm.module.flags = !{!0} +!0 = !{i32 2, !"Debug Info Version", i32 3} +!1 = distinct !DISubprogram() diff --git a/llvm/test/Verifier/RemoveDI/llvm.dbg.declare-expression.ll b/llvm/test/Verifier/RemoveDI/llvm.dbg.declare-expression.ll new file mode 100644 index 000000000000..b52c15cb3f88 --- /dev/null +++ b/llvm/test/Verifier/RemoveDI/llvm.dbg.declare-expression.ll @@ -0,0 +1,16 @@ +; RUN: llvm-as -disable-output <%s 2>&1 | FileCheck %s +; CHECK: invalid #dbg record expression +; CHECK-NEXT: #dbg_declare({{.*}}) +; CHECK-NEXT: !{} +; CHECK: warning: ignoring invalid debug info + +define void @foo(i32 %a) { +entry: + %s = alloca i32 + #dbg_declare(ptr %s, !DILocalVariable(scope: !1), !{}, !DILocation(scope: !1)) + ret void +} + +!llvm.module.flags = !{!0} +!0 = !{i32 2, !"Debug Info Version", i32 3} +!1 = distinct !DISubprogram() diff --git a/llvm/test/Verifier/RemoveDI/llvm.dbg.declare-variable.ll b/llvm/test/Verifier/RemoveDI/llvm.dbg.declare-variable.ll new file mode 100644 index 000000000000..db2b0e0a54e2 --- /dev/null +++ b/llvm/test/Verifier/RemoveDI/llvm.dbg.declare-variable.ll @@ -0,0 +1,17 @@ +; RUN: llvm-as -disable-output <%s 2>&1 | FileCheck %s +; CHECK: invalid #dbg record variable +; CHECK-NEXT: #dbg_declare({{.*}}) +; CHECK-NEXT: !{} +; CHECK: warning: ignoring invalid debug info + +define void @foo(i32 %a) { +entry: + %s = alloca i32 + #dbg_declare(ptr %s, !{}, !DIExpression(), !DILocation(scope: !1)) + ret void +} + + +!llvm.module.flags = !{!0} +!0 = !{i32 2, !"Debug Info Version", i32 3} +!1 = distinct !DISubprogram() diff --git a/llvm/test/Verifier/RemoveDI/llvm.dbg.intrinsic-dbg-attachment.ll b/llvm/test/Verifier/RemoveDI/llvm.dbg.intrinsic-dbg-attachment.ll new file mode 100644 index 000000000000..1839821ab140 --- /dev/null +++ b/llvm/test/Verifier/RemoveDI/llvm.dbg.intrinsic-dbg-attachment.ll @@ -0,0 +1,55 @@ +; RUN: llvm-as -disable-output <%s 2>&1 | FileCheck %s +define void @foo() { +entry: + #dbg_value( + ptr undef, + !DILocalVariable(scope: !1), + !DIExpression(), + !{}) +; CHECK-LABEL: invalid #dbg record DILocation +; CHECK-NEXT: #dbg_value({{.*}}) + + #dbg_declare( + ptr undef, + !DILocalVariable(scope: !1), + !DIExpression(), + !{}) +; CHECK-LABEL: invalid #dbg record DILocation +; CHECK-NEXT: #dbg_declare({{.*}}) + + #dbg_value( + ptr undef, + !DILocalVariable(scope: !1), + !DIExpression(), + !DILocation(scope: !2)) +; CHECK-LABEL: mismatched subprogram between #dbg record variable and DILocation +; CHECK-NEXT: #dbg_value({{[^,]+}}, ![[VAR:[0-9]+]], {{[^,]+}}, ![[LOC:[0-9]+]] +; CHECK-NEXT: label %entry +; CHECK-NEXT: ptr @foo +; CHECK-NEXT: ![[VAR]] = !DILocalVariable({{.*}}scope: ![[VARSP:[0-9]+]] +; CHECK-NEXT: ![[VARSP]] = distinct !DISubprogram( +; CHECK-NEXT: ![[LOC]] = !DILocation({{.*}}scope: ![[LOCSP:[0-9]+]] +; CHECK-NEXT: ![[LOCSP]] = distinct !DISubprogram( + + #dbg_declare( + ptr undef, + !DILocalVariable(scope: !1), + !DIExpression(), + !DILocation(scope: !2)) +; CHECK-LABEL: mismatched subprogram between #dbg record variable and DILocation +; CHECK-NEXT: #dbg_declare({{[^,]+}}, ![[VAR:[0-9]+]], {{.*[^,]+}}, ![[LOC:[0-9]+]] +; CHECK-NEXT: label %entry +; CHECK-NEXT: ptr @foo +; CHECK-NEXT: ![[VAR]] = !DILocalVariable({{.*}}scope: ![[VARSP:[0-9]+]] +; CHECK-NEXT: ![[VARSP]] = distinct !DISubprogram( +; CHECK-NEXT: ![[LOC]] = !DILocation({{.*}}scope: ![[LOCSP:[0-9]+]] +; CHECK-NEXT: ![[LOCSP]] = distinct !DISubprogram( + + ret void +} + + +!llvm.module.flags = !{!0} +!0 = !{i32 2, !"Debug Info Version", i32 3} +!1 = distinct !DISubprogram(name: "foo") +!2 = distinct !DISubprogram(name: "bar") diff --git a/llvm/test/Verifier/RemoveDI/llvm.dbg.value-expression.ll b/llvm/test/Verifier/RemoveDI/llvm.dbg.value-expression.ll new file mode 100644 index 000000000000..cbd93c1ce6a4 --- /dev/null +++ b/llvm/test/Verifier/RemoveDI/llvm.dbg.value-expression.ll @@ -0,0 +1,16 @@ +; RUN: llvm-as -disable-output <%s 2>&1 | FileCheck %s +; CHECK: invalid #dbg record expression +; CHECK-NEXT: #dbg_value({{.*}}) +; CHECK-NEXT: !{} +; CHECK: warning: ignoring invalid debug info + +define void @foo(i32 %a) { +entry: + %s = alloca i32 + #dbg_value(ptr %s, !DILocalVariable(scope: !1), !{}, !DILocation(scope: !1)) + ret void +} + +!llvm.module.flags = !{!0} +!0 = !{i32 2, !"Debug Info Version", i32 3} +!1 = distinct !DISubprogram() diff --git a/llvm/test/Verifier/RemoveDI/llvm.dbg.value-value.ll b/llvm/test/Verifier/RemoveDI/llvm.dbg.value-value.ll new file mode 100644 index 000000000000..b6fcde250526 --- /dev/null +++ b/llvm/test/Verifier/RemoveDI/llvm.dbg.value-value.ll @@ -0,0 +1,17 @@ +; RUN: llvm-as -disable-output <%s 2>&1 | FileCheck %s +; CHECK: invalid #dbg record address/value +; CHECK-NEXT: #dbg_value({{.*}}) +; CHECK-NEXT: !"" +; CHECK: warning: ignoring invalid debug info + +define void @foo(i32 %a) { +entry: + %s = alloca i32 + #dbg_value(!"", !DILocalVariable(scope: !1), !DIExpression(), !DILocation(scope: !1)) + ret void +} + + +!llvm.module.flags = !{!0} +!0 = !{i32 2, !"Debug Info Version", i32 3} +!1 = distinct !DISubprogram() diff --git a/llvm/test/Verifier/RemoveDI/llvm.dbg.value-variable.ll b/llvm/test/Verifier/RemoveDI/llvm.dbg.value-variable.ll new file mode 100644 index 000000000000..0a5fe79453d7 --- /dev/null +++ b/llvm/test/Verifier/RemoveDI/llvm.dbg.value-variable.ll @@ -0,0 +1,17 @@ +; RUN: llvm-as -disable-output <%s 2>&1 | FileCheck %s +; CHECK: invalid #dbg record variable +; CHECK-NEXT: #dbg_value({{.*}}) +; CHECK-NEXT: !{} +; CHECK: warning: ignoring invalid debug info + +define void @foo(i32 %a) { +entry: + %s = alloca i32 + #dbg_value(ptr %s, !{}, !DIExpression(), !DILocation(scope: !1)) + ret void +} + + +!llvm.module.flags = !{!0} +!0 = !{i32 2, !"Debug Info Version", i32 3} +!1 = distinct !DISubprogram() diff --git a/llvm/test/Verifier/RemoveDI/set1.ll b/llvm/test/Verifier/RemoveDI/set1.ll new file mode 100644 index 000000000000..d54ba8876c36 --- /dev/null +++ b/llvm/test/Verifier/RemoveDI/set1.ll @@ -0,0 +1,62 @@ +; RUN: llvm-as -disable-output <%s 2>&1 | FileCheck %s + +define void @Main__Test() #0 !dbg !17 { +entry: + %as = alloca i64, align 8 + %bs = alloca i64, align 8 + br label %second, !dbg !21 + +second: ; preds = %entry + #dbg_declare(ptr %as, !22, !DIExpression(), !25) + #dbg_declare(ptr %bs, !26, !DIExpression(), !25) + store i64 36028797018972298, ptr %as, align 8, !dbg !28 + store i64 85, ptr %bs, align 8, !dbg !29 + ret void, !dbg !21 +} + +; Function Attrs: nofree nosync nounwind readnone speculatable willreturn + +!llvm.ident = !{!0} +!llvm.dbg.cu = !{!1} +!llvm.module.flags = !{!14, !15, !16} + +!0 = !{!"versions- cm3: d5.10.0 llvm: 12.0"} +!1 = distinct !DICompileUnit(language: DW_LANG_Modula3, file: !2, producer: "cm3", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !3) +!2 = !DIFile(filename: "Main.m3", directory: "/home/peter/cm3/settest/src") +!3 = !{!4} +!4 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum", scope: !2, file: !2, line: 11, size: 8, align: 8, elements: !5) +!5 = !{!6, !7, !8, !9, !10, !11, !12, !13} +!6 = !DIEnumerator(name: "alpha", value: 0) +!7 = !DIEnumerator(name: "beta", value: 1) +!8 = !DIEnumerator(name: "gamma", value: 2) +!9 = !DIEnumerator(name: "delta", value: 3) +!10 = !DIEnumerator(name: "epsilon", value: 4) +!11 = !DIEnumerator(name: "theta", value: 5) +!12 = !DIEnumerator(name: "psi", value: 6) +!13 = !DIEnumerator(name: "zeta", value: 7) +!14 = !{i64 2, !"Dwarf Version", i64 4} +!15 = !{i64 2, !"Debug Info Version", i64 3} +!16 = !{i64 2, !"wchar_size", i64 2} +!17 = distinct !DISubprogram(name: "Test", linkageName: "Main__Test", scope: !2, file: !2, line: 11, type: !18, scopeLine: 11, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !20) +!18 = !DISubroutineType(types: !19) +!19 = !{null} +!20 = !{} +!21 = !DILocation(line: 20, scope: !17) +!22 = !DILocalVariable(name: "as", scope: !17, file: !2, line: 11, type: !23) +; CHECK: invalid set base type +!23 = !DIDerivedType(tag: DW_TAG_set_type, name: "SS", scope: !2, file: !2, line: 11, baseType: !24, size: 64, align: 64) +!24 = !DIBasicType(name: "SR", size: 8, encoding: DW_ATE_signed) +!25 = !DILocation(line: 11, scope: !17) +!26 = !DILocalVariable(name: "bs", scope: !17, file: !2, line: 11, type: !27) +!27 = !DIDerivedType(tag: DW_TAG_set_type, name: "ST", scope: !2, file: !2, line: 11, baseType: !23, size: 64, align: 64) +!28 = !DILocation(line: 17, scope: !17) +!29 = !DILocation(line: 18, scope: !17) +!30 = distinct !DISubprogram(name: "Main_M3", linkageName: "Main_M3", scope: !2, file: !2, line: 22, type: !31, scopeLine: 22, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !20) +!31 = !DISubroutineType(types: !32) +!32 = !{!33, !35} +!33 = !DIDerivedType(tag: DW_TAG_pointer_type, name: "ADDR", baseType: !34, size: 64, align: 64) +!34 = !DICompositeType(tag: DW_TAG_class_type, name: "ADDR__HeapObject", scope: !2, file: !2, line: 22, size: 64, align: 64, elements: !19, identifier: "AJWxb1") +!35 = !DIBasicType(name: "INTEGER", size: 64, encoding: DW_ATE_signed) +!36 = !DILocation(line: 23, scope: !30) +!37 = !DILocalVariable(name: "mode", arg: 1, scope: !30, file: !2, line: 22, type: !35) +!38 = !DILocation(line: 22, scope: !30) -- GitLab From 4b70d17bcffaffd75a5d8c420396f8dc755b4652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Thu, 7 Mar 2024 14:27:04 +0100 Subject: [PATCH 436/929] [clang-repl] Names declared in if conditions and for-init statements are local to the inner context (#84150) Make TopLevelStmtDecl a DeclContext so that variables defined in statements are attached to the TopLevelDeclContext. This fixes redefinition errors from variables declared in if conditions and for-init statements. These must be local to the inner context (C++ 3.3.2p4), but they had generated definitions on global scope instead. This PR makes the TopLevelStmtDecl looking more like a FunctionDecl and that's fine because the FunctionDecl is very close in terms of semantics. Additionally, ActOnForStmt() requires a CompoundScope when processing a NullStmt body. --------- Co-authored-by: Vassil Vassilev --- clang/include/clang/AST/Decl.h | 16 ++++++++++------ clang/include/clang/AST/DeclBase.h | 1 + clang/include/clang/Basic/DeclNodes.td | 2 +- clang/include/clang/Sema/Sema.h | 3 ++- clang/lib/AST/Decl.cpp | 11 ++++++++--- clang/lib/AST/DeclBase.cpp | 1 + clang/lib/Parse/ParseDecl.cpp | 24 ++++++++++++++++-------- clang/lib/Sema/SemaDecl.cpp | 16 +++++++++++++--- clang/test/Interpreter/execute-stmts.cpp | 24 +++++++++++++++++++++++- 9 files changed, 75 insertions(+), 23 deletions(-) diff --git a/clang/include/clang/AST/Decl.h b/clang/include/clang/AST/Decl.h index 61117cc5ce71..a5879591f4c6 100644 --- a/clang/include/clang/AST/Decl.h +++ b/clang/include/clang/AST/Decl.h @@ -4419,7 +4419,7 @@ public: /// /// \note This is used in libInterpreter, clang -cc1 -fincremental-extensions /// and in tools such as clang-repl. -class TopLevelStmtDecl : public Decl { +class TopLevelStmtDecl : public Decl, public DeclContext { friend class ASTDeclReader; friend class ASTDeclWriter; @@ -4427,7 +4427,7 @@ class TopLevelStmtDecl : public Decl { bool IsSemiMissing = false; TopLevelStmtDecl(DeclContext *DC, SourceLocation L, Stmt *S) - : Decl(TopLevelStmt, DC, L), Statement(S) {} + : Decl(TopLevelStmt, DC, L), DeclContext(TopLevelStmt), Statement(S) {} virtual void anchor(); @@ -4438,15 +4438,19 @@ public: SourceRange getSourceRange() const override LLVM_READONLY; Stmt *getStmt() { return Statement; } const Stmt *getStmt() const { return Statement; } - void setStmt(Stmt *S) { - assert(IsSemiMissing && "Operation supported for printing values only!"); - Statement = S; - } + void setStmt(Stmt *S); bool isSemiMissing() const { return IsSemiMissing; } void setSemiMissing(bool Missing = true) { IsSemiMissing = Missing; } static bool classof(const Decl *D) { return classofKind(D->getKind()); } static bool classofKind(Kind K) { return K == TopLevelStmt; } + + static DeclContext *castToDeclContext(const TopLevelStmtDecl *D) { + return static_cast(const_cast(D)); + } + static TopLevelStmtDecl *castFromDeclContext(const DeclContext *DC) { + return static_cast(const_cast(DC)); + } }; /// Represents a block literal declaration, which is like an diff --git a/clang/include/clang/AST/DeclBase.h b/clang/include/clang/AST/DeclBase.h index 9a4736019d1b..76810a86a78a 100644 --- a/clang/include/clang/AST/DeclBase.h +++ b/clang/include/clang/AST/DeclBase.h @@ -2120,6 +2120,7 @@ public: case Decl::Block: case Decl::Captured: case Decl::ObjCMethod: + case Decl::TopLevelStmt: return true; default: return getDeclKind() >= Decl::firstFunction && diff --git a/clang/include/clang/Basic/DeclNodes.td b/clang/include/clang/Basic/DeclNodes.td index 8b1f415dd5fe..48396e85c5ad 100644 --- a/clang/include/clang/Basic/DeclNodes.td +++ b/clang/include/clang/Basic/DeclNodes.td @@ -95,7 +95,7 @@ def LinkageSpec : DeclNode, DeclContext; def Export : DeclNode, DeclContext; def ObjCPropertyImpl : DeclNode; def FileScopeAsm : DeclNode; -def TopLevelStmt : DeclNode; +def TopLevelStmt : DeclNode, DeclContext; def AccessSpec : DeclNode; def Friend : DeclNode; def FriendTemplate : DeclNode; diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index 2d949f3fc9a7..592c7871a4a5 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -3263,7 +3263,8 @@ public: Decl *ActOnFileScopeAsmDecl(Expr *expr, SourceLocation AsmLoc, SourceLocation RParenLoc); - Decl *ActOnTopLevelStmtDecl(Stmt *Statement); + TopLevelStmtDecl *ActOnStartTopLevelStmtDecl(Scope *S); + void ActOnFinishTopLevelStmtDecl(TopLevelStmtDecl *D, Stmt *Statement); void ActOnPopScope(SourceLocation Loc, Scope *S); diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp index 59c039f1f8da..d681791d3920 100644 --- a/clang/lib/AST/Decl.cpp +++ b/clang/lib/AST/Decl.cpp @@ -5552,14 +5552,13 @@ FileScopeAsmDecl *FileScopeAsmDecl::CreateDeserialized(ASTContext &C, void TopLevelStmtDecl::anchor() {} TopLevelStmtDecl *TopLevelStmtDecl::Create(ASTContext &C, Stmt *Statement) { - assert(Statement); assert(C.getLangOpts().IncrementalExtensions && "Must be used only in incremental mode"); - SourceLocation BeginLoc = Statement->getBeginLoc(); + SourceLocation Loc = Statement ? Statement->getBeginLoc() : SourceLocation(); DeclContext *DC = C.getTranslationUnitDecl(); - return new (C, DC) TopLevelStmtDecl(DC, BeginLoc, Statement); + return new (C, DC) TopLevelStmtDecl(DC, Loc, Statement); } TopLevelStmtDecl *TopLevelStmtDecl::CreateDeserialized(ASTContext &C, @@ -5572,6 +5571,12 @@ SourceRange TopLevelStmtDecl::getSourceRange() const { return SourceRange(getLocation(), Statement->getEndLoc()); } +void TopLevelStmtDecl::setStmt(Stmt *S) { + assert(S); + Statement = S; + setLocation(Statement->getBeginLoc()); +} + void EmptyDecl::anchor() {} EmptyDecl *EmptyDecl::Create(ASTContext &C, DeclContext *DC, SourceLocation L) { diff --git a/clang/lib/AST/DeclBase.cpp b/clang/lib/AST/DeclBase.cpp index 10fe8bb97ce6..fcedb3cfd176 100644 --- a/clang/lib/AST/DeclBase.cpp +++ b/clang/lib/AST/DeclBase.cpp @@ -1352,6 +1352,7 @@ DeclContext *DeclContext::getPrimaryContext() { case Decl::ExternCContext: case Decl::LinkageSpec: case Decl::Export: + case Decl::TopLevelStmt: case Decl::Block: case Decl::Captured: case Decl::OMPDeclareReduction: diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp index 81f1c7112694..64b234eb460d 100644 --- a/clang/lib/Parse/ParseDecl.cpp +++ b/clang/lib/Parse/ParseDecl.cpp @@ -5678,24 +5678,32 @@ Parser::DeclGroupPtrTy Parser::ParseTopLevelStmtDecl() { // Parse a top-level-stmt. Parser::StmtVector Stmts; ParsedStmtContext SubStmtCtx = ParsedStmtContext(); - Actions.PushFunctionScope(); + ParseScope FnScope(this, Scope::FnScope | Scope::DeclScope | + Scope::CompoundStmtScope); + TopLevelStmtDecl *TLSD = Actions.ActOnStartTopLevelStmtDecl(getCurScope()); StmtResult R = ParseStatementOrDeclaration(Stmts, SubStmtCtx); - Actions.PopFunctionScopeInfo(); if (!R.isUsable()) return nullptr; - SmallVector DeclsInGroup; - DeclsInGroup.push_back(Actions.ActOnTopLevelStmtDecl(R.get())); + Actions.ActOnFinishTopLevelStmtDecl(TLSD, R.get()); if (Tok.is(tok::annot_repl_input_end) && Tok.getAnnotationValue() != nullptr) { ConsumeAnnotationToken(); - cast(DeclsInGroup.back())->setSemiMissing(); + TLSD->setSemiMissing(); } - // Currently happens for things like -fms-extensions and use `__if_exists`. - for (Stmt *S : Stmts) - DeclsInGroup.push_back(Actions.ActOnTopLevelStmtDecl(S)); + SmallVector DeclsInGroup; + DeclsInGroup.push_back(TLSD); + + // Currently happens for things like -fms-extensions and use `__if_exists`. + for (Stmt *S : Stmts) { + // Here we should be safe as `__if_exists` and friends are not introducing + // new variables which need to live outside file scope. + TopLevelStmtDecl *D = Actions.ActOnStartTopLevelStmtDecl(getCurScope()); + Actions.ActOnFinishTopLevelStmtDecl(D, S); + DeclsInGroup.push_back(D); + } return Actions.BuildDeclaratorGroup(DeclsInGroup); } diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 6b81ee183cc4..67e56a917a51 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -20519,12 +20519,22 @@ Decl *Sema::ActOnFileScopeAsmDecl(Expr *expr, return New; } -Decl *Sema::ActOnTopLevelStmtDecl(Stmt *Statement) { - auto *New = TopLevelStmtDecl::Create(Context, Statement); - Context.getTranslationUnitDecl()->addDecl(New); +TopLevelStmtDecl *Sema::ActOnStartTopLevelStmtDecl(Scope *S) { + auto *New = TopLevelStmtDecl::Create(Context, /*Statement=*/nullptr); + CurContext->addDecl(New); + PushDeclContext(S, New); + PushFunctionScope(); + PushCompoundScope(false); return New; } +void Sema::ActOnFinishTopLevelStmtDecl(TopLevelStmtDecl *D, Stmt *Statement) { + D->setStmt(Statement); + PopCompoundScope(); + PopFunctionScopeInfo(); + PopDeclContext(); +} + void Sema::ActOnPragmaRedefineExtname(IdentifierInfo* Name, IdentifierInfo* AliasName, SourceLocation PragmaLoc, diff --git a/clang/test/Interpreter/execute-stmts.cpp b/clang/test/Interpreter/execute-stmts.cpp index 2d4c17e0c91e..433c6811777d 100644 --- a/clang/test/Interpreter/execute-stmts.cpp +++ b/clang/test/Interpreter/execute-stmts.cpp @@ -9,7 +9,6 @@ //CODEGEN-CHECK-COUNT-2: define internal void @__stmts__ //CODEGEN-CHECK-NOT: define internal void @__stmts__ - extern "C" int printf(const char*,...); template T call() { printf("called\n"); return T(); } @@ -41,3 +40,26 @@ for (; i > 4; --i) { printf("i = %d\n", i); }; int j = i; printf("j = %d\n", j); // CHECK-NEXT: j = 4 + +{i = 0; printf("i = %d (global scope)\n", i);} +// CHECK-NEXT: i = 0 + +while (int i = 1) { printf("i = %d (while condition)\n", i--); break; } +// CHECK-NEXT: i = 1 + +if (int i = 2) printf("i = %d (if condition)\n", i); +// CHECK-NEXT: i = 2 + +switch (int i = 3) { default: printf("i = %d (switch condition)\n", i); } +// CHECK-NEXT: i = 3 + +for (int i = 4; i > 3; --i) printf("i = %d (for-init)\n", i); +// CHECK-NEXT: i = 4 + +for (const auto &i : "5") printf("i = %c (range-based for-init)\n", i); +// CHECK-NEXT: i = 5 + +int *aa=nullptr; +if (auto *b=aa) *b += 1; +while (auto *b=aa) ; +for (auto *b=aa; b; *b+=1) ; -- GitLab From 2acccf6717996bea8ade96dafdfc3343e9604694 Mon Sep 17 00:00:00 2001 From: aniplcc <157880614+aniplcc@users.noreply.github.com> Date: Thu, 7 Mar 2024 19:09:04 +0530 Subject: [PATCH 437/929] [Clang] Update value for __cpp_implicit_move (#84216) (#84228) Fixes #84216 --- clang/lib/Frontend/InitPreprocessor.cpp | 2 +- clang/test/Lexer/cxx-features.cpp | 2 +- clang/test/SemaCXX/cxx2b-p2266-disable-with-msvc-compat.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/clang/lib/Frontend/InitPreprocessor.cpp b/clang/lib/Frontend/InitPreprocessor.cpp index 9b979d810fa1..48ad92063bd4 100644 --- a/clang/lib/Frontend/InitPreprocessor.cpp +++ b/clang/lib/Frontend/InitPreprocessor.cpp @@ -736,7 +736,7 @@ static void InitializeCPlusPlusFeatureTestMacros(const LangOptions &LangOpts, } // C++23 features. if (LangOpts.CPlusPlus23) { - Builder.defineMacro("__cpp_implicit_move", "202011L"); + Builder.defineMacro("__cpp_implicit_move", "202207L"); Builder.defineMacro("__cpp_size_t_suffix", "202011L"); Builder.defineMacro("__cpp_if_consteval", "202106L"); Builder.defineMacro("__cpp_multidimensional_subscript", "202211L"); diff --git a/clang/test/Lexer/cxx-features.cpp b/clang/test/Lexer/cxx-features.cpp index 2650a3a82252..9496746c6fd6 100644 --- a/clang/test/Lexer/cxx-features.cpp +++ b/clang/test/Lexer/cxx-features.cpp @@ -45,7 +45,7 @@ #endif -#if check(implicit_move, 0, 0, 0, 0, 0, 202011, 202011) +#if check(implicit_move, 0, 0, 0, 0, 0, 202207, 202207) #error "wrong value for __cpp_implicit_move" #endif diff --git a/clang/test/SemaCXX/cxx2b-p2266-disable-with-msvc-compat.cpp b/clang/test/SemaCXX/cxx2b-p2266-disable-with-msvc-compat.cpp index d40491834d39..9323dea24bd7 100644 --- a/clang/test/SemaCXX/cxx2b-p2266-disable-with-msvc-compat.cpp +++ b/clang/test/SemaCXX/cxx2b-p2266-disable-with-msvc-compat.cpp @@ -9,7 +9,7 @@ #if __INCLUDE_LEVEL__ == 0 -#if __cpluscplus > 202002L && __cpp_implicit_move < 202011L +#if __cpluscplus > 202002L && __cpp_implicit_move < 202207L #error "__cpp_implicit_move not defined correctly" #endif -- GitLab From c669c0383cf982bec279f567662cc918576b6f34 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 7 Mar 2024 13:39:37 +0000 Subject: [PATCH 438/929] [TTI] SK_ExtractSubvector - Ensure we use the src / subvector types in the correct order Fixes typo in #84156, fixes buildbot assertion (most targets don't seem to care so tricky to create a testcase). --- llvm/include/llvm/Analysis/TargetTransformInfoImpl.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h index 095c2ff1e58b..7f661bb4a1df 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -1382,9 +1382,9 @@ public: SmallVector ExtractMask(Mask.size()); std::iota(ExtractMask.begin(), ExtractMask.end(), 0); - return ShuffleCost + TargetTTI->getShuffleCost( - TTI::SK_ExtractSubvector, VecTy, ExtractMask, - CostKind, 0, VecSrcTy, Operands); + return ShuffleCost + TargetTTI->getShuffleCost(TTI::SK_ExtractSubvector, + VecSrcTy, ExtractMask, + CostKind, 0, VecTy); } if (Shuffle->isIdentity()) -- GitLab From 48dd118f56e007a173b30019e860f0bd373a8ff8 Mon Sep 17 00:00:00 2001 From: cor3ntin Date: Thu, 7 Mar 2024 14:40:30 +0100 Subject: [PATCH 439/929] [Clang] Fix approved revision of P2266 --- clang/www/cxx_status.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html index 5ed27cdd43b3..fa00e7685610 100755 --- a/clang/www/cxx_status.html +++ b/clang/www/cxx_status.html @@ -197,7 +197,7 @@ C++23, informally referred to as C++26.

Simpler implicit move - P2266R1 + P2266R3 Clang 13 -- GitLab From 597be90f8b72fde59505f3650c20cf9e57b47d57 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Thu, 7 Mar 2024 07:40:38 -0600 Subject: [PATCH 440/929] [Clang][NFC] Remove '--' separator in the linker wrapper usage (#84253) Summary: The very first version of the `clang-linker-wrapper` used `--` as a separator for the host and device arguments. I moved away from this towards a commandline parsing implementation years ago but never got around to officially removing this. --- clang/lib/Driver/ToolChains/Clang.cpp | 1 - clang/test/Driver/amdgpu-openmp-toolchain.c | 2 +- clang/test/Driver/linker-wrapper-image.c | 18 ++++----- clang/test/Driver/linker-wrapper-libs.c | 14 +++---- clang/test/Driver/linker-wrapper.c | 40 +++++++++---------- clang/test/Driver/openmp-offload-gpu.c | 2 +- clang/test/Driver/openmp-offload-infer.c | 2 +- .../clang-linker-wrapper/LinkerWrapperOpts.td | 4 -- 8 files changed, 39 insertions(+), 44 deletions(-) diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index deb2dac80afe..e63e8a8e2e0e 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -8892,7 +8892,6 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA, // Add the linker arguments to be forwarded by the wrapper. CmdArgs.push_back(Args.MakeArgString(Twine("--linker-path=") + LinkCommand->getExecutable())); - CmdArgs.push_back("--"); for (const char *LinkArg : LinkCommand->getArguments()) CmdArgs.push_back(LinkArg); diff --git a/clang/test/Driver/amdgpu-openmp-toolchain.c b/clang/test/Driver/amdgpu-openmp-toolchain.c index 4975e2f8a523..849afb871ddb 100644 --- a/clang/test/Driver/amdgpu-openmp-toolchain.c +++ b/clang/test/Driver/amdgpu-openmp-toolchain.c @@ -11,7 +11,7 @@ // CHECK: "-cc1" "-triple" "x86_64-unknown-linux-gnu"{{.*}}"-emit-llvm-bc"{{.*}}"-x" "c" // CHECK: "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"{{.*}}"-target-cpu" "gfx906"{{.*}}"-fcuda-is-device"{{.*}} // CHECK: "-cc1" "-triple" "x86_64-unknown-linux-gnu"{{.*}}"-emit-obj" -// CHECK: clang-linker-wrapper{{.*}}"--"{{.*}} "-o" "a.out" +// CHECK: clang-linker-wrapper{{.*}} "-o" "a.out" // RUN: %clang -ccc-print-phases --target=x86_64-unknown-linux-gnu -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target=amdgcn-amd-amdhsa -march=gfx906 %s 2>&1 \ // RUN: | FileCheck --check-prefix=CHECK-PHASES %s diff --git a/clang/test/Driver/linker-wrapper-image.c b/clang/test/Driver/linker-wrapper-image.c index 08f860f6cab0..754752641352 100644 --- a/clang/test/Driver/linker-wrapper-image.c +++ b/clang/test/Driver/linker-wrapper-image.c @@ -8,11 +8,11 @@ // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o \ // RUN: -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --print-wrapped-module --dry-run --host-triple=x86_64-unknown-linux-gnu \ -// RUN: --linker-path=/usr/bin/ld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefixes=OPENMP,OPENMP-ELF +// RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefixes=OPENMP,OPENMP-ELF // RUN: clang-linker-wrapper --print-wrapped-module --dry-run -r --host-triple=x86_64-unknown-linux-gnu \ -// RUN: --linker-path=/usr/bin/ld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefixes=OPENMP-ELF,OPENMP-REL +// RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefixes=OPENMP-ELF,OPENMP-REL // RUN: clang-linker-wrapper --print-wrapped-module --dry-run --host-triple=x86_64-unknown-windows-gnu \ -// RUN: --linker-path=/usr/bin/ld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefixes=OPENMP,OPENMP-COFF +// RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefixes=OPENMP,OPENMP-COFF // OPENMP-ELF: @__start_omp_offloading_entries = external hidden constant [0 x %struct.__tgt_offload_entry] // OPENMP-ELF-NEXT: @__stop_omp_offloading_entries = external hidden constant [0 x %struct.__tgt_offload_entry] @@ -45,11 +45,11 @@ // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o \ // RUN: -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --print-wrapped-module --dry-run --host-triple=x86_64-unknown-linux-gnu \ -// RUN: --linker-path=/usr/bin/ld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefixes=CUDA,CUDA-ELF +// RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefixes=CUDA,CUDA-ELF // RUN: clang-linker-wrapper --print-wrapped-module --dry-run -r --host-triple=x86_64-unknown-linux-gnu \ -// RUN: --linker-path=/usr/bin/ld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefixes=CUDA,CUDA-ELF +// RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefixes=CUDA,CUDA-ELF // RUN: clang-linker-wrapper --print-wrapped-module --dry-run --host-triple=x86_64-unknown-windows-gnu \ -// RUN: --linker-path=/usr/bin/ld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefixes=CUDA,CUDA-COFF +// RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefixes=CUDA,CUDA-COFF // CUDA-ELF: @__start_cuda_offloading_entries = external hidden constant [0 x %struct.__tgt_offload_entry] // CUDA-ELF-NEXT: @__stop_cuda_offloading_entries = external hidden constant [0 x %struct.__tgt_offload_entry] @@ -145,11 +145,11 @@ // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o \ // RUN: -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --print-wrapped-module --dry-run --host-triple=x86_64-unknown-linux-gnu \ -// RUN: --linker-path=/usr/bin/ld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefixes=HIP,HIP-ELF +// RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefixes=HIP,HIP-ELF // RUN: clang-linker-wrapper --print-wrapped-module --dry-run --host-triple=x86_64-unknown-linux-gnu -r \ -// RUN: --linker-path=/usr/bin/ld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefixes=HIP,HIP-ELF +// RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefixes=HIP,HIP-ELF // RUN: clang-linker-wrapper --print-wrapped-module --dry-run --host-triple=x86_64-unknown-windows-gnu \ -// RUN: --linker-path=/usr/bin/ld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefixes=HIP,HIP-COFF +// RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefixes=HIP,HIP-COFF // HIP-ELF: @__start_hip_offloading_entries = external hidden constant [0 x %struct.__tgt_offload_entry] // HIP-ELF-NEXT: @__stop_hip_offloading_entries = external hidden constant [0 x %struct.__tgt_offload_entry] diff --git a/clang/test/Driver/linker-wrapper-libs.c b/clang/test/Driver/linker-wrapper-libs.c index 2073092bdbcf..9a78200d7d3c 100644 --- a/clang/test/Driver/linker-wrapper-libs.c +++ b/clang/test/Driver/linker-wrapper-libs.c @@ -43,7 +43,7 @@ int bar() { return weak; } // RUN: --image=file=%t.elf.o,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030 // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \ -// RUN: --linker-path=/usr/bin/ld -- %t.o %t.a -o a.out 2>&1 \ +// RUN: --linker-path=/usr/bin/ld %t.o %t.a -o a.out 2>&1 \ // RUN: | FileCheck %s --check-prefix=LIBRARY-RESOLVES // LIBRARY-RESOLVES: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx1030 {{.*}}.o {{.*}}.o @@ -65,7 +65,7 @@ int bar() { return weak; } // RUN: --image=file=%t.elf.o,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030 // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \ -// RUN: --linker-path=/usr/bin/ld -- %t.o %t.a -o a.out 2>&1 \ +// RUN: --linker-path=/usr/bin/ld %t.o %t.a -o a.out 2>&1 \ // RUN: | FileCheck %s --check-prefix=LIBRARY-GLOBAL // LIBRARY-GLOBAL: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx1030 {{.*}}.o {{.*}}.o @@ -88,7 +88,7 @@ int bar() { return weak; } // RUN: --image=file=%t.elf.o,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030 // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \ -// RUN: --linker-path=/usr/bin/ld -- %t.o %t.a -o a.out 2>&1 \ +// RUN: --linker-path=/usr/bin/ld %t.o %t.a -o a.out 2>&1 \ // RUN: | FileCheck %s --check-prefix=LIBRARY-GLOBAL-NONE // LIBRARY-GLOBAL-NONE-NOT: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx1030 {{.*}}.o {{.*}}.o @@ -109,7 +109,7 @@ int bar() { return weak; } // RUN: --image=file=%t.elf.o,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030 // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \ -// RUN: --linker-path=/usr/bin/ld -- %t.o %t.a -o a.out 2>&1 \ +// RUN: --linker-path=/usr/bin/ld %t.o %t.a -o a.out 2>&1 \ // RUN: | FileCheck %s --check-prefix=LIBRARY-WEAK // LIBRARY-WEAK: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx1030 @@ -131,7 +131,7 @@ int bar() { return weak; } // RUN: --image=file=%t.elf.o,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030 // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \ -// RUN: --linker-path=/usr/bin/ld -- %t.o %t.a -o a.out 2>&1 \ +// RUN: --linker-path=/usr/bin/ld %t.o %t.a -o a.out 2>&1 \ // RUN: | FileCheck %s --check-prefix=LIBRARY-HIDDEN // LIBRARY-HIDDEN: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx1030 @@ -154,7 +154,7 @@ int bar() { return weak; } // RUN: --image=file=%t.elf.o,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030 // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \ -// RUN: --linker-path=/usr/bin/ld -- %t.o %t.a %t.a -o a.out 2>&1 \ +// RUN: --linker-path=/usr/bin/ld %t.o %t.a %t.a -o a.out 2>&1 \ // RUN: | FileCheck %s --check-prefix=LIBRARY-GLOBAL-DEFINED // LIBRARY-GLOBAL-DEFINED: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx1030 {{.*}}.o {{.*}}.o @@ -178,7 +178,7 @@ int bar() { return weak; } // RUN: --image=file=%t.elf.o,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030 // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \ -// RUN: --linker-path=/usr/bin/ld -- %t.o --whole-archive %t.a -o a.out 2>&1 \ +// RUN: --linker-path=/usr/bin/ld %t.o --whole-archive %t.a -o a.out 2>&1 \ // RUN: | FileCheck %s --check-prefix=LIBRARY-WHOLE-ARCHIVE // LIBRARY-WHOLE-ARCHIVE: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx1030 {{.*}}.o {{.*}}.o diff --git a/clang/test/Driver/linker-wrapper.c b/clang/test/Driver/linker-wrapper.c index 83df2b84adef..c37f01189d08 100644 --- a/clang/test/Driver/linker-wrapper.c +++ b/clang/test/Driver/linker-wrapper.c @@ -16,10 +16,10 @@ __attribute__((visibility("protected"), used)) int x; // RUN: --image=file=%t.elf.o,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70 // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \ -// RUN: --linker-path=/usr/bin/ld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=NVPTX-LINK +// RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=NVPTX-LINK // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-llvm-bc -o %t.o -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \ -// RUN: --linker-path=/usr/bin/ld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=NVPTX-LINK +// RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=NVPTX-LINK // NVPTX-LINK: clang{{.*}} -o {{.*}}.img --target=nvptx64-nvidia-cuda -march=sm_70 -O2 {{.*}}.o {{.*}}.o @@ -28,7 +28,7 @@ __attribute__((visibility("protected"), used)) int x; // RUN: --image=file=%t.elf.o,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70 // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run --device-debug -O0 \ -// RUN: --linker-path=/usr/bin/ld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=NVPTX-LINK-DEBUG +// RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=NVPTX-LINK-DEBUG // NVPTX-LINK-DEBUG: clang{{.*}} -o {{.*}}.img --target=nvptx64-nvidia-cuda -march=sm_70 -O2 {{.*}}.o {{.*}}.o -g @@ -37,7 +37,7 @@ __attribute__((visibility("protected"), used)) int x; // RUN: --image=file=%t.elf.o,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx908 // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \ -// RUN: --linker-path=/usr/bin/ld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=AMDGPU-LINK +// RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=AMDGPU-LINK // AMDGPU-LINK: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx908 -O2 -Wl,--no-undefined {{.*}}.o {{.*}}.o @@ -46,7 +46,7 @@ __attribute__((visibility("protected"), used)) int x; // RUN: --image=file=%t.amdgpu.bc,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030 // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run --save-temps -O2 \ -// RUN: --linker-path=/usr/bin/ld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=AMDGPU-LTO-TEMPS +// RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=AMDGPU-LTO-TEMPS // AMDGPU-LTO-TEMPS: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx1030 -O2 -Wl,--no-undefined {{.*}}.s -save-temps @@ -56,14 +56,14 @@ __attribute__((visibility("protected"), used)) int x; // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out // RUN: llvm-ar rcs %t.a %t.o // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \ -// RUN: --linker-path=/usr/bin/ld.lld -- --whole-archive %t.a --no-whole-archive \ +// RUN: --linker-path=/usr/bin/ld.lld --whole-archive %t.a --no-whole-archive \ // RUN: %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=CPU-LINK // CPU-LINK: clang{{.*}} -o {{.*}}.img --target=x86_64-unknown-linux-gnu -march=native -O2 -Wl,--no-undefined {{.*}}.o {{.*}}.o -Wl,-Bsymbolic -shared -Wl,--whole-archive {{.*}}.a -Wl,--no-whole-archive // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o // RUN: clang-linker-wrapper --dry-run --host-triple=x86_64-unknown-linux-gnu -mllvm -openmp-opt-disable \ -// RUN: --linker-path=/usr/bin/ld.lld -- -a -b -c %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=HOST-LINK +// RUN: --linker-path=/usr/bin/ld.lld -a -b -c %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=HOST-LINK // HOST-LINK: ld.lld{{.*}}-a -b -c {{.*}}.o -o a.out // HOST-LINK-NOT: ld.lld{{.*}}-abc @@ -77,7 +77,7 @@ __attribute__((visibility("protected"), used)) int x; // RUN: --image=file=%t.elf.o,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70 // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t-obj.o -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \ -// RUN: --linker-path=/usr/bin/ld -- %t.a %t-obj.o -o a.out 2>&1 | FileCheck %s --check-prefix=STATIC-LIBRARY +// RUN: --linker-path=/usr/bin/ld %t.a %t-obj.o -o a.out 2>&1 | FileCheck %s --check-prefix=STATIC-LIBRARY // STATIC-LIBRARY: clang{{.*}} -march=sm_70 // STATIC-LIBRARY-NOT: clang{{.*}} -march=sm_50 @@ -89,7 +89,7 @@ __attribute__((visibility("protected"), used)) int x; // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o \ // RUN: -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --dry-run --host-triple=x86_64-unknown-linux-gnu \ -// RUN: --linker-path=/usr/bin/ld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=CUDA +// RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=CUDA // CUDA: clang{{.*}} -o [[IMG_SM52:.+]] --target=nvptx64-nvidia-cuda -march=sm_52 // CUDA: clang{{.*}} -o [[IMG_SM70:.+]] --target=nvptx64-nvidia-cuda -march=sm_70 @@ -104,7 +104,7 @@ __attribute__((visibility("protected"), used)) int x; // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o \ // RUN: -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --dry-run --host-triple=x86_64-unknown-linux-gnu --wrapper-jobs=4 \ -// RUN: --linker-path=/usr/bin/ld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=CUDA-PAR +// RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=CUDA-PAR // CUDA-PAR: fatbinary{{.*}}-64 --create {{.*}}.fatbin @@ -115,7 +115,7 @@ __attribute__((visibility("protected"), used)) int x; // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o \ // RUN: -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --dry-run --host-triple=x86_64-unknown-linux-gnu \ -// RUN: --linker-path=/usr/bin/ld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=HIP +// RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=HIP // HIP: clang{{.*}} -o [[IMG_GFX908:.+]] --target=amdgcn-amd-amdhsa -mcpu=gfx908 // HIP: clang{{.*}} -o [[IMG_GFX90A:.+]] --target=amdgcn-amd-amdhsa -mcpu=gfx90a @@ -127,14 +127,14 @@ __attribute__((visibility("protected"), used)) int x; // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o \ // RUN: -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --dry-run --host-triple=x86_64-unknown-linux-gnu \ -// RUN: --linker-path=/usr/bin/ld --device-linker=a --device-linker=nvptx64-nvidia-cuda=b -- \ +// RUN: --linker-path=/usr/bin/ld --device-linker=a --device-linker=nvptx64-nvidia-cuda=b \ // RUN: %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=LINKER-ARGS // LINKER-ARGS: clang{{.*}}--target=amdgcn-amd-amdhsa{{.*}}a // LINKER-ARGS: clang{{.*}}--target=nvptx64-nvidia-cuda{{.*}}a b // RUN: not clang-linker-wrapper --dry-run --host-triple=x86_64-unknown-linux-gnu -ldummy \ -// RUN: --linker-path=/usr/bin/ld --device-linker=a --device-linker=nvptx64-nvidia-cuda=b -- \ +// RUN: --linker-path=/usr/bin/ld --device-linker=a --device-linker=nvptx64-nvidia-cuda=b \ // RUN: -o a.out 2>&1 | FileCheck %s --check-prefix=MISSING-LIBRARY // MISSING-LIBRARY: error: unable to find library -ldummy @@ -144,7 +144,7 @@ __attribute__((visibility("protected"), used)) int x; // RUN: --image=file=%t.amdgpu.bc,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx908 // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run --clang-backend \ -// RUN: --linker-path=/usr/bin/ld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=CLANG-BACKEND +// RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=CLANG-BACKEND // CLANG-BACKEND: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx908 -O2 -Wl,--no-undefined {{.*}}.bc @@ -152,7 +152,7 @@ __attribute__((visibility("protected"), used)) int x; // RUN: --image=file=%t.elf.o,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70 // RUN: %clang -cc1 %s -triple x86_64-unknown-windows-msvc -emit-obj -o %t.o -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-windows-msvc --dry-run \ -// RUN: --linker-path=/usr/bin/lld-link -- %t.o -libpath:./ -out:a.exe 2>&1 | FileCheck %s --check-prefix=COFF +// RUN: --linker-path=/usr/bin/lld-link %t.o -libpath:./ -out:a.exe 2>&1 | FileCheck %s --check-prefix=COFF // COFF: "/usr/bin/lld-link" {{.*}}.o -libpath:./ -out:a.exe {{.*}}openmp.image.wrapper{{.*}} @@ -167,7 +167,7 @@ __attribute__((visibility("protected"), used)) int x; // RUN: --image=file=%t.elf.o,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx90a:xnack- // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t-off.o -fembed-offload-object=%t-off.out // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \ -// RUN: --linker-path=/usr/bin/ld -- %t-on.o %t-off.o %t.a -o a.out 2>&1 | FileCheck %s --check-prefix=AMD-TARGET-ID +// RUN: --linker-path=/usr/bin/ld %t-on.o %t-off.o %t.a -o a.out 2>&1 | FileCheck %s --check-prefix=AMD-TARGET-ID // AMD-TARGET-ID: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx90a:xnack+ -O2 -Wl,--no-undefined {{.*}}.o {{.*}}.o // AMD-TARGET-ID: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx90a:xnack- -O2 -Wl,--no-undefined {{.*}}.o {{.*}}.o @@ -183,7 +183,7 @@ __attribute__((visibility("protected"), used)) int x; // RUN: --image=file=%t.elf.o,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx908 // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t2.o -fembed-offload-object=%t2.out // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \ -// RUN: --linker-path=/usr/bin/ld -- %t1.o %t2.o %t.a -o a.out 2>&1 | FileCheck %s --check-prefix=ARCH-ALL +// RUN: --linker-path=/usr/bin/ld %t1.o %t2.o %t.a -o a.out 2>&1 | FileCheck %s --check-prefix=ARCH-ALL // ARCH-ALL: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx908 -O2 -Wl,--no-undefined {{.*}}.o {{.*}}.o // ARCH-ALL: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx90a -O2 -Wl,--no-undefined {{.*}}.o {{.*}}.o @@ -193,7 +193,7 @@ __attribute__((visibility("protected"), used)) int x; // RUN: --image=file=%t.elf.o,kind=openmp,triple=x86_64-unknown-linux-gnu // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \ -// RUN: --linker-path=/usr/bin/ld.lld -- -r %t.o \ +// RUN: --linker-path=/usr/bin/ld.lld -r %t.o \ // RUN: %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=RELOCATABLE-LINK // RELOCATABLE-LINK: clang{{.*}} -o {{.*}}.img --target=x86_64-unknown-linux-gnu @@ -205,7 +205,7 @@ __attribute__((visibility("protected"), used)) int x; // RUN: --image=file=%t.elf.o,kind=hip,triple=amdgcn-amd-amdhsa,arch=gfx90a // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \ -// RUN: --linker-path=/usr/bin/ld.lld -- -r %t.o \ +// RUN: --linker-path=/usr/bin/ld.lld -r %t.o \ // RUN: %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=RELOCATABLE-LINK-HIP // RELOCATABLE-LINK-HIP: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa @@ -218,7 +218,7 @@ __attribute__((visibility("protected"), used)) int x; // RUN: --image=file=%t.elf.o,kind=cuda,triple=nvptx64-nvidia-cuda,arch=sm_89 // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \ -// RUN: --linker-path=/usr/bin/ld.lld -- -r %t.o \ +// RUN: --linker-path=/usr/bin/ld.lld -r %t.o \ // RUN: %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=RELOCATABLE-LINK-CUDA // RELOCATABLE-LINK-CUDA: clang{{.*}} -o {{.*}}.img --target=nvptx64-nvidia-cuda diff --git a/clang/test/Driver/openmp-offload-gpu.c b/clang/test/Driver/openmp-offload-gpu.c index f7b06c9ec595..d705be44e595 100644 --- a/clang/test/Driver/openmp-offload-gpu.c +++ b/clang/test/Driver/openmp-offload-gpu.c @@ -233,7 +233,7 @@ // CHECK: "-cc1" "-triple" "x86_64-unknown-linux-gnu"{{.*}}"-emit-llvm-bc"{{.*}}"-x" "c" // CHECK: "-cc1" "-triple" "nvptx64-nvidia-cuda" "-aux-triple" "x86_64-unknown-linux-gnu"{{.*}}"-target-cpu" "sm_52" // CHECK: "-cc1" "-triple" "x86_64-unknown-linux-gnu"{{.*}}"-emit-obj" -// CHECK: clang-linker-wrapper{{.*}}"--"{{.*}} "-o" "a.out" +// CHECK: clang-linker-wrapper{{.*}} "-o" "a.out" // RUN: %clang -ccc-print-phases --target=x86_64-unknown-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target=nvptx64-nvidia-cuda -march=sm_52 %s 2>&1 \ // RUN: | FileCheck --check-prefix=CHECK-PHASES %s diff --git a/clang/test/Driver/openmp-offload-infer.c b/clang/test/Driver/openmp-offload-infer.c index 9a949f52e2e9..50333293eb7d 100644 --- a/clang/test/Driver/openmp-offload-infer.c +++ b/clang/test/Driver/openmp-offload-infer.c @@ -13,7 +13,7 @@ // CHECK: "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"{{.*}}"-target-cpu" "gfx803" // CHECK: "-cc1" "-triple" "nvptx64-nvidia-cuda" "-aux-triple" "x86_64-unknown-linux-gnu"{{.*}}"-target-cpu" "sm_52" // CHECK: "-cc1" "-triple" "x86_64-unknown-linux-gnu"{{.*}}"-emit-obj" -// CHECK: clang-linker-wrapper{{.*}}"--"{{.*}} "-o" "a.out" +// CHECK: clang-linker-wrapper{{.*}} "-o" "a.out" // RUN: %clang -### --target=x86_64-unknown-linux-gnu -ccc-print-bindings -fopenmp=libomp \ // RUN: --offload-arch=sm_70 --offload-arch=gfx908:sramecc+:xnack- \ diff --git a/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td b/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td index 763426570c2a..2c6a788cf23a 100644 --- a/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td +++ b/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td @@ -84,10 +84,6 @@ def linker_arg_EQ : Joined<["--"], "linker-arg=">, Flags<[DeviceOnlyOption, HelpHidden]>, HelpText<"An extra argument to be passed to the linker">; -// Separator between the linker wrapper and host linker flags. -def separator : Flag<["--"], "">, Flags<[WrapperOnlyOption]>, - HelpText<"The separator for the wrapped linker arguments">; - // Arguments for the LLVM backend. def mllvm : Separate<["-"], "mllvm">, Flags<[WrapperOnlyOption]>, MetaVarName<"">, HelpText<"Arguments passed to the LLVM invocation">; -- GitLab From 4cfd4a7896b5fd50274ec8573c259d7ad41741de Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 7 Mar 2024 13:53:02 +0000 Subject: [PATCH 441/929] [LAA] Add test case for #82665. Test case for https://github.com/llvm/llvm-project/issues/82665. --- .../underlying-object-loop-varying-phi.ll | 175 ++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 llvm/test/Analysis/LoopAccessAnalysis/underlying-object-loop-varying-phi.ll diff --git a/llvm/test/Analysis/LoopAccessAnalysis/underlying-object-loop-varying-phi.ll b/llvm/test/Analysis/LoopAccessAnalysis/underlying-object-loop-varying-phi.ll new file mode 100644 index 000000000000..1a5a6ac08d40 --- /dev/null +++ b/llvm/test/Analysis/LoopAccessAnalysis/underlying-object-loop-varying-phi.ll @@ -0,0 +1,175 @@ +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -passes='print' -disable-output %s 2>&1 | FileCheck %s + +target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" + +; Test case for https://github.com/llvm/llvm-project/issues/82665. +define void @indirect_ptr_recurrences_read_write(ptr %A, ptr %B) { +; CHECK-LABEL: 'indirect_ptr_recurrences_read_write' +; CHECK-NEXT: loop: +; CHECK-NEXT: Memory dependences are safe +; CHECK-NEXT: Dependences: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ] + %ptr.recur = phi ptr [ %A, %entry ], [ %ptr.next, %loop ] + %gep.B = getelementptr inbounds ptr, ptr %B, i64 %iv + %ptr.next = load ptr, ptr %gep.B, align 8, !tbaa !6 + %l = load i32, ptr %ptr.recur, align 4, !tbaa !10 + %xor = xor i32 %l, 1 + store i32 %xor, ptr %ptr.recur, align 4, !tbaa !10 + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 5 + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + +define i32 @indirect_ptr_recurrences_read_only_loop(ptr %A, ptr %B) { +; CHECK-LABEL: 'indirect_ptr_recurrences_read_only_loop' +; CHECK-NEXT: loop: +; CHECK-NEXT: Memory dependences are safe +; CHECK-NEXT: Dependences: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ] + %ptr.recur = phi ptr [ %A, %entry ], [ %ptr.next, %loop ] + %red = phi i32 [ 0, %entry ], [ %xor, %loop ] + %gep.B = getelementptr inbounds ptr, ptr %B, i64 %iv + %ptr.next = load ptr, ptr %gep.B, align 8, !tbaa !6 + %l = load i32, ptr %ptr.recur, align 4, !tbaa !10 + %xor = xor i32 %l, 1 + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 5 + br i1 %ec, label %exit, label %loop + +exit: + ret i32 %xor +} + +define void @indirect_ptr_recurrences_read_write_may_alias_no_tbaa(ptr %A, ptr %B) { +; CHECK-LABEL: 'indirect_ptr_recurrences_read_write_may_alias_no_tbaa' +; CHECK-NEXT: loop: +; CHECK-NEXT: Report: cannot identify array bounds +; CHECK-NEXT: Dependences: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ] + %ptr.recur = phi ptr [ %A, %entry ], [ %ptr.next, %loop ] + %gep.B = getelementptr inbounds ptr, ptr %B, i64 %iv + %ptr.next = load ptr, ptr %gep.B, align 8, !tbaa !6 + %l = load i32, ptr %ptr.recur, align 4 + %xor = xor i32 %l, 1 + store i32 %xor, ptr %ptr.recur, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 5 + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + +define void @indirect_ptr_recurrences_read_write_may_alias_different_obj(ptr %A, ptr %B, ptr %C) { +; CHECK-LABEL: 'indirect_ptr_recurrences_read_write_may_alias_different_obj' +; CHECK-NEXT: loop: +; CHECK-NEXT: Report: cannot identify array bounds +; CHECK-NEXT: Dependences: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ] + %ptr.recur = phi ptr [ %A, %entry ], [ %ptr.next, %loop ] + %gep.B = getelementptr inbounds ptr, ptr %B, i64 %iv + %ptr.next = load ptr, ptr %gep.B, align 8, !tbaa !6 + %l = load i32, ptr %ptr.recur, align 4 + %xor = xor i32 %l, 1 + %gep.C = getelementptr inbounds ptr, ptr %C, i64 %iv + store i32 %xor, ptr %gep.C, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 5 + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + +define void @indirect_ptr_recurrences_read_write_may_noalias_different_obj(ptr %A, ptr %B, ptr noalias %C) { +; CHECK-LABEL: 'indirect_ptr_recurrences_read_write_may_noalias_different_obj' +; CHECK-NEXT: loop: +; CHECK-NEXT: Memory dependences are safe +; CHECK-NEXT: Dependences: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ] + %ptr.recur = phi ptr [ %A, %entry ], [ %ptr.next, %loop ] + %gep.B = getelementptr inbounds ptr, ptr %B, i64 %iv + %ptr.next = load ptr, ptr %gep.B, align 8, !tbaa !6 + %l = load i32, ptr %ptr.recur, align 4 + %xor = xor i32 %l, 1 + %gep.C = getelementptr inbounds ptr, ptr %C, i64 %iv + store i32 %xor, ptr %gep.C, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 5 + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + + +!6 = !{!7, !7, i64 0} +!7 = !{!"any pointer", !8, i64 0} +!8 = !{!"omnipotent char", !9, i64 0} +!9 = !{!"Simple C/C++ TBAA"} +!10 = !{!11, !11, i64 0} +!11 = !{!"int", !8, i64 0} -- GitLab From 61b13e0dfe1b476d9bf0fe477983be8471cfd26b Mon Sep 17 00:00:00 2001 From: "Yaxun (Sam) Liu" Date: Thu, 7 Mar 2024 09:26:16 -0500 Subject: [PATCH 442/929] [ClangOffloadBundler] fix unbundling archive (#84195) When unbundling an archive, need to save the content of each object file to a temporary file before passing it to llvm-objcopy, instead of passing the original input archive file to llvm-objcopy. Also allows extracting host bundles for archives. Fixes: https://github.com/llvm/llvm-project/issues/83509 --- clang/lib/Driver/OffloadBundler.cpp | 30 ++++++++++++++++----- clang/test/Driver/clang-offload-bundler.c | 32 +++++++++++++++++++++++ 2 files changed, 55 insertions(+), 7 deletions(-) diff --git a/clang/lib/Driver/OffloadBundler.cpp b/clang/lib/Driver/OffloadBundler.cpp index 99a34d25cfcd..f9eadfaec88d 100644 --- a/clang/lib/Driver/OffloadBundler.cpp +++ b/clang/lib/Driver/OffloadBundler.cpp @@ -590,7 +590,8 @@ public: // Copy fat object contents to the output when extracting host bundle. std::string ModifiedContent; if (Content.size() == 1u && Content.front() == 0) { - auto HostBundleOrErr = getHostBundle(); + auto HostBundleOrErr = getHostBundle( + StringRef(Input.getBufferStart(), Input.getBufferSize())); if (!HostBundleOrErr) return HostBundleOrErr.takeError(); @@ -700,7 +701,7 @@ private: return Error::success(); } - Expected getHostBundle() { + Expected getHostBundle(StringRef Input) { TempFileHandlerRAII TempFiles; auto ModifiedObjPathOrErr = TempFiles.Create(std::nullopt); @@ -715,7 +716,24 @@ private: ObjcopyArgs.push_back("--regex"); ObjcopyArgs.push_back("--remove-section=__CLANG_OFFLOAD_BUNDLE__.*"); ObjcopyArgs.push_back("--"); - ObjcopyArgs.push_back(BundlerConfig.InputFileNames.front()); + + StringRef ObjcopyInputFileName; + // When unbundling an archive, the content of each object file in the + // archive is passed to this function by parameter Input, which is different + // from the content of the original input archive file, therefore it needs + // to be saved to a temporary file before passed to llvm-objcopy. Otherwise, + // Input is the same as the content of the original input file, therefore + // temporary file is not needed. + if (StringRef(BundlerConfig.FilesType).starts_with("a")) { + auto InputFileOrErr = + TempFiles.Create(ArrayRef(Input.data(), Input.size())); + if (!InputFileOrErr) + return InputFileOrErr.takeError(); + ObjcopyInputFileName = *InputFileOrErr; + } else + ObjcopyInputFileName = BundlerConfig.InputFileNames.front(); + + ObjcopyArgs.push_back(ObjcopyInputFileName); ObjcopyArgs.push_back(ModifiedObjPath); if (Error Err = executeObjcopy(BundlerConfig.ObjcopyPath, ObjcopyArgs)) @@ -1628,10 +1646,8 @@ Error OffloadBundler::UnbundleArchive() { while (!CodeObject.empty()) { SmallVector CompatibleTargets; auto CodeObjectInfo = OffloadTargetInfo(CodeObject, BundlerConfig); - if (CodeObjectInfo.hasHostKind()) { - // Do nothing, we don't extract host code yet. - } else if (getCompatibleOffloadTargets(CodeObjectInfo, CompatibleTargets, - BundlerConfig)) { + if (getCompatibleOffloadTargets(CodeObjectInfo, CompatibleTargets, + BundlerConfig)) { std::string BundleData; raw_string_ostream DataStream(BundleData); if (Error Err = FileHandler->ReadBundle(DataStream, CodeObjectBuffer)) diff --git a/clang/test/Driver/clang-offload-bundler.c b/clang/test/Driver/clang-offload-bundler.c index 9d8b81ee9806..f3cd2493e052 100644 --- a/clang/test/Driver/clang-offload-bundler.c +++ b/clang/test/Driver/clang-offload-bundler.c @@ -13,6 +13,19 @@ // RUN: obj2yaml %t.o > %t.o.yaml // RUN: %clang -O0 -target %itanium_abi_triple %s -emit-ast -o %t.ast +// RUN: echo 'void a() {}' >%t.a.cpp +// RUN: echo 'void b() {}' >%t.b.cpp +// RUN: %clang -target %itanium_abi_triple %t.a.cpp -c -o %t.a.o +// RUN: %clang -target %itanium_abi_triple %t.b.cpp -c -o %t.b.o +// +// Remove .llvm_addrsig section since its offset changes after llvm-objcopy +// removes clang-offload-bundler sections, therefore not good for comparison. +// +// RUN: llvm-objcopy --remove-section=.llvm_addrsig %t.a.o +// RUN: llvm-objcopy --remove-section=.llvm_addrsig %t.b.o +// RUN: obj2yaml %t.a.o > %t.a.yaml +// RUN: obj2yaml %t.b.o > %t.b.yaml + // // Generate an empty file to help with the checks of empty files. // @@ -414,6 +427,25 @@ // HIP-AR-906-DAG: hip_bundle1-hip-amdgcn-amd-amdhsa--gfx906 // HIP-AR-906-DAG: hip_bundle2-hip-amdgcn-amd-amdhsa--gfx906 +// +// Check unbundling archive for host target +// +// RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,hip-amdgcn-amd-amdhsa--gfx900 \ +// RUN: -input=%t.a.o -input=%t.tgt1 -output=%t.a.bundled.o +// RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,hip-amdgcn-amd-amdhsa--gfx900 \ +// RUN: -input=%t.b.o -input=%t.tgt1 -output=%t.b.bundled.o +// RUN: rm -f %t.bundled.a +// RUN: llvm-ar cr %t.bundled.a %t.a.bundled.o %t.b.bundled.o +// RUN: cp %t.bundled.a %t.bundled.a.bak +// RUN: clang-offload-bundler -unbundle --targets=host-%itanium_abi_triple -type=a -input=%t.bundled.a -output=%t.host.a +// RUN: rm -f *%itanium_abi_triple*.a.bundled.o *%itanium_abi_triple*.b.bundled.o +// RUN: llvm-ar -x %t.host.a +// RUN: diff %t.bundled.a %t.bundled.a.bak +// RUN: obj2yaml *%itanium_abi_triple*.a.bundled.o > %t.a.unbundled.yaml +// RUN: diff %t.a.unbundled.yaml %t.a.yaml +// RUN: obj2yaml *%itanium_abi_triple*.b.bundled.o > %t.b.unbundled.yaml +// RUN: diff %t.b.unbundled.yaml %t.b.yaml +// // Check clang-offload-bundler reporting an error when trying to unbundle an archive but // the input file is not an archive. // -- GitLab From e4d4cfa5a0111372dff2b01126545cf3139ee40b Mon Sep 17 00:00:00 2001 From: Lei Huang Date: Thu, 7 Mar 2024 09:32:43 -0500 Subject: [PATCH 443/929] [libc++] Fixes time formatter test output for Linux on PowerPC (#75526) Fix output to match actual. --- .../test/std/time/time.syn/formatter.file_time.pass.cpp | 8 ++++++-- .../test/std/time/time.syn/formatter.local_time.pass.cpp | 8 ++++++-- libcxx/test/std/time/time.syn/formatter.sys_time.pass.cpp | 8 ++++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/libcxx/test/std/time/time.syn/formatter.file_time.pass.cpp b/libcxx/test/std/time/time.syn/formatter.file_time.pass.cpp index aec6e78d994d..b07282593d75 100644 --- a/libcxx/test/std/time/time.syn/formatter.file_time.pass.cpp +++ b/libcxx/test/std/time/time.syn/formatter.file_time.pass.cpp @@ -821,7 +821,9 @@ static void test_valid_values_date_time() { // Use the global locale (fr_FR) check( // https://sourceware.org/bugzilla/show_bug.cgi?id=24054 -#if defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 29 +#if defined(__powerpc__) && defined(__linux__) + SV("%c='jeu. 01 janv. 1970 00:00:00 UTC'\t%Ec='jeu. 01 janv. 1970 00:00:00 UTC'\n"), +#elif defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 29 SV("%c='jeu. 01 janv. 1970 00:00:00 GMT'\t%Ec='jeu. 01 janv. 1970 00:00:00 GMT'\n"), #elif defined(_AIX) SV("%c=' 1 janvier 1970 à 00:00:00 UTC'\t%Ec=' 1 janvier 1970 à 00:00:00 UTC'\n"), @@ -839,7 +841,9 @@ static void test_valid_values_date_time() { check( // https://sourceware.org/bugzilla/show_bug.cgi?id=24054 -#if defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 29 +#if defined(__powerpc__) && defined(__linux__) + SV("%c='ven. 13 févr. 2009 23:31:30 UTC'\t%Ec='ven. 13 févr. 2009 23:31:30 UTC'\n"), +#elif defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 29 SV("%c='ven. 13 févr. 2009 23:31:30 GMT'\t%Ec='ven. 13 févr. 2009 23:31:30 GMT'\n"), #elif defined(_AIX) SV("%c='13 février 2009 à 23:31:30 UTC'\t%Ec='13 février 2009 à 23:31:30 UTC'\n"), diff --git a/libcxx/test/std/time/time.syn/formatter.local_time.pass.cpp b/libcxx/test/std/time/time.syn/formatter.local_time.pass.cpp index 71dd7fba9fb7..45c3a12ea35c 100644 --- a/libcxx/test/std/time/time.syn/formatter.local_time.pass.cpp +++ b/libcxx/test/std/time/time.syn/formatter.local_time.pass.cpp @@ -820,7 +820,9 @@ static void test_valid_values_date_time() { // Use the global locale (fr_FR) check( // https://sourceware.org/bugzilla/show_bug.cgi?id=24054 -#if defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 29 +#if defined(__powerpc__) && defined(__linux__) + SV("%c='jeu. 01 janv. 1970 00:00:00 UTC'\t%Ec='jeu. 01 janv. 1970 00:00:00 UTC'\n"), +#elif defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 29 SV("%c='jeu. 01 janv. 1970 00:00:00 GMT'\t%Ec='jeu. 01 janv. 1970 00:00:00 GMT'\n"), #elif defined(_AIX) SV("%c=' 1 janvier 1970 à 00:00:00 UTC'\t%Ec=' 1 janvier 1970 à 00:00:00 UTC'\n"), @@ -838,7 +840,9 @@ static void test_valid_values_date_time() { check( // https://sourceware.org/bugzilla/show_bug.cgi?id=24054 -#if defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 29 +#if defined(__powerpc__) && defined(__linux__) + SV("%c='ven. 13 févr. 2009 23:31:30 UTC'\t%Ec='ven. 13 févr. 2009 23:31:30 UTC'\n"), +#elif defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 29 SV("%c='ven. 13 févr. 2009 23:31:30 GMT'\t%Ec='ven. 13 févr. 2009 23:31:30 GMT'\n"), #elif defined(_AIX) SV("%c='13 février 2009 à 23:31:30 UTC'\t%Ec='13 février 2009 à 23:31:30 UTC'\n"), diff --git a/libcxx/test/std/time/time.syn/formatter.sys_time.pass.cpp b/libcxx/test/std/time/time.syn/formatter.sys_time.pass.cpp index ebc45c7e8735..2fed270cbade 100644 --- a/libcxx/test/std/time/time.syn/formatter.sys_time.pass.cpp +++ b/libcxx/test/std/time/time.syn/formatter.sys_time.pass.cpp @@ -817,7 +817,9 @@ static void test_valid_values_date_time() { // Use the global locale (fr_FR) check( // https://sourceware.org/bugzilla/show_bug.cgi?id=24054 -#if defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 29 +#if defined(__powerpc__) && defined(__linux__) + SV("%c='jeu. 01 janv. 1970 00:00:00 UTC'\t%Ec='jeu. 01 janv. 1970 00:00:00 UTC'\n"), +#elif defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 29 SV("%c='jeu. 01 janv. 1970 00:00:00 GMT'\t%Ec='jeu. 01 janv. 1970 00:00:00 GMT'\n"), #elif defined(_AIX) SV("%c=' 1 janvier 1970 à 00:00:00 UTC'\t%Ec=' 1 janvier 1970 à 00:00:00 UTC'\n"), @@ -835,7 +837,9 @@ static void test_valid_values_date_time() { check( // https://sourceware.org/bugzilla/show_bug.cgi?id=24054 -#if defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 29 +#if defined(__powerpc__) && defined(__linux__) + SV("%c='ven. 13 févr. 2009 23:31:30 UTC'\t%Ec='ven. 13 févr. 2009 23:31:30 UTC'\n"), +#elif defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 29 SV("%c='ven. 13 févr. 2009 23:31:30 GMT'\t%Ec='ven. 13 févr. 2009 23:31:30 GMT'\n"), #elif defined(_AIX) SV("%c='13 février 2009 à 23:31:30 UTC'\t%Ec='13 février 2009 à 23:31:30 UTC'\n"), -- GitLab From f355cd6f6c51580316e1e88ef5534bd2f8cfa498 Mon Sep 17 00:00:00 2001 From: Marius Brehler Date: Thu, 7 Mar 2024 15:48:11 +0100 Subject: [PATCH 444/929] [mlir][EmitC] Allow further ops within expressions (#84284) This adds the `CExpression` trait to additional ops to allow to use these ops within the expression operation. Furthermore, the operator precedence is defined for those ops. --- mlir/include/mlir/Dialect/EmitC/IR/EmitC.td | 22 +++++++++-------- mlir/lib/Target/Cpp/TranslateToCpp.cpp | 26 ++++++++++++++------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td index 02ab73fa2ca5..db0e2d10960d 100644 --- a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td +++ b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td @@ -106,7 +106,7 @@ def EmitC_ApplyOp : EmitC_Op<"apply", [CExpression]> { let hasVerifier = 1; } -def EmitC_BitwiseAndOp : EmitC_BinaryOp<"bitwise_and", []> { +def EmitC_BitwiseAndOp : EmitC_BinaryOp<"bitwise_and", [CExpression]> { let summary = "Bitwise and operation"; let description = [{ With the `bitwise_and` operation the bitwise operator & (and) can @@ -124,7 +124,8 @@ def EmitC_BitwiseAndOp : EmitC_BinaryOp<"bitwise_and", []> { }]; } -def EmitC_BitwiseLeftShiftOp : EmitC_BinaryOp<"bitwise_left_shift", []> { +def EmitC_BitwiseLeftShiftOp : EmitC_BinaryOp<"bitwise_left_shift", + [CExpression]> { let summary = "Bitwise left shift operation"; let description = [{ With the `bitwise_left_shift` operation the bitwise operator << @@ -142,7 +143,7 @@ def EmitC_BitwiseLeftShiftOp : EmitC_BinaryOp<"bitwise_left_shift", []> { }]; } -def EmitC_BitwiseNotOp : EmitC_UnaryOp<"bitwise_not", []> { +def EmitC_BitwiseNotOp : EmitC_UnaryOp<"bitwise_not", [CExpression]> { let summary = "Bitwise not operation"; let description = [{ With the `bitwise_not` operation the bitwise operator ~ (not) can @@ -160,7 +161,7 @@ def EmitC_BitwiseNotOp : EmitC_UnaryOp<"bitwise_not", []> { }]; } -def EmitC_BitwiseOrOp : EmitC_BinaryOp<"bitwise_or", []> { +def EmitC_BitwiseOrOp : EmitC_BinaryOp<"bitwise_or", [CExpression]> { let summary = "Bitwise or operation"; let description = [{ With the `bitwise_or` operation the bitwise operator | (or) @@ -178,7 +179,8 @@ def EmitC_BitwiseOrOp : EmitC_BinaryOp<"bitwise_or", []> { }]; } -def EmitC_BitwiseRightShiftOp : EmitC_BinaryOp<"bitwise_right_shift", []> { +def EmitC_BitwiseRightShiftOp : EmitC_BinaryOp<"bitwise_right_shift", + [CExpression]> { let summary = "Bitwise right shift operation"; let description = [{ With the `bitwise_right_shift` operation the bitwise operator >> @@ -196,7 +198,7 @@ def EmitC_BitwiseRightShiftOp : EmitC_BinaryOp<"bitwise_right_shift", []> { }]; } -def EmitC_BitwiseXorOp : EmitC_BinaryOp<"bitwise_xor", []> { +def EmitC_BitwiseXorOp : EmitC_BinaryOp<"bitwise_xor", [CExpression]> { let summary = "Bitwise xor operation"; let description = [{ With the `bitwise_xor` operation the bitwise operator ^ (xor) @@ -515,7 +517,7 @@ def EmitC_ForOp : EmitC_Op<"for", } def EmitC_CallOp : EmitC_Op<"call", - [CallOpInterface, + [CallOpInterface, CExpression, DeclareOpInterfaceMethods]> { let summary = "call operation"; let description = [{ @@ -771,7 +773,7 @@ def EmitC_LiteralOp : EmitC_Op<"literal", [Pure]> { let assemblyFormat = "$value attr-dict `:` type($result)"; } -def EmitC_LogicalAndOp : EmitC_BinaryOp<"logical_and", []> { +def EmitC_LogicalAndOp : EmitC_BinaryOp<"logical_and", [CExpression]> { let summary = "Logical and operation"; let description = [{ With the `logical_and` operation the logical operator && (and) can @@ -792,7 +794,7 @@ def EmitC_LogicalAndOp : EmitC_BinaryOp<"logical_and", []> { let assemblyFormat = "operands attr-dict `:` type(operands)"; } -def EmitC_LogicalNotOp : EmitC_UnaryOp<"logical_not", []> { +def EmitC_LogicalNotOp : EmitC_UnaryOp<"logical_not", [CExpression]> { let summary = "Logical not operation"; let description = [{ With the `logical_not` operation the logical operator ! (negation) can @@ -813,7 +815,7 @@ def EmitC_LogicalNotOp : EmitC_UnaryOp<"logical_not", []> { let assemblyFormat = "operands attr-dict `:` type(operands)"; } -def EmitC_LogicalOrOp : EmitC_BinaryOp<"logical_or", []> { +def EmitC_LogicalOrOp : EmitC_BinaryOp<"logical_or", [CExpression]> { let summary = "Logical or operation"; let description = [{ With the `logical_or` operation the logical operator || (inclusive or) diff --git a/mlir/lib/Target/Cpp/TranslateToCpp.cpp b/mlir/lib/Target/Cpp/TranslateToCpp.cpp index 4bc707c43ad9..95513cb0fb2e 100644 --- a/mlir/lib/Target/Cpp/TranslateToCpp.cpp +++ b/mlir/lib/Target/Cpp/TranslateToCpp.cpp @@ -71,9 +71,17 @@ inline LogicalResult interleaveCommaWithError(const Container &c, /// imply higher precedence. static FailureOr getOperatorPrecedence(Operation *operation) { return llvm::TypeSwitch>(operation) - .Case([&](auto op) { return 11; }) - .Case([&](auto op) { return 13; }) - .Case([&](auto op) { return 13; }) + .Case([&](auto op) { return 12; }) + .Case([&](auto op) { return 15; }) + .Case([&](auto op) { return 7; }) + .Case([&](auto op) { return 11; }) + .Case([&](auto op) { return 15; }) + .Case([&](auto op) { return 5; }) + .Case([&](auto op) { return 11; }) + .Case([&](auto op) { return 6; }) + .Case([&](auto op) { return 16; }) + .Case([&](auto op) { return 16; }) + .Case([&](auto op) { return 15; }) .Case([&](auto op) -> FailureOr { switch (op.getPredicate()) { case emitc::CmpPredicate::eq: @@ -89,11 +97,13 @@ static FailureOr getOperatorPrecedence(Operation *operation) { } return op->emitError("unsupported cmp predicate"); }) - .Case([&](auto op) { return 12; }) - .Case([&](auto op) { return 12; }) - .Case([&](auto op) { return 12; }) - .Case([&](auto op) { return 11; }) - .Case([&](auto op) { return 14; }) + .Case([&](auto op) { return 13; }) + .Case([&](auto op) { return 4; }) + .Case([&](auto op) { return 15; }) + .Case([&](auto op) { return 3; }) + .Case([&](auto op) { return 13; }) + .Case([&](auto op) { return 13; }) + .Case([&](auto op) { return 12; }) .Default([](auto op) { return op->emitError("unsupported operation"); }); } -- GitLab From 043a020688765ad1ed27df718f908cfd0dc353a3 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Thu, 7 Mar 2024 08:47:57 -0600 Subject: [PATCH 445/929] [libc] Fix missing standard definitions in the GPU config Summary: Some dependencies on the standard C extensions are added transitively. This patch adds the new values. --- libc/config/gpu/api.td | 1 + 1 file changed, 1 insertion(+) diff --git a/libc/config/gpu/api.td b/libc/config/gpu/api.td index dbd212be56a3..607b8b6d5900 100644 --- a/libc/config/gpu/api.td +++ b/libc/config/gpu/api.td @@ -4,6 +4,7 @@ include "spec/stdc.td" include "spec/posix.td" include "spec/gpu_ext.td" include "spec/gnu_ext.td" +include "spec/stdc_ext.td" include "spec/llvm_libc_ext.td" def AssertMacro : MacroDef<"assert"> { -- GitLab From 2b8aaef09e2fd0b2a5581e198a73579c6939c717 Mon Sep 17 00:00:00 2001 From: Michael Maitland Date: Thu, 7 Mar 2024 09:50:29 -0500 Subject: [PATCH 446/929] [GISEL] Add IRTranslation for shufflevector on scalable vector types (#80378) This patch is stacked on https://github.com/llvm/llvm-project/pull/80372, https://github.com/llvm/llvm-project/pull/80307, and https://github.com/llvm/llvm-project/pull/80306. ShuffleVector on scalable vector types gets IRTranslate'd to G_SPLAT_VECTOR since a ShuffleVector that has operates on scalable vectors is a splat vector where the value of the splat vector is the 0th element of the first operand, because the index mask operand is the zeroinitializer (undef and poison are treated as zeroinitializer here). This is analogous to what happens in SelectionDAG for ShuffleVector. `buildSplatVector` is renamed to`buildBuildVectorSplatVector`. I did not make this a separate patch because it would cause problems to revert that change without reverting this change too. --- llvm/docs/GlobalISel/GenericOpcode.rst | 5 + .../CodeGen/GlobalISel/MachineIRBuilder.h | 12 +- llvm/include/llvm/Support/TargetOpcodes.def | 3 + llvm/include/llvm/Target/GenericOpcodes.td | 7 + llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp | 4 +- llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 27 +- .../CodeGen/GlobalISel/LegalizerHelper.cpp | 2 +- .../CodeGen/GlobalISel/MachineIRBuilder.cpp | 16 +- llvm/lib/CodeGen/MachineVerifier.cpp | 18 + llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 3 +- .../GlobalISel/legalizer-info-validation.mir | 3 + .../GlobalISel/irtranslator/shufflevector.ll | 1774 +++++++++++++++++ .../MachineVerifier/test_g_splat_vector.mir | 27 + .../GlobalISel/LegalizerHelperTest.cpp | 4 +- .../CodeGen/GlobalISel/PatternMatchTest.cpp | 6 +- 15 files changed, 1890 insertions(+), 21 deletions(-) create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/shufflevector.ll create mode 100644 llvm/test/MachineVerifier/test_g_splat_vector.mir diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst index 33b0152bd7b4..dda367607d04 100644 --- a/llvm/docs/GlobalISel/GenericOpcode.rst +++ b/llvm/docs/GlobalISel/GenericOpcode.rst @@ -639,6 +639,11 @@ Concatenate two vectors and shuffle the elements according to the mask operand. The mask operand should be an IR Constant which exactly matches the corresponding mask for the IR shufflevector instruction. +G_SPLAT_VECTOR +^^^^^^^^^^^^^^^^ + +Create a vector where all elements are the scalar from the source operand. + Vector Reduction Operations --------------------------- diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index 1387a0a37561..6762b1b360d5 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -1063,8 +1063,7 @@ public: /// Build and insert \p Res = G_BUILD_VECTOR with \p Src replicated to fill /// the number of elements - MachineInstrBuilder buildSplatVector(const DstOp &Res, - const SrcOp &Src); + MachineInstrBuilder buildSplatBuildVector(const DstOp &Res, const SrcOp &Src); /// Build and insert \p Res = G_BUILD_VECTOR_TRUNC \p Op0, ... /// @@ -1099,6 +1098,15 @@ public: MachineInstrBuilder buildShuffleVector(const DstOp &Res, const SrcOp &Src1, const SrcOp &Src2, ArrayRef Mask); + /// Build and insert \p Res = G_SPLAT_VECTOR \p Val + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p Res must be a generic virtual register with vector type. + /// \pre \p Val must be a generic virtual register with scalar type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildSplatVector(const DstOp &Res, const SrcOp &Val); + /// Build and insert \p Res = G_CONCAT_VECTORS \p Op0, ... /// /// G_CONCAT_VECTORS creates a vector from the concatenation of 2 or more diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def index 6aded2ceebe1..94fba491148b 100644 --- a/llvm/include/llvm/Support/TargetOpcodes.def +++ b/llvm/include/llvm/Support/TargetOpcodes.def @@ -736,6 +736,9 @@ HANDLE_TARGET_OPCODE(G_EXTRACT_VECTOR_ELT) /// Generic shufflevector. HANDLE_TARGET_OPCODE(G_SHUFFLE_VECTOR) +/// Generic splatvector. +HANDLE_TARGET_OPCODE(G_SPLAT_VECTOR) + /// Generic count trailing zeroes. HANDLE_TARGET_OPCODE(G_CTTZ) diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td index d2036e478d18..d967885aa2d7 100644 --- a/llvm/include/llvm/Target/GenericOpcodes.td +++ b/llvm/include/llvm/Target/GenericOpcodes.td @@ -1450,6 +1450,13 @@ def G_SHUFFLE_VECTOR: GenericInstruction { let hasSideEffects = false; } +// Generic splatvector. +def G_SPLAT_VECTOR: GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type1:$val); + let hasSideEffects = false; +} + //------------------------------------------------------------------------------ // Vector reductions //------------------------------------------------------------------------------ diff --git a/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp index 64e2d517e3b9..1869e0d41a51 100644 --- a/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp @@ -309,7 +309,7 @@ MachineInstrBuilder CSEMIRBuilder::buildConstant(const DstOp &Res, // For vectors, CSE the element only for now. LLT Ty = Res.getLLTTy(*getMRI()); if (Ty.isVector()) - return buildSplatVector(Res, buildConstant(Ty.getElementType(), Val)); + return buildSplatBuildVector(Res, buildConstant(Ty.getElementType(), Val)); FoldingSetNodeID ID; GISelInstProfileBuilder ProfBuilder(ID, *getMRI()); @@ -336,7 +336,7 @@ MachineInstrBuilder CSEMIRBuilder::buildFConstant(const DstOp &Res, // For vectors, CSE the element only for now. LLT Ty = Res.getLLTTy(*getMRI()); if (Ty.isVector()) - return buildSplatVector(Res, buildFConstant(Ty.getElementType(), Val)); + return buildSplatBuildVector(Res, buildFConstant(Ty.getElementType(), Val)); FoldingSetNodeID ID; GISelInstProfileBuilder ProfBuilder(ID, *getMRI()); diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 7c986dbbc2c7..365870f540da 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -1598,10 +1598,10 @@ bool IRTranslator::translateGetElementPtr(const User &U, // We might need to splat the base pointer into a vector if the offsets // are vectors. if (WantSplatVector && !PtrTy.isVector()) { - BaseReg = - MIRBuilder - .buildSplatVector(LLT::fixed_vector(VectorWidth, PtrTy), BaseReg) - .getReg(0); + BaseReg = MIRBuilder + .buildSplatBuildVector(LLT::fixed_vector(VectorWidth, PtrTy), + BaseReg) + .getReg(0); PtrIRTy = FixedVectorType::get(PtrIRTy, VectorWidth); PtrTy = getLLTForType(*PtrIRTy, *DL); OffsetIRTy = DL->getIndexType(PtrIRTy); @@ -1639,8 +1639,10 @@ bool IRTranslator::translateGetElementPtr(const User &U, LLT IdxTy = MRI->getType(IdxReg); if (IdxTy != OffsetTy) { if (!IdxTy.isVector() && WantSplatVector) { - IdxReg = MIRBuilder.buildSplatVector( - OffsetTy.changeElementType(IdxTy), IdxReg).getReg(0); + IdxReg = MIRBuilder + .buildSplatBuildVector(OffsetTy.changeElementType(IdxTy), + IdxReg) + .getReg(0); } IdxReg = MIRBuilder.buildSExtOrTrunc(OffsetTy, IdxReg).getReg(0); @@ -2997,6 +2999,19 @@ bool IRTranslator::translateExtractElement(const User &U, bool IRTranslator::translateShuffleVector(const User &U, MachineIRBuilder &MIRBuilder) { + // A ShuffleVector that has operates on scalable vectors is a splat vector + // where the value of the splat vector is the 0th element of the first + // operand, since the index mask operand is the zeroinitializer (undef and + // poison are treated as zeroinitializer here). + if (U.getOperand(0)->getType()->isScalableTy()) { + Value *Op0 = U.getOperand(0); + auto SplatVal = MIRBuilder.buildExtractVectorElementConstant( + LLT::scalar(Op0->getType()->getScalarSizeInBits()), + getOrCreateVReg(*Op0), 0); + MIRBuilder.buildSplatVector(getOrCreateVReg(U), SplatVal); + return true; + } + ArrayRef Mask; if (auto *SVI = dyn_cast(&U)) Mask = SVI->getShuffleMask(); diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index 1d016e684c48..2ec47f72aca3 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -8391,7 +8391,7 @@ static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB) { // For vector types create a G_BUILD_VECTOR. if (Ty.isVector()) - Val = MIB.buildSplatVector(Ty, Val).getReg(0); + Val = MIB.buildSplatBuildVector(Ty, Val).getReg(0); return Val; } diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp index cdd605a5221a..a5a136e2effc 100644 --- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -326,7 +326,7 @@ MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res, auto Const = buildInstr(TargetOpcode::G_CONSTANT) .addDef(getMRI()->createGenericVirtualRegister(EltTy)) .addCImm(&Val); - return buildSplatVector(Res, Const); + return buildSplatBuildVector(Res, Const); } auto Const = buildInstr(TargetOpcode::G_CONSTANT); @@ -363,7 +363,7 @@ MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res, .addDef(getMRI()->createGenericVirtualRegister(EltTy)) .addFPImm(&Val); - return buildSplatVector(Res, Const); + return buildSplatBuildVector(Res, Const); } auto Const = buildInstr(TargetOpcode::G_FCONSTANT); @@ -711,8 +711,8 @@ MachineIRBuilder::buildBuildVectorConstant(const DstOp &Res, return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec); } -MachineInstrBuilder MachineIRBuilder::buildSplatVector(const DstOp &Res, - const SrcOp &Src) { +MachineInstrBuilder MachineIRBuilder::buildSplatBuildVector(const DstOp &Res, + const SrcOp &Src) { SmallVector TmpVec(Res.getLLTTy(*getMRI()).getNumElements(), Src); return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec); } @@ -742,6 +742,14 @@ MachineInstrBuilder MachineIRBuilder::buildShuffleSplat(const DstOp &Res, return buildShuffleVector(DstTy, InsElt, UndefVec, ZeroMask); } +MachineInstrBuilder MachineIRBuilder::buildSplatVector(const DstOp &Res, + const SrcOp &Src) { + LLT DstTy = Res.getLLTTy(*getMRI()); + assert(Src.getLLTTy(*getMRI()) == DstTy.getElementType() && + "Expected Src to match Dst elt ty"); + return buildInstr(TargetOpcode::G_SPLAT_VECTOR, Res, Src); +} + MachineInstrBuilder MachineIRBuilder::buildShuffleVector(const DstOp &Res, const SrcOp &Src1, const SrcOp &Src2, diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp index 1d0757c5d7f5..ecb3bd33bdfd 100644 --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -1640,6 +1640,24 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) { break; } + + case TargetOpcode::G_SPLAT_VECTOR: { + LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); + LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); + + if (!DstTy.isScalableVector()) + report("Destination type must be a scalable vector", MI); + + if (!SrcTy.isScalar()) + report("Source type must be a scalar", MI); + + if (DstTy.getScalarType() != SrcTy) + report("Element type of the destination must be the same type as the " + "source type", + MI); + + break; + } case TargetOpcode::G_DYN_STACKALLOC: { const MachineOperand &DstOp = MI->getOperand(0); const MachineOperand &AllocOp = MI->getOperand(1); diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 750d70c03eab..4713bd605c24 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -20920,7 +20920,8 @@ bool RISCVTargetLowering::fallBackToDAGISel(const Instruction &Inst) const { unsigned Op = Inst.getOpcode(); if (Op == Instruction::Add || Op == Instruction::Sub || Op == Instruction::And || Op == Instruction::Or || - Op == Instruction::Xor || Op == Instruction::InsertElement) + Op == Instruction::Xor || Op == Instruction::InsertElement || + Op == Instruction::Xor || Op == Instruction::ShuffleVector) return false; if (Inst.getType()->isScalableTy()) diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir index d87704cf45d5..7774158e15ec 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir @@ -625,6 +625,9 @@ # DEBUG-NEXT: G_SHUFFLE_VECTOR (opcode {{[0-9]+}}): 2 type indices, 0 imm indices # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected +# DEBUG-NEXT: G_SPLAT_VECTOR (opcode 217): 2 type indices, 0 imm indices +# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined # DEBUG-NEXT: G_CTTZ (opcode {{[0-9]+}}): 2 type indices, 0 imm indices # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/shufflevector.ll b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/shufflevector.ll new file mode 100644 index 000000000000..df7778899b0d --- /dev/null +++ b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/shufflevector.ll @@ -0,0 +1,1774 @@ +; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=riscv32 -mattr=+v -global-isel -stop-after=irtranslator \ +; RUN: -verify-machineinstrs < %s | FileCheck -check-prefixes=RV32 %s +; RUN: llc -mtriple=riscv64 -mattr=+v -global-isel -stop-after=irtranslator \ +; RUN: -verify-machineinstrs < %s | FileCheck -check-prefixes=RV64 %s + +define @shufflevector_nxv1i1_0() { + ; RV32-LABEL: name: shufflevector_nxv1i1_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv1i1_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv1i1_1() { + ; RV32-LABEL: name: shufflevector_nxv1i1_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv1i1_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv1i1_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv1i1_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v0 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv1i1_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v0 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv2i1_0() { + ; RV32-LABEL: name: shufflevector_nxv2i1_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv2i1_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv2i1_1() { + ; RV32-LABEL: name: shufflevector_nxv2i1_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv2i1_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv2i1_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv2i1_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v0 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv2i1_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v0 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv4i1_0() { + ; RV32-LABEL: name: shufflevector_nxv4i1_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv4i1_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv4i1_1() { + ; RV32-LABEL: name: shufflevector_nxv4i1_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv4i1_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv4i1_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv4i1_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v0 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv4i1_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v0 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv8i1_0() { + ; RV32-LABEL: name: shufflevector_nxv8i1_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv8i1_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv8i1_1() { + ; RV32-LABEL: name: shufflevector_nxv8i1_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv8i1_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv8i1_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv8i1_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v0 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv8i1_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v0 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv16i1_0() { + ; RV32-LABEL: name: shufflevector_nxv16i1_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv16i1_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv16i1_1() { + ; RV32-LABEL: name: shufflevector_nxv16i1_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv16i1_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv16i1_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv16i1_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v0 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv16i1_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v0 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv1i8_0() { + ; RV32-LABEL: name: shufflevector_nxv1i8_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i8_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv1i8_1() { + ; RV32-LABEL: name: shufflevector_nxv1i8_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i8_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv1i8_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv1i8_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i8_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv2i8_0() { + ; RV32-LABEL: name: shufflevector_nxv2i8_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv2i8_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv2i8_1() { + ; RV32-LABEL: name: shufflevector_nxv2i8_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv2i8_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv2i8_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv2i8_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv2i8_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv4i8_0() { + ; RV32-LABEL: name: shufflevector_nxv4i8_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv4i8_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv4i8_1() { + ; RV32-LABEL: name: shufflevector_nxv4i8_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv4i8_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv4i8_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv4i8_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv4i8_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv8i8_0() { + ; RV32-LABEL: name: shufflevector_nxv8i8_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv8i8_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv8i8_1() { + ; RV32-LABEL: name: shufflevector_nxv8i8_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv8i8_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv8i8_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv8i8_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv8i8_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv16i8_0() { + ; RV32-LABEL: name: shufflevector_nxv16i8_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv16i8_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv16i8_1() { + ; RV32-LABEL: name: shufflevector_nxv16i8_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv16i8_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv16i8_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv16i8_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8m2 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv16i8_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8m2 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv1i16_0() { + ; RV32-LABEL: name: shufflevector_nxv1i16_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i16_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv1i16_1() { + ; RV32-LABEL: name: shufflevector_nxv1i16_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i16_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv1i16_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv1i16_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i16_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv2i16_0() { + ; RV32-LABEL: name: shufflevector_nxv2i16_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv2i16_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv2i16_1() { + ; RV32-LABEL: name: shufflevector_nxv2i16_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv2i16_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv2i16_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv2i16_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv2i16_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv4i16_0() { + ; RV32-LABEL: name: shufflevector_nxv4i16_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv4i16_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv4i16_1() { + ; RV32-LABEL: name: shufflevector_nxv4i16_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv4i16_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv4i16_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv4i16_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv4i16_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv8i16_0() { + ; RV32-LABEL: name: shufflevector_nxv8i16_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv8i16_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv8i16_1() { + ; RV32-LABEL: name: shufflevector_nxv8i16_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv8i16_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv8i16_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv8i16_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8m2 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv8i16_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8m2 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv16i16_0() { + ; RV32-LABEL: name: shufflevector_nxv16i16_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m4 + ; + ; RV64-LABEL: name: shufflevector_nxv16i16_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m4 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv16i16_1() { + ; RV32-LABEL: name: shufflevector_nxv16i16_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m4 + ; + ; RV64-LABEL: name: shufflevector_nxv16i16_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m4 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv16i16_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv16i16_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8m4 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m4 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m4 + ; + ; RV64-LABEL: name: shufflevector_nxv16i16_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8m4 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m4 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m4 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv1i32_0() { + ; RV32-LABEL: name: shufflevector_nxv1i32_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i32_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv1i32_1() { + ; RV32-LABEL: name: shufflevector_nxv1i32_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i32_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv1i32_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv1i32_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i32_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv2i32_0() { + ; RV32-LABEL: name: shufflevector_nxv2i32_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv2i32_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv2i32_1() { + ; RV32-LABEL: name: shufflevector_nxv2i32_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv2i32_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv2i32_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv2i32_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv2i32_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv4i32_0() { + ; RV32-LABEL: name: shufflevector_nxv4i32_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv4i32_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv4i32_1() { + ; RV32-LABEL: name: shufflevector_nxv4i32_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv4i32_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv4i32_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv4i32_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8m2 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv4i32_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8m2 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv8i32_0() { + ; RV32-LABEL: name: shufflevector_nxv8i32_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m4 + ; + ; RV64-LABEL: name: shufflevector_nxv8i32_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m4 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv8i32_1() { + ; RV32-LABEL: name: shufflevector_nxv8i32_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m4 + ; + ; RV64-LABEL: name: shufflevector_nxv8i32_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m4 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv8i32_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv8i32_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8m4 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m4 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m4 + ; + ; RV64-LABEL: name: shufflevector_nxv8i32_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8m4 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m4 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m4 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv16i32_0() { + ; RV32-LABEL: name: shufflevector_nxv16i32_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m8 + ; + ; RV64-LABEL: name: shufflevector_nxv16i32_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv16i32_1() { + ; RV32-LABEL: name: shufflevector_nxv16i32_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m8 + ; + ; RV64-LABEL: name: shufflevector_nxv16i32_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv16i32_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv16i32_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8m8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m8 + ; + ; RV64-LABEL: name: shufflevector_nxv16i32_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8m8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv1i64_0() { + ; RV32-LABEL: name: shufflevector_nxv1i64_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i64_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv1i64_1() { + ; RV32-LABEL: name: shufflevector_nxv1i64_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i64_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv1i64_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv1i64_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i64_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv2i64_0() { + ; RV32-LABEL: name: shufflevector_nxv2i64_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv2i64_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv2i64_1() { + ; RV32-LABEL: name: shufflevector_nxv2i64_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv2i64_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv2i64_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv2i64_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8m2 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv2i64_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8m2 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv4i64_0() { + ; RV32-LABEL: name: shufflevector_nxv4i64_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m4 + ; + ; RV64-LABEL: name: shufflevector_nxv4i64_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m4 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv4i64_1() { + ; RV32-LABEL: name: shufflevector_nxv4i64_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m4 + ; + ; RV64-LABEL: name: shufflevector_nxv4i64_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m4 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv4i64_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv4i64_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8m4 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m4 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m4 + ; + ; RV64-LABEL: name: shufflevector_nxv4i64_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8m4 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m4 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m4 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv8i64_0() { + ; RV32-LABEL: name: shufflevector_nxv8i64_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m8 + ; + ; RV64-LABEL: name: shufflevector_nxv8i64_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv8i64_1() { + ; RV32-LABEL: name: shufflevector_nxv8i64_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m8 + ; + ; RV64-LABEL: name: shufflevector_nxv8i64_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv8i64_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv8i64_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8m8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m8 + ; + ; RV64-LABEL: name: shufflevector_nxv8i64_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8m8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv16i64_0() { + ; RV32-LABEL: name: shufflevector_nxv16i64_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: [[UV:%[0-9]+]]:_(), [[UV1:%[0-9]+]]:_() = G_UNMERGE_VALUES [[SPLAT_VECTOR]]() + ; RV32-NEXT: $v8m8 = COPY [[UV]]() + ; RV32-NEXT: $v16m8 = COPY [[UV1]]() + ; RV32-NEXT: PseudoRET implicit $v8m8, implicit $v16m8 + ; + ; RV64-LABEL: name: shufflevector_nxv16i64_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: [[UV:%[0-9]+]]:_(), [[UV1:%[0-9]+]]:_() = G_UNMERGE_VALUES [[SPLAT_VECTOR]]() + ; RV64-NEXT: $v8m8 = COPY [[UV]]() + ; RV64-NEXT: $v16m8 = COPY [[UV1]]() + ; RV64-NEXT: PseudoRET implicit $v8m8, implicit $v16m8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv16i64_1() { + ; RV32-LABEL: name: shufflevector_nxv16i64_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: [[UV:%[0-9]+]]:_(), [[UV1:%[0-9]+]]:_() = G_UNMERGE_VALUES [[SPLAT_VECTOR]]() + ; RV32-NEXT: $v8m8 = COPY [[UV]]() + ; RV32-NEXT: $v16m8 = COPY [[UV1]]() + ; RV32-NEXT: PseudoRET implicit $v8m8, implicit $v16m8 + ; + ; RV64-LABEL: name: shufflevector_nxv16i64_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: [[UV:%[0-9]+]]:_(), [[UV1:%[0-9]+]]:_() = G_UNMERGE_VALUES [[SPLAT_VECTOR]]() + ; RV64-NEXT: $v8m8 = COPY [[UV]]() + ; RV64-NEXT: $v16m8 = COPY [[UV1]]() + ; RV64-NEXT: PseudoRET implicit $v8m8, implicit $v16m8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv16i64_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv16i64_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8m8, $v16m8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m8 + ; RV32-NEXT: [[COPY1:%[0-9]+]]:_() = COPY $v16m8 + ; RV32-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_() = G_CONCAT_VECTORS [[COPY]](), [[COPY1]]() + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[CONCAT_VECTORS]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: [[UV:%[0-9]+]]:_(), [[UV1:%[0-9]+]]:_() = G_UNMERGE_VALUES [[SPLAT_VECTOR]]() + ; RV32-NEXT: $v8m8 = COPY [[UV]]() + ; RV32-NEXT: $v16m8 = COPY [[UV1]]() + ; RV32-NEXT: PseudoRET implicit $v8m8, implicit $v16m8 + ; + ; RV64-LABEL: name: shufflevector_nxv16i64_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8m8, $v16m8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m8 + ; RV64-NEXT: [[COPY1:%[0-9]+]]:_() = COPY $v16m8 + ; RV64-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_() = G_CONCAT_VECTORS [[COPY]](), [[COPY1]]() + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[CONCAT_VECTORS]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: [[UV:%[0-9]+]]:_(), [[UV1:%[0-9]+]]:_() = G_UNMERGE_VALUES [[SPLAT_VECTOR]]() + ; RV64-NEXT: $v8m8 = COPY [[UV]]() + ; RV64-NEXT: $v16m8 = COPY [[UV1]]() + ; RV64-NEXT: PseudoRET implicit $v8m8, implicit $v16m8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + + + diff --git a/llvm/test/MachineVerifier/test_g_splat_vector.mir b/llvm/test/MachineVerifier/test_g_splat_vector.mir new file mode 100644 index 000000000000..0d1d8a3e6dcc --- /dev/null +++ b/llvm/test/MachineVerifier/test_g_splat_vector.mir @@ -0,0 +1,27 @@ +# RUN: not --crash llc -o - -mtriple=arm64 -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s +# REQUIRES: aarch64-registered-target +--- +name: g_splat_vector +tracksRegLiveness: true +liveins: +body: | + bb.0: + %0:_(s32) = G_CONSTANT i32 0 + %1:_(<2 x s32>) = G_IMPLICIT_DEF + %2:_() = G_IMPLICIT_DEF + + ; CHECK: Destination type must be a scalable vector + %3:_(s32) = G_SPLAT_VECTOR %0 + + ; CHECK: Destination type must be a scalable vector + %4:_(<2 x s32>) = G_SPLAT_VECTOR %0 + + ; CHECK: Source type must be a scalar + %5:_() = G_SPLAT_VECTOR %1 + + ; CHECK: Source type must be a scalar + %6:_() = G_SPLAT_VECTOR %2 + + ; CHECK: Element type of the destination must be the same type as the source type + %7:_() = G_SPLAT_VECTOR %0 +... diff --git a/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp b/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp index 73837279701a..33155d2c9a96 100644 --- a/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp +++ b/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp @@ -147,9 +147,9 @@ TEST_F(AArch64GISelMITest, LowerRotatesVector) { LLT S32 = LLT::scalar(32); LLT V4S32 = LLT::fixed_vector(4, S32); auto SrcTrunc = B.buildTrunc(S32, Copies[0]); - auto Src = B.buildSplatVector(V4S32, SrcTrunc); + auto Src = B.buildSplatBuildVector(V4S32, SrcTrunc); auto AmtTrunc = B.buildTrunc(S32, Copies[1]); - auto Amt = B.buildSplatVector(V4S32, AmtTrunc); + auto Amt = B.buildSplatBuildVector(V4S32, AmtTrunc); auto ROTR = B.buildInstr(TargetOpcode::G_ROTR, {V4S32}, {Src, Amt}); AInfo Info(MF->getSubtarget()); diff --git a/llvm/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp b/llvm/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp index f52e49df0bcd..59a86fa5646f 100644 --- a/llvm/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp +++ b/llvm/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp @@ -61,7 +61,7 @@ TEST_F(AArch64GISelMITest, MatchIntConstantSplat) { LLT v4s64 = LLT::fixed_vector(4, s64); MachineInstrBuilder FortyTwoSplat = - B.buildSplatVector(v4s64, B.buildConstant(s64, 42)); + B.buildSplatBuildVector(v4s64, B.buildConstant(s64, 42)); int64_t Cst; EXPECT_TRUE(mi_match(FortyTwoSplat.getReg(0), *MRI, m_ICstOrSplat(Cst))); EXPECT_EQ(Cst, 42); @@ -625,7 +625,7 @@ TEST_F(AArch64GISelMITest, MatchSpecificConstantSplat) { LLT v4s64 = LLT::fixed_vector(4, s64); MachineInstrBuilder FortyTwoSplat = - B.buildSplatVector(v4s64, B.buildConstant(s64, 42)); + B.buildSplatBuildVector(v4s64, B.buildConstant(s64, 42)); MachineInstrBuilder FortyTwo = B.buildConstant(s64, 42); EXPECT_TRUE(mi_match(FortyTwoSplat.getReg(0), *MRI, m_SpecificICstSplat(42))); @@ -655,7 +655,7 @@ TEST_F(AArch64GISelMITest, MatchSpecificConstantOrSplat) { LLT v4s64 = LLT::fixed_vector(4, s64); MachineInstrBuilder FortyTwoSplat = - B.buildSplatVector(v4s64, B.buildConstant(s64, 42)); + B.buildSplatBuildVector(v4s64, B.buildConstant(s64, 42)); MachineInstrBuilder FortyTwo = B.buildConstant(s64, 42); EXPECT_TRUE( -- GitLab From 3239b4dcfebbaa3eeaff9258893a6674050d8354 Mon Sep 17 00:00:00 2001 From: Jordan Rupprecht Date: Thu, 7 Mar 2024 08:53:18 -0600 Subject: [PATCH 447/929] [lldb][test] Enforce `pexpect` system availability by default (#84270) This switches the default of `LLDB_TEST_USE_VENDOR_PACKAGES` from `ON` to `OFF` in preparation for eventually deleting it. All known LLDB buildbots have this package installed, so flipping the default will uncover any other users. If this breaks anything, the preferred fix is to install `pexpect` on the host system. The second fix is to build with cmake option `-DLLDB_TEST_USE_VENDOR_PACKAGES=ON` as a temporary measure until `pexpect` can be installed. If neither of those work, reverting this patch is OK. --- lldb/cmake/modules/LLDBConfig.cmake | 2 +- lldb/test/CMakeLists.txt | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lldb/cmake/modules/LLDBConfig.cmake b/lldb/cmake/modules/LLDBConfig.cmake index 93c8ffe4b7d8..5d62213c3f58 100644 --- a/lldb/cmake/modules/LLDBConfig.cmake +++ b/lldb/cmake/modules/LLDBConfig.cmake @@ -68,7 +68,7 @@ option(LLDB_SKIP_DSYM "Whether to skip generating a dSYM when installing lldb." option(LLDB_ENFORCE_STRICT_TEST_REQUIREMENTS "Fail to configure if certain requirements are not met for testing." OFF) option(LLDB_TEST_USE_VENDOR_PACKAGES - "Use packages from lldb/third_party/Python/module instead of system deps." ON) + "Use packages from lldb/third_party/Python/module instead of system deps." OFF) set(LLDB_GLOBAL_INIT_DIRECTORY "" CACHE STRING "Path to the global lldbinit directory. Relative paths are resolved relative to the diff --git a/lldb/test/CMakeLists.txt b/lldb/test/CMakeLists.txt index 950643a5b8cc..0ef2eb1c42ce 100644 --- a/lldb/test/CMakeLists.txt +++ b/lldb/test/CMakeLists.txt @@ -34,7 +34,9 @@ endif() # The "pexpect" package should come from the system environment, not from the # LLDB tree. However, we delay the deletion of it from the tree in case # users/buildbots don't have the package yet and need some time to install it. -if (NOT LLDB_TEST_USE_VENDOR_PACKAGES) +# Windows is configured to skip all pexpect tests, and guards all +# "import pexpect" calls, so we do not need pexpect installed there. +if (NOT LLDB_TEST_USE_VENDOR_PACKAGES AND NOT WIN32) unset(PY_pexpect_FOUND CACHE) lldb_find_python_module(pexpect) if (NOT PY_pexpect_FOUND) -- GitLab From 03588a27261f7ebea15af49268d2ec901fe1979e Mon Sep 17 00:00:00 2001 From: David Spickett Date: Thu, 7 Mar 2024 14:51:23 +0000 Subject: [PATCH 448/929] [lldb][test][FreeBSD] xfail TestPlatformConnect on AArch64 Details in the linked issue. Might fail on other architectures but I can't confirm, they can add to this if it does. --- .../API/commands/platform/connect/TestPlatformConnect.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lldb/test/API/commands/platform/connect/TestPlatformConnect.py b/lldb/test/API/commands/platform/connect/TestPlatformConnect.py index 6a0f036c0070..fc6c2ee98df4 100644 --- a/lldb/test/API/commands/platform/connect/TestPlatformConnect.py +++ b/lldb/test/API/commands/platform/connect/TestPlatformConnect.py @@ -13,6 +13,13 @@ class TestPlatformProcessConnect(TestBase): @expectedFailureAll(hostoslist=["windows"], triple=".*-android") @skipIfDarwin # lldb-server not found correctly @expectedFailureAll(oslist=["windows"]) # process modules not loaded + # lldb-server platform times out waiting for the gdbserver port number to be + # written to the pipe, yet it seems the gdbserver already has written it. + @expectedFailureAll( + archs=["aarch64"], + oslist=["freebsd"], + bugnumber="https://github.com/llvm/llvm-project/issues/84327", + ) @add_test_categories(["lldb-server"]) def test_platform_process_connect(self): self.build() -- GitLab From 9e0f5909d0af3911b19bb1f97fb400c3ce431f63 Mon Sep 17 00:00:00 2001 From: SahilPatidar Date: Thu, 7 Mar 2024 20:45:17 +0530 Subject: [PATCH 449/929] [DAG] Fix Failure to reassociate SMAX/SMIN/UMAX/UMIN (#82175) Resolve #58110 --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 4 + .../RISCV/rvv/fixed-vectors-strided-vpload.ll | 84 ++++++------ .../RISCV/rvv/fixed-vectors-trunc-vp.ll | 121 +++++++++--------- .../CodeGen/RISCV/rvv/fixed-vectors-vpload.ll | 30 ++--- llvm/test/CodeGen/X86/combine-smin.ll | 6 - llvm/test/CodeGen/X86/combine-umax.ll | 8 +- llvm/test/CodeGen/X86/combine-umin.ll | 8 +- 7 files changed, 127 insertions(+), 134 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index cdcb71146404..5476ef879714 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -5562,6 +5562,10 @@ SDValue DAGCombiner::visitIMINMAX(SDNode *N) { if (SDValue FoldedVOp = SimplifyVBinOp(N, DL)) return FoldedVOp; + // reassociate minmax + if (SDValue RMINMAX = reassociateOps(Opcode, DL, N0, N1, N->getFlags())) + return RMINMAX; + // Is sign bits are zero, flip between UMIN/UMAX and SMIN/SMAX. // Only do this if the current op isn't legal and the flipped is. if (!TLI.isOperationLegal(Opcode, VT) && diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-vpload.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-vpload.ll index 2ae031798f5b..2ae058128eaa 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-vpload.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-vpload.ll @@ -549,36 +549,36 @@ define <33 x double> @strided_load_v33f64(ptr %ptr, i64 %stride, <33 x i1> %mask ; CHECK-RV32-NEXT: # %bb.1: ; CHECK-RV32-NEXT: li a3, 32 ; CHECK-RV32-NEXT: .LBB42_2: -; CHECK-RV32-NEXT: mul a5, a3, a2 -; CHECK-RV32-NEXT: addi a6, a4, -32 -; CHECK-RV32-NEXT: sltu a4, a4, a6 -; CHECK-RV32-NEXT: addi a4, a4, -1 -; CHECK-RV32-NEXT: and a6, a4, a6 -; CHECK-RV32-NEXT: li a4, 16 -; CHECK-RV32-NEXT: add a5, a1, a5 -; CHECK-RV32-NEXT: bltu a6, a4, .LBB42_4 +; CHECK-RV32-NEXT: mul a6, a3, a2 +; CHECK-RV32-NEXT: addi a5, a4, -32 +; CHECK-RV32-NEXT: sltu a7, a4, a5 +; CHECK-RV32-NEXT: addi a7, a7, -1 +; CHECK-RV32-NEXT: and a7, a7, a5 +; CHECK-RV32-NEXT: li a5, 16 +; CHECK-RV32-NEXT: add a6, a1, a6 +; CHECK-RV32-NEXT: bltu a7, a5, .LBB42_4 ; CHECK-RV32-NEXT: # %bb.3: -; CHECK-RV32-NEXT: li a6, 16 +; CHECK-RV32-NEXT: li a7, 16 ; CHECK-RV32-NEXT: .LBB42_4: ; CHECK-RV32-NEXT: vsetivli zero, 4, e8, mf2, ta, ma ; CHECK-RV32-NEXT: vslidedown.vi v0, v8, 4 -; CHECK-RV32-NEXT: vsetvli zero, a6, e64, m8, ta, ma -; CHECK-RV32-NEXT: vlse64.v v16, (a5), a2, v0.t -; CHECK-RV32-NEXT: addi a5, a3, -16 -; CHECK-RV32-NEXT: sltu a6, a3, a5 -; CHECK-RV32-NEXT: addi a6, a6, -1 -; CHECK-RV32-NEXT: and a5, a6, a5 -; CHECK-RV32-NEXT: bltu a3, a4, .LBB42_6 +; CHECK-RV32-NEXT: vsetvli zero, a7, e64, m8, ta, ma +; CHECK-RV32-NEXT: vlse64.v v16, (a6), a2, v0.t +; CHECK-RV32-NEXT: addi a6, a3, -16 +; CHECK-RV32-NEXT: sltu a3, a3, a6 +; CHECK-RV32-NEXT: addi a3, a3, -1 +; CHECK-RV32-NEXT: and a3, a3, a6 +; CHECK-RV32-NEXT: bltu a4, a5, .LBB42_6 ; CHECK-RV32-NEXT: # %bb.5: -; CHECK-RV32-NEXT: li a3, 16 +; CHECK-RV32-NEXT: li a4, 16 ; CHECK-RV32-NEXT: .LBB42_6: -; CHECK-RV32-NEXT: mul a4, a3, a2 -; CHECK-RV32-NEXT: add a4, a1, a4 +; CHECK-RV32-NEXT: mul a5, a4, a2 +; CHECK-RV32-NEXT: add a5, a1, a5 ; CHECK-RV32-NEXT: vsetivli zero, 2, e8, mf4, ta, ma ; CHECK-RV32-NEXT: vslidedown.vi v0, v8, 2 -; CHECK-RV32-NEXT: vsetvli zero, a5, e64, m8, ta, ma -; CHECK-RV32-NEXT: vlse64.v v24, (a4), a2, v0.t ; CHECK-RV32-NEXT: vsetvli zero, a3, e64, m8, ta, ma +; CHECK-RV32-NEXT: vlse64.v v24, (a5), a2, v0.t +; CHECK-RV32-NEXT: vsetvli zero, a4, e64, m8, ta, ma ; CHECK-RV32-NEXT: vmv1r.v v0, v8 ; CHECK-RV32-NEXT: vlse64.v v8, (a1), a2, v0.t ; CHECK-RV32-NEXT: vsetivli zero, 16, e64, m8, ta, ma @@ -599,36 +599,36 @@ define <33 x double> @strided_load_v33f64(ptr %ptr, i64 %stride, <33 x i1> %mask ; CHECK-RV64-NEXT: # %bb.1: ; CHECK-RV64-NEXT: li a4, 32 ; CHECK-RV64-NEXT: .LBB42_2: -; CHECK-RV64-NEXT: mul a5, a4, a2 -; CHECK-RV64-NEXT: addi a6, a3, -32 -; CHECK-RV64-NEXT: sltu a3, a3, a6 -; CHECK-RV64-NEXT: addi a3, a3, -1 -; CHECK-RV64-NEXT: and a6, a3, a6 -; CHECK-RV64-NEXT: li a3, 16 -; CHECK-RV64-NEXT: add a5, a1, a5 -; CHECK-RV64-NEXT: bltu a6, a3, .LBB42_4 +; CHECK-RV64-NEXT: mul a6, a4, a2 +; CHECK-RV64-NEXT: addi a5, a3, -32 +; CHECK-RV64-NEXT: sltu a7, a3, a5 +; CHECK-RV64-NEXT: addi a7, a7, -1 +; CHECK-RV64-NEXT: and a7, a7, a5 +; CHECK-RV64-NEXT: li a5, 16 +; CHECK-RV64-NEXT: add a6, a1, a6 +; CHECK-RV64-NEXT: bltu a7, a5, .LBB42_4 ; CHECK-RV64-NEXT: # %bb.3: -; CHECK-RV64-NEXT: li a6, 16 +; CHECK-RV64-NEXT: li a7, 16 ; CHECK-RV64-NEXT: .LBB42_4: ; CHECK-RV64-NEXT: vsetivli zero, 4, e8, mf2, ta, ma ; CHECK-RV64-NEXT: vslidedown.vi v0, v8, 4 -; CHECK-RV64-NEXT: vsetvli zero, a6, e64, m8, ta, ma -; CHECK-RV64-NEXT: vlse64.v v16, (a5), a2, v0.t -; CHECK-RV64-NEXT: addi a5, a4, -16 -; CHECK-RV64-NEXT: sltu a6, a4, a5 -; CHECK-RV64-NEXT: addi a6, a6, -1 -; CHECK-RV64-NEXT: and a5, a6, a5 -; CHECK-RV64-NEXT: bltu a4, a3, .LBB42_6 +; CHECK-RV64-NEXT: vsetvli zero, a7, e64, m8, ta, ma +; CHECK-RV64-NEXT: vlse64.v v16, (a6), a2, v0.t +; CHECK-RV64-NEXT: addi a6, a4, -16 +; CHECK-RV64-NEXT: sltu a4, a4, a6 +; CHECK-RV64-NEXT: addi a4, a4, -1 +; CHECK-RV64-NEXT: and a4, a4, a6 +; CHECK-RV64-NEXT: bltu a3, a5, .LBB42_6 ; CHECK-RV64-NEXT: # %bb.5: -; CHECK-RV64-NEXT: li a4, 16 +; CHECK-RV64-NEXT: li a3, 16 ; CHECK-RV64-NEXT: .LBB42_6: -; CHECK-RV64-NEXT: mul a3, a4, a2 -; CHECK-RV64-NEXT: add a3, a1, a3 +; CHECK-RV64-NEXT: mul a5, a3, a2 +; CHECK-RV64-NEXT: add a5, a1, a5 ; CHECK-RV64-NEXT: vsetivli zero, 2, e8, mf4, ta, ma ; CHECK-RV64-NEXT: vslidedown.vi v0, v8, 2 -; CHECK-RV64-NEXT: vsetvli zero, a5, e64, m8, ta, ma -; CHECK-RV64-NEXT: vlse64.v v24, (a3), a2, v0.t ; CHECK-RV64-NEXT: vsetvli zero, a4, e64, m8, ta, ma +; CHECK-RV64-NEXT: vlse64.v v24, (a5), a2, v0.t +; CHECK-RV64-NEXT: vsetvli zero, a3, e64, m8, ta, ma ; CHECK-RV64-NEXT: vmv1r.v v0, v8 ; CHECK-RV64-NEXT: vlse64.v v8, (a1), a2, v0.t ; CHECK-RV64-NEXT: vsetivli zero, 16, e64, m8, ta, ma diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-vp.ll index e7b747372391..4f16ce28bbb7 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-vp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-vp.ll @@ -310,23 +310,24 @@ define <128 x i32> @vtrunc_v128i32_v128i64(<128 x i64> %a, <128 x i1> %m, i32 ze ; CHECK-NEXT: add a5, sp, a5 ; CHECK-NEXT: addi a5, a5, 16 ; CHECK-NEXT: vs8r.v v16, (a5) # Unknown-size Folded Spill +; CHECK-NEXT: mv a6, a7 ; CHECK-NEXT: bltu a7, a3, .LBB16_4 ; CHECK-NEXT: # %bb.3: -; CHECK-NEXT: li a7, 64 +; CHECK-NEXT: li a6, 64 ; CHECK-NEXT: .LBB16_4: ; CHECK-NEXT: addi a5, a1, 384 ; CHECK-NEXT: li a3, 32 ; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma ; CHECK-NEXT: vle64.v v8, (a1) -; CHECK-NEXT: csrr a6, vlenb -; CHECK-NEXT: slli a6, a6, 3 -; CHECK-NEXT: add a6, sp, a6 -; CHECK-NEXT: addi a6, a6, 16 -; CHECK-NEXT: vs8r.v v8, (a6) # Unknown-size Folded Spill -; CHECK-NEXT: addi a6, a7, -32 -; CHECK-NEXT: sltu t0, a7, a6 -; CHECK-NEXT: addi t0, t0, -1 -; CHECK-NEXT: and a6, t0, a6 +; CHECK-NEXT: csrr t0, vlenb +; CHECK-NEXT: slli t0, t0, 3 +; CHECK-NEXT: add t0, sp, t0 +; CHECK-NEXT: addi t0, t0, 16 +; CHECK-NEXT: vs8r.v v8, (t0) # Unknown-size Folded Spill +; CHECK-NEXT: addi t0, a6, -32 +; CHECK-NEXT: sltu a6, a6, t0 +; CHECK-NEXT: addi a6, a6, -1 +; CHECK-NEXT: and a6, a6, t0 ; CHECK-NEXT: addi t0, a6, -16 ; CHECK-NEXT: sltu t1, a6, t0 ; CHECK-NEXT: addi t1, t1, -1 @@ -364,14 +365,15 @@ define <128 x i32> @vtrunc_v128i32_v128i64(<128 x i64> %a, <128 x i1> %m, i32 ze ; CHECK-NEXT: add a5, sp, a5 ; CHECK-NEXT: addi a5, a5, 16 ; CHECK-NEXT: vs8r.v v16, (a5) # Unknown-size Folded Spill +; CHECK-NEXT: mv a5, a4 ; CHECK-NEXT: bltu a4, a3, .LBB16_8 ; CHECK-NEXT: # %bb.7: -; CHECK-NEXT: li a4, 32 +; CHECK-NEXT: li a5, 32 ; CHECK-NEXT: .LBB16_8: ; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma ; CHECK-NEXT: vle64.v v24, (a1) -; CHECK-NEXT: addi a1, a4, -16 -; CHECK-NEXT: sltu a5, a4, a1 +; CHECK-NEXT: addi a1, a5, -16 +; CHECK-NEXT: sltu a5, a5, a1 ; CHECK-NEXT: addi a5, a5, -1 ; CHECK-NEXT: and a1, a5, a1 ; CHECK-NEXT: vsetvli zero, a1, e32, m4, ta, ma @@ -387,62 +389,63 @@ define <128 x i32> @vtrunc_v128i32_v128i64(<128 x i64> %a, <128 x i1> %m, i32 ze ; CHECK-NEXT: vmv1r.v v0, v5 ; CHECK-NEXT: vnsrl.wi v8, v24, 0, v0.t ; CHECK-NEXT: vmv.v.v v0, v8 +; CHECK-NEXT: mv a1, a7 ; CHECK-NEXT: bltu a7, a3, .LBB16_12 ; CHECK-NEXT: # %bb.11: -; CHECK-NEXT: li a7, 32 +; CHECK-NEXT: li a1, 32 ; CHECK-NEXT: .LBB16_12: ; CHECK-NEXT: vsetvli zero, a3, e32, m8, ta, ma -; CHECK-NEXT: csrr a1, vlenb -; CHECK-NEXT: li a4, 24 -; CHECK-NEXT: mul a1, a1, a4 -; CHECK-NEXT: add a1, sp, a1 -; CHECK-NEXT: addi a1, a1, 16 -; CHECK-NEXT: vl8r.v v8, (a1) # Unknown-size Folded Reload +; CHECK-NEXT: csrr a4, vlenb +; CHECK-NEXT: li a5, 24 +; CHECK-NEXT: mul a4, a4, a5 +; CHECK-NEXT: add a4, sp, a4 +; CHECK-NEXT: addi a4, a4, 16 +; CHECK-NEXT: vl8r.v v8, (a4) # Unknown-size Folded Reload ; CHECK-NEXT: vmv4r.v v24, v8 -; CHECK-NEXT: csrr a1, vlenb -; CHECK-NEXT: li a4, 56 -; CHECK-NEXT: mul a1, a1, a4 -; CHECK-NEXT: add a1, sp, a1 -; CHECK-NEXT: addi a1, a1, 16 -; CHECK-NEXT: vl8r.v v8, (a1) # Unknown-size Folded Reload +; CHECK-NEXT: csrr a4, vlenb +; CHECK-NEXT: li a5, 56 +; CHECK-NEXT: mul a4, a4, a5 +; CHECK-NEXT: add a4, sp, a4 +; CHECK-NEXT: addi a4, a4, 16 +; CHECK-NEXT: vl8r.v v8, (a4) # Unknown-size Folded Reload ; CHECK-NEXT: vslideup.vi v8, v24, 16 -; CHECK-NEXT: csrr a1, vlenb -; CHECK-NEXT: li a4, 56 -; CHECK-NEXT: mul a1, a1, a4 -; CHECK-NEXT: add a1, sp, a1 -; CHECK-NEXT: addi a1, a1, 16 -; CHECK-NEXT: vs8r.v v8, (a1) # Unknown-size Folded Spill -; CHECK-NEXT: csrr a1, vlenb -; CHECK-NEXT: slli a1, a1, 4 -; CHECK-NEXT: add a1, sp, a1 -; CHECK-NEXT: addi a1, a1, 16 -; CHECK-NEXT: vl8r.v v8, (a1) # Unknown-size Folded Reload +; CHECK-NEXT: csrr a4, vlenb +; CHECK-NEXT: li a5, 56 +; CHECK-NEXT: mul a4, a4, a5 +; CHECK-NEXT: add a4, sp, a4 +; CHECK-NEXT: addi a4, a4, 16 +; CHECK-NEXT: vs8r.v v8, (a4) # Unknown-size Folded Spill +; CHECK-NEXT: csrr a4, vlenb +; CHECK-NEXT: slli a4, a4, 4 +; CHECK-NEXT: add a4, sp, a4 +; CHECK-NEXT: addi a4, a4, 16 +; CHECK-NEXT: vl8r.v v8, (a4) # Unknown-size Folded Reload ; CHECK-NEXT: vmv4r.v v24, v8 -; CHECK-NEXT: csrr a1, vlenb -; CHECK-NEXT: li a4, 48 -; CHECK-NEXT: mul a1, a1, a4 -; CHECK-NEXT: add a1, sp, a1 -; CHECK-NEXT: addi a1, a1, 16 -; CHECK-NEXT: vl8r.v v8, (a1) # Unknown-size Folded Reload +; CHECK-NEXT: csrr a4, vlenb +; CHECK-NEXT: li a5, 48 +; CHECK-NEXT: mul a4, a4, a5 +; CHECK-NEXT: add a4, sp, a4 +; CHECK-NEXT: addi a4, a4, 16 +; CHECK-NEXT: vl8r.v v8, (a4) # Unknown-size Folded Reload ; CHECK-NEXT: vslideup.vi v8, v24, 16 -; CHECK-NEXT: csrr a1, vlenb -; CHECK-NEXT: li a4, 48 -; CHECK-NEXT: mul a1, a1, a4 -; CHECK-NEXT: add a1, sp, a1 -; CHECK-NEXT: addi a1, a1, 16 -; CHECK-NEXT: vs8r.v v8, (a1) # Unknown-size Folded Spill +; CHECK-NEXT: csrr a4, vlenb +; CHECK-NEXT: li a5, 48 +; CHECK-NEXT: mul a4, a4, a5 +; CHECK-NEXT: add a4, sp, a4 +; CHECK-NEXT: addi a4, a4, 16 +; CHECK-NEXT: vs8r.v v8, (a4) # Unknown-size Folded Spill ; CHECK-NEXT: vmv4r.v v8, v0 ; CHECK-NEXT: vslideup.vi v8, v16, 16 -; CHECK-NEXT: csrr a1, vlenb -; CHECK-NEXT: li a4, 24 -; CHECK-NEXT: mul a1, a1, a4 -; CHECK-NEXT: add a1, sp, a1 -; CHECK-NEXT: addi a1, a1, 16 -; CHECK-NEXT: vs8r.v v8, (a1) # Unknown-size Folded Spill -; CHECK-NEXT: addi a1, a7, -16 -; CHECK-NEXT: sltu a4, a7, a1 -; CHECK-NEXT: addi a4, a4, -1 -; CHECK-NEXT: and a1, a4, a1 +; CHECK-NEXT: csrr a4, vlenb +; CHECK-NEXT: li a5, 24 +; CHECK-NEXT: mul a4, a4, a5 +; CHECK-NEXT: add a4, sp, a4 +; CHECK-NEXT: addi a4, a4, 16 +; CHECK-NEXT: vs8r.v v8, (a4) # Unknown-size Folded Spill +; CHECK-NEXT: addi a4, a1, -16 +; CHECK-NEXT: sltu a1, a1, a4 +; CHECK-NEXT: addi a1, a1, -1 +; CHECK-NEXT: and a1, a1, a4 ; CHECK-NEXT: vsetvli zero, a1, e32, m4, ta, ma ; CHECK-NEXT: vmv1r.v v0, v6 ; CHECK-NEXT: csrr a1, vlenb diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpload.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpload.ll index bb213c9276a3..618b875be566 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpload.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpload.ll @@ -418,20 +418,20 @@ define <33 x double> @vpload_v33f64(ptr %ptr, <33 x i1> %m, i32 zeroext %evl) { ; CHECK-NEXT: li a3, 32 ; CHECK-NEXT: .LBB32_2: ; CHECK-NEXT: addi a4, a3, -16 -; CHECK-NEXT: sltu a5, a3, a4 -; CHECK-NEXT: addi a5, a5, -1 -; CHECK-NEXT: and a4, a5, a4 -; CHECK-NEXT: addi a5, a1, 128 +; CHECK-NEXT: sltu a3, a3, a4 +; CHECK-NEXT: addi a3, a3, -1 +; CHECK-NEXT: and a3, a3, a4 +; CHECK-NEXT: addi a4, a1, 128 ; CHECK-NEXT: vsetivli zero, 2, e8, mf4, ta, ma ; CHECK-NEXT: vslidedown.vi v0, v8, 2 -; CHECK-NEXT: vsetvli zero, a4, e64, m8, ta, ma -; CHECK-NEXT: vle64.v v16, (a5), v0.t -; CHECK-NEXT: addi a4, a2, -32 -; CHECK-NEXT: sltu a2, a2, a4 -; CHECK-NEXT: addi a2, a2, -1 -; CHECK-NEXT: and a4, a2, a4 -; CHECK-NEXT: li a2, 16 -; CHECK-NEXT: bltu a4, a2, .LBB32_4 +; CHECK-NEXT: vsetvli zero, a3, e64, m8, ta, ma +; CHECK-NEXT: vle64.v v16, (a4), v0.t +; CHECK-NEXT: addi a3, a2, -32 +; CHECK-NEXT: sltu a4, a2, a3 +; CHECK-NEXT: addi a4, a4, -1 +; CHECK-NEXT: and a4, a4, a3 +; CHECK-NEXT: li a3, 16 +; CHECK-NEXT: bltu a4, a3, .LBB32_4 ; CHECK-NEXT: # %bb.3: ; CHECK-NEXT: li a4, 16 ; CHECK-NEXT: .LBB32_4: @@ -440,11 +440,11 @@ define <33 x double> @vpload_v33f64(ptr %ptr, <33 x i1> %m, i32 zeroext %evl) { ; CHECK-NEXT: vslidedown.vi v0, v8, 4 ; CHECK-NEXT: vsetvli zero, a4, e64, m8, ta, ma ; CHECK-NEXT: vle64.v v24, (a5), v0.t -; CHECK-NEXT: bltu a3, a2, .LBB32_6 +; CHECK-NEXT: bltu a2, a3, .LBB32_6 ; CHECK-NEXT: # %bb.5: -; CHECK-NEXT: li a3, 16 +; CHECK-NEXT: li a2, 16 ; CHECK-NEXT: .LBB32_6: -; CHECK-NEXT: vsetvli zero, a3, e64, m8, ta, ma +; CHECK-NEXT: vsetvli zero, a2, e64, m8, ta, ma ; CHECK-NEXT: vmv1r.v v0, v8 ; CHECK-NEXT: vle64.v v8, (a1), v0.t ; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma diff --git a/llvm/test/CodeGen/X86/combine-smin.ll b/llvm/test/CodeGen/X86/combine-smin.ll index 87ae495f945e..b58934256a20 100644 --- a/llvm/test/CodeGen/X86/combine-smin.ll +++ b/llvm/test/CodeGen/X86/combine-smin.ll @@ -70,9 +70,6 @@ define <16 x i8> @test_v16i8_reassociation(<16 x i8> %a) { ; SSE2-LABEL: test_v16i8_reassociation: ; SSE2: # %bb.0: ; SSE2-NEXT: pxor %xmm1, %xmm1 -; SSE2-NEXT: pxor %xmm2, %xmm2 -; SSE2-NEXT: pcmpgtb %xmm0, %xmm2 -; SSE2-NEXT: pand %xmm2, %xmm0 ; SSE2-NEXT: pcmpgtb %xmm0, %xmm1 ; SSE2-NEXT: pand %xmm1, %xmm0 ; SSE2-NEXT: retq @@ -81,21 +78,18 @@ define <16 x i8> @test_v16i8_reassociation(<16 x i8> %a) { ; SSE41: # %bb.0: ; SSE41-NEXT: pxor %xmm1, %xmm1 ; SSE41-NEXT: pminsb %xmm1, %xmm0 -; SSE41-NEXT: pminsb %xmm1, %xmm0 ; SSE41-NEXT: retq ; ; SSE42-LABEL: test_v16i8_reassociation: ; SSE42: # %bb.0: ; SSE42-NEXT: pxor %xmm1, %xmm1 ; SSE42-NEXT: pminsb %xmm1, %xmm0 -; SSE42-NEXT: pminsb %xmm1, %xmm0 ; SSE42-NEXT: retq ; ; AVX-LABEL: test_v16i8_reassociation: ; AVX: # %bb.0: ; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; AVX-NEXT: vpminsb %xmm1, %xmm0, %xmm0 -; AVX-NEXT: vpminsb %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq %1 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> %a, <16 x i8> zeroinitializer) %2 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> %1, <16 x i8> zeroinitializer) diff --git a/llvm/test/CodeGen/X86/combine-umax.ll b/llvm/test/CodeGen/X86/combine-umax.ll index 52bb9ee7fcb9..25f8ec891a24 100644 --- a/llvm/test/CodeGen/X86/combine-umax.ll +++ b/llvm/test/CodeGen/X86/combine-umax.ll @@ -45,16 +45,12 @@ define <8 x i16> @test_v8i16_nosignbit(<8 x i16> %a, <8 x i16> %b) { define <16 x i8> @test_v16i8_reassociation(<16 x i8> %a) { ; SSE-LABEL: test_v16i8_reassociation: ; SSE: # %bb.0: -; SSE-NEXT: movdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] -; SSE-NEXT: pmaxub %xmm1, %xmm0 -; SSE-NEXT: pmaxub %xmm1, %xmm0 +; SSE-NEXT: pmaxub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; SSE-NEXT: retq ; ; AVX-LABEL: test_v16i8_reassociation: ; AVX: # %bb.0: -; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] -; AVX-NEXT: vpmaxub %xmm1, %xmm0, %xmm0 -; AVX-NEXT: vpmaxub %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpmaxub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 ; AVX-NEXT: retq %1 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> %a, <16 x i8> ) %2 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> %1, <16 x i8> ) diff --git a/llvm/test/CodeGen/X86/combine-umin.ll b/llvm/test/CodeGen/X86/combine-umin.ll index 5b3b7f942805..76dbcb50bf8c 100644 --- a/llvm/test/CodeGen/X86/combine-umin.ll +++ b/llvm/test/CodeGen/X86/combine-umin.ll @@ -62,16 +62,12 @@ define <8 x i16> @test_v8i16_nosignbit(<8 x i16> %a, <8 x i16> %b) { define <16 x i8> @test_v16i8_reassociation(<16 x i8> %a) { ; SSE-LABEL: test_v16i8_reassociation: ; SSE: # %bb.0: -; SSE-NEXT: movdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] -; SSE-NEXT: pminub %xmm1, %xmm0 -; SSE-NEXT: pminub %xmm1, %xmm0 +; SSE-NEXT: pminub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; SSE-NEXT: retq ; ; AVX-LABEL: test_v16i8_reassociation: ; AVX: # %bb.0: -; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] -; AVX-NEXT: vpminub %xmm1, %xmm0, %xmm0 -; AVX-NEXT: vpminub %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpminub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 ; AVX-NEXT: retq %1 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> %a, <16 x i8> ) %2 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> %1, <16 x i8> ) -- GitLab From 552da2484390bb002522fc18124ac9fc19ab4b59 Mon Sep 17 00:00:00 2001 From: Michael Maitland Date: Thu, 7 Mar 2024 10:16:31 -0500 Subject: [PATCH 450/929] Revert "[GISEL] Add IRTranslation for shufflevector on scalable vector types" (#84330) Reverts llvm/llvm-project#80378 causing Buildbot failures that did not show up with check-llvm or CI. --- llvm/docs/GlobalISel/GenericOpcode.rst | 5 - .../CodeGen/GlobalISel/MachineIRBuilder.h | 12 +- llvm/include/llvm/Support/TargetOpcodes.def | 3 - llvm/include/llvm/Target/GenericOpcodes.td | 7 - llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp | 4 +- llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 27 +- .../CodeGen/GlobalISel/LegalizerHelper.cpp | 2 +- .../CodeGen/GlobalISel/MachineIRBuilder.cpp | 16 +- llvm/lib/CodeGen/MachineVerifier.cpp | 18 - llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 3 +- .../GlobalISel/legalizer-info-validation.mir | 3 - .../GlobalISel/irtranslator/shufflevector.ll | 1774 ----------------- .../MachineVerifier/test_g_splat_vector.mir | 27 - .../GlobalISel/LegalizerHelperTest.cpp | 4 +- .../CodeGen/GlobalISel/PatternMatchTest.cpp | 6 +- 15 files changed, 21 insertions(+), 1890 deletions(-) delete mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/shufflevector.ll delete mode 100644 llvm/test/MachineVerifier/test_g_splat_vector.mir diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst index dda367607d04..33b0152bd7b4 100644 --- a/llvm/docs/GlobalISel/GenericOpcode.rst +++ b/llvm/docs/GlobalISel/GenericOpcode.rst @@ -639,11 +639,6 @@ Concatenate two vectors and shuffle the elements according to the mask operand. The mask operand should be an IR Constant which exactly matches the corresponding mask for the IR shufflevector instruction. -G_SPLAT_VECTOR -^^^^^^^^^^^^^^^^ - -Create a vector where all elements are the scalar from the source operand. - Vector Reduction Operations --------------------------- diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index 6762b1b360d5..1387a0a37561 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -1063,7 +1063,8 @@ public: /// Build and insert \p Res = G_BUILD_VECTOR with \p Src replicated to fill /// the number of elements - MachineInstrBuilder buildSplatBuildVector(const DstOp &Res, const SrcOp &Src); + MachineInstrBuilder buildSplatVector(const DstOp &Res, + const SrcOp &Src); /// Build and insert \p Res = G_BUILD_VECTOR_TRUNC \p Op0, ... /// @@ -1098,15 +1099,6 @@ public: MachineInstrBuilder buildShuffleVector(const DstOp &Res, const SrcOp &Src1, const SrcOp &Src2, ArrayRef Mask); - /// Build and insert \p Res = G_SPLAT_VECTOR \p Val - /// - /// \pre setBasicBlock or setMI must have been called. - /// \pre \p Res must be a generic virtual register with vector type. - /// \pre \p Val must be a generic virtual register with scalar type. - /// - /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildSplatVector(const DstOp &Res, const SrcOp &Val); - /// Build and insert \p Res = G_CONCAT_VECTORS \p Op0, ... /// /// G_CONCAT_VECTORS creates a vector from the concatenation of 2 or more diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def index 94fba491148b..6aded2ceebe1 100644 --- a/llvm/include/llvm/Support/TargetOpcodes.def +++ b/llvm/include/llvm/Support/TargetOpcodes.def @@ -736,9 +736,6 @@ HANDLE_TARGET_OPCODE(G_EXTRACT_VECTOR_ELT) /// Generic shufflevector. HANDLE_TARGET_OPCODE(G_SHUFFLE_VECTOR) -/// Generic splatvector. -HANDLE_TARGET_OPCODE(G_SPLAT_VECTOR) - /// Generic count trailing zeroes. HANDLE_TARGET_OPCODE(G_CTTZ) diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td index d967885aa2d7..d2036e478d18 100644 --- a/llvm/include/llvm/Target/GenericOpcodes.td +++ b/llvm/include/llvm/Target/GenericOpcodes.td @@ -1450,13 +1450,6 @@ def G_SHUFFLE_VECTOR: GenericInstruction { let hasSideEffects = false; } -// Generic splatvector. -def G_SPLAT_VECTOR: GenericInstruction { - let OutOperandList = (outs type0:$dst); - let InOperandList = (ins type1:$val); - let hasSideEffects = false; -} - //------------------------------------------------------------------------------ // Vector reductions //------------------------------------------------------------------------------ diff --git a/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp index 1869e0d41a51..64e2d517e3b9 100644 --- a/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp @@ -309,7 +309,7 @@ MachineInstrBuilder CSEMIRBuilder::buildConstant(const DstOp &Res, // For vectors, CSE the element only for now. LLT Ty = Res.getLLTTy(*getMRI()); if (Ty.isVector()) - return buildSplatBuildVector(Res, buildConstant(Ty.getElementType(), Val)); + return buildSplatVector(Res, buildConstant(Ty.getElementType(), Val)); FoldingSetNodeID ID; GISelInstProfileBuilder ProfBuilder(ID, *getMRI()); @@ -336,7 +336,7 @@ MachineInstrBuilder CSEMIRBuilder::buildFConstant(const DstOp &Res, // For vectors, CSE the element only for now. LLT Ty = Res.getLLTTy(*getMRI()); if (Ty.isVector()) - return buildSplatBuildVector(Res, buildFConstant(Ty.getElementType(), Val)); + return buildSplatVector(Res, buildFConstant(Ty.getElementType(), Val)); FoldingSetNodeID ID; GISelInstProfileBuilder ProfBuilder(ID, *getMRI()); diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 365870f540da..7c986dbbc2c7 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -1598,10 +1598,10 @@ bool IRTranslator::translateGetElementPtr(const User &U, // We might need to splat the base pointer into a vector if the offsets // are vectors. if (WantSplatVector && !PtrTy.isVector()) { - BaseReg = MIRBuilder - .buildSplatBuildVector(LLT::fixed_vector(VectorWidth, PtrTy), - BaseReg) - .getReg(0); + BaseReg = + MIRBuilder + .buildSplatVector(LLT::fixed_vector(VectorWidth, PtrTy), BaseReg) + .getReg(0); PtrIRTy = FixedVectorType::get(PtrIRTy, VectorWidth); PtrTy = getLLTForType(*PtrIRTy, *DL); OffsetIRTy = DL->getIndexType(PtrIRTy); @@ -1639,10 +1639,8 @@ bool IRTranslator::translateGetElementPtr(const User &U, LLT IdxTy = MRI->getType(IdxReg); if (IdxTy != OffsetTy) { if (!IdxTy.isVector() && WantSplatVector) { - IdxReg = MIRBuilder - .buildSplatBuildVector(OffsetTy.changeElementType(IdxTy), - IdxReg) - .getReg(0); + IdxReg = MIRBuilder.buildSplatVector( + OffsetTy.changeElementType(IdxTy), IdxReg).getReg(0); } IdxReg = MIRBuilder.buildSExtOrTrunc(OffsetTy, IdxReg).getReg(0); @@ -2999,19 +2997,6 @@ bool IRTranslator::translateExtractElement(const User &U, bool IRTranslator::translateShuffleVector(const User &U, MachineIRBuilder &MIRBuilder) { - // A ShuffleVector that has operates on scalable vectors is a splat vector - // where the value of the splat vector is the 0th element of the first - // operand, since the index mask operand is the zeroinitializer (undef and - // poison are treated as zeroinitializer here). - if (U.getOperand(0)->getType()->isScalableTy()) { - Value *Op0 = U.getOperand(0); - auto SplatVal = MIRBuilder.buildExtractVectorElementConstant( - LLT::scalar(Op0->getType()->getScalarSizeInBits()), - getOrCreateVReg(*Op0), 0); - MIRBuilder.buildSplatVector(getOrCreateVReg(U), SplatVal); - return true; - } - ArrayRef Mask; if (auto *SVI = dyn_cast(&U)) Mask = SVI->getShuffleMask(); diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index 2ec47f72aca3..1d016e684c48 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -8391,7 +8391,7 @@ static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB) { // For vector types create a G_BUILD_VECTOR. if (Ty.isVector()) - Val = MIB.buildSplatBuildVector(Ty, Val).getReg(0); + Val = MIB.buildSplatVector(Ty, Val).getReg(0); return Val; } diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp index a5a136e2effc..cdd605a5221a 100644 --- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -326,7 +326,7 @@ MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res, auto Const = buildInstr(TargetOpcode::G_CONSTANT) .addDef(getMRI()->createGenericVirtualRegister(EltTy)) .addCImm(&Val); - return buildSplatBuildVector(Res, Const); + return buildSplatVector(Res, Const); } auto Const = buildInstr(TargetOpcode::G_CONSTANT); @@ -363,7 +363,7 @@ MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res, .addDef(getMRI()->createGenericVirtualRegister(EltTy)) .addFPImm(&Val); - return buildSplatBuildVector(Res, Const); + return buildSplatVector(Res, Const); } auto Const = buildInstr(TargetOpcode::G_FCONSTANT); @@ -711,8 +711,8 @@ MachineIRBuilder::buildBuildVectorConstant(const DstOp &Res, return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec); } -MachineInstrBuilder MachineIRBuilder::buildSplatBuildVector(const DstOp &Res, - const SrcOp &Src) { +MachineInstrBuilder MachineIRBuilder::buildSplatVector(const DstOp &Res, + const SrcOp &Src) { SmallVector TmpVec(Res.getLLTTy(*getMRI()).getNumElements(), Src); return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec); } @@ -742,14 +742,6 @@ MachineInstrBuilder MachineIRBuilder::buildShuffleSplat(const DstOp &Res, return buildShuffleVector(DstTy, InsElt, UndefVec, ZeroMask); } -MachineInstrBuilder MachineIRBuilder::buildSplatVector(const DstOp &Res, - const SrcOp &Src) { - LLT DstTy = Res.getLLTTy(*getMRI()); - assert(Src.getLLTTy(*getMRI()) == DstTy.getElementType() && - "Expected Src to match Dst elt ty"); - return buildInstr(TargetOpcode::G_SPLAT_VECTOR, Res, Src); -} - MachineInstrBuilder MachineIRBuilder::buildShuffleVector(const DstOp &Res, const SrcOp &Src1, const SrcOp &Src2, diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp index ecb3bd33bdfd..1d0757c5d7f5 100644 --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -1640,24 +1640,6 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) { break; } - - case TargetOpcode::G_SPLAT_VECTOR: { - LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); - LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); - - if (!DstTy.isScalableVector()) - report("Destination type must be a scalable vector", MI); - - if (!SrcTy.isScalar()) - report("Source type must be a scalar", MI); - - if (DstTy.getScalarType() != SrcTy) - report("Element type of the destination must be the same type as the " - "source type", - MI); - - break; - } case TargetOpcode::G_DYN_STACKALLOC: { const MachineOperand &DstOp = MI->getOperand(0); const MachineOperand &AllocOp = MI->getOperand(1); diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 4713bd605c24..750d70c03eab 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -20920,8 +20920,7 @@ bool RISCVTargetLowering::fallBackToDAGISel(const Instruction &Inst) const { unsigned Op = Inst.getOpcode(); if (Op == Instruction::Add || Op == Instruction::Sub || Op == Instruction::And || Op == Instruction::Or || - Op == Instruction::Xor || Op == Instruction::InsertElement || - Op == Instruction::Xor || Op == Instruction::ShuffleVector) + Op == Instruction::Xor || Op == Instruction::InsertElement) return false; if (Inst.getType()->isScalableTy()) diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir index 7774158e15ec..d87704cf45d5 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir @@ -625,9 +625,6 @@ # DEBUG-NEXT: G_SHUFFLE_VECTOR (opcode {{[0-9]+}}): 2 type indices, 0 imm indices # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected -# DEBUG-NEXT: G_SPLAT_VECTOR (opcode 217): 2 type indices, 0 imm indices -# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined -# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined # DEBUG-NEXT: G_CTTZ (opcode {{[0-9]+}}): 2 type indices, 0 imm indices # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/shufflevector.ll b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/shufflevector.ll deleted file mode 100644 index df7778899b0d..000000000000 --- a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/shufflevector.ll +++ /dev/null @@ -1,1774 +0,0 @@ -; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4 -; RUN: llc -mtriple=riscv32 -mattr=+v -global-isel -stop-after=irtranslator \ -; RUN: -verify-machineinstrs < %s | FileCheck -check-prefixes=RV32 %s -; RUN: llc -mtriple=riscv64 -mattr=+v -global-isel -stop-after=irtranslator \ -; RUN: -verify-machineinstrs < %s | FileCheck -check-prefixes=RV64 %s - -define @shufflevector_nxv1i1_0() { - ; RV32-LABEL: name: shufflevector_nxv1i1_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v0 - ; - ; RV64-LABEL: name: shufflevector_nxv1i1_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v0 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv1i1_1() { - ; RV32-LABEL: name: shufflevector_nxv1i1_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v0 - ; - ; RV64-LABEL: name: shufflevector_nxv1i1_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v0 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv1i1_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv1i1_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v0 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v0 - ; - ; RV64-LABEL: name: shufflevector_nxv1i1_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v0 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v0 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv2i1_0() { - ; RV32-LABEL: name: shufflevector_nxv2i1_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v0 - ; - ; RV64-LABEL: name: shufflevector_nxv2i1_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v0 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv2i1_1() { - ; RV32-LABEL: name: shufflevector_nxv2i1_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v0 - ; - ; RV64-LABEL: name: shufflevector_nxv2i1_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v0 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv2i1_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv2i1_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v0 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v0 - ; - ; RV64-LABEL: name: shufflevector_nxv2i1_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v0 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v0 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv4i1_0() { - ; RV32-LABEL: name: shufflevector_nxv4i1_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v0 - ; - ; RV64-LABEL: name: shufflevector_nxv4i1_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v0 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv4i1_1() { - ; RV32-LABEL: name: shufflevector_nxv4i1_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v0 - ; - ; RV64-LABEL: name: shufflevector_nxv4i1_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v0 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv4i1_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv4i1_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v0 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v0 - ; - ; RV64-LABEL: name: shufflevector_nxv4i1_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v0 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v0 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv8i1_0() { - ; RV32-LABEL: name: shufflevector_nxv8i1_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v0 - ; - ; RV64-LABEL: name: shufflevector_nxv8i1_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v0 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv8i1_1() { - ; RV32-LABEL: name: shufflevector_nxv8i1_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v0 - ; - ; RV64-LABEL: name: shufflevector_nxv8i1_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v0 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv8i1_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv8i1_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v0 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v0 - ; - ; RV64-LABEL: name: shufflevector_nxv8i1_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v0 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v0 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv16i1_0() { - ; RV32-LABEL: name: shufflevector_nxv16i1_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v0 - ; - ; RV64-LABEL: name: shufflevector_nxv16i1_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v0 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv16i1_1() { - ; RV32-LABEL: name: shufflevector_nxv16i1_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v0 - ; - ; RV64-LABEL: name: shufflevector_nxv16i1_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v0 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv16i1_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv16i1_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v0 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v0 - ; - ; RV64-LABEL: name: shufflevector_nxv16i1_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v0 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) - ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v0 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv1i8_0() { - ; RV32-LABEL: name: shufflevector_nxv1i8_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv1i8_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv1i8_1() { - ; RV32-LABEL: name: shufflevector_nxv1i8_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv1i8_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv1i8_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv1i8_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v8 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv1i8_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v8 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv2i8_0() { - ; RV32-LABEL: name: shufflevector_nxv2i8_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv2i8_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv2i8_1() { - ; RV32-LABEL: name: shufflevector_nxv2i8_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv2i8_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv2i8_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv2i8_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v8 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv2i8_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v8 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv4i8_0() { - ; RV32-LABEL: name: shufflevector_nxv4i8_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv4i8_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv4i8_1() { - ; RV32-LABEL: name: shufflevector_nxv4i8_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv4i8_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv4i8_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv4i8_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v8 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv4i8_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v8 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv8i8_0() { - ; RV32-LABEL: name: shufflevector_nxv8i8_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv8i8_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv8i8_1() { - ; RV32-LABEL: name: shufflevector_nxv8i8_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv8i8_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv8i8_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv8i8_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v8 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv8i8_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v8 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv16i8_0() { - ; RV32-LABEL: name: shufflevector_nxv16i8_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m2 - ; - ; RV64-LABEL: name: shufflevector_nxv16i8_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m2 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv16i8_1() { - ; RV32-LABEL: name: shufflevector_nxv16i8_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m2 - ; - ; RV64-LABEL: name: shufflevector_nxv16i8_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m2 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv16i8_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv16i8_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v8m2 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m2 - ; - ; RV64-LABEL: name: shufflevector_nxv16i8_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v8m2 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) - ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m2 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv1i16_0() { - ; RV32-LABEL: name: shufflevector_nxv1i16_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv1i16_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv1i16_1() { - ; RV32-LABEL: name: shufflevector_nxv1i16_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv1i16_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv1i16_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv1i16_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v8 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv1i16_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v8 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv2i16_0() { - ; RV32-LABEL: name: shufflevector_nxv2i16_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv2i16_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv2i16_1() { - ; RV32-LABEL: name: shufflevector_nxv2i16_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv2i16_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv2i16_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv2i16_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v8 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv2i16_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v8 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv4i16_0() { - ; RV32-LABEL: name: shufflevector_nxv4i16_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv4i16_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv4i16_1() { - ; RV32-LABEL: name: shufflevector_nxv4i16_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv4i16_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv4i16_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv4i16_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v8 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv4i16_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v8 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv8i16_0() { - ; RV32-LABEL: name: shufflevector_nxv8i16_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m2 - ; - ; RV64-LABEL: name: shufflevector_nxv8i16_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m2 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv8i16_1() { - ; RV32-LABEL: name: shufflevector_nxv8i16_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m2 - ; - ; RV64-LABEL: name: shufflevector_nxv8i16_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m2 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv8i16_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv8i16_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v8m2 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m2 - ; - ; RV64-LABEL: name: shufflevector_nxv8i16_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v8m2 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m2 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv16i16_0() { - ; RV32-LABEL: name: shufflevector_nxv16i16_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m4 - ; - ; RV64-LABEL: name: shufflevector_nxv16i16_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m4 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv16i16_1() { - ; RV32-LABEL: name: shufflevector_nxv16i16_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m4 - ; - ; RV64-LABEL: name: shufflevector_nxv16i16_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m4 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv16i16_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv16i16_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v8m4 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m4 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m4 - ; - ; RV64-LABEL: name: shufflevector_nxv16i16_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v8m4 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m4 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) - ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m4 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv1i32_0() { - ; RV32-LABEL: name: shufflevector_nxv1i32_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv1i32_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv1i32_1() { - ; RV32-LABEL: name: shufflevector_nxv1i32_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv1i32_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv1i32_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv1i32_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v8 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv1i32_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v8 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv2i32_0() { - ; RV32-LABEL: name: shufflevector_nxv2i32_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv2i32_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv2i32_1() { - ; RV32-LABEL: name: shufflevector_nxv2i32_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv2i32_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv2i32_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv2i32_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v8 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv2i32_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v8 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv4i32_0() { - ; RV32-LABEL: name: shufflevector_nxv4i32_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m2 - ; - ; RV64-LABEL: name: shufflevector_nxv4i32_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m2 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv4i32_1() { - ; RV32-LABEL: name: shufflevector_nxv4i32_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m2 - ; - ; RV64-LABEL: name: shufflevector_nxv4i32_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m2 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv4i32_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv4i32_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v8m2 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m2 - ; - ; RV64-LABEL: name: shufflevector_nxv4i32_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v8m2 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m2 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv8i32_0() { - ; RV32-LABEL: name: shufflevector_nxv8i32_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m4 - ; - ; RV64-LABEL: name: shufflevector_nxv8i32_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m4 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv8i32_1() { - ; RV32-LABEL: name: shufflevector_nxv8i32_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m4 - ; - ; RV64-LABEL: name: shufflevector_nxv8i32_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m4 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv8i32_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv8i32_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v8m4 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m4 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m4 - ; - ; RV64-LABEL: name: shufflevector_nxv8i32_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v8m4 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m4 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m4 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv16i32_0() { - ; RV32-LABEL: name: shufflevector_nxv16i32_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV32-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m8 - ; - ; RV64-LABEL: name: shufflevector_nxv16i32_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV64-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m8 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv16i32_1() { - ; RV32-LABEL: name: shufflevector_nxv16i32_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV32-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m8 - ; - ; RV64-LABEL: name: shufflevector_nxv16i32_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV64-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m8 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv16i32_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv16i32_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v8m8 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m8 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV32-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m8 - ; - ; RV64-LABEL: name: shufflevector_nxv16i32_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v8m8 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m8 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) - ; RV64-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m8 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv1i64_0() { - ; RV32-LABEL: name: shufflevector_nxv1i64_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv1i64_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv1i64_1() { - ; RV32-LABEL: name: shufflevector_nxv1i64_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv1i64_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv1i64_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv1i64_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v8 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8 - ; - ; RV64-LABEL: name: shufflevector_nxv1i64_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v8 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv2i64_0() { - ; RV32-LABEL: name: shufflevector_nxv2i64_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m2 - ; - ; RV64-LABEL: name: shufflevector_nxv2i64_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m2 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv2i64_1() { - ; RV32-LABEL: name: shufflevector_nxv2i64_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m2 - ; - ; RV64-LABEL: name: shufflevector_nxv2i64_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m2 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv2i64_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv2i64_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v8m2 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m2 - ; - ; RV64-LABEL: name: shufflevector_nxv2i64_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v8m2 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m2 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv4i64_0() { - ; RV32-LABEL: name: shufflevector_nxv4i64_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m4 - ; - ; RV64-LABEL: name: shufflevector_nxv4i64_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m4 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv4i64_1() { - ; RV32-LABEL: name: shufflevector_nxv4i64_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m4 - ; - ; RV64-LABEL: name: shufflevector_nxv4i64_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m4 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv4i64_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv4i64_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v8m4 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m4 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m4 - ; - ; RV64-LABEL: name: shufflevector_nxv4i64_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v8m4 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m4 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m4 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv8i64_0() { - ; RV32-LABEL: name: shufflevector_nxv8i64_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV32-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m8 - ; - ; RV64-LABEL: name: shufflevector_nxv8i64_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV64-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m8 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv8i64_1() { - ; RV32-LABEL: name: shufflevector_nxv8i64_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV32-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m8 - ; - ; RV64-LABEL: name: shufflevector_nxv8i64_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV64-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m8 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv8i64_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv8i64_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v8m8 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m8 - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV32-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() - ; RV32-NEXT: PseudoRET implicit $v8m8 - ; - ; RV64-LABEL: name: shufflevector_nxv8i64_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v8m8 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m8 - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV64-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() - ; RV64-NEXT: PseudoRET implicit $v8m8 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - -define @shufflevector_nxv16i64_0() { - ; RV32-LABEL: name: shufflevector_nxv16i64_0 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV32-NEXT: [[UV:%[0-9]+]]:_(), [[UV1:%[0-9]+]]:_() = G_UNMERGE_VALUES [[SPLAT_VECTOR]]() - ; RV32-NEXT: $v8m8 = COPY [[UV]]() - ; RV32-NEXT: $v16m8 = COPY [[UV1]]() - ; RV32-NEXT: PseudoRET implicit $v8m8, implicit $v16m8 - ; - ; RV64-LABEL: name: shufflevector_nxv16i64_0 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV64-NEXT: [[UV:%[0-9]+]]:_(), [[UV1:%[0-9]+]]:_() = G_UNMERGE_VALUES [[SPLAT_VECTOR]]() - ; RV64-NEXT: $v8m8 = COPY [[UV]]() - ; RV64-NEXT: $v16m8 = COPY [[UV1]]() - ; RV64-NEXT: PseudoRET implicit $v8m8, implicit $v16m8 - %a = shufflevector poison, poison, poison - ret %a -} - -define @shufflevector_nxv16i64_1() { - ; RV32-LABEL: name: shufflevector_nxv16i64_1 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV32-NEXT: [[UV:%[0-9]+]]:_(), [[UV1:%[0-9]+]]:_() = G_UNMERGE_VALUES [[SPLAT_VECTOR]]() - ; RV32-NEXT: $v8m8 = COPY [[UV]]() - ; RV32-NEXT: $v16m8 = COPY [[UV1]]() - ; RV32-NEXT: PseudoRET implicit $v8m8, implicit $v16m8 - ; - ; RV64-LABEL: name: shufflevector_nxv16i64_1 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV64-NEXT: [[UV:%[0-9]+]]:_(), [[UV1:%[0-9]+]]:_() = G_UNMERGE_VALUES [[SPLAT_VECTOR]]() - ; RV64-NEXT: $v8m8 = COPY [[UV]]() - ; RV64-NEXT: $v16m8 = COPY [[UV1]]() - ; RV64-NEXT: PseudoRET implicit $v8m8, implicit $v16m8 - %a = shufflevector undef, undef, undef - ret %a -} - -define @shufflevector_nxv16i64_2( %a) { - ; RV32-LABEL: name: shufflevector_nxv16i64_2 - ; RV32: bb.1 (%ir-block.0): - ; RV32-NEXT: liveins: $v8m8, $v16m8 - ; RV32-NEXT: {{ $}} - ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m8 - ; RV32-NEXT: [[COPY1:%[0-9]+]]:_() = COPY $v16m8 - ; RV32-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_() = G_CONCAT_VECTORS [[COPY]](), [[COPY1]]() - ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[CONCAT_VECTORS]](), [[C]](s64) - ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV32-NEXT: [[UV:%[0-9]+]]:_(), [[UV1:%[0-9]+]]:_() = G_UNMERGE_VALUES [[SPLAT_VECTOR]]() - ; RV32-NEXT: $v8m8 = COPY [[UV]]() - ; RV32-NEXT: $v16m8 = COPY [[UV1]]() - ; RV32-NEXT: PseudoRET implicit $v8m8, implicit $v16m8 - ; - ; RV64-LABEL: name: shufflevector_nxv16i64_2 - ; RV64: bb.1 (%ir-block.0): - ; RV64-NEXT: liveins: $v8m8, $v16m8 - ; RV64-NEXT: {{ $}} - ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m8 - ; RV64-NEXT: [[COPY1:%[0-9]+]]:_() = COPY $v16m8 - ; RV64-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_() = G_CONCAT_VECTORS [[COPY]](), [[COPY1]]() - ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 - ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[CONCAT_VECTORS]](), [[C]](s64) - ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) - ; RV64-NEXT: [[UV:%[0-9]+]]:_(), [[UV1:%[0-9]+]]:_() = G_UNMERGE_VALUES [[SPLAT_VECTOR]]() - ; RV64-NEXT: $v8m8 = COPY [[UV]]() - ; RV64-NEXT: $v16m8 = COPY [[UV1]]() - ; RV64-NEXT: PseudoRET implicit $v8m8, implicit $v16m8 - %b = shufflevector %a , poison, zeroinitializer - ret %b -} - - - diff --git a/llvm/test/MachineVerifier/test_g_splat_vector.mir b/llvm/test/MachineVerifier/test_g_splat_vector.mir deleted file mode 100644 index 0d1d8a3e6dcc..000000000000 --- a/llvm/test/MachineVerifier/test_g_splat_vector.mir +++ /dev/null @@ -1,27 +0,0 @@ -# RUN: not --crash llc -o - -mtriple=arm64 -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s -# REQUIRES: aarch64-registered-target ---- -name: g_splat_vector -tracksRegLiveness: true -liveins: -body: | - bb.0: - %0:_(s32) = G_CONSTANT i32 0 - %1:_(<2 x s32>) = G_IMPLICIT_DEF - %2:_() = G_IMPLICIT_DEF - - ; CHECK: Destination type must be a scalable vector - %3:_(s32) = G_SPLAT_VECTOR %0 - - ; CHECK: Destination type must be a scalable vector - %4:_(<2 x s32>) = G_SPLAT_VECTOR %0 - - ; CHECK: Source type must be a scalar - %5:_() = G_SPLAT_VECTOR %1 - - ; CHECK: Source type must be a scalar - %6:_() = G_SPLAT_VECTOR %2 - - ; CHECK: Element type of the destination must be the same type as the source type - %7:_() = G_SPLAT_VECTOR %0 -... diff --git a/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp b/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp index 33155d2c9a96..73837279701a 100644 --- a/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp +++ b/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp @@ -147,9 +147,9 @@ TEST_F(AArch64GISelMITest, LowerRotatesVector) { LLT S32 = LLT::scalar(32); LLT V4S32 = LLT::fixed_vector(4, S32); auto SrcTrunc = B.buildTrunc(S32, Copies[0]); - auto Src = B.buildSplatBuildVector(V4S32, SrcTrunc); + auto Src = B.buildSplatVector(V4S32, SrcTrunc); auto AmtTrunc = B.buildTrunc(S32, Copies[1]); - auto Amt = B.buildSplatBuildVector(V4S32, AmtTrunc); + auto Amt = B.buildSplatVector(V4S32, AmtTrunc); auto ROTR = B.buildInstr(TargetOpcode::G_ROTR, {V4S32}, {Src, Amt}); AInfo Info(MF->getSubtarget()); diff --git a/llvm/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp b/llvm/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp index 59a86fa5646f..f52e49df0bcd 100644 --- a/llvm/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp +++ b/llvm/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp @@ -61,7 +61,7 @@ TEST_F(AArch64GISelMITest, MatchIntConstantSplat) { LLT v4s64 = LLT::fixed_vector(4, s64); MachineInstrBuilder FortyTwoSplat = - B.buildSplatBuildVector(v4s64, B.buildConstant(s64, 42)); + B.buildSplatVector(v4s64, B.buildConstant(s64, 42)); int64_t Cst; EXPECT_TRUE(mi_match(FortyTwoSplat.getReg(0), *MRI, m_ICstOrSplat(Cst))); EXPECT_EQ(Cst, 42); @@ -625,7 +625,7 @@ TEST_F(AArch64GISelMITest, MatchSpecificConstantSplat) { LLT v4s64 = LLT::fixed_vector(4, s64); MachineInstrBuilder FortyTwoSplat = - B.buildSplatBuildVector(v4s64, B.buildConstant(s64, 42)); + B.buildSplatVector(v4s64, B.buildConstant(s64, 42)); MachineInstrBuilder FortyTwo = B.buildConstant(s64, 42); EXPECT_TRUE(mi_match(FortyTwoSplat.getReg(0), *MRI, m_SpecificICstSplat(42))); @@ -655,7 +655,7 @@ TEST_F(AArch64GISelMITest, MatchSpecificConstantOrSplat) { LLT v4s64 = LLT::fixed_vector(4, s64); MachineInstrBuilder FortyTwoSplat = - B.buildSplatBuildVector(v4s64, B.buildConstant(s64, 42)); + B.buildSplatVector(v4s64, B.buildConstant(s64, 42)); MachineInstrBuilder FortyTwo = B.buildConstant(s64, 42); EXPECT_TRUE( -- GitLab From 4ce52e2d576937fe930294cae883a0daa17eeced Mon Sep 17 00:00:00 2001 From: Alexey Bataev <5361294+alexey-bataev@users.noreply.github.com> Date: Thu, 7 Mar 2024 10:36:41 -0500 Subject: [PATCH 451/929] [SLP]Improve minbitwidth analysis. This improves overall analysis for minbitwidth in SLP. It allows to analyze the trees with store/insertelement root nodes. Also, instead of using single minbitwidth, detected from the very first analysis stage, it tries to detect the best one for each trunc/ext subtree in the graph and use it for the subtree. Results in better code and less vector register pressure. Metric: size..text Program size..text results results0 diff test-suite :: SingleSource/Benchmarks/Adobe-C++/simple_types_loop_invariant.test 92549.00 92609.00 0.1% test-suite :: External/SPEC/CINT2017speed/625.x264_s/625.x264_s.test 663381.00 663493.00 0.0% test-suite :: External/SPEC/CINT2017rate/525.x264_r/525.x264_r.test 663381.00 663493.00 0.0% test-suite :: MultiSource/Benchmarks/Bullet/bullet.test 307182.00 307214.00 0.0% test-suite :: External/SPEC/CFP2017speed/638.imagick_s/638.imagick_s.test 1394420.00 1394484.00 0.0% test-suite :: External/SPEC/CFP2017rate/538.imagick_r/538.imagick_r.test 1394420.00 1394484.00 0.0% test-suite :: External/SPEC/CFP2017rate/510.parest_r/510.parest_r.test 2040257.00 2040273.00 0.0% test-suite :: External/SPEC/CFP2017rate/526.blender_r/526.blender_r.test 12396098.00 12395858.00 -0.0% test-suite :: External/SPEC/CINT2006/445.gobmk/445.gobmk.test 909944.00 909768.00 -0.0% SingleSource/Benchmarks/Adobe-C++/simple_types_loop_invariant - 4 scalar instructions remain scalar (good). Spec2017/x264 - the whole function idct4x4dc is vectorized using <16 x i16> instead of <16 x i32>, also zext/trunc are removed. In other places last vector zext/sext removed and replaced by extractelement + scalar zext/sext pair. MultiSource/Benchmarks/Bullet/bullet - reduce or <4 x i32> replaced by reduce or <4 x i8> Spec2017/imagick - Removed extra zext from 2 packs of the operations. Spec2017/parest - Removed extra zext, replaced by extractelement+scalar zext Spec2017/blender - the whole bunch of vector zext/sext replaced by extractelement+scalar zext/sext, some extra code vectorized in smaller types. Spec2006/gobmk - fixed cost estimation, some small code remains scalar. Reviewers: RKSimon Pull Request: https://github.com/llvm/llvm-project/pull/84334 --- .../Transforms/Vectorize/SLPVectorizer.cpp | 634 ++++++++++++------ .../SLPVectorizer/AArch64/ext-trunc.ll | 9 +- .../SLPVectorizer/AArch64/getelementptr2.ll | 4 +- .../SLPVectorizer/AArch64/reduce-add-i64.ll | 20 +- .../SLPVectorizer/RISCV/reductions.ll | 7 +- .../Transforms/SLPVectorizer/X86/PR35777.ll | 9 +- .../X86/int-bitcast-minbitwidth.ll | 2 +- ...minbitwidth-multiuse-with-insertelement.ll | 17 +- .../X86/minbitwidth-transformed-operand.ll | 21 +- .../SLPVectorizer/X86/minimum-sizes.ll | 43 +- .../SLPVectorizer/X86/phi-undef-input.ll | 24 +- .../Transforms/SLPVectorizer/X86/resched.ll | 32 +- .../X86/reused-reductions-with-minbitwidth.ll | 10 +- .../X86/store-insertelement-minbitwidth.ll | 22 +- .../SLPVectorizer/alt-cmp-vectorize.ll | 4 +- 15 files changed, 553 insertions(+), 305 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 36dc9094538a..1889bc09e850 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -1085,6 +1085,9 @@ public: BS->clear(); } MinBWs.clear(); + ReductionBitWidth = 0; + CastMaxMinBWSizes.reset(); + TruncNodes.clear(); InstrElementSize.clear(); UserIgnoreList = nullptr; PostponedGathers.clear(); @@ -2287,6 +2290,7 @@ public: void clearReductionData() { AnalyzedReductionsRoots.clear(); AnalyzedReductionVals.clear(); + AnalyzedMinBWVals.clear(); } /// Checks if the given value is gathered in one of the nodes. bool isAnyGathered(const SmallDenseSet &Vals) const { @@ -2307,9 +2311,11 @@ private: /// constant and to be demoted. Required to correctly identify constant nodes /// to be demoted. bool collectValuesToDemote( - Value *V, SmallVectorImpl &ToDemote, + Value *V, bool IsProfitableToDemoteRoot, unsigned &BitWidth, + SmallVectorImpl &ToDemote, DenseMap> &DemotedConsts, - SmallVectorImpl &Roots, DenseSet &Visited) const; + DenseSet &Visited, unsigned &MaxDepthLevel, + bool &IsProfitableToDemote) const; /// Check if the operands on the edges \p Edges of the \p UserTE allows /// reordering (i.e. the operands can be reordered because they have only one @@ -2375,6 +2381,10 @@ private: /// \ returns the graph entry for the \p Idx operand of the \p E entry. const TreeEntry *getOperandEntry(const TreeEntry *E, unsigned Idx) const; + /// \returns Cast context for the given graph node. + TargetTransformInfo::CastContextHint + getCastContextHint(const TreeEntry &TE) const; + /// \returns the cost of the vectorizable entry. InstructionCost getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, @@ -2925,11 +2935,18 @@ private: } assert(!BundleMember && "Bundle and VL out of sync"); } else { - MustGather.insert(VL.begin(), VL.end()); // Build a map for gathered scalars to the nodes where they are used. + bool AllConstsOrCasts = true; for (Value *V : VL) - if (!isConstant(V)) + if (!isConstant(V)) { + auto *I = dyn_cast(V); + AllConstsOrCasts &= I && I->getType()->isIntegerTy(); ValueToGatherNodes.try_emplace(V).first->getSecond().insert(Last); + } + if (AllConstsOrCasts) + CastMaxMinBWSizes = + std::make_pair(std::numeric_limits::max(), 1); + MustGather.insert(VL.begin(), VL.end()); } if (UserTreeIdx.UserTE) @@ -3054,6 +3071,10 @@ private: /// Set of hashes for the list of reduction values already being analyzed. DenseSet AnalyzedReductionVals; + /// Values, already been analyzed for mininmal bitwidth and found to be + /// non-profitable. + DenseSet AnalyzedMinBWVals; + /// A list of values that need to extracted out of the tree. /// This list holds pairs of (Internal Scalar : External User). External User /// can be nullptr, it means that this Internal Scalar will be used later, @@ -3629,6 +3650,18 @@ private: /// value must be signed-extended, rather than zero-extended, back to its /// original width. DenseMap> MinBWs; + + /// Final size of the reduced vector, if the current graph represents the + /// input for the reduction and it was possible to narrow the size of the + /// reduction. + unsigned ReductionBitWidth = 0; + + /// If the tree contains any zext/sext/trunc nodes, contains max-min pair of + /// type sizes, used in the tree. + std::optional> CastMaxMinBWSizes; + + /// Indices of the vectorized trunc nodes. + DenseSet TruncNodes; }; } // end namespace slpvectorizer @@ -6539,8 +6572,29 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, case Instruction::Trunc: case Instruction::FPTrunc: case Instruction::BitCast: { + auto [PrevMaxBW, PrevMinBW] = CastMaxMinBWSizes.value_or( + std::make_pair(std::numeric_limits::min(), + std::numeric_limits::max())); + if (ShuffleOrOp == Instruction::ZExt || + ShuffleOrOp == Instruction::SExt) { + CastMaxMinBWSizes = std::make_pair( + std::max(DL->getTypeSizeInBits(VL0->getType()), + PrevMaxBW), + std::min( + DL->getTypeSizeInBits(VL0->getOperand(0)->getType()), + PrevMinBW)); + } else if (ShuffleOrOp == Instruction::Trunc) { + CastMaxMinBWSizes = std::make_pair( + std::max( + DL->getTypeSizeInBits(VL0->getOperand(0)->getType()), + PrevMaxBW), + std::min(DL->getTypeSizeInBits(VL0->getType()), + PrevMinBW)); + TruncNodes.insert(VectorizableTree.size()); + } TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, ReuseShuffleIndicies); + LLVM_DEBUG(dbgs() << "SLP: added a vector of casts.\n"); TE->setOperandsInOrder(); @@ -8362,6 +8416,22 @@ const BoUpSLP::TreeEntry *BoUpSLP::getOperandEntry(const TreeEntry *E, return It->get(); } +TTI::CastContextHint BoUpSLP::getCastContextHint(const TreeEntry &TE) const { + if (TE.State == TreeEntry::ScatterVectorize || + TE.State == TreeEntry::StridedVectorize) + return TTI::CastContextHint::GatherScatter; + if (TE.State == TreeEntry::Vectorize && TE.getOpcode() == Instruction::Load && + !TE.isAltShuffle()) { + if (TE.ReorderIndices.empty()) + return TTI::CastContextHint::Normal; + SmallVector Mask; + inversePermutation(TE.ReorderIndices, Mask); + if (ShuffleVectorInst::isReverseMask(Mask, Mask.size())) + return TTI::CastContextHint::Reversed; + } + return TTI::CastContextHint::None; +} + InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, SmallPtrSetImpl &CheckedExtracts) { @@ -8384,6 +8454,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, // If we have computed a smaller type for the expression, update VecTy so // that the costs will be accurate. auto It = MinBWs.find(E); + Type *OrigScalarTy = ScalarTy; if (It != MinBWs.end()) { ScalarTy = IntegerType::get(F->getContext(), It->second.first); VecTy = FixedVectorType::get(ScalarTy, VL.size()); @@ -8441,24 +8512,11 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, UsedScalars.set(I); } auto GetCastContextHint = [&](Value *V) { - if (const TreeEntry *OpTE = getTreeEntry(V)) { - if (OpTE->State == TreeEntry::ScatterVectorize || - OpTE->State == TreeEntry::StridedVectorize) - return TTI::CastContextHint::GatherScatter; - if (OpTE->State == TreeEntry::Vectorize && - OpTE->getOpcode() == Instruction::Load && !OpTE->isAltShuffle()) { - if (OpTE->ReorderIndices.empty()) - return TTI::CastContextHint::Normal; - SmallVector Mask; - inversePermutation(OpTE->ReorderIndices, Mask); - if (ShuffleVectorInst::isReverseMask(Mask, Mask.size())) - return TTI::CastContextHint::Reversed; - } - } else { - InstructionsState SrcState = getSameOpcode(E->getOperand(0), *TLI); - if (SrcState.getOpcode() == Instruction::Load && !SrcState.isAltShuffle()) - return TTI::CastContextHint::GatherScatter; - } + if (const TreeEntry *OpTE = getTreeEntry(V)) + return getCastContextHint(*OpTE); + InstructionsState SrcState = getSameOpcode(E->getOperand(0), *TLI); + if (SrcState.getOpcode() == Instruction::Load && !SrcState.isAltShuffle()) + return TTI::CastContextHint::GatherScatter; return TTI::CastContextHint::None; }; auto GetCostDiff = @@ -8507,8 +8565,6 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, TTI::CastContextHint CCH = GetCastContextHint(VL0); VecCost += TTI->getCastInstrCost(VecOpcode, UserVecTy, VecTy, CCH, CostKind); - ScalarCost += Sz * TTI->getCastInstrCost(VecOpcode, UserScalarTy, - ScalarTy, CCH, CostKind); } } } @@ -8525,7 +8581,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, InstructionCost ScalarCost = 0; InstructionCost VecCost = 0; std::tie(ScalarCost, VecCost) = getGEPCosts( - *TTI, Ptrs, BasePtr, E->getOpcode(), CostKind, ScalarTy, VecTy); + *TTI, Ptrs, BasePtr, E->getOpcode(), CostKind, OrigScalarTy, VecTy); LLVM_DEBUG(dumpTreeCosts(E, 0, VecCost, ScalarCost, "Calculated GEPs cost for Tree")); @@ -8572,7 +8628,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, NumElts = ATy->getNumElements(); else NumElts = AggregateTy->getStructNumElements(); - SrcVecTy = FixedVectorType::get(ScalarTy, NumElts); + SrcVecTy = FixedVectorType::get(OrigScalarTy, NumElts); } if (I->hasOneUse()) { Instruction *Ext = I->user_back(); @@ -8740,13 +8796,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, } } auto GetScalarCost = [&](unsigned Idx) -> InstructionCost { - // Do not count cost here if minimum bitwidth is in effect and it is just - // a bitcast (here it is just a noop). - if (VecOpcode != Opcode && VecOpcode == Instruction::BitCast) - return TTI::TCC_Free; - auto *VI = VL0->getOpcode() == Opcode - ? cast(UniqueValues[Idx]) - : nullptr; + auto *VI = cast(UniqueValues[Idx]); return TTI->getCastInstrCost(Opcode, VL0->getType(), VL0->getOperand(0)->getType(), TTI::getCastContextHint(VI), CostKind, VI); @@ -8789,7 +8839,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, ? CmpInst::BAD_FCMP_PREDICATE : CmpInst::BAD_ICMP_PREDICATE; - return TTI->getCmpSelInstrCost(E->getOpcode(), ScalarTy, + return TTI->getCmpSelInstrCost(E->getOpcode(), OrigScalarTy, Builder.getInt1Ty(), CurrentPred, CostKind, VI); }; @@ -8844,7 +8894,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(VI->getOperand(OpIdx)); SmallVector Operands(VI->operand_values()); - return TTI->getArithmeticInstrCost(ShuffleOrOp, ScalarTy, CostKind, + return TTI->getArithmeticInstrCost(ShuffleOrOp, OrigScalarTy, CostKind, Op1Info, Op2Info, Operands, VI); }; auto GetVectorCost = [=](InstructionCost CommonCost) { @@ -8863,9 +8913,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, case Instruction::Load: { auto GetScalarCost = [&](unsigned Idx) { auto *VI = cast(UniqueValues[Idx]); - return TTI->getMemoryOpCost(Instruction::Load, ScalarTy, VI->getAlign(), - VI->getPointerAddressSpace(), CostKind, - TTI::OperandValueInfo(), VI); + return TTI->getMemoryOpCost(Instruction::Load, OrigScalarTy, + VI->getAlign(), VI->getPointerAddressSpace(), + CostKind, TTI::OperandValueInfo(), VI); }; auto *LI0 = cast(VL0); auto GetVectorCost = [&](InstructionCost CommonCost) { @@ -8908,9 +8958,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, auto GetScalarCost = [=](unsigned Idx) { auto *VI = cast(VL[Idx]); TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(VI->getValueOperand()); - return TTI->getMemoryOpCost(Instruction::Store, ScalarTy, VI->getAlign(), - VI->getPointerAddressSpace(), CostKind, - OpInfo, VI); + return TTI->getMemoryOpCost(Instruction::Store, OrigScalarTy, + VI->getAlign(), VI->getPointerAddressSpace(), + CostKind, OpInfo, VI); }; auto *BaseSI = cast(IsReorder ? VL[E->ReorderIndices.front()] : VL0); @@ -9772,6 +9822,44 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef VectorizedVals) { Cost -= InsertCost; } + // Add the cost for reduced value resize (if required). + if (ReductionBitWidth != 0) { + assert(UserIgnoreList && "Expected reduction tree."); + const TreeEntry &E = *VectorizableTree.front().get(); + auto It = MinBWs.find(&E); + if (It != MinBWs.end() && It->second.first != ReductionBitWidth) { + unsigned SrcSize = It->second.first; + unsigned DstSize = ReductionBitWidth; + unsigned Opcode = Instruction::Trunc; + if (SrcSize < DstSize) + Opcode = It->second.second ? Instruction::SExt : Instruction::ZExt; + auto *SrcVecTy = + FixedVectorType::get(Builder.getIntNTy(SrcSize), E.getVectorFactor()); + auto *DstVecTy = + FixedVectorType::get(Builder.getIntNTy(DstSize), E.getVectorFactor()); + TTI::CastContextHint CCH = getCastContextHint(E); + InstructionCost CastCost; + switch (E.getOpcode()) { + case Instruction::SExt: + case Instruction::ZExt: + case Instruction::Trunc: { + const TreeEntry *OpTE = getOperandEntry(&E, 0); + CCH = getCastContextHint(*OpTE); + break; + } + default: + break; + } + CastCost += TTI->getCastInstrCost(Opcode, DstVecTy, SrcVecTy, CCH, + TTI::TCK_RecipThroughput); + Cost += CastCost; + LLVM_DEBUG(dbgs() << "SLP: Adding cost " << CastCost + << " for final resize for reduction from " << SrcVecTy + << " to " << DstVecTy << "\n"; + dbgs() << "SLP: Current total cost = " << Cost << "\n"); + } + } + #ifndef NDEBUG SmallString<256> Str; { @@ -9992,6 +10080,30 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry( // tree node for each gathered value - we have just a permutation of the // single vector. If we have 2 different sets, we're in situation where we // have a permutation of 2 input vectors. + // Filter out entries with larger bitwidth of elements. + Type *ScalarTy = VL.front()->getType(); + unsigned BitWidth = 0; + if (ScalarTy->isIntegerTy()) { + // Check if the used TEs supposed to be resized and choose the best + // candidates. + BitWidth = DL->getTypeStoreSize(ScalarTy); + if (TEUseEI.UserTE->getOpcode() != Instruction::Select || + TEUseEI.EdgeIdx != 0) { + auto UserIt = MinBWs.find(TEUseEI.UserTE); + if (UserIt != MinBWs.end()) + BitWidth = UserIt->second.second; + } + } + auto CheckBitwidth = [&](const TreeEntry &TE) { + Type *ScalarTy = TE.Scalars.front()->getType(); + if (!ScalarTy->isIntegerTy()) + return true; + unsigned TEBitWidth = DL->getTypeStoreSize(ScalarTy); + auto UserIt = MinBWs.find(TEUseEI.UserTE); + if (UserIt != MinBWs.end()) + TEBitWidth = UserIt->second.second; + return BitWidth == TEBitWidth; + }; SmallVector> UsedTEs; DenseMap UsedValuesEntry; for (Value *V : VL) { @@ -10026,6 +10138,8 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry( continue; } + if (!CheckBitwidth(*TEPtr)) + continue; // Check if the user node of the TE comes after user node of TEPtr, // otherwise TEPtr depends on TE. if ((TEInsertBlock != InsertPt->getParent() || @@ -10042,8 +10156,8 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry( continue; VTE = *It->getSecond().begin(); // Iterate through all vectorized nodes. - auto *MIt = find_if(It->getSecond(), [](const TreeEntry *MTE) { - return MTE->State == TreeEntry::Vectorize; + auto *MIt = find_if(It->getSecond(), [&](const TreeEntry *MTE) { + return MTE->State == TreeEntry::Vectorize && CheckBitwidth(*MTE); }); if (MIt == It->getSecond().end()) continue; @@ -10053,10 +10167,7 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry( Instruction &LastBundleInst = getLastInstructionInBundle(VTE); if (&LastBundleInst == TEInsertPt || !CheckOrdering(&LastBundleInst)) continue; - auto It = MinBWs.find(VTE); - // If vectorize node is demoted - do not match. - if (It != MinBWs.end() && - It->second.first != DL->getTypeSizeInBits(V->getType())) + if (!CheckBitwidth(*VTE)) continue; VToTEs.insert(VTE); } @@ -12929,7 +13040,21 @@ Value *BoUpSLP::vectorizeTree( Builder.ClearInsertionPoint(); InstrElementSize.clear(); - return VectorizableTree[0]->VectorizedValue; + const TreeEntry &RootTE = *VectorizableTree.front().get(); + Value *Vec = RootTE.VectorizedValue; + if (auto It = MinBWs.find(&RootTE); ReductionBitWidth != 0 && + It != MinBWs.end() && + ReductionBitWidth != It->second.first) { + IRBuilder<>::InsertPointGuard Guard(Builder); + Builder.SetInsertPoint(ReductionRoot->getParent(), + ReductionRoot->getIterator()); + Vec = Builder.CreateIntCast( + Vec, + VectorType::get(Builder.getIntNTy(ReductionBitWidth), + cast(Vec->getType())->getElementCount()), + It->second.second); + } + return Vec; } void BoUpSLP::optimizeGatherSequence() { @@ -13749,23 +13874,42 @@ unsigned BoUpSLP::getVectorElementSize(Value *V) { // smaller type with a truncation. We collect the values that will be demoted // in ToDemote and additional roots that require investigating in Roots. bool BoUpSLP::collectValuesToDemote( - Value *V, SmallVectorImpl &ToDemote, + Value *V, bool IsProfitableToDemoteRoot, unsigned &BitWidth, + SmallVectorImpl &ToDemote, DenseMap> &DemotedConsts, - SmallVectorImpl &Roots, DenseSet &Visited) const { + DenseSet &Visited, unsigned &MaxDepthLevel, + bool &IsProfitableToDemote) const { // We can always demote constants. - if (isa(V)) + if (isa(V)) { + MaxDepthLevel = 1; return true; + } // If the value is not a vectorized instruction in the expression and not used // by the insertelement instruction and not used in multiple vector nodes, it // cannot be demoted. + // TODO: improve handling of gathered values and others. auto *I = dyn_cast(V); - if (!I || !getTreeEntry(I) || MultiNodeScalars.contains(I) || - !Visited.insert(I).second || all_of(I->users(), [&](User *U) { + if (!I || !Visited.insert(I).second || !getTreeEntry(I) || + MultiNodeScalars.contains(I) || all_of(I->users(), [&](User *U) { return isa(U) && !getTreeEntry(U); })) return false; + auto IsPotentiallyTruncated = [&](Value *V, unsigned &BitWidth) -> bool { + if (MultiNodeScalars.contains(V)) + return false; + uint32_t OrigBitWidth = DL->getTypeSizeInBits(V->getType()); + APInt Mask = APInt::getBitsSetFrom(OrigBitWidth, BitWidth); + if (MaskedValueIsZero(V, Mask, SimplifyQuery(*DL))) + return true; + auto NumSignBits = ComputeNumSignBits(V, *DL, 0, AC, nullptr, DT); + unsigned BitWidth1 = OrigBitWidth - NumSignBits; + if (!isKnownNonNegative(V, SimplifyQuery(*DL))) + ++BitWidth1; + BitWidth = std::max(BitWidth, BitWidth1); + return BitWidth > 0 && OrigBitWidth >= (BitWidth * 2); + }; unsigned Start = 0; unsigned End = I->getNumOperands(); switch (I->getOpcode()) { @@ -13773,12 +13917,14 @@ bool BoUpSLP::collectValuesToDemote( // We can always demote truncations and extensions. Since truncations can // seed additional demotion, we save the truncated value. case Instruction::Trunc: - Roots.push_back(I->getOperand(0)); + MaxDepthLevel = 1; + if (IsProfitableToDemoteRoot) + IsProfitableToDemote = true; break; case Instruction::ZExt: case Instruction::SExt: - if (isa(I->getOperand(0))) - return false; + MaxDepthLevel = 1; + IsProfitableToDemote = true; break; // We can demote certain binary operations if we can demote both of their @@ -13788,23 +13934,32 @@ bool BoUpSLP::collectValuesToDemote( case Instruction::Mul: case Instruction::And: case Instruction::Or: - case Instruction::Xor: - if (!collectValuesToDemote(I->getOperand(0), ToDemote, DemotedConsts, Roots, - Visited) || - !collectValuesToDemote(I->getOperand(1), ToDemote, DemotedConsts, Roots, - Visited)) + case Instruction::Xor: { + unsigned Level1, Level2; + if (!collectValuesToDemote(I->getOperand(0), IsProfitableToDemoteRoot, + BitWidth, ToDemote, DemotedConsts, Visited, + Level1, IsProfitableToDemote) || + !collectValuesToDemote(I->getOperand(1), IsProfitableToDemoteRoot, + BitWidth, ToDemote, DemotedConsts, Visited, + Level2, IsProfitableToDemote)) return false; + MaxDepthLevel = std::max(Level1, Level2); break; + } // We can demote selects if we can demote their true and false values. case Instruction::Select: { Start = 1; + unsigned Level1, Level2; SelectInst *SI = cast(I); - if (!collectValuesToDemote(SI->getTrueValue(), ToDemote, DemotedConsts, - Roots, Visited) || - !collectValuesToDemote(SI->getFalseValue(), ToDemote, DemotedConsts, - Roots, Visited)) + if (!collectValuesToDemote(SI->getTrueValue(), IsProfitableToDemoteRoot, + BitWidth, ToDemote, DemotedConsts, Visited, + Level1, IsProfitableToDemote) || + !collectValuesToDemote(SI->getFalseValue(), IsProfitableToDemoteRoot, + BitWidth, ToDemote, DemotedConsts, Visited, + Level2, IsProfitableToDemote)) return false; + MaxDepthLevel = std::max(Level1, Level2); break; } @@ -13813,171 +13968,262 @@ bool BoUpSLP::collectValuesToDemote( case Instruction::PHI: { PHINode *PN = cast(I); for (Value *IncValue : PN->incoming_values()) - if (!collectValuesToDemote(IncValue, ToDemote, DemotedConsts, Roots, - Visited)) + if (!collectValuesToDemote(IncValue, IsProfitableToDemoteRoot, BitWidth, + ToDemote, DemotedConsts, Visited, + MaxDepthLevel, IsProfitableToDemote)) return false; break; } // Otherwise, conservatively give up. default: - return false; + return IsProfitableToDemote && IsPotentiallyTruncated(I, BitWidth); } + ++MaxDepthLevel; // Gather demoted constant operands. for (unsigned Idx : seq(Start, End)) if (isa(I->getOperand(Idx))) DemotedConsts.try_emplace(I).first->getSecond().push_back(Idx); // Record the value that we can demote. ToDemote.push_back(V); - return true; + return IsProfitableToDemote; } void BoUpSLP::computeMinimumValueSizes() { // We only attempt to truncate integer expressions. - auto &TreeRoot = VectorizableTree[0]->Scalars; - auto *TreeRootIT = dyn_cast(TreeRoot[0]->getType()); - if (!TreeRootIT || VectorizableTree.front()->State == TreeEntry::NeedToGather) + bool IsStoreOrInsertElt = + VectorizableTree.front()->getOpcode() == Instruction::Store || + VectorizableTree.front()->getOpcode() == Instruction::InsertElement; + if ((IsStoreOrInsertElt || UserIgnoreList) && TruncNodes.size() <= 1 && + (!CastMaxMinBWSizes || CastMaxMinBWSizes->second == 0 || + CastMaxMinBWSizes->first / CastMaxMinBWSizes->second <= 2)) return; + unsigned NodeIdx = 0; + if (IsStoreOrInsertElt && + VectorizableTree.front()->State != TreeEntry::NeedToGather) + NodeIdx = 1; + // Ensure the roots of the vectorizable tree don't form a cycle. - if (!VectorizableTree.front()->UserTreeIndices.empty()) + if (VectorizableTree[NodeIdx]->State == TreeEntry::NeedToGather || + (NodeIdx == 0 && !VectorizableTree[NodeIdx]->UserTreeIndices.empty()) || + (NodeIdx != 0 && any_of(VectorizableTree[NodeIdx]->UserTreeIndices, + [NodeIdx](const EdgeInfo &EI) { + return EI.UserTE->Idx > + static_cast(NodeIdx); + }))) return; - // Conservatively determine if we can actually truncate the roots of the - // expression. Collect the values that can be demoted in ToDemote and - // additional roots that require investigating in Roots. - SmallVector ToDemote; - DenseMap> DemotedConsts; - SmallVector Roots; - for (auto *Root : TreeRoot) { - DenseSet Visited; - if (!collectValuesToDemote(Root, ToDemote, DemotedConsts, Roots, Visited)) - return; - } - - // The maximum bit width required to represent all the values that can be - // demoted without loss of precision. It would be safe to truncate the roots - // of the expression to this width. - auto MaxBitWidth = 1u; - - // We first check if all the bits of the roots are demanded. If they're not, - // we can truncate the roots to this narrower type. - for (auto *Root : TreeRoot) { - auto Mask = DB->getDemandedBits(cast(Root)); - MaxBitWidth = std::max(Mask.getBitWidth() - Mask.countl_zero(), - MaxBitWidth); - } - - // True if the roots can be zero-extended back to their original type, rather - // than sign-extended. We know that if the leading bits are not demanded, we - // can safely zero-extend. So we initialize IsKnownPositive to True. - bool IsKnownPositive = true; - - // If all the bits of the roots are demanded, we can try a little harder to - // compute a narrower type. This can happen, for example, if the roots are - // getelementptr indices. InstCombine promotes these indices to the pointer - // width. Thus, all their bits are technically demanded even though the - // address computation might be vectorized in a smaller type. - // - // We start by looking at each entry that can be demoted. We compute the - // maximum bit width required to store the scalar by using ValueTracking to - // compute the number of high-order bits we can truncate. - if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType()) && - all_of(TreeRoot, [](Value *V) { - return all_of(V->users(), - [](User *U) { return isa(U); }); - })) { - MaxBitWidth = 8u; + // The first value node for store/insertelement is sext/zext/trunc? Skip it, + // resize to the final type. + bool IsProfitableToDemoteRoot = !IsStoreOrInsertElt; + if (NodeIdx != 0 && + VectorizableTree[NodeIdx]->State == TreeEntry::Vectorize && + (VectorizableTree[NodeIdx]->getOpcode() == Instruction::ZExt || + VectorizableTree[NodeIdx]->getOpcode() == Instruction::SExt || + VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc)) { + assert(IsStoreOrInsertElt && "Expected store/insertelement seeded graph."); + ++NodeIdx; + IsProfitableToDemoteRoot = true; + } + + // Analyzed in reduction already and not profitable - exit. + if (AnalyzedMinBWVals.contains(VectorizableTree[NodeIdx]->Scalars.front())) + return; + SmallVector ToDemote; + DenseMap> DemotedConsts; + auto ComputeMaxBitWidth = [&](ArrayRef TreeRoot, unsigned VF, + bool IsTopRoot, bool IsProfitableToDemoteRoot, + unsigned Opcode, unsigned Limit) { + ToDemote.clear(); + auto *TreeRootIT = dyn_cast(TreeRoot[0]->getType()); + if (!TreeRootIT || !Opcode) + return 0u; + + if (AnalyzedMinBWVals.contains(TreeRoot.front())) + return 0u; + + unsigned NumParts = TTI->getNumberOfParts( + FixedVectorType::get(TreeRoot.front()->getType(), VF)); + + // The maximum bit width required to represent all the values that can be + // demoted without loss of precision. It would be safe to truncate the roots + // of the expression to this width. + unsigned MaxBitWidth = 1u; + + // True if the roots can be zero-extended back to their original type, + // rather than sign-extended. We know that if the leading bits are not + // demanded, we can safely zero-extend. So we initialize IsKnownPositive to + // True. // Determine if the sign bit of all the roots is known to be zero. If not, // IsKnownPositive is set to False. - IsKnownPositive = llvm::all_of(TreeRoot, [&](Value *R) { + bool IsKnownPositive = all_of(TreeRoot, [&](Value *R) { KnownBits Known = computeKnownBits(R, *DL); return Known.isNonNegative(); }); - // Determine the maximum number of bits required to store the scalar - // values. - for (auto *Scalar : ToDemote) { - auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, nullptr, DT); - auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType()); - MaxBitWidth = std::max(NumTypeBits - NumSignBits, MaxBitWidth); - } - - // If we can't prove that the sign bit is zero, we must add one to the - // maximum bit width to account for the unknown sign bit. This preserves - // the existing sign bit so we can safely sign-extend the root back to the - // original type. Otherwise, if we know the sign bit is zero, we will - // zero-extend the root instead. - // - // FIXME: This is somewhat suboptimal, as there will be cases where adding - // one to the maximum bit width will yield a larger-than-necessary - // type. In general, we need to add an extra bit only if we can't - // prove that the upper bit of the original type is equal to the - // upper bit of the proposed smaller type. If these two bits are the - // same (either zero or one) we know that sign-extending from the - // smaller type will result in the same value. Here, since we can't - // yet prove this, we are just making the proposed smaller type - // larger to ensure correctness. - if (!IsKnownPositive) - ++MaxBitWidth; - } - - // Round MaxBitWidth up to the next power-of-two. - MaxBitWidth = llvm::bit_ceil(MaxBitWidth); - - // If the maximum bit width we compute is less than the with of the roots' - // type, we can proceed with the narrowing. Otherwise, do nothing. - if (MaxBitWidth >= TreeRootIT->getBitWidth()) - return; + // We first check if all the bits of the roots are demanded. If they're not, + // we can truncate the roots to this narrower type. + for (auto *Root : TreeRoot) { + unsigned NumSignBits = ComputeNumSignBits(Root, *DL, 0, AC, nullptr, DT); + TypeSize NumTypeBits = DL->getTypeSizeInBits(Root->getType()); + unsigned BitWidth1 = NumTypeBits - NumSignBits; + // If we can't prove that the sign bit is zero, we must add one to the + // maximum bit width to account for the unknown sign bit. This preserves + // the existing sign bit so we can safely sign-extend the root back to the + // original type. Otherwise, if we know the sign bit is zero, we will + // zero-extend the root instead. + // + // FIXME: This is somewhat suboptimal, as there will be cases where adding + // one to the maximum bit width will yield a larger-than-necessary + // type. In general, we need to add an extra bit only if we can't + // prove that the upper bit of the original type is equal to the + // upper bit of the proposed smaller type. If these two bits are + // the same (either zero or one) we know that sign-extending from + // the smaller type will result in the same value. Here, since we + // can't yet prove this, we are just making the proposed smaller + // type larger to ensure correctness. + if (!IsKnownPositive) + ++BitWidth1; + + APInt Mask = DB->getDemandedBits(cast(Root)); + unsigned BitWidth2 = Mask.getBitWidth() - Mask.countl_zero(); + MaxBitWidth = + std::max(std::min(BitWidth1, BitWidth2), MaxBitWidth); + } + + if (MaxBitWidth < 8 && MaxBitWidth > 1) + MaxBitWidth = 8; + + // If the original type is large, but reduced type does not improve the reg + // use - ignore it. + if (NumParts > 1 && + NumParts == + TTI->getNumberOfParts(FixedVectorType::get( + IntegerType::get(F->getContext(), bit_ceil(MaxBitWidth)), VF))) + return 0u; + + bool IsProfitableToDemote = Opcode == Instruction::Trunc || + Opcode == Instruction::SExt || + Opcode == Instruction::ZExt || NumParts > 1; + // Conservatively determine if we can actually truncate the roots of the + // expression. Collect the values that can be demoted in ToDemote and + // additional roots that require investigating in Roots. + for (auto *Root : TreeRoot) { + DenseSet Visited; + unsigned MaxDepthLevel; + bool NeedToDemote = IsProfitableToDemote; + + if (!collectValuesToDemote(Root, IsProfitableToDemoteRoot, MaxBitWidth, + ToDemote, DemotedConsts, Visited, + MaxDepthLevel, NeedToDemote) || + (MaxDepthLevel <= Limit && + !(((Opcode == Instruction::SExt || Opcode == Instruction::ZExt) && + (!IsTopRoot || !(IsStoreOrInsertElt || UserIgnoreList) || + DL->getTypeSizeInBits(Root->getType()) / + DL->getTypeSizeInBits( + cast(Root)->getOperand(0)->getType()) > + 2)) || + (Opcode == Instruction::Trunc && + (!IsTopRoot || !(IsStoreOrInsertElt || UserIgnoreList) || + DL->getTypeSizeInBits( + cast(Root)->getOperand(0)->getType()) / + DL->getTypeSizeInBits(Root->getType()) > + 2))))) + return 0u; + } + // Round MaxBitWidth up to the next power-of-two. + MaxBitWidth = bit_ceil(MaxBitWidth); + + return MaxBitWidth; + }; // If we can truncate the root, we must collect additional values that might // be demoted as a result. That is, those seeded by truncations we will // modify. - while (!Roots.empty()) { - DenseSet Visited; - collectValuesToDemote(Roots.pop_back_val(), ToDemote, DemotedConsts, Roots, - Visited); - } - - // Check that all users are marked for demotion. - DenseSet Demoted(ToDemote.begin(), ToDemote.end()); - DenseSet Visited; - for (Value *V: ToDemote) { - const TreeEntry *TE = getTreeEntry(V); - assert(TE && "Expected vectorized scalar."); - if (!Visited.insert(TE).second) - continue; - if (!all_of(TE->UserTreeIndices, [&](const EdgeInfo &EI) { - return all_of(EI.UserTE->Scalars, - [&](Value *V) { return Demoted.contains(V); }); - })) - return; - } - // Finally, map the values we can demote to the maximum bit with we computed. - for (auto *Scalar : ToDemote) { - auto *TE = getTreeEntry(Scalar); - assert(TE && "Expected vectorized scalar."); - if (MinBWs.contains(TE)) + // Add reduction ops sizes, if any. + if (UserIgnoreList && + isa(VectorizableTree.front()->Scalars.front()->getType())) { + for (Value *V : *UserIgnoreList) { + auto NumSignBits = ComputeNumSignBits(V, *DL, 0, AC, nullptr, DT); + auto NumTypeBits = DL->getTypeSizeInBits(V->getType()); + unsigned BitWidth1 = NumTypeBits - NumSignBits; + if (!isKnownNonNegative(V, SimplifyQuery(*DL))) + ++BitWidth1; + auto Mask = DB->getDemandedBits(cast(V)); + unsigned BitWidth2 = Mask.getBitWidth() - Mask.countl_zero(); + ReductionBitWidth = + std::max(std::min(BitWidth1, BitWidth2), ReductionBitWidth); + } + if (ReductionBitWidth < 8 && ReductionBitWidth > 1) + ReductionBitWidth = 8; + + ReductionBitWidth = bit_ceil(ReductionBitWidth); + } + bool IsTopRoot = NodeIdx == 0; + while (NodeIdx < VectorizableTree.size() && + VectorizableTree[NodeIdx]->State == TreeEntry::Vectorize && + VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc) + ++NodeIdx; + while (NodeIdx < VectorizableTree.size()) { + ArrayRef TreeRoot = VectorizableTree[NodeIdx]->Scalars; + unsigned Limit = 2; + unsigned Opcode = VectorizableTree[NodeIdx]->getOpcode(); + if (IsTopRoot && + ReductionBitWidth == + DL->getTypeSizeInBits( + VectorizableTree.front()->Scalars.front()->getType())) + Limit = 3; + unsigned MaxBitWidth = ComputeMaxBitWidth( + TreeRoot, VectorizableTree[NodeIdx]->getVectorFactor(), IsTopRoot, + IsProfitableToDemoteRoot, Opcode, Limit); + IsTopRoot = false; + IsProfitableToDemoteRoot = true; + + if (TruncNodes.empty()) { + NodeIdx = VectorizableTree.size(); + } else { + NodeIdx = *TruncNodes.begin() + 1; + TruncNodes.erase(TruncNodes.begin()); + } + + // If the maximum bit width we compute is less than the with of the roots' + // type, we can proceed with the narrowing. Otherwise, do nothing. + if (MaxBitWidth == 0 || + MaxBitWidth >= + cast(TreeRoot.front()->getType())->getBitWidth()) { + if (UserIgnoreList) + AnalyzedMinBWVals.insert(TreeRoot.begin(), TreeRoot.end()); continue; - bool IsSigned = any_of(TE->Scalars, [&](Value *R) { - KnownBits Known = computeKnownBits(R, *DL); - return !Known.isNonNegative(); - }); - MinBWs.try_emplace(TE, MaxBitWidth, IsSigned); - const auto *I = cast(Scalar); - auto DCIt = DemotedConsts.find(I); - if (DCIt != DemotedConsts.end()) { - for (unsigned Idx : DCIt->getSecond()) { - // Check that all instructions operands are demoted. - if (all_of(TE->Scalars, [&](Value *V) { - auto SIt = DemotedConsts.find(cast(V)); - return SIt != DemotedConsts.end() && - is_contained(SIt->getSecond(), Idx); - })) { + } + + // Finally, map the values we can demote to the maximum bit with we + // computed. + for (Value *Scalar : ToDemote) { + TreeEntry *TE = getTreeEntry(Scalar); + assert(TE && "Expected vectorized scalar."); + if (MinBWs.contains(TE)) + continue; + bool IsSigned = any_of(TE->Scalars, [&](Value *R) { + return !isKnownNonNegative(R, SimplifyQuery(*DL)); + }); + MinBWs.try_emplace(TE, MaxBitWidth, IsSigned); + const auto *I = cast(Scalar); + auto DCIt = DemotedConsts.find(I); + if (DCIt != DemotedConsts.end()) { + for (unsigned Idx : DCIt->getSecond()) { + // Check that all instructions operands are demoted. const TreeEntry *CTE = getOperandEntry(TE, Idx); - MinBWs.try_emplace(CTE, MaxBitWidth, IsSigned); + if (all_of(TE->Scalars, + [&](Value *V) { + auto SIt = DemotedConsts.find(cast(V)); + return SIt != DemotedConsts.end() && + is_contained(SIt->getSecond(), Idx); + }) || + all_of(CTE->Scalars, Constant::classof)) + MinBWs.try_emplace(CTE, MaxBitWidth, IsSigned); } } } diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll index cef791633655..5e3fd156666f 100644 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll @@ -17,12 +17,13 @@ define void @test1(<4 x i16> %a, <4 x i16> %b, ptr %p) { ; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i64 [[S0]] ; CHECK-NEXT: [[LOAD0:%.*]] = load i64, ptr [[GEP0]], align 4 ; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <4 x i32> [[SUB0]], <4 x i32> poison, <2 x i32> -; CHECK-NEXT: [[TMP1:%.*]] = sext <2 x i32> [[TMP0]] to <2 x i64> -; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0 +; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i32> [[TMP0]], i32 0 +; CHECK-NEXT: [[TMP2:%.*]] = sext i32 [[TMP1]] to i64 ; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP2]] ; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[GEP1]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1 -; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP3]] +; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP0]], i32 1 +; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[TMP3]] to i64 +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP4]] ; CHECK-NEXT: [[LOAD2:%.*]] = load i64, ptr [[GEP2]], align 4 ; CHECK-NEXT: [[E3:%.*]] = extractelement <4 x i32> [[SUB0]], i32 3 ; CHECK-NEXT: [[S3:%.*]] = sext i32 [[E3]] to i64 diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll index 47485e514ec2..1cce52060c47 100644 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ;test_i16_extend NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer,dce,instcombine -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s +; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer,dce,instcombine -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s ; RUN: cat %t | FileCheck -check-prefix=YAML %s -; RUN: opt -S -mtriple=aarch64--linux-gnu -passes='slp-vectorizer,dce,instcombine' -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s +; RUN: opt -S -mtriple=aarch64--linux-gnu -passes='slp-vectorizer,dce,instcombine' -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s ; RUN: cat %t | FileCheck -check-prefix=YAML %s diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll index d67fdc1cd6aa..a7a7f642ced5 100644 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll @@ -28,21 +28,11 @@ entry: define i64 @red_zext_ld_4xi64(ptr %ptr) { ; CHECK-LABEL: @red_zext_ld_4xi64( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[LD0:%.*]] = load i8, ptr [[PTR:%.*]], align 1 -; CHECK-NEXT: [[ZEXT:%.*]] = zext i8 [[LD0]] to i64 -; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 1 -; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[GEP]], align 1 -; CHECK-NEXT: [[ZEXT_1:%.*]] = zext i8 [[LD1]] to i64 -; CHECK-NEXT: [[ADD_1:%.*]] = add nuw nsw i64 [[ZEXT]], [[ZEXT_1]] -; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 2 -; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[GEP_1]], align 1 -; CHECK-NEXT: [[ZEXT_2:%.*]] = zext i8 [[LD2]] to i64 -; CHECK-NEXT: [[ADD_2:%.*]] = add nuw nsw i64 [[ADD_1]], [[ZEXT_2]] -; CHECK-NEXT: [[GEP_2:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 3 -; CHECK-NEXT: [[LD3:%.*]] = load i8, ptr [[GEP_2]], align 1 -; CHECK-NEXT: [[ZEXT_3:%.*]] = zext i8 [[LD3]] to i64 -; CHECK-NEXT: [[ADD_3:%.*]] = add nuw nsw i64 [[ADD_2]], [[ZEXT_3]] -; CHECK-NEXT: ret i64 [[ADD_3]] +; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[PTR:%.*]], align 1 +; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i16> +; CHECK-NEXT: [[TMP2:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]]) +; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[TMP2]] to i64 +; CHECK-NEXT: ret i64 [[TMP3]] ; entry: %ld0 = load i8, ptr %ptr diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll index 000e7a56df37..500f10659f04 100644 --- a/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll @@ -802,9 +802,10 @@ define i64 @red_zext_ld_4xi64(ptr %ptr) { ; CHECK-LABEL: @red_zext_ld_4xi64( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[PTR:%.*]], align 1 -; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i64> -; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP1]]) -; CHECK-NEXT: ret i64 [[TMP2]] +; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i16> +; CHECK-NEXT: [[TMP2:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]]) +; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[TMP2]] to i64 +; CHECK-NEXT: ret i64 [[TMP3]] ; entry: %ld0 = load i8, ptr %ptr diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll index 4565d4928ba4..05511f843a68 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll @@ -15,11 +15,12 @@ define { i64, i64 } @patatino(double %arg) { ; CHECK-NEXT: [[TMP6:%.*]] = load <2 x double>, ptr getelementptr inbounds ([6 x double], ptr @global, i64 0, i64 4), align 16 ; CHECK-NEXT: [[TMP7:%.*]] = fadd <2 x double> [[TMP6]], [[TMP5]] ; CHECK-NEXT: [[TMP8:%.*]] = fptosi <2 x double> [[TMP7]] to <2 x i32> -; CHECK-NEXT: [[TMP9:%.*]] = sext <2 x i32> [[TMP8]] to <2 x i64> -; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i64> [[TMP9]], i32 0 +; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[TMP8]], i32 0 +; CHECK-NEXT: [[TMP10:%.*]] = sext i32 [[TMP9]] to i64 ; CHECK-NEXT: [[T16:%.*]] = insertvalue { i64, i64 } undef, i64 [[TMP10]], 0 -; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i64> [[TMP9]], i32 1 -; CHECK-NEXT: [[T17:%.*]] = insertvalue { i64, i64 } [[T16]], i64 [[TMP11]], 1 +; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[TMP8]], i32 1 +; CHECK-NEXT: [[TMP12:%.*]] = sext i32 [[TMP11]] to i64 +; CHECK-NEXT: [[T17:%.*]] = insertvalue { i64, i64 } [[T16]], i64 [[TMP12]], 1 ; CHECK-NEXT: ret { i64, i64 } [[T17]] ; bb: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll index a0af8e36b36c..5ee801607653 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 -; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-3 < %s | FileCheck %s +; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-6 < %s | FileCheck %s define void @t(i64 %v) { ; CHECK-LABEL: define void @t( diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll index 6e512fcbb739..6051638562b5 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll @@ -6,18 +6,17 @@ define void @test(i8 %0) { ; CHECK-SAME: i8 [[TMP0:%.*]]) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> , i8 [[TMP0]], i32 1 -; CHECK-NEXT: [[TMP2:%.*]] = sext <2 x i8> [[TMP1]] to <2 x i16> -; CHECK-NEXT: [[TMP3:%.*]] = sext <2 x i16> [[TMP2]] to <2 x i32> -; CHECK-NEXT: [[TMP4:%.*]] = mul <2 x i16> [[TMP2]], zeroinitializer -; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i16> [[TMP4]], i32 0 -; CHECK-NEXT: [[TMP6:%.*]] = zext i16 [[TMP5]] to i32 -; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i16> [[TMP4]], i32 1 -; CHECK-NEXT: [[TMP8:%.*]] = zext i16 [[TMP7]] to i32 -; CHECK-NEXT: [[ADD:%.*]] = or i32 [[TMP6]], [[TMP8]] +; CHECK-NEXT: [[TMP2:%.*]] = sext <2 x i8> [[TMP1]] to <2 x i32> +; CHECK-NEXT: [[TMP3:%.*]] = mul <2 x i8> [[TMP1]], zeroinitializer +; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[TMP3]], i32 0 +; CHECK-NEXT: [[TMP5:%.*]] = zext i8 [[TMP4]] to i32 +; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i8> [[TMP3]], i32 1 +; CHECK-NEXT: [[TMP7:%.*]] = zext i8 [[TMP6]] to i32 +; CHECK-NEXT: [[ADD:%.*]] = or i32 [[TMP5]], [[TMP7]] ; CHECK-NEXT: [[SHR:%.*]] = lshr i32 [[ADD]], 1 ; CHECK-NEXT: [[CONV9:%.*]] = trunc i32 [[SHR]] to i8 ; CHECK-NEXT: store i8 [[CONV9]], ptr null, align 1 -; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <8 x i32> +; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> ; CHECK-NEXT: ret void ; entry: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll index 2c834616becc..4acd63078b82 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll @@ -6,15 +6,20 @@ define void @test(i64 %d.promoted.i) { ; CHECK-SAME: i64 [[D_PROMOTED_I:%.*]]) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[AND_1_I:%.*]] = and i64 0, [[D_PROMOTED_I]] +; CHECK-NEXT: [[TMP0:%.*]] = insertelement <8 x i64> , i64 [[AND_1_I]], i32 1 +; CHECK-NEXT: [[TMP1:%.*]] = trunc <8 x i64> [[TMP0]] to <8 x i1> +; CHECK-NEXT: [[TMP2:%.*]] = mul <8 x i1> [[TMP1]], zeroinitializer ; CHECK-NEXT: [[AND_1_I_1:%.*]] = and i64 0, 0 -; CHECK-NEXT: [[TMP0:%.*]] = insertelement <16 x i64> , i64 [[AND_1_I_1]], i32 1 -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <16 x i64> [[TMP0]], i64 [[AND_1_I]], i32 9 -; CHECK-NEXT: [[TMP2:%.*]] = trunc <16 x i64> [[TMP1]] to <16 x i1> -; CHECK-NEXT: [[TMP3:%.*]] = mul <16 x i1> [[TMP2]], zeroinitializer -; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP3]]) -; CHECK-NEXT: [[TMP5:%.*]] = zext i1 [[TMP4]] to i32 -; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[TMP5]], 0 -; CHECK-NEXT: store i32 [[TMP6]], ptr null, align 4 +; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i64> , i64 [[AND_1_I_1]], i32 1 +; CHECK-NEXT: [[TMP4:%.*]] = trunc <8 x i64> [[TMP3]] to <8 x i1> +; CHECK-NEXT: [[TMP5:%.*]] = mul <8 x i1> [[TMP4]], zeroinitializer +; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP5]]) +; CHECK-NEXT: [[TMP7:%.*]] = zext i1 [[TMP6]] to i32 +; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP2]]) +; CHECK-NEXT: [[TMP9:%.*]] = zext i1 [[TMP8]] to i32 +; CHECK-NEXT: [[OP_RDX:%.*]] = or i32 [[TMP7]], [[TMP9]] +; CHECK-NEXT: [[TMP10:%.*]] = and i32 [[OP_RDX]], 0 +; CHECK-NEXT: store i32 [[TMP10]], ptr null, align 4 ; CHECK-NEXT: ret void ; entry: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll index 651631de2c35..a316415dcc6b 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll @@ -17,12 +17,15 @@ target triple = "x86_64-unknown-linux-gnu" define i8 @PR31243_zext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) { ; SSE-LABEL: @PR31243_zext( ; SSE-NEXT: entry: -; SSE-NEXT: [[TMP0:%.*]] = or i8 [[V0:%.*]], 1 -; SSE-NEXT: [[TMP1:%.*]] = or i8 [[V1:%.*]], 1 -; SSE-NEXT: [[TMP2:%.*]] = zext i8 [[TMP0]] to i64 -; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP2]] -; SSE-NEXT: [[TMP3:%.*]] = zext i8 [[TMP1]] to i64 -; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP3]] +; SSE-NEXT: [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0 +; SSE-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1 +; SSE-NEXT: [[TMP2:%.*]] = or <2 x i8> [[TMP1]], +; SSE-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0 +; SSE-NEXT: [[TMP4:%.*]] = zext i8 [[TMP3]] to i64 +; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]] +; SSE-NEXT: [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1 +; SSE-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i64 +; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]] ; SSE-NEXT: [[T6:%.*]] = load i8, ptr [[T4]], align 1 ; SSE-NEXT: [[T7:%.*]] = load i8, ptr [[T5]], align 1 ; SSE-NEXT: [[T8:%.*]] = add i8 [[T6]], [[T7]] @@ -73,12 +76,15 @@ entry: define i8 @PR31243_sext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) { ; SSE-LABEL: @PR31243_sext( ; SSE-NEXT: entry: -; SSE-NEXT: [[TMP0:%.*]] = or i8 [[V0:%.*]], 1 -; SSE-NEXT: [[TMP1:%.*]] = or i8 [[V1:%.*]], 1 -; SSE-NEXT: [[TMP2:%.*]] = sext i8 [[TMP0]] to i64 -; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP2]] -; SSE-NEXT: [[TMP3:%.*]] = sext i8 [[TMP1]] to i64 -; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP3]] +; SSE-NEXT: [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0 +; SSE-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1 +; SSE-NEXT: [[TMP2:%.*]] = or <2 x i8> [[TMP1]], +; SSE-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0 +; SSE-NEXT: [[TMP4:%.*]] = sext i8 [[TMP3]] to i64 +; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]] +; SSE-NEXT: [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1 +; SSE-NEXT: [[TMP6:%.*]] = sext i8 [[TMP5]] to i64 +; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]] ; SSE-NEXT: [[T6:%.*]] = load i8, ptr [[T4]], align 1 ; SSE-NEXT: [[T7:%.*]] = load i8, ptr [[T5]], align 1 ; SSE-NEXT: [[T8:%.*]] = add i8 [[T6]], [[T7]] @@ -89,13 +95,12 @@ define i8 @PR31243_sext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) { ; AVX-NEXT: [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0 ; AVX-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1 ; AVX-NEXT: [[TMP2:%.*]] = or <2 x i8> [[TMP1]], -; AVX-NEXT: [[TMP3:%.*]] = sext <2 x i8> [[TMP2]] to <2 x i16> -; AVX-NEXT: [[TMP4:%.*]] = extractelement <2 x i16> [[TMP3]], i64 0 -; AVX-NEXT: [[TMP5:%.*]] = sext i16 [[TMP4]] to i64 -; AVX-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP5]] -; AVX-NEXT: [[TMP6:%.*]] = extractelement <2 x i16> [[TMP3]], i64 1 -; AVX-NEXT: [[TMP7:%.*]] = sext i16 [[TMP6]] to i64 -; AVX-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP7]] +; AVX-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0 +; AVX-NEXT: [[TMP4:%.*]] = sext i8 [[TMP3]] to i64 +; AVX-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]] +; AVX-NEXT: [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1 +; AVX-NEXT: [[TMP6:%.*]] = sext i8 [[TMP5]] to i64 +; AVX-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]] ; AVX-NEXT: [[T6:%.*]] = load i8, ptr [[T4]], align 1 ; AVX-NEXT: [[T7:%.*]] = load i8, ptr [[T5]], align 1 ; AVX-NEXT: [[T8:%.*]] = add i8 [[T6]], [[T7]] diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll index 88f75c37846e..3cc32c1fc7b2 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll @@ -15,8 +15,8 @@ define i32 @phi3UndefInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %arg3) { ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> -; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) +; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) +; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: @@ -52,8 +52,8 @@ define i32 @phi2UndefInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %arg3) { ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> -; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) +; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) +; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: @@ -89,8 +89,8 @@ define i32 @phi1UndefInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %arg3) { ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> -; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) +; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) +; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: @@ -127,8 +127,8 @@ define i32 @phi1Undef1PoisonInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %ar ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> -; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) +; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) +; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: @@ -165,8 +165,8 @@ define i32 @phi1Undef2PoisonInputs(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %a ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> -; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) +; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) +; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: @@ -202,8 +202,8 @@ define i32 @phi1Undef1PoisonGapInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> -; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) +; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) +; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/resched.ll b/llvm/test/Transforms/SLPVectorizer/X86/resched.ll index 78c6d9516a3d..b7237cbb02bb 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/resched.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/resched.ll @@ -11,26 +11,26 @@ define fastcc void @_ZN12_GLOBAL__N_127PolynomialMultiplyRecognize9recognizeEv() ; CHECK: if.then22.i: ; CHECK-NEXT: [[SUB_I:%.*]] = add nsw i32 undef, -1 ; CHECK-NEXT: [[CONV31_I:%.*]] = and i32 undef, [[SUB_I]] -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[CONV31_I]], i32 0 -; CHECK-NEXT: [[SHUFFLE1:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP2:%.*]] = lshr <4 x i32> [[SHUFFLE1]], +; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[CONV31_I]], i32 0 +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = lshr <4 x i32> [[TMP1]], ; CHECK-NEXT: [[SHR_4_I_I:%.*]] = lshr i32 [[CONV31_I]], 5 ; CHECK-NEXT: [[SHR_5_I_I:%.*]] = lshr i32 [[CONV31_I]], 6 ; CHECK-NEXT: [[SHR_6_I_I:%.*]] = lshr i32 [[CONV31_I]], 7 ; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i32> poison, i32 [[CONV31_I]], i32 0 -; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> poison, <8 x i32> zeroinitializer -; CHECK-NEXT: [[TMP4:%.*]] = lshr <8 x i32> [[SHUFFLE]], -; CHECK-NEXT: [[TMP5:%.*]] = insertelement <16 x i32> poison, i32 [[SUB_I]], i32 0 -; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <16 x i32> -; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> [[TMP6]], <16 x i32> -; CHECK-NEXT: [[TMP8:%.*]] = insertelement <16 x i32> [[TMP7]], i32 [[SHR_4_I_I]], i32 5 -; CHECK-NEXT: [[TMP9:%.*]] = insertelement <16 x i32> [[TMP8]], i32 [[SHR_5_I_I]], i32 6 -; CHECK-NEXT: [[TMP10:%.*]] = insertelement <16 x i32> [[TMP9]], i32 [[SHR_6_I_I]], i32 7 -; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <16 x i32> -; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <16 x i32> [[TMP10]], <16 x i32> [[TMP11]], <16 x i32> -; CHECK-NEXT: [[TMP13:%.*]] = trunc <16 x i32> [[TMP12]] to <16 x i8> -; CHECK-NEXT: [[TMP14:%.*]] = and <16 x i8> [[TMP13]], -; CHECK-NEXT: store <16 x i8> [[TMP14]], ptr undef, align 1 +; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> poison, <8 x i32> zeroinitializer +; CHECK-NEXT: [[TMP5:%.*]] = lshr <8 x i32> [[TMP4]], +; CHECK-NEXT: [[TMP6:%.*]] = insertelement <16 x i32> poison, i32 [[SUB_I]], i32 0 +; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <16 x i32> +; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <16 x i32> [[TMP6]], <16 x i32> [[TMP7]], <16 x i32> +; CHECK-NEXT: [[TMP9:%.*]] = insertelement <16 x i32> [[TMP8]], i32 [[SHR_4_I_I]], i32 5 +; CHECK-NEXT: [[TMP10:%.*]] = insertelement <16 x i32> [[TMP9]], i32 [[SHR_5_I_I]], i32 6 +; CHECK-NEXT: [[TMP11:%.*]] = insertelement <16 x i32> [[TMP10]], i32 [[SHR_6_I_I]], i32 7 +; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> poison, <16 x i32> +; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <16 x i32> [[TMP11]], <16 x i32> [[TMP12]], <16 x i32> +; CHECK-NEXT: [[TMP14:%.*]] = trunc <16 x i32> [[TMP13]] to <16 x i8> +; CHECK-NEXT: [[TMP15:%.*]] = and <16 x i8> [[TMP14]], +; CHECK-NEXT: store <16 x i8> [[TMP15]], ptr undef, align 1 ; CHECK-NEXT: unreachable ; CHECK: if.end50.i: ; CHECK-NEXT: ret void diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll index 5d22b5a4873b..1d1fcec2a7ae 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll @@ -7,12 +7,10 @@ define i1 @test(i1 %cmp5.not.31) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i1> , i1 [[CMP5_NOT_31]], i32 0 ; CHECK-NEXT: [[TMP1:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> zeroinitializer, <4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP2:%.*]] = trunc <4 x i32> [[TMP1]] to <4 x i1> -; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32> -; CHECK-NEXT: [[TMP4:%.*]] = mul <4 x i32> [[TMP3]], -; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]]) -; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[TMP5]], 0 -; CHECK-NEXT: [[CMP_NOT_I_I:%.*]] = icmp eq i32 [[TMP6]], 0 +; CHECK-NEXT: [[TMP2:%.*]] = mul <4 x i32> [[TMP1]], +; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]]) +; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[TMP3]], 0 +; CHECK-NEXT: [[CMP_NOT_I_I:%.*]] = icmp eq i32 [[TMP4]], 0 ; CHECK-NEXT: ret i1 [[CMP_NOT_I_I]] ; entry: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll index c1dd90d0e9a7..2f6868d8dfd6 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll @@ -8,17 +8,18 @@ ; YAML-NEXT: Function: stores ; YAML-NEXT: Args: ; YAML-NEXT: - String: 'Stores SLP vectorized with cost ' -; YAML-NEXT: - Cost: '-3' +; YAML-NEXT: - Cost: '-7' ; YAML-NEXT: - String: ' and with tree size ' ; YAML-NEXT: - TreeSize: '6' define void @stores(ptr noalias %in, ptr noalias %inn, ptr noalias %out) { ; CHECK-LABEL: @stores( ; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[IN:%.*]], align 1 ; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[INN:%.*]], align 1 -; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i64> -; CHECK-NEXT: [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i64> -; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[TMP3]], [[TMP4]] -; CHECK-NEXT: store <4 x i64> [[TMP5]], ptr [[OUT:%.*]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i16> +; CHECK-NEXT: [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i16> +; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i16> [[TMP3]], [[TMP4]] +; CHECK-NEXT: [[TMP6:%.*]] = zext <4 x i16> [[TMP5]] to <4 x i64> +; CHECK-NEXT: store <4 x i64> [[TMP6]], ptr [[OUT:%.*]], align 4 ; CHECK-NEXT: ret void ; %load.1 = load i8, ptr %in, align 1 @@ -63,17 +64,18 @@ define void @stores(ptr noalias %in, ptr noalias %inn, ptr noalias %out) { ; YAML-NEXT: Function: insertelems ; YAML-NEXT: Args: ; YAML-NEXT: - String: 'SLP vectorized with cost ' -; YAML-NEXT: - Cost: '-5' +; YAML-NEXT: - Cost: '-9' ; YAML-NEXT: - String: ' and with tree size ' ; YAML-NEXT: - TreeSize: '6' define <4 x i64> @insertelems(ptr noalias %in, ptr noalias %inn) { ; CHECK-LABEL: @insertelems( ; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[IN:%.*]], align 1 ; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[INN:%.*]], align 1 -; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i64> -; CHECK-NEXT: [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i64> -; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[TMP3]], [[TMP4]] -; CHECK-NEXT: ret <4 x i64> [[TMP5]] +; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i16> +; CHECK-NEXT: [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i16> +; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i16> [[TMP3]], [[TMP4]] +; CHECK-NEXT: [[TMP6:%.*]] = zext <4 x i16> [[TMP5]] to <4 x i64> +; CHECK-NEXT: ret <4 x i64> [[TMP6]] ; %load.1 = load i8, ptr %in, align 1 %gep.1 = getelementptr inbounds i8, ptr %in, i64 1 diff --git a/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll b/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll index 061fbdb45a13..ff6f0bdd3db8 100644 --- a/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll +++ b/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll @@ -10,8 +10,8 @@ define i32 @alt_cmp(i16 %call46) { ; CHECK-NEXT: [[TMP2:%.*]] = icmp ult <4 x i16> [[TMP0]], [[TMP1]] ; CHECK-NEXT: [[TMP3:%.*]] = icmp ugt <4 x i16> [[TMP0]], [[TMP1]] ; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i1> [[TMP2]], <4 x i1> [[TMP3]], <4 x i32> -; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i1> [[TMP4]] to <4 x i16> -; CHECK-NEXT: [[TMP6:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP5]]) +; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) +; CHECK-NEXT: [[TMP6:%.*]] = zext i1 [[TMP5]] to i16 ; CHECK-NEXT: [[OP_RDX:%.*]] = or i16 [[TMP6]], 0 ; CHECK-NEXT: [[EXT:%.*]] = zext i16 [[OP_RDX]] to i32 ; CHECK-NEXT: ret i32 [[EXT]] -- GitLab From a213df5d3895f323ef0d2d9affc1020414576caa Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Thu, 7 Mar 2024 09:45:14 -0600 Subject: [PATCH 452/929] [LinkerWrapper] Use the correct empty file on Windows (#84322) Summary: The clang-offload-bundler uses an empty file to control the bundles made for embedding. Previously this still used `/dev/null` by mistake even on Windows. --- clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp index 576e8f2cd7f8..0a783db8962b 100644 --- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp +++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp @@ -410,7 +410,11 @@ fatbinary(ArrayRef> InputFiles, Targets.push_back(Saver.save("hipv4-amdgcn-amd-amdhsa--" + Arch)); CmdArgs.push_back(Saver.save(llvm::join(Targets, ","))); +#ifdef _WIN32 + CmdArgs.push_back("-input=NUL"); +#else CmdArgs.push_back("-input=/dev/null"); +#endif for (const auto &[File, Arch] : InputFiles) CmdArgs.push_back(Saver.save("-input=" + File)); -- GitLab From d6b3be375ffed14fefc93c2031cd56e680afd0c1 Mon Sep 17 00:00:00 2001 From: sylvain-audi <62035306+sylvain-audi@users.noreply.github.com> Date: Thu, 7 Mar 2024 10:54:41 -0500 Subject: [PATCH 453/929] [NFC][Asan] Prepare AddressSanitizer to detect inserted runtime calls (#84223) This is in preparation for an upcoming commit that will add "funclet" OpBundle to the inserted runtime calls where the function's EH personality requires it. See PR https://github.com/llvm/llvm-project/pull/82533 --- .../Instrumentation/AddressSanitizer.cpp | 184 +++++++++++------- 1 file changed, 111 insertions(+), 73 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp index f22f53b8cd8f..c95a50a033b1 100644 --- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -642,6 +642,23 @@ static uint64_t GetCtorAndDtorPriority(Triple &TargetTriple) { } namespace { +/// Helper RAII class to post-process inserted asan runtime calls during a +/// pass on a single Function. This is a no-op implementation, for a first NFC +/// commit. Coming up: detect and add "funclet" opBundle to function calls that +/// need them. +class RuntimeCallInserter { + Function *OwnerFn = nullptr; + +public: + RuntimeCallInserter(Function &Fn) : OwnerFn(&Fn) {} + + CallInst *createRuntimeCall(IRBuilder<> &IRB, FunctionCallee Callee, + ArrayRef Args = {}, + const Twine &Name = "") { + assert(IRB.GetInsertBlock()->getParent() == OwnerFn); + return IRB.CreateCall(Callee, Args, Name, nullptr); + } +}; /// AddressSanitizer: instrument the code in module to find memory bugs. struct AddressSanitizer { @@ -691,12 +708,14 @@ struct AddressSanitizer { void instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis, InterestingMemoryOperand &O, bool UseCalls, - const DataLayout &DL); - void instrumentPointerComparisonOrSubtraction(Instruction *I); + const DataLayout &DL, RuntimeCallInserter &RTCI); + void instrumentPointerComparisonOrSubtraction(Instruction *I, + RuntimeCallInserter &RTCI); void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore, Value *Addr, MaybeAlign Alignment, uint32_t TypeStoreSize, bool IsWrite, - Value *SizeArgument, bool UseCalls, uint32_t Exp); + Value *SizeArgument, bool UseCalls, uint32_t Exp, + RuntimeCallInserter &RTCI); Instruction *instrumentAMDGPUAddress(Instruction *OrigIns, Instruction *InsertBefore, Value *Addr, uint32_t TypeStoreSize, bool IsWrite, @@ -707,20 +726,22 @@ struct AddressSanitizer { Instruction *InsertBefore, Value *Addr, TypeSize TypeStoreSize, bool IsWrite, Value *SizeArgument, bool UseCalls, - uint32_t Exp); + uint32_t Exp, + RuntimeCallInserter &RTCI); void instrumentMaskedLoadOrStore(AddressSanitizer *Pass, const DataLayout &DL, Type *IntptrTy, Value *Mask, Value *EVL, Value *Stride, Instruction *I, Value *Addr, MaybeAlign Alignment, unsigned Granularity, Type *OpType, bool IsWrite, Value *SizeArgument, bool UseCalls, - uint32_t Exp); + uint32_t Exp, RuntimeCallInserter &RTCI); Value *createSlowPathCmp(IRBuilder<> &IRB, Value *AddrLong, Value *ShadowValue, uint32_t TypeStoreSize); Instruction *generateCrashCode(Instruction *InsertBefore, Value *Addr, bool IsWrite, size_t AccessSizeIndex, - Value *SizeArgument, uint32_t Exp); - void instrumentMemIntrinsic(MemIntrinsic *MI); + Value *SizeArgument, uint32_t Exp, + RuntimeCallInserter &RTCI); + void instrumentMemIntrinsic(MemIntrinsic *MI, RuntimeCallInserter &RTCI); Value *memToShadow(Value *Shadow, IRBuilder<> &IRB); bool suppressInstrumentationSiteForDebug(int &Instrumented); bool instrumentFunction(Function &F, const TargetLibraryInfo *TLI); @@ -912,6 +933,7 @@ private: struct FunctionStackPoisoner : public InstVisitor { Function &F; AddressSanitizer &ASan; + RuntimeCallInserter &RTCI; DIBuilder DIB; LLVMContext *C; Type *IntptrTy; @@ -948,10 +970,12 @@ struct FunctionStackPoisoner : public InstVisitor { bool HasReturnsTwiceCall = false; bool PoisonStack; - FunctionStackPoisoner(Function &F, AddressSanitizer &ASan) - : F(F), ASan(ASan), DIB(*F.getParent(), /*AllowUnresolved*/ false), - C(ASan.C), IntptrTy(ASan.IntptrTy), - IntptrPtrTy(PointerType::get(IntptrTy, 0)), Mapping(ASan.Mapping), + FunctionStackPoisoner(Function &F, AddressSanitizer &ASan, + RuntimeCallInserter &RTCI) + : F(F), ASan(ASan), RTCI(RTCI), + DIB(*F.getParent(), /*AllowUnresolved*/ false), C(ASan.C), + IntptrTy(ASan.IntptrTy), IntptrPtrTy(PointerType::get(IntptrTy, 0)), + Mapping(ASan.Mapping), PoisonStack(ClStack && !Triple(F.getParent()->getTargetTriple()).isAMDGPU()) {} @@ -1034,8 +1058,8 @@ struct FunctionStackPoisoner : public InstVisitor { DynamicAreaOffset); } - IRB.CreateCall( - AsanAllocasUnpoisonFunc, + RTCI.createRuntimeCall( + IRB, AsanAllocasUnpoisonFunc, {IRB.CreateLoad(IntptrTy, DynamicAllocaLayout), DynamicAreaPtr}); } @@ -1251,16 +1275,18 @@ Value *AddressSanitizer::memToShadow(Value *Shadow, IRBuilder<> &IRB) { } // Instrument memset/memmove/memcpy -void AddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) { +void AddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI, + RuntimeCallInserter &RTCI) { InstrumentationIRBuilder IRB(MI); if (isa(MI)) { - IRB.CreateCall(isa(MI) ? AsanMemmove : AsanMemcpy, - {IRB.CreateAddrSpaceCast(MI->getOperand(0), PtrTy), - IRB.CreateAddrSpaceCast(MI->getOperand(1), PtrTy), - IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)}); + RTCI.createRuntimeCall( + IRB, isa(MI) ? AsanMemmove : AsanMemcpy, + {IRB.CreateAddrSpaceCast(MI->getOperand(0), PtrTy), + IRB.CreateAddrSpaceCast(MI->getOperand(1), PtrTy), + IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)}); } else if (isa(MI)) { - IRB.CreateCall( - AsanMemset, + RTCI.createRuntimeCall( + IRB, AsanMemset, {IRB.CreateAddrSpaceCast(MI->getOperand(0), PtrTy), IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false), IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)}); @@ -1498,7 +1524,7 @@ bool AddressSanitizer::GlobalIsLinkerInitialized(GlobalVariable *G) { } void AddressSanitizer::instrumentPointerComparisonOrSubtraction( - Instruction *I) { + Instruction *I, RuntimeCallInserter &RTCI) { IRBuilder<> IRB(I); FunctionCallee F = isa(I) ? AsanPtrCmpFunction : AsanPtrSubFunction; Value *Param[2] = {I->getOperand(0), I->getOperand(1)}; @@ -1506,7 +1532,7 @@ void AddressSanitizer::instrumentPointerComparisonOrSubtraction( if (i->getType()->isPointerTy()) i = IRB.CreatePointerCast(i, IntptrTy); } - IRB.CreateCall(F, Param); + RTCI.createRuntimeCall(IRB, F, Param); } static void doInstrumentAddress(AddressSanitizer *Pass, Instruction *I, @@ -1514,7 +1540,7 @@ static void doInstrumentAddress(AddressSanitizer *Pass, Instruction *I, MaybeAlign Alignment, unsigned Granularity, TypeSize TypeStoreSize, bool IsWrite, Value *SizeArgument, bool UseCalls, - uint32_t Exp) { + uint32_t Exp, RuntimeCallInserter &RTCI) { // Instrument a 1-, 2-, 4-, 8-, or 16- byte access with one check // if the data is properly aligned. if (!TypeStoreSize.isScalable()) { @@ -1529,18 +1555,19 @@ static void doInstrumentAddress(AddressSanitizer *Pass, Instruction *I, *Alignment >= FixedSize / 8) return Pass->instrumentAddress(I, InsertBefore, Addr, Alignment, FixedSize, IsWrite, nullptr, UseCalls, - Exp); + Exp, RTCI); } } Pass->instrumentUnusualSizeOrAlignment(I, InsertBefore, Addr, TypeStoreSize, - IsWrite, nullptr, UseCalls, Exp); + IsWrite, nullptr, UseCalls, Exp, RTCI); } void AddressSanitizer::instrumentMaskedLoadOrStore( AddressSanitizer *Pass, const DataLayout &DL, Type *IntptrTy, Value *Mask, Value *EVL, Value *Stride, Instruction *I, Value *Addr, MaybeAlign Alignment, unsigned Granularity, Type *OpType, bool IsWrite, - Value *SizeArgument, bool UseCalls, uint32_t Exp) { + Value *SizeArgument, bool UseCalls, uint32_t Exp, + RuntimeCallInserter &RTCI) { auto *VTy = cast(OpType); TypeSize ElemTypeSize = DL.getTypeStoreSizeInBits(VTy->getScalarType()); auto Zero = ConstantInt::get(IntptrTy, 0); @@ -1595,15 +1622,16 @@ void AddressSanitizer::instrumentMaskedLoadOrStore( } else { InstrumentedAddress = IRB.CreateGEP(VTy, Addr, {Zero, Index}); } - doInstrumentAddress(Pass, I, &*IRB.GetInsertPoint(), - InstrumentedAddress, Alignment, Granularity, - ElemTypeSize, IsWrite, SizeArgument, UseCalls, Exp); + doInstrumentAddress(Pass, I, &*IRB.GetInsertPoint(), InstrumentedAddress, + Alignment, Granularity, ElemTypeSize, IsWrite, + SizeArgument, UseCalls, Exp, RTCI); }); } void AddressSanitizer::instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis, InterestingMemoryOperand &O, bool UseCalls, - const DataLayout &DL) { + const DataLayout &DL, + RuntimeCallInserter &RTCI) { Value *Addr = O.getPtr(); // Optimization experiments. @@ -1649,11 +1677,11 @@ void AddressSanitizer::instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis, instrumentMaskedLoadOrStore(this, DL, IntptrTy, O.MaybeMask, O.MaybeEVL, O.MaybeStride, O.getInsn(), Addr, O.Alignment, Granularity, O.OpType, O.IsWrite, nullptr, - UseCalls, Exp); + UseCalls, Exp, RTCI); } else { doInstrumentAddress(this, O.getInsn(), O.getInsn(), Addr, O.Alignment, - Granularity, O.TypeStoreSize, O.IsWrite, nullptr, UseCalls, - Exp); + Granularity, O.TypeStoreSize, O.IsWrite, nullptr, + UseCalls, Exp, RTCI); } } @@ -1661,24 +1689,25 @@ Instruction *AddressSanitizer::generateCrashCode(Instruction *InsertBefore, Value *Addr, bool IsWrite, size_t AccessSizeIndex, Value *SizeArgument, - uint32_t Exp) { + uint32_t Exp, + RuntimeCallInserter &RTCI) { InstrumentationIRBuilder IRB(InsertBefore); Value *ExpVal = Exp == 0 ? nullptr : ConstantInt::get(IRB.getInt32Ty(), Exp); CallInst *Call = nullptr; if (SizeArgument) { if (Exp == 0) - Call = IRB.CreateCall(AsanErrorCallbackSized[IsWrite][0], - {Addr, SizeArgument}); + Call = RTCI.createRuntimeCall(IRB, AsanErrorCallbackSized[IsWrite][0], + {Addr, SizeArgument}); else - Call = IRB.CreateCall(AsanErrorCallbackSized[IsWrite][1], - {Addr, SizeArgument, ExpVal}); + Call = RTCI.createRuntimeCall(IRB, AsanErrorCallbackSized[IsWrite][1], + {Addr, SizeArgument, ExpVal}); } else { if (Exp == 0) - Call = - IRB.CreateCall(AsanErrorCallback[IsWrite][0][AccessSizeIndex], Addr); + Call = RTCI.createRuntimeCall( + IRB, AsanErrorCallback[IsWrite][0][AccessSizeIndex], Addr); else - Call = IRB.CreateCall(AsanErrorCallback[IsWrite][1][AccessSizeIndex], - {Addr, ExpVal}); + Call = RTCI.createRuntimeCall( + IRB, AsanErrorCallback[IsWrite][1][AccessSizeIndex], {Addr, ExpVal}); } Call->setCannotMerge(); @@ -1754,7 +1783,8 @@ void AddressSanitizer::instrumentAddress(Instruction *OrigIns, MaybeAlign Alignment, uint32_t TypeStoreSize, bool IsWrite, Value *SizeArgument, bool UseCalls, - uint32_t Exp) { + uint32_t Exp, + RuntimeCallInserter &RTCI) { if (TargetTriple.isAMDGPU()) { InsertBefore = instrumentAMDGPUAddress(OrigIns, InsertBefore, Addr, TypeStoreSize, IsWrite, SizeArgument); @@ -1779,11 +1809,12 @@ void AddressSanitizer::instrumentAddress(Instruction *OrigIns, Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy); if (UseCalls) { if (Exp == 0) - IRB.CreateCall(AsanMemoryAccessCallback[IsWrite][0][AccessSizeIndex], - AddrLong); + RTCI.createRuntimeCall( + IRB, AsanMemoryAccessCallback[IsWrite][0][AccessSizeIndex], AddrLong); else - IRB.CreateCall(AsanMemoryAccessCallback[IsWrite][1][AccessSizeIndex], - {AddrLong, ConstantInt::get(IRB.getInt32Ty(), Exp)}); + RTCI.createRuntimeCall( + IRB, AsanMemoryAccessCallback[IsWrite][1][AccessSizeIndex], + {AddrLong, ConstantInt::get(IRB.getInt32Ty(), Exp)}); return; } @@ -1830,8 +1861,8 @@ void AddressSanitizer::instrumentAddress(Instruction *OrigIns, CrashTerm = SplitBlockAndInsertIfThen(Cmp, InsertBefore, !Recover); } - Instruction *Crash = generateCrashCode(CrashTerm, AddrLong, IsWrite, - AccessSizeIndex, SizeArgument, Exp); + Instruction *Crash = generateCrashCode( + CrashTerm, AddrLong, IsWrite, AccessSizeIndex, SizeArgument, Exp, RTCI); if (OrigIns->getDebugLoc()) Crash->setDebugLoc(OrigIns->getDebugLoc()); } @@ -1841,8 +1872,9 @@ void AddressSanitizer::instrumentAddress(Instruction *OrigIns, // and the last bytes. We call __asan_report_*_n(addr, real_size) to be able // to report the actual access size. void AddressSanitizer::instrumentUnusualSizeOrAlignment( - Instruction *I, Instruction *InsertBefore, Value *Addr, TypeSize TypeStoreSize, - bool IsWrite, Value *SizeArgument, bool UseCalls, uint32_t Exp) { + Instruction *I, Instruction *InsertBefore, Value *Addr, + TypeSize TypeStoreSize, bool IsWrite, Value *SizeArgument, bool UseCalls, + uint32_t Exp, RuntimeCallInserter &RTCI) { InstrumentationIRBuilder IRB(InsertBefore); Value *NumBits = IRB.CreateTypeSize(IntptrTy, TypeStoreSize); Value *Size = IRB.CreateLShr(NumBits, ConstantInt::get(IntptrTy, 3)); @@ -1850,19 +1882,21 @@ void AddressSanitizer::instrumentUnusualSizeOrAlignment( Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy); if (UseCalls) { if (Exp == 0) - IRB.CreateCall(AsanMemoryAccessCallbackSized[IsWrite][0], - {AddrLong, Size}); + RTCI.createRuntimeCall(IRB, AsanMemoryAccessCallbackSized[IsWrite][0], + {AddrLong, Size}); else - IRB.CreateCall(AsanMemoryAccessCallbackSized[IsWrite][1], - {AddrLong, Size, ConstantInt::get(IRB.getInt32Ty(), Exp)}); + RTCI.createRuntimeCall( + IRB, AsanMemoryAccessCallbackSized[IsWrite][1], + {AddrLong, Size, ConstantInt::get(IRB.getInt32Ty(), Exp)}); } else { Value *SizeMinusOne = IRB.CreateSub(Size, ConstantInt::get(IntptrTy, 1)); Value *LastByte = IRB.CreateIntToPtr( IRB.CreateAdd(AddrLong, SizeMinusOne), Addr->getType()); - instrumentAddress(I, InsertBefore, Addr, {}, 8, IsWrite, Size, false, Exp); + instrumentAddress(I, InsertBefore, Addr, {}, 8, IsWrite, Size, false, Exp, + RTCI); instrumentAddress(I, InsertBefore, LastByte, {}, 8, IsWrite, Size, false, - Exp); + Exp, RTCI); } } @@ -2881,6 +2915,8 @@ bool AddressSanitizer::instrumentFunction(Function &F, FunctionStateRAII CleanupObj(this); + RuntimeCallInserter RTCI(F); + FunctionModified |= maybeInsertDynamicShadowAtFunctionEntry(F); // We can't instrument allocas used with llvm.localescape. Only static allocas @@ -2963,27 +2999,27 @@ bool AddressSanitizer::instrumentFunction(Function &F, for (auto &Operand : OperandsToInstrument) { if (!suppressInstrumentationSiteForDebug(NumInstrumented)) instrumentMop(ObjSizeVis, Operand, UseCalls, - F.getParent()->getDataLayout()); + F.getParent()->getDataLayout(), RTCI); FunctionModified = true; } for (auto *Inst : IntrinToInstrument) { if (!suppressInstrumentationSiteForDebug(NumInstrumented)) - instrumentMemIntrinsic(Inst); + instrumentMemIntrinsic(Inst, RTCI); FunctionModified = true; } - FunctionStackPoisoner FSP(F, *this); + FunctionStackPoisoner FSP(F, *this, RTCI); bool ChangedStack = FSP.runOnFunction(); // We must unpoison the stack before NoReturn calls (throw, _exit, etc). // See e.g. https://github.com/google/sanitizers/issues/37 for (auto *CI : NoReturnCalls) { IRBuilder<> IRB(CI); - IRB.CreateCall(AsanHandleNoReturnFunc, {}); + RTCI.createRuntimeCall(IRB, AsanHandleNoReturnFunc, {}); } for (auto *Inst : PointerComparisonsOrSubtracts) { - instrumentPointerComparisonOrSubtraction(Inst); + instrumentPointerComparisonOrSubtraction(Inst, RTCI); FunctionModified = true; } @@ -3128,9 +3164,10 @@ void FunctionStackPoisoner::copyToShadow(ArrayRef ShadowMask, if (j - i >= ASan.MaxInlinePoisoningSize) { copyToShadowInline(ShadowMask, ShadowBytes, Done, i, IRB, ShadowBase); - IRB.CreateCall(AsanSetShadowFunc[Val], - {IRB.CreateAdd(ShadowBase, ConstantInt::get(IntptrTy, i)), - ConstantInt::get(IntptrTy, j - i)}); + RTCI.createRuntimeCall( + IRB, AsanSetShadowFunc[Val], + {IRB.CreateAdd(ShadowBase, ConstantInt::get(IntptrTy, i)), + ConstantInt::get(IntptrTy, j - i)}); Done = j; } } @@ -3417,8 +3454,8 @@ void FunctionStackPoisoner::processStaticAllocas() { StackMallocIdx = StackMallocSizeClass(LocalStackSize); assert(StackMallocIdx <= kMaxAsanStackMallocSizeClass); Value *FakeStackValue = - IRBIf.CreateCall(AsanStackMallocFunc[StackMallocIdx], - ConstantInt::get(IntptrTy, LocalStackSize)); + RTCI.createRuntimeCall(IRBIf, AsanStackMallocFunc[StackMallocIdx], + ConstantInt::get(IntptrTy, LocalStackSize)); IRB.SetInsertPoint(InsBefore); FakeStack = createPHI(IRB, UseAfterReturnIsEnabled, FakeStackValue, Term, ConstantInt::get(IntptrTy, 0)); @@ -3428,7 +3465,8 @@ void FunctionStackPoisoner::processStaticAllocas() { // void *LocalStackBase = (FakeStack) ? FakeStack : // alloca(LocalStackSize); StackMallocIdx = StackMallocSizeClass(LocalStackSize); - FakeStack = IRB.CreateCall(AsanStackMallocFunc[StackMallocIdx], + FakeStack = + RTCI.createRuntimeCall(IRB, AsanStackMallocFunc[StackMallocIdx], ConstantInt::get(IntptrTy, LocalStackSize)); } Value *NoFakeStack = @@ -3563,8 +3601,8 @@ void FunctionStackPoisoner::processStaticAllocas() { IRBPoison.CreateIntToPtr(SavedFlagPtr, IRBPoison.getPtrTy())); } else { // For larger frames call __asan_stack_free_*. - IRBPoison.CreateCall( - AsanStackFreeFunc[StackMallocIdx], + RTCI.createRuntimeCall( + IRBPoison, AsanStackFreeFunc[StackMallocIdx], {FakeStack, ConstantInt::get(IntptrTy, LocalStackSize)}); } @@ -3585,8 +3623,8 @@ void FunctionStackPoisoner::poisonAlloca(Value *V, uint64_t Size, // For now just insert the call to ASan runtime. Value *AddrArg = IRB.CreatePointerCast(V, IntptrTy); Value *SizeArg = ConstantInt::get(IntptrTy, Size); - IRB.CreateCall( - DoPoison ? AsanPoisonStackMemoryFunc : AsanUnpoisonStackMemoryFunc, + RTCI.createRuntimeCall( + IRB, DoPoison ? AsanPoisonStackMemoryFunc : AsanUnpoisonStackMemoryFunc, {AddrArg, SizeArg}); } @@ -3647,7 +3685,7 @@ void FunctionStackPoisoner::handleDynamicAllocaCall(AllocaInst *AI) { ConstantInt::get(IntptrTy, Alignment.value())); // Insert __asan_alloca_poison call for new created alloca. - IRB.CreateCall(AsanAllocaPoisonFunc, {NewAddress, OldSize}); + RTCI.createRuntimeCall(IRB, AsanAllocaPoisonFunc, {NewAddress, OldSize}); // Store the last alloca's address to DynamicAllocaLayout. We'll need this // for unpoisoning stuff. -- GitLab From 8aed911fe91bb6cbfb95789683dadf3e77ea713a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Thu, 7 Mar 2024 15:31:40 +0100 Subject: [PATCH 454/929] [clang][Interp] Implement complex comparisons --- clang/lib/AST/Interp/ByteCodeExprGen.cpp | 106 ++++++++++++++++++++++- clang/lib/AST/Interp/ByteCodeExprGen.h | 2 + clang/test/AST/Interp/complex.c | 8 +- clang/test/AST/Interp/complex.cpp | 47 ++++++++++ 4 files changed, 157 insertions(+), 6 deletions(-) diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp index d887170cbc5d..8872579e12dc 100644 --- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp +++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp @@ -393,12 +393,16 @@ bool ByteCodeExprGen::VisitBinaryOperator(const BinaryOperator *BO) { if (BO->isLogicalOp()) return this->VisitLogicalBinOp(BO); - if (BO->getType()->isAnyComplexType()) - return this->VisitComplexBinOp(BO); - const Expr *LHS = BO->getLHS(); const Expr *RHS = BO->getRHS(); + if (BO->getType()->isAnyComplexType()) + return this->VisitComplexBinOp(BO); + if ((LHS->getType()->isAnyComplexType() || + RHS->getType()->isAnyComplexType()) && + BO->isComparisonOp()) + return this->emitComplexComparison(LHS, RHS, BO); + if (BO->isPtrMemOp()) return this->visit(RHS); @@ -3410,6 +3414,102 @@ bool ByteCodeExprGen::emitComplexBoolCast(const Expr *E) { return true; } +template +bool ByteCodeExprGen::emitComplexComparison(const Expr *LHS, + const Expr *RHS, + const BinaryOperator *E) { + assert(E->isComparisonOp()); + assert(!Initializing); + assert(!DiscardResult); + + PrimType ElemT; + bool LHSIsComplex; + unsigned LHSOffset; + if (LHS->getType()->isAnyComplexType()) { + LHSIsComplex = true; + ElemT = classifyComplexElementType(LHS->getType()); + LHSOffset = allocateLocalPrimitive(LHS, PT_Ptr, /*IsConst=*/true, + /*IsExtended=*/false); + if (!this->visit(LHS)) + return false; + if (!this->emitSetLocal(PT_Ptr, LHSOffset, E)) + return false; + } else { + LHSIsComplex = false; + PrimType LHST = classifyPrim(LHS->getType()); + LHSOffset = this->allocateLocalPrimitive(LHS, LHST, true, false); + if (!this->visit(LHS)) + return false; + if (!this->emitSetLocal(LHST, LHSOffset, E)) + return false; + } + + bool RHSIsComplex; + unsigned RHSOffset; + if (RHS->getType()->isAnyComplexType()) { + RHSIsComplex = true; + ElemT = classifyComplexElementType(RHS->getType()); + RHSOffset = allocateLocalPrimitive(RHS, PT_Ptr, /*IsConst=*/true, + /*IsExtended=*/false); + if (!this->visit(RHS)) + return false; + if (!this->emitSetLocal(PT_Ptr, RHSOffset, E)) + return false; + } else { + RHSIsComplex = false; + PrimType RHST = classifyPrim(RHS->getType()); + RHSOffset = this->allocateLocalPrimitive(RHS, RHST, true, false); + if (!this->visit(RHS)) + return false; + if (!this->emitSetLocal(RHST, RHSOffset, E)) + return false; + } + + auto getElem = [&](unsigned LocalOffset, unsigned Index, + bool IsComplex) -> bool { + if (IsComplex) { + if (!this->emitGetLocal(PT_Ptr, LocalOffset, E)) + return false; + return this->emitArrayElemPop(ElemT, Index, E); + } + return this->emitGetLocal(ElemT, LocalOffset, E); + }; + + for (unsigned I = 0; I != 2; ++I) { + // Get both values. + if (!getElem(LHSOffset, I, LHSIsComplex)) + return false; + if (!getElem(RHSOffset, I, RHSIsComplex)) + return false; + // And compare them. + if (!this->emitEQ(ElemT, E)) + return false; + + if (!this->emitCastBoolUint8(E)) + return false; + } + + // We now have two bool values on the stack. Compare those. + if (!this->emitAddUint8(E)) + return false; + if (!this->emitConstUint8(2, E)) + return false; + + if (E->getOpcode() == BO_EQ) { + if (!this->emitEQUint8(E)) + return false; + } else if (E->getOpcode() == BO_NE) { + if (!this->emitNEUint8(E)) + return false; + } else + return false; + + // In C, this returns an int. + if (PrimType ResT = classifyPrim(E->getType()); ResT != PT_Bool) + return this->emitCast(PT_Bool, ResT, E); + return true; +} + /// When calling this, we have a pointer of the local-to-destroy /// on the stack. /// Emit destruction of record types (or arrays of record types). diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.h b/clang/lib/AST/Interp/ByteCodeExprGen.h index acbbcc3dc961..5977bb5e6ff2 100644 --- a/clang/lib/AST/Interp/ByteCodeExprGen.h +++ b/clang/lib/AST/Interp/ByteCodeExprGen.h @@ -268,6 +268,8 @@ private: bool emitComplexReal(const Expr *SubExpr); bool emitComplexBoolCast(const Expr *E); + bool emitComplexComparison(const Expr *LHS, const Expr *RHS, + const BinaryOperator *E); bool emitRecordDestruction(const Record *R); bool emitDestruction(const Descriptor *Desc); diff --git a/clang/test/AST/Interp/complex.c b/clang/test/AST/Interp/complex.c index b07d0241da12..c9c2efb59745 100644 --- a/clang/test/AST/Interp/complex.c +++ b/clang/test/AST/Interp/complex.c @@ -1,9 +1,6 @@ // RUN: %clang_cc1 -fexperimental-new-constant-interpreter -verify=expected,both -Wno-unused-value %s // RUN: %clang_cc1 -verify=ref,both -Wno-unused-value %s -// expected-no-diagnostics -// ref-no-diagnostics - void blah() { __complex__ unsigned xx; __complex__ signed yy; @@ -12,3 +9,8 @@ void blah() { /// The following line calls into the constant interpreter. result = xx * yy; } + + +_Static_assert((0.0 + 0.0j) == (0.0 + 0.0j), ""); +_Static_assert((0.0 + 0.0j) != (0.0 + 0.0j), ""); // both-error {{static assertion}} \ + // both-note {{evaluates to}} diff --git a/clang/test/AST/Interp/complex.cpp b/clang/test/AST/Interp/complex.cpp index 8acce7b734d8..6a42afc68d26 100644 --- a/clang/test/AST/Interp/complex.cpp +++ b/clang/test/AST/Interp/complex.cpp @@ -266,3 +266,50 @@ namespace Builtin { constexpr _Complex float C = __builtin_complex(10.0f, 20.0); // both-error {{arguments are of different types}} } + +namespace Cmp { + static_assert((0.0 + 0.0j) == (0.0 + 0.0j)); + static_assert((0.0 + 0.0j) != (0.0 + 0.0j)); // both-error {{static assertion}} \ + // both-note {{evaluates to}} + + static_assert((0.0 + 0.0j) == 0.0); + static_assert(0.0 == (0.0 + 0.0j)); + static_assert(0.0 == 0.0j); + static_assert((0.0 + 1.0j) != 0.0); + static_assert(1.0 != (0.0 + 0.0j)); + static_assert(0.0 != 1.0j); + + // Walk around the complex plane stepping between angular differences and + // equality. + static_assert((1.0 + 0.0j) == (0.0 + 0.0j)); // both-error {{static assertion}} \ + // both-note {{evaluates to}} + static_assert((1.0 + 0.0j) == (1.0 + 0.0j)); + static_assert((1.0 + 1.0j) == (1.0 + 0.0j)); // both-error {{static assertion}} \ + // both-note {{evaluates to}} + static_assert((1.0 + 1.0j) == (1.0 + 1.0j)); + static_assert((0.0 + 1.0j) == (1.0 + 1.0j)); // both-error {{static assertion}} \ + // both-note {{evaluates to}} + static_assert((0.0 + 1.0j) == (0.0 + 1.0j)); + static_assert((-1.0 + 1.0j) == (0.0 + 1.0j)); // both-error {{static assertion}} \ + // both-note {{evaluates to}} + static_assert((-1.0 + 1.0j) == (-1.0 + 1.0j)); + static_assert((-1.0 + 0.0j) == (-1.0 + 1.0j)); // both-error {{static assertion}} \ + // both-note {{evaluates to}} + static_assert((-1.0 + 0.0j) == (-1.0 + 0.0j)); + static_assert((-1.0 - 1.0j) == (-1.0 + 0.0j)); // both-error {{static assertion}} \ + // both-note {{evaluates to}} + static_assert((-1.0 - 1.0j) == (-1.0 - 1.0j)); + static_assert((0.0 - 1.0j) == (-1.0 - 1.0j)); // both-error {{static assertion}} \ + // both-note {{evaluates to}} + static_assert((0.0 - 1.0j) == (0.0 - 1.0j)); + static_assert((1.0 - 1.0j) == (0.0 - 1.0j)); // both-error {{static assertion}} \ + // both-note {{evaluates to}} + static_assert((1.0 - 1.0j) == (1.0 - 1.0j)); + + /// Make sure these are rejected before reaching the constexpr interpreter. + static_assert((0.0 + 0.0j) & (0.0 + 0.0j)); // both-error {{invalid operands to binary expression}} + static_assert((0.0 + 0.0j) | (0.0 + 0.0j)); // both-error {{invalid operands to binary expression}} + static_assert((0.0 + 0.0j) < (0.0 + 0.0j)); // both-error {{invalid operands to binary expression}} + static_assert((0.0 + 0.0j) > (0.0 + 0.0j)); // both-error {{invalid operands to binary expression}} + static_assert((0.0 + 0.0j) ^ (0.0 + 0.0j)); // both-error {{invalid operands to binary expression}} +} -- GitLab From 5d59fa90ce225814739d9b51ba37e1cca9204cad Mon Sep 17 00:00:00 2001 From: "Oleksandr \"Alex\" Zinenko" Date: Thu, 7 Mar 2024 17:14:08 +0100 Subject: [PATCH 455/929] Reapply "[mlir][py] better support for arith.constant construction" (#84142) Arithmetic constants for vector types can be constructed from objects implementing Python buffer protocol such as `array.array`. Note that until Python 3.12, there is no typing support for buffer protocol implementers, so the annotations use array explicitly. Reverts llvm/llvm-project#84103 --- mlir/python/mlir/dialects/arith.py | 30 ++++++++++++++-- mlir/test/python/dialects/arith_dialect.py | 40 ++++++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/mlir/python/mlir/dialects/arith.py b/mlir/python/mlir/dialects/arith.py index 61c6917393f1..92da5df9bce6 100644 --- a/mlir/python/mlir/dialects/arith.py +++ b/mlir/python/mlir/dialects/arith.py @@ -5,6 +5,8 @@ from ._arith_ops_gen import * from ._arith_ops_gen import _Dialect from ._arith_enum_gen import * +from array import array as _array +from typing import overload try: from ..ir import * @@ -43,13 +45,37 @@ def _is_float_type(type: Type): class ConstantOp(ConstantOp): """Specialization for the constant op class.""" + @overload + def __init__(self, value: Attribute, *, loc=None, ip=None): + ... + + @overload def __init__( - self, result: Type, value: Union[int, float, Attribute], *, loc=None, ip=None + self, result: Type, value: Union[int, float, _array], *, loc=None, ip=None ): + ... + + def __init__(self, result, value, *, loc=None, ip=None): + if value is None: + assert isinstance(result, Attribute) + super().__init__(result, loc=loc, ip=ip) + return + if isinstance(value, int): super().__init__(IntegerAttr.get(result, value), loc=loc, ip=ip) elif isinstance(value, float): super().__init__(FloatAttr.get(result, value), loc=loc, ip=ip) + elif isinstance(value, _array): + if 8 * value.itemsize != result.element_type.width: + raise ValueError( + f"Mismatching array element ({8 * value.itemsize}) and type ({result.element_type.width}) width." + ) + if value.typecode in ["i", "l", "q"]: + super().__init__(DenseIntElementsAttr.get(value, type=result)) + elif value.typecode in ["f", "d"]: + super().__init__(DenseFPElementsAttr.get(value, type=result)) + else: + raise ValueError(f'Unsupported typecode: "{value.typecode}".') else: super().__init__(value, loc=loc, ip=ip) @@ -79,6 +105,6 @@ class ConstantOp(ConstantOp): def constant( - result: Type, value: Union[int, float, Attribute], *, loc=None, ip=None + result: Type, value: Union[int, float, Attribute, _array], *, loc=None, ip=None ) -> Value: return _get_op_result_or_op_results(ConstantOp(result, value, loc=loc, ip=ip)) diff --git a/mlir/test/python/dialects/arith_dialect.py b/mlir/test/python/dialects/arith_dialect.py index 8bb80eed2b81..c9af5e7b46db 100644 --- a/mlir/test/python/dialects/arith_dialect.py +++ b/mlir/test/python/dialects/arith_dialect.py @@ -4,6 +4,7 @@ from functools import partialmethod from mlir.ir import * import mlir.dialects.arith as arith import mlir.dialects.func as func +from array import array def run(f): @@ -92,3 +93,42 @@ def testArithValue(): b = a * a # CHECK: ArithValue(%2 = arith.mulf %cst_1, %cst_1 : f64) print(b) + + +# CHECK-LABEL: TEST: testArrayConstantConstruction +@run +def testArrayConstantConstruction(): + with Context(), Location.unknown(): + module = Module.create() + with InsertionPoint(module.body): + i32_array = array("i", [1, 2, 3, 4]) + i32 = IntegerType.get_signless(32) + vec_i32 = VectorType.get([2, 2], i32) + arith.constant(vec_i32, i32_array) + arith.ConstantOp(vec_i32, DenseIntElementsAttr.get(i32_array, type=vec_i32)) + + # "q" is the equivalent of `long long` in C and requires at least + # 64 bit width integers on both Linux and Windows. + i64_array = array("q", [5, 6, 7, 8]) + i64 = IntegerType.get_signless(64) + vec_i64 = VectorType.get([1, 4], i64) + arith.constant(vec_i64, i64_array) + arith.ConstantOp(vec_i64, DenseIntElementsAttr.get(i64_array, type=vec_i64)) + + f32_array = array("f", [1.0, 2.0, 3.0, 4.0]) + f32 = F32Type.get() + vec_f32 = VectorType.get([4, 1], f32) + arith.constant(vec_f32, f32_array) + arith.ConstantOp(vec_f32, DenseFPElementsAttr.get(f32_array, type=vec_f32)) + + f64_array = array("d", [1.0, 2.0, 3.0, 4.0]) + f64 = F64Type.get() + vec_f64 = VectorType.get([2, 1, 2], f64) + arith.constant(vec_f64, f64_array) + arith.ConstantOp(vec_f64, DenseFPElementsAttr.get(f64_array, type=vec_f64)) + + # CHECK-COUNT-2: arith.constant dense<[{{\[}}1, 2], [3, 4]]> : vector<2x2xi32> + # CHECK-COUNT-2: arith.constant dense<[{{\[}}5, 6, 7, 8]]> : vector<1x4xi64> + # CHECK-COUNT-2: arith.constant dense<[{{\[}}1.000000e+00], [2.000000e+00], [3.000000e+00], [4.000000e+00]]> : vector<4x1xf32> + # CHECK-COUNT-2: arith.constant dense<[{{\[}}[1.000000e+00, 2.000000e+00]], [{{\[}}3.000000e+00, 4.000000e+00]]]> : vector<2x1x2xf64> + print(module) -- GitLab From c03fd37d9b61bc6063e4d6e983846f877e83ac67 Mon Sep 17 00:00:00 2001 From: Anchu Rajendran S Date: Thu, 7 Mar 2024 08:23:58 -0800 Subject: [PATCH 456/929] [flang] Changes to map variables in link clause of declare target (#83643) As per the OpenMP standard, "If a variable appears in a link clause on a declare target directive that does not have a device_type clause with the nohost device-type-description then it is treated as if it had appeared in a map clause with a map-type of tofrom" is an implicit mapping rule. Before this change, such variables were mapped as to by default. --- flang/lib/Lower/OpenMP/OpenMP.cpp | 16 +++- .../OpenMP/declare-target-link-tarop-cap.f90 | 55 +++++++++++++ .../declare-target-array-in-target-region.f90 | 33 -------- .../declare-target-vars-in-target-region.f90 | 81 +++++++++++++++++++ 4 files changed, 151 insertions(+), 34 deletions(-) create mode 100644 flang/test/Lower/OpenMP/declare-target-link-tarop-cap.f90 delete mode 100644 openmp/libomptarget/test/offloading/fortran/declare-target-array-in-target-region.f90 create mode 100644 openmp/libomptarget/test/offloading/fortran/declare-target-vars-in-target-region.f90 diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp index 185e0316870e..5cff95c7d125 100644 --- a/flang/lib/Lower/OpenMP/OpenMP.cpp +++ b/flang/lib/Lower/OpenMP/OpenMP.cpp @@ -1120,7 +1120,21 @@ genTargetOp(Fortran::lower::AbstractConverter &converter, if (auto refType = baseOp.getType().dyn_cast()) eleType = refType.getElementType(); - if (fir::isa_trivial(eleType) || fir::isa_char(eleType)) { + // If a variable is specified in declare target link and if device + // type is not specified as `nohost`, it needs to be mapped tofrom + mlir::ModuleOp mod = converter.getFirOpBuilder().getModule(); + mlir::Operation *op = mod.lookupSymbol(converter.mangleName(sym)); + auto declareTargetOp = + llvm::dyn_cast_if_present(op); + if (declareTargetOp && declareTargetOp.isDeclareTarget()) { + if (declareTargetOp.getDeclareTargetCaptureClause() == + mlir::omp::DeclareTargetCaptureClause::link && + declareTargetOp.getDeclareTargetDeviceType() != + mlir::omp::DeclareTargetDeviceType::nohost) { + mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO; + mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM; + } + } else if (fir::isa_trivial(eleType) || fir::isa_char(eleType)) { captureKind = mlir::omp::VariableCaptureKind::ByCopy; } else if (!fir::isa_builtin_cptr_type(eleType)) { mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO; diff --git a/flang/test/Lower/OpenMP/declare-target-link-tarop-cap.f90 b/flang/test/Lower/OpenMP/declare-target-link-tarop-cap.f90 new file mode 100644 index 000000000000..7cd059716157 --- /dev/null +++ b/flang/test/Lower/OpenMP/declare-target-link-tarop-cap.f90 @@ -0,0 +1,55 @@ +!RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s +!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-is-device %s -o - | FileCheck %s +!RUN: bbc -emit-hlfir -fopenmp %s -o - | FileCheck %s +!RUN: bbc -emit-hlfir -fopenmp -fopenmp-is-target-device %s -o - | FileCheck %s + +program test_link + + integer :: test_int = 1 + !$omp declare target link(test_int) + + integer :: test_array_1d(3) = (/1,2,3/) + !$omp declare target link(test_array_1d) + + integer, pointer :: test_ptr1 + !$omp declare target link(test_ptr1) + + integer, target :: test_target = 1 + !$omp declare target link(test_target) + + integer, pointer :: test_ptr2 + !$omp declare target link(test_ptr2) + + !CHECK-DAG: {{%.*}} = omp.map_info var_ptr({{%.*}} : !fir.ref, i32) map_clauses(implicit, tofrom) capture(ByRef) -> !fir.ref {name = "test_int"} + !$omp target + test_int = test_int + 1 + !$omp end target + + + !CHECK-DAG: {{%.*}} = omp.map_info var_ptr({{%.*}} : !fir.ref>, !fir.array<3xi32>) map_clauses(implicit, tofrom) capture(ByRef) bounds({{%.*}}) -> !fir.ref> {name = "test_array_1d"} + !$omp target + do i = 1,3 + test_array_1d(i) = i * 2 + end do + !$omp end target + + allocate(test_ptr1) + test_ptr1 = 1 + !CHECK-DAG: {{%.*}} = omp.map_info var_ptr({{%.*}} : !fir.ref>>, !fir.box>) map_clauses(implicit, tofrom) capture(ByRef) members({{%.*}} : !fir.llvm_ptr>) -> !fir.ref>> {name = "test_ptr1"} + !$omp target + test_ptr1 = test_ptr1 + 1 + !$omp end target + + !CHECK-DAG: {{%.*}} = omp.map_info var_ptr({{%.*}} : !fir.ref, i32) map_clauses(implicit, tofrom) capture(ByRef) -> !fir.ref {name = "test_target"} + !$omp target + test_target = test_target + 1 + !$omp end target + + + !CHECK-DAG: {{%.*}} = omp.map_info var_ptr({{%.*}} : !fir.ref>>, !fir.box>) map_clauses(implicit, tofrom) capture(ByRef) members({{%.*}} : !fir.llvm_ptr>) -> !fir.ref>> {name = "test_ptr2"} + test_ptr2 => test_target + !$omp target + test_ptr2 = test_ptr2 + 1 + !$omp end target + +end diff --git a/openmp/libomptarget/test/offloading/fortran/declare-target-array-in-target-region.f90 b/openmp/libomptarget/test/offloading/fortran/declare-target-array-in-target-region.f90 deleted file mode 100644 index c09146198768..000000000000 --- a/openmp/libomptarget/test/offloading/fortran/declare-target-array-in-target-region.f90 +++ /dev/null @@ -1,33 +0,0 @@ -! Offloading test with a target region mapping a declare target -! Fortran array writing some values to it and checking the host -! correctly receives the updates made on the device. -! REQUIRES: flang -! UNSUPPORTED: nvptx64-nvidia-cuda-LTO -! UNSUPPORTED: aarch64-unknown-linux-gnu -! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO -! UNSUPPORTED: x86_64-pc-linux-gnu -! UNSUPPORTED: x86_64-pc-linux-gnu-LTO - -! RUN: %libomptarget-compile-fortran-run-and-check-generic -module test_0 - implicit none - INTEGER :: sp(10) = (/0,0,0,0,0,0,0,0,0,0/) - !$omp declare target link(sp) -end module test_0 - -program main - use test_0 - integer :: i = 1 - integer :: j = 11 -!$omp target map(tofrom:sp, i, j) - do while (i <= j) - sp(i) = i; - i = i + 1 - end do -!$omp end target - -PRINT *, sp(:) - -end program - -! CHECK: 1 2 3 4 5 6 7 8 9 10 diff --git a/openmp/libomptarget/test/offloading/fortran/declare-target-vars-in-target-region.f90 b/openmp/libomptarget/test/offloading/fortran/declare-target-vars-in-target-region.f90 new file mode 100644 index 000000000000..f524deac3bcc --- /dev/null +++ b/openmp/libomptarget/test/offloading/fortran/declare-target-vars-in-target-region.f90 @@ -0,0 +1,81 @@ +! Offloading test with a target region mapping a declare target +! Fortran array writing some values to it and checking the host +! correctly receives the updates made on the device. +! REQUIRES: flang +! UNSUPPORTED: nvptx64-nvidia-cuda-LTO +! UNSUPPORTED: aarch64-unknown-linux-gnu +! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO +! UNSUPPORTED: x86_64-pc-linux-gnu +! UNSUPPORTED: x86_64-pc-linux-gnu-LTO + +! RUN: %libomptarget-compile-fortran-run-and-check-generic +module test_0 + implicit none + INTEGER :: arr1(10) = (/0,0,0,0,0,0,0,0,0,0/) + INTEGER :: arr2(10) = (/0,0,0,0,0,0,0,0,0,0/) + !$omp declare target link(arr1) enter(arr2) + INTEGER :: scalar = 1 + !$omp declare target link(scalar) +end module test_0 + +subroutine test_with_array_link_and_tofrom() + use test_0 + integer :: i = 1 + integer :: j = 11 + !$omp target map(tofrom:arr1, i, j) + do while (i <= j) + arr1(i) = i; + i = i + 1 + end do + !$omp end target + + ! CHECK: 1 2 3 4 5 6 7 8 9 10 + PRINT *, arr1(:) +end subroutine test_with_array_link_and_tofrom + +subroutine test_with_array_link_only() + use test_0 + integer :: i = 1 + integer :: j = 11 + !$omp target map(i, j) + do while (i <= j) + arr1(i) = i + 1; + i = i + 1 + end do + !$omp end target + + ! CHECK: 2 3 4 5 6 7 8 9 10 11 + PRINT *, arr1(:) +end subroutine test_with_array_link_only + +subroutine test_with_array_enter_only() + use test_0 + integer :: i = 1 + integer :: j = 11 + !$omp target map(i, j) + do while (i <= j) + arr2(i) = i + 1; + i = i + 1 + end do + !$omp end target + + ! CHECK: 0 0 0 0 0 0 0 0 0 0 + PRINT *, arr2(:) +end subroutine test_with_array_enter_only + +subroutine test_with_scalar_link_only() + use test_0 + !$omp target + scalar = 10 + !$omp end target + + ! CHECK: 10 + PRINT *, scalar +end subroutine test_with_scalar_link_only + +program main + call test_with_array_link_and_tofrom() + call test_with_array_link_only() + call test_with_array_enter_only() + call test_with_scalar_link_only() +end program -- GitLab From 904a6aedca422d43f4f893bb97b2990e86b909e4 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Thu, 7 Mar 2024 08:31:16 -0800 Subject: [PATCH 457/929] [SLP][NFC]Add lshr version of the test with casting, NFC. --- .../X86/reorder-possible-strided-node.ll | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reorder-possible-strided-node.ll b/llvm/test/Transforms/SLPVectorizer/X86/reorder-possible-strided-node.ll index b9ef17c49b75..6f5d3d3785e0 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/reorder-possible-strided-node.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/reorder-possible-strided-node.ll @@ -54,3 +54,57 @@ entry: store i32 %conv27, ptr getelementptr inbounds ([4 x i32], ptr null, i64 8, i64 3), align 4 ret void } + +define void @test1() { +; CHECK-LABEL: define void @test1( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[ARRAYIDX22:%.*]] = getelementptr i32, ptr null, i64 60 +; CHECK-NEXT: [[TMP0:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> getelementptr (i32, <4 x ptr> zeroinitializer, <4 x i64> ), i32 4, <4 x i1> , <4 x i32> poison) +; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[ARRAYIDX22]], align 4 +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> +; CHECK-NEXT: [[TMP3:%.*]] = mul <4 x i32> [[TMP2]], [[TMP0]] +; CHECK-NEXT: [[TMP4:%.*]] = sext <4 x i32> [[TMP3]] to <4 x i64> +; CHECK-NEXT: [[TMP5:%.*]] = lshr <4 x i64> [[TMP4]], zeroinitializer +; CHECK-NEXT: [[TMP6:%.*]] = trunc <4 x i64> [[TMP5]] to <4 x i32> +; CHECK-NEXT: store <4 x i32> [[TMP6]], ptr getelementptr inbounds ([4 x i32], ptr null, i64 8, i64 0), align 16 +; CHECK-NEXT: ret void +; +entry: + %arrayidx1 = getelementptr i32, ptr null, i64 1 + %0 = load i32, ptr %arrayidx1, align 4 + %arrayidx2 = getelementptr i32, ptr null, i64 63 + %1 = load i32, ptr %arrayidx2, align 4 + %mul = mul i32 %1, %0 + %conv = sext i32 %mul to i64 + %shr = lshr i64 %conv, 0 + %conv3 = trunc i64 %shr to i32 + store i32 %conv3, ptr getelementptr inbounds ([4 x i32], ptr null, i64 8, i64 0), align 16 + %arrayidx5 = getelementptr i32, ptr null, i64 33 + %2 = load i32, ptr %arrayidx5, align 4 + %arrayidx6 = getelementptr i32, ptr null, i64 62 + %3 = load i32, ptr %arrayidx6, align 4 + %mul7 = mul i32 %3, %2 + %conv8 = sext i32 %mul7 to i64 + %shr10 = lshr i64 %conv8, 0 + %conv11 = trunc i64 %shr10 to i32 + store i32 %conv11, ptr getelementptr inbounds ([4 x i32], ptr null, i64 8, i64 1), align 4 + %arrayidx13 = getelementptr i32, ptr null, i64 7 + %4 = load i32, ptr %arrayidx13, align 4 + %arrayidx14 = getelementptr i32, ptr null, i64 61 + %5 = load i32, ptr %arrayidx14, align 4 + %mul15 = mul i32 %5, %4 + %conv16 = sext i32 %mul15 to i64 + %shr18 = lshr i64 %conv16, 0 + %conv19 = trunc i64 %shr18 to i32 + store i32 %conv19, ptr getelementptr inbounds ([4 x i32], ptr null, i64 8, i64 2), align 8 + %6 = load i32, ptr null, align 4 + %arrayidx22 = getelementptr i32, ptr null, i64 60 + %7 = load i32, ptr %arrayidx22, align 4 + %mul23 = mul i32 %7, %6 + %conv24 = sext i32 %mul23 to i64 + %shr26 = lshr i64 %conv24, 0 + %conv27 = trunc i64 %shr26 to i32 + store i32 %conv27, ptr getelementptr inbounds ([4 x i32], ptr null, i64 8, i64 3), align 4 + ret void +} -- GitLab From 101a13df71734b06116846a3a39c0880eb33456d Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Thu, 7 Mar 2024 08:38:04 -0800 Subject: [PATCH 458/929] [libc][stdbit] implement stdc_bit_floor (C23) (#84233) --- libc/config/linux/x86_64/entrypoints.txt | 5 +++++ libc/docs/stdbit.rst | 12 +++++----- libc/include/llvm-libc-macros/stdbit-macros.h | 20 +++++++++++++++++ libc/spec/stdc.td | 10 +++++++-- libc/src/__support/CPP/bit.h | 2 +- libc/src/stdbit/CMakeLists.txt | 1 + libc/src/stdbit/stdc_bit_floor_uc.cpp | 20 +++++++++++++++++ libc/src/stdbit/stdc_bit_floor_uc.h | 18 +++++++++++++++ libc/src/stdbit/stdc_bit_floor_ui.cpp | 20 +++++++++++++++++ libc/src/stdbit/stdc_bit_floor_ui.h | 18 +++++++++++++++ libc/src/stdbit/stdc_bit_floor_ul.cpp | 20 +++++++++++++++++ libc/src/stdbit/stdc_bit_floor_ul.h | 18 +++++++++++++++ libc/src/stdbit/stdc_bit_floor_ull.cpp | 21 ++++++++++++++++++ libc/src/stdbit/stdc_bit_floor_ull.h | 18 +++++++++++++++ libc/src/stdbit/stdc_bit_floor_us.cpp | 20 +++++++++++++++++ libc/src/stdbit/stdc_bit_floor_us.h | 18 +++++++++++++++ libc/test/include/stdbit_test.cpp | 17 ++++++++++++++ libc/test/src/stdbit/CMakeLists.txt | 1 + .../src/stdbit/stdc_bit_floor_uc_test.cpp | 22 +++++++++++++++++++ .../src/stdbit/stdc_bit_floor_ui_test.cpp | 21 ++++++++++++++++++ .../src/stdbit/stdc_bit_floor_ul_test.cpp | 21 ++++++++++++++++++ .../src/stdbit/stdc_bit_floor_ull_test.cpp | 21 ++++++++++++++++++ .../src/stdbit/stdc_bit_floor_us_test.cpp | 22 +++++++++++++++++++ 23 files changed, 357 insertions(+), 9 deletions(-) create mode 100644 libc/src/stdbit/stdc_bit_floor_uc.cpp create mode 100644 libc/src/stdbit/stdc_bit_floor_uc.h create mode 100644 libc/src/stdbit/stdc_bit_floor_ui.cpp create mode 100644 libc/src/stdbit/stdc_bit_floor_ui.h create mode 100644 libc/src/stdbit/stdc_bit_floor_ul.cpp create mode 100644 libc/src/stdbit/stdc_bit_floor_ul.h create mode 100644 libc/src/stdbit/stdc_bit_floor_ull.cpp create mode 100644 libc/src/stdbit/stdc_bit_floor_ull.h create mode 100644 libc/src/stdbit/stdc_bit_floor_us.cpp create mode 100644 libc/src/stdbit/stdc_bit_floor_us.h create mode 100644 libc/test/src/stdbit/stdc_bit_floor_uc_test.cpp create mode 100644 libc/test/src/stdbit/stdc_bit_floor_ui_test.cpp create mode 100644 libc/test/src/stdbit/stdc_bit_floor_ul_test.cpp create mode 100644 libc/test/src/stdbit/stdc_bit_floor_ull_test.cpp create mode 100644 libc/test/src/stdbit/stdc_bit_floor_us_test.cpp diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index 705ec10960c4..bd2006ddb7e9 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -152,6 +152,11 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdbit.stdc_bit_width_ui libc.src.stdbit.stdc_bit_width_ul libc.src.stdbit.stdc_bit_width_ull + libc.src.stdbit.stdc_bit_floor_uc + libc.src.stdbit.stdc_bit_floor_us + libc.src.stdbit.stdc_bit_floor_ui + libc.src.stdbit.stdc_bit_floor_ul + libc.src.stdbit.stdc_bit_floor_ull # stdlib.h entrypoints libc.src.stdlib.abs diff --git a/libc/docs/stdbit.rst b/libc/docs/stdbit.rst index ccd1393ef5d9..3ec46cf8d8ff 100644 --- a/libc/docs/stdbit.rst +++ b/libc/docs/stdbit.rst @@ -91,11 +91,11 @@ stdc_bit_width_us |check| stdc_bit_width_ui |check| stdc_bit_width_ul |check| stdc_bit_width_ull |check| -stdc_bit_floor_uc -stdc_bit_floor_us -stdc_bit_floor_ui -stdc_bit_floor_ul -stdc_bit_floor_ull +stdc_bit_floor_uc |check| +stdc_bit_floor_us |check| +stdc_bit_floor_ui |check| +stdc_bit_floor_ul |check| +stdc_bit_floor_ull |check| stdc_bit_ceil_uc stdc_bit_ceil_us stdc_bit_ceil_ui @@ -126,7 +126,7 @@ stdc_count_zeros |check| stdc_count_ones |check| stdc_has_single_bit |check| stdc_bit_width |check| -stdc_bit_floor +stdc_bit_floor |check| stdc_bit_ceil ========================= ========= diff --git a/libc/include/llvm-libc-macros/stdbit-macros.h b/libc/include/llvm-libc-macros/stdbit-macros.h index 104418ca4856..5b51068f866b 100644 --- a/libc/include/llvm-libc-macros/stdbit-macros.h +++ b/libc/include/llvm-libc-macros/stdbit-macros.h @@ -181,6 +181,19 @@ inline unsigned stdc_bit_width(unsigned long x) { return stdc_bit_width_ul(x); } inline unsigned stdc_bit_width(unsigned long long x) { return stdc_bit_width_ull(x); } +inline unsigned char stdc_bit_floor(unsigned char x) { + return stdc_bit_floor_uc(x); +} +inline unsigned short stdc_bit_floor(unsigned short x) { + return stdc_bit_floor_us(x); +} +inline unsigned stdc_bit_floor(unsigned x) { return stdc_bit_floor_ui(x); } +inline unsigned long stdc_bit_floor(unsigned long x) { + return stdc_bit_floor_ul(x); +} +inline unsigned long long stdc_bit_floor(unsigned long long x) { + return stdc_bit_floor_ull(x); +} #else #define stdc_leading_zeros(x) \ _Generic((x), \ @@ -266,6 +279,13 @@ inline unsigned stdc_bit_width(unsigned long long x) { unsigned: stdc_bit_width_ui, \ unsigned long: stdc_bit_width_ul, \ unsigned long long: stdc_bit_width_ull)(x) +#define stdc_bit_floor(x) \ + _Generic((x), \ + unsigned char: stdc_bit_floor_ui, \ + unsigned short: stdc_bit_floor_us, \ + unsigned: stdc_bit_floor_ui, \ + unsigned long: stdc_bit_floor_ul, \ + unsigned long long: stdc_bit_floor_ull)(x) #endif // __cplusplus #endif // __LLVM_LIBC_MACROS_STDBIT_MACROS_H diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index cfebc60a0a9a..a3a856a4ec85 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -801,7 +801,8 @@ def StdC : StandardSpec<"stdc"> { Macro<"stdc_count_zeros">, Macro<"stdc_count_ones">, Macro<"stdc_has_single_bit">, - Macro<"std_bit_width"> + Macro<"std_bit_width">, + Macro<"std_bit_floor"> ], // Macros [], // Types [], // Enumerations @@ -860,7 +861,12 @@ def StdC : StandardSpec<"stdc"> { FunctionSpec<"stdc_bit_width_us", RetValSpec, [ArgSpec]>, FunctionSpec<"stdc_bit_width_ui", RetValSpec, [ArgSpec]>, FunctionSpec<"stdc_bit_width_ul", RetValSpec, [ArgSpec]>, - FunctionSpec<"stdc_bit_width_ull", RetValSpec, [ArgSpec]> + FunctionSpec<"stdc_bit_width_ull", RetValSpec, [ArgSpec]>, + FunctionSpec<"stdc_bit_floor_uc", RetValSpec, [ArgSpec]>, + FunctionSpec<"stdc_bit_floor_us", RetValSpec, [ArgSpec]>, + FunctionSpec<"stdc_bit_floor_ui", RetValSpec, [ArgSpec]>, + FunctionSpec<"stdc_bit_floor_ul", RetValSpec, [ArgSpec]>, + FunctionSpec<"stdc_bit_floor_ull", RetValSpec, [ArgSpec]> ] // Functions >; diff --git a/libc/src/__support/CPP/bit.h b/libc/src/__support/CPP/bit.h index bc2f595845a9..6b625b0c97a3 100644 --- a/libc/src/__support/CPP/bit.h +++ b/libc/src/__support/CPP/bit.h @@ -178,7 +178,7 @@ template bit_floor(T value) { if (!value) return 0; - return T(1) << (cpp::bit_width(value) - 1); + return static_cast(T(1) << (cpp::bit_width(value) - 1)); } /// Returns the smallest integral power of two no smaller than value if value is diff --git a/libc/src/stdbit/CMakeLists.txt b/libc/src/stdbit/CMakeLists.txt index f077baeee6d2..7ab4fee4454a 100644 --- a/libc/src/stdbit/CMakeLists.txt +++ b/libc/src/stdbit/CMakeLists.txt @@ -11,6 +11,7 @@ set(prefixes count_ones has_single_bit bit_width + bit_floor ) set(suffixes c s i l ll) foreach(prefix IN LISTS prefixes) diff --git a/libc/src/stdbit/stdc_bit_floor_uc.cpp b/libc/src/stdbit/stdc_bit_floor_uc.cpp new file mode 100644 index 000000000000..6cb04c9eb43e --- /dev/null +++ b/libc/src/stdbit/stdc_bit_floor_uc.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of stdc_bit_floor_uc -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdbit/stdc_bit_floor_uc.h" + +#include "src/__support/CPP/bit.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(unsigned char, stdc_bit_floor_uc, (unsigned char value)) { + return cpp::bit_floor(value); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_bit_floor_uc.h b/libc/src/stdbit/stdc_bit_floor_uc.h new file mode 100644 index 000000000000..d6f53c5f6997 --- /dev/null +++ b/libc/src/stdbit/stdc_bit_floor_uc.h @@ -0,0 +1,18 @@ +//===-- Implementation header for stdc_bit_floor_uc -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDBIT_STDC_BIT_FLOOR_UC_H +#define LLVM_LIBC_SRC_STDBIT_STDC_BIT_FLOOR_UC_H + +namespace LIBC_NAMESPACE { + +unsigned char stdc_bit_floor_uc(unsigned char value); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDBIT_STDC_BIT_FLOOR_UC_H diff --git a/libc/src/stdbit/stdc_bit_floor_ui.cpp b/libc/src/stdbit/stdc_bit_floor_ui.cpp new file mode 100644 index 000000000000..149b63f190cf --- /dev/null +++ b/libc/src/stdbit/stdc_bit_floor_ui.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of stdc_bit_floor_ui -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdbit/stdc_bit_floor_ui.h" + +#include "src/__support/CPP/bit.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(unsigned, stdc_bit_floor_ui, (unsigned value)) { + return cpp::bit_floor(value); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_bit_floor_ui.h b/libc/src/stdbit/stdc_bit_floor_ui.h new file mode 100644 index 000000000000..fcc606386f86 --- /dev/null +++ b/libc/src/stdbit/stdc_bit_floor_ui.h @@ -0,0 +1,18 @@ +//===-- Implementation header for stdc_bit_floor_ui -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDBIT_STDC_BIT_FLOOR_UI_H +#define LLVM_LIBC_SRC_STDBIT_STDC_BIT_FLOOR_UI_H + +namespace LIBC_NAMESPACE { + +unsigned stdc_bit_floor_ui(unsigned value); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDBIT_STDC_BIT_FLOOR_UI_H diff --git a/libc/src/stdbit/stdc_bit_floor_ul.cpp b/libc/src/stdbit/stdc_bit_floor_ul.cpp new file mode 100644 index 000000000000..a29a04454568 --- /dev/null +++ b/libc/src/stdbit/stdc_bit_floor_ul.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of stdc_bit_floor_ul -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdbit/stdc_bit_floor_ul.h" + +#include "src/__support/CPP/bit.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(unsigned long, stdc_bit_floor_ul, (unsigned long value)) { + return cpp::bit_floor(value); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_bit_floor_ul.h b/libc/src/stdbit/stdc_bit_floor_ul.h new file mode 100644 index 000000000000..08327aa60c90 --- /dev/null +++ b/libc/src/stdbit/stdc_bit_floor_ul.h @@ -0,0 +1,18 @@ +//===-- Implementation header for stdc_bit_floor_ul -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDBIT_STDC_BIT_FLOOR_UL_H +#define LLVM_LIBC_SRC_STDBIT_STDC_BIT_FLOOR_UL_H + +namespace LIBC_NAMESPACE { + +unsigned long stdc_bit_floor_ul(unsigned long value); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDBIT_STDC_BIT_FLOOR_UL_H diff --git a/libc/src/stdbit/stdc_bit_floor_ull.cpp b/libc/src/stdbit/stdc_bit_floor_ull.cpp new file mode 100644 index 000000000000..d1084b635732 --- /dev/null +++ b/libc/src/stdbit/stdc_bit_floor_ull.cpp @@ -0,0 +1,21 @@ +//===-- Implementation of stdc_bit_floor_ull ------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdbit/stdc_bit_floor_ull.h" + +#include "src/__support/CPP/bit.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(unsigned long long, stdc_bit_floor_ull, + (unsigned long long value)) { + return cpp::bit_floor(value); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_bit_floor_ull.h b/libc/src/stdbit/stdc_bit_floor_ull.h new file mode 100644 index 000000000000..8f360b23855a --- /dev/null +++ b/libc/src/stdbit/stdc_bit_floor_ull.h @@ -0,0 +1,18 @@ +//===-- Implementation header for stdc_bit_floor_ull ------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDBIT_STDC_BIT_FLOOR_ULL_H +#define LLVM_LIBC_SRC_STDBIT_STDC_BIT_FLOOR_ULL_H + +namespace LIBC_NAMESPACE { + +unsigned long long stdc_bit_floor_ull(unsigned long long value); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDBIT_STDC_BIT_FLOOR_ULL_H diff --git a/libc/src/stdbit/stdc_bit_floor_us.cpp b/libc/src/stdbit/stdc_bit_floor_us.cpp new file mode 100644 index 000000000000..d1357a980e3a --- /dev/null +++ b/libc/src/stdbit/stdc_bit_floor_us.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of stdc_bit_floor_us -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdbit/stdc_bit_floor_us.h" + +#include "src/__support/CPP/bit.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(unsigned short, stdc_bit_floor_us, (unsigned short value)) { + return cpp::bit_floor(value); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_bit_floor_us.h b/libc/src/stdbit/stdc_bit_floor_us.h new file mode 100644 index 000000000000..fcd0b9e3c549 --- /dev/null +++ b/libc/src/stdbit/stdc_bit_floor_us.h @@ -0,0 +1,18 @@ +//===-- Implementation header for stdc_bit_floor_us -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDBIT_STDC_BIT_FLOOR_US_H +#define LLVM_LIBC_SRC_STDBIT_STDC_BIT_FLOOR_US_H + +namespace LIBC_NAMESPACE { + +unsigned short stdc_bit_floor_us(unsigned short value); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDBIT_STDC_BIT_FLOOR_US_H diff --git a/libc/test/include/stdbit_test.cpp b/libc/test/include/stdbit_test.cpp index dfb7c97e3d9e..20820d52fbde 100644 --- a/libc/test/include/stdbit_test.cpp +++ b/libc/test/include/stdbit_test.cpp @@ -91,6 +91,13 @@ unsigned stdc_bit_width_us(unsigned short) noexcept { return 0x4BU; } unsigned stdc_bit_width_ui(unsigned) noexcept { return 0x4CU; } unsigned stdc_bit_width_ul(unsigned long) noexcept { return 0x4DU; } unsigned stdc_bit_width_ull(unsigned long long) noexcept { return 0x4EU; } +unsigned char stdc_bit_floor_uc(unsigned char) noexcept { return 0x5AU; } +unsigned short stdc_bit_floor_us(unsigned short) noexcept { return 0x5BU; } +unsigned stdc_bit_floor_ui(unsigned) noexcept { return 0x5CU; } +unsigned long stdc_bit_floor_ul(unsigned long) noexcept { return 0x5DU; } +unsigned long long stdc_bit_floor_ull(unsigned long long) noexcept { + return 0x5EU; +} } #include "include/llvm-libc-macros/stdbit-macros.h" @@ -190,3 +197,13 @@ TEST(LlvmLibcStdbitTest, TypeGenericMacroBitWidth) { EXPECT_EQ(stdc_bit_width(1UL), 0x4DU); EXPECT_EQ(stdc_bit_width(1ULL), 0x4EU); } + +TEST(LlvmLibcStdbitTest, TypeGenericMacroBitFloor) { + EXPECT_EQ(stdc_bit_floor(static_cast(0U)), + static_cast(0x5AU)); + EXPECT_EQ(stdc_bit_floor(static_cast(0U)), + static_cast(0x5BU)); + EXPECT_EQ(stdc_bit_floor(0U), 0x5CU); + EXPECT_EQ(stdc_bit_floor(0UL), 0x5DUL); + EXPECT_EQ(stdc_bit_floor(0ULL), 0x5EULL); +} diff --git a/libc/test/src/stdbit/CMakeLists.txt b/libc/test/src/stdbit/CMakeLists.txt index f7e17d732299..3aed56c0e923 100644 --- a/libc/test/src/stdbit/CMakeLists.txt +++ b/libc/test/src/stdbit/CMakeLists.txt @@ -13,6 +13,7 @@ set(prefixes count_ones has_single_bit bit_width + bit_floor ) set(suffixes c s i l ll) foreach(prefix IN LISTS prefixes) diff --git a/libc/test/src/stdbit/stdc_bit_floor_uc_test.cpp b/libc/test/src/stdbit/stdc_bit_floor_uc_test.cpp new file mode 100644 index 000000000000..254abd043d6e --- /dev/null +++ b/libc/test/src/stdbit/stdc_bit_floor_uc_test.cpp @@ -0,0 +1,22 @@ +//===-- Unittests for stdc_bit_floor_uc -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/CPP/limits.h" +#include "src/stdbit/stdc_bit_floor_uc.h" +#include "test/UnitTest/Test.h" + +TEST(LlvmLibcStdcBitfloorUcTest, Zero) { + EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_floor_uc(0U), + static_cast(0)); +} + +TEST(LlvmLibcStdcBitfloorUcTest, Ones) { + for (unsigned i = 0U; i != UCHAR_WIDTH; ++i) + EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_floor_uc(UCHAR_MAX >> i), + static_cast(1 << (UCHAR_WIDTH - i - 1))); +} diff --git a/libc/test/src/stdbit/stdc_bit_floor_ui_test.cpp b/libc/test/src/stdbit/stdc_bit_floor_ui_test.cpp new file mode 100644 index 000000000000..53790402a9bd --- /dev/null +++ b/libc/test/src/stdbit/stdc_bit_floor_ui_test.cpp @@ -0,0 +1,21 @@ +//===-- Unittests for stdc_bit_floor_ui -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/CPP/limits.h" +#include "src/stdbit/stdc_bit_floor_ui.h" +#include "test/UnitTest/Test.h" + +TEST(LlvmLibcStdcBitfloorUiTest, Zero) { + EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_floor_ui(0U), 0U); +} + +TEST(LlvmLibcStdcBitfloorUiTest, Ones) { + for (unsigned i = 0U; i != INT_WIDTH; ++i) + EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_floor_ui(UINT_MAX >> i), + 1U << (UINT_WIDTH - i - 1)); +} diff --git a/libc/test/src/stdbit/stdc_bit_floor_ul_test.cpp b/libc/test/src/stdbit/stdc_bit_floor_ul_test.cpp new file mode 100644 index 000000000000..1c574437e02b --- /dev/null +++ b/libc/test/src/stdbit/stdc_bit_floor_ul_test.cpp @@ -0,0 +1,21 @@ +//===-- Unittests for stdc_bit_floor_ul -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/CPP/limits.h" +#include "src/stdbit/stdc_bit_floor_ul.h" +#include "test/UnitTest/Test.h" + +TEST(LlvmLibcStdcBitfloorUlTest, Zero) { + EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_floor_ul(0UL), 0UL); +} + +TEST(LlvmLibcStdcBitfloorUlTest, Ones) { + for (unsigned i = 0U; i != ULONG_WIDTH; ++i) + EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_floor_ul(ULONG_MAX >> i), + 1UL << (ULONG_WIDTH - i - 1)); +} diff --git a/libc/test/src/stdbit/stdc_bit_floor_ull_test.cpp b/libc/test/src/stdbit/stdc_bit_floor_ull_test.cpp new file mode 100644 index 000000000000..4717d427a40a --- /dev/null +++ b/libc/test/src/stdbit/stdc_bit_floor_ull_test.cpp @@ -0,0 +1,21 @@ +//===-- Unittests for stdc_bit_floor_ull ----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/CPP/limits.h" +#include "src/stdbit/stdc_bit_floor_ull.h" +#include "test/UnitTest/Test.h" + +TEST(LlvmLibcStdcBitfloorUllTest, Zero) { + EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_floor_ull(0ULL), 0ULL); +} + +TEST(LlvmLibcStdcBitfloorUllTest, Ones) { + for (unsigned i = 0U; i != ULLONG_WIDTH; ++i) + EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_floor_ull(ULLONG_MAX >> i), + 1ULL << (ULLONG_WIDTH - i - 1)); +} diff --git a/libc/test/src/stdbit/stdc_bit_floor_us_test.cpp b/libc/test/src/stdbit/stdc_bit_floor_us_test.cpp new file mode 100644 index 000000000000..4df87fb079ba --- /dev/null +++ b/libc/test/src/stdbit/stdc_bit_floor_us_test.cpp @@ -0,0 +1,22 @@ +//===-- Unittests for stdc_bit_floor_us -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/CPP/limits.h" +#include "src/stdbit/stdc_bit_floor_us.h" +#include "test/UnitTest/Test.h" + +TEST(LlvmLibcStdcBitfloorUsTest, Zero) { + EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_floor_us(0U), + static_cast(0)); +} + +TEST(LlvmLibcStdcBitfloorUsTest, Ones) { + for (unsigned i = 0U; i != USHRT_WIDTH; ++i) + EXPECT_EQ(LIBC_NAMESPACE::stdc_bit_floor_us(USHRT_MAX >> i), + static_cast(1 << (USHRT_WIDTH - i - 1))); +} -- GitLab From b1f2e19dc27d339cbeb3373066e73a4c91d133dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Thu, 7 Mar 2024 17:29:42 +0100 Subject: [PATCH 459/929] [clang][Interp][NFC] Use ArrayElem{,Pop} ops more often Instead of the longer ArrayElemPtr + Load. --- clang/lib/AST/Interp/ByteCodeExprGen.cpp | 57 +++++++++--------------- clang/lib/AST/Interp/Interp.h | 14 ++++++ clang/lib/AST/Interp/Opcodes.td | 8 ++++ 3 files changed, 42 insertions(+), 37 deletions(-) diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp index 8872579e12dc..712218f5de2e 100644 --- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp +++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp @@ -314,11 +314,7 @@ bool ByteCodeExprGen::VisitCastExpr(const CastExpr *CE) { for (unsigned I = 0; I != 2; ++I) { if (!this->emitGetLocal(PT_Ptr, *SubExprOffset, CE)) return false; - if (!this->emitConstUint8(I, CE)) - return false; - if (!this->emitArrayElemPtrPopUint8(CE)) - return false; - if (!this->emitLoadPop(SourceElemT, CE)) + if (!this->emitArrayElemPop(SourceElemT, I, CE)) return false; // Do the cast. @@ -729,11 +725,8 @@ bool ByteCodeExprGen::VisitComplexBinOp(const BinaryOperator *E) { if (IsComplex) { if (!this->emitGetLocal(PT_Ptr, Offset, E)) return false; - if (!this->emitConstUint8(ElemIndex, E)) - return false; - if (!this->emitArrayElemPtrPopUint8(E)) - return false; - return this->emitLoadPop(classifyComplexElementType(E->getType()), E); + return this->emitArrayElemPop(classifyComplexElementType(E->getType()), + ElemIndex, E); } if (ElemIndex == 0) return this->emitGetLocal(classifyPrim(E->getType()), Offset, E); @@ -3127,16 +3120,16 @@ bool ByteCodeExprGen::VisitUnaryOperator(const UnaryOperator *E) { if (!this->visit(SubExpr)) return false; - if (!this->emitConstUint8(1, E)) - return false; - if (!this->emitArrayElemPtrPopUint8(E)) - return false; + + if (SubExpr->isLValue()) { + if (!this->emitConstUint8(1, E)) + return false; + return this->emitArrayElemPtrPopUint8(E); + } // Since our _Complex implementation does not map to a primitive type, // we sometimes have to do the lvalue-to-rvalue conversion here manually. - if (!SubExpr->isLValue()) - return this->emitLoadPop(classifyPrim(E->getType()), E); - return true; + return this->emitArrayElemPop(classifyPrim(E->getType()), 1, E); } case UO_Extension: return this->delegate(SubExpr); @@ -3347,17 +3340,15 @@ bool ByteCodeExprGen::emitComplexReal(const Expr *SubExpr) { if (!this->visit(SubExpr)) return false; - if (!this->emitConstUint8(0, SubExpr)) - return false; - if (!this->emitArrayElemPtrPopUint8(SubExpr)) - return false; + if (SubExpr->isLValue()) { + if (!this->emitConstUint8(0, SubExpr)) + return false; + return this->emitArrayElemPtrPopUint8(SubExpr); + } - // Since our _Complex implementation does not map to a primitive type, - // we sometimes have to do the lvalue-to-rvalue conversion here manually. - if (!SubExpr->isLValue()) - return this->emitLoadPop(classifyComplexElementType(SubExpr->getType()), - SubExpr); - return true; + // Rvalue, load the actual element. + return this->emitArrayElemPop(classifyComplexElementType(SubExpr->getType()), + 0, SubExpr); } template @@ -3366,11 +3357,7 @@ bool ByteCodeExprGen::emitComplexBoolCast(const Expr *E) { PrimType ElemT = classifyComplexElementType(E->getType()); // We emit the expression (__real(E) != 0 || __imag(E) != 0) // for us, that means (bool)E[0] || (bool)E[1] - if (!this->emitConstUint8(0, E)) - return false; - if (!this->emitArrayElemPtrUint8(E)) - return false; - if (!this->emitLoadPop(ElemT, E)) + if (!this->emitArrayElem(ElemT, 0, E)) return false; if (ElemT == PT_Float) { if (!this->emitCastFloatingIntegral(PT_Bool, E)) @@ -3385,11 +3372,7 @@ bool ByteCodeExprGen::emitComplexBoolCast(const Expr *E) { if (!this->jumpTrue(LabelTrue)) return false; - if (!this->emitConstUint8(1, E)) - return false; - if (!this->emitArrayElemPtrPopUint8(E)) - return false; - if (!this->emitLoadPop(ElemT, E)) + if (!this->emitArrayElemPop(ElemT, 1, E)) return false; if (ElemT == PT_Float) { if (!this->emitCastFloatingIntegral(PT_Bool, E)) diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h index 43cbc2ff292c..bb220657c2da 100644 --- a/clang/lib/AST/Interp/Interp.h +++ b/clang/lib/AST/Interp/Interp.h @@ -1959,10 +1959,24 @@ inline bool ArrayElemPtrPop(InterpState &S, CodePtr OpPC) { return NarrowPtr(S, OpPC); } +template ::T> +inline bool ArrayElem(InterpState &S, CodePtr OpPC, uint32_t Index) { + const Pointer &Ptr = S.Stk.peek(); + + if (!CheckLoad(S, OpPC, Ptr)) + return false; + + S.Stk.push(Ptr.atIndex(Index).deref()); + return true; +} + template ::T> inline bool ArrayElemPop(InterpState &S, CodePtr OpPC, uint32_t Index) { const Pointer &Ptr = S.Stk.pop(); + if (!CheckLoad(S, OpPC, Ptr)) + return false; + S.Stk.push(Ptr.atIndex(Index).deref()); return true; } diff --git a/clang/lib/AST/Interp/Opcodes.td b/clang/lib/AST/Interp/Opcodes.td index ffc54646f027..9b99aa0ccb55 100644 --- a/clang/lib/AST/Interp/Opcodes.td +++ b/clang/lib/AST/Interp/Opcodes.td @@ -368,6 +368,14 @@ def ArrayElemPop : Opcode { let HasGroup = 1; } +def ArrayElem : Opcode { + let Args = [ArgUint32]; + let Types = [AllTypeClass]; + let HasGroup = 1; +} + + + //===----------------------------------------------------------------------===// // Direct field accessors //===----------------------------------------------------------------------===// -- GitLab From 5c752df1e10b7af0684e549601f0a8dccffcfcf0 Mon Sep 17 00:00:00 2001 From: Gheorghe-Teodor Bercea Date: Thu, 7 Mar 2024 12:01:42 -0500 Subject: [PATCH 460/929] [libomptarget][nextgen-plugin][NFC] Clean-up InputSignal checks (#83458) Clean-up InputSignal checks. --- .../plugins-nextgen/amdgpu/src/rtl.cpp | 28 +++---------------- 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp b/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp index 81634ae1edc4..fce7454bf280 100644 --- a/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp +++ b/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp @@ -715,16 +715,12 @@ struct AMDGPUQueueTy { std::lock_guard Lock(Mutex); assert(Queue && "Interacted with a non-initialized queue!"); - // Avoid defining the input dependency if already satisfied. - if (InputSignal && !InputSignal->load()) - InputSignal = nullptr; - // Add a barrier packet before the kernel packet in case there is a pending // preceding operation. The barrier packet will delay the processing of // subsequent queue's packets until the barrier input signal are satisfied. // No need output signal needed because the dependency is already guaranteed // by the queue barrier itself. - if (InputSignal) + if (InputSignal && InputSignal->load()) if (auto Err = pushBarrierImpl(nullptr, InputSignal)) return Err; @@ -1254,12 +1250,8 @@ public: // Consume stream slot and compute dependencies. auto [Curr, InputSignal] = consume(OutputSignal); - // Avoid defining the input dependency if already satisfied. - if (InputSignal && !InputSignal->load()) - InputSignal = nullptr; - // Issue the async memory copy. - if (InputSignal) { + if (InputSignal && InputSignal->load()) { hsa_signal_t InputSignalRaw = InputSignal->get(); return utils::asyncMemCopy(UseMultipleSdmaEngines, Dst, Agent, Src, Agent, CopySize, 1, &InputSignalRaw, @@ -1293,17 +1285,13 @@ public: // Consume stream slot and compute dependencies. auto [Curr, InputSignal] = consume(OutputSignals[0]); - // Avoid defining the input dependency if already satisfied. - if (InputSignal && !InputSignal->load()) - InputSignal = nullptr; - // Setup the post action for releasing the intermediate buffer. if (auto Err = Slots[Curr].schedReleaseBuffer(Inter, MemoryManager)) return Err; // Issue the first step: device to host transfer. Avoid defining the input // dependency if already satisfied. - if (InputSignal) { + if (InputSignal && InputSignal->load()) { hsa_signal_t InputSignalRaw = InputSignal->get(); if (auto Err = utils::asyncMemCopy( UseMultipleSdmaEngines, Inter, Agent, Src, Agent, CopySize, 1, @@ -1361,12 +1349,8 @@ public: // Consume stream slot and compute dependencies. auto [Curr, InputSignal] = consume(OutputSignal); - // Avoid defining the input dependency if already satisfied. - if (InputSignal && !InputSignal->load()) - InputSignal = nullptr; - // Issue the first step: host to host transfer. - if (InputSignal) { + if (InputSignal && InputSignal->load()) { // The std::memcpy is done asynchronously using an async handler. We store // the function's information in the action but it is not actually a // post action. @@ -1429,10 +1413,6 @@ public: // Consume stream slot and compute dependencies. auto [Curr, InputSignal] = consume(OutputSignal); - // Avoid defining the input dependency if already satisfied. - if (InputSignal && !InputSignal->load()) - InputSignal = nullptr; - // The agents need to have access to the corresponding memory // This is presently only true if the pointers were originally // allocated by this runtime or the caller made the appropriate -- GitLab From d1fc59c3b5c5ce292a6060d7a5545094cdf1b5fc Mon Sep 17 00:00:00 2001 From: Benjamin Maxwell Date: Thu, 7 Mar 2024 17:04:12 +0000 Subject: [PATCH 461/929] [mlir][ArmSME] Rewrite illegal `shape_casts` to `vector.transpose` ops (#82985) This adds a rewrite that converts illegal 2D unit-dim `shape_casts` into `vector.transpose` ops. E.g. ```mlir // Case 1: %a = vector.shape_cast %0 : vector<[4]x1xf32> to vector<1x[4]xf32> // Case 2: %b = vector.shape_cast %1 : vector<[4]x1xf32> to vector<[4]xf32> ``` Becomes: ```mlir // Case 1: %a = vector.transpose %0 : [1, 0] vector<[4]x1xf32> to vector<1x[4]xf32> // Case 2: %t = vector.transpose %1 : [1, 0] vector<[4]x1xf32> to vector<1x[4]xf32> %b = vector.shape_cast %t : vector<1x[4]xf32> to vector<[4]xf32> ``` Various lowerings and drop unit-dims patterns add such shape_casts, however, if they do not cancel out (which they likely won't if we've reached the vector-legalization pass) they will prevent lowering the IR. Rewriting them as a transpose gives `LiftIllegalVectorTransposeToMemory` a chance to eliminate the illegal types. --- .../ArmSME/Transforms/VectorLegalization.cpp | 85 ++++++++++++++++--- .../Dialect/ArmSME/vector-legalization.mlir | 45 ++++++++++ 2 files changed, 116 insertions(+), 14 deletions(-) diff --git a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp index 11f8bc04b218..31500c62c0d6 100644 --- a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp +++ b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp @@ -46,6 +46,8 @@ static constexpr StringLiteral kMatchFailureUnsupportedMaskOp( "op mask is unsupported for legalization/decomposition"); static constexpr StringLiteral kMatchFailureNonPermutationMap("op affine map is not a permutation"); +static constexpr StringLiteral kMatchFailureNotIllegalToLegal( + "expected transpose from illegal type to legal type"); /// An SMESubTile represents a single SME-sized sub-tile from decomposing a /// larger vector type. The (`row`, `col`) are the position of the tile in the @@ -416,6 +418,17 @@ struct FoldExtractFromVectorOfSMELikeCreateMasks } }; +/// A vector type where no fixed dimension comes after a scalable dimension. +bool isLegalVectorType(VectorType vType) { + bool seenFixedDim = false; + for (bool scalableFlag : llvm::reverse(vType.getScalableDims())) { + seenFixedDim |= !scalableFlag; + if (seenFixedDim && scalableFlag) + return false; + } + return true; +} + /// Lifts an illegal vector.transpose and vector.transfer_read to a /// memref.subview + memref.transpose, followed by a legal read. /// @@ -448,16 +461,6 @@ struct LiftIllegalVectorTransposeToMemory : public OpRewritePattern { using OpRewritePattern::OpRewritePattern; - static bool isIllegalVectorType(VectorType vType) { - bool seenFixedDim = false; - for (bool scalableFlag : llvm::reverse(vType.getScalableDims())) { - seenFixedDim |= !scalableFlag; - if (seenFixedDim && scalableFlag) - return true; - } - return false; - } - static Value getExtensionSource(Operation *op) { if (isa_and_present(op)) return op->getOperand(0); @@ -468,9 +471,9 @@ struct LiftIllegalVectorTransposeToMemory PatternRewriter &rewriter) const override { auto sourceType = transposeOp.getSourceVectorType(); auto resultType = transposeOp.getResultVectorType(); - if (!isIllegalVectorType(sourceType) || isIllegalVectorType(resultType)) - return rewriter.notifyMatchFailure( - transposeOp, "expected transpose from illegal type to legal type"); + if (isLegalVectorType(sourceType) || !isLegalVectorType(resultType)) + return rewriter.notifyMatchFailure(transposeOp, + kMatchFailureNotIllegalToLegal); // Look through extend for transfer_read. Value maybeRead = transposeOp.getVector(); @@ -556,6 +559,59 @@ struct LiftIllegalVectorTransposeToMemory } }; +/// A rewrite to turn unit dim transpose-like vector.shape_casts into +/// vector.transposes. The shape_cast has to be from an illegal vector type to a +/// legal one (as defined by isLegalVectorType). +/// +/// The reasoning for this is if we've got to this pass and we still have +/// shape_casts of illegal types, then they likely will not cancel out. Turning +/// them into transposes gives LiftIllegalVectorTransposeToMemory a chance to +/// eliminate them. +/// +/// Example: +/// +/// BEFORE: +/// ```mlir +/// %0 = vector.shape_cast %a : vector<[4]x1xf32> to vector<1x[4]xf32> +/// ``` +/// +/// AFTER: +/// ```mlir +/// %0 = vector.transpose %0, [1, 0] : vector<[4]x1xf32> to vector<1x[4]xf32> +/// ``` +struct ConvertIllegalShapeCastOpsToTransposes + : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(vector::ShapeCastOp shapeCastOp, + PatternRewriter &rewriter) const override { + auto sourceType = shapeCastOp.getSourceVectorType(); + auto resultType = shapeCastOp.getResultVectorType(); + if (isLegalVectorType(sourceType) || !isLegalVectorType(resultType)) + return rewriter.notifyMatchFailure(shapeCastOp, + kMatchFailureNotIllegalToLegal); + + // Note: If we know that `sourceType` is an illegal vector type (and 2D) + // then dim 0 is scalable and dim 1 is fixed. + if (sourceType.getRank() != 2 || sourceType.getDimSize(1) != 1) + return rewriter.notifyMatchFailure( + shapeCastOp, "expected source to be a 2D scalable vector with a " + "trailing unit dim"); + + auto loc = shapeCastOp.getLoc(); + auto transpose = rewriter.create( + loc, shapeCastOp.getSource(), ArrayRef{1, 0}); + + if (resultType.getRank() == 1) + rewriter.replaceOpWithNewOp(shapeCastOp, resultType, + transpose); + else + rewriter.replaceOp(shapeCastOp, transpose); + + return success(); + } +}; + struct VectorLegalizationPass : public arm_sme::impl::VectorLegalizationBase { void runOnOperation() override { @@ -576,7 +632,8 @@ struct VectorLegalizationPass }); patterns.add(context); + LiftIllegalVectorTransposeToMemory, + ConvertIllegalShapeCastOpsToTransposes>(context); // Note: High benefit to ensure masked outer products are lowered first. patterns.add( converter, context, 1024); diff --git a/mlir/test/Dialect/ArmSME/vector-legalization.mlir b/mlir/test/Dialect/ArmSME/vector-legalization.mlir index bf0b58ff4cf0..f8be697548c1 100644 --- a/mlir/test/Dialect/ArmSME/vector-legalization.mlir +++ b/mlir/test/Dialect/ArmSME/vector-legalization.mlir @@ -388,3 +388,48 @@ func.func @illegal_transpose_no_defining_source_op(%vec: vector<[4]x1xf32>) -> v %0 = vector.transpose %vec, [1, 0] : vector<[4]x1xf32> to vector<1x[4]xf32> return %0 : vector<1x[4]xf32> } + +// ----- + +// CHECK-LABEL: @illegal_shape_cast_to_transpose_2d( +// CHECK-SAME: %[[VEC:.*]]: vector<[4]x1xf32>) +func.func @illegal_shape_cast_to_transpose_2d(%vec: vector<[4]x1xf32>) -> vector<1x[4]xf32> { + // CHECK: vector.transpose %[[VEC]], [1, 0] : vector<[4]x1xf32> to vector<1x[4]xf32> + %0 = vector.shape_cast %vec : vector<[4]x1xf32> to vector<1x[4]xf32> + return %0 : vector<1x[4]xf32> +} + +// ----- + +// CHECK-LABEL: @illegal_shape_cast_to_transpose_1d( +// CHECK-SAME: %[[VEC:.*]]: vector<[4]x1xf32>) +func.func @illegal_shape_cast_to_transpose_1d(%vec: vector<[4]x1xf32>) -> vector<[4]xf32> { + // CHECK: %[[TRANSPOSE:.*]] = vector.transpose %[[VEC]], [1, 0] : vector<[4]x1xf32> to vector<1x[4]xf32> + // CHECK: vector.shape_cast %[[TRANSPOSE]] : vector<1x[4]xf32> to vector<[4]xf32> + %0 = vector.shape_cast %vec : vector<[4]x1xf32> to vector<[4]xf32> + return %0 : vector<[4]xf32> +} + +// ----- + +// CHECK-LABEL: @lift_illegal_2d_shape_cast_to_memory +func.func @lift_illegal_2d_shape_cast_to_memory(%a: index, %b: index, %memref: memref) -> vector<1x[4]xf32> { + // CHECK: vector.transfer_read {{.*}} : memref, vector<1x[4]xf32> + // CHECK-NOT: vector.shape_cast + %pad = arith.constant 0.0 : f32 + %illegalRead = vector.transfer_read %memref[%a, %b], %pad {in_bounds = [false, true]}: memref, vector<[4]x1xf32> + %cast = vector.shape_cast %illegalRead : vector<[4]x1xf32> to vector<1x[4]xf32> + return %cast : vector<1x[4]xf32> +} + +// ----- + +// CHECK-LABEL: @lift_illegal_1d_shape_cast_to_memory +func.func @lift_illegal_1d_shape_cast_to_memory(%a: index, %b: index, %memref: memref) -> vector<[4]xf32> { + // CHECK: vector.transfer_read {{.*}} : memref, vector<1x[4]xf32> + // CHECK-NOT: vector.shape_cast {{.*}} : vector<[4]x1xf32> to vector<[4]xf32> + %pad = arith.constant 0.0 : f32 + %illegalRead = vector.transfer_read %memref[%a, %b], %pad {in_bounds = [false, true]}: memref, vector<[4]x1xf32> + %cast = vector.shape_cast %illegalRead : vector<[4]x1xf32> to vector<[4]xf32> + return %cast : vector<[4]xf32> +} -- GitLab From 630289f77d67703673928ae38d3e5ba900e9ff62 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Thu, 7 Mar 2024 11:04:40 -0600 Subject: [PATCH 462/929] [HIP] Do not include the CUID module hash with the new driver (#84332) Summary: The new driver does not need this hash and it can lead to redefined symbol errors when the CUID hash isn't set. --- clang/lib/CodeGen/CodeGenModule.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index d02875c6a86d..967319bdfc45 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -916,7 +916,7 @@ void CodeGenModule::Release() { llvm::ConstantArray::get(ATy, UsedArray), "__clang_gpu_used_external"); addCompilerUsedGlobal(GV); } - if (LangOpts.HIP) { + if (LangOpts.HIP && !getLangOpts().OffloadingNewDriver) { // Emit a unique ID so that host and device binaries from the same // compilation unit can be associated. auto *GV = new llvm::GlobalVariable( -- GitLab From 8f79cdd8da97c131ae7d8a3210bb69cb6654903d Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Thu, 7 Mar 2024 17:02:23 +0000 Subject: [PATCH 463/929] [AArch64] Add -verify-machineinstrs to a test This would have helped identify problems with #83905 which only showed up in an LLVM_ENABLE_EXPENSIVE_CHECKS build. --- llvm/test/CodeGen/AArch64/stack-probing-no-scratch-reg.mir | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/test/CodeGen/AArch64/stack-probing-no-scratch-reg.mir b/llvm/test/CodeGen/AArch64/stack-probing-no-scratch-reg.mir index f50bd9ab4b8a..f2d79bd72069 100644 --- a/llvm/test/CodeGen/AArch64/stack-probing-no-scratch-reg.mir +++ b/llvm/test/CodeGen/AArch64/stack-probing-no-scratch-reg.mir @@ -1,5 +1,5 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4 -# RUN: llc %s --start-before=shrink-wrap -stop-after=prologepilog -o - | FileCheck %s +# RUN: llc %s --start-before=shrink-wrap -stop-after=prologepilog -verify-machineinstrs -o - | FileCheck %s --- | target triple = "aarch64-linux" -- GitLab From 96049fcf4e5f2eb0271bdfa89e113eef9c5fa9f6 Mon Sep 17 00:00:00 2001 From: Michael Maitland Date: Thu, 7 Mar 2024 09:50:29 -0500 Subject: [PATCH 464/929] [GISEL] Add IRTranslation for shufflevector on scalable vector types (#80378) Recommits llvm/llvm-project#80378 which was reverted in llvm/llvm-project#84330. The problem was that the change in llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir used 217 as an opcode instead of a regex. --- llvm/docs/GlobalISel/GenericOpcode.rst | 5 + .../CodeGen/GlobalISel/MachineIRBuilder.h | 12 +- llvm/include/llvm/Support/TargetOpcodes.def | 3 + llvm/include/llvm/Target/GenericOpcodes.td | 7 + llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp | 4 +- llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 27 +- .../CodeGen/GlobalISel/LegalizerHelper.cpp | 2 +- .../CodeGen/GlobalISel/MachineIRBuilder.cpp | 16 +- llvm/lib/CodeGen/MachineVerifier.cpp | 18 + llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 3 +- .../GlobalISel/legalizer-info-validation.mir | 3 + .../GlobalISel/irtranslator/shufflevector.ll | 1774 +++++++++++++++++ .../MachineVerifier/test_g_splat_vector.mir | 27 + .../GlobalISel/LegalizerHelperTest.cpp | 4 +- .../CodeGen/GlobalISel/PatternMatchTest.cpp | 6 +- 15 files changed, 1890 insertions(+), 21 deletions(-) create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/shufflevector.ll create mode 100644 llvm/test/MachineVerifier/test_g_splat_vector.mir diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst index 33b0152bd7b4..dda367607d04 100644 --- a/llvm/docs/GlobalISel/GenericOpcode.rst +++ b/llvm/docs/GlobalISel/GenericOpcode.rst @@ -639,6 +639,11 @@ Concatenate two vectors and shuffle the elements according to the mask operand. The mask operand should be an IR Constant which exactly matches the corresponding mask for the IR shufflevector instruction. +G_SPLAT_VECTOR +^^^^^^^^^^^^^^^^ + +Create a vector where all elements are the scalar from the source operand. + Vector Reduction Operations --------------------------- diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index 1387a0a37561..6762b1b360d5 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -1063,8 +1063,7 @@ public: /// Build and insert \p Res = G_BUILD_VECTOR with \p Src replicated to fill /// the number of elements - MachineInstrBuilder buildSplatVector(const DstOp &Res, - const SrcOp &Src); + MachineInstrBuilder buildSplatBuildVector(const DstOp &Res, const SrcOp &Src); /// Build and insert \p Res = G_BUILD_VECTOR_TRUNC \p Op0, ... /// @@ -1099,6 +1098,15 @@ public: MachineInstrBuilder buildShuffleVector(const DstOp &Res, const SrcOp &Src1, const SrcOp &Src2, ArrayRef Mask); + /// Build and insert \p Res = G_SPLAT_VECTOR \p Val + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p Res must be a generic virtual register with vector type. + /// \pre \p Val must be a generic virtual register with scalar type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildSplatVector(const DstOp &Res, const SrcOp &Val); + /// Build and insert \p Res = G_CONCAT_VECTORS \p Op0, ... /// /// G_CONCAT_VECTORS creates a vector from the concatenation of 2 or more diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def index 6aded2ceebe1..94fba491148b 100644 --- a/llvm/include/llvm/Support/TargetOpcodes.def +++ b/llvm/include/llvm/Support/TargetOpcodes.def @@ -736,6 +736,9 @@ HANDLE_TARGET_OPCODE(G_EXTRACT_VECTOR_ELT) /// Generic shufflevector. HANDLE_TARGET_OPCODE(G_SHUFFLE_VECTOR) +/// Generic splatvector. +HANDLE_TARGET_OPCODE(G_SPLAT_VECTOR) + /// Generic count trailing zeroes. HANDLE_TARGET_OPCODE(G_CTTZ) diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td index d2036e478d18..d967885aa2d7 100644 --- a/llvm/include/llvm/Target/GenericOpcodes.td +++ b/llvm/include/llvm/Target/GenericOpcodes.td @@ -1450,6 +1450,13 @@ def G_SHUFFLE_VECTOR: GenericInstruction { let hasSideEffects = false; } +// Generic splatvector. +def G_SPLAT_VECTOR: GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type1:$val); + let hasSideEffects = false; +} + //------------------------------------------------------------------------------ // Vector reductions //------------------------------------------------------------------------------ diff --git a/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp index 64e2d517e3b9..1869e0d41a51 100644 --- a/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp @@ -309,7 +309,7 @@ MachineInstrBuilder CSEMIRBuilder::buildConstant(const DstOp &Res, // For vectors, CSE the element only for now. LLT Ty = Res.getLLTTy(*getMRI()); if (Ty.isVector()) - return buildSplatVector(Res, buildConstant(Ty.getElementType(), Val)); + return buildSplatBuildVector(Res, buildConstant(Ty.getElementType(), Val)); FoldingSetNodeID ID; GISelInstProfileBuilder ProfBuilder(ID, *getMRI()); @@ -336,7 +336,7 @@ MachineInstrBuilder CSEMIRBuilder::buildFConstant(const DstOp &Res, // For vectors, CSE the element only for now. LLT Ty = Res.getLLTTy(*getMRI()); if (Ty.isVector()) - return buildSplatVector(Res, buildFConstant(Ty.getElementType(), Val)); + return buildSplatBuildVector(Res, buildFConstant(Ty.getElementType(), Val)); FoldingSetNodeID ID; GISelInstProfileBuilder ProfBuilder(ID, *getMRI()); diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 7c986dbbc2c7..365870f540da 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -1598,10 +1598,10 @@ bool IRTranslator::translateGetElementPtr(const User &U, // We might need to splat the base pointer into a vector if the offsets // are vectors. if (WantSplatVector && !PtrTy.isVector()) { - BaseReg = - MIRBuilder - .buildSplatVector(LLT::fixed_vector(VectorWidth, PtrTy), BaseReg) - .getReg(0); + BaseReg = MIRBuilder + .buildSplatBuildVector(LLT::fixed_vector(VectorWidth, PtrTy), + BaseReg) + .getReg(0); PtrIRTy = FixedVectorType::get(PtrIRTy, VectorWidth); PtrTy = getLLTForType(*PtrIRTy, *DL); OffsetIRTy = DL->getIndexType(PtrIRTy); @@ -1639,8 +1639,10 @@ bool IRTranslator::translateGetElementPtr(const User &U, LLT IdxTy = MRI->getType(IdxReg); if (IdxTy != OffsetTy) { if (!IdxTy.isVector() && WantSplatVector) { - IdxReg = MIRBuilder.buildSplatVector( - OffsetTy.changeElementType(IdxTy), IdxReg).getReg(0); + IdxReg = MIRBuilder + .buildSplatBuildVector(OffsetTy.changeElementType(IdxTy), + IdxReg) + .getReg(0); } IdxReg = MIRBuilder.buildSExtOrTrunc(OffsetTy, IdxReg).getReg(0); @@ -2997,6 +2999,19 @@ bool IRTranslator::translateExtractElement(const User &U, bool IRTranslator::translateShuffleVector(const User &U, MachineIRBuilder &MIRBuilder) { + // A ShuffleVector that has operates on scalable vectors is a splat vector + // where the value of the splat vector is the 0th element of the first + // operand, since the index mask operand is the zeroinitializer (undef and + // poison are treated as zeroinitializer here). + if (U.getOperand(0)->getType()->isScalableTy()) { + Value *Op0 = U.getOperand(0); + auto SplatVal = MIRBuilder.buildExtractVectorElementConstant( + LLT::scalar(Op0->getType()->getScalarSizeInBits()), + getOrCreateVReg(*Op0), 0); + MIRBuilder.buildSplatVector(getOrCreateVReg(U), SplatVal); + return true; + } + ArrayRef Mask; if (auto *SVI = dyn_cast(&U)) Mask = SVI->getShuffleMask(); diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index 1d016e684c48..2ec47f72aca3 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -8391,7 +8391,7 @@ static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB) { // For vector types create a G_BUILD_VECTOR. if (Ty.isVector()) - Val = MIB.buildSplatVector(Ty, Val).getReg(0); + Val = MIB.buildSplatBuildVector(Ty, Val).getReg(0); return Val; } diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp index cdd605a5221a..a5a136e2effc 100644 --- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -326,7 +326,7 @@ MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res, auto Const = buildInstr(TargetOpcode::G_CONSTANT) .addDef(getMRI()->createGenericVirtualRegister(EltTy)) .addCImm(&Val); - return buildSplatVector(Res, Const); + return buildSplatBuildVector(Res, Const); } auto Const = buildInstr(TargetOpcode::G_CONSTANT); @@ -363,7 +363,7 @@ MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res, .addDef(getMRI()->createGenericVirtualRegister(EltTy)) .addFPImm(&Val); - return buildSplatVector(Res, Const); + return buildSplatBuildVector(Res, Const); } auto Const = buildInstr(TargetOpcode::G_FCONSTANT); @@ -711,8 +711,8 @@ MachineIRBuilder::buildBuildVectorConstant(const DstOp &Res, return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec); } -MachineInstrBuilder MachineIRBuilder::buildSplatVector(const DstOp &Res, - const SrcOp &Src) { +MachineInstrBuilder MachineIRBuilder::buildSplatBuildVector(const DstOp &Res, + const SrcOp &Src) { SmallVector TmpVec(Res.getLLTTy(*getMRI()).getNumElements(), Src); return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec); } @@ -742,6 +742,14 @@ MachineInstrBuilder MachineIRBuilder::buildShuffleSplat(const DstOp &Res, return buildShuffleVector(DstTy, InsElt, UndefVec, ZeroMask); } +MachineInstrBuilder MachineIRBuilder::buildSplatVector(const DstOp &Res, + const SrcOp &Src) { + LLT DstTy = Res.getLLTTy(*getMRI()); + assert(Src.getLLTTy(*getMRI()) == DstTy.getElementType() && + "Expected Src to match Dst elt ty"); + return buildInstr(TargetOpcode::G_SPLAT_VECTOR, Res, Src); +} + MachineInstrBuilder MachineIRBuilder::buildShuffleVector(const DstOp &Res, const SrcOp &Src1, const SrcOp &Src2, diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp index 1d0757c5d7f5..ecb3bd33bdfd 100644 --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -1640,6 +1640,24 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) { break; } + + case TargetOpcode::G_SPLAT_VECTOR: { + LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); + LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); + + if (!DstTy.isScalableVector()) + report("Destination type must be a scalable vector", MI); + + if (!SrcTy.isScalar()) + report("Source type must be a scalar", MI); + + if (DstTy.getScalarType() != SrcTy) + report("Element type of the destination must be the same type as the " + "source type", + MI); + + break; + } case TargetOpcode::G_DYN_STACKALLOC: { const MachineOperand &DstOp = MI->getOperand(0); const MachineOperand &AllocOp = MI->getOperand(1); diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 750d70c03eab..4713bd605c24 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -20920,7 +20920,8 @@ bool RISCVTargetLowering::fallBackToDAGISel(const Instruction &Inst) const { unsigned Op = Inst.getOpcode(); if (Op == Instruction::Add || Op == Instruction::Sub || Op == Instruction::And || Op == Instruction::Or || - Op == Instruction::Xor || Op == Instruction::InsertElement) + Op == Instruction::Xor || Op == Instruction::InsertElement || + Op == Instruction::Xor || Op == Instruction::ShuffleVector) return false; if (Inst.getType()->isScalableTy()) diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir index d87704cf45d5..ecad3f115134 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir @@ -625,6 +625,9 @@ # DEBUG-NEXT: G_SHUFFLE_VECTOR (opcode {{[0-9]+}}): 2 type indices, 0 imm indices # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected +# DEBUG-NEXT: G_SPLAT_VECTOR (opcode {{[0-9]+}}): 2 type indices, 0 imm indices +# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined # DEBUG-NEXT: G_CTTZ (opcode {{[0-9]+}}): 2 type indices, 0 imm indices # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/shufflevector.ll b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/shufflevector.ll new file mode 100644 index 000000000000..df7778899b0d --- /dev/null +++ b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/shufflevector.ll @@ -0,0 +1,1774 @@ +; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=riscv32 -mattr=+v -global-isel -stop-after=irtranslator \ +; RUN: -verify-machineinstrs < %s | FileCheck -check-prefixes=RV32 %s +; RUN: llc -mtriple=riscv64 -mattr=+v -global-isel -stop-after=irtranslator \ +; RUN: -verify-machineinstrs < %s | FileCheck -check-prefixes=RV64 %s + +define @shufflevector_nxv1i1_0() { + ; RV32-LABEL: name: shufflevector_nxv1i1_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv1i1_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv1i1_1() { + ; RV32-LABEL: name: shufflevector_nxv1i1_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv1i1_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv1i1_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv1i1_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v0 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv1i1_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v0 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv2i1_0() { + ; RV32-LABEL: name: shufflevector_nxv2i1_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv2i1_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv2i1_1() { + ; RV32-LABEL: name: shufflevector_nxv2i1_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv2i1_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv2i1_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv2i1_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v0 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv2i1_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v0 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv4i1_0() { + ; RV32-LABEL: name: shufflevector_nxv4i1_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv4i1_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv4i1_1() { + ; RV32-LABEL: name: shufflevector_nxv4i1_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv4i1_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv4i1_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv4i1_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v0 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv4i1_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v0 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv8i1_0() { + ; RV32-LABEL: name: shufflevector_nxv8i1_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv8i1_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv8i1_1() { + ; RV32-LABEL: name: shufflevector_nxv8i1_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv8i1_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv8i1_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv8i1_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v0 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv8i1_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v0 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv16i1_0() { + ; RV32-LABEL: name: shufflevector_nxv16i1_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv16i1_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv16i1_1() { + ; RV32-LABEL: name: shufflevector_nxv16i1_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv16i1_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv16i1_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv16i1_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v0 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV32-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v0 + ; + ; RV64-LABEL: name: shufflevector_nxv16i1_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v0 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v0 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s1) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s1) + ; RV64-NEXT: $v0 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v0 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv1i8_0() { + ; RV32-LABEL: name: shufflevector_nxv1i8_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i8_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv1i8_1() { + ; RV32-LABEL: name: shufflevector_nxv1i8_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i8_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv1i8_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv1i8_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i8_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv2i8_0() { + ; RV32-LABEL: name: shufflevector_nxv2i8_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv2i8_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv2i8_1() { + ; RV32-LABEL: name: shufflevector_nxv2i8_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv2i8_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv2i8_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv2i8_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv2i8_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv4i8_0() { + ; RV32-LABEL: name: shufflevector_nxv4i8_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv4i8_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv4i8_1() { + ; RV32-LABEL: name: shufflevector_nxv4i8_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv4i8_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv4i8_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv4i8_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv4i8_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv8i8_0() { + ; RV32-LABEL: name: shufflevector_nxv8i8_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv8i8_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv8i8_1() { + ; RV32-LABEL: name: shufflevector_nxv8i8_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv8i8_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv8i8_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv8i8_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv8i8_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv16i8_0() { + ; RV32-LABEL: name: shufflevector_nxv16i8_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv16i8_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv16i8_1() { + ; RV32-LABEL: name: shufflevector_nxv16i8_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv16i8_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv16i8_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv16i8_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8m2 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv16i8_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8m2 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s8) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s8) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv1i16_0() { + ; RV32-LABEL: name: shufflevector_nxv1i16_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i16_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv1i16_1() { + ; RV32-LABEL: name: shufflevector_nxv1i16_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i16_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv1i16_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv1i16_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i16_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv2i16_0() { + ; RV32-LABEL: name: shufflevector_nxv2i16_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv2i16_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv2i16_1() { + ; RV32-LABEL: name: shufflevector_nxv2i16_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv2i16_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv2i16_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv2i16_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv2i16_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv4i16_0() { + ; RV32-LABEL: name: shufflevector_nxv4i16_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv4i16_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv4i16_1() { + ; RV32-LABEL: name: shufflevector_nxv4i16_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv4i16_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv4i16_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv4i16_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv4i16_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv8i16_0() { + ; RV32-LABEL: name: shufflevector_nxv8i16_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv8i16_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv8i16_1() { + ; RV32-LABEL: name: shufflevector_nxv8i16_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv8i16_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv8i16_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv8i16_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8m2 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv8i16_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8m2 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv16i16_0() { + ; RV32-LABEL: name: shufflevector_nxv16i16_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m4 + ; + ; RV64-LABEL: name: shufflevector_nxv16i16_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m4 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv16i16_1() { + ; RV32-LABEL: name: shufflevector_nxv16i16_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m4 + ; + ; RV64-LABEL: name: shufflevector_nxv16i16_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m4 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv16i16_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv16i16_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8m4 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m4 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m4 + ; + ; RV64-LABEL: name: shufflevector_nxv16i16_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8m4 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m4 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s16) + ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m4 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv1i32_0() { + ; RV32-LABEL: name: shufflevector_nxv1i32_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i32_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv1i32_1() { + ; RV32-LABEL: name: shufflevector_nxv1i32_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i32_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv1i32_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv1i32_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i32_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv2i32_0() { + ; RV32-LABEL: name: shufflevector_nxv2i32_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv2i32_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv2i32_1() { + ; RV32-LABEL: name: shufflevector_nxv2i32_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv2i32_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv2i32_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv2i32_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv2i32_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv4i32_0() { + ; RV32-LABEL: name: shufflevector_nxv4i32_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv4i32_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv4i32_1() { + ; RV32-LABEL: name: shufflevector_nxv4i32_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv4i32_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv4i32_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv4i32_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8m2 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv4i32_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8m2 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv8i32_0() { + ; RV32-LABEL: name: shufflevector_nxv8i32_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m4 + ; + ; RV64-LABEL: name: shufflevector_nxv8i32_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m4 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv8i32_1() { + ; RV32-LABEL: name: shufflevector_nxv8i32_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m4 + ; + ; RV64-LABEL: name: shufflevector_nxv8i32_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m4 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv8i32_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv8i32_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8m4 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m4 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m4 + ; + ; RV64-LABEL: name: shufflevector_nxv8i32_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8m4 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m4 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m4 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv16i32_0() { + ; RV32-LABEL: name: shufflevector_nxv16i32_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m8 + ; + ; RV64-LABEL: name: shufflevector_nxv16i32_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv16i32_1() { + ; RV32-LABEL: name: shufflevector_nxv16i32_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m8 + ; + ; RV64-LABEL: name: shufflevector_nxv16i32_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv16i32_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv16i32_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8m8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV32-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m8 + ; + ; RV64-LABEL: name: shufflevector_nxv16i32_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8m8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s32) + ; RV64-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv1i64_0() { + ; RV32-LABEL: name: shufflevector_nxv1i64_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i64_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv1i64_1() { + ; RV32-LABEL: name: shufflevector_nxv1i64_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i64_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv1i64_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv1i64_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8 + ; + ; RV64-LABEL: name: shufflevector_nxv1i64_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv2i64_0() { + ; RV32-LABEL: name: shufflevector_nxv2i64_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv2i64_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv2i64_1() { + ; RV32-LABEL: name: shufflevector_nxv2i64_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv2i64_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv2i64_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv2i64_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8m2 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m2 + ; + ; RV64-LABEL: name: shufflevector_nxv2i64_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8m2 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m2 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8m2 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m2 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv4i64_0() { + ; RV32-LABEL: name: shufflevector_nxv4i64_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m4 + ; + ; RV64-LABEL: name: shufflevector_nxv4i64_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m4 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv4i64_1() { + ; RV32-LABEL: name: shufflevector_nxv4i64_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m4 + ; + ; RV64-LABEL: name: shufflevector_nxv4i64_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m4 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv4i64_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv4i64_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8m4 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m4 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m4 + ; + ; RV64-LABEL: name: shufflevector_nxv4i64_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8m4 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m4 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8m4 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m4 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv8i64_0() { + ; RV32-LABEL: name: shufflevector_nxv8i64_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m8 + ; + ; RV64-LABEL: name: shufflevector_nxv8i64_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv8i64_1() { + ; RV32-LABEL: name: shufflevector_nxv8i64_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m8 + ; + ; RV64-LABEL: name: shufflevector_nxv8i64_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv8i64_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv8i64_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8m8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m8 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV32-NEXT: PseudoRET implicit $v8m8 + ; + ; RV64-LABEL: name: shufflevector_nxv8i64_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8m8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m8 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[COPY]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: $v8m8 = COPY [[SPLAT_VECTOR]]() + ; RV64-NEXT: PseudoRET implicit $v8m8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + +define @shufflevector_nxv16i64_0() { + ; RV32-LABEL: name: shufflevector_nxv16i64_0 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: [[UV:%[0-9]+]]:_(), [[UV1:%[0-9]+]]:_() = G_UNMERGE_VALUES [[SPLAT_VECTOR]]() + ; RV32-NEXT: $v8m8 = COPY [[UV]]() + ; RV32-NEXT: $v16m8 = COPY [[UV1]]() + ; RV32-NEXT: PseudoRET implicit $v8m8, implicit $v16m8 + ; + ; RV64-LABEL: name: shufflevector_nxv16i64_0 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: [[UV:%[0-9]+]]:_(), [[UV1:%[0-9]+]]:_() = G_UNMERGE_VALUES [[SPLAT_VECTOR]]() + ; RV64-NEXT: $v8m8 = COPY [[UV]]() + ; RV64-NEXT: $v16m8 = COPY [[UV1]]() + ; RV64-NEXT: PseudoRET implicit $v8m8, implicit $v16m8 + %a = shufflevector poison, poison, poison + ret %a +} + +define @shufflevector_nxv16i64_1() { + ; RV32-LABEL: name: shufflevector_nxv16i64_1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: [[UV:%[0-9]+]]:_(), [[UV1:%[0-9]+]]:_() = G_UNMERGE_VALUES [[SPLAT_VECTOR]]() + ; RV32-NEXT: $v8m8 = COPY [[UV]]() + ; RV32-NEXT: $v16m8 = COPY [[UV1]]() + ; RV32-NEXT: PseudoRET implicit $v8m8, implicit $v16m8 + ; + ; RV64-LABEL: name: shufflevector_nxv16i64_1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: [[DEF:%[0-9]+]]:_() = G_IMPLICIT_DEF + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[DEF]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: [[UV:%[0-9]+]]:_(), [[UV1:%[0-9]+]]:_() = G_UNMERGE_VALUES [[SPLAT_VECTOR]]() + ; RV64-NEXT: $v8m8 = COPY [[UV]]() + ; RV64-NEXT: $v16m8 = COPY [[UV1]]() + ; RV64-NEXT: PseudoRET implicit $v8m8, implicit $v16m8 + %a = shufflevector undef, undef, undef + ret %a +} + +define @shufflevector_nxv16i64_2( %a) { + ; RV32-LABEL: name: shufflevector_nxv16i64_2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $v8m8, $v16m8 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m8 + ; RV32-NEXT: [[COPY1:%[0-9]+]]:_() = COPY $v16m8 + ; RV32-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_() = G_CONCAT_VECTORS [[COPY]](), [[COPY1]]() + ; RV32-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV32-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[CONCAT_VECTORS]](), [[C]](s64) + ; RV32-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV32-NEXT: [[UV:%[0-9]+]]:_(), [[UV1:%[0-9]+]]:_() = G_UNMERGE_VALUES [[SPLAT_VECTOR]]() + ; RV32-NEXT: $v8m8 = COPY [[UV]]() + ; RV32-NEXT: $v16m8 = COPY [[UV1]]() + ; RV32-NEXT: PseudoRET implicit $v8m8, implicit $v16m8 + ; + ; RV64-LABEL: name: shufflevector_nxv16i64_2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $v8m8, $v16m8 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_() = COPY $v8m8 + ; RV64-NEXT: [[COPY1:%[0-9]+]]:_() = COPY $v16m8 + ; RV64-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_() = G_CONCAT_VECTORS [[COPY]](), [[COPY1]]() + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; RV64-NEXT: [[EVEC:%[0-9]+]]:_(s64) = G_EXTRACT_VECTOR_ELT [[CONCAT_VECTORS]](), [[C]](s64) + ; RV64-NEXT: [[SPLAT_VECTOR:%[0-9]+]]:_() = G_SPLAT_VECTOR [[EVEC]](s64) + ; RV64-NEXT: [[UV:%[0-9]+]]:_(), [[UV1:%[0-9]+]]:_() = G_UNMERGE_VALUES [[SPLAT_VECTOR]]() + ; RV64-NEXT: $v8m8 = COPY [[UV]]() + ; RV64-NEXT: $v16m8 = COPY [[UV1]]() + ; RV64-NEXT: PseudoRET implicit $v8m8, implicit $v16m8 + %b = shufflevector %a , poison, zeroinitializer + ret %b +} + + + diff --git a/llvm/test/MachineVerifier/test_g_splat_vector.mir b/llvm/test/MachineVerifier/test_g_splat_vector.mir new file mode 100644 index 000000000000..0d1d8a3e6dcc --- /dev/null +++ b/llvm/test/MachineVerifier/test_g_splat_vector.mir @@ -0,0 +1,27 @@ +# RUN: not --crash llc -o - -mtriple=arm64 -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s +# REQUIRES: aarch64-registered-target +--- +name: g_splat_vector +tracksRegLiveness: true +liveins: +body: | + bb.0: + %0:_(s32) = G_CONSTANT i32 0 + %1:_(<2 x s32>) = G_IMPLICIT_DEF + %2:_() = G_IMPLICIT_DEF + + ; CHECK: Destination type must be a scalable vector + %3:_(s32) = G_SPLAT_VECTOR %0 + + ; CHECK: Destination type must be a scalable vector + %4:_(<2 x s32>) = G_SPLAT_VECTOR %0 + + ; CHECK: Source type must be a scalar + %5:_() = G_SPLAT_VECTOR %1 + + ; CHECK: Source type must be a scalar + %6:_() = G_SPLAT_VECTOR %2 + + ; CHECK: Element type of the destination must be the same type as the source type + %7:_() = G_SPLAT_VECTOR %0 +... diff --git a/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp b/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp index 73837279701a..33155d2c9a96 100644 --- a/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp +++ b/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp @@ -147,9 +147,9 @@ TEST_F(AArch64GISelMITest, LowerRotatesVector) { LLT S32 = LLT::scalar(32); LLT V4S32 = LLT::fixed_vector(4, S32); auto SrcTrunc = B.buildTrunc(S32, Copies[0]); - auto Src = B.buildSplatVector(V4S32, SrcTrunc); + auto Src = B.buildSplatBuildVector(V4S32, SrcTrunc); auto AmtTrunc = B.buildTrunc(S32, Copies[1]); - auto Amt = B.buildSplatVector(V4S32, AmtTrunc); + auto Amt = B.buildSplatBuildVector(V4S32, AmtTrunc); auto ROTR = B.buildInstr(TargetOpcode::G_ROTR, {V4S32}, {Src, Amt}); AInfo Info(MF->getSubtarget()); diff --git a/llvm/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp b/llvm/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp index f52e49df0bcd..59a86fa5646f 100644 --- a/llvm/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp +++ b/llvm/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp @@ -61,7 +61,7 @@ TEST_F(AArch64GISelMITest, MatchIntConstantSplat) { LLT v4s64 = LLT::fixed_vector(4, s64); MachineInstrBuilder FortyTwoSplat = - B.buildSplatVector(v4s64, B.buildConstant(s64, 42)); + B.buildSplatBuildVector(v4s64, B.buildConstant(s64, 42)); int64_t Cst; EXPECT_TRUE(mi_match(FortyTwoSplat.getReg(0), *MRI, m_ICstOrSplat(Cst))); EXPECT_EQ(Cst, 42); @@ -625,7 +625,7 @@ TEST_F(AArch64GISelMITest, MatchSpecificConstantSplat) { LLT v4s64 = LLT::fixed_vector(4, s64); MachineInstrBuilder FortyTwoSplat = - B.buildSplatVector(v4s64, B.buildConstant(s64, 42)); + B.buildSplatBuildVector(v4s64, B.buildConstant(s64, 42)); MachineInstrBuilder FortyTwo = B.buildConstant(s64, 42); EXPECT_TRUE(mi_match(FortyTwoSplat.getReg(0), *MRI, m_SpecificICstSplat(42))); @@ -655,7 +655,7 @@ TEST_F(AArch64GISelMITest, MatchSpecificConstantOrSplat) { LLT v4s64 = LLT::fixed_vector(4, s64); MachineInstrBuilder FortyTwoSplat = - B.buildSplatVector(v4s64, B.buildConstant(s64, 42)); + B.buildSplatBuildVector(v4s64, B.buildConstant(s64, 42)); MachineInstrBuilder FortyTwo = B.buildConstant(s64, 42); EXPECT_TRUE( -- GitLab From 3714f937b835c06c8c32ca4f3f61ba2317db2296 Mon Sep 17 00:00:00 2001 From: Edgar Date: Thu, 7 Mar 2024 18:10:46 +0100 Subject: [PATCH 465/929] [MLIR] Add llvm (debug) attributes to CAPI (#83992) This PR adds the following to the mlir c api: - The disctinct mlir builtin attribute. - LLVM attributes (mostly debug related ones) --- mlir/include/mlir-c/BuiltinAttributes.h | 4 + mlir/include/mlir-c/Dialect/LLVM.h | 231 ++++++++++++++++++++++++ mlir/lib/CAPI/Dialect/LLVM.cpp | 207 +++++++++++++++++++++ mlir/lib/CAPI/IR/BuiltinAttributes.cpp | 4 + mlir/test/CAPI/ir.c | 4 + mlir/test/CAPI/llvm.c | 113 +++++++++++- 6 files changed, 562 insertions(+), 1 deletion(-) diff --git a/mlir/include/mlir-c/BuiltinAttributes.h b/mlir/include/mlir-c/BuiltinAttributes.h index 01d1b6008f5e..231eb83b5e26 100644 --- a/mlir/include/mlir-c/BuiltinAttributes.h +++ b/mlir/include/mlir-c/BuiltinAttributes.h @@ -266,6 +266,10 @@ mlirSymbolRefAttrGetNestedReference(MlirAttribute attr, intptr_t pos); /// Returns the typeID of an SymbolRef attribute. MLIR_CAPI_EXPORTED MlirTypeID mlirSymbolRefAttrGetTypeID(void); +/// Creates a DisctinctAttr with the referenced attribute. +MLIR_CAPI_EXPORTED MlirAttribute +mlirDisctinctAttrCreate(MlirAttribute referencedAttr); + //===----------------------------------------------------------------------===// // Flat SymbolRef attribute. //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir-c/Dialect/LLVM.h b/mlir/include/mlir-c/Dialect/LLVM.h index ac216b01f364..d823afb659c8 100644 --- a/mlir/include/mlir-c/Dialect/LLVM.h +++ b/mlir/include/mlir-c/Dialect/LLVM.h @@ -11,6 +11,7 @@ #define MLIR_C_DIALECT_LLVM_H #include "mlir-c/IR.h" +#include "mlir-c/Support.h" #ifdef __cplusplus extern "C" { @@ -98,6 +99,236 @@ MLIR_CAPI_EXPORTED MlirLogicalResult mlirLLVMStructTypeSetBody(MlirType structType, intptr_t nFieldTypes, MlirType const *fieldTypes, bool isPacked); +enum MlirLLVMCConv { + MlirLLVMCConvC = 0, + MlirLLVMCConvFast = 8, + MlirLLVMCConvCold = 9, + MlirLLVMCConvGHC = 10, + MlirLLVMCConvHiPE = 11, + MlirLLVMCConvAnyReg = 13, + MlirLLVMCConvPreserveMost = 14, + MlirLLVMCConvPreserveAll = 15, + MlirLLVMCConvSwift = 16, + MlirLLVMCConvCXX_FAST_TLS = 17, + MlirLLVMCConvTail = 18, + MlirLLVMCConvCFGuard_Check = 19, + MlirLLVMCConvSwiftTail = 20, + MlirLLVMCConvX86_StdCall = 64, + MlirLLVMCConvX86_FastCall = 65, + MlirLLVMCConvARM_APCS = 66, + MlirLLVMCConvARM_AAPCS = 67, + MlirLLVMCConvARM_AAPCS_VFP = 68, + MlirLLVMCConvMSP430_INTR = 69, + MlirLLVMCConvX86_ThisCall = 70, + MlirLLVMCConvPTX_Kernel = 71, + MlirLLVMCConvPTX_Device = 72, + MlirLLVMCConvSPIR_FUNC = 75, + MlirLLVMCConvSPIR_KERNEL = 76, + MlirLLVMCConvIntel_OCL_BI = 77, + MlirLLVMCConvX86_64_SysV = 78, + MlirLLVMCConvWin64 = 79, + MlirLLVMCConvX86_VectorCall = 80, + MlirLLVMCConvDUMMY_HHVM = 81, + MlirLLVMCConvDUMMY_HHVM_C = 82, + MlirLLVMCConvX86_INTR = 83, + MlirLLVMCConvAVR_INTR = 84, + MlirLLVMCConvAVR_BUILTIN = 86, + MlirLLVMCConvAMDGPU_VS = 87, + MlirLLVMCConvAMDGPU_GS = 88, + MlirLLVMCConvAMDGPU_CS = 90, + MlirLLVMCConvAMDGPU_KERNEL = 91, + MlirLLVMCConvX86_RegCall = 92, + MlirLLVMCConvAMDGPU_HS = 93, + MlirLLVMCConvMSP430_BUILTIN = 94, + MlirLLVMCConvAMDGPU_LS = 95, + MlirLLVMCConvAMDGPU_ES = 96, + MlirLLVMCConvAArch64_VectorCall = 97, + MlirLLVMCConvAArch64_SVE_VectorCall = 98, + MlirLLVMCConvWASM_EmscriptenInvoke = 99, + MlirLLVMCConvAMDGPU_Gfx = 100, + MlirLLVMCConvM68k_INTR = 101, +}; +typedef enum MlirLLVMCConv MlirLLVMCConv; + +/// Creates a LLVM CConv attribute. +MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMCConvAttrGet(MlirContext ctx, + MlirLLVMCConv cconv); + +enum MlirLLVMComdat { + MlirLLVMComdatAny = 0, + MlirLLVMComdatExactMatch = 1, + MlirLLVMComdatLargest = 2, + MlirLLVMComdatNoDeduplicate = 3, + MlirLLVMComdatSameSize = 4, +}; +typedef enum MlirLLVMComdat MlirLLVMComdat; + +/// Creates a LLVM Comdat attribute. +MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMComdatAttrGet(MlirContext ctx, + MlirLLVMComdat comdat); + +enum MlirLLVMLinkage { + MlirLLVMLinkagePrivate = 0, + MlirLLVMLinkageInternal = 1, + MlirLLVMLinkageAvailableExternally = 2, + MlirLLVMLinkageLinkonce = 3, + MlirLLVMLinkageWeak = 4, + MlirLLVMLinkageCommon = 5, + MlirLLVMLinkageAppending = 6, + MlirLLVMLinkageExternWeak = 7, + MlirLLVMLinkageLinkonceODR = 8, + MlirLLVMLinkageWeakODR = 9, + MlirLLVMLinkageExternal = 10, +}; +typedef enum MlirLLVMLinkage MlirLLVMLinkage; + +/// Creates a LLVM Linkage attribute. +MLIR_CAPI_EXPORTED MlirAttribute +mlirLLVMLinkageAttrGet(MlirContext ctx, MlirLLVMLinkage linkage); + +/// Creates a LLVM DINullType attribute. +MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMDINullTypeAttrGet(MlirContext ctx); + +/// Creates a LLVM DIExpressionElem attribute. +MLIR_CAPI_EXPORTED MlirAttribute +mlirLLVMDIExpressionElemAttrGet(MlirContext ctx, unsigned int opcode, + intptr_t nArguments, uint64_t const *arguments); + +/// Creates a LLVM DIExpression attribute. +MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMDIExpressionAttrGet( + MlirContext ctx, intptr_t nOperations, MlirAttribute const *operations); + +enum MlirLLVMTypeEncoding { + MlirLLVMTypeEncodingAddress = 0x1, + MlirLLVMTypeEncodingBoolean = 0x2, + MlirLLVMTypeEncodingComplexFloat = 0x31, + MlirLLVMTypeEncodingFloatT = 0x4, + MlirLLVMTypeEncodingSigned = 0x5, + MlirLLVMTypeEncodingSignedChar = 0x6, + MlirLLVMTypeEncodingUnsigned = 0x7, + MlirLLVMTypeEncodingUnsignedChar = 0x08, + MlirLLVMTypeEncodingImaginaryFloat = 0x09, + MlirLLVMTypeEncodingPackedDecimal = 0x0a, + MlirLLVMTypeEncodingNumericString = 0x0b, + MlirLLVMTypeEncodingEdited = 0x0c, + MlirLLVMTypeEncodingSignedFixed = 0x0d, + MlirLLVMTypeEncodingUnsignedFixed = 0x0e, + MlirLLVMTypeEncodingDecimalFloat = 0x0f, + MlirLLVMTypeEncodingUTF = 0x10, + MlirLLVMTypeEncodingUCS = 0x11, + MlirLLVMTypeEncodingASCII = 0x12, + MlirLLVMTypeEncodingLoUser = 0x80, + MlirLLVMTypeEncodingHiUser = 0xff, +}; +typedef enum MlirLLVMTypeEncoding MlirLLVMTypeEncoding; + +/// Creates a LLVM DIBasicType attribute. +MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMDIBasicTypeAttrGet( + MlirContext ctx, unsigned int tag, MlirAttribute name, uint64_t sizeInBits, + MlirLLVMTypeEncoding encoding); + +/// Creates a LLVM DICompositeType attribute. +MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMDICompositeTypeAttrGet( + MlirContext ctx, unsigned int tag, MlirAttribute name, MlirAttribute file, + uint32_t line, MlirAttribute scope, MlirAttribute baseType, int64_t flags, + uint64_t sizeInBits, uint64_t alignInBits, intptr_t nElements, + MlirAttribute const *elements); + +/// Creates a LLVM DIDerivedType attribute. +MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMDIDerivedTypeAttrGet( + MlirContext ctx, unsigned int tag, MlirAttribute name, + MlirAttribute baseType, uint64_t sizeInBits, uint32_t alignInBits, + uint64_t offsetInBits); + +/// Gets the base type from a LLVM DIDerivedType attribute. +MLIR_CAPI_EXPORTED MlirAttribute +mlirLLVMDIDerivedTypeAttrGetBaseType(MlirAttribute diDerivedType); + +/// Creates a LLVM DIFileAttr attribute. +MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMDIFileAttrGet(MlirContext ctx, + MlirAttribute name, + MlirAttribute directory); + +enum MlirLLVMDIEmissionKind { + MlirLLVMDIEmissionKindNone = 0, + MlirLLVMDIEmissionKindFull = 1, + MlirLLVMDIEmissionKindLineTablesOnly = 2, + MlirLLVMDIEmissionKindDebugDirectivesOnly = 3, +}; +typedef enum MlirLLVMDIEmissionKind MlirLLVMDIEmissionKind; + +/// Creates a LLVM DICompileUnit attribute. +MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMDICompileUnitAttrGet( + MlirContext ctx, MlirAttribute id, unsigned int sourceLanguage, + MlirAttribute file, MlirAttribute producer, bool isOptimized, + MlirLLVMDIEmissionKind emissionKind); + +/// Creates a LLVM DIFlags attribute. +MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMDIFlagsAttrGet(MlirContext ctx, + uint64_t value); + +/// Creates a LLVM DILexicalBlock attribute. +MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMDILexicalBlockAttrGet( + MlirContext ctx, MlirAttribute scope, MlirAttribute file, unsigned int line, + unsigned int column); + +/// Creates a LLVM DILexicalBlockFile attribute. +MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMDILexicalBlockFileAttrGet( + MlirContext ctx, MlirAttribute scope, MlirAttribute file, + unsigned int discriminator); + +/// Creates a LLVM DILocalVariableAttr attribute. +MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMDILocalVariableAttrGet( + MlirContext ctx, MlirAttribute scope, MlirAttribute name, + MlirAttribute diFile, unsigned int line, unsigned int arg, + unsigned int alignInBits, MlirAttribute diType); + +/// Creates a LLVM DISubprogramAttr attribute. +MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMDISubprogramAttrGet( + MlirContext ctx, MlirAttribute id, MlirAttribute compileUnit, + MlirAttribute scope, MlirAttribute name, MlirAttribute linkageName, + MlirAttribute file, unsigned int line, unsigned int scopeLine, + uint64_t subprogramFlags, MlirAttribute type); + +/// Gets the scope from this DISubprogramAttr. +MLIR_CAPI_EXPORTED MlirAttribute +mlirLLVMDISubprogramAttrGetScope(MlirAttribute diSubprogram); + +/// Gets the line from this DISubprogramAttr. +MLIR_CAPI_EXPORTED unsigned int +mlirLLVMDISubprogramAttrGetLine(MlirAttribute diSubprogram); + +/// Gets the scope line from this DISubprogram. +MLIR_CAPI_EXPORTED unsigned int +mlirLLVMDISubprogramAttrGetScopeLine(MlirAttribute diSubprogram); + +/// Gets the compile unit from this DISubprogram. +MLIR_CAPI_EXPORTED MlirAttribute +mlirLLVMDISubprogramAttrGetCompileUnit(MlirAttribute diSubprogram); + +/// Gets the file from this DISubprogramAttr. +MLIR_CAPI_EXPORTED MlirAttribute +mlirLLVMDISubprogramAttrGetFile(MlirAttribute diSubprogram); + +/// Gets the type from this DISubprogramAttr. +MLIR_CAPI_EXPORTED MlirAttribute +mlirLLVMDISubprogramAttrGetType(MlirAttribute diSubprogram); + +/// Creates a LLVM DISubroutineTypeAttr attribute. +MLIR_CAPI_EXPORTED MlirAttribute +mlirLLVMDISubroutineTypeAttrGet(MlirContext ctx, unsigned int callingConvention, + intptr_t nTypes, MlirAttribute const *types); + +/// Creates a LLVM DIModuleAttr attribute. +MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMDIModuleAttrGet( + MlirContext ctx, MlirAttribute file, MlirAttribute scope, + MlirAttribute name, MlirAttribute configMacros, MlirAttribute includePath, + MlirAttribute apinotes, unsigned int line, bool isDecl); + +/// Gets the scope of this DIModuleAttr. +MLIR_CAPI_EXPORTED MlirAttribute +mlirLLVMDIModuleAttrGetScope(MlirAttribute diModule); + #ifdef __cplusplus } #endif diff --git a/mlir/lib/CAPI/Dialect/LLVM.cpp b/mlir/lib/CAPI/Dialect/LLVM.cpp index 642018a814ca..2d938ce5f483 100644 --- a/mlir/lib/CAPI/Dialect/LLVM.cpp +++ b/mlir/lib/CAPI/Dialect/LLVM.cpp @@ -7,9 +7,16 @@ //===----------------------------------------------------------------------===// #include "mlir-c/Dialect/LLVM.h" +#include "mlir-c/IR.h" +#include "mlir-c/Support.h" #include "mlir/CAPI/Registration.h" +#include "mlir/CAPI/Wrap.h" +#include "mlir/Dialect/LLVMIR/LLVMAttrs.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" #include "mlir/Dialect/LLVMIR/LLVMTypes.h" +#include "llvm-c/Core.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/SmallVectorExtras.h" using namespace mlir; using namespace mlir::LLVM; @@ -110,3 +117,203 @@ MlirLogicalResult mlirLLVMStructTypeSetBody(MlirType structType, cast(unwrap(structType)) .setBody(unwrapList(nFieldTypes, fieldTypes, fields), isPacked)); } + +MlirAttribute mlirLLVMDIExpressionElemAttrGet(MlirContext ctx, + unsigned int opcode, + intptr_t nArguments, + uint64_t const *arguments) { + auto list = ArrayRef(arguments, nArguments); + return wrap(DIExpressionElemAttr::get(unwrap(ctx), opcode, list)); +} + +MlirAttribute mlirLLVMDIExpressionAttrGet(MlirContext ctx, intptr_t nOperations, + MlirAttribute const *operations) { + SmallVector attrStorage; + attrStorage.reserve(nOperations); + + return wrap(DIExpressionAttr::get( + unwrap(ctx), + llvm::map_to_vector( + unwrapList(nOperations, operations, attrStorage), + [](Attribute a) { return a.cast(); }))); +} + +MlirAttribute mlirLLVMDINullTypeAttrGet(MlirContext ctx) { + return wrap(DINullTypeAttr::get(unwrap(ctx))); +} + +MlirAttribute mlirLLVMDIBasicTypeAttrGet(MlirContext ctx, unsigned int tag, + MlirAttribute name, + uint64_t sizeInBits, + MlirLLVMTypeEncoding encoding) { + + return wrap(DIBasicTypeAttr::get( + unwrap(ctx), tag, cast(unwrap(name)), sizeInBits, encoding)); +} + +MlirAttribute mlirLLVMDICompositeTypeAttrGet( + MlirContext ctx, unsigned int tag, MlirAttribute name, MlirAttribute file, + uint32_t line, MlirAttribute scope, MlirAttribute baseType, int64_t flags, + uint64_t sizeInBits, uint64_t alignInBits, intptr_t nElements, + MlirAttribute const *elements) { + SmallVector elementsStorage; + elementsStorage.reserve(nElements); + + return wrap(DICompositeTypeAttr::get( + unwrap(ctx), tag, cast(unwrap(name)), + cast(unwrap(file)), line, cast(unwrap(scope)), + cast(unwrap(baseType)), DIFlags(flags), sizeInBits, + alignInBits, + llvm::map_to_vector(unwrapList(nElements, elements, elementsStorage), + [](Attribute a) { return a.cast(); }))); +} + +MlirAttribute mlirLLVMDIDerivedTypeAttrGet(MlirContext ctx, unsigned int tag, + MlirAttribute name, + MlirAttribute baseType, + uint64_t sizeInBits, + uint32_t alignInBits, + uint64_t offsetInBits) { + return wrap(DIDerivedTypeAttr::get(unwrap(ctx), tag, + cast(unwrap(name)), + cast(unwrap(baseType)), + sizeInBits, alignInBits, offsetInBits)); +} + +MlirAttribute +mlirLLVMDIDerivedTypeAttrGetBaseType(MlirAttribute diDerivedType) { + return wrap(cast(unwrap(diDerivedType)).getBaseType()); +} + +MlirAttribute mlirLLVMCConvAttrGet(MlirContext ctx, MlirLLVMCConv cconv) { + return wrap(CConvAttr::get(unwrap(ctx), CConv(cconv))); +} + +MlirAttribute mlirLLVMComdatAttrGet(MlirContext ctx, MlirLLVMComdat comdat) { + return wrap(ComdatAttr::get(unwrap(ctx), comdat::Comdat(comdat))); +} + +MlirAttribute mlirLLVMLinkageAttrGet(MlirContext ctx, MlirLLVMLinkage linkage) { + return wrap(LinkageAttr::get(unwrap(ctx), linkage::Linkage(linkage))); +} + +MlirAttribute mlirLLVMDIFileAttrGet(MlirContext ctx, MlirAttribute name, + MlirAttribute directory) { + return wrap(DIFileAttr::get(unwrap(ctx), cast(unwrap(name)), + cast(unwrap(directory)))); +} + +MlirAttribute +mlirLLVMDICompileUnitAttrGet(MlirContext ctx, MlirAttribute id, + unsigned int sourceLanguage, MlirAttribute file, + MlirAttribute producer, bool isOptimized, + MlirLLVMDIEmissionKind emissionKind) { + return wrap(DICompileUnitAttr::get( + unwrap(ctx), cast(unwrap(id)), sourceLanguage, + cast(unwrap(file)), cast(unwrap(producer)), + isOptimized, DIEmissionKind(emissionKind))); +} + +MlirAttribute mlirLLVMDIFlagsAttrGet(MlirContext ctx, uint64_t value) { + return wrap(DIFlagsAttr::get(unwrap(ctx), DIFlags(value))); +} + +MlirAttribute mlirLLVMDILexicalBlockAttrGet(MlirContext ctx, + MlirAttribute scope, + MlirAttribute file, + unsigned int line, + unsigned int column) { + return wrap( + DILexicalBlockAttr::get(unwrap(ctx), cast(unwrap(scope)), + cast(unwrap(file)), line, column)); +} + +MlirAttribute mlirLLVMDILexicalBlockFileAttrGet(MlirContext ctx, + MlirAttribute scope, + MlirAttribute file, + unsigned int discriminator) { + return wrap(DILexicalBlockFileAttr::get( + unwrap(ctx), cast(unwrap(scope)), + cast(unwrap(file)), discriminator)); +} + +MlirAttribute +mlirLLVMDILocalVariableAttrGet(MlirContext ctx, MlirAttribute scope, + MlirAttribute name, MlirAttribute diFile, + unsigned int line, unsigned int arg, + unsigned int alignInBits, MlirAttribute diType) { + return wrap(DILocalVariableAttr::get( + unwrap(ctx), cast(unwrap(scope)), + cast(unwrap(name)), cast(unwrap(diFile)), line, + arg, alignInBits, cast(unwrap(diType)))); +} + +MlirAttribute mlirLLVMDISubroutineTypeAttrGet(MlirContext ctx, + unsigned int callingConvention, + intptr_t nTypes, + MlirAttribute const *types) { + SmallVector attrStorage; + attrStorage.reserve(nTypes); + + return wrap(DISubroutineTypeAttr::get( + unwrap(ctx), callingConvention, + llvm::map_to_vector(unwrapList(nTypes, types, attrStorage), + [](Attribute a) { return a.cast(); }))); +} + +MlirAttribute mlirLLVMDISubprogramAttrGet( + MlirContext ctx, MlirAttribute id, MlirAttribute compileUnit, + MlirAttribute scope, MlirAttribute name, MlirAttribute linkageName, + MlirAttribute file, unsigned int line, unsigned int scopeLine, + uint64_t subprogramFlags, MlirAttribute type) { + return wrap(DISubprogramAttr::get( + unwrap(ctx), cast(unwrap(id)), + cast(unwrap(compileUnit)), + cast(unwrap(scope)), cast(unwrap(name)), + cast(unwrap(linkageName)), cast(unwrap(file)), + line, scopeLine, DISubprogramFlags(subprogramFlags), + cast(unwrap(type)))); +} + +MlirAttribute mlirLLVMDISubprogramAttrGetScope(MlirAttribute diSubprogram) { + return wrap(cast(unwrap(diSubprogram)).getScope()); +} + +unsigned int mlirLLVMDISubprogramAttrGetLine(MlirAttribute diSubprogram) { + return cast(unwrap(diSubprogram)).getLine(); +} + +unsigned int mlirLLVMDISubprogramAttrGetScopeLine(MlirAttribute diSubprogram) { + return cast(unwrap(diSubprogram)).getScopeLine(); +} + +MlirAttribute +mlirLLVMDISubprogramAttrGetCompileUnit(MlirAttribute diSubprogram) { + return wrap(cast(unwrap(diSubprogram)).getCompileUnit()); +} + +MlirAttribute mlirLLVMDISubprogramAttrGetFile(MlirAttribute diSubprogram) { + return wrap(cast(unwrap(diSubprogram)).getFile()); +} + +MlirAttribute mlirLLVMDISubprogramAttrGetType(MlirAttribute diSubprogram) { + return wrap(cast(unwrap(diSubprogram)).getType()); +} + +MlirAttribute mlirLLVMDIModuleAttrGet(MlirContext ctx, MlirAttribute file, + MlirAttribute scope, MlirAttribute name, + MlirAttribute configMacros, + MlirAttribute includePath, + MlirAttribute apinotes, unsigned int line, + bool isDecl) { + return wrap(DIModuleAttr::get( + unwrap(ctx), cast(unwrap(file)), + cast(unwrap(scope)), cast(unwrap(name)), + cast(unwrap(configMacros)), + cast(unwrap(includePath)), cast(unwrap(apinotes)), + line, isDecl)); +} + +MlirAttribute mlirLLVMDIModuleAttrGetScope(MlirAttribute diModule) { + return wrap(cast(unwrap(diModule)).getScope()); +} diff --git a/mlir/lib/CAPI/IR/BuiltinAttributes.cpp b/mlir/lib/CAPI/IR/BuiltinAttributes.cpp index b3066ee0c28b..726af884668b 100644 --- a/mlir/lib/CAPI/IR/BuiltinAttributes.cpp +++ b/mlir/lib/CAPI/IR/BuiltinAttributes.cpp @@ -289,6 +289,10 @@ MlirTypeID mlirSymbolRefAttrGetTypeID(void) { return wrap(SymbolRefAttr::getTypeID()); } +MlirAttribute mlirDisctinctAttrCreate(MlirAttribute referencedAttr) { + return wrap(mlir::DistinctAttr::create(unwrap(referencedAttr))); +} + //===----------------------------------------------------------------------===// // Flat SymbolRef attribute. //===----------------------------------------------------------------------===// diff --git a/mlir/test/CAPI/ir.c b/mlir/test/CAPI/ir.c index a9850c0a132e..8e79338c57a2 100644 --- a/mlir/test/CAPI/ir.c +++ b/mlir/test/CAPI/ir.c @@ -1482,6 +1482,10 @@ int printAffineMap(MlirContext ctx) { // CHECK: (d0, d1, d2) -> (d0) // CHECK: (d0, d1, d2) -> (d2) + // CHECK: distinct[0]<"foo"> + mlirAttributeDump(mlirDisctinctAttrCreate( + mlirStringAttrGet(ctx, mlirStringRefCreateFromCString("foo")))); + return 0; } diff --git a/mlir/test/CAPI/llvm.c b/mlir/test/CAPI/llvm.c index 1817988dd67d..2fd98b29f487 100644 --- a/mlir/test/CAPI/llvm.c +++ b/mlir/test/CAPI/llvm.c @@ -10,9 +10,12 @@ // RUN: mlir-capi-llvm-test 2>&1 | FileCheck %s #include "mlir-c/Dialect/LLVM.h" +#include "mlir-c/BuiltinAttributes.h" #include "mlir-c/BuiltinTypes.h" #include "mlir-c/IR.h" #include "mlir-c/Support.h" +#include "llvm-c/Core.h" +#include "llvm-c/DebugInfo.h" #include #include @@ -77,7 +80,7 @@ static void testTypeCreation(MlirContext ctx) { // CHECK-LABEL: testStructTypeCreation static int testStructTypeCreation(MlirContext ctx) { - fprintf(stderr, "testStructTypeCreation"); + fprintf(stderr, "testStructTypeCreation\n"); // CHECK: !llvm.struct<()> mlirTypeDump(mlirLLVMStructTypeLiteralGet(ctx, /*nFieldTypes=*/0, @@ -225,12 +228,120 @@ static int testStructTypeCreation(MlirContext ctx) { return 0; } +// CHECK-LABEL: testLLVMAttributes +static void testLLVMAttributes(MlirContext ctx) { + fprintf(stderr, "testLLVMAttributes\n"); + + // CHECK: #llvm.linkage + mlirAttributeDump(mlirLLVMLinkageAttrGet(ctx, MlirLLVMLinkageInternal)); + // CHECK: #llvm.cconv + mlirAttributeDump(mlirLLVMCConvAttrGet(ctx, MlirLLVMCConvC)); + // CHECK: #llvm + mlirAttributeDump(mlirLLVMComdatAttrGet(ctx, MlirLLVMComdatAny)); +} + +// CHECK-LABEL: testDebugInfoAttributes +static void testDebugInfoAttributes(MlirContext ctx) { + fprintf(stderr, "testDebugInfoAttributes\n"); + + MlirAttribute foo = + mlirStringAttrGet(ctx, mlirStringRefCreateFromCString("foo")); + MlirAttribute bar = + mlirStringAttrGet(ctx, mlirStringRefCreateFromCString("bar")); + MlirAttribute id = mlirDisctinctAttrCreate(foo); + + // CHECK: #llvm.di_null_type + mlirAttributeDump(mlirLLVMDINullTypeAttrGet(ctx)); + + // CHECK: #llvm.di_basic_type + MlirAttribute di_type = + mlirLLVMDIBasicTypeAttrGet(ctx, 0, foo, 64, MlirLLVMTypeEncodingSigned); + mlirAttributeDump(di_type); + + MlirAttribute file = mlirLLVMDIFileAttrGet(ctx, foo, bar); + + // CHECK: #llvm.di_file<"foo" in "bar"> + mlirAttributeDump(file); + + MlirAttribute compile_unit = + mlirLLVMDICompileUnitAttrGet(ctx, id, LLVMDWARFSourceLanguageC99, file, + foo, false, MlirLLVMDIEmissionKindFull); + + // CHECK: #llvm.di_compile_unit<{{.*}}> + mlirAttributeDump(compile_unit); + + MlirAttribute di_module = mlirLLVMDIModuleAttrGet( + ctx, file, compile_unit, foo, + mlirStringAttrGet(ctx, mlirStringRefCreateFromCString("")), bar, foo, 1, + 0); + // CHECK: #llvm.di_module<{{.*}}> + mlirAttributeDump(di_module); + + // CHECK: #llvm.di_compile_unit<{{.*}}> + mlirAttributeDump(mlirLLVMDIModuleAttrGetScope(di_module)); + + // CHECK: 1 : i32 + mlirAttributeDump(mlirLLVMDIFlagsAttrGet(ctx, 0x1)); + + // CHECK: #llvm.di_lexical_block<{{.*}}> + mlirAttributeDump( + mlirLLVMDILexicalBlockAttrGet(ctx, compile_unit, file, 1, 2)); + + // CHECK: #llvm.di_lexical_block_file<{{.*}}> + mlirAttributeDump( + mlirLLVMDILexicalBlockFileAttrGet(ctx, compile_unit, file, 3)); + + // CHECK: #llvm.di_local_variable<{{.*}}> + mlirAttributeDump(mlirLLVMDILocalVariableAttrGet(ctx, compile_unit, foo, file, + 1, 0, 8, di_type)); + // CHECK: #llvm.di_derived_type<{{.*}}> + mlirAttributeDump( + mlirLLVMDIDerivedTypeAttrGet(ctx, 0, bar, di_type, 64, 8, 0)); + + // CHECK: #llvm.di_composite_type<{{.*}}> + mlirAttributeDump(mlirLLVMDICompositeTypeAttrGet( + ctx, 0, foo, file, 1, compile_unit, di_type, 0, 64, 8, 1, &di_type)); + + MlirAttribute subroutine_type = + mlirLLVMDISubroutineTypeAttrGet(ctx, 0x0, 1, &di_type); + + // CHECK: #llvm.di_subroutine_type<{{.*}}> + mlirAttributeDump(subroutine_type); + + MlirAttribute di_subprogram = + mlirLLVMDISubprogramAttrGet(ctx, id, compile_unit, compile_unit, foo, bar, + file, 1, 2, 0, subroutine_type); + // CHECK: #llvm.di_subprogram<{{.*}}> + mlirAttributeDump(di_subprogram); + + // CHECK: #llvm.di_compile_unit<{{.*}}> + mlirAttributeDump(mlirLLVMDISubprogramAttrGetScope(di_subprogram)); + + // CHECK: #llvm.di_file<{{.*}}> + mlirAttributeDump(mlirLLVMDISubprogramAttrGetFile(di_subprogram)); + + // CHECK: #llvm.di_subroutine_type<{{.*}}> + mlirAttributeDump(mlirLLVMDISubprogramAttrGetType(di_subprogram)); + + MlirAttribute expression_elem = + mlirLLVMDIExpressionElemAttrGet(ctx, 1, 1, &(uint64_t){1}); + + // CHECK: #llvm + mlirAttributeDump(expression_elem); + + // CHECK: #llvm.di_expression<[(1)]> + mlirAttributeDump(mlirLLVMDIExpressionAttrGet(ctx, 1, &expression_elem)); +} + int main(void) { MlirContext ctx = mlirContextCreate(); mlirDialectHandleRegisterDialect(mlirGetDialectHandle__llvm__(), ctx); mlirContextGetOrLoadDialect(ctx, mlirStringRefCreateFromCString("llvm")); testTypeCreation(ctx); int result = testStructTypeCreation(ctx); + testLLVMAttributes(ctx); + testDebugInfoAttributes(ctx); mlirContextDestroy(ctx); if (result) fprintf(stderr, "FAILED: code %d", result); -- GitLab From 6157538d9e4eea135fb863b972c577f648c21641 Mon Sep 17 00:00:00 2001 From: Jon Chesterfield Date: Thu, 7 Mar 2024 17:36:28 +0000 Subject: [PATCH 466/929] [InstCombine] ptrmask of gep for dynamic pointer aligment (#80002) Targets the dynamic realignment pattern of `(Ptr + Align - 1) & -Align;` as implemented by gep then ptrmask. Specifically, when the pointer already has alignment information, dynamically realigning it to less than is already known should be a no-op. Discovered while writing test cases for another patch. For the zero low bits of a known aligned pointer, adding the gep index then removing it with a mask is a no-op. Folding the ptrmask effect entirely into the gep is the ideal result as that unblocks other optimisations that are not aware of ptrmask. In some other cases the gep is known to be dead and is removed without changing the ptrmask. In the least effective case, this transform creates a new gep with a rounded-down index and still leaves the ptrmask unchanged. That simplified gep is still a minor improvement, geps are cheap and ptrmask occurs in address calculation contexts so I don't think it's worth special casing to avoid the extra instruction. --- .../InstCombineSimplifyDemanded.cpp | 38 ++++ llvm/test/Transforms/InstCombine/ptrmask.ll | 164 +++++++++++++++++- 2 files changed, 198 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp index 1b963a7de4a8..c691c8b1c55b 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp @@ -997,6 +997,44 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask, I, 1, (DemandedMask & ~LHSKnown.Zero).zextOrTrunc(MaskWidth))) return I; + // Combine: + // (ptrmask (getelementptr i8, ptr p, imm i), imm mask) + // -> (ptrmask (getelementptr i8, ptr p, imm (i & mask)), imm mask) + // where only the low bits known to be zero in the pointer are changed + Value *InnerPtr; + uint64_t GEPIndex; + uint64_t PtrMaskImmediate; + if (match(I, m_Intrinsic( + m_PtrAdd(m_Value(InnerPtr), m_ConstantInt(GEPIndex)), + m_ConstantInt(PtrMaskImmediate)))) { + + LHSKnown = computeKnownBits(InnerPtr, Depth + 1, I); + if (!LHSKnown.isZero()) { + const unsigned trailingZeros = LHSKnown.countMinTrailingZeros(); + uint64_t PointerAlignBits = (uint64_t(1) << trailingZeros) - 1; + + uint64_t HighBitsGEPIndex = GEPIndex & ~PointerAlignBits; + uint64_t MaskedLowBitsGEPIndex = + GEPIndex & PointerAlignBits & PtrMaskImmediate; + + uint64_t MaskedGEPIndex = HighBitsGEPIndex | MaskedLowBitsGEPIndex; + + if (MaskedGEPIndex != GEPIndex) { + auto *GEP = cast(II->getArgOperand(0)); + Builder.SetInsertPoint(I); + Type *GEPIndexType = + DL.getIndexType(GEP->getPointerOperand()->getType()); + Value *MaskedGEP = Builder.CreateGEP( + GEP->getSourceElementType(), InnerPtr, + ConstantInt::get(GEPIndexType, MaskedGEPIndex), + GEP->getName(), GEP->isInBounds()); + + replaceOperand(*I, 0, MaskedGEP); + return I; + } + } + } + break; } diff --git a/llvm/test/Transforms/InstCombine/ptrmask.ll b/llvm/test/Transforms/InstCombine/ptrmask.ll index afeb5d5251d0..4631b81cd1ce 100644 --- a/llvm/test/Transforms/InstCombine/ptrmask.ll +++ b/llvm/test/Transforms/InstCombine/ptrmask.ll @@ -80,12 +80,12 @@ define ptr addrspace(1) @ptrmask_combine_consecutive_preserve_attrs_todo2(ptr ad define ptr @ptrmask_combine_add_nonnull(ptr %p) { ; CHECK-LABEL: define ptr @ptrmask_combine_add_nonnull ; CHECK-SAME: (ptr [[P:%.*]]) { -; CHECK-NEXT: [[PM0:%.*]] = call align 64 ptr @llvm.ptrmask.p0.i64(ptr [[P]], i64 -64) -; CHECK-NEXT: [[PGEP:%.*]] = getelementptr i8, ptr [[PM0]], i64 33 -; CHECK-NEXT: [[R:%.*]] = call nonnull align 32 ptr @llvm.ptrmask.p0.i64(ptr [[PGEP]], i64 -32) +; CHECK-NEXT: [[PM0:%.*]] = call align 4 ptr @llvm.ptrmask.p0.i64(ptr [[P]], i64 -60) +; CHECK-NEXT: [[PGEP1:%.*]] = getelementptr i8, ptr [[PM0]], i64 32 +; CHECK-NEXT: [[R:%.*]] = call nonnull align 32 ptr @llvm.ptrmask.p0.i64(ptr [[PGEP1]], i64 -32) ; CHECK-NEXT: ret ptr [[R]] ; - %pm0 = call ptr @llvm.ptrmask.p0.i64(ptr %p, i64 -64) + %pm0 = call ptr @llvm.ptrmask.p0.i64(ptr %p, i64 -60) %pgep = getelementptr i8, ptr %pm0, i64 33 %r = call ptr @llvm.ptrmask.p0.i64(ptr %pgep, i64 -16) ret ptr %r @@ -287,6 +287,162 @@ define ptr addrspace(1) @ptrmask_maintain_provenance_i32(ptr addrspace(1) %p0) { ret ptr addrspace(1) %r } +define ptr @ptrmask_is_nop0(ptr align 8 %p) { +; CHECK-LABEL: define ptr @ptrmask_is_nop0 +; CHECK-SAME: (ptr align 8 [[P:%.*]]) { +; CHECK-NEXT: ret ptr [[P]] +; + %pm = call ptr @llvm.ptrmask.p0.i64(ptr %p, i64 -8) + ret ptr %pm +} + +define ptr @ptrmask_is_nop1(ptr align 8 %p) { +; CHECK-LABEL: define ptr @ptrmask_is_nop1 +; CHECK-SAME: (ptr align 8 [[P:%.*]]) { +; CHECK-NEXT: ret ptr [[P]] +; + %pm = call ptr @llvm.ptrmask.p0.i64(ptr %p, i64 -4) + ret ptr %pm +} + +define ptr @ptrmask_to_modified_gep0(ptr align 8 %p) { +; CHECK-LABEL: define ptr @ptrmask_to_modified_gep0 +; CHECK-SAME: (ptr align 8 [[P:%.*]]) { +; CHECK-NEXT: [[PM:%.*]] = call align 16 ptr @llvm.ptrmask.p0.i64(ptr [[P]], i64 -16) +; CHECK-NEXT: ret ptr [[PM]] +; + %gep = getelementptr i8, ptr %p, i32 5 + %pm = call ptr @llvm.ptrmask.p0.i64(ptr %gep, i64 -16) + ret ptr %pm +} + +define ptr @ptrmask_to_modified_gep1(ptr align 8 %p) { +; CHECK-LABEL: define ptr @ptrmask_to_modified_gep1 +; CHECK-SAME: (ptr align 8 [[P:%.*]]) { +; CHECK-NEXT: ret ptr [[P]] +; + %gep = getelementptr i8, ptr %p, i32 6 + %pm = call ptr @llvm.ptrmask.p0.i64(ptr %gep, i64 -8) + ret ptr %pm +} + +define ptr @ptrmask_to_modified_gep2(ptr align 16 %p) { +; CHECK-LABEL: define ptr @ptrmask_to_modified_gep2 +; CHECK-SAME: (ptr align 16 [[P:%.*]]) { +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i8, ptr [[P]], i64 12 +; CHECK-NEXT: ret ptr [[GEP1]] +; + %gep = getelementptr i8, ptr %p, i32 15 + %pm = call ptr @llvm.ptrmask.p0.i64(ptr %gep, i64 -4) + ret ptr %pm +} + +define ptr @ptrmask_to_modified_gep4(ptr align 8 %p) { +; CHECK-LABEL: define ptr @ptrmask_to_modified_gep4 +; CHECK-SAME: (ptr align 8 [[P:%.*]]) { +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i8, ptr [[P]], i64 24 +; CHECK-NEXT: [[PM:%.*]] = call align 16 ptr @llvm.ptrmask.p0.i64(ptr [[GEP1]], i64 -16) +; CHECK-NEXT: ret ptr [[PM]] +; + %gep = getelementptr i8, ptr %p, i32 29 + %pm = call ptr @llvm.ptrmask.p0.i64(ptr %gep, i64 -16) + ret ptr %pm +} + +define ptr @ptrmask_to_modified_gep5(ptr align 8 %p) { +; CHECK-LABEL: define ptr @ptrmask_to_modified_gep5 +; CHECK-SAME: (ptr align 8 [[P:%.*]]) { +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i8, ptr [[P]], i64 24 +; CHECK-NEXT: ret ptr [[GEP1]] +; + %gep = getelementptr i8, ptr %p, i32 30 + %pm = call ptr @llvm.ptrmask.p0.i64(ptr %gep, i64 -8) + ret ptr %pm +} + +define ptr @ptrmask_to_modified_gep6(ptr align 16 %p) { +; CHECK-LABEL: define ptr @ptrmask_to_modified_gep6 +; CHECK-SAME: (ptr align 16 [[P:%.*]]) { +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i8, ptr [[P]], i64 28 +; CHECK-NEXT: ret ptr [[GEP1]] +; + %gep = getelementptr i8, ptr %p, i32 31 + %pm = call ptr @llvm.ptrmask.p0.i64(ptr %gep, i64 -4) + ret ptr %pm +} + +define ptr @ptrmask_to_modified_gep_indirect0(ptr align 16 %p) { +; CHECK-LABEL: define ptr @ptrmask_to_modified_gep_indirect0 +; CHECK-SAME: (ptr align 16 [[P:%.*]]) { +; 44 from 4*sizeof(i32) + (31 & -4) +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i8, ptr [[P]], i64 44 +; CHECK-NEXT: ret ptr [[GEP1]] +; + %gep0 = getelementptr i32, ptr %p, i32 4 + %gep1 = getelementptr i8, ptr %gep0, i32 31 + %pm = call ptr @llvm.ptrmask.p0.i64(ptr %gep1, i64 -4) + ret ptr %pm +} + +define ptr @ptrmask_to_modified_gep_indirect1(ptr %p) { +; CHECK-LABEL: define ptr @ptrmask_to_modified_gep_indirect1 +; CHECK-SAME: (ptr [[P:%.*]]) { + +; CHECK-NEXT: [[R:%.*]] = call align 16 ptr @llvm.ptrmask.p0.i64(ptr [[P]], i64 -16) +; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[R]], i64 32 +; CHECK-NEXT: ret ptr [[GEP]] +; + %pm0 = call ptr @llvm.ptrmask.p0.i64(ptr %p, i64 -16) + %pgep = getelementptr i8, ptr %pm0, i64 33 + %r = call ptr @llvm.ptrmask.p0.i64(ptr %pgep, i64 -16) + ret ptr %r +} + +define ptr @ptrmask_to_modified_gep_zero_argument() { +; CHECK-LABEL: define ptr @ptrmask_to_modified_gep_zero_argument() { +; CHECK-NEXT: [[P:%.*]] = call nonnull align 4 ptr @llvm.ptrmask.p0.i64(ptr nonnull inttoptr (i64 31 to ptr), i64 28) +; CHECK-NEXT: ret ptr [[P]] +; + %gep = getelementptr inbounds i8, ptr null, i32 31 + %pm = call ptr @llvm.ptrmask.p0.i64(ptr %gep, i64 -4) + ret ptr %pm +} + +define ptr @ptrmask_to_preserves_inbounds(ptr align 16 %p) { +; CHECK-LABEL: define ptr @ptrmask_to_preserves_inbounds +; CHECK-SAME: (ptr align 16 [[P:%.*]]) { +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 28 +; CHECK-NEXT: ret ptr [[GEP1]] +; + %gep = getelementptr inbounds i8, ptr %p, i32 31 + %pm = call ptr @llvm.ptrmask.p0.i64(ptr %gep, i64 -4) + ret ptr %pm +} + +define ptr @ptrmask_of_gep_requires_i8(ptr align 8 %p) { +; CHECK-LABEL: define ptr @ptrmask_of_gep_requires_i8 +; CHECK-SAME: (ptr align 8 [[P:%.*]]) { +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i8, ptr [[P]], i64 8 +; CHECK-NEXT: [[PM:%.*]] = call align 16 ptr @llvm.ptrmask.p0.i64(ptr [[GEP1]], i64 -16) +; CHECK-NEXT: ret ptr [[PM]] +; + %gep = getelementptr i16, ptr %p, i32 5 + %pm = call ptr @llvm.ptrmask.p0.i64(ptr %gep, i64 -16) + ret ptr %pm +} + +define <2 x ptr> @ptrmask_of_gep_vector_type_unimplemented(<2 x ptr> align 8 %p) { +; CHECK-LABEL: define <2 x ptr> @ptrmask_of_gep_vector_type_unimplemented +; CHECK-SAME: (<2 x ptr> align 8 [[P:%.*]]) { +; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, <2 x ptr> [[P]], i64 17 +; CHECK-NEXT: [[PM:%.*]] = call align 32 <2 x ptr> @llvm.ptrmask.v2p0.v2i64(<2 x ptr> [[GEP]], <2 x i64> ) +; CHECK-NEXT: ret <2 x ptr> [[PM]] +; + %gep = getelementptr i8, <2 x ptr> %p, i32 17 + %pm = call <2 x ptr> @llvm.ptrmask.v2p0.v2i64(<2 x ptr> %gep, <2 x i64> ) + ret <2 x ptr> %pm +} + define ptr @ptrmask_is_useless0(i64 %i, i64 %m) { ; CHECK-LABEL: define ptr @ptrmask_is_useless0 ; CHECK-SAME: (i64 [[I:%.*]], i64 [[M:%.*]]) { -- GitLab From 36f866c6ec3f6671fd4178ed4e49fd632a335cc2 Mon Sep 17 00:00:00 2001 From: jeffreytan81 Date: Thu, 7 Mar 2024 09:37:27 -0800 Subject: [PATCH 467/929] Fix vfork test strcmp buildbot failure (#84224) The buildbot seems to complain about `strcmp` function not available in the vfork patch (https://github.com/llvm/llvm-project/pull/81564): https://lab.llvm.org/buildbot/#/builders/68/builds/70093/steps/6/logs/stdio Unfortunately, I can't reproduce the failure on my linux machine so this is a guessing fix. If anyone has a way to reproduce and very this fix, please feel free to merge this change. Co-authored-by: jeffreytan81 --- lldb/test/API/functionalities/fork/concurrent_vfork/main.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/lldb/test/API/functionalities/fork/concurrent_vfork/main.cpp b/lldb/test/API/functionalities/fork/concurrent_vfork/main.cpp index 2f3a95dc5c6e..d72051e4ee84 100644 --- a/lldb/test/API/functionalities/fork/concurrent_vfork/main.cpp +++ b/lldb/test/API/functionalities/fork/concurrent_vfork/main.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include -- GitLab From ee24409c40ff35c3221892d9723331c233ca9f0e Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Thu, 7 Mar 2024 09:25:11 -0800 Subject: [PATCH 468/929] Revert "[AArch64][GlobalISel] Fix incorrect selection of monotonic s32->s64 anyext load." This reverts commit 7524ad9aa7b1b5003fe554a6ac8e434d50027dfb. Broke sanitizer build bots, e.g. https://lab.llvm.org/buildbot/#/builders/5/builds/41588/steps/9/logs/stdio --- .../GISel/AArch64InstructionSelector.cpp | 9 +++-- .../GlobalISel/select-atomic-load-store.mir | 33 +++---------------- 2 files changed, 8 insertions(+), 34 deletions(-) diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp index 48b73dced09b..665288379239 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp @@ -2997,14 +2997,13 @@ bool AArch64InstructionSelector::select(MachineInstr &I) { } } - if (IsZExtLoad || - (isa(LdSt) && ValTy == LLT::scalar(64) && MemSizeInBits == 32)) { - // The any/zextload from a smaller type to i32 should be handled by the + if (IsZExtLoad) { + // The zextload from a smaller type to i32 should be handled by the // importer. if (MRI.getType(LoadStore->getOperand(0).getReg()).getSizeInBits() != 64) return false; - // If we have an extending load then change the load's type to be a - // narrower reg and zero_extend with SUBREG_TO_REG. + // If we have a ZEXTLOAD then change the load's type to be a narrower reg + // and zero_extend with SUBREG_TO_REG. Register LdReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass); Register DstReg = LoadStore->getOperand(0).getReg(); LoadStore->getOperand(0).setReg(LdReg); diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-atomic-load-store.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-atomic-load-store.mir index 6b4bbb85b2ec..5787f914b965 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/select-atomic-load-store.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-atomic-load-store.mir @@ -9,11 +9,6 @@ ret i8 %v } - define i32 @anyext_load_monotonic_i32() { - %v = load atomic i32, ptr null monotonic, align 4 - ret i32 %v - } - ... --- name: load_acq_i8 @@ -30,33 +25,13 @@ body: | ; CHECK-LABEL: name: load_acq_i8 ; CHECK: liveins: $x0 - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0 - ; CHECK-NEXT: [[LDARB:%[0-9]+]]:gpr32 = LDARB [[COPY]] :: (load acquire (s8) from %ir.ptr, align 8) - ; CHECK-NEXT: $w0 = COPY [[LDARB]] - ; CHECK-NEXT: RET_ReallyLR implicit $w0 + ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0 + ; CHECK: [[LDARB:%[0-9]+]]:gpr32 = LDARB [[COPY]] :: (load acquire (s8) from %ir.ptr, align 8) + ; CHECK: $w0 = COPY [[LDARB]] + ; CHECK: RET_ReallyLR implicit $w0 %0:gpr(p0) = COPY $x0 %2:gpr(s32) = G_LOAD %0(p0) :: (load acquire (s8) from %ir.ptr, align 8) $w0 = COPY %2(s32) RET_ReallyLR implicit $w0 ... ---- -name: anyext_load_monotonic_i32 -legalized: true -regBankSelected: true -tracksRegLiveness: true -body: | - bb.1: - ; CHECK-LABEL: name: anyext_load_monotonic_i32 - ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $xzr - ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load monotonic (s32) from `ptr null`) - ; CHECK-NEXT: %ld:gpr64all = SUBREG_TO_REG 0, [[LDRWui]], %subreg.sub_32 - ; CHECK-NEXT: $x0 = COPY %ld - ; CHECK-NEXT: RET_ReallyLR implicit $x0 - %1:gpr(p0) = G_CONSTANT i64 0 - %ld:gpr(s64) = G_LOAD %1(p0) :: (load monotonic (s32) from `ptr null`) - $x0 = COPY %ld(s64) - RET_ReallyLR implicit $x0 - -... -- GitLab From e9901d8c94fdcd0d299d1abfdc8f0a5936aa7a50 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Thu, 7 Mar 2024 11:44:31 -0600 Subject: [PATCH 469/929] [LinkerWrapper] Accept compression arguments for HIP fatbins (#84337) Summary: The HIP toolchain has support for compressing the final output. We should respect that when we create the executable. --- clang/lib/Driver/ToolChains/Clang.cpp | 4 ++++ clang/test/Driver/linker-wrapper.c | 4 ++-- clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp | 3 +++ clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td | 2 ++ 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index e63e8a8e2e0e..fa17f6295d6e 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -8895,6 +8895,10 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA, for (const char *LinkArg : LinkCommand->getArguments()) CmdArgs.push_back(LinkArg); + if (Args.hasFlag(options::OPT_offload_compress, + options::OPT_no_offload_compress, false)) + CmdArgs.push_back("--compress"); + const char *Exec = Args.MakeArgString(getToolChain().GetProgramPath("clang-linker-wrapper")); diff --git a/clang/test/Driver/linker-wrapper.c b/clang/test/Driver/linker-wrapper.c index c37f01189d08..0e6fd80b4298 100644 --- a/clang/test/Driver/linker-wrapper.c +++ b/clang/test/Driver/linker-wrapper.c @@ -114,12 +114,12 @@ __attribute__((visibility("protected"), used)) int x; // RUN: --image=file=%t.elf.o,kind=hip,triple=amdgcn-amd-amdhsa,arch=gfx908 // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o \ // RUN: -fembed-offload-object=%t.out -// RUN: clang-linker-wrapper --dry-run --host-triple=x86_64-unknown-linux-gnu \ +// RUN: clang-linker-wrapper --dry-run --host-triple=x86_64-unknown-linux-gnu --compress \ // RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=HIP // HIP: clang{{.*}} -o [[IMG_GFX908:.+]] --target=amdgcn-amd-amdhsa -mcpu=gfx908 // HIP: clang{{.*}} -o [[IMG_GFX90A:.+]] --target=amdgcn-amd-amdhsa -mcpu=gfx90a -// HIP: clang-offload-bundler{{.*}}-type=o -bundle-align=4096 -targets=host-x86_64-unknown-linux,hipv4-amdgcn-amd-amdhsa--gfx90a,hipv4-amdgcn-amd-amdhsa--gfx908 -input=/dev/null -input=[[IMG_GFX90A]] -input=[[IMG_GFX908]] -output={{.*}}.hipfb +// HIP: clang-offload-bundler{{.*}}-type=o -bundle-align=4096 -compress -targets=host-x86_64-unknown-linux,hipv4-amdgcn-amd-amdhsa--gfx90a,hipv4-amdgcn-amd-amdhsa--gfx908 -input=/dev/null -input=[[IMG_GFX90A]] -input=[[IMG_GFX908]] -output={{.*}}.hipfb // RUN: clang-offload-packager -o %t.out \ // RUN: --image=file=%t.elf.o,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx908 \ diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp index 0a783db8962b..7e6e289c50d8 100644 --- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp +++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp @@ -405,6 +405,9 @@ fatbinary(ArrayRef> InputFiles, CmdArgs.push_back("-type=o"); CmdArgs.push_back("-bundle-align=4096"); + if (Args.hasArg(OPT_compress)) + CmdArgs.push_back("-compress"); + SmallVector Targets = {"-targets=host-x86_64-unknown-linux"}; for (const auto &[File, Arch] : InputFiles) Targets.push_back(Saver.save("hipv4-amdgcn-amd-amdhsa--" + Arch)); diff --git a/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td b/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td index 2c6a788cf23a..473fb19d9223 100644 --- a/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td +++ b/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td @@ -58,6 +58,8 @@ def print_wrapped_module : Flag<["--"], "print-wrapped-module">, HelpText<"Print the wrapped module's IR for testing">; def save_temps : Flag<["--"], "save-temps">, Flags<[WrapperOnlyOption]>, HelpText<"Save intermediate results">; +def compress : Flag<["--"], "compress">, + Flags<[WrapperOnlyOption]>, HelpText<"Compress bundled files">; def wrapper_time_trace_eq : Joined<["--"], "wrapper-time-trace=">, Flags<[WrapperOnlyOption]>, MetaVarName<"">, -- GitLab From ea49e04b35bc8e4bed7ee4db4074d201f780a15c Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Thu, 7 Mar 2024 09:55:00 -0800 Subject: [PATCH 470/929] [lldb] Don't report all progress event as completed. (#84281) Currently, progress events reported by the ProgressManager and broadcast to eBroadcastBitProgressCategory always specify they're complete. The problem is that the ProgressManager reports kNonDeterministicTotal for both the total and the completed number of (sub)events. Because the values are the same, the event reports itself as complete. This patch fixes the issue by reporting 0 as the completed value for the start event and kNonDeterministicTotal for the end event. --- lldb/include/lldb/Core/Progress.h | 9 +++++++-- lldb/source/Core/Progress.cpp | 16 +++++++++------- lldb/unittests/Core/ProgressReportTest.cpp | 4 ++-- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/lldb/include/lldb/Core/Progress.h b/lldb/include/lldb/Core/Progress.h index c6fc861fb71d..c38f6dd0a140 100644 --- a/lldb/include/lldb/Core/Progress.h +++ b/lldb/include/lldb/Core/Progress.h @@ -148,9 +148,14 @@ public: static ProgressManager &Instance(); - static void ReportProgress(const Progress::ProgressData &); - private: + enum class EventType { + Begin, + End, + }; + static void ReportProgress(const Progress::ProgressData &progress_data, + EventType type); + llvm::StringMap> m_progress_category_map; std::mutex m_progress_map_mutex; diff --git a/lldb/source/Core/Progress.cpp b/lldb/source/Core/Progress.cpp index 9dcd7cf75ae0..b4b5e98b7ba4 100644 --- a/lldb/source/Core/Progress.cpp +++ b/lldb/source/Core/Progress.cpp @@ -97,7 +97,7 @@ void ProgressManager::Increment(const Progress::ProgressData &progress_data) { // initial progress report. if (!m_progress_category_map.contains(progress_data.title)) { m_progress_category_map[progress_data.title].second = progress_data; - ReportProgress(progress_data); + ReportProgress(progress_data, EventType::Begin); } m_progress_category_map[progress_data.title].first++; } @@ -110,7 +110,7 @@ void ProgressManager::Decrement(const Progress::ProgressData &progress_data) { return; if (pos->second.first <= 1) { - ReportProgress(pos->second.second); + ReportProgress(pos->second.second, EventType::End); m_progress_category_map.erase(progress_data.title); } else { --pos->second.first; @@ -118,12 +118,14 @@ void ProgressManager::Decrement(const Progress::ProgressData &progress_data) { } void ProgressManager::ReportProgress( - const Progress::ProgressData &progress_data) { + const Progress::ProgressData &progress_data, EventType type) { // The category bit only keeps track of when progress report categories have // started and ended, so clear the details and reset other fields when // broadcasting to it since that bit doesn't need that information. - Debugger::ReportProgress( - progress_data.progress_id, progress_data.title, "", - Progress::kNonDeterministicTotal, Progress::kNonDeterministicTotal, - progress_data.debugger_id, Debugger::eBroadcastBitProgressCategory); + const uint64_t completed = + (type == EventType::Begin) ? 0 : Progress::kNonDeterministicTotal; + Debugger::ReportProgress(progress_data.progress_id, progress_data.title, "", + completed, Progress::kNonDeterministicTotal, + progress_data.debugger_id, + Debugger::eBroadcastBitProgressCategory); } diff --git a/lldb/unittests/Core/ProgressReportTest.cpp b/lldb/unittests/Core/ProgressReportTest.cpp index 98cbc475ce28..e0253cbc4ec5 100644 --- a/lldb/unittests/Core/ProgressReportTest.cpp +++ b/lldb/unittests/Core/ProgressReportTest.cpp @@ -168,7 +168,7 @@ TEST_F(ProgressReportTest, TestProgressManager) { ASSERT_EQ(data->GetDetails(), ""); ASSERT_FALSE(data->IsFinite()); - ASSERT_TRUE(data->GetCompleted()); + ASSERT_FALSE(data->GetCompleted()); ASSERT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal); ASSERT_EQ(data->GetMessage(), "Progress report 1"); @@ -199,7 +199,7 @@ TEST_F(ProgressReportTest, TestProgressManager) { ASSERT_EQ(data->GetDetails(), ""); ASSERT_FALSE(data->IsFinite()); - ASSERT_TRUE(data->GetCompleted()); + ASSERT_FALSE(data->GetCompleted()); ASSERT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal); ASSERT_EQ(data->GetMessage(), "Overlapping report 1"); -- GitLab From cfdfeb440cb2e25d1537616118a6c5509d96f2ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Thu, 7 Mar 2024 18:54:25 +0100 Subject: [PATCH 471/929] [clang][Interp][NFC] Remove unneeded forward declaration We already import Record.h. --- clang/lib/AST/Interp/Program.h | 1 - 1 file changed, 1 deletion(-) diff --git a/clang/lib/AST/Interp/Program.h b/clang/lib/AST/Interp/Program.h index 045bf7ab7745..50bdb575e805 100644 --- a/clang/lib/AST/Interp/Program.h +++ b/clang/lib/AST/Interp/Program.h @@ -34,7 +34,6 @@ class VarDecl; namespace interp { class Context; -class Record; /// The program contains and links the bytecode for all functions. class Program final { -- GitLab From a435e1f63bbd8c6d0ff140ccc890c25787091490 Mon Sep 17 00:00:00 2001 From: Razvan Lupusoru Date: Thu, 7 Mar 2024 10:06:47 -0800 Subject: [PATCH 472/929] [acc] Add attribute for combined constructs (#80319) Combined constructs are decomposed into separate operations. However, this does not adhere to `acc` dialect's goal to be able to regenerate semantically equivalent clauses as user's intent. Thus, add an attribute to keep track of the combined constructs. --- mlir/include/mlir/Dialect/OpenACC/OpenACC.h | 4 ++ .../mlir/Dialect/OpenACC/OpenACCOps.td | 34 +++++++++++-- mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp | 51 +++++++++++++++++++ mlir/test/Dialect/OpenACC/invalid.mlir | 40 +++++++++++++++ mlir/test/Dialect/OpenACC/ops.mlir | 46 +++++++++++++++-- 5 files changed, 168 insertions(+), 7 deletions(-) diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACC.h b/mlir/include/mlir/Dialect/OpenACC/OpenACC.h index bb3b9617c24e..0c8e0b458782 100644 --- a/mlir/include/mlir/Dialect/OpenACC/OpenACC.h +++ b/mlir/include/mlir/Dialect/OpenACC/OpenACC.h @@ -133,6 +133,10 @@ static constexpr StringLiteral getRoutineInfoAttrName() { return StringLiteral("acc.routine_info"); } +static constexpr StringLiteral getCombinedConstructsAttrName() { + return CombinedConstructsTypeAttr::name; +} + struct RuntimeCounters : public mlir::SideEffects::Resource::Base { mlir::StringRef getName() final { return "AccRuntimeCounters"; } diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td index 6da7a742bbed..b5ad46361fa6 100644 --- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td +++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td @@ -218,6 +218,24 @@ def GangArgTypeArrayAttr : let constBuilderCall = ?; } +// Combined constructs enumerations +def OpenACC_KernelsLoop : I32EnumAttrCase<"KernelsLoop", 1, "kernels_loop">; +def OpenACC_ParallelLoop : I32EnumAttrCase<"ParallelLoop", 2, "parallel_loop">; +def OpenACC_SerialLoop : I32EnumAttrCase<"SerialLoop", 3, "serial_loop">; + +def OpenACC_CombinedConstructsType : I32EnumAttr<"CombinedConstructsType", + "Differentiate between combined constructs", + [OpenACC_KernelsLoop, OpenACC_ParallelLoop, OpenACC_SerialLoop]> { + let genSpecializedAttr = 0; + let cppNamespace = "::mlir::acc"; +} + +def OpenACC_CombinedConstructsAttr : EnumAttr { + let assemblyFormat = [{ ```<` $value `>` }]; +} + // Define a resource for the OpenACC runtime counters. def OpenACC_RuntimeCounters : Resource<"::mlir::acc::RuntimeCounters">; @@ -933,7 +951,8 @@ def OpenACC_ParallelOp : OpenACC_Op<"parallel", Variadic:$gangFirstPrivateOperands, OptionalAttr:$firstprivatizations, Variadic:$dataClauseOperands, - OptionalAttr:$defaultAttr); + OptionalAttr:$defaultAttr, + UnitAttr:$combined); let regions = (region AnyRegion:$region); @@ -993,6 +1012,7 @@ def OpenACC_ParallelOp : OpenACC_Op<"parallel", }]; let assemblyFormat = [{ + ( `combined` `(` `loop` `)` $combined^)? oilist( `dataOperands` `(` $dataClauseOperands `:` type($dataClauseOperands) `)` | `async` `(` custom($asyncOperands, @@ -1068,7 +1088,8 @@ def OpenACC_SerialOp : OpenACC_Op<"serial", Variadic:$gangFirstPrivateOperands, OptionalAttr:$firstprivatizations, Variadic:$dataClauseOperands, - OptionalAttr:$defaultAttr); + OptionalAttr:$defaultAttr, + UnitAttr:$combined); let regions = (region AnyRegion:$region); @@ -1109,6 +1130,7 @@ def OpenACC_SerialOp : OpenACC_Op<"serial", }]; let assemblyFormat = [{ + ( `combined` `(` `loop` `)` $combined^)? oilist( `dataOperands` `(` $dataClauseOperands `:` type($dataClauseOperands) `)` | `async` `(` custom($asyncOperands, @@ -1182,7 +1204,8 @@ def OpenACC_KernelsOp : OpenACC_Op<"kernels", Optional:$selfCond, UnitAttr:$selfAttr, Variadic:$dataClauseOperands, - OptionalAttr:$defaultAttr); + OptionalAttr:$defaultAttr, + UnitAttr:$combined); let regions = (region AnyRegion:$region); @@ -1242,6 +1265,7 @@ def OpenACC_KernelsOp : OpenACC_Op<"kernels", }]; let assemblyFormat = [{ + ( `combined` `(` `loop` `)` $combined^)? oilist( `dataOperands` `(` $dataClauseOperands `:` type($dataClauseOperands) `)` | `async` `(` custom($asyncOperands, @@ -1573,7 +1597,8 @@ def OpenACC_LoopOp : OpenACC_Op<"loop", Variadic:$privateOperands, OptionalAttr:$privatizations, Variadic:$reductionOperands, - OptionalAttr:$reductionRecipes + OptionalAttr:$reductionRecipes, + OptionalAttr:$combined ); let results = (outs Variadic:$results); @@ -1665,6 +1690,7 @@ def OpenACC_LoopOp : OpenACC_Op<"loop", let hasCustomAssemblyFormat = 1; let assemblyFormat = [{ + custom($combined) oilist( `gang` `` custom($gangOperands, type($gangOperands), $gangOperandsArgType, $gangOperandsDeviceType, diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp index 18187e7d4f66..c09a3403f9a3 100644 --- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp +++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp @@ -1283,6 +1283,50 @@ static void printDeviceTypeOperandsWithKeywordOnly( p << ")"; } +static ParseResult +parseCombinedConstructsLoop(mlir::OpAsmParser &parser, + mlir::acc::CombinedConstructsTypeAttr &attr) { + if (succeeded(parser.parseOptionalKeyword("combined"))) { + if (parser.parseLParen()) + return failure(); + if (succeeded(parser.parseOptionalKeyword("kernels"))) { + attr = mlir::acc::CombinedConstructsTypeAttr::get( + parser.getContext(), mlir::acc::CombinedConstructsType::KernelsLoop); + } else if (succeeded(parser.parseOptionalKeyword("parallel"))) { + attr = mlir::acc::CombinedConstructsTypeAttr::get( + parser.getContext(), mlir::acc::CombinedConstructsType::ParallelLoop); + } else if (succeeded(parser.parseOptionalKeyword("serial"))) { + attr = mlir::acc::CombinedConstructsTypeAttr::get( + parser.getContext(), mlir::acc::CombinedConstructsType::SerialLoop); + } else { + parser.emitError(parser.getCurrentLocation(), + "expected compute construct name"); + return failure(); + } + if (parser.parseRParen()) + return failure(); + } + return success(); +} + +static void +printCombinedConstructsLoop(mlir::OpAsmPrinter &p, mlir::Operation *op, + mlir::acc::CombinedConstructsTypeAttr attr) { + if (attr) { + switch (attr.getValue()) { + case mlir::acc::CombinedConstructsType::KernelsLoop: + p << "combined(kernels)"; + break; + case mlir::acc::CombinedConstructsType::ParallelLoop: + p << "combined(parallel)"; + break; + case mlir::acc::CombinedConstructsType::SerialLoop: + p << "combined(serial)"; + break; + }; + } +} + //===----------------------------------------------------------------------===// // SerialOp //===----------------------------------------------------------------------===// @@ -1851,6 +1895,13 @@ LogicalResult acc::LoopOp::verify() { "reductions", false))) return failure(); + if (getCombined().has_value() && + (getCombined().value() != acc::CombinedConstructsType::ParallelLoop && + getCombined().value() != acc::CombinedConstructsType::KernelsLoop && + getCombined().value() != acc::CombinedConstructsType::SerialLoop)) { + return emitError("unexpected combined constructs attribute"); + } + // Check non-empty body(). if (getRegion().empty()) return emitError("expected non-empty body."); diff --git a/mlir/test/Dialect/OpenACC/invalid.mlir b/mlir/test/Dialect/OpenACC/invalid.mlir index 70747b7e2acf..ec5430420524 100644 --- a/mlir/test/Dialect/OpenACC/invalid.mlir +++ b/mlir/test/Dialect/OpenACC/invalid.mlir @@ -738,3 +738,43 @@ func.func @acc_atomic_capture(%x: memref, %y: memref, %v: memref, acc.terminator } } + +// ----- + +func.func @acc_combined() { + // expected-error @below {{expected 'loop'}} + acc.parallel combined() { + } + + return +} + +// ----- + +func.func @acc_combined() { + // expected-error @below {{expected compute construct name}} + acc.loop combined(loop) { + } + + return +} + +// ----- + +func.func @acc_combined() { + // expected-error @below {{expected 'loop'}} + acc.parallel combined(parallel loop) { + } + + return +} + +// ----- + +func.func @acc_combined() { + // expected-error @below {{expected ')'}} + acc.loop combined(parallel loop) { + } + + return +} diff --git a/mlir/test/Dialect/OpenACC/ops.mlir b/mlir/test/Dialect/OpenACC/ops.mlir index 1739b3de3e65..2ef2178cb2b6 100644 --- a/mlir/test/Dialect/OpenACC/ops.mlir +++ b/mlir/test/Dialect/OpenACC/ops.mlir @@ -1846,9 +1846,49 @@ func.func @acc_atomic_capture(%v: memref, %x: memref, %expr: i32) { // ----- -%c2 = arith.constant 2 : i32 -%c1 = arith.constant 1 : i32 -acc.parallel num_gangs({%c2 : i32} [#acc.device_type], {%c1 : i32, %c1 : i32, %c1 : i32} [#acc.device_type]) { +// CHECK-LABEL: func.func @acc_num_gangs +func.func @acc_num_gangs() { + %c2 = arith.constant 2 : i32 + %c1 = arith.constant 1 : i32 + acc.parallel num_gangs({%c2 : i32} [#acc.device_type], {%c1 : i32, %c1 : i32, %c1 : i32} [#acc.device_type]) { + } + + return } // CHECK: acc.parallel num_gangs({%c2{{.*}} : i32} [#acc.device_type], {%c1{{.*}} : i32, %c1{{.*}} : i32, %c1{{.*}} : i32} [#acc.device_type]) + +// ----- + +// CHECK-LABEL: func.func @acc_combined +func.func @acc_combined() { + acc.parallel combined(loop) { + acc.loop combined(parallel) { + acc.yield + } + acc.terminator + } + + acc.kernels combined(loop) { + acc.loop combined(kernels) { + acc.yield + } + acc.terminator + } + + acc.serial combined(loop) { + acc.loop combined(serial) { + acc.yield + } + acc.terminator + } + + return +} + +// CHECK: acc.parallel combined(loop) +// CHECK: acc.loop combined(parallel) +// CHECK: acc.kernels combined(loop) +// CHECK: acc.loop combined(kernels) +// CHECK: acc.serial combined(loop) +// CHECK: acc.loop combined(serial) -- GitLab From 6515930b0cc4aa2e11e75728ef6cbeecbe5caec2 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Thu, 7 Mar 2024 10:13:14 -0800 Subject: [PATCH 473/929] [lldb] Minor cleanup in StoringDiagnosticConsumer (#84263) Removes an unused field. Retypes unshared smart pointers to `unique_ptr`. --- .../Clang/ClangModulesDeclVendor.cpp | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp index 2d778e410b0e..024fc75a5dd5 100644 --- a/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp +++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp @@ -67,12 +67,11 @@ private: std::vector m_diagnostics; /// The DiagnosticPrinter used for creating the full diagnostic messages /// that are stored in m_diagnostics. - std::shared_ptr m_diag_printer; + std::unique_ptr m_diag_printer; /// Output stream of m_diag_printer. - std::shared_ptr m_os; + std::unique_ptr m_os; /// Output string filled by m_os. Will be reused for different diagnostics. std::string m_output; - Log *m_log; /// A Progress with explicitly managed lifetime. std::unique_ptr m_current_progress_up; std::vector m_module_build_stack; @@ -134,12 +133,10 @@ private: } // anonymous namespace StoringDiagnosticConsumer::StoringDiagnosticConsumer() { - m_log = GetLog(LLDBLog::Expressions); - - clang::DiagnosticOptions *m_options = new clang::DiagnosticOptions(); - m_os = std::make_shared(m_output); + auto *options = new clang::DiagnosticOptions(); + m_os = std::make_unique(m_output); m_diag_printer = - std::make_shared(*m_os, m_options); + std::make_unique(*m_os, options); } void StoringDiagnosticConsumer::HandleDiagnostic( -- GitLab From 9e4f289bd6c905a2a436b3311ca49ad2d6328060 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timm=20B=C3=A4der?= Date: Thu, 7 Mar 2024 19:37:24 +0100 Subject: [PATCH 474/929] [clang][Interp][NFC] Add [[nodiscard]] attribute to emit functions --- clang/lib/AST/Interp/ByteCodeExprGen.cpp | 3 ++- clang/utils/TableGen/ClangOpcodesEmitter.cpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp index 712218f5de2e..a384e191464f 100644 --- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp +++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp @@ -1646,7 +1646,8 @@ bool ByteCodeExprGen::VisitMaterializeTemporaryExpr( SubExpr, *SubExprT, /*IsConst=*/true, /*IsExtended=*/true)) { if (!this->visit(SubExpr)) return false; - this->emitSetLocal(*SubExprT, *LocalIndex, E); + if (!this->emitSetLocal(*SubExprT, *LocalIndex, E)) + return false; return this->emitGetPtrLocal(*LocalIndex, E); } } else { diff --git a/clang/utils/TableGen/ClangOpcodesEmitter.cpp b/clang/utils/TableGen/ClangOpcodesEmitter.cpp index 1c41301ab3ae..120e1e2efa32 100644 --- a/clang/utils/TableGen/ClangOpcodesEmitter.cpp +++ b/clang/utils/TableGen/ClangOpcodesEmitter.cpp @@ -274,7 +274,7 @@ void ClangOpcodesEmitter::EmitGroup(raw_ostream &OS, StringRef N, // Emit the prototype of the group emitter in the header. OS << "#if defined(GET_EVAL_PROTO) || defined(GET_LINK_PROTO)\n"; - OS << "bool " << EmitFuncName << "("; + OS << "[[nodiscard]] bool " << EmitFuncName << "("; for (size_t I = 0, N = Types->size(); I < N; ++I) OS << "PrimType, "; for (auto *Arg : Args) -- GitLab From 41572177d129bf19f13f077a30b582fd3b8f790c Mon Sep 17 00:00:00 2001 From: Vlad Serebrennikov Date: Thu, 7 Mar 2024 22:38:44 +0400 Subject: [PATCH 475/929] [clang] Add CodeGen tests for CWG 5xx issues (#84303) This patch covers [CWG519](https://cplusplus.github.io/CWG/issues/519.html) "Null pointer preservation in `void*` conversions", [CWG571](https://cplusplus.github.io/CWG/issues/571.html) "References declared const". --- clang/test/CXX/drs/dr519.cpp | 36 ++++++++++++++++++++++++++++++++++++ clang/test/CXX/drs/dr571.cpp | 20 ++++++++++++++++++++ clang/test/CXX/drs/dr5xx.cpp | 19 ++----------------- clang/www/cxx_dr_status.html | 4 ++-- 4 files changed, 60 insertions(+), 19 deletions(-) create mode 100644 clang/test/CXX/drs/dr519.cpp create mode 100644 clang/test/CXX/drs/dr571.cpp diff --git a/clang/test/CXX/drs/dr519.cpp b/clang/test/CXX/drs/dr519.cpp new file mode 100644 index 000000000000..67c01d95ef7c --- /dev/null +++ b/clang/test/CXX/drs/dr519.cpp @@ -0,0 +1,36 @@ +// RUN: %clang_cc1 -std=c++98 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++11 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++14 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++17 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++20 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++23 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++2c %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK + +namespace dr519 { // dr519: 2.7 +void f() { + int *a = 0; + void *v = a; + bool c1 = v == static_cast(0); + + void *w = 0; + int *b = static_cast(w); + bool c2 = b == static_cast(0); +} +} // namespace dr519 + +// We're checking that `null`s that were initially stored in `a` and `w` +// are simply copied over all the way to respective comparisons with `null`. + +// CHECK-LABEL: define {{.*}} void @dr519::f()() +// CHECK: store ptr null, ptr [[A:%.+]], +// CHECK-NEXT: [[TEMP_A:%.+]] = load ptr, ptr [[A]] +// CHECK-NEXT: store ptr [[TEMP_A]], ptr [[V:%.+]], +// CHECK-NEXT: [[TEMP_V:%.+]] = load ptr, ptr [[V]] +// CHECK-NEXT: {{.+}} = icmp eq ptr [[TEMP_V]], null + +// CHECK: store ptr null, ptr [[W:%.+]], +// CHECK-NEXT: [[TEMP_W:%.+]] = load ptr, ptr [[W]] +// CHECK-NEXT: store ptr [[TEMP_W]], ptr [[B:%.+]], +// CHECK-NEXT: [[TEMP_B:%.+]] = load ptr, ptr [[B]] +// CHECK-NEXT: {{.+}} = icmp eq ptr [[TEMP_B]], null +// CHECK-LABEL: } diff --git a/clang/test/CXX/drs/dr571.cpp b/clang/test/CXX/drs/dr571.cpp new file mode 100644 index 000000000000..19a85b7ddc35 --- /dev/null +++ b/clang/test/CXX/drs/dr571.cpp @@ -0,0 +1,20 @@ +// RUN: %clang_cc1 -std=c++98 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++11 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++14 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++17 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++20 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++23 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK +// RUN: %clang_cc1 -std=c++2c %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK + +namespace dr571 { // dr571: 2.7 + typedef int &ir; + int n; + const ir r = n; + // expected-warning@-1 {{'const' qualifier on reference type 'ir' (aka 'int &') has no effect}} + ir r2 = n; +} + +// Entities have external linkage by default. + +// CHECK: @dr571::r = constant ptr @dr571::n +// CHECK: @dr571::r2 = constant ptr @dr571::n diff --git a/clang/test/CXX/drs/dr5xx.cpp b/clang/test/CXX/drs/dr5xx.cpp index 0e1de342f670..426b368b390a 100644 --- a/clang/test/CXX/drs/dr5xx.cpp +++ b/clang/test/CXX/drs/dr5xx.cpp @@ -141,15 +141,7 @@ namespace dr518 { // dr518: yes c++11 // cxx98-error@-1 {{commas at the end of enumerator lists are a C++11 extension}} } -namespace dr519 { // dr519: yes -// FIXME: Add a codegen test. -#if __cplusplus >= 201103L -#define fold(x) (__builtin_constant_p(x) ? (x) : (x)) - int test[fold((int*)(void*)0) ? -1 : 1]; -#undef fold -#endif -} - +// dr519 is in dr519.cpp // dr520: na // dr521: no @@ -800,14 +792,7 @@ namespace dr570 { // dr570: dup 633 // expected-note@#dr570-r {{previous definition is here}} } -namespace dr571 { // dr571 unknown - // FIXME: Add a codegen test. - typedef int &ir; - int n; - // FIXME: Test if this has internal linkage. - const ir r = n; - // expected-warning@-1 {{'const' qualifier on reference type 'ir' (aka 'int &') has no effect}} -} +// dr571 is in dr571.cpp namespace dr572 { // dr572: yes enum E { a = 1, b = 2 }; diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html index 774c71bc1cb6..503472a2cae4 100755 --- a/clang/www/cxx_dr_status.html +++ b/clang/www/cxx_dr_status.html @@ -3154,7 +3154,7 @@ of class templates 519 CD1 Null pointer preservation in void* conversions - Yes + Clang 2.7 520 @@ -3468,7 +3468,7 @@ and POD class 571 CD2 References declared const - Unknown + Clang 2.7 572 -- GitLab From a6a6fca7911feab8325129ea57247303b3c8d558 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Thu, 7 Mar 2024 10:52:43 -0800 Subject: [PATCH 476/929] [ubsan][pgo] Pass to remove ubsan checks based on profile data (#83471) UBSAN checks can be too expensive to be used in release binaries. However not all code affect performace in the same way. Removing small number of checks in hot code we can performance loss, preserving most of the checks. --- .../Instrumentation/RemoveTrapsPass.h | 32 ++ llvm/lib/Passes/PassBuilder.cpp | 1 + llvm/lib/Passes/PassRegistry.def | 1 + .../Transforms/Instrumentation/CMakeLists.txt | 1 + .../Instrumentation/RemoveTrapsPass.cpp | 104 +++++ .../Transforms/RemoveTraps/remove-traps.ll | 397 ++++++++++++++++++ 6 files changed, 536 insertions(+) create mode 100644 llvm/include/llvm/Transforms/Instrumentation/RemoveTrapsPass.h create mode 100644 llvm/lib/Transforms/Instrumentation/RemoveTrapsPass.cpp create mode 100644 llvm/test/Transforms/RemoveTraps/remove-traps.ll diff --git a/llvm/include/llvm/Transforms/Instrumentation/RemoveTrapsPass.h b/llvm/include/llvm/Transforms/Instrumentation/RemoveTrapsPass.h new file mode 100644 index 000000000000..58f6bbcec5dc --- /dev/null +++ b/llvm/include/llvm/Transforms/Instrumentation/RemoveTrapsPass.h @@ -0,0 +1,32 @@ +//===- RemoveTrapsPass.h ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// This file provides the interface for the pass responsible for removing +/// expensive ubsan checks. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_UBSANOPTIMIZATIONPASS_H +#define LLVM_TRANSFORMS_INSTRUMENTATION_UBSANOPTIMIZATIONPASS_H + +#include "llvm/IR/Function.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Pass.h" + +namespace llvm { + +// This pass is responsible for removing optional traps, like llvm.ubsantrap +// from the hot code. +class RemoveTrapsPass : public PassInfoMixin { +public: + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; + +} // namespace llvm + +#endif diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index e0bc57f8bf72..4d1eb10d2d41 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -177,6 +177,7 @@ #include "llvm/Transforms/Instrumentation/PGOForceFunctionAttrs.h" #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h" #include "llvm/Transforms/Instrumentation/PoisonChecking.h" +#include "llvm/Transforms/Instrumentation/RemoveTrapsPass.h" #include "llvm/Transforms/Instrumentation/SanitizerBinaryMetadata.h" #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h" #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index a345e8d72d93..41f16d0915bf 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -422,6 +422,7 @@ FUNCTION_PASS("print", UniformityInfoPrinterPass(dbgs())) FUNCTION_PASS("reassociate", ReassociatePass()) FUNCTION_PASS("redundant-dbg-inst-elim", RedundantDbgInstEliminationPass()) FUNCTION_PASS("reg2mem", RegToMemPass()) +FUNCTION_PASS("remove-traps", RemoveTrapsPass()) FUNCTION_PASS("safe-stack", SafeStackPass(TM)) FUNCTION_PASS("scalarize-masked-mem-intrin", ScalarizeMaskedMemIntrinPass()) FUNCTION_PASS("scalarizer", ScalarizerPass()) diff --git a/llvm/lib/Transforms/Instrumentation/CMakeLists.txt b/llvm/lib/Transforms/Instrumentation/CMakeLists.txt index ee9aa73ff034..b23a6ed1f084 100644 --- a/llvm/lib/Transforms/Instrumentation/CMakeLists.txt +++ b/llvm/lib/Transforms/Instrumentation/CMakeLists.txt @@ -17,6 +17,7 @@ add_llvm_component_library(LLVMInstrumentation PGOInstrumentation.cpp PGOMemOPSizeOpt.cpp PoisonChecking.cpp + RemoveTrapsPass.cpp SanitizerCoverage.cpp SanitizerBinaryMetadata.cpp ValueProfileCollector.cpp diff --git a/llvm/lib/Transforms/Instrumentation/RemoveTrapsPass.cpp b/llvm/lib/Transforms/Instrumentation/RemoveTrapsPass.cpp new file mode 100644 index 000000000000..d87f7482a21d --- /dev/null +++ b/llvm/lib/Transforms/Instrumentation/RemoveTrapsPass.cpp @@ -0,0 +1,104 @@ +//===- RemoveTrapsPass.cpp --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Instrumentation/RemoveTrapsPass.h" + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/ProfileSummaryInfo.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/Support/RandomNumberGenerator.h" +#include +#include + +using namespace llvm; + +#define DEBUG_TYPE "remove-traps" + +static cl::opt HotPercentileCutoff( + "remove-traps-percentile-cutoff-hot", cl::init(0), + cl::desc("Alternative hot percentile cuttoff. By default " + "`-profile-summary-cutoff-hot` is used.")); + +static cl::opt + RandomRate("remove-traps-random-rate", cl::init(0.0), + cl::desc("Probability value in the range [0.0, 1.0] of " + "unconditional pseudo-random checks removal.")); + +STATISTIC(NumChecksTotal, "Number of checks"); +STATISTIC(NumChecksRemoved, "Number of removed checks"); + +static bool removeUbsanTraps(Function &F, const BlockFrequencyInfo &BFI, + const ProfileSummaryInfo *PSI) { + SmallVector Remove; + std::unique_ptr Rng; + + auto ShouldRemove = [&](bool IsHot) { + if (!RandomRate.getNumOccurrences()) + return IsHot; + if (!Rng) + Rng = F.getParent()->createRNG(F.getName()); + std::bernoulli_distribution D(RandomRate); + return D(*Rng); + }; + + for (BasicBlock &BB : F) { + for (Instruction &I : BB) { + IntrinsicInst *II = dyn_cast(&I); + if (!II) + continue; + auto ID = II->getIntrinsicID(); + switch (ID) { + case Intrinsic::ubsantrap: { + ++NumChecksTotal; + + bool IsHot = false; + if (PSI) { + uint64_t Count = 0; + for (const auto *PR : predecessors(&BB)) + Count += BFI.getBlockProfileCount(PR).value_or(0); + + IsHot = + HotPercentileCutoff.getNumOccurrences() + ? (HotPercentileCutoff > 0 && + PSI->isHotCountNthPercentile(HotPercentileCutoff, Count)) + : PSI->isHotCount(Count); + } + + if (ShouldRemove(IsHot)) { + Remove.push_back(II); + ++NumChecksRemoved; + } + break; + } + default: + break; + } + } + } + + for (IntrinsicInst *I : Remove) + I->eraseFromParent(); + + return !Remove.empty(); +} + +PreservedAnalyses RemoveTrapsPass::run(Function &F, + FunctionAnalysisManager &AM) { + if (F.isDeclaration()) + return PreservedAnalyses::all(); + auto &MAMProxy = AM.getResult(F); + ProfileSummaryInfo *PSI = + MAMProxy.getCachedResult(*F.getParent()); + BlockFrequencyInfo &BFI = AM.getResult(F); + + return removeUbsanTraps(F, BFI, PSI) ? PreservedAnalyses::none() + : PreservedAnalyses::all(); +} diff --git a/llvm/test/Transforms/RemoveTraps/remove-traps.ll b/llvm/test/Transforms/RemoveTraps/remove-traps.ll new file mode 100644 index 000000000000..71549e7d9b41 --- /dev/null +++ b/llvm/test/Transforms/RemoveTraps/remove-traps.ll @@ -0,0 +1,397 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -passes='function(remove-traps)' -S | FileCheck %s --check-prefixes=NOPROFILE +; RUN: opt < %s -passes='function(remove-traps)' -remove-traps-random-rate=1 -S | FileCheck %s --check-prefixes=ALL +; RUN: opt < %s -passes='require,function(remove-traps)' -S | FileCheck %s --check-prefixes=HOT +; RUN: opt < %s -passes='require,function(remove-traps)' -remove-traps-percentile-cutoff-hot=700000 -S | FileCheck %s --check-prefixes=HOT70 + +target triple = "x86_64-pc-linux-gnu" + +declare void @llvm.ubsantrap(i8 immarg) + +define dso_local noundef i32 @simple(ptr noundef readonly %0) { +; NOPROFILE-LABEL: define dso_local noundef i32 @simple( +; NOPROFILE-SAME: ptr noundef readonly [[TMP0:%.*]]) { +; NOPROFILE-NEXT: [[TMP2:%.*]] = icmp eq ptr [[TMP0]], null +; NOPROFILE-NEXT: br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP4:%.*]] +; NOPROFILE: 3: +; NOPROFILE-NEXT: tail call void @llvm.ubsantrap(i8 22) +; NOPROFILE-NEXT: unreachable +; NOPROFILE: 4: +; NOPROFILE-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP0]], align 4 +; NOPROFILE-NEXT: ret i32 [[TMP5]] +; +; ALL-LABEL: define dso_local noundef i32 @simple( +; ALL-SAME: ptr noundef readonly [[TMP0:%.*]]) { +; ALL-NEXT: [[TMP2:%.*]] = icmp eq ptr [[TMP0]], null +; ALL-NEXT: br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP4:%.*]] +; ALL: 3: +; ALL-NEXT: unreachable +; ALL: 4: +; ALL-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP0]], align 4 +; ALL-NEXT: ret i32 [[TMP5]] +; +; HOT-LABEL: define dso_local noundef i32 @simple( +; HOT-SAME: ptr noundef readonly [[TMP0:%.*]]) { +; HOT-NEXT: [[TMP2:%.*]] = icmp eq ptr [[TMP0]], null +; HOT-NEXT: br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP4:%.*]] +; HOT: 3: +; HOT-NEXT: tail call void @llvm.ubsantrap(i8 22) +; HOT-NEXT: unreachable +; HOT: 4: +; HOT-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP0]], align 4 +; HOT-NEXT: ret i32 [[TMP5]] +; +; HOT70-LABEL: define dso_local noundef i32 @simple( +; HOT70-SAME: ptr noundef readonly [[TMP0:%.*]]) { +; HOT70-NEXT: [[TMP2:%.*]] = icmp eq ptr [[TMP0]], null +; HOT70-NEXT: br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP4:%.*]] +; HOT70: 3: +; HOT70-NEXT: tail call void @llvm.ubsantrap(i8 22) +; HOT70-NEXT: unreachable +; HOT70: 4: +; HOT70-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP0]], align 4 +; HOT70-NEXT: ret i32 [[TMP5]] +; + %2 = icmp eq ptr %0, null + br i1 %2, label %3, label %4 + +3: + tail call void @llvm.ubsantrap(i8 22) + unreachable + +4: + %5 = load i32, ptr %0, align 4 + ret i32 %5 +} + + +define dso_local noundef i32 @hot(ptr noundef readonly %0) !prof !36 { +; NOPROFILE-LABEL: define dso_local noundef i32 @hot( +; NOPROFILE-SAME: ptr noundef readonly [[TMP0:%.*]]) !prof [[PROF16:![0-9]+]] { +; NOPROFILE-NEXT: [[TMP2:%.*]] = icmp eq ptr [[TMP0]], null +; NOPROFILE-NEXT: br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP4:%.*]] +; NOPROFILE: 3: +; NOPROFILE-NEXT: tail call void @llvm.ubsantrap(i8 22) +; NOPROFILE-NEXT: unreachable +; NOPROFILE: 4: +; NOPROFILE-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP0]], align 4 +; NOPROFILE-NEXT: ret i32 [[TMP5]] +; +; ALL-LABEL: define dso_local noundef i32 @hot( +; ALL-SAME: ptr noundef readonly [[TMP0:%.*]]) !prof [[PROF16:![0-9]+]] { +; ALL-NEXT: [[TMP2:%.*]] = icmp eq ptr [[TMP0]], null +; ALL-NEXT: br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP4:%.*]] +; ALL: 3: +; ALL-NEXT: unreachable +; ALL: 4: +; ALL-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP0]], align 4 +; ALL-NEXT: ret i32 [[TMP5]] +; +; HOT-LABEL: define dso_local noundef i32 @hot( +; HOT-SAME: ptr noundef readonly [[TMP0:%.*]]) !prof [[PROF16:![0-9]+]] { +; HOT-NEXT: [[TMP2:%.*]] = icmp eq ptr [[TMP0]], null +; HOT-NEXT: br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP4:%.*]] +; HOT: 3: +; HOT-NEXT: unreachable +; HOT: 4: +; HOT-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP0]], align 4 +; HOT-NEXT: ret i32 [[TMP5]] +; +; HOT70-LABEL: define dso_local noundef i32 @hot( +; HOT70-SAME: ptr noundef readonly [[TMP0:%.*]]) !prof [[PROF16:![0-9]+]] { +; HOT70-NEXT: [[TMP2:%.*]] = icmp eq ptr [[TMP0]], null +; HOT70-NEXT: br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP4:%.*]] +; HOT70: 3: +; HOT70-NEXT: tail call void @llvm.ubsantrap(i8 22) +; HOT70-NEXT: unreachable +; HOT70: 4: +; HOT70-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP0]], align 4 +; HOT70-NEXT: ret i32 [[TMP5]] +; + %2 = icmp eq ptr %0, null + br i1 %2, label %3, label %4 + +3: + tail call void @llvm.ubsantrap(i8 22) + unreachable + +4: + %5 = load i32, ptr %0, align 4 + ret i32 %5 +} + +define dso_local noundef i32 @veryHot(ptr noundef readonly %0) !prof !39 { +; NOPROFILE-LABEL: define dso_local noundef i32 @veryHot( +; NOPROFILE-SAME: ptr noundef readonly [[TMP0:%.*]]) !prof [[PROF17:![0-9]+]] { +; NOPROFILE-NEXT: [[TMP2:%.*]] = icmp eq ptr [[TMP0]], null +; NOPROFILE-NEXT: br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP4:%.*]] +; NOPROFILE: 3: +; NOPROFILE-NEXT: tail call void @llvm.ubsantrap(i8 22) +; NOPROFILE-NEXT: unreachable +; NOPROFILE: 4: +; NOPROFILE-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP0]], align 4 +; NOPROFILE-NEXT: ret i32 [[TMP5]] +; +; ALL-LABEL: define dso_local noundef i32 @veryHot( +; ALL-SAME: ptr noundef readonly [[TMP0:%.*]]) !prof [[PROF17:![0-9]+]] { +; ALL-NEXT: [[TMP2:%.*]] = icmp eq ptr [[TMP0]], null +; ALL-NEXT: br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP4:%.*]] +; ALL: 3: +; ALL-NEXT: unreachable +; ALL: 4: +; ALL-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP0]], align 4 +; ALL-NEXT: ret i32 [[TMP5]] +; +; HOT-LABEL: define dso_local noundef i32 @veryHot( +; HOT-SAME: ptr noundef readonly [[TMP0:%.*]]) !prof [[PROF17:![0-9]+]] { +; HOT-NEXT: [[TMP2:%.*]] = icmp eq ptr [[TMP0]], null +; HOT-NEXT: br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP4:%.*]] +; HOT: 3: +; HOT-NEXT: unreachable +; HOT: 4: +; HOT-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP0]], align 4 +; HOT-NEXT: ret i32 [[TMP5]] +; +; HOT70-LABEL: define dso_local noundef i32 @veryHot( +; HOT70-SAME: ptr noundef readonly [[TMP0:%.*]]) !prof [[PROF17:![0-9]+]] { +; HOT70-NEXT: [[TMP2:%.*]] = icmp eq ptr [[TMP0]], null +; HOT70-NEXT: br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP4:%.*]] +; HOT70: 3: +; HOT70-NEXT: unreachable +; HOT70: 4: +; HOT70-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP0]], align 4 +; HOT70-NEXT: ret i32 [[TMP5]] +; + %2 = icmp eq ptr %0, null + br i1 %2, label %3, label %4 + +3: + tail call void @llvm.ubsantrap(i8 22) + unreachable + +4: + %5 = load i32, ptr %0, align 4 + ret i32 %5 +} + + +define dso_local noundef i32 @branchColdFnHot(i32 noundef %0, ptr noundef readonly %1) !prof !39 { +; NOPROFILE-LABEL: define dso_local noundef i32 @branchColdFnHot( +; NOPROFILE-SAME: i32 noundef [[TMP0:%.*]], ptr noundef readonly [[TMP1:%.*]]) !prof [[PROF17]] { +; NOPROFILE-NEXT: [[TMP3:%.*]] = icmp eq i32 [[TMP0]], 0 +; NOPROFILE-NEXT: br i1 [[TMP3]], label [[TMP9:%.*]], label [[TMP4:%.*]], !prof [[PROF18:![0-9]+]] +; NOPROFILE: 4: +; NOPROFILE-NEXT: [[TMP5:%.*]] = icmp eq ptr [[TMP1]], null +; NOPROFILE-NEXT: br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP7:%.*]] +; NOPROFILE: 6: +; NOPROFILE-NEXT: tail call void @llvm.ubsantrap(i8 22) +; NOPROFILE-NEXT: unreachable +; NOPROFILE: 7: +; NOPROFILE-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP1]], align 4 +; NOPROFILE-NEXT: br label [[TMP9]] +; NOPROFILE: 9: +; NOPROFILE-NEXT: [[TMP10:%.*]] = phi i32 [ [[TMP8]], [[TMP7]] ], [ 0, [[TMP2:%.*]] ] +; NOPROFILE-NEXT: ret i32 [[TMP10]] +; +; ALL-LABEL: define dso_local noundef i32 @branchColdFnHot( +; ALL-SAME: i32 noundef [[TMP0:%.*]], ptr noundef readonly [[TMP1:%.*]]) !prof [[PROF17]] { +; ALL-NEXT: [[TMP3:%.*]] = icmp eq i32 [[TMP0]], 0 +; ALL-NEXT: br i1 [[TMP3]], label [[TMP9:%.*]], label [[TMP4:%.*]], !prof [[PROF18:![0-9]+]] +; ALL: 4: +; ALL-NEXT: [[TMP5:%.*]] = icmp eq ptr [[TMP1]], null +; ALL-NEXT: br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP7:%.*]] +; ALL: 6: +; ALL-NEXT: unreachable +; ALL: 7: +; ALL-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP1]], align 4 +; ALL-NEXT: br label [[TMP9]] +; ALL: 9: +; ALL-NEXT: [[TMP10:%.*]] = phi i32 [ [[TMP8]], [[TMP7]] ], [ 0, [[TMP2:%.*]] ] +; ALL-NEXT: ret i32 [[TMP10]] +; +; HOT-LABEL: define dso_local noundef i32 @branchColdFnHot( +; HOT-SAME: i32 noundef [[TMP0:%.*]], ptr noundef readonly [[TMP1:%.*]]) !prof [[PROF17]] { +; HOT-NEXT: [[TMP3:%.*]] = icmp eq i32 [[TMP0]], 0 +; HOT-NEXT: br i1 [[TMP3]], label [[TMP9:%.*]], label [[TMP4:%.*]], !prof [[PROF18:![0-9]+]] +; HOT: 4: +; HOT-NEXT: [[TMP5:%.*]] = icmp eq ptr [[TMP1]], null +; HOT-NEXT: br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP7:%.*]] +; HOT: 6: +; HOT-NEXT: tail call void @llvm.ubsantrap(i8 22) +; HOT-NEXT: unreachable +; HOT: 7: +; HOT-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP1]], align 4 +; HOT-NEXT: br label [[TMP9]] +; HOT: 9: +; HOT-NEXT: [[TMP10:%.*]] = phi i32 [ [[TMP8]], [[TMP7]] ], [ 0, [[TMP2:%.*]] ] +; HOT-NEXT: ret i32 [[TMP10]] +; +; HOT70-LABEL: define dso_local noundef i32 @branchColdFnHot( +; HOT70-SAME: i32 noundef [[TMP0:%.*]], ptr noundef readonly [[TMP1:%.*]]) !prof [[PROF17]] { +; HOT70-NEXT: [[TMP3:%.*]] = icmp eq i32 [[TMP0]], 0 +; HOT70-NEXT: br i1 [[TMP3]], label [[TMP9:%.*]], label [[TMP4:%.*]], !prof [[PROF18:![0-9]+]] +; HOT70: 4: +; HOT70-NEXT: [[TMP5:%.*]] = icmp eq ptr [[TMP1]], null +; HOT70-NEXT: br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP7:%.*]] +; HOT70: 6: +; HOT70-NEXT: tail call void @llvm.ubsantrap(i8 22) +; HOT70-NEXT: unreachable +; HOT70: 7: +; HOT70-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP1]], align 4 +; HOT70-NEXT: br label [[TMP9]] +; HOT70: 9: +; HOT70-NEXT: [[TMP10:%.*]] = phi i32 [ [[TMP8]], [[TMP7]] ], [ 0, [[TMP2:%.*]] ] +; HOT70-NEXT: ret i32 [[TMP10]] +; + %3 = icmp eq i32 %0, 0 + br i1 %3, label %9, label %4, !prof !38 + +4: + %5 = icmp eq ptr %1, null + br i1 %5, label %6, label %7 + +6: + tail call void @llvm.ubsantrap(i8 22) #2 + unreachable + +7: + %8 = load i32, ptr %1, align 4 + br label %9 + +9: + %10 = phi i32 [ %8, %7 ], [ 0, %2 ] + ret i32 %10 +} + +define dso_local noundef i32 @branchHotFnCold(i32 noundef %0, ptr noundef readonly %1) !prof !36 { +; NOPROFILE-LABEL: define dso_local noundef i32 @branchHotFnCold( +; NOPROFILE-SAME: i32 noundef [[TMP0:%.*]], ptr noundef readonly [[TMP1:%.*]]) !prof [[PROF16]] { +; NOPROFILE-NEXT: [[TMP3:%.*]] = icmp eq i32 [[TMP0]], 0 +; NOPROFILE-NEXT: br i1 [[TMP3]], label [[TMP9:%.*]], label [[TMP4:%.*]], !prof [[PROF19:![0-9]+]] +; NOPROFILE: 4: +; NOPROFILE-NEXT: [[TMP5:%.*]] = icmp eq ptr [[TMP1]], null +; NOPROFILE-NEXT: br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP7:%.*]] +; NOPROFILE: 6: +; NOPROFILE-NEXT: tail call void @llvm.ubsantrap(i8 22) +; NOPROFILE-NEXT: unreachable +; NOPROFILE: 7: +; NOPROFILE-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP1]], align 4 +; NOPROFILE-NEXT: br label [[TMP9]] +; NOPROFILE: 9: +; NOPROFILE-NEXT: [[TMP10:%.*]] = phi i32 [ [[TMP8]], [[TMP7]] ], [ 0, [[TMP2:%.*]] ] +; NOPROFILE-NEXT: ret i32 [[TMP10]] +; +; ALL-LABEL: define dso_local noundef i32 @branchHotFnCold( +; ALL-SAME: i32 noundef [[TMP0:%.*]], ptr noundef readonly [[TMP1:%.*]]) !prof [[PROF16]] { +; ALL-NEXT: [[TMP3:%.*]] = icmp eq i32 [[TMP0]], 0 +; ALL-NEXT: br i1 [[TMP3]], label [[TMP9:%.*]], label [[TMP4:%.*]], !prof [[PROF19:![0-9]+]] +; ALL: 4: +; ALL-NEXT: [[TMP5:%.*]] = icmp eq ptr [[TMP1]], null +; ALL-NEXT: br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP7:%.*]] +; ALL: 6: +; ALL-NEXT: unreachable +; ALL: 7: +; ALL-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP1]], align 4 +; ALL-NEXT: br label [[TMP9]] +; ALL: 9: +; ALL-NEXT: [[TMP10:%.*]] = phi i32 [ [[TMP8]], [[TMP7]] ], [ 0, [[TMP2:%.*]] ] +; ALL-NEXT: ret i32 [[TMP10]] +; +; HOT-LABEL: define dso_local noundef i32 @branchHotFnCold( +; HOT-SAME: i32 noundef [[TMP0:%.*]], ptr noundef readonly [[TMP1:%.*]]) !prof [[PROF16]] { +; HOT-NEXT: [[TMP3:%.*]] = icmp eq i32 [[TMP0]], 0 +; HOT-NEXT: br i1 [[TMP3]], label [[TMP9:%.*]], label [[TMP4:%.*]], !prof [[PROF19:![0-9]+]] +; HOT: 4: +; HOT-NEXT: [[TMP5:%.*]] = icmp eq ptr [[TMP1]], null +; HOT-NEXT: br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP7:%.*]] +; HOT: 6: +; HOT-NEXT: unreachable +; HOT: 7: +; HOT-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP1]], align 4 +; HOT-NEXT: br label [[TMP9]] +; HOT: 9: +; HOT-NEXT: [[TMP10:%.*]] = phi i32 [ [[TMP8]], [[TMP7]] ], [ 0, [[TMP2:%.*]] ] +; HOT-NEXT: ret i32 [[TMP10]] +; +; HOT70-LABEL: define dso_local noundef i32 @branchHotFnCold( +; HOT70-SAME: i32 noundef [[TMP0:%.*]], ptr noundef readonly [[TMP1:%.*]]) !prof [[PROF16]] { +; HOT70-NEXT: [[TMP3:%.*]] = icmp eq i32 [[TMP0]], 0 +; HOT70-NEXT: br i1 [[TMP3]], label [[TMP9:%.*]], label [[TMP4:%.*]], !prof [[PROF19:![0-9]+]] +; HOT70: 4: +; HOT70-NEXT: [[TMP5:%.*]] = icmp eq ptr [[TMP1]], null +; HOT70-NEXT: br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP7:%.*]] +; HOT70: 6: +; HOT70-NEXT: tail call void @llvm.ubsantrap(i8 22) +; HOT70-NEXT: unreachable +; HOT70: 7: +; HOT70-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP1]], align 4 +; HOT70-NEXT: br label [[TMP9]] +; HOT70: 9: +; HOT70-NEXT: [[TMP10:%.*]] = phi i32 [ [[TMP8]], [[TMP7]] ], [ 0, [[TMP2:%.*]] ] +; HOT70-NEXT: ret i32 [[TMP10]] +; + %3 = icmp eq i32 %0, 0 + br i1 %3, label %9, label %4, !prof !37 + +4: + %5 = icmp eq ptr %1, null + br i1 %5, label %6, label %7 + +6: + tail call void @llvm.ubsantrap(i8 22) #2 + unreachable + +7: + %8 = load i32, ptr %1, align 4 + br label %9 + +9: + %10 = phi i32 [ %8, %7 ], [ 0, %2 ] + ret i32 %10 +} + +!llvm.module.flags = !{!6} +!6 = !{i32 1, !"ProfileSummary", !7} +!7 = !{!8, !9, !10, !11, !12, !13, !14, !17} +!8 = !{!"ProfileFormat", !"InstrProf"} +!9 = !{!"TotalCount", i64 30000} +!10 = !{!"MaxCount", i64 10000} +!11 = !{!"MaxInternalCount", i64 10000} +!12 = !{!"MaxFunctionCount", i64 10000} +!13 = !{!"NumCounts", i64 3} +!14 = !{!"NumFunctions", i64 5} +!17 = !{!"DetailedSummary", !18} +!18 = !{!19, !29, !30, !32, !34} +!19 = !{i32 10000, i64 10000, i32 3} +!29 = !{i32 950000, i64 5000, i32 3} +!30 = !{i32 990000, i64 500, i32 4} +!32 = !{i32 999900, i64 250, i32 4} +!34 = !{i32 999999, i64 1, i32 6} + +!36 = !{!"function_entry_count", i64 1000} +!39 = !{!"function_entry_count", i64 7000} + +!37 = !{!"branch_weights", i32 1, i32 1000} +!38 = !{!"branch_weights", i32 1000, i32 1} + +;. +; NOPROFILE: [[PROF16]] = !{!"function_entry_count", i64 1000} +; NOPROFILE: [[PROF17]] = !{!"function_entry_count", i64 7000} +; NOPROFILE: [[PROF18]] = !{!"branch_weights", i32 1000, i32 1} +; NOPROFILE: [[PROF19]] = !{!"branch_weights", i32 1, i32 1000} +;. +; ALL: [[PROF16]] = !{!"function_entry_count", i64 1000} +; ALL: [[PROF17]] = !{!"function_entry_count", i64 7000} +; ALL: [[PROF18]] = !{!"branch_weights", i32 1000, i32 1} +; ALL: [[PROF19]] = !{!"branch_weights", i32 1, i32 1000} +;. +; HOT: [[PROF16]] = !{!"function_entry_count", i64 1000} +; HOT: [[PROF17]] = !{!"function_entry_count", i64 7000} +; HOT: [[PROF18]] = !{!"branch_weights", i32 1000, i32 1} +; HOT: [[PROF19]] = !{!"branch_weights", i32 1, i32 1000} +;. +; HOT70: [[PROF16]] = !{!"function_entry_count", i64 1000} +; HOT70: [[PROF17]] = !{!"function_entry_count", i64 7000} +; HOT70: [[PROF18]] = !{!"branch_weights", i32 1000, i32 1} +; HOT70: [[PROF19]] = !{!"branch_weights", i32 1, i32 1000} +;. -- GitLab From 54c955b828bbdcf46586556339cbd3cf8f205b4f Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Thu, 29 Feb 2024 15:06:28 -0800 Subject: [PATCH 477/929] Change the return type of ValueObject::CalculateNumChildren to uint32_t. In the end this value comes from TypeSystem::GetNumChildren which returns a uint32_t, so ValueObject should be consistent with that. --- lldb/include/lldb/Core/ValueObject.h | 2 +- lldb/include/lldb/Core/ValueObjectCast.h | 2 +- lldb/include/lldb/Core/ValueObjectChild.h | 2 +- lldb/include/lldb/Core/ValueObjectConstResult.h | 2 +- lldb/include/lldb/Core/ValueObjectDynamicValue.h | 2 +- lldb/include/lldb/Core/ValueObjectMemory.h | 2 +- lldb/include/lldb/Core/ValueObjectRegister.h | 4 ++-- lldb/include/lldb/Core/ValueObjectSyntheticFilter.h | 2 +- lldb/include/lldb/Core/ValueObjectVTable.h | 2 +- lldb/include/lldb/Core/ValueObjectVariable.h | 2 +- lldb/include/lldb/Target/StackFrameRecognizer.h | 2 +- lldb/source/Core/ValueObjectCast.cpp | 2 +- lldb/source/Core/ValueObjectChild.cpp | 2 +- lldb/source/Core/ValueObjectConstResult.cpp | 2 +- lldb/source/Core/ValueObjectDynamicValue.cpp | 2 +- lldb/source/Core/ValueObjectMemory.cpp | 2 +- lldb/source/Core/ValueObjectRegister.cpp | 4 ++-- lldb/source/Core/ValueObjectSyntheticFilter.cpp | 2 +- lldb/source/Core/ValueObjectVTable.cpp | 4 ++-- lldb/source/Core/ValueObjectVariable.cpp | 2 +- 20 files changed, 23 insertions(+), 23 deletions(-) diff --git a/lldb/include/lldb/Core/ValueObject.h b/lldb/include/lldb/Core/ValueObject.h index 4c0b0b2dae6c..05dd64f5634f 100644 --- a/lldb/include/lldb/Core/ValueObject.h +++ b/lldb/include/lldb/Core/ValueObject.h @@ -958,7 +958,7 @@ protected: int32_t synthetic_index); /// Should only be called by ValueObject::GetNumChildren(). - virtual size_t CalculateNumChildren(uint32_t max = UINT32_MAX) = 0; + virtual uint32_t CalculateNumChildren(uint32_t max = UINT32_MAX) = 0; void SetNumChildren(size_t num_children); diff --git a/lldb/include/lldb/Core/ValueObjectCast.h b/lldb/include/lldb/Core/ValueObjectCast.h index fe053c12d9c3..51c647680d52 100644 --- a/lldb/include/lldb/Core/ValueObjectCast.h +++ b/lldb/include/lldb/Core/ValueObjectCast.h @@ -33,7 +33,7 @@ public: std::optional GetByteSize() override; - size_t CalculateNumChildren(uint32_t max) override; + uint32_t CalculateNumChildren(uint32_t max) override; lldb::ValueType GetValueType() const override; diff --git a/lldb/include/lldb/Core/ValueObjectChild.h b/lldb/include/lldb/Core/ValueObjectChild.h index 46b14e6840f0..47a13be08bb8 100644 --- a/lldb/include/lldb/Core/ValueObjectChild.h +++ b/lldb/include/lldb/Core/ValueObjectChild.h @@ -39,7 +39,7 @@ public: lldb::ValueType GetValueType() const override; - size_t CalculateNumChildren(uint32_t max) override; + uint32_t CalculateNumChildren(uint32_t max) override; ConstString GetTypeName() override; diff --git a/lldb/include/lldb/Core/ValueObjectConstResult.h b/lldb/include/lldb/Core/ValueObjectConstResult.h index d61df859bebc..9f1246cf2a78 100644 --- a/lldb/include/lldb/Core/ValueObjectConstResult.h +++ b/lldb/include/lldb/Core/ValueObjectConstResult.h @@ -67,7 +67,7 @@ public: lldb::ValueType GetValueType() const override; - size_t CalculateNumChildren(uint32_t max) override; + uint32_t CalculateNumChildren(uint32_t max) override; ConstString GetTypeName() override; diff --git a/lldb/include/lldb/Core/ValueObjectDynamicValue.h b/lldb/include/lldb/Core/ValueObjectDynamicValue.h index 2758b4e5bb56..21a9b409fd5b 100644 --- a/lldb/include/lldb/Core/ValueObjectDynamicValue.h +++ b/lldb/include/lldb/Core/ValueObjectDynamicValue.h @@ -43,7 +43,7 @@ public: ConstString GetDisplayTypeName() override; - size_t CalculateNumChildren(uint32_t max) override; + uint32_t CalculateNumChildren(uint32_t max) override; lldb::ValueType GetValueType() const override; diff --git a/lldb/include/lldb/Core/ValueObjectMemory.h b/lldb/include/lldb/Core/ValueObjectMemory.h index 3c01df388d2e..a74b325546b0 100644 --- a/lldb/include/lldb/Core/ValueObjectMemory.h +++ b/lldb/include/lldb/Core/ValueObjectMemory.h @@ -47,7 +47,7 @@ public: ConstString GetDisplayTypeName() override; - size_t CalculateNumChildren(uint32_t max) override; + uint32_t CalculateNumChildren(uint32_t max) override; lldb::ValueType GetValueType() const override; diff --git a/lldb/include/lldb/Core/ValueObjectRegister.h b/lldb/include/lldb/Core/ValueObjectRegister.h index 2e47eee3d7f7..6c470c1a6865 100644 --- a/lldb/include/lldb/Core/ValueObjectRegister.h +++ b/lldb/include/lldb/Core/ValueObjectRegister.h @@ -47,7 +47,7 @@ public: ConstString GetQualifiedTypeName() override; - size_t CalculateNumChildren(uint32_t max) override; + uint32_t CalculateNumChildren(uint32_t max) override; ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member, int32_t synthetic_index) override; @@ -95,7 +95,7 @@ public: ConstString GetTypeName() override; - size_t CalculateNumChildren(uint32_t max) override; + uint32_t CalculateNumChildren(uint32_t max) override; bool SetValueFromCString(const char *value_str, Status &error) override; diff --git a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h b/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h index 67596232eafd..57794072ff92 100644 --- a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h +++ b/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h @@ -47,7 +47,7 @@ public: bool MightHaveChildren() override; - size_t CalculateNumChildren(uint32_t max) override; + uint32_t CalculateNumChildren(uint32_t max) override; lldb::ValueType GetValueType() const override; diff --git a/lldb/include/lldb/Core/ValueObjectVTable.h b/lldb/include/lldb/Core/ValueObjectVTable.h index 217ff8d0d334..e7e14fc83d78 100644 --- a/lldb/include/lldb/Core/ValueObjectVTable.h +++ b/lldb/include/lldb/Core/ValueObjectVTable.h @@ -64,7 +64,7 @@ public: std::optional GetByteSize() override; - size_t CalculateNumChildren(uint32_t max) override; + uint32_t CalculateNumChildren(uint32_t max) override; ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member, int32_t synthetic_index) override; diff --git a/lldb/include/lldb/Core/ValueObjectVariable.h b/lldb/include/lldb/Core/ValueObjectVariable.h index bba28ce567b2..da270300df0b 100644 --- a/lldb/include/lldb/Core/ValueObjectVariable.h +++ b/lldb/include/lldb/Core/ValueObjectVariable.h @@ -46,7 +46,7 @@ public: ConstString GetDisplayTypeName() override; - size_t CalculateNumChildren(uint32_t max) override; + uint32_t CalculateNumChildren(uint32_t max) override; lldb::ValueType GetValueType() const override; diff --git a/lldb/include/lldb/Target/StackFrameRecognizer.h b/lldb/include/lldb/Target/StackFrameRecognizer.h index 419f0c0aac1f..e111f4a4dc70 100644 --- a/lldb/include/lldb/Target/StackFrameRecognizer.h +++ b/lldb/include/lldb/Target/StackFrameRecognizer.h @@ -164,7 +164,7 @@ class ValueObjectRecognizerSynthesizedValue : public ValueObject { m_value = m_parent->GetValue(); return true; } - size_t CalculateNumChildren(uint32_t max = UINT32_MAX) override { + uint32_t CalculateNumChildren(uint32_t max = UINT32_MAX) override { return m_parent->GetNumChildren(max); } CompilerType GetCompilerTypeImpl() override { diff --git a/lldb/source/Core/ValueObjectCast.cpp b/lldb/source/Core/ValueObjectCast.cpp index 0882d4b36776..a5c555f86b13 100644 --- a/lldb/source/Core/ValueObjectCast.cpp +++ b/lldb/source/Core/ValueObjectCast.cpp @@ -41,7 +41,7 @@ ValueObjectCast::~ValueObjectCast() = default; CompilerType ValueObjectCast::GetCompilerTypeImpl() { return m_cast_type; } -size_t ValueObjectCast::CalculateNumChildren(uint32_t max) { +uint32_t ValueObjectCast::CalculateNumChildren(uint32_t max) { ExecutionContext exe_ctx(GetExecutionContextRef()); auto children_count = GetCompilerType().GetNumChildren( true, &exe_ctx); diff --git a/lldb/source/Core/ValueObjectChild.cpp b/lldb/source/Core/ValueObjectChild.cpp index 39067387dc97..2e55dd7726bd 100644 --- a/lldb/source/Core/ValueObjectChild.cpp +++ b/lldb/source/Core/ValueObjectChild.cpp @@ -49,7 +49,7 @@ lldb::ValueType ValueObjectChild::GetValueType() const { return m_parent->GetValueType(); } -size_t ValueObjectChild::CalculateNumChildren(uint32_t max) { +uint32_t ValueObjectChild::CalculateNumChildren(uint32_t max) { ExecutionContext exe_ctx(GetExecutionContextRef()); auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx); return children_count <= max ? children_count : max; diff --git a/lldb/source/Core/ValueObjectConstResult.cpp b/lldb/source/Core/ValueObjectConstResult.cpp index 693da1a551f8..5c7aa4452b70 100644 --- a/lldb/source/Core/ValueObjectConstResult.cpp +++ b/lldb/source/Core/ValueObjectConstResult.cpp @@ -216,7 +216,7 @@ std::optional ValueObjectConstResult::GetByteSize() { void ValueObjectConstResult::SetByteSize(size_t size) { m_byte_size = size; } -size_t ValueObjectConstResult::CalculateNumChildren(uint32_t max) { +uint32_t ValueObjectConstResult::CalculateNumChildren(uint32_t max) { ExecutionContext exe_ctx(GetExecutionContextRef()); auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx); return children_count <= max ? children_count : max; diff --git a/lldb/source/Core/ValueObjectDynamicValue.cpp b/lldb/source/Core/ValueObjectDynamicValue.cpp index e6e30dce9d1e..4e64760371ae 100644 --- a/lldb/source/Core/ValueObjectDynamicValue.cpp +++ b/lldb/source/Core/ValueObjectDynamicValue.cpp @@ -85,7 +85,7 @@ ConstString ValueObjectDynamicValue::GetDisplayTypeName() { return m_parent->GetDisplayTypeName(); } -size_t ValueObjectDynamicValue::CalculateNumChildren(uint32_t max) { +uint32_t ValueObjectDynamicValue::CalculateNumChildren(uint32_t max) { const bool success = UpdateValueIfNeeded(false); if (success && m_dynamic_type_info.HasType()) { ExecutionContext exe_ctx(GetExecutionContextRef()); diff --git a/lldb/source/Core/ValueObjectMemory.cpp b/lldb/source/Core/ValueObjectMemory.cpp index 3f125a7bee8c..7f68236c7884 100644 --- a/lldb/source/Core/ValueObjectMemory.cpp +++ b/lldb/source/Core/ValueObjectMemory.cpp @@ -126,7 +126,7 @@ ConstString ValueObjectMemory::GetDisplayTypeName() { return m_compiler_type.GetDisplayTypeName(); } -size_t ValueObjectMemory::CalculateNumChildren(uint32_t max) { +uint32_t ValueObjectMemory::CalculateNumChildren(uint32_t max) { if (m_type_sp) { auto child_count = m_type_sp->GetNumChildren(true); return child_count <= max ? child_count : max; diff --git a/lldb/source/Core/ValueObjectRegister.cpp b/lldb/source/Core/ValueObjectRegister.cpp index c2b84c113473..d4c144cc7edb 100644 --- a/lldb/source/Core/ValueObjectRegister.cpp +++ b/lldb/source/Core/ValueObjectRegister.cpp @@ -74,7 +74,7 @@ ConstString ValueObjectRegisterSet::GetQualifiedTypeName() { return ConstString(); } -size_t ValueObjectRegisterSet::CalculateNumChildren(uint32_t max) { +uint32_t ValueObjectRegisterSet::CalculateNumChildren(uint32_t max) { const RegisterSet *reg_set = m_reg_ctx_sp->GetRegisterSet(m_reg_set_idx); if (reg_set) { auto reg_count = reg_set->num_registers; @@ -220,7 +220,7 @@ ConstString ValueObjectRegister::GetTypeName() { return m_type_name; } -size_t ValueObjectRegister::CalculateNumChildren(uint32_t max) { +uint32_t ValueObjectRegister::CalculateNumChildren(uint32_t max) { ExecutionContext exe_ctx(GetExecutionContextRef()); auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx); return children_count <= max ? children_count : max; diff --git a/lldb/source/Core/ValueObjectSyntheticFilter.cpp b/lldb/source/Core/ValueObjectSyntheticFilter.cpp index e8b4b02d11a0..ae358fba4bd8 100644 --- a/lldb/source/Core/ValueObjectSyntheticFilter.cpp +++ b/lldb/source/Core/ValueObjectSyntheticFilter.cpp @@ -84,7 +84,7 @@ ConstString ValueObjectSynthetic::GetDisplayTypeName() { return m_parent->GetDisplayTypeName(); } -size_t ValueObjectSynthetic::CalculateNumChildren(uint32_t max) { +uint32_t ValueObjectSynthetic::CalculateNumChildren(uint32_t max) { Log *log = GetLog(LLDBLog::DataFormatters); UpdateValueIfNeeded(); diff --git a/lldb/source/Core/ValueObjectVTable.cpp b/lldb/source/Core/ValueObjectVTable.cpp index 177ae4167a1d..4d1cbb8d2f6f 100644 --- a/lldb/source/Core/ValueObjectVTable.cpp +++ b/lldb/source/Core/ValueObjectVTable.cpp @@ -33,7 +33,7 @@ public: std::optional GetByteSize() override { return m_addr_size; }; - size_t CalculateNumChildren(uint32_t max) override { return 0; }; + uint32_t CalculateNumChildren(uint32_t max) override { return 0; }; ValueType GetValueType() const override { return eValueTypeVTableEntry; }; @@ -159,7 +159,7 @@ std::optional ValueObjectVTable::GetByteSize() { return std::nullopt; } -size_t ValueObjectVTable::CalculateNumChildren(uint32_t max) { +uint32_t ValueObjectVTable::CalculateNumChildren(uint32_t max) { if (UpdateValueIfNeeded(false)) return m_num_vtable_entries <= max ? m_num_vtable_entries : max; return 0; diff --git a/lldb/source/Core/ValueObjectVariable.cpp b/lldb/source/Core/ValueObjectVariable.cpp index 9f8df847f28a..dc62bb6358dc 100644 --- a/lldb/source/Core/ValueObjectVariable.cpp +++ b/lldb/source/Core/ValueObjectVariable.cpp @@ -94,7 +94,7 @@ ConstString ValueObjectVariable::GetQualifiedTypeName() { return ConstString(); } -size_t ValueObjectVariable::CalculateNumChildren(uint32_t max) { +uint32_t ValueObjectVariable::CalculateNumChildren(uint32_t max) { CompilerType type(GetCompilerType()); if (!type.IsValid()) -- GitLab From 3d7c5b80e38b01223811eb557a5e9953cfa2154d Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Thu, 29 Feb 2024 15:35:35 -0800 Subject: [PATCH 478/929] Change the return type of SyntheticFrontend::CalculateNumChildren to int32_t This way it is consistent with ValueObject and TypeSystem. --- .../lldb/DataFormatters/TypeSynthetic.h | 12 +++++----- .../lldb/DataFormatters/VectorIterator.h | 2 +- .../Core/ValueObjectSyntheticFilter.cpp | 2 +- lldb/source/DataFormatters/TypeSynthetic.cpp | 4 ++-- lldb/source/DataFormatters/VectorType.cpp | 2 +- .../Language/CPlusPlus/BlockPointer.cpp | 2 +- .../Plugins/Language/CPlusPlus/Coroutines.cpp | 2 +- .../Plugins/Language/CPlusPlus/Coroutines.h | 2 +- .../Language/CPlusPlus/GenericBitset.cpp | 2 +- .../Language/CPlusPlus/GenericOptional.cpp | 2 +- .../Plugins/Language/CPlusPlus/LibCxx.cpp | 8 +++---- .../Plugins/Language/CPlusPlus/LibCxx.h | 8 +++---- .../Language/CPlusPlus/LibCxxAtomic.cpp | 4 ++-- .../CPlusPlus/LibCxxInitializerList.cpp | 4 ++-- .../Plugins/Language/CPlusPlus/LibCxxList.cpp | 8 +++---- .../Plugins/Language/CPlusPlus/LibCxxMap.cpp | 4 ++-- .../Language/CPlusPlus/LibCxxQueue.cpp | 2 +- .../CPlusPlus/LibCxxRangesRefView.cpp | 2 +- .../Plugins/Language/CPlusPlus/LibCxxSpan.cpp | 4 ++-- .../Language/CPlusPlus/LibCxxTuple.cpp | 2 +- .../Language/CPlusPlus/LibCxxUnorderedMap.cpp | 4 ++-- .../Language/CPlusPlus/LibCxxValarray.cpp | 4 ++-- .../Language/CPlusPlus/LibCxxVariant.cpp | 2 +- .../Language/CPlusPlus/LibCxxVector.cpp | 8 +++---- .../Plugins/Language/CPlusPlus/LibStdcpp.cpp | 10 ++++---- .../Language/CPlusPlus/LibStdcppTuple.cpp | 4 ++-- .../CPlusPlus/LibStdcppUniquePointer.cpp | 4 ++-- lldb/source/Plugins/Language/ObjC/Cocoa.cpp | 2 +- lldb/source/Plugins/Language/ObjC/NSArray.cpp | 18 +++++++------- .../Plugins/Language/ObjC/NSDictionary.cpp | 24 +++++++++---------- lldb/source/Plugins/Language/ObjC/NSError.cpp | 2 +- .../Plugins/Language/ObjC/NSException.cpp | 2 +- .../Plugins/Language/ObjC/NSIndexPath.cpp | 2 +- lldb/source/Plugins/Language/ObjC/NSSet.cpp | 14 +++++------ 34 files changed, 89 insertions(+), 89 deletions(-) diff --git a/lldb/include/lldb/DataFormatters/TypeSynthetic.h b/lldb/include/lldb/DataFormatters/TypeSynthetic.h index 23cc054b399a..7bb011c15794 100644 --- a/lldb/include/lldb/DataFormatters/TypeSynthetic.h +++ b/lldb/include/lldb/DataFormatters/TypeSynthetic.h @@ -38,9 +38,9 @@ public: virtual ~SyntheticChildrenFrontEnd() = default; - virtual size_t CalculateNumChildren() = 0; + virtual uint32_t CalculateNumChildren() = 0; - virtual size_t CalculateNumChildren(uint32_t max) { + virtual uint32_t CalculateNumChildren(uint32_t max) { auto count = CalculateNumChildren(); return count <= max ? count : max; } @@ -109,7 +109,7 @@ public: ~SyntheticValueProviderFrontEnd() override = default; - size_t CalculateNumChildren() override { return 0; } + uint32_t CalculateNumChildren() override { return 0; } lldb::ValueObjectSP GetChildAtIndex(size_t idx) override { return nullptr; } @@ -322,7 +322,7 @@ public: ~FrontEnd() override = default; - size_t CalculateNumChildren() override { return filter->GetCount(); } + uint32_t CalculateNumChildren() override { return filter->GetCount(); } lldb::ValueObjectSP GetChildAtIndex(size_t idx) override { if (idx >= filter->GetCount()) @@ -426,9 +426,9 @@ public: bool IsValid(); - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; - size_t CalculateNumChildren(uint32_t max) override; + uint32_t CalculateNumChildren(uint32_t max) override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; diff --git a/lldb/include/lldb/DataFormatters/VectorIterator.h b/lldb/include/lldb/DataFormatters/VectorIterator.h index 5f774bb72c3a..88500b0bfdd4 100644 --- a/lldb/include/lldb/DataFormatters/VectorIterator.h +++ b/lldb/include/lldb/DataFormatters/VectorIterator.h @@ -24,7 +24,7 @@ public: VectorIteratorSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp, llvm::ArrayRef item_names); - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; diff --git a/lldb/source/Core/ValueObjectSyntheticFilter.cpp b/lldb/source/Core/ValueObjectSyntheticFilter.cpp index ae358fba4bd8..b03bd9a80e50 100644 --- a/lldb/source/Core/ValueObjectSyntheticFilter.cpp +++ b/lldb/source/Core/ValueObjectSyntheticFilter.cpp @@ -31,7 +31,7 @@ public: DummySyntheticFrontEnd(ValueObject &backend) : SyntheticChildrenFrontEnd(backend) {} - size_t CalculateNumChildren() override { return m_backend.GetNumChildren(); } + uint32_t CalculateNumChildren() override { return m_backend.GetNumChildren(); } lldb::ValueObjectSP GetChildAtIndex(size_t idx) override { return m_backend.GetChildAtIndex(idx); diff --git a/lldb/source/DataFormatters/TypeSynthetic.cpp b/lldb/source/DataFormatters/TypeSynthetic.cpp index 8a6f132a3957..a05fbe9a73c2 100644 --- a/lldb/source/DataFormatters/TypeSynthetic.cpp +++ b/lldb/source/DataFormatters/TypeSynthetic.cpp @@ -178,13 +178,13 @@ bool ScriptedSyntheticChildren::FrontEnd::IsValid() { return (m_wrapper_sp && m_wrapper_sp->IsValid() && m_interpreter); } -size_t ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren() { +uint32_t ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren() { if (!m_wrapper_sp || m_interpreter == nullptr) return 0; return m_interpreter->CalculateNumChildren(m_wrapper_sp, UINT32_MAX); } -size_t ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren(uint32_t max) { +uint32_t ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren(uint32_t max) { if (!m_wrapper_sp || m_interpreter == nullptr) return 0; return m_interpreter->CalculateNumChildren(m_wrapper_sp, max); diff --git a/lldb/source/DataFormatters/VectorType.cpp b/lldb/source/DataFormatters/VectorType.cpp index c94ca68319ee..6f1f0e5b058f 100644 --- a/lldb/source/DataFormatters/VectorType.cpp +++ b/lldb/source/DataFormatters/VectorType.cpp @@ -224,7 +224,7 @@ public: ~VectorTypeSyntheticFrontEnd() override = default; - size_t CalculateNumChildren() override { return m_num_children; } + uint32_t CalculateNumChildren() override { return m_num_children; } lldb::ValueObjectSP GetChildAtIndex(size_t idx) override { if (idx >= CalculateNumChildren()) diff --git a/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp b/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp index 2e43aa3fa1d8..f8e47efa7d91 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp @@ -74,7 +74,7 @@ public: ~BlockPointerSyntheticFrontEnd() override = default; - size_t CalculateNumChildren() override { + uint32_t CalculateNumChildren() override { const bool omit_empty_base_classes = false; return m_block_struct_type.GetNumChildren(omit_empty_base_classes, nullptr); } diff --git a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp index 742017438bcf..ea1218350f6c 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp @@ -104,7 +104,7 @@ lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd:: lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd:: ~StdlibCoroutineHandleSyntheticFrontEnd() = default; -size_t lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd:: +uint32_t lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd:: CalculateNumChildren() { if (!m_resume_ptr_sp || !m_destroy_ptr_sp) return 0; diff --git a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h index d38c7ecefa6e..8ec2c14d2e9e 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h +++ b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h @@ -34,7 +34,7 @@ public: ~StdlibCoroutineHandleSyntheticFrontEnd() override; - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; diff --git a/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp b/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp index ac3166385235..c8fc3af4a9d4 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp @@ -34,7 +34,7 @@ public: bool MightHaveChildren() override { return true; } lldb::ChildCacheState Update() override; - size_t CalculateNumChildren() override { return m_elements.size(); } + uint32_t CalculateNumChildren() override { return m_elements.size(); } ValueObjectSP GetChildAtIndex(size_t idx) override; private: diff --git a/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp b/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp index 57331eaa9868..34ae1a248921 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp @@ -41,7 +41,7 @@ public: } bool MightHaveChildren() override { return true; } - size_t CalculateNumChildren() override { return m_has_value ? 1U : 0U; } + uint32_t CalculateNumChildren() override { return m_has_value ? 1U : 0U; } ValueObjectSP GetChildAtIndex(size_t idx) override; lldb::ChildCacheState Update() override; diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp index 7893aa7cc1f9..710b235ce957 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp @@ -351,7 +351,7 @@ lldb_private::formatters::LibCxxMapIteratorSyntheticFrontEnd::Update() { return lldb::ChildCacheState::eRefetch; } -size_t lldb_private::formatters::LibCxxMapIteratorSyntheticFrontEnd:: +uint32_t lldb_private::formatters::LibCxxMapIteratorSyntheticFrontEnd:: CalculateNumChildren() { return 2; } @@ -509,7 +509,7 @@ lldb::ChildCacheState lldb_private::formatters:: return lldb::ChildCacheState::eRefetch; } -size_t lldb_private::formatters::LibCxxUnorderedMapIteratorSyntheticFrontEnd:: +uint32_t lldb_private::formatters::LibCxxUnorderedMapIteratorSyntheticFrontEnd:: CalculateNumChildren() { return 2; } @@ -566,7 +566,7 @@ lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd:: Update(); } -size_t lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd:: +uint32_t lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd:: CalculateNumChildren() { return (m_cntrl ? 1 : 0); } @@ -661,7 +661,7 @@ lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEndCreator( : nullptr); } -size_t lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEnd:: +uint32_t lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEnd:: CalculateNumChildren() { if (m_value_ptr_sp) return m_deleter_sp ? 2 : 1; diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h index d823fbd76222..af4f313a9050 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h @@ -87,7 +87,7 @@ class LibCxxMapIteratorSyntheticFrontEnd : public SyntheticChildrenFrontEnd { public: LibCxxMapIteratorSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp); - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -135,7 +135,7 @@ public: ~LibCxxUnorderedMapIteratorSyntheticFrontEnd() override = default; - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -166,7 +166,7 @@ class LibcxxSharedPtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd { public: LibcxxSharedPtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp); - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -186,7 +186,7 @@ class LibcxxUniquePtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd { public: LibcxxUniquePtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp); - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp index c81b1e8012f6..484d756a300e 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp @@ -90,7 +90,7 @@ public: ~LibcxxStdAtomicSyntheticFrontEnd() override = default; - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -124,7 +124,7 @@ bool lldb_private::formatters::LibcxxStdAtomicSyntheticFrontEnd:: return true; } -size_t lldb_private::formatters::LibcxxStdAtomicSyntheticFrontEnd:: +uint32_t lldb_private::formatters::LibcxxStdAtomicSyntheticFrontEnd:: CalculateNumChildren() { return m_real_child ? 1 : 0; } diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp index 3c33f94f9237..ef94815b3afc 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp @@ -26,7 +26,7 @@ public: ~LibcxxInitializerListSyntheticFrontEnd() override; - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -59,7 +59,7 @@ lldb_private::formatters::LibcxxInitializerListSyntheticFrontEnd:: // delete m_start; } -size_t lldb_private::formatters::LibcxxInitializerListSyntheticFrontEnd:: +uint32_t lldb_private::formatters::LibcxxInitializerListSyntheticFrontEnd:: CalculateNumChildren() { m_num_elements = 0; ValueObjectSP size_sp(m_backend.GetChildMemberWithName("__size_")); diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp index e28ef818b10f..d05a64b1eba5 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp @@ -136,7 +136,7 @@ class ForwardListFrontEnd : public AbstractListFrontEnd { public: ForwardListFrontEnd(ValueObject &valobj); - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; ValueObjectSP GetChildAtIndex(size_t idx) override; lldb::ChildCacheState Update() override; }; @@ -147,7 +147,7 @@ public: ~ListFrontEnd() override = default; - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -240,7 +240,7 @@ ForwardListFrontEnd::ForwardListFrontEnd(ValueObject &valobj) Update(); } -size_t ForwardListFrontEnd::CalculateNumChildren() { +uint32_t ForwardListFrontEnd::CalculateNumChildren() { if (m_count != UINT32_MAX) return m_count; @@ -308,7 +308,7 @@ ListFrontEnd::ListFrontEnd(lldb::ValueObjectSP valobj_sp) Update(); } -size_t ListFrontEnd::CalculateNumChildren() { +uint32_t ListFrontEnd::CalculateNumChildren() { if (m_count != UINT32_MAX) return m_count; if (!m_head || !m_tail || m_node_address == 0) diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp index d208acfc9da4..28a9a26c9d2d 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp @@ -177,7 +177,7 @@ public: ~LibcxxStdMapSyntheticFrontEnd() override = default; - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -209,7 +209,7 @@ lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd:: Update(); } -size_t lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd:: +uint32_t lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd:: CalculateNumChildren() { if (m_count != UINT32_MAX) return m_count; diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp index 83f93b16fc9a..beab453b788a 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp @@ -28,7 +28,7 @@ public: bool MightHaveChildren() override { return true; } lldb::ChildCacheState Update() override; - size_t CalculateNumChildren() override { + uint32_t CalculateNumChildren() override { return m_container_sp ? m_container_sp->GetNumChildren() : 0; } diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp index c032d67c66cb..cda6c5d79c31 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp @@ -27,7 +27,7 @@ public: ~LibcxxStdRangesRefViewSyntheticFrontEnd() override = default; - size_t CalculateNumChildren() override { + uint32_t CalculateNumChildren() override { // __range_ will be the sole child of this type return 1; } diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp index 4ddfaef9c0ad..5aaf1ec2d56e 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp @@ -27,7 +27,7 @@ public: ~LibcxxStdSpanSyntheticFrontEnd() override = default; - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -73,7 +73,7 @@ lldb_private::formatters::LibcxxStdSpanSyntheticFrontEnd:: Update(); } -size_t lldb_private::formatters::LibcxxStdSpanSyntheticFrontEnd:: +uint32_t lldb_private::formatters::LibcxxStdSpanSyntheticFrontEnd:: CalculateNumChildren() { return m_num_elements; } diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp index 546871012d2b..5c42d8551d4f 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp @@ -26,7 +26,7 @@ public: bool MightHaveChildren() override { return true; } lldb::ChildCacheState Update() override; - size_t CalculateNumChildren() override { return m_elements.size(); } + uint32_t CalculateNumChildren() override { return m_elements.size(); } ValueObjectSP GetChildAtIndex(size_t idx) override; private: diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp index 4cac52f235a1..0be73a9e633d 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp @@ -33,7 +33,7 @@ public: ~LibcxxStdUnorderedMapSyntheticFrontEnd() override = default; - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -62,7 +62,7 @@ lldb_private::formatters::LibcxxStdUnorderedMapSyntheticFrontEnd:: Update(); } -size_t lldb_private::formatters::LibcxxStdUnorderedMapSyntheticFrontEnd:: +uint32_t lldb_private::formatters::LibcxxStdUnorderedMapSyntheticFrontEnd:: CalculateNumChildren() { return m_num_elements; } diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp index 7c8fd25fd9f2..eae7711c8cd9 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp @@ -24,7 +24,7 @@ public: ~LibcxxStdValarraySyntheticFrontEnd() override; - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -63,7 +63,7 @@ lldb_private::formatters::LibcxxStdValarraySyntheticFrontEnd:: // delete m_finish; } -size_t lldb_private::formatters::LibcxxStdValarraySyntheticFrontEnd:: +uint32_t lldb_private::formatters::LibcxxStdValarraySyntheticFrontEnd:: CalculateNumChildren() { if (!m_start || !m_finish) return 0; diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp index ecbb7cf0ca2b..c704e1b8335e 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp @@ -205,7 +205,7 @@ public: bool MightHaveChildren() override { return true; } lldb::ChildCacheState Update() override; - size_t CalculateNumChildren() override { return m_size; } + uint32_t CalculateNumChildren() override { return m_size; } ValueObjectSP GetChildAtIndex(size_t idx) override; private: diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp index 0c3c3f02b60c..62ff76b82a2a 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp @@ -25,7 +25,7 @@ public: ~LibcxxStdVectorSyntheticFrontEnd() override; - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -46,7 +46,7 @@ class LibcxxVectorBoolSyntheticFrontEnd : public SyntheticChildrenFrontEnd { public: LibcxxVectorBoolSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp); - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -82,7 +82,7 @@ lldb_private::formatters::LibcxxStdVectorSyntheticFrontEnd:: // delete m_finish; } -size_t lldb_private::formatters::LibcxxStdVectorSyntheticFrontEnd:: +uint32_t lldb_private::formatters::LibcxxStdVectorSyntheticFrontEnd:: CalculateNumChildren() { if (!m_start || !m_finish) return 0; @@ -165,7 +165,7 @@ lldb_private::formatters::LibcxxVectorBoolSyntheticFrontEnd:: } } -size_t lldb_private::formatters::LibcxxVectorBoolSyntheticFrontEnd:: +uint32_t lldb_private::formatters::LibcxxVectorBoolSyntheticFrontEnd:: CalculateNumChildren() { return m_count; } diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp index 411551839e1e..b2249d23961f 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp @@ -43,7 +43,7 @@ class LibstdcppMapIteratorSyntheticFrontEnd : public SyntheticChildrenFrontEnd { public: explicit LibstdcppMapIteratorSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp); - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -64,7 +64,7 @@ class LibStdcppSharedPtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd { public: explicit LibStdcppSharedPtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp); - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -132,7 +132,7 @@ lldb::ChildCacheState LibstdcppMapIteratorSyntheticFrontEnd::Update() { return lldb::ChildCacheState::eReuse; } -size_t LibstdcppMapIteratorSyntheticFrontEnd::CalculateNumChildren() { +uint32_t LibstdcppMapIteratorSyntheticFrontEnd::CalculateNumChildren() { return 2; } @@ -219,7 +219,7 @@ lldb::ChildCacheState VectorIteratorSyntheticFrontEnd::Update() { return lldb::ChildCacheState::eRefetch; } -size_t VectorIteratorSyntheticFrontEnd::CalculateNumChildren() { return 1; } +uint32_t VectorIteratorSyntheticFrontEnd::CalculateNumChildren() { return 1; } lldb::ValueObjectSP VectorIteratorSyntheticFrontEnd::GetChildAtIndex(size_t idx) { @@ -371,7 +371,7 @@ LibStdcppSharedPtrSyntheticFrontEnd::LibStdcppSharedPtrSyntheticFrontEnd( Update(); } -size_t LibStdcppSharedPtrSyntheticFrontEnd::CalculateNumChildren() { return 1; } +uint32_t LibStdcppSharedPtrSyntheticFrontEnd::CalculateNumChildren() { return 1; } lldb::ValueObjectSP LibStdcppSharedPtrSyntheticFrontEnd::GetChildAtIndex(size_t idx) { diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp index 189f9561e52a..ba3116d01a72 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp @@ -26,7 +26,7 @@ class LibStdcppTupleSyntheticFrontEnd : public SyntheticChildrenFrontEnd { public: explicit LibStdcppTupleSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp); - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -95,7 +95,7 @@ LibStdcppTupleSyntheticFrontEnd::GetChildAtIndex(size_t idx) { return lldb::ValueObjectSP(); } -size_t LibStdcppTupleSyntheticFrontEnd::CalculateNumChildren() { +uint32_t LibStdcppTupleSyntheticFrontEnd::CalculateNumChildren() { return m_members.size(); } diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp index 3b0f6329d0e3..aca18ee694c0 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp @@ -26,7 +26,7 @@ class LibStdcppUniquePtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd { public: explicit LibStdcppUniquePtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp); - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -135,7 +135,7 @@ LibStdcppUniquePtrSyntheticFrontEnd::GetChildAtIndex(size_t idx) { return lldb::ValueObjectSP(); } -size_t LibStdcppUniquePtrSyntheticFrontEnd::CalculateNumChildren() { +uint32_t LibStdcppUniquePtrSyntheticFrontEnd::CalculateNumChildren() { if (m_del_obj) return 2; return 1; diff --git a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp index 64047dc53236..05b324a30f2d 100644 --- a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp +++ b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp @@ -1038,7 +1038,7 @@ public: ~ObjCClassSyntheticChildrenFrontEnd() override = default; - size_t CalculateNumChildren() override { return 0; } + uint32_t CalculateNumChildren() override { return 0; } lldb::ValueObjectSP GetChildAtIndex(size_t idx) override { return lldb::ValueObjectSP(); diff --git a/lldb/source/Plugins/Language/ObjC/NSArray.cpp b/lldb/source/Plugins/Language/ObjC/NSArray.cpp index 09bf7a23d609..9ee12e50b8ac 100644 --- a/lldb/source/Plugins/Language/ObjC/NSArray.cpp +++ b/lldb/source/Plugins/Language/ObjC/NSArray.cpp @@ -50,7 +50,7 @@ public: ~NSArrayMSyntheticFrontEndBase() override = default; - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -214,7 +214,7 @@ public: ~GenericNSArrayISyntheticFrontEnd() override; - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -302,7 +302,7 @@ public: ~NSArray0SyntheticFrontEnd() override = default; - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -319,7 +319,7 @@ public: ~NSArray1SyntheticFrontEnd() override = default; - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -477,8 +477,8 @@ lldb_private::formatters:: : NSArrayMSyntheticFrontEndBase(valobj_sp), m_data_32(nullptr), m_data_64(nullptr) {} -size_t -lldb_private::formatters::NSArrayMSyntheticFrontEndBase::CalculateNumChildren() { +uint32_t lldb_private::formatters::NSArrayMSyntheticFrontEndBase:: + CalculateNumChildren() { return GetUsedCount(); } @@ -634,7 +634,7 @@ lldb_private::formatters::GenericNSArrayISyntheticFrontEnd:: } template -size_t +uint32_t lldb_private::formatters::GenericNSArrayISyntheticFrontEnd:: CalculateNumChildren() { return m_data_32 ? m_data_32->used : m_data_64->used; @@ -719,7 +719,7 @@ lldb_private::formatters::NSArray0SyntheticFrontEnd::GetIndexOfChildWithName( return UINT32_MAX; } -size_t +uint32_t lldb_private::formatters::NSArray0SyntheticFrontEnd::CalculateNumChildren() { return 0; } @@ -754,7 +754,7 @@ lldb_private::formatters::NSArray1SyntheticFrontEnd::GetIndexOfChildWithName( return UINT32_MAX; } -size_t +uint32_t lldb_private::formatters::NSArray1SyntheticFrontEnd::CalculateNumChildren() { return 1; } diff --git a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp index 9c252a98de83..3bb2e4fac02e 100644 --- a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp +++ b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp @@ -103,7 +103,7 @@ public: ~NSDictionaryISyntheticFrontEnd() override; - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -144,7 +144,7 @@ class NSConstantDictionarySyntheticFrontEnd : public SyntheticChildrenFrontEnd { public: NSConstantDictionarySyntheticFrontEnd(lldb::ValueObjectSP valobj_sp); - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -176,7 +176,7 @@ class NSCFDictionarySyntheticFrontEnd : public SyntheticChildrenFrontEnd { public: NSCFDictionarySyntheticFrontEnd(lldb::ValueObjectSP valobj_sp); - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -209,7 +209,7 @@ public: ~NSDictionary1SyntheticFrontEnd() override = default; - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -230,7 +230,7 @@ public: ~GenericNSDictionaryMSyntheticFrontEnd() override; - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -263,7 +263,7 @@ namespace Foundation1100 { ~NSDictionaryMSyntheticFrontEnd() override; - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -606,7 +606,7 @@ size_t lldb_private::formatters::NSDictionaryISyntheticFrontEnd:: return idx; } -size_t lldb_private::formatters::NSDictionaryISyntheticFrontEnd:: +uint32_t lldb_private::formatters::NSDictionaryISyntheticFrontEnd:: CalculateNumChildren() { if (!m_data_32 && !m_data_64) return 0; @@ -744,7 +744,7 @@ size_t lldb_private::formatters::NSCFDictionarySyntheticFrontEnd:: return idx; } -size_t lldb_private::formatters::NSCFDictionarySyntheticFrontEnd:: +uint32_t lldb_private::formatters::NSCFDictionarySyntheticFrontEnd:: CalculateNumChildren() { if (!m_hashtable.IsValid()) return 0; @@ -880,7 +880,7 @@ size_t lldb_private::formatters::NSConstantDictionarySyntheticFrontEnd:: return idx; } -size_t lldb_private::formatters::NSConstantDictionarySyntheticFrontEnd:: +uint32_t lldb_private::formatters::NSConstantDictionarySyntheticFrontEnd:: CalculateNumChildren() { return m_size; } @@ -994,7 +994,7 @@ size_t lldb_private::formatters::NSDictionary1SyntheticFrontEnd:: return name == g_zero ? 0 : UINT32_MAX; } -size_t lldb_private::formatters::NSDictionary1SyntheticFrontEnd:: +uint32_t lldb_private::formatters::NSDictionary1SyntheticFrontEnd:: CalculateNumChildren() { return 1; } @@ -1087,7 +1087,7 @@ size_t lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd< } template -size_t +uint32_t lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd::CalculateNumChildren() { if (!m_data_32 && !m_data_64) return 0; @@ -1250,7 +1250,7 @@ lldb_private::formatters::Foundation1100:: return idx; } -size_t +uint32_t lldb_private::formatters::Foundation1100:: NSDictionaryMSyntheticFrontEnd::CalculateNumChildren() { if (!m_data_32 && !m_data_64) diff --git a/lldb/source/Plugins/Language/ObjC/NSError.cpp b/lldb/source/Plugins/Language/ObjC/NSError.cpp index ce52ae542a50..786d2bc51871 100644 --- a/lldb/source/Plugins/Language/ObjC/NSError.cpp +++ b/lldb/source/Plugins/Language/ObjC/NSError.cpp @@ -116,7 +116,7 @@ public: // no need to delete m_child_ptr - it's kept alive by the cluster manager on // our behalf - size_t CalculateNumChildren() override { + uint32_t CalculateNumChildren() override { if (m_child_ptr) return 1; if (m_child_sp) diff --git a/lldb/source/Plugins/Language/ObjC/NSException.cpp b/lldb/source/Plugins/Language/ObjC/NSException.cpp index e8011e5d2ca0..e47939c71801 100644 --- a/lldb/source/Plugins/Language/ObjC/NSException.cpp +++ b/lldb/source/Plugins/Language/ObjC/NSException.cpp @@ -123,7 +123,7 @@ public: ~NSExceptionSyntheticFrontEnd() override = default; - size_t CalculateNumChildren() override { + uint32_t CalculateNumChildren() override { return 4; } diff --git a/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp b/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp index 69e6ab1055d8..45c020f8d1b3 100644 --- a/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp +++ b/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp @@ -40,7 +40,7 @@ public: ~NSIndexPathSyntheticFrontEnd() override = default; - size_t CalculateNumChildren() override { return m_impl.GetNumIndexes(); } + uint32_t CalculateNumChildren() override { return m_impl.GetNumIndexes(); } lldb::ValueObjectSP GetChildAtIndex(size_t idx) override { return m_impl.GetIndexAtIndex(idx, m_uint_star_type); diff --git a/lldb/source/Plugins/Language/ObjC/NSSet.cpp b/lldb/source/Plugins/Language/ObjC/NSSet.cpp index ede64852d9a8..97f19c1ac937 100644 --- a/lldb/source/Plugins/Language/ObjC/NSSet.cpp +++ b/lldb/source/Plugins/Language/ObjC/NSSet.cpp @@ -46,7 +46,7 @@ public: ~NSSetISyntheticFrontEnd() override; - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -84,7 +84,7 @@ class NSCFSetSyntheticFrontEnd : public SyntheticChildrenFrontEnd { public: NSCFSetSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp); - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -117,7 +117,7 @@ public: ~GenericNSSetMSyntheticFrontEnd() override; - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -233,7 +233,7 @@ public: ~NSSetCodeRunningSyntheticFrontEnd() override; - size_t CalculateNumChildren() override; + uint32_t CalculateNumChildren() override; lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; @@ -419,7 +419,7 @@ lldb_private::formatters::NSSetISyntheticFrontEnd::GetIndexOfChildWithName( return idx; } -size_t +uint32_t lldb_private::formatters::NSSetISyntheticFrontEnd::CalculateNumChildren() { if (!m_data_32 && !m_data_64) return 0; @@ -555,7 +555,7 @@ lldb_private::formatters::NSCFSetSyntheticFrontEnd::GetIndexOfChildWithName( return idx; } -size_t +uint32_t lldb_private::formatters::NSCFSetSyntheticFrontEnd::CalculateNumChildren() { if (!m_hashtable.IsValid()) return 0; @@ -696,7 +696,7 @@ lldb_private::formatters:: } template -size_t +uint32_t lldb_private::formatters:: GenericNSSetMSyntheticFrontEnd::CalculateNumChildren() { if (!m_data_32 && !m_data_64) -- GitLab From e710523e408ce64c15fddf9f7dbe1248795c20d7 Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Mon, 4 Mar 2024 17:50:47 -0800 Subject: [PATCH 479/929] Change GetChildAtIndex to take a uint32_t --- lldb/include/lldb/Core/ValueObject.h | 4 ++-- .../lldb/Core/ValueObjectSyntheticFilter.h | 2 +- .../lldb/DataFormatters/TypeSynthetic.h | 8 +++---- .../lldb/DataFormatters/VectorIterator.h | 2 +- lldb/source/Core/ValueObject.cpp | 2 +- .../Core/ValueObjectSyntheticFilter.cpp | 14 +++++------ lldb/source/DataFormatters/TypeSynthetic.cpp | 2 +- lldb/source/DataFormatters/VectorType.cpp | 2 +- .../Language/CPlusPlus/BlockPointer.cpp | 2 +- .../Plugins/Language/CPlusPlus/Coroutines.cpp | 2 +- .../Plugins/Language/CPlusPlus/Coroutines.h | 2 +- .../Language/CPlusPlus/GenericBitset.cpp | 4 ++-- .../Language/CPlusPlus/GenericOptional.cpp | 4 ++-- .../Plugins/Language/CPlusPlus/LibCxx.cpp | 8 +++---- .../Plugins/Language/CPlusPlus/LibCxx.h | 8 +++---- .../Language/CPlusPlus/LibCxxAtomic.cpp | 4 ++-- .../CPlusPlus/LibCxxInitializerList.cpp | 4 ++-- .../Plugins/Language/CPlusPlus/LibCxxList.cpp | 8 +++---- .../Plugins/Language/CPlusPlus/LibCxxMap.cpp | 4 ++-- .../Language/CPlusPlus/LibCxxQueue.cpp | 2 +- .../CPlusPlus/LibCxxRangesRefView.cpp | 2 +- .../Plugins/Language/CPlusPlus/LibCxxSpan.cpp | 4 ++-- .../Language/CPlusPlus/LibCxxTuple.cpp | 4 ++-- .../Language/CPlusPlus/LibCxxUnorderedMap.cpp | 4 ++-- .../Language/CPlusPlus/LibCxxValarray.cpp | 4 ++-- .../Language/CPlusPlus/LibCxxVariant.cpp | 4 ++-- .../Language/CPlusPlus/LibCxxVector.cpp | 8 +++---- .../Plugins/Language/CPlusPlus/LibStdcpp.cpp | 10 ++++---- .../Language/CPlusPlus/LibStdcppTuple.cpp | 4 ++-- .../CPlusPlus/LibStdcppUniquePointer.cpp | 4 ++-- lldb/source/Plugins/Language/ObjC/Cocoa.cpp | 2 +- lldb/source/Plugins/Language/ObjC/NSArray.cpp | 16 ++++++------- .../Plugins/Language/ObjC/NSDictionary.cpp | 24 +++++++++---------- lldb/source/Plugins/Language/ObjC/NSError.cpp | 2 +- .../Plugins/Language/ObjC/NSException.cpp | 2 +- .../Plugins/Language/ObjC/NSIndexPath.cpp | 2 +- lldb/source/Plugins/Language/ObjC/NSSet.cpp | 15 ++++++------ 37 files changed, 100 insertions(+), 99 deletions(-) diff --git a/lldb/include/lldb/Core/ValueObject.h b/lldb/include/lldb/Core/ValueObject.h index 05dd64f5634f..757ae0601c24 100644 --- a/lldb/include/lldb/Core/ValueObject.h +++ b/lldb/include/lldb/Core/ValueObject.h @@ -465,7 +465,7 @@ public: /// Returns a unique id for this ValueObject. lldb::user_id_t GetID() const { return m_id.GetID(); } - virtual lldb::ValueObjectSP GetChildAtIndex(size_t idx, + virtual lldb::ValueObjectSP GetChildAtIndex(uint32_t idx, bool can_create = true); // The method always creates missing children in the path, if necessary. @@ -791,7 +791,7 @@ protected: return (m_children.find(idx) != m_children.end()); } - ValueObject *GetChildAtIndex(size_t idx) { + ValueObject *GetChildAtIndex(uint32_t idx) { std::lock_guard guard(m_mutex); const auto iter = m_children.find(idx); return ((iter == m_children.end()) ? nullptr : iter->second); diff --git a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h b/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h index 57794072ff92..1e54babc94f3 100644 --- a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h +++ b/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h @@ -51,7 +51,7 @@ public: lldb::ValueType GetValueType() const override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx, + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx, bool can_create = true) override; lldb::ValueObjectSP GetChildMemberWithName(llvm::StringRef name, diff --git a/lldb/include/lldb/DataFormatters/TypeSynthetic.h b/lldb/include/lldb/DataFormatters/TypeSynthetic.h index 7bb011c15794..38f3ce0fa5f0 100644 --- a/lldb/include/lldb/DataFormatters/TypeSynthetic.h +++ b/lldb/include/lldb/DataFormatters/TypeSynthetic.h @@ -45,7 +45,7 @@ public: return count <= max ? count : max; } - virtual lldb::ValueObjectSP GetChildAtIndex(size_t idx) = 0; + virtual lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) = 0; virtual size_t GetIndexOfChildWithName(ConstString name) = 0; @@ -111,7 +111,7 @@ public: uint32_t CalculateNumChildren() override { return 0; } - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override { return nullptr; } + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override { return nullptr; } size_t GetIndexOfChildWithName(ConstString name) override { return UINT32_MAX; @@ -324,7 +324,7 @@ public: uint32_t CalculateNumChildren() override { return filter->GetCount(); } - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override { + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override { if (idx >= filter->GetCount()) return lldb::ValueObjectSP(); return m_backend.GetSyntheticExpressionPathChild( @@ -430,7 +430,7 @@ public: uint32_t CalculateNumChildren(uint32_t max) override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; diff --git a/lldb/include/lldb/DataFormatters/VectorIterator.h b/lldb/include/lldb/DataFormatters/VectorIterator.h index 88500b0bfdd4..7711b9de95db 100644 --- a/lldb/include/lldb/DataFormatters/VectorIterator.h +++ b/lldb/include/lldb/DataFormatters/VectorIterator.h @@ -26,7 +26,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; diff --git a/lldb/source/Core/ValueObject.cpp b/lldb/source/Core/ValueObject.cpp index 840b100c70dd..dcf7ccadef88 100644 --- a/lldb/source/Core/ValueObject.cpp +++ b/lldb/source/Core/ValueObject.cpp @@ -372,7 +372,7 @@ bool ValueObject::IsLogicalTrue(Status &error) { return ret; } -ValueObjectSP ValueObject::GetChildAtIndex(size_t idx, bool can_create) { +ValueObjectSP ValueObject::GetChildAtIndex(uint32_t idx, bool can_create) { ValueObjectSP child_sp; // We may need to update our value if we are dynamic if (IsPossibleDynamicType()) diff --git a/lldb/source/Core/ValueObjectSyntheticFilter.cpp b/lldb/source/Core/ValueObjectSyntheticFilter.cpp index b03bd9a80e50..7f8a9a34cb35 100644 --- a/lldb/source/Core/ValueObjectSyntheticFilter.cpp +++ b/lldb/source/Core/ValueObjectSyntheticFilter.cpp @@ -33,7 +33,7 @@ public: uint32_t CalculateNumChildren() override { return m_backend.GetNumChildren(); } - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override { + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override { return m_backend.GetChildAtIndex(idx); } @@ -236,13 +236,13 @@ bool ValueObjectSynthetic::UpdateValue() { return true; } -lldb::ValueObjectSP ValueObjectSynthetic::GetChildAtIndex(size_t idx, +lldb::ValueObjectSP ValueObjectSynthetic::GetChildAtIndex(uint32_t idx, bool can_create) { Log *log = GetLog(LLDBLog::DataFormatters); LLDB_LOGF(log, "[ValueObjectSynthetic::GetChildAtIndex] name=%s, retrieving " - "child at index %zu", + "child at index %u", GetName().AsCString(), idx); UpdateValueIfNeeded(); @@ -261,7 +261,7 @@ lldb::ValueObjectSP ValueObjectSynthetic::GetChildAtIndex(size_t idx, if (can_create && m_synth_filter_up != nullptr) { LLDB_LOGF(log, "[ValueObjectSynthetic::GetChildAtIndex] name=%s, child at " - "index %zu not cached and will be created", + "index %u not cached and will be created", GetName().AsCString(), idx); lldb::ValueObjectSP synth_guy = m_synth_filter_up->GetChildAtIndex(idx); @@ -269,7 +269,7 @@ lldb::ValueObjectSP ValueObjectSynthetic::GetChildAtIndex(size_t idx, LLDB_LOGF( log, "[ValueObjectSynthetic::GetChildAtIndex] name=%s, child at index " - "%zu created as %p (is " + "%u created as %p (is " "synthetic: %s)", GetName().AsCString(), idx, static_cast(synth_guy.get()), synth_guy.get() @@ -291,7 +291,7 @@ lldb::ValueObjectSP ValueObjectSynthetic::GetChildAtIndex(size_t idx, } else { LLDB_LOGF(log, "[ValueObjectSynthetic::GetChildAtIndex] name=%s, child at " - "index %zu not cached and cannot " + "index %u not cached and cannot " "be created (can_create = %s, synth_filter = %p)", GetName().AsCString(), idx, can_create ? "yes" : "no", static_cast(m_synth_filter_up.get())); @@ -301,7 +301,7 @@ lldb::ValueObjectSP ValueObjectSynthetic::GetChildAtIndex(size_t idx, } else { LLDB_LOGF(log, "[ValueObjectSynthetic::GetChildAtIndex] name=%s, child at " - "index %zu cached as %p", + "index %u cached as %p", GetName().AsCString(), idx, static_cast(valobj)); return valobj->GetSP(); diff --git a/lldb/source/DataFormatters/TypeSynthetic.cpp b/lldb/source/DataFormatters/TypeSynthetic.cpp index a05fbe9a73c2..0ae38c4d31f2 100644 --- a/lldb/source/DataFormatters/TypeSynthetic.cpp +++ b/lldb/source/DataFormatters/TypeSynthetic.cpp @@ -167,7 +167,7 @@ ScriptedSyntheticChildren::FrontEnd::FrontEnd(std::string pclass, ScriptedSyntheticChildren::FrontEnd::~FrontEnd() = default; lldb::ValueObjectSP -ScriptedSyntheticChildren::FrontEnd::GetChildAtIndex(size_t idx) { +ScriptedSyntheticChildren::FrontEnd::GetChildAtIndex(uint32_t idx) { if (!m_wrapper_sp || !m_interpreter) return lldb::ValueObjectSP(); diff --git a/lldb/source/DataFormatters/VectorType.cpp b/lldb/source/DataFormatters/VectorType.cpp index 6f1f0e5b058f..a0626a8cba77 100644 --- a/lldb/source/DataFormatters/VectorType.cpp +++ b/lldb/source/DataFormatters/VectorType.cpp @@ -226,7 +226,7 @@ public: uint32_t CalculateNumChildren() override { return m_num_children; } - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override { + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override { if (idx >= CalculateNumChildren()) return {}; std::optional size = m_child_type.GetByteSize(nullptr); diff --git a/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp b/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp index f8e47efa7d91..ef0f67d1e9f9 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp @@ -79,7 +79,7 @@ public: return m_block_struct_type.GetNumChildren(omit_empty_base_classes, nullptr); } - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override { + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override { if (!m_block_struct_type.IsValid()) { return lldb::ValueObjectSP(); } diff --git a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp index ea1218350f6c..3827f9c21eff 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp @@ -113,7 +113,7 @@ uint32_t lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd:: } lldb::ValueObjectSP lldb_private::formatters:: - StdlibCoroutineHandleSyntheticFrontEnd::GetChildAtIndex(size_t idx) { + StdlibCoroutineHandleSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) { switch (idx) { case 0: return m_resume_ptr_sp; diff --git a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h index 8ec2c14d2e9e..5c6a80b57ff4 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h +++ b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h @@ -36,7 +36,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; diff --git a/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp b/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp index c8fc3af4a9d4..6a9da1d17c76 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp @@ -35,7 +35,7 @@ public: bool MightHaveChildren() override { return true; } lldb::ChildCacheState Update() override; uint32_t CalculateNumChildren() override { return m_elements.size(); } - ValueObjectSP GetChildAtIndex(size_t idx) override; + ValueObjectSP GetChildAtIndex(uint32_t idx) override; private: llvm::StringRef GetDataContainerMemberName(); @@ -97,7 +97,7 @@ lldb::ChildCacheState GenericBitsetFrontEnd::Update() { return lldb::ChildCacheState::eRefetch; } -ValueObjectSP GenericBitsetFrontEnd::GetChildAtIndex(size_t idx) { +ValueObjectSP GenericBitsetFrontEnd::GetChildAtIndex(uint32_t idx) { if (idx >= m_elements.size() || !m_first) return ValueObjectSP(); diff --git a/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp b/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp index 34ae1a248921..c06afb53eb8a 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp @@ -43,7 +43,7 @@ public: bool MightHaveChildren() override { return true; } uint32_t CalculateNumChildren() override { return m_has_value ? 1U : 0U; } - ValueObjectSP GetChildAtIndex(size_t idx) override; + ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; private: @@ -81,7 +81,7 @@ lldb::ChildCacheState GenericOptionalFrontend::Update() { return lldb::ChildCacheState::eRefetch; } -ValueObjectSP GenericOptionalFrontend::GetChildAtIndex(size_t _idx) { +ValueObjectSP GenericOptionalFrontend::GetChildAtIndex(uint32_t _idx) { if (!m_has_value) return ValueObjectSP(); diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp index 710b235ce957..bba887fec3ac 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp @@ -358,7 +358,7 @@ uint32_t lldb_private::formatters::LibCxxMapIteratorSyntheticFrontEnd:: lldb::ValueObjectSP lldb_private::formatters::LibCxxMapIteratorSyntheticFrontEnd::GetChildAtIndex( - size_t idx) { + uint32_t idx) { if (m_pair_ptr) return m_pair_ptr->GetChildAtIndex(idx); if (m_pair_sp) @@ -515,7 +515,7 @@ uint32_t lldb_private::formatters::LibCxxUnorderedMapIteratorSyntheticFrontEnd:: } lldb::ValueObjectSP lldb_private::formatters:: - LibCxxUnorderedMapIteratorSyntheticFrontEnd::GetChildAtIndex(size_t idx) { + LibCxxUnorderedMapIteratorSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) { if (m_pair_sp) return m_pair_sp->GetChildAtIndex(idx); return lldb::ValueObjectSP(); @@ -573,7 +573,7 @@ uint32_t lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd:: lldb::ValueObjectSP lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::GetChildAtIndex( - size_t idx) { + uint32_t idx) { if (!m_cntrl) return lldb::ValueObjectSP(); @@ -670,7 +670,7 @@ uint32_t lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEnd:: lldb::ValueObjectSP lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEnd::GetChildAtIndex( - size_t idx) { + uint32_t idx) { if (!m_value_ptr_sp) return lldb::ValueObjectSP(); diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h index af4f313a9050..ad2f58508ab7 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h @@ -89,7 +89,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -137,7 +137,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -168,7 +168,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -188,7 +188,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp index 484d756a300e..8e4c36103a74 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp @@ -92,7 +92,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -131,7 +131,7 @@ uint32_t lldb_private::formatters::LibcxxStdAtomicSyntheticFrontEnd:: lldb::ValueObjectSP lldb_private::formatters::LibcxxStdAtomicSyntheticFrontEnd::GetChildAtIndex( - size_t idx) { + uint32_t idx) { if (idx == 0) return m_real_child->GetSP()->Clone(ConstString("Value")); return nullptr; diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp index ef94815b3afc..00012dfc056e 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp @@ -28,7 +28,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -69,7 +69,7 @@ uint32_t lldb_private::formatters::LibcxxInitializerListSyntheticFrontEnd:: } lldb::ValueObjectSP lldb_private::formatters:: - LibcxxInitializerListSyntheticFrontEnd::GetChildAtIndex(size_t idx) { + LibcxxInitializerListSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) { if (!m_start) return lldb::ValueObjectSP(); diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp index d05a64b1eba5..17f6b737d9f6 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp @@ -137,7 +137,7 @@ public: ForwardListFrontEnd(ValueObject &valobj); uint32_t CalculateNumChildren() override; - ValueObjectSP GetChildAtIndex(size_t idx) override; + ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; }; @@ -149,7 +149,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -253,7 +253,7 @@ uint32_t ForwardListFrontEnd::CalculateNumChildren() { return m_count; } -ValueObjectSP ForwardListFrontEnd::GetChildAtIndex(size_t idx) { +ValueObjectSP ForwardListFrontEnd::GetChildAtIndex(uint32_t idx) { if (idx >= CalculateNumChildren()) return nullptr; @@ -343,7 +343,7 @@ uint32_t ListFrontEnd::CalculateNumChildren() { } } -lldb::ValueObjectSP ListFrontEnd::GetChildAtIndex(size_t idx) { +lldb::ValueObjectSP ListFrontEnd::GetChildAtIndex(uint32_t idx) { static ConstString g_value("__value_"); static ConstString g_next("__next_"); diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp index 28a9a26c9d2d..6d24eb03779c 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp @@ -179,7 +179,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -308,7 +308,7 @@ void lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::GetValueOffset( lldb::ValueObjectSP lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::GetChildAtIndex( - size_t idx) { + uint32_t idx) { static ConstString g_cc_("__cc_"), g_cc("__cc"); static ConstString g_nc("__nc"); diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp index beab453b788a..fbadee89b7b7 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp @@ -32,7 +32,7 @@ public: return m_container_sp ? m_container_sp->GetNumChildren() : 0; } - ValueObjectSP GetChildAtIndex(size_t idx) override { + ValueObjectSP GetChildAtIndex(uint32_t idx) override { return m_container_sp ? m_container_sp->GetChildAtIndex(idx) : nullptr; } diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp index cda6c5d79c31..74f54f767356 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp @@ -32,7 +32,7 @@ public: return 1; } - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override { + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override { // Since we only have a single child, return it assert(idx == 0); return m_range_sp; diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp index 5aaf1ec2d56e..af2b51d2b540 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp @@ -29,7 +29,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; /// Determines properties of the std::span<> associated with this object // @@ -80,7 +80,7 @@ uint32_t lldb_private::formatters::LibcxxStdSpanSyntheticFrontEnd:: lldb::ValueObjectSP lldb_private::formatters::LibcxxStdSpanSyntheticFrontEnd::GetChildAtIndex( - size_t idx) { + uint32_t idx) { if (!m_start) return {}; diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp index 5c42d8551d4f..62bb7d619267 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp @@ -27,7 +27,7 @@ public: bool MightHaveChildren() override { return true; } lldb::ChildCacheState Update() override; uint32_t CalculateNumChildren() override { return m_elements.size(); } - ValueObjectSP GetChildAtIndex(size_t idx) override; + ValueObjectSP GetChildAtIndex(uint32_t idx) override; private: // The lifetime of a ValueObject and all its derivative ValueObjects @@ -58,7 +58,7 @@ lldb::ChildCacheState TupleFrontEnd::Update() { return lldb::ChildCacheState::eRefetch; } -ValueObjectSP TupleFrontEnd::GetChildAtIndex(size_t idx) { +ValueObjectSP TupleFrontEnd::GetChildAtIndex(uint32_t idx) { if (idx >= m_elements.size()) return ValueObjectSP(); if (!m_base) diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp index 0be73a9e633d..b3c364294335 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp @@ -35,7 +35,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -93,7 +93,7 @@ static bool isUnorderedMap(ConstString type_name) { } lldb::ValueObjectSP lldb_private::formatters:: - LibcxxStdUnorderedMapSyntheticFrontEnd::GetChildAtIndex(size_t idx) { + LibcxxStdUnorderedMapSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) { if (idx >= CalculateNumChildren()) return lldb::ValueObjectSP(); if (m_tree == nullptr) diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp index eae7711c8cd9..463c7b8d7ce3 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp @@ -26,7 +26,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -84,7 +84,7 @@ uint32_t lldb_private::formatters::LibcxxStdValarraySyntheticFrontEnd:: lldb::ValueObjectSP lldb_private::formatters::LibcxxStdValarraySyntheticFrontEnd::GetChildAtIndex( - size_t idx) { + uint32_t idx) { if (!m_start || !m_finish) return lldb::ValueObjectSP(); diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp index c704e1b8335e..1f62062f09be 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp @@ -206,7 +206,7 @@ public: bool MightHaveChildren() override { return true; } lldb::ChildCacheState Update() override; uint32_t CalculateNumChildren() override { return m_size; } - ValueObjectSP GetChildAtIndex(size_t idx) override; + ValueObjectSP GetChildAtIndex(uint32_t idx) override; private: size_t m_size = 0; @@ -233,7 +233,7 @@ lldb::ChildCacheState VariantFrontEnd::Update() { return lldb::ChildCacheState::eRefetch; } -ValueObjectSP VariantFrontEnd::GetChildAtIndex(size_t idx) { +ValueObjectSP VariantFrontEnd::GetChildAtIndex(uint32_t idx) { if (idx >= m_size) return {}; diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp index 62ff76b82a2a..fcf727ad2ea0 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp @@ -27,7 +27,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -48,7 +48,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -103,7 +103,7 @@ uint32_t lldb_private::formatters::LibcxxStdVectorSyntheticFrontEnd:: lldb::ValueObjectSP lldb_private::formatters::LibcxxStdVectorSyntheticFrontEnd::GetChildAtIndex( - size_t idx) { + uint32_t idx) { if (!m_start || !m_finish) return lldb::ValueObjectSP(); @@ -172,7 +172,7 @@ uint32_t lldb_private::formatters::LibcxxVectorBoolSyntheticFrontEnd:: lldb::ValueObjectSP lldb_private::formatters::LibcxxVectorBoolSyntheticFrontEnd::GetChildAtIndex( - size_t idx) { + uint32_t idx) { auto iter = m_children.find(idx), end = m_children.end(); if (iter != end) return iter->second; diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp index b2249d23961f..5abb3d50674b 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp @@ -45,7 +45,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -66,7 +66,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -137,7 +137,7 @@ uint32_t LibstdcppMapIteratorSyntheticFrontEnd::CalculateNumChildren() { } lldb::ValueObjectSP -LibstdcppMapIteratorSyntheticFrontEnd::GetChildAtIndex(size_t idx) { +LibstdcppMapIteratorSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) { if (m_pair_address != 0 && m_pair_type) { if (!m_pair_sp) m_pair_sp = CreateValueObjectFromAddress("pair", m_pair_address, @@ -222,7 +222,7 @@ lldb::ChildCacheState VectorIteratorSyntheticFrontEnd::Update() { uint32_t VectorIteratorSyntheticFrontEnd::CalculateNumChildren() { return 1; } lldb::ValueObjectSP -VectorIteratorSyntheticFrontEnd::GetChildAtIndex(size_t idx) { +VectorIteratorSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) { if (idx == 0) return m_item_sp; return lldb::ValueObjectSP(); @@ -374,7 +374,7 @@ LibStdcppSharedPtrSyntheticFrontEnd::LibStdcppSharedPtrSyntheticFrontEnd( uint32_t LibStdcppSharedPtrSyntheticFrontEnd::CalculateNumChildren() { return 1; } lldb::ValueObjectSP -LibStdcppSharedPtrSyntheticFrontEnd::GetChildAtIndex(size_t idx) { +LibStdcppSharedPtrSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) { if (idx == 0) return m_ptr_obj->GetSP(); if (idx == 1) { diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp index ba3116d01a72..64d2ec9d943a 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp @@ -28,7 +28,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -89,7 +89,7 @@ lldb::ChildCacheState LibStdcppTupleSyntheticFrontEnd::Update() { bool LibStdcppTupleSyntheticFrontEnd::MightHaveChildren() { return true; } lldb::ValueObjectSP -LibStdcppTupleSyntheticFrontEnd::GetChildAtIndex(size_t idx) { +LibStdcppTupleSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) { if (idx < m_members.size() && m_members[idx]) return m_members[idx]->GetSP(); return lldb::ValueObjectSP(); diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp index aca18ee694c0..3a48fe412e07 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp @@ -28,7 +28,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -116,7 +116,7 @@ lldb::ChildCacheState LibStdcppUniquePtrSyntheticFrontEnd::Update() { bool LibStdcppUniquePtrSyntheticFrontEnd::MightHaveChildren() { return true; } lldb::ValueObjectSP -LibStdcppUniquePtrSyntheticFrontEnd::GetChildAtIndex(size_t idx) { +LibStdcppUniquePtrSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) { if (idx == 0 && m_ptr_obj) return m_ptr_obj->GetSP(); if (idx == 1 && m_del_obj) diff --git a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp index 05b324a30f2d..cb740f8e71e1 100644 --- a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp +++ b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp @@ -1040,7 +1040,7 @@ public: uint32_t CalculateNumChildren() override { return 0; } - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override { + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override { return lldb::ValueObjectSP(); } diff --git a/lldb/source/Plugins/Language/ObjC/NSArray.cpp b/lldb/source/Plugins/Language/ObjC/NSArray.cpp index 9ee12e50b8ac..7f060b2613d6 100644 --- a/lldb/source/Plugins/Language/ObjC/NSArray.cpp +++ b/lldb/source/Plugins/Language/ObjC/NSArray.cpp @@ -52,7 +52,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override = 0; @@ -216,7 +216,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -304,7 +304,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -321,7 +321,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -484,7 +484,7 @@ uint32_t lldb_private::formatters::NSArrayMSyntheticFrontEndBase:: lldb::ValueObjectSP lldb_private::formatters::NSArrayMSyntheticFrontEndBase::GetChildAtIndex( - size_t idx) { + uint32_t idx) { if (idx >= CalculateNumChildren()) return lldb::ValueObjectSP(); lldb::addr_t object_at_idx = GetDataAddress(); @@ -684,7 +684,7 @@ lldb_private::formatters::GenericNSArrayISyntheticFrontEnd:: template lldb::ValueObjectSP lldb_private::formatters::GenericNSArrayISyntheticFrontEnd:: - GetChildAtIndex(size_t idx) { + GetChildAtIndex(uint32_t idx) { if (idx >= CalculateNumChildren()) return lldb::ValueObjectSP(); lldb::addr_t object_at_idx; @@ -735,7 +735,7 @@ bool lldb_private::formatters::NSArray0SyntheticFrontEnd::MightHaveChildren() { lldb::ValueObjectSP lldb_private::formatters::NSArray0SyntheticFrontEnd::GetChildAtIndex( - size_t idx) { + uint32_t idx) { return lldb::ValueObjectSP(); } @@ -770,7 +770,7 @@ bool lldb_private::formatters::NSArray1SyntheticFrontEnd::MightHaveChildren() { lldb::ValueObjectSP lldb_private::formatters::NSArray1SyntheticFrontEnd::GetChildAtIndex( - size_t idx) { + uint32_t idx) { static const ConstString g_zero("[0]"); if (idx == 0) { diff --git a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp index 3bb2e4fac02e..da94eda1529c 100644 --- a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp +++ b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp @@ -105,7 +105,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -146,7 +146,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -178,7 +178,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -211,7 +211,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -232,7 +232,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -265,7 +265,7 @@ namespace Foundation1100 { uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -655,7 +655,7 @@ bool lldb_private::formatters::NSDictionaryISyntheticFrontEnd:: lldb::ValueObjectSP lldb_private::formatters::NSDictionaryISyntheticFrontEnd::GetChildAtIndex( - size_t idx) { + uint32_t idx) { uint32_t num_children = CalculateNumChildren(); if (idx >= num_children) @@ -777,7 +777,7 @@ bool lldb_private::formatters::NSCFDictionarySyntheticFrontEnd:: lldb::ValueObjectSP lldb_private::formatters::NSCFDictionarySyntheticFrontEnd::GetChildAtIndex( - size_t idx) { + uint32_t idx) { lldb::addr_t m_keys_ptr = m_hashtable.GetKeyPointer(); lldb::addr_t m_values_ptr = m_hashtable.GetValuePointer(); @@ -920,7 +920,7 @@ bool lldb_private::formatters::NSConstantDictionarySyntheticFrontEnd:: } lldb::ValueObjectSP lldb_private::formatters:: - NSConstantDictionarySyntheticFrontEnd::GetChildAtIndex(size_t idx) { + NSConstantDictionarySyntheticFrontEnd::GetChildAtIndex(uint32_t idx) { uint32_t num_children = CalculateNumChildren(); if (idx >= num_children) @@ -1012,7 +1012,7 @@ bool lldb_private::formatters::NSDictionary1SyntheticFrontEnd:: lldb::ValueObjectSP lldb_private::formatters::NSDictionary1SyntheticFrontEnd::GetChildAtIndex( - size_t idx) { + uint32_t idx) { if (idx != 0) return lldb::ValueObjectSP(); @@ -1140,7 +1140,7 @@ lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd:: template lldb::ValueObjectSP lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd< - D32, D64>::GetChildAtIndex(size_t idx) { + D32, D64>::GetChildAtIndex(uint32_t idx) { lldb::addr_t m_keys_ptr; lldb::addr_t m_values_ptr; if (m_data_32) { @@ -1300,7 +1300,7 @@ lldb_private::formatters::Foundation1100:: lldb::ValueObjectSP lldb_private::formatters::Foundation1100:: - NSDictionaryMSyntheticFrontEnd::GetChildAtIndex(size_t idx) { + NSDictionaryMSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) { lldb::addr_t m_keys_ptr = (m_data_32 ? m_data_32->_keys_addr : m_data_64->_keys_addr); lldb::addr_t m_values_ptr = diff --git a/lldb/source/Plugins/Language/ObjC/NSError.cpp b/lldb/source/Plugins/Language/ObjC/NSError.cpp index 786d2bc51871..b034e799b716 100644 --- a/lldb/source/Plugins/Language/ObjC/NSError.cpp +++ b/lldb/source/Plugins/Language/ObjC/NSError.cpp @@ -124,7 +124,7 @@ public: return 0; } - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override { + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override { if (idx != 0) return lldb::ValueObjectSP(); diff --git a/lldb/source/Plugins/Language/ObjC/NSException.cpp b/lldb/source/Plugins/Language/ObjC/NSException.cpp index e47939c71801..09d3a1b42b74 100644 --- a/lldb/source/Plugins/Language/ObjC/NSException.cpp +++ b/lldb/source/Plugins/Language/ObjC/NSException.cpp @@ -127,7 +127,7 @@ public: return 4; } - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override { + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override { switch (idx) { case 0: return m_name_sp; case 1: return m_reason_sp; diff --git a/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp b/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp index 45c020f8d1b3..10bb907c58ed 100644 --- a/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp +++ b/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp @@ -42,7 +42,7 @@ public: uint32_t CalculateNumChildren() override { return m_impl.GetNumIndexes(); } - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override { + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override { return m_impl.GetIndexAtIndex(idx, m_uint_star_type); } diff --git a/lldb/source/Plugins/Language/ObjC/NSSet.cpp b/lldb/source/Plugins/Language/ObjC/NSSet.cpp index 97f19c1ac937..c965a2a13400 100644 --- a/lldb/source/Plugins/Language/ObjC/NSSet.cpp +++ b/lldb/source/Plugins/Language/ObjC/NSSet.cpp @@ -48,7 +48,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -86,7 +86,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -119,7 +119,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -235,7 +235,7 @@ public: uint32_t CalculateNumChildren() override; - lldb::ValueObjectSP GetChildAtIndex(size_t idx) override; + lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override; lldb::ChildCacheState Update() override; @@ -466,7 +466,8 @@ bool lldb_private::formatters::NSSetISyntheticFrontEnd::MightHaveChildren() { } lldb::ValueObjectSP -lldb_private::formatters::NSSetISyntheticFrontEnd::GetChildAtIndex(size_t idx) { +lldb_private::formatters::NSSetISyntheticFrontEnd::GetChildAtIndex( + uint32_t idx) { uint32_t num_children = CalculateNumChildren(); if (idx >= num_children) @@ -587,7 +588,7 @@ bool lldb_private::formatters::NSCFSetSyntheticFrontEnd::MightHaveChildren() { lldb::ValueObjectSP lldb_private::formatters::NSCFSetSyntheticFrontEnd::GetChildAtIndex( - size_t idx) { + uint32_t idx) { lldb::addr_t m_values_ptr = m_hashtable.GetValuePointer(); const uint32_t num_children = CalculateNumChildren(); @@ -748,7 +749,7 @@ lldb_private::formatters:: template lldb::ValueObjectSP lldb_private::formatters:: - GenericNSSetMSyntheticFrontEnd::GetChildAtIndex(size_t idx) { + GenericNSSetMSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) { lldb::addr_t m_objs_addr = (m_data_32 ? m_data_32->_objs_addr : m_data_64->_objs_addr); -- GitLab From 6f299417769ade1635c91f974a8745e237cc9adf Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 7 Mar 2024 18:57:09 +0000 Subject: [PATCH 480/929] [TBAA] Add extra tests to copy structs with union members. Adds extra test coverage for TBAA generation for copies of structs with union members. --- clang/test/CodeGen/tbaa-struct.cpp | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/clang/test/CodeGen/tbaa-struct.cpp b/clang/test/CodeGen/tbaa-struct.cpp index 883c982be26c..63e409794644 100644 --- a/clang/test/CodeGen/tbaa-struct.cpp +++ b/clang/test/CodeGen/tbaa-struct.cpp @@ -151,6 +151,38 @@ void copy10(NamedBitfields3 *a1, NamedBitfields3 *a2) { *a1 = *a2; } +union U2 { + double d; + float f; +}; + +struct UnionMember1 { + U2 u; + int p; +}; + +void copy11(UnionMember1 *a1, UnionMember1 *a2) { +// CHECK-LABEL: _Z6copy11P12UnionMember1S0_ +// CHECK: tail call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(16) %a1, ptr noundef nonnull align 8 dereferenceable(16) %a2, i64 16, i1 false), +// CHECK-OLD-SAME: !tbaa.struct [[TS9:!.*]] +// CHECK-NEW-SAME: !tbaa [[TAG_UnionMember1:!.+]], !tbaa.struct + *a1 = *a2; +} + +struct UnionMember2 { + int p; + U2 u; +}; + +void copy12(UnionMember2 *a1, UnionMember2 *a2) { +// CHECK-LABEL: _Z6copy12P12UnionMember2S0_ +// CHECK: tail call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(16) %a1, ptr noundef nonnull align 8 dereferenceable(16) %a2, i64 16, i1 false), +// CHECK-OLD-SAME: !tbaa.struct [[TS10:!.*]] +// CHECK-NEW-SAME: !tbaa [[TAG_UnionMember2:!.+]], !tbaa.struct + + *a1 = *a2; +} + // CHECK-OLD: [[TS]] = !{i64 0, i64 2, !{{.*}}, i64 4, i64 4, !{{.*}}, i64 8, i64 1, !{{.*}}, i64 12, i64 4, !{{.*}}} // CHECK-OLD: [[CHAR:!.*]] = !{!"omnipotent char", !{{.*}}} // CHECK-OLD: [[TAG_INT:!.*]] = !{[[INT:!.*]], [[INT]], i64 0} @@ -167,6 +199,10 @@ void copy10(NamedBitfields3 *a1, NamedBitfields3 *a2) { // CHECK-OLD [[DOUBLE]] = !{!"double", [[CHAR]], i64 0} // CHECK-OLD: [[TS7]] = !{i64 0, i64 1, [[TAG_CHAR]], i64 1, i64 1, [[TAG_CHAR]], i64 2, i64 1, [[TAG_CHAR]], i64 3, i64 1, [[TAG_CHAR]], i64 4, i64 1, [[TAG_CHAR]], i64 8, i64 8, [[TAG_DOUBLE]], i64 16, i64 1, [[TAG_CHAR]]} // CHECK-OLD: [[TS8]] = !{i64 0, i64 4, [[TAG_CHAR]], i64 8, i64 8, [[TAG_DOUBLE]]} +// CHECK-OLD: [[TS9]] = !{i64 0, i64 8, [[TAG_DOUBLE]], i64 0, i64 4, [[TAG_FLOAT:!.+]], i64 8, i64 4, [[TAG_INT]]} +// CHECK-OLD: [[TAG_FLOAT]] = !{[[FLOAT:!.+]], [[FLOAT]], i64 0} +// CHECK-OLD: [[FLOAT]] = !{!"float", [[CHAR]], i64 0} +// CHECK-OLD: [[TS10]] = !{i64 0, i64 4, [[TAG_INT]], i64 8, i64 8, [[TAG_DOUBLE]], i64 8, i64 4, [[TAG_FLOAT:!.+]]} // CHECK-NEW-DAG: [[TYPE_char:!.*]] = !{{{.*}}, i64 1, !"omnipotent char"} // CHECK-NEW-DAG: [[TAG_char]] = !{[[TYPE_char]], [[TYPE_char]], i64 0, i64 0} @@ -188,3 +224,7 @@ void copy10(NamedBitfields3 *a1, NamedBitfields3 *a2) { // CHECK-NEW-DAG: [[TYPE_NamedBitfields2]] = !{[[TYPE_char]], i64 24, !"_ZTS15NamedBitfields2", [[TYPE_char]], i64 0, i64 1, [[TYPE_char]], i64 1, i64 1, [[TYPE_char]], i64 2, i64 1, [[TYPE_int]], i64 3, i64 4, [[TYPE_int]], i64 3, i64 4, [[TYPE_char]], i64 4, i64 1, [[TYPE_double]], i64 8, i64 8, [[TYPE_int]], i64 16, i64 4} // CHECK-NEW-DAG: [[TAG_NamedBitfields3]] = !{[[TYPE_NamedBitfields3:!.+]], [[TYPE_NamedBitfields3]], i64 0, i64 16} // CHECK-NEW-DAG: [[TYPE_NamedBitfields3]] = !{[[TYPE_char]], i64 16, !"_ZTS15NamedBitfields3", [[TYPE_int]], i64 1, i64 4, [[TYPE_int]], i64 2, i64 4, [[TYPE_double]], i64 8, i64 8} +// CHECK-NEW-DAG: [[TAG_UnionMember1]] = !{[[TYPE_UnionMember1:!.+]], [[TYPE_UnionMember1]], i64 0, i64 16} +// CHECK-NEW-DAG: [[TYPE_UnionMember1]] = !{[[TYPE_char]], i64 16, !"_ZTS12UnionMember1", [[TYPE_char]], i64 0, i64 8, [[TYPE_int]], i64 8, i64 4} +// CHECK-NEW-DAG: [[TAG_UnionMember2]] = !{[[TYPE_UnionMember2:!.+]], [[TYPE_UnionMember2]], i64 0, i64 16} +// CHECK-NEW-DAG: [[TYPE_UnionMember2]] = !{[[TYPE_char]], i64 16, !"_ZTS12UnionMember2", [[TYPE_int]], i64 0, i64 4, [[TYPE_char]], i64 8, i64 8} -- GitLab From 7fc583c9a5ddf447b2b53007778cb034a186d4b5 Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Tue, 5 Mar 2024 16:29:00 -0800 Subject: [PATCH 481/929] Change Get|SetNumChildren to use unint32_t --- lldb/include/lldb/Core/ValueObject.h | 4 ++-- lldb/source/Core/ValueObject.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lldb/include/lldb/Core/ValueObject.h b/lldb/include/lldb/Core/ValueObject.h index 757ae0601c24..b4d2c8098edc 100644 --- a/lldb/include/lldb/Core/ValueObject.h +++ b/lldb/include/lldb/Core/ValueObject.h @@ -476,7 +476,7 @@ public: virtual size_t GetIndexOfChildWithName(llvm::StringRef name); - size_t GetNumChildren(uint32_t max = UINT32_MAX); + uint32_t GetNumChildren(uint32_t max = UINT32_MAX); const Value &GetValue() const { return m_value; } @@ -960,7 +960,7 @@ protected: /// Should only be called by ValueObject::GetNumChildren(). virtual uint32_t CalculateNumChildren(uint32_t max = UINT32_MAX) = 0; - void SetNumChildren(size_t num_children); + void SetNumChildren(uint32_t num_children); void SetValueDidChange(bool value_changed) { m_flags.m_value_did_change = value_changed; diff --git a/lldb/source/Core/ValueObject.cpp b/lldb/source/Core/ValueObject.cpp index dcf7ccadef88..0ed7f03be25c 100644 --- a/lldb/source/Core/ValueObject.cpp +++ b/lldb/source/Core/ValueObject.cpp @@ -440,7 +440,7 @@ ValueObjectSP ValueObject::GetChildMemberWithName(llvm::StringRef name, return child_sp; } -size_t ValueObject::GetNumChildren(uint32_t max) { +uint32_t ValueObject::GetNumChildren(uint32_t max) { UpdateValueIfNeeded(); if (max < UINT32_MAX) { @@ -470,7 +470,7 @@ bool ValueObject::MightHaveChildren() { } // Should only be called by ValueObject::GetNumChildren() -void ValueObject::SetNumChildren(size_t num_children) { +void ValueObject::SetNumChildren(uint32_t num_children) { m_flags.m_children_count_valid = true; m_children.SetChildrenCount(num_children); } -- GitLab From c103d573e7fc236c0c9e2fde41a843ea62d960f4 Mon Sep 17 00:00:00 2001 From: Guillaume Chatelet Date: Thu, 7 Mar 2024 20:00:05 +0100 Subject: [PATCH 482/929] [libc] Fix forward missing `BigInt` specialization of `mask_leading_ones` / `mask_trailing_ones` (#84325) #84299 broke the arm32 build, this patch fixes it forward. --- libc/src/__support/UInt.h | 56 +++++++++++++++ libc/src/__support/math_extras.h | 13 ++-- .../stdio/printf_core/float_dec_converter.h | 7 +- libc/test/UnitTest/CMakeLists.txt | 1 + libc/test/UnitTest/StringUtils.h | 3 +- libc/test/src/__support/CMakeLists.txt | 2 + libc/test/src/__support/math_extras_test.cpp | 69 +++++++++++++------ .../libc/test/src/__support/BUILD.bazel | 6 +- 8 files changed, 123 insertions(+), 34 deletions(-) diff --git a/libc/src/__support/UInt.h b/libc/src/__support/UInt.h index b3d8f00b9a01..925de8764715 100644 --- a/libc/src/__support/UInt.h +++ b/libc/src/__support/UInt.h @@ -1056,4 +1056,60 @@ rotr(T value, int rotate) { } // namespace LIBC_NAMESPACE::cpp +namespace LIBC_NAMESPACE { + +// Specialization of mask_trailing_ones ('math_extras.h') for BigInt. +template +LIBC_INLINE constexpr cpp::enable_if_t, T> +mask_trailing_ones() { + static_assert(!T::SIGNED); + if (count == 0) + return T(); + constexpr unsigned T_BITS = CHAR_BIT * sizeof(T); + static_assert(count <= T_BITS && "Invalid bit index"); + using word_type = typename T::word_type; + T out; + constexpr int CHUNK_INDEX_CONTAINING_BIT = + static_cast(count / T::WORD_SIZE); + int index = 0; + for (auto &word : out.val) { + if (index < CHUNK_INDEX_CONTAINING_BIT) + word = -1; + else if (index > CHUNK_INDEX_CONTAINING_BIT) + word = 0; + else + word = mask_trailing_ones(); + ++index; + } + return out; +} + +// Specialization of mask_leading_ones ('math_extras.h') for BigInt. +template +LIBC_INLINE constexpr cpp::enable_if_t, T> +mask_leading_ones() { + static_assert(!T::SIGNED); + if (count == 0) + return T(); + constexpr unsigned T_BITS = CHAR_BIT * sizeof(T); + static_assert(count <= T_BITS && "Invalid bit index"); + using word_type = typename T::word_type; + T out; + constexpr int CHUNK_INDEX_CONTAINING_BIT = + static_cast((T::BITS - count - 1ULL) / T::WORD_SIZE); + int index = 0; + for (auto &word : out.val) { + if (index < CHUNK_INDEX_CONTAINING_BIT) + word = 0; + else if (index > CHUNK_INDEX_CONTAINING_BIT) + word = -1; + else + word = mask_leading_ones(); + ++index; + } + return out; +} + +} // namespace LIBC_NAMESPACE + #endif // LLVM_LIBC_SRC___SUPPORT_UINT_H diff --git a/libc/src/__support/math_extras.h b/libc/src/__support/math_extras.h index 7a89fbb11b2a..c6b458ddecda 100644 --- a/libc/src/__support/math_extras.h +++ b/libc/src/__support/math_extras.h @@ -20,21 +20,18 @@ namespace LIBC_NAMESPACE { // Create a bitmask with the count right-most bits set to 1, and all other bits // set to 0. Only unsigned types are allowed. template -LIBC_INLINE constexpr T mask_trailing_ones() { - static_assert(cpp::is_unsigned_v); +LIBC_INLINE constexpr cpp::enable_if_t, T> +mask_trailing_ones() { constexpr unsigned T_BITS = CHAR_BIT * sizeof(T); static_assert(count <= T_BITS && "Invalid bit index"); - // It's important not to initialize T with -1, since T may be BigInt which - // will take -1 as a uint64_t and only initialize the low 64 bits. - constexpr T ALL_ZEROES(0); - constexpr T ALL_ONES(~ALL_ZEROES); // bitwise NOT performs integer promotion. - return count == 0 ? 0 : (ALL_ONES >> (T_BITS - count)); + return count == 0 ? 0 : (T(-1) >> (T_BITS - count)); } // Create a bitmask with the count left-most bits set to 1, and all other bits // set to 0. Only unsigned types are allowed. template -LIBC_INLINE constexpr T mask_leading_ones() { +LIBC_INLINE constexpr cpp::enable_if_t, T> +mask_leading_ones() { constexpr T MASK(mask_trailing_ones()); return T(~MASK); // bitwise NOT performs integer promotion. } diff --git a/libc/src/stdio/printf_core/float_dec_converter.h b/libc/src/stdio/printf_core/float_dec_converter.h index a6c68329e660..27d229a3e42c 100644 --- a/libc/src/stdio/printf_core/float_dec_converter.h +++ b/libc/src/stdio/printf_core/float_dec_converter.h @@ -12,6 +12,7 @@ #include "src/__support/CPP/string_view.h" #include "src/__support/FPUtil/FPBits.h" #include "src/__support/FPUtil/rounding_mode.h" +#include "src/__support/UInt.h" // cpp::is_big_int_v #include "src/__support/float_to_string.h" #include "src/__support/integer_to_string.h" #include "src/__support/libc_assert.h" @@ -33,7 +34,8 @@ using ExponentString = // Returns true if value is divisible by 2^p. template -LIBC_INLINE constexpr cpp::enable_if_t, bool> +LIBC_INLINE constexpr cpp::enable_if_t< + cpp::is_integral_v || cpp::is_big_int_v, bool> multiple_of_power_of_2(T value, uint32_t p) { return (value & ((T(1) << p) - 1)) == 0; } @@ -76,7 +78,8 @@ LIBC_INLINE RoundDirection get_round_direction(int last_digit, bool truncated, } template -LIBC_INLINE constexpr cpp::enable_if_t, bool> +LIBC_INLINE constexpr cpp::enable_if_t< + cpp::is_integral_v || cpp::is_big_int_v, bool> zero_after_digits(int32_t base_2_exp, int32_t digits_after_point, T mantissa, const int32_t mant_width) { const int32_t required_twos = -base_2_exp - digits_after_point - 1; diff --git a/libc/test/UnitTest/CMakeLists.txt b/libc/test/UnitTest/CMakeLists.txt index 36837c553efc..8a35f1204eb5 100644 --- a/libc/test/UnitTest/CMakeLists.txt +++ b/libc/test/UnitTest/CMakeLists.txt @@ -104,6 +104,7 @@ add_header_library( DEPENDS libc.src.__support.CPP.string libc.src.__support.CPP.type_traits + libc.src.__support.uint ) add_unittest_framework_library( diff --git a/libc/test/UnitTest/StringUtils.h b/libc/test/UnitTest/StringUtils.h index 54cff97ceafb..1e3ba5715d23 100644 --- a/libc/test/UnitTest/StringUtils.h +++ b/libc/test/UnitTest/StringUtils.h @@ -11,12 +11,13 @@ #include "src/__support/CPP/string.h" #include "src/__support/CPP/type_traits.h" +#include "src/__support/UInt.h" namespace LIBC_NAMESPACE { // Return the first N hex digits of an integer as a string in upper case. template -cpp::enable_if_t, cpp::string> +cpp::enable_if_t || cpp::is_big_int_v, cpp::string> int_to_hex(T value, size_t length = sizeof(T) * 2) { cpp::string s(length, '0'); diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt index 8c861b576f9b..adbacb9728cc 100644 --- a/libc/test/src/__support/CMakeLists.txt +++ b/libc/test/src/__support/CMakeLists.txt @@ -27,7 +27,9 @@ add_libc_test( SRCS math_extras_test.cpp DEPENDS + libc.src.__support.integer_literals libc.src.__support.math_extras + libc.src.__support.uint128 ) add_libc_test( diff --git a/libc/test/src/__support/math_extras_test.cpp b/libc/test/src/__support/math_extras_test.cpp index e55d995592cc..ed064363d446 100644 --- a/libc/test/src/__support/math_extras_test.cpp +++ b/libc/test/src/__support/math_extras_test.cpp @@ -6,34 +6,59 @@ // //===----------------------------------------------------------------------===// +#include "src/__support/UInt128.h" // UInt128 +#include "src/__support/integer_literals.h" #include "src/__support/math_extras.h" #include "test/UnitTest/Test.h" namespace LIBC_NAMESPACE { TEST(LlvmLibcBlockMathExtrasTest, mask_trailing_ones) { - EXPECT_EQ(uint8_t(0), (mask_leading_ones())); - EXPECT_EQ(uint8_t(0), (mask_trailing_ones())); - EXPECT_EQ(uint16_t(0), (mask_leading_ones())); - EXPECT_EQ(uint16_t(0), (mask_trailing_ones())); - EXPECT_EQ(uint32_t(0), (mask_leading_ones())); - EXPECT_EQ(uint32_t(0), (mask_trailing_ones())); - EXPECT_EQ(uint64_t(0), (mask_leading_ones())); - EXPECT_EQ(uint64_t(0), (mask_trailing_ones())); - - EXPECT_EQ(uint32_t(0x00000003), (mask_trailing_ones())); - EXPECT_EQ(uint32_t(0xC0000000), (mask_leading_ones())); - - EXPECT_EQ(uint32_t(0x000007FF), (mask_trailing_ones())); - EXPECT_EQ(uint32_t(0xFFE00000), (mask_leading_ones())); - - EXPECT_EQ(uint32_t(0xFFFFFFFF), (mask_trailing_ones())); - EXPECT_EQ(uint32_t(0xFFFFFFFF), (mask_leading_ones())); - EXPECT_EQ(uint64_t(0xFFFFFFFFFFFFFFFF), (mask_trailing_ones())); - EXPECT_EQ(uint64_t(0xFFFFFFFFFFFFFFFF), (mask_leading_ones())); - - EXPECT_EQ(uint64_t(0x0000FFFFFFFFFFFF), (mask_trailing_ones())); - EXPECT_EQ(uint64_t(0xFFFFFFFFFFFF0000), (mask_leading_ones())); + EXPECT_EQ(0_u8, (mask_leading_ones())); + EXPECT_EQ(0_u8, (mask_trailing_ones())); + EXPECT_EQ(0_u16, (mask_leading_ones())); + EXPECT_EQ(0_u16, (mask_trailing_ones())); + EXPECT_EQ(0_u32, (mask_leading_ones())); + EXPECT_EQ(0_u32, (mask_trailing_ones())); + EXPECT_EQ(0_u64, (mask_leading_ones())); + EXPECT_EQ(0_u64, (mask_trailing_ones())); + + EXPECT_EQ(0x00000003_u32, (mask_trailing_ones())); + EXPECT_EQ(0xC0000000_u32, (mask_leading_ones())); + + EXPECT_EQ(0x000007FF_u32, (mask_trailing_ones())); + EXPECT_EQ(0xFFE00000_u32, (mask_leading_ones())); + + EXPECT_EQ(0xFFFFFFFF_u32, (mask_trailing_ones())); + EXPECT_EQ(0xFFFFFFFF_u32, (mask_leading_ones())); + EXPECT_EQ(0xFFFFFFFFFFFFFFFF_u64, (mask_trailing_ones())); + EXPECT_EQ(0xFFFFFFFFFFFFFFFF_u64, (mask_leading_ones())); + + EXPECT_EQ(0x0000FFFFFFFFFFFF_u64, (mask_trailing_ones())); + EXPECT_EQ(0xFFFFFFFFFFFF0000_u64, (mask_leading_ones())); + + EXPECT_EQ(0_u128, (mask_trailing_ones())); + EXPECT_EQ(0_u128, (mask_leading_ones())); + + EXPECT_EQ(0x00000000000000007FFFFFFFFFFFFFFF_u128, + (mask_trailing_ones())); + EXPECT_EQ(0xFFFFFFFFFFFFFFFE0000000000000000_u128, + (mask_leading_ones())); + + EXPECT_EQ(0x0000000000000000FFFFFFFFFFFFFFFF_u128, + (mask_trailing_ones())); + EXPECT_EQ(0xFFFFFFFFFFFFFFFF0000000000000000_u128, + (mask_leading_ones())); + + EXPECT_EQ(0x0000000000000001FFFFFFFFFFFFFFFF_u128, + (mask_trailing_ones())); + EXPECT_EQ(0xFFFFFFFFFFFFFFFF8000000000000000_u128, + (mask_leading_ones())); + + EXPECT_EQ(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF_u128, + (mask_trailing_ones())); + EXPECT_EQ(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF_u128, + (mask_leading_ones())); } } // namespace LIBC_NAMESPACE diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel index 8e94a84f586f..19d4c7869799 100644 --- a/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel @@ -13,7 +13,11 @@ licenses(["notice"]) libc_test( name = "math_extras_test", srcs = ["math_extras_test.cpp"], - deps = ["//libc:__support_math_extras"], + deps = [ + "//libc:__support_integer_literals", + "//libc:__support_math_extras", + "//libc:__support_uint128", + ], ) # This test is currently disabled because of an issue in -- GitLab From 6e692e726a5bc4385ab64635e9c06b7574332e4d Mon Sep 17 00:00:00 2001 From: Yinying Li Date: Thu, 7 Mar 2024 14:02:01 -0500 Subject: [PATCH 483/929] [mlir][sparse] Migrate to sparse_tensor.print (#83946) Continuous efforts following #83506. --- .../SparseTensor/CPU/sparse_matvec.mlir | 4 +- .../SparseTensor/CPU/sparse_mttkrp.mlir | 4 +- .../SparseTensor/CPU/sparse_out_mult_elt.mlir | 19 +++-- .../CPU/sparse_out_reduction.mlir | 29 +++---- .../SparseTensor/CPU/sparse_out_simple.mlir | 19 +++-- .../Dialect/SparseTensor/CPU/sparse_pack.mlir | 4 +- .../SparseTensor/CPU/sparse_pack_d.mlir | 53 +++++++----- .../SparseTensor/CPU/sparse_pooling_nhwc.mlir | 27 ++++-- .../CPU/sparse_quantized_matmul.mlir | 4 +- .../CPU/sparse_reduce_custom.mlir | 83 +++++++++---------- .../CPU/sparse_reduce_custom_prod.mlir | 78 +++++++++-------- .../CPU/sparse_reduce_custom_sum.mlir | 4 +- .../SparseTensor/CPU/sparse_reductions.mlir | 4 +- .../CPU/sparse_reductions_min.mlir | 4 +- .../CPU/sparse_reductions_prod.mlir | 4 +- .../SparseTensor/CPU/sparse_reshape.mlir | 53 ++++++++---- .../CPU/sparse_rewrite_push_back.mlir | 4 +- .../CPU/sparse_rewrite_sort_coo.mlir | 4 +- .../CPU/sparse_sampled_matmul.mlir | 4 +- .../CPU/sparse_sampled_mm_fusion.mlir | 34 +++++--- 20 files changed, 249 insertions(+), 190 deletions(-) diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matvec.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matvec.mlir index bdad713709af..b9d1148301dd 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matvec.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matvec.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -88,7 +88,7 @@ module { // // Main driver that reads matrix from file and calls the sparse kernel. // - func.func @entry() { + func.func @main() { %i0 = arith.constant 0 : i32 %c0 = arith.constant 0 : index %c1 = arith.constant 1 : index diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_mttkrp.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_mttkrp.mlir index 30e620b9d610..d615cb66c3d0 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_mttkrp.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_mttkrp.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -84,7 +84,7 @@ module { // // Main driver that reads matrix from file and calls the sparse kernel. // - func.func @entry() { + func.func @main() { %f0 = arith.constant 0.0 : f64 %cst0 = arith.constant 0 : index %cst1 = arith.constant 1 : index diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_mult_elt.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_mult_elt.mlir index f7bcd1122d46..c30c6b9b5cc2 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_mult_elt.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_mult_elt.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -60,7 +60,7 @@ module { } // Driver method to call and verify kernel. - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %f0 = arith.constant 0.0 : f32 @@ -84,11 +84,18 @@ module { // // Verify results. Only two entries stored in result! // - // CHECK: ( 14, 20, 0, 0 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 2 + // CHECK-NEXT: dim = ( 32, 16 ) + // CHECK-NEXT: lvl = ( 32, 16 ) + // CHECK-NEXT: pos[0] : ( 0, 2 + // CHECK-NEXT: crd[0] : ( 2, 31 + // CHECK-NEXT: pos[1] : ( 0, 1, 2 + // CHECK-NEXT: crd[1] : ( 2, 0 + // CHECK-NEXT: values : ( 14, 20 + // CHECK-NEXT: ---- // - %val = sparse_tensor.values %0 : tensor<32x16xf32, #DCSR> to memref - %vv = vector.transfer_read %val[%c0], %f0: memref, vector<4xf32> - vector.print %vv : vector<4xf32> + sparse_tensor.print %0 : tensor<32x16xf32, #DCSR> // Release the resources. bufferization.dealloc_tensor %sta : tensor<32x16xf32, #DCSR> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_reduction.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_reduction.mlir index 17def3f52c00..74f0e7698bc1 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_reduction.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_reduction.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -70,10 +70,7 @@ module { } // Driver method to call and verify tensor kernel. - func.func @entry() { - %c0 = arith.constant 0 : index - %i0 = arith.constant 0 : i32 - + func.func @main() { // Setup very sparse 3-d tensors. %t1 = arith.constant sparse< [ [1,1,3], [2,0,0], [2,2,1], [2,2,2], [2,2,3] ], [ 1, 2, 3, 4, 5 ] @@ -94,23 +91,23 @@ module { // // Verify results. Only two entries stored in result. Correct structure. // - // CHECK: ( 7, 69, 0, 0 ) - // CHECK-NEXT: ( ( 0, 0, 0 ), ( 0, 7, 0 ), ( 0, 0, 69 ) ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 2 + // CHECK-NEXT: dim = ( 3, 3 ) + // CHECK-NEXT: lvl = ( 3, 3 ) + // CHECK-NEXT: pos[0] : ( 0, 2 + // CHECK-NEXT: crd[0] : ( 1, 2 + // CHECK-NEXT: pos[1] : ( 0, 1, 2 + // CHECK-NEXT: crd[1] : ( 1, 2 + // CHECK-NEXT: values : ( 7, 69 + // CHECK-NEXT: ---- // - %val = sparse_tensor.values %0 - : tensor to memref - %vv = vector.transfer_read %val[%c0], %i0: memref, vector<4xi32> - vector.print %vv : vector<4xi32> - %dm = sparse_tensor.convert %0 - : tensor to tensor - %vm = vector.transfer_read %dm[%c0, %c0], %i0: tensor, vector<3x3xi32> - vector.print %vm : vector<3x3xi32> + sparse_tensor.print %0 : tensor // Release the resources. bufferization.dealloc_tensor %st1 : tensor bufferization.dealloc_tensor %st2 : tensor bufferization.dealloc_tensor %0 : tensor - bufferization.dealloc_tensor %dm : tensor return } diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_simple.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_simple.mlir index e2d8c4fd4628..88513c80219a 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_simple.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_simple.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -70,7 +70,7 @@ module { // // Main driver that reads matrix from file and calls the sparse kernel. // - func.func @entry() { + func.func @main() { %d0 = arith.constant 0.0 : f64 %c0 = arith.constant 0 : index @@ -83,11 +83,18 @@ module { // Print the result for verification. // - // CHECK: ( 1, 1.96, 4, 6.25, 9, 16.81, 16, 27.04, 25 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 5, 5 ) + // CHECK-NEXT: lvl = ( 5, 5 ) + // CHECK-NEXT: pos[0] : ( 0, 5 + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4 + // CHECK-NEXT: pos[1] : ( 0, 2, 4, 5, 7, 9 + // CHECK-NEXT: crd[1] : ( 0, 3, 1, 4, 2, 0, 3, 1, 4 + // CHECK-NEXT: values : ( 1, 1.96, 4, 6.25, 9, 16.81, 16, 27.04, 25 + // CHECK-NEXT: ---- // - %m = sparse_tensor.values %0 : tensor to memref - %v = vector.transfer_read %m[%c0], %d0: memref, vector<9xf64> - vector.print %v : vector<9xf64> + sparse_tensor.print %0 : tensor // Release the resources. bufferization.dealloc_tensor %x : tensor diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir index b792d00681dd..7cde6b93d325 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -54,7 +54,7 @@ module { // // Main driver. // - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %f0 = arith.constant 0.0 : f64 %i0 = arith.constant 0 : i32 diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir index 8a65e2449c15..aa1bd04fde87 100755 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -48,7 +48,7 @@ module { // // Main driver. // - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %f0 = arith.constant 0.0 : f32 @@ -107,26 +107,39 @@ module { // // Verify. // - // CHECK: ( ( ( 1, 2 ), ( 3, 4 ), ( 0, 0 ) ), ( ( 0, 0 ), ( 0, 0 ), ( 0, 0 ) ), ( ( 0, 0 ), ( 5, 0 ), ( 6, 7 ) ), ( ( 0, 0 ), ( 8, 0 ), ( 0, 0 ) ) ) - // CHECK: ( ( ( 1, 2 ), ( 0, 3 ), ( 4, 0 ) ), ( ( 5, 6 ), ( 0, 0 ), ( 0, 7 ) ), ( ( 8, 9 ), ( 10, 11 ), ( 12, 13 ) ), ( ( 14, 0 ), ( 0, 15 ), ( 0, 16 ) ) ) - // CHECK: ( ( ( 1, 2 ), ( 0, 3 ), ( 4, 0 ) ), ( ( 5, 6 ), ( 0, 0 ), ( 0, 7 ) ), ( ( 8, 9 ), ( 10, 11 ), ( 12, 13 ) ), ( ( 14, 0 ), ( 0, 15 ), ( 0, 16 ) ) ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 8 + // CHECK-NEXT: dim = ( 4, 3, 2 ) + // CHECK-NEXT: lvl = ( 4, 3, 2 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 0, 2, 3 + // CHECK-NEXT: pos[1] : ( 0, 2, 4, 5 + // CHECK-NEXT: crd[1] : ( 0, 1, 1, 2, 1 + // CHECK-NEXT: pos[2] : ( 0, 2, 4, 5, 7, 8 + // CHECK-NEXT: crd[2] : ( 0, 1, 0, 1, 0, 0, 1, 0 + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 16 + // CHECK-NEXT: dim = ( 4, 3, 2 ) + // CHECK-NEXT: lvl = ( 4, 3, 2 ) + // CHECK-NEXT: pos[2] : ( 0, 2, 3, 4, 6, 6, 7, 9, 11, 13, 14, 15, 16 + // CHECK-NEXT: crd[2] : ( 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1 + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 22 + // CHECK-NEXT: dim = ( 4, 3, 2 ) + // CHECK-NEXT: lvl = ( 4, 3, 2 ) + // CHECK-NEXT: pos[1] : ( 0, 3, 5, 8, 11 + // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 2, 0, 1, 2, 0, 1, 2 + // CHECK-NEXT: values : ( 1, 2, 0, 3, 4, 0, 5, 6, 0, 7, 8, 9, 10, 11, 12, 13, 14, 0, 0, 15, 0, 16 + // CHECK-NEXT: ---- // + sparse_tensor.print %s0 : tensor<4x3x2xf32, #CCC> + sparse_tensor.print %s1 : tensor<4x3x2xf32, #BatchedCSR> + sparse_tensor.print %s2 : tensor<4x3x2xf32, #CSRDense> - %d0 = sparse_tensor.convert %s0 : tensor<4x3x2xf32, #CCC> to tensor<4x3x2xf32> - %v0 = vector.transfer_read %d0[%c0, %c0, %c0], %f0 : tensor<4x3x2xf32>, vector<4x3x2xf32> - vector.print %v0 : vector<4x3x2xf32> - - %d1 = sparse_tensor.convert %s1 : tensor<4x3x2xf32, #BatchedCSR> to tensor<4x3x2xf32> - %v1 = vector.transfer_read %d1[%c0, %c0, %c0], %f0 : tensor<4x3x2xf32>, vector<4x3x2xf32> - vector.print %v1 : vector<4x3x2xf32> - - %d2 = sparse_tensor.convert %s2 : tensor<4x3x2xf32, #CSRDense> to tensor<4x3x2xf32> - %v2 = vector.transfer_read %d1[%c0, %c0, %c0], %f0 : tensor<4x3x2xf32>, vector<4x3x2xf32> - vector.print %v2 : vector<4x3x2xf32> - - bufferization.dealloc_tensor %d0 : tensor<4x3x2xf32> - bufferization.dealloc_tensor %d1 : tensor<4x3x2xf32> - bufferization.dealloc_tensor %d2 : tensor<4x3x2xf32> // FIXME: doing this explicitly crashes runtime // bufferization.dealloc_tensor %s0 : tensor<4x3x2xf32, #CCC> // bufferization.dealloc_tensor %s1 : tensor<4x3x2xf32, #BatchedCSR> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pooling_nhwc.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pooling_nhwc.mlir index 3ce089d7a7cf..39699fbdb14e 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pooling_nhwc.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pooling_nhwc.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -47,7 +47,7 @@ func.func @pooling_nhwc_sum(%input: tensor<1x4x4x1xf32>, %filter: tensor<2x2xf32 } -func.func @entry() { +func.func @main() { %c0 = arith.constant 0 : index %zero = arith.constant 0.00000e+00 : f32 @@ -76,17 +76,26 @@ func.func @entry() { // // Sparse pooling should have the same output. // - - // CHECK-NEXT: ( ( ( ( 6 ), ( 6 ), ( 6 ) ), ( ( 6 ), ( 6 ), ( 6 ) ), ( ( 6 ), ( 6 ), ( 6 ) ) ) ) - %s1 = sparse_tensor.convert %CCCC_ret : tensor<1x3x3x1xf32, #CCCC> to tensor<1x3x3x1xf32> - %v1 = vector.transfer_read %s1[%c0, %c0, %c0, %c0], %zero - : tensor<1x3x3x1xf32>, vector<1x3x3x1xf32> - vector.print %v1 : vector<1x3x3x1xf32> + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 1, 3, 3, 1 ) + // CHECK-NEXT: lvl = ( 1, 3, 3, 1 ) + // CHECK-NEXT: pos[0] : ( 0, 1 + // CHECK-NEXT: crd[0] : ( 0 + // CHECK-NEXT: pos[1] : ( 0, 3 + // CHECK-NEXT: crd[1] : ( 0, 1, 2 + // CHECK-NEXT: pos[2] : ( 0, 3, 6, 9 + // CHECK-NEXT: crd[2] : ( 0, 1, 2, 0, 1, 2, 0, 1, 2 + // CHECK-NEXT: pos[3] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 + // CHECK-NEXT: crd[3] : ( 0, 0, 0, 0, 0, 0, 0, 0, 0 + // CHECK-NEXT: values : ( 6, 6, 6, 6, 6, 6, 6, 6, 6 + // CHECK-NEXT: ---- + // + sparse_tensor.print %CCCC_ret : tensor<1x3x3x1xf32, #CCCC> // Releases resources. bufferization.dealloc_tensor %in_CCCC : tensor<1x4x4x1xf32, #CCCC> bufferization.dealloc_tensor %CCCC_ret : tensor<1x3x3x1xf32, #CCCC> bufferization.dealloc_tensor %dense_ret : tensor<1x3x3x1xf32> - bufferization.dealloc_tensor %s1 : tensor<1x3x3x1xf32> return } diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_quantized_matmul.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_quantized_matmul.mlir index b322d965f2dc..873322929232 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_quantized_matmul.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_quantized_matmul.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -49,7 +49,7 @@ module { return %0: tensor<5x6xi32> } - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %i0 = arith.constant 0 : i32 diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom.mlir index 17219cde035d..a927a5dfb94b 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -114,39 +114,8 @@ module { return %0 : tensor } - // Dumps a sparse vector of type f64. - func.func @dump_vec(%arg0: tensor) { - // Dump the values array to verify only sparse contents are stored. - %c0 = arith.constant 0 : index - %d0 = arith.constant 0.0 : f64 - %0 = sparse_tensor.values %arg0 : tensor to memref - %1 = vector.transfer_read %0[%c0], %d0: memref, vector<8xf64> - vector.print %1 : vector<8xf64> - // Dump the dense vector to verify structure is correct. - %dv = sparse_tensor.convert %arg0 : tensor to tensor - %2 = vector.transfer_read %dv[%c0], %d0: tensor, vector<16xf64> - vector.print %2 : vector<16xf64> - bufferization.dealloc_tensor %dv : tensor - return - } - - // Dump a sparse matrix. - func.func @dump_mat(%arg0: tensor) { - // Dump the values array to verify only sparse contents are stored. - %c0 = arith.constant 0 : index - %d0 = arith.constant 0.0 : f64 - %0 = sparse_tensor.values %arg0 : tensor to memref - %1 = vector.transfer_read %0[%c0], %d0: memref, vector<16xf64> - vector.print %1 : vector<16xf64> - %dm = sparse_tensor.convert %arg0 : tensor to tensor - %2 = vector.transfer_read %dm[%c0, %c0], %d0: tensor, vector<5x5xf64> - vector.print %2 : vector<5x5xf64> - bufferization.dealloc_tensor %dm : tensor - return - } - // Driver method to call and verify vector kernels. - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index // Setup sparse matrices. @@ -171,19 +140,43 @@ module { // // Verify the results. // - // CHECK: ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( ( 1, 2, 0, 0, 0 ), ( 3, 0, 0, 0, 0 ), ( 0, 0, 4, 5, 6 ), ( 7, 0, 8, 9, 0 ), ( 0, 0, 0, 0, 0 ) ) - // CHECK-NEXT: ( 6, 5, 4, 3, 2, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( ( 6, 0, 0, 0, 0 ), ( 0, 0, 0, 5, 0 ), ( 4, 0, 0, 3, 0 ), ( 0, 2, 0, 0, 0 ), ( 0, 11, 0, 0, 0 ) ) - // CHECK-NEXT: ( 7, 7, 9, 8, 7, 7, 12, 11, 11, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( ( 7, 0, 0, 7, 0 ), ( 9, 0, 0, 0, 0 ), ( 8, 7, 0, 7, 0 ), ( 12, 11, 0, 11, 0 ), ( 0, 0, 0, 0, 0 ) ) - // CHECK-NEXT: ( 7, 7, 9, 8, 7, 7, 12, 11, 11, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( ( 7, 0, 0, 7, 0 ), ( 9, 0, 0, 0, 0 ), ( 8, 7, 0, 7, 0 ), ( 12, 11, 0, 11, 0 ), ( 0, 0, 0, 0, 0 ) ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 4, 5 ) + // CHECK-NEXT: lvl = ( 4, 5 ) + // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9 + // CHECK-NEXT: crd[1] : ( 0, 1, 0, 2, 3, 4, 0, 2, 3 + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 6 + // CHECK-NEXT: dim = ( 5, 4 ) + // CHECK-NEXT: lvl = ( 5, 4 ) + // CHECK-NEXT: pos[1] : ( 0, 1, 2, 4, 5, 6 + // CHECK-NEXT: crd[1] : ( 0, 3, 0, 3, 1, 1 + // CHECK-NEXT: values : ( 6, 5, 4, 3, 2, 11 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 4, 4 ) + // CHECK-NEXT: lvl = ( 4, 4 ) + // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9 + // CHECK-NEXT: crd[1] : ( 0, 3, 0, 0, 1, 3, 0, 1, 3 + // CHECK-NEXT: values : ( 7, 7, 9, 8, 7, 7, 12, 11, 11 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 4, 4 ) + // CHECK-NEXT: lvl = ( 4, 4 ) + // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9 + // CHECK-NEXT: crd[1] : ( 0, 3, 0, 0, 1, 3, 0, 1, 3 + // CHECK-NEXT: values : ( 7, 7, 9, 8, 7, 7, 12, 11, 11 + // CHECK-NEXT: ---- // - call @dump_mat(%sm1) : (tensor) -> () - call @dump_mat(%sm2r) : (tensor) -> () - call @dump_mat(%5) : (tensor) -> () - call @dump_mat(%6) : (tensor) -> () + sparse_tensor.print %sm1 : tensor + sparse_tensor.print %sm2r : tensor + sparse_tensor.print %5 : tensor + sparse_tensor.print %6 : tensor // Release the resources. bufferization.dealloc_tensor %sm1 : tensor diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_prod.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_prod.mlir index 6e2c572cf21b..18bf6a71c530 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_prod.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_prod.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -89,39 +89,9 @@ module { return %0 : tensor } - // Dumps a sparse vector of type f64. - func.func @dump_vec(%arg0: tensor) { - // Dump the values array to verify only sparse contents are stored. - %c0 = arith.constant 0 : index - %d0 = arith.constant 0.0 : f64 - %0 = sparse_tensor.values %arg0 : tensor to memref - %1 = vector.transfer_read %0[%c0], %d0: memref, vector<8xf64> - vector.print %1 : vector<8xf64> - // Dump the dense vector to verify structure is correct. - %dv = sparse_tensor.convert %arg0 : tensor to tensor - %2 = vector.transfer_read %dv[%c0], %d0: tensor, vector<16xf64> - vector.print %2 : vector<16xf64> - bufferization.dealloc_tensor %dv : tensor - return - } - - // Dump a sparse matrix. - func.func @dump_mat(%arg0: tensor) { - // Dump the values array to verify only sparse contents are stored. - %c0 = arith.constant 0 : index - %d0 = arith.constant 0.0 : f64 - %0 = sparse_tensor.values %arg0 : tensor to memref - %1 = vector.transfer_read %0[%c0], %d0: memref, vector<16xf64> - vector.print %1 : vector<16xf64> - %dm = sparse_tensor.convert %arg0 : tensor to tensor - %2 = vector.transfer_read %dm[%c0, %c0], %d0: tensor, vector<5x5xf64> - vector.print %2 : vector<5x5xf64> - bufferization.dealloc_tensor %dm : tensor - return - } // Driver method to call and verify vector kernels. - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index // Setup sparse matrices. @@ -144,15 +114,43 @@ module { // // Verify the results. // - // CHECK: ( 2, 3, 120, 504, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 2, 3, 120, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 6, 5, 12, 2, 11, 0, 0, 0 ) - // CHECK-NEXT: ( 6, 5, 12, 2, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 4, 5 ) + // CHECK-NEXT: lvl = ( 4, 5 ) + // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9 + // CHECK-NEXT: crd[1] : ( 0, 1, 0, 2, 3, 4, 0, 2, 3 + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 6 + // CHECK-NEXT: dim = ( 5, 4 ) + // CHECK-NEXT: lvl = ( 5, 4 ) + // CHECK-NEXT: pos[1] : ( 0, 1, 2, 4, 5, 6 + // CHECK-NEXT: crd[1] : ( 0, 3, 0, 3, 1, 1 + // CHECK-NEXT: values : ( 6, 5, 4, 3, 2, 11 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 4 + // CHECK-NEXT: dim = ( 4 ) + // CHECK-NEXT: lvl = ( 4 ) + // CHECK-NEXT: pos[0] : ( 0, 4 + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 + // CHECK-NEXT: values : ( 2, 3, 120, 504 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 5 + // CHECK-NEXT: dim = ( 5 ) + // CHECK-NEXT: lvl = ( 5 ) + // CHECK-NEXT: pos[0] : ( 0, 5 + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4 + // CHECK-NEXT: values : ( 6, 5, 12, 2, 11 + // CHECK-NEXT: ---- // - call @dump_mat(%sm1) : (tensor) -> () - call @dump_mat(%sm2r) : (tensor) -> () - call @dump_vec(%1) : (tensor) -> () - call @dump_vec(%2) : (tensor) -> () + sparse_tensor.print %sm1 : tensor + sparse_tensor.print %sm2r : tensor + sparse_tensor.print %1 : tensor + sparse_tensor.print %2 : tensor // Release the resources. bufferization.dealloc_tensor %sm1 : tensor diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_sum.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_sum.mlir index 80c35676e804..8588ebd98cc9 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_sum.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_sum.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -145,7 +145,7 @@ module { return } - func.func @entry() { + func.func @main() { %ri = arith.constant dense<0> : tensor // Sparse vector of length 8 with 2 stored elements (and thus 6 implicit zeros). diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions.mlir index 4ad23d1c0312..96ec8bad8b55 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -106,7 +106,7 @@ module { return } - func.func @entry() { + func.func @main() { %ri = arith.constant dense< 7 > : tensor %rf = arith.constant dense< 2.0 > : tensor diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_min.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_min.mlir index 198920024221..16c1d7df7e89 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_min.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_min.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -82,7 +82,7 @@ module { return } - func.func @entry() { + func.func @main() { %ri = arith.constant dense<999> : tensor // Vectors with a few zeros. diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_prod.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_prod.mlir index 7ec30787fea4..4797fbb8f531 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_prod.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_prod.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -162,7 +162,7 @@ module { return } - func.func @entry() { + func.func @main() { // Note: Constants bufferize to read-only buffers. %ri = arith.constant dense< 7 > : tensor %rf = arith.constant dense< 2.0 > : tensor diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reshape.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reshape.mlir index b551f9545dc4..4c26ebe6e401 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reshape.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reshape.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -63,7 +63,7 @@ module { } - func.func @entry() { + func.func @main() { %m = arith.constant dense <[ [ 1.1, 0.0, 1.3, 0.0 ], [ 2.1, 0.0, 2.3, 0.0 ], [ 3.1, 0.0, 3.3, 0.0 ]]> : tensor<3x4xf64> @@ -76,20 +76,41 @@ module { %c0 = arith.constant 0 : index %df = arith.constant -1.0 : f64 - // CHECK: ( 1.1, 1.3, 2.1, 2.3, 3.1, 3.3 - %b0 = sparse_tensor.values %reshaped0: tensor<2x6xf64, #SparseMatrix> to memref - %v0 = vector.transfer_read %b0[%c0], %df: memref, vector<12xf64> - vector.print %v0 : vector<12xf64> - - // CHECK: ( 1.1, 1.3, 2.1, 2.3, 3.1, 3.3 - %b1 = sparse_tensor.values %reshaped1: tensor<12xf64, #SparseVector> to memref - %v1 = vector.transfer_read %b1[%c0], %df: memref, vector<12xf64> - vector.print %v1 : vector<12xf64> - - // CHECK: ( 1.1, 1.3, 2.1, 2.3, 3.1, 3.3 - %b2 = sparse_tensor.values %reshaped2: tensor<2x3x2xf64, #Sparse3dTensor> to memref - %v2 = vector.transfer_read %b2[%c0], %df: memref, vector<12xf64> - vector.print %v2: vector<12xf64> + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 6 + // CHECK-NEXT: dim = ( 2, 6 ) + // CHECK-NEXT: lvl = ( 2, 6 ) + // CHECK-NEXT: pos[0] : ( 0, 2 + // CHECK-NEXT: crd[0] : ( 0, 1 + // CHECK-NEXT: pos[1] : ( 0, 3, 6 + // CHECK-NEXT: crd[1] : ( 0, 2, 4, 0, 2, 4 + // CHECK-NEXT: values : ( 1.1, 1.3, 2.1, 2.3, 3.1, 3.3 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 6 + // CHECK-NEXT: dim = ( 12 ) + // CHECK-NEXT: lvl = ( 12 ) + // CHECK-NEXT: pos[0] : ( 0, 6 + // CHECK-NEXT: crd[0] : ( 0, 2, 4, 6, 8, 10 + // CHECK-NEXT: values : ( 1.1, 1.3, 2.1, 2.3, 3.1, 3.3 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 6 + // CHECK-NEXT: dim = ( 2, 3, 2 ) + // CHECK-NEXT: lvl = ( 2, 3, 2 ) + // CHECK-NEXT: pos[0] : ( 0, 2 + // CHECK-NEXT: crd[0] : ( 0, 1 + // CHECK-NEXT: pos[1] : ( 0, 3, 6 + // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2 + // CHECK-NEXT: pos[2] : ( 0, 1, 2, 3, 4, 5, 6 + // CHECK-NEXT: crd[2] : ( 0, 0, 0, 0, 0, 0 + // CHECK-NEXT: values : ( 1.1, 1.3, 2.1, 2.3, 3.1, 3.3 + // CHECK-NEXT: ---- + // + sparse_tensor.print %reshaped0: tensor<2x6xf64, #SparseMatrix> + sparse_tensor.print %reshaped1: tensor<12xf64, #SparseVector> + sparse_tensor.print %reshaped2: tensor<2x3x2xf64, #Sparse3dTensor> bufferization.dealloc_tensor %sm : tensor<3x4xf64, #SparseMatrix> bufferization.dealloc_tensor %reshaped0 : tensor<2x6xf64, #SparseMatrix> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_push_back.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_push_back.mlir index c2e83fc61c4b..1536249e60f2 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_push_back.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_push_back.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -28,7 +28,7 @@ // RUN: %if mlir_arm_sve_tests %{ %{compile_sve} | %{run_sve} | FileCheck %s %} module { - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %c1 = arith.constant 1 : index %c10 = arith.constant 10 : index diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_sort_coo.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_sort_coo.mlir index e7dd0ad32a24..0682bc6f314f 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_sort_coo.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_sort_coo.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -64,7 +64,7 @@ module { } // The main driver. - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : i32 %c1 = arith.constant 1 : i32 %c2 = arith.constant 2 : i32 diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_matmul.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_matmul.mlir index 3330d2249707..085b36a36870 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_matmul.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_matmul.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -81,7 +81,7 @@ module { // // Main driver that reads matrix from file and calls the sparse kernel. // - func.func @entry() { + func.func @main() { %d0 = arith.constant 0.0 : f32 %c0 = arith.constant 0 : index %c1 = arith.constant 1 : index diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_mm_fusion.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_mm_fusion.mlir index afaf36c4072c..20a8c5f812de 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_mm_fusion.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_mm_fusion.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -169,7 +169,7 @@ module { // // Main driver. // - func.func @entry() { + func.func @main() { %d0 = arith.constant 0.0 : f64 %c0 = arith.constant 0 : index @@ -207,22 +207,36 @@ module { // CHECK-SAME: ( 0, 0, 0, 0, 0, 0, 0, 0 ), ( 0, 0, 0, 0, 0, 0, 0, 0 ), // CHECK-SAME: ( 0, 0, 0, 0, 0, 0, 0, 0 ), ( 0, 0, 0, 0, 0, 0, 0, 192 ) ) // - // CHECK-NEXT: ( 96, 192, 0, 0 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 2 + // CHECK-NEXT: dim = ( 8, 8 ) + // CHECK-NEXT: lvl = ( 8, 8 ) + // CHECK-NEXT: pos[0] : ( 0, 2 + // CHECK-NEXT: crd[0] : ( 0, 7 + // CHECK-NEXT: pos[1] : ( 0, 1, 2 + // CHECK-NEXT: crd[1] : ( 0, 7 + // CHECK-NEXT: values : ( 96, 192 + // CHECK-NEXT: ---- // - // CHECK-NEXT: ( 96, 192, 0, 0 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 2 + // CHECK-NEXT: dim = ( 8, 8 ) + // CHECK-NEXT: lvl = ( 8, 8 ) + // CHECK-NEXT: pos[0] : ( 0, 2 + // CHECK-NEXT: crd[0] : ( 0, 7 + // CHECK-NEXT: pos[1] : ( 0, 1, 2 + // CHECK-NEXT: crd[1] : ( 0, 7 + // CHECK-NEXT: values : ( 96, 192 + // CHECK-NEXT: ---- // - %m2 = sparse_tensor.values %2 : tensor<8x8xf64, #SM> to memref - %m3 = sparse_tensor.values %3 : tensor<8x8xf64, #SM> to memref %v0 = vector.transfer_read %0[%c0, %c0], %d0 : tensor<8x8xf64>, vector<8x8xf64> %v1 = vector.transfer_read %1[%c0, %c0], %d0 : tensor<8x8xf64>, vector<8x8xf64> - %v2 = vector.transfer_read %m2[%c0], %d0 : memref, vector<4xf64> - %v3 = vector.transfer_read %m3[%c0], %d0 : memref, vector<4xf64> vector.print %v0 : vector<8x8xf64> vector.print %v1 : vector<8x8xf64> - vector.print %v2 : vector<4xf64> - vector.print %v3 : vector<4xf64> + sparse_tensor.print %2 : tensor<8x8xf64, #SM> + sparse_tensor.print %3 : tensor<8x8xf64, #SM> // Release the resources. bufferization.dealloc_tensor %s : tensor<8x8xf64, #SM> -- GitLab From 4cb5a96af646e18f9fc8c1b337299d5465f0a4d6 Mon Sep 17 00:00:00 2001 From: Yinying Li Date: Thu, 7 Mar 2024 14:02:20 -0500 Subject: [PATCH 484/929] [mlir][sparse] Migrate more tests to sparse_tensor.print (#84249) Continuous efforts following #83946. --- .../SparseTensor/CPU/sparse_scale.mlir | 17 +- .../SparseTensor/CPU/sparse_scf_nested.mlir | 46 ++-- .../SparseTensor/CPU/sparse_select.mlir | 83 ++++---- .../CPU/sparse_semiring_select.mlir | 26 ++- .../Dialect/SparseTensor/CPU/sparse_sign.mlir | 17 +- .../SparseTensor/CPU/sparse_sorted_coo.mlir | 200 ++++++------------ .../Dialect/SparseTensor/CPU/sparse_spmm.mlir | 4 +- .../SparseTensor/CPU/sparse_storage.mlir | 197 ++++++----------- .../CPU/sparse_strided_conv_2d_nhwc_hwcf.mlir | 4 +- .../Dialect/SparseTensor/CPU/sparse_sum.mlir | 4 +- .../SparseTensor/CPU/sparse_sum_bf16.mlir | 4 +- .../SparseTensor/CPU/sparse_sum_c32.mlir | 4 +- .../SparseTensor/CPU/sparse_sum_f16.mlir | 4 +- .../Dialect/SparseTensor/CPU/sparse_tanh.mlir | 37 +--- .../SparseTensor/CPU/sparse_tensor_mul.mlir | 36 ++-- .../SparseTensor/CPU/sparse_tensor_ops.mlir | 44 ++-- .../SparseTensor/CPU/sparse_transpose.mlir | 45 ++-- .../CPU/sparse_transpose_coo.mlir | 36 ++-- .../SparseTensor/CPU/sparse_unary.mlir | 126 ++++++----- .../SparseTensor/CPU/sparse_vector_ops.mlir | 104 +++++---- 20 files changed, 478 insertions(+), 560 deletions(-) diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scale.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scale.mlir index 6ec13fd623b5..4e9090ae201d 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scale.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scale.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -65,7 +65,7 @@ module { // and then calls the sparse scaling kernel with the sparse tensor // as input argument. // - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %f0 = arith.constant 0.0 : f32 @@ -88,11 +88,16 @@ module { // Print the resulting compacted values for verification. // - // CHECK: ( 2, 2, 2, 4, 6, 8, 2, 10, 2, 2, 12, 2, 14, 2, 2, 16 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 16 + // CHECK-NEXT: dim = ( 8, 8 ) + // CHECK-NEXT: lvl = ( 8, 8 ) + // CHECK-NEXT: pos[1] : ( 0, 3, 4, 5, 6, 8, 11, 14, 16 + // CHECK-NEXT: crd[1] : ( 0, 2, 7, 1, 2, 3, 1, 4, 1, 2, 5, 2, 6, 7, 2, 7 + // CHECK-NEXT: values : ( 2, 2, 2, 4, 6, 8, 2, 10, 2, 2, 12, 2, 14, 2, 2, 16 + // CHECK-NEXT: ---- // - %m = sparse_tensor.values %2 : tensor<8x8xf32, #CSR> to memref - %v = vector.transfer_read %m[%c0], %f0: memref, vector<16xf32> - vector.print %v : vector<16xf32> + sparse_tensor.print %2 : tensor<8x8xf32, #CSR> // Release the resources. bufferization.dealloc_tensor %1 : tensor<8x8xf32, #CSR> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scf_nested.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scf_nested.mlir index 439144fedeeb..dd8396dc23b0 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scf_nested.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scf_nested.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -68,17 +68,7 @@ module @func_sparse.2 { return %1 : tensor<2x3x4xf64, #SparseMatrix> } - func.func @dump(%arg0: tensor<2x3x4xf64, #SparseMatrix>) { - %d0 = arith.constant 0.0 : f64 - %c0 = arith.constant 0 : index - %dm = sparse_tensor.convert %arg0 : tensor<2x3x4xf64, #SparseMatrix> to tensor<2x3x4xf64> - %0 = vector.transfer_read %dm[%c0, %c0, %c0], %d0: tensor<2x3x4xf64>, vector<2x3x4xf64> - vector.print %0 : vector<2x3x4xf64> - bufferization.dealloc_tensor %dm : tensor<2x3x4xf64> - return - } - - func.func public @entry() { + func.func public @main() { %src = arith.constant dense<[ [ [ 1.0, 2.0, 3.0, 4.0 ], [ 5.0, 6.0, 7.0, 8.0 ], @@ -96,10 +86,34 @@ module @func_sparse.2 { %sm_t = call @condition(%t, %sm) : (i1, tensor<2x3x4xf64, #SparseMatrix>) -> tensor<2x3x4xf64, #SparseMatrix> %sm_f = call @condition(%f, %sm) : (i1, tensor<2x3x4xf64, #SparseMatrix>) -> tensor<2x3x4xf64, #SparseMatrix> - // CHECK: ( ( ( 0, 1, 2, 3 ), ( 4, 5, 6, 7 ), ( 8, 9, 10, 11 ) ), ( ( 12, 13, 14, 15 ), ( 16, 17, 18, 19 ), ( 20, 21, 22, 23 ) ) ) - // CHECK-NEXT: ( ( ( 2, 3, 4, 5 ), ( 6, 7, 8, 9 ), ( 10, 11, 12, 13 ) ), ( ( 14, 15, 16, 17 ), ( 18, 19, 20, 21 ), ( 22, 23, 24, 25 ) ) ) - call @dump(%sm_t) : (tensor<2x3x4xf64, #SparseMatrix>) -> () - call @dump(%sm_f) : (tensor<2x3x4xf64, #SparseMatrix>) -> () + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 2, 3, 4 ) + // CHECK-NEXT: lvl = ( 2, 3, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 2 + // CHECK-NEXT: crd[0] : ( 0, 1 + // CHECK-NEXT: pos[1] : ( 0, 3, 6 + // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2 + // CHECK-NEXT: pos[2] : ( 0, 4, 8, 12, 16, 20, 24 + // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 + // CHECK-NEXT: values : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 2, 3, 4 ) + // CHECK-NEXT: lvl = ( 2, 3, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 2 + // CHECK-NEXT: crd[0] : ( 0, 1 + // CHECK-NEXT: pos[1] : ( 0, 3, 6 + // CHECK-NEXT: crd[1] : ( 0, 1, 2, 0, 1, 2 + // CHECK-NEXT: pos[2] : ( 0, 4, 8, 12, 16, 20, 24 + // CHECK-NEXT: crd[2] : ( 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 + // CHECK-NEXT: values : ( 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 + // CHECK-NEXT: ---- + // + sparse_tensor.print %sm_t : tensor<2x3x4xf64, #SparseMatrix> + sparse_tensor.print %sm_f : tensor<2x3x4xf64, #SparseMatrix> bufferization.dealloc_tensor %sm : tensor<2x3x4xf64, #SparseMatrix> bufferization.dealloc_tensor %sm_t : tensor<2x3x4xf64, #SparseMatrix> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_select.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_select.mlir index 533afb6644ae..68bc17175e3b 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_select.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_select.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -97,39 +97,8 @@ module { return %0 : tensor } - // Dumps a sparse vector of type f64. - func.func @dump_vec(%arg0: tensor) { - // Dump the values array to verify only sparse contents are stored. - %c0 = arith.constant 0 : index - %d0 = arith.constant 0.0 : f64 - %0 = sparse_tensor.values %arg0 : tensor to memref - %1 = vector.transfer_read %0[%c0], %d0: memref, vector<8xf64> - vector.print %1 : vector<8xf64> - // Dump the dense vector to verify structure is correct. - %dv = sparse_tensor.convert %arg0 : tensor to tensor - %2 = vector.transfer_read %dv[%c0], %d0: tensor, vector<16xf64> - vector.print %2 : vector<16xf64> - bufferization.dealloc_tensor %dv : tensor - return - } - - // Dump a sparse matrix. - func.func @dump_mat(%arg0: tensor) { - // Dump the values array to verify only sparse contents are stored. - %c0 = arith.constant 0 : index - %d0 = arith.constant 0.0 : f64 - %0 = sparse_tensor.values %arg0 : tensor to memref - %1 = vector.transfer_read %0[%c0], %d0: memref, vector<16xf64> - vector.print %1 : vector<16xf64> - %dm = sparse_tensor.convert %arg0 : tensor to tensor - %2 = vector.transfer_read %dm[%c0, %c0], %d0: tensor, vector<5x5xf64> - vector.print %2 : vector<5x5xf64> - bufferization.dealloc_tensor %dm : tensor - return - } - // Driver method to call and verify vector kernels. - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index // Setup sparse matrices. @@ -151,19 +120,43 @@ module { // // Verify the results. // - // CHECK: ( 1, 2, -4, 0, 5, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 0, 2, 0, -4, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 1, 2, 3, 4, 5, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( ( 0, 0, 0, 1, 0 ), ( 0, 0, 0, 0, 2 ), ( 0, 3, 0, 4, 0 ), ( 0, 0, 0, 5, 6 ), ( 0, 0, 7, 0, 0 ) ) - // CHECK-NEXT: ( 1, 2, 5, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 0, 2, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 1, 2, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( ( 0, 0, 0, 1, 0 ), ( 0, 0, 0, 0, 2 ), ( 0, 0, 0, 4, 0 ), ( 0, 0, 0, 0, 6 ), ( 0, 0, 0, 0, 0 ) ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 5 + // CHECK-NEXT: dim = ( 10 ) + // CHECK-NEXT: lvl = ( 10 ) + // CHECK-NEXT: pos[0] : ( 0, 5 + // CHECK-NEXT: crd[0] : ( 1, 3, 5, 7, 9 + // CHECK-NEXT: values : ( 1, 2, -4, 0, 5 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 7 + // CHECK-NEXT: dim = ( 5, 5 ) + // CHECK-NEXT: lvl = ( 5, 5 ) + // CHECK-NEXT: pos[1] : ( 0, 1, 2, 4, 6, 7 + // CHECK-NEXT: crd[1] : ( 3, 4, 1, 3, 3, 4, 2 + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 3 + // CHECK-NEXT: dim = ( 10 ) + // CHECK-NEXT: lvl = ( 10 ) + // CHECK-NEXT: pos[0] : ( 0, 3 + // CHECK-NEXT: crd[0] : ( 1, 3, 9 + // CHECK-NEXT: values : ( 1, 2, 5 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 4 + // CHECK-NEXT: dim = ( 5, 5 ) + // CHECK-NEXT: lvl = ( 5, 5 ) + // CHECK-NEXT: pos[1] : ( 0, 1, 2, 3, 4, 4 + // CHECK-NEXT: crd[1] : ( 3, 4, 3, 4 + // CHECK-NEXT: values : ( 1, 2, 4, 6 + // CHECK-NEXT: ---- // - call @dump_vec(%sv1) : (tensor) -> () - call @dump_mat(%sm1) : (tensor) -> () - call @dump_vec(%1) : (tensor) -> () - call @dump_mat(%2) : (tensor) -> () + sparse_tensor.print %sv1 : tensor + sparse_tensor.print %sm1 : tensor + sparse_tensor.print %1 : tensor + sparse_tensor.print %2 : tensor // Release the resources. bufferization.dealloc_tensor %sv1 : tensor diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_semiring_select.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_semiring_select.mlir index 6244be0ba7ab..f4435c81117b 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_semiring_select.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_semiring_select.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -60,7 +60,7 @@ module { } // Driver method to call and verify vector kernels. - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %f0 = arith.constant 0.0 : f64 @@ -86,20 +86,24 @@ module { tensor<5x5xf64, #DCSR>) -> tensor<5x5xf64, #DCSR> - // CHECK: ( ( 0.1, 1.1, 0, 0, 0 ), - // CHECK-SAME: ( 0, 1.1, 2.2, 0, 0 ), - // CHECK-SAME: ( 0, 0, 2.1, 3.3, 0 ), - // CHECK-SAME: ( 0, 0, 0, 3.1, 4.4 ), - // CHECK-SAME: ( 0, 0, 0, 0, 4.1 ) ) - %r = sparse_tensor.convert %1 : tensor<5x5xf64, #DCSR> to tensor<5x5xf64> - %v2 = vector.transfer_read %r[%c0, %c0], %f0 : tensor<5x5xf64>, vector<5x5xf64> - vector.print %v2 : vector<5x5xf64> + // + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 5, 5 ) + // CHECK-NEXT: lvl = ( 5, 5 ) + // CHECK-NEXT: pos[0] : ( 0, 5 + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4 + // CHECK-NEXT: pos[1] : ( 0, 2, 4, 6, 8, 9 + // CHECK-NEXT: crd[1] : ( 0, 1, 1, 2, 2, 3, 3, 4, 4 + // CHECK-NEXT: values : ( 0.1, 1.1, 1.1, 2.2, 2.1, 3.3, 3.1, 4.4, 4.1 + // CHECK-NEXT: ---- + // + sparse_tensor.print %1 : tensor<5x5xf64, #DCSR> // Release the resources. bufferization.dealloc_tensor %sl: tensor<5x5xf64, #DCSR> bufferization.dealloc_tensor %sr: tensor<5x5xf64, #DCSR> bufferization.dealloc_tensor %1: tensor<5x5xf64, #DCSR> - bufferization.dealloc_tensor %r : tensor<5x5xf64> return } diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sign.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sign.mlir index 08e75dfa2c02..c09374918b7d 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sign.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sign.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -79,7 +79,7 @@ module { } // Driver method to call and verify sign kernel. - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %du = arith.constant 0.0 : f64 @@ -110,11 +110,16 @@ module { // // Verify the results. // - // CHECK: ( -1, 1, -1, 1, 1, -1, nan, -nan, 1, -1, -0, 0, 0 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 12 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 12 + // CHECK-NEXT: crd[0] : ( 0, 3, 5, 11, 13, 17, 18, 20, 21, 28, 29, 31 + // CHECK-NEXT: values : ( -1, 1, -1, 1, 1, -1, nan, -nan, 1, -1, -0, 0 + // CHECK-NEXT: ---- // - %1 = sparse_tensor.values %0 : tensor to memref - %2 = vector.transfer_read %1[%c0], %du: memref, vector<13xf64> - vector.print %2 : vector<13xf64> + sparse_tensor.print %0 : tensor // Release the resources. bufferization.dealloc_tensor %sv1 : tensor diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sorted_coo.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sorted_coo.mlir index e0111f692601..7b3f9a2ce0e0 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sorted_coo.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sorted_coo.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -35,19 +35,19 @@ !Filename = !llvm.ptr #SortedCOO = #sparse_tensor.encoding<{ - map = (d0, d1) -> (d0 : compressed(nonunique), d1 : singleton) + map = (d0, d1) -> (d0 : compressed(nonunique), d1 : singleton(soa)) }> #SortedCOOPermuted = #sparse_tensor.encoding<{ - map = (d0, d1) -> (d1 : compressed(nonunique), d0 : singleton), + map = (d0, d1) -> (d1 : compressed(nonunique), d0 : singleton(soa)), }> #SortedCOO3D = #sparse_tensor.encoding<{ - map = (d0, d1, d2) -> (d0 : compressed(nonunique), d1 : singleton(nonunique), d2 : singleton) + map = (d0, d1, d2) -> (d0 : compressed(nonunique), d1 : singleton(nonunique, soa), d2 : singleton(soa)) }> #SortedCOO3DPermuted = #sparse_tensor.encoding<{ - map = (d0, d1, d2) -> (d2 : compressed(nonunique), d0 : singleton(nonunique), d1 : singleton) + map = (d0, d1, d2) -> (d2 : compressed(nonunique), d0 : singleton(nonunique, soa), d1 : singleton(soa)) }> @@ -82,29 +82,7 @@ module { return %0 : tensor } - func.func @dumpi(%arg0: memref) { - %c0 = arith.constant 0 : index - %v = vector.transfer_read %arg0[%c0], %c0: memref, vector<20xindex> - vector.print %v : vector<20xindex> - return - } - - func.func @dumpsi(%arg0: memref>) { - %c0 = arith.constant 0 : index - %v = vector.transfer_read %arg0[%c0], %c0: memref>, vector<20xindex> - vector.print %v : vector<20xindex> - return - } - - func.func @dumpf(%arg0: memref) { - %c0 = arith.constant 0 : index - %nan = arith.constant 0x0 : f64 - %v = vector.transfer_read %arg0[%c0], %nan: memref, vector<20xf64> - vector.print %v : vector<20xf64> - return - } - - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %c1 = arith.constant 1 : index @@ -125,130 +103,88 @@ module { %4 = sparse_tensor.convert %m : tensor<5x4xf64> to tensor // - // CHECK: ( 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 0, 0, 0, 1, 1, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 126, 127, 254, 1, 253, 2, 0, 1, 3, 98, 126, 127, 128, 249, 253, 255, 0, 0, 0 ) - // CHECK-NEXT: ( -1, 2, -3, 4, -5, 6, -7, 8, -9, 10, -11, 12, -13, 14, -15, 16, -17, 0, 0, 0 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 17 + // CHECK-NEXT: dim = ( 4, 256 ) + // CHECK-NEXT: lvl = ( 4, 256 ) + // CHECK-NEXT: pos[0] : ( 0, 17 + // CHECK-NEXT: crd[0] : ( 0, 0, 0, 0, 1, 1, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 + // CHECK-NEXT: crd[1] : ( 0, 126, 127, 254, 1, 253, 2, 0, 1, 3, 98, 126, 127, 128, 249, 253, 255 + // CHECK-NEXT: values : ( -1, 2, -3, 4, -5, 6, -7, 8, -9, 10, -11, 12, -13, 14, -15, 16, -17 + // CHECK-NEXT: ---- // - %p0 = sparse_tensor.positions %0 { level = 0 : index } - : tensor to memref - %i00 = sparse_tensor.coordinates %0 { level = 0 : index } - : tensor to memref> - %i01 = sparse_tensor.coordinates %0 { level = 1 : index } - : tensor to memref> - %v0 = sparse_tensor.values %0 - : tensor to memref - call @dumpi(%p0) : (memref) -> () - call @dumpsi(%i00) : (memref>) -> () - call @dumpsi(%i01) : (memref>) -> () - call @dumpf(%v0) : (memref) -> () + sparse_tensor.print %0 : tensor // - // CHECK-NEXT: ( 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 0, 1, 1, 2, 3, 98, 126, 126, 127, 127, 128, 249, 253, 253, 254, 255, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 3, 1, 3, 2, 3, 3, 0, 3, 0, 3, 3, 3, 1, 3, 0, 3, 0, 0, 0 ) - // CHECK-NEXT: ( -1, 8, -5, -9, -7, 10, -11, 2, 12, -3, -13, 14, -15, 6, 16, 4, -17, 0, 0, 0 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 17 + // CHECK-NEXT: dim = ( 4, 256 ) + // CHECK-NEXT: lvl = ( 256, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 17 + // CHECK-NEXT: crd[0] : ( 0, 0, 1, 1, 2, 3, 98, 126, 126, 127, 127, 128, 249, 253, 253, 254, 255 + // CHECK-NEXT: crd[1] : ( 0, 3, 1, 3, 2, 3, 3, 0, 3, 0, 3, 3, 3, 1, 3, 0, 3 + // CHECK-NEXT: values : ( -1, 8, -5, -9, -7, 10, -11, 2, 12, -3, -13, 14, -15, 6, 16, 4, -17 + // CHECK-NEXT: ---- // - %p1 = sparse_tensor.positions %1 { level = 0 : index } - : tensor to memref - %i10 = sparse_tensor.coordinates %1 { level = 0 : index } - : tensor to memref> - %i11 = sparse_tensor.coordinates %1 { level = 1 : index } - : tensor to memref> - %v1 = sparse_tensor.values %1 - : tensor to memref - call @dumpi(%p1) : (memref) -> () - call @dumpsi(%i10) : (memref>) -> () - call @dumpsi(%i11) : (memref>) -> () - call @dumpf(%v1) : (memref) -> () + sparse_tensor.print %1 : tensor // - // CHECK-NEXT: ( 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 0, 1, 1, 2, 2, 2, 2, 0, 0, 0, 1, 1, 1, 1, 2, 2, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 0, 1, 1, 2, 2, 2, 2, 0, 0, 0, 1, 1, 1, 1, 2, 2, 0, 0, 0 ) - // CHECK-NEXT: ( 3, 63, 11, 100, 66, 61, 13, 43, 77, 10, 46, 61, 53, 3, 75, 22, 18, 0, 0, 0 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 17 + // CHECK-NEXT: dim = ( 2, 3, 4 ) + // CHECK-NEXT: lvl = ( 2, 3, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 17 + // CHECK-NEXT: crd[0] : ( 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1 + // CHECK-NEXT: crd[1] : ( 0, 0, 1, 1, 2, 2, 2, 2, 0, 0, 0, 1, 1, 1, 1, 2, 2 + // CHECK-NEXT: crd[2] : ( 2, 3, 1, 2, 0, 1, 2, 3, 0, 2, 3, 0, 1, 2, 3, 1, 2 + // CHECK-NEXT: values : ( 3, 63, 11, 100, 66, 61, 13, 43, 77, 10, 46, 61, 53, 3, 75, 22, 18 + // CHECK-NEXT: ---- // - %p2 = sparse_tensor.positions %2 { level = 0 : index } - : tensor to memref - %i20 = sparse_tensor.coordinates %2 { level = 0 : index } - : tensor to memref> - %i21 = sparse_tensor.coordinates %2 { level = 1 : index } - : tensor to memref> - %i22 = sparse_tensor.coordinates %2 { level = 2 : index } - : tensor to memref> - %v2 = sparse_tensor.values %2 - : tensor to memref - call @dumpi(%p2) : (memref) -> () - call @dumpsi(%i20) : (memref>) -> () - call @dumpsi(%i21) : (memref>) -> () - call @dumpsi(%i21) : (memref>) -> () - call @dumpf(%v2) : (memref) -> () + sparse_tensor.print %2 : tensor // - // CHECK-NEXT: ( 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0 ) - // CHECK-NEXT: ( 66, 77, 61, 11, 61, 53, 22, 3, 100, 13, 10, 3, 18, 63, 43, 46, 75, 0, 0, 0 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 17 + // CHECK-NEXT: dim = ( 2, 3, 4 ) + // CHECK-NEXT: lvl = ( 4, 2, 3 ) + // CHECK-NEXT: pos[0] : ( 0, 17 + // CHECK-NEXT: crd[0] : ( 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3 + // CHECK-NEXT: crd[1] : ( 0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1 + // CHECK-NEXT: crd[2] : ( 2, 0, 1, 1, 2, 1, 2, 0, 1, 2, 0, 1, 2, 0, 2, 0, 1 + // CHECK-NEXT: values : ( 66, 77, 61, 11, 61, 53, 22, 3, 100, 13, 10, 3, 18, 63, 43, 46, 75 + // CHECK-NEXT: ---- // - %p3 = sparse_tensor.positions %3 { level = 0 : index } - : tensor to memref - %i30 = sparse_tensor.coordinates %3 { level = 0 : index } - : tensor to memref> - %i31 = sparse_tensor.coordinates %3 { level = 1 : index } - : tensor to memref> - %i32 = sparse_tensor.coordinates %3 { level = 2 : index } - : tensor to memref> - %v3 = sparse_tensor.values %3 - : tensor to memref - call @dumpi(%p3) : (memref) -> () - call @dumpsi(%i30) : (memref>) -> () - call @dumpsi(%i31) : (memref>) -> () - call @dumpsi(%i31) : (memref>) -> () - call @dumpf(%v3) : (memref) -> () + sparse_tensor.print %3 : tensor // - // CHECK-NEXT: ( 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 2, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 3, 0, 3, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 6, 5, 4, 3, 2, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 6 + // CHECK-NEXT: dim = ( 5, 4 ) + // CHECK-NEXT: lvl = ( 5, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 6 + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 2, 3, 4 + // CHECK-NEXT: crd[1] : ( 0, 3, 0, 3, 1, 1 + // CHECK-NEXT: values : ( 6, 5, 4, 3, 2, 11 + // CHECK-NEXT: ---- // - %p4 = sparse_tensor.positions %4 { level = 0 : index } - : tensor to memref - %i40 = sparse_tensor.coordinates %4 { level = 0 : index } - : tensor to memref> - %i41 = sparse_tensor.coordinates %4 { level = 1 : index } - : tensor to memref> - %v4 = sparse_tensor.values %4 - : tensor to memref - call @dumpi(%p4) : (memref) -> () - call @dumpsi(%i40) : (memref>) -> () - call @dumpsi(%i41) : (memref>) -> () - call @dumpf(%v4) : (memref) -> () + sparse_tensor.print %4 : tensor // And last but not least, an actual operation applied to COO. // Note that this performs the operation "in place". %5 = call @sparse_scale(%4) : (tensor) -> tensor // - // CHECK-NEXT: ( 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 1, 2, 2, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 3, 0, 3, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 12, 10, 8, 6, 4, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 6 + // CHECK-NEXT: dim = ( 5, 4 ) + // CHECK-NEXT: lvl = ( 5, 4 ) + // CHECK-NEXT: pos[0] : ( 0, 6 + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 2, 3, 4 + // CHECK-NEXT: crd[1] : ( 0, 3, 0, 3, 1, 1 + // CHECK-NEXT: values : ( 12, 10, 8, 6, 4, 22 + // CHECK-NEXT: ---- // - %p5 = sparse_tensor.positions %5 { level = 0 : index } - : tensor to memref - %i50 = sparse_tensor.coordinates %5 { level = 0 : index } - : tensor to memref> - %i51 = sparse_tensor.coordinates %5 { level = 1 : index } - : tensor to memref> - %v5 = sparse_tensor.values %5 - : tensor to memref - call @dumpi(%p5) : (memref) -> () - call @dumpsi(%i50) : (memref>) -> () - call @dumpsi(%i51) : (memref>) -> () - call @dumpf(%v5) : (memref) -> () + sparse_tensor.print %5 : tensor // Release the resources. bufferization.dealloc_tensor %0 : tensor diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_spmm.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_spmm.mlir index 573b1a2aac25..ca8bcd7744c8 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_spmm.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_spmm.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -76,7 +76,7 @@ module { // // Main driver that reads matrix from file and calls the sparse kernel. // - func.func @entry() { + func.func @main() { %i0 = arith.constant 0.0 : f64 %c0 = arith.constant 0 : index %c1 = arith.constant 1 : index diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_storage.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_storage.mlir index 8ca95f2139e4..2ee189de7906 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_storage.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_storage.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -72,7 +72,7 @@ module { // are typically not concerned with such details, but the test ensures // everything is working "under the hood". // - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %c1 = arith.constant 1 : index %d0 = arith.constant 0.0 : f64 @@ -107,166 +107,103 @@ module { // // Inspect storage scheme of Dense. // - // CHECK: ( 1, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, - // CHECK-SAME: 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, - // CHECK-SAME: 0, 0, 0, 0, 6, 0, 0, 0, 0, 7, 8, 0, 0, 0, 0, 9, - // CHECK-SAME: 0, 0, 10, 0, 0, 0, 11, 12, 0, 13, 14, 0, 0, 0, 15, 16, - // CHECK-SAME: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 80 + // CHECK-NEXT: dim = ( 10, 8 ) + // CHECK-NEXT: lvl = ( 10, 8 ) + // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 7, 8, 0, 0, 0, 0, 9, 0, 0, 10, 0, 0, 0, 11, 12, 0, 13, 14, 0, 0, 0, 15, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0 + // CHECK-NEXT: ---- // - %5 = sparse_tensor.values %0 : tensor<10x8xf64, #Dense> to memref - %6 = vector.transfer_read %5[%c0], %d0: memref, vector<80xf64> - vector.print %6 : vector<80xf64> + sparse_tensor.print %0 : tensor<10x8xf64, #Dense> // // Inspect storage scheme of CSR. // - // positions(1) - // indices(1) - // values // - // CHECK: ( 0, 3, 3, 4, 5, 6, 9, 12, 16, 16, 17 ) - // CHECK: ( 0, 2, 7, 2, 3, 4, 1, 2, 7, 2, 6, 7, 1, 2, 6, 7, 6 ) - // CHECK: ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 17 + // CHECK-NEXT: dim = ( 10, 8 ) + // CHECK-NEXT: lvl = ( 10, 8 ) + // CHECK-NEXT: pos[1] : ( 0, 3, 3, 4, 5, 6, 9, 12, 16, 16, 17 + // CHECK-NEXT: crd[1] : ( 0, 2, 7, 2, 3, 4, 1, 2, 7, 2, 6, 7, 1, 2, 6, 7, 6 + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 + // CHECK-NEXT: ---- // - %7 = sparse_tensor.positions %1 { level = 1 : index } : tensor<10x8xf64, #CSR> to memref - %8 = vector.transfer_read %7[%c0], %c0: memref, vector<11xindex> - vector.print %8 : vector<11xindex> - %9 = sparse_tensor.coordinates %1 { level = 1 : index } : tensor<10x8xf64, #CSR> to memref - %10 = vector.transfer_read %9[%c0], %c0: memref, vector<17xindex> - vector.print %10 : vector<17xindex> - %11 = sparse_tensor.values %1 : tensor<10x8xf64, #CSR> to memref - %12 = vector.transfer_read %11[%c0], %d0: memref, vector<17xf64> - vector.print %12 : vector<17xf64> + sparse_tensor.print %1 : tensor<10x8xf64, #CSR> // // Inspect storage scheme of DCSR. // - // positions(0) - // indices(0) - // positions(1) - // indices(1) - // values + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 17 + // CHECK-NEXT: dim = ( 10, 8 ) + // CHECK-NEXT: lvl = ( 10, 8 ) + // CHECK-NEXT: pos[0] : ( 0, 8 + // CHECK-NEXT: crd[0] : ( 0, 2, 3, 4, 5, 6, 7, 9 + // CHECK-NEXT: pos[1] : ( 0, 3, 4, 5, 6, 9, 12, 16, 17 + // CHECK-NEXT: crd[1] : ( 0, 2, 7, 2, 3, 4, 1, 2, 7, 2, 6, 7, 1, 2, 6, 7, 6 + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 + // CHECK-NEXT: ---- // - // CHECK: ( 0, 8 ) - // CHECK: ( 0, 2, 3, 4, 5, 6, 7, 9 ) - // CHECK: ( 0, 3, 4, 5, 6, 9, 12, 16, 17 ) - // CHECK: ( 0, 2, 7, 2, 3, 4, 1, 2, 7, 2, 6, 7, 1, 2, 6, 7, 6 ) - // CHECK: ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 ) - // - %13 = sparse_tensor.positions %2 { level = 0 : index } : tensor<10x8xf64, #DCSR> to memref - %14 = vector.transfer_read %13[%c0], %c0: memref, vector<2xindex> - vector.print %14 : vector<2xindex> - %15 = sparse_tensor.coordinates %2 { level = 0 : index } : tensor<10x8xf64, #DCSR> to memref - %16 = vector.transfer_read %15[%c0], %c0: memref, vector<8xindex> - vector.print %16 : vector<8xindex> - %17 = sparse_tensor.positions %2 { level = 1 : index } : tensor<10x8xf64, #DCSR> to memref - %18 = vector.transfer_read %17[%c0], %c0: memref, vector<9xindex> - vector.print %18 : vector<9xindex> - %19 = sparse_tensor.coordinates %2 { level = 1 : index } : tensor<10x8xf64, #DCSR> to memref - %20 = vector.transfer_read %19[%c0], %c0: memref, vector<17xindex> - vector.print %20 : vector<17xindex> - %21 = sparse_tensor.values %2 : tensor<10x8xf64, #DCSR> to memref - %22 = vector.transfer_read %21[%c0], %d0: memref, vector<17xf64> - vector.print %22 : vector<17xf64> + sparse_tensor.print %2 : tensor<10x8xf64, #DCSR> // // Inspect storage scheme of CSC. // - // positions(1) - // indices(1) - // values - // - // CHECK: ( 0, 1, 3, 8, 9, 10, 10, 13, 17 ) - // CHECK: ( 0, 5, 7, 0, 2, 5, 6, 7, 3, 4, 6, 7, 9, 0, 5, 6, 7 ) - // CHECK: ( 1, 7, 13, 2, 4, 8, 10, 14, 5, 6, 11, 15, 17, 3, 9, 12, 16 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 17 + // CHECK-NEXT: dim = ( 10, 8 ) + // CHECK-NEXT: lvl = ( 8, 10 ) + // CHECK-NEXT: pos[1] : ( 0, 1, 3, 8, 9, 10, 10, 13, 17 + // CHECK-NEXT: crd[1] : ( 0, 5, 7, 0, 2, 5, 6, 7, 3, 4, 6, 7, 9, 0, 5, 6, 7 + // CHECK-NEXT: values : ( 1, 7, 13, 2, 4, 8, 10, 14, 5, 6, 11, 15, 17, 3, 9, 12, 16 + // CHECK-NEXT: ---- // - %23 = sparse_tensor.positions %3 { level = 1 : index } : tensor<10x8xf64, #CSC> to memref - %24 = vector.transfer_read %23[%c0], %c0: memref, vector<9xindex> - vector.print %24 : vector<9xindex> - %25 = sparse_tensor.coordinates %3 { level = 1 : index } : tensor<10x8xf64, #CSC> to memref - %26 = vector.transfer_read %25[%c0], %c0: memref, vector<17xindex> - vector.print %26 : vector<17xindex> - %27 = sparse_tensor.values %3 : tensor<10x8xf64, #CSC> to memref - %28 = vector.transfer_read %27[%c0], %d0: memref, vector<17xf64> - vector.print %28 : vector<17xf64> + sparse_tensor.print %3 : tensor<10x8xf64, #CSC> // // Inspect storage scheme of DCSC. // - // positions(0) - // indices(0) - // positions(1) - // indices(1) - // values - // - // CHECK: ( 0, 7 ) - // CHECK: ( 0, 1, 2, 3, 4, 6, 7 ) - // CHECK: ( 0, 1, 3, 8, 9, 10, 13, 17 ) - // CHECK: ( 0, 5, 7, 0, 2, 5, 6, 7, 3, 4, 6, 7, 9, 0, 5, 6, 7 ) - // CHECK: ( 1, 7, 13, 2, 4, 8, 10, 14, 5, 6, 11, 15, 17, 3, 9, 12, 16 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 17 + // CHECK-NEXT: dim = ( 10, 8 ) + // CHECK-NEXT: lvl = ( 8, 10 ) + // CHECK-NEXT: pos[0] : ( 0, 7 + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 6, 7 + // CHECK-NEXT: pos[1] : ( 0, 1, 3, 8, 9, 10, 13, 17 + // CHECK-NEXT: crd[1] : ( 0, 5, 7, 0, 2, 5, 6, 7, 3, 4, 6, 7, 9, 0, 5, 6, 7 + // CHECK-NEXT: values : ( 1, 7, 13, 2, 4, 8, 10, 14, 5, 6, 11, 15, 17, 3, 9, 12, 16 + // CHECK-NEXT: ---- // - %29 = sparse_tensor.positions %4 { level = 0 : index } : tensor<10x8xf64, #DCSC> to memref - %30 = vector.transfer_read %29[%c0], %c0: memref, vector<2xindex> - vector.print %30 : vector<2xindex> - %31 = sparse_tensor.coordinates %4 { level = 0 : index } : tensor<10x8xf64, #DCSC> to memref - %32 = vector.transfer_read %31[%c0], %c0: memref, vector<7xindex> - vector.print %32 : vector<7xindex> - %33 = sparse_tensor.positions %4 { level = 1 : index } : tensor<10x8xf64, #DCSC> to memref - %34 = vector.transfer_read %33[%c0], %c0: memref, vector<8xindex> - vector.print %34 : vector<8xindex> - %35 = sparse_tensor.coordinates %4 { level = 1 : index } : tensor<10x8xf64, #DCSC> to memref - %36 = vector.transfer_read %35[%c0], %c0: memref, vector<17xindex> - vector.print %36 : vector<17xindex> - %37 = sparse_tensor.values %4 : tensor<10x8xf64, #DCSC> to memref - %38 = vector.transfer_read %37[%c0], %d0: memref, vector<17xf64> - vector.print %38 : vector<17xf64> + sparse_tensor.print %4 : tensor<10x8xf64, #DCSC> // // Inspect storage scheme of BlockRow. // - // positions(0) - // indices(0) - // values + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 64 + // CHECK-NEXT: dim = ( 10, 8 ) + // CHECK-NEXT: lvl = ( 10, 8 ) + // CHECK-NEXT: pos[0] : ( 0, 8 + // CHECK-NEXT: crd[0] : ( 0, 2, 3, 4, 5, 6, 7, 9 + // CHECK-NEXT: values : ( 1, 0, 2, 0, 0, 0, 0, 3, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 7, 8, 0, 0, 0, 0, 9, 0, 0, 10, 0, 0, 0, 11, 12, 0, 13, 14, 0, 0, 0, 15, 16, 0, 0, 0, 0, 0, 0, 17, 0 + // CHECK-NEXT: ---- // - // CHECK: ( 0, 8 ) - // CHECK: ( 0, 2, 3, 4, 5, 6, 7, 9 ) - // CHECK: ( 1, 0, 2, 0, 0, 0, 0, 3, 0, 0, 4, 0, 0, 0, 0, 0, - // CHECK-SAME: 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, - // CHECK-SAME: 0, 7, 8, 0, 0, 0, 0, 9, 0, 0, 10, 0, 0, 0, 11, 12, - // CHECK-SAME: 0, 13, 14, 0, 0, 0, 15, 16, 0, 0, 0, 0, 0, 0, 17, 0 ) - // - %39 = sparse_tensor.positions %x { level = 0 : index } : tensor<10x8xf64, #BlockRow> to memref - %40 = vector.transfer_read %39[%c0], %c0: memref, vector<2xindex> - vector.print %40 : vector<2xindex> - %41 = sparse_tensor.coordinates %x { level = 0 : index } : tensor<10x8xf64, #BlockRow> to memref - %42 = vector.transfer_read %41[%c0], %c0: memref, vector<8xindex> - vector.print %42 : vector<8xindex> - %43 = sparse_tensor.values %x : tensor<10x8xf64, #BlockRow> to memref - %44 = vector.transfer_read %43[%c0], %d0: memref, vector<64xf64> - vector.print %44 : vector<64xf64> + sparse_tensor.print %x : tensor<10x8xf64, #BlockRow> // // Inspect storage scheme of BlockCol. // - // positions(0) - // indices(0) - // values - // - // CHECK: ( 0, 7 ) - // CHECK: ( 0, 1, 2, 3, 4, 6, 7 ) - // CHECK: ( 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 13, 0, 0, 2, 0, 4, 0, - // CHECK-SAME: 0, 8, 10, 14, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, - // CHECK-SAME: 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 15, 0, 17, 3, 0, 0, 0, 0, 9, 12, 16, 0, 0 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 70 + // CHECK-NEXT: dim = ( 10, 8 ) + // CHECK-NEXT: lvl = ( 8, 10 ) + // CHECK-NEXT: pos[0] : ( 0, 7 + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 6, 7 + // CHECK-NEXT: values : ( 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 13, 0, 0, 2, 0, 4, 0, 0, 8, 10, 14, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 15, 0, 17, 3, 0, 0, 0, 0, 9, 12, 16, 0, 0 + // CHECK-NEXT: ---- // - %45 = sparse_tensor.positions %y { level = 0 : index } : tensor<10x8xf64, #BlockCol> to memref - %46 = vector.transfer_read %45[%c0], %c0: memref, vector<2xindex> - vector.print %46 : vector<2xindex> - %47 = sparse_tensor.coordinates %y { level = 0 : index } : tensor<10x8xf64, #BlockCol> to memref - %48 = vector.transfer_read %47[%c0], %c0: memref, vector<7xindex> - vector.print %48 : vector<7xindex> - %49 = sparse_tensor.values %y : tensor<10x8xf64, #BlockCol> to memref - %50 = vector.transfer_read %49[%c0], %d0: memref, vector<70xf64> - vector.print %50 : vector<70xf64> + sparse_tensor.print %y : tensor<10x8xf64, #BlockCol> // Release the resources. bufferization.dealloc_tensor %0 : tensor<10x8xf64, #Dense> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_strided_conv_2d_nhwc_hwcf.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_strided_conv_2d_nhwc_hwcf.mlir index 5184083f665d..2b2b8536fe39 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_strided_conv_2d_nhwc_hwcf.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_strided_conv_2d_nhwc_hwcf.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -78,7 +78,7 @@ func.func @conv_2d_nhwc_hwcf_dual_CDCC(%arg0: tensor, %arg1: } -func.func @entry() { +func.func @main() { %c0 = arith.constant 0 : index %c1 = arith.constant 1 : index %c3 = arith.constant 3 : index diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum.mlir index e6cbff231024..d1c58bfb6d59 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -75,7 +75,7 @@ module { // // Main driver that reads matrix from file and calls the sparse kernel. // - func.func @entry() { + func.func @main() { %d0 = arith.constant 0.0 : f64 %c0 = arith.constant 0 : index diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_bf16.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_bf16.mlir index ee00a19a4123..16a8b50ab08e 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_bf16.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_bf16.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -67,7 +67,7 @@ module { // // Main driver that reads matrix from file and calls the sparse kernel. // - func.func @entry() { + func.func @main() { // Setup input sparse matrix from compressed constant. %d = arith.constant dense <[ [ 1.1, 1.2, 0.0, 1.4 ], diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_c32.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_c32.mlir index 5fdf636ef123..f95c163a57c1 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_c32.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_c32.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -75,7 +75,7 @@ module { // // Main driver that reads matrix from file and calls the sparse kernel. // - func.func @entry() { + func.func @main() { //%d0 = arith.constant 0.0 : complex %d0 = complex.constant [0.0 : f64, 0.0 : f64] : complex %c0 = arith.constant 0 : index diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_f16.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_f16.mlir index 6a3469522949..30be587c8f61 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_f16.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_f16.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -66,7 +66,7 @@ module { // // Main driver that reads matrix from file and calls the sparse kernel. // - func.func @entry() { + func.func @main() { // Setup input sparse matrix from compressed constant. %d = arith.constant dense <[ [ 1.1, 1.2, 0.0, 1.4 ], diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tanh.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tanh.mlir index 336044d56600..29bc744c9920 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tanh.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tanh.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -56,28 +56,8 @@ module { return %0 : tensor } - // Dumps a sparse vector of type f64. - func.func @dump_vec_f64(%arg0: tensor) { - // Dump the values array to verify only sparse contents are stored. - %c0 = arith.constant 0 : index - %d0 = arith.constant -1.0 : f64 - %n = sparse_tensor.number_of_entries %arg0: tensor - vector.print %n : index - %0 = sparse_tensor.values %arg0 - : tensor to memref - %1 = vector.transfer_read %0[%c0], %d0: memref, vector<9xf64> - vector.print %1 : vector<9xf64> - // Dump the dense vector to verify structure is correct. - %dv = sparse_tensor.convert %arg0 - : tensor to tensor - %3 = vector.transfer_read %dv[%c0], %d0: tensor, vector<32xf64> - vector.print %3 : vector<32xf64> - bufferization.dealloc_tensor %dv : tensor - return - } - // Driver method to call and verify vector kernels. - func.func @entry() { + func.func @main() { // Setup sparse vector. %v1 = arith.constant sparse< [ [0], [3], [11], [17], [20], [21], [28], [29], [31] ], @@ -93,11 +73,16 @@ module { // // Verify the results (within some precision). // - // CHECK: 9 - // CHECK-NEXT: {{( -0.761[0-9]*, 0.761[0-9]*, 0.96[0-9]*, 0.99[0-9]*, 0.99[0-9]*, 0.99[0-9]*, 0.99[0-9]*, 0.99[0-9]*, 1 )}} - // CHECK-NEXT: {{( -0.761[0-9]*, 0, 0, 0.761[0-9]*, 0, 0, 0, 0, 0, 0, 0, 0.96[0-9]*, 0, 0, 0, 0, 0, 0.99[0-9]*, 0, 0, 0.99[0-9]*, 0.99[0-9]*, 0, 0, 0, 0, 0, 0, 0.99[0-9]*, 0.99[0-9]*, 0, 1 )}} + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 9 + // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31 + // CHECK-NEXT: values : ({{ -0.761[0-9]*, 0.761[0-9]*, 0.96[0-9]*, 0.99[0-9]*, 0.99[0-9]*, 0.99[0-9]*, 0.99[0-9]*, 0.99[0-9]*, 1}} + // CHECK-NEXT: ---- // - call @dump_vec_f64(%0) : (tensor) -> () + sparse_tensor.print %0 : tensor // Release the resources. bufferization.dealloc_tensor %sv1 : tensor diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_mul.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_mul.mlir index d53b03025f55..67155201c584 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_mul.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_mul.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -67,7 +67,7 @@ module { } // Driver method to call and verify tensor multiplication kernel. - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %default_val = arith.constant -1.0 : f64 @@ -103,30 +103,28 @@ module { %0 = call @tensor_mul(%sta, %stb) : (tensor, tensor) -> tensor - // Verify results // - // CHECK: 4 - // CHECK-NEXT: ( 2.4, 3.5, 2, 8 ) - // CHECK-NEXT: ( ( ( 0, 0, 0, 0, 0 ), ( 0, 0, 0, 0, 0 ), ( 2.4, 0, 3.5, 0, 0 ) ), - // CHECK-SAME: ( ( 0, 0, 0, 0, 0 ), ( 0, 0, 0, 0, 0 ), ( 0, 0, 0, 0, 0 ) ), - // CHECK-SAME: ( ( 2, 0, 0, 0, 0 ), ( 0, 0, 0, 0, 0 ), ( 0, 0, 8, 0, 0 ) ) ) + // Verify results. // - %n = sparse_tensor.number_of_entries %0 : tensor - vector.print %n : index - %m1 = sparse_tensor.values %0 : tensor to memref - %v1 = vector.transfer_read %m1[%c0], %default_val: memref, vector<4xf64> - vector.print %v1 : vector<4xf64> - - // Print %0 in dense form. - %dt = sparse_tensor.convert %0 : tensor to tensor - %v2 = vector.transfer_read %dt[%c0, %c0, %c0], %default_val: tensor, vector<3x3x5xf64> - vector.print %v2 : vector<3x3x5xf64> + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 4 + // CHECK-NEXT: dim = ( 3, 3, 5 ) + // CHECK-NEXT: lvl = ( 3, 3, 5 ) + // CHECK-NEXT: pos[0] : ( 0, 2 + // CHECK-NEXT: crd[0] : ( 0, 2 + // CHECK-NEXT: pos[1] : ( 0, 1, 3 + // CHECK-NEXT: crd[1] : ( 2, 0, 2 + // CHECK-NEXT: pos[2] : ( 0, 2, 3, 4 + // CHECK-NEXT: crd[2] : ( 0, 2, 0, 2 + // CHECK-NEXT: values : ( 2.4, 3.5, 2, 8 + // CHECK-NEXT: ---- + // + sparse_tensor.print %0 : tensor // Release the resources. bufferization.dealloc_tensor %sta : tensor bufferization.dealloc_tensor %stb : tensor bufferization.dealloc_tensor %0 : tensor - bufferization.dealloc_tensor %dt : tensor return } diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_ops.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_ops.mlir index 6ef6b393019a..356808ebee3f 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_ops.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_ops.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -67,7 +67,7 @@ module { } // Driver method to call and verify tensor kernel. - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %d1 = arith.constant -1.0 : f64 @@ -90,22 +90,34 @@ module { // Call sparse vector kernels. %0 = call @tensor_scale(%st) : (tensor) -> tensor + // // Sanity check on stored values. // - // CHECK: 5 - // CHECK-NEXT: ( 1, 2, 3, 4, 5 ) - // CHECK-NEXT: 24 - // CHECK-NEXT: ( 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6, 8, 0, 0, 0, 0, 10 ) - %m1 = sparse_tensor.values %st : tensor to memref - %m2 = sparse_tensor.values %0 : tensor to memref - %n1 = sparse_tensor.number_of_entries %st : tensor - %n2 = sparse_tensor.number_of_entries %0 : tensor - %v1 = vector.transfer_read %m1[%c0], %d1: memref, vector<5xf64> - %v2 = vector.transfer_read %m2[%c0], %d1: memref, vector<24xf64> - vector.print %n1 : index - vector.print %v1 : vector<5xf64> - vector.print %n2 : index - vector.print %v2 : vector<24xf64> + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 5 + // CHECK-NEXT: dim = ( 3, 4, 8 ) + // CHECK-NEXT: lvl = ( 3, 4, 8 ) + // CHECK-NEXT: pos[0] : ( 0, 2 + // CHECK-NEXT: crd[0] : ( 0, 2 + // CHECK-NEXT: pos[1] : ( 0, 2, 3 + // CHECK-NEXT: crd[1] : ( 0, 3, 2 + // CHECK-NEXT: pos[2] : ( 0, 1, 2, 5 + // CHECK-NEXT: crd[2] : ( 0, 7, 1, 2, 7 + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 24 + // CHECK-NEXT: dim = ( 3, 4, 8 ) + // CHECK-NEXT: lvl = ( 3, 4, 8 ) + // CHECK-NEXT: pos[0] : ( 0, 2 + // CHECK-NEXT: crd[0] : ( 0, 2 + // CHECK-NEXT: pos[1] : ( 0, 2, 3 + // CHECK-NEXT: crd[1] : ( 0, 3, 2 + // CHECK-NEXT: values : ( 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6, 8, 0, 0, 0, 0, 10 + // CHECK-NEXT: ---- + // + sparse_tensor.print %st : tensor + sparse_tensor.print %0 : tensor // Release the resources. bufferization.dealloc_tensor %st : tensor diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose.mlir index 185f6161493e..549c2082fcb3 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -92,7 +92,7 @@ module { // // Main driver. // - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %c1 = arith.constant 1 : index %c4 = arith.constant 4 : index @@ -115,26 +115,29 @@ module { // // Verify result. // - // CHECK: ( 1.1, 0, 3.1 ) - // CHECK-NEXT: ( 1.2, 0, 0 ) - // CHECK-NEXT: ( 0, 0, 3.3 ) - // CHECK-NEXT: ( 1.4, 0, 3.4 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 6 + // CHECK-NEXT: dim = ( 4, 3 ) + // CHECK-NEXT: lvl = ( 4, 3 ) + // CHECK-NEXT: pos[0] : ( 0, 4 + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 + // CHECK-NEXT: pos[1] : ( 0, 2, 3, 4, 6 + // CHECK-NEXT: crd[1] : ( 0, 2, 0, 2, 0, 2 + // CHECK-NEXT: values : ( 1.1, 3.1, 1.2, 3.3, 1.4, 3.4 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 6 + // CHECK-NEXT: dim = ( 4, 3 ) + // CHECK-NEXT: lvl = ( 4, 3 ) + // CHECK-NEXT: pos[0] : ( 0, 4 + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 + // CHECK-NEXT: pos[1] : ( 0, 2, 3, 4, 6 + // CHECK-NEXT: crd[1] : ( 0, 2, 0, 2, 0, 2 + // CHECK-NEXT: values : ( 1.1, 3.1, 1.2, 3.3, 1.4, 3.4 + // CHECK-NEXT: ---- // - // CHECK-NEXT: ( 1.1, 0, 3.1 ) - // CHECK-NEXT: ( 1.2, 0, 0 ) - // CHECK-NEXT: ( 0, 0, 3.3 ) - // CHECK-NEXT: ( 1.4, 0, 3.4 ) - // - %x = sparse_tensor.convert %0 : tensor<4x3xf64, #DCSR> to tensor<4x3xf64> - scf.for %i = %c0 to %c4 step %c1 { - %v1 = vector.transfer_read %x[%i, %c0], %du: tensor<4x3xf64>, vector<3xf64> - vector.print %v1 : vector<3xf64> - } - %y = sparse_tensor.convert %1 : tensor<4x3xf64, #DCSR> to tensor<4x3xf64> - scf.for %i = %c0 to %c4 step %c1 { - %v2 = vector.transfer_read %y[%i, %c0], %du: tensor<4x3xf64>, vector<3xf64> - vector.print %v2 : vector<3xf64> - } + sparse_tensor.print %0 : tensor<4x3xf64, #DCSR> + sparse_tensor.print %1 : tensor<4x3xf64, #DCSR> // Release resources. bufferization.dealloc_tensor %a : tensor<3x4xf64, #DCSR> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose_coo.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose_coo.mlir index dba897334830..cc6f6a068746 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose_coo.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose_coo.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -31,7 +31,7 @@ // RUN: %if mlir_arm_sve_tests %{ %{compile_sve} | %{run_sve} | FileCheck %s %} #SortedCOO = #sparse_tensor.encoding<{ - map = (d0, d1) -> (d0 : compressed(nonunique), d1 : singleton) + map = (d0, d1) -> (d0 : compressed(nonunique), d1 : singleton(soa)) }> module { @@ -52,7 +52,7 @@ module { return %1 : tensor<5x10xf32, #SortedCOO> } - func.func @entry() { + func.func @main() { %f0 = arith.constant 0.0 : f32 %c0 = arith.constant 0 : index %c1 = arith.constant 1 : index @@ -79,17 +79,27 @@ module { // // Verify original and transposed sorted COO. // - // CHECK: ( 10, 20, 30, 40, 50, 11, 21, 31, 41, 51, 12, 22, 32, 42, 52, 13, 23, 33, 43, 53, 14, 24, 34, 44, 54, 15, 25, 35, 45, 55, 16, 26, 36, 46, 56, 17, 27, 37, 47, 57, 18, 28, 38, 48, 58, 19, 29, 39, 49, 59 ) - // CHECK-NEXT: ( 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 50 + // CHECK-NEXT: dim = ( 10, 5 ) + // CHECK-NEXT: lvl = ( 10, 5 ) + // CHECK-NEXT: pos[0] : ( 0, 50 + // CHECK-NEXT: crd[0] : ( 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9 + // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4 + // CHECK-NEXT: values : ( 10, 20, 30, 40, 50, 11, 21, 31, 41, 51, 12, 22, 32, 42, 52, 13, 23, 33, 43, 53, 14, 24, 34, 44, 54, 15, 25, 35, 45, 55, 16, 26, 36, 46, 56, 17, 27, 37, 47, 57, 18, 28, 38, 48, 58, 19, 29, 39, 49, 59 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 50 + // CHECK-NEXT: dim = ( 5, 10 ) + // CHECK-NEXT: lvl = ( 5, 10 ) + // CHECK-NEXT: pos[0] : ( 0, 50 + // CHECK-NEXT: crd[0] : ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 + // CHECK-NEXT: crd[1] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 + // CHECK-NEXT: values : ( 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59 + // CHECK-NEXT: ---- // - %va = sparse_tensor.values %SA - : tensor<10x5xf32, #SortedCOO> to memref - %vat = sparse_tensor.values %SAT - : tensor<5x10xf32, #SortedCOO> to memref - %v1 = vector.transfer_read %va[%c0], %f0 : memref, vector<50xf32> - %v2 = vector.transfer_read %vat[%c0], %f0 : memref, vector<50xf32> - vector.print %v1 : vector<50xf32> - vector.print %v2 : vector<50xf32> + sparse_tensor.print %SA : tensor<10x5xf32, #SortedCOO> + sparse_tensor.print %SAT : tensor<5x10xf32, #SortedCOO> // Release resources. bufferization.dealloc_tensor %SA : tensor<10x5xf32, #SortedCOO> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_unary.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_unary.mlir index e03f99253b78..3da1e35818cf 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_unary.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_unary.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -204,54 +204,8 @@ module { return %0 : tensor } - // Dumps a sparse vector of type f64. - func.func @dump_vec_f64(%arg0: tensor) { - // Dump the values array to verify only sparse contents are stored. - %c0 = arith.constant 0 : index - %d0 = arith.constant 0.0 : f64 - %0 = sparse_tensor.values %arg0 : tensor to memref - %1 = vector.transfer_read %0[%c0], %d0: memref, vector<32xf64> - vector.print %1 : vector<32xf64> - // Dump the dense vector to verify structure is correct. - %dv = sparse_tensor.convert %arg0 : tensor to tensor - %3 = vector.transfer_read %dv[%c0], %d0: tensor, vector<32xf64> - vector.print %3 : vector<32xf64> - bufferization.dealloc_tensor %dv : tensor - return - } - - // Dumps a sparse vector of type i32. - func.func @dump_vec_i32(%arg0: tensor) { - // Dump the values array to verify only sparse contents are stored. - %c0 = arith.constant 0 : index - %d0 = arith.constant 0 : i32 - %0 = sparse_tensor.values %arg0 : tensor to memref - %1 = vector.transfer_read %0[%c0], %d0: memref, vector<24xi32> - vector.print %1 : vector<24xi32> - // Dump the dense vector to verify structure is correct. - %dv = sparse_tensor.convert %arg0 : tensor to tensor - %3 = vector.transfer_read %dv[%c0], %d0: tensor, vector<32xi32> - vector.print %3 : vector<32xi32> - bufferization.dealloc_tensor %dv : tensor - return - } - - // Dump a sparse matrix. - func.func @dump_mat(%arg0: tensor) { - %c0 = arith.constant 0 : index - %d0 = arith.constant 0.0 : f64 - %0 = sparse_tensor.values %arg0 : tensor to memref - %1 = vector.transfer_read %0[%c0], %d0: memref, vector<16xf64> - vector.print %1 : vector<16xf64> - %dm = sparse_tensor.convert %arg0 : tensor to tensor - %3 = vector.transfer_read %dm[%c0, %c0], %d0: tensor, vector<4x8xf64> - vector.print %3 : vector<4x8xf64> - bufferization.dealloc_tensor %dm : tensor - return - } - // Driver method to call and verify vector kernels. - func.func @entry() { + func.func @main() { %cmu = arith.constant -99 : i32 %c0 = arith.constant 0 : index @@ -289,26 +243,66 @@ module { // // Verify the results. // - // CHECK: ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0, 7, 8, 0, 9 ) - // CHECK-NEXT: ( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0 ) - // CHECK-NEXT: ( 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0 ) - // CHECK-NEXT: ( -1, 1, 1, -2, 1, 1, 1, 1, 1, 1, 1, -3, 1, 1, 1, 1, 1, -4, 1, 1, -5, -6, 1, 1, 1, 1, 1, 1, -7, -8, 1, -9 ) - // CHECK-NEXT: ( -1, 1, 1, -2, 1, 1, 1, 1, 1, 1, 1, -3, 1, 1, 1, 1, 1, -4, 1, 1, -5, -6, 1, 1, 1, 1, 1, 1, -7, -8, 1, -9 ) - // CHECK-NEXT: ( 0, 6, 33, 68, 100, 126, 196, 232, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 68, 0, 0, 100, 126, 0, 0, 0, 0, 0, 0, 196, 232, 0, 279 ) - // CHECK-NEXT: ( 3, 3, 3, 4, 5, 6, 7, 7, 7, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( ( 3, 3, 0, 0, 0, 0, 0, 0 ), ( 0, 0, 0, 0, 0, 0, 0, 3 ), ( 0, 0, 4, 0, 5, 0, 0, 6 ), ( 7, 0, 7, 7, 0, 0, 0, 0 ) ) - // CHECK-NEXT: ( 99, 99, 99, 99, 5, 6, 99, 99, 99, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( ( 99, 99, 0, 0, 0, 0, 0, 0 ), ( 0, 0, 0, 0, 0, 0, 0, 99 ), ( 0, 0, 99, 0, 5, 0, 0, 6 ), ( 99, 0, 99, 99, 0, 0, 0, 0 ) ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 9 + // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31 + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 23 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 23 + // CHECK-NEXT: crd[0] : ( 1, 2, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 18, 19, 22, 23, 24, 25, 26, 27, 30 + // CHECK-NEXT: values : ( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 32 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 32 + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + // CHECK-NEXT: values : ( -1, 1, 1, -2, 1, 1, 1, 1, 1, 1, 1, -3, 1, 1, 1, 1, 1, -4, 1, 1, -5, -6, 1, 1, 1, 1, 1, 1, -7, -8, 1, -9 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 9 + // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31 + // CHECK-NEXT: values : ( 0, 6, 33, 68, 100, 126, 196, 232, 279 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 4, 8 ) + // CHECK-NEXT: pos[0] : ( 0, 4 + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 + // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9 + // CHECK-NEXT: crd[1] : ( 0, 1, 7, 2, 4, 7, 0, 2, 3 + // CHECK-NEXT: values : ( 3, 3, 3, 4, 5, 6, 7, 7, 7 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 4, 8 ) + // CHECK-NEXT: lvl = ( 4, 8 ) + // CHECK-NEXT: pos[0] : ( 0, 4 + // CHECK-NEXT: crd[0] : ( 0, 1, 2, 3 + // CHECK-NEXT: pos[1] : ( 0, 2, 3, 6, 9 + // CHECK-NEXT: crd[1] : ( 0, 1, 7, 2, 4, 7, 0, 2, 3 + // CHECK-NEXT: values : ( 99, 99, 99, 99, 5, 6, 99, 99, 99 + // CHECK-NEXT: ---- // CHECK-NEXT: ( 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0 ) // - call @dump_vec_f64(%sv1) : (tensor) -> () - call @dump_vec_i32(%0) : (tensor) -> () - call @dump_vec_f64(%1) : (tensor) -> () - call @dump_vec_f64(%2) : (tensor) -> () - call @dump_mat(%3) : (tensor) -> () - call @dump_mat(%4) : (tensor) -> () + sparse_tensor.print %sv1 : tensor + sparse_tensor.print %0 : tensor + sparse_tensor.print %1 : tensor + sparse_tensor.print %2 : tensor + sparse_tensor.print %3 : tensor + sparse_tensor.print %4 : tensor %v = vector.transfer_read %5[%c0], %cmu: tensor, vector<32xi32> vector.print %v : vector<32xi32> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_vector_ops.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_vector_ops.mlir index d9ca2dca8534..553323331641 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_vector_ops.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_vector_ops.mlir @@ -10,7 +10,7 @@ // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils -// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run_opts} = -e main -entry-point-result=void // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} // DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} // @@ -162,24 +162,8 @@ module { return %0 : tensor } - // Dumps a sparse vector. - func.func @dump(%arg0: tensor) { - // Dump the values array to verify only sparse contents are stored. - %c0 = arith.constant 0 : index - %d0 = arith.constant 0.0 : f64 - %0 = sparse_tensor.values %arg0 : tensor to memref - %1 = vector.transfer_read %0[%c0], %d0: memref, vector<16xf64> - vector.print %1 : vector<16xf64> - // Dump the dense vector to verify structure is correct. - %dv = sparse_tensor.convert %arg0 : tensor to tensor - %2 = vector.transfer_read %dv[%c0], %d0: tensor, vector<32xf64> - vector.print %2 : vector<32xf64> - bufferization.dealloc_tensor %dv : tensor - return - } - // Driver method to call and verify vector kernels. - func.func @entry() { + func.func @main() { %c0 = arith.constant 0 : index %d1 = arith.constant 1.1 : f64 @@ -221,31 +205,69 @@ module { // // Verify the results. // - // CHECK: ( 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0, 7, 8, 0, 9 ) - // CHECK-NEXT: ( 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 11, 0, 12, 13, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 15, 0, 16, 0, 0, 17, 0, 0, 0, 0, 0, 0, 18, 19, 0, 20 ) - // CHECK-NEXT: ( 2, 4, 6, 8, 10, 12, 14, 16, 18, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 8, 0, 0, 10, 12, 0, 0, 0, 0, 0, 0, 14, 16, 0, 18 ) - // CHECK-NEXT: ( 2, 4, 6, 8, 10, 12, 14, 16, 18, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 8, 0, 0, 10, 12, 0, 0, 0, 0, 0, 0, 14, 16, 0, 18 ) - // CHECK-NEXT: ( 2, 11, 16, 13, 14, 6, 15, 8, 16, 10, 29, 32, 35, 38, 0, 0 ) - // CHECK-NEXT: ( 2, 11, 0, 16, 13, 0, 0, 0, 0, 0, 14, 6, 0, 0, 0, 0, 15, 8, 16, 0, 10, 29, 0, 0, 0, 0, 0, 0, 32, 35, 0, 38 ) - // CHECK-NEXT: ( 48, 204, 252, 304, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) - // CHECK-NEXT: ( 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 0, 0, 0, 0, 0, 252, 304, 0, 360 ) - // CHECK-NEXT: ( 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 0, 0, 0, 0, 0, 252, 304, 0, 360 ) + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 9 + // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31 + // CHECK-NEXT: values : ( 1, 2, 3, 4, 5, 6, 7, 8, 9 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 10 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 10 + // CHECK-NEXT: crd[0] : ( 1, 3, 4, 10, 16, 18, 21, 28, 29, 31 + // CHECK-NEXT: values : ( 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 9 + // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31 + // CHECK-NEXT: values : ( 2, 4, 6, 8, 10, 12, 14, 16, 18 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 9 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 9 + // CHECK-NEXT: crd[0] : ( 0, 3, 11, 17, 20, 21, 28, 29, 31 + // CHECK-NEXT: values : ( 2, 4, 6, 8, 10, 12, 14, 16, 18 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 14 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 14 + // CHECK-NEXT: crd[0] : ( 0, 1, 3, 4, 10, 11, 16, 17, 18, 20, 21, 28, 29, 31 + // CHECK-NEXT: values : ( 2, 11, 16, 13, 14, 6, 15, 8, 16, 10, 29, 32, 35, 38 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 5 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: pos[0] : ( 0, 5 + // CHECK-NEXT: crd[0] : ( 3, 21, 28, 29, 31 + // CHECK-NEXT: values : ( 48, 204, 252, 304, 360 + // CHECK-NEXT: ---- + // CHECK: ---- Sparse Tensor ---- + // CHECK-NEXT: nse = 32 + // CHECK-NEXT: dim = ( 32 ) + // CHECK-NEXT: lvl = ( 32 ) + // CHECK-NEXT: values : ( 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 0, 0, 0, 0, 0, 252, 304, 0, 360 + // CHECK-NEXT: ---- // CHECK-NEXT: 1169.1 // - - call @dump(%sv1) : (tensor) -> () - call @dump(%sv2) : (tensor) -> () - call @dump(%0) : (tensor) -> () - call @dump(%1) : (tensor) -> () - call @dump(%2) : (tensor) -> () - call @dump(%3) : (tensor) -> () - %m4 = sparse_tensor.values %4 : tensor to memref - %v4 = vector.load %m4[%c0]: memref, vector<32xf64> - vector.print %v4 : vector<32xf64> + sparse_tensor.print %sv1 : tensor + sparse_tensor.print %sv2 : tensor + sparse_tensor.print %0 : tensor + sparse_tensor.print %1 : tensor + sparse_tensor.print %2 : tensor + sparse_tensor.print %3 : tensor + sparse_tensor.print %4 : tensor %v5 = tensor.extract %5[] : tensor vector.print %v5 : f64 -- GitLab From fc837f7a2dbdfca472bd1275362052facfe331a0 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Thu, 7 Mar 2024 19:05:50 +0000 Subject: [PATCH 485/929] [gn build] Port a6a6fca7911f --- .../gn/secondary/llvm/lib/Transforms/Instrumentation/BUILD.gn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/Instrumentation/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/Instrumentation/BUILD.gn index 00e1888da64d..131308db2aa5 100644 --- a/llvm/utils/gn/secondary/llvm/lib/Transforms/Instrumentation/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/Instrumentation/BUILD.gn @@ -23,12 +23,13 @@ static_library("Instrumentation") { "InstrProfiling.cpp", "Instrumentation.cpp", "KCFI.cpp", - "PGOForceFunctionAttrs.cpp", "MemProfiler.cpp", "MemorySanitizer.cpp", + "PGOForceFunctionAttrs.cpp", "PGOInstrumentation.cpp", "PGOMemOPSizeOpt.cpp", "PoisonChecking.cpp", + "RemoveTrapsPass.cpp", "SanitizerBinaryMetadata.cpp", "SanitizerCoverage.cpp", "ThreadSanitizer.cpp", -- GitLab From 3e73a080fa23594c81ad1dc61a65a11c9c488c5b Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Tue, 5 Mar 2024 19:47:38 -0600 Subject: [PATCH 486/929] [X86] Add tests for folding `(icmp ult (add x,-C),2)` -> `(or (icmp eq X,C), (icmp eq X,C+1))`; NFC --- llvm/test/CodeGen/X86/eq-or-eq-range-of-2.ll | 786 +++++++++++++++++++ 1 file changed, 786 insertions(+) create mode 100644 llvm/test/CodeGen/X86/eq-or-eq-range-of-2.ll diff --git a/llvm/test/CodeGen/X86/eq-or-eq-range-of-2.ll b/llvm/test/CodeGen/X86/eq-or-eq-range-of-2.ll new file mode 100644 index 000000000000..3578a49759bf --- /dev/null +++ b/llvm/test/CodeGen/X86/eq-or-eq-range-of-2.ll @@ -0,0 +1,786 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE41 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE2 + +declare void @use.v4.i32(<4 x i32>) + +define <4 x i32> @eq_or_eq_ult_2(<4 x i32> %x) { +; AVX512-LABEL: eq_or_eq_ult_2: +; AVX512: # %bb.0: +; AVX512-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0 +; AVX512-NEXT: vpcmpltud {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %k1 +; AVX512-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0 +; AVX512-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z} +; AVX512-NEXT: retq +; +; AVX1-LABEL: eq_or_eq_ult_2: +; AVX1: # %bb.0: +; AVX1-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 +; AVX1-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: eq_or_eq_ult_2: +; AVX2: # %bb.0: +; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [4294967291,4294967291,4294967291,4294967291] +; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [1,1,1,1] +; AVX2-NEXT: vpminud %xmm1, %xmm0, %xmm1 +; AVX2-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq +; +; SSE41-LABEL: eq_or_eq_ult_2: +; SSE41: # %bb.0: +; SSE41-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE41-NEXT: pmovsxbd {{.*#+}} xmm1 = [1,1,1,1] +; SSE41-NEXT: pminud %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE2-LABEL: eq_or_eq_ult_2: +; SSE2: # %bb.0: +; SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [2147483650,2147483650,2147483650,2147483650] +; SSE2-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: retq + %x_adj = add <4 x i32> %x, + %cmp = icmp ult <4 x i32> %x_adj, + %r = sext <4 x i1> %cmp to <4 x i32> + ret <4 x i32> %r +} + +define <4 x i32> @eq_or_eq_ult_2_only_transform_sse2(<4 x i32> %x) { +; AVX512-LABEL: eq_or_eq_ult_2_only_transform_sse2: +; AVX512: # %bb.0: +; AVX512-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpcmpltud {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %k1 +; AVX512-NEXT: vmovdqa32 %xmm1, %xmm0 {%k1} {z} +; AVX512-NEXT: retq +; +; AVX1-LABEL: eq_or_eq_ult_2_only_transform_sse2: +; AVX1: # %bb.0: +; AVX1-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 +; AVX1-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: eq_or_eq_ult_2_only_transform_sse2: +; AVX2: # %bb.0: +; AVX2-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [1,1,1,1] +; AVX2-NEXT: vpminud %xmm1, %xmm0, %xmm1 +; AVX2-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq +; +; SSE41-LABEL: eq_or_eq_ult_2_only_transform_sse2: +; SSE41: # %bb.0: +; SSE41-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE41-NEXT: paddd %xmm1, %xmm0 +; SSE41-NEXT: pmovsxbd {{.*#+}} xmm1 = [1,1,1,1] +; SSE41-NEXT: pminud %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE2-LABEL: eq_or_eq_ult_2_only_transform_sse2: +; SSE2: # %bb.0: +; SSE2-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE2-NEXT: paddd %xmm0, %xmm1 +; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [2147483650,2147483650,2147483650,2147483650] +; SSE2-NEXT: pcmpgtd %xmm1, %xmm0 +; SSE2-NEXT: retq + %x_adj = add <4 x i32> %x, + %cmp = icmp ult <4 x i32> %x_adj, + %r = sext <4 x i1> %cmp to <4 x i32> + ret <4 x i32> %r +} + +define <4 x i32> @eq_or_eq_ult_2_fail_multiuse(<4 x i32> %x) { +; AVX512-LABEL: eq_or_eq_ult_2_fail_multiuse: +; AVX512: # %bb.0: +; AVX512-NEXT: subq $24, %rsp +; AVX512-NEXT: .cfi_def_cfa_offset 32 +; AVX512-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill +; AVX512-NEXT: callq use.v4.i32@PLT +; AVX512-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload +; AVX512-NEXT: vpcmpltud {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %k1 +; AVX512-NEXT: vmovdqa32 {{.*#+}} xmm0 {%k1} {z} = [4294967295,4294967295,4294967295,4294967295] +; AVX512-NEXT: addq $24, %rsp +; AVX512-NEXT: .cfi_def_cfa_offset 8 +; AVX512-NEXT: retq +; +; AVX1-LABEL: eq_or_eq_ult_2_fail_multiuse: +; AVX1: # %bb.0: +; AVX1-NEXT: subq $24, %rsp +; AVX1-NEXT: .cfi_def_cfa_offset 32 +; AVX1-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill +; AVX1-NEXT: callq use.v4.i32@PLT +; AVX1-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload +; AVX1-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm0 +; AVX1-NEXT: vpcmpeqd %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: addq $24, %rsp +; AVX1-NEXT: .cfi_def_cfa_offset 8 +; AVX1-NEXT: retq +; +; AVX2-LABEL: eq_or_eq_ult_2_fail_multiuse: +; AVX2: # %bb.0: +; AVX2-NEXT: subq $24, %rsp +; AVX2-NEXT: .cfi_def_cfa_offset 32 +; AVX2-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill +; AVX2-NEXT: callq use.v4.i32@PLT +; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm0 = [1,1,1,1] +; AVX2-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload +; AVX2-NEXT: vpminud %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: vpcmpeqd %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: addq $24, %rsp +; AVX2-NEXT: .cfi_def_cfa_offset 8 +; AVX2-NEXT: retq +; +; SSE41-LABEL: eq_or_eq_ult_2_fail_multiuse: +; SSE41: # %bb.0: +; SSE41-NEXT: subq $24, %rsp +; SSE41-NEXT: .cfi_def_cfa_offset 32 +; SSE41-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE41-NEXT: paddd %xmm1, %xmm0 +; SSE41-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill +; SSE41-NEXT: callq use.v4.i32@PLT +; SSE41-NEXT: pmovsxbd {{.*#+}} xmm0 = [1,1,1,1] +; SSE41-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload +; SSE41-NEXT: pminud %xmm1, %xmm0 +; SSE41-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE41-NEXT: addq $24, %rsp +; SSE41-NEXT: .cfi_def_cfa_offset 8 +; SSE41-NEXT: retq +; +; SSE2-LABEL: eq_or_eq_ult_2_fail_multiuse: +; SSE2: # %bb.0: +; SSE2-NEXT: subq $24, %rsp +; SSE2-NEXT: .cfi_def_cfa_offset 32 +; SSE2-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE2-NEXT: paddd %xmm0, %xmm1 +; SSE2-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill +; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: callq use.v4.i32@PLT +; SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload +; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [2147483650,2147483650,2147483650,2147483650] +; SSE2-NEXT: pcmpgtd %xmm1, %xmm0 +; SSE2-NEXT: addq $24, %rsp +; SSE2-NEXT: .cfi_def_cfa_offset 8 +; SSE2-NEXT: retq + %x_adj = add <4 x i32> %x, + call void @use.v4.i32(<4 x i32> %x_adj) + %cmp = icmp ult <4 x i32> %x_adj, + %r = sext <4 x i1> %cmp to <4 x i32> + ret <4 x i32> %r +} + +define <4 x i32> @eq_or_eq_ult_3_fail(<4 x i32> %x) { +; AVX512-LABEL: eq_or_eq_ult_3_fail: +; AVX512: # %bb.0: +; AVX512-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpcmpltud {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %k1 +; AVX512-NEXT: vmovdqa32 %xmm1, %xmm0 {%k1} {z} +; AVX512-NEXT: retq +; +; AVX1-LABEL: eq_or_eq_ult_3_fail: +; AVX1: # %bb.0: +; AVX1-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 +; AVX1-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: eq_or_eq_ult_3_fail: +; AVX2: # %bb.0: +; AVX2-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [2,2,2,2] +; AVX2-NEXT: vpminud %xmm1, %xmm0, %xmm1 +; AVX2-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq +; +; SSE41-LABEL: eq_or_eq_ult_3_fail: +; SSE41: # %bb.0: +; SSE41-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE41-NEXT: paddd %xmm1, %xmm0 +; SSE41-NEXT: pmovsxbd {{.*#+}} xmm1 = [2,2,2,2] +; SSE41-NEXT: pminud %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE2-LABEL: eq_or_eq_ult_3_fail: +; SSE2: # %bb.0: +; SSE2-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE2-NEXT: paddd %xmm0, %xmm1 +; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [2147483651,2147483651,2147483651,2147483651] +; SSE2-NEXT: pcmpgtd %xmm1, %xmm0 +; SSE2-NEXT: retq + %x_adj = add <4 x i32> %x, + %cmp = icmp ult <4 x i32> %x_adj, + %r = sext <4 x i1> %cmp to <4 x i32> + ret <4 x i32> %r +} + +define <4 x i32> @eq_or_eq_ugt_m3(<4 x i32> %x) { +; AVX512-LABEL: eq_or_eq_ugt_m3: +; AVX512: # %bb.0: +; AVX512-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX512-NEXT: vpcmpnleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %k1 +; AVX512-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0 +; AVX512-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z} +; AVX512-NEXT: retq +; +; AVX1-LABEL: eq_or_eq_ugt_m3: +; AVX1: # %bb.0: +; AVX1-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vpmaxud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 +; AVX1-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: eq_or_eq_ugt_m3: +; AVX2: # %bb.0: +; AVX2-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [4294967294,4294967294,4294967294,4294967294] +; AVX2-NEXT: vpmaxud %xmm1, %xmm0, %xmm1 +; AVX2-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq +; +; SSE41-LABEL: eq_or_eq_ugt_m3: +; SSE41: # %bb.0: +; SSE41-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE41-NEXT: pmovsxbd {{.*#+}} xmm1 = [4294967294,4294967294,4294967294,4294967294] +; SSE41-NEXT: pmaxud %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE2-LABEL: eq_or_eq_ugt_m3: +; SSE2: # %bb.0: +; SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: retq + %x_adj = add <4 x i32> %x, + %cmp = icmp ugt <4 x i32> %x_adj, + %r = sext <4 x i1> %cmp to <4 x i32> + ret <4 x i32> %r +} + +define <4 x i32> @eq_or_eq_ule_1(<4 x i32> %x) { +; AVX512-LABEL: eq_or_eq_ule_1: +; AVX512: # %bb.0: +; AVX512-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX512-NEXT: vpcmpleud {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %k1 +; AVX512-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0 +; AVX512-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z} +; AVX512-NEXT: retq +; +; AVX1-LABEL: eq_or_eq_ule_1: +; AVX1: # %bb.0: +; AVX1-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 +; AVX1-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: eq_or_eq_ule_1: +; AVX2: # %bb.0: +; AVX2-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [1,1,1,1] +; AVX2-NEXT: vpminud %xmm1, %xmm0, %xmm1 +; AVX2-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq +; +; SSE41-LABEL: eq_or_eq_ule_1: +; SSE41: # %bb.0: +; SSE41-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE41-NEXT: pmovsxbd {{.*#+}} xmm1 = [1,1,1,1] +; SSE41-NEXT: pminud %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE2-LABEL: eq_or_eq_ule_1: +; SSE2: # %bb.0: +; SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE2-NEXT: pxor %xmm1, %xmm0 +; SSE2-NEXT: retq + %x_adj = add <4 x i32> %x, + %cmp = icmp ule <4 x i32> %x_adj, + %r = sext <4 x i1> %cmp to <4 x i32> + ret <4 x i32> %r +} + +define <2 x i64> @eq_or_eq_uge_m2_i64(<2 x i64> %x) { +; AVX512-LABEL: eq_or_eq_uge_m2_i64: +; AVX512: # %bb.0: +; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX512-NEXT: vpcmpnltuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %k1 +; AVX512-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0 +; AVX512-NEXT: vmovdqa64 %xmm0, %xmm0 {%k1} {z} +; AVX512-NEXT: retq +; +; AVX1-LABEL: eq_or_eq_uge_m2_i64: +; AVX1: # %bb.0: +; AVX1-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vpxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vmovddup {{.*#+}} xmm1 = [9223372036854775806,9223372036854775806] +; AVX1-NEXT: # xmm1 = mem[0,0] +; AVX1-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: eq_or_eq_uge_m2_i64: +; AVX2: # %bb.0: +; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX2-NEXT: vpxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX2-NEXT: vpbroadcastq {{.*#+}} xmm1 = [9223372036854775806,9223372036854775806] +; AVX2-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX2-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq +; +; SSE41-LABEL: eq_or_eq_uge_m2_i64: +; SSE41: # %bb.0: +; SSE41-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE41-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE41-NEXT: movdqa {{.*#+}} xmm1 = [9223372034707292158,9223372034707292158] +; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2] +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE41-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE41-NEXT: pand %xmm2, %xmm0 +; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] +; SSE41-NEXT: por %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE41-NEXT: pxor %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE2-LABEL: eq_or_eq_uge_m2_i64: +; SSE2: # %bb.0: +; SSE2-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [9223372034707292158,9223372034707292158] +; SSE2-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2] +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE2-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: pand %xmm2, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] +; SSE2-NEXT: por %xmm0, %xmm1 +; SSE2-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE2-NEXT: pxor %xmm1, %xmm0 +; SSE2-NEXT: retq + %x_adj = add <2 x i64> %x, + %cmp = icmp uge <2 x i64> %x_adj, + %r = sext <2 x i1> %cmp to <2 x i64> + ret <2 x i64> %r +} + + +define <2 x i64> @eq_or_eq_uge_m2_i64_m1(<2 x i64> %x) { +; AVX512-LABEL: eq_or_eq_uge_m2_i64_m1: +; AVX512: # %bb.0: +; AVX512-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpcmpnltuq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %k1 +; AVX512-NEXT: vmovdqa64 %xmm1, %xmm0 {%k1} {z} +; AVX512-NEXT: retq +; +; AVX1-LABEL: eq_or_eq_uge_m2_i64_m1: +; AVX1: # %bb.0: +; AVX1-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vmovddup {{.*#+}} xmm2 = [9223372036854775806,9223372036854775806] +; AVX1-NEXT: # xmm2 = mem[0,0] +; AVX1-NEXT: vpcmpgtq %xmm0, %xmm2, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: eq_or_eq_uge_m2_i64_m1: +; AVX2: # %bb.0: +; AVX2-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX2-NEXT: vpbroadcastq {{.*#+}} xmm2 = [9223372036854775806,9223372036854775806] +; AVX2-NEXT: vpcmpgtq %xmm0, %xmm2, %xmm0 +; AVX2-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq +; +; SSE41-LABEL: eq_or_eq_uge_m2_i64_m1: +; SSE41: # %bb.0: +; SSE41-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE41-NEXT: paddq %xmm1, %xmm0 +; SSE41-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [9223372034707292158,9223372034707292158] +; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] +; SSE41-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 +; SSE41-NEXT: pand %xmm3, %xmm4 +; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE41-NEXT: por %xmm4, %xmm0 +; SSE41-NEXT: pxor %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE2-LABEL: eq_or_eq_uge_m2_i64_m1: +; SSE2: # %bb.0: +; SSE2-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE2-NEXT: paddq %xmm1, %xmm0 +; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [9223372034707292158,9223372034707292158] +; SSE2-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] +; SSE2-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 +; SSE2-NEXT: pand %xmm3, %xmm4 +; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE2-NEXT: por %xmm4, %xmm0 +; SSE2-NEXT: pxor %xmm1, %xmm0 +; SSE2-NEXT: retq + %x_adj = add <2 x i64> %x, + %cmp = icmp uge <2 x i64> %x_adj, + %r = sext <2 x i1> %cmp to <2 x i64> + ret <2 x i64> %r +} + +define <4 x i32> @eq_or_eq_uge_2_fail_(<4 x i32> %x) { +; AVX512-LABEL: eq_or_eq_uge_2_fail_: +; AVX512: # %bb.0: +; AVX512-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX512-NEXT: vpcmpnltud {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %k1 +; AVX512-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0 +; AVX512-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z} +; AVX512-NEXT: retq +; +; AVX1-LABEL: eq_or_eq_uge_2_fail_: +; AVX1: # %bb.0: +; AVX1-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vpmaxud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 +; AVX1-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: eq_or_eq_uge_2_fail_: +; AVX2: # %bb.0: +; AVX2-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [2,2,2,2] +; AVX2-NEXT: vpmaxud %xmm1, %xmm0, %xmm1 +; AVX2-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq +; +; SSE41-LABEL: eq_or_eq_uge_2_fail_: +; SSE41: # %bb.0: +; SSE41-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE41-NEXT: pmovsxbd {{.*#+}} xmm1 = [2,2,2,2] +; SSE41-NEXT: pmaxud %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE2-LABEL: eq_or_eq_uge_2_fail_: +; SSE2: # %bb.0: +; SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [2147483650,2147483650,2147483650,2147483650] +; SSE2-NEXT: pcmpgtd %xmm0, %xmm1 +; SSE2-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE2-NEXT: pxor %xmm1, %xmm0 +; SSE2-NEXT: retq + %x_adj = add <4 x i32> %x, + %cmp = icmp uge <4 x i32> %x_adj, + %r = sext <4 x i1> %cmp to <4 x i32> + ret <4 x i32> %r +} + + +define <8 x i32> @eq_or_eq_ult_2_256(<8 x i32> %x) { +; AVX512-LABEL: eq_or_eq_ult_2_256: +; AVX512: # %bb.0: +; AVX512-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0 +; AVX512-NEXT: vpcmpltud {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %k1 +; AVX512-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0 +; AVX512-NEXT: vmovdqa32 %ymm0, %ymm0 {%k1} {z} +; AVX512-NEXT: retq +; +; AVX1-LABEL: eq_or_eq_ult_2_256: +; AVX1: # %bb.0: +; AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [4294967291,4294967291,4294967291,4294967291] +; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [1,1,1,1] +; AVX1-NEXT: vpminud %xmm1, %xmm0, %xmm3 +; AVX1-NEXT: vpcmpeqd %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpminud %xmm1, %xmm2, %xmm1 +; AVX1-NEXT: vpcmpeqd %xmm1, %xmm2, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: eq_or_eq_ult_2_256: +; AVX2: # %bb.0: +; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm1 = [4294967291,4294967291,4294967291,4294967291,4294967291,4294967291,4294967291,4294967291] +; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm1 = [1,1,1,1,1,1,1,1] +; AVX2-NEXT: vpminud %ymm1, %ymm0, %ymm1 +; AVX2-NEXT: vpcmpeqd %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; SSE41-LABEL: eq_or_eq_ult_2_256: +; SSE41: # %bb.0: +; SSE41-NEXT: pmovsxbd {{.*#+}} xmm2 = [4294967291,4294967291,4294967291,4294967291] +; SSE41-NEXT: paddd %xmm2, %xmm1 +; SSE41-NEXT: paddd %xmm2, %xmm0 +; SSE41-NEXT: pmovsxbd {{.*#+}} xmm2 = [1,1,1,1] +; SSE41-NEXT: movdqa %xmm0, %xmm3 +; SSE41-NEXT: pminud %xmm2, %xmm3 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 +; SSE41-NEXT: pminud %xmm1, %xmm2 +; SSE41-NEXT: pcmpeqd %xmm2, %xmm1 +; SSE41-NEXT: retq +; +; SSE2-LABEL: eq_or_eq_ult_2_256: +; SSE2: # %bb.0: +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [4294967291,4294967291,4294967291,4294967291] +; SSE2-NEXT: paddd %xmm2, %xmm1 +; SSE2-NEXT: paddd %xmm2, %xmm0 +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: pxor %xmm4, %xmm0 +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483650,2147483650,2147483650,2147483650] +; SSE2-NEXT: movdqa %xmm2, %xmm3 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE2-NEXT: pxor %xmm4, %xmm1 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm3, %xmm0 +; SSE2-NEXT: movdqa %xmm2, %xmm1 +; SSE2-NEXT: retq + %x_adj = add <8 x i32> %x, + %cmp = icmp ult <8 x i32> %x_adj, + %r = sext <8 x i1> %cmp to <8 x i32> + ret <8 x i32> %r +} + + +define <8 x i32> @eq_or_eq_ult_2_256_m1(<8 x i32> %x) { +; AVX512-LABEL: eq_or_eq_ult_2_256_m1: +; AVX512: # %bb.0: +; AVX512-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1 +; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: vpcmpltud {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %k1 +; AVX512-NEXT: vmovdqa32 %ymm1, %ymm0 {%k1} {z} +; AVX512-NEXT: retq +; +; AVX1-LABEL: eq_or_eq_ult_2_256_m1: +; AVX1: # %bb.0: +; AVX1-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [1,1,1,1] +; AVX1-NEXT: vpminud %xmm1, %xmm0, %xmm3 +; AVX1-NEXT: vpcmpeqd %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpminud %xmm1, %xmm2, %xmm1 +; AVX1-NEXT: vpcmpeqd %xmm1, %xmm2, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: eq_or_eq_ult_2_256_m1: +; AVX2: # %bb.0: +; AVX2-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1 +; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm1 = [1,1,1,1,1,1,1,1] +; AVX2-NEXT: vpminud %ymm1, %ymm0, %ymm1 +; AVX2-NEXT: vpcmpeqd %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; SSE41-LABEL: eq_or_eq_ult_2_256_m1: +; SSE41: # %bb.0: +; SSE41-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE41-NEXT: paddd %xmm2, %xmm1 +; SSE41-NEXT: paddd %xmm2, %xmm0 +; SSE41-NEXT: pmovsxbd {{.*#+}} xmm2 = [1,1,1,1] +; SSE41-NEXT: movdqa %xmm0, %xmm3 +; SSE41-NEXT: pminud %xmm2, %xmm3 +; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 +; SSE41-NEXT: pminud %xmm1, %xmm2 +; SSE41-NEXT: pcmpeqd %xmm2, %xmm1 +; SSE41-NEXT: retq +; +; SSE2-LABEL: eq_or_eq_ult_2_256_m1: +; SSE2: # %bb.0: +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: paddd %xmm2, %xmm1 +; SSE2-NEXT: paddd %xmm2, %xmm0 +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: pxor %xmm4, %xmm0 +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483650,2147483650,2147483650,2147483650] +; SSE2-NEXT: movdqa %xmm2, %xmm3 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm3 +; SSE2-NEXT: pxor %xmm4, %xmm1 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm3, %xmm0 +; SSE2-NEXT: movdqa %xmm2, %xmm1 +; SSE2-NEXT: retq + %x_adj = add <8 x i32> %x, + %cmp = icmp ult <8 x i32> %x_adj, + %r = sext <8 x i1> %cmp to <8 x i32> + ret <8 x i32> %r +} + + +define <32 x i8> @eq_or_eq_ult_2_256_i8_m1(<32 x i8> %x) { +; AVX512-LABEL: eq_or_eq_ult_2_256_i8_m1: +; AVX512: # %bb.0: +; AVX512-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1 +; AVX512-NEXT: vpaddb %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: vpminub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm1 +; AVX512-NEXT: vpcmpeqb %ymm1, %ymm0, %ymm0 +; AVX512-NEXT: retq +; +; AVX1-LABEL: eq_or_eq_ult_2_256_i8_m1: +; AVX1: # %bb.0: +; AVX1-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm2 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; AVX1-NEXT: vpminub %xmm1, %xmm0, %xmm3 +; AVX1-NEXT: vpcmpeqb %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpminub %xmm1, %xmm2, %xmm1 +; AVX1-NEXT: vpcmpeqb %xmm1, %xmm2, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: eq_or_eq_ult_2_256_i8_m1: +; AVX2: # %bb.0: +; AVX2-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1 +; AVX2-NEXT: vpaddb %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vpminub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm1 +; AVX2-NEXT: vpcmpeqb %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: retq +; +; SSE41-LABEL: eq_or_eq_ult_2_256_i8_m1: +; SSE41: # %bb.0: +; SSE41-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE41-NEXT: paddb %xmm2, %xmm1 +; SSE41-NEXT: paddb %xmm2, %xmm0 +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; SSE41-NEXT: movdqa %xmm0, %xmm3 +; SSE41-NEXT: pminub %xmm2, %xmm3 +; SSE41-NEXT: pcmpeqb %xmm3, %xmm0 +; SSE41-NEXT: pminub %xmm1, %xmm2 +; SSE41-NEXT: pcmpeqb %xmm2, %xmm1 +; SSE41-NEXT: retq +; +; SSE2-LABEL: eq_or_eq_ult_2_256_i8_m1: +; SSE2: # %bb.0: +; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 +; SSE2-NEXT: paddb %xmm2, %xmm1 +; SSE2-NEXT: paddb %xmm2, %xmm0 +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: pminub %xmm2, %xmm3 +; SSE2-NEXT: pcmpeqb %xmm3, %xmm0 +; SSE2-NEXT: pminub %xmm1, %xmm2 +; SSE2-NEXT: pcmpeqb %xmm2, %xmm1 +; SSE2-NEXT: retq + %x_adj = add <32 x i8> %x, + %cmp = icmp ult <32 x i8> %x_adj, + %r = sext <32 x i1> %cmp to <32 x i8> + ret <32 x i8> %r +} + + +define <16 x i8> @eq_or_eq_ult_2_128_i8_m1(<16 x i8> %x) { +; AVX512-LABEL: eq_or_eq_ult_2_128_i8_m1: +; AVX512: # %bb.0: +; AVX512-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX512-NEXT: vpaddb %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpminub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 +; AVX512-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; AVX-LABEL: eq_or_eq_ult_2_128_i8_m1: +; AVX: # %bb.0: +; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpaddb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpminub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 +; AVX-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; SSE41-LABEL: eq_or_eq_ult_2_128_i8_m1: +; SSE41: # %bb.0: +; SSE41-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE41-NEXT: paddb %xmm1, %xmm0 +; SSE41-NEXT: movdqa {{.*#+}} xmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; SSE41-NEXT: pminub %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqb %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE2-LABEL: eq_or_eq_ult_2_128_i8_m1: +; SSE2: # %bb.0: +; SSE2-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE2-NEXT: paddb %xmm1, %xmm0 +; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; SSE2-NEXT: pminub %xmm0, %xmm1 +; SSE2-NEXT: pcmpeqb %xmm1, %xmm0 +; SSE2-NEXT: retq + %x_adj = add <16 x i8> %x, + %cmp = icmp ult <16 x i8> %x_adj, + %r = sext <16 x i1> %cmp to <16 x i8> + ret <16 x i8> %r +} + + +define <16 x i8> @eq_or_eq_ult_2_128_i8(<16 x i8> %x) { +; AVX512-LABEL: eq_or_eq_ult_2_128_i8: +; AVX512: # %bb.0: +; AVX512-NEXT: vpaddb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX512-NEXT: vpminub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 +; AVX512-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: retq +; +; AVX-LABEL: eq_or_eq_ult_2_128_i8: +; AVX: # %bb.0: +; AVX-NEXT: vpaddb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: vpminub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 +; AVX-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq +; +; SSE41-LABEL: eq_or_eq_ult_2_128_i8: +; SSE41: # %bb.0: +; SSE41-NEXT: paddb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE41-NEXT: movdqa {{.*#+}} xmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; SSE41-NEXT: pminub %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqb %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; SSE2-LABEL: eq_or_eq_ult_2_128_i8: +; SSE2: # %bb.0: +; SSE2-NEXT: paddb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; SSE2-NEXT: pminub %xmm0, %xmm1 +; SSE2-NEXT: pcmpeqb %xmm1, %xmm0 +; SSE2-NEXT: retq + %x_adj = add <16 x i8> %x, + %cmp = icmp ult <16 x i8> %x_adj, + %r = sext <16 x i1> %cmp to <16 x i8> + ret <16 x i8> %r +} -- GitLab From 9f96db8e310f79ec450c9cf6e6311f576dfd1d51 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Tue, 5 Mar 2024 19:44:41 -0600 Subject: [PATCH 487/929] [X86] Fold `(icmp ult (add x,-C),2)` -> `(or (icmp eq X,C), (icmp eq X,C+1))` for Vectors This is undoing a middle-end transform which does the opposite. Since X86 doesn't have unsigned vector comparison instructions pre-AVX512, the simplified form gets worse codegen. Fixes #66479 Proofs: https://alive2.llvm.org/ce/z/UCz3wt Closes #84104 Closes #66479 --- llvm/lib/Target/X86/X86ISelLowering.cpp | 63 ++++ llvm/test/CodeGen/X86/eq-or-eq-range-of-2.ll | 374 ++++++++----------- 2 files changed, 214 insertions(+), 223 deletions(-) diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 94c4bbc4a099..e1e6c22eb8cc 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -53441,6 +53441,69 @@ static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG, truncateAVX512SetCCNoBWI(VT, OpVT, LHS, RHS, CC, DL, DAG, Subtarget)) return R; + // In the middle end transforms: + // `(or (icmp eq X, C), (icmp eq X, C+1))` + // -> `(icmp ult (add x, -C), 2)` + // Likewise inverted cases with `ugt`. + // + // Since x86, pre avx512, doesn't have unsigned vector compares, this results + // in worse codegen. So, undo the middle-end transform and go back to `(or + // (icmp eq), (icmp eq))` form. + // Also skip AVX1 with ymm vectors, as the umin approach combines better than + // the xmm approach. + // + // NB: We don't handle the similiar simplication of `(and (icmp ne), (icmp + // ne))` as it doesn't end up instruction positive. + // TODO: We might want to do this for avx512 as well if we `sext` the result. + if (VT.isVector() && OpVT.isVector() && OpVT.isInteger() && + ISD::isUnsignedIntSetCC(CC) && LHS.getOpcode() == ISD::ADD && + !Subtarget.hasAVX512() && + (OpVT.getSizeInBits() <= 128 || !Subtarget.hasAVX() || + Subtarget.hasAVX2()) && + LHS.hasOneUse()) { + + APInt CmpC; + SDValue AddC = LHS.getOperand(1); + if (ISD::isConstantSplatVector(RHS.getNode(), CmpC) && + DAG.isConstantIntBuildVectorOrConstantInt(AddC)) { + // See which form we have depending on the constant/condition. + SDValue C0 = SDValue(); + SDValue C1 = SDValue(); + + // If we had `(add x, -1)` and can lower with `umin`, don't transform as + // we will end up generating an additional constant. Keeping in the + // current form has a slight latency cost, but it probably worth saving a + // constant. + if (ISD::isConstantSplatVectorAllOnes(AddC.getNode()) && + DAG.getTargetLoweringInfo().isOperationLegal(ISD::UMIN, OpVT)) { + // Pass + } + // Normal Cases + else if ((CC == ISD::SETULT && CmpC == 2) || + (CC == ISD::SETULE && CmpC == 1)) { + // These will constant fold. + C0 = DAG.getNegative(AddC, DL, OpVT); + C1 = DAG.getNode(ISD::SUB, DL, OpVT, C0, + DAG.getAllOnesConstant(DL, OpVT)); + } + // Inverted Cases + else if ((CC == ISD::SETUGT && (-CmpC) == 3) || + (CC == ISD::SETUGE && (-CmpC) == 2)) { + // These will constant fold. + C0 = DAG.getNOT(DL, AddC, OpVT); + C1 = DAG.getNode(ISD::ADD, DL, OpVT, C0, + DAG.getAllOnesConstant(DL, OpVT)); + } + if (C0 && C1) { + SDValue NewLHS = + DAG.getSetCC(DL, VT, LHS.getOperand(0), C0, ISD::SETEQ); + SDValue NewRHS = + DAG.getSetCC(DL, VT, LHS.getOperand(0), C1, ISD::SETEQ); + return DAG.getNode(ISD::OR, DL, VT, NewLHS, NewRHS); + } + } + } + // For an SSE1-only target, lower a comparison of v4f32 to X86ISD::CMPP early // to avoid scalarization via legalization because v4i32 is not a legal type. if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32 && diff --git a/llvm/test/CodeGen/X86/eq-or-eq-range-of-2.ll b/llvm/test/CodeGen/X86/eq-or-eq-range-of-2.ll index 3578a49759bf..527995bc2139 100644 --- a/llvm/test/CodeGen/X86/eq-or-eq-range-of-2.ll +++ b/llvm/test/CodeGen/X86/eq-or-eq-range-of-2.ll @@ -18,35 +18,34 @@ define <4 x i32> @eq_or_eq_ult_2(<4 x i32> %x) { ; ; AVX1-LABEL: eq_or_eq_ult_2: ; AVX1: # %bb.0: -; AVX1-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX1-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 -; AVX1-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 +; AVX1-NEXT: vpcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: eq_or_eq_ult_2: ; AVX2: # %bb.0: -; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [4294967291,4294967291,4294967291,4294967291] -; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0 -; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [1,1,1,1] -; AVX2-NEXT: vpminud %xmm1, %xmm0, %xmm1 -; AVX2-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [6,6,6,6] +; AVX2-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm1 +; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [5,5,5,5] +; AVX2-NEXT: vpcmpeqd %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 ; AVX2-NEXT: retq ; ; SSE41-LABEL: eq_or_eq_ult_2: ; SSE41: # %bb.0: -; SSE41-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE41-NEXT: pmovsxbd {{.*#+}} xmm1 = [1,1,1,1] -; SSE41-NEXT: pminud %xmm0, %xmm1 -; SSE41-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE41-NEXT: pmovsxbd {{.*#+}} xmm1 = [6,6,6,6] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: retq ; ; SSE2-LABEL: eq_or_eq_ult_2: ; SSE2: # %bb.0: -; SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [2147483650,2147483650,2147483650,2147483650] -; SSE2-NEXT: pcmpgtd %xmm0, %xmm1 -; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [6,6,6,6] +; SSE2-NEXT: pcmpeqd %xmm0, %xmm1 +; SSE2-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: por %xmm1, %xmm0 ; SSE2-NEXT: retq %x_adj = add <4 x i32> %x, %cmp = icmp ult <4 x i32> %x_adj, @@ -91,11 +90,10 @@ define <4 x i32> @eq_or_eq_ult_2_only_transform_sse2(<4 x i32> %x) { ; ; SSE2-LABEL: eq_or_eq_ult_2_only_transform_sse2: ; SSE2: # %bb.0: -; SSE2-NEXT: pcmpeqd %xmm1, %xmm1 -; SSE2-NEXT: paddd %xmm0, %xmm1 -; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [2147483650,2147483650,2147483650,2147483650] -; SSE2-NEXT: pcmpgtd %xmm1, %xmm0 +; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [2,2,2,2] +; SSE2-NEXT: pcmpeqd %xmm0, %xmm1 +; SSE2-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: por %xmm1, %xmm0 ; SSE2-NEXT: retq %x_adj = add <4 x i32> %x, %cmp = icmp ult <4 x i32> %x_adj, @@ -247,34 +245,27 @@ define <4 x i32> @eq_or_eq_ugt_m3(<4 x i32> %x) { ; AVX512-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z} ; AVX512-NEXT: retq ; -; AVX1-LABEL: eq_or_eq_ugt_m3: -; AVX1: # %bb.0: -; AVX1-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX1-NEXT: vpmaxud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 -; AVX1-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 -; AVX1-NEXT: retq -; -; AVX2-LABEL: eq_or_eq_ugt_m3: -; AVX2: # %bb.0: -; AVX2-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [4294967294,4294967294,4294967294,4294967294] -; AVX2-NEXT: vpmaxud %xmm1, %xmm0, %xmm1 -; AVX2-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 -; AVX2-NEXT: retq +; AVX-LABEL: eq_or_eq_ugt_m3: +; AVX: # %bb.0: +; AVX-NEXT: vpcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 +; AVX-NEXT: vpcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq ; ; SSE41-LABEL: eq_or_eq_ugt_m3: ; SSE41: # %bb.0: -; SSE41-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE41-NEXT: pmovsxbd {{.*#+}} xmm1 = [4294967294,4294967294,4294967294,4294967294] -; SSE41-NEXT: pmaxud %xmm0, %xmm1 -; SSE41-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE41-NEXT: pmovsxbd {{.*#+}} xmm1 = [9,12,9,9] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: retq ; ; SSE2-LABEL: eq_or_eq_ugt_m3: ; SSE2: # %bb.0: -; SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [9,12,9,9] +; SSE2-NEXT: pcmpeqd %xmm0, %xmm1 +; SSE2-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: por %xmm1, %xmm0 ; SSE2-NEXT: retq %x_adj = add <4 x i32> %x, %cmp = icmp ugt <4 x i32> %x_adj, @@ -291,36 +282,27 @@ define <4 x i32> @eq_or_eq_ule_1(<4 x i32> %x) { ; AVX512-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z} ; AVX512-NEXT: retq ; -; AVX1-LABEL: eq_or_eq_ule_1: -; AVX1: # %bb.0: -; AVX1-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX1-NEXT: vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 -; AVX1-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 -; AVX1-NEXT: retq -; -; AVX2-LABEL: eq_or_eq_ule_1: -; AVX2: # %bb.0: -; AVX2-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [1,1,1,1] -; AVX2-NEXT: vpminud %xmm1, %xmm0, %xmm1 -; AVX2-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 -; AVX2-NEXT: retq +; AVX-LABEL: eq_or_eq_ule_1: +; AVX: # %bb.0: +; AVX-NEXT: vpcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 +; AVX-NEXT: vpcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq ; ; SSE41-LABEL: eq_or_eq_ule_1: ; SSE41: # %bb.0: -; SSE41-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE41-NEXT: pmovsxbd {{.*#+}} xmm1 = [1,1,1,1] -; SSE41-NEXT: pminud %xmm0, %xmm1 -; SSE41-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE41-NEXT: pmovsxbd {{.*#+}} xmm1 = [0,4294967295,4294967294,4294967293] +; SSE41-NEXT: pcmpeqd %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: retq ; ; SSE2-LABEL: eq_or_eq_ule_1: ; SSE2: # %bb.0: -; SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: pcmpeqd %xmm1, %xmm1 -; SSE2-NEXT: pxor %xmm1, %xmm0 +; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [0,4294967295,4294967294,4294967293] +; SSE2-NEXT: pcmpeqd %xmm0, %xmm1 +; SSE2-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: por %xmm1, %xmm0 ; SSE2-NEXT: retq %x_adj = add <4 x i32> %x, %cmp = icmp ule <4 x i32> %x_adj, @@ -337,57 +319,31 @@ define <2 x i64> @eq_or_eq_uge_m2_i64(<2 x i64> %x) { ; AVX512-NEXT: vmovdqa64 %xmm0, %xmm0 {%k1} {z} ; AVX512-NEXT: retq ; -; AVX1-LABEL: eq_or_eq_uge_m2_i64: -; AVX1: # %bb.0: -; AVX1-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX1-NEXT: vpxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX1-NEXT: vmovddup {{.*#+}} xmm1 = [9223372036854775806,9223372036854775806] -; AVX1-NEXT: # xmm1 = mem[0,0] -; AVX1-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm0 -; AVX1-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 -; AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm0 -; AVX1-NEXT: retq -; -; AVX2-LABEL: eq_or_eq_uge_m2_i64: -; AVX2: # %bb.0: -; AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX2-NEXT: vpxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX2-NEXT: vpbroadcastq {{.*#+}} xmm1 = [9223372036854775806,9223372036854775806] -; AVX2-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm0 -; AVX2-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 -; AVX2-NEXT: vpxor %xmm1, %xmm0, %xmm0 -; AVX2-NEXT: retq +; AVX-LABEL: eq_or_eq_uge_m2_i64: +; AVX: # %bb.0: +; AVX-NEXT: vpcmpeqq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 +; AVX-NEXT: vpcmpeqq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq ; ; SSE41-LABEL: eq_or_eq_uge_m2_i64: ; SSE41: # %bb.0: -; SSE41-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE41-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE41-NEXT: movdqa {{.*#+}} xmm1 = [9223372034707292158,9223372034707292158] -; SSE41-NEXT: pcmpgtd %xmm0, %xmm1 -; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2] -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE41-NEXT: pand %xmm2, %xmm0 -; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE41-NEXT: por %xmm0, %xmm1 -; SSE41-NEXT: pcmpeqd %xmm0, %xmm0 -; SSE41-NEXT: pxor %xmm1, %xmm0 +; SSE41-NEXT: pmovsxbq {{.*#+}} xmm1 = [18446744073709551613,18446744073709551612] +; SSE41-NEXT: pcmpeqq %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: retq ; ; SSE2-LABEL: eq_or_eq_uge_m2_i64: ; SSE2: # %bb.0: -; SSE2-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [9223372034707292158,9223372034707292158] -; SSE2-NEXT: pcmpgtd %xmm0, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2] -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [18446744073709551613,18446744073709551612] +; SSE2-NEXT: pcmpeqd %xmm0, %xmm1 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,0,3,2] +; SSE2-NEXT: pand %xmm1, %xmm2 ; SSE2-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: pand %xmm2, %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] -; SSE2-NEXT: por %xmm0, %xmm1 -; SSE2-NEXT: pcmpeqd %xmm0, %xmm0 -; SSE2-NEXT: pxor %xmm1, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2] +; SSE2-NEXT: pand %xmm1, %xmm0 +; SSE2-NEXT: por %xmm2, %xmm0 ; SSE2-NEXT: retq %x_adj = add <2 x i64> %x, %cmp = icmp uge <2 x i64> %x_adj, @@ -405,57 +361,35 @@ define <2 x i64> @eq_or_eq_uge_m2_i64_m1(<2 x i64> %x) { ; AVX512-NEXT: vmovdqa64 %xmm1, %xmm0 {%k1} {z} ; AVX512-NEXT: retq ; -; AVX1-LABEL: eq_or_eq_uge_m2_i64_m1: -; AVX1: # %bb.0: -; AVX1-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 -; AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0 -; AVX1-NEXT: vpxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX1-NEXT: vmovddup {{.*#+}} xmm2 = [9223372036854775806,9223372036854775806] -; AVX1-NEXT: # xmm2 = mem[0,0] -; AVX1-NEXT: vpcmpgtq %xmm0, %xmm2, %xmm0 -; AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm0 -; AVX1-NEXT: retq -; -; AVX2-LABEL: eq_or_eq_uge_m2_i64_m1: -; AVX2: # %bb.0: -; AVX2-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 -; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0 -; AVX2-NEXT: vpxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX2-NEXT: vpbroadcastq {{.*#+}} xmm2 = [9223372036854775806,9223372036854775806] -; AVX2-NEXT: vpcmpgtq %xmm0, %xmm2, %xmm0 -; AVX2-NEXT: vpxor %xmm1, %xmm0, %xmm0 -; AVX2-NEXT: retq +; AVX-LABEL: eq_or_eq_uge_m2_i64_m1: +; AVX: # %bb.0: +; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpcmpeqq %xmm1, %xmm0, %xmm1 +; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX-NEXT: vpcmpeqq %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq ; ; SSE41-LABEL: eq_or_eq_uge_m2_i64_m1: ; SSE41: # %bb.0: ; SSE41-NEXT: pcmpeqd %xmm1, %xmm1 -; SSE41-NEXT: paddq %xmm1, %xmm0 -; SSE41-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [9223372034707292158,9223372034707292158] -; SSE41-NEXT: pcmpgtd %xmm0, %xmm2 -; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] -; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE41-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 -; SSE41-NEXT: pand %xmm3, %xmm4 -; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] -; SSE41-NEXT: por %xmm4, %xmm0 -; SSE41-NEXT: pxor %xmm1, %xmm0 +; SSE41-NEXT: pcmpeqq %xmm0, %xmm1 +; SSE41-NEXT: pxor %xmm2, %xmm2 +; SSE41-NEXT: pcmpeqq %xmm2, %xmm0 +; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: retq ; ; SSE2-LABEL: eq_or_eq_uge_m2_i64_m1: ; SSE2: # %bb.0: ; SSE2-NEXT: pcmpeqd %xmm1, %xmm1 -; SSE2-NEXT: paddq %xmm1, %xmm0 -; SSE2-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [9223372034707292158,9223372034707292158] -; SSE2-NEXT: pcmpgtd %xmm0, %xmm2 -; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] -; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3] -; SSE2-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 -; SSE2-NEXT: pand %xmm3, %xmm4 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] -; SSE2-NEXT: por %xmm4, %xmm0 -; SSE2-NEXT: pxor %xmm1, %xmm0 +; SSE2-NEXT: pcmpeqd %xmm0, %xmm1 +; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,0,3,2] +; SSE2-NEXT: pand %xmm1, %xmm2 +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2] +; SSE2-NEXT: pand %xmm1, %xmm0 +; SSE2-NEXT: por %xmm2, %xmm0 ; SSE2-NEXT: retq %x_adj = add <2 x i64> %x, %cmp = icmp uge <2 x i64> %x_adj, @@ -536,40 +470,37 @@ define <8 x i32> @eq_or_eq_ult_2_256(<8 x i32> %x) { ; ; AVX2-LABEL: eq_or_eq_ult_2_256: ; AVX2: # %bb.0: -; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm1 = [4294967291,4294967291,4294967291,4294967291,4294967291,4294967291,4294967291,4294967291] -; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0 -; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm1 = [1,1,1,1,1,1,1,1] -; AVX2-NEXT: vpminud %ymm1, %ymm0, %ymm1 -; AVX2-NEXT: vpcmpeqd %ymm1, %ymm0, %ymm0 +; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm1 = [6,6,6,6,6,6,6,6] +; AVX2-NEXT: vpcmpeqd %ymm1, %ymm0, %ymm1 +; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm2 = [5,5,5,5,5,5,5,5] +; AVX2-NEXT: vpcmpeqd %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq ; ; SSE41-LABEL: eq_or_eq_ult_2_256: ; SSE41: # %bb.0: -; SSE41-NEXT: pmovsxbd {{.*#+}} xmm2 = [4294967291,4294967291,4294967291,4294967291] -; SSE41-NEXT: paddd %xmm2, %xmm1 -; SSE41-NEXT: paddd %xmm2, %xmm0 -; SSE41-NEXT: pmovsxbd {{.*#+}} xmm2 = [1,1,1,1] +; SSE41-NEXT: pmovsxbd {{.*#+}} xmm2 = [6,6,6,6] ; SSE41-NEXT: movdqa %xmm0, %xmm3 -; SSE41-NEXT: pminud %xmm2, %xmm3 -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pminud %xmm1, %xmm2 -; SSE41-NEXT: pcmpeqd %xmm2, %xmm1 +; SSE41-NEXT: pcmpeqd %xmm2, %xmm3 +; SSE41-NEXT: pmovsxbd {{.*#+}} xmm4 = [5,5,5,5] +; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 +; SSE41-NEXT: por %xmm3, %xmm0 +; SSE41-NEXT: pcmpeqd %xmm1, %xmm2 +; SSE41-NEXT: pcmpeqd %xmm4, %xmm1 +; SSE41-NEXT: por %xmm2, %xmm1 ; SSE41-NEXT: retq ; ; SSE2-LABEL: eq_or_eq_ult_2_256: ; SSE2: # %bb.0: -; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [4294967291,4294967291,4294967291,4294967291] -; SSE2-NEXT: paddd %xmm2, %xmm1 -; SSE2-NEXT: paddd %xmm2, %xmm0 -; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648] -; SSE2-NEXT: pxor %xmm4, %xmm0 -; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483650,2147483650,2147483650,2147483650] -; SSE2-NEXT: movdqa %xmm2, %xmm3 -; SSE2-NEXT: pcmpgtd %xmm0, %xmm3 -; SSE2-NEXT: pxor %xmm4, %xmm1 -; SSE2-NEXT: pcmpgtd %xmm1, %xmm2 -; SSE2-NEXT: movdqa %xmm3, %xmm0 -; SSE2-NEXT: movdqa %xmm2, %xmm1 +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [6,6,6,6] +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm3 +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [5,5,5,5] +; SSE2-NEXT: pcmpeqd %xmm4, %xmm0 +; SSE2-NEXT: por %xmm3, %xmm0 +; SSE2-NEXT: pcmpeqd %xmm1, %xmm2 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm1 +; SSE2-NEXT: por %xmm2, %xmm1 ; SSE2-NEXT: retq %x_adj = add <8 x i32> %x, %cmp = icmp ult <8 x i32> %x_adj, @@ -612,31 +543,28 @@ define <8 x i32> @eq_or_eq_ult_2_256_m1(<8 x i32> %x) { ; ; SSE41-LABEL: eq_or_eq_ult_2_256_m1: ; SSE41: # %bb.0: -; SSE41-NEXT: pcmpeqd %xmm2, %xmm2 -; SSE41-NEXT: paddd %xmm2, %xmm1 -; SSE41-NEXT: paddd %xmm2, %xmm0 -; SSE41-NEXT: pmovsxbd {{.*#+}} xmm2 = [1,1,1,1] +; SSE41-NEXT: pmovsxbd {{.*#+}} xmm2 = [2,2,2,2] ; SSE41-NEXT: movdqa %xmm0, %xmm3 -; SSE41-NEXT: pminud %xmm2, %xmm3 -; SSE41-NEXT: pcmpeqd %xmm3, %xmm0 -; SSE41-NEXT: pminud %xmm1, %xmm2 -; SSE41-NEXT: pcmpeqd %xmm2, %xmm1 +; SSE41-NEXT: pcmpeqd %xmm2, %xmm3 +; SSE41-NEXT: pmovsxbd {{.*#+}} xmm4 = [1,1,1,1] +; SSE41-NEXT: pcmpeqd %xmm4, %xmm0 +; SSE41-NEXT: por %xmm3, %xmm0 +; SSE41-NEXT: pcmpeqd %xmm1, %xmm2 +; SSE41-NEXT: pcmpeqd %xmm4, %xmm1 +; SSE41-NEXT: por %xmm2, %xmm1 ; SSE41-NEXT: retq ; ; SSE2-LABEL: eq_or_eq_ult_2_256_m1: ; SSE2: # %bb.0: -; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 -; SSE2-NEXT: paddd %xmm2, %xmm1 -; SSE2-NEXT: paddd %xmm2, %xmm0 -; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648] -; SSE2-NEXT: pxor %xmm4, %xmm0 -; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483650,2147483650,2147483650,2147483650] -; SSE2-NEXT: movdqa %xmm2, %xmm3 -; SSE2-NEXT: pcmpgtd %xmm0, %xmm3 -; SSE2-NEXT: pxor %xmm4, %xmm1 -; SSE2-NEXT: pcmpgtd %xmm1, %xmm2 -; SSE2-NEXT: movdqa %xmm3, %xmm0 -; SSE2-NEXT: movdqa %xmm2, %xmm1 +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2,2,2,2] +; SSE2-NEXT: movdqa %xmm0, %xmm3 +; SSE2-NEXT: pcmpeqd %xmm2, %xmm3 +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [1,1,1,1] +; SSE2-NEXT: pcmpeqd %xmm4, %xmm0 +; SSE2-NEXT: por %xmm3, %xmm0 +; SSE2-NEXT: pcmpeqd %xmm1, %xmm2 +; SSE2-NEXT: pcmpeqd %xmm4, %xmm1 +; SSE2-NEXT: por %xmm2, %xmm1 ; SSE2-NEXT: retq %x_adj = add <8 x i32> %x, %cmp = icmp ult <8 x i32> %x_adj, @@ -678,28 +606,28 @@ define <32 x i8> @eq_or_eq_ult_2_256_i8_m1(<32 x i8> %x) { ; ; SSE41-LABEL: eq_or_eq_ult_2_256_i8_m1: ; SSE41: # %bb.0: -; SSE41-NEXT: pcmpeqd %xmm2, %xmm2 -; SSE41-NEXT: paddb %xmm2, %xmm1 -; SSE41-NEXT: paddb %xmm2, %xmm0 -; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2] ; SSE41-NEXT: movdqa %xmm0, %xmm3 -; SSE41-NEXT: pminub %xmm2, %xmm3 -; SSE41-NEXT: pcmpeqb %xmm3, %xmm0 -; SSE41-NEXT: pminub %xmm1, %xmm2 -; SSE41-NEXT: pcmpeqb %xmm2, %xmm1 +; SSE41-NEXT: pcmpeqb %xmm2, %xmm3 +; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; SSE41-NEXT: pcmpeqb %xmm4, %xmm0 +; SSE41-NEXT: por %xmm3, %xmm0 +; SSE41-NEXT: pcmpeqb %xmm1, %xmm2 +; SSE41-NEXT: pcmpeqb %xmm4, %xmm1 +; SSE41-NEXT: por %xmm2, %xmm1 ; SSE41-NEXT: retq ; ; SSE2-LABEL: eq_or_eq_ult_2_256_i8_m1: ; SSE2: # %bb.0: -; SSE2-NEXT: pcmpeqd %xmm2, %xmm2 -; SSE2-NEXT: paddb %xmm2, %xmm1 -; SSE2-NEXT: paddb %xmm2, %xmm0 -; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2] ; SSE2-NEXT: movdqa %xmm0, %xmm3 -; SSE2-NEXT: pminub %xmm2, %xmm3 -; SSE2-NEXT: pcmpeqb %xmm3, %xmm0 -; SSE2-NEXT: pminub %xmm1, %xmm2 -; SSE2-NEXT: pcmpeqb %xmm2, %xmm1 +; SSE2-NEXT: pcmpeqb %xmm2, %xmm3 +; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +; SSE2-NEXT: pcmpeqb %xmm4, %xmm0 +; SSE2-NEXT: por %xmm3, %xmm0 +; SSE2-NEXT: pcmpeqb %xmm1, %xmm2 +; SSE2-NEXT: pcmpeqb %xmm4, %xmm1 +; SSE2-NEXT: por %xmm2, %xmm1 ; SSE2-NEXT: retq %x_adj = add <32 x i8> %x, %cmp = icmp ult <32 x i8> %x_adj, @@ -759,25 +687,25 @@ define <16 x i8> @eq_or_eq_ult_2_128_i8(<16 x i8> %x) { ; ; AVX-LABEL: eq_or_eq_ult_2_128_i8: ; AVX: # %bb.0: -; AVX-NEXT: vpaddb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX-NEXT: vpminub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 -; AVX-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpcmpeqb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 +; AVX-NEXT: vpcmpeqb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: vpor %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq ; ; SSE41-LABEL: eq_or_eq_ult_2_128_i8: ; SSE41: # %bb.0: -; SSE41-NEXT: paddb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE41-NEXT: movdqa {{.*#+}} xmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] -; SSE41-NEXT: pminub %xmm0, %xmm1 -; SSE41-NEXT: pcmpeqb %xmm1, %xmm0 +; SSE41-NEXT: movdqa {{.*#+}} xmm1 = [232,232,232,232,232,232,232,232,232,232,232,232,232,232,232,232] +; SSE41-NEXT: pcmpeqb %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE41-NEXT: por %xmm1, %xmm0 ; SSE41-NEXT: retq ; ; SSE2-LABEL: eq_or_eq_ult_2_128_i8: ; SSE2: # %bb.0: -; SSE2-NEXT: paddb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] -; SSE2-NEXT: pminub %xmm0, %xmm1 -; SSE2-NEXT: pcmpeqb %xmm1, %xmm0 +; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [232,232,232,232,232,232,232,232,232,232,232,232,232,232,232,232] +; SSE2-NEXT: pcmpeqb %xmm0, %xmm1 +; SSE2-NEXT: pcmpeqb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE2-NEXT: por %xmm1, %xmm0 ; SSE2-NEXT: retq %x_adj = add <16 x i8> %x, %cmp = icmp ult <16 x i8> %x_adj, -- GitLab From 00f412168cf6aee234615a11d1424fc58221eb78 Mon Sep 17 00:00:00 2001 From: Hongyu Chen Date: Thu, 7 Mar 2024 11:15:16 -0800 Subject: [PATCH 488/929] [ORC][JITLink] Add Intel VTune support to JITLink (#83957) [ORC] Re-land https://github.com/llvm/llvm-project/pull/81826 This patch adds two plugins: VTuneSupportPlugin.cpp and JITLoaderVTune.cpp. The testing is done in a manner similar to llvm-jitlistener. Currently, we only support the old version of Intel VTune API. --- .../Orc/Debugging/VTuneSupportPlugin.h | 66 ++++++ .../Orc/Shared/VTuneSharedStructs.h | 102 ++++++++ .../Orc/TargetProcess/JITLoaderVTune.h | 31 +++ .../Orc/Debugging/CMakeLists.txt | 1 + .../Orc/Debugging/VTuneSupportPlugin.cpp | 185 +++++++++++++++ .../Orc/TargetProcess/CMakeLists.txt | 9 + .../Orc/TargetProcess/JITLoaderVTune.cpp | 224 ++++++++++++++++++ .../JITLink/x86-64/ELF_vtune.s | 52 ++++ .../JITLink/x86-64/lit.local.cfg | 3 + llvm/tools/llvm-jitlink/llvm-jitlink.cpp | 19 +- 10 files changed, 691 insertions(+), 1 deletion(-) create mode 100644 llvm/include/llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h create mode 100644 llvm/include/llvm/ExecutionEngine/Orc/Shared/VTuneSharedStructs.h create mode 100644 llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h create mode 100644 llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp create mode 100644 llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.cpp create mode 100644 llvm/test/ExecutionEngine/JITLink/x86-64/ELF_vtune.s diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h b/llvm/include/llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h new file mode 100644 index 000000000000..9deb38a1a71f --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h @@ -0,0 +1,66 @@ +//===--- VTuneSupportPlugin.h -- Support for VTune profiler ---*- C++ -*---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Handles support for registering code with VIntel Tune's Amplifier JIT API. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_DEBUGGING_VTUNESUPPORT_H +#define LLVM_EXECUTIONENGINE_ORC_DEBUGGING_VTUNESUPPORT_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" +#include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h" +#include "llvm/ExecutionEngine/Orc/Shared/VTuneSharedStructs.h" + +namespace llvm { + +namespace orc { + +class VTuneSupportPlugin : public ObjectLinkingLayer::Plugin { +public: + VTuneSupportPlugin(ExecutorProcessControl &EPC, ExecutorAddr RegisterImplAddr, + ExecutorAddr UnregisterImplAddr, bool EmitDebugInfo) + : EPC(EPC), RegisterVTuneImplAddr(RegisterImplAddr), + UnregisterVTuneImplAddr(UnregisterImplAddr), + EmitDebugInfo(EmitDebugInfo) {} + + void modifyPassConfig(MaterializationResponsibility &MR, + jitlink::LinkGraph &G, + jitlink::PassConfiguration &Config) override; + + Error notifyEmitted(MaterializationResponsibility &MR) override; + Error notifyFailed(MaterializationResponsibility &MR) override; + Error notifyRemovingResources(JITDylib &JD, ResourceKey K) override; + void notifyTransferringResources(JITDylib &JD, ResourceKey DstKey, + ResourceKey SrcKey) override; + + static Expected> + Create(ExecutorProcessControl &EPC, JITDylib &JD, bool EmitDebugInfo, + bool TestMode = false); + +private: + ExecutorProcessControl &EPC; + ExecutorAddr RegisterVTuneImplAddr; + ExecutorAddr UnregisterVTuneImplAddr; + std::mutex PluginMutex; + uint64_t NextMethodID = 0; + DenseMap> + PendingMethodIDs; + DenseMap>> + LoadedMethodIDs; + bool EmitDebugInfo; +}; + +} // end namespace orc + +} // end namespace llvm + +#endif diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/VTuneSharedStructs.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/VTuneSharedStructs.h new file mode 100644 index 000000000000..667d3446faff --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/VTuneSharedStructs.h @@ -0,0 +1,102 @@ +//===-------------------- VTuneSharedStructs.h ------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Structs and serialization to share VTune-related information +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_SHARED_VTUNESHAREDSTRUCTS_H +#define LLVM_EXECUTIONENGINE_ORC_SHARED_VTUNESHAREDSTRUCTS_H + +namespace llvm { +namespace orc { + +using VTuneLineTable = std::vector>; + +// SI = String Index, 1-indexed into the VTuneMethodBatch::Strings table. +// SI == 0 means replace with nullptr. + +// MI = Method Index, 1-indexed into the VTuneMethodBatch::Methods table. +// MI == 0 means this is a parent method and was not inlined. + +struct VTuneMethodInfo { + VTuneLineTable LineTable; + ExecutorAddr LoadAddr; + uint64_t LoadSize; + uint64_t MethodID; + uint32_t NameSI; + uint32_t ClassFileSI; + uint32_t SourceFileSI; + uint32_t ParentMI; +}; + +using VTuneMethodTable = std::vector; +using VTuneStringTable = std::vector; + +struct VTuneMethodBatch { + VTuneMethodTable Methods; + VTuneStringTable Strings; +}; + +using VTuneUnloadedMethodIDs = SmallVector>; + +namespace shared { + +using SPSVTuneLineTable = SPSSequence>; +using SPSVTuneMethodInfo = + SPSTuple; +using SPSVTuneMethodTable = SPSSequence; +using SPSVTuneStringTable = SPSSequence; +using SPSVTuneMethodBatch = SPSTuple; +using SPSVTuneUnloadedMethodIDs = SPSSequence>; + +template <> class SPSSerializationTraits { +public: + static size_t size(const VTuneMethodInfo &MI) { + return SPSVTuneMethodInfo::AsArgList::size( + MI.LineTable, MI.LoadAddr, MI.LoadSize, MI.MethodID, MI.NameSI, + MI.ClassFileSI, MI.SourceFileSI, MI.ParentMI); + } + + static bool deserialize(SPSInputBuffer &IB, VTuneMethodInfo &MI) { + return SPSVTuneMethodInfo::AsArgList::deserialize( + IB, MI.LineTable, MI.LoadAddr, MI.LoadSize, MI.MethodID, MI.NameSI, + MI.ClassFileSI, MI.SourceFileSI, MI.ParentMI); + } + + static bool serialize(SPSOutputBuffer &OB, const VTuneMethodInfo &MI) { + return SPSVTuneMethodInfo::AsArgList::serialize( + OB, MI.LineTable, MI.LoadAddr, MI.LoadSize, MI.MethodID, MI.NameSI, + MI.ClassFileSI, MI.SourceFileSI, MI.ParentMI); + } +}; + +template <> +class SPSSerializationTraits { +public: + static size_t size(const VTuneMethodBatch &MB) { + return SPSVTuneMethodBatch::AsArgList::size(MB.Methods, MB.Strings); + } + + static bool deserialize(SPSInputBuffer &IB, VTuneMethodBatch &MB) { + return SPSVTuneMethodBatch::AsArgList::deserialize(IB, MB.Methods, + MB.Strings); + } + + static bool serialize(SPSOutputBuffer &OB, const VTuneMethodBatch &MB) { + return SPSVTuneMethodBatch::AsArgList::serialize(OB, MB.Methods, + MB.Strings); + } +}; + +} // end namespace shared +} // end namespace orc +} // end namespace llvm + +#endif diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h new file mode 100644 index 000000000000..afb7df592faf --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h @@ -0,0 +1,31 @@ + +//===------ JITLoaderVTune.h --- Register profiler objects ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Register objects for access by profilers via the perf JIT interface. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_JITLOADERVTUNE_H +#define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_JITLOADERVTUNE_H + +#include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" +#include + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_registerVTuneImpl(const char *Data, uint64_t Size); + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_unregisterVTuneImpl(const char *Data, uint64_t Size); + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_test_registerVTuneImpl(const char *Data, uint64_t Size); + +#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_JITLOADERVTUNE_H + + diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/Debugging/CMakeLists.txt index 5bf23a7ec0bc..ed52692662a8 100644 --- a/llvm/lib/ExecutionEngine/Orc/Debugging/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/Orc/Debugging/CMakeLists.txt @@ -8,6 +8,7 @@ add_llvm_component_library(LLVMOrcDebugging DebuggerSupportPlugin.cpp LLJITUtilsCBindings.cpp PerfSupportPlugin.cpp + VTuneSupportPlugin.cpp ADDITIONAL_HEADER_DIRS ${LLVM_MAIN_INCLUDE_DIR}/llvm/ExecutionEngine/Orc/Debugging/ diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp new file mode 100644 index 000000000000..30a9728c8c20 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.cpp @@ -0,0 +1,185 @@ +//===--- VTuneSupportPlugin.cpp -- Support for VTune profiler --*- C++ -*--===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Handles support for registering code with VIntel Tune's Amplfiier JIT API. +// +//===----------------------------------------------------------------------===// +#include "llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h" +#include "llvm/DebugInfo/DWARF/DWARFContext.h" +#include "llvm/ExecutionEngine/Orc/Debugging/DebugInfoSupport.h" + +using namespace llvm; +using namespace llvm::orc; +using namespace llvm::jitlink; + +static constexpr StringRef RegisterVTuneImplName = "llvm_orc_registerVTuneImpl"; +static constexpr StringRef UnregisterVTuneImplName = + "llvm_orc_unregisterVTuneImpl"; +static constexpr StringRef RegisterTestVTuneImplName = + "llvm_orc_test_registerVTuneImpl"; + +static VTuneMethodBatch getMethodBatch(LinkGraph &G, bool EmitDebugInfo) { + VTuneMethodBatch Batch; + std::unique_ptr DC; + StringMap> DCBacking; + if (EmitDebugInfo) { + auto EDC = createDWARFContext(G); + if (!EDC) { + EmitDebugInfo = false; + } else { + DC = std::move(EDC->first); + DCBacking = std::move(EDC->second); + } + } + + auto GetStringIdx = [Deduplicator = StringMap(), + &Batch](StringRef S) mutable { + auto I = Deduplicator.find(S); + if (I != Deduplicator.end()) + return I->second; + + Batch.Strings.push_back(S.str()); + return Deduplicator[S] = Batch.Strings.size(); + }; + for (auto Sym : G.defined_symbols()) { + if (!Sym->isCallable()) + continue; + + Batch.Methods.push_back(VTuneMethodInfo()); + auto &Method = Batch.Methods.back(); + Method.MethodID = 0; + Method.ParentMI = 0; + Method.LoadAddr = Sym->getAddress(); + Method.LoadSize = Sym->getSize(); + Method.NameSI = GetStringIdx(Sym->getName()); + Method.ClassFileSI = 0; + Method.SourceFileSI = 0; + + if (!EmitDebugInfo) + continue; + + auto &Section = Sym->getBlock().getSection(); + auto Addr = Sym->getAddress(); + auto SAddr = + object::SectionedAddress{Addr.getValue(), Section.getOrdinal()}; + DILineInfoTable LinesInfo = DC->getLineInfoForAddressRange( + SAddr, Sym->getSize(), + DILineInfoSpecifier::FileLineInfoKind::AbsoluteFilePath); + Method.SourceFileSI = Batch.Strings.size(); + Batch.Strings.push_back(DC->getLineInfoForAddress(SAddr).FileName); + for (auto &LInfo : LinesInfo) { + Method.LineTable.push_back( + std::pair{/*unsigned*/ Sym->getOffset(), + /*DILineInfo*/ LInfo.second.Line}); + } + } + return Batch; +} + +void VTuneSupportPlugin::modifyPassConfig(MaterializationResponsibility &MR, + LinkGraph &G, + PassConfiguration &Config) { + Config.PostFixupPasses.push_back([this, MR = &MR](LinkGraph &G) { + // the object file is generated but not linked yet + auto Batch = getMethodBatch(G, EmitDebugInfo); + if (Batch.Methods.empty()) { + return Error::success(); + } + { + std::lock_guard Lock(PluginMutex); + uint64_t Allocated = Batch.Methods.size(); + uint64_t Start = NextMethodID; + NextMethodID += Allocated; + for (size_t i = Start; i < NextMethodID; ++i) { + Batch.Methods[i - Start].MethodID = i; + } + this->PendingMethodIDs[MR] = {Start, Allocated}; + } + G.allocActions().push_back( + {cantFail(shared::WrapperFunctionCall::Create< + shared::SPSArgList>( + RegisterVTuneImplAddr, Batch)), + {}}); + return Error::success(); + }); +} + +Error VTuneSupportPlugin::notifyEmitted(MaterializationResponsibility &MR) { + if (auto Err = MR.withResourceKeyDo([this, MR = &MR](ResourceKey K) { + std::lock_guard Lock(PluginMutex); + auto I = PendingMethodIDs.find(MR); + if (I == PendingMethodIDs.end()) + return; + + LoadedMethodIDs[K].push_back(I->second); + PendingMethodIDs.erase(I); + })) { + return Err; + } + return Error::success(); +} + +Error VTuneSupportPlugin::notifyFailed(MaterializationResponsibility &MR) { + std::lock_guard Lock(PluginMutex); + PendingMethodIDs.erase(&MR); + return Error::success(); +} + +Error VTuneSupportPlugin::notifyRemovingResources(JITDylib &JD, ResourceKey K) { + // Unregistration not required if not provided + if (!UnregisterVTuneImplAddr) { + return Error::success(); + } + VTuneUnloadedMethodIDs UnloadedIDs; + { + std::lock_guard Lock(PluginMutex); + auto I = LoadedMethodIDs.find(K); + if (I == LoadedMethodIDs.end()) + return Error::success(); + + UnloadedIDs = std::move(I->second); + LoadedMethodIDs.erase(I); + } + if (auto Err = EPC.callSPSWrapper( + UnregisterVTuneImplAddr, UnloadedIDs)) + return Err; + + return Error::success(); +} + +void VTuneSupportPlugin::notifyTransferringResources(JITDylib &JD, + ResourceKey DstKey, + ResourceKey SrcKey) { + std::lock_guard Lock(PluginMutex); + auto I = LoadedMethodIDs.find(SrcKey); + if (I == LoadedMethodIDs.end()) + return; + + auto &Dest = LoadedMethodIDs[DstKey]; + Dest.insert(Dest.end(), I->second.begin(), I->second.end()); + LoadedMethodIDs.erase(SrcKey); +} + +Expected> +VTuneSupportPlugin::Create(ExecutorProcessControl &EPC, JITDylib &JD, + bool EmitDebugInfo, bool TestMode) { + auto &ES = EPC.getExecutionSession(); + auto RegisterImplName = + ES.intern(TestMode ? RegisterTestVTuneImplName : RegisterVTuneImplName); + auto UnregisterImplName = ES.intern(UnregisterVTuneImplName); + SymbolLookupSet SLS{RegisterImplName, UnregisterImplName}; + auto Res = ES.lookup(makeJITDylibSearchOrder({&JD}), std::move(SLS)); + if (!Res) + return Res.takeError(); + ExecutorAddr RegisterImplAddr( + Res->find(RegisterImplName)->second.getAddress()); + ExecutorAddr UnregisterImplAddr( + Res->find(UnregisterImplName)->second.getAddress()); + return std::make_unique( + EPC, RegisterImplAddr, UnregisterImplAddr, EmitDebugInfo); +} diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt index f2005dc1775e..3d1dfe758c79 100644 --- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/CMakeLists.txt @@ -2,10 +2,18 @@ if( CMAKE_HOST_UNIX AND HAVE_LIBRT ) set(rt_lib rt) endif() +set(intel_jit_profiling ) +if( LLVM_USE_INTEL_JITEVENTS ) + set(intel_jit_profiling IntelJITProfiling) + include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../IntelJITProfiling) + include_directories(${PROJECT_BINARY_DIR}/ittapi/include/ ) +endif() + add_llvm_component_library(LLVMOrcTargetProcess ExecutorSharedMemoryMapperService.cpp JITLoaderGDB.cpp JITLoaderPerf.cpp + JITLoaderVTune.cpp OrcRTBootstrap.cpp RegisterEHFrames.cpp SimpleExecutorDylibManager.cpp @@ -21,6 +29,7 @@ add_llvm_component_library(LLVMOrcTargetProcess ${rt_lib} LINK_COMPONENTS + ${intel_jit_profiling} OrcShared Support TargetParser diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.cpp new file mode 100644 index 000000000000..d346214d3ae2 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.cpp @@ -0,0 +1,224 @@ +//===------- JITLoaderVTune.cpp - Register profiler objects -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Register objects for access by profilers via the VTune JIT interface. +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h" +#include "llvm/ExecutionEngine/Orc/Shared/VTuneSharedStructs.h" +#include + +#if LLVM_USE_INTEL_JITEVENTS +#include "IntelJITEventsWrapper.h" +#include "ittnotify.h" + +using namespace llvm; +using namespace llvm::orc; + +namespace { +class JITEventWrapper { +public: + static std::unique_ptr Wrapper; +}; +std::unique_ptr JITEventWrapper::Wrapper; +} // namespace + +static Error registerJITLoaderVTuneRegisterImpl(const VTuneMethodBatch &MB) { + const size_t StringsSize = MB.Strings.size(); + + for (const auto &MethodInfo : MB.Methods) { + iJIT_Method_Load MethodMessage; + memset(&MethodMessage, 0, sizeof(iJIT_Method_Load)); + + MethodMessage.method_id = MethodInfo.MethodID; + if (MethodInfo.NameSI != 0 && MethodInfo.NameSI < StringsSize) { + MethodMessage.method_name = + const_cast(MB.Strings.at(MethodInfo.NameSI).data()); + } else { + MethodMessage.method_name = NULL; + } + if (MethodInfo.ClassFileSI != 0 && MethodInfo.ClassFileSI < StringsSize) { + MethodMessage.class_file_name = + const_cast(MB.Strings.at(MethodInfo.ClassFileSI).data()); + } else { + MethodMessage.class_file_name = NULL; + } + if (MethodInfo.SourceFileSI != 0 && MethodInfo.SourceFileSI < StringsSize) { + MethodMessage.source_file_name = + const_cast(MB.Strings.at(MethodInfo.SourceFileSI).data()); + } else { + MethodMessage.source_file_name = NULL; + } + + MethodMessage.method_load_address = MethodInfo.LoadAddr.toPtr(); + MethodMessage.method_size = MethodInfo.LoadSize; + MethodMessage.class_id = 0; + + MethodMessage.user_data = NULL; + MethodMessage.user_data_size = 0; + MethodMessage.env = iJDE_JittingAPI; + + std::vector LineInfo; + for (const auto &LInfo : MethodInfo.LineTable) { + LineInfo.push_back(LineNumberInfo{LInfo.first, LInfo.second}); + } + + if (LineInfo.size() == 0) { + MethodMessage.line_number_size = 0; + MethodMessage.line_number_table = 0; + } else { + MethodMessage.line_number_size = LineInfo.size(); + MethodMessage.line_number_table = &*LineInfo.begin(); + } + JITEventWrapper::Wrapper->iJIT_NotifyEvent( + iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, &MethodMessage); + } + + return Error::success(); +} + +static void registerJITLoaderVTuneUnregisterImpl( + const std::vector> &UM) { + for (auto &Method : UM) { + JITEventWrapper::Wrapper->iJIT_NotifyEvent( + iJVM_EVENT_TYPE_METHOD_UNLOAD_START, + const_cast(&Method.first)); + } +} + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_registerVTuneImpl(const char *Data, uint64_t Size) { + using namespace orc::shared; + if (!JITEventWrapper::Wrapper) + JITEventWrapper::Wrapper.reset(new IntelJITEventsWrapper); + + return WrapperFunction::handle( + Data, Size, registerJITLoaderVTuneRegisterImpl) + .release(); +} + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_unregisterVTuneImpl(const char *Data, uint64_t Size) { + using namespace orc::shared; + return WrapperFunction::handle( + Data, Size, registerJITLoaderVTuneUnregisterImpl) + .release(); +} + +// For Testing: following code comes from llvm-jitlistener.cpp in llvm tools +namespace { +using SourceLocations = std::vector>; +using NativeCodeMap = std::map; +NativeCodeMap ReportedDebugFuncs; +} // namespace + +static int NotifyEvent(iJIT_JVM_EVENT EventType, void *EventSpecificData) { + switch (EventType) { + case iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED: { + if (!EventSpecificData) { + errs() << "Error: The JIT event listener did not provide a event data."; + return -1; + } + iJIT_Method_Load *msg = static_cast(EventSpecificData); + + ReportedDebugFuncs[msg->method_id]; + + outs() << "Method load [" << msg->method_id << "]: " << msg->method_name + << ", Size = " << msg->method_size << "\n"; + + for (unsigned int i = 0; i < msg->line_number_size; ++i) { + if (!msg->line_number_table) { + errs() << "A function with a non-zero line count had no line table."; + return -1; + } + std::pair loc( + std::string(msg->source_file_name), + msg->line_number_table[i].LineNumber); + ReportedDebugFuncs[msg->method_id].push_back(loc); + outs() << " Line info @ " << msg->line_number_table[i].Offset << ": " + << msg->source_file_name << ", line " + << msg->line_number_table[i].LineNumber << "\n"; + } + outs() << "\n"; + } break; + case iJVM_EVENT_TYPE_METHOD_UNLOAD_START: { + if (!EventSpecificData) { + errs() << "Error: The JIT event listener did not provide a event data."; + return -1; + } + unsigned int UnloadId = + *reinterpret_cast(EventSpecificData); + assert(1 == ReportedDebugFuncs.erase(UnloadId)); + outs() << "Method unload [" << UnloadId << "]\n"; + } break; + default: + break; + } + return 0; +} + +static iJIT_IsProfilingActiveFlags IsProfilingActive(void) { + // for testing, pretend we have an Intel Parallel Amplifier XE 2011 + // instance attached + return iJIT_SAMPLING_ON; +} + +static unsigned int GetNewMethodID(void) { + static unsigned int id = 0; + return ++id; +} + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_test_registerVTuneImpl(const char *Data, uint64_t Size) { + using namespace orc::shared; + JITEventWrapper::Wrapper.reset(new IntelJITEventsWrapper( + NotifyEvent, NULL, NULL, IsProfilingActive, 0, 0, GetNewMethodID)); + return WrapperFunction::handle( + Data, Size, registerJITLoaderVTuneRegisterImpl) + .release(); +} + +#else + +using namespace llvm; +using namespace llvm::orc; + +static Error unsupportedBatch(const VTuneMethodBatch &MB) { + return llvm::make_error("unsupported for Intel VTune", + inconvertibleErrorCode()); +} + +static void unsuppported(const std::vector> &UM) { + +} + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_registerVTuneImpl(const char *Data, uint64_t Size) { + using namespace orc::shared; + return WrapperFunction::handle( + Data, Size, unsupportedBatch) + .release(); +} + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_unregisterVTuneImpl(const char *Data, uint64_t Size) { + using namespace orc::shared; + return WrapperFunction::handle(Data, Size, + unsuppported) + .release(); +} + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_test_registerVTuneImpl(const char *Data, uint64_t Size) { + using namespace orc::shared; + return WrapperFunction::handle( + Data, Size, unsupportedBatch) + .release(); +} + +#endif diff --git a/llvm/test/ExecutionEngine/JITLink/x86-64/ELF_vtune.s b/llvm/test/ExecutionEngine/JITLink/x86-64/ELF_vtune.s new file mode 100644 index 000000000000..1c95bde51e12 --- /dev/null +++ b/llvm/test/ExecutionEngine/JITLink/x86-64/ELF_vtune.s @@ -0,0 +1,52 @@ +# REQUIRES: native && x86_64-linux && intel-jitevents + +# RUN: rm -rf %t && mkdir %t +# RUN: llvm-mc -triple=x86_64-unknown-linux \ +# RUN: -filetype=obj -o %t/ELF_x86-64_vtune.o %s +# RUN: llvm-jitlink -vtune-support %t/ELF_x86-64_vtune.o | \ +# RUN: FileCheck %s + +# CHECK: Method load [0]: {{.*}}, Size = {{[0-9]+}} +# CHECK: Method unload [0] + .file "test.c" + .text + .globl main + .type main, @function +main: +.LFB0: + .cfi_startproc + endbr64 + pushq %rbp + .cfi_def_cfa_offset 16 + .cfi_offset 6, -16 + movq %rsp, %rbp + .cfi_def_cfa_register 6 + movl %edi, -4(%rbp) + movq %rsi, -16(%rbp) + movl -4(%rbp), %ebx + addl $1, %ebx + movl $0, %eax + popq %rbp + .cfi_def_cfa 7, 8 + ret + .cfi_endproc +.LFE0: + .size main, .-main + .ident "GCC: (Ubuntu 9.4.0-1ubuntu1~20.04.2) 9.4.0" + .section .note.GNU-stack,"",@progbits + .section .note.gnu.property,"a" + .align 8 + .long 1f - 0f + .long 4f - 1f + .long 5 +0: + .string "GNU" +1: + .align 8 + .long 0xc0000002 + .long 3f - 2f +2: + .long 0x3 +3: + .align 8 +4: diff --git a/llvm/test/ExecutionEngine/JITLink/x86-64/lit.local.cfg b/llvm/test/ExecutionEngine/JITLink/x86-64/lit.local.cfg index 42bf50dcc13c..d5a1ad626b65 100644 --- a/llvm/test/ExecutionEngine/JITLink/x86-64/lit.local.cfg +++ b/llvm/test/ExecutionEngine/JITLink/x86-64/lit.local.cfg @@ -1,2 +1,5 @@ if not "X86" in config.root.targets: config.unsupported = True + +if config.llvm_use_intel_jitevents: + config.available_features.add("intel-jitevents") diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp index f0b8310a32ef..09b2a5900eb0 100644 --- a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp +++ b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp @@ -21,6 +21,7 @@ #include "llvm/ExecutionEngine/Orc/Debugging/DebugInfoSupport.h" #include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.h" #include "llvm/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.h" +#include "llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h" #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h" #include "llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h" #include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h" @@ -34,6 +35,7 @@ #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderPerf.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" @@ -148,6 +150,10 @@ static cl::opt PerfSupport("perf-support", cl::init(false), cl::Hidden, cl::cat(JITLinkCategory)); +static cl::opt VTuneSupport("vtune-support", + cl::desc("Enable vtune profiling support"), + cl::init(false), cl::Hidden, + cl::cat(JITLinkCategory)); static cl::opt NoProcessSymbols("no-process-syms", cl::desc("Do not resolve to llvm-jitlink process symbols"), @@ -264,7 +270,10 @@ static LLVM_ATTRIBUTE_USED void linkComponents() { << (void *)&llvm_orc_registerJITLoaderGDBAllocAction << '\n' << (void *)&llvm_orc_registerJITLoaderPerfStart << '\n' << (void *)&llvm_orc_registerJITLoaderPerfEnd << '\n' - << (void *)&llvm_orc_registerJITLoaderPerfImpl << '\n'; + << (void *)&llvm_orc_registerJITLoaderPerfImpl << '\n' + << (void *)&llvm_orc_registerVTuneImpl << '\n' + << (void *)&llvm_orc_unregisterVTuneImpl << '\n' + << (void *)&llvm_orc_test_registerVTuneImpl << '\n'; } static bool UseTestResultOverride = false; @@ -1004,6 +1013,14 @@ Session::Session(std::unique_ptr EPC, Error &Err) this->ES.getExecutorProcessControl(), *ProcessSymsJD, true, true))); } + if (VTuneSupport && TT.isOSBinFormatELF()) { + ObjLayer.addPlugin(ExitOnErr(DebugInfoPreservationPlugin::Create())); + ObjLayer.addPlugin(ExitOnErr( + VTuneSupportPlugin::Create(this->ES.getExecutorProcessControl(), + *ProcessSymsJD, /*EmitDebugInfo=*/true, + /*TestMode=*/true))); + } + // Set up the platform. if (!OrcRuntime.empty()) { assert(ProcessSymsJD && "ProcessSymsJD should have been set"); -- GitLab From d64632becd159a4b816af2d5a01fd3531bd45f65 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Thu, 7 Mar 2024 19:18:25 +0000 Subject: [PATCH 489/929] [gn build] Port 00f412168cf6 --- .../gn/secondary/llvm/lib/ExecutionEngine/Orc/Debugging/BUILD.gn | 1 + .../llvm/lib/ExecutionEngine/Orc/TargetProcess/BUILD.gn | 1 + 2 files changed, 2 insertions(+) diff --git a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Debugging/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Debugging/BUILD.gn index 1d3fc6cfdfaa..5610679ff333 100644 --- a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Debugging/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Debugging/BUILD.gn @@ -13,6 +13,7 @@ static_library("Debugging") { "DebuggerSupportPlugin.cpp", "LLJITUtilsCBindings.cpp", "PerfSupportPlugin.cpp", + "VTuneSupportPlugin.cpp", ] if (current_os == "linux") { libs = [ "rt" ] diff --git a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/TargetProcess/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/TargetProcess/BUILD.gn index f34855e8e1cc..d62f5042c946 100644 --- a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/TargetProcess/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/TargetProcess/BUILD.gn @@ -9,6 +9,7 @@ static_library("TargetProcess") { "ExecutorSharedMemoryMapperService.cpp", "JITLoaderGDB.cpp", "JITLoaderPerf.cpp", + "JITLoaderVTune.cpp", "OrcRTBootstrap.cpp", "RegisterEHFrames.cpp", "SimpleExecutorDylibManager.cpp", -- GitLab From 4d31fbbb5af6528387fd5efd90363a408713108b Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Thu, 7 Mar 2024 11:07:59 -0800 Subject: [PATCH 490/929] [ORC] Propagate defineMaterializing failure when resource tracker is defunct. Remove an overly aggressive cantFail: This call to defineMaterializing should never fail with a duplicate symbols error (since all new symbols shoul be weak), but may fail if the tracker has become defunct in the mean time. In that case we need to propagate the error. --- llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp index fffa95ee72b7..6ac256dff9b4 100644 --- a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -455,9 +455,10 @@ private: ProcessSymbol(Sym); // Attempt to claim all weak defs that we're not already responsible for. - // This cannot fail -- any clashes will just result in rejection of our - // claim, at which point we'll externalize that symbol. - cantFail(MR->defineMaterializing(std::move(NewSymbolsToClaim))); + // This may fail if the resource tracker has become defunct, but should + // always succeed otherwise. + if (auto Err = MR->defineMaterializing(std::move(NewSymbolsToClaim))) + return Err; // Walk the list of symbols that we just tried to claim. Symbols that we're // responsible for are marked live. Symbols that we're not responsible for -- GitLab From 9286665f7667a00062ef7bb8d1eec0fa02d2602c Mon Sep 17 00:00:00 2001 From: Changpeng Fang Date: Thu, 7 Mar 2024 11:43:33 -0800 Subject: [PATCH 491/929] AMDGPU: Use OtherPredicates for v_dot2_bf16_bf16(f16_f16) pseudo (#84354) This is because SubtargetPredicate is not copied from pseudo to dpp16 and dpp8 real. Actually this is the common issue for insts with _Realtriple_ --- We should avoid using SubtargetPredicate to define pseudo: the predicate will be lost in real. --- llvm/lib/Target/AMDGPU/VOP3Instructions.td | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td index 334cfad478f1..3340ded9d360 100644 --- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td +++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td @@ -893,7 +893,7 @@ let SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0 in { defm V_MINIMUMMAXIMUM_F16 : VOP3Inst<"v_minimummaximum_f16", VOP3_Profile>; } // End SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0 -let SubtargetPredicate = HasDot9Insts, IsDOT=1 in { +let OtherPredicates = [HasDot9Insts], IsDOT=1 in { defm V_DOT2_F16_F16 : VOP3Inst<"v_dot2_f16_f16", VOP3_DOT_Profile, int_amdgcn_fdot2_f16_f16>; defm V_DOT2_BF16_BF16 : VOP3Inst<"v_dot2_bf16_bf16", VOP3_DOT_Profile, int_amdgcn_fdot2_bf16_bf16>; } -- GitLab From b8b434b3e150f1c79b114893e36f8e447e560b80 Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Thu, 7 Mar 2024 20:46:10 +0100 Subject: [PATCH 492/929] [GISEL] Silence unused variable warning. NFC --- llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp index a5a136e2effc..28e5bf85ca9c 100644 --- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -744,8 +744,7 @@ MachineInstrBuilder MachineIRBuilder::buildShuffleSplat(const DstOp &Res, MachineInstrBuilder MachineIRBuilder::buildSplatVector(const DstOp &Res, const SrcOp &Src) { - LLT DstTy = Res.getLLTTy(*getMRI()); - assert(Src.getLLTTy(*getMRI()) == DstTy.getElementType() && + assert(Src.getLLTTy(*getMRI()) == Res.getLLTTy(*getMRI()).getElementType() && "Expected Src to match Dst elt ty"); return buildInstr(TargetOpcode::G_SPLAT_VECTOR, Res, Src); } -- GitLab From 4f85f620b6a92e657dc9f719158dbdcae561ead7 Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Thu, 7 Mar 2024 20:54:02 +0100 Subject: [PATCH 493/929] [bazel] Port 3714f937b835c06c8c32ca4f3f61ba2317db2296 --- utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 9d6ca4ed932f..7a6bc2dc3202 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -542,6 +542,7 @@ mlir_c_api_cc_library( includes = ["include"], deps = [ ":LLVMDialect", + "//llvm:Support", ], ) -- GitLab From 57a337378f37fa3813992842714c9b06fae20af2 Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Thu, 7 Mar 2024 14:57:35 -0500 Subject: [PATCH 494/929] [libc][c23] add memset_explicit (#83577) --- libc/config/linux/aarch64/entrypoints.txt | 1 + libc/config/linux/x86_64/entrypoints.txt | 1 + libc/spec/stdc.td | 5 +++ libc/src/string/CMakeLists.txt | 11 +++++++ libc/src/string/memset_explicit.cpp | 25 +++++++++++++++ libc/src/string/memset_explicit.h | 20 ++++++++++++ libc/test/src/string/CMakeLists.txt | 10 ++++++ libc/test/src/string/memset_explicit_test.cpp | 31 +++++++++++++++++++ 8 files changed, 104 insertions(+) create mode 100644 libc/src/string/memset_explicit.cpp create mode 100644 libc/src/string/memset_explicit.h create mode 100644 libc/test/src/string/memset_explicit_test.cpp diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt index 06832a41221d..c32773f67cda 100644 --- a/libc/config/linux/aarch64/entrypoints.txt +++ b/libc/config/linux/aarch64/entrypoints.txt @@ -51,6 +51,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.string.mempcpy libc.src.string.memrchr libc.src.string.memset + libc.src.string.memset_explicit libc.src.string.rindex libc.src.string.stpcpy libc.src.string.stpncpy diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index bd2006ddb7e9..1f36f127e3c4 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -51,6 +51,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.string.mempcpy libc.src.string.memrchr libc.src.string.memset + libc.src.string.memset_explicit libc.src.string.rindex libc.src.string.stpcpy libc.src.string.stpncpy diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index a3a856a4ec85..e09cce0efd9b 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -234,6 +234,11 @@ def StdC : StandardSpec<"stdc"> { RetValSpec, [ArgSpec, ArgSpec, ArgSpec] >, + FunctionSpec< + "memset_explicit", + RetValSpec, + [ArgSpec, ArgSpec, ArgSpec] + >, FunctionSpec< "strcpy", RetValSpec, diff --git a/libc/src/string/CMakeLists.txt b/libc/src/string/CMakeLists.txt index 1c893280e8a3..56588ffafb86 100644 --- a/libc/src/string/CMakeLists.txt +++ b/libc/src/string/CMakeLists.txt @@ -441,6 +441,17 @@ add_entrypoint_object( .memory_utils.inline_memcpy ) +add_entrypoint_object( + memset_explicit + SRCS + memset_explicit.cpp + HDRS + memset_explicit.h + DEPENDS + .string_utils + .memory_utils.inline_memset +) + # Helper to define a function with multiple implementations # - Computes flags to satisfy required/rejected features and arch, # - Declares an entry point, diff --git a/libc/src/string/memset_explicit.cpp b/libc/src/string/memset_explicit.cpp new file mode 100644 index 000000000000..a8656d1e791e --- /dev/null +++ b/libc/src/string/memset_explicit.cpp @@ -0,0 +1,25 @@ +//===-- Implementation of memset_explicit ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/string/memset_explicit.h" +#include "src/__support/common.h" +#include "src/string/memory_utils/inline_memset.h" + +namespace LIBC_NAMESPACE { + +[[gnu::noinline]] LLVM_LIBC_FUNCTION(void *, memset_explicit, + (void *dst, int value, size_t count)) { + // Use the inline memset function to set the memory. + inline_memset(dst, static_cast(value), count); + // avoid dead store elimination + // The asm itself should also be sufficient to behave as a compiler barrier. + asm("" : : "r"(dst) : "memory"); + return dst; +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/string/memset_explicit.h b/libc/src/string/memset_explicit.h new file mode 100644 index 000000000000..f6c189761a12 --- /dev/null +++ b/libc/src/string/memset_explicit.h @@ -0,0 +1,20 @@ +//===-- Implementation header for memset_explicit ---------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STRING_MEMSET_EXPLICIT_H +#define LLVM_LIBC_SRC_STRING_MEMSET_EXPLICIT_H + +#include // size_t + +namespace LIBC_NAMESPACE { + +[[gnu::noinline]] void *memset_explicit(void *ptr, int value, size_t count); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STRING_MEMSET_EXPLICIT_H diff --git a/libc/test/src/string/CMakeLists.txt b/libc/test/src/string/CMakeLists.txt index 6088289532d7..c1caec5fd912 100644 --- a/libc/test/src/string/CMakeLists.txt +++ b/libc/test/src/string/CMakeLists.txt @@ -418,6 +418,16 @@ add_libc_test( libc.src.string.strxfrm ) +add_libc_test( + memset_explicit_test + SUITE + libc-string-tests + SRCS + memset_explicit_test.cpp + DEPENDS + libc.src.string.memset_explicit +) + # Tests all implementations that can run on the target CPU. function(add_libc_multi_impl_test name) get_property(fq_implementations GLOBAL PROPERTY ${name}_implementations) diff --git a/libc/test/src/string/memset_explicit_test.cpp b/libc/test/src/string/memset_explicit_test.cpp new file mode 100644 index 000000000000..bb5111bd639e --- /dev/null +++ b/libc/test/src/string/memset_explicit_test.cpp @@ -0,0 +1,31 @@ +//===-- Unittests for memset_explicit -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "memory_utils/memory_check_utils.h" +#include "src/string/memset_explicit.h" +#include "test/UnitTest/Test.h" + +namespace LIBC_NAMESPACE { + +// Apply the same tests as memset + +static inline void Adaptor(cpp::span p1, uint8_t value, size_t size) { + LIBC_NAMESPACE::memset_explicit(p1.begin(), value, size); +} + +TEST(LlvmLibcmemsetExplicitTest, SizeSweep) { + static constexpr size_t kMaxSize = 400; + Buffer DstBuffer(kMaxSize); + for (size_t size = 0; size < kMaxSize; ++size) { + const char value = size % 10; + auto dst = DstBuffer.span().subspan(0, size); + ASSERT_TRUE((CheckMemset(dst, value, size))); + } +} + +} // namespace LIBC_NAMESPACE -- GitLab From 308a2360725948fd6c77d005110c169ab1a8322c Mon Sep 17 00:00:00 2001 From: Clement Courbet Date: Thu, 7 Mar 2024 21:01:46 +0100 Subject: [PATCH 495/929] [clang-tidy] `isOnlyUsedAsConst`: Handle static method calls. (#84005) ... using method syntax: ``` struct S { static void f() }; void DoIt(S& s) { s.f(); // Does not mutate `s` through the `this` parameter. } ``` --- clang-tools-extra/clang-tidy/utils/DeclRefExprUtils.cpp | 9 +++++---- .../unittests/clang-tidy/DeclRefExprUtilsTest.cpp | 9 +++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/clang-tools-extra/clang-tidy/utils/DeclRefExprUtils.cpp b/clang-tools-extra/clang-tidy/utils/DeclRefExprUtils.cpp index f0ffa517047b..a48e45e13568 100644 --- a/clang-tools-extra/clang-tidy/utils/DeclRefExprUtils.cpp +++ b/clang-tools-extra/clang-tidy/utils/DeclRefExprUtils.cpp @@ -155,15 +155,16 @@ AST_MATCHER_P(DeclRefExpr, doesNotMutateObject, int, Indirections) { if (const auto *const Member = dyn_cast(P)) { if (const auto *const Method = dyn_cast(Member->getMemberDecl())) { - if (!Method->isConst()) { - // The method can mutate our variable. - return false; + if (Method->isConst() || Method->isStatic()) { + // The method call cannot mutate our variable. + continue; } - continue; + return false; } Stack.emplace_back(Member, 0); continue; } + if (const auto *const Op = dyn_cast(P)) { switch (Op->getOpcode()) { case UO_AddrOf: diff --git a/clang-tools-extra/unittests/clang-tidy/DeclRefExprUtilsTest.cpp b/clang-tools-extra/unittests/clang-tidy/DeclRefExprUtilsTest.cpp index 4c9e81ea0f61..3d9f51e2e17b 100644 --- a/clang-tools-extra/unittests/clang-tidy/DeclRefExprUtilsTest.cpp +++ b/clang-tools-extra/unittests/clang-tidy/DeclRefExprUtilsTest.cpp @@ -51,6 +51,8 @@ template void RunTest(StringRef Snippet) { void constMethod() const; void nonConstMethod(); + static void staticMethod(); + void operator()(ConstTag) const; void operator()(NonConstTag); @@ -109,10 +111,12 @@ TEST(ConstReferenceDeclRefExprsTest, ConstValueVar) { useConstPtr(&/*const*/target); useConstPtrConstRef(&/*const*/target); /*const*/target.constMethod(); + /*const*/target.staticMethod(); /*const*/target(ConstTag{}); /*const*/target[42]; useConstRef((/*const*/target)); (/*const*/target).constMethod(); + /*const*/target.staticMethod(); (void)(/*const*/target == /*const*/target); (void)/*const*/target; (void)&/*const*/target; @@ -140,6 +144,7 @@ TEST(ConstReferenceDeclRefExprsTest, ConstRefVar) { useConstPtr(&/*const*/target); useConstPtrConstRef(&/*const*/target); /*const*/target.constMethod(); + /*const*/target.staticMethod(); /*const*/target(ConstTag{}); /*const*/target[42]; useConstRef((/*const*/target)); @@ -179,6 +184,7 @@ TEST(ConstReferenceDeclRefExprsTest, ValueVar) { useConstPtr(&/*const*/target); useConstPtrConstRef(&/*const*/target); /*const*/target.constMethod(); + /*const*/target.staticMethod(); target.nonConstMethod(); /*const*/target(ConstTag{}); target[42]; @@ -218,6 +224,7 @@ TEST(ConstReferenceDeclRefExprsTest, RefVar) { useConstPtr(&/*const*/target); useConstPtrConstRef(&/*const*/target); /*const*/target.constMethod(); + /*const*/target.staticMethod(); target.nonConstMethod(); /*const*/target(ConstTag{}); target[42]; @@ -256,6 +263,7 @@ TEST(ConstReferenceDeclRefExprsTest, PtrVar) { useConstPtrConstRef(/*const*/target); usePtrConstPtr(&target); /*const*/target->constMethod(); + /*const*/target->staticMethod(); target->nonConstMethod(); (*/*const*/target)(ConstTag{}); (*target)[42]; @@ -292,6 +300,7 @@ TEST(ConstReferenceDeclRefExprsTest, ConstPtrVar) { useConstPtrConstPtr(&/*const*/target); useConstPtrConstRef(/*const*/target); /*const*/target->constMethod(); + /*const*/target->staticMethod(); (*/*const*/target)(ConstTag{}); (*/*const*/target)[42]; /*const*/target->operator[](42); -- GitLab From a10fd16270b6fecf99b793318872e208c8b1abab Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Thu, 7 Mar 2024 11:43:35 -0800 Subject: [PATCH 496/929] [CVP] Add test coverage for an upcoming generalization of expandUDivOrURem --- .../udiv-expansion.ll | 68 ++++++++++++++ .../urem-expansion.ll | 93 +++++++++++++++++++ 2 files changed, 161 insertions(+) diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/udiv-expansion.ll b/llvm/test/Transforms/CorrelatedValuePropagation/udiv-expansion.ll index a2a767084fbf..a5fc26ebab00 100644 --- a/llvm/test/Transforms/CorrelatedValuePropagation/udiv-expansion.ll +++ b/llvm/test/Transforms/CorrelatedValuePropagation/udiv-expansion.ll @@ -90,6 +90,74 @@ define i8 @constant.divisor.v7(i8 %x) { ret i8 %div } +define i8 @constant.divisor.v6to8(i8 %x) { +; CHECK-LABEL: @constant.divisor.v6to8( +; CHECK-NEXT: [[CMP_X_LOWER:%.*]] = icmp uge i8 [[X:%.*]], 6 +; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_X_LOWER]]) +; CHECK-NEXT: [[CMP_X_UPPER:%.*]] = icmp ult i8 [[X]], 9 +; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_X_UPPER]]) +; CHECK-NEXT: [[DIV:%.*]] = udiv i8 [[X]], 3 +; CHECK-NEXT: ret i8 2 +; + %cmp.x.lower = icmp uge i8 %x, 6 + call void @llvm.assume(i1 %cmp.x.lower) + %cmp.x.upper = icmp ult i8 %x, 9 + call void @llvm.assume(i1 %cmp.x.upper) + %div = udiv i8 %x, 3 + ret i8 %div +} + +define i8 @constant.divisor.v9to11(i8 %x) { +; CHECK-LABEL: @constant.divisor.v9to11( +; CHECK-NEXT: [[CMP_X_LOWER:%.*]] = icmp uge i8 [[X:%.*]], 9 +; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_X_LOWER]]) +; CHECK-NEXT: [[CMP_X_UPPER:%.*]] = icmp ult i8 [[X]], 12 +; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_X_UPPER]]) +; CHECK-NEXT: [[DIV:%.*]] = udiv i8 [[X]], 3 +; CHECK-NEXT: ret i8 3 +; + %cmp.x.lower = icmp uge i8 %x, 9 + call void @llvm.assume(i1 %cmp.x.lower) + %cmp.x.upper = icmp ult i8 %x, 12 + call void @llvm.assume(i1 %cmp.x.upper) + %div = udiv i8 %x, 3 + ret i8 %div +} + +define i8 @constant.divisor.v12to14(i8 %x) { +; CHECK-LABEL: @constant.divisor.v12to14( +; CHECK-NEXT: [[CMP_X_LOWER:%.*]] = icmp uge i8 [[X:%.*]], 12 +; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_X_LOWER]]) +; CHECK-NEXT: [[CMP_X_UPPER:%.*]] = icmp ult i8 [[X]], 15 +; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_X_UPPER]]) +; CHECK-NEXT: [[DIV:%.*]] = udiv i8 [[X]], 3 +; CHECK-NEXT: ret i8 4 +; + %cmp.x.lower = icmp uge i8 %x, 12 + call void @llvm.assume(i1 %cmp.x.lower) + %cmp.x.upper = icmp ult i8 %x, 15 + call void @llvm.assume(i1 %cmp.x.upper) + %div = udiv i8 %x, 3 + ret i8 %div +} + +define i8 @constant.divisor.v6to11(i8 %x) { +; CHECK-LABEL: @constant.divisor.v6to11( +; CHECK-NEXT: [[CMP_X_LOWER:%.*]] = icmp uge i8 [[X:%.*]], 6 +; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_X_LOWER]]) +; CHECK-NEXT: [[CMP_X_UPPER:%.*]] = icmp ult i8 [[X]], 12 +; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_X_UPPER]]) +; CHECK-NEXT: [[DIV:%.*]] = udiv i8 [[X]], 3 +; CHECK-NEXT: ret i8 [[DIV]] +; + %cmp.x.lower = icmp uge i8 %x, 6 + call void @llvm.assume(i1 %cmp.x.lower) + %cmp.x.upper = icmp ult i8 %x, 12 + call void @llvm.assume(i1 %cmp.x.upper) + %div = udiv i8 %x, 3 + ret i8 %div +} + ; Both are variable. Bounds are known define i8 @variable.v3(i8 %x, i8 %y) { diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll b/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll index cd0ba2f189dc..2af8c8f23bbd 100644 --- a/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll +++ b/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll @@ -100,6 +100,74 @@ define i8 @constant.divisor.v7(i8 %x) { ret i8 %rem } +define i8 @constant.divisor.v6to8(i8 %x) { +; CHECK-LABEL: @constant.divisor.v6to8( +; CHECK-NEXT: [[CMP_X_LOWER:%.*]] = icmp uge i8 [[X:%.*]], 6 +; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_X_LOWER]]) +; CHECK-NEXT: [[CMP_X_UPPER:%.*]] = icmp ult i8 [[X]], 9 +; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_X_UPPER]]) +; CHECK-NEXT: [[REM:%.*]] = urem i8 [[X]], 3 +; CHECK-NEXT: ret i8 [[REM]] +; + %cmp.x.lower = icmp uge i8 %x, 6 + call void @llvm.assume(i1 %cmp.x.lower) + %cmp.x.upper = icmp ult i8 %x, 9 + call void @llvm.assume(i1 %cmp.x.upper) + %rem = urem i8 %x, 3 + ret i8 %rem +} + +define i8 @constant.divisor.v9to11(i8 %x) { +; CHECK-LABEL: @constant.divisor.v9to11( +; CHECK-NEXT: [[CMP_X_LOWER:%.*]] = icmp uge i8 [[X:%.*]], 9 +; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_X_LOWER]]) +; CHECK-NEXT: [[CMP_X_UPPER:%.*]] = icmp ult i8 [[X]], 12 +; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_X_UPPER]]) +; CHECK-NEXT: [[REM:%.*]] = urem i8 [[X]], 3 +; CHECK-NEXT: ret i8 [[REM]] +; + %cmp.x.lower = icmp uge i8 %x, 9 + call void @llvm.assume(i1 %cmp.x.lower) + %cmp.x.upper = icmp ult i8 %x, 12 + call void @llvm.assume(i1 %cmp.x.upper) + %rem = urem i8 %x, 3 + ret i8 %rem +} + +define i8 @constant.divisor.v12to14(i8 %x) { +; CHECK-LABEL: @constant.divisor.v12to14( +; CHECK-NEXT: [[CMP_X_LOWER:%.*]] = icmp uge i8 [[X:%.*]], 12 +; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_X_LOWER]]) +; CHECK-NEXT: [[CMP_X_UPPER:%.*]] = icmp ult i8 [[X]], 15 +; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_X_UPPER]]) +; CHECK-NEXT: [[REM:%.*]] = urem i8 [[X]], 3 +; CHECK-NEXT: ret i8 [[REM]] +; + %cmp.x.lower = icmp uge i8 %x, 12 + call void @llvm.assume(i1 %cmp.x.lower) + %cmp.x.upper = icmp ult i8 %x, 15 + call void @llvm.assume(i1 %cmp.x.upper) + %rem = urem i8 %x, 3 + ret i8 %rem +} + +define i8 @constant.divisor.v6to11(i8 %x) { +; CHECK-LABEL: @constant.divisor.v6to11( +; CHECK-NEXT: [[CMP_X_LOWER:%.*]] = icmp uge i8 [[X:%.*]], 6 +; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_X_LOWER]]) +; CHECK-NEXT: [[CMP_X_UPPER:%.*]] = icmp ult i8 [[X]], 12 +; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_X_UPPER]]) +; CHECK-NEXT: [[REM:%.*]] = urem i8 [[X]], 3 +; CHECK-NEXT: ret i8 [[REM]] +; + %cmp.x.lower = icmp uge i8 %x, 6 + call void @llvm.assume(i1 %cmp.x.lower) + %cmp.x.upper = icmp ult i8 %x, 12 + call void @llvm.assume(i1 %cmp.x.upper) + %rem = urem i8 %x, 3 + ret i8 %rem +} + ; Both are variable. Bounds are known define i8 @variable.v3(i8 %x, i8 %y) { @@ -226,6 +294,31 @@ define i8 @variable.v7(i8 %x, i8 %y) { ret i8 %rem } +define i8 @variable.v6to8.v3to4(i8 %x, i8 %y) { +; CHECK-LABEL: @variable.v6to8.v3to4( +; CHECK-NEXT: [[CMP_X_LOWER:%.*]] = icmp uge i8 [[X:%.*]], 6 +; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_X_LOWER]]) +; CHECK-NEXT: [[CMP_X_UPPER:%.*]] = icmp ult i8 [[X]], 8 +; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_X_UPPER]]) +; CHECK-NEXT: [[CMP_Y_LOWER:%.*]] = icmp uge i8 [[Y:%.*]], 3 +; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_Y_LOWER]]) +; CHECK-NEXT: [[CMP_Y_UPPER:%.*]] = icmp ule i8 [[Y]], 4 +; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_Y_UPPER]]) +; CHECK-NEXT: [[REM:%.*]] = urem i8 [[X]], [[Y]] +; CHECK-NEXT: ret i8 [[REM]] +; + %cmp.x.lower = icmp uge i8 %x, 6 + call void @llvm.assume(i1 %cmp.x.lower) + %cmp.x.upper = icmp ult i8 %x, 8 + call void @llvm.assume(i1 %cmp.x.upper) + %cmp.y.lower = icmp uge i8 %y, 3 + call void @llvm.assume(i1 %cmp.y.lower) + %cmp.y.upper = icmp ule i8 %y, 4 + call void @llvm.assume(i1 %cmp.y.upper) + %rem = urem i8 %x, %y + ret i8 %rem +} + ; Constant divisor define i8 @large.divisor.v0(i8 %x) { -- GitLab From 292a28df6c55679fad0589dea35278a8c66b2ae1 Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Thu, 7 Mar 2024 15:12:21 -0500 Subject: [PATCH 497/929] [libc++] Enable availability based on the compiler instead of __has_extension (#84065) __has_extension(...) doesn't work as intended when -pedantic-errors is used with Clang. With that flag, __has_extension(...) is equivalent to __has_feature(...), which means that checks like __has_extension(pragma_clang_attribute_external_declaration) will return 0. In turn, this has the effect of disabling availability markup in libc++, which is undesirable. rdar://124078119 --- libcxx/include/__availability | 7 +++--- ...lity-with-pedantic-errors.compile.pass.cpp | 22 +++++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 libcxx/test/libcxx/vendor/apple/availability-with-pedantic-errors.compile.pass.cpp diff --git a/libcxx/include/__availability b/libcxx/include/__availability index 78438c55a3b7..bb3ed0a8da52 100644 --- a/libcxx/include/__availability +++ b/libcxx/include/__availability @@ -72,11 +72,10 @@ # endif #endif -// Availability markup is disabled when building the library, or when the compiler +// Availability markup is disabled when building the library, or when a non-Clang +// compiler is used because only Clang supports the necessary attributes. // doesn't support the proper attributes. -#if defined(_LIBCPP_BUILDING_LIBRARY) || defined(_LIBCXXABI_BUILDING_LIBRARY) || \ - !__has_feature(attribute_availability_with_strict) || !__has_feature(attribute_availability_in_templates) || \ - !__has_extension(pragma_clang_attribute_external_declaration) +#if defined(_LIBCPP_BUILDING_LIBRARY) || defined(_LIBCXXABI_BUILDING_LIBRARY) || !defined(_LIBCPP_COMPILER_CLANG_BASED) # if !defined(_LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS) # define _LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS # endif diff --git a/libcxx/test/libcxx/vendor/apple/availability-with-pedantic-errors.compile.pass.cpp b/libcxx/test/libcxx/vendor/apple/availability-with-pedantic-errors.compile.pass.cpp new file mode 100644 index 000000000000..c55a0a4d6e5d --- /dev/null +++ b/libcxx/test/libcxx/vendor/apple/availability-with-pedantic-errors.compile.pass.cpp @@ -0,0 +1,22 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// REQUIRES: stdlib=apple-libc++ + +// Test that using -pedantic-errors doesn't turn off availability annotations. +// This used to be the case because we used __has_extension(...) to enable the +// availability annotations, and -pedantic-errors changes the behavior of +// __has_extension(...) in an incompatible way. + +// ADDITIONAL_COMPILE_FLAGS: -pedantic-errors + +#include <__availability> + +#if defined(_LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS) +# error Availability annotations should be enabled on Apple platforms in the system configuration! +#endif -- GitLab From d0b702279819fe2fd3b3d2bfa274c895ac49f23b Mon Sep 17 00:00:00 2001 From: lntue <35648136+lntue@users.noreply.github.com> Date: Thu, 7 Mar 2024 15:23:37 -0500 Subject: [PATCH 498/929] [libc] Refactor stdfix extension from llvm_libc_ext.td to llvm_libc_stdfix_ext.td. (#84365) This fixes runtime build for armv6 baremetal targets: https://github.com/llvm/llvm-project/pull/83959#issuecomment-1984221249 --- libc/config/baremetal/api.td | 2 +- libc/config/linux/api.td | 1 + libc/spec/llvm_libc_ext.td | 20 -------------------- libc/spec/llvm_libc_stdfix_ext.td | 24 ++++++++++++++++++++++++ 4 files changed, 26 insertions(+), 21 deletions(-) create mode 100644 libc/spec/llvm_libc_stdfix_ext.td diff --git a/libc/config/baremetal/api.td b/libc/config/baremetal/api.td index 008eb45386f2..33b3a03828e9 100644 --- a/libc/config/baremetal/api.td +++ b/libc/config/baremetal/api.td @@ -2,7 +2,7 @@ include "config/public_api.td" include "spec/stdc.td" include "spec/stdc_ext.td" -include "spec/llvm_libc_ext.td" +include "spec/llvm_libc_stdfix_ext.td" def AssertMacro : MacroDef<"assert"> { let Defn = [{ diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td index 526fd03f94f6..75432a2a2986 100644 --- a/libc/config/linux/api.td +++ b/libc/config/linux/api.td @@ -7,6 +7,7 @@ include "spec/gnu_ext.td" include "spec/bsd_ext.td" include "spec/stdc_ext.td" include "spec/llvm_libc_ext.td" +include "spec/llvm_libc_stdfix_ext.td" def AssertMacro : MacroDef<"assert"> { let Defn = [{ diff --git a/libc/spec/llvm_libc_ext.td b/libc/spec/llvm_libc_ext.td index 3241ec055037..ca61d4ef371a 100644 --- a/libc/spec/llvm_libc_ext.td +++ b/libc/spec/llvm_libc_ext.td @@ -51,29 +51,9 @@ def LLVMLibcExt : StandardSpec<"llvm_libc_ext"> { ] >; - HeaderSpec StdFix = HeaderSpec< - "stdfix.h", - [], // macros - [], // types - [], // enums - [ // functions - GuardedFunctionSpec<"sqrtuhr", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, - GuardedFunctionSpec<"sqrtur", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, - GuardedFunctionSpec<"sqrtulr", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, - - GuardedFunctionSpec<"sqrtuhk", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, - GuardedFunctionSpec<"sqrtuk", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, - GuardedFunctionSpec<"sqrtulk", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, - - GuardedFunctionSpec<"uhksqrtus", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, - GuardedFunctionSpec<"uksqrtui", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, - ] - >; - let Headers = [ Assert, Sched, - StdFix, Strings, ]; } diff --git a/libc/spec/llvm_libc_stdfix_ext.td b/libc/spec/llvm_libc_stdfix_ext.td new file mode 100644 index 000000000000..75bde47810a6 --- /dev/null +++ b/libc/spec/llvm_libc_stdfix_ext.td @@ -0,0 +1,24 @@ +def LLVMLibcStdfixExt : StandardSpec<"llvm_libc_stdfix_ext"> { + HeaderSpec StdFix = HeaderSpec< + "stdfix.h", + [], // macros + [], // types + [], // enums + [ // functions + GuardedFunctionSpec<"sqrtuhr", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, + GuardedFunctionSpec<"sqrtur", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, + GuardedFunctionSpec<"sqrtulr", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, + + GuardedFunctionSpec<"sqrtuhk", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, + GuardedFunctionSpec<"sqrtuk", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, + GuardedFunctionSpec<"sqrtulk", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, + + GuardedFunctionSpec<"uhksqrtus", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, + GuardedFunctionSpec<"uksqrtui", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, + ] + >; + + let Headers = [ + StdFix, + ]; +} -- GitLab From a41226b05510a6f40d99fc622d78853460dc5599 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Pettersson?= Date: Thu, 7 Mar 2024 21:32:49 +0100 Subject: [PATCH 499/929] [ValueTracking] Fix KnownBits conflict for calls (range vs returned) (#84353) If a function only exits for certain input values we can still derive that an argument is "returned". We can also derive range metadata that describe the possible value range returned by the function. However, it turns out that those two analyses can result in conflicting information. Example: declare i16 @foo(i16 returned) ... %A = call i16 @foo(i16 4095), !range !{i16 32, i16 33} To avoid "Bits known to be one AND zero?" assertion failures we know make sure to discard the known bits for this kind of scenario. --- llvm/lib/Analysis/ValueTracking.cpp | 6 ++++++ llvm/unittests/Analysis/ValueTrackingTest.cpp | 14 ++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index 52ae9f034e5d..6d0e79e11eed 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -1476,6 +1476,12 @@ static void computeKnownBitsFromOperator(const Operator *I, if (RV->getType() == I->getType()) { computeKnownBits(RV, Known2, Depth + 1, Q); Known = Known.unionWith(Known2); + // If the function doesn't return properly for all input values + // (e.g. unreachable exits) then there might be conflicts between the + // argument value and the range metadata. Simply discard the known bits + // in case of conflicts. + if (Known.hasConflict()) + Known.resetAll(); } } if (const IntrinsicInst *II = dyn_cast(I)) { diff --git a/llvm/unittests/Analysis/ValueTrackingTest.cpp b/llvm/unittests/Analysis/ValueTrackingTest.cpp index 9e0abe7a16df..6c6897d83a25 100644 --- a/llvm/unittests/Analysis/ValueTrackingTest.cpp +++ b/llvm/unittests/Analysis/ValueTrackingTest.cpp @@ -2359,6 +2359,20 @@ TEST_F(ComputeKnownBitsTest, ComputeKnownBitsFreeze) { EXPECT_EQ(Known.One.getZExtValue(), 0u); } +TEST_F(ComputeKnownBitsTest, ComputeKnownBitsReturnedRangeConflict) { + parseAssembly( + "declare i16 @foo(i16 returned)\n" + "\n" + "define i16 @test() {\n" + " %A = call i16 @foo(i16 4095), !range !{i16 32, i16 33}\n" + " ret i16 %A\n" + "}\n"); + // The call returns 32 according to range metadata, but 4095 according to the + // returned arg operand. Given the conflicting information we expect that the + // known bits information simply is cleared. + expectKnownBits(/*zero*/ 0u, /*one*/ 0u); +} + TEST_F(ComputeKnownBitsTest, ComputeKnownBitsAddWithRange) { parseAssembly("define void @test(ptr %p) {\n" " %A = load i64, ptr %p, !range !{i64 64, i64 65536}\n" -- GitLab From 458636690afdd223ffa72f49164f30449b588892 Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Thu, 7 Mar 2024 12:41:06 -0800 Subject: [PATCH 500/929] [lldb] Do some gardening in ProgressReportTest (NFC) (#84278) - Factor our common setup code. - Split the ProgressManager test into separate tests as they test separate things. - Fix usage of EXPECT (which continues on failure) and ASSERT (which halts on failure). We must use the latter when calling GetEvent as otherwise we'll try to dereference a null EventSP. --- lldb/unittests/Core/ProgressReportTest.cpp | 199 ++++++++++----------- 1 file changed, 96 insertions(+), 103 deletions(-) diff --git a/lldb/unittests/Core/ProgressReportTest.cpp b/lldb/unittests/Core/ProgressReportTest.cpp index e0253cbc4ec5..1f993180fd83 100644 --- a/lldb/unittests/Core/ProgressReportTest.cpp +++ b/lldb/unittests/Core/ProgressReportTest.cpp @@ -22,9 +22,29 @@ using namespace lldb; using namespace lldb_private; +static std::chrono::milliseconds TIMEOUT(100); + class ProgressReportTest : public ::testing::Test { - SubsystemRAII subsystems; +public: + ListenerSP CreateListenerFor(uint32_t bit) { + // Set up the debugger, make sure that was done properly. + ArchSpec arch("x86_64-apple-macosx-"); + Platform::SetHostPlatform( + PlatformRemoteMacOSX::CreateInstance(true, &arch)); + + m_debugger_sp = Debugger::CreateInstance(); + + // Get the debugger's broadcaster. + Broadcaster &broadcaster = m_debugger_sp->GetBroadcaster(); + + // Create a listener, make sure it can receive events and that it's + // listening to the correct broadcast bit. + m_listener_sp = Listener::MakeListener("progress-listener"); + m_listener_sp->StartListeningForEvents(&broadcaster, bit); + return m_listener_sp; + } +protected: // The debugger's initialization function can't be called with no arguments // so calling it using SubsystemRAII will cause the test build to fail as // SubsystemRAII will call Initialize with no arguments. As such we set it up @@ -33,30 +53,14 @@ class ProgressReportTest : public ::testing::Test { std::call_once(TestUtilities::g_debugger_initialize_flag, []() { Debugger::Initialize(nullptr); }); }; + + DebuggerSP m_debugger_sp; + ListenerSP m_listener_sp; + SubsystemRAII subsystems; }; TEST_F(ProgressReportTest, TestReportCreation) { - std::chrono::milliseconds timeout(100); - - // Set up the debugger, make sure that was done properly. - ArchSpec arch("x86_64-apple-macosx-"); - Platform::SetHostPlatform(PlatformRemoteMacOSX::CreateInstance(true, &arch)); - - DebuggerSP debugger_sp = Debugger::CreateInstance(); - ASSERT_TRUE(debugger_sp); - - // Get the debugger's broadcaster. - Broadcaster &broadcaster = debugger_sp->GetBroadcaster(); - - // Create a listener, make sure it can receive events and that it's - // listening to the correct broadcast bit. - ListenerSP listener_sp = Listener::MakeListener("progress-listener"); - - listener_sp->StartListeningForEvents(&broadcaster, - Debugger::eBroadcastBitProgress); - EXPECT_TRUE( - broadcaster.EventTypeHasListeners(Debugger::eBroadcastBitProgress)); - + ListenerSP listener_sp = CreateListenerFor(Debugger::eBroadcastBitProgress); EventSP event_sp; const ProgressEventData *data; @@ -73,82 +77,64 @@ TEST_F(ProgressReportTest, TestReportCreation) { // in this order: // Starting progress: 1, 2, 3 // Ending progress: 3, 2, 1 - EXPECT_TRUE(listener_sp->GetEvent(event_sp, timeout)); + ASSERT_TRUE(listener_sp->GetEvent(event_sp, TIMEOUT)); data = ProgressEventData::GetEventDataFromEvent(event_sp.get()); - ASSERT_EQ(data->GetDetails(), "Starting report 1"); - ASSERT_FALSE(data->IsFinite()); - ASSERT_FALSE(data->GetCompleted()); - ASSERT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal); - ASSERT_EQ(data->GetMessage(), "Progress report 1: Starting report 1"); + EXPECT_EQ(data->GetDetails(), "Starting report 1"); + EXPECT_FALSE(data->IsFinite()); + EXPECT_FALSE(data->GetCompleted()); + EXPECT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal); + EXPECT_EQ(data->GetMessage(), "Progress report 1: Starting report 1"); - EXPECT_TRUE(listener_sp->GetEvent(event_sp, timeout)); + ASSERT_TRUE(listener_sp->GetEvent(event_sp, TIMEOUT)); data = ProgressEventData::GetEventDataFromEvent(event_sp.get()); - ASSERT_EQ(data->GetDetails(), "Starting report 2"); - ASSERT_FALSE(data->IsFinite()); - ASSERT_FALSE(data->GetCompleted()); - ASSERT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal); - ASSERT_EQ(data->GetMessage(), "Progress report 2: Starting report 2"); + EXPECT_EQ(data->GetDetails(), "Starting report 2"); + EXPECT_FALSE(data->IsFinite()); + EXPECT_FALSE(data->GetCompleted()); + EXPECT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal); + EXPECT_EQ(data->GetMessage(), "Progress report 2: Starting report 2"); - EXPECT_TRUE(listener_sp->GetEvent(event_sp, timeout)); + ASSERT_TRUE(listener_sp->GetEvent(event_sp, TIMEOUT)); data = ProgressEventData::GetEventDataFromEvent(event_sp.get()); - ASSERT_EQ(data->GetDetails(), "Starting report 3"); - ASSERT_FALSE(data->IsFinite()); - ASSERT_FALSE(data->GetCompleted()); - ASSERT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal); - ASSERT_EQ(data->GetMessage(), "Progress report 3: Starting report 3"); + + EXPECT_EQ(data->GetDetails(), "Starting report 3"); + EXPECT_FALSE(data->IsFinite()); + EXPECT_FALSE(data->GetCompleted()); + EXPECT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal); + EXPECT_EQ(data->GetMessage(), "Progress report 3: Starting report 3"); // Progress report objects should be destroyed at this point so // get each report from the queue and check that they've been // destroyed in reverse order. - EXPECT_TRUE(listener_sp->GetEvent(event_sp, timeout)); + ASSERT_TRUE(listener_sp->GetEvent(event_sp, TIMEOUT)); data = ProgressEventData::GetEventDataFromEvent(event_sp.get()); - ASSERT_EQ(data->GetTitle(), "Progress report 3"); - ASSERT_TRUE(data->GetCompleted()); - ASSERT_FALSE(data->IsFinite()); - ASSERT_EQ(data->GetMessage(), "Progress report 3: Starting report 3"); + EXPECT_EQ(data->GetTitle(), "Progress report 3"); + EXPECT_TRUE(data->GetCompleted()); + EXPECT_FALSE(data->IsFinite()); + EXPECT_EQ(data->GetMessage(), "Progress report 3: Starting report 3"); - EXPECT_TRUE(listener_sp->GetEvent(event_sp, timeout)); + ASSERT_TRUE(listener_sp->GetEvent(event_sp, TIMEOUT)); data = ProgressEventData::GetEventDataFromEvent(event_sp.get()); - ASSERT_EQ(data->GetTitle(), "Progress report 2"); - ASSERT_TRUE(data->GetCompleted()); - ASSERT_FALSE(data->IsFinite()); - ASSERT_EQ(data->GetMessage(), "Progress report 2: Starting report 2"); + EXPECT_EQ(data->GetTitle(), "Progress report 2"); + EXPECT_TRUE(data->GetCompleted()); + EXPECT_FALSE(data->IsFinite()); + EXPECT_EQ(data->GetMessage(), "Progress report 2: Starting report 2"); - EXPECT_TRUE(listener_sp->GetEvent(event_sp, timeout)); + ASSERT_TRUE(listener_sp->GetEvent(event_sp, TIMEOUT)); data = ProgressEventData::GetEventDataFromEvent(event_sp.get()); - ASSERT_EQ(data->GetTitle(), "Progress report 1"); - ASSERT_TRUE(data->GetCompleted()); - ASSERT_FALSE(data->IsFinite()); - ASSERT_EQ(data->GetMessage(), "Progress report 1: Starting report 1"); + EXPECT_EQ(data->GetTitle(), "Progress report 1"); + EXPECT_TRUE(data->GetCompleted()); + EXPECT_FALSE(data->IsFinite()); + EXPECT_EQ(data->GetMessage(), "Progress report 1: Starting report 1"); } TEST_F(ProgressReportTest, TestProgressManager) { - std::chrono::milliseconds timeout(100); - - // Set up the debugger, make sure that was done properly. - ArchSpec arch("x86_64-apple-macosx-"); - Platform::SetHostPlatform(PlatformRemoteMacOSX::CreateInstance(true, &arch)); - - DebuggerSP debugger_sp = Debugger::CreateInstance(); - ASSERT_TRUE(debugger_sp); - - // Get the debugger's broadcaster. - Broadcaster &broadcaster = debugger_sp->GetBroadcaster(); - - // Create a listener, make sure it can receive events and that it's - // listening to the correct broadcast bit. - ListenerSP listener_sp = Listener::MakeListener("progress-category-listener"); - - listener_sp->StartListeningForEvents(&broadcaster, - Debugger::eBroadcastBitProgressCategory); - EXPECT_TRUE(broadcaster.EventTypeHasListeners( - Debugger::eBroadcastBitProgressCategory)); - + ListenerSP listener_sp = + CreateListenerFor(Debugger::eBroadcastBitProgressCategory); EventSP event_sp; const ProgressEventData *data; @@ -160,28 +146,35 @@ TEST_F(ProgressReportTest, TestProgressManager) { Progress progress1("Progress report 1", "Starting report 1"); Progress progress2("Progress report 1", "Starting report 2"); Progress progress3("Progress report 1", "Starting report 3"); - EXPECT_TRUE(listener_sp->GetEvent(event_sp, timeout)); - EXPECT_FALSE(listener_sp->GetEvent(event_sp, timeout)); + ASSERT_TRUE(listener_sp->GetEvent(event_sp, TIMEOUT)); + ASSERT_FALSE(listener_sp->GetEvent(event_sp, TIMEOUT)); } data = ProgressEventData::GetEventDataFromEvent(event_sp.get()); - ASSERT_EQ(data->GetDetails(), ""); - ASSERT_FALSE(data->IsFinite()); - ASSERT_FALSE(data->GetCompleted()); - ASSERT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal); - ASSERT_EQ(data->GetMessage(), "Progress report 1"); + EXPECT_EQ(data->GetDetails(), ""); + EXPECT_FALSE(data->IsFinite()); + EXPECT_FALSE(data->GetCompleted()); + EXPECT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal); + EXPECT_EQ(data->GetMessage(), "Progress report 1"); // Pop another event from the queue, this should be the event for the final // report for this category. - EXPECT_TRUE(listener_sp->GetEvent(event_sp, timeout)); - + ASSERT_TRUE(listener_sp->GetEvent(event_sp, TIMEOUT)); data = ProgressEventData::GetEventDataFromEvent(event_sp.get()); - ASSERT_EQ(data->GetDetails(), ""); - ASSERT_FALSE(data->IsFinite()); - ASSERT_TRUE(data->GetCompleted()); - ASSERT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal); - ASSERT_EQ(data->GetMessage(), "Progress report 1"); + + EXPECT_EQ(data->GetDetails(), ""); + EXPECT_FALSE(data->IsFinite()); + EXPECT_TRUE(data->GetCompleted()); + EXPECT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal); + EXPECT_EQ(data->GetMessage(), "Progress report 1"); +} + +TEST_F(ProgressReportTest, TestOverlappingEvents) { + ListenerSP listener_sp = + CreateListenerFor(Debugger::eBroadcastBitProgressCategory); + EventSP event_sp; + const ProgressEventData *data; // Create two progress reports of the same category that overlap with each // other. Here we want to ensure that the ID broadcasted for the initial and @@ -192,28 +185,28 @@ TEST_F(ProgressReportTest, TestProgressManager) { std::make_unique("Overlapping report 1", "Starting report 2"); overlap_progress1.reset(); - EXPECT_TRUE(listener_sp->GetEvent(event_sp, timeout)); + ASSERT_TRUE(listener_sp->GetEvent(event_sp, TIMEOUT)); data = ProgressEventData::GetEventDataFromEvent(event_sp.get()); // Get the ID used in the first report for this category. uint64_t expected_progress_id = data->GetID(); - ASSERT_EQ(data->GetDetails(), ""); - ASSERT_FALSE(data->IsFinite()); - ASSERT_FALSE(data->GetCompleted()); - ASSERT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal); - ASSERT_EQ(data->GetMessage(), "Overlapping report 1"); + EXPECT_EQ(data->GetDetails(), ""); + EXPECT_FALSE(data->IsFinite()); + EXPECT_FALSE(data->GetCompleted()); + EXPECT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal); + EXPECT_EQ(data->GetMessage(), "Overlapping report 1"); overlap_progress2.reset(); - EXPECT_TRUE(listener_sp->GetEvent(event_sp, timeout)); + ASSERT_TRUE(listener_sp->GetEvent(event_sp, TIMEOUT)); data = ProgressEventData::GetEventDataFromEvent(event_sp.get()); - ASSERT_EQ(data->GetDetails(), ""); - ASSERT_FALSE(data->IsFinite()); - ASSERT_TRUE(data->GetCompleted()); - ASSERT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal); - ASSERT_EQ(data->GetMessage(), "Overlapping report 1"); + EXPECT_EQ(data->GetDetails(), ""); + EXPECT_FALSE(data->IsFinite()); + EXPECT_TRUE(data->GetCompleted()); + EXPECT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal); + EXPECT_EQ(data->GetMessage(), "Overlapping report 1"); // The progress ID for the final report should be the same as that for the // initial report. - ASSERT_EQ(data->GetID(), expected_progress_id); + EXPECT_EQ(data->GetID(), expected_progress_id); } -- GitLab From 11185715a28c6592ca6fe247fe693b305c85627a Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Thu, 7 Mar 2024 12:44:25 -0800 Subject: [PATCH 501/929] Revert "[SLP]Improve minbitwidth analysis." This reverts commit 4ce52e2d576937fe930294cae883a0daa17eeced to fix issues detected by https://lab.llvm.org/buildbot/#/builders/74/builds/26470/steps/12/logs/stdio. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 634 ++++++------------ .../SLPVectorizer/AArch64/ext-trunc.ll | 9 +- .../SLPVectorizer/AArch64/getelementptr2.ll | 4 +- .../SLPVectorizer/AArch64/reduce-add-i64.ll | 20 +- .../SLPVectorizer/RISCV/reductions.ll | 7 +- .../Transforms/SLPVectorizer/X86/PR35777.ll | 9 +- .../X86/int-bitcast-minbitwidth.ll | 2 +- ...minbitwidth-multiuse-with-insertelement.ll | 17 +- .../X86/minbitwidth-transformed-operand.ll | 21 +- .../SLPVectorizer/X86/minimum-sizes.ll | 43 +- .../SLPVectorizer/X86/phi-undef-input.ll | 24 +- .../Transforms/SLPVectorizer/X86/resched.ll | 32 +- .../X86/reused-reductions-with-minbitwidth.ll | 10 +- .../X86/store-insertelement-minbitwidth.ll | 22 +- .../SLPVectorizer/alt-cmp-vectorize.ll | 4 +- 15 files changed, 305 insertions(+), 553 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 1889bc09e850..36dc9094538a 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -1085,9 +1085,6 @@ public: BS->clear(); } MinBWs.clear(); - ReductionBitWidth = 0; - CastMaxMinBWSizes.reset(); - TruncNodes.clear(); InstrElementSize.clear(); UserIgnoreList = nullptr; PostponedGathers.clear(); @@ -2290,7 +2287,6 @@ public: void clearReductionData() { AnalyzedReductionsRoots.clear(); AnalyzedReductionVals.clear(); - AnalyzedMinBWVals.clear(); } /// Checks if the given value is gathered in one of the nodes. bool isAnyGathered(const SmallDenseSet &Vals) const { @@ -2311,11 +2307,9 @@ private: /// constant and to be demoted. Required to correctly identify constant nodes /// to be demoted. bool collectValuesToDemote( - Value *V, bool IsProfitableToDemoteRoot, unsigned &BitWidth, - SmallVectorImpl &ToDemote, + Value *V, SmallVectorImpl &ToDemote, DenseMap> &DemotedConsts, - DenseSet &Visited, unsigned &MaxDepthLevel, - bool &IsProfitableToDemote) const; + SmallVectorImpl &Roots, DenseSet &Visited) const; /// Check if the operands on the edges \p Edges of the \p UserTE allows /// reordering (i.e. the operands can be reordered because they have only one @@ -2381,10 +2375,6 @@ private: /// \ returns the graph entry for the \p Idx operand of the \p E entry. const TreeEntry *getOperandEntry(const TreeEntry *E, unsigned Idx) const; - /// \returns Cast context for the given graph node. - TargetTransformInfo::CastContextHint - getCastContextHint(const TreeEntry &TE) const; - /// \returns the cost of the vectorizable entry. InstructionCost getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, @@ -2935,18 +2925,11 @@ private: } assert(!BundleMember && "Bundle and VL out of sync"); } else { + MustGather.insert(VL.begin(), VL.end()); // Build a map for gathered scalars to the nodes where they are used. - bool AllConstsOrCasts = true; for (Value *V : VL) - if (!isConstant(V)) { - auto *I = dyn_cast(V); - AllConstsOrCasts &= I && I->getType()->isIntegerTy(); + if (!isConstant(V)) ValueToGatherNodes.try_emplace(V).first->getSecond().insert(Last); - } - if (AllConstsOrCasts) - CastMaxMinBWSizes = - std::make_pair(std::numeric_limits::max(), 1); - MustGather.insert(VL.begin(), VL.end()); } if (UserTreeIdx.UserTE) @@ -3071,10 +3054,6 @@ private: /// Set of hashes for the list of reduction values already being analyzed. DenseSet AnalyzedReductionVals; - /// Values, already been analyzed for mininmal bitwidth and found to be - /// non-profitable. - DenseSet AnalyzedMinBWVals; - /// A list of values that need to extracted out of the tree. /// This list holds pairs of (Internal Scalar : External User). External User /// can be nullptr, it means that this Internal Scalar will be used later, @@ -3650,18 +3629,6 @@ private: /// value must be signed-extended, rather than zero-extended, back to its /// original width. DenseMap> MinBWs; - - /// Final size of the reduced vector, if the current graph represents the - /// input for the reduction and it was possible to narrow the size of the - /// reduction. - unsigned ReductionBitWidth = 0; - - /// If the tree contains any zext/sext/trunc nodes, contains max-min pair of - /// type sizes, used in the tree. - std::optional> CastMaxMinBWSizes; - - /// Indices of the vectorized trunc nodes. - DenseSet TruncNodes; }; } // end namespace slpvectorizer @@ -6572,29 +6539,8 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, case Instruction::Trunc: case Instruction::FPTrunc: case Instruction::BitCast: { - auto [PrevMaxBW, PrevMinBW] = CastMaxMinBWSizes.value_or( - std::make_pair(std::numeric_limits::min(), - std::numeric_limits::max())); - if (ShuffleOrOp == Instruction::ZExt || - ShuffleOrOp == Instruction::SExt) { - CastMaxMinBWSizes = std::make_pair( - std::max(DL->getTypeSizeInBits(VL0->getType()), - PrevMaxBW), - std::min( - DL->getTypeSizeInBits(VL0->getOperand(0)->getType()), - PrevMinBW)); - } else if (ShuffleOrOp == Instruction::Trunc) { - CastMaxMinBWSizes = std::make_pair( - std::max( - DL->getTypeSizeInBits(VL0->getOperand(0)->getType()), - PrevMaxBW), - std::min(DL->getTypeSizeInBits(VL0->getType()), - PrevMinBW)); - TruncNodes.insert(VectorizableTree.size()); - } TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, ReuseShuffleIndicies); - LLVM_DEBUG(dbgs() << "SLP: added a vector of casts.\n"); TE->setOperandsInOrder(); @@ -8416,22 +8362,6 @@ const BoUpSLP::TreeEntry *BoUpSLP::getOperandEntry(const TreeEntry *E, return It->get(); } -TTI::CastContextHint BoUpSLP::getCastContextHint(const TreeEntry &TE) const { - if (TE.State == TreeEntry::ScatterVectorize || - TE.State == TreeEntry::StridedVectorize) - return TTI::CastContextHint::GatherScatter; - if (TE.State == TreeEntry::Vectorize && TE.getOpcode() == Instruction::Load && - !TE.isAltShuffle()) { - if (TE.ReorderIndices.empty()) - return TTI::CastContextHint::Normal; - SmallVector Mask; - inversePermutation(TE.ReorderIndices, Mask); - if (ShuffleVectorInst::isReverseMask(Mask, Mask.size())) - return TTI::CastContextHint::Reversed; - } - return TTI::CastContextHint::None; -} - InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, SmallPtrSetImpl &CheckedExtracts) { @@ -8454,7 +8384,6 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, // If we have computed a smaller type for the expression, update VecTy so // that the costs will be accurate. auto It = MinBWs.find(E); - Type *OrigScalarTy = ScalarTy; if (It != MinBWs.end()) { ScalarTy = IntegerType::get(F->getContext(), It->second.first); VecTy = FixedVectorType::get(ScalarTy, VL.size()); @@ -8512,11 +8441,24 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, UsedScalars.set(I); } auto GetCastContextHint = [&](Value *V) { - if (const TreeEntry *OpTE = getTreeEntry(V)) - return getCastContextHint(*OpTE); - InstructionsState SrcState = getSameOpcode(E->getOperand(0), *TLI); - if (SrcState.getOpcode() == Instruction::Load && !SrcState.isAltShuffle()) - return TTI::CastContextHint::GatherScatter; + if (const TreeEntry *OpTE = getTreeEntry(V)) { + if (OpTE->State == TreeEntry::ScatterVectorize || + OpTE->State == TreeEntry::StridedVectorize) + return TTI::CastContextHint::GatherScatter; + if (OpTE->State == TreeEntry::Vectorize && + OpTE->getOpcode() == Instruction::Load && !OpTE->isAltShuffle()) { + if (OpTE->ReorderIndices.empty()) + return TTI::CastContextHint::Normal; + SmallVector Mask; + inversePermutation(OpTE->ReorderIndices, Mask); + if (ShuffleVectorInst::isReverseMask(Mask, Mask.size())) + return TTI::CastContextHint::Reversed; + } + } else { + InstructionsState SrcState = getSameOpcode(E->getOperand(0), *TLI); + if (SrcState.getOpcode() == Instruction::Load && !SrcState.isAltShuffle()) + return TTI::CastContextHint::GatherScatter; + } return TTI::CastContextHint::None; }; auto GetCostDiff = @@ -8565,6 +8507,8 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, TTI::CastContextHint CCH = GetCastContextHint(VL0); VecCost += TTI->getCastInstrCost(VecOpcode, UserVecTy, VecTy, CCH, CostKind); + ScalarCost += Sz * TTI->getCastInstrCost(VecOpcode, UserScalarTy, + ScalarTy, CCH, CostKind); } } } @@ -8581,7 +8525,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, InstructionCost ScalarCost = 0; InstructionCost VecCost = 0; std::tie(ScalarCost, VecCost) = getGEPCosts( - *TTI, Ptrs, BasePtr, E->getOpcode(), CostKind, OrigScalarTy, VecTy); + *TTI, Ptrs, BasePtr, E->getOpcode(), CostKind, ScalarTy, VecTy); LLVM_DEBUG(dumpTreeCosts(E, 0, VecCost, ScalarCost, "Calculated GEPs cost for Tree")); @@ -8628,7 +8572,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, NumElts = ATy->getNumElements(); else NumElts = AggregateTy->getStructNumElements(); - SrcVecTy = FixedVectorType::get(OrigScalarTy, NumElts); + SrcVecTy = FixedVectorType::get(ScalarTy, NumElts); } if (I->hasOneUse()) { Instruction *Ext = I->user_back(); @@ -8796,7 +8740,13 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, } } auto GetScalarCost = [&](unsigned Idx) -> InstructionCost { - auto *VI = cast(UniqueValues[Idx]); + // Do not count cost here if minimum bitwidth is in effect and it is just + // a bitcast (here it is just a noop). + if (VecOpcode != Opcode && VecOpcode == Instruction::BitCast) + return TTI::TCC_Free; + auto *VI = VL0->getOpcode() == Opcode + ? cast(UniqueValues[Idx]) + : nullptr; return TTI->getCastInstrCost(Opcode, VL0->getType(), VL0->getOperand(0)->getType(), TTI::getCastContextHint(VI), CostKind, VI); @@ -8839,7 +8789,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, ? CmpInst::BAD_FCMP_PREDICATE : CmpInst::BAD_ICMP_PREDICATE; - return TTI->getCmpSelInstrCost(E->getOpcode(), OrigScalarTy, + return TTI->getCmpSelInstrCost(E->getOpcode(), ScalarTy, Builder.getInt1Ty(), CurrentPred, CostKind, VI); }; @@ -8894,7 +8844,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(VI->getOperand(OpIdx)); SmallVector Operands(VI->operand_values()); - return TTI->getArithmeticInstrCost(ShuffleOrOp, OrigScalarTy, CostKind, + return TTI->getArithmeticInstrCost(ShuffleOrOp, ScalarTy, CostKind, Op1Info, Op2Info, Operands, VI); }; auto GetVectorCost = [=](InstructionCost CommonCost) { @@ -8913,9 +8863,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, case Instruction::Load: { auto GetScalarCost = [&](unsigned Idx) { auto *VI = cast(UniqueValues[Idx]); - return TTI->getMemoryOpCost(Instruction::Load, OrigScalarTy, - VI->getAlign(), VI->getPointerAddressSpace(), - CostKind, TTI::OperandValueInfo(), VI); + return TTI->getMemoryOpCost(Instruction::Load, ScalarTy, VI->getAlign(), + VI->getPointerAddressSpace(), CostKind, + TTI::OperandValueInfo(), VI); }; auto *LI0 = cast(VL0); auto GetVectorCost = [&](InstructionCost CommonCost) { @@ -8958,9 +8908,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, auto GetScalarCost = [=](unsigned Idx) { auto *VI = cast(VL[Idx]); TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(VI->getValueOperand()); - return TTI->getMemoryOpCost(Instruction::Store, OrigScalarTy, - VI->getAlign(), VI->getPointerAddressSpace(), - CostKind, OpInfo, VI); + return TTI->getMemoryOpCost(Instruction::Store, ScalarTy, VI->getAlign(), + VI->getPointerAddressSpace(), CostKind, + OpInfo, VI); }; auto *BaseSI = cast(IsReorder ? VL[E->ReorderIndices.front()] : VL0); @@ -9822,44 +9772,6 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef VectorizedVals) { Cost -= InsertCost; } - // Add the cost for reduced value resize (if required). - if (ReductionBitWidth != 0) { - assert(UserIgnoreList && "Expected reduction tree."); - const TreeEntry &E = *VectorizableTree.front().get(); - auto It = MinBWs.find(&E); - if (It != MinBWs.end() && It->second.first != ReductionBitWidth) { - unsigned SrcSize = It->second.first; - unsigned DstSize = ReductionBitWidth; - unsigned Opcode = Instruction::Trunc; - if (SrcSize < DstSize) - Opcode = It->second.second ? Instruction::SExt : Instruction::ZExt; - auto *SrcVecTy = - FixedVectorType::get(Builder.getIntNTy(SrcSize), E.getVectorFactor()); - auto *DstVecTy = - FixedVectorType::get(Builder.getIntNTy(DstSize), E.getVectorFactor()); - TTI::CastContextHint CCH = getCastContextHint(E); - InstructionCost CastCost; - switch (E.getOpcode()) { - case Instruction::SExt: - case Instruction::ZExt: - case Instruction::Trunc: { - const TreeEntry *OpTE = getOperandEntry(&E, 0); - CCH = getCastContextHint(*OpTE); - break; - } - default: - break; - } - CastCost += TTI->getCastInstrCost(Opcode, DstVecTy, SrcVecTy, CCH, - TTI::TCK_RecipThroughput); - Cost += CastCost; - LLVM_DEBUG(dbgs() << "SLP: Adding cost " << CastCost - << " for final resize for reduction from " << SrcVecTy - << " to " << DstVecTy << "\n"; - dbgs() << "SLP: Current total cost = " << Cost << "\n"); - } - } - #ifndef NDEBUG SmallString<256> Str; { @@ -10080,30 +9992,6 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry( // tree node for each gathered value - we have just a permutation of the // single vector. If we have 2 different sets, we're in situation where we // have a permutation of 2 input vectors. - // Filter out entries with larger bitwidth of elements. - Type *ScalarTy = VL.front()->getType(); - unsigned BitWidth = 0; - if (ScalarTy->isIntegerTy()) { - // Check if the used TEs supposed to be resized and choose the best - // candidates. - BitWidth = DL->getTypeStoreSize(ScalarTy); - if (TEUseEI.UserTE->getOpcode() != Instruction::Select || - TEUseEI.EdgeIdx != 0) { - auto UserIt = MinBWs.find(TEUseEI.UserTE); - if (UserIt != MinBWs.end()) - BitWidth = UserIt->second.second; - } - } - auto CheckBitwidth = [&](const TreeEntry &TE) { - Type *ScalarTy = TE.Scalars.front()->getType(); - if (!ScalarTy->isIntegerTy()) - return true; - unsigned TEBitWidth = DL->getTypeStoreSize(ScalarTy); - auto UserIt = MinBWs.find(TEUseEI.UserTE); - if (UserIt != MinBWs.end()) - TEBitWidth = UserIt->second.second; - return BitWidth == TEBitWidth; - }; SmallVector> UsedTEs; DenseMap UsedValuesEntry; for (Value *V : VL) { @@ -10138,8 +10026,6 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry( continue; } - if (!CheckBitwidth(*TEPtr)) - continue; // Check if the user node of the TE comes after user node of TEPtr, // otherwise TEPtr depends on TE. if ((TEInsertBlock != InsertPt->getParent() || @@ -10156,8 +10042,8 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry( continue; VTE = *It->getSecond().begin(); // Iterate through all vectorized nodes. - auto *MIt = find_if(It->getSecond(), [&](const TreeEntry *MTE) { - return MTE->State == TreeEntry::Vectorize && CheckBitwidth(*MTE); + auto *MIt = find_if(It->getSecond(), [](const TreeEntry *MTE) { + return MTE->State == TreeEntry::Vectorize; }); if (MIt == It->getSecond().end()) continue; @@ -10167,7 +10053,10 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry( Instruction &LastBundleInst = getLastInstructionInBundle(VTE); if (&LastBundleInst == TEInsertPt || !CheckOrdering(&LastBundleInst)) continue; - if (!CheckBitwidth(*VTE)) + auto It = MinBWs.find(VTE); + // If vectorize node is demoted - do not match. + if (It != MinBWs.end() && + It->second.first != DL->getTypeSizeInBits(V->getType())) continue; VToTEs.insert(VTE); } @@ -13040,21 +12929,7 @@ Value *BoUpSLP::vectorizeTree( Builder.ClearInsertionPoint(); InstrElementSize.clear(); - const TreeEntry &RootTE = *VectorizableTree.front().get(); - Value *Vec = RootTE.VectorizedValue; - if (auto It = MinBWs.find(&RootTE); ReductionBitWidth != 0 && - It != MinBWs.end() && - ReductionBitWidth != It->second.first) { - IRBuilder<>::InsertPointGuard Guard(Builder); - Builder.SetInsertPoint(ReductionRoot->getParent(), - ReductionRoot->getIterator()); - Vec = Builder.CreateIntCast( - Vec, - VectorType::get(Builder.getIntNTy(ReductionBitWidth), - cast(Vec->getType())->getElementCount()), - It->second.second); - } - return Vec; + return VectorizableTree[0]->VectorizedValue; } void BoUpSLP::optimizeGatherSequence() { @@ -13874,42 +13749,23 @@ unsigned BoUpSLP::getVectorElementSize(Value *V) { // smaller type with a truncation. We collect the values that will be demoted // in ToDemote and additional roots that require investigating in Roots. bool BoUpSLP::collectValuesToDemote( - Value *V, bool IsProfitableToDemoteRoot, unsigned &BitWidth, - SmallVectorImpl &ToDemote, + Value *V, SmallVectorImpl &ToDemote, DenseMap> &DemotedConsts, - DenseSet &Visited, unsigned &MaxDepthLevel, - bool &IsProfitableToDemote) const { + SmallVectorImpl &Roots, DenseSet &Visited) const { // We can always demote constants. - if (isa(V)) { - MaxDepthLevel = 1; + if (isa(V)) return true; - } // If the value is not a vectorized instruction in the expression and not used // by the insertelement instruction and not used in multiple vector nodes, it // cannot be demoted. - // TODO: improve handling of gathered values and others. auto *I = dyn_cast(V); - if (!I || !Visited.insert(I).second || !getTreeEntry(I) || - MultiNodeScalars.contains(I) || all_of(I->users(), [&](User *U) { + if (!I || !getTreeEntry(I) || MultiNodeScalars.contains(I) || + !Visited.insert(I).second || all_of(I->users(), [&](User *U) { return isa(U) && !getTreeEntry(U); })) return false; - auto IsPotentiallyTruncated = [&](Value *V, unsigned &BitWidth) -> bool { - if (MultiNodeScalars.contains(V)) - return false; - uint32_t OrigBitWidth = DL->getTypeSizeInBits(V->getType()); - APInt Mask = APInt::getBitsSetFrom(OrigBitWidth, BitWidth); - if (MaskedValueIsZero(V, Mask, SimplifyQuery(*DL))) - return true; - auto NumSignBits = ComputeNumSignBits(V, *DL, 0, AC, nullptr, DT); - unsigned BitWidth1 = OrigBitWidth - NumSignBits; - if (!isKnownNonNegative(V, SimplifyQuery(*DL))) - ++BitWidth1; - BitWidth = std::max(BitWidth, BitWidth1); - return BitWidth > 0 && OrigBitWidth >= (BitWidth * 2); - }; unsigned Start = 0; unsigned End = I->getNumOperands(); switch (I->getOpcode()) { @@ -13917,14 +13773,12 @@ bool BoUpSLP::collectValuesToDemote( // We can always demote truncations and extensions. Since truncations can // seed additional demotion, we save the truncated value. case Instruction::Trunc: - MaxDepthLevel = 1; - if (IsProfitableToDemoteRoot) - IsProfitableToDemote = true; + Roots.push_back(I->getOperand(0)); break; case Instruction::ZExt: case Instruction::SExt: - MaxDepthLevel = 1; - IsProfitableToDemote = true; + if (isa(I->getOperand(0))) + return false; break; // We can demote certain binary operations if we can demote both of their @@ -13934,32 +13788,23 @@ bool BoUpSLP::collectValuesToDemote( case Instruction::Mul: case Instruction::And: case Instruction::Or: - case Instruction::Xor: { - unsigned Level1, Level2; - if (!collectValuesToDemote(I->getOperand(0), IsProfitableToDemoteRoot, - BitWidth, ToDemote, DemotedConsts, Visited, - Level1, IsProfitableToDemote) || - !collectValuesToDemote(I->getOperand(1), IsProfitableToDemoteRoot, - BitWidth, ToDemote, DemotedConsts, Visited, - Level2, IsProfitableToDemote)) + case Instruction::Xor: + if (!collectValuesToDemote(I->getOperand(0), ToDemote, DemotedConsts, Roots, + Visited) || + !collectValuesToDemote(I->getOperand(1), ToDemote, DemotedConsts, Roots, + Visited)) return false; - MaxDepthLevel = std::max(Level1, Level2); break; - } // We can demote selects if we can demote their true and false values. case Instruction::Select: { Start = 1; - unsigned Level1, Level2; SelectInst *SI = cast(I); - if (!collectValuesToDemote(SI->getTrueValue(), IsProfitableToDemoteRoot, - BitWidth, ToDemote, DemotedConsts, Visited, - Level1, IsProfitableToDemote) || - !collectValuesToDemote(SI->getFalseValue(), IsProfitableToDemoteRoot, - BitWidth, ToDemote, DemotedConsts, Visited, - Level2, IsProfitableToDemote)) + if (!collectValuesToDemote(SI->getTrueValue(), ToDemote, DemotedConsts, + Roots, Visited) || + !collectValuesToDemote(SI->getFalseValue(), ToDemote, DemotedConsts, + Roots, Visited)) return false; - MaxDepthLevel = std::max(Level1, Level2); break; } @@ -13968,262 +13813,171 @@ bool BoUpSLP::collectValuesToDemote( case Instruction::PHI: { PHINode *PN = cast(I); for (Value *IncValue : PN->incoming_values()) - if (!collectValuesToDemote(IncValue, IsProfitableToDemoteRoot, BitWidth, - ToDemote, DemotedConsts, Visited, - MaxDepthLevel, IsProfitableToDemote)) + if (!collectValuesToDemote(IncValue, ToDemote, DemotedConsts, Roots, + Visited)) return false; break; } // Otherwise, conservatively give up. default: - return IsProfitableToDemote && IsPotentiallyTruncated(I, BitWidth); + return false; } - ++MaxDepthLevel; // Gather demoted constant operands. for (unsigned Idx : seq(Start, End)) if (isa(I->getOperand(Idx))) DemotedConsts.try_emplace(I).first->getSecond().push_back(Idx); // Record the value that we can demote. ToDemote.push_back(V); - return IsProfitableToDemote; + return true; } void BoUpSLP::computeMinimumValueSizes() { // We only attempt to truncate integer expressions. - bool IsStoreOrInsertElt = - VectorizableTree.front()->getOpcode() == Instruction::Store || - VectorizableTree.front()->getOpcode() == Instruction::InsertElement; - if ((IsStoreOrInsertElt || UserIgnoreList) && TruncNodes.size() <= 1 && - (!CastMaxMinBWSizes || CastMaxMinBWSizes->second == 0 || - CastMaxMinBWSizes->first / CastMaxMinBWSizes->second <= 2)) + auto &TreeRoot = VectorizableTree[0]->Scalars; + auto *TreeRootIT = dyn_cast(TreeRoot[0]->getType()); + if (!TreeRootIT || VectorizableTree.front()->State == TreeEntry::NeedToGather) return; - unsigned NodeIdx = 0; - if (IsStoreOrInsertElt && - VectorizableTree.front()->State != TreeEntry::NeedToGather) - NodeIdx = 1; - // Ensure the roots of the vectorizable tree don't form a cycle. - if (VectorizableTree[NodeIdx]->State == TreeEntry::NeedToGather || - (NodeIdx == 0 && !VectorizableTree[NodeIdx]->UserTreeIndices.empty()) || - (NodeIdx != 0 && any_of(VectorizableTree[NodeIdx]->UserTreeIndices, - [NodeIdx](const EdgeInfo &EI) { - return EI.UserTE->Idx > - static_cast(NodeIdx); - }))) - return; - - // The first value node for store/insertelement is sext/zext/trunc? Skip it, - // resize to the final type. - bool IsProfitableToDemoteRoot = !IsStoreOrInsertElt; - if (NodeIdx != 0 && - VectorizableTree[NodeIdx]->State == TreeEntry::Vectorize && - (VectorizableTree[NodeIdx]->getOpcode() == Instruction::ZExt || - VectorizableTree[NodeIdx]->getOpcode() == Instruction::SExt || - VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc)) { - assert(IsStoreOrInsertElt && "Expected store/insertelement seeded graph."); - ++NodeIdx; - IsProfitableToDemoteRoot = true; - } - - // Analyzed in reduction already and not profitable - exit. - if (AnalyzedMinBWVals.contains(VectorizableTree[NodeIdx]->Scalars.front())) + if (!VectorizableTree.front()->UserTreeIndices.empty()) return; - SmallVector ToDemote; + // Conservatively determine if we can actually truncate the roots of the + // expression. Collect the values that can be demoted in ToDemote and + // additional roots that require investigating in Roots. + SmallVector ToDemote; DenseMap> DemotedConsts; - auto ComputeMaxBitWidth = [&](ArrayRef TreeRoot, unsigned VF, - bool IsTopRoot, bool IsProfitableToDemoteRoot, - unsigned Opcode, unsigned Limit) { - ToDemote.clear(); - auto *TreeRootIT = dyn_cast(TreeRoot[0]->getType()); - if (!TreeRootIT || !Opcode) - return 0u; - - if (AnalyzedMinBWVals.contains(TreeRoot.front())) - return 0u; - - unsigned NumParts = TTI->getNumberOfParts( - FixedVectorType::get(TreeRoot.front()->getType(), VF)); - - // The maximum bit width required to represent all the values that can be - // demoted without loss of precision. It would be safe to truncate the roots - // of the expression to this width. - unsigned MaxBitWidth = 1u; - - // True if the roots can be zero-extended back to their original type, - // rather than sign-extended. We know that if the leading bits are not - // demanded, we can safely zero-extend. So we initialize IsKnownPositive to - // True. + SmallVector Roots; + for (auto *Root : TreeRoot) { + DenseSet Visited; + if (!collectValuesToDemote(Root, ToDemote, DemotedConsts, Roots, Visited)) + return; + } + + // The maximum bit width required to represent all the values that can be + // demoted without loss of precision. It would be safe to truncate the roots + // of the expression to this width. + auto MaxBitWidth = 1u; + + // We first check if all the bits of the roots are demanded. If they're not, + // we can truncate the roots to this narrower type. + for (auto *Root : TreeRoot) { + auto Mask = DB->getDemandedBits(cast(Root)); + MaxBitWidth = std::max(Mask.getBitWidth() - Mask.countl_zero(), + MaxBitWidth); + } + + // True if the roots can be zero-extended back to their original type, rather + // than sign-extended. We know that if the leading bits are not demanded, we + // can safely zero-extend. So we initialize IsKnownPositive to True. + bool IsKnownPositive = true; + + // If all the bits of the roots are demanded, we can try a little harder to + // compute a narrower type. This can happen, for example, if the roots are + // getelementptr indices. InstCombine promotes these indices to the pointer + // width. Thus, all their bits are technically demanded even though the + // address computation might be vectorized in a smaller type. + // + // We start by looking at each entry that can be demoted. We compute the + // maximum bit width required to store the scalar by using ValueTracking to + // compute the number of high-order bits we can truncate. + if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType()) && + all_of(TreeRoot, [](Value *V) { + return all_of(V->users(), + [](User *U) { return isa(U); }); + })) { + MaxBitWidth = 8u; + // Determine if the sign bit of all the roots is known to be zero. If not, // IsKnownPositive is set to False. - bool IsKnownPositive = all_of(TreeRoot, [&](Value *R) { + IsKnownPositive = llvm::all_of(TreeRoot, [&](Value *R) { KnownBits Known = computeKnownBits(R, *DL); return Known.isNonNegative(); }); - // We first check if all the bits of the roots are demanded. If they're not, - // we can truncate the roots to this narrower type. - for (auto *Root : TreeRoot) { - unsigned NumSignBits = ComputeNumSignBits(Root, *DL, 0, AC, nullptr, DT); - TypeSize NumTypeBits = DL->getTypeSizeInBits(Root->getType()); - unsigned BitWidth1 = NumTypeBits - NumSignBits; - // If we can't prove that the sign bit is zero, we must add one to the - // maximum bit width to account for the unknown sign bit. This preserves - // the existing sign bit so we can safely sign-extend the root back to the - // original type. Otherwise, if we know the sign bit is zero, we will - // zero-extend the root instead. - // - // FIXME: This is somewhat suboptimal, as there will be cases where adding - // one to the maximum bit width will yield a larger-than-necessary - // type. In general, we need to add an extra bit only if we can't - // prove that the upper bit of the original type is equal to the - // upper bit of the proposed smaller type. If these two bits are - // the same (either zero or one) we know that sign-extending from - // the smaller type will result in the same value. Here, since we - // can't yet prove this, we are just making the proposed smaller - // type larger to ensure correctness. - if (!IsKnownPositive) - ++BitWidth1; - - APInt Mask = DB->getDemandedBits(cast(Root)); - unsigned BitWidth2 = Mask.getBitWidth() - Mask.countl_zero(); - MaxBitWidth = - std::max(std::min(BitWidth1, BitWidth2), MaxBitWidth); - } - - if (MaxBitWidth < 8 && MaxBitWidth > 1) - MaxBitWidth = 8; - - // If the original type is large, but reduced type does not improve the reg - // use - ignore it. - if (NumParts > 1 && - NumParts == - TTI->getNumberOfParts(FixedVectorType::get( - IntegerType::get(F->getContext(), bit_ceil(MaxBitWidth)), VF))) - return 0u; - - bool IsProfitableToDemote = Opcode == Instruction::Trunc || - Opcode == Instruction::SExt || - Opcode == Instruction::ZExt || NumParts > 1; - // Conservatively determine if we can actually truncate the roots of the - // expression. Collect the values that can be demoted in ToDemote and - // additional roots that require investigating in Roots. - for (auto *Root : TreeRoot) { - DenseSet Visited; - unsigned MaxDepthLevel; - bool NeedToDemote = IsProfitableToDemote; - - if (!collectValuesToDemote(Root, IsProfitableToDemoteRoot, MaxBitWidth, - ToDemote, DemotedConsts, Visited, - MaxDepthLevel, NeedToDemote) || - (MaxDepthLevel <= Limit && - !(((Opcode == Instruction::SExt || Opcode == Instruction::ZExt) && - (!IsTopRoot || !(IsStoreOrInsertElt || UserIgnoreList) || - DL->getTypeSizeInBits(Root->getType()) / - DL->getTypeSizeInBits( - cast(Root)->getOperand(0)->getType()) > - 2)) || - (Opcode == Instruction::Trunc && - (!IsTopRoot || !(IsStoreOrInsertElt || UserIgnoreList) || - DL->getTypeSizeInBits( - cast(Root)->getOperand(0)->getType()) / - DL->getTypeSizeInBits(Root->getType()) > - 2))))) - return 0u; - } - // Round MaxBitWidth up to the next power-of-two. - MaxBitWidth = bit_ceil(MaxBitWidth); - - return MaxBitWidth; - }; + // Determine the maximum number of bits required to store the scalar + // values. + for (auto *Scalar : ToDemote) { + auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, nullptr, DT); + auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType()); + MaxBitWidth = std::max(NumTypeBits - NumSignBits, MaxBitWidth); + } + + // If we can't prove that the sign bit is zero, we must add one to the + // maximum bit width to account for the unknown sign bit. This preserves + // the existing sign bit so we can safely sign-extend the root back to the + // original type. Otherwise, if we know the sign bit is zero, we will + // zero-extend the root instead. + // + // FIXME: This is somewhat suboptimal, as there will be cases where adding + // one to the maximum bit width will yield a larger-than-necessary + // type. In general, we need to add an extra bit only if we can't + // prove that the upper bit of the original type is equal to the + // upper bit of the proposed smaller type. If these two bits are the + // same (either zero or one) we know that sign-extending from the + // smaller type will result in the same value. Here, since we can't + // yet prove this, we are just making the proposed smaller type + // larger to ensure correctness. + if (!IsKnownPositive) + ++MaxBitWidth; + } + + // Round MaxBitWidth up to the next power-of-two. + MaxBitWidth = llvm::bit_ceil(MaxBitWidth); + + // If the maximum bit width we compute is less than the with of the roots' + // type, we can proceed with the narrowing. Otherwise, do nothing. + if (MaxBitWidth >= TreeRootIT->getBitWidth()) + return; // If we can truncate the root, we must collect additional values that might // be demoted as a result. That is, those seeded by truncations we will // modify. - // Add reduction ops sizes, if any. - if (UserIgnoreList && - isa(VectorizableTree.front()->Scalars.front()->getType())) { - for (Value *V : *UserIgnoreList) { - auto NumSignBits = ComputeNumSignBits(V, *DL, 0, AC, nullptr, DT); - auto NumTypeBits = DL->getTypeSizeInBits(V->getType()); - unsigned BitWidth1 = NumTypeBits - NumSignBits; - if (!isKnownNonNegative(V, SimplifyQuery(*DL))) - ++BitWidth1; - auto Mask = DB->getDemandedBits(cast(V)); - unsigned BitWidth2 = Mask.getBitWidth() - Mask.countl_zero(); - ReductionBitWidth = - std::max(std::min(BitWidth1, BitWidth2), ReductionBitWidth); - } - if (ReductionBitWidth < 8 && ReductionBitWidth > 1) - ReductionBitWidth = 8; - - ReductionBitWidth = bit_ceil(ReductionBitWidth); - } - bool IsTopRoot = NodeIdx == 0; - while (NodeIdx < VectorizableTree.size() && - VectorizableTree[NodeIdx]->State == TreeEntry::Vectorize && - VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc) - ++NodeIdx; - while (NodeIdx < VectorizableTree.size()) { - ArrayRef TreeRoot = VectorizableTree[NodeIdx]->Scalars; - unsigned Limit = 2; - unsigned Opcode = VectorizableTree[NodeIdx]->getOpcode(); - if (IsTopRoot && - ReductionBitWidth == - DL->getTypeSizeInBits( - VectorizableTree.front()->Scalars.front()->getType())) - Limit = 3; - unsigned MaxBitWidth = ComputeMaxBitWidth( - TreeRoot, VectorizableTree[NodeIdx]->getVectorFactor(), IsTopRoot, - IsProfitableToDemoteRoot, Opcode, Limit); - IsTopRoot = false; - IsProfitableToDemoteRoot = true; - - if (TruncNodes.empty()) { - NodeIdx = VectorizableTree.size(); - } else { - NodeIdx = *TruncNodes.begin() + 1; - TruncNodes.erase(TruncNodes.begin()); - } - - // If the maximum bit width we compute is less than the with of the roots' - // type, we can proceed with the narrowing. Otherwise, do nothing. - if (MaxBitWidth == 0 || - MaxBitWidth >= - cast(TreeRoot.front()->getType())->getBitWidth()) { - if (UserIgnoreList) - AnalyzedMinBWVals.insert(TreeRoot.begin(), TreeRoot.end()); + while (!Roots.empty()) { + DenseSet Visited; + collectValuesToDemote(Roots.pop_back_val(), ToDemote, DemotedConsts, Roots, + Visited); + } + + // Check that all users are marked for demotion. + DenseSet Demoted(ToDemote.begin(), ToDemote.end()); + DenseSet Visited; + for (Value *V: ToDemote) { + const TreeEntry *TE = getTreeEntry(V); + assert(TE && "Expected vectorized scalar."); + if (!Visited.insert(TE).second) continue; - } - - // Finally, map the values we can demote to the maximum bit with we - // computed. - for (Value *Scalar : ToDemote) { - TreeEntry *TE = getTreeEntry(Scalar); - assert(TE && "Expected vectorized scalar."); - if (MinBWs.contains(TE)) - continue; - bool IsSigned = any_of(TE->Scalars, [&](Value *R) { - return !isKnownNonNegative(R, SimplifyQuery(*DL)); - }); - MinBWs.try_emplace(TE, MaxBitWidth, IsSigned); - const auto *I = cast(Scalar); - auto DCIt = DemotedConsts.find(I); - if (DCIt != DemotedConsts.end()) { - for (unsigned Idx : DCIt->getSecond()) { - // Check that all instructions operands are demoted. + if (!all_of(TE->UserTreeIndices, [&](const EdgeInfo &EI) { + return all_of(EI.UserTE->Scalars, + [&](Value *V) { return Demoted.contains(V); }); + })) + return; + } + // Finally, map the values we can demote to the maximum bit with we computed. + for (auto *Scalar : ToDemote) { + auto *TE = getTreeEntry(Scalar); + assert(TE && "Expected vectorized scalar."); + if (MinBWs.contains(TE)) + continue; + bool IsSigned = any_of(TE->Scalars, [&](Value *R) { + KnownBits Known = computeKnownBits(R, *DL); + return !Known.isNonNegative(); + }); + MinBWs.try_emplace(TE, MaxBitWidth, IsSigned); + const auto *I = cast(Scalar); + auto DCIt = DemotedConsts.find(I); + if (DCIt != DemotedConsts.end()) { + for (unsigned Idx : DCIt->getSecond()) { + // Check that all instructions operands are demoted. + if (all_of(TE->Scalars, [&](Value *V) { + auto SIt = DemotedConsts.find(cast(V)); + return SIt != DemotedConsts.end() && + is_contained(SIt->getSecond(), Idx); + })) { const TreeEntry *CTE = getOperandEntry(TE, Idx); - if (all_of(TE->Scalars, - [&](Value *V) { - auto SIt = DemotedConsts.find(cast(V)); - return SIt != DemotedConsts.end() && - is_contained(SIt->getSecond(), Idx); - }) || - all_of(CTE->Scalars, Constant::classof)) - MinBWs.try_emplace(CTE, MaxBitWidth, IsSigned); + MinBWs.try_emplace(CTE, MaxBitWidth, IsSigned); } } } diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll index 5e3fd156666f..cef791633655 100644 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll @@ -17,13 +17,12 @@ define void @test1(<4 x i16> %a, <4 x i16> %b, ptr %p) { ; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i64 [[S0]] ; CHECK-NEXT: [[LOAD0:%.*]] = load i64, ptr [[GEP0]], align 4 ; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <4 x i32> [[SUB0]], <4 x i32> poison, <2 x i32> -; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i32> [[TMP0]], i32 0 -; CHECK-NEXT: [[TMP2:%.*]] = sext i32 [[TMP1]] to i64 +; CHECK-NEXT: [[TMP1:%.*]] = sext <2 x i32> [[TMP0]] to <2 x i64> +; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0 ; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP2]] ; CHECK-NEXT: [[LOAD1:%.*]] = load i64, ptr [[GEP1]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP0]], i32 1 -; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[TMP3]] to i64 -; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP4]] +; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1 +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP3]] ; CHECK-NEXT: [[LOAD2:%.*]] = load i64, ptr [[GEP2]], align 4 ; CHECK-NEXT: [[E3:%.*]] = extractelement <4 x i32> [[SUB0]], i32 3 ; CHECK-NEXT: [[S3:%.*]] = sext i32 [[E3]] to i64 diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll index 1cce52060c47..47485e514ec2 100644 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ;test_i16_extend NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer,dce,instcombine -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s +; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer,dce,instcombine -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s ; RUN: cat %t | FileCheck -check-prefix=YAML %s -; RUN: opt -S -mtriple=aarch64--linux-gnu -passes='slp-vectorizer,dce,instcombine' -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s +; RUN: opt -S -mtriple=aarch64--linux-gnu -passes='slp-vectorizer,dce,instcombine' -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s ; RUN: cat %t | FileCheck -check-prefix=YAML %s diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll index a7a7f642ced5..d67fdc1cd6aa 100644 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll @@ -28,11 +28,21 @@ entry: define i64 @red_zext_ld_4xi64(ptr %ptr) { ; CHECK-LABEL: @red_zext_ld_4xi64( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[PTR:%.*]], align 1 -; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i16> -; CHECK-NEXT: [[TMP2:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]]) -; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[TMP2]] to i64 -; CHECK-NEXT: ret i64 [[TMP3]] +; CHECK-NEXT: [[LD0:%.*]] = load i8, ptr [[PTR:%.*]], align 1 +; CHECK-NEXT: [[ZEXT:%.*]] = zext i8 [[LD0]] to i64 +; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 1 +; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[GEP]], align 1 +; CHECK-NEXT: [[ZEXT_1:%.*]] = zext i8 [[LD1]] to i64 +; CHECK-NEXT: [[ADD_1:%.*]] = add nuw nsw i64 [[ZEXT]], [[ZEXT_1]] +; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 2 +; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[GEP_1]], align 1 +; CHECK-NEXT: [[ZEXT_2:%.*]] = zext i8 [[LD2]] to i64 +; CHECK-NEXT: [[ADD_2:%.*]] = add nuw nsw i64 [[ADD_1]], [[ZEXT_2]] +; CHECK-NEXT: [[GEP_2:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 3 +; CHECK-NEXT: [[LD3:%.*]] = load i8, ptr [[GEP_2]], align 1 +; CHECK-NEXT: [[ZEXT_3:%.*]] = zext i8 [[LD3]] to i64 +; CHECK-NEXT: [[ADD_3:%.*]] = add nuw nsw i64 [[ADD_2]], [[ZEXT_3]] +; CHECK-NEXT: ret i64 [[ADD_3]] ; entry: %ld0 = load i8, ptr %ptr diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll index 500f10659f04..000e7a56df37 100644 --- a/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll @@ -802,10 +802,9 @@ define i64 @red_zext_ld_4xi64(ptr %ptr) { ; CHECK-LABEL: @red_zext_ld_4xi64( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[PTR:%.*]], align 1 -; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i16> -; CHECK-NEXT: [[TMP2:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]]) -; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[TMP2]] to i64 -; CHECK-NEXT: ret i64 [[TMP3]] +; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i64> +; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP1]]) +; CHECK-NEXT: ret i64 [[TMP2]] ; entry: %ld0 = load i8, ptr %ptr diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll index 05511f843a68..4565d4928ba4 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll @@ -15,12 +15,11 @@ define { i64, i64 } @patatino(double %arg) { ; CHECK-NEXT: [[TMP6:%.*]] = load <2 x double>, ptr getelementptr inbounds ([6 x double], ptr @global, i64 0, i64 4), align 16 ; CHECK-NEXT: [[TMP7:%.*]] = fadd <2 x double> [[TMP6]], [[TMP5]] ; CHECK-NEXT: [[TMP8:%.*]] = fptosi <2 x double> [[TMP7]] to <2 x i32> -; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[TMP8]], i32 0 -; CHECK-NEXT: [[TMP10:%.*]] = sext i32 [[TMP9]] to i64 +; CHECK-NEXT: [[TMP9:%.*]] = sext <2 x i32> [[TMP8]] to <2 x i64> +; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i64> [[TMP9]], i32 0 ; CHECK-NEXT: [[T16:%.*]] = insertvalue { i64, i64 } undef, i64 [[TMP10]], 0 -; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[TMP8]], i32 1 -; CHECK-NEXT: [[TMP12:%.*]] = sext i32 [[TMP11]] to i64 -; CHECK-NEXT: [[T17:%.*]] = insertvalue { i64, i64 } [[T16]], i64 [[TMP12]], 1 +; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i64> [[TMP9]], i32 1 +; CHECK-NEXT: [[T17:%.*]] = insertvalue { i64, i64 } [[T16]], i64 [[TMP11]], 1 ; CHECK-NEXT: ret { i64, i64 } [[T17]] ; bb: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll index 5ee801607653..a0af8e36b36c 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 -; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-6 < %s | FileCheck %s +; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-3 < %s | FileCheck %s define void @t(i64 %v) { ; CHECK-LABEL: define void @t( diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll index 6051638562b5..6e512fcbb739 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll @@ -6,17 +6,18 @@ define void @test(i8 %0) { ; CHECK-SAME: i8 [[TMP0:%.*]]) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> , i8 [[TMP0]], i32 1 -; CHECK-NEXT: [[TMP2:%.*]] = sext <2 x i8> [[TMP1]] to <2 x i32> -; CHECK-NEXT: [[TMP3:%.*]] = mul <2 x i8> [[TMP1]], zeroinitializer -; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[TMP3]], i32 0 -; CHECK-NEXT: [[TMP5:%.*]] = zext i8 [[TMP4]] to i32 -; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i8> [[TMP3]], i32 1 -; CHECK-NEXT: [[TMP7:%.*]] = zext i8 [[TMP6]] to i32 -; CHECK-NEXT: [[ADD:%.*]] = or i32 [[TMP5]], [[TMP7]] +; CHECK-NEXT: [[TMP2:%.*]] = sext <2 x i8> [[TMP1]] to <2 x i16> +; CHECK-NEXT: [[TMP3:%.*]] = sext <2 x i16> [[TMP2]] to <2 x i32> +; CHECK-NEXT: [[TMP4:%.*]] = mul <2 x i16> [[TMP2]], zeroinitializer +; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i16> [[TMP4]], i32 0 +; CHECK-NEXT: [[TMP6:%.*]] = zext i16 [[TMP5]] to i32 +; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i16> [[TMP4]], i32 1 +; CHECK-NEXT: [[TMP8:%.*]] = zext i16 [[TMP7]] to i32 +; CHECK-NEXT: [[ADD:%.*]] = or i32 [[TMP6]], [[TMP8]] ; CHECK-NEXT: [[SHR:%.*]] = lshr i32 [[ADD]], 1 ; CHECK-NEXT: [[CONV9:%.*]] = trunc i32 [[SHR]] to i8 ; CHECK-NEXT: store i8 [[CONV9]], ptr null, align 1 -; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> +; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <8 x i32> ; CHECK-NEXT: ret void ; entry: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll index 4acd63078b82..2c834616becc 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll @@ -6,20 +6,15 @@ define void @test(i64 %d.promoted.i) { ; CHECK-SAME: i64 [[D_PROMOTED_I:%.*]]) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[AND_1_I:%.*]] = and i64 0, [[D_PROMOTED_I]] -; CHECK-NEXT: [[TMP0:%.*]] = insertelement <8 x i64> , i64 [[AND_1_I]], i32 1 -; CHECK-NEXT: [[TMP1:%.*]] = trunc <8 x i64> [[TMP0]] to <8 x i1> -; CHECK-NEXT: [[TMP2:%.*]] = mul <8 x i1> [[TMP1]], zeroinitializer ; CHECK-NEXT: [[AND_1_I_1:%.*]] = and i64 0, 0 -; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i64> , i64 [[AND_1_I_1]], i32 1 -; CHECK-NEXT: [[TMP4:%.*]] = trunc <8 x i64> [[TMP3]] to <8 x i1> -; CHECK-NEXT: [[TMP5:%.*]] = mul <8 x i1> [[TMP4]], zeroinitializer -; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP5]]) -; CHECK-NEXT: [[TMP7:%.*]] = zext i1 [[TMP6]] to i32 -; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP2]]) -; CHECK-NEXT: [[TMP9:%.*]] = zext i1 [[TMP8]] to i32 -; CHECK-NEXT: [[OP_RDX:%.*]] = or i32 [[TMP7]], [[TMP9]] -; CHECK-NEXT: [[TMP10:%.*]] = and i32 [[OP_RDX]], 0 -; CHECK-NEXT: store i32 [[TMP10]], ptr null, align 4 +; CHECK-NEXT: [[TMP0:%.*]] = insertelement <16 x i64> , i64 [[AND_1_I_1]], i32 1 +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <16 x i64> [[TMP0]], i64 [[AND_1_I]], i32 9 +; CHECK-NEXT: [[TMP2:%.*]] = trunc <16 x i64> [[TMP1]] to <16 x i1> +; CHECK-NEXT: [[TMP3:%.*]] = mul <16 x i1> [[TMP2]], zeroinitializer +; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP3]]) +; CHECK-NEXT: [[TMP5:%.*]] = zext i1 [[TMP4]] to i32 +; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[TMP5]], 0 +; CHECK-NEXT: store i32 [[TMP6]], ptr null, align 4 ; CHECK-NEXT: ret void ; entry: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll index a316415dcc6b..651631de2c35 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll @@ -17,15 +17,12 @@ target triple = "x86_64-unknown-linux-gnu" define i8 @PR31243_zext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) { ; SSE-LABEL: @PR31243_zext( ; SSE-NEXT: entry: -; SSE-NEXT: [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0 -; SSE-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1 -; SSE-NEXT: [[TMP2:%.*]] = or <2 x i8> [[TMP1]], -; SSE-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0 -; SSE-NEXT: [[TMP4:%.*]] = zext i8 [[TMP3]] to i64 -; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]] -; SSE-NEXT: [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1 -; SSE-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i64 -; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]] +; SSE-NEXT: [[TMP0:%.*]] = or i8 [[V0:%.*]], 1 +; SSE-NEXT: [[TMP1:%.*]] = or i8 [[V1:%.*]], 1 +; SSE-NEXT: [[TMP2:%.*]] = zext i8 [[TMP0]] to i64 +; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP2]] +; SSE-NEXT: [[TMP3:%.*]] = zext i8 [[TMP1]] to i64 +; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP3]] ; SSE-NEXT: [[T6:%.*]] = load i8, ptr [[T4]], align 1 ; SSE-NEXT: [[T7:%.*]] = load i8, ptr [[T5]], align 1 ; SSE-NEXT: [[T8:%.*]] = add i8 [[T6]], [[T7]] @@ -76,15 +73,12 @@ entry: define i8 @PR31243_sext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) { ; SSE-LABEL: @PR31243_sext( ; SSE-NEXT: entry: -; SSE-NEXT: [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0 -; SSE-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1 -; SSE-NEXT: [[TMP2:%.*]] = or <2 x i8> [[TMP1]], -; SSE-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0 -; SSE-NEXT: [[TMP4:%.*]] = sext i8 [[TMP3]] to i64 -; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]] -; SSE-NEXT: [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1 -; SSE-NEXT: [[TMP6:%.*]] = sext i8 [[TMP5]] to i64 -; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]] +; SSE-NEXT: [[TMP0:%.*]] = or i8 [[V0:%.*]], 1 +; SSE-NEXT: [[TMP1:%.*]] = or i8 [[V1:%.*]], 1 +; SSE-NEXT: [[TMP2:%.*]] = sext i8 [[TMP0]] to i64 +; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP2]] +; SSE-NEXT: [[TMP3:%.*]] = sext i8 [[TMP1]] to i64 +; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP3]] ; SSE-NEXT: [[T6:%.*]] = load i8, ptr [[T4]], align 1 ; SSE-NEXT: [[T7:%.*]] = load i8, ptr [[T5]], align 1 ; SSE-NEXT: [[T8:%.*]] = add i8 [[T6]], [[T7]] @@ -95,12 +89,13 @@ define i8 @PR31243_sext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) { ; AVX-NEXT: [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0 ; AVX-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1 ; AVX-NEXT: [[TMP2:%.*]] = or <2 x i8> [[TMP1]], -; AVX-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0 -; AVX-NEXT: [[TMP4:%.*]] = sext i8 [[TMP3]] to i64 -; AVX-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]] -; AVX-NEXT: [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1 -; AVX-NEXT: [[TMP6:%.*]] = sext i8 [[TMP5]] to i64 -; AVX-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]] +; AVX-NEXT: [[TMP3:%.*]] = sext <2 x i8> [[TMP2]] to <2 x i16> +; AVX-NEXT: [[TMP4:%.*]] = extractelement <2 x i16> [[TMP3]], i64 0 +; AVX-NEXT: [[TMP5:%.*]] = sext i16 [[TMP4]] to i64 +; AVX-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP5]] +; AVX-NEXT: [[TMP6:%.*]] = extractelement <2 x i16> [[TMP3]], i64 1 +; AVX-NEXT: [[TMP7:%.*]] = sext i16 [[TMP6]] to i64 +; AVX-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP7]] ; AVX-NEXT: [[T6:%.*]] = load i8, ptr [[T4]], align 1 ; AVX-NEXT: [[T7:%.*]] = load i8, ptr [[T5]], align 1 ; AVX-NEXT: [[T8:%.*]] = add i8 [[T6]], [[T7]] diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll index 3cc32c1fc7b2..88f75c37846e 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll @@ -15,8 +15,8 @@ define i32 @phi3UndefInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %arg3) { ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) -; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 +; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> +; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: @@ -52,8 +52,8 @@ define i32 @phi2UndefInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %arg3) { ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) -; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 +; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> +; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: @@ -89,8 +89,8 @@ define i32 @phi1UndefInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %arg3) { ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) -; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 +; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> +; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: @@ -127,8 +127,8 @@ define i32 @phi1Undef1PoisonInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %ar ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) -; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 +; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> +; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: @@ -165,8 +165,8 @@ define i32 @phi1Undef2PoisonInputs(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %a ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) -; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 +; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> +; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: @@ -202,8 +202,8 @@ define i32 @phi1Undef1PoisonGapInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: ; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]]) -; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32 +; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32> +; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]]) ; CHECK-NEXT: ret i32 [[TMP6]] ; entry: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/resched.ll b/llvm/test/Transforms/SLPVectorizer/X86/resched.ll index b7237cbb02bb..78c6d9516a3d 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/resched.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/resched.ll @@ -11,26 +11,26 @@ define fastcc void @_ZN12_GLOBAL__N_127PolynomialMultiplyRecognize9recognizeEv() ; CHECK: if.then22.i: ; CHECK-NEXT: [[SUB_I:%.*]] = add nsw i32 undef, -1 ; CHECK-NEXT: [[CONV31_I:%.*]] = and i32 undef, [[SUB_I]] -; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[CONV31_I]], i32 0 -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP2:%.*]] = lshr <4 x i32> [[TMP1]], +; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[CONV31_I]], i32 0 +; CHECK-NEXT: [[SHUFFLE1:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = lshr <4 x i32> [[SHUFFLE1]], ; CHECK-NEXT: [[SHR_4_I_I:%.*]] = lshr i32 [[CONV31_I]], 5 ; CHECK-NEXT: [[SHR_5_I_I:%.*]] = lshr i32 [[CONV31_I]], 6 ; CHECK-NEXT: [[SHR_6_I_I:%.*]] = lshr i32 [[CONV31_I]], 7 ; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i32> poison, i32 [[CONV31_I]], i32 0 -; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> poison, <8 x i32> zeroinitializer -; CHECK-NEXT: [[TMP5:%.*]] = lshr <8 x i32> [[TMP4]], -; CHECK-NEXT: [[TMP6:%.*]] = insertelement <16 x i32> poison, i32 [[SUB_I]], i32 0 -; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <16 x i32> -; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <16 x i32> [[TMP6]], <16 x i32> [[TMP7]], <16 x i32> -; CHECK-NEXT: [[TMP9:%.*]] = insertelement <16 x i32> [[TMP8]], i32 [[SHR_4_I_I]], i32 5 -; CHECK-NEXT: [[TMP10:%.*]] = insertelement <16 x i32> [[TMP9]], i32 [[SHR_5_I_I]], i32 6 -; CHECK-NEXT: [[TMP11:%.*]] = insertelement <16 x i32> [[TMP10]], i32 [[SHR_6_I_I]], i32 7 -; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> poison, <16 x i32> -; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <16 x i32> [[TMP11]], <16 x i32> [[TMP12]], <16 x i32> -; CHECK-NEXT: [[TMP14:%.*]] = trunc <16 x i32> [[TMP13]] to <16 x i8> -; CHECK-NEXT: [[TMP15:%.*]] = and <16 x i8> [[TMP14]], -; CHECK-NEXT: store <16 x i8> [[TMP15]], ptr undef, align 1 +; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> poison, <8 x i32> zeroinitializer +; CHECK-NEXT: [[TMP4:%.*]] = lshr <8 x i32> [[SHUFFLE]], +; CHECK-NEXT: [[TMP5:%.*]] = insertelement <16 x i32> poison, i32 [[SUB_I]], i32 0 +; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <16 x i32> +; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> [[TMP6]], <16 x i32> +; CHECK-NEXT: [[TMP8:%.*]] = insertelement <16 x i32> [[TMP7]], i32 [[SHR_4_I_I]], i32 5 +; CHECK-NEXT: [[TMP9:%.*]] = insertelement <16 x i32> [[TMP8]], i32 [[SHR_5_I_I]], i32 6 +; CHECK-NEXT: [[TMP10:%.*]] = insertelement <16 x i32> [[TMP9]], i32 [[SHR_6_I_I]], i32 7 +; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <16 x i32> +; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <16 x i32> [[TMP10]], <16 x i32> [[TMP11]], <16 x i32> +; CHECK-NEXT: [[TMP13:%.*]] = trunc <16 x i32> [[TMP12]] to <16 x i8> +; CHECK-NEXT: [[TMP14:%.*]] = and <16 x i8> [[TMP13]], +; CHECK-NEXT: store <16 x i8> [[TMP14]], ptr undef, align 1 ; CHECK-NEXT: unreachable ; CHECK: if.end50.i: ; CHECK-NEXT: ret void diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll index 1d1fcec2a7ae..5d22b5a4873b 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll @@ -7,10 +7,12 @@ define i1 @test(i1 %cmp5.not.31) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i1> , i1 [[CMP5_NOT_31]], i32 0 ; CHECK-NEXT: [[TMP1:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> zeroinitializer, <4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP2:%.*]] = mul <4 x i32> [[TMP1]], -; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]]) -; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[TMP3]], 0 -; CHECK-NEXT: [[CMP_NOT_I_I:%.*]] = icmp eq i32 [[TMP4]], 0 +; CHECK-NEXT: [[TMP2:%.*]] = trunc <4 x i32> [[TMP1]] to <4 x i1> +; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32> +; CHECK-NEXT: [[TMP4:%.*]] = mul <4 x i32> [[TMP3]], +; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]]) +; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[TMP5]], 0 +; CHECK-NEXT: [[CMP_NOT_I_I:%.*]] = icmp eq i32 [[TMP6]], 0 ; CHECK-NEXT: ret i1 [[CMP_NOT_I_I]] ; entry: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll index 2f6868d8dfd6..c1dd90d0e9a7 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll @@ -8,18 +8,17 @@ ; YAML-NEXT: Function: stores ; YAML-NEXT: Args: ; YAML-NEXT: - String: 'Stores SLP vectorized with cost ' -; YAML-NEXT: - Cost: '-7' +; YAML-NEXT: - Cost: '-3' ; YAML-NEXT: - String: ' and with tree size ' ; YAML-NEXT: - TreeSize: '6' define void @stores(ptr noalias %in, ptr noalias %inn, ptr noalias %out) { ; CHECK-LABEL: @stores( ; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[IN:%.*]], align 1 ; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[INN:%.*]], align 1 -; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i16> -; CHECK-NEXT: [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i16> -; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i16> [[TMP3]], [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = zext <4 x i16> [[TMP5]] to <4 x i64> -; CHECK-NEXT: store <4 x i64> [[TMP6]], ptr [[OUT:%.*]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i64> +; CHECK-NEXT: [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i64> +; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[TMP3]], [[TMP4]] +; CHECK-NEXT: store <4 x i64> [[TMP5]], ptr [[OUT:%.*]], align 4 ; CHECK-NEXT: ret void ; %load.1 = load i8, ptr %in, align 1 @@ -64,18 +63,17 @@ define void @stores(ptr noalias %in, ptr noalias %inn, ptr noalias %out) { ; YAML-NEXT: Function: insertelems ; YAML-NEXT: Args: ; YAML-NEXT: - String: 'SLP vectorized with cost ' -; YAML-NEXT: - Cost: '-9' +; YAML-NEXT: - Cost: '-5' ; YAML-NEXT: - String: ' and with tree size ' ; YAML-NEXT: - TreeSize: '6' define <4 x i64> @insertelems(ptr noalias %in, ptr noalias %inn) { ; CHECK-LABEL: @insertelems( ; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[IN:%.*]], align 1 ; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[INN:%.*]], align 1 -; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i16> -; CHECK-NEXT: [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i16> -; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i16> [[TMP3]], [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = zext <4 x i16> [[TMP5]] to <4 x i64> -; CHECK-NEXT: ret <4 x i64> [[TMP6]] +; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i64> +; CHECK-NEXT: [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i64> +; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[TMP3]], [[TMP4]] +; CHECK-NEXT: ret <4 x i64> [[TMP5]] ; %load.1 = load i8, ptr %in, align 1 %gep.1 = getelementptr inbounds i8, ptr %in, i64 1 diff --git a/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll b/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll index ff6f0bdd3db8..061fbdb45a13 100644 --- a/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll +++ b/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll @@ -10,8 +10,8 @@ define i32 @alt_cmp(i16 %call46) { ; CHECK-NEXT: [[TMP2:%.*]] = icmp ult <4 x i16> [[TMP0]], [[TMP1]] ; CHECK-NEXT: [[TMP3:%.*]] = icmp ugt <4 x i16> [[TMP0]], [[TMP1]] ; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i1> [[TMP2]], <4 x i1> [[TMP3]], <4 x i32> -; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) -; CHECK-NEXT: [[TMP6:%.*]] = zext i1 [[TMP5]] to i16 +; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i1> [[TMP4]] to <4 x i16> +; CHECK-NEXT: [[TMP6:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP5]]) ; CHECK-NEXT: [[OP_RDX:%.*]] = or i16 [[TMP6]], 0 ; CHECK-NEXT: [[EXT:%.*]] = zext i16 [[OP_RDX]] to i32 ; CHECK-NEXT: ret i32 [[EXT]] -- GitLab From ecf7db8b52d7061ef8f14c1f7b6fcc370072d087 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Thu, 7 Mar 2024 12:55:13 -0800 Subject: [PATCH 502/929] [lldb] Disable shell tests affected by ld_new bug (#84246) Equivalent to the changes made in https://github.com/llvm/llvm-project/pull/83941, except to support shell tests. --- .../test/Shell/Unwind/eh-frame-dwarf-unwind.test | 2 +- .../Unwind/thread-step-out-ret-addr-check.test | 2 +- lldb/test/Shell/lit.cfg.py | 16 ++++++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/lldb/test/Shell/Unwind/eh-frame-dwarf-unwind.test b/lldb/test/Shell/Unwind/eh-frame-dwarf-unwind.test index 3df9906394f4..7b5d6650fe2f 100644 --- a/lldb/test/Shell/Unwind/eh-frame-dwarf-unwind.test +++ b/lldb/test/Shell/Unwind/eh-frame-dwarf-unwind.test @@ -1,7 +1,7 @@ # Test handing of dwarf expressions specifying the location of registers, if # those expressions refer to the frame's CFA value. -# UNSUPPORTED: system-windows +# UNSUPPORTED: system-windows, ld_new-bug # REQUIRES: target-x86_64, native # RUN: %clang_host %p/Inputs/call-asm.c %p/Inputs/eh-frame-dwarf-unwind.s -o %t diff --git a/lldb/test/Shell/Unwind/thread-step-out-ret-addr-check.test b/lldb/test/Shell/Unwind/thread-step-out-ret-addr-check.test index 682b0e5332b1..9bc7c78f79b2 100644 --- a/lldb/test/Shell/Unwind/thread-step-out-ret-addr-check.test +++ b/lldb/test/Shell/Unwind/thread-step-out-ret-addr-check.test @@ -2,7 +2,7 @@ # points to non-executable memory. # REQUIRES: target-x86_64 -# UNSUPPORTED: system-windows +# UNSUPPORTED: system-windows, ld_new-bug # RUN: %clang_host %p/Inputs/call-asm.c -x assembler-with-cpp %p/Inputs/thread-step-out-ret-addr-check.s -o %t # RUN: not %lldb %t -s %s -b 2>&1 | FileCheck %s diff --git a/lldb/test/Shell/lit.cfg.py b/lldb/test/Shell/lit.cfg.py index d75c1f532e14..31afe5151c06 100644 --- a/lldb/test/Shell/lit.cfg.py +++ b/lldb/test/Shell/lit.cfg.py @@ -1,5 +1,6 @@ # -*- Python -*- +import json import os import platform import re @@ -179,3 +180,18 @@ if can_set_dbregs: if "LD_PRELOAD" in os.environ: config.available_features.add("ld_preload-present") + +# Determine if a specific version of Xcode's linker contains a bug. We want to +# skip affected tests if they contain this bug. +if platform.system() == "Darwin": + try: + raw_version_details = subprocess.check_output( + ("xcrun", "ld", "-version_details") + ) + version_details = json.loads(raw_version_details) + version = version_details.get("version", "0") + version_tuple = tuple(int(x) for x in version.split(".")) + if (1000,) <= version_tuple <= (1109,): + config.available_features.add("ld_new-bug") + except: + pass -- GitLab From 641b98a0d1e20da9500aa012ced41e53967a423f Mon Sep 17 00:00:00 2001 From: Amara Emerson Date: Thu, 7 Mar 2024 12:56:05 -0800 Subject: [PATCH 503/929] [GlobalISel] Fix crash in tryFoldAndOrOrICmpsUsingRanges() with pointer types. --- .../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 3 + .../GlobalISel/combine-logic-of-compare.mir | 63 +++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp index 2f18a64ca285..4862207d53f4 100644 --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -6713,6 +6713,9 @@ bool CombinerHelper::tryFoldAndOrOrICmpsUsingRanges(GLogicalBinOp *Logic, LLT CmpTy = MRI.getType(Cmp1->getReg(0)); LLT CmpOperandTy = MRI.getType(R1); + if (CmpOperandTy.isPointer()) + return false; + // We build ands, adds, and constants of type CmpOperandTy. // They must be legal to build. if (!isLegalOrBeforeLegalizer({TargetOpcode::G_AND, CmpOperandTy}) || diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-logic-of-compare.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-logic-of-compare.mir index d050823e3b94..1eb445c03efc 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-logic-of-compare.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-logic-of-compare.mir @@ -406,3 +406,66 @@ body: | %zext:_(<2 x s64>) = G_ZEXT %and(<2 x s1>) $q0 = COPY %zext ... +--- +name: test_dont_combine_pointers +body: | + ; CHECK-LABEL: name: test_dont_combine_pointers + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[C:%[0-9]+]]:_(p0) = G_CONSTANT i64 0 + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -8 + ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[C1]](s64) + ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -16 + ; CHECK-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p0) = G_INTTOPTR [[C2]](s64) + ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s1) = G_CONSTANT i1 false + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: successors: %bb.2(0x60000000), %bb.3(0x20000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[C]](p0) :: (load (p0)) + ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[LOAD]](p0), [[INTTOPTR]] + ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[LOAD]](p0), [[INTTOPTR1]] + ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s1) = G_AND [[ICMP]], [[ICMP1]] + ; CHECK-NEXT: G_BRCOND [[AND]](s1), %bb.3 + ; CHECK-NEXT: G_BR %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: successors: %bb.1(0x55555555), %bb.3(0x2aaaaaab) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: G_BRCOND [[C3]](s1), %bb.1 + ; CHECK-NEXT: G_BR %bb.3 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.3: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: G_BR %bb.1 + bb.1: + %1:_(p0) = G_CONSTANT i64 0 + %3:_(s64) = G_CONSTANT i64 -8 + %2:_(p0) = G_INTTOPTR %3(s64) + %6:_(s64) = G_CONSTANT i64 -16 + %5:_(p0) = G_INTTOPTR %6(s64) + %10:_(s1) = G_CONSTANT i1 false + + bb.2: + successors: %bb.4(0x60000000), %bb.3(0x20000000) + + %0:_(p0) = G_LOAD %1(p0) :: (load (p0)) + %4:_(s1) = G_ICMP intpred(eq), %0(p0), %2 + %7:_(s1) = G_ICMP intpred(eq), %0(p0), %5 + %8:_(s1) = G_OR %4, %7 + %9:_(s1) = G_SELECT %8(s1), %10, %10 + G_BRCOND %8(s1), %bb.4 + G_BR %bb.3 + + bb.4: + successors: %bb.2(0x55555555), %bb.3(0x2aaaaaab) + + G_BRCOND %10(s1), %bb.2 + G_BR %bb.3 + + bb.3: + G_BR %bb.2 + +... -- GitLab From 143afb405a7e12e3fe1622b92f046ab2380c8981 Mon Sep 17 00:00:00 2001 From: Maksim Panchenko Date: Thu, 7 Mar 2024 13:04:02 -0800 Subject: [PATCH 504/929] [BOLT] Add reading support for Linux kernel .altinstructions section (#84283) Read .altinstructions and annotate instructions that have alternative sequences with "AltInst" annotation. Note that some instructions may have more than one alternatives, in which case they will have multiple annotations in the form "AltInst", "AltInst2", "AltInst3", etc. --- bolt/lib/Rewrite/LinuxKernelRewriter.cpp | 141 +++++++++++++++++++++++ bolt/test/X86/linux-alt-instruction.s | 92 +++++++++++++++ 2 files changed, 233 insertions(+) create mode 100644 bolt/test/X86/linux-alt-instruction.s diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp index 964a47346592..ecfbea3cb511 100644 --- a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp +++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp @@ -27,6 +27,21 @@ using namespace bolt; namespace opts { +static cl::opt + AltInstHasPadLen("alt-inst-has-padlen", + cl::desc("specify that .altinstructions has padlen field"), + cl::init(false), cl::Hidden, cl::cat(BoltCategory)); + +static cl::opt + AltInstFeatureSize("alt-inst-feature-size", + cl::desc("size of feature field in .altinstructions"), + cl::init(2), cl::Hidden, cl::cat(BoltCategory)); + +static cl::opt + DumpAltInstructions("dump-alt-instructions", + cl::desc("dump Linux alternative instructions info"), + cl::init(false), cl::Hidden, cl::cat(BoltCategory)); + static cl::opt DumpExceptions("dump-linux-exceptions", cl::desc("dump Linux kernel exception table"), @@ -157,6 +172,9 @@ class LinuxKernelRewriter final : public MetadataRewriter { /// Alignment of paravirtual patch structures. static constexpr size_t PARA_PATCH_ALIGN = 8; + /// .altinstructions section. + ErrorOr AltInstrSection = std::errc::bad_address; + /// Section containing Linux bug table. ErrorOr BugTableSection = std::errc::bad_address; @@ -205,6 +223,9 @@ class LinuxKernelRewriter final : public MetadataRewriter { Error readBugTable(); + /// Read alternative instruction info from .altinstructions. + Error readAltInstructions(); + /// Mark instructions referenced by kernel metadata. Error markInstructions(); @@ -232,6 +253,9 @@ public: if (Error E = readBugTable()) return E; + if (Error E = readAltInstructions()) + return E; + return Error::success(); } @@ -1132,6 +1156,123 @@ Error LinuxKernelRewriter::readBugTable() { return Error::success(); } +/// The kernel can replace certain instruction sequences depending on hardware +/// it is running on and features specified during boot time. The information +/// about alternative instruction sequences is stored in .altinstructions +/// section. The format of entries in this section is defined in +/// arch/x86/include/asm/alternative.h: +/// +/// struct alt_instr { +/// s32 instr_offset; +/// s32 repl_offset; +/// uXX feature; +/// u8 instrlen; +/// u8 replacementlen; +/// u8 padlen; // present in older kernels +/// } __packed; +/// +/// Note the structures is packed. +Error LinuxKernelRewriter::readAltInstructions() { + AltInstrSection = BC.getUniqueSectionByName(".altinstructions"); + if (!AltInstrSection) + return Error::success(); + + const uint64_t Address = AltInstrSection->getAddress(); + DataExtractor DE = DataExtractor(AltInstrSection->getContents(), + BC.AsmInfo->isLittleEndian(), + BC.AsmInfo->getCodePointerSize()); + uint64_t EntryID = 0; + DataExtractor::Cursor Cursor(0); + while (Cursor && !DE.eof(Cursor)) { + const uint64_t OrgInstAddress = + Address + Cursor.tell() + (int32_t)DE.getU32(Cursor); + const uint64_t AltInstAddress = + Address + Cursor.tell() + (int32_t)DE.getU32(Cursor); + const uint64_t Feature = DE.getUnsigned(Cursor, opts::AltInstFeatureSize); + const uint8_t OrgSize = DE.getU8(Cursor); + const uint8_t AltSize = DE.getU8(Cursor); + + // Older kernels may have the padlen field. + const uint8_t PadLen = opts::AltInstHasPadLen ? DE.getU8(Cursor) : 0; + + if (!Cursor) + return createStringError(errc::executable_format_error, + "out of bounds while reading .altinstructions"); + + ++EntryID; + + if (opts::DumpAltInstructions) { + BC.outs() << "Alternative instruction entry: " << EntryID + << "\n\tOrg: 0x" << Twine::utohexstr(OrgInstAddress) + << "\n\tAlt: 0x" << Twine::utohexstr(AltInstAddress) + << "\n\tFeature: 0x" << Twine::utohexstr(Feature) + << "\n\tOrgSize: " << (int)OrgSize + << "\n\tAltSize: " << (int)AltSize << '\n'; + if (opts::AltInstHasPadLen) + BC.outs() << "\tPadLen: " << (int)PadLen << '\n'; + } + + if (AltSize > OrgSize) + return createStringError(errc::executable_format_error, + "error reading .altinstructions"); + + BinaryFunction *BF = BC.getBinaryFunctionContainingAddress(OrgInstAddress); + if (!BF && opts::Verbosity) { + BC.outs() << "BOLT-INFO: no function matches address 0x" + << Twine::utohexstr(OrgInstAddress) + << " of instruction from .altinstructions\n"; + } + + BinaryFunction *AltBF = + BC.getBinaryFunctionContainingAddress(AltInstAddress); + if (AltBF && BC.shouldEmit(*AltBF)) { + BC.errs() + << "BOLT-WARNING: alternative instruction sequence found in function " + << *AltBF << '\n'; + AltBF->setIgnored(); + } + + if (!BF || !BC.shouldEmit(*BF)) + continue; + + if (OrgInstAddress + OrgSize > BF->getAddress() + BF->getSize()) + return createStringError(errc::executable_format_error, + "error reading .altinstructions"); + + MCInst *Inst = + BF->getInstructionAtOffset(OrgInstAddress - BF->getAddress()); + if (!Inst) + return createStringError(errc::executable_format_error, + "no instruction at address 0x%" PRIx64 + " referenced by .altinstructions entry %d", + OrgInstAddress, EntryID); + + // There could be more than one alternative instruction sequences for the + // same original instruction. Annotate each alternative separately. + std::string AnnotationName = "AltInst"; + unsigned N = 2; + while (BC.MIB->hasAnnotation(*Inst, AnnotationName)) + AnnotationName = "AltInst" + std::to_string(N++); + + BC.MIB->addAnnotation(*Inst, AnnotationName, EntryID); + + // Annotate all instructions from the original sequence. Note that it's not + // the most efficient way to look for instructions in the address range, + // but since alternative instructions are uncommon, it will do for now. + for (uint32_t Offset = 1; Offset < OrgSize; ++Offset) { + Inst = BF->getInstructionAtOffset(OrgInstAddress + Offset - + BF->getAddress()); + if (Inst) + BC.MIB->addAnnotation(*Inst, AnnotationName, EntryID); + } + } + + BC.outs() << "BOLT-INFO: parsed " << EntryID + << " alternative instruction entries\n"; + + return Error::success(); +} + } // namespace std::unique_ptr diff --git a/bolt/test/X86/linux-alt-instruction.s b/bolt/test/X86/linux-alt-instruction.s new file mode 100644 index 000000000000..96e77545b654 --- /dev/null +++ b/bolt/test/X86/linux-alt-instruction.s @@ -0,0 +1,92 @@ +# REQUIRES: system-linux + +## Check that BOLT correctly parses the Linux kernel .altinstructions section +## and annotates alternative instructions. + +# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o +# RUN: %clang %cflags -nostdlib %t.o -o %t.exe \ +# RUN: -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr,--no-pie +# RUN: llvm-bolt %t.exe --print-normalized --keep-nops -o %t.out \ +# RUN: --alt-inst-feature-size=2 | FileCheck %s + +## Older kernels used to have padlen field in alt_instr. Check compatibility. + +# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown --defsym PADLEN=1 \ +# RUN: %s -o %t.o +# RUN: %clang %cflags -nostdlib %t.o -o %t.exe \ +# RUN: -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr,--no-pie +# RUN: llvm-bolt %t.exe --print-normalized --keep-nops --alt-inst-has-padlen \ +# RUN: -o %t.out | FileCheck %s + +## Check with a larger size of "feature" field in alt_instr. + +# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown \ +# RUN: --defsym FEATURE_SIZE_4=1 %s -o %t.o +# RUN: %clang %cflags -nostdlib %t.o -o %t.exe \ +# RUN: -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr,--no-pie +# RUN: llvm-bolt %t.exe --print-normalized --keep-nops \ +# RUN: --alt-inst-feature-size=4 -o %t.out | FileCheck %s + +# CHECK: BOLT-INFO: Linux kernel binary detected +# CHECK: BOLT-INFO: parsed 2 alternative instruction entries + + .text + .globl _start + .type _start, %function +_start: +# CHECK: Binary Function "_start" +.L0: + rdtsc +# CHECK: rdtsc +# CHECK-SAME: AltInst: 1 +# CHECK-SAME: AltInst2: 2 + nop +# CHECK-NEXT: nop +# CHECK-SAME: AltInst: 1 +# CHECK-SAME: AltInst2: 2 + nop + nop +.L1: + ret + .size _start, .-_start + + .section .altinstr_replacement,"ax",@progbits +.A0: + lfence + rdtsc +.A1: + rdtscp +.Ae: + +## Alternative instruction info. + .section .altinstructions,"a",@progbits + + .long .L0 - . # org instruction + .long .A0 - . # alt instruction +.ifdef FEATURE_SIZE_4 + .long 0x72 # feature flags +.else + .word 0x72 # feature flags +.endif + .byte .L1 - .L0 # org size + .byte .A1 - .A0 # alt size +.ifdef PADLEN + .byte 0 +.endif + + .long .L0 - . # org instruction + .long .A1 - . # alt instruction +.ifdef FEATURE_SIZE_4 + .long 0x3b # feature flags +.else + .word 0x3b # feature flags +.endif + .byte .L1 - .L0 # org size + .byte .Ae - .A1 # alt size +.ifdef PADLEN + .byte 0 +.endif + +## Fake Linux Kernel sections. + .section __ksymtab,"a",@progbits + .section __ksymtab_gpl,"a",@progbits -- GitLab From 9cf9cb271bf86bda4996be9a31fa413381f2f5e3 Mon Sep 17 00:00:00 2001 From: Cyndy Ishida Date: Thu, 7 Mar 2024 13:06:30 -0800 Subject: [PATCH 505/929] [clang] Upstream visionOS Availability & DarwinSDKInfo APIs (#84279) Admittedly a bit awkward, `visionos` is the correct and accepted spelling for annotating availability for xrOS target triples. This patch detects errors and handles cases when `xros` is mistakenly passed. In addition, add APIs for introduced/deprecated/obsoleted versioning in DarwinSDKInfo mappings. --- clang/include/clang/Basic/Attr.td | 8 ++++ clang/include/clang/Basic/DarwinSDKInfo.h | 24 ++++++++++++ clang/lib/Parse/ParseDecl.cpp | 5 ++- clang/lib/Parse/ParseExpr.cpp | 3 +- .../test/CodeGen/attr-availability-visionos.c | 10 +++++ clang/test/Sema/attr-availability-visionos.c | 39 +++++++++++++++++++ clang/unittests/Basic/DarwinSDKInfoTest.cpp | 10 +++++ 7 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 clang/test/CodeGen/attr-availability-visionos.c create mode 100644 clang/test/Sema/attr-availability-visionos.c diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td index fa191c7378db..ebb616fbe253 100644 --- a/clang/include/clang/Basic/Attr.td +++ b/clang/include/clang/Basic/Attr.td @@ -983,6 +983,8 @@ def Availability : InheritableAttr { .Case("watchos_app_extension", "watchOS (App Extension)") .Case("maccatalyst", "macCatalyst") .Case("maccatalyst_app_extension", "macCatalyst (App Extension)") + .Case("xros", "visionOS") + .Case("xros_app_extension", "visionOS (App Extension)") .Case("swift", "Swift") .Case("shadermodel", "HLSL ShaderModel") .Case("ohos", "OpenHarmony OS") @@ -1000,6 +1002,8 @@ static llvm::StringRef getPlatformNameSourceSpelling(llvm::StringRef Platform) { .Case("watchos_app_extension", "watchOSApplicationExtension") .Case("maccatalyst", "macCatalyst") .Case("maccatalyst_app_extension", "macCatalystApplicationExtension") + .Case("xros", "visionOS") + .Case("xros_app_extension", "visionOSApplicationExtension") .Case("zos", "z/OS") .Case("shadermodel", "ShaderModel") .Default(Platform); @@ -1016,6 +1020,10 @@ static llvm::StringRef canonicalizePlatformName(llvm::StringRef Platform) { .Case("watchOSApplicationExtension", "watchos_app_extension") .Case("macCatalyst", "maccatalyst") .Case("macCatalystApplicationExtension", "maccatalyst_app_extension") + .Case("visionOS", "xros") + .Case("visionOSApplicationExtension", "xros_app_extension") + .Case("visionos", "xros") + .Case("visionos_app_extension", "xros_app_extension") .Case("ShaderModel", "shadermodel") .Default(Platform); } }]; diff --git a/clang/include/clang/Basic/DarwinSDKInfo.h b/clang/include/clang/Basic/DarwinSDKInfo.h index dedfbd934a7b..db20b968a898 100644 --- a/clang/include/clang/Basic/DarwinSDKInfo.h +++ b/clang/include/clang/Basic/DarwinSDKInfo.h @@ -105,6 +105,30 @@ public: map(const VersionTuple &Key, const VersionTuple &MinimumValue, std::optional MaximumValue) const; + /// Remap the 'introduced' availability version. + /// If None is returned, the 'unavailable' availability should be used + /// instead. + std::optional + mapIntroducedAvailabilityVersion(const VersionTuple &Key) const { + // API_TO_BE_DEPRECATED is 100000. + if (Key.getMajor() == 100000) + return VersionTuple(100000); + // Use None for maximum to force unavailable behavior for + return map(Key, MinimumValue, std::nullopt); + } + + /// Remap the 'deprecated' and 'obsoleted' availability version. + /// If None is returned for 'obsoleted', the 'unavailable' availability + /// should be used instead. If None is returned for 'deprecated', the + /// 'deprecated' version should be dropped. + std::optional + mapDeprecatedObsoletedAvailabilityVersion(const VersionTuple &Key) const { + // API_TO_BE_DEPRECATED is 100000. + if (Key.getMajor() == 100000) + return VersionTuple(100000); + return map(Key, MinimumValue, MaximumValue); + } + static std::optional parseJSON(const llvm::json::Object &Obj, VersionTuple MaximumDeploymentTarget); diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp index 64b234eb460d..dd179414a141 100644 --- a/clang/lib/Parse/ParseDecl.cpp +++ b/clang/lib/Parse/ParseDecl.cpp @@ -1234,8 +1234,11 @@ void Parser::ParseAvailabilityAttribute( } IdentifierLoc *Platform = ParseIdentifierLoc(); if (const IdentifierInfo *const Ident = Platform->Ident) { + // Disallow xrOS for availability attributes. + if (Ident->getName().contains("xrOS") || Ident->getName().contains("xros")) + Diag(Platform->Loc, diag::warn_availability_unknown_platform) << Ident; // Canonicalize platform name from "macosx" to "macos". - if (Ident->getName() == "macosx") + else if (Ident->getName() == "macosx") Platform->Ident = PP.getIdentifierInfo("macos"); // Canonicalize platform name from "macosx_app_extension" to // "macos_app_extension". diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp index 4bf954b5cc4d..1f07eddb0fb3 100644 --- a/clang/lib/Parse/ParseExpr.cpp +++ b/clang/lib/Parse/ParseExpr.cpp @@ -3863,7 +3863,8 @@ std::optional Parser::ParseAvailabilitySpec() { StringRef Platform = AvailabilityAttr::canonicalizePlatformName(GivenPlatform); - if (AvailabilityAttr::getPrettyPlatformName(Platform).empty()) { + if (AvailabilityAttr::getPrettyPlatformName(Platform).empty() || + (GivenPlatform.contains("xros") || GivenPlatform.contains("xrOS"))) { Diag(PlatformIdentifier->Loc, diag::err_avail_query_unrecognized_platform_name) << GivenPlatform; diff --git a/clang/test/CodeGen/attr-availability-visionos.c b/clang/test/CodeGen/attr-availability-visionos.c new file mode 100644 index 000000000000..09b98fb4a7d5 --- /dev/null +++ b/clang/test/CodeGen/attr-availability-visionos.c @@ -0,0 +1,10 @@ +// RUN: %clang_cc1 -triple arm64-apple-xros1 -emit-llvm -o - %s 2>&1 | FileCheck %s + +__attribute__((availability(visionOS, introduced=1.1))) +void introduced_1_1(); + +void use() { + if (__builtin_available(visionOS 1.2, *)) + introduced_1_1(); + // CHECK: call i32 @__isPlatformVersionAtLeast(i32 11, i32 1, i32 2, i32 0) +} diff --git a/clang/test/Sema/attr-availability-visionos.c b/clang/test/Sema/attr-availability-visionos.c new file mode 100644 index 000000000000..2c388c5d5290 --- /dev/null +++ b/clang/test/Sema/attr-availability-visionos.c @@ -0,0 +1,39 @@ +// RUN: %clang_cc1 -triple arm64-apple-xros1 -fapplication-extension -verify=visionos %s 2>&1 + +__attribute__((availability(xros, unavailable))) // visionos-warning {{unknown platform 'xros' in availability macro}} +void xros_unavail(); // visionos-note {{}} + +__attribute__((availability(xros_app_extension, unavailable))) // visionos-warning {{unknown platform 'xros_app_extension' in availability macro}} +void xros_ext_unavail(); // visionos-note {{}} + +__attribute__((availability(visionOSApplicationExtension, unavailable))) +void visionos_ext_unavail(); // visionos-note {{}} + +void use() { + xros_unavail(); // visionos-error {{'xros_unavail' is unavailable: not available on visionOS}} + xros_ext_unavail(); // visionos-error {{'xros_ext_unavail' is unavailable: not available on visionOS}} + visionos_ext_unavail(); // visionos-error {{'visionos_ext_unavail' is unavailable: not available on visionOS}} +} + +__attribute__((availability(visionOS, introduced=1.0))) +void visionos_introduced_1(); + +__attribute__((availability(visionos, introduced=1.1))) +void visionos_introduced_1_1(); // visionos-note 4 {{'visionos_introduced_1_1' has been marked as being introduced in visionOS 1.1 here, but the deployment target is visionOS 1}} + +void use2() { + if (__builtin_available(iOS 16.1, *)) + visionos_introduced_1_1(); // visionos-warning {{'visionos_introduced_1_1' is only available on visionOS 1.1 or newer}} visionos-note {{enclose}} + + if (__builtin_available(xrOS 1.1, *)) // visionos-error {{unrecognized platform name xrOS}} + visionos_introduced_1_1(); // visionos-warning {{'visionos_introduced_1_1' is only available on visionOS 1.1 or newer}} visionos-note {{enclose}} + + if (__builtin_available(xros_app_extension 1, *)) // visionos-error {{unrecognized platform name xros_app_extension}} + visionos_introduced_1_1(); // visionos-warning {{'visionos_introduced_1_1' is only available on visionOS 1.1 or newer}} visionos-note {{enclose}} + + if (__builtin_available(visionOS 1.1, *)) + visionos_introduced_1_1(); + + visionos_introduced_1(); + visionos_introduced_1_1(); // visionos-warning {{'visionos_introduced_1_1' is only available on visionOS 1.1 or newer}} visionos-note {{enclose}} +} diff --git a/clang/unittests/Basic/DarwinSDKInfoTest.cpp b/clang/unittests/Basic/DarwinSDKInfoTest.cpp index 5f24e6eae515..7214f3bc8e19 100644 --- a/clang/unittests/Basic/DarwinSDKInfoTest.cpp +++ b/clang/unittests/Basic/DarwinSDKInfoTest.cpp @@ -168,6 +168,16 @@ TEST(DarwinSDKInfoTest, ParseAndTestMappingIOSDerived) { EXPECT_EQ( *Mapping->map(VersionTuple(13, 0), VersionTuple(), VersionTuple(99, 99)), VersionTuple(99, 99)); + + // Verify introduced, deprecated, and obsoleted mappings. + EXPECT_EQ(Mapping->mapIntroducedAvailabilityVersion(VersionTuple(10, 1)), + VersionTuple(10.0)); + EXPECT_EQ(Mapping->mapDeprecatedObsoletedAvailabilityVersion( + VersionTuple(100000, 0)), + VersionTuple(100000)); + EXPECT_EQ( + Mapping->mapDeprecatedObsoletedAvailabilityVersion(VersionTuple(13.0)), + VersionTuple(15, 0, 99)); } TEST(DarwinSDKInfoTest, MissingKeys) { -- GitLab From ced1fac8a32e35b63733bda27c7f5b9a2b635403 Mon Sep 17 00:00:00 2001 From: Yinying Li Date: Thu, 7 Mar 2024 16:13:45 -0500 Subject: [PATCH 506/929] [mlir][sparse] Move n:m printing into toMLIRString (#84264) --- .../include/mlir/Dialect/SparseTensor/IR/Enums.h | 4 ++++ .../SparseTensor/IR/SparseTensorDialect.cpp | 16 ++-------------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/Enums.h b/mlir/include/mlir/Dialect/SparseTensor/IR/Enums.h index 5563cb907e93..33f613a46bad 100644 --- a/mlir/include/mlir/Dialect/SparseTensor/IR/Enums.h +++ b/mlir/include/mlir/Dialect/SparseTensor/IR/Enums.h @@ -360,6 +360,10 @@ public: std::string toMLIRString() const { std::string lvlStr = toFormatString(getLvlFmt()); std::string propStr = ""; + if (isa()) { + lvlStr += + "[" + std::to_string(getN()) + ", " + std::to_string(getM()) + "]"; + } if (isa()) propStr += toPropString(LevelPropNonDefault::Nonunique); diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp index 6ba8b46370b0..c19907a945d3 100644 --- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp +++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp @@ -646,28 +646,16 @@ void SparseTensorEncodingAttr::printDimensions( } } -std::string getNOutOfMString(LevelType lt) { - if (isNOutOfMLT(lt)) { - unsigned n = getN(lt); - unsigned m = getM(lt); - auto output = "[" + std::to_string(n) + ", " + std::to_string(m) + "]"; - return output; - } - return ""; -} - void SparseTensorEncodingAttr::printLevels(AffineMap &map, AsmPrinter &printer, ArrayRef lvlTypes) const { for (unsigned i = 0, n = map.getNumResults() - 1; i < n; i++) { map.getResult(i).print(printer.getStream()); - printer << " : " << toMLIRString(lvlTypes[i]) - << getNOutOfMString(lvlTypes[i]) << ", "; + printer << " : " << toMLIRString(lvlTypes[i]) << ", "; } if (map.getNumResults() >= 1) { auto lastIndex = map.getNumResults() - 1; map.getResult(lastIndex).print(printer.getStream()); - printer << " : " << toMLIRString(lvlTypes[lastIndex]) - << getNOutOfMString(lvlTypes[lastIndex]); + printer << " : " << toMLIRString(lvlTypes[lastIndex]); } } -- GitLab From 167b90d0401d0fe488195c7e3d6fc1edc8fc5d94 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 7 Mar 2024 21:40:23 +0000 Subject: [PATCH 507/929] [TBAA] Add test showing tbaa.struct being generated with relaxed-alias. Add test showing that tbaa.struct is generated when using TSan with relaxed-aliasing. --- ...tbaa-struct-relaxed-aliasing-with-tsan.cpp | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 clang/test/CodeGen/tbaa-struct-relaxed-aliasing-with-tsan.cpp diff --git a/clang/test/CodeGen/tbaa-struct-relaxed-aliasing-with-tsan.cpp b/clang/test/CodeGen/tbaa-struct-relaxed-aliasing-with-tsan.cpp new file mode 100644 index 000000000000..931ff2476cd1 --- /dev/null +++ b/clang/test/CodeGen/tbaa-struct-relaxed-aliasing-with-tsan.cpp @@ -0,0 +1,26 @@ +// RUN: %clang_cc1 -triple x86_64-apple-darwin -emit-llvm -o - -O1 -relaxed-aliasing -fsanitize=thread -disable-llvm-optzns %s | \ +// RUN: FileCheck %s +// RUN: %clang_cc1 -triple x86_64-apple-darwin -new-struct-path-tbaa \ +// RUN: -emit-llvm -o - -O1 -relaxed-aliasing -fsanitize=thread -disable-llvm-optzns %s | \ +// RUN: FileCheck %s +// +// Check that we do not create tbaa for instructions generated for copies. +// FIXME: !tbaa.struct is generated with null node as tag. + +// CHECK: !tbaa.struct +// CHECK-NOT: !tbaa + +struct A { + short s; + int i; + char c; + int j; +}; + +void copyStruct(A *a1, A *a2) { + *a1 = *a2; +} + +void copyInt(int *a, int *b) { + *a = *b; +} -- GitLab From a0c7714525b696d90d2021249f9105c24ca7adcc Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Thu, 7 Mar 2024 14:03:58 -0800 Subject: [PATCH 508/929] [RISCV] Split div vs rem scheduling information [nfc] (#84385) Allows a processor to define different latencies for the two operations. --- llvm/lib/Target/RISCV/RISCVInstrInfoM.td | 8 ++++---- llvm/lib/Target/RISCV/RISCVSchedRocket.td | 12 ++++++++++++ llvm/lib/Target/RISCV/RISCVSchedSiFive7.td | 13 +++++++++++++ llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td | 12 ++++++++++++ llvm/lib/Target/RISCV/RISCVSchedSyntacoreSCR1.td | 6 +++++- llvm/lib/Target/RISCV/RISCVSchedXiangShanNanHu.td | 6 +++++- llvm/lib/Target/RISCV/RISCVSchedule.td | 8 ++++++-- 7 files changed, 57 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoM.td b/llvm/lib/Target/RISCV/RISCVInstrInfoM.td index 6b43d4393f76..8ea1560e5b37 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoM.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoM.td @@ -41,9 +41,9 @@ def DIV : ALU_rr<0b0000001, 0b100, "div">, def DIVU : ALU_rr<0b0000001, 0b101, "divu">, Sched<[WriteIDiv, ReadIDiv, ReadIDiv]>; def REM : ALU_rr<0b0000001, 0b110, "rem">, - Sched<[WriteIDiv, ReadIDiv, ReadIDiv]>; + Sched<[WriteIRem, ReadIRem, ReadIRem]>; def REMU : ALU_rr<0b0000001, 0b111, "remu">, - Sched<[WriteIDiv, ReadIDiv, ReadIDiv]>; + Sched<[WriteIRem, ReadIRem, ReadIRem]>; } // Predicates = [HasStdExtM] let Predicates = [HasStdExtMOrZmmul, IsRV64], IsSignExtendingOpW = 1 in { @@ -57,9 +57,9 @@ def DIVW : ALUW_rr<0b0000001, 0b100, "divw">, def DIVUW : ALUW_rr<0b0000001, 0b101, "divuw">, Sched<[WriteIDiv32, ReadIDiv32, ReadIDiv32]>; def REMW : ALUW_rr<0b0000001, 0b110, "remw">, - Sched<[WriteIDiv32, ReadIDiv32, ReadIDiv32]>; + Sched<[WriteIRem32, ReadIRem32, ReadIRem32]>; def REMUW : ALUW_rr<0b0000001, 0b111, "remuw">, - Sched<[WriteIDiv32, ReadIDiv32, ReadIDiv32]>; + Sched<[WriteIRem32, ReadIRem32, ReadIRem32]>; } // Predicates = [HasStdExtM, IsRV64] //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/RISCV/RISCVSchedRocket.td b/llvm/lib/Target/RISCV/RISCVSchedRocket.td index 60fa1a848306..e74c7aab7474 100644 --- a/llvm/lib/Target/RISCV/RISCVSchedRocket.td +++ b/llvm/lib/Target/RISCV/RISCVSchedRocket.td @@ -77,6 +77,16 @@ def : WriteRes { let ReleaseAtCycles = [33]; } +// Integer remainder +def : WriteRes { + let Latency = 34; + let ReleaseAtCycles = [34]; +} +def : WriteRes { + let Latency = 33; + let ReleaseAtCycles = [33]; +} + // Memory def : WriteRes; def : WriteRes; @@ -189,6 +199,8 @@ def : ReadAdvance; def : ReadAdvance; def : ReadAdvance; def : ReadAdvance; +def : ReadAdvance; +def : ReadAdvance; def : ReadAdvance; def : ReadAdvance; def : ReadAdvance; diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td index 0430d603620b..b21a56bdcdd2 100644 --- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td +++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td @@ -189,6 +189,7 @@ class SiFive7AnyToGPRBypass WriteREV8, WriteORCB, WriteSFB, WriteIMul, WriteIMul32, WriteIDiv, WriteIDiv32, + WriteIRem, WriteIRem32, WriteLDB, WriteLDH, WriteLDW, WriteLDD]>; // SiFive7 machine model for scheduling and other instruction cost heuristics. @@ -273,6 +274,16 @@ def : WriteRes { let ReleaseAtCycles = [1, 33]; } +// Integer remainder +def : WriteRes { + let Latency = 66; + let ReleaseAtCycles = [1, 65]; +} +def : WriteRes { + let Latency = 34; + let ReleaseAtCycles = [1, 33]; +} + // Bitmanip let Latency = 3 in { // Rotates are in the late-B ALU. @@ -946,6 +957,8 @@ def : SiFive7AnyToGPRBypass; def : SiFive7AnyToGPRBypass; def : ReadAdvance; def : ReadAdvance; +def : ReadAdvance; +def : ReadAdvance; def : ReadAdvance; def : ReadAdvance; def : ReadAdvance; diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td index 01398dea14a3..d02d34a0fb9c 100644 --- a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td +++ b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td @@ -86,6 +86,16 @@ def : WriteRes { let ReleaseAtCycles = [1, 19]; } +// Integer remainder +def : WriteRes { + let Latency = 35; + let ReleaseAtCycles = [1, 34]; +} +def : WriteRes { + let Latency = 20; + let ReleaseAtCycles = [1, 19]; +} + let Latency = 1 in { // Bitmanip def : WriteRes; @@ -258,6 +268,8 @@ def : ReadAdvance; def : ReadAdvance; def : ReadAdvance; def : ReadAdvance; +def : ReadAdvance; +def : ReadAdvance; def : ReadAdvance; def : ReadAdvance; def : ReadAdvance; diff --git a/llvm/lib/Target/RISCV/RISCVSchedSyntacoreSCR1.td b/llvm/lib/Target/RISCV/RISCVSchedSyntacoreSCR1.td index f2c07810867b..9625d17e0b26 100644 --- a/llvm/lib/Target/RISCV/RISCVSchedSyntacoreSCR1.td +++ b/llvm/lib/Target/RISCV/RISCVSchedSyntacoreSCR1.td @@ -54,10 +54,12 @@ def : WriteRes; def : WriteRes; def : WriteRes; -// Integer division: latency 33, inverse throughput 33 +// Integer division/remainder: latency 33, inverse throughput 33 let Latency = 33, ReleaseAtCycles = [33] in { def : WriteRes; def : WriteRes; +def : WriteRes; +def : WriteRes; } // Load/store instructions on SCR1 have latency 2 and inverse throughput 2 @@ -147,6 +149,8 @@ def : ReadAdvance; def : ReadAdvance; def : ReadAdvance; def : ReadAdvance; +def : ReadAdvance; +def : ReadAdvance; def : ReadAdvance; def : ReadAdvance; def : ReadAdvance; diff --git a/llvm/lib/Target/RISCV/RISCVSchedXiangShanNanHu.td b/llvm/lib/Target/RISCV/RISCVSchedXiangShanNanHu.td index 667b5983cb40..ef491edf3671 100644 --- a/llvm/lib/Target/RISCV/RISCVSchedXiangShanNanHu.td +++ b/llvm/lib/Target/RISCV/RISCVSchedXiangShanNanHu.td @@ -64,11 +64,13 @@ def : WriteRes; def : WriteRes; } -// Integer division +// Integer division/remainder // SRT16 algorithm let Latency = 20, ReleaseAtCycles = [20] in { def : WriteRes; def : WriteRes; +def : WriteRes; +def : WriteRes; } // Zb* @@ -221,6 +223,8 @@ def : XS2LoadToALUBypass; def : XS2LoadToALUBypass; def : ReadAdvance; def : ReadAdvance; +def : ReadAdvance; +def : ReadAdvance; def : ReadAdvance; def : ReadAdvance; def : ReadAdvance; diff --git a/llvm/lib/Target/RISCV/RISCVSchedule.td b/llvm/lib/Target/RISCV/RISCVSchedule.td index 593921bfcc67..1d19624342d2 100644 --- a/llvm/lib/Target/RISCV/RISCVSchedule.td +++ b/llvm/lib/Target/RISCV/RISCVSchedule.td @@ -13,8 +13,10 @@ def WriteShiftImm : SchedWrite; // 32 or 64-bit shift by immediate operatio def WriteShiftImm32 : SchedWrite; // 32-bit shift by immediate operations on RV64Ix def WriteShiftReg : SchedWrite; // 32 or 64-bit shift by immediate operations def WriteShiftReg32 : SchedWrite; // 32-bit shift by immediate operations on RV64Ix -def WriteIDiv : SchedWrite; // 32-bit or 64-bit divide and remainder -def WriteIDiv32 : SchedWrite; // 32-bit divide and remainder on RV64I +def WriteIDiv : SchedWrite; // 32-bit or 64-bit divide +def WriteIDiv32 : SchedWrite; // 32-bit divide on RV64I +def WriteIRem : SchedWrite; // 32-bit or 64-bit remainder +def WriteIRem32 : SchedWrite; // 32-bit remainder on RV64I def WriteIMul : SchedWrite; // 32-bit or 64-bit multiply def WriteIMul32 : SchedWrite; // 32-bit multiply on RV64I def WriteJmp : SchedWrite; // Jump @@ -135,6 +137,8 @@ def ReadShiftReg : SchedRead; def ReadShiftReg32 : SchedRead; // 32-bit shift by register operations on RV64Ix def ReadIDiv : SchedRead; def ReadIDiv32 : SchedRead; +def ReadIRem : SchedRead; +def ReadIRem32 : SchedRead; def ReadIMul : SchedRead; def ReadIMul32 : SchedRead; def ReadAtomicBA : SchedRead; -- GitLab From f78129e2bbafdd04a71bc09fc44e0797dd08db05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Thu, 7 Mar 2024 23:05:46 +0100 Subject: [PATCH 509/929] [Orc] Add NotifyCreated callback for LLJITBuilder (#84175) This is useful to attach generators to JITDylibs or inject initial symbol definitions. --- llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h b/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h index 76d16e63df28..d5682fcaa28b 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h @@ -311,6 +311,8 @@ public: using PlatformSetupFunction = unique_function(LLJIT &J)>; + using NotifyCreatedFunction = std::function; + std::unique_ptr EPC; std::unique_ptr ES; std::optional JTMB; @@ -321,6 +323,7 @@ public: CompileFunctionCreator CreateCompileFunction; unique_function PrePlatformSetup; PlatformSetupFunction SetUpPlatform; + NotifyCreatedFunction NotifyCreated; unsigned NumCompileThreads = 0; /// Called prior to JIT class construcion to fix up defaults. @@ -441,6 +444,16 @@ public: return impl(); } + /// Set up a callback after successful construction of the JIT. + /// + /// This is useful to attach generators to JITDylibs or inject initial symbol + /// definitions. + SetterImpl & + setNotifyCreatedCallback(LLJITBuilderState::NotifyCreatedFunction Callback) { + impl().NotifyCreated = std::move(Callback); + return impl(); + } + /// Set the number of compile threads to use. /// /// If set to zero, compilation will be performed on the execution thread when @@ -474,6 +487,11 @@ public: std::unique_ptr J(new JITType(impl(), Err)); if (Err) return std::move(Err); + + if (impl().NotifyCreated) + if (Error Err = impl().NotifyCreated(*J)) + return Err; + return std::move(J); } -- GitLab From 2a4a852a67eab2f8d0533c23719b1bd08d6edea9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Wed, 6 Mar 2024 16:46:56 +0100 Subject: [PATCH 510/929] Reland [clang-repl] Expose setter for triple in IncrementalCompilerBuilder (#84174) With out-of-process execution the target triple can be different from the one on the host. We need an interface to configure it. Relanding this with cleanup-fixes in the unittest. --- clang/include/clang/Interpreter/Interpreter.h | 5 +- clang/lib/Interpreter/Interpreter.cpp | 12 +++-- clang/unittests/Interpreter/CMakeLists.txt | 1 + .../IncrementalCompilerBuilderTest.cpp | 47 +++++++++++++++++++ 4 files changed, 59 insertions(+), 6 deletions(-) create mode 100644 clang/unittests/Interpreter/IncrementalCompilerBuilderTest.cpp diff --git a/clang/include/clang/Interpreter/Interpreter.h b/clang/include/clang/Interpreter/Interpreter.h index 292fa566ae70..c8f932e95c47 100644 --- a/clang/include/clang/Interpreter/Interpreter.h +++ b/clang/include/clang/Interpreter/Interpreter.h @@ -48,6 +48,8 @@ public: UserArgs = Args; } + void SetTargetTriple(std::string TT) { TargetTriple = TT; } + // General C++ llvm::Expected> CreateCpp(); @@ -62,11 +64,12 @@ public: private: static llvm::Expected> - create(std::vector &ClangArgv); + create(std::string TT, std::vector &ClangArgv); llvm::Expected> createCuda(bool device); std::vector UserArgs; + std::optional TargetTriple; llvm::StringRef OffloadArch; llvm::StringRef CudaSDKPath; diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp index 9f97a3c6b0be..37696b289764 100644 --- a/clang/lib/Interpreter/Interpreter.cpp +++ b/clang/lib/Interpreter/Interpreter.cpp @@ -132,7 +132,8 @@ CreateCI(const llvm::opt::ArgStringList &Argv) { } // anonymous namespace llvm::Expected> -IncrementalCompilerBuilder::create(std::vector &ClangArgv) { +IncrementalCompilerBuilder::create(std::string TT, + std::vector &ClangArgv) { // If we don't know ClangArgv0 or the address of main() at this point, try // to guess it anyway (it's possible on some platforms). @@ -162,8 +163,7 @@ IncrementalCompilerBuilder::create(std::vector &ClangArgv) { TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer; DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer); - driver::Driver Driver(/*MainBinaryName=*/ClangArgv[0], - llvm::sys::getProcessTriple(), Diags); + driver::Driver Driver(/*MainBinaryName=*/ClangArgv[0], TT, Diags); Driver.setCheckInputsExist(false); // the input comes from mem buffers llvm::ArrayRef RF = llvm::ArrayRef(ClangArgv); std::unique_ptr Compilation(Driver.BuildCompilation(RF)); @@ -185,7 +185,8 @@ IncrementalCompilerBuilder::CreateCpp() { Argv.push_back("-xc++"); Argv.insert(Argv.end(), UserArgs.begin(), UserArgs.end()); - return IncrementalCompilerBuilder::create(Argv); + std::string TT = TargetTriple ? *TargetTriple : llvm::sys::getProcessTriple(); + return IncrementalCompilerBuilder::create(TT, Argv); } llvm::Expected> @@ -213,7 +214,8 @@ IncrementalCompilerBuilder::createCuda(bool device) { Argv.insert(Argv.end(), UserArgs.begin(), UserArgs.end()); - return IncrementalCompilerBuilder::create(Argv); + std::string TT = TargetTriple ? *TargetTriple : llvm::sys::getProcessTriple(); + return IncrementalCompilerBuilder::create(TT, Argv); } llvm::Expected> diff --git a/clang/unittests/Interpreter/CMakeLists.txt b/clang/unittests/Interpreter/CMakeLists.txt index 712641afb976..0ddedb283e07 100644 --- a/clang/unittests/Interpreter/CMakeLists.txt +++ b/clang/unittests/Interpreter/CMakeLists.txt @@ -7,6 +7,7 @@ set(LLVM_LINK_COMPONENTS ) add_clang_unittest(ClangReplInterpreterTests + IncrementalCompilerBuilderTest.cpp IncrementalProcessingTest.cpp InterpreterTest.cpp CodeCompletionTest.cpp diff --git a/clang/unittests/Interpreter/IncrementalCompilerBuilderTest.cpp b/clang/unittests/Interpreter/IncrementalCompilerBuilderTest.cpp new file mode 100644 index 000000000000..f729566f7efd --- /dev/null +++ b/clang/unittests/Interpreter/IncrementalCompilerBuilderTest.cpp @@ -0,0 +1,47 @@ +//=== unittests/Interpreter/IncrementalCompilerBuilderTest.cpp ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/Basic/TargetOptions.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Interpreter/Interpreter.h" +#include "clang/Lex/PreprocessorOptions.h" +#include "llvm/Support/Error.h" +#include "gtest/gtest.h" + +using namespace llvm; +using namespace clang; + +namespace { + +// Usually FrontendAction takes the raw pointers and wraps them back into +// unique_ptrs in InitializeFileRemapping() +static void cleanupRemappedFileBuffers(CompilerInstance &CI) { + for (const auto &RB : CI.getPreprocessorOpts().RemappedFileBuffers) { + delete RB.second; + } + CI.getPreprocessorOpts().clearRemappedFiles(); +} + +TEST(IncrementalCompilerBuilder, SetCompilerArgs) { + std::vector ClangArgv = {"-Xclang", "-ast-dump-all"}; + auto CB = clang::IncrementalCompilerBuilder(); + CB.SetCompilerArgs(ClangArgv); + auto CI = cantFail(CB.CreateCpp()); + EXPECT_TRUE(CI->getFrontendOpts().ASTDumpAll); + cleanupRemappedFileBuffers(*CI); +} + +TEST(IncrementalCompilerBuilder, SetTargetTriple) { + auto CB = clang::IncrementalCompilerBuilder(); + CB.SetTargetTriple("armv6-none-eabi"); + auto CI = cantFail(CB.CreateCpp()); + EXPECT_EQ(CI->getTargetOpts().Triple, "armv6-none-unknown-eabi"); + cleanupRemappedFileBuffers(*CI); +} + +} // end anonymous namespace -- GitLab From 23d2c388303982e4341f248120915328a6444b51 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Thu, 7 Mar 2024 22:18:39 +0000 Subject: [PATCH 511/929] [gn build] Port 2a4a852a67ea --- llvm/utils/gn/secondary/clang/unittests/Interpreter/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/clang/unittests/Interpreter/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/Interpreter/BUILD.gn index 441d57187cd2..a20066436a3b 100644 --- a/llvm/utils/gn/secondary/clang/unittests/Interpreter/BUILD.gn +++ b/llvm/utils/gn/secondary/clang/unittests/Interpreter/BUILD.gn @@ -12,6 +12,7 @@ unittest("ClangReplInterpreterTests") { ] sources = [ "CodeCompletionTest.cpp", + "IncrementalCompilerBuilderTest.cpp", "IncrementalProcessingTest.cpp", "InterpreterTest.cpp", ] -- GitLab From 49b1fc4f831a047bd6ffde9ba19612c329dc5166 Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Thu, 7 Mar 2024 14:37:10 -0800 Subject: [PATCH 512/929] [CVP] Freeze Y when expanding urem x, y with X < 2Y (#84390) We're going from a single use to two independent uses, we need these two to see consistent values for undef. As an example, consider x = 0x2 when y = 0b00u1. If the sub use picks 0b0001 and the cmp use picks 0b0011, that would be incorrect. --- .../Scalar/CorrelatedValuePropagation.cpp | 9 ++++++--- .../urem-expansion.ll | 20 +++++++++++-------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp index 7a2011888ab0..de3bfb57b538 100644 --- a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp +++ b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp @@ -805,9 +805,12 @@ static bool expandUDivOrURem(BinaryOperator *Instr, const ConstantRange &XCR, Value *FrozenX = X; if (!isGuaranteedNotToBeUndef(X)) FrozenX = B.CreateFreeze(X, X->getName() + ".frozen"); - auto *AdjX = B.CreateNUWSub(FrozenX, Y, Instr->getName() + ".urem"); - auto *Cmp = - B.CreateICmp(ICmpInst::ICMP_ULT, FrozenX, Y, Instr->getName() + ".cmp"); + Value *FrozenY = Y; + if (!isGuaranteedNotToBeUndef(Y)) + FrozenY = B.CreateFreeze(Y, Y->getName() + ".frozen"); + auto *AdjX = B.CreateNUWSub(FrozenX, FrozenY, Instr->getName() + ".urem"); + auto *Cmp = B.CreateICmp(ICmpInst::ICMP_ULT, FrozenX, FrozenY, + Instr->getName() + ".cmp"); ExpandedOp = B.CreateSelect(Cmp, FrozenX, AdjX); } else { auto *Cmp = diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll b/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll index 2af8c8f23bbd..8e276d010fdd 100644 --- a/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll +++ b/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll @@ -198,8 +198,9 @@ define i8 @variable.v4(i8 %x, i8 %y) { ; CHECK-NEXT: [[CMP_Y_UPPER:%.*]] = icmp ule i8 [[Y]], 4 ; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_Y_UPPER]]) ; CHECK-NEXT: [[X_FROZEN:%.*]] = freeze i8 [[X]] -; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X_FROZEN]], [[Y]] -; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X_FROZEN]], [[Y]] +; CHECK-NEXT: [[Y_FROZEN:%.*]] = freeze i8 [[Y]] +; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X_FROZEN]], [[Y_FROZEN]] +; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X_FROZEN]], [[Y_FROZEN]] ; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X_FROZEN]], i8 [[REM_UREM]] ; CHECK-NEXT: ret i8 [[REM]] ; @@ -217,8 +218,9 @@ define i8 @variable.v4.range(ptr %x.ptr, ptr %y.ptr) { ; CHECK-NEXT: [[X:%.*]] = load i8, ptr [[X_PTR:%.*]], align 1, !range [[RNG0]] ; CHECK-NEXT: [[Y:%.*]] = load i8, ptr [[Y_PTR:%.*]], align 1, !range [[RNG1:![0-9]+]] ; CHECK-NEXT: [[X_FROZEN:%.*]] = freeze i8 [[X]] -; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X_FROZEN]], [[Y]] -; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X_FROZEN]], [[Y]] +; CHECK-NEXT: [[Y_FROZEN:%.*]] = freeze i8 [[Y]] +; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X_FROZEN]], [[Y_FROZEN]] +; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X_FROZEN]], [[Y_FROZEN]] ; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X_FROZEN]], i8 [[REM_UREM]] ; CHECK-NEXT: ret i8 [[REM]] ; @@ -236,8 +238,9 @@ define i8 @variable.v5(i8 %x, i8 %y) { ; CHECK-NEXT: [[CMP_Y_UPPER:%.*]] = icmp ule i8 [[Y]], 4 ; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_Y_UPPER]]) ; CHECK-NEXT: [[X_FROZEN:%.*]] = freeze i8 [[X]] -; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X_FROZEN]], [[Y]] -; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X_FROZEN]], [[Y]] +; CHECK-NEXT: [[Y_FROZEN:%.*]] = freeze i8 [[Y]] +; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X_FROZEN]], [[Y_FROZEN]] +; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X_FROZEN]], [[Y_FROZEN]] ; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X_FROZEN]], i8 [[REM_UREM]] ; CHECK-NEXT: ret i8 [[REM]] ; @@ -259,8 +262,9 @@ define i8 @variable.v6(i8 %x, i8 %y) { ; CHECK-NEXT: [[CMP_Y_UPPER:%.*]] = icmp ule i8 [[Y]], 4 ; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_Y_UPPER]]) ; CHECK-NEXT: [[X_FROZEN:%.*]] = freeze i8 [[X]] -; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X_FROZEN]], [[Y]] -; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X_FROZEN]], [[Y]] +; CHECK-NEXT: [[Y_FROZEN:%.*]] = freeze i8 [[Y]] +; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X_FROZEN]], [[Y_FROZEN]] +; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X_FROZEN]], [[Y_FROZEN]] ; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X_FROZEN]], i8 [[REM_UREM]] ; CHECK-NEXT: ret i8 [[REM]] ; -- GitLab From 48673825f47cbac9cd7c61299ca8d01579314ae0 Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Thu, 7 Mar 2024 14:28:20 -0800 Subject: [PATCH 513/929] [ORC] Deallocate FinalizedAllocs on error paths in notifyEmitted. If notifyEmitted encounters a failure (either because some plugin returned one, or because the ResourceTracker was defunct) then we need to deallocate the FinalizedAlloc manually. No testcase yet: This requires a concurrent setup -- we'll need to build some infrastructure to coordinate links and deliberately injected failures in order to reliably test this. --- llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp index 6ac256dff9b4..131728fd7e7e 100644 --- a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -720,14 +720,22 @@ Error ObjectLinkingLayer::notifyEmitted(MaterializationResponsibility &MR, for (auto &P : Plugins) Err = joinErrors(std::move(Err), P->notifyEmitted(MR)); - if (Err) + if (Err) { + if (FA) + Err = joinErrors(std::move(Err), MemMgr.deallocate(std::move(FA))); return Err; + } if (!FA) return Error::success(); - return MR.withResourceKeyDo( + Err = MR.withResourceKeyDo( [&](ResourceKey K) { Allocs[K].push_back(std::move(FA)); }); + + if (Err) + Err = joinErrors(std::move(Err), MemMgr.deallocate(std::move(FA))); + + return Err; } Error ObjectLinkingLayer::handleRemoveResources(JITDylib &JD, ResourceKey K) { -- GitLab From 69b8bc71110aca64c74a14800e800f4b151d5d6f Mon Sep 17 00:00:00 2001 From: dyung Date: Thu, 7 Mar 2024 14:43:12 -0800 Subject: [PATCH 514/929] [Dexter] Extend XFAIL of Dexter tests to all MacOS architectures. (#83936) I am trying to bring up a MacOS buildbot targeting x86 and noticed that two Dexter tests were failing, cross-project-tests/debuginfo-tests/llgdb-tests/static-member.cpp and cross-project-tests/debuginfo-tests/llgdb-tests/static-member-2.cpp. Looking in the history for these tests, they were XFAILed for Apple Silicon in 9c46606 and are failing similar on x86 for me, so we should extend the XFAIL to all MacOS architectures. --- .../debuginfo-tests/llgdb-tests/static-member-2.cpp | 2 +- .../debuginfo-tests/llgdb-tests/static-member.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cross-project-tests/debuginfo-tests/llgdb-tests/static-member-2.cpp b/cross-project-tests/debuginfo-tests/llgdb-tests/static-member-2.cpp index 79ff74cb2d0a..3f11ae018fc8 100644 --- a/cross-project-tests/debuginfo-tests/llgdb-tests/static-member-2.cpp +++ b/cross-project-tests/debuginfo-tests/llgdb-tests/static-member-2.cpp @@ -2,7 +2,7 @@ // RUN: %clangxx %target_itanium_abi_host_triple %t -o %t.out // RUN: %test_debuginfo %s %t.out // XFAIL: gdb-clang-incompatibility -// XFAIL: system-darwin && target-aarch64 +// XFAIL: system-darwin // DEBUGGER: delete breakpoints // DEBUGGER: break static-member.cpp:33 diff --git a/cross-project-tests/debuginfo-tests/llgdb-tests/static-member.cpp b/cross-project-tests/debuginfo-tests/llgdb-tests/static-member.cpp index abfa8e3337f6..57316dfd6404 100644 --- a/cross-project-tests/debuginfo-tests/llgdb-tests/static-member.cpp +++ b/cross-project-tests/debuginfo-tests/llgdb-tests/static-member.cpp @@ -2,7 +2,7 @@ // RUN: %clangxx %target_itanium_abi_host_triple %t -o %t.out // RUN: %test_debuginfo %s %t.out // XFAIL: !system-darwin && gdb-clang-incompatibility -// XFAIL: system-darwin && target-aarch64 +// XFAIL: system-darwin // DEBUGGER: delete breakpoints // DEBUGGER: break static-member.cpp:33 // DEBUGGER: r -- GitLab From 3a56b5a27d711aaa141c354706638bd94f7460a3 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Thu, 7 Mar 2024 16:53:41 -0600 Subject: [PATCH 515/929] [CUDA] Include PTX in non-RDC mode using the new driver (#84367) Summary: The old driver embed PTX in rdc-mode and so does the `nvcc` compiler. The new drivers currently does not do this, so we should keep it consistent in this case. This simply requires adding the assembler output as an input to the offloading action that gets fed to fatbin. --- clang/docs/ReleaseNotes.rst | 3 +++ clang/lib/Driver/Driver.cpp | 8 ++++++++ clang/lib/Driver/ToolChains/Cuda.cpp | 22 ++++++++++++---------- clang/test/Driver/cuda-phases.cu | 25 +++++++++++++------------ 4 files changed, 36 insertions(+), 22 deletions(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 1b901a27fd19..42c4a7c4d4bd 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -391,6 +391,9 @@ RISC-V Support CUDA/HIP Language Changes ^^^^^^^^^^^^^^^^^^^^^^^^^ +- PTX is no longer included by default when compiling for CUDA. Using + ``--cuda-include-ptx=all`` will return the old behavior. + CUDA Support ^^^^^^^^^^^^ diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp index cecd34acbc92..96e6ad77f5e5 100644 --- a/clang/lib/Driver/Driver.cpp +++ b/clang/lib/Driver/Driver.cpp @@ -4625,7 +4625,15 @@ Action *Driver::BuildOffloadingActions(Compilation &C, DDeps.add(*A, *TCAndArch->first, TCAndArch->second.data(), Kind); OffloadAction::DeviceDependences DDep; DDep.add(*A, *TCAndArch->first, TCAndArch->second.data(), Kind); + + // Compiling CUDA in non-RDC mode uses the PTX output if available. + for (Action *Input : A->getInputs()) + if (Kind == Action::OFK_Cuda && A->getType() == types::TY_Object && + !Args.hasFlag(options::OPT_fgpu_rdc, options::OPT_fno_gpu_rdc, + false)) + DDep.add(*Input, *TCAndArch->first, TCAndArch->second.data(), Kind); OffloadActions.push_back(C.MakeAction(DDep, A->getType())); + ++TCAndArch; } } diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp index 177fd6310e7e..c6007d3cfab8 100644 --- a/clang/lib/Driver/ToolChains/Cuda.cpp +++ b/clang/lib/Driver/ToolChains/Cuda.cpp @@ -503,18 +503,20 @@ void NVPTX::Assembler::ConstructJob(Compilation &C, const JobAction &JA, Exec, CmdArgs, Inputs, Output)); } -static bool shouldIncludePTX(const ArgList &Args, const char *gpu_arch) { - bool includePTX = true; - for (Arg *A : Args) { - if (!(A->getOption().matches(options::OPT_cuda_include_ptx_EQ) || - A->getOption().matches(options::OPT_no_cuda_include_ptx_EQ))) - continue; +static bool shouldIncludePTX(const ArgList &Args, StringRef InputArch) { + // The new driver does not include PTX by default to avoid overhead. + bool includePTX = !Args.hasFlag(options::OPT_offload_new_driver, + options::OPT_no_offload_new_driver, false); + for (Arg *A : Args.filtered(options::OPT_cuda_include_ptx_EQ, + options::OPT_no_cuda_include_ptx_EQ)) { A->claim(); const StringRef ArchStr = A->getValue(); - if (ArchStr == "all" || ArchStr == gpu_arch) { - includePTX = A->getOption().matches(options::OPT_cuda_include_ptx_EQ); - continue; - } + if (A->getOption().matches(options::OPT_cuda_include_ptx_EQ) && + (ArchStr == "all" || ArchStr == InputArch)) + includePTX = true; + else if (A->getOption().matches(options::OPT_no_cuda_include_ptx_EQ) && + (ArchStr == "all" || ArchStr == InputArch)) + includePTX = false; } return includePTX; } diff --git a/clang/test/Driver/cuda-phases.cu b/clang/test/Driver/cuda-phases.cu index 9a231091de2b..a1c3c9b51b1e 100644 --- a/clang/test/Driver/cuda-phases.cu +++ b/clang/test/Driver/cuda-phases.cu @@ -244,31 +244,32 @@ // NEW-DRIVER-RDC-NEXT: 18: assembler, {17}, object, (host-cuda) // NEW-DRIVER-RDC-NEXT: 19: clang-linker-wrapper, {18}, image, (host-cuda) -// RUN: %clang -### -target powerpc64le-ibm-linux-gnu -ccc-print-phases --offload-new-driver -fgpu-rdc \ +// RUN: %clang -### -target powerpc64le-ibm-linux-gnu -ccc-print-phases --offload-new-driver \ // RUN: --offload-arch=sm_52 --offload-arch=sm_70 %s 2>&1 | FileCheck --check-prefix=NEW-DRIVER %s -// NEW-DRIVER: 0: input, "[[INPUT:.+]]", cuda -// NEW-DRIVER-NEXT: 1: preprocessor, {0}, cuda-cpp-output -// NEW-DRIVER-NEXT: 2: compiler, {1}, ir -// NEW-DRIVER-NEXT: 3: input, "[[INPUT]]", cuda, (device-cuda, sm_52) +// NEW-DRIVER: 0: input, "[[CUDA:.+]]", cuda, (host-cuda) +// NEW-DRIVER-NEXT: 1: preprocessor, {0}, cuda-cpp-output, (host-cuda) +// NEW-DRIVER-NEXT: 2: compiler, {1}, ir, (host-cuda) +// NEW-DRIVER-NEXT: 3: input, "[[CUDA]]", cuda, (device-cuda, sm_52) // NEW-DRIVER-NEXT: 4: preprocessor, {3}, cuda-cpp-output, (device-cuda, sm_52) // NEW-DRIVER-NEXT: 5: compiler, {4}, ir, (device-cuda, sm_52) // NEW-DRIVER-NEXT: 6: backend, {5}, assembler, (device-cuda, sm_52) // NEW-DRIVER-NEXT: 7: assembler, {6}, object, (device-cuda, sm_52) -// NEW-DRIVER-NEXT: 8: offload, "device-cuda (nvptx64-nvidia-cuda:sm_52)" {7}, object -// NEW-DRIVER-NEXT: 9: input, "[[INPUT]]", cuda, (device-cuda, sm_70) +// NEW-DRIVER-NEXT: 8: offload, "device-cuda (nvptx64-nvidia-cuda:sm_52)" {7}, "device-cuda (nvptx64-nvidia-cuda:sm_52)" {6}, object +// NEW-DRIVER-NEXT: 9: input, "[[CUDA]]", cuda, (device-cuda, sm_70) // NEW-DRIVER-NEXT: 10: preprocessor, {9}, cuda-cpp-output, (device-cuda, sm_70) // NEW-DRIVER-NEXT: 11: compiler, {10}, ir, (device-cuda, sm_70) // NEW-DRIVER-NEXT: 12: backend, {11}, assembler, (device-cuda, sm_70) // NEW-DRIVER-NEXT: 13: assembler, {12}, object, (device-cuda, sm_70) -// NEW-DRIVER-NEXT: 14: offload, "device-cuda (nvptx64-nvidia-cuda:sm_70)" {13}, object -// NEW-DRIVER-NEXT: 15: clang-offload-packager, {8, 14}, image -// NEW-DRIVER-NEXT: 16: offload, "host-cuda (powerpc64le-ibm-linux-gnu)" {2}, "device-cuda (powerpc64le-ibm-linux-gnu)" {15}, ir +// NEW-DRIVER-NEXT: 14: offload, "device-cuda (nvptx64-nvidia-cuda:sm_70)" {13}, "device-cuda (nvptx64-nvidia-cuda:sm_70)" {12}, object +// NEW-DRIVER-NEXT: 15: linker, {8, 14}, cuda-fatbin, (device-cuda) +// NEW-DRIVER-NEXT: 16: offload, "host-cuda (powerpc64le-ibm-linux-gnu)" {2}, "device-cuda (nvptx64-nvidia-cuda)" {15}, ir // NEW-DRIVER-NEXT: 17: backend, {16}, assembler, (host-cuda) // NEW-DRIVER-NEXT: 18: assembler, {17}, object, (host-cuda) // NEW-DRIVER-NEXT: 19: clang-linker-wrapper, {18}, image, (host-cuda) // RUN: %clang -### --target=powerpc64le-ibm-linux-gnu -ccc-print-phases --offload-new-driver \ // RUN: --offload-arch=sm_52 --offload-arch=sm_70 %s %S/Inputs/empty.cpp 2>&1 | FileCheck --check-prefix=NON-CUDA-INPUT %s + // NON-CUDA-INPUT: 0: input, "[[CUDA:.+]]", cuda, (host-cuda) // NON-CUDA-INPUT-NEXT: 1: preprocessor, {0}, cuda-cpp-output, (host-cuda) // NON-CUDA-INPUT-NEXT: 2: compiler, {1}, ir, (host-cuda) @@ -277,13 +278,13 @@ // NON-CUDA-INPUT-NEXT: 5: compiler, {4}, ir, (device-cuda, sm_52) // NON-CUDA-INPUT-NEXT: 6: backend, {5}, assembler, (device-cuda, sm_52) // NON-CUDA-INPUT-NEXT: 7: assembler, {6}, object, (device-cuda, sm_52) -// NON-CUDA-INPUT-NEXT: 8: offload, "device-cuda (nvptx64-nvidia-cuda:sm_52)" {7}, object +// NON-CUDA-INPUT-NEXT: 8: offload, "device-cuda (nvptx64-nvidia-cuda:sm_52)" {7}, "device-cuda (nvptx64-nvidia-cuda:sm_52)" {6}, object // NON-CUDA-INPUT-NEXT: 9: input, "[[CUDA]]", cuda, (device-cuda, sm_70) // NON-CUDA-INPUT-NEXT: 10: preprocessor, {9}, cuda-cpp-output, (device-cuda, sm_70) // NON-CUDA-INPUT-NEXT: 11: compiler, {10}, ir, (device-cuda, sm_70) // NON-CUDA-INPUT-NEXT: 12: backend, {11}, assembler, (device-cuda, sm_70) // NON-CUDA-INPUT-NEXT: 13: assembler, {12}, object, (device-cuda, sm_70) -// NON-CUDA-INPUT-NEXT: 14: offload, "device-cuda (nvptx64-nvidia-cuda:sm_70)" {13}, object +// NON-CUDA-INPUT-NEXT: 14: offload, "device-cuda (nvptx64-nvidia-cuda:sm_70)" {13}, "device-cuda (nvptx64-nvidia-cuda:sm_70)" {12}, object // NON-CUDA-INPUT-NEXT: 15: linker, {8, 14}, cuda-fatbin, (device-cuda) // NON-CUDA-INPUT-NEXT: 16: offload, "host-cuda (powerpc64le-ibm-linux-gnu)" {2}, "device-cuda (nvptx64-nvidia-cuda)" {15}, ir // NON-CUDA-INPUT-NEXT: 17: backend, {16}, assembler, (host-cuda) -- GitLab From 14171b87a3b5a403f39d78da964595175636a0ae Mon Sep 17 00:00:00 2001 From: lntue <35648136+lntue@users.noreply.github.com> Date: Thu, 7 Mar 2024 17:58:28 -0500 Subject: [PATCH 516/929] [libc][stdfix] Add exp function for short _Accum and _Accum types. (#84391) --- libc/config/baremetal/arm/entrypoints.txt | 2 + libc/config/baremetal/riscv/entrypoints.txt | 2 + libc/config/linux/x86_64/entrypoints.txt | 2 + libc/docs/math/stdfix.rst | 2 +- libc/spec/llvm_libc_stdfix_ext.td | 3 + libc/src/__support/fixed_point/fx_rep.h | 24 ++--- libc/src/stdfix/CMakeLists.txt | 26 +++++ libc/src/stdfix/exphk.cpp | 92 +++++++++++++++++ libc/src/stdfix/exphk.h | 20 ++++ libc/src/stdfix/expk.cpp | 104 ++++++++++++++++++++ libc/src/stdfix/expk.h | 20 ++++ libc/test/src/stdfix/CMakeLists.txt | 36 +++++++ libc/test/src/stdfix/ExpTest.h | 77 +++++++++++++++ libc/test/src/stdfix/exphk_test.cpp | 13 +++ libc/test/src/stdfix/expk_test.cpp | 13 +++ 15 files changed, 423 insertions(+), 13 deletions(-) create mode 100644 libc/src/stdfix/exphk.cpp create mode 100644 libc/src/stdfix/exphk.h create mode 100644 libc/src/stdfix/expk.cpp create mode 100644 libc/src/stdfix/expk.h create mode 100644 libc/test/src/stdfix/ExpTest.h create mode 100644 libc/test/src/stdfix/exphk_test.cpp create mode 100644 libc/test/src/stdfix/expk_test.cpp diff --git a/libc/config/baremetal/arm/entrypoints.txt b/libc/config/baremetal/arm/entrypoints.txt index 99796ad5edf5..6e4fdb036264 100644 --- a/libc/config/baremetal/arm/entrypoints.txt +++ b/libc/config/baremetal/arm/entrypoints.txt @@ -288,6 +288,8 @@ if(LIBC_COMPILER_HAS_FIXED_POINT) libc.src.stdfix.absr libc.src.stdfix.abslk libc.src.stdfix.abslr + libc.src.stdfix.exphk + libc.src.stdfix.expk libc.src.stdfix.roundhk libc.src.stdfix.roundhr libc.src.stdfix.roundk diff --git a/libc/config/baremetal/riscv/entrypoints.txt b/libc/config/baremetal/riscv/entrypoints.txt index 99796ad5edf5..6e4fdb036264 100644 --- a/libc/config/baremetal/riscv/entrypoints.txt +++ b/libc/config/baremetal/riscv/entrypoints.txt @@ -288,6 +288,8 @@ if(LIBC_COMPILER_HAS_FIXED_POINT) libc.src.stdfix.absr libc.src.stdfix.abslk libc.src.stdfix.abslr + libc.src.stdfix.exphk + libc.src.stdfix.expk libc.src.stdfix.roundhk libc.src.stdfix.roundhr libc.src.stdfix.roundk diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index 1f36f127e3c4..0b77a9e170aa 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -483,6 +483,8 @@ if(LIBC_COMPILER_HAS_FIXED_POINT) libc.src.stdfix.absr libc.src.stdfix.abslk libc.src.stdfix.abslr + libc.src.stdfix.exphk + libc.src.stdfix.expk libc.src.stdfix.roundhk libc.src.stdfix.roundhr libc.src.stdfix.roundk diff --git a/libc/docs/math/stdfix.rst b/libc/docs/math/stdfix.rst index 5e39d5c01d1e..d8dcb0cfa4c5 100644 --- a/libc/docs/math/stdfix.rst +++ b/libc/docs/math/stdfix.rst @@ -110,7 +110,7 @@ floating point types, but are not part of the ISO/IEC TR 18037:2008 spec. +===============+================+=============+===============+============+================+=============+================+=============+===============+============+================+=============+ | cos | | | | | | | | | | | | | +---------------+----------------+-------------+---------------+------------+----------------+-------------+----------------+-------------+---------------+------------+----------------+-------------+ -| exp | | | | | | | | | | | | | +| exp | | | | | | | | |check| | | |check| | | | +---------------+----------------+-------------+---------------+------------+----------------+-------------+----------------+-------------+---------------+------------+----------------+-------------+ | log | | | | | | | | | | | | | +---------------+----------------+-------------+---------------+------------+----------------+-------------+----------------+-------------+---------------+------------+----------------+-------------+ diff --git a/libc/spec/llvm_libc_stdfix_ext.td b/libc/spec/llvm_libc_stdfix_ext.td index 75bde47810a6..7bc7ec546408 100644 --- a/libc/spec/llvm_libc_stdfix_ext.td +++ b/libc/spec/llvm_libc_stdfix_ext.td @@ -5,6 +5,9 @@ def LLVMLibcStdfixExt : StandardSpec<"llvm_libc_stdfix_ext"> { [], // types [], // enums [ // functions + GuardedFunctionSpec<"exphk", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, + GuardedFunctionSpec<"expk", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, + GuardedFunctionSpec<"sqrtuhr", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, GuardedFunctionSpec<"sqrtur", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, GuardedFunctionSpec<"sqrtulr", RetValSpec, [ArgSpec], "LIBC_COMPILER_HAS_FIXED_POINT">, diff --git a/libc/src/__support/fixed_point/fx_rep.h b/libc/src/__support/fixed_point/fx_rep.h index 042cd2b20714..f13640a6c019 100644 --- a/libc/src/__support/fixed_point/fx_rep.h +++ b/libc/src/__support/fixed_point/fx_rep.h @@ -45,7 +45,7 @@ template <> struct FXRep { SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; LIBC_INLINE static constexpr Type MIN() { return SFRACT_MIN; } - LIBC_INLINE static constexpr Type MAX() { return SFRACT_MIN; } + LIBC_INLINE static constexpr Type MAX() { return SFRACT_MAX; } LIBC_INLINE static constexpr Type ZERO() { return 0.0HR; } LIBC_INLINE static constexpr Type EPS() { return SFRACT_EPSILON; } LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5HR; } @@ -65,7 +65,7 @@ template <> struct FXRep { SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; LIBC_INLINE static constexpr Type MIN() { return USFRACT_MIN; } - LIBC_INLINE static constexpr Type MAX() { return USFRACT_MIN; } + LIBC_INLINE static constexpr Type MAX() { return USFRACT_MAX; } LIBC_INLINE static constexpr Type ZERO() { return 0.0UHR; } LIBC_INLINE static constexpr Type EPS() { return USFRACT_EPSILON; } LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5UHR; } @@ -85,7 +85,7 @@ template <> struct FXRep { SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; LIBC_INLINE static constexpr Type MIN() { return FRACT_MIN; } - LIBC_INLINE static constexpr Type MAX() { return FRACT_MIN; } + LIBC_INLINE static constexpr Type MAX() { return FRACT_MAX; } LIBC_INLINE static constexpr Type ZERO() { return 0.0R; } LIBC_INLINE static constexpr Type EPS() { return FRACT_EPSILON; } LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5R; } @@ -105,7 +105,7 @@ template <> struct FXRep { SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; LIBC_INLINE static constexpr Type MIN() { return UFRACT_MIN; } - LIBC_INLINE static constexpr Type MAX() { return UFRACT_MIN; } + LIBC_INLINE static constexpr Type MAX() { return UFRACT_MAX; } LIBC_INLINE static constexpr Type ZERO() { return 0.0UR; } LIBC_INLINE static constexpr Type EPS() { return UFRACT_EPSILON; } LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5UR; } @@ -125,7 +125,7 @@ template <> struct FXRep { SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; LIBC_INLINE static constexpr Type MIN() { return LFRACT_MIN; } - LIBC_INLINE static constexpr Type MAX() { return LFRACT_MIN; } + LIBC_INLINE static constexpr Type MAX() { return LFRACT_MAX; } LIBC_INLINE static constexpr Type ZERO() { return 0.0LR; } LIBC_INLINE static constexpr Type EPS() { return LFRACT_EPSILON; } LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5LR; } @@ -145,7 +145,7 @@ template <> struct FXRep { SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; LIBC_INLINE static constexpr Type MIN() { return ULFRACT_MIN; } - LIBC_INLINE static constexpr Type MAX() { return ULFRACT_MIN; } + LIBC_INLINE static constexpr Type MAX() { return ULFRACT_MAX; } LIBC_INLINE static constexpr Type ZERO() { return 0.0ULR; } LIBC_INLINE static constexpr Type EPS() { return ULFRACT_EPSILON; } LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5ULR; } @@ -165,7 +165,7 @@ template <> struct FXRep { SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; LIBC_INLINE static constexpr Type MIN() { return SACCUM_MIN; } - LIBC_INLINE static constexpr Type MAX() { return SACCUM_MIN; } + LIBC_INLINE static constexpr Type MAX() { return SACCUM_MAX; } LIBC_INLINE static constexpr Type ZERO() { return 0.0HK; } LIBC_INLINE static constexpr Type EPS() { return SACCUM_EPSILON; } LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5HK; } @@ -185,7 +185,7 @@ template <> struct FXRep { SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; LIBC_INLINE static constexpr Type MIN() { return USACCUM_MIN; } - LIBC_INLINE static constexpr Type MAX() { return USACCUM_MIN; } + LIBC_INLINE static constexpr Type MAX() { return USACCUM_MAX; } LIBC_INLINE static constexpr Type ZERO() { return 0.0UHK; } LIBC_INLINE static constexpr Type EPS() { return USACCUM_EPSILON; } LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5UHK; } @@ -205,7 +205,7 @@ template <> struct FXRep { SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; LIBC_INLINE static constexpr Type MIN() { return ACCUM_MIN; } - LIBC_INLINE static constexpr Type MAX() { return ACCUM_MIN; } + LIBC_INLINE static constexpr Type MAX() { return ACCUM_MAX; } LIBC_INLINE static constexpr Type ZERO() { return 0.0K; } LIBC_INLINE static constexpr Type EPS() { return ACCUM_EPSILON; } LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5K; } @@ -225,7 +225,7 @@ template <> struct FXRep { SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; LIBC_INLINE static constexpr Type MIN() { return UACCUM_MIN; } - LIBC_INLINE static constexpr Type MAX() { return UACCUM_MIN; } + LIBC_INLINE static constexpr Type MAX() { return UACCUM_MAX; } LIBC_INLINE static constexpr Type ZERO() { return 0.0UK; } LIBC_INLINE static constexpr Type EPS() { return UACCUM_EPSILON; } LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5UK; } @@ -245,7 +245,7 @@ template <> struct FXRep { SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; LIBC_INLINE static constexpr Type MIN() { return LACCUM_MIN; } - LIBC_INLINE static constexpr Type MAX() { return LACCUM_MIN; } + LIBC_INLINE static constexpr Type MAX() { return LACCUM_MAX; } LIBC_INLINE static constexpr Type ZERO() { return 0.0LK; } LIBC_INLINE static constexpr Type EPS() { return LACCUM_EPSILON; } LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5LK; } @@ -265,7 +265,7 @@ template <> struct FXRep { SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; LIBC_INLINE static constexpr Type MIN() { return ULACCUM_MIN; } - LIBC_INLINE static constexpr Type MAX() { return ULACCUM_MIN; } + LIBC_INLINE static constexpr Type MAX() { return ULACCUM_MAX; } LIBC_INLINE static constexpr Type ZERO() { return 0.0ULK; } LIBC_INLINE static constexpr Type EPS() { return ULACCUM_EPSILON; } LIBC_INLINE static constexpr Type ONE_HALF() { return 0.5ULK; } diff --git a/libc/src/stdfix/CMakeLists.txt b/libc/src/stdfix/CMakeLists.txt index 3a1cb66b7abc..10d76ae31349 100644 --- a/libc/src/stdfix/CMakeLists.txt +++ b/libc/src/stdfix/CMakeLists.txt @@ -67,3 +67,29 @@ add_entrypoint_object( DEPENDS libc.src.__support.fixed_point.sqrt ) + +add_entrypoint_object( + exphk + HDRS + exphk.h + SRCS + exphk.cpp + COMPILE_OPTIONS + -O3 + DEPENDS + libc.src.__support.fixed_point.fx_rep + libc.src.__support.CPP.bit +) + +add_entrypoint_object( + expk + HDRS + expk.h + SRCS + expk.cpp + COMPILE_OPTIONS + -O3 + DEPENDS + libc.src.__support.fixed_point.fx_rep + libc.src.__support.CPP.bit +) diff --git a/libc/src/stdfix/exphk.cpp b/libc/src/stdfix/exphk.cpp new file mode 100644 index 000000000000..19a972b390c7 --- /dev/null +++ b/libc/src/stdfix/exphk.cpp @@ -0,0 +1,92 @@ +//===-- Implementation of exphk function ----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "exphk.h" +#include "src/__support/CPP/bit.h" +#include "src/__support/common.h" +#include "src/__support/fixed_point/fx_bits.h" + +namespace LIBC_NAMESPACE { + +namespace { + +// Look up tables for exp(hi) and exp(mid). +// Generated with Sollya: +// > for i from 0 to 89 do { +// hi = floor(i/8) - 5; +// m = i/8 - floor(i/8) - 0.5; +// e_hi = nearestint(exp(hi) * 2^7) * 2^-7; +// e_mid = nearestint(exp(m) * 2^7) * 2^-7; +// print(hi, e_hi, m, e_mid); +// }; +// Notice that when i = 88 and 89, e_hi will overflow short accum range. +static constexpr short accum EXP_HI[12] = { + 0x1.0p-7hk, 0x1.0p-6hk, 0x1.8p-5hk, 0x1.1p-3hk, 0x1.78p-2hk, 0x1.0p0hk, + 0x1.5cp1hk, 0x1.d9p2hk, 0x1.416p4hk, 0x1.b4dp5hk, 0x1.28d4p7hk, SACCUM_MAX, +}; + +static constexpr short accum EXP_MID[8] = { + 0x1.38p-1hk, 0x1.6p-1hk, 0x1.9p-1hk, 0x1.c4p-1hk, + 0x1.0p0hk, 0x1.22p0hk, 0x1.48p0hk, 0x1.74p0hk, +}; + +} // anonymous namespace + +LLVM_LIBC_FUNCTION(short accum, exphk, (short accum x)) { + using FXRep = fixed_point::FXRep; + using StorageType = typename FXRep::StorageType; + // Output overflow + if (LIBC_UNLIKELY(x >= 0x1.64p2hk)) + return FXRep::MAX(); + // Lower bound where exp(x) -> 0: + // floor(log(2^-8) * 2^7) * 2^-7 + if (LIBC_UNLIKELY(x <= -0x1.63p2hk)) + return FXRep::ZERO(); + + // Current range of x: + // -0x1.628p2 <= x <= 0x1.638p2 + // Range reduction: + // x = hi + mid + lo, + // where: + // hi is an integer + // mid * 2^3 is an integer + // |lo| <= 2^-4. + // Then exp(x) = exp(hi + mid + lo) = exp(hi) * exp(mid) * exp(lo) + // ~ exp(hi) * exp(mid) * (1 + lo) + // with relative errors < |lo|^2 <= 2^-8. + // exp(hi) and exp(mid) are extracted from small lookup tables. + + // Round-to-nearest 1/8, tie-to-(+Int): + constexpr short accum ONE_SIXTEENTH = 0x1.0p-4hk; + // x_rounded = floor(x + 1/16). + short accum x_rounded = ((x + ONE_SIXTEENTH) >> (FXRep::FRACTION_LEN - 3)) + << (FXRep::FRACTION_LEN - 3); + short accum lo = x - x_rounded; + + // Range of x_rounded: + // x_rounded >= floor((-0x1.628p2 + 0x1.0p-4) * 2^3) * 2^-3 + // = -0x1.6p2 = -5.5 + // To get the indices, we shift the values so that it start with 0. + // Range of indices: 0 <= indices <= 89 + StorageType indices = cpp::bit_cast((x_rounded + 0x1.6p2hk) >> + (FXRep::FRACTION_LEN - 3)); + // So we have the following relation: + // indices = (hi + mid + 44/8) * 8 + // That implies: + // hi + mid = indices/8 - 5.5 + // So for lookup tables, we can use the upper 4 bits to get: + // exp( floor(indices / 8) - 5 ) + // and lower 3 bits for: + // exp( (indices - floor(indices)) - 0.5 ) + short accum exp_hi = EXP_HI[indices >> 3]; + short accum exp_mid = EXP_MID[indices & 0x7]; + // exp(x) ~ exp(hi) * exp(mid) * (1 + lo); + return (exp_hi * (exp_mid * (0x1.0p0hk + lo))); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdfix/exphk.h b/libc/src/stdfix/exphk.h new file mode 100644 index 000000000000..da03bb76d53f --- /dev/null +++ b/libc/src/stdfix/exphk.h @@ -0,0 +1,20 @@ +//===-- Implementation header for exphk -------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDFIX_EXPHK_H +#define LLVM_LIBC_SRC_STDFIX_EXPHK_H + +#include "include/llvm-libc-macros/stdfix-macros.h" + +namespace LIBC_NAMESPACE { + +short accum exphk(short accum x); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDFIX_EXPHK_H diff --git a/libc/src/stdfix/expk.cpp b/libc/src/stdfix/expk.cpp new file mode 100644 index 000000000000..57227fd27769 --- /dev/null +++ b/libc/src/stdfix/expk.cpp @@ -0,0 +1,104 @@ +//===-- Implementation of expk function ----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "expk.h" +#include "src/__support/CPP/bit.h" +#include "src/__support/common.h" +#include "src/__support/fixed_point/fx_bits.h" + +namespace LIBC_NAMESPACE { + +namespace { + +// Look up tables for exp(hi) and exp(mid). +// Generated with Sollya: +// > for i from 0 to 23 do { +// hi = i - 11; +// e_hi = nearestint(exp(hi) * 2^15) * 2^-15; +// print(e_hi, "k,"); +// }; +static constexpr accum EXP_HI[24] = { + 0x1p-15k, 0x1p-15k, 0x1p-13k, 0x1.6p-12k, + 0x1.ep-11k, 0x1.44p-9k, 0x1.bap-8k, 0x1.2cp-6k, + 0x1.97cp-5k, 0x1.153p-3k, 0x1.78b8p-2k, 0x1p0k, + 0x1.5bf1p1k, 0x1.d8e68p2k, 0x1.415e6p4k, 0x1.b4c9p5k, + 0x1.28d388p7k, 0x1.936dc6p8k, 0x1.1228858p10k, 0x1.749ea7cp11k, + 0x1.fa7157cp12k, 0x1.5829dcf8p14k, 0x1.d3c4489p15k, ACCUM_MAX, +}; + +// Generated with Sollya: +// > for i from 0 to 15 do { +// m = i/16 - 0.0625; +// e_m = nearestint(exp(m) * 2^15) * 2^-15; +// print(e_m, "k,"); +// }; +static constexpr accum EXP_MID[16] = { + 0x1.e0fcp-1k, 0x1p0k, 0x1.1082p0k, 0x1.2216p0k, + 0x1.34ccp0k, 0x1.48b6p0k, 0x1.5deap0k, 0x1.747ap0k, + 0x1.8c8p0k, 0x1.a612p0k, 0x1.c14cp0k, 0x1.de46p0k, + 0x1.fd1ep0k, 0x1.0efap1k, 0x1.2074p1k, 0x1.330ep1k, +}; + +} // anonymous namespace + +LLVM_LIBC_FUNCTION(accum, expk, (accum x)) { + using FXRep = fixed_point::FXRep; + using StorageType = typename FXRep::StorageType; + // Output overflow + // > floor(log(2^16) * 2^15) * 2^-15 + if (LIBC_UNLIKELY(x >= 0x1.62e4p3k)) + return FXRep::MAX(); + // Lower bound where exp(x) -> 0: + // floor(log(2^-16) * 2^15) * 2^-15 + if (LIBC_UNLIKELY(x <= -0x1.62e44p3k)) + return FXRep::ZERO(); + + // Current range of x: + // -0x1.62e4p3 <= x <= 0x1.62e3cp3 + // Range reduction: + // x = hi + mid + lo, + // where: + // hi is an integer + // mid * 2^4 is an integer + // |lo| <= 2^-5. + // Then exp(x) = exp(hi + mid + lo) = exp(hi) * exp(mid) * exp(lo) + // ~ exp(hi) * exp(mid) * (1 + lo + lo^2 / 2) + // with relative errors < |lo|^3/2 <= 2^-16. + // exp(hi) and exp(mid) are extracted from small lookup tables. + + // Round-to-nearest 1/16, tie-to-(+Int): + constexpr accum ONE_THIRTY_SECOND = 0x1.0p-5k; + // x_rounded = floor(x + 1/16). + accum x_rounded = ((x + ONE_THIRTY_SECOND) >> (FXRep::FRACTION_LEN - 4)) + << (FXRep::FRACTION_LEN - 4); + accum lo = x - x_rounded; + + // Range of x_rounded: + // x_rounded >= floor((-0x1.62e4p3 + 0x1.0p-5) * 2^4) * 2^-4 + // = -0x1.62p3 = -11.0625 + // To get the indices, we shift the values so that it start with 0. + // Range of indices: 0 <= indices <= 355. + StorageType indices = cpp::bit_cast((x_rounded + 0x1.62p3k) >> + (FXRep::FRACTION_LEN - 4)); + // So we have the following relation: + // indices = (hi + mid + 177/16) * 16 + // That implies: + // hi + mid = indices/16 - 11.0625 + // So for lookup tables, we can use the upper 4 bits to get: + // exp( floor(indices / 16) - 11 ) + // and lower 4 bits for: + // exp( (indices - floor(indices)) - 0.0625 ) + accum exp_hi = EXP_HI[indices >> 4]; + accum exp_mid = EXP_MID[indices & 0xf]; + // exp(x) ~ exp(hi) * exp(mid) * (1 + lo); + accum l1 = 0x1.0p0k + (lo >> 1); // = 1 + lo / 2 + accum l2 = 0x1.0p0k + lo * l1; // = 1 + lo * (1 + lo / 2) = 1 + lo + lo^2/2 + return (exp_hi * (exp_mid * l2)); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdfix/expk.h b/libc/src/stdfix/expk.h new file mode 100644 index 000000000000..4526686a200b --- /dev/null +++ b/libc/src/stdfix/expk.h @@ -0,0 +1,20 @@ +//===-- Implementation header for expk --------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDFIX_EXPK_H +#define LLVM_LIBC_SRC_STDFIX_EXPK_H + +#include "include/llvm-libc-macros/stdfix-macros.h" + +namespace LIBC_NAMESPACE { + +accum expk(accum x); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDFIX_EXPK_H diff --git a/libc/test/src/stdfix/CMakeLists.txt b/libc/test/src/stdfix/CMakeLists.txt index d3e122884eb4..74a1fb13127c 100644 --- a/libc/test/src/stdfix/CMakeLists.txt +++ b/libc/test/src/stdfix/CMakeLists.txt @@ -96,3 +96,39 @@ add_libc_test( libc.src.__support.FPUtil.basic_operations libc.src.__support.FPUtil.sqrt ) + +add_libc_test( + exphk_test + SUITE + libc-stdfix-tests + HDRS + ExpTest.h + SRCS + exphk_test.cpp + COMPILE_OPTIONS + -O3 + DEPENDS + libc.src.stdfix.exphk + libc.src.math.exp + libc.src.__support.CPP.bit + libc.src.__support.fixed_point.fx_rep + libc.src.__support.FPUtil.basic_operations +) + +add_libc_test( + expk_test + SUITE + libc-stdfix-tests + HDRS + ExpTest.h + SRCS + expk_test.cpp + COMPILE_OPTIONS + -O3 + DEPENDS + libc.src.stdfix.expk + libc.src.math.exp + libc.src.__support.CPP.bit + libc.src.__support.fixed_point.fx_rep + libc.src.__support.FPUtil.basic_operations +) diff --git a/libc/test/src/stdfix/ExpTest.h b/libc/test/src/stdfix/ExpTest.h new file mode 100644 index 000000000000..e588cebf621b --- /dev/null +++ b/libc/test/src/stdfix/ExpTest.h @@ -0,0 +1,77 @@ +//===-- Utility class to test integer sqrt ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "test/UnitTest/FPMatcher.h" +#include "test/UnitTest/Test.h" + +#include "src/__support/CPP/bit.h" +#include "src/__support/FPUtil/BasicOperations.h" +#include "src/__support/fixed_point/fx_rep.h" +#include "src/__support/fixed_point/sqrt.h" + +#include "src/math/exp.h" + +template class ExpTest : public LIBC_NAMESPACE::testing::Test { + + using FXRep = LIBC_NAMESPACE::fixed_point::FXRep; + static constexpr T zero = FXRep::ZERO(); + static constexpr T one = static_cast(1); + static constexpr T eps = FXRep::EPS(); + +public: + typedef T (*ExpFunc)(T); + + void test_special_numbers(ExpFunc func) { + EXPECT_EQ(one, func(T(0))); + EXPECT_EQ(FXRep::MAX(), func(T(30))); + EXPECT_EQ(zero, func(T(-30))); + } + + void test_range_with_step(ExpFunc func, T step, bool rel_error) { + constexpr int COUNT = 255; + constexpr double ERR = 3.0 * static_cast(eps); + double x_d = 0.0; + T x = step; + for (int i = 0; i < COUNT; ++i) { + x += step; + x_d = static_cast(x); + double y_d = static_cast(func(x)); + double result = LIBC_NAMESPACE::exp(x_d); + double errors = rel_error + ? LIBC_NAMESPACE::fputil::abs((y_d / result) - 1.0) + : LIBC_NAMESPACE::fputil::abs(y_d - result); + if (errors > ERR) { + // Print out the failure input and output. + EXPECT_EQ(x, T(0)); + EXPECT_EQ(func(x), zero); + } + ASSERT_TRUE(errors <= ERR); + } + } + + void test_positive_range(ExpFunc func) { + test_range_with_step(func, T(0x1.0p-6), /*rel_error*/ true); + } + + void test_negative_range(ExpFunc func) { + test_range_with_step(func, T(-0x1.0p-6), /*rel_error*/ false); + } +}; + +#define LIST_EXP_TESTS(Name, T, func) \ + using LlvmLibcExp##Name##Test = ExpTest; \ + TEST_F(LlvmLibcExp##Name##Test, SpecialNumbers) { \ + test_special_numbers(&func); \ + } \ + TEST_F(LlvmLibcExp##Name##Test, PositiveRange) { \ + test_positive_range(&func); \ + } \ + TEST_F(LlvmLibcExp##Name##Test, NegativeRange) { \ + test_negative_range(&func); \ + } \ + static_assert(true, "Require semicolon.") diff --git a/libc/test/src/stdfix/exphk_test.cpp b/libc/test/src/stdfix/exphk_test.cpp new file mode 100644 index 000000000000..24e92dc902fa --- /dev/null +++ b/libc/test/src/stdfix/exphk_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for exphk -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "ExpTest.h" + +#include "src/stdfix/exphk.h" + +LIST_EXP_TESTS(hk, short accum, LIBC_NAMESPACE::exphk); diff --git a/libc/test/src/stdfix/expk_test.cpp b/libc/test/src/stdfix/expk_test.cpp new file mode 100644 index 000000000000..bc322037af04 --- /dev/null +++ b/libc/test/src/stdfix/expk_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for expk ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "ExpTest.h" + +#include "src/stdfix/expk.h" + +LIST_EXP_TESTS(k, accum, LIBC_NAMESPACE::expk); -- GitLab From 909ab0e0d1903ad2329ca9fdf248d21330f9437f Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Thu, 7 Mar 2024 15:03:51 -0800 Subject: [PATCH 517/929] [RISCV] Insert a freeze before converting select to AND/OR. (#84232) Select blocks poison, but AND/OR do not. We need to insert a freeze to block poison propagation. This creates suboptimal codegen which I will try to fix with other patches. I'm prioritizing the correctness fix since we have 2 bug reports. Fixes #84200 and #84350 --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 14 +- llvm/test/CodeGen/RISCV/alu64.ll | 3 +- .../CodeGen/RISCV/atomicrmw-uinc-udec-wrap.ll | 16 +- llvm/test/CodeGen/RISCV/bfloat-convert.ll | 169 +- llvm/test/CodeGen/RISCV/double-convert.ll | 137 +- .../CodeGen/RISCV/double-round-conv-sat.ll | 804 +++++----- llvm/test/CodeGen/RISCV/float-convert.ll | 236 +-- .../CodeGen/RISCV/float-round-conv-sat.ll | 636 ++++---- llvm/test/CodeGen/RISCV/forced-atomics.ll | 5 +- llvm/test/CodeGen/RISCV/fpclamptosat.ll | 350 +++-- llvm/test/CodeGen/RISCV/half-convert.ll | 370 +++-- .../test/CodeGen/RISCV/half-round-conv-sat.ll | 1356 +++++++++-------- llvm/test/CodeGen/RISCV/iabs.ll | 88 +- llvm/test/CodeGen/RISCV/pr84200.ll | 22 + llvm/test/CodeGen/RISCV/rv32zbb-zbkb.ll | 16 +- llvm/test/CodeGen/RISCV/rv32zbs.ll | 40 +- .../RISCV/rv64-legal-i32/rv64xtheadbb.ll | 16 +- .../CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll | 13 +- .../CodeGen/RISCV/rvv/fpclamptosat_vec.ll | 617 ++++---- .../CodeGen/RISCV/rvv/vec3-setcc-crash.ll | 26 +- .../CodeGen/RISCV/signed-truncation-check.ll | 9 +- 21 files changed, 2587 insertions(+), 2356 deletions(-) create mode 100644 llvm/test/CodeGen/RISCV/pr84200.ll diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 4713bd605c24..9b748cdcf745 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -7246,25 +7246,25 @@ static SDValue combineSelectToBinOp(SDNode *N, SelectionDAG &DAG, // (select c, -1, y) -> -c | y if (isAllOnesConstant(TrueV)) { SDValue Neg = DAG.getNegative(CondV, DL, VT); - return DAG.getNode(ISD::OR, DL, VT, Neg, FalseV); + return DAG.getNode(ISD::OR, DL, VT, Neg, DAG.getFreeze(FalseV)); } // (select c, y, -1) -> (c-1) | y if (isAllOnesConstant(FalseV)) { SDValue Neg = DAG.getNode(ISD::ADD, DL, VT, CondV, DAG.getAllOnesConstant(DL, VT)); - return DAG.getNode(ISD::OR, DL, VT, Neg, TrueV); + return DAG.getNode(ISD::OR, DL, VT, Neg, DAG.getFreeze(TrueV)); } // (select c, 0, y) -> (c-1) & y if (isNullConstant(TrueV)) { SDValue Neg = DAG.getNode(ISD::ADD, DL, VT, CondV, DAG.getAllOnesConstant(DL, VT)); - return DAG.getNode(ISD::AND, DL, VT, Neg, FalseV); + return DAG.getNode(ISD::AND, DL, VT, Neg, DAG.getFreeze(FalseV)); } // (select c, y, 0) -> -c & y if (isNullConstant(FalseV)) { SDValue Neg = DAG.getNegative(CondV, DL, VT); - return DAG.getNode(ISD::AND, DL, VT, Neg, TrueV); + return DAG.getNode(ISD::AND, DL, VT, Neg, DAG.getFreeze(TrueV)); } } @@ -7290,13 +7290,13 @@ static SDValue combineSelectToBinOp(SDNode *N, SelectionDAG &DAG, // (select !x, x, y) -> x & y if (std::optional MatchResult = matchSetCC(LHS, RHS, CC, TrueV)) { return DAG.getNode(*MatchResult ? ISD::OR : ISD::AND, DL, VT, TrueV, - FalseV); + DAG.getFreeze(FalseV)); } // (select x, y, x) -> x & y // (select !x, y, x) -> x | y if (std::optional MatchResult = matchSetCC(LHS, RHS, CC, FalseV)) { - return DAG.getNode(*MatchResult ? ISD::AND : ISD::OR, DL, VT, TrueV, - FalseV); + return DAG.getNode(*MatchResult ? ISD::AND : ISD::OR, DL, VT, + DAG.getFreeze(TrueV), FalseV); } } diff --git a/llvm/test/CodeGen/RISCV/alu64.ll b/llvm/test/CodeGen/RISCV/alu64.ll index f032756e007b..e16f6abcca24 100644 --- a/llvm/test/CodeGen/RISCV/alu64.ll +++ b/llvm/test/CodeGen/RISCV/alu64.ll @@ -58,7 +58,8 @@ define i64 @sltiu(i64 %a) nounwind { ; RV32I-LABEL: sltiu: ; RV32I: # %bb.0: ; RV32I-NEXT: sltiu a0, a0, 3 -; RV32I-NEXT: seqz a1, a1 +; RV32I-NEXT: snez a1, a1 +; RV32I-NEXT: addi a1, a1, -1 ; RV32I-NEXT: and a0, a1, a0 ; RV32I-NEXT: li a1, 0 ; RV32I-NEXT: ret diff --git a/llvm/test/CodeGen/RISCV/atomicrmw-uinc-udec-wrap.ll b/llvm/test/CodeGen/RISCV/atomicrmw-uinc-udec-wrap.ll index aa962d68fc52..5914e45a1533 100644 --- a/llvm/test/CodeGen/RISCV/atomicrmw-uinc-udec-wrap.ll +++ b/llvm/test/CodeGen/RISCV/atomicrmw-uinc-udec-wrap.ll @@ -372,10 +372,10 @@ define i32 @atomicrmw_uinc_wrap_i32(ptr %ptr, i32 %val) { ; RV32IA-NEXT: # =>This Loop Header: Depth=1 ; RV32IA-NEXT: # Child Loop BB2_3 Depth 2 ; RV32IA-NEXT: mv a3, a2 -; RV32IA-NEXT: addi a2, a2, 1 -; RV32IA-NEXT: sltu a4, a3, a1 -; RV32IA-NEXT: neg a4, a4 -; RV32IA-NEXT: and a4, a4, a2 +; RV32IA-NEXT: addi a4, a2, 1 +; RV32IA-NEXT: sltu a2, a2, a1 +; RV32IA-NEXT: neg a2, a2 +; RV32IA-NEXT: and a4, a2, a4 ; RV32IA-NEXT: .LBB2_3: # %atomicrmw.start ; RV32IA-NEXT: # Parent Loop BB2_1 Depth=1 ; RV32IA-NEXT: # => This Inner Loop Header: Depth=2 @@ -607,10 +607,10 @@ define i64 @atomicrmw_uinc_wrap_i64(ptr %ptr, i64 %val) { ; RV64IA-NEXT: # =>This Loop Header: Depth=1 ; RV64IA-NEXT: # Child Loop BB3_3 Depth 2 ; RV64IA-NEXT: mv a3, a2 -; RV64IA-NEXT: addi a2, a2, 1 -; RV64IA-NEXT: sltu a4, a3, a1 -; RV64IA-NEXT: neg a4, a4 -; RV64IA-NEXT: and a4, a4, a2 +; RV64IA-NEXT: addi a4, a2, 1 +; RV64IA-NEXT: sltu a2, a2, a1 +; RV64IA-NEXT: neg a2, a2 +; RV64IA-NEXT: and a4, a2, a4 ; RV64IA-NEXT: .LBB3_3: # %atomicrmw.start ; RV64IA-NEXT: # Parent Loop BB3_1 Depth=1 ; RV64IA-NEXT: # => This Inner Loop Header: Depth=2 diff --git a/llvm/test/CodeGen/RISCV/bfloat-convert.ll b/llvm/test/CodeGen/RISCV/bfloat-convert.ll index d533607ad54e..0216d00be218 100644 --- a/llvm/test/CodeGen/RISCV/bfloat-convert.ll +++ b/llvm/test/CodeGen/RISCV/bfloat-convert.ll @@ -456,121 +456,142 @@ define i64 @fcvt_l_bf16(bfloat %a) nounwind { define i64 @fcvt_l_bf16_sat(bfloat %a) nounwind { ; RV32IZFBFMIN-LABEL: fcvt_l_bf16_sat: ; RV32IZFBFMIN: # %bb.0: # %start -; RV32IZFBFMIN-NEXT: addi sp, sp, -16 -; RV32IZFBFMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFBFMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFBFMIN-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFBFMIN-NEXT: addi sp, sp, -32 +; RV32IZFBFMIN-NEXT: sw ra, 28(sp) # 4-byte Folded Spill +; RV32IZFBFMIN-NEXT: sw s0, 24(sp) # 4-byte Folded Spill +; RV32IZFBFMIN-NEXT: sw s1, 20(sp) # 4-byte Folded Spill +; RV32IZFBFMIN-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IZFBFMIN-NEXT: sw s3, 12(sp) # 4-byte Folded Spill +; RV32IZFBFMIN-NEXT: fsw fs0, 8(sp) # 4-byte Folded Spill +; RV32IZFBFMIN-NEXT: lui a0, %hi(.LCPI10_0) +; RV32IZFBFMIN-NEXT: flw fa5, %lo(.LCPI10_0)(a0) ; RV32IZFBFMIN-NEXT: fcvt.s.bf16 fs0, fa0 +; RV32IZFBFMIN-NEXT: flt.s s0, fa5, fs0 +; RV32IZFBFMIN-NEXT: neg s1, s0 ; RV32IZFBFMIN-NEXT: lui a0, 913408 ; RV32IZFBFMIN-NEXT: fmv.w.x fa5, a0 -; RV32IZFBFMIN-NEXT: fle.s s0, fa5, fs0 +; RV32IZFBFMIN-NEXT: fle.s s2, fa5, fs0 +; RV32IZFBFMIN-NEXT: neg s3, s2 ; RV32IZFBFMIN-NEXT: fmv.s fa0, fs0 ; RV32IZFBFMIN-NEXT: call __fixsfdi +; RV32IZFBFMIN-NEXT: and a0, s3, a0 +; RV32IZFBFMIN-NEXT: or a0, s1, a0 +; RV32IZFBFMIN-NEXT: feq.s a2, fs0, fs0 +; RV32IZFBFMIN-NEXT: neg a2, a2 ; RV32IZFBFMIN-NEXT: lui a4, 524288 -; RV32IZFBFMIN-NEXT: lui a2, 524288 -; RV32IZFBFMIN-NEXT: beqz s0, .LBB10_2 +; RV32IZFBFMIN-NEXT: li a5, 1 +; RV32IZFBFMIN-NEXT: lui a3, 524288 +; RV32IZFBFMIN-NEXT: bne s2, a5, .LBB10_2 ; RV32IZFBFMIN-NEXT: # %bb.1: # %start -; RV32IZFBFMIN-NEXT: mv a2, a1 +; RV32IZFBFMIN-NEXT: mv a3, a1 ; RV32IZFBFMIN-NEXT: .LBB10_2: # %start -; RV32IZFBFMIN-NEXT: lui a1, %hi(.LCPI10_0) -; RV32IZFBFMIN-NEXT: flw fa5, %lo(.LCPI10_0)(a1) -; RV32IZFBFMIN-NEXT: flt.s a3, fa5, fs0 -; RV32IZFBFMIN-NEXT: beqz a3, .LBB10_4 +; RV32IZFBFMIN-NEXT: and a0, a2, a0 +; RV32IZFBFMIN-NEXT: beqz s0, .LBB10_4 ; RV32IZFBFMIN-NEXT: # %bb.3: -; RV32IZFBFMIN-NEXT: addi a2, a4, -1 +; RV32IZFBFMIN-NEXT: addi a3, a4, -1 ; RV32IZFBFMIN-NEXT: .LBB10_4: # %start -; RV32IZFBFMIN-NEXT: feq.s a1, fs0, fs0 -; RV32IZFBFMIN-NEXT: neg a4, a1 -; RV32IZFBFMIN-NEXT: and a1, a4, a2 -; RV32IZFBFMIN-NEXT: neg a2, a3 -; RV32IZFBFMIN-NEXT: neg a3, s0 -; RV32IZFBFMIN-NEXT: and a0, a3, a0 -; RV32IZFBFMIN-NEXT: or a0, a2, a0 -; RV32IZFBFMIN-NEXT: and a0, a4, a0 -; RV32IZFBFMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFBFMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFBFMIN-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload -; RV32IZFBFMIN-NEXT: addi sp, sp, 16 +; RV32IZFBFMIN-NEXT: and a1, a2, a3 +; RV32IZFBFMIN-NEXT: lw ra, 28(sp) # 4-byte Folded Reload +; RV32IZFBFMIN-NEXT: lw s0, 24(sp) # 4-byte Folded Reload +; RV32IZFBFMIN-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; RV32IZFBFMIN-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; RV32IZFBFMIN-NEXT: lw s3, 12(sp) # 4-byte Folded Reload +; RV32IZFBFMIN-NEXT: flw fs0, 8(sp) # 4-byte Folded Reload +; RV32IZFBFMIN-NEXT: addi sp, sp, 32 ; RV32IZFBFMIN-NEXT: ret ; ; R32IDZFBFMIN-LABEL: fcvt_l_bf16_sat: ; R32IDZFBFMIN: # %bb.0: # %start -; R32IDZFBFMIN-NEXT: addi sp, sp, -16 -; R32IDZFBFMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; R32IDZFBFMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; R32IDZFBFMIN-NEXT: addi sp, sp, -32 +; R32IDZFBFMIN-NEXT: sw ra, 28(sp) # 4-byte Folded Spill +; R32IDZFBFMIN-NEXT: sw s0, 24(sp) # 4-byte Folded Spill +; R32IDZFBFMIN-NEXT: sw s1, 20(sp) # 4-byte Folded Spill +; R32IDZFBFMIN-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; R32IDZFBFMIN-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; R32IDZFBFMIN-NEXT: fsd fs0, 0(sp) # 8-byte Folded Spill +; R32IDZFBFMIN-NEXT: lui a0, %hi(.LCPI10_0) +; R32IDZFBFMIN-NEXT: flw fa5, %lo(.LCPI10_0)(a0) ; R32IDZFBFMIN-NEXT: fcvt.s.bf16 fs0, fa0 +; R32IDZFBFMIN-NEXT: flt.s s0, fa5, fs0 +; R32IDZFBFMIN-NEXT: neg s1, s0 ; R32IDZFBFMIN-NEXT: lui a0, 913408 ; R32IDZFBFMIN-NEXT: fmv.w.x fa5, a0 -; R32IDZFBFMIN-NEXT: fle.s s0, fa5, fs0 +; R32IDZFBFMIN-NEXT: fle.s s2, fa5, fs0 +; R32IDZFBFMIN-NEXT: neg s3, s2 ; R32IDZFBFMIN-NEXT: fmv.s fa0, fs0 ; R32IDZFBFMIN-NEXT: call __fixsfdi +; R32IDZFBFMIN-NEXT: and a0, s3, a0 +; R32IDZFBFMIN-NEXT: or a0, s1, a0 +; R32IDZFBFMIN-NEXT: feq.s a2, fs0, fs0 +; R32IDZFBFMIN-NEXT: neg a2, a2 ; R32IDZFBFMIN-NEXT: lui a4, 524288 -; R32IDZFBFMIN-NEXT: lui a2, 524288 -; R32IDZFBFMIN-NEXT: beqz s0, .LBB10_2 +; R32IDZFBFMIN-NEXT: li a5, 1 +; R32IDZFBFMIN-NEXT: lui a3, 524288 +; R32IDZFBFMIN-NEXT: bne s2, a5, .LBB10_2 ; R32IDZFBFMIN-NEXT: # %bb.1: # %start -; R32IDZFBFMIN-NEXT: mv a2, a1 +; R32IDZFBFMIN-NEXT: mv a3, a1 ; R32IDZFBFMIN-NEXT: .LBB10_2: # %start -; R32IDZFBFMIN-NEXT: lui a1, %hi(.LCPI10_0) -; R32IDZFBFMIN-NEXT: flw fa5, %lo(.LCPI10_0)(a1) -; R32IDZFBFMIN-NEXT: flt.s a3, fa5, fs0 -; R32IDZFBFMIN-NEXT: beqz a3, .LBB10_4 +; R32IDZFBFMIN-NEXT: and a0, a2, a0 +; R32IDZFBFMIN-NEXT: beqz s0, .LBB10_4 ; R32IDZFBFMIN-NEXT: # %bb.3: -; R32IDZFBFMIN-NEXT: addi a2, a4, -1 +; R32IDZFBFMIN-NEXT: addi a3, a4, -1 ; R32IDZFBFMIN-NEXT: .LBB10_4: # %start -; R32IDZFBFMIN-NEXT: feq.s a1, fs0, fs0 -; R32IDZFBFMIN-NEXT: neg a4, a1 -; R32IDZFBFMIN-NEXT: and a1, a4, a2 -; R32IDZFBFMIN-NEXT: neg a2, a3 -; R32IDZFBFMIN-NEXT: neg a3, s0 -; R32IDZFBFMIN-NEXT: and a0, a3, a0 -; R32IDZFBFMIN-NEXT: or a0, a2, a0 -; R32IDZFBFMIN-NEXT: and a0, a4, a0 -; R32IDZFBFMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; R32IDZFBFMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; R32IDZFBFMIN-NEXT: and a1, a2, a3 +; R32IDZFBFMIN-NEXT: lw ra, 28(sp) # 4-byte Folded Reload +; R32IDZFBFMIN-NEXT: lw s0, 24(sp) # 4-byte Folded Reload +; R32IDZFBFMIN-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; R32IDZFBFMIN-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; R32IDZFBFMIN-NEXT: lw s3, 12(sp) # 4-byte Folded Reload ; R32IDZFBFMIN-NEXT: fld fs0, 0(sp) # 8-byte Folded Reload -; R32IDZFBFMIN-NEXT: addi sp, sp, 16 +; R32IDZFBFMIN-NEXT: addi sp, sp, 32 ; R32IDZFBFMIN-NEXT: ret ; ; RV32ID-LABEL: fcvt_l_bf16_sat: ; RV32ID: # %bb.0: # %start -; RV32ID-NEXT: addi sp, sp, -16 -; RV32ID-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32ID-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32ID-NEXT: addi sp, sp, -32 +; RV32ID-NEXT: sw ra, 28(sp) # 4-byte Folded Spill +; RV32ID-NEXT: sw s0, 24(sp) # 4-byte Folded Spill +; RV32ID-NEXT: sw s1, 20(sp) # 4-byte Folded Spill +; RV32ID-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32ID-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32ID-NEXT: fsd fs0, 0(sp) # 8-byte Folded Spill +; RV32ID-NEXT: lui a0, %hi(.LCPI10_0) +; RV32ID-NEXT: flw fa5, %lo(.LCPI10_0)(a0) ; RV32ID-NEXT: fmv.x.w a0, fa0 ; RV32ID-NEXT: slli a0, a0, 16 ; RV32ID-NEXT: fmv.w.x fs0, a0 +; RV32ID-NEXT: flt.s s0, fa5, fs0 +; RV32ID-NEXT: neg s1, s0 ; RV32ID-NEXT: lui a0, 913408 ; RV32ID-NEXT: fmv.w.x fa5, a0 -; RV32ID-NEXT: fle.s s0, fa5, fs0 +; RV32ID-NEXT: fle.s s2, fa5, fs0 +; RV32ID-NEXT: neg s3, s2 ; RV32ID-NEXT: fmv.s fa0, fs0 ; RV32ID-NEXT: call __fixsfdi +; RV32ID-NEXT: and a0, s3, a0 +; RV32ID-NEXT: or a0, s1, a0 +; RV32ID-NEXT: feq.s a2, fs0, fs0 +; RV32ID-NEXT: neg a2, a2 ; RV32ID-NEXT: lui a4, 524288 -; RV32ID-NEXT: lui a2, 524288 -; RV32ID-NEXT: beqz s0, .LBB10_2 +; RV32ID-NEXT: li a5, 1 +; RV32ID-NEXT: lui a3, 524288 +; RV32ID-NEXT: bne s2, a5, .LBB10_2 ; RV32ID-NEXT: # %bb.1: # %start -; RV32ID-NEXT: mv a2, a1 +; RV32ID-NEXT: mv a3, a1 ; RV32ID-NEXT: .LBB10_2: # %start -; RV32ID-NEXT: lui a1, %hi(.LCPI10_0) -; RV32ID-NEXT: flw fa5, %lo(.LCPI10_0)(a1) -; RV32ID-NEXT: flt.s a3, fa5, fs0 -; RV32ID-NEXT: beqz a3, .LBB10_4 +; RV32ID-NEXT: and a0, a2, a0 +; RV32ID-NEXT: beqz s0, .LBB10_4 ; RV32ID-NEXT: # %bb.3: -; RV32ID-NEXT: addi a2, a4, -1 +; RV32ID-NEXT: addi a3, a4, -1 ; RV32ID-NEXT: .LBB10_4: # %start -; RV32ID-NEXT: feq.s a1, fs0, fs0 -; RV32ID-NEXT: neg a4, a1 -; RV32ID-NEXT: and a1, a4, a2 -; RV32ID-NEXT: neg a2, a3 -; RV32ID-NEXT: neg a3, s0 -; RV32ID-NEXT: and a0, a3, a0 -; RV32ID-NEXT: or a0, a2, a0 -; RV32ID-NEXT: and a0, a4, a0 -; RV32ID-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32ID-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32ID-NEXT: and a1, a2, a3 +; RV32ID-NEXT: lw ra, 28(sp) # 4-byte Folded Reload +; RV32ID-NEXT: lw s0, 24(sp) # 4-byte Folded Reload +; RV32ID-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; RV32ID-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; RV32ID-NEXT: lw s3, 12(sp) # 4-byte Folded Reload ; RV32ID-NEXT: fld fs0, 0(sp) # 8-byte Folded Reload -; RV32ID-NEXT: addi sp, sp, 16 +; RV32ID-NEXT: addi sp, sp, 32 ; RV32ID-NEXT: ret ; ; CHECK64ZFBFMIN-LABEL: fcvt_l_bf16_sat: @@ -654,7 +675,8 @@ define i64 @fcvt_lu_bf16_sat(bfloat %a) nounwind { ; CHECK32ZFBFMIN-NEXT: neg s0, a0 ; CHECK32ZFBFMIN-NEXT: fmv.w.x fa5, zero ; CHECK32ZFBFMIN-NEXT: fle.s a0, fa5, fa0 -; CHECK32ZFBFMIN-NEXT: neg s1, a0 +; CHECK32ZFBFMIN-NEXT: xori a0, a0, 1 +; CHECK32ZFBFMIN-NEXT: addi s1, a0, -1 ; CHECK32ZFBFMIN-NEXT: call __fixunssfdi ; CHECK32ZFBFMIN-NEXT: and a0, s1, a0 ; CHECK32ZFBFMIN-NEXT: or a0, s0, a0 @@ -681,7 +703,8 @@ define i64 @fcvt_lu_bf16_sat(bfloat %a) nounwind { ; RV32ID-NEXT: neg s0, a0 ; RV32ID-NEXT: fmv.w.x fa5, zero ; RV32ID-NEXT: fle.s a0, fa5, fa0 -; RV32ID-NEXT: neg s1, a0 +; RV32ID-NEXT: xori a0, a0, 1 +; RV32ID-NEXT: addi s1, a0, -1 ; RV32ID-NEXT: call __fixunssfdi ; RV32ID-NEXT: and a0, s1, a0 ; RV32ID-NEXT: or a0, s0, a0 diff --git a/llvm/test/CodeGen/RISCV/double-convert.ll b/llvm/test/CodeGen/RISCV/double-convert.ll index eb8ffe75ef76..f2e37f55521b 100644 --- a/llvm/test/CodeGen/RISCV/double-convert.ll +++ b/llvm/test/CodeGen/RISCV/double-convert.ll @@ -749,40 +749,47 @@ define i64 @fcvt_l_d(double %a) nounwind { define i64 @fcvt_l_d_sat(double %a) nounwind { ; RV32IFD-LABEL: fcvt_l_d_sat: ; RV32IFD: # %bb.0: # %start -; RV32IFD-NEXT: addi sp, sp, -16 -; RV32IFD-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IFD-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: addi sp, sp, -32 +; RV32IFD-NEXT: sw ra, 28(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s0, 24(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s1, 20(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32IFD-NEXT: fsd fs0, 0(sp) # 8-byte Folded Spill ; RV32IFD-NEXT: lui a0, %hi(.LCPI12_0) ; RV32IFD-NEXT: fld fa5, %lo(.LCPI12_0)(a0) +; RV32IFD-NEXT: lui a0, %hi(.LCPI12_1) +; RV32IFD-NEXT: fld fa4, %lo(.LCPI12_1)(a0) ; RV32IFD-NEXT: fmv.d fs0, fa0 -; RV32IFD-NEXT: fle.d s0, fa5, fa0 +; RV32IFD-NEXT: flt.d s0, fa5, fa0 +; RV32IFD-NEXT: neg s1, s0 +; RV32IFD-NEXT: fle.d s2, fa4, fa0 +; RV32IFD-NEXT: neg s3, s2 ; RV32IFD-NEXT: call __fixdfdi +; RV32IFD-NEXT: and a0, s3, a0 +; RV32IFD-NEXT: or a0, s1, a0 +; RV32IFD-NEXT: feq.d a2, fs0, fs0 +; RV32IFD-NEXT: neg a2, a2 ; RV32IFD-NEXT: lui a4, 524288 -; RV32IFD-NEXT: lui a2, 524288 -; RV32IFD-NEXT: beqz s0, .LBB12_2 +; RV32IFD-NEXT: li a5, 1 +; RV32IFD-NEXT: lui a3, 524288 +; RV32IFD-NEXT: bne s2, a5, .LBB12_2 ; RV32IFD-NEXT: # %bb.1: # %start -; RV32IFD-NEXT: mv a2, a1 +; RV32IFD-NEXT: mv a3, a1 ; RV32IFD-NEXT: .LBB12_2: # %start -; RV32IFD-NEXT: lui a1, %hi(.LCPI12_1) -; RV32IFD-NEXT: fld fa5, %lo(.LCPI12_1)(a1) -; RV32IFD-NEXT: flt.d a3, fa5, fs0 -; RV32IFD-NEXT: beqz a3, .LBB12_4 +; RV32IFD-NEXT: and a0, a2, a0 +; RV32IFD-NEXT: beqz s0, .LBB12_4 ; RV32IFD-NEXT: # %bb.3: -; RV32IFD-NEXT: addi a2, a4, -1 +; RV32IFD-NEXT: addi a3, a4, -1 ; RV32IFD-NEXT: .LBB12_4: # %start -; RV32IFD-NEXT: feq.d a1, fs0, fs0 -; RV32IFD-NEXT: neg a4, a1 -; RV32IFD-NEXT: and a1, a4, a2 -; RV32IFD-NEXT: neg a2, a3 -; RV32IFD-NEXT: neg a3, s0 -; RV32IFD-NEXT: and a0, a3, a0 -; RV32IFD-NEXT: or a0, a2, a0 -; RV32IFD-NEXT: and a0, a4, a0 -; RV32IFD-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IFD-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: and a1, a2, a3 +; RV32IFD-NEXT: lw ra, 28(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s0, 24(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s3, 12(sp) # 4-byte Folded Reload ; RV32IFD-NEXT: fld fs0, 0(sp) # 8-byte Folded Reload -; RV32IFD-NEXT: addi sp, sp, 16 +; RV32IFD-NEXT: addi sp, sp, 32 ; RV32IFD-NEXT: ret ; ; RV64IFD-LABEL: fcvt_l_d_sat: @@ -800,40 +807,45 @@ define i64 @fcvt_l_d_sat(double %a) nounwind { ; RV32IZFINXZDINX-NEXT: sw ra, 28(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s0, 24(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s1, 20(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw s0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw s1, 12(sp) -; RV32IZFINXZDINX-NEXT: call __fixdfdi +; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s3, 12(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) +; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) +; RV32IZFINXZDINX-NEXT: lw s0, 0(sp) +; RV32IZFINXZDINX-NEXT: lw s1, 4(sp) ; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI12_0) ; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI12_0+4)(a2) ; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI12_0)(a2) -; RV32IZFINXZDINX-NEXT: fle.d a2, a2, s0 +; RV32IZFINXZDINX-NEXT: fle.d s2, a2, s0 +; RV32IZFINXZDINX-NEXT: neg s3, s2 +; RV32IZFINXZDINX-NEXT: call __fixdfdi +; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI12_1) +; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI12_1+4)(a2) +; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI12_1)(a2) +; RV32IZFINXZDINX-NEXT: and a0, s3, a0 +; RV32IZFINXZDINX-NEXT: flt.d a3, a2, s0 +; RV32IZFINXZDINX-NEXT: neg a2, a3 +; RV32IZFINXZDINX-NEXT: or a0, a2, a0 +; RV32IZFINXZDINX-NEXT: feq.d a2, s0, s0 +; RV32IZFINXZDINX-NEXT: neg a2, a2 ; RV32IZFINXZDINX-NEXT: lui a5, 524288 -; RV32IZFINXZDINX-NEXT: lui a3, 524288 -; RV32IZFINXZDINX-NEXT: beqz a2, .LBB12_2 +; RV32IZFINXZDINX-NEXT: li a6, 1 +; RV32IZFINXZDINX-NEXT: lui a4, 524288 +; RV32IZFINXZDINX-NEXT: bne s2, a6, .LBB12_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: # %start -; RV32IZFINXZDINX-NEXT: mv a3, a1 +; RV32IZFINXZDINX-NEXT: mv a4, a1 ; RV32IZFINXZDINX-NEXT: .LBB12_2: # %start -; RV32IZFINXZDINX-NEXT: lui a1, %hi(.LCPI12_1) -; RV32IZFINXZDINX-NEXT: lw a6, %lo(.LCPI12_1)(a1) -; RV32IZFINXZDINX-NEXT: lw a7, %lo(.LCPI12_1+4)(a1) -; RV32IZFINXZDINX-NEXT: flt.d a4, a6, s0 -; RV32IZFINXZDINX-NEXT: beqz a4, .LBB12_4 +; RV32IZFINXZDINX-NEXT: and a0, a2, a0 +; RV32IZFINXZDINX-NEXT: beqz a3, .LBB12_4 ; RV32IZFINXZDINX-NEXT: # %bb.3: -; RV32IZFINXZDINX-NEXT: addi a3, a5, -1 +; RV32IZFINXZDINX-NEXT: addi a4, a5, -1 ; RV32IZFINXZDINX-NEXT: .LBB12_4: # %start -; RV32IZFINXZDINX-NEXT: feq.d a1, s0, s0 -; RV32IZFINXZDINX-NEXT: neg a5, a1 -; RV32IZFINXZDINX-NEXT: and a1, a5, a3 -; RV32IZFINXZDINX-NEXT: neg a2, a2 -; RV32IZFINXZDINX-NEXT: and a0, a2, a0 -; RV32IZFINXZDINX-NEXT: neg a2, a4 -; RV32IZFINXZDINX-NEXT: or a0, a2, a0 -; RV32IZFINXZDINX-NEXT: and a0, a5, a0 +; RV32IZFINXZDINX-NEXT: and a1, a2, a4 ; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s3, 12(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: addi sp, sp, 32 ; RV32IZFINXZDINX-NEXT: ret ; @@ -1013,23 +1025,23 @@ define i64 @fcvt_lu_d_sat(double %a) nounwind { ; RV32IFD-NEXT: addi sp, sp, -16 ; RV32IFD-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IFD-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IFD-NEXT: fsd fs0, 0(sp) # 8-byte Folded Spill -; RV32IFD-NEXT: fmv.d fs0, fa0 +; RV32IFD-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: lui a0, %hi(.LCPI14_0) +; RV32IFD-NEXT: fld fa5, %lo(.LCPI14_0)(a0) +; RV32IFD-NEXT: flt.d a0, fa5, fa0 +; RV32IFD-NEXT: neg s0, a0 ; RV32IFD-NEXT: fcvt.d.w fa5, zero ; RV32IFD-NEXT: fle.d a0, fa5, fa0 -; RV32IFD-NEXT: neg s0, a0 +; RV32IFD-NEXT: xori a0, a0, 1 +; RV32IFD-NEXT: addi s1, a0, -1 ; RV32IFD-NEXT: call __fixunsdfdi -; RV32IFD-NEXT: lui a2, %hi(.LCPI14_0) -; RV32IFD-NEXT: fld fa5, %lo(.LCPI14_0)(a2) -; RV32IFD-NEXT: and a0, s0, a0 -; RV32IFD-NEXT: flt.d a2, fa5, fs0 -; RV32IFD-NEXT: neg a2, a2 -; RV32IFD-NEXT: or a0, a2, a0 -; RV32IFD-NEXT: and a1, s0, a1 -; RV32IFD-NEXT: or a1, a2, a1 +; RV32IFD-NEXT: and a0, s1, a0 +; RV32IFD-NEXT: or a0, s0, a0 +; RV32IFD-NEXT: and a1, s1, a1 +; RV32IFD-NEXT: or a1, s0, a1 ; RV32IFD-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IFD-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IFD-NEXT: fld fs0, 0(sp) # 8-byte Folded Reload +; RV32IFD-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IFD-NEXT: addi sp, sp, 16 ; RV32IFD-NEXT: ret ; @@ -1054,11 +1066,12 @@ define i64 @fcvt_lu_d_sat(double %a) nounwind { ; RV32IZFINXZDINX-NEXT: lw s1, 12(sp) ; RV32IZFINXZDINX-NEXT: call __fixunsdfdi ; RV32IZFINXZDINX-NEXT: fcvt.d.w a2, zero -; RV32IZFINXZDINX-NEXT: lui a4, %hi(.LCPI14_0) -; RV32IZFINXZDINX-NEXT: lw a5, %lo(.LCPI14_0+4)(a4) -; RV32IZFINXZDINX-NEXT: lw a4, %lo(.LCPI14_0)(a4) ; RV32IZFINXZDINX-NEXT: fle.d a2, a2, s0 -; RV32IZFINXZDINX-NEXT: neg a2, a2 +; RV32IZFINXZDINX-NEXT: lui a3, %hi(.LCPI14_0) +; RV32IZFINXZDINX-NEXT: lw a4, %lo(.LCPI14_0)(a3) +; RV32IZFINXZDINX-NEXT: lw a5, %lo(.LCPI14_0+4)(a3) +; RV32IZFINXZDINX-NEXT: xori a2, a2, 1 +; RV32IZFINXZDINX-NEXT: addi a2, a2, -1 ; RV32IZFINXZDINX-NEXT: and a0, a2, a0 ; RV32IZFINXZDINX-NEXT: flt.d a3, a4, s0 ; RV32IZFINXZDINX-NEXT: neg a3, a3 diff --git a/llvm/test/CodeGen/RISCV/double-round-conv-sat.ll b/llvm/test/CodeGen/RISCV/double-round-conv-sat.ll index b8c6e8450240..ff2d8e006300 100644 --- a/llvm/test/CodeGen/RISCV/double-round-conv-sat.ll +++ b/llvm/test/CodeGen/RISCV/double-round-conv-sat.ll @@ -50,41 +50,48 @@ define signext i32 @test_floor_si32(double %x) { define i64 @test_floor_si64(double %x) nounwind { ; RV32IFD-LABEL: test_floor_si64: ; RV32IFD: # %bb.0: -; RV32IFD-NEXT: addi sp, sp, -16 -; RV32IFD-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IFD-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: addi sp, sp, -32 +; RV32IFD-NEXT: sw ra, 28(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s0, 24(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s1, 20(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32IFD-NEXT: fsd fs0, 0(sp) # 8-byte Folded Spill ; RV32IFD-NEXT: call floor ; RV32IFD-NEXT: lui a0, %hi(.LCPI1_0) ; RV32IFD-NEXT: fld fa5, %lo(.LCPI1_0)(a0) +; RV32IFD-NEXT: lui a0, %hi(.LCPI1_1) +; RV32IFD-NEXT: fld fa4, %lo(.LCPI1_1)(a0) ; RV32IFD-NEXT: fmv.d fs0, fa0 -; RV32IFD-NEXT: fle.d s0, fa5, fa0 +; RV32IFD-NEXT: flt.d s0, fa5, fa0 +; RV32IFD-NEXT: neg s1, s0 +; RV32IFD-NEXT: fle.d s2, fa4, fa0 +; RV32IFD-NEXT: neg s3, s2 ; RV32IFD-NEXT: call __fixdfdi +; RV32IFD-NEXT: and a0, s3, a0 +; RV32IFD-NEXT: or a0, s1, a0 +; RV32IFD-NEXT: feq.d a2, fs0, fs0 +; RV32IFD-NEXT: neg a2, a2 ; RV32IFD-NEXT: lui a4, 524288 -; RV32IFD-NEXT: lui a2, 524288 -; RV32IFD-NEXT: beqz s0, .LBB1_2 +; RV32IFD-NEXT: li a5, 1 +; RV32IFD-NEXT: lui a3, 524288 +; RV32IFD-NEXT: bne s2, a5, .LBB1_2 ; RV32IFD-NEXT: # %bb.1: -; RV32IFD-NEXT: mv a2, a1 +; RV32IFD-NEXT: mv a3, a1 ; RV32IFD-NEXT: .LBB1_2: -; RV32IFD-NEXT: lui a1, %hi(.LCPI1_1) -; RV32IFD-NEXT: fld fa5, %lo(.LCPI1_1)(a1) -; RV32IFD-NEXT: flt.d a3, fa5, fs0 -; RV32IFD-NEXT: beqz a3, .LBB1_4 +; RV32IFD-NEXT: and a0, a2, a0 +; RV32IFD-NEXT: beqz s0, .LBB1_4 ; RV32IFD-NEXT: # %bb.3: -; RV32IFD-NEXT: addi a2, a4, -1 +; RV32IFD-NEXT: addi a3, a4, -1 ; RV32IFD-NEXT: .LBB1_4: -; RV32IFD-NEXT: feq.d a1, fs0, fs0 -; RV32IFD-NEXT: neg a4, a1 -; RV32IFD-NEXT: and a1, a4, a2 -; RV32IFD-NEXT: neg a2, a3 -; RV32IFD-NEXT: neg a3, s0 -; RV32IFD-NEXT: and a0, a3, a0 -; RV32IFD-NEXT: or a0, a2, a0 -; RV32IFD-NEXT: and a0, a4, a0 -; RV32IFD-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IFD-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: and a1, a2, a3 +; RV32IFD-NEXT: lw ra, 28(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s0, 24(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s3, 12(sp) # 4-byte Folded Reload ; RV32IFD-NEXT: fld fs0, 0(sp) # 8-byte Folded Reload -; RV32IFD-NEXT: addi sp, sp, 16 +; RV32IFD-NEXT: addi sp, sp, 32 ; RV32IFD-NEXT: ret ; ; RV64IFD-LABEL: test_floor_si64: @@ -101,44 +108,47 @@ define i64 @test_floor_si64(double %x) nounwind { ; RV32IZFINXZDINX-NEXT: addi sp, sp, -32 ; RV32IZFINXZDINX-NEXT: sw ra, 28(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s0, 24(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s2, 20(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s3, 16(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s1, 20(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call floor -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw s2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw s3, 12(sp) +; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) +; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) +; RV32IZFINXZDINX-NEXT: lw s0, 0(sp) +; RV32IZFINXZDINX-NEXT: lw s1, 4(sp) ; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI1_0) ; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI1_0+4)(a2) ; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI1_0)(a2) -; RV32IZFINXZDINX-NEXT: fle.d s0, a2, s2 +; RV32IZFINXZDINX-NEXT: fle.d s2, a2, s0 +; RV32IZFINXZDINX-NEXT: neg s3, s2 ; RV32IZFINXZDINX-NEXT: call __fixdfdi +; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI1_1) +; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI1_1+4)(a2) +; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI1_1)(a2) +; RV32IZFINXZDINX-NEXT: and a0, s3, a0 +; RV32IZFINXZDINX-NEXT: flt.d a3, a2, s0 +; RV32IZFINXZDINX-NEXT: neg a2, a3 +; RV32IZFINXZDINX-NEXT: or a0, a2, a0 +; RV32IZFINXZDINX-NEXT: feq.d a2, s0, s0 +; RV32IZFINXZDINX-NEXT: neg a2, a2 +; RV32IZFINXZDINX-NEXT: lui a5, 524288 +; RV32IZFINXZDINX-NEXT: li a6, 1 ; RV32IZFINXZDINX-NEXT: lui a4, 524288 -; RV32IZFINXZDINX-NEXT: lui a2, 524288 -; RV32IZFINXZDINX-NEXT: beqz s0, .LBB1_2 +; RV32IZFINXZDINX-NEXT: bne s2, a6, .LBB1_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: -; RV32IZFINXZDINX-NEXT: mv a2, a1 +; RV32IZFINXZDINX-NEXT: mv a4, a1 ; RV32IZFINXZDINX-NEXT: .LBB1_2: -; RV32IZFINXZDINX-NEXT: lui a1, %hi(.LCPI1_1) -; RV32IZFINXZDINX-NEXT: lw a6, %lo(.LCPI1_1)(a1) -; RV32IZFINXZDINX-NEXT: lw a7, %lo(.LCPI1_1+4)(a1) -; RV32IZFINXZDINX-NEXT: flt.d a3, a6, s2 +; RV32IZFINXZDINX-NEXT: and a0, a2, a0 ; RV32IZFINXZDINX-NEXT: beqz a3, .LBB1_4 ; RV32IZFINXZDINX-NEXT: # %bb.3: -; RV32IZFINXZDINX-NEXT: addi a2, a4, -1 +; RV32IZFINXZDINX-NEXT: addi a4, a5, -1 ; RV32IZFINXZDINX-NEXT: .LBB1_4: -; RV32IZFINXZDINX-NEXT: feq.d a1, s2, s2 -; RV32IZFINXZDINX-NEXT: neg a4, a1 -; RV32IZFINXZDINX-NEXT: and a1, a4, a2 -; RV32IZFINXZDINX-NEXT: neg a2, s0 -; RV32IZFINXZDINX-NEXT: and a0, a2, a0 -; RV32IZFINXZDINX-NEXT: neg a2, a3 -; RV32IZFINXZDINX-NEXT: or a0, a2, a0 -; RV32IZFINXZDINX-NEXT: and a0, a4, a0 +; RV32IZFINXZDINX-NEXT: and a1, a2, a4 ; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s2, 20(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s3, 16(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s3, 12(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: addi sp, sp, 32 ; RV32IZFINXZDINX-NEXT: ret ; @@ -208,7 +218,8 @@ define i64 @test_floor_ui64(double %x) nounwind { ; RV32IFD-NEXT: neg s0, a0 ; RV32IFD-NEXT: fcvt.d.w fa5, zero ; RV32IFD-NEXT: fle.d a0, fa5, fa0 -; RV32IFD-NEXT: neg s1, a0 +; RV32IFD-NEXT: xori a0, a0, 1 +; RV32IFD-NEXT: addi s1, a0, -1 ; RV32IFD-NEXT: call __fixunsdfdi ; RV32IFD-NEXT: and a0, s1, a0 ; RV32IFD-NEXT: or a0, s0, a0 @@ -235,29 +246,28 @@ define i64 @test_floor_ui64(double %x) nounwind { ; RV32IZFINXZDINX-NEXT: sw ra, 28(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s0, 24(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s1, 20(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call floor ; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) ; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: lw s0, 8(sp) ; RV32IZFINXZDINX-NEXT: lw s1, 12(sp) +; RV32IZFINXZDINX-NEXT: call __fixunsdfdi ; RV32IZFINXZDINX-NEXT: fcvt.d.w a2, zero ; RV32IZFINXZDINX-NEXT: fle.d a2, a2, s0 -; RV32IZFINXZDINX-NEXT: neg s2, a2 -; RV32IZFINXZDINX-NEXT: call __fixunsdfdi -; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI3_0) -; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI3_0+4)(a2) -; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI3_0)(a2) -; RV32IZFINXZDINX-NEXT: and a0, s2, a0 -; RV32IZFINXZDINX-NEXT: flt.d a2, a2, s0 -; RV32IZFINXZDINX-NEXT: neg a2, a2 -; RV32IZFINXZDINX-NEXT: or a0, a2, a0 -; RV32IZFINXZDINX-NEXT: and a1, s2, a1 -; RV32IZFINXZDINX-NEXT: or a1, a2, a1 +; RV32IZFINXZDINX-NEXT: lui a3, %hi(.LCPI3_0) +; RV32IZFINXZDINX-NEXT: lw a4, %lo(.LCPI3_0)(a3) +; RV32IZFINXZDINX-NEXT: lw a5, %lo(.LCPI3_0+4)(a3) +; RV32IZFINXZDINX-NEXT: xori a2, a2, 1 +; RV32IZFINXZDINX-NEXT: addi a2, a2, -1 +; RV32IZFINXZDINX-NEXT: and a0, a2, a0 +; RV32IZFINXZDINX-NEXT: flt.d a3, a4, s0 +; RV32IZFINXZDINX-NEXT: neg a3, a3 +; RV32IZFINXZDINX-NEXT: or a0, a3, a0 +; RV32IZFINXZDINX-NEXT: and a1, a2, a1 +; RV32IZFINXZDINX-NEXT: or a1, a3, a1 ; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s2, 16(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: addi sp, sp, 32 ; RV32IZFINXZDINX-NEXT: ret ; @@ -316,41 +326,48 @@ define signext i32 @test_ceil_si32(double %x) { define i64 @test_ceil_si64(double %x) nounwind { ; RV32IFD-LABEL: test_ceil_si64: ; RV32IFD: # %bb.0: -; RV32IFD-NEXT: addi sp, sp, -16 -; RV32IFD-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IFD-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: addi sp, sp, -32 +; RV32IFD-NEXT: sw ra, 28(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s0, 24(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s1, 20(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32IFD-NEXT: fsd fs0, 0(sp) # 8-byte Folded Spill ; RV32IFD-NEXT: call ceil ; RV32IFD-NEXT: lui a0, %hi(.LCPI5_0) ; RV32IFD-NEXT: fld fa5, %lo(.LCPI5_0)(a0) +; RV32IFD-NEXT: lui a0, %hi(.LCPI5_1) +; RV32IFD-NEXT: fld fa4, %lo(.LCPI5_1)(a0) ; RV32IFD-NEXT: fmv.d fs0, fa0 -; RV32IFD-NEXT: fle.d s0, fa5, fa0 +; RV32IFD-NEXT: flt.d s0, fa5, fa0 +; RV32IFD-NEXT: neg s1, s0 +; RV32IFD-NEXT: fle.d s2, fa4, fa0 +; RV32IFD-NEXT: neg s3, s2 ; RV32IFD-NEXT: call __fixdfdi +; RV32IFD-NEXT: and a0, s3, a0 +; RV32IFD-NEXT: or a0, s1, a0 +; RV32IFD-NEXT: feq.d a2, fs0, fs0 +; RV32IFD-NEXT: neg a2, a2 ; RV32IFD-NEXT: lui a4, 524288 -; RV32IFD-NEXT: lui a2, 524288 -; RV32IFD-NEXT: beqz s0, .LBB5_2 +; RV32IFD-NEXT: li a5, 1 +; RV32IFD-NEXT: lui a3, 524288 +; RV32IFD-NEXT: bne s2, a5, .LBB5_2 ; RV32IFD-NEXT: # %bb.1: -; RV32IFD-NEXT: mv a2, a1 +; RV32IFD-NEXT: mv a3, a1 ; RV32IFD-NEXT: .LBB5_2: -; RV32IFD-NEXT: lui a1, %hi(.LCPI5_1) -; RV32IFD-NEXT: fld fa5, %lo(.LCPI5_1)(a1) -; RV32IFD-NEXT: flt.d a3, fa5, fs0 -; RV32IFD-NEXT: beqz a3, .LBB5_4 +; RV32IFD-NEXT: and a0, a2, a0 +; RV32IFD-NEXT: beqz s0, .LBB5_4 ; RV32IFD-NEXT: # %bb.3: -; RV32IFD-NEXT: addi a2, a4, -1 +; RV32IFD-NEXT: addi a3, a4, -1 ; RV32IFD-NEXT: .LBB5_4: -; RV32IFD-NEXT: feq.d a1, fs0, fs0 -; RV32IFD-NEXT: neg a4, a1 -; RV32IFD-NEXT: and a1, a4, a2 -; RV32IFD-NEXT: neg a2, a3 -; RV32IFD-NEXT: neg a3, s0 -; RV32IFD-NEXT: and a0, a3, a0 -; RV32IFD-NEXT: or a0, a2, a0 -; RV32IFD-NEXT: and a0, a4, a0 -; RV32IFD-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IFD-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: and a1, a2, a3 +; RV32IFD-NEXT: lw ra, 28(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s0, 24(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s3, 12(sp) # 4-byte Folded Reload ; RV32IFD-NEXT: fld fs0, 0(sp) # 8-byte Folded Reload -; RV32IFD-NEXT: addi sp, sp, 16 +; RV32IFD-NEXT: addi sp, sp, 32 ; RV32IFD-NEXT: ret ; ; RV64IFD-LABEL: test_ceil_si64: @@ -367,44 +384,47 @@ define i64 @test_ceil_si64(double %x) nounwind { ; RV32IZFINXZDINX-NEXT: addi sp, sp, -32 ; RV32IZFINXZDINX-NEXT: sw ra, 28(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s0, 24(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s2, 20(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s3, 16(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s1, 20(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call ceil -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw s2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw s3, 12(sp) +; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) +; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) +; RV32IZFINXZDINX-NEXT: lw s0, 0(sp) +; RV32IZFINXZDINX-NEXT: lw s1, 4(sp) ; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI5_0) ; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI5_0+4)(a2) ; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI5_0)(a2) -; RV32IZFINXZDINX-NEXT: fle.d s0, a2, s2 +; RV32IZFINXZDINX-NEXT: fle.d s2, a2, s0 +; RV32IZFINXZDINX-NEXT: neg s3, s2 ; RV32IZFINXZDINX-NEXT: call __fixdfdi +; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI5_1) +; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI5_1+4)(a2) +; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI5_1)(a2) +; RV32IZFINXZDINX-NEXT: and a0, s3, a0 +; RV32IZFINXZDINX-NEXT: flt.d a3, a2, s0 +; RV32IZFINXZDINX-NEXT: neg a2, a3 +; RV32IZFINXZDINX-NEXT: or a0, a2, a0 +; RV32IZFINXZDINX-NEXT: feq.d a2, s0, s0 +; RV32IZFINXZDINX-NEXT: neg a2, a2 +; RV32IZFINXZDINX-NEXT: lui a5, 524288 +; RV32IZFINXZDINX-NEXT: li a6, 1 ; RV32IZFINXZDINX-NEXT: lui a4, 524288 -; RV32IZFINXZDINX-NEXT: lui a2, 524288 -; RV32IZFINXZDINX-NEXT: beqz s0, .LBB5_2 +; RV32IZFINXZDINX-NEXT: bne s2, a6, .LBB5_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: -; RV32IZFINXZDINX-NEXT: mv a2, a1 +; RV32IZFINXZDINX-NEXT: mv a4, a1 ; RV32IZFINXZDINX-NEXT: .LBB5_2: -; RV32IZFINXZDINX-NEXT: lui a1, %hi(.LCPI5_1) -; RV32IZFINXZDINX-NEXT: lw a6, %lo(.LCPI5_1)(a1) -; RV32IZFINXZDINX-NEXT: lw a7, %lo(.LCPI5_1+4)(a1) -; RV32IZFINXZDINX-NEXT: flt.d a3, a6, s2 +; RV32IZFINXZDINX-NEXT: and a0, a2, a0 ; RV32IZFINXZDINX-NEXT: beqz a3, .LBB5_4 ; RV32IZFINXZDINX-NEXT: # %bb.3: -; RV32IZFINXZDINX-NEXT: addi a2, a4, -1 +; RV32IZFINXZDINX-NEXT: addi a4, a5, -1 ; RV32IZFINXZDINX-NEXT: .LBB5_4: -; RV32IZFINXZDINX-NEXT: feq.d a1, s2, s2 -; RV32IZFINXZDINX-NEXT: neg a4, a1 -; RV32IZFINXZDINX-NEXT: and a1, a4, a2 -; RV32IZFINXZDINX-NEXT: neg a2, s0 -; RV32IZFINXZDINX-NEXT: and a0, a2, a0 -; RV32IZFINXZDINX-NEXT: neg a2, a3 -; RV32IZFINXZDINX-NEXT: or a0, a2, a0 -; RV32IZFINXZDINX-NEXT: and a0, a4, a0 +; RV32IZFINXZDINX-NEXT: and a1, a2, a4 ; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s2, 20(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s3, 16(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s3, 12(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: addi sp, sp, 32 ; RV32IZFINXZDINX-NEXT: ret ; @@ -474,7 +494,8 @@ define i64 @test_ceil_ui64(double %x) nounwind { ; RV32IFD-NEXT: neg s0, a0 ; RV32IFD-NEXT: fcvt.d.w fa5, zero ; RV32IFD-NEXT: fle.d a0, fa5, fa0 -; RV32IFD-NEXT: neg s1, a0 +; RV32IFD-NEXT: xori a0, a0, 1 +; RV32IFD-NEXT: addi s1, a0, -1 ; RV32IFD-NEXT: call __fixunsdfdi ; RV32IFD-NEXT: and a0, s1, a0 ; RV32IFD-NEXT: or a0, s0, a0 @@ -501,29 +522,28 @@ define i64 @test_ceil_ui64(double %x) nounwind { ; RV32IZFINXZDINX-NEXT: sw ra, 28(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s0, 24(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s1, 20(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call ceil ; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) ; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: lw s0, 8(sp) ; RV32IZFINXZDINX-NEXT: lw s1, 12(sp) +; RV32IZFINXZDINX-NEXT: call __fixunsdfdi ; RV32IZFINXZDINX-NEXT: fcvt.d.w a2, zero ; RV32IZFINXZDINX-NEXT: fle.d a2, a2, s0 -; RV32IZFINXZDINX-NEXT: neg s2, a2 -; RV32IZFINXZDINX-NEXT: call __fixunsdfdi -; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI7_0) -; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI7_0+4)(a2) -; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI7_0)(a2) -; RV32IZFINXZDINX-NEXT: and a0, s2, a0 -; RV32IZFINXZDINX-NEXT: flt.d a2, a2, s0 -; RV32IZFINXZDINX-NEXT: neg a2, a2 -; RV32IZFINXZDINX-NEXT: or a0, a2, a0 -; RV32IZFINXZDINX-NEXT: and a1, s2, a1 -; RV32IZFINXZDINX-NEXT: or a1, a2, a1 +; RV32IZFINXZDINX-NEXT: lui a3, %hi(.LCPI7_0) +; RV32IZFINXZDINX-NEXT: lw a4, %lo(.LCPI7_0)(a3) +; RV32IZFINXZDINX-NEXT: lw a5, %lo(.LCPI7_0+4)(a3) +; RV32IZFINXZDINX-NEXT: xori a2, a2, 1 +; RV32IZFINXZDINX-NEXT: addi a2, a2, -1 +; RV32IZFINXZDINX-NEXT: and a0, a2, a0 +; RV32IZFINXZDINX-NEXT: flt.d a3, a4, s0 +; RV32IZFINXZDINX-NEXT: neg a3, a3 +; RV32IZFINXZDINX-NEXT: or a0, a3, a0 +; RV32IZFINXZDINX-NEXT: and a1, a2, a1 +; RV32IZFINXZDINX-NEXT: or a1, a3, a1 ; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s2, 16(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: addi sp, sp, 32 ; RV32IZFINXZDINX-NEXT: ret ; @@ -582,41 +602,48 @@ define signext i32 @test_trunc_si32(double %x) { define i64 @test_trunc_si64(double %x) nounwind { ; RV32IFD-LABEL: test_trunc_si64: ; RV32IFD: # %bb.0: -; RV32IFD-NEXT: addi sp, sp, -16 -; RV32IFD-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IFD-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: addi sp, sp, -32 +; RV32IFD-NEXT: sw ra, 28(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s0, 24(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s1, 20(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32IFD-NEXT: fsd fs0, 0(sp) # 8-byte Folded Spill ; RV32IFD-NEXT: call trunc ; RV32IFD-NEXT: lui a0, %hi(.LCPI9_0) ; RV32IFD-NEXT: fld fa5, %lo(.LCPI9_0)(a0) +; RV32IFD-NEXT: lui a0, %hi(.LCPI9_1) +; RV32IFD-NEXT: fld fa4, %lo(.LCPI9_1)(a0) ; RV32IFD-NEXT: fmv.d fs0, fa0 -; RV32IFD-NEXT: fle.d s0, fa5, fa0 +; RV32IFD-NEXT: flt.d s0, fa5, fa0 +; RV32IFD-NEXT: neg s1, s0 +; RV32IFD-NEXT: fle.d s2, fa4, fa0 +; RV32IFD-NEXT: neg s3, s2 ; RV32IFD-NEXT: call __fixdfdi +; RV32IFD-NEXT: and a0, s3, a0 +; RV32IFD-NEXT: or a0, s1, a0 +; RV32IFD-NEXT: feq.d a2, fs0, fs0 +; RV32IFD-NEXT: neg a2, a2 ; RV32IFD-NEXT: lui a4, 524288 -; RV32IFD-NEXT: lui a2, 524288 -; RV32IFD-NEXT: beqz s0, .LBB9_2 +; RV32IFD-NEXT: li a5, 1 +; RV32IFD-NEXT: lui a3, 524288 +; RV32IFD-NEXT: bne s2, a5, .LBB9_2 ; RV32IFD-NEXT: # %bb.1: -; RV32IFD-NEXT: mv a2, a1 +; RV32IFD-NEXT: mv a3, a1 ; RV32IFD-NEXT: .LBB9_2: -; RV32IFD-NEXT: lui a1, %hi(.LCPI9_1) -; RV32IFD-NEXT: fld fa5, %lo(.LCPI9_1)(a1) -; RV32IFD-NEXT: flt.d a3, fa5, fs0 -; RV32IFD-NEXT: beqz a3, .LBB9_4 +; RV32IFD-NEXT: and a0, a2, a0 +; RV32IFD-NEXT: beqz s0, .LBB9_4 ; RV32IFD-NEXT: # %bb.3: -; RV32IFD-NEXT: addi a2, a4, -1 +; RV32IFD-NEXT: addi a3, a4, -1 ; RV32IFD-NEXT: .LBB9_4: -; RV32IFD-NEXT: feq.d a1, fs0, fs0 -; RV32IFD-NEXT: neg a4, a1 -; RV32IFD-NEXT: and a1, a4, a2 -; RV32IFD-NEXT: neg a2, a3 -; RV32IFD-NEXT: neg a3, s0 -; RV32IFD-NEXT: and a0, a3, a0 -; RV32IFD-NEXT: or a0, a2, a0 -; RV32IFD-NEXT: and a0, a4, a0 -; RV32IFD-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IFD-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: and a1, a2, a3 +; RV32IFD-NEXT: lw ra, 28(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s0, 24(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s3, 12(sp) # 4-byte Folded Reload ; RV32IFD-NEXT: fld fs0, 0(sp) # 8-byte Folded Reload -; RV32IFD-NEXT: addi sp, sp, 16 +; RV32IFD-NEXT: addi sp, sp, 32 ; RV32IFD-NEXT: ret ; ; RV64IFD-LABEL: test_trunc_si64: @@ -633,44 +660,47 @@ define i64 @test_trunc_si64(double %x) nounwind { ; RV32IZFINXZDINX-NEXT: addi sp, sp, -32 ; RV32IZFINXZDINX-NEXT: sw ra, 28(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s0, 24(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s2, 20(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s3, 16(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s1, 20(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call trunc -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw s2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw s3, 12(sp) +; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) +; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) +; RV32IZFINXZDINX-NEXT: lw s0, 0(sp) +; RV32IZFINXZDINX-NEXT: lw s1, 4(sp) ; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI9_0) ; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI9_0+4)(a2) ; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI9_0)(a2) -; RV32IZFINXZDINX-NEXT: fle.d s0, a2, s2 +; RV32IZFINXZDINX-NEXT: fle.d s2, a2, s0 +; RV32IZFINXZDINX-NEXT: neg s3, s2 ; RV32IZFINXZDINX-NEXT: call __fixdfdi +; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI9_1) +; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI9_1+4)(a2) +; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI9_1)(a2) +; RV32IZFINXZDINX-NEXT: and a0, s3, a0 +; RV32IZFINXZDINX-NEXT: flt.d a3, a2, s0 +; RV32IZFINXZDINX-NEXT: neg a2, a3 +; RV32IZFINXZDINX-NEXT: or a0, a2, a0 +; RV32IZFINXZDINX-NEXT: feq.d a2, s0, s0 +; RV32IZFINXZDINX-NEXT: neg a2, a2 +; RV32IZFINXZDINX-NEXT: lui a5, 524288 +; RV32IZFINXZDINX-NEXT: li a6, 1 ; RV32IZFINXZDINX-NEXT: lui a4, 524288 -; RV32IZFINXZDINX-NEXT: lui a2, 524288 -; RV32IZFINXZDINX-NEXT: beqz s0, .LBB9_2 +; RV32IZFINXZDINX-NEXT: bne s2, a6, .LBB9_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: -; RV32IZFINXZDINX-NEXT: mv a2, a1 +; RV32IZFINXZDINX-NEXT: mv a4, a1 ; RV32IZFINXZDINX-NEXT: .LBB9_2: -; RV32IZFINXZDINX-NEXT: lui a1, %hi(.LCPI9_1) -; RV32IZFINXZDINX-NEXT: lw a6, %lo(.LCPI9_1)(a1) -; RV32IZFINXZDINX-NEXT: lw a7, %lo(.LCPI9_1+4)(a1) -; RV32IZFINXZDINX-NEXT: flt.d a3, a6, s2 +; RV32IZFINXZDINX-NEXT: and a0, a2, a0 ; RV32IZFINXZDINX-NEXT: beqz a3, .LBB9_4 ; RV32IZFINXZDINX-NEXT: # %bb.3: -; RV32IZFINXZDINX-NEXT: addi a2, a4, -1 +; RV32IZFINXZDINX-NEXT: addi a4, a5, -1 ; RV32IZFINXZDINX-NEXT: .LBB9_4: -; RV32IZFINXZDINX-NEXT: feq.d a1, s2, s2 -; RV32IZFINXZDINX-NEXT: neg a4, a1 -; RV32IZFINXZDINX-NEXT: and a1, a4, a2 -; RV32IZFINXZDINX-NEXT: neg a2, s0 -; RV32IZFINXZDINX-NEXT: and a0, a2, a0 -; RV32IZFINXZDINX-NEXT: neg a2, a3 -; RV32IZFINXZDINX-NEXT: or a0, a2, a0 -; RV32IZFINXZDINX-NEXT: and a0, a4, a0 +; RV32IZFINXZDINX-NEXT: and a1, a2, a4 ; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s2, 20(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s3, 16(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s3, 12(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: addi sp, sp, 32 ; RV32IZFINXZDINX-NEXT: ret ; @@ -740,7 +770,8 @@ define i64 @test_trunc_ui64(double %x) nounwind { ; RV32IFD-NEXT: neg s0, a0 ; RV32IFD-NEXT: fcvt.d.w fa5, zero ; RV32IFD-NEXT: fle.d a0, fa5, fa0 -; RV32IFD-NEXT: neg s1, a0 +; RV32IFD-NEXT: xori a0, a0, 1 +; RV32IFD-NEXT: addi s1, a0, -1 ; RV32IFD-NEXT: call __fixunsdfdi ; RV32IFD-NEXT: and a0, s1, a0 ; RV32IFD-NEXT: or a0, s0, a0 @@ -767,29 +798,28 @@ define i64 @test_trunc_ui64(double %x) nounwind { ; RV32IZFINXZDINX-NEXT: sw ra, 28(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s0, 24(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s1, 20(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call trunc ; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) ; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: lw s0, 8(sp) ; RV32IZFINXZDINX-NEXT: lw s1, 12(sp) +; RV32IZFINXZDINX-NEXT: call __fixunsdfdi ; RV32IZFINXZDINX-NEXT: fcvt.d.w a2, zero ; RV32IZFINXZDINX-NEXT: fle.d a2, a2, s0 -; RV32IZFINXZDINX-NEXT: neg s2, a2 -; RV32IZFINXZDINX-NEXT: call __fixunsdfdi -; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI11_0) -; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI11_0+4)(a2) -; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI11_0)(a2) -; RV32IZFINXZDINX-NEXT: and a0, s2, a0 -; RV32IZFINXZDINX-NEXT: flt.d a2, a2, s0 -; RV32IZFINXZDINX-NEXT: neg a2, a2 -; RV32IZFINXZDINX-NEXT: or a0, a2, a0 -; RV32IZFINXZDINX-NEXT: and a1, s2, a1 -; RV32IZFINXZDINX-NEXT: or a1, a2, a1 +; RV32IZFINXZDINX-NEXT: lui a3, %hi(.LCPI11_0) +; RV32IZFINXZDINX-NEXT: lw a4, %lo(.LCPI11_0)(a3) +; RV32IZFINXZDINX-NEXT: lw a5, %lo(.LCPI11_0+4)(a3) +; RV32IZFINXZDINX-NEXT: xori a2, a2, 1 +; RV32IZFINXZDINX-NEXT: addi a2, a2, -1 +; RV32IZFINXZDINX-NEXT: and a0, a2, a0 +; RV32IZFINXZDINX-NEXT: flt.d a3, a4, s0 +; RV32IZFINXZDINX-NEXT: neg a3, a3 +; RV32IZFINXZDINX-NEXT: or a0, a3, a0 +; RV32IZFINXZDINX-NEXT: and a1, a2, a1 +; RV32IZFINXZDINX-NEXT: or a1, a3, a1 ; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s2, 16(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: addi sp, sp, 32 ; RV32IZFINXZDINX-NEXT: ret ; @@ -848,41 +878,48 @@ define signext i32 @test_round_si32(double %x) { define i64 @test_round_si64(double %x) nounwind { ; RV32IFD-LABEL: test_round_si64: ; RV32IFD: # %bb.0: -; RV32IFD-NEXT: addi sp, sp, -16 -; RV32IFD-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IFD-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: addi sp, sp, -32 +; RV32IFD-NEXT: sw ra, 28(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s0, 24(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s1, 20(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32IFD-NEXT: fsd fs0, 0(sp) # 8-byte Folded Spill ; RV32IFD-NEXT: call round ; RV32IFD-NEXT: lui a0, %hi(.LCPI13_0) ; RV32IFD-NEXT: fld fa5, %lo(.LCPI13_0)(a0) +; RV32IFD-NEXT: lui a0, %hi(.LCPI13_1) +; RV32IFD-NEXT: fld fa4, %lo(.LCPI13_1)(a0) ; RV32IFD-NEXT: fmv.d fs0, fa0 -; RV32IFD-NEXT: fle.d s0, fa5, fa0 +; RV32IFD-NEXT: flt.d s0, fa5, fa0 +; RV32IFD-NEXT: neg s1, s0 +; RV32IFD-NEXT: fle.d s2, fa4, fa0 +; RV32IFD-NEXT: neg s3, s2 ; RV32IFD-NEXT: call __fixdfdi +; RV32IFD-NEXT: and a0, s3, a0 +; RV32IFD-NEXT: or a0, s1, a0 +; RV32IFD-NEXT: feq.d a2, fs0, fs0 +; RV32IFD-NEXT: neg a2, a2 ; RV32IFD-NEXT: lui a4, 524288 -; RV32IFD-NEXT: lui a2, 524288 -; RV32IFD-NEXT: beqz s0, .LBB13_2 +; RV32IFD-NEXT: li a5, 1 +; RV32IFD-NEXT: lui a3, 524288 +; RV32IFD-NEXT: bne s2, a5, .LBB13_2 ; RV32IFD-NEXT: # %bb.1: -; RV32IFD-NEXT: mv a2, a1 +; RV32IFD-NEXT: mv a3, a1 ; RV32IFD-NEXT: .LBB13_2: -; RV32IFD-NEXT: lui a1, %hi(.LCPI13_1) -; RV32IFD-NEXT: fld fa5, %lo(.LCPI13_1)(a1) -; RV32IFD-NEXT: flt.d a3, fa5, fs0 -; RV32IFD-NEXT: beqz a3, .LBB13_4 +; RV32IFD-NEXT: and a0, a2, a0 +; RV32IFD-NEXT: beqz s0, .LBB13_4 ; RV32IFD-NEXT: # %bb.3: -; RV32IFD-NEXT: addi a2, a4, -1 +; RV32IFD-NEXT: addi a3, a4, -1 ; RV32IFD-NEXT: .LBB13_4: -; RV32IFD-NEXT: feq.d a1, fs0, fs0 -; RV32IFD-NEXT: neg a4, a1 -; RV32IFD-NEXT: and a1, a4, a2 -; RV32IFD-NEXT: neg a2, a3 -; RV32IFD-NEXT: neg a3, s0 -; RV32IFD-NEXT: and a0, a3, a0 -; RV32IFD-NEXT: or a0, a2, a0 -; RV32IFD-NEXT: and a0, a4, a0 -; RV32IFD-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IFD-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: and a1, a2, a3 +; RV32IFD-NEXT: lw ra, 28(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s0, 24(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s3, 12(sp) # 4-byte Folded Reload ; RV32IFD-NEXT: fld fs0, 0(sp) # 8-byte Folded Reload -; RV32IFD-NEXT: addi sp, sp, 16 +; RV32IFD-NEXT: addi sp, sp, 32 ; RV32IFD-NEXT: ret ; ; RV64IFD-LABEL: test_round_si64: @@ -899,44 +936,47 @@ define i64 @test_round_si64(double %x) nounwind { ; RV32IZFINXZDINX-NEXT: addi sp, sp, -32 ; RV32IZFINXZDINX-NEXT: sw ra, 28(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s0, 24(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s2, 20(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s3, 16(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s1, 20(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call round -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw s2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw s3, 12(sp) +; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) +; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) +; RV32IZFINXZDINX-NEXT: lw s0, 0(sp) +; RV32IZFINXZDINX-NEXT: lw s1, 4(sp) ; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI13_0) ; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI13_0+4)(a2) ; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI13_0)(a2) -; RV32IZFINXZDINX-NEXT: fle.d s0, a2, s2 +; RV32IZFINXZDINX-NEXT: fle.d s2, a2, s0 +; RV32IZFINXZDINX-NEXT: neg s3, s2 ; RV32IZFINXZDINX-NEXT: call __fixdfdi +; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI13_1) +; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI13_1+4)(a2) +; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI13_1)(a2) +; RV32IZFINXZDINX-NEXT: and a0, s3, a0 +; RV32IZFINXZDINX-NEXT: flt.d a3, a2, s0 +; RV32IZFINXZDINX-NEXT: neg a2, a3 +; RV32IZFINXZDINX-NEXT: or a0, a2, a0 +; RV32IZFINXZDINX-NEXT: feq.d a2, s0, s0 +; RV32IZFINXZDINX-NEXT: neg a2, a2 +; RV32IZFINXZDINX-NEXT: lui a5, 524288 +; RV32IZFINXZDINX-NEXT: li a6, 1 ; RV32IZFINXZDINX-NEXT: lui a4, 524288 -; RV32IZFINXZDINX-NEXT: lui a2, 524288 -; RV32IZFINXZDINX-NEXT: beqz s0, .LBB13_2 +; RV32IZFINXZDINX-NEXT: bne s2, a6, .LBB13_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: -; RV32IZFINXZDINX-NEXT: mv a2, a1 +; RV32IZFINXZDINX-NEXT: mv a4, a1 ; RV32IZFINXZDINX-NEXT: .LBB13_2: -; RV32IZFINXZDINX-NEXT: lui a1, %hi(.LCPI13_1) -; RV32IZFINXZDINX-NEXT: lw a6, %lo(.LCPI13_1)(a1) -; RV32IZFINXZDINX-NEXT: lw a7, %lo(.LCPI13_1+4)(a1) -; RV32IZFINXZDINX-NEXT: flt.d a3, a6, s2 +; RV32IZFINXZDINX-NEXT: and a0, a2, a0 ; RV32IZFINXZDINX-NEXT: beqz a3, .LBB13_4 ; RV32IZFINXZDINX-NEXT: # %bb.3: -; RV32IZFINXZDINX-NEXT: addi a2, a4, -1 +; RV32IZFINXZDINX-NEXT: addi a4, a5, -1 ; RV32IZFINXZDINX-NEXT: .LBB13_4: -; RV32IZFINXZDINX-NEXT: feq.d a1, s2, s2 -; RV32IZFINXZDINX-NEXT: neg a4, a1 -; RV32IZFINXZDINX-NEXT: and a1, a4, a2 -; RV32IZFINXZDINX-NEXT: neg a2, s0 -; RV32IZFINXZDINX-NEXT: and a0, a2, a0 -; RV32IZFINXZDINX-NEXT: neg a2, a3 -; RV32IZFINXZDINX-NEXT: or a0, a2, a0 -; RV32IZFINXZDINX-NEXT: and a0, a4, a0 +; RV32IZFINXZDINX-NEXT: and a1, a2, a4 ; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s2, 20(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s3, 16(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s3, 12(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: addi sp, sp, 32 ; RV32IZFINXZDINX-NEXT: ret ; @@ -1006,7 +1046,8 @@ define i64 @test_round_ui64(double %x) nounwind { ; RV32IFD-NEXT: neg s0, a0 ; RV32IFD-NEXT: fcvt.d.w fa5, zero ; RV32IFD-NEXT: fle.d a0, fa5, fa0 -; RV32IFD-NEXT: neg s1, a0 +; RV32IFD-NEXT: xori a0, a0, 1 +; RV32IFD-NEXT: addi s1, a0, -1 ; RV32IFD-NEXT: call __fixunsdfdi ; RV32IFD-NEXT: and a0, s1, a0 ; RV32IFD-NEXT: or a0, s0, a0 @@ -1033,29 +1074,28 @@ define i64 @test_round_ui64(double %x) nounwind { ; RV32IZFINXZDINX-NEXT: sw ra, 28(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s0, 24(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s1, 20(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call round ; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) ; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: lw s0, 8(sp) ; RV32IZFINXZDINX-NEXT: lw s1, 12(sp) +; RV32IZFINXZDINX-NEXT: call __fixunsdfdi ; RV32IZFINXZDINX-NEXT: fcvt.d.w a2, zero ; RV32IZFINXZDINX-NEXT: fle.d a2, a2, s0 -; RV32IZFINXZDINX-NEXT: neg s2, a2 -; RV32IZFINXZDINX-NEXT: call __fixunsdfdi -; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI15_0) -; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI15_0+4)(a2) -; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI15_0)(a2) -; RV32IZFINXZDINX-NEXT: and a0, s2, a0 -; RV32IZFINXZDINX-NEXT: flt.d a2, a2, s0 -; RV32IZFINXZDINX-NEXT: neg a2, a2 -; RV32IZFINXZDINX-NEXT: or a0, a2, a0 -; RV32IZFINXZDINX-NEXT: and a1, s2, a1 -; RV32IZFINXZDINX-NEXT: or a1, a2, a1 +; RV32IZFINXZDINX-NEXT: lui a3, %hi(.LCPI15_0) +; RV32IZFINXZDINX-NEXT: lw a4, %lo(.LCPI15_0)(a3) +; RV32IZFINXZDINX-NEXT: lw a5, %lo(.LCPI15_0+4)(a3) +; RV32IZFINXZDINX-NEXT: xori a2, a2, 1 +; RV32IZFINXZDINX-NEXT: addi a2, a2, -1 +; RV32IZFINXZDINX-NEXT: and a0, a2, a0 +; RV32IZFINXZDINX-NEXT: flt.d a3, a4, s0 +; RV32IZFINXZDINX-NEXT: neg a3, a3 +; RV32IZFINXZDINX-NEXT: or a0, a3, a0 +; RV32IZFINXZDINX-NEXT: and a1, a2, a1 +; RV32IZFINXZDINX-NEXT: or a1, a3, a1 ; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s2, 16(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: addi sp, sp, 32 ; RV32IZFINXZDINX-NEXT: ret ; @@ -1114,41 +1154,48 @@ define signext i32 @test_roundeven_si32(double %x) { define i64 @test_roundeven_si64(double %x) nounwind { ; RV32IFD-LABEL: test_roundeven_si64: ; RV32IFD: # %bb.0: -; RV32IFD-NEXT: addi sp, sp, -16 -; RV32IFD-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IFD-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: addi sp, sp, -32 +; RV32IFD-NEXT: sw ra, 28(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s0, 24(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s1, 20(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32IFD-NEXT: fsd fs0, 0(sp) # 8-byte Folded Spill ; RV32IFD-NEXT: call roundeven ; RV32IFD-NEXT: lui a0, %hi(.LCPI17_0) ; RV32IFD-NEXT: fld fa5, %lo(.LCPI17_0)(a0) +; RV32IFD-NEXT: lui a0, %hi(.LCPI17_1) +; RV32IFD-NEXT: fld fa4, %lo(.LCPI17_1)(a0) ; RV32IFD-NEXT: fmv.d fs0, fa0 -; RV32IFD-NEXT: fle.d s0, fa5, fa0 +; RV32IFD-NEXT: flt.d s0, fa5, fa0 +; RV32IFD-NEXT: neg s1, s0 +; RV32IFD-NEXT: fle.d s2, fa4, fa0 +; RV32IFD-NEXT: neg s3, s2 ; RV32IFD-NEXT: call __fixdfdi +; RV32IFD-NEXT: and a0, s3, a0 +; RV32IFD-NEXT: or a0, s1, a0 +; RV32IFD-NEXT: feq.d a2, fs0, fs0 +; RV32IFD-NEXT: neg a2, a2 ; RV32IFD-NEXT: lui a4, 524288 -; RV32IFD-NEXT: lui a2, 524288 -; RV32IFD-NEXT: beqz s0, .LBB17_2 +; RV32IFD-NEXT: li a5, 1 +; RV32IFD-NEXT: lui a3, 524288 +; RV32IFD-NEXT: bne s2, a5, .LBB17_2 ; RV32IFD-NEXT: # %bb.1: -; RV32IFD-NEXT: mv a2, a1 +; RV32IFD-NEXT: mv a3, a1 ; RV32IFD-NEXT: .LBB17_2: -; RV32IFD-NEXT: lui a1, %hi(.LCPI17_1) -; RV32IFD-NEXT: fld fa5, %lo(.LCPI17_1)(a1) -; RV32IFD-NEXT: flt.d a3, fa5, fs0 -; RV32IFD-NEXT: beqz a3, .LBB17_4 +; RV32IFD-NEXT: and a0, a2, a0 +; RV32IFD-NEXT: beqz s0, .LBB17_4 ; RV32IFD-NEXT: # %bb.3: -; RV32IFD-NEXT: addi a2, a4, -1 +; RV32IFD-NEXT: addi a3, a4, -1 ; RV32IFD-NEXT: .LBB17_4: -; RV32IFD-NEXT: feq.d a1, fs0, fs0 -; RV32IFD-NEXT: neg a4, a1 -; RV32IFD-NEXT: and a1, a4, a2 -; RV32IFD-NEXT: neg a2, a3 -; RV32IFD-NEXT: neg a3, s0 -; RV32IFD-NEXT: and a0, a3, a0 -; RV32IFD-NEXT: or a0, a2, a0 -; RV32IFD-NEXT: and a0, a4, a0 -; RV32IFD-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IFD-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: and a1, a2, a3 +; RV32IFD-NEXT: lw ra, 28(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s0, 24(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s3, 12(sp) # 4-byte Folded Reload ; RV32IFD-NEXT: fld fs0, 0(sp) # 8-byte Folded Reload -; RV32IFD-NEXT: addi sp, sp, 16 +; RV32IFD-NEXT: addi sp, sp, 32 ; RV32IFD-NEXT: ret ; ; RV64IFD-LABEL: test_roundeven_si64: @@ -1165,44 +1212,47 @@ define i64 @test_roundeven_si64(double %x) nounwind { ; RV32IZFINXZDINX-NEXT: addi sp, sp, -32 ; RV32IZFINXZDINX-NEXT: sw ra, 28(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s0, 24(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s2, 20(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s3, 16(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s1, 20(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call roundeven -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw s2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw s3, 12(sp) +; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) +; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) +; RV32IZFINXZDINX-NEXT: lw s0, 0(sp) +; RV32IZFINXZDINX-NEXT: lw s1, 4(sp) ; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI17_0) ; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI17_0+4)(a2) ; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI17_0)(a2) -; RV32IZFINXZDINX-NEXT: fle.d s0, a2, s2 +; RV32IZFINXZDINX-NEXT: fle.d s2, a2, s0 +; RV32IZFINXZDINX-NEXT: neg s3, s2 ; RV32IZFINXZDINX-NEXT: call __fixdfdi +; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI17_1) +; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI17_1+4)(a2) +; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI17_1)(a2) +; RV32IZFINXZDINX-NEXT: and a0, s3, a0 +; RV32IZFINXZDINX-NEXT: flt.d a3, a2, s0 +; RV32IZFINXZDINX-NEXT: neg a2, a3 +; RV32IZFINXZDINX-NEXT: or a0, a2, a0 +; RV32IZFINXZDINX-NEXT: feq.d a2, s0, s0 +; RV32IZFINXZDINX-NEXT: neg a2, a2 +; RV32IZFINXZDINX-NEXT: lui a5, 524288 +; RV32IZFINXZDINX-NEXT: li a6, 1 ; RV32IZFINXZDINX-NEXT: lui a4, 524288 -; RV32IZFINXZDINX-NEXT: lui a2, 524288 -; RV32IZFINXZDINX-NEXT: beqz s0, .LBB17_2 +; RV32IZFINXZDINX-NEXT: bne s2, a6, .LBB17_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: -; RV32IZFINXZDINX-NEXT: mv a2, a1 +; RV32IZFINXZDINX-NEXT: mv a4, a1 ; RV32IZFINXZDINX-NEXT: .LBB17_2: -; RV32IZFINXZDINX-NEXT: lui a1, %hi(.LCPI17_1) -; RV32IZFINXZDINX-NEXT: lw a6, %lo(.LCPI17_1)(a1) -; RV32IZFINXZDINX-NEXT: lw a7, %lo(.LCPI17_1+4)(a1) -; RV32IZFINXZDINX-NEXT: flt.d a3, a6, s2 +; RV32IZFINXZDINX-NEXT: and a0, a2, a0 ; RV32IZFINXZDINX-NEXT: beqz a3, .LBB17_4 ; RV32IZFINXZDINX-NEXT: # %bb.3: -; RV32IZFINXZDINX-NEXT: addi a2, a4, -1 +; RV32IZFINXZDINX-NEXT: addi a4, a5, -1 ; RV32IZFINXZDINX-NEXT: .LBB17_4: -; RV32IZFINXZDINX-NEXT: feq.d a1, s2, s2 -; RV32IZFINXZDINX-NEXT: neg a4, a1 -; RV32IZFINXZDINX-NEXT: and a1, a4, a2 -; RV32IZFINXZDINX-NEXT: neg a2, s0 -; RV32IZFINXZDINX-NEXT: and a0, a2, a0 -; RV32IZFINXZDINX-NEXT: neg a2, a3 -; RV32IZFINXZDINX-NEXT: or a0, a2, a0 -; RV32IZFINXZDINX-NEXT: and a0, a4, a0 +; RV32IZFINXZDINX-NEXT: and a1, a2, a4 ; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s2, 20(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s3, 16(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s3, 12(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: addi sp, sp, 32 ; RV32IZFINXZDINX-NEXT: ret ; @@ -1272,7 +1322,8 @@ define i64 @test_roundeven_ui64(double %x) nounwind { ; RV32IFD-NEXT: neg s0, a0 ; RV32IFD-NEXT: fcvt.d.w fa5, zero ; RV32IFD-NEXT: fle.d a0, fa5, fa0 -; RV32IFD-NEXT: neg s1, a0 +; RV32IFD-NEXT: xori a0, a0, 1 +; RV32IFD-NEXT: addi s1, a0, -1 ; RV32IFD-NEXT: call __fixunsdfdi ; RV32IFD-NEXT: and a0, s1, a0 ; RV32IFD-NEXT: or a0, s0, a0 @@ -1299,29 +1350,28 @@ define i64 @test_roundeven_ui64(double %x) nounwind { ; RV32IZFINXZDINX-NEXT: sw ra, 28(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s0, 24(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s1, 20(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call roundeven ; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) ; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: lw s0, 8(sp) ; RV32IZFINXZDINX-NEXT: lw s1, 12(sp) +; RV32IZFINXZDINX-NEXT: call __fixunsdfdi ; RV32IZFINXZDINX-NEXT: fcvt.d.w a2, zero ; RV32IZFINXZDINX-NEXT: fle.d a2, a2, s0 -; RV32IZFINXZDINX-NEXT: neg s2, a2 -; RV32IZFINXZDINX-NEXT: call __fixunsdfdi -; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI19_0) -; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI19_0+4)(a2) -; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI19_0)(a2) -; RV32IZFINXZDINX-NEXT: and a0, s2, a0 -; RV32IZFINXZDINX-NEXT: flt.d a2, a2, s0 -; RV32IZFINXZDINX-NEXT: neg a2, a2 -; RV32IZFINXZDINX-NEXT: or a0, a2, a0 -; RV32IZFINXZDINX-NEXT: and a1, s2, a1 -; RV32IZFINXZDINX-NEXT: or a1, a2, a1 +; RV32IZFINXZDINX-NEXT: lui a3, %hi(.LCPI19_0) +; RV32IZFINXZDINX-NEXT: lw a4, %lo(.LCPI19_0)(a3) +; RV32IZFINXZDINX-NEXT: lw a5, %lo(.LCPI19_0+4)(a3) +; RV32IZFINXZDINX-NEXT: xori a2, a2, 1 +; RV32IZFINXZDINX-NEXT: addi a2, a2, -1 +; RV32IZFINXZDINX-NEXT: and a0, a2, a0 +; RV32IZFINXZDINX-NEXT: flt.d a3, a4, s0 +; RV32IZFINXZDINX-NEXT: neg a3, a3 +; RV32IZFINXZDINX-NEXT: or a0, a3, a0 +; RV32IZFINXZDINX-NEXT: and a1, a2, a1 +; RV32IZFINXZDINX-NEXT: or a1, a3, a1 ; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s2, 16(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: addi sp, sp, 32 ; RV32IZFINXZDINX-NEXT: ret ; @@ -1380,41 +1430,48 @@ define signext i32 @test_rint_si32(double %x) { define i64 @test_rint_si64(double %x) nounwind { ; RV32IFD-LABEL: test_rint_si64: ; RV32IFD: # %bb.0: -; RV32IFD-NEXT: addi sp, sp, -16 -; RV32IFD-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IFD-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: addi sp, sp, -32 +; RV32IFD-NEXT: sw ra, 28(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s0, 24(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s1, 20(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32IFD-NEXT: fsd fs0, 0(sp) # 8-byte Folded Spill ; RV32IFD-NEXT: call rint ; RV32IFD-NEXT: lui a0, %hi(.LCPI21_0) ; RV32IFD-NEXT: fld fa5, %lo(.LCPI21_0)(a0) +; RV32IFD-NEXT: lui a0, %hi(.LCPI21_1) +; RV32IFD-NEXT: fld fa4, %lo(.LCPI21_1)(a0) ; RV32IFD-NEXT: fmv.d fs0, fa0 -; RV32IFD-NEXT: fle.d s0, fa5, fa0 +; RV32IFD-NEXT: flt.d s0, fa5, fa0 +; RV32IFD-NEXT: neg s1, s0 +; RV32IFD-NEXT: fle.d s2, fa4, fa0 +; RV32IFD-NEXT: neg s3, s2 ; RV32IFD-NEXT: call __fixdfdi +; RV32IFD-NEXT: and a0, s3, a0 +; RV32IFD-NEXT: or a0, s1, a0 +; RV32IFD-NEXT: feq.d a2, fs0, fs0 +; RV32IFD-NEXT: neg a2, a2 ; RV32IFD-NEXT: lui a4, 524288 -; RV32IFD-NEXT: lui a2, 524288 -; RV32IFD-NEXT: beqz s0, .LBB21_2 +; RV32IFD-NEXT: li a5, 1 +; RV32IFD-NEXT: lui a3, 524288 +; RV32IFD-NEXT: bne s2, a5, .LBB21_2 ; RV32IFD-NEXT: # %bb.1: -; RV32IFD-NEXT: mv a2, a1 +; RV32IFD-NEXT: mv a3, a1 ; RV32IFD-NEXT: .LBB21_2: -; RV32IFD-NEXT: lui a1, %hi(.LCPI21_1) -; RV32IFD-NEXT: fld fa5, %lo(.LCPI21_1)(a1) -; RV32IFD-NEXT: flt.d a3, fa5, fs0 -; RV32IFD-NEXT: beqz a3, .LBB21_4 +; RV32IFD-NEXT: and a0, a2, a0 +; RV32IFD-NEXT: beqz s0, .LBB21_4 ; RV32IFD-NEXT: # %bb.3: -; RV32IFD-NEXT: addi a2, a4, -1 +; RV32IFD-NEXT: addi a3, a4, -1 ; RV32IFD-NEXT: .LBB21_4: -; RV32IFD-NEXT: feq.d a1, fs0, fs0 -; RV32IFD-NEXT: neg a4, a1 -; RV32IFD-NEXT: and a1, a4, a2 -; RV32IFD-NEXT: neg a2, a3 -; RV32IFD-NEXT: neg a3, s0 -; RV32IFD-NEXT: and a0, a3, a0 -; RV32IFD-NEXT: or a0, a2, a0 -; RV32IFD-NEXT: and a0, a4, a0 -; RV32IFD-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IFD-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: and a1, a2, a3 +; RV32IFD-NEXT: lw ra, 28(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s0, 24(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: lw s3, 12(sp) # 4-byte Folded Reload ; RV32IFD-NEXT: fld fs0, 0(sp) # 8-byte Folded Reload -; RV32IFD-NEXT: addi sp, sp, 16 +; RV32IFD-NEXT: addi sp, sp, 32 ; RV32IFD-NEXT: ret ; ; RV64IFD-LABEL: test_rint_si64: @@ -1431,44 +1488,47 @@ define i64 @test_rint_si64(double %x) nounwind { ; RV32IZFINXZDINX-NEXT: addi sp, sp, -32 ; RV32IZFINXZDINX-NEXT: sw ra, 28(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s0, 24(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s2, 20(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s3, 16(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s1, 20(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call rint -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw s2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw s3, 12(sp) +; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) +; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) +; RV32IZFINXZDINX-NEXT: lw s0, 0(sp) +; RV32IZFINXZDINX-NEXT: lw s1, 4(sp) ; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI21_0) ; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI21_0+4)(a2) ; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI21_0)(a2) -; RV32IZFINXZDINX-NEXT: fle.d s0, a2, s2 +; RV32IZFINXZDINX-NEXT: fle.d s2, a2, s0 +; RV32IZFINXZDINX-NEXT: neg s3, s2 ; RV32IZFINXZDINX-NEXT: call __fixdfdi +; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI21_1) +; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI21_1+4)(a2) +; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI21_1)(a2) +; RV32IZFINXZDINX-NEXT: and a0, s3, a0 +; RV32IZFINXZDINX-NEXT: flt.d a3, a2, s0 +; RV32IZFINXZDINX-NEXT: neg a2, a3 +; RV32IZFINXZDINX-NEXT: or a0, a2, a0 +; RV32IZFINXZDINX-NEXT: feq.d a2, s0, s0 +; RV32IZFINXZDINX-NEXT: neg a2, a2 +; RV32IZFINXZDINX-NEXT: lui a5, 524288 +; RV32IZFINXZDINX-NEXT: li a6, 1 ; RV32IZFINXZDINX-NEXT: lui a4, 524288 -; RV32IZFINXZDINX-NEXT: lui a2, 524288 -; RV32IZFINXZDINX-NEXT: beqz s0, .LBB21_2 +; RV32IZFINXZDINX-NEXT: bne s2, a6, .LBB21_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: -; RV32IZFINXZDINX-NEXT: mv a2, a1 +; RV32IZFINXZDINX-NEXT: mv a4, a1 ; RV32IZFINXZDINX-NEXT: .LBB21_2: -; RV32IZFINXZDINX-NEXT: lui a1, %hi(.LCPI21_1) -; RV32IZFINXZDINX-NEXT: lw a6, %lo(.LCPI21_1)(a1) -; RV32IZFINXZDINX-NEXT: lw a7, %lo(.LCPI21_1+4)(a1) -; RV32IZFINXZDINX-NEXT: flt.d a3, a6, s2 +; RV32IZFINXZDINX-NEXT: and a0, a2, a0 ; RV32IZFINXZDINX-NEXT: beqz a3, .LBB21_4 ; RV32IZFINXZDINX-NEXT: # %bb.3: -; RV32IZFINXZDINX-NEXT: addi a2, a4, -1 +; RV32IZFINXZDINX-NEXT: addi a4, a5, -1 ; RV32IZFINXZDINX-NEXT: .LBB21_4: -; RV32IZFINXZDINX-NEXT: feq.d a1, s2, s2 -; RV32IZFINXZDINX-NEXT: neg a4, a1 -; RV32IZFINXZDINX-NEXT: and a1, a4, a2 -; RV32IZFINXZDINX-NEXT: neg a2, s0 -; RV32IZFINXZDINX-NEXT: and a0, a2, a0 -; RV32IZFINXZDINX-NEXT: neg a2, a3 -; RV32IZFINXZDINX-NEXT: or a0, a2, a0 -; RV32IZFINXZDINX-NEXT: and a0, a4, a0 +; RV32IZFINXZDINX-NEXT: and a1, a2, a4 ; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s2, 20(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s3, 16(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s3, 12(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: addi sp, sp, 32 ; RV32IZFINXZDINX-NEXT: ret ; @@ -1538,7 +1598,8 @@ define i64 @test_rint_ui64(double %x) nounwind { ; RV32IFD-NEXT: neg s0, a0 ; RV32IFD-NEXT: fcvt.d.w fa5, zero ; RV32IFD-NEXT: fle.d a0, fa5, fa0 -; RV32IFD-NEXT: neg s1, a0 +; RV32IFD-NEXT: xori a0, a0, 1 +; RV32IFD-NEXT: addi s1, a0, -1 ; RV32IFD-NEXT: call __fixunsdfdi ; RV32IFD-NEXT: and a0, s1, a0 ; RV32IFD-NEXT: or a0, s0, a0 @@ -1565,29 +1626,28 @@ define i64 @test_rint_ui64(double %x) nounwind { ; RV32IZFINXZDINX-NEXT: sw ra, 28(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s0, 24(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s1, 20(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call rint ; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) ; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: lw s0, 8(sp) ; RV32IZFINXZDINX-NEXT: lw s1, 12(sp) +; RV32IZFINXZDINX-NEXT: call __fixunsdfdi ; RV32IZFINXZDINX-NEXT: fcvt.d.w a2, zero ; RV32IZFINXZDINX-NEXT: fle.d a2, a2, s0 -; RV32IZFINXZDINX-NEXT: neg s2, a2 -; RV32IZFINXZDINX-NEXT: call __fixunsdfdi -; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI23_0) -; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI23_0+4)(a2) -; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI23_0)(a2) -; RV32IZFINXZDINX-NEXT: and a0, s2, a0 -; RV32IZFINXZDINX-NEXT: flt.d a2, a2, s0 -; RV32IZFINXZDINX-NEXT: neg a2, a2 -; RV32IZFINXZDINX-NEXT: or a0, a2, a0 -; RV32IZFINXZDINX-NEXT: and a1, s2, a1 -; RV32IZFINXZDINX-NEXT: or a1, a2, a1 +; RV32IZFINXZDINX-NEXT: lui a3, %hi(.LCPI23_0) +; RV32IZFINXZDINX-NEXT: lw a4, %lo(.LCPI23_0)(a3) +; RV32IZFINXZDINX-NEXT: lw a5, %lo(.LCPI23_0+4)(a3) +; RV32IZFINXZDINX-NEXT: xori a2, a2, 1 +; RV32IZFINXZDINX-NEXT: addi a2, a2, -1 +; RV32IZFINXZDINX-NEXT: and a0, a2, a0 +; RV32IZFINXZDINX-NEXT: flt.d a3, a4, s0 +; RV32IZFINXZDINX-NEXT: neg a3, a3 +; RV32IZFINXZDINX-NEXT: or a0, a3, a0 +; RV32IZFINXZDINX-NEXT: and a1, a2, a1 +; RV32IZFINXZDINX-NEXT: or a1, a3, a1 ; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s2, 16(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: addi sp, sp, 32 ; RV32IZFINXZDINX-NEXT: ret ; diff --git a/llvm/test/CodeGen/RISCV/float-convert.ll b/llvm/test/CodeGen/RISCV/float-convert.ll index f1e444b5b624..1a0e4e182911 100644 --- a/llvm/test/CodeGen/RISCV/float-convert.ll +++ b/llvm/test/CodeGen/RISCV/float-convert.ll @@ -275,24 +275,26 @@ define i32 @fcvt_wu_s_sat(float %a) nounwind { ; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32I-NEXT: sw s0, 8(sp) # 4-byte Folded Spill ; RV32I-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32I-NEXT: sw s2, 0(sp) # 4-byte Folded Spill ; RV32I-NEXT: mv s0, a0 +; RV32I-NEXT: lui a1, 325632 +; RV32I-NEXT: addi a1, a1, -1 +; RV32I-NEXT: call __gtsf2 +; RV32I-NEXT: sgtz a0, a0 +; RV32I-NEXT: neg s1, a0 +; RV32I-NEXT: mv a0, s0 ; RV32I-NEXT: li a1, 0 ; RV32I-NEXT: call __gesf2 ; RV32I-NEXT: slti a0, a0, 0 -; RV32I-NEXT: addi s1, a0, -1 +; RV32I-NEXT: addi s2, a0, -1 ; RV32I-NEXT: mv a0, s0 ; RV32I-NEXT: call __fixunssfsi -; RV32I-NEXT: and s1, s1, a0 -; RV32I-NEXT: lui a1, 325632 -; RV32I-NEXT: addi a1, a1, -1 -; RV32I-NEXT: mv a0, s0 -; RV32I-NEXT: call __gtsf2 -; RV32I-NEXT: sgtz a0, a0 -; RV32I-NEXT: neg a0, a0 -; RV32I-NEXT: or a0, a0, s1 +; RV32I-NEXT: and a0, s2, a0 +; RV32I-NEXT: or a0, s1, a0 ; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32I-NEXT: lw s0, 8(sp) # 4-byte Folded Reload ; RV32I-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32I-NEXT: lw s2, 0(sp) # 4-byte Folded Reload ; RV32I-NEXT: addi sp, sp, 16 ; RV32I-NEXT: ret ; @@ -613,40 +615,47 @@ define i64 @fcvt_l_s(float %a) nounwind { define i64 @fcvt_l_s_sat(float %a) nounwind { ; RV32IF-LABEL: fcvt_l_s_sat: ; RV32IF: # %bb.0: # %start -; RV32IF-NEXT: addi sp, sp, -16 -; RV32IF-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IF-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IF-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IF-NEXT: addi sp, sp, -32 +; RV32IF-NEXT: sw ra, 28(sp) # 4-byte Folded Spill +; RV32IF-NEXT: sw s0, 24(sp) # 4-byte Folded Spill +; RV32IF-NEXT: sw s1, 20(sp) # 4-byte Folded Spill +; RV32IF-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IF-NEXT: sw s3, 12(sp) # 4-byte Folded Spill +; RV32IF-NEXT: fsw fs0, 8(sp) # 4-byte Folded Spill +; RV32IF-NEXT: lui a0, %hi(.LCPI12_0) +; RV32IF-NEXT: flw fa5, %lo(.LCPI12_0)(a0) ; RV32IF-NEXT: fmv.s fs0, fa0 +; RV32IF-NEXT: flt.s s0, fa5, fa0 +; RV32IF-NEXT: neg s1, s0 ; RV32IF-NEXT: lui a0, 913408 ; RV32IF-NEXT: fmv.w.x fa5, a0 -; RV32IF-NEXT: fle.s s0, fa5, fa0 +; RV32IF-NEXT: fle.s s2, fa5, fa0 +; RV32IF-NEXT: neg s3, s2 ; RV32IF-NEXT: call __fixsfdi +; RV32IF-NEXT: and a0, s3, a0 +; RV32IF-NEXT: or a0, s1, a0 +; RV32IF-NEXT: feq.s a2, fs0, fs0 +; RV32IF-NEXT: neg a2, a2 ; RV32IF-NEXT: lui a4, 524288 -; RV32IF-NEXT: lui a2, 524288 -; RV32IF-NEXT: beqz s0, .LBB12_2 +; RV32IF-NEXT: li a5, 1 +; RV32IF-NEXT: lui a3, 524288 +; RV32IF-NEXT: bne s2, a5, .LBB12_2 ; RV32IF-NEXT: # %bb.1: # %start -; RV32IF-NEXT: mv a2, a1 +; RV32IF-NEXT: mv a3, a1 ; RV32IF-NEXT: .LBB12_2: # %start -; RV32IF-NEXT: lui a1, %hi(.LCPI12_0) -; RV32IF-NEXT: flw fa5, %lo(.LCPI12_0)(a1) -; RV32IF-NEXT: flt.s a3, fa5, fs0 -; RV32IF-NEXT: beqz a3, .LBB12_4 +; RV32IF-NEXT: and a0, a2, a0 +; RV32IF-NEXT: beqz s0, .LBB12_4 ; RV32IF-NEXT: # %bb.3: -; RV32IF-NEXT: addi a2, a4, -1 +; RV32IF-NEXT: addi a3, a4, -1 ; RV32IF-NEXT: .LBB12_4: # %start -; RV32IF-NEXT: feq.s a1, fs0, fs0 -; RV32IF-NEXT: neg a4, a1 -; RV32IF-NEXT: and a1, a4, a2 -; RV32IF-NEXT: neg a2, a3 -; RV32IF-NEXT: neg a3, s0 -; RV32IF-NEXT: and a0, a3, a0 -; RV32IF-NEXT: or a0, a2, a0 -; RV32IF-NEXT: and a0, a4, a0 -; RV32IF-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IF-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IF-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload -; RV32IF-NEXT: addi sp, sp, 16 +; RV32IF-NEXT: and a1, a2, a3 +; RV32IF-NEXT: lw ra, 28(sp) # 4-byte Folded Reload +; RV32IF-NEXT: lw s0, 24(sp) # 4-byte Folded Reload +; RV32IF-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; RV32IF-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; RV32IF-NEXT: lw s3, 12(sp) # 4-byte Folded Reload +; RV32IF-NEXT: flw fs0, 8(sp) # 4-byte Folded Reload +; RV32IF-NEXT: addi sp, sp, 32 ; RV32IF-NEXT: ret ; ; RV64IF-LABEL: fcvt_l_s_sat: @@ -664,35 +673,38 @@ define i64 @fcvt_l_s_sat(float %a) nounwind { ; RV32IZFINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill ; RV32IZFINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32IZFINX-NEXT: sw s2, 0(sp) # 4-byte Folded Spill ; RV32IZFINX-NEXT: mv s0, a0 ; RV32IZFINX-NEXT: lui a0, 913408 ; RV32IZFINX-NEXT: fle.s s1, a0, s0 +; RV32IZFINX-NEXT: neg s2, s1 ; RV32IZFINX-NEXT: mv a0, s0 ; RV32IZFINX-NEXT: call __fixsfdi +; RV32IZFINX-NEXT: lui a2, %hi(.LCPI12_0) +; RV32IZFINX-NEXT: lw a2, %lo(.LCPI12_0)(a2) +; RV32IZFINX-NEXT: and a0, s2, a0 +; RV32IZFINX-NEXT: flt.s a3, a2, s0 +; RV32IZFINX-NEXT: neg a2, a3 +; RV32IZFINX-NEXT: or a0, a2, a0 +; RV32IZFINX-NEXT: feq.s a2, s0, s0 +; RV32IZFINX-NEXT: neg a2, a2 +; RV32IZFINX-NEXT: lui a5, 524288 +; RV32IZFINX-NEXT: li a6, 1 ; RV32IZFINX-NEXT: lui a4, 524288 -; RV32IZFINX-NEXT: lui a2, 524288 -; RV32IZFINX-NEXT: beqz s1, .LBB12_2 +; RV32IZFINX-NEXT: bne s1, a6, .LBB12_2 ; RV32IZFINX-NEXT: # %bb.1: # %start -; RV32IZFINX-NEXT: mv a2, a1 +; RV32IZFINX-NEXT: mv a4, a1 ; RV32IZFINX-NEXT: .LBB12_2: # %start -; RV32IZFINX-NEXT: lui a1, %hi(.LCPI12_0) -; RV32IZFINX-NEXT: lw a1, %lo(.LCPI12_0)(a1) -; RV32IZFINX-NEXT: flt.s a3, a1, s0 +; RV32IZFINX-NEXT: and a0, a2, a0 ; RV32IZFINX-NEXT: beqz a3, .LBB12_4 ; RV32IZFINX-NEXT: # %bb.3: -; RV32IZFINX-NEXT: addi a2, a4, -1 +; RV32IZFINX-NEXT: addi a4, a5, -1 ; RV32IZFINX-NEXT: .LBB12_4: # %start -; RV32IZFINX-NEXT: feq.s a1, s0, s0 -; RV32IZFINX-NEXT: neg a4, a1 -; RV32IZFINX-NEXT: and a1, a4, a2 -; RV32IZFINX-NEXT: neg a2, s1 -; RV32IZFINX-NEXT: and a0, a2, a0 -; RV32IZFINX-NEXT: neg a2, a3 -; RV32IZFINX-NEXT: or a0, a2, a0 -; RV32IZFINX-NEXT: and a0, a4, a0 +; RV32IZFINX-NEXT: and a1, a2, a4 ; RV32IZFINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IZFINX-NEXT: lw s2, 0(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: addi sp, sp, 16 ; RV32IZFINX-NEXT: ret ; @@ -863,23 +875,23 @@ define i64 @fcvt_lu_s_sat(float %a) nounwind { ; RV32IF-NEXT: addi sp, sp, -16 ; RV32IF-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IF-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IF-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill -; RV32IF-NEXT: fmv.s fs0, fa0 +; RV32IF-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32IF-NEXT: lui a0, %hi(.LCPI14_0) +; RV32IF-NEXT: flw fa5, %lo(.LCPI14_0)(a0) +; RV32IF-NEXT: flt.s a0, fa5, fa0 +; RV32IF-NEXT: neg s0, a0 ; RV32IF-NEXT: fmv.w.x fa5, zero ; RV32IF-NEXT: fle.s a0, fa5, fa0 -; RV32IF-NEXT: neg s0, a0 +; RV32IF-NEXT: xori a0, a0, 1 +; RV32IF-NEXT: addi s1, a0, -1 ; RV32IF-NEXT: call __fixunssfdi -; RV32IF-NEXT: lui a2, %hi(.LCPI14_0) -; RV32IF-NEXT: flw fa5, %lo(.LCPI14_0)(a2) -; RV32IF-NEXT: and a0, s0, a0 -; RV32IF-NEXT: flt.s a2, fa5, fs0 -; RV32IF-NEXT: neg a2, a2 -; RV32IF-NEXT: or a0, a2, a0 -; RV32IF-NEXT: and a1, s0, a1 -; RV32IF-NEXT: or a1, a2, a1 +; RV32IF-NEXT: and a0, s1, a0 +; RV32IF-NEXT: or a0, s0, a0 +; RV32IF-NEXT: and a1, s1, a1 +; RV32IF-NEXT: or a1, s0, a1 ; RV32IF-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IF-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IF-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IF-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IF-NEXT: addi sp, sp, 16 ; RV32IF-NEXT: ret ; @@ -898,19 +910,18 @@ define i64 @fcvt_lu_s_sat(float %a) nounwind { ; RV32IZFINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill ; RV32IZFINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill -; RV32IZFINX-NEXT: mv s0, a0 -; RV32IZFINX-NEXT: fle.s a0, zero, a0 -; RV32IZFINX-NEXT: neg s1, a0 -; RV32IZFINX-NEXT: mv a0, s0 +; RV32IZFINX-NEXT: lui a1, %hi(.LCPI14_0) +; RV32IZFINX-NEXT: lw a1, %lo(.LCPI14_0)(a1) +; RV32IZFINX-NEXT: flt.s a1, a1, a0 +; RV32IZFINX-NEXT: neg s0, a1 +; RV32IZFINX-NEXT: fle.s a1, zero, a0 +; RV32IZFINX-NEXT: xori a1, a1, 1 +; RV32IZFINX-NEXT: addi s1, a1, -1 ; RV32IZFINX-NEXT: call __fixunssfdi -; RV32IZFINX-NEXT: lui a2, %hi(.LCPI14_0) -; RV32IZFINX-NEXT: lw a2, %lo(.LCPI14_0)(a2) ; RV32IZFINX-NEXT: and a0, s1, a0 -; RV32IZFINX-NEXT: flt.s a2, a2, s0 -; RV32IZFINX-NEXT: neg a2, a2 -; RV32IZFINX-NEXT: or a0, a2, a0 +; RV32IZFINX-NEXT: or a0, s0, a0 ; RV32IZFINX-NEXT: and a1, s1, a1 -; RV32IZFINX-NEXT: or a1, a2, a1 +; RV32IZFINX-NEXT: or a1, s0, a1 ; RV32IZFINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload @@ -928,36 +939,33 @@ define i64 @fcvt_lu_s_sat(float %a) nounwind { ; ; RV32I-LABEL: fcvt_lu_s_sat: ; RV32I: # %bb.0: # %start -; RV32I-NEXT: addi sp, sp, -32 -; RV32I-NEXT: sw ra, 28(sp) # 4-byte Folded Spill -; RV32I-NEXT: sw s0, 24(sp) # 4-byte Folded Spill -; RV32I-NEXT: sw s1, 20(sp) # 4-byte Folded Spill -; RV32I-NEXT: sw s2, 16(sp) # 4-byte Folded Spill -; RV32I-NEXT: sw s3, 12(sp) # 4-byte Folded Spill +; RV32I-NEXT: addi sp, sp, -16 +; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32I-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32I-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32I-NEXT: sw s2, 0(sp) # 4-byte Folded Spill ; RV32I-NEXT: mv s0, a0 +; RV32I-NEXT: lui a1, 391168 +; RV32I-NEXT: addi a1, a1, -1 +; RV32I-NEXT: call __gtsf2 +; RV32I-NEXT: sgtz a0, a0 +; RV32I-NEXT: neg s1, a0 +; RV32I-NEXT: mv a0, s0 ; RV32I-NEXT: li a1, 0 ; RV32I-NEXT: call __gesf2 ; RV32I-NEXT: slti a0, a0, 0 ; RV32I-NEXT: addi s2, a0, -1 ; RV32I-NEXT: mv a0, s0 ; RV32I-NEXT: call __fixunssfdi -; RV32I-NEXT: mv s1, a1 -; RV32I-NEXT: and s3, s2, a0 -; RV32I-NEXT: lui a1, 391168 -; RV32I-NEXT: addi a1, a1, -1 -; RV32I-NEXT: mv a0, s0 -; RV32I-NEXT: call __gtsf2 -; RV32I-NEXT: sgtz a0, a0 -; RV32I-NEXT: neg a1, a0 -; RV32I-NEXT: or a0, a1, s3 -; RV32I-NEXT: and a2, s2, s1 -; RV32I-NEXT: or a1, a1, a2 -; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload -; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload -; RV32I-NEXT: lw s1, 20(sp) # 4-byte Folded Reload -; RV32I-NEXT: lw s2, 16(sp) # 4-byte Folded Reload -; RV32I-NEXT: lw s3, 12(sp) # 4-byte Folded Reload -; RV32I-NEXT: addi sp, sp, 32 +; RV32I-NEXT: and a0, s2, a0 +; RV32I-NEXT: or a0, s1, a0 +; RV32I-NEXT: and a1, s2, a1 +; RV32I-NEXT: or a1, s1, a1 +; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32I-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32I-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32I-NEXT: lw s2, 0(sp) # 4-byte Folded Reload +; RV32I-NEXT: addi sp, sp, 16 ; RV32I-NEXT: ret ; ; RV64I-LABEL: fcvt_lu_s_sat: @@ -966,24 +974,26 @@ define i64 @fcvt_lu_s_sat(float %a) nounwind { ; RV64I-NEXT: sd ra, 24(sp) # 8-byte Folded Spill ; RV64I-NEXT: sd s0, 16(sp) # 8-byte Folded Spill ; RV64I-NEXT: sd s1, 8(sp) # 8-byte Folded Spill +; RV64I-NEXT: sd s2, 0(sp) # 8-byte Folded Spill ; RV64I-NEXT: mv s0, a0 +; RV64I-NEXT: lui a1, 391168 +; RV64I-NEXT: addiw a1, a1, -1 +; RV64I-NEXT: call __gtsf2 +; RV64I-NEXT: sgtz a0, a0 +; RV64I-NEXT: neg s1, a0 +; RV64I-NEXT: mv a0, s0 ; RV64I-NEXT: li a1, 0 ; RV64I-NEXT: call __gesf2 ; RV64I-NEXT: slti a0, a0, 0 -; RV64I-NEXT: addi s1, a0, -1 +; RV64I-NEXT: addi s2, a0, -1 ; RV64I-NEXT: mv a0, s0 ; RV64I-NEXT: call __fixunssfdi -; RV64I-NEXT: and s1, s1, a0 -; RV64I-NEXT: lui a1, 391168 -; RV64I-NEXT: addiw a1, a1, -1 -; RV64I-NEXT: mv a0, s0 -; RV64I-NEXT: call __gtsf2 -; RV64I-NEXT: sgtz a0, a0 -; RV64I-NEXT: neg a0, a0 -; RV64I-NEXT: or a0, a0, s1 +; RV64I-NEXT: and a0, s2, a0 +; RV64I-NEXT: or a0, s1, a0 ; RV64I-NEXT: ld ra, 24(sp) # 8-byte Folded Reload ; RV64I-NEXT: ld s0, 16(sp) # 8-byte Folded Reload ; RV64I-NEXT: ld s1, 8(sp) # 8-byte Folded Reload +; RV64I-NEXT: ld s2, 0(sp) # 8-byte Folded Reload ; RV64I-NEXT: addi sp, sp, 32 ; RV64I-NEXT: ret start: @@ -2089,24 +2099,26 @@ define zeroext i32 @fcvt_wu_s_sat_zext(float %a) nounwind { ; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32I-NEXT: sw s0, 8(sp) # 4-byte Folded Spill ; RV32I-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32I-NEXT: sw s2, 0(sp) # 4-byte Folded Spill ; RV32I-NEXT: mv s0, a0 +; RV32I-NEXT: lui a1, 325632 +; RV32I-NEXT: addi a1, a1, -1 +; RV32I-NEXT: call __gtsf2 +; RV32I-NEXT: sgtz a0, a0 +; RV32I-NEXT: neg s1, a0 +; RV32I-NEXT: mv a0, s0 ; RV32I-NEXT: li a1, 0 ; RV32I-NEXT: call __gesf2 ; RV32I-NEXT: slti a0, a0, 0 -; RV32I-NEXT: addi s1, a0, -1 +; RV32I-NEXT: addi s2, a0, -1 ; RV32I-NEXT: mv a0, s0 ; RV32I-NEXT: call __fixunssfsi -; RV32I-NEXT: and s1, s1, a0 -; RV32I-NEXT: lui a1, 325632 -; RV32I-NEXT: addi a1, a1, -1 -; RV32I-NEXT: mv a0, s0 -; RV32I-NEXT: call __gtsf2 -; RV32I-NEXT: sgtz a0, a0 -; RV32I-NEXT: neg a0, a0 -; RV32I-NEXT: or a0, a0, s1 +; RV32I-NEXT: and a0, s2, a0 +; RV32I-NEXT: or a0, s1, a0 ; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32I-NEXT: lw s0, 8(sp) # 4-byte Folded Reload ; RV32I-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32I-NEXT: lw s2, 0(sp) # 4-byte Folded Reload ; RV32I-NEXT: addi sp, sp, 16 ; RV32I-NEXT: ret ; diff --git a/llvm/test/CodeGen/RISCV/float-round-conv-sat.ll b/llvm/test/CodeGen/RISCV/float-round-conv-sat.ll index 5e99c7eb9056..f91aac11876d 100644 --- a/llvm/test/CodeGen/RISCV/float-round-conv-sat.ll +++ b/llvm/test/CodeGen/RISCV/float-round-conv-sat.ll @@ -37,7 +37,8 @@ define i64 @test_floor_si64(float %x) nounwind { ; RV32IF-NEXT: addi sp, sp, -16 ; RV32IF-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IF-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IF-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IF-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32IF-NEXT: fsw fs0, 0(sp) # 4-byte Folded Spill ; RV32IF-NEXT: fmv.s fs0, fa0 ; RV32IF-NEXT: lui a0, 307200 ; RV32IF-NEXT: fmv.w.x fa5, a0 @@ -52,32 +53,34 @@ define i64 @test_floor_si64(float %x) nounwind { ; RV32IF-NEXT: lui a0, 913408 ; RV32IF-NEXT: fmv.w.x fa5, a0 ; RV32IF-NEXT: fle.s s0, fa5, fs0 +; RV32IF-NEXT: neg s1, s0 ; RV32IF-NEXT: fmv.s fa0, fs0 ; RV32IF-NEXT: call __fixsfdi +; RV32IF-NEXT: lui a2, %hi(.LCPI1_0) +; RV32IF-NEXT: flw fa5, %lo(.LCPI1_0)(a2) +; RV32IF-NEXT: and a0, s1, a0 +; RV32IF-NEXT: flt.s a3, fa5, fs0 +; RV32IF-NEXT: neg a2, a3 +; RV32IF-NEXT: or a0, a2, a0 +; RV32IF-NEXT: feq.s a2, fs0, fs0 +; RV32IF-NEXT: neg a2, a2 +; RV32IF-NEXT: lui a5, 524288 +; RV32IF-NEXT: li a6, 1 ; RV32IF-NEXT: lui a4, 524288 -; RV32IF-NEXT: lui a2, 524288 -; RV32IF-NEXT: beqz s0, .LBB1_4 +; RV32IF-NEXT: bne s0, a6, .LBB1_4 ; RV32IF-NEXT: # %bb.3: -; RV32IF-NEXT: mv a2, a1 +; RV32IF-NEXT: mv a4, a1 ; RV32IF-NEXT: .LBB1_4: -; RV32IF-NEXT: lui a1, %hi(.LCPI1_0) -; RV32IF-NEXT: flw fa5, %lo(.LCPI1_0)(a1) -; RV32IF-NEXT: flt.s a3, fa5, fs0 +; RV32IF-NEXT: and a0, a2, a0 ; RV32IF-NEXT: beqz a3, .LBB1_6 ; RV32IF-NEXT: # %bb.5: -; RV32IF-NEXT: addi a2, a4, -1 +; RV32IF-NEXT: addi a4, a5, -1 ; RV32IF-NEXT: .LBB1_6: -; RV32IF-NEXT: feq.s a1, fs0, fs0 -; RV32IF-NEXT: neg a4, a1 -; RV32IF-NEXT: and a1, a4, a2 -; RV32IF-NEXT: neg a2, s0 -; RV32IF-NEXT: and a0, a2, a0 -; RV32IF-NEXT: neg a2, a3 -; RV32IF-NEXT: or a0, a2, a0 -; RV32IF-NEXT: and a0, a4, a0 +; RV32IF-NEXT: and a1, a2, a4 ; RV32IF-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IF-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IF-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IF-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IF-NEXT: flw fs0, 0(sp) # 4-byte Folded Reload ; RV32IF-NEXT: addi sp, sp, 16 ; RV32IF-NEXT: ret ; @@ -115,23 +118,24 @@ define i64 @test_floor_si64(float %x) nounwind { ; RV32IZFINX-NEXT: lui a2, %hi(.LCPI1_0) ; RV32IZFINX-NEXT: lw a2, %lo(.LCPI1_0)(a2) ; RV32IZFINX-NEXT: and a0, s2, a0 -; RV32IZFINX-NEXT: flt.s a4, a2, s0 -; RV32IZFINX-NEXT: neg a2, a4 +; RV32IZFINX-NEXT: flt.s a3, a2, s0 +; RV32IZFINX-NEXT: neg a2, a3 ; RV32IZFINX-NEXT: or a0, a2, a0 ; RV32IZFINX-NEXT: feq.s a2, s0, s0 ; RV32IZFINX-NEXT: neg a2, a2 ; RV32IZFINX-NEXT: lui a5, 524288 -; RV32IZFINX-NEXT: lui a3, 524288 -; RV32IZFINX-NEXT: beqz s1, .LBB1_4 +; RV32IZFINX-NEXT: li a6, 1 +; RV32IZFINX-NEXT: lui a4, 524288 +; RV32IZFINX-NEXT: bne s1, a6, .LBB1_4 ; RV32IZFINX-NEXT: # %bb.3: -; RV32IZFINX-NEXT: mv a3, a1 +; RV32IZFINX-NEXT: mv a4, a1 ; RV32IZFINX-NEXT: .LBB1_4: ; RV32IZFINX-NEXT: and a0, a2, a0 -; RV32IZFINX-NEXT: beqz a4, .LBB1_6 +; RV32IZFINX-NEXT: beqz a3, .LBB1_6 ; RV32IZFINX-NEXT: # %bb.5: -; RV32IZFINX-NEXT: addi a3, a5, -1 +; RV32IZFINX-NEXT: addi a4, a5, -1 ; RV32IZFINX-NEXT: .LBB1_6: -; RV32IZFINX-NEXT: and a1, a2, a3 +; RV32IZFINX-NEXT: and a1, a2, a4 ; RV32IZFINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload @@ -180,8 +184,7 @@ define i64 @test_floor_ui64(float %x) nounwind { ; RV32IF: # %bb.0: ; RV32IF-NEXT: addi sp, sp, -16 ; RV32IF-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IF-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IF-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IF-NEXT: fsw fs0, 8(sp) # 4-byte Folded Spill ; RV32IF-NEXT: fmv.s fs0, fa0 ; RV32IF-NEXT: lui a0, 307200 ; RV32IF-NEXT: fmv.w.x fa5, a0 @@ -193,22 +196,22 @@ define i64 @test_floor_ui64(float %x) nounwind { ; RV32IF-NEXT: fcvt.s.w fa5, a0, rdn ; RV32IF-NEXT: fsgnj.s fs0, fa5, fs0 ; RV32IF-NEXT: .LBB3_2: -; RV32IF-NEXT: fmv.w.x fa5, zero -; RV32IF-NEXT: fle.s a0, fa5, fs0 -; RV32IF-NEXT: neg s0, a0 ; RV32IF-NEXT: fmv.s fa0, fs0 ; RV32IF-NEXT: call __fixunssfdi -; RV32IF-NEXT: lui a2, %hi(.LCPI3_0) -; RV32IF-NEXT: flw fa5, %lo(.LCPI3_0)(a2) -; RV32IF-NEXT: and a0, s0, a0 -; RV32IF-NEXT: flt.s a2, fa5, fs0 -; RV32IF-NEXT: neg a2, a2 -; RV32IF-NEXT: or a0, a2, a0 -; RV32IF-NEXT: and a1, s0, a1 -; RV32IF-NEXT: or a1, a2, a1 +; RV32IF-NEXT: fmv.w.x fa5, zero +; RV32IF-NEXT: fle.s a2, fa5, fs0 +; RV32IF-NEXT: lui a3, %hi(.LCPI3_0) +; RV32IF-NEXT: flw fa5, %lo(.LCPI3_0)(a3) +; RV32IF-NEXT: xori a2, a2, 1 +; RV32IF-NEXT: addi a2, a2, -1 +; RV32IF-NEXT: and a0, a2, a0 +; RV32IF-NEXT: flt.s a3, fa5, fs0 +; RV32IF-NEXT: neg a3, a3 +; RV32IF-NEXT: or a0, a3, a0 +; RV32IF-NEXT: and a1, a2, a1 +; RV32IF-NEXT: or a1, a3, a1 ; RV32IF-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IF-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IF-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IF-NEXT: flw fs0, 8(sp) # 4-byte Folded Reload ; RV32IF-NEXT: addi sp, sp, 16 ; RV32IF-NEXT: ret ; @@ -226,7 +229,6 @@ define i64 @test_floor_ui64(float %x) nounwind { ; RV32IZFINX-NEXT: addi sp, sp, -16 ; RV32IZFINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZFINX-NEXT: mv s0, a0 ; RV32IZFINX-NEXT: lui a0, 307200 ; RV32IZFINX-NEXT: fabs.s a1, s0 @@ -237,21 +239,21 @@ define i64 @test_floor_ui64(float %x) nounwind { ; RV32IZFINX-NEXT: fcvt.s.w a0, a0, rdn ; RV32IZFINX-NEXT: fsgnj.s s0, a0, s0 ; RV32IZFINX-NEXT: .LBB3_2: -; RV32IZFINX-NEXT: fle.s a0, zero, s0 -; RV32IZFINX-NEXT: neg s1, a0 ; RV32IZFINX-NEXT: mv a0, s0 ; RV32IZFINX-NEXT: call __fixunssfdi -; RV32IZFINX-NEXT: lui a2, %hi(.LCPI3_0) -; RV32IZFINX-NEXT: lw a2, %lo(.LCPI3_0)(a2) -; RV32IZFINX-NEXT: and a0, s1, a0 -; RV32IZFINX-NEXT: flt.s a2, a2, s0 -; RV32IZFINX-NEXT: neg a2, a2 -; RV32IZFINX-NEXT: or a0, a2, a0 -; RV32IZFINX-NEXT: and a1, s1, a1 -; RV32IZFINX-NEXT: or a1, a2, a1 +; RV32IZFINX-NEXT: fle.s a2, zero, s0 +; RV32IZFINX-NEXT: lui a3, %hi(.LCPI3_0) +; RV32IZFINX-NEXT: lw a3, %lo(.LCPI3_0)(a3) +; RV32IZFINX-NEXT: xori a2, a2, 1 +; RV32IZFINX-NEXT: addi a2, a2, -1 +; RV32IZFINX-NEXT: and a0, a2, a0 +; RV32IZFINX-NEXT: flt.s a3, a3, s0 +; RV32IZFINX-NEXT: neg a3, a3 +; RV32IZFINX-NEXT: or a0, a3, a0 +; RV32IZFINX-NEXT: and a1, a2, a1 +; RV32IZFINX-NEXT: or a1, a3, a1 ; RV32IZFINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: addi sp, sp, 16 ; RV32IZFINX-NEXT: ret ; @@ -297,7 +299,8 @@ define i64 @test_ceil_si64(float %x) nounwind { ; RV32IF-NEXT: addi sp, sp, -16 ; RV32IF-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IF-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IF-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IF-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32IF-NEXT: fsw fs0, 0(sp) # 4-byte Folded Spill ; RV32IF-NEXT: fmv.s fs0, fa0 ; RV32IF-NEXT: lui a0, 307200 ; RV32IF-NEXT: fmv.w.x fa5, a0 @@ -312,32 +315,34 @@ define i64 @test_ceil_si64(float %x) nounwind { ; RV32IF-NEXT: lui a0, 913408 ; RV32IF-NEXT: fmv.w.x fa5, a0 ; RV32IF-NEXT: fle.s s0, fa5, fs0 +; RV32IF-NEXT: neg s1, s0 ; RV32IF-NEXT: fmv.s fa0, fs0 ; RV32IF-NEXT: call __fixsfdi +; RV32IF-NEXT: lui a2, %hi(.LCPI5_0) +; RV32IF-NEXT: flw fa5, %lo(.LCPI5_0)(a2) +; RV32IF-NEXT: and a0, s1, a0 +; RV32IF-NEXT: flt.s a3, fa5, fs0 +; RV32IF-NEXT: neg a2, a3 +; RV32IF-NEXT: or a0, a2, a0 +; RV32IF-NEXT: feq.s a2, fs0, fs0 +; RV32IF-NEXT: neg a2, a2 +; RV32IF-NEXT: lui a5, 524288 +; RV32IF-NEXT: li a6, 1 ; RV32IF-NEXT: lui a4, 524288 -; RV32IF-NEXT: lui a2, 524288 -; RV32IF-NEXT: beqz s0, .LBB5_4 +; RV32IF-NEXT: bne s0, a6, .LBB5_4 ; RV32IF-NEXT: # %bb.3: -; RV32IF-NEXT: mv a2, a1 +; RV32IF-NEXT: mv a4, a1 ; RV32IF-NEXT: .LBB5_4: -; RV32IF-NEXT: lui a1, %hi(.LCPI5_0) -; RV32IF-NEXT: flw fa5, %lo(.LCPI5_0)(a1) -; RV32IF-NEXT: flt.s a3, fa5, fs0 +; RV32IF-NEXT: and a0, a2, a0 ; RV32IF-NEXT: beqz a3, .LBB5_6 ; RV32IF-NEXT: # %bb.5: -; RV32IF-NEXT: addi a2, a4, -1 +; RV32IF-NEXT: addi a4, a5, -1 ; RV32IF-NEXT: .LBB5_6: -; RV32IF-NEXT: feq.s a1, fs0, fs0 -; RV32IF-NEXT: neg a4, a1 -; RV32IF-NEXT: and a1, a4, a2 -; RV32IF-NEXT: neg a2, s0 -; RV32IF-NEXT: and a0, a2, a0 -; RV32IF-NEXT: neg a2, a3 -; RV32IF-NEXT: or a0, a2, a0 -; RV32IF-NEXT: and a0, a4, a0 +; RV32IF-NEXT: and a1, a2, a4 ; RV32IF-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IF-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IF-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IF-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IF-NEXT: flw fs0, 0(sp) # 4-byte Folded Reload ; RV32IF-NEXT: addi sp, sp, 16 ; RV32IF-NEXT: ret ; @@ -375,23 +380,24 @@ define i64 @test_ceil_si64(float %x) nounwind { ; RV32IZFINX-NEXT: lui a2, %hi(.LCPI5_0) ; RV32IZFINX-NEXT: lw a2, %lo(.LCPI5_0)(a2) ; RV32IZFINX-NEXT: and a0, s2, a0 -; RV32IZFINX-NEXT: flt.s a4, a2, s0 -; RV32IZFINX-NEXT: neg a2, a4 +; RV32IZFINX-NEXT: flt.s a3, a2, s0 +; RV32IZFINX-NEXT: neg a2, a3 ; RV32IZFINX-NEXT: or a0, a2, a0 ; RV32IZFINX-NEXT: feq.s a2, s0, s0 ; RV32IZFINX-NEXT: neg a2, a2 ; RV32IZFINX-NEXT: lui a5, 524288 -; RV32IZFINX-NEXT: lui a3, 524288 -; RV32IZFINX-NEXT: beqz s1, .LBB5_4 +; RV32IZFINX-NEXT: li a6, 1 +; RV32IZFINX-NEXT: lui a4, 524288 +; RV32IZFINX-NEXT: bne s1, a6, .LBB5_4 ; RV32IZFINX-NEXT: # %bb.3: -; RV32IZFINX-NEXT: mv a3, a1 +; RV32IZFINX-NEXT: mv a4, a1 ; RV32IZFINX-NEXT: .LBB5_4: ; RV32IZFINX-NEXT: and a0, a2, a0 -; RV32IZFINX-NEXT: beqz a4, .LBB5_6 +; RV32IZFINX-NEXT: beqz a3, .LBB5_6 ; RV32IZFINX-NEXT: # %bb.5: -; RV32IZFINX-NEXT: addi a3, a5, -1 +; RV32IZFINX-NEXT: addi a4, a5, -1 ; RV32IZFINX-NEXT: .LBB5_6: -; RV32IZFINX-NEXT: and a1, a2, a3 +; RV32IZFINX-NEXT: and a1, a2, a4 ; RV32IZFINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload @@ -440,8 +446,7 @@ define i64 @test_ceil_ui64(float %x) nounwind { ; RV32IF: # %bb.0: ; RV32IF-NEXT: addi sp, sp, -16 ; RV32IF-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IF-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IF-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IF-NEXT: fsw fs0, 8(sp) # 4-byte Folded Spill ; RV32IF-NEXT: fmv.s fs0, fa0 ; RV32IF-NEXT: lui a0, 307200 ; RV32IF-NEXT: fmv.w.x fa5, a0 @@ -453,22 +458,22 @@ define i64 @test_ceil_ui64(float %x) nounwind { ; RV32IF-NEXT: fcvt.s.w fa5, a0, rup ; RV32IF-NEXT: fsgnj.s fs0, fa5, fs0 ; RV32IF-NEXT: .LBB7_2: -; RV32IF-NEXT: fmv.w.x fa5, zero -; RV32IF-NEXT: fle.s a0, fa5, fs0 -; RV32IF-NEXT: neg s0, a0 ; RV32IF-NEXT: fmv.s fa0, fs0 ; RV32IF-NEXT: call __fixunssfdi -; RV32IF-NEXT: lui a2, %hi(.LCPI7_0) -; RV32IF-NEXT: flw fa5, %lo(.LCPI7_0)(a2) -; RV32IF-NEXT: and a0, s0, a0 -; RV32IF-NEXT: flt.s a2, fa5, fs0 -; RV32IF-NEXT: neg a2, a2 -; RV32IF-NEXT: or a0, a2, a0 -; RV32IF-NEXT: and a1, s0, a1 -; RV32IF-NEXT: or a1, a2, a1 +; RV32IF-NEXT: fmv.w.x fa5, zero +; RV32IF-NEXT: fle.s a2, fa5, fs0 +; RV32IF-NEXT: lui a3, %hi(.LCPI7_0) +; RV32IF-NEXT: flw fa5, %lo(.LCPI7_0)(a3) +; RV32IF-NEXT: xori a2, a2, 1 +; RV32IF-NEXT: addi a2, a2, -1 +; RV32IF-NEXT: and a0, a2, a0 +; RV32IF-NEXT: flt.s a3, fa5, fs0 +; RV32IF-NEXT: neg a3, a3 +; RV32IF-NEXT: or a0, a3, a0 +; RV32IF-NEXT: and a1, a2, a1 +; RV32IF-NEXT: or a1, a3, a1 ; RV32IF-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IF-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IF-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IF-NEXT: flw fs0, 8(sp) # 4-byte Folded Reload ; RV32IF-NEXT: addi sp, sp, 16 ; RV32IF-NEXT: ret ; @@ -486,7 +491,6 @@ define i64 @test_ceil_ui64(float %x) nounwind { ; RV32IZFINX-NEXT: addi sp, sp, -16 ; RV32IZFINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZFINX-NEXT: mv s0, a0 ; RV32IZFINX-NEXT: lui a0, 307200 ; RV32IZFINX-NEXT: fabs.s a1, s0 @@ -497,21 +501,21 @@ define i64 @test_ceil_ui64(float %x) nounwind { ; RV32IZFINX-NEXT: fcvt.s.w a0, a0, rup ; RV32IZFINX-NEXT: fsgnj.s s0, a0, s0 ; RV32IZFINX-NEXT: .LBB7_2: -; RV32IZFINX-NEXT: fle.s a0, zero, s0 -; RV32IZFINX-NEXT: neg s1, a0 ; RV32IZFINX-NEXT: mv a0, s0 ; RV32IZFINX-NEXT: call __fixunssfdi -; RV32IZFINX-NEXT: lui a2, %hi(.LCPI7_0) -; RV32IZFINX-NEXT: lw a2, %lo(.LCPI7_0)(a2) -; RV32IZFINX-NEXT: and a0, s1, a0 -; RV32IZFINX-NEXT: flt.s a2, a2, s0 -; RV32IZFINX-NEXT: neg a2, a2 -; RV32IZFINX-NEXT: or a0, a2, a0 -; RV32IZFINX-NEXT: and a1, s1, a1 -; RV32IZFINX-NEXT: or a1, a2, a1 +; RV32IZFINX-NEXT: fle.s a2, zero, s0 +; RV32IZFINX-NEXT: lui a3, %hi(.LCPI7_0) +; RV32IZFINX-NEXT: lw a3, %lo(.LCPI7_0)(a3) +; RV32IZFINX-NEXT: xori a2, a2, 1 +; RV32IZFINX-NEXT: addi a2, a2, -1 +; RV32IZFINX-NEXT: and a0, a2, a0 +; RV32IZFINX-NEXT: flt.s a3, a3, s0 +; RV32IZFINX-NEXT: neg a3, a3 +; RV32IZFINX-NEXT: or a0, a3, a0 +; RV32IZFINX-NEXT: and a1, a2, a1 +; RV32IZFINX-NEXT: or a1, a3, a1 ; RV32IZFINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: addi sp, sp, 16 ; RV32IZFINX-NEXT: ret ; @@ -557,7 +561,8 @@ define i64 @test_trunc_si64(float %x) nounwind { ; RV32IF-NEXT: addi sp, sp, -16 ; RV32IF-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IF-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IF-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IF-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32IF-NEXT: fsw fs0, 0(sp) # 4-byte Folded Spill ; RV32IF-NEXT: fmv.s fs0, fa0 ; RV32IF-NEXT: lui a0, 307200 ; RV32IF-NEXT: fmv.w.x fa5, a0 @@ -572,32 +577,34 @@ define i64 @test_trunc_si64(float %x) nounwind { ; RV32IF-NEXT: lui a0, 913408 ; RV32IF-NEXT: fmv.w.x fa5, a0 ; RV32IF-NEXT: fle.s s0, fa5, fs0 +; RV32IF-NEXT: neg s1, s0 ; RV32IF-NEXT: fmv.s fa0, fs0 ; RV32IF-NEXT: call __fixsfdi +; RV32IF-NEXT: lui a2, %hi(.LCPI9_0) +; RV32IF-NEXT: flw fa5, %lo(.LCPI9_0)(a2) +; RV32IF-NEXT: and a0, s1, a0 +; RV32IF-NEXT: flt.s a3, fa5, fs0 +; RV32IF-NEXT: neg a2, a3 +; RV32IF-NEXT: or a0, a2, a0 +; RV32IF-NEXT: feq.s a2, fs0, fs0 +; RV32IF-NEXT: neg a2, a2 +; RV32IF-NEXT: lui a5, 524288 +; RV32IF-NEXT: li a6, 1 ; RV32IF-NEXT: lui a4, 524288 -; RV32IF-NEXT: lui a2, 524288 -; RV32IF-NEXT: beqz s0, .LBB9_4 +; RV32IF-NEXT: bne s0, a6, .LBB9_4 ; RV32IF-NEXT: # %bb.3: -; RV32IF-NEXT: mv a2, a1 +; RV32IF-NEXT: mv a4, a1 ; RV32IF-NEXT: .LBB9_4: -; RV32IF-NEXT: lui a1, %hi(.LCPI9_0) -; RV32IF-NEXT: flw fa5, %lo(.LCPI9_0)(a1) -; RV32IF-NEXT: flt.s a3, fa5, fs0 +; RV32IF-NEXT: and a0, a2, a0 ; RV32IF-NEXT: beqz a3, .LBB9_6 ; RV32IF-NEXT: # %bb.5: -; RV32IF-NEXT: addi a2, a4, -1 +; RV32IF-NEXT: addi a4, a5, -1 ; RV32IF-NEXT: .LBB9_6: -; RV32IF-NEXT: feq.s a1, fs0, fs0 -; RV32IF-NEXT: neg a4, a1 -; RV32IF-NEXT: and a1, a4, a2 -; RV32IF-NEXT: neg a2, s0 -; RV32IF-NEXT: and a0, a2, a0 -; RV32IF-NEXT: neg a2, a3 -; RV32IF-NEXT: or a0, a2, a0 -; RV32IF-NEXT: and a0, a4, a0 +; RV32IF-NEXT: and a1, a2, a4 ; RV32IF-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IF-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IF-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IF-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IF-NEXT: flw fs0, 0(sp) # 4-byte Folded Reload ; RV32IF-NEXT: addi sp, sp, 16 ; RV32IF-NEXT: ret ; @@ -635,23 +642,24 @@ define i64 @test_trunc_si64(float %x) nounwind { ; RV32IZFINX-NEXT: lui a2, %hi(.LCPI9_0) ; RV32IZFINX-NEXT: lw a2, %lo(.LCPI9_0)(a2) ; RV32IZFINX-NEXT: and a0, s2, a0 -; RV32IZFINX-NEXT: flt.s a4, a2, s0 -; RV32IZFINX-NEXT: neg a2, a4 +; RV32IZFINX-NEXT: flt.s a3, a2, s0 +; RV32IZFINX-NEXT: neg a2, a3 ; RV32IZFINX-NEXT: or a0, a2, a0 ; RV32IZFINX-NEXT: feq.s a2, s0, s0 ; RV32IZFINX-NEXT: neg a2, a2 ; RV32IZFINX-NEXT: lui a5, 524288 -; RV32IZFINX-NEXT: lui a3, 524288 -; RV32IZFINX-NEXT: beqz s1, .LBB9_4 +; RV32IZFINX-NEXT: li a6, 1 +; RV32IZFINX-NEXT: lui a4, 524288 +; RV32IZFINX-NEXT: bne s1, a6, .LBB9_4 ; RV32IZFINX-NEXT: # %bb.3: -; RV32IZFINX-NEXT: mv a3, a1 +; RV32IZFINX-NEXT: mv a4, a1 ; RV32IZFINX-NEXT: .LBB9_4: ; RV32IZFINX-NEXT: and a0, a2, a0 -; RV32IZFINX-NEXT: beqz a4, .LBB9_6 +; RV32IZFINX-NEXT: beqz a3, .LBB9_6 ; RV32IZFINX-NEXT: # %bb.5: -; RV32IZFINX-NEXT: addi a3, a5, -1 +; RV32IZFINX-NEXT: addi a4, a5, -1 ; RV32IZFINX-NEXT: .LBB9_6: -; RV32IZFINX-NEXT: and a1, a2, a3 +; RV32IZFINX-NEXT: and a1, a2, a4 ; RV32IZFINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload @@ -700,8 +708,7 @@ define i64 @test_trunc_ui64(float %x) nounwind { ; RV32IF: # %bb.0: ; RV32IF-NEXT: addi sp, sp, -16 ; RV32IF-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IF-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IF-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IF-NEXT: fsw fs0, 8(sp) # 4-byte Folded Spill ; RV32IF-NEXT: fmv.s fs0, fa0 ; RV32IF-NEXT: lui a0, 307200 ; RV32IF-NEXT: fmv.w.x fa5, a0 @@ -713,22 +720,22 @@ define i64 @test_trunc_ui64(float %x) nounwind { ; RV32IF-NEXT: fcvt.s.w fa5, a0, rtz ; RV32IF-NEXT: fsgnj.s fs0, fa5, fs0 ; RV32IF-NEXT: .LBB11_2: -; RV32IF-NEXT: fmv.w.x fa5, zero -; RV32IF-NEXT: fle.s a0, fa5, fs0 -; RV32IF-NEXT: neg s0, a0 ; RV32IF-NEXT: fmv.s fa0, fs0 ; RV32IF-NEXT: call __fixunssfdi -; RV32IF-NEXT: lui a2, %hi(.LCPI11_0) -; RV32IF-NEXT: flw fa5, %lo(.LCPI11_0)(a2) -; RV32IF-NEXT: and a0, s0, a0 -; RV32IF-NEXT: flt.s a2, fa5, fs0 -; RV32IF-NEXT: neg a2, a2 -; RV32IF-NEXT: or a0, a2, a0 -; RV32IF-NEXT: and a1, s0, a1 -; RV32IF-NEXT: or a1, a2, a1 +; RV32IF-NEXT: fmv.w.x fa5, zero +; RV32IF-NEXT: fle.s a2, fa5, fs0 +; RV32IF-NEXT: lui a3, %hi(.LCPI11_0) +; RV32IF-NEXT: flw fa5, %lo(.LCPI11_0)(a3) +; RV32IF-NEXT: xori a2, a2, 1 +; RV32IF-NEXT: addi a2, a2, -1 +; RV32IF-NEXT: and a0, a2, a0 +; RV32IF-NEXT: flt.s a3, fa5, fs0 +; RV32IF-NEXT: neg a3, a3 +; RV32IF-NEXT: or a0, a3, a0 +; RV32IF-NEXT: and a1, a2, a1 +; RV32IF-NEXT: or a1, a3, a1 ; RV32IF-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IF-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IF-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IF-NEXT: flw fs0, 8(sp) # 4-byte Folded Reload ; RV32IF-NEXT: addi sp, sp, 16 ; RV32IF-NEXT: ret ; @@ -746,7 +753,6 @@ define i64 @test_trunc_ui64(float %x) nounwind { ; RV32IZFINX-NEXT: addi sp, sp, -16 ; RV32IZFINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZFINX-NEXT: mv s0, a0 ; RV32IZFINX-NEXT: lui a0, 307200 ; RV32IZFINX-NEXT: fabs.s a1, s0 @@ -757,21 +763,21 @@ define i64 @test_trunc_ui64(float %x) nounwind { ; RV32IZFINX-NEXT: fcvt.s.w a0, a0, rtz ; RV32IZFINX-NEXT: fsgnj.s s0, a0, s0 ; RV32IZFINX-NEXT: .LBB11_2: -; RV32IZFINX-NEXT: fle.s a0, zero, s0 -; RV32IZFINX-NEXT: neg s1, a0 ; RV32IZFINX-NEXT: mv a0, s0 ; RV32IZFINX-NEXT: call __fixunssfdi -; RV32IZFINX-NEXT: lui a2, %hi(.LCPI11_0) -; RV32IZFINX-NEXT: lw a2, %lo(.LCPI11_0)(a2) -; RV32IZFINX-NEXT: and a0, s1, a0 -; RV32IZFINX-NEXT: flt.s a2, a2, s0 -; RV32IZFINX-NEXT: neg a2, a2 -; RV32IZFINX-NEXT: or a0, a2, a0 -; RV32IZFINX-NEXT: and a1, s1, a1 -; RV32IZFINX-NEXT: or a1, a2, a1 +; RV32IZFINX-NEXT: fle.s a2, zero, s0 +; RV32IZFINX-NEXT: lui a3, %hi(.LCPI11_0) +; RV32IZFINX-NEXT: lw a3, %lo(.LCPI11_0)(a3) +; RV32IZFINX-NEXT: xori a2, a2, 1 +; RV32IZFINX-NEXT: addi a2, a2, -1 +; RV32IZFINX-NEXT: and a0, a2, a0 +; RV32IZFINX-NEXT: flt.s a3, a3, s0 +; RV32IZFINX-NEXT: neg a3, a3 +; RV32IZFINX-NEXT: or a0, a3, a0 +; RV32IZFINX-NEXT: and a1, a2, a1 +; RV32IZFINX-NEXT: or a1, a3, a1 ; RV32IZFINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: addi sp, sp, 16 ; RV32IZFINX-NEXT: ret ; @@ -817,7 +823,8 @@ define i64 @test_round_si64(float %x) nounwind { ; RV32IF-NEXT: addi sp, sp, -16 ; RV32IF-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IF-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IF-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IF-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32IF-NEXT: fsw fs0, 0(sp) # 4-byte Folded Spill ; RV32IF-NEXT: fmv.s fs0, fa0 ; RV32IF-NEXT: lui a0, 307200 ; RV32IF-NEXT: fmv.w.x fa5, a0 @@ -832,32 +839,34 @@ define i64 @test_round_si64(float %x) nounwind { ; RV32IF-NEXT: lui a0, 913408 ; RV32IF-NEXT: fmv.w.x fa5, a0 ; RV32IF-NEXT: fle.s s0, fa5, fs0 +; RV32IF-NEXT: neg s1, s0 ; RV32IF-NEXT: fmv.s fa0, fs0 ; RV32IF-NEXT: call __fixsfdi +; RV32IF-NEXT: lui a2, %hi(.LCPI13_0) +; RV32IF-NEXT: flw fa5, %lo(.LCPI13_0)(a2) +; RV32IF-NEXT: and a0, s1, a0 +; RV32IF-NEXT: flt.s a3, fa5, fs0 +; RV32IF-NEXT: neg a2, a3 +; RV32IF-NEXT: or a0, a2, a0 +; RV32IF-NEXT: feq.s a2, fs0, fs0 +; RV32IF-NEXT: neg a2, a2 +; RV32IF-NEXT: lui a5, 524288 +; RV32IF-NEXT: li a6, 1 ; RV32IF-NEXT: lui a4, 524288 -; RV32IF-NEXT: lui a2, 524288 -; RV32IF-NEXT: beqz s0, .LBB13_4 +; RV32IF-NEXT: bne s0, a6, .LBB13_4 ; RV32IF-NEXT: # %bb.3: -; RV32IF-NEXT: mv a2, a1 +; RV32IF-NEXT: mv a4, a1 ; RV32IF-NEXT: .LBB13_4: -; RV32IF-NEXT: lui a1, %hi(.LCPI13_0) -; RV32IF-NEXT: flw fa5, %lo(.LCPI13_0)(a1) -; RV32IF-NEXT: flt.s a3, fa5, fs0 +; RV32IF-NEXT: and a0, a2, a0 ; RV32IF-NEXT: beqz a3, .LBB13_6 ; RV32IF-NEXT: # %bb.5: -; RV32IF-NEXT: addi a2, a4, -1 +; RV32IF-NEXT: addi a4, a5, -1 ; RV32IF-NEXT: .LBB13_6: -; RV32IF-NEXT: feq.s a1, fs0, fs0 -; RV32IF-NEXT: neg a4, a1 -; RV32IF-NEXT: and a1, a4, a2 -; RV32IF-NEXT: neg a2, s0 -; RV32IF-NEXT: and a0, a2, a0 -; RV32IF-NEXT: neg a2, a3 -; RV32IF-NEXT: or a0, a2, a0 -; RV32IF-NEXT: and a0, a4, a0 +; RV32IF-NEXT: and a1, a2, a4 ; RV32IF-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IF-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IF-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IF-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IF-NEXT: flw fs0, 0(sp) # 4-byte Folded Reload ; RV32IF-NEXT: addi sp, sp, 16 ; RV32IF-NEXT: ret ; @@ -895,23 +904,24 @@ define i64 @test_round_si64(float %x) nounwind { ; RV32IZFINX-NEXT: lui a2, %hi(.LCPI13_0) ; RV32IZFINX-NEXT: lw a2, %lo(.LCPI13_0)(a2) ; RV32IZFINX-NEXT: and a0, s2, a0 -; RV32IZFINX-NEXT: flt.s a4, a2, s0 -; RV32IZFINX-NEXT: neg a2, a4 +; RV32IZFINX-NEXT: flt.s a3, a2, s0 +; RV32IZFINX-NEXT: neg a2, a3 ; RV32IZFINX-NEXT: or a0, a2, a0 ; RV32IZFINX-NEXT: feq.s a2, s0, s0 ; RV32IZFINX-NEXT: neg a2, a2 ; RV32IZFINX-NEXT: lui a5, 524288 -; RV32IZFINX-NEXT: lui a3, 524288 -; RV32IZFINX-NEXT: beqz s1, .LBB13_4 +; RV32IZFINX-NEXT: li a6, 1 +; RV32IZFINX-NEXT: lui a4, 524288 +; RV32IZFINX-NEXT: bne s1, a6, .LBB13_4 ; RV32IZFINX-NEXT: # %bb.3: -; RV32IZFINX-NEXT: mv a3, a1 +; RV32IZFINX-NEXT: mv a4, a1 ; RV32IZFINX-NEXT: .LBB13_4: ; RV32IZFINX-NEXT: and a0, a2, a0 -; RV32IZFINX-NEXT: beqz a4, .LBB13_6 +; RV32IZFINX-NEXT: beqz a3, .LBB13_6 ; RV32IZFINX-NEXT: # %bb.5: -; RV32IZFINX-NEXT: addi a3, a5, -1 +; RV32IZFINX-NEXT: addi a4, a5, -1 ; RV32IZFINX-NEXT: .LBB13_6: -; RV32IZFINX-NEXT: and a1, a2, a3 +; RV32IZFINX-NEXT: and a1, a2, a4 ; RV32IZFINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload @@ -960,8 +970,7 @@ define i64 @test_round_ui64(float %x) nounwind { ; RV32IF: # %bb.0: ; RV32IF-NEXT: addi sp, sp, -16 ; RV32IF-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IF-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IF-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IF-NEXT: fsw fs0, 8(sp) # 4-byte Folded Spill ; RV32IF-NEXT: fmv.s fs0, fa0 ; RV32IF-NEXT: lui a0, 307200 ; RV32IF-NEXT: fmv.w.x fa5, a0 @@ -973,22 +982,22 @@ define i64 @test_round_ui64(float %x) nounwind { ; RV32IF-NEXT: fcvt.s.w fa5, a0, rmm ; RV32IF-NEXT: fsgnj.s fs0, fa5, fs0 ; RV32IF-NEXT: .LBB15_2: -; RV32IF-NEXT: fmv.w.x fa5, zero -; RV32IF-NEXT: fle.s a0, fa5, fs0 -; RV32IF-NEXT: neg s0, a0 ; RV32IF-NEXT: fmv.s fa0, fs0 ; RV32IF-NEXT: call __fixunssfdi -; RV32IF-NEXT: lui a2, %hi(.LCPI15_0) -; RV32IF-NEXT: flw fa5, %lo(.LCPI15_0)(a2) -; RV32IF-NEXT: and a0, s0, a0 -; RV32IF-NEXT: flt.s a2, fa5, fs0 -; RV32IF-NEXT: neg a2, a2 -; RV32IF-NEXT: or a0, a2, a0 -; RV32IF-NEXT: and a1, s0, a1 -; RV32IF-NEXT: or a1, a2, a1 +; RV32IF-NEXT: fmv.w.x fa5, zero +; RV32IF-NEXT: fle.s a2, fa5, fs0 +; RV32IF-NEXT: lui a3, %hi(.LCPI15_0) +; RV32IF-NEXT: flw fa5, %lo(.LCPI15_0)(a3) +; RV32IF-NEXT: xori a2, a2, 1 +; RV32IF-NEXT: addi a2, a2, -1 +; RV32IF-NEXT: and a0, a2, a0 +; RV32IF-NEXT: flt.s a3, fa5, fs0 +; RV32IF-NEXT: neg a3, a3 +; RV32IF-NEXT: or a0, a3, a0 +; RV32IF-NEXT: and a1, a2, a1 +; RV32IF-NEXT: or a1, a3, a1 ; RV32IF-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IF-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IF-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IF-NEXT: flw fs0, 8(sp) # 4-byte Folded Reload ; RV32IF-NEXT: addi sp, sp, 16 ; RV32IF-NEXT: ret ; @@ -1006,7 +1015,6 @@ define i64 @test_round_ui64(float %x) nounwind { ; RV32IZFINX-NEXT: addi sp, sp, -16 ; RV32IZFINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZFINX-NEXT: mv s0, a0 ; RV32IZFINX-NEXT: lui a0, 307200 ; RV32IZFINX-NEXT: fabs.s a1, s0 @@ -1017,21 +1025,21 @@ define i64 @test_round_ui64(float %x) nounwind { ; RV32IZFINX-NEXT: fcvt.s.w a0, a0, rmm ; RV32IZFINX-NEXT: fsgnj.s s0, a0, s0 ; RV32IZFINX-NEXT: .LBB15_2: -; RV32IZFINX-NEXT: fle.s a0, zero, s0 -; RV32IZFINX-NEXT: neg s1, a0 ; RV32IZFINX-NEXT: mv a0, s0 ; RV32IZFINX-NEXT: call __fixunssfdi -; RV32IZFINX-NEXT: lui a2, %hi(.LCPI15_0) -; RV32IZFINX-NEXT: lw a2, %lo(.LCPI15_0)(a2) -; RV32IZFINX-NEXT: and a0, s1, a0 -; RV32IZFINX-NEXT: flt.s a2, a2, s0 -; RV32IZFINX-NEXT: neg a2, a2 -; RV32IZFINX-NEXT: or a0, a2, a0 -; RV32IZFINX-NEXT: and a1, s1, a1 -; RV32IZFINX-NEXT: or a1, a2, a1 +; RV32IZFINX-NEXT: fle.s a2, zero, s0 +; RV32IZFINX-NEXT: lui a3, %hi(.LCPI15_0) +; RV32IZFINX-NEXT: lw a3, %lo(.LCPI15_0)(a3) +; RV32IZFINX-NEXT: xori a2, a2, 1 +; RV32IZFINX-NEXT: addi a2, a2, -1 +; RV32IZFINX-NEXT: and a0, a2, a0 +; RV32IZFINX-NEXT: flt.s a3, a3, s0 +; RV32IZFINX-NEXT: neg a3, a3 +; RV32IZFINX-NEXT: or a0, a3, a0 +; RV32IZFINX-NEXT: and a1, a2, a1 +; RV32IZFINX-NEXT: or a1, a3, a1 ; RV32IZFINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: addi sp, sp, 16 ; RV32IZFINX-NEXT: ret ; @@ -1077,7 +1085,8 @@ define i64 @test_roundeven_si64(float %x) nounwind { ; RV32IF-NEXT: addi sp, sp, -16 ; RV32IF-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IF-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IF-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IF-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32IF-NEXT: fsw fs0, 0(sp) # 4-byte Folded Spill ; RV32IF-NEXT: fmv.s fs0, fa0 ; RV32IF-NEXT: lui a0, 307200 ; RV32IF-NEXT: fmv.w.x fa5, a0 @@ -1092,32 +1101,34 @@ define i64 @test_roundeven_si64(float %x) nounwind { ; RV32IF-NEXT: lui a0, 913408 ; RV32IF-NEXT: fmv.w.x fa5, a0 ; RV32IF-NEXT: fle.s s0, fa5, fs0 +; RV32IF-NEXT: neg s1, s0 ; RV32IF-NEXT: fmv.s fa0, fs0 ; RV32IF-NEXT: call __fixsfdi +; RV32IF-NEXT: lui a2, %hi(.LCPI17_0) +; RV32IF-NEXT: flw fa5, %lo(.LCPI17_0)(a2) +; RV32IF-NEXT: and a0, s1, a0 +; RV32IF-NEXT: flt.s a3, fa5, fs0 +; RV32IF-NEXT: neg a2, a3 +; RV32IF-NEXT: or a0, a2, a0 +; RV32IF-NEXT: feq.s a2, fs0, fs0 +; RV32IF-NEXT: neg a2, a2 +; RV32IF-NEXT: lui a5, 524288 +; RV32IF-NEXT: li a6, 1 ; RV32IF-NEXT: lui a4, 524288 -; RV32IF-NEXT: lui a2, 524288 -; RV32IF-NEXT: beqz s0, .LBB17_4 +; RV32IF-NEXT: bne s0, a6, .LBB17_4 ; RV32IF-NEXT: # %bb.3: -; RV32IF-NEXT: mv a2, a1 +; RV32IF-NEXT: mv a4, a1 ; RV32IF-NEXT: .LBB17_4: -; RV32IF-NEXT: lui a1, %hi(.LCPI17_0) -; RV32IF-NEXT: flw fa5, %lo(.LCPI17_0)(a1) -; RV32IF-NEXT: flt.s a3, fa5, fs0 +; RV32IF-NEXT: and a0, a2, a0 ; RV32IF-NEXT: beqz a3, .LBB17_6 ; RV32IF-NEXT: # %bb.5: -; RV32IF-NEXT: addi a2, a4, -1 +; RV32IF-NEXT: addi a4, a5, -1 ; RV32IF-NEXT: .LBB17_6: -; RV32IF-NEXT: feq.s a1, fs0, fs0 -; RV32IF-NEXT: neg a4, a1 -; RV32IF-NEXT: and a1, a4, a2 -; RV32IF-NEXT: neg a2, s0 -; RV32IF-NEXT: and a0, a2, a0 -; RV32IF-NEXT: neg a2, a3 -; RV32IF-NEXT: or a0, a2, a0 -; RV32IF-NEXT: and a0, a4, a0 +; RV32IF-NEXT: and a1, a2, a4 ; RV32IF-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IF-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IF-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IF-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IF-NEXT: flw fs0, 0(sp) # 4-byte Folded Reload ; RV32IF-NEXT: addi sp, sp, 16 ; RV32IF-NEXT: ret ; @@ -1155,23 +1166,24 @@ define i64 @test_roundeven_si64(float %x) nounwind { ; RV32IZFINX-NEXT: lui a2, %hi(.LCPI17_0) ; RV32IZFINX-NEXT: lw a2, %lo(.LCPI17_0)(a2) ; RV32IZFINX-NEXT: and a0, s2, a0 -; RV32IZFINX-NEXT: flt.s a4, a2, s0 -; RV32IZFINX-NEXT: neg a2, a4 +; RV32IZFINX-NEXT: flt.s a3, a2, s0 +; RV32IZFINX-NEXT: neg a2, a3 ; RV32IZFINX-NEXT: or a0, a2, a0 ; RV32IZFINX-NEXT: feq.s a2, s0, s0 ; RV32IZFINX-NEXT: neg a2, a2 ; RV32IZFINX-NEXT: lui a5, 524288 -; RV32IZFINX-NEXT: lui a3, 524288 -; RV32IZFINX-NEXT: beqz s1, .LBB17_4 +; RV32IZFINX-NEXT: li a6, 1 +; RV32IZFINX-NEXT: lui a4, 524288 +; RV32IZFINX-NEXT: bne s1, a6, .LBB17_4 ; RV32IZFINX-NEXT: # %bb.3: -; RV32IZFINX-NEXT: mv a3, a1 +; RV32IZFINX-NEXT: mv a4, a1 ; RV32IZFINX-NEXT: .LBB17_4: ; RV32IZFINX-NEXT: and a0, a2, a0 -; RV32IZFINX-NEXT: beqz a4, .LBB17_6 +; RV32IZFINX-NEXT: beqz a3, .LBB17_6 ; RV32IZFINX-NEXT: # %bb.5: -; RV32IZFINX-NEXT: addi a3, a5, -1 +; RV32IZFINX-NEXT: addi a4, a5, -1 ; RV32IZFINX-NEXT: .LBB17_6: -; RV32IZFINX-NEXT: and a1, a2, a3 +; RV32IZFINX-NEXT: and a1, a2, a4 ; RV32IZFINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload @@ -1220,8 +1232,7 @@ define i64 @test_roundeven_ui64(float %x) nounwind { ; RV32IF: # %bb.0: ; RV32IF-NEXT: addi sp, sp, -16 ; RV32IF-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IF-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IF-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IF-NEXT: fsw fs0, 8(sp) # 4-byte Folded Spill ; RV32IF-NEXT: fmv.s fs0, fa0 ; RV32IF-NEXT: lui a0, 307200 ; RV32IF-NEXT: fmv.w.x fa5, a0 @@ -1233,22 +1244,22 @@ define i64 @test_roundeven_ui64(float %x) nounwind { ; RV32IF-NEXT: fcvt.s.w fa5, a0, rne ; RV32IF-NEXT: fsgnj.s fs0, fa5, fs0 ; RV32IF-NEXT: .LBB19_2: -; RV32IF-NEXT: fmv.w.x fa5, zero -; RV32IF-NEXT: fle.s a0, fa5, fs0 -; RV32IF-NEXT: neg s0, a0 ; RV32IF-NEXT: fmv.s fa0, fs0 ; RV32IF-NEXT: call __fixunssfdi -; RV32IF-NEXT: lui a2, %hi(.LCPI19_0) -; RV32IF-NEXT: flw fa5, %lo(.LCPI19_0)(a2) -; RV32IF-NEXT: and a0, s0, a0 -; RV32IF-NEXT: flt.s a2, fa5, fs0 -; RV32IF-NEXT: neg a2, a2 -; RV32IF-NEXT: or a0, a2, a0 -; RV32IF-NEXT: and a1, s0, a1 -; RV32IF-NEXT: or a1, a2, a1 +; RV32IF-NEXT: fmv.w.x fa5, zero +; RV32IF-NEXT: fle.s a2, fa5, fs0 +; RV32IF-NEXT: lui a3, %hi(.LCPI19_0) +; RV32IF-NEXT: flw fa5, %lo(.LCPI19_0)(a3) +; RV32IF-NEXT: xori a2, a2, 1 +; RV32IF-NEXT: addi a2, a2, -1 +; RV32IF-NEXT: and a0, a2, a0 +; RV32IF-NEXT: flt.s a3, fa5, fs0 +; RV32IF-NEXT: neg a3, a3 +; RV32IF-NEXT: or a0, a3, a0 +; RV32IF-NEXT: and a1, a2, a1 +; RV32IF-NEXT: or a1, a3, a1 ; RV32IF-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IF-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IF-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IF-NEXT: flw fs0, 8(sp) # 4-byte Folded Reload ; RV32IF-NEXT: addi sp, sp, 16 ; RV32IF-NEXT: ret ; @@ -1266,7 +1277,6 @@ define i64 @test_roundeven_ui64(float %x) nounwind { ; RV32IZFINX-NEXT: addi sp, sp, -16 ; RV32IZFINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZFINX-NEXT: mv s0, a0 ; RV32IZFINX-NEXT: lui a0, 307200 ; RV32IZFINX-NEXT: fabs.s a1, s0 @@ -1277,21 +1287,21 @@ define i64 @test_roundeven_ui64(float %x) nounwind { ; RV32IZFINX-NEXT: fcvt.s.w a0, a0, rne ; RV32IZFINX-NEXT: fsgnj.s s0, a0, s0 ; RV32IZFINX-NEXT: .LBB19_2: -; RV32IZFINX-NEXT: fle.s a0, zero, s0 -; RV32IZFINX-NEXT: neg s1, a0 ; RV32IZFINX-NEXT: mv a0, s0 ; RV32IZFINX-NEXT: call __fixunssfdi -; RV32IZFINX-NEXT: lui a2, %hi(.LCPI19_0) -; RV32IZFINX-NEXT: lw a2, %lo(.LCPI19_0)(a2) -; RV32IZFINX-NEXT: and a0, s1, a0 -; RV32IZFINX-NEXT: flt.s a2, a2, s0 -; RV32IZFINX-NEXT: neg a2, a2 -; RV32IZFINX-NEXT: or a0, a2, a0 -; RV32IZFINX-NEXT: and a1, s1, a1 -; RV32IZFINX-NEXT: or a1, a2, a1 +; RV32IZFINX-NEXT: fle.s a2, zero, s0 +; RV32IZFINX-NEXT: lui a3, %hi(.LCPI19_0) +; RV32IZFINX-NEXT: lw a3, %lo(.LCPI19_0)(a3) +; RV32IZFINX-NEXT: xori a2, a2, 1 +; RV32IZFINX-NEXT: addi a2, a2, -1 +; RV32IZFINX-NEXT: and a0, a2, a0 +; RV32IZFINX-NEXT: flt.s a3, a3, s0 +; RV32IZFINX-NEXT: neg a3, a3 +; RV32IZFINX-NEXT: or a0, a3, a0 +; RV32IZFINX-NEXT: and a1, a2, a1 +; RV32IZFINX-NEXT: or a1, a3, a1 ; RV32IZFINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: addi sp, sp, 16 ; RV32IZFINX-NEXT: ret ; @@ -1337,7 +1347,8 @@ define i64 @test_rint_si64(float %x) nounwind { ; RV32IF-NEXT: addi sp, sp, -16 ; RV32IF-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IF-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IF-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IF-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32IF-NEXT: fsw fs0, 0(sp) # 4-byte Folded Spill ; RV32IF-NEXT: fmv.s fs0, fa0 ; RV32IF-NEXT: lui a0, 307200 ; RV32IF-NEXT: fmv.w.x fa5, a0 @@ -1352,32 +1363,34 @@ define i64 @test_rint_si64(float %x) nounwind { ; RV32IF-NEXT: lui a0, 913408 ; RV32IF-NEXT: fmv.w.x fa5, a0 ; RV32IF-NEXT: fle.s s0, fa5, fs0 +; RV32IF-NEXT: neg s1, s0 ; RV32IF-NEXT: fmv.s fa0, fs0 ; RV32IF-NEXT: call __fixsfdi +; RV32IF-NEXT: lui a2, %hi(.LCPI21_0) +; RV32IF-NEXT: flw fa5, %lo(.LCPI21_0)(a2) +; RV32IF-NEXT: and a0, s1, a0 +; RV32IF-NEXT: flt.s a3, fa5, fs0 +; RV32IF-NEXT: neg a2, a3 +; RV32IF-NEXT: or a0, a2, a0 +; RV32IF-NEXT: feq.s a2, fs0, fs0 +; RV32IF-NEXT: neg a2, a2 +; RV32IF-NEXT: lui a5, 524288 +; RV32IF-NEXT: li a6, 1 ; RV32IF-NEXT: lui a4, 524288 -; RV32IF-NEXT: lui a2, 524288 -; RV32IF-NEXT: beqz s0, .LBB21_4 +; RV32IF-NEXT: bne s0, a6, .LBB21_4 ; RV32IF-NEXT: # %bb.3: -; RV32IF-NEXT: mv a2, a1 +; RV32IF-NEXT: mv a4, a1 ; RV32IF-NEXT: .LBB21_4: -; RV32IF-NEXT: lui a1, %hi(.LCPI21_0) -; RV32IF-NEXT: flw fa5, %lo(.LCPI21_0)(a1) -; RV32IF-NEXT: flt.s a3, fa5, fs0 +; RV32IF-NEXT: and a0, a2, a0 ; RV32IF-NEXT: beqz a3, .LBB21_6 ; RV32IF-NEXT: # %bb.5: -; RV32IF-NEXT: addi a2, a4, -1 +; RV32IF-NEXT: addi a4, a5, -1 ; RV32IF-NEXT: .LBB21_6: -; RV32IF-NEXT: feq.s a1, fs0, fs0 -; RV32IF-NEXT: neg a4, a1 -; RV32IF-NEXT: and a1, a4, a2 -; RV32IF-NEXT: neg a2, s0 -; RV32IF-NEXT: and a0, a2, a0 -; RV32IF-NEXT: neg a2, a3 -; RV32IF-NEXT: or a0, a2, a0 -; RV32IF-NEXT: and a0, a4, a0 +; RV32IF-NEXT: and a1, a2, a4 ; RV32IF-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IF-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IF-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IF-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IF-NEXT: flw fs0, 0(sp) # 4-byte Folded Reload ; RV32IF-NEXT: addi sp, sp, 16 ; RV32IF-NEXT: ret ; @@ -1415,23 +1428,24 @@ define i64 @test_rint_si64(float %x) nounwind { ; RV32IZFINX-NEXT: lui a2, %hi(.LCPI21_0) ; RV32IZFINX-NEXT: lw a2, %lo(.LCPI21_0)(a2) ; RV32IZFINX-NEXT: and a0, s2, a0 -; RV32IZFINX-NEXT: flt.s a4, a2, s0 -; RV32IZFINX-NEXT: neg a2, a4 +; RV32IZFINX-NEXT: flt.s a3, a2, s0 +; RV32IZFINX-NEXT: neg a2, a3 ; RV32IZFINX-NEXT: or a0, a2, a0 ; RV32IZFINX-NEXT: feq.s a2, s0, s0 ; RV32IZFINX-NEXT: neg a2, a2 ; RV32IZFINX-NEXT: lui a5, 524288 -; RV32IZFINX-NEXT: lui a3, 524288 -; RV32IZFINX-NEXT: beqz s1, .LBB21_4 +; RV32IZFINX-NEXT: li a6, 1 +; RV32IZFINX-NEXT: lui a4, 524288 +; RV32IZFINX-NEXT: bne s1, a6, .LBB21_4 ; RV32IZFINX-NEXT: # %bb.3: -; RV32IZFINX-NEXT: mv a3, a1 +; RV32IZFINX-NEXT: mv a4, a1 ; RV32IZFINX-NEXT: .LBB21_4: ; RV32IZFINX-NEXT: and a0, a2, a0 -; RV32IZFINX-NEXT: beqz a4, .LBB21_6 +; RV32IZFINX-NEXT: beqz a3, .LBB21_6 ; RV32IZFINX-NEXT: # %bb.5: -; RV32IZFINX-NEXT: addi a3, a5, -1 +; RV32IZFINX-NEXT: addi a4, a5, -1 ; RV32IZFINX-NEXT: .LBB21_6: -; RV32IZFINX-NEXT: and a1, a2, a3 +; RV32IZFINX-NEXT: and a1, a2, a4 ; RV32IZFINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload @@ -1480,8 +1494,7 @@ define i64 @test_rint_ui64(float %x) nounwind { ; RV32IF: # %bb.0: ; RV32IF-NEXT: addi sp, sp, -16 ; RV32IF-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IF-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IF-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IF-NEXT: fsw fs0, 8(sp) # 4-byte Folded Spill ; RV32IF-NEXT: fmv.s fs0, fa0 ; RV32IF-NEXT: lui a0, 307200 ; RV32IF-NEXT: fmv.w.x fa5, a0 @@ -1493,22 +1506,22 @@ define i64 @test_rint_ui64(float %x) nounwind { ; RV32IF-NEXT: fcvt.s.w fa5, a0 ; RV32IF-NEXT: fsgnj.s fs0, fa5, fs0 ; RV32IF-NEXT: .LBB23_2: -; RV32IF-NEXT: fmv.w.x fa5, zero -; RV32IF-NEXT: fle.s a0, fa5, fs0 -; RV32IF-NEXT: neg s0, a0 ; RV32IF-NEXT: fmv.s fa0, fs0 ; RV32IF-NEXT: call __fixunssfdi -; RV32IF-NEXT: lui a2, %hi(.LCPI23_0) -; RV32IF-NEXT: flw fa5, %lo(.LCPI23_0)(a2) -; RV32IF-NEXT: and a0, s0, a0 -; RV32IF-NEXT: flt.s a2, fa5, fs0 -; RV32IF-NEXT: neg a2, a2 -; RV32IF-NEXT: or a0, a2, a0 -; RV32IF-NEXT: and a1, s0, a1 -; RV32IF-NEXT: or a1, a2, a1 +; RV32IF-NEXT: fmv.w.x fa5, zero +; RV32IF-NEXT: fle.s a2, fa5, fs0 +; RV32IF-NEXT: lui a3, %hi(.LCPI23_0) +; RV32IF-NEXT: flw fa5, %lo(.LCPI23_0)(a3) +; RV32IF-NEXT: xori a2, a2, 1 +; RV32IF-NEXT: addi a2, a2, -1 +; RV32IF-NEXT: and a0, a2, a0 +; RV32IF-NEXT: flt.s a3, fa5, fs0 +; RV32IF-NEXT: neg a3, a3 +; RV32IF-NEXT: or a0, a3, a0 +; RV32IF-NEXT: and a1, a2, a1 +; RV32IF-NEXT: or a1, a3, a1 ; RV32IF-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IF-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IF-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IF-NEXT: flw fs0, 8(sp) # 4-byte Folded Reload ; RV32IF-NEXT: addi sp, sp, 16 ; RV32IF-NEXT: ret ; @@ -1526,7 +1539,6 @@ define i64 @test_rint_ui64(float %x) nounwind { ; RV32IZFINX-NEXT: addi sp, sp, -16 ; RV32IZFINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZFINX-NEXT: mv s0, a0 ; RV32IZFINX-NEXT: lui a0, 307200 ; RV32IZFINX-NEXT: fabs.s a1, s0 @@ -1537,21 +1549,21 @@ define i64 @test_rint_ui64(float %x) nounwind { ; RV32IZFINX-NEXT: fcvt.s.w a0, a0 ; RV32IZFINX-NEXT: fsgnj.s s0, a0, s0 ; RV32IZFINX-NEXT: .LBB23_2: -; RV32IZFINX-NEXT: fle.s a0, zero, s0 -; RV32IZFINX-NEXT: neg s1, a0 ; RV32IZFINX-NEXT: mv a0, s0 ; RV32IZFINX-NEXT: call __fixunssfdi -; RV32IZFINX-NEXT: lui a2, %hi(.LCPI23_0) -; RV32IZFINX-NEXT: lw a2, %lo(.LCPI23_0)(a2) -; RV32IZFINX-NEXT: and a0, s1, a0 -; RV32IZFINX-NEXT: flt.s a2, a2, s0 -; RV32IZFINX-NEXT: neg a2, a2 -; RV32IZFINX-NEXT: or a0, a2, a0 -; RV32IZFINX-NEXT: and a1, s1, a1 -; RV32IZFINX-NEXT: or a1, a2, a1 +; RV32IZFINX-NEXT: fle.s a2, zero, s0 +; RV32IZFINX-NEXT: lui a3, %hi(.LCPI23_0) +; RV32IZFINX-NEXT: lw a3, %lo(.LCPI23_0)(a3) +; RV32IZFINX-NEXT: xori a2, a2, 1 +; RV32IZFINX-NEXT: addi a2, a2, -1 +; RV32IZFINX-NEXT: and a0, a2, a0 +; RV32IZFINX-NEXT: flt.s a3, a3, s0 +; RV32IZFINX-NEXT: neg a3, a3 +; RV32IZFINX-NEXT: or a0, a3, a0 +; RV32IZFINX-NEXT: and a1, a2, a1 +; RV32IZFINX-NEXT: or a1, a3, a1 ; RV32IZFINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZFINX-NEXT: addi sp, sp, 16 ; RV32IZFINX-NEXT: ret ; diff --git a/llvm/test/CodeGen/RISCV/forced-atomics.ll b/llvm/test/CodeGen/RISCV/forced-atomics.ll index f6a53a9d76dd..2b198afb47a9 100644 --- a/llvm/test/CodeGen/RISCV/forced-atomics.ll +++ b/llvm/test/CodeGen/RISCV/forced-atomics.ll @@ -3567,8 +3567,8 @@ define i64 @rmw64_umax_seq_cst(ptr %p) nounwind { ; RV32-NEXT: # in Loop: Header=BB51_2 Depth=1 ; RV32-NEXT: neg a3, a0 ; RV32-NEXT: and a3, a3, a1 -; RV32-NEXT: sw a4, 0(sp) ; RV32-NEXT: sw a1, 4(sp) +; RV32-NEXT: sw a4, 0(sp) ; RV32-NEXT: mv a1, sp ; RV32-NEXT: li a4, 5 ; RV32-NEXT: li a5, 5 @@ -3672,7 +3672,8 @@ define i64 @rmw64_umin_seq_cst(ptr %p) nounwind { ; RV32-NEXT: .LBB52_2: # %atomicrmw.start ; RV32-NEXT: # =>This Inner Loop Header: Depth=1 ; RV32-NEXT: sltiu a0, a4, 2 -; RV32-NEXT: seqz a2, a1 +; RV32-NEXT: snez a2, a1 +; RV32-NEXT: addi a2, a2, -1 ; RV32-NEXT: and a0, a2, a0 ; RV32-NEXT: mv a2, a4 ; RV32-NEXT: bnez a0, .LBB52_1 diff --git a/llvm/test/CodeGen/RISCV/fpclamptosat.ll b/llvm/test/CodeGen/RISCV/fpclamptosat.ll index 9e93ad0043a7..6bfacc3e9814 100644 --- a/llvm/test/CodeGen/RISCV/fpclamptosat.ll +++ b/llvm/test/CodeGen/RISCV/fpclamptosat.ll @@ -115,7 +115,8 @@ define i32 @utest_f64i32(double %x) { ; RV32IF-NEXT: .cfi_offset ra, -4 ; RV32IF-NEXT: call __fixunsdfdi ; RV32IF-NEXT: sltiu a2, a0, -1 -; RV32IF-NEXT: seqz a1, a1 +; RV32IF-NEXT: snez a1, a1 +; RV32IF-NEXT: addi a1, a1, -1 ; RV32IF-NEXT: and a1, a1, a2 ; RV32IF-NEXT: addi a1, a1, -1 ; RV32IF-NEXT: or a0, a1, a0 @@ -430,7 +431,8 @@ define i32 @utesth_f16i32(half %x) { ; RV32-NEXT: call __extendhfsf2 ; RV32-NEXT: call __fixunssfdi ; RV32-NEXT: sltiu a2, a0, -1 -; RV32-NEXT: seqz a1, a1 +; RV32-NEXT: snez a1, a1 +; RV32-NEXT: addi a1, a1, -1 ; RV32-NEXT: and a1, a1, a2 ; RV32-NEXT: addi a1, a1, -1 ; RV32-NEXT: or a0, a1, a0 @@ -1043,8 +1045,8 @@ define i64 @stest_f64i64(double %x) { ; RV32IF-NEXT: mv a1, a0 ; RV32IF-NEXT: addi a0, sp, 8 ; RV32IF-NEXT: call __fixdfti -; RV32IF-NEXT: lw a0, 20(sp) -; RV32IF-NEXT: lw a2, 16(sp) +; RV32IF-NEXT: lw a0, 16(sp) +; RV32IF-NEXT: lw a2, 20(sp) ; RV32IF-NEXT: lw a1, 12(sp) ; RV32IF-NEXT: lw a4, 8(sp) ; RV32IF-NEXT: lui a3, 524288 @@ -1052,25 +1054,25 @@ define i64 @stest_f64i64(double %x) { ; RV32IF-NEXT: beq a1, a5, .LBB18_2 ; RV32IF-NEXT: # %bb.1: # %entry ; RV32IF-NEXT: sltu a6, a1, a5 -; RV32IF-NEXT: or a7, a2, a0 +; RV32IF-NEXT: or a7, a0, a2 ; RV32IF-NEXT: bnez a7, .LBB18_3 ; RV32IF-NEXT: j .LBB18_4 ; RV32IF-NEXT: .LBB18_2: ; RV32IF-NEXT: sltiu a6, a4, -1 -; RV32IF-NEXT: or a7, a2, a0 +; RV32IF-NEXT: or a7, a0, a2 ; RV32IF-NEXT: beqz a7, .LBB18_4 ; RV32IF-NEXT: .LBB18_3: # %entry -; RV32IF-NEXT: slti a6, a0, 0 +; RV32IF-NEXT: slti a6, a2, 0 ; RV32IF-NEXT: .LBB18_4: # %entry -; RV32IF-NEXT: neg a7, a6 -; RV32IF-NEXT: addi t0, a6, -1 +; RV32IF-NEXT: addi a7, a6, -1 +; RV32IF-NEXT: neg t0, a6 ; RV32IF-NEXT: bnez a6, .LBB18_6 ; RV32IF-NEXT: # %bb.5: # %entry ; RV32IF-NEXT: mv a1, a5 ; RV32IF-NEXT: .LBB18_6: # %entry -; RV32IF-NEXT: or a4, t0, a4 -; RV32IF-NEXT: and a5, a7, a0 -; RV32IF-NEXT: and a2, a7, a2 +; RV32IF-NEXT: or a4, a7, a4 +; RV32IF-NEXT: and a2, t0, a2 +; RV32IF-NEXT: and a5, t0, a0 ; RV32IF-NEXT: beq a1, a3, .LBB18_8 ; RV32IF-NEXT: # %bb.7: # %entry ; RV32IF-NEXT: sltu a0, a3, a1 @@ -1078,11 +1080,11 @@ define i64 @stest_f64i64(double %x) { ; RV32IF-NEXT: .LBB18_8: ; RV32IF-NEXT: snez a0, a4 ; RV32IF-NEXT: .LBB18_9: # %entry -; RV32IF-NEXT: and a2, a2, a5 +; RV32IF-NEXT: and a5, a5, a2 ; RV32IF-NEXT: li a3, -1 -; RV32IF-NEXT: beq a2, a3, .LBB18_11 +; RV32IF-NEXT: beq a5, a3, .LBB18_11 ; RV32IF-NEXT: # %bb.10: # %entry -; RV32IF-NEXT: slti a0, a5, 0 +; RV32IF-NEXT: slti a0, a2, 0 ; RV32IF-NEXT: xori a0, a0, 1 ; RV32IF-NEXT: .LBB18_11: # %entry ; RV32IF-NEXT: bnez a0, .LBB18_13 @@ -1142,8 +1144,8 @@ define i64 @stest_f64i64(double %x) { ; RV32IFD-NEXT: .cfi_offset ra, -4 ; RV32IFD-NEXT: addi a0, sp, 8 ; RV32IFD-NEXT: call __fixdfti -; RV32IFD-NEXT: lw a0, 20(sp) -; RV32IFD-NEXT: lw a2, 16(sp) +; RV32IFD-NEXT: lw a0, 16(sp) +; RV32IFD-NEXT: lw a2, 20(sp) ; RV32IFD-NEXT: lw a1, 12(sp) ; RV32IFD-NEXT: lw a4, 8(sp) ; RV32IFD-NEXT: lui a3, 524288 @@ -1151,25 +1153,25 @@ define i64 @stest_f64i64(double %x) { ; RV32IFD-NEXT: beq a1, a5, .LBB18_2 ; RV32IFD-NEXT: # %bb.1: # %entry ; RV32IFD-NEXT: sltu a6, a1, a5 -; RV32IFD-NEXT: or a7, a2, a0 +; RV32IFD-NEXT: or a7, a0, a2 ; RV32IFD-NEXT: bnez a7, .LBB18_3 ; RV32IFD-NEXT: j .LBB18_4 ; RV32IFD-NEXT: .LBB18_2: ; RV32IFD-NEXT: sltiu a6, a4, -1 -; RV32IFD-NEXT: or a7, a2, a0 +; RV32IFD-NEXT: or a7, a0, a2 ; RV32IFD-NEXT: beqz a7, .LBB18_4 ; RV32IFD-NEXT: .LBB18_3: # %entry -; RV32IFD-NEXT: slti a6, a0, 0 +; RV32IFD-NEXT: slti a6, a2, 0 ; RV32IFD-NEXT: .LBB18_4: # %entry -; RV32IFD-NEXT: neg a7, a6 -; RV32IFD-NEXT: addi t0, a6, -1 +; RV32IFD-NEXT: addi a7, a6, -1 +; RV32IFD-NEXT: neg t0, a6 ; RV32IFD-NEXT: bnez a6, .LBB18_6 ; RV32IFD-NEXT: # %bb.5: # %entry ; RV32IFD-NEXT: mv a1, a5 ; RV32IFD-NEXT: .LBB18_6: # %entry -; RV32IFD-NEXT: or a4, t0, a4 -; RV32IFD-NEXT: and a5, a7, a0 -; RV32IFD-NEXT: and a2, a7, a2 +; RV32IFD-NEXT: or a4, a7, a4 +; RV32IFD-NEXT: and a2, t0, a2 +; RV32IFD-NEXT: and a5, t0, a0 ; RV32IFD-NEXT: beq a1, a3, .LBB18_8 ; RV32IFD-NEXT: # %bb.7: # %entry ; RV32IFD-NEXT: sltu a0, a3, a1 @@ -1177,11 +1179,11 @@ define i64 @stest_f64i64(double %x) { ; RV32IFD-NEXT: .LBB18_8: ; RV32IFD-NEXT: snez a0, a4 ; RV32IFD-NEXT: .LBB18_9: # %entry -; RV32IFD-NEXT: and a2, a2, a5 +; RV32IFD-NEXT: and a5, a5, a2 ; RV32IFD-NEXT: li a3, -1 -; RV32IFD-NEXT: beq a2, a3, .LBB18_11 +; RV32IFD-NEXT: beq a5, a3, .LBB18_11 ; RV32IFD-NEXT: # %bb.10: # %entry -; RV32IFD-NEXT: slti a0, a5, 0 +; RV32IFD-NEXT: slti a0, a2, 0 ; RV32IFD-NEXT: xori a0, a0, 1 ; RV32IFD-NEXT: .LBB18_11: # %entry ; RV32IFD-NEXT: bnez a0, .LBB18_13 @@ -1227,8 +1229,10 @@ define i64 @utest_f64i64(double %x) { ; RV32IF-NEXT: lw a1, 20(sp) ; RV32IF-NEXT: lw a2, 12(sp) ; RV32IF-NEXT: lw a3, 8(sp) -; RV32IF-NEXT: or a4, a1, a0 -; RV32IF-NEXT: seqz a4, a4 +; RV32IF-NEXT: seqz a4, a0 +; RV32IF-NEXT: snez a5, a1 +; RV32IF-NEXT: addi a5, a5, -1 +; RV32IF-NEXT: and a4, a5, a4 ; RV32IF-NEXT: xori a0, a0, 1 ; RV32IF-NEXT: or a0, a0, a1 ; RV32IF-NEXT: seqz a0, a0 @@ -1267,8 +1271,10 @@ define i64 @utest_f64i64(double %x) { ; RV32IFD-NEXT: lw a1, 20(sp) ; RV32IFD-NEXT: lw a2, 12(sp) ; RV32IFD-NEXT: lw a3, 8(sp) -; RV32IFD-NEXT: or a4, a1, a0 -; RV32IFD-NEXT: seqz a4, a4 +; RV32IFD-NEXT: seqz a4, a0 +; RV32IFD-NEXT: snez a5, a1 +; RV32IFD-NEXT: addi a5, a5, -1 +; RV32IFD-NEXT: and a4, a5, a4 ; RV32IFD-NEXT: xori a0, a0, 1 ; RV32IFD-NEXT: or a0, a0, a1 ; RV32IFD-NEXT: seqz a0, a0 @@ -1440,8 +1446,8 @@ define i64 @stest_f32i64(float %x) { ; RV32-NEXT: .cfi_offset ra, -4 ; RV32-NEXT: addi a0, sp, 8 ; RV32-NEXT: call __fixsfti -; RV32-NEXT: lw a0, 20(sp) -; RV32-NEXT: lw a2, 16(sp) +; RV32-NEXT: lw a0, 16(sp) +; RV32-NEXT: lw a2, 20(sp) ; RV32-NEXT: lw a1, 12(sp) ; RV32-NEXT: lw a4, 8(sp) ; RV32-NEXT: lui a3, 524288 @@ -1449,25 +1455,25 @@ define i64 @stest_f32i64(float %x) { ; RV32-NEXT: beq a1, a5, .LBB21_2 ; RV32-NEXT: # %bb.1: # %entry ; RV32-NEXT: sltu a6, a1, a5 -; RV32-NEXT: or a7, a2, a0 +; RV32-NEXT: or a7, a0, a2 ; RV32-NEXT: bnez a7, .LBB21_3 ; RV32-NEXT: j .LBB21_4 ; RV32-NEXT: .LBB21_2: ; RV32-NEXT: sltiu a6, a4, -1 -; RV32-NEXT: or a7, a2, a0 +; RV32-NEXT: or a7, a0, a2 ; RV32-NEXT: beqz a7, .LBB21_4 ; RV32-NEXT: .LBB21_3: # %entry -; RV32-NEXT: slti a6, a0, 0 +; RV32-NEXT: slti a6, a2, 0 ; RV32-NEXT: .LBB21_4: # %entry -; RV32-NEXT: neg a7, a6 -; RV32-NEXT: addi t0, a6, -1 +; RV32-NEXT: addi a7, a6, -1 +; RV32-NEXT: neg t0, a6 ; RV32-NEXT: bnez a6, .LBB21_6 ; RV32-NEXT: # %bb.5: # %entry ; RV32-NEXT: mv a1, a5 ; RV32-NEXT: .LBB21_6: # %entry -; RV32-NEXT: or a4, t0, a4 -; RV32-NEXT: and a5, a7, a0 -; RV32-NEXT: and a2, a7, a2 +; RV32-NEXT: or a4, a7, a4 +; RV32-NEXT: and a2, t0, a2 +; RV32-NEXT: and a5, t0, a0 ; RV32-NEXT: beq a1, a3, .LBB21_8 ; RV32-NEXT: # %bb.7: # %entry ; RV32-NEXT: sltu a0, a3, a1 @@ -1475,11 +1481,11 @@ define i64 @stest_f32i64(float %x) { ; RV32-NEXT: .LBB21_8: ; RV32-NEXT: snez a0, a4 ; RV32-NEXT: .LBB21_9: # %entry -; RV32-NEXT: and a2, a2, a5 +; RV32-NEXT: and a5, a5, a2 ; RV32-NEXT: li a3, -1 -; RV32-NEXT: beq a2, a3, .LBB21_11 +; RV32-NEXT: beq a5, a3, .LBB21_11 ; RV32-NEXT: # %bb.10: # %entry -; RV32-NEXT: slti a0, a5, 0 +; RV32-NEXT: slti a0, a2, 0 ; RV32-NEXT: xori a0, a0, 1 ; RV32-NEXT: .LBB21_11: # %entry ; RV32-NEXT: bnez a0, .LBB21_13 @@ -1523,8 +1529,10 @@ define i64 @utest_f32i64(float %x) { ; RV32-NEXT: lw a1, 20(sp) ; RV32-NEXT: lw a2, 12(sp) ; RV32-NEXT: lw a3, 8(sp) -; RV32-NEXT: or a4, a1, a0 -; RV32-NEXT: seqz a4, a4 +; RV32-NEXT: seqz a4, a0 +; RV32-NEXT: snez a5, a1 +; RV32-NEXT: addi a5, a5, -1 +; RV32-NEXT: and a4, a5, a4 ; RV32-NEXT: xori a0, a0, 1 ; RV32-NEXT: or a0, a0, a1 ; RV32-NEXT: seqz a0, a0 @@ -1657,8 +1665,8 @@ define i64 @stest_f16i64(half %x) { ; RV32-NEXT: call __extendhfsf2 ; RV32-NEXT: addi a0, sp, 8 ; RV32-NEXT: call __fixsfti -; RV32-NEXT: lw a0, 20(sp) -; RV32-NEXT: lw a2, 16(sp) +; RV32-NEXT: lw a0, 16(sp) +; RV32-NEXT: lw a2, 20(sp) ; RV32-NEXT: lw a1, 12(sp) ; RV32-NEXT: lw a4, 8(sp) ; RV32-NEXT: lui a3, 524288 @@ -1666,25 +1674,25 @@ define i64 @stest_f16i64(half %x) { ; RV32-NEXT: beq a1, a5, .LBB24_2 ; RV32-NEXT: # %bb.1: # %entry ; RV32-NEXT: sltu a6, a1, a5 -; RV32-NEXT: or a7, a2, a0 +; RV32-NEXT: or a7, a0, a2 ; RV32-NEXT: bnez a7, .LBB24_3 ; RV32-NEXT: j .LBB24_4 ; RV32-NEXT: .LBB24_2: ; RV32-NEXT: sltiu a6, a4, -1 -; RV32-NEXT: or a7, a2, a0 +; RV32-NEXT: or a7, a0, a2 ; RV32-NEXT: beqz a7, .LBB24_4 ; RV32-NEXT: .LBB24_3: # %entry -; RV32-NEXT: slti a6, a0, 0 +; RV32-NEXT: slti a6, a2, 0 ; RV32-NEXT: .LBB24_4: # %entry -; RV32-NEXT: neg a7, a6 -; RV32-NEXT: addi t0, a6, -1 +; RV32-NEXT: addi a7, a6, -1 +; RV32-NEXT: neg t0, a6 ; RV32-NEXT: bnez a6, .LBB24_6 ; RV32-NEXT: # %bb.5: # %entry ; RV32-NEXT: mv a1, a5 ; RV32-NEXT: .LBB24_6: # %entry -; RV32-NEXT: or a4, t0, a4 -; RV32-NEXT: and a5, a7, a0 -; RV32-NEXT: and a2, a7, a2 +; RV32-NEXT: or a4, a7, a4 +; RV32-NEXT: and a2, t0, a2 +; RV32-NEXT: and a5, t0, a0 ; RV32-NEXT: beq a1, a3, .LBB24_8 ; RV32-NEXT: # %bb.7: # %entry ; RV32-NEXT: sltu a0, a3, a1 @@ -1692,11 +1700,11 @@ define i64 @stest_f16i64(half %x) { ; RV32-NEXT: .LBB24_8: ; RV32-NEXT: snez a0, a4 ; RV32-NEXT: .LBB24_9: # %entry -; RV32-NEXT: and a2, a2, a5 +; RV32-NEXT: and a5, a5, a2 ; RV32-NEXT: li a3, -1 -; RV32-NEXT: beq a2, a3, .LBB24_11 +; RV32-NEXT: beq a5, a3, .LBB24_11 ; RV32-NEXT: # %bb.10: # %entry -; RV32-NEXT: slti a0, a5, 0 +; RV32-NEXT: slti a0, a2, 0 ; RV32-NEXT: xori a0, a0, 1 ; RV32-NEXT: .LBB24_11: # %entry ; RV32-NEXT: bnez a0, .LBB24_13 @@ -1772,8 +1780,10 @@ define i64 @utesth_f16i64(half %x) { ; RV32-NEXT: lw a1, 20(sp) ; RV32-NEXT: lw a2, 12(sp) ; RV32-NEXT: lw a3, 8(sp) -; RV32-NEXT: or a4, a1, a0 -; RV32-NEXT: seqz a4, a4 +; RV32-NEXT: seqz a4, a0 +; RV32-NEXT: snez a5, a1 +; RV32-NEXT: addi a5, a5, -1 +; RV32-NEXT: and a4, a5, a4 ; RV32-NEXT: xori a0, a0, 1 ; RV32-NEXT: or a0, a0, a1 ; RV32-NEXT: seqz a0, a0 @@ -2891,8 +2901,8 @@ define i64 @stest_f64i64_mm(double %x) { ; RV32IF-NEXT: mv a1, a0 ; RV32IF-NEXT: addi a0, sp, 8 ; RV32IF-NEXT: call __fixdfti -; RV32IF-NEXT: lw a0, 20(sp) -; RV32IF-NEXT: lw a2, 16(sp) +; RV32IF-NEXT: lw a0, 16(sp) +; RV32IF-NEXT: lw a2, 20(sp) ; RV32IF-NEXT: lw a1, 12(sp) ; RV32IF-NEXT: lw a4, 8(sp) ; RV32IF-NEXT: lui a3, 524288 @@ -2900,25 +2910,25 @@ define i64 @stest_f64i64_mm(double %x) { ; RV32IF-NEXT: beq a1, a5, .LBB45_2 ; RV32IF-NEXT: # %bb.1: # %entry ; RV32IF-NEXT: sltu a6, a1, a5 -; RV32IF-NEXT: or a7, a2, a0 +; RV32IF-NEXT: or a7, a0, a2 ; RV32IF-NEXT: bnez a7, .LBB45_3 ; RV32IF-NEXT: j .LBB45_4 ; RV32IF-NEXT: .LBB45_2: ; RV32IF-NEXT: sltiu a6, a4, -1 -; RV32IF-NEXT: or a7, a2, a0 +; RV32IF-NEXT: or a7, a0, a2 ; RV32IF-NEXT: beqz a7, .LBB45_4 ; RV32IF-NEXT: .LBB45_3: # %entry -; RV32IF-NEXT: slti a6, a0, 0 +; RV32IF-NEXT: slti a6, a2, 0 ; RV32IF-NEXT: .LBB45_4: # %entry -; RV32IF-NEXT: neg a7, a6 -; RV32IF-NEXT: addi t0, a6, -1 +; RV32IF-NEXT: addi a7, a6, -1 +; RV32IF-NEXT: neg t0, a6 ; RV32IF-NEXT: bnez a6, .LBB45_6 ; RV32IF-NEXT: # %bb.5: # %entry ; RV32IF-NEXT: mv a1, a5 ; RV32IF-NEXT: .LBB45_6: # %entry -; RV32IF-NEXT: or a4, t0, a4 -; RV32IF-NEXT: and a5, a7, a0 -; RV32IF-NEXT: and a2, a7, a2 +; RV32IF-NEXT: or a4, a7, a4 +; RV32IF-NEXT: and a2, t0, a2 +; RV32IF-NEXT: and a5, t0, a0 ; RV32IF-NEXT: beq a1, a3, .LBB45_8 ; RV32IF-NEXT: # %bb.7: # %entry ; RV32IF-NEXT: sltu a0, a3, a1 @@ -2926,11 +2936,11 @@ define i64 @stest_f64i64_mm(double %x) { ; RV32IF-NEXT: .LBB45_8: ; RV32IF-NEXT: snez a0, a4 ; RV32IF-NEXT: .LBB45_9: # %entry -; RV32IF-NEXT: and a2, a2, a5 +; RV32IF-NEXT: and a5, a5, a2 ; RV32IF-NEXT: li a3, -1 -; RV32IF-NEXT: beq a2, a3, .LBB45_11 +; RV32IF-NEXT: beq a5, a3, .LBB45_11 ; RV32IF-NEXT: # %bb.10: # %entry -; RV32IF-NEXT: slti a0, a5, 0 +; RV32IF-NEXT: slti a0, a2, 0 ; RV32IF-NEXT: xori a0, a0, 1 ; RV32IF-NEXT: .LBB45_11: # %entry ; RV32IF-NEXT: bnez a0, .LBB45_13 @@ -2990,8 +3000,8 @@ define i64 @stest_f64i64_mm(double %x) { ; RV32IFD-NEXT: .cfi_offset ra, -4 ; RV32IFD-NEXT: addi a0, sp, 8 ; RV32IFD-NEXT: call __fixdfti -; RV32IFD-NEXT: lw a0, 20(sp) -; RV32IFD-NEXT: lw a2, 16(sp) +; RV32IFD-NEXT: lw a0, 16(sp) +; RV32IFD-NEXT: lw a2, 20(sp) ; RV32IFD-NEXT: lw a1, 12(sp) ; RV32IFD-NEXT: lw a4, 8(sp) ; RV32IFD-NEXT: lui a3, 524288 @@ -2999,25 +3009,25 @@ define i64 @stest_f64i64_mm(double %x) { ; RV32IFD-NEXT: beq a1, a5, .LBB45_2 ; RV32IFD-NEXT: # %bb.1: # %entry ; RV32IFD-NEXT: sltu a6, a1, a5 -; RV32IFD-NEXT: or a7, a2, a0 +; RV32IFD-NEXT: or a7, a0, a2 ; RV32IFD-NEXT: bnez a7, .LBB45_3 ; RV32IFD-NEXT: j .LBB45_4 ; RV32IFD-NEXT: .LBB45_2: ; RV32IFD-NEXT: sltiu a6, a4, -1 -; RV32IFD-NEXT: or a7, a2, a0 +; RV32IFD-NEXT: or a7, a0, a2 ; RV32IFD-NEXT: beqz a7, .LBB45_4 ; RV32IFD-NEXT: .LBB45_3: # %entry -; RV32IFD-NEXT: slti a6, a0, 0 +; RV32IFD-NEXT: slti a6, a2, 0 ; RV32IFD-NEXT: .LBB45_4: # %entry -; RV32IFD-NEXT: neg a7, a6 -; RV32IFD-NEXT: addi t0, a6, -1 +; RV32IFD-NEXT: addi a7, a6, -1 +; RV32IFD-NEXT: neg t0, a6 ; RV32IFD-NEXT: bnez a6, .LBB45_6 ; RV32IFD-NEXT: # %bb.5: # %entry ; RV32IFD-NEXT: mv a1, a5 ; RV32IFD-NEXT: .LBB45_6: # %entry -; RV32IFD-NEXT: or a4, t0, a4 -; RV32IFD-NEXT: and a5, a7, a0 -; RV32IFD-NEXT: and a2, a7, a2 +; RV32IFD-NEXT: or a4, a7, a4 +; RV32IFD-NEXT: and a2, t0, a2 +; RV32IFD-NEXT: and a5, t0, a0 ; RV32IFD-NEXT: beq a1, a3, .LBB45_8 ; RV32IFD-NEXT: # %bb.7: # %entry ; RV32IFD-NEXT: sltu a0, a3, a1 @@ -3025,11 +3035,11 @@ define i64 @stest_f64i64_mm(double %x) { ; RV32IFD-NEXT: .LBB45_8: ; RV32IFD-NEXT: snez a0, a4 ; RV32IFD-NEXT: .LBB45_9: # %entry -; RV32IFD-NEXT: and a2, a2, a5 +; RV32IFD-NEXT: and a5, a5, a2 ; RV32IFD-NEXT: li a3, -1 -; RV32IFD-NEXT: beq a2, a3, .LBB45_11 +; RV32IFD-NEXT: beq a5, a3, .LBB45_11 ; RV32IFD-NEXT: # %bb.10: # %entry -; RV32IFD-NEXT: slti a0, a5, 0 +; RV32IFD-NEXT: slti a0, a2, 0 ; RV32IFD-NEXT: xori a0, a0, 1 ; RV32IFD-NEXT: .LBB45_11: # %entry ; RV32IFD-NEXT: bnez a0, .LBB45_13 @@ -3073,8 +3083,10 @@ define i64 @utest_f64i64_mm(double %x) { ; RV32IF-NEXT: lw a1, 20(sp) ; RV32IF-NEXT: lw a2, 12(sp) ; RV32IF-NEXT: lw a3, 8(sp) -; RV32IF-NEXT: or a4, a1, a0 -; RV32IF-NEXT: seqz a4, a4 +; RV32IF-NEXT: seqz a4, a0 +; RV32IF-NEXT: snez a5, a1 +; RV32IF-NEXT: addi a5, a5, -1 +; RV32IF-NEXT: and a4, a5, a4 ; RV32IF-NEXT: xori a0, a0, 1 ; RV32IF-NEXT: or a0, a0, a1 ; RV32IF-NEXT: seqz a0, a0 @@ -3113,8 +3125,10 @@ define i64 @utest_f64i64_mm(double %x) { ; RV32IFD-NEXT: lw a1, 20(sp) ; RV32IFD-NEXT: lw a2, 12(sp) ; RV32IFD-NEXT: lw a3, 8(sp) -; RV32IFD-NEXT: or a4, a1, a0 -; RV32IFD-NEXT: seqz a4, a4 +; RV32IFD-NEXT: seqz a4, a0 +; RV32IFD-NEXT: snez a5, a1 +; RV32IFD-NEXT: addi a5, a5, -1 +; RV32IFD-NEXT: and a4, a5, a4 ; RV32IFD-NEXT: xori a0, a0, 1 ; RV32IFD-NEXT: or a0, a0, a1 ; RV32IFD-NEXT: seqz a0, a0 @@ -3144,30 +3158,30 @@ define i64 @ustest_f64i64_mm(double %x) { ; RV32IF-NEXT: mv a1, a0 ; RV32IF-NEXT: addi a0, sp, 8 ; RV32IF-NEXT: call __fixdfti -; RV32IF-NEXT: lw a0, 8(sp) -; RV32IF-NEXT: lw a1, 12(sp) -; RV32IF-NEXT: lw a2, 20(sp) +; RV32IF-NEXT: lw a0, 20(sp) +; RV32IF-NEXT: lw a1, 8(sp) +; RV32IF-NEXT: lw a2, 12(sp) ; RV32IF-NEXT: lw a3, 16(sp) -; RV32IF-NEXT: beqz a2, .LBB47_2 +; RV32IF-NEXT: beqz a0, .LBB47_2 ; RV32IF-NEXT: # %bb.1: # %entry -; RV32IF-NEXT: slti a4, a2, 0 +; RV32IF-NEXT: slti a4, a0, 0 ; RV32IF-NEXT: j .LBB47_3 ; RV32IF-NEXT: .LBB47_2: ; RV32IF-NEXT: seqz a4, a3 ; RV32IF-NEXT: .LBB47_3: # %entry ; RV32IF-NEXT: xori a3, a3, 1 -; RV32IF-NEXT: or a3, a3, a2 +; RV32IF-NEXT: or a3, a3, a0 ; RV32IF-NEXT: seqz a3, a3 ; RV32IF-NEXT: addi a3, a3, -1 ; RV32IF-NEXT: and a3, a3, a4 ; RV32IF-NEXT: neg a3, a3 +; RV32IF-NEXT: and a2, a3, a2 ; RV32IF-NEXT: and a1, a3, a1 ; RV32IF-NEXT: and a0, a3, a0 -; RV32IF-NEXT: and a2, a3, a2 -; RV32IF-NEXT: slti a2, a2, 0 -; RV32IF-NEXT: addi a2, a2, -1 -; RV32IF-NEXT: and a0, a2, a0 -; RV32IF-NEXT: and a1, a2, a1 +; RV32IF-NEXT: slti a0, a0, 0 +; RV32IF-NEXT: addi a3, a0, -1 +; RV32IF-NEXT: and a0, a3, a1 +; RV32IF-NEXT: and a1, a3, a2 ; RV32IF-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32IF-NEXT: addi sp, sp, 32 ; RV32IF-NEXT: ret @@ -3202,30 +3216,30 @@ define i64 @ustest_f64i64_mm(double %x) { ; RV32IFD-NEXT: .cfi_offset ra, -4 ; RV32IFD-NEXT: addi a0, sp, 8 ; RV32IFD-NEXT: call __fixdfti -; RV32IFD-NEXT: lw a0, 8(sp) -; RV32IFD-NEXT: lw a1, 12(sp) -; RV32IFD-NEXT: lw a2, 20(sp) +; RV32IFD-NEXT: lw a0, 20(sp) +; RV32IFD-NEXT: lw a1, 8(sp) +; RV32IFD-NEXT: lw a2, 12(sp) ; RV32IFD-NEXT: lw a3, 16(sp) -; RV32IFD-NEXT: beqz a2, .LBB47_2 +; RV32IFD-NEXT: beqz a0, .LBB47_2 ; RV32IFD-NEXT: # %bb.1: # %entry -; RV32IFD-NEXT: slti a4, a2, 0 +; RV32IFD-NEXT: slti a4, a0, 0 ; RV32IFD-NEXT: j .LBB47_3 ; RV32IFD-NEXT: .LBB47_2: ; RV32IFD-NEXT: seqz a4, a3 ; RV32IFD-NEXT: .LBB47_3: # %entry ; RV32IFD-NEXT: xori a3, a3, 1 -; RV32IFD-NEXT: or a3, a3, a2 +; RV32IFD-NEXT: or a3, a3, a0 ; RV32IFD-NEXT: seqz a3, a3 ; RV32IFD-NEXT: addi a3, a3, -1 ; RV32IFD-NEXT: and a3, a3, a4 ; RV32IFD-NEXT: neg a3, a3 +; RV32IFD-NEXT: and a2, a3, a2 ; RV32IFD-NEXT: and a1, a3, a1 ; RV32IFD-NEXT: and a0, a3, a0 -; RV32IFD-NEXT: and a2, a3, a2 -; RV32IFD-NEXT: slti a2, a2, 0 -; RV32IFD-NEXT: addi a2, a2, -1 -; RV32IFD-NEXT: and a0, a2, a0 -; RV32IFD-NEXT: and a1, a2, a1 +; RV32IFD-NEXT: slti a0, a0, 0 +; RV32IFD-NEXT: addi a3, a0, -1 +; RV32IFD-NEXT: and a0, a3, a1 +; RV32IFD-NEXT: and a1, a3, a2 ; RV32IFD-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32IFD-NEXT: addi sp, sp, 32 ; RV32IFD-NEXT: ret @@ -3246,8 +3260,8 @@ define i64 @stest_f32i64_mm(float %x) { ; RV32-NEXT: .cfi_offset ra, -4 ; RV32-NEXT: addi a0, sp, 8 ; RV32-NEXT: call __fixsfti -; RV32-NEXT: lw a0, 20(sp) -; RV32-NEXT: lw a2, 16(sp) +; RV32-NEXT: lw a0, 16(sp) +; RV32-NEXT: lw a2, 20(sp) ; RV32-NEXT: lw a1, 12(sp) ; RV32-NEXT: lw a4, 8(sp) ; RV32-NEXT: lui a3, 524288 @@ -3255,25 +3269,25 @@ define i64 @stest_f32i64_mm(float %x) { ; RV32-NEXT: beq a1, a5, .LBB48_2 ; RV32-NEXT: # %bb.1: # %entry ; RV32-NEXT: sltu a6, a1, a5 -; RV32-NEXT: or a7, a2, a0 +; RV32-NEXT: or a7, a0, a2 ; RV32-NEXT: bnez a7, .LBB48_3 ; RV32-NEXT: j .LBB48_4 ; RV32-NEXT: .LBB48_2: ; RV32-NEXT: sltiu a6, a4, -1 -; RV32-NEXT: or a7, a2, a0 +; RV32-NEXT: or a7, a0, a2 ; RV32-NEXT: beqz a7, .LBB48_4 ; RV32-NEXT: .LBB48_3: # %entry -; RV32-NEXT: slti a6, a0, 0 +; RV32-NEXT: slti a6, a2, 0 ; RV32-NEXT: .LBB48_4: # %entry -; RV32-NEXT: neg a7, a6 -; RV32-NEXT: addi t0, a6, -1 +; RV32-NEXT: addi a7, a6, -1 +; RV32-NEXT: neg t0, a6 ; RV32-NEXT: bnez a6, .LBB48_6 ; RV32-NEXT: # %bb.5: # %entry ; RV32-NEXT: mv a1, a5 ; RV32-NEXT: .LBB48_6: # %entry -; RV32-NEXT: or a4, t0, a4 -; RV32-NEXT: and a5, a7, a0 -; RV32-NEXT: and a2, a7, a2 +; RV32-NEXT: or a4, a7, a4 +; RV32-NEXT: and a2, t0, a2 +; RV32-NEXT: and a5, t0, a0 ; RV32-NEXT: beq a1, a3, .LBB48_8 ; RV32-NEXT: # %bb.7: # %entry ; RV32-NEXT: sltu a0, a3, a1 @@ -3281,11 +3295,11 @@ define i64 @stest_f32i64_mm(float %x) { ; RV32-NEXT: .LBB48_8: ; RV32-NEXT: snez a0, a4 ; RV32-NEXT: .LBB48_9: # %entry -; RV32-NEXT: and a2, a2, a5 +; RV32-NEXT: and a5, a5, a2 ; RV32-NEXT: li a3, -1 -; RV32-NEXT: beq a2, a3, .LBB48_11 +; RV32-NEXT: beq a5, a3, .LBB48_11 ; RV32-NEXT: # %bb.10: # %entry -; RV32-NEXT: slti a0, a5, 0 +; RV32-NEXT: slti a0, a2, 0 ; RV32-NEXT: xori a0, a0, 1 ; RV32-NEXT: .LBB48_11: # %entry ; RV32-NEXT: bnez a0, .LBB48_13 @@ -3327,8 +3341,10 @@ define i64 @utest_f32i64_mm(float %x) { ; RV32-NEXT: lw a1, 20(sp) ; RV32-NEXT: lw a2, 12(sp) ; RV32-NEXT: lw a3, 8(sp) -; RV32-NEXT: or a4, a1, a0 -; RV32-NEXT: seqz a4, a4 +; RV32-NEXT: seqz a4, a0 +; RV32-NEXT: snez a5, a1 +; RV32-NEXT: addi a5, a5, -1 +; RV32-NEXT: and a4, a5, a4 ; RV32-NEXT: xori a0, a0, 1 ; RV32-NEXT: or a0, a0, a1 ; RV32-NEXT: seqz a0, a0 @@ -3370,30 +3386,30 @@ define i64 @ustest_f32i64_mm(float %x) { ; RV32-NEXT: .cfi_offset ra, -4 ; RV32-NEXT: addi a0, sp, 8 ; RV32-NEXT: call __fixsfti -; RV32-NEXT: lw a0, 8(sp) -; RV32-NEXT: lw a1, 12(sp) -; RV32-NEXT: lw a2, 20(sp) +; RV32-NEXT: lw a0, 20(sp) +; RV32-NEXT: lw a1, 8(sp) +; RV32-NEXT: lw a2, 12(sp) ; RV32-NEXT: lw a3, 16(sp) -; RV32-NEXT: beqz a2, .LBB50_2 +; RV32-NEXT: beqz a0, .LBB50_2 ; RV32-NEXT: # %bb.1: # %entry -; RV32-NEXT: slti a4, a2, 0 +; RV32-NEXT: slti a4, a0, 0 ; RV32-NEXT: j .LBB50_3 ; RV32-NEXT: .LBB50_2: ; RV32-NEXT: seqz a4, a3 ; RV32-NEXT: .LBB50_3: # %entry ; RV32-NEXT: xori a3, a3, 1 -; RV32-NEXT: or a3, a3, a2 +; RV32-NEXT: or a3, a3, a0 ; RV32-NEXT: seqz a3, a3 ; RV32-NEXT: addi a3, a3, -1 ; RV32-NEXT: and a3, a3, a4 ; RV32-NEXT: neg a3, a3 +; RV32-NEXT: and a2, a3, a2 ; RV32-NEXT: and a1, a3, a1 ; RV32-NEXT: and a0, a3, a0 -; RV32-NEXT: and a2, a3, a2 -; RV32-NEXT: slti a2, a2, 0 -; RV32-NEXT: addi a2, a2, -1 -; RV32-NEXT: and a0, a2, a0 -; RV32-NEXT: and a1, a2, a1 +; RV32-NEXT: slti a0, a0, 0 +; RV32-NEXT: addi a3, a0, -1 +; RV32-NEXT: and a0, a3, a1 +; RV32-NEXT: and a1, a3, a2 ; RV32-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32-NEXT: addi sp, sp, 32 ; RV32-NEXT: ret @@ -3437,8 +3453,8 @@ define i64 @stest_f16i64_mm(half %x) { ; RV32-NEXT: call __extendhfsf2 ; RV32-NEXT: addi a0, sp, 8 ; RV32-NEXT: call __fixsfti -; RV32-NEXT: lw a0, 20(sp) -; RV32-NEXT: lw a2, 16(sp) +; RV32-NEXT: lw a0, 16(sp) +; RV32-NEXT: lw a2, 20(sp) ; RV32-NEXT: lw a1, 12(sp) ; RV32-NEXT: lw a4, 8(sp) ; RV32-NEXT: lui a3, 524288 @@ -3446,25 +3462,25 @@ define i64 @stest_f16i64_mm(half %x) { ; RV32-NEXT: beq a1, a5, .LBB51_2 ; RV32-NEXT: # %bb.1: # %entry ; RV32-NEXT: sltu a6, a1, a5 -; RV32-NEXT: or a7, a2, a0 +; RV32-NEXT: or a7, a0, a2 ; RV32-NEXT: bnez a7, .LBB51_3 ; RV32-NEXT: j .LBB51_4 ; RV32-NEXT: .LBB51_2: ; RV32-NEXT: sltiu a6, a4, -1 -; RV32-NEXT: or a7, a2, a0 +; RV32-NEXT: or a7, a0, a2 ; RV32-NEXT: beqz a7, .LBB51_4 ; RV32-NEXT: .LBB51_3: # %entry -; RV32-NEXT: slti a6, a0, 0 +; RV32-NEXT: slti a6, a2, 0 ; RV32-NEXT: .LBB51_4: # %entry -; RV32-NEXT: neg a7, a6 -; RV32-NEXT: addi t0, a6, -1 +; RV32-NEXT: addi a7, a6, -1 +; RV32-NEXT: neg t0, a6 ; RV32-NEXT: bnez a6, .LBB51_6 ; RV32-NEXT: # %bb.5: # %entry ; RV32-NEXT: mv a1, a5 ; RV32-NEXT: .LBB51_6: # %entry -; RV32-NEXT: or a4, t0, a4 -; RV32-NEXT: and a5, a7, a0 -; RV32-NEXT: and a2, a7, a2 +; RV32-NEXT: or a4, a7, a4 +; RV32-NEXT: and a2, t0, a2 +; RV32-NEXT: and a5, t0, a0 ; RV32-NEXT: beq a1, a3, .LBB51_8 ; RV32-NEXT: # %bb.7: # %entry ; RV32-NEXT: sltu a0, a3, a1 @@ -3472,11 +3488,11 @@ define i64 @stest_f16i64_mm(half %x) { ; RV32-NEXT: .LBB51_8: ; RV32-NEXT: snez a0, a4 ; RV32-NEXT: .LBB51_9: # %entry -; RV32-NEXT: and a2, a2, a5 +; RV32-NEXT: and a5, a5, a2 ; RV32-NEXT: li a3, -1 -; RV32-NEXT: beq a2, a3, .LBB51_11 +; RV32-NEXT: beq a5, a3, .LBB51_11 ; RV32-NEXT: # %bb.10: # %entry -; RV32-NEXT: slti a0, a5, 0 +; RV32-NEXT: slti a0, a2, 0 ; RV32-NEXT: xori a0, a0, 1 ; RV32-NEXT: .LBB51_11: # %entry ; RV32-NEXT: bnez a0, .LBB51_13 @@ -3550,8 +3566,10 @@ define i64 @utesth_f16i64_mm(half %x) { ; RV32-NEXT: lw a1, 20(sp) ; RV32-NEXT: lw a2, 12(sp) ; RV32-NEXT: lw a3, 8(sp) -; RV32-NEXT: or a4, a1, a0 -; RV32-NEXT: seqz a4, a4 +; RV32-NEXT: seqz a4, a0 +; RV32-NEXT: snez a5, a1 +; RV32-NEXT: addi a5, a5, -1 +; RV32-NEXT: and a4, a5, a4 ; RV32-NEXT: xori a0, a0, 1 ; RV32-NEXT: or a0, a0, a1 ; RV32-NEXT: seqz a0, a0 @@ -3595,30 +3613,30 @@ define i64 @ustest_f16i64_mm(half %x) { ; RV32-NEXT: call __extendhfsf2 ; RV32-NEXT: addi a0, sp, 8 ; RV32-NEXT: call __fixsfti -; RV32-NEXT: lw a0, 8(sp) -; RV32-NEXT: lw a1, 12(sp) -; RV32-NEXT: lw a2, 20(sp) +; RV32-NEXT: lw a0, 20(sp) +; RV32-NEXT: lw a1, 8(sp) +; RV32-NEXT: lw a2, 12(sp) ; RV32-NEXT: lw a3, 16(sp) -; RV32-NEXT: beqz a2, .LBB53_2 +; RV32-NEXT: beqz a0, .LBB53_2 ; RV32-NEXT: # %bb.1: # %entry -; RV32-NEXT: slti a4, a2, 0 +; RV32-NEXT: slti a4, a0, 0 ; RV32-NEXT: j .LBB53_3 ; RV32-NEXT: .LBB53_2: ; RV32-NEXT: seqz a4, a3 ; RV32-NEXT: .LBB53_3: # %entry ; RV32-NEXT: xori a3, a3, 1 -; RV32-NEXT: or a3, a3, a2 +; RV32-NEXT: or a3, a3, a0 ; RV32-NEXT: seqz a3, a3 ; RV32-NEXT: addi a3, a3, -1 ; RV32-NEXT: and a3, a3, a4 ; RV32-NEXT: neg a3, a3 +; RV32-NEXT: and a2, a3, a2 ; RV32-NEXT: and a1, a3, a1 ; RV32-NEXT: and a0, a3, a0 -; RV32-NEXT: and a2, a3, a2 -; RV32-NEXT: slti a2, a2, 0 -; RV32-NEXT: addi a2, a2, -1 -; RV32-NEXT: and a0, a2, a0 -; RV32-NEXT: and a1, a2, a1 +; RV32-NEXT: slti a0, a0, 0 +; RV32-NEXT: addi a3, a0, -1 +; RV32-NEXT: and a0, a3, a1 +; RV32-NEXT: and a1, a3, a2 ; RV32-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32-NEXT: addi sp, sp, 32 ; RV32-NEXT: ret diff --git a/llvm/test/CodeGen/RISCV/half-convert.ll b/llvm/test/CodeGen/RISCV/half-convert.ll index daaceed3941c..518cd7da2ab7 100644 --- a/llvm/test/CodeGen/RISCV/half-convert.ll +++ b/llvm/test/CodeGen/RISCV/half-convert.ll @@ -2145,41 +2145,48 @@ define i64 @fcvt_l_h(half %a) nounwind { define i64 @fcvt_l_h_sat(half %a) nounwind { ; RV32IZFH-LABEL: fcvt_l_h_sat: ; RV32IZFH: # %bb.0: # %start -; RV32IZFH-NEXT: addi sp, sp, -16 -; RV32IZFH-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFH-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFH-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: addi sp, sp, -32 +; RV32IZFH-NEXT: sw ra, 28(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: sw s0, 24(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: sw s1, 20(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: sw s3, 12(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: fsw fs0, 8(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: lui a0, %hi(.LCPI10_0) +; RV32IZFH-NEXT: flw fa5, %lo(.LCPI10_0)(a0) ; RV32IZFH-NEXT: fcvt.s.h fs0, fa0 +; RV32IZFH-NEXT: flt.s s0, fa5, fs0 +; RV32IZFH-NEXT: neg s1, s0 ; RV32IZFH-NEXT: lui a0, 913408 ; RV32IZFH-NEXT: fmv.w.x fa5, a0 -; RV32IZFH-NEXT: fle.s s0, fa5, fs0 +; RV32IZFH-NEXT: fle.s s2, fa5, fs0 +; RV32IZFH-NEXT: neg s3, s2 ; RV32IZFH-NEXT: fmv.s fa0, fs0 ; RV32IZFH-NEXT: call __fixsfdi +; RV32IZFH-NEXT: and a0, s3, a0 +; RV32IZFH-NEXT: or a0, s1, a0 +; RV32IZFH-NEXT: feq.s a2, fs0, fs0 +; RV32IZFH-NEXT: neg a2, a2 ; RV32IZFH-NEXT: lui a4, 524288 -; RV32IZFH-NEXT: lui a2, 524288 -; RV32IZFH-NEXT: beqz s0, .LBB10_2 +; RV32IZFH-NEXT: li a5, 1 +; RV32IZFH-NEXT: lui a3, 524288 +; RV32IZFH-NEXT: bne s2, a5, .LBB10_2 ; RV32IZFH-NEXT: # %bb.1: # %start -; RV32IZFH-NEXT: mv a2, a1 +; RV32IZFH-NEXT: mv a3, a1 ; RV32IZFH-NEXT: .LBB10_2: # %start -; RV32IZFH-NEXT: lui a1, %hi(.LCPI10_0) -; RV32IZFH-NEXT: flw fa5, %lo(.LCPI10_0)(a1) -; RV32IZFH-NEXT: flt.s a3, fa5, fs0 -; RV32IZFH-NEXT: beqz a3, .LBB10_4 +; RV32IZFH-NEXT: and a0, a2, a0 +; RV32IZFH-NEXT: beqz s0, .LBB10_4 ; RV32IZFH-NEXT: # %bb.3: -; RV32IZFH-NEXT: addi a2, a4, -1 +; RV32IZFH-NEXT: addi a3, a4, -1 ; RV32IZFH-NEXT: .LBB10_4: # %start -; RV32IZFH-NEXT: feq.s a1, fs0, fs0 -; RV32IZFH-NEXT: neg a4, a1 -; RV32IZFH-NEXT: and a1, a4, a2 -; RV32IZFH-NEXT: neg a2, a3 -; RV32IZFH-NEXT: neg a3, s0 -; RV32IZFH-NEXT: and a0, a3, a0 -; RV32IZFH-NEXT: or a0, a2, a0 -; RV32IZFH-NEXT: and a0, a4, a0 -; RV32IZFH-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFH-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFH-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload -; RV32IZFH-NEXT: addi sp, sp, 16 +; RV32IZFH-NEXT: and a1, a2, a3 +; RV32IZFH-NEXT: lw ra, 28(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: lw s0, 24(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: lw s3, 12(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: flw fs0, 8(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: addi sp, sp, 32 ; RV32IZFH-NEXT: ret ; ; RV64IZFH-LABEL: fcvt_l_h_sat: @@ -2193,41 +2200,48 @@ define i64 @fcvt_l_h_sat(half %a) nounwind { ; ; RV32IDZFH-LABEL: fcvt_l_h_sat: ; RV32IDZFH: # %bb.0: # %start -; RV32IDZFH-NEXT: addi sp, sp, -16 -; RV32IDZFH-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IDZFH-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32IDZFH-NEXT: addi sp, sp, -32 +; RV32IDZFH-NEXT: sw ra, 28(sp) # 4-byte Folded Spill +; RV32IDZFH-NEXT: sw s0, 24(sp) # 4-byte Folded Spill +; RV32IDZFH-NEXT: sw s1, 20(sp) # 4-byte Folded Spill +; RV32IDZFH-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IDZFH-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32IDZFH-NEXT: fsd fs0, 0(sp) # 8-byte Folded Spill +; RV32IDZFH-NEXT: lui a0, %hi(.LCPI10_0) +; RV32IDZFH-NEXT: flw fa5, %lo(.LCPI10_0)(a0) ; RV32IDZFH-NEXT: fcvt.s.h fs0, fa0 +; RV32IDZFH-NEXT: flt.s s0, fa5, fs0 +; RV32IDZFH-NEXT: neg s1, s0 ; RV32IDZFH-NEXT: lui a0, 913408 ; RV32IDZFH-NEXT: fmv.w.x fa5, a0 -; RV32IDZFH-NEXT: fle.s s0, fa5, fs0 +; RV32IDZFH-NEXT: fle.s s2, fa5, fs0 +; RV32IDZFH-NEXT: neg s3, s2 ; RV32IDZFH-NEXT: fmv.s fa0, fs0 ; RV32IDZFH-NEXT: call __fixsfdi +; RV32IDZFH-NEXT: and a0, s3, a0 +; RV32IDZFH-NEXT: or a0, s1, a0 +; RV32IDZFH-NEXT: feq.s a2, fs0, fs0 +; RV32IDZFH-NEXT: neg a2, a2 ; RV32IDZFH-NEXT: lui a4, 524288 -; RV32IDZFH-NEXT: lui a2, 524288 -; RV32IDZFH-NEXT: beqz s0, .LBB10_2 +; RV32IDZFH-NEXT: li a5, 1 +; RV32IDZFH-NEXT: lui a3, 524288 +; RV32IDZFH-NEXT: bne s2, a5, .LBB10_2 ; RV32IDZFH-NEXT: # %bb.1: # %start -; RV32IDZFH-NEXT: mv a2, a1 +; RV32IDZFH-NEXT: mv a3, a1 ; RV32IDZFH-NEXT: .LBB10_2: # %start -; RV32IDZFH-NEXT: lui a1, %hi(.LCPI10_0) -; RV32IDZFH-NEXT: flw fa5, %lo(.LCPI10_0)(a1) -; RV32IDZFH-NEXT: flt.s a3, fa5, fs0 -; RV32IDZFH-NEXT: beqz a3, .LBB10_4 +; RV32IDZFH-NEXT: and a0, a2, a0 +; RV32IDZFH-NEXT: beqz s0, .LBB10_4 ; RV32IDZFH-NEXT: # %bb.3: -; RV32IDZFH-NEXT: addi a2, a4, -1 +; RV32IDZFH-NEXT: addi a3, a4, -1 ; RV32IDZFH-NEXT: .LBB10_4: # %start -; RV32IDZFH-NEXT: feq.s a1, fs0, fs0 -; RV32IDZFH-NEXT: neg a4, a1 -; RV32IDZFH-NEXT: and a1, a4, a2 -; RV32IDZFH-NEXT: neg a2, a3 -; RV32IDZFH-NEXT: neg a3, s0 -; RV32IDZFH-NEXT: and a0, a3, a0 -; RV32IDZFH-NEXT: or a0, a2, a0 -; RV32IDZFH-NEXT: and a0, a4, a0 -; RV32IDZFH-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IDZFH-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32IDZFH-NEXT: and a1, a2, a3 +; RV32IDZFH-NEXT: lw ra, 28(sp) # 4-byte Folded Reload +; RV32IDZFH-NEXT: lw s0, 24(sp) # 4-byte Folded Reload +; RV32IDZFH-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; RV32IDZFH-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; RV32IDZFH-NEXT: lw s3, 12(sp) # 4-byte Folded Reload ; RV32IDZFH-NEXT: fld fs0, 0(sp) # 8-byte Folded Reload -; RV32IDZFH-NEXT: addi sp, sp, 16 +; RV32IDZFH-NEXT: addi sp, sp, 32 ; RV32IDZFH-NEXT: ret ; ; RV64IDZFH-LABEL: fcvt_l_h_sat: @@ -2263,8 +2277,9 @@ define i64 @fcvt_l_h_sat(half %a) nounwind { ; RV32IZHINX-NEXT: feq.s a2, s0, s0 ; RV32IZHINX-NEXT: neg a2, a2 ; RV32IZHINX-NEXT: lui a4, 524288 +; RV32IZHINX-NEXT: li a5, 1 ; RV32IZHINX-NEXT: lui a3, 524288 -; RV32IZHINX-NEXT: beqz s3, .LBB10_2 +; RV32IZHINX-NEXT: bne s3, a5, .LBB10_2 ; RV32IZHINX-NEXT: # %bb.1: # %start ; RV32IZHINX-NEXT: mv a3, a1 ; RV32IZHINX-NEXT: .LBB10_2: # %start @@ -2316,8 +2331,9 @@ define i64 @fcvt_l_h_sat(half %a) nounwind { ; RV32IZDINXZHINX-NEXT: feq.s a2, s0, s0 ; RV32IZDINXZHINX-NEXT: neg a2, a2 ; RV32IZDINXZHINX-NEXT: lui a4, 524288 +; RV32IZDINXZHINX-NEXT: li a5, 1 ; RV32IZDINXZHINX-NEXT: lui a3, 524288 -; RV32IZDINXZHINX-NEXT: beqz s3, .LBB10_2 +; RV32IZDINXZHINX-NEXT: bne s3, a5, .LBB10_2 ; RV32IZDINXZHINX-NEXT: # %bb.1: # %start ; RV32IZDINXZHINX-NEXT: mv a3, a1 ; RV32IZDINXZHINX-NEXT: .LBB10_2: # %start @@ -2448,42 +2464,48 @@ define i64 @fcvt_l_h_sat(half %a) nounwind { ; ; RV32ID-ILP32-LABEL: fcvt_l_h_sat: ; RV32ID-ILP32: # %bb.0: # %start -; RV32ID-ILP32-NEXT: addi sp, sp, -16 -; RV32ID-ILP32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32ID-ILP32-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32ID-ILP32-NEXT: addi sp, sp, -32 +; RV32ID-ILP32-NEXT: sw ra, 28(sp) # 4-byte Folded Spill +; RV32ID-ILP32-NEXT: sw s0, 24(sp) # 4-byte Folded Spill +; RV32ID-ILP32-NEXT: sw s1, 20(sp) # 4-byte Folded Spill +; RV32ID-ILP32-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32ID-ILP32-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32ID-ILP32-NEXT: call __extendhfsf2 +; RV32ID-ILP32-NEXT: lui a1, %hi(.LCPI10_0) +; RV32ID-ILP32-NEXT: flw fa5, %lo(.LCPI10_0)(a1) ; RV32ID-ILP32-NEXT: fmv.w.x fa4, a0 +; RV32ID-ILP32-NEXT: fsw fa4, 8(sp) # 4-byte Folded Spill +; RV32ID-ILP32-NEXT: flt.s s0, fa5, fa4 +; RV32ID-ILP32-NEXT: neg s1, s0 ; RV32ID-ILP32-NEXT: lui a1, 913408 ; RV32ID-ILP32-NEXT: fmv.w.x fa5, a1 -; RV32ID-ILP32-NEXT: fsw fa4, 4(sp) # 4-byte Folded Spill -; RV32ID-ILP32-NEXT: fle.s s0, fa5, fa4 +; RV32ID-ILP32-NEXT: fle.s s2, fa5, fa4 +; RV32ID-ILP32-NEXT: neg s3, s2 ; RV32ID-ILP32-NEXT: call __fixsfdi +; RV32ID-ILP32-NEXT: and a0, s3, a0 +; RV32ID-ILP32-NEXT: or a0, s1, a0 +; RV32ID-ILP32-NEXT: flw fa5, 8(sp) # 4-byte Folded Reload +; RV32ID-ILP32-NEXT: feq.s a2, fa5, fa5 +; RV32ID-ILP32-NEXT: neg a2, a2 ; RV32ID-ILP32-NEXT: lui a4, 524288 -; RV32ID-ILP32-NEXT: lui a2, 524288 -; RV32ID-ILP32-NEXT: beqz s0, .LBB10_2 +; RV32ID-ILP32-NEXT: li a5, 1 +; RV32ID-ILP32-NEXT: lui a3, 524288 +; RV32ID-ILP32-NEXT: bne s2, a5, .LBB10_2 ; RV32ID-ILP32-NEXT: # %bb.1: # %start -; RV32ID-ILP32-NEXT: mv a2, a1 +; RV32ID-ILP32-NEXT: mv a3, a1 ; RV32ID-ILP32-NEXT: .LBB10_2: # %start -; RV32ID-ILP32-NEXT: lui a1, %hi(.LCPI10_0) -; RV32ID-ILP32-NEXT: flw fa5, %lo(.LCPI10_0)(a1) -; RV32ID-ILP32-NEXT: flw fa4, 4(sp) # 4-byte Folded Reload -; RV32ID-ILP32-NEXT: flt.s a3, fa5, fa4 -; RV32ID-ILP32-NEXT: fmv.s fa5, fa4 -; RV32ID-ILP32-NEXT: beqz a3, .LBB10_4 +; RV32ID-ILP32-NEXT: and a0, a2, a0 +; RV32ID-ILP32-NEXT: beqz s0, .LBB10_4 ; RV32ID-ILP32-NEXT: # %bb.3: -; RV32ID-ILP32-NEXT: addi a2, a4, -1 +; RV32ID-ILP32-NEXT: addi a3, a4, -1 ; RV32ID-ILP32-NEXT: .LBB10_4: # %start -; RV32ID-ILP32-NEXT: feq.s a1, fa5, fa5 -; RV32ID-ILP32-NEXT: neg a4, a1 -; RV32ID-ILP32-NEXT: and a1, a4, a2 -; RV32ID-ILP32-NEXT: neg a2, a3 -; RV32ID-ILP32-NEXT: neg a3, s0 -; RV32ID-ILP32-NEXT: and a0, a3, a0 -; RV32ID-ILP32-NEXT: or a0, a2, a0 -; RV32ID-ILP32-NEXT: and a0, a4, a0 -; RV32ID-ILP32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32ID-ILP32-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32ID-ILP32-NEXT: addi sp, sp, 16 +; RV32ID-ILP32-NEXT: and a1, a2, a3 +; RV32ID-ILP32-NEXT: lw ra, 28(sp) # 4-byte Folded Reload +; RV32ID-ILP32-NEXT: lw s0, 24(sp) # 4-byte Folded Reload +; RV32ID-ILP32-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; RV32ID-ILP32-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; RV32ID-ILP32-NEXT: lw s3, 12(sp) # 4-byte Folded Reload +; RV32ID-ILP32-NEXT: addi sp, sp, 32 ; RV32ID-ILP32-NEXT: ret ; ; RV64ID-LP64-LABEL: fcvt_l_h_sat: @@ -2503,41 +2525,48 @@ define i64 @fcvt_l_h_sat(half %a) nounwind { ; ; RV32ID-LABEL: fcvt_l_h_sat: ; RV32ID: # %bb.0: # %start -; RV32ID-NEXT: addi sp, sp, -16 -; RV32ID-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32ID-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32ID-NEXT: addi sp, sp, -32 +; RV32ID-NEXT: sw ra, 28(sp) # 4-byte Folded Spill +; RV32ID-NEXT: sw s0, 24(sp) # 4-byte Folded Spill +; RV32ID-NEXT: sw s1, 20(sp) # 4-byte Folded Spill +; RV32ID-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32ID-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32ID-NEXT: fsd fs0, 0(sp) # 8-byte Folded Spill ; RV32ID-NEXT: call __extendhfsf2 +; RV32ID-NEXT: lui a0, %hi(.LCPI10_0) +; RV32ID-NEXT: flw fa5, %lo(.LCPI10_0)(a0) ; RV32ID-NEXT: fmv.s fs0, fa0 +; RV32ID-NEXT: flt.s s0, fa5, fa0 +; RV32ID-NEXT: neg s1, s0 ; RV32ID-NEXT: lui a0, 913408 ; RV32ID-NEXT: fmv.w.x fa5, a0 -; RV32ID-NEXT: fle.s s0, fa5, fa0 +; RV32ID-NEXT: fle.s s2, fa5, fa0 +; RV32ID-NEXT: neg s3, s2 ; RV32ID-NEXT: call __fixsfdi +; RV32ID-NEXT: and a0, s3, a0 +; RV32ID-NEXT: or a0, s1, a0 +; RV32ID-NEXT: feq.s a2, fs0, fs0 +; RV32ID-NEXT: neg a2, a2 ; RV32ID-NEXT: lui a4, 524288 -; RV32ID-NEXT: lui a2, 524288 -; RV32ID-NEXT: beqz s0, .LBB10_2 +; RV32ID-NEXT: li a5, 1 +; RV32ID-NEXT: lui a3, 524288 +; RV32ID-NEXT: bne s2, a5, .LBB10_2 ; RV32ID-NEXT: # %bb.1: # %start -; RV32ID-NEXT: mv a2, a1 +; RV32ID-NEXT: mv a3, a1 ; RV32ID-NEXT: .LBB10_2: # %start -; RV32ID-NEXT: lui a1, %hi(.LCPI10_0) -; RV32ID-NEXT: flw fa5, %lo(.LCPI10_0)(a1) -; RV32ID-NEXT: flt.s a3, fa5, fs0 -; RV32ID-NEXT: beqz a3, .LBB10_4 +; RV32ID-NEXT: and a0, a2, a0 +; RV32ID-NEXT: beqz s0, .LBB10_4 ; RV32ID-NEXT: # %bb.3: -; RV32ID-NEXT: addi a2, a4, -1 +; RV32ID-NEXT: addi a3, a4, -1 ; RV32ID-NEXT: .LBB10_4: # %start -; RV32ID-NEXT: feq.s a1, fs0, fs0 -; RV32ID-NEXT: neg a4, a1 -; RV32ID-NEXT: and a1, a4, a2 -; RV32ID-NEXT: neg a2, a3 -; RV32ID-NEXT: neg a3, s0 -; RV32ID-NEXT: and a0, a3, a0 -; RV32ID-NEXT: or a0, a2, a0 -; RV32ID-NEXT: and a0, a4, a0 -; RV32ID-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32ID-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32ID-NEXT: and a1, a2, a3 +; RV32ID-NEXT: lw ra, 28(sp) # 4-byte Folded Reload +; RV32ID-NEXT: lw s0, 24(sp) # 4-byte Folded Reload +; RV32ID-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; RV32ID-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; RV32ID-NEXT: lw s3, 12(sp) # 4-byte Folded Reload ; RV32ID-NEXT: fld fs0, 0(sp) # 8-byte Folded Reload -; RV32ID-NEXT: addi sp, sp, 16 +; RV32ID-NEXT: addi sp, sp, 32 ; RV32ID-NEXT: ret ; ; RV64ID-LABEL: fcvt_l_h_sat: @@ -2556,41 +2585,48 @@ define i64 @fcvt_l_h_sat(half %a) nounwind { ; ; RV32IFZFHMIN-LABEL: fcvt_l_h_sat: ; RV32IFZFHMIN: # %bb.0: # %start -; RV32IFZFHMIN-NEXT: addi sp, sp, -16 -; RV32IFZFHMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IFZFHMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IFZFHMIN-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IFZFHMIN-NEXT: addi sp, sp, -32 +; RV32IFZFHMIN-NEXT: sw ra, 28(sp) # 4-byte Folded Spill +; RV32IFZFHMIN-NEXT: sw s0, 24(sp) # 4-byte Folded Spill +; RV32IFZFHMIN-NEXT: sw s1, 20(sp) # 4-byte Folded Spill +; RV32IFZFHMIN-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IFZFHMIN-NEXT: sw s3, 12(sp) # 4-byte Folded Spill +; RV32IFZFHMIN-NEXT: fsw fs0, 8(sp) # 4-byte Folded Spill +; RV32IFZFHMIN-NEXT: lui a0, %hi(.LCPI10_0) +; RV32IFZFHMIN-NEXT: flw fa5, %lo(.LCPI10_0)(a0) ; RV32IFZFHMIN-NEXT: fcvt.s.h fs0, fa0 +; RV32IFZFHMIN-NEXT: flt.s s0, fa5, fs0 +; RV32IFZFHMIN-NEXT: neg s1, s0 ; RV32IFZFHMIN-NEXT: lui a0, 913408 ; RV32IFZFHMIN-NEXT: fmv.w.x fa5, a0 -; RV32IFZFHMIN-NEXT: fle.s s0, fa5, fs0 +; RV32IFZFHMIN-NEXT: fle.s s2, fa5, fs0 +; RV32IFZFHMIN-NEXT: neg s3, s2 ; RV32IFZFHMIN-NEXT: fmv.s fa0, fs0 ; RV32IFZFHMIN-NEXT: call __fixsfdi +; RV32IFZFHMIN-NEXT: and a0, s3, a0 +; RV32IFZFHMIN-NEXT: or a0, s1, a0 +; RV32IFZFHMIN-NEXT: feq.s a2, fs0, fs0 +; RV32IFZFHMIN-NEXT: neg a2, a2 ; RV32IFZFHMIN-NEXT: lui a4, 524288 -; RV32IFZFHMIN-NEXT: lui a2, 524288 -; RV32IFZFHMIN-NEXT: beqz s0, .LBB10_2 +; RV32IFZFHMIN-NEXT: li a5, 1 +; RV32IFZFHMIN-NEXT: lui a3, 524288 +; RV32IFZFHMIN-NEXT: bne s2, a5, .LBB10_2 ; RV32IFZFHMIN-NEXT: # %bb.1: # %start -; RV32IFZFHMIN-NEXT: mv a2, a1 +; RV32IFZFHMIN-NEXT: mv a3, a1 ; RV32IFZFHMIN-NEXT: .LBB10_2: # %start -; RV32IFZFHMIN-NEXT: lui a1, %hi(.LCPI10_0) -; RV32IFZFHMIN-NEXT: flw fa5, %lo(.LCPI10_0)(a1) -; RV32IFZFHMIN-NEXT: flt.s a3, fa5, fs0 -; RV32IFZFHMIN-NEXT: beqz a3, .LBB10_4 +; RV32IFZFHMIN-NEXT: and a0, a2, a0 +; RV32IFZFHMIN-NEXT: beqz s0, .LBB10_4 ; RV32IFZFHMIN-NEXT: # %bb.3: -; RV32IFZFHMIN-NEXT: addi a2, a4, -1 +; RV32IFZFHMIN-NEXT: addi a3, a4, -1 ; RV32IFZFHMIN-NEXT: .LBB10_4: # %start -; RV32IFZFHMIN-NEXT: feq.s a1, fs0, fs0 -; RV32IFZFHMIN-NEXT: neg a4, a1 -; RV32IFZFHMIN-NEXT: and a1, a4, a2 -; RV32IFZFHMIN-NEXT: neg a2, a3 -; RV32IFZFHMIN-NEXT: neg a3, s0 -; RV32IFZFHMIN-NEXT: and a0, a3, a0 -; RV32IFZFHMIN-NEXT: or a0, a2, a0 -; RV32IFZFHMIN-NEXT: and a0, a4, a0 -; RV32IFZFHMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IFZFHMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IFZFHMIN-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload -; RV32IFZFHMIN-NEXT: addi sp, sp, 16 +; RV32IFZFHMIN-NEXT: and a1, a2, a3 +; RV32IFZFHMIN-NEXT: lw ra, 28(sp) # 4-byte Folded Reload +; RV32IFZFHMIN-NEXT: lw s0, 24(sp) # 4-byte Folded Reload +; RV32IFZFHMIN-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; RV32IFZFHMIN-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; RV32IFZFHMIN-NEXT: lw s3, 12(sp) # 4-byte Folded Reload +; RV32IFZFHMIN-NEXT: flw fs0, 8(sp) # 4-byte Folded Reload +; RV32IFZFHMIN-NEXT: addi sp, sp, 32 ; RV32IFZFHMIN-NEXT: ret ; ; CHECK64-IZFHMIN-LABEL: fcvt_l_h_sat: @@ -2605,41 +2641,48 @@ define i64 @fcvt_l_h_sat(half %a) nounwind { ; ; RV32IDZFHMIN-LABEL: fcvt_l_h_sat: ; RV32IDZFHMIN: # %bb.0: # %start -; RV32IDZFHMIN-NEXT: addi sp, sp, -16 -; RV32IDZFHMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IDZFHMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32IDZFHMIN-NEXT: addi sp, sp, -32 +; RV32IDZFHMIN-NEXT: sw ra, 28(sp) # 4-byte Folded Spill +; RV32IDZFHMIN-NEXT: sw s0, 24(sp) # 4-byte Folded Spill +; RV32IDZFHMIN-NEXT: sw s1, 20(sp) # 4-byte Folded Spill +; RV32IDZFHMIN-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IDZFHMIN-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32IDZFHMIN-NEXT: fsd fs0, 0(sp) # 8-byte Folded Spill +; RV32IDZFHMIN-NEXT: lui a0, %hi(.LCPI10_0) +; RV32IDZFHMIN-NEXT: flw fa5, %lo(.LCPI10_0)(a0) ; RV32IDZFHMIN-NEXT: fcvt.s.h fs0, fa0 +; RV32IDZFHMIN-NEXT: flt.s s0, fa5, fs0 +; RV32IDZFHMIN-NEXT: neg s1, s0 ; RV32IDZFHMIN-NEXT: lui a0, 913408 ; RV32IDZFHMIN-NEXT: fmv.w.x fa5, a0 -; RV32IDZFHMIN-NEXT: fle.s s0, fa5, fs0 +; RV32IDZFHMIN-NEXT: fle.s s2, fa5, fs0 +; RV32IDZFHMIN-NEXT: neg s3, s2 ; RV32IDZFHMIN-NEXT: fmv.s fa0, fs0 ; RV32IDZFHMIN-NEXT: call __fixsfdi +; RV32IDZFHMIN-NEXT: and a0, s3, a0 +; RV32IDZFHMIN-NEXT: or a0, s1, a0 +; RV32IDZFHMIN-NEXT: feq.s a2, fs0, fs0 +; RV32IDZFHMIN-NEXT: neg a2, a2 ; RV32IDZFHMIN-NEXT: lui a4, 524288 -; RV32IDZFHMIN-NEXT: lui a2, 524288 -; RV32IDZFHMIN-NEXT: beqz s0, .LBB10_2 +; RV32IDZFHMIN-NEXT: li a5, 1 +; RV32IDZFHMIN-NEXT: lui a3, 524288 +; RV32IDZFHMIN-NEXT: bne s2, a5, .LBB10_2 ; RV32IDZFHMIN-NEXT: # %bb.1: # %start -; RV32IDZFHMIN-NEXT: mv a2, a1 +; RV32IDZFHMIN-NEXT: mv a3, a1 ; RV32IDZFHMIN-NEXT: .LBB10_2: # %start -; RV32IDZFHMIN-NEXT: lui a1, %hi(.LCPI10_0) -; RV32IDZFHMIN-NEXT: flw fa5, %lo(.LCPI10_0)(a1) -; RV32IDZFHMIN-NEXT: flt.s a3, fa5, fs0 -; RV32IDZFHMIN-NEXT: beqz a3, .LBB10_4 +; RV32IDZFHMIN-NEXT: and a0, a2, a0 +; RV32IDZFHMIN-NEXT: beqz s0, .LBB10_4 ; RV32IDZFHMIN-NEXT: # %bb.3: -; RV32IDZFHMIN-NEXT: addi a2, a4, -1 +; RV32IDZFHMIN-NEXT: addi a3, a4, -1 ; RV32IDZFHMIN-NEXT: .LBB10_4: # %start -; RV32IDZFHMIN-NEXT: feq.s a1, fs0, fs0 -; RV32IDZFHMIN-NEXT: neg a4, a1 -; RV32IDZFHMIN-NEXT: and a1, a4, a2 -; RV32IDZFHMIN-NEXT: neg a2, a3 -; RV32IDZFHMIN-NEXT: neg a3, s0 -; RV32IDZFHMIN-NEXT: and a0, a3, a0 -; RV32IDZFHMIN-NEXT: or a0, a2, a0 -; RV32IDZFHMIN-NEXT: and a0, a4, a0 -; RV32IDZFHMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IDZFHMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32IDZFHMIN-NEXT: and a1, a2, a3 +; RV32IDZFHMIN-NEXT: lw ra, 28(sp) # 4-byte Folded Reload +; RV32IDZFHMIN-NEXT: lw s0, 24(sp) # 4-byte Folded Reload +; RV32IDZFHMIN-NEXT: lw s1, 20(sp) # 4-byte Folded Reload +; RV32IDZFHMIN-NEXT: lw s2, 16(sp) # 4-byte Folded Reload +; RV32IDZFHMIN-NEXT: lw s3, 12(sp) # 4-byte Folded Reload ; RV32IDZFHMIN-NEXT: fld fs0, 0(sp) # 8-byte Folded Reload -; RV32IDZFHMIN-NEXT: addi sp, sp, 16 +; RV32IDZFHMIN-NEXT: addi sp, sp, 32 ; RV32IDZFHMIN-NEXT: ret ; ; CHECK32-IZHINXMIN-LABEL: fcvt_l_h_sat: @@ -2666,8 +2709,9 @@ define i64 @fcvt_l_h_sat(half %a) nounwind { ; CHECK32-IZHINXMIN-NEXT: feq.s a2, s0, s0 ; CHECK32-IZHINXMIN-NEXT: neg a2, a2 ; CHECK32-IZHINXMIN-NEXT: lui a4, 524288 +; CHECK32-IZHINXMIN-NEXT: li a5, 1 ; CHECK32-IZHINXMIN-NEXT: lui a3, 524288 -; CHECK32-IZHINXMIN-NEXT: beqz s3, .LBB10_2 +; CHECK32-IZHINXMIN-NEXT: bne s3, a5, .LBB10_2 ; CHECK32-IZHINXMIN-NEXT: # %bb.1: # %start ; CHECK32-IZHINXMIN-NEXT: mv a3, a1 ; CHECK32-IZHINXMIN-NEXT: .LBB10_2: # %start @@ -2720,8 +2764,9 @@ define i64 @fcvt_l_h_sat(half %a) nounwind { ; CHECK32-IZDINXZHINXMIN-NEXT: feq.s a2, s0, s0 ; CHECK32-IZDINXZHINXMIN-NEXT: neg a2, a2 ; CHECK32-IZDINXZHINXMIN-NEXT: lui a4, 524288 +; CHECK32-IZDINXZHINXMIN-NEXT: li a5, 1 ; CHECK32-IZDINXZHINXMIN-NEXT: lui a3, 524288 -; CHECK32-IZDINXZHINXMIN-NEXT: beqz s3, .LBB10_2 +; CHECK32-IZDINXZHINXMIN-NEXT: bne s3, a5, .LBB10_2 ; CHECK32-IZDINXZHINXMIN-NEXT: # %bb.1: # %start ; CHECK32-IZDINXZHINXMIN-NEXT: mv a3, a1 ; CHECK32-IZDINXZHINXMIN-NEXT: .LBB10_2: # %start @@ -2939,7 +2984,8 @@ define i64 @fcvt_lu_h_sat(half %a) nounwind { ; RV32IZFH-NEXT: neg s0, a0 ; RV32IZFH-NEXT: fmv.w.x fa5, zero ; RV32IZFH-NEXT: fle.s a0, fa5, fa0 -; RV32IZFH-NEXT: neg s1, a0 +; RV32IZFH-NEXT: xori a0, a0, 1 +; RV32IZFH-NEXT: addi s1, a0, -1 ; RV32IZFH-NEXT: call __fixunssfdi ; RV32IZFH-NEXT: and a0, s1, a0 ; RV32IZFH-NEXT: or a0, s0, a0 @@ -2973,7 +3019,8 @@ define i64 @fcvt_lu_h_sat(half %a) nounwind { ; RV32IDZFH-NEXT: neg s0, a0 ; RV32IDZFH-NEXT: fmv.w.x fa5, zero ; RV32IDZFH-NEXT: fle.s a0, fa5, fa0 -; RV32IDZFH-NEXT: neg s1, a0 +; RV32IDZFH-NEXT: xori a0, a0, 1 +; RV32IDZFH-NEXT: addi s1, a0, -1 ; RV32IDZFH-NEXT: call __fixunssfdi ; RV32IDZFH-NEXT: and a0, s1, a0 ; RV32IDZFH-NEXT: or a0, s0, a0 @@ -3006,7 +3053,8 @@ define i64 @fcvt_lu_h_sat(half %a) nounwind { ; RV32IZHINX-NEXT: flt.s a1, a1, a0 ; RV32IZHINX-NEXT: neg s0, a1 ; RV32IZHINX-NEXT: fle.s a1, zero, a0 -; RV32IZHINX-NEXT: neg s1, a1 +; RV32IZHINX-NEXT: xori a1, a1, 1 +; RV32IZHINX-NEXT: addi s1, a1, -1 ; RV32IZHINX-NEXT: call __fixunssfdi ; RV32IZHINX-NEXT: and a0, s1, a0 ; RV32IZHINX-NEXT: or a0, s0, a0 @@ -3039,7 +3087,8 @@ define i64 @fcvt_lu_h_sat(half %a) nounwind { ; RV32IZDINXZHINX-NEXT: flt.s a1, a1, a0 ; RV32IZDINXZHINX-NEXT: neg s0, a1 ; RV32IZDINXZHINX-NEXT: fle.s a1, zero, a0 -; RV32IZDINXZHINX-NEXT: neg s1, a1 +; RV32IZDINXZHINX-NEXT: xori a1, a1, 1 +; RV32IZDINXZHINX-NEXT: addi s1, a1, -1 ; RV32IZDINXZHINX-NEXT: call __fixunssfdi ; RV32IZDINXZHINX-NEXT: and a0, s1, a0 ; RV32IZDINXZHINX-NEXT: or a0, s0, a0 @@ -3138,7 +3187,8 @@ define i64 @fcvt_lu_h_sat(half %a) nounwind { ; RV32ID-ILP32-NEXT: neg s0, a1 ; RV32ID-ILP32-NEXT: fmv.w.x fa5, zero ; RV32ID-ILP32-NEXT: fle.s a1, fa5, fa4 -; RV32ID-ILP32-NEXT: neg s1, a1 +; RV32ID-ILP32-NEXT: xori a1, a1, 1 +; RV32ID-ILP32-NEXT: addi s1, a1, -1 ; RV32ID-ILP32-NEXT: call __fixunssfdi ; RV32ID-ILP32-NEXT: and a0, s1, a0 ; RV32ID-ILP32-NEXT: or a0, s0, a0 @@ -3178,7 +3228,8 @@ define i64 @fcvt_lu_h_sat(half %a) nounwind { ; RV32ID-NEXT: neg s0, a0 ; RV32ID-NEXT: fmv.w.x fa5, zero ; RV32ID-NEXT: fle.s a0, fa5, fa0 -; RV32ID-NEXT: neg s1, a0 +; RV32ID-NEXT: xori a0, a0, 1 +; RV32ID-NEXT: addi s1, a0, -1 ; RV32ID-NEXT: call __fixunssfdi ; RV32ID-NEXT: and a0, s1, a0 ; RV32ID-NEXT: or a0, s0, a0 @@ -3217,7 +3268,8 @@ define i64 @fcvt_lu_h_sat(half %a) nounwind { ; CHECK32-IZFHMIN-NEXT: neg s0, a0 ; CHECK32-IZFHMIN-NEXT: fmv.w.x fa5, zero ; CHECK32-IZFHMIN-NEXT: fle.s a0, fa5, fa0 -; CHECK32-IZFHMIN-NEXT: neg s1, a0 +; CHECK32-IZFHMIN-NEXT: xori a0, a0, 1 +; CHECK32-IZFHMIN-NEXT: addi s1, a0, -1 ; CHECK32-IZFHMIN-NEXT: call __fixunssfdi ; CHECK32-IZFHMIN-NEXT: and a0, s1, a0 ; CHECK32-IZFHMIN-NEXT: or a0, s0, a0 @@ -3251,7 +3303,8 @@ define i64 @fcvt_lu_h_sat(half %a) nounwind { ; CHECK32-IZHINXMIN-NEXT: flt.s a1, a1, a0 ; CHECK32-IZHINXMIN-NEXT: neg s0, a1 ; CHECK32-IZHINXMIN-NEXT: fle.s a1, zero, a0 -; CHECK32-IZHINXMIN-NEXT: neg s1, a1 +; CHECK32-IZHINXMIN-NEXT: xori a1, a1, 1 +; CHECK32-IZHINXMIN-NEXT: addi s1, a1, -1 ; CHECK32-IZHINXMIN-NEXT: call __fixunssfdi ; CHECK32-IZHINXMIN-NEXT: and a0, s1, a0 ; CHECK32-IZHINXMIN-NEXT: or a0, s0, a0 @@ -3285,7 +3338,8 @@ define i64 @fcvt_lu_h_sat(half %a) nounwind { ; CHECK32-IZDINXZHINXMIN-NEXT: flt.s a1, a1, a0 ; CHECK32-IZDINXZHINXMIN-NEXT: neg s0, a1 ; CHECK32-IZDINXZHINXMIN-NEXT: fle.s a1, zero, a0 -; CHECK32-IZDINXZHINXMIN-NEXT: neg s1, a1 +; CHECK32-IZDINXZHINXMIN-NEXT: xori a1, a1, 1 +; CHECK32-IZDINXZHINXMIN-NEXT: addi s1, a1, -1 ; CHECK32-IZDINXZHINXMIN-NEXT: call __fixunssfdi ; CHECK32-IZDINXZHINXMIN-NEXT: and a0, s1, a0 ; CHECK32-IZDINXZHINXMIN-NEXT: or a0, s0, a0 diff --git a/llvm/test/CodeGen/RISCV/half-round-conv-sat.ll b/llvm/test/CodeGen/RISCV/half-round-conv-sat.ll index 3f385909b0b5..647af5f5b874 100644 --- a/llvm/test/CodeGen/RISCV/half-round-conv-sat.ll +++ b/llvm/test/CodeGen/RISCV/half-round-conv-sat.ll @@ -108,38 +108,41 @@ define i64 @test_floor_si64(half %x) nounwind { ; RV32IZFH-NEXT: addi sp, sp, -16 ; RV32IZFH-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFH-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFH-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: fsw fs0, 0(sp) # 4-byte Folded Spill ; RV32IZFH-NEXT: fcvt.s.h fs0, fa0 ; RV32IZFH-NEXT: lui a0, 913408 ; RV32IZFH-NEXT: fmv.w.x fa5, a0 ; RV32IZFH-NEXT: fle.s s0, fa5, fs0 +; RV32IZFH-NEXT: neg s1, s0 ; RV32IZFH-NEXT: fmv.s fa0, fs0 ; RV32IZFH-NEXT: call __fixsfdi +; RV32IZFH-NEXT: lui a2, %hi(.LCPI1_1) +; RV32IZFH-NEXT: flw fa5, %lo(.LCPI1_1)(a2) +; RV32IZFH-NEXT: and a0, s1, a0 +; RV32IZFH-NEXT: flt.s a3, fa5, fs0 +; RV32IZFH-NEXT: neg a2, a3 +; RV32IZFH-NEXT: or a0, a2, a0 +; RV32IZFH-NEXT: feq.s a2, fs0, fs0 +; RV32IZFH-NEXT: neg a2, a2 +; RV32IZFH-NEXT: lui a5, 524288 +; RV32IZFH-NEXT: li a6, 1 ; RV32IZFH-NEXT: lui a4, 524288 -; RV32IZFH-NEXT: lui a2, 524288 -; RV32IZFH-NEXT: beqz s0, .LBB1_4 +; RV32IZFH-NEXT: bne s0, a6, .LBB1_4 ; RV32IZFH-NEXT: # %bb.3: -; RV32IZFH-NEXT: mv a2, a1 +; RV32IZFH-NEXT: mv a4, a1 ; RV32IZFH-NEXT: .LBB1_4: -; RV32IZFH-NEXT: lui a1, %hi(.LCPI1_1) -; RV32IZFH-NEXT: flw fa5, %lo(.LCPI1_1)(a1) -; RV32IZFH-NEXT: flt.s a3, fa5, fs0 -; RV32IZFH-NEXT: beqz a3, .LBB1_6 -; RV32IZFH-NEXT: # %bb.5: -; RV32IZFH-NEXT: addi a2, a4, -1 -; RV32IZFH-NEXT: .LBB1_6: -; RV32IZFH-NEXT: feq.s a1, fs0, fs0 -; RV32IZFH-NEXT: neg a4, a1 -; RV32IZFH-NEXT: and a1, a4, a2 -; RV32IZFH-NEXT: neg a2, s0 ; RV32IZFH-NEXT: and a0, a2, a0 -; RV32IZFH-NEXT: neg a2, a3 -; RV32IZFH-NEXT: or a0, a2, a0 -; RV32IZFH-NEXT: and a0, a4, a0 ; RV32IZFH-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFH-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFH-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: flw fs0, 0(sp) # 4-byte Folded Reload ; RV32IZFH-NEXT: addi sp, sp, 16 +; RV32IZFH-NEXT: beqz a3, .LBB1_6 +; RV32IZFH-NEXT: # %bb.5: +; RV32IZFH-NEXT: addi a4, a5, -1 +; RV32IZFH-NEXT: .LBB1_6: +; RV32IZFH-NEXT: and a1, a2, a4 ; RV32IZFH-NEXT: ret ; ; RV64IZFH-LABEL: test_floor_si64: @@ -177,16 +180,17 @@ define i64 @test_floor_si64(half %x) nounwind { ; RV32IZHINX-NEXT: lui a2, %hi(.LCPI1_1) ; RV32IZHINX-NEXT: lw a2, %lo(.LCPI1_1)(a2) ; RV32IZHINX-NEXT: and a0, s2, a0 -; RV32IZHINX-NEXT: flt.s a4, a2, s0 -; RV32IZHINX-NEXT: neg a2, a4 +; RV32IZHINX-NEXT: flt.s a3, a2, s0 +; RV32IZHINX-NEXT: neg a2, a3 ; RV32IZHINX-NEXT: or a0, a2, a0 ; RV32IZHINX-NEXT: feq.s a2, s0, s0 ; RV32IZHINX-NEXT: neg a2, a2 ; RV32IZHINX-NEXT: lui a5, 524288 -; RV32IZHINX-NEXT: lui a3, 524288 -; RV32IZHINX-NEXT: beqz s1, .LBB1_4 +; RV32IZHINX-NEXT: li a6, 1 +; RV32IZHINX-NEXT: lui a4, 524288 +; RV32IZHINX-NEXT: bne s1, a6, .LBB1_4 ; RV32IZHINX-NEXT: # %bb.3: -; RV32IZHINX-NEXT: mv a3, a1 +; RV32IZHINX-NEXT: mv a4, a1 ; RV32IZHINX-NEXT: .LBB1_4: ; RV32IZHINX-NEXT: and a0, a2, a0 ; RV32IZHINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload @@ -194,11 +198,11 @@ define i64 @test_floor_si64(half %x) nounwind { ; RV32IZHINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: lw s2, 0(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: addi sp, sp, 16 -; RV32IZHINX-NEXT: beqz a4, .LBB1_6 +; RV32IZHINX-NEXT: beqz a3, .LBB1_6 ; RV32IZHINX-NEXT: # %bb.5: -; RV32IZHINX-NEXT: addi a3, a5, -1 +; RV32IZHINX-NEXT: addi a4, a5, -1 ; RV32IZHINX-NEXT: .LBB1_6: -; RV32IZHINX-NEXT: and a1, a2, a3 +; RV32IZHINX-NEXT: and a1, a2, a4 ; RV32IZHINX-NEXT: ret ; ; RV64IZHINX-LABEL: test_floor_si64: @@ -236,39 +240,42 @@ define i64 @test_floor_si64(half %x) nounwind { ; RV32IZFHMIN-NEXT: addi sp, sp, -16 ; RV32IZFHMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFHMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFHMIN-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFHMIN-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32IZFHMIN-NEXT: fsw fs0, 0(sp) # 4-byte Folded Spill ; RV32IZFHMIN-NEXT: fcvt.h.s fa5, fa5 ; RV32IZFHMIN-NEXT: fcvt.s.h fs0, fa5 ; RV32IZFHMIN-NEXT: lui a0, 913408 ; RV32IZFHMIN-NEXT: fmv.w.x fa5, a0 ; RV32IZFHMIN-NEXT: fle.s s0, fa5, fs0 +; RV32IZFHMIN-NEXT: neg s1, s0 ; RV32IZFHMIN-NEXT: fmv.s fa0, fs0 ; RV32IZFHMIN-NEXT: call __fixsfdi +; RV32IZFHMIN-NEXT: lui a2, %hi(.LCPI1_0) +; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI1_0)(a2) +; RV32IZFHMIN-NEXT: and a0, s1, a0 +; RV32IZFHMIN-NEXT: flt.s a3, fa5, fs0 +; RV32IZFHMIN-NEXT: neg a2, a3 +; RV32IZFHMIN-NEXT: or a0, a2, a0 +; RV32IZFHMIN-NEXT: feq.s a2, fs0, fs0 +; RV32IZFHMIN-NEXT: neg a2, a2 +; RV32IZFHMIN-NEXT: lui a5, 524288 +; RV32IZFHMIN-NEXT: li a6, 1 ; RV32IZFHMIN-NEXT: lui a4, 524288 -; RV32IZFHMIN-NEXT: lui a2, 524288 -; RV32IZFHMIN-NEXT: beqz s0, .LBB1_4 +; RV32IZFHMIN-NEXT: bne s0, a6, .LBB1_4 ; RV32IZFHMIN-NEXT: # %bb.3: -; RV32IZFHMIN-NEXT: mv a2, a1 +; RV32IZFHMIN-NEXT: mv a4, a1 ; RV32IZFHMIN-NEXT: .LBB1_4: -; RV32IZFHMIN-NEXT: lui a1, %hi(.LCPI1_0) -; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI1_0)(a1) -; RV32IZFHMIN-NEXT: flt.s a3, fa5, fs0 -; RV32IZFHMIN-NEXT: beqz a3, .LBB1_6 -; RV32IZFHMIN-NEXT: # %bb.5: -; RV32IZFHMIN-NEXT: addi a2, a4, -1 -; RV32IZFHMIN-NEXT: .LBB1_6: -; RV32IZFHMIN-NEXT: feq.s a1, fs0, fs0 -; RV32IZFHMIN-NEXT: neg a4, a1 -; RV32IZFHMIN-NEXT: and a1, a4, a2 -; RV32IZFHMIN-NEXT: neg a2, s0 ; RV32IZFHMIN-NEXT: and a0, a2, a0 -; RV32IZFHMIN-NEXT: neg a2, a3 -; RV32IZFHMIN-NEXT: or a0, a2, a0 -; RV32IZFHMIN-NEXT: and a0, a4, a0 ; RV32IZFHMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFHMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFHMIN-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFHMIN-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IZFHMIN-NEXT: flw fs0, 0(sp) # 4-byte Folded Reload ; RV32IZFHMIN-NEXT: addi sp, sp, 16 +; RV32IZFHMIN-NEXT: beqz a3, .LBB1_6 +; RV32IZFHMIN-NEXT: # %bb.5: +; RV32IZFHMIN-NEXT: addi a4, a5, -1 +; RV32IZFHMIN-NEXT: .LBB1_6: +; RV32IZFHMIN-NEXT: and a1, a2, a4 ; RV32IZFHMIN-NEXT: ret ; ; RV64IZFHMIN-LABEL: test_floor_si64: @@ -320,16 +327,17 @@ define i64 @test_floor_si64(half %x) nounwind { ; RV32IZHINXMIN-NEXT: lui a2, %hi(.LCPI1_0) ; RV32IZHINXMIN-NEXT: lw a2, %lo(.LCPI1_0)(a2) ; RV32IZHINXMIN-NEXT: and a0, s2, a0 -; RV32IZHINXMIN-NEXT: flt.s a4, a2, s0 -; RV32IZHINXMIN-NEXT: neg a2, a4 +; RV32IZHINXMIN-NEXT: flt.s a3, a2, s0 +; RV32IZHINXMIN-NEXT: neg a2, a3 ; RV32IZHINXMIN-NEXT: or a0, a2, a0 ; RV32IZHINXMIN-NEXT: feq.s a2, s0, s0 ; RV32IZHINXMIN-NEXT: neg a2, a2 ; RV32IZHINXMIN-NEXT: lui a5, 524288 -; RV32IZHINXMIN-NEXT: lui a3, 524288 -; RV32IZHINXMIN-NEXT: beqz s1, .LBB1_4 +; RV32IZHINXMIN-NEXT: li a6, 1 +; RV32IZHINXMIN-NEXT: lui a4, 524288 +; RV32IZHINXMIN-NEXT: bne s1, a6, .LBB1_4 ; RV32IZHINXMIN-NEXT: # %bb.3: -; RV32IZHINXMIN-NEXT: mv a3, a1 +; RV32IZHINXMIN-NEXT: mv a4, a1 ; RV32IZHINXMIN-NEXT: .LBB1_4: ; RV32IZHINXMIN-NEXT: and a0, a2, a0 ; RV32IZHINXMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload @@ -337,11 +345,11 @@ define i64 @test_floor_si64(half %x) nounwind { ; RV32IZHINXMIN-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: lw s2, 0(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: addi sp, sp, 16 -; RV32IZHINXMIN-NEXT: beqz a4, .LBB1_6 +; RV32IZHINXMIN-NEXT: beqz a3, .LBB1_6 ; RV32IZHINXMIN-NEXT: # %bb.5: -; RV32IZHINXMIN-NEXT: addi a3, a5, -1 +; RV32IZHINXMIN-NEXT: addi a4, a5, -1 ; RV32IZHINXMIN-NEXT: .LBB1_6: -; RV32IZHINXMIN-NEXT: and a1, a2, a3 +; RV32IZHINXMIN-NEXT: and a1, a2, a4 ; RV32IZHINXMIN-NEXT: ret ; ; RV64IZHINXMIN-LABEL: test_floor_si64: @@ -413,7 +421,7 @@ define signext i32 @test_floor_ui32(half %x) { ; RV64IZHINX-NEXT: fcvt.wu.h a1, a0, rtz ; RV64IZHINX-NEXT: feq.h a0, a0, a0 ; RV64IZHINX-NEXT: seqz a0, a0 -; RV64IZHINX-NEXT: addi a0, a0, -1 +; RV64IZHINX-NEXT: addiw a0, a0, -1 ; RV64IZHINX-NEXT: and a0, a1, a0 ; RV64IZHINX-NEXT: ret ; @@ -457,7 +465,7 @@ define signext i32 @test_floor_ui32(half %x) { ; RV64IZFHMIN-NEXT: fcvt.wu.s a0, fa5, rtz ; RV64IZFHMIN-NEXT: feq.s a1, fa5, fa5 ; RV64IZFHMIN-NEXT: seqz a1, a1 -; RV64IZFHMIN-NEXT: addi a1, a1, -1 +; RV64IZFHMIN-NEXT: addiw a1, a1, -1 ; RV64IZFHMIN-NEXT: and a0, a0, a1 ; RV64IZFHMIN-NEXT: ret ; @@ -499,7 +507,7 @@ define signext i32 @test_floor_ui32(half %x) { ; RV64IZHINXMIN-NEXT: fcvt.wu.s a1, a0, rtz ; RV64IZHINXMIN-NEXT: feq.s a0, a0, a0 ; RV64IZHINXMIN-NEXT: seqz a0, a0 -; RV64IZHINXMIN-NEXT: addi a0, a0, -1 +; RV64IZHINXMIN-NEXT: addiw a0, a0, -1 ; RV64IZHINXMIN-NEXT: and a0, a1, a0 ; RV64IZHINXMIN-NEXT: ret %a = call half @llvm.floor.f16(half %x) @@ -522,25 +530,24 @@ define i64 @test_floor_ui64(half %x) nounwind { ; RV32IZFH-NEXT: .LBB3_2: ; RV32IZFH-NEXT: addi sp, sp, -16 ; RV32IZFH-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFH-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFH-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: fsw fs0, 8(sp) # 4-byte Folded Spill ; RV32IZFH-NEXT: fcvt.s.h fs0, fa0 -; RV32IZFH-NEXT: fmv.w.x fa5, zero -; RV32IZFH-NEXT: fle.s a0, fa5, fs0 -; RV32IZFH-NEXT: neg s0, a0 ; RV32IZFH-NEXT: fmv.s fa0, fs0 ; RV32IZFH-NEXT: call __fixunssfdi -; RV32IZFH-NEXT: lui a2, %hi(.LCPI3_1) -; RV32IZFH-NEXT: flw fa5, %lo(.LCPI3_1)(a2) -; RV32IZFH-NEXT: and a0, s0, a0 -; RV32IZFH-NEXT: flt.s a2, fa5, fs0 -; RV32IZFH-NEXT: neg a2, a2 -; RV32IZFH-NEXT: or a0, a2, a0 -; RV32IZFH-NEXT: and a1, s0, a1 -; RV32IZFH-NEXT: or a1, a2, a1 +; RV32IZFH-NEXT: fmv.w.x fa5, zero +; RV32IZFH-NEXT: fle.s a2, fa5, fs0 +; RV32IZFH-NEXT: lui a3, %hi(.LCPI3_1) +; RV32IZFH-NEXT: flw fa5, %lo(.LCPI3_1)(a3) +; RV32IZFH-NEXT: xori a2, a2, 1 +; RV32IZFH-NEXT: addi a2, a2, -1 +; RV32IZFH-NEXT: and a0, a2, a0 +; RV32IZFH-NEXT: flt.s a3, fa5, fs0 +; RV32IZFH-NEXT: neg a3, a3 +; RV32IZFH-NEXT: or a0, a3, a0 +; RV32IZFH-NEXT: and a1, a2, a1 +; RV32IZFH-NEXT: or a1, a3, a1 ; RV32IZFH-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFH-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFH-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: flw fs0, 8(sp) # 4-byte Folded Reload ; RV32IZFH-NEXT: addi sp, sp, 16 ; RV32IZFH-NEXT: ret ; @@ -568,23 +575,22 @@ define i64 @test_floor_ui64(half %x) nounwind { ; RV32IZHINX-NEXT: addi sp, sp, -16 ; RV32IZHINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZHINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZHINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZHINX-NEXT: fcvt.s.h s0, a0 -; RV32IZHINX-NEXT: fle.s a0, zero, s0 -; RV32IZHINX-NEXT: neg s1, a0 ; RV32IZHINX-NEXT: mv a0, s0 ; RV32IZHINX-NEXT: call __fixunssfdi -; RV32IZHINX-NEXT: lui a2, %hi(.LCPI3_1) -; RV32IZHINX-NEXT: lw a2, %lo(.LCPI3_1)(a2) -; RV32IZHINX-NEXT: and a0, s1, a0 -; RV32IZHINX-NEXT: flt.s a2, a2, s0 -; RV32IZHINX-NEXT: neg a2, a2 -; RV32IZHINX-NEXT: or a0, a2, a0 -; RV32IZHINX-NEXT: and a1, s1, a1 -; RV32IZHINX-NEXT: or a1, a2, a1 +; RV32IZHINX-NEXT: fle.s a2, zero, s0 +; RV32IZHINX-NEXT: lui a3, %hi(.LCPI3_1) +; RV32IZHINX-NEXT: lw a3, %lo(.LCPI3_1)(a3) +; RV32IZHINX-NEXT: xori a2, a2, 1 +; RV32IZHINX-NEXT: addi a2, a2, -1 +; RV32IZHINX-NEXT: and a0, a2, a0 +; RV32IZHINX-NEXT: flt.s a3, a3, s0 +; RV32IZHINX-NEXT: neg a3, a3 +; RV32IZHINX-NEXT: or a0, a3, a0 +; RV32IZHINX-NEXT: and a1, a2, a1 +; RV32IZHINX-NEXT: or a1, a3, a1 ; RV32IZHINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZHINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: addi sp, sp, 16 ; RV32IZHINX-NEXT: ret ; @@ -622,26 +628,25 @@ define i64 @test_floor_ui64(half %x) nounwind { ; RV32IZFHMIN-NEXT: .LBB3_2: ; RV32IZFHMIN-NEXT: addi sp, sp, -16 ; RV32IZFHMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFHMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFHMIN-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFHMIN-NEXT: fsw fs0, 8(sp) # 4-byte Folded Spill ; RV32IZFHMIN-NEXT: fcvt.h.s fa5, fa5 ; RV32IZFHMIN-NEXT: fcvt.s.h fs0, fa5 -; RV32IZFHMIN-NEXT: fmv.w.x fa5, zero -; RV32IZFHMIN-NEXT: fle.s a0, fa5, fs0 -; RV32IZFHMIN-NEXT: neg s0, a0 ; RV32IZFHMIN-NEXT: fmv.s fa0, fs0 ; RV32IZFHMIN-NEXT: call __fixunssfdi -; RV32IZFHMIN-NEXT: lui a2, %hi(.LCPI3_0) -; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI3_0)(a2) -; RV32IZFHMIN-NEXT: and a0, s0, a0 -; RV32IZFHMIN-NEXT: flt.s a2, fa5, fs0 -; RV32IZFHMIN-NEXT: neg a2, a2 -; RV32IZFHMIN-NEXT: or a0, a2, a0 -; RV32IZFHMIN-NEXT: and a1, s0, a1 -; RV32IZFHMIN-NEXT: or a1, a2, a1 +; RV32IZFHMIN-NEXT: fmv.w.x fa5, zero +; RV32IZFHMIN-NEXT: fle.s a2, fa5, fs0 +; RV32IZFHMIN-NEXT: lui a3, %hi(.LCPI3_0) +; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI3_0)(a3) +; RV32IZFHMIN-NEXT: xori a2, a2, 1 +; RV32IZFHMIN-NEXT: addi a2, a2, -1 +; RV32IZFHMIN-NEXT: and a0, a2, a0 +; RV32IZFHMIN-NEXT: flt.s a3, fa5, fs0 +; RV32IZFHMIN-NEXT: neg a3, a3 +; RV32IZFHMIN-NEXT: or a0, a3, a0 +; RV32IZFHMIN-NEXT: and a1, a2, a1 +; RV32IZFHMIN-NEXT: or a1, a3, a1 ; RV32IZFHMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFHMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFHMIN-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFHMIN-NEXT: flw fs0, 8(sp) # 4-byte Folded Reload ; RV32IZFHMIN-NEXT: addi sp, sp, 16 ; RV32IZFHMIN-NEXT: ret ; @@ -682,24 +687,23 @@ define i64 @test_floor_ui64(half %x) nounwind { ; RV32IZHINXMIN-NEXT: addi sp, sp, -16 ; RV32IZHINXMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZHINXMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZHINXMIN-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZHINXMIN-NEXT: fcvt.h.s a0, a0 ; RV32IZHINXMIN-NEXT: fcvt.s.h s0, a0 -; RV32IZHINXMIN-NEXT: fle.s a0, zero, s0 -; RV32IZHINXMIN-NEXT: neg s1, a0 ; RV32IZHINXMIN-NEXT: mv a0, s0 ; RV32IZHINXMIN-NEXT: call __fixunssfdi -; RV32IZHINXMIN-NEXT: lui a2, %hi(.LCPI3_0) -; RV32IZHINXMIN-NEXT: lw a2, %lo(.LCPI3_0)(a2) -; RV32IZHINXMIN-NEXT: and a0, s1, a0 -; RV32IZHINXMIN-NEXT: flt.s a2, a2, s0 -; RV32IZHINXMIN-NEXT: neg a2, a2 -; RV32IZHINXMIN-NEXT: or a0, a2, a0 -; RV32IZHINXMIN-NEXT: and a1, s1, a1 -; RV32IZHINXMIN-NEXT: or a1, a2, a1 +; RV32IZHINXMIN-NEXT: fle.s a2, zero, s0 +; RV32IZHINXMIN-NEXT: lui a3, %hi(.LCPI3_0) +; RV32IZHINXMIN-NEXT: lw a3, %lo(.LCPI3_0)(a3) +; RV32IZHINXMIN-NEXT: xori a2, a2, 1 +; RV32IZHINXMIN-NEXT: addi a2, a2, -1 +; RV32IZHINXMIN-NEXT: and a0, a2, a0 +; RV32IZHINXMIN-NEXT: flt.s a3, a3, s0 +; RV32IZHINXMIN-NEXT: neg a3, a3 +; RV32IZHINXMIN-NEXT: or a0, a3, a0 +; RV32IZHINXMIN-NEXT: and a1, a2, a1 +; RV32IZHINXMIN-NEXT: or a1, a3, a1 ; RV32IZHINXMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZHINXMIN-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: addi sp, sp, 16 ; RV32IZHINXMIN-NEXT: ret ; @@ -820,38 +824,41 @@ define i64 @test_ceil_si64(half %x) nounwind { ; RV32IZFH-NEXT: addi sp, sp, -16 ; RV32IZFH-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFH-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFH-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: fsw fs0, 0(sp) # 4-byte Folded Spill ; RV32IZFH-NEXT: fcvt.s.h fs0, fa0 ; RV32IZFH-NEXT: lui a0, 913408 ; RV32IZFH-NEXT: fmv.w.x fa5, a0 ; RV32IZFH-NEXT: fle.s s0, fa5, fs0 +; RV32IZFH-NEXT: neg s1, s0 ; RV32IZFH-NEXT: fmv.s fa0, fs0 ; RV32IZFH-NEXT: call __fixsfdi +; RV32IZFH-NEXT: lui a2, %hi(.LCPI5_1) +; RV32IZFH-NEXT: flw fa5, %lo(.LCPI5_1)(a2) +; RV32IZFH-NEXT: and a0, s1, a0 +; RV32IZFH-NEXT: flt.s a3, fa5, fs0 +; RV32IZFH-NEXT: neg a2, a3 +; RV32IZFH-NEXT: or a0, a2, a0 +; RV32IZFH-NEXT: feq.s a2, fs0, fs0 +; RV32IZFH-NEXT: neg a2, a2 +; RV32IZFH-NEXT: lui a5, 524288 +; RV32IZFH-NEXT: li a6, 1 ; RV32IZFH-NEXT: lui a4, 524288 -; RV32IZFH-NEXT: lui a2, 524288 -; RV32IZFH-NEXT: beqz s0, .LBB5_4 +; RV32IZFH-NEXT: bne s0, a6, .LBB5_4 ; RV32IZFH-NEXT: # %bb.3: -; RV32IZFH-NEXT: mv a2, a1 +; RV32IZFH-NEXT: mv a4, a1 ; RV32IZFH-NEXT: .LBB5_4: -; RV32IZFH-NEXT: lui a1, %hi(.LCPI5_1) -; RV32IZFH-NEXT: flw fa5, %lo(.LCPI5_1)(a1) -; RV32IZFH-NEXT: flt.s a3, fa5, fs0 -; RV32IZFH-NEXT: beqz a3, .LBB5_6 -; RV32IZFH-NEXT: # %bb.5: -; RV32IZFH-NEXT: addi a2, a4, -1 -; RV32IZFH-NEXT: .LBB5_6: -; RV32IZFH-NEXT: feq.s a1, fs0, fs0 -; RV32IZFH-NEXT: neg a4, a1 -; RV32IZFH-NEXT: and a1, a4, a2 -; RV32IZFH-NEXT: neg a2, s0 ; RV32IZFH-NEXT: and a0, a2, a0 -; RV32IZFH-NEXT: neg a2, a3 -; RV32IZFH-NEXT: or a0, a2, a0 -; RV32IZFH-NEXT: and a0, a4, a0 ; RV32IZFH-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFH-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFH-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: flw fs0, 0(sp) # 4-byte Folded Reload ; RV32IZFH-NEXT: addi sp, sp, 16 +; RV32IZFH-NEXT: beqz a3, .LBB5_6 +; RV32IZFH-NEXT: # %bb.5: +; RV32IZFH-NEXT: addi a4, a5, -1 +; RV32IZFH-NEXT: .LBB5_6: +; RV32IZFH-NEXT: and a1, a2, a4 ; RV32IZFH-NEXT: ret ; ; RV64IZFH-LABEL: test_ceil_si64: @@ -889,16 +896,17 @@ define i64 @test_ceil_si64(half %x) nounwind { ; RV32IZHINX-NEXT: lui a2, %hi(.LCPI5_1) ; RV32IZHINX-NEXT: lw a2, %lo(.LCPI5_1)(a2) ; RV32IZHINX-NEXT: and a0, s2, a0 -; RV32IZHINX-NEXT: flt.s a4, a2, s0 -; RV32IZHINX-NEXT: neg a2, a4 +; RV32IZHINX-NEXT: flt.s a3, a2, s0 +; RV32IZHINX-NEXT: neg a2, a3 ; RV32IZHINX-NEXT: or a0, a2, a0 ; RV32IZHINX-NEXT: feq.s a2, s0, s0 ; RV32IZHINX-NEXT: neg a2, a2 ; RV32IZHINX-NEXT: lui a5, 524288 -; RV32IZHINX-NEXT: lui a3, 524288 -; RV32IZHINX-NEXT: beqz s1, .LBB5_4 +; RV32IZHINX-NEXT: li a6, 1 +; RV32IZHINX-NEXT: lui a4, 524288 +; RV32IZHINX-NEXT: bne s1, a6, .LBB5_4 ; RV32IZHINX-NEXT: # %bb.3: -; RV32IZHINX-NEXT: mv a3, a1 +; RV32IZHINX-NEXT: mv a4, a1 ; RV32IZHINX-NEXT: .LBB5_4: ; RV32IZHINX-NEXT: and a0, a2, a0 ; RV32IZHINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload @@ -906,11 +914,11 @@ define i64 @test_ceil_si64(half %x) nounwind { ; RV32IZHINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: lw s2, 0(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: addi sp, sp, 16 -; RV32IZHINX-NEXT: beqz a4, .LBB5_6 +; RV32IZHINX-NEXT: beqz a3, .LBB5_6 ; RV32IZHINX-NEXT: # %bb.5: -; RV32IZHINX-NEXT: addi a3, a5, -1 +; RV32IZHINX-NEXT: addi a4, a5, -1 ; RV32IZHINX-NEXT: .LBB5_6: -; RV32IZHINX-NEXT: and a1, a2, a3 +; RV32IZHINX-NEXT: and a1, a2, a4 ; RV32IZHINX-NEXT: ret ; ; RV64IZHINX-LABEL: test_ceil_si64: @@ -948,39 +956,42 @@ define i64 @test_ceil_si64(half %x) nounwind { ; RV32IZFHMIN-NEXT: addi sp, sp, -16 ; RV32IZFHMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFHMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFHMIN-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFHMIN-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32IZFHMIN-NEXT: fsw fs0, 0(sp) # 4-byte Folded Spill ; RV32IZFHMIN-NEXT: fcvt.h.s fa5, fa5 ; RV32IZFHMIN-NEXT: fcvt.s.h fs0, fa5 ; RV32IZFHMIN-NEXT: lui a0, 913408 ; RV32IZFHMIN-NEXT: fmv.w.x fa5, a0 ; RV32IZFHMIN-NEXT: fle.s s0, fa5, fs0 +; RV32IZFHMIN-NEXT: neg s1, s0 ; RV32IZFHMIN-NEXT: fmv.s fa0, fs0 ; RV32IZFHMIN-NEXT: call __fixsfdi +; RV32IZFHMIN-NEXT: lui a2, %hi(.LCPI5_0) +; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI5_0)(a2) +; RV32IZFHMIN-NEXT: and a0, s1, a0 +; RV32IZFHMIN-NEXT: flt.s a3, fa5, fs0 +; RV32IZFHMIN-NEXT: neg a2, a3 +; RV32IZFHMIN-NEXT: or a0, a2, a0 +; RV32IZFHMIN-NEXT: feq.s a2, fs0, fs0 +; RV32IZFHMIN-NEXT: neg a2, a2 +; RV32IZFHMIN-NEXT: lui a5, 524288 +; RV32IZFHMIN-NEXT: li a6, 1 ; RV32IZFHMIN-NEXT: lui a4, 524288 -; RV32IZFHMIN-NEXT: lui a2, 524288 -; RV32IZFHMIN-NEXT: beqz s0, .LBB5_4 +; RV32IZFHMIN-NEXT: bne s0, a6, .LBB5_4 ; RV32IZFHMIN-NEXT: # %bb.3: -; RV32IZFHMIN-NEXT: mv a2, a1 +; RV32IZFHMIN-NEXT: mv a4, a1 ; RV32IZFHMIN-NEXT: .LBB5_4: -; RV32IZFHMIN-NEXT: lui a1, %hi(.LCPI5_0) -; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI5_0)(a1) -; RV32IZFHMIN-NEXT: flt.s a3, fa5, fs0 -; RV32IZFHMIN-NEXT: beqz a3, .LBB5_6 -; RV32IZFHMIN-NEXT: # %bb.5: -; RV32IZFHMIN-NEXT: addi a2, a4, -1 -; RV32IZFHMIN-NEXT: .LBB5_6: -; RV32IZFHMIN-NEXT: feq.s a1, fs0, fs0 -; RV32IZFHMIN-NEXT: neg a4, a1 -; RV32IZFHMIN-NEXT: and a1, a4, a2 -; RV32IZFHMIN-NEXT: neg a2, s0 ; RV32IZFHMIN-NEXT: and a0, a2, a0 -; RV32IZFHMIN-NEXT: neg a2, a3 -; RV32IZFHMIN-NEXT: or a0, a2, a0 -; RV32IZFHMIN-NEXT: and a0, a4, a0 ; RV32IZFHMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFHMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFHMIN-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFHMIN-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IZFHMIN-NEXT: flw fs0, 0(sp) # 4-byte Folded Reload ; RV32IZFHMIN-NEXT: addi sp, sp, 16 +; RV32IZFHMIN-NEXT: beqz a3, .LBB5_6 +; RV32IZFHMIN-NEXT: # %bb.5: +; RV32IZFHMIN-NEXT: addi a4, a5, -1 +; RV32IZFHMIN-NEXT: .LBB5_6: +; RV32IZFHMIN-NEXT: and a1, a2, a4 ; RV32IZFHMIN-NEXT: ret ; ; RV64IZFHMIN-LABEL: test_ceil_si64: @@ -1032,16 +1043,17 @@ define i64 @test_ceil_si64(half %x) nounwind { ; RV32IZHINXMIN-NEXT: lui a2, %hi(.LCPI5_0) ; RV32IZHINXMIN-NEXT: lw a2, %lo(.LCPI5_0)(a2) ; RV32IZHINXMIN-NEXT: and a0, s2, a0 -; RV32IZHINXMIN-NEXT: flt.s a4, a2, s0 -; RV32IZHINXMIN-NEXT: neg a2, a4 +; RV32IZHINXMIN-NEXT: flt.s a3, a2, s0 +; RV32IZHINXMIN-NEXT: neg a2, a3 ; RV32IZHINXMIN-NEXT: or a0, a2, a0 ; RV32IZHINXMIN-NEXT: feq.s a2, s0, s0 ; RV32IZHINXMIN-NEXT: neg a2, a2 ; RV32IZHINXMIN-NEXT: lui a5, 524288 -; RV32IZHINXMIN-NEXT: lui a3, 524288 -; RV32IZHINXMIN-NEXT: beqz s1, .LBB5_4 +; RV32IZHINXMIN-NEXT: li a6, 1 +; RV32IZHINXMIN-NEXT: lui a4, 524288 +; RV32IZHINXMIN-NEXT: bne s1, a6, .LBB5_4 ; RV32IZHINXMIN-NEXT: # %bb.3: -; RV32IZHINXMIN-NEXT: mv a3, a1 +; RV32IZHINXMIN-NEXT: mv a4, a1 ; RV32IZHINXMIN-NEXT: .LBB5_4: ; RV32IZHINXMIN-NEXT: and a0, a2, a0 ; RV32IZHINXMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload @@ -1049,11 +1061,11 @@ define i64 @test_ceil_si64(half %x) nounwind { ; RV32IZHINXMIN-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: lw s2, 0(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: addi sp, sp, 16 -; RV32IZHINXMIN-NEXT: beqz a4, .LBB5_6 +; RV32IZHINXMIN-NEXT: beqz a3, .LBB5_6 ; RV32IZHINXMIN-NEXT: # %bb.5: -; RV32IZHINXMIN-NEXT: addi a3, a5, -1 +; RV32IZHINXMIN-NEXT: addi a4, a5, -1 ; RV32IZHINXMIN-NEXT: .LBB5_6: -; RV32IZHINXMIN-NEXT: and a1, a2, a3 +; RV32IZHINXMIN-NEXT: and a1, a2, a4 ; RV32IZHINXMIN-NEXT: ret ; ; RV64IZHINXMIN-LABEL: test_ceil_si64: @@ -1125,7 +1137,7 @@ define signext i32 @test_ceil_ui32(half %x) { ; RV64IZHINX-NEXT: fcvt.wu.h a1, a0, rtz ; RV64IZHINX-NEXT: feq.h a0, a0, a0 ; RV64IZHINX-NEXT: seqz a0, a0 -; RV64IZHINX-NEXT: addi a0, a0, -1 +; RV64IZHINX-NEXT: addiw a0, a0, -1 ; RV64IZHINX-NEXT: and a0, a1, a0 ; RV64IZHINX-NEXT: ret ; @@ -1169,7 +1181,7 @@ define signext i32 @test_ceil_ui32(half %x) { ; RV64IZFHMIN-NEXT: fcvt.wu.s a0, fa5, rtz ; RV64IZFHMIN-NEXT: feq.s a1, fa5, fa5 ; RV64IZFHMIN-NEXT: seqz a1, a1 -; RV64IZFHMIN-NEXT: addi a1, a1, -1 +; RV64IZFHMIN-NEXT: addiw a1, a1, -1 ; RV64IZFHMIN-NEXT: and a0, a0, a1 ; RV64IZFHMIN-NEXT: ret ; @@ -1211,7 +1223,7 @@ define signext i32 @test_ceil_ui32(half %x) { ; RV64IZHINXMIN-NEXT: fcvt.wu.s a1, a0, rtz ; RV64IZHINXMIN-NEXT: feq.s a0, a0, a0 ; RV64IZHINXMIN-NEXT: seqz a0, a0 -; RV64IZHINXMIN-NEXT: addi a0, a0, -1 +; RV64IZHINXMIN-NEXT: addiw a0, a0, -1 ; RV64IZHINXMIN-NEXT: and a0, a1, a0 ; RV64IZHINXMIN-NEXT: ret %a = call half @llvm.ceil.f16(half %x) @@ -1234,25 +1246,24 @@ define i64 @test_ceil_ui64(half %x) nounwind { ; RV32IZFH-NEXT: .LBB7_2: ; RV32IZFH-NEXT: addi sp, sp, -16 ; RV32IZFH-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFH-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFH-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: fsw fs0, 8(sp) # 4-byte Folded Spill ; RV32IZFH-NEXT: fcvt.s.h fs0, fa0 -; RV32IZFH-NEXT: fmv.w.x fa5, zero -; RV32IZFH-NEXT: fle.s a0, fa5, fs0 -; RV32IZFH-NEXT: neg s0, a0 ; RV32IZFH-NEXT: fmv.s fa0, fs0 ; RV32IZFH-NEXT: call __fixunssfdi -; RV32IZFH-NEXT: lui a2, %hi(.LCPI7_1) -; RV32IZFH-NEXT: flw fa5, %lo(.LCPI7_1)(a2) -; RV32IZFH-NEXT: and a0, s0, a0 -; RV32IZFH-NEXT: flt.s a2, fa5, fs0 -; RV32IZFH-NEXT: neg a2, a2 -; RV32IZFH-NEXT: or a0, a2, a0 -; RV32IZFH-NEXT: and a1, s0, a1 -; RV32IZFH-NEXT: or a1, a2, a1 +; RV32IZFH-NEXT: fmv.w.x fa5, zero +; RV32IZFH-NEXT: fle.s a2, fa5, fs0 +; RV32IZFH-NEXT: lui a3, %hi(.LCPI7_1) +; RV32IZFH-NEXT: flw fa5, %lo(.LCPI7_1)(a3) +; RV32IZFH-NEXT: xori a2, a2, 1 +; RV32IZFH-NEXT: addi a2, a2, -1 +; RV32IZFH-NEXT: and a0, a2, a0 +; RV32IZFH-NEXT: flt.s a3, fa5, fs0 +; RV32IZFH-NEXT: neg a3, a3 +; RV32IZFH-NEXT: or a0, a3, a0 +; RV32IZFH-NEXT: and a1, a2, a1 +; RV32IZFH-NEXT: or a1, a3, a1 ; RV32IZFH-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFH-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFH-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: flw fs0, 8(sp) # 4-byte Folded Reload ; RV32IZFH-NEXT: addi sp, sp, 16 ; RV32IZFH-NEXT: ret ; @@ -1280,23 +1291,22 @@ define i64 @test_ceil_ui64(half %x) nounwind { ; RV32IZHINX-NEXT: addi sp, sp, -16 ; RV32IZHINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZHINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZHINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZHINX-NEXT: fcvt.s.h s0, a0 -; RV32IZHINX-NEXT: fle.s a0, zero, s0 -; RV32IZHINX-NEXT: neg s1, a0 ; RV32IZHINX-NEXT: mv a0, s0 ; RV32IZHINX-NEXT: call __fixunssfdi -; RV32IZHINX-NEXT: lui a2, %hi(.LCPI7_1) -; RV32IZHINX-NEXT: lw a2, %lo(.LCPI7_1)(a2) -; RV32IZHINX-NEXT: and a0, s1, a0 -; RV32IZHINX-NEXT: flt.s a2, a2, s0 -; RV32IZHINX-NEXT: neg a2, a2 -; RV32IZHINX-NEXT: or a0, a2, a0 -; RV32IZHINX-NEXT: and a1, s1, a1 -; RV32IZHINX-NEXT: or a1, a2, a1 +; RV32IZHINX-NEXT: fle.s a2, zero, s0 +; RV32IZHINX-NEXT: lui a3, %hi(.LCPI7_1) +; RV32IZHINX-NEXT: lw a3, %lo(.LCPI7_1)(a3) +; RV32IZHINX-NEXT: xori a2, a2, 1 +; RV32IZHINX-NEXT: addi a2, a2, -1 +; RV32IZHINX-NEXT: and a0, a2, a0 +; RV32IZHINX-NEXT: flt.s a3, a3, s0 +; RV32IZHINX-NEXT: neg a3, a3 +; RV32IZHINX-NEXT: or a0, a3, a0 +; RV32IZHINX-NEXT: and a1, a2, a1 +; RV32IZHINX-NEXT: or a1, a3, a1 ; RV32IZHINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZHINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: addi sp, sp, 16 ; RV32IZHINX-NEXT: ret ; @@ -1334,26 +1344,25 @@ define i64 @test_ceil_ui64(half %x) nounwind { ; RV32IZFHMIN-NEXT: .LBB7_2: ; RV32IZFHMIN-NEXT: addi sp, sp, -16 ; RV32IZFHMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFHMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFHMIN-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFHMIN-NEXT: fsw fs0, 8(sp) # 4-byte Folded Spill ; RV32IZFHMIN-NEXT: fcvt.h.s fa5, fa5 ; RV32IZFHMIN-NEXT: fcvt.s.h fs0, fa5 -; RV32IZFHMIN-NEXT: fmv.w.x fa5, zero -; RV32IZFHMIN-NEXT: fle.s a0, fa5, fs0 -; RV32IZFHMIN-NEXT: neg s0, a0 ; RV32IZFHMIN-NEXT: fmv.s fa0, fs0 ; RV32IZFHMIN-NEXT: call __fixunssfdi -; RV32IZFHMIN-NEXT: lui a2, %hi(.LCPI7_0) -; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI7_0)(a2) -; RV32IZFHMIN-NEXT: and a0, s0, a0 -; RV32IZFHMIN-NEXT: flt.s a2, fa5, fs0 -; RV32IZFHMIN-NEXT: neg a2, a2 -; RV32IZFHMIN-NEXT: or a0, a2, a0 -; RV32IZFHMIN-NEXT: and a1, s0, a1 -; RV32IZFHMIN-NEXT: or a1, a2, a1 +; RV32IZFHMIN-NEXT: fmv.w.x fa5, zero +; RV32IZFHMIN-NEXT: fle.s a2, fa5, fs0 +; RV32IZFHMIN-NEXT: lui a3, %hi(.LCPI7_0) +; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI7_0)(a3) +; RV32IZFHMIN-NEXT: xori a2, a2, 1 +; RV32IZFHMIN-NEXT: addi a2, a2, -1 +; RV32IZFHMIN-NEXT: and a0, a2, a0 +; RV32IZFHMIN-NEXT: flt.s a3, fa5, fs0 +; RV32IZFHMIN-NEXT: neg a3, a3 +; RV32IZFHMIN-NEXT: or a0, a3, a0 +; RV32IZFHMIN-NEXT: and a1, a2, a1 +; RV32IZFHMIN-NEXT: or a1, a3, a1 ; RV32IZFHMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFHMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFHMIN-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFHMIN-NEXT: flw fs0, 8(sp) # 4-byte Folded Reload ; RV32IZFHMIN-NEXT: addi sp, sp, 16 ; RV32IZFHMIN-NEXT: ret ; @@ -1394,24 +1403,23 @@ define i64 @test_ceil_ui64(half %x) nounwind { ; RV32IZHINXMIN-NEXT: addi sp, sp, -16 ; RV32IZHINXMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZHINXMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZHINXMIN-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZHINXMIN-NEXT: fcvt.h.s a0, a0 ; RV32IZHINXMIN-NEXT: fcvt.s.h s0, a0 -; RV32IZHINXMIN-NEXT: fle.s a0, zero, s0 -; RV32IZHINXMIN-NEXT: neg s1, a0 ; RV32IZHINXMIN-NEXT: mv a0, s0 ; RV32IZHINXMIN-NEXT: call __fixunssfdi -; RV32IZHINXMIN-NEXT: lui a2, %hi(.LCPI7_0) -; RV32IZHINXMIN-NEXT: lw a2, %lo(.LCPI7_0)(a2) -; RV32IZHINXMIN-NEXT: and a0, s1, a0 -; RV32IZHINXMIN-NEXT: flt.s a2, a2, s0 -; RV32IZHINXMIN-NEXT: neg a2, a2 -; RV32IZHINXMIN-NEXT: or a0, a2, a0 -; RV32IZHINXMIN-NEXT: and a1, s1, a1 -; RV32IZHINXMIN-NEXT: or a1, a2, a1 +; RV32IZHINXMIN-NEXT: fle.s a2, zero, s0 +; RV32IZHINXMIN-NEXT: lui a3, %hi(.LCPI7_0) +; RV32IZHINXMIN-NEXT: lw a3, %lo(.LCPI7_0)(a3) +; RV32IZHINXMIN-NEXT: xori a2, a2, 1 +; RV32IZHINXMIN-NEXT: addi a2, a2, -1 +; RV32IZHINXMIN-NEXT: and a0, a2, a0 +; RV32IZHINXMIN-NEXT: flt.s a3, a3, s0 +; RV32IZHINXMIN-NEXT: neg a3, a3 +; RV32IZHINXMIN-NEXT: or a0, a3, a0 +; RV32IZHINXMIN-NEXT: and a1, a2, a1 +; RV32IZHINXMIN-NEXT: or a1, a3, a1 ; RV32IZHINXMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZHINXMIN-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: addi sp, sp, 16 ; RV32IZHINXMIN-NEXT: ret ; @@ -1532,38 +1540,41 @@ define i64 @test_trunc_si64(half %x) nounwind { ; RV32IZFH-NEXT: addi sp, sp, -16 ; RV32IZFH-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFH-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFH-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: fsw fs0, 0(sp) # 4-byte Folded Spill ; RV32IZFH-NEXT: fcvt.s.h fs0, fa0 ; RV32IZFH-NEXT: lui a0, 913408 ; RV32IZFH-NEXT: fmv.w.x fa5, a0 ; RV32IZFH-NEXT: fle.s s0, fa5, fs0 +; RV32IZFH-NEXT: neg s1, s0 ; RV32IZFH-NEXT: fmv.s fa0, fs0 ; RV32IZFH-NEXT: call __fixsfdi +; RV32IZFH-NEXT: lui a2, %hi(.LCPI9_1) +; RV32IZFH-NEXT: flw fa5, %lo(.LCPI9_1)(a2) +; RV32IZFH-NEXT: and a0, s1, a0 +; RV32IZFH-NEXT: flt.s a3, fa5, fs0 +; RV32IZFH-NEXT: neg a2, a3 +; RV32IZFH-NEXT: or a0, a2, a0 +; RV32IZFH-NEXT: feq.s a2, fs0, fs0 +; RV32IZFH-NEXT: neg a2, a2 +; RV32IZFH-NEXT: lui a5, 524288 +; RV32IZFH-NEXT: li a6, 1 ; RV32IZFH-NEXT: lui a4, 524288 -; RV32IZFH-NEXT: lui a2, 524288 -; RV32IZFH-NEXT: beqz s0, .LBB9_4 +; RV32IZFH-NEXT: bne s0, a6, .LBB9_4 ; RV32IZFH-NEXT: # %bb.3: -; RV32IZFH-NEXT: mv a2, a1 +; RV32IZFH-NEXT: mv a4, a1 ; RV32IZFH-NEXT: .LBB9_4: -; RV32IZFH-NEXT: lui a1, %hi(.LCPI9_1) -; RV32IZFH-NEXT: flw fa5, %lo(.LCPI9_1)(a1) -; RV32IZFH-NEXT: flt.s a3, fa5, fs0 -; RV32IZFH-NEXT: beqz a3, .LBB9_6 -; RV32IZFH-NEXT: # %bb.5: -; RV32IZFH-NEXT: addi a2, a4, -1 -; RV32IZFH-NEXT: .LBB9_6: -; RV32IZFH-NEXT: feq.s a1, fs0, fs0 -; RV32IZFH-NEXT: neg a4, a1 -; RV32IZFH-NEXT: and a1, a4, a2 -; RV32IZFH-NEXT: neg a2, s0 ; RV32IZFH-NEXT: and a0, a2, a0 -; RV32IZFH-NEXT: neg a2, a3 -; RV32IZFH-NEXT: or a0, a2, a0 -; RV32IZFH-NEXT: and a0, a4, a0 ; RV32IZFH-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFH-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFH-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: flw fs0, 0(sp) # 4-byte Folded Reload ; RV32IZFH-NEXT: addi sp, sp, 16 +; RV32IZFH-NEXT: beqz a3, .LBB9_6 +; RV32IZFH-NEXT: # %bb.5: +; RV32IZFH-NEXT: addi a4, a5, -1 +; RV32IZFH-NEXT: .LBB9_6: +; RV32IZFH-NEXT: and a1, a2, a4 ; RV32IZFH-NEXT: ret ; ; RV64IZFH-LABEL: test_trunc_si64: @@ -1601,16 +1612,17 @@ define i64 @test_trunc_si64(half %x) nounwind { ; RV32IZHINX-NEXT: lui a2, %hi(.LCPI9_1) ; RV32IZHINX-NEXT: lw a2, %lo(.LCPI9_1)(a2) ; RV32IZHINX-NEXT: and a0, s2, a0 -; RV32IZHINX-NEXT: flt.s a4, a2, s0 -; RV32IZHINX-NEXT: neg a2, a4 +; RV32IZHINX-NEXT: flt.s a3, a2, s0 +; RV32IZHINX-NEXT: neg a2, a3 ; RV32IZHINX-NEXT: or a0, a2, a0 ; RV32IZHINX-NEXT: feq.s a2, s0, s0 ; RV32IZHINX-NEXT: neg a2, a2 ; RV32IZHINX-NEXT: lui a5, 524288 -; RV32IZHINX-NEXT: lui a3, 524288 -; RV32IZHINX-NEXT: beqz s1, .LBB9_4 +; RV32IZHINX-NEXT: li a6, 1 +; RV32IZHINX-NEXT: lui a4, 524288 +; RV32IZHINX-NEXT: bne s1, a6, .LBB9_4 ; RV32IZHINX-NEXT: # %bb.3: -; RV32IZHINX-NEXT: mv a3, a1 +; RV32IZHINX-NEXT: mv a4, a1 ; RV32IZHINX-NEXT: .LBB9_4: ; RV32IZHINX-NEXT: and a0, a2, a0 ; RV32IZHINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload @@ -1618,11 +1630,11 @@ define i64 @test_trunc_si64(half %x) nounwind { ; RV32IZHINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: lw s2, 0(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: addi sp, sp, 16 -; RV32IZHINX-NEXT: beqz a4, .LBB9_6 +; RV32IZHINX-NEXT: beqz a3, .LBB9_6 ; RV32IZHINX-NEXT: # %bb.5: -; RV32IZHINX-NEXT: addi a3, a5, -1 +; RV32IZHINX-NEXT: addi a4, a5, -1 ; RV32IZHINX-NEXT: .LBB9_6: -; RV32IZHINX-NEXT: and a1, a2, a3 +; RV32IZHINX-NEXT: and a1, a2, a4 ; RV32IZHINX-NEXT: ret ; ; RV64IZHINX-LABEL: test_trunc_si64: @@ -1660,39 +1672,42 @@ define i64 @test_trunc_si64(half %x) nounwind { ; RV32IZFHMIN-NEXT: addi sp, sp, -16 ; RV32IZFHMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFHMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFHMIN-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFHMIN-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32IZFHMIN-NEXT: fsw fs0, 0(sp) # 4-byte Folded Spill ; RV32IZFHMIN-NEXT: fcvt.h.s fa5, fa5 ; RV32IZFHMIN-NEXT: fcvt.s.h fs0, fa5 ; RV32IZFHMIN-NEXT: lui a0, 913408 ; RV32IZFHMIN-NEXT: fmv.w.x fa5, a0 ; RV32IZFHMIN-NEXT: fle.s s0, fa5, fs0 +; RV32IZFHMIN-NEXT: neg s1, s0 ; RV32IZFHMIN-NEXT: fmv.s fa0, fs0 ; RV32IZFHMIN-NEXT: call __fixsfdi +; RV32IZFHMIN-NEXT: lui a2, %hi(.LCPI9_0) +; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI9_0)(a2) +; RV32IZFHMIN-NEXT: and a0, s1, a0 +; RV32IZFHMIN-NEXT: flt.s a3, fa5, fs0 +; RV32IZFHMIN-NEXT: neg a2, a3 +; RV32IZFHMIN-NEXT: or a0, a2, a0 +; RV32IZFHMIN-NEXT: feq.s a2, fs0, fs0 +; RV32IZFHMIN-NEXT: neg a2, a2 +; RV32IZFHMIN-NEXT: lui a5, 524288 +; RV32IZFHMIN-NEXT: li a6, 1 ; RV32IZFHMIN-NEXT: lui a4, 524288 -; RV32IZFHMIN-NEXT: lui a2, 524288 -; RV32IZFHMIN-NEXT: beqz s0, .LBB9_4 +; RV32IZFHMIN-NEXT: bne s0, a6, .LBB9_4 ; RV32IZFHMIN-NEXT: # %bb.3: -; RV32IZFHMIN-NEXT: mv a2, a1 +; RV32IZFHMIN-NEXT: mv a4, a1 ; RV32IZFHMIN-NEXT: .LBB9_4: -; RV32IZFHMIN-NEXT: lui a1, %hi(.LCPI9_0) -; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI9_0)(a1) -; RV32IZFHMIN-NEXT: flt.s a3, fa5, fs0 -; RV32IZFHMIN-NEXT: beqz a3, .LBB9_6 -; RV32IZFHMIN-NEXT: # %bb.5: -; RV32IZFHMIN-NEXT: addi a2, a4, -1 -; RV32IZFHMIN-NEXT: .LBB9_6: -; RV32IZFHMIN-NEXT: feq.s a1, fs0, fs0 -; RV32IZFHMIN-NEXT: neg a4, a1 -; RV32IZFHMIN-NEXT: and a1, a4, a2 -; RV32IZFHMIN-NEXT: neg a2, s0 ; RV32IZFHMIN-NEXT: and a0, a2, a0 -; RV32IZFHMIN-NEXT: neg a2, a3 -; RV32IZFHMIN-NEXT: or a0, a2, a0 -; RV32IZFHMIN-NEXT: and a0, a4, a0 ; RV32IZFHMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFHMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFHMIN-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFHMIN-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IZFHMIN-NEXT: flw fs0, 0(sp) # 4-byte Folded Reload ; RV32IZFHMIN-NEXT: addi sp, sp, 16 +; RV32IZFHMIN-NEXT: beqz a3, .LBB9_6 +; RV32IZFHMIN-NEXT: # %bb.5: +; RV32IZFHMIN-NEXT: addi a4, a5, -1 +; RV32IZFHMIN-NEXT: .LBB9_6: +; RV32IZFHMIN-NEXT: and a1, a2, a4 ; RV32IZFHMIN-NEXT: ret ; ; RV64IZFHMIN-LABEL: test_trunc_si64: @@ -1744,16 +1759,17 @@ define i64 @test_trunc_si64(half %x) nounwind { ; RV32IZHINXMIN-NEXT: lui a2, %hi(.LCPI9_0) ; RV32IZHINXMIN-NEXT: lw a2, %lo(.LCPI9_0)(a2) ; RV32IZHINXMIN-NEXT: and a0, s2, a0 -; RV32IZHINXMIN-NEXT: flt.s a4, a2, s0 -; RV32IZHINXMIN-NEXT: neg a2, a4 +; RV32IZHINXMIN-NEXT: flt.s a3, a2, s0 +; RV32IZHINXMIN-NEXT: neg a2, a3 ; RV32IZHINXMIN-NEXT: or a0, a2, a0 ; RV32IZHINXMIN-NEXT: feq.s a2, s0, s0 ; RV32IZHINXMIN-NEXT: neg a2, a2 ; RV32IZHINXMIN-NEXT: lui a5, 524288 -; RV32IZHINXMIN-NEXT: lui a3, 524288 -; RV32IZHINXMIN-NEXT: beqz s1, .LBB9_4 +; RV32IZHINXMIN-NEXT: li a6, 1 +; RV32IZHINXMIN-NEXT: lui a4, 524288 +; RV32IZHINXMIN-NEXT: bne s1, a6, .LBB9_4 ; RV32IZHINXMIN-NEXT: # %bb.3: -; RV32IZHINXMIN-NEXT: mv a3, a1 +; RV32IZHINXMIN-NEXT: mv a4, a1 ; RV32IZHINXMIN-NEXT: .LBB9_4: ; RV32IZHINXMIN-NEXT: and a0, a2, a0 ; RV32IZHINXMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload @@ -1761,11 +1777,11 @@ define i64 @test_trunc_si64(half %x) nounwind { ; RV32IZHINXMIN-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: lw s2, 0(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: addi sp, sp, 16 -; RV32IZHINXMIN-NEXT: beqz a4, .LBB9_6 +; RV32IZHINXMIN-NEXT: beqz a3, .LBB9_6 ; RV32IZHINXMIN-NEXT: # %bb.5: -; RV32IZHINXMIN-NEXT: addi a3, a5, -1 +; RV32IZHINXMIN-NEXT: addi a4, a5, -1 ; RV32IZHINXMIN-NEXT: .LBB9_6: -; RV32IZHINXMIN-NEXT: and a1, a2, a3 +; RV32IZHINXMIN-NEXT: and a1, a2, a4 ; RV32IZHINXMIN-NEXT: ret ; ; RV64IZHINXMIN-LABEL: test_trunc_si64: @@ -1837,7 +1853,7 @@ define signext i32 @test_trunc_ui32(half %x) { ; RV64IZHINX-NEXT: fcvt.wu.h a1, a0, rtz ; RV64IZHINX-NEXT: feq.h a0, a0, a0 ; RV64IZHINX-NEXT: seqz a0, a0 -; RV64IZHINX-NEXT: addi a0, a0, -1 +; RV64IZHINX-NEXT: addiw a0, a0, -1 ; RV64IZHINX-NEXT: and a0, a1, a0 ; RV64IZHINX-NEXT: ret ; @@ -1881,7 +1897,7 @@ define signext i32 @test_trunc_ui32(half %x) { ; RV64IZFHMIN-NEXT: fcvt.wu.s a0, fa5, rtz ; RV64IZFHMIN-NEXT: feq.s a1, fa5, fa5 ; RV64IZFHMIN-NEXT: seqz a1, a1 -; RV64IZFHMIN-NEXT: addi a1, a1, -1 +; RV64IZFHMIN-NEXT: addiw a1, a1, -1 ; RV64IZFHMIN-NEXT: and a0, a0, a1 ; RV64IZFHMIN-NEXT: ret ; @@ -1923,7 +1939,7 @@ define signext i32 @test_trunc_ui32(half %x) { ; RV64IZHINXMIN-NEXT: fcvt.wu.s a1, a0, rtz ; RV64IZHINXMIN-NEXT: feq.s a0, a0, a0 ; RV64IZHINXMIN-NEXT: seqz a0, a0 -; RV64IZHINXMIN-NEXT: addi a0, a0, -1 +; RV64IZHINXMIN-NEXT: addiw a0, a0, -1 ; RV64IZHINXMIN-NEXT: and a0, a1, a0 ; RV64IZHINXMIN-NEXT: ret %a = call half @llvm.trunc.f16(half %x) @@ -1946,25 +1962,24 @@ define i64 @test_trunc_ui64(half %x) nounwind { ; RV32IZFH-NEXT: .LBB11_2: ; RV32IZFH-NEXT: addi sp, sp, -16 ; RV32IZFH-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFH-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFH-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: fsw fs0, 8(sp) # 4-byte Folded Spill ; RV32IZFH-NEXT: fcvt.s.h fs0, fa0 -; RV32IZFH-NEXT: fmv.w.x fa5, zero -; RV32IZFH-NEXT: fle.s a0, fa5, fs0 -; RV32IZFH-NEXT: neg s0, a0 ; RV32IZFH-NEXT: fmv.s fa0, fs0 ; RV32IZFH-NEXT: call __fixunssfdi -; RV32IZFH-NEXT: lui a2, %hi(.LCPI11_1) -; RV32IZFH-NEXT: flw fa5, %lo(.LCPI11_1)(a2) -; RV32IZFH-NEXT: and a0, s0, a0 -; RV32IZFH-NEXT: flt.s a2, fa5, fs0 -; RV32IZFH-NEXT: neg a2, a2 -; RV32IZFH-NEXT: or a0, a2, a0 -; RV32IZFH-NEXT: and a1, s0, a1 -; RV32IZFH-NEXT: or a1, a2, a1 +; RV32IZFH-NEXT: fmv.w.x fa5, zero +; RV32IZFH-NEXT: fle.s a2, fa5, fs0 +; RV32IZFH-NEXT: lui a3, %hi(.LCPI11_1) +; RV32IZFH-NEXT: flw fa5, %lo(.LCPI11_1)(a3) +; RV32IZFH-NEXT: xori a2, a2, 1 +; RV32IZFH-NEXT: addi a2, a2, -1 +; RV32IZFH-NEXT: and a0, a2, a0 +; RV32IZFH-NEXT: flt.s a3, fa5, fs0 +; RV32IZFH-NEXT: neg a3, a3 +; RV32IZFH-NEXT: or a0, a3, a0 +; RV32IZFH-NEXT: and a1, a2, a1 +; RV32IZFH-NEXT: or a1, a3, a1 ; RV32IZFH-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFH-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFH-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: flw fs0, 8(sp) # 4-byte Folded Reload ; RV32IZFH-NEXT: addi sp, sp, 16 ; RV32IZFH-NEXT: ret ; @@ -1992,23 +2007,22 @@ define i64 @test_trunc_ui64(half %x) nounwind { ; RV32IZHINX-NEXT: addi sp, sp, -16 ; RV32IZHINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZHINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZHINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZHINX-NEXT: fcvt.s.h s0, a0 -; RV32IZHINX-NEXT: fle.s a0, zero, s0 -; RV32IZHINX-NEXT: neg s1, a0 ; RV32IZHINX-NEXT: mv a0, s0 ; RV32IZHINX-NEXT: call __fixunssfdi -; RV32IZHINX-NEXT: lui a2, %hi(.LCPI11_1) -; RV32IZHINX-NEXT: lw a2, %lo(.LCPI11_1)(a2) -; RV32IZHINX-NEXT: and a0, s1, a0 -; RV32IZHINX-NEXT: flt.s a2, a2, s0 -; RV32IZHINX-NEXT: neg a2, a2 -; RV32IZHINX-NEXT: or a0, a2, a0 -; RV32IZHINX-NEXT: and a1, s1, a1 -; RV32IZHINX-NEXT: or a1, a2, a1 +; RV32IZHINX-NEXT: fle.s a2, zero, s0 +; RV32IZHINX-NEXT: lui a3, %hi(.LCPI11_1) +; RV32IZHINX-NEXT: lw a3, %lo(.LCPI11_1)(a3) +; RV32IZHINX-NEXT: xori a2, a2, 1 +; RV32IZHINX-NEXT: addi a2, a2, -1 +; RV32IZHINX-NEXT: and a0, a2, a0 +; RV32IZHINX-NEXT: flt.s a3, a3, s0 +; RV32IZHINX-NEXT: neg a3, a3 +; RV32IZHINX-NEXT: or a0, a3, a0 +; RV32IZHINX-NEXT: and a1, a2, a1 +; RV32IZHINX-NEXT: or a1, a3, a1 ; RV32IZHINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZHINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: addi sp, sp, 16 ; RV32IZHINX-NEXT: ret ; @@ -2046,26 +2060,25 @@ define i64 @test_trunc_ui64(half %x) nounwind { ; RV32IZFHMIN-NEXT: .LBB11_2: ; RV32IZFHMIN-NEXT: addi sp, sp, -16 ; RV32IZFHMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFHMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFHMIN-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFHMIN-NEXT: fsw fs0, 8(sp) # 4-byte Folded Spill ; RV32IZFHMIN-NEXT: fcvt.h.s fa5, fa5 ; RV32IZFHMIN-NEXT: fcvt.s.h fs0, fa5 -; RV32IZFHMIN-NEXT: fmv.w.x fa5, zero -; RV32IZFHMIN-NEXT: fle.s a0, fa5, fs0 -; RV32IZFHMIN-NEXT: neg s0, a0 ; RV32IZFHMIN-NEXT: fmv.s fa0, fs0 ; RV32IZFHMIN-NEXT: call __fixunssfdi -; RV32IZFHMIN-NEXT: lui a2, %hi(.LCPI11_0) -; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI11_0)(a2) -; RV32IZFHMIN-NEXT: and a0, s0, a0 -; RV32IZFHMIN-NEXT: flt.s a2, fa5, fs0 -; RV32IZFHMIN-NEXT: neg a2, a2 -; RV32IZFHMIN-NEXT: or a0, a2, a0 -; RV32IZFHMIN-NEXT: and a1, s0, a1 -; RV32IZFHMIN-NEXT: or a1, a2, a1 +; RV32IZFHMIN-NEXT: fmv.w.x fa5, zero +; RV32IZFHMIN-NEXT: fle.s a2, fa5, fs0 +; RV32IZFHMIN-NEXT: lui a3, %hi(.LCPI11_0) +; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI11_0)(a3) +; RV32IZFHMIN-NEXT: xori a2, a2, 1 +; RV32IZFHMIN-NEXT: addi a2, a2, -1 +; RV32IZFHMIN-NEXT: and a0, a2, a0 +; RV32IZFHMIN-NEXT: flt.s a3, fa5, fs0 +; RV32IZFHMIN-NEXT: neg a3, a3 +; RV32IZFHMIN-NEXT: or a0, a3, a0 +; RV32IZFHMIN-NEXT: and a1, a2, a1 +; RV32IZFHMIN-NEXT: or a1, a3, a1 ; RV32IZFHMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFHMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFHMIN-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFHMIN-NEXT: flw fs0, 8(sp) # 4-byte Folded Reload ; RV32IZFHMIN-NEXT: addi sp, sp, 16 ; RV32IZFHMIN-NEXT: ret ; @@ -2106,24 +2119,23 @@ define i64 @test_trunc_ui64(half %x) nounwind { ; RV32IZHINXMIN-NEXT: addi sp, sp, -16 ; RV32IZHINXMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZHINXMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZHINXMIN-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZHINXMIN-NEXT: fcvt.h.s a0, a0 ; RV32IZHINXMIN-NEXT: fcvt.s.h s0, a0 -; RV32IZHINXMIN-NEXT: fle.s a0, zero, s0 -; RV32IZHINXMIN-NEXT: neg s1, a0 ; RV32IZHINXMIN-NEXT: mv a0, s0 ; RV32IZHINXMIN-NEXT: call __fixunssfdi -; RV32IZHINXMIN-NEXT: lui a2, %hi(.LCPI11_0) -; RV32IZHINXMIN-NEXT: lw a2, %lo(.LCPI11_0)(a2) -; RV32IZHINXMIN-NEXT: and a0, s1, a0 -; RV32IZHINXMIN-NEXT: flt.s a2, a2, s0 -; RV32IZHINXMIN-NEXT: neg a2, a2 -; RV32IZHINXMIN-NEXT: or a0, a2, a0 -; RV32IZHINXMIN-NEXT: and a1, s1, a1 -; RV32IZHINXMIN-NEXT: or a1, a2, a1 +; RV32IZHINXMIN-NEXT: fle.s a2, zero, s0 +; RV32IZHINXMIN-NEXT: lui a3, %hi(.LCPI11_0) +; RV32IZHINXMIN-NEXT: lw a3, %lo(.LCPI11_0)(a3) +; RV32IZHINXMIN-NEXT: xori a2, a2, 1 +; RV32IZHINXMIN-NEXT: addi a2, a2, -1 +; RV32IZHINXMIN-NEXT: and a0, a2, a0 +; RV32IZHINXMIN-NEXT: flt.s a3, a3, s0 +; RV32IZHINXMIN-NEXT: neg a3, a3 +; RV32IZHINXMIN-NEXT: or a0, a3, a0 +; RV32IZHINXMIN-NEXT: and a1, a2, a1 +; RV32IZHINXMIN-NEXT: or a1, a3, a1 ; RV32IZHINXMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZHINXMIN-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: addi sp, sp, 16 ; RV32IZHINXMIN-NEXT: ret ; @@ -2244,38 +2256,41 @@ define i64 @test_round_si64(half %x) nounwind { ; RV32IZFH-NEXT: addi sp, sp, -16 ; RV32IZFH-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFH-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFH-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: fsw fs0, 0(sp) # 4-byte Folded Spill ; RV32IZFH-NEXT: fcvt.s.h fs0, fa0 ; RV32IZFH-NEXT: lui a0, 913408 ; RV32IZFH-NEXT: fmv.w.x fa5, a0 ; RV32IZFH-NEXT: fle.s s0, fa5, fs0 +; RV32IZFH-NEXT: neg s1, s0 ; RV32IZFH-NEXT: fmv.s fa0, fs0 ; RV32IZFH-NEXT: call __fixsfdi +; RV32IZFH-NEXT: lui a2, %hi(.LCPI13_1) +; RV32IZFH-NEXT: flw fa5, %lo(.LCPI13_1)(a2) +; RV32IZFH-NEXT: and a0, s1, a0 +; RV32IZFH-NEXT: flt.s a3, fa5, fs0 +; RV32IZFH-NEXT: neg a2, a3 +; RV32IZFH-NEXT: or a0, a2, a0 +; RV32IZFH-NEXT: feq.s a2, fs0, fs0 +; RV32IZFH-NEXT: neg a2, a2 +; RV32IZFH-NEXT: lui a5, 524288 +; RV32IZFH-NEXT: li a6, 1 ; RV32IZFH-NEXT: lui a4, 524288 -; RV32IZFH-NEXT: lui a2, 524288 -; RV32IZFH-NEXT: beqz s0, .LBB13_4 +; RV32IZFH-NEXT: bne s0, a6, .LBB13_4 ; RV32IZFH-NEXT: # %bb.3: -; RV32IZFH-NEXT: mv a2, a1 +; RV32IZFH-NEXT: mv a4, a1 ; RV32IZFH-NEXT: .LBB13_4: -; RV32IZFH-NEXT: lui a1, %hi(.LCPI13_1) -; RV32IZFH-NEXT: flw fa5, %lo(.LCPI13_1)(a1) -; RV32IZFH-NEXT: flt.s a3, fa5, fs0 -; RV32IZFH-NEXT: beqz a3, .LBB13_6 -; RV32IZFH-NEXT: # %bb.5: -; RV32IZFH-NEXT: addi a2, a4, -1 -; RV32IZFH-NEXT: .LBB13_6: -; RV32IZFH-NEXT: feq.s a1, fs0, fs0 -; RV32IZFH-NEXT: neg a4, a1 -; RV32IZFH-NEXT: and a1, a4, a2 -; RV32IZFH-NEXT: neg a2, s0 ; RV32IZFH-NEXT: and a0, a2, a0 -; RV32IZFH-NEXT: neg a2, a3 -; RV32IZFH-NEXT: or a0, a2, a0 -; RV32IZFH-NEXT: and a0, a4, a0 ; RV32IZFH-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFH-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFH-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: flw fs0, 0(sp) # 4-byte Folded Reload ; RV32IZFH-NEXT: addi sp, sp, 16 +; RV32IZFH-NEXT: beqz a3, .LBB13_6 +; RV32IZFH-NEXT: # %bb.5: +; RV32IZFH-NEXT: addi a4, a5, -1 +; RV32IZFH-NEXT: .LBB13_6: +; RV32IZFH-NEXT: and a1, a2, a4 ; RV32IZFH-NEXT: ret ; ; RV64IZFH-LABEL: test_round_si64: @@ -2313,16 +2328,17 @@ define i64 @test_round_si64(half %x) nounwind { ; RV32IZHINX-NEXT: lui a2, %hi(.LCPI13_1) ; RV32IZHINX-NEXT: lw a2, %lo(.LCPI13_1)(a2) ; RV32IZHINX-NEXT: and a0, s2, a0 -; RV32IZHINX-NEXT: flt.s a4, a2, s0 -; RV32IZHINX-NEXT: neg a2, a4 +; RV32IZHINX-NEXT: flt.s a3, a2, s0 +; RV32IZHINX-NEXT: neg a2, a3 ; RV32IZHINX-NEXT: or a0, a2, a0 ; RV32IZHINX-NEXT: feq.s a2, s0, s0 ; RV32IZHINX-NEXT: neg a2, a2 ; RV32IZHINX-NEXT: lui a5, 524288 -; RV32IZHINX-NEXT: lui a3, 524288 -; RV32IZHINX-NEXT: beqz s1, .LBB13_4 +; RV32IZHINX-NEXT: li a6, 1 +; RV32IZHINX-NEXT: lui a4, 524288 +; RV32IZHINX-NEXT: bne s1, a6, .LBB13_4 ; RV32IZHINX-NEXT: # %bb.3: -; RV32IZHINX-NEXT: mv a3, a1 +; RV32IZHINX-NEXT: mv a4, a1 ; RV32IZHINX-NEXT: .LBB13_4: ; RV32IZHINX-NEXT: and a0, a2, a0 ; RV32IZHINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload @@ -2330,11 +2346,11 @@ define i64 @test_round_si64(half %x) nounwind { ; RV32IZHINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: lw s2, 0(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: addi sp, sp, 16 -; RV32IZHINX-NEXT: beqz a4, .LBB13_6 +; RV32IZHINX-NEXT: beqz a3, .LBB13_6 ; RV32IZHINX-NEXT: # %bb.5: -; RV32IZHINX-NEXT: addi a3, a5, -1 +; RV32IZHINX-NEXT: addi a4, a5, -1 ; RV32IZHINX-NEXT: .LBB13_6: -; RV32IZHINX-NEXT: and a1, a2, a3 +; RV32IZHINX-NEXT: and a1, a2, a4 ; RV32IZHINX-NEXT: ret ; ; RV64IZHINX-LABEL: test_round_si64: @@ -2372,39 +2388,42 @@ define i64 @test_round_si64(half %x) nounwind { ; RV32IZFHMIN-NEXT: addi sp, sp, -16 ; RV32IZFHMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFHMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFHMIN-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFHMIN-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32IZFHMIN-NEXT: fsw fs0, 0(sp) # 4-byte Folded Spill ; RV32IZFHMIN-NEXT: fcvt.h.s fa5, fa5 ; RV32IZFHMIN-NEXT: fcvt.s.h fs0, fa5 ; RV32IZFHMIN-NEXT: lui a0, 913408 ; RV32IZFHMIN-NEXT: fmv.w.x fa5, a0 ; RV32IZFHMIN-NEXT: fle.s s0, fa5, fs0 +; RV32IZFHMIN-NEXT: neg s1, s0 ; RV32IZFHMIN-NEXT: fmv.s fa0, fs0 ; RV32IZFHMIN-NEXT: call __fixsfdi +; RV32IZFHMIN-NEXT: lui a2, %hi(.LCPI13_0) +; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI13_0)(a2) +; RV32IZFHMIN-NEXT: and a0, s1, a0 +; RV32IZFHMIN-NEXT: flt.s a3, fa5, fs0 +; RV32IZFHMIN-NEXT: neg a2, a3 +; RV32IZFHMIN-NEXT: or a0, a2, a0 +; RV32IZFHMIN-NEXT: feq.s a2, fs0, fs0 +; RV32IZFHMIN-NEXT: neg a2, a2 +; RV32IZFHMIN-NEXT: lui a5, 524288 +; RV32IZFHMIN-NEXT: li a6, 1 ; RV32IZFHMIN-NEXT: lui a4, 524288 -; RV32IZFHMIN-NEXT: lui a2, 524288 -; RV32IZFHMIN-NEXT: beqz s0, .LBB13_4 +; RV32IZFHMIN-NEXT: bne s0, a6, .LBB13_4 ; RV32IZFHMIN-NEXT: # %bb.3: -; RV32IZFHMIN-NEXT: mv a2, a1 +; RV32IZFHMIN-NEXT: mv a4, a1 ; RV32IZFHMIN-NEXT: .LBB13_4: -; RV32IZFHMIN-NEXT: lui a1, %hi(.LCPI13_0) -; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI13_0)(a1) -; RV32IZFHMIN-NEXT: flt.s a3, fa5, fs0 -; RV32IZFHMIN-NEXT: beqz a3, .LBB13_6 -; RV32IZFHMIN-NEXT: # %bb.5: -; RV32IZFHMIN-NEXT: addi a2, a4, -1 -; RV32IZFHMIN-NEXT: .LBB13_6: -; RV32IZFHMIN-NEXT: feq.s a1, fs0, fs0 -; RV32IZFHMIN-NEXT: neg a4, a1 -; RV32IZFHMIN-NEXT: and a1, a4, a2 -; RV32IZFHMIN-NEXT: neg a2, s0 ; RV32IZFHMIN-NEXT: and a0, a2, a0 -; RV32IZFHMIN-NEXT: neg a2, a3 -; RV32IZFHMIN-NEXT: or a0, a2, a0 -; RV32IZFHMIN-NEXT: and a0, a4, a0 ; RV32IZFHMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFHMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFHMIN-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFHMIN-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IZFHMIN-NEXT: flw fs0, 0(sp) # 4-byte Folded Reload ; RV32IZFHMIN-NEXT: addi sp, sp, 16 +; RV32IZFHMIN-NEXT: beqz a3, .LBB13_6 +; RV32IZFHMIN-NEXT: # %bb.5: +; RV32IZFHMIN-NEXT: addi a4, a5, -1 +; RV32IZFHMIN-NEXT: .LBB13_6: +; RV32IZFHMIN-NEXT: and a1, a2, a4 ; RV32IZFHMIN-NEXT: ret ; ; RV64IZFHMIN-LABEL: test_round_si64: @@ -2456,16 +2475,17 @@ define i64 @test_round_si64(half %x) nounwind { ; RV32IZHINXMIN-NEXT: lui a2, %hi(.LCPI13_0) ; RV32IZHINXMIN-NEXT: lw a2, %lo(.LCPI13_0)(a2) ; RV32IZHINXMIN-NEXT: and a0, s2, a0 -; RV32IZHINXMIN-NEXT: flt.s a4, a2, s0 -; RV32IZHINXMIN-NEXT: neg a2, a4 +; RV32IZHINXMIN-NEXT: flt.s a3, a2, s0 +; RV32IZHINXMIN-NEXT: neg a2, a3 ; RV32IZHINXMIN-NEXT: or a0, a2, a0 ; RV32IZHINXMIN-NEXT: feq.s a2, s0, s0 ; RV32IZHINXMIN-NEXT: neg a2, a2 ; RV32IZHINXMIN-NEXT: lui a5, 524288 -; RV32IZHINXMIN-NEXT: lui a3, 524288 -; RV32IZHINXMIN-NEXT: beqz s1, .LBB13_4 +; RV32IZHINXMIN-NEXT: li a6, 1 +; RV32IZHINXMIN-NEXT: lui a4, 524288 +; RV32IZHINXMIN-NEXT: bne s1, a6, .LBB13_4 ; RV32IZHINXMIN-NEXT: # %bb.3: -; RV32IZHINXMIN-NEXT: mv a3, a1 +; RV32IZHINXMIN-NEXT: mv a4, a1 ; RV32IZHINXMIN-NEXT: .LBB13_4: ; RV32IZHINXMIN-NEXT: and a0, a2, a0 ; RV32IZHINXMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload @@ -2473,11 +2493,11 @@ define i64 @test_round_si64(half %x) nounwind { ; RV32IZHINXMIN-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: lw s2, 0(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: addi sp, sp, 16 -; RV32IZHINXMIN-NEXT: beqz a4, .LBB13_6 +; RV32IZHINXMIN-NEXT: beqz a3, .LBB13_6 ; RV32IZHINXMIN-NEXT: # %bb.5: -; RV32IZHINXMIN-NEXT: addi a3, a5, -1 +; RV32IZHINXMIN-NEXT: addi a4, a5, -1 ; RV32IZHINXMIN-NEXT: .LBB13_6: -; RV32IZHINXMIN-NEXT: and a1, a2, a3 +; RV32IZHINXMIN-NEXT: and a1, a2, a4 ; RV32IZHINXMIN-NEXT: ret ; ; RV64IZHINXMIN-LABEL: test_round_si64: @@ -2549,7 +2569,7 @@ define signext i32 @test_round_ui32(half %x) { ; RV64IZHINX-NEXT: fcvt.wu.h a1, a0, rtz ; RV64IZHINX-NEXT: feq.h a0, a0, a0 ; RV64IZHINX-NEXT: seqz a0, a0 -; RV64IZHINX-NEXT: addi a0, a0, -1 +; RV64IZHINX-NEXT: addiw a0, a0, -1 ; RV64IZHINX-NEXT: and a0, a1, a0 ; RV64IZHINX-NEXT: ret ; @@ -2593,7 +2613,7 @@ define signext i32 @test_round_ui32(half %x) { ; RV64IZFHMIN-NEXT: fcvt.wu.s a0, fa5, rtz ; RV64IZFHMIN-NEXT: feq.s a1, fa5, fa5 ; RV64IZFHMIN-NEXT: seqz a1, a1 -; RV64IZFHMIN-NEXT: addi a1, a1, -1 +; RV64IZFHMIN-NEXT: addiw a1, a1, -1 ; RV64IZFHMIN-NEXT: and a0, a0, a1 ; RV64IZFHMIN-NEXT: ret ; @@ -2635,7 +2655,7 @@ define signext i32 @test_round_ui32(half %x) { ; RV64IZHINXMIN-NEXT: fcvt.wu.s a1, a0, rtz ; RV64IZHINXMIN-NEXT: feq.s a0, a0, a0 ; RV64IZHINXMIN-NEXT: seqz a0, a0 -; RV64IZHINXMIN-NEXT: addi a0, a0, -1 +; RV64IZHINXMIN-NEXT: addiw a0, a0, -1 ; RV64IZHINXMIN-NEXT: and a0, a1, a0 ; RV64IZHINXMIN-NEXT: ret %a = call half @llvm.round.f16(half %x) @@ -2658,25 +2678,24 @@ define i64 @test_round_ui64(half %x) nounwind { ; RV32IZFH-NEXT: .LBB15_2: ; RV32IZFH-NEXT: addi sp, sp, -16 ; RV32IZFH-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFH-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFH-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: fsw fs0, 8(sp) # 4-byte Folded Spill ; RV32IZFH-NEXT: fcvt.s.h fs0, fa0 -; RV32IZFH-NEXT: fmv.w.x fa5, zero -; RV32IZFH-NEXT: fle.s a0, fa5, fs0 -; RV32IZFH-NEXT: neg s0, a0 ; RV32IZFH-NEXT: fmv.s fa0, fs0 ; RV32IZFH-NEXT: call __fixunssfdi -; RV32IZFH-NEXT: lui a2, %hi(.LCPI15_1) -; RV32IZFH-NEXT: flw fa5, %lo(.LCPI15_1)(a2) -; RV32IZFH-NEXT: and a0, s0, a0 -; RV32IZFH-NEXT: flt.s a2, fa5, fs0 -; RV32IZFH-NEXT: neg a2, a2 -; RV32IZFH-NEXT: or a0, a2, a0 -; RV32IZFH-NEXT: and a1, s0, a1 -; RV32IZFH-NEXT: or a1, a2, a1 +; RV32IZFH-NEXT: fmv.w.x fa5, zero +; RV32IZFH-NEXT: fle.s a2, fa5, fs0 +; RV32IZFH-NEXT: lui a3, %hi(.LCPI15_1) +; RV32IZFH-NEXT: flw fa5, %lo(.LCPI15_1)(a3) +; RV32IZFH-NEXT: xori a2, a2, 1 +; RV32IZFH-NEXT: addi a2, a2, -1 +; RV32IZFH-NEXT: and a0, a2, a0 +; RV32IZFH-NEXT: flt.s a3, fa5, fs0 +; RV32IZFH-NEXT: neg a3, a3 +; RV32IZFH-NEXT: or a0, a3, a0 +; RV32IZFH-NEXT: and a1, a2, a1 +; RV32IZFH-NEXT: or a1, a3, a1 ; RV32IZFH-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFH-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFH-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: flw fs0, 8(sp) # 4-byte Folded Reload ; RV32IZFH-NEXT: addi sp, sp, 16 ; RV32IZFH-NEXT: ret ; @@ -2704,23 +2723,22 @@ define i64 @test_round_ui64(half %x) nounwind { ; RV32IZHINX-NEXT: addi sp, sp, -16 ; RV32IZHINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZHINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZHINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZHINX-NEXT: fcvt.s.h s0, a0 -; RV32IZHINX-NEXT: fle.s a0, zero, s0 -; RV32IZHINX-NEXT: neg s1, a0 ; RV32IZHINX-NEXT: mv a0, s0 ; RV32IZHINX-NEXT: call __fixunssfdi -; RV32IZHINX-NEXT: lui a2, %hi(.LCPI15_1) -; RV32IZHINX-NEXT: lw a2, %lo(.LCPI15_1)(a2) -; RV32IZHINX-NEXT: and a0, s1, a0 -; RV32IZHINX-NEXT: flt.s a2, a2, s0 -; RV32IZHINX-NEXT: neg a2, a2 -; RV32IZHINX-NEXT: or a0, a2, a0 -; RV32IZHINX-NEXT: and a1, s1, a1 -; RV32IZHINX-NEXT: or a1, a2, a1 +; RV32IZHINX-NEXT: fle.s a2, zero, s0 +; RV32IZHINX-NEXT: lui a3, %hi(.LCPI15_1) +; RV32IZHINX-NEXT: lw a3, %lo(.LCPI15_1)(a3) +; RV32IZHINX-NEXT: xori a2, a2, 1 +; RV32IZHINX-NEXT: addi a2, a2, -1 +; RV32IZHINX-NEXT: and a0, a2, a0 +; RV32IZHINX-NEXT: flt.s a3, a3, s0 +; RV32IZHINX-NEXT: neg a3, a3 +; RV32IZHINX-NEXT: or a0, a3, a0 +; RV32IZHINX-NEXT: and a1, a2, a1 +; RV32IZHINX-NEXT: or a1, a3, a1 ; RV32IZHINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZHINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: addi sp, sp, 16 ; RV32IZHINX-NEXT: ret ; @@ -2758,26 +2776,25 @@ define i64 @test_round_ui64(half %x) nounwind { ; RV32IZFHMIN-NEXT: .LBB15_2: ; RV32IZFHMIN-NEXT: addi sp, sp, -16 ; RV32IZFHMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFHMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFHMIN-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFHMIN-NEXT: fsw fs0, 8(sp) # 4-byte Folded Spill ; RV32IZFHMIN-NEXT: fcvt.h.s fa5, fa5 ; RV32IZFHMIN-NEXT: fcvt.s.h fs0, fa5 -; RV32IZFHMIN-NEXT: fmv.w.x fa5, zero -; RV32IZFHMIN-NEXT: fle.s a0, fa5, fs0 -; RV32IZFHMIN-NEXT: neg s0, a0 ; RV32IZFHMIN-NEXT: fmv.s fa0, fs0 ; RV32IZFHMIN-NEXT: call __fixunssfdi -; RV32IZFHMIN-NEXT: lui a2, %hi(.LCPI15_0) -; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI15_0)(a2) -; RV32IZFHMIN-NEXT: and a0, s0, a0 -; RV32IZFHMIN-NEXT: flt.s a2, fa5, fs0 -; RV32IZFHMIN-NEXT: neg a2, a2 -; RV32IZFHMIN-NEXT: or a0, a2, a0 -; RV32IZFHMIN-NEXT: and a1, s0, a1 -; RV32IZFHMIN-NEXT: or a1, a2, a1 +; RV32IZFHMIN-NEXT: fmv.w.x fa5, zero +; RV32IZFHMIN-NEXT: fle.s a2, fa5, fs0 +; RV32IZFHMIN-NEXT: lui a3, %hi(.LCPI15_0) +; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI15_0)(a3) +; RV32IZFHMIN-NEXT: xori a2, a2, 1 +; RV32IZFHMIN-NEXT: addi a2, a2, -1 +; RV32IZFHMIN-NEXT: and a0, a2, a0 +; RV32IZFHMIN-NEXT: flt.s a3, fa5, fs0 +; RV32IZFHMIN-NEXT: neg a3, a3 +; RV32IZFHMIN-NEXT: or a0, a3, a0 +; RV32IZFHMIN-NEXT: and a1, a2, a1 +; RV32IZFHMIN-NEXT: or a1, a3, a1 ; RV32IZFHMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFHMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFHMIN-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFHMIN-NEXT: flw fs0, 8(sp) # 4-byte Folded Reload ; RV32IZFHMIN-NEXT: addi sp, sp, 16 ; RV32IZFHMIN-NEXT: ret ; @@ -2818,24 +2835,23 @@ define i64 @test_round_ui64(half %x) nounwind { ; RV32IZHINXMIN-NEXT: addi sp, sp, -16 ; RV32IZHINXMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZHINXMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZHINXMIN-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZHINXMIN-NEXT: fcvt.h.s a0, a0 ; RV32IZHINXMIN-NEXT: fcvt.s.h s0, a0 -; RV32IZHINXMIN-NEXT: fle.s a0, zero, s0 -; RV32IZHINXMIN-NEXT: neg s1, a0 ; RV32IZHINXMIN-NEXT: mv a0, s0 ; RV32IZHINXMIN-NEXT: call __fixunssfdi -; RV32IZHINXMIN-NEXT: lui a2, %hi(.LCPI15_0) -; RV32IZHINXMIN-NEXT: lw a2, %lo(.LCPI15_0)(a2) -; RV32IZHINXMIN-NEXT: and a0, s1, a0 -; RV32IZHINXMIN-NEXT: flt.s a2, a2, s0 -; RV32IZHINXMIN-NEXT: neg a2, a2 -; RV32IZHINXMIN-NEXT: or a0, a2, a0 -; RV32IZHINXMIN-NEXT: and a1, s1, a1 -; RV32IZHINXMIN-NEXT: or a1, a2, a1 +; RV32IZHINXMIN-NEXT: fle.s a2, zero, s0 +; RV32IZHINXMIN-NEXT: lui a3, %hi(.LCPI15_0) +; RV32IZHINXMIN-NEXT: lw a3, %lo(.LCPI15_0)(a3) +; RV32IZHINXMIN-NEXT: xori a2, a2, 1 +; RV32IZHINXMIN-NEXT: addi a2, a2, -1 +; RV32IZHINXMIN-NEXT: and a0, a2, a0 +; RV32IZHINXMIN-NEXT: flt.s a3, a3, s0 +; RV32IZHINXMIN-NEXT: neg a3, a3 +; RV32IZHINXMIN-NEXT: or a0, a3, a0 +; RV32IZHINXMIN-NEXT: and a1, a2, a1 +; RV32IZHINXMIN-NEXT: or a1, a3, a1 ; RV32IZHINXMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZHINXMIN-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: addi sp, sp, 16 ; RV32IZHINXMIN-NEXT: ret ; @@ -2956,38 +2972,41 @@ define i64 @test_roundeven_si64(half %x) nounwind { ; RV32IZFH-NEXT: addi sp, sp, -16 ; RV32IZFH-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFH-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFH-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: fsw fs0, 0(sp) # 4-byte Folded Spill ; RV32IZFH-NEXT: fcvt.s.h fs0, fa0 ; RV32IZFH-NEXT: lui a0, 913408 ; RV32IZFH-NEXT: fmv.w.x fa5, a0 ; RV32IZFH-NEXT: fle.s s0, fa5, fs0 +; RV32IZFH-NEXT: neg s1, s0 ; RV32IZFH-NEXT: fmv.s fa0, fs0 ; RV32IZFH-NEXT: call __fixsfdi +; RV32IZFH-NEXT: lui a2, %hi(.LCPI17_1) +; RV32IZFH-NEXT: flw fa5, %lo(.LCPI17_1)(a2) +; RV32IZFH-NEXT: and a0, s1, a0 +; RV32IZFH-NEXT: flt.s a3, fa5, fs0 +; RV32IZFH-NEXT: neg a2, a3 +; RV32IZFH-NEXT: or a0, a2, a0 +; RV32IZFH-NEXT: feq.s a2, fs0, fs0 +; RV32IZFH-NEXT: neg a2, a2 +; RV32IZFH-NEXT: lui a5, 524288 +; RV32IZFH-NEXT: li a6, 1 ; RV32IZFH-NEXT: lui a4, 524288 -; RV32IZFH-NEXT: lui a2, 524288 -; RV32IZFH-NEXT: beqz s0, .LBB17_4 +; RV32IZFH-NEXT: bne s0, a6, .LBB17_4 ; RV32IZFH-NEXT: # %bb.3: -; RV32IZFH-NEXT: mv a2, a1 +; RV32IZFH-NEXT: mv a4, a1 ; RV32IZFH-NEXT: .LBB17_4: -; RV32IZFH-NEXT: lui a1, %hi(.LCPI17_1) -; RV32IZFH-NEXT: flw fa5, %lo(.LCPI17_1)(a1) -; RV32IZFH-NEXT: flt.s a3, fa5, fs0 -; RV32IZFH-NEXT: beqz a3, .LBB17_6 -; RV32IZFH-NEXT: # %bb.5: -; RV32IZFH-NEXT: addi a2, a4, -1 -; RV32IZFH-NEXT: .LBB17_6: -; RV32IZFH-NEXT: feq.s a1, fs0, fs0 -; RV32IZFH-NEXT: neg a4, a1 -; RV32IZFH-NEXT: and a1, a4, a2 -; RV32IZFH-NEXT: neg a2, s0 ; RV32IZFH-NEXT: and a0, a2, a0 -; RV32IZFH-NEXT: neg a2, a3 -; RV32IZFH-NEXT: or a0, a2, a0 -; RV32IZFH-NEXT: and a0, a4, a0 ; RV32IZFH-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFH-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFH-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: flw fs0, 0(sp) # 4-byte Folded Reload ; RV32IZFH-NEXT: addi sp, sp, 16 +; RV32IZFH-NEXT: beqz a3, .LBB17_6 +; RV32IZFH-NEXT: # %bb.5: +; RV32IZFH-NEXT: addi a4, a5, -1 +; RV32IZFH-NEXT: .LBB17_6: +; RV32IZFH-NEXT: and a1, a2, a4 ; RV32IZFH-NEXT: ret ; ; RV64IZFH-LABEL: test_roundeven_si64: @@ -3025,16 +3044,17 @@ define i64 @test_roundeven_si64(half %x) nounwind { ; RV32IZHINX-NEXT: lui a2, %hi(.LCPI17_1) ; RV32IZHINX-NEXT: lw a2, %lo(.LCPI17_1)(a2) ; RV32IZHINX-NEXT: and a0, s2, a0 -; RV32IZHINX-NEXT: flt.s a4, a2, s0 -; RV32IZHINX-NEXT: neg a2, a4 +; RV32IZHINX-NEXT: flt.s a3, a2, s0 +; RV32IZHINX-NEXT: neg a2, a3 ; RV32IZHINX-NEXT: or a0, a2, a0 ; RV32IZHINX-NEXT: feq.s a2, s0, s0 ; RV32IZHINX-NEXT: neg a2, a2 ; RV32IZHINX-NEXT: lui a5, 524288 -; RV32IZHINX-NEXT: lui a3, 524288 -; RV32IZHINX-NEXT: beqz s1, .LBB17_4 +; RV32IZHINX-NEXT: li a6, 1 +; RV32IZHINX-NEXT: lui a4, 524288 +; RV32IZHINX-NEXT: bne s1, a6, .LBB17_4 ; RV32IZHINX-NEXT: # %bb.3: -; RV32IZHINX-NEXT: mv a3, a1 +; RV32IZHINX-NEXT: mv a4, a1 ; RV32IZHINX-NEXT: .LBB17_4: ; RV32IZHINX-NEXT: and a0, a2, a0 ; RV32IZHINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload @@ -3042,11 +3062,11 @@ define i64 @test_roundeven_si64(half %x) nounwind { ; RV32IZHINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: lw s2, 0(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: addi sp, sp, 16 -; RV32IZHINX-NEXT: beqz a4, .LBB17_6 +; RV32IZHINX-NEXT: beqz a3, .LBB17_6 ; RV32IZHINX-NEXT: # %bb.5: -; RV32IZHINX-NEXT: addi a3, a5, -1 +; RV32IZHINX-NEXT: addi a4, a5, -1 ; RV32IZHINX-NEXT: .LBB17_6: -; RV32IZHINX-NEXT: and a1, a2, a3 +; RV32IZHINX-NEXT: and a1, a2, a4 ; RV32IZHINX-NEXT: ret ; ; RV64IZHINX-LABEL: test_roundeven_si64: @@ -3084,39 +3104,42 @@ define i64 @test_roundeven_si64(half %x) nounwind { ; RV32IZFHMIN-NEXT: addi sp, sp, -16 ; RV32IZFHMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFHMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFHMIN-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFHMIN-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32IZFHMIN-NEXT: fsw fs0, 0(sp) # 4-byte Folded Spill ; RV32IZFHMIN-NEXT: fcvt.h.s fa5, fa5 ; RV32IZFHMIN-NEXT: fcvt.s.h fs0, fa5 ; RV32IZFHMIN-NEXT: lui a0, 913408 ; RV32IZFHMIN-NEXT: fmv.w.x fa5, a0 ; RV32IZFHMIN-NEXT: fle.s s0, fa5, fs0 +; RV32IZFHMIN-NEXT: neg s1, s0 ; RV32IZFHMIN-NEXT: fmv.s fa0, fs0 ; RV32IZFHMIN-NEXT: call __fixsfdi +; RV32IZFHMIN-NEXT: lui a2, %hi(.LCPI17_0) +; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI17_0)(a2) +; RV32IZFHMIN-NEXT: and a0, s1, a0 +; RV32IZFHMIN-NEXT: flt.s a3, fa5, fs0 +; RV32IZFHMIN-NEXT: neg a2, a3 +; RV32IZFHMIN-NEXT: or a0, a2, a0 +; RV32IZFHMIN-NEXT: feq.s a2, fs0, fs0 +; RV32IZFHMIN-NEXT: neg a2, a2 +; RV32IZFHMIN-NEXT: lui a5, 524288 +; RV32IZFHMIN-NEXT: li a6, 1 ; RV32IZFHMIN-NEXT: lui a4, 524288 -; RV32IZFHMIN-NEXT: lui a2, 524288 -; RV32IZFHMIN-NEXT: beqz s0, .LBB17_4 +; RV32IZFHMIN-NEXT: bne s0, a6, .LBB17_4 ; RV32IZFHMIN-NEXT: # %bb.3: -; RV32IZFHMIN-NEXT: mv a2, a1 +; RV32IZFHMIN-NEXT: mv a4, a1 ; RV32IZFHMIN-NEXT: .LBB17_4: -; RV32IZFHMIN-NEXT: lui a1, %hi(.LCPI17_0) -; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI17_0)(a1) -; RV32IZFHMIN-NEXT: flt.s a3, fa5, fs0 -; RV32IZFHMIN-NEXT: beqz a3, .LBB17_6 -; RV32IZFHMIN-NEXT: # %bb.5: -; RV32IZFHMIN-NEXT: addi a2, a4, -1 -; RV32IZFHMIN-NEXT: .LBB17_6: -; RV32IZFHMIN-NEXT: feq.s a1, fs0, fs0 -; RV32IZFHMIN-NEXT: neg a4, a1 -; RV32IZFHMIN-NEXT: and a1, a4, a2 -; RV32IZFHMIN-NEXT: neg a2, s0 ; RV32IZFHMIN-NEXT: and a0, a2, a0 -; RV32IZFHMIN-NEXT: neg a2, a3 -; RV32IZFHMIN-NEXT: or a0, a2, a0 -; RV32IZFHMIN-NEXT: and a0, a4, a0 ; RV32IZFHMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFHMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFHMIN-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFHMIN-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IZFHMIN-NEXT: flw fs0, 0(sp) # 4-byte Folded Reload ; RV32IZFHMIN-NEXT: addi sp, sp, 16 +; RV32IZFHMIN-NEXT: beqz a3, .LBB17_6 +; RV32IZFHMIN-NEXT: # %bb.5: +; RV32IZFHMIN-NEXT: addi a4, a5, -1 +; RV32IZFHMIN-NEXT: .LBB17_6: +; RV32IZFHMIN-NEXT: and a1, a2, a4 ; RV32IZFHMIN-NEXT: ret ; ; RV64IZFHMIN-LABEL: test_roundeven_si64: @@ -3168,16 +3191,17 @@ define i64 @test_roundeven_si64(half %x) nounwind { ; RV32IZHINXMIN-NEXT: lui a2, %hi(.LCPI17_0) ; RV32IZHINXMIN-NEXT: lw a2, %lo(.LCPI17_0)(a2) ; RV32IZHINXMIN-NEXT: and a0, s2, a0 -; RV32IZHINXMIN-NEXT: flt.s a4, a2, s0 -; RV32IZHINXMIN-NEXT: neg a2, a4 +; RV32IZHINXMIN-NEXT: flt.s a3, a2, s0 +; RV32IZHINXMIN-NEXT: neg a2, a3 ; RV32IZHINXMIN-NEXT: or a0, a2, a0 ; RV32IZHINXMIN-NEXT: feq.s a2, s0, s0 ; RV32IZHINXMIN-NEXT: neg a2, a2 ; RV32IZHINXMIN-NEXT: lui a5, 524288 -; RV32IZHINXMIN-NEXT: lui a3, 524288 -; RV32IZHINXMIN-NEXT: beqz s1, .LBB17_4 +; RV32IZHINXMIN-NEXT: li a6, 1 +; RV32IZHINXMIN-NEXT: lui a4, 524288 +; RV32IZHINXMIN-NEXT: bne s1, a6, .LBB17_4 ; RV32IZHINXMIN-NEXT: # %bb.3: -; RV32IZHINXMIN-NEXT: mv a3, a1 +; RV32IZHINXMIN-NEXT: mv a4, a1 ; RV32IZHINXMIN-NEXT: .LBB17_4: ; RV32IZHINXMIN-NEXT: and a0, a2, a0 ; RV32IZHINXMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload @@ -3185,11 +3209,11 @@ define i64 @test_roundeven_si64(half %x) nounwind { ; RV32IZHINXMIN-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: lw s2, 0(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: addi sp, sp, 16 -; RV32IZHINXMIN-NEXT: beqz a4, .LBB17_6 +; RV32IZHINXMIN-NEXT: beqz a3, .LBB17_6 ; RV32IZHINXMIN-NEXT: # %bb.5: -; RV32IZHINXMIN-NEXT: addi a3, a5, -1 +; RV32IZHINXMIN-NEXT: addi a4, a5, -1 ; RV32IZHINXMIN-NEXT: .LBB17_6: -; RV32IZHINXMIN-NEXT: and a1, a2, a3 +; RV32IZHINXMIN-NEXT: and a1, a2, a4 ; RV32IZHINXMIN-NEXT: ret ; ; RV64IZHINXMIN-LABEL: test_roundeven_si64: @@ -3261,7 +3285,7 @@ define signext i32 @test_roundeven_ui32(half %x) { ; RV64IZHINX-NEXT: fcvt.wu.h a1, a0, rtz ; RV64IZHINX-NEXT: feq.h a0, a0, a0 ; RV64IZHINX-NEXT: seqz a0, a0 -; RV64IZHINX-NEXT: addi a0, a0, -1 +; RV64IZHINX-NEXT: addiw a0, a0, -1 ; RV64IZHINX-NEXT: and a0, a1, a0 ; RV64IZHINX-NEXT: ret ; @@ -3305,7 +3329,7 @@ define signext i32 @test_roundeven_ui32(half %x) { ; RV64IZFHMIN-NEXT: fcvt.wu.s a0, fa5, rtz ; RV64IZFHMIN-NEXT: feq.s a1, fa5, fa5 ; RV64IZFHMIN-NEXT: seqz a1, a1 -; RV64IZFHMIN-NEXT: addi a1, a1, -1 +; RV64IZFHMIN-NEXT: addiw a1, a1, -1 ; RV64IZFHMIN-NEXT: and a0, a0, a1 ; RV64IZFHMIN-NEXT: ret ; @@ -3347,7 +3371,7 @@ define signext i32 @test_roundeven_ui32(half %x) { ; RV64IZHINXMIN-NEXT: fcvt.wu.s a1, a0, rtz ; RV64IZHINXMIN-NEXT: feq.s a0, a0, a0 ; RV64IZHINXMIN-NEXT: seqz a0, a0 -; RV64IZHINXMIN-NEXT: addi a0, a0, -1 +; RV64IZHINXMIN-NEXT: addiw a0, a0, -1 ; RV64IZHINXMIN-NEXT: and a0, a1, a0 ; RV64IZHINXMIN-NEXT: ret %a = call half @llvm.roundeven.f16(half %x) @@ -3370,25 +3394,24 @@ define i64 @test_roundeven_ui64(half %x) nounwind { ; RV32IZFH-NEXT: .LBB19_2: ; RV32IZFH-NEXT: addi sp, sp, -16 ; RV32IZFH-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFH-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFH-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: fsw fs0, 8(sp) # 4-byte Folded Spill ; RV32IZFH-NEXT: fcvt.s.h fs0, fa0 -; RV32IZFH-NEXT: fmv.w.x fa5, zero -; RV32IZFH-NEXT: fle.s a0, fa5, fs0 -; RV32IZFH-NEXT: neg s0, a0 ; RV32IZFH-NEXT: fmv.s fa0, fs0 ; RV32IZFH-NEXT: call __fixunssfdi -; RV32IZFH-NEXT: lui a2, %hi(.LCPI19_1) -; RV32IZFH-NEXT: flw fa5, %lo(.LCPI19_1)(a2) -; RV32IZFH-NEXT: and a0, s0, a0 -; RV32IZFH-NEXT: flt.s a2, fa5, fs0 -; RV32IZFH-NEXT: neg a2, a2 -; RV32IZFH-NEXT: or a0, a2, a0 -; RV32IZFH-NEXT: and a1, s0, a1 -; RV32IZFH-NEXT: or a1, a2, a1 +; RV32IZFH-NEXT: fmv.w.x fa5, zero +; RV32IZFH-NEXT: fle.s a2, fa5, fs0 +; RV32IZFH-NEXT: lui a3, %hi(.LCPI19_1) +; RV32IZFH-NEXT: flw fa5, %lo(.LCPI19_1)(a3) +; RV32IZFH-NEXT: xori a2, a2, 1 +; RV32IZFH-NEXT: addi a2, a2, -1 +; RV32IZFH-NEXT: and a0, a2, a0 +; RV32IZFH-NEXT: flt.s a3, fa5, fs0 +; RV32IZFH-NEXT: neg a3, a3 +; RV32IZFH-NEXT: or a0, a3, a0 +; RV32IZFH-NEXT: and a1, a2, a1 +; RV32IZFH-NEXT: or a1, a3, a1 ; RV32IZFH-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFH-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFH-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: flw fs0, 8(sp) # 4-byte Folded Reload ; RV32IZFH-NEXT: addi sp, sp, 16 ; RV32IZFH-NEXT: ret ; @@ -3416,23 +3439,22 @@ define i64 @test_roundeven_ui64(half %x) nounwind { ; RV32IZHINX-NEXT: addi sp, sp, -16 ; RV32IZHINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZHINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZHINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZHINX-NEXT: fcvt.s.h s0, a0 -; RV32IZHINX-NEXT: fle.s a0, zero, s0 -; RV32IZHINX-NEXT: neg s1, a0 ; RV32IZHINX-NEXT: mv a0, s0 ; RV32IZHINX-NEXT: call __fixunssfdi -; RV32IZHINX-NEXT: lui a2, %hi(.LCPI19_1) -; RV32IZHINX-NEXT: lw a2, %lo(.LCPI19_1)(a2) -; RV32IZHINX-NEXT: and a0, s1, a0 -; RV32IZHINX-NEXT: flt.s a2, a2, s0 -; RV32IZHINX-NEXT: neg a2, a2 -; RV32IZHINX-NEXT: or a0, a2, a0 -; RV32IZHINX-NEXT: and a1, s1, a1 -; RV32IZHINX-NEXT: or a1, a2, a1 +; RV32IZHINX-NEXT: fle.s a2, zero, s0 +; RV32IZHINX-NEXT: lui a3, %hi(.LCPI19_1) +; RV32IZHINX-NEXT: lw a3, %lo(.LCPI19_1)(a3) +; RV32IZHINX-NEXT: xori a2, a2, 1 +; RV32IZHINX-NEXT: addi a2, a2, -1 +; RV32IZHINX-NEXT: and a0, a2, a0 +; RV32IZHINX-NEXT: flt.s a3, a3, s0 +; RV32IZHINX-NEXT: neg a3, a3 +; RV32IZHINX-NEXT: or a0, a3, a0 +; RV32IZHINX-NEXT: and a1, a2, a1 +; RV32IZHINX-NEXT: or a1, a3, a1 ; RV32IZHINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZHINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: addi sp, sp, 16 ; RV32IZHINX-NEXT: ret ; @@ -3470,26 +3492,25 @@ define i64 @test_roundeven_ui64(half %x) nounwind { ; RV32IZFHMIN-NEXT: .LBB19_2: ; RV32IZFHMIN-NEXT: addi sp, sp, -16 ; RV32IZFHMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFHMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFHMIN-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFHMIN-NEXT: fsw fs0, 8(sp) # 4-byte Folded Spill ; RV32IZFHMIN-NEXT: fcvt.h.s fa5, fa5 ; RV32IZFHMIN-NEXT: fcvt.s.h fs0, fa5 -; RV32IZFHMIN-NEXT: fmv.w.x fa5, zero -; RV32IZFHMIN-NEXT: fle.s a0, fa5, fs0 -; RV32IZFHMIN-NEXT: neg s0, a0 ; RV32IZFHMIN-NEXT: fmv.s fa0, fs0 ; RV32IZFHMIN-NEXT: call __fixunssfdi -; RV32IZFHMIN-NEXT: lui a2, %hi(.LCPI19_0) -; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI19_0)(a2) -; RV32IZFHMIN-NEXT: and a0, s0, a0 -; RV32IZFHMIN-NEXT: flt.s a2, fa5, fs0 -; RV32IZFHMIN-NEXT: neg a2, a2 -; RV32IZFHMIN-NEXT: or a0, a2, a0 -; RV32IZFHMIN-NEXT: and a1, s0, a1 -; RV32IZFHMIN-NEXT: or a1, a2, a1 +; RV32IZFHMIN-NEXT: fmv.w.x fa5, zero +; RV32IZFHMIN-NEXT: fle.s a2, fa5, fs0 +; RV32IZFHMIN-NEXT: lui a3, %hi(.LCPI19_0) +; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI19_0)(a3) +; RV32IZFHMIN-NEXT: xori a2, a2, 1 +; RV32IZFHMIN-NEXT: addi a2, a2, -1 +; RV32IZFHMIN-NEXT: and a0, a2, a0 +; RV32IZFHMIN-NEXT: flt.s a3, fa5, fs0 +; RV32IZFHMIN-NEXT: neg a3, a3 +; RV32IZFHMIN-NEXT: or a0, a3, a0 +; RV32IZFHMIN-NEXT: and a1, a2, a1 +; RV32IZFHMIN-NEXT: or a1, a3, a1 ; RV32IZFHMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFHMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFHMIN-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFHMIN-NEXT: flw fs0, 8(sp) # 4-byte Folded Reload ; RV32IZFHMIN-NEXT: addi sp, sp, 16 ; RV32IZFHMIN-NEXT: ret ; @@ -3530,24 +3551,23 @@ define i64 @test_roundeven_ui64(half %x) nounwind { ; RV32IZHINXMIN-NEXT: addi sp, sp, -16 ; RV32IZHINXMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZHINXMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZHINXMIN-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZHINXMIN-NEXT: fcvt.h.s a0, a0 ; RV32IZHINXMIN-NEXT: fcvt.s.h s0, a0 -; RV32IZHINXMIN-NEXT: fle.s a0, zero, s0 -; RV32IZHINXMIN-NEXT: neg s1, a0 ; RV32IZHINXMIN-NEXT: mv a0, s0 ; RV32IZHINXMIN-NEXT: call __fixunssfdi -; RV32IZHINXMIN-NEXT: lui a2, %hi(.LCPI19_0) -; RV32IZHINXMIN-NEXT: lw a2, %lo(.LCPI19_0)(a2) -; RV32IZHINXMIN-NEXT: and a0, s1, a0 -; RV32IZHINXMIN-NEXT: flt.s a2, a2, s0 -; RV32IZHINXMIN-NEXT: neg a2, a2 -; RV32IZHINXMIN-NEXT: or a0, a2, a0 -; RV32IZHINXMIN-NEXT: and a1, s1, a1 -; RV32IZHINXMIN-NEXT: or a1, a2, a1 +; RV32IZHINXMIN-NEXT: fle.s a2, zero, s0 +; RV32IZHINXMIN-NEXT: lui a3, %hi(.LCPI19_0) +; RV32IZHINXMIN-NEXT: lw a3, %lo(.LCPI19_0)(a3) +; RV32IZHINXMIN-NEXT: xori a2, a2, 1 +; RV32IZHINXMIN-NEXT: addi a2, a2, -1 +; RV32IZHINXMIN-NEXT: and a0, a2, a0 +; RV32IZHINXMIN-NEXT: flt.s a3, a3, s0 +; RV32IZHINXMIN-NEXT: neg a3, a3 +; RV32IZHINXMIN-NEXT: or a0, a3, a0 +; RV32IZHINXMIN-NEXT: and a1, a2, a1 +; RV32IZHINXMIN-NEXT: or a1, a3, a1 ; RV32IZHINXMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZHINXMIN-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: addi sp, sp, 16 ; RV32IZHINXMIN-NEXT: ret ; @@ -3668,38 +3688,41 @@ define i64 @test_rint_si64(half %x) nounwind { ; RV32IZFH-NEXT: addi sp, sp, -16 ; RV32IZFH-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFH-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFH-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: fsw fs0, 0(sp) # 4-byte Folded Spill ; RV32IZFH-NEXT: fcvt.s.h fs0, fa0 ; RV32IZFH-NEXT: lui a0, 913408 ; RV32IZFH-NEXT: fmv.w.x fa5, a0 ; RV32IZFH-NEXT: fle.s s0, fa5, fs0 +; RV32IZFH-NEXT: neg s1, s0 ; RV32IZFH-NEXT: fmv.s fa0, fs0 ; RV32IZFH-NEXT: call __fixsfdi +; RV32IZFH-NEXT: lui a2, %hi(.LCPI21_1) +; RV32IZFH-NEXT: flw fa5, %lo(.LCPI21_1)(a2) +; RV32IZFH-NEXT: and a0, s1, a0 +; RV32IZFH-NEXT: flt.s a3, fa5, fs0 +; RV32IZFH-NEXT: neg a2, a3 +; RV32IZFH-NEXT: or a0, a2, a0 +; RV32IZFH-NEXT: feq.s a2, fs0, fs0 +; RV32IZFH-NEXT: neg a2, a2 +; RV32IZFH-NEXT: lui a5, 524288 +; RV32IZFH-NEXT: li a6, 1 ; RV32IZFH-NEXT: lui a4, 524288 -; RV32IZFH-NEXT: lui a2, 524288 -; RV32IZFH-NEXT: beqz s0, .LBB21_4 +; RV32IZFH-NEXT: bne s0, a6, .LBB21_4 ; RV32IZFH-NEXT: # %bb.3: -; RV32IZFH-NEXT: mv a2, a1 +; RV32IZFH-NEXT: mv a4, a1 ; RV32IZFH-NEXT: .LBB21_4: -; RV32IZFH-NEXT: lui a1, %hi(.LCPI21_1) -; RV32IZFH-NEXT: flw fa5, %lo(.LCPI21_1)(a1) -; RV32IZFH-NEXT: flt.s a3, fa5, fs0 -; RV32IZFH-NEXT: beqz a3, .LBB21_6 -; RV32IZFH-NEXT: # %bb.5: -; RV32IZFH-NEXT: addi a2, a4, -1 -; RV32IZFH-NEXT: .LBB21_6: -; RV32IZFH-NEXT: feq.s a1, fs0, fs0 -; RV32IZFH-NEXT: neg a4, a1 -; RV32IZFH-NEXT: and a1, a4, a2 -; RV32IZFH-NEXT: neg a2, s0 ; RV32IZFH-NEXT: and a0, a2, a0 -; RV32IZFH-NEXT: neg a2, a3 -; RV32IZFH-NEXT: or a0, a2, a0 -; RV32IZFH-NEXT: and a0, a4, a0 ; RV32IZFH-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFH-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFH-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: flw fs0, 0(sp) # 4-byte Folded Reload ; RV32IZFH-NEXT: addi sp, sp, 16 +; RV32IZFH-NEXT: beqz a3, .LBB21_6 +; RV32IZFH-NEXT: # %bb.5: +; RV32IZFH-NEXT: addi a4, a5, -1 +; RV32IZFH-NEXT: .LBB21_6: +; RV32IZFH-NEXT: and a1, a2, a4 ; RV32IZFH-NEXT: ret ; ; RV64IZFH-LABEL: test_rint_si64: @@ -3737,16 +3760,17 @@ define i64 @test_rint_si64(half %x) nounwind { ; RV32IZHINX-NEXT: lui a2, %hi(.LCPI21_1) ; RV32IZHINX-NEXT: lw a2, %lo(.LCPI21_1)(a2) ; RV32IZHINX-NEXT: and a0, s2, a0 -; RV32IZHINX-NEXT: flt.s a4, a2, s0 -; RV32IZHINX-NEXT: neg a2, a4 +; RV32IZHINX-NEXT: flt.s a3, a2, s0 +; RV32IZHINX-NEXT: neg a2, a3 ; RV32IZHINX-NEXT: or a0, a2, a0 ; RV32IZHINX-NEXT: feq.s a2, s0, s0 ; RV32IZHINX-NEXT: neg a2, a2 ; RV32IZHINX-NEXT: lui a5, 524288 -; RV32IZHINX-NEXT: lui a3, 524288 -; RV32IZHINX-NEXT: beqz s1, .LBB21_4 +; RV32IZHINX-NEXT: li a6, 1 +; RV32IZHINX-NEXT: lui a4, 524288 +; RV32IZHINX-NEXT: bne s1, a6, .LBB21_4 ; RV32IZHINX-NEXT: # %bb.3: -; RV32IZHINX-NEXT: mv a3, a1 +; RV32IZHINX-NEXT: mv a4, a1 ; RV32IZHINX-NEXT: .LBB21_4: ; RV32IZHINX-NEXT: and a0, a2, a0 ; RV32IZHINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload @@ -3754,11 +3778,11 @@ define i64 @test_rint_si64(half %x) nounwind { ; RV32IZHINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: lw s2, 0(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: addi sp, sp, 16 -; RV32IZHINX-NEXT: beqz a4, .LBB21_6 +; RV32IZHINX-NEXT: beqz a3, .LBB21_6 ; RV32IZHINX-NEXT: # %bb.5: -; RV32IZHINX-NEXT: addi a3, a5, -1 +; RV32IZHINX-NEXT: addi a4, a5, -1 ; RV32IZHINX-NEXT: .LBB21_6: -; RV32IZHINX-NEXT: and a1, a2, a3 +; RV32IZHINX-NEXT: and a1, a2, a4 ; RV32IZHINX-NEXT: ret ; ; RV64IZHINX-LABEL: test_rint_si64: @@ -3796,39 +3820,42 @@ define i64 @test_rint_si64(half %x) nounwind { ; RV32IZFHMIN-NEXT: addi sp, sp, -16 ; RV32IZFHMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFHMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFHMIN-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFHMIN-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32IZFHMIN-NEXT: fsw fs0, 0(sp) # 4-byte Folded Spill ; RV32IZFHMIN-NEXT: fcvt.h.s fa5, fa5 ; RV32IZFHMIN-NEXT: fcvt.s.h fs0, fa5 ; RV32IZFHMIN-NEXT: lui a0, 913408 ; RV32IZFHMIN-NEXT: fmv.w.x fa5, a0 ; RV32IZFHMIN-NEXT: fle.s s0, fa5, fs0 +; RV32IZFHMIN-NEXT: neg s1, s0 ; RV32IZFHMIN-NEXT: fmv.s fa0, fs0 ; RV32IZFHMIN-NEXT: call __fixsfdi +; RV32IZFHMIN-NEXT: lui a2, %hi(.LCPI21_0) +; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI21_0)(a2) +; RV32IZFHMIN-NEXT: and a0, s1, a0 +; RV32IZFHMIN-NEXT: flt.s a3, fa5, fs0 +; RV32IZFHMIN-NEXT: neg a2, a3 +; RV32IZFHMIN-NEXT: or a0, a2, a0 +; RV32IZFHMIN-NEXT: feq.s a2, fs0, fs0 +; RV32IZFHMIN-NEXT: neg a2, a2 +; RV32IZFHMIN-NEXT: lui a5, 524288 +; RV32IZFHMIN-NEXT: li a6, 1 ; RV32IZFHMIN-NEXT: lui a4, 524288 -; RV32IZFHMIN-NEXT: lui a2, 524288 -; RV32IZFHMIN-NEXT: beqz s0, .LBB21_4 +; RV32IZFHMIN-NEXT: bne s0, a6, .LBB21_4 ; RV32IZFHMIN-NEXT: # %bb.3: -; RV32IZFHMIN-NEXT: mv a2, a1 +; RV32IZFHMIN-NEXT: mv a4, a1 ; RV32IZFHMIN-NEXT: .LBB21_4: -; RV32IZFHMIN-NEXT: lui a1, %hi(.LCPI21_0) -; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI21_0)(a1) -; RV32IZFHMIN-NEXT: flt.s a3, fa5, fs0 -; RV32IZFHMIN-NEXT: beqz a3, .LBB21_6 -; RV32IZFHMIN-NEXT: # %bb.5: -; RV32IZFHMIN-NEXT: addi a2, a4, -1 -; RV32IZFHMIN-NEXT: .LBB21_6: -; RV32IZFHMIN-NEXT: feq.s a1, fs0, fs0 -; RV32IZFHMIN-NEXT: neg a4, a1 -; RV32IZFHMIN-NEXT: and a1, a4, a2 -; RV32IZFHMIN-NEXT: neg a2, s0 ; RV32IZFHMIN-NEXT: and a0, a2, a0 -; RV32IZFHMIN-NEXT: neg a2, a3 -; RV32IZFHMIN-NEXT: or a0, a2, a0 -; RV32IZFHMIN-NEXT: and a0, a4, a0 ; RV32IZFHMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZFHMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFHMIN-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFHMIN-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IZFHMIN-NEXT: flw fs0, 0(sp) # 4-byte Folded Reload ; RV32IZFHMIN-NEXT: addi sp, sp, 16 +; RV32IZFHMIN-NEXT: beqz a3, .LBB21_6 +; RV32IZFHMIN-NEXT: # %bb.5: +; RV32IZFHMIN-NEXT: addi a4, a5, -1 +; RV32IZFHMIN-NEXT: .LBB21_6: +; RV32IZFHMIN-NEXT: and a1, a2, a4 ; RV32IZFHMIN-NEXT: ret ; ; RV64IZFHMIN-LABEL: test_rint_si64: @@ -3880,16 +3907,17 @@ define i64 @test_rint_si64(half %x) nounwind { ; RV32IZHINXMIN-NEXT: lui a2, %hi(.LCPI21_0) ; RV32IZHINXMIN-NEXT: lw a2, %lo(.LCPI21_0)(a2) ; RV32IZHINXMIN-NEXT: and a0, s2, a0 -; RV32IZHINXMIN-NEXT: flt.s a4, a2, s0 -; RV32IZHINXMIN-NEXT: neg a2, a4 +; RV32IZHINXMIN-NEXT: flt.s a3, a2, s0 +; RV32IZHINXMIN-NEXT: neg a2, a3 ; RV32IZHINXMIN-NEXT: or a0, a2, a0 ; RV32IZHINXMIN-NEXT: feq.s a2, s0, s0 ; RV32IZHINXMIN-NEXT: neg a2, a2 ; RV32IZHINXMIN-NEXT: lui a5, 524288 -; RV32IZHINXMIN-NEXT: lui a3, 524288 -; RV32IZHINXMIN-NEXT: beqz s1, .LBB21_4 +; RV32IZHINXMIN-NEXT: li a6, 1 +; RV32IZHINXMIN-NEXT: lui a4, 524288 +; RV32IZHINXMIN-NEXT: bne s1, a6, .LBB21_4 ; RV32IZHINXMIN-NEXT: # %bb.3: -; RV32IZHINXMIN-NEXT: mv a3, a1 +; RV32IZHINXMIN-NEXT: mv a4, a1 ; RV32IZHINXMIN-NEXT: .LBB21_4: ; RV32IZHINXMIN-NEXT: and a0, a2, a0 ; RV32IZHINXMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload @@ -3897,11 +3925,11 @@ define i64 @test_rint_si64(half %x) nounwind { ; RV32IZHINXMIN-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: lw s2, 0(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: addi sp, sp, 16 -; RV32IZHINXMIN-NEXT: beqz a4, .LBB21_6 +; RV32IZHINXMIN-NEXT: beqz a3, .LBB21_6 ; RV32IZHINXMIN-NEXT: # %bb.5: -; RV32IZHINXMIN-NEXT: addi a3, a5, -1 +; RV32IZHINXMIN-NEXT: addi a4, a5, -1 ; RV32IZHINXMIN-NEXT: .LBB21_6: -; RV32IZHINXMIN-NEXT: and a1, a2, a3 +; RV32IZHINXMIN-NEXT: and a1, a2, a4 ; RV32IZHINXMIN-NEXT: ret ; ; RV64IZHINXMIN-LABEL: test_rint_si64: @@ -3973,7 +4001,7 @@ define signext i32 @test_rint_ui32(half %x) { ; RV64IZHINX-NEXT: fcvt.wu.h a1, a0, rtz ; RV64IZHINX-NEXT: feq.h a0, a0, a0 ; RV64IZHINX-NEXT: seqz a0, a0 -; RV64IZHINX-NEXT: addi a0, a0, -1 +; RV64IZHINX-NEXT: addiw a0, a0, -1 ; RV64IZHINX-NEXT: and a0, a1, a0 ; RV64IZHINX-NEXT: ret ; @@ -4017,7 +4045,7 @@ define signext i32 @test_rint_ui32(half %x) { ; RV64IZFHMIN-NEXT: fcvt.wu.s a0, fa5, rtz ; RV64IZFHMIN-NEXT: feq.s a1, fa5, fa5 ; RV64IZFHMIN-NEXT: seqz a1, a1 -; RV64IZFHMIN-NEXT: addi a1, a1, -1 +; RV64IZFHMIN-NEXT: addiw a1, a1, -1 ; RV64IZFHMIN-NEXT: and a0, a0, a1 ; RV64IZFHMIN-NEXT: ret ; @@ -4059,7 +4087,7 @@ define signext i32 @test_rint_ui32(half %x) { ; RV64IZHINXMIN-NEXT: fcvt.wu.s a1, a0, rtz ; RV64IZHINXMIN-NEXT: feq.s a0, a0, a0 ; RV64IZHINXMIN-NEXT: seqz a0, a0 -; RV64IZHINXMIN-NEXT: addi a0, a0, -1 +; RV64IZHINXMIN-NEXT: addiw a0, a0, -1 ; RV64IZHINXMIN-NEXT: and a0, a1, a0 ; RV64IZHINXMIN-NEXT: ret %a = call half @llvm.rint.f16(half %x) @@ -4082,25 +4110,24 @@ define i64 @test_rint_ui64(half %x) nounwind { ; RV32IZFH-NEXT: .LBB23_2: ; RV32IZFH-NEXT: addi sp, sp, -16 ; RV32IZFH-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFH-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFH-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFH-NEXT: fsw fs0, 8(sp) # 4-byte Folded Spill ; RV32IZFH-NEXT: fcvt.s.h fs0, fa0 -; RV32IZFH-NEXT: fmv.w.x fa5, zero -; RV32IZFH-NEXT: fle.s a0, fa5, fs0 -; RV32IZFH-NEXT: neg s0, a0 ; RV32IZFH-NEXT: fmv.s fa0, fs0 ; RV32IZFH-NEXT: call __fixunssfdi -; RV32IZFH-NEXT: lui a2, %hi(.LCPI23_1) -; RV32IZFH-NEXT: flw fa5, %lo(.LCPI23_1)(a2) -; RV32IZFH-NEXT: and a0, s0, a0 -; RV32IZFH-NEXT: flt.s a2, fa5, fs0 -; RV32IZFH-NEXT: neg a2, a2 -; RV32IZFH-NEXT: or a0, a2, a0 -; RV32IZFH-NEXT: and a1, s0, a1 -; RV32IZFH-NEXT: or a1, a2, a1 +; RV32IZFH-NEXT: fmv.w.x fa5, zero +; RV32IZFH-NEXT: fle.s a2, fa5, fs0 +; RV32IZFH-NEXT: lui a3, %hi(.LCPI23_1) +; RV32IZFH-NEXT: flw fa5, %lo(.LCPI23_1)(a3) +; RV32IZFH-NEXT: xori a2, a2, 1 +; RV32IZFH-NEXT: addi a2, a2, -1 +; RV32IZFH-NEXT: and a0, a2, a0 +; RV32IZFH-NEXT: flt.s a3, fa5, fs0 +; RV32IZFH-NEXT: neg a3, a3 +; RV32IZFH-NEXT: or a0, a3, a0 +; RV32IZFH-NEXT: and a1, a2, a1 +; RV32IZFH-NEXT: or a1, a3, a1 ; RV32IZFH-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFH-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFH-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFH-NEXT: flw fs0, 8(sp) # 4-byte Folded Reload ; RV32IZFH-NEXT: addi sp, sp, 16 ; RV32IZFH-NEXT: ret ; @@ -4128,23 +4155,22 @@ define i64 @test_rint_ui64(half %x) nounwind { ; RV32IZHINX-NEXT: addi sp, sp, -16 ; RV32IZHINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZHINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZHINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZHINX-NEXT: fcvt.s.h s0, a0 -; RV32IZHINX-NEXT: fle.s a0, zero, s0 -; RV32IZHINX-NEXT: neg s1, a0 ; RV32IZHINX-NEXT: mv a0, s0 ; RV32IZHINX-NEXT: call __fixunssfdi -; RV32IZHINX-NEXT: lui a2, %hi(.LCPI23_1) -; RV32IZHINX-NEXT: lw a2, %lo(.LCPI23_1)(a2) -; RV32IZHINX-NEXT: and a0, s1, a0 -; RV32IZHINX-NEXT: flt.s a2, a2, s0 -; RV32IZHINX-NEXT: neg a2, a2 -; RV32IZHINX-NEXT: or a0, a2, a0 -; RV32IZHINX-NEXT: and a1, s1, a1 -; RV32IZHINX-NEXT: or a1, a2, a1 +; RV32IZHINX-NEXT: fle.s a2, zero, s0 +; RV32IZHINX-NEXT: lui a3, %hi(.LCPI23_1) +; RV32IZHINX-NEXT: lw a3, %lo(.LCPI23_1)(a3) +; RV32IZHINX-NEXT: xori a2, a2, 1 +; RV32IZHINX-NEXT: addi a2, a2, -1 +; RV32IZHINX-NEXT: and a0, a2, a0 +; RV32IZHINX-NEXT: flt.s a3, a3, s0 +; RV32IZHINX-NEXT: neg a3, a3 +; RV32IZHINX-NEXT: or a0, a3, a0 +; RV32IZHINX-NEXT: and a1, a2, a1 +; RV32IZHINX-NEXT: or a1, a3, a1 ; RV32IZHINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZHINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINX-NEXT: addi sp, sp, 16 ; RV32IZHINX-NEXT: ret ; @@ -4182,26 +4208,25 @@ define i64 @test_rint_ui64(half %x) nounwind { ; RV32IZFHMIN-NEXT: .LBB23_2: ; RV32IZFHMIN-NEXT: addi sp, sp, -16 ; RV32IZFHMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFHMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZFHMIN-NEXT: fsw fs0, 4(sp) # 4-byte Folded Spill +; RV32IZFHMIN-NEXT: fsw fs0, 8(sp) # 4-byte Folded Spill ; RV32IZFHMIN-NEXT: fcvt.h.s fa5, fa5 ; RV32IZFHMIN-NEXT: fcvt.s.h fs0, fa5 -; RV32IZFHMIN-NEXT: fmv.w.x fa5, zero -; RV32IZFHMIN-NEXT: fle.s a0, fa5, fs0 -; RV32IZFHMIN-NEXT: neg s0, a0 ; RV32IZFHMIN-NEXT: fmv.s fa0, fs0 ; RV32IZFHMIN-NEXT: call __fixunssfdi -; RV32IZFHMIN-NEXT: lui a2, %hi(.LCPI23_0) -; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI23_0)(a2) -; RV32IZFHMIN-NEXT: and a0, s0, a0 -; RV32IZFHMIN-NEXT: flt.s a2, fa5, fs0 -; RV32IZFHMIN-NEXT: neg a2, a2 -; RV32IZFHMIN-NEXT: or a0, a2, a0 -; RV32IZFHMIN-NEXT: and a1, s0, a1 -; RV32IZFHMIN-NEXT: or a1, a2, a1 +; RV32IZFHMIN-NEXT: fmv.w.x fa5, zero +; RV32IZFHMIN-NEXT: fle.s a2, fa5, fs0 +; RV32IZFHMIN-NEXT: lui a3, %hi(.LCPI23_0) +; RV32IZFHMIN-NEXT: flw fa5, %lo(.LCPI23_0)(a3) +; RV32IZFHMIN-NEXT: xori a2, a2, 1 +; RV32IZFHMIN-NEXT: addi a2, a2, -1 +; RV32IZFHMIN-NEXT: and a0, a2, a0 +; RV32IZFHMIN-NEXT: flt.s a3, fa5, fs0 +; RV32IZFHMIN-NEXT: neg a3, a3 +; RV32IZFHMIN-NEXT: or a0, a3, a0 +; RV32IZFHMIN-NEXT: and a1, a2, a1 +; RV32IZFHMIN-NEXT: or a1, a3, a1 ; RV32IZFHMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFHMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZFHMIN-NEXT: flw fs0, 4(sp) # 4-byte Folded Reload +; RV32IZFHMIN-NEXT: flw fs0, 8(sp) # 4-byte Folded Reload ; RV32IZFHMIN-NEXT: addi sp, sp, 16 ; RV32IZFHMIN-NEXT: ret ; @@ -4242,24 +4267,23 @@ define i64 @test_rint_ui64(half %x) nounwind { ; RV32IZHINXMIN-NEXT: addi sp, sp, -16 ; RV32IZHINXMIN-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZHINXMIN-NEXT: sw s0, 8(sp) # 4-byte Folded Spill -; RV32IZHINXMIN-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZHINXMIN-NEXT: fcvt.h.s a0, a0 ; RV32IZHINXMIN-NEXT: fcvt.s.h s0, a0 -; RV32IZHINXMIN-NEXT: fle.s a0, zero, s0 -; RV32IZHINXMIN-NEXT: neg s1, a0 ; RV32IZHINXMIN-NEXT: mv a0, s0 ; RV32IZHINXMIN-NEXT: call __fixunssfdi -; RV32IZHINXMIN-NEXT: lui a2, %hi(.LCPI23_0) -; RV32IZHINXMIN-NEXT: lw a2, %lo(.LCPI23_0)(a2) -; RV32IZHINXMIN-NEXT: and a0, s1, a0 -; RV32IZHINXMIN-NEXT: flt.s a2, a2, s0 -; RV32IZHINXMIN-NEXT: neg a2, a2 -; RV32IZHINXMIN-NEXT: or a0, a2, a0 -; RV32IZHINXMIN-NEXT: and a1, s1, a1 -; RV32IZHINXMIN-NEXT: or a1, a2, a1 +; RV32IZHINXMIN-NEXT: fle.s a2, zero, s0 +; RV32IZHINXMIN-NEXT: lui a3, %hi(.LCPI23_0) +; RV32IZHINXMIN-NEXT: lw a3, %lo(.LCPI23_0)(a3) +; RV32IZHINXMIN-NEXT: xori a2, a2, 1 +; RV32IZHINXMIN-NEXT: addi a2, a2, -1 +; RV32IZHINXMIN-NEXT: and a0, a2, a0 +; RV32IZHINXMIN-NEXT: flt.s a3, a3, s0 +; RV32IZHINXMIN-NEXT: neg a3, a3 +; RV32IZHINXMIN-NEXT: or a0, a3, a0 +; RV32IZHINXMIN-NEXT: and a1, a2, a1 +; RV32IZHINXMIN-NEXT: or a1, a3, a1 ; RV32IZHINXMIN-NEXT: lw ra, 12(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: lw s0, 8(sp) # 4-byte Folded Reload -; RV32IZHINXMIN-NEXT: lw s1, 4(sp) # 4-byte Folded Reload ; RV32IZHINXMIN-NEXT: addi sp, sp, 16 ; RV32IZHINXMIN-NEXT: ret ; diff --git a/llvm/test/CodeGen/RISCV/iabs.ll b/llvm/test/CodeGen/RISCV/iabs.ll index cb64e24128b5..98c886333d69 100644 --- a/llvm/test/CodeGen/RISCV/iabs.ll +++ b/llvm/test/CodeGen/RISCV/iabs.ll @@ -302,56 +302,56 @@ define i128 @abs128(i128 %x) { ; RV32I-LABEL: abs128: ; RV32I: # %bb.0: ; RV32I-NEXT: lw a2, 12(a1) -; RV32I-NEXT: lw a3, 4(a1) -; RV32I-NEXT: lw a4, 0(a1) +; RV32I-NEXT: lw a3, 0(a1) +; RV32I-NEXT: lw a4, 4(a1) ; RV32I-NEXT: lw a1, 8(a1) ; RV32I-NEXT: bgez a2, .LBB8_2 ; RV32I-NEXT: # %bb.1: ; RV32I-NEXT: neg a5, a1 -; RV32I-NEXT: or a6, a4, a3 -; RV32I-NEXT: snez a6, a6 -; RV32I-NEXT: sltu a7, a5, a6 +; RV32I-NEXT: snez a6, a4 +; RV32I-NEXT: snez a7, a3 +; RV32I-NEXT: or a6, a7, a6 +; RV32I-NEXT: sltu t0, a5, a6 ; RV32I-NEXT: snez a1, a1 ; RV32I-NEXT: add a1, a2, a1 ; RV32I-NEXT: neg a1, a1 -; RV32I-NEXT: sub a2, a1, a7 +; RV32I-NEXT: sub a2, a1, t0 ; RV32I-NEXT: sub a1, a5, a6 -; RV32I-NEXT: snez a5, a4 -; RV32I-NEXT: neg a3, a3 -; RV32I-NEXT: sub a3, a3, a5 ; RV32I-NEXT: neg a4, a4 +; RV32I-NEXT: sub a4, a4, a7 +; RV32I-NEXT: neg a3, a3 ; RV32I-NEXT: .LBB8_2: -; RV32I-NEXT: sw a4, 0(a0) +; RV32I-NEXT: sw a3, 0(a0) +; RV32I-NEXT: sw a4, 4(a0) ; RV32I-NEXT: sw a1, 8(a0) -; RV32I-NEXT: sw a3, 4(a0) ; RV32I-NEXT: sw a2, 12(a0) ; RV32I-NEXT: ret ; ; RV32ZBB-LABEL: abs128: ; RV32ZBB: # %bb.0: ; RV32ZBB-NEXT: lw a2, 12(a1) -; RV32ZBB-NEXT: lw a3, 4(a1) -; RV32ZBB-NEXT: lw a4, 0(a1) +; RV32ZBB-NEXT: lw a3, 0(a1) +; RV32ZBB-NEXT: lw a4, 4(a1) ; RV32ZBB-NEXT: lw a1, 8(a1) ; RV32ZBB-NEXT: bgez a2, .LBB8_2 ; RV32ZBB-NEXT: # %bb.1: ; RV32ZBB-NEXT: neg a5, a1 -; RV32ZBB-NEXT: or a6, a4, a3 -; RV32ZBB-NEXT: snez a6, a6 -; RV32ZBB-NEXT: sltu a7, a5, a6 +; RV32ZBB-NEXT: snez a6, a4 +; RV32ZBB-NEXT: snez a7, a3 +; RV32ZBB-NEXT: or a6, a7, a6 +; RV32ZBB-NEXT: sltu t0, a5, a6 ; RV32ZBB-NEXT: snez a1, a1 ; RV32ZBB-NEXT: add a1, a2, a1 ; RV32ZBB-NEXT: neg a1, a1 -; RV32ZBB-NEXT: sub a2, a1, a7 +; RV32ZBB-NEXT: sub a2, a1, t0 ; RV32ZBB-NEXT: sub a1, a5, a6 -; RV32ZBB-NEXT: snez a5, a4 -; RV32ZBB-NEXT: neg a3, a3 -; RV32ZBB-NEXT: sub a3, a3, a5 ; RV32ZBB-NEXT: neg a4, a4 +; RV32ZBB-NEXT: sub a4, a4, a7 +; RV32ZBB-NEXT: neg a3, a3 ; RV32ZBB-NEXT: .LBB8_2: -; RV32ZBB-NEXT: sw a4, 0(a0) +; RV32ZBB-NEXT: sw a3, 0(a0) +; RV32ZBB-NEXT: sw a4, 4(a0) ; RV32ZBB-NEXT: sw a1, 8(a0) -; RV32ZBB-NEXT: sw a3, 4(a0) ; RV32ZBB-NEXT: sw a2, 12(a0) ; RV32ZBB-NEXT: ret ; @@ -384,56 +384,56 @@ define i128 @select_abs128(i128 %x) { ; RV32I-LABEL: select_abs128: ; RV32I: # %bb.0: ; RV32I-NEXT: lw a2, 12(a1) -; RV32I-NEXT: lw a3, 4(a1) -; RV32I-NEXT: lw a4, 0(a1) +; RV32I-NEXT: lw a3, 0(a1) +; RV32I-NEXT: lw a4, 4(a1) ; RV32I-NEXT: lw a1, 8(a1) ; RV32I-NEXT: bgez a2, .LBB9_2 ; RV32I-NEXT: # %bb.1: ; RV32I-NEXT: neg a5, a1 -; RV32I-NEXT: or a6, a4, a3 -; RV32I-NEXT: snez a6, a6 -; RV32I-NEXT: sltu a7, a5, a6 +; RV32I-NEXT: snez a6, a4 +; RV32I-NEXT: snez a7, a3 +; RV32I-NEXT: or a6, a7, a6 +; RV32I-NEXT: sltu t0, a5, a6 ; RV32I-NEXT: snez a1, a1 ; RV32I-NEXT: add a1, a2, a1 ; RV32I-NEXT: neg a1, a1 -; RV32I-NEXT: sub a2, a1, a7 +; RV32I-NEXT: sub a2, a1, t0 ; RV32I-NEXT: sub a1, a5, a6 -; RV32I-NEXT: snez a5, a4 -; RV32I-NEXT: neg a3, a3 -; RV32I-NEXT: sub a3, a3, a5 ; RV32I-NEXT: neg a4, a4 +; RV32I-NEXT: sub a4, a4, a7 +; RV32I-NEXT: neg a3, a3 ; RV32I-NEXT: .LBB9_2: -; RV32I-NEXT: sw a4, 0(a0) +; RV32I-NEXT: sw a3, 0(a0) +; RV32I-NEXT: sw a4, 4(a0) ; RV32I-NEXT: sw a1, 8(a0) -; RV32I-NEXT: sw a3, 4(a0) ; RV32I-NEXT: sw a2, 12(a0) ; RV32I-NEXT: ret ; ; RV32ZBB-LABEL: select_abs128: ; RV32ZBB: # %bb.0: ; RV32ZBB-NEXT: lw a2, 12(a1) -; RV32ZBB-NEXT: lw a3, 4(a1) -; RV32ZBB-NEXT: lw a4, 0(a1) +; RV32ZBB-NEXT: lw a3, 0(a1) +; RV32ZBB-NEXT: lw a4, 4(a1) ; RV32ZBB-NEXT: lw a1, 8(a1) ; RV32ZBB-NEXT: bgez a2, .LBB9_2 ; RV32ZBB-NEXT: # %bb.1: ; RV32ZBB-NEXT: neg a5, a1 -; RV32ZBB-NEXT: or a6, a4, a3 -; RV32ZBB-NEXT: snez a6, a6 -; RV32ZBB-NEXT: sltu a7, a5, a6 +; RV32ZBB-NEXT: snez a6, a4 +; RV32ZBB-NEXT: snez a7, a3 +; RV32ZBB-NEXT: or a6, a7, a6 +; RV32ZBB-NEXT: sltu t0, a5, a6 ; RV32ZBB-NEXT: snez a1, a1 ; RV32ZBB-NEXT: add a1, a2, a1 ; RV32ZBB-NEXT: neg a1, a1 -; RV32ZBB-NEXT: sub a2, a1, a7 +; RV32ZBB-NEXT: sub a2, a1, t0 ; RV32ZBB-NEXT: sub a1, a5, a6 -; RV32ZBB-NEXT: snez a5, a4 -; RV32ZBB-NEXT: neg a3, a3 -; RV32ZBB-NEXT: sub a3, a3, a5 ; RV32ZBB-NEXT: neg a4, a4 +; RV32ZBB-NEXT: sub a4, a4, a7 +; RV32ZBB-NEXT: neg a3, a3 ; RV32ZBB-NEXT: .LBB9_2: -; RV32ZBB-NEXT: sw a4, 0(a0) +; RV32ZBB-NEXT: sw a3, 0(a0) +; RV32ZBB-NEXT: sw a4, 4(a0) ; RV32ZBB-NEXT: sw a1, 8(a0) -; RV32ZBB-NEXT: sw a3, 4(a0) ; RV32ZBB-NEXT: sw a2, 12(a0) ; RV32ZBB-NEXT: ret ; diff --git a/llvm/test/CodeGen/RISCV/pr84200.ll b/llvm/test/CodeGen/RISCV/pr84200.ll new file mode 100644 index 000000000000..19a102b84ed0 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/pr84200.ll @@ -0,0 +1,22 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc < %s -mtriple=riscv64 | FileCheck %s + +; The sub nuw produces poison if the input is not 0 or 1. We must insert a +; freeze before converting the sub to AND so that we don't propagate poison. +define i64 @foo(i64 %1) { +; CHECK-LABEL: foo: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: li a1, 1 +; CHECK-NEXT: sub a1, a1, a0 +; CHECK-NEXT: sltiu a0, a0, 2 +; CHECK-NEXT: xori a1, a1, 1 +; CHECK-NEXT: neg a0, a0 +; CHECK-NEXT: and a0, a0, a1 +; CHECK-NEXT: ret +entry: + %.urem.i = sub nuw i64 1, %1 + %.cmp.i = icmp ugt i64 %1, 1 + %2 = xor i64 %.urem.i, 1 + %3 = select i1 %.cmp.i, i64 0, i64 %2 + ret i64 %3 +} diff --git a/llvm/test/CodeGen/RISCV/rv32zbb-zbkb.ll b/llvm/test/CodeGen/RISCV/rv32zbb-zbkb.ll index 71040bf2646d..4e958f5699ad 100644 --- a/llvm/test/CodeGen/RISCV/rv32zbb-zbkb.ll +++ b/llvm/test/CodeGen/RISCV/rv32zbb-zbkb.ll @@ -298,14 +298,14 @@ define i32 @not_shl_one_i32(i32 %x) { define i64 @not_shl_one_i64(i64 %x) { ; CHECK-LABEL: not_shl_one_i64: ; CHECK: # %bb.0: -; CHECK-NEXT: li a1, 1 -; CHECK-NEXT: sll a1, a1, a0 -; CHECK-NEXT: addi a0, a0, -32 -; CHECK-NEXT: slti a0, a0, 0 -; CHECK-NEXT: neg a2, a0 -; CHECK-NEXT: and a2, a2, a1 -; CHECK-NEXT: addi a0, a0, -1 -; CHECK-NEXT: and a1, a0, a1 +; CHECK-NEXT: addi a1, a0, -32 +; CHECK-NEXT: slti a1, a1, 0 +; CHECK-NEXT: neg a2, a1 +; CHECK-NEXT: li a3, 1 +; CHECK-NEXT: sll a0, a3, a0 +; CHECK-NEXT: and a2, a2, a0 +; CHECK-NEXT: addi a1, a1, -1 +; CHECK-NEXT: and a1, a1, a0 ; CHECK-NEXT: not a0, a2 ; CHECK-NEXT: not a1, a1 ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/RISCV/rv32zbs.ll b/llvm/test/CodeGen/RISCV/rv32zbs.ll index ccda8f4e5dd0..30aba61ba474 100644 --- a/llvm/test/CodeGen/RISCV/rv32zbs.ll +++ b/llvm/test/CodeGen/RISCV/rv32zbs.ll @@ -48,20 +48,20 @@ define i32 @bclr_i32_no_mask(i32 %a, i32 %b) nounwind { define i64 @bclr_i64(i64 %a, i64 %b) nounwind { ; RV32I-LABEL: bclr_i64: ; RV32I: # %bb.0: -; RV32I-NEXT: li a3, 1 -; RV32I-NEXT: sll a4, a3, a2 -; RV32I-NEXT: andi a2, a2, 63 -; RV32I-NEXT: addi a5, a2, -32 -; RV32I-NEXT: slti a5, a5, 0 -; RV32I-NEXT: neg a6, a5 -; RV32I-NEXT: and a4, a6, a4 -; RV32I-NEXT: sll a2, a3, a2 -; RV32I-NEXT: addi a5, a5, -1 +; RV32I-NEXT: andi a3, a2, 63 +; RV32I-NEXT: addi a4, a3, -32 +; RV32I-NEXT: slti a4, a4, 0 +; RV32I-NEXT: neg a5, a4 +; RV32I-NEXT: li a6, 1 +; RV32I-NEXT: sll a2, a6, a2 ; RV32I-NEXT: and a2, a5, a2 -; RV32I-NEXT: not a3, a4 +; RV32I-NEXT: sll a3, a6, a3 +; RV32I-NEXT: addi a4, a4, -1 +; RV32I-NEXT: and a3, a4, a3 ; RV32I-NEXT: not a2, a2 -; RV32I-NEXT: and a0, a3, a0 -; RV32I-NEXT: and a1, a2, a1 +; RV32I-NEXT: not a3, a3 +; RV32I-NEXT: and a0, a2, a0 +; RV32I-NEXT: and a1, a3, a1 ; RV32I-NEXT: ret ; ; RV32ZBSNOZBB-LABEL: bclr_i64: @@ -186,14 +186,14 @@ define i64 @bset_i64(i64 %a, i64 %b) nounwind { define signext i64 @bset_i64_zero(i64 signext %a) nounwind { ; RV32I-LABEL: bset_i64_zero: ; RV32I: # %bb.0: -; RV32I-NEXT: li a1, 1 -; RV32I-NEXT: sll a1, a1, a0 -; RV32I-NEXT: addi a0, a0, -32 -; RV32I-NEXT: slti a2, a0, 0 -; RV32I-NEXT: neg a0, a2 -; RV32I-NEXT: and a0, a0, a1 -; RV32I-NEXT: addi a2, a2, -1 -; RV32I-NEXT: and a1, a2, a1 +; RV32I-NEXT: addi a1, a0, -32 +; RV32I-NEXT: slti a1, a1, 0 +; RV32I-NEXT: neg a2, a1 +; RV32I-NEXT: li a3, 1 +; RV32I-NEXT: sll a3, a3, a0 +; RV32I-NEXT: and a0, a2, a3 +; RV32I-NEXT: addi a1, a1, -1 +; RV32I-NEXT: and a1, a1, a3 ; RV32I-NEXT: ret ; ; RV32ZBS-LABEL: bset_i64_zero: diff --git a/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64xtheadbb.ll b/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64xtheadbb.ll index 4ec7f2660b2a..73bfc6480b4d 100644 --- a/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64xtheadbb.ll +++ b/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64xtheadbb.ll @@ -489,7 +489,7 @@ define signext i32 @findFirstSet_i32(i32 signext %a) nounwind { ; RV64I-NEXT: add a0, a1, a0 ; RV64I-NEXT: lbu a0, 0(a0) ; RV64I-NEXT: snez a1, s0 -; RV64I-NEXT: addi a1, a1, -1 +; RV64I-NEXT: addiw a1, a1, -1 ; RV64I-NEXT: or a0, a1, a0 ; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload ; RV64I-NEXT: ld s0, 0(sp) # 8-byte Folded Reload @@ -513,7 +513,7 @@ define signext i32 @findFirstSet_i32(i32 signext %a) nounwind { ; RV64XTHEADBB-NEXT: add a0, a1, a0 ; RV64XTHEADBB-NEXT: lbu a0, 0(a0) ; RV64XTHEADBB-NEXT: snez a1, s0 -; RV64XTHEADBB-NEXT: addi a1, a1, -1 +; RV64XTHEADBB-NEXT: addiw a1, a1, -1 ; RV64XTHEADBB-NEXT: or a0, a1, a0 ; RV64XTHEADBB-NEXT: ld ra, 8(sp) # 8-byte Folded Reload ; RV64XTHEADBB-NEXT: ld s0, 0(sp) # 8-byte Folded Reload @@ -542,12 +542,10 @@ define signext i32 @ffs_i32(i32 signext %a) nounwind { ; RV64I-NEXT: addi a1, a1, %lo(.LCPI9_0) ; RV64I-NEXT: add a0, a1, a0 ; RV64I-NEXT: lbu a0, 0(a0) -; RV64I-NEXT: addi a0, a0, 1 +; RV64I-NEXT: addiw a0, a0, 1 ; RV64I-NEXT: seqz a1, s0 -; RV64I-NEXT: addi a1, a1, -1 +; RV64I-NEXT: addiw a1, a1, -1 ; RV64I-NEXT: and a0, a1, a0 -; RV64I-NEXT: slli a0, a0, 32 -; RV64I-NEXT: srli a0, a0, 32 ; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload ; RV64I-NEXT: ld s0, 0(sp) # 8-byte Folded Reload ; RV64I-NEXT: addi sp, sp, 16 @@ -569,12 +567,10 @@ define signext i32 @ffs_i32(i32 signext %a) nounwind { ; RV64XTHEADBB-NEXT: addi a1, a1, %lo(.LCPI9_0) ; RV64XTHEADBB-NEXT: add a0, a1, a0 ; RV64XTHEADBB-NEXT: lbu a0, 0(a0) -; RV64XTHEADBB-NEXT: addi a0, a0, 1 +; RV64XTHEADBB-NEXT: addiw a0, a0, 1 ; RV64XTHEADBB-NEXT: seqz a1, s0 -; RV64XTHEADBB-NEXT: addi a1, a1, -1 +; RV64XTHEADBB-NEXT: addiw a1, a1, -1 ; RV64XTHEADBB-NEXT: and a0, a1, a0 -; RV64XTHEADBB-NEXT: slli a0, a0, 32 -; RV64XTHEADBB-NEXT: srli a0, a0, 32 ; RV64XTHEADBB-NEXT: ld ra, 8(sp) # 8-byte Folded Reload ; RV64XTHEADBB-NEXT: ld s0, 0(sp) # 8-byte Folded Reload ; RV64XTHEADBB-NEXT: addi sp, sp, 16 diff --git a/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll b/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll index 68ce66cbe853..7feef4dad411 100644 --- a/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll +++ b/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zbb.ll @@ -444,7 +444,7 @@ define signext i32 @findFirstSet_i32(i32 signext %a) nounwind { ; RV64I-NEXT: add a0, a1, a0 ; RV64I-NEXT: lbu a0, 0(a0) ; RV64I-NEXT: snez a1, s0 -; RV64I-NEXT: addi a1, a1, -1 +; RV64I-NEXT: addiw a1, a1, -1 ; RV64I-NEXT: or a0, a1, a0 ; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload ; RV64I-NEXT: ld s0, 0(sp) # 8-byte Folded Reload @@ -481,12 +481,10 @@ define signext i32 @ffs_i32(i32 signext %a) nounwind { ; RV64I-NEXT: addi a1, a1, %lo(.LCPI9_0) ; RV64I-NEXT: add a0, a1, a0 ; RV64I-NEXT: lbu a0, 0(a0) -; RV64I-NEXT: addi a0, a0, 1 +; RV64I-NEXT: addiw a0, a0, 1 ; RV64I-NEXT: seqz a1, s0 -; RV64I-NEXT: addi a1, a1, -1 +; RV64I-NEXT: addiw a1, a1, -1 ; RV64I-NEXT: and a0, a1, a0 -; RV64I-NEXT: slli a0, a0, 32 -; RV64I-NEXT: srli a0, a0, 32 ; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload ; RV64I-NEXT: ld s0, 0(sp) # 8-byte Folded Reload ; RV64I-NEXT: addi sp, sp, 16 @@ -495,11 +493,10 @@ define signext i32 @ffs_i32(i32 signext %a) nounwind { ; RV64ZBB-LABEL: ffs_i32: ; RV64ZBB: # %bb.0: ; RV64ZBB-NEXT: ctzw a1, a0 -; RV64ZBB-NEXT: addi a1, a1, 1 +; RV64ZBB-NEXT: addiw a1, a1, 1 ; RV64ZBB-NEXT: seqz a0, a0 -; RV64ZBB-NEXT: addi a0, a0, -1 +; RV64ZBB-NEXT: addiw a0, a0, -1 ; RV64ZBB-NEXT: and a0, a0, a1 -; RV64ZBB-NEXT: zext.h a0, a0 ; RV64ZBB-NEXT: ret %1 = call i32 @llvm.cttz.i32(i32 %a, i1 true) %2 = add i32 %1, 1 diff --git a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll index cbdabab65cc6..3ada24bd9846 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll @@ -89,17 +89,17 @@ entry: define <2 x i32> @ustest_f64i32(<2 x double> %x) { ; CHECK-NOV-LABEL: ustest_f64i32: ; CHECK-NOV: # %bb.0: # %entry -; CHECK-NOV-NEXT: fcvt.l.d a0, fa0, rtz +; CHECK-NOV-NEXT: fcvt.l.d a1, fa1, rtz ; CHECK-NOV-NEXT: li a2, -1 ; CHECK-NOV-NEXT: srli a2, a2, 32 -; CHECK-NOV-NEXT: fcvt.l.d a1, fa1, rtz -; CHECK-NOV-NEXT: blt a0, a2, .LBB2_2 +; CHECK-NOV-NEXT: fcvt.l.d a0, fa0, rtz +; CHECK-NOV-NEXT: blt a1, a2, .LBB2_2 ; CHECK-NOV-NEXT: # %bb.1: # %entry -; CHECK-NOV-NEXT: mv a0, a2 +; CHECK-NOV-NEXT: mv a1, a2 ; CHECK-NOV-NEXT: .LBB2_2: # %entry -; CHECK-NOV-NEXT: blt a1, a2, .LBB2_4 +; CHECK-NOV-NEXT: blt a0, a2, .LBB2_4 ; CHECK-NOV-NEXT: # %bb.3: # %entry -; CHECK-NOV-NEXT: mv a1, a2 +; CHECK-NOV-NEXT: mv a0, a2 ; CHECK-NOV-NEXT: .LBB2_4: # %entry ; CHECK-NOV-NEXT: sgtz a2, a1 ; CHECK-NOV-NEXT: sgtz a3, a0 @@ -257,46 +257,46 @@ entry: define <4 x i32> @ustest_f32i32(<4 x float> %x) { ; CHECK-NOV-LABEL: ustest_f32i32: ; CHECK-NOV: # %bb.0: # %entry -; CHECK-NOV-NEXT: fcvt.l.s a1, fa0, rtz +; CHECK-NOV-NEXT: fcvt.l.s a1, fa3, rtz ; CHECK-NOV-NEXT: li a4, -1 ; CHECK-NOV-NEXT: srli a4, a4, 32 -; CHECK-NOV-NEXT: fcvt.l.s a2, fa1, rtz +; CHECK-NOV-NEXT: fcvt.l.s a2, fa2, rtz ; CHECK-NOV-NEXT: bge a1, a4, .LBB5_6 ; CHECK-NOV-NEXT: # %bb.1: # %entry -; CHECK-NOV-NEXT: fcvt.l.s a3, fa2, rtz +; CHECK-NOV-NEXT: fcvt.l.s a3, fa1, rtz ; CHECK-NOV-NEXT: bge a2, a4, .LBB5_7 ; CHECK-NOV-NEXT: .LBB5_2: # %entry -; CHECK-NOV-NEXT: fcvt.l.s a5, fa3, rtz +; CHECK-NOV-NEXT: fcvt.l.s a5, fa0, rtz ; CHECK-NOV-NEXT: bge a3, a4, .LBB5_8 ; CHECK-NOV-NEXT: .LBB5_3: # %entry ; CHECK-NOV-NEXT: blt a5, a4, .LBB5_5 ; CHECK-NOV-NEXT: .LBB5_4: # %entry ; CHECK-NOV-NEXT: mv a5, a4 ; CHECK-NOV-NEXT: .LBB5_5: # %entry -; CHECK-NOV-NEXT: sgtz a4, a5 -; CHECK-NOV-NEXT: sgtz a6, a3 -; CHECK-NOV-NEXT: sgtz a7, a2 -; CHECK-NOV-NEXT: sgtz t0, a1 +; CHECK-NOV-NEXT: sgtz a4, a1 +; CHECK-NOV-NEXT: sgtz a6, a2 +; CHECK-NOV-NEXT: sgtz a7, a3 +; CHECK-NOV-NEXT: sgtz t0, a5 ; CHECK-NOV-NEXT: negw t0, t0 -; CHECK-NOV-NEXT: and a1, t0, a1 +; CHECK-NOV-NEXT: and a5, t0, a5 ; CHECK-NOV-NEXT: negw a7, a7 -; CHECK-NOV-NEXT: and a2, a7, a2 +; CHECK-NOV-NEXT: and a3, a7, a3 ; CHECK-NOV-NEXT: negw a6, a6 -; CHECK-NOV-NEXT: and a3, a6, a3 +; CHECK-NOV-NEXT: and a2, a6, a2 ; CHECK-NOV-NEXT: negw a4, a4 -; CHECK-NOV-NEXT: and a4, a4, a5 -; CHECK-NOV-NEXT: sw a4, 12(a0) -; CHECK-NOV-NEXT: sw a3, 8(a0) -; CHECK-NOV-NEXT: sw a2, 4(a0) -; CHECK-NOV-NEXT: sw a1, 0(a0) +; CHECK-NOV-NEXT: and a1, a4, a1 +; CHECK-NOV-NEXT: sw a1, 12(a0) +; CHECK-NOV-NEXT: sw a2, 8(a0) +; CHECK-NOV-NEXT: sw a3, 4(a0) +; CHECK-NOV-NEXT: sw a5, 0(a0) ; CHECK-NOV-NEXT: ret ; CHECK-NOV-NEXT: .LBB5_6: # %entry ; CHECK-NOV-NEXT: mv a1, a4 -; CHECK-NOV-NEXT: fcvt.l.s a3, fa2, rtz +; CHECK-NOV-NEXT: fcvt.l.s a3, fa1, rtz ; CHECK-NOV-NEXT: blt a2, a4, .LBB5_2 ; CHECK-NOV-NEXT: .LBB5_7: # %entry ; CHECK-NOV-NEXT: mv a2, a4 -; CHECK-NOV-NEXT: fcvt.l.s a5, fa3, rtz +; CHECK-NOV-NEXT: fcvt.l.s a5, fa0, rtz ; CHECK-NOV-NEXT: blt a3, a4, .LBB5_3 ; CHECK-NOV-NEXT: .LBB5_8: # %entry ; CHECK-NOV-NEXT: mv a3, a4 @@ -686,10 +686,10 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) { ; CHECK-NOV-NEXT: .cfi_offset fs0, -48 ; CHECK-NOV-NEXT: .cfi_offset fs1, -56 ; CHECK-NOV-NEXT: .cfi_offset fs2, -64 -; CHECK-NOV-NEXT: lhu s1, 0(a1) -; CHECK-NOV-NEXT: lhu s2, 24(a1) -; CHECK-NOV-NEXT: lhu s3, 16(a1) -; CHECK-NOV-NEXT: lhu a1, 8(a1) +; CHECK-NOV-NEXT: lhu s1, 24(a1) +; CHECK-NOV-NEXT: lhu s2, 0(a1) +; CHECK-NOV-NEXT: lhu s3, 8(a1) +; CHECK-NOV-NEXT: lhu a1, 16(a1) ; CHECK-NOV-NEXT: mv s0, a0 ; CHECK-NOV-NEXT: fmv.w.x fa0, a1 ; CHECK-NOV-NEXT: call __extendhfsf2 @@ -718,22 +718,22 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) { ; CHECK-NOV-NEXT: .LBB8_4: # %entry ; CHECK-NOV-NEXT: mv a3, a2 ; CHECK-NOV-NEXT: .LBB8_5: # %entry -; CHECK-NOV-NEXT: sgtz a2, a3 -; CHECK-NOV-NEXT: sgtz a4, a1 -; CHECK-NOV-NEXT: sgtz a5, s1 -; CHECK-NOV-NEXT: sgtz a6, a0 +; CHECK-NOV-NEXT: sgtz a2, a0 +; CHECK-NOV-NEXT: sgtz a4, s1 +; CHECK-NOV-NEXT: sgtz a5, a1 +; CHECK-NOV-NEXT: sgtz a6, a3 ; CHECK-NOV-NEXT: negw a6, a6 -; CHECK-NOV-NEXT: and a0, a6, a0 +; CHECK-NOV-NEXT: and a3, a6, a3 ; CHECK-NOV-NEXT: negw a5, a5 -; CHECK-NOV-NEXT: and a5, a5, s1 +; CHECK-NOV-NEXT: and a1, a5, a1 ; CHECK-NOV-NEXT: negw a4, a4 -; CHECK-NOV-NEXT: and a1, a4, a1 +; CHECK-NOV-NEXT: and a4, a4, s1 ; CHECK-NOV-NEXT: negw a2, a2 -; CHECK-NOV-NEXT: and a2, a2, a3 -; CHECK-NOV-NEXT: sw a2, 12(s0) -; CHECK-NOV-NEXT: sw a1, 8(s0) -; CHECK-NOV-NEXT: sw a5, 4(s0) -; CHECK-NOV-NEXT: sw a0, 0(s0) +; CHECK-NOV-NEXT: and a0, a2, a0 +; CHECK-NOV-NEXT: sw a0, 12(s0) +; CHECK-NOV-NEXT: sw a4, 8(s0) +; CHECK-NOV-NEXT: sw a1, 4(s0) +; CHECK-NOV-NEXT: sw a3, 0(s0) ; CHECK-NOV-NEXT: ld ra, 56(sp) # 8-byte Folded Reload ; CHECK-NOV-NEXT: ld s0, 48(sp) # 8-byte Folded Reload ; CHECK-NOV-NEXT: ld s1, 40(sp) # 8-byte Folded Reload @@ -929,17 +929,17 @@ entry: define <2 x i16> @ustest_f64i16(<2 x double> %x) { ; CHECK-NOV-LABEL: ustest_f64i16: ; CHECK-NOV: # %bb.0: # %entry -; CHECK-NOV-NEXT: fcvt.w.d a0, fa0, rtz +; CHECK-NOV-NEXT: fcvt.w.d a1, fa1, rtz ; CHECK-NOV-NEXT: lui a2, 16 ; CHECK-NOV-NEXT: addiw a2, a2, -1 -; CHECK-NOV-NEXT: fcvt.w.d a1, fa1, rtz -; CHECK-NOV-NEXT: blt a0, a2, .LBB11_2 +; CHECK-NOV-NEXT: fcvt.w.d a0, fa0, rtz +; CHECK-NOV-NEXT: blt a1, a2, .LBB11_2 ; CHECK-NOV-NEXT: # %bb.1: # %entry -; CHECK-NOV-NEXT: mv a0, a2 +; CHECK-NOV-NEXT: mv a1, a2 ; CHECK-NOV-NEXT: .LBB11_2: # %entry -; CHECK-NOV-NEXT: blt a1, a2, .LBB11_4 +; CHECK-NOV-NEXT: blt a0, a2, .LBB11_4 ; CHECK-NOV-NEXT: # %bb.3: # %entry -; CHECK-NOV-NEXT: mv a1, a2 +; CHECK-NOV-NEXT: mv a0, a2 ; CHECK-NOV-NEXT: .LBB11_4: # %entry ; CHECK-NOV-NEXT: sgtz a2, a1 ; CHECK-NOV-NEXT: sgtz a3, a0 @@ -1101,46 +1101,46 @@ entry: define <4 x i16> @ustest_f32i16(<4 x float> %x) { ; CHECK-NOV-LABEL: ustest_f32i16: ; CHECK-NOV: # %bb.0: # %entry -; CHECK-NOV-NEXT: fcvt.w.s a1, fa0, rtz +; CHECK-NOV-NEXT: fcvt.w.s a1, fa3, rtz ; CHECK-NOV-NEXT: lui a4, 16 ; CHECK-NOV-NEXT: addiw a4, a4, -1 -; CHECK-NOV-NEXT: fcvt.w.s a2, fa1, rtz +; CHECK-NOV-NEXT: fcvt.w.s a2, fa2, rtz ; CHECK-NOV-NEXT: bge a1, a4, .LBB14_6 ; CHECK-NOV-NEXT: # %bb.1: # %entry -; CHECK-NOV-NEXT: fcvt.w.s a3, fa2, rtz +; CHECK-NOV-NEXT: fcvt.w.s a3, fa1, rtz ; CHECK-NOV-NEXT: bge a2, a4, .LBB14_7 ; CHECK-NOV-NEXT: .LBB14_2: # %entry -; CHECK-NOV-NEXT: fcvt.w.s a5, fa3, rtz +; CHECK-NOV-NEXT: fcvt.w.s a5, fa0, rtz ; CHECK-NOV-NEXT: bge a3, a4, .LBB14_8 ; CHECK-NOV-NEXT: .LBB14_3: # %entry ; CHECK-NOV-NEXT: blt a5, a4, .LBB14_5 ; CHECK-NOV-NEXT: .LBB14_4: # %entry ; CHECK-NOV-NEXT: mv a5, a4 ; CHECK-NOV-NEXT: .LBB14_5: # %entry -; CHECK-NOV-NEXT: sgtz a4, a5 -; CHECK-NOV-NEXT: sgtz a6, a3 -; CHECK-NOV-NEXT: sgtz a7, a2 -; CHECK-NOV-NEXT: sgtz t0, a1 +; CHECK-NOV-NEXT: sgtz a4, a1 +; CHECK-NOV-NEXT: sgtz a6, a2 +; CHECK-NOV-NEXT: sgtz a7, a3 +; CHECK-NOV-NEXT: sgtz t0, a5 ; CHECK-NOV-NEXT: negw t0, t0 -; CHECK-NOV-NEXT: and a1, t0, a1 +; CHECK-NOV-NEXT: and a5, t0, a5 ; CHECK-NOV-NEXT: negw a7, a7 -; CHECK-NOV-NEXT: and a2, a7, a2 +; CHECK-NOV-NEXT: and a3, a7, a3 ; CHECK-NOV-NEXT: negw a6, a6 -; CHECK-NOV-NEXT: and a3, a6, a3 +; CHECK-NOV-NEXT: and a2, a6, a2 ; CHECK-NOV-NEXT: negw a4, a4 -; CHECK-NOV-NEXT: and a4, a4, a5 -; CHECK-NOV-NEXT: sh a4, 6(a0) -; CHECK-NOV-NEXT: sh a3, 4(a0) -; CHECK-NOV-NEXT: sh a2, 2(a0) -; CHECK-NOV-NEXT: sh a1, 0(a0) +; CHECK-NOV-NEXT: and a1, a4, a1 +; CHECK-NOV-NEXT: sh a1, 6(a0) +; CHECK-NOV-NEXT: sh a2, 4(a0) +; CHECK-NOV-NEXT: sh a3, 2(a0) +; CHECK-NOV-NEXT: sh a5, 0(a0) ; CHECK-NOV-NEXT: ret ; CHECK-NOV-NEXT: .LBB14_6: # %entry ; CHECK-NOV-NEXT: mv a1, a4 -; CHECK-NOV-NEXT: fcvt.w.s a3, fa2, rtz +; CHECK-NOV-NEXT: fcvt.w.s a3, fa1, rtz ; CHECK-NOV-NEXT: blt a2, a4, .LBB14_2 ; CHECK-NOV-NEXT: .LBB14_7: # %entry ; CHECK-NOV-NEXT: mv a2, a4 -; CHECK-NOV-NEXT: fcvt.w.s a5, fa3, rtz +; CHECK-NOV-NEXT: fcvt.w.s a5, fa0, rtz ; CHECK-NOV-NEXT: blt a3, a4, .LBB14_3 ; CHECK-NOV-NEXT: .LBB14_8: # %entry ; CHECK-NOV-NEXT: mv a3, a4 @@ -1871,14 +1871,14 @@ define <8 x i16> @ustest_f16i16(<8 x half> %x) { ; CHECK-NOV-NEXT: .cfi_offset fs4, -112 ; CHECK-NOV-NEXT: .cfi_offset fs5, -120 ; CHECK-NOV-NEXT: .cfi_offset fs6, -128 -; CHECK-NOV-NEXT: lhu s1, 0(a1) -; CHECK-NOV-NEXT: lhu s2, 56(a1) -; CHECK-NOV-NEXT: lhu s3, 48(a1) -; CHECK-NOV-NEXT: lhu s4, 40(a1) -; CHECK-NOV-NEXT: lhu s5, 32(a1) -; CHECK-NOV-NEXT: lhu s6, 24(a1) -; CHECK-NOV-NEXT: lhu s7, 16(a1) -; CHECK-NOV-NEXT: lhu a1, 8(a1) +; CHECK-NOV-NEXT: lhu s1, 56(a1) +; CHECK-NOV-NEXT: lhu s2, 0(a1) +; CHECK-NOV-NEXT: lhu s3, 8(a1) +; CHECK-NOV-NEXT: lhu s4, 16(a1) +; CHECK-NOV-NEXT: lhu s5, 24(a1) +; CHECK-NOV-NEXT: lhu s6, 32(a1) +; CHECK-NOV-NEXT: lhu s7, 40(a1) +; CHECK-NOV-NEXT: lhu a1, 48(a1) ; CHECK-NOV-NEXT: mv s0, a0 ; CHECK-NOV-NEXT: fmv.w.x fa0, a1 ; CHECK-NOV-NEXT: call __extendhfsf2 @@ -1931,38 +1931,38 @@ define <8 x i16> @ustest_f16i16(<8 x half> %x) { ; CHECK-NOV-NEXT: .LBB17_8: # %entry ; CHECK-NOV-NEXT: mv a7, a3 ; CHECK-NOV-NEXT: .LBB17_9: # %entry -; CHECK-NOV-NEXT: sgtz a3, a7 -; CHECK-NOV-NEXT: sgtz t0, a6 -; CHECK-NOV-NEXT: sgtz t1, a5 -; CHECK-NOV-NEXT: sgtz t2, a4 -; CHECK-NOV-NEXT: sgtz t3, a2 -; CHECK-NOV-NEXT: sgtz t4, a1 -; CHECK-NOV-NEXT: sgtz t5, s1 -; CHECK-NOV-NEXT: sgtz t6, a0 +; CHECK-NOV-NEXT: sgtz a3, a0 +; CHECK-NOV-NEXT: sgtz t0, s1 +; CHECK-NOV-NEXT: sgtz t1, a1 +; CHECK-NOV-NEXT: sgtz t2, a2 +; CHECK-NOV-NEXT: sgtz t3, a4 +; CHECK-NOV-NEXT: sgtz t4, a5 +; CHECK-NOV-NEXT: sgtz t5, a6 +; CHECK-NOV-NEXT: sgtz t6, a7 ; CHECK-NOV-NEXT: negw t6, t6 -; CHECK-NOV-NEXT: and a0, t6, a0 +; CHECK-NOV-NEXT: and a7, t6, a7 ; CHECK-NOV-NEXT: negw t5, t5 -; CHECK-NOV-NEXT: and t5, t5, s1 +; CHECK-NOV-NEXT: and a6, t5, a6 ; CHECK-NOV-NEXT: negw t4, t4 -; CHECK-NOV-NEXT: and a1, t4, a1 +; CHECK-NOV-NEXT: and a5, t4, a5 ; CHECK-NOV-NEXT: negw t3, t3 -; CHECK-NOV-NEXT: and a2, t3, a2 +; CHECK-NOV-NEXT: and a4, t3, a4 ; CHECK-NOV-NEXT: negw t2, t2 -; CHECK-NOV-NEXT: and a4, t2, a4 +; CHECK-NOV-NEXT: and a2, t2, a2 ; CHECK-NOV-NEXT: negw t1, t1 -; CHECK-NOV-NEXT: and a5, t1, a5 +; CHECK-NOV-NEXT: and a1, t1, a1 ; CHECK-NOV-NEXT: negw t0, t0 -; CHECK-NOV-NEXT: and a6, t0, a6 +; CHECK-NOV-NEXT: and t0, t0, s1 ; CHECK-NOV-NEXT: negw a3, a3 -; CHECK-NOV-NEXT: and a3, a3, a7 -; CHECK-NOV-NEXT: sh a3, 14(s0) -; CHECK-NOV-NEXT: sh a6, 12(s0) -; CHECK-NOV-NEXT: sh a5, 10(s0) -; CHECK-NOV-NEXT: sh a4, 8(s0) -; CHECK-NOV-NEXT: sh a2, 6(s0) -; CHECK-NOV-NEXT: sh a1, 4(s0) -; CHECK-NOV-NEXT: sh t5, 2(s0) -; CHECK-NOV-NEXT: sh a0, 0(s0) +; CHECK-NOV-NEXT: and a0, a3, a0 +; CHECK-NOV-NEXT: sh a0, 14(s0) +; CHECK-NOV-NEXT: sh t0, 12(s0) +; CHECK-NOV-NEXT: sh a1, 10(s0) +; CHECK-NOV-NEXT: sh a2, 8(s0) +; CHECK-NOV-NEXT: sh a4, 6(s0) +; CHECK-NOV-NEXT: sh a5, 4(s0) +; CHECK-NOV-NEXT: sh a6, 2(s0) +; CHECK-NOV-NEXT: sh a7, 0(s0) ; CHECK-NOV-NEXT: ld ra, 120(sp) # 8-byte Folded Reload ; CHECK-NOV-NEXT: ld s0, 112(sp) # 8-byte Folded Reload ; CHECK-NOV-NEXT: ld s1, 104(sp) # 8-byte Folded Reload @@ -2190,66 +2190,65 @@ define <2 x i64> @stest_f64i64(<2 x double> %x) { ; CHECK-NOV-NEXT: .cfi_offset s0, -16 ; CHECK-NOV-NEXT: .cfi_offset s1, -24 ; CHECK-NOV-NEXT: .cfi_offset fs0, -32 -; CHECK-NOV-NEXT: fmv.d fs0, fa1 +; CHECK-NOV-NEXT: fmv.d fs0, fa0 +; CHECK-NOV-NEXT: fmv.d fa0, fa1 ; CHECK-NOV-NEXT: call __fixdfti ; CHECK-NOV-NEXT: mv s0, a0 ; CHECK-NOV-NEXT: mv s1, a1 ; CHECK-NOV-NEXT: fmv.d fa0, fs0 ; CHECK-NOV-NEXT: call __fixdfti -; CHECK-NOV-NEXT: mv a2, a0 -; CHECK-NOV-NEXT: li a0, -1 -; CHECK-NOV-NEXT: srli a3, a0, 1 -; CHECK-NOV-NEXT: beqz a1, .LBB18_3 +; CHECK-NOV-NEXT: li a2, -1 +; CHECK-NOV-NEXT: srli a3, a2, 1 +; CHECK-NOV-NEXT: beqz s1, .LBB18_3 ; CHECK-NOV-NEXT: # %bb.1: # %entry -; CHECK-NOV-NEXT: slti a4, a1, 0 -; CHECK-NOV-NEXT: bnez s1, .LBB18_4 +; CHECK-NOV-NEXT: slti a4, s1, 0 +; CHECK-NOV-NEXT: bnez a1, .LBB18_4 ; CHECK-NOV-NEXT: .LBB18_2: -; CHECK-NOV-NEXT: sltu a5, s0, a3 +; CHECK-NOV-NEXT: sltu a5, a0, a3 ; CHECK-NOV-NEXT: beqz a5, .LBB18_5 ; CHECK-NOV-NEXT: j .LBB18_6 ; CHECK-NOV-NEXT: .LBB18_3: -; CHECK-NOV-NEXT: sltu a4, a2, a3 -; CHECK-NOV-NEXT: beqz s1, .LBB18_2 +; CHECK-NOV-NEXT: sltu a4, s0, a3 +; CHECK-NOV-NEXT: beqz a1, .LBB18_2 ; CHECK-NOV-NEXT: .LBB18_4: # %entry -; CHECK-NOV-NEXT: slti a5, s1, 0 +; CHECK-NOV-NEXT: slti a5, a1, 0 ; CHECK-NOV-NEXT: bnez a5, .LBB18_6 ; CHECK-NOV-NEXT: .LBB18_5: # %entry -; CHECK-NOV-NEXT: mv s0, a3 +; CHECK-NOV-NEXT: mv a0, a3 ; CHECK-NOV-NEXT: .LBB18_6: # %entry ; CHECK-NOV-NEXT: neg a6, a5 ; CHECK-NOV-NEXT: neg a5, a4 -; CHECK-NOV-NEXT: and a5, a5, a1 +; CHECK-NOV-NEXT: and a5, a5, s1 ; CHECK-NOV-NEXT: bnez a4, .LBB18_8 ; CHECK-NOV-NEXT: # %bb.7: # %entry -; CHECK-NOV-NEXT: mv a2, a3 +; CHECK-NOV-NEXT: mv s0, a3 ; CHECK-NOV-NEXT: .LBB18_8: # %entry -; CHECK-NOV-NEXT: and a4, a6, s1 -; CHECK-NOV-NEXT: slli a1, a0, 63 -; CHECK-NOV-NEXT: beq a5, a0, .LBB18_11 +; CHECK-NOV-NEXT: and a4, a6, a1 +; CHECK-NOV-NEXT: slli a1, a2, 63 +; CHECK-NOV-NEXT: beq a5, a2, .LBB18_11 ; CHECK-NOV-NEXT: # %bb.9: # %entry ; CHECK-NOV-NEXT: slti a3, a5, 0 ; CHECK-NOV-NEXT: xori a3, a3, 1 -; CHECK-NOV-NEXT: bne a4, a0, .LBB18_12 +; CHECK-NOV-NEXT: bne a4, a2, .LBB18_12 ; CHECK-NOV-NEXT: .LBB18_10: -; CHECK-NOV-NEXT: sltu a0, a1, s0 -; CHECK-NOV-NEXT: beqz a0, .LBB18_13 +; CHECK-NOV-NEXT: sltu a2, a1, a0 +; CHECK-NOV-NEXT: beqz a2, .LBB18_13 ; CHECK-NOV-NEXT: j .LBB18_14 ; CHECK-NOV-NEXT: .LBB18_11: -; CHECK-NOV-NEXT: sltu a3, a1, a2 -; CHECK-NOV-NEXT: beq a4, a0, .LBB18_10 +; CHECK-NOV-NEXT: sltu a3, a1, s0 +; CHECK-NOV-NEXT: beq a4, a2, .LBB18_10 ; CHECK-NOV-NEXT: .LBB18_12: # %entry -; CHECK-NOV-NEXT: slti a0, a4, 0 -; CHECK-NOV-NEXT: xori a0, a0, 1 -; CHECK-NOV-NEXT: bnez a0, .LBB18_14 +; CHECK-NOV-NEXT: slti a2, a4, 0 +; CHECK-NOV-NEXT: xori a2, a2, 1 +; CHECK-NOV-NEXT: bnez a2, .LBB18_14 ; CHECK-NOV-NEXT: .LBB18_13: # %entry -; CHECK-NOV-NEXT: mv s0, a1 +; CHECK-NOV-NEXT: mv a0, a1 ; CHECK-NOV-NEXT: .LBB18_14: # %entry ; CHECK-NOV-NEXT: bnez a3, .LBB18_16 ; CHECK-NOV-NEXT: # %bb.15: # %entry -; CHECK-NOV-NEXT: mv a2, a1 +; CHECK-NOV-NEXT: mv s0, a1 ; CHECK-NOV-NEXT: .LBB18_16: # %entry -; CHECK-NOV-NEXT: mv a0, s0 -; CHECK-NOV-NEXT: mv a1, a2 +; CHECK-NOV-NEXT: mv a1, s0 ; CHECK-NOV-NEXT: ld ra, 24(sp) # 8-byte Folded Reload ; CHECK-NOV-NEXT: ld s0, 16(sp) # 8-byte Folded Reload ; CHECK-NOV-NEXT: ld s1, 8(sp) # 8-byte Folded Reload @@ -2274,43 +2273,43 @@ define <2 x i64> @stest_f64i64(<2 x double> %x) { ; CHECK-V-NEXT: addi a0, sp, 32 ; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; CHECK-V-NEXT: vslidedown.vi v9, v8, 1 -; CHECK-V-NEXT: vfmv.f.s fa0, v9 +; CHECK-V-NEXT: vfmv.f.s fa0, v8 ; CHECK-V-NEXT: call __fixdfti ; CHECK-V-NEXT: mv s0, a0 ; CHECK-V-NEXT: mv s1, a1 ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: addi a0, sp, 32 ; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslidedown.vi v8, v8, 1 ; CHECK-V-NEXT: vfmv.f.s fa0, v8 ; CHECK-V-NEXT: call __fixdfti ; CHECK-V-NEXT: li a2, -1 ; CHECK-V-NEXT: srli a3, a2, 1 -; CHECK-V-NEXT: beqz a1, .LBB18_3 +; CHECK-V-NEXT: beqz s1, .LBB18_3 ; CHECK-V-NEXT: # %bb.1: # %entry -; CHECK-V-NEXT: slti a4, a1, 0 -; CHECK-V-NEXT: bnez s1, .LBB18_4 +; CHECK-V-NEXT: slti a4, s1, 0 +; CHECK-V-NEXT: bnez a1, .LBB18_4 ; CHECK-V-NEXT: .LBB18_2: -; CHECK-V-NEXT: sltu a5, s0, a3 +; CHECK-V-NEXT: sltu a5, a0, a3 ; CHECK-V-NEXT: beqz a5, .LBB18_5 ; CHECK-V-NEXT: j .LBB18_6 ; CHECK-V-NEXT: .LBB18_3: -; CHECK-V-NEXT: sltu a4, a0, a3 -; CHECK-V-NEXT: beqz s1, .LBB18_2 +; CHECK-V-NEXT: sltu a4, s0, a3 +; CHECK-V-NEXT: beqz a1, .LBB18_2 ; CHECK-V-NEXT: .LBB18_4: # %entry -; CHECK-V-NEXT: slti a5, s1, 0 +; CHECK-V-NEXT: slti a5, a1, 0 ; CHECK-V-NEXT: bnez a5, .LBB18_6 ; CHECK-V-NEXT: .LBB18_5: # %entry -; CHECK-V-NEXT: mv s0, a3 +; CHECK-V-NEXT: mv a0, a3 ; CHECK-V-NEXT: .LBB18_6: # %entry ; CHECK-V-NEXT: neg a6, a5 ; CHECK-V-NEXT: neg a5, a4 -; CHECK-V-NEXT: and a5, a5, a1 +; CHECK-V-NEXT: and a5, a5, s1 ; CHECK-V-NEXT: bnez a4, .LBB18_8 ; CHECK-V-NEXT: # %bb.7: # %entry -; CHECK-V-NEXT: mv a0, a3 +; CHECK-V-NEXT: mv s0, a3 ; CHECK-V-NEXT: .LBB18_8: # %entry -; CHECK-V-NEXT: and a4, a6, s1 +; CHECK-V-NEXT: and a4, a6, a1 ; CHECK-V-NEXT: slli a1, a2, 63 ; CHECK-V-NEXT: beq a5, a2, .LBB18_11 ; CHECK-V-NEXT: # %bb.9: # %entry @@ -2318,26 +2317,26 @@ define <2 x i64> @stest_f64i64(<2 x double> %x) { ; CHECK-V-NEXT: xori a3, a3, 1 ; CHECK-V-NEXT: bne a4, a2, .LBB18_12 ; CHECK-V-NEXT: .LBB18_10: -; CHECK-V-NEXT: sltu a2, a1, s0 +; CHECK-V-NEXT: sltu a2, a1, a0 ; CHECK-V-NEXT: beqz a2, .LBB18_13 ; CHECK-V-NEXT: j .LBB18_14 ; CHECK-V-NEXT: .LBB18_11: -; CHECK-V-NEXT: sltu a3, a1, a0 +; CHECK-V-NEXT: sltu a3, a1, s0 ; CHECK-V-NEXT: beq a4, a2, .LBB18_10 ; CHECK-V-NEXT: .LBB18_12: # %entry ; CHECK-V-NEXT: slti a2, a4, 0 ; CHECK-V-NEXT: xori a2, a2, 1 ; CHECK-V-NEXT: bnez a2, .LBB18_14 ; CHECK-V-NEXT: .LBB18_13: # %entry -; CHECK-V-NEXT: mv s0, a1 +; CHECK-V-NEXT: mv a0, a1 ; CHECK-V-NEXT: .LBB18_14: # %entry ; CHECK-V-NEXT: bnez a3, .LBB18_16 ; CHECK-V-NEXT: # %bb.15: # %entry -; CHECK-V-NEXT: mv a0, a1 +; CHECK-V-NEXT: mv s0, a1 ; CHECK-V-NEXT: .LBB18_16: # %entry ; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: vmv.s.x v9, s0 +; CHECK-V-NEXT: vmv.s.x v8, s0 +; CHECK-V-NEXT: vmv.s.x v9, a0 ; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb ; CHECK-V-NEXT: slli a0, a0, 1 @@ -2370,19 +2369,19 @@ define <2 x i64> @utest_f64i64(<2 x double> %x) { ; CHECK-NOV-NEXT: .cfi_offset s0, -16 ; CHECK-NOV-NEXT: .cfi_offset s1, -24 ; CHECK-NOV-NEXT: .cfi_offset fs0, -32 -; CHECK-NOV-NEXT: fmv.d fs0, fa1 +; CHECK-NOV-NEXT: fmv.d fs0, fa0 +; CHECK-NOV-NEXT: fmv.d fa0, fa1 ; CHECK-NOV-NEXT: call __fixunsdfti ; CHECK-NOV-NEXT: mv s0, a0 ; CHECK-NOV-NEXT: mv s1, a1 ; CHECK-NOV-NEXT: fmv.d fa0, fs0 ; CHECK-NOV-NEXT: call __fixunsdfti -; CHECK-NOV-NEXT: snez a1, a1 ; CHECK-NOV-NEXT: snez a2, s1 -; CHECK-NOV-NEXT: addi a2, a2, -1 -; CHECK-NOV-NEXT: and a2, a2, s0 +; CHECK-NOV-NEXT: snez a1, a1 ; CHECK-NOV-NEXT: addi a1, a1, -1 -; CHECK-NOV-NEXT: and a1, a1, a0 -; CHECK-NOV-NEXT: mv a0, a2 +; CHECK-NOV-NEXT: and a0, a1, a0 +; CHECK-NOV-NEXT: addi a1, a2, -1 +; CHECK-NOV-NEXT: and a1, a1, s0 ; CHECK-NOV-NEXT: ld ra, 24(sp) # 8-byte Folded Reload ; CHECK-NOV-NEXT: ld s0, 16(sp) # 8-byte Folded Reload ; CHECK-NOV-NEXT: ld s1, 8(sp) # 8-byte Folded Reload @@ -2407,25 +2406,25 @@ define <2 x i64> @utest_f64i64(<2 x double> %x) { ; CHECK-V-NEXT: addi a0, sp, 32 ; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma -; CHECK-V-NEXT: vslidedown.vi v9, v8, 1 -; CHECK-V-NEXT: vfmv.f.s fa0, v9 +; CHECK-V-NEXT: vfmv.f.s fa0, v8 ; CHECK-V-NEXT: call __fixunsdfti ; CHECK-V-NEXT: mv s0, a0 ; CHECK-V-NEXT: mv s1, a1 ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: addi a0, sp, 32 ; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslidedown.vi v8, v8, 1 ; CHECK-V-NEXT: vfmv.f.s fa0, v8 ; CHECK-V-NEXT: call __fixunsdfti -; CHECK-V-NEXT: snez a1, a1 ; CHECK-V-NEXT: snez a2, s1 -; CHECK-V-NEXT: addi a2, a2, -1 -; CHECK-V-NEXT: and a2, a2, s0 +; CHECK-V-NEXT: snez a1, a1 ; CHECK-V-NEXT: addi a1, a1, -1 ; CHECK-V-NEXT: and a0, a1, a0 +; CHECK-V-NEXT: addi a2, a2, -1 +; CHECK-V-NEXT: and a2, a2, s0 ; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: vmv.s.x v9, a2 +; CHECK-V-NEXT: vmv.s.x v8, a2 +; CHECK-V-NEXT: vmv.s.x v9, a0 ; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb ; CHECK-V-NEXT: slli a0, a0, 1 @@ -2467,32 +2466,32 @@ define <2 x i64> @ustest_f64i64(<2 x double> %x) { ; CHECK-NOV-NEXT: # %bb.1: # %entry ; CHECK-NOV-NEXT: li a2, 1 ; CHECK-NOV-NEXT: .LBB20_2: # %entry -; CHECK-NOV-NEXT: slti a3, a1, 1 ; CHECK-NOV-NEXT: slti a4, s1, 1 +; CHECK-NOV-NEXT: slti a3, a1, 1 ; CHECK-NOV-NEXT: blez a1, .LBB20_4 ; CHECK-NOV-NEXT: # %bb.3: # %entry ; CHECK-NOV-NEXT: li a1, 1 ; CHECK-NOV-NEXT: .LBB20_4: # %entry -; CHECK-NOV-NEXT: neg a4, a4 ; CHECK-NOV-NEXT: neg a3, a3 ; CHECK-NOV-NEXT: and a3, a3, a0 +; CHECK-NOV-NEXT: neg a0, a4 ; CHECK-NOV-NEXT: beqz a1, .LBB20_7 ; CHECK-NOV-NEXT: # %bb.5: # %entry ; CHECK-NOV-NEXT: sgtz a1, a1 -; CHECK-NOV-NEXT: and a4, a4, s0 +; CHECK-NOV-NEXT: and a0, a0, s0 ; CHECK-NOV-NEXT: bnez a2, .LBB20_8 ; CHECK-NOV-NEXT: .LBB20_6: -; CHECK-NOV-NEXT: snez a0, a4 +; CHECK-NOV-NEXT: snez a2, a0 ; CHECK-NOV-NEXT: j .LBB20_9 ; CHECK-NOV-NEXT: .LBB20_7: ; CHECK-NOV-NEXT: snez a1, a3 -; CHECK-NOV-NEXT: and a4, a4, s0 +; CHECK-NOV-NEXT: and a0, a0, s0 ; CHECK-NOV-NEXT: beqz a2, .LBB20_6 ; CHECK-NOV-NEXT: .LBB20_8: # %entry -; CHECK-NOV-NEXT: sgtz a0, a2 +; CHECK-NOV-NEXT: sgtz a2, a2 ; CHECK-NOV-NEXT: .LBB20_9: # %entry -; CHECK-NOV-NEXT: neg a0, a0 -; CHECK-NOV-NEXT: and a0, a0, a4 +; CHECK-NOV-NEXT: neg a2, a2 +; CHECK-NOV-NEXT: and a0, a2, a0 ; CHECK-NOV-NEXT: neg a1, a1 ; CHECK-NOV-NEXT: and a1, a1, a3 ; CHECK-NOV-NEXT: ld ra, 24(sp) # 8-byte Folded Reload @@ -2534,15 +2533,15 @@ define <2 x i64> @ustest_f64i64(<2 x double> %x) { ; CHECK-V-NEXT: # %bb.1: # %entry ; CHECK-V-NEXT: li a2, 1 ; CHECK-V-NEXT: .LBB20_2: # %entry -; CHECK-V-NEXT: slti a4, a1, 1 ; CHECK-V-NEXT: slti a3, s1, 1 +; CHECK-V-NEXT: slti a4, a1, 1 ; CHECK-V-NEXT: blez a1, .LBB20_4 ; CHECK-V-NEXT: # %bb.3: # %entry ; CHECK-V-NEXT: li a1, 1 ; CHECK-V-NEXT: .LBB20_4: # %entry -; CHECK-V-NEXT: neg a3, a3 ; CHECK-V-NEXT: neg a4, a4 ; CHECK-V-NEXT: and a0, a4, a0 +; CHECK-V-NEXT: neg a3, a3 ; CHECK-V-NEXT: beqz a1, .LBB20_7 ; CHECK-V-NEXT: # %bb.5: # %entry ; CHECK-V-NEXT: sgtz a1, a1 @@ -2597,66 +2596,65 @@ define <2 x i64> @stest_f32i64(<2 x float> %x) { ; CHECK-NOV-NEXT: .cfi_offset s0, -16 ; CHECK-NOV-NEXT: .cfi_offset s1, -24 ; CHECK-NOV-NEXT: .cfi_offset fs0, -32 -; CHECK-NOV-NEXT: fmv.s fs0, fa1 +; CHECK-NOV-NEXT: fmv.s fs0, fa0 +; CHECK-NOV-NEXT: fmv.s fa0, fa1 ; CHECK-NOV-NEXT: call __fixsfti ; CHECK-NOV-NEXT: mv s0, a0 ; CHECK-NOV-NEXT: mv s1, a1 ; CHECK-NOV-NEXT: fmv.s fa0, fs0 ; CHECK-NOV-NEXT: call __fixsfti -; CHECK-NOV-NEXT: mv a2, a0 -; CHECK-NOV-NEXT: li a0, -1 -; CHECK-NOV-NEXT: srli a3, a0, 1 -; CHECK-NOV-NEXT: beqz a1, .LBB21_3 +; CHECK-NOV-NEXT: li a2, -1 +; CHECK-NOV-NEXT: srli a3, a2, 1 +; CHECK-NOV-NEXT: beqz s1, .LBB21_3 ; CHECK-NOV-NEXT: # %bb.1: # %entry -; CHECK-NOV-NEXT: slti a4, a1, 0 -; CHECK-NOV-NEXT: bnez s1, .LBB21_4 +; CHECK-NOV-NEXT: slti a4, s1, 0 +; CHECK-NOV-NEXT: bnez a1, .LBB21_4 ; CHECK-NOV-NEXT: .LBB21_2: -; CHECK-NOV-NEXT: sltu a5, s0, a3 +; CHECK-NOV-NEXT: sltu a5, a0, a3 ; CHECK-NOV-NEXT: beqz a5, .LBB21_5 ; CHECK-NOV-NEXT: j .LBB21_6 ; CHECK-NOV-NEXT: .LBB21_3: -; CHECK-NOV-NEXT: sltu a4, a2, a3 -; CHECK-NOV-NEXT: beqz s1, .LBB21_2 +; CHECK-NOV-NEXT: sltu a4, s0, a3 +; CHECK-NOV-NEXT: beqz a1, .LBB21_2 ; CHECK-NOV-NEXT: .LBB21_4: # %entry -; CHECK-NOV-NEXT: slti a5, s1, 0 +; CHECK-NOV-NEXT: slti a5, a1, 0 ; CHECK-NOV-NEXT: bnez a5, .LBB21_6 ; CHECK-NOV-NEXT: .LBB21_5: # %entry -; CHECK-NOV-NEXT: mv s0, a3 +; CHECK-NOV-NEXT: mv a0, a3 ; CHECK-NOV-NEXT: .LBB21_6: # %entry ; CHECK-NOV-NEXT: neg a6, a5 ; CHECK-NOV-NEXT: neg a5, a4 -; CHECK-NOV-NEXT: and a5, a5, a1 +; CHECK-NOV-NEXT: and a5, a5, s1 ; CHECK-NOV-NEXT: bnez a4, .LBB21_8 ; CHECK-NOV-NEXT: # %bb.7: # %entry -; CHECK-NOV-NEXT: mv a2, a3 +; CHECK-NOV-NEXT: mv s0, a3 ; CHECK-NOV-NEXT: .LBB21_8: # %entry -; CHECK-NOV-NEXT: and a4, a6, s1 -; CHECK-NOV-NEXT: slli a1, a0, 63 -; CHECK-NOV-NEXT: beq a5, a0, .LBB21_11 +; CHECK-NOV-NEXT: and a4, a6, a1 +; CHECK-NOV-NEXT: slli a1, a2, 63 +; CHECK-NOV-NEXT: beq a5, a2, .LBB21_11 ; CHECK-NOV-NEXT: # %bb.9: # %entry ; CHECK-NOV-NEXT: slti a3, a5, 0 ; CHECK-NOV-NEXT: xori a3, a3, 1 -; CHECK-NOV-NEXT: bne a4, a0, .LBB21_12 +; CHECK-NOV-NEXT: bne a4, a2, .LBB21_12 ; CHECK-NOV-NEXT: .LBB21_10: -; CHECK-NOV-NEXT: sltu a0, a1, s0 -; CHECK-NOV-NEXT: beqz a0, .LBB21_13 +; CHECK-NOV-NEXT: sltu a2, a1, a0 +; CHECK-NOV-NEXT: beqz a2, .LBB21_13 ; CHECK-NOV-NEXT: j .LBB21_14 ; CHECK-NOV-NEXT: .LBB21_11: -; CHECK-NOV-NEXT: sltu a3, a1, a2 -; CHECK-NOV-NEXT: beq a4, a0, .LBB21_10 +; CHECK-NOV-NEXT: sltu a3, a1, s0 +; CHECK-NOV-NEXT: beq a4, a2, .LBB21_10 ; CHECK-NOV-NEXT: .LBB21_12: # %entry -; CHECK-NOV-NEXT: slti a0, a4, 0 -; CHECK-NOV-NEXT: xori a0, a0, 1 -; CHECK-NOV-NEXT: bnez a0, .LBB21_14 +; CHECK-NOV-NEXT: slti a2, a4, 0 +; CHECK-NOV-NEXT: xori a2, a2, 1 +; CHECK-NOV-NEXT: bnez a2, .LBB21_14 ; CHECK-NOV-NEXT: .LBB21_13: # %entry -; CHECK-NOV-NEXT: mv s0, a1 +; CHECK-NOV-NEXT: mv a0, a1 ; CHECK-NOV-NEXT: .LBB21_14: # %entry ; CHECK-NOV-NEXT: bnez a3, .LBB21_16 ; CHECK-NOV-NEXT: # %bb.15: # %entry -; CHECK-NOV-NEXT: mv a2, a1 +; CHECK-NOV-NEXT: mv s0, a1 ; CHECK-NOV-NEXT: .LBB21_16: # %entry -; CHECK-NOV-NEXT: mv a0, s0 -; CHECK-NOV-NEXT: mv a1, a2 +; CHECK-NOV-NEXT: mv a1, s0 ; CHECK-NOV-NEXT: ld ra, 24(sp) # 8-byte Folded Reload ; CHECK-NOV-NEXT: ld s0, 16(sp) # 8-byte Folded Reload ; CHECK-NOV-NEXT: ld s1, 8(sp) # 8-byte Folded Reload @@ -2681,43 +2679,43 @@ define <2 x i64> @stest_f32i64(<2 x float> %x) { ; CHECK-V-NEXT: addi a0, sp, 32 ; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: vsetivli zero, 1, e32, mf2, ta, ma -; CHECK-V-NEXT: vslidedown.vi v9, v8, 1 -; CHECK-V-NEXT: vfmv.f.s fa0, v9 +; CHECK-V-NEXT: vfmv.f.s fa0, v8 ; CHECK-V-NEXT: call __fixsfti ; CHECK-V-NEXT: mv s0, a0 ; CHECK-V-NEXT: mv s1, a1 ; CHECK-V-NEXT: vsetivli zero, 1, e32, mf2, ta, ma ; CHECK-V-NEXT: addi a0, sp, 32 ; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslidedown.vi v8, v8, 1 ; CHECK-V-NEXT: vfmv.f.s fa0, v8 ; CHECK-V-NEXT: call __fixsfti ; CHECK-V-NEXT: li a2, -1 ; CHECK-V-NEXT: srli a3, a2, 1 -; CHECK-V-NEXT: beqz a1, .LBB21_3 +; CHECK-V-NEXT: beqz s1, .LBB21_3 ; CHECK-V-NEXT: # %bb.1: # %entry -; CHECK-V-NEXT: slti a4, a1, 0 -; CHECK-V-NEXT: bnez s1, .LBB21_4 +; CHECK-V-NEXT: slti a4, s1, 0 +; CHECK-V-NEXT: bnez a1, .LBB21_4 ; CHECK-V-NEXT: .LBB21_2: -; CHECK-V-NEXT: sltu a5, s0, a3 +; CHECK-V-NEXT: sltu a5, a0, a3 ; CHECK-V-NEXT: beqz a5, .LBB21_5 ; CHECK-V-NEXT: j .LBB21_6 ; CHECK-V-NEXT: .LBB21_3: -; CHECK-V-NEXT: sltu a4, a0, a3 -; CHECK-V-NEXT: beqz s1, .LBB21_2 +; CHECK-V-NEXT: sltu a4, s0, a3 +; CHECK-V-NEXT: beqz a1, .LBB21_2 ; CHECK-V-NEXT: .LBB21_4: # %entry -; CHECK-V-NEXT: slti a5, s1, 0 +; CHECK-V-NEXT: slti a5, a1, 0 ; CHECK-V-NEXT: bnez a5, .LBB21_6 ; CHECK-V-NEXT: .LBB21_5: # %entry -; CHECK-V-NEXT: mv s0, a3 +; CHECK-V-NEXT: mv a0, a3 ; CHECK-V-NEXT: .LBB21_6: # %entry ; CHECK-V-NEXT: neg a6, a5 ; CHECK-V-NEXT: neg a5, a4 -; CHECK-V-NEXT: and a5, a5, a1 +; CHECK-V-NEXT: and a5, a5, s1 ; CHECK-V-NEXT: bnez a4, .LBB21_8 ; CHECK-V-NEXT: # %bb.7: # %entry -; CHECK-V-NEXT: mv a0, a3 +; CHECK-V-NEXT: mv s0, a3 ; CHECK-V-NEXT: .LBB21_8: # %entry -; CHECK-V-NEXT: and a4, a6, s1 +; CHECK-V-NEXT: and a4, a6, a1 ; CHECK-V-NEXT: slli a1, a2, 63 ; CHECK-V-NEXT: beq a5, a2, .LBB21_11 ; CHECK-V-NEXT: # %bb.9: # %entry @@ -2725,26 +2723,26 @@ define <2 x i64> @stest_f32i64(<2 x float> %x) { ; CHECK-V-NEXT: xori a3, a3, 1 ; CHECK-V-NEXT: bne a4, a2, .LBB21_12 ; CHECK-V-NEXT: .LBB21_10: -; CHECK-V-NEXT: sltu a2, a1, s0 +; CHECK-V-NEXT: sltu a2, a1, a0 ; CHECK-V-NEXT: beqz a2, .LBB21_13 ; CHECK-V-NEXT: j .LBB21_14 ; CHECK-V-NEXT: .LBB21_11: -; CHECK-V-NEXT: sltu a3, a1, a0 +; CHECK-V-NEXT: sltu a3, a1, s0 ; CHECK-V-NEXT: beq a4, a2, .LBB21_10 ; CHECK-V-NEXT: .LBB21_12: # %entry ; CHECK-V-NEXT: slti a2, a4, 0 ; CHECK-V-NEXT: xori a2, a2, 1 ; CHECK-V-NEXT: bnez a2, .LBB21_14 ; CHECK-V-NEXT: .LBB21_13: # %entry -; CHECK-V-NEXT: mv s0, a1 +; CHECK-V-NEXT: mv a0, a1 ; CHECK-V-NEXT: .LBB21_14: # %entry ; CHECK-V-NEXT: bnez a3, .LBB21_16 ; CHECK-V-NEXT: # %bb.15: # %entry -; CHECK-V-NEXT: mv a0, a1 +; CHECK-V-NEXT: mv s0, a1 ; CHECK-V-NEXT: .LBB21_16: # %entry ; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: vmv.s.x v9, s0 +; CHECK-V-NEXT: vmv.s.x v8, s0 +; CHECK-V-NEXT: vmv.s.x v9, a0 ; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb ; CHECK-V-NEXT: slli a0, a0, 1 @@ -2777,19 +2775,19 @@ define <2 x i64> @utest_f32i64(<2 x float> %x) { ; CHECK-NOV-NEXT: .cfi_offset s0, -16 ; CHECK-NOV-NEXT: .cfi_offset s1, -24 ; CHECK-NOV-NEXT: .cfi_offset fs0, -32 -; CHECK-NOV-NEXT: fmv.s fs0, fa1 +; CHECK-NOV-NEXT: fmv.s fs0, fa0 +; CHECK-NOV-NEXT: fmv.s fa0, fa1 ; CHECK-NOV-NEXT: call __fixunssfti ; CHECK-NOV-NEXT: mv s0, a0 ; CHECK-NOV-NEXT: mv s1, a1 ; CHECK-NOV-NEXT: fmv.s fa0, fs0 ; CHECK-NOV-NEXT: call __fixunssfti -; CHECK-NOV-NEXT: snez a1, a1 ; CHECK-NOV-NEXT: snez a2, s1 -; CHECK-NOV-NEXT: addi a2, a2, -1 -; CHECK-NOV-NEXT: and a2, a2, s0 +; CHECK-NOV-NEXT: snez a1, a1 ; CHECK-NOV-NEXT: addi a1, a1, -1 -; CHECK-NOV-NEXT: and a1, a1, a0 -; CHECK-NOV-NEXT: mv a0, a2 +; CHECK-NOV-NEXT: and a0, a1, a0 +; CHECK-NOV-NEXT: addi a1, a2, -1 +; CHECK-NOV-NEXT: and a1, a1, s0 ; CHECK-NOV-NEXT: ld ra, 24(sp) # 8-byte Folded Reload ; CHECK-NOV-NEXT: ld s0, 16(sp) # 8-byte Folded Reload ; CHECK-NOV-NEXT: ld s1, 8(sp) # 8-byte Folded Reload @@ -2814,25 +2812,25 @@ define <2 x i64> @utest_f32i64(<2 x float> %x) { ; CHECK-V-NEXT: addi a0, sp, 32 ; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: vsetivli zero, 1, e32, mf2, ta, ma -; CHECK-V-NEXT: vslidedown.vi v9, v8, 1 -; CHECK-V-NEXT: vfmv.f.s fa0, v9 +; CHECK-V-NEXT: vfmv.f.s fa0, v8 ; CHECK-V-NEXT: call __fixunssfti ; CHECK-V-NEXT: mv s0, a0 ; CHECK-V-NEXT: mv s1, a1 ; CHECK-V-NEXT: vsetivli zero, 1, e32, mf2, ta, ma ; CHECK-V-NEXT: addi a0, sp, 32 ; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslidedown.vi v8, v8, 1 ; CHECK-V-NEXT: vfmv.f.s fa0, v8 ; CHECK-V-NEXT: call __fixunssfti -; CHECK-V-NEXT: snez a1, a1 ; CHECK-V-NEXT: snez a2, s1 -; CHECK-V-NEXT: addi a2, a2, -1 -; CHECK-V-NEXT: and a2, a2, s0 +; CHECK-V-NEXT: snez a1, a1 ; CHECK-V-NEXT: addi a1, a1, -1 ; CHECK-V-NEXT: and a0, a1, a0 +; CHECK-V-NEXT: addi a2, a2, -1 +; CHECK-V-NEXT: and a2, a2, s0 ; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: vmv.s.x v9, a2 +; CHECK-V-NEXT: vmv.s.x v8, a2 +; CHECK-V-NEXT: vmv.s.x v9, a0 ; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb ; CHECK-V-NEXT: slli a0, a0, 1 @@ -2874,32 +2872,32 @@ define <2 x i64> @ustest_f32i64(<2 x float> %x) { ; CHECK-NOV-NEXT: # %bb.1: # %entry ; CHECK-NOV-NEXT: li a2, 1 ; CHECK-NOV-NEXT: .LBB23_2: # %entry -; CHECK-NOV-NEXT: slti a3, a1, 1 ; CHECK-NOV-NEXT: slti a4, s1, 1 +; CHECK-NOV-NEXT: slti a3, a1, 1 ; CHECK-NOV-NEXT: blez a1, .LBB23_4 ; CHECK-NOV-NEXT: # %bb.3: # %entry ; CHECK-NOV-NEXT: li a1, 1 ; CHECK-NOV-NEXT: .LBB23_4: # %entry -; CHECK-NOV-NEXT: neg a4, a4 ; CHECK-NOV-NEXT: neg a3, a3 ; CHECK-NOV-NEXT: and a3, a3, a0 +; CHECK-NOV-NEXT: neg a0, a4 ; CHECK-NOV-NEXT: beqz a1, .LBB23_7 ; CHECK-NOV-NEXT: # %bb.5: # %entry ; CHECK-NOV-NEXT: sgtz a1, a1 -; CHECK-NOV-NEXT: and a4, a4, s0 +; CHECK-NOV-NEXT: and a0, a0, s0 ; CHECK-NOV-NEXT: bnez a2, .LBB23_8 ; CHECK-NOV-NEXT: .LBB23_6: -; CHECK-NOV-NEXT: snez a0, a4 +; CHECK-NOV-NEXT: snez a2, a0 ; CHECK-NOV-NEXT: j .LBB23_9 ; CHECK-NOV-NEXT: .LBB23_7: ; CHECK-NOV-NEXT: snez a1, a3 -; CHECK-NOV-NEXT: and a4, a4, s0 +; CHECK-NOV-NEXT: and a0, a0, s0 ; CHECK-NOV-NEXT: beqz a2, .LBB23_6 ; CHECK-NOV-NEXT: .LBB23_8: # %entry -; CHECK-NOV-NEXT: sgtz a0, a2 +; CHECK-NOV-NEXT: sgtz a2, a2 ; CHECK-NOV-NEXT: .LBB23_9: # %entry -; CHECK-NOV-NEXT: neg a0, a0 -; CHECK-NOV-NEXT: and a0, a0, a4 +; CHECK-NOV-NEXT: neg a2, a2 +; CHECK-NOV-NEXT: and a0, a2, a0 ; CHECK-NOV-NEXT: neg a1, a1 ; CHECK-NOV-NEXT: and a1, a1, a3 ; CHECK-NOV-NEXT: ld ra, 24(sp) # 8-byte Folded Reload @@ -2941,15 +2939,15 @@ define <2 x i64> @ustest_f32i64(<2 x float> %x) { ; CHECK-V-NEXT: # %bb.1: # %entry ; CHECK-V-NEXT: li a2, 1 ; CHECK-V-NEXT: .LBB23_2: # %entry -; CHECK-V-NEXT: slti a4, a1, 1 ; CHECK-V-NEXT: slti a3, s1, 1 +; CHECK-V-NEXT: slti a4, a1, 1 ; CHECK-V-NEXT: blez a1, .LBB23_4 ; CHECK-V-NEXT: # %bb.3: # %entry ; CHECK-V-NEXT: li a1, 1 ; CHECK-V-NEXT: .LBB23_4: # %entry -; CHECK-V-NEXT: neg a3, a3 ; CHECK-V-NEXT: neg a4, a4 ; CHECK-V-NEXT: and a0, a4, a0 +; CHECK-V-NEXT: neg a3, a3 ; CHECK-V-NEXT: beqz a1, .LBB23_7 ; CHECK-V-NEXT: # %bb.5: # %entry ; CHECK-V-NEXT: sgtz a1, a1 @@ -3004,8 +3002,8 @@ define <2 x i64> @stest_f16i64(<2 x half> %x) { ; CHECK-NOV-NEXT: .cfi_offset s0, -16 ; CHECK-NOV-NEXT: .cfi_offset s1, -24 ; CHECK-NOV-NEXT: .cfi_offset s2, -32 -; CHECK-NOV-NEXT: mv s2, a1 -; CHECK-NOV-NEXT: fmv.w.x fa0, a0 +; CHECK-NOV-NEXT: mv s2, a0 +; CHECK-NOV-NEXT: fmv.w.x fa0, a1 ; CHECK-NOV-NEXT: call __extendhfsf2 ; CHECK-NOV-NEXT: call __fixsfti ; CHECK-NOV-NEXT: mv s0, a0 @@ -3013,60 +3011,58 @@ define <2 x i64> @stest_f16i64(<2 x half> %x) { ; CHECK-NOV-NEXT: fmv.w.x fa0, s2 ; CHECK-NOV-NEXT: call __extendhfsf2 ; CHECK-NOV-NEXT: call __fixsfti -; CHECK-NOV-NEXT: mv a2, a0 -; CHECK-NOV-NEXT: li a0, -1 -; CHECK-NOV-NEXT: srli a3, a0, 1 -; CHECK-NOV-NEXT: beqz a1, .LBB24_3 +; CHECK-NOV-NEXT: li a2, -1 +; CHECK-NOV-NEXT: srli a3, a2, 1 +; CHECK-NOV-NEXT: beqz s1, .LBB24_3 ; CHECK-NOV-NEXT: # %bb.1: # %entry -; CHECK-NOV-NEXT: slti a4, a1, 0 -; CHECK-NOV-NEXT: bnez s1, .LBB24_4 +; CHECK-NOV-NEXT: slti a4, s1, 0 +; CHECK-NOV-NEXT: bnez a1, .LBB24_4 ; CHECK-NOV-NEXT: .LBB24_2: -; CHECK-NOV-NEXT: sltu a5, s0, a3 +; CHECK-NOV-NEXT: sltu a5, a0, a3 ; CHECK-NOV-NEXT: beqz a5, .LBB24_5 ; CHECK-NOV-NEXT: j .LBB24_6 ; CHECK-NOV-NEXT: .LBB24_3: -; CHECK-NOV-NEXT: sltu a4, a2, a3 -; CHECK-NOV-NEXT: beqz s1, .LBB24_2 +; CHECK-NOV-NEXT: sltu a4, s0, a3 +; CHECK-NOV-NEXT: beqz a1, .LBB24_2 ; CHECK-NOV-NEXT: .LBB24_4: # %entry -; CHECK-NOV-NEXT: slti a5, s1, 0 +; CHECK-NOV-NEXT: slti a5, a1, 0 ; CHECK-NOV-NEXT: bnez a5, .LBB24_6 ; CHECK-NOV-NEXT: .LBB24_5: # %entry -; CHECK-NOV-NEXT: mv s0, a3 +; CHECK-NOV-NEXT: mv a0, a3 ; CHECK-NOV-NEXT: .LBB24_6: # %entry ; CHECK-NOV-NEXT: neg a6, a5 ; CHECK-NOV-NEXT: neg a5, a4 -; CHECK-NOV-NEXT: and a5, a5, a1 +; CHECK-NOV-NEXT: and a5, a5, s1 ; CHECK-NOV-NEXT: bnez a4, .LBB24_8 ; CHECK-NOV-NEXT: # %bb.7: # %entry -; CHECK-NOV-NEXT: mv a2, a3 +; CHECK-NOV-NEXT: mv s0, a3 ; CHECK-NOV-NEXT: .LBB24_8: # %entry -; CHECK-NOV-NEXT: and a4, a6, s1 -; CHECK-NOV-NEXT: slli a1, a0, 63 -; CHECK-NOV-NEXT: beq a5, a0, .LBB24_11 +; CHECK-NOV-NEXT: and a4, a6, a1 +; CHECK-NOV-NEXT: slli a1, a2, 63 +; CHECK-NOV-NEXT: beq a5, a2, .LBB24_11 ; CHECK-NOV-NEXT: # %bb.9: # %entry ; CHECK-NOV-NEXT: slti a3, a5, 0 ; CHECK-NOV-NEXT: xori a3, a3, 1 -; CHECK-NOV-NEXT: bne a4, a0, .LBB24_12 +; CHECK-NOV-NEXT: bne a4, a2, .LBB24_12 ; CHECK-NOV-NEXT: .LBB24_10: -; CHECK-NOV-NEXT: sltu a0, a1, s0 -; CHECK-NOV-NEXT: beqz a0, .LBB24_13 +; CHECK-NOV-NEXT: sltu a2, a1, a0 +; CHECK-NOV-NEXT: beqz a2, .LBB24_13 ; CHECK-NOV-NEXT: j .LBB24_14 ; CHECK-NOV-NEXT: .LBB24_11: -; CHECK-NOV-NEXT: sltu a3, a1, a2 -; CHECK-NOV-NEXT: beq a4, a0, .LBB24_10 +; CHECK-NOV-NEXT: sltu a3, a1, s0 +; CHECK-NOV-NEXT: beq a4, a2, .LBB24_10 ; CHECK-NOV-NEXT: .LBB24_12: # %entry -; CHECK-NOV-NEXT: slti a0, a4, 0 -; CHECK-NOV-NEXT: xori a0, a0, 1 -; CHECK-NOV-NEXT: bnez a0, .LBB24_14 +; CHECK-NOV-NEXT: slti a2, a4, 0 +; CHECK-NOV-NEXT: xori a2, a2, 1 +; CHECK-NOV-NEXT: bnez a2, .LBB24_14 ; CHECK-NOV-NEXT: .LBB24_13: # %entry -; CHECK-NOV-NEXT: mv s0, a1 +; CHECK-NOV-NEXT: mv a0, a1 ; CHECK-NOV-NEXT: .LBB24_14: # %entry ; CHECK-NOV-NEXT: bnez a3, .LBB24_16 ; CHECK-NOV-NEXT: # %bb.15: # %entry -; CHECK-NOV-NEXT: mv a2, a1 +; CHECK-NOV-NEXT: mv s0, a1 ; CHECK-NOV-NEXT: .LBB24_16: # %entry -; CHECK-NOV-NEXT: mv a0, s0 -; CHECK-NOV-NEXT: mv a1, a2 +; CHECK-NOV-NEXT: mv a1, s0 ; CHECK-NOV-NEXT: ld ra, 24(sp) # 8-byte Folded Reload ; CHECK-NOV-NEXT: ld s0, 16(sp) # 8-byte Folded Reload ; CHECK-NOV-NEXT: ld s1, 8(sp) # 8-byte Folded Reload @@ -3086,8 +3082,8 @@ define <2 x i64> @stest_f16i64(<2 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s0, -16 ; CHECK-V-NEXT: .cfi_offset s1, -24 ; CHECK-V-NEXT: .cfi_offset s2, -32 -; CHECK-V-NEXT: mv s2, a1 -; CHECK-V-NEXT: fmv.w.x fa0, a0 +; CHECK-V-NEXT: mv s2, a0 +; CHECK-V-NEXT: fmv.w.x fa0, a1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: call __fixsfti ; CHECK-V-NEXT: mv s0, a0 @@ -3097,31 +3093,31 @@ define <2 x i64> @stest_f16i64(<2 x half> %x) { ; CHECK-V-NEXT: call __fixsfti ; CHECK-V-NEXT: li a2, -1 ; CHECK-V-NEXT: srli a3, a2, 1 -; CHECK-V-NEXT: beqz a1, .LBB24_3 +; CHECK-V-NEXT: beqz s1, .LBB24_3 ; CHECK-V-NEXT: # %bb.1: # %entry -; CHECK-V-NEXT: slti a4, a1, 0 -; CHECK-V-NEXT: bnez s1, .LBB24_4 +; CHECK-V-NEXT: slti a4, s1, 0 +; CHECK-V-NEXT: bnez a1, .LBB24_4 ; CHECK-V-NEXT: .LBB24_2: -; CHECK-V-NEXT: sltu a5, s0, a3 +; CHECK-V-NEXT: sltu a5, a0, a3 ; CHECK-V-NEXT: beqz a5, .LBB24_5 ; CHECK-V-NEXT: j .LBB24_6 ; CHECK-V-NEXT: .LBB24_3: -; CHECK-V-NEXT: sltu a4, a0, a3 -; CHECK-V-NEXT: beqz s1, .LBB24_2 +; CHECK-V-NEXT: sltu a4, s0, a3 +; CHECK-V-NEXT: beqz a1, .LBB24_2 ; CHECK-V-NEXT: .LBB24_4: # %entry -; CHECK-V-NEXT: slti a5, s1, 0 +; CHECK-V-NEXT: slti a5, a1, 0 ; CHECK-V-NEXT: bnez a5, .LBB24_6 ; CHECK-V-NEXT: .LBB24_5: # %entry -; CHECK-V-NEXT: mv s0, a3 +; CHECK-V-NEXT: mv a0, a3 ; CHECK-V-NEXT: .LBB24_6: # %entry ; CHECK-V-NEXT: neg a6, a5 ; CHECK-V-NEXT: neg a5, a4 -; CHECK-V-NEXT: and a5, a5, a1 +; CHECK-V-NEXT: and a5, a5, s1 ; CHECK-V-NEXT: bnez a4, .LBB24_8 ; CHECK-V-NEXT: # %bb.7: # %entry -; CHECK-V-NEXT: mv a0, a3 +; CHECK-V-NEXT: mv s0, a3 ; CHECK-V-NEXT: .LBB24_8: # %entry -; CHECK-V-NEXT: and a4, a6, s1 +; CHECK-V-NEXT: and a4, a6, a1 ; CHECK-V-NEXT: slli a1, a2, 63 ; CHECK-V-NEXT: beq a5, a2, .LBB24_11 ; CHECK-V-NEXT: # %bb.9: # %entry @@ -3129,26 +3125,26 @@ define <2 x i64> @stest_f16i64(<2 x half> %x) { ; CHECK-V-NEXT: xori a3, a3, 1 ; CHECK-V-NEXT: bne a4, a2, .LBB24_12 ; CHECK-V-NEXT: .LBB24_10: -; CHECK-V-NEXT: sltu a2, a1, s0 +; CHECK-V-NEXT: sltu a2, a1, a0 ; CHECK-V-NEXT: beqz a2, .LBB24_13 ; CHECK-V-NEXT: j .LBB24_14 ; CHECK-V-NEXT: .LBB24_11: -; CHECK-V-NEXT: sltu a3, a1, a0 +; CHECK-V-NEXT: sltu a3, a1, s0 ; CHECK-V-NEXT: beq a4, a2, .LBB24_10 ; CHECK-V-NEXT: .LBB24_12: # %entry ; CHECK-V-NEXT: slti a2, a4, 0 ; CHECK-V-NEXT: xori a2, a2, 1 ; CHECK-V-NEXT: bnez a2, .LBB24_14 ; CHECK-V-NEXT: .LBB24_13: # %entry -; CHECK-V-NEXT: mv s0, a1 +; CHECK-V-NEXT: mv a0, a1 ; CHECK-V-NEXT: .LBB24_14: # %entry ; CHECK-V-NEXT: bnez a3, .LBB24_16 ; CHECK-V-NEXT: # %bb.15: # %entry -; CHECK-V-NEXT: mv a0, a1 +; CHECK-V-NEXT: mv s0, a1 ; CHECK-V-NEXT: .LBB24_16: # %entry ; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v9, a0 -; CHECK-V-NEXT: vmv.s.x v8, s0 +; CHECK-V-NEXT: vmv.s.x v9, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: ld ra, 24(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 16(sp) # 8-byte Folded Reload @@ -3179,8 +3175,8 @@ define <2 x i64> @utesth_f16i64(<2 x half> %x) { ; CHECK-NOV-NEXT: .cfi_offset s0, -16 ; CHECK-NOV-NEXT: .cfi_offset s1, -24 ; CHECK-NOV-NEXT: .cfi_offset s2, -32 -; CHECK-NOV-NEXT: mv s0, a1 -; CHECK-NOV-NEXT: fmv.w.x fa0, a0 +; CHECK-NOV-NEXT: mv s0, a0 +; CHECK-NOV-NEXT: fmv.w.x fa0, a1 ; CHECK-NOV-NEXT: call __extendhfsf2 ; CHECK-NOV-NEXT: call __fixunssfti ; CHECK-NOV-NEXT: mv s1, a0 @@ -3188,13 +3184,12 @@ define <2 x i64> @utesth_f16i64(<2 x half> %x) { ; CHECK-NOV-NEXT: fmv.w.x fa0, s0 ; CHECK-NOV-NEXT: call __extendhfsf2 ; CHECK-NOV-NEXT: call __fixunssfti -; CHECK-NOV-NEXT: snez a1, a1 ; CHECK-NOV-NEXT: snez a2, s2 -; CHECK-NOV-NEXT: addi a2, a2, -1 -; CHECK-NOV-NEXT: and a2, a2, s1 +; CHECK-NOV-NEXT: snez a1, a1 ; CHECK-NOV-NEXT: addi a1, a1, -1 -; CHECK-NOV-NEXT: and a1, a1, a0 -; CHECK-NOV-NEXT: mv a0, a2 +; CHECK-NOV-NEXT: and a0, a1, a0 +; CHECK-NOV-NEXT: addi a1, a2, -1 +; CHECK-NOV-NEXT: and a1, a1, s1 ; CHECK-NOV-NEXT: ld ra, 24(sp) # 8-byte Folded Reload ; CHECK-NOV-NEXT: ld s0, 16(sp) # 8-byte Folded Reload ; CHECK-NOV-NEXT: ld s1, 8(sp) # 8-byte Folded Reload @@ -3214,8 +3209,8 @@ define <2 x i64> @utesth_f16i64(<2 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s0, -16 ; CHECK-V-NEXT: .cfi_offset s1, -24 ; CHECK-V-NEXT: .cfi_offset s2, -32 -; CHECK-V-NEXT: mv s0, a1 -; CHECK-V-NEXT: fmv.w.x fa0, a0 +; CHECK-V-NEXT: mv s0, a0 +; CHECK-V-NEXT: fmv.w.x fa0, a1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: call __fixunssfti ; CHECK-V-NEXT: mv s1, a0 @@ -3223,15 +3218,15 @@ define <2 x i64> @utesth_f16i64(<2 x half> %x) { ; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: call __fixunssfti -; CHECK-V-NEXT: snez a1, a1 ; CHECK-V-NEXT: snez a2, s2 -; CHECK-V-NEXT: addi a2, a2, -1 -; CHECK-V-NEXT: and a2, a2, s1 +; CHECK-V-NEXT: snez a1, a1 ; CHECK-V-NEXT: addi a1, a1, -1 ; CHECK-V-NEXT: and a0, a1, a0 +; CHECK-V-NEXT: addi a2, a2, -1 +; CHECK-V-NEXT: and a2, a2, s1 ; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v9, a0 -; CHECK-V-NEXT: vmv.s.x v8, a2 +; CHECK-V-NEXT: vmv.s.x v9, a2 +; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: ld ra, 24(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 16(sp) # 8-byte Folded Reload @@ -3274,32 +3269,32 @@ define <2 x i64> @ustest_f16i64(<2 x half> %x) { ; CHECK-NOV-NEXT: # %bb.1: # %entry ; CHECK-NOV-NEXT: li a2, 1 ; CHECK-NOV-NEXT: .LBB26_2: # %entry -; CHECK-NOV-NEXT: slti a3, a1, 1 ; CHECK-NOV-NEXT: slti a4, s1, 1 +; CHECK-NOV-NEXT: slti a3, a1, 1 ; CHECK-NOV-NEXT: blez a1, .LBB26_4 ; CHECK-NOV-NEXT: # %bb.3: # %entry ; CHECK-NOV-NEXT: li a1, 1 ; CHECK-NOV-NEXT: .LBB26_4: # %entry -; CHECK-NOV-NEXT: neg a4, a4 ; CHECK-NOV-NEXT: neg a3, a3 ; CHECK-NOV-NEXT: and a3, a3, a0 +; CHECK-NOV-NEXT: neg a0, a4 ; CHECK-NOV-NEXT: beqz a1, .LBB26_7 ; CHECK-NOV-NEXT: # %bb.5: # %entry ; CHECK-NOV-NEXT: sgtz a1, a1 -; CHECK-NOV-NEXT: and a4, a4, s0 +; CHECK-NOV-NEXT: and a0, a0, s0 ; CHECK-NOV-NEXT: bnez a2, .LBB26_8 ; CHECK-NOV-NEXT: .LBB26_6: -; CHECK-NOV-NEXT: snez a0, a4 +; CHECK-NOV-NEXT: snez a2, a0 ; CHECK-NOV-NEXT: j .LBB26_9 ; CHECK-NOV-NEXT: .LBB26_7: ; CHECK-NOV-NEXT: snez a1, a3 -; CHECK-NOV-NEXT: and a4, a4, s0 +; CHECK-NOV-NEXT: and a0, a0, s0 ; CHECK-NOV-NEXT: beqz a2, .LBB26_6 ; CHECK-NOV-NEXT: .LBB26_8: # %entry -; CHECK-NOV-NEXT: sgtz a0, a2 +; CHECK-NOV-NEXT: sgtz a2, a2 ; CHECK-NOV-NEXT: .LBB26_9: # %entry -; CHECK-NOV-NEXT: neg a0, a0 -; CHECK-NOV-NEXT: and a0, a0, a4 +; CHECK-NOV-NEXT: neg a2, a2 +; CHECK-NOV-NEXT: and a0, a2, a0 ; CHECK-NOV-NEXT: neg a1, a1 ; CHECK-NOV-NEXT: and a1, a1, a3 ; CHECK-NOV-NEXT: ld ra, 24(sp) # 8-byte Folded Reload @@ -3335,15 +3330,15 @@ define <2 x i64> @ustest_f16i64(<2 x half> %x) { ; CHECK-V-NEXT: # %bb.1: # %entry ; CHECK-V-NEXT: li a2, 1 ; CHECK-V-NEXT: .LBB26_2: # %entry -; CHECK-V-NEXT: slti a4, a1, 1 ; CHECK-V-NEXT: slti a3, s1, 1 +; CHECK-V-NEXT: slti a4, a1, 1 ; CHECK-V-NEXT: blez a1, .LBB26_4 ; CHECK-V-NEXT: # %bb.3: # %entry ; CHECK-V-NEXT: li a1, 1 ; CHECK-V-NEXT: .LBB26_4: # %entry -; CHECK-V-NEXT: neg a3, a3 ; CHECK-V-NEXT: neg a4, a4 ; CHECK-V-NEXT: and a0, a4, a0 +; CHECK-V-NEXT: neg a3, a3 ; CHECK-V-NEXT: beqz a1, .LBB26_7 ; CHECK-V-NEXT: # %bb.5: # %entry ; CHECK-V-NEXT: sgtz a1, a1 diff --git a/llvm/test/CodeGen/RISCV/rvv/vec3-setcc-crash.ll b/llvm/test/CodeGen/RISCV/rvv/vec3-setcc-crash.ll index 81076e41a7cb..122ac13cb257 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vec3-setcc-crash.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vec3-setcc-crash.ll @@ -13,18 +13,18 @@ define void @vec3_setcc_crash(ptr %in, ptr %out) { ; RV32: # %bb.0: ; RV32-NEXT: lw a0, 0(a0) ; RV32-NEXT: srli a2, a0, 16 -; RV32-NEXT: srli a3, a0, 8 -; RV32-NEXT: slli a4, a0, 16 -; RV32-NEXT: srai a4, a4, 24 +; RV32-NEXT: slli a3, a0, 16 +; RV32-NEXT: srli a4, a3, 24 +; RV32-NEXT: srai a3, a3, 24 ; RV32-NEXT: slli a5, a0, 24 ; RV32-NEXT: srai a5, a5, 24 ; RV32-NEXT: slli a6, a0, 8 ; RV32-NEXT: srai a6, a6, 24 ; RV32-NEXT: sgtz a6, a6 ; RV32-NEXT: sgtz a5, a5 -; RV32-NEXT: sgtz a4, a4 -; RV32-NEXT: neg a4, a4 -; RV32-NEXT: and a3, a4, a3 +; RV32-NEXT: sgtz a3, a3 +; RV32-NEXT: neg a3, a3 +; RV32-NEXT: and a3, a3, a4 ; RV32-NEXT: slli a3, a3, 8 ; RV32-NEXT: neg a4, a5 ; RV32-NEXT: and a0, a4, a0 @@ -39,19 +39,19 @@ define void @vec3_setcc_crash(ptr %in, ptr %out) { ; RV64-LABEL: vec3_setcc_crash: ; RV64: # %bb.0: ; RV64-NEXT: lw a0, 0(a0) -; RV64-NEXT: srli a2, a0, 16 -; RV64-NEXT: srli a3, a0, 8 -; RV64-NEXT: slli a4, a0, 48 -; RV64-NEXT: srai a4, a4, 56 +; RV64-NEXT: srliw a2, a0, 16 +; RV64-NEXT: slli a3, a0, 48 +; RV64-NEXT: srli a4, a3, 56 +; RV64-NEXT: srai a3, a3, 56 ; RV64-NEXT: slli a5, a0, 56 ; RV64-NEXT: srai a5, a5, 56 ; RV64-NEXT: slli a6, a0, 40 ; RV64-NEXT: srai a6, a6, 56 ; RV64-NEXT: sgtz a6, a6 ; RV64-NEXT: sgtz a5, a5 -; RV64-NEXT: sgtz a4, a4 -; RV64-NEXT: negw a4, a4 -; RV64-NEXT: and a3, a4, a3 +; RV64-NEXT: sgtz a3, a3 +; RV64-NEXT: negw a3, a3 +; RV64-NEXT: and a3, a3, a4 ; RV64-NEXT: slli a3, a3, 8 ; RV64-NEXT: negw a4, a5 ; RV64-NEXT: and a0, a4, a0 diff --git a/llvm/test/CodeGen/RISCV/signed-truncation-check.ll b/llvm/test/CodeGen/RISCV/signed-truncation-check.ll index de36bcdb9106..069b2febc334 100644 --- a/llvm/test/CodeGen/RISCV/signed-truncation-check.ll +++ b/llvm/test/CodeGen/RISCV/signed-truncation-check.ll @@ -422,7 +422,8 @@ define i1 @add_ugecmp_i64_i16(i64 %x) nounwind { ; RV32I-NEXT: lui a1, 1048560 ; RV32I-NEXT: addi a1, a1, -1 ; RV32I-NEXT: sltu a1, a1, a2 -; RV32I-NEXT: seqz a0, a0 +; RV32I-NEXT: snez a0, a0 +; RV32I-NEXT: addi a0, a0, -1 ; RV32I-NEXT: and a0, a0, a1 ; RV32I-NEXT: ret ; @@ -462,7 +463,8 @@ define i1 @add_ugecmp_i64_i8(i64 %x) nounwind { ; RV32I-NEXT: addi a2, a0, -128 ; RV32I-NEXT: sltu a0, a2, a0 ; RV32I-NEXT: add a0, a1, a0 -; RV32I-NEXT: seqz a0, a0 +; RV32I-NEXT: snez a0, a0 +; RV32I-NEXT: addi a0, a0, -1 ; RV32I-NEXT: sltiu a1, a2, -256 ; RV32I-NEXT: xori a1, a1, 1 ; RV32I-NEXT: and a0, a0, a1 @@ -691,7 +693,8 @@ define i1 @add_ultcmp_i64_i8(i64 %x) nounwind { ; RV32I-NEXT: addi a2, a0, 128 ; RV32I-NEXT: sltu a0, a2, a0 ; RV32I-NEXT: add a0, a1, a0 -; RV32I-NEXT: seqz a0, a0 +; RV32I-NEXT: snez a0, a0 +; RV32I-NEXT: addi a0, a0, -1 ; RV32I-NEXT: sltiu a1, a2, 256 ; RV32I-NEXT: and a0, a0, a1 ; RV32I-NEXT: ret -- GitLab From d93a126090b6e772d3b96f201cdd44ea0d6360ef Mon Sep 17 00:00:00 2001 From: Alex Langford Date: Thu, 7 Mar 2024 15:04:42 -0800 Subject: [PATCH 518/929] [lldb] Add ability to detect darwin host linker version to xfail tests (#83941) When Apple released its new linker, it had a subtle bug that caused LLDB's TLS tests to fail. Unfortunately this means that TLS tests are not going to work on machines that have affected versions of the linker, so we should annotate the tests so that they only work when we are confident the linker has the required fix. I'm not completely satisfied with this implementation. That being said, I believe that adding suport for linker versions in general is a non-trivial change that would require far more thought. There are a few challenges involved: - LLDB's testing infra takes an argument to change the compiler, but there's no way to switch out the linker. - There's no standard way to ask a compiler what linker it will use. - There's no standard way to ask a linker what its version is. Many platforms have the same name for their linker (ld). - Some platforms automatically switch out the linker underneath you. We do this for Windows tests (where we use LLD no matter what). Given that this is affecting the tests on our CI, I think this is an acceptable solution in the interim. --- .../Python/lldbsuite/test/lldbplatformutil.py | 27 +++++++++++++++++++ .../API/lang/c/tls_globals/TestTlsGlobals.py | 1 + 2 files changed, 28 insertions(+) diff --git a/lldb/packages/Python/lldbsuite/test/lldbplatformutil.py b/lldb/packages/Python/lldbsuite/test/lldbplatformutil.py index c4d063d3cc77..187d16aa1baa 100644 --- a/lldb/packages/Python/lldbsuite/test/lldbplatformutil.py +++ b/lldb/packages/Python/lldbsuite/test/lldbplatformutil.py @@ -3,6 +3,7 @@ architecture and/or the platform dependent nature of the tests. """ # System modules import itertools +import json import re import subprocess import sys @@ -16,6 +17,7 @@ from . import configuration from . import lldbtest_config import lldbsuite.test.lldbplatform as lldbplatform from lldbsuite.test.builders import get_builder +from lldbsuite.test.lldbutil import is_exe def check_first_register_readable(test_case): @@ -333,3 +335,28 @@ def expectedCompiler(compilers): return True return False + + +# This is a helper function to determine if a specific version of Xcode's linker +# contains a TLS bug. We want to skip TLS tests if they contain this bug, but +# adding a linker/linker_version conditions to a decorator is challenging due to +# the number of ways linkers can enter the build process. +def xcode15LinkerBug(): + """Returns true iff a test is running on a darwin platform and the host linker is between versions 1000 and 1109.""" + darwin_platforms = lldbplatform.translate(lldbplatform.darwin_all) + if getPlatform() not in darwin_platforms: + return False + + try: + raw_version_details = subprocess.check_output( + ("xcrun", "ld", "-version_details") + ) + version_details = json.loads(raw_version_details) + version = version_details.get("version", "0") + version_tuple = tuple(int(x) for x in version.split(".")) + if (1000,) <= version_tuple <= (1109,): + return True + except: + pass + + return False diff --git a/lldb/test/API/lang/c/tls_globals/TestTlsGlobals.py b/lldb/test/API/lang/c/tls_globals/TestTlsGlobals.py index dfe29b451df0..2bffd2eea123 100644 --- a/lldb/test/API/lang/c/tls_globals/TestTlsGlobals.py +++ b/lldb/test/API/lang/c/tls_globals/TestTlsGlobals.py @@ -40,6 +40,7 @@ class TlsGlobalTestCase(TestBase): @skipIfWindows @skipIf(oslist=["linux"], archs=["arm", "aarch64"]) @skipIf(oslist=no_match([lldbplatformutil.getDarwinOSTriples(), "linux"])) + @expectedFailureIf(lldbplatformutil.xcode15LinkerBug()) def test(self): """Test thread-local storage.""" self.build() -- GitLab From 1c01651bda46426f497c2948fe52cc25acf0e76d Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Thu, 7 Mar 2024 15:09:17 -0800 Subject: [PATCH 519/929] [libc][docs] add page linking to talks (#84393) --- libc/docs/index.rst | 1 + libc/docs/talks.rst | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 libc/docs/talks.rst diff --git a/libc/docs/index.rst b/libc/docs/index.rst index a50eb080c9ee..370fcd843974 100644 --- a/libc/docs/index.rst +++ b/libc/docs/index.rst @@ -78,6 +78,7 @@ stages there is no ABI stability in any form. dev/index.rst porting contributing + talks .. toctree:: :hidden: diff --git a/libc/docs/talks.rst b/libc/docs/talks.rst new file mode 100644 index 000000000000..6daae9f1e7b6 --- /dev/null +++ b/libc/docs/talks.rst @@ -0,0 +1,29 @@ +===== +Talks +===== +---- +2023 +---- +* Math functions in LLVM libc or yet another correctly rounded libm - Tue Ly + + * `video `__ +* The LLVM C Library for GPUs - Joseph Huber + + * `slides `__ + * `video `__ + +---- +2022 +---- +* Using LLVM's libc - Sivachandra Reddy, Michael Jones, Tue Ly + + * `slides `__ + * `video `__ +* Using modern CPU instructions to improve LLVM's libc math library - Tue Ly + + * `slides `__ + * `video `__ +* Approximating at Scale: How strto float in LLVM’s libc is faster - Michael Jones + + * `slides `__ + * `video `__ -- GitLab From 10edabbcf331fdd53d27c5195de1b692a0063721 Mon Sep 17 00:00:00 2001 From: Evgenii Kudriashov Date: Fri, 8 Mar 2024 02:10:53 +0300 Subject: [PATCH 520/929] [X86][GlobalISel] Enable G_SDIV/G_UDIV/G_SREM/G_UREM (#81615) * Create a libcall for s64 type for 32 bit targets. * Fix a bug in REM selection: SUBREG_TO_REG is not intended to produce a value from super registers. * Replace selector tests by end-to-end tests. Other passes check the selected MIR better. --- .../X86/GISel/X86InstructionSelector.cpp | 9 +- .../lib/Target/X86/GISel/X86LegalizerInfo.cpp | 1 + .../CodeGen/X86/GlobalISel/legalize-sdiv.mir | 127 ++++++++ .../CodeGen/X86/GlobalISel/legalize-srem.mir | 127 ++++++++ .../CodeGen/X86/GlobalISel/legalize-udiv.mir | 127 ++++++++ .../CodeGen/X86/GlobalISel/legalize-urem.mir | 127 ++++++++ .../X86/GlobalISel/x86-legalize-sdiv.mir | 114 -------- .../X86/GlobalISel/x86-legalize-srem.mir | 211 -------------- .../X86/GlobalISel/x86-legalize-udiv.mir | 195 ------------- .../X86/GlobalISel/x86-legalize-urem.mir | 211 -------------- .../X86/GlobalISel/x86-select-sdiv.mir | 130 --------- .../X86/GlobalISel/x86-select-srem.mir | 213 -------------- .../X86/GlobalISel/x86-select-udiv.mir | 215 -------------- .../X86/GlobalISel/x86-select-urem.mir | 215 -------------- .../X86/GlobalISel/x86_64-legalize-sdiv.mir | 145 ---------- .../X86/GlobalISel/x86_64-legalize-srem.mir | 253 ---------------- .../X86/GlobalISel/x86_64-legalize-udiv.mir | 253 ---------------- .../X86/GlobalISel/x86_64-legalize-urem.mir | 253 ---------------- .../X86/GlobalISel/x86_64-select-sdiv.mir | 164 ----------- .../X86/GlobalISel/x86_64-select-srem.mir | 270 ----------------- .../X86/GlobalISel/x86_64-select-udiv.mir | 267 ----------------- .../X86/GlobalISel/x86_64-select-urem.mir | 273 ------------------ llvm/test/CodeGen/X86/isel-sdiv.ll | 116 ++++++++ llvm/test/CodeGen/X86/isel-srem.ll | 150 ++++++++++ llvm/test/CodeGen/X86/isel-udiv.ll | 116 ++++++++ llvm/test/CodeGen/X86/isel-urem.ll | 150 ++++++++++ 26 files changed, 1044 insertions(+), 3388 deletions(-) create mode 100644 llvm/test/CodeGen/X86/GlobalISel/legalize-sdiv.mir create mode 100644 llvm/test/CodeGen/X86/GlobalISel/legalize-srem.mir create mode 100644 llvm/test/CodeGen/X86/GlobalISel/legalize-udiv.mir create mode 100644 llvm/test/CodeGen/X86/GlobalISel/legalize-urem.mir delete mode 100644 llvm/test/CodeGen/X86/GlobalISel/x86-legalize-sdiv.mir delete mode 100644 llvm/test/CodeGen/X86/GlobalISel/x86-legalize-srem.mir delete mode 100644 llvm/test/CodeGen/X86/GlobalISel/x86-legalize-udiv.mir delete mode 100644 llvm/test/CodeGen/X86/GlobalISel/x86-legalize-urem.mir delete mode 100644 llvm/test/CodeGen/X86/GlobalISel/x86-select-sdiv.mir delete mode 100644 llvm/test/CodeGen/X86/GlobalISel/x86-select-srem.mir delete mode 100644 llvm/test/CodeGen/X86/GlobalISel/x86-select-udiv.mir delete mode 100644 llvm/test/CodeGen/X86/GlobalISel/x86-select-urem.mir delete mode 100644 llvm/test/CodeGen/X86/GlobalISel/x86_64-legalize-sdiv.mir delete mode 100644 llvm/test/CodeGen/X86/GlobalISel/x86_64-legalize-srem.mir delete mode 100644 llvm/test/CodeGen/X86/GlobalISel/x86_64-legalize-udiv.mir delete mode 100644 llvm/test/CodeGen/X86/GlobalISel/x86_64-legalize-urem.mir delete mode 100644 llvm/test/CodeGen/X86/GlobalISel/x86_64-select-sdiv.mir delete mode 100644 llvm/test/CodeGen/X86/GlobalISel/x86_64-select-srem.mir delete mode 100644 llvm/test/CodeGen/X86/GlobalISel/x86_64-select-udiv.mir delete mode 100644 llvm/test/CodeGen/X86/GlobalISel/x86_64-select-urem.mir create mode 100644 llvm/test/CodeGen/X86/isel-sdiv.ll create mode 100644 llvm/test/CodeGen/X86/isel-srem.ll create mode 100644 llvm/test/CodeGen/X86/isel-udiv.ll create mode 100644 llvm/test/CodeGen/X86/isel-urem.ll diff --git a/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp b/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp index 26932ba2c8e2..8e0f61a85566 100644 --- a/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp +++ b/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp @@ -1778,12 +1778,9 @@ bool X86InstructionSelector::selectMulDivRem(MachineInstr &I, .addImm(8); // Now reference the 8-bit subreg of the result. - BuildMI(*I.getParent(), I, I.getDebugLoc(), - TII.get(TargetOpcode::SUBREG_TO_REG)) - .addDef(DstReg) - .addImm(0) - .addReg(ResultSuperReg) - .addImm(X86::sub_8bit); + BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), + DstReg) + .addReg(ResultSuperReg, 0, X86::sub_8bit); } else { BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), DstReg) diff --git a/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp b/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp index 2e33adaed7a8..06389842ebb1 100644 --- a/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp +++ b/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp @@ -213,6 +213,7 @@ X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI, return typeInSet(0, {s8, s16, s32})(Query) || (Is64Bit && typeInSet(0, {s64})(Query)); }) + .libcallFor({s64}) .clampScalar(0, s8, sMaxScalar); // integer shifts diff --git a/llvm/test/CodeGen/X86/GlobalISel/legalize-sdiv.mir b/llvm/test/CodeGen/X86/GlobalISel/legalize-sdiv.mir new file mode 100644 index 000000000000..95c69209a2c3 --- /dev/null +++ b/llvm/test/CodeGen/X86/GlobalISel/legalize-sdiv.mir @@ -0,0 +1,127 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=x86_64-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,X64 +# RUN: llc -mtriple=i686-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,X86 + +... +--- +name: test_sdiv_i8 +tracksRegLiveness: true +body: | + bb.1: + liveins: $edi, $esi + + ; CHECK-LABEL: name: test_sdiv_i8 + ; CHECK: liveins: $edi, $esi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $edi + ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32) + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi + ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32) + ; CHECK-NEXT: [[SDIV:%[0-9]+]]:_(s8) = G_SDIV [[TRUNC]], [[TRUNC1]] + ; CHECK-NEXT: $al = COPY [[SDIV]](s8) + ; CHECK-NEXT: RET 0, implicit $al + %2:_(s32) = COPY $edi + %0:_(s8) = G_TRUNC %2(s32) + %3:_(s32) = COPY $esi + %1:_(s8) = G_TRUNC %3(s32) + %4:_(s8) = G_SDIV %0, %1 + $al = COPY %4(s8) + RET 0, implicit $al + +... +--- +name: test_sdiv_i16 +tracksRegLiveness: true +body: | + bb.1: + liveins: $edi, $esi + + ; CHECK-LABEL: name: test_sdiv_i16 + ; CHECK: liveins: $edi, $esi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $edi + ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32) + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi + ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32) + ; CHECK-NEXT: [[SDIV:%[0-9]+]]:_(s16) = G_SDIV [[TRUNC]], [[TRUNC1]] + ; CHECK-NEXT: $ax = COPY [[SDIV]](s16) + ; CHECK-NEXT: RET 0, implicit $ax + %2:_(s32) = COPY $edi + %0:_(s16) = G_TRUNC %2(s32) + %3:_(s32) = COPY $esi + %1:_(s16) = G_TRUNC %3(s32) + %4:_(s16) = G_SDIV %0, %1 + $ax = COPY %4(s16) + RET 0, implicit $ax + +... +--- +name: test_sdiv_i32 +tracksRegLiveness: true +body: | + bb.1: + liveins: $edi, $esi + + ; CHECK-LABEL: name: test_sdiv_i32 + ; CHECK: liveins: $edi, $esi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $edi + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi + ; CHECK-NEXT: [[SDIV:%[0-9]+]]:_(s32) = G_SDIV [[COPY]], [[COPY1]] + ; CHECK-NEXT: $eax = COPY [[SDIV]](s32) + ; CHECK-NEXT: RET 0, implicit $eax + %0:_(s32) = COPY $edi + %1:_(s32) = COPY $esi + %2:_(s32) = G_SDIV %0, %1 + $eax = COPY %2(s32) + RET 0, implicit $eax + +... +--- +name: test_sdiv_i64 +tracksRegLiveness: true +body: | + bb.1: + ; X64-LABEL: name: test_sdiv_i64 + ; X64: [[DEF:%[0-9]+]]:_(s64) = IMPLICIT_DEF + ; X64-NEXT: [[DEF1:%[0-9]+]]:_(s64) = IMPLICIT_DEF + ; X64-NEXT: [[SDIV:%[0-9]+]]:_(s64) = G_SDIV [[DEF]], [[DEF1]] + ; X64-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY [[SDIV]](s64) + ; X64-NEXT: RET 0, implicit [[COPY]](s64) + ; + ; X86-LABEL: name: test_sdiv_i64 + ; X86: [[DEF:%[0-9]+]]:_(s64) = IMPLICIT_DEF + ; X86-NEXT: [[DEF1:%[0-9]+]]:_(s64) = IMPLICIT_DEF + ; X86-NEXT: ADJCALLSTACKDOWN32 16, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp + ; X86-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](s64) + ; X86-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $esp + ; X86-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 + ; X86-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s32) + ; X86-NEXT: G_STORE [[UV]](s32), [[PTR_ADD]](p0) :: (store (s32) into stack, align 1) + ; X86-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $esp + ; X86-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; X86-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C1]](s32) + ; X86-NEXT: G_STORE [[UV1]](s32), [[PTR_ADD1]](p0) :: (store (s32) into stack + 4, align 1) + ; X86-NEXT: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF1]](s64) + ; X86-NEXT: [[COPY2:%[0-9]+]]:_(p0) = COPY $esp + ; X86-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 8 + ; X86-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY2]], [[C2]](s32) + ; X86-NEXT: G_STORE [[UV2]](s32), [[PTR_ADD2]](p0) :: (store (s32) into stack + 8, align 1) + ; X86-NEXT: [[COPY3:%[0-9]+]]:_(p0) = COPY $esp + ; X86-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 12 + ; X86-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY3]], [[C3]](s32) + ; X86-NEXT: G_STORE [[UV3]](s32), [[PTR_ADD3]](p0) :: (store (s32) into stack + 12, align 1) + ; X86-NEXT: CALLpcrel32 &__divdi3, csr_32, implicit $esp, implicit $ssp, implicit-def $eax, implicit-def $edx + ; X86-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $eax + ; X86-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $edx + ; X86-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32) + ; X86-NEXT: ADJCALLSTACKUP32 16, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp + ; X86-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY [[MV]](s64) + ; X86-NEXT: RET 0, implicit [[COPY6]](s64) + %0:_(s64) = IMPLICIT_DEF + %1:_(s64) = IMPLICIT_DEF + %2:_(s64) = G_SDIV %0, %1 + %3:_(s64) = COPY %2(s64) + RET 0, implicit %3 + +... diff --git a/llvm/test/CodeGen/X86/GlobalISel/legalize-srem.mir b/llvm/test/CodeGen/X86/GlobalISel/legalize-srem.mir new file mode 100644 index 000000000000..ab7d89de5aa0 --- /dev/null +++ b/llvm/test/CodeGen/X86/GlobalISel/legalize-srem.mir @@ -0,0 +1,127 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=x86_64-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,X64 +# RUN: llc -mtriple=i686-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,X86 + +... +--- +name: test_srem_i8 +tracksRegLiveness: true +body: | + bb.1: + liveins: $edi, $esi + + ; CHECK-LABEL: name: test_srem_i8 + ; CHECK: liveins: $edi, $esi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $edi + ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32) + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi + ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32) + ; CHECK-NEXT: [[SREM:%[0-9]+]]:_(s8) = G_SREM [[TRUNC]], [[TRUNC1]] + ; CHECK-NEXT: $al = COPY [[SREM]](s8) + ; CHECK-NEXT: RET 0, implicit $al + %2:_(s32) = COPY $edi + %0:_(s8) = G_TRUNC %2(s32) + %3:_(s32) = COPY $esi + %1:_(s8) = G_TRUNC %3(s32) + %4:_(s8) = G_SREM %0, %1 + $al = COPY %4(s8) + RET 0, implicit $al + +... +--- +name: test_srem_i16 +tracksRegLiveness: true +body: | + bb.1: + liveins: $edi, $esi + + ; CHECK-LABEL: name: test_srem_i16 + ; CHECK: liveins: $edi, $esi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $edi + ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32) + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi + ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32) + ; CHECK-NEXT: [[SREM:%[0-9]+]]:_(s16) = G_SREM [[TRUNC]], [[TRUNC1]] + ; CHECK-NEXT: $ax = COPY [[SREM]](s16) + ; CHECK-NEXT: RET 0, implicit $ax + %2:_(s32) = COPY $edi + %0:_(s16) = G_TRUNC %2(s32) + %3:_(s32) = COPY $esi + %1:_(s16) = G_TRUNC %3(s32) + %4:_(s16) = G_SREM %0, %1 + $ax = COPY %4(s16) + RET 0, implicit $ax + +... +--- +name: test_srem_i32 +tracksRegLiveness: true +body: | + bb.1: + liveins: $edi, $esi + + ; CHECK-LABEL: name: test_srem_i32 + ; CHECK: liveins: $edi, $esi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $edi + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi + ; CHECK-NEXT: [[SREM:%[0-9]+]]:_(s32) = G_SREM [[COPY]], [[COPY1]] + ; CHECK-NEXT: $eax = COPY [[SREM]](s32) + ; CHECK-NEXT: RET 0, implicit $eax + %0:_(s32) = COPY $edi + %1:_(s32) = COPY $esi + %2:_(s32) = G_SREM %0, %1 + $eax = COPY %2(s32) + RET 0, implicit $eax + +... +--- +name: test_srem_i64 +tracksRegLiveness: true +body: | + bb.1: + ; X64-LABEL: name: test_srem_i64 + ; X64: [[DEF:%[0-9]+]]:_(s64) = IMPLICIT_DEF + ; X64-NEXT: [[DEF1:%[0-9]+]]:_(s64) = IMPLICIT_DEF + ; X64-NEXT: [[SREM:%[0-9]+]]:_(s64) = G_SREM [[DEF]], [[DEF1]] + ; X64-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY [[SREM]](s64) + ; X64-NEXT: RET 0, implicit [[COPY]](s64) + ; + ; X86-LABEL: name: test_srem_i64 + ; X86: [[DEF:%[0-9]+]]:_(s64) = IMPLICIT_DEF + ; X86-NEXT: [[DEF1:%[0-9]+]]:_(s64) = IMPLICIT_DEF + ; X86-NEXT: ADJCALLSTACKDOWN32 16, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp + ; X86-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](s64) + ; X86-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $esp + ; X86-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 + ; X86-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s32) + ; X86-NEXT: G_STORE [[UV]](s32), [[PTR_ADD]](p0) :: (store (s32) into stack, align 1) + ; X86-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $esp + ; X86-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; X86-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C1]](s32) + ; X86-NEXT: G_STORE [[UV1]](s32), [[PTR_ADD1]](p0) :: (store (s32) into stack + 4, align 1) + ; X86-NEXT: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF1]](s64) + ; X86-NEXT: [[COPY2:%[0-9]+]]:_(p0) = COPY $esp + ; X86-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 8 + ; X86-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY2]], [[C2]](s32) + ; X86-NEXT: G_STORE [[UV2]](s32), [[PTR_ADD2]](p0) :: (store (s32) into stack + 8, align 1) + ; X86-NEXT: [[COPY3:%[0-9]+]]:_(p0) = COPY $esp + ; X86-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 12 + ; X86-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY3]], [[C3]](s32) + ; X86-NEXT: G_STORE [[UV3]](s32), [[PTR_ADD3]](p0) :: (store (s32) into stack + 12, align 1) + ; X86-NEXT: CALLpcrel32 &__moddi3, csr_32, implicit $esp, implicit $ssp, implicit-def $eax, implicit-def $edx + ; X86-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $eax + ; X86-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $edx + ; X86-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32) + ; X86-NEXT: ADJCALLSTACKUP32 16, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp + ; X86-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY [[MV]](s64) + ; X86-NEXT: RET 0, implicit [[COPY6]](s64) + %0:_(s64) = IMPLICIT_DEF + %1:_(s64) = IMPLICIT_DEF + %2:_(s64) = G_SREM %0, %1 + %3:_(s64) = COPY %2(s64) + RET 0, implicit %3 + +... diff --git a/llvm/test/CodeGen/X86/GlobalISel/legalize-udiv.mir b/llvm/test/CodeGen/X86/GlobalISel/legalize-udiv.mir new file mode 100644 index 000000000000..233fada9c6c8 --- /dev/null +++ b/llvm/test/CodeGen/X86/GlobalISel/legalize-udiv.mir @@ -0,0 +1,127 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=x86_64-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,X64 +# RUN: llc -mtriple=i686-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,X86 + +... +--- +name: test_udiv_i8 +tracksRegLiveness: true +body: | + bb.1: + liveins: $edi, $esi + + ; CHECK-LABEL: name: test_udiv_i8 + ; CHECK: liveins: $edi, $esi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $edi + ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32) + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi + ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32) + ; CHECK-NEXT: [[UDIV:%[0-9]+]]:_(s8) = G_UDIV [[TRUNC]], [[TRUNC1]] + ; CHECK-NEXT: $al = COPY [[UDIV]](s8) + ; CHECK-NEXT: RET 0, implicit $al + %2:_(s32) = COPY $edi + %0:_(s8) = G_TRUNC %2(s32) + %3:_(s32) = COPY $esi + %1:_(s8) = G_TRUNC %3(s32) + %4:_(s8) = G_UDIV %0, %1 + $al = COPY %4(s8) + RET 0, implicit $al + +... +--- +name: test_udiv_i16 +tracksRegLiveness: true +body: | + bb.1: + liveins: $edi, $esi + + ; CHECK-LABEL: name: test_udiv_i16 + ; CHECK: liveins: $edi, $esi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $edi + ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32) + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi + ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32) + ; CHECK-NEXT: [[UDIV:%[0-9]+]]:_(s16) = G_UDIV [[TRUNC]], [[TRUNC1]] + ; CHECK-NEXT: $ax = COPY [[UDIV]](s16) + ; CHECK-NEXT: RET 0, implicit $ax + %2:_(s32) = COPY $edi + %0:_(s16) = G_TRUNC %2(s32) + %3:_(s32) = COPY $esi + %1:_(s16) = G_TRUNC %3(s32) + %4:_(s16) = G_UDIV %0, %1 + $ax = COPY %4(s16) + RET 0, implicit $ax + +... +--- +name: test_udiv_i32 +tracksRegLiveness: true +body: | + bb.1: + liveins: $edi, $esi + + ; CHECK-LABEL: name: test_udiv_i32 + ; CHECK: liveins: $edi, $esi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $edi + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi + ; CHECK-NEXT: [[UDIV:%[0-9]+]]:_(s32) = G_UDIV [[COPY]], [[COPY1]] + ; CHECK-NEXT: $eax = COPY [[UDIV]](s32) + ; CHECK-NEXT: RET 0, implicit $eax + %0:_(s32) = COPY $edi + %1:_(s32) = COPY $esi + %2:_(s32) = G_UDIV %0, %1 + $eax = COPY %2(s32) + RET 0, implicit $eax + +... +--- +name: test_udiv_i64 +tracksRegLiveness: true +body: | + bb.1: + ; X64-LABEL: name: test_udiv_i64 + ; X64: [[DEF:%[0-9]+]]:_(s64) = IMPLICIT_DEF + ; X64-NEXT: [[DEF1:%[0-9]+]]:_(s64) = IMPLICIT_DEF + ; X64-NEXT: [[UDIV:%[0-9]+]]:_(s64) = G_UDIV [[DEF]], [[DEF1]] + ; X64-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY [[UDIV]](s64) + ; X64-NEXT: RET 0, implicit [[COPY]](s64) + ; + ; X86-LABEL: name: test_udiv_i64 + ; X86: [[DEF:%[0-9]+]]:_(s64) = IMPLICIT_DEF + ; X86-NEXT: [[DEF1:%[0-9]+]]:_(s64) = IMPLICIT_DEF + ; X86-NEXT: ADJCALLSTACKDOWN32 16, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp + ; X86-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](s64) + ; X86-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $esp + ; X86-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 + ; X86-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s32) + ; X86-NEXT: G_STORE [[UV]](s32), [[PTR_ADD]](p0) :: (store (s32) into stack, align 1) + ; X86-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $esp + ; X86-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; X86-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C1]](s32) + ; X86-NEXT: G_STORE [[UV1]](s32), [[PTR_ADD1]](p0) :: (store (s32) into stack + 4, align 1) + ; X86-NEXT: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF1]](s64) + ; X86-NEXT: [[COPY2:%[0-9]+]]:_(p0) = COPY $esp + ; X86-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 8 + ; X86-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY2]], [[C2]](s32) + ; X86-NEXT: G_STORE [[UV2]](s32), [[PTR_ADD2]](p0) :: (store (s32) into stack + 8, align 1) + ; X86-NEXT: [[COPY3:%[0-9]+]]:_(p0) = COPY $esp + ; X86-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 12 + ; X86-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY3]], [[C3]](s32) + ; X86-NEXT: G_STORE [[UV3]](s32), [[PTR_ADD3]](p0) :: (store (s32) into stack + 12, align 1) + ; X86-NEXT: CALLpcrel32 &__udivdi3, csr_32, implicit $esp, implicit $ssp, implicit-def $eax, implicit-def $edx + ; X86-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $eax + ; X86-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $edx + ; X86-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32) + ; X86-NEXT: ADJCALLSTACKUP32 16, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp + ; X86-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY [[MV]](s64) + ; X86-NEXT: RET 0, implicit [[COPY6]](s64) + %0:_(s64) = IMPLICIT_DEF + %1:_(s64) = IMPLICIT_DEF + %2:_(s64) = G_UDIV %0, %1 + %3:_(s64) = COPY %2(s64) + RET 0, implicit %3 + +... diff --git a/llvm/test/CodeGen/X86/GlobalISel/legalize-urem.mir b/llvm/test/CodeGen/X86/GlobalISel/legalize-urem.mir new file mode 100644 index 000000000000..85f6063dbd1e --- /dev/null +++ b/llvm/test/CodeGen/X86/GlobalISel/legalize-urem.mir @@ -0,0 +1,127 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=x86_64-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,X64 +# RUN: llc -mtriple=i686-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,X86 + +... +--- +name: test_urem_i8 +tracksRegLiveness: true +body: | + bb.1: + liveins: $edi, $esi + + ; CHECK-LABEL: name: test_urem_i8 + ; CHECK: liveins: $edi, $esi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $edi + ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32) + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi + ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32) + ; CHECK-NEXT: [[UREM:%[0-9]+]]:_(s8) = G_UREM [[TRUNC]], [[TRUNC1]] + ; CHECK-NEXT: $al = COPY [[UREM]](s8) + ; CHECK-NEXT: RET 0, implicit $al + %2:_(s32) = COPY $edi + %0:_(s8) = G_TRUNC %2(s32) + %3:_(s32) = COPY $esi + %1:_(s8) = G_TRUNC %3(s32) + %4:_(s8) = G_UREM %0, %1 + $al = COPY %4(s8) + RET 0, implicit $al + +... +--- +name: test_urem_i16 +tracksRegLiveness: true +body: | + bb.1: + liveins: $edi, $esi + + ; CHECK-LABEL: name: test_urem_i16 + ; CHECK: liveins: $edi, $esi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $edi + ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32) + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi + ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32) + ; CHECK-NEXT: [[UREM:%[0-9]+]]:_(s16) = G_UREM [[TRUNC]], [[TRUNC1]] + ; CHECK-NEXT: $ax = COPY [[UREM]](s16) + ; CHECK-NEXT: RET 0, implicit $ax + %2:_(s32) = COPY $edi + %0:_(s16) = G_TRUNC %2(s32) + %3:_(s32) = COPY $esi + %1:_(s16) = G_TRUNC %3(s32) + %4:_(s16) = G_UREM %0, %1 + $ax = COPY %4(s16) + RET 0, implicit $ax + +... +--- +name: test_urem_i32 +tracksRegLiveness: true +body: | + bb.1: + liveins: $edi, $esi + + ; CHECK-LABEL: name: test_urem_i32 + ; CHECK: liveins: $edi, $esi + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $edi + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi + ; CHECK-NEXT: [[UREM:%[0-9]+]]:_(s32) = G_UREM [[COPY]], [[COPY1]] + ; CHECK-NEXT: $eax = COPY [[UREM]](s32) + ; CHECK-NEXT: RET 0, implicit $eax + %0:_(s32) = COPY $edi + %1:_(s32) = COPY $esi + %2:_(s32) = G_UREM %0, %1 + $eax = COPY %2(s32) + RET 0, implicit $eax + +... +--- +name: test_urem_i64 +tracksRegLiveness: true +body: | + bb.1: + ; X64-LABEL: name: test_urem_i64 + ; X64: [[DEF:%[0-9]+]]:_(s64) = IMPLICIT_DEF + ; X64-NEXT: [[DEF1:%[0-9]+]]:_(s64) = IMPLICIT_DEF + ; X64-NEXT: [[UREM:%[0-9]+]]:_(s64) = G_UREM [[DEF]], [[DEF1]] + ; X64-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY [[UREM]](s64) + ; X64-NEXT: RET 0, implicit [[COPY]](s64) + ; + ; X86-LABEL: name: test_urem_i64 + ; X86: [[DEF:%[0-9]+]]:_(s64) = IMPLICIT_DEF + ; X86-NEXT: [[DEF1:%[0-9]+]]:_(s64) = IMPLICIT_DEF + ; X86-NEXT: ADJCALLSTACKDOWN32 16, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp + ; X86-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](s64) + ; X86-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $esp + ; X86-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 + ; X86-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C]](s32) + ; X86-NEXT: G_STORE [[UV]](s32), [[PTR_ADD]](p0) :: (store (s32) into stack, align 1) + ; X86-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $esp + ; X86-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; X86-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY1]], [[C1]](s32) + ; X86-NEXT: G_STORE [[UV1]](s32), [[PTR_ADD1]](p0) :: (store (s32) into stack + 4, align 1) + ; X86-NEXT: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF1]](s64) + ; X86-NEXT: [[COPY2:%[0-9]+]]:_(p0) = COPY $esp + ; X86-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 8 + ; X86-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY2]], [[C2]](s32) + ; X86-NEXT: G_STORE [[UV2]](s32), [[PTR_ADD2]](p0) :: (store (s32) into stack + 8, align 1) + ; X86-NEXT: [[COPY3:%[0-9]+]]:_(p0) = COPY $esp + ; X86-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 12 + ; X86-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY3]], [[C3]](s32) + ; X86-NEXT: G_STORE [[UV3]](s32), [[PTR_ADD3]](p0) :: (store (s32) into stack + 12, align 1) + ; X86-NEXT: CALLpcrel32 &__umoddi3, csr_32, implicit $esp, implicit $ssp, implicit-def $eax, implicit-def $edx + ; X86-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $eax + ; X86-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $edx + ; X86-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32) + ; X86-NEXT: ADJCALLSTACKUP32 16, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp + ; X86-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY [[MV]](s64) + ; X86-NEXT: RET 0, implicit [[COPY6]](s64) + %0:_(s64) = IMPLICIT_DEF + %1:_(s64) = IMPLICIT_DEF + %2:_(s64) = G_UREM %0, %1 + %3:_(s64) = COPY %2(s64) + RET 0, implicit %3 + +... diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86-legalize-sdiv.mir b/llvm/test/CodeGen/X86/GlobalISel/x86-legalize-sdiv.mir deleted file mode 100644 index 80382db94272..000000000000 --- a/llvm/test/CodeGen/X86/GlobalISel/x86-legalize-sdiv.mir +++ /dev/null @@ -1,114 +0,0 @@ -# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple=i686-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s - ---- | - ; ModuleID = 'sdiv.ll' - source_filename = "sdiv.ll" - target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" - - define i8 @test_sdiv_i8(i8 %arg1, i8 %arg2) { - %res = sdiv i8 %arg1, %arg2 - ret i8 %res - } - - define i16 @test_sdiv_i16(i16 %arg1, i16 %arg2) { - %res = sdiv i16 %arg1, %arg2 - ret i16 %res - } - - define i32 @test_sdiv_i32(i32 %arg1, i32 %arg2) { - %res = sdiv i32 %arg1, %arg2 - ret i32 %res - } - -... ---- -name: test_sdiv_i8 -alignment: 16 -tracksRegLiveness: true -registers: - - { id: 0, class: _ } - - { id: 1, class: _ } - - { id: 2, class: _ } - - { id: 3, class: _ } - - { id: 4, class: _ } -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_sdiv_i8 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $edi - ; CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32) - ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi - ; CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32) - ; CHECK: [[SDIV:%[0-9]+]]:_(s8) = G_SDIV [[TRUNC]], [[TRUNC1]] - ; CHECK: $al = COPY [[SDIV]](s8) - ; CHECK: RET 0, implicit $al - %2:_(s32) = COPY $edi - %0:_(s8) = G_TRUNC %2(s32) - %3:_(s32) = COPY $esi - %1:_(s8) = G_TRUNC %3(s32) - %4:_(s8) = G_SDIV %0, %1 - $al = COPY %4(s8) - RET 0, implicit $al - -... ---- -name: test_sdiv_i16 -alignment: 16 -tracksRegLiveness: true -registers: - - { id: 0, class: _ } - - { id: 1, class: _ } - - { id: 2, class: _ } - - { id: 3, class: _ } - - { id: 4, class: _ } -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_sdiv_i16 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $edi - ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32) - ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi - ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32) - ; CHECK: [[SDIV:%[0-9]+]]:_(s16) = G_SDIV [[TRUNC]], [[TRUNC1]] - ; CHECK: $ax = COPY [[SDIV]](s16) - ; CHECK: RET 0, implicit $ax - %2:_(s32) = COPY $edi - %0:_(s16) = G_TRUNC %2(s32) - %3:_(s32) = COPY $esi - %1:_(s16) = G_TRUNC %3(s32) - %4:_(s16) = G_SDIV %0, %1 - $ax = COPY %4(s16) - RET 0, implicit $ax - -... ---- -name: test_sdiv_i32 -alignment: 16 -tracksRegLiveness: true -registers: - - { id: 0, class: _ } - - { id: 1, class: _ } - - { id: 2, class: _ } -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_sdiv_i32 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $edi - ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi - ; CHECK: [[SDIV:%[0-9]+]]:_(s32) = G_SDIV [[COPY]], [[COPY1]] - ; CHECK: $eax = COPY [[SDIV]](s32) - ; CHECK: RET 0, implicit $eax - %0:_(s32) = COPY $edi - %1:_(s32) = COPY $esi - %2:_(s32) = G_SDIV %0, %1 - $eax = COPY %2(s32) - RET 0, implicit $eax - -... diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86-legalize-srem.mir b/llvm/test/CodeGen/X86/GlobalISel/x86-legalize-srem.mir deleted file mode 100644 index 965bf635d6fe..000000000000 --- a/llvm/test/CodeGen/X86/GlobalISel/x86-legalize-srem.mir +++ /dev/null @@ -1,211 +0,0 @@ -# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple=i686-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s - ---- | - ; ModuleID = 'srem.ll' - source_filename = "srem.ll" - target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128" - target triple = "i386--linux-gnu" - - define i8 @test_srem_i8(i8 %arg1, i8 %arg2) { - %res = srem i8 %arg1, %arg2 - ret i8 %res - } - - define i16 @test_srem_i16(i16 %arg1, i16 %arg2) { - %res = srem i16 %arg1, %arg2 - ret i16 %res - } - - define i32 @test_srem_i32(i32 %arg1, i32 %arg2) { - %res = srem i32 %arg1, %arg2 - ret i32 %res - } - -... ---- -name: test_srem_i8 -alignment: 16 -exposesReturnsTwice: false -legalized: false -regBankSelected: false -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: _, preferred-register: '' } - - { id: 1, class: _, preferred-register: '' } - - { id: 2, class: _, preferred-register: '' } - - { id: 3, class: _, preferred-register: '' } - - { id: 4, class: _, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 4 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: - - { id: 0, type: default, offset: 4, size: 1, alignment: 4, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } - - { id: 1, type: default, offset: 0, size: 1, alignment: 16, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } -stack: -constants: -body: | - bb.1 (%ir-block.0): - ; CHECK-LABEL: name: test_srem_i8 - ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0 - ; CHECK: [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (s8) from %fixed-stack.0, align 16) - ; CHECK: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 - ; CHECK: [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load (s8) from %fixed-stack.1, align 4) - ; CHECK: [[SREM:%[0-9]+]]:_(s8) = G_SREM [[LOAD]], [[LOAD1]] - ; CHECK: $al = COPY [[SREM]](s8) - ; CHECK: RET 0, implicit $al - %2:_(p0) = G_FRAME_INDEX %fixed-stack.1 - %0:_(s8) = G_LOAD %2(p0) :: (invariant load (s8) from %fixed-stack.1, align 16) - %3:_(p0) = G_FRAME_INDEX %fixed-stack.0 - %1:_(s8) = G_LOAD %3(p0) :: (invariant load (s8) from %fixed-stack.0, align 4) - %4:_(s8) = G_SREM %0, %1 - $al = COPY %4(s8) - RET 0, implicit $al - -... ---- -name: test_srem_i16 -alignment: 16 -exposesReturnsTwice: false -legalized: false -regBankSelected: false -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: _, preferred-register: '' } - - { id: 1, class: _, preferred-register: '' } - - { id: 2, class: _, preferred-register: '' } - - { id: 3, class: _, preferred-register: '' } - - { id: 4, class: _, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 4 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: - - { id: 0, type: default, offset: 4, size: 2, alignment: 4, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } - - { id: 1, type: default, offset: 0, size: 2, alignment: 16, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } -stack: -constants: -body: | - bb.1 (%ir-block.0): - ; CHECK-LABEL: name: test_srem_i16 - ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0 - ; CHECK: [[LOAD:%[0-9]+]]:_(s16) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (s16) from %fixed-stack.0, align 16) - ; CHECK: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 - ; CHECK: [[LOAD1:%[0-9]+]]:_(s16) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load (s16) from %fixed-stack.1, align 4) - ; CHECK: [[SREM:%[0-9]+]]:_(s16) = G_SREM [[LOAD]], [[LOAD1]] - ; CHECK: $ax = COPY [[SREM]](s16) - ; CHECK: RET 0, implicit $ax - %2:_(p0) = G_FRAME_INDEX %fixed-stack.1 - %0:_(s16) = G_LOAD %2(p0) :: (invariant load (s16) from %fixed-stack.1, align 16) - %3:_(p0) = G_FRAME_INDEX %fixed-stack.0 - %1:_(s16) = G_LOAD %3(p0) :: (invariant load (s16) from %fixed-stack.0, align 4) - %4:_(s16) = G_SREM %0, %1 - $ax = COPY %4(s16) - RET 0, implicit $ax - -... ---- -name: test_srem_i32 -alignment: 16 -exposesReturnsTwice: false -legalized: false -regBankSelected: false -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: _, preferred-register: '' } - - { id: 1, class: _, preferred-register: '' } - - { id: 2, class: _, preferred-register: '' } - - { id: 3, class: _, preferred-register: '' } - - { id: 4, class: _, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 4 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: - - { id: 0, type: default, offset: 4, size: 4, alignment: 4, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } - - { id: 1, type: default, offset: 0, size: 4, alignment: 16, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } -stack: -constants: -body: | - bb.1 (%ir-block.0): - ; CHECK-LABEL: name: test_srem_i32 - ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0 - ; CHECK: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (s32) from %fixed-stack.0, align 16) - ; CHECK: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 - ; CHECK: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load (s32) from %fixed-stack.1) - ; CHECK: [[SREM:%[0-9]+]]:_(s32) = G_SREM [[LOAD]], [[LOAD1]] - ; CHECK: $eax = COPY [[SREM]](s32) - ; CHECK: RET 0, implicit $eax - %2:_(p0) = G_FRAME_INDEX %fixed-stack.1 - %0:_(s32) = G_LOAD %2(p0) :: (invariant load (s32) from %fixed-stack.1, align 16) - %3:_(p0) = G_FRAME_INDEX %fixed-stack.0 - %1:_(s32) = G_LOAD %3(p0) :: (invariant load (s32) from %fixed-stack.0, align 4) - %4:_(s32) = G_SREM %0, %1 - $eax = COPY %4(s32) - RET 0, implicit $eax - -... diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86-legalize-udiv.mir b/llvm/test/CodeGen/X86/GlobalISel/x86-legalize-udiv.mir deleted file mode 100644 index 85c9b6d9e86b..000000000000 --- a/llvm/test/CodeGen/X86/GlobalISel/x86-legalize-udiv.mir +++ /dev/null @@ -1,195 +0,0 @@ -# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple=i686-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s - ---- | - ; ModuleID = 'udiv.ll' - source_filename = "udiv.ll" - target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" - - define i8 @test_udiv_i8(i8 %arg1, i8 %arg2) { - %res = udiv i8 %arg1, %arg2 - ret i8 %res - } - - define i16 @test_udiv_i16(i16 %arg1, i16 %arg2) { - %res = udiv i16 %arg1, %arg2 - ret i16 %res - } - - define i32 @test_udiv_i32(i32 %arg1, i32 %arg2) { - %res = udiv i32 %arg1, %arg2 - ret i32 %res - } - -... ---- -name: test_udiv_i8 -alignment: 16 -exposesReturnsTwice: false -legalized: false -regBankSelected: false -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: _, preferred-register: '' } - - { id: 1, class: _, preferred-register: '' } - - { id: 2, class: _, preferred-register: '' } - - { id: 3, class: _, preferred-register: '' } - - { id: 4, class: _, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_udiv_i8 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $edi - ; CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32) - ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi - ; CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32) - ; CHECK: [[UDIV:%[0-9]+]]:_(s8) = G_UDIV [[TRUNC]], [[TRUNC1]] - ; CHECK: $al = COPY [[UDIV]](s8) - ; CHECK: RET 0, implicit $al - %2:_(s32) = COPY $edi - %0:_(s8) = G_TRUNC %2(s32) - %3:_(s32) = COPY $esi - %1:_(s8) = G_TRUNC %3(s32) - %4:_(s8) = G_UDIV %0, %1 - $al = COPY %4(s8) - RET 0, implicit $al - -... ---- -name: test_udiv_i16 -alignment: 16 -exposesReturnsTwice: false -legalized: false -regBankSelected: false -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: _, preferred-register: '' } - - { id: 1, class: _, preferred-register: '' } - - { id: 2, class: _, preferred-register: '' } - - { id: 3, class: _, preferred-register: '' } - - { id: 4, class: _, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_udiv_i16 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $edi - ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32) - ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi - ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32) - ; CHECK: [[UDIV:%[0-9]+]]:_(s16) = G_UDIV [[TRUNC]], [[TRUNC1]] - ; CHECK: $ax = COPY [[UDIV]](s16) - ; CHECK: RET 0, implicit $ax - %2:_(s32) = COPY $edi - %0:_(s16) = G_TRUNC %2(s32) - %3:_(s32) = COPY $esi - %1:_(s16) = G_TRUNC %3(s32) - %4:_(s16) = G_UDIV %0, %1 - $ax = COPY %4(s16) - RET 0, implicit $ax - -... ---- -name: test_udiv_i32 -alignment: 16 -exposesReturnsTwice: false -legalized: false -regBankSelected: false -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: _, preferred-register: '' } - - { id: 1, class: _, preferred-register: '' } - - { id: 2, class: _, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_udiv_i32 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $edi - ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi - ; CHECK: [[UDIV:%[0-9]+]]:_(s32) = G_UDIV [[COPY]], [[COPY1]] - ; CHECK: $eax = COPY [[UDIV]](s32) - ; CHECK: RET 0, implicit $eax - %0:_(s32) = COPY $edi - %1:_(s32) = COPY $esi - %2:_(s32) = G_UDIV %0, %1 - $eax = COPY %2(s32) - RET 0, implicit $eax - -... diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86-legalize-urem.mir b/llvm/test/CodeGen/X86/GlobalISel/x86-legalize-urem.mir deleted file mode 100644 index b6496216ac56..000000000000 --- a/llvm/test/CodeGen/X86/GlobalISel/x86-legalize-urem.mir +++ /dev/null @@ -1,211 +0,0 @@ -# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple=i686-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s - ---- | - ; ModuleID = 'urem.ll' - source_filename = "urem.ll" - target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128" - target triple = "i386--linux-gnu" - - define i8 @test_urem_i8(i8 %arg1, i8 %arg2) { - %res = urem i8 %arg1, %arg2 - ret i8 %res - } - - define i16 @test_urem_i16(i16 %arg1, i16 %arg2) { - %res = urem i16 %arg1, %arg2 - ret i16 %res - } - - define i32 @test_urem_i32(i32 %arg1, i32 %arg2) { - %res = urem i32 %arg1, %arg2 - ret i32 %res - } - -... ---- -name: test_urem_i8 -alignment: 16 -exposesReturnsTwice: false -legalized: false -regBankSelected: false -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: _, preferred-register: '' } - - { id: 1, class: _, preferred-register: '' } - - { id: 2, class: _, preferred-register: '' } - - { id: 3, class: _, preferred-register: '' } - - { id: 4, class: _, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 4 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: - - { id: 0, type: default, offset: 4, size: 1, alignment: 4, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } - - { id: 1, type: default, offset: 0, size: 1, alignment: 16, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } -stack: -constants: -body: | - bb.1 (%ir-block.0): - ; CHECK-LABEL: name: test_urem_i8 - ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0 - ; CHECK: [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (s8) from %fixed-stack.0, align 16) - ; CHECK: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 - ; CHECK: [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load (s8) from %fixed-stack.1, align 4) - ; CHECK: [[UREM:%[0-9]+]]:_(s8) = G_UREM [[LOAD]], [[LOAD1]] - ; CHECK: $al = COPY [[UREM]](s8) - ; CHECK: RET 0, implicit $al - %2:_(p0) = G_FRAME_INDEX %fixed-stack.1 - %0:_(s8) = G_LOAD %2(p0) :: (invariant load (s8) from %fixed-stack.1, align 16) - %3:_(p0) = G_FRAME_INDEX %fixed-stack.0 - %1:_(s8) = G_LOAD %3(p0) :: (invariant load (s8) from %fixed-stack.0, align 4) - %4:_(s8) = G_UREM %0, %1 - $al = COPY %4(s8) - RET 0, implicit $al - -... ---- -name: test_urem_i16 -alignment: 16 -exposesReturnsTwice: false -legalized: false -regBankSelected: false -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: _, preferred-register: '' } - - { id: 1, class: _, preferred-register: '' } - - { id: 2, class: _, preferred-register: '' } - - { id: 3, class: _, preferred-register: '' } - - { id: 4, class: _, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 4 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: - - { id: 0, type: default, offset: 4, size: 2, alignment: 4, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } - - { id: 1, type: default, offset: 0, size: 2, alignment: 16, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } -stack: -constants: -body: | - bb.1 (%ir-block.0): - ; CHECK-LABEL: name: test_urem_i16 - ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0 - ; CHECK: [[LOAD:%[0-9]+]]:_(s16) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (s16) from %fixed-stack.0, align 16) - ; CHECK: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 - ; CHECK: [[LOAD1:%[0-9]+]]:_(s16) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load (s16) from %fixed-stack.1, align 4) - ; CHECK: [[UREM:%[0-9]+]]:_(s16) = G_UREM [[LOAD]], [[LOAD1]] - ; CHECK: $ax = COPY [[UREM]](s16) - ; CHECK: RET 0, implicit $ax - %2:_(p0) = G_FRAME_INDEX %fixed-stack.1 - %0:_(s16) = G_LOAD %2(p0) :: (invariant load (s16) from %fixed-stack.1, align 16) - %3:_(p0) = G_FRAME_INDEX %fixed-stack.0 - %1:_(s16) = G_LOAD %3(p0) :: (invariant load (s16) from %fixed-stack.0, align 4) - %4:_(s16) = G_UREM %0, %1 - $ax = COPY %4(s16) - RET 0, implicit $ax - -... ---- -name: test_urem_i32 -alignment: 16 -exposesReturnsTwice: false -legalized: false -regBankSelected: false -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: _, preferred-register: '' } - - { id: 1, class: _, preferred-register: '' } - - { id: 2, class: _, preferred-register: '' } - - { id: 3, class: _, preferred-register: '' } - - { id: 4, class: _, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 4 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: - - { id: 0, type: default, offset: 4, size: 4, alignment: 4, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } - - { id: 1, type: default, offset: 0, size: 4, alignment: 16, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } -stack: -constants: -body: | - bb.1 (%ir-block.0): - ; CHECK-LABEL: name: test_urem_i32 - ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0 - ; CHECK: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (s32) from %fixed-stack.0, align 16) - ; CHECK: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 - ; CHECK: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load (s32) from %fixed-stack.1) - ; CHECK: [[UREM:%[0-9]+]]:_(s32) = G_UREM [[LOAD]], [[LOAD1]] - ; CHECK: $eax = COPY [[UREM]](s32) - ; CHECK: RET 0, implicit $eax - %2:_(p0) = G_FRAME_INDEX %fixed-stack.1 - %0:_(s32) = G_LOAD %2(p0) :: (invariant load (s32) from %fixed-stack.1, align 16) - %3:_(p0) = G_FRAME_INDEX %fixed-stack.0 - %1:_(s32) = G_LOAD %3(p0) :: (invariant load (s32) from %fixed-stack.0, align 4) - %4:_(s32) = G_UREM %0, %1 - $eax = COPY %4(s32) - RET 0, implicit $eax - -... diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86-select-sdiv.mir b/llvm/test/CodeGen/X86/GlobalISel/x86-select-sdiv.mir deleted file mode 100644 index 653d867492dc..000000000000 --- a/llvm/test/CodeGen/X86/GlobalISel/x86-select-sdiv.mir +++ /dev/null @@ -1,130 +0,0 @@ -# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple=i386-linux-gnu -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s - ---- | - ; ModuleID = 'sdiv.ll' - source_filename = "sdiv.ll" - target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" - - define i8 @test_sdiv_i8(i8 %arg1, i8 %arg2) { - %res = sdiv i8 %arg1, %arg2 - ret i8 %res - } - - define i16 @test_sdiv_i16(i16 %arg1, i16 %arg2) { - %res = sdiv i16 %arg1, %arg2 - ret i16 %res - } - - define i32 @test_sdiv_i32(i32 %arg1, i32 %arg2) { - %res = sdiv i32 %arg1, %arg2 - ret i32 %res - } - -... ---- -name: test_sdiv_i8 -alignment: 16 -legalized: true -regBankSelected: true -tracksRegLiveness: true -registers: - - { id: 0, class: gpr } - - { id: 1, class: gpr } - - { id: 2, class: gpr } - - { id: 3, class: gpr } - - { id: 4, class: gpr } -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_sdiv_i8 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:gr32 = COPY $edi - ; CHECK: [[COPY1:%[0-9]+]]:gr32_abcd = COPY [[COPY]] - ; CHECK: [[COPY2:%[0-9]+]]:gr8_abcd_l = COPY [[COPY1]].sub_8bit - ; CHECK: [[COPY3:%[0-9]+]]:gr32 = COPY $esi - ; CHECK: [[COPY4:%[0-9]+]]:gr32_abcd = COPY [[COPY3]] - ; CHECK: [[COPY5:%[0-9]+]]:gr8_abcd_l = COPY [[COPY4]].sub_8bit - ; CHECK: $ax = MOVSX16rr8 [[COPY2]] - ; CHECK: IDIV8r [[COPY5]], implicit-def $al, implicit-def $ah, implicit-def $eflags, implicit $ax - ; CHECK: [[COPY6:%[0-9]+]]:gr8 = COPY $al - ; CHECK: $al = COPY [[COPY6]] - ; CHECK: RET 0, implicit $al - %2:gpr(s32) = COPY $edi - %0:gpr(s8) = G_TRUNC %2(s32) - %3:gpr(s32) = COPY $esi - %1:gpr(s8) = G_TRUNC %3(s32) - %4:gpr(s8) = G_SDIV %0, %1 - $al = COPY %4(s8) - RET 0, implicit $al - -... ---- -name: test_sdiv_i16 -alignment: 16 -legalized: true -regBankSelected: true -tracksRegLiveness: true -registers: - - { id: 0, class: gpr } - - { id: 1, class: gpr } - - { id: 2, class: gpr } - - { id: 3, class: gpr } - - { id: 4, class: gpr } -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_sdiv_i16 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:gr32 = COPY $edi - ; CHECK: [[COPY1:%[0-9]+]]:gr16 = COPY [[COPY]].sub_16bit - ; CHECK: [[COPY2:%[0-9]+]]:gr32 = COPY $esi - ; CHECK: [[COPY3:%[0-9]+]]:gr16 = COPY [[COPY2]].sub_16bit - ; CHECK: $ax = COPY [[COPY1]] - ; CHECK: CWD implicit-def $ax, implicit-def $dx, implicit $ax - ; CHECK: IDIV16r [[COPY3]], implicit-def $ax, implicit-def $dx, implicit-def $eflags, implicit $ax, implicit $dx - ; CHECK: [[COPY4:%[0-9]+]]:gr16 = COPY $ax - ; CHECK: $ax = COPY [[COPY4]] - ; CHECK: RET 0, implicit $ax - %2:gpr(s32) = COPY $edi - %0:gpr(s16) = G_TRUNC %2(s32) - %3:gpr(s32) = COPY $esi - %1:gpr(s16) = G_TRUNC %3(s32) - %4:gpr(s16) = G_SDIV %0, %1 - $ax = COPY %4(s16) - RET 0, implicit $ax - -... ---- -name: test_sdiv_i32 -alignment: 16 -legalized: true -regBankSelected: true -tracksRegLiveness: true -registers: - - { id: 0, class: gpr } - - { id: 1, class: gpr } - - { id: 2, class: gpr } -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_sdiv_i32 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:gr32 = COPY $edi - ; CHECK: [[COPY1:%[0-9]+]]:gr32 = COPY $esi - ; CHECK: $eax = COPY [[COPY]] - ; CHECK: CDQ implicit-def $eax, implicit-def $edx, implicit $eax - ; CHECK: IDIV32r [[COPY1]], implicit-def $eax, implicit-def $edx, implicit-def $eflags, implicit $eax, implicit $edx - ; CHECK: [[COPY2:%[0-9]+]]:gr32 = COPY $eax - ; CHECK: $eax = COPY [[COPY2]] - ; CHECK: RET 0, implicit $eax - %0:gpr(s32) = COPY $edi - %1:gpr(s32) = COPY $esi - %2:gpr(s32) = G_SDIV %0, %1 - $eax = COPY %2(s32) - RET 0, implicit $eax - -... diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86-select-srem.mir b/llvm/test/CodeGen/X86/GlobalISel/x86-select-srem.mir deleted file mode 100644 index a7f5badcdef0..000000000000 --- a/llvm/test/CodeGen/X86/GlobalISel/x86-select-srem.mir +++ /dev/null @@ -1,213 +0,0 @@ -# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple=i386-linux-gnu -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s - ---- | - ; ModuleID = 'srem.ll' - source_filename = "srem.ll" - target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128" - target triple = "i386--linux-gnu" - - define i8 @test_srem_i8(i8 %arg1, i8 %arg2) { - %res = srem i8 %arg1, %arg2 - ret i8 %res - } - - define i16 @test_srem_i16(i16 %arg1, i16 %arg2) { - %res = srem i16 %arg1, %arg2 - ret i16 %res - } - - define i32 @test_srem_i32(i32 %arg1, i32 %arg2) { - %res = srem i32 %arg1, %arg2 - ret i32 %res - } - -... ---- -name: test_srem_i8 -alignment: 16 -exposesReturnsTwice: false -legalized: true -regBankSelected: true -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: gpr, preferred-register: '' } - - { id: 1, class: gpr, preferred-register: '' } - - { id: 2, class: gpr, preferred-register: '' } - - { id: 3, class: gpr, preferred-register: '' } - - { id: 4, class: gpr, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 4 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: - - { id: 0, type: default, offset: 4, size: 1, alignment: 4, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } - - { id: 1, type: default, offset: 0, size: 1, alignment: 16, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } -stack: -constants: -body: | - bb.1 (%ir-block.0): - ; CHECK-LABEL: name: test_srem_i8 - ; CHECK: [[MOV8rm:%[0-9]+]]:gr8 = MOV8rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load (s8) from %fixed-stack.0, align 16) - ; CHECK: [[MOV8rm1:%[0-9]+]]:gr8 = MOV8rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load (s8) from %fixed-stack.1, align 4) - ; CHECK: $ax = MOVSX16rr8 [[MOV8rm]] - ; CHECK: IDIV8r [[MOV8rm1]], implicit-def $al, implicit-def $ah, implicit-def $eflags, implicit $ax - ; CHECK: [[COPY:%[0-9]+]]:gr8 = COPY $ah - ; CHECK: $al = COPY [[COPY]] - ; CHECK: RET 0, implicit $al - %2:gpr(p0) = G_FRAME_INDEX %fixed-stack.1 - %0:gpr(s8) = G_LOAD %2(p0) :: (invariant load (s8) from %fixed-stack.1, align 16) - %3:gpr(p0) = G_FRAME_INDEX %fixed-stack.0 - %1:gpr(s8) = G_LOAD %3(p0) :: (invariant load (s8) from %fixed-stack.0, align 4) - %4:gpr(s8) = G_SREM %0, %1 - $al = COPY %4(s8) - RET 0, implicit $al - -... ---- -name: test_srem_i16 -alignment: 16 -exposesReturnsTwice: false -legalized: true -regBankSelected: true -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: gpr, preferred-register: '' } - - { id: 1, class: gpr, preferred-register: '' } - - { id: 2, class: gpr, preferred-register: '' } - - { id: 3, class: gpr, preferred-register: '' } - - { id: 4, class: gpr, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 4 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: - - { id: 0, type: default, offset: 4, size: 2, alignment: 4, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } - - { id: 1, type: default, offset: 0, size: 2, alignment: 16, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } -stack: -constants: -body: | - bb.1 (%ir-block.0): - ; CHECK-LABEL: name: test_srem_i16 - ; CHECK: [[MOV16rm:%[0-9]+]]:gr16 = MOV16rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load (s16) from %fixed-stack.0, align 16) - ; CHECK: [[MOV16rm1:%[0-9]+]]:gr16 = MOV16rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load (s16) from %fixed-stack.1, align 4) - ; CHECK: $ax = COPY [[MOV16rm]] - ; CHECK: CWD implicit-def $ax, implicit-def $dx, implicit $ax - ; CHECK: IDIV16r [[MOV16rm1]], implicit-def $ax, implicit-def $dx, implicit-def $eflags, implicit $ax, implicit $dx - ; CHECK: [[COPY:%[0-9]+]]:gr16 = COPY $dx - ; CHECK: $ax = COPY [[COPY]] - ; CHECK: RET 0, implicit $ax - %2:gpr(p0) = G_FRAME_INDEX %fixed-stack.1 - %0:gpr(s16) = G_LOAD %2(p0) :: (invariant load (s16) from %fixed-stack.1, align 16) - %3:gpr(p0) = G_FRAME_INDEX %fixed-stack.0 - %1:gpr(s16) = G_LOAD %3(p0) :: (invariant load (s16) from %fixed-stack.0, align 4) - %4:gpr(s16) = G_SREM %0, %1 - $ax = COPY %4(s16) - RET 0, implicit $ax - -... ---- -name: test_srem_i32 -alignment: 16 -exposesReturnsTwice: false -legalized: true -regBankSelected: true -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: gpr, preferred-register: '' } - - { id: 1, class: gpr, preferred-register: '' } - - { id: 2, class: gpr, preferred-register: '' } - - { id: 3, class: gpr, preferred-register: '' } - - { id: 4, class: gpr, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 4 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: - - { id: 0, type: default, offset: 4, size: 4, alignment: 4, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } - - { id: 1, type: default, offset: 0, size: 4, alignment: 16, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } -stack: -constants: -body: | - bb.1 (%ir-block.0): - ; CHECK-LABEL: name: test_srem_i32 - ; CHECK: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load (s32) from %fixed-stack.0, align 16) - ; CHECK: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load (s32) from %fixed-stack.1) - ; CHECK: $eax = COPY [[MOV32rm]] - ; CHECK: CDQ implicit-def $eax, implicit-def $edx, implicit $eax - ; CHECK: IDIV32r [[MOV32rm1]], implicit-def $eax, implicit-def $edx, implicit-def $eflags, implicit $eax, implicit $edx - ; CHECK: [[COPY:%[0-9]+]]:gr32 = COPY $edx - ; CHECK: $eax = COPY [[COPY]] - ; CHECK: RET 0, implicit $eax - %2:gpr(p0) = G_FRAME_INDEX %fixed-stack.1 - %0:gpr(s32) = G_LOAD %2(p0) :: (invariant load (s32) from %fixed-stack.1, align 16) - %3:gpr(p0) = G_FRAME_INDEX %fixed-stack.0 - %1:gpr(s32) = G_LOAD %3(p0) :: (invariant load (s32) from %fixed-stack.0, align 4) - %4:gpr(s32) = G_SREM %0, %1 - $eax = COPY %4(s32) - RET 0, implicit $eax - -... diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86-select-udiv.mir b/llvm/test/CodeGen/X86/GlobalISel/x86-select-udiv.mir deleted file mode 100644 index 1a960f9ad9e2..000000000000 --- a/llvm/test/CodeGen/X86/GlobalISel/x86-select-udiv.mir +++ /dev/null @@ -1,215 +0,0 @@ -# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple=i386-linux-gnu -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s - ---- | - ; ModuleID = 'udiv.ll' - source_filename = "udiv.ll" - target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128" - target triple = "i386--linux-gnu" - - define i8 @test_udiv_i8(i8 %arg1, i8 %arg2) { - %res = udiv i8 %arg1, %arg2 - ret i8 %res - } - - define i16 @test_udiv_i16(i16 %arg1, i16 %arg2) { - %res = udiv i16 %arg1, %arg2 - ret i16 %res - } - - define i32 @test_udiv_i32(i32 %arg1, i32 %arg2) { - %res = udiv i32 %arg1, %arg2 - ret i32 %res - } - -... ---- -name: test_udiv_i8 -alignment: 16 -exposesReturnsTwice: false -legalized: true -regBankSelected: true -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: gpr, preferred-register: '' } - - { id: 1, class: gpr, preferred-register: '' } - - { id: 2, class: gpr, preferred-register: '' } - - { id: 3, class: gpr, preferred-register: '' } - - { id: 4, class: gpr, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 4 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: - - { id: 0, type: default, offset: 4, size: 1, alignment: 4, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } - - { id: 1, type: default, offset: 0, size: 1, alignment: 16, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } -stack: -constants: -body: | - bb.1 (%ir-block.0): - ; CHECK-LABEL: name: test_udiv_i8 - ; CHECK: [[MOV8rm:%[0-9]+]]:gr8 = MOV8rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load (s8) from %fixed-stack.0, align 16) - ; CHECK: [[MOV8rm1:%[0-9]+]]:gr8 = MOV8rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load (s8) from %fixed-stack.1, align 4) - ; CHECK: $ax = MOVZX16rr8 [[MOV8rm]] - ; CHECK: DIV8r [[MOV8rm1]], implicit-def $al, implicit-def $ah, implicit-def $eflags, implicit $ax - ; CHECK: [[COPY:%[0-9]+]]:gr8 = COPY $al - ; CHECK: $al = COPY [[COPY]] - ; CHECK: RET 0, implicit $al - %2:gpr(p0) = G_FRAME_INDEX %fixed-stack.1 - %0:gpr(s8) = G_LOAD %2(p0) :: (invariant load (s8) from %fixed-stack.1, align 16) - %3:gpr(p0) = G_FRAME_INDEX %fixed-stack.0 - %1:gpr(s8) = G_LOAD %3(p0) :: (invariant load (s8) from %fixed-stack.0, align 4) - %4:gpr(s8) = G_UDIV %0, %1 - $al = COPY %4(s8) - RET 0, implicit $al - -... ---- -name: test_udiv_i16 -alignment: 16 -exposesReturnsTwice: false -legalized: true -regBankSelected: true -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: gpr, preferred-register: '' } - - { id: 1, class: gpr, preferred-register: '' } - - { id: 2, class: gpr, preferred-register: '' } - - { id: 3, class: gpr, preferred-register: '' } - - { id: 4, class: gpr, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 4 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: - - { id: 0, type: default, offset: 4, size: 2, alignment: 4, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } - - { id: 1, type: default, offset: 0, size: 2, alignment: 16, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } -stack: -constants: -body: | - bb.1 (%ir-block.0): - ; CHECK-LABEL: name: test_udiv_i16 - ; CHECK: [[MOV16rm:%[0-9]+]]:gr16 = MOV16rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load (s16) from %fixed-stack.0, align 16) - ; CHECK: [[MOV16rm1:%[0-9]+]]:gr16 = MOV16rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load (s16) from %fixed-stack.1, align 4) - ; CHECK: $ax = COPY [[MOV16rm]] - ; CHECK: [[MOV32r0_:%[0-9]+]]:gr32 = MOV32r0 implicit-def $eflags - ; CHECK: $dx = COPY [[MOV32r0_]].sub_16bit - ; CHECK: DIV16r [[MOV16rm1]], implicit-def $ax, implicit-def $dx, implicit-def $eflags, implicit $ax, implicit $dx - ; CHECK: [[COPY:%[0-9]+]]:gr16 = COPY $ax - ; CHECK: $ax = COPY [[COPY]] - ; CHECK: RET 0, implicit $ax - %2:gpr(p0) = G_FRAME_INDEX %fixed-stack.1 - %0:gpr(s16) = G_LOAD %2(p0) :: (invariant load (s16) from %fixed-stack.1, align 16) - %3:gpr(p0) = G_FRAME_INDEX %fixed-stack.0 - %1:gpr(s16) = G_LOAD %3(p0) :: (invariant load (s16) from %fixed-stack.0, align 4) - %4:gpr(s16) = G_UDIV %0, %1 - $ax = COPY %4(s16) - RET 0, implicit $ax - -... ---- -name: test_udiv_i32 -alignment: 16 -exposesReturnsTwice: false -legalized: true -regBankSelected: true -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: gpr, preferred-register: '' } - - { id: 1, class: gpr, preferred-register: '' } - - { id: 2, class: gpr, preferred-register: '' } - - { id: 3, class: gpr, preferred-register: '' } - - { id: 4, class: gpr, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 4 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: - - { id: 0, type: default, offset: 4, size: 4, alignment: 4, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } - - { id: 1, type: default, offset: 0, size: 4, alignment: 16, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } -stack: -constants: -body: | - bb.1 (%ir-block.0): - ; CHECK-LABEL: name: test_udiv_i32 - ; CHECK: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load (s32) from %fixed-stack.0) - ; CHECK: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load (s32) from %fixed-stack.1, align 16) - ; CHECK: $eax = COPY [[MOV32rm]] - ; CHECK: [[MOV32r0_:%[0-9]+]]:gr32 = MOV32r0 implicit-def $eflags - ; CHECK: $edx = COPY [[MOV32r0_]] - ; CHECK: DIV32r [[MOV32rm1]], implicit-def $eax, implicit-def $edx, implicit-def $eflags, implicit $eax, implicit $edx - ; CHECK: [[COPY:%[0-9]+]]:gr32 = COPY $eax - ; CHECK: $eax = COPY [[COPY]] - ; CHECK: RET 0, implicit $eax - %2:gpr(p0) = G_FRAME_INDEX %fixed-stack.1 - %0:gpr(s32) = G_LOAD %2(p0) :: (invariant load (s32) from %fixed-stack.1, align 4) - %3:gpr(p0) = G_FRAME_INDEX %fixed-stack.0 - %1:gpr(s32) = G_LOAD %3(p0) :: (invariant load (s32) from %fixed-stack.0, align 16) - %4:gpr(s32) = G_UDIV %0, %1 - $eax = COPY %4(s32) - RET 0, implicit $eax - -... diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86-select-urem.mir b/llvm/test/CodeGen/X86/GlobalISel/x86-select-urem.mir deleted file mode 100644 index 23d2892ad911..000000000000 --- a/llvm/test/CodeGen/X86/GlobalISel/x86-select-urem.mir +++ /dev/null @@ -1,215 +0,0 @@ -# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple=i386-linux-gnu -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s - ---- | - ; ModuleID = 'urem.ll' - source_filename = "urem.ll" - target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128" - target triple = "i386--linux-gnu" - - define i8 @test_urem_i8(i8 %arg1, i8 %arg2) { - %res = urem i8 %arg1, %arg2 - ret i8 %res - } - - define i16 @test_urem_i16(i16 %arg1, i16 %arg2) { - %res = urem i16 %arg1, %arg2 - ret i16 %res - } - - define i32 @test_urem_i32(i32 %arg1, i32 %arg2) { - %res = urem i32 %arg1, %arg2 - ret i32 %res - } - -... ---- -name: test_urem_i8 -alignment: 16 -exposesReturnsTwice: false -legalized: true -regBankSelected: true -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: gpr, preferred-register: '' } - - { id: 1, class: gpr, preferred-register: '' } - - { id: 2, class: gpr, preferred-register: '' } - - { id: 3, class: gpr, preferred-register: '' } - - { id: 4, class: gpr, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 4 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: - - { id: 0, type: default, offset: 4, size: 1, alignment: 4, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } - - { id: 1, type: default, offset: 0, size: 1, alignment: 16, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } -stack: -constants: -body: | - bb.1 (%ir-block.0): - ; CHECK-LABEL: name: test_urem_i8 - ; CHECK: [[MOV8rm:%[0-9]+]]:gr8 = MOV8rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load (s8) from %fixed-stack.0, align 16) - ; CHECK: [[MOV8rm1:%[0-9]+]]:gr8 = MOV8rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load (s8) from %fixed-stack.1, align 4) - ; CHECK: $ax = MOVZX16rr8 [[MOV8rm]] - ; CHECK: DIV8r [[MOV8rm1]], implicit-def $al, implicit-def $ah, implicit-def $eflags, implicit $ax - ; CHECK: [[COPY:%[0-9]+]]:gr8 = COPY $ah - ; CHECK: $al = COPY [[COPY]] - ; CHECK: RET 0, implicit $al - %2:gpr(p0) = G_FRAME_INDEX %fixed-stack.1 - %0:gpr(s8) = G_LOAD %2(p0) :: (invariant load (s8) from %fixed-stack.1, align 16) - %3:gpr(p0) = G_FRAME_INDEX %fixed-stack.0 - %1:gpr(s8) = G_LOAD %3(p0) :: (invariant load (s8) from %fixed-stack.0, align 4) - %4:gpr(s8) = G_UREM %0, %1 - $al = COPY %4(s8) - RET 0, implicit $al - -... ---- -name: test_urem_i16 -alignment: 16 -exposesReturnsTwice: false -legalized: true -regBankSelected: true -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: gpr, preferred-register: '' } - - { id: 1, class: gpr, preferred-register: '' } - - { id: 2, class: gpr, preferred-register: '' } - - { id: 3, class: gpr, preferred-register: '' } - - { id: 4, class: gpr, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 4 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: - - { id: 0, type: default, offset: 4, size: 2, alignment: 4, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } - - { id: 1, type: default, offset: 0, size: 2, alignment: 16, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } -stack: -constants: -body: | - bb.1 (%ir-block.0): - ; CHECK-LABEL: name: test_urem_i16 - ; CHECK: [[MOV16rm:%[0-9]+]]:gr16 = MOV16rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load (s16) from %fixed-stack.0, align 16) - ; CHECK: [[MOV16rm1:%[0-9]+]]:gr16 = MOV16rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load (s16) from %fixed-stack.1, align 4) - ; CHECK: $ax = COPY [[MOV16rm]] - ; CHECK: [[MOV32r0_:%[0-9]+]]:gr32 = MOV32r0 implicit-def $eflags - ; CHECK: $dx = COPY [[MOV32r0_]].sub_16bit - ; CHECK: DIV16r [[MOV16rm1]], implicit-def $ax, implicit-def $dx, implicit-def $eflags, implicit $ax, implicit $dx - ; CHECK: [[COPY:%[0-9]+]]:gr16 = COPY $dx - ; CHECK: $ax = COPY [[COPY]] - ; CHECK: RET 0, implicit $ax - %2:gpr(p0) = G_FRAME_INDEX %fixed-stack.1 - %0:gpr(s16) = G_LOAD %2(p0) :: (invariant load (s16) from %fixed-stack.1, align 16) - %3:gpr(p0) = G_FRAME_INDEX %fixed-stack.0 - %1:gpr(s16) = G_LOAD %3(p0) :: (invariant load (s16) from %fixed-stack.0, align 4) - %4:gpr(s16) = G_UREM %0, %1 - $ax = COPY %4(s16) - RET 0, implicit $ax - -... ---- -name: test_urem_i32 -alignment: 16 -exposesReturnsTwice: false -legalized: true -regBankSelected: true -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: gpr, preferred-register: '' } - - { id: 1, class: gpr, preferred-register: '' } - - { id: 2, class: gpr, preferred-register: '' } - - { id: 3, class: gpr, preferred-register: '' } - - { id: 4, class: gpr, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 4 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: - - { id: 0, type: default, offset: 4, size: 4, alignment: 4, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } - - { id: 1, type: default, offset: 0, size: 4, alignment: 16, stack-id: default, - isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, - debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } -stack: -constants: -body: | - bb.1 (%ir-block.0): - ; CHECK-LABEL: name: test_urem_i32 - ; CHECK: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load (s32) from %fixed-stack.0, align 16) - ; CHECK: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load (s32) from %fixed-stack.1) - ; CHECK: $eax = COPY [[MOV32rm]] - ; CHECK: [[MOV32r0_:%[0-9]+]]:gr32 = MOV32r0 implicit-def $eflags - ; CHECK: $edx = COPY [[MOV32r0_]] - ; CHECK: DIV32r [[MOV32rm1]], implicit-def $eax, implicit-def $edx, implicit-def $eflags, implicit $eax, implicit $edx - ; CHECK: [[COPY:%[0-9]+]]:gr32 = COPY $edx - ; CHECK: $eax = COPY [[COPY]] - ; CHECK: RET 0, implicit $eax - %2:gpr(p0) = G_FRAME_INDEX %fixed-stack.1 - %0:gpr(s32) = G_LOAD %2(p0) :: (invariant load (s32) from %fixed-stack.1, align 16) - %3:gpr(p0) = G_FRAME_INDEX %fixed-stack.0 - %1:gpr(s32) = G_LOAD %3(p0) :: (invariant load (s32) from %fixed-stack.0, align 4) - %4:gpr(s32) = G_UREM %0, %1 - $eax = COPY %4(s32) - RET 0, implicit $eax - -... diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86_64-legalize-sdiv.mir b/llvm/test/CodeGen/X86/GlobalISel/x86_64-legalize-sdiv.mir deleted file mode 100644 index faccc3750c80..000000000000 --- a/llvm/test/CodeGen/X86/GlobalISel/x86_64-legalize-sdiv.mir +++ /dev/null @@ -1,145 +0,0 @@ -# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple=x86_64-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s - ---- | - ; ModuleID = 'sdiv.ll' - source_filename = "sdiv.ll" - target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" - - define i8 @test_sdiv_i8(i8 %arg1, i8 %arg2) { - %res = sdiv i8 %arg1, %arg2 - ret i8 %res - } - - define i16 @test_sdiv_i16(i16 %arg1, i16 %arg2) { - %res = sdiv i16 %arg1, %arg2 - ret i16 %res - } - - define i32 @test_sdiv_i32(i32 %arg1, i32 %arg2) { - %res = sdiv i32 %arg1, %arg2 - ret i32 %res - } - - define i64 @test_sdiv_i64(i64 %arg1, i64 %arg2) { - %res = sdiv i64 %arg1, %arg2 - ret i64 %res - } - -... ---- -name: test_sdiv_i8 -alignment: 16 -tracksRegLiveness: true -registers: - - { id: 0, class: _ } - - { id: 1, class: _ } - - { id: 2, class: _ } - - { id: 3, class: _ } - - { id: 4, class: _ } -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_sdiv_i8 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $edi - ; CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32) - ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi - ; CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32) - ; CHECK: [[SDIV:%[0-9]+]]:_(s8) = G_SDIV [[TRUNC]], [[TRUNC1]] - ; CHECK: $al = COPY [[SDIV]](s8) - ; CHECK: RET 0, implicit $al - %2:_(s32) = COPY $edi - %0:_(s8) = G_TRUNC %2(s32) - %3:_(s32) = COPY $esi - %1:_(s8) = G_TRUNC %3(s32) - %4:_(s8) = G_SDIV %0, %1 - $al = COPY %4(s8) - RET 0, implicit $al - -... ---- -name: test_sdiv_i16 -alignment: 16 -tracksRegLiveness: true -registers: - - { id: 0, class: _ } - - { id: 1, class: _ } - - { id: 2, class: _ } - - { id: 3, class: _ } - - { id: 4, class: _ } -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_sdiv_i16 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $edi - ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32) - ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi - ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32) - ; CHECK: [[SDIV:%[0-9]+]]:_(s16) = G_SDIV [[TRUNC]], [[TRUNC1]] - ; CHECK: $ax = COPY [[SDIV]](s16) - ; CHECK: RET 0, implicit $ax - %2:_(s32) = COPY $edi - %0:_(s16) = G_TRUNC %2(s32) - %3:_(s32) = COPY $esi - %1:_(s16) = G_TRUNC %3(s32) - %4:_(s16) = G_SDIV %0, %1 - $ax = COPY %4(s16) - RET 0, implicit $ax - -... ---- -name: test_sdiv_i32 -alignment: 16 -tracksRegLiveness: true -registers: - - { id: 0, class: _ } - - { id: 1, class: _ } - - { id: 2, class: _ } -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_sdiv_i32 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $edi - ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi - ; CHECK: [[SDIV:%[0-9]+]]:_(s32) = G_SDIV [[COPY]], [[COPY1]] - ; CHECK: $eax = COPY [[SDIV]](s32) - ; CHECK: RET 0, implicit $eax - %0:_(s32) = COPY $edi - %1:_(s32) = COPY $esi - %2:_(s32) = G_SDIV %0, %1 - $eax = COPY %2(s32) - RET 0, implicit $eax - -... ---- -name: test_sdiv_i64 -alignment: 16 -tracksRegLiveness: true -registers: - - { id: 0, class: _ } - - { id: 1, class: _ } - - { id: 2, class: _ } -body: | - bb.1 (%ir-block.0): - liveins: $rdi, $rsi - - ; CHECK-LABEL: name: test_sdiv_i64 - ; CHECK: liveins: $rdi, $rsi - ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $rdi - ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $rsi - ; CHECK: [[SDIV:%[0-9]+]]:_(s64) = G_SDIV [[COPY]], [[COPY1]] - ; CHECK: $rax = COPY [[SDIV]](s64) - ; CHECK: RET 0, implicit $rax - %0:_(s64) = COPY $rdi - %1:_(s64) = COPY $rsi - %2:_(s64) = G_SDIV %0, %1 - $rax = COPY %2(s64) - RET 0, implicit $rax - -... diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86_64-legalize-srem.mir b/llvm/test/CodeGen/X86/GlobalISel/x86_64-legalize-srem.mir deleted file mode 100644 index f02442f2b850..000000000000 --- a/llvm/test/CodeGen/X86/GlobalISel/x86_64-legalize-srem.mir +++ /dev/null @@ -1,253 +0,0 @@ -# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple=x86_64-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s - ---- | - ; ModuleID = 'srem.ll' - source_filename = "srem.ll" - target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" - - define i8 @test_srem_i8(i8 %arg1, i8 %arg2) { - %res = srem i8 %arg1, %arg2 - ret i8 %res - } - - define i16 @test_srem_i16(i16 %arg1, i16 %arg2) { - %res = srem i16 %arg1, %arg2 - ret i16 %res - } - - define i32 @test_srem_i32(i32 %arg1, i32 %arg2) { - %res = srem i32 %arg1, %arg2 - ret i32 %res - } - - define i64 @test_srem_i64(i64 %arg1, i64 %arg2) { - %res = srem i64 %arg1, %arg2 - ret i64 %res - } - -... ---- -name: test_srem_i8 -alignment: 16 -exposesReturnsTwice: false -legalized: false -regBankSelected: false -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: _, preferred-register: '' } - - { id: 1, class: _, preferred-register: '' } - - { id: 2, class: _, preferred-register: '' } - - { id: 3, class: _, preferred-register: '' } - - { id: 4, class: _, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_srem_i8 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $edi - ; CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32) - ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi - ; CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32) - ; CHECK: [[SREM:%[0-9]+]]:_(s8) = G_SREM [[TRUNC]], [[TRUNC1]] - ; CHECK: $al = COPY [[SREM]](s8) - ; CHECK: RET 0, implicit $al - %2:_(s32) = COPY $edi - %0:_(s8) = G_TRUNC %2(s32) - %3:_(s32) = COPY $esi - %1:_(s8) = G_TRUNC %3(s32) - %4:_(s8) = G_SREM %0, %1 - $al = COPY %4(s8) - RET 0, implicit $al - -... ---- -name: test_srem_i16 -alignment: 16 -exposesReturnsTwice: false -legalized: false -regBankSelected: false -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: _, preferred-register: '' } - - { id: 1, class: _, preferred-register: '' } - - { id: 2, class: _, preferred-register: '' } - - { id: 3, class: _, preferred-register: '' } - - { id: 4, class: _, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_srem_i16 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $edi - ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32) - ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi - ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32) - ; CHECK: [[SREM:%[0-9]+]]:_(s16) = G_SREM [[TRUNC]], [[TRUNC1]] - ; CHECK: $ax = COPY [[SREM]](s16) - ; CHECK: RET 0, implicit $ax - %2:_(s32) = COPY $edi - %0:_(s16) = G_TRUNC %2(s32) - %3:_(s32) = COPY $esi - %1:_(s16) = G_TRUNC %3(s32) - %4:_(s16) = G_SREM %0, %1 - $ax = COPY %4(s16) - RET 0, implicit $ax - -... ---- -name: test_srem_i32 -alignment: 16 -exposesReturnsTwice: false -legalized: false -regBankSelected: false -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: _, preferred-register: '' } - - { id: 1, class: _, preferred-register: '' } - - { id: 2, class: _, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_srem_i32 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $edi - ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi - ; CHECK: [[SREM:%[0-9]+]]:_(s32) = G_SREM [[COPY]], [[COPY1]] - ; CHECK: $eax = COPY [[SREM]](s32) - ; CHECK: RET 0, implicit $eax - %0:_(s32) = COPY $edi - %1:_(s32) = COPY $esi - %2:_(s32) = G_SREM %0, %1 - $eax = COPY %2(s32) - RET 0, implicit $eax - -... ---- -name: test_srem_i64 -alignment: 16 -exposesReturnsTwice: false -legalized: false -regBankSelected: false -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: _, preferred-register: '' } - - { id: 1, class: _, preferred-register: '' } - - { id: 2, class: _, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $rdi, $rsi - - ; CHECK-LABEL: name: test_srem_i64 - ; CHECK: liveins: $rdi, $rsi - ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $rdi - ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $rsi - ; CHECK: [[SREM:%[0-9]+]]:_(s64) = G_SREM [[COPY]], [[COPY1]] - ; CHECK: $rax = COPY [[SREM]](s64) - ; CHECK: RET 0, implicit $rax - %0:_(s64) = COPY $rdi - %1:_(s64) = COPY $rsi - %2:_(s64) = G_SREM %0, %1 - $rax = COPY %2(s64) - RET 0, implicit $rax - -... diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86_64-legalize-udiv.mir b/llvm/test/CodeGen/X86/GlobalISel/x86_64-legalize-udiv.mir deleted file mode 100644 index 35073e2bcb1b..000000000000 --- a/llvm/test/CodeGen/X86/GlobalISel/x86_64-legalize-udiv.mir +++ /dev/null @@ -1,253 +0,0 @@ -# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple=x86_64-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s - ---- | - ; ModuleID = 'udiv.ll' - source_filename = "udiv.ll" - target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" - - define i8 @test_udiv_i8(i8 %arg1, i8 %arg2) { - %res = udiv i8 %arg1, %arg2 - ret i8 %res - } - - define i16 @test_udiv_i16(i16 %arg1, i16 %arg2) { - %res = udiv i16 %arg1, %arg2 - ret i16 %res - } - - define i32 @test_udiv_i32(i32 %arg1, i32 %arg2) { - %res = udiv i32 %arg1, %arg2 - ret i32 %res - } - - define i64 @test_udiv_i64(i64 %arg1, i64 %arg2) { - %res = udiv i64 %arg1, %arg2 - ret i64 %res - } - -... ---- -name: test_udiv_i8 -alignment: 16 -exposesReturnsTwice: false -legalized: false -regBankSelected: false -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: _, preferred-register: '' } - - { id: 1, class: _, preferred-register: '' } - - { id: 2, class: _, preferred-register: '' } - - { id: 3, class: _, preferred-register: '' } - - { id: 4, class: _, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_udiv_i8 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $edi - ; CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32) - ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi - ; CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32) - ; CHECK: [[UDIV:%[0-9]+]]:_(s8) = G_UDIV [[TRUNC]], [[TRUNC1]] - ; CHECK: $al = COPY [[UDIV]](s8) - ; CHECK: RET 0, implicit $al - %2:_(s32) = COPY $edi - %0:_(s8) = G_TRUNC %2(s32) - %3:_(s32) = COPY $esi - %1:_(s8) = G_TRUNC %3(s32) - %4:_(s8) = G_UDIV %0, %1 - $al = COPY %4(s8) - RET 0, implicit $al - -... ---- -name: test_udiv_i16 -alignment: 16 -exposesReturnsTwice: false -legalized: false -regBankSelected: false -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: _, preferred-register: '' } - - { id: 1, class: _, preferred-register: '' } - - { id: 2, class: _, preferred-register: '' } - - { id: 3, class: _, preferred-register: '' } - - { id: 4, class: _, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_udiv_i16 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $edi - ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32) - ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi - ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32) - ; CHECK: [[UDIV:%[0-9]+]]:_(s16) = G_UDIV [[TRUNC]], [[TRUNC1]] - ; CHECK: $ax = COPY [[UDIV]](s16) - ; CHECK: RET 0, implicit $ax - %2:_(s32) = COPY $edi - %0:_(s16) = G_TRUNC %2(s32) - %3:_(s32) = COPY $esi - %1:_(s16) = G_TRUNC %3(s32) - %4:_(s16) = G_UDIV %0, %1 - $ax = COPY %4(s16) - RET 0, implicit $ax - -... ---- -name: test_udiv_i32 -alignment: 16 -exposesReturnsTwice: false -legalized: false -regBankSelected: false -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: _, preferred-register: '' } - - { id: 1, class: _, preferred-register: '' } - - { id: 2, class: _, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_udiv_i32 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $edi - ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi - ; CHECK: [[UDIV:%[0-9]+]]:_(s32) = G_UDIV [[COPY]], [[COPY1]] - ; CHECK: $eax = COPY [[UDIV]](s32) - ; CHECK: RET 0, implicit $eax - %0:_(s32) = COPY $edi - %1:_(s32) = COPY $esi - %2:_(s32) = G_UDIV %0, %1 - $eax = COPY %2(s32) - RET 0, implicit $eax - -... ---- -name: test_udiv_i64 -alignment: 16 -exposesReturnsTwice: false -legalized: false -regBankSelected: false -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: _, preferred-register: '' } - - { id: 1, class: _, preferred-register: '' } - - { id: 2, class: _, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $rdi, $rsi - - ; CHECK-LABEL: name: test_udiv_i64 - ; CHECK: liveins: $rdi, $rsi - ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $rdi - ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $rsi - ; CHECK: [[UDIV:%[0-9]+]]:_(s64) = G_UDIV [[COPY]], [[COPY1]] - ; CHECK: $rax = COPY [[UDIV]](s64) - ; CHECK: RET 0, implicit $rax - %0:_(s64) = COPY $rdi - %1:_(s64) = COPY $rsi - %2:_(s64) = G_UDIV %0, %1 - $rax = COPY %2(s64) - RET 0, implicit $rax - -... diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86_64-legalize-urem.mir b/llvm/test/CodeGen/X86/GlobalISel/x86_64-legalize-urem.mir deleted file mode 100644 index c0ca5ae74fc3..000000000000 --- a/llvm/test/CodeGen/X86/GlobalISel/x86_64-legalize-urem.mir +++ /dev/null @@ -1,253 +0,0 @@ -# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple=x86_64-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s - ---- | - ; ModuleID = 'urem.ll' - source_filename = "urem.ll" - target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" - - define i8 @test_urem_i8(i8 %arg1, i8 %arg2) { - %res = urem i8 %arg1, %arg2 - ret i8 %res - } - - define i16 @test_urem_i16(i16 %arg1, i16 %arg2) { - %res = urem i16 %arg1, %arg2 - ret i16 %res - } - - define i32 @test_urem_i32(i32 %arg1, i32 %arg2) { - %res = urem i32 %arg1, %arg2 - ret i32 %res - } - - define i64 @test_urem_i64(i64 %arg1, i64 %arg2) { - %res = urem i64 %arg1, %arg2 - ret i64 %res - } - -... ---- -name: test_urem_i8 -alignment: 16 -exposesReturnsTwice: false -legalized: false -regBankSelected: false -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: _, preferred-register: '' } - - { id: 1, class: _, preferred-register: '' } - - { id: 2, class: _, preferred-register: '' } - - { id: 3, class: _, preferred-register: '' } - - { id: 4, class: _, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_urem_i8 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $edi - ; CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32) - ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi - ; CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32) - ; CHECK: [[UREM:%[0-9]+]]:_(s8) = G_UREM [[TRUNC]], [[TRUNC1]] - ; CHECK: $al = COPY [[UREM]](s8) - ; CHECK: RET 0, implicit $al - %2:_(s32) = COPY $edi - %0:_(s8) = G_TRUNC %2(s32) - %3:_(s32) = COPY $esi - %1:_(s8) = G_TRUNC %3(s32) - %4:_(s8) = G_UREM %0, %1 - $al = COPY %4(s8) - RET 0, implicit $al - -... ---- -name: test_urem_i16 -alignment: 16 -exposesReturnsTwice: false -legalized: false -regBankSelected: false -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: _, preferred-register: '' } - - { id: 1, class: _, preferred-register: '' } - - { id: 2, class: _, preferred-register: '' } - - { id: 3, class: _, preferred-register: '' } - - { id: 4, class: _, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_urem_i16 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $edi - ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32) - ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi - ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32) - ; CHECK: [[UREM:%[0-9]+]]:_(s16) = G_UREM [[TRUNC]], [[TRUNC1]] - ; CHECK: $ax = COPY [[UREM]](s16) - ; CHECK: RET 0, implicit $ax - %2:_(s32) = COPY $edi - %0:_(s16) = G_TRUNC %2(s32) - %3:_(s32) = COPY $esi - %1:_(s16) = G_TRUNC %3(s32) - %4:_(s16) = G_UREM %0, %1 - $ax = COPY %4(s16) - RET 0, implicit $ax - -... ---- -name: test_urem_i32 -alignment: 16 -exposesReturnsTwice: false -legalized: false -regBankSelected: false -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: _, preferred-register: '' } - - { id: 1, class: _, preferred-register: '' } - - { id: 2, class: _, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_urem_i32 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $edi - ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi - ; CHECK: [[UREM:%[0-9]+]]:_(s32) = G_UREM [[COPY]], [[COPY1]] - ; CHECK: $eax = COPY [[UREM]](s32) - ; CHECK: RET 0, implicit $eax - %0:_(s32) = COPY $edi - %1:_(s32) = COPY $esi - %2:_(s32) = G_UREM %0, %1 - $eax = COPY %2(s32) - RET 0, implicit $eax - -... ---- -name: test_urem_i64 -alignment: 16 -exposesReturnsTwice: false -legalized: false -regBankSelected: false -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: _, preferred-register: '' } - - { id: 1, class: _, preferred-register: '' } - - { id: 2, class: _, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $rdi, $rsi - - ; CHECK-LABEL: name: test_urem_i64 - ; CHECK: liveins: $rdi, $rsi - ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $rdi - ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $rsi - ; CHECK: [[UREM:%[0-9]+]]:_(s64) = G_UREM [[COPY]], [[COPY1]] - ; CHECK: $rax = COPY [[UREM]](s64) - ; CHECK: RET 0, implicit $rax - %0:_(s64) = COPY $rdi - %1:_(s64) = COPY $rsi - %2:_(s64) = G_UREM %0, %1 - $rax = COPY %2(s64) - RET 0, implicit $rax - -... diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86_64-select-sdiv.mir b/llvm/test/CodeGen/X86/GlobalISel/x86_64-select-sdiv.mir deleted file mode 100644 index d3a1608be52a..000000000000 --- a/llvm/test/CodeGen/X86/GlobalISel/x86_64-select-sdiv.mir +++ /dev/null @@ -1,164 +0,0 @@ -# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple=x86_64-linux-gnu -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s - ---- | - ; ModuleID = 'sdiv.ll' - source_filename = "sdiv.ll" - target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" - - define i8 @test_sdiv_i8(i8 %arg1, i8 %arg2) { - %res = sdiv i8 %arg1, %arg2 - ret i8 %res - } - - define i16 @test_sdiv_i16(i16 %arg1, i16 %arg2) { - %res = sdiv i16 %arg1, %arg2 - ret i16 %res - } - - define i32 @test_sdiv_i32(i32 %arg1, i32 %arg2) { - %res = sdiv i32 %arg1, %arg2 - ret i32 %res - } - - define i64 @test_sdiv_i64(i64 %arg1, i64 %arg2) { - %res = sdiv i64 %arg1, %arg2 - ret i64 %res - } - -... ---- -name: test_sdiv_i8 -alignment: 16 -legalized: true -regBankSelected: true -tracksRegLiveness: true -registers: - - { id: 0, class: gpr } - - { id: 1, class: gpr } - - { id: 2, class: gpr } - - { id: 3, class: gpr } - - { id: 4, class: gpr } -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_sdiv_i8 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:gr32 = COPY $edi - ; CHECK: [[COPY1:%[0-9]+]]:gr8 = COPY [[COPY]].sub_8bit - ; CHECK: [[COPY2:%[0-9]+]]:gr32 = COPY $esi - ; CHECK: [[COPY3:%[0-9]+]]:gr8 = COPY [[COPY2]].sub_8bit - ; CHECK: $ax = MOVSX16rr8 [[COPY1]] - ; CHECK: IDIV8r [[COPY3]], implicit-def $al, implicit-def $ah, implicit-def $eflags, implicit $ax - ; CHECK: [[COPY4:%[0-9]+]]:gr8 = COPY $al - ; CHECK: $al = COPY [[COPY4]] - ; CHECK: RET 0, implicit $al - %2:gpr(s32) = COPY $edi - %0:gpr(s8) = G_TRUNC %2(s32) - %3:gpr(s32) = COPY $esi - %1:gpr(s8) = G_TRUNC %3(s32) - %4:gpr(s8) = G_SDIV %0, %1 - $al = COPY %4(s8) - RET 0, implicit $al - -... ---- -name: test_sdiv_i16 -alignment: 16 -legalized: true -regBankSelected: true -tracksRegLiveness: true -registers: - - { id: 0, class: gpr } - - { id: 1, class: gpr } - - { id: 2, class: gpr } - - { id: 3, class: gpr } - - { id: 4, class: gpr } -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_sdiv_i16 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:gr32 = COPY $edi - ; CHECK: [[COPY1:%[0-9]+]]:gr16 = COPY [[COPY]].sub_16bit - ; CHECK: [[COPY2:%[0-9]+]]:gr32 = COPY $esi - ; CHECK: [[COPY3:%[0-9]+]]:gr16 = COPY [[COPY2]].sub_16bit - ; CHECK: $ax = COPY [[COPY1]] - ; CHECK: CWD implicit-def $ax, implicit-def $dx, implicit $ax - ; CHECK: IDIV16r [[COPY3]], implicit-def $ax, implicit-def $dx, implicit-def $eflags, implicit $ax, implicit $dx - ; CHECK: [[COPY4:%[0-9]+]]:gr16 = COPY $ax - ; CHECK: $ax = COPY [[COPY4]] - ; CHECK: RET 0, implicit $ax - %2:gpr(s32) = COPY $edi - %0:gpr(s16) = G_TRUNC %2(s32) - %3:gpr(s32) = COPY $esi - %1:gpr(s16) = G_TRUNC %3(s32) - %4:gpr(s16) = G_SDIV %0, %1 - $ax = COPY %4(s16) - RET 0, implicit $ax - -... ---- -name: test_sdiv_i32 -alignment: 16 -legalized: true -regBankSelected: true -tracksRegLiveness: true -registers: - - { id: 0, class: gpr } - - { id: 1, class: gpr } - - { id: 2, class: gpr } -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_sdiv_i32 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:gr32 = COPY $edi - ; CHECK: [[COPY1:%[0-9]+]]:gr32 = COPY $esi - ; CHECK: $eax = COPY [[COPY]] - ; CHECK: CDQ implicit-def $eax, implicit-def $edx, implicit $eax - ; CHECK: IDIV32r [[COPY1]], implicit-def $eax, implicit-def $edx, implicit-def $eflags, implicit $eax, implicit $edx - ; CHECK: [[COPY2:%[0-9]+]]:gr32 = COPY $eax - ; CHECK: $eax = COPY [[COPY2]] - ; CHECK: RET 0, implicit $eax - %0:gpr(s32) = COPY $edi - %1:gpr(s32) = COPY $esi - %2:gpr(s32) = G_SDIV %0, %1 - $eax = COPY %2(s32) - RET 0, implicit $eax - -... ---- -name: test_sdiv_i64 -alignment: 16 -legalized: true -regBankSelected: true -tracksRegLiveness: true -registers: - - { id: 0, class: gpr } - - { id: 1, class: gpr } - - { id: 2, class: gpr } -body: | - bb.1 (%ir-block.0): - liveins: $rdi, $rsi - - ; CHECK-LABEL: name: test_sdiv_i64 - ; CHECK: liveins: $rdi, $rsi - ; CHECK: [[COPY:%[0-9]+]]:gr64 = COPY $rdi - ; CHECK: [[COPY1:%[0-9]+]]:gr64 = COPY $rsi - ; CHECK: $rax = COPY [[COPY]] - ; CHECK: CQO implicit-def $rax, implicit-def $rdx, implicit $rax - ; CHECK: IDIV64r [[COPY1]], implicit-def $rax, implicit-def $rdx, implicit-def $eflags, implicit $rax, implicit $rdx - ; CHECK: [[COPY2:%[0-9]+]]:gr64 = COPY $rax - ; CHECK: $rax = COPY [[COPY2]] - ; CHECK: RET 0, implicit $rax - %0:gpr(s64) = COPY $rdi - %1:gpr(s64) = COPY $rsi - %2:gpr(s64) = G_SDIV %0, %1 - $rax = COPY %2(s64) - RET 0, implicit $rax - -... diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86_64-select-srem.mir b/llvm/test/CodeGen/X86/GlobalISel/x86_64-select-srem.mir deleted file mode 100644 index 0988883145bc..000000000000 --- a/llvm/test/CodeGen/X86/GlobalISel/x86_64-select-srem.mir +++ /dev/null @@ -1,270 +0,0 @@ -# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple=x86_64-linux-gnu -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s - ---- | - ; ModuleID = 'srem.ll' - source_filename = "srem.ll" - target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" - - define i8 @test_srem_i8(i8 %arg1, i8 %arg2) { - %res = srem i8 %arg1, %arg2 - ret i8 %res - } - - define i16 @test_srem_i16(i16 %arg1, i16 %arg2) { - %res = srem i16 %arg1, %arg2 - ret i16 %res - } - - define i32 @test_srem_i32(i32 %arg1, i32 %arg2) { - %res = srem i32 %arg1, %arg2 - ret i32 %res - } - - define i64 @test_srem_i64(i64 %arg1, i64 %arg2) { - %res = srem i64 %arg1, %arg2 - ret i64 %res - } - -... ---- -name: test_srem_i8 -alignment: 16 -exposesReturnsTwice: false -legalized: true -regBankSelected: true -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: gpr, preferred-register: '' } - - { id: 1, class: gpr, preferred-register: '' } - - { id: 2, class: gpr, preferred-register: '' } - - { id: 3, class: gpr, preferred-register: '' } - - { id: 4, class: gpr, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_srem_i8 - ; CHECK: liveins: $edi, $esi - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $edi - ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr8 = COPY [[COPY]].sub_8bit - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY $esi - ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr8 = COPY [[COPY2]].sub_8bit - ; CHECK-NEXT: $ax = MOVSX16rr8 [[COPY1]] - ; CHECK-NEXT: IDIV8r [[COPY3]], implicit-def $al, implicit-def $ah, implicit-def $eflags, implicit $ax - ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr16 = COPY $ax - ; CHECK-NEXT: [[SHR16ri:%[0-9]+]]:gr16 = SHR16ri [[COPY4]], 8, implicit-def $eflags - ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gr8 = SUBREG_TO_REG 0, [[SHR16ri]], %subreg.sub_8bit - ; CHECK-NEXT: $al = COPY [[SUBREG_TO_REG]] - ; CHECK-NEXT: RET 0, implicit $al - %2:gpr(s32) = COPY $edi - %0:gpr(s8) = G_TRUNC %2(s32) - %3:gpr(s32) = COPY $esi - %1:gpr(s8) = G_TRUNC %3(s32) - %4:gpr(s8) = G_SREM %0, %1 - $al = COPY %4(s8) - RET 0, implicit $al - -... ---- -name: test_srem_i16 -alignment: 16 -exposesReturnsTwice: false -legalized: true -regBankSelected: true -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: gpr, preferred-register: '' } - - { id: 1, class: gpr, preferred-register: '' } - - { id: 2, class: gpr, preferred-register: '' } - - { id: 3, class: gpr, preferred-register: '' } - - { id: 4, class: gpr, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_srem_i16 - ; CHECK: liveins: $edi, $esi - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $edi - ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr16 = COPY [[COPY]].sub_16bit - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY $esi - ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr16 = COPY [[COPY2]].sub_16bit - ; CHECK-NEXT: $ax = COPY [[COPY1]] - ; CHECK-NEXT: CWD implicit-def $ax, implicit-def $dx, implicit $ax - ; CHECK-NEXT: IDIV16r [[COPY3]], implicit-def $ax, implicit-def $dx, implicit-def $eflags, implicit $ax, implicit $dx - ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr16 = COPY $dx - ; CHECK-NEXT: $ax = COPY [[COPY4]] - ; CHECK-NEXT: RET 0, implicit $ax - %2:gpr(s32) = COPY $edi - %0:gpr(s16) = G_TRUNC %2(s32) - %3:gpr(s32) = COPY $esi - %1:gpr(s16) = G_TRUNC %3(s32) - %4:gpr(s16) = G_SREM %0, %1 - $ax = COPY %4(s16) - RET 0, implicit $ax - -... ---- -name: test_srem_i32 -alignment: 16 -exposesReturnsTwice: false -legalized: true -regBankSelected: true -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: gpr, preferred-register: '' } - - { id: 1, class: gpr, preferred-register: '' } - - { id: 2, class: gpr, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_srem_i32 - ; CHECK: liveins: $edi, $esi - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $edi - ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $esi - ; CHECK-NEXT: $eax = COPY [[COPY]] - ; CHECK-NEXT: CDQ implicit-def $eax, implicit-def $edx, implicit $eax - ; CHECK-NEXT: IDIV32r [[COPY1]], implicit-def $eax, implicit-def $edx, implicit-def $eflags, implicit $eax, implicit $edx - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY $edx - ; CHECK-NEXT: $eax = COPY [[COPY2]] - ; CHECK-NEXT: RET 0, implicit $eax - %0:gpr(s32) = COPY $edi - %1:gpr(s32) = COPY $esi - %2:gpr(s32) = G_SREM %0, %1 - $eax = COPY %2(s32) - RET 0, implicit $eax - -... ---- -name: test_srem_i64 -alignment: 16 -exposesReturnsTwice: false -legalized: true -regBankSelected: true -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: gpr, preferred-register: '' } - - { id: 1, class: gpr, preferred-register: '' } - - { id: 2, class: gpr, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $rdi, $rsi - - ; CHECK-LABEL: name: test_srem_i64 - ; CHECK: liveins: $rdi, $rsi - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi - ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rsi - ; CHECK-NEXT: $rax = COPY [[COPY]] - ; CHECK-NEXT: CQO implicit-def $rax, implicit-def $rdx, implicit $rax - ; CHECK-NEXT: IDIV64r [[COPY1]], implicit-def $rax, implicit-def $rdx, implicit-def $eflags, implicit $rax, implicit $rdx - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64 = COPY $rdx - ; CHECK-NEXT: $rax = COPY [[COPY2]] - ; CHECK-NEXT: RET 0, implicit $rax - %0:gpr(s64) = COPY $rdi - %1:gpr(s64) = COPY $rsi - %2:gpr(s64) = G_SREM %0, %1 - $rax = COPY %2(s64) - RET 0, implicit $rax - -... diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86_64-select-udiv.mir b/llvm/test/CodeGen/X86/GlobalISel/x86_64-select-udiv.mir deleted file mode 100644 index 71c03fd6e28f..000000000000 --- a/llvm/test/CodeGen/X86/GlobalISel/x86_64-select-udiv.mir +++ /dev/null @@ -1,267 +0,0 @@ -# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple=x86_64-linux-gnu -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s - ---- | - ; ModuleID = 'udiv.ll' - source_filename = "udiv.ll" - target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" - - define i8 @test_udiv_i8(i8 %arg1, i8 %arg2) { - %res = udiv i8 %arg1, %arg2 - ret i8 %res - } - - define i16 @test_udiv_i16(i16 %arg1, i16 %arg2) { - %res = udiv i16 %arg1, %arg2 - ret i16 %res - } - - define i32 @test_udiv_i32(i32 %arg1, i32 %arg2) { - %res = udiv i32 %arg1, %arg2 - ret i32 %res - } - - define i64 @test_udiv_i64(i64 %arg1, i64 %arg2) { - %res = udiv i64 %arg1, %arg2 - ret i64 %res - } - -... ---- -name: test_udiv_i8 -alignment: 16 -exposesReturnsTwice: false -legalized: true -regBankSelected: true -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: gpr, preferred-register: '' } - - { id: 1, class: gpr, preferred-register: '' } - - { id: 2, class: gpr, preferred-register: '' } - - { id: 3, class: gpr, preferred-register: '' } - - { id: 4, class: gpr, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_udiv_i8 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:gr32 = COPY $edi - ; CHECK: [[COPY1:%[0-9]+]]:gr8 = COPY [[COPY]].sub_8bit - ; CHECK: [[COPY2:%[0-9]+]]:gr32 = COPY $esi - ; CHECK: [[COPY3:%[0-9]+]]:gr8 = COPY [[COPY2]].sub_8bit - ; CHECK: $ax = MOVZX16rr8 [[COPY1]] - ; CHECK: DIV8r [[COPY3]], implicit-def $al, implicit-def $ah, implicit-def $eflags, implicit $ax - ; CHECK: [[COPY4:%[0-9]+]]:gr8 = COPY $al - ; CHECK: $al = COPY [[COPY4]] - ; CHECK: RET 0, implicit $al - %2:gpr(s32) = COPY $edi - %0:gpr(s8) = G_TRUNC %2(s32) - %3:gpr(s32) = COPY $esi - %1:gpr(s8) = G_TRUNC %3(s32) - %4:gpr(s8) = G_UDIV %0, %1 - $al = COPY %4(s8) - RET 0, implicit $al - -... ---- -name: test_udiv_i16 -alignment: 16 -exposesReturnsTwice: false -legalized: true -regBankSelected: true -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: gpr, preferred-register: '' } - - { id: 1, class: gpr, preferred-register: '' } - - { id: 2, class: gpr, preferred-register: '' } - - { id: 3, class: gpr, preferred-register: '' } - - { id: 4, class: gpr, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_udiv_i16 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:gr32 = COPY $edi - ; CHECK: [[COPY1:%[0-9]+]]:gr16 = COPY [[COPY]].sub_16bit - ; CHECK: [[COPY2:%[0-9]+]]:gr32 = COPY $esi - ; CHECK: [[COPY3:%[0-9]+]]:gr16 = COPY [[COPY2]].sub_16bit - ; CHECK: $ax = COPY [[COPY1]] - ; CHECK: [[MOV32r0_:%[0-9]+]]:gr32 = MOV32r0 implicit-def $eflags - ; CHECK: $dx = COPY [[MOV32r0_]].sub_16bit - ; CHECK: DIV16r [[COPY3]], implicit-def $ax, implicit-def $dx, implicit-def $eflags, implicit $ax, implicit $dx - ; CHECK: [[COPY4:%[0-9]+]]:gr16 = COPY $ax - ; CHECK: $ax = COPY [[COPY4]] - ; CHECK: RET 0, implicit $ax - %2:gpr(s32) = COPY $edi - %0:gpr(s16) = G_TRUNC %2(s32) - %3:gpr(s32) = COPY $esi - %1:gpr(s16) = G_TRUNC %3(s32) - %4:gpr(s16) = G_UDIV %0, %1 - $ax = COPY %4(s16) - RET 0, implicit $ax - -... ---- -name: test_udiv_i32 -alignment: 16 -exposesReturnsTwice: false -legalized: true -regBankSelected: true -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: gpr, preferred-register: '' } - - { id: 1, class: gpr, preferred-register: '' } - - { id: 2, class: gpr, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_udiv_i32 - ; CHECK: liveins: $edi, $esi - ; CHECK: [[COPY:%[0-9]+]]:gr32 = COPY $edi - ; CHECK: [[COPY1:%[0-9]+]]:gr32 = COPY $esi - ; CHECK: $eax = COPY [[COPY]] - ; CHECK: [[MOV32r0_:%[0-9]+]]:gr32 = MOV32r0 implicit-def $eflags - ; CHECK: $edx = COPY [[MOV32r0_]] - ; CHECK: DIV32r [[COPY1]], implicit-def $eax, implicit-def $edx, implicit-def $eflags, implicit $eax, implicit $edx - ; CHECK: [[COPY2:%[0-9]+]]:gr32 = COPY $eax - ; CHECK: $eax = COPY [[COPY2]] - ; CHECK: RET 0, implicit $eax - %0:gpr(s32) = COPY $edi - %1:gpr(s32) = COPY $esi - %2:gpr(s32) = G_UDIV %0, %1 - $eax = COPY %2(s32) - RET 0, implicit $eax - -... ---- -name: test_udiv_i64 -alignment: 16 -exposesReturnsTwice: false -legalized: true -regBankSelected: true -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: gpr, preferred-register: '' } - - { id: 1, class: gpr, preferred-register: '' } - - { id: 2, class: gpr, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $rdi, $rsi - - ; CHECK-LABEL: name: test_udiv_i64 - ; CHECK: liveins: $rdi, $rsi - ; CHECK: [[COPY:%[0-9]+]]:gr64 = COPY $rdi - ; CHECK: [[COPY1:%[0-9]+]]:gr64 = COPY $rsi - ; CHECK: $rax = COPY [[COPY]] - ; CHECK: [[MOV32r0_:%[0-9]+]]:gr32 = MOV32r0 implicit-def $eflags - ; CHECK: $rdx = SUBREG_TO_REG 0, [[MOV32r0_]], %subreg.sub_32bit - ; CHECK: DIV64r [[COPY1]], implicit-def $rax, implicit-def $rdx, implicit-def $eflags, implicit $rax, implicit $rdx - ; CHECK: [[COPY2:%[0-9]+]]:gr64 = COPY $rax - ; CHECK: $rax = COPY [[COPY2]] - ; CHECK: RET 0, implicit $rax - %0:gpr(s64) = COPY $rdi - %1:gpr(s64) = COPY $rsi - %2:gpr(s64) = G_UDIV %0, %1 - $rax = COPY %2(s64) - RET 0, implicit $rax - -... diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86_64-select-urem.mir b/llvm/test/CodeGen/X86/GlobalISel/x86_64-select-urem.mir deleted file mode 100644 index 657cf4999497..000000000000 --- a/llvm/test/CodeGen/X86/GlobalISel/x86_64-select-urem.mir +++ /dev/null @@ -1,273 +0,0 @@ -# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple=x86_64-linux-gnu -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s - ---- | - ; ModuleID = 'urem.ll' - source_filename = "urem.ll" - target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" - - define i8 @test_urem_i8(i8 %arg1, i8 %arg2) { - %res = urem i8 %arg1, %arg2 - ret i8 %res - } - - define i16 @test_urem_i16(i16 %arg1, i16 %arg2) { - %res = urem i16 %arg1, %arg2 - ret i16 %res - } - - define i32 @test_urem_i32(i32 %arg1, i32 %arg2) { - %res = urem i32 %arg1, %arg2 - ret i32 %res - } - - define i64 @test_urem_i64(i64 %arg1, i64 %arg2) { - %res = urem i64 %arg1, %arg2 - ret i64 %res - } - -... ---- -name: test_urem_i8 -alignment: 16 -exposesReturnsTwice: false -legalized: true -regBankSelected: true -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: gpr, preferred-register: '' } - - { id: 1, class: gpr, preferred-register: '' } - - { id: 2, class: gpr, preferred-register: '' } - - { id: 3, class: gpr, preferred-register: '' } - - { id: 4, class: gpr, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_urem_i8 - ; CHECK: liveins: $edi, $esi - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $edi - ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr8 = COPY [[COPY]].sub_8bit - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY $esi - ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr8 = COPY [[COPY2]].sub_8bit - ; CHECK-NEXT: $ax = MOVZX16rr8 [[COPY1]] - ; CHECK-NEXT: DIV8r [[COPY3]], implicit-def $al, implicit-def $ah, implicit-def $eflags, implicit $ax - ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr16 = COPY $ax - ; CHECK-NEXT: [[SHR16ri:%[0-9]+]]:gr16 = SHR16ri [[COPY4]], 8, implicit-def $eflags - ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gr8 = SUBREG_TO_REG 0, [[SHR16ri]], %subreg.sub_8bit - ; CHECK-NEXT: $al = COPY [[SUBREG_TO_REG]] - ; CHECK-NEXT: RET 0, implicit $al - %2:gpr(s32) = COPY $edi - %0:gpr(s8) = G_TRUNC %2(s32) - %3:gpr(s32) = COPY $esi - %1:gpr(s8) = G_TRUNC %3(s32) - %4:gpr(s8) = G_UREM %0, %1 - $al = COPY %4(s8) - RET 0, implicit $al - -... ---- -name: test_urem_i16 -alignment: 16 -exposesReturnsTwice: false -legalized: true -regBankSelected: true -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: gpr, preferred-register: '' } - - { id: 1, class: gpr, preferred-register: '' } - - { id: 2, class: gpr, preferred-register: '' } - - { id: 3, class: gpr, preferred-register: '' } - - { id: 4, class: gpr, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_urem_i16 - ; CHECK: liveins: $edi, $esi - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $edi - ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr16 = COPY [[COPY]].sub_16bit - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY $esi - ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr16 = COPY [[COPY2]].sub_16bit - ; CHECK-NEXT: $ax = COPY [[COPY1]] - ; CHECK-NEXT: [[MOV32r0_:%[0-9]+]]:gr32 = MOV32r0 implicit-def $eflags - ; CHECK-NEXT: $dx = COPY [[MOV32r0_]].sub_16bit - ; CHECK-NEXT: DIV16r [[COPY3]], implicit-def $ax, implicit-def $dx, implicit-def $eflags, implicit $ax, implicit $dx - ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr16 = COPY $dx - ; CHECK-NEXT: $ax = COPY [[COPY4]] - ; CHECK-NEXT: RET 0, implicit $ax - %2:gpr(s32) = COPY $edi - %0:gpr(s16) = G_TRUNC %2(s32) - %3:gpr(s32) = COPY $esi - %1:gpr(s16) = G_TRUNC %3(s32) - %4:gpr(s16) = G_UREM %0, %1 - $ax = COPY %4(s16) - RET 0, implicit $ax - -... ---- -name: test_urem_i32 -alignment: 16 -exposesReturnsTwice: false -legalized: true -regBankSelected: true -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: gpr, preferred-register: '' } - - { id: 1, class: gpr, preferred-register: '' } - - { id: 2, class: gpr, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $edi, $esi - - ; CHECK-LABEL: name: test_urem_i32 - ; CHECK: liveins: $edi, $esi - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $edi - ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $esi - ; CHECK-NEXT: $eax = COPY [[COPY]] - ; CHECK-NEXT: [[MOV32r0_:%[0-9]+]]:gr32 = MOV32r0 implicit-def $eflags - ; CHECK-NEXT: $edx = COPY [[MOV32r0_]] - ; CHECK-NEXT: DIV32r [[COPY1]], implicit-def $eax, implicit-def $edx, implicit-def $eflags, implicit $eax, implicit $edx - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY $edx - ; CHECK-NEXT: $eax = COPY [[COPY2]] - ; CHECK-NEXT: RET 0, implicit $eax - %0:gpr(s32) = COPY $edi - %1:gpr(s32) = COPY $esi - %2:gpr(s32) = G_UREM %0, %1 - $eax = COPY %2(s32) - RET 0, implicit $eax - -... ---- -name: test_urem_i64 -alignment: 16 -exposesReturnsTwice: false -legalized: true -regBankSelected: true -selected: false -failedISel: false -tracksRegLiveness: true -registers: - - { id: 0, class: gpr, preferred-register: '' } - - { id: 1, class: gpr, preferred-register: '' } - - { id: 2, class: gpr, preferred-register: '' } -liveins: -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - stackProtector: '' - maxCallFrameSize: 4294967295 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false - localFrameSize: 0 - savePoint: '' - restorePoint: '' -fixedStack: -stack: -constants: -body: | - bb.1 (%ir-block.0): - liveins: $rdi, $rsi - - ; CHECK-LABEL: name: test_urem_i64 - ; CHECK: liveins: $rdi, $rsi - ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi - ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rsi - ; CHECK-NEXT: $rax = COPY [[COPY]] - ; CHECK-NEXT: [[MOV32r0_:%[0-9]+]]:gr32 = MOV32r0 implicit-def $eflags - ; CHECK-NEXT: $rdx = SUBREG_TO_REG 0, [[MOV32r0_]], %subreg.sub_32bit - ; CHECK-NEXT: DIV64r [[COPY1]], implicit-def $rax, implicit-def $rdx, implicit-def $eflags, implicit $rax, implicit $rdx - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64 = COPY $rdx - ; CHECK-NEXT: $rax = COPY [[COPY2]] - ; CHECK-NEXT: RET 0, implicit $rax - %0:gpr(s64) = COPY $rdi - %1:gpr(s64) = COPY $rsi - %2:gpr(s64) = G_UREM %0, %1 - $rax = COPY %2(s64) - RET 0, implicit $rax - -... diff --git a/llvm/test/CodeGen/X86/isel-sdiv.ll b/llvm/test/CodeGen/X86/isel-sdiv.ll new file mode 100644 index 000000000000..6a6b2da8dc2f --- /dev/null +++ b/llvm/test/CodeGen/X86/isel-sdiv.ll @@ -0,0 +1,116 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -global-isel=0 -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=X64 +; RUN: llc < %s -fast-isel -fast-isel-abort=1 -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=X64 +; RUN: llc < %s -global-isel -global-isel-abort=1 -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=X64 +; RUN: llc < %s -global-isel=0 -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=X86,DAG-X86 +; RUN: llc < %s -fast-isel -fast-isel-abort=1 -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=X86,DAG-X86 +; RUN: llc < %s -global-isel -global-isel-abort=1 -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=X86,GISEL-X86 + +define i8 @test_sdiv_i8(i8 %arg1, i8 %arg2) nounwind { +; X64-LABEL: test_sdiv_i8: +; X64: # %bb.0: +; X64-NEXT: movsbl %dil, %eax +; X64-NEXT: idivb %sil +; X64-NEXT: retq +; +; DAG-X86-LABEL: test_sdiv_i8: +; DAG-X86: # %bb.0: +; DAG-X86-NEXT: movsbl {{[0-9]+}}(%esp), %eax +; DAG-X86-NEXT: idivb {{[0-9]+}}(%esp) +; DAG-X86-NEXT: retl +; +; GISEL-X86-LABEL: test_sdiv_i8: +; GISEL-X86: # %bb.0: +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; GISEL-X86-NEXT: cbtw +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; GISEL-X86-NEXT: idivb %cl +; GISEL-X86-NEXT: retl + %ret = sdiv i8 %arg1, %arg2 + ret i8 %ret +} + +define i16 @test_sdiv_i16(i16 %arg1, i16 %arg2) nounwind { +; X64-LABEL: test_sdiv_i16: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: # kill: def $ax killed $ax killed $eax +; X64-NEXT: cwtd +; X64-NEXT: idivw %si +; X64-NEXT: retq +; +; DAG-X86-LABEL: test_sdiv_i16: +; DAG-X86: # %bb.0: +; DAG-X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax +; DAG-X86-NEXT: cwtd +; DAG-X86-NEXT: idivw {{[0-9]+}}(%esp) +; DAG-X86-NEXT: retl +; +; GISEL-X86-LABEL: test_sdiv_i16: +; GISEL-X86: # %bb.0: +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; GISEL-X86-NEXT: # kill: def $ax killed $ax killed $eax +; GISEL-X86-NEXT: cwtd +; GISEL-X86-NEXT: idivw %cx +; GISEL-X86-NEXT: retl + %ret = sdiv i16 %arg1, %arg2 + ret i16 %ret +} + +define i32 @test_sdiv_i32(i32 %arg1, i32 %arg2) nounwind { +; X64-LABEL: test_sdiv_i32: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: cltd +; X64-NEXT: idivl %esi +; X64-NEXT: retq +; +; X86-LABEL: test_sdiv_i32: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: cltd +; X86-NEXT: idivl {{[0-9]+}}(%esp) +; X86-NEXT: retl + %ret = sdiv i32 %arg1, %arg2 + ret i32 %ret +} + +define i64 @test_sdiv_i64(i64 %arg1, i64 %arg2) nounwind { +; X64-LABEL: test_sdiv_i64: +; X64: # %bb.0: +; X64-NEXT: movq %rdi, %rax +; X64-NEXT: cqto +; X64-NEXT: idivq %rsi +; X64-NEXT: retq +; +; DAG-X86-LABEL: test_sdiv_i64: +; DAG-X86: # %bb.0: +; DAG-X86-NEXT: subl $12, %esp +; DAG-X86-NEXT: pushl {{[0-9]+}}(%esp) +; DAG-X86-NEXT: pushl {{[0-9]+}}(%esp) +; DAG-X86-NEXT: pushl {{[0-9]+}}(%esp) +; DAG-X86-NEXT: pushl {{[0-9]+}}(%esp) +; DAG-X86-NEXT: calll __divdi3 +; DAG-X86-NEXT: addl $28, %esp +; DAG-X86-NEXT: retl +; +; GISEL-X86-LABEL: test_sdiv_i64: +; GISEL-X86: # %bb.0: +; GISEL-X86-NEXT: pushl %esi +; GISEL-X86-NEXT: subl $24, %esp +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; GISEL-X86-NEXT: movl %eax, (%esp) +; GISEL-X86-NEXT: movl %ecx, {{[0-9]+}}(%esp) +; GISEL-X86-NEXT: movl %edx, {{[0-9]+}}(%esp) +; GISEL-X86-NEXT: movl %esi, {{[0-9]+}}(%esp) +; GISEL-X86-NEXT: calll __divdi3 +; GISEL-X86-NEXT: addl $24, %esp +; GISEL-X86-NEXT: popl %esi +; GISEL-X86-NEXT: retl + %ret = sdiv i64 %arg1, %arg2 + ret i64 %ret +} diff --git a/llvm/test/CodeGen/X86/isel-srem.ll b/llvm/test/CodeGen/X86/isel-srem.ll new file mode 100644 index 000000000000..56716e10a9d9 --- /dev/null +++ b/llvm/test/CodeGen/X86/isel-srem.ll @@ -0,0 +1,150 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -global-isel=0 -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=X64,SDAG-X64 +; RUN: llc < %s -fast-isel -fast-isel-abort=1 -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=X64,FAST-X64 +; RUN: llc < %s -global-isel -global-isel-abort=1 -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=X64,GISEL-X64 +; RUN: llc < %s -global-isel=0 -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=X86,DAG-X86,SDAG-X86 +; RUN: llc < %s -fast-isel -fast-isel-abort=1 -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=X86,DAG-X86,FAST-X86 +; RUN: llc < %s -global-isel -global-isel-abort=1 -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=X86,GISEL-X86 + +define i8 @test_srem_i8(i8 %arg1, i8 %arg2) nounwind { +; SDAG-X64-LABEL: test_srem_i8: +; SDAG-X64: # %bb.0: +; SDAG-X64-NEXT: movsbl %dil, %eax +; SDAG-X64-NEXT: idivb %sil +; SDAG-X64-NEXT: movsbl %ah, %eax +; SDAG-X64-NEXT: # kill: def $al killed $al killed $eax +; SDAG-X64-NEXT: retq +; +; FAST-X64-LABEL: test_srem_i8: +; FAST-X64: # %bb.0: +; FAST-X64-NEXT: movsbl %dil, %eax +; FAST-X64-NEXT: idivb %sil +; FAST-X64-NEXT: shrw $8, %ax +; FAST-X64-NEXT: # kill: def $al killed $al killed $ax +; FAST-X64-NEXT: retq +; +; GISEL-X64-LABEL: test_srem_i8: +; GISEL-X64: # %bb.0: +; GISEL-X64-NEXT: movsbl %dil, %eax +; GISEL-X64-NEXT: idivb %sil +; GISEL-X64-NEXT: shrw $8, %ax +; GISEL-X64-NEXT: # kill: def $al killed $al killed $ax +; GISEL-X64-NEXT: retq +; +; SDAG-X86-LABEL: test_srem_i8: +; SDAG-X86: # %bb.0: +; SDAG-X86-NEXT: movsbl {{[0-9]+}}(%esp), %eax +; SDAG-X86-NEXT: idivb {{[0-9]+}}(%esp) +; SDAG-X86-NEXT: movsbl %ah, %eax +; SDAG-X86-NEXT: # kill: def $al killed $al killed $eax +; SDAG-X86-NEXT: retl +; +; FAST-X86-LABEL: test_srem_i8: +; FAST-X86: # %bb.0: +; FAST-X86-NEXT: movsbl {{[0-9]+}}(%esp), %eax +; FAST-X86-NEXT: idivb {{[0-9]+}}(%esp) +; FAST-X86-NEXT: movb %ah, %al +; FAST-X86-NEXT: retl +; +; GISEL-X86-LABEL: test_srem_i8: +; GISEL-X86: # %bb.0: +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; GISEL-X86-NEXT: cbtw +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; GISEL-X86-NEXT: idivb %cl +; GISEL-X86-NEXT: movb %ah, %al +; GISEL-X86-NEXT: retl + %ret = srem i8 %arg1, %arg2 + ret i8 %ret +} + +define i16 @test_srem_i16(i16 %arg1, i16 %arg2) nounwind { +; X64-LABEL: test_srem_i16: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: # kill: def $ax killed $ax killed $eax +; X64-NEXT: cwtd +; X64-NEXT: idivw %si +; X64-NEXT: movl %edx, %eax +; X64-NEXT: retq +; +; DAG-X86-LABEL: test_srem_i16: +; DAG-X86: # %bb.0: +; DAG-X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax +; DAG-X86-NEXT: cwtd +; DAG-X86-NEXT: idivw {{[0-9]+}}(%esp) +; DAG-X86-NEXT: movl %edx, %eax +; DAG-X86-NEXT: retl +; +; GISEL-X86-LABEL: test_srem_i16: +; GISEL-X86: # %bb.0: +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; GISEL-X86-NEXT: # kill: def $ax killed $ax killed $eax +; GISEL-X86-NEXT: cwtd +; GISEL-X86-NEXT: idivw %cx +; GISEL-X86-NEXT: movl %edx, %eax +; GISEL-X86-NEXT: retl + %ret = srem i16 %arg1, %arg2 + ret i16 %ret +} + +define i32 @test_srem_i32(i32 %arg1, i32 %arg2) nounwind { +; X64-LABEL: test_srem_i32: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: cltd +; X64-NEXT: idivl %esi +; X64-NEXT: movl %edx, %eax +; X64-NEXT: retq +; +; X86-LABEL: test_srem_i32: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: cltd +; X86-NEXT: idivl {{[0-9]+}}(%esp) +; X86-NEXT: movl %edx, %eax +; X86-NEXT: retl + %ret = srem i32 %arg1, %arg2 + ret i32 %ret +} + +define i64 @test_srem_i64(i64 %arg1, i64 %arg2) nounwind { +; X64-LABEL: test_srem_i64: +; X64: # %bb.0: +; X64-NEXT: movq %rdi, %rax +; X64-NEXT: cqto +; X64-NEXT: idivq %rsi +; X64-NEXT: movq %rdx, %rax +; X64-NEXT: retq +; +; DAG-X86-LABEL: test_srem_i64: +; DAG-X86: # %bb.0: +; DAG-X86-NEXT: subl $12, %esp +; DAG-X86-NEXT: pushl {{[0-9]+}}(%esp) +; DAG-X86-NEXT: pushl {{[0-9]+}}(%esp) +; DAG-X86-NEXT: pushl {{[0-9]+}}(%esp) +; DAG-X86-NEXT: pushl {{[0-9]+}}(%esp) +; DAG-X86-NEXT: calll __moddi3 +; DAG-X86-NEXT: addl $28, %esp +; DAG-X86-NEXT: retl +; +; GISEL-X86-LABEL: test_srem_i64: +; GISEL-X86: # %bb.0: +; GISEL-X86-NEXT: pushl %esi +; GISEL-X86-NEXT: subl $24, %esp +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; GISEL-X86-NEXT: movl %eax, (%esp) +; GISEL-X86-NEXT: movl %ecx, {{[0-9]+}}(%esp) +; GISEL-X86-NEXT: movl %edx, {{[0-9]+}}(%esp) +; GISEL-X86-NEXT: movl %esi, {{[0-9]+}}(%esp) +; GISEL-X86-NEXT: calll __moddi3 +; GISEL-X86-NEXT: addl $24, %esp +; GISEL-X86-NEXT: popl %esi +; GISEL-X86-NEXT: retl + %ret = srem i64 %arg1, %arg2 + ret i64 %ret +} diff --git a/llvm/test/CodeGen/X86/isel-udiv.ll b/llvm/test/CodeGen/X86/isel-udiv.ll new file mode 100644 index 000000000000..b56b8b112fe4 --- /dev/null +++ b/llvm/test/CodeGen/X86/isel-udiv.ll @@ -0,0 +1,116 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -global-isel=0 -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=X64 +; RUN: llc < %s -fast-isel -fast-isel-abort=1 -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=X64 +; RUN: llc < %s -global-isel -global-isel-abort=1 -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=X64 +; RUN: llc < %s -global-isel=0 -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=X86,DAG-X86 +; RUN: llc < %s -fast-isel -fast-isel-abort=1 -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=X86,DAG-X86 +; RUN: llc < %s -global-isel -global-isel-abort=1 -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=X86,GISEL-X86 + +define i8 @test_udiv_i8(i8 %arg1, i8 %arg2) nounwind { +; X64-LABEL: test_udiv_i8: +; X64: # %bb.0: +; X64-NEXT: movzbl %dil, %eax +; X64-NEXT: divb %sil +; X64-NEXT: retq +; +; DAG-X86-LABEL: test_udiv_i8: +; DAG-X86: # %bb.0: +; DAG-X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax +; DAG-X86-NEXT: divb {{[0-9]+}}(%esp) +; DAG-X86-NEXT: retl +; +; GISEL-X86-LABEL: test_udiv_i8: +; GISEL-X86: # %bb.0: +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; GISEL-X86-NEXT: movzbl %al, %eax +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; GISEL-X86-NEXT: divb %cl +; GISEL-X86-NEXT: retl + %ret = udiv i8 %arg1, %arg2 + ret i8 %ret +} + +define i16 @test_udiv_i16(i16 %arg1, i16 %arg2) nounwind { +; X64-LABEL: test_udiv_i16: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: # kill: def $ax killed $ax killed $eax +; X64-NEXT: xorl %edx, %edx +; X64-NEXT: divw %si +; X64-NEXT: retq +; +; DAG-X86-LABEL: test_udiv_i16: +; DAG-X86: # %bb.0: +; DAG-X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax +; DAG-X86-NEXT: xorl %edx, %edx +; DAG-X86-NEXT: divw {{[0-9]+}}(%esp) +; DAG-X86-NEXT: retl +; +; GISEL-X86-LABEL: test_udiv_i16: +; GISEL-X86: # %bb.0: +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; GISEL-X86-NEXT: # kill: def $ax killed $ax killed $eax +; GISEL-X86-NEXT: xorl %edx, %edx +; GISEL-X86-NEXT: divw %cx +; GISEL-X86-NEXT: retl + %ret = udiv i16 %arg1, %arg2 + ret i16 %ret +} + +define i32 @test_udiv_i32(i32 %arg1, i32 %arg2) nounwind { +; X64-LABEL: test_udiv_i32: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: xorl %edx, %edx +; X64-NEXT: divl %esi +; X64-NEXT: retq +; +; X86-LABEL: test_udiv_i32: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: xorl %edx, %edx +; X86-NEXT: divl {{[0-9]+}}(%esp) +; X86-NEXT: retl + %ret = udiv i32 %arg1, %arg2 + ret i32 %ret +} + +define i64 @test_udiv_i64(i64 %arg1, i64 %arg2) nounwind { +; X64-LABEL: test_udiv_i64: +; X64: # %bb.0: +; X64-NEXT: movq %rdi, %rax +; X64-NEXT: xorl %edx, %edx +; X64-NEXT: divq %rsi +; X64-NEXT: retq +; +; DAG-X86-LABEL: test_udiv_i64: +; DAG-X86: # %bb.0: +; DAG-X86-NEXT: subl $12, %esp +; DAG-X86-NEXT: pushl {{[0-9]+}}(%esp) +; DAG-X86-NEXT: pushl {{[0-9]+}}(%esp) +; DAG-X86-NEXT: pushl {{[0-9]+}}(%esp) +; DAG-X86-NEXT: pushl {{[0-9]+}}(%esp) +; DAG-X86-NEXT: calll __udivdi3 +; DAG-X86-NEXT: addl $28, %esp +; DAG-X86-NEXT: retl +; +; GISEL-X86-LABEL: test_udiv_i64: +; GISEL-X86: # %bb.0: +; GISEL-X86-NEXT: pushl %esi +; GISEL-X86-NEXT: subl $24, %esp +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; GISEL-X86-NEXT: movl %eax, (%esp) +; GISEL-X86-NEXT: movl %ecx, {{[0-9]+}}(%esp) +; GISEL-X86-NEXT: movl %edx, {{[0-9]+}}(%esp) +; GISEL-X86-NEXT: movl %esi, {{[0-9]+}}(%esp) +; GISEL-X86-NEXT: calll __udivdi3 +; GISEL-X86-NEXT: addl $24, %esp +; GISEL-X86-NEXT: popl %esi +; GISEL-X86-NEXT: retl + %ret = udiv i64 %arg1, %arg2 + ret i64 %ret +} diff --git a/llvm/test/CodeGen/X86/isel-urem.ll b/llvm/test/CodeGen/X86/isel-urem.ll new file mode 100644 index 000000000000..50b9c1250ff8 --- /dev/null +++ b/llvm/test/CodeGen/X86/isel-urem.ll @@ -0,0 +1,150 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -global-isel=0 -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=X64,SDAG-X64 +; RUN: llc < %s -fast-isel -fast-isel-abort=1 -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=X64,FAST-X64 +; RUN: llc < %s -global-isel -global-isel-abort=1 -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=X64,GISEL-X64 +; RUN: llc < %s -global-isel=0 -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=X86,DAG-X86,SDAG-X86 +; RUN: llc < %s -fast-isel -fast-isel-abort=1 -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=X86,DAG-X86,FAST-X86 +; RUN: llc < %s -global-isel -global-isel-abort=1 -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=X86,GISEL-X86 + +define i8 @test_urem_i8(i8 %arg1, i8 %arg2) nounwind { +; SDAG-X64-LABEL: test_urem_i8: +; SDAG-X64: # %bb.0: +; SDAG-X64-NEXT: movzbl %dil, %eax +; SDAG-X64-NEXT: divb %sil +; SDAG-X64-NEXT: movzbl %ah, %eax +; SDAG-X64-NEXT: # kill: def $al killed $al killed $eax +; SDAG-X64-NEXT: retq +; +; FAST-X64-LABEL: test_urem_i8: +; FAST-X64: # %bb.0: +; FAST-X64-NEXT: movzbl %dil, %eax +; FAST-X64-NEXT: divb %sil +; FAST-X64-NEXT: shrw $8, %ax +; FAST-X64-NEXT: # kill: def $al killed $al killed $ax +; FAST-X64-NEXT: retq +; +; GISEL-X64-LABEL: test_urem_i8: +; GISEL-X64: # %bb.0: +; GISEL-X64-NEXT: movzbl %dil, %eax +; GISEL-X64-NEXT: divb %sil +; GISEL-X64-NEXT: shrw $8, %ax +; GISEL-X64-NEXT: # kill: def $al killed $al killed $ax +; GISEL-X64-NEXT: retq +; +; SDAG-X86-LABEL: test_urem_i8: +; SDAG-X86: # %bb.0: +; SDAG-X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax +; SDAG-X86-NEXT: divb {{[0-9]+}}(%esp) +; SDAG-X86-NEXT: movzbl %ah, %eax +; SDAG-X86-NEXT: # kill: def $al killed $al killed $eax +; SDAG-X86-NEXT: retl +; +; FAST-X86-LABEL: test_urem_i8: +; FAST-X86: # %bb.0: +; FAST-X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax +; FAST-X86-NEXT: divb {{[0-9]+}}(%esp) +; FAST-X86-NEXT: movb %ah, %al +; FAST-X86-NEXT: retl +; +; GISEL-X86-LABEL: test_urem_i8: +; GISEL-X86: # %bb.0: +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; GISEL-X86-NEXT: movzbl %al, %eax +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; GISEL-X86-NEXT: divb %cl +; GISEL-X86-NEXT: movb %ah, %al +; GISEL-X86-NEXT: retl + %ret = urem i8 %arg1, %arg2 + ret i8 %ret +} + +define i16 @test_urem_i16(i16 %arg1, i16 %arg2) nounwind { +; X64-LABEL: test_urem_i16: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: # kill: def $ax killed $ax killed $eax +; X64-NEXT: xorl %edx, %edx +; X64-NEXT: divw %si +; X64-NEXT: movl %edx, %eax +; X64-NEXT: retq +; +; DAG-X86-LABEL: test_urem_i16: +; DAG-X86: # %bb.0: +; DAG-X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax +; DAG-X86-NEXT: xorl %edx, %edx +; DAG-X86-NEXT: divw {{[0-9]+}}(%esp) +; DAG-X86-NEXT: movl %edx, %eax +; DAG-X86-NEXT: retl +; +; GISEL-X86-LABEL: test_urem_i16: +; GISEL-X86: # %bb.0: +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; GISEL-X86-NEXT: # kill: def $ax killed $ax killed $eax +; GISEL-X86-NEXT: xorl %edx, %edx +; GISEL-X86-NEXT: divw %cx +; GISEL-X86-NEXT: movl %edx, %eax +; GISEL-X86-NEXT: retl + %ret = urem i16 %arg1, %arg2 + ret i16 %ret +} + +define i32 @test_urem_i32(i32 %arg1, i32 %arg2) nounwind { +; X64-LABEL: test_urem_i32: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: xorl %edx, %edx +; X64-NEXT: divl %esi +; X64-NEXT: movl %edx, %eax +; X64-NEXT: retq +; +; X86-LABEL: test_urem_i32: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: xorl %edx, %edx +; X86-NEXT: divl {{[0-9]+}}(%esp) +; X86-NEXT: movl %edx, %eax +; X86-NEXT: retl + %ret = urem i32 %arg1, %arg2 + ret i32 %ret +} + +define i64 @test_urem_i64(i64 %arg1, i64 %arg2) nounwind { +; X64-LABEL: test_urem_i64: +; X64: # %bb.0: +; X64-NEXT: movq %rdi, %rax +; X64-NEXT: xorl %edx, %edx +; X64-NEXT: divq %rsi +; X64-NEXT: movq %rdx, %rax +; X64-NEXT: retq +; +; DAG-X86-LABEL: test_urem_i64: +; DAG-X86: # %bb.0: +; DAG-X86-NEXT: subl $12, %esp +; DAG-X86-NEXT: pushl {{[0-9]+}}(%esp) +; DAG-X86-NEXT: pushl {{[0-9]+}}(%esp) +; DAG-X86-NEXT: pushl {{[0-9]+}}(%esp) +; DAG-X86-NEXT: pushl {{[0-9]+}}(%esp) +; DAG-X86-NEXT: calll __umoddi3 +; DAG-X86-NEXT: addl $28, %esp +; DAG-X86-NEXT: retl +; +; GISEL-X86-LABEL: test_urem_i64: +; GISEL-X86: # %bb.0: +; GISEL-X86-NEXT: pushl %esi +; GISEL-X86-NEXT: subl $24, %esp +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; GISEL-X86-NEXT: movl %eax, (%esp) +; GISEL-X86-NEXT: movl %ecx, {{[0-9]+}}(%esp) +; GISEL-X86-NEXT: movl %edx, {{[0-9]+}}(%esp) +; GISEL-X86-NEXT: movl %esi, {{[0-9]+}}(%esp) +; GISEL-X86-NEXT: calll __umoddi3 +; GISEL-X86-NEXT: addl $24, %esp +; GISEL-X86-NEXT: popl %esi +; GISEL-X86-NEXT: retl + %ret = urem i64 %arg1, %arg2 + ret i64 %ret +} -- GitLab From 5d33f7176b002da244823ca0e6b524777890dd9d Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Thu, 7 Mar 2024 15:19:27 -0800 Subject: [PATCH 521/929] Fix build: llvm::Error needs to be moved for implicit conversion to Expected. I don't know why the premerge setup didn't fail on this, but many builbots are broken right now. --- llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h b/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h index d5682fcaa28b..810a38f4a6ac 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h @@ -490,7 +490,7 @@ public: if (impl().NotifyCreated) if (Error Err = impl().NotifyCreated(*J)) - return Err; + return std::move(Err); return std::move(J); } -- GitLab From 5669660f37ef1800f4a7852577364b024d75e3d8 Mon Sep 17 00:00:00 2001 From: Chao Chen <116223022+chencha3@users.noreply.github.com> Date: Thu, 7 Mar 2024 17:25:59 -0600 Subject: [PATCH 522/929] [MLIR] XeGPU dialect for Intel GPU - core definitions and base classes (#78483) This PR follows our previous [RFC ](https://discourse.llvm.org/t/rfc-add-xegpu-dialect-for-intel-gpus/75723) to add XeGPU dialect definition for Intel GPUs. It contains dialect, type, attributes and operators definitions, as well as testcases for semantic checks. The lowering and optimization passes will be issued with separated passes. --------- Co-authored-by: Mehdi Amini --- mlir/include/mlir/Dialect/CMakeLists.txt | 1 + .../include/mlir/Dialect/XeGPU/CMakeLists.txt | 1 + .../mlir/Dialect/XeGPU/IR/CMakeLists.txt | 14 +++++++ mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.h | 29 ++++++++++++++ mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.td | 14 +++++++ .../mlir/Dialect/XeGPU/IR/XeGPUAttrs.td | 20 ++++++++++ .../mlir/Dialect/XeGPU/IR/XeGPUDialect.td | 30 ++++++++++++++ .../include/mlir/Dialect/XeGPU/IR/XeGPUOps.td | 26 +++++++++++++ .../mlir/Dialect/XeGPU/IR/XeGPUTypes.td | 33 ++++++++++++++++ mlir/include/mlir/InitAllDialects.h | 4 +- mlir/lib/Dialect/CMakeLists.txt | 1 + mlir/lib/Dialect/XeGPU/CMakeLists.txt | 1 + mlir/lib/Dialect/XeGPU/IR/CMakeLists.txt | 15 +++++++ mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp | 39 +++++++++++++++++++ mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp | 23 +++++++++++ 15 files changed, 250 insertions(+), 1 deletion(-) create mode 100644 mlir/include/mlir/Dialect/XeGPU/CMakeLists.txt create mode 100644 mlir/include/mlir/Dialect/XeGPU/IR/CMakeLists.txt create mode 100644 mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.h create mode 100644 mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.td create mode 100644 mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td create mode 100644 mlir/include/mlir/Dialect/XeGPU/IR/XeGPUDialect.td create mode 100644 mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td create mode 100644 mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td create mode 100644 mlir/lib/Dialect/XeGPU/CMakeLists.txt create mode 100644 mlir/lib/Dialect/XeGPU/IR/CMakeLists.txt create mode 100644 mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp create mode 100644 mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp diff --git a/mlir/include/mlir/Dialect/CMakeLists.txt b/mlir/include/mlir/Dialect/CMakeLists.txt index 9788e24e4a1d..2da79011fa26 100644 --- a/mlir/include/mlir/Dialect/CMakeLists.txt +++ b/mlir/include/mlir/Dialect/CMakeLists.txt @@ -40,3 +40,4 @@ add_subdirectory(UB) add_subdirectory(Utils) add_subdirectory(Vector) add_subdirectory(X86Vector) +add_subdirectory(XeGPU) diff --git a/mlir/include/mlir/Dialect/XeGPU/CMakeLists.txt b/mlir/include/mlir/Dialect/XeGPU/CMakeLists.txt new file mode 100644 index 000000000000..f33061b2d87c --- /dev/null +++ b/mlir/include/mlir/Dialect/XeGPU/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(IR) diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/XeGPU/IR/CMakeLists.txt new file mode 100644 index 000000000000..f1740e9ed929 --- /dev/null +++ b/mlir/include/mlir/Dialect/XeGPU/IR/CMakeLists.txt @@ -0,0 +1,14 @@ +add_mlir_dialect(XeGPU xegpu) +add_mlir_doc(XeGPU XeGPU Dialects/ -gen-dialect-doc -dialect=xegpu) + +set(LLVM_TARGET_DEFINITIONS XeGPU.td) +mlir_tablegen(XeGPUAttrs.h.inc -gen-attrdef-decls) +mlir_tablegen(XeGPUAttrs.cpp.inc -gen-attrdef-defs) +add_public_tablegen_target(MLIRXeGPUAttrsIncGen) +add_dependencies(mlir-headers MLIRXeGPUAttrsIncGen) + +set(LLVM_TARGET_DEFINITIONS XeGPU.td) +mlir_tablegen(XeGPUEnums.h.inc -gen-enum-decls) +mlir_tablegen(XeGPUEnums.cpp.inc -gen-enum-defs) +add_public_tablegen_target(MLIRXeGPUEnumsIncGen) +add_dependencies(mlir-headers MLIRXeGPUEnumsIncGen) diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.h b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.h new file mode 100644 index 000000000000..7aaa4ecc7ee7 --- /dev/null +++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.h @@ -0,0 +1,29 @@ +//===- XeGPU.h - MLIR dialect for XeGPU -------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_XEGPU_IR_XEGPU_H +#define MLIR_DIALECT_XEGPU_IR_XEGPU_H + +#include + +namespace mlir { +namespace xegpu { +// placeholder +} // namespace xegpu +} // namespace mlir + +#include +#include +#define GET_ATTRDEF_CLASSES +#include +#define GET_TYPEDEF_CLASSES +#include +#define GET_OP_CLASSES +#include + +#endif // MLIR_DIALECT_XEGPU_IR_XEGPU_H diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.td new file mode 100644 index 000000000000..232e96287071 --- /dev/null +++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.td @@ -0,0 +1,14 @@ +//===- XeGPU.td - XeGPU dialect definition ------------------*- tablegen -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_XEGPU_IR_XEGPU_TD +#define MLIR_DIALECT_XEGPU_IR_XEGPU_TD + +include "mlir/Dialect/XeGPU/IR/XeGPUOps.td" + +#endif // MLIR_DIALECT_XEGPU_IR_XEGPU_TD diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td new file mode 100644 index 000000000000..bb325c272e33 --- /dev/null +++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td @@ -0,0 +1,20 @@ +//===- XeGPUAttrs.td - XeGPU dialect attributes definition --*- tablegen -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_XEGPU_IR_XEGPUATTRS_TD +#define MLIR_DIALECT_XEGPU_IR_XEGPUATTRS_TD + +include "mlir/Dialect/XeGPU/IR/XeGPUDialect.td" + +class XeGPUAttr traits = [], + string baseCppClass = "::mlir::Attribute"> + : AttrDef { + let mnemonic = attrMnemonic; +} + +#endif // MLIR_DIALECT_XEGPU_IR_XEGPUATTRS_TD diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUDialect.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUDialect.td new file mode 100644 index 000000000000..3851275ad30a --- /dev/null +++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUDialect.td @@ -0,0 +1,30 @@ +//===- XeGPUDialect.td - XeGPU dialect definition -----------*- tablegen -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_XEGPU_IR_XEGPUDIALECT_TD +#define MLIR_DIALECT_XEGPU_IR_XEGPUDIALECT_TD + +include "mlir/IR/OpBase.td" + +def XeGPU_Dialect : Dialect { + let name = "xegpu"; + let cppNamespace = "::mlir::xegpu"; + let summary = "The XeGPU dialect that models Intel GPU's ISA"; + let description = [{ + The XeGPU dialect models Intel Xe ISA semantics but works at vector and + TensorDesc data type. It provides 1:1 mappings to match Xe instructions + like DPAS and 2D block load. The matrix size being processed at this level + exactly matches the hardware instructions or the intrinsic supported by + the lower-level GPU compiler. + }]; + + // let useDefaultTypePrinterParser = true; + // let useDefaultAttributePrinterParser = true; +} + +#endif // MLIR_DIALECT_XEGPU_IR_XEGPUDIALECT_TD diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td new file mode 100644 index 000000000000..5825ef9195b0 --- /dev/null +++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td @@ -0,0 +1,26 @@ +//===- XeGPUOps.td - XeGPU dialect operations definition ----*- tablegen -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_XEGPU_IR_XEGPUOPS_TD +#define MLIR_DIALECT_XEGPU_IR_XEGPUOPS_TD + +include "mlir/Dialect/XeGPU/IR/XeGPUAttrs.td" +include "mlir/Dialect/XeGPU/IR/XeGPUDialect.td" +include "mlir/Dialect/XeGPU/IR/XeGPUTypes.td" + + +// Base class for dialect operations. This operation inherits from the base +// `Op` class in OpBase.td, and provides: +// * The parent dialect of the operation. +// * The mnemonic for the operation, or the name without the dialect prefix. +// * A list of traits for the operation. +class XeGPU_Op traits = []>: + Op; + + +#endif // MLIR_DIALECT_XEGPU_IR_XEGPUOPS_TD diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td new file mode 100644 index 000000000000..1d75bb4e2906 --- /dev/null +++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td @@ -0,0 +1,33 @@ +//===- XeGPUTypes.td - XeGPU dialect types definition -------*- tablegen -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_XEGPU_IR_XEGPUTYPES_TD +#define MLIR_DIALECT_XEGPU_IR_XEGPUTYPES_TD + +include "mlir/IR/BuiltinTypes.td" +include "mlir/Dialect/XeGPU/IR/XeGPUAttrs.td" +include "mlir/Dialect/XeGPU/IR/XeGPUDialect.td" + +def XeGPU_IntType: AnyTypeOf<[I1, I8, I16, I32, I64, SI1, SI8, SI16, SI32, SI64, UI1, UI8, UI16, UI32, UI64]>; +def XeGPU_FloatType: AnyTypeOf<[F16, F32, F64, BF16, TF32]>; +def XeGPU_ScalarType: AnyTypeOf<[XeGPU_IntType, XeGPU_FloatType]>; +def XeGPU_BaseAddrType: AnyTypeOf<[MemRefRankOf<[XeGPU_ScalarType], [1, 2]>, UI64, UI32, I64, I32]>; +def XeGPU_DpasOpType: VectorOfRankAndType<[2, 3], [XeGPU_ScalarType]>; +def XeGPU_OffsetType: VectorOfRankAndType<[1], [Index]>; +def XeGPU_MaskType: AnyTypeOf<[VectorOfRankAndType<[1,2], [I1]>, I1]>; +def XeGPU_ValueType: AnyTypeOf<[VectorOfRankAndType<[1,2,3,4], [XeGPU_ScalarType]>, XeGPU_ScalarType]>; +def XeGPU_Vector2DType: VectorOfRankAndType<[2], [XeGPU_ScalarType]>; + +// common base class for types in XeGPU dialect +class XeGPUTypeDef traits = [], + string baseCppClass = "::mlir::Type"> + : TypeDef { + let mnemonic = typeMnemonic; +} + +#endif // MLIR_DIALECT_XEGPU_IR_XEGPUTYPES_TD diff --git a/mlir/include/mlir/InitAllDialects.h b/mlir/include/mlir/InitAllDialects.h index e508d51205f3..838bd03622a6 100644 --- a/mlir/include/mlir/InitAllDialects.h +++ b/mlir/include/mlir/InitAllDialects.h @@ -89,6 +89,7 @@ #include "mlir/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.h" #include "mlir/Dialect/Vector/Transforms/SubsetOpInterfaceImpl.h" #include "mlir/Dialect/X86Vector/X86VectorDialect.h" +#include "mlir/Dialect/XeGPU/IR/XeGPU.h" #include "mlir/IR/Dialect.h" #include "mlir/Interfaces/CastInterfaces.h" #include "mlir/Target/LLVM/NVVM/Target.h" @@ -141,7 +142,8 @@ inline void registerAllDialects(DialectRegistry ®istry) { transform::TransformDialect, ub::UBDialect, vector::VectorDialect, - x86vector::X86VectorDialect>(); + x86vector::X86VectorDialect, + xegpu::XeGPUDialect>(); // clang-format on // Register all external models. diff --git a/mlir/lib/Dialect/CMakeLists.txt b/mlir/lib/Dialect/CMakeLists.txt index c72107939cf4..b1ba5a3bc881 100644 --- a/mlir/lib/Dialect/CMakeLists.txt +++ b/mlir/lib/Dialect/CMakeLists.txt @@ -40,6 +40,7 @@ add_subdirectory(UB) add_subdirectory(Utils) add_subdirectory(Vector) add_subdirectory(X86Vector) +add_subdirectory(XeGPU) set(LLVM_OPTIONAL_SOURCES Traits.cpp diff --git a/mlir/lib/Dialect/XeGPU/CMakeLists.txt b/mlir/lib/Dialect/XeGPU/CMakeLists.txt new file mode 100644 index 000000000000..f33061b2d87c --- /dev/null +++ b/mlir/lib/Dialect/XeGPU/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(IR) diff --git a/mlir/lib/Dialect/XeGPU/IR/CMakeLists.txt b/mlir/lib/Dialect/XeGPU/IR/CMakeLists.txt new file mode 100644 index 000000000000..2e99f39ed86d --- /dev/null +++ b/mlir/lib/Dialect/XeGPU/IR/CMakeLists.txt @@ -0,0 +1,15 @@ +add_mlir_dialect_library(MLIRXeGPUDialect + XeGPUDialect.cpp + XeGPUOps.cpp + + ADDITIONAL_HEADER_DIRS + ${PROJECT_SOURCE_DIR}/include/mlir/Dialect/XeGPU + + DEPENDS + MLIRXeGPUIncGen + MLIRXeGPUAttrsIncGen + MLIRXeGPUEnumsIncGen + + LINK_LIBS PUBLIC + MLIRIR +) diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp new file mode 100644 index 000000000000..4f839ee77347 --- /dev/null +++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp @@ -0,0 +1,39 @@ +//===- XeGPUDialect.cpp - MLIR XeGPU dialect implementation -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include + +namespace mlir { +namespace xegpu { + +void XeGPUDialect::initialize() { + addTypes< +#define GET_TYPEDEF_LIST +#include + >(); + addOperations< +#define GET_OP_LIST +#include + >(); + addAttributes< +#define GET_ATTRDEF_LIST +#include + >(); +} + +// this file is for position occupation, +// we will add functions in following PRs. + +} // namespace xegpu +} // namespace mlir + +#include +#define GET_ATTRDEF_CLASSES +#include +#define GET_TYPEDEF_CLASSES +#include diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp new file mode 100644 index 000000000000..0e89ac4df6ef --- /dev/null +++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp @@ -0,0 +1,23 @@ +//===- XeGPUOps.cpp - MLIR XeGPU ops implementation -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include + +#define DEBUG_TYPE "xegpu" + +namespace mlir { +namespace xegpu { +// this file is for position occupation, +// we will add functions in following PRs. + +} // namespace xegpu +} // namespace mlir + +#include +#define GET_OP_CLASSES +#include -- GitLab From a9b0d7590b9e08151243b97aa75366e988e0d6c8 Mon Sep 17 00:00:00 2001 From: Maksim Panchenko Date: Thu, 7 Mar 2024 15:29:38 -0800 Subject: [PATCH 523/929] [BOLT] Properly propagate Cursor errors (#84378) Handle out-of-bounds reading errors correctly in LinuxKernelRewriter. --- bolt/lib/Rewrite/LinuxKernelRewriter.cpp | 30 ++++++++++++++++-------- bolt/test/X86/linux-alt-instruction.s | 5 ++++ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp index ecfbea3cb511..331a61e7c3c2 100644 --- a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp +++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp @@ -500,7 +500,8 @@ Error LinuxKernelRewriter::readORCTables() { // Consume the status of the cursor. if (!IPCursor) return createStringError(errc::executable_format_error, - "out of bounds while reading ORC IP table"); + "out of bounds while reading ORC IP table: %s", + toString(IPCursor.takeError()).c_str()); if (IP < PrevIP && opts::Verbosity) BC.errs() << "BOLT-WARNING: out of order IP 0x" << Twine::utohexstr(IP) @@ -522,7 +523,8 @@ Error LinuxKernelRewriter::readORCTables() { // Consume the status of the cursor. if (!ORCCursor) return createStringError(errc::executable_format_error, - "out of bounds while reading ORC"); + "out of bounds while reading ORC: %s", + toString(ORCCursor.takeError()).c_str()); if (Entry.ORC == NullORC) continue; @@ -843,7 +845,8 @@ Error LinuxKernelRewriter::readStaticCalls() { // Consume the status of the cursor. if (!Cursor) return createStringError(errc::executable_format_error, - "out of bounds while reading static calls"); + "out of bounds while reading static calls: %s", + toString(Cursor.takeError()).c_str()); ++EntryID; @@ -954,8 +957,10 @@ Error LinuxKernelRewriter::readExceptionTable() { // Consume the status of the cursor. if (!Cursor) - return createStringError(errc::executable_format_error, - "out of bounds while reading exception table"); + return createStringError( + errc::executable_format_error, + "out of bounds while reading exception table: %s", + toString(Cursor.takeError()).c_str()); ++EntryID; @@ -1061,8 +1066,10 @@ Error LinuxKernelRewriter::readParaInstructions() { const uint8_t Len = DE.getU8(Cursor); if (!Cursor) - return createStringError(errc::executable_format_error, - "out of bounds while reading .parainstructions"); + return createStringError( + errc::executable_format_error, + "out of bounds while reading .parainstructions: %s", + toString(Cursor.takeError()).c_str()); ++EntryID; @@ -1129,7 +1136,8 @@ Error LinuxKernelRewriter::readBugTable() { if (!Cursor) return createStringError(errc::executable_format_error, - "out of bounds while reading __bug_table"); + "out of bounds while reading __bug_table: %s", + toString(Cursor.takeError()).c_str()); ++EntryID; @@ -1196,8 +1204,10 @@ Error LinuxKernelRewriter::readAltInstructions() { const uint8_t PadLen = opts::AltInstHasPadLen ? DE.getU8(Cursor) : 0; if (!Cursor) - return createStringError(errc::executable_format_error, - "out of bounds while reading .altinstructions"); + return createStringError( + errc::executable_format_error, + "out of bounds while reading .altinstructions: %s", + toString(Cursor.takeError()).c_str()); ++EntryID; diff --git a/bolt/test/X86/linux-alt-instruction.s b/bolt/test/X86/linux-alt-instruction.s index 96e77545b654..5dcc6fe3ab0c 100644 --- a/bolt/test/X86/linux-alt-instruction.s +++ b/bolt/test/X86/linux-alt-instruction.s @@ -27,6 +27,11 @@ # RUN: llvm-bolt %t.exe --print-normalized --keep-nops \ # RUN: --alt-inst-feature-size=4 -o %t.out | FileCheck %s +## Check that out-of-bounds read is handled properly. + +# RUN: not llvm-bolt %t.exe --print-normalized --keep-nops \ +# RUN: --alt-inst-feature-size=2 -o %t.out + # CHECK: BOLT-INFO: Linux kernel binary detected # CHECK: BOLT-INFO: parsed 2 alternative instruction entries -- GitLab From 50ae8a2a38b618d76193bed04b1d7df6890d5c8a Mon Sep 17 00:00:00 2001 From: Cyndy Ishida Date: Thu, 7 Mar 2024 15:32:44 -0800 Subject: [PATCH 524/929] [InstallAPI] Collect global functions (#83952) * Include whether functions are inlinable as they impact whether to add them into the tbd file and for future verification. * Fix how clang arguments got passed along, previously spacing was passed along to CC1 causing search path inputs to look non-existent. --- clang/include/clang/InstallAPI/Frontend.h | 5 +- clang/include/clang/InstallAPI/Visitor.h | 3 + clang/lib/InstallAPI/Frontend.cpp | 5 +- clang/lib/InstallAPI/Visitor.cpp | 78 +++++++++++++++++++++++ clang/test/InstallAPI/functions.test | 78 +++++++++++++++++++++++ clang/tools/clang-installapi/Options.cpp | 4 +- llvm/include/llvm/TextAPI/Record.h | 6 +- llvm/include/llvm/TextAPI/RecordsSlice.h | 5 +- llvm/lib/TextAPI/RecordsSlice.cpp | 6 +- llvm/unittests/TextAPI/RecordTests.cpp | 3 +- 10 files changed, 182 insertions(+), 11 deletions(-) create mode 100644 clang/test/InstallAPI/functions.test diff --git a/clang/include/clang/InstallAPI/Frontend.h b/clang/include/clang/InstallAPI/Frontend.h index 8774321e990c..cbc2b159ebd1 100644 --- a/clang/include/clang/InstallAPI/Frontend.h +++ b/clang/include/clang/InstallAPI/Frontend.h @@ -50,12 +50,15 @@ public: /// \param D The pointer to the declaration from traversing AST. /// \param Access The intended access level of symbol. /// \param Flags The flags that describe attributes of the symbol. + /// \param Inlined Whether declaration is inlined, only applicable to + /// functions. /// \return The non-owning pointer to added record in slice. GlobalRecord *addGlobal(StringRef Name, RecordLinkage Linkage, GlobalRecord::Kind GV, const clang::AvailabilityInfo Avail, const Decl *D, const HeaderType Access, - SymbolFlags Flags = SymbolFlags::None); + SymbolFlags Flags = SymbolFlags::None, + bool Inlined = false); /// Add ObjC Class record with attributes from AST. /// diff --git a/clang/include/clang/InstallAPI/Visitor.h b/clang/include/clang/InstallAPI/Visitor.h index ff0a9957aa86..71d4d9894f42 100644 --- a/clang/include/clang/InstallAPI/Visitor.h +++ b/clang/include/clang/InstallAPI/Visitor.h @@ -37,6 +37,9 @@ public: /// Collect global variables. bool VisitVarDecl(const VarDecl *D); + /// Collect global functions. + bool VisitFunctionDecl(const FunctionDecl *D); + /// Collect Objective-C Interface declarations. /// Every Objective-C class has an interface declaration that lists all the /// ivars, properties, and methods of the class. diff --git a/clang/lib/InstallAPI/Frontend.cpp b/clang/lib/InstallAPI/Frontend.cpp index 240a80e1d3d8..1edbdf5bb983 100644 --- a/clang/lib/InstallAPI/Frontend.cpp +++ b/clang/lib/InstallAPI/Frontend.cpp @@ -19,9 +19,10 @@ namespace clang::installapi { GlobalRecord *FrontendRecordsSlice::addGlobal( StringRef Name, RecordLinkage Linkage, GlobalRecord::Kind GV, const clang::AvailabilityInfo Avail, const Decl *D, const HeaderType Access, - SymbolFlags Flags) { + SymbolFlags Flags, bool Inlined) { - auto *GR = llvm::MachO::RecordsSlice::addGlobal(Name, Linkage, GV, Flags); + auto *GR = + llvm::MachO::RecordsSlice::addGlobal(Name, Linkage, GV, Flags, Inlined); FrontendRecords.insert({GR, FrontendAttrs{Avail, D, Access}}); return GR; } diff --git a/clang/lib/InstallAPI/Visitor.cpp b/clang/lib/InstallAPI/Visitor.cpp index fbe6f1dabe00..1f2ef08e5aa2 100644 --- a/clang/lib/InstallAPI/Visitor.cpp +++ b/clang/lib/InstallAPI/Visitor.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "clang/InstallAPI/Visitor.h" +#include "clang/AST/ParentMapContext.h" #include "clang/Basic/Linkage.h" #include "clang/InstallAPI/Frontend.h" #include "llvm/ADT/SmallString.h" @@ -27,6 +28,31 @@ static bool isExported(const NamedDecl *D) { (LV.getVisibility() == DefaultVisibility); } +static bool isInlined(const FunctionDecl *D) { + bool HasInlineAttribute = false; + bool NoCXXAttr = + (!D->getASTContext().getLangOpts().CPlusPlus && + !D->getASTContext().getTargetInfo().getCXXABI().isMicrosoft() && + !D->hasAttr()); + + // Check all redeclarations to find an inline attribute or keyword. + for (const auto *RD : D->redecls()) { + if (!RD->isInlined()) + continue; + HasInlineAttribute = true; + if (!(NoCXXAttr || RD->hasAttr())) + continue; + if (RD->doesThisDeclarationHaveABody() && + RD->isInlineDefinitionExternallyVisible()) + return false; + } + + if (!HasInlineAttribute) + return false; + + return true; +} + static SymbolFlags getFlags(bool WeakDef, bool ThreadLocal) { SymbolFlags Result = SymbolFlags::None; if (WeakDef) @@ -204,4 +230,56 @@ bool InstallAPIVisitor::VisitVarDecl(const VarDecl *D) { return true; } +bool InstallAPIVisitor::VisitFunctionDecl(const FunctionDecl *D) { + if (const CXXMethodDecl *M = dyn_cast(D)) { + // Skip member function in class templates. + if (M->getParent()->getDescribedClassTemplate() != nullptr) + return true; + + // Skip methods in CXX RecordDecls. + for (auto P : D->getASTContext().getParents(*M)) { + if (P.get()) + return true; + } + + // Skip CXX ConstructorDecls and DestructorDecls. + if (isa(M) || isa(M)) + return true; + } + + // Skip templated functions. + switch (D->getTemplatedKind()) { + case FunctionDecl::TK_NonTemplate: + case FunctionDecl::TK_DependentNonTemplate: + break; + case FunctionDecl::TK_MemberSpecialization: + case FunctionDecl::TK_FunctionTemplateSpecialization: + if (auto *TempInfo = D->getTemplateSpecializationInfo()) { + if (!TempInfo->isExplicitInstantiationOrSpecialization()) + return true; + } + break; + case FunctionDecl::TK_FunctionTemplate: + case FunctionDecl::TK_DependentFunctionTemplateSpecialization: + return true; + } + + auto Access = getAccessForDecl(D); + if (!Access) + return true; + auto Name = getMangledName(D); + const AvailabilityInfo Avail = AvailabilityInfo::createFromDecl(D); + const bool ExplicitInstantiation = D->getTemplateSpecializationKind() == + TSK_ExplicitInstantiationDeclaration; + const bool WeakDef = ExplicitInstantiation || D->hasAttr(); + const bool Inlined = isInlined(D); + const RecordLinkage Linkage = (Inlined || !isExported(D)) + ? RecordLinkage::Internal + : RecordLinkage::Exported; + Ctx.Slice->addGlobal(Name, Linkage, GlobalRecord::Kind::Function, Avail, D, + *Access, getFlags(WeakDef, /*ThreadLocal=*/false), + Inlined); + return true; +} + } // namespace clang::installapi diff --git a/clang/test/InstallAPI/functions.test b/clang/test/InstallAPI/functions.test new file mode 100644 index 000000000000..527965303cb3 --- /dev/null +++ b/clang/test/InstallAPI/functions.test @@ -0,0 +1,78 @@ +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed -e "s|DSTROOT|%/t|g" %t/inputs.json.in > %t/inputs.json + +// RUN: clang-installapi -target arm64-apple-macos13.1 \ +// RUN: -I%t/usr/include -I%t/usr/local/include \ +// RUN: -install_name @rpath/lib/libfunctions.dylib \ +// RUN: %t/inputs.json -o %t/outputs.tbd 2>&1 | FileCheck %s --allow-empty +// RUN: llvm-readtapi -compare %t/outputs.tbd %t/expected.tbd 2>&1 | FileCheck %s --allow-empty + +// CHECK-NOT: error: +// CHECK-NOT: warning: + +//--- usr/include/functions.h +inline int inlined_func(void) { return 1;} +int public(int a); + +//--- usr/local/include/private_functions.h +__attribute__((visibility("hidden"))) +void hidden(void); + +//--- inputs.json.in +{ + "headers": [ { + "path" : "DSTROOT/usr/include/functions.h", + "type" : "public" + }, + { + "path" : "DSTROOT/usr/local/include/private_functions.h", + "type" : "private" + } + ], + "version": "3" +} + +//--- expected.tbd +{ + "main_library": { + "compatibility_versions": [ + { + "version": "0" + } + ], + "current_versions": [ + { + "version": "0" + } + ], + "exported_symbols": [ + { + "text": { + "global": [ + "_public" + ] + } + } + ], + "flags": [ + { + "attributes": [ + "not_app_extension_safe" + ] + } + ], + "install_names": [ + { + "name": "@rpath/lib/libfunctions.dylib" + } + ], + "target_info": [ + { + "min_deployment": "13.1", + "target": "arm64-macos" + } + ] + }, + "tapi_tbd_version": 5 +} diff --git a/clang/tools/clang-installapi/Options.cpp b/clang/tools/clang-installapi/Options.cpp index 7d45e999448d..b9c36eab2ad3 100644 --- a/clang/tools/clang-installapi/Options.cpp +++ b/clang/tools/clang-installapi/Options.cpp @@ -112,7 +112,9 @@ Options::Options(DiagnosticsEngine &Diag, FileManager *FM, for (const Arg *A : ArgList) { if (A->isClaimed()) continue; - FrontendArgs.emplace_back(A->getAsString(ArgList)); + + FrontendArgs.emplace_back(A->getSpelling()); + llvm::copy(A->getValues(), std::back_inserter(FrontendArgs)); } FrontendArgs.push_back("-fsyntax-only"); } diff --git a/llvm/include/llvm/TextAPI/Record.h b/llvm/include/llvm/TextAPI/Record.h index 867d6a235883..98639b064eaa 100644 --- a/llvm/include/llvm/TextAPI/Record.h +++ b/llvm/include/llvm/TextAPI/Record.h @@ -103,8 +103,8 @@ public: }; GlobalRecord(StringRef Name, RecordLinkage Linkage, SymbolFlags Flags, - Kind GV) - : Record({Name, Linkage, Flags}), GV(GV) {} + Kind GV, bool Inlined) + : Record({Name, Linkage, Flags}), GV(GV), Inlined(Inlined) {} bool isFunction() const { return GV == Kind::Function; } bool isVariable() const { return GV == Kind::Variable; } @@ -112,9 +112,11 @@ public: if (GV == Kind::Unknown) GV = V; } + bool isInlined() const { return Inlined; } private: Kind GV; + bool Inlined = false; }; // Define Objective-C instance variable records. diff --git a/llvm/include/llvm/TextAPI/RecordsSlice.h b/llvm/include/llvm/TextAPI/RecordsSlice.h index 57b23e5ea29e..f934cf7607f1 100644 --- a/llvm/include/llvm/TextAPI/RecordsSlice.h +++ b/llvm/include/llvm/TextAPI/RecordsSlice.h @@ -53,10 +53,13 @@ public: /// \param Linkage The linkage of symbol. /// \param GV The kind of global. /// \param Flags The flags that describe attributes of the symbol. + /// \param Inlined Whether declaration is inlined, only applicable to + /// functions. /// \return The non-owning pointer to added record in slice. GlobalRecord *addGlobal(StringRef Name, RecordLinkage Linkage, GlobalRecord::Kind GV, - SymbolFlags Flags = SymbolFlags::None); + SymbolFlags Flags = SymbolFlags::None, + bool Inlined = false); /// Add ObjC Class record. /// diff --git a/llvm/lib/TextAPI/RecordsSlice.cpp b/llvm/lib/TextAPI/RecordsSlice.cpp index db52a2cdd85c..111a1fa6eaf4 100644 --- a/llvm/lib/TextAPI/RecordsSlice.cpp +++ b/llvm/lib/TextAPI/RecordsSlice.cpp @@ -171,8 +171,8 @@ ObjCIVarRecord *RecordsSlice::findObjCIVar(bool IsScopedName, } GlobalRecord *RecordsSlice::addGlobal(StringRef Name, RecordLinkage Linkage, - GlobalRecord::Kind GV, - SymbolFlags Flags) { + GlobalRecord::Kind GV, SymbolFlags Flags, + bool Inlined) { if (GV == GlobalRecord::Kind::Function) Flags |= SymbolFlags::Text; else if (GV == GlobalRecord::Kind::Variable) @@ -182,7 +182,7 @@ GlobalRecord *RecordsSlice::addGlobal(StringRef Name, RecordLinkage Linkage, auto Result = Globals.insert({Name, nullptr}); if (Result.second) Result.first->second = - std::make_unique(Name, Linkage, Flags, GV); + std::make_unique(Name, Linkage, Flags, GV, Inlined); else { updateLinkage(Result.first->second.get(), Linkage); updateFlags(Result.first->second.get(), Flags); diff --git a/llvm/unittests/TextAPI/RecordTests.cpp b/llvm/unittests/TextAPI/RecordTests.cpp index 37289eca1bdf..89ffbc4275e0 100644 --- a/llvm/unittests/TextAPI/RecordTests.cpp +++ b/llvm/unittests/TextAPI/RecordTests.cpp @@ -19,7 +19,7 @@ TEST(TAPIRecord, Simple) { GlobalRecord API{"_sym", RecordLinkage::Rexported, SymbolFlags::Rexported | SymbolFlags::Text | SymbolFlags::ThreadLocalValue, - GlobalRecord::Kind::Function}; + GlobalRecord::Kind::Function, /*Inlined=*/false}; EXPECT_TRUE(API.isExported()); EXPECT_TRUE(API.isText()); EXPECT_TRUE(API.isRexported()); @@ -30,6 +30,7 @@ TEST(TAPIRecord, Simple) { EXPECT_FALSE(API.isWeakDefined()); EXPECT_FALSE(API.isWeakReferenced()); EXPECT_FALSE(API.isVariable()); + EXPECT_FALSE(API.isInlined()); } TEST(TAPIRecord, SimpleObjC) { -- GitLab From 3712edbdbb79e0169acf0c57e111f3195006c013 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Thu, 7 Mar 2024 15:37:45 -0800 Subject: [PATCH 525/929] [libc] finish documenting c23 additions (#84383) - [libc] finish documenting c23 additions - sort according to appearance in Annex B and section 7 --- libc/docs/c23.rst | 112 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 79 insertions(+), 33 deletions(-) diff --git a/libc/docs/c23.rst b/libc/docs/c23.rst index ec9d40947cc5..24cef8539393 100644 --- a/libc/docs/c23.rst +++ b/libc/docs/c23.rst @@ -15,36 +15,15 @@ Implementation Status (It's helpful to review 'Annex B (Informative) Library Summary' for these.) -New headers: - -* stdbit.h -* stdckdint.h (|check|, macros are only defined with `__GNUC__` builtins) Additions: -* uchar.h - - * mbrtoc8 - * c8rtomb - * char*_t - -* string.h - - * memset_explicit - * memccpy - * strdup - * strndup - -* time.h - - * gmtime_r - * localtime_r - * timegm - * timespec_getres - * strftime conversion specifiers +* fenv.h - * 0b - * 0B + * fesetexcept + * fetestexceptflag + * fegetmode + * fesetmode * math.h * acospi* @@ -96,20 +75,87 @@ Additions: * dfmal * fsqrt* * dsqrtl -* fenv.h - - * fesetexcept - * fetestexceptflag - * fegetmode - * fesetmode +* stdbit.h (New header) +* stdckdint.h (New header) |check| * stddef.h * unreachable * stdlib.h + * strfromd + * strfromf + * strfroml * free_sized * free_aligned_sized * memalignment +* string.h + + * memset_explicit |check| + * memccpy + * strdup + * strndup * tgmath.h - * + * acospi + * asinpi + * atan2pi + * atanpi + * compoundn + * cospi + * erf + * exp10m1 + * exp10 + * exp2m1 + * fmaximum + * fmaximum_mag + * fmaximum_num + * fmaximum_mag_num + * fminimum + * fminimum_mag + * fminimum_num + * fminimum_mag_num + * fromfpx + * fromfp + * llogb + * log10p1 + * log2p1 + * logp1 + * nextdown + * nextup + * pown + * powr + * rootn + * roundeven + * rsqrt + * scalbn + * sinpi + * tanpi + * ufromfpx + * ufromfp + * fadd + * dadd + * fsub + * dsub + * fmul + * dmul + * fdiv + * ddiv + * ffma + * dfma + * fsqrt + * dsqrt +* time.h + + * gmtime_r + * localtime_r + * timegm + * timespec_getres + * strftime conversion specifiers + + * 0b + * 0B +* uchar.h + + * mbrtoc8 + * c8rtomb + * char*_t -- GitLab From 293ec4865bfcb6df2091ef4bcce706a566794b5c Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Thu, 7 Mar 2024 15:38:16 -0800 Subject: [PATCH 526/929] [libc] rename cpp::count_ones to cpp::popcount to better mirror std:: (#84388) libc/src/__support/CPP/bit.h and cpp:: is meant to mirror std::. Fix the TODO. --- libc/src/__support/CPP/bit.h | 10 ++++------ libc/src/__support/UInt.h | 2 +- libc/src/stdbit/stdc_count_ones_uc.cpp | 2 +- libc/src/stdbit/stdc_count_ones_ui.cpp | 2 +- libc/src/stdbit/stdc_count_ones_ul.cpp | 2 +- libc/src/stdbit/stdc_count_ones_ull.cpp | 2 +- libc/src/stdbit/stdc_count_ones_us.cpp | 2 +- libc/test/src/__support/CPP/bit_test.cpp | 4 ++-- 8 files changed, 12 insertions(+), 14 deletions(-) diff --git a/libc/src/__support/CPP/bit.h b/libc/src/__support/CPP/bit.h index 6b625b0c97a3..9c74a346949f 100644 --- a/libc/src/__support/CPP/bit.h +++ b/libc/src/__support/CPP/bit.h @@ -269,14 +269,12 @@ first_trailing_one(T value) { return value == cpp::numeric_limits::max() ? 0 : countr_zero(value) + 1; } -/// Count number of 1's aka population count or hamming weight. +/// Count number of 1's aka population count or Hamming weight. /// /// Only unsigned integral types are allowed. -// TODO: rename as 'popcount' to follow the standard -// https://en.cppreference.com/w/cpp/numeric/popcount template [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> -count_ones(T value) { +popcount(T value) { int count = 0; for (int i = 0; i != cpp::numeric_limits::digits; ++i) if ((value >> i) & 0x1) @@ -285,7 +283,7 @@ count_ones(T value) { } #define ADD_SPECIALIZATION(TYPE, BUILTIN) \ template <> \ - [[nodiscard]] LIBC_INLINE constexpr int count_ones(TYPE value) { \ + [[nodiscard]] LIBC_INLINE constexpr int popcount(TYPE value) { \ return BUILTIN(value); \ } ADD_SPECIALIZATION(unsigned char, __builtin_popcount) @@ -300,7 +298,7 @@ ADD_SPECIALIZATION(unsigned long long, __builtin_popcountll) template [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> count_zeros(T value) { - return count_ones(static_cast(~value)); + return popcount(static_cast(~value)); } } // namespace LIBC_NAMESPACE::cpp diff --git a/libc/src/__support/UInt.h b/libc/src/__support/UInt.h index 925de8764715..e899a79684b7 100644 --- a/libc/src/__support/UInt.h +++ b/libc/src/__support/UInt.h @@ -979,7 +979,7 @@ has_single_bit(T value) { for (auto word : value.val) { if (word == 0) continue; - bits += count_ones(word); + bits += popcount(word); if (bits > 1) return false; } diff --git a/libc/src/stdbit/stdc_count_ones_uc.cpp b/libc/src/stdbit/stdc_count_ones_uc.cpp index 5a7314caa3ba..1e998ff521b7 100644 --- a/libc/src/stdbit/stdc_count_ones_uc.cpp +++ b/libc/src/stdbit/stdc_count_ones_uc.cpp @@ -14,7 +14,7 @@ namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_count_ones_uc, (unsigned char value)) { - return static_cast(cpp::count_ones(value)); + return static_cast(cpp::popcount(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_count_ones_ui.cpp b/libc/src/stdbit/stdc_count_ones_ui.cpp index 289f4bac31f7..e457dd793db3 100644 --- a/libc/src/stdbit/stdc_count_ones_ui.cpp +++ b/libc/src/stdbit/stdc_count_ones_ui.cpp @@ -14,7 +14,7 @@ namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_count_ones_ui, (unsigned value)) { - return static_cast(cpp::count_ones(value)); + return static_cast(cpp::popcount(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_count_ones_ul.cpp b/libc/src/stdbit/stdc_count_ones_ul.cpp index 83f3279d7919..ed86653fc7ee 100644 --- a/libc/src/stdbit/stdc_count_ones_ul.cpp +++ b/libc/src/stdbit/stdc_count_ones_ul.cpp @@ -14,7 +14,7 @@ namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_count_ones_ul, (unsigned long value)) { - return static_cast(cpp::count_ones(value)); + return static_cast(cpp::popcount(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_count_ones_ull.cpp b/libc/src/stdbit/stdc_count_ones_ull.cpp index 104788aaf212..c5ecc3cda647 100644 --- a/libc/src/stdbit/stdc_count_ones_ull.cpp +++ b/libc/src/stdbit/stdc_count_ones_ull.cpp @@ -14,7 +14,7 @@ namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_count_ones_ull, (unsigned long long value)) { - return static_cast(cpp::count_ones(value)); + return static_cast(cpp::popcount(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/stdbit/stdc_count_ones_us.cpp b/libc/src/stdbit/stdc_count_ones_us.cpp index 4b6ff0b94b62..465c5c374e7c 100644 --- a/libc/src/stdbit/stdc_count_ones_us.cpp +++ b/libc/src/stdbit/stdc_count_ones_us.cpp @@ -14,7 +14,7 @@ namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(unsigned, stdc_count_ones_us, (unsigned short value)) { - return static_cast(cpp::count_ones(value)); + return static_cast(cpp::popcount(value)); } } // namespace LIBC_NAMESPACE diff --git a/libc/test/src/__support/CPP/bit_test.cpp b/libc/test/src/__support/CPP/bit_test.cpp index 25a80ca9209c..d3f56d5bad83 100644 --- a/libc/test/src/__support/CPP/bit_test.cpp +++ b/libc/test/src/__support/CPP/bit_test.cpp @@ -260,9 +260,9 @@ TYPED_TEST(LlvmLibcBitTest, CountZeros, UnsignedTypesNoBigInt) { } TYPED_TEST(LlvmLibcBitTest, CountOnes, UnsignedTypesNoBigInt) { - EXPECT_EQ(count_ones(T(0)), 0); + EXPECT_EQ(popcount(T(0)), 0); for (int i = 0; i != cpp::numeric_limits::digits; ++i) - EXPECT_EQ(count_ones(cpp::numeric_limits::max() >> i), + EXPECT_EQ(popcount(cpp::numeric_limits::max() >> i), cpp::numeric_limits::digits - i); } -- GitLab From f862265733d65efbfd819408b594b3b2854491d2 Mon Sep 17 00:00:00 2001 From: Changpeng Fang Date: Thu, 7 Mar 2024 15:39:41 -0800 Subject: [PATCH 527/929] AMDGPU: Use True16Predicate for UseRealTrue16Insts in VOP2 Reals (#84394) We can not use OtherPredicates or SubtargetPredicate because they should be copied from pseudo to real, and we should not override them. --- llvm/lib/Target/AMDGPU/VOP2Instructions.td | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/VOP2Instructions.td b/llvm/lib/Target/AMDGPU/VOP2Instructions.td index 13fe79b47596..53578682e002 100644 --- a/llvm/lib/Target/AMDGPU/VOP2Instructions.td +++ b/llvm/lib/Target/AMDGPU/VOP2Instructions.td @@ -112,7 +112,7 @@ class VOP2_Real : VOP2_Real { let AssemblerPredicate = Gen.AssemblerPredicate; - let OtherPredicates = !if(ps.Pfl.IsRealTrue16, [UseRealTrue16Insts], []); + let True16Predicate = !if(ps.Pfl.IsRealTrue16, UseRealTrue16Insts, NoTrue16Predicate); let DecoderNamespace = Gen.DecoderNamespace# !if(ps.Pfl.IsRealTrue16, "", "_FAKE16"); } @@ -1272,7 +1272,7 @@ class VOP2_DPP16_Gen op, VOP2_DPP_Pseudo ps, GFXGen Gen, string opName = ps.OpName, VOPProfile p = ps.Pfl> : VOP2_DPP16 { let AssemblerPredicate = Gen.AssemblerPredicate; - let OtherPredicates = !if(ps.Pfl.IsRealTrue16, [UseRealTrue16Insts], []); + let True16Predicate = !if(ps.Pfl.IsRealTrue16, UseRealTrue16Insts, NoTrue16Predicate); let DecoderNamespace = Gen.DecoderNamespace# !if(ps.Pfl.IsRealTrue16, "", "_FAKE16"); } @@ -1301,7 +1301,7 @@ class VOP2_DPP8_Gen op, VOP2_Pseudo ps, GFXGen Gen, VOPProfile p = ps.Pfl> : VOP2_DPP8 { let AssemblerPredicate = Gen.AssemblerPredicate; - let OtherPredicates = !if(ps.Pfl.IsRealTrue16, [UseRealTrue16Insts], []); + let True16Predicate = !if(ps.Pfl.IsRealTrue16, UseRealTrue16Insts, NoTrue16Predicate); let DecoderNamespace = Gen.DecoderNamespace# !if(ps.Pfl.IsRealTrue16, "", "_FAKE16"); } -- GitLab From a01e9ce86f4c1bc9af819902db9f287b6d23f54f Mon Sep 17 00:00:00 2001 From: Amara Emerson Date: Thu, 7 Mar 2024 15:38:33 -0800 Subject: [PATCH 528/929] [AArc64][GlobalISel] Fix legalizer assert for G_INSERT_VECTOR_ELT We should moreElements <3 x s1> to <4 x s1> before we try to widen the element, otherwise we end up with a <3 x s21> nonsense type. --- .../AArch64/GISel/AArch64LegalizerInfo.cpp | 1 + .../GlobalISel/legalize-insert-vector-elt.mir | 70 +++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp index ad389cfc75aa..36adada27965 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp @@ -884,6 +884,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) getActionDefinitionsBuilder(G_INSERT_VECTOR_ELT) .legalIf(typeInSet(0, {v16s8, v8s8, v8s16, v4s16, v4s32, v2s32, v2s64})) + .moreElementsToNextPow2(0) .widenVectorEltsToVectorMinSize(0, 64); getActionDefinitionsBuilder(G_BUILD_VECTOR) diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-insert-vector-elt.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-insert-vector-elt.mir index 6f6cf2cc165b..e12353c7ef5b 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-insert-vector-elt.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-insert-vector-elt.mir @@ -216,3 +216,73 @@ body: | $q0 = COPY %2(<2 x s64>) RET_ReallyLR ... +--- +name: v3s8_crash +body: | + ; CHECK-LABEL: name: v3s8_crash + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: liveins: $w1, $w2, $w3, $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1 + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $w2 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $w3 + ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF + ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32) + ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32) + ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY3]](s32) + ; CHECK-NEXT: [[DEF2:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF + ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16), [[TRUNC2]](s16), [[DEF2]](s16) + ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 0 + ; CHECK-NEXT: [[IVEC:%[0-9]+]]:_(<4 x s16>) = G_INSERT_VECTOR_ELT [[BUILD_VECTOR]], [[C2]](s16), [[C1]](s64) + ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[IVEC]](<4 x s16>) + ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[UV]](s16) + ; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s8) = G_TRUNC [[UV1]](s16) + ; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s8) = G_TRUNC [[UV2]](s16) + ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[TRUNC3]](s8), [[TRUNC4]](s8), [[TRUNC5]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8) + ; CHECK-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[C]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8) + ; CHECK-NEXT: [[SHUF:%[0-9]+]]:_(<16 x s8>) = G_SHUFFLE_VECTOR [[BUILD_VECTOR1]](<16 x s8>), [[BUILD_VECTOR2]], shufflemask(0, 16, 16, 16, 1, 16, 16, 16, 2, 16, 16, 16, undef, undef, undef, undef) + ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<4 x s32>) = G_BITCAST [[SHUF]](<16 x s8>) + ; CHECK-NEXT: [[UITOFP:%[0-9]+]]:_(<4 x s32>) = G_UITOFP [[BITCAST]](<4 x s32>) + ; CHECK-NEXT: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UITOFP]](<4 x s32>) + ; CHECK-NEXT: G_STORE [[UV4]](s32), [[COPY]](p0) :: (store (s32), align 16) + ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 4 + ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C3]](s64) + ; CHECK-NEXT: G_STORE [[UV5]](s32), [[PTR_ADD]](p0) :: (store (s32) into unknown-address + 4) + ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64) + ; CHECK-NEXT: G_STORE [[UV6]](s32), [[PTR_ADD1]](p0) :: (store (s32) into unknown-address + 8, align 8) + ; CHECK-NEXT: G_BR %bb.1 + bb.1: + liveins: $w1, $w2, $w3, $x0 + + %0:_(p0) = COPY $x0 + %2:_(s32) = COPY $w1 + %3:_(s32) = COPY $w2 + %4:_(s32) = COPY $w3 + %5:_(<3 x s32>) = G_BUILD_VECTOR %2(s32), %3(s32), %4(s32) + %1:_(<3 x s8>) = G_TRUNC %5(<3 x s32>) + %8:_(s64) = G_CONSTANT i64 0 + %11:_(s8) = G_IMPLICIT_DEF + %7:_(s8) = G_CONSTANT i8 0 + %10:_(<3 x s8>) = G_BUILD_VECTOR %7(s8), %11(s8), %11(s8) + + bb.2: + %14:_(s64) = G_CONSTANT i64 0 + %15:_(s8) = G_CONSTANT i8 0 + %6:_(<3 x s8>) = G_INSERT_VECTOR_ELT %1, %15(s8), %14(s64) + %9:_(<12 x s8>) = G_SHUFFLE_VECTOR %6(<3 x s8>), %10, shufflemask(0, 3, 3, 3, 1, 3, 3, 3, 2, 3, 3, 3) + %12:_(<3 x s32>) = G_BITCAST %9(<12 x s8>) + %13:_(<3 x s32>) = G_UITOFP %12(<3 x s32>) + G_STORE %13(<3 x s32>), %0(p0) :: (store (<3 x s32>)) + G_BR %bb.2 + +... -- GitLab From 3e5afba8ef9319956d288ff755df3c442433eb88 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Thu, 7 Mar 2024 15:41:12 -0800 Subject: [PATCH 529/929] [NFC] [hwasan] be consistent about how to get integer types (#84396) --- llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp index 236ee8910d46..289183ecf0f2 100644 --- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -412,8 +412,8 @@ private: Type *VoidTy = Type::getVoidTy(M.getContext()); Type *IntptrTy; PointerType *PtrTy; - Type *Int8Ty; - Type *Int32Ty; + Type *Int8Ty = Type::getInt8Ty(M.getContext()); + Type *Int32Ty = Type::getInt32Ty(M.getContext()); Type *Int64Ty = Type::getInt64Ty(M.getContext()); bool CompileKernel; @@ -615,8 +615,6 @@ void HWAddressSanitizer::initializeModule() { IRBuilder<> IRB(*C); IntptrTy = IRB.getIntPtrTy(DL); PtrTy = IRB.getPtrTy(); - Int8Ty = IRB.getInt8Ty(); - Int32Ty = IRB.getInt32Ty(); HwasanCtorFunction = nullptr; -- GitLab From ddf79deb42d901fbb732e56464efbf93bc444070 Mon Sep 17 00:00:00 2001 From: Jie Fu Date: Fri, 8 Mar 2024 07:42:01 +0800 Subject: [PATCH 530/929] [Asan] Fix -Wunused-private-field in non-assertion builds (NFC) llvm-project/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp:650:13: error: private field 'OwnerFn' is not used [-Werror,-Wunused-private-field] Function *OwnerFn = nullptr; ^ 1 error generated. --- llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp index c95a50a033b1..db75eec21a37 100644 --- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -656,6 +656,7 @@ public: ArrayRef Args = {}, const Twine &Name = "") { assert(IRB.GetInsertBlock()->getParent() == OwnerFn); + (void)OwnerFn; return IRB.CreateCall(Callee, Args, Name, nullptr); } }; -- GitLab From 8bf8d36f8e82a1e2d32f33dbe7369d9cecd57f46 Mon Sep 17 00:00:00 2001 From: David CARLIER Date: Thu, 7 Mar 2024 23:53:27 +0000 Subject: [PATCH 531/929] [compiler-rt][fuzzer] Reland "SetThreadName windows implementation" (#83562) Following-up on GH-76761. --- compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp | 25 ++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp b/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp index 71770166805f..0dbcec8b5f22 100644 --- a/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp @@ -21,10 +21,15 @@ #include #include #include +// clang-format off #include - -// This must be included after windows.h. +// These must be included after windows.h. +// archicture need to be set before including +// libloaderapi +#include +#include #include +// clang-format on namespace fuzzer { @@ -234,8 +239,20 @@ size_t PageSize() { } void SetThreadName(std::thread &thread, const std::string &name) { - // TODO ? - // to UTF-8 then SetThreadDescription ? + typedef HRESULT(WINAPI * proc)(HANDLE, PCWSTR); + HMODULE kbase = GetModuleHandleA("KernelBase.dll"); + proc ThreadNameProc = + reinterpret_cast(GetProcAddress(kbase, "SetThreadDescription")); + if (proc) { + std::wstring buf; + auto sz = MultiByteToWideChar(CP_UTF8, 0, name.data(), -1, nullptr, 0); + if (sz > 0) { + buf.resize(sz); + if (MultiByteToWideChar(CP_UTF8, 0, name.data(), -1, &buf[0], sz) > 0) { + (void)ThreadNameProc(thread.native_handle(), buf.c_str()); + } + } + } } } // namespace fuzzer -- GitLab From 26fa4409572ad81c5522165ba2a831845f4d0635 Mon Sep 17 00:00:00 2001 From: Amara Emerson Date: Thu, 7 Mar 2024 15:50:25 -0800 Subject: [PATCH 532/929] [GlobalISel] Fix yet another pointer type invalid combining issue, this time in tryFoldSelectOfConstants() --- .../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 3 ++ .../AArch64/GlobalISel/combine-select.mir | 37 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp index 4862207d53f4..ab055b723dbb 100644 --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -6369,6 +6369,9 @@ bool CombinerHelper::tryFoldSelectOfConstants(GSelect *Select, if (CondTy != LLT::scalar(1)) return false; + if (TrueTy.isPointer()) + return false; + // Both are scalars. std::optional TrueOpt = getIConstantVRegValWithLookThrough(True, MRI); diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir index 7b73c8cec477..2bf7e84a379b 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir @@ -859,3 +859,40 @@ body: | RET_ReallyLR implicit $x0 ... +--- +name: dont_combine_pointer_type_select_of_constant +alignment: 4 +liveins: + - { reg: '$w0' } + - { reg: '$x1' } +body: | + bb.1: + liveins: $w0, $x1 + + ; CHECK-LABEL: name: dont_combine_pointer_type_select_of_constant + ; CHECK: liveins: $w0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0 + ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32) + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1 + ; CHECK-NEXT: [[ASSERT_ZEXT:%[0-9]+]]:_(s8) = G_ASSERT_ZEXT [[TRUNC]], 1 + ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s1) = G_TRUNC [[ASSERT_ZEXT]](s8) + ; CHECK-NEXT: [[C:%[0-9]+]]:_(p0) = G_CONSTANT i64 0 + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1 + ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[C1]](s64) + ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(p0) = G_SELECT [[TRUNC1]](s1), [[C]], [[INTTOPTR]] + ; CHECK-NEXT: G_STORE [[SELECT]](p0), [[COPY1]](p0) :: (store (p0)) + ; CHECK-NEXT: RET_ReallyLR + %3:_(s32) = COPY $w0 + %2:_(s8) = G_TRUNC %3(s32) + %1:_(p0) = COPY $x1 + %4:_(s8) = G_ASSERT_ZEXT %2, 1 + %0:_(s1) = G_TRUNC %4(s8) + %6:_(p0) = G_CONSTANT i64 0 + %8:_(s64) = G_CONSTANT i64 -1 + %7:_(p0) = G_INTTOPTR %8(s64) + %5:_(p0) = G_SELECT %0(s1), %6, %7 + G_STORE %5(p0), %1(p0) :: (store (p0)) + RET_ReallyLR + +... -- GitLab From b408241d0ad9ce009b49018fe1e9838887abf3c1 Mon Sep 17 00:00:00 2001 From: David CARLIER Date: Fri, 8 Mar 2024 00:00:26 +0000 Subject: [PATCH 533/929] [compiler-rt] adding fchmodat2 syscall introduced in Linux 6.6. (#82275) --- .../lib/sanitizer_common/sanitizer_common_syscalls.inc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc index c10943b3e487..b3161690f3ce 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc +++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc @@ -2808,6 +2808,15 @@ PRE_SYSCALL(fchownat) POST_SYSCALL(fchownat) (long res, long dfd, const void *filename, long user, long group, long flag) {} +PRE_SYSCALL(fchmodat2)(long dfd, const void *filename, long mode, long flag) { + if (filename) + PRE_READ(filename, + __sanitizer::internal_strlen((const char *)filename) + 1); +} + +POST_SYSCALL(fchmodat2) +(long res, long dfd, const void *filename, long mode, long flag) {} + PRE_SYSCALL(openat)(long dfd, const void *filename, long flags, long mode) { if (filename) PRE_READ(filename, -- GitLab From 487cfbe494413e12123b55dead5ef8742ef49fb2 Mon Sep 17 00:00:00 2001 From: OverMighty Date: Fri, 8 Mar 2024 00:01:37 +0000 Subject: [PATCH 534/929] [Clang] Implement constexpr support for `__builtin_popcountg` (#84318) --- clang/docs/LanguageExtensions.rst | 1 + clang/docs/ReleaseNotes.rst | 5 +++++ clang/include/clang/Basic/Builtins.td | 2 +- clang/lib/AST/ExprConstant.cpp | 1 + clang/test/Sema/constant-builtins-2.c | 7 +++++++ 5 files changed, 15 insertions(+), 1 deletion(-) diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst index 2b54dffd058a..06af93fd3c15 100644 --- a/clang/docs/LanguageExtensions.rst +++ b/clang/docs/LanguageExtensions.rst @@ -5378,6 +5378,7 @@ The following builtin intrinsics can be used in constant expressions: * ``__builtin_popcount`` * ``__builtin_popcountl`` * ``__builtin_popcountll`` +* ``__builtin_popcountg`` * ``__builtin_rotateleft8`` * ``__builtin_rotateleft16`` * ``__builtin_rotateleft32`` diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 42c4a7c4d4bd..fa23c215790f 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -157,6 +157,11 @@ Non-comprehensive list of changes in this release - ``__builtin_addc``, ``__builtin_subc``, and the other sizes of those builtins are now constexpr and may be used in constant expressions. +- Added ``__builtin_popcountg`` as a type-generic alternative to + ``__builtin_popcount{,l,ll}`` with support for any unsigned integer type. Like + the previous builtins, this new builtin is constexpr and may be used in + constant expressions. + New Compiler Flags ------------------ diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td index a81131d82c4c..9c703377ca8d 100644 --- a/clang/include/clang/Basic/Builtins.td +++ b/clang/include/clang/Basic/Builtins.td @@ -706,7 +706,7 @@ def Popcount : Builtin, BitInt_Long_LongLongTemplate { def Popcountg : Builtin { let Spellings = ["__builtin_popcountg"]; - let Attributes = [NoThrow, Const, CustomTypeChecking]; + let Attributes = [NoThrow, Const, Constexpr, CustomTypeChecking]; let Prototype = "int(...)"; } diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp index d8ca35740fbc..4a7c7755e1d6 100644 --- a/clang/lib/AST/ExprConstant.cpp +++ b/clang/lib/AST/ExprConstant.cpp @@ -12483,6 +12483,7 @@ bool IntExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E, case Builtin::BI__builtin_popcount: case Builtin::BI__builtin_popcountl: case Builtin::BI__builtin_popcountll: + case Builtin::BI__builtin_popcountg: case Builtin::BI__popcnt16: // Microsoft variants of popcount case Builtin::BI__popcnt: case Builtin::BI__popcnt64: { diff --git a/clang/test/Sema/constant-builtins-2.c b/clang/test/Sema/constant-builtins-2.c index 2bdd7b06daab..0935abe4c65f 100644 --- a/clang/test/Sema/constant-builtins-2.c +++ b/clang/test/Sema/constant-builtins-2.c @@ -237,6 +237,13 @@ char popcount7[__builtin_popcountl(~0L) == BITSIZE(long) ? 1 : -1]; char popcount8[__builtin_popcountll(0LL) == 0 ? 1 : -1]; char popcount9[__builtin_popcountll(0xF0F0LL) == 8 ? 1 : -1]; char popcount10[__builtin_popcountll(~0LL) == BITSIZE(long long) ? 1 : -1]; +char popcount11[__builtin_popcountg(0U) == 0 ? 1 : -1]; +char popcount12[__builtin_popcountg(0xF0F0U) == 8 ? 1 : -1]; +char popcount13[__builtin_popcountg(~0U) == BITSIZE(int) ? 1 : -1]; +char popcount14[__builtin_popcountg(~0UL) == BITSIZE(long) ? 1 : -1]; +char popcount15[__builtin_popcountg(~0ULL) == BITSIZE(long long) ? 1 : -1]; +char popcount16[__builtin_popcountg(~(unsigned __int128)0) == BITSIZE(__int128) ? 1 : -1]; +char popcount17[__builtin_popcountg(~(unsigned _BitInt(128))0) == BITSIZE(_BitInt(128)) ? 1 : -1]; char parity1[__builtin_parity(0) == 0 ? 1 : -1]; char parity2[__builtin_parity(0xb821) == 0 ? 1 : -1]; -- GitLab From e932fe880b69a6cd13b4f29678c7f143540f1999 Mon Sep 17 00:00:00 2001 From: David CARLIER Date: Fri, 8 Mar 2024 00:15:22 +0000 Subject: [PATCH 535/929] [compiler-rt][Fuzzer] fix windows typo (#84407) --- compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp b/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp index 0dbcec8b5f22..db80eb383885 100644 --- a/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp +++ b/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp @@ -243,7 +243,7 @@ void SetThreadName(std::thread &thread, const std::string &name) { HMODULE kbase = GetModuleHandleA("KernelBase.dll"); proc ThreadNameProc = reinterpret_cast(GetProcAddress(kbase, "SetThreadDescription")); - if (proc) { + if (ThreadNameProc) { std::wstring buf; auto sz = MultiByteToWideChar(CP_UTF8, 0, name.data(), -1, nullptr, 0); if (sz > 0) { -- GitLab From 0d4978f3cf8f917d88c19ec0ba3b1b3ef092cef1 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Thu, 7 Mar 2024 15:45:26 -0800 Subject: [PATCH 536/929] [RISCV] Update some tests I missed in 909ab0e0d1903ad2329ca9fdf248d21330f9437f. NFC --- llvm/test/CodeGen/RISCV/forced-atomics.ll | 2 +- llvm/test/CodeGen/RISCV/fpclamptosat.ll | 60 ++++++++++++----------- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/llvm/test/CodeGen/RISCV/forced-atomics.ll b/llvm/test/CodeGen/RISCV/forced-atomics.ll index 2b198afb47a9..659e0748dd53 100644 --- a/llvm/test/CodeGen/RISCV/forced-atomics.ll +++ b/llvm/test/CodeGen/RISCV/forced-atomics.ll @@ -3567,8 +3567,8 @@ define i64 @rmw64_umax_seq_cst(ptr %p) nounwind { ; RV32-NEXT: # in Loop: Header=BB51_2 Depth=1 ; RV32-NEXT: neg a3, a0 ; RV32-NEXT: and a3, a3, a1 -; RV32-NEXT: sw a1, 4(sp) ; RV32-NEXT: sw a4, 0(sp) +; RV32-NEXT: sw a1, 4(sp) ; RV32-NEXT: mv a1, sp ; RV32-NEXT: li a4, 5 ; RV32-NEXT: li a5, 5 diff --git a/llvm/test/CodeGen/RISCV/fpclamptosat.ll b/llvm/test/CodeGen/RISCV/fpclamptosat.ll index 6bfacc3e9814..630d16e7c888 100644 --- a/llvm/test/CodeGen/RISCV/fpclamptosat.ll +++ b/llvm/test/CodeGen/RISCV/fpclamptosat.ll @@ -1324,8 +1324,8 @@ define i64 @ustest_f64i64(double %x) { ; RV32IF-NEXT: # %bb.4: # %entry ; RV32IF-NEXT: li a0, 1 ; RV32IF-NEXT: .LBB20_5: # %entry -; RV32IF-NEXT: lw a3, 8(sp) -; RV32IF-NEXT: lw a4, 12(sp) +; RV32IF-NEXT: lw a4, 8(sp) +; RV32IF-NEXT: lw a3, 12(sp) ; RV32IF-NEXT: and a5, a2, a1 ; RV32IF-NEXT: beqz a5, .LBB20_7 ; RV32IF-NEXT: # %bb.6: # %entry @@ -1334,17 +1334,18 @@ define i64 @ustest_f64i64(double %x) { ; RV32IF-NEXT: .LBB20_7: ; RV32IF-NEXT: snez a1, a0 ; RV32IF-NEXT: .LBB20_8: # %entry -; RV32IF-NEXT: and a4, a2, a4 +; RV32IF-NEXT: and a3, a2, a3 ; RV32IF-NEXT: or a0, a0, a5 -; RV32IF-NEXT: and a2, a2, a3 +; RV32IF-NEXT: and a2, a2, a4 ; RV32IF-NEXT: bnez a0, .LBB20_10 ; RV32IF-NEXT: # %bb.9: -; RV32IF-NEXT: or a0, a2, a4 -; RV32IF-NEXT: snez a1, a0 +; RV32IF-NEXT: snez a0, a3 +; RV32IF-NEXT: snez a1, a2 +; RV32IF-NEXT: or a1, a1, a0 ; RV32IF-NEXT: .LBB20_10: # %entry ; RV32IF-NEXT: neg a1, a1 ; RV32IF-NEXT: and a0, a1, a2 -; RV32IF-NEXT: and a1, a1, a4 +; RV32IF-NEXT: and a1, a1, a3 ; RV32IF-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32IF-NEXT: addi sp, sp, 32 ; RV32IF-NEXT: ret @@ -1403,8 +1404,8 @@ define i64 @ustest_f64i64(double %x) { ; RV32IFD-NEXT: # %bb.4: # %entry ; RV32IFD-NEXT: li a0, 1 ; RV32IFD-NEXT: .LBB20_5: # %entry -; RV32IFD-NEXT: lw a3, 8(sp) -; RV32IFD-NEXT: lw a4, 12(sp) +; RV32IFD-NEXT: lw a4, 8(sp) +; RV32IFD-NEXT: lw a3, 12(sp) ; RV32IFD-NEXT: and a5, a2, a1 ; RV32IFD-NEXT: beqz a5, .LBB20_7 ; RV32IFD-NEXT: # %bb.6: # %entry @@ -1413,17 +1414,18 @@ define i64 @ustest_f64i64(double %x) { ; RV32IFD-NEXT: .LBB20_7: ; RV32IFD-NEXT: snez a1, a0 ; RV32IFD-NEXT: .LBB20_8: # %entry -; RV32IFD-NEXT: and a4, a2, a4 +; RV32IFD-NEXT: and a3, a2, a3 ; RV32IFD-NEXT: or a0, a0, a5 -; RV32IFD-NEXT: and a2, a2, a3 +; RV32IFD-NEXT: and a2, a2, a4 ; RV32IFD-NEXT: bnez a0, .LBB20_10 ; RV32IFD-NEXT: # %bb.9: -; RV32IFD-NEXT: or a0, a2, a4 -; RV32IFD-NEXT: snez a1, a0 +; RV32IFD-NEXT: snez a0, a3 +; RV32IFD-NEXT: snez a1, a2 +; RV32IFD-NEXT: or a1, a1, a0 ; RV32IFD-NEXT: .LBB20_10: # %entry ; RV32IFD-NEXT: neg a1, a1 ; RV32IFD-NEXT: and a0, a1, a2 -; RV32IFD-NEXT: and a1, a1, a4 +; RV32IFD-NEXT: and a1, a1, a3 ; RV32IFD-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32IFD-NEXT: addi sp, sp, 32 ; RV32IFD-NEXT: ret @@ -1594,8 +1596,8 @@ define i64 @ustest_f32i64(float %x) { ; RV32-NEXT: # %bb.4: # %entry ; RV32-NEXT: li a0, 1 ; RV32-NEXT: .LBB23_5: # %entry -; RV32-NEXT: lw a3, 8(sp) -; RV32-NEXT: lw a4, 12(sp) +; RV32-NEXT: lw a4, 8(sp) +; RV32-NEXT: lw a3, 12(sp) ; RV32-NEXT: and a5, a2, a1 ; RV32-NEXT: beqz a5, .LBB23_7 ; RV32-NEXT: # %bb.6: # %entry @@ -1604,17 +1606,18 @@ define i64 @ustest_f32i64(float %x) { ; RV32-NEXT: .LBB23_7: ; RV32-NEXT: snez a1, a0 ; RV32-NEXT: .LBB23_8: # %entry -; RV32-NEXT: and a4, a2, a4 +; RV32-NEXT: and a3, a2, a3 ; RV32-NEXT: or a0, a0, a5 -; RV32-NEXT: and a2, a2, a3 +; RV32-NEXT: and a2, a2, a4 ; RV32-NEXT: bnez a0, .LBB23_10 ; RV32-NEXT: # %bb.9: -; RV32-NEXT: or a0, a2, a4 -; RV32-NEXT: snez a1, a0 +; RV32-NEXT: snez a0, a3 +; RV32-NEXT: snez a1, a2 +; RV32-NEXT: or a1, a1, a0 ; RV32-NEXT: .LBB23_10: # %entry ; RV32-NEXT: neg a1, a1 ; RV32-NEXT: and a0, a1, a2 -; RV32-NEXT: and a1, a1, a4 +; RV32-NEXT: and a1, a1, a3 ; RV32-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32-NEXT: addi sp, sp, 32 ; RV32-NEXT: ret @@ -1847,8 +1850,8 @@ define i64 @ustest_f16i64(half %x) { ; RV32-NEXT: # %bb.4: # %entry ; RV32-NEXT: li a0, 1 ; RV32-NEXT: .LBB26_5: # %entry -; RV32-NEXT: lw a3, 8(sp) -; RV32-NEXT: lw a4, 12(sp) +; RV32-NEXT: lw a4, 8(sp) +; RV32-NEXT: lw a3, 12(sp) ; RV32-NEXT: and a5, a2, a1 ; RV32-NEXT: beqz a5, .LBB26_7 ; RV32-NEXT: # %bb.6: # %entry @@ -1857,17 +1860,18 @@ define i64 @ustest_f16i64(half %x) { ; RV32-NEXT: .LBB26_7: ; RV32-NEXT: snez a1, a0 ; RV32-NEXT: .LBB26_8: # %entry -; RV32-NEXT: and a4, a2, a4 +; RV32-NEXT: and a3, a2, a3 ; RV32-NEXT: or a0, a0, a5 -; RV32-NEXT: and a2, a2, a3 +; RV32-NEXT: and a2, a2, a4 ; RV32-NEXT: bnez a0, .LBB26_10 ; RV32-NEXT: # %bb.9: -; RV32-NEXT: or a0, a2, a4 -; RV32-NEXT: snez a1, a0 +; RV32-NEXT: snez a0, a3 +; RV32-NEXT: snez a1, a2 +; RV32-NEXT: or a1, a1, a0 ; RV32-NEXT: .LBB26_10: # %entry ; RV32-NEXT: neg a1, a1 ; RV32-NEXT: and a0, a1, a2 -; RV32-NEXT: and a1, a1, a4 +; RV32-NEXT: and a1, a1, a3 ; RV32-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32-NEXT: addi sp, sp, 32 ; RV32-NEXT: ret -- GitLab From fcd2d483251605f1b6cdace0ce5baf5dfd31b880 Mon Sep 17 00:00:00 2001 From: vadikp-intel Date: Thu, 7 Mar 2024 16:28:03 -0800 Subject: [PATCH 537/929] [OpenMP] runtime support for efficient partitioning of collapsed triangular loops (#83939) This PR adds OMP runtime support for more efficient partitioning of certain types of collapsed loops that can be used by compilers that support loop collapsing (i.e. MSVC) to achieve more optimal thread load balancing. In particular, this PR addresses double nested upper and lower isosceles triangular loops of the following types 1. lower triangular 'less_than' for (int i=0; i --- openmp/runtime/src/kmp_collapse.cpp | 311 ++++++++++++++++++ openmp/runtime/src/kmp_collapse.h | 11 +- .../omp_for_collapse_LowerTriangularLess.c | 124 +++++++ ...mp_for_collapse_LowerTriangularLessEqual.c | 124 +++++++ .../for/omp_for_collapse_UpperTriangular.c | 124 +++++++ 5 files changed, 692 insertions(+), 2 deletions(-) create mode 100644 openmp/runtime/test/worksharing/for/omp_for_collapse_LowerTriangularLess.c create mode 100644 openmp/runtime/test/worksharing/for/omp_for_collapse_LowerTriangularLessEqual.c create mode 100644 openmp/runtime/test/worksharing/for/omp_for_collapse_UpperTriangular.c diff --git a/openmp/runtime/src/kmp_collapse.cpp b/openmp/runtime/src/kmp_collapse.cpp index 2c410ca9b603..569d2c150831 100644 --- a/openmp/runtime/src/kmp_collapse.cpp +++ b/openmp/runtime/src/kmp_collapse.cpp @@ -1272,6 +1272,304 @@ void kmp_calc_original_ivs_for_end( } } +/************************************************************************** + * Identify nested loop structure - loops come in the canonical form + * Lower triangle matrix: i = 0; i <= N; i++ {0,0}:{N,0} + * j = 0; j <= 0/-1+1*i; j++ {0,0}:{0/-1,1} + * Upper Triangle matrix + * i = 0; i <= N; i++ {0,0}:{N,0} + * j = 0+1*i; j <= N; j++ {0,1}:{N,0} + * ************************************************************************/ +nested_loop_type_t +kmp_identify_nested_loop_structure(/*in*/ bounds_info_t *original_bounds_nest, + /*in*/ kmp_index_t n) { + // only 2-level nested loops are supported + if (n != 2) { + return nested_loop_type_unkown; + } + // loops must be canonical + KMP_ASSERT( + (original_bounds_nest[0].comparison == comparison_t::comp_less_or_eq) && + (original_bounds_nest[1].comparison == comparison_t::comp_less_or_eq)); + // check outer loop bounds: for triangular need to be {0,0}:{N,0} + kmp_uint64 outer_lb0_u64 = kmp_fix_iv(original_bounds_nest[0].loop_iv_type, + original_bounds_nest[0].lb0_u64); + kmp_uint64 outer_ub0_u64 = kmp_fix_iv(original_bounds_nest[0].loop_iv_type, + original_bounds_nest[0].ub0_u64); + kmp_uint64 outer_lb1_u64 = kmp_fix_iv(original_bounds_nest[0].loop_iv_type, + original_bounds_nest[0].lb1_u64); + kmp_uint64 outer_ub1_u64 = kmp_fix_iv(original_bounds_nest[0].loop_iv_type, + original_bounds_nest[0].ub1_u64); + if (outer_lb0_u64 != 0 || outer_lb1_u64 != 0 || outer_ub1_u64 != 0) { + return nested_loop_type_unkown; + } + // check inner bounds to determine triangle type + kmp_uint64 inner_lb0_u64 = kmp_fix_iv(original_bounds_nest[1].loop_iv_type, + original_bounds_nest[1].lb0_u64); + kmp_uint64 inner_ub0_u64 = kmp_fix_iv(original_bounds_nest[1].loop_iv_type, + original_bounds_nest[1].ub0_u64); + kmp_uint64 inner_lb1_u64 = kmp_fix_iv(original_bounds_nest[1].loop_iv_type, + original_bounds_nest[1].lb1_u64); + kmp_uint64 inner_ub1_u64 = kmp_fix_iv(original_bounds_nest[1].loop_iv_type, + original_bounds_nest[1].ub1_u64); + // lower triangle loop inner bounds need to be {0,0}:{0/-1,1} + if (inner_lb0_u64 == 0 && inner_lb1_u64 == 0 && + (inner_ub0_u64 == 0 || inner_ub0_u64 == -1) && inner_ub1_u64 == 1) { + return nested_loop_type_lower_triangular_matrix; + } + // upper triangle loop inner bounds need to be {0,1}:{N,0} + if (inner_lb0_u64 == 0 && inner_lb1_u64 == 1 && + inner_ub0_u64 == outer_ub0_u64 && inner_ub1_u64 == 0) { + return nested_loop_type_upper_triangular_matrix; + } + return nested_loop_type_unkown; +} + +/************************************************************************** + * SQRT Approximation: https://math.mit.edu/~stevenj/18.335/newton-sqrt.pdf + * Start point is x so the result is always > sqrt(x) + * The method has uniform convergence, PRECISION is set to 0.1 + * ************************************************************************/ +#define level_of_precision 0.1 +double sqrt_newton_approx(/*in*/ kmp_uint64 x) { + double sqrt_old = 0.; + double sqrt_new = (double)x; + do { + sqrt_old = sqrt_new; + sqrt_new = (sqrt_old + x / sqrt_old) / 2; + } while ((sqrt_old - sqrt_new) > level_of_precision); + return sqrt_new; +} + +/************************************************************************** + * Handle lower triangle matrix in the canonical form + * i = 0; i <= N; i++ {0,0}:{N,0} + * j = 0; j <= 0/-1 + 1*i; j++ {0,0}:{0/-1,1} + * ************************************************************************/ +void kmp_handle_lower_triangle_matrix( + /*in*/ kmp_uint32 nth, + /*in*/ kmp_uint32 tid, + /*in */ kmp_index_t n, + /*in/out*/ bounds_info_t *original_bounds_nest, + /*out*/ bounds_info_t *chunk_bounds_nest) { + + // transfer loop types from the original loop to the chunks + for (kmp_index_t i = 0; i < n; ++i) { + chunk_bounds_nest[i] = original_bounds_nest[i]; + } + // cleanup iv variables + kmp_uint64 outer_ub0 = kmp_fix_iv(original_bounds_nest[0].loop_iv_type, + original_bounds_nest[0].ub0_u64); + kmp_uint64 outer_lb0 = kmp_fix_iv(original_bounds_nest[0].loop_iv_type, + original_bounds_nest[0].lb0_u64); + kmp_uint64 inner_ub0 = kmp_fix_iv(original_bounds_nest[1].loop_iv_type, + original_bounds_nest[1].ub0_u64); + // calculate the chunk's lower and upper bounds + // the total number of iterations in the loop is the sum of the arithmetic + // progression from the outer lower to outer upper bound (inclusive since the + // loop is canonical) note that less_than inner loops (inner_ub0 = -1) + // effectively make the progression 1-based making N = (outer_ub0 - inner_lb0 + // + 1) -> N - 1 + kmp_uint64 outer_iters = (outer_ub0 - outer_lb0 + 1) + inner_ub0; + kmp_uint64 iter_total = outer_iters * (outer_iters + 1) / 2; + // the current thread's number of iterations: + // each thread gets an equal number of iterations: total number of iterations + // divided by the number of threads plus, if there's a remainder, + // the first threads with the number up to the remainder get an additional + // iteration each to cover it + kmp_uint64 iter_current = + iter_total / nth + ((tid < (iter_total % nth)) ? 1 : 0); + // cumulative number of iterations executed by all the previous threads: + // threads with the tid below the remainder will have (iter_total/nth+1) + // elements, and so will all threads before them so the cumulative number of + // iterations executed by the all previous will be the current thread's number + // of iterations multiplied by the number of previous threads which is equal + // to the current thread's tid; threads with the number equal or above the + // remainder will have (iter_total/nth) elements so the cumulative number of + // iterations previously executed is its number of iterations multipled by the + // number of previous threads which is again equal to the current thread's tid + // PLUS all the remainder iterations that will have been executed by the + // previous threads + kmp_uint64 iter_before_current = + tid * iter_current + ((tid < iter_total % nth) ? 0 : (iter_total % nth)); + // cumulative number of iterations executed with the current thread is + // the cumulative number executed before it plus its own + kmp_uint64 iter_with_current = iter_before_current + iter_current; + // calculate the outer loop lower bound (lbo) which is the max outer iv value + // that gives the number of iterations that is equal or just below the total + // number of iterations executed by the previous threads, for less_than + // (1-based) inner loops (inner_ub0 == -1) it will be i.e. + // lbo*(lbo-1)/2<=iter_before_current => lbo^2-lbo-2*iter_before_current<=0 + // for less_than_equal (0-based) inner loops (inner_ub == 0) it will be: + // i.e. lbo*(lbo+1)/2<=iter_before_current => + // lbo^2+lbo-2*iter_before_current<=0 both cases can be handled similarily + // using a parameter to control the equation sign + kmp_int64 inner_adjustment = 1 + 2 * inner_ub0; + kmp_uint64 lower_bound_outer = + (kmp_uint64)(sqrt_newton_approx(inner_adjustment * inner_adjustment + + 8 * iter_before_current) + + inner_adjustment) / + 2 - + inner_adjustment; + // calculate the inner loop lower bound which is the remaining number of + // iterations required to hit the total number of iterations executed by the + // previous threads giving the starting point of this thread + kmp_uint64 lower_bound_inner = + iter_before_current - + ((lower_bound_outer + inner_adjustment) * lower_bound_outer) / 2; + // calculate the outer loop upper bound using the same approach as for the + // inner bound except using the total number of iterations executed with the + // current thread + kmp_uint64 upper_bound_outer = + (kmp_uint64)(sqrt_newton_approx(inner_adjustment * inner_adjustment + + 8 * iter_with_current) + + inner_adjustment) / + 2 - + inner_adjustment; + // calculate the inner loop upper bound which is the remaining number of + // iterations required to hit the total number of iterations executed after + // the current thread giving the starting point of the next thread + kmp_uint64 upper_bound_inner = + iter_with_current - + ((upper_bound_outer + inner_adjustment) * upper_bound_outer) / 2; + // adjust the upper bounds down by 1 element to point at the last iteration of + // the current thread the first iteration of the next thread + if (upper_bound_inner == 0) { + // {n,0} => {n-1,n-1} + upper_bound_outer -= 1; + upper_bound_inner = upper_bound_outer; + } else { + // {n,m} => {n,m-1} (m!=0) + upper_bound_inner -= 1; + } + + // assign the values, zeroing out lb1 and ub1 values since the iteration space + // is now one-dimensional + chunk_bounds_nest[0].lb0_u64 = lower_bound_outer; + chunk_bounds_nest[1].lb0_u64 = lower_bound_inner; + chunk_bounds_nest[0].ub0_u64 = upper_bound_outer; + chunk_bounds_nest[1].ub0_u64 = upper_bound_inner; + chunk_bounds_nest[0].lb1_u64 = 0; + chunk_bounds_nest[0].ub1_u64 = 0; + chunk_bounds_nest[1].lb1_u64 = 0; + chunk_bounds_nest[1].ub1_u64 = 0; + +#if 0 + printf("tid/nth = %d/%d : From [%llu, %llu] To [%llu, %llu] : Chunks %llu/%llu\n", + tid, nth, chunk_bounds_nest[0].lb0_u64, chunk_bounds_nest[1].lb0_u64, + chunk_bounds_nest[0].ub0_u64, chunk_bounds_nest[1].ub0_u64, iter_current, iter_total); +#endif +} + +/************************************************************************** + * Handle upper triangle matrix in the canonical form + * i = 0; i <= N; i++ {0,0}:{N,0} + * j = 0+1*i; j <= N; j++ {0,1}:{N,0} + * ************************************************************************/ +void kmp_handle_upper_triangle_matrix( + /*in*/ kmp_uint32 nth, + /*in*/ kmp_uint32 tid, + /*in */ kmp_index_t n, + /*in/out*/ bounds_info_t *original_bounds_nest, + /*out*/ bounds_info_t *chunk_bounds_nest) { + + // transfer loop types from the original loop to the chunks + for (kmp_index_t i = 0; i < n; ++i) { + chunk_bounds_nest[i] = original_bounds_nest[i]; + } + // cleanup iv variables + kmp_uint64 outer_ub0 = kmp_fix_iv(original_bounds_nest[0].loop_iv_type, + original_bounds_nest[0].ub0_u64); + kmp_uint64 outer_lb0 = kmp_fix_iv(original_bounds_nest[0].loop_iv_type, + original_bounds_nest[0].lb0_u64); + kmp_uint64 inner_ub0 = kmp_fix_iv(original_bounds_nest[1].loop_iv_type, + original_bounds_nest[1].ub0_u64); + // calculate the chunk's lower and upper bounds + // the total number of iterations in the loop is the sum of the arithmetic + // progression from the outer lower to outer upper bound (inclusive since the + // loop is canonical) note that less_than inner loops (inner_ub0 = -1) + // effectively make the progression 1-based making N = (outer_ub0 - inner_lb0 + // + 1) -> N - 1 + kmp_uint64 outer_iters = (outer_ub0 - outer_lb0 + 1); + kmp_uint64 iter_total = outer_iters * (outer_iters + 1) / 2; + // the current thread's number of iterations: + // each thread gets an equal number of iterations: total number of iterations + // divided by the number of threads plus, if there's a remainder, + // the first threads with the number up to the remainder get an additional + // iteration each to cover it + kmp_uint64 iter_current = + iter_total / nth + ((tid < (iter_total % nth)) ? 1 : 0); + // cumulative number of iterations executed by all the previous threads: + // threads with the tid below the remainder will have (iter_total/nth+1) + // elements, and so will all threads before them so the cumulative number of + // iterations executed by the all previous will be the current thread's number + // of iterations multiplied by the number of previous threads which is equal + // to the current thread's tid; threads with the number equal or above the + // remainder will have (iter_total/nth) elements so the cumulative number of + // iterations previously executed is its number of iterations multipled by the + // number of previous threads which is again equal to the current thread's tid + // PLUS all the remainder iterations that will have been executed by the + // previous threads + kmp_uint64 iter_before_current = + tid * iter_current + ((tid < iter_total % nth) ? 0 : (iter_total % nth)); + // cumulative number of iterations executed with the current thread is + // the cumulative number executed before it plus its own + kmp_uint64 iter_with_current = iter_before_current + iter_current; + // calculate the outer loop lower bound (lbo) which is the max outer iv value + // that gives the number of iterations that is equal or just below the total + // number of iterations executed by the previous threads, for less_than + // (1-based) inner loops (inner_ub0 == -1) it will be i.e. + // lbo*(lbo-1)/2<=iter_before_current => lbo^2-lbo-2*iter_before_current<=0 + // for less_than_equal (0-based) inner loops (inner_ub == 0) it will be: + // i.e. lbo*(lbo+1)/2<=iter_before_current => + // lbo^2+lbo-2*iter_before_current<=0 both cases can be handled similarily + // using a parameter to control the equatio sign + kmp_uint64 lower_bound_outer = + (kmp_uint64)(sqrt_newton_approx(1 + 8 * iter_before_current) + 1) / 2 - 1; + ; + // calculate the inner loop lower bound which is the remaining number of + // iterations required to hit the total number of iterations executed by the + // previous threads giving the starting point of this thread + kmp_uint64 lower_bound_inner = + iter_before_current - ((lower_bound_outer + 1) * lower_bound_outer) / 2; + // calculate the outer loop upper bound using the same approach as for the + // inner bound except using the total number of iterations executed with the + // current thread + kmp_uint64 upper_bound_outer = + (kmp_uint64)(sqrt_newton_approx(1 + 8 * iter_with_current) + 1) / 2 - 1; + // calculate the inner loop upper bound which is the remaining number of + // iterations required to hit the total number of iterations executed after + // the current thread giving the starting point of the next thread + kmp_uint64 upper_bound_inner = + iter_with_current - ((upper_bound_outer + 1) * upper_bound_outer) / 2; + // adjust the upper bounds down by 1 element to point at the last iteration of + // the current thread the first iteration of the next thread + if (upper_bound_inner == 0) { + // {n,0} => {n-1,n-1} + upper_bound_outer -= 1; + upper_bound_inner = upper_bound_outer; + } else { + // {n,m} => {n,m-1} (m!=0) + upper_bound_inner -= 1; + } + + // assign the values, zeroing out lb1 and ub1 values since the iteration space + // is now one-dimensional + chunk_bounds_nest[0].lb0_u64 = (outer_iters - 1) - upper_bound_outer; + chunk_bounds_nest[1].lb0_u64 = (outer_iters - 1) - upper_bound_inner; + chunk_bounds_nest[0].ub0_u64 = (outer_iters - 1) - lower_bound_outer; + chunk_bounds_nest[1].ub0_u64 = (outer_iters - 1) - lower_bound_inner; + chunk_bounds_nest[0].lb1_u64 = 0; + chunk_bounds_nest[0].ub1_u64 = 0; + chunk_bounds_nest[1].lb1_u64 = 0; + chunk_bounds_nest[1].ub1_u64 = 0; + +#if 0 + printf("tid/nth = %d/%d : From [%llu, %llu] To [%llu, %llu] : Chunks %llu/%llu\n", + tid, nth, chunk_bounds_nest[0].lb0_u64, chunk_bounds_nest[1].lb0_u64, + chunk_bounds_nest[0].ub0_u64, chunk_bounds_nest[1].ub0_u64, iter_current, iter_total); +#endif +} //----------Init API for non-rectangular loops-------------------------------- // Init API for collapsed loops (static, no chunks defined). @@ -1334,6 +1632,19 @@ __kmpc_for_collapsed_init(ident_t *loc, kmp_int32 gtid, KMP_DEBUG_ASSERT(tid < nth); + // Handle special cases + nested_loop_type_t loop_type = + kmp_identify_nested_loop_structure(original_bounds_nest, n); + if (loop_type == nested_loop_type_lower_triangular_matrix) { + kmp_handle_lower_triangle_matrix(nth, tid, n, original_bounds_nest, + chunk_bounds_nest); + return TRUE; + } else if (loop_type == nested_loop_type_upper_triangular_matrix) { + kmp_handle_upper_triangle_matrix(nth, tid, n, original_bounds_nest, + chunk_bounds_nest); + return TRUE; + } + CollapseAllocator original_ivs_start(n); if (!kmp_calc_original_ivs_for_start(original_bounds_nest, n, diff --git a/openmp/runtime/src/kmp_collapse.h b/openmp/runtime/src/kmp_collapse.h index e4870185645d..1044478554a0 100644 --- a/openmp/runtime/src/kmp_collapse.h +++ b/openmp/runtime/src/kmp_collapse.h @@ -45,6 +45,13 @@ enum loop_type_t : kmp_int32 { loop_type_int64 = 7 }; +// Defining loop types to handle special cases +enum nested_loop_type_t : kmp_int32 { + nested_loop_type_unkown = 0, + nested_loop_type_lower_triangular_matrix = 1, + nested_loop_type_upper_triangular_matrix = 2 +}; + /*! @ingroup WORK_SHARING * Describes the structure for rectangular nested loops. @@ -124,14 +131,14 @@ struct bounds_info_t { // It's represented in kmp_uint64, but each dimention is calculated in // that loop IV type. Also dimentions have to be converted to those types // when used in generated code. -typedef kmp_uint64* kmp_point_t; +typedef kmp_uint64 *kmp_point_t; // Array: Number of loop iterations on each nesting level to achieve some point, // in expanded space or in original space. // OMPTODO: move from using iterations to using offsets (iterations multiplied // by steps). For those we need to be careful with the types, as step can be // negative, but it'll remove multiplications and divisions in several places. -typedef kmp_loop_nest_iv_t* kmp_iterations_t; +typedef kmp_loop_nest_iv_t *kmp_iterations_t; // Internal struct with additional info: template struct bounds_info_internalXX_template { diff --git a/openmp/runtime/test/worksharing/for/omp_for_collapse_LowerTriangularLess.c b/openmp/runtime/test/worksharing/for/omp_for_collapse_LowerTriangularLess.c new file mode 100644 index 000000000000..9d742066cf1f --- /dev/null +++ b/openmp/runtime/test/worksharing/for/omp_for_collapse_LowerTriangularLess.c @@ -0,0 +1,124 @@ +// RUN: %libomp-compile-and-run +#include +#include +#include +#include "omp.h" + +#ifndef MAX_BOUND +#define MAX_BOUND 64 +#endif +#ifndef _MSC_VER +#define NO_EFFICIENCY_CHECK +#endif + +/* To ensure Correctness, only valid iterations are executed and are executed + only once. Stores the number of times an iteration is executed. */ +unsigned *execution_count = NULL; +/* Stores the number of iterations executed by each thread. */ +unsigned *iterations_per_thread = NULL; + +unsigned *Alloc(unsigned bound1, unsigned bound2) { + return (unsigned *)(malloc(bound1 * bound2 * sizeof(unsigned))); +} + +void ZeroOut(unsigned *p, unsigned bound1, unsigned bound2) { + memset(p, 0, bound1 * bound2 * sizeof(unsigned)); +} + +void Free(unsigned *p) { free((void *)p); } + +unsigned *Index(unsigned *p, unsigned i, unsigned j, unsigned bound2) { + return &p[i * bound2 + j]; +} + +int test(unsigned upper_bound) { + + unsigned total_iterations = upper_bound * (upper_bound - 1) / 2; + unsigned num_threads = omp_get_max_threads(); + unsigned lower_per_chunk = total_iterations / num_threads; + unsigned upper_per_chunk = + lower_per_chunk + ((total_iterations % num_threads) ? 1 : 0); + int i, j; + + omp_set_num_threads(num_threads); + + ZeroOut(execution_count, upper_bound, upper_bound); + ZeroOut(iterations_per_thread, num_threads, 1); + +#ifdef VERBOSE + fprintf(stderr, + "INFO: Using %6d threads for %6d outer iterations with %6d [%6d:%6d] " + "chunks " + "loop type lower triangle <,< - ", + num_threads, upper_bound, total_iterations, lower_per_chunk, + upper_per_chunk); +#endif + +#pragma omp parallel shared(iterations_per_thread, execution_count) + { /* begin of parallel */ + /* Lower triangular execution_count matrix */ +#pragma omp for schedule(static) collapse(2) + for (i = 0; i < upper_bound; i++) { + for (j = 0; j < i; j++) { + (*Index(iterations_per_thread, omp_get_thread_num(), 0, 1))++; + (*Index(execution_count, i, j, upper_bound))++; + } + } /* end of for*/ + } /* end of parallel */ + + /* check the execution_count array */ + for (i = 0; i < upper_bound; i++) { + for (j = 0; j < i; j++) { + unsigned value = *Index(execution_count, i, j, upper_bound); + /* iteration with j<=i are valid, but should have been executed only once + */ + if (value != 1) { + fprintf(stderr, "ERROR: valid iteration [%i,%i] executed %i times.\n", + i, j, value); + return 0; + } + } + for (j = i; j < upper_bound; j++) { + unsigned value = *Index(execution_count, i, j, upper_bound); + /* iteration with j>=i are invalid and should not have been executed + */ + if (value > 0) { + fprintf(stderr, "ERROR: invalid iteration [%i,%i] executed %i times.\n", + i, j, value); + return 0; + } + } + } + +#ifndef NO_EFFICIENCY_CHECK + /* Ensure the number of iterations executed by each thread is within bounds */ + for (i = 0; i < num_threads; i++) { + unsigned value = *Index(iterations_per_thread, i, 0, 1); + if (value < lower_per_chunk || value > upper_per_chunk) { + fprintf(stderr, + "ERROR: Inefficient Collapse thread %d of %d assigned %i " + "iterations; must be between %d and %d\n", + i, num_threads, value, lower_per_chunk, upper_per_chunk); + return 0; + } + } +#endif +#ifdef VERBOSE + fprintf(stderr, "PASSED\r\n"); +#endif + return 1; +} + +int main() { + + execution_count = Alloc(MAX_BOUND, MAX_BOUND); + iterations_per_thread = Alloc(omp_get_max_threads(), 1); + + for (unsigned j = 0; j < MAX_BOUND; j++) { + if (!test(j)) + return 1; + } + Free(execution_count); + Free(iterations_per_thread); + return 0; +} diff --git a/openmp/runtime/test/worksharing/for/omp_for_collapse_LowerTriangularLessEqual.c b/openmp/runtime/test/worksharing/for/omp_for_collapse_LowerTriangularLessEqual.c new file mode 100644 index 000000000000..154ee0f69daa --- /dev/null +++ b/openmp/runtime/test/worksharing/for/omp_for_collapse_LowerTriangularLessEqual.c @@ -0,0 +1,124 @@ +// RUN: %libomp-compile-and-run +#include +#include +#include +#include "omp.h" + +#ifndef MAX_BOUND +#define MAX_BOUND 64 +#endif +#ifndef _MSC_VER +#define NO_EFFICIENCY_CHECK +#endif + +/* To ensure Correctness, only valid iterations are executed and are executed + only once. Stores the number of times an iteration is executed. */ +unsigned *execution_count = NULL; +/* Stores the number of iterations executed by each thread. */ +unsigned *iterations_per_thread = NULL; + +unsigned *Alloc(unsigned bound1, unsigned bound2) { + return (unsigned *)(malloc(bound1 * bound2 * sizeof(unsigned))); +} + +void ZeroOut(unsigned *p, unsigned bound1, unsigned bound2) { + memset(p, 0, bound1 * bound2 * sizeof(unsigned)); +} + +void Free(unsigned *p) { free((void *)p); } + +unsigned *Index(unsigned *p, unsigned i, unsigned j, unsigned bound2) { + return &p[i * bound2 + j]; +} + +int test(int upper_bound) { + + unsigned total_iterations = upper_bound * (upper_bound + 1) / 2; + unsigned num_threads = omp_get_max_threads(); + unsigned lower_per_chunk = total_iterations / num_threads; + unsigned upper_per_chunk = + lower_per_chunk + ((total_iterations % num_threads) ? 1 : 0); + int i, j; + + omp_set_num_threads(num_threads); + + ZeroOut(execution_count, upper_bound, upper_bound); + ZeroOut(iterations_per_thread, num_threads, 1); + +#ifdef VERBOSE + fprintf(stderr, + "INFO: Using %6d threads for %6d outer iterations with %6d [%6d:%6d] " + "chunks " + "loop type lower triangle <,<= - ", + num_threads, upper_bound, total_iterations, lower_per_chunk, + upper_per_chunk); +#endif + +#pragma omp parallel shared(iterations_per_thread, execution_count) + { /* begin of parallel */ + /* Lower triangular execution_count matrix */ +#pragma omp for schedule(static) collapse(2) + for (i = 0; i < upper_bound; i++) { + for (j = 0; j <= i; j++) { + (*Index(iterations_per_thread, omp_get_thread_num(), 0, 1))++; + (*Index(execution_count, i, j, upper_bound))++; + } + } /* end of for*/ + } /* end of parallel */ + + /* check the execution_count array */ + for (i = 0; i < upper_bound; i++) { + for (j = 0; j <= i; j++) { + unsigned value = *Index(execution_count, i, j, upper_bound); + /* iteration with j<=i are valid, but should have been executed only once + */ + if (value != 1) { + fprintf(stderr, "ERROR: valid iteration [%i,%i] executed %i times.\n", + i, j, value); + return 0; + } + } + for (j = i + 1; j < upper_bound; j++) { + unsigned value = *Index(execution_count, i, j, upper_bound); + /* iteration with j>=i are invalid and should not have been executed + */ + if (value > 0) { + fprintf(stderr, "ERROR: invalid iteration [%i,%i] executed %i times.\n", + i, j, value); + return 0; + } + } + } + +#ifndef NO_EFFICIENCY_CHECK + /* Ensure the number of iterations executed by each thread is within bounds */ + for (i = 0; i < num_threads; i++) { + unsigned value = *Index(iterations_per_thread, i, 0, 1); + if (value < lower_per_chunk || value > upper_per_chunk) { + fprintf(stderr, + "ERROR: Inefficient Collapse thread %d of %d assigned %i " + "iterations; must be between %d and %d\n", + i, num_threads, value, lower_per_chunk, upper_per_chunk); + return 0; + } + } +#endif +#ifdef VERBOSE + fprintf(stderr, "PASSED\r\n"); +#endif + return 1; +} + +int main() { + + execution_count = Alloc(MAX_BOUND, MAX_BOUND); + iterations_per_thread = Alloc(omp_get_max_threads(), 1); + + for (unsigned j = 0; j < MAX_BOUND; j++) { + if (!test(j)) + return 1; + } + Free(execution_count); + Free(iterations_per_thread); + return 0; +} diff --git a/openmp/runtime/test/worksharing/for/omp_for_collapse_UpperTriangular.c b/openmp/runtime/test/worksharing/for/omp_for_collapse_UpperTriangular.c new file mode 100644 index 000000000000..452410025be0 --- /dev/null +++ b/openmp/runtime/test/worksharing/for/omp_for_collapse_UpperTriangular.c @@ -0,0 +1,124 @@ +// RUN: %libomp-compile-and-run +#include +#include +#include +#include "omp.h" + +#ifndef MAX_BOUND +#define MAX_BOUND 64 +#endif +#ifndef _MSC_VER +#define NO_EFFICIENCY_CHECK +#endif + +/* To ensure Correctness, only valid iterations are executed and are executed + only once. Stores the number of times an iteration is executed. */ +unsigned *execution_count = NULL; +/* Stores the number of iterations executed by each thread. */ +unsigned *iterations_per_thread = NULL; + +unsigned *Alloc(unsigned bound1, unsigned bound2) { + return (unsigned *)(malloc(bound1 * bound2 * sizeof(unsigned))); +} + +void ZeroOut(unsigned *p, unsigned bound1, unsigned bound2) { + memset(p, 0, bound1 * bound2 * sizeof(unsigned)); +} + +void Free(unsigned *p) { free((void *)p); } + +unsigned *Index(unsigned *p, unsigned i, unsigned j, unsigned bound2) { + return &p[i * bound2 + j]; +} + +int test(unsigned upper_bound) { + + unsigned total_iterations = upper_bound * (upper_bound + 1) / 2; + unsigned num_threads = omp_get_max_threads(); + unsigned lower_per_chunk = total_iterations / num_threads; + unsigned upper_per_chunk = + lower_per_chunk + ((total_iterations % num_threads) ? 1 : 0); + int i, j; + + omp_set_num_threads(num_threads); + + ZeroOut(execution_count, upper_bound, upper_bound); + ZeroOut(iterations_per_thread, num_threads, 1); + +#ifdef VERBOSE + fprintf(stderr, + "INFO: Using %6d threads for %6d outer iterations with %6d [%6d:%6d] " + "chunks " + "loop type upper triangle <,< - ", + num_threads, upper_bound, total_iterations, lower_per_chunk, + upper_per_chunk); +#endif + +#pragma omp parallel shared(iterations_per_thread, execution_count) + { /* begin of parallel */ + /* Lower triangular execution_count matrix */ +#pragma omp for schedule(static) collapse(2) + for (i = 0; i < upper_bound; i++) { + for (j = i; j < upper_bound; j++) { + (*Index(iterations_per_thread, omp_get_thread_num(), 0, 1))++; + (*Index(execution_count, i, j, upper_bound))++; + } + } /* end of for*/ + } /* end of parallel */ + + /* check the execution_count array */ + for (i = 0; i < upper_bound; i++) { + for (j = i; j < upper_bound; j++) { + unsigned value = *Index(execution_count, i, j, upper_bound); + /* iteration with j<=i are valid, but should have been executed only once + */ + if (value != 1) { + fprintf(stderr, "ERROR: valid iteration [%i,%i] executed %i times.\n", + i, j, value); + return 0; + } + } + for (j = 0; j < i; j++) { + unsigned value = *Index(execution_count, i, j, upper_bound); + /* iteration with j>=i are invalid and should not have been executed + */ + if (value > 0) { + fprintf(stderr, "ERROR: invalid iteration [%i,%i] executed %i times.\n", + i, j, value); + return 0; + } + } + } + +#ifndef NO_EFFICIENCY_CHECK + /* Ensure the number of iterations executed by each thread is within bounds */ + for (i = 0; i < num_threads; i++) { + unsigned value = *Index(iterations_per_thread, i, 0, 1); + if (value < lower_per_chunk || value > upper_per_chunk) { + fprintf(stderr, + "ERROR: Inefficient Collapse thread %d of %d assigned %i " + "iterations; must be between %d and %d\n", + i, num_threads, value, lower_per_chunk, upper_per_chunk); + return 0; + } + } +#endif +#ifdef VERBOSE + fprintf(stderr, "PASSED\r\n"); +#endif + return 1; +} + +int main() { + + execution_count = Alloc(MAX_BOUND, MAX_BOUND); + iterations_per_thread = Alloc(omp_get_max_threads(), 1); + + for (unsigned j = 0; j < MAX_BOUND; j++) { + if (!test(j)) + return 1; + } + Free(execution_count); + Free(iterations_per_thread); + return 0; +} -- GitLab From 0cd7942c7f7a6f0c8a749c5f0d6d758e0a6fd9d9 Mon Sep 17 00:00:00 2001 From: Igor Kudrin Date: Thu, 7 Mar 2024 16:34:20 -0800 Subject: [PATCH 538/929] [llvm-dwarfdump] Fix parsing DW_CFA_AARCH64_negate_ra_state (#84128) The saved state of the AARCH64_DWARF_PAUTH_RA_STATE register was not updated, so `llvm-dwarfdump` continued to dump it as `reg34=1` even if the correct value is `0`: ``` > llvm-dwarfdump -v test.o ... 0000002c 00000024 00000030 FDE cie=00000000 pc=00000030...00000064 Format: DWARF32 DW_CFA_advance_loc: 4 DW_CFA_AARCH64_negate_ra_state: DW_CFA_advance_loc: 4 DW_CFA_def_cfa_offset: +16 DW_CFA_offset: W30 -16 DW_CFA_remember_state: DW_CFA_advance_loc: 16 DW_CFA_def_cfa_offset: +0 DW_CFA_advance_loc: 4 DW_CFA_AARCH64_negate_ra_state: DW_CFA_restore: W30 DW_CFA_advance_loc: 4 DW_CFA_restore_state: DW_CFA_advance_loc: 12 DW_CFA_def_cfa_offset: +0 DW_CFA_advance_loc: 4 DW_CFA_AARCH64_negate_ra_state: DW_CFA_restore: W30 DW_CFA_nop: 0x30: CFA=WSP 0x34: CFA=WSP: reg34=1 0x38: CFA=WSP+16: W30=[CFA-16], reg34=1 0x48: CFA=WSP: W30=[CFA-16], reg34=1 0x4c: CFA=WSP: reg34=1 <--- should be '=0' 0x50: CFA=WSP+16: W30=[CFA-16], reg34=1 0x5c: CFA=WSP: W30=[CFA-16], reg34=1 0x60: CFA=WSP: reg34=1 <--- should be '=0' ``` --- llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp | 2 ++ ...sign-return-address-cfi-negate-ra-state.ll | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp b/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp index aae1668c1639..0c968703f80b 100644 --- a/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp +++ b/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp @@ -630,6 +630,8 @@ Error UnwindTable::parseRows(const CFIProgram &CFIP, UnwindRow &Row, if (LRLoc->getLocation() == UnwindLocation::Constant) { // Toggle the constant value from 0 to 1 or 1 to 0. LRLoc->setConstant(LRLoc->getConstant() ^ 1); + Row.getRegisterLocations().setRegisterLocation( + AArch64DWARFPAuthRaState, *LRLoc); } else { return createStringError( errc::invalid_argument, diff --git a/llvm/test/CodeGen/AArch64/sign-return-address-cfi-negate-ra-state.ll b/llvm/test/CodeGen/AArch64/sign-return-address-cfi-negate-ra-state.ll index da2c2985acf9..9464e3447993 100644 --- a/llvm/test/CodeGen/AArch64/sign-return-address-cfi-negate-ra-state.ll +++ b/llvm/test/CodeGen/AArch64/sign-return-address-cfi-negate-ra-state.ll @@ -213,6 +213,10 @@ attributes #0 = { "sign-return-address"="all" } ; CHECK-DUMP-NOT: DW_CFA_remember_state ; CHECK-DUMP-NOT: DW_CFA_restore_state +; CHECK-DUMP: CFA=WSP{{$}} +; CHECK-DUMP: reg34=1 +; CHECK-DUMP-NOT: reg34=0 + ; baz_async ; CHECK-DUMP-LABEL: FDE ; CHECK-DUMP: Format: DWARF32 @@ -222,9 +226,24 @@ attributes #0 = { "sign-return-address"="all" } ; CHECK-DUMP: DW_CFA_restore_state: ; CHECK-DUMP: DW_CFA_AARCH64_negate_ra_state: +; CHECK-DUMP: CFA=WSP{{$}} +;; First DW_CFA_AARCH64_negate_ra_state: +; CHECK-DUMP: reg34=1 +;; Second DW_CFA_AARCH64_negate_ra_state: +; CHECK-DUMP: reg34=0 +;; DW_CFA_restore_state: +; CHECK-DUMP: reg34=1 +;; Third DW_CFA_AARCH64_negate_ra_state: +; CHECK-DUMP: reg34=0 +; CHECK-DUMP-NOT: reg34= + ; baz_sync ; CHECK-DUMP-LABEL: FDE ; CHECK-DUMP: DW_CFA_AARCH64_negate_ra_state: ; CHECK-DUMP-NOT: DW_CFA_AARCH64_negate_ra_state ; CHECK-DUMP-NOT: DW_CFA_remember_state ; CHECK-DUMP-NOT: DW_CFA_restore_state + +; CHECK-DUMP: CFA=WSP{{$}} +; CHECK-DUMP: reg34=1 +; CHECK-DUMP-NOT: reg34=0 -- GitLab From fe8476472467acd15a4d3771313e5532d1eb032f Mon Sep 17 00:00:00 2001 From: Igor Kudrin Date: Thu, 7 Mar 2024 16:34:36 -0800 Subject: [PATCH 539/929] [DWARF] Dump an updated location for DW_CFA_advance_loc* (#84274) When dumping FDEs, `readelf` prints new location values after `DW_CFA_advance_loc(*)` instructions, which looks quite convenient: ``` > readelf -wf test.o ... ... FDE ... pc=0000000000000030..0000000000000064 DW_CFA_advance_loc: 4 to 0000000000000034 ... DW_CFA_advance_loc: 4 to 0000000000000038 ... ``` This patch makes `llvm-dwarfdump` and `llvm-readobj` do the same. --- .../llvm/DebugInfo/DWARF/DWARFDebugFrame.h | 6 ++-- llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp | 19 +++++++---- .../dwarfdump-debug-frame-simple.test | 6 ++-- llvm/test/tools/llvm-readobj/ELF/unwind.test | 34 +++++++++---------- llvm/tools/llvm-readobj/DwarfCFIEHPrinter.h | 7 ++-- 5 files changed, 41 insertions(+), 31 deletions(-) diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h index bc35f2ab988e..c7c558850a28 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h @@ -454,8 +454,8 @@ public: /// where a problem occurred in case an error is returned. Error parse(DWARFDataExtractor Data, uint64_t *Offset, uint64_t EndOffset); - void dump(raw_ostream &OS, DIDumpOptions DumpOpts, - unsigned IndentLevel = 1) const; + void dump(raw_ostream &OS, DIDumpOptions DumpOpts, unsigned IndentLevel, + std::optional InitialLocation) const; void addInstruction(const Instruction &I) { Instructions.push_back(I); } @@ -524,7 +524,7 @@ private: /// Print \p Opcode's operand number \p OperandIdx which has value \p Operand. void printOperand(raw_ostream &OS, DIDumpOptions DumpOpts, const Instruction &Instr, unsigned OperandIdx, - uint64_t Operand) const; + uint64_t Operand, std::optional &Address) const; }; /// An entry in either debug_frame or eh_frame. This entry can be a CIE or an diff --git a/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp b/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp index 0c968703f80b..aff26824dda1 100644 --- a/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp +++ b/llvm/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp @@ -860,7 +860,8 @@ CFIProgram::getOperandTypes() { /// Print \p Opcode's operand number \p OperandIdx which has value \p Operand. void CFIProgram::printOperand(raw_ostream &OS, DIDumpOptions DumpOpts, const Instruction &Instr, unsigned OperandIdx, - uint64_t Operand) const { + uint64_t Operand, + std::optional &Address) const { assert(OperandIdx < MaxOperands); uint8_t Opcode = Instr.Opcode; OperandType Type = getOperandTypes()[Opcode][OperandIdx]; @@ -879,6 +880,7 @@ void CFIProgram::printOperand(raw_ostream &OS, DIDumpOptions DumpOpts, break; case OT_Address: OS << format(" %" PRIx64, Operand); + Address = Operand; break; case OT_Offset: // The offsets are all encoded in a unsigned form, but in practice @@ -890,7 +892,11 @@ void CFIProgram::printOperand(raw_ostream &OS, DIDumpOptions DumpOpts, if (CodeAlignmentFactor) OS << format(" %" PRId64, Operand * CodeAlignmentFactor); else - OS << format(" %" PRId64 "*code_alignment_factor" , Operand); + OS << format(" %" PRId64 "*code_alignment_factor", Operand); + if (Address && CodeAlignmentFactor) { + *Address += Operand * CodeAlignmentFactor; + OS << format(" to 0x%" PRIx64, *Address); + } break; case OT_SignedFactDataOffset: if (DataAlignmentFactor) @@ -920,13 +926,14 @@ void CFIProgram::printOperand(raw_ostream &OS, DIDumpOptions DumpOpts, } void CFIProgram::dump(raw_ostream &OS, DIDumpOptions DumpOpts, - unsigned IndentLevel) const { + unsigned IndentLevel, + std::optional Address) const { for (const auto &Instr : Instructions) { uint8_t Opcode = Instr.Opcode; OS.indent(2 * IndentLevel); OS << callFrameString(Opcode) << ":"; for (unsigned i = 0; i < Instr.Ops.size(); ++i) - printOperand(OS, DumpOpts, Instr, i, Instr.Ops[i]); + printOperand(OS, DumpOpts, Instr, i, Instr.Ops[i], Address); OS << '\n'; } } @@ -977,7 +984,7 @@ void CIE::dump(raw_ostream &OS, DIDumpOptions DumpOpts) const { OS << "\n"; } OS << "\n"; - CFIs.dump(OS, DumpOpts); + CFIs.dump(OS, DumpOpts, /*IndentLevel=*/1, /*InitialLocation=*/{}); OS << "\n"; if (Expected RowsOrErr = UnwindTable::create(this)) @@ -1005,7 +1012,7 @@ void FDE::dump(raw_ostream &OS, DIDumpOptions DumpOpts) const { OS << " Format: " << FormatString(IsDWARF64) << "\n"; if (LSDAAddress) OS << format(" LSDA Address: %016" PRIx64 "\n", *LSDAAddress); - CFIs.dump(OS, DumpOpts); + CFIs.dump(OS, DumpOpts, /*IndentLevel=*/1, InitialLocation); OS << "\n"; if (Expected RowsOrErr = UnwindTable::create(this)) diff --git a/llvm/test/DebugInfo/dwarfdump-debug-frame-simple.test b/llvm/test/DebugInfo/dwarfdump-debug-frame-simple.test index 6c049af43efe..2cd281c8d0af 100644 --- a/llvm/test/DebugInfo/dwarfdump-debug-frame-simple.test +++ b/llvm/test/DebugInfo/dwarfdump-debug-frame-simple.test @@ -12,15 +12,15 @@ ; FRAMES-NEXT: DW_CFA_nop: ; FRAMES: 00000014 00000010 00000000 FDE cie=00000000 pc=00000000...00000022 -; FRAMES: DW_CFA_advance_loc: 3 +; FRAMES: DW_CFA_advance_loc: 3 to 0x3 ; FRAMES-NEXT: DW_CFA_def_cfa_offset: +12 ; FRAMES-NEXT: DW_CFA_nop: ; FRAMES: 00000028 00000014 00000000 FDE cie=00000000 pc=00000030...00000080 -; FRAMES: DW_CFA_advance_loc: 1 +; FRAMES: DW_CFA_advance_loc: 1 to 0x31 ; FRAMES-NEXT: DW_CFA_def_cfa_offset: +8 ; FRAMES-NEXT: DW_CFA_offset: {{reg5|EBP}} -8 -; FRAMES-NEXT: DW_CFA_advance_loc: 2 +; FRAMES-NEXT: DW_CFA_advance_loc: 2 to 0x33 ; FRAMES-NEXT: DW_CFA_def_cfa_register: {{reg5|EBP}} ; FRAMES-NOT: CIE diff --git a/llvm/test/tools/llvm-readobj/ELF/unwind.test b/llvm/test/tools/llvm-readobj/ELF/unwind.test index 2deb1a587d24..2e51ec2a61a6 100644 --- a/llvm/test/tools/llvm-readobj/ELF/unwind.test +++ b/llvm/test/tools/llvm-readobj/ELF/unwind.test @@ -96,9 +96,9 @@ # CHECK: Program: # CHECK-NEXT: DW_CFA_def_cfa_offset: +16 -# CHECK-NEXT: DW_CFA_advance_loc: 6 +# CHECK-NEXT: DW_CFA_advance_loc: 6 to 0x4004a6 # CHECK-NEXT: DW_CFA_def_cfa_offset: +24 -# CHECK-NEXT: DW_CFA_advance_loc: 10 +# CHECK-NEXT: DW_CFA_advance_loc: 10 to 0x4004b0 # CHECK-NEXT: DW_CFA_def_cfa_expression: DW_OP_breg7 +8, DW_OP_breg16 +0, DW_OP_lit15, DW_OP_and, DW_OP_lit11, DW_OP_ge, DW_OP_lit3, DW_OP_shl, DW_OP_plus # CHECK-NEXT: DW_CFA_nop: # CHECK-NEXT: DW_CFA_nop: @@ -110,12 +110,12 @@ # CHECK-NEXT: address_range: 0x10 (end : 0x4005c6) # CHECK: Program: -# CHECK-NEXT: DW_CFA_advance_loc: 1 +# CHECK-NEXT: DW_CFA_advance_loc: 1 to 0x4005b7 # CHECK-NEXT: DW_CFA_def_cfa_offset: +16 # CHECK-NEXT: DW_CFA_offset: reg6 -16 -# CHECK-NEXT: DW_CFA_advance_loc: 3 +# CHECK-NEXT: DW_CFA_advance_loc: 3 to 0x4005ba # CHECK-NEXT: DW_CFA_def_cfa_register: reg6 -# CHECK-NEXT: DW_CFA_advance_loc: 11 +# CHECK-NEXT: DW_CFA_advance_loc: 11 to 0x4005c5 # CHECK-NEXT: DW_CFA_def_cfa: reg7 +8 # CHECK-NEXT: DW_CFA_nop: # CHECK-NEXT: DW_CFA_nop: @@ -126,15 +126,15 @@ # CHECK-NEXT: address_range: 0xc7f (end : 0x40124f) # CHECK: Program: -# CHECK-NEXT: DW_CFA_advance_loc: 5 +# CHECK-NEXT: DW_CFA_advance_loc: 5 to 0x4005d5 # CHECK-NEXT: DW_CFA_def_cfa: reg10 +0 -# CHECK-NEXT: DW_CFA_advance_loc: 9 +# CHECK-NEXT: DW_CFA_advance_loc: 9 to 0x4005de # CHECK-NEXT: DW_CFA_expression: reg6 DW_OP_breg6 +0 -# CHECK-NEXT: DW_CFA_advance_loc: 5 +# CHECK-NEXT: DW_CFA_advance_loc: 5 to 0x4005e3 # CHECK-NEXT: DW_CFA_def_cfa_expression: DW_OP_breg6 -8, DW_OP_deref -# CHECK-NEXT: DW_CFA_advance_loc2: 3174 +# CHECK-NEXT: DW_CFA_advance_loc2: 3174 to 0x401249 # CHECK-NEXT: DW_CFA_def_cfa: reg10 +0 -# CHECK-NEXT: DW_CFA_advance_loc: 5 +# CHECK-NEXT: DW_CFA_advance_loc: 5 to 0x40124e # CHECK-NEXT: DW_CFA_def_cfa: reg7 +8 # CHECK-NEXT: DW_CFA_nop: # CHECK-NEXT: DW_CFA_nop: @@ -146,21 +146,21 @@ # CHECK-NEXT: address_range: 0x66 (end : 0x4012b6) # CHECK: Program: -# CHECK-NEXT: DW_CFA_advance_loc: 1 +# CHECK-NEXT: DW_CFA_advance_loc: 1 to 0x401251 # CHECK-NEXT: DW_CFA_def_cfa_offset: +16 # CHECK-NEXT: DW_CFA_offset: reg6 -16 -# CHECK-NEXT: DW_CFA_advance_loc: 3 +# CHECK-NEXT: DW_CFA_advance_loc: 3 to 0x401254 # CHECK-NEXT: DW_CFA_def_cfa_register: reg6 -# CHECK-NEXT: DW_CFA_advance_loc: 2 +# CHECK-NEXT: DW_CFA_advance_loc: 2 to 0x401256 # CHECK-NEXT: DW_CFA_offset: reg15 -24 -# CHECK-NEXT: DW_CFA_advance_loc: 5 +# CHECK-NEXT: DW_CFA_advance_loc: 5 to 0x40125b # CHECK-NEXT: DW_CFA_offset: reg14 -32 -# CHECK-NEXT: DW_CFA_advance_loc: 7 +# CHECK-NEXT: DW_CFA_advance_loc: 7 to 0x401262 # CHECK-NEXT: DW_CFA_offset: reg13 -40 # CHECK-NEXT: DW_CFA_offset: reg12 -48 -# CHECK-NEXT: DW_CFA_advance_loc: 8 +# CHECK-NEXT: DW_CFA_advance_loc: 8 to 0x40126a # CHECK-NEXT: DW_CFA_offset: reg3 -56 -# CHECK-NEXT: DW_CFA_advance_loc1: 75 +# CHECK-NEXT: DW_CFA_advance_loc1: 75 to 0x4012b5 # CHECK-NEXT: DW_CFA_def_cfa: reg7 +8 # CHECK-NEXT: DW_CFA_nop: # CHECK-NEXT: DW_CFA_nop: diff --git a/llvm/tools/llvm-readobj/DwarfCFIEHPrinter.h b/llvm/tools/llvm-readobj/DwarfCFIEHPrinter.h index 687d97abd023..2e89463e68d5 100644 --- a/llvm/tools/llvm-readobj/DwarfCFIEHPrinter.h +++ b/llvm/tools/llvm-readobj/DwarfCFIEHPrinter.h @@ -196,6 +196,7 @@ void PrinterContext::printEHFrame(const Elf_Shdr *EHFrameShdr) const { reportError(std::move(E), ObjF.getFileName()); for (const dwarf::FrameEntry &Entry : EHFrame) { + std::optional InitialLocation; if (const dwarf::CIE *CIE = dyn_cast(&Entry)) { W.startLine() << format("[0x%" PRIx64 "] CIE length=%" PRIu64 "\n", Address + CIE->getOffset(), CIE->getLength()); @@ -214,8 +215,9 @@ void PrinterContext::printEHFrame(const Elf_Shdr *EHFrameShdr) const { Address + FDE->getLinkedCIE()->getOffset()); W.indent(); + InitialLocation = FDE->getInitialLocation(); W.startLine() << format("initial_location: 0x%" PRIx64 "\n", - FDE->getInitialLocation()); + *InitialLocation); W.startLine() << format( "address_range: 0x%" PRIx64 " (end : 0x%" PRIx64 ")\n", FDE->getAddressRange(), @@ -227,7 +229,8 @@ void PrinterContext::printEHFrame(const Elf_Shdr *EHFrameShdr) const { W.indent(); auto DumpOpts = DIDumpOptions(); DumpOpts.IsEH = true; - Entry.cfis().dump(W.getOStream(), DumpOpts, W.getIndentLevel()); + Entry.cfis().dump(W.getOStream(), DumpOpts, W.getIndentLevel(), + InitialLocation); W.unindent(); W.unindent(); W.getOStream() << "\n"; -- GitLab From 99512b1728bcf47dbf28f8a4cf5d296109fb0630 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Thu, 7 Mar 2024 16:37:41 -0800 Subject: [PATCH 540/929] [Object] Convert tests to opaque pointers (NFC) Link: https://discourse.llvm.org/t/enabling-opaque-pointers-by-default/61322 --- llvm/test/Object/Inputs/small.ll | 10 +++++----- llvm/test/Object/Inputs/trivial.ll | 10 +++++----- llvm/test/Object/X86/irsymtab-bad-alias.ll | 4 ++-- llvm/test/Object/X86/nm-ir.ll | 10 +++++----- llvm/test/Object/dllimport-globalref.ll | 2 +- llvm/test/Object/dllimport.ll | 2 +- llvm/test/Object/mangle-ir.ll | 4 ++-- llvm/test/Object/objc-swift-mixed-imageinfo-macho.ll | 8 ++++---- 8 files changed, 25 insertions(+), 25 deletions(-) diff --git a/llvm/test/Object/Inputs/small.ll b/llvm/test/Object/Inputs/small.ll index ef68a8c324a3..677f20ade4c5 100644 --- a/llvm/test/Object/Inputs/small.ll +++ b/llvm/test/Object/Inputs/small.ll @@ -4,15 +4,15 @@ target triple = "i386-pc-windows" define i32 @main() nounwind { entry: - %call = tail call i32 @puts(i8* getelementptr inbounds ([13 x i8], [13 x i8]* @.str, i32 0, i32 0)) nounwind - tail call void bitcast (void (...)* @SomeOtherFunction to void ()*)() nounwind + %call = tail call i32 @puts(ptr @.str) nounwind + tail call void @SomeOtherFunction() nounwind ret i32 0 } -declare i32 @puts(i8* nocapture) nounwind +declare i32 @puts(ptr nocapture) nounwind declare void @SomeOtherFunction(...) @var = global i32 0 -@llvm.used = appending global [1 x i8*] [i8* bitcast (i32* @var to i8*)], section "llvm.metadata" -@llvm.global_ctors = appending global [1 x { i32, void ()*, i8* }] [{ i32, void ()*, i8* } { i32 65535, void ()* null, i8* null }] +@llvm.used = appending global [1 x ptr] [ptr @var], section "llvm.metadata" +@llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 65535, ptr null, ptr null }] diff --git a/llvm/test/Object/Inputs/trivial.ll b/llvm/test/Object/Inputs/trivial.ll index 82eabc6389fb..1a6a76298b23 100644 --- a/llvm/test/Object/Inputs/trivial.ll +++ b/llvm/test/Object/Inputs/trivial.ll @@ -5,15 +5,15 @@ define i32 @main() nounwind { entry: - %call = tail call i32 @puts(i8* getelementptr inbounds ([13 x i8], [13 x i8]* @.str, i32 0, i32 0)) nounwind - tail call void bitcast (void (...)* @SomeOtherFunction to void ()*)() nounwind + %call = tail call i32 @puts(ptr @.str) nounwind + tail call void @SomeOtherFunction() nounwind ret i32 0 } -declare i32 @puts(i8* nocapture) nounwind +declare i32 @puts(ptr nocapture) nounwind declare void @SomeOtherFunction(...) @var = global i32 0 -@llvm.used = appending global [1 x i8*] [i8* bitcast (i32* @var to i8*)], section "llvm.metadata" -@llvm.global_ctors = appending global [1 x { i32, void ()*, i8* }] [{ i32, void ()*, i8* } { i32 65535, void ()* null, i8* null }] +@llvm.used = appending global [1 x ptr] [ptr @var], section "llvm.metadata" +@llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 65535, ptr null, ptr null }] diff --git a/llvm/test/Object/X86/irsymtab-bad-alias.ll b/llvm/test/Object/X86/irsymtab-bad-alias.ll index c54436d59219..7f204d1dd157 100644 --- a/llvm/test/Object/X86/irsymtab-bad-alias.ll +++ b/llvm/test/Object/X86/irsymtab-bad-alias.ll @@ -11,5 +11,5 @@ target triple = "x86_64-unknown-linux-gnu" @g1 = global i32 1 @g2 = global i32 2 -@a = alias i32, inttoptr(i32 sub (i32 ptrtoint (i32* @g1 to i32), - i32 ptrtoint (i32* @g2 to i32)) to i32*) +@a = alias i32, inttoptr(i32 sub (i32 ptrtoint (ptr @g1 to i32), + i32 ptrtoint (ptr @g2 to i32)) to ptr) diff --git a/llvm/test/Object/X86/nm-ir.ll b/llvm/test/Object/X86/nm-ir.ll index e57c6d9a11c6..0324efb2948d 100644 --- a/llvm/test/Object/X86/nm-ir.ll +++ b/llvm/test/Object/X86/nm-ir.ll @@ -29,15 +29,15 @@ module asm ".long undef_asm_sym" @g3 = common global i32 0 @g4 = private global i32 42 -@a1 = alias i32, i32* @g1 -@a2 = internal alias i32, i32* @g1 +@a1 = alias i32, ptr @g1 +@a2 = internal alias i32, ptr @g1 -define void ()* @f1() { +define ptr @f1() { call void @f5() - ret void ()* null + ret ptr null } -@ifunc_f1 = ifunc void (), void ()* ()* @f1 +@ifunc_f1 = ifunc void (), ptr @f1 define internal void @f2() { ret void diff --git a/llvm/test/Object/dllimport-globalref.ll b/llvm/test/Object/dllimport-globalref.ll index dd518bc2266c..0a95be20a9d1 100644 --- a/llvm/test/Object/dllimport-globalref.ll +++ b/llvm/test/Object/dllimport-globalref.ll @@ -11,4 +11,4 @@ target triple = "x86_64-pc-windows-msvc" ; CHECK: U f declare dllimport void @f() -@fp = constant void ()* @f +@fp = constant ptr @f diff --git a/llvm/test/Object/dllimport.ll b/llvm/test/Object/dllimport.ll index afdb4562cc9f..52f583fa2487 100644 --- a/llvm/test/Object/dllimport.ll +++ b/llvm/test/Object/dllimport.ll @@ -12,6 +12,6 @@ declare dllimport void @f() define void @g() { call void @f() - store i32 42, i32* @v + store i32 42, ptr @v ret void } diff --git a/llvm/test/Object/mangle-ir.ll b/llvm/test/Object/mangle-ir.ll index bd7c3d93b7c9..76442f070385 100644 --- a/llvm/test/Object/mangle-ir.ll +++ b/llvm/test/Object/mangle-ir.ll @@ -7,8 +7,8 @@ target datalayout = "m:o" ; CHECK-NOT: memcpy define void @f() { - tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* null, i8* null, i64 0, i1 false) + tail call void @llvm.memcpy.p0.p0.i64(ptr null, ptr null, i64 0, i1 false) ret void } -declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1) +declare void @llvm.memcpy.p0.p0.i64(ptr nocapture, ptr nocapture readonly, i64, i1) diff --git a/llvm/test/Object/objc-swift-mixed-imageinfo-macho.ll b/llvm/test/Object/objc-swift-mixed-imageinfo-macho.ll index d2518f46cc27..c506c9687ec2 100644 --- a/llvm/test/Object/objc-swift-mixed-imageinfo-macho.ll +++ b/llvm/test/Object/objc-swift-mixed-imageinfo-macho.ll @@ -5,11 +5,11 @@ target triple = "x86_64-apple-macosx10.15.0" -@llvm.used = appending global [1 x i8*] [i8* bitcast (i16* @__swift_reflection_version to i8*)], section "llvm.metadata", align 8 +@llvm.used = appending global [1 x ptr] [ptr @__swift_reflection_version], section "llvm.metadata", align 8 @__swift_reflection_version = linkonce_odr hidden constant i16 3 -define i32 @main(i32 %0, i8** %1) #0 { - %3 = bitcast i8** %1 to i8* +define i32 @main(i32 %0, ptr %1) #0 { + %3 = bitcast ptr %1 to ptr ret i32 0 } @@ -25,7 +25,7 @@ attributes #0 = { "frame-pointer"="all" "target-cpu"="penryn" "target-features"= !1 = !{!"-lswiftSwiftOnoneSupport"} !2 = !{!"-lswiftCore"} !3 = !{!"-lobjc"} -!4 = !{[1 x i8*]* @llvm.used, null, null, i1 false, i1 true} +!4 = !{ptr @llvm.used, null, null, i1 false, i1 true} !5 = !{i32 2, !"SDK Version", [2 x i32] [i32 10, i32 15]} !6 = !{i32 1, !"Objective-C Version", i32 2} !7 = !{i32 1, !"Objective-C Image Info Version", i32 0} -- GitLab From 30f098ef9dfb121bccaef6975b13788b6f940e47 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Thu, 7 Mar 2024 16:39:57 -0800 Subject: [PATCH 541/929] [Instrumentation] Convert tests to opaque pointers (NFC) Link: https://discourse.llvm.org/t/enabling-opaque-pointers-by-default/61322 --- .../Instrumentation/AddressSanitizer/aarch64be.ll | 4 ++-- .../AddressSanitizer/program-addrspace.ll | 2 +- .../InstrProfiling/before-value-profile-lowering.ll | 12 ++++++------ .../InstrProfiling/timestamp-coverage.ll | 8 ++++---- .../test/Instrumentation/InstrProfiling/timestamp.ll | 8 ++++---- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/llvm/test/Instrumentation/AddressSanitizer/aarch64be.ll b/llvm/test/Instrumentation/AddressSanitizer/aarch64be.ll index eb522a0f3f31..aeb1b0e8ebe7 100644 --- a/llvm/test/Instrumentation/AddressSanitizer/aarch64be.ll +++ b/llvm/test/Instrumentation/AddressSanitizer/aarch64be.ll @@ -2,9 +2,9 @@ ; RUN: opt < %s -passes=asan -S -mtriple=aarch64_be-linux-gnu | FileCheck --check-prefix=CHECK-AARCH64BE %s ; REQUIRES: aarch64-registered-target -define i32 @read_4_bytes(i32* %a) sanitize_address { +define i32 @read_4_bytes(ptr %a) sanitize_address { entry: - %tmp1 = load i32, i32* %a, align 4 + %tmp1 = load i32, ptr %a, align 4 ret i32 %tmp1 } diff --git a/llvm/test/Instrumentation/AddressSanitizer/program-addrspace.ll b/llvm/test/Instrumentation/AddressSanitizer/program-addrspace.ll index adfe21135e7a..1d5bfb09ead9 100644 --- a/llvm/test/Instrumentation/AddressSanitizer/program-addrspace.ll +++ b/llvm/test/Instrumentation/AddressSanitizer/program-addrspace.ll @@ -16,7 +16,7 @@ target datalayout = "P1" define i1 @b(i64 %c) addrspace(1) { %cast = inttoptr i64 %c to ptr addrspace(42) - %cmp = icmp ugt ptr addrspace(42) %cast, getelementptr inbounds ([1 x i32], ptr addrspace(42) @a, i64 0, i64 0) + %cmp = icmp ugt ptr addrspace(42) %cast, @a ret i1 %cmp } diff --git a/llvm/test/Instrumentation/InstrProfiling/before-value-profile-lowering.ll b/llvm/test/Instrumentation/InstrProfiling/before-value-profile-lowering.ll index 5dfec433f4ec..870e74ccfdac 100644 --- a/llvm/test/Instrumentation/InstrProfiling/before-value-profile-lowering.ll +++ b/llvm/test/Instrumentation/InstrProfiling/before-value-profile-lowering.ll @@ -7,17 +7,17 @@ target triple = "x86_64-unknown-linux-gnu" -declare void @llvm.instrprof.increment.step(i8*, i64, i32, i32, i64) +declare void @llvm.instrprof.increment.step(ptr, i64, i32, i32, i64) -declare void @llvm.instrprof.value.profile(i8*, i64, i64, i32, i32) +declare void @llvm.instrprof.value.profile(ptr, i64, i64, i32, i32) ; CHECK: @__profd_foo = private global @__profn_foo = private constant [3 x i8] c"foo" -define i32 @foo(i32 ()* ) { - %2 = ptrtoint i32 ()* %0 to i64 - call void @llvm.instrprof.value.profile(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 0, i64 %2, i32 0, i32 0) - call void @llvm.instrprof.increment.step(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 0, i32 1, i32 0, i64 0) +define i32 @foo(ptr ) { + %2 = ptrtoint ptr %0 to i64 + call void @llvm.instrprof.value.profile(ptr @__profn_foo, i64 0, i64 %2, i32 0, i32 0) + call void @llvm.instrprof.increment.step(ptr @__profn_foo, i64 0, i32 1, i32 0, i64 0) %3 = tail call i32 %0() ret i32 %3 } diff --git a/llvm/test/Instrumentation/InstrProfiling/timestamp-coverage.ll b/llvm/test/Instrumentation/InstrProfiling/timestamp-coverage.ll index ab9b664a2cff..d40cc2ac02c1 100644 --- a/llvm/test/Instrumentation/InstrProfiling/timestamp-coverage.ll +++ b/llvm/test/Instrumentation/InstrProfiling/timestamp-coverage.ll @@ -6,11 +6,11 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK: @__profc_foo = private global [9 x i8] c"\FF\FF\FF\FF\FF\FF\FF\FF\FF", section "__llvm_prf_cnts", comdat, align 8 define void @_Z3foov() { - call void @llvm.instrprof.timestamp(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 12345678, i32 9, i32 0) + call void @llvm.instrprof.timestamp(ptr @__profn_foo, i64 12345678, i32 9, i32 0) ; CHECK: call void @__llvm_profile_set_timestamp(ptr @__profc_foo) - call void @llvm.instrprof.cover(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 12345678, i32 9, i32 8) + call void @llvm.instrprof.cover(ptr @__profn_foo, i64 12345678, i32 9, i32 8) ret void } -declare void @llvm.instrprof.timestamp(i8*, i64, i32, i32) -declare void @llvm.instrprof.cover(i8*, i64, i32, i32) +declare void @llvm.instrprof.timestamp(ptr, i64, i32, i32) +declare void @llvm.instrprof.cover(ptr, i64, i32, i32) diff --git a/llvm/test/Instrumentation/InstrProfiling/timestamp.ll b/llvm/test/Instrumentation/InstrProfiling/timestamp.ll index aa2393695d6b..c08ba4485fc5 100644 --- a/llvm/test/Instrumentation/InstrProfiling/timestamp.ll +++ b/llvm/test/Instrumentation/InstrProfiling/timestamp.ll @@ -6,11 +6,11 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK: @__profc_foo = private global [2 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat, align 8 define void @_Z3foov() { - call void @llvm.instrprof.timestamp(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 12345678, i32 2, i32 0) + call void @llvm.instrprof.timestamp(ptr @__profn_foo, i64 12345678, i32 2, i32 0) ; CHECK: call void @__llvm_profile_set_timestamp(ptr @__profc_foo) - call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 12345678, i32 2, i32 1) + call void @llvm.instrprof.increment(ptr @__profn_foo, i64 12345678, i32 2, i32 1) ret void } -declare void @llvm.instrprof.timestamp(i8*, i64, i32, i32) -declare void @llvm.instrprof.increment(i8*, i64, i32, i32) +declare void @llvm.instrprof.timestamp(ptr, i64, i32, i32) +declare void @llvm.instrprof.increment(ptr, i64, i32, i32) -- GitLab From 9d3bf9b639eafeded82c6be295031262735d1dac Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Thu, 7 Mar 2024 16:41:50 -0800 Subject: [PATCH 542/929] [NFC] [hwasan] consistent naming for cl::opt --- .../Instrumentation/HWAddressSanitizer.cpp | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp index 289183ecf0f2..6bae679e11be 100644 --- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -187,15 +187,15 @@ static cl::opt cl::desc("Use selective instrumentation"), cl::Hidden, cl::init(false)); -static cl::opt HotPercentileCutoff( +static cl::opt ClHotPercentileCutoff( "hwasan-percentile-cutoff-hot", cl::init(0), cl::desc("Alternative hot percentile cuttoff." "By default `-profile-summary-cutoff-hot` is used.")); static cl::opt - RandomSkipRate("hwasan-random-skip-rate", cl::init(0), - cl::desc("Probability value in the range [0.0, 1.0] " - "to skip instrumentation of a function.")); + ClRandomSkipRate("hwasan-random-skip-rate", cl::init(0), + cl::desc("Probability value in the range [0.0, 1.0] " + "to skip instrumentation of a function.")); STATISTIC(NumTotalFuncs, "Number of total funcs"); STATISTIC(NumInstrumentedFuncs, "Number of instrumented funcs"); @@ -301,7 +301,7 @@ public: ? ClEnableKhwasan : CompileKernel; this->Rng = - RandomSkipRate.getNumOccurrences() ? M.createRNG("hwasan") : nullptr; + ClRandomSkipRate.getNumOccurrences() ? M.createRNG("hwasan") : nullptr; initializeModule(); } @@ -1537,8 +1537,8 @@ void HWAddressSanitizer::sanitizeFunction(Function &F, NumTotalFuncs++; if (CSelectiveInstrumentation) { - if (RandomSkipRate.getNumOccurrences()) { - std::bernoulli_distribution D(RandomSkipRate); + if (ClRandomSkipRate.getNumOccurrences()) { + std::bernoulli_distribution D(ClRandomSkipRate); if (D(*Rng)) return; } else { @@ -1547,10 +1547,10 @@ void HWAddressSanitizer::sanitizeFunction(Function &F, MAMProxy.getCachedResult(*F.getParent()); if (PSI && PSI->hasProfileSummary()) { auto &BFI = FAM.getResult(F); - if ((HotPercentileCutoff.getNumOccurrences() && - HotPercentileCutoff >= 0) + if ((ClHotPercentileCutoff.getNumOccurrences() && + ClHotPercentileCutoff >= 0) ? PSI->isFunctionHotInCallGraphNthPercentile( - HotPercentileCutoff, &F, BFI) + ClHotPercentileCutoff, &F, BFI) : PSI->isFunctionHotInCallGraph(&F, BFI)) return; } else { -- GitLab From fb582b6ace781ff6991775d6dcd4df98aa16698f Mon Sep 17 00:00:00 2001 From: Boian Petkantchin Date: Thu, 7 Mar 2024 17:05:44 -0800 Subject: [PATCH 543/929] [mlir] Implement Mesh's ShardingInterface for Linalg ops (#82284) Allows linalg structured operations to be handled during spmdization and sharding propagation. There is only support for projected permutation indexing maps. --- .../Dialect/Linalg/Transforms/AllInterfaces.h | 26 ++ .../Transforms/MeshShardingInterfaceImpl.h | 20 + mlir/include/mlir/Dialect/Mesh/IR/MeshBase.td | 6 + mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td | 4 + .../Mesh/Interfaces/ShardingInterfaceImpl.h | 18 + .../mlir/Dialect/Mesh/Transforms/Transforms.h | 6 + mlir/include/mlir/IR/Dialect.h | 8 + mlir/include/mlir/InitAllDialects.h | 10 +- mlir/lib/Dialect/Linalg/IR/CMakeLists.txt | 1 + mlir/lib/Dialect/Linalg/IR/LinalgDialect.cpp | 7 + .../Linalg/Transforms/AllInterfaces.cpp | 24 ++ .../Dialect/Linalg/Transforms/CMakeLists.txt | 5 + .../Transforms/MeshShardingInterfaceImpl.cpp | 352 ++++++++++++++++++ .../Linalg/Transforms/TilingInterfaceImpl.cpp | 8 - mlir/lib/Dialect/Mesh/IR/MeshOps.cpp | 7 + .../Mesh/Interfaces/ShardingInterface.cpp | 89 ++++- .../Dialect/Mesh/Transforms/Transforms.cpp | 13 + .../test/Dialect/Linalg/mesh-spmdization.mlir | 165 ++++++++ .../llvm-project-overlay/mlir/BUILD.bazel | 4 + 19 files changed, 754 insertions(+), 19 deletions(-) create mode 100644 mlir/include/mlir/Dialect/Linalg/Transforms/AllInterfaces.h create mode 100644 mlir/include/mlir/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.h create mode 100644 mlir/lib/Dialect/Linalg/Transforms/AllInterfaces.cpp create mode 100644 mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp create mode 100644 mlir/test/Dialect/Linalg/mesh-spmdization.mlir diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/AllInterfaces.h b/mlir/include/mlir/Dialect/Linalg/Transforms/AllInterfaces.h new file mode 100644 index 000000000000..a69751e072b7 --- /dev/null +++ b/mlir/include/mlir/Dialect/Linalg/Transforms/AllInterfaces.h @@ -0,0 +1,26 @@ +//===- AllInterfaces.h - ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines a common entry point for registering all external +// interface implementations to the linalg dialect. +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_LINALG_TRANSFORMS_ALLINTERFACES_H +#define MLIR_DIALECT_LINALG_TRANSFORMS_ALLINTERFACES_H + +namespace mlir { +class DialectRegistry; + +namespace linalg { +void registerAllDialectInterfaceImplementations(DialectRegistry ®istry); +} // namespace linalg + +} // namespace mlir + +#endif // MLIR_DIALECT_LINALG_TRANSFORMS_ALLINTERFACES_H diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.h b/mlir/include/mlir/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.h new file mode 100644 index 000000000000..c57501ea86b7 --- /dev/null +++ b/mlir/include/mlir/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.h @@ -0,0 +1,20 @@ +//===- MeshShardingInterfaceImpl.h ----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_LINALG_MESHSHARDINGINTERFACEIMPL_H +#define MLIR_DIALECT_LINALG_MESHSHARDINGINTERFACEIMPL_H + +namespace mlir { +class DialectRegistry; + +namespace linalg { +void registerMeshShardingInterfaceExternalModels(DialectRegistry ®istry); +} // namespace linalg +} // namespace mlir + +#endif // MLIR_DIALECT_LINALG_MESHSHARDINGINTERFACEIMPL_H diff --git a/mlir/include/mlir/Dialect/Mesh/IR/MeshBase.td b/mlir/include/mlir/Dialect/Mesh/IR/MeshBase.td index fc2acc70381e..9d9b5892e1a5 100644 --- a/mlir/include/mlir/Dialect/Mesh/IR/MeshBase.td +++ b/mlir/include/mlir/Dialect/Mesh/IR/MeshBase.td @@ -46,6 +46,12 @@ def Mesh_ReductionKind : I32EnumAttr<"ReductionKind", I32EnumAttrCase<"Sum", 1, "sum">, I32EnumAttrCase<"Max", 2, "max">, I32EnumAttrCase<"Min", 3, "min">, + I32EnumAttrCase<"Product", 4, "product">, + // Arithmetic mean. + I32EnumAttrCase<"Average", 5, "average">, + I32EnumAttrCase<"BitwiseAnd", 6, "bitwise_and">, + I32EnumAttrCase<"BitwiseOr", 7, "bitwise_or">, + I32EnumAttrCase<"BitwiseXor", 8, "bitwise_xor">, I32EnumAttrCase<"Generic", 100, "generic"> ]> { let genSpecializedAttr = 0; diff --git a/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td b/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td index b9cd15e20626..8e1e47546358 100644 --- a/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td +++ b/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td @@ -353,6 +353,10 @@ def Mesh_AllReduceOp : Mesh_CollectiveCommunicationOpBase<"all_reduce", [ attr-dict `:` type($input) `->` type($result) }]; let hasCanonicalizer = 1; + let builders = [ + OpBuilder<(ins "Value":$input, "StringRef":$mesh, + "ArrayRef":$meshAxes, "ReductionKind":$reduction)> + ]; } def Mesh_AllSliceOp : Mesh_CollectiveCommunicationOpBase<"all_slice", [ diff --git a/mlir/include/mlir/Dialect/Mesh/Interfaces/ShardingInterfaceImpl.h b/mlir/include/mlir/Dialect/Mesh/Interfaces/ShardingInterfaceImpl.h index ffc9b6fb18be..ab4df2ab028d 100644 --- a/mlir/include/mlir/Dialect/Mesh/Interfaces/ShardingInterfaceImpl.h +++ b/mlir/include/mlir/Dialect/Mesh/Interfaces/ShardingInterfaceImpl.h @@ -22,6 +22,24 @@ class SymbolTableCollection; namespace mesh { +// Retrieve the mesh axes corresponding to each operation loop iterator based +// on the provided shardings for the op's operands and results. +// Assumes that the indexingMaps are projected permutations. +ShardingArray getMeshAxisAssignmentForLoopIterators( + ArrayRef operandShardings, + ArrayRef resultShardings, + ArrayRef loopIteratorTypes, + ArrayRef indexingMaps); + +bool isAtLeastOneReductionIteratorSharded( + ArrayRef loopIteratorTypes, + ArrayRef> meshAxisAssignmentForLoopIterators); + +// Get the set of mesh axes that correspond to reduction loop iterators. +SmallVector getReductionMeshAxes( + ArrayRef loopIteratorTypes, + ArrayRef> meshAxisAssignmentForLoopIterators); + // Inserts a clone of the operation that has all ranked tensor // arguments/results sharded. void spmdizeTriviallyShardableOperation( diff --git a/mlir/include/mlir/Dialect/Mesh/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Mesh/Transforms/Transforms.h index aeab28961a4e..be82e2af399d 100644 --- a/mlir/include/mlir/Dialect/Mesh/Transforms/Transforms.h +++ b/mlir/include/mlir/Dialect/Mesh/Transforms/Transforms.h @@ -13,6 +13,7 @@ #include "mlir/IR/BuiltinTypes.h" #include "mlir/IR/Value.h" #include "mlir/Support/LLVM.h" +#include "llvm/ADT/ArrayRef.h" namespace mlir { class RewritePatternSet; @@ -37,6 +38,11 @@ TypedValue createCollectiveProcessGroupSize(MeshOp mesh, ArrayRef axes, ImplicitLocOpBuilder &builder); +// Get process linear index along the given mesh axes. +TypedValue createProcessLinearIndex(StringRef mesh, + ArrayRef meshAxes, + ImplicitLocOpBuilder &builder); + } // namespace mesh } // namespace mlir diff --git a/mlir/include/mlir/IR/Dialect.h b/mlir/include/mlir/IR/Dialect.h index 50f6f6de5c28..6c8a170a03c7 100644 --- a/mlir/include/mlir/IR/Dialect.h +++ b/mlir/include/mlir/IR/Dialect.h @@ -216,6 +216,14 @@ public: {TypeID::get(), InterfaceT::getInterfaceID()}); } + // Declare the same interface for multiple types. + // Example: + // declarePromisedInterfaces() + template + void declarePromisedInterfaces() { + (declarePromisedInterface(), ...); + } + /// Checks if the given interface, which is attempting to be used, is a /// promised interface of this dialect that has yet to be implemented. If so, /// emits a fatal error. `interfaceName` is an optional string that contains a diff --git a/mlir/include/mlir/InitAllDialects.h b/mlir/include/mlir/InitAllDialects.h index 838bd03622a6..21775e11e071 100644 --- a/mlir/include/mlir/InitAllDialects.h +++ b/mlir/include/mlir/InitAllDialects.h @@ -43,10 +43,7 @@ #include "mlir/Dialect/LLVMIR/NVVMDialect.h" #include "mlir/Dialect/LLVMIR/ROCDLDialect.h" #include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/Linalg/IR/ValueBoundsOpInterfaceImpl.h" -#include "mlir/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.h" -#include "mlir/Dialect/Linalg/Transforms/SubsetInsertionOpInterfaceImpl.h" -#include "mlir/Dialect/Linalg/Transforms/TilingInterfaceImpl.h" +#include "mlir/Dialect/Linalg/Transforms/AllInterfaces.h" #include "mlir/Dialect/MLProgram/IR/MLProgram.h" #include "mlir/Dialect/MLProgram/Transforms/BufferizableOpInterfaceImpl.h" #include "mlir/Dialect/MPI/IR/MPI.h" @@ -157,10 +154,7 @@ inline void registerAllDialects(DialectRegistry ®istry) { cf::registerBufferizableOpInterfaceExternalModels(registry); cf::registerBufferDeallocationOpInterfaceExternalModels(registry); gpu::registerBufferDeallocationOpInterfaceExternalModels(registry); - linalg::registerBufferizableOpInterfaceExternalModels(registry); - linalg::registerSubsetOpInterfaceExternalModels(registry); - linalg::registerTilingInterfaceExternalModels(registry); - linalg::registerValueBoundsOpInterfaceExternalModels(registry); + linalg::registerAllDialectInterfaceImplementations(registry); memref::registerAllocationOpInterfaceExternalModels(registry); memref::registerRuntimeVerifiableOpInterfaceExternalModels(registry); memref::registerValueBoundsOpInterfaceExternalModels(registry); diff --git a/mlir/lib/Dialect/Linalg/IR/CMakeLists.txt b/mlir/lib/Dialect/Linalg/IR/CMakeLists.txt index f0ac1899bb02..c187563b8f0c 100644 --- a/mlir/lib/Dialect/Linalg/IR/CMakeLists.txt +++ b/mlir/lib/Dialect/Linalg/IR/CMakeLists.txt @@ -25,6 +25,7 @@ add_mlir_dialect_library(MLIRLinalgDialect MLIRInferTypeOpInterface MLIRIR MLIRParser + MLIRShardingInterface MLIRSideEffectInterfaces MLIRSparseTensorDialect MLIRSCFDialect diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgDialect.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgDialect.cpp index 5069d43e7db9..027058d4de63 100644 --- a/mlir/lib/Dialect/Linalg/IR/LinalgDialect.cpp +++ b/mlir/lib/Dialect/Linalg/IR/LinalgDialect.cpp @@ -16,6 +16,7 @@ #include "mlir/Dialect/Linalg/IR/Linalg.h" #include "mlir/Dialect/Math/IR/Math.h" #include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/Mesh/Interfaces/ShardingInterface.h" #include "mlir/Dialect/Tensor/IR/Tensor.h" #include "mlir/IR/BuiltinTypes.h" #include "mlir/IR/Dialect.h" @@ -118,6 +119,12 @@ void mlir::linalg::LinalgDialect::initialize() { >(namedStructuredOpRegionBuilders); addInterfaces(); + + declarePromisedInterface(); + declarePromisedInterfaces(); } LogicalResult LinalgDialect::verifyOperationAttribute(Operation *op, diff --git a/mlir/lib/Dialect/Linalg/Transforms/AllInterfaces.cpp b/mlir/lib/Dialect/Linalg/Transforms/AllInterfaces.cpp new file mode 100644 index 000000000000..281d9f220448 --- /dev/null +++ b/mlir/lib/Dialect/Linalg/Transforms/AllInterfaces.cpp @@ -0,0 +1,24 @@ +//===- AllInterfaces.cpp - ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "mlir/Dialect/Linalg/Transforms/AllInterfaces.h" + +#include "mlir/Dialect/Linalg/IR/ValueBoundsOpInterfaceImpl.h" +#include "mlir/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.h" +#include "mlir/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.h" +#include "mlir/Dialect/Linalg/Transforms/SubsetInsertionOpInterfaceImpl.h" +#include "mlir/Dialect/Linalg/Transforms/TilingInterfaceImpl.h" + +void mlir::linalg::registerAllDialectInterfaceImplementations( + DialectRegistry ®istry) { + registerBufferizableOpInterfaceExternalModels(registry); + registerMeshShardingInterfaceExternalModels(registry); + registerSubsetOpInterfaceExternalModels(registry); + registerTilingInterfaceExternalModels(registry); + registerValueBoundsOpInterfaceExternalModels(registry); +} diff --git a/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt index 4f47e3b87184..513c54de5d7b 100644 --- a/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt +++ b/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt @@ -1,4 +1,5 @@ add_mlir_dialect_library(MLIRLinalgTransforms + AllInterfaces.cpp BubbleUpExtractSlice.cpp BufferizableOpInterfaceImpl.cpp Bufferize.cpp @@ -21,6 +22,7 @@ add_mlir_dialect_library(MLIRLinalgTransforms InlineScalarOperands.cpp Interchange.cpp Loops.cpp + MeshShardingInterfaceImpl.cpp NamedOpConversions.cpp Padding.cpp Promotion.cpp @@ -61,12 +63,15 @@ add_mlir_dialect_library(MLIRLinalgTransforms MLIRIR MLIRMemRefDialect MLIRMemRefTransforms + MLIRMeshDialect + MLIRMeshTransforms MLIRLinalgDialect MLIRLinalgUtils MLIRSCFDialect MLIRSCFTransforms MLIRSCFUtils MLIRPass + MLIRShardingInterface MLIRSubsetOpInterface MLIRSparseTensorDialect MLIRTensorDialect diff --git a/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp new file mode 100644 index 000000000000..7ac45dc3eb3e --- /dev/null +++ b/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp @@ -0,0 +1,352 @@ +//===- MeshShardingInterfaceImpl.cpp --------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "mlir/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.h" + +#include "mlir/Analysis/SliceAnalysis.h" +#include "mlir/Dialect/Affine/IR/AffineOps.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/Linalg/IR/LinalgInterfaces.h" +#include "mlir/Dialect/Mesh/IR/MeshOps.h" +#include "mlir/Dialect/Mesh/Interfaces/ShardingInterface.h" +#include "mlir/Dialect/Mesh/Interfaces/ShardingInterfaceImpl.h" +#include "mlir/Dialect/Mesh/Transforms/Transforms.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/Dialect/Tensor/IR/Tensor.h" +#include "mlir/Dialect/Utils/StructuredOpsUtils.h" +#include "mlir/IR/AffineExpr.h" +#include "mlir/IR/DialectRegistry.h" +#include "mlir/IR/IRMapping.h" +#include "mlir/IR/ImplicitLocOpBuilder.h" +#include "mlir/IR/MLIRContext.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/Operation.h" +#include "mlir/IR/SymbolTable.h" +#include "mlir/IR/Value.h" +#include "mlir/Interfaces/TilingInterface.h" +#include "mlir/Support/LogicalResult.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/TypeSwitch.h" +#include +#include +#include + +namespace mlir::linalg { + +using MeshAxis = mesh::MeshAxis; +using ReductionKind = mesh::ReductionKind; +using MeshShardingAttr = mesh::MeshShardingAttr; +using ShardingArray = mesh::ShardingArray; +using MeshOp = mesh::MeshOp; + +// Returns the corresponding mesh reduction kind for the given arith op. +static ReductionKind getReductionKind(Operation *op) { + return llvm::TypeSwitch(op) + // Floating-point operations. + .Case([](arith::AddFOp op) { return ReductionKind::Sum; }) + .Case([](arith::MulFOp op) { return ReductionKind::Product; }) + // TODO: handle maxnumf and minnumf. + .Case([](arith::MaximumFOp op) { return ReductionKind::Max; }) + .Case([](arith::MinimumFOp op) { return ReductionKind::Min; }) + // Integer operations. + .Case([](arith::AddIOp op) { return ReductionKind::Sum; }) + .Case([](arith::OrIOp op) { return ReductionKind::BitwiseOr; }) + .Case([](arith::XOrIOp op) { return ReductionKind::BitwiseXor; }) + .Case([](arith::AndIOp op) { return ReductionKind::Sum; }) + // TODO: handle signless, signed and unsigned types properly. + // It is assumed that the element type of the collective operands and + // result drive the meaning of the reduction kind, whether it is signed + // or unsigned. + // The reduction op inside the linalg op may have different result type + // from the element type of the linalg op's result. + // Also signed and unsigned Arith dialect ops may accept signed, unsigned + // or signless operands. + // Maybe expand the reduction kinds. + .Case([](arith::MaxUIOp op) { return ReductionKind::Max; }) + .Case([](arith::MinUIOp op) { return ReductionKind::Min; }) + .Case([](arith::MaxSIOp op) { return ReductionKind::Max; }) + .Case([](arith::MinSIOp op) { return ReductionKind::Min; }) + .Case([](arith::MulIOp op) { return ReductionKind::Product; }) + .Default([](Operation *op) { return ReductionKind::Generic; }); +} + +static std::optional getCombinerOp(LinalgOp op) { + SmallVector combinerOps; + Value reducedValue = matchReduction(op.getRegionOutputArgs(), 0, combinerOps); + if (!reducedValue || combinerOps.size() != 1) { + return std::nullopt; + } + + return combinerOps[0]; +} + +static ReductionKind getReductionKindOfLinalgOp(LinalgOp op) { + std::optional reductionOp = getCombinerOp(op); + if (!reductionOp) { + return ReductionKind::Generic; + } + Type resultElementType = + llvm::cast(op->getResult(0).getType()).getElementType(); + // TODO: handle case when result type of the reduction op does not match the + // element type of the result tensor. + // Would it makes sense at all? + assert(resultElementType == reductionOp.value()->getResult(0).getType()); + return getReductionKind(reductionOp.value()); +} + +static MeshOp getMesh(Operation *op, + ArrayRef operandShardings, + ArrayRef resultShardings, + SymbolTableCollection &symbolTable) { + for (MeshShardingAttr sharding : operandShardings) { + if (sharding) { + return mesh::getMesh(op, sharding.getMesh(), symbolTable); + } + } + + for (MeshShardingAttr sharding : resultShardings) { + if (sharding) { + return mesh::getMesh(op, sharding.getMesh(), symbolTable); + } + } + + assert(false); +} + +// Choose the operand based on the current process index along the reduction +// mesh axes. +// We need to use the initial value only once to avoid including it in the +// reduction multiple times. +// In each process group only the leading process with linear index 0 would use +// the original operand. +// The other processes would use the reduction operation neutral tensor. +static Value createDestinationPassingStyleInitOperand( + LinalgOp op, Value spmdizedOperand, ArrayRef reductionMeshAxes, + MeshOp meshOp, ImplicitLocOpBuilder &builder) { + Value processLinearIndexInReductionGroup = mesh::createProcessLinearIndex( + meshOp.getSymName(), reductionMeshAxes, builder); + Value zero = builder.create(0); + Value isLeadProcess = builder.create( + builder.getI1Type(), arith::CmpIPredicate::eq, + processLinearIndexInReductionGroup, zero); + scf::IfOp ifOp = builder.create(spmdizedOperand.getType(), + isLeadProcess, true, true); + // Then block. + { + OpBuilder::InsertionGuard insertionGuard(builder); + builder.setInsertionPointToEnd(&ifOp.getThenRegion().front()); + builder.create(spmdizedOperand); + } + + // Else block. + { + OpBuilder::InsertionGuard insertionGuard(builder); + builder.setInsertionPointToEnd(&ifOp.getElseRegion().front()); + SmallVector shape = + tensor::getMixedSizes(builder, builder.getLoc(), spmdizedOperand); + PartialReductionOpInterface partialReductionIface = + llvm::cast(op.getOperation()); + FailureOr reductionNeutralTensorOp = + partialReductionIface.generateInitialTensorForPartialReduction( + builder, builder.getLoc(), shape, {}); + assert(succeeded(reductionNeutralTensorOp)); + builder.create( + reductionNeutralTensorOp.value()->getResult(0)); + } + return ifOp.getResult(0); +} + +// Create the DPS init operands for the spmdized Linalg op. +// Return all the new spmdized operands. +static SmallVector createDestinationPassingStyleInitOperands( + LinalgOp op, MeshOp meshOp, ArrayRef spmdizedOperands, + ArrayRef reductionMeshAxes, IRMapping &spmdizationMap, + ImplicitLocOpBuilder &builder) { + // TODO: add support for multiple destination passing style initial value + // operands. + // PartialReductionOpInterface::generateInitialTensorForPartialReduction + // needs to also support multiple DPS initial operands. + SmallVector newOperands = llvm::to_vector(spmdizedOperands); + auto operandIdx = op.getDpsInitOperand(0)->getOperandNumber(); + Value spmdizedInitOperand = + spmdizationMap.lookup(op->getOperands()[operandIdx]); + newOperands[operandIdx] = createDestinationPassingStyleInitOperand( + op, spmdizedInitOperand, reductionMeshAxes, meshOp, builder); + return newOperands; +} + +static void createAllReduceForResultWithoutPartialSharding( + Value unshardedLinalgOpResult, ArrayRef opReductionMeshAxes, + MeshShardingAttr resultSharding, ReductionKind reductionKind, + IRMapping &spmdizationMap, ImplicitLocOpBuilder &builder) { + SmallVector allReduceMeshAxes; + llvm::copy_if(opReductionMeshAxes, std::back_inserter(allReduceMeshAxes), + [&resultSharding](MeshAxis axis) { + return !llvm::is_contained(resultSharding.getPartialAxes(), + axis); + }); + if (allReduceMeshAxes.empty()) { + return; + } + + Value spmdizedLinalgOpResult = spmdizationMap.lookup(unshardedLinalgOpResult); + Value reducedValue = builder.create( + spmdizedLinalgOpResult, resultSharding.getMesh().getValue(), + allReduceMeshAxes, reductionKind); + spmdizationMap.map(unshardedLinalgOpResult, reducedValue); +} + +static void createAllReduceForResultsWithoutPartialShardings( + LinalgOp unshardedOp, ArrayRef opReductionMeshAxes, + ArrayRef resultShardings, IRMapping &spmdizationMap, + ImplicitLocOpBuilder &builder) { + ReductionKind reductionKind = getReductionKindOfLinalgOp(unshardedOp); + for (auto [unshardedLinalgOpResult, resultSharding] : + llvm::zip_equal(unshardedOp->getResults(), resultShardings)) { + createAllReduceForResultWithoutPartialSharding( + unshardedLinalgOpResult, opReductionMeshAxes, resultSharding, + reductionKind, spmdizationMap, builder); + } +} + +static void spmdizeLinalgOpWithShardedReduction( + LinalgOp op, ArrayRef spmdizedOperands, + ArrayRef operandShardings, + ArrayRef resultShardings, + ArrayRef loopIteratorTypes, + ArrayRef> meshAxisAssignmentForLoopIterators, + IRMapping &spmdizationMap, SymbolTableCollection &symbolTable, + ImplicitLocOpBuilder &builder) { + MeshOp mesh = getMesh(op, operandShardings, resultShardings, symbolTable); + SmallVector reductionMeshAxes = mesh::getReductionMeshAxes( + loopIteratorTypes, meshAxisAssignmentForLoopIterators); + SmallVector spmdizedLinalgOpOperands = + createDestinationPassingStyleInitOperands(op, mesh, spmdizedOperands, + reductionMeshAxes, + spmdizationMap, builder); + // We must not change the operand mappings of the original spmdizationMap as + // they are the mappings for the whole spmdization blob and may be used by + // others. + IRMapping internalSpmdizationMap; + for (auto [unshardedOperand, spmdizedOperand] : + llvm::zip_equal(op->getOperands(), spmdizedLinalgOpOperands)) { + internalSpmdizationMap.map(unshardedOperand, spmdizedOperand); + } + spmdizeTriviallyShardableOperation( + *op, spmdizedLinalgOpOperands, operandShardings, resultShardings, + internalSpmdizationMap, symbolTable, builder); + for (Value result : op->getResults()) { + spmdizationMap.map(result, internalSpmdizationMap.lookup(result)); + } + + // Handle partial shardings. + createAllReduceForResultsWithoutPartialShardings( + op, reductionMeshAxes, resultShardings, spmdizationMap, builder); +} + +namespace { + +// ShardingInterface for ops that implement LinalgStructuredInterface. +// The supported ops are only those where the indexing maps are projected +// permutations. +template +struct StructuredOpShardingInterface + : public mesh::ShardingInterface::ExternalModel< + StructuredOpShardingInterface, Op> { + SmallVector getLoopIteratorTypes(Operation *op) const { + return llvm::cast(op).getIteratorTypesArray(); + } + + SmallVector getIndexingMaps(Operation *op) const { + LinalgOp linalgOp = llvm::cast(op); + SmallVector res = linalgOp.getIndexingMapsArray(); + + // Results must have the same indexing as destination passing style initial + // operands. + for (int64_t i = 0; i < linalgOp.getNumDpsInits(); ++i) { + res.push_back(res[linalgOp.getDpsInitOperand(i)->getOperandNumber()]); + } + + return res; + } + + LogicalResult spmdize(Operation *op, ArrayRef spmdizedOperands, + ArrayRef operandShardings, + ArrayRef resultShardings, + IRMapping &spmdizationMap, + SymbolTableCollection &symbolTable, + OpBuilder &builder) const { + LinalgOp linalgOp = llvm::cast(op); + + SmallVector indexingMaps = linalgOp.getIndexingMapsArray(); + bool allIndexingMapsAreProjectedPermutation = + llvm::all_of(indexingMaps, [](AffineMap map) { + return map.isProjectedPermutation(); + }); + if (!allIndexingMapsAreProjectedPermutation) { + // TODO: handle non-projected permutations. + return op->emitOpError() + << "supports indexing maps that are only projected permutation."; + } + + SmallVector loopIteratorTypes = + linalgOp.getIteratorTypesArray(); + ShardingArray meshAxisAssignmentForLoopIterators = + getMeshAxisAssignmentForLoopIterators(operandShardings, resultShardings, + loopIteratorTypes, indexingMaps); + if (mesh::isAtLeastOneReductionIteratorSharded( + loopIteratorTypes, meshAxisAssignmentForLoopIterators)) { + ImplicitLocOpBuilder implicitLocBuilder(op->getLoc(), builder); + spmdizeLinalgOpWithShardedReduction( + linalgOp, spmdizedOperands, operandShardings, resultShardings, + loopIteratorTypes, meshAxisAssignmentForLoopIterators, spmdizationMap, + symbolTable, implicitLocBuilder); + } else { + spmdizeTriviallyShardableOperation(*op, spmdizedOperands, + operandShardings, resultShardings, + spmdizationMap, symbolTable, builder); + } + + return success(); + } +}; + +} // namespace + +template +static void registerOne(MLIRContext *ctx) { + OpType::template attachInterface>(*ctx); +} + +/// Variadic helper function. +template +static void registerAll(MLIRContext *ctx) { + (registerOne(ctx), ...); +} + +void registerMeshShardingInterfaceExternalModels(DialectRegistry ®istry) { + registry.addExtension(+[](MLIRContext *ctx, LinalgDialect *dialect) { + DialectRegistry registry; + registry.insert(); + ctx->appendDialectRegistry(registry); + for (StringRef name : registry.getDialectNames()) + ctx->getOrLoadDialect(name); + + registerOne(ctx); + registerAll< +#define GET_OP_LIST +#include "mlir/Dialect/Linalg/IR/LinalgStructuredOps.cpp.inc" + >(ctx); + }); +} + +} // namespace mlir::linalg diff --git a/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp index 8b3119f02e8f..bd870d4f982e 100644 --- a/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp +++ b/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp @@ -275,14 +275,6 @@ struct LinalgOpPartialReductionInterface ArrayRef oldShape = linalgOp.getShape(linalgOp.getDpsInitOperand(0)); - // Extend tile size vector to the rank of the output tensor. - SmallVector tileSizeVector = - getValueOrCreateConstantIndexOp(b, loc, sizes); - if (tileSizeVector.size() < oldShape.size()) { - auto zero = b.create(loc, 0); - tileSizeVector.append(oldShape.size() - tileSizeVector.size(), zero); - } - // Calculate the new shape, we insert the new dimensions based on the index // of the reduction dimensions. SmallVector newOutputShape; diff --git a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp index 50163880e85f..03f11ad1f949 100644 --- a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp +++ b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp @@ -647,6 +647,13 @@ void AllReduceOp::getCanonicalizationPatterns(RewritePatternSet &patterns, patterns.add>(context); } +void AllReduceOp::build(OpBuilder &odsBuilder, OperationState &odsState, + Value input, StringRef mesh, + ArrayRef meshAxes, ReductionKind reduction) { + build(odsBuilder, odsState, input.getType(), mesh, meshAxes, input, + reduction); +} + void AllReduceOp::getAsmResultNames( function_ref setNameFn) { setNameFn(getResult(), "all_reduce"); diff --git a/mlir/lib/Dialect/Mesh/Interfaces/ShardingInterface.cpp b/mlir/lib/Dialect/Mesh/Interfaces/ShardingInterface.cpp index fe3d7c44413f..9acee5aa8d86 100644 --- a/mlir/lib/Dialect/Mesh/Interfaces/ShardingInterface.cpp +++ b/mlir/lib/Dialect/Mesh/Interfaces/ShardingInterface.cpp @@ -539,8 +539,9 @@ static bool areValuesCompatibleWithFullReplicationShardings( if (std::size(values) != std::size(shardings)) { return false; } - return llvm::all_of(llvm::zip(std::forward(values), - std::forward(shardings)), + return llvm::all_of(llvm::zip_equal( + std::forward(values), + std::forward(shardings)), [](auto valueAndSharding) { return isValueCompatibleWithFullReplicationSharding( std::get<0>(valueAndSharding), @@ -563,6 +564,88 @@ void mesh::spmdizeFullyReplicatedOperation( builder.clone(op, spmdizationMap); } +static void updateMeshAxisAssignmentForLoopIterators( + ArrayRef meshAxesAssignmentForTensorAxis, AffineExpr indexingExpr, + SmallVector>> + &meshAxesAssignmentForLoopIterators) { + AffineDimExpr affineDimExpr = cast(indexingExpr); + unsigned loopIteratorIdx = affineDimExpr.getPosition(); + if (meshAxesAssignmentForLoopIterators[loopIteratorIdx]) { + assert(llvm::equal(meshAxesAssignmentForTensorAxis, + *meshAxesAssignmentForLoopIterators[loopIteratorIdx])); + } else { + meshAxesAssignmentForLoopIterators[loopIteratorIdx] = + llvm::to_vector(meshAxesAssignmentForTensorAxis); + } +} + +ShardingArray mesh::getMeshAxisAssignmentForLoopIterators( + ArrayRef operandShardings, + ArrayRef resultShardings, + ArrayRef loopIteratorTypes, + ArrayRef indexingMaps) { + SmallVector>> + meshAxisAssignmentForLoopIterators(loopIteratorTypes.size()); + SmallVector operatorAndResultShardings; + operatorAndResultShardings.reserve(operandShardings.size() + + resultShardings.size()); + llvm::append_range(operatorAndResultShardings, operandShardings); + for (auto [sharding, affineMap] : + llvm::zip_equal(operatorAndResultShardings, indexingMaps)) { + if (!sharding) { + continue; + } + for (auto [meshAxesAssignmentForTensorAxis, indexingExpr] : + llvm::zip(sharding.getSplitAxes(), affineMap.getResults())) { + updateMeshAxisAssignmentForLoopIterators( + meshAxesAssignmentForTensorAxis.asArrayRef(), indexingExpr, + meshAxisAssignmentForLoopIterators); + } + // Missing trailing split axes means replication on those tensor dimensions. + for (unsigned i = sharding.getSplitAxes().size(); + i < affineMap.getNumResults(); ++i) { + updateMeshAxisAssignmentForLoopIterators( + {}, affineMap.getResults()[i], meshAxisAssignmentForLoopIterators); + } + } + + ShardingArray res; + llvm::transform(meshAxisAssignmentForLoopIterators, std::back_inserter(res), + [](std::optional> &axes) { + if (!axes) { + return SmallVector(); + }; + return std::move(*axes); + }); + return res; +} + +bool mesh::isAtLeastOneReductionIteratorSharded( + ArrayRef loopIteratorTypes, + ArrayRef> meshAxisAssignmentForLoopIterators) { + for (auto [loopIteratorType, meshAxisAssignment] : + llvm::zip_equal(loopIteratorTypes, meshAxisAssignmentForLoopIterators)) { + if (loopIteratorType == utils::IteratorType::reduction && + !meshAxisAssignment.empty()) { + return true; + } + } + return false; +} + +SmallVector mesh::getReductionMeshAxes( + ArrayRef loopIteratorTypes, + ArrayRef> meshAxisAssignmentForLoopIterators) { + SmallVector meshAxes; + for (auto [loopIteratorType, meshAxisAssignment] : + llvm::zip_equal(loopIteratorTypes, meshAxisAssignmentForLoopIterators)) { + if (loopIteratorType == utils::IteratorType::reduction) { + llvm::append_range(meshAxes, meshAxisAssignment); + } + } + return meshAxes; +} + void mesh::spmdizeTriviallyShardableOperation( Operation &op, ArrayRef spmdizedOperands, ArrayRef operandShardings, @@ -572,7 +655,7 @@ void mesh::spmdizeTriviallyShardableOperation( Operation *newOp = builder.clone(op, spmdizationMap); // Set the result types to the sharded counterparts. for (auto [oldResult, newResult, sharding] : - llvm::zip(op.getResults(), newOp->getResults(), resultShardings)) { + llvm::zip_equal(op.getResults(), newOp->getResults(), resultShardings)) { newResult.setType(shardType(newResult.getType(), getMesh(&op, sharding.getMesh(), symbolTable), sharding)); diff --git a/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp b/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp index d59b9119dea5..cb13ee404751 100644 --- a/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp +++ b/mlir/lib/Dialect/Mesh/Transforms/Transforms.cpp @@ -208,4 +208,17 @@ createCollectiveProcessGroupSize(MeshOp mesh, ArrayRef axes, .cast>(); } +TypedValue createProcessLinearIndex(StringRef mesh, + ArrayRef meshAxes, + ImplicitLocOpBuilder &builder) { + ResultRange processInGroupMultiIndex = + builder.create(mesh, meshAxes).getResults(); + Operation::result_range processGroupShape = + builder.create(mesh, meshAxes).getResult(); + OpFoldResult processInGroupLinearIndex = affine::linearizeIndex( + llvm::to_vector_of(processInGroupMultiIndex), + llvm::to_vector_of(processGroupShape), builder); + return cast>(processInGroupLinearIndex.get()); +} + } // namespace mlir::mesh diff --git a/mlir/test/Dialect/Linalg/mesh-spmdization.mlir b/mlir/test/Dialect/Linalg/mesh-spmdization.mlir new file mode 100644 index 000000000000..6d21def8de27 --- /dev/null +++ b/mlir/test/Dialect/Linalg/mesh-spmdization.mlir @@ -0,0 +1,165 @@ +// RUN: mlir-opt \ +// RUN: --mesh-spmdization \ +// RUN: --test-constant-fold \ +// RUN: --split-input-file \ +// RUN: %s | FileCheck %s + +// CHECK: #[[$MAP_IDENTITY_1D:.*]] = affine_map<(d0) -> (d0)> +#map_identity_1d = affine_map<(d0) -> (d0)> + +mesh.mesh @mesh_1d(shape = 2) + +// CHECK-LABEL: func @elementwise_static_1d_mesh_static_1d_tensor +func.func @elementwise_static_1d_mesh_static_1d_tensor( + // CHECK-SAME: %[[IN1:[A-Za-z0-9_]+]]: tensor<1xi8>, + %in1: tensor<2xi8>, + // CHECK-SAME: %[[IN2:[A-Za-z0-9_]+]]: tensor<1xi8>, + %in2: tensor<2xi8>, + // CHECK-SAME: %[[DPS_OUT:[A-Za-z0-9_]+]]: tensor<1xi8> + %dps_out: tensor<2xi8> +// CHECK-SAME: -> tensor<1xi8> { +) -> tensor<2xi8> { + %in1_shared1 = mesh.shard %in1 to <@mesh_1d, [[0]]> : tensor<2xi8> + %in1_shared2 = mesh.shard %in1_shared1 to <@mesh_1d, [[0]]> annotate_for_users: tensor<2xi8> + %in2_shared1 = mesh.shard %in2 to <@mesh_1d, [[0]]> : tensor<2xi8> + %in2_shared2 = mesh.shard %in2_shared1 to <@mesh_1d, [[0]]> annotate_for_users: tensor<2xi8> + %dps_out_shared1 = mesh.shard %dps_out to <@mesh_1d, [[0]]> : tensor<2xi8> + %dps_out_shared2 = mesh.shard %dps_out_shared1 to <@mesh_1d, [[0]]> annotate_for_users: tensor<2xi8> + // CHECK: %[[RES:.*]] = linalg.generic { + // CHECK-SAME: indexing_maps = [#[[$MAP_IDENTITY_1D]], #[[$MAP_IDENTITY_1D]], #[[$MAP_IDENTITY_1D]]], + // CHECK-SAME: iterator_types = ["parallel"]} + // CHECK-SAME: ins(%[[IN1]], %[[IN2]] : tensor<1xi8>, tensor<1xi8>) + // CHECK-SAME: outs(%[[DPS_OUT]] : tensor<1xi8>) { + %res = linalg.generic { + indexing_maps = [#map_identity_1d, #map_identity_1d, #map_identity_1d], + iterator_types = ["parallel"] + } ins(%in1_shared2, %in2_shared2 : tensor<2xi8>, tensor<2xi8>) + outs(%dps_out_shared2 : tensor<2xi8>) { + ^bb0(%in1_scalar: i8, %in2_scalar: i8, %out: i8): + %res_scalar = arith.muli %in1_scalar, %in2_scalar : i8 + linalg.yield %res_scalar : i8 + } -> tensor<2xi8> + %res_shared1 = mesh.shard %res to <@mesh_1d, [[0]]> : tensor<2xi8> + %res_shared2 = mesh.shard %res_shared1 to <@mesh_1d, [[0]]> annotate_for_users: tensor<2xi8> + // CHECK: return %[[RES]] : tensor<1xi8> + return %res_shared2 : tensor<2xi8> +} + +// ----- + +mesh.mesh @mesh_1d(shape = 4) + +// CHECK-LABEL: func @matmul_1d_mesh_static_tensors_parallel_iterator_sharding +func.func @matmul_1d_mesh_static_tensors_parallel_iterator_sharding( + // CHECK-SAME: %[[IN1:[A-Za-z0-9_]+]]: tensor<1x3xi8>, + %in1: tensor<4x3xi8>, +// CHECK-SAME: %[[IN2:[A-Za-z0-9_]+]]: tensor<3x8xi8>, + %in2: tensor<3x8xi8>, +// CHECK-SAME: %[[DPS_OUT:[A-Za-z0-9_]+]]: tensor<1x8xi8> + %dps_out: tensor<4x8xi8> +// CHECK-SAME: -> tensor<1x8xi8> { +) -> tensor<4x8xi8> { + %in1_shared1 = mesh.shard %in1 to <@mesh_1d, [[0]]> : tensor<4x3xi8> + %in1_shared2 = mesh.shard %in1_shared1 to <@mesh_1d, [[0]]> annotate_for_users: tensor<4x3xi8> + %in2_shared1 = mesh.shard %in2 to <@mesh_1d, [[]]> : tensor<3x8xi8> + %in2_shared2 = mesh.shard %in2_shared1 to <@mesh_1d, [[]]> annotate_for_users: tensor<3x8xi8> + %dps_out_shared1 = mesh.shard %dps_out to <@mesh_1d, [[0]]> : tensor<4x8xi8> + %dps_out_shared2 = mesh.shard %dps_out_shared1 to <@mesh_1d, [[0]]> annotate_for_users: tensor<4x8xi8> + // CHECK: %[[RES:.*]] = linalg.matmul + // CHECK-SAME: ins(%[[IN1]], %[[IN2]] : tensor<1x3xi8>, tensor<3x8xi8>) + // CHECK-SAME: outs(%[[DPS_OUT]] : tensor<1x8xi8>) + // CHECK-SAME: -> tensor<1x8xi8> + %res = linalg.matmul ins(%in1_shared2, %in2_shared2 : tensor<4x3xi8>, tensor<3x8xi8>) + outs(%dps_out_shared2 : tensor<4x8xi8>) -> tensor<4x8xi8> + %res_shared1 = mesh.shard %res to <@mesh_1d, [[0]]> : tensor<4x8xi8> + %res_shared2 = mesh.shard %res_shared1 to <@mesh_1d, [[0]]> annotate_for_users: tensor<4x8xi8> + // CHECK: return %[[RES]] : tensor<1x8xi8> + return %res_shared2 : tensor<4x8xi8> +} + +// ----- + +mesh.mesh @mesh_1d(shape = 3) + +// CHECK-LABEL: func @matmul_1d_mesh_static_tensors_reduction_iterator_sharding +func.func @matmul_1d_mesh_static_tensors_reduction_iterator_sharding( + // CHECK-SAME: %[[IN1:[A-Za-z0-9_]+]]: tensor<4x2xi8>, + %in1: tensor<4x6xi8>, +// CHECK-SAME: %[[IN2:[A-Za-z0-9_]+]]: tensor<2x8xi8>, + %in2: tensor<6x8xi8>, +// CHECK-SAME: %[[DPS_OUT:[A-Za-z0-9_]+]]: tensor<4x8xi8> + %dps_out: tensor<4x8xi8> +// CHECK-SAME: -> tensor<4x8xi8> { +) -> tensor<4x8xi8> { + %in1_shared1 = mesh.shard %in1 to <@mesh_1d, [[], [0]]> : tensor<4x6xi8> + %in1_shared2 = mesh.shard %in1_shared1 to <@mesh_1d, [[], [0]]> annotate_for_users: tensor<4x6xi8> + %in2_shared1 = mesh.shard %in2 to <@mesh_1d, [[0]]> : tensor<6x8xi8> + %in2_shared2 = mesh.shard %in2_shared1 to <@mesh_1d, [[0]]> annotate_for_users: tensor<6x8xi8> + %dps_out_shared1 = mesh.shard %dps_out to <@mesh_1d, [[]]> : tensor<4x8xi8> + %dps_out_shared2 = mesh.shard %dps_out_shared1 to <@mesh_1d, [[]]> annotate_for_users: tensor<4x8xi8> + // CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index + // CHECK-DAG: %[[C0_I8:.*]] = arith.constant 0 : i8 + // CHECK-DAG: %[[PROCESS_IDX:.*]] = mesh.process_multi_index on @mesh_1d axes = [0] : index + // CHECK-DAG: %[[MESH_SIZE:.*]] = mesh.mesh_shape @mesh_1d axes = [0] : index + // CHECK: %[[DPS_INIT_OPERAND_CONDITION:.*]] = arith.cmpi eq, %[[PROCESS_IDX]], %[[C0]] : index + // CHECK: %[[DPS_INIT_OPERAND:.*]] = scf.if %[[DPS_INIT_OPERAND_CONDITION]] -> (tensor<4x8xi8>) { + // CHECK: scf.yield %[[DPS_OUT]] : tensor<4x8xi8> + // CHECK: } else { + // CHECK-DAG: %[[EMPTY_TENSOR:.*]] = tensor.empty() : tensor<4x8xi8> + // CHECK: %[[NEUTRAL_ELEMENT_FILLED_TENSOR:.*]] = linalg.fill ins(%[[C0_I8]] : i8) + // CHECK-SAME: outs(%[[EMPTY_TENSOR]] : tensor<4x8xi8>) -> tensor<4x8xi8> + // CHECK: scf.yield %[[NEUTRAL_ELEMENT_FILLED_TENSOR]] : tensor<4x8xi8> + // CHECK: } + // CHECK: %[[SHARDED_MATMUL:.*]] = linalg.matmul ins(%[[IN1]], %[[IN2]] : tensor<4x2xi8>, tensor<2x8xi8>) + // CHECK-SAME: outs(%[[DPS_INIT_OPERAND]] : tensor<4x8xi8>) -> tensor<4x8xi8> + // CHECK: %[[ALL_REDUCED:.*]] = mesh.all_reduce %[[SHARDED_MATMUL]] on @mesh_1d mesh_axes = [0] : tensor<4x8xi8> -> tensor<4x8xi8> + %res = linalg.matmul ins(%in1_shared2, %in2_shared2 : tensor<4x6xi8>, tensor<6x8xi8>) + outs(%dps_out_shared2 : tensor<4x8xi8>) -> tensor<4x8xi8> + %res_shared1 = mesh.shard %res to <@mesh_1d, [[]]> : tensor<4x8xi8> + %res_shared2 = mesh.shard %res_shared1 to <@mesh_1d, [[]]> annotate_for_users: tensor<4x8xi8> + // CHECK: return %[[ALL_REDUCED]] : tensor<4x8xi8> + return %res_shared2 : tensor<4x8xi8> +} + +// ----- + +mesh.mesh @mesh_1d(shape = 3) + +// CHECK-LABEL: func @matmul_1d_mesh_static_tensors_reduction_iterator_sharding_with_partial_result +func.func @matmul_1d_mesh_static_tensors_reduction_iterator_sharding_with_partial_result( + // CHECK-SAME: %[[IN1:[A-Za-z0-9_]+]]: tensor<4x2xi8>, + %in1: tensor<4x6xi8>, +// CHECK-SAME: %[[IN2:[A-Za-z0-9_]+]]: tensor<2x8xi8>, + %in2: tensor<6x8xi8>, +// CHECK-SAME: %[[DPS_OUT:[A-Za-z0-9_]+]]: tensor<4x8xi8> + %dps_out: tensor<4x8xi8> +// CHECK-SAME: -> tensor<4x8xi8> { +) -> tensor<4x8xi8> { + %in1_shared1 = mesh.shard %in1 to <@mesh_1d, [[], [0]]> : tensor<4x6xi8> + %in1_shared2 = mesh.shard %in1_shared1 to <@mesh_1d, [[], [0]]> annotate_for_users: tensor<4x6xi8> + %in2_shared1 = mesh.shard %in2 to <@mesh_1d, [[0]]> : tensor<6x8xi8> + %in2_shared2 = mesh.shard %in2_shared1 to <@mesh_1d, [[0]]> annotate_for_users: tensor<6x8xi8> + %dps_out_shared1 = mesh.shard %dps_out to <@mesh_1d, [[]]> : tensor<4x8xi8> + %dps_out_shared2 = mesh.shard %dps_out_shared1 to <@mesh_1d, [[]]> annotate_for_users: tensor<4x8xi8> + // CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index + // CHECK-DAG: %[[C0_I8:.*]] = arith.constant 0 : i8 + // CHECK-DAG: %[[PROCESS_IDX:.*]] = mesh.process_multi_index on @mesh_1d axes = [0] : index + // CHECK-DAG: %[[MESH_SIZE:.*]] = mesh.mesh_shape @mesh_1d axes = [0] : index + // CHECK: %[[DPS_INIT_OPERAND_CONDITION:.*]] = arith.cmpi eq, %[[PROCESS_IDX]], %[[C0]] : index + // CHECK: %[[DPS_INIT_OPERAND:.*]] = scf.if %[[DPS_INIT_OPERAND_CONDITION]] -> (tensor<4x8xi8>) { + // CHECK: scf.yield %[[DPS_OUT]] : tensor<4x8xi8> + // CHECK: } else { + // CHECK-DAG: %[[EMPTY_TENSOR:.*]] = tensor.empty() : tensor<4x8xi8> + // CHECK: %[[NEUTRAL_ELEMENT_FILLED_TENSOR:.*]] = linalg.fill ins(%[[C0_I8]] : i8) + // CHECK-SAME: outs(%[[EMPTY_TENSOR]] : tensor<4x8xi8>) -> tensor<4x8xi8> + // CHECK: scf.yield %[[NEUTRAL_ELEMENT_FILLED_TENSOR]] : tensor<4x8xi8> + // CHECK: } + // CHECK: %[[SHARDED_MATMUL:.*]] = linalg.matmul ins(%[[IN1]], %[[IN2]] : tensor<4x2xi8>, tensor<2x8xi8>) + // CHECK-SAME: outs(%[[DPS_INIT_OPERAND]] : tensor<4x8xi8>) -> tensor<4x8xi8> + %res = linalg.matmul ins(%in1_shared2, %in2_shared2 : tensor<4x6xi8>, tensor<6x8xi8>) + outs(%dps_out_shared2 : tensor<4x8xi8>) -> tensor<4x8xi8> + %res_shared1 = mesh.shard %res to <@mesh_1d, [[]], partial = sum[0]> : tensor<4x8xi8> + %res_shared2 = mesh.shard %res_shared1 to <@mesh_1d, [[]], partial = sum[0]> annotate_for_users: tensor<4x8xi8> + // CHECK: return %[[SHARDED_MATMUL]] : tensor<4x8xi8> + return %res_shared2 : tensor<4x8xi8> +} diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 7a6bc2dc3202..2cfe61844703 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -10841,6 +10841,7 @@ cc_library( ":MemRefDialect", ":Parser", ":SCFDialect", + ":MeshShardingInterface", ":SideEffectInterfaces", ":SparseTensorDialect", ":Support", @@ -10994,10 +10995,13 @@ cc_library( ":MathDialect", ":MemRefDialect", ":MemRefTransforms", + ":MeshDialect", + ":MeshTransforms", ":Pass", ":SCFDialect", ":SCFTransforms", ":SCFUtils", + ":MeshShardingInterface", ":SparseTensorDialect", ":SubsetOpInterface", ":Support", -- GitLab From ddaf040ea924b1bdd4e093f583018c262da3cc7f Mon Sep 17 00:00:00 2001 From: Matthias Springer Date: Fri, 8 Mar 2024 10:06:24 +0900 Subject: [PATCH 544/929] [mlir][Transforms][NFC] Make signature conversion more efficient (#83922) During block signature conversion, a new block is inserted and ops are moved from the old block to the new block. This commit changes the implementation such that ops are moved in bulk (`splice`) instead of one-by-one; that's what `splitBlock` is doing. This also makes it possible to pass the new block argument types directly to `createBlock` instead of using `addArgument` (which bypasses the rewriter). This doesn't change anything from a technical point of view (there is no rewriter API for adding arguments at the moment), but the implementation reads a bit nicer. --- .../Transforms/Utils/DialectConversion.cpp | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp index d7dc902a9a5e..8b2d71408a56 100644 --- a/mlir/lib/Transforms/Utils/DialectConversion.cpp +++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp @@ -1281,7 +1281,7 @@ Block *ConversionPatternRewriterImpl::applySignatureConversion( ConversionPatternRewriter &rewriter, Block *block, const TypeConverter *converter, TypeConverter::SignatureConversion &signatureConversion) { - MLIRContext *ctx = rewriter.getContext(); + OpBuilder::InsertionGuard g(rewriter); // If no arguments are being changed or added, there is nothing to do. unsigned origArgCount = block->getNumArguments(); @@ -1289,14 +1289,9 @@ Block *ConversionPatternRewriterImpl::applySignatureConversion( if (llvm::equal(block->getArgumentTypes(), convertedTypes)) return block; - // Split the block at the beginning to get a new block to use for the updated - // signature. - Block *newBlock = rewriter.splitBlock(block, block->begin()); - block->replaceAllUsesWith(newBlock); - - // Map all new arguments to the location of the argument they originate from. + // Compute the locations of all block arguments in the new block. SmallVector newLocs(convertedTypes.size(), - Builder(ctx).getUnknownLoc()); + rewriter.getUnknownLoc()); for (unsigned i = 0; i < origArgCount; ++i) { auto inputMap = signatureConversion.getInputMapping(i); if (!inputMap || inputMap->replacementValue) @@ -1306,9 +1301,16 @@ Block *ConversionPatternRewriterImpl::applySignatureConversion( newLocs[inputMap->inputNo + j] = origLoc; } - SmallVector newArgRange( - newBlock->addArguments(convertedTypes, newLocs)); - ArrayRef newArgs(newArgRange); + // Insert a new block with the converted block argument types and move all ops + // from the old block to the new block. + Block *newBlock = + rewriter.createBlock(block->getParent(), std::next(block->getIterator()), + convertedTypes, newLocs); + appendRewrite(newBlock, block, newBlock->end()); + newBlock->getOperations().splice(newBlock->end(), block->getOperations()); + + // Replace all uses of the old block with the new block. + block->replaceAllUsesWith(newBlock); // Remap each of the original arguments as determined by the signature // conversion. @@ -1333,7 +1335,8 @@ Block *ConversionPatternRewriterImpl::applySignatureConversion( } // Otherwise, this is a 1->1+ mapping. - auto replArgs = newArgs.slice(inputMap->inputNo, inputMap->size); + auto replArgs = + newBlock->getArguments().slice(inputMap->inputNo, inputMap->size); Value newArg; // If this is a 1->1 mapping and the types of new and replacement arguments -- GitLab From e7a22e72de79352c4639664f1ac678555a4c20e4 Mon Sep 17 00:00:00 2001 From: Chen Zheng Date: Thu, 7 Mar 2024 20:19:26 -0500 Subject: [PATCH 545/929] [PPC] precommit cases for issue 74915 --- llvm/test/CodeGen/PowerPC/pr74951.ll | 54 ++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 llvm/test/CodeGen/PowerPC/pr74951.ll diff --git a/llvm/test/CodeGen/PowerPC/pr74951.ll b/llvm/test/CodeGen/PowerPC/pr74951.ll new file mode 100644 index 000000000000..a0d19fc09cc2 --- /dev/null +++ b/llvm/test/CodeGen/PowerPC/pr74951.ll @@ -0,0 +1,54 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc < %s -verify-machineinstrs -ppc-asm-full-reg-names -mtriple=powerpc64-ibm-aix-xcoff | FileCheck %s + +%struct.anon = type { i32 } + +@b = local_unnamed_addr global %struct.anon { i32 -1 }, align 4 +@g = local_unnamed_addr global [1 x i1] zeroinitializer, align 1 + +define noundef signext i32 @main() { +; CHECK-LABEL: main: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: ld r3, L..C0(r2) # @b +; CHECK-NEXT: lwz r3, 0(r3) +; CHECK-NEXT: extsw r4, r3 +; CHECK-NEXT: neg r4, r4 +; CHECK-NEXT: andi. r5, r3, 65535 +; CHECK-NEXT: rldicl r4, r4, 1, 63 +; CHECK-NEXT: bne cr0, L..BB0_4 +; CHECK-NEXT: # %bb.1: # %lor.rhs.i.i +; CHECK-NEXT: xori r5, r4, 1 +; CHECK-NEXT: cmpw r3, r5 +; CHECK-NEXT: crnot 4*cr5+lt, eq +; CHECK-NEXT: li r3, 1 +; CHECK-NEXT: bc 12, 4*cr5+lt, L..BB0_3 +; CHECK-NEXT: # %bb.2: # %lor.rhs.i.i +; CHECK-NEXT: li r3, 0 +; CHECK-NEXT: L..BB0_3: # %lor.rhs.i.i +; CHECK-NEXT: ld r5, L..C1(r2) # @g +; CHECK-NEXT: stb r3, 0(r5) +; CHECK-NEXT: L..BB0_4: # %g.exit +; CHECK-NEXT: ld r5, L..C1(r2) # @g +; CHECK-NEXT: li r3, 0 +; CHECK-NEXT: stb r4, 0(r5) +; CHECK-NEXT: blr +entry: + %0 = load i32, ptr @b, align 4 + %conv4.i = sext i32 %0 to i64 + %cmp.i = icmp slt i32 %0, 1 + %conv.i = zext i1 %cmp.i to i32 + %cmp1.i = icmp ne i32 %0, %conv.i + %conv3.i = trunc i32 %0 to i16 + %tobool.not.i.i = icmp eq i16 %conv3.i, 0 + br i1 %tobool.not.i.i, label %lor.rhs.i.i, label %g.exit + +lor.rhs.i.i: ; preds = %entry + store i1 %cmp1.i, ptr @g, align 1 + br label %g.exit + +g.exit: ; preds = %lor.end.i.i + %4 = trunc i64 %conv4.i to i32 + %cmp.i9.i = icmp sgt i32 %4, 0 + store i1 %cmp.i9.i, ptr @g, align 1 + ret i32 0 +} -- GitLab From 60a20bd6973c8fc7aa9a19465ed042604e07fb17 Mon Sep 17 00:00:00 2001 From: Matthias Springer Date: Fri, 8 Mar 2024 10:34:45 +0900 Subject: [PATCH 546/929] [mlir][Transforms] Add listener support to dialect conversion (#83425) This commit adds listener support to the dialect conversion. Similarly to the greedy pattern rewrite driver, an optional listener can be specified in the configuration object. Listeners are notified only if the dialect conversion succeeds. In case of a failure, where some IR changes are first performed and then rolled back, no notifications are sent. Due to the fact that some kinds of rewrite are reflected in the IR immediately and some in a delayed fashion, there are certain limitations when attaching a listener; these are documented in `ConversionConfig`. To summarize, users are always notified about all rewrites that happened, but the notifications are sent all at once at the very end, and not interleaved with the actual IR changes. This change is in preparation improvements to `transform.apply_conversion_patterns`, which currently invalidates all handles. In the future, it can use a listener to update handles accordingly, similar to `transform.apply_patterns`. --- .../mlir/Transforms/DialectConversion.h | 33 +++ .../Transforms/Utils/DialectConversion.cpp | 225 ++++++++++++++---- mlir/test/Transforms/test-legalizer.mlir | 71 +++++- mlir/test/lib/Dialect/Test/TestPatterns.cpp | 28 ++- 4 files changed, 302 insertions(+), 55 deletions(-) diff --git a/mlir/include/mlir/Transforms/DialectConversion.h b/mlir/include/mlir/Transforms/DialectConversion.h index 01fde101ef3c..83198c9b0db5 100644 --- a/mlir/include/mlir/Transforms/DialectConversion.h +++ b/mlir/include/mlir/Transforms/DialectConversion.h @@ -1085,6 +1085,39 @@ struct ConversionConfig { /// IR during an analysis conversion and only pre-existing operations are /// added to the set. DenseSet *legalizableOps = nullptr; + + /// An optional listener that is notified about all IR modifications in case + /// dialect conversion succeeds. If the dialect conversion fails and no IR + /// modifications are visible (i.e., they were all rolled back), no + /// notifications are sent. + /// + /// Note: Notifications are sent in a delayed fashion, when the dialect + /// conversion is guaranteed to succeed. At that point, some IR modifications + /// may already have been materialized. Consequently, operations/blocks that + /// are passed to listener callbacks should not be accessed. (Ops/blocks are + /// guaranteed to be valid pointers and accessing op names is allowed. But + /// there are no guarantees about the state of ops/blocks at the time that a + /// callback is triggered.) + /// + /// Example: Consider a dialect conversion a new op ("test.foo") is created + /// and inserted, and later moved to another block. (Moving ops also triggers + /// "notifyOperationInserted".) + /// + /// (1) notifyOperationInserted: "test.foo" (into block "b1") + /// (2) notifyOperationInserted: "test.foo" (moved to another block "b2") + /// + /// When querying "op->getBlock()" during the first "notifyOperationInserted", + /// "b2" would be returned because "moving an op" is a kind of rewrite that is + /// immediately performed by the dialect conversion (and rolled back upon + /// failure). + // + // Note: When receiving a "notifyBlockInserted"/"notifyOperationInserted" + // callback, the previous region/block is provided to the callback, but not + // the iterator pointing to the exact location within the region/block. That + // is because these notifications are sent with a delay (after the IR has + // already been modified) and iterators into past IR state cannot be + // represented at the moment. + RewriterBase::Listener *listener = nullptr; }; //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp index 8b2d71408a56..c1a261eab848 100644 --- a/mlir/lib/Transforms/Utils/DialectConversion.cpp +++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp @@ -204,14 +204,22 @@ public: /// Roll back the rewrite. Operations may be erased during rollback. virtual void rollback() = 0; - /// Commit the rewrite. Operations/blocks may be unlinked during the commit - /// phase, but they must not be erased yet. This is because internal dialect - /// conversion state (such as `mapping`) may still be using them. Operations/ - /// blocks must be erased during cleanup. - virtual void commit() {} + /// Commit the rewrite. At this point, it is certain that the dialect + /// conversion will succeed. All IR modifications, except for operation/block + /// erasure, must be performed through the given rewriter. + /// + /// Instead of erasing operations/blocks, they should merely be unlinked + /// commit phase and finally be erased during the cleanup phase. This is + /// because internal dialect conversion state (such as `mapping`) may still + /// be using them. + /// + /// Any IR modification that was already performed before the commit phase + /// (e.g., insertion of an op) must be communicated to the listener that may + /// be attached to the given rewriter. + virtual void commit(RewriterBase &rewriter) {} /// Cleanup operations/blocks. Cleanup is called after commit. - virtual void cleanup() {} + virtual void cleanup(RewriterBase &rewriter) {} Kind getKind() const { return kind; } @@ -221,12 +229,6 @@ protected: IRRewrite(Kind kind, ConversionPatternRewriterImpl &rewriterImpl) : kind(kind), rewriterImpl(rewriterImpl) {} - /// Erase the given op (unless it was already erased). - void eraseOp(Operation *op); - - /// Erase the given block (unless it was already erased). - void eraseBlock(Block *block); - const ConversionConfig &getConfig() const; const Kind kind; @@ -265,6 +267,12 @@ public: return rewrite->getKind() == Kind::CreateBlock; } + void commit(RewriterBase &rewriter) override { + // The block was already created and inserted. Just inform the listener. + if (auto *listener = rewriter.getListener()) + listener->notifyBlockInserted(block, /*previous=*/{}, /*previousIt=*/{}); + } + void rollback() override { // Unlink all of the operations within this block, they will be deleted // separately. @@ -311,10 +319,19 @@ public: block = nullptr; } - void cleanup() override { + void commit(RewriterBase &rewriter) override { // Erase the block. assert(block && "expected block"); assert(block->empty() && "expected empty block"); + + // Notify the listener that the block is about to be erased. + if (auto *listener = + dyn_cast_or_null(rewriter.getListener())) + listener->notifyBlockErased(block); + } + + void cleanup(RewriterBase &rewriter) override { + // Erase the block. block->dropAllDefinedValueUses(); delete block; block = nullptr; @@ -341,6 +358,13 @@ public: firstInlinedInst(sourceBlock->empty() ? nullptr : &sourceBlock->front()), lastInlinedInst(sourceBlock->empty() ? nullptr : &sourceBlock->back()) { + // If a listener is attached to the dialect conversion, ops must be moved + // one-by-one. When they are moved in bulk, notifications cannot be sent + // because the ops that used to be in the source block at the time of the + // inlining (before the "commit" phase) are unknown at the time when + // notifications are sent (which is during the "commit" phase). + assert(!getConfig().listener && + "InlineBlockRewrite not supported if listener is attached"); } static bool classof(const IRRewrite *rewrite) { @@ -382,6 +406,16 @@ public: return rewrite->getKind() == Kind::MoveBlock; } + void commit(RewriterBase &rewriter) override { + // The block was already moved. Just inform the listener. + if (auto *listener = rewriter.getListener()) { + // Note: `previousIt` cannot be passed because this is a delayed + // notification and iterators into past IR state cannot be represented. + listener->notifyBlockInserted(block, /*previous=*/region, + /*previousIt=*/{}); + } + } + void rollback() override { // Move the block back to its original position. Region::iterator before = @@ -437,7 +471,7 @@ public: LogicalResult materializeLiveConversions(function_ref findLiveUser); - void commit() override; + void commit(RewriterBase &rewriter) override; void rollback() override; @@ -466,7 +500,7 @@ public: return rewrite->getKind() == Kind::ReplaceBlockArg; } - void commit() override; + void commit(RewriterBase &rewriter) override; void rollback() override; @@ -506,6 +540,17 @@ public: return rewrite->getKind() == Kind::MoveOperation; } + void commit(RewriterBase &rewriter) override { + // The operation was already moved. Just inform the listener. + if (auto *listener = rewriter.getListener()) { + // Note: `previousIt` cannot be passed because this is a delayed + // notification and iterators into past IR state cannot be represented. + listener->notifyOperationInserted( + op, /*previous=*/OpBuilder::InsertPoint(/*insertBlock=*/block, + /*insertPt=*/{})); + } + } + void rollback() override { // Move the operation back to its original position. Block::iterator before = @@ -549,7 +594,12 @@ public: "rewrite was neither committed nor rolled back"); } - void commit() override { + void commit(RewriterBase &rewriter) override { + // Notify the listener that the operation was modified in-place. + if (auto *listener = + dyn_cast_or_null(rewriter.getListener())) + listener->notifyOperationModified(op); + if (propertiesStorage) { OpaqueProperties propCopy(propertiesStorage); // Note: The operation may have been erased in the mean time, so @@ -600,11 +650,11 @@ public: return rewrite->getKind() == Kind::ReplaceOperation; } - void commit() override; + void commit(RewriterBase &rewriter) override; void rollback() override; - void cleanup() override; + void cleanup(RewriterBase &rewriter) override; const TypeConverter *getConverter() const { return converter; } @@ -629,6 +679,12 @@ public: return rewrite->getKind() == Kind::CreateOperation; } + void commit(RewriterBase &rewriter) override { + // The operation was already created and inserted. Just inform the listener. + if (auto *listener = rewriter.getListener()) + listener->notifyOperationInserted(op, /*previous=*/{}); + } + void rollback() override; }; @@ -666,7 +722,7 @@ public: void rollback() override; - void cleanup() override; + void cleanup(RewriterBase &rewriter) override; /// Return the type converter of this materialization (which may be null). const TypeConverter *getConverter() const { @@ -735,7 +791,7 @@ namespace detail { struct ConversionPatternRewriterImpl : public RewriterBase::Listener { explicit ConversionPatternRewriterImpl(MLIRContext *ctx, const ConversionConfig &config) - : eraseRewriter(ctx), config(config) {} + : context(ctx), config(config) {} //===--------------------------------------------------------------------===// // State Management @@ -900,6 +956,7 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener { } void notifyOperationErased(Operation *op) override { erased.insert(op); } + void notifyBlockErased(Block *block) override { erased.insert(block); } /// Pointers to all erased operations and blocks. @@ -910,8 +967,8 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener { // State //===--------------------------------------------------------------------===// - /// This rewriter must be used for erasing ops/blocks. - SingleEraseRewriter eraseRewriter; + /// MLIR context. + MLIRContext *context; // Mapping between replaced values that differ in type. This happens when // replacing a value with one of a different type. @@ -955,19 +1012,19 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener { } // namespace detail } // namespace mlir -void IRRewrite::eraseOp(Operation *op) { - rewriterImpl.eraseRewriter.eraseOp(op); -} - -void IRRewrite::eraseBlock(Block *block) { - rewriterImpl.eraseRewriter.eraseBlock(block); -} - const ConversionConfig &IRRewrite::getConfig() const { return rewriterImpl.config; } -void BlockTypeConversionRewrite::commit() { +void BlockTypeConversionRewrite::commit(RewriterBase &rewriter) { + // Inform the listener about all IR modifications that have already taken + // place: References to the original block have been replaced with the new + // block. + if (auto *listener = dyn_cast_or_null( + rewriter.getListener())) + for (Operation *op : block->getUsers()) + listener->notifyOperationModified(op); + // Process the remapping for each of the original arguments. for (auto [origArg, info] : llvm::zip_equal(origBlock->getArguments(), argInfo)) { @@ -975,7 +1032,7 @@ void BlockTypeConversionRewrite::commit() { if (!info) { if (Value newArg = rewriterImpl.mapping.lookupOrNull(origArg, origArg.getType())) - origArg.replaceAllUsesWith(newArg); + rewriter.replaceAllUsesWith(origArg, newArg); continue; } @@ -985,8 +1042,8 @@ void BlockTypeConversionRewrite::commit() { // If the argument is still used, replace it with the generated cast. if (!origArg.use_empty()) { - origArg.replaceAllUsesWith( - rewriterImpl.mapping.lookupOrDefault(castValue, origArg.getType())); + rewriter.replaceAllUsesWith(origArg, rewriterImpl.mapping.lookupOrDefault( + castValue, origArg.getType())); } } } @@ -1042,13 +1099,13 @@ LogicalResult BlockTypeConversionRewrite::materializeLiveConversions( return success(); } -void ReplaceBlockArgRewrite::commit() { +void ReplaceBlockArgRewrite::commit(RewriterBase &rewriter) { Value repl = rewriterImpl.mapping.lookupOrNull(arg, arg.getType()); if (!repl) return; if (isa(repl)) { - arg.replaceAllUsesWith(repl); + rewriter.replaceAllUsesWith(arg, repl); return; } @@ -1057,7 +1114,7 @@ void ReplaceBlockArgRewrite::commit() { // replacement value. Operation *replOp = cast(repl).getOwner(); Block *replBlock = replOp->getBlock(); - arg.replaceUsesWithIf(repl, [&](OpOperand &operand) { + rewriter.replaceUsesWithIf(arg, repl, [&](OpOperand &operand) { Operation *user = operand.getOwner(); return user->getBlock() != replBlock || replOp->isBeforeInBlock(user); }); @@ -1065,14 +1122,40 @@ void ReplaceBlockArgRewrite::commit() { void ReplaceBlockArgRewrite::rollback() { rewriterImpl.mapping.erase(arg); } -void ReplaceOperationRewrite::commit() { - for (OpResult result : op->getResults()) - if (Value newValue = - rewriterImpl.mapping.lookupOrNull(result, result.getType())) - result.replaceAllUsesWith(newValue); +void ReplaceOperationRewrite::commit(RewriterBase &rewriter) { + auto *listener = dyn_cast_or_null( + rewriter.getListener()); + + // Compute replacement values. + SmallVector replacements = + llvm::map_to_vector(op->getResults(), [&](OpResult result) { + return rewriterImpl.mapping.lookupOrNull(result, result.getType()); + }); + + // Notify the listener that the operation is about to be replaced. + if (listener) + listener->notifyOperationReplaced(op, replacements); + + // Replace all uses with the new values. + for (auto [result, newValue] : + llvm::zip_equal(op->getResults(), replacements)) + if (newValue) + rewriter.replaceAllUsesWith(result, newValue); + + // The original op will be erased, so remove it from the set of unlegalized + // ops. if (getConfig().unlegalizedOps) getConfig().unlegalizedOps->erase(op); + + // Notify the listener that the operation (and its nested operations) was + // erased. + if (listener) { + op->walk( + [&](Operation *op) { listener->notifyOperationErased(op); }); + } + // Do not erase the operation yet. It may still be referenced in `mapping`. + // Just unlink it for now and erase it during cleanup. op->getBlock()->getOperations().remove(op); } @@ -1081,7 +1164,9 @@ void ReplaceOperationRewrite::rollback() { rewriterImpl.mapping.erase(result); } -void ReplaceOperationRewrite::cleanup() { eraseOp(op); } +void ReplaceOperationRewrite::cleanup(RewriterBase &rewriter) { + rewriter.eraseOp(op); +} void CreateOperationRewrite::rollback() { for (Region ®ion : op->getRegions()) { @@ -1100,14 +1185,20 @@ void UnresolvedMaterializationRewrite::rollback() { op->erase(); } -void UnresolvedMaterializationRewrite::cleanup() { eraseOp(op); } +void UnresolvedMaterializationRewrite::cleanup(RewriterBase &rewriter) { + rewriter.eraseOp(op); +} void ConversionPatternRewriterImpl::applyRewrites() { // Commit all rewrites. + IRRewriter rewriter(context, config.listener); for (auto &rewrite : rewrites) - rewrite->commit(); + rewrite->commit(rewriter); + + // Clean up all rewrites. + SingleEraseRewriter eraseRewriter(context); for (auto &rewrite : rewrites) - rewrite->cleanup(); + rewrite->cleanup(eraseRewriter); } //===----------------------------------------------------------------------===// @@ -1306,8 +1397,21 @@ Block *ConversionPatternRewriterImpl::applySignatureConversion( Block *newBlock = rewriter.createBlock(block->getParent(), std::next(block->getIterator()), convertedTypes, newLocs); - appendRewrite(newBlock, block, newBlock->end()); - newBlock->getOperations().splice(newBlock->end(), block->getOperations()); + + // If a listener is attached to the dialect conversion, ops cannot be moved + // to the destination block in bulk ("fast path"). This is because at the time + // the notifications are sent, it is unknown which ops were moved. Instead, + // ops should be moved one-by-one ("slow path"), so that a separate + // `MoveOperationRewrite` is enqueued for each moved op. Moving ops in bulk is + // a bit more efficient, so we try to do that when possible. + bool fastPath = !config.listener; + if (fastPath) { + appendRewrite(newBlock, block, newBlock->end()); + newBlock->getOperations().splice(newBlock->end(), block->getOperations()); + } else { + while (!block->empty()) + rewriter.moveOpBefore(&block->front(), newBlock, newBlock->end()); + } // Replace all uses of the old block with the new block. block->replaceAllUsesWith(newBlock); @@ -1645,10 +1749,31 @@ void ConversionPatternRewriter::inlineBlockBefore(Block *source, Block *dest, "expected 'source' to have no predecessors"); #endif // NDEBUG - impl->notifyBlockBeingInlined(dest, source, before); + // If a listener is attached to the dialect conversion, ops cannot be moved + // to the destination block in bulk ("fast path"). This is because at the time + // the notifications are sent, it is unknown which ops were moved. Instead, + // ops should be moved one-by-one ("slow path"), so that a separate + // `MoveOperationRewrite` is enqueued for each moved op. Moving ops in bulk is + // a bit more efficient, so we try to do that when possible. + bool fastPath = !impl->config.listener; + + if (fastPath) + impl->notifyBlockBeingInlined(dest, source, before); + + // Replace all uses of block arguments. for (auto it : llvm::zip(source->getArguments(), argValues)) replaceUsesOfBlockArgument(std::get<0>(it), std::get<1>(it)); - dest->getOperations().splice(before, source->getOperations()); + + if (fastPath) { + // Move all ops at once. + dest->getOperations().splice(before, source->getOperations()); + } else { + // Move op by op. + while (!source->empty()) + moveOpBefore(&source->front(), dest, before); + } + + // Erase the source block. eraseBlock(source); } diff --git a/mlir/test/Transforms/test-legalizer.mlir b/mlir/test/Transforms/test-legalizer.mlir index ccdc9fe78ea0..d552f0346644 100644 --- a/mlir/test/Transforms/test-legalizer.mlir +++ b/mlir/test/Transforms/test-legalizer.mlir @@ -1,5 +1,10 @@ // RUN: mlir-opt -allow-unregistered-dialect -split-input-file -test-legalize-patterns -verify-diagnostics %s | FileCheck %s +// CHECK: notifyOperationInserted: test.legal_op_a, was unlinked +// CHECK-NEXT: notifyOperationReplaced: test.illegal_op_a +// CHECK-NEXT: notifyOperationModified: func.return +// CHECK-NEXT: notifyOperationErased: test.illegal_op_a + // CHECK-LABEL: verifyDirectPattern func.func @verifyDirectPattern() -> i32 { // CHECK-NEXT: "test.legal_op_a"() <{status = "Success"} @@ -8,6 +13,16 @@ func.func @verifyDirectPattern() -> i32 { return %result : i32 } +// ----- + +// CHECK: notifyOperationInserted: test.illegal_op_e, was unlinked +// CHECK-NEXT: notifyOperationReplaced: test.illegal_op_c +// CHECK-NEXT: notifyOperationModified: func.return +// CHECK-NEXT: notifyOperationErased: test.illegal_op_c +// CHECK-NEXT: notifyOperationInserted: test.legal_op_a, was unlinked +// CHECK-NEXT: notifyOperationReplaced: test.illegal_op_e +// CHECK-NEXT: notifyOperationErased: test.illegal_op_e + // CHECK-LABEL: verifyLargerBenefit func.func @verifyLargerBenefit() -> i32 { // CHECK-NEXT: "test.legal_op_a"() <{status = "Success"} @@ -16,16 +31,24 @@ func.func @verifyLargerBenefit() -> i32 { return %result : i32 } +// ----- + +// CHECK: notifyOperationModified: func.func +// Note: No block insertion because this function is external and no block +// signature conversion is performed. + // CHECK-LABEL: func private @remap_input_1_to_0() func.func private @remap_input_1_to_0(i16) +// ----- + // CHECK-LABEL: func @remap_input_1_to_1(%arg0: f64) func.func @remap_input_1_to_1(%arg0: i64) { // CHECK-NEXT: "test.valid"{{.*}} : (f64) "test.invalid"(%arg0) : (i64) -> () } -// CHECK-LABEL: func @remap_call_1_to_1(%arg0: f64) +// CHECK: func @remap_call_1_to_1(%arg0: f64) func.func @remap_call_1_to_1(%arg0: i64) { // CHECK-NEXT: call @remap_input_1_to_1(%arg0) : (f64) -> () call @remap_input_1_to_1(%arg0) : (i64) -> () @@ -33,12 +56,36 @@ func.func @remap_call_1_to_1(%arg0: i64) { return } +// ----- + +// Block signature conversion: new block is inserted. +// CHECK: notifyBlockInserted into func.func: was unlinked + +// Contents of the old block are moved to the new block. +// CHECK-NEXT: notifyOperationInserted: test.return, was linked, exact position unknown + +// The new block arguments are used in "test.return". +// CHECK-NEXT: notifyOperationModified: test.return + +// The old block is erased. +// CHECK-NEXT: notifyBlockErased + +// The function op gets a new type attribute. +// CHECK-NEXT: notifyOperationModified: func.func + +// "test.return" is replaced. +// CHECK-NEXT: notifyOperationInserted: test.return, was unlinked +// CHECK-NEXT: notifyOperationReplaced: test.return +// CHECK-NEXT: notifyOperationErased: test.return + // CHECK-LABEL: func @remap_input_1_to_N({{.*}}f16, {{.*}}f16) func.func @remap_input_1_to_N(%arg0: f32) -> f32 { // CHECK-NEXT: "test.return"{{.*}} : (f16, f16) -> () "test.return"(%arg0) : (f32) -> () } +// ----- + // CHECK-LABEL: func @remap_input_1_to_N_remaining_use(%arg0: f16, %arg1: f16) func.func @remap_input_1_to_N_remaining_use(%arg0: f32) { // CHECK-NEXT: [[CAST:%.*]] = "test.cast"(%arg0, %arg1) : (f16, f16) -> f32 @@ -54,6 +101,8 @@ func.func @remap_materialize_1_to_1(%arg0: i42) { "test.return"(%arg0) : (i42) -> () } +// ----- + // CHECK-LABEL: func @remap_input_to_self func.func @remap_input_to_self(%arg0: index) { // CHECK-NOT: test.cast @@ -68,6 +117,8 @@ func.func @remap_multi(%arg0: i64, %unused: i16, %arg1: i64) -> (i64, i64) { "test.invalid"(%arg0, %arg1) : (i64, i64) -> () } +// ----- + // CHECK-LABEL: func @no_remap_nested func.func @no_remap_nested() { // CHECK-NEXT: "foo.region" @@ -82,6 +133,8 @@ func.func @no_remap_nested() { return } +// ----- + // CHECK-LABEL: func @remap_moved_region_args func.func @remap_moved_region_args() { // CHECK-NEXT: return @@ -96,6 +149,8 @@ func.func @remap_moved_region_args() { return } +// ----- + // CHECK-LABEL: func @remap_cloned_region_args func.func @remap_cloned_region_args() { // CHECK-NEXT: return @@ -122,6 +177,8 @@ func.func @remap_drop_region() { return } +// ----- + // CHECK-LABEL: func @dropped_input_in_use func.func @dropped_input_in_use(%arg: i16, %arg2: i64) { // CHECK-NEXT: "test.cast"{{.*}} : () -> i16 @@ -130,6 +187,8 @@ func.func @dropped_input_in_use(%arg: i16, %arg2: i64) { "work"(%arg) : (i16) -> () } +// ----- + // CHECK-LABEL: func @up_to_date_replacement func.func @up_to_date_replacement(%arg: i8) -> i8 { // CHECK-NEXT: return @@ -139,6 +198,8 @@ func.func @up_to_date_replacement(%arg: i8) -> i8 { return %repl_2 : i8 } +// ----- + // CHECK-LABEL: func @remove_foldable_op // CHECK-SAME: (%[[ARG_0:[a-z0-9]*]]: i32) func.func @remove_foldable_op(%arg0 : i32) -> (i32) { @@ -150,6 +211,8 @@ func.func @remove_foldable_op(%arg0 : i32) -> (i32) { return %0 : i32 } +// ----- + // CHECK-LABEL: @create_block func.func @create_block() { // Check that we created a block with arguments. @@ -161,6 +224,12 @@ func.func @create_block() { return } +// ----- + +// CHECK: notifyOperationModified: test.recursive_rewrite +// CHECK-NEXT: notifyOperationModified: test.recursive_rewrite +// CHECK-NEXT: notifyOperationModified: test.recursive_rewrite + // CHECK-LABEL: @bounded_recursion func.func @bounded_recursion() { // CHECK: test.recursive_rewrite 0 diff --git a/mlir/test/lib/Dialect/Test/TestPatterns.cpp b/mlir/test/lib/Dialect/Test/TestPatterns.cpp index 27eae2ffd694..2da184bc3d85 100644 --- a/mlir/test/lib/Dialect/Test/TestPatterns.cpp +++ b/mlir/test/lib/Dialect/Test/TestPatterns.cpp @@ -327,8 +327,12 @@ struct TestPatternDriver struct DumpNotifications : public RewriterBase::Listener { void notifyBlockInserted(Block *block, Region *previous, Region::iterator previousIt) override { - llvm::outs() << "notifyBlockInserted into " - << block->getParentOp()->getName() << ": "; + llvm::outs() << "notifyBlockInserted"; + if (block->getParentOp()) { + llvm::outs() << " into " << block->getParentOp()->getName() << ": "; + } else { + llvm::outs() << " into unknown op: "; + } if (previous == nullptr) { llvm::outs() << "was unlinked\n"; } else { @@ -341,7 +345,9 @@ struct DumpNotifications : public RewriterBase::Listener { if (!previous.isSet()) { llvm::outs() << ", was unlinked\n"; } else { - if (previous.getPoint() == previous.getBlock()->end()) { + if (!previous.getPoint().getNodePtr()) { + llvm::outs() << ", was linked, exact position unknown\n"; + } else if (previous.getPoint() == previous.getBlock()->end()) { llvm::outs() << ", was last in block\n"; } else { llvm::outs() << ", previous = " << previous.getPoint()->getName() @@ -349,9 +355,18 @@ struct DumpNotifications : public RewriterBase::Listener { } } } + void notifyBlockErased(Block *block) override { + llvm::outs() << "notifyBlockErased\n"; + } void notifyOperationErased(Operation *op) override { llvm::outs() << "notifyOperationErased: " << op->getName() << "\n"; } + void notifyOperationModified(Operation *op) override { + llvm::outs() << "notifyOperationModified: " << op->getName() << "\n"; + } + void notifyOperationReplaced(Operation *op, ValueRange values) override { + llvm::outs() << "notifyOperationReplaced: " << op->getName() << "\n"; + } }; struct TestStrictPatternDriver @@ -1153,6 +1168,8 @@ struct TestLegalizePatternDriver if (mode == ConversionMode::Partial) { DenseSet unlegalizedOps; ConversionConfig config; + DumpNotifications dumpNotifications; + config.listener = &dumpNotifications; config.unlegalizedOps = &unlegalizedOps; if (failed(applyPartialConversion(getOperation(), target, std::move(patterns), config))) { @@ -1171,8 +1188,11 @@ struct TestLegalizePatternDriver return (bool)op->getAttrOfType("test.dynamically_legal"); }); + ConversionConfig config; + DumpNotifications dumpNotifications; + config.listener = &dumpNotifications; if (failed(applyFullConversion(getOperation(), target, - std::move(patterns)))) { + std::move(patterns), config))) { getOperation()->emitRemark() << "applyFullConversion failed"; } return; -- GitLab From 474a73d979bdab8782c17829d72386e0da39eb39 Mon Sep 17 00:00:00 2001 From: Jie Fu Date: Fri, 8 Mar 2024 09:38:29 +0800 Subject: [PATCH 547/929] [mlir] Fix build failure in MeshShardingInterfaceImpl.cpp (NFC) llvm-project/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp:96:8: error: unused variable 'resultElementType' [-Werror,-Wunused-variable] Type resultElementType = ^ llvm-project/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp:122:1: error: non-void function does not return a value in all control paths [-Werror,-Wreturn-type] } ^ 2 errors generated. --- .../Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp index 7ac45dc3eb3e..146e88076566 100644 --- a/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp +++ b/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp @@ -93,7 +93,7 @@ static ReductionKind getReductionKindOfLinalgOp(LinalgOp op) { if (!reductionOp) { return ReductionKind::Generic; } - Type resultElementType = + [[maybe_unused]] Type resultElementType = llvm::cast(op->getResult(0).getType()).getElementType(); // TODO: handle case when result type of the reduction op does not match the // element type of the result tensor. @@ -119,6 +119,7 @@ static MeshOp getMesh(Operation *op, } assert(false); + return nullptr; } // Choose the operand based on the current process index along the reduction -- GitLab From da4957be2365831c94eab0b52612367c29f1d299 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Thu, 7 Mar 2024 17:28:37 -0800 Subject: [PATCH 548/929] [NFC] [hwasan] use for_each and move comment --- llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp b/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp index 2ffe89a24584..bfe474d82045 100644 --- a/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp +++ b/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp @@ -12,6 +12,7 @@ #include "llvm/Transforms/Utils/MemoryTaggingSupport.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/Analysis/CFG.h" #include "llvm/Analysis/PostDominators.h" #include "llvm/Analysis/StackSafetyAnalysis.h" @@ -69,14 +70,12 @@ bool forAllReachableExits(const DominatorTree &DT, const PostDominatorTree &PDT, ++NumCoveredExits; } } - // If there's a mix of covered and non-covered exits, just put the untag - // on exits, so we avoid the redundancy of untagging twice. if (NumCoveredExits == ReachableRetVec.size()) { - for (auto *End : Ends) - Callback(End); + for_each(Ends, Callback); } else { - for (auto *RI : ReachableRetVec) - Callback(RI); + // If there's a mix of covered and non-covered exits, just put the untag + // on exits, so we avoid the redundancy of untagging twice. + for_each(ReachableRetVec, Callback); // We may have inserted untag outside of the lifetime interval. // Signal the caller to remove the lifetime end call for this alloca. return false; -- GitLab From cc34e56b865f1fc9e894b75fc958f09dff0fcdea Mon Sep 17 00:00:00 2001 From: Chen Zheng Date: Thu, 7 Mar 2024 20:51:47 -0500 Subject: [PATCH 549/929] [PPC][NFC] add an option to expose the bug in 74951 --- llvm/test/CodeGen/PowerPC/pr74951.ll | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/llvm/test/CodeGen/PowerPC/pr74951.ll b/llvm/test/CodeGen/PowerPC/pr74951.ll index a0d19fc09cc2..c1b2e3ee0dd6 100644 --- a/llvm/test/CodeGen/PowerPC/pr74951.ll +++ b/llvm/test/CodeGen/PowerPC/pr74951.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 -; RUN: llc < %s -verify-machineinstrs -ppc-asm-full-reg-names -mtriple=powerpc64-ibm-aix-xcoff | FileCheck %s +; RUN: llc < %s -start-before=codegenprepare -verify-machineinstrs -ppc-asm-full-reg-names \ +; RUN: -mtriple=powerpc64-ibm-aix-xcoff | FileCheck %s %struct.anon = type { i32 } @@ -11,26 +12,28 @@ define noundef signext i32 @main() { ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: ld r3, L..C0(r2) # @b ; CHECK-NEXT: lwz r3, 0(r3) -; CHECK-NEXT: extsw r4, r3 -; CHECK-NEXT: neg r4, r4 -; CHECK-NEXT: andi. r5, r3, 65535 -; CHECK-NEXT: rldicl r4, r4, 1, 63 +; CHECK-NEXT: andi. r4, r3, 65535 ; CHECK-NEXT: bne cr0, L..BB0_4 ; CHECK-NEXT: # %bb.1: # %lor.rhs.i.i -; CHECK-NEXT: xori r5, r4, 1 -; CHECK-NEXT: cmpw r3, r5 +; CHECK-NEXT: extsw r4, r3 +; CHECK-NEXT: neg r5, r4 +; CHECK-NEXT: rldicl r5, r5, 1, 63 +; CHECK-NEXT: xori r5, r5, 1 +; CHECK-NEXT: cmpw r4, r5 ; CHECK-NEXT: crnot 4*cr5+lt, eq -; CHECK-NEXT: li r3, 1 +; CHECK-NEXT: li r4, 1 ; CHECK-NEXT: bc 12, 4*cr5+lt, L..BB0_3 ; CHECK-NEXT: # %bb.2: # %lor.rhs.i.i -; CHECK-NEXT: li r3, 0 +; CHECK-NEXT: li r4, 0 ; CHECK-NEXT: L..BB0_3: # %lor.rhs.i.i ; CHECK-NEXT: ld r5, L..C1(r2) # @g -; CHECK-NEXT: stb r3, 0(r5) +; CHECK-NEXT: stb r4, 0(r5) ; CHECK-NEXT: L..BB0_4: # %g.exit -; CHECK-NEXT: ld r5, L..C1(r2) # @g +; CHECK-NEXT: ld r4, L..C1(r2) # @g +; CHECK-NEXT: neg r3, r3 +; CHECK-NEXT: rldicl r5, r3, 1, 63 ; CHECK-NEXT: li r3, 0 -; CHECK-NEXT: stb r4, 0(r5) +; CHECK-NEXT: stb r5, 0(r4) ; CHECK-NEXT: blr entry: %0 = load i32, ptr @b, align 4 -- GitLab From da00c60dae0040185dc45039c4397f6e746548e9 Mon Sep 17 00:00:00 2001 From: Chuanqi Xu Date: Fri, 8 Mar 2024 10:12:51 +0800 Subject: [PATCH 550/929] [C++20] [Modules] Introduce reduced BMI (#75894) Close https://github.com/llvm/llvm-project/issues/71034 See https://discourse.llvm.org/t/rfc-c-20-modules-introduce-thin-bmi-and-decls-hash/74755 This patch introduces reduced BMI, which doesn't contain the definitions of functions and variables if its definitions won't contribute to the ABI. Testing is a big part of the patch. We want to make sure the reduced BMI contains the same behavior with the existing and relatively stable fatBMI. This is pretty helpful for further reduction. The user interfaces part it left to following patches to ease the reviewing. --- clang/include/clang/Driver/Options.td | 4 +- .../include/clang/Frontend/FrontendActions.h | 15 ++++++- .../include/clang/Frontend/FrontendOptions.h | 6 ++- clang/include/clang/Serialization/ASTWriter.h | 32 ++++++++++++- clang/lib/Frontend/CompilerInvocation.cpp | 3 ++ clang/lib/Frontend/FrontendActions.cpp | 37 ++++++++++++--- .../ExecuteCompilerInvocation.cpp | 2 + clang/lib/Serialization/ASTWriter.cpp | 32 +++++++------ clang/lib/Serialization/ASTWriterDecl.cpp | 45 ++++++++++++++++--- clang/lib/Serialization/GeneratePCH.cpp | 37 ++++++++++++++- clang/test/CXX/basic/basic.link/p10-ex2.cpp | 2 + .../p4-friend-in-reachable-class.cpp | 5 ++- .../test/Modules/InheritDefaultArguments.cppm | 3 ++ clang/test/Modules/Reachability-Private.cpp | 10 +++++ .../Modules/Reachability-func-default-arg.cpp | 3 ++ clang/test/Modules/Reachability-func-ret.cpp | 3 ++ .../Reachability-template-default-arg.cpp | 3 ++ .../Reachability-template-instantiation.cpp | 4 ++ .../Modules/Reachability-using-templates.cpp | 3 ++ clang/test/Modules/Reachability-using.cpp | 3 ++ clang/test/Modules/concept.cppm | 1 - clang/test/Modules/concept_differ.cppm | 5 +++ clang/test/Modules/ctor.arg.dep.cppm | 4 ++ clang/test/Modules/cxx20-10-1-ex1.cpp | 16 +++++++ clang/test/Modules/cxx20-10-1-ex2.cpp | 36 ++++++++++++--- clang/test/Modules/cxx20-10-2-ex2.cpp | 12 +++++ clang/test/Modules/cxx20-10-2-ex5.cpp | 12 +++++ clang/test/Modules/cxx20-10-3-ex1.cpp | 14 ++++++ clang/test/Modules/cxx20-10-3-ex2.cpp | 10 +++++ clang/test/Modules/cxx20-10-5-ex1.cpp | 12 +++++ .../Modules/cxx20-import-diagnostics-a.cpp | 39 ++++++++++++++++ .../Modules/cxx20-import-diagnostics-b.cpp | 25 +++++++++++ .../Modules/cxx20-module-file-info-macros.cpp | 3 ++ clang/test/Modules/deduction-guide.cppm | 3 ++ clang/test/Modules/deduction-guide2.cppm | 3 ++ clang/test/Modules/deduction-guide3.cppm | 3 ++ clang/test/Modules/derived_class.cpp | 3 ++ ...duplicated-module-file-eq-module-name.cppm | 4 ++ clang/test/Modules/enum-class.cppm | 3 ++ .../explicitly-specialized-template.cpp | 3 ++ .../test/Modules/export-language-linkage.cppm | 8 +++- clang/test/Modules/ftime-trace.cppm | 9 ++++ .../inconsistent-deduction-guide-linkage.cppm | 6 +++ clang/test/Modules/inconsistent-export.cppm | 13 ++++++ clang/test/Modules/inherited_arg.cppm | 8 ++++ .../Modules/instantiation-argdep-lookup.cppm | 3 ++ clang/test/Modules/lambdas.cppm | 15 +++++++ .../Modules/merge-concepts-cxx-modules.cpp | 12 +++++ .../Modules/merge-constrained-friends.cpp | 3 ++ clang/test/Modules/merge-lambdas.cppm | 4 ++ .../Modules/merge-requires-with-lambdas.cppm | 19 ++++++++ .../merge-var-template-spec-cxx-modules.cppm | 5 +++ clang/test/Modules/mismatch-diagnostics.cpp | 11 +++++ .../module-init-duplicated-import.cppm | 11 +++++ clang/test/Modules/named-modules-adl-2.cppm | 4 ++ clang/test/Modules/named-modules-adl-3.cppm | 14 ++++++ clang/test/Modules/named-modules-adl.cppm | 3 ++ .../Modules/no-duplicate-codegen-in-GMF.cppm | 10 +++++ clang/test/Modules/pair-unambiguous-ctor.cppm | 9 ++++ .../test/Modules/partial_specialization.cppm | 3 ++ .../test/Modules/placement-new-reachable.cpp | 3 ++ clang/test/Modules/polluted-operator.cppm | 3 ++ clang/test/Modules/pr54457.cppm | 3 ++ clang/test/Modules/pr56916.cppm | 12 +++++ clang/test/Modules/pr58532.cppm | 6 +++ clang/test/Modules/pr58716.cppm | 2 +- clang/test/Modules/pr59719.cppm | 3 ++ clang/test/Modules/pr59780.cppm | 10 +++++ clang/test/Modules/pr59999.cppm | 13 ++++++ clang/test/Modules/pr60036.cppm | 14 ++++++ clang/test/Modules/pr60085.cppm | 17 +++++++ clang/test/Modules/pr60275.cppm | 7 ++- clang/test/Modules/pr60486.cppm | 3 ++ clang/test/Modules/pr60693.cppm | 4 ++ clang/test/Modules/pr60775.cppm | 13 ++++++ clang/test/Modules/pr60890.cppm | 6 +++ clang/test/Modules/pr61065.cppm | 13 ++++++ clang/test/Modules/pr61065_2.cppm | 15 +++++++ clang/test/Modules/pr61067.cppm | 14 ++++++ clang/test/Modules/pr61317.cppm | 9 ++++ clang/test/Modules/pr61783.cppm | 8 ++++ clang/test/Modules/pr61892.cppm | 40 ++++++++--------- clang/test/Modules/pr62158.cppm | 9 ++++ clang/test/Modules/pr62359.cppm | 16 +++++++ clang/test/Modules/pr62589.cppm | 3 ++ clang/test/Modules/pr62705.cppm | 8 ++++ clang/test/Modules/pr62796.cppm | 4 ++ clang/test/Modules/pr62943.cppm | 12 +++++ clang/test/Modules/pr63544.cppm | 12 +++++ clang/test/Modules/pr63595.cppm | 10 +++++ clang/test/Modules/pr67627.cppm | 4 ++ clang/test/Modules/pr67893.cppm | 9 ++++ clang/test/Modules/predefined.cpp | 3 ++ clang/test/Modules/preferred_name.cppm | 10 +++++ clang/test/Modules/redefinition-merges.cppm | 6 +++ .../redundant-template-default-arg.cpp | 3 ++ .../redundant-template-default-arg2.cpp | 3 ++ .../redundant-template-default-arg3.cpp | 3 ++ clang/test/Modules/search-partitions.cpp | 16 +++++++ ...unction-definition-for-template-class.cppm | 12 +++++ .../template-function-specialization.cpp | 5 ++- clang/test/Modules/template-lambdas.cppm | 15 +++++++ clang/test/Modules/template-pack.cppm | 3 ++ .../Modules/template_default_argument.cpp | 3 ++ clang/unittests/Sema/SemaNoloadLookupTest.cpp | 9 ++-- .../Serialization/ForceCheckFileInputTest.cpp | 10 +++-- .../Serialization/NoCommentsTest.cpp | 9 ++-- .../Serialization/VarDeclConstantInitTest.cpp | 13 +++--- 108 files changed, 977 insertions(+), 84 deletions(-) diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index bef38738fde8..5b3d366dbcf9 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -7414,7 +7414,9 @@ def ast_view : Flag<["-"], "ast-view">, def emit_module : Flag<["-"], "emit-module">, HelpText<"Generate pre-compiled module file from a module map">; def emit_module_interface : Flag<["-"], "emit-module-interface">, - HelpText<"Generate pre-compiled module file from a C++ module interface">; + HelpText<"Generate pre-compiled module file from a standard C++ module interface unit">; +def emit_reduced_module_interface : Flag<["-"], "emit-reduced-module-interface">, + HelpText<"Generate reduced prebuilt module interface from a standard C++ module interface unit">; def emit_header_unit : Flag<["-"], "emit-header-unit">, HelpText<"Generate C++20 header units from header files">; def emit_pch : Flag<["-"], "emit-pch">, diff --git a/clang/include/clang/Frontend/FrontendActions.h b/clang/include/clang/Frontend/FrontendActions.h index fcce31ac0590..8441af2ee3e7 100644 --- a/clang/include/clang/Frontend/FrontendActions.h +++ b/clang/include/clang/Frontend/FrontendActions.h @@ -118,6 +118,9 @@ class GenerateModuleAction : public ASTFrontendAction { CreateOutputFile(CompilerInstance &CI, StringRef InFile) = 0; protected: + std::vector> + CreateMultiplexConsumer(CompilerInstance &CI, StringRef InFile); + std::unique_ptr CreateASTConsumer(CompilerInstance &CI, StringRef InFile) override; @@ -147,8 +150,10 @@ private: CreateOutputFile(CompilerInstance &CI, StringRef InFile) override; }; +/// Generates full BMI (which contains full information to generate the object +/// files) for C++20 Named Modules. class GenerateModuleInterfaceAction : public GenerateModuleAction { -private: +protected: bool BeginSourceFileAction(CompilerInstance &CI) override; std::unique_ptr CreateASTConsumer(CompilerInstance &CI, @@ -158,6 +163,14 @@ private: CreateOutputFile(CompilerInstance &CI, StringRef InFile) override; }; +/// Only generates the reduced BMI. This action is mainly used by tests. +class GenerateReducedModuleInterfaceAction + : public GenerateModuleInterfaceAction { +private: + std::unique_ptr CreateASTConsumer(CompilerInstance &CI, + StringRef InFile) override; +}; + class GenerateHeaderUnitAction : public GenerateModuleAction { private: diff --git a/clang/include/clang/Frontend/FrontendOptions.h b/clang/include/clang/Frontend/FrontendOptions.h index 53a8681cfdbb..8085dbcbf671 100644 --- a/clang/include/clang/Frontend/FrontendOptions.h +++ b/clang/include/clang/Frontend/FrontendOptions.h @@ -85,9 +85,13 @@ enum ActionKind { /// Generate pre-compiled module from a module map. GenerateModule, - /// Generate pre-compiled module from a C++ module interface file. + /// Generate pre-compiled module from a standard C++ module interface unit. GenerateModuleInterface, + /// Generate reduced module interface for a standard C++ module interface + /// unit. + GenerateReducedModuleInterface, + /// Generate a C++20 header unit module from a header file. GenerateHeaderUnit, diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h index 5e2f305b294c..e5db486a71a4 100644 --- a/clang/include/clang/Serialization/ASTWriter.h +++ b/clang/include/clang/Serialization/ASTWriter.h @@ -166,6 +166,10 @@ private: /// Indicates that the AST contained compiler errors. bool ASTHasCompilerErrors = false; + /// Indicates that we're going to generate the reduced BMI for C++20 + /// named modules. + bool GeneratingReducedBMI = false; + /// Mapping from input file entries to the index into the /// offset table where information about that input file is stored. llvm::DenseMap InputFileIDs; @@ -596,7 +600,8 @@ public: ASTWriter(llvm::BitstreamWriter &Stream, SmallVectorImpl &Buffer, InMemoryModuleCache &ModuleCache, ArrayRef> Extensions, - bool IncludeTimestamps = true, bool BuildingImplicitModule = false); + bool IncludeTimestamps = true, bool BuildingImplicitModule = false, + bool GeneratingReducedBMI = false); ~ASTWriter() override; ASTContext &getASTContext() const { @@ -856,6 +861,13 @@ protected: const ASTWriter &getWriter() const { return Writer; } SmallVectorImpl &getPCH() const { return Buffer->Data; } + bool isComplete() const { return Buffer->IsComplete; } + PCHBuffer *getBufferPtr() { return Buffer.get(); } + StringRef getOutputFile() const { return OutputFile; } + DiagnosticsEngine &getDiagnostics() const { + return SemaPtr->getDiagnostics(); + } + public: PCHGenerator(const Preprocessor &PP, InMemoryModuleCache &ModuleCache, StringRef OutputFile, StringRef isysroot, @@ -863,7 +875,8 @@ public: ArrayRef> Extensions, bool AllowASTWithErrors = false, bool IncludeTimestamps = true, bool BuildingImplicitModule = false, - bool ShouldCacheASTInMemory = false); + bool ShouldCacheASTInMemory = false, + bool GeneratingReducedBMI = false); ~PCHGenerator() override; void InitializeSema(Sema &S) override { SemaPtr = &S; } @@ -873,6 +886,21 @@ public: bool hasEmittedPCH() const { return Buffer->IsComplete; } }; +class ReducedBMIGenerator : public PCHGenerator { +public: + ReducedBMIGenerator(const Preprocessor &PP, InMemoryModuleCache &ModuleCache, + StringRef OutputFile, std::shared_ptr Buffer, + bool IncludeTimestamps); + + void HandleTranslationUnit(ASTContext &Ctx) override; +}; + +/// If we can elide the definition of \param D in reduced BMI. +/// +/// Generally, we can elide the definition of a declaration if it won't affect +/// the ABI. e.g., the non-inline function bodies. +bool CanElideDeclDef(const Decl *D); + /// A simple helper class to pack several bits in order into (a) 32 bit /// integer(s). class BitsPacker { diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index 691f3b989b81..451bdb9386f5 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -2556,6 +2556,8 @@ static const auto &getFrontendActionTable() { {frontend::GenerateModule, OPT_emit_module}, {frontend::GenerateModuleInterface, OPT_emit_module_interface}, + {frontend::GenerateReducedModuleInterface, + OPT_emit_reduced_module_interface}, {frontend::GenerateHeaderUnit, OPT_emit_header_unit}, {frontend::GeneratePCH, OPT_emit_pch}, {frontend::GenerateInterfaceStubs, OPT_emit_interface_stubs}, @@ -4280,6 +4282,7 @@ static bool isStrictlyPreprocessorAction(frontend::ActionKind Action) { case frontend::FixIt: case frontend::GenerateModule: case frontend::GenerateModuleInterface: + case frontend::GenerateReducedModuleInterface: case frontend::GenerateHeaderUnit: case frontend::GeneratePCH: case frontend::GenerateInterfaceStubs: diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp index b9ed5dedfa42..cd9b9923421c 100644 --- a/clang/lib/Frontend/FrontendActions.cpp +++ b/clang/lib/Frontend/FrontendActions.cpp @@ -184,12 +184,12 @@ bool GeneratePCHAction::BeginSourceFileAction(CompilerInstance &CI) { return true; } -std::unique_ptr -GenerateModuleAction::CreateASTConsumer(CompilerInstance &CI, - StringRef InFile) { +std::vector> +GenerateModuleAction::CreateMultiplexConsumer(CompilerInstance &CI, + StringRef InFile) { std::unique_ptr OS = CreateOutputFile(CI, InFile); if (!OS) - return nullptr; + return {}; std::string OutputFile = CI.getFrontendOpts().OutputFile; std::string Sysroot; @@ -210,6 +210,17 @@ GenerateModuleAction::CreateASTConsumer(CompilerInstance &CI, +CI.getFrontendOpts().BuildingImplicitModule)); Consumers.push_back(CI.getPCHContainerWriter().CreatePCHContainerGenerator( CI, std::string(InFile), OutputFile, std::move(OS), Buffer)); + return std::move(Consumers); +} + +std::unique_ptr +GenerateModuleAction::CreateASTConsumer(CompilerInstance &CI, + StringRef InFile) { + std::vector> Consumers = + CreateMultiplexConsumer(CI, InFile); + if (Consumers.empty()) + return nullptr; + return std::make_unique(std::move(Consumers)); } @@ -265,7 +276,12 @@ GenerateModuleInterfaceAction::CreateASTConsumer(CompilerInstance &CI, CI.getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true; CI.getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings = true; - return GenerateModuleAction::CreateASTConsumer(CI, InFile); + std::vector> Consumers = + CreateMultiplexConsumer(CI, InFile); + if (Consumers.empty()) + return nullptr; + + return std::make_unique(std::move(Consumers)); } std::unique_ptr @@ -274,6 +290,16 @@ GenerateModuleInterfaceAction::CreateOutputFile(CompilerInstance &CI, return CI.createDefaultOutputFile(/*Binary=*/true, InFile, "pcm"); } +std::unique_ptr +GenerateReducedModuleInterfaceAction::CreateASTConsumer(CompilerInstance &CI, + StringRef InFile) { + auto Buffer = std::make_shared(); + return std::make_unique( + CI.getPreprocessor(), CI.getModuleCache(), + CI.getFrontendOpts().OutputFile, Buffer, + /*IncludeTimestamps=*/+CI.getFrontendOpts().IncludeTimestamps); +} + bool GenerateHeaderUnitAction::BeginSourceFileAction(CompilerInstance &CI) { if (!CI.getLangOpts().CPlusPlusModules) { CI.getDiagnostics().Report(diag::err_module_interface_requires_cpp_modules); @@ -839,7 +865,6 @@ void DumpModuleInfoAction::ExecuteAction() { const LangOptions &LO = getCurrentASTUnit().getLangOpts(); if (LO.CPlusPlusModules && !LO.CurrentModule.empty()) { - ASTReader *R = getCurrentASTUnit().getASTReader().get(); unsigned SubModuleCount = R->getTotalNumSubmodules(); serialization::ModuleFile &MF = R->getModuleManager().getPrimaryModule(); diff --git a/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp b/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp index 925879a68cbd..2446aee571f4 100644 --- a/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp +++ b/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp @@ -65,6 +65,8 @@ CreateFrontendBaseAction(CompilerInstance &CI) { return std::make_unique(); case GenerateModuleInterface: return std::make_unique(); + case GenerateReducedModuleInterface: + return std::make_unique(); case GenerateHeaderUnit: return std::make_unique(); case GeneratePCH: return std::make_unique(); diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index a9edc7e68b53..6904c924c2fd 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -4623,10 +4623,12 @@ ASTWriter::ASTWriter(llvm::BitstreamWriter &Stream, SmallVectorImpl &Buffer, InMemoryModuleCache &ModuleCache, ArrayRef> Extensions, - bool IncludeTimestamps, bool BuildingImplicitModule) + bool IncludeTimestamps, bool BuildingImplicitModule, + bool GeneratingReducedBMI) : Stream(Stream), Buffer(Buffer), ModuleCache(ModuleCache), IncludeTimestamps(IncludeTimestamps), - BuildingImplicitModule(BuildingImplicitModule) { + BuildingImplicitModule(BuildingImplicitModule), + GeneratingReducedBMI(GeneratingReducedBMI) { for (const auto &Ext : Extensions) { if (auto Writer = Ext->createExtensionWriter(*this)) ModuleFileExtensionWriters.push_back(std::move(Writer)); @@ -5457,18 +5459,20 @@ void ASTWriter::WriteDeclUpdatesBlocks(RecordDataImpl &OffsetsRecord) { // Add a trailing update record, if any. These must go last because we // lazily load their attached statement. - if (HasUpdatedBody) { - const auto *Def = cast(D); - Record.push_back(UPD_CXX_ADDED_FUNCTION_DEFINITION); - Record.push_back(Def->isInlined()); - Record.AddSourceLocation(Def->getInnerLocStart()); - Record.AddFunctionDefinition(Def); - } else if (HasAddedVarDefinition) { - const auto *VD = cast(D); - Record.push_back(UPD_CXX_ADDED_VAR_DEFINITION); - Record.push_back(VD->isInline()); - Record.push_back(VD->isInlineSpecified()); - Record.AddVarDeclInit(VD); + if (!GeneratingReducedBMI || !CanElideDeclDef(D)) { + if (HasUpdatedBody) { + const auto *Def = cast(D); + Record.push_back(UPD_CXX_ADDED_FUNCTION_DEFINITION); + Record.push_back(Def->isInlined()); + Record.AddSourceLocation(Def->getInnerLocStart()); + Record.AddFunctionDefinition(Def); + } else if (HasAddedVarDefinition) { + const auto *VD = cast(D); + Record.push_back(UPD_CXX_ADDED_VAR_DEFINITION); + Record.push_back(VD->isInline()); + Record.push_back(VD->isInlineSpecified()); + Record.AddVarDeclInit(VD); + } } OffsetsRecord.push_back(GetDeclRef(D)); diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp index e73800100e3c..e1862de4a35b 100644 --- a/clang/lib/Serialization/ASTWriterDecl.cpp +++ b/clang/lib/Serialization/ASTWriterDecl.cpp @@ -16,6 +16,7 @@ #include "clang/AST/DeclTemplate.h" #include "clang/AST/DeclVisitor.h" #include "clang/AST/Expr.h" +#include "clang/AST/ODRHash.h" #include "clang/AST/OpenMPClause.h" #include "clang/AST/PrettyDeclStackTrace.h" #include "clang/Basic/SourceManager.h" @@ -40,11 +41,14 @@ namespace clang { serialization::DeclCode Code; unsigned AbbrevToUse; + bool GeneratingReducedBMI = false; + public: ASTDeclWriter(ASTWriter &Writer, ASTContext &Context, - ASTWriter::RecordDataImpl &Record) + ASTWriter::RecordDataImpl &Record, bool GeneratingReducedBMI) : Writer(Writer), Context(Context), Record(Writer, Record), - Code((serialization::DeclCode)0), AbbrevToUse(0) {} + Code((serialization::DeclCode)0), AbbrevToUse(0), + GeneratingReducedBMI(GeneratingReducedBMI) {} uint64_t Emit(Decl *D) { if (!Code) @@ -270,6 +274,27 @@ namespace clang { }; } +bool clang::CanElideDeclDef(const Decl *D) { + if (auto *FD = dyn_cast(D)) { + if (FD->isInlined() || FD->isConstexpr()) + return false; + + if (FD->isDependentContext()) + return false; + } + + if (auto *VD = dyn_cast(D)) { + if (!VD->getDeclContext()->getRedeclContext()->isFileContext() || + VD->isInline() || VD->isConstexpr() || isa(VD)) + return false; + + if (VD->getTemplateSpecializationKind() == TSK_ImplicitInstantiation) + return false; + } + + return true; +} + void ASTDeclWriter::Visit(Decl *D) { DeclVisitor::Visit(D); @@ -285,9 +310,12 @@ void ASTDeclWriter::Visit(Decl *D) { // have been written. We want it last because we will not read it back when // retrieving it from the AST, we'll just lazily set the offset. if (auto *FD = dyn_cast(D)) { - Record.push_back(FD->doesThisDeclarationHaveABody()); - if (FD->doesThisDeclarationHaveABody()) - Record.AddFunctionDefinition(FD); + if (!GeneratingReducedBMI || !CanElideDeclDef(FD)) { + Record.push_back(FD->doesThisDeclarationHaveABody()); + if (FD->doesThisDeclarationHaveABody()) + Record.AddFunctionDefinition(FD); + } else + Record.push_back(0); } // Similar to FunctionDecls, handle VarDecl's initializer here and write it @@ -295,7 +323,10 @@ void ASTDeclWriter::Visit(Decl *D) { // we have finished recursive deserialization, because it can recursively // refer back to the variable. if (auto *VD = dyn_cast(D)) { - Record.AddVarDeclInit(VD); + if (!GeneratingReducedBMI || !CanElideDeclDef(VD)) + Record.AddVarDeclInit(VD); + else + Record.push_back(0); } // And similarly for FieldDecls. We already serialized whether there is a @@ -2729,7 +2760,7 @@ void ASTWriter::WriteDecl(ASTContext &Context, Decl *D) { assert(ID >= FirstDeclID && "invalid decl ID"); RecordData Record; - ASTDeclWriter W(*this, Context, Record); + ASTDeclWriter W(*this, Context, Record, GeneratingReducedBMI); // Build a record for this declaration W.Visit(D); diff --git a/clang/lib/Serialization/GeneratePCH.cpp b/clang/lib/Serialization/GeneratePCH.cpp index cf8084333811..2b511b2d5a90 100644 --- a/clang/lib/Serialization/GeneratePCH.cpp +++ b/clang/lib/Serialization/GeneratePCH.cpp @@ -12,9 +12,11 @@ //===----------------------------------------------------------------------===// #include "clang/AST/ASTContext.h" +#include "clang/Frontend/FrontendDiagnostic.h" #include "clang/Lex/HeaderSearch.h" #include "clang/Lex/Preprocessor.h" #include "clang/Sema/SemaConsumer.h" +#include "clang/Serialization/ASTReader.h" #include "clang/Serialization/ASTWriter.h" #include "llvm/Bitstream/BitstreamWriter.h" @@ -25,11 +27,12 @@ PCHGenerator::PCHGenerator( StringRef OutputFile, StringRef isysroot, std::shared_ptr Buffer, ArrayRef> Extensions, bool AllowASTWithErrors, bool IncludeTimestamps, - bool BuildingImplicitModule, bool ShouldCacheASTInMemory) + bool BuildingImplicitModule, bool ShouldCacheASTInMemory, + bool GeneratingReducedBMI) : PP(PP), OutputFile(OutputFile), isysroot(isysroot.str()), SemaPtr(nullptr), Buffer(std::move(Buffer)), Stream(this->Buffer->Data), Writer(Stream, this->Buffer->Data, ModuleCache, Extensions, - IncludeTimestamps, BuildingImplicitModule), + IncludeTimestamps, BuildingImplicitModule, GeneratingReducedBMI), AllowASTWithErrors(AllowASTWithErrors), ShouldCacheASTInMemory(ShouldCacheASTInMemory) { this->Buffer->IsComplete = false; @@ -78,3 +81,33 @@ ASTMutationListener *PCHGenerator::GetASTMutationListener() { ASTDeserializationListener *PCHGenerator::GetASTDeserializationListener() { return &Writer; } + +ReducedBMIGenerator::ReducedBMIGenerator(const Preprocessor &PP, + InMemoryModuleCache &ModuleCache, + StringRef OutputFile, + std::shared_ptr Buffer, + bool IncludeTimestamps) + : PCHGenerator( + PP, ModuleCache, OutputFile, llvm::StringRef(), Buffer, + /*Extensions=*/ArrayRef>(), + /*AllowASTWithErrors*/ false, /*IncludeTimestamps=*/IncludeTimestamps, + /*BuildingImplicitModule=*/false, /*ShouldCacheASTInMemory=*/false, + /*GeneratingReducedBMI=*/true) {} + +void ReducedBMIGenerator::HandleTranslationUnit(ASTContext &Ctx) { + PCHGenerator::HandleTranslationUnit(Ctx); + + if (!isComplete()) + return; + + std::error_code EC; + auto OS = std::make_unique(getOutputFile(), EC); + if (EC) { + getDiagnostics().Report(diag::err_fe_unable_to_open_output) + << getOutputFile() << EC.message() << "\n"; + return; + } + + *OS << getBufferPtr()->Data; + OS->flush(); +} diff --git a/clang/test/CXX/basic/basic.link/p10-ex2.cpp b/clang/test/CXX/basic/basic.link/p10-ex2.cpp index 95fdb56f78d6..e985ce37a934 100644 --- a/clang/test/CXX/basic/basic.link/p10-ex2.cpp +++ b/clang/test/CXX/basic/basic.link/p10-ex2.cpp @@ -5,7 +5,9 @@ // // RUN: %clang_cc1 -std=c++20 M.cpp -fsyntax-only -DTEST_INTERFACE -verify // RUN: %clang_cc1 -std=c++20 M.cpp -emit-module-interface -o M.pcm +// RUN: %clang_cc1 -std=c++20 M.cpp -emit-reduced-module-interface -o M.reduced.pcm // RUN: %clang_cc1 -std=c++20 useM.cpp -fsyntax-only -fmodule-file=M=M.pcm -verify +// RUN: %clang_cc1 -std=c++20 useM.cpp -fsyntax-only -fmodule-file=M=M.reduced.pcm -verify //--- decls.h int f(); // #1, attached to the global module diff --git a/clang/test/CXX/basic/basic.lookup/basic.lookup.argdep/p4-friend-in-reachable-class.cpp b/clang/test/CXX/basic/basic.lookup/basic.lookup.argdep/p4-friend-in-reachable-class.cpp index 638057cbd681..3c120654f2ee 100644 --- a/clang/test/CXX/basic/basic.lookup/basic.lookup.argdep/p4-friend-in-reachable-class.cpp +++ b/clang/test/CXX/basic/basic.lookup/basic.lookup.argdep/p4-friend-in-reachable-class.cpp @@ -8,7 +8,10 @@ // RUN: split-file %s %t // // RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/Friend-in-reachable-class.cppm -o %t/X.pcm -// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/Friend-in-reachable-class.cppm \ +// RUN: -o %t/X.reduced.pcm +// RUN: %clang_cc1 -std=c++20 -fmodule-file=X=%t/X.pcm %t/Use.cpp -verify -fsyntax-only +// RUN: %clang_cc1 -std=c++20 -fmodule-file=X=%t/X.reduced.pcm %t/Use.cpp -verify -fsyntax-only // //--- Friend-in-reachable-class.cppm module; diff --git a/clang/test/Modules/InheritDefaultArguments.cppm b/clang/test/Modules/InheritDefaultArguments.cppm index 0afb46319ff8..0ef6390204c4 100644 --- a/clang/test/Modules/InheritDefaultArguments.cppm +++ b/clang/test/Modules/InheritDefaultArguments.cppm @@ -5,6 +5,9 @@ // RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-module-interface -o %t/A.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t -I%t %t/Use.cppm -verify -fsyntax-only +// RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-reduced-module-interface -o %t/A.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t -I%t %t/Use.cppm -verify -fsyntax-only + //--- foo.h template class Templ; diff --git a/clang/test/Modules/Reachability-Private.cpp b/clang/test/Modules/Reachability-Private.cpp index 9a7c3ba231f1..3ce108dc5c55 100644 --- a/clang/test/Modules/Reachability-Private.cpp +++ b/clang/test/Modules/Reachability-Private.cpp @@ -9,6 +9,16 @@ // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp \ // RUN: -DTEST_BADINLINE -verify -fsyntax-only +// Test again with reduced BMI. +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 %t/Private.cppm -emit-reduced-module-interface \ +// RUN: -o %t/Private.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp \ +// RUN: -DTEST_BADINLINE -verify -fsyntax-only + //--- Private.cppm export module Private; #ifdef TEST_BADINLINE diff --git a/clang/test/Modules/Reachability-func-default-arg.cpp b/clang/test/Modules/Reachability-func-default-arg.cpp index 0d6d8655d532..bc0cafdebb7a 100644 --- a/clang/test/Modules/Reachability-func-default-arg.cpp +++ b/clang/test/Modules/Reachability-func-default-arg.cpp @@ -4,6 +4,9 @@ // // RUN: %clang_cc1 -std=c++20 %t/func_default_arg.cppm -emit-module-interface -o %t/func_default_arg.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only + +// RUN: %clang_cc1 -std=c++20 %t/func_default_arg.cppm -emit-reduced-module-interface -o %t/func_default_arg.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only // //--- func_default_arg.cppm export module func_default_arg; diff --git a/clang/test/Modules/Reachability-func-ret.cpp b/clang/test/Modules/Reachability-func-ret.cpp index ca5bbc68d759..7d34387726f6 100644 --- a/clang/test/Modules/Reachability-func-ret.cpp +++ b/clang/test/Modules/Reachability-func-ret.cpp @@ -4,6 +4,9 @@ // // RUN: %clang_cc1 -std=c++20 %t/func_ret.cppm -emit-module-interface -o %t/func_ret.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only + +// RUN: %clang_cc1 -std=c++20 %t/func_ret.cppm -emit-reduced-module-interface -o %t/func_ret.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only // //--- func_ret.cppm export module func_ret; diff --git a/clang/test/Modules/Reachability-template-default-arg.cpp b/clang/test/Modules/Reachability-template-default-arg.cpp index 6fb109e41fcf..35c647d0d344 100644 --- a/clang/test/Modules/Reachability-template-default-arg.cpp +++ b/clang/test/Modules/Reachability-template-default-arg.cpp @@ -4,6 +4,9 @@ // // RUN: %clang_cc1 -std=c++20 %t/template_default_arg.cppm -emit-module-interface -o %t/template_default_arg.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -fsyntax-only -verify + +// RUN: %clang_cc1 -std=c++20 %t/template_default_arg.cppm -emit-reduced-module-interface -o %t/template_default_arg.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -fsyntax-only -verify // //--- template_default_arg.cppm export module template_default_arg; diff --git a/clang/test/Modules/Reachability-template-instantiation.cpp b/clang/test/Modules/Reachability-template-instantiation.cpp index 2170c7b92a37..6f363ed00b6e 100644 --- a/clang/test/Modules/Reachability-template-instantiation.cpp +++ b/clang/test/Modules/Reachability-template-instantiation.cpp @@ -5,6 +5,10 @@ // RUN: %clang_cc1 -std=c++20 %t/Templ.cppm -emit-module-interface -o %t/Templ.pcm // RUN: %clang_cc1 -std=c++20 %t/Use.cppm -fprebuilt-module-path=%t -emit-module-interface -o %t/Use.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t -I%t %t/Use.cpp -verify -fsyntax-only + +// RUN: %clang_cc1 -std=c++20 %t/Templ.cppm -emit-reduced-module-interface -o %t/Templ.pcm +// RUN: %clang_cc1 -std=c++20 %t/Use.cppm -fprebuilt-module-path=%t -emit-reduced-module-interface -o %t/Use.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t -I%t %t/Use.cpp -verify -fsyntax-only // //--- Templ.h #ifndef TEMPL_H diff --git a/clang/test/Modules/Reachability-using-templates.cpp b/clang/test/Modules/Reachability-using-templates.cpp index f530e15bd4d2..65601c1cfe4e 100644 --- a/clang/test/Modules/Reachability-using-templates.cpp +++ b/clang/test/Modules/Reachability-using-templates.cpp @@ -4,6 +4,9 @@ // // RUN: %clang_cc1 -std=c++20 %t/mod.templates.cppm -emit-module-interface -o %t/mod.templates.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -fsyntax-only -verify + +// RUN: %clang_cc1 -std=c++20 %t/mod.templates.cppm -emit-reduced-module-interface -o %t/mod.templates.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -fsyntax-only -verify // //--- mod.templates.cppm export module mod.templates; diff --git a/clang/test/Modules/Reachability-using.cpp b/clang/test/Modules/Reachability-using.cpp index 642b97dd8432..8301bfbedf87 100644 --- a/clang/test/Modules/Reachability-using.cpp +++ b/clang/test/Modules/Reachability-using.cpp @@ -4,6 +4,9 @@ // // RUN: %clang_cc1 -std=c++20 %t/mod.cppm -emit-module-interface -o %t/mod.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -fsyntax-only -verify + +// RUN: %clang_cc1 -std=c++20 %t/mod.cppm -emit-reduced-module-interface -o %t/mod.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -fsyntax-only -verify // //--- mod.cppm export module mod; diff --git a/clang/test/Modules/concept.cppm b/clang/test/Modules/concept.cppm index 0fdb5ea89680..4464cf7c0a41 100644 --- a/clang/test/Modules/concept.cppm +++ b/clang/test/Modules/concept.cppm @@ -11,7 +11,6 @@ // RUN: %clang_cc1 -std=c++20 -fskip-odr-check-in-gmf -fprebuilt-module-path=%t -I%t \ // RUN: -DDIFFERENT -DSKIP_ODR_CHECK_IN_GMF %t/B.cppm -verify - //--- foo.h #ifndef FOO_H #define FOO_H diff --git a/clang/test/Modules/concept_differ.cppm b/clang/test/Modules/concept_differ.cppm index ccb29d26e53d..525ee2d4edcc 100644 --- a/clang/test/Modules/concept_differ.cppm +++ b/clang/test/Modules/concept_differ.cppm @@ -5,6 +5,11 @@ // RUN: %clang_cc1 -x c++ -std=c++20 %t/A.cppm -I%t -emit-module-interface -o %t/A.pcm // RUN: %clang_cc1 -x c++ -std=c++20 %t/B.cppm -I%t -emit-module-interface -o %t/B.pcm // RUN: %clang_cc1 -x c++ -std=c++20 -fprebuilt-module-path=%t %t/foo.cpp -verify +// +// RUN: rm %t/A.pcm %t/B.pcm +// RUN: %clang_cc1 -x c++ -std=c++20 %t/A.cppm -I%t -emit-reduced-module-interface -o %t/A.pcm +// RUN: %clang_cc1 -x c++ -std=c++20 %t/B.cppm -I%t -emit-reduced-module-interface -o %t/B.pcm +// RUN: %clang_cc1 -x c++ -std=c++20 -fprebuilt-module-path=%t %t/foo.cpp -verify //--- foo.h template diff --git a/clang/test/Modules/ctor.arg.dep.cppm b/clang/test/Modules/ctor.arg.dep.cppm index 0e5b1a694f6a..10924bfe0f1b 100644 --- a/clang/test/Modules/ctor.arg.dep.cppm +++ b/clang/test/Modules/ctor.arg.dep.cppm @@ -5,6 +5,10 @@ // RUN: %clang_cc1 -std=c++20 %t/A.cppm -I%t -emit-module-interface -o %t/A.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only // +// RUN: rm %t/A.pcm +// RUN: %clang_cc1 -std=c++20 %t/A.cppm -I%t -emit-reduced-module-interface -o %t/A.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only +// //--- foo.h namespace ns { diff --git a/clang/test/Modules/cxx20-10-1-ex1.cpp b/clang/test/Modules/cxx20-10-1-ex1.cpp index b330e0a6c9a9..4445b19ea86c 100644 --- a/clang/test/Modules/cxx20-10-1-ex1.cpp +++ b/clang/test/Modules/cxx20-10-1-ex1.cpp @@ -19,6 +19,22 @@ // RUN: -fmodule-file=A=%t/A.pcm -fmodule-file=A:Foo=%t/A_Foo.pcm \ // RUN: -fmodule-file=A:Internals=%t/A_Internals.pcm -o %t/ex1.o +// RUN: rm %t/A_Internals.pcm %t/A_Foo.pcm %t/A.pcm +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/std10-1-ex1-tu1.cpp \ +// RUN: -o %t/A_Internals.pcm + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/std10-1-ex1-tu2.cpp \ +// RUN: -fmodule-file=A:Internals=%t/A_Internals.pcm -o %t/A_Foo.pcm + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/std10-1-ex1-tu3.cpp \ +// RUN: -fmodule-file=A:Internals=%t/A_Internals.pcm \ +// RUN: -fmodule-file=A:Foo=%t/A_Foo.pcm -o %t/A.pcm + +// RUN: %clang_cc1 -std=c++20 -emit-obj %t/std10-1-ex1-tu4.cpp \ +// RUN: -fmodule-file=A:Internals=%t/A_Internals.pcm \ +// RUN: -fmodule-file=A:Foo=%t/A_Foo.pcm \ +// RUN: -fmodule-file=A=%t/A.pcm -o %t/ex1.o + // expected-no-diagnostics //--- std10-1-ex1-tu1.cpp diff --git a/clang/test/Modules/cxx20-10-1-ex2.cpp b/clang/test/Modules/cxx20-10-1-ex2.cpp index 8b908d5fa2ed..fc61d89926d4 100644 --- a/clang/test/Modules/cxx20-10-1-ex2.cpp +++ b/clang/test/Modules/cxx20-10-1-ex2.cpp @@ -5,26 +5,50 @@ // RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/std10-1-ex2-tu1.cpp \ // RUN: -o %t/B_Y.pcm - +// // RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/std10-1-ex2-tu2.cpp \ // RUN: -fmodule-file=B:Y=%t/B_Y.pcm -o %t/B.pcm - +// // RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/std10-1-ex2-tu3.cpp \ // RUN: -o %t/B_X1.pcm -verify - +// // RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/std10-1-ex2-tu4.cpp \ // RUN: -fmodule-file=B=%t/B.pcm -fmodule-file=B:Y=%t/B_Y.pcm -o %t/B_X2.pcm - +// // RUN: %clang_cc1 -std=c++20 -emit-obj %t/std10-1-ex2-tu5.cpp \ // RUN: -fmodule-file=B=%t/B.pcm -fmodule-file=B:Y=%t/B_Y.pcm -o %t/b_tu5.o - +// // RUN: %clang_cc1 -std=c++20 -S %t/std10-1-ex2-tu6.cpp \ // RUN: -fmodule-file=B=%t/B.pcm -fmodule-file=B:Y=%t/B_Y.pcm -o %t/b_tu6.s -verify - +// // RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/std10-1-ex2-tu7.cpp \ // RUN: -fmodule-file=B:X2=%t/B_X2.pcm -fmodule-file=B=%t/B.pcm \ // RUN: -fmodule-file=B:Y=%t/B_Y.pcm -o %t/B_X3.pcm -verify +// Test again with reduced BMI. +// RUN: rm %t/B_X2.pcm %t/B.pcm %t/B_Y.pcm +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/std10-1-ex2-tu1.cpp \ +// RUN: -o %t/B_Y.pcm +// +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/std10-1-ex2-tu2.cpp \ +// RUN: -fmodule-file=B:Y=%t/B_Y.pcm -o %t/B.pcm +// +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/std10-1-ex2-tu3.cpp \ +// RUN: -o %t/B_X1.pcm -verify +// +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/std10-1-ex2-tu4.cpp \ +// RUN: -fmodule-file=B=%t/B.pcm -fmodule-file=B:Y=%t/B_Y.pcm -o %t/B_X2.pcm +// +// RUN: %clang_cc1 -std=c++20 -emit-obj %t/std10-1-ex2-tu5.cpp \ +// RUN: -fmodule-file=B=%t/B.pcm -fmodule-file=B:Y=%t/B_Y.pcm -o %t/b_tu5.o +// +// RUN: %clang_cc1 -std=c++20 -S %t/std10-1-ex2-tu6.cpp \ +// RUN: -fmodule-file=B=%t/B.pcm -fmodule-file=B:Y=%t/B_Y.pcm -o %t/b_tu6.s -verify +// +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/std10-1-ex2-tu7.cpp \ +// RUN: -fmodule-file=B:X2=%t/B_X2.pcm -fmodule-file=B=%t/B.pcm \ +// RUN: -fmodule-file=B:Y=%t/B_Y.pcm -o %t/B_X3.pcm -verify + //--- std10-1-ex2-tu1.cpp module B:Y; int y(); diff --git a/clang/test/Modules/cxx20-10-2-ex2.cpp b/clang/test/Modules/cxx20-10-2-ex2.cpp index bc66d6a2ec1a..b48d96478b9a 100644 --- a/clang/test/Modules/cxx20-10-2-ex2.cpp +++ b/clang/test/Modules/cxx20-10-2-ex2.cpp @@ -14,6 +14,18 @@ // RUN: -fmodule-file=%t/std-10-2-ex2-c.pcm -fmodule-file=X=%t/X.pcm \ // RUN: -pedantic-errors -verify -o %t/M.pcm +// Test again with reduced BMI. +// RUN: %clang_cc1 -std=c++20 -emit-header-unit -I %t \ +// RUN: -xc++-user-header std-10-2-ex2-c.h -o %t/std-10-2-ex2-c.pcm + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/std-10-2-ex2-tu1.cpp \ +// RUN: -o %t/X.pcm + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/std-10-2-ex2-tu2.cpp \ +// RUN: -fmodule-file=%t/std-10-2-ex2-c.pcm -fmodule-file=X=%t/X.pcm \ +// RUN: -pedantic-errors -verify -o %t/M.pcm + + //--- std-10-2-ex2-b.h int f(); diff --git a/clang/test/Modules/cxx20-10-2-ex5.cpp b/clang/test/Modules/cxx20-10-2-ex5.cpp index 49c5934c8f21..f22256807239 100644 --- a/clang/test/Modules/cxx20-10-2-ex5.cpp +++ b/clang/test/Modules/cxx20-10-2-ex5.cpp @@ -13,6 +13,18 @@ // RUN: %clang_cc1 -std=c++20 -emit-obj %t/std-10-2-ex5-tu3.cpp \ // RUN: -fmodule-file=M=%t/M.pcm -verify -o %t/main.o +// Test again with reduced BMI. +// RUN: rm %t/M.pcm +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/std-10-2-ex5-tu1.cpp \ +// RUN: -o %t/M.pcm + +// RUN: %clang_cc1 -std=c++20 -emit-obj %t/std-10-2-ex5-tu2.cpp \ +// RUN: -fmodule-file=M=%t/M.pcm -o %t/tu-2.o + +// RUN: %clang_cc1 -std=c++20 -emit-obj %t/std-10-2-ex5-tu3.cpp \ +// RUN: -fmodule-file=M=%t/M.pcm -verify -o %t/main.o + + //--- std-10-2-ex5-tu1.cpp export module M; export struct X { diff --git a/clang/test/Modules/cxx20-10-3-ex1.cpp b/clang/test/Modules/cxx20-10-3-ex1.cpp index 5d6e2554f753..99b88c7e442f 100644 --- a/clang/test/Modules/cxx20-10-3-ex1.cpp +++ b/clang/test/Modules/cxx20-10-3-ex1.cpp @@ -14,6 +14,20 @@ // RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/std10-3-ex1-tu4.cpp \ // RUN: -fmodule-file=M:Part=%t/M_Part.pcm -o %t/M.pcm +// Test again with reduced BMI. +// RUN: rm %t/M_PartImpl.pcm %t/M.pcm %t/M_Part.pcm +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/std10-3-ex1-tu1.cpp \ +// RUN: -o %t/M_PartImpl.pcm + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/std10-3-ex1-tu2.cpp \ +// RUN: -fmodule-file=M:PartImpl=%t/M_PartImpl.pcm -o %t/M.pcm -verify + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/std10-3-ex1-tu3.cpp \ +// RUN: -o %t/M_Part.pcm + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/std10-3-ex1-tu4.cpp \ +// RUN: -fmodule-file=M:Part=%t/M_Part.pcm -o %t/M.pcm + //--- std10-3-ex1-tu1.cpp module M:PartImpl; diff --git a/clang/test/Modules/cxx20-10-3-ex2.cpp b/clang/test/Modules/cxx20-10-3-ex2.cpp index b1d6d669c0a0..40566c00f578 100644 --- a/clang/test/Modules/cxx20-10-3-ex2.cpp +++ b/clang/test/Modules/cxx20-10-3-ex2.cpp @@ -11,6 +11,16 @@ // RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/std10-3-ex2-tu3.cpp \ // RUN: -o %t/M.pcm -verify +// Test again with reduced BMI. +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/std10-3-ex2-tu1.cpp \ +// RUN: -o %t/M.pcm + +// RUN: %clang_cc1 -std=c++20 -S %t/std10-3-ex2-tu2.cpp \ +// RUN: -fmodule-file=M=%t/M.pcm -o %t/tu_8.s -verify + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/std10-3-ex2-tu3.cpp \ +// RUN: -o %t/M.pcm -verify + //--- std10-3-ex2-tu1.cpp export module M; diff --git a/clang/test/Modules/cxx20-10-5-ex1.cpp b/clang/test/Modules/cxx20-10-5-ex1.cpp index a83162c5c150..0435b3a64c07 100644 --- a/clang/test/Modules/cxx20-10-5-ex1.cpp +++ b/clang/test/Modules/cxx20-10-5-ex1.cpp @@ -11,6 +11,18 @@ // RUN: %clang_cc1 -std=c++20 std-10-5-ex1-use.cpp -fmodule-file=A=A.pcm \ // RUN: -fsyntax-only -verify +// Test again with reduced BMI. +// RUN: rm A.pcm +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface std-10-5-ex1-interface.cpp \ +// RUN: -DBAD_FWD_DECL -fsyntax-only -verify + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface std-10-5-ex1-interface.cpp \ +// RUN: -o A.pcm + +// RUN: %clang_cc1 -std=c++20 std-10-5-ex1-use.cpp -fmodule-file=A=A.pcm \ +// RUN: -fsyntax-only -verify + + //--- std-10-5-ex1-interface.cpp export module A; diff --git a/clang/test/Modules/cxx20-import-diagnostics-a.cpp b/clang/test/Modules/cxx20-import-diagnostics-a.cpp index a5cf44ed82d5..1b38259e0358 100644 --- a/clang/test/Modules/cxx20-import-diagnostics-a.cpp +++ b/clang/test/Modules/cxx20-import-diagnostics-a.cpp @@ -36,6 +36,45 @@ // RUN: %clang_cc1 -std=c++20 -emit-obj %t/import-diags-tu11.cpp \ // RUN: -fmodule-file=C=%t/C.pcm -o %t/impl.o +// Test again with reduced BMI. +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/import-diags-tu1.cpp \ +// RUN: -o %t/B.pcm + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/import-diags-tu2.cpp \ +// RUN: -o %t/C.pcm + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/import-diags-tu3.cpp \ +// RUN: -fmodule-file=B=%t/B.pcm -fmodule-file=C=%t/C.pcm -o %t/AOK1.pcm + +// RUN: %clang_cc1 -std=c++20 -S %t/import-diags-tu4.cpp \ +// RUN: -fmodule-file=AOK1=%t/AOK1.pcm -fmodule-file=B=%t/B.pcm \ +// RUN: -fmodule-file=C=%t/C.pcm -o %t/tu_3.s -verify + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/import-diags-tu5.cpp \ +// RUN: -fmodule-file=B=%t/B.pcm -fmodule-file=C=%t/C.pcm -o %t/BC.pcm -verify + +// RUN: %clang_cc1 -std=c++20 -S %t/import-diags-tu6.cpp \ +// RUN: -fmodule-file=B=%t/B.pcm -fmodule-file=C=%t/C.pcm -o %t/tu_5.s -verify + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/import-diags-tu7.cpp \ +// RUN: -fmodule-file=B=%t/B.pcm -o %t/D.pcm -verify + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/import-diags-tu8.cpp \ +// RUN: -fmodule-file=B=%t/B.pcm -o %t/D.pcm -verify + +// RUN: %clang_cc1 -std=c++20 -S %t/import-diags-tu9.cpp \ +// RUN: -fmodule-file=B=%t/B.pcm -o %t/tu_8.s -verify + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/import-diags-tu10.cpp \ +// RUN: -o %t/B.pcm -verify + +// RUN: %clang_cc1 -std=c++20 -emit-obj %t/import-diags-tu11.cpp \ +// RUN: -fmodule-file=C=%t/C.pcm -o %t/impl.o + // Test diagnostics for incorrect module import sequences. //--- import-diags-tu1.cpp diff --git a/clang/test/Modules/cxx20-import-diagnostics-b.cpp b/clang/test/Modules/cxx20-import-diagnostics-b.cpp index 7d432633552a..db522d7babd3 100644 --- a/clang/test/Modules/cxx20-import-diagnostics-b.cpp +++ b/clang/test/Modules/cxx20-import-diagnostics-b.cpp @@ -22,6 +22,31 @@ // RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/g.cpp \ // RUN: -fmodule-file=a=%t/a.pcm -o %t/g.pcm -verify +// Test again with reduced BMI. +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/a.cpp -o %t/a.pcm + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/c.cpp \ +// RUN: -fmodule-file=a=%t/a.pcm -o %t/c.pcm + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/d.cpp \ +// RUN: -fmodule-file=a=%t/a.pcm -o %t/d.pcm + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/e.cpp \ +// RUN: -fmodule-file=a=%t/a.pcm -o %t/e.pcm + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/a-part.cpp \ +// RUN: -o %t/a-part.pcm + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/f.cpp \ +// RUN: -fmodule-file=a=%t/a.pcm -o %t/f.pcm -verify + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/g.cpp \ +// RUN: -fmodule-file=a=%t/a.pcm -o %t/g.pcm -verify + //--- a.cpp export module a; diff --git a/clang/test/Modules/cxx20-module-file-info-macros.cpp b/clang/test/Modules/cxx20-module-file-info-macros.cpp index bc7df1c9f50b..3b67e9b9acd4 100644 --- a/clang/test/Modules/cxx20-module-file-info-macros.cpp +++ b/clang/test/Modules/cxx20-module-file-info-macros.cpp @@ -17,6 +17,9 @@ // RUN: %clang_cc1 -std=c++20 %t/named_module.cppm -emit-module-interface -o %t/M.pcm // RUN: %clang_cc1 -module-file-info %t/M.pcm | FileCheck %t/named_module.cppm +// RUN: %clang_cc1 -std=c++20 %t/named_module.cppm -emit-reduced-module-interface -o %t/M.pcm +// RUN: %clang_cc1 -module-file-info %t/M.pcm | FileCheck %t/named_module.cppm + //--- foo.h #pragma once #define FOO diff --git a/clang/test/Modules/deduction-guide.cppm b/clang/test/Modules/deduction-guide.cppm index 9c959a71365d..02ac2c0053cf 100644 --- a/clang/test/Modules/deduction-guide.cppm +++ b/clang/test/Modules/deduction-guide.cppm @@ -5,6 +5,9 @@ // RUN: %clang_cc1 -std=c++20 %t/Templ.cppm -emit-module-interface -o %t/Templ.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only +// RUN: %clang_cc1 -std=c++20 %t/Templ.cppm -emit-reduced-module-interface -o %t/Templ.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only + //--- foo.h template class Templ { diff --git a/clang/test/Modules/deduction-guide2.cppm b/clang/test/Modules/deduction-guide2.cppm index a163c3656831..889670b973f0 100644 --- a/clang/test/Modules/deduction-guide2.cppm +++ b/clang/test/Modules/deduction-guide2.cppm @@ -5,6 +5,9 @@ // RUN: %clang_cc1 -std=c++20 %t/Templ.cppm -emit-module-interface -o %t/Templ.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only +// RUN: %clang_cc1 -std=c++20 %t/Templ.cppm -emit-reduced-module-interface -o %t/Templ.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only + //--- Templ.cppm export module Templ; export template diff --git a/clang/test/Modules/deduction-guide3.cppm b/clang/test/Modules/deduction-guide3.cppm index 8fa08a0625d7..1165dd40bcfb 100644 --- a/clang/test/Modules/deduction-guide3.cppm +++ b/clang/test/Modules/deduction-guide3.cppm @@ -5,6 +5,9 @@ // RUN: %clang_cc1 -std=c++20 %t/Templ.cppm -emit-module-interface -o %t/Templ.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only +// RUN: %clang_cc1 -std=c++20 %t/Templ.cppm -emit-reduced-module-interface -o %t/Templ.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only + //--- Templ.cppm export module Templ; template diff --git a/clang/test/Modules/derived_class.cpp b/clang/test/Modules/derived_class.cpp index ee9e0ae4637e..e0c5a652eba4 100644 --- a/clang/test/Modules/derived_class.cpp +++ b/clang/test/Modules/derived_class.cpp @@ -4,6 +4,9 @@ // // RUN: %clang_cc1 -std=c++20 %t/foo.cppm -emit-module-interface -o %t/foo.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -fsyntax-only -verify + +// RUN: %clang_cc1 -std=c++20 %t/foo.cppm -emit-reduced-module-interface -o %t/foo.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -fsyntax-only -verify // //--- bar.h struct bar_base { diff --git a/clang/test/Modules/duplicated-module-file-eq-module-name.cppm b/clang/test/Modules/duplicated-module-file-eq-module-name.cppm index e86dbe2b941e..57ffb560ab54 100644 --- a/clang/test/Modules/duplicated-module-file-eq-module-name.cppm +++ b/clang/test/Modules/duplicated-module-file-eq-module-name.cppm @@ -8,6 +8,10 @@ // RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-module-interface -o %t/a.pcm // RUN: %clang_cc1 -std=c++20 %t/u.cpp -fmodule-file=a=%t/unexist.pcm \ // RUN: -fmodule-file=a=%t/a.pcm -verify -fsyntax-only +// +// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/a.pcm +// RUN: %clang_cc1 -std=c++20 %t/u.cpp -fmodule-file=a=%t/unexist.pcm \ +// RUN: -fmodule-file=a=%t/a.pcm -verify -fsyntax-only //--- a.cppm export module a; diff --git a/clang/test/Modules/enum-class.cppm b/clang/test/Modules/enum-class.cppm index 01ae8c0d8814..992eb9d5e551 100644 --- a/clang/test/Modules/enum-class.cppm +++ b/clang/test/Modules/enum-class.cppm @@ -6,6 +6,9 @@ // // RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-module-interface -o %t/A.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only +// +// RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-reduced-module-interface -o %t/A.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only //--- foo.h enum class foo { diff --git a/clang/test/Modules/explicitly-specialized-template.cpp b/clang/test/Modules/explicitly-specialized-template.cpp index 89677254ea73..2450bbe31bd9 100644 --- a/clang/test/Modules/explicitly-specialized-template.cpp +++ b/clang/test/Modules/explicitly-specialized-template.cpp @@ -5,6 +5,9 @@ // RUN: %clang_cc1 -std=c++20 %t/X.cppm -emit-module-interface -o %t/X.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -fsyntax-only -verify // +// RUN: %clang_cc1 -std=c++20 %t/X.cppm -emit-reduced-module-interface -o %t/X.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -fsyntax-only -verify +// //--- foo.h #ifndef FOO_H #define FOO_H diff --git a/clang/test/Modules/export-language-linkage.cppm b/clang/test/Modules/export-language-linkage.cppm index 462b28d36cb4..f389d9604ef3 100644 --- a/clang/test/Modules/export-language-linkage.cppm +++ b/clang/test/Modules/export-language-linkage.cppm @@ -8,6 +8,11 @@ // RUN: %clang_cc1 -std=c++20 %t/c.cppm -emit-module-interface -o %t/c.pcm // RUN: %clang_cc1 -std=c++20 %t/d.cpp -fsyntax-only -verify -fmodule-file=c=%t/c.pcm +// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/a.pcm +// RUN: %clang_cc1 -std=c++20 %t/b.cpp -fmodule-file=a=%t/a.pcm -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 %t/c.cppm -fsyntax-only -verify +// RUN: %clang_cc1 -module-file-info %t/a.pcm | FileCheck %t/a.cppm + //--- a.cppm export module a; export extern "C++" int foo() { return 43; } @@ -43,6 +48,7 @@ int use() { } //--- c.cppm +// expected-no-diagnostics export module c; extern "C++" { export int f(); @@ -59,5 +65,5 @@ int use() { int use_of_nonexported() { return h(); // expected-error {{declaration of 'h' must be imported from module 'c' before it is required}} - // expected-note@c.cppm:4 {{declaration here is not visible}} + // expected-note@c.cppm:5 {{declaration here is not visible}} } diff --git a/clang/test/Modules/ftime-trace.cppm b/clang/test/Modules/ftime-trace.cppm index 48cd4113ec78..8882e85be151 100644 --- a/clang/test/Modules/ftime-trace.cppm +++ b/clang/test/Modules/ftime-trace.cppm @@ -9,5 +9,14 @@ // RUN: %clang_cc1 -std=c++20 %t/a.pcm -ftime-trace=%t/a.json -o - // RUN: ls %t | grep "a.json" +// Test again with reduced BMI. +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/a.pcm +// RUN: %clang_cc1 -std=c++20 %t/a.pcm -ftime-trace=%t/a.json -o - +// RUN: ls %t | grep "a.json" + //--- a.cppm export module a; diff --git a/clang/test/Modules/inconsistent-deduction-guide-linkage.cppm b/clang/test/Modules/inconsistent-deduction-guide-linkage.cppm index abcbec07f97d..3991e47ce215 100644 --- a/clang/test/Modules/inconsistent-deduction-guide-linkage.cppm +++ b/clang/test/Modules/inconsistent-deduction-guide-linkage.cppm @@ -8,6 +8,12 @@ // RUN: %clang_cc1 -std=c++20 %t/D.cppm -I%t -emit-module-interface -o %t/D.pcm // RUN: %clang_cc1 -std=c++20 -fsyntax-only %t/D-part.cppm -I%t -fprebuilt-module-path=%t -verify +// RUN: %clang_cc1 -std=c++20 %t/B.cppm -I%t -emit-reduced-module-interface -o %t/B.pcm +// RUN: %clang_cc1 -std=c++20 -fsyntax-only %t/A.cppm -I%t -fprebuilt-module-path=%t -verify +// +// RUN: %clang_cc1 -std=c++20 %t/D.cppm -I%t -emit-reduced-module-interface -o %t/D.pcm +// RUN: %clang_cc1 -std=c++20 -fsyntax-only %t/D-part.cppm -I%t -fprebuilt-module-path=%t -verify + //--- A.cppm module; export module baz:A; diff --git a/clang/test/Modules/inconsistent-export.cppm b/clang/test/Modules/inconsistent-export.cppm index 5e94d2b37b75..0c74ba903770 100644 --- a/clang/test/Modules/inconsistent-export.cppm +++ b/clang/test/Modules/inconsistent-export.cppm @@ -9,6 +9,19 @@ // RUN: -fprebuilt-module-path=%t // RUN: %clang_cc1 -std=c++20 %t/use.cppm -fprebuilt-module-path=%t -emit-obj +// Test again with reduced BMI. +// RUN: rm -fr %t +// RUN: mkdir %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/m-a.pcm +// RUN: %clang_cc1 -std=c++20 %t/b.cppm -emit-reduced-module-interface -o %t/m-b.pcm \ +// RUN: -fprebuilt-module-path=%t +// RUN: %clang_cc1 -std=c++20 %t/m.cppm -emit-reduced-module-interface -o %t/m.pcm \ +// RUN: -fprebuilt-module-path=%t +// RUN: %clang_cc1 -std=c++20 %t/use.cppm -fprebuilt-module-path=%t -emit-obj + + //--- a.cppm export module m:a; namespace n { diff --git a/clang/test/Modules/inherited_arg.cppm b/clang/test/Modules/inherited_arg.cppm index eb66b70cdce3..a9b6efabb1e6 100644 --- a/clang/test/Modules/inherited_arg.cppm +++ b/clang/test/Modules/inherited_arg.cppm @@ -7,6 +7,14 @@ // RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-module-interface -fprebuilt-module-path=%t -o %t/A.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only +// Test again with reduced BMI. +// +// RUN: %clang_cc1 -std=c++20 %t/A-B.cppm -I%t -emit-reduced-module-interface -o %t/A-B.pcm +// RUN: %clang_cc1 -std=c++20 %t/A-C.cppm -I%t -emit-reduced-module-interface -o %t/A-C.pcm +// RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-reduced-module-interface -fprebuilt-module-path=%t -o %t/A.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only + + //--- foo.h template class pair {}; diff --git a/clang/test/Modules/instantiation-argdep-lookup.cppm b/clang/test/Modules/instantiation-argdep-lookup.cppm index fc9009a5bc13..62dabfb6efdd 100644 --- a/clang/test/Modules/instantiation-argdep-lookup.cppm +++ b/clang/test/Modules/instantiation-argdep-lookup.cppm @@ -5,6 +5,9 @@ // RUN: %clang_cc1 -std=c++20 %t/A.cppm -I%t -emit-module-interface -o %t/A.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only // +// RUN: %clang_cc1 -std=c++20 %t/A.cppm -I%t -emit-reduced-module-interface -o %t/A.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only +// //--- foo.h namespace ns { diff --git a/clang/test/Modules/lambdas.cppm b/clang/test/Modules/lambdas.cppm index 7f00cf6f8682..be614b051916 100644 --- a/clang/test/Modules/lambdas.cppm +++ b/clang/test/Modules/lambdas.cppm @@ -11,6 +11,21 @@ // RUN: -o %t/lambdas2.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -fsyntax-only \ // RUN: -verify -DUSE_LAMBDA2 +// +// Test again with reduced BMI. +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 %t/lambdas.cppm -emit-reduced-module-interface \ +// RUN: -o %t/lambdas.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -fsyntax-only \ +// RUN: -verify +// +// RUN: %clang_cc1 -std=c++20 %t/lambdas2.cppm -emit-reduced-module-interface \ +// RUN: -o %t/lambdas2.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -fsyntax-only \ +// RUN: -verify -DUSE_LAMBDA2 //--- lambdas.h auto l1 = []() constexpr -> int { diff --git a/clang/test/Modules/merge-concepts-cxx-modules.cpp b/clang/test/Modules/merge-concepts-cxx-modules.cpp index 3d4f8435531a..0127e8baad6b 100644 --- a/clang/test/Modules/merge-concepts-cxx-modules.cpp +++ b/clang/test/Modules/merge-concepts-cxx-modules.cpp @@ -8,6 +8,18 @@ // RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/conflicting.cppm -o %t/conflicting.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cppm -fsyntax-only -verify +// Test again with reduced BMI. +// RUN: rm -rf %t +// RUN: mkdir %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/same_as.cppm -o %t/same_as.pcm +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface -fprebuilt-module-path=%t %t/concepts.cppm -o %t/concepts.pcm +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface -fprebuilt-module-path=%t %t/format.cppm -o %t/format.pcm +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/conflicting.cppm -o %t/conflicting.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cppm -fsyntax-only -verify + + //--- same_as.cppm export module same_as; export template diff --git a/clang/test/Modules/merge-constrained-friends.cpp b/clang/test/Modules/merge-constrained-friends.cpp index 8f0e9ed83cf2..d0317b99801e 100644 --- a/clang/test/Modules/merge-constrained-friends.cpp +++ b/clang/test/Modules/merge-constrained-friends.cpp @@ -5,6 +5,9 @@ // RUN: %clang_cc1 -std=c++23 %t/A.cppm -emit-module-interface -o %t/A.pcm // RUN: %clang_cc1 -std=c++23 %t/Use.cpp -fprebuilt-module-path=%t -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++23 %t/A.cppm -emit-reduced-module-interface -o %t/A.pcm +// RUN: %clang_cc1 -std=c++23 %t/Use.cpp -fprebuilt-module-path=%t -fsyntax-only -verify + //--- A.cppm module; export module A; diff --git a/clang/test/Modules/merge-lambdas.cppm b/clang/test/Modules/merge-lambdas.cppm index a1d04ab4e234..4363e452c2bc 100644 --- a/clang/test/Modules/merge-lambdas.cppm +++ b/clang/test/Modules/merge-lambdas.cppm @@ -6,6 +6,10 @@ // RUN: %clang_cc1 -std=c++20 %t/B.cppm -emit-module-interface -o %t/B.pcm // RUN: %clang_cc1 -std=c++20 %t/use.cppm -fprebuilt-module-path=%t -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-reduced-module-interface -o %t/A.pcm +// RUN: %clang_cc1 -std=c++20 %t/B.cppm -emit-reduced-module-interface -o %t/B.pcm +// RUN: %clang_cc1 -std=c++20 %t/use.cppm -fprebuilt-module-path=%t -fsyntax-only -verify + //--- lambda.h inline auto cmp = [](auto l, auto r) { return l < r; diff --git a/clang/test/Modules/merge-requires-with-lambdas.cppm b/clang/test/Modules/merge-requires-with-lambdas.cppm index 576749204768..c4d6e0539f41 100644 --- a/clang/test/Modules/merge-requires-with-lambdas.cppm +++ b/clang/test/Modules/merge-requires-with-lambdas.cppm @@ -17,6 +17,25 @@ // RUN: %clang_cc1 -std=c++20 %t/A3.cppm -emit-module-interface -o %t/A3.pcm // RUN: %clang_cc1 -std=c++20 %t/TestA3.cpp -fprebuilt-module-path=%t -fsyntax-only -verify +// Test again with reduced BMI. +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-reduced-module-interface -o %t/A.pcm +// RUN: %clang_cc1 -std=c++20 %t/A0.cppm -emit-reduced-module-interface -o %t/A0.pcm +// RUN: %clang_cc1 -std=c++20 %t/TestA.cpp -fprebuilt-module-path=%t -fsyntax-only -verify +// +// RUN: %clang_cc1 -std=c++20 %t/A1.cppm -emit-reduced-module-interface -o %t/A1.pcm +// RUN: %clang_cc1 -std=c++20 %t/TestA1.cpp -fprebuilt-module-path=%t -fsyntax-only -verify +// +// RUN: %clang_cc1 -std=c++20 %t/A2.cppm -emit-reduced-module-interface -o %t/A2.pcm +// RUN: %clang_cc1 -std=c++20 %t/TestA2.cpp -fprebuilt-module-path=%t -fsyntax-only -verify +// +// RUN: %clang_cc1 -std=c++20 %t/A3.cppm -emit-reduced-module-interface -o %t/A3.pcm +// RUN: %clang_cc1 -std=c++20 %t/TestA3.cpp -fprebuilt-module-path=%t -fsyntax-only -verify + + //--- A.h template concept A = requires(const _Tp& __t) { [](const __Up&) {}(__t); }; diff --git a/clang/test/Modules/merge-var-template-spec-cxx-modules.cppm b/clang/test/Modules/merge-var-template-spec-cxx-modules.cppm index a451bfe7804d..db3f4cd51871 100644 --- a/clang/test/Modules/merge-var-template-spec-cxx-modules.cppm +++ b/clang/test/Modules/merge-var-template-spec-cxx-modules.cppm @@ -7,6 +7,11 @@ // RUN: %clang_cc1 -std=c++20 -emit-module-interface -fprebuilt-module-path=%t %t/reexport2.cppm -o %t/reexport2.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/use.cppm -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/var_def.cppm -o %t/var_def.pcm +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface -fprebuilt-module-path=%t %t/reexport1.cppm -o %t/reexport1.pcm +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface -fprebuilt-module-path=%t %t/reexport2.cppm -o %t/reexport2.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/use.cppm -fsyntax-only -verify + //--- use.cppm import reexport1; import reexport2; diff --git a/clang/test/Modules/mismatch-diagnostics.cpp b/clang/test/Modules/mismatch-diagnostics.cpp index f8ce987cfba5..5a026aa1f6c0 100644 --- a/clang/test/Modules/mismatch-diagnostics.cpp +++ b/clang/test/Modules/mismatch-diagnostics.cpp @@ -13,6 +13,17 @@ // RUN: -fprebuilt-module-path=%t/prebuilt_modules -DCHECK_MISMATCH \ // RUN: %t/use.cpp 2>&1 | FileCheck %s +// Test again with reduced BMI. +// RUN: %clang_cc1 -triple %itanium_abi_triple \ +// RUN: -std=c++20 -fprebuilt-module-path=%t/prebuilt-modules \ +// RUN: -emit-reduced-module-interface -pthread -DBUILD_MODULE \ +// RUN: %t/mismatching_module.cppm -o \ +// RUN: %t/prebuilt_modules/mismatching_module.pcm +// +// RUN: not %clang_cc1 -triple %itanium_abi_triple -std=c++20 \ +// RUN: -fprebuilt-module-path=%t/prebuilt_modules -DCHECK_MISMATCH \ +// RUN: %t/use.cpp 2>&1 | FileCheck %s + //--- mismatching_module.cppm export module mismatching_module; diff --git a/clang/test/Modules/module-init-duplicated-import.cppm b/clang/test/Modules/module-init-duplicated-import.cppm index 7adce1177956..1326402bb4de 100644 --- a/clang/test/Modules/module-init-duplicated-import.cppm +++ b/clang/test/Modules/module-init-duplicated-import.cppm @@ -9,6 +9,17 @@ // RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++20 %t/m.pcm \ // RUN: -fmodule-file=a=%t/a.pcm -S -emit-llvm -o - | FileCheck %t/m.cppm +// Test again with reduced BMI. +// Note that we can't use reduced BMI here for m.cppm since it is required +// to generate the backend code. +// RUN: rm %t/a.pcm %t/m.pcm +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++20 %t/a.cppm \ +// RUN: -emit-reduced-module-interface -o %t/a.pcm +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++20 %t/m.cppm \ +// RUN: -emit-module-interface -fmodule-file=a=%t/a.pcm -o %t/m.pcm +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++20 %t/m.pcm \ +// RUN: -fmodule-file=a=%t/a.pcm -S -emit-llvm -o - | FileCheck %t/m.cppm + //--- a.cppm export module a; export struct A { diff --git a/clang/test/Modules/named-modules-adl-2.cppm b/clang/test/Modules/named-modules-adl-2.cppm index 655acfcd93f6..a14b9a68d74e 100644 --- a/clang/test/Modules/named-modules-adl-2.cppm +++ b/clang/test/Modules/named-modules-adl-2.cppm @@ -6,6 +6,10 @@ // RUN: %clang_cc1 -std=c++20 %t/b.cppm -fmodule-file=a=%t/a.pcm -emit-module-interface -o %t/b.pcm // RUN: %clang_cc1 -std=c++20 %t/c.cppm -fmodule-file=a=%t/a.pcm -fmodule-file=b=%t/b.pcm -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/a.pcm +// RUN: %clang_cc1 -std=c++20 %t/b.cppm -fmodule-file=a=%t/a.pcm -emit-reduced-module-interface -o %t/b.pcm +// RUN: %clang_cc1 -std=c++20 %t/c.cppm -fmodule-file=a=%t/a.pcm -fmodule-file=b=%t/b.pcm -fsyntax-only -verify + //--- a.cppm export module a; diff --git a/clang/test/Modules/named-modules-adl-3.cppm b/clang/test/Modules/named-modules-adl-3.cppm index 2fc2962c926b..d70946fa068b 100644 --- a/clang/test/Modules/named-modules-adl-3.cppm +++ b/clang/test/Modules/named-modules-adl-3.cppm @@ -14,6 +14,20 @@ // RUN: %clang_cc1 -std=c++20 -DEXPORT_OPERATOR %t/c.cppm -fmodule-file=a=%t/a.pcm \ // RUN: -fmodule-file=b=%t/b.pcm -fsyntax-only -verify +// Test again with reduced BMI. +// +// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/a.pcm +// RUN: %clang_cc1 -std=c++20 %t/b.cppm -fmodule-file=a=%t/a.pcm -emit-reduced-module-interface \ +// RUN: -o %t/b.pcm +// RUN: %clang_cc1 -std=c++20 %t/c.cppm -fmodule-file=a=%t/a.pcm -fmodule-file=b=%t/b.pcm \ +// RUN: -fsyntax-only -verify +// +// RUN: %clang_cc1 -std=c++20 -DEXPORT_OPERATOR %t/a.cppm -emit-reduced-module-interface -o %t/a.pcm +// RUN: %clang_cc1 -std=c++20 -DEXPORT_OPERATOR %t/b.cppm -fmodule-file=a=%t/a.pcm \ +// RUN: -emit-reduced-module-interface -o %t/b.pcm +// RUN: %clang_cc1 -std=c++20 -DEXPORT_OPERATOR %t/c.cppm -fmodule-file=a=%t/a.pcm \ +// RUN: -fmodule-file=b=%t/b.pcm -fsyntax-only -verify + //--- foo.h namespace n { diff --git a/clang/test/Modules/named-modules-adl.cppm b/clang/test/Modules/named-modules-adl.cppm index d5133ef36726..ef250023f91e 100644 --- a/clang/test/Modules/named-modules-adl.cppm +++ b/clang/test/Modules/named-modules-adl.cppm @@ -5,6 +5,9 @@ // RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-module-interface -o %t/a.pcm // RUN: %clang_cc1 -std=c++20 %t/b.cppm -fmodule-file=a=%t/a.pcm -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/a.pcm +// RUN: %clang_cc1 -std=c++20 %t/b.cppm -fmodule-file=a=%t/a.pcm -fsyntax-only -verify + //--- a.h namespace n { diff --git a/clang/test/Modules/no-duplicate-codegen-in-GMF.cppm b/clang/test/Modules/no-duplicate-codegen-in-GMF.cppm index a743b64cb18d..36a2d8bc8c95 100644 --- a/clang/test/Modules/no-duplicate-codegen-in-GMF.cppm +++ b/clang/test/Modules/no-duplicate-codegen-in-GMF.cppm @@ -10,6 +10,16 @@ // RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/B.pcm -S -emit-llvm -o - \ // RUN: -fprebuilt-module-path=%t | FileCheck %t/B.cppm +// Test again with reduced BMI. Note that we need to generate full BMI for B.cppm +// since it is required to generate backend codes. +// RUN: rm %t/A.pcm %t/B.pcm +// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/A.cppm -emit-reduced-module-interface -o %t/A.pcm +// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/B.cppm -emit-module-interface -o %t/B.pcm \ +// RUN: -fprebuilt-module-path=%t +// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/B.pcm -S -emit-llvm -o - \ +// RUN: -fprebuilt-module-path=%t | FileCheck %t/B.cppm + + //--- foo.h template diff --git a/clang/test/Modules/pair-unambiguous-ctor.cppm b/clang/test/Modules/pair-unambiguous-ctor.cppm index eb242244260c..24fb15959577 100644 --- a/clang/test/Modules/pair-unambiguous-ctor.cppm +++ b/clang/test/Modules/pair-unambiguous-ctor.cppm @@ -10,6 +10,15 @@ // RUN: %clang_cc1 -std=c++20 %t/algorithm.cppm -I%t -emit-module-interface -o %t/std-algorithm.pcm // RUN: %clang_cc1 -std=c++20 %t/Use.cppm -I%t -fprebuilt-module-path=%t -emit-module-interface -verify -o %t/Use.pcm +// Test again with reduced BMI. +// RUN: rm -fr %t +// RUN: mkdir %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 %t/string.cppm -I%t -emit-reduced-module-interface -o %t/std-string.pcm +// RUN: %clang_cc1 -std=c++20 %t/algorithm.cppm -I%t -emit-reduced-module-interface -o %t/std-algorithm.pcm +// RUN: %clang_cc1 -std=c++20 %t/Use.cppm -I%t -fprebuilt-module-path=%t -emit-reduced-module-interface -verify -o %t/Use.pcm + //--- Use.cppm // expected-no-diagnostics module; diff --git a/clang/test/Modules/partial_specialization.cppm b/clang/test/Modules/partial_specialization.cppm index 3a0185717211..1d65a375643a 100644 --- a/clang/test/Modules/partial_specialization.cppm +++ b/clang/test/Modules/partial_specialization.cppm @@ -5,6 +5,9 @@ // RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/A.cppm -o %t/A.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -fsyntax-only -verify // +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/A.cppm -o %t/A.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -fsyntax-only -verify +// //--- foo.h template inline constexpr bool IsSame = false; diff --git a/clang/test/Modules/placement-new-reachable.cpp b/clang/test/Modules/placement-new-reachable.cpp index 29263173d78f..6b495a60306b 100644 --- a/clang/test/Modules/placement-new-reachable.cpp +++ b/clang/test/Modules/placement-new-reachable.cpp @@ -5,6 +5,9 @@ // RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-module-interface -o %t/A.pcm // RUN: %clang_cc1 -std=c++20 %t/Use.cpp -fprebuilt-module-path=%t -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-reduced-module-interface -o %t/A.pcm +// RUN: %clang_cc1 -std=c++20 %t/Use.cpp -fprebuilt-module-path=%t -fsyntax-only -verify + //--- placement.h namespace std { using size_t = decltype(sizeof(0)); diff --git a/clang/test/Modules/polluted-operator.cppm b/clang/test/Modules/polluted-operator.cppm index 721ca061c939..2179fa098064 100644 --- a/clang/test/Modules/polluted-operator.cppm +++ b/clang/test/Modules/polluted-operator.cppm @@ -11,6 +11,9 @@ // RUN: %clang_cc1 -std=c++20 -fskip-odr-check-in-gmf %t/b.cppm -fprebuilt-module-path=%t \ // RUN: -emit-module-interface -DSKIP_ODR_CHECK_IN_GMF -o %t/b.pcm -verify +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/a.cppm -o %t/a.pcm +// RUN: %clang_cc1 -std=c++20 %t/b.cppm -fprebuilt-module-path=%t -emit-reduced-module-interface -o %t/b.pcm -verify + //--- foo.h namespace std diff --git a/clang/test/Modules/pr54457.cppm b/clang/test/Modules/pr54457.cppm index ed67ec106537..d55bdfbf3b75 100644 --- a/clang/test/Modules/pr54457.cppm +++ b/clang/test/Modules/pr54457.cppm @@ -9,6 +9,9 @@ // RUN: %clang_cc1 -std=c++20 %t/C.cppm -emit-module-interface -o %t/C.pcm // RUN: %clang_cc1 -std=c++20 %t/UseC.cppm -fprebuilt-module-path=%t -verify -S -o - +// RUN: %clang_cc1 -std=c++20 %t/C.cppm -emit-reduced-module-interface -o %t/C.pcm +// RUN: %clang_cc1 -std=c++20 %t/UseC.cppm -fprebuilt-module-path=%t -verify -S -o - + //--- A.cppm // expected-no-diagnostics export module A; diff --git a/clang/test/Modules/pr56916.cppm b/clang/test/Modules/pr56916.cppm index a435b06d5cf1..09cea6720427 100644 --- a/clang/test/Modules/pr56916.cppm +++ b/clang/test/Modules/pr56916.cppm @@ -8,6 +8,18 @@ // RUN: -fprebuilt-module-path=%t // RUN: %clang_cc1 -std=c++20 %t/Use.cpp -fsyntax-only -fprebuilt-module-path=%t -verify +// Test again with reduced BMI. +// RUN: rm -rf %t +// RUN: mkdir %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-reduced-module-interface -o %t/M-A.pcm +// RUN: %clang_cc1 -std=c++20 %t/B.cppm -emit-reduced-module-interface -o %t/M-B.pcm +// RUN: %clang_cc1 -std=c++20 %t/M.cppm -emit-reduced-module-interface -o %t/M.pcm \ +// RUN: -fprebuilt-module-path=%t +// RUN: %clang_cc1 -std=c++20 %t/Use.cpp -fsyntax-only -fprebuilt-module-path=%t -verify + + //--- foo.h template class Templ { diff --git a/clang/test/Modules/pr58532.cppm b/clang/test/Modules/pr58532.cppm index cf530b4ac2cc..35bebb41431e 100644 --- a/clang/test/Modules/pr58532.cppm +++ b/clang/test/Modules/pr58532.cppm @@ -7,6 +7,12 @@ // RUN: %clang_cc1 -std=c++20 %t/implementation.cpp -fmodule-file=m=%t/m.pcm \ // RUN: -fsyntax-only -verify +// Test again with reduced BMI. +// RUN: %clang_cc1 -std=c++20 %t/interface.cppm -emit-reduced-module-interface \ +// RUN: -o %t/m.pcm +// RUN: %clang_cc1 -std=c++20 %t/implementation.cpp -fmodule-file=m=%t/m.pcm \ +// RUN: -fsyntax-only -verify + //--- invisible.h #pragma once // This breaks things. const int kInvisibleSymbol = 0; diff --git a/clang/test/Modules/pr58716.cppm b/clang/test/Modules/pr58716.cppm index 3f97fca7d5e8..177802fe3afc 100644 --- a/clang/test/Modules/pr58716.cppm +++ b/clang/test/Modules/pr58716.cppm @@ -8,7 +8,7 @@ // // RUN: %clang_cc1 -triple=x86_64-linux-gnu -std=c++20 -emit-module-interface %t/m.cppm -o %t/m.pcm // RUN: %clang_cc1 -triple=x86_64-linux-gnu -std=c++20 %t/m.pcm -S -emit-llvm -o - | FileCheck %t/m.cppm -// + //--- m.cppm module; #include "fail.h" diff --git a/clang/test/Modules/pr59719.cppm b/clang/test/Modules/pr59719.cppm index 5aea8992a0ca..5a600c8e36a4 100644 --- a/clang/test/Modules/pr59719.cppm +++ b/clang/test/Modules/pr59719.cppm @@ -7,6 +7,9 @@ // RUN: %clang_cc1 -std=c++20 %t/data.cppm -emit-module-interface -o %t/data.pcm // RUN: %clang_cc1 -std=c++20 %t/main.cpp -fprebuilt-module-path=%t -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 %t/data.cppm -emit-reduced-module-interface -o %t/data.pcm +// RUN: %clang_cc1 -std=c++20 %t/main.cpp -fprebuilt-module-path=%t -fsyntax-only -verify + //--- foo.h namespace std { diff --git a/clang/test/Modules/pr59780.cppm b/clang/test/Modules/pr59780.cppm index d4bbd52c13f1..ee81ca575d7b 100644 --- a/clang/test/Modules/pr59780.cppm +++ b/clang/test/Modules/pr59780.cppm @@ -9,6 +9,16 @@ // RUN: -triple %itanium_abi_triple -emit-llvm -o - | FileCheck %t/use.cpp // RUN: %clang_cc1 -std=c++20 %t/a.pcm -triple %itanium_abi_triple -emit-llvm -o - | FileCheck %t/a.cppm +// Test again with reduced BMI. +// RUN: %clang_cc1 -std=c++20 %t/a.cppm -triple %itanium_abi_triple -emit-module-interface \ +// RUN: -o %t/a.full.pcm +// RUN: %clang_cc1 -std=c++20 %t/a.cppm -triple %itanium_abi_triple -emit-reduced-module-interface \ +// RUN: -o %t/a.pcm +// RUN: %clang_cc1 -std=c++20 %t/use.cpp -fprebuilt-module-path=%t -S \ +// RUN: -triple %itanium_abi_triple -emit-llvm -o - | FileCheck %t/use.cpp +// RUN: %clang_cc1 -std=c++20 %t/a.full.pcm -triple %itanium_abi_triple -emit-llvm -o - | FileCheck %t/a.cppm + + //--- a.cppm export module a; diff --git a/clang/test/Modules/pr59999.cppm b/clang/test/Modules/pr59999.cppm index 23710de9fe1c..54452c26de47 100644 --- a/clang/test/Modules/pr59999.cppm +++ b/clang/test/Modules/pr59999.cppm @@ -11,6 +11,19 @@ // RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/Object.pcm \ // RUN: -fmodule-file=Module=%t/Module.pcm -S -emit-llvm -o - | FileCheck %t/Object.cppm +// Test again with reduced BMI. +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/Module.cppm \ +// RUN: -emit-reduced-module-interface -o %t/Module.pcm +// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/Object.cppm \ +// RUN: -fmodule-file=Module=%t/Module.pcm -emit-module-interface -o %t/Object.pcm +// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/Object.pcm \ +// RUN: -fmodule-file=Module=%t/Module.pcm -S -emit-llvm -o - | FileCheck %t/Object.cppm + + //--- Module.cppm export module Module; diff --git a/clang/test/Modules/pr60036.cppm b/clang/test/Modules/pr60036.cppm index 297132cfde60..ffbc5fd56c27 100644 --- a/clang/test/Modules/pr60036.cppm +++ b/clang/test/Modules/pr60036.cppm @@ -24,6 +24,20 @@ // RUN: -fmodule-file=c=%t/c.pcm -fmodule-file=d=%t/d.pcm -fmodule-file=e=%t/e.pcm \ // RUN: -fmodule-file=f=%t/f.pcm -verify -fsyntax-only +// Test again with reduced BMI +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/a.pcm +// RUN: %clang_cc1 -std=c++20 %t/b.cppm -emit-reduced-module-interface -fprebuilt-module-path=%t -o %t/b.pcm +// RUN: %clang_cc1 -std=c++20 %t/c.cppm -emit-reduced-module-interface -fprebuilt-module-path=%t -o %t/c.pcm +// RUN: %clang_cc1 -std=c++20 %t/d.cppm -emit-reduced-module-interface -fprebuilt-module-path=%t -o %t/d.pcm +// RUN: %clang_cc1 -std=c++20 %t/e.cppm -emit-reduced-module-interface -fprebuilt-module-path=%t -o %t/e.pcm +// RUN: %clang_cc1 -std=c++20 %t/f.cppm -emit-reduced-module-interface -fprebuilt-module-path=%t -o %t/f.pcm +// RUN: %clang_cc1 -std=c++20 %t/g.cppm -fprebuilt-module-path=%t -verify -fsyntax-only + + //--- a.cppm export module a; diff --git a/clang/test/Modules/pr60085.cppm b/clang/test/Modules/pr60085.cppm index fd6fd914a543..37d8b09350b4 100644 --- a/clang/test/Modules/pr60085.cppm +++ b/clang/test/Modules/pr60085.cppm @@ -14,6 +14,23 @@ // RUN: -S -emit-llvm -disable-llvm-passes -o - -fprebuilt-module-path=%t \ // RUN: | FileCheck %t/a.cppm +// Test again with reduced BMI. +// RUN: rm -rf %t +// RUN: mkdir %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/d.cppm \ +// RUN: -emit-reduced-module-interface -o %t/d.pcm +// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/c.cppm \ +// RUN: -emit-reduced-module-interface -o %t/c.pcm -fprebuilt-module-path=%t +// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/b.cppm \ +// RUN: -emit-reduced-module-interface -o %t/b.pcm -fprebuilt-module-path=%t +// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/a.cppm \ +// RUN: -emit-module-interface -o %t/a.pcm -fprebuilt-module-path=%t +// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/a.pcm \ +// RUN: -S -emit-llvm -disable-llvm-passes -o - -fprebuilt-module-path=%t \ +// RUN: | FileCheck %t/a.cppm + //--- d.cppm export module d; diff --git a/clang/test/Modules/pr60275.cppm b/clang/test/Modules/pr60275.cppm index 57b31c6952be..eb1ebc0e4330 100644 --- a/clang/test/Modules/pr60275.cppm +++ b/clang/test/Modules/pr60275.cppm @@ -5,7 +5,12 @@ // RUN: split-file %s %t // // RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple -emit-module-interface %t/a.cppm -o %t/a.pcm -// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/b.cpp -fmodule-file=%t/a.pcm -emit-llvm -o - | FileCheck %t/b.cpp +// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/b.cpp -fmodule-file=a=%t/a.pcm -emit-llvm -o - | FileCheck %t/b.cpp + +// Test again with reduced BMI +// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple -emit-reduced-module-interface %t/a.cppm -o %t/a.pcm +// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/b.cpp -fmodule-file=a=%t/a.pcm -emit-llvm -o - | FileCheck %t/b.cpp + //--- foo.h consteval void global() {} diff --git a/clang/test/Modules/pr60486.cppm b/clang/test/Modules/pr60486.cppm index 13802a4917e6..1100662c4321 100644 --- a/clang/test/Modules/pr60486.cppm +++ b/clang/test/Modules/pr60486.cppm @@ -7,6 +7,9 @@ // RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-module-interface -o %t/a.pcm // RUN: %clang_cc1 -std=c++20 -fmodule-file=a=%t/a.pcm %t/b.cppm -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/a.pcm +// RUN: %clang_cc1 -std=c++20 -fmodule-file=a=%t/a.pcm %t/b.cppm -fsyntax-only -verify + //--- foo.h template struct s { diff --git a/clang/test/Modules/pr60693.cppm b/clang/test/Modules/pr60693.cppm index c50791083a5b..6fb3de60e59b 100644 --- a/clang/test/Modules/pr60693.cppm +++ b/clang/test/Modules/pr60693.cppm @@ -7,6 +7,10 @@ // RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/a.cppm -emit-module-interface -o %t/a.pcm // RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple -fmodule-file=a=%t/a.pcm %t/c.cpp -S -emit-llvm -disable-llvm-passes -o - | FileCheck %t/c.cpp +// Test again with reduced BMI +// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/a.cppm -emit-reduced-module-interface -o %t/a.pcm +// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple -fmodule-file=a=%t/a.pcm %t/c.cpp -S -emit-llvm -disable-llvm-passes -o - | FileCheck %t/c.cpp + //--- a.cppm export module a; diff --git a/clang/test/Modules/pr60775.cppm b/clang/test/Modules/pr60775.cppm index 4db027ba3600..35eb92512f42 100644 --- a/clang/test/Modules/pr60775.cppm +++ b/clang/test/Modules/pr60775.cppm @@ -12,6 +12,19 @@ // RUN: %clang_cc1 -std=c++20 %t/f.cppm -emit-module-interface -fmodule-file=c=%t/c.pcm -o %t/f.pcm // RUN: %clang_cc1 -std=c++20 %t/g.cpp -fmodule-file=f=%t/f.pcm -fmodule-file=c=%t/c.pcm -verify -fsyntax-only +// Test again with reduced BMI +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 %t/a.cppm -I%t -emit-reduced-module-interface -o %t/a.pcm +// RUN: %clang_cc1 -std=c++20 %t/b.cpp -fmodule-file=a=%t/a.pcm -verify -fsyntax-only +// RUN: %clang_cc1 -std=c++20 %t/c.cppm -I%t -emit-reduced-module-interface -o %t/c.pcm +// RUN: %clang_cc1 -std=c++20 %t/d.cppm -emit-reduced-module-interface -fmodule-file=c=%t/c.pcm -o %t/d.pcm +// RUN: %clang_cc1 -std=c++20 %t/e.cpp -fmodule-file=d=%t/d.pcm -fmodule-file=c=%t/c.pcm -verify -fsyntax-only +// RUN: %clang_cc1 -std=c++20 %t/f.cppm -emit-reduced-module-interface -fmodule-file=c=%t/c.pcm -o %t/f.pcm +// RUN: %clang_cc1 -std=c++20 %t/g.cpp -fmodule-file=f=%t/f.pcm -fmodule-file=c=%t/c.pcm -verify -fsyntax-only + //--- initializer_list.h namespace std { typedef decltype(sizeof(int)) size_t; diff --git a/clang/test/Modules/pr60890.cppm b/clang/test/Modules/pr60890.cppm index 2560bec5b433..488b512aaac2 100644 --- a/clang/test/Modules/pr60890.cppm +++ b/clang/test/Modules/pr60890.cppm @@ -9,6 +9,12 @@ // RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/c.cppm -fprebuilt-module-path=%t -o %t/c.pcm // RUN: %clang_cc1 -std=c++20 %t/d.cpp -fprebuilt-module-path=%t -S -emit-llvm -o - +// Test again with reduced BMI +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/a.cppm -o %t/a.pcm +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/b.cppm -fprebuilt-module-path=%t -o %t/b.pcm +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/c.cppm -fprebuilt-module-path=%t -o %t/c.pcm +// RUN: %clang_cc1 -std=c++20 %t/d.cpp -fprebuilt-module-path=%t -S -emit-llvm -o - + //--- a.cppm export module a; diff --git a/clang/test/Modules/pr61065.cppm b/clang/test/Modules/pr61065.cppm index cf6fcdda78cd..c79d7ac4457a 100644 --- a/clang/test/Modules/pr61065.cppm +++ b/clang/test/Modules/pr61065.cppm @@ -10,6 +10,19 @@ // DISABLED: -fprebuilt-module-path=%t // DISABLED: %clang_cc1 -std=c++20 %t/d.cpp -fsyntax-only -verify -fprebuilt-module-path=%t +// Test again with reduced BMI +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/a.pcm +// RUN: %clang_cc1 -std=c++20 %t/b.cppm -emit-reduced-module-interface -o %t/b.pcm \ +// RUN: -fprebuilt-module-path=%t +// DISABLED: %clang_cc1 -std=c++20 %t/c.cppm -emit-reduced-module-interface -o %t/c.pcm \ +// DISABLED: -fprebuilt-module-path=%t +// DISABLED: %clang_cc1 -std=c++20 %t/d.cpp -fsyntax-only -verify -fprebuilt-module-path=%t + + //--- a.cppm export module a; diff --git a/clang/test/Modules/pr61065_2.cppm b/clang/test/Modules/pr61065_2.cppm index 10cc1a06b7e4..e898f4086af1 100644 --- a/clang/test/Modules/pr61065_2.cppm +++ b/clang/test/Modules/pr61065_2.cppm @@ -11,6 +11,21 @@ // RUN: -fprebuilt-module-path=%t // RUN: %clang_cc1 -std=c++20 %t/e.cpp -fsyntax-only -verify -fprebuilt-module-path=%t +// Test again with reduced BMI +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/a.pcm +// RUN: %clang_cc1 -std=c++20 %t/b.cppm -emit-reduced-module-interface -o %t/b.pcm \ +// RUN: -fprebuilt-module-path=%t +// RUN: %clang_cc1 -std=c++20 %t/c.cppm -emit-reduced-module-interface -o %t/c.pcm \ +// RUN: -fprebuilt-module-path=%t +// RUN: %clang_cc1 -std=c++20 %t/d.cppm -emit-reduced-module-interface -o %t/d.pcm \ +// RUN: -fprebuilt-module-path=%t +// RUN: %clang_cc1 -std=c++20 %t/e.cpp -fsyntax-only -verify -fprebuilt-module-path=%t + + //--- a.cppm export module a; diff --git a/clang/test/Modules/pr61067.cppm b/clang/test/Modules/pr61067.cppm index baee4b83de56..b7f9d22e2538 100644 --- a/clang/test/Modules/pr61067.cppm +++ b/clang/test/Modules/pr61067.cppm @@ -12,6 +12,20 @@ // RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/c.cpp -fmodule-file=a=%t/a.pcm \ // RUN: -S -emit-llvm -disable-llvm-passes -o - | FileCheck %t/c.cpp +// Test again with reduced BMI +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/a.cppm \ +// RUN: -emit-reduced-module-interface -o %t/a.pcm +// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/b.cppm \ +// RUN: -emit-module-interface -fmodule-file=a=%t/a.pcm -o %t/b.pcm +// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/b.pcm -S \ +// RUN: -emit-llvm -fmodule-file=a=%t/a.pcm -disable-llvm-passes -o - | FileCheck %t/b.cppm +// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/c.cpp -fmodule-file=a=%t/a.pcm \ +// RUN: -S -emit-llvm -disable-llvm-passes -o - | FileCheck %t/c.cpp + //--- a.cppm export module a; diff --git a/clang/test/Modules/pr61317.cppm b/clang/test/Modules/pr61317.cppm index 4b54d26dc5a6..9ed20e794706 100644 --- a/clang/test/Modules/pr61317.cppm +++ b/clang/test/Modules/pr61317.cppm @@ -8,6 +8,15 @@ // RUN: -fprebuilt-module-path=%t // RUN: %clang_cc1 -std=c++20 %t/Use.cpp -fprebuilt-module-path=%t -fsyntax-only -verify +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-reduced-module-interface -o %t/A.pcm +// RUN: %clang_cc1 -std=c++20 %t/B.cppm -emit-reduced-module-interface -o %t/B.pcm \ +// RUN: -fprebuilt-module-path=%t +// RUN: %clang_cc1 -std=c++20 %t/Use.cpp -fprebuilt-module-path=%t -fsyntax-only -verify + //--- foo.h #ifndef _FOO #define _FOO diff --git a/clang/test/Modules/pr61783.cppm b/clang/test/Modules/pr61783.cppm index 9cf773b0b282..c3bc853d2dee 100644 --- a/clang/test/Modules/pr61783.cppm +++ b/clang/test/Modules/pr61783.cppm @@ -9,6 +9,14 @@ // RUN: %clang_cc1 -std=c++20 -triple x86_64-pc-windows-msvc19.11.0 -fms-extensions %t/user.cpp -fmodule-file=mod=%t/mod.pcm \ // RUN: -S -emit-llvm -o - | FileCheck %t/user.cpp +// Test again with reduced BMI +// RUN: %clang_cc1 -std=c++20 -triple x86_64-pc-windows-msvc19.11.0 -fms-extensions %t/mod.cppm -emit-reduced-module-interface \ +// RUN: -o %t/mod.pcm +// RUN: %clang_cc1 -std=c++20 -triple x86_64-pc-windows-msvc19.11.0 -fms-extensions %t/mod.pcm -S -emit-llvm -o - | \ +// RUN: FileCheck %t/mod.cppm +// RUN: %clang_cc1 -std=c++20 -triple x86_64-pc-windows-msvc19.11.0 -fms-extensions %t/user.cpp -fmodule-file=mod=%t/mod.pcm \ +// RUN: -S -emit-llvm -o - | FileCheck %t/user.cpp + //--- mod.cppm module; diff --git a/clang/test/Modules/pr61892.cppm b/clang/test/Modules/pr61892.cppm index 99d02f36b2b5..7b8905036cd4 100644 --- a/clang/test/Modules/pr61892.cppm +++ b/clang/test/Modules/pr61892.cppm @@ -2,11 +2,25 @@ // RUN: mkdir -p %t // RUN: split-file %s %t // +// RUNX: %clang_cc1 -std=c++20 -triple %itanium_abi_triple \ +// RUNX: -emit-module-interface %t/a.cppm -o %t/a.pcm +// RUNX: %clang_cc1 -std=c++20 -triple %itanium_abi_triple \ +// RUNX: %t/b.cpp -fmodule-file=a=%t/a.pcm -disable-llvm-passes \ +// RUNX: -emit-llvm -o - | FileCheck %t/b.cpp +// RUNX: %clang_cc1 -std=c++20 -triple %itanium_abi_triple \ +// RUNX: %t/c.cpp -fmodule-file=a=%t/a.pcm -disable-llvm-passes \ +// RUNX: -emit-llvm -o - | FileCheck %t/c.cpp + +// Test again with reduced BMI. +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t +// // RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple \ -// RUN: -emit-module-interface %t/a.cppm -o %t/a.pcm -// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple \ -// RUN: %t/b.cpp -fmodule-file=a=%t/a.pcm -disable-llvm-passes \ -// RUN: -emit-llvm -o - | FileCheck %t/b.cpp +// RUN: -emit-reduced-module-interface %t/a.cppm -o %t/a.pcm +// RUNX: %clang_cc1 -std=c++20 -triple %itanium_abi_triple \ +// RUNX: %t/b.cpp -fmodule-file=a=%t/a.pcm -disable-llvm-passes \ +// RUNX: -emit-llvm -o - | FileCheck %t/b.cpp // RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple \ // RUN: %t/c.cpp -fmodule-file=a=%t/a.pcm -disable-llvm-passes \ // RUN: -emit-llvm -o - | FileCheck %t/c.cpp @@ -23,20 +37,10 @@ struct integer { export template int a = static_cast(integer()); -struct s { - ~s(); - operator int() const; -}; - -export template -auto d = s(); - int aa() { - return a + d; + return a; } -int dynamic_func(); -export inline int dynamic_var = dynamic_func(); //--- b.cpp import a; @@ -53,13 +57,9 @@ void b() {} //--- c.cpp import a; int c() { - return a + d + dynamic_var; + return a; } // The used variables are generated normally // CHECK-DAG: @_ZW1a1aIvE = -// CHECK-DAG: @_ZW1a1dIvE = -// CHECK-DAG: @_ZW1a11dynamic_var = linkonce_odr // CHECK-DAG: @_ZGVW1a1aIvE = -// CHECk-DAG: @_ZGVW1a1dIvE = -// CHECK-DAG: @_ZGVW1a11dynamic_var = linkonce_odr diff --git a/clang/test/Modules/pr62158.cppm b/clang/test/Modules/pr62158.cppm index 7a0761df7715..bb488fff108f 100644 --- a/clang/test/Modules/pr62158.cppm +++ b/clang/test/Modules/pr62158.cppm @@ -6,6 +6,15 @@ // RUN: %clang_cc1 -std=c++20 %t/main.cpp -fmodule-file=lib=%t/lib.pcm \ // RUN: -verify -fsyntax-only +// Test again with reduced BMI +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/lib.cppm -o %t/lib.pcm +// RUN: %clang_cc1 -std=c++20 %t/main.cpp -fmodule-file=lib=%t/lib.pcm \ +// RUN: -verify -fsyntax-only + //--- header.h namespace lib::inline __1 { template diff --git a/clang/test/Modules/pr62359.cppm b/clang/test/Modules/pr62359.cppm index 4632457e57f1..69acc3ce303a 100644 --- a/clang/test/Modules/pr62359.cppm +++ b/clang/test/Modules/pr62359.cppm @@ -12,6 +12,22 @@ // RUN: %clang_cc1 -std=c++20 -fopenmp %t/use.cpp -fmodule-file=hello=%t/Hello.pcm -fsyntax-only -verify // RUN: %clang_cc1 -std=c++20 -fopenmp %t/use2.cpp -fmodule-file=hello=%t/Hello.pcm -fsyntax-only -verify +// Test again with reduced BMI +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/Hello.cppm -o %t/Hello.pcm +// RUN: not %clang_cc1 -std=c++20 -fopenmp %t/use.cpp -fmodule-file=hello=%t/Hello.pcm -fsyntax-only \ +// RUN: 2>&1 | FileCheck %t/use.cpp +// RUN: not %clang_cc1 -std=c++20 -fopenmp %t/use2.cpp -fmodule-file=hello=%t/Hello.pcm -fsyntax-only \ +// RUN: 2>&1 | FileCheck %t/use2.cpp +// +// RUN: %clang_cc1 -std=c++20 -fopenmp -emit-reduced-module-interface %t/Hello.cppm -o %t/Hello.pcm +// RUN: %clang_cc1 -std=c++20 -fopenmp %t/use.cpp -fmodule-file=hello=%t/Hello.pcm -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 -fopenmp %t/use2.cpp -fmodule-file=hello=%t/Hello.pcm -fsyntax-only -verify + + //--- Hello.cppm export module hello; export void hello() { diff --git a/clang/test/Modules/pr62589.cppm b/clang/test/Modules/pr62589.cppm index 4164c3405ac0..c5aec3ed8184 100644 --- a/clang/test/Modules/pr62589.cppm +++ b/clang/test/Modules/pr62589.cppm @@ -5,6 +5,9 @@ // RUN: %clang_cc1 -std=c++23 -emit-module-interface %t/a.cppm -o %t/a.pcm // RUN: %clang_cc1 -std=c++23 %t/b.cpp -fmodule-file=a=%t/a.pcm -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++23 -emit-reduced-module-interface %t/a.cppm -o %t/a.pcm +// RUN: %clang_cc1 -std=c++23 %t/b.cpp -fmodule-file=a=%t/a.pcm -fsyntax-only -verify + //--- foo.h class TypeA {}; diff --git a/clang/test/Modules/pr62705.cppm b/clang/test/Modules/pr62705.cppm index 00769d2277f4..9d996ae297d7 100644 --- a/clang/test/Modules/pr62705.cppm +++ b/clang/test/Modules/pr62705.cppm @@ -10,6 +10,14 @@ // RUN: %clang_cc1 %t/b.pcm -std=c++20 -triple %itanium_abi_triple \ // RUN: -fmodule-file=a=%t/a.pcm -emit-llvm -o - | FileCheck %t/b.cppm +// RUN: %clang_cc1 %t/a.cppm -std=c++20 -triple %itanium_abi_triple \ +// RUN: -emit-reduced-module-interface -o %t/a.pcm +// RUN: %clang_cc1 %t/b.cppm -std=c++20 -triple %itanium_abi_triple \ +// RUN: -emit-module-interface -o %t/b.pcm \ +// RUN: -fmodule-file=a=%t/a.pcm +// RUN: %clang_cc1 %t/b.pcm -std=c++20 -triple %itanium_abi_triple \ +// RUN: -fmodule-file=a=%t/a.pcm -emit-llvm -o - | FileCheck %t/b.cppm + //--- foo.h namespace n { diff --git a/clang/test/Modules/pr62796.cppm b/clang/test/Modules/pr62796.cppm index f96e54bc6ade..58b72164e88b 100644 --- a/clang/test/Modules/pr62796.cppm +++ b/clang/test/Modules/pr62796.cppm @@ -6,6 +6,10 @@ // RUN: %clang_cc1 -std=c++20 %t/Use.cpp -fmodule-file=Fibonacci.Cache=%t/Cache.pcm \ // RUN: -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/Cache.cppm -o %t/Cache.pcm +// RUN: %clang_cc1 -std=c++20 %t/Use.cpp -fmodule-file=Fibonacci.Cache=%t/Cache.pcm \ +// RUN: -fsyntax-only -verify + //--- Cache.cppm export module Fibonacci.Cache; diff --git a/clang/test/Modules/pr62943.cppm b/clang/test/Modules/pr62943.cppm index 27868b78220f..c3a373814a43 100644 --- a/clang/test/Modules/pr62943.cppm +++ b/clang/test/Modules/pr62943.cppm @@ -9,6 +9,18 @@ // RUN: %clang_cc1 -std=c++20 %t/use.cpp -fprebuilt-module-path=%t \ // RUN: -fsyntax-only -verify +// Test again with reduced BMI. +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/a.pcm +// RUN: %clang_cc1 -std=c++20 %t/b.cppm -emit-reduced-module-interface -o %t/b.pcm +// RUN: %clang_cc1 -std=c++20 %t/c.cppm -emit-reduced-module-interface \ +// RUN: -fprebuilt-module-path=%t -o %t/c.pcm +// RUN: %clang_cc1 -std=c++20 %t/use.cpp -fprebuilt-module-path=%t \ +// RUN: -fsyntax-only -verify + //--- foo.h #ifndef FOO_H #define FOO_H diff --git a/clang/test/Modules/pr63544.cppm b/clang/test/Modules/pr63544.cppm index 16224cfd0109..f079abaed09d 100644 --- a/clang/test/Modules/pr63544.cppm +++ b/clang/test/Modules/pr63544.cppm @@ -8,6 +8,18 @@ // RUN: -fprebuilt-module-path=%t // RUN: %clang_cc1 -std=c++23 %t/pr63544.cpp -fprebuilt-module-path=%t -fsyntax-only -verify +// Test again with reduced BMI. +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++23 %t/a.cppm -emit-reduced-module-interface -o %t/m-a.pcm +// RUN: %clang_cc1 -std=c++23 %t/b.cppm -emit-reduced-module-interface -o %t/m-b.pcm +// RUN: %clang_cc1 -std=c++23 %t/m.cppm -emit-reduced-module-interface -o %t/m.pcm \ +// RUN: -fprebuilt-module-path=%t +// RUN: %clang_cc1 -std=c++23 %t/pr63544.cpp -fprebuilt-module-path=%t -fsyntax-only -verify + + //--- foo.h namespace std { diff --git a/clang/test/Modules/pr63595.cppm b/clang/test/Modules/pr63595.cppm index 13a5f84a3e71..7c5395e065de 100644 --- a/clang/test/Modules/pr63595.cppm +++ b/clang/test/Modules/pr63595.cppm @@ -6,6 +6,16 @@ // RUN: %clang_cc1 -std=c++20 -emit-module-interface -I%t %t/module2.cppm -o %t/module2.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/merge.cpp -verify -fsyntax-only +// Test again with reduced BMI. +// RUN: rm -rf %t +// RUN: mkdir %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface -I%t %t/module1.cppm -o %t/module1.pcm +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface -I%t %t/module2.cppm -o %t/module2.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/merge.cpp -verify -fsyntax-only + + //--- header.h namespace NS { template diff --git a/clang/test/Modules/pr67627.cppm b/clang/test/Modules/pr67627.cppm index 3d4410229080..d3f8496c47c2 100644 --- a/clang/test/Modules/pr67627.cppm +++ b/clang/test/Modules/pr67627.cppm @@ -5,6 +5,10 @@ // RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-module-interface -o %t/A.pcm // RUN: %clang_cc1 -std=c++20 %t/B.cppm -fmodule-file=A=%t/A.pcm -fsyntax-only -verify +// RUN: rm %t/A.pcm +// RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-reduced-module-interface -o %t/A.pcm +// RUN: %clang_cc1 -std=c++20 %t/B.cppm -fmodule-file=A=%t/A.pcm -fsyntax-only -verify + //--- A.cppm export module A; diff --git a/clang/test/Modules/pr67893.cppm b/clang/test/Modules/pr67893.cppm index 00b024ecc2eb..58990cec01d6 100644 --- a/clang/test/Modules/pr67893.cppm +++ b/clang/test/Modules/pr67893.cppm @@ -9,6 +9,15 @@ // RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++20 %t/m.pcm \ // RUN: -fprebuilt-module-path=%t -S -emit-llvm -o - | FileCheck %t/m.cppm +// Test again with reduced BMI +// +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++20 %t/a.cppm \ +// RUN: -emit-reduced-module-interface -o %t/a.pcm +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++20 %t/m.cppm \ +// RUN: -emit-reduced-module-interface -fprebuilt-module-path=%t -o %t/m.pcm +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++20 %t/m.pcm \ +// RUN: -fprebuilt-module-path=%t -S -emit-llvm -o - | FileCheck %t/m.cppm + //--- a.cppm export module a; export struct A { diff --git a/clang/test/Modules/predefined.cpp b/clang/test/Modules/predefined.cpp index fbe0c4e23ca5..8f897f5ace93 100644 --- a/clang/test/Modules/predefined.cpp +++ b/clang/test/Modules/predefined.cpp @@ -5,6 +5,9 @@ // RUN: %clang_cc1 -x c++ -std=c++20 -emit-module-interface a.h -o a.pcm -fms-extensions -verify // RUN: %clang_cc1 -std=c++20 a.cpp -fmodule-file=A=a.pcm -fms-extensions -fsyntax-only -verify +// RUN: %clang_cc1 -x c++ -std=c++20 -emit-reduced-module-interface a.h -o a.pcm -fms-extensions -verify +// RUN: %clang_cc1 -std=c++20 a.cpp -fmodule-file=A=a.pcm -fms-extensions -fsyntax-only -verify + //--- a.h // expected-no-diagnostics diff --git a/clang/test/Modules/preferred_name.cppm b/clang/test/Modules/preferred_name.cppm index 46ad96cb1abc..2f1705867845 100644 --- a/clang/test/Modules/preferred_name.cppm +++ b/clang/test/Modules/preferred_name.cppm @@ -8,6 +8,16 @@ // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t -I%t %t/Use.cppm -verify -fsyntax-only // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t -I%t %t/Use1.cpp -verify -fsyntax-only // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t -I%t %t/Use2.cpp -verify -fsyntax-only + +// Test again with reduced BMI. +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-reduced-module-interface -o %t/A.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t -I%t %t/Use.cppm -verify -fsyntax-only +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t -I%t %t/Use1.cpp -verify -fsyntax-only +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t -I%t %t/Use2.cpp -verify -fsyntax-only // //--- foo.h template diff --git a/clang/test/Modules/redefinition-merges.cppm b/clang/test/Modules/redefinition-merges.cppm index 9ab4006f985f..13032b22ee60 100644 --- a/clang/test/Modules/redefinition-merges.cppm +++ b/clang/test/Modules/redefinition-merges.cppm @@ -12,6 +12,12 @@ // RUN: %clang_cc1 -std=c++20 -I%t %t/M.cppm -emit-module-interface -o %t/M.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use1.cpp -verify -fsyntax-only // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use2.cpp -verify -fsyntax-only + +// / Test again with reduced BMI. +// RUN: %clang_cc1 -std=c++20 -I%t %t/M.cppm -emit-reduced-module-interface -o %t/M.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use1.cpp -verify -fsyntax-only +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use2.cpp -verify -fsyntax-only + // //--- foo.h #ifndef FOO diff --git a/clang/test/Modules/redundant-template-default-arg.cpp b/clang/test/Modules/redundant-template-default-arg.cpp index 6807b45e5139..20a806c4c818 100644 --- a/clang/test/Modules/redundant-template-default-arg.cpp +++ b/clang/test/Modules/redundant-template-default-arg.cpp @@ -5,6 +5,9 @@ // RUN: %clang_cc1 -std=c++20 %t/foo.cppm -I%t -emit-module-interface -o %t/foo.pcm // RUN: %clang_cc1 -fprebuilt-module-path=%t -std=c++20 %t/use.cpp -I%t -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 %t/foo.cppm -I%t -emit-reduced-module-interface -o %t/foo.pcm +// RUN: %clang_cc1 -fprebuilt-module-path=%t -std=c++20 %t/use.cpp -I%t -fsyntax-only -verify + //--- foo.h template T u; diff --git a/clang/test/Modules/redundant-template-default-arg2.cpp b/clang/test/Modules/redundant-template-default-arg2.cpp index 41deb112cfa6..ae1f0c7e69cc 100644 --- a/clang/test/Modules/redundant-template-default-arg2.cpp +++ b/clang/test/Modules/redundant-template-default-arg2.cpp @@ -5,6 +5,9 @@ // RUN: %clang_cc1 -std=c++20 %t/foo.cppm -I%t -emit-module-interface -o %t/foo.pcm // RUN: %clang_cc1 -fprebuilt-module-path=%t -std=c++20 %t/use.cpp -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 %t/foo.cppm -I%t -emit-reduced-module-interface -o %t/foo.pcm +// RUN: %clang_cc1 -fprebuilt-module-path=%t -std=c++20 %t/use.cpp -fsyntax-only -verify + //--- foo.cppm export module foo; export template diff --git a/clang/test/Modules/redundant-template-default-arg3.cpp b/clang/test/Modules/redundant-template-default-arg3.cpp index 8bb222ac91ff..e4464c40e976 100644 --- a/clang/test/Modules/redundant-template-default-arg3.cpp +++ b/clang/test/Modules/redundant-template-default-arg3.cpp @@ -5,6 +5,9 @@ // RUN: %clang_cc1 -std=c++20 %t/foo.cppm -I%t -emit-module-interface -o %t/foo.pcm // RUN: %clang_cc1 -fprebuilt-module-path=%t -std=c++20 %t/use.cpp -I%t/. -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 %t/foo.cppm -I%t -emit-reduced-module-interface -o %t/foo.pcm +// RUN: %clang_cc1 -fprebuilt-module-path=%t -std=c++20 %t/use.cpp -I%t/. -fsyntax-only -verify + //--- foo.h template T v; diff --git a/clang/test/Modules/search-partitions.cpp b/clang/test/Modules/search-partitions.cpp index 571160def7e9..92732958db94 100644 --- a/clang/test/Modules/search-partitions.cpp +++ b/clang/test/Modules/search-partitions.cpp @@ -14,6 +14,22 @@ // RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/moduleA.cpp \ // RUN: -fprebuilt-module-path=%t +// Test again with reduced BMI +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/partition1.cpp \ +// RUN: -o %t/A-Part1.pcm + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/partition2.cpp \ +// RUN: -o %t/A-Part2.pcm + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/partition3.cpp \ +// RUN: -o %t/A-Part3.pcm + +// RUN: %clang_cc1 -std=c++20 -fsyntax-only %t/moduleA.cpp -fprebuilt-module-path=%t + // expected-no-diagnostics //--- partition1.cpp diff --git a/clang/test/Modules/seperated-member-function-definition-for-template-class.cppm b/clang/test/Modules/seperated-member-function-definition-for-template-class.cppm index e32da39d9df1..1465c33c3625 100644 --- a/clang/test/Modules/seperated-member-function-definition-for-template-class.cppm +++ b/clang/test/Modules/seperated-member-function-definition-for-template-class.cppm @@ -12,6 +12,18 @@ // RUN: -fprebuilt-module-path=%t // RUN: %clang_cc1 -std=c++20 %t/use.cpp -fsyntax-only -verify -fprebuilt-module-path=%t +// Test again with reduced BMI +// RUN: rm -rf %t +// RUN: mkdir %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 %t/base.cppm -emit-reduced-module-interface -o %t/package-base.pcm +// RUN: %clang_cc1 -std=c++20 %t/child.cppm -emit-reduced-module-interface -o %t/package-child.pcm \ +// RUN: -fprebuilt-module-path=%t +// RUN: %clang_cc1 -std=c++20 %t/package.cppm -emit-reduced-module-interface -o %t/package.pcm \ +// RUN: -fprebuilt-module-path=%t +// RUN: %clang_cc1 -std=c++20 %t/use.cpp -fsyntax-only -verify -fprebuilt-module-path=%t + //--- base.cppm export module package:base; diff --git a/clang/test/Modules/template-function-specialization.cpp b/clang/test/Modules/template-function-specialization.cpp index 3eac92e7edb9..1b6bf2de6ba1 100644 --- a/clang/test/Modules/template-function-specialization.cpp +++ b/clang/test/Modules/template-function-specialization.cpp @@ -4,7 +4,10 @@ // // RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/foo.cppm -o %t/foo.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only -// + +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/foo.cppm -o %t/foo.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only + //--- foo.cppm module; # 3 __FILE__ 1 // use the next physical line number here (and below) diff --git a/clang/test/Modules/template-lambdas.cppm b/clang/test/Modules/template-lambdas.cppm index 69117a1a04fc..e82cb1f3ad85 100644 --- a/clang/test/Modules/template-lambdas.cppm +++ b/clang/test/Modules/template-lambdas.cppm @@ -12,6 +12,21 @@ // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -fsyntax-only \ // RUN: -verify -DUSE_LAMBDA2 +// Test again with reduced BMI +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 %t/template_lambdas.cppm -emit-reduced-module-interface \ +// RUN: -o %t/lambdas.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -fsyntax-only \ +// RUN: -verify +// +// RUN: %clang_cc1 -std=c++20 %t/template_lambdas2.cppm -emit-reduced-module-interface \ +// RUN: -o %t/lambdas2.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -fsyntax-only \ +// RUN: -verify -DUSE_LAMBDA2 + //--- lambdas.h auto l1 = []() constexpr -> int { return I; diff --git a/clang/test/Modules/template-pack.cppm b/clang/test/Modules/template-pack.cppm index eca17f31f015..278c1c2d54cc 100644 --- a/clang/test/Modules/template-pack.cppm +++ b/clang/test/Modules/template-pack.cppm @@ -5,6 +5,9 @@ // RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/a.cppm -o %t/a.pcm // RUN: %clang_cc1 -std=c++20 %t/b.cppm -fprebuilt-module-path=%t -fsyntax-only -verify +// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/a.cppm -o %t/a.pcm +// RUN: %clang_cc1 -std=c++20 %t/b.cppm -fprebuilt-module-path=%t -fsyntax-only -verify + //--- foo.h namespace std diff --git a/clang/test/Modules/template_default_argument.cpp b/clang/test/Modules/template_default_argument.cpp index 5a7d1c04cf18..202f8dd40d7a 100644 --- a/clang/test/Modules/template_default_argument.cpp +++ b/clang/test/Modules/template_default_argument.cpp @@ -4,6 +4,9 @@ // // RUN: %clang_cc1 -std=c++20 %t/B.cppm -emit-module-interface -o %t/B.pcm // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -fsyntax-only -verify + +// RUN: %clang_cc1 -std=c++20 %t/B.cppm -emit-reduced-module-interface -o %t/B.pcm +// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -fsyntax-only -verify // //--- templ.h template diff --git a/clang/unittests/Sema/SemaNoloadLookupTest.cpp b/clang/unittests/Sema/SemaNoloadLookupTest.cpp index b24c72cba407..cf89c7331e4e 100644 --- a/clang/unittests/Sema/SemaNoloadLookupTest.cpp +++ b/clang/unittests/Sema/SemaNoloadLookupTest.cpp @@ -64,7 +64,7 @@ public: CIOpts.VFS = llvm::vfs::createPhysicalFileSystem(); std::string CacheBMIPath = - llvm::Twine(TestDir + "/" + ModuleName + " .pcm").str(); + llvm::Twine(TestDir + "/" + ModuleName + ".pcm").str(); std::string PrebuiltModulePath = "-fprebuilt-module-path=" + TestDir.str().str(); const char *Args[] = {"clang++", @@ -75,9 +75,7 @@ public: TestDir.c_str(), "-I", TestDir.c_str(), - FileName.c_str(), - "-o", - CacheBMIPath.c_str()}; + FileName.c_str()}; std::shared_ptr Invocation = createInvocation(Args, CIOpts); EXPECT_TRUE(Invocation); @@ -85,7 +83,8 @@ public: CompilerInstance Instance; Instance.setDiagnostics(Diags.get()); Instance.setInvocation(Invocation); - GenerateModuleInterfaceAction Action; + Instance.getFrontendOpts().OutputFile = CacheBMIPath; + GenerateReducedModuleInterfaceAction Action; EXPECT_TRUE(Instance.ExecuteAction(Action)); EXPECT_FALSE(Diags->hasErrorOccurred()); diff --git a/clang/unittests/Serialization/ForceCheckFileInputTest.cpp b/clang/unittests/Serialization/ForceCheckFileInputTest.cpp index ed0daa43436e..ad8892b8c8be 100644 --- a/clang/unittests/Serialization/ForceCheckFileInputTest.cpp +++ b/clang/unittests/Serialization/ForceCheckFileInputTest.cpp @@ -69,9 +69,9 @@ export int aa = 43; CIOpts.Diags = Diags; CIOpts.VFS = llvm::vfs::createPhysicalFileSystem(); - const char *Args[] = { - "clang++", "-std=c++20", "--precompile", "-working-directory", - TestDir.c_str(), "a.cppm", "-o", BMIPath.c_str()}; + const char *Args[] = {"clang++", "-std=c++20", + "--precompile", "-working-directory", + TestDir.c_str(), "a.cppm"}; std::shared_ptr Invocation = createInvocation(Args, CIOpts); EXPECT_TRUE(Invocation); @@ -88,6 +88,8 @@ export int aa = 43; Instance.setDiagnostics(Diags.get()); Instance.setInvocation(Invocation); + Instance.getFrontendOpts().OutputFile = BMIPath; + if (auto VFSWithRemapping = createVFSFromCompilerInvocation( Instance.getInvocation(), Instance.getDiagnostics(), CIOpts.VFS)) CIOpts.VFS = VFSWithRemapping; @@ -95,7 +97,7 @@ export int aa = 43; Instance.getHeaderSearchOpts().ValidateASTInputFilesContent = true; - GenerateModuleInterfaceAction Action; + GenerateReducedModuleInterfaceAction Action; EXPECT_TRUE(Instance.ExecuteAction(Action)); EXPECT_FALSE(Diags->hasErrorOccurred()); } diff --git a/clang/unittests/Serialization/NoCommentsTest.cpp b/clang/unittests/Serialization/NoCommentsTest.cpp index 2632a6337807..a0a564aeff9a 100644 --- a/clang/unittests/Serialization/NoCommentsTest.cpp +++ b/clang/unittests/Serialization/NoCommentsTest.cpp @@ -90,9 +90,9 @@ void foo() {} CIOpts.VFS = llvm::vfs::createPhysicalFileSystem(); std::string CacheBMIPath = llvm::Twine(TestDir + "/Comments.pcm").str(); - const char *Args[] = { - "clang++", "-std=c++20", "--precompile", "-working-directory", - TestDir.c_str(), "Comments.cppm", "-o", CacheBMIPath.c_str()}; + const char *Args[] = {"clang++", "-std=c++20", + "--precompile", "-working-directory", + TestDir.c_str(), "Comments.cppm"}; std::shared_ptr Invocation = createInvocation(Args, CIOpts); ASSERT_TRUE(Invocation); @@ -100,7 +100,8 @@ void foo() {} CompilerInstance Instance; Instance.setDiagnostics(Diags.get()); Instance.setInvocation(Invocation); - GenerateModuleInterfaceAction Action; + Instance.getFrontendOpts().OutputFile = CacheBMIPath; + GenerateReducedModuleInterfaceAction Action; ASSERT_TRUE(Instance.ExecuteAction(Action)); ASSERT_FALSE(Diags->hasErrorOccurred()); diff --git a/clang/unittests/Serialization/VarDeclConstantInitTest.cpp b/clang/unittests/Serialization/VarDeclConstantInitTest.cpp index 7efa1c1d64a9..5cbbfb9ff003 100644 --- a/clang/unittests/Serialization/VarDeclConstantInitTest.cpp +++ b/clang/unittests/Serialization/VarDeclConstantInitTest.cpp @@ -96,10 +96,9 @@ export namespace Fibonacci CIOpts.Diags = Diags; CIOpts.VFS = llvm::vfs::createPhysicalFileSystem(); - std::string CacheBMIPath = llvm::Twine(TestDir + "/Cached.pcm").str(); - const char *Args[] = { - "clang++", "-std=c++20", "--precompile", "-working-directory", - TestDir.c_str(), "Cached.cppm", "-o", CacheBMIPath.c_str()}; + const char *Args[] = {"clang++", "-std=c++20", + "--precompile", "-working-directory", + TestDir.c_str(), "Cached.cppm"}; std::shared_ptr Invocation = createInvocation(Args, CIOpts); ASSERT_TRUE(Invocation); @@ -108,7 +107,11 @@ export namespace Fibonacci CompilerInstance Instance; Instance.setDiagnostics(Diags.get()); Instance.setInvocation(Invocation); - GenerateModuleInterfaceAction Action; + + std::string CacheBMIPath = llvm::Twine(TestDir + "/Cached.pcm").str(); + Instance.getFrontendOpts().OutputFile = CacheBMIPath; + + GenerateReducedModuleInterfaceAction Action; ASSERT_TRUE(Instance.ExecuteAction(Action)); ASSERT_FALSE(Diags->hasErrorOccurred()); -- GitLab From a4703e5bccc63eca45e0e4bff8f04f82d8622ebe Mon Sep 17 00:00:00 2001 From: Jie Fu Date: Fri, 8 Mar 2024 10:19:07 +0800 Subject: [PATCH 551/929] [clang] Remove std::move in GenerateModuleAction::CreateMultiplexConsumer (NFC) llvm-project/clang/lib/Frontend/FrontendActions.cpp:213:10: error: moving a local object in a return statement prevents copy elision [-Werror,-Wpessimizing-move] 213 | return std::move(Consumers); | ^ /Users/jiefu/llvm-project/clang/lib/Frontend/FrontendActions.cpp:213:10: note: remove std::move call here 213 | return std::move(Consumers); | ^~~~~~~~~~ ~ 1 error generated. --- clang/lib/Frontend/FrontendActions.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp index cd9b9923421c..50338bfa670f 100644 --- a/clang/lib/Frontend/FrontendActions.cpp +++ b/clang/lib/Frontend/FrontendActions.cpp @@ -210,7 +210,7 @@ GenerateModuleAction::CreateMultiplexConsumer(CompilerInstance &CI, +CI.getFrontendOpts().BuildingImplicitModule)); Consumers.push_back(CI.getPCHContainerWriter().CreatePCHContainerGenerator( CI, std::string(InFile), OutputFile, std::move(OS), Buffer)); - return std::move(Consumers); + return Consumers; } std::unique_ptr -- GitLab From 448419007e1bb8a9e2edfe2c1c87fef6d104442a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolai=20H=C3=A4hnle?= Date: Thu, 23 Nov 2023 05:34:05 +0100 Subject: [PATCH 552/929] update_test_checks: precommit a test case The test case demonstrates how meta variables are needlessly renamed, making diffs harder to read. --- .../Inputs/stable_ir_values.ll | 22 ++++++++++ .../Inputs/stable_ir_values.ll.expected | 23 ++++++++++ .../Inputs/stable_ir_values2.ll | 30 +++++++++++++ .../Inputs/stable_ir_values2.ll.expected | 26 ++++++++++++ .../Inputs/stable_ir_values3.ll | 38 +++++++++++++++++ .../Inputs/stable_ir_values3.ll.expected | 38 +++++++++++++++++ .../Inputs/stable_ir_values4.ll | 41 ++++++++++++++++++ .../Inputs/stable_ir_values4.ll.expected | 42 +++++++++++++++++++ .../update_test_checks/stable_ir_values.test | 2 + .../update_test_checks/stable_ir_values2.test | 2 + .../update_test_checks/stable_ir_values3.test | 2 + .../update_test_checks/stable_ir_values4.test | 2 + 12 files changed, 268 insertions(+) create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll.expected create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll.expected create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values4.ll create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values4.ll.expected create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values.test create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values2.test create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values3.test create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values4.test diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll new file mode 100644 index 000000000000..8457bf7dc40a --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll @@ -0,0 +1,22 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 +; RUN: opt < %s -S | FileCheck %s + +; The assumption underlying this test is that there are pre-existing check lines +; but something has changed, and we would like to avoid needless changes of +; meta variable names so that diffs end up being easier to read, e.g. avoid +; changing X_I33 into X_I34 or renumbering the various TMP variables. + +define i32 @func({i32, i32} %x, i32 %y) { +; CHECK-LABEL: define i32 @func( +; CHECK-SAME: { i32, i32 } [[X:%.*]], i32 [[Y:%.*]]) { +; CHECK-NEXT: [[X_I33:%.*]] = extractvalue { i32, i32 } [[X]], 0 +; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[X_I33]], [[Y]] +; CHECK-NEXT: [[TMP2:%.*]] = mul i32 [[TMP1]], 3 +; CHECK-NEXT: ret i32 [[TMP2]] +; + %x.i34 = extractvalue {i32, i32} %x, 0 + %1 = add i32 %y, 1 + %2 = add i32 %x.i34, %1 + %3 = mul i32 %2, 3 + ret i32 %3 +} diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected new file mode 100644 index 000000000000..5142e3ed32ba --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected @@ -0,0 +1,23 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 +; RUN: opt < %s -S | FileCheck %s + +; The assumption underlying this test is that there are pre-existing check lines +; but something has changed, and we would like to avoid needless changes of +; meta variable names so that diffs end up being easier to read, e.g. avoid +; changing X_I33 into X_I34 or renumbering the various TMP variables. + +define i32 @func({i32, i32} %x, i32 %y) { +; CHECK-LABEL: define i32 @func( +; CHECK-SAME: { i32, i32 } [[X:%.*]], i32 [[Y:%.*]]) { +; CHECK-NEXT: [[X_I34:%.*]] = extractvalue { i32, i32 } [[X]], 0 +; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[Y]], 1 +; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[X_I34]], [[TMP1]] +; CHECK-NEXT: [[TMP3:%.*]] = mul i32 [[TMP2]], 3 +; CHECK-NEXT: ret i32 [[TMP3]] +; + %x.i34 = extractvalue {i32, i32} %x, 0 + %1 = add i32 %y, 1 + %2 = add i32 %x.i34, %1 + %3 = mul i32 %2, 3 + ret i32 %3 +} diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll new file mode 100644 index 000000000000..d05c26241f87 --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll @@ -0,0 +1,30 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 +; RUN: opt < %s -S | FileCheck %s + +define i32 @func(i32 %x) { +; CHECK-LABEL: define i32 @func( +; CHECK-SAME: i32 [[X:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[X]], 0 +; CHECK-NEXT: [[TMP2:%.*]] = call i32 @foo(i1 [[TMP1]]) +; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[X]], 1 +; CHECK-NEXT: [[TMP4:%.*]] = call i32 @foo(i1 [[TMP3]]) +; CHECK-NEXT: [[TMP5:%.*]] = icmp ne i32 [[TMP4]], 0 +; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 [[TMP2]] +; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[X]], 2 +; CHECK-NEXT: [[TMP8:%.*]] = call i32 @foo(i1 [[TMP7]]) +; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i32 [[TMP8]], 0 +; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[TMP6]] +; CHECK-NEXT: ret i32 [[TMP10]] +; + %1 = icmp eq i32 %x, 0 + %2 = call i32 @foo(i1 %1) + + %3 = icmp eq i32 %x, 2 + %4 = call i32 @foo(i1 %3) + %5 = icmp ne i32 %4, 0 + %6 = select i1 %5, i32 %4, i32 %2 + + ret i32 %6 +} + +declare i32 @foo(i1) diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll.expected new file mode 100644 index 000000000000..53f60bda8ee5 --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll.expected @@ -0,0 +1,26 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 +; RUN: opt < %s -S | FileCheck %s + +define i32 @func(i32 %x) { +; CHECK-LABEL: define i32 @func( +; CHECK-SAME: i32 [[X:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[X]], 0 +; CHECK-NEXT: [[TMP2:%.*]] = call i32 @foo(i1 [[TMP1]]) +; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[X]], 2 +; CHECK-NEXT: [[TMP4:%.*]] = call i32 @foo(i1 [[TMP3]]) +; CHECK-NEXT: [[TMP5:%.*]] = icmp ne i32 [[TMP4]], 0 +; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 [[TMP2]] +; CHECK-NEXT: ret i32 [[TMP6]] +; + %1 = icmp eq i32 %x, 0 + %2 = call i32 @foo(i1 %1) + + %3 = icmp eq i32 %x, 2 + %4 = call i32 @foo(i1 %3) + %5 = icmp ne i32 %4, 0 + %6 = select i1 %5, i32 %4, i32 %2 + + ret i32 %6 +} + +declare i32 @foo(i1) diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll new file mode 100644 index 000000000000..3b449291d0e7 --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll @@ -0,0 +1,38 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 +; RUN: opt < %s -S | FileCheck %s + +; Test that we don't regress diff quality by trying to keep variable names +; stable (and messing up the matching). + +define i32 @func(i32 %x) { +; CHECK-LABEL: define i32 @func( +; CHECK-SAME: i32 [[X:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[X]], 0 +; CHECK-NEXT: [[TMP2:%.*]] = call i32 @foo(i1 [[TMP1]]) +; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[X]], 1 +; CHECK-NEXT: [[TMP4:%.*]] = call i32 @foo(i1 [[TMP3]]) +; CHECK-NEXT: [[TMP5:%.*]] = icmp ne i32 [[TMP4]], 0 +; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 [[TMP2]] +; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[X]], 2 +; CHECK-NEXT: [[TMP8:%.*]] = call i32 @foo(i1 [[TMP7]]) +; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i32 [[TMP8]], 0 +; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[TMP6]] +; CHECK-NEXT: ret i32 [[TMP10]] +; + %1 = icmp eq i32 %x, 0 + %2 = call i32 @foo(i1 %1) + + %3 = icmp eq i32 %x, 2 + %4 = call i32 @foo(i1 %3) + %5 = icmp ne i32 %4, 0 + %6 = select i1 %5, i32 %4, i32 %2 + + %7 = icmp eq i32 %x, 1 + %8 = call i32 @foo(i1 %7) + %9 = icmp ne i32 %8, 0 + %10 = select i1 %9, i32 %8, i32 %6 + + ret i32 %10 +} + +declare i32 @foo(i1) diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll.expected new file mode 100644 index 000000000000..3d0f772505a6 --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll.expected @@ -0,0 +1,38 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 +; RUN: opt < %s -S | FileCheck %s + +; Test that we don't regress diff quality by trying to keep variable names +; stable (and messing up the matching). + +define i32 @func(i32 %x) { +; CHECK-LABEL: define i32 @func( +; CHECK-SAME: i32 [[X:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[X]], 0 +; CHECK-NEXT: [[TMP2:%.*]] = call i32 @foo(i1 [[TMP1]]) +; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[X]], 2 +; CHECK-NEXT: [[TMP4:%.*]] = call i32 @foo(i1 [[TMP3]]) +; CHECK-NEXT: [[TMP5:%.*]] = icmp ne i32 [[TMP4]], 0 +; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 [[TMP2]] +; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[X]], 1 +; CHECK-NEXT: [[TMP8:%.*]] = call i32 @foo(i1 [[TMP7]]) +; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i32 [[TMP8]], 0 +; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[TMP6]] +; CHECK-NEXT: ret i32 [[TMP10]] +; + %1 = icmp eq i32 %x, 0 + %2 = call i32 @foo(i1 %1) + + %3 = icmp eq i32 %x, 2 + %4 = call i32 @foo(i1 %3) + %5 = icmp ne i32 %4, 0 + %6 = select i1 %5, i32 %4, i32 %2 + + %7 = icmp eq i32 %x, 1 + %8 = call i32 @foo(i1 %7) + %9 = icmp ne i32 %8, 0 + %10 = select i1 %9, i32 %8, i32 %6 + + ret i32 %10 +} + +declare i32 @foo(i1) diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values4.ll b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values4.ll new file mode 100644 index 000000000000..e3d8452f9631 --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values4.ll @@ -0,0 +1,41 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 +; RUN: opt < %s -S | FileCheck %s + +; A test that hits the quadratic runtime prevention in the diff algorithm and +; a more complex case of name conflict avoidance. + +define i32 @func(i32 %x) { +; CHECK-LABEL: define i32 @func( +; CHECK-SAME: i32 [[X:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = mul i32 [[X]], 3 +; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], 4 +; CHECK-NEXT: [[TMP3:%.*]] = call i32 @foo(i32 [[TMP2]]) +; CHECK-NEXT: [[TMP4:%.*]] = call i32 @foo(i32 [[TMP3]]) +; CHECK-NEXT: [[TMP5:%.*]] = call i32 @foo(i32 [[TMP4]]) +; CHECK-NEXT: [[TMP6:%.*]] = call i32 @foo(i32 [[TMP5]]) +; CHECK-NEXT: [[TMP7:%.*]] = call i32 @foo(i32 [[TMP6]]) +; CHECK-NEXT: [[TMP8:%.*]] = call i32 @foo(i32 [[TMP7]]) +; CHECK-NEXT: [[TMP9:%.*]] = call i32 @foo(i32 [[TMP8]]) +; CHECK-NEXT: [[TMP10:%.*]] = call i32 @foo(i32 [[TMP9]]) +; CHECK-NEXT: [[TMP11:%.*]] = call i32 @foo(i32 [[TMP10]]) +; CHECK-NEXT: [[TMP12:%.*]] = call i32 @foo(i32 [[TMP11]]) +; CHECK-NEXT: ret i32 [[TMP12]] +; + %1 = mul i32 %x, 3 + %2 = call i32 @foo(i32 %1) + %3 = call i32 @foo(i32 %2) + %4 = call i32 @foo(i32 %3) + %5 = call i32 @foo(i32 %4) + %6 = call i32 @foo(i32 %5) + %7 = call i32 @foo(i32 %6) + %8 = xor i32 %7, 1 + %9 = call i32 @foo(i32 %8) + %10 = add i32 %9, 1 + %11 = call i32 @foo(i32 %10) + %12 = call i32 @foo(i32 %11) + %13 = call i32 @foo(i32 %12) + + ret i32 %13 +} + +declare i32 @foo(i1) diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values4.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values4.ll.expected new file mode 100644 index 000000000000..5962bdafd9ea --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values4.ll.expected @@ -0,0 +1,42 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 +; RUN: opt < %s -S | FileCheck %s + +; A test that hits the quadratic runtime prevention in the diff algorithm and +; a more complex case of name conflict avoidance. + +define i32 @func(i32 %x) { +; CHECK-LABEL: define i32 @func( +; CHECK-SAME: i32 [[X:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = mul i32 [[X]], 3 +; CHECK-NEXT: [[TMP2:%.*]] = call i32 @foo(i32 [[TMP1]]) +; CHECK-NEXT: [[TMP3:%.*]] = call i32 @foo(i32 [[TMP2]]) +; CHECK-NEXT: [[TMP4:%.*]] = call i32 @foo(i32 [[TMP3]]) +; CHECK-NEXT: [[TMP5:%.*]] = call i32 @foo(i32 [[TMP4]]) +; CHECK-NEXT: [[TMP6:%.*]] = call i32 @foo(i32 [[TMP5]]) +; CHECK-NEXT: [[TMP7:%.*]] = call i32 @foo(i32 [[TMP6]]) +; CHECK-NEXT: [[TMP8:%.*]] = xor i32 [[TMP7]], 1 +; CHECK-NEXT: [[TMP9:%.*]] = call i32 @foo(i32 [[TMP8]]) +; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[TMP9]], 1 +; CHECK-NEXT: [[TMP11:%.*]] = call i32 @foo(i32 [[TMP10]]) +; CHECK-NEXT: [[TMP12:%.*]] = call i32 @foo(i32 [[TMP11]]) +; CHECK-NEXT: [[TMP13:%.*]] = call i32 @foo(i32 [[TMP12]]) +; CHECK-NEXT: ret i32 [[TMP13]] +; + %1 = mul i32 %x, 3 + %2 = call i32 @foo(i32 %1) + %3 = call i32 @foo(i32 %2) + %4 = call i32 @foo(i32 %3) + %5 = call i32 @foo(i32 %4) + %6 = call i32 @foo(i32 %5) + %7 = call i32 @foo(i32 %6) + %8 = xor i32 %7, 1 + %9 = call i32 @foo(i32 %8) + %10 = add i32 %9, 1 + %11 = call i32 @foo(i32 %10) + %12 = call i32 @foo(i32 %11) + %13 = call i32 @foo(i32 %12) + + ret i32 %13 +} + +declare i32 @foo(i1) diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values.test b/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values.test new file mode 100644 index 000000000000..c6287a6b29ca --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values.test @@ -0,0 +1,2 @@ +# RUN: cp -f %S/Inputs/stable_ir_values.ll %t.ll && %update_test_checks %t.ll +# RUN: diff -u %t.ll %S/Inputs/stable_ir_values.ll.expected diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values2.test b/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values2.test new file mode 100644 index 000000000000..3cebcd52f005 --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values2.test @@ -0,0 +1,2 @@ +# RUN: cp -f %S/Inputs/stable_ir_values2.ll %t.ll && %update_test_checks %t.ll +# RUN: diff -u %t.ll %S/Inputs/stable_ir_values2.ll.expected diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values3.test b/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values3.test new file mode 100644 index 000000000000..83bc80128541 --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values3.test @@ -0,0 +1,2 @@ +# RUN: cp -f %S/Inputs/stable_ir_values3.ll %t.ll && %update_test_checks %t.ll +# RUN: diff -u %t.ll %S/Inputs/stable_ir_values3.ll.expected diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values4.test b/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values4.test new file mode 100644 index 000000000000..89f252f80780 --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values4.test @@ -0,0 +1,2 @@ +# RUN: cp -f %S/Inputs/stable_ir_values4.ll %t.ll && %update_test_checks %t.ll +# RUN: diff -u %t.ll %S/Inputs/stable_ir_values4.ll.expected -- GitLab From fc1e5f978561d6357083f9b5bc981e52d93374b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolai=20H=C3=A4hnle?= Date: Sat, 23 Dec 2023 14:07:47 +0100 Subject: [PATCH 553/929] update_test_checks: simplify is_local_def_ir_value The match argument is unused. --- llvm/utils/UpdateTestChecks/common.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py index 53777523ec2a..72d21cf4ca7d 100644 --- a/llvm/utils/UpdateTestChecks/common.py +++ b/llvm/utils/UpdateTestChecks/common.py @@ -933,7 +933,7 @@ class NamelessValue: self.variable_mapping = {} # Return true if this kind of IR value is "local", basically if it matches '%{{.*}}'. - def is_local_def_ir_value_match(self, match): + def is_local_def_ir_value(self): return self.ir_prefix == "%" # Return true if this kind of IR value is "global", basically if it matches '#{{.*}}'. @@ -949,7 +949,7 @@ class NamelessValue: # Return the IR regexp we use for this kind or IR value, e.g., [\w.-]+? for locals def get_ir_regex_from_ir_value_re_match(self, match): # for backwards compatibility we check locals with '.*' - if self.is_local_def_ir_value_match(match): + if self.is_local_def_ir_value(): return ".*" return self.ir_regexp @@ -990,7 +990,7 @@ class NamelessValue: else: regex = self.get_ir_regex_from_ir_value_re_match(match) capture_start = "[[" - if self.is_local_def_ir_value_match(match): + if self.is_local_def_ir_value(): return capture_start + varname + ":" + prefix + regex + "]]" return prefix + capture_start + varname + ":" + regex + "]]" @@ -999,7 +999,7 @@ class NamelessValue: if var_prefix is None: var_prefix = self.check_prefix capture_start = "[[#" if self.is_number else "[[" - if self.is_local_def_ir_value_match(match): + if self.is_local_def_ir_value(): return capture_start + self.get_value_name(var, var_prefix) + "]]" prefix = self.get_ir_prefix_from_ir_value_match(match)[0] return prefix + capture_start + self.get_value_name(var, var_prefix) + "]]" @@ -1209,7 +1209,7 @@ def generalize_check_lines_common( " with scripted FileCheck name." % (var,) ) key = (var, nameless_value.check_key) - is_local_def = nameless_value.is_local_def_ir_value_match(match) + is_local_def = nameless_value.is_local_def_ir_value() if is_local_def and key in vars_seen: rv = nameless_value.get_value_use(var, match) elif not is_local_def and key in global_vars_seen: -- GitLab From 5747f9d3e9f3a49c27984b9341e123b9ca82e741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolai=20H=C3=A4hnle?= Date: Sat, 23 Dec 2023 14:20:43 +0100 Subject: [PATCH 554/929] update_test_checks: simplify get_ir_regex The match argument isn't used. --- llvm/utils/UpdateTestChecks/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py index 72d21cf4ca7d..a3365fef5f6e 100644 --- a/llvm/utils/UpdateTestChecks/common.py +++ b/llvm/utils/UpdateTestChecks/common.py @@ -947,7 +947,7 @@ class NamelessValue: return re.search(self.ir_prefix, match[0])[0], self.check_prefix # Return the IR regexp we use for this kind or IR value, e.g., [\w.-]+? for locals - def get_ir_regex_from_ir_value_re_match(self, match): + def get_ir_regex(self): # for backwards compatibility we check locals with '.*' if self.is_local_def_ir_value(): return ".*" @@ -988,7 +988,7 @@ class NamelessValue: regex = "" # always capture a number in the default format capture_start = "[[#" else: - regex = self.get_ir_regex_from_ir_value_re_match(match) + regex = self.get_ir_regex() capture_start = "[[" if self.is_local_def_ir_value(): return capture_start + varname + ":" + prefix + regex + "]]" -- GitLab From fb02f9ac84a6151e41aba8f7391edd132a9aaf14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolai=20H=C3=A4hnle?= Date: Thu, 23 Nov 2023 06:46:07 +0100 Subject: [PATCH 555/929] update_test_checks: keep meta variables stable by default Prior to this change, running UTC on larger tests, especially tests with unnamed IR values, often resulted in a spuriously large diff because e.g. TMPnn variables in the CHECK lines were renumbered. This change attempts to reduce the diff by keeping those variable names the same. There are cases in which this "drift" of variable names can end up being more confusing. The old behavior can be re-enabled with the --reset-variable-names command line argument. The improvement may not be immediately apparent in the diff of this change. The point is that the diff of stable_ir_values.ll against stable_ir_values.ll.expected after this change is smaller. Ideally, we'd also keep meta variables for "global" objects stable, e.g. for attributes (#nn) and metadata (!nn). However, that would require a much more substantial refactoring of how we generate check lines, so I left it for future work. --- .../Inputs/stable_ir_values.ll.expected | 10 +- .../Inputs/stable_ir_values.ll.expected.reset | 23 + .../Inputs/stable_ir_values2.ll.expected | 12 +- .../Inputs/stable_ir_values3.ll | 3 - .../Inputs/stable_ir_values3.ll.expected | 3 - .../Inputs/stable_ir_values4.ll.expected | 8 +- .../update_test_checks/stable_ir_values.test | 3 + llvm/utils/UpdateTestChecks/common.py | 513 +++++++++++++++++- llvm/utils/update_test_checks.py | 22 +- 9 files changed, 554 insertions(+), 43 deletions(-) create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected.reset diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected index 5142e3ed32ba..3549a4d76aa7 100644 --- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected @@ -9,11 +9,11 @@ define i32 @func({i32, i32} %x, i32 %y) { ; CHECK-LABEL: define i32 @func( ; CHECK-SAME: { i32, i32 } [[X:%.*]], i32 [[Y:%.*]]) { -; CHECK-NEXT: [[X_I34:%.*]] = extractvalue { i32, i32 } [[X]], 0 -; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[Y]], 1 -; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[X_I34]], [[TMP1]] -; CHECK-NEXT: [[TMP3:%.*]] = mul i32 [[TMP2]], 3 -; CHECK-NEXT: ret i32 [[TMP3]] +; CHECK-NEXT: [[X_I33:%.*]] = extractvalue { i32, i32 } [[X]], 0 +; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[Y]], 1 +; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[X_I33]], [[TMP3]] +; CHECK-NEXT: [[TMP2:%.*]] = mul i32 [[TMP1]], 3 +; CHECK-NEXT: ret i32 [[TMP2]] ; %x.i34 = extractvalue {i32, i32} %x, 0 %1 = add i32 %y, 1 diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected.reset b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected.reset new file mode 100644 index 000000000000..5142e3ed32ba --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected.reset @@ -0,0 +1,23 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 +; RUN: opt < %s -S | FileCheck %s + +; The assumption underlying this test is that there are pre-existing check lines +; but something has changed, and we would like to avoid needless changes of +; meta variable names so that diffs end up being easier to read, e.g. avoid +; changing X_I33 into X_I34 or renumbering the various TMP variables. + +define i32 @func({i32, i32} %x, i32 %y) { +; CHECK-LABEL: define i32 @func( +; CHECK-SAME: { i32, i32 } [[X:%.*]], i32 [[Y:%.*]]) { +; CHECK-NEXT: [[X_I34:%.*]] = extractvalue { i32, i32 } [[X]], 0 +; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[Y]], 1 +; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[X_I34]], [[TMP1]] +; CHECK-NEXT: [[TMP3:%.*]] = mul i32 [[TMP2]], 3 +; CHECK-NEXT: ret i32 [[TMP3]] +; + %x.i34 = extractvalue {i32, i32} %x, 0 + %1 = add i32 %y, 1 + %2 = add i32 %x.i34, %1 + %3 = mul i32 %2, 3 + ret i32 %3 +} diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll.expected index 53f60bda8ee5..6311a55a1f9d 100644 --- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll.expected +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll.expected @@ -5,12 +5,12 @@ define i32 @func(i32 %x) { ; CHECK-LABEL: define i32 @func( ; CHECK-SAME: i32 [[X:%.*]]) { ; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[X]], 0 -; CHECK-NEXT: [[TMP2:%.*]] = call i32 @foo(i1 [[TMP1]]) -; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[X]], 2 -; CHECK-NEXT: [[TMP4:%.*]] = call i32 @foo(i1 [[TMP3]]) -; CHECK-NEXT: [[TMP5:%.*]] = icmp ne i32 [[TMP4]], 0 -; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 [[TMP2]] -; CHECK-NEXT: ret i32 [[TMP6]] +; CHECK-NEXT: [[TMP6:%.*]] = call i32 @foo(i1 [[TMP1]]) +; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[X]], 2 +; CHECK-NEXT: [[TMP8:%.*]] = call i32 @foo(i1 [[TMP7]]) +; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i32 [[TMP8]], 0 +; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[TMP6]] +; CHECK-NEXT: ret i32 [[TMP10]] ; %1 = icmp eq i32 %x, 0 %2 = call i32 @foo(i1 %1) diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll index 3b449291d0e7..a4f4fc67f78d 100644 --- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll @@ -1,9 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 ; RUN: opt < %s -S | FileCheck %s -; Test that we don't regress diff quality by trying to keep variable names -; stable (and messing up the matching). - define i32 @func(i32 %x) { ; CHECK-LABEL: define i32 @func( ; CHECK-SAME: i32 [[X:%.*]]) { diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll.expected index 3d0f772505a6..08d3c22172ee 100644 --- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll.expected +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll.expected @@ -1,9 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 ; RUN: opt < %s -S | FileCheck %s -; Test that we don't regress diff quality by trying to keep variable names -; stable (and messing up the matching). - define i32 @func(i32 %x) { ; CHECK-LABEL: define i32 @func( ; CHECK-SAME: i32 [[X:%.*]]) { diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values4.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values4.ll.expected index 5962bdafd9ea..e3fa51598c48 100644 --- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values4.ll.expected +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values4.ll.expected @@ -15,12 +15,12 @@ define i32 @func(i32 %x) { ; CHECK-NEXT: [[TMP6:%.*]] = call i32 @foo(i32 [[TMP5]]) ; CHECK-NEXT: [[TMP7:%.*]] = call i32 @foo(i32 [[TMP6]]) ; CHECK-NEXT: [[TMP8:%.*]] = xor i32 [[TMP7]], 1 -; CHECK-NEXT: [[TMP9:%.*]] = call i32 @foo(i32 [[TMP8]]) -; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[TMP9]], 1 +; CHECK-NEXT: [[TMP13:%.*]] = call i32 @foo(i32 [[TMP8]]) +; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[TMP13]], 1 +; CHECK-NEXT: [[TMP10:%.*]] = call i32 @foo(i32 [[TMP9]]) ; CHECK-NEXT: [[TMP11:%.*]] = call i32 @foo(i32 [[TMP10]]) ; CHECK-NEXT: [[TMP12:%.*]] = call i32 @foo(i32 [[TMP11]]) -; CHECK-NEXT: [[TMP13:%.*]] = call i32 @foo(i32 [[TMP12]]) -; CHECK-NEXT: ret i32 [[TMP13]] +; CHECK-NEXT: ret i32 [[TMP12]] ; %1 = mul i32 %x, 3 %2 = call i32 @foo(i32 %1) diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values.test b/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values.test index c6287a6b29ca..4dfaf5d25c8a 100644 --- a/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values.test +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values.test @@ -1,2 +1,5 @@ # RUN: cp -f %S/Inputs/stable_ir_values.ll %t.ll && %update_test_checks %t.ll # RUN: diff -u %t.ll %S/Inputs/stable_ir_values.ll.expected +# Now test that we can reset all the names +# RUN: %update_test_checks %t.ll --reset-variable-names +# RUN: diff -u %t.ll %S/Inputs/stable_ir_values.ll.expected.reset diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py index a3365fef5f6e..f766d541c79c 100644 --- a/llvm/utils/UpdateTestChecks/common.py +++ b/llvm/utils/UpdateTestChecks/common.py @@ -1,6 +1,8 @@ from __future__ import print_function import argparse +import bisect +import collections import copy import glob import itertools @@ -10,7 +12,7 @@ import subprocess import sys import shlex -from typing import List +from typing import List, Mapping, Set ##### Common utilities for update_*test_checks.py @@ -420,6 +422,48 @@ def should_add_line_to_output( return True +def collect_original_check_lines(ti: TestInfo, prefix_set: set): + """ + Collect pre-existing check lines into a dictionary `result` which is + returned. + + result[func_name][prefix] is filled with a list of right-hand-sides of check + lines. + """ + result = {} + + current_function = None + for input_line_info in ti.ro_iterlines(): + input_line = input_line_info.line + if current_function is not None: + if input_line == "": + continue + if input_line.lstrip().startswith(";"): + m = CHECK_RE.match(input_line) + if ( + m is not None + and m.group(1) in prefix_set + and m.group(2) not in ["LABEL", "SAME"] + ): + if m.group(1) not in current_function: + current_function[m.group(1)] = [] + current_function[m.group(1)].append(input_line[m.end() :].strip()) + continue + current_function = None + + m = IR_FUNCTION_RE.match(input_line) + if m is not None: + func_name = m.group(1) + if ti.args.function is not None and func_name != ti.args.function: + # When filtering on a specific function, skip all others. + continue + + assert func_name not in result + current_function = result[func_name] = {} + + return result + + # Perform lit-like substitutions def getSubstitutions(sourcepath): sourcedir = os.path.dirname(sourcepath) @@ -491,7 +535,7 @@ RUN_LINE_RE = re.compile(r"^\s*(?://|[;#])\s*RUN:\s*(.*)$") CHECK_PREFIX_RE = re.compile(r"--?check-prefix(?:es)?[= ](\S+)") PREFIX_RE = re.compile("^[a-zA-Z0-9_-]+$") CHECK_RE = re.compile( - r"^\s*(?://|[;#])\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL|-SAME|-EMPTY)?:" + r"^\s*(?://|[;#])\s*([^:]+?)(?:-(NEXT|NOT|DAG|LABEL|SAME|EMPTY))?:" ) CHECK_SAME_RE = re.compile(r"^\s*(?://|[;#])\s*([^:]+?)(?:-SAME)?:") @@ -1187,20 +1231,325 @@ def may_clash_with_default_check_prefix_name(check_prefix, var): ) +def find_diff_matching(lhs: List[str], rhs: List[str]) -> List[tuple]: + """ + Find a large ordered matching between strings in lhs and rhs. + + Think of this as finding the *unchanged* lines in a diff, where the entries + of lhs and rhs are lines of the files being diffed. + + Returns a list of matched (lhs_idx, rhs_idx) pairs. + """ + + if not lhs or not rhs: + return [] + + # Collect matches in reverse order. + matches = [] + + # First, collect a set of candidate matching edges. We limit this to a + # constant multiple of the input size to avoid quadratic runtime. + patterns = collections.defaultdict(lambda: ([], [])) + + for idx in range(len(lhs)): + patterns[lhs[idx]][0].append(idx) + for idx in range(len(rhs)): + patterns[rhs[idx]][1].append(idx) + + multiple_patterns = [] + + candidates = [] + for pattern in patterns.values(): + if not pattern[0] or not pattern[1]: + continue + + if len(pattern[0]) == len(pattern[1]) == 1: + candidates.append((pattern[0][0], pattern[1][0])) + else: + multiple_patterns.append(pattern) + + multiple_patterns.sort(key=lambda pattern: len(pattern[0]) * len(pattern[1])) + + for pattern in multiple_patterns: + if len(candidates) + len(pattern[0]) * len(pattern[1]) > 2 * ( + len(lhs) + len(rhs) + ): + break + for lhs_idx in pattern[0]: + for rhs_idx in pattern[1]: + candidates.append((lhs_idx, rhs_idx)) + + if not candidates: + # The LHS and RHS either share nothing in common, or lines are just too + # identical. In that case, let's give up and not match anything. + return [] + + # Compute a maximal crossing-free matching via an algorithm that is + # inspired by a mixture of dynamic programming and line-sweeping in + # discrete geometry. + # + # I would be surprised if this algorithm didn't exist somewhere in the + # literature, but I found it without consciously recalling any + # references, so you'll have to make do with the explanation below. + # Sorry. + # + # The underlying graph is bipartite: + # - nodes on the LHS represent lines in the original check + # - nodes on the RHS represent lines in the new (updated) check + # + # Nodes are implicitly sorted by the corresponding line number. + # Edges (unique_matches) are sorted by the line number on the LHS. + # + # Here's the geometric intuition for the algorithm. + # + # * Plot the edges as points in the plane, with the original line + # number on the X axis and the updated line number on the Y axis. + # * The goal is to find a longest "chain" of points where each point + # is strictly above and to the right of the previous point. + # * The algorithm proceeds by sweeping a vertical line from left to + # right. + # * The algorithm maintains a table where `table[N]` answers the + # question "What is currently the 'best' way to build a chain of N+1 + # points to the left of the vertical line". Here, 'best' means + # that the last point of the chain is a as low as possible (minimal + # Y coordinate). + # * `table[N]` is `(y, point_idx)` where `point_idx` is the index of + # the last point in the chain and `y` is its Y coordinate + # * A key invariant is that the Y values in the table are + # monotonically increasing + # * Thanks to these properties, the table can be used to answer the + # question "What is the longest chain that can be built to the left + # of the vertical line using only points below a certain Y value", + # using a binary search over the table. + # * The algorithm also builds a backlink structure in which every point + # links back to the previous point on a best (longest) chain ending + # at that point + # + # The core loop of the algorithm sweeps the line and updates the table + # and backlink structure for every point that we cross during the sweep. + # Therefore, the algorithm is trivially O(M log M) in the number of + # points. + candidates.sort(key=lambda candidate: (candidate[0], -candidate[1])) + + backlinks = [] + table = [] + for _, rhs_idx in candidates: + candidate_idx = len(backlinks) + ti = bisect.bisect_left(table, rhs_idx, key=lambda entry: entry[0]) + + # Update the table to record a best chain ending in the current point. + # There always is one, and if any of the previously visited points had + # a higher Y coordinate, then there is always a previously recorded best + # chain that can be improved upon by using the current point. + # + # There is only one case where there is some ambiguity. If the + # pre-existing entry table[ti] has the same Y coordinate / rhs_idx as + # the current point (this can only happen if the same line appeared + # multiple times on the LHS), then we could choose to keep the + # previously recorded best chain instead. That would bias the algorithm + # differently but should have no systematic impact on the quality of the + # result. + if ti < len(table): + table[ti] = (rhs_idx, candidate_idx) + else: + table.append((rhs_idx, candidate_idx)) + if ti > 0: + backlinks.append(table[ti - 1][1]) + else: + backlinks.append(None) + + # Commit to names in the matching by walking the backlinks. Recursively + # attempt to fill in more matches in-betweem. + match_idx = table[-1][1] + while match_idx is not None: + current = candidates[match_idx] + matches.append(current) + match_idx = backlinks[match_idx] + + matches.reverse() + return matches + + +VARIABLE_TAG = "[[@@]]" +METAVAR_RE = re.compile(r"\[\[([A-Z0-9_]+)(?::[^]]+)?\]\]") +NUMERIC_SUFFIX_RE = re.compile(r"[0-9]*$") + + +class CheckValueInfo: + def __init__( + self, + nameless_value: NamelessValue, + var: str, + prefix: str, + ): + self.nameless_value = nameless_value + self.var = var + self.prefix = prefix + + +# Represent a check line in a way that allows us to compare check lines while +# ignoring some or all of the FileCheck variable names. +class CheckLineInfo: + def __init__(self, line, values): + # Line with all FileCheck variable name occurrences replaced by VARIABLE_TAG + self.line: str = line + + # Information on each FileCheck variable name occurrences in the line + self.values: List[CheckValueInfo] = values + + def __repr__(self): + return f"CheckLineInfo(line={self.line}, self.values={self.values})" + + +def remap_metavar_names( + old_line_infos: List[CheckLineInfo], + new_line_infos: List[CheckLineInfo], + committed_names: Set[str], +) -> Mapping[str, str]: + """ + Map all FileCheck variable names that appear in new_line_infos to new + FileCheck variable names in an attempt to reduce the diff from old_line_infos + to new_line_infos. + + This is done by: + * Matching old check lines and new check lines using a diffing algorithm + applied after replacing names with wildcards. + * Committing to variable names such that the matched lines become equal + (without wildcards) if possible + * This is done recursively to handle cases where many lines are equal + after wildcard replacement + """ + # Initialize uncommitted identity mappings + new_mapping = {} + for line in new_line_infos: + for value in line.values: + new_mapping[value.var] = value.var + + # Recursively commit to the identity mapping or find a better one + def recurse(old_begin, old_end, new_begin, new_end): + if old_begin == old_end or new_begin == new_end: + return + + # Find a matching of lines where uncommitted names are replaced + # with a placeholder. + def diffify_line(line, mapper): + values = [] + for value in line.values: + mapped = mapper(value.var) + values.append(mapped if mapped in committed_names else "?") + return line.line.strip() + " @@@ " + " @ ".join(values) + + lhs_lines = [ + diffify_line(line, lambda x: x) + for line in old_line_infos[old_begin:old_end] + ] + rhs_lines = [ + diffify_line(line, lambda x: new_mapping[x]) + for line in new_line_infos[new_begin:new_end] + ] + + candidate_matches = find_diff_matching(lhs_lines, rhs_lines) + + # Apply commits greedily on a match-by-match basis + matches = [(-1, -1)] + committed_anything = False + for lhs_idx, rhs_idx in candidate_matches: + lhs_line = old_line_infos[lhs_idx] + rhs_line = new_line_infos[rhs_idx] + + local_commits = {} + + for lhs_value, rhs_value in zip(lhs_line.values, rhs_line.values): + if new_mapping[rhs_value.var] in committed_names: + # The new value has already been committed. If it was mapped + # to the same name as the original value, we can consider + # committing other values from this line. Otherwise, we + # should ignore this line. + if new_mapping[rhs_value.var] == lhs_value.var: + continue + else: + break + + if rhs_value.var in local_commits: + # Same, but for a possible commit happening on the same line + if local_commits[rhs_value.var] == lhs_value.var: + continue + else: + break + + if lhs_value.var in committed_names: + # We can't map this value because the name we would map it to has already been + # committed for something else. Give up on this line. + break + + local_commits[rhs_value.var] = lhs_value.var + else: + # No reason not to add any commitments for this line + for rhs_var, lhs_var in local_commits.items(): + new_mapping[rhs_var] = lhs_var + committed_names.add(lhs_var) + committed_anything = True + + if ( + lhs_var != rhs_var + and lhs_var in new_mapping + and new_mapping[lhs_var] == lhs_var + ): + new_mapping[lhs_var] = "conflict_" + lhs_var + + matches.append((lhs_idx, rhs_idx)) + + matches.append((old_end, new_end)) + + # Recursively handle sequences between matches + if committed_anything: + for (lhs_prev, rhs_prev), (lhs_next, rhs_next) in zip(matches, matches[1:]): + recurse(lhs_prev + 1, lhs_next, rhs_prev + 1, rhs_next) + + recurse(0, len(old_line_infos), 0, len(new_line_infos)) + + # Commit to remaining names and resolve conflicts + for new_name, mapped_name in new_mapping.items(): + if mapped_name in committed_names: + continue + if not mapped_name.startswith("conflict_"): + assert mapped_name == new_name + committed_names.add(mapped_name) + + for new_name, mapped_name in new_mapping.items(): + if mapped_name in committed_names: + continue + assert mapped_name.startswith("conflict_") + + m = NUMERIC_SUFFIX_RE.search(new_name) + base_name = new_name[: m.start()] + suffix = int(new_name[m.start() :]) if m.start() != m.end() else 1 + while True: + candidate = f"{base_name}{suffix}" + if candidate not in committed_names: + new_mapping[new_name] = candidate + committed_names.add(candidate) + break + suffix += 1 + + return new_mapping + + def generalize_check_lines_common( lines, is_analyze, vars_seen, global_vars_seen, nameless_values, - nameless_value_regex, + nameless_value_regex: re.Pattern, is_asm, preserve_names, + original_check_lines=None, ): # This gets called for each match that occurs in # a line. We transform variables we haven't seen # into defs, and variables we have seen into uses. - def transform_line_vars(match): + def transform_line_vars(match, transform_locals=True): var = get_name_from_ir_value_match(match) nameless_value = get_nameless_value_from_match(match, nameless_values) if may_clash_with_default_check_prefix_name(nameless_value.check_prefix, var): @@ -1210,6 +1559,8 @@ def generalize_check_lines_common( ) key = (var, nameless_value.check_key) is_local_def = nameless_value.is_local_def_ir_value() + if is_local_def and not transform_locals: + return None if is_local_def and key in vars_seen: rv = nameless_value.get_value_use(var, match) elif not is_local_def and key in global_vars_seen: @@ -1228,13 +1579,15 @@ def generalize_check_lines_common( # including the commas and spaces. return match.group(1) + rv + match.group(match.lastindex) - lines_with_def = [] + def transform_non_local_line_vars(match): + return transform_line_vars(match, False) + multiple_braces_re = re.compile(r"({{+)|(}}+)") def escape_braces(match_obj): return '{{' + re.escape(match_obj.group(0)) + '}}' - for i, line in enumerate(lines): - if not is_asm and not is_analyze: + if not is_asm and not is_analyze: + for i, line in enumerate(lines): # An IR variable named '%.' matches the FileCheck regex string. line = line.replace("%.", "%dot") for regex in _global_hex_value_regex: @@ -1252,25 +1605,136 @@ def generalize_check_lines_common( # Ignore any comments, since the check lines will too. scrubbed_line = SCRUB_IR_COMMENT_RE.sub(r"", line) lines[i] = scrubbed_line - if not preserve_names: - # It can happen that two matches are back-to-back and for some reason sub - # will not replace both of them. For now we work around this by - # substituting until there is no more match. - changed = True - while changed: - (lines[i], changed) = nameless_value_regex.subn( - transform_line_vars, lines[i], count=1 - ) - if is_analyze: + + if not preserve_names: + if is_asm: + for i, _ in enumerate(lines): + # It can happen that two matches are back-to-back and for some reason sub + # will not replace both of them. For now we work around this by + # substituting until there is no more match. + changed = True + while changed: + (lines[i], changed) = nameless_value_regex.subn( + transform_line_vars, lines[i], count=1 + ) + else: + # LLVM IR case. Start by handling global meta variables (global IR variables, + # metadata, attributes) + for i, _ in enumerate(lines): + start = 0 + while True: + m = nameless_value_regex.search(lines[i][start:]) + if m is None: + break + start += m.start() + sub = transform_non_local_line_vars(m) + if sub is not None: + lines[i] = ( + lines[i][:start] + sub + lines[i][start + len(m.group(0)) :] + ) + start += 1 + + # Collect information about new check lines and original check lines (if any) + new_line_infos = [] + for line in lines: + filtered_line = "" + values = [] + while True: + m = nameless_value_regex.search(line) + if m is None: + filtered_line += line + break + + var = get_name_from_ir_value_match(m) + nameless_value = get_nameless_value_from_match(m, nameless_values) + var = nameless_value.get_value_name( + var, nameless_value.check_prefix + ) + + # Replace with a [[@@]] tag, but be sure to keep the spaces and commas. + filtered_line += ( + line[: m.start()] + + m.group(1) + + VARIABLE_TAG + + m.group(m.lastindex) + ) + line = line[m.end() :] + values.append( + CheckValueInfo( + nameless_value=nameless_value, + var=var, + prefix=nameless_value.get_ir_prefix_from_ir_value_match(m)[ + 0 + ], + ) + ) + new_line_infos.append(CheckLineInfo(filtered_line, values)) + + orig_line_infos = [] + for line in original_check_lines or []: + filtered_line = "" + values = [] + while True: + m = METAVAR_RE.search(line) + if m is None: + filtered_line += line + break + + # Replace with a [[@@]] tag, but be sure to keep the spaces and commas. + filtered_line += line[: m.start()] + VARIABLE_TAG + line = line[m.end() :] + values.append( + CheckValueInfo( + nameless_value=None, + var=m.group(1), + prefix=None, + ) + ) + orig_line_infos.append(CheckLineInfo(filtered_line, values)) + + # Compute the variable name mapping + committed_names = set(vars_seen) + + mapping = remap_metavar_names( + orig_line_infos, new_line_infos, committed_names + ) + + for i, line_info in enumerate(new_line_infos): + line_template = line_info.line + line = "" + + for value in line_info.values: + idx = line_template.find(VARIABLE_TAG) + line += line_template[:idx] + line_template = line_template[idx + len(VARIABLE_TAG) :] + + key = (mapping[value.var], nameless_value.check_key) + is_local_def = nameless_value.is_local_def_ir_value() + if is_local_def: + if mapping[value.var] in vars_seen: + line += f"[[{mapping[value.var]}]]" + else: + line += f"[[{mapping[value.var]}:{value.prefix}{value.nameless_value.get_ir_regex()}]]" + vars_seen.add(mapping[value.var]) + else: + raise RuntimeError("not implemented") + + line += line_template + + lines[i] = line + + if is_analyze: + for i, _ in enumerate(lines): # Escape multiple {{ or }} as {{}} denotes a FileCheck regex. scrubbed_line = multiple_braces_re.sub(escape_braces, lines[i]) lines[i] = scrubbed_line + return lines # Replace IR value defs and uses with FileCheck variables. def generalize_check_lines( - lines, is_analyze, vars_seen, global_vars_seen, preserve_names + lines, is_analyze, vars_seen, global_vars_seen, preserve_names, original_check_lines ): return generalize_check_lines_common( lines, @@ -1281,6 +1745,7 @@ def generalize_check_lines( IR_VALUE_RE, False, preserve_names, + original_check_lines=original_check_lines, ) @@ -1337,6 +1802,7 @@ def add_checks( global_vars_seen_dict, is_filtered, preserve_names=False, + original_check_lines: Mapping[str, List[str]] = {}, ): # prefix_exclusions are prefixes we cannot use to print the function because it doesn't exist in run lines that use these prefixes as well. prefix_exclusions = set() @@ -1409,6 +1875,7 @@ def add_checks( vars_seen, global_vars_seen, preserve_names, + original_check_lines=[], )[0] func_name_separator = func_dict[checkprefix][func_name].func_name_separator if "[[" in args_and_sig: @@ -1516,7 +1983,12 @@ def add_checks( # to variable naming fashions. else: func_body = generalize_check_lines( - func_body, False, vars_seen, global_vars_seen, preserve_names + func_body, + False, + vars_seen, + global_vars_seen, + preserve_names, + original_check_lines=original_check_lines.get(checkprefix), ) # This could be selectively enabled with an optional invocation argument. @@ -1578,6 +2050,7 @@ def add_ir_checks( version, global_vars_seen_dict, is_filtered, + original_check_lines={}, ): # Label format is based on IR string. if function_sig and version > 1: @@ -1602,6 +2075,7 @@ def add_ir_checks( global_vars_seen_dict, is_filtered, preserve_names, + original_check_lines=original_check_lines, ) @@ -1890,6 +2364,7 @@ def get_autogennote_suffix(parser, args): "llvm_bin", "verbose", "force_update", + "reset_variable_names", ): continue value = getattr(args, action.dest) diff --git a/llvm/utils/update_test_checks.py b/llvm/utils/update_test_checks.py index b5077d793513..04808ce6bb1c 100755 --- a/llvm/utils/update_test_checks.py +++ b/llvm/utils/update_test_checks.py @@ -85,6 +85,12 @@ def main(): choices=["none", "smart", "all"], help="Check global entries (global variables, metadata, attribute sets, ...) for functions", ) + parser.add_argument( + "--reset-variable-names", + action="store_true", + help="Reset all variable names to correspond closely to the variable names in IR. " + "This tends to result in larger diffs.", + ) parser.add_argument("tests", nargs="+") initial_args = common.parse_commandline_args(parser) @@ -170,13 +176,19 @@ def main(): ) builder.processed_prefixes(prefixes) + prefix_set = set( + [prefix for prefixes, _, _ in prefix_list for prefix in prefixes] + ) + + if not ti.args.reset_variable_names: + original_check_lines = common.collect_original_check_lines(ti, prefix_set) + else: + original_check_lines = {} + func_dict = builder.finish_and_get_func_dict() is_in_function = False is_in_function_start = False has_checked_pre_function_globals = False - prefix_set = set( - [prefix for prefixes, _, _ in prefix_list for prefix in prefixes] - ) common.debug("Rewriting FileCheck prefixes:", str(prefix_set)) output_lines = [] @@ -230,6 +242,7 @@ def main(): args.version, global_vars_seen_dict, is_filtered=builder.is_filtered(), + original_check_lines=original_check_lines.get(func, {}), ), ) ) @@ -261,6 +274,9 @@ def main(): args.version, global_vars_seen_dict, is_filtered=builder.is_filtered(), + original_check_lines=original_check_lines.get( + func_name, {} + ), ) ) is_in_function_start = False -- GitLab From 51b7ef937504ce45ef6a024aadfdee4147bffab5 Mon Sep 17 00:00:00 2001 From: lntue <35648136+lntue@users.noreply.github.com> Date: Thu, 7 Mar 2024 22:04:59 -0500 Subject: [PATCH 556/929] [libc][NFC] Fix a typo in test/src/stdfix/RoundTest.h. (#84411) --- libc/test/src/stdfix/RoundTest.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libc/test/src/stdfix/RoundTest.h b/libc/test/src/stdfix/RoundTest.h index 06343addbef2..d3ae04db9749 100644 --- a/libc/test/src/stdfix/RoundTest.h +++ b/libc/test/src/stdfix/RoundTest.h @@ -28,7 +28,7 @@ public: void testSpecialNumbers(RoundFunc func) { EXPECT_EQ(zero, func(zero, FXRep::FRACTION_LEN - 5)); - EXPECT_EQ(max, func(min, 0)); + EXPECT_EQ(min, func(min, 0)); EXPECT_EQ(max, func(max, FXRep::FRACTION_LEN)); EXPECT_EQ(one, func(half, 0)); -- GitLab From 66bd3cd75b32ccfa8d228c200cf4fbf72d49fd1f Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Thu, 7 Mar 2024 19:09:18 -0800 Subject: [PATCH 557/929] [AMDGPU,test] Change llc -march= to -mtriple= PR #75982 had been created before these tests were added, therefore some test were not updated. --- .../AMDGPU/GlobalISel/combine-fpneg-one-fneg.mir | 2 +- .../wmma-gfx12-w32-f16-f32-matrix-modifiers.ll | 2 +- .../AMDGPU/GlobalISel/wmma-gfx12-w32-imm.ll | 2 +- .../GlobalISel/wmma-gfx12-w32-iu-modifiers.ll | 2 +- .../wmma-gfx12-w32-swmmac-index_key.ll | 2 +- .../CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32.ll | 2 +- .../wmma-gfx12-w64-f16-f32-matrix-modifiers.ll | 2 +- .../AMDGPU/GlobalISel/wmma-gfx12-w64-imm.ll | 2 +- .../GlobalISel/wmma-gfx12-w64-iu-modifiers.ll | 2 +- .../wmma-gfx12-w64-swmmac-index_key.ll | 2 +- .../CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64.ll | 2 +- llvm/test/CodeGen/AMDGPU/atomics_cond_sub.ll | 4 ++-- .../CodeGen/AMDGPU/generic-targets-require-v6.ll | 16 ++++++++-------- .../CodeGen/AMDGPU/gfx12_scalar_subword_loads.ll | 4 ++-- .../test/CodeGen/AMDGPU/insert-waitcnts-hang.mir | 2 +- .../AMDGPU/llvm.amdgcn.atomic.cond.sub.ll | 2 +- llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dot4.f32.ll | 4 ++-- .../AMDGPU/llvm.amdgcn.global.load.tr-w32.ll | 4 ++-- .../AMDGPU/llvm.amdgcn.global.load.tr-w64.ll | 4 ++-- .../CodeGen/AMDGPU/llvm.amdgcn.s.wait.gfx12.ll | 4 ++-- .../CodeGen/AMDGPU/spill-regpressure-less.mir | 2 +- .../AMDGPU/wait-before-stores-with-scope_sys.ll | 4 ++-- .../wmma-gfx12-w32-f16-f32-matrix-modifiers.ll | 2 +- llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-imm.ll | 2 +- .../AMDGPU/wmma-gfx12-w32-iu-modifiers.ll | 2 +- .../AMDGPU/wmma-gfx12-w32-swmmac-index_key.ll | 2 +- llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32.ll | 2 +- .../wmma-gfx12-w64-f16-f32-matrix-modifiers.ll | 2 +- llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-imm.ll | 2 +- .../AMDGPU/wmma-gfx12-w64-iu-modifiers.ll | 2 +- .../AMDGPU/wmma-gfx12-w64-swmmac-index_key.ll | 2 +- llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64.ll | 2 +- .../CodeGen/AMDGPU/wmma-hazards-gfx12-w32.mir | 2 +- .../CodeGen/AMDGPU/wmma-hazards-gfx12-w64.mir | 2 +- .../convergencectrl/AMDGPU/basic.mir | 2 +- .../convergencectrl/AMDGPU/cycles.mir | 2 +- .../convergencectrl/AMDGPU/mixed2.mir | 2 +- .../convergencectrl/AMDGPU/region-nesting.mir | 2 +- 38 files changed, 52 insertions(+), 52 deletions(-) diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fpneg-one-fneg.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fpneg-one-fneg.mir index 8ec2778992e2..bdfc7c2b25c2 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fpneg-one-fneg.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fpneg-one-fneg.mir @@ -1,5 +1,5 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4 -# RUN: llc -march=amdgcn -mcpu=gfx1010 -run-pass=amdgpu-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK +# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=amdgpu-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK --- name: test_neg_one_f16_sgpr diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll index b29ae366ca1a..e500aae7e0f3 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 +; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 define amdgpu_ps void @test_wmma_f32_16x16x16_f16_negA(<8 x half> %A, <8 x half> %B, <8 x float> %C, ptr addrspace(1) %out) { ; GFX12-LABEL: test_wmma_f32_16x16x16_f16_negA: diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-imm.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-imm.ll index 6251dfdc392e..3037c1ec2829 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-imm.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-imm.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 +; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 define amdgpu_ps void @test_wmma_f32_16x16x16_f16_imm(<8 x half> %A, <8 x half> %B, ptr addrspace(1) %out) { ; GFX12-LABEL: test_wmma_f32_16x16x16_f16_imm: diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-iu-modifiers.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-iu-modifiers.ll index fe6d16bd8b5e..086144873a04 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-iu-modifiers.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-iu-modifiers.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 +; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 define amdgpu_ps void @test_wmma_i32_16x16x16_iu8_zext_src0(<2 x i32> %A, <2 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out) { ; GFX12-LABEL: test_wmma_i32_16x16x16_iu8_zext_src0: diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-swmmac-index_key.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-swmmac-index_key.ll index c80d7a6d9a83..a6e1f5ef12b4 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-swmmac-index_key.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-swmmac-index_key.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 +; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 define amdgpu_ps void @test_swmmac_f32_16x16x32_f16_index_key(<8 x half> %A, <16 x half> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out0, ptr addrspace(1) %out1) { ; GFX12-LABEL: test_swmmac_f32_16x16x32_f16_index_key: diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32.ll index c4edc5b72b2f..3aa81da317d6 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 +; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 define amdgpu_ps void @test_wmma_f32_16x16x16_f16(<8 x half> %A, <8 x half> %B, <8 x float> %C, ptr addrspace(1) %out) { ; GFX12-LABEL: test_wmma_f32_16x16x16_f16: diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-f16-f32-matrix-modifiers.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-f16-f32-matrix-modifiers.ll index e2831afe68e7..6c232b680ebf 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-f16-f32-matrix-modifiers.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-f16-f32-matrix-modifiers.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 +; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 define amdgpu_ps void @test_wmma_f32_16x16x16_f16_negA(<4 x half> %A, <4 x half> %B, <4 x float> %C, ptr addrspace(1) %out) { ; GFX12-LABEL: test_wmma_f32_16x16x16_f16_negA: diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-imm.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-imm.ll index c4d70fd5f063..717a4fc823d5 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-imm.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-imm.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 +; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 define amdgpu_ps void @test_wmma_f32_16x16x16_f16_imm(<4 x half> %A, <4 x half> %B, ptr addrspace(1) %out) { ; GFX12-LABEL: test_wmma_f32_16x16x16_f16_imm: diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-iu-modifiers.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-iu-modifiers.ll index 7e1d09805df3..1ef50cbd0fc7 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-iu-modifiers.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-iu-modifiers.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 +; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 define amdgpu_ps void @test_wmma_i32_16x16x16_iu8_zext_src0(i32 %A, i32 %B, <4 x i32> %C, ptr addrspace(1) %out) { ; GFX12-LABEL: test_wmma_i32_16x16x16_iu8_zext_src0: diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-swmmac-index_key.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-swmmac-index_key.ll index b6f1828dce25..0bd255e5e1af 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-swmmac-index_key.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-swmmac-index_key.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 +; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 define amdgpu_ps void @test_swmmac_f32_16x16x32_f16_index_key(<4 x half> %A, <8 x half> %B, <4 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(1) %out2, ptr addrspace(1) %out3) { ; GFX12-LABEL: test_swmmac_f32_16x16x32_f16_index_key: diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64.ll index 0d1871a18d40..7399fa0a341e 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 +; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 define amdgpu_ps void @test_wmma_f32_16x16x16_f16(<4 x half> %A, <4 x half> %B, <4 x float> %C, ptr addrspace(1) %out) { ; GFX12-LABEL: test_wmma_f32_16x16x16_f16: diff --git a/llvm/test/CodeGen/AMDGPU/atomics_cond_sub.ll b/llvm/test/CodeGen/AMDGPU/atomics_cond_sub.ll index 0772f9d0199f..2c69ae58f0e6 100644 --- a/llvm/test/CodeGen/AMDGPU/atomics_cond_sub.ll +++ b/llvm/test/CodeGen/AMDGPU/atomics_cond_sub.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12-SDAG %s -; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12-GISEL %s +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12-SDAG %s +; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12-GISEL %s declare i32 @llvm.amdgcn.atomic.cond.sub.u32.p3(ptr addrspace(3), i32) declare i32 @llvm.amdgcn.atomic.cond.sub.u32.p1(ptr addrspace(1), i32) diff --git a/llvm/test/CodeGen/AMDGPU/generic-targets-require-v6.ll b/llvm/test/CodeGen/AMDGPU/generic-targets-require-v6.ll index 482f61624ec7..15a696bb3af0 100644 --- a/llvm/test/CodeGen/AMDGPU/generic-targets-require-v6.ll +++ b/llvm/test/CodeGen/AMDGPU/generic-targets-require-v6.ll @@ -1,12 +1,12 @@ -; RUN: not llc -march=amdgcn -mcpu=gfx9-generic --amdhsa-code-object-version=5 -o - %s 2>&1 | FileCheck --check-prefix=GFX9-V5 %s -; RUN: not llc -march=amdgcn -mcpu=gfx10-1-generic --amdhsa-code-object-version=5 -o - %s 2>&1 | FileCheck --check-prefix=GFX101-V5 %s -; RUN: not llc -march=amdgcn -mcpu=gfx10-3-generic --amdhsa-code-object-version=5 -o - %s 2>&1 | FileCheck --check-prefix=GFX103-V5 %s -; RUN: not llc -march=amdgcn -mcpu=gfx11-generic --amdhsa-code-object-version=5 -o - %s 2>&1 | FileCheck --check-prefix=GFX11-V5 %s +; RUN: not llc -mtriple=amdgcn -mcpu=gfx9-generic --amdhsa-code-object-version=5 -o - %s 2>&1 | FileCheck --check-prefix=GFX9-V5 %s +; RUN: not llc -mtriple=amdgcn -mcpu=gfx10-1-generic --amdhsa-code-object-version=5 -o - %s 2>&1 | FileCheck --check-prefix=GFX101-V5 %s +; RUN: not llc -mtriple=amdgcn -mcpu=gfx10-3-generic --amdhsa-code-object-version=5 -o - %s 2>&1 | FileCheck --check-prefix=GFX103-V5 %s +; RUN: not llc -mtriple=amdgcn -mcpu=gfx11-generic --amdhsa-code-object-version=5 -o - %s 2>&1 | FileCheck --check-prefix=GFX11-V5 %s -; RUN: llc -march=amdgcn -mcpu=gfx9-generic --amdhsa-code-object-version=6 -o - %s -; RUN: llc -march=amdgcn -mcpu=gfx10-1-generic --amdhsa-code-object-version=6 -o - %s -; RUN: llc -march=amdgcn -mcpu=gfx10-3-generic --amdhsa-code-object-version=6 -o - %s -; RUN: llc -march=amdgcn -mcpu=gfx11-generic --amdhsa-code-object-version=6 -o - %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx9-generic --amdhsa-code-object-version=6 -o - %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx10-1-generic --amdhsa-code-object-version=6 -o - %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx10-3-generic --amdhsa-code-object-version=6 -o - %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx11-generic --amdhsa-code-object-version=6 -o - %s ; GFX9-V5: gfx9-generic is only available on code object version 6 or better ; GFX101-V5: gfx10-1-generic is only available on code object version 6 or better diff --git a/llvm/test/CodeGen/AMDGPU/gfx12_scalar_subword_loads.ll b/llvm/test/CodeGen/AMDGPU/gfx12_scalar_subword_loads.ll index 6c324ddc6546..c69207c0472e 100644 --- a/llvm/test/CodeGen/AMDGPU/gfx12_scalar_subword_loads.ll +++ b/llvm/test/CodeGen/AMDGPU/gfx12_scalar_subword_loads.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,DAG %s -; RUN: llc -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GISEL %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,DAG %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GISEL %s define amdgpu_ps void @test_s_load_i8(ptr addrspace(4) inreg %in, ptr addrspace(1) %out) { ; GCN-LABEL: test_s_load_i8: diff --git a/llvm/test/CodeGen/AMDGPU/insert-waitcnts-hang.mir b/llvm/test/CodeGen/AMDGPU/insert-waitcnts-hang.mir index 993933b2b5c7..28d79efc00b0 100644 --- a/llvm/test/CodeGen/AMDGPU/insert-waitcnts-hang.mir +++ b/llvm/test/CodeGen/AMDGPU/insert-waitcnts-hang.mir @@ -1,5 +1,5 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4 -# RUN: llc -march=amdgcn -mcpu=gfx1100 -run-pass si-insert-waitcnts %s -o - | FileCheck %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass si-insert-waitcnts %s -o - | FileCheck %s --- name: test diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.cond.sub.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.cond.sub.ll index 11bafa197a2f..9b63a8a3efcf 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.cond.sub.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.cond.sub.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s -check-prefix=GFX12 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s -check-prefix=GFX12 define float @raw_buffer_atomic_cond_sub_return(<4 x i32> inreg %rsrc, i32 inreg %data) #0 { ; GFX12-LABEL: raw_buffer_atomic_cond_sub_return: diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dot4.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dot4.f32.ll index 02e27152bf5c..8ea10f4496a2 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dot4.f32.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dot4.f32.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12 %s -; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12 %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12 %s +; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12 %s define float @test_amdgcn_dot4_f32_fp8_bf8(i32 %a, i32 %b, float %c) { ; GFX12-LABEL: test_amdgcn_dot4_f32_fp8_bf8: diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w32.ll index 8f1e6f3ac1a0..b4415c12926a 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w32.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w32.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=+wavefrontsize32,-wavefrontsize64 < %s | FileCheck -check-prefixes=GFX12-SDAG-W32 %s -; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=+wavefrontsize32,-wavefrontsize64 < %s | FileCheck -check-prefixes=GFX12-GISEL-W32 %s +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=+wavefrontsize32,-wavefrontsize64 < %s | FileCheck -check-prefixes=GFX12-SDAG-W32 %s +; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=+wavefrontsize32,-wavefrontsize64 < %s | FileCheck -check-prefixes=GFX12-GISEL-W32 %s declare <2 x i32> @llvm.amdgcn.global.load.tr.v2i32.p1(ptr addrspace(1)) declare <8 x i16> @llvm.amdgcn.global.load.tr.v8i16.p1(ptr addrspace(1)) diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w64.ll index d5a45fb838fc..7ad1416789de 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w64.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w64.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=-wavefrontsize32,+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX12-SDAG-W64 %s -; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=-wavefrontsize32,+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX12-GISEL-W64 %s +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=-wavefrontsize32,+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX12-SDAG-W64 %s +; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=-wavefrontsize32,+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX12-GISEL-W64 %s declare i32 @llvm.amdgcn.global.load.tr.i32.p1(ptr addrspace(1)) declare <4 x i16> @llvm.amdgcn.global.load.tr.v4i16.p1(ptr addrspace(1)) diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.wait.gfx12.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.wait.gfx12.ll index f03dbb9eb164..ff8f28dae3f8 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.wait.gfx12.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.wait.gfx12.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s -check-prefix=GFX12 -; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s -check-prefix=GFX12 +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s -check-prefix=GFX12 +; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s -check-prefix=GFX12 define amdgpu_ps void @test_bvhcnt() { ; GFX12-LABEL: test_bvhcnt: diff --git a/llvm/test/CodeGen/AMDGPU/spill-regpressure-less.mir b/llvm/test/CodeGen/AMDGPU/spill-regpressure-less.mir index f50688240fe8..ed57caadea5c 100644 --- a/llvm/test/CodeGen/AMDGPU/spill-regpressure-less.mir +++ b/llvm/test/CodeGen/AMDGPU/spill-regpressure-less.mir @@ -1,5 +1,5 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3 -# RUN: llc -march=amdgcn -mcpu=gfx90a -run-pass=machine-scheduler -verify-misched -o - %s | FileCheck -check-prefix=GCN %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=machine-scheduler -verify-misched -o - %s | FileCheck -check-prefix=GCN %s --- | define amdgpu_kernel void @spill_regpressure_less() #0 { diff --git a/llvm/test/CodeGen/AMDGPU/wait-before-stores-with-scope_sys.ll b/llvm/test/CodeGen/AMDGPU/wait-before-stores-with-scope_sys.ll index e6fbe97f8dc0..96fa2a45a2dd 100644 --- a/llvm/test/CodeGen/AMDGPU/wait-before-stores-with-scope_sys.ll +++ b/llvm/test/CodeGen/AMDGPU/wait-before-stores-with-scope_sys.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 -; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX12 %s -; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX12 %s +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX12 %s +; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX12 %s define amdgpu_ps void @intrinsic_store_system_scope(i32 %val, <4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) { ; GFX12-LABEL: intrinsic_store_system_scope: diff --git a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll index 5f662ac088a3..cb3d76cd9c0b 100644 --- a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll +++ b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 define amdgpu_ps void @test_wmma_f32_16x16x16_f16_negA(<8 x half> %A, <8 x half> %B, <8 x float> %C, ptr addrspace(1) %out) { ; GFX12-LABEL: test_wmma_f32_16x16x16_f16_negA: diff --git a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-imm.ll b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-imm.ll index c80e5e0e3506..c4adc8c32128 100644 --- a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-imm.ll +++ b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-imm.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 define amdgpu_ps void @test_wmma_f32_16x16x16_f16_imm(<8 x half> %A, <8 x half> %B, ptr addrspace(1) %out) { ; GFX12-LABEL: test_wmma_f32_16x16x16_f16_imm: diff --git a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-iu-modifiers.ll b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-iu-modifiers.ll index 5426458e6b1d..dbb4db05a35c 100644 --- a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-iu-modifiers.ll +++ b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-iu-modifiers.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 define amdgpu_ps void @test_wmma_i32_16x16x16_iu8_zext_src0(<2 x i32> %A, <2 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out) { ; GFX12-LABEL: test_wmma_i32_16x16x16_iu8_zext_src0: diff --git a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-swmmac-index_key.ll b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-swmmac-index_key.ll index b0213abba904..009288dbdf53 100644 --- a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-swmmac-index_key.ll +++ b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-swmmac-index_key.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 define amdgpu_ps void @test_swmmac_f32_16x16x32_f16_index_key(<8 x half> %A, <16 x half> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out0, ptr addrspace(1) %out1) { ; GFX12-LABEL: test_swmmac_f32_16x16x32_f16_index_key: diff --git a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32.ll b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32.ll index a66747567dd3..1012287838f1 100644 --- a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32.ll +++ b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 define amdgpu_ps void @test_wmma_f32_16x16x16_f16(<8 x half> %A, <8 x half> %B, <8 x float> %C, ptr addrspace(1) %out) { ; GFX12-LABEL: test_wmma_f32_16x16x16_f16: diff --git a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-f16-f32-matrix-modifiers.ll b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-f16-f32-matrix-modifiers.ll index 1e82e74d92c4..ab1121a70552 100644 --- a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-f16-f32-matrix-modifiers.ll +++ b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-f16-f32-matrix-modifiers.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -march=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 define amdgpu_ps void @test_wmma_f32_16x16x16_f16_negA(<4 x half> %A, <4 x half> %B, <4 x float> %C, ptr addrspace(1) %out) { ; GFX12-LABEL: test_wmma_f32_16x16x16_f16_negA: diff --git a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-imm.ll b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-imm.ll index 19b0e697183f..462fc01e8e79 100644 --- a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-imm.ll +++ b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-imm.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -march=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 define amdgpu_ps void @test_wmma_f32_16x16x16_f16_imm(<4 x half> %A, <4 x half> %B, ptr addrspace(1) %out) { ; GFX12-LABEL: test_wmma_f32_16x16x16_f16_imm: diff --git a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-iu-modifiers.ll b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-iu-modifiers.ll index fa5eb3605e67..161d222d10ff 100644 --- a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-iu-modifiers.ll +++ b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-iu-modifiers.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -march=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 define amdgpu_ps void @test_wmma_i32_16x16x16_iu8_zext_src0(i32 %A, i32 %B, <4 x i32> %C, ptr addrspace(1) %out) { ; GFX12-LABEL: test_wmma_i32_16x16x16_iu8_zext_src0: diff --git a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-swmmac-index_key.ll b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-swmmac-index_key.ll index 861eb1aaa333..511a116a78be 100644 --- a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-swmmac-index_key.ll +++ b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-swmmac-index_key.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -march=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 define amdgpu_ps void @test_swmmac_f32_16x16x32_f16_index_key(<4 x half> %A, <8 x half> %B, <4 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(1) %out2, ptr addrspace(1) %out3) { ; GFX12-LABEL: test_swmmac_f32_16x16x32_f16_index_key: diff --git a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64.ll b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64.ll index a05a8f4117ec..5fde11cb4b1b 100644 --- a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64.ll +++ b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -march=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX12 define amdgpu_ps void @test_wmma_f32_16x16x16_f16(<4 x half> %A, <4 x half> %B, <4 x float> %C, ptr addrspace(1) %out) { ; GFX12-LABEL: test_wmma_f32_16x16x16_f16: diff --git a/llvm/test/CodeGen/AMDGPU/wmma-hazards-gfx12-w32.mir b/llvm/test/CodeGen/AMDGPU/wmma-hazards-gfx12-w32.mir index 47a1e06c5d7d..ef85de201294 100644 --- a/llvm/test/CodeGen/AMDGPU/wmma-hazards-gfx12-w32.mir +++ b/llvm/test/CodeGen/AMDGPU/wmma-hazards-gfx12-w32.mir @@ -1,5 +1,5 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass post-RA-hazard-rec %s -o - | FileCheck -check-prefix=GFX12 %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass post-RA-hazard-rec %s -o - | FileCheck -check-prefix=GFX12 %s # D0 overlaps A1, B1, C1 or Index1. Overlap starts at vgpr0. # $D0 = wmma0 $A0, $B0, $C0 or $D0 = swmmac0 $A0, $B0, $C0, $Index0 diff --git a/llvm/test/CodeGen/AMDGPU/wmma-hazards-gfx12-w64.mir b/llvm/test/CodeGen/AMDGPU/wmma-hazards-gfx12-w64.mir index 34c37aa91ab8..277db33e940d 100644 --- a/llvm/test/CodeGen/AMDGPU/wmma-hazards-gfx12-w64.mir +++ b/llvm/test/CodeGen/AMDGPU/wmma-hazards-gfx12-w64.mir @@ -1,5 +1,5 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=-wavefrontsize32,+wavefrontsize64 -run-pass post-RA-hazard-rec %s -o - | FileCheck -check-prefix=GFX12 %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=-wavefrontsize32,+wavefrontsize64 -run-pass post-RA-hazard-rec %s -o - | FileCheck -check-prefix=GFX12 %s # D0 overlaps A1, B1, C1 or Index1. Overlap starts at vgpr0. # $D0 = wmma0 $A0, $B0, $C0 or $D0 = swmmac0 $A0, $B0, $C0, $Index0 diff --git a/llvm/test/MachineVerifier/convergencectrl/AMDGPU/basic.mir b/llvm/test/MachineVerifier/convergencectrl/AMDGPU/basic.mir index 94d0ddad2594..cb06d90ccd7f 100644 --- a/llvm/test/MachineVerifier/convergencectrl/AMDGPU/basic.mir +++ b/llvm/test/MachineVerifier/convergencectrl/AMDGPU/basic.mir @@ -1,4 +1,4 @@ -# RUN: not --crash llc -march=amdgcn -run-pass=none -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s +# RUN: not --crash llc -mtriple=amdgcn -run-pass=none -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s --- name: basic tracksRegLiveness: true diff --git a/llvm/test/MachineVerifier/convergencectrl/AMDGPU/cycles.mir b/llvm/test/MachineVerifier/convergencectrl/AMDGPU/cycles.mir index 87cf3e604929..d935d8ea4be5 100644 --- a/llvm/test/MachineVerifier/convergencectrl/AMDGPU/cycles.mir +++ b/llvm/test/MachineVerifier/convergencectrl/AMDGPU/cycles.mir @@ -1,4 +1,4 @@ -# RUN: not --crash llc -march=amdgcn -run-pass=none -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s +# RUN: not --crash llc -mtriple=amdgcn -run-pass=none -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s --- name: cycles body: | diff --git a/llvm/test/MachineVerifier/convergencectrl/AMDGPU/mixed2.mir b/llvm/test/MachineVerifier/convergencectrl/AMDGPU/mixed2.mir index c70a48bf2130..7893837126e7 100644 --- a/llvm/test/MachineVerifier/convergencectrl/AMDGPU/mixed2.mir +++ b/llvm/test/MachineVerifier/convergencectrl/AMDGPU/mixed2.mir @@ -1,4 +1,4 @@ -# RUN: not --crash llc -march=amdgcn -run-pass=none -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s +# RUN: not --crash llc -mtriple=amdgcn -run-pass=none -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s --- name: mixed2 body: | diff --git a/llvm/test/MachineVerifier/convergencectrl/AMDGPU/region-nesting.mir b/llvm/test/MachineVerifier/convergencectrl/AMDGPU/region-nesting.mir index 9e869acb3e93..e9588d25d774 100644 --- a/llvm/test/MachineVerifier/convergencectrl/AMDGPU/region-nesting.mir +++ b/llvm/test/MachineVerifier/convergencectrl/AMDGPU/region-nesting.mir @@ -1,4 +1,4 @@ -# RUN: not --crash llc -march=amdgcn -run-pass=none -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s +# RUN: not --crash llc -mtriple=amdgcn -run-pass=none -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s --- name: region_nesting body: | -- GitLab From b565126b4dbef0d9f5c9f3ef8f3489ff6581218f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolai=20H=C3=A4hnle?= Date: Fri, 8 Mar 2024 04:24:28 +0100 Subject: [PATCH 558/929] Revert "update_test_checks: keep meta variables stable by default" This reverts commit fb02f9ac84a6151e41aba8f7391edd132a9aaf14. Looks like some Python version incompatibility, will investigate. --- .../Inputs/stable_ir_values.ll.expected | 10 +- .../Inputs/stable_ir_values.ll.expected.reset | 23 - .../Inputs/stable_ir_values2.ll.expected | 12 +- .../Inputs/stable_ir_values3.ll | 3 + .../Inputs/stable_ir_values3.ll.expected | 3 + .../Inputs/stable_ir_values4.ll.expected | 8 +- .../update_test_checks/stable_ir_values.test | 3 - llvm/utils/UpdateTestChecks/common.py | 513 +----------------- llvm/utils/update_test_checks.py | 22 +- 9 files changed, 43 insertions(+), 554 deletions(-) delete mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected.reset diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected index 3549a4d76aa7..5142e3ed32ba 100644 --- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected @@ -9,11 +9,11 @@ define i32 @func({i32, i32} %x, i32 %y) { ; CHECK-LABEL: define i32 @func( ; CHECK-SAME: { i32, i32 } [[X:%.*]], i32 [[Y:%.*]]) { -; CHECK-NEXT: [[X_I33:%.*]] = extractvalue { i32, i32 } [[X]], 0 -; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[Y]], 1 -; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[X_I33]], [[TMP3]] -; CHECK-NEXT: [[TMP2:%.*]] = mul i32 [[TMP1]], 3 -; CHECK-NEXT: ret i32 [[TMP2]] +; CHECK-NEXT: [[X_I34:%.*]] = extractvalue { i32, i32 } [[X]], 0 +; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[Y]], 1 +; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[X_I34]], [[TMP1]] +; CHECK-NEXT: [[TMP3:%.*]] = mul i32 [[TMP2]], 3 +; CHECK-NEXT: ret i32 [[TMP3]] ; %x.i34 = extractvalue {i32, i32} %x, 0 %1 = add i32 %y, 1 diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected.reset b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected.reset deleted file mode 100644 index 5142e3ed32ba..000000000000 --- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected.reset +++ /dev/null @@ -1,23 +0,0 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt < %s -S | FileCheck %s - -; The assumption underlying this test is that there are pre-existing check lines -; but something has changed, and we would like to avoid needless changes of -; meta variable names so that diffs end up being easier to read, e.g. avoid -; changing X_I33 into X_I34 or renumbering the various TMP variables. - -define i32 @func({i32, i32} %x, i32 %y) { -; CHECK-LABEL: define i32 @func( -; CHECK-SAME: { i32, i32 } [[X:%.*]], i32 [[Y:%.*]]) { -; CHECK-NEXT: [[X_I34:%.*]] = extractvalue { i32, i32 } [[X]], 0 -; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[Y]], 1 -; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[X_I34]], [[TMP1]] -; CHECK-NEXT: [[TMP3:%.*]] = mul i32 [[TMP2]], 3 -; CHECK-NEXT: ret i32 [[TMP3]] -; - %x.i34 = extractvalue {i32, i32} %x, 0 - %1 = add i32 %y, 1 - %2 = add i32 %x.i34, %1 - %3 = mul i32 %2, 3 - ret i32 %3 -} diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll.expected index 6311a55a1f9d..53f60bda8ee5 100644 --- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll.expected +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll.expected @@ -5,12 +5,12 @@ define i32 @func(i32 %x) { ; CHECK-LABEL: define i32 @func( ; CHECK-SAME: i32 [[X:%.*]]) { ; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[X]], 0 -; CHECK-NEXT: [[TMP6:%.*]] = call i32 @foo(i1 [[TMP1]]) -; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[X]], 2 -; CHECK-NEXT: [[TMP8:%.*]] = call i32 @foo(i1 [[TMP7]]) -; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i32 [[TMP8]], 0 -; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[TMP6]] -; CHECK-NEXT: ret i32 [[TMP10]] +; CHECK-NEXT: [[TMP2:%.*]] = call i32 @foo(i1 [[TMP1]]) +; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[X]], 2 +; CHECK-NEXT: [[TMP4:%.*]] = call i32 @foo(i1 [[TMP3]]) +; CHECK-NEXT: [[TMP5:%.*]] = icmp ne i32 [[TMP4]], 0 +; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 [[TMP2]] +; CHECK-NEXT: ret i32 [[TMP6]] ; %1 = icmp eq i32 %x, 0 %2 = call i32 @foo(i1 %1) diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll index a4f4fc67f78d..3b449291d0e7 100644 --- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll @@ -1,6 +1,9 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 ; RUN: opt < %s -S | FileCheck %s +; Test that we don't regress diff quality by trying to keep variable names +; stable (and messing up the matching). + define i32 @func(i32 %x) { ; CHECK-LABEL: define i32 @func( ; CHECK-SAME: i32 [[X:%.*]]) { diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll.expected index 08d3c22172ee..3d0f772505a6 100644 --- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll.expected +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll.expected @@ -1,6 +1,9 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 ; RUN: opt < %s -S | FileCheck %s +; Test that we don't regress diff quality by trying to keep variable names +; stable (and messing up the matching). + define i32 @func(i32 %x) { ; CHECK-LABEL: define i32 @func( ; CHECK-SAME: i32 [[X:%.*]]) { diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values4.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values4.ll.expected index e3fa51598c48..5962bdafd9ea 100644 --- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values4.ll.expected +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values4.ll.expected @@ -15,12 +15,12 @@ define i32 @func(i32 %x) { ; CHECK-NEXT: [[TMP6:%.*]] = call i32 @foo(i32 [[TMP5]]) ; CHECK-NEXT: [[TMP7:%.*]] = call i32 @foo(i32 [[TMP6]]) ; CHECK-NEXT: [[TMP8:%.*]] = xor i32 [[TMP7]], 1 -; CHECK-NEXT: [[TMP13:%.*]] = call i32 @foo(i32 [[TMP8]]) -; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[TMP13]], 1 -; CHECK-NEXT: [[TMP10:%.*]] = call i32 @foo(i32 [[TMP9]]) +; CHECK-NEXT: [[TMP9:%.*]] = call i32 @foo(i32 [[TMP8]]) +; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[TMP9]], 1 ; CHECK-NEXT: [[TMP11:%.*]] = call i32 @foo(i32 [[TMP10]]) ; CHECK-NEXT: [[TMP12:%.*]] = call i32 @foo(i32 [[TMP11]]) -; CHECK-NEXT: ret i32 [[TMP12]] +; CHECK-NEXT: [[TMP13:%.*]] = call i32 @foo(i32 [[TMP12]]) +; CHECK-NEXT: ret i32 [[TMP13]] ; %1 = mul i32 %x, 3 %2 = call i32 @foo(i32 %1) diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values.test b/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values.test index 4dfaf5d25c8a..c6287a6b29ca 100644 --- a/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values.test +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values.test @@ -1,5 +1,2 @@ # RUN: cp -f %S/Inputs/stable_ir_values.ll %t.ll && %update_test_checks %t.ll # RUN: diff -u %t.ll %S/Inputs/stable_ir_values.ll.expected -# Now test that we can reset all the names -# RUN: %update_test_checks %t.ll --reset-variable-names -# RUN: diff -u %t.ll %S/Inputs/stable_ir_values.ll.expected.reset diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py index f766d541c79c..a3365fef5f6e 100644 --- a/llvm/utils/UpdateTestChecks/common.py +++ b/llvm/utils/UpdateTestChecks/common.py @@ -1,8 +1,6 @@ from __future__ import print_function import argparse -import bisect -import collections import copy import glob import itertools @@ -12,7 +10,7 @@ import subprocess import sys import shlex -from typing import List, Mapping, Set +from typing import List ##### Common utilities for update_*test_checks.py @@ -422,48 +420,6 @@ def should_add_line_to_output( return True -def collect_original_check_lines(ti: TestInfo, prefix_set: set): - """ - Collect pre-existing check lines into a dictionary `result` which is - returned. - - result[func_name][prefix] is filled with a list of right-hand-sides of check - lines. - """ - result = {} - - current_function = None - for input_line_info in ti.ro_iterlines(): - input_line = input_line_info.line - if current_function is not None: - if input_line == "": - continue - if input_line.lstrip().startswith(";"): - m = CHECK_RE.match(input_line) - if ( - m is not None - and m.group(1) in prefix_set - and m.group(2) not in ["LABEL", "SAME"] - ): - if m.group(1) not in current_function: - current_function[m.group(1)] = [] - current_function[m.group(1)].append(input_line[m.end() :].strip()) - continue - current_function = None - - m = IR_FUNCTION_RE.match(input_line) - if m is not None: - func_name = m.group(1) - if ti.args.function is not None and func_name != ti.args.function: - # When filtering on a specific function, skip all others. - continue - - assert func_name not in result - current_function = result[func_name] = {} - - return result - - # Perform lit-like substitutions def getSubstitutions(sourcepath): sourcedir = os.path.dirname(sourcepath) @@ -535,7 +491,7 @@ RUN_LINE_RE = re.compile(r"^\s*(?://|[;#])\s*RUN:\s*(.*)$") CHECK_PREFIX_RE = re.compile(r"--?check-prefix(?:es)?[= ](\S+)") PREFIX_RE = re.compile("^[a-zA-Z0-9_-]+$") CHECK_RE = re.compile( - r"^\s*(?://|[;#])\s*([^:]+?)(?:-(NEXT|NOT|DAG|LABEL|SAME|EMPTY))?:" + r"^\s*(?://|[;#])\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL|-SAME|-EMPTY)?:" ) CHECK_SAME_RE = re.compile(r"^\s*(?://|[;#])\s*([^:]+?)(?:-SAME)?:") @@ -1231,325 +1187,20 @@ def may_clash_with_default_check_prefix_name(check_prefix, var): ) -def find_diff_matching(lhs: List[str], rhs: List[str]) -> List[tuple]: - """ - Find a large ordered matching between strings in lhs and rhs. - - Think of this as finding the *unchanged* lines in a diff, where the entries - of lhs and rhs are lines of the files being diffed. - - Returns a list of matched (lhs_idx, rhs_idx) pairs. - """ - - if not lhs or not rhs: - return [] - - # Collect matches in reverse order. - matches = [] - - # First, collect a set of candidate matching edges. We limit this to a - # constant multiple of the input size to avoid quadratic runtime. - patterns = collections.defaultdict(lambda: ([], [])) - - for idx in range(len(lhs)): - patterns[lhs[idx]][0].append(idx) - for idx in range(len(rhs)): - patterns[rhs[idx]][1].append(idx) - - multiple_patterns = [] - - candidates = [] - for pattern in patterns.values(): - if not pattern[0] or not pattern[1]: - continue - - if len(pattern[0]) == len(pattern[1]) == 1: - candidates.append((pattern[0][0], pattern[1][0])) - else: - multiple_patterns.append(pattern) - - multiple_patterns.sort(key=lambda pattern: len(pattern[0]) * len(pattern[1])) - - for pattern in multiple_patterns: - if len(candidates) + len(pattern[0]) * len(pattern[1]) > 2 * ( - len(lhs) + len(rhs) - ): - break - for lhs_idx in pattern[0]: - for rhs_idx in pattern[1]: - candidates.append((lhs_idx, rhs_idx)) - - if not candidates: - # The LHS and RHS either share nothing in common, or lines are just too - # identical. In that case, let's give up and not match anything. - return [] - - # Compute a maximal crossing-free matching via an algorithm that is - # inspired by a mixture of dynamic programming and line-sweeping in - # discrete geometry. - # - # I would be surprised if this algorithm didn't exist somewhere in the - # literature, but I found it without consciously recalling any - # references, so you'll have to make do with the explanation below. - # Sorry. - # - # The underlying graph is bipartite: - # - nodes on the LHS represent lines in the original check - # - nodes on the RHS represent lines in the new (updated) check - # - # Nodes are implicitly sorted by the corresponding line number. - # Edges (unique_matches) are sorted by the line number on the LHS. - # - # Here's the geometric intuition for the algorithm. - # - # * Plot the edges as points in the plane, with the original line - # number on the X axis and the updated line number on the Y axis. - # * The goal is to find a longest "chain" of points where each point - # is strictly above and to the right of the previous point. - # * The algorithm proceeds by sweeping a vertical line from left to - # right. - # * The algorithm maintains a table where `table[N]` answers the - # question "What is currently the 'best' way to build a chain of N+1 - # points to the left of the vertical line". Here, 'best' means - # that the last point of the chain is a as low as possible (minimal - # Y coordinate). - # * `table[N]` is `(y, point_idx)` where `point_idx` is the index of - # the last point in the chain and `y` is its Y coordinate - # * A key invariant is that the Y values in the table are - # monotonically increasing - # * Thanks to these properties, the table can be used to answer the - # question "What is the longest chain that can be built to the left - # of the vertical line using only points below a certain Y value", - # using a binary search over the table. - # * The algorithm also builds a backlink structure in which every point - # links back to the previous point on a best (longest) chain ending - # at that point - # - # The core loop of the algorithm sweeps the line and updates the table - # and backlink structure for every point that we cross during the sweep. - # Therefore, the algorithm is trivially O(M log M) in the number of - # points. - candidates.sort(key=lambda candidate: (candidate[0], -candidate[1])) - - backlinks = [] - table = [] - for _, rhs_idx in candidates: - candidate_idx = len(backlinks) - ti = bisect.bisect_left(table, rhs_idx, key=lambda entry: entry[0]) - - # Update the table to record a best chain ending in the current point. - # There always is one, and if any of the previously visited points had - # a higher Y coordinate, then there is always a previously recorded best - # chain that can be improved upon by using the current point. - # - # There is only one case where there is some ambiguity. If the - # pre-existing entry table[ti] has the same Y coordinate / rhs_idx as - # the current point (this can only happen if the same line appeared - # multiple times on the LHS), then we could choose to keep the - # previously recorded best chain instead. That would bias the algorithm - # differently but should have no systematic impact on the quality of the - # result. - if ti < len(table): - table[ti] = (rhs_idx, candidate_idx) - else: - table.append((rhs_idx, candidate_idx)) - if ti > 0: - backlinks.append(table[ti - 1][1]) - else: - backlinks.append(None) - - # Commit to names in the matching by walking the backlinks. Recursively - # attempt to fill in more matches in-betweem. - match_idx = table[-1][1] - while match_idx is not None: - current = candidates[match_idx] - matches.append(current) - match_idx = backlinks[match_idx] - - matches.reverse() - return matches - - -VARIABLE_TAG = "[[@@]]" -METAVAR_RE = re.compile(r"\[\[([A-Z0-9_]+)(?::[^]]+)?\]\]") -NUMERIC_SUFFIX_RE = re.compile(r"[0-9]*$") - - -class CheckValueInfo: - def __init__( - self, - nameless_value: NamelessValue, - var: str, - prefix: str, - ): - self.nameless_value = nameless_value - self.var = var - self.prefix = prefix - - -# Represent a check line in a way that allows us to compare check lines while -# ignoring some or all of the FileCheck variable names. -class CheckLineInfo: - def __init__(self, line, values): - # Line with all FileCheck variable name occurrences replaced by VARIABLE_TAG - self.line: str = line - - # Information on each FileCheck variable name occurrences in the line - self.values: List[CheckValueInfo] = values - - def __repr__(self): - return f"CheckLineInfo(line={self.line}, self.values={self.values})" - - -def remap_metavar_names( - old_line_infos: List[CheckLineInfo], - new_line_infos: List[CheckLineInfo], - committed_names: Set[str], -) -> Mapping[str, str]: - """ - Map all FileCheck variable names that appear in new_line_infos to new - FileCheck variable names in an attempt to reduce the diff from old_line_infos - to new_line_infos. - - This is done by: - * Matching old check lines and new check lines using a diffing algorithm - applied after replacing names with wildcards. - * Committing to variable names such that the matched lines become equal - (without wildcards) if possible - * This is done recursively to handle cases where many lines are equal - after wildcard replacement - """ - # Initialize uncommitted identity mappings - new_mapping = {} - for line in new_line_infos: - for value in line.values: - new_mapping[value.var] = value.var - - # Recursively commit to the identity mapping or find a better one - def recurse(old_begin, old_end, new_begin, new_end): - if old_begin == old_end or new_begin == new_end: - return - - # Find a matching of lines where uncommitted names are replaced - # with a placeholder. - def diffify_line(line, mapper): - values = [] - for value in line.values: - mapped = mapper(value.var) - values.append(mapped if mapped in committed_names else "?") - return line.line.strip() + " @@@ " + " @ ".join(values) - - lhs_lines = [ - diffify_line(line, lambda x: x) - for line in old_line_infos[old_begin:old_end] - ] - rhs_lines = [ - diffify_line(line, lambda x: new_mapping[x]) - for line in new_line_infos[new_begin:new_end] - ] - - candidate_matches = find_diff_matching(lhs_lines, rhs_lines) - - # Apply commits greedily on a match-by-match basis - matches = [(-1, -1)] - committed_anything = False - for lhs_idx, rhs_idx in candidate_matches: - lhs_line = old_line_infos[lhs_idx] - rhs_line = new_line_infos[rhs_idx] - - local_commits = {} - - for lhs_value, rhs_value in zip(lhs_line.values, rhs_line.values): - if new_mapping[rhs_value.var] in committed_names: - # The new value has already been committed. If it was mapped - # to the same name as the original value, we can consider - # committing other values from this line. Otherwise, we - # should ignore this line. - if new_mapping[rhs_value.var] == lhs_value.var: - continue - else: - break - - if rhs_value.var in local_commits: - # Same, but for a possible commit happening on the same line - if local_commits[rhs_value.var] == lhs_value.var: - continue - else: - break - - if lhs_value.var in committed_names: - # We can't map this value because the name we would map it to has already been - # committed for something else. Give up on this line. - break - - local_commits[rhs_value.var] = lhs_value.var - else: - # No reason not to add any commitments for this line - for rhs_var, lhs_var in local_commits.items(): - new_mapping[rhs_var] = lhs_var - committed_names.add(lhs_var) - committed_anything = True - - if ( - lhs_var != rhs_var - and lhs_var in new_mapping - and new_mapping[lhs_var] == lhs_var - ): - new_mapping[lhs_var] = "conflict_" + lhs_var - - matches.append((lhs_idx, rhs_idx)) - - matches.append((old_end, new_end)) - - # Recursively handle sequences between matches - if committed_anything: - for (lhs_prev, rhs_prev), (lhs_next, rhs_next) in zip(matches, matches[1:]): - recurse(lhs_prev + 1, lhs_next, rhs_prev + 1, rhs_next) - - recurse(0, len(old_line_infos), 0, len(new_line_infos)) - - # Commit to remaining names and resolve conflicts - for new_name, mapped_name in new_mapping.items(): - if mapped_name in committed_names: - continue - if not mapped_name.startswith("conflict_"): - assert mapped_name == new_name - committed_names.add(mapped_name) - - for new_name, mapped_name in new_mapping.items(): - if mapped_name in committed_names: - continue - assert mapped_name.startswith("conflict_") - - m = NUMERIC_SUFFIX_RE.search(new_name) - base_name = new_name[: m.start()] - suffix = int(new_name[m.start() :]) if m.start() != m.end() else 1 - while True: - candidate = f"{base_name}{suffix}" - if candidate not in committed_names: - new_mapping[new_name] = candidate - committed_names.add(candidate) - break - suffix += 1 - - return new_mapping - - def generalize_check_lines_common( lines, is_analyze, vars_seen, global_vars_seen, nameless_values, - nameless_value_regex: re.Pattern, + nameless_value_regex, is_asm, preserve_names, - original_check_lines=None, ): # This gets called for each match that occurs in # a line. We transform variables we haven't seen # into defs, and variables we have seen into uses. - def transform_line_vars(match, transform_locals=True): + def transform_line_vars(match): var = get_name_from_ir_value_match(match) nameless_value = get_nameless_value_from_match(match, nameless_values) if may_clash_with_default_check_prefix_name(nameless_value.check_prefix, var): @@ -1559,8 +1210,6 @@ def generalize_check_lines_common( ) key = (var, nameless_value.check_key) is_local_def = nameless_value.is_local_def_ir_value() - if is_local_def and not transform_locals: - return None if is_local_def and key in vars_seen: rv = nameless_value.get_value_use(var, match) elif not is_local_def and key in global_vars_seen: @@ -1579,15 +1228,13 @@ def generalize_check_lines_common( # including the commas and spaces. return match.group(1) + rv + match.group(match.lastindex) - def transform_non_local_line_vars(match): - return transform_line_vars(match, False) - + lines_with_def = [] multiple_braces_re = re.compile(r"({{+)|(}}+)") def escape_braces(match_obj): return '{{' + re.escape(match_obj.group(0)) + '}}' - if not is_asm and not is_analyze: - for i, line in enumerate(lines): + for i, line in enumerate(lines): + if not is_asm and not is_analyze: # An IR variable named '%.' matches the FileCheck regex string. line = line.replace("%.", "%dot") for regex in _global_hex_value_regex: @@ -1605,136 +1252,25 @@ def generalize_check_lines_common( # Ignore any comments, since the check lines will too. scrubbed_line = SCRUB_IR_COMMENT_RE.sub(r"", line) lines[i] = scrubbed_line - - if not preserve_names: - if is_asm: - for i, _ in enumerate(lines): - # It can happen that two matches are back-to-back and for some reason sub - # will not replace both of them. For now we work around this by - # substituting until there is no more match. - changed = True - while changed: - (lines[i], changed) = nameless_value_regex.subn( - transform_line_vars, lines[i], count=1 - ) - else: - # LLVM IR case. Start by handling global meta variables (global IR variables, - # metadata, attributes) - for i, _ in enumerate(lines): - start = 0 - while True: - m = nameless_value_regex.search(lines[i][start:]) - if m is None: - break - start += m.start() - sub = transform_non_local_line_vars(m) - if sub is not None: - lines[i] = ( - lines[i][:start] + sub + lines[i][start + len(m.group(0)) :] - ) - start += 1 - - # Collect information about new check lines and original check lines (if any) - new_line_infos = [] - for line in lines: - filtered_line = "" - values = [] - while True: - m = nameless_value_regex.search(line) - if m is None: - filtered_line += line - break - - var = get_name_from_ir_value_match(m) - nameless_value = get_nameless_value_from_match(m, nameless_values) - var = nameless_value.get_value_name( - var, nameless_value.check_prefix - ) - - # Replace with a [[@@]] tag, but be sure to keep the spaces and commas. - filtered_line += ( - line[: m.start()] - + m.group(1) - + VARIABLE_TAG - + m.group(m.lastindex) - ) - line = line[m.end() :] - values.append( - CheckValueInfo( - nameless_value=nameless_value, - var=var, - prefix=nameless_value.get_ir_prefix_from_ir_value_match(m)[ - 0 - ], - ) - ) - new_line_infos.append(CheckLineInfo(filtered_line, values)) - - orig_line_infos = [] - for line in original_check_lines or []: - filtered_line = "" - values = [] - while True: - m = METAVAR_RE.search(line) - if m is None: - filtered_line += line - break - - # Replace with a [[@@]] tag, but be sure to keep the spaces and commas. - filtered_line += line[: m.start()] + VARIABLE_TAG - line = line[m.end() :] - values.append( - CheckValueInfo( - nameless_value=None, - var=m.group(1), - prefix=None, - ) - ) - orig_line_infos.append(CheckLineInfo(filtered_line, values)) - - # Compute the variable name mapping - committed_names = set(vars_seen) - - mapping = remap_metavar_names( - orig_line_infos, new_line_infos, committed_names - ) - - for i, line_info in enumerate(new_line_infos): - line_template = line_info.line - line = "" - - for value in line_info.values: - idx = line_template.find(VARIABLE_TAG) - line += line_template[:idx] - line_template = line_template[idx + len(VARIABLE_TAG) :] - - key = (mapping[value.var], nameless_value.check_key) - is_local_def = nameless_value.is_local_def_ir_value() - if is_local_def: - if mapping[value.var] in vars_seen: - line += f"[[{mapping[value.var]}]]" - else: - line += f"[[{mapping[value.var]}:{value.prefix}{value.nameless_value.get_ir_regex()}]]" - vars_seen.add(mapping[value.var]) - else: - raise RuntimeError("not implemented") - - line += line_template - - lines[i] = line - - if is_analyze: - for i, _ in enumerate(lines): + if not preserve_names: + # It can happen that two matches are back-to-back and for some reason sub + # will not replace both of them. For now we work around this by + # substituting until there is no more match. + changed = True + while changed: + (lines[i], changed) = nameless_value_regex.subn( + transform_line_vars, lines[i], count=1 + ) + if is_analyze: # Escape multiple {{ or }} as {{}} denotes a FileCheck regex. scrubbed_line = multiple_braces_re.sub(escape_braces, lines[i]) lines[i] = scrubbed_line - return lines # Replace IR value defs and uses with FileCheck variables. def generalize_check_lines( - lines, is_analyze, vars_seen, global_vars_seen, preserve_names, original_check_lines + lines, is_analyze, vars_seen, global_vars_seen, preserve_names ): return generalize_check_lines_common( lines, @@ -1745,7 +1281,6 @@ def generalize_check_lines( IR_VALUE_RE, False, preserve_names, - original_check_lines=original_check_lines, ) @@ -1802,7 +1337,6 @@ def add_checks( global_vars_seen_dict, is_filtered, preserve_names=False, - original_check_lines: Mapping[str, List[str]] = {}, ): # prefix_exclusions are prefixes we cannot use to print the function because it doesn't exist in run lines that use these prefixes as well. prefix_exclusions = set() @@ -1875,7 +1409,6 @@ def add_checks( vars_seen, global_vars_seen, preserve_names, - original_check_lines=[], )[0] func_name_separator = func_dict[checkprefix][func_name].func_name_separator if "[[" in args_and_sig: @@ -1983,12 +1516,7 @@ def add_checks( # to variable naming fashions. else: func_body = generalize_check_lines( - func_body, - False, - vars_seen, - global_vars_seen, - preserve_names, - original_check_lines=original_check_lines.get(checkprefix), + func_body, False, vars_seen, global_vars_seen, preserve_names ) # This could be selectively enabled with an optional invocation argument. @@ -2050,7 +1578,6 @@ def add_ir_checks( version, global_vars_seen_dict, is_filtered, - original_check_lines={}, ): # Label format is based on IR string. if function_sig and version > 1: @@ -2075,7 +1602,6 @@ def add_ir_checks( global_vars_seen_dict, is_filtered, preserve_names, - original_check_lines=original_check_lines, ) @@ -2364,7 +1890,6 @@ def get_autogennote_suffix(parser, args): "llvm_bin", "verbose", "force_update", - "reset_variable_names", ): continue value = getattr(args, action.dest) diff --git a/llvm/utils/update_test_checks.py b/llvm/utils/update_test_checks.py index 04808ce6bb1c..b5077d793513 100755 --- a/llvm/utils/update_test_checks.py +++ b/llvm/utils/update_test_checks.py @@ -85,12 +85,6 @@ def main(): choices=["none", "smart", "all"], help="Check global entries (global variables, metadata, attribute sets, ...) for functions", ) - parser.add_argument( - "--reset-variable-names", - action="store_true", - help="Reset all variable names to correspond closely to the variable names in IR. " - "This tends to result in larger diffs.", - ) parser.add_argument("tests", nargs="+") initial_args = common.parse_commandline_args(parser) @@ -176,19 +170,13 @@ def main(): ) builder.processed_prefixes(prefixes) - prefix_set = set( - [prefix for prefixes, _, _ in prefix_list for prefix in prefixes] - ) - - if not ti.args.reset_variable_names: - original_check_lines = common.collect_original_check_lines(ti, prefix_set) - else: - original_check_lines = {} - func_dict = builder.finish_and_get_func_dict() is_in_function = False is_in_function_start = False has_checked_pre_function_globals = False + prefix_set = set( + [prefix for prefixes, _, _ in prefix_list for prefix in prefixes] + ) common.debug("Rewriting FileCheck prefixes:", str(prefix_set)) output_lines = [] @@ -242,7 +230,6 @@ def main(): args.version, global_vars_seen_dict, is_filtered=builder.is_filtered(), - original_check_lines=original_check_lines.get(func, {}), ), ) ) @@ -274,9 +261,6 @@ def main(): args.version, global_vars_seen_dict, is_filtered=builder.is_filtered(), - original_check_lines=original_check_lines.get( - func_name, {} - ), ) ) is_in_function_start = False -- GitLab From 3846019d8f6379ea1a8bf3a0fdfb0202de8e2f2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolai=20H=C3=A4hnle?= Date: Thu, 23 Nov 2023 06:46:07 +0100 Subject: [PATCH 559/929] update_test_checks: keep meta variables stable by default Resubmitting this after previous revert with the following changes: - Split table into table_rhs_idx and table_candidate_idx so that bisect.bisect_left can be used without the `key` argument, which was introduced in Python 3.10 - Remove a re.Pattern type annotation Original commit message: Prior to this change, running UTC on larger tests, especially tests with unnamed IR values, often resulted in a spuriously large diff because e.g. TMPnn variables in the CHECK lines were renumbered. This change attempts to reduce the diff by keeping those variable names the same. There are cases in which this "drift" of variable names can end up being more confusing. The old behavior can be re-enabled with the --reset-variable-names command line argument. The improvement may not be immediately apparent in the diff of this change. The point is that the diff of stable_ir_values.ll against stable_ir_values.ll.expected after this change is smaller. Ideally, we'd also keep meta variables for "global" objects stable, e.g. for attributes (#nn) and metadata (!nn). However, that would require a much more substantial refactoring of how we generate check lines, so I left it for future work. --- .../Inputs/stable_ir_values.ll.expected | 10 +- .../Inputs/stable_ir_values.ll.expected.reset | 23 + .../Inputs/stable_ir_values2.ll.expected | 12 +- .../Inputs/stable_ir_values3.ll | 3 - .../Inputs/stable_ir_values3.ll.expected | 3 - .../Inputs/stable_ir_values4.ll.expected | 8 +- .../update_test_checks/stable_ir_values.test | 3 + llvm/utils/UpdateTestChecks/common.py | 514 +++++++++++++++++- llvm/utils/update_test_checks.py | 22 +- 9 files changed, 556 insertions(+), 42 deletions(-) create mode 100644 llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected.reset diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected index 5142e3ed32ba..3549a4d76aa7 100644 --- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected @@ -9,11 +9,11 @@ define i32 @func({i32, i32} %x, i32 %y) { ; CHECK-LABEL: define i32 @func( ; CHECK-SAME: { i32, i32 } [[X:%.*]], i32 [[Y:%.*]]) { -; CHECK-NEXT: [[X_I34:%.*]] = extractvalue { i32, i32 } [[X]], 0 -; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[Y]], 1 -; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[X_I34]], [[TMP1]] -; CHECK-NEXT: [[TMP3:%.*]] = mul i32 [[TMP2]], 3 -; CHECK-NEXT: ret i32 [[TMP3]] +; CHECK-NEXT: [[X_I33:%.*]] = extractvalue { i32, i32 } [[X]], 0 +; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[Y]], 1 +; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[X_I33]], [[TMP3]] +; CHECK-NEXT: [[TMP2:%.*]] = mul i32 [[TMP1]], 3 +; CHECK-NEXT: ret i32 [[TMP2]] ; %x.i34 = extractvalue {i32, i32} %x, 0 %1 = add i32 %y, 1 diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected.reset b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected.reset new file mode 100644 index 000000000000..5142e3ed32ba --- /dev/null +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values.ll.expected.reset @@ -0,0 +1,23 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 +; RUN: opt < %s -S | FileCheck %s + +; The assumption underlying this test is that there are pre-existing check lines +; but something has changed, and we would like to avoid needless changes of +; meta variable names so that diffs end up being easier to read, e.g. avoid +; changing X_I33 into X_I34 or renumbering the various TMP variables. + +define i32 @func({i32, i32} %x, i32 %y) { +; CHECK-LABEL: define i32 @func( +; CHECK-SAME: { i32, i32 } [[X:%.*]], i32 [[Y:%.*]]) { +; CHECK-NEXT: [[X_I34:%.*]] = extractvalue { i32, i32 } [[X]], 0 +; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[Y]], 1 +; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[X_I34]], [[TMP1]] +; CHECK-NEXT: [[TMP3:%.*]] = mul i32 [[TMP2]], 3 +; CHECK-NEXT: ret i32 [[TMP3]] +; + %x.i34 = extractvalue {i32, i32} %x, 0 + %1 = add i32 %y, 1 + %2 = add i32 %x.i34, %1 + %3 = mul i32 %2, 3 + ret i32 %3 +} diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll.expected index 53f60bda8ee5..6311a55a1f9d 100644 --- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll.expected +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values2.ll.expected @@ -5,12 +5,12 @@ define i32 @func(i32 %x) { ; CHECK-LABEL: define i32 @func( ; CHECK-SAME: i32 [[X:%.*]]) { ; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[X]], 0 -; CHECK-NEXT: [[TMP2:%.*]] = call i32 @foo(i1 [[TMP1]]) -; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[X]], 2 -; CHECK-NEXT: [[TMP4:%.*]] = call i32 @foo(i1 [[TMP3]]) -; CHECK-NEXT: [[TMP5:%.*]] = icmp ne i32 [[TMP4]], 0 -; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 [[TMP2]] -; CHECK-NEXT: ret i32 [[TMP6]] +; CHECK-NEXT: [[TMP6:%.*]] = call i32 @foo(i1 [[TMP1]]) +; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[X]], 2 +; CHECK-NEXT: [[TMP8:%.*]] = call i32 @foo(i1 [[TMP7]]) +; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i32 [[TMP8]], 0 +; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[TMP6]] +; CHECK-NEXT: ret i32 [[TMP10]] ; %1 = icmp eq i32 %x, 0 %2 = call i32 @foo(i1 %1) diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll index 3b449291d0e7..a4f4fc67f78d 100644 --- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll @@ -1,9 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 ; RUN: opt < %s -S | FileCheck %s -; Test that we don't regress diff quality by trying to keep variable names -; stable (and messing up the matching). - define i32 @func(i32 %x) { ; CHECK-LABEL: define i32 @func( ; CHECK-SAME: i32 [[X:%.*]]) { diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll.expected index 3d0f772505a6..08d3c22172ee 100644 --- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll.expected +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values3.ll.expected @@ -1,9 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 ; RUN: opt < %s -S | FileCheck %s -; Test that we don't regress diff quality by trying to keep variable names -; stable (and messing up the matching). - define i32 @func(i32 %x) { ; CHECK-LABEL: define i32 @func( ; CHECK-SAME: i32 [[X:%.*]]) { diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values4.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values4.ll.expected index 5962bdafd9ea..e3fa51598c48 100644 --- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values4.ll.expected +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/stable_ir_values4.ll.expected @@ -15,12 +15,12 @@ define i32 @func(i32 %x) { ; CHECK-NEXT: [[TMP6:%.*]] = call i32 @foo(i32 [[TMP5]]) ; CHECK-NEXT: [[TMP7:%.*]] = call i32 @foo(i32 [[TMP6]]) ; CHECK-NEXT: [[TMP8:%.*]] = xor i32 [[TMP7]], 1 -; CHECK-NEXT: [[TMP9:%.*]] = call i32 @foo(i32 [[TMP8]]) -; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[TMP9]], 1 +; CHECK-NEXT: [[TMP13:%.*]] = call i32 @foo(i32 [[TMP8]]) +; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[TMP13]], 1 +; CHECK-NEXT: [[TMP10:%.*]] = call i32 @foo(i32 [[TMP9]]) ; CHECK-NEXT: [[TMP11:%.*]] = call i32 @foo(i32 [[TMP10]]) ; CHECK-NEXT: [[TMP12:%.*]] = call i32 @foo(i32 [[TMP11]]) -; CHECK-NEXT: [[TMP13:%.*]] = call i32 @foo(i32 [[TMP12]]) -; CHECK-NEXT: ret i32 [[TMP13]] +; CHECK-NEXT: ret i32 [[TMP12]] ; %1 = mul i32 %x, 3 %2 = call i32 @foo(i32 %1) diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values.test b/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values.test index c6287a6b29ca..4dfaf5d25c8a 100644 --- a/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values.test +++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/stable_ir_values.test @@ -1,2 +1,5 @@ # RUN: cp -f %S/Inputs/stable_ir_values.ll %t.ll && %update_test_checks %t.ll # RUN: diff -u %t.ll %S/Inputs/stable_ir_values.ll.expected +# Now test that we can reset all the names +# RUN: %update_test_checks %t.ll --reset-variable-names +# RUN: diff -u %t.ll %S/Inputs/stable_ir_values.ll.expected.reset diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py index a3365fef5f6e..ecb19d233a8d 100644 --- a/llvm/utils/UpdateTestChecks/common.py +++ b/llvm/utils/UpdateTestChecks/common.py @@ -1,6 +1,8 @@ from __future__ import print_function import argparse +import bisect +import collections import copy import glob import itertools @@ -10,7 +12,7 @@ import subprocess import sys import shlex -from typing import List +from typing import List, Mapping, Set ##### Common utilities for update_*test_checks.py @@ -420,6 +422,48 @@ def should_add_line_to_output( return True +def collect_original_check_lines(ti: TestInfo, prefix_set: set): + """ + Collect pre-existing check lines into a dictionary `result` which is + returned. + + result[func_name][prefix] is filled with a list of right-hand-sides of check + lines. + """ + result = {} + + current_function = None + for input_line_info in ti.ro_iterlines(): + input_line = input_line_info.line + if current_function is not None: + if input_line == "": + continue + if input_line.lstrip().startswith(";"): + m = CHECK_RE.match(input_line) + if ( + m is not None + and m.group(1) in prefix_set + and m.group(2) not in ["LABEL", "SAME"] + ): + if m.group(1) not in current_function: + current_function[m.group(1)] = [] + current_function[m.group(1)].append(input_line[m.end() :].strip()) + continue + current_function = None + + m = IR_FUNCTION_RE.match(input_line) + if m is not None: + func_name = m.group(1) + if ti.args.function is not None and func_name != ti.args.function: + # When filtering on a specific function, skip all others. + continue + + assert func_name not in result + current_function = result[func_name] = {} + + return result + + # Perform lit-like substitutions def getSubstitutions(sourcepath): sourcedir = os.path.dirname(sourcepath) @@ -491,7 +535,7 @@ RUN_LINE_RE = re.compile(r"^\s*(?://|[;#])\s*RUN:\s*(.*)$") CHECK_PREFIX_RE = re.compile(r"--?check-prefix(?:es)?[= ](\S+)") PREFIX_RE = re.compile("^[a-zA-Z0-9_-]+$") CHECK_RE = re.compile( - r"^\s*(?://|[;#])\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL|-SAME|-EMPTY)?:" + r"^\s*(?://|[;#])\s*([^:]+?)(?:-(NEXT|NOT|DAG|LABEL|SAME|EMPTY))?:" ) CHECK_SAME_RE = re.compile(r"^\s*(?://|[;#])\s*([^:]+?)(?:-SAME)?:") @@ -1187,6 +1231,313 @@ def may_clash_with_default_check_prefix_name(check_prefix, var): ) +def find_diff_matching(lhs: List[str], rhs: List[str]) -> List[tuple]: + """ + Find a large ordered matching between strings in lhs and rhs. + + Think of this as finding the *unchanged* lines in a diff, where the entries + of lhs and rhs are lines of the files being diffed. + + Returns a list of matched (lhs_idx, rhs_idx) pairs. + """ + + if not lhs or not rhs: + return [] + + # Collect matches in reverse order. + matches = [] + + # First, collect a set of candidate matching edges. We limit this to a + # constant multiple of the input size to avoid quadratic runtime. + patterns = collections.defaultdict(lambda: ([], [])) + + for idx in range(len(lhs)): + patterns[lhs[idx]][0].append(idx) + for idx in range(len(rhs)): + patterns[rhs[idx]][1].append(idx) + + multiple_patterns = [] + + candidates = [] + for pattern in patterns.values(): + if not pattern[0] or not pattern[1]: + continue + + if len(pattern[0]) == len(pattern[1]) == 1: + candidates.append((pattern[0][0], pattern[1][0])) + else: + multiple_patterns.append(pattern) + + multiple_patterns.sort(key=lambda pattern: len(pattern[0]) * len(pattern[1])) + + for pattern in multiple_patterns: + if len(candidates) + len(pattern[0]) * len(pattern[1]) > 2 * ( + len(lhs) + len(rhs) + ): + break + for lhs_idx in pattern[0]: + for rhs_idx in pattern[1]: + candidates.append((lhs_idx, rhs_idx)) + + if not candidates: + # The LHS and RHS either share nothing in common, or lines are just too + # identical. In that case, let's give up and not match anything. + return [] + + # Compute a maximal crossing-free matching via an algorithm that is + # inspired by a mixture of dynamic programming and line-sweeping in + # discrete geometry. + # + # I would be surprised if this algorithm didn't exist somewhere in the + # literature, but I found it without consciously recalling any + # references, so you'll have to make do with the explanation below. + # Sorry. + # + # The underlying graph is bipartite: + # - nodes on the LHS represent lines in the original check + # - nodes on the RHS represent lines in the new (updated) check + # + # Nodes are implicitly sorted by the corresponding line number. + # Edges (unique_matches) are sorted by the line number on the LHS. + # + # Here's the geometric intuition for the algorithm. + # + # * Plot the edges as points in the plane, with the original line + # number on the X axis and the updated line number on the Y axis. + # * The goal is to find a longest "chain" of points where each point + # is strictly above and to the right of the previous point. + # * The algorithm proceeds by sweeping a vertical line from left to + # right. + # * The algorithm maintains a table where `table[N]` answers the + # question "What is currently the 'best' way to build a chain of N+1 + # points to the left of the vertical line". Here, 'best' means + # that the last point of the chain is a as low as possible (minimal + # Y coordinate). + # * `table[N]` is `(y, point_idx)` where `point_idx` is the index of + # the last point in the chain and `y` is its Y coordinate + # * A key invariant is that the Y values in the table are + # monotonically increasing + # * Thanks to these properties, the table can be used to answer the + # question "What is the longest chain that can be built to the left + # of the vertical line using only points below a certain Y value", + # using a binary search over the table. + # * The algorithm also builds a backlink structure in which every point + # links back to the previous point on a best (longest) chain ending + # at that point + # + # The core loop of the algorithm sweeps the line and updates the table + # and backlink structure for every point that we cross during the sweep. + # Therefore, the algorithm is trivially O(M log M) in the number of + # points. + candidates.sort(key=lambda candidate: (candidate[0], -candidate[1])) + + backlinks = [] + table_rhs_idx = [] + table_candidate_idx = [] + for _, rhs_idx in candidates: + candidate_idx = len(backlinks) + ti = bisect.bisect_left(table_rhs_idx, rhs_idx) + + # Update the table to record a best chain ending in the current point. + # There always is one, and if any of the previously visited points had + # a higher Y coordinate, then there is always a previously recorded best + # chain that can be improved upon by using the current point. + # + # There is only one case where there is some ambiguity. If the + # pre-existing entry table[ti] has the same Y coordinate / rhs_idx as + # the current point (this can only happen if the same line appeared + # multiple times on the LHS), then we could choose to keep the + # previously recorded best chain instead. That would bias the algorithm + # differently but should have no systematic impact on the quality of the + # result. + if ti < len(table_rhs_idx): + table_rhs_idx[ti] = rhs_idx + table_candidate_idx[ti] = candidate_idx + else: + table_rhs_idx.append(rhs_idx) + table_candidate_idx.append(candidate_idx) + if ti > 0: + backlinks.append(table_candidate_idx[ti - 1]) + else: + backlinks.append(None) + + # Commit to names in the matching by walking the backlinks. Recursively + # attempt to fill in more matches in-betweem. + match_idx = table_candidate_idx[-1] + while match_idx is not None: + current = candidates[match_idx] + matches.append(current) + match_idx = backlinks[match_idx] + + matches.reverse() + return matches + + +VARIABLE_TAG = "[[@@]]" +METAVAR_RE = re.compile(r"\[\[([A-Z0-9_]+)(?::[^]]+)?\]\]") +NUMERIC_SUFFIX_RE = re.compile(r"[0-9]*$") + + +class CheckValueInfo: + def __init__( + self, + nameless_value: NamelessValue, + var: str, + prefix: str, + ): + self.nameless_value = nameless_value + self.var = var + self.prefix = prefix + + +# Represent a check line in a way that allows us to compare check lines while +# ignoring some or all of the FileCheck variable names. +class CheckLineInfo: + def __init__(self, line, values): + # Line with all FileCheck variable name occurrences replaced by VARIABLE_TAG + self.line: str = line + + # Information on each FileCheck variable name occurrences in the line + self.values: List[CheckValueInfo] = values + + def __repr__(self): + return f"CheckLineInfo(line={self.line}, self.values={self.values})" + + +def remap_metavar_names( + old_line_infos: List[CheckLineInfo], + new_line_infos: List[CheckLineInfo], + committed_names: Set[str], +) -> Mapping[str, str]: + """ + Map all FileCheck variable names that appear in new_line_infos to new + FileCheck variable names in an attempt to reduce the diff from old_line_infos + to new_line_infos. + + This is done by: + * Matching old check lines and new check lines using a diffing algorithm + applied after replacing names with wildcards. + * Committing to variable names such that the matched lines become equal + (without wildcards) if possible + * This is done recursively to handle cases where many lines are equal + after wildcard replacement + """ + # Initialize uncommitted identity mappings + new_mapping = {} + for line in new_line_infos: + for value in line.values: + new_mapping[value.var] = value.var + + # Recursively commit to the identity mapping or find a better one + def recurse(old_begin, old_end, new_begin, new_end): + if old_begin == old_end or new_begin == new_end: + return + + # Find a matching of lines where uncommitted names are replaced + # with a placeholder. + def diffify_line(line, mapper): + values = [] + for value in line.values: + mapped = mapper(value.var) + values.append(mapped if mapped in committed_names else "?") + return line.line.strip() + " @@@ " + " @ ".join(values) + + lhs_lines = [ + diffify_line(line, lambda x: x) + for line in old_line_infos[old_begin:old_end] + ] + rhs_lines = [ + diffify_line(line, lambda x: new_mapping[x]) + for line in new_line_infos[new_begin:new_end] + ] + + candidate_matches = find_diff_matching(lhs_lines, rhs_lines) + + # Apply commits greedily on a match-by-match basis + matches = [(-1, -1)] + committed_anything = False + for lhs_idx, rhs_idx in candidate_matches: + lhs_line = old_line_infos[lhs_idx] + rhs_line = new_line_infos[rhs_idx] + + local_commits = {} + + for lhs_value, rhs_value in zip(lhs_line.values, rhs_line.values): + if new_mapping[rhs_value.var] in committed_names: + # The new value has already been committed. If it was mapped + # to the same name as the original value, we can consider + # committing other values from this line. Otherwise, we + # should ignore this line. + if new_mapping[rhs_value.var] == lhs_value.var: + continue + else: + break + + if rhs_value.var in local_commits: + # Same, but for a possible commit happening on the same line + if local_commits[rhs_value.var] == lhs_value.var: + continue + else: + break + + if lhs_value.var in committed_names: + # We can't map this value because the name we would map it to has already been + # committed for something else. Give up on this line. + break + + local_commits[rhs_value.var] = lhs_value.var + else: + # No reason not to add any commitments for this line + for rhs_var, lhs_var in local_commits.items(): + new_mapping[rhs_var] = lhs_var + committed_names.add(lhs_var) + committed_anything = True + + if ( + lhs_var != rhs_var + and lhs_var in new_mapping + and new_mapping[lhs_var] == lhs_var + ): + new_mapping[lhs_var] = "conflict_" + lhs_var + + matches.append((lhs_idx, rhs_idx)) + + matches.append((old_end, new_end)) + + # Recursively handle sequences between matches + if committed_anything: + for (lhs_prev, rhs_prev), (lhs_next, rhs_next) in zip(matches, matches[1:]): + recurse(lhs_prev + 1, lhs_next, rhs_prev + 1, rhs_next) + + recurse(0, len(old_line_infos), 0, len(new_line_infos)) + + # Commit to remaining names and resolve conflicts + for new_name, mapped_name in new_mapping.items(): + if mapped_name in committed_names: + continue + if not mapped_name.startswith("conflict_"): + assert mapped_name == new_name + committed_names.add(mapped_name) + + for new_name, mapped_name in new_mapping.items(): + if mapped_name in committed_names: + continue + assert mapped_name.startswith("conflict_") + + m = NUMERIC_SUFFIX_RE.search(new_name) + base_name = new_name[: m.start()] + suffix = int(new_name[m.start() :]) if m.start() != m.end() else 1 + while True: + candidate = f"{base_name}{suffix}" + if candidate not in committed_names: + new_mapping[new_name] = candidate + committed_names.add(candidate) + break + suffix += 1 + + return new_mapping + + def generalize_check_lines_common( lines, is_analyze, @@ -1196,11 +1547,12 @@ def generalize_check_lines_common( nameless_value_regex, is_asm, preserve_names, + original_check_lines=None, ): # This gets called for each match that occurs in # a line. We transform variables we haven't seen # into defs, and variables we have seen into uses. - def transform_line_vars(match): + def transform_line_vars(match, transform_locals=True): var = get_name_from_ir_value_match(match) nameless_value = get_nameless_value_from_match(match, nameless_values) if may_clash_with_default_check_prefix_name(nameless_value.check_prefix, var): @@ -1210,6 +1562,8 @@ def generalize_check_lines_common( ) key = (var, nameless_value.check_key) is_local_def = nameless_value.is_local_def_ir_value() + if is_local_def and not transform_locals: + return None if is_local_def and key in vars_seen: rv = nameless_value.get_value_use(var, match) elif not is_local_def and key in global_vars_seen: @@ -1228,13 +1582,15 @@ def generalize_check_lines_common( # including the commas and spaces. return match.group(1) + rv + match.group(match.lastindex) - lines_with_def = [] + def transform_non_local_line_vars(match): + return transform_line_vars(match, False) + multiple_braces_re = re.compile(r"({{+)|(}}+)") def escape_braces(match_obj): return '{{' + re.escape(match_obj.group(0)) + '}}' - for i, line in enumerate(lines): - if not is_asm and not is_analyze: + if not is_asm and not is_analyze: + for i, line in enumerate(lines): # An IR variable named '%.' matches the FileCheck regex string. line = line.replace("%.", "%dot") for regex in _global_hex_value_regex: @@ -1252,25 +1608,136 @@ def generalize_check_lines_common( # Ignore any comments, since the check lines will too. scrubbed_line = SCRUB_IR_COMMENT_RE.sub(r"", line) lines[i] = scrubbed_line - if not preserve_names: - # It can happen that two matches are back-to-back and for some reason sub - # will not replace both of them. For now we work around this by - # substituting until there is no more match. - changed = True - while changed: - (lines[i], changed) = nameless_value_regex.subn( - transform_line_vars, lines[i], count=1 - ) - if is_analyze: + + if not preserve_names: + if is_asm: + for i, _ in enumerate(lines): + # It can happen that two matches are back-to-back and for some reason sub + # will not replace both of them. For now we work around this by + # substituting until there is no more match. + changed = True + while changed: + (lines[i], changed) = nameless_value_regex.subn( + transform_line_vars, lines[i], count=1 + ) + else: + # LLVM IR case. Start by handling global meta variables (global IR variables, + # metadata, attributes) + for i, _ in enumerate(lines): + start = 0 + while True: + m = nameless_value_regex.search(lines[i][start:]) + if m is None: + break + start += m.start() + sub = transform_non_local_line_vars(m) + if sub is not None: + lines[i] = ( + lines[i][:start] + sub + lines[i][start + len(m.group(0)) :] + ) + start += 1 + + # Collect information about new check lines and original check lines (if any) + new_line_infos = [] + for line in lines: + filtered_line = "" + values = [] + while True: + m = nameless_value_regex.search(line) + if m is None: + filtered_line += line + break + + var = get_name_from_ir_value_match(m) + nameless_value = get_nameless_value_from_match(m, nameless_values) + var = nameless_value.get_value_name( + var, nameless_value.check_prefix + ) + + # Replace with a [[@@]] tag, but be sure to keep the spaces and commas. + filtered_line += ( + line[: m.start()] + + m.group(1) + + VARIABLE_TAG + + m.group(m.lastindex) + ) + line = line[m.end() :] + values.append( + CheckValueInfo( + nameless_value=nameless_value, + var=var, + prefix=nameless_value.get_ir_prefix_from_ir_value_match(m)[ + 0 + ], + ) + ) + new_line_infos.append(CheckLineInfo(filtered_line, values)) + + orig_line_infos = [] + for line in original_check_lines or []: + filtered_line = "" + values = [] + while True: + m = METAVAR_RE.search(line) + if m is None: + filtered_line += line + break + + # Replace with a [[@@]] tag, but be sure to keep the spaces and commas. + filtered_line += line[: m.start()] + VARIABLE_TAG + line = line[m.end() :] + values.append( + CheckValueInfo( + nameless_value=None, + var=m.group(1), + prefix=None, + ) + ) + orig_line_infos.append(CheckLineInfo(filtered_line, values)) + + # Compute the variable name mapping + committed_names = set(vars_seen) + + mapping = remap_metavar_names( + orig_line_infos, new_line_infos, committed_names + ) + + for i, line_info in enumerate(new_line_infos): + line_template = line_info.line + line = "" + + for value in line_info.values: + idx = line_template.find(VARIABLE_TAG) + line += line_template[:idx] + line_template = line_template[idx + len(VARIABLE_TAG) :] + + key = (mapping[value.var], nameless_value.check_key) + is_local_def = nameless_value.is_local_def_ir_value() + if is_local_def: + if mapping[value.var] in vars_seen: + line += f"[[{mapping[value.var]}]]" + else: + line += f"[[{mapping[value.var]}:{value.prefix}{value.nameless_value.get_ir_regex()}]]" + vars_seen.add(mapping[value.var]) + else: + raise RuntimeError("not implemented") + + line += line_template + + lines[i] = line + + if is_analyze: + for i, _ in enumerate(lines): # Escape multiple {{ or }} as {{}} denotes a FileCheck regex. scrubbed_line = multiple_braces_re.sub(escape_braces, lines[i]) lines[i] = scrubbed_line + return lines # Replace IR value defs and uses with FileCheck variables. def generalize_check_lines( - lines, is_analyze, vars_seen, global_vars_seen, preserve_names + lines, is_analyze, vars_seen, global_vars_seen, preserve_names, original_check_lines ): return generalize_check_lines_common( lines, @@ -1281,6 +1748,7 @@ def generalize_check_lines( IR_VALUE_RE, False, preserve_names, + original_check_lines=original_check_lines, ) @@ -1337,6 +1805,7 @@ def add_checks( global_vars_seen_dict, is_filtered, preserve_names=False, + original_check_lines: Mapping[str, List[str]] = {}, ): # prefix_exclusions are prefixes we cannot use to print the function because it doesn't exist in run lines that use these prefixes as well. prefix_exclusions = set() @@ -1409,6 +1878,7 @@ def add_checks( vars_seen, global_vars_seen, preserve_names, + original_check_lines=[], )[0] func_name_separator = func_dict[checkprefix][func_name].func_name_separator if "[[" in args_and_sig: @@ -1516,7 +1986,12 @@ def add_checks( # to variable naming fashions. else: func_body = generalize_check_lines( - func_body, False, vars_seen, global_vars_seen, preserve_names + func_body, + False, + vars_seen, + global_vars_seen, + preserve_names, + original_check_lines=original_check_lines.get(checkprefix), ) # This could be selectively enabled with an optional invocation argument. @@ -1578,6 +2053,7 @@ def add_ir_checks( version, global_vars_seen_dict, is_filtered, + original_check_lines={}, ): # Label format is based on IR string. if function_sig and version > 1: @@ -1602,6 +2078,7 @@ def add_ir_checks( global_vars_seen_dict, is_filtered, preserve_names, + original_check_lines=original_check_lines, ) @@ -1890,6 +2367,7 @@ def get_autogennote_suffix(parser, args): "llvm_bin", "verbose", "force_update", + "reset_variable_names", ): continue value = getattr(args, action.dest) diff --git a/llvm/utils/update_test_checks.py b/llvm/utils/update_test_checks.py index b5077d793513..04808ce6bb1c 100755 --- a/llvm/utils/update_test_checks.py +++ b/llvm/utils/update_test_checks.py @@ -85,6 +85,12 @@ def main(): choices=["none", "smart", "all"], help="Check global entries (global variables, metadata, attribute sets, ...) for functions", ) + parser.add_argument( + "--reset-variable-names", + action="store_true", + help="Reset all variable names to correspond closely to the variable names in IR. " + "This tends to result in larger diffs.", + ) parser.add_argument("tests", nargs="+") initial_args = common.parse_commandline_args(parser) @@ -170,13 +176,19 @@ def main(): ) builder.processed_prefixes(prefixes) + prefix_set = set( + [prefix for prefixes, _, _ in prefix_list for prefix in prefixes] + ) + + if not ti.args.reset_variable_names: + original_check_lines = common.collect_original_check_lines(ti, prefix_set) + else: + original_check_lines = {} + func_dict = builder.finish_and_get_func_dict() is_in_function = False is_in_function_start = False has_checked_pre_function_globals = False - prefix_set = set( - [prefix for prefixes, _, _ in prefix_list for prefix in prefixes] - ) common.debug("Rewriting FileCheck prefixes:", str(prefix_set)) output_lines = [] @@ -230,6 +242,7 @@ def main(): args.version, global_vars_seen_dict, is_filtered=builder.is_filtered(), + original_check_lines=original_check_lines.get(func, {}), ), ) ) @@ -261,6 +274,9 @@ def main(): args.version, global_vars_seen_dict, is_filtered=builder.is_filtered(), + original_check_lines=original_check_lines.get( + func_name, {} + ), ) ) is_in_function_start = False -- GitLab From d9c855014123a313006adfc873be6f10e997be61 Mon Sep 17 00:00:00 2001 From: Slava Zakharin Date: Thu, 7 Mar 2024 19:52:28 -0800 Subject: [PATCH 560/929] [flang] Fixed build issues after f20ea05. (#84377) Older versions of clang do not have __builtin_complex, but they may define `__GNUC__`. --- flang/runtime/complex-reduction.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flang/runtime/complex-reduction.c b/flang/runtime/complex-reduction.c index 72c31ce08b87..c91d12539911 100644 --- a/flang/runtime/complex-reduction.c +++ b/flang/runtime/complex-reduction.c @@ -82,7 +82,8 @@ static long_double_Complex_t CMPLXL(long double r, long double i) { * supports __builtin_complex. For Clang, require >=12.0. * Otherwise, rely on the memory layout compatibility. */ -#if (defined(__clang_major__) && (__clang_major__ >= 12)) || defined(__GNUC__) +#if (defined(__clang_major__) && (__clang_major__ >= 12)) || \ + (defined(__GNUC__) && !defined(__clang__)) #define CMPLXF128 __builtin_complex #else static CFloat128ComplexType CMPLXF128(CFloat128Type r, CFloat128Type i) { -- GitLab From 23c658ac4183272221ef358575dca0d386096d36 Mon Sep 17 00:00:00 2001 From: David Green Date: Fri, 8 Mar 2024 07:09:33 +0000 Subject: [PATCH 561/929] [AArch64] Ensure Neoverse V1 scheduling model includes all SVE pseudos. (#84187) With the many pseudos used in SVE codegen it can be too easy to miss instructions. This enables the existing test we have for checking the scheduling info of the pseudos matches the real instructions, and adjusts the scheduling info in the NeoverseV1 model to make sure all are handled. In the cases I could I opted to use the same info as in the NeoverseV2 model, to keep the differences smaller. --- .../Target/AArch64/AArch64SchedNeoverseV1.td | 105 ++++++++++-------- .../Target/AArch64/AArch64SchedNeoverseV2.td | 6 +- .../AArch64/AArch64SVESchedPseudoTest.cpp | 4 + 3 files changed, 67 insertions(+), 48 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td index e50a401f8b2a..c7dfd64b2fb2 100644 --- a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td +++ b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td @@ -1372,18 +1372,18 @@ def : InstRW<[V1Write_3c_2M0], (instregex "^PTRUES_[BHSD]$")>; // Arithmetic, basic // Logical def : InstRW<[V1Write_2c_1V01], - (instregex "^(ABS|CNOT|NEG)_ZPmZ_[BHSD]$", - "^(ADD|SUB)_Z(I|P[mZ]Z|ZZ)_[BHSD]$", + (instregex "^(ABS|CNOT|NEG)_ZPmZ_[BHSD]", + "^(ADD|SUB)_Z(I|P[mZ]Z|ZZ)_[BHSD]", "^ADR_[SU]XTW_ZZZ_D_[0123]$", "^ADR_LSL_ZZZ_[SD]_[0123]$", - "^[SU]ABD_ZP[mZ]Z_[BHSD]$", - "^[SU](MAX|MIN)_Z(I|P[mZ]Z)_[BHSD]$", + "^[SU]ABD_ZP[mZ]Z_[BHSD]", + "^[SU](MAX|MIN)_Z(I|P[mZ]Z)_[BHSD]", "^[SU]Q(ADD|SUB)_Z(I|ZZ)_[BHSD]$", - "^SUBR_Z(I|P[mZ]Z)_[BHSD]$", + "^SUBR_Z(I|P[mZ]Z)_[BHSD]", "^(AND|EOR|ORR)_ZI$", - "^(AND|BIC|EOR|EOR(BT|TB)?|ORR)_ZZZ$", + "^(AND|BIC|EOR|EOR(BT|TB)?|ORR)_ZP?ZZ", "^EOR(BT|TB)_ZZZ_[BHSD]$", - "^(AND|BIC|EOR|NOT|ORR)_ZPmZ_[BHSD]$")>; + "^(AND|BIC|EOR|NOT|ORR)_ZPmZ_[BHSD]")>; // Arithmetic, shift def : InstRW<[V1Write_2c_1V1], @@ -1394,10 +1394,10 @@ def : InstRW<[V1Write_2c_1V1], "^(ASRR|LSLR|LSRR)_ZPmZ_[BHSD]")>; // Arithmetic, shift right for divide -def : InstRW<[V1Write_4c_1V1], (instregex "^ASRD_ZP[mZ]I_[BHSD]$")>; +def : InstRW<[V1Write_4c_1V1], (instregex "^ASRD_(ZPmI|ZPZI)_[BHSD]")>; // Count/reverse bits -def : InstRW<[V1Write_2c_1V01], (instregex "^(CLS|CLZ|CNT|RBIT)_ZPmZ_[BHSD]$")>; +def : InstRW<[V1Write_2c_1V01], (instregex "^(CLS|CLZ|CNT|RBIT)_ZPmZ_[BHSD]")>; // Broadcast logical bitmask immediate to vector def : InstRW<[V1Write_2c_1V01], (instrs DUPM_ZI)>; @@ -1420,10 +1420,10 @@ def : InstRW<[V1Write_3c_1V0], (instregex "^[SU]CVTF_ZPmZ_Dto[HSD]", "^[SU]CVTF_ZPmZ_StoD")>; // Convert to floating point, 32b to single or half -def : InstRW<[V1Write_4c_2V0], (instregex "^[SU]CVTF_ZPmZ_Sto[HS]$")>; +def : InstRW<[V1Write_4c_2V0], (instregex "^[SU]CVTF_ZPmZ_Sto[HS]")>; // Convert to floating point, 16b to half -def : InstRW<[V1Write_6c_4V0], (instregex "^[SU]CVTF_ZPmZ_HtoH$")>; +def : InstRW<[V1Write_6c_4V0], (instregex "^[SU]CVTF_ZPmZ_HtoH")>; // Copy, scalar def : InstRW<[V1Write_5c_1M0_1V01], (instregex "^CPY_ZPmR_[BHSD]$")>; @@ -1432,10 +1432,12 @@ def : InstRW<[V1Write_5c_1M0_1V01], (instregex "^CPY_ZPmR_[BHSD]$")>; def : InstRW<[V1Write_2c_1V01], (instregex "^CPY_ZP([mz]I|mV)_[BHSD]$")>; // Divides, 32 bit -def : InstRW<[V1Write_12c7_1V0], (instregex "^[SU]DIVR?_ZPmZ_S$")>; +def : InstRW<[V1Write_12c7_1V0], (instregex "^[SU]DIVR?_ZPmZ_S", + "^[SU]DIV_ZPZZ_S")>; // Divides, 64 bit -def : InstRW<[V1Write_20c7_1V0], (instregex "^[SU]DIVR?_ZPmZ_D$")>; +def : InstRW<[V1Write_20c7_1V0], (instregex "^[SU]DIVR?_ZPmZ_D", + "^[SU]DIV_ZPZZ_D")>; // Dot product, 8 bit def : InstRW<[V1Write_3c_1V01], (instregex "^[SU]DOT_ZZZI?_S$")>; @@ -1454,9 +1456,9 @@ def : InstRW<[V1Write_2c_1V01], (instregex "^DUP_ZI_[BHSD]$", def : InstRW<[V1Write_3c_1M0], (instregex "^DUP_ZR_[BHSD]$")>; // Extend, sign or zero -def : InstRW<[V1Write_2c_1V1], (instregex "^[SU]XTB_ZPmZ_[HSD]$", - "^[SU]XTH_ZPmZ_[SD]$", - "^[SU]XTW_ZPmZ_[D]$")>; +def : InstRW<[V1Write_2c_1V1], (instregex "^[SU]XTB_ZPmZ_[HSD]", + "^[SU]XTH_ZPmZ_[SD]", + "^[SU]XTW_ZPmZ_[D]")>; // Extract def : InstRW<[V1Write_2c_1V01], (instrs EXT_ZZI)>; @@ -1489,18 +1491,22 @@ def : InstRW<[V1Write_2c_1V01], (instregex "^MOVPRFX_ZP[mz]Z_[BHSD]$", def : InstRW<[V1Write_3c_1V01], (instrs SMMLA_ZZZ, UMMLA_ZZZ, USMMLA_ZZZ)>; // Multiply, B, H, S element size -def : InstRW<[V1Write_4c_1V0], (instregex "^MUL_(ZI|ZPmZ)_[BHS]$", - "^[SU]MULH_(ZPmZ|ZZZ)_[BHS]$")>; +def : InstRW<[V1Write_4c_1V0], (instregex "^MUL_(ZI|ZPmZ|ZZZI|ZZZ)_[BHS]", + "^MUL_ZPZZ_[BHS]", + "^[SU]MULH_(ZPmZ|ZZZ)_[BHS]", + "^[SU]MULH_ZPZZ_[BHS]")>; // Multiply, D element size // Multiply accumulate, D element size -def : InstRW<[V1Write_5c_2V0], (instregex "^MUL_(ZI|ZPmZ)_D$", - "^[SU]MULH_ZPmZ_D$", - "^(MLA|MLS|MAD|MSB)_ZPmZZ_D$")>; +def : InstRW<[V1Write_5c_2V0], (instregex "^MUL_(ZI|ZPmZ|ZZZI|ZZZ)_D", + "^MUL_ZPZZ_D", + "^[SU]MULH_(ZPmZ|ZZZ)_D", + "^[SU]MULH_ZPZZ_D", + "^(MLA|MLS|MAD|MSB)_(ZPmZZ|ZPZZZ)_D")>; // Multiply accumulate, B, H, S element size // NOTE: This is not specified in the SOG. -def : InstRW<[V1Write_4c_1V0], (instregex "^(ML[AS]|MAD|MSB)_ZPmZZ_[BHS]")>; +def : InstRW<[V1Write_4c_1V0], (instregex "^(ML[AS]|MAD|MSB)_(ZPmZZ|ZPZZZ)_[BHS]")>; // Predicate counting vector def : InstRW<[V1Write_2c_1V0], (instregex "^([SU]Q)?(DEC|INC)[HWD]_ZPiI$")>; @@ -1547,12 +1553,17 @@ def : InstRW<[V1Write_2c_1V01], (instregex "^SEL_ZPZZ_[BHSD]$", // ----------------------------------------------------------------------------- // Floating point absolute value/difference +def : InstRW<[V1Write_2c_1V01], (instregex "^FAB[SD]_ZPmZ_[HSD]", + "^FABD_ZPZZ_[HSD]", + "^FABS_ZPmZ_[HSD]")>; + // Floating point arithmetic -def : InstRW<[V1Write_2c_1V01], (instregex "^FAB[SD]_ZPmZ_[HSD]$", - "^F(ADD|SUB)_(ZPm[IZ]|ZZZ)_[HSD]$", - "^FADDP_ZPmZZ_[HSD]$", - "^FNEG_ZPmZ_[HSD]$", - "^FSUBR_ZPm[IZ]_[HSD]$")>; +def : InstRW<[V1Write_2c_1V01], (instregex "^F(ADD|SUB)_(ZPm[IZ]|ZZZ)_[HSD]", + "^F(ADD|SUB)_ZPZ[IZ]_[HSD]", + "^FADDP_ZPmZZ_[HSD]", + "^FNEG_ZPmZ_[HSD]", + "^FSUBR_ZPm[IZ]_[HSD]", + "^FSUBR_(ZPZI|ZPZZ)_[HSD]")>; // Floating point associative add, F16 def : InstRW<[V1Write_19c_18V0], (instrs FADDA_VPZ_H)>; @@ -1577,40 +1588,44 @@ def : InstRW<[V1Write_5c_1V01], (instregex "^FCMLA_ZPmZZ_[HSD]$", // Floating point convert, long or narrow (F16 to F32 or F32 to F16) // Floating point convert to integer, F32 -def : InstRW<[V1Write_4c_2V0], (instregex "^FCVT_ZPmZ_(HtoS|StoH)$", - "^FCVTZ[SU]_ZPmZ_(HtoS|StoS)$")>; +def : InstRW<[V1Write_4c_2V0], (instregex "^FCVT_ZPmZ_(HtoS|StoH)", + "^FCVTZ[SU]_ZPmZ_(HtoS|StoS)")>; // Floating point convert, long or narrow (F16 to F64, F32 to F64, F64 to F32 or F64 to F16) // Floating point convert to integer, F64 -def : InstRW<[V1Write_3c_1V0], (instregex "^FCVT_ZPmZ_(HtoD|StoD|DtoS|DtoH)$", - "^FCVTZ[SU]_ZPmZ_(HtoD|StoD|DtoS|DtoD)$")>; +def : InstRW<[V1Write_3c_1V0], (instregex "^FCVT_ZPmZ_(HtoD|StoD|DtoS|DtoH)", + "^FCVTZ[SU]_ZPmZ_(HtoD|StoD|DtoS|DtoD)")>; // Floating point convert to integer, F16 -def : InstRW<[V1Write_6c_4V0], (instregex "^FCVTZ[SU]_ZPmZ_HtoH$")>; +def : InstRW<[V1Write_6c_4V0], (instregex "^FCVTZ[SU]_ZPmZ_HtoH")>; // Floating point copy def : InstRW<[V1Write_2c_1V01], (instregex "^FCPY_ZPmI_[HSD]$", "^FDUP_ZI_[HSD]$")>; // Floating point divide, F16 -def : InstRW<[V1Write_13c10_1V0], (instregex "^FDIVR?_ZPmZ_H$")>; +def : InstRW<[V1Write_13c10_1V0], (instregex "^FDIVR?_(ZPmZ|ZPZZ)_H")>; // Floating point divide, F32 -def : InstRW<[V1Write_10c7_1V0], (instregex "^FDIVR?_ZPmZ_S$")>; +def : InstRW<[V1Write_10c7_1V0], (instregex "^FDIVR?_(ZPmZ|ZPZZ)_S")>; // Floating point divide, F64 -def : InstRW<[V1Write_15c7_1V0], (instregex "^FDIVR?_ZPmZ_D$")>; +def : InstRW<[V1Write_15c7_1V0], (instregex "^FDIVR?_(ZPmZ|ZPZZ)_D")>; // Floating point min/max -def : InstRW<[V1Write_2c_1V01], (instregex "^F(MAX|MIN)(NM)?_ZPm[IZ]_[HSD]$")>; +def : InstRW<[V1Write_2c_1V01], (instregex "^F(MAX|MIN)(NM)?_ZPm[IZ]_[HSD]", + "^F(MAX|MIN)(NM)?_ZPZ[IZ]_[HSD]")>; // Floating point multiply -def : InstRW<[V1Write_3c_1V01], (instregex "^F(SCALE|MULX)_ZPmZ_[HSD]$", - "^FMUL_(ZPm[IZ]|ZZZI?)_[HSD]$")>; +def : InstRW<[V1Write_3c_1V01], (instregex "^(FSCALE|FMULX)_ZPmZ_[HSD]", + "^FMULX_ZPZZ_[HSD]", + "^FMUL_(ZPm[IZ]|ZZZI?)_[HSD]", + "^FMUL_ZPZ[IZ]_[HSD]")>; // Floating point multiply accumulate // Floating point reciprocal step def : InstRW<[V1Write_4c_1V01], (instregex "^F(N?M(AD|SB)|N?ML[AS])_ZPmZZ_[HSD]$", + "^FN?ML[AS]_ZPZZZ_[HSD]", "^FML[AS]_ZZZI_[HSD]$", "^F(RECPS|RSQRTS)_ZZZ_[HSD]$")>; @@ -1624,7 +1639,7 @@ def : InstRW<[V1Write_4c_2V0], (instrs FRECPE_ZZ_S, FRSQRTE_ZZ_S)>; def : InstRW<[V1Write_3c_1V0], (instrs FRECPE_ZZ_D, FRSQRTE_ZZ_D)>; // Floating point reciprocal exponent -def : InstRW<[V1Write_3c_1V0], (instregex "^FRECPX_ZPmZ_[HSD]$")>; +def : InstRW<[V1Write_3c_1V0], (instregex "^FRECPX_ZPmZ_[HSD]")>; // Floating point reduction, F16 def : InstRW<[V1Write_13c_6V01], (instregex "^F(ADD|((MAX|MIN)(NM)?))V_VPZ_H$")>; @@ -1636,22 +1651,22 @@ def : InstRW<[V1Write_11c_1V_5V01], (instregex "^F(ADD|((MAX|MIN)(NM)?))V_VPZ_S$ def : InstRW<[V1Write_9c_1V_4V01], (instregex "^F(ADD|((MAX|MIN)(NM)?))V_VPZ_D$")>; // Floating point round to integral, F16 -def : InstRW<[V1Write_6c_1V0], (instregex "^FRINT[AIMNPXZ]_ZPmZ_H$")>; +def : InstRW<[V1Write_6c_1V0], (instregex "^FRINT[AIMNPXZ]_ZPmZ_H")>; // Floating point round to integral, F32 -def : InstRW<[V1Write_4c_1V0], (instregex "^FRINT[AIMNPXZ]_ZPmZ_S$")>; +def : InstRW<[V1Write_4c_1V0], (instregex "^FRINT[AIMNPXZ]_ZPmZ_S")>; // Floating point round to integral, F64 -def : InstRW<[V1Write_3c_1V0], (instregex "^FRINT[AIMNPXZ]_ZPmZ_D$")>; +def : InstRW<[V1Write_3c_1V0], (instregex "^FRINT[AIMNPXZ]_ZPmZ_D")>; // Floating point square root, F16 -def : InstRW<[V1Write_13c10_1V0], (instrs FSQRT_ZPmZ_H)>; +def : InstRW<[V1Write_13c10_1V0], (instregex "^FSQRT_ZPmZ_H")>; // Floating point square root, F32 -def : InstRW<[V1Write_10c7_1V0], (instrs FSQRT_ZPmZ_S)>; +def : InstRW<[V1Write_10c7_1V0], (instregex "^FSQRT_ZPmZ_S")>; // Floating point square root, F64 -def : InstRW<[V1Write_16c7_1V0], (instrs FSQRT_ZPmZ_D)>; +def : InstRW<[V1Write_16c7_1V0], (instregex "^FSQRT_ZPmZ_D")>; // Floating point trigonometric def : InstRW<[V1Write_3c_1V01], (instregex "^FEXPA_ZZ_[HSD]$", diff --git a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td index 807ce40bc5ea..f10b94523d2e 100644 --- a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td +++ b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td @@ -2567,13 +2567,13 @@ def : InstRW<[V2Write_4cyc_2V02], (instregex "^FRINT[AIMNPXZ]_ZPmZ_S")>; def : InstRW<[V2Write_3cyc_1V02], (instregex "^FRINT[AIMNPXZ]_ZPmZ_D")>; // Floating point square root, F16 -def : InstRW<[V2Write_13cyc_1V0_12rc], (instregex "^FSQRT_ZPmZ_H", "^FSQRT_ZPmZ_H")>; +def : InstRW<[V2Write_13cyc_1V0_12rc], (instregex "^FSQRT_ZPmZ_H")>; // Floating point square root, F32 -def : InstRW<[V2Write_10cyc_1V0_9rc], (instregex "^FSQRT_ZPmZ_S", "^FSQRT_ZPmZ_S")>; +def : InstRW<[V2Write_10cyc_1V0_9rc], (instregex "^FSQRT_ZPmZ_S")>; // Floating point square root, F64 -def : InstRW<[V2Write_16cyc_1V0_14rc], (instregex "^FSQRT_ZPmZ_D", "^FSQRT_ZPmZ_D")>; +def : InstRW<[V2Write_16cyc_1V0_14rc], (instregex "^FSQRT_ZPmZ_D")>; // Floating point trigonometric exponentiation def : InstRW<[V2Write_3cyc_1V1], (instregex "^FEXPA_ZZ_[HSD]")>; diff --git a/llvm/unittests/Target/AArch64/AArch64SVESchedPseudoTest.cpp b/llvm/unittests/Target/AArch64/AArch64SVESchedPseudoTest.cpp index 9d8633353e1f..6098d4e62392 100644 --- a/llvm/unittests/Target/AArch64/AArch64SVESchedPseudoTest.cpp +++ b/llvm/unittests/Target/AArch64/AArch64SVESchedPseudoTest.cpp @@ -107,6 +107,10 @@ TEST(AArch64SVESchedPseudoTesta510, IsCorrect) { runSVEPseudoTestForCPU("cortex-a510"); } +TEST(AArch64SVESchedPseudoTestv1, IsCorrect) { + runSVEPseudoTestForCPU("neoverse-v1"); +} + TEST(AArch64SVESchedPseudoTestv2, IsCorrect) { runSVEPseudoTestForCPU("neoverse-v2"); } -- GitLab From 2d539db246fd9d26201255b84d04dacf2782eddf Mon Sep 17 00:00:00 2001 From: martinboehme Date: Fri, 8 Mar 2024 08:19:02 +0100 Subject: [PATCH 562/929] [clang][dataflow] When analyzing ctors, don't initialize fields of `*this` with values. (#84164) This is the constructor's job, and we want to be able to test that it does this. --- .../FlowSensitive/DataflowEnvironment.h | 5 ++ .../FlowSensitive/DataflowEnvironment.cpp | 21 ++++++- .../TypeErasedDataflowAnalysis.cpp | 2 +- .../Analysis/FlowSensitive/TestingSupport.cpp | 5 +- .../Analysis/FlowSensitive/TransferTest.cpp | 63 +++++++++++++++++++ 5 files changed, 92 insertions(+), 4 deletions(-) diff --git a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h index e8f009ef6c79..2330697299fd 100644 --- a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h +++ b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h @@ -445,6 +445,11 @@ public: return createObjectInternal(&D, D.getType(), InitExpr); } + /// Initializes the fields (including synthetic fields) of `Loc` with values, + /// unless values of the field type are not supported or we hit one of the + /// limits at which we stop producing values. + void initializeFieldsWithValues(RecordStorageLocation &Loc); + /// Assigns `Val` as the value of `Loc` in the environment. void setValue(const StorageLocation &Loc, Value &Val); diff --git a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp index 62332a18c44a..1d2bd9a9b08a 100644 --- a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp +++ b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp @@ -432,8 +432,15 @@ void Environment::initialize() { } } else if (MethodDecl->isImplicitObjectMemberFunction()) { QualType ThisPointeeType = MethodDecl->getFunctionObjectParameterType(); - setThisPointeeStorageLocation( - cast(createObject(ThisPointeeType))); + auto &ThisLoc = + cast(createStorageLocation(ThisPointeeType)); + setThisPointeeStorageLocation(ThisLoc); + refreshRecordValue(ThisLoc, *this); + // Initialize fields of `*this` with values, but only if we're not + // analyzing a constructor; after all, it's the constructor's job to do + // this (and we want to be able to test that). + if (!isa(MethodDecl)) + initializeFieldsWithValues(ThisLoc); } } } @@ -819,6 +826,16 @@ PointerValue &Environment::getOrCreateNullPointerValue(QualType PointeeType) { return DACtx->getOrCreateNullPointerValue(PointeeType); } +void Environment::initializeFieldsWithValues(RecordStorageLocation &Loc) { + llvm::DenseSet Visited; + int CreatedValuesCount = 0; + initializeFieldsWithValues(Loc, Visited, 0, CreatedValuesCount); + if (CreatedValuesCount > MaxCompositeValueSize) { + llvm::errs() << "Attempting to initialize a huge value of type: " + << Loc.getType() << '\n'; + } +} + void Environment::setValue(const StorageLocation &Loc, Value &Val) { assert(!isa(&Val) || &cast(&Val)->getLoc() == &Loc); diff --git a/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp b/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp index a9f39e153d0c..939247c047c6 100644 --- a/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp +++ b/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp @@ -406,7 +406,6 @@ builtinTransferInitializer(const CFGInitializer &Elt, } } assert(Member != nullptr); - assert(MemberLoc != nullptr); // FIXME: Instead of these case distinctions, we would ideally want to be able // to simply use `Environment::createObject()` here, the same way that we do @@ -422,6 +421,7 @@ builtinTransferInitializer(const CFGInitializer &Elt, ParentLoc->setChild(*Member, InitExprLoc); } else if (auto *InitExprVal = Env.getValue(*InitExpr)) { + assert(MemberLoc != nullptr); if (Member->getType()->isRecordType()) { auto *InitValStruct = cast(InitExprVal); // FIXME: Rather than performing a copy here, we should really be diff --git a/clang/unittests/Analysis/FlowSensitive/TestingSupport.cpp b/clang/unittests/Analysis/FlowSensitive/TestingSupport.cpp index 09f5524e152c..5c4d42c6ccdc 100644 --- a/clang/unittests/Analysis/FlowSensitive/TestingSupport.cpp +++ b/clang/unittests/Analysis/FlowSensitive/TestingSupport.cpp @@ -179,7 +179,10 @@ llvm::Error test::checkDataflowWithNoopAnalysis( // -fnodelayed-template-parsing is the default everywhere but on Windows. // Set it explicitly so that tests behave the same on Windows as on other // platforms. - "-fsyntax-only", "-fno-delayed-template-parsing", + // Set -Wno-unused-value because it's often desirable in tests to write + // expressions with unused value, and we don't want the output to be + // cluttered with warnings about them. + "-fsyntax-only", "-fno-delayed-template-parsing", "-Wno-unused-value", "-std=" + std::string(LangStandard::getLangStandardForKind(Std).getName())}; AnalysisInputs AI( diff --git a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp index f534ccb12547..9fde4179db1c 100644 --- a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp +++ b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp @@ -1476,6 +1476,69 @@ TEST(TransferTest, BaseClassInitializer) { llvm::Succeeded()); } +TEST(TransferTest, FieldsDontHaveValuesInConstructor) { + // In a constructor, unlike in regular member functions, we don't want fields + // to be pre-initialized with values, because doing so is the job of the + // constructor. + std::string Code = R"( + struct target { + target() { + 0; + // [[p]] + // Mention the field so it is modeled; + Val; + } + + int Val; + }; + )"; + runDataflow( + Code, + [](const llvm::StringMap> &Results, + ASTContext &ASTCtx) { + const Environment &Env = getEnvironmentAtAnnotation(Results, "p"); + EXPECT_EQ(getFieldValue(Env.getThisPointeeStorageLocation(), "Val", + ASTCtx, Env), + nullptr); + }); +} + +TEST(TransferTest, FieldsDontHaveValuesInConstructorWithBaseClass) { + // See above, but for a class with a base class. + std::string Code = R"( + struct Base { + int BaseVal; + }; + + struct target : public Base { + target() { + 0; + // [[p]] + // Mention the fields so they are modeled. + BaseVal; + Val; + } + + int Val; + }; + )"; + runDataflow( + Code, + [](const llvm::StringMap> &Results, + ASTContext &ASTCtx) { + const Environment &Env = getEnvironmentAtAnnotation(Results, "p"); + // FIXME: The field of the base class should already have been + // initialized with a value by the base constructor. This test documents + // the current buggy behavior. + EXPECT_EQ(getFieldValue(Env.getThisPointeeStorageLocation(), "BaseVal", + ASTCtx, Env), + nullptr); + EXPECT_EQ(getFieldValue(Env.getThisPointeeStorageLocation(), "Val", + ASTCtx, Env), + nullptr); + }); +} + TEST(TransferTest, StructModeledFieldsWithAccessor) { std::string Code = R"( class S { -- GitLab From 9b74c43d70f4b39d6fea7b542d77f2b652e4d651 Mon Sep 17 00:00:00 2001 From: martinboehme Date: Fri, 8 Mar 2024 08:19:41 +0100 Subject: [PATCH 563/929] [clang][dataflow] Add context-sensitive test for returning a record by value. (#84317) I'm making some changes to `Environment::getResultObjectLocation()`, with the ultimate goal of eliminating `RecordValue` entirely, and I'd like to make sure I don't break this behavior (and I've realized we don't have a test for it yet). --- .../Analysis/FlowSensitive/TransferTest.cpp | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp index 9fde4179db1c..a8c282f140b4 100644 --- a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp +++ b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp @@ -5735,6 +5735,39 @@ TEST(TransferTest, ContextSensitiveReturnInt) { {BuiltinOptions{ContextSensitiveOptions{}}}); } +TEST(TransferTest, ContextSensitiveReturnRecord) { + std::string Code = R"( + struct S { + bool B; + }; + + S makeS(bool BVal) { return {BVal}; } + + void target() { + S FalseS = makeS(false); + S TrueS = makeS(true); + // [[p]] + } + )"; + runDataflow( + Code, + [](const llvm::StringMap> &Results, + ASTContext &ASTCtx) { + const Environment &Env = getEnvironmentAtAnnotation(Results, "p"); + + auto &FalseSLoc = + getLocForDecl(ASTCtx, Env, "FalseS"); + auto &TrueSLoc = + getLocForDecl(ASTCtx, Env, "TrueS"); + + EXPECT_EQ(getFieldValue(&FalseSLoc, "B", ASTCtx, Env), + &Env.getBoolLiteralValue(false)); + EXPECT_EQ(getFieldValue(&TrueSLoc, "B", ASTCtx, Env), + &Env.getBoolLiteralValue(true)); + }, + {BuiltinOptions{ContextSensitiveOptions{}}}); +} + TEST(TransferTest, ContextSensitiveMethodLiteral) { std::string Code = R"( class MyClass { -- GitLab From bfa6444a332f82843f9fa44821d68fcc772e0272 Mon Sep 17 00:00:00 2001 From: Alexander Richardson Date: Thu, 7 Mar 2024 23:24:59 -0800 Subject: [PATCH 564/929] [compiler-rt] Fix incorrect usages of check_cxx_compiler_flag These checks have been broken since 6afe972195454a1110ed8d20c6f2a547e6366379. The check_cxx_compiler_flag macro only takes two arguments and passing three to it ends up calling `cmake_check_compiler_flag(CXX "${_FLAG}" ${_RESULT})` with ${_FLAG} equal to `-Werror` and the result variable being the actually tested compiler flag. I noticed this because some of the flags that I know should be supported were being flagged as not supported. `--debug-trycompile` shows the following surprising line in the generated CMakeLists.txt: `add_definitions([==[-D-Wempty-body]==] [==[-Werror]==])` which then results in the following error while running the check: ``` FAILED: CMakeFiles/cmTC_72736.dir/src.cxx.o tmp/upstream-llvm-readonly/bin/clang++ -nodefaultlibs -std=c++17 -fcolor-diagnostics -D-Wempty-body -Werror -MD -MT CMakeFiles/cmTC_72736.dir/src.cxx.o -MF CMakeFiles/cmTC_72736.dir/src.cxx.o.d -o CMakeFiles/cmTC_72736.dir/src.cxx.o -c .../cmake-build-all-sanitizers/CMakeFiles/CMakeScratch/TryCompile-nyh3QR/src.cxx In file included from :450: :1:9: error: macro name must be an identifier 1 | #define -Wempty-body 1 | ^ 1 error generated. ``` It would be great if CMake could be a bit more helpful here so I've filed https://gitlab.kitware.com/cmake/cmake/-/issues/25735. See also https://reviews.llvm.org/D146920. Reviewed By: nikic Pull Request: https://github.com/llvm/llvm-project/pull/83779 --- compiler-rt/cmake/config-ix.cmake | 32 +++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/compiler-rt/cmake/config-ix.cmake b/compiler-rt/cmake/config-ix.cmake index 2ca18ebb4ad4..4f47142850a5 100644 --- a/compiler-rt/cmake/config-ix.cmake +++ b/compiler-rt/cmake/config-ix.cmake @@ -103,7 +103,7 @@ check_cxx_compiler_flag("-Werror -msse4.2" COMPILER_RT_HAS_MSSE4_2_FLAG) check_cxx_compiler_flag(--sysroot=. COMPILER_RT_HAS_SYSROOT_FLAG) check_cxx_compiler_flag("-Werror -mcrc" COMPILER_RT_HAS_MCRC_FLAG) check_cxx_compiler_flag(-fno-partial-inlining COMPILER_RT_HAS_FNO_PARTIAL_INLINING_FLAG) -check_cxx_compiler_flag(-Werror -ftrivial-auto-var-init=pattern COMPILER_RT_HAS_TRIVIAL_AUTO_INIT) +check_cxx_compiler_flag("-Werror -ftrivial-auto-var-init=pattern" COMPILER_RT_HAS_TRIVIAL_AUTO_INIT) if(NOT WIN32 AND NOT CYGWIN) # MinGW warns if -fvisibility-inlines-hidden is used. @@ -150,21 +150,21 @@ check_cxx_compiler_flag(/wd4391 COMPILER_RT_HAS_WD4391_FLAG) check_cxx_compiler_flag(/wd4722 COMPILER_RT_HAS_WD4722_FLAG) check_cxx_compiler_flag(/wd4800 COMPILER_RT_HAS_WD4800_FLAG) -check_cxx_compiler_flag(-Werror -Warray-bounds COMPILER_RT_HAS_ARRAY_BOUNDS_FLAG) -check_cxx_compiler_flag(-Werror -Wuninitialized COMPILER_RT_HAS_UNINITIALIZED_FLAG) -check_cxx_compiler_flag(-Werror -Wshadow COMPILER_RT_HAS_SHADOW_FLAG) -check_cxx_compiler_flag(-Werror -Wempty-body COMPILER_RT_HAS_EMPTY_BODY_FLAG) -check_cxx_compiler_flag(-Werror -Wsizeof-pointer-memaccess COMPILER_RT_HAS_SIZEOF_POINTER_MEMACCESS_FLAG) -check_cxx_compiler_flag(-Werror -Wsizeof-array-argument COMPILER_RT_HAS_SIZEOF_ARRAY_ARGUMENT_FLAG) -check_cxx_compiler_flag(-Werror -Wsuspicious-memaccess COMPILER_RT_HAS_SUSPICIOUS_MEMACCESS_FLAG) -check_cxx_compiler_flag(-Werror -Wbuiltin-memcpy-chk-size COMPILER_RT_HAS_BUILTIN_MEMCPY_CHK_SIZE_FLAG) -check_cxx_compiler_flag(-Werror -Warray-bounds-pointer-arithmetic COMPILER_RT_HAS_ARRAY_BOUNDS_POINTER_ARITHMETIC_FLAG) -check_cxx_compiler_flag(-Werror -Wreturn-stack-address COMPILER_RT_HAS_RETURN_STACK_ADDRESS_FLAG) -check_cxx_compiler_flag(-Werror -Wsizeof-array-decay COMPILER_RT_HAS_SIZEOF_ARRAY_DECAY_FLAG) -check_cxx_compiler_flag(-Werror -Wformat-insufficient-args COMPILER_RT_HAS_FORMAT_INSUFFICIENT_ARGS_FLAG) -check_cxx_compiler_flag(-Werror -Wformat-security COMPILER_RT_HAS_BUILTIN_FORMAL_SECURITY_FLAG) -check_cxx_compiler_flag(-Werror -Wsizeof-array-div COMPILER_RT_HAS_SIZEOF_ARRAY_DIV_FLAG) -check_cxx_compiler_flag(-Werror -Wsizeof-pointer-div COMPILER_RT_HAS_SIZEOF_POINTER_DIV_FLAG) +check_cxx_compiler_flag("-Werror -Warray-bounds" COMPILER_RT_HAS_ARRAY_BOUNDS_FLAG) +check_cxx_compiler_flag("-Werror -Wuninitialized" COMPILER_RT_HAS_UNINITIALIZED_FLAG) +check_cxx_compiler_flag("-Werror -Wshadow" COMPILER_RT_HAS_SHADOW_FLAG) +check_cxx_compiler_flag("-Werror -Wempty-body" COMPILER_RT_HAS_EMPTY_BODY_FLAG) +check_cxx_compiler_flag("-Werror -Wsizeof-pointer-memaccess" COMPILER_RT_HAS_SIZEOF_POINTER_MEMACCESS_FLAG) +check_cxx_compiler_flag("-Werror -Wsizeof-array-argument" COMPILER_RT_HAS_SIZEOF_ARRAY_ARGUMENT_FLAG) +check_cxx_compiler_flag("-Werror -Wsuspicious-memaccess" COMPILER_RT_HAS_SUSPICIOUS_MEMACCESS_FLAG) +check_cxx_compiler_flag("-Werror -Wbuiltin-memcpy-chk-size" COMPILER_RT_HAS_BUILTIN_MEMCPY_CHK_SIZE_FLAG) +check_cxx_compiler_flag("-Werror -Warray-bounds-pointer-arithmetic" COMPILER_RT_HAS_ARRAY_BOUNDS_POINTER_ARITHMETIC_FLAG) +check_cxx_compiler_flag("-Werror -Wreturn-stack-address" COMPILER_RT_HAS_RETURN_STACK_ADDRESS_FLAG) +check_cxx_compiler_flag("-Werror -Wsizeof-array-decay" COMPILER_RT_HAS_SIZEOF_ARRAY_DECAY_FLAG) +check_cxx_compiler_flag("-Werror -Wformat-insufficient-args" COMPILER_RT_HAS_FORMAT_INSUFFICIENT_ARGS_FLAG) +check_cxx_compiler_flag("-Werror -Wformat-security" COMPILER_RT_HAS_BUILTIN_FORMAL_SECURITY_FLAG) +check_cxx_compiler_flag("-Werror -Wsizeof-array-div" COMPILER_RT_HAS_SIZEOF_ARRAY_DIV_FLAG) +check_cxx_compiler_flag("-Werror -Wsizeof-pointer-div" COMPILER_RT_HAS_SIZEOF_POINTER_DIV_FLAG) # Symbols. check_symbol_exists(__func__ "" COMPILER_RT_HAS_FUNC_SYMBOL) -- GitLab From 780a20984b6af067eed5870eb7c00ea076b6072a Mon Sep 17 00:00:00 2001 From: Matthias Gehre Date: Fri, 8 Mar 2024 08:27:46 +0100 Subject: [PATCH 565/929] [PDLL]: Fix crash when negation doesn't apply to native constraint (#84331) Fixes that ``` Pattern { let tuple = (attr<"3 : i34">); not tuple.0; erase _; } ``` would crash the PDLL parser because it expected a native constraint after `not`. --- mlir/lib/Tools/PDLL/Parser/Parser.cpp | 2 ++ mlir/test/mlir-pdll/Parser/expr-failure.pdll | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/mlir/lib/Tools/PDLL/Parser/Parser.cpp b/mlir/lib/Tools/PDLL/Parser/Parser.cpp index 206781ed1466..9f931f4fce00 100644 --- a/mlir/lib/Tools/PDLL/Parser/Parser.cpp +++ b/mlir/lib/Tools/PDLL/Parser/Parser.cpp @@ -1968,6 +1968,8 @@ FailureOr Parser::parseNegatedExpr() { FailureOr identifierExpr = parseIdentifierExpr(); if (failed(identifierExpr)) return failure(); + if (!curToken.is(Token::l_paren)) + return emitError("expected `(` after function name"); return parseCallExpr(*identifierExpr, /*isNegated = */ true); } diff --git a/mlir/test/mlir-pdll/Parser/expr-failure.pdll b/mlir/test/mlir-pdll/Parser/expr-failure.pdll index 253d770c83e5..898fe8680a76 100644 --- a/mlir/test/mlir-pdll/Parser/expr-failure.pdll +++ b/mlir/test/mlir-pdll/Parser/expr-failure.pdll @@ -205,6 +205,23 @@ Pattern { // ----- +Pattern { + // CHECK: expected native constraint + not attr<"0 : i1"> + erase _; +} + +// ----- + +Pattern { + let tuple = (attr<"3 : i34">); + // CHECK: expected `(` after function name + not tuple.0; + erase _; +} + +// ----- + Pattern { // CHECK: expected expression let tuple = (10 = _: Value); -- GitLab From f6b825f51ec8a67c4ace43aaacc27bfd4a78f706 Mon Sep 17 00:00:00 2001 From: Amara Emerson Date: Thu, 7 Mar 2024 23:15:58 -0800 Subject: [PATCH 566/929] Revert "Revert "[AArch64][GlobalISel] Fix incorrect selection of monotonic s32->s64 anyext load."" Attempt 2. The first one was trying to call isa<> on an MI reference that was free'd. This reverts commit ee24409c40ff35c3221892d9723331c233ca9f0e. --- .../GISel/AArch64InstructionSelector.cpp | 9 ++--- .../GlobalISel/select-atomic-load-store.mir | 33 ++++++++++++++++--- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp index 665288379239..0f3c3cb96e6c 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp @@ -2997,13 +2997,14 @@ bool AArch64InstructionSelector::select(MachineInstr &I) { } } - if (IsZExtLoad) { - // The zextload from a smaller type to i32 should be handled by the + if (IsZExtLoad || (Opcode == TargetOpcode::G_LOAD && + ValTy == LLT::scalar(64) && MemSizeInBits == 32)) { + // The any/zextload from a smaller type to i32 should be handled by the // importer. if (MRI.getType(LoadStore->getOperand(0).getReg()).getSizeInBits() != 64) return false; - // If we have a ZEXTLOAD then change the load's type to be a narrower reg - // and zero_extend with SUBREG_TO_REG. + // If we have an extending load then change the load's type to be a + // narrower reg and zero_extend with SUBREG_TO_REG. Register LdReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass); Register DstReg = LoadStore->getOperand(0).getReg(); LoadStore->getOperand(0).setReg(LdReg); diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-atomic-load-store.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-atomic-load-store.mir index 5787f914b965..6b4bbb85b2ec 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/select-atomic-load-store.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-atomic-load-store.mir @@ -9,6 +9,11 @@ ret i8 %v } + define i32 @anyext_load_monotonic_i32() { + %v = load atomic i32, ptr null monotonic, align 4 + ret i32 %v + } + ... --- name: load_acq_i8 @@ -25,13 +30,33 @@ body: | ; CHECK-LABEL: name: load_acq_i8 ; CHECK: liveins: $x0 - ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0 - ; CHECK: [[LDARB:%[0-9]+]]:gpr32 = LDARB [[COPY]] :: (load acquire (s8) from %ir.ptr, align 8) - ; CHECK: $w0 = COPY [[LDARB]] - ; CHECK: RET_ReallyLR implicit $w0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0 + ; CHECK-NEXT: [[LDARB:%[0-9]+]]:gpr32 = LDARB [[COPY]] :: (load acquire (s8) from %ir.ptr, align 8) + ; CHECK-NEXT: $w0 = COPY [[LDARB]] + ; CHECK-NEXT: RET_ReallyLR implicit $w0 %0:gpr(p0) = COPY $x0 %2:gpr(s32) = G_LOAD %0(p0) :: (load acquire (s8) from %ir.ptr, align 8) $w0 = COPY %2(s32) RET_ReallyLR implicit $w0 ... +--- +name: anyext_load_monotonic_i32 +legalized: true +regBankSelected: true +tracksRegLiveness: true +body: | + bb.1: + ; CHECK-LABEL: name: anyext_load_monotonic_i32 + ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $xzr + ; CHECK-NEXT: [[LDRWui:%[0-9]+]]:gpr32 = LDRWui [[COPY]], 0 :: (load monotonic (s32) from `ptr null`) + ; CHECK-NEXT: %ld:gpr64all = SUBREG_TO_REG 0, [[LDRWui]], %subreg.sub_32 + ; CHECK-NEXT: $x0 = COPY %ld + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %1:gpr(p0) = G_CONSTANT i64 0 + %ld:gpr(s64) = G_LOAD %1(p0) :: (load monotonic (s32) from `ptr null`) + $x0 = COPY %ld(s64) + RET_ReallyLR implicit $x0 + +... -- GitLab From b6a340023d383d1e77cb8d91d92c096f791fa8c0 Mon Sep 17 00:00:00 2001 From: Vlad Serebrennikov Date: Fri, 8 Mar 2024 11:31:00 +0400 Subject: [PATCH 567/929] [clang] Respect field alignment in layout compatibility of structs (#84313) This patch implements [CWG2586](https://cplusplus.github.io/CWG/issues/2583.html) "Common initial sequence should consider over-alignment". Note that alignment of union members doesn't have to match, as layout compatibility of unions is not defined in terms of common initial sequence (http://eel.is/c++draft/class.mem.general#25). --- clang/docs/ReleaseNotes.rst | 4 ++++ clang/lib/Sema/SemaChecking.cpp | 23 +++++++++++++++++++++-- clang/test/CXX/drs/dr25xx.cpp | 26 ++++++++++++++++++++++++++ clang/test/SemaCXX/type-traits.cpp | 13 ++++++++++++- clang/www/cxx_dr_status.html | 2 +- 5 files changed, 64 insertions(+), 4 deletions(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index fa23c215790f..fe7bbe437831 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -115,6 +115,10 @@ Resolutions to C++ Defect Reports of two types. (`CWG1719: Layout compatibility and cv-qualification revisited `_). +- Alignment of members is now respected when evaluating layout compatibility + of structs. + (`CWG2583: Common initial sequence should consider over-alignment `_). + - ``[[no_unique_address]]`` is now respected when evaluating layout compatibility of two types. (`CWG2759: [[no_unique_address] and common initial sequence `_). diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index 3597f93a0171..b34b8df00201 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -19184,8 +19184,22 @@ static bool isLayoutCompatible(ASTContext &C, EnumDecl *ED1, EnumDecl *ED2) { } /// Check if two fields are layout-compatible. +/// Can be used on union members, which are exempt from alignment requirement +/// of common initial sequence. static bool isLayoutCompatible(ASTContext &C, FieldDecl *Field1, - FieldDecl *Field2) { + FieldDecl *Field2, + bool AreUnionMembers = false) { + const Type *Field1Parent = Field1->getParent()->getTypeForDecl(); + const Type *Field2Parent = Field2->getParent()->getTypeForDecl(); + assert(((Field1Parent->isStructureOrClassType() && + Field2Parent->isStructureOrClassType()) || + (Field1Parent->isUnionType() && Field2Parent->isUnionType())) && + "Can't evaluate layout compatibility between a struct field and a " + "union field."); + assert(((!AreUnionMembers && Field1Parent->isStructureOrClassType()) || + (AreUnionMembers && Field1Parent->isUnionType())) && + "AreUnionMembers should be 'true' for union fields (only)."); + if (!isLayoutCompatible(C, Field1->getType(), Field2->getType())) return false; @@ -19204,6 +19218,11 @@ static bool isLayoutCompatible(ASTContext &C, FieldDecl *Field1, if (Field1->hasAttr() || Field2->hasAttr()) return false; + + if (!AreUnionMembers && + Field1->getMaxAlignment() != Field2->getMaxAlignment()) + return false; + return true; } @@ -19265,7 +19284,7 @@ static bool isLayoutCompatibleUnion(ASTContext &C, RecordDecl *RD1, E = UnmatchedFields.end(); for ( ; I != E; ++I) { - if (isLayoutCompatible(C, Field1, *I)) { + if (isLayoutCompatible(C, Field1, *I, /*IsUnionMember=*/true)) { bool Result = UnmatchedFields.erase(*I); (void) Result; assert(Result); diff --git a/clang/test/CXX/drs/dr25xx.cpp b/clang/test/CXX/drs/dr25xx.cpp index 9fc7cf59485c..46532486e50e 100644 --- a/clang/test/CXX/drs/dr25xx.cpp +++ b/clang/test/CXX/drs/dr25xx.cpp @@ -211,6 +211,32 @@ namespace dr2565 { // dr2565: 16 open 2023-06-07 #endif } +namespace dr2583 { // dr2583: 19 +#if __cplusplus >= 201103L +struct A { + int i; + char c; +}; + +struct B { + int i; + alignas(8) char c; +}; + +union U { + A a; + B b; +}; + +union V { + A a; + alignas(64) B b; +}; + +static_assert(!__is_layout_compatible(A, B), ""); +static_assert(__is_layout_compatible(U, V), ""); +#endif +} // namespace dr2583 namespace dr2598 { // dr2598: 18 #if __cplusplus >= 201103L diff --git a/clang/test/SemaCXX/type-traits.cpp b/clang/test/SemaCXX/type-traits.cpp index 23c339ebdf08..831de2589dcb 100644 --- a/clang/test/SemaCXX/type-traits.cpp +++ b/clang/test/SemaCXX/type-traits.cpp @@ -1681,6 +1681,16 @@ union UnionLayout3 { [[no_unique_address]] CEmptyStruct d; }; +union UnionNoOveralignedMembers { + int a; + double b; +}; + +union UnionWithOveralignedMembers { + int a; + alignas(16) double b; +}; + struct StructWithAnonUnion { union { int a; @@ -1771,7 +1781,8 @@ void is_layout_compatible(int n) static_assert(__is_layout_compatible(CStruct, CStructNoUniqueAddress) != bool(__has_cpp_attribute(no_unique_address)), ""); static_assert(__is_layout_compatible(CStructNoUniqueAddress, CStructNoUniqueAddress2) != bool(__has_cpp_attribute(no_unique_address)), ""); static_assert(__is_layout_compatible(CStruct, CStructAlignment), ""); - static_assert(__is_layout_compatible(CStruct, CStructAlignedMembers), ""); // FIXME: alignment of members impact common initial sequence + static_assert(!__is_layout_compatible(CStruct, CStructAlignedMembers), ""); + static_assert(__is_layout_compatible(UnionNoOveralignedMembers, UnionWithOveralignedMembers), ""); static_assert(__is_layout_compatible(CStructWithBitfelds, CStructWithBitfelds), ""); static_assert(__is_layout_compatible(CStructWithBitfelds, CStructWithBitfelds2), ""); static_assert(!__is_layout_compatible(CStructWithBitfelds, CStructWithBitfelds3), ""); diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html index 503472a2cae4..c20a5d021e9d 100755 --- a/clang/www/cxx_dr_status.html +++ b/clang/www/cxx_dr_status.html @@ -15306,7 +15306,7 @@ and POD class 2583 C++23 Common initial sequence should consider over-alignment - Unknown + Clang 19 2584 -- GitLab From fb1be9b33ca3ed1b7ea54b15bd77fd868726b57c Mon Sep 17 00:00:00 2001 From: Vyacheslav Levytskyy Date: Fri, 8 Mar 2024 08:31:56 +0100 Subject: [PATCH 568/929] [SPIR-V] Insert a bitcast before load/store instruction to keep SPIR-V code valid (#84069) This PR introduces a step after instruction selection where instructions can be traversed from the perspective of their validity from the specification point of view. The PR adds also a way to correct load/store when there is a type mismatch contradicting the specification -- an additional bitcast is inserted to keep types consistent. Correspondent test cases are added and existing test cases are corrected. This PR helps to successfully validate with the `spirv-val` tool (https://github.com/KhronosGroup/SPIRV-Tools) some output that previously led to validation errors and crashes of back translation from SPIRV to LLVM IR from the side of SPIRV Translator project (https://github.com/KhronosGroup/SPIRV-LLVM-Translator). The added step of bringing instructions to required by the specification type correspondence can be (should be and will be) extended beyond load/store instructions to ensure validity rules of other SPIRV instructions related to type inference. --- llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 4 +- llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp | 93 ++++++++++--------- llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h | 1 + llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 80 ++++++++++++++++ llvm/lib/Target/SPIRV/SPIRVISelLowering.h | 12 ++- .../SPIRV/constant/global-constants.ll | 1 + .../SPIRV/pointers/bitcast-fix-load.ll | 21 +++++ .../SPIRV/pointers/bitcast-fix-store.ll | 31 +++++++ llvm/test/CodeGen/SPIRV/spirv-load-store.ll | 11 ++- 9 files changed, 203 insertions(+), 51 deletions(-) create mode 100644 llvm/test/CodeGen/SPIRV/pointers/bitcast-fix-load.ll create mode 100644 llvm/test/CodeGen/SPIRV/pointers/bitcast-fix-store.ll diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index 5c432d682732..575e903d05bb 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -20,6 +20,7 @@ #include "llvm/IR/InstIterator.h" #include "llvm/IR/InstVisitor.h" #include "llvm/IR/IntrinsicsSPIRV.h" +#include "llvm/IR/TypedPointerType.h" #include @@ -446,7 +447,8 @@ void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I, for (unsigned OpIdx = 0; OpIdx < CI->arg_size(); OpIdx++) { Value *ArgOperand = CI->getArgOperand(OpIdx); - if (!isa(ArgOperand->getType())) + if (!isa(ArgOperand->getType()) && + !isa(ArgOperand->getType())) continue; // Constants (nulls/undefs) are handled in insertAssignPtrTypeIntrs() diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp index e88298f52fbe..8556581996fe 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -20,6 +20,7 @@ #include "SPIRVSubtarget.h" #include "SPIRVTargetMachine.h" #include "SPIRVUtils.h" +#include "llvm/IR/TypedPointerType.h" using namespace llvm; SPIRVGlobalRegistry::SPIRVGlobalRegistry(unsigned PointerSize) @@ -420,9 +421,10 @@ Register SPIRVGlobalRegistry::getOrCreateConstNullPtr(MachineIRBuilder &MIRBuilder, SPIRVType *SpvType) { const Type *LLVMTy = getTypeForSPIRVType(SpvType); - const PointerType *LLVMPtrTy = cast(LLVMTy); + const TypedPointerType *LLVMPtrTy = cast(LLVMTy); // Find a constant in DT or build a new one. - Constant *CP = ConstantPointerNull::get(const_cast(LLVMPtrTy)); + Constant *CP = ConstantPointerNull::get(PointerType::get( + LLVMPtrTy->getElementType(), LLVMPtrTy->getAddressSpace())); Register Res = DT.find(CP, CurMF); if (!Res.isValid()) { LLT LLTy = LLT::pointer(LLVMPtrTy->getAddressSpace(), PointerSize); @@ -517,6 +519,13 @@ Register SPIRVGlobalRegistry::buildGlobalVariable( LLT RegLLTy = LLT::pointer(MRI->getType(ResVReg).getAddressSpace(), 32); MRI->setType(Reg, RegLLTy); assignSPIRVTypeToVReg(BaseType, Reg, MIRBuilder.getMF()); + } else { + // Our knowledge about the type may be updated. + // If that's the case, we need to update a type + // associated with the register. + SPIRVType *DefType = getSPIRVTypeForVReg(ResVReg); + if (!DefType || DefType != BaseType) + assignSPIRVTypeToVReg(BaseType, Reg, MIRBuilder.getMF()); } // If it's a global variable with name, output OpName for it. @@ -705,33 +714,37 @@ SPIRVType *SPIRVGlobalRegistry::createSPIRVType( } return getOpTypeFunction(RetTy, ParamTypes, MIRBuilder); } - if (auto PType = dyn_cast(Ty)) { - SPIRVType *SpvElementType; - // At the moment, all opaque pointers correspond to i8 element type. - // TODO: change the implementation once opaque pointers are supported - // in the SPIR-V specification. - SpvElementType = getOrCreateSPIRVIntegerType(8, MIRBuilder); - // Get access to information about available extensions - const SPIRVSubtarget *ST = - static_cast(&MIRBuilder.getMF().getSubtarget()); - auto SC = addressSpaceToStorageClass(PType->getAddressSpace(), *ST); - // Null pointer means we have a loop in type definitions, make and - // return corresponding OpTypeForwardPointer. - if (SpvElementType == nullptr) { - if (!ForwardPointerTypes.contains(Ty)) - ForwardPointerTypes[PType] = getOpTypeForwardPointer(SC, MIRBuilder); - return ForwardPointerTypes[PType]; - } - // If we have forward pointer associated with this type, use its register - // operand to create OpTypePointer. - if (ForwardPointerTypes.contains(PType)) { - Register Reg = getSPIRVTypeID(ForwardPointerTypes[PType]); - return getOpTypePointer(SC, SpvElementType, MIRBuilder, Reg); - } - - return getOrCreateSPIRVPointerType(SpvElementType, MIRBuilder, SC); + unsigned AddrSpace = 0xFFFF; + if (auto PType = dyn_cast(Ty)) + AddrSpace = PType->getAddressSpace(); + else if (auto PType = dyn_cast(Ty)) + AddrSpace = PType->getAddressSpace(); + else + report_fatal_error("Unable to convert LLVM type to SPIRVType", true); + SPIRVType *SpvElementType; + // At the moment, all opaque pointers correspond to i8 element type. + // TODO: change the implementation once opaque pointers are supported + // in the SPIR-V specification. + SpvElementType = getOrCreateSPIRVIntegerType(8, MIRBuilder); + // Get access to information about available extensions + const SPIRVSubtarget *ST = + static_cast(&MIRBuilder.getMF().getSubtarget()); + auto SC = addressSpaceToStorageClass(AddrSpace, *ST); + // Null pointer means we have a loop in type definitions, make and + // return corresponding OpTypeForwardPointer. + if (SpvElementType == nullptr) { + if (!ForwardPointerTypes.contains(Ty)) + ForwardPointerTypes[Ty] = getOpTypeForwardPointer(SC, MIRBuilder); + return ForwardPointerTypes[Ty]; + } + // If we have forward pointer associated with this type, use its register + // operand to create OpTypePointer. + if (ForwardPointerTypes.contains(Ty)) { + Register Reg = getSPIRVTypeID(ForwardPointerTypes[Ty]); + return getOpTypePointer(SC, SpvElementType, MIRBuilder, Reg); } - llvm_unreachable("Unable to convert LLVM type to SPIRVType"); + + return getOrCreateSPIRVPointerType(SpvElementType, MIRBuilder, SC); } SPIRVType *SPIRVGlobalRegistry::restOfCreateSPIRVType( @@ -1139,11 +1152,13 @@ SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVPointerType( SPIRV::StorageClass::StorageClass SC) { const Type *PointerElementType = getTypeForSPIRVType(BaseType); unsigned AddressSpace = storageClassToAddressSpace(SC); - Type *LLVMTy = - PointerType::get(const_cast(PointerElementType), AddressSpace); + Type *LLVMTy = TypedPointerType::get(const_cast(PointerElementType), + AddressSpace); + // check if this type is already available Register Reg = DT.find(PointerElementType, AddressSpace, CurMF); if (Reg.isValid()) return getSPIRVTypeForVReg(Reg); + // create a new type auto MIB = BuildMI(MIRBuilder.getMBB(), MIRBuilder.getInsertPt(), MIRBuilder.getDebugLoc(), MIRBuilder.getTII().get(SPIRV::OpTypePointer)) @@ -1155,22 +1170,10 @@ SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVPointerType( } SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVPointerType( - SPIRVType *BaseType, MachineInstr &I, const SPIRVInstrInfo &TII, + SPIRVType *BaseType, MachineInstr &I, const SPIRVInstrInfo &, SPIRV::StorageClass::StorageClass SC) { - const Type *PointerElementType = getTypeForSPIRVType(BaseType); - unsigned AddressSpace = storageClassToAddressSpace(SC); - Type *LLVMTy = - PointerType::get(const_cast(PointerElementType), AddressSpace); - Register Reg = DT.find(PointerElementType, AddressSpace, CurMF); - if (Reg.isValid()) - return getSPIRVTypeForVReg(Reg); - MachineBasicBlock &BB = *I.getParent(); - auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpTypePointer)) - .addDef(createTypeVReg(CurMF->getRegInfo())) - .addImm(static_cast(SC)) - .addUse(getSPIRVTypeID(BaseType)); - DT.add(PointerElementType, AddressSpace, CurMF, getSPIRVTypeID(MIB)); - return finishCreatingSPIRVType(LLVMTy, MIB); + MachineIRBuilder MIRBuilder(I); + return getOrCreateSPIRVPointerType(BaseType, MIRBuilder, SC); } Register SPIRVGlobalRegistry::getOrCreateUndef(MachineInstr &I, diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h index f5a83072c19d..9c0061d13fd0 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h @@ -34,6 +34,7 @@ class SPIRVGlobalRegistry { DenseMap> VRegToTypeMap; + // Map LLVM Type* to SPIRVGeneralDuplicatesTracker DT; DenseMap SPIRVToLLVMType; diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp index 33c6aa242969..61748070fc0f 100644 --- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp @@ -12,6 +12,13 @@ #include "SPIRVISelLowering.h" #include "SPIRV.h" +#include "SPIRVInstrInfo.h" +#include "SPIRVRegisterBankInfo.h" +#include "SPIRVRegisterInfo.h" +#include "SPIRVSubtarget.h" +#include "SPIRVTargetMachine.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/IR/IntrinsicsSPIRV.h" #define DEBUG_TYPE "spirv-lower" @@ -74,3 +81,76 @@ bool SPIRVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, } return false; } + +// Insert a bitcast before the instruction to keep SPIR-V code valid +// when there is a type mismatch between results and operand types. +static void validatePtrTypes(const SPIRVSubtarget &STI, + MachineRegisterInfo *MRI, SPIRVGlobalRegistry &GR, + MachineInstr &I, SPIRVType *ResType, + unsigned OpIdx) { + Register OpReg = I.getOperand(OpIdx).getReg(); + SPIRVType *TypeInst = MRI->getVRegDef(OpReg); + SPIRVType *OpType = GR.getSPIRVTypeForVReg( + TypeInst && TypeInst->getOpcode() == SPIRV::OpFunctionParameter + ? TypeInst->getOperand(1).getReg() + : OpReg); + if (!ResType || !OpType || OpType->getOpcode() != SPIRV::OpTypePointer) + return; + SPIRVType *ElemType = GR.getSPIRVTypeForVReg(OpType->getOperand(2).getReg()); + if (!ElemType || ElemType == ResType) + return; + // There is a type mismatch between results and operand types + // and we insert a bitcast before the instruction to keep SPIR-V code valid + SPIRV::StorageClass::StorageClass SC = + static_cast( + OpType->getOperand(1).getImm()); + MachineInstr *PrevI = I.getPrevNode(); + MachineBasicBlock &MBB = *I.getParent(); + MachineBasicBlock::iterator InsPt = + PrevI ? PrevI->getIterator() : MBB.begin(); + MachineIRBuilder MIB(MBB, InsPt); + SPIRVType *NewPtrType = GR.getOrCreateSPIRVPointerType(ResType, MIB, SC); + if (!GR.isBitcastCompatible(NewPtrType, OpType)) + report_fatal_error( + "insert validation bitcast: incompatible result and operand types"); + Register NewReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); + bool Res = MIB.buildInstr(SPIRV::OpBitcast) + .addDef(NewReg) + .addUse(GR.getSPIRVTypeID(NewPtrType)) + .addUse(OpReg) + .constrainAllUses(*STI.getInstrInfo(), *STI.getRegisterInfo(), + *STI.getRegBankInfo()); + if (!Res) + report_fatal_error("insert validation bitcast: cannot constrain all uses"); + MRI->setRegClass(NewReg, &SPIRV::IDRegClass); + GR.assignSPIRVTypeToVReg(NewPtrType, NewReg, MIB.getMF()); + I.getOperand(OpIdx).setReg(NewReg); +} + +// TODO: the logic of inserting additional bitcast's is to be moved +// to pre-IRTranslation passes eventually +void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const { + MachineRegisterInfo *MRI = &MF.getRegInfo(); + SPIRVGlobalRegistry &GR = *STI.getSPIRVGlobalRegistry(); + GR.setCurrentFunc(MF); + for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) { + MachineBasicBlock *MBB = &*I; + for (MachineBasicBlock::iterator MBBI = MBB->begin(), MBBE = MBB->end(); + MBBI != MBBE;) { + MachineInstr &MI = *MBBI++; + switch (MI.getOpcode()) { + case SPIRV::OpLoad: + // OpLoad , ptr %Op implies that %Op is a pointer to + validatePtrTypes(STI, MRI, GR, MI, + GR.getSPIRVTypeForVReg(MI.getOperand(0).getReg()), 2); + break; + case SPIRV::OpStore: + // OpStore ptr %Op, implies that %Op points to the 's type + validatePtrTypes(STI, MRI, GR, MI, + GR.getSPIRVTypeForVReg(MI.getOperand(1).getReg()), 0); + break; + } + } + } + TargetLowering::finalizeLowering(MF); +} diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h index d34f802e9d88..b01571bfc1ee 100644 --- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h +++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h @@ -14,16 +14,19 @@ #ifndef LLVM_LIB_TARGET_SPIRV_SPIRVISELLOWERING_H #define LLVM_LIB_TARGET_SPIRV_SPIRVISELLOWERING_H +#include "SPIRVGlobalRegistry.h" #include "llvm/CodeGen/TargetLowering.h" namespace llvm { class SPIRVSubtarget; class SPIRVTargetLowering : public TargetLowering { + const SPIRVSubtarget &STI; + public: explicit SPIRVTargetLowering(const TargetMachine &TM, - const SPIRVSubtarget &STI) - : TargetLowering(TM) {} + const SPIRVSubtarget &ST) + : TargetLowering(TM), STI(ST) {} // Stop IRTranslator breaking up FMA instrs to preserve types information. bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, @@ -47,6 +50,11 @@ public: bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override; + + // Call the default implementation and finalize target lowering by inserting + // extra instructions required to preserve validity of SPIR-V code imposed by + // the standard. + void finalizeLowering(MachineFunction &MF) const override; }; } // namespace llvm diff --git a/llvm/test/CodeGen/SPIRV/constant/global-constants.ll b/llvm/test/CodeGen/SPIRV/constant/global-constants.ll index 916c70628d01..74e28cbe7acb 100644 --- a/llvm/test/CodeGen/SPIRV/constant/global-constants.ll +++ b/llvm/test/CodeGen/SPIRV/constant/global-constants.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} @global = addrspace(1) constant i32 1 ; OpenCL global memory @constant = addrspace(2) constant i32 2 ; OpenCL constant memory diff --git a/llvm/test/CodeGen/SPIRV/pointers/bitcast-fix-load.ll b/llvm/test/CodeGen/SPIRV/pointers/bitcast-fix-load.ll new file mode 100644 index 000000000000..a30d0792e399 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/pointers/bitcast-fix-load.ll @@ -0,0 +1,21 @@ +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; CHECK-DAG: %[[#TYLONG:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#TYSTRUCTLONG:]] = OpTypeStruct %[[#TYLONG]] +; CHECK-DAG: %[[#TYARRAY:]] = OpTypeArray %[[#TYSTRUCTLONG]] %[[#]] +; CHECK-DAG: %[[#TYSTRUCT:]] = OpTypeStruct %[[#TYARRAY]] +; CHECK-DAG: %[[#TYSTRUCTPTR:]] = OpTypePointer Function %[[#TYSTRUCT]] +; CHECK-DAG: %[[#TYLONGPTR:]] = OpTypePointer Function %[[#TYLONG]] +; CHECK: %[[#PTRTOSTRUCT:]] = OpFunctionParameter %[[#TYSTRUCTPTR]] +; CHECK: %[[#PTRTOLONG:]] = OpBitcast %[[#TYLONGPTR]] %[[#PTRTOSTRUCT]] +; CHECK: OpLoad %[[#TYLONG]] %[[#PTRTOLONG]] + +%struct.S = type { i32 } +%struct.__wrapper_class = type { [7 x %struct.S] } + +define spir_kernel void @foo(ptr noundef byval(%struct.__wrapper_class) align 4 %_arg_Arr) { +entry: + %val = load i32, ptr %_arg_Arr + ret void +} diff --git a/llvm/test/CodeGen/SPIRV/pointers/bitcast-fix-store.ll b/llvm/test/CodeGen/SPIRV/pointers/bitcast-fix-store.ll new file mode 100644 index 000000000000..4701f02ea33a --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/pointers/bitcast-fix-store.ll @@ -0,0 +1,31 @@ +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; CHECK-DAG: %[[#TYLONG:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#TYLONGPTR:]] = OpTypePointer Function %[[#TYLONG]] +; CHECK-DAG: %[[#TYSTRUCT:]] = OpTypeStruct %[[#TYLONG]] +; CHECK-DAG: %[[#CONST:]] = OpConstant %[[#TYLONG]] 3 +; CHECK-DAG: %[[#TYSTRUCTPTR:]] = OpTypePointer Function %[[#TYSTRUCT]] +; CHECK: OpFunction +; CHECK: %[[#ARGPTR1:]] = OpFunctionParameter %[[#TYLONGPTR]] +; CHECK: OpStore %[[#ARGPTR1]] %[[#CONST:]] +; CHECK: OpFunction +; CHECK: %[[#OBJ:]] = OpFunctionParameter %[[#TYSTRUCT]] +; CHECK: %[[#ARGPTR2:]] = OpFunctionParameter %[[#TYLONGPTR]] +; CHECK: %[[#PTRTOSTRUCT:]] = OpBitcast %[[#TYSTRUCTPTR]] %[[#ARGPTR2]] +; CHECK: OpStore %[[#PTRTOSTRUCT]] %[[#OBJ]] + +%struct.S = type { i32 } +%struct.__wrapper_class = type { [7 x %struct.S] } + +define spir_kernel void @foo(%struct.S %arg, ptr %ptr) { +entry: + store %struct.S %arg, ptr %ptr + ret void +} + +define spir_kernel void @bar(ptr %ptr) { +entry: + store i32 3, ptr %ptr + ret void +} diff --git a/llvm/test/CodeGen/SPIRV/spirv-load-store.ll b/llvm/test/CodeGen/SPIRV/spirv-load-store.ll index a82bf0ab2e01..918861731246 100644 --- a/llvm/test/CodeGen/SPIRV/spirv-load-store.ll +++ b/llvm/test/CodeGen/SPIRV/spirv-load-store.ll @@ -1,9 +1,14 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ;; Translate SPIR-V friendly OpLoad and OpStore calls -; CHECK: %[[#CONST:]] = OpConstant %[[#]] 42 -; CHECK: OpStore %[[#PTR:]] %[[#CONST]] Volatile|Aligned 4 -; CHECK: %[[#]] = OpLoad %[[#]] %[[#PTR]] +; CHECK-DAG: %[[#TYLONG:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#TYFLOAT:]] = OpTypeFloat 64 +; CHECK-DAG: %[[#TYFLOATPTR:]] = OpTypePointer CrossWorkgroup %[[#TYFLOAT]] +; CHECK-DAG: %[[#CONST:]] = OpConstant %[[#TYLONG]] 42 +; CHECK: OpStore %[[#PTRTOLONG:]] %[[#CONST]] Volatile|Aligned 4 +; CHECK: %[[#PTRTOFLOAT:]] = OpBitcast %[[#TYFLOATPTR]] %[[#PTRTOLONG]] +; CHECK: OpLoad %[[#TYFLOAT]] %[[#PTRTOFLOAT]] define weak_odr dso_local spir_kernel void @foo(i32 addrspace(1)* %var) { entry: -- GitLab From df9be017b7828e0a1dbb4f1f507a92266b61e680 Mon Sep 17 00:00:00 2001 From: Marius Brehler Date: Fri, 8 Mar 2024 08:34:56 +0100 Subject: [PATCH 569/929] [mlir][EmitC] Add `unary_{minus,plus}` operators (#84329) This adds operations for the unary minus and the unary plus operator. --- mlir/include/mlir/Dialect/EmitC/IR/EmitC.td | 36 +++++++++++++++++++++ mlir/lib/Target/Cpp/TranslateToCpp.cpp | 17 +++++++++- mlir/test/Dialect/EmitC/ops.mlir | 6 ++++ mlir/test/Target/Cpp/unary_operators.mlir | 12 +++++++ 4 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 mlir/test/Target/Cpp/unary_operators.mlir diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td index db0e2d10960d..ac1e38a5506d 100644 --- a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td +++ b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td @@ -908,6 +908,42 @@ def EmitC_SubOp : EmitC_BinaryOp<"sub", [CExpression]> { let hasVerifier = 1; } +def EmitC_UnaryMinusOp : EmitC_UnaryOp<"unary_minus", [CExpression]> { + let summary = "Unary minus operation"; + let description = [{ + With the `unary_minus` operation the unary operator - (minus) can be + applied. + + Example: + + ```mlir + %0 = emitc.unary_plus %arg0 : (i32) -> i32 + ``` + ```c++ + // Code emitted for the operation above. + int32_t v2 = -v1; + ``` + }]; +} + +def EmitC_UnaryPlusOp : EmitC_UnaryOp<"unary_plus", [CExpression]> { + let summary = "Unary plus operation"; + let description = [{ + With the `unary_plus` operation the unary operator + (plus) can be + applied. + + Example: + + ```mlir + %0 = emitc.unary_plus %arg0 : (i32) -> i32 + ``` + ```c++ + // Code emitted for the operation above. + int32_t v2 = +v1; + ``` + }]; +} + def EmitC_VariableOp : EmitC_Op<"variable", []> { let summary = "Variable operation"; let description = [{ diff --git a/mlir/lib/Target/Cpp/TranslateToCpp.cpp b/mlir/lib/Target/Cpp/TranslateToCpp.cpp index 95513cb0fb2e..b99d0ede8bf4 100644 --- a/mlir/lib/Target/Cpp/TranslateToCpp.cpp +++ b/mlir/lib/Target/Cpp/TranslateToCpp.cpp @@ -104,6 +104,8 @@ static FailureOr getOperatorPrecedence(Operation *operation) { .Case([&](auto op) { return 13; }) .Case([&](auto op) { return 13; }) .Case([&](auto op) { return 12; }) + .Case([&](auto op) { return 15; }) + .Case([&](auto op) { return 15; }) .Default([](auto op) { return op->emitError("unsupported operation"); }); } @@ -652,6 +654,18 @@ static LogicalResult printOperation(CppEmitter &emitter, return printBinaryOperation(emitter, operation, "^"); } +static LogicalResult printOperation(CppEmitter &emitter, + emitc::UnaryPlusOp unaryPlusOp) { + Operation *operation = unaryPlusOp.getOperation(); + return printUnaryOperation(emitter, operation, "+"); +} + +static LogicalResult printOperation(CppEmitter &emitter, + emitc::UnaryMinusOp unaryMinusOp) { + Operation *operation = unaryMinusOp.getOperation(); + return printUnaryOperation(emitter, operation, "-"); +} + static LogicalResult printOperation(CppEmitter &emitter, emitc::CastOp castOp) { raw_ostream &os = emitter.ostream(); Operation &op = *castOp.getOperation(); @@ -1371,7 +1385,8 @@ LogicalResult CppEmitter::emitOperation(Operation &op, bool trailingSemicolon) { emitc::ExpressionOp, emitc::ForOp, emitc::FuncOp, emitc::IfOp, emitc::IncludeOp, emitc::LogicalAndOp, emitc::LogicalNotOp, emitc::LogicalOrOp, emitc::MulOp, emitc::RemOp, emitc::ReturnOp, - emitc::SubOp, emitc::VariableOp, emitc::VerbatimOp>( + emitc::SubOp, emitc::UnaryMinusOp, emitc::UnaryPlusOp, + emitc::VariableOp, emitc::VerbatimOp>( [&](auto op) { return printOperation(*this, op); }) // Func ops. .Case( diff --git a/mlir/test/Dialect/EmitC/ops.mlir b/mlir/test/Dialect/EmitC/ops.mlir index f852390f03e2..122b1d9ef105 100644 --- a/mlir/test/Dialect/EmitC/ops.mlir +++ b/mlir/test/Dialect/EmitC/ops.mlir @@ -134,6 +134,12 @@ func.func @logical(%arg0: i32, %arg1: i32) { return } +func.func @unary(%arg0: i32) { + %0 = emitc.unary_minus %arg0 : (i32) -> i32 + %1 = emitc.unary_plus %arg0 : (i32) -> i32 + return +} + func.func @test_if(%arg0: i1, %arg1: f32) { emitc.if %arg0 { %0 = emitc.call_opaque "func_const"(%arg1) : (f32) -> i32 diff --git a/mlir/test/Target/Cpp/unary_operators.mlir b/mlir/test/Target/Cpp/unary_operators.mlir new file mode 100644 index 000000000000..8a89437a41cc --- /dev/null +++ b/mlir/test/Target/Cpp/unary_operators.mlir @@ -0,0 +1,12 @@ +// RUN: mlir-translate -mlir-to-cpp %s | FileCheck %s + +func.func @unary(%arg0: i32) -> () { + %0 = emitc.unary_minus %arg0 : (i32) -> i32 + %1 = emitc.unary_plus %arg0 : (i32) -> i32 + + return +} + +// CHECK-LABEL: void unary +// CHECK-NEXT: int32_t [[V1:[^ ]*]] = -[[V0:[^ ]*]]; +// CHECK-NEXT: int32_t [[V2:[^ ]*]] = +[[V0]]; -- GitLab From d72b7f913300493480f78d4f23104d0f51d50b62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20K=C3=A9ri?= Date: Fri, 8 Mar 2024 08:56:10 +0100 Subject: [PATCH 570/929] [clang][analyzer] Fix StreamChecker `ftell` and `fgetpos` at indeterminate file position. (#84191) These functions should not be allowed if the file position is indeterminate (they return the file position). This condition is now checked, and tests are improved to check it. --- .../StaticAnalyzer/Checkers/StreamChecker.cpp | 91 ++++++++++++------- clang/test/Analysis/stream-error.c | 27 +++++- 2 files changed, 80 insertions(+), 38 deletions(-) diff --git a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp index 2ec47bf55df7..10972158f398 100644 --- a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp @@ -307,64 +307,64 @@ private: {{{"fclose"}, 1}, {&StreamChecker::preDefault, &StreamChecker::evalFclose, 0}}, {{{"fread"}, 4}, - {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, true), + {&StreamChecker::preRead, std::bind(&StreamChecker::evalFreadFwrite, _1, _2, _3, _4, true), 3}}, {{{"fwrite"}, 4}, - {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, false), + {&StreamChecker::preWrite, std::bind(&StreamChecker::evalFreadFwrite, _1, _2, _3, _4, false), 3}}, {{{"fgetc"}, 1}, - {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, true), + {&StreamChecker::preRead, std::bind(&StreamChecker::evalFgetx, _1, _2, _3, _4, true), 0}}, {{{"fgets"}, 3}, - {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, true), + {&StreamChecker::preRead, std::bind(&StreamChecker::evalFgetx, _1, _2, _3, _4, false), 2}}, {{{"getc"}, 1}, - {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, true), + {&StreamChecker::preRead, std::bind(&StreamChecker::evalFgetx, _1, _2, _3, _4, true), 0}}, {{{"fputc"}, 2}, - {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, false), + {&StreamChecker::preWrite, std::bind(&StreamChecker::evalFputx, _1, _2, _3, _4, true), 1}}, {{{"fputs"}, 2}, - {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, false), + {&StreamChecker::preWrite, std::bind(&StreamChecker::evalFputx, _1, _2, _3, _4, false), 1}}, {{{"putc"}, 2}, - {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, false), + {&StreamChecker::preWrite, std::bind(&StreamChecker::evalFputx, _1, _2, _3, _4, true), 1}}, {{{"fprintf"}}, - {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, false), + {&StreamChecker::preWrite, std::bind(&StreamChecker::evalFprintf, _1, _2, _3, _4), 0}}, {{{"vfprintf"}, 3}, - {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, false), + {&StreamChecker::preWrite, std::bind(&StreamChecker::evalFprintf, _1, _2, _3, _4), 0}}, {{{"fscanf"}}, - {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, true), + {&StreamChecker::preRead, std::bind(&StreamChecker::evalFscanf, _1, _2, _3, _4), 0}}, {{{"vfscanf"}, 3}, - {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, true), + {&StreamChecker::preRead, std::bind(&StreamChecker::evalFscanf, _1, _2, _3, _4), 0}}, {{{"ungetc"}, 2}, - {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, false), + {&StreamChecker::preWrite, std::bind(&StreamChecker::evalUngetc, _1, _2, _3, _4), 1}}, {{{"getdelim"}, 4}, - {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, true), + {&StreamChecker::preRead, std::bind(&StreamChecker::evalGetdelim, _1, _2, _3, _4), 3}}, {{{"getline"}, 3}, - {std::bind(&StreamChecker::preReadWrite, _1, _2, _3, _4, true), + {&StreamChecker::preRead, std::bind(&StreamChecker::evalGetdelim, _1, _2, _3, _4), 2}}, {{{"fseek"}, 3}, {&StreamChecker::preFseek, &StreamChecker::evalFseek, 0}}, {{{"fseeko"}, 3}, {&StreamChecker::preFseek, &StreamChecker::evalFseek, 0}}, {{{"ftell"}, 1}, - {&StreamChecker::preDefault, &StreamChecker::evalFtell, 0}}, + {&StreamChecker::preWrite, &StreamChecker::evalFtell, 0}}, {{{"ftello"}, 1}, - {&StreamChecker::preDefault, &StreamChecker::evalFtell, 0}}, + {&StreamChecker::preWrite, &StreamChecker::evalFtell, 0}}, {{{"fflush"}, 1}, {&StreamChecker::preFflush, &StreamChecker::evalFflush, 0}}, {{{"rewind"}, 1}, {&StreamChecker::preDefault, &StreamChecker::evalRewind, 0}}, {{{"fgetpos"}, 2}, - {&StreamChecker::preDefault, &StreamChecker::evalFgetpos, 0}}, + {&StreamChecker::preWrite, &StreamChecker::evalFgetpos, 0}}, {{{"fsetpos"}, 2}, {&StreamChecker::preDefault, &StreamChecker::evalFsetpos, 0}}, {{{"clearerr"}, 1}, @@ -384,12 +384,18 @@ private: CallDescriptionMap FnTestDescriptions = { {{{"StreamTesterChecker_make_feof_stream"}, 1}, {nullptr, - std::bind(&StreamChecker::evalSetFeofFerror, _1, _2, _3, _4, ErrorFEof), + std::bind(&StreamChecker::evalSetFeofFerror, _1, _2, _3, _4, ErrorFEof, + false), 0}}, {{{"StreamTesterChecker_make_ferror_stream"}, 1}, {nullptr, std::bind(&StreamChecker::evalSetFeofFerror, _1, _2, _3, _4, - ErrorFError), + ErrorFError, false), + 0}}, + {{{"StreamTesterChecker_make_ferror_indeterminate_stream"}, 1}, + {nullptr, + std::bind(&StreamChecker::evalSetFeofFerror, _1, _2, _3, _4, + ErrorFError, true), 0}}, }; @@ -415,8 +421,11 @@ private: void evalFclose(const FnDescription *Desc, const CallEvent &Call, CheckerContext &C) const; - void preReadWrite(const FnDescription *Desc, const CallEvent &Call, - CheckerContext &C, bool IsRead) const; + void preRead(const FnDescription *Desc, const CallEvent &Call, + CheckerContext &C) const; + + void preWrite(const FnDescription *Desc, const CallEvent &Call, + CheckerContext &C) const; void evalFreadFwrite(const FnDescription *Desc, const CallEvent &Call, CheckerContext &C, bool IsFread) const; @@ -467,8 +476,8 @@ private: const StreamErrorState &ErrorKind) const; void evalSetFeofFerror(const FnDescription *Desc, const CallEvent &Call, - CheckerContext &C, - const StreamErrorState &ErrorKind) const; + CheckerContext &C, const StreamErrorState &ErrorKind, + bool Indeterminate) const; void preFflush(const FnDescription *Desc, const CallEvent &Call, CheckerContext &C) const; @@ -849,9 +858,8 @@ void StreamChecker::evalFclose(const FnDescription *Desc, const CallEvent &Call, C.addTransition(E.bindReturnValue(State, C, *EofVal)); } -void StreamChecker::preReadWrite(const FnDescription *Desc, - const CallEvent &Call, CheckerContext &C, - bool IsRead) const { +void StreamChecker::preRead(const FnDescription *Desc, const CallEvent &Call, + CheckerContext &C) const { ProgramStateRef State = C.getState(); SVal StreamVal = getStreamArg(Desc, Call); State = ensureStreamNonNull(StreamVal, Call.getArgExpr(Desc->StreamArgNo), C, @@ -865,11 +873,6 @@ void StreamChecker::preReadWrite(const FnDescription *Desc, if (!State) return; - if (!IsRead) { - C.addTransition(State); - return; - } - SymbolRef Sym = StreamVal.getAsSymbol(); if (Sym && State->get(Sym)) { const StreamState *SS = State->get(Sym); @@ -880,6 +883,24 @@ void StreamChecker::preReadWrite(const FnDescription *Desc, } } +void StreamChecker::preWrite(const FnDescription *Desc, const CallEvent &Call, + CheckerContext &C) const { + ProgramStateRef State = C.getState(); + SVal StreamVal = getStreamArg(Desc, Call); + State = ensureStreamNonNull(StreamVal, Call.getArgExpr(Desc->StreamArgNo), C, + State); + if (!State) + return; + State = ensureStreamOpened(StreamVal, C, State); + if (!State) + return; + State = ensureNoFilePositionIndeterminate(StreamVal, C, State); + if (!State) + return; + + C.addTransition(State); +} + void StreamChecker::evalFreadFwrite(const FnDescription *Desc, const CallEvent &Call, CheckerContext &C, bool IsFread) const { @@ -1496,14 +1517,16 @@ void StreamChecker::preDefault(const FnDescription *Desc, const CallEvent &Call, void StreamChecker::evalSetFeofFerror(const FnDescription *Desc, const CallEvent &Call, CheckerContext &C, - const StreamErrorState &ErrorKind) const { + const StreamErrorState &ErrorKind, + bool Indeterminate) const { ProgramStateRef State = C.getState(); SymbolRef StreamSym = getStreamArg(Desc, Call).getAsSymbol(); assert(StreamSym && "Operation not permitted on non-symbolic stream value."); const StreamState *SS = State->get(StreamSym); assert(SS && "Stream should be tracked by the checker."); State = State->set( - StreamSym, StreamState::getOpened(SS->LastOperation, ErrorKind)); + StreamSym, + StreamState::getOpened(SS->LastOperation, ErrorKind, Indeterminate)); C.addTransition(State); } diff --git a/clang/test/Analysis/stream-error.c b/clang/test/Analysis/stream-error.c index ac31083bfc69..88f7de4234ff 100644 --- a/clang/test/Analysis/stream-error.c +++ b/clang/test/Analysis/stream-error.c @@ -11,6 +11,7 @@ void clang_analyzer_dump(int); void clang_analyzer_warnIfReached(void); void StreamTesterChecker_make_feof_stream(FILE *); void StreamTesterChecker_make_ferror_stream(FILE *); +void StreamTesterChecker_make_ferror_indeterminate_stream(FILE *); void error_fopen(void) { FILE *F = fopen("file", "r"); @@ -52,6 +53,8 @@ void stream_error_feof(void) { clearerr(F); clang_analyzer_eval(feof(F)); // expected-warning {{FALSE}} clang_analyzer_eval(ferror(F)); // expected-warning {{FALSE}} + StreamTesterChecker_make_ferror_indeterminate_stream(F); + clang_analyzer_eval(feof(F)); // expected-warning {{FALSE}} fclose(F); } @@ -65,6 +68,8 @@ void stream_error_ferror(void) { clearerr(F); clang_analyzer_eval(feof(F)); // expected-warning {{FALSE}} clang_analyzer_eval(ferror(F)); // expected-warning {{FALSE}} + StreamTesterChecker_make_ferror_indeterminate_stream(F); + clang_analyzer_eval(ferror(F)); // expected-warning {{TRUE}} fclose(F); } @@ -233,7 +238,7 @@ void error_fscanf(int *A) { fscanf(F, "ccc"); // expected-warning {{Stream might be already closed}} } -void error_ungetc() { +void error_ungetc(int TestIndeterminate) { FILE *F = tmpfile(); if (!F) return; @@ -245,8 +250,12 @@ void error_ungetc() { clang_analyzer_eval(Ret == 'X'); // expected-warning {{TRUE}} } fputc('Y', F); // no-warning + if (TestIndeterminate) { + StreamTesterChecker_make_ferror_indeterminate_stream(F); + ungetc('X', F); // expected-warning {{might be 'indeterminate'}} + } fclose(F); - ungetc('A', F); // expected-warning {{Stream might be already closed}} + ungetc('A', F); // expected-warning {{Stream might be already closed}} } void error_getdelim(char *P, size_t Sz) { @@ -449,7 +458,7 @@ void error_fseeko_0(void) { fclose(F); } -void error_ftell(void) { +void error_ftell(int TestIndeterminate) { FILE *F = fopen("file", "r"); if (!F) return; @@ -467,10 +476,14 @@ void error_ftell(void) { rc = ftell(F); clang_analyzer_eval(feof(F)); // expected-warning {{FALSE}} clang_analyzer_eval(ferror(F)); // expected-warning {{TRUE}} + if (TestIndeterminate) { + StreamTesterChecker_make_ferror_indeterminate_stream(F); + ftell(F); // expected-warning {{might be 'indeterminate'}} + } fclose(F); } -void error_ftello(void) { +void error_ftello(int TestIndeterminate) { FILE *F = fopen("file", "r"); if (!F) return; @@ -488,6 +501,10 @@ void error_ftello(void) { rc = ftello(F); clang_analyzer_eval(feof(F)); // expected-warning {{FALSE}} clang_analyzer_eval(ferror(F)); // expected-warning {{TRUE}} + if (TestIndeterminate) { + StreamTesterChecker_make_ferror_indeterminate_stream(F); + ftell(F); // expected-warning {{might be 'indeterminate'}} + } fclose(F); } @@ -506,6 +523,8 @@ void error_fileno(void) { N = fileno(F); clang_analyzer_eval(feof(F)); // expected-warning {{FALSE}} clang_analyzer_eval(ferror(F)); // expected-warning {{TRUE}} + StreamTesterChecker_make_ferror_indeterminate_stream(F); + fileno(F); // no warning fclose(F); } -- GitLab From 9f5be5f0092a636274953389cd5771c45ac0a568 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Fri, 8 Mar 2024 00:13:11 -0800 Subject: [PATCH 571/929] [Github] Add repository checks to release-binaries workflow (#84437) This patch adds repository checks to the release-binaries workflow jobs. People were observing that the job was running on a schedule in their forks. This only happens on old forks, but those probably exist in great number given how prolific LLVM is. This is also good practice anyways, on top of solving the direct problem of these jobs running with the cron schedule on people's forks. --- .github/workflows/release-binaries.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index ebf6fa41898d..e487b0f1d4b5 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -35,6 +35,7 @@ jobs: prepare: name: Prepare to build binaries runs-on: ubuntu-22.04 + if: github.repository == 'llvm/llvm-project' outputs: release-version: ${{ steps.vars.outputs.release-version }} flags: ${{ steps.vars.outputs.flags }} @@ -85,6 +86,7 @@ jobs: name: "Fill Cache ${{ matrix.os }}" needs: prepare runs-on: ${{ matrix.os }} + if: github.repository == 'llvm/llvm-project' strategy: matrix: os: @@ -119,6 +121,7 @@ jobs: - prepare - fill-cache runs-on: ${{ matrix.target.runs-on }} + if: github.repository == 'llvm/llvm-project' strategy: fail-fast: false matrix: -- GitLab From 0ddb1221471ae481cc0c7dfd57b71d98e7e60a27 Mon Sep 17 00:00:00 2001 From: Tina Jung <126699487+TinaAMD@users.noreply.github.com> Date: Fri, 8 Mar 2024 09:16:10 +0100 Subject: [PATCH 572/929] [mlir][emitc] Arith to EmitC conversion: constants (#83798) * Add a conversion from `arith.constant` to `emitc.constant`. * Drop the translation for `arith.constant`s. --- mlir/docs/Dialects/emitc.md | 2 - .../Conversion/ArithToEmitC/ArithToEmitC.cpp | 16 ++++++++ .../ArithToEmitC/ArithToEmitCPass.cpp | 1 - mlir/lib/Target/Cpp/CMakeLists.txt | 1 - mlir/lib/Target/Cpp/TranslateRegistration.cpp | 4 +- mlir/lib/Target/Cpp/TranslateToCpp.cpp | 12 ------ .../ArithToEmitC/arith-to-emitc.mlir | 24 ++++++++++- mlir/test/Target/Cpp/call.mlir | 2 +- mlir/test/Target/Cpp/const.mlir | 20 +++++++++ mlir/test/Target/Cpp/for.mlir | 10 ++--- mlir/test/Target/Cpp/if.mlir | 2 +- mlir/test/Target/Cpp/stdops.mlir | 41 ++----------------- .../llvm-project-overlay/mlir/BUILD.bazel | 1 - 13 files changed, 70 insertions(+), 66 deletions(-) diff --git a/mlir/docs/Dialects/emitc.md b/mlir/docs/Dialects/emitc.md index b227a8c4599a..1158bc683af0 100644 --- a/mlir/docs/Dialects/emitc.md +++ b/mlir/docs/Dialects/emitc.md @@ -30,5 +30,3 @@ translating the following operations: * `func.call` * `func.func` * `func.return` -* 'arith' Dialect - * `arith.constant` diff --git a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp index 6909534d4790..40dce001a3b2 100644 --- a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp +++ b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp @@ -24,6 +24,21 @@ using namespace mlir; //===----------------------------------------------------------------------===// namespace { +class ArithConstantOpConversionPattern + : public OpConversionPattern { +public: + using OpConversionPattern::OpConversionPattern; + + LogicalResult + matchAndRewrite(arith::ConstantOp arithConst, + arith::ConstantOp::Adaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + rewriter.replaceOpWithNewOp( + arithConst, arithConst.getType(), adaptor.getValue()); + return success(); + } +}; + template class ArithOpConversion final : public OpConversionPattern { public: @@ -51,6 +66,7 @@ void mlir::populateArithToEmitCPatterns(TypeConverter &typeConverter, // clang-format off patterns.add< + ArithConstantOpConversionPattern, ArithOpConversion, ArithOpConversion, ArithOpConversion, diff --git a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitCPass.cpp b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitCPass.cpp index b377c063a7aa..45a088ed144f 100644 --- a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitCPass.cpp +++ b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitCPass.cpp @@ -38,7 +38,6 @@ void ConvertArithToEmitC::runOnOperation() { target.addLegalDialect(); target.addIllegalDialect(); - target.addLegalOp(); RewritePatternSet patterns(&getContext()); diff --git a/mlir/lib/Target/Cpp/CMakeLists.txt b/mlir/lib/Target/Cpp/CMakeLists.txt index 5521e7909a8a..d8f372cf1624 100644 --- a/mlir/lib/Target/Cpp/CMakeLists.txt +++ b/mlir/lib/Target/Cpp/CMakeLists.txt @@ -6,7 +6,6 @@ add_mlir_translation_library(MLIRTargetCpp ${EMITC_MAIN_INCLUDE_DIR}/emitc/Target/Cpp LINK_LIBS PUBLIC - MLIRArithDialect MLIRControlFlowDialect MLIREmitCDialect MLIRFuncDialect diff --git a/mlir/lib/Target/Cpp/TranslateRegistration.cpp b/mlir/lib/Target/Cpp/TranslateRegistration.cpp index b486e5429ea6..4104b177d7d9 100644 --- a/mlir/lib/Target/Cpp/TranslateRegistration.cpp +++ b/mlir/lib/Target/Cpp/TranslateRegistration.cpp @@ -6,7 +6,6 @@ // //===----------------------------------------------------------------------===// -#include "mlir/Dialect/Arith/IR/Arith.h" #include "mlir/Dialect/ControlFlow/IR/ControlFlow.h" #include "mlir/Dialect/EmitC/IR/EmitC.h" #include "mlir/Dialect/Func/IR/FuncOps.h" @@ -41,8 +40,7 @@ void registerToCppTranslation() { }, [](DialectRegistry ®istry) { // clang-format off - registry.insert(assignOp.getVar().getDefiningOp()); @@ -1391,9 +1382,6 @@ LogicalResult CppEmitter::emitOperation(Operation &op, bool trailingSemicolon) { // Func ops. .Case( [&](auto op) { return printOperation(*this, op); }) - // Arithmetic ops. - .Case( - [&](auto op) { return printOperation(*this, op); }) .Case([&](auto op) { return success(); }) .Default([&](Operation *) { return op.emitOpError("unable to find printer for op"); diff --git a/mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir b/mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir index 6a56474a5c48..2886810c01e9 100644 --- a/mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir +++ b/mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir @@ -1,4 +1,26 @@ -// RUN: mlir-opt -convert-arith-to-emitc %s | FileCheck %s +// RUN: mlir-opt -split-input-file -convert-arith-to-emitc %s | FileCheck %s + +// CHECK-LABEL: arith_constants +func.func @arith_constants() { + // CHECK: emitc.constant + // CHECK-SAME: value = 0 : index + %c_index = arith.constant 0 : index + // CHECK: emitc.constant + // CHECK-SAME: value = 0 : i32 + %c_signless_int_32 = arith.constant 0 : i32 + // CHECK: emitc.constant + // CHECK-SAME: value = 0.{{0+}}e+00 : f32 + %c_float_32 = arith.constant 0.0 : f32 + // CHECK: emitc.constant + // CHECK-SAME: value = dense<0> : tensor + %c_tensor_single_value = arith.constant dense<0> : tensor + // CHECK: emitc.constant + // CHECK-SAME: value{{.*}}[1, 2], [-3, 9], [0, 0], [2, -1]{{.*}}tensor<4x2xi64> + %c_tensor_value = arith.constant dense<[[1, 2], [-3, 9], [0, 0], [2, -1]]> : tensor<4x2xi64> + return +} + +// ----- func.func @arith_ops(%arg0: f32, %arg1: f32) { // CHECK: [[V0:[^ ]*]] = emitc.add %arg0, %arg1 : (f32, f32) -> f32 diff --git a/mlir/test/Target/Cpp/call.mlir b/mlir/test/Target/Cpp/call.mlir index 2bcdc8720518..e3ac392f30b6 100644 --- a/mlir/test/Target/Cpp/call.mlir +++ b/mlir/test/Target/Cpp/call.mlir @@ -18,7 +18,7 @@ func.func @emitc_call_opaque() { func.func @emitc_call_opaque_two_results() { - %0 = arith.constant 0 : index + %0 = "emitc.constant"() <{value = 0 : index}> : () -> index %1:2 = emitc.call_opaque "two_results" () : () -> (i32, i32) return } diff --git a/mlir/test/Target/Cpp/const.mlir b/mlir/test/Target/Cpp/const.mlir index 28a547909a0a..524d564b3b94 100644 --- a/mlir/test/Target/Cpp/const.mlir +++ b/mlir/test/Target/Cpp/const.mlir @@ -8,6 +8,11 @@ func.func @emitc_constant() { %c3 = "emitc.constant"(){value = -1 : si8} : () -> si8 %c4 = "emitc.constant"(){value = 255 : ui8} : () -> ui8 %c5 = "emitc.constant"(){value = #emitc.opaque<"CHAR_MIN">} : () -> !emitc.opaque<"char"> + %c6 = "emitc.constant"(){value = 2 : index} : () -> index + %c7 = "emitc.constant"(){value = 2.0 : f32} : () -> f32 + %c8 = "emitc.constant"(){value = dense<0> : tensor} : () -> tensor + %c9 = "emitc.constant"(){value = dense<[0, 1]> : tensor<2xindex>} : () -> tensor<2xindex> + %c10 = "emitc.constant"(){value = dense<[[0.0, 1.0], [2.0, 3.0]]> : tensor<2x2xf32>} : () -> tensor<2x2xf32> return } // CPP-DEFAULT: void emitc_constant() { @@ -17,6 +22,11 @@ func.func @emitc_constant() { // CPP-DEFAULT-NEXT: int8_t [[V3:[^ ]*]] = -1; // CPP-DEFAULT-NEXT: uint8_t [[V4:[^ ]*]] = 255; // CPP-DEFAULT-NEXT: char [[V5:[^ ]*]] = CHAR_MIN; +// CPP-DEFAULT-NEXT: size_t [[V6:[^ ]*]] = 2; +// CPP-DEFAULT-NEXT: float [[V7:[^ ]*]] = (float)2.000000000e+00; +// CPP-DEFAULT-NEXT: Tensor [[V8:[^ ]*]] = {0}; +// CPP-DEFAULT-NEXT: Tensor [[V9:[^ ]*]] = {0, 1}; +// CPP-DEFAULT-NEXT: Tensor [[V10:[^ ]*]] = {(float)0.0e+00, (float)1.000000000e+00, (float)2.000000000e+00, (float)3.000000000e+00}; // CPP-DECLTOP: void emitc_constant() { // CPP-DECLTOP-NEXT: int32_t [[V0:[^ ]*]]; @@ -25,9 +35,19 @@ func.func @emitc_constant() { // CPP-DECLTOP-NEXT: int8_t [[V3:[^ ]*]]; // CPP-DECLTOP-NEXT: uint8_t [[V4:[^ ]*]]; // CPP-DECLTOP-NEXT: char [[V5:[^ ]*]]; +// CPP-DECLTOP-NEXT: size_t [[V6:[^ ]*]]; +// CPP-DECLTOP-NEXT: float [[V7:[^ ]*]]; +// CPP-DECLTOP-NEXT: Tensor [[V8:[^ ]*]]; +// CPP-DECLTOP-NEXT: Tensor [[V9:[^ ]*]]; +// CPP-DECLTOP-NEXT: Tensor [[V10:[^ ]*]]; // CPP-DECLTOP-NEXT: [[V0]] = INT_MAX; // CPP-DECLTOP-NEXT: [[V1]] = 42; // CPP-DECLTOP-NEXT: [[V2]] = -1; // CPP-DECLTOP-NEXT: [[V3]] = -1; // CPP-DECLTOP-NEXT: [[V4]] = 255; // CPP-DECLTOP-NEXT: [[V5]] = CHAR_MIN; +// CPP-DECLTOP-NEXT: [[V6]] = 2; +// CPP-DECLTOP-NEXT: [[V7]] = (float)2.000000000e+00; +// CPP-DECLTOP-NEXT: [[V8]] = {0}; +// CPP-DECLTOP-NEXT: [[V9]] = {0, 1}; +// CPP-DECLTOP-NEXT: [[V10]] = {(float)0.0e+00, (float)1.000000000e+00, (float)2.000000000e+00, (float)3.000000000e+00}; diff --git a/mlir/test/Target/Cpp/for.mlir b/mlir/test/Target/Cpp/for.mlir index b9bd3d98465a..5225f3ddaff2 100644 --- a/mlir/test/Target/Cpp/for.mlir +++ b/mlir/test/Target/Cpp/for.mlir @@ -33,12 +33,12 @@ func.func @test_for(%arg0 : index, %arg1 : index, %arg2 : index) { // CPP-DECLTOP-NEXT: return; func.func @test_for_yield() { - %start = arith.constant 0 : index - %stop = arith.constant 10 : index - %step = arith.constant 1 : index + %start = "emitc.constant"() <{value = 0 : index}> : () -> index + %stop = "emitc.constant"() <{value = 10 : index}> : () -> index + %step = "emitc.constant"() <{value = 1 : index}> : () -> index - %s0 = arith.constant 0 : i32 - %p0 = arith.constant 1.0 : f32 + %s0 = "emitc.constant"() <{value = 0 : i32}> : () -> i32 + %p0 = "emitc.constant"() <{value = 1.0 : f32}> : () -> f32 %0 = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> i32 %1 = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> f32 diff --git a/mlir/test/Target/Cpp/if.mlir b/mlir/test/Target/Cpp/if.mlir index 743f8ad39688..7b0e2da85d0e 100644 --- a/mlir/test/Target/Cpp/if.mlir +++ b/mlir/test/Target/Cpp/if.mlir @@ -49,7 +49,7 @@ func.func @test_if_else(%arg0: i1, %arg1: f32) { func.func @test_if_yield(%arg0: i1, %arg1: f32) { - %0 = arith.constant 0 : i8 + %0 = "emitc.constant"() <{value = 0 : i8}> : () -> i8 %x = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> i32 %y = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> f64 emitc.if %arg0 { diff --git a/mlir/test/Target/Cpp/stdops.mlir b/mlir/test/Target/Cpp/stdops.mlir index 0723188a62c6..cc6bdbe37698 100644 --- a/mlir/test/Target/Cpp/stdops.mlir +++ b/mlir/test/Target/Cpp/stdops.mlir @@ -1,37 +1,6 @@ // RUN: mlir-translate -mlir-to-cpp %s | FileCheck %s -check-prefix=CPP-DEFAULT // RUN: mlir-translate -mlir-to-cpp -declare-variables-at-top %s | FileCheck %s -check-prefix=CPP-DECLTOP -func.func @std_constant() { - %c0 = arith.constant 0 : i32 - %c1 = arith.constant 2 : index - %c2 = arith.constant 2.0 : f32 - %c3 = arith.constant dense<0> : tensor - %c4 = arith.constant dense<[0, 1]> : tensor<2xindex> - %c5 = arith.constant dense<[[0.0, 1.0], [2.0, 3.0]]> : tensor<2x2xf32> - return -} -// CPP-DEFAULT: void std_constant() { -// CPP-DEFAULT-NEXT: int32_t [[V0:[^ ]*]] = 0; -// CPP-DEFAULT-NEXT: size_t [[V1:[^ ]*]] = 2; -// CPP-DEFAULT-NEXT: float [[V2:[^ ]*]] = (float)2.000000000e+00; -// CPP-DEFAULT-NEXT: Tensor [[V3:[^ ]*]] = {0}; -// CPP-DEFAULT-NEXT: Tensor [[V4:[^ ]*]] = {0, 1}; -// CPP-DEFAULT-NEXT: Tensor [[V5:[^ ]*]] = {(float)0.0e+00, (float)1.000000000e+00, (float)2.000000000e+00, (float)3.000000000e+00}; - -// CPP-DECLTOP: void std_constant() { -// CPP-DECLTOP-NEXT: int32_t [[V0:[^ ]*]]; -// CPP-DECLTOP-NEXT: size_t [[V1:[^ ]*]]; -// CPP-DECLTOP-NEXT: float [[V2:[^ ]*]]; -// CPP-DECLTOP-NEXT: Tensor [[V3:[^ ]*]]; -// CPP-DECLTOP-NEXT: Tensor [[V4:[^ ]*]]; -// CPP-DECLTOP-NEXT: Tensor [[V5:[^ ]*]]; -// CPP-DECLTOP-NEXT: [[V0]] = 0; -// CPP-DECLTOP-NEXT: [[V1]] = 2; -// CPP-DECLTOP-NEXT: [[V2]] = (float)2.000000000e+00; -// CPP-DECLTOP-NEXT: [[V3]] = {0}; -// CPP-DECLTOP-NEXT: [[V4]] = {0, 1}; -// CPP-DECLTOP-NEXT: [[V5]] = {(float)0.0e+00, (float)1.000000000e+00, (float)2.000000000e+00, (float)3.000000000e+00}; - func.func @std_call() { %0 = call @one_result () : () -> i32 %1 = call @one_result () : () -> i32 @@ -49,13 +18,11 @@ func.func @std_call() { func.func @std_call_two_results() { - %c = arith.constant 0 : i8 %0:2 = call @two_results () : () -> (i32, f32) %1:2 = call @two_results () : () -> (i32, f32) return } // CPP-DEFAULT: void std_call_two_results() { -// CPP-DEFAULT-NEXT: int8_t [[V0:[^ ]*]] = 0; // CPP-DEFAULT-NEXT: int32_t [[V1:[^ ]*]]; // CPP-DEFAULT-NEXT: float [[V2:[^ ]*]]; // CPP-DEFAULT-NEXT: std::tie([[V1]], [[V2]]) = two_results(); @@ -64,18 +31,16 @@ func.func @std_call_two_results() { // CPP-DEFAULT-NEXT: std::tie([[V3]], [[V4]]) = two_results(); // CPP-DECLTOP: void std_call_two_results() { -// CPP-DECLTOP-NEXT: int8_t [[V0:[^ ]*]]; // CPP-DECLTOP-NEXT: int32_t [[V1:[^ ]*]]; // CPP-DECLTOP-NEXT: float [[V2:[^ ]*]]; // CPP-DECLTOP-NEXT: int32_t [[V3:[^ ]*]]; // CPP-DECLTOP-NEXT: float [[V4:[^ ]*]]; -// CPP-DECLTOP-NEXT: [[V0]] = 0; // CPP-DECLTOP-NEXT: std::tie([[V1]], [[V2]]) = two_results(); // CPP-DECLTOP-NEXT: std::tie([[V3]], [[V4]]) = two_results(); func.func @one_result() -> i32 { - %0 = arith.constant 0 : i32 + %0 = "emitc.constant"() <{value = 0 : i32}> : () -> i32 return %0 : i32 } // CPP-DEFAULT: int32_t one_result() { @@ -89,8 +54,8 @@ func.func @one_result() -> i32 { func.func @two_results() -> (i32, f32) { - %0 = arith.constant 0 : i32 - %1 = arith.constant 1.0 : f32 + %0 = "emitc.constant"() <{value = 0 : i32}> : () -> i32 + %1 = "emitc.constant"() <{value = 1.0 : f32}> : () -> f32 return %0, %1 : i32, f32 } // CPP-DEFAULT: std::tuple two_results() { diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 2cfe61844703..a2f170222684 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -1769,7 +1769,6 @@ cc_library( ]), hdrs = glob(["include/mlir/Target/Cpp/*.h"]), deps = [ - ":ArithDialect", ":ControlFlowDialect", ":EmitCDialect", ":FuncDialect", -- GitLab From 4b1910b11d30d2382cb969dc09aa3456659d42fb Mon Sep 17 00:00:00 2001 From: Pierre van Houtryve Date: Fri, 8 Mar 2024 09:39:10 +0100 Subject: [PATCH 573/929] [GlobalISel][AMDGPU] Import patterns with multiple defs (#84171) Fixes #63216 --- llvm/lib/Target/AMDGPU/VOP3Instructions.td | 23 +- .../AMDGPU/GlobalISel/mul-known-bits.i64.ll | 162 ++--- llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll | 235 ++++--- .../test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll | 592 +++++++++--------- .../test/CodeGen/AMDGPU/GlobalISel/udivrem.ll | 553 ++++++++-------- .../CodeGen/AMDGPU/integer-mad-patterns.ll | 305 +++++---- ...lobalISelEmitter-multiple-output-reject.td | 14 - .../GlobalISelEmitter-multiple-output.td | 30 + llvm/utils/TableGen/GlobalISelEmitter.cpp | 27 +- 9 files changed, 983 insertions(+), 958 deletions(-) delete mode 100644 llvm/test/TableGen/GlobalISelEmitter-multiple-output-reject.td diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td index 3340ded9d360..647595d9ccab 100644 --- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td +++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td @@ -728,25 +728,34 @@ def : OpSelBinOpClampPat; def : OpSelBinOpClampPat; } // End SubtargetPredicate = isGFX9Plus -// FIXME: GlobalISel in general does not handle instructions with 2 results, -// so it cannot use these patterns. multiclass IMAD32_Pats { def : GCNPat < (ThreeOpFrag i32:$src0, i32:$src1, i32:$src2), - (EXTRACT_SUBREG (inst $src0, $src1, + (EXTRACT_SUBREG (inst i32:$src0, i32:$src1, (REG_SEQUENCE SReg_64, // Use scalar and let it be legalized $src2, sub0, (i32 (IMPLICIT_DEF)), sub1), 0 /* clamp */), sub0) >; + + // GISel-specific pattern that avoids creating a SGPR->VGPR copy if + // $src2 is a VGPR. + def : GCNPat < + (ThreeOpFrag i32:$src0, i32:$src1, VGPR_32:$src2), + (EXTRACT_SUBREG (inst i32:$src0, i32:$src1, + (REG_SEQUENCE VReg_64, + $src2, sub0, + (i32 (IMPLICIT_DEF)), sub1), + 0 /* clamp */), + sub0) + >; + // Immediate src2 in the pattern above will not fold because it would be partially // undef. Hence define specialized pattern for this case. - // FIXME: GlobalISel pattern exporter fails to export a pattern like this and asserts, - // make it SDAG only. def : GCNPat < - (ThreeOpFragSDAG i32:$src0, i32:$src1, (i32 imm:$src2)), - (EXTRACT_SUBREG (inst $src0, $src1, (i64 (as_i64imm $src2)), 0 /* clamp */), sub0) + (ThreeOpFrag i32:$src0, i32:$src1, (i32 imm:$src2)), + (EXTRACT_SUBREG (inst i32:$src0, i32:$src1, (i64 (as_i64imm $src2)), 0 /* clamp */), sub0) >; } diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul-known-bits.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul-known-bits.i64.ll index d671a1d87b63..1140ef88ac7f 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul-known-bits.i64.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul-known-bits.i64.ll @@ -8,34 +8,35 @@ define amdgpu_kernel void @v_mul_i64_no_zext(ptr addrspace(1) %out, ptr addrspac ; GFX10-LABEL: v_mul_i64_no_zext: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x2c -; GFX10-NEXT: v_lshlrev_b32_e32 v6, 3, v0 +; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_clause 0x1 -; GFX10-NEXT: global_load_dwordx2 v[0:1], v6, s[0:1] -; GFX10-NEXT: global_load_dwordx2 v[2:3], v6, s[2:3] +; GFX10-NEXT: global_load_dwordx2 v[0:1], v7, s[0:1] +; GFX10-NEXT: global_load_dwordx2 v[2:3], v7, s[2:3] ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mad_u64_u32 v[4:5], s0, v0, v2, 0 -; GFX10-NEXT: v_mul_lo_u32 v0, v0, v3 -; GFX10-NEXT: v_mul_lo_u32 v1, v1, v2 -; GFX10-NEXT: v_add3_u32 v5, v5, v0, v1 -; GFX10-NEXT: global_store_dwordx2 v6, v[4:5], s[2:3] +; GFX10-NEXT: v_mad_u64_u32 v[5:6], s0, v0, v3, v[5:6] +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s0, v1, v2, v[5:6] +; GFX10-NEXT: v_mov_b32_e32 v5, v0 +; GFX10-NEXT: global_store_dwordx2 v7, v[4:5], s[2:3] ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: v_mul_i64_no_zext: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x2c -; GFX11-NEXT: v_lshlrev_b32_e32 v6, 3, v0 +; GFX11-NEXT: v_lshlrev_b32_e32 v9, 3, v0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_clause 0x1 -; GFX11-NEXT: global_load_b64 v[0:1], v6, s[0:1] -; GFX11-NEXT: global_load_b64 v[2:3], v6, s[2:3] +; GFX11-NEXT: global_load_b64 v[0:1], v9, s[0:1] +; GFX11-NEXT: global_load_b64 v[2:3], v9, s[2:3] ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_mad_u64_u32 v[4:5], null, v0, v2, 0 -; GFX11-NEXT: v_mul_lo_u32 v0, v0, v3 -; GFX11-NEXT: v_mul_lo_u32 v1, v1, v2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_mad_u64_u32 v[6:7], null, v0, v3, v[5:6] +; GFX11-NEXT: v_mad_u64_u32 v[7:8], null, v1, v2, v[6:7] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_add3_u32 v5, v5, v0, v1 -; GFX11-NEXT: global_store_b64 v6, v[4:5], s[2:3] +; GFX11-NEXT: v_mov_b32_e32 v5, v7 +; GFX11-NEXT: global_store_b64 v9, v[4:5], s[2:3] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_endpgm @@ -64,8 +65,9 @@ define amdgpu_kernel void @v_mul_i64_zext_src1(ptr addrspace(1) %out, ptr addrsp ; GFX10-NEXT: global_load_dword v4, v3, s[2:3] ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mad_u64_u32 v[2:3], s0, v0, v4, 0 -; GFX10-NEXT: v_mul_lo_u32 v0, v1, v4 -; GFX10-NEXT: v_add_nc_u32_e32 v3, v3, v0 +; GFX10-NEXT: v_mov_b32_e32 v0, v3 +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s0, v1, v4, v[0:1] +; GFX10-NEXT: v_mov_b32_e32 v3, v0 ; GFX10-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-NEXT: global_store_dwordx2 v0, v[2:3], s[4:5] ; GFX10-NEXT: s_endpgm @@ -79,12 +81,13 @@ define amdgpu_kernel void @v_mul_i64_zext_src1(ptr addrspace(1) %out, ptr addrsp ; GFX11-NEXT: v_lshlrev_b32_e32 v2, 2, v0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[0:1], v1, s[6:7] -; GFX11-NEXT: global_load_b32 v4, v2, s[0:1] +; GFX11-NEXT: global_load_b32 v5, v2, s[0:1] ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_mad_u64_u32 v[2:3], null, v0, v4, 0 -; GFX11-NEXT: v_mul_lo_u32 v0, v1, v4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_add_nc_u32 v3, v3, v0 +; GFX11-NEXT: v_mad_u64_u32 v[2:3], null, v0, v5, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v0, v3 +; GFX11-NEXT: v_mad_u64_u32 v[3:4], null, v1, v5, v[0:1] +; GFX11-NEXT: v_mov_b32_e32 v0, 0 ; GFX11-NEXT: global_store_b64 v0, v[2:3], s[4:5] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -114,8 +117,9 @@ define amdgpu_kernel void @v_mul_i64_zext_src0(ptr addrspace(1) %out, ptr addrsp ; GFX10-NEXT: global_load_dwordx2 v[0:1], v3, s[2:3] ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mad_u64_u32 v[2:3], s0, v4, v0, 0 -; GFX10-NEXT: v_mul_lo_u32 v0, v4, v1 -; GFX10-NEXT: v_add_nc_u32_e32 v3, v3, v0 +; GFX10-NEXT: v_mov_b32_e32 v0, v3 +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s0, v4, v1, v[0:1] +; GFX10-NEXT: v_mov_b32_e32 v3, v0 ; GFX10-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-NEXT: global_store_dwordx2 v0, v[2:3], s[4:5] ; GFX10-NEXT: s_endpgm @@ -128,13 +132,14 @@ define amdgpu_kernel void @v_mul_i64_zext_src0(ptr addrspace(1) %out, ptr addrsp ; GFX11-NEXT: v_lshlrev_b32_e32 v1, 2, v0 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, 3, v0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: global_load_b32 v4, v1, s[6:7] +; GFX11-NEXT: global_load_b32 v5, v1, s[6:7] ; GFX11-NEXT: global_load_b64 v[0:1], v0, s[0:1] ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_mad_u64_u32 v[2:3], null, v4, v0, 0 -; GFX11-NEXT: v_mul_lo_u32 v0, v4, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_add_nc_u32 v3, v3, v0 +; GFX11-NEXT: v_mad_u64_u32 v[2:3], null, v5, v0, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v0, v3 +; GFX11-NEXT: v_mad_u64_u32 v[3:4], null, v5, v1, v[0:1] +; GFX11-NEXT: v_mov_b32_e32 v0, 0 ; GFX11-NEXT: global_store_b64 v0, v[2:3], s[4:5] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -211,8 +216,9 @@ define amdgpu_kernel void @v_mul_i64_masked_src0_hi(ptr addrspace(1) %out, ptr a ; GFX10-NEXT: global_load_dwordx2 v[0:1], v2, s[2:3] ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mad_u64_u32 v[2:3], s0, v4, v0, 0 -; GFX10-NEXT: v_mul_lo_u32 v0, v4, v1 -; GFX10-NEXT: v_add_nc_u32_e32 v3, v3, v0 +; GFX10-NEXT: v_mov_b32_e32 v0, v3 +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s0, v4, v1, v[0:1] +; GFX10-NEXT: v_mov_b32_e32 v3, v0 ; GFX10-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-NEXT: global_store_dwordx2 v0, v[2:3], s[4:5] ; GFX10-NEXT: s_endpgm @@ -225,13 +231,14 @@ define amdgpu_kernel void @v_mul_i64_masked_src0_hi(ptr addrspace(1) %out, ptr a ; GFX11-NEXT: v_lshlrev_b32_e32 v0, 3, v0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_clause 0x1 -; GFX11-NEXT: global_load_b32 v4, v0, s[6:7] +; GFX11-NEXT: global_load_b32 v5, v0, s[6:7] ; GFX11-NEXT: global_load_b64 v[0:1], v0, s[0:1] ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_mad_u64_u32 v[2:3], null, v4, v0, 0 -; GFX11-NEXT: v_mul_lo_u32 v0, v4, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_add_nc_u32 v3, v3, v0 +; GFX11-NEXT: v_mad_u64_u32 v[2:3], null, v5, v0, 0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v0, v3 +; GFX11-NEXT: v_mad_u64_u32 v[3:4], null, v5, v1, v[0:1] +; GFX11-NEXT: v_mov_b32_e32 v0, 0 ; GFX11-NEXT: global_store_b64 v0, v[2:3], s[4:5] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -390,15 +397,16 @@ define amdgpu_kernel void @v_mul_i64_partially_masked_src0(ptr addrspace(1) %out ; GFX10-NEXT: global_load_dwordx2 v[0:1], v4, s[6:7] ; GFX10-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3] ; GFX10-NEXT: s_waitcnt vmcnt(1) -; GFX10-NEXT: v_and_b32_e32 v4, 0xfff00000, v0 -; GFX10-NEXT: v_and_b32_e32 v5, 0xf00f, v1 +; GFX10-NEXT: v_and_b32_e32 v6, 0xfff00000, v0 ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_mad_u64_u32 v[0:1], s0, v4, v2, 0 -; GFX10-NEXT: v_mul_lo_u32 v3, v4, v3 -; GFX10-NEXT: v_mul_lo_u32 v2, v5, v2 -; GFX10-NEXT: v_add3_u32 v1, v1, v3, v2 -; GFX10-NEXT: v_mov_b32_e32 v2, 0 -; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5] +; GFX10-NEXT: v_mad_u64_u32 v[4:5], s0, v6, v2, 0 +; GFX10-NEXT: v_mov_b32_e32 v0, v5 +; GFX10-NEXT: v_mad_u64_u32 v[5:6], s0, v6, v3, v[0:1] +; GFX10-NEXT: v_and_b32_e32 v0, 0xf00f, v1 +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s0, v0, v2, v[5:6] +; GFX10-NEXT: v_mov_b32_e32 v5, v0 +; GFX10-NEXT: v_mov_b32_e32 v0, 0 +; GFX10-NEXT: global_store_dwordx2 v0, v[4:5], s[4:5] ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: v_mul_i64_partially_masked_src0: @@ -412,17 +420,18 @@ define amdgpu_kernel void @v_mul_i64_partially_masked_src0(ptr addrspace(1) %out ; GFX11-NEXT: global_load_b64 v[0:1], v2, s[6:7] ; GFX11-NEXT: global_load_b64 v[2:3], v2, s[0:1] ; GFX11-NEXT: s_waitcnt vmcnt(1) -; GFX11-NEXT: v_and_b32_e32 v4, 0xfff00000, v0 -; GFX11-NEXT: v_and_b32_e32 v5, 0xf00f, v1 +; GFX11-NEXT: v_and_b32_e32 v7, 0xfff00000, v0 ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v4, v2, 0 -; GFX11-NEXT: v_mul_lo_u32 v3, v4, v3 -; GFX11-NEXT: v_mul_lo_u32 v2, v5, v2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_mad_u64_u32 v[4:5], null, v7, v2, 0 +; GFX11-NEXT: v_mov_b32_e32 v0, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_mad_u64_u32 v[5:6], null, v7, v3, v[0:1] +; GFX11-NEXT: v_and_b32_e32 v3, 0xf00f, v1 +; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v3, v2, v[5:6] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_add3_u32 v1, v1, v3, v2 -; GFX11-NEXT: v_mov_b32_e32 v2, 0 -; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5] +; GFX11-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v0, 0 +; GFX11-NEXT: global_store_b64 v0, v[4:5], s[4:5] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_endpgm @@ -491,27 +500,31 @@ define amdgpu_kernel void @v_mul64_masked_before_and_in_branch(ptr addrspace(1) ; GFX10-NEXT: s_clause 0x1 ; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 -; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v0 +; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_clause 0x1 -; GFX10-NEXT: global_load_dwordx2 v[2:3], v4, s[6:7] -; GFX10-NEXT: global_load_dwordx2 v[0:1], v4, s[2:3] +; GFX10-NEXT: global_load_dwordx2 v[2:3], v0, s[6:7] +; GFX10-NEXT: global_load_dwordx2 v[4:5], v0, s[2:3] +; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GFX10-NEXT: s_waitcnt vmcnt(1) ; GFX10-NEXT: v_cmp_ge_u64_e32 vcc_lo, 0, v[2:3] -; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_mul_lo_u32 v1, v2, v1 ; GFX10-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX10-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX10-NEXT: s_cbranch_execz .LBB10_2 ; GFX10-NEXT: ; %bb.1: ; %else -; GFX10-NEXT: v_mad_u64_u32 v[2:3], s1, v2, v0, 0 -; GFX10-NEXT: v_add_nc_u32_e32 v3, v3, v1 -; GFX10-NEXT: v_mov_b32_e32 v0, v2 -; GFX10-NEXT: v_mov_b32_e32 v1, v3 -; GFX10-NEXT: ; %bb.2: ; %Flow +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s1, v2, v4, 0 +; GFX10-NEXT: v_mad_u64_u32 v[1:2], s1, v2, v5, v[1:2] +; GFX10-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX10-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX10-NEXT: .LBB10_2: ; %Flow ; GFX10-NEXT: s_andn2_saveexec_b32 s0, s0 +; GFX10-NEXT: s_cbranch_execz .LBB10_4 ; GFX10-NEXT: ; %bb.3: ; %if +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: v_mul_lo_u32 v1, v2, v5 ; GFX10-NEXT: v_mov_b32_e32 v0, 0 -; GFX10-NEXT: ; %bb.4: ; %endif +; GFX10-NEXT: .LBB10_4: ; %endif ; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX10-NEXT: v_mov_b32_e32 v2, 0 ; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5] @@ -526,22 +539,29 @@ define amdgpu_kernel void @v_mul64_masked_before_and_in_branch(ptr addrspace(1) ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_clause 0x1 ; GFX11-NEXT: global_load_b64 v[2:3], v0, s[6:7] -; GFX11-NEXT: global_load_b64 v[0:1], v0, s[0:1] +; GFX11-NEXT: global_load_b64 v[4:5], v0, s[0:1] ; GFX11-NEXT: s_mov_b32 s0, exec_lo -; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_mul_lo_u32 v1, v2, v1 +; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1 +; GFX11-NEXT: s_waitcnt vmcnt(1) ; GFX11-NEXT: v_cmpx_ge_u64_e32 0, v[2:3] ; GFX11-NEXT: s_xor_b32 s0, exec_lo, s0 +; GFX11-NEXT: s_cbranch_execz .LBB10_2 ; GFX11-NEXT: ; %bb.1: ; %else -; GFX11-NEXT: v_mad_u64_u32 v[2:3], null, v2, v0, 0 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v2, v4, 0 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_add_nc_u32_e32 v3, v3, v1 -; GFX11-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3 -; GFX11-NEXT: ; %bb.2: ; %Flow +; GFX11-NEXT: v_mad_u64_u32 v[3:4], null, v2, v5, v[1:2] +; GFX11-NEXT: ; implicit-def: $vgpr4_vgpr5 +; GFX11-NEXT: v_mov_b32_e32 v1, v3 +; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3 +; GFX11-NEXT: .LBB10_2: ; %Flow ; GFX11-NEXT: s_and_not1_saveexec_b32 s0, s0 +; GFX11-NEXT: s_cbranch_execz .LBB10_4 ; GFX11-NEXT: ; %bb.3: ; %if +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_mul_lo_u32 v1, v2, v5 ; GFX11-NEXT: v_mov_b32_e32 v0, 0 -; GFX11-NEXT: ; %bb.4: ; %endif +; GFX11-NEXT: .LBB10_4: ; %endif ; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0 ; GFX11-NEXT: v_mov_b32_e32 v2, 0 ; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5] diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll index 4c1935d06517..2d81452f9ef3 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll @@ -483,19 +483,18 @@ define i64 @v_mul_i64(i64 %num, i64 %den) { ; GFX10-NEXT: v_mov_b32_e32 v4, v0 ; GFX10-NEXT: v_mov_b32_e32 v5, v1 ; GFX10-NEXT: v_mad_u64_u32 v[0:1], s4, v4, v2, 0 -; GFX10-NEXT: v_mul_lo_u32 v3, v4, v3 -; GFX10-NEXT: v_mul_lo_u32 v2, v5, v2 -; GFX10-NEXT: v_add3_u32 v1, v1, v3, v2 +; GFX10-NEXT: v_mad_u64_u32 v[3:4], s4, v4, v3, v[1:2] +; GFX10-NEXT: v_mad_u64_u32 v[1:2], s4, v5, v2, v[3:4] ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_mul_i64: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, v1 -; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v4, v2, 0 -; GFX11-NEXT: v_mul_lo_u32 v3, v4, v3 -; GFX11-NEXT: v_mul_lo_u32 v2, v5, v2 -; GFX11-NEXT: v_add3_u32 v1, v1, v3, v2 +; GFX11-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, v2 +; GFX11-NEXT: v_mov_b32_e32 v6, v1 +; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v4, v5, 0 +; GFX11-NEXT: v_mad_u64_u32 v[7:8], null, v4, v3, v[1:2] +; GFX11-NEXT: v_mad_u64_u32 v[1:2], null, v6, v5, v[7:8] ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_mul_i64: @@ -506,11 +505,10 @@ define i64 @v_mul_i64(i64 %num, i64 %den) { ; GFX12-NEXT: s_wait_bvhcnt 0x0 ; GFX12-NEXT: s_wait_kmcnt 0x0 ; GFX12-NEXT: v_mul_hi_u32 v4, v0, v2 -; GFX12-NEXT: v_mul_lo_u32 v3, v0, v3 -; GFX12-NEXT: v_mul_lo_u32 v1, v1, v2 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX12-NEXT: v_mad_co_u64_u32 v[3:4], null, v0, v3, v[4:5] ; GFX12-NEXT: v_mul_lo_u32 v0, v0, v2 -; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX12-NEXT: v_add3_u32 v1, v4, v3, v1 +; GFX12-NEXT: v_mad_co_u64_u32 v[1:2], null, v1, v2, v[3:4] ; GFX12-NEXT: s_setpc_b64 s[30:31] %result = mul i64 %num, %den ret i64 %result @@ -653,11 +651,11 @@ define i96 @v_mul_i96(i96 %num, i96 %den) { ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v6, v0 ; GFX10-NEXT: v_mov_b32_e32 v7, v1 -; GFX10-NEXT: v_mul_lo_u32 v2, v2, v3 -; GFX10-NEXT: v_mul_lo_u32 v5, v6, v5 -; GFX10-NEXT: v_mul_lo_u32 v8, v7, v4 +; GFX10-NEXT: v_mul_lo_u32 v0, v6, v5 +; GFX10-NEXT: v_mad_u64_u32 v[8:9], s4, v7, v4, v[0:1] ; GFX10-NEXT: v_mad_u64_u32 v[0:1], s4, v6, v3, 0 -; GFX10-NEXT: v_add3_u32 v2, v5, v8, v2 +; GFX10-NEXT: v_mad_u64_u32 v[8:9], s4, v2, v3, v[8:9] +; GFX10-NEXT: v_mov_b32_e32 v2, v8 ; GFX10-NEXT: v_mad_u64_u32 v[1:2], s4, v6, v4, v[1:2] ; GFX10-NEXT: v_mad_u64_u32 v[1:2], s4, v7, v3, v[1:2] ; GFX10-NEXT: s_setpc_b64 s[30:31] @@ -666,11 +664,11 @@ define i96 @v_mul_i96(i96 %num, i96 %den) { ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_dual_mov_b32 v6, v0 :: v_dual_mov_b32 v7, v1 -; GFX11-NEXT: v_mul_lo_u32 v2, v2, v3 -; GFX11-NEXT: v_mul_lo_u32 v5, v6, v5 -; GFX11-NEXT: v_mul_lo_u32 v8, v7, v4 +; GFX11-NEXT: v_mul_lo_u32 v0, v6, v5 +; GFX11-NEXT: v_mad_u64_u32 v[8:9], null, v7, v4, v[0:1] ; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v6, v3, 0 -; GFX11-NEXT: v_add3_u32 v2, v5, v8, v2 +; GFX11-NEXT: v_mad_u64_u32 v[9:10], null, v2, v3, v[8:9] +; GFX11-NEXT: v_mov_b32_e32 v2, v9 ; GFX11-NEXT: v_mad_u64_u32 v[1:2], null, v6, v4, v[1:2] ; GFX11-NEXT: v_mad_u64_u32 v[1:2], null, v7, v3, v[1:2] ; GFX11-NEXT: s_setpc_b64 s[30:31] @@ -683,15 +681,15 @@ define i96 @v_mul_i96(i96 %num, i96 %den) { ; GFX12-NEXT: s_wait_bvhcnt 0x0 ; GFX12-NEXT: s_wait_kmcnt 0x0 ; GFX12-NEXT: v_dual_mov_b32 v6, v0 :: v_dual_mov_b32 v7, v1 -; GFX12-NEXT: v_mul_lo_u32 v2, v2, v3 -; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX12-NEXT: v_mul_lo_u32 v5, v6, v5 -; GFX12-NEXT: v_mul_lo_u32 v8, v7, v4 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_mul_lo_u32 v0, v6, v5 +; GFX12-NEXT: v_mad_co_u64_u32 v[8:9], null, v7, v4, v[0:1] ; GFX12-NEXT: v_mad_co_u64_u32 v[0:1], null, v6, v3, 0 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-NEXT: v_add3_u32 v2, v5, v8, v2 +; GFX12-NEXT: v_mad_co_u64_u32 v[8:9], null, v2, v3, v[8:9] +; GFX12-NEXT: v_mov_b32_e32 v2, v8 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-NEXT: v_mad_co_u64_u32 v[1:2], null, v6, v4, v[1:2] -; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-NEXT: v_mad_co_u64_u32 v[1:2], null, v7, v3, v[1:2] ; GFX12-NEXT: s_setpc_b64 s[30:31] %result = mul i96 %num, %den @@ -978,7 +976,6 @@ define i128 @v_mul_i128(i128 %num, i128 %den) { ; GFX10-NEXT: v_mov_b32_e32 v8, v0 ; GFX10-NEXT: v_mov_b32_e32 v9, v1 ; GFX10-NEXT: v_mov_b32_e32 v10, v2 -; GFX10-NEXT: v_mul_lo_u32 v3, v3, v4 ; GFX10-NEXT: v_mad_u64_u32 v[0:1], s4, v8, v6, 0 ; GFX10-NEXT: v_mul_lo_u32 v7, v8, v7 ; GFX10-NEXT: v_mul_lo_u32 v6, v9, v6 @@ -987,32 +984,31 @@ define i128 @v_mul_i128(i128 %num, i128 %den) { ; GFX10-NEXT: v_mad_u64_u32 v[11:12], s4, v10, v4, v[11:12] ; GFX10-NEXT: v_mov_b32_e32 v2, v11 ; GFX10-NEXT: v_mad_u64_u32 v[1:2], vcc_lo, v8, v5, v[1:2] -; GFX10-NEXT: v_mul_lo_u32 v5, v10, v5 ; GFX10-NEXT: v_mad_u64_u32 v[1:2], s4, v9, v4, v[1:2] ; GFX10-NEXT: v_add_co_ci_u32_e64 v7, s4, v12, v7, s4 -; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v7, v6, vcc_lo -; GFX10-NEXT: v_add3_u32 v3, v4, v5, v3 +; GFX10-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, v7, v6, vcc_lo +; GFX10-NEXT: v_mad_u64_u32 v[5:6], s4, v10, v5, v[6:7] +; GFX10-NEXT: v_mad_u64_u32 v[3:4], s4, v3, v4, v[5:6] ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_mul_i128: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_dual_mov_b32 v8, v0 :: v_dual_mov_b32 v9, v1 -; GFX11-NEXT: v_mov_b32_e32 v10, v2 -; GFX11-NEXT: v_mul_lo_u32 v3, v3, v4 +; GFX11-NEXT: v_dual_mov_b32 v10, v2 :: v_dual_mov_b32 v11, v4 +; GFX11-NEXT: v_mov_b32_e32 v12, v3 ; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v8, v6, 0 -; GFX11-NEXT: v_mul_lo_u32 v7, v8, v7 -; GFX11-NEXT: v_mul_lo_u32 v6, v9, v6 -; GFX11-NEXT: v_mad_u64_u32 v[11:12], null, v9, v5, v[0:1] -; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v8, v4, 0 -; GFX11-NEXT: v_mad_u64_u32 v[11:12], null, v10, v4, v[11:12] -; GFX11-NEXT: v_mov_b32_e32 v2, v11 +; GFX11-NEXT: v_mul_lo_u32 v4, v9, v6 +; GFX11-NEXT: v_mul_lo_u32 v6, v8, v7 +; GFX11-NEXT: v_mad_u64_u32 v[2:3], null, v9, v5, v[0:1] +; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v8, v11, 0 +; GFX11-NEXT: v_mad_u64_u32 v[2:3], null, v10, v11, v[2:3] ; GFX11-NEXT: v_mad_u64_u32 v[1:2], vcc_lo, v8, v5, v[1:2] -; GFX11-NEXT: v_mul_lo_u32 v5, v10, v5 -; GFX11-NEXT: v_mad_u64_u32 v[1:2], s0, v9, v4, v[1:2] -; GFX11-NEXT: v_add_co_ci_u32_e64 v7, s0, v12, v7, s0 -; GFX11-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v7, v6, vcc_lo -; GFX11-NEXT: v_add3_u32 v3, v4, v5, v3 +; GFX11-NEXT: v_mad_u64_u32 v[1:2], s0, v9, v11, v[1:2] +; GFX11-NEXT: v_add_co_ci_u32_e64 v3, s0, v3, v6, s0 +; GFX11-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v4, vcc_lo +; GFX11-NEXT: v_mad_u64_u32 v[6:7], null, v10, v5, v[3:4] +; GFX11-NEXT: v_mad_u64_u32 v[3:4], null, v12, v11, v[6:7] ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_mul_i128: @@ -1024,25 +1020,25 @@ define i128 @v_mul_i128(i128 %num, i128 %den) { ; GFX12-NEXT: s_wait_kmcnt 0x0 ; GFX12-NEXT: v_dual_mov_b32 v8, v0 :: v_dual_mov_b32 v9, v1 ; GFX12-NEXT: v_mov_b32_e32 v10, v2 -; GFX12-NEXT: v_mul_lo_u32 v3, v3, v4 -; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4) ; GFX12-NEXT: v_mad_co_u64_u32 v[0:1], null, v8, v6, 0 ; GFX12-NEXT: v_mul_lo_u32 v7, v8, v7 ; GFX12-NEXT: v_mul_lo_u32 v6, v9, v6 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX12-NEXT: v_mad_co_u64_u32 v[11:12], null, v9, v5, v[0:1] ; GFX12-NEXT: v_mad_co_u64_u32 v[0:1], null, v8, v4, 0 -; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-NEXT: v_mad_co_u64_u32 v[11:12], null, v10, v4, v[11:12] +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-NEXT: v_mov_b32_e32 v2, v11 -; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX12-NEXT: v_mad_co_u64_u32 v[1:2], vcc_lo, v8, v5, v[1:2] -; GFX12-NEXT: v_mul_lo_u32 v5, v10, v5 -; GFX12-NEXT: v_mad_co_u64_u32 v[1:2], s0, v9, v4, v[1:2] ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_mad_co_u64_u32 v[1:2], s0, v9, v4, v[1:2] ; GFX12-NEXT: v_add_co_ci_u32_e64 v7, s0, v12, v7, s0 -; GFX12-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v7, v6, vcc_lo +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, v7, v6, vcc_lo +; GFX12-NEXT: v_mad_co_u64_u32 v[5:6], null, v10, v5, v[6:7] ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-NEXT: v_add3_u32 v3, v4, v5, v3 +; GFX12-NEXT: v_mad_co_u64_u32 v[3:4], null, v3, v4, v[5:6] ; GFX12-NEXT: s_setpc_b64 s[30:31] %result = mul i128 %num, %den ret i128 %result @@ -2248,7 +2244,6 @@ define i256 @v_mul_i256(i256 %num, i256 %den) { ; GFX10-NEXT: v_mov_b32_e32 v17, v1 ; GFX10-NEXT: v_mul_lo_u32 v27, v6, v9 ; GFX10-NEXT: v_mul_lo_u32 v28, v5, v10 -; GFX10-NEXT: v_mul_lo_u32 v7, v7, v8 ; GFX10-NEXT: v_mad_u64_u32 v[0:1], s4, v16, v14, 0 ; GFX10-NEXT: v_mad_u64_u32 v[18:19], s4, v16, v12, 0 ; GFX10-NEXT: v_mul_lo_u32 v30, v17, v14 @@ -2308,78 +2303,78 @@ define i256 @v_mul_i256(i256 %num, i256 %den) { ; GFX10-NEXT: v_add_co_ci_u32_e64 v9, s6, v9, v25, s7 ; GFX10-NEXT: v_add_co_ci_u32_e64 v9, s5, v9, v20, s5 ; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v9, v28, vcc_lo -; GFX10-NEXT: v_add_co_ci_u32_e64 v8, vcc_lo, v9, v27, s4 -; GFX10-NEXT: v_add_nc_u32_e32 v7, v8, v7 +; GFX10-NEXT: v_add_co_ci_u32_e64 v9, vcc_lo, v9, v27, s4 +; GFX10-NEXT: v_mad_u64_u32 v[7:8], s4, v7, v8, v[9:10] ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_mul_i256: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_dual_mov_b32 v16, v0 :: v_dual_mov_b32 v17, v1 -; GFX11-NEXT: v_mul_lo_u32 v7, v7, v8 -; GFX11-NEXT: v_mul_lo_u32 v27, v6, v9 -; GFX11-NEXT: v_mul_lo_u32 v28, v5, v10 +; GFX11-NEXT: v_dual_mov_b32 v18, v8 :: v_dual_mov_b32 v19, v7 +; GFX11-NEXT: v_mul_lo_u32 v30, v4, v11 ; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v16, v14, 0 -; GFX11-NEXT: v_mad_u64_u32 v[18:19], null, v16, v12, 0 -; GFX11-NEXT: v_mul_lo_u32 v30, v17, v14 +; GFX11-NEXT: v_mad_u64_u32 v[7:8], null, v16, v12, 0 +; GFX11-NEXT: v_mul_lo_u32 v29, v17, v14 +; GFX11-NEXT: v_mul_lo_u32 v28, v5, v10 ; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v17, v13, v[0:1] -; GFX11-NEXT: v_mad_u64_u32 v[18:19], s0, v17, v11, v[18:19] +; GFX11-NEXT: v_mad_u64_u32 v[7:8], s0, v17, v11, v[7:8] ; GFX11-NEXT: v_cndmask_b32_e64 v20, 0, 1, s0 ; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v2, v12, v[0:1] -; GFX11-NEXT: v_mad_u64_u32 v[18:19], vcc_lo, v2, v10, v[18:19] +; GFX11-NEXT: v_mad_u64_u32 v[7:8], vcc_lo, v2, v10, v[7:8] ; GFX11-NEXT: v_add_co_ci_u32_e32 v22, vcc_lo, 0, v20, vcc_lo ; GFX11-NEXT: v_mad_u64_u32 v[20:21], null, v16, v10, 0 ; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v3, v11, v[0:1] -; GFX11-NEXT: v_mad_u64_u32 v[18:19], vcc_lo, v3, v9, v[18:19] +; GFX11-NEXT: v_mad_u64_u32 v[7:8], vcc_lo, v3, v9, v[7:8] ; GFX11-NEXT: v_add_co_ci_u32_e32 v24, vcc_lo, 0, v22, vcc_lo ; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v4, v10, v[0:1] -; GFX11-NEXT: v_mad_u64_u32 v[18:19], vcc_lo, v4, v8, v[18:19] -; GFX11-NEXT: v_add_co_ci_u32_e32 v26, vcc_lo, 0, v24, vcc_lo +; GFX11-NEXT: v_mad_u64_u32 v[7:8], vcc_lo, v4, v18, v[7:8] +; GFX11-NEXT: v_add_co_ci_u32_e32 v27, vcc_lo, 0, v24, vcc_lo ; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v5, v9, v[0:1] -; GFX11-NEXT: v_mad_u64_u32 v[22:23], null, v6, v8, v[0:1] +; GFX11-NEXT: v_mad_u64_u32 v[22:23], null, v6, v18, v[0:1] ; GFX11-NEXT: v_mad_u64_u32 v[0:1], s0, v17, v9, v[20:21] -; GFX11-NEXT: v_cndmask_b32_e64 v25, 0, 1, s0 -; GFX11-NEXT: v_mov_b32_e32 v20, v22 -; GFX11-NEXT: v_mad_u64_u32 v[21:22], vcc_lo, v2, v8, v[0:1] -; GFX11-NEXT: v_add_co_ci_u32_e32 v29, vcc_lo, 0, v25, vcc_lo -; GFX11-NEXT: v_mad_u64_u32 v[0:1], s0, v16, v13, v[19:20] -; GFX11-NEXT: v_mov_b32_e32 v20, v18 -; GFX11-NEXT: v_mov_b32_e32 v19, v22 -; GFX11-NEXT: v_mul_lo_u32 v22, v16, v15 -; GFX11-NEXT: v_mad_u64_u32 v[24:25], vcc_lo, v17, v12, v[0:1] -; GFX11-NEXT: v_mad_u64_u32 v[14:15], s2, v16, v11, v[19:20] -; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v16, v8, 0 -; GFX11-NEXT: v_mul_lo_u32 v20, v4, v11 -; GFX11-NEXT: v_cndmask_b32_e64 v6, 0, 1, s2 -; GFX11-NEXT: v_mad_u64_u32 v[18:19], s1, v2, v11, v[24:25] -; GFX11-NEXT: v_mul_lo_u32 v25, v3, v12 -; GFX11-NEXT: v_mad_u64_u32 v[11:12], s2, v17, v10, v[14:15] -; GFX11-NEXT: v_mov_b32_e32 v14, v21 -; GFX11-NEXT: v_add_co_ci_u32_e64 v6, s2, 0, v6, s2 -; GFX11-NEXT: v_mad_u64_u32 v[18:19], s3, v3, v10, v[18:19] -; GFX11-NEXT: v_mul_lo_u32 v24, v2, v13 -; GFX11-NEXT: v_mov_b32_e32 v13, v1 -; GFX11-NEXT: v_mad_u64_u32 v[1:2], s2, v2, v9, v[11:12] -; GFX11-NEXT: v_add_co_ci_u32_e64 v6, s2, 0, v6, s2 -; GFX11-NEXT: v_mad_u64_u32 v[10:11], s2, v4, v9, v[18:19] -; GFX11-NEXT: v_mad_u64_u32 v[12:13], s4, v16, v9, v[13:14] -; GFX11-NEXT: v_cndmask_b32_e64 v9, 0, 1, s4 -; GFX11-NEXT: v_mad_u64_u32 v[3:4], s4, v3, v8, v[1:2] -; GFX11-NEXT: v_add_co_ci_u32_e64 v14, s4, 0, v6, s4 -; GFX11-NEXT: v_mad_u64_u32 v[5:6], s4, v5, v8, v[10:11] -; GFX11-NEXT: v_mad_u64_u32 v[1:2], s5, v17, v8, v[12:13] -; GFX11-NEXT: v_add_co_ci_u32_e64 v3, s5, v9, v3, s5 -; GFX11-NEXT: v_add_co_ci_u32_e64 v4, s5, v29, v4, s5 -; GFX11-NEXT: v_add_co_ci_u32_e64 v5, s5, v14, v5, s5 -; GFX11-NEXT: v_add_co_ci_u32_e64 v6, s5, v26, v6, s5 -; GFX11-NEXT: v_add_co_ci_u32_e64 v9, s5, v23, v22, s5 -; GFX11-NEXT: v_add_co_ci_u32_e64 v9, s4, v9, v30, s4 -; GFX11-NEXT: v_add_co_ci_u32_e64 v9, s2, v9, v24, s2 -; GFX11-NEXT: v_add_co_ci_u32_e64 v9, s2, v9, v25, s3 -; GFX11-NEXT: v_add_co_ci_u32_e64 v9, s1, v9, v20, s1 -; GFX11-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v9, v28, vcc_lo -; GFX11-NEXT: v_add_co_ci_u32_e64 v8, vcc_lo, v9, v27, s0 -; GFX11-NEXT: v_add_nc_u32_e32 v7, v8, v7 +; GFX11-NEXT: v_mov_b32_e32 v20, v8 +; GFX11-NEXT: v_cndmask_b32_e64 v26, 0, 1, s0 +; GFX11-NEXT: v_mov_b32_e32 v21, v22 +; GFX11-NEXT: v_mul_lo_u32 v22, v6, v9 +; GFX11-NEXT: v_mad_u64_u32 v[24:25], vcc_lo, v2, v18, v[0:1] +; GFX11-NEXT: v_add_co_ci_u32_e32 v26, vcc_lo, 0, v26, vcc_lo +; GFX11-NEXT: v_mad_u64_u32 v[0:1], s0, v16, v13, v[20:21] +; GFX11-NEXT: v_mov_b32_e32 v6, v25 +; GFX11-NEXT: v_mul_lo_u32 v25, v16, v15 +; GFX11-NEXT: v_mad_u64_u32 v[20:21], vcc_lo, v17, v12, v[0:1] +; GFX11-NEXT: v_mad_u64_u32 v[6:7], s2, v16, v11, v[6:7] +; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v16, v18, 0 +; GFX11-NEXT: v_cndmask_b32_e64 v8, 0, 1, s2 +; GFX11-NEXT: v_mad_u64_u32 v[14:15], s1, v2, v11, v[20:21] +; GFX11-NEXT: v_mad_u64_u32 v[6:7], s2, v17, v10, v[6:7] +; GFX11-NEXT: v_mul_lo_u32 v20, v2, v13 +; GFX11-NEXT: v_add_co_ci_u32_e64 v8, s2, 0, v8, s2 +; GFX11-NEXT: v_mov_b32_e32 v11, v1 +; GFX11-NEXT: v_mad_u64_u32 v[13:14], s3, v3, v10, v[14:15] +; GFX11-NEXT: v_mad_u64_u32 v[1:2], s2, v2, v9, v[6:7] +; GFX11-NEXT: v_mul_lo_u32 v21, v3, v12 +; GFX11-NEXT: v_mov_b32_e32 v12, v24 +; GFX11-NEXT: v_add_co_ci_u32_e64 v10, s2, 0, v8, s2 +; GFX11-NEXT: v_mad_u64_u32 v[6:7], s2, v4, v9, v[13:14] +; GFX11-NEXT: v_mad_u64_u32 v[8:9], s4, v16, v9, v[11:12] +; GFX11-NEXT: v_cndmask_b32_e64 v11, 0, 1, s4 +; GFX11-NEXT: v_mad_u64_u32 v[3:4], s4, v3, v18, v[1:2] +; GFX11-NEXT: v_add_co_ci_u32_e64 v10, s4, 0, v10, s4 +; GFX11-NEXT: v_mad_u64_u32 v[5:6], s4, v5, v18, v[6:7] +; GFX11-NEXT: v_mad_u64_u32 v[1:2], s5, v17, v18, v[8:9] +; GFX11-NEXT: v_add_co_ci_u32_e64 v3, s5, v11, v3, s5 +; GFX11-NEXT: v_add_co_ci_u32_e64 v4, s5, v26, v4, s5 +; GFX11-NEXT: v_add_co_ci_u32_e64 v5, s5, v10, v5, s5 +; GFX11-NEXT: v_add_co_ci_u32_e64 v6, s5, v27, v6, s5 +; GFX11-NEXT: v_add_co_ci_u32_e64 v7, s5, v23, v25, s5 +; GFX11-NEXT: v_add_co_ci_u32_e64 v7, s4, v7, v29, s4 +; GFX11-NEXT: v_add_co_ci_u32_e64 v7, s2, v7, v20, s2 +; GFX11-NEXT: v_add_co_ci_u32_e64 v7, s2, v7, v21, s3 +; GFX11-NEXT: v_add_co_ci_u32_e64 v7, s1, v7, v30, s1 +; GFX11-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v7, v28, vcc_lo +; GFX11-NEXT: v_add_co_ci_u32_e64 v9, vcc_lo, v7, v22, s0 +; GFX11-NEXT: v_mad_u64_u32 v[7:8], null, v19, v18, v[9:10] ; GFX11-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-LABEL: v_mul_i256: @@ -2391,9 +2386,8 @@ define i256 @v_mul_i256(i256 %num, i256 %den) { ; GFX12-NEXT: s_wait_kmcnt 0x0 ; GFX12-NEXT: v_dual_mov_b32 v16, v0 :: v_dual_mov_b32 v17, v1 ; GFX12-NEXT: v_mul_lo_u32 v27, v6, v9 -; GFX12-NEXT: v_mul_lo_u32 v7, v7, v8 ; GFX12-NEXT: v_mul_lo_u32 v28, v5, v10 -; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX12-NEXT: v_mad_co_u64_u32 v[0:1], null, v16, v14, 0 ; GFX12-NEXT: v_mad_co_u64_u32 v[18:19], null, v16, v12, 0 ; GFX12-NEXT: v_mul_lo_u32 v30, v17, v14 @@ -2472,9 +2466,9 @@ define i256 @v_mul_i256(i256 %num, i256 %den) { ; GFX12-NEXT: v_add_co_ci_u32_e64 v9, s1, v9, v20, s1 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v9, v28, vcc_lo -; GFX12-NEXT: v_add_co_ci_u32_e64 v8, vcc_lo, v9, v27, s0 +; GFX12-NEXT: v_add_co_ci_u32_e64 v9, vcc_lo, v9, v27, s0 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-NEXT: v_add_nc_u32_e32 v7, v8, v7 +; GFX12-NEXT: v_mad_co_u64_u32 v[7:8], null, v7, v8, v[9:10] ; GFX12-NEXT: s_setpc_b64 s[30:31] %result = mul i256 %num, %den ret i256 %result @@ -2684,23 +2678,22 @@ define amdgpu_ps void @s_mul_u64_sext_with_vregs(ptr addrspace(1) %out, ptr addr ; ; GFX10-LABEL: s_mul_u64_sext_with_vregs: ; GFX10: ; %bb.0: -; GFX10-NEXT: global_load_dword v2, v[2:3], off +; GFX10-NEXT: global_load_dword v4, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_ashrrev_i32_e32 v4, 31, v2 -; GFX10-NEXT: v_mad_u64_u32 v[2:3], s0, 0x50, v2, 0 -; GFX10-NEXT: v_mul_lo_u32 v4, 0x50, v4 -; GFX10-NEXT: v_add_nc_u32_e32 v3, v3, v4 +; GFX10-NEXT: v_mad_u64_u32 v[2:3], s0, 0x50, v4, 0 +; GFX10-NEXT: v_ashrrev_i32_e32 v4, 31, v4 +; GFX10-NEXT: v_mad_u64_u32 v[3:4], s0, 0x50, v4, v[3:4] ; GFX10-NEXT: global_store_dwordx2 v[0:1], v[2:3], off ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: s_mul_u64_sext_with_vregs: ; GFX11: ; %bb.0: -; GFX11-NEXT: global_load_b32 v2, v[2:3], off +; GFX11-NEXT: global_load_b32 v4, v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_ashrrev_i32_e32 v4, 31, v2 -; GFX11-NEXT: v_mad_u64_u32 v[2:3], null, 0x50, v2, 0 -; GFX11-NEXT: v_mul_lo_u32 v4, 0x50, v4 -; GFX11-NEXT: v_add_nc_u32_e32 v3, v3, v4 +; GFX11-NEXT: v_mad_u64_u32 v[2:3], null, 0x50, v4, 0 +; GFX11-NEXT: v_ashrrev_i32_e32 v6, 31, v4 +; GFX11-NEXT: v_mad_u64_u32 v[4:5], null, 0x50, v6, v[3:4] +; GFX11-NEXT: v_mov_b32_e32 v3, v4 ; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll index 5297df3bedf8..b666f4552166 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll @@ -474,97 +474,94 @@ define amdgpu_kernel void @sdivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1) ; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s9 ; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s8 ; GFX10-NEXT: s_sub_u32 s10, 0, s8 -; GFX10-NEXT: s_subb_u32 s11, 0, s9 ; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f800000, v0 ; GFX10-NEXT: v_add_f32_e32 v0, v0, v1 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX10-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 ; GFX10-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 -; GFX10-NEXT: v_trunc_f32_e32 v1, v1 -; GFX10-NEXT: v_mul_f32_e32 v2, 0xcf800000, v1 -; GFX10-NEXT: v_add_f32_e32 v0, v2, v0 -; GFX10-NEXT: v_cvt_u32_f32_e32 v2, v1 +; GFX10-NEXT: v_trunc_f32_e32 v2, v1 +; GFX10-NEXT: v_mul_f32_e32 v1, 0xcf800000, v2 +; GFX10-NEXT: v_cvt_u32_f32_e32 v4, v2 +; GFX10-NEXT: v_add_f32_e32 v0, v1, v0 ; GFX10-NEXT: v_cvt_u32_f32_e32 v3, v0 -; GFX10-NEXT: v_mul_lo_u32 v4, s10, v2 -; GFX10-NEXT: v_mad_u64_u32 v[0:1], s14, s10, v3, 0 -; GFX10-NEXT: v_mul_lo_u32 v5, s11, v3 -; GFX10-NEXT: v_mul_hi_u32 v6, v2, v0 -; GFX10-NEXT: v_add3_u32 v1, v1, v4, v5 -; GFX10-NEXT: v_mul_lo_u32 v4, v2, v0 +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s11, s10, v3, 0 +; GFX10-NEXT: v_mad_u64_u32 v[1:2], s11, s10, v4, v[1:2] +; GFX10-NEXT: s_subb_u32 s11, 0, s9 +; GFX10-NEXT: v_mul_hi_u32 v6, v4, v0 +; GFX10-NEXT: v_mad_u64_u32 v[1:2], s14, s11, v3, v[1:2] +; GFX10-NEXT: v_mul_lo_u32 v2, v4, v0 ; GFX10-NEXT: v_mul_hi_u32 v0, v3, v0 ; GFX10-NEXT: v_mul_lo_u32 v5, v3, v1 -; GFX10-NEXT: v_mul_lo_u32 v7, v2, v1 +; GFX10-NEXT: v_mul_lo_u32 v7, v4, v1 ; GFX10-NEXT: v_mul_hi_u32 v8, v3, v1 -; GFX10-NEXT: v_mul_hi_u32 v1, v2, v1 -; GFX10-NEXT: v_add_co_u32 v4, s14, v4, v5 +; GFX10-NEXT: v_mul_hi_u32 v1, v4, v1 +; GFX10-NEXT: v_add_co_u32 v2, s14, v2, v5 ; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s14 ; GFX10-NEXT: v_add_co_u32 v6, s14, v7, v6 ; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, s14 -; GFX10-NEXT: v_add_co_u32 v0, s14, v4, v0 +; GFX10-NEXT: v_add_co_u32 v0, s14, v2, v0 ; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s14 -; GFX10-NEXT: v_add_co_u32 v4, s14, v6, v8 +; GFX10-NEXT: v_add_co_u32 v2, s14, v6, v8 ; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s14 ; GFX10-NEXT: v_add_nc_u32_e32 v0, v5, v0 ; GFX10-NEXT: v_add_nc_u32_e32 v5, v7, v6 -; GFX10-NEXT: v_add_co_u32 v0, s14, v4, v0 -; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, s14 +; GFX10-NEXT: v_add_co_u32 v0, s14, v2, v0 +; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, s14 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, v3, v0 -; GFX10-NEXT: v_add3_u32 v1, v5, v4, v1 -; GFX10-NEXT: v_mul_lo_u32 v4, s11, v3 -; GFX10-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v1, vcc_lo +; GFX10-NEXT: v_add3_u32 v1, v5, v2, v1 +; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v1, vcc_lo ; GFX10-NEXT: v_mad_u64_u32 v[0:1], s14, s10, v3, 0 -; GFX10-NEXT: v_mul_lo_u32 v5, s10, v2 -; GFX10-NEXT: v_mul_hi_u32 v6, v2, v0 -; GFX10-NEXT: v_add3_u32 v1, v1, v5, v4 -; GFX10-NEXT: v_mul_lo_u32 v4, v2, v0 +; GFX10-NEXT: v_mad_u64_u32 v[1:2], s10, s10, v4, v[1:2] +; GFX10-NEXT: v_mul_hi_u32 v6, v4, v0 +; GFX10-NEXT: v_mad_u64_u32 v[1:2], s10, s11, v3, v[1:2] +; GFX10-NEXT: v_mul_lo_u32 v2, v4, v0 ; GFX10-NEXT: v_mul_hi_u32 v0, v3, v0 ; GFX10-NEXT: v_mul_lo_u32 v5, v3, v1 -; GFX10-NEXT: v_mul_lo_u32 v7, v2, v1 +; GFX10-NEXT: v_mul_lo_u32 v7, v4, v1 ; GFX10-NEXT: v_mul_hi_u32 v8, v3, v1 -; GFX10-NEXT: v_mul_hi_u32 v1, v2, v1 -; GFX10-NEXT: v_add_co_u32 v4, s10, v4, v5 +; GFX10-NEXT: v_mul_hi_u32 v1, v4, v1 +; GFX10-NEXT: v_add_co_u32 v2, s10, v2, v5 ; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s10 ; GFX10-NEXT: v_add_co_u32 v6, s10, v7, v6 ; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, s10 -; GFX10-NEXT: v_add_co_u32 v0, s10, v4, v0 +; GFX10-NEXT: v_add_co_u32 v0, s10, v2, v0 ; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s10 -; GFX10-NEXT: v_add_co_u32 v4, s10, v6, v8 +; GFX10-NEXT: v_add_co_u32 v2, s10, v6, v8 ; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s10 ; GFX10-NEXT: v_add_nc_u32_e32 v0, v5, v0 ; GFX10-NEXT: v_add_nc_u32_e32 v5, v7, v6 -; GFX10-NEXT: v_add_co_u32 v0, s10, v4, v0 -; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, s10 +; GFX10-NEXT: v_add_co_u32 v0, s10, v2, v0 +; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, s10 ; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v3, v0 -; GFX10-NEXT: v_add3_u32 v1, v5, v4, v1 -; GFX10-NEXT: v_mul_hi_u32 v4, s1, v0 -; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v2, v1, vcc_lo +; GFX10-NEXT: v_add3_u32 v1, v5, v2, v1 ; GFX10-NEXT: v_mul_lo_u32 v2, s1, v0 -; GFX10-NEXT: v_mul_hi_u32 v0, s0, v0 +; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v4, v1, vcc_lo +; GFX10-NEXT: v_mul_hi_u32 v4, s0, v0 +; GFX10-NEXT: v_mul_hi_u32 v0, s1, v0 ; GFX10-NEXT: v_mul_lo_u32 v3, s0, v1 ; GFX10-NEXT: v_mul_lo_u32 v5, s1, v1 -; GFX10-NEXT: v_mul_hi_u32 v6, s0, v1 -; GFX10-NEXT: v_mul_hi_u32 v1, s1, v1 ; GFX10-NEXT: v_add_co_u32 v2, s10, v2, v3 -; GFX10-NEXT: v_cndmask_b32_e64 v3, 0, 1, s10 -; GFX10-NEXT: v_add_co_u32 v4, s10, v5, v4 -; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s10 -; GFX10-NEXT: v_add_co_u32 v0, s10, v2, v0 -; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s10 -; GFX10-NEXT: v_add_co_u32 v2, s10, v4, v6 +; GFX10-NEXT: v_mul_hi_u32 v3, s0, v1 +; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s10 +; GFX10-NEXT: v_add_co_u32 v2, s10, v2, v4 +; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, s10 +; GFX10-NEXT: v_add_co_u32 v0, s10, v5, v0 ; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, s10 -; GFX10-NEXT: v_add_nc_u32_e32 v0, v3, v0 -; GFX10-NEXT: v_add_nc_u32_e32 v3, v5, v4 -; GFX10-NEXT: v_add_co_u32 v2, s10, v2, v0 -; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s10 -; GFX10-NEXT: v_mul_lo_u32 v4, s9, v2 -; GFX10-NEXT: v_add3_u32 v3, v3, v0, v1 -; GFX10-NEXT: v_mad_u64_u32 v[0:1], s10, s8, v2, 0 -; GFX10-NEXT: v_mul_lo_u32 v5, s8, v3 -; GFX10-NEXT: v_add3_u32 v1, v1, v5, v4 -; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, v2, 1 -; GFX10-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, 0, v3, vcc_lo -; GFX10-NEXT: v_sub_nc_u32_e32 v6, s1, v1 +; GFX10-NEXT: v_add_nc_u32_e32 v2, v6, v2 +; GFX10-NEXT: v_add_co_u32 v0, s10, v0, v3 +; GFX10-NEXT: v_cndmask_b32_e64 v3, 0, 1, s10 +; GFX10-NEXT: v_add_co_u32 v5, s10, v0, v2 +; GFX10-NEXT: v_mul_hi_u32 v2, s1, v1 +; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s10 +; GFX10-NEXT: v_add_nc_u32_e32 v3, v4, v3 +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s10, s8, v5, 0 +; GFX10-NEXT: v_add3_u32 v3, v3, v6, v2 +; GFX10-NEXT: v_mad_u64_u32 v[1:2], s10, s8, v3, v[1:2] +; GFX10-NEXT: v_mad_u64_u32 v[1:2], s10, s9, v5, v[1:2] +; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v5, 1 +; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v3, vcc_lo ; GFX10-NEXT: v_sub_co_u32 v0, vcc_lo, s0, v0 +; GFX10-NEXT: v_sub_nc_u32_e32 v6, s1, v1 ; GFX10-NEXT: v_sub_co_ci_u32_e64 v1, s0, s1, v1, vcc_lo ; GFX10-NEXT: v_subrev_co_ci_u32_e32 v6, vcc_lo, s9, v6, vcc_lo ; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s8, v0 @@ -578,8 +575,8 @@ define amdgpu_kernel void @sdivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1) ; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, -1, s0 ; GFX10-NEXT: v_cmp_le_u32_e64 s0, s9, v9 ; GFX10-NEXT: v_cndmask_b32_e64 v12, 0, -1, s0 -; GFX10-NEXT: v_add_co_u32 v13, s0, v4, 1 -; GFX10-NEXT: v_add_co_ci_u32_e64 v14, s0, 0, v5, s0 +; GFX10-NEXT: v_add_co_u32 v13, s0, v2, 1 +; GFX10-NEXT: v_add_co_ci_u32_e64 v14, s0, 0, v4, s0 ; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s9, v9 ; GFX10-NEXT: v_cndmask_b32_e64 v11, v12, v11, s0 ; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s9, v1 @@ -587,14 +584,14 @@ define amdgpu_kernel void @sdivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1) ; GFX10-NEXT: v_cndmask_b32_e64 v7, v10, v7, s0 ; GFX10-NEXT: v_sub_co_u32 v10, s0, v8, s8 ; GFX10-NEXT: v_subrev_co_ci_u32_e64 v6, s0, 0, v6, s0 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v13, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v13, vcc_lo ; GFX10-NEXT: v_cmp_ne_u32_e64 s0, 0, v7 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v5, v14, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v14, vcc_lo ; GFX10-NEXT: v_cndmask_b32_e32 v7, v8, v10, vcc_lo ; GFX10-NEXT: v_cndmask_b32_e32 v6, v9, v6, vcc_lo ; GFX10-NEXT: s_xor_b64 s[8:9], s[2:3], s[12:13] -; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v4, s0 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v5, s0 +; GFX10-NEXT: v_cndmask_b32_e64 v2, v5, v2, s0 +; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v4, s0 ; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v7, s0 ; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v6, s0 ; GFX10-NEXT: v_mov_b32_e32 v4, 0 @@ -1902,25 +1899,25 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1 ; GFX10-NEXT: s_xor_b64 s[6:7], s[0:1], s[16:17] ; GFX10-NEXT: s_xor_b64 s[0:1], s[12:13], s[4:5] ; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s7 -; GFX10-NEXT: s_sub_u32 s20, 0, s6 -; GFX10-NEXT: s_subb_u32 s21, 0, s7 +; GFX10-NEXT: s_sub_u32 s21, 0, s6 +; GFX10-NEXT: s_subb_u32 s20, 0, s7 ; GFX10-NEXT: s_ashr_i32 s12, s15, 31 -; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s6 ; GFX10-NEXT: s_xor_b64 s[18:19], s[4:5], s[16:17] ; GFX10-NEXT: s_ashr_i32 s16, s3, 31 -; GFX10-NEXT: v_mul_f32_e32 v1, 0x4f800000, v1 ; GFX10-NEXT: s_add_u32 s14, s14, s12 ; GFX10-NEXT: s_addc_u32 s15, s15, s12 +; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s6 +; GFX10-NEXT: v_mul_f32_e32 v1, 0x4f800000, v1 ; GFX10-NEXT: s_add_u32 s2, s2, s16 ; GFX10-NEXT: s_mov_b32 s17, s16 ; GFX10-NEXT: s_addc_u32 s3, s3, s16 -; GFX10-NEXT: v_add_f32_e32 v0, v1, v0 -; GFX10-NEXT: s_xor_b64 s[2:3], s[2:3], s[16:17] ; GFX10-NEXT: s_mov_b32 s13, s12 +; GFX10-NEXT: s_xor_b64 s[2:3], s[2:3], s[16:17] +; GFX10-NEXT: v_add_f32_e32 v0, v1, v0 ; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s3 ; GFX10-NEXT: v_cvt_f32_u32_e32 v2, s2 -; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX10-NEXT: s_xor_b64 s[14:15], s[14:15], s[12:13] +; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX10-NEXT: v_mul_f32_e32 v1, 0x4f800000, v1 ; GFX10-NEXT: v_add_f32_e32 v1, v1, v2 ; GFX10-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 @@ -1929,259 +1926,256 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1 ; GFX10-NEXT: v_trunc_f32_e32 v2, v2 ; GFX10-NEXT: v_mul_f32_e32 v3, 0x5f7ffffc, v1 ; GFX10-NEXT: v_mul_f32_e32 v1, 0xcf800000, v2 -; GFX10-NEXT: v_cvt_u32_f32_e32 v5, v2 ; GFX10-NEXT: v_mul_f32_e32 v4, 0x2f800000, v3 +; GFX10-NEXT: v_cvt_u32_f32_e32 v9, v2 ; GFX10-NEXT: v_add_f32_e32 v0, v1, v0 -; GFX10-NEXT: v_mul_lo_u32 v7, s20, v5 -; GFX10-NEXT: v_trunc_f32_e32 v4, v4 -; GFX10-NEXT: v_cvt_u32_f32_e32 v6, v0 -; GFX10-NEXT: v_mul_f32_e32 v2, 0xcf800000, v4 -; GFX10-NEXT: v_mad_u64_u32 v[0:1], s5, s20, v6, 0 -; GFX10-NEXT: v_mul_lo_u32 v8, s21, v6 -; GFX10-NEXT: v_add_f32_e32 v2, v2, v3 -; GFX10-NEXT: v_cvt_u32_f32_e32 v3, v4 +; GFX10-NEXT: v_trunc_f32_e32 v6, v4 +; GFX10-NEXT: v_cvt_u32_f32_e32 v7, v0 +; GFX10-NEXT: v_mul_f32_e32 v4, 0xcf800000, v6 +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s5, s21, v7, 0 +; GFX10-NEXT: v_add_f32_e32 v3, v4, v3 ; GFX10-NEXT: s_sub_u32 s5, 0, s2 +; GFX10-NEXT: v_cvt_u32_f32_e32 v8, v3 +; GFX10-NEXT: v_mul_hi_u32 v10, v9, v0 +; GFX10-NEXT: v_mad_u64_u32 v[2:3], s22, s5, v8, 0 +; GFX10-NEXT: v_mad_u64_u32 v[4:5], s22, s21, v9, v[1:2] +; GFX10-NEXT: v_cvt_u32_f32_e32 v5, v6 +; GFX10-NEXT: v_mov_b32_e32 v1, v3 +; GFX10-NEXT: v_mul_hi_u32 v6, v7, v0 ; GFX10-NEXT: s_subb_u32 s22, 0, s3 -; GFX10-NEXT: v_cvt_u32_f32_e32 v4, v2 -; GFX10-NEXT: v_mul_lo_u32 v9, s5, v3 -; GFX10-NEXT: v_add3_u32 v7, v1, v7, v8 -; GFX10-NEXT: v_mul_lo_u32 v10, v5, v0 -; GFX10-NEXT: v_mul_hi_u32 v11, v6, v0 -; GFX10-NEXT: v_mad_u64_u32 v[1:2], s23, s5, v4, 0 -; GFX10-NEXT: v_mul_lo_u32 v8, s22, v4 -; GFX10-NEXT: v_mul_lo_u32 v12, v6, v7 -; GFX10-NEXT: v_mul_hi_u32 v0, v5, v0 -; GFX10-NEXT: v_mul_lo_u32 v13, v5, v7 -; GFX10-NEXT: v_mul_hi_u32 v14, v6, v7 -; GFX10-NEXT: v_mul_hi_u32 v7, v5, v7 -; GFX10-NEXT: v_add3_u32 v2, v2, v9, v8 -; GFX10-NEXT: v_add_co_u32 v10, s23, v10, v12 -; GFX10-NEXT: v_cndmask_b32_e64 v12, 0, 1, s23 -; GFX10-NEXT: v_add_co_u32 v0, s23, v13, v0 -; GFX10-NEXT: v_mul_lo_u32 v8, v3, v1 +; GFX10-NEXT: v_mul_hi_u32 v12, v8, v2 +; GFX10-NEXT: v_mul_lo_u32 v11, v5, v2 +; GFX10-NEXT: v_mad_u64_u32 v[3:4], s23, s20, v7, v[4:5] +; GFX10-NEXT: v_mul_lo_u32 v4, v9, v0 +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s23, s5, v5, v[1:2] +; GFX10-NEXT: v_mul_hi_u32 v2, v5, v2 +; GFX10-NEXT: v_mul_lo_u32 v13, v7, v3 +; GFX10-NEXT: v_mul_lo_u32 v14, v9, v3 +; GFX10-NEXT: v_mul_hi_u32 v15, v7, v3 +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s23, s22, v8, v[0:1] +; GFX10-NEXT: v_mul_hi_u32 v1, v9, v3 +; GFX10-NEXT: v_add_co_u32 v3, s23, v4, v13 +; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, s23 +; GFX10-NEXT: v_add_co_u32 v10, s23, v14, v10 ; GFX10-NEXT: v_cndmask_b32_e64 v13, 0, 1, s23 -; GFX10-NEXT: v_mul_lo_u32 v15, v4, v2 -; GFX10-NEXT: v_add_co_u32 v10, s23, v10, v11 -; GFX10-NEXT: v_mul_hi_u32 v9, v4, v1 -; GFX10-NEXT: v_mul_hi_u32 v1, v3, v1 +; GFX10-NEXT: v_mul_lo_u32 v14, v8, v0 +; GFX10-NEXT: v_add_co_u32 v3, s23, v3, v6 +; GFX10-NEXT: v_cndmask_b32_e64 v3, 0, 1, s23 +; GFX10-NEXT: v_add_co_u32 v6, s23, v10, v15 +; GFX10-NEXT: v_mul_lo_u32 v15, v5, v0 ; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, 1, s23 -; GFX10-NEXT: v_add_co_u32 v0, s23, v0, v14 -; GFX10-NEXT: v_mul_lo_u32 v14, v3, v2 +; GFX10-NEXT: v_mul_hi_u32 v16, v8, v0 +; GFX10-NEXT: v_mul_hi_u32 v17, v5, v0 +; GFX10-NEXT: v_add_nc_u32_e32 v0, v4, v3 +; GFX10-NEXT: v_add_co_u32 v4, s23, v11, v14 +; GFX10-NEXT: v_add_nc_u32_e32 v3, v13, v10 +; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, 1, s23 +; GFX10-NEXT: v_add_co_u32 v2, s23, v15, v2 ; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, 1, s23 -; GFX10-NEXT: v_add_nc_u32_e32 v10, v12, v10 -; GFX10-NEXT: v_add_co_u32 v8, s23, v8, v15 +; GFX10-NEXT: v_add_co_u32 v0, s23, v6, v0 +; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s23 +; GFX10-NEXT: v_add_co_u32 v4, s23, v4, v12 +; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, s23 +; GFX10-NEXT: v_add_co_u32 v2, s23, v2, v16 +; GFX10-NEXT: v_add3_u32 v1, v3, v6, v1 +; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v7, v0 +; GFX10-NEXT: v_add_nc_u32_e32 v3, v10, v4 ; GFX10-NEXT: v_cndmask_b32_e64 v12, 0, 1, s23 -; GFX10-NEXT: v_mul_hi_u32 v16, v4, v2 -; GFX10-NEXT: v_add_nc_u32_e32 v11, v13, v11 -; GFX10-NEXT: v_add_co_u32 v1, s23, v14, v1 -; GFX10-NEXT: v_cndmask_b32_e64 v13, 0, 1, s23 -; GFX10-NEXT: v_add_co_u32 v0, s23, v0, v10 -; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, 1, s23 -; GFX10-NEXT: v_add_co_u32 v8, s23, v8, v9 -; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, s23 -; GFX10-NEXT: v_add_co_u32 v9, s23, v1, v16 -; GFX10-NEXT: v_add3_u32 v7, v11, v10, v7 -; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, s23 -; GFX10-NEXT: v_add_co_u32 v6, vcc_lo, v6, v0 -; GFX10-NEXT: v_add_nc_u32_e32 v8, v12, v8 -; GFX10-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, v5, v7, vcc_lo -; GFX10-NEXT: v_mul_hi_u32 v2, v3, v2 -; GFX10-NEXT: v_add_nc_u32_e32 v10, v13, v1 -; GFX10-NEXT: v_mad_u64_u32 v[0:1], s23, s20, v6, 0 -; GFX10-NEXT: v_add_co_u32 v7, s23, v9, v8 -; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, s23 -; GFX10-NEXT: v_mul_lo_u32 v9, s21, v6 -; GFX10-NEXT: v_mul_lo_u32 v11, s20, v5 -; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, v4, v7 -; GFX10-NEXT: v_add3_u32 v2, v10, v8, v2 -; GFX10-NEXT: v_mul_lo_u32 v8, v5, v0 -; GFX10-NEXT: v_mul_hi_u32 v10, v6, v0 -; GFX10-NEXT: v_mul_hi_u32 v0, v5, v0 -; GFX10-NEXT: v_add3_u32 v7, v1, v11, v9 -; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v2, vcc_lo -; GFX10-NEXT: v_mad_u64_u32 v[1:2], s20, s5, v4, 0 -; GFX10-NEXT: v_mul_lo_u32 v12, v6, v7 -; GFX10-NEXT: v_mul_lo_u32 v9, s22, v4 -; GFX10-NEXT: v_mul_lo_u32 v11, s5, v3 -; GFX10-NEXT: v_mul_lo_u32 v13, v5, v7 -; GFX10-NEXT: v_mul_hi_u32 v14, v6, v7 -; GFX10-NEXT: v_mul_hi_u32 v7, v5, v7 -; GFX10-NEXT: v_mul_lo_u32 v15, v3, v1 -; GFX10-NEXT: v_mul_hi_u32 v16, v4, v1 -; GFX10-NEXT: v_add_co_u32 v8, s5, v8, v12 -; GFX10-NEXT: v_add3_u32 v2, v2, v11, v9 -; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, 1, s5 -; GFX10-NEXT: v_add_co_u32 v0, s5, v13, v0 +; GFX10-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v9, v1, vcc_lo +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s23, s21, v6, 0 +; GFX10-NEXT: v_add_co_u32 v2, s23, v2, v3 +; GFX10-NEXT: v_add_nc_u32_e32 v4, v11, v12 +; GFX10-NEXT: v_cndmask_b32_e64 v3, 0, 1, s23 +; GFX10-NEXT: v_mov_b32_e32 v10, 0 +; GFX10-NEXT: v_add_co_u32 v8, vcc_lo, v8, v2 +; GFX10-NEXT: v_mul_hi_u32 v11, v7, v0 +; GFX10-NEXT: v_add3_u32 v3, v4, v3, v17 +; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v5, v3, vcc_lo +; GFX10-NEXT: v_mad_u64_u32 v[2:3], s23, s5, v8, 0 +; GFX10-NEXT: v_mad_u64_u32 v[4:5], s21, s21, v7, v[1:2] +; GFX10-NEXT: v_mov_b32_e32 v1, v3 +; GFX10-NEXT: v_mul_lo_u32 v12, v9, v2 +; GFX10-NEXT: v_mul_hi_u32 v13, v8, v2 +; GFX10-NEXT: v_mad_u64_u32 v[3:4], s20, s20, v6, v[4:5] +; GFX10-NEXT: v_mul_lo_u32 v4, v7, v0 +; GFX10-NEXT: v_mul_hi_u32 v5, v6, v0 +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s5, s5, v9, v[1:2] +; GFX10-NEXT: v_mul_hi_u32 v2, v9, v2 +; GFX10-NEXT: v_mul_lo_u32 v14, v6, v3 +; GFX10-NEXT: v_mul_lo_u32 v15, v7, v3 +; GFX10-NEXT: v_mul_hi_u32 v16, v6, v3 +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s5, s22, v8, v[0:1] +; GFX10-NEXT: v_mul_hi_u32 v1, v7, v3 +; GFX10-NEXT: v_add_co_u32 v3, s5, v4, v14 +; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, s5 +; GFX10-NEXT: v_add_co_u32 v11, s5, v15, v11 +; GFX10-NEXT: v_cndmask_b32_e64 v14, 0, 1, s5 +; GFX10-NEXT: v_add_co_u32 v3, s5, v3, v5 +; GFX10-NEXT: v_mul_lo_u32 v15, v8, v0 +; GFX10-NEXT: v_cndmask_b32_e64 v3, 0, 1, s5 +; GFX10-NEXT: v_add_co_u32 v5, s5, v11, v16 +; GFX10-NEXT: v_mul_lo_u32 v16, v9, v0 ; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, 1, s5 -; GFX10-NEXT: v_add_co_u32 v8, s5, v8, v10 -; GFX10-NEXT: v_mul_lo_u32 v12, v4, v2 -; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, s5 -; GFX10-NEXT: v_add_co_u32 v0, s5, v0, v14 -; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, 1, s5 -; GFX10-NEXT: v_mul_hi_u32 v1, v3, v1 -; GFX10-NEXT: v_add_nc_u32_e32 v8, v9, v8 -; GFX10-NEXT: v_mul_lo_u32 v13, v3, v2 -; GFX10-NEXT: v_mul_hi_u32 v14, v4, v2 -; GFX10-NEXT: v_add_nc_u32_e32 v9, v11, v10 -; GFX10-NEXT: v_add_co_u32 v10, s5, v15, v12 -; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, 1, s5 -; GFX10-NEXT: v_add_co_u32 v0, s5, v0, v8 -; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, s5 -; GFX10-NEXT: v_add_co_u32 v1, s5, v13, v1 +; GFX10-NEXT: v_add_nc_u32_e32 v3, v4, v3 +; GFX10-NEXT: v_mul_hi_u32 v17, v8, v0 +; GFX10-NEXT: v_mul_hi_u32 v0, v9, v0 +; GFX10-NEXT: v_add_nc_u32_e32 v4, v14, v11 +; GFX10-NEXT: v_add_co_u32 v11, s5, v12, v15 ; GFX10-NEXT: v_cndmask_b32_e64 v12, 0, 1, s5 -; GFX10-NEXT: v_add_co_u32 v10, s5, v10, v16 -; GFX10-NEXT: v_add3_u32 v7, v9, v8, v7 -; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, 1, s5 -; GFX10-NEXT: v_add_co_u32 v1, s5, v1, v14 -; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v6, v0 -; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, s5 -; GFX10-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, v5, v7, vcc_lo -; GFX10-NEXT: v_add_nc_u32_e32 v6, v11, v10 -; GFX10-NEXT: v_mul_hi_u32 v9, s0, v0 -; GFX10-NEXT: v_add_nc_u32_e32 v7, v12, v8 -; GFX10-NEXT: v_mul_lo_u32 v8, s1, v0 -; GFX10-NEXT: v_mul_lo_u32 v10, s0, v5 -; GFX10-NEXT: v_mul_hi_u32 v0, s1, v0 -; GFX10-NEXT: v_mul_lo_u32 v11, s1, v5 -; GFX10-NEXT: v_add_co_u32 v1, s5, v1, v6 +; GFX10-NEXT: v_add_co_u32 v2, s5, v16, v2 +; GFX10-NEXT: v_cndmask_b32_e64 v14, 0, 1, s5 +; GFX10-NEXT: v_add_co_u32 v3, s5, v5, v3 +; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s5 +; GFX10-NEXT: v_add_co_u32 v11, s5, v11, v13 +; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, 1, s5 +; GFX10-NEXT: v_add_co_u32 v2, s5, v2, v17 +; GFX10-NEXT: v_add3_u32 v1, v4, v5, v1 +; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, v6, v3 +; GFX10-NEXT: v_add_nc_u32_e32 v4, v12, v11 +; GFX10-NEXT: v_cndmask_b32_e64 v13, 0, 1, s5 +; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v7, v1, vcc_lo +; GFX10-NEXT: v_mul_lo_u32 v6, s1, v3 +; GFX10-NEXT: v_add_co_u32 v2, s5, v2, v4 +; GFX10-NEXT: v_add_nc_u32_e32 v5, v14, v13 +; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, s5 +; GFX10-NEXT: v_mul_lo_u32 v11, s0, v1 +; GFX10-NEXT: v_mul_hi_u32 v7, s0, v3 +; GFX10-NEXT: v_mul_hi_u32 v3, s1, v3 +; GFX10-NEXT: v_mul_lo_u32 v12, s1, v1 +; GFX10-NEXT: v_add3_u32 v0, v5, v4, v0 +; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v8, v2 +; GFX10-NEXT: v_mul_hi_u32 v4, s0, v1 +; GFX10-NEXT: v_mul_hi_u32 v5, s1, v1 +; GFX10-NEXT: v_add_co_u32 v1, s5, v6, v11 +; GFX10-NEXT: v_add_co_ci_u32_e32 v8, vcc_lo, v9, v0, vcc_lo ; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s5 -; GFX10-NEXT: v_mul_hi_u32 v12, s0, v5 -; GFX10-NEXT: v_mul_hi_u32 v2, v3, v2 -; GFX10-NEXT: v_add_co_u32 v8, s5, v8, v10 -; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, 1, s5 -; GFX10-NEXT: v_add_co_u32 v0, s5, v11, v0 +; GFX10-NEXT: v_add_co_u32 v3, s5, v12, v3 ; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, 1, s5 -; GFX10-NEXT: v_add_co_u32 v8, s5, v8, v9 -; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, s5 -; GFX10-NEXT: v_add_co_u32 v0, s5, v0, v12 +; GFX10-NEXT: v_add_co_u32 v1, s5, v1, v7 +; GFX10-NEXT: v_mul_lo_u32 v0, s15, v2 +; GFX10-NEXT: v_mul_lo_u32 v12, s14, v8 +; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, s5 +; GFX10-NEXT: v_add_co_u32 v3, s5, v3, v4 +; GFX10-NEXT: v_mul_hi_u32 v9, s14, v2 +; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, s5 +; GFX10-NEXT: v_mul_hi_u32 v2, s15, v2 +; GFX10-NEXT: v_mul_lo_u32 v7, s15, v8 +; GFX10-NEXT: v_add_nc_u32_e32 v1, v6, v1 +; GFX10-NEXT: v_add_co_u32 v6, s5, v0, v12 +; GFX10-NEXT: v_mul_hi_u32 v13, s14, v8 +; GFX10-NEXT: v_add_nc_u32_e32 v4, v11, v4 +; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, 1, s5 +; GFX10-NEXT: v_add_co_u32 v12, s5, v3, v1 +; GFX10-NEXT: v_add_co_u32 v2, s20, v7, v2 +; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, s5 +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s5, s6, v12, 0 +; GFX10-NEXT: v_add_co_u32 v6, s5, v6, v9 +; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s5 +; GFX10-NEXT: v_add_co_u32 v9, s5, v2, v13 +; GFX10-NEXT: v_cndmask_b32_e64 v3, 0, 1, s20 +; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, s5 +; GFX10-NEXT: v_add3_u32 v4, v4, v7, v5 +; GFX10-NEXT: v_add_nc_u32_e32 v6, v11, v6 +; GFX10-NEXT: v_mul_hi_u32 v5, s15, v8 +; GFX10-NEXT: v_add_co_u32 v7, vcc_lo, v12, 1 +; GFX10-NEXT: v_add_nc_u32_e32 v3, v3, v2 +; GFX10-NEXT: v_mad_u64_u32 v[1:2], s5, s6, v4, v[1:2] +; GFX10-NEXT: v_add_co_u32 v6, s5, v9, v6 ; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, 1, s5 -; GFX10-NEXT: v_mul_hi_u32 v5, s1, v5 -; GFX10-NEXT: v_add_nc_u32_e32 v8, v10, v8 -; GFX10-NEXT: v_add3_u32 v2, v7, v6, v2 -; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, v4, v1 -; GFX10-NEXT: v_add_nc_u32_e32 v6, v11, v9 -; GFX10-NEXT: v_add_co_u32 v7, s5, v0, v8 -; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s5 -; GFX10-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v3, v2, vcc_lo -; GFX10-NEXT: v_mul_lo_u32 v8, s7, v7 -; GFX10-NEXT: v_mul_lo_u32 v3, s15, v4 -; GFX10-NEXT: v_add3_u32 v5, v6, v0, v5 -; GFX10-NEXT: v_mad_u64_u32 v[0:1], s5, s6, v7, 0 -; GFX10-NEXT: v_mul_lo_u32 v6, s14, v2 -; GFX10-NEXT: v_mul_hi_u32 v10, s14, v4 -; GFX10-NEXT: v_mul_lo_u32 v9, s6, v5 -; GFX10-NEXT: v_mul_hi_u32 v4, s15, v4 -; GFX10-NEXT: v_mul_lo_u32 v11, s15, v2 -; GFX10-NEXT: v_add_co_u32 v3, s5, v3, v6 -; GFX10-NEXT: v_add3_u32 v1, v1, v9, v8 -; GFX10-NEXT: v_sub_co_u32 v6, vcc_lo, s0, v0 -; GFX10-NEXT: v_add_co_u32 v0, s0, v7, 1 +; GFX10-NEXT: v_add_co_ci_u32_e32 v8, vcc_lo, 0, v4, vcc_lo +; GFX10-NEXT: v_add_co_u32 v11, vcc_lo, v7, 1 +; GFX10-NEXT: v_mad_u64_u32 v[1:2], s5, s7, v12, v[1:2] +; GFX10-NEXT: v_add3_u32 v5, v3, v9, v5 +; GFX10-NEXT: v_mad_u64_u32 v[2:3], s5, s2, v6, 0 +; GFX10-NEXT: v_add_co_ci_u32_e32 v13, vcc_lo, 0, v8, vcc_lo +; GFX10-NEXT: v_sub_co_u32 v14, vcc_lo, s0, v0 ; GFX10-NEXT: v_sub_nc_u32_e32 v9, s1, v1 -; GFX10-NEXT: v_add_co_ci_u32_e64 v8, s0, 0, v5, s0 -; GFX10-NEXT: v_sub_co_ci_u32_e64 v12, s0, s1, v1, vcc_lo -; GFX10-NEXT: v_subrev_co_ci_u32_e32 v9, vcc_lo, s7, v9, vcc_lo -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s6, v6 -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s7, v12 -; GFX10-NEXT: v_cndmask_b32_e64 v17, 0, 1, s5 -; GFX10-NEXT: v_add_co_u32 v4, s1, v11, v4 -; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, -1, s0 -; GFX10-NEXT: v_cndmask_b32_e64 v13, 0, -1, vcc_lo -; GFX10-NEXT: v_sub_co_u32 v14, vcc_lo, v6, s6 -; GFX10-NEXT: v_subrev_co_ci_u32_e64 v15, s0, 0, v9, vcc_lo -; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s7, v12 +; GFX10-NEXT: v_sub_co_ci_u32_e64 v15, s0, s1, v1, vcc_lo +; GFX10-NEXT: v_mov_b32_e32 v0, v3 ; GFX10-NEXT: v_subrev_co_ci_u32_e32 v9, vcc_lo, s7, v9, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v1, v13, v1, s0 +; GFX10-NEXT: v_sub_co_u32 v3, vcc_lo, v14, s6 +; GFX10-NEXT: v_subrev_co_ci_u32_e64 v16, s0, 0, v9, vcc_lo ; GFX10-NEXT: v_cmp_le_u32_e64 s0, s6, v14 -; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1 -; GFX10-NEXT: v_cndmask_b32_e64 v13, 0, -1, s0 +; GFX10-NEXT: v_subrev_co_ci_u32_e32 v9, vcc_lo, s7, v9, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v17, 0, -1, s0 +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s6, v3 +; GFX10-NEXT: v_cndmask_b32_e64 v18, 0, -1, s0 +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s7, v16 +; GFX10-NEXT: v_cndmask_b32_e64 v19, 0, -1, s0 ; GFX10-NEXT: v_cmp_le_u32_e64 s0, s7, v15 -; GFX10-NEXT: v_cndmask_b32_e64 v16, 0, -1, s0 -; GFX10-NEXT: v_add_co_u32 v3, s0, v3, v10 -; GFX10-NEXT: v_mul_hi_u32 v10, s14, v2 -; GFX10-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0 +; GFX10-NEXT: v_cndmask_b32_e64 v20, 0, -1, s0 +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s0, s2, v5, v[0:1] +; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s7, v16 +; GFX10-NEXT: v_cndmask_b32_e64 v1, v19, v18, s0 ; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s7, v15 -; GFX10-NEXT: v_mul_hi_u32 v2, s15, v2 -; GFX10-NEXT: v_add_nc_u32_e32 v3, v17, v3 -; GFX10-NEXT: v_cndmask_b32_e64 v11, v16, v13, s0 -; GFX10-NEXT: v_add_co_u32 v4, s0, v4, v10 -; GFX10-NEXT: v_cndmask_b32_e64 v13, 0, 1, s1 -; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v16, s0, v0, 1 -; GFX10-NEXT: v_add_co_ci_u32_e64 v17, s0, 0, v8, s0 -; GFX10-NEXT: v_add_co_u32 v3, s0, v4, v3 -; GFX10-NEXT: v_add_nc_u32_e32 v10, v13, v10 -; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0 -; GFX10-NEXT: v_cmp_ne_u32_e64 s0, 0, v11 -; GFX10-NEXT: v_mul_lo_u32 v13, s3, v3 -; GFX10-NEXT: v_add3_u32 v2, v10, v4, v2 -; GFX10-NEXT: v_cndmask_b32_e64 v11, v0, v16, s0 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v8, v17, s0 -; GFX10-NEXT: v_mad_u64_u32 v[0:1], s1, s2, v3, 0 -; GFX10-NEXT: v_mul_lo_u32 v8, s2, v2 -; GFX10-NEXT: v_sub_co_u32 v10, s1, v14, s6 -; GFX10-NEXT: v_subrev_co_ci_u32_e64 v9, s1, 0, v9, s1 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v10, v14, v10, s0 +; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1 +; GFX10-NEXT: v_cndmask_b32_e64 v17, v20, v17, s0 +; GFX10-NEXT: v_sub_co_u32 v1, s0, v3, s6 +; GFX10-NEXT: v_subrev_co_ci_u32_e64 v9, s0, 0, v9, s0 ; GFX10-NEXT: v_cndmask_b32_e32 v7, v7, v11, vcc_lo -; GFX10-NEXT: v_add3_u32 v1, v1, v8, v13 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v15, v9, s0 -; GFX10-NEXT: v_sub_co_u32 v8, s0, s14, v0 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v6, v10, vcc_lo -; GFX10-NEXT: v_sub_co_ci_u32_e64 v9, s1, s15, v1, s0 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v12, v5, vcc_lo -; GFX10-NEXT: v_sub_nc_u32_e32 v1, s15, v1 -; GFX10-NEXT: v_xor_b32_e32 v0, s18, v7 -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s3, v9 +; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc_lo +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s1, s3, v6, v[0:1] +; GFX10-NEXT: v_cmp_ne_u32_e64 s0, 0, v17 +; GFX10-NEXT: v_cndmask_b32_e32 v8, v8, v13, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v1, v12, v7, s0 +; GFX10-NEXT: v_cndmask_b32_e32 v7, v16, v9, vcc_lo +; GFX10-NEXT: v_sub_co_u32 v2, vcc_lo, s14, v2 +; GFX10-NEXT: v_cndmask_b32_e64 v4, v4, v8, s0 +; GFX10-NEXT: v_sub_co_ci_u32_e64 v8, s1, s15, v0, vcc_lo +; GFX10-NEXT: v_sub_nc_u32_e32 v0, s15, v0 +; GFX10-NEXT: v_cndmask_b32_e64 v3, v14, v3, s0 +; GFX10-NEXT: v_cndmask_b32_e64 v7, v15, v7, s0 +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s3, v8 +; GFX10-NEXT: v_xor_b32_e32 v1, s18, v1 +; GFX10-NEXT: v_subrev_co_ci_u32_e32 v11, vcc_lo, s3, v0, vcc_lo +; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s2, v2 ; GFX10-NEXT: v_xor_b32_e32 v4, s19, v4 -; GFX10-NEXT: v_xor_b32_e32 v5, s4, v5 -; GFX10-NEXT: v_mov_b32_e32 v16, 0 -; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, -1, vcc_lo -; GFX10-NEXT: v_subrev_co_ci_u32_e64 v10, vcc_lo, s3, v1, s0 -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s2, v8 -; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, -1, vcc_lo -; GFX10-NEXT: v_sub_co_u32 v12, vcc_lo, v8, s2 -; GFX10-NEXT: v_subrev_co_ci_u32_e64 v13, s0, 0, v10, vcc_lo -; GFX10-NEXT: v_sub_co_u32 v0, s0, v0, s18 +; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, -1, s0 +; GFX10-NEXT: v_xor_b32_e32 v3, s4, v3 +; GFX10-NEXT: v_xor_b32_e32 v7, s4, v7 +; GFX10-NEXT: v_cndmask_b32_e64 v12, 0, -1, vcc_lo +; GFX10-NEXT: v_sub_co_u32 v13, vcc_lo, v2, s2 +; GFX10-NEXT: v_subrev_co_ci_u32_e64 v14, s0, 0, v11, vcc_lo +; GFX10-NEXT: v_sub_co_u32 v0, s0, v1, s18 ; GFX10-NEXT: v_subrev_co_ci_u32_e64 v1, s0, s19, v4, s0 -; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s3, v9 -; GFX10-NEXT: v_xor_b32_e32 v4, s4, v6 -; GFX10-NEXT: v_subrev_co_ci_u32_e32 v10, vcc_lo, s3, v10, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v6, v7, v11, s0 -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s3, v13 -; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, -1, s0 -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s2, v12 -; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, -1, s0 -; GFX10-NEXT: v_add_co_u32 v14, s0, v3, 1 -; GFX10-NEXT: v_add_co_ci_u32_e64 v15, s0, 0, v2, s0 -; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s3, v13 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v7, v11, s0 -; GFX10-NEXT: v_add_co_u32 v11, s0, v14, 1 -; GFX10-NEXT: v_add_co_ci_u32_e64 v17, s0, 0, v15, s0 -; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v7 -; GFX10-NEXT: v_sub_co_u32 v7, s0, v12, s2 -; GFX10-NEXT: v_subrev_co_ci_u32_e64 v10, s0, 0, v10, s0 -; GFX10-NEXT: v_cndmask_b32_e32 v11, v14, v11, vcc_lo -; GFX10-NEXT: v_cmp_ne_u32_e64 s0, 0, v6 -; GFX10-NEXT: v_cndmask_b32_e32 v14, v15, v17, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v6, v12, v7, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v7, v13, v10, vcc_lo -; GFX10-NEXT: v_sub_co_u32 v4, vcc_lo, v4, s4 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v11, s0 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v14, s0 -; GFX10-NEXT: v_cndmask_b32_e64 v6, v8, v6, s0 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v9, v7, s0 +; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s3, v8 +; GFX10-NEXT: v_subrev_co_ci_u32_e32 v11, vcc_lo, s3, v11, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v4, v9, v12, s0 +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s3, v14 +; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, -1, s0 +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s2, v13 +; GFX10-NEXT: v_cndmask_b32_e64 v12, 0, -1, s0 +; GFX10-NEXT: v_add_co_u32 v15, s0, v6, 1 +; GFX10-NEXT: v_add_co_ci_u32_e64 v16, s0, 0, v5, s0 +; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s3, v14 +; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v12, s0 +; GFX10-NEXT: v_add_co_u32 v12, s0, v15, 1 +; GFX10-NEXT: v_add_co_ci_u32_e64 v17, s0, 0, v16, s0 +; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v9 +; GFX10-NEXT: v_sub_co_u32 v9, s0, v13, s2 +; GFX10-NEXT: v_subrev_co_ci_u32_e64 v11, s0, 0, v11, s0 +; GFX10-NEXT: v_cndmask_b32_e32 v12, v15, v12, vcc_lo +; GFX10-NEXT: v_cmp_ne_u32_e64 s0, 0, v4 +; GFX10-NEXT: v_cndmask_b32_e32 v15, v16, v17, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v4, v13, v9, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v9, v14, v11, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v12, s0 +; GFX10-NEXT: v_cndmask_b32_e64 v11, v5, v15, s0 +; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v4, s0 +; GFX10-NEXT: v_cndmask_b32_e64 v8, v8, v9, s0 ; GFX10-NEXT: s_xor_b64 s[0:1], s[12:13], s[16:17] -; GFX10-NEXT: v_subrev_co_ci_u32_e32 v5, vcc_lo, s4, v5, vcc_lo -; GFX10-NEXT: v_xor_b32_e32 v3, s0, v3 -; GFX10-NEXT: v_xor_b32_e32 v8, s1, v2 -; GFX10-NEXT: v_xor_b32_e32 v6, s12, v6 -; GFX10-NEXT: v_xor_b32_e32 v7, s12, v7 +; GFX10-NEXT: v_sub_co_u32 v4, vcc_lo, v3, s4 +; GFX10-NEXT: v_xor_b32_e32 v3, s0, v6 +; GFX10-NEXT: v_xor_b32_e32 v6, s1, v11 +; GFX10-NEXT: v_subrev_co_ci_u32_e32 v5, vcc_lo, s4, v7, vcc_lo +; GFX10-NEXT: v_xor_b32_e32 v7, s12, v2 +; GFX10-NEXT: v_xor_b32_e32 v8, s12, v8 ; GFX10-NEXT: v_sub_co_u32 v2, vcc_lo, v3, s0 -; GFX10-NEXT: v_subrev_co_ci_u32_e32 v3, vcc_lo, s1, v8, vcc_lo -; GFX10-NEXT: v_sub_co_u32 v6, vcc_lo, v6, s12 -; GFX10-NEXT: v_subrev_co_ci_u32_e32 v7, vcc_lo, s12, v7, vcc_lo -; GFX10-NEXT: global_store_dwordx4 v16, v[0:3], s[8:9] -; GFX10-NEXT: global_store_dwordx4 v16, v[4:7], s[10:11] +; GFX10-NEXT: v_subrev_co_ci_u32_e32 v3, vcc_lo, s1, v6, vcc_lo +; GFX10-NEXT: v_sub_co_u32 v6, vcc_lo, v7, s12 +; GFX10-NEXT: v_subrev_co_ci_u32_e32 v7, vcc_lo, s12, v8, vcc_lo +; GFX10-NEXT: global_store_dwordx4 v10, v[0:3], s[8:9] +; GFX10-NEXT: global_store_dwordx4 v10, v[4:7], s[10:11] ; GFX10-NEXT: s_endpgm %div = sdiv <2 x i64> %x, %y store <2 x i64> %div, ptr addrspace(1) %out0 diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll index 8b4e218f7894..a58397eccaba 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll @@ -389,97 +389,94 @@ define amdgpu_kernel void @udivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1) ; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s11 ; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s10 ; GFX10-NEXT: s_sub_u32 s0, 0, s10 -; GFX10-NEXT: s_subb_u32 s1, 0, s11 ; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f800000, v0 ; GFX10-NEXT: v_add_f32_e32 v0, v0, v1 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX10-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 ; GFX10-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 -; GFX10-NEXT: v_trunc_f32_e32 v1, v1 -; GFX10-NEXT: v_mul_f32_e32 v2, 0xcf800000, v1 -; GFX10-NEXT: v_add_f32_e32 v0, v2, v0 -; GFX10-NEXT: v_cvt_u32_f32_e32 v2, v1 +; GFX10-NEXT: v_trunc_f32_e32 v2, v1 +; GFX10-NEXT: v_mul_f32_e32 v1, 0xcf800000, v2 +; GFX10-NEXT: v_cvt_u32_f32_e32 v4, v2 +; GFX10-NEXT: v_add_f32_e32 v0, v1, v0 ; GFX10-NEXT: v_cvt_u32_f32_e32 v3, v0 -; GFX10-NEXT: v_mul_lo_u32 v4, s0, v2 -; GFX10-NEXT: v_mad_u64_u32 v[0:1], s2, s0, v3, 0 -; GFX10-NEXT: v_mul_lo_u32 v5, s1, v3 -; GFX10-NEXT: v_mul_hi_u32 v6, v2, v0 -; GFX10-NEXT: v_add3_u32 v1, v1, v4, v5 -; GFX10-NEXT: v_mul_lo_u32 v4, v2, v0 +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s1, s0, v3, 0 +; GFX10-NEXT: v_mad_u64_u32 v[1:2], s1, s0, v4, v[1:2] +; GFX10-NEXT: s_subb_u32 s1, 0, s11 +; GFX10-NEXT: v_mul_hi_u32 v6, v4, v0 +; GFX10-NEXT: v_mad_u64_u32 v[1:2], s2, s1, v3, v[1:2] +; GFX10-NEXT: v_mul_lo_u32 v2, v4, v0 ; GFX10-NEXT: v_mul_hi_u32 v0, v3, v0 ; GFX10-NEXT: v_mul_lo_u32 v5, v3, v1 -; GFX10-NEXT: v_mul_lo_u32 v7, v2, v1 +; GFX10-NEXT: v_mul_lo_u32 v7, v4, v1 ; GFX10-NEXT: v_mul_hi_u32 v8, v3, v1 -; GFX10-NEXT: v_mul_hi_u32 v1, v2, v1 -; GFX10-NEXT: v_add_co_u32 v4, s2, v4, v5 +; GFX10-NEXT: v_mul_hi_u32 v1, v4, v1 +; GFX10-NEXT: v_add_co_u32 v2, s2, v2, v5 ; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s2 ; GFX10-NEXT: v_add_co_u32 v6, s2, v7, v6 ; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, s2 -; GFX10-NEXT: v_add_co_u32 v0, s2, v4, v0 +; GFX10-NEXT: v_add_co_u32 v0, s2, v2, v0 ; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s2 -; GFX10-NEXT: v_add_co_u32 v4, s2, v6, v8 +; GFX10-NEXT: v_add_co_u32 v2, s2, v6, v8 ; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s2 ; GFX10-NEXT: v_add_nc_u32_e32 v0, v5, v0 ; GFX10-NEXT: v_add_nc_u32_e32 v5, v7, v6 -; GFX10-NEXT: v_add_co_u32 v0, s2, v4, v0 -; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, s2 +; GFX10-NEXT: v_add_co_u32 v0, s2, v2, v0 +; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, s2 ; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, v3, v0 -; GFX10-NEXT: v_add3_u32 v1, v5, v4, v1 -; GFX10-NEXT: v_mul_lo_u32 v4, s1, v3 -; GFX10-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v1, vcc_lo +; GFX10-NEXT: v_add3_u32 v1, v5, v2, v1 +; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v1, vcc_lo ; GFX10-NEXT: v_mad_u64_u32 v[0:1], s2, s0, v3, 0 -; GFX10-NEXT: v_mul_lo_u32 v5, s0, v2 -; GFX10-NEXT: v_mul_hi_u32 v6, v2, v0 -; GFX10-NEXT: v_add3_u32 v1, v1, v5, v4 -; GFX10-NEXT: v_mul_lo_u32 v4, v2, v0 +; GFX10-NEXT: v_mad_u64_u32 v[1:2], s0, s0, v4, v[1:2] +; GFX10-NEXT: v_mul_hi_u32 v6, v4, v0 +; GFX10-NEXT: v_mad_u64_u32 v[1:2], s0, s1, v3, v[1:2] +; GFX10-NEXT: v_mul_lo_u32 v2, v4, v0 ; GFX10-NEXT: v_mul_hi_u32 v0, v3, v0 ; GFX10-NEXT: v_mul_lo_u32 v5, v3, v1 -; GFX10-NEXT: v_mul_lo_u32 v7, v2, v1 +; GFX10-NEXT: v_mul_lo_u32 v7, v4, v1 ; GFX10-NEXT: v_mul_hi_u32 v8, v3, v1 -; GFX10-NEXT: v_mul_hi_u32 v1, v2, v1 -; GFX10-NEXT: v_add_co_u32 v4, s0, v4, v5 +; GFX10-NEXT: v_mul_hi_u32 v1, v4, v1 +; GFX10-NEXT: v_add_co_u32 v2, s0, v2, v5 ; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0 ; GFX10-NEXT: v_add_co_u32 v6, s0, v7, v6 ; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v0, s0, v4, v0 +; GFX10-NEXT: v_add_co_u32 v0, s0, v2, v0 ; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v4, s0, v6, v8 +; GFX10-NEXT: v_add_co_u32 v2, s0, v6, v8 ; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0 ; GFX10-NEXT: v_add_nc_u32_e32 v0, v5, v0 ; GFX10-NEXT: v_add_nc_u32_e32 v5, v7, v6 -; GFX10-NEXT: v_add_co_u32 v0, s0, v4, v0 -; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v0, s0, v2, v0 +; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 ; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v3, v0 -; GFX10-NEXT: v_add3_u32 v1, v5, v4, v1 -; GFX10-NEXT: v_mul_hi_u32 v4, s9, v0 -; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v2, v1, vcc_lo +; GFX10-NEXT: v_add3_u32 v1, v5, v2, v1 ; GFX10-NEXT: v_mul_lo_u32 v2, s9, v0 -; GFX10-NEXT: v_mul_hi_u32 v0, s8, v0 +; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v4, v1, vcc_lo +; GFX10-NEXT: v_mul_hi_u32 v4, s8, v0 +; GFX10-NEXT: v_mul_hi_u32 v0, s9, v0 ; GFX10-NEXT: v_mul_lo_u32 v3, s8, v1 ; GFX10-NEXT: v_mul_lo_u32 v5, s9, v1 -; GFX10-NEXT: v_mul_hi_u32 v6, s8, v1 -; GFX10-NEXT: v_mul_hi_u32 v1, s9, v1 ; GFX10-NEXT: v_add_co_u32 v2, s0, v2, v3 -; GFX10-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v4, s0, v5, v4 -; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v0, s0, v2, v0 -; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v2, s0, v4, v6 +; GFX10-NEXT: v_mul_hi_u32 v3, s8, v1 +; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v2, s0, v2, v4 +; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v0, s0, v5, v0 ; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0 -; GFX10-NEXT: v_add_nc_u32_e32 v0, v3, v0 -; GFX10-NEXT: v_add_nc_u32_e32 v3, v5, v4 -; GFX10-NEXT: v_add_co_u32 v2, s0, v2, v0 -; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0 -; GFX10-NEXT: v_mul_lo_u32 v4, s11, v2 -; GFX10-NEXT: v_add3_u32 v3, v3, v0, v1 -; GFX10-NEXT: v_mad_u64_u32 v[0:1], s0, s10, v2, 0 -; GFX10-NEXT: v_mul_lo_u32 v5, s10, v3 -; GFX10-NEXT: v_add3_u32 v1, v1, v5, v4 -; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, v2, 1 -; GFX10-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, 0, v3, vcc_lo -; GFX10-NEXT: v_sub_nc_u32_e32 v6, s9, v1 +; GFX10-NEXT: v_add_nc_u32_e32 v2, v6, v2 +; GFX10-NEXT: v_add_co_u32 v0, s0, v0, v3 +; GFX10-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v5, s0, v0, v2 +; GFX10-NEXT: v_mul_hi_u32 v2, s9, v1 +; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0 +; GFX10-NEXT: v_add_nc_u32_e32 v3, v4, v3 +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s0, s10, v5, 0 +; GFX10-NEXT: v_add3_u32 v3, v3, v6, v2 +; GFX10-NEXT: v_mad_u64_u32 v[1:2], s0, s10, v3, v[1:2] +; GFX10-NEXT: v_mad_u64_u32 v[1:2], s0, s11, v5, v[1:2] +; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v5, 1 +; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v3, vcc_lo ; GFX10-NEXT: v_sub_co_u32 v7, vcc_lo, s8, v0 +; GFX10-NEXT: v_sub_nc_u32_e32 v6, s9, v1 ; GFX10-NEXT: v_sub_co_ci_u32_e64 v8, s0, s9, v1, vcc_lo ; GFX10-NEXT: v_subrev_co_ci_u32_e32 v0, vcc_lo, s11, v6, vcc_lo ; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s10, v7 @@ -493,8 +490,8 @@ define amdgpu_kernel void @udivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1) ; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, -1, s0 ; GFX10-NEXT: v_cmp_le_u32_e64 s0, s11, v9 ; GFX10-NEXT: v_cndmask_b32_e64 v12, 0, -1, s0 -; GFX10-NEXT: v_add_co_u32 v13, s0, v4, 1 -; GFX10-NEXT: v_add_co_ci_u32_e64 v14, s0, 0, v5, s0 +; GFX10-NEXT: v_add_co_u32 v13, s0, v2, 1 +; GFX10-NEXT: v_add_co_ci_u32_e64 v14, s0, 0, v4, s0 ; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s11, v9 ; GFX10-NEXT: v_cndmask_b32_e64 v11, v12, v11, s0 ; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s11, v8 @@ -502,14 +499,14 @@ define amdgpu_kernel void @udivrem_i64(ptr addrspace(1) %out0, ptr addrspace(1) ; GFX10-NEXT: v_cndmask_b32_e64 v1, v10, v1, s0 ; GFX10-NEXT: v_sub_co_u32 v10, s0, v6, s10 ; GFX10-NEXT: v_subrev_co_ci_u32_e64 v0, s0, 0, v0, s0 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v13, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v5, v5, v14, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v13, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v14, vcc_lo ; GFX10-NEXT: v_cmp_ne_u32_e64 s0, 0, v1 ; GFX10-NEXT: v_cndmask_b32_e32 v6, v6, v10, vcc_lo ; GFX10-NEXT: v_cndmask_b32_e32 v9, v9, v0, vcc_lo ; GFX10-NEXT: v_mov_b32_e32 v10, 0 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v2, v4, s0 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v3, v5, s0 +; GFX10-NEXT: v_cndmask_b32_e64 v0, v5, v2, s0 +; GFX10-NEXT: v_cndmask_b32_e64 v1, v3, v4, s0 ; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v6, s0 ; GFX10-NEXT: v_cndmask_b32_e64 v3, v8, v9, s0 ; GFX10-NEXT: global_store_dwordx2 v10, v[0:1], s[4:5] @@ -1523,8 +1520,6 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1 ; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f800000, v0 ; GFX10-NEXT: v_mul_f32_e32 v1, 0x4f800000, v1 ; GFX10-NEXT: s_subb_u32 s1, 0, s13 -; GFX10-NEXT: s_sub_u32 s2, 0, s14 -; GFX10-NEXT: s_subb_u32 s3, 0, s15 ; GFX10-NEXT: v_add_f32_e32 v0, v0, v2 ; GFX10-NEXT: v_add_f32_e32 v1, v1, v3 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0 @@ -1533,243 +1528,239 @@ define amdgpu_kernel void @udivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1 ; GFX10-NEXT: v_mul_f32_e32 v1, 0x5f7ffffc, v1 ; GFX10-NEXT: v_mul_f32_e32 v2, 0x2f800000, v0 ; GFX10-NEXT: v_mul_f32_e32 v3, 0x2f800000, v1 -; GFX10-NEXT: v_trunc_f32_e32 v2, v2 -; GFX10-NEXT: v_trunc_f32_e32 v3, v3 -; GFX10-NEXT: v_mul_f32_e32 v4, 0xcf800000, v2 -; GFX10-NEXT: v_mul_f32_e32 v5, 0xcf800000, v3 -; GFX10-NEXT: v_cvt_u32_f32_e32 v6, v3 -; GFX10-NEXT: v_add_f32_e32 v0, v4, v0 -; GFX10-NEXT: v_add_f32_e32 v1, v5, v1 -; GFX10-NEXT: v_cvt_u32_f32_e32 v4, v2 -; GFX10-NEXT: v_mul_lo_u32 v10, s2, v6 -; GFX10-NEXT: v_cvt_u32_f32_e32 v5, v0 +; GFX10-NEXT: v_trunc_f32_e32 v4, v2 +; GFX10-NEXT: v_trunc_f32_e32 v5, v3 +; GFX10-NEXT: v_mul_f32_e32 v2, 0xcf800000, v4 +; GFX10-NEXT: v_mul_f32_e32 v3, 0xcf800000, v5 +; GFX10-NEXT: v_cvt_u32_f32_e32 v9, v4 +; GFX10-NEXT: v_cvt_u32_f32_e32 v10, v5 +; GFX10-NEXT: v_add_f32_e32 v0, v2, v0 +; GFX10-NEXT: v_add_f32_e32 v1, v3, v1 +; GFX10-NEXT: v_cvt_u32_f32_e32 v7, v0 ; GFX10-NEXT: v_cvt_u32_f32_e32 v8, v1 -; GFX10-NEXT: v_mul_lo_u32 v7, s0, v4 -; GFX10-NEXT: v_mad_u64_u32 v[0:1], s6, s0, v5, 0 -; GFX10-NEXT: v_mul_lo_u32 v9, s1, v5 -; GFX10-NEXT: v_mad_u64_u32 v[2:3], s6, s2, v8, 0 -; GFX10-NEXT: v_mul_lo_u32 v11, s3, v8 -; GFX10-NEXT: v_add3_u32 v1, v1, v7, v9 -; GFX10-NEXT: v_mul_lo_u32 v7, v4, v0 -; GFX10-NEXT: v_mul_hi_u32 v9, v5, v0 -; GFX10-NEXT: v_add3_u32 v3, v3, v10, v11 -; GFX10-NEXT: v_mul_hi_u32 v0, v4, v0 -; GFX10-NEXT: v_mul_lo_u32 v12, v5, v1 -; GFX10-NEXT: v_mul_lo_u32 v13, v4, v1 -; GFX10-NEXT: v_mul_lo_u32 v10, v6, v2 -; GFX10-NEXT: v_mul_lo_u32 v15, v8, v3 -; GFX10-NEXT: v_mul_hi_u32 v11, v8, v2 -; GFX10-NEXT: v_mul_hi_u32 v2, v6, v2 -; GFX10-NEXT: v_mul_lo_u32 v16, v6, v3 -; GFX10-NEXT: v_mul_hi_u32 v14, v5, v1 -; GFX10-NEXT: v_add_co_u32 v7, s6, v7, v12 +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s2, s0, v7, 0 +; GFX10-NEXT: s_sub_u32 s2, 0, s14 +; GFX10-NEXT: v_mad_u64_u32 v[2:3], s3, s2, v8, 0 +; GFX10-NEXT: v_mul_hi_u32 v11, v9, v0 +; GFX10-NEXT: v_mad_u64_u32 v[4:5], s3, s0, v9, v[1:2] +; GFX10-NEXT: v_mad_u64_u32 v[5:6], s3, s2, v10, v[3:4] +; GFX10-NEXT: v_mul_lo_u32 v6, v9, v0 +; GFX10-NEXT: s_subb_u32 s3, 0, s15 +; GFX10-NEXT: v_mad_u64_u32 v[3:4], s6, s1, v7, v[4:5] +; GFX10-NEXT: v_mul_hi_u32 v4, v7, v0 +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s6, s3, v8, v[5:6] +; GFX10-NEXT: v_mul_lo_u32 v1, v10, v2 +; GFX10-NEXT: v_mul_hi_u32 v5, v8, v2 +; GFX10-NEXT: v_mul_hi_u32 v2, v10, v2 +; GFX10-NEXT: v_mul_lo_u32 v12, v7, v3 +; GFX10-NEXT: v_mul_lo_u32 v13, v9, v3 +; GFX10-NEXT: v_mul_hi_u32 v14, v7, v3 +; GFX10-NEXT: v_mul_lo_u32 v15, v8, v0 +; GFX10-NEXT: v_mul_lo_u32 v16, v10, v0 +; GFX10-NEXT: v_mul_hi_u32 v17, v8, v0 +; GFX10-NEXT: v_mul_hi_u32 v3, v9, v3 +; GFX10-NEXT: v_mul_hi_u32 v0, v10, v0 +; GFX10-NEXT: v_add_co_u32 v6, s6, v6, v12 ; GFX10-NEXT: v_cndmask_b32_e64 v12, 0, 1, s6 -; GFX10-NEXT: v_add_co_u32 v0, s6, v13, v0 +; GFX10-NEXT: v_add_co_u32 v11, s6, v13, v11 ; GFX10-NEXT: v_cndmask_b32_e64 v13, 0, 1, s6 -; GFX10-NEXT: v_add_co_u32 v10, s6, v10, v15 +; GFX10-NEXT: v_add_co_u32 v1, s6, v1, v15 ; GFX10-NEXT: v_cndmask_b32_e64 v15, 0, 1, s6 ; GFX10-NEXT: v_add_co_u32 v2, s6, v16, v2 ; GFX10-NEXT: v_cndmask_b32_e64 v16, 0, 1, s6 -; GFX10-NEXT: v_add_co_u32 v7, s6, v7, v9 -; GFX10-NEXT: v_mul_hi_u32 v17, v8, v3 -; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, s6 -; GFX10-NEXT: v_add_co_u32 v0, s6, v0, v14 -; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, 1, s6 -; GFX10-NEXT: v_add_co_u32 v10, s6, v10, v11 -; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, 1, s6 -; GFX10-NEXT: v_add_nc_u32_e32 v7, v12, v7 -; GFX10-NEXT: v_add_co_u32 v2, s6, v2, v17 -; GFX10-NEXT: v_mul_hi_u32 v1, v4, v1 -; GFX10-NEXT: v_add_nc_u32_e32 v10, v15, v10 +; GFX10-NEXT: v_add_co_u32 v4, s6, v6, v4 +; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, s6 +; GFX10-NEXT: v_add_co_u32 v6, s6, v11, v14 ; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, 1, s6 -; GFX10-NEXT: v_add_co_u32 v0, s6, v0, v7 -; GFX10-NEXT: v_add_nc_u32_e32 v9, v13, v9 -; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, s6 -; GFX10-NEXT: v_mul_hi_u32 v3, v6, v3 -; GFX10-NEXT: v_add_co_u32 v2, s6, v2, v10 -; GFX10-NEXT: v_add_nc_u32_e32 v11, v16, v11 -; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, 1, s6 -; GFX10-NEXT: v_add3_u32 v1, v9, v7, v1 -; GFX10-NEXT: v_add_co_u32 v5, vcc_lo, v5, v0 -; GFX10-NEXT: v_add3_u32 v3, v11, v10, v3 -; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v1, vcc_lo -; GFX10-NEXT: v_add_co_u32 v8, vcc_lo, v8, v2 -; GFX10-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, v6, v3, vcc_lo -; GFX10-NEXT: v_mad_u64_u32 v[0:1], s6, s0, v5, 0 -; GFX10-NEXT: v_mul_lo_u32 v7, s1, v5 -; GFX10-NEXT: v_mul_lo_u32 v9, s0, v4 -; GFX10-NEXT: v_mad_u64_u32 v[2:3], s0, s2, v8, 0 -; GFX10-NEXT: v_mul_lo_u32 v10, s3, v8 -; GFX10-NEXT: v_mul_lo_u32 v11, s2, v6 -; GFX10-NEXT: v_mul_lo_u32 v12, v4, v0 -; GFX10-NEXT: v_mul_hi_u32 v13, v5, v0 -; GFX10-NEXT: v_mul_hi_u32 v0, v4, v0 -; GFX10-NEXT: v_add3_u32 v1, v1, v9, v7 -; GFX10-NEXT: v_mul_lo_u32 v7, v6, v2 -; GFX10-NEXT: v_mul_hi_u32 v9, v8, v2 -; GFX10-NEXT: v_mul_hi_u32 v2, v6, v2 -; GFX10-NEXT: v_add3_u32 v3, v3, v11, v10 -; GFX10-NEXT: v_mul_lo_u32 v10, v5, v1 -; GFX10-NEXT: v_mul_lo_u32 v11, v4, v1 -; GFX10-NEXT: v_mul_hi_u32 v14, v5, v1 +; GFX10-NEXT: v_add_co_u32 v1, s6, v1, v5 +; GFX10-NEXT: v_add_nc_u32_e32 v4, v12, v4 +; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, s6 +; GFX10-NEXT: v_add_co_u32 v2, s6, v2, v17 +; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s6 +; GFX10-NEXT: v_add_co_u32 v4, s6, v6, v4 +; GFX10-NEXT: v_add_nc_u32_e32 v1, v15, v1 +; GFX10-NEXT: v_add_nc_u32_e32 v11, v13, v11 +; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s6 +; GFX10-NEXT: v_add_nc_u32_e32 v5, v16, v5 +; GFX10-NEXT: v_add_co_u32 v7, vcc_lo, v7, v4 +; GFX10-NEXT: v_add_co_u32 v1, s6, v2, v1 +; GFX10-NEXT: v_add3_u32 v3, v11, v6, v3 +; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, s6 +; GFX10-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v9, v3, vcc_lo +; GFX10-NEXT: v_add3_u32 v2, v5, v2, v0 +; GFX10-NEXT: v_add_co_u32 v8, vcc_lo, v8, v1 +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s6, s0, v7, 0 +; GFX10-NEXT: v_add_co_ci_u32_e32 v10, vcc_lo, v10, v2, vcc_lo +; GFX10-NEXT: v_mad_u64_u32 v[2:3], s6, s2, v8, 0 ; GFX10-NEXT: s_load_dwordx8 s[4:11], s[4:5], 0x0 -; GFX10-NEXT: v_mul_lo_u32 v15, v8, v3 -; GFX10-NEXT: v_mul_lo_u32 v16, v6, v3 -; GFX10-NEXT: v_mul_hi_u32 v17, v8, v3 -; GFX10-NEXT: v_mul_hi_u32 v1, v4, v1 -; GFX10-NEXT: v_add_co_u32 v10, s0, v12, v10 +; GFX10-NEXT: v_mul_hi_u32 v11, v9, v0 +; GFX10-NEXT: v_mad_u64_u32 v[4:5], s0, s0, v9, v[1:2] +; GFX10-NEXT: v_mad_u64_u32 v[5:6], s0, s2, v10, v[3:4] +; GFX10-NEXT: v_mul_lo_u32 v6, v9, v0 +; GFX10-NEXT: v_mad_u64_u32 v[3:4], s0, s1, v7, v[4:5] +; GFX10-NEXT: v_mul_hi_u32 v4, v7, v0 +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s0, s3, v8, v[5:6] +; GFX10-NEXT: v_mul_lo_u32 v1, v10, v2 +; GFX10-NEXT: v_mul_hi_u32 v5, v8, v2 +; GFX10-NEXT: v_mul_hi_u32 v2, v10, v2 +; GFX10-NEXT: v_mul_lo_u32 v12, v7, v3 +; GFX10-NEXT: v_mul_lo_u32 v13, v9, v3 +; GFX10-NEXT: v_mul_hi_u32 v14, v7, v3 +; GFX10-NEXT: v_mul_lo_u32 v15, v8, v0 +; GFX10-NEXT: v_mul_lo_u32 v16, v10, v0 +; GFX10-NEXT: v_mul_hi_u32 v17, v8, v0 +; GFX10-NEXT: v_mul_hi_u32 v3, v9, v3 +; GFX10-NEXT: v_mul_hi_u32 v0, v10, v0 +; GFX10-NEXT: v_add_co_u32 v6, s0, v6, v12 ; GFX10-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v0, s0, v11, v0 -; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v7, s0, v7, v15 +; GFX10-NEXT: v_add_co_u32 v11, s0, v13, v11 +; GFX10-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v1, s0, v1, v15 ; GFX10-NEXT: v_cndmask_b32_e64 v15, 0, 1, s0 ; GFX10-NEXT: v_add_co_u32 v2, s0, v16, v2 ; GFX10-NEXT: v_cndmask_b32_e64 v16, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v10, s0, v10, v13 -; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v0, s0, v0, v14 -; GFX10-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v7, s0, v7, v9 -; GFX10-NEXT: v_add_nc_u32_e32 v9, v12, v10 -; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v4, s0, v6, v4 +; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v6, s0, v11, v14 +; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v1, s0, v1, v5 +; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0 +; GFX10-NEXT: v_add_nc_u32_e32 v4, v12, v4 ; GFX10-NEXT: v_add_co_u32 v2, s0, v2, v17 -; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v0, s0, v0, v9 -; GFX10-NEXT: v_add_nc_u32_e32 v11, v11, v13 -; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, 1, s0 -; GFX10-NEXT: v_add_nc_u32_e32 v7, v15, v7 -; GFX10-NEXT: v_mul_hi_u32 v3, v6, v3 -; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v5, v0 -; GFX10-NEXT: v_add3_u32 v1, v11, v9, v1 -; GFX10-NEXT: v_add_co_u32 v2, s0, v2, v7 -; GFX10-NEXT: v_add_nc_u32_e32 v10, v16, v10 -; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0 -; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v4, v1, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0 +; GFX10-NEXT: v_add_nc_u32_e32 v1, v15, v1 +; GFX10-NEXT: v_add_co_u32 v4, s0, v6, v4 +; GFX10-NEXT: v_add_nc_u32_e32 v11, v13, v11 +; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v1, s0, v2, v1 +; GFX10-NEXT: v_add_nc_u32_e32 v5, v16, v5 +; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 +; GFX10-NEXT: v_add3_u32 v3, v11, v6, v3 +; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, v7, v4 +; GFX10-NEXT: v_add3_u32 v0, v5, v2, v0 +; GFX10-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v9, v3, vcc_lo +; GFX10-NEXT: v_add_co_u32 v1, vcc_lo, v8, v1 +; GFX10-NEXT: v_add_co_ci_u32_e32 v0, vcc_lo, v10, v0, vcc_lo ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_mul_lo_u32 v4, s9, v0 -; GFX10-NEXT: v_mul_hi_u32 v5, s8, v0 -; GFX10-NEXT: v_add3_u32 v3, v10, v7, v3 -; GFX10-NEXT: v_mul_lo_u32 v7, s8, v1 -; GFX10-NEXT: v_mul_hi_u32 v0, s9, v0 -; GFX10-NEXT: v_mul_lo_u32 v9, s9, v1 -; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v8, v2 -; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v6, v3, vcc_lo -; GFX10-NEXT: v_mul_hi_u32 v6, s8, v1 -; GFX10-NEXT: v_add_co_u32 v4, s0, v4, v7 -; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v0, s0, v9, v0 +; GFX10-NEXT: v_mul_lo_u32 v3, s9, v4 +; GFX10-NEXT: v_mul_lo_u32 v8, s8, v2 +; GFX10-NEXT: v_mul_hi_u32 v5, s8, v4 +; GFX10-NEXT: v_mul_hi_u32 v4, s9, v4 +; GFX10-NEXT: v_mul_lo_u32 v9, s9, v2 +; GFX10-NEXT: v_mul_lo_u32 v6, s11, v1 +; GFX10-NEXT: v_mul_hi_u32 v10, s8, v2 +; GFX10-NEXT: v_mul_hi_u32 v11, s9, v2 +; GFX10-NEXT: v_mul_lo_u32 v2, s10, v0 +; GFX10-NEXT: v_mul_hi_u32 v7, s10, v1 +; GFX10-NEXT: v_mul_hi_u32 v1, s11, v1 +; GFX10-NEXT: v_mul_lo_u32 v12, s11, v0 +; GFX10-NEXT: v_mul_hi_u32 v13, s10, v0 +; GFX10-NEXT: v_mul_hi_u32 v14, s11, v0 +; GFX10-NEXT: v_add_co_u32 v0, s0, v3, v8 +; GFX10-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v4, s0, v9, v4 ; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v4, s0, v4, v5 -; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v0, s0, v0, v6 -; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0 -; GFX10-NEXT: v_mul_hi_u32 v1, s9, v1 -; GFX10-NEXT: v_add_nc_u32_e32 v4, v7, v4 -; GFX10-NEXT: v_mul_lo_u32 v6, s11, v2 -; GFX10-NEXT: v_mul_lo_u32 v7, s10, v3 -; GFX10-NEXT: v_add_nc_u32_e32 v5, v8, v5 -; GFX10-NEXT: v_mul_hi_u32 v8, s10, v2 -; GFX10-NEXT: v_add_co_u32 v4, s0, v0, v4 +; GFX10-NEXT: v_add_co_u32 v2, s0, v6, v2 +; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v1, s0, v12, v1 +; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v0, s0, v0, v5 ; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0 -; GFX10-NEXT: v_mul_hi_u32 v2, s11, v2 -; GFX10-NEXT: v_mul_lo_u32 v9, s11, v3 -; GFX10-NEXT: v_mul_hi_u32 v10, s10, v3 -; GFX10-NEXT: v_add_co_u32 v6, s0, v6, v7 -; GFX10-NEXT: v_add3_u32 v5, v5, v0, v1 +; GFX10-NEXT: v_add_co_u32 v4, s0, v4, v10 +; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v2, s0, v2, v7 +; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 +; GFX10-NEXT: v_add_nc_u32_e32 v0, v3, v0 +; GFX10-NEXT: v_add_co_u32 v1, s0, v1, v13 ; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0 -; GFX10-NEXT: v_mad_u64_u32 v[0:1], s0, s12, v4, 0 -; GFX10-NEXT: v_mul_lo_u32 v11, s13, v4 -; GFX10-NEXT: v_mul_lo_u32 v12, s12, v5 -; GFX10-NEXT: v_add_co_u32 v2, s0, v9, v2 -; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v6, s0, v6, v8 +; GFX10-NEXT: v_add_nc_u32_e32 v2, v6, v2 +; GFX10-NEXT: v_add_nc_u32_e32 v5, v8, v5 +; GFX10-NEXT: v_add_co_u32 v8, s0, v4, v0 +; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v10, s0, v1, v2 +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s1, s12, v8, 0 ; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v2, s0, v2, v10 -; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, s0 -; GFX10-NEXT: v_add3_u32 v1, v1, v12, v11 -; GFX10-NEXT: v_add_nc_u32_e32 v6, v7, v6 -; GFX10-NEXT: v_mul_hi_u32 v3, s11, v3 -; GFX10-NEXT: v_mov_b32_e32 v10, 0 -; GFX10-NEXT: v_add_nc_u32_e32 v7, v9, v8 -; GFX10-NEXT: v_sub_nc_u32_e32 v8, s9, v1 -; GFX10-NEXT: v_sub_co_u32 v9, vcc_lo, s8, v0 -; GFX10-NEXT: v_sub_co_ci_u32_e64 v11, s0, s9, v1, vcc_lo -; GFX10-NEXT: v_subrev_co_ci_u32_e32 v0, vcc_lo, s13, v8, vcc_lo -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s12, v9 -; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, -1, vcc_lo -; GFX10-NEXT: v_sub_co_u32 v8, vcc_lo, v9, s12 -; GFX10-NEXT: v_subrev_co_ci_u32_e64 v12, s0, 0, v0, vcc_lo -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s13, v11 -; GFX10-NEXT: v_cndmask_b32_e64 v13, 0, -1, s0 -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s12, v8 -; GFX10-NEXT: v_cndmask_b32_e64 v14, 0, -1, s0 -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s13, v12 -; GFX10-NEXT: v_cndmask_b32_e64 v15, 0, -1, s0 -; GFX10-NEXT: v_add_co_u32 v16, s0, v4, 1 -; GFX10-NEXT: v_add_co_ci_u32_e64 v17, s0, 0, v5, s0 -; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s13, v11 -; GFX10-NEXT: v_cndmask_b32_e64 v13, v13, v1, s0 -; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s13, v12 -; GFX10-NEXT: v_cndmask_b32_e64 v14, v15, v14, s0 -; GFX10-NEXT: v_add_co_u32 v2, s0, v2, v6 -; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v6, s0, v16, 1 -; GFX10-NEXT: v_add_co_ci_u32_e64 v15, s0, 0, v17, s0 -; GFX10-NEXT: v_add3_u32 v3, v7, v1, v3 -; GFX10-NEXT: v_subrev_co_ci_u32_e32 v7, vcc_lo, s13, v0, vcc_lo -; GFX10-NEXT: v_mad_u64_u32 v[0:1], s0, s14, v2, 0 -; GFX10-NEXT: v_mul_lo_u32 v18, s14, v3 -; GFX10-NEXT: v_mul_lo_u32 v19, s15, v2 -; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v14 -; GFX10-NEXT: v_sub_co_u32 v14, s0, v8, s12 -; GFX10-NEXT: v_subrev_co_ci_u32_e64 v7, s0, 0, v7, s0 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v16, v6, vcc_lo -; GFX10-NEXT: v_cmp_ne_u32_e64 s0, 0, v13 -; GFX10-NEXT: v_add3_u32 v16, v1, v18, v19 -; GFX10-NEXT: v_cndmask_b32_e32 v15, v17, v15, vcc_lo -; GFX10-NEXT: v_sub_co_u32 v13, s1, s10, v0 -; GFX10-NEXT: v_cndmask_b32_e64 v0, v4, v6, s0 -; GFX10-NEXT: v_sub_nc_u32_e32 v4, s11, v16 -; GFX10-NEXT: v_sub_co_ci_u32_e64 v17, s2, s11, v16, s1 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v5, v15, s0 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v14, vcc_lo -; GFX10-NEXT: v_subrev_co_ci_u32_e64 v8, s1, s15, v4, s1 -; GFX10-NEXT: v_cmp_le_u32_e64 s1, s14, v13 -; GFX10-NEXT: v_cmp_le_u32_e64 s2, s15, v17 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v12, v7, vcc_lo -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, s15, v17 -; GFX10-NEXT: v_cndmask_b32_e64 v4, v9, v5, s0 -; GFX10-NEXT: v_cndmask_b32_e64 v14, 0, -1, s1 -; GFX10-NEXT: v_sub_co_u32 v15, s1, v13, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, -1, s2 -; GFX10-NEXT: v_subrev_co_ci_u32_e64 v16, s2, 0, v8, s1 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v6, v14, vcc_lo -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s15, v16 -; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc_lo +; GFX10-NEXT: v_mad_u64_u32 v[2:3], s0, s14, v10, 0 +; GFX10-NEXT: v_add_nc_u32_e32 v7, v9, v7 +; GFX10-NEXT: v_add3_u32 v9, v5, v4, v11 +; GFX10-NEXT: v_add_co_u32 v12, vcc_lo, v8, 1 +; GFX10-NEXT: v_mov_b32_e32 v11, 0 +; GFX10-NEXT: v_add3_u32 v7, v7, v6, v14 +; GFX10-NEXT: v_mad_u64_u32 v[4:5], s0, s12, v9, v[1:2] +; GFX10-NEXT: v_add_co_ci_u32_e32 v13, vcc_lo, 0, v9, vcc_lo +; GFX10-NEXT: v_mad_u64_u32 v[5:6], s0, s14, v7, v[3:4] +; GFX10-NEXT: v_mad_u64_u32 v[3:4], s0, s13, v8, v[4:5] +; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, v12, 1 +; GFX10-NEXT: v_add_co_ci_u32_e32 v6, vcc_lo, 0, v13, vcc_lo +; GFX10-NEXT: v_sub_co_u32 v14, vcc_lo, s8, v0 +; GFX10-NEXT: v_mad_u64_u32 v[0:1], s0, s15, v10, v[5:6] +; GFX10-NEXT: v_sub_co_ci_u32_e64 v5, s0, s9, v3, vcc_lo +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s12, v14 +; GFX10-NEXT: v_sub_nc_u32_e32 v1, s9, v3 +; GFX10-NEXT: v_cndmask_b32_e64 v3, 0, -1, s0 +; GFX10-NEXT: v_sub_co_u32 v15, s0, s10, v2 +; GFX10-NEXT: v_subrev_co_ci_u32_e32 v1, vcc_lo, s13, v1, vcc_lo +; GFX10-NEXT: v_sub_co_ci_u32_e64 v16, s1, s11, v0, s0 ; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s14, v15 -; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, -1, vcc_lo -; GFX10-NEXT: v_add_co_u32 v12, vcc_lo, v2, 1 -; GFX10-NEXT: v_add_co_ci_u32_e32 v14, vcc_lo, 0, v3, vcc_lo -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, s15, v16 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v6, v9, vcc_lo -; GFX10-NEXT: v_add_co_u32 v9, vcc_lo, v12, 1 -; GFX10-NEXT: v_add_co_ci_u32_e32 v18, vcc_lo, 0, v14, vcc_lo -; GFX10-NEXT: v_subrev_co_ci_u32_e64 v8, vcc_lo, s15, v8, s1 -; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v6 +; GFX10-NEXT: v_sub_nc_u32_e32 v0, s11, v0 +; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc_lo +; GFX10-NEXT: v_sub_co_u32 v17, vcc_lo, v14, s12 +; GFX10-NEXT: v_subrev_co_ci_u32_e64 v18, s1, 0, v1, vcc_lo +; GFX10-NEXT: v_cmp_le_u32_e64 s1, s13, v5 +; GFX10-NEXT: v_subrev_co_ci_u32_e64 v23, s0, s15, v0, s0 +; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s13, v18 +; GFX10-NEXT: v_subrev_co_ci_u32_e32 v1, vcc_lo, s13, v1, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v19, 0, -1, s1 +; GFX10-NEXT: v_cmp_le_u32_e64 s1, s12, v17 +; GFX10-NEXT: v_cndmask_b32_e64 v20, 0, -1, s1 +; GFX10-NEXT: v_cmp_le_u32_e64 s1, s13, v18 +; GFX10-NEXT: v_cndmask_b32_e64 v21, 0, -1, s1 +; GFX10-NEXT: v_cmp_le_u32_e64 s1, s15, v16 +; GFX10-NEXT: v_cndmask_b32_e64 v0, v21, v20, s0 +; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s13, v5 +; GFX10-NEXT: v_cndmask_b32_e64 v22, 0, -1, s1 +; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0 +; GFX10-NEXT: v_cndmask_b32_e64 v3, v19, v3, s0 +; GFX10-NEXT: v_sub_co_u32 v0, s0, v17, s12 +; GFX10-NEXT: v_subrev_co_ci_u32_e64 v19, s0, 0, v1, s0 +; GFX10-NEXT: v_cndmask_b32_e32 v1, v12, v4, vcc_lo +; GFX10-NEXT: v_cmp_ne_u32_e64 s0, 0, v3 +; GFX10-NEXT: v_cndmask_b32_e32 v3, v13, v6, vcc_lo ; GFX10-NEXT: v_sub_co_u32 v6, s1, v15, s14 -; GFX10-NEXT: v_subrev_co_ci_u32_e64 v8, s1, 0, v8, s1 -; GFX10-NEXT: v_cndmask_b32_e32 v9, v12, v9, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v12, v14, v18, vcc_lo -; GFX10-NEXT: v_cmp_ne_u32_e64 s1, 0, v5 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v15, v6, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v8, v16, v8, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v5, v11, v7, s0 -; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v9, s1 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v12, s1 -; GFX10-NEXT: v_cndmask_b32_e64 v6, v13, v6, s1 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v17, v8, s1 -; GFX10-NEXT: global_store_dwordx4 v10, v[0:3], s[4:5] -; GFX10-NEXT: global_store_dwordx4 v10, v[4:7], s[6:7] +; GFX10-NEXT: v_cndmask_b32_e32 v4, v17, v0, vcc_lo +; GFX10-NEXT: v_subrev_co_ci_u32_e64 v12, s2, 0, v23, s1 +; GFX10-NEXT: v_cndmask_b32_e64 v0, v8, v1, s0 +; GFX10-NEXT: v_cndmask_b32_e64 v1, v9, v3, s0 +; GFX10-NEXT: v_cndmask_b32_e32 v3, v18, v19, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, s15, v16 +; GFX10-NEXT: v_cndmask_b32_e64 v4, v14, v4, s0 +; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v3, s0 +; GFX10-NEXT: v_cndmask_b32_e32 v2, v22, v2, vcc_lo +; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s15, v12 +; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc_lo +; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s14, v6 +; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, -1, vcc_lo +; GFX10-NEXT: v_add_co_u32 v13, vcc_lo, v10, 1 +; GFX10-NEXT: v_add_co_ci_u32_e32 v14, vcc_lo, 0, v7, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, s15, v12 +; GFX10-NEXT: v_cndmask_b32_e32 v8, v8, v9, vcc_lo +; GFX10-NEXT: v_add_co_u32 v9, vcc_lo, v13, 1 +; GFX10-NEXT: v_add_co_ci_u32_e32 v17, vcc_lo, 0, v14, vcc_lo +; GFX10-NEXT: v_subrev_co_ci_u32_e64 v18, vcc_lo, s15, v23, s1 +; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8 +; GFX10-NEXT: v_sub_co_u32 v8, s1, v6, s14 +; GFX10-NEXT: v_subrev_co_ci_u32_e64 v18, s1, 0, v18, s1 +; GFX10-NEXT: v_cndmask_b32_e32 v9, v13, v9, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v13, v14, v17, vcc_lo +; GFX10-NEXT: v_cmp_ne_u32_e64 s1, 0, v2 +; GFX10-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v8, v12, v18, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v2, v10, v9, s1 +; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v13, s1 +; GFX10-NEXT: v_cndmask_b32_e64 v6, v15, v6, s1 +; GFX10-NEXT: v_cndmask_b32_e64 v7, v16, v8, s1 +; GFX10-NEXT: global_store_dwordx4 v11, v[0:3], s[4:5] +; GFX10-NEXT: global_store_dwordx4 v11, v[4:7], s[6:7] ; GFX10-NEXT: s_endpgm %div = udiv <2 x i64> %x, %y store <2 x i64> %div, ptr addrspace(1) %out0 diff --git a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll index 8874240fae8d..526ee5a51745 100644 --- a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll +++ b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll @@ -4418,30 +4418,28 @@ define i64 @clpeak_imad_pat_i64(i64 %x, i64 %y) { ; GFX10-GISEL-LABEL: clpeak_imad_pat_i64: ; GFX10-GISEL: ; %bb.0: ; %entry ; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-GISEL-NEXT: v_add_co_u32 v4, vcc_lo, v0, 1 -; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo -; GFX10-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v4, v2, 0 -; GFX10-GISEL-NEXT: v_mul_lo_u32 v6, v4, v3 -; GFX10-GISEL-NEXT: v_mul_lo_u32 v7, v5, v2 -; GFX10-GISEL-NEXT: v_add3_u32 v1, v1, v6, v7 -; GFX10-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v4 -; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v1, v5, vcc_lo -; GFX10-GISEL-NEXT: v_mad_u64_u32 v[4:5], null, v6, v2, 0 -; GFX10-GISEL-NEXT: v_mul_lo_u32 v3, v6, v3 -; GFX10-GISEL-NEXT: v_mul_lo_u32 v2, v7, v2 -; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 1 -; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo -; GFX10-GISEL-NEXT: v_mul_lo_u32 v1, v4, v1 -; GFX10-GISEL-NEXT: v_add3_u32 v5, v5, v3, v2 -; GFX10-GISEL-NEXT: v_mad_u64_u32 v[2:3], null, v4, v0, 0 -; GFX10-GISEL-NEXT: v_add_co_u32 v4, vcc_lo, v4, 1 -; GFX10-GISEL-NEXT: v_mul_lo_u32 v0, v5, v0 -; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, 0, v5, vcc_lo -; GFX10-GISEL-NEXT: v_add3_u32 v3, v3, v1, v0 -; GFX10-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v2, v4, 0 -; GFX10-GISEL-NEXT: v_mul_lo_u32 v2, v2, v5 -; GFX10-GISEL-NEXT: v_mul_lo_u32 v3, v3, v4 -; GFX10-GISEL-NEXT: v_add3_u32 v1, v1, v2, v3 +; GFX10-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, 1 +; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, 0, v1, vcc_lo +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v6, v2, 0 +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[4:5], null, v6, v3, v[1:2] +; GFX10-GISEL-NEXT: v_add_co_u32 v8, vcc_lo, v0, v6 +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[4:5], null, v7, v2, v[4:5] +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[5:6], null, v8, v2, 0 +; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, v4, v7, vcc_lo +; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, v6 +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[6:7], null, v8, v3, v[1:2] +; GFX10-GISEL-NEXT: v_add_co_u32 v8, vcc_lo, v0, 1 +; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v10, vcc_lo, 0, v4, vcc_lo +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[3:4], null, v5, v8, 0 +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[6:7], null, v9, v2, v[6:7] +; GFX10-GISEL-NEXT: v_add_co_u32 v7, vcc_lo, v5, 1 +; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, v4 +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v3, v7, 0 +; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, 0, v6, vcc_lo +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[4:5], null, v5, v10, v[2:3] +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[1:2], null, v3, v9, v[1:2] +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[4:5], null, v6, v8, v[4:5] +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[1:2], null, v4, v7, v[1:2] ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-SDAG-LABEL: clpeak_imad_pat_i64: @@ -4479,38 +4477,36 @@ define i64 @clpeak_imad_pat_i64(i64 %x, i64 %y) { ; GFX11-GISEL-LABEL: clpeak_imad_pat_i64: ; GFX11-GISEL: ; %bb.0: ; %entry ; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-GISEL-NEXT: v_add_co_u32 v4, vcc_lo, v0, 1 -; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo +; GFX11-GISEL-NEXT: v_add_co_u32 v7, vcc_lo, v0, 1 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v8, vcc_lo, 0, v1, vcc_lo +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v7, v2, 0 +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[4:5], null, v7, v3, v[1:2] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[5:6], null, v8, v2, v[4:5] +; GFX11-GISEL-NEXT: v_add_co_u32 v4, vcc_lo, v0, v7 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[6:7], null, v4, v2, 0 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v10, vcc_lo, v5, v8, vcc_lo +; GFX11-GISEL-NEXT: v_add_co_u32 v11, vcc_lo, v0, 1 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v12, vcc_lo, 0, v5, vcc_lo +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, v7 +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[7:8], null, v4, v3, v[1:2] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[3:4], null, v6, v11, 0 +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[8:9], null, v10, v2, v[7:8] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, v4 +; GFX11-GISEL-NEXT: v_add_co_u32 v9, vcc_lo, v6, 1 +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[4:5], null, v6, v12, v[2:3] ; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v4, v2, 0 -; GFX11-GISEL-NEXT: v_mul_lo_u32 v6, v4, v3 -; GFX11-GISEL-NEXT: v_mul_lo_u32 v7, v5, v2 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v6, v7 -; GFX11-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v4 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v1, v5, vcc_lo -; GFX11-GISEL-NEXT: v_mad_u64_u32 v[4:5], null, v6, v2, 0 -; GFX11-GISEL-NEXT: v_mul_lo_u32 v3, v6, v3 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v7, v2 -; GFX11-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 1 -; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo -; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v4, v1 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-GISEL-NEXT: v_add3_u32 v5, v5, v3, v2 -; GFX11-GISEL-NEXT: v_mad_u64_u32 v[2:3], null, v4, v0, 0 -; GFX11-GISEL-NEXT: v_add_co_u32 v4, vcc_lo, v4, 1 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v5, v0 -; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, 0, v5, vcc_lo -; GFX11-GISEL-NEXT: v_add3_u32 v3, v3, v1, v0 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v2, v4, 0 -; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v2, v5 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-NEXT: v_mul_lo_u32 v3, v3, v4 -; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v2, v3 +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v3, v9, 0 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v10, vcc_lo, 0, v8, vcc_lo +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[5:6], null, v8, v11, v[4:5] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[6:7], null, v3, v10, v[1:2] +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[1:2], null, v5, v9, v[6:7] ; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y18 = add i64 %x, 1 @@ -5116,54 +5112,51 @@ define <2 x i64> @clpeak_imad_pat_v2i64(<2 x i64> %x, <2 x i64> %y) { ; GFX10-GISEL-LABEL: clpeak_imad_pat_v2i64: ; GFX10-GISEL: ; %bb.0: ; %entry ; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-GISEL-NEXT: v_add_co_u32 v8, vcc_lo, v0, 1 -; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, 0, v1, vcc_lo -; GFX10-GISEL-NEXT: v_add_co_u32 v10, vcc_lo, v2, 1 -; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v11, vcc_lo, 0, v3, vcc_lo -; GFX10-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v8, v4, 0 -; GFX10-GISEL-NEXT: v_mul_lo_u32 v12, v8, v5 -; GFX10-GISEL-NEXT: v_mul_lo_u32 v13, v9, v4 -; GFX10-GISEL-NEXT: v_mad_u64_u32 v[2:3], null, v10, v6, 0 -; GFX10-GISEL-NEXT: v_mul_lo_u32 v14, v10, v7 -; GFX10-GISEL-NEXT: v_mul_lo_u32 v15, v11, v6 -; GFX10-GISEL-NEXT: v_add3_u32 v1, v1, v12, v13 -; GFX10-GISEL-NEXT: v_add3_u32 v12, v3, v14, v15 -; GFX10-GISEL-NEXT: v_add_co_u32 v3, vcc_lo, v0, v8 -; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v13, vcc_lo, v1, v9, vcc_lo -; GFX10-GISEL-NEXT: v_add_co_u32 v10, vcc_lo, v2, v10 -; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v11, vcc_lo, v12, v11, vcc_lo -; GFX10-GISEL-NEXT: v_mad_u64_u32 v[8:9], null, v3, v4, 0 -; GFX10-GISEL-NEXT: v_mul_lo_u32 v5, v3, v5 -; GFX10-GISEL-NEXT: v_mul_lo_u32 v13, v13, v4 -; GFX10-GISEL-NEXT: v_mad_u64_u32 v[3:4], null, v10, v6, 0 -; GFX10-GISEL-NEXT: v_mul_lo_u32 v7, v10, v7 -; GFX10-GISEL-NEXT: v_mul_lo_u32 v6, v11, v6 -; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 1 -; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo -; GFX10-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, 1 -; GFX10-GISEL-NEXT: v_add3_u32 v9, v9, v5, v13 -; GFX10-GISEL-NEXT: v_add3_u32 v10, v4, v7, v6 -; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v11, vcc_lo, 0, v12, vcc_lo -; GFX10-GISEL-NEXT: v_mad_u64_u32 v[4:5], null, v8, v0, 0 -; GFX10-GISEL-NEXT: v_mul_lo_u32 v1, v8, v1 -; GFX10-GISEL-NEXT: v_mul_lo_u32 v0, v9, v0 -; GFX10-GISEL-NEXT: v_mad_u64_u32 v[6:7], null, v3, v2, 0 -; GFX10-GISEL-NEXT: v_mul_lo_u32 v11, v3, v11 -; GFX10-GISEL-NEXT: v_mul_lo_u32 v2, v10, v2 -; GFX10-GISEL-NEXT: v_add_co_u32 v8, vcc_lo, v8, 1 -; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, 0, v9, vcc_lo -; GFX10-GISEL-NEXT: v_add_co_u32 v12, vcc_lo, v3, 1 -; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v10, vcc_lo, 0, v10, vcc_lo -; GFX10-GISEL-NEXT: v_add3_u32 v3, v5, v1, v0 -; GFX10-GISEL-NEXT: v_add3_u32 v5, v7, v11, v2 -; GFX10-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v4, v8, 0 -; GFX10-GISEL-NEXT: v_mul_lo_u32 v4, v4, v9 -; GFX10-GISEL-NEXT: v_mul_lo_u32 v7, v3, v8 -; GFX10-GISEL-NEXT: v_mad_u64_u32 v[2:3], null, v6, v12, 0 -; GFX10-GISEL-NEXT: v_mul_lo_u32 v6, v6, v10 -; GFX10-GISEL-NEXT: v_mul_lo_u32 v5, v5, v12 -; GFX10-GISEL-NEXT: v_add3_u32 v1, v1, v4, v7 -; GFX10-GISEL-NEXT: v_add3_u32 v3, v3, v6, v5 +; GFX10-GISEL-NEXT: v_add_co_u32 v12, vcc_lo, v0, 1 +; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v13, vcc_lo, 0, v1, vcc_lo +; GFX10-GISEL-NEXT: v_add_co_u32 v14, vcc_lo, v2, 1 +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v12, v4, 0 +; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v15, vcc_lo, 0, v3, vcc_lo +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[2:3], null, v14, v6, 0 +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[8:9], null, v12, v5, v[1:2] +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[9:10], null, v14, v7, v[3:4] +; GFX10-GISEL-NEXT: v_add_co_u32 v3, vcc_lo, v0, v12 +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[10:11], null, v13, v4, v[8:9] +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[11:12], null, v3, v4, 0 +; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v16, vcc_lo, v10, v13, vcc_lo +; GFX10-GISEL-NEXT: v_add_co_u32 v17, vcc_lo, v2, v14 +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[8:9], null, v15, v6, v[9:10] +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[13:14], null, v17, v6, 0 +; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v18, vcc_lo, v8, v15, vcc_lo +; GFX10-GISEL-NEXT: v_add_co_u32 v19, vcc_lo, v0, 1 +; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, v12 +; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, v14 +; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v20, vcc_lo, 0, v10, vcc_lo +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[9:10], null, v11, v19, 0 +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[14:15], null, v3, v5, v[0:1] +; GFX10-GISEL-NEXT: v_add_co_u32 v15, vcc_lo, v2, 1 +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v17, v7, v[1:2] +; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v12, vcc_lo, 0, v8, vcc_lo +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[7:8], null, v13, v15, 0 +; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, v10 +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[4:5], null, v16, v4, v[14:15] +; GFX10-GISEL-NEXT: v_add_co_u32 v14, vcc_lo, v11, 1 +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[5:6], null, v18, v6, v[0:1] +; GFX10-GISEL-NEXT: v_mov_b32_e32 v6, v8 +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[10:11], null, v11, v20, v[1:2] +; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v16, vcc_lo, 0, v4, vcc_lo +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[11:12], null, v13, v12, v[6:7] +; GFX10-GISEL-NEXT: v_add_co_u32 v17, vcc_lo, v13, 1 +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v9, v14, 0 +; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v18, vcc_lo, 0, v5, vcc_lo +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[12:13], null, v4, v19, v[10:11] +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[2:3], null, v7, v17, 0 +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[4:5], null, v5, v15, v[11:12] +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[5:6], null, v9, v16, v[1:2] +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[7:8], null, v7, v18, v[3:4] +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[5:6], null, v12, v14, v[5:6] +; GFX10-GISEL-NEXT: v_mad_u64_u32 v[3:4], null, v4, v17, v[7:8] +; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-SDAG-LABEL: clpeak_imad_pat_v2i64: @@ -5224,60 +5217,62 @@ define <2 x i64> @clpeak_imad_pat_v2i64(<2 x i64> %x, <2 x i64> %y) { ; GFX11-GISEL-LABEL: clpeak_imad_pat_v2i64: ; GFX11-GISEL: ; %bb.0: ; %entry ; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-GISEL-NEXT: v_add_co_u32 v8, vcc_lo, v0, 1 -; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, 0, v1, vcc_lo -; GFX11-GISEL-NEXT: v_add_co_u32 v10, vcc_lo, v2, 1 -; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v11, vcc_lo, 0, v3, vcc_lo -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v8, v4, 0 -; GFX11-GISEL-NEXT: v_mul_lo_u32 v12, v8, v5 -; GFX11-GISEL-NEXT: v_mul_lo_u32 v13, v9, v4 -; GFX11-GISEL-NEXT: v_mad_u64_u32 v[2:3], null, v10, v6, 0 -; GFX11-GISEL-NEXT: v_mul_lo_u32 v14, v10, v7 -; GFX11-GISEL-NEXT: v_mul_lo_u32 v15, v11, v6 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v12, v13 -; GFX11-GISEL-NEXT: v_add3_u32 v12, v3, v14, v15 -; GFX11-GISEL-NEXT: v_add_co_u32 v3, vcc_lo, v0, v8 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v13, vcc_lo, v1, v9, vcc_lo -; GFX11-GISEL-NEXT: v_add_co_u32 v10, vcc_lo, v2, v10 -; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v11, vcc_lo, v12, v11, vcc_lo -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_add_co_u32 v13, vcc_lo, v0, 1 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v14, vcc_lo, 0, v1, vcc_lo +; GFX11-GISEL-NEXT: v_add_co_u32 v15, vcc_lo, v2, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v13, v4, 0 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v16, vcc_lo, 0, v3, vcc_lo +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[2:3], null, v15, v6, 0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[8:9], null, v13, v5, v[1:2] +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[9:10], null, v15, v7, v[3:4] +; GFX11-GISEL-NEXT: v_add_co_u32 v3, vcc_lo, v0, v13 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[10:11], null, v14, v4, v[8:9] +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[11:12], null, v16, v6, v[9:10] +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v17, vcc_lo, v10, v14, vcc_lo +; GFX11-GISEL-NEXT: v_add_co_u32 v18, vcc_lo, v2, v15 ; GFX11-GISEL-NEXT: v_mad_u64_u32 v[8:9], null, v3, v4, 0 -; GFX11-GISEL-NEXT: v_mul_lo_u32 v5, v3, v5 -; GFX11-GISEL-NEXT: v_mul_lo_u32 v13, v13, v4 -; GFX11-GISEL-NEXT: v_mad_u64_u32 v[3:4], null, v10, v6, 0 -; GFX11-GISEL-NEXT: v_mul_lo_u32 v7, v10, v7 -; GFX11-GISEL-NEXT: v_mul_lo_u32 v6, v11, v6 -; GFX11-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 1 -; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo -; GFX11-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, 1 -; GFX11-GISEL-NEXT: v_add3_u32 v9, v9, v5, v13 -; GFX11-GISEL-NEXT: v_add3_u32 v10, v4, v7, v6 -; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v11, vcc_lo, 0, v12, vcc_lo -; GFX11-GISEL-NEXT: v_mad_u64_u32 v[4:5], null, v8, v0, 0 -; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v8, v1 -; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v9, v0 -; GFX11-GISEL-NEXT: v_mad_u64_u32 v[6:7], null, v3, v2, 0 -; GFX11-GISEL-NEXT: v_mul_lo_u32 v11, v3, v11 -; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v10, v2 -; GFX11-GISEL-NEXT: v_add_co_u32 v8, vcc_lo, v8, 1 -; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, 0, v9, vcc_lo -; GFX11-GISEL-NEXT: v_add_co_u32 v12, vcc_lo, v3, 1 -; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v10, vcc_lo, 0, v10, vcc_lo -; GFX11-GISEL-NEXT: v_add3_u32 v3, v5, v1, v0 -; GFX11-GISEL-NEXT: v_add3_u32 v5, v7, v11, v2 -; GFX11-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v4, v8, 0 -; GFX11-GISEL-NEXT: v_mul_lo_u32 v4, v4, v9 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) -; GFX11-GISEL-NEXT: v_mul_lo_u32 v7, v3, v8 -; GFX11-GISEL-NEXT: v_mad_u64_u32 v[2:3], null, v6, v12, 0 -; GFX11-GISEL-NEXT: v_mul_lo_u32 v6, v6, v10 -; GFX11-GISEL-NEXT: v_mul_lo_u32 v5, v5, v12 -; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v4, v7 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v19, vcc_lo, v11, v16, vcc_lo +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[12:13], null, v18, v6, 0 +; GFX11-GISEL-NEXT: v_add_co_u32 v20, vcc_lo, v0, 1 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v21, vcc_lo, 0, v10, vcc_lo +; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, v9 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[9:10], null, v8, v20, 0 +; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, v13 +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[13:14], null, v3, v5, v[0:1] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, v10 +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[14:15], null, v18, v7, v[1:2] +; GFX11-GISEL-NEXT: v_add_co_u32 v18, vcc_lo, v2, 1 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v22, vcc_lo, 0, v11, vcc_lo +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[15:16], null, v17, v4, v[13:14] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[4:5], null, v12, v18, 0 +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[10:11], null, v19, v6, v[14:15] +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[6:7], null, v8, v21, v[0:1] +; GFX11-GISEL-NEXT: v_add_co_u32 v14, vcc_lo, v8, 1 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v16, vcc_lo, 0, v15, vcc_lo +; GFX11-GISEL-NEXT: v_add_co_u32 v17, vcc_lo, v12, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[7:8], null, v12, v22, v[5:6] +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v9, v14, 0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[2:3], null, v4, v17, 0 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v19, vcc_lo, 0, v10, vcc_lo +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[11:12], null, v15, v20, v[6:7] +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[12:13], null, v10, v18, v[7:8] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[7:8], null, v9, v16, v[1:2] +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[8:9], null, v4, v19, v[3:4] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[5:6], null, v11, v14, v[7:8] +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[3:4], null, v12, v17, v[8:9] ; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-GISEL-NEXT: v_add3_u32 v3, v3, v6, v5 +; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, v5 ; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y18 = add <2 x i64> %x, diff --git a/llvm/test/TableGen/GlobalISelEmitter-multiple-output-reject.td b/llvm/test/TableGen/GlobalISelEmitter-multiple-output-reject.td deleted file mode 100644 index bce2d8cb096b..000000000000 --- a/llvm/test/TableGen/GlobalISelEmitter-multiple-output-reject.td +++ /dev/null @@ -1,14 +0,0 @@ -// RUN: llvm-tblgen -gen-global-isel -optimize-match-table=false -warn-on-skipped-patterns -I %p/../../include -I %p/Common %s -o - < %s 2>&1 | FileCheck %s - -include "llvm/Target/Target.td" -include "GlobalISelEmitterCommon.td" - -// Test when the inner instruction in the output pattern has two outs - -def TwoOutsInstr : I<(outs GPR32:$out1, GPR32:$out2), (ins GPR32:$src), []>; -def OtherInstr : I<(outs GPR32:$dst), (ins GPR32:$src), []>; - -def : Pat<(i32 (add i32:$src, i32:$src)), - (OtherInstr (TwoOutsInstr GPR32:$src))>; - -// CHECK: warning: Skipped pattern: Dst pattern child only supported with exactly one result diff --git a/llvm/test/TableGen/GlobalISelEmitter-multiple-output.td b/llvm/test/TableGen/GlobalISelEmitter-multiple-output.td index e063ebd4c2bb..94c9f60eabd3 100644 --- a/llvm/test/TableGen/GlobalISelEmitter-multiple-output.td +++ b/llvm/test/TableGen/GlobalISelEmitter-multiple-output.td @@ -117,3 +117,33 @@ def : Pat<(i32 (add i32:$src, i32:$src)), // CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0, // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, + +//----------------------------------------------------------------------------- +// Test when the inner instruction in the output pattern has two outs + +// CHECK: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_ADD), +// CHECK-NEXT: // MIs[0] DstI[dst] +// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, +// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), +// CHECK-NEXT: // MIs[0] src +// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, +// CHECK-NEXT: // MIs[0] src +// CHECK-NEXT: GIM_CheckIsSameOperand, /*MI*/0, /*OpIdx*/2, /*OtherMI*/0, /*OtherOpIdx*/1, +// CHECK-NEXT: // (add:{ *:[i32] } i32:{ *:[i32] }:$src, i32:{ *:[i32] }:$src) => (OtherInstr:{ *:[i32] } (TwoOutsInstr:{ *:[i32] }:{ *:[i32] } GPR32:{ *:[i32] }:$src)) +// CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/1, /*TypeID*/GILLT_s32, +// CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32, +// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(MyTarget::TwoOutsInstr), +// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define), +// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/1, /*TempRegFlags*/GIMT_Encode2(RegState::Define|RegState::Dead), +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/1, // src +// CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/1, +// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::OtherInstr), +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] +// CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0, +// CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, +// CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, + +def TwoOutsInstr : I<(outs GPR32:$out1, GPR32:$out2), (ins GPR32:$src), []>; + +def : Pat<(i32 (add i32:$src, i32:$src)), + (OtherInstr (TwoOutsInstr GPR32:$src))>; diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp index 6ca6cfb18ccd..e86057422cd7 100644 --- a/llvm/utils/TableGen/GlobalISelEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -295,9 +295,8 @@ static Expected getInstResultType(const TreePatternNode &Dst, // below, we only expect one explicit def here. assert(Dst.getOperator()->isSubClassOf("Instruction")); CodeGenInstruction &InstInfo = Target.getInstruction(Dst.getOperator()); - if (InstInfo.Operands.NumDefs != 1) - return failedImport( - "Dst pattern child only supported with exactly one result"); + if (!InstInfo.Operands.NumDefs) + return failedImport("Dst pattern child needs a def"); ArrayRef ChildTypes = Dst.getExtTypes(); if (ChildTypes.size() < 1) @@ -408,9 +407,11 @@ private: createInstructionRenderer(action_iterator InsertPt, RuleMatcher &M, const TreePatternNode &Dst); - Expected importExplicitDefRenderers( - action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder, - const TreePatternNode &Src, const TreePatternNode &Dst); + Expected + importExplicitDefRenderers(action_iterator InsertPt, RuleMatcher &M, + BuildMIAction &DstMIBuilder, + const TreePatternNode &Src, + const TreePatternNode &Dst, unsigned Start = 0); Expected importExplicitUseRenderers( action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder, @@ -1379,6 +1380,14 @@ GlobalISelEmitter::createAndImportSubInstructionRenderer( // Assign the result to TempReg. DstMIBuilder.addRenderer(TempRegID, true); + // Handle additional (ignored) results. + if (DstMIBuilder.getCGI()->Operands.NumDefs > 1) { + InsertPtOrError = importExplicitDefRenderers( + std::prev(*InsertPtOrError), M, DstMIBuilder, Src, Dst, /*Start=*/1); + if (auto Error = InsertPtOrError.takeError()) + return std::move(Error); + } + InsertPtOrError = importExplicitUseRenderers(InsertPtOrError.get(), M, DstMIBuilder, Dst, Src); if (auto Error = InsertPtOrError.takeError()) @@ -1507,14 +1516,14 @@ Expected GlobalISelEmitter::createInstructionRenderer( Expected GlobalISelEmitter::importExplicitDefRenderers( action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder, - const TreePatternNode &Src, const TreePatternNode &Dst) { + const TreePatternNode &Src, const TreePatternNode &Dst, unsigned Start) { const CodeGenInstruction *DstI = DstMIBuilder.getCGI(); const unsigned SrcNumDefs = Src.getExtTypes().size(); const unsigned DstNumDefs = DstI->Operands.NumDefs; if (DstNumDefs == 0) return InsertPt; - for (unsigned I = 0; I < SrcNumDefs; ++I) { + for (unsigned I = Start; I < SrcNumDefs; ++I) { std::string OpName = getMangledRootDefName(DstI->Operands[I].Name); // CopyRenderer saves a StringRef, so cannot pass OpName itself - // let's use a string with an appropriate lifetime. @@ -1800,8 +1809,6 @@ GlobalISelEmitter::inferRegClassFromPattern(const TreePatternNode &N) { // Don't want to try and infer things when there could potentially be more // than one candidate register class. auto &Inst = Target.getInstruction(OpRec); - if (Inst.Operands.NumDefs > 1) - return std::nullopt; // Handle any special-case instructions which we can safely infer register // classes from. -- GitLab From 6a8e6c9a31933dec276fe26e18090832a9c5d0d0 Mon Sep 17 00:00:00 2001 From: Guillaume Chatelet Date: Fri, 8 Mar 2024 09:41:23 +0100 Subject: [PATCH 574/929] [libc][NFC] Move `BigInt` out of the `cpp` namespace (#84445) As noted in https://github.com/llvm/llvm-project/pull/84035#discussion_r1516817755 only files under the CPP folder should be in the `cpp` namespace. --- libc/src/__support/FPUtil/dyadic_float.h | 2 +- libc/src/__support/UInt.h | 67 +++++++++--------- libc/src/__support/UInt128.h | 4 +- libc/src/__support/float_to_string.h | 70 +++++++++---------- libc/src/__support/integer_literals.h | 8 +-- libc/src/__support/integer_to_string.h | 6 +- libc/src/__support/str_to_float.h | 2 +- .../stdio/printf_core/float_dec_converter.h | 10 +-- libc/test/UnitTest/LibcTest.cpp | 12 ++-- libc/test/UnitTest/LibcTest.h | 10 +-- libc/test/UnitTest/StringUtils.h | 2 +- libc/test/UnitTest/TestLogger.cpp | 10 +-- libc/test/src/__support/CPP/bit_test.cpp | 2 +- libc/test/src/__support/CPP/limits_test.cpp | 7 +- .../src/__support/integer_literals_test.cpp | 2 +- .../src/__support/integer_to_string_test.cpp | 2 +- libc/test/src/__support/uint_test.cpp | 26 +++---- 17 files changed, 119 insertions(+), 123 deletions(-) diff --git a/libc/src/__support/FPUtil/dyadic_float.h b/libc/src/__support/FPUtil/dyadic_float.h index f25fa9b3026c..73fd7381c3c8 100644 --- a/libc/src/__support/FPUtil/dyadic_float.h +++ b/libc/src/__support/FPUtil/dyadic_float.h @@ -31,7 +31,7 @@ namespace LIBC_NAMESPACE::fputil { // To simplify and improve the efficiency, many functions will assume that the // inputs are normal. template struct DyadicFloat { - using MantissaType = LIBC_NAMESPACE::cpp::UInt; + using MantissaType = LIBC_NAMESPACE::UInt; Sign sign = Sign::POS; int exponent = 0; diff --git a/libc/src/__support/UInt.h b/libc/src/__support/UInt.h index e899a79684b7..cf0c5a669ae8 100644 --- a/libc/src/__support/UInt.h +++ b/libc/src/__support/UInt.h @@ -23,16 +23,16 @@ #include // For size_t #include -namespace LIBC_NAMESPACE::cpp { +namespace LIBC_NAMESPACE { namespace internal { template struct half_width; -template <> struct half_width : type_identity {}; -template <> struct half_width : type_identity {}; -template <> struct half_width : type_identity {}; +template <> struct half_width : cpp::type_identity {}; +template <> struct half_width : cpp::type_identity {}; +template <> struct half_width : cpp::type_identity {}; #ifdef __SIZEOF_INT128__ -template <> struct half_width<__uint128_t> : type_identity {}; +template <> struct half_width<__uint128_t> : cpp::type_identity {}; #endif // __SIZEOF_INT128__ template using half_width_t = typename half_width::type; @@ -40,7 +40,7 @@ template using half_width_t = typename half_width::type; template struct BigInt { - static_assert(is_integral_v && is_unsigned_v, + static_assert(cpp::is_integral_v && cpp::is_unsigned_v, "WordType must be unsigned integer."); using word_type = WordType; @@ -76,7 +76,7 @@ struct BigInt { WordType sign = 0; if constexpr (Signed && OtherSigned) { sign = static_cast( - -static_cast>(other.is_neg())); + -static_cast>(other.is_neg())); } for (; i < WORD_COUNT; ++i) val[i] = sign; @@ -84,7 +84,7 @@ struct BigInt { } // Construct a BigInt from a C array. - template = 0> + template = 0> LIBC_INLINE constexpr BigInt(const WordType (&nums)[N]) { size_t min_wordcount = N < WORD_COUNT ? N : WORD_COUNT; size_t i = 0; @@ -97,7 +97,7 @@ struct BigInt { } // Initialize the first word to |v| and the rest to 0. - template >> + template >> LIBC_INLINE constexpr BigInt(T v) { val[0] = static_cast(v); @@ -406,7 +406,7 @@ struct BigInt { // div takes another BigInt of the same size and divides this by it. The value // of this will be set to the quotient, and the return value is the remainder. - LIBC_INLINE constexpr optional div(const BigInt &other) { + LIBC_INLINE constexpr cpp::optional div(const BigInt &other) { BigInt remainder(0); if (*this < other) { remainder = *this; @@ -417,7 +417,7 @@ struct BigInt { return remainder; } if (other == 0) { - return nullopt; + return cpp::nullopt; } BigInt quotient(0); @@ -448,12 +448,12 @@ struct BigInt { // Since the remainder of each division step < x < 2^(WORD_SIZE / 2), the // computation of each step is now properly contained within WordType. // And finally we perform some extra alignment steps for the remaining bits. - LIBC_INLINE constexpr optional + LIBC_INLINE constexpr cpp::optional div_uint_half_times_pow_2(internal::half_width_t x, size_t e) { BigInt remainder(0); if (x == 0) { - return nullopt; + return cpp::nullopt; } if (e >= Bits) { remainder = *this; @@ -463,7 +463,7 @@ struct BigInt { BigInt quotient(0); WordType x_word = static_cast(x); - constexpr size_t LOG2_WORD_SIZE = bit_width(WORD_SIZE) - 1; + constexpr size_t LOG2_WORD_SIZE = cpp::bit_width(WORD_SIZE) - 1; constexpr size_t HALF_WORD_SIZE = WORD_SIZE >> 1; constexpr WordType HALF_MASK = ((WordType(1) << HALF_WORD_SIZE) - 1); // lower = smallest multiple of WORD_SIZE that is >= e. @@ -592,7 +592,7 @@ struct BigInt { int leading_zeroes = 0; for (auto i = val.size(); i > 0;) { --i; - const int zeroes = countl_zero(val[i]); + const int zeroes = cpp::countl_zero(val[i]); leading_zeroes += zeroes; if (zeroes != word_digits) break; @@ -605,7 +605,7 @@ struct BigInt { constexpr int word_digits = cpp::numeric_limits::digits; int trailing_zeroes = 0; for (auto word : val) { - const int zeroes = countr_zero(word); + const int zeroes = cpp::countr_zero(word); trailing_zeroes += zeroes; if (zeroes != word_digits) break; @@ -913,7 +913,7 @@ template using Int = BigInt>; // Provides limits of U/Int<128>. -template <> class numeric_limits> { +template <> class cpp::numeric_limits> { public: LIBC_INLINE static constexpr UInt<128> max() { return UInt<128>({0xffff'ffff'ffff'ffff, 0xffff'ffff'ffff'ffff}); @@ -924,7 +924,7 @@ public: LIBC_INLINE_VAR static constexpr int digits = 128; }; -template <> class numeric_limits> { +template <> class cpp::numeric_limits> { public: LIBC_INLINE static constexpr Int<128> max() { return Int<128>({0xffff'ffff'ffff'ffff, 0x7fff'ffff'ffff'ffff}); @@ -937,7 +937,7 @@ public: LIBC_INLINE_VAR static constexpr int digits = 128; }; -// type traits to determine whether a T is a cpp::BigInt. +// type traits to determine whether a T is a BigInt. template struct is_big_int : cpp::false_type {}; template @@ -946,6 +946,8 @@ struct is_big_int> : cpp::true_type {}; template LIBC_INLINE_VAR constexpr bool is_big_int_v = is_big_int::value; +namespace cpp { + // Specialization of cpp::bit_cast ('bit.h') from T to BigInt. template LIBC_INLINE constexpr cpp::enable_if_t< @@ -973,7 +975,7 @@ bit_cast(const UInt &from) { // Specialization of cpp::has_single_bit ('bit.h') for BigInt. template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, bool> +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, bool> has_single_bit(T value) { int bits = 0; for (auto word : value.val) { @@ -988,21 +990,21 @@ has_single_bit(T value) { // Specialization of cpp::countr_zero ('bit.h') for BigInt. template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> countr_zero(const T &value) { return value.ctz(); } // Specialization of cpp::countl_zero ('bit.h') for BigInt. template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> countl_zero(const T &value) { return value.clz(); } // Specialization of cpp::countl_one ('bit.h') for BigInt. template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> countl_one(T value) { // TODO : Implement a faster version not involving operator~. return cpp::countl_zero(~value); @@ -1010,7 +1012,7 @@ countl_one(T value) { // Specialization of cpp::countr_one ('bit.h') for BigInt. template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> countr_one(T value) { // TODO : Implement a faster version not involving operator~. return cpp::countr_zero(~value); @@ -1018,19 +1020,19 @@ countr_one(T value) { // Specialization of cpp::bit_width ('bit.h') for BigInt. template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> bit_width(T value) { return cpp::numeric_limits::digits - cpp::countl_zero(value); } // Forward-declare rotr so that rotl can use it. template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> rotr(T value, int rotate); // Specialization of cpp::rotl ('bit.h') for BigInt. template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> rotl(T value, int rotate) { constexpr unsigned N = cpp::numeric_limits::digits; rotate = rotate % N; @@ -1043,7 +1045,7 @@ rotl(T value, int rotate) { // Specialization of cpp::rotr ('bit.h') for BigInt. template -[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> +[[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> rotr(T value, int rotate) { constexpr unsigned N = cpp::numeric_limits::digits; rotate = rotate % N; @@ -1054,13 +1056,11 @@ rotr(T value, int rotate) { return (value >> rotate) | (value << (N - rotate)); } -} // namespace LIBC_NAMESPACE::cpp - -namespace LIBC_NAMESPACE { +} // namespace cpp // Specialization of mask_trailing_ones ('math_extras.h') for BigInt. template -LIBC_INLINE constexpr cpp::enable_if_t, T> +LIBC_INLINE constexpr cpp::enable_if_t, T> mask_trailing_ones() { static_assert(!T::SIGNED); if (count == 0) @@ -1086,8 +1086,7 @@ mask_trailing_ones() { // Specialization of mask_leading_ones ('math_extras.h') for BigInt. template -LIBC_INLINE constexpr cpp::enable_if_t, T> -mask_leading_ones() { +LIBC_INLINE constexpr cpp::enable_if_t, T> mask_leading_ones() { static_assert(!T::SIGNED); if (count == 0) return T(); diff --git a/libc/src/__support/UInt128.h b/libc/src/__support/UInt128.h index 0558e5095f9f..06696b7a6158 100644 --- a/libc/src/__support/UInt128.h +++ b/libc/src/__support/UInt128.h @@ -15,8 +15,8 @@ using UInt128 = __uint128_t; using Int128 = __int128_t; #else -using UInt128 = LIBC_NAMESPACE::cpp::UInt<128>; -using Int128 = LIBC_NAMESPACE::cpp::Int<128>; +using UInt128 = LIBC_NAMESPACE::UInt<128>; +using Int128 = LIBC_NAMESPACE::Int<128>; #endif #endif // LLVM_LIBC_SRC___SUPPORT_UINT128_H diff --git a/libc/src/__support/float_to_string.h b/libc/src/__support/float_to_string.h index 27476433a945..1287c3e9a84f 100644 --- a/libc/src/__support/float_to_string.h +++ b/libc/src/__support/float_to_string.h @@ -179,8 +179,8 @@ LIBC_INLINE constexpr uint32_t length_for_num(uint32_t idx, // TODO: Fix long doubles (needs bigger table or alternate algorithm.) // Currently the table values are generated, which is very slow. template -LIBC_INLINE constexpr cpp::UInt get_table_positive(int exponent, - size_t i) { +LIBC_INLINE constexpr UInt get_table_positive(int exponent, + size_t i) { // INT_SIZE is the size of int that is used for the internal calculations of // this function. It should be large enough to hold 2^(exponent+constant), so // ~1000 for double and ~16000 for long double. Be warned that the time @@ -191,17 +191,17 @@ LIBC_INLINE constexpr cpp::UInt get_table_positive(int exponent, if (shift_amount < 0) { return 1; } - cpp::UInt num(0); + UInt num(0); // MOD_SIZE is one of the limiting factors for how big the constant argument // can get, since it needs to be small enough to fit in the result UInt, // otherwise we'll get truncation on return. - constexpr cpp::UInt MOD_SIZE = - (cpp::UInt(EXP10_9) + constexpr UInt MOD_SIZE = + (UInt(EXP10_9) << (CALC_SHIFT_CONST + (IDX_SIZE > 1 ? IDX_SIZE : 0))); - num = cpp::UInt(1) << (shift_amount); + num = UInt(1) << (shift_amount); if (i > 0) { - cpp::UInt fives(EXP5_9); + UInt fives(EXP5_9); fives.pow_n(i); num = num / fives; } @@ -217,8 +217,7 @@ LIBC_INLINE constexpr cpp::UInt get_table_positive(int exponent, } template -LIBC_INLINE cpp::UInt get_table_positive_df(int exponent, - size_t i) { +LIBC_INLINE UInt get_table_positive_df(int exponent, size_t i) { static_assert(INT_SIZE == 256, "Only 256 is supported as an int size right now."); // This version uses dyadic floats with 256 bit mantissas to perform the same @@ -233,11 +232,11 @@ LIBC_INLINE cpp::UInt get_table_positive_df(int exponent, return 1; } fputil::DyadicFloat num(false, 0, 1); - constexpr cpp::UInt MOD_SIZE = - (cpp::UInt(EXP10_9) + constexpr UInt MOD_SIZE = + (UInt(EXP10_9) << (CALC_SHIFT_CONST + (IDX_SIZE > 1 ? IDX_SIZE : 0))); - constexpr cpp::UInt FIVE_EXP_MINUS_NINE_MANT{ + constexpr UInt FIVE_EXP_MINUS_NINE_MANT{ {0xf387295d242602a7, 0xfdd7645e011abac9, 0x31680a88f8953030, 0x89705f4136b4a597}}; @@ -251,7 +250,7 @@ LIBC_INLINE cpp::UInt get_table_positive_df(int exponent, num = mul_pow_2(num, shift_amount); // Adding one is part of the formula. - cpp::UInt int_num = static_cast>(num) + 1; + UInt int_num = static_cast>(num) + 1; if (int_num > MOD_SIZE) { auto rem = int_num @@ -261,7 +260,7 @@ LIBC_INLINE cpp::UInt get_table_positive_df(int exponent, int_num = rem; } - cpp::UInt result = int_num; + UInt result = int_num; return result; } @@ -275,11 +274,11 @@ LIBC_INLINE cpp::UInt get_table_positive_df(int exponent, // The formula being used looks more like this: // floor(10^(9*(-i)) * 2^(c_0 + (-e))) % (10^9 * 2^c_0) template -LIBC_INLINE cpp::UInt get_table_negative(int exponent, size_t i) { +LIBC_INLINE UInt get_table_negative(int exponent, size_t i) { int shift_amount = CALC_SHIFT_CONST - exponent; - cpp::UInt num(1); - constexpr cpp::UInt MOD_SIZE = - (cpp::UInt(EXP10_9) + UInt num(1); + constexpr UInt MOD_SIZE = + (UInt(EXP10_9) << (CALC_SHIFT_CONST + (IDX_SIZE > 1 ? IDX_SIZE : 0))); size_t ten_blocks = i; @@ -298,12 +297,12 @@ LIBC_INLINE cpp::UInt get_table_negative(int exponent, size_t i) { } if (five_blocks > 0) { - cpp::UInt fives(EXP5_9); + UInt fives(EXP5_9); fives.pow_n(five_blocks); num = fives; } if (ten_blocks > 0) { - cpp::UInt tens(EXP10_9); + UInt tens(EXP10_9); tens.pow_n(ten_blocks); if (five_blocks <= 0) { num = tens; @@ -327,8 +326,7 @@ LIBC_INLINE cpp::UInt get_table_negative(int exponent, size_t i) { } template -LIBC_INLINE cpp::UInt get_table_negative_df(int exponent, - size_t i) { +LIBC_INLINE UInt get_table_negative_df(int exponent, size_t i) { static_assert(INT_SIZE == 256, "Only 256 is supported as an int size right now."); // This version uses dyadic floats with 256 bit mantissas to perform the same @@ -341,11 +339,11 @@ LIBC_INLINE cpp::UInt get_table_negative_df(int exponent, int shift_amount = CALC_SHIFT_CONST - exponent; fputil::DyadicFloat num(false, 0, 1); - constexpr cpp::UInt MOD_SIZE = - (cpp::UInt(EXP10_9) + constexpr UInt MOD_SIZE = + (UInt(EXP10_9) << (CALC_SHIFT_CONST + (IDX_SIZE > 1 ? IDX_SIZE : 0))); - constexpr cpp::UInt TEN_EXP_NINE_MANT(EXP10_9); + constexpr UInt TEN_EXP_NINE_MANT(EXP10_9); static const fputil::DyadicFloat TEN_EXP_NINE(false, 0, TEN_EXP_NINE_MANT); @@ -356,7 +354,7 @@ LIBC_INLINE cpp::UInt get_table_negative_df(int exponent, } num = mul_pow_2(num, shift_amount); - cpp::UInt int_num = static_cast>(num); + UInt int_num = static_cast>(num); if (int_num > MOD_SIZE) { auto rem = int_num @@ -366,16 +364,16 @@ LIBC_INLINE cpp::UInt get_table_negative_df(int exponent, int_num = rem; } - cpp::UInt result = int_num; + UInt result = int_num; return result; } -LIBC_INLINE uint32_t fast_uint_mod_1e9(const cpp::UInt &val) { +LIBC_INLINE uint32_t fast_uint_mod_1e9(const UInt &val) { // The formula for mult_const is: // 1 + floor((2^(bits in target integer size + log_2(divider))) / divider) // Where divider is 10^9 and target integer size is 128. - const cpp::UInt mult_const( + const UInt mult_const( {0x31680A88F8953031u, 0x89705F4136B4A597u, 0}); const auto middle = (mult_const * val); const uint64_t result = static_cast(middle[2]); @@ -385,9 +383,9 @@ LIBC_INLINE uint32_t fast_uint_mod_1e9(const cpp::UInt &val) { } LIBC_INLINE uint32_t mul_shift_mod_1e9(const FPBits::StorageType mantissa, - const cpp::UInt &large, + const UInt &large, const int32_t shift_amount) { - cpp::UInt val(large); + UInt val(large); val = (val * mantissa) >> shift_amount; return static_cast( val.div_uint_half_times_pow_2(static_cast(EXP10_9), 0).value()); @@ -452,7 +450,7 @@ public: const uint32_t pos_exp = idx * IDX_SIZE; - cpp::UInt val; + UInt val; #if defined(LIBC_COPT_FLOAT_TO_STR_USE_DYADIC_FLOAT) // ----------------------- DYADIC FLOAT CALC MODE ------------------------ @@ -502,7 +500,7 @@ public: if (exponent < 0) { const int32_t idx = -exponent / IDX_SIZE; - cpp::UInt val; + UInt val; const uint32_t pos_exp = static_cast(idx * IDX_SIZE); @@ -643,7 +641,7 @@ template <> class FloatToString { internal::div_ceil(sizeof(long double) * CHAR_BIT, UINT_WORD_SIZE) * UINT_WORD_SIZE; - using wide_int = cpp::UInt; + using wide_int = UInt; // float_as_fixed represents the floating point number as a fixed point number // with the point EXTRA_INT_WIDTH bits from the left of the number. This can @@ -658,7 +656,7 @@ template <> class FloatToString { size_t block_buffer_valid = 0; template - LIBC_INLINE static constexpr BlockInt grab_digits(cpp::UInt &int_num) { + LIBC_INLINE static constexpr BlockInt grab_digits(UInt &int_num) { auto wide_result = int_num.div_uint_half_times_pow_2(EXP5_9, 9); // the optional only comes into effect when dividing by 0, which will // never happen here. Thus, we just assert that it has value. @@ -714,7 +712,7 @@ template <> class FloatToString { // If there are still digits above the decimal point, handle those. if (float_as_fixed.clz() < static_cast(EXTRA_INT_WIDTH)) { - cpp::UInt above_decimal_point = + UInt above_decimal_point = float_as_fixed >> FLOAT_AS_INT_WIDTH; size_t positive_int_block_index = 0; diff --git a/libc/src/__support/integer_literals.h b/libc/src/__support/integer_literals.h index 08bc6973b158..de1f88fbd3f3 100644 --- a/libc/src/__support/integer_literals.h +++ b/libc/src/__support/integer_literals.h @@ -115,13 +115,13 @@ template struct Parser { } }; -// Specialization for cpp::UInt. +// Specialization for UInt. // Because this code runs at compile time we try to make it efficient. For // binary and hexadecimal formats we read digits by chunks of 64 bits and // produce the BigInt internal representation direcly. For decimal numbers we // go the slow path and use slower BigInt arithmetic. -template struct Parser> { - using UIntT = cpp::UInt; +template struct Parser> { + using UIntT = UInt; template static constexpr UIntT parse(const char *str) { const DigitBuffer buffer(str); if constexpr (base == 10) { @@ -166,7 +166,7 @@ LIBC_INLINE constexpr UInt128 operator""_u128(const char *x) { } LIBC_INLINE constexpr auto operator""_u256(const char *x) { - return internal::parse_with_prefix>(x); + return internal::parse_with_prefix>(x); } } // namespace LIBC_NAMESPACE diff --git a/libc/src/__support/integer_to_string.h b/libc/src/__support/integer_to_string.h index a5872dce6520..ac0bdd688aea 100644 --- a/libc/src/__support/integer_to_string.h +++ b/libc/src/__support/integer_to_string.h @@ -158,7 +158,7 @@ struct IntegerWriterUnsigned>> { }; template -struct IntegerWriterUnsigned>> { +struct IntegerWriterUnsigned>> { using type = typename T::unsigned_type; }; @@ -176,7 +176,7 @@ template using Custom = details::Fmt; // See file header for documentation. template class IntegerToString { - static_assert(cpp::is_integral_v || cpp::is_big_int_v); + static_assert(cpp::is_integral_v || is_big_int_v); LIBC_INLINE static constexpr size_t compute_buffer_size() { constexpr auto MAX_DIGITS = []() -> size_t { @@ -221,7 +221,7 @@ template class IntegerToString { // An internal stateless structure that handles the number formatting logic. struct IntegerWriter { - static_assert(cpp::is_integral_v || cpp::is_big_int_v); + static_assert(cpp::is_integral_v || is_big_int_v); using UNSIGNED_T = typename details::IntegerWriterUnsigned::type; LIBC_INLINE static char digit_char(uint8_t digit) { diff --git a/libc/src/__support/str_to_float.h b/libc/src/__support/str_to_float.h index 6caf8e62a454..073e1dc67217 100644 --- a/libc/src/__support/str_to_float.h +++ b/libc/src/__support/str_to_float.h @@ -524,7 +524,7 @@ clinger_fast_path(ExpandedFloat init_num, FPBits result; T float_mantissa; - if constexpr (cpp::is_same_v>) { + if constexpr (cpp::is_same_v>) { float_mantissa = static_cast(fputil::DyadicFloat<128>( Sign::POS, 0, fputil::DyadicFloat<128>::MantissaType( diff --git a/libc/src/stdio/printf_core/float_dec_converter.h b/libc/src/stdio/printf_core/float_dec_converter.h index 27d229a3e42c..5270fc9de037 100644 --- a/libc/src/stdio/printf_core/float_dec_converter.h +++ b/libc/src/stdio/printf_core/float_dec_converter.h @@ -12,7 +12,7 @@ #include "src/__support/CPP/string_view.h" #include "src/__support/FPUtil/FPBits.h" #include "src/__support/FPUtil/rounding_mode.h" -#include "src/__support/UInt.h" // cpp::is_big_int_v +#include "src/__support/UInt.h" // is_big_int_v #include "src/__support/float_to_string.h" #include "src/__support/integer_to_string.h" #include "src/__support/libc_assert.h" @@ -34,8 +34,8 @@ using ExponentString = // Returns true if value is divisible by 2^p. template -LIBC_INLINE constexpr cpp::enable_if_t< - cpp::is_integral_v || cpp::is_big_int_v, bool> +LIBC_INLINE constexpr cpp::enable_if_t || is_big_int_v, + bool> multiple_of_power_of_2(T value, uint32_t p) { return (value & ((T(1) << p) - 1)) == 0; } @@ -78,8 +78,8 @@ LIBC_INLINE RoundDirection get_round_direction(int last_digit, bool truncated, } template -LIBC_INLINE constexpr cpp::enable_if_t< - cpp::is_integral_v || cpp::is_big_int_v, bool> +LIBC_INLINE constexpr cpp::enable_if_t || is_big_int_v, + bool> zero_after_digits(int32_t base_2_exp, int32_t digits_after_point, T mantissa, const int32_t mant_width) { const int32_t required_twos = -base_2_exp - digits_after_point - 1; diff --git a/libc/test/UnitTest/LibcTest.cpp b/libc/test/UnitTest/LibcTest.cpp index 0340f7ed3710..7dc85d0d4e2d 100644 --- a/libc/test/UnitTest/LibcTest.cpp +++ b/libc/test/UnitTest/LibcTest.cpp @@ -39,7 +39,7 @@ TestLogger &operator<<(TestLogger &logger, Location Loc) { // digits. template cpp::enable_if_t<(cpp::is_integral_v && (sizeof(T) > sizeof(uint64_t))) || - cpp::is_big_int_v, + is_big_int_v, cpp::string> describeValue(T Value) { static_assert(sizeof(T) % 8 == 0, "Unsupported size of UInt"); @@ -221,12 +221,12 @@ TEST_SPECIALIZATION(bool); TEST_SPECIALIZATION(__uint128_t); #endif -TEST_SPECIALIZATION(LIBC_NAMESPACE::cpp::Int<128>); +TEST_SPECIALIZATION(LIBC_NAMESPACE::Int<128>); -TEST_SPECIALIZATION(LIBC_NAMESPACE::cpp::UInt<128>); -TEST_SPECIALIZATION(LIBC_NAMESPACE::cpp::UInt<192>); -TEST_SPECIALIZATION(LIBC_NAMESPACE::cpp::UInt<256>); -TEST_SPECIALIZATION(LIBC_NAMESPACE::cpp::UInt<320>); +TEST_SPECIALIZATION(LIBC_NAMESPACE::UInt<128>); +TEST_SPECIALIZATION(LIBC_NAMESPACE::UInt<192>); +TEST_SPECIALIZATION(LIBC_NAMESPACE::UInt<256>); +TEST_SPECIALIZATION(LIBC_NAMESPACE::UInt<320>); TEST_SPECIALIZATION(LIBC_NAMESPACE::cpp::string_view); TEST_SPECIALIZATION(LIBC_NAMESPACE::cpp::string); diff --git a/libc/test/UnitTest/LibcTest.h b/libc/test/UnitTest/LibcTest.h index d26d6490bcb5..a813a59d2d67 100644 --- a/libc/test/UnitTest/LibcTest.h +++ b/libc/test/UnitTest/LibcTest.h @@ -125,11 +125,11 @@ protected: // is the result of the |Cond| operation on |LHS| and |RHS|. Though not bad, // |Cond| on mismatched |LHS| and |RHS| types can potentially succeed because // of type promotion. - template || - cpp::is_big_int_v || - cpp::is_fixed_point_v, - int> = 0> + template < + typename ValType, + cpp::enable_if_t || is_big_int_v || + cpp::is_fixed_point_v, + int> = 0> bool test(TestCond Cond, ValType LHS, ValType RHS, const char *LHSStr, const char *RHSStr, internal::Location Loc) { return internal::test(Ctx, Cond, LHS, RHS, LHSStr, RHSStr, Loc); diff --git a/libc/test/UnitTest/StringUtils.h b/libc/test/UnitTest/StringUtils.h index 1e3ba5715d23..cab0b58f9690 100644 --- a/libc/test/UnitTest/StringUtils.h +++ b/libc/test/UnitTest/StringUtils.h @@ -17,7 +17,7 @@ namespace LIBC_NAMESPACE { // Return the first N hex digits of an integer as a string in upper case. template -cpp::enable_if_t || cpp::is_big_int_v, cpp::string> +cpp::enable_if_t || is_big_int_v, cpp::string> int_to_hex(T value, size_t length = sizeof(T) * 2) { cpp::string s(length, '0'); diff --git a/libc/test/UnitTest/TestLogger.cpp b/libc/test/UnitTest/TestLogger.cpp index 469b3a11d57d..32f0ec52041f 100644 --- a/libc/test/UnitTest/TestLogger.cpp +++ b/libc/test/UnitTest/TestLogger.cpp @@ -48,7 +48,7 @@ template <> TestLogger &TestLogger::operator<<(void *addr) { } template TestLogger &TestLogger::operator<<(T t) { - if constexpr (cpp::is_big_int_v || + if constexpr (is_big_int_v || (cpp::is_integral_v && cpp::is_unsigned_v && (sizeof(T) > sizeof(uint64_t)))) { static_assert(sizeof(T) % 8 == 0, "Unsupported size of UInt"); @@ -75,10 +75,10 @@ template TestLogger & #ifdef __SIZEOF_INT128__ template TestLogger &TestLogger::operator<< <__uint128_t>(__uint128_t); #endif -template TestLogger &TestLogger::operator<< >(cpp::UInt<128>); -template TestLogger &TestLogger::operator<< >(cpp::UInt<192>); -template TestLogger &TestLogger::operator<< >(cpp::UInt<256>); -template TestLogger &TestLogger::operator<< >(cpp::UInt<320>); +template TestLogger &TestLogger::operator<< >(UInt<128>); +template TestLogger &TestLogger::operator<< >(UInt<192>); +template TestLogger &TestLogger::operator<< >(UInt<256>); +template TestLogger &TestLogger::operator<< >(UInt<320>); // TODO: Add floating point formatting once it's supported by StringStream. diff --git a/libc/test/src/__support/CPP/bit_test.cpp b/libc/test/src/__support/CPP/bit_test.cpp index d3f56d5bad83..da7bb3389b2f 100644 --- a/libc/test/src/__support/CPP/bit_test.cpp +++ b/libc/test/src/__support/CPP/bit_test.cpp @@ -26,7 +26,7 @@ using UnsignedTypes = testing::TypeList< __uint128_t, #endif unsigned char, unsigned short, unsigned int, unsigned long, - unsigned long long, cpp::UInt<128>>; + unsigned long long, UInt<128>>; TYPED_TEST(LlvmLibcBitTest, HasSingleBit, UnsignedTypes) { constexpr auto ZERO = T(0); diff --git a/libc/test/src/__support/CPP/limits_test.cpp b/libc/test/src/__support/CPP/limits_test.cpp index 12641b7b51b6..d83e307fb196 100644 --- a/libc/test/src/__support/CPP/limits_test.cpp +++ b/libc/test/src/__support/CPP/limits_test.cpp @@ -32,11 +32,10 @@ TEST(LlvmLibcLimitsTest, LimitsFollowSpec) { } TEST(LlvmLibcLimitsTest, UInt128Limits) { - auto umax128 = cpp::numeric_limits>::max(); - auto umax64 = - LIBC_NAMESPACE::cpp::UInt<128>(cpp::numeric_limits::max()); + auto umax128 = cpp::numeric_limits>::max(); + auto umax64 = LIBC_NAMESPACE::UInt<128>(cpp::numeric_limits::max()); EXPECT_GT(umax128, umax64); - ASSERT_EQ(~LIBC_NAMESPACE::cpp::UInt<128>(0), umax128); + ASSERT_EQ(~LIBC_NAMESPACE::UInt<128>(0), umax128); #ifdef __SIZEOF_INT128__ ASSERT_EQ(~__uint128_t(0), cpp::numeric_limits<__uint128_t>::max()); #endif diff --git a/libc/test/src/__support/integer_literals_test.cpp b/libc/test/src/__support/integer_literals_test.cpp index 10c3625a0e5a..b8b399d9cf40 100644 --- a/libc/test/src/__support/integer_literals_test.cpp +++ b/libc/test/src/__support/integer_literals_test.cpp @@ -104,7 +104,7 @@ TEST(LlvmLibcIntegerLiteralTest, u128) { } TEST(LlvmLibcIntegerLiteralTest, u256) { - using UInt256 = LIBC_NAMESPACE::cpp::UInt<256>; + using UInt256 = LIBC_NAMESPACE::UInt<256>; const UInt256 ZERO = 0; const UInt256 U8_MAX = UINT8_MAX; const UInt256 U16_MAX = UINT16_MAX; diff --git a/libc/test/src/__support/integer_to_string_test.cpp b/libc/test/src/__support/integer_to_string_test.cpp index 2a19c5bf7549..a2a80c81b9f6 100644 --- a/libc/test/src/__support/integer_to_string_test.cpp +++ b/libc/test/src/__support/integer_to_string_test.cpp @@ -228,7 +228,7 @@ TEST(LlvmLibcIntegerToStringTest, UINT64_Base_36) { } TEST(LlvmLibcIntegerToStringTest, UINT256_Base_16) { - using UInt256 = LIBC_NAMESPACE::cpp::UInt<256>; + using UInt256 = LIBC_NAMESPACE::UInt<256>; using type = IntegerToString>; EXPECT( type, diff --git a/libc/test/src/__support/uint_test.cpp b/libc/test/src/__support/uint_test.cpp index 963c553b10d0..eb1db9729cf4 100644 --- a/libc/test/src/__support/uint_test.cpp +++ b/libc/test/src/__support/uint_test.cpp @@ -14,19 +14,19 @@ namespace LIBC_NAMESPACE { -using LL_UInt64 = cpp::UInt<64>; -// We want to test cpp::UInt<128> explicitly. So, for +using LL_UInt64 = UInt<64>; +// We want to test UInt<128> explicitly. So, for // convenience, we use a sugar which does not conflict with the UInt128 type // which can resolve to __uint128_t if the platform has it. -using LL_UInt128 = cpp::UInt<128>; -using LL_UInt192 = cpp::UInt<192>; -using LL_UInt256 = cpp::UInt<256>; -using LL_UInt320 = cpp::UInt<320>; -using LL_UInt512 = cpp::UInt<512>; -using LL_UInt1024 = cpp::UInt<1024>; +using LL_UInt128 = UInt<128>; +using LL_UInt192 = UInt<192>; +using LL_UInt256 = UInt<256>; +using LL_UInt320 = UInt<320>; +using LL_UInt512 = UInt<512>; +using LL_UInt1024 = UInt<1024>; -using LL_Int128 = cpp::Int<128>; -using LL_Int192 = cpp::Int<192>; +using LL_Int128 = Int<128>; +using LL_Int192 = Int<192>; TEST(LlvmLibcUIntClassTest, BitCastToFromDouble) { static_assert(cpp::is_trivially_copyable::value); @@ -677,8 +677,8 @@ TEST(LlvmLibcUIntClassTest, ConstructorFromUInt128Tests) { } TEST(LlvmLibcUIntClassTest, WordTypeUInt128Tests) { - using LL_UInt256_128 = cpp::BigInt<256, false, __uint128_t>; - using LL_UInt128_128 = cpp::BigInt<128, false, __uint128_t>; + using LL_UInt256_128 = BigInt<256, false, __uint128_t>; + using LL_UInt128_128 = BigInt<128, false, __uint128_t>; LL_UInt256_128 a(1); @@ -710,7 +710,7 @@ TEST(LlvmLibcUIntClassTest, WordTypeUInt128Tests) { #endif // __SIZEOF_INT128__ TEST(LlvmLibcUIntClassTest, OtherWordTypeTests) { - using LL_UInt96 = cpp::BigInt<96, false, uint32_t>; + using LL_UInt96 = BigInt<96, false, uint32_t>; LL_UInt96 a(1); -- GitLab From 80a9574d36834f7e0814e603894347e997435d54 Mon Sep 17 00:00:00 2001 From: Alex Richardson Date: Fri, 8 Mar 2024 00:45:02 -0800 Subject: [PATCH 575/929] [compiler-rt] Unbreak GCC builds after bfa6444a332f82843 GCC complains if we pass -Werror=format-security without -Wformat. Reported at https://github.com/llvm/llvm-project/pull/83779#issuecomment-1985246797 --- compiler-rt/cmake/Modules/CompilerRTUtils.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler-rt/cmake/Modules/CompilerRTUtils.cmake b/compiler-rt/cmake/Modules/CompilerRTUtils.cmake index 25e7823716fc..e8e5f612d5b0 100644 --- a/compiler-rt/cmake/Modules/CompilerRTUtils.cmake +++ b/compiler-rt/cmake/Modules/CompilerRTUtils.cmake @@ -583,7 +583,8 @@ function(add_security_warnings out_flags macosx_sdk_version) append_list_if(COMPILER_RT_HAS_RETURN_STACK_ADDRESS_FLAG -Werror=return-stack-address flags) append_list_if(COMPILER_RT_HAS_SIZEOF_ARRAY_DECAY_FLAG -Werror=sizeof-array-decay flags) append_list_if(COMPILER_RT_HAS_FORMAT_INSUFFICIENT_ARGS_FLAG -Werror=format-insufficient-args flags) - append_list_if(COMPILER_RT_HAS_BUILTIN_FORMAL_SECURITY_FLAG -Werror=format-security flags) + # GCC complains if we pass -Werror=format-security without -Wformat + append_list_if(COMPILER_RT_HAS_BUILTIN_FORMAL_SECURITY_FLAG -Wformat -Werror=format-security flags) append_list_if(COMPILER_RT_HAS_SIZEOF_ARRAY_DIV_FLAG -Werror=sizeof-array-div) append_list_if(COMPILER_RT_HAS_SIZEOF_POINTER_DIV_FLAG -Werror=sizeof-pointer-div) -- GitLab From 851ab41d33fcbc72bc334dfc2d5d4c0902ccbb23 Mon Sep 17 00:00:00 2001 From: OverMighty Date: Fri, 8 Mar 2024 08:50:21 +0000 Subject: [PATCH 576/929] [clang][test] Fix constant __builtin_popcountg test requiring __int128 (#84412) See https://lab.llvm.org/buildbot/#/builders/245/builds/21611/steps/5/logs/FAIL__Clang__constant-builtins-2_c. --- clang/test/Sema/constant-builtins-2.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clang/test/Sema/constant-builtins-2.c b/clang/test/Sema/constant-builtins-2.c index 0935abe4c65f..6dd1d88759c7 100644 --- a/clang/test/Sema/constant-builtins-2.c +++ b/clang/test/Sema/constant-builtins-2.c @@ -242,7 +242,9 @@ char popcount12[__builtin_popcountg(0xF0F0U) == 8 ? 1 : -1]; char popcount13[__builtin_popcountg(~0U) == BITSIZE(int) ? 1 : -1]; char popcount14[__builtin_popcountg(~0UL) == BITSIZE(long) ? 1 : -1]; char popcount15[__builtin_popcountg(~0ULL) == BITSIZE(long long) ? 1 : -1]; +#ifdef __SIZEOF_INT128__ char popcount16[__builtin_popcountg(~(unsigned __int128)0) == BITSIZE(__int128) ? 1 : -1]; +#endif char popcount17[__builtin_popcountg(~(unsigned _BitInt(128))0) == BITSIZE(_BitInt(128)) ? 1 : -1]; char parity1[__builtin_parity(0) == 0 ? 1 : -1]; -- GitLab From dd36138e9c23c462ce5379a3d83530fb4ebec9e7 Mon Sep 17 00:00:00 2001 From: Mariya Podchishchaeva Date: Fri, 8 Mar 2024 11:52:42 +0300 Subject: [PATCH 577/929] [clang] Error on explicit specialization of lambda call operator (#84343) Fixes https://github.com/llvm/llvm-project/issues/83267 --- clang/docs/ReleaseNotes.rst | 4 ++++ .../clang/Basic/DiagnosticSemaKinds.td | 2 ++ clang/lib/Sema/SemaDecl.cpp | 22 +++++++++++++------ clang/test/SemaCXX/lambda-expressions.cpp | 15 +++++++++++++ 4 files changed, 36 insertions(+), 7 deletions(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index fe7bbe437831..225ca85e1811 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -250,6 +250,10 @@ Bug Fixes in This Version for variables created through copy initialization having side-effects in C++17 and later. Fixes (#GH64356) (#GH79518). +- Clang now emits errors for explicit specializations/instatiations of lambda call + operator. + Fixes (#GH83267). + Bug Fixes to Compiler Builtins ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index c8dfdc08f5ea..a80c8b75e863 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -8170,6 +8170,8 @@ let CategoryName = "Lambda Issue" in { def warn_cxx11_compat_generic_lambda : Warning< "generic lambdas are incompatible with C++11">, InGroup, DefaultIgnore; + def err_lambda_explicit_spec : Error< + "lambda call operator should not be explicitly specialized or instantiated">; // C++17 '*this' captures. def warn_cxx14_compat_star_this_lambda_capture : Warning< diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 67e56a917a51..910d0dfbf9f6 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -15795,10 +15795,19 @@ Decl *Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope, Decl *D, // captures during transformation of nested lambdas, it is necessary to // have the LSI properly restored. if (isGenericLambdaCallOperatorSpecialization(FD)) { - assert(inTemplateInstantiation() && - "There should be an active template instantiation on the stack " - "when instantiating a generic lambda!"); - RebuildLambdaScopeInfo(cast(D)); + // C++2c 7.5.5.2p17 A member of a closure type shall not be explicitly + // instantiated, explicitly specialized. + if (FD->getTemplateSpecializationInfo() + ->isExplicitInstantiationOrSpecialization()) { + Diag(FD->getLocation(), diag::err_lambda_explicit_spec); + FD->setInvalidDecl(); + PushFunctionScope(); + } else { + assert(inTemplateInstantiation() && + "There should be an active template instantiation on the stack " + "when instantiating a generic lambda!"); + RebuildLambdaScopeInfo(cast(D)); + } } else { // Enter a new function scope PushFunctionScope(); @@ -16317,9 +16326,8 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body, } } - assert( - (FD == getCurFunctionDecl() || getCurLambda()->CallOperator == FD) && - "Function parsing confused"); + assert((FD == getCurFunctionDecl(/*AllowLambdas=*/true)) && + "Function parsing confused"); } else if (ObjCMethodDecl *MD = dyn_cast_or_null(dcl)) { assert(MD == getCurMethodDecl() && "Method parsing confused"); MD->setBody(Body); diff --git a/clang/test/SemaCXX/lambda-expressions.cpp b/clang/test/SemaCXX/lambda-expressions.cpp index 41cf5a46c389..0516a5da31ae 100644 --- a/clang/test/SemaCXX/lambda-expressions.cpp +++ b/clang/test/SemaCXX/lambda-expressions.cpp @@ -732,3 +732,18 @@ void GH67492() { constexpr auto test = 42; auto lambda = (test, []() noexcept(true) {}); } + +namespace GH83267 { +auto l = [](auto a) { return 1; }; +using type = decltype(l); + +template<> +auto type::operator()(int a) const { // expected-error{{lambda call operator should not be explicitly specialized or instantiated}} + return c; // expected-error {{use of undeclared identifier 'c'}} +} + +auto ll = [](auto a) { return 1; }; // expected-error{{lambda call operator should not be explicitly specialized or instantiated}} +using t = decltype(ll); +template auto t::operator()(int a) const; // expected-note {{in instantiation}} + +} -- GitLab From 881df557501d339c7a14b16d68e43da5c732b424 Mon Sep 17 00:00:00 2001 From: Jie Fu Date: Fri, 8 Mar 2024 17:03:51 +0800 Subject: [PATCH 578/929] [clang] Fix -Wunused-variable in SemaChecking.cpp (NFC) llvm-project/clang/lib/Sema/SemaChecking.cpp:19192:15: error: unused variable 'Field1Parent' [-Werror,-Wunused-variable] const Type *Field1Parent = Field1->getParent()->getTypeForDecl(); ^ llvm-project/clang/lib/Sema/SemaChecking.cpp:19193:15: error: unused variable 'Field2Parent' [-Werror,-Wunused-variable] const Type *Field2Parent = Field2->getParent()->getTypeForDecl(); ^ 2 errors generated. --- clang/lib/Sema/SemaChecking.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index b34b8df00201..a5f42b630c3f 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -19189,8 +19189,10 @@ static bool isLayoutCompatible(ASTContext &C, EnumDecl *ED1, EnumDecl *ED2) { static bool isLayoutCompatible(ASTContext &C, FieldDecl *Field1, FieldDecl *Field2, bool AreUnionMembers = false) { - const Type *Field1Parent = Field1->getParent()->getTypeForDecl(); - const Type *Field2Parent = Field2->getParent()->getTypeForDecl(); + [[maybe_unused]] const Type *Field1Parent = + Field1->getParent()->getTypeForDecl(); + [[maybe_unused]] const Type *Field2Parent = + Field2->getParent()->getTypeForDecl(); assert(((Field1Parent->isStructureOrClassType() && Field2Parent->isStructureOrClassType()) || (Field1Parent->isUnionType() && Field2Parent->isUnionType())) && -- GitLab From ef1eb0315e5830e2c3ea76d4f1ec6c50842da7bb Mon Sep 17 00:00:00 2001 From: Sizov Nikita Date: Fri, 8 Mar 2024 12:45:58 +0300 Subject: [PATCH 579/929] [AArch64] Add neon bici test for haddu and shadd (#84073) Add neon bici test for haddu and shadd, prerequisite for #76644 --- .../AArch64/aarch64-known-bits-hadd.ll | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 llvm/test/CodeGen/AArch64/aarch64-known-bits-hadd.ll diff --git a/llvm/test/CodeGen/AArch64/aarch64-known-bits-hadd.ll b/llvm/test/CodeGen/AArch64/aarch64-known-bits-hadd.ll new file mode 100644 index 000000000000..017f38277489 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/aarch64-known-bits-hadd.ll @@ -0,0 +1,129 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=aarch64 < %s | FileCheck %s + +declare <8 x i16> @llvm.aarch64.neon.uhadd.v8i16(<8 x i16>, <8 x i16>) +declare <8 x i16> @llvm.aarch64.neon.urhadd.v8i16(<8 x i16>, <8 x i16>) +declare <8 x i16> @llvm.aarch64.neon.shadd.v8i16(<8 x i16>, <8 x i16>) +declare <8 x i16> @llvm.aarch64.neon.srhadd.v8i16(<8 x i16>, <8 x i16>) + +define <8 x i16> @haddu_zext(<8 x i8> %a0, <8 x i8> %a1) { +; CHECK-LABEL: haddu_zext: +; CHECK: // %bb.0: +; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-NEXT: uhadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: bic v0.8h, #254, lsl #8 +; CHECK-NEXT: ret + %x0 = zext <8 x i8> %a0 to <8 x i16> + %x1 = zext <8 x i8> %a1 to <8 x i16> + %hadd = call <8 x i16> @llvm.aarch64.neon.uhadd.v8i16(<8 x i16> %x0, <8 x i16> %x1) + %res = and <8 x i16> %hadd, + ret <8 x i16> %res +} + +define <8 x i16> @rhaddu_zext(<8 x i8> %a0, <8 x i8> %a1) { +; CHECK-LABEL: rhaddu_zext: +; CHECK: // %bb.0: +; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-NEXT: urhadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: bic v0.8h, #254, lsl #8 +; CHECK-NEXT: ret + %x0 = zext <8 x i8> %a0 to <8 x i16> + %x1 = zext <8 x i8> %a1 to <8 x i16> + %hadd = call <8 x i16> @llvm.aarch64.neon.urhadd.v8i16(<8 x i16> %x0, <8 x i16> %x1) + %res = and <8 x i16> %hadd, + ret <8 x i16> %res +} + +define <8 x i16> @hadds_zext(<8 x i8> %a0, <8 x i8> %a1) { +; CHECK-LABEL: hadds_zext: +; CHECK: // %bb.0: +; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-NEXT: shadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: bic v0.8h, #254, lsl #8 +; CHECK-NEXT: ret + %x0 = zext <8 x i8> %a0 to <8 x i16> + %x1 = zext <8 x i8> %a1 to <8 x i16> + %hadd = call <8 x i16> @llvm.aarch64.neon.shadd.v8i16(<8 x i16> %x0, <8 x i16> %x1) + %res = and <8 x i16> %hadd, + ret <8 x i16> %res +} + +define <8 x i16> @shaddu_zext(<8 x i8> %a0, <8 x i8> %a1) { +; CHECK-LABEL: shaddu_zext: +; CHECK: // %bb.0: +; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-NEXT: srhadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: bic v0.8h, #254, lsl #8 +; CHECK-NEXT: ret + %x0 = zext <8 x i8> %a0 to <8 x i16> + %x1 = zext <8 x i8> %a1 to <8 x i16> + %hadd = call <8 x i16> @llvm.aarch64.neon.srhadd.v8i16(<8 x i16> %x0, <8 x i16> %x1) + %res = and <8 x i16> %hadd, + ret <8 x i16> %res +} + +; ; negative tests + +define <8 x i16> @haddu_sext(<8 x i8> %a0, <8 x i8> %a1) { +; CHECK-LABEL: haddu_sext: +; CHECK: // %bb.0: +; CHECK-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-NEXT: uhadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: bic v0.8h, #254, lsl #8 +; CHECK-NEXT: ret + %x0 = sext <8 x i8> %a0 to <8 x i16> + %x1 = sext <8 x i8> %a1 to <8 x i16> + %hadd = call <8 x i16> @llvm.aarch64.neon.uhadd.v8i16(<8 x i16> %x0, <8 x i16> %x1) + %res = and <8 x i16> %hadd, + ret <8 x i16> %res +} + +define <8 x i16> @urhadd_sext(<8 x i8> %a0, <8 x i8> %a1) { +; CHECK-LABEL: urhadd_sext: +; CHECK: // %bb.0: +; CHECK-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-NEXT: urhadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: bic v0.8h, #254, lsl #8 +; CHECK-NEXT: ret + %x0 = sext <8 x i8> %a0 to <8 x i16> + %x1 = sext <8 x i8> %a1 to <8 x i16> + %hadd = call <8 x i16> @llvm.aarch64.neon.urhadd.v8i16(<8 x i16> %x0, <8 x i16> %x1) + %res = and <8 x i16> %hadd, + ret <8 x i16> %res +} + +define <8 x i16> @hadds_sext(<8 x i8> %a0, <8 x i8> %a1) { +; CHECK-LABEL: hadds_sext: +; CHECK: // %bb.0: +; CHECK-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-NEXT: shadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: bic v0.8h, #254, lsl #8 +; CHECK-NEXT: ret + %x0 = sext <8 x i8> %a0 to <8 x i16> + %x1 = sext <8 x i8> %a1 to <8 x i16> + %hadd = call <8 x i16> @llvm.aarch64.neon.shadd.v8i16(<8 x i16> %x0, <8 x i16> %x1) + %res = and <8 x i16> %hadd, + ret <8 x i16> %res +} + +define <8 x i16> @shaddu_sext(<8 x i8> %a0, <8 x i8> %a1) { +; CHECK-LABEL: shaddu_sext: +; CHECK: // %bb.0: +; CHECK-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-NEXT: srhadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: bic v0.8h, #254, lsl #8 +; CHECK-NEXT: ret + %x0 = sext <8 x i8> %a0 to <8 x i16> + %x1 = sext <8 x i8> %a1 to <8 x i16> + %hadd = call <8 x i16> @llvm.aarch64.neon.srhadd.v8i16(<8 x i16> %x0, <8 x i16> %x1) + %res = and <8 x i16> %hadd, + ret <8 x i16> %res +} -- GitLab From d36d805373c27103edf13164260ef679efd30dc1 Mon Sep 17 00:00:00 2001 From: David Green Date: Fri, 8 Mar 2024 09:52:49 +0000 Subject: [PATCH 580/929] [AArch64] Ensure Neoverse-N2 scheduling model includes all SVE pseudos. Similar to #84187, this enables the existing test we have for checking the scheduling info of the pseudos matches the real instructions, and adjusts the scheduling info in the NeoverseN2 model to make sure all cases were handled. --- .../Target/AArch64/AArch64SchedNeoverseN2.td | 210 +++++++++--------- .../AArch64/AArch64SVESchedPseudoTest.cpp | 4 + 2 files changed, 114 insertions(+), 100 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td b/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td index c18045e7c8f9..a4ac344510de 100644 --- a/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td +++ b/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td @@ -1577,7 +1577,8 @@ def : InstRW<[N2Write_2cyc_1M], (instregex "^(ZIP|UZP)[12]_PPP_[BHSDQ]$")>; // ----------------------------------------------------------------------------- // Arithmetic, absolute diff -def : InstRW<[N2Write_2cyc_1V], (instregex "^[SU]ABD_ZPmZ_[BHSD]$")>; +def : InstRW<[N2Write_2cyc_1V], (instregex "^[SU]ABD_ZPmZ_[BHSD]", + "^[SU]ABD_ZPZZ_[BHSD]")>; // Arithmetic, absolute diff accum def : InstRW<[N2Write_4cyc_1V1], (instregex "^[SU]ABA_ZZZ_[BHSD]$")>; @@ -1590,24 +1591,25 @@ def : InstRW<[N2Write_2cyc_1V], (instregex "^[SU]ABDL[TB]_ZZZ_[HSD]$")>; // Arithmetic, basic def : InstRW<[N2Write_2cyc_1V], - (instregex "^(ABS|ADD|CNOT|NEG|SUB|SUBR)_ZPmZ_[BHSD]$", - "^(ADD|SUB)_ZZZ_[BHSD]$", - "^(ADD|SUB|SUBR)_ZI_[BHSD]$", - "^ADR_[SU]XTW_ZZZ_D_[0123]$", - "^ADR_LSL_ZZZ_[SD]_[0123]$", - "^[SU](ADD|SUB)[LW][BT]_ZZZ_[HSD]$", - "^SADDLBT_ZZZ_[HSD]$", - "^[SU]H(ADD|SUB|SUBR)_ZPmZ_[BHSD]$", - "^SSUBL(BT|TB)_ZZZ_[HSD]$")>; + (instregex "^(ABS|ADD|CNOT|NEG|SUB|SUBR)_ZPmZ_[BHSD]", + "^(ADD|SUB)_ZZZ_[BHSD]", + "^(ADD|SUB|SUBR)_ZPZZ_[BHSD]", + "^(ADD|SUB|SUBR)_ZI_[BHSD]", + "^ADR_[SU]XTW_ZZZ_D_[0123]", + "^ADR_LSL_ZZZ_[SD]_[0123]", + "^[SU](ADD|SUB)[LW][BT]_ZZZ_[HSD]", + "^SADDLBT_ZZZ_[HSD]", + "^[SU]H(ADD|SUB|SUBR)_ZPmZ_[BHSD]", + "^SSUBL(BT|TB)_ZZZ_[HSD]")>; // Arithmetic, complex def : InstRW<[N2Write_2cyc_1V], - (instregex "^R?(ADD|SUB)HN[BT]_ZZZ_[BHS]$", - "^SQ(ABS|ADD|NEG|SUB|SUBR)_ZPmZ_[BHSD]$", - "^[SU]Q(ADD|SUB)_ZZZ_[BHSD]$", - "^[SU]Q(ADD|SUB)_ZI_[BHSD]$", - "^(SRH|SUQ|UQ|USQ|URH)ADD_ZPmZ_[BHSD]$", - "^(UQSUB|UQSUBR)_ZPmZ_[BHSD]$")>; + (instregex "^R?(ADD|SUB)HN[BT]_ZZZ_[BHS]", + "^SQ(ABS|ADD|NEG|SUB|SUBR)_ZPmZ_[BHSD]", + "^[SU]Q(ADD|SUB)_ZZZ_[BHSD]", + "^[SU]Q(ADD|SUB)_ZI_[BHSD]", + "^(SRH|SUQ|UQ|USQ|URH)ADD_ZPmZ_[BHSD]", + "^(UQSUB|UQSUBR)_ZPmZ_[BHSD]")>; // Arithmetic, large integer def : InstRW<[N2Write_2cyc_1V], (instregex "^(AD|SB)CL[BT]_ZZZ_[SD]$")>; @@ -1620,12 +1622,13 @@ def : InstRW<[N2Write_4cyc_1V1], (instregex "^[SU]ADALP_ZPmZ_[HSD]$")>; // Arithmetic, shift def : InstRW<[N2Write_2cyc_1V1], - (instregex "^(ASR|LSL|LSR)_WIDE_ZPmZ_[BHS]$", - "^(ASR|LSL|LSR)_WIDE_ZZZ_[BHS]$", - "^(ASR|LSL|LSR)_ZPmI_[BHSD]$", - "^(ASR|LSL|LSR)_ZPmZ_[BHSD]$", - "^(ASR|LSL|LSR)_ZZI_[BHSD]$", - "^(ASRR|LSLR|LSRR)_ZPmZ_[BHSD]$")>; + (instregex "^(ASR|LSL|LSR)_WIDE_ZPmZ_[BHS]", + "^(ASR|LSL|LSR)_WIDE_ZZZ_[BHS]", + "^(ASR|LSL|LSR)_ZPmI_[BHSD]", + "^(ASR|LSL|LSR)_ZPmZ_[BHSD]", + "^(ASR|LSL|LSR)_ZZI_[BHSD]", + "^(ASR|LSL|LSR)_ZPZ[IZ]_[BHSD]", + "^(ASRR|LSLR|LSRR)_ZPmZ_[BHSD]")>; // Arithmetic, shift and accumulate def : InstRW<[N2Write_4cyc_1V1], @@ -1638,29 +1641,29 @@ def : InstRW<[N2Write_2cyc_1V1], // Arithmetic, shift complex def : InstRW<[N2Write_4cyc_1V1], - (instregex "^(SQ)?RSHRU?N[BT]_ZZI_[BHS]$", - "^(SQRSHL|SQRSHLR|SQSHL|SQSHLR|UQRSHL|UQRSHLR|UQSHL|UQSHLR)_ZPmZ_[BHSD]$", - "^(SQSHL|SQSHLU|UQSHL)_ZPmI_[BHSD]$", - "^SQSHRU?N[BT]_ZZI_[BHS]$", - "^UQR?SHRN[BT]_ZZI_[BHS]$")>; + (instregex "^(SQ)?RSHRU?N[BT]_ZZI_[BHS]", + "^(SQRSHL|SQRSHLR|SQSHL|SQSHLR|UQRSHL|UQRSHLR|UQSHL|UQSHLR)_ZPmZ_[BHSD]", + "^[SU]QR?SHL_ZPZZ_[BHSD]", + "^(SQSHL|SQSHLU|UQSHL)_(ZPmI|ZPZI)_[BHSD]", + "^SQSHRU?N[BT]_ZZI_[BHS]", + "^UQR?SHRN[BT]_ZZI_[BHS]")>; // Arithmetic, shift right for divide -def : InstRW<[N2Write_4cyc_1V1], (instregex "^ASRD_ZPmI_[BHSD]$")>; +def : InstRW<[N2Write_4cyc_1V1], (instregex "^ASRD_(ZPmI|ZPZI)_[BHSD]")>; // Arithmetic, shift rounding -def : InstRW<[N2Write_4cyc_1V1], - (instregex "^(SRSHL|SRSHLR|URSHL|URSHLR)_ZPmZ_[BHSD]$", - "^[SU]RSHR_ZPmI_[BHSD]$")>; +def : InstRW<[N2Write_4cyc_1V1], (instregex "^[SU]RSHLR?_ZPmZ_[BHSD]", + "^[SU]RSHL_ZPZZ_[BHSD]", + "^[SU]RSHR_(ZPmI|ZPZI)_[BHSD]")>; // Bit manipulation -def : InstRW<[N2Write_6cyc_2V1], - (instregex "^(BDEP|BEXT|BGRP)_ZZZ_[BHSD]$")>; +def : InstRW<[N2Write_6cyc_2V1], (instregex "^(BDEP|BEXT|BGRP)_ZZZ_[BHSD]")>; // Bitwise select def : InstRW<[N2Write_2cyc_1V], (instregex "^(BSL|BSL1N|BSL2N|NBSL)_ZZZZ$")>; // Count/reverse bits -def : InstRW<[N2Write_2cyc_1V], (instregex "^(CLS|CLZ|CNT|RBIT)_ZPmZ_[BHSD]$")>; +def : InstRW<[N2Write_2cyc_1V], (instregex "^(CLS|CLZ|CNT|RBIT)_ZPmZ_[BHSD]")>; // Broadcast logical bitmask immediate to vector def : InstRW<[N2Write_2cyc_1V], (instrs DUPM_ZI)>; @@ -1695,19 +1698,14 @@ def : InstRW<[N2Write_3cyc_1V1], (instregex "^CLAST[AB]_[VZ]PZ_[BHSD]$", "^SPLICE_ZPZZ?_[BHSD]$")>; // Convert to floating point, 64b to float or convert to double -def : InstRW<[N2Write_3cyc_1V0], (instregex "^[SU]CVTF_ZPmZ_Dto[SD]$")>; - -// Convert to floating point, 64b to half -def : InstRW<[N2Write_3cyc_1V0], (instregex "^[SU]CVTF_ZPmZ_DtoH$")>; +def : InstRW<[N2Write_3cyc_1V0], (instregex "^[SU]CVTF_ZPmZ_Dto[HSD]", + "^[SU]CVTF_ZPmZ_StoD")>; // Convert to floating point, 32b to single or half -def : InstRW<[N2Write_4cyc_2V0], (instregex "^[SU]CVTF_ZPmZ_Sto[HS]$")>; - -// Convert to floating point, 32b to double -def : InstRW<[N2Write_3cyc_1V0], (instregex "^[SU]CVTF_ZPmZ_StoD$")>; +def : InstRW<[N2Write_4cyc_2V0], (instregex "^[SU]CVTF_ZPmZ_Sto[HS]")>; // Convert to floating point, 16b to half -def : InstRW<[N2Write_6cyc_4V0], (instregex "^[SU]CVTF_ZPmZ_HtoH$")>; +def : InstRW<[N2Write_6cyc_4V0], (instregex "^[SU]CVTF_ZPmZ_HtoH")>; // Copy, scalar def : InstRW<[N2Write_5cyc_1M0_1V], (instregex "^CPY_ZPmR_[BHSD]$")>; @@ -1717,10 +1715,12 @@ def : InstRW<[N2Write_2cyc_1V], (instregex "^CPY_ZPm[IV]_[BHSD]$", "^CPY_ZPzI_[BHSD]$")>; // Divides, 32 bit -def : InstRW<[N2Write_12cyc_1V0], (instregex "^[SU]DIVR?_ZPmZ_S$")>; +def : InstRW<[N2Write_12cyc_1V0], (instregex "^[SU]DIVR?_ZPmZ_S", + "^[SU]DIV_ZPZZ_S")>; // Divides, 64 bit -def : InstRW<[N2Write_20cyc_1V0], (instregex "^[SU]DIVR?_ZPmZ_D$")>; +def : InstRW<[N2Write_20cyc_1V0], (instregex "^[SU]DIVR?_ZPmZ_D", + "^[SU]DIV_ZPZZ_D")>; // Dot product, 8 bit def : InstRW<[N2Write_3cyc_1V], (instregex "^[SU]DOT_ZZZI?_S$")>; @@ -1739,9 +1739,9 @@ def : InstRW<[N2Write_2cyc_1V], (instregex "^DUP_ZI_[BHSD]$", def : InstRW<[N2Write_3cyc_1M0], (instregex "^DUP_ZR_[BHSD]$")>; // Extend, sign or zero -def : InstRW<[N2Write_2cyc_1V1], (instregex "^[SU]XTB_ZPmZ_[HSD]$", - "^[SU]XTH_ZPmZ_[SD]$", - "^[SU]XTW_ZPmZ_[D]$")>; +def : InstRW<[N2Write_2cyc_1V1], (instregex "^[SU]XTB_ZPmZ_[HSD]", + "^[SU]XTH_ZPmZ_[SD]", + "^[SU]XTW_ZPmZ_[D]")>; // Extract def : InstRW<[N2Write_2cyc_1V], (instrs EXT_ZZI, EXT_ZZI_B)>; @@ -1778,14 +1778,16 @@ def : InstRW<[N2Write_8cyc_2M0_2V0], (instregex "^INDEX_(IR|RI|RR)_D$")>; // Logical def : InstRW<[N2Write_2cyc_1V], - (instregex "^(AND|EOR|ORR)_ZI$", - "^(AND|BIC|EOR|EOR(BT|TB)?|ORR)_ZZZ$", - "^EOR(BT|TB)_ZZZ_[BHSD]$", - "^(AND|BIC|EOR|NOT|ORR)_ZPmZ_[BHSD]$")>; + (instregex "^(AND|EOR|ORR)_ZI", + "^(AND|BIC|EOR|ORR)_ZZZ", + "^EOR(BT|TB)_ZZZ_[BHSD]", + "^(AND|BIC|EOR|NOT|ORR)_(ZPmZ|ZPZZ)_[BHSD]", + "^NOT_ZPmZ_[BHSD]")>; // Max/min, basic and pairwise -def : InstRW<[N2Write_2cyc_1V], (instregex "^[SU](MAX|MIN)_ZI_[BHSD]$", - "^[SU](MAX|MIN)P?_ZPmZ_[BHSD]$")>; +def : InstRW<[N2Write_2cyc_1V], (instregex "^[SU](MAX|MIN)_ZI_[BHSD]", + "^[SU](MAX|MIN)P?_ZPmZ_[BHSD]", + "^[SU](MAX|MIN)_ZPZZ_[BHSD]")>; // Matching operations def : InstRW<[N2Write_2cyc_1V0_1M], (instregex "^N?MATCH_PPzZZ_[BH]$")>; @@ -1798,12 +1800,16 @@ def : InstRW<[N2Write_2cyc_1V], (instregex "^MOVPRFX_ZP[mz]Z_[BHSD]$", "^MOVPRFX_ZZ$")>; // Multiply, B, H, S element size -def : InstRW<[N2Write_4cyc_1V0], (instregex "^MUL_(ZI|ZPmZ|ZZZI|ZZZ)_[BHS]$", - "^[SU]MULH_(ZPmZ|ZZZ)_[BHS]$")>; +def : InstRW<[N2Write_4cyc_1V0], (instregex "^MUL_(ZI|ZPmZ|ZZZI|ZZZ)_[BHS]", + "^MUL_ZPZZ_[BHS]", + "^[SU]MULH_(ZPmZ|ZZZ)_[BHS]", + "^[SU]MULH_ZPZZ_[BHS]")>; // Multiply, D element size -def : InstRW<[N2Write_5cyc_2V0], (instregex "^MUL_(ZI|ZPmZ|ZZZI|ZZZ)_D$", - "^[SU]MULH_(ZPmZ|ZZZ)_D$")>; +def : InstRW<[N2Write_5cyc_2V0], (instregex "^MUL_(ZI|ZPmZ|ZZZI|ZZZ)_D", + "^MUL_ZPZZ_D", + "^[SU]MULH_(ZPmZ|ZZZ)_D", + "^[SU]MULH_ZPZZ_D")>; // Multiply long def : InstRW<[N2Write_4cyc_1V0], (instregex "^[SU]MULL[BT]_ZZZI_[SD]$", @@ -1811,11 +1817,11 @@ def : InstRW<[N2Write_4cyc_1V0], (instregex "^[SU]MULL[BT]_ZZZI_[SD]$", // Multiply accumulate, B, H, S element size def : InstRW<[N2Write_4cyc_1V0], (instregex "^ML[AS]_ZZZI_[BHS]$", - "^(ML[AS]|MAD|MSB)_ZPmZZ_[BHS]$")>; + "^(ML[AS]|MAD|MSB)_(ZPmZZ|ZPZZZ)_[BHS]")>; // Multiply accumulate, D element size def : InstRW<[N2Write_5cyc_2V0], (instregex "^ML[AS]_ZZZI_D$", - "^(ML[AS]|MAD|MSB)_ZPmZZ_D$")>; + "^(ML[AS]|MAD|MSB)_(ZPmZZ|ZPZZZ)_D")>; // Multiply accumulate long def : InstRW<[N2Write_4cyc_1V0], (instregex "^[SU]ML[AS]L[BT]_ZZZ_[HSD]$", @@ -1864,7 +1870,7 @@ def : InstRW<[N2Write_2cyc_1V0], (instregex "^(DEC|INC|SQDEC|SQINC|UQDEC|UQINC)[HWD]_ZPiI$")>; // Reciprocal estimate -def : InstRW<[N2Write_4cyc_2V0], (instrs URECPE_ZPmZ_S, URSQRTE_ZPmZ_S)>; +def : InstRW<[N2Write_4cyc_2V0], (instregex "^URECPE_ZPmZ_S", "^URSQRTE_ZPmZ_S")>; // Reduction, arithmetic, B form def : InstRW<[N2Write_11cyc_2V_2V1], (instregex "^[SU](ADD|MAX|MIN)V_VPZ_B")>; @@ -1909,13 +1915,17 @@ def : InstRW<[N2Write_2cyc_1V], (instregex "^(UZP|ZIP)[12]_ZZZ_[BHSDQ]$")>; // ----------------------------------------------------------------------------- // Floating point absolute value/difference -def : InstRW<[N2Write_2cyc_1V], (instregex "^FAB[SD]_ZPmZ_[HSD]$")>; +def : InstRW<[N2Write_2cyc_1V], (instregex "^FAB[SD]_ZPmZ_[HSD]", + "^FABD_ZPZZ_[HSD]", + "^FABS_ZPmZ_[HSD]")>; // Floating point arithmetic -def : InstRW<[N2Write_2cyc_1V], (instregex "^F(ADD|SUB)_(ZPm[IZ]|ZZZ)_[HSD]$", - "^FADDP_ZPmZZ_[HSD]$", - "^FNEG_ZPmZ_[HSD]$", - "^FSUBR_ZPm[IZ]_[HSD]$")>; +def : InstRW<[N2Write_2cyc_1V], (instregex "^F(ADD|SUB)_(ZPm[IZ]|ZZZ)_[HSD]", + "^F(ADD|SUB)_ZPZ[IZ]_[HSD]", + "^FADDP_ZPmZZ_[HSD]", + "^FNEG_ZPmZ_[HSD]", + "^FSUBR_ZPm[IZ]_[HSD]", + "^FSUBR_(ZPZI|ZPZZ)_[HSD]")>; // Floating point associative add, F16 def : InstRW<[N2Write_10cyc_1V1], (instrs FADDA_VPZ_H)>; @@ -1940,80 +1950,80 @@ def : InstRW<[N2Write_5cyc_1V], (instregex "^FCMLA_ZPmZZ_[HSD]$", "^FCMLA_ZZZI_[HS]$")>; // Floating point convert, long or narrow (F16 to F32 or F32 to F16) -def : InstRW<[N2Write_4cyc_2V0], (instregex "^FCVT_ZPmZ_(HtoS|StoH)$", - "^FCVTLT_ZPmZ_HtoS$", - "^FCVTNT_ZPmZ_StoH$")>; +def : InstRW<[N2Write_4cyc_2V0], (instregex "^FCVT_ZPmZ_(HtoS|StoH)", + "^FCVTLT_ZPmZ_HtoS", + "^FCVTNT_ZPmZ_StoH")>; // Floating point convert, long or narrow (F16 to F64, F32 to F64, F64 to F32 // or F64 to F16) -def : InstRW<[N2Write_3cyc_1V0], (instregex "^FCVT_ZPmZ_(HtoD|StoD|DtoS|DtoH)$", - "^FCVTLT_ZPmZ_StoD$", - "^FCVTNT_ZPmZ_DtoS$")>; +def : InstRW<[N2Write_3cyc_1V0], (instregex "^FCVT_ZPmZ_(HtoD|StoD|DtoS|DtoH)", + "^FCVTLT_ZPmZ_StoD", + "^FCVTNT_ZPmZ_DtoS")>; // Floating point convert, round to odd def : InstRW<[N2Write_3cyc_1V0], (instrs FCVTX_ZPmZ_DtoS, FCVTXNT_ZPmZ_DtoS)>; // Floating point base2 log, F16 -def : InstRW<[N2Write_6cyc_4V0], (instrs FLOGB_ZPmZ_H)>; +def : InstRW<[N2Write_6cyc_4V0], (instregex "^FLOGB_(ZPmZ|ZPZZ)_H")>; // Floating point base2 log, F32 -def : InstRW<[N2Write_4cyc_2V0], (instrs FLOGB_ZPmZ_S)>; +def : InstRW<[N2Write_4cyc_2V0], (instregex "^FLOGB_(ZPmZ|ZPZZ)_S")>; // Floating point base2 log, F64 -def : InstRW<[N2Write_3cyc_1V0], (instrs FLOGB_ZPmZ_D)>; +def : InstRW<[N2Write_3cyc_1V0], (instregex "^FLOGB_(ZPmZ|ZPZZ)_D")>; // Floating point convert to integer, F16 -def : InstRW<[N2Write_6cyc_4V0], (instregex "^FCVTZ[SU]_ZPmZ_HtoH$")>; +def : InstRW<[N2Write_6cyc_4V0], (instregex "^FCVTZ[SU]_ZPmZ_HtoH")>; // Floating point convert to integer, F32 -def : InstRW<[N2Write_4cyc_2V0], (instregex "^FCVTZ[SU]_ZPmZ_(HtoS|StoS)$")>; +def : InstRW<[N2Write_4cyc_2V0], (instregex "^FCVTZ[SU]_ZPmZ_(HtoS|StoS)")>; // Floating point convert to integer, F64 def : InstRW<[N2Write_3cyc_1V0], - (instregex "^FCVTZ[SU]_ZPmZ_(HtoD|StoD|DtoS|DtoD)$")>; + (instregex "^FCVTZ[SU]_ZPmZ_(HtoD|StoD|DtoS|DtoD)")>; // Floating point copy def : InstRW<[N2Write_2cyc_1V], (instregex "^FCPY_ZPmI_[HSD]$", "^FDUP_ZI_[HSD]$")>; // Floating point divide, F16 -def : InstRW<[N2Write_13cyc_1V0], (instregex "^FDIVR?_ZPmZ_H$")>; +def : InstRW<[N2Write_13cyc_1V0], (instregex "^FDIVR?_(ZPmZ|ZPZZ)_H")>; // Floating point divide, F32 -def : InstRW<[N2Write_10cyc_1V0], (instregex "^FDIVR?_ZPmZ_S$")>; +def : InstRW<[N2Write_10cyc_1V0], (instregex "^FDIVR?_(ZPmZ|ZPZZ)_S")>; // Floating point divide, F64 -def : InstRW<[N2Write_15cyc_1V0], (instregex "^FDIVR?_ZPmZ_D$")>; +def : InstRW<[N2Write_15cyc_1V0], (instregex "^FDIVR?_(ZPmZ|ZPZZ)_D")>; // Floating point min/max pairwise -def : InstRW<[N2Write_2cyc_1V], (instregex "^F(MAX|MIN)(NM)?P_ZPmZZ_[HSD]$")>; +def : InstRW<[N2Write_2cyc_1V], (instregex "^F(MAX|MIN)(NM)?P_ZPmZZ_[HSD]")>; // Floating point min/max -def : InstRW<[N2Write_2cyc_1V], (instregex "^F(MAX|MIN)(NM)?_ZPm[IZ]_[HSD]$")>; +def : InstRW<[N2Write_2cyc_1V], (instregex "^F(MAX|MIN)(NM)?_ZPm[IZ]_[HSD]", + "^F(MAX|MIN)(NM)?_ZPZ[IZ]_[HSD]")>; // Floating point multiply -def : InstRW<[N2Write_3cyc_1V], (instregex "^(FSCALE|FMULX)_ZPmZ_[HSD]$", - "^FMUL_(ZPm[IZ]|ZZZI?)_[HSD]$")>; +def : InstRW<[N2Write_3cyc_1V], (instregex "^(FSCALE|FMULX)_ZPmZ_[HSD]", + "^FMULX_ZPZZ_[HSD]", + "^FMUL_(ZPm[IZ]|ZZZI?)_[HSD]", + "^FMUL_ZPZ[IZ]_[HSD]")>; // Floating point multiply accumulate -def : InstRW<[N2Write_4cyc_1V], - (instregex "^FML[AS]_(ZPmZZ|ZZZI)_[HSD]$", - "^(FMAD|FNMAD|FNML[AS]|FN?MSB)_ZPmZZ_[HSD]$")>; +def : InstRW<[N2Write_4cyc_1V], (instregex "^F(N?M(AD|SB)|N?ML[AS])_ZPmZZ_[HSD]$", + "^FN?ML[AS]_ZPZZZ_[HSD]", + "^FML[AS]_ZZZI_[HSD]$")>; // Floating point multiply add/sub accumulate long def : InstRW<[N2Write_4cyc_1V], (instregex "^FML[AS]L[BT]_ZZZI?_SHH$")>; // Floating point reciprocal estimate, F16 -def : InstRW<[N2Write_6cyc_4V0], (instrs FRECPE_ZZ_H, FRECPX_ZPmZ_H, - FRSQRTE_ZZ_H)>; +def : InstRW<[N2Write_6cyc_4V0], (instregex "^FR(ECP|SQRT)E_ZZ_H", "^FRECPX_ZPmZ_H")>; // Floating point reciprocal estimate, F32 -def : InstRW<[N2Write_4cyc_2V0], (instrs FRECPE_ZZ_S, FRECPX_ZPmZ_S, - FRSQRTE_ZZ_S)>; +def : InstRW<[N2Write_4cyc_2V0], (instregex "^FR(ECP|SQRT)E_ZZ_S", "^FRECPX_ZPmZ_S")>; // Floating point reciprocal estimate, F64 -def : InstRW<[N2Write_3cyc_1V0], (instrs FRECPE_ZZ_D, FRECPX_ZPmZ_D, - FRSQRTE_ZZ_D)>; +def : InstRW<[N2Write_3cyc_1V0], (instregex "^FR(ECP|SQRT)E_ZZ_D", "^FRECPX_ZPmZ_D")>; // Floating point reciprocal step def : InstRW<[N2Write_4cyc_1V0], (instregex "^F(RECPS|RSQRTS)_ZZZ_[HSD]$")>; @@ -2031,22 +2041,22 @@ def : InstRW<[N2Write_2cyc_1V], (instregex "^(FADDV|FMAXNMV|FMAXV|FMINNMV|FMINV)_VPZ_D$")>; // Floating point round to integral, F16 -def : InstRW<[N2Write_6cyc_4V0], (instregex "^FRINT[AIMNPXZ]_ZPmZ_H$")>; +def : InstRW<[N2Write_6cyc_4V0], (instregex "^FRINT[AIMNPXZ]_ZPmZ_H")>; // Floating point round to integral, F32 -def : InstRW<[N2Write_4cyc_2V0], (instregex "^FRINT[AIMNPXZ]_ZPmZ_S$")>; +def : InstRW<[N2Write_4cyc_2V0], (instregex "^FRINT[AIMNPXZ]_ZPmZ_S")>; // Floating point round to integral, F64 -def : InstRW<[N2Write_3cyc_1V0], (instregex "^FRINT[AIMNPXZ]_ZPmZ_D$")>; +def : InstRW<[N2Write_3cyc_1V0], (instregex "^FRINT[AIMNPXZ]_ZPmZ_D")>; // Floating point square root, F16 -def : InstRW<[N2Write_13cyc_1V0], (instrs FSQRT_ZPmZ_H)>; +def : InstRW<[N2Write_13cyc_1V0], (instregex "^FSQRT_ZPmZ_H")>; // Floating point square root, F32 -def : InstRW<[N2Write_10cyc_1V0], (instrs FSQRT_ZPmZ_S)>; +def : InstRW<[N2Write_10cyc_1V0], (instregex "^FSQRT_ZPmZ_S")>; // Floating point square root, F64 -def : InstRW<[N2Write_16cyc_1V0], (instrs FSQRT_ZPmZ_D)>; +def : InstRW<[N2Write_16cyc_1V0], (instregex "^FSQRT_ZPmZ_D")>; // Floating point trigonometric exponentiation def : InstRW<[N2Write_3cyc_1V1], (instregex "^FEXPA_ZZ_[HSD]$")>; diff --git a/llvm/unittests/Target/AArch64/AArch64SVESchedPseudoTest.cpp b/llvm/unittests/Target/AArch64/AArch64SVESchedPseudoTest.cpp index 6098d4e62392..a527bdeef724 100644 --- a/llvm/unittests/Target/AArch64/AArch64SVESchedPseudoTest.cpp +++ b/llvm/unittests/Target/AArch64/AArch64SVESchedPseudoTest.cpp @@ -107,6 +107,10 @@ TEST(AArch64SVESchedPseudoTesta510, IsCorrect) { runSVEPseudoTestForCPU("cortex-a510"); } +TEST(AArch64SVESchedPseudoTestn1, IsCorrect) { + runSVEPseudoTestForCPU("neoverse-n2"); +} + TEST(AArch64SVESchedPseudoTestv1, IsCorrect) { runSVEPseudoTestForCPU("neoverse-v1"); } -- GitLab From 3a62edcf527bfc927f148c99b9d37d3d199db611 Mon Sep 17 00:00:00 2001 From: zhongyunde 00443407 Date: Wed, 6 Mar 2024 14:09:18 -0500 Subject: [PATCH 581/929] [AArch64] MachineCombiner msub matching Pattern should be sorted in priority order since the pattern evalutor stops checking as soon as it finds a faster sequence. so for a * b - c * d, we prefer to match the 2nd operands of sub, which can be use msub to fold them. Refer to https://www.slideshare.net/chimerawang/instruction-combine-in-llvm Fix https://github.com/llvm/llvm-project/issues/84152 --- llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 2 +- llvm/test/CodeGen/AArch64/scalar-mla-mls.ll | 31 ++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 llvm/test/CodeGen/AArch64/scalar-mla-mls.ll diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp index 5df691f35275..5893f76dbd55 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -6110,8 +6110,8 @@ static bool getMaddPatterns(MachineInstr &Root, setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULADDX_OP2); break; case AArch64::SUBWrr: - setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1); setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULSUBW_OP2); + setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1); break; case AArch64::SUBXrr: setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1); diff --git a/llvm/test/CodeGen/AArch64/scalar-mla-mls.ll b/llvm/test/CodeGen/AArch64/scalar-mla-mls.ll new file mode 100644 index 000000000000..36ac36701fa8 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/scalar-mla-mls.ll @@ -0,0 +1,31 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -mattr=+neon | FileCheck %s + +define ptr @test_scalar_msub(ptr %a, ptr %b) { +; CHECK-LABEL: test_scalar_msub: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ldp w8, w11, [x1] +; CHECK-NEXT: ldp w9, w10, [x0] +; CHECK-NEXT: mul w12, w8, w9 +; CHECK-NEXT: mul w8, w10, w8 +; CHECK-NEXT: madd w8, w11, w9, w8 +; CHECK-NEXT: msub w9, w11, w10, w12 +; CHECK-NEXT: stp w9, w8, [x0] +; CHECK-NEXT: ret +entry: + %0 = load i32, ptr %a, align 4 + %1 = load i32, ptr %b, align 4 + %mul = mul nsw i32 %1, %0 + %_M_imag = getelementptr inbounds i8, ptr %a, i64 4 + %2 = load i32, ptr %_M_imag, align 4 + %_M_imag.i = getelementptr inbounds i8, ptr %b, i64 4 + %3 = load i32, ptr %_M_imag.i, align 4 + %mul3 = mul nsw i32 %3, %2 + %sub = sub nsw i32 %mul, %mul3 + %mul6 = mul nsw i32 %3, %0 + %mul9 = mul nsw i32 %2, %1 + %add = add nsw i32 %mul6, %mul9 + store i32 %add, ptr %_M_imag, align 4 + store i32 %sub, ptr %a, align 4 + ret ptr %a +} -- GitLab From a110a1c0ed9850be168cd0e29f05179e80941b04 Mon Sep 17 00:00:00 2001 From: zhongyunde 00443407 Date: Fri, 8 Mar 2024 11:27:48 -0500 Subject: [PATCH 582/929] [AArch64] MachineCombiner msub matching for i64 --- llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 2 +- llvm/test/CodeGen/AArch64/scalar-mla-mls.ll | 32 ++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp index 5893f76dbd55..02943b8a4ab1 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -6114,8 +6114,8 @@ static bool getMaddPatterns(MachineInstr &Root, setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1); break; case AArch64::SUBXrr: - setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1); setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULSUBX_OP2); + setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1); break; case AArch64::ADDWri: setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDWI_OP1); diff --git a/llvm/test/CodeGen/AArch64/scalar-mla-mls.ll b/llvm/test/CodeGen/AArch64/scalar-mla-mls.ll index 36ac36701fa8..c8ba50ae0cc3 100644 --- a/llvm/test/CodeGen/AArch64/scalar-mla-mls.ll +++ b/llvm/test/CodeGen/AArch64/scalar-mla-mls.ll @@ -29,3 +29,35 @@ entry: store i32 %sub, ptr %a, align 4 ret ptr %a } + +define ptr @test_scalar_msub_i64(ptr %a, ptr %b) { +; CHECK-LABEL: test_scalar_msub_i64: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ldr x8, [x1] +; CHECK-NEXT: ldur x9, [x0, #4] +; CHECK-NEXT: ldr x10, [x0] +; CHECK-NEXT: ldur x12, [x1, #4] +; CHECK-NEXT: mul x11, x9, x8 +; CHECK-NEXT: mul x8, x8, x10 +; CHECK-NEXT: madd x10, x12, x10, x11 +; CHECK-NEXT: msub x8, x12, x9, x8 +; CHECK-NEXT: stur x10, [x0, #4] +; CHECK-NEXT: str x8, [x0] +; CHECK-NEXT: ret +entry: + %0 = load i64, ptr %a, align 8 + %1 = load i64, ptr %b, align 8 + %mul = mul nsw i64 %1, %0 + %_M_imag = getelementptr inbounds i8, ptr %a, i64 4 + %2 = load i64, ptr %_M_imag, align 8 + %_M_imag.i = getelementptr inbounds i8, ptr %b, i64 4 + %3 = load i64, ptr %_M_imag.i, align 8 + %mul3 = mul nsw i64 %3, %2 + %sub = sub nsw i64 %mul, %mul3 + %mul6 = mul nsw i64 %3, %0 + %mul9 = mul nsw i64 %2, %1 + %add = add nsw i64 %mul6, %mul9 + store i64 %add, ptr %_M_imag, align 8 + store i64 %sub, ptr %a, align 8 + ret ptr %a +} -- GitLab From 860a40057da2fbb626fdf17feec18070d95b687b Mon Sep 17 00:00:00 2001 From: Tom Eccles Date: Fri, 8 Mar 2024 10:33:43 +0000 Subject: [PATCH 583/929] [flang][NFC] move loadIfRef to FIRBuilder (#84306) This will be useful for OpenMP too. I changed the definition slightly to use `fir::isa_ref_type` (which also includes llvm pointers) because I think it reads better using the common type helpers. There shouldn't be any llvm pointers in lowering so this isn't a functional change. --- flang/include/flang/Optimizer/Builder/FIRBuilder.h | 4 ++++ flang/lib/Lower/OpenACC.cpp | 12 ++---------- flang/lib/Optimizer/Builder/FIRBuilder.cpp | 7 +++++++ 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/flang/include/flang/Optimizer/Builder/FIRBuilder.h b/flang/include/flang/Optimizer/Builder/FIRBuilder.h index bd9b67b14b96..d61bf681be61 100644 --- a/flang/include/flang/Optimizer/Builder/FIRBuilder.h +++ b/flang/include/flang/Optimizer/Builder/FIRBuilder.h @@ -309,6 +309,10 @@ public: void createStoreWithConvert(mlir::Location loc, mlir::Value val, mlir::Value addr); + /// Create a fir.load if \p val is a reference or pointer type. Return the + /// result of the load if it was created, otherwise return \p val + mlir::Value loadIfRef(mlir::Location loc, mlir::Value val); + /// Create a new FuncOp. If the function may have already been created, use /// `addNamedFunction` instead. mlir::func::FuncOp createFunction(mlir::Location loc, llvm::StringRef name, diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp index 151077d81ba1..d2c6006ecf91 100644 --- a/flang/lib/Lower/OpenACC.cpp +++ b/flang/lib/Lower/OpenACC.cpp @@ -1041,14 +1041,6 @@ static mlir::Value genLogicalCombiner(fir::FirOpBuilder &builder, return builder.create(loc, value1.getType(), combined); } -static mlir::Value loadIfRef(fir::FirOpBuilder &builder, mlir::Location loc, - mlir::Value value) { - if (mlir::isa( - value.getType())) - return builder.create(loc, value); - return value; -} - static mlir::Value genComparisonCombiner(fir::FirOpBuilder &builder, mlir::Location loc, mlir::arith::CmpIPredicate pred, @@ -1066,8 +1058,8 @@ static mlir::Value genScalarCombiner(fir::FirOpBuilder &builder, mlir::acc::ReductionOperator op, mlir::Type ty, mlir::Value value1, mlir::Value value2) { - value1 = loadIfRef(builder, loc, value1); - value2 = loadIfRef(builder, loc, value2); + value1 = builder.loadIfRef(loc, value1); + value2 = builder.loadIfRef(loc, value2); if (op == mlir::acc::ReductionOperator::AccAdd) { if (ty.isIntOrIndex()) return builder.create(loc, value1, value2); diff --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp index 788c99e40105..12da7412888a 100644 --- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp +++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp @@ -16,6 +16,7 @@ #include "flang/Optimizer/Builder/Todo.h" #include "flang/Optimizer/Dialect/FIRAttr.h" #include "flang/Optimizer/Dialect/FIROpsSupport.h" +#include "flang/Optimizer/Dialect/FIRType.h" #include "flang/Optimizer/Support/FatalError.h" #include "flang/Optimizer/Support/InternalNames.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" @@ -404,6 +405,12 @@ void fir::FirOpBuilder::createStoreWithConvert(mlir::Location loc, create(loc, cast, addr); } +mlir::Value fir::FirOpBuilder::loadIfRef(mlir::Location loc, mlir::Value val) { + if (fir::isa_ref_type(val.getType())) + return create(loc, val); + return val; +} + fir::StringLitOp fir::FirOpBuilder::createStringLitOp(mlir::Location loc, llvm::StringRef data) { auto type = fir::CharacterType::get(getContext(), 1, data.size()); -- GitLab From c76d853c176231b991cfc744e7cb69443ddc2268 Mon Sep 17 00:00:00 2001 From: Tom Eccles Date: Fri, 8 Mar 2024 10:34:00 +0000 Subject: [PATCH 584/929] [flang][TBAABuilder] not all loads and stores are inside of functions (#84305) TBAA builder assumed that all loads/stores are inside of functions and hit an assertion once it found loads and stores inside of an omp::ReductionDeclareOp. For now just don't add TBAA tags to those loads and stores. They would end up in a different TBAA tree to the host function after OpenMPIRBuilder inlines them anyway so there isn't an easy way of making this work. --- flang/lib/Optimizer/CodeGen/TBAABuilder.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flang/lib/Optimizer/CodeGen/TBAABuilder.cpp b/flang/lib/Optimizer/CodeGen/TBAABuilder.cpp index 8e7f59f76383..b1b0e9b766a6 100644 --- a/flang/lib/Optimizer/CodeGen/TBAABuilder.cpp +++ b/flang/lib/Optimizer/CodeGen/TBAABuilder.cpp @@ -102,7 +102,8 @@ void TBAABuilder::attachTBAATag(AliasAnalysisOpInterface op, Type baseFIRType, return; mlir::LLVM::LLVMFuncOp func = op->getParentOfType(); - assert(func && "func.func should have already been converted to llvm.func"); + if (!func) + return; ++tagAttachmentCounter; if (tagAttachmentLimit != kTagAttachmentUnlimited && -- GitLab From 419e8da2dadc7eaa1499935f55303c3f132948c4 Mon Sep 17 00:00:00 2001 From: Christian Sigg Date: Fri, 8 Mar 2024 11:37:10 +0100 Subject: [PATCH 585/929] [mlir][bazel] Fix BUILD after 5669660f37ef1800f4a7852577364b024d75e3d8 --- .../llvm-project-overlay/mlir/BUILD.bazel | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index a2f170222684..cca548d51348 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -3637,6 +3637,107 @@ cc_library( ], ) +##---------------------------------------------------------------------------## +# XeGPU dialect. +##---------------------------------------------------------------------------## + +td_library( + name = "XeGPUTdFiles", + srcs = glob(["include/mlir/Dialect/XeGPU/IR/*.td"]), + includes = ["include"], + deps = [ + ":BuiltinDialectTdFiles", + ":OpBaseTdFiles", + ], +) + +gentbl_cc_library( + name = "XeGPUIncGen", + tbl_outs = [ + ( + [ + "-gen-dialect-decls", + "-dialect=xegpu", + ], + "include/mlir/Dialect/XeGPU/IR/XeGPUDialect.h.inc", + ), + ( + [ + "-gen-dialect-defs", + "-dialect=xegpu", + ], + "include/mlir/Dialect/XeGPU/IR/XeGPUDialect.cpp.inc", + ), + ( + ["-gen-op-decls"], + "include/mlir/Dialect/XeGPU/IR/XeGPU.h.inc", + ), + ( + ["-gen-op-defs"], + "include/mlir/Dialect/XeGPU/IR/XeGPU.cpp.inc", + ), + ( + ["-gen-op-doc"], + "g3doc/Dialects/XeGPU/XeGPU.md", + ), + ( + [ + "-gen-typedef-decls", + "-typedefs-dialect=xegpu", + ], + "include/mlir/Dialect/XeGPU/IR/XeGPUTypes.h.inc", + ), + ( + [ + "-gen-typedef-defs", + "-typedefs-dialect=xegpu", + ], + "include/mlir/Dialect/XeGPU/IR/XeGPUTypes.cpp.inc", + ), + ( + ["-gen-enum-decls"], + "include/mlir/Dialect/XeGPU/IR/XeGPUEnums.h.inc", + ), + ( + ["-gen-enum-defs"], + "include/mlir/Dialect/XeGPU/IR/XeGPUEnums.cpp.inc", + ), + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=xegpu", + ], + "include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=xegpu", + ], + "include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.cpp.inc", + ), + ], + tblgen = ":mlir-tblgen", + td_file = "include/mlir/Dialect/XeGPU/IR/XeGPU.td", + deps = [":XeGPUTdFiles"], +) + +cc_library( + name = "XeGPUDialect", + srcs = [ + "lib/Dialect/XeGPU/IR/XeGPUDialect.cpp", + "lib/Dialect/XeGPU/IR/XeGPUOps.cpp", + ], + hdrs = ["include/mlir/Dialect/XeGPU/IR/XeGPU.h"], + includes = ["include"], + deps = [ + ":IR", + ":XeGPUIncGen", + "//llvm:Core", + "//llvm:Support", + ], +) + td_library( name = "FuncTdFiles", srcs = [ -- GitLab From dd547d5cca64c1f97d00af0c3a555c2b341161b0 Mon Sep 17 00:00:00 2001 From: Christian Sigg Date: Fri, 8 Mar 2024 11:39:59 +0100 Subject: [PATCH 586/929] [mlir][bazel] Fix BUILD after 5669660f37ef1800f4a7852577364b024d75e3d8 Second attempt. --- utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index cca548d51348..cadcbcc5a092 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -9269,6 +9269,7 @@ cc_library( ":VectorTransforms", ":X86VectorDialect", ":X86VectorTransforms", + ":XeGPUDialect", ":config", ], ) -- GitLab From 0f8cb6818d788c34d70d586f701469a5adabfce1 Mon Sep 17 00:00:00 2001 From: David Spickett Date: Fri, 8 Mar 2024 10:42:31 +0000 Subject: [PATCH 587/929] [lldb][Docs] Update the build guide * gmake is needed on FreeBSD * pkg can install libxml2 on FreeBSD * Make the swig note into an RST note box. --- lldb/docs/resources/build.rst | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lldb/docs/resources/build.rst b/lldb/docs/resources/build.rst index 55fe73c52f61..fe8e293db642 100644 --- a/lldb/docs/resources/build.rst +++ b/lldb/docs/resources/build.rst @@ -33,6 +33,9 @@ scripting support. * `Python `_ * `SWIG `_ 4 or later. +If you are on FreeBSD or NetBSD, you will need to install ``gmake`` for building +the test programs. On other platforms ``make`` is used. + .. _Optional Dependencies: Optional Dependencies @@ -71,16 +74,16 @@ commands below. $ yum install libedit-devel libxml2-devel ncurses-devel python-devel swig $ sudo apt-get install build-essential swig python3-dev libedit-dev libncurses5-dev - $ pkg install swig python + $ pkg install swig python libxml2 $ pkgin install swig python36 cmake ninja-build $ brew install swig cmake ninja -Note that there's an `incompatibility -`_ between Python version 3.7 and later -and swig versions older than 4.0.0 which makes builds of LLDB using debug -versions of python unusable. This primarily affects Windows, as debug builds of -LLDB must use debug python as well. - +.. note:: + There is an `incompatibility + `_ between Python version 3.7 and later + and swig versions older than 4.0.0 which makes builds of LLDB using debug + versions of python unusable. This primarily affects Windows, as debug builds of + LLDB must use debug python as well. Windows ******* -- GitLab From 50633868538d4fd6875a7de57804b3dde5c86108 Mon Sep 17 00:00:00 2001 From: Christian Sigg Date: Fri, 8 Mar 2024 11:47:38 +0100 Subject: [PATCH 588/929] [clang][bazel] Fix BUILD after da00c60dae0040185dc45039c4397f6e746548e9 --- utils/bazel/llvm-project-overlay/clang/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel index 6d3904f84673..339706077b52 100644 --- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel @@ -2092,6 +2092,7 @@ cc_library( deps = [ ":ast", ":basic", + ":frontend", ":lex", ":sema", ":serialization_attr_gen", -- GitLab From 005e182ecbf0099005f7d3c33c2b7556abb515e4 Mon Sep 17 00:00:00 2001 From: Christian Sigg Date: Fri, 8 Mar 2024 11:55:19 +0100 Subject: [PATCH 589/929] [clang][bazel] Fix BUILD after da00c60dae0040185dc45039c4397f6e746548e9 Second attempt. This also reverts 50633868538d4fd6875a7de57804b3dde5c86108. --- utils/bazel/llvm-project-overlay/clang/BUILD.bazel | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel index 339706077b52..d6b124f9d8e4 100644 --- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel @@ -2079,6 +2079,7 @@ cc_library( "include/clang/Serialization/AttrPCHRead.inc", "include/clang/Serialization/AttrPCHWrite.inc", ] + glob([ + "include/clang/Frontend/*.h", "lib/Serialization/*.cpp", "lib/Serialization/*.h", ]), @@ -2090,12 +2091,14 @@ cc_library( "include/clang/Serialization/*.def", ]), deps = [ + ":apinotes", ":ast", ":basic", - ":frontend", + ":driver", ":lex", ":sema", ":serialization_attr_gen", + ":static_analyzer_core_options", ":type_nodes_gen", "//llvm:BitReader", "//llvm:BitWriter", -- GitLab From ee94bd20ba09975de493675dd6a0b7fc7dd5cece Mon Sep 17 00:00:00 2001 From: Krystian Stasiowski Date: Fri, 8 Mar 2024 05:57:04 -0500 Subject: [PATCH 590/929] Revert "[Clang][Sema] Fix crash when using name of UnresolvedUsingValueDecl with template arguments (#83842)" (#84457) This reverts commit a642eb89bdaf10c6b4994fc1187de27b441236ed (see #83842) --- clang/docs/ReleaseNotes.rst | 2 -- clang/lib/Sema/SemaDecl.cpp | 5 +--- clang/lib/Sema/SemaTemplate.cpp | 10 +++---- .../unqual-unresolved-using-value.cpp | 30 ------------------- 4 files changed, 5 insertions(+), 42 deletions(-) delete mode 100644 clang/test/SemaTemplate/unqual-unresolved-using-value.cpp diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 225ca85e1811..0ee2801766a9 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -332,8 +332,6 @@ Bug Fixes to C++ Support our attention by an attempt to fix in (#GH77703). Fixes (#GH83385). - Fix evaluation of some immediate calls in default arguments. Fixes (#GH80630) -- Fix a crash when an explicit template argument list is used with a name for which lookup - finds a non-template function and a dependent using declarator. - Fixed an issue where the ``RequiresExprBody`` was involved in the lambda dependency calculation. (#GH56556), (#GH82849). - Fix a bug where overload resolution falsely reported an ambiguity when it was comparing diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 910d0dfbf9f6..1f4a041e88df 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -1110,9 +1110,7 @@ Corrected: // unqualified-id followed by a < and name lookup finds either one // or more functions or finds nothing. if (!IsFilteredTemplateName) - FilterAcceptableTemplateNames(Result, - /*AllowFunctionTemplates=*/true, - /*AllowDependent=*/true); + FilterAcceptableTemplateNames(Result); bool IsFunctionTemplate; bool IsVarTemplate; @@ -1122,7 +1120,6 @@ Corrected: Template = Context.getOverloadedTemplateName(Result.begin(), Result.end()); } else if (!Result.empty()) { - assert(!Result.isUnresolvableResult()); auto *TD = cast(getAsTemplateNameDecl( *Result.begin(), /*AllowFunctionTemplates=*/true, /*AllowDependent=*/false)); diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp index 7e91815c2d52..83eee83aa6ca 100644 --- a/clang/lib/Sema/SemaTemplate.cpp +++ b/clang/lib/Sema/SemaTemplate.cpp @@ -491,20 +491,18 @@ bool Sema::LookupTemplateName(LookupResult &Found, // To keep our behavior consistent, we apply the "finds nothing" part in // all language modes, and diagnose the empty lookup in ActOnCallExpr if we // successfully form a call to an undeclared template-id. - bool AnyFunctions = - getLangOpts().CPlusPlus20 && llvm::any_of(Found, [](NamedDecl *ND) { + bool AllFunctions = + getLangOpts().CPlusPlus20 && llvm::all_of(Found, [](NamedDecl *ND) { return isa(ND->getUnderlyingDecl()); }); - if (AnyFunctions || (Found.empty() && !IsDependent)) { + if (AllFunctions || (Found.empty() && !IsDependent)) { // If lookup found any functions, or if this is a name that can only be // used for a function, then strongly assume this is a function // template-id. *ATK = (Found.empty() && Found.getLookupName().isIdentifier()) ? AssumedTemplateKind::FoundNothing : AssumedTemplateKind::FoundFunctions; - FilterAcceptableTemplateNames(Found, - /*AllowFunctionTemplates*/ true, - /*AllowDependent*/ true); + Found.clear(); return false; } } diff --git a/clang/test/SemaTemplate/unqual-unresolved-using-value.cpp b/clang/test/SemaTemplate/unqual-unresolved-using-value.cpp deleted file mode 100644 index 688e7a0a10b7..000000000000 --- a/clang/test/SemaTemplate/unqual-unresolved-using-value.cpp +++ /dev/null @@ -1,30 +0,0 @@ -// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify %s - -template -struct A : T { - using T::f; - using T::g; - using T::h; - - void f(); - void g(); - - void i() { - f(); - g(); // expected-error{{no member named 'g' in 'A'}} - h(); // expected-error{{expected '(' for function-style cast or type construction}} - // expected-error@-1{{expected expression}} - } -}; - -struct B { - template - void f(); - - void g(); - - template - void h(); -}; - -template struct A; // expected-note{{in instantiation of member function 'A::i' requested here}} -- GitLab From 6a0618a0289cb0c23ef3e5c820418650cc1d0fdc Mon Sep 17 00:00:00 2001 From: Amirreza Ashouri Date: Fri, 8 Mar 2024 14:27:35 +0330 Subject: [PATCH 591/929] [clang][NFC] Refactor `clang/test/SemaCXX/type-traits.cpp` to use modern `static_assert` (#77584) static_assert is supported in all modes, so the T/F macros aren't needed anymore. --- clang/test/SemaCXX/type-traits.cpp | 4745 ++++++++++++++-------------- 1 file changed, 2364 insertions(+), 2381 deletions(-) diff --git a/clang/test/SemaCXX/type-traits.cpp b/clang/test/SemaCXX/type-traits.cpp index 831de2589dcb..f50f51cc7098 100644 --- a/clang/test/SemaCXX/type-traits.cpp +++ b/clang/test/SemaCXX/type-traits.cpp @@ -1,10 +1,8 @@ -// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -verify -std=gnu++11 -fblocks -Wno-deprecated-builtins -Wno-defaulted-function-deleted %s -// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -verify -std=gnu++14 -fblocks -Wno-deprecated-builtins -Wno-defaulted-function-deleted %s -// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -verify -std=gnu++17 -fblocks -Wno-deprecated-builtins -Wno-defaulted-function-deleted %s -// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -verify -std=gnu++20 -fblocks -Wno-deprecated-builtins -Wno-defaulted-function-deleted %s +// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -verify -std=gnu++11 -fblocks -Wno-deprecated-builtins -Wno-defaulted-function-deleted -Wno-c++17-extensions %s +// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -verify -std=gnu++14 -fblocks -Wno-deprecated-builtins -Wno-defaulted-function-deleted -Wno-c++17-extensions %s +// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -verify -std=gnu++17 -fblocks -Wno-deprecated-builtins -Wno-defaulted-function-deleted %s +// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -verify -std=gnu++20 -fblocks -Wno-deprecated-builtins -Wno-defaulted-function-deleted %s -#define T(b) (b) ? 1 : -1 -#define F(b) (b) ? -1 : 1 struct NonPOD { NonPOD(int); }; typedef NonPOD NonPODAr[10]; @@ -214,56 +212,56 @@ struct HasVirtBase : virtual ACompleteType {}; void is_pod() { - { int arr[T(__is_pod(int))]; } - { int arr[T(__is_pod(Enum))]; } - { int arr[T(__is_pod(POD))]; } - { int arr[T(__is_pod(Int))]; } - { int arr[T(__is_pod(IntAr))]; } - { int arr[T(__is_pod(Statics))]; } - { int arr[T(__is_pod(Empty))]; } - { int arr[T(__is_pod(EmptyUnion))]; } - { int arr[T(__is_pod(Union))]; } - { int arr[T(__is_pod(HasFunc))]; } - { int arr[T(__is_pod(HasOp))]; } - { int arr[T(__is_pod(HasConv))]; } - { int arr[T(__is_pod(HasAssign))]; } - { int arr[T(__is_pod(IntArNB))]; } - { int arr[T(__is_pod(HasAnonymousUnion))]; } - { int arr[T(__is_pod(Vector))]; } - { int arr[T(__is_pod(VectorExt))]; } - { int arr[T(__is_pod(Derives))]; } - { int arr[T(__is_pod(DerivesAr))]; } - { int arr[T(__is_pod(DerivesArNB))]; } - { int arr[T(__is_pod(DerivesEmpty))]; } - { int arr[T(__is_pod(HasPriv))]; } - { int arr[T(__is_pod(HasProt))]; } - { int arr[T(__is_pod(DerivesHasPriv))]; } - { int arr[T(__is_pod(DerivesHasProt))]; } - - { int arr[F(__is_pod(HasCons))]; } - { int arr[F(__is_pod(HasCopyAssign))]; } - { int arr[F(__is_pod(HasMoveAssign))]; } - { int arr[F(__is_pod(HasDest))]; } - { int arr[F(__is_pod(HasRef))]; } - { int arr[F(__is_pod(HasVirt))]; } - { int arr[F(__is_pod(DerivesHasCons))]; } - { int arr[F(__is_pod(DerivesHasCopyAssign))]; } - { int arr[F(__is_pod(DerivesHasMoveAssign))]; } - { int arr[F(__is_pod(DerivesHasDest))]; } - { int arr[F(__is_pod(DerivesHasRef))]; } - { int arr[F(__is_pod(DerivesHasVirt))]; } - { int arr[F(__is_pod(NonPOD))]; } - { int arr[F(__is_pod(HasNonPOD))]; } - { int arr[F(__is_pod(NonPODAr))]; } - { int arr[F(__is_pod(NonPODArNB))]; } - { int arr[F(__is_pod(void))]; } - { int arr[F(__is_pod(cvoid))]; } -// { int arr[F(__is_pod(NonPODUnion))]; } - - { int arr[T(__is_pod(ACompleteType))]; } - { int arr[F(__is_pod(AnIncompleteType))]; } // expected-error {{incomplete type}} - { int arr[F(__is_pod(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[F(__is_pod(AnIncompleteType[1]))]; } // expected-error {{incomplete type}} + static_assert(__is_pod(int)); + static_assert(__is_pod(Enum)); + static_assert(__is_pod(POD)); + static_assert(__is_pod(Int)); + static_assert(__is_pod(IntAr)); + static_assert(__is_pod(Statics)); + static_assert(__is_pod(Empty)); + static_assert(__is_pod(EmptyUnion)); + static_assert(__is_pod(Union)); + static_assert(__is_pod(HasFunc)); + static_assert(__is_pod(HasOp)); + static_assert(__is_pod(HasConv)); + static_assert(__is_pod(HasAssign)); + static_assert(__is_pod(IntArNB)); + static_assert(__is_pod(HasAnonymousUnion)); + static_assert(__is_pod(Vector)); + static_assert(__is_pod(VectorExt)); + static_assert(__is_pod(Derives)); + static_assert(__is_pod(DerivesAr)); + static_assert(__is_pod(DerivesArNB)); + static_assert(__is_pod(DerivesEmpty)); + static_assert(__is_pod(HasPriv)); + static_assert(__is_pod(HasProt)); + static_assert(__is_pod(DerivesHasPriv)); + static_assert(__is_pod(DerivesHasProt)); + + static_assert(!__is_pod(HasCons)); + static_assert(!__is_pod(HasCopyAssign)); + static_assert(!__is_pod(HasMoveAssign)); + static_assert(!__is_pod(HasDest)); + static_assert(!__is_pod(HasRef)); + static_assert(!__is_pod(HasVirt)); + static_assert(!__is_pod(DerivesHasCons)); + static_assert(!__is_pod(DerivesHasCopyAssign)); + static_assert(!__is_pod(DerivesHasMoveAssign)); + static_assert(!__is_pod(DerivesHasDest)); + static_assert(!__is_pod(DerivesHasRef)); + static_assert(!__is_pod(DerivesHasVirt)); + static_assert(!__is_pod(NonPOD)); + static_assert(!__is_pod(HasNonPOD)); + static_assert(!__is_pod(NonPODAr)); + static_assert(!__is_pod(NonPODArNB)); + static_assert(!__is_pod(void)); + static_assert(!__is_pod(cvoid)); +// static_assert(!__is_pod(NonPODUnion)); + + static_assert(__is_pod(ACompleteType)); + static_assert(!__is_pod(AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_pod(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__is_pod(AnIncompleteType[1])); // expected-error {{incomplete type}} } typedef Empty EmptyAr[10]; @@ -275,56 +273,56 @@ struct DerivesVirt : virtual POD {}; void is_empty() { - { int arr[T(__is_empty(Empty))]; } - { int arr[T(__is_empty(DerivesEmpty))]; } - { int arr[T(__is_empty(HasCons))]; } - { int arr[T(__is_empty(HasCopyAssign))]; } - { int arr[T(__is_empty(HasMoveAssign))]; } - { int arr[T(__is_empty(HasDest))]; } - { int arr[T(__is_empty(HasFunc))]; } - { int arr[T(__is_empty(HasOp))]; } - { int arr[T(__is_empty(HasConv))]; } - { int arr[T(__is_empty(HasAssign))]; } - { int arr[T(__is_empty(Bit0))]; } - { int arr[T(__is_empty(Bit0Cons))]; } - - { int arr[F(__is_empty(Int))]; } - { int arr[F(__is_empty(POD))]; } - { int arr[F(__is_empty(EmptyUnion))]; } - { int arr[F(__is_empty(IncompleteUnion))]; } - { int arr[F(__is_empty(EmptyAr))]; } - { int arr[F(__is_empty(HasRef))]; } - { int arr[F(__is_empty(HasVirt))]; } - { int arr[F(__is_empty(AnonBitOnly))]; } - { int arr[F(__is_empty(BitOnly))]; } - { int arr[F(__is_empty(void))]; } - { int arr[F(__is_empty(IntArNB))]; } - { int arr[F(__is_empty(HasAnonymousUnion))]; } -// { int arr[F(__is_empty(DerivesVirt))]; } - - { int arr[T(__is_empty(ACompleteType))]; } - { int arr[F(__is_empty(AnIncompleteType))]; } // expected-error {{incomplete type}} - { int arr[F(__is_empty(AnIncompleteType[]))]; } - { int arr[F(__is_empty(AnIncompleteType[1]))]; } + static_assert(__is_empty(Empty)); + static_assert(__is_empty(DerivesEmpty)); + static_assert(__is_empty(HasCons)); + static_assert(__is_empty(HasCopyAssign)); + static_assert(__is_empty(HasMoveAssign)); + static_assert(__is_empty(HasDest)); + static_assert(__is_empty(HasFunc)); + static_assert(__is_empty(HasOp)); + static_assert(__is_empty(HasConv)); + static_assert(__is_empty(HasAssign)); + static_assert(__is_empty(Bit0)); + static_assert(__is_empty(Bit0Cons)); + + static_assert(!__is_empty(Int)); + static_assert(!__is_empty(POD)); + static_assert(!__is_empty(EmptyUnion)); + static_assert(!__is_empty(IncompleteUnion)); + static_assert(!__is_empty(EmptyAr)); + static_assert(!__is_empty(HasRef)); + static_assert(!__is_empty(HasVirt)); + static_assert(!__is_empty(AnonBitOnly)); + static_assert(!__is_empty(BitOnly)); + static_assert(!__is_empty(void)); + static_assert(!__is_empty(IntArNB)); + static_assert(!__is_empty(HasAnonymousUnion)); +// static_assert(!__is_empty(DerivesVirt)); + + static_assert(__is_empty(ACompleteType)); + static_assert(!__is_empty(AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_empty(AnIncompleteType[])); + static_assert(!__is_empty(AnIncompleteType[1])); } typedef Derives ClassType; void is_class() { - { int arr[T(__is_class(Derives))]; } - { int arr[T(__is_class(HasPriv))]; } - { int arr[T(__is_class(ClassType))]; } - { int arr[T(__is_class(HasAnonymousUnion))]; } - - { int arr[F(__is_class(int))]; } - { int arr[F(__is_class(Enum))]; } - { int arr[F(__is_class(Int))]; } - { int arr[F(__is_class(IntAr))]; } - { int arr[F(__is_class(DerivesAr))]; } - { int arr[F(__is_class(Union))]; } - { int arr[F(__is_class(cvoid))]; } - { int arr[F(__is_class(IntArNB))]; } + static_assert(__is_class(Derives)); + static_assert(__is_class(HasPriv)); + static_assert(__is_class(ClassType)); + static_assert(__is_class(HasAnonymousUnion)); + + static_assert(!__is_class(int)); + static_assert(!__is_class(Enum)); + static_assert(!__is_class(Int)); + static_assert(!__is_class(IntAr)); + static_assert(!__is_class(DerivesAr)); + static_assert(!__is_class(Union)); + static_assert(!__is_class(cvoid)); + static_assert(!__is_class(IntArNB)); } typedef Union UnionAr[10]; @@ -332,17 +330,17 @@ typedef Union UnionType; void is_union() { - { int arr[T(__is_union(Union))]; } - { int arr[T(__is_union(UnionType))]; } - - { int arr[F(__is_union(int))]; } - { int arr[F(__is_union(Enum))]; } - { int arr[F(__is_union(Int))]; } - { int arr[F(__is_union(IntAr))]; } - { int arr[F(__is_union(UnionAr))]; } - { int arr[F(__is_union(cvoid))]; } - { int arr[F(__is_union(IntArNB))]; } - { int arr[F(__is_union(HasAnonymousUnion))]; } + static_assert(__is_union(Union)); + static_assert(__is_union(UnionType)); + + static_assert(!__is_union(int)); + static_assert(!__is_union(Enum)); + static_assert(!__is_union(Int)); + static_assert(!__is_union(IntAr)); + static_assert(!__is_union(UnionAr)); + static_assert(!__is_union(cvoid)); + static_assert(!__is_union(IntArNB)); + static_assert(!__is_union(HasAnonymousUnion)); } typedef Enum EnumType; @@ -350,57 +348,57 @@ typedef EnumClass EnumClassType; void is_enum() { - { int arr[T(__is_enum(Enum))]; } - { int arr[T(__is_enum(EnumType))]; } - { int arr[T(__is_enum(SignedEnum))]; } - { int arr[T(__is_enum(UnsignedEnum))]; } - - { int arr[T(__is_enum(EnumClass))]; } - { int arr[T(__is_enum(EnumClassType))]; } - { int arr[T(__is_enum(SignedEnumClass))]; } - { int arr[T(__is_enum(UnsignedEnumClass))]; } - - { int arr[F(__is_enum(int))]; } - { int arr[F(__is_enum(Union))]; } - { int arr[F(__is_enum(Int))]; } - { int arr[F(__is_enum(IntAr))]; } - { int arr[F(__is_enum(UnionAr))]; } - { int arr[F(__is_enum(Derives))]; } - { int arr[F(__is_enum(ClassType))]; } - { int arr[F(__is_enum(cvoid))]; } - { int arr[F(__is_enum(IntArNB))]; } - { int arr[F(__is_enum(HasAnonymousUnion))]; } - { int arr[F(__is_enum(AnIncompleteType))]; } - { int arr[F(__is_enum(AnIncompleteTypeAr))]; } - { int arr[F(__is_enum(AnIncompleteTypeArMB))]; } - { int arr[F(__is_enum(AnIncompleteTypeArNB))]; } + static_assert(__is_enum(Enum)); + static_assert(__is_enum(EnumType)); + static_assert(__is_enum(SignedEnum)); + static_assert(__is_enum(UnsignedEnum)); + + static_assert(__is_enum(EnumClass)); + static_assert(__is_enum(EnumClassType)); + static_assert(__is_enum(SignedEnumClass)); + static_assert(__is_enum(UnsignedEnumClass)); + + static_assert(!__is_enum(int)); + static_assert(!__is_enum(Union)); + static_assert(!__is_enum(Int)); + static_assert(!__is_enum(IntAr)); + static_assert(!__is_enum(UnionAr)); + static_assert(!__is_enum(Derives)); + static_assert(!__is_enum(ClassType)); + static_assert(!__is_enum(cvoid)); + static_assert(!__is_enum(IntArNB)); + static_assert(!__is_enum(HasAnonymousUnion)); + static_assert(!__is_enum(AnIncompleteType)); + static_assert(!__is_enum(AnIncompleteTypeAr)); + static_assert(!__is_enum(AnIncompleteTypeArMB)); + static_assert(!__is_enum(AnIncompleteTypeArNB)); } void is_scoped_enum() { - static_assert(!__is_scoped_enum(Enum), ""); - static_assert(!__is_scoped_enum(EnumType), ""); - static_assert(!__is_scoped_enum(SignedEnum), ""); - static_assert(!__is_scoped_enum(UnsignedEnum), ""); - - static_assert(__is_scoped_enum(EnumClass), ""); - static_assert(__is_scoped_enum(EnumClassType), ""); - static_assert(__is_scoped_enum(SignedEnumClass), ""); - static_assert(__is_scoped_enum(UnsignedEnumClass), ""); - - static_assert(!__is_scoped_enum(int), ""); - static_assert(!__is_scoped_enum(Union), ""); - static_assert(!__is_scoped_enum(Int), ""); - static_assert(!__is_scoped_enum(IntAr), ""); - static_assert(!__is_scoped_enum(UnionAr), ""); - static_assert(!__is_scoped_enum(Derives), ""); - static_assert(!__is_scoped_enum(ClassType), ""); - static_assert(!__is_scoped_enum(cvoid), ""); - static_assert(!__is_scoped_enum(IntArNB), ""); - static_assert(!__is_scoped_enum(HasAnonymousUnion), ""); - static_assert(!__is_scoped_enum(AnIncompleteType), ""); - static_assert(!__is_scoped_enum(AnIncompleteTypeAr), ""); - static_assert(!__is_scoped_enum(AnIncompleteTypeArMB), ""); - static_assert(!__is_scoped_enum(AnIncompleteTypeArNB), ""); + static_assert(!__is_scoped_enum(Enum)); + static_assert(!__is_scoped_enum(EnumType)); + static_assert(!__is_scoped_enum(SignedEnum)); + static_assert(!__is_scoped_enum(UnsignedEnum)); + + static_assert(__is_scoped_enum(EnumClass)); + static_assert(__is_scoped_enum(EnumClassType)); + static_assert(__is_scoped_enum(SignedEnumClass)); + static_assert(__is_scoped_enum(UnsignedEnumClass)); + + static_assert(!__is_scoped_enum(int)); + static_assert(!__is_scoped_enum(Union)); + static_assert(!__is_scoped_enum(Int)); + static_assert(!__is_scoped_enum(IntAr)); + static_assert(!__is_scoped_enum(UnionAr)); + static_assert(!__is_scoped_enum(Derives)); + static_assert(!__is_scoped_enum(ClassType)); + static_assert(!__is_scoped_enum(cvoid)); + static_assert(!__is_scoped_enum(IntArNB)); + static_assert(!__is_scoped_enum(HasAnonymousUnion)); + static_assert(!__is_scoped_enum(AnIncompleteType)); + static_assert(!__is_scoped_enum(AnIncompleteTypeAr)); + static_assert(!__is_scoped_enum(AnIncompleteTypeArMB)); + static_assert(!__is_scoped_enum(AnIncompleteTypeArNB)); } struct FinalClass final { @@ -421,20 +419,20 @@ struct PotentiallyFinal final { }; void is_final() { - { int arr[T(__is_final(FinalClass))]; } - { int arr[T(__is_final(PotentiallyFinal))]; } - { int arr[T(__is_final(PotentiallyFinal))]; } - - { int arr[F(__is_final(int))]; } - { int arr[F(__is_final(Union))]; } - { int arr[F(__is_final(Int))]; } - { int arr[F(__is_final(IntAr))]; } - { int arr[F(__is_final(UnionAr))]; } - { int arr[F(__is_final(Derives))]; } - { int arr[F(__is_final(ClassType))]; } - { int arr[F(__is_final(cvoid))]; } - { int arr[F(__is_final(IntArNB))]; } - { int arr[F(__is_final(HasAnonymousUnion))]; } + static_assert(__is_final(FinalClass)); + static_assert(__is_final(PotentiallyFinal)); + static_assert(__is_final(PotentiallyFinal)); + + static_assert(!__is_final(int)); + static_assert(!__is_final(Union)); + static_assert(!__is_final(Int)); + static_assert(!__is_final(IntAr)); + static_assert(!__is_final(UnionAr)); + static_assert(!__is_final(Derives)); + static_assert(!__is_final(ClassType)); + static_assert(!__is_final(cvoid)); + static_assert(!__is_final(IntArNB)); + static_assert(!__is_final(HasAnonymousUnion)); } @@ -443,78 +441,78 @@ struct InheritPolymorph : Polymorph {}; void is_polymorphic() { - { int arr[T(__is_polymorphic(Polymorph))]; } - { int arr[T(__is_polymorphic(InheritPolymorph))]; } - - { int arr[F(__is_polymorphic(int))]; } - { int arr[F(__is_polymorphic(Union))]; } - { int arr[F(__is_polymorphic(IncompleteUnion))]; } - { int arr[F(__is_polymorphic(Int))]; } - { int arr[F(__is_polymorphic(IntAr))]; } - { int arr[F(__is_polymorphic(UnionAr))]; } - { int arr[F(__is_polymorphic(Derives))]; } - { int arr[F(__is_polymorphic(ClassType))]; } - { int arr[F(__is_polymorphic(Enum))]; } - { int arr[F(__is_polymorphic(cvoid))]; } - { int arr[F(__is_polymorphic(IntArNB))]; } + static_assert(__is_polymorphic(Polymorph)); + static_assert(__is_polymorphic(InheritPolymorph)); + + static_assert(!__is_polymorphic(int)); + static_assert(!__is_polymorphic(Union)); + static_assert(!__is_polymorphic(IncompleteUnion)); + static_assert(!__is_polymorphic(Int)); + static_assert(!__is_polymorphic(IntAr)); + static_assert(!__is_polymorphic(UnionAr)); + static_assert(!__is_polymorphic(Derives)); + static_assert(!__is_polymorphic(ClassType)); + static_assert(!__is_polymorphic(Enum)); + static_assert(!__is_polymorphic(cvoid)); + static_assert(!__is_polymorphic(IntArNB)); } void is_integral() { - int t01[T(__is_integral(bool))]; - int t02[T(__is_integral(char))]; - int t03[T(__is_integral(signed char))]; - int t04[T(__is_integral(unsigned char))]; - //int t05[T(__is_integral(char16_t))]; - //int t06[T(__is_integral(char32_t))]; - int t07[T(__is_integral(wchar_t))]; - int t08[T(__is_integral(short))]; - int t09[T(__is_integral(unsigned short))]; - int t10[T(__is_integral(int))]; - int t11[T(__is_integral(unsigned int))]; - int t12[T(__is_integral(long))]; - int t13[T(__is_integral(unsigned long))]; - - int t21[F(__is_integral(float))]; - int t22[F(__is_integral(double))]; - int t23[F(__is_integral(long double))]; - int t24[F(__is_integral(Union))]; - int t25[F(__is_integral(UnionAr))]; - int t26[F(__is_integral(Derives))]; - int t27[F(__is_integral(ClassType))]; - int t28[F(__is_integral(Enum))]; - int t29[F(__is_integral(void))]; - int t30[F(__is_integral(cvoid))]; - int t31[F(__is_integral(IntArNB))]; + static_assert(__is_integral(bool)); + static_assert(__is_integral(char)); + static_assert(__is_integral(signed char)); + static_assert(__is_integral(unsigned char)); + //static_assert(__is_integral(char16_t)); + //static_assert(__is_integral(char32_t)); + static_assert(__is_integral(wchar_t)); + static_assert(__is_integral(short)); + static_assert(__is_integral(unsigned short)); + static_assert(__is_integral(int)); + static_assert(__is_integral(unsigned int)); + static_assert(__is_integral(long)); + static_assert(__is_integral(unsigned long)); + + static_assert(!__is_integral(float)); + static_assert(!__is_integral(double)); + static_assert(!__is_integral(long double)); + static_assert(!__is_integral(Union)); + static_assert(!__is_integral(UnionAr)); + static_assert(!__is_integral(Derives)); + static_assert(!__is_integral(ClassType)); + static_assert(!__is_integral(Enum)); + static_assert(!__is_integral(void)); + static_assert(!__is_integral(cvoid)); + static_assert(!__is_integral(IntArNB)); } void is_floating_point() { - int t01[T(__is_floating_point(float))]; - int t02[T(__is_floating_point(double))]; - int t03[T(__is_floating_point(long double))]; - - int t11[F(__is_floating_point(bool))]; - int t12[F(__is_floating_point(char))]; - int t13[F(__is_floating_point(signed char))]; - int t14[F(__is_floating_point(unsigned char))]; - //int t15[F(__is_floating_point(char16_t))]; - //int t16[F(__is_floating_point(char32_t))]; - int t17[F(__is_floating_point(wchar_t))]; - int t18[F(__is_floating_point(short))]; - int t19[F(__is_floating_point(unsigned short))]; - int t20[F(__is_floating_point(int))]; - int t21[F(__is_floating_point(unsigned int))]; - int t22[F(__is_floating_point(long))]; - int t23[F(__is_floating_point(unsigned long))]; - int t24[F(__is_floating_point(Union))]; - int t25[F(__is_floating_point(UnionAr))]; - int t26[F(__is_floating_point(Derives))]; - int t27[F(__is_floating_point(ClassType))]; - int t28[F(__is_floating_point(Enum))]; - int t29[F(__is_floating_point(void))]; - int t30[F(__is_floating_point(cvoid))]; - int t31[F(__is_floating_point(IntArNB))]; + static_assert(__is_floating_point(float)); + static_assert(__is_floating_point(double)); + static_assert(__is_floating_point(long double)); + + static_assert(!__is_floating_point(bool)); + static_assert(!__is_floating_point(char)); + static_assert(!__is_floating_point(signed char)); + static_assert(!__is_floating_point(unsigned char)); + //static_assert(!__is_floating_point(char16_t)); + //static_assert(!__is_floating_point(char32_t)); + static_assert(!__is_floating_point(wchar_t)); + static_assert(!__is_floating_point(short)); + static_assert(!__is_floating_point(unsigned short)); + static_assert(!__is_floating_point(int)); + static_assert(!__is_floating_point(unsigned int)); + static_assert(!__is_floating_point(long)); + static_assert(!__is_floating_point(unsigned long)); + static_assert(!__is_floating_point(Union)); + static_assert(!__is_floating_point(UnionAr)); + static_assert(!__is_floating_point(Derives)); + static_assert(!__is_floating_point(ClassType)); + static_assert(!__is_floating_point(Enum)); + static_assert(!__is_floating_point(void)); + static_assert(!__is_floating_point(cvoid)); + static_assert(!__is_floating_point(IntArNB)); } template @@ -537,274 +535,274 @@ void is_aggregate() __is_aggregate(IncompleteUnion); // expected-error {{incomplete type}} // Valid since LWG3823 - static_assert(__is_aggregate(AnIncompleteType[]), ""); - static_assert(__is_aggregate(AnIncompleteType[1]), ""); - static_assert(__is_aggregate(AnIncompleteTypeAr), ""); - static_assert(__is_aggregate(AnIncompleteTypeArNB), ""); - static_assert(__is_aggregate(AnIncompleteTypeArMB), ""); - - static_assert(!__is_aggregate(NonPOD), ""); - static_assert(__is_aggregate(NonPODAr), ""); - static_assert(__is_aggregate(NonPODArNB), ""); - static_assert(__is_aggregate(NonPODArMB), ""); - - static_assert(!__is_aggregate(Enum), ""); - static_assert(__is_aggregate(POD), ""); - static_assert(__is_aggregate(Empty), ""); - static_assert(__is_aggregate(EmptyAr), ""); - static_assert(__is_aggregate(EmptyArNB), ""); - static_assert(__is_aggregate(EmptyArMB), ""); - static_assert(!__is_aggregate(void), ""); - static_assert(!__is_aggregate(const volatile void), ""); - static_assert(!__is_aggregate(int), ""); - static_assert(__is_aggregate(IntAr), ""); - static_assert(__is_aggregate(IntArNB), ""); - static_assert(__is_aggregate(EmptyUnion), ""); - static_assert(__is_aggregate(Union), ""); - static_assert(__is_aggregate(Statics), ""); - static_assert(__is_aggregate(HasFunc), ""); - static_assert(__is_aggregate(HasOp), ""); - static_assert(__is_aggregate(HasAssign), ""); - static_assert(__is_aggregate(HasAnonymousUnion), ""); - - static_assert(__is_aggregate(Derives) == TrueAfterCpp14, ""); - static_assert(__is_aggregate(DerivesAr), ""); - static_assert(__is_aggregate(DerivesArNB), ""); - static_assert(!__is_aggregate(HasCons), ""); + static_assert(__is_aggregate(AnIncompleteType[])); + static_assert(__is_aggregate(AnIncompleteType[1])); + static_assert(__is_aggregate(AnIncompleteTypeAr)); + static_assert(__is_aggregate(AnIncompleteTypeArNB)); + static_assert(__is_aggregate(AnIncompleteTypeArMB)); + + static_assert(!__is_aggregate(NonPOD)); + static_assert(__is_aggregate(NonPODAr)); + static_assert(__is_aggregate(NonPODArNB)); + static_assert(__is_aggregate(NonPODArMB)); + + static_assert(!__is_aggregate(Enum)); + static_assert(__is_aggregate(POD)); + static_assert(__is_aggregate(Empty)); + static_assert(__is_aggregate(EmptyAr)); + static_assert(__is_aggregate(EmptyArNB)); + static_assert(__is_aggregate(EmptyArMB)); + static_assert(!__is_aggregate(void)); + static_assert(!__is_aggregate(const volatile void)); + static_assert(!__is_aggregate(int)); + static_assert(__is_aggregate(IntAr)); + static_assert(__is_aggregate(IntArNB)); + static_assert(__is_aggregate(EmptyUnion)); + static_assert(__is_aggregate(Union)); + static_assert(__is_aggregate(Statics)); + static_assert(__is_aggregate(HasFunc)); + static_assert(__is_aggregate(HasOp)); + static_assert(__is_aggregate(HasAssign)); + static_assert(__is_aggregate(HasAnonymousUnion)); + + static_assert(__is_aggregate(Derives) == TrueAfterCpp14); + static_assert(__is_aggregate(DerivesAr)); + static_assert(__is_aggregate(DerivesArNB)); + static_assert(!__is_aggregate(HasCons)); #if __cplusplus >= 202002L - static_assert(!__is_aggregate(HasDefaultCons), ""); + static_assert(!__is_aggregate(HasDefaultCons)); #else - static_assert(__is_aggregate(HasDefaultCons), ""); + static_assert(__is_aggregate(HasDefaultCons)); #endif - static_assert(!__is_aggregate(HasExplicitDefaultCons), ""); - static_assert(!__is_aggregate(HasInheritedCons), ""); - static_assert(__is_aggregate(HasNoInheritedCons) == TrueAfterCpp14, ""); - static_assert(__is_aggregate(HasCopyAssign), ""); - static_assert(!__is_aggregate(NonTrivialDefault), ""); - static_assert(__is_aggregate(HasDest), ""); - static_assert(!__is_aggregate(HasPriv), ""); - static_assert(!__is_aggregate(HasProt), ""); - static_assert(__is_aggregate(HasRefAggregate), ""); - static_assert(__is_aggregate(HasNonPOD), ""); - static_assert(!__is_aggregate(HasVirt), ""); - static_assert(__is_aggregate(VirtAr), ""); - static_assert(__is_aggregate(HasInClassInit) == TrueAfterCpp11, ""); - static_assert(!__is_aggregate(HasPrivateBase), ""); - static_assert(!__is_aggregate(HasProtectedBase), ""); - static_assert(!__is_aggregate(HasVirtBase), ""); - - static_assert(__is_aggregate(AggregateTemplate), ""); - static_assert(!__is_aggregate(NonAggregateTemplate), ""); - - static_assert(__is_aggregate(Vector), ""); // Extension supported by GCC and Clang - static_assert(__is_aggregate(VectorExt), ""); - static_assert(__is_aggregate(ComplexInt), ""); - static_assert(__is_aggregate(ComplexFloat), ""); + static_assert(!__is_aggregate(HasExplicitDefaultCons)); + static_assert(!__is_aggregate(HasInheritedCons)); + static_assert(__is_aggregate(HasNoInheritedCons) == TrueAfterCpp14); + static_assert(__is_aggregate(HasCopyAssign)); + static_assert(!__is_aggregate(NonTrivialDefault)); + static_assert(__is_aggregate(HasDest)); + static_assert(!__is_aggregate(HasPriv)); + static_assert(!__is_aggregate(HasProt)); + static_assert(__is_aggregate(HasRefAggregate)); + static_assert(__is_aggregate(HasNonPOD)); + static_assert(!__is_aggregate(HasVirt)); + static_assert(__is_aggregate(VirtAr)); + static_assert(__is_aggregate(HasInClassInit) == TrueAfterCpp11); + static_assert(!__is_aggregate(HasPrivateBase)); + static_assert(!__is_aggregate(HasProtectedBase)); + static_assert(!__is_aggregate(HasVirtBase)); + + static_assert(__is_aggregate(AggregateTemplate)); + static_assert(!__is_aggregate(NonAggregateTemplate)); + + static_assert(__is_aggregate(Vector)); // Extension supported by GCC and Clang + static_assert(__is_aggregate(VectorExt)); + static_assert(__is_aggregate(ComplexInt)); + static_assert(__is_aggregate(ComplexFloat)); } void is_arithmetic() { - int t01[T(__is_arithmetic(float))]; - int t02[T(__is_arithmetic(double))]; - int t03[T(__is_arithmetic(long double))]; - int t11[T(__is_arithmetic(bool))]; - int t12[T(__is_arithmetic(char))]; - int t13[T(__is_arithmetic(signed char))]; - int t14[T(__is_arithmetic(unsigned char))]; - //int t15[T(__is_arithmetic(char16_t))]; - //int t16[T(__is_arithmetic(char32_t))]; - int t17[T(__is_arithmetic(wchar_t))]; - int t18[T(__is_arithmetic(short))]; - int t19[T(__is_arithmetic(unsigned short))]; - int t20[T(__is_arithmetic(int))]; - int t21[T(__is_arithmetic(unsigned int))]; - int t22[T(__is_arithmetic(long))]; - int t23[T(__is_arithmetic(unsigned long))]; - - int t24[F(__is_arithmetic(Union))]; - int t25[F(__is_arithmetic(UnionAr))]; - int t26[F(__is_arithmetic(Derives))]; - int t27[F(__is_arithmetic(ClassType))]; - int t28[F(__is_arithmetic(Enum))]; - int t29[F(__is_arithmetic(void))]; - int t30[F(__is_arithmetic(cvoid))]; - int t31[F(__is_arithmetic(IntArNB))]; + static_assert(__is_arithmetic(float)); + static_assert(__is_arithmetic(double)); + static_assert(__is_arithmetic(long double)); + static_assert(__is_arithmetic(bool)); + static_assert(__is_arithmetic(char)); + static_assert(__is_arithmetic(signed char)); + static_assert(__is_arithmetic(unsigned char)); + //static_assert(__is_arithmetic(char16_t)); + //static_assert(__is_arithmetic(char32_t)); + static_assert(__is_arithmetic(wchar_t)); + static_assert(__is_arithmetic(short)); + static_assert(__is_arithmetic(unsigned short)); + static_assert(__is_arithmetic(int)); + static_assert(__is_arithmetic(unsigned int)); + static_assert(__is_arithmetic(long)); + static_assert(__is_arithmetic(unsigned long)); + + static_assert(!__is_arithmetic(Union)); + static_assert(!__is_arithmetic(UnionAr)); + static_assert(!__is_arithmetic(Derives)); + static_assert(!__is_arithmetic(ClassType)); + static_assert(!__is_arithmetic(Enum)); + static_assert(!__is_arithmetic(void)); + static_assert(!__is_arithmetic(cvoid)); + static_assert(!__is_arithmetic(IntArNB)); } void is_complete_type() { - int t01[T(__is_complete_type(float))]; - int t02[T(__is_complete_type(double))]; - int t03[T(__is_complete_type(long double))]; - int t11[T(__is_complete_type(bool))]; - int t12[T(__is_complete_type(char))]; - int t13[T(__is_complete_type(signed char))]; - int t14[T(__is_complete_type(unsigned char))]; - //int t15[T(__is_complete_type(char16_t))]; - //int t16[T(__is_complete_type(char32_t))]; - int t17[T(__is_complete_type(wchar_t))]; - int t18[T(__is_complete_type(short))]; - int t19[T(__is_complete_type(unsigned short))]; - int t20[T(__is_complete_type(int))]; - int t21[T(__is_complete_type(unsigned int))]; - int t22[T(__is_complete_type(long))]; - int t23[T(__is_complete_type(unsigned long))]; - int t24[T(__is_complete_type(ACompleteType))]; - - int t30[F(__is_complete_type(AnIncompleteType))]; + static_assert(__is_complete_type(float)); + static_assert(__is_complete_type(double)); + static_assert(__is_complete_type(long double)); + static_assert(__is_complete_type(bool)); + static_assert(__is_complete_type(char)); + static_assert(__is_complete_type(signed char)); + static_assert(__is_complete_type(unsigned char)); + //static_assert(__is_complete_type(char16_t)); + //static_assert(__is_complete_type(char32_t)); + static_assert(__is_complete_type(wchar_t)); + static_assert(__is_complete_type(short)); + static_assert(__is_complete_type(unsigned short)); + static_assert(__is_complete_type(int)); + static_assert(__is_complete_type(unsigned int)); + static_assert(__is_complete_type(long)); + static_assert(__is_complete_type(unsigned long)); + static_assert(__is_complete_type(ACompleteType)); + + static_assert(!__is_complete_type(AnIncompleteType)); } void is_void() { - int t01[T(__is_void(void))]; - int t02[T(__is_void(cvoid))]; - - int t10[F(__is_void(float))]; - int t11[F(__is_void(double))]; - int t12[F(__is_void(long double))]; - int t13[F(__is_void(bool))]; - int t14[F(__is_void(char))]; - int t15[F(__is_void(signed char))]; - int t16[F(__is_void(unsigned char))]; - int t17[F(__is_void(wchar_t))]; - int t18[F(__is_void(short))]; - int t19[F(__is_void(unsigned short))]; - int t20[F(__is_void(int))]; - int t21[F(__is_void(unsigned int))]; - int t22[F(__is_void(long))]; - int t23[F(__is_void(unsigned long))]; - int t24[F(__is_void(Union))]; - int t25[F(__is_void(UnionAr))]; - int t26[F(__is_void(Derives))]; - int t27[F(__is_void(ClassType))]; - int t28[F(__is_void(Enum))]; - int t29[F(__is_void(IntArNB))]; - int t30[F(__is_void(void*))]; - int t31[F(__is_void(cvoid*))]; + static_assert(__is_void(void)); + static_assert(__is_void(cvoid)); + + static_assert(!__is_void(float)); + static_assert(!__is_void(double)); + static_assert(!__is_void(long double)); + static_assert(!__is_void(bool)); + static_assert(!__is_void(char)); + static_assert(!__is_void(signed char)); + static_assert(!__is_void(unsigned char)); + static_assert(!__is_void(wchar_t)); + static_assert(!__is_void(short)); + static_assert(!__is_void(unsigned short)); + static_assert(!__is_void(int)); + static_assert(!__is_void(unsigned int)); + static_assert(!__is_void(long)); + static_assert(!__is_void(unsigned long)); + static_assert(!__is_void(Union)); + static_assert(!__is_void(UnionAr)); + static_assert(!__is_void(Derives)); + static_assert(!__is_void(ClassType)); + static_assert(!__is_void(Enum)); + static_assert(!__is_void(IntArNB)); + static_assert(!__is_void(void*)); + static_assert(!__is_void(cvoid*)); } void is_array() { - int t01[T(__is_array(IntAr))]; - int t02[T(__is_array(IntArNB))]; - int t03[T(__is_array(UnionAr))]; - - int t10[F(__is_array(void))]; - int t11[F(__is_array(cvoid))]; - int t12[F(__is_array(float))]; - int t13[F(__is_array(double))]; - int t14[F(__is_array(long double))]; - int t15[F(__is_array(bool))]; - int t16[F(__is_array(char))]; - int t17[F(__is_array(signed char))]; - int t18[F(__is_array(unsigned char))]; - int t19[F(__is_array(wchar_t))]; - int t20[F(__is_array(short))]; - int t21[F(__is_array(unsigned short))]; - int t22[F(__is_array(int))]; - int t23[F(__is_array(unsigned int))]; - int t24[F(__is_array(long))]; - int t25[F(__is_array(unsigned long))]; - int t26[F(__is_array(Union))]; - int t27[F(__is_array(Derives))]; - int t28[F(__is_array(ClassType))]; - int t29[F(__is_array(Enum))]; - int t30[F(__is_array(void*))]; - int t31[F(__is_array(cvoid*))]; + static_assert(__is_array(IntAr)); + static_assert(__is_array(IntArNB)); + static_assert(__is_array(UnionAr)); + + static_assert(!__is_array(void)); + static_assert(!__is_array(cvoid)); + static_assert(!__is_array(float)); + static_assert(!__is_array(double)); + static_assert(!__is_array(long double)); + static_assert(!__is_array(bool)); + static_assert(!__is_array(char)); + static_assert(!__is_array(signed char)); + static_assert(!__is_array(unsigned char)); + static_assert(!__is_array(wchar_t)); + static_assert(!__is_array(short)); + static_assert(!__is_array(unsigned short)); + static_assert(!__is_array(int)); + static_assert(!__is_array(unsigned int)); + static_assert(!__is_array(long)); + static_assert(!__is_array(unsigned long)); + static_assert(!__is_array(Union)); + static_assert(!__is_array(Derives)); + static_assert(!__is_array(ClassType)); + static_assert(!__is_array(Enum)); + static_assert(!__is_array(void*)); + static_assert(!__is_array(cvoid*)); } void is_bounded_array(int n) { - static_assert(__is_bounded_array(IntAr), ""); - static_assert(!__is_bounded_array(IntArNB), ""); - static_assert(__is_bounded_array(UnionAr), ""); - - static_assert(!__is_bounded_array(void), ""); - static_assert(!__is_bounded_array(cvoid), ""); - static_assert(!__is_bounded_array(float), ""); - static_assert(!__is_bounded_array(double), ""); - static_assert(!__is_bounded_array(long double), ""); - static_assert(!__is_bounded_array(bool), ""); - static_assert(!__is_bounded_array(char), ""); - static_assert(!__is_bounded_array(signed char), ""); - static_assert(!__is_bounded_array(unsigned char), ""); - static_assert(!__is_bounded_array(wchar_t), ""); - static_assert(!__is_bounded_array(short), ""); - static_assert(!__is_bounded_array(unsigned short), ""); - static_assert(!__is_bounded_array(int), ""); - static_assert(!__is_bounded_array(unsigned int), ""); - static_assert(!__is_bounded_array(long), ""); - static_assert(!__is_bounded_array(unsigned long), ""); - static_assert(!__is_bounded_array(Union), ""); - static_assert(!__is_bounded_array(Derives), ""); - static_assert(!__is_bounded_array(ClassType), ""); - static_assert(!__is_bounded_array(Enum), ""); - static_assert(!__is_bounded_array(void *), ""); - static_assert(!__is_bounded_array(cvoid *), ""); + static_assert(__is_bounded_array(IntAr)); + static_assert(!__is_bounded_array(IntArNB)); + static_assert(__is_bounded_array(UnionAr)); + + static_assert(!__is_bounded_array(void)); + static_assert(!__is_bounded_array(cvoid)); + static_assert(!__is_bounded_array(float)); + static_assert(!__is_bounded_array(double)); + static_assert(!__is_bounded_array(long double)); + static_assert(!__is_bounded_array(bool)); + static_assert(!__is_bounded_array(char)); + static_assert(!__is_bounded_array(signed char)); + static_assert(!__is_bounded_array(unsigned char)); + static_assert(!__is_bounded_array(wchar_t)); + static_assert(!__is_bounded_array(short)); + static_assert(!__is_bounded_array(unsigned short)); + static_assert(!__is_bounded_array(int)); + static_assert(!__is_bounded_array(unsigned int)); + static_assert(!__is_bounded_array(long)); + static_assert(!__is_bounded_array(unsigned long)); + static_assert(!__is_bounded_array(Union)); + static_assert(!__is_bounded_array(Derives)); + static_assert(!__is_bounded_array(ClassType)); + static_assert(!__is_bounded_array(Enum)); + static_assert(!__is_bounded_array(void *)); + static_assert(!__is_bounded_array(cvoid *)); int t32[n]; (void)__is_bounded_array(decltype(t32)); // expected-error{{variable length arrays are not supported for '__is_bounded_array'}} } void is_unbounded_array(int n) { - static_assert(!__is_unbounded_array(IntAr), ""); - static_assert(__is_unbounded_array(IntArNB), ""); - static_assert(!__is_unbounded_array(UnionAr), ""); - - static_assert(!__is_unbounded_array(void), ""); - static_assert(!__is_unbounded_array(cvoid), ""); - static_assert(!__is_unbounded_array(float), ""); - static_assert(!__is_unbounded_array(double), ""); - static_assert(!__is_unbounded_array(long double), ""); - static_assert(!__is_unbounded_array(bool), ""); - static_assert(!__is_unbounded_array(char), ""); - static_assert(!__is_unbounded_array(signed char), ""); - static_assert(!__is_unbounded_array(unsigned char), ""); - static_assert(!__is_unbounded_array(wchar_t), ""); - static_assert(!__is_unbounded_array(short), ""); - static_assert(!__is_unbounded_array(unsigned short), ""); - static_assert(!__is_unbounded_array(int), ""); - static_assert(!__is_unbounded_array(unsigned int), ""); - static_assert(!__is_unbounded_array(long), ""); - static_assert(!__is_unbounded_array(unsigned long), ""); - static_assert(!__is_unbounded_array(Union), ""); - static_assert(!__is_unbounded_array(Derives), ""); - static_assert(!__is_unbounded_array(ClassType), ""); - static_assert(!__is_unbounded_array(Enum), ""); - static_assert(!__is_unbounded_array(void *), ""); - static_assert(!__is_unbounded_array(cvoid *), ""); + static_assert(!__is_unbounded_array(IntAr)); + static_assert(__is_unbounded_array(IntArNB)); + static_assert(!__is_unbounded_array(UnionAr)); + + static_assert(!__is_unbounded_array(void)); + static_assert(!__is_unbounded_array(cvoid)); + static_assert(!__is_unbounded_array(float)); + static_assert(!__is_unbounded_array(double)); + static_assert(!__is_unbounded_array(long double)); + static_assert(!__is_unbounded_array(bool)); + static_assert(!__is_unbounded_array(char)); + static_assert(!__is_unbounded_array(signed char)); + static_assert(!__is_unbounded_array(unsigned char)); + static_assert(!__is_unbounded_array(wchar_t)); + static_assert(!__is_unbounded_array(short)); + static_assert(!__is_unbounded_array(unsigned short)); + static_assert(!__is_unbounded_array(int)); + static_assert(!__is_unbounded_array(unsigned int)); + static_assert(!__is_unbounded_array(long)); + static_assert(!__is_unbounded_array(unsigned long)); + static_assert(!__is_unbounded_array(Union)); + static_assert(!__is_unbounded_array(Derives)); + static_assert(!__is_unbounded_array(ClassType)); + static_assert(!__is_unbounded_array(Enum)); + static_assert(!__is_unbounded_array(void *)); + static_assert(!__is_unbounded_array(cvoid *)); int t32[n]; (void)__is_unbounded_array(decltype(t32)); // expected-error{{variable length arrays are not supported for '__is_unbounded_array'}} } void is_referenceable() { - static_assert(__is_referenceable(int), ""); - static_assert(__is_referenceable(const int), ""); - static_assert(__is_referenceable(volatile int), ""); - static_assert(__is_referenceable(const volatile int), ""); - static_assert(__is_referenceable(int *), ""); - static_assert(__is_referenceable(int &), ""); - static_assert(__is_referenceable(int &&), ""); - static_assert(__is_referenceable(int (*)()), ""); - static_assert(__is_referenceable(int (&)()), ""); - static_assert(__is_referenceable(int(&&)()), ""); - static_assert(__is_referenceable(IntAr), ""); - static_assert(__is_referenceable(IntArNB), ""); - static_assert(__is_referenceable(decltype(nullptr)), ""); - static_assert(__is_referenceable(Empty), ""); - static_assert(__is_referenceable(Union), ""); - static_assert(__is_referenceable(Derives), ""); - static_assert(__is_referenceable(Enum), ""); - static_assert(__is_referenceable(EnumClass), ""); - static_assert(__is_referenceable(int Empty::*), ""); - static_assert(__is_referenceable(int(Empty::*)()), ""); - static_assert(__is_referenceable(AnIncompleteType), ""); - static_assert(__is_referenceable(struct AnIncompleteType), ""); + static_assert(__is_referenceable(int)); + static_assert(__is_referenceable(const int)); + static_assert(__is_referenceable(volatile int)); + static_assert(__is_referenceable(const volatile int)); + static_assert(__is_referenceable(int *)); + static_assert(__is_referenceable(int &)); + static_assert(__is_referenceable(int &&)); + static_assert(__is_referenceable(int (*)())); + static_assert(__is_referenceable(int (&)())); + static_assert(__is_referenceable(int(&&)())); + static_assert(__is_referenceable(IntAr)); + static_assert(__is_referenceable(IntArNB)); + static_assert(__is_referenceable(decltype(nullptr))); + static_assert(__is_referenceable(Empty)); + static_assert(__is_referenceable(Union)); + static_assert(__is_referenceable(Derives)); + static_assert(__is_referenceable(Enum)); + static_assert(__is_referenceable(EnumClass)); + static_assert(__is_referenceable(int Empty::*)); + static_assert(__is_referenceable(int(Empty::*)())); + static_assert(__is_referenceable(AnIncompleteType)); + static_assert(__is_referenceable(struct AnIncompleteType)); using function_type = void(int); - static_assert(__is_referenceable(function_type), ""); + static_assert(__is_referenceable(function_type)); - static_assert(!__is_referenceable(void), ""); + static_assert(!__is_referenceable(void)); } template void tmpl_func(T&) {} @@ -817,152 +815,152 @@ template struct type_wrapper { void is_function() { - int t01[T(__is_function(type_wrapper::type))]; - int t02[T(__is_function(typeof(tmpl_func)))]; + static_assert(__is_function(type_wrapper::type)); + static_assert(__is_function(typeof(tmpl_func))); typedef void (*ptr_to_func_type)(void); - int t10[F(__is_function(void))]; - int t11[F(__is_function(cvoid))]; - int t12[F(__is_function(float))]; - int t13[F(__is_function(double))]; - int t14[F(__is_function(long double))]; - int t15[F(__is_function(bool))]; - int t16[F(__is_function(char))]; - int t17[F(__is_function(signed char))]; - int t18[F(__is_function(unsigned char))]; - int t19[F(__is_function(wchar_t))]; - int t20[F(__is_function(short))]; - int t21[F(__is_function(unsigned short))]; - int t22[F(__is_function(int))]; - int t23[F(__is_function(unsigned int))]; - int t24[F(__is_function(long))]; - int t25[F(__is_function(unsigned long))]; - int t26[F(__is_function(Union))]; - int t27[F(__is_function(Derives))]; - int t28[F(__is_function(ClassType))]; - int t29[F(__is_function(Enum))]; - int t30[F(__is_function(void*))]; - int t31[F(__is_function(cvoid*))]; - int t32[F(__is_function(void(*)()))]; - int t33[F(__is_function(ptr_to_func_type))]; - int t34[F(__is_function(type_wrapper::ptrtype))]; - int t35[F(__is_function(type_wrapper::reftype))]; + static_assert(!__is_function(void)); + static_assert(!__is_function(cvoid)); + static_assert(!__is_function(float)); + static_assert(!__is_function(double)); + static_assert(!__is_function(long double)); + static_assert(!__is_function(bool)); + static_assert(!__is_function(char)); + static_assert(!__is_function(signed char)); + static_assert(!__is_function(unsigned char)); + static_assert(!__is_function(wchar_t)); + static_assert(!__is_function(short)); + static_assert(!__is_function(unsigned short)); + static_assert(!__is_function(int)); + static_assert(!__is_function(unsigned int)); + static_assert(!__is_function(long)); + static_assert(!__is_function(unsigned long)); + static_assert(!__is_function(Union)); + static_assert(!__is_function(Derives)); + static_assert(!__is_function(ClassType)); + static_assert(!__is_function(Enum)); + static_assert(!__is_function(void*)); + static_assert(!__is_function(cvoid*)); + static_assert(!__is_function(void(*)())); + static_assert(!__is_function(ptr_to_func_type)); + static_assert(!__is_function(type_wrapper::ptrtype)); + static_assert(!__is_function(type_wrapper::reftype)); } void is_reference() { - int t01[T(__is_reference(int&))]; - int t02[T(__is_reference(const int&))]; - int t03[T(__is_reference(void *&))]; + static_assert(__is_reference(int&)); + static_assert(__is_reference(const int&)); + static_assert(__is_reference(void *&)); - int t10[F(__is_reference(int))]; - int t11[F(__is_reference(const int))]; - int t12[F(__is_reference(void *))]; + static_assert(!__is_reference(int)); + static_assert(!__is_reference(const int)); + static_assert(!__is_reference(void *)); } void is_lvalue_reference() { - int t01[T(__is_lvalue_reference(int&))]; - int t02[T(__is_lvalue_reference(void *&))]; - int t03[T(__is_lvalue_reference(const int&))]; - int t04[T(__is_lvalue_reference(void * const &))]; - - int t10[F(__is_lvalue_reference(int))]; - int t11[F(__is_lvalue_reference(const int))]; - int t12[F(__is_lvalue_reference(void *))]; + static_assert(__is_lvalue_reference(int&)); + static_assert(__is_lvalue_reference(void *&)); + static_assert(__is_lvalue_reference(const int&)); + static_assert(__is_lvalue_reference(void * const &)); + + static_assert(!__is_lvalue_reference(int)); + static_assert(!__is_lvalue_reference(const int)); + static_assert(!__is_lvalue_reference(void *)); } #if __has_feature(cxx_rvalue_references) void is_rvalue_reference() { - int t01[T(__is_rvalue_reference(const int&&))]; - int t02[T(__is_rvalue_reference(void * const &&))]; - - int t10[F(__is_rvalue_reference(int&))]; - int t11[F(__is_rvalue_reference(void *&))]; - int t12[F(__is_rvalue_reference(const int&))]; - int t13[F(__is_rvalue_reference(void * const &))]; - int t14[F(__is_rvalue_reference(int))]; - int t15[F(__is_rvalue_reference(const int))]; - int t16[F(__is_rvalue_reference(void *))]; + static_assert(__is_rvalue_reference(const int&&)); + static_assert(__is_rvalue_reference(void * const &&)); + + static_assert(!__is_rvalue_reference(int&)); + static_assert(!__is_rvalue_reference(void *&)); + static_assert(!__is_rvalue_reference(const int&)); + static_assert(!__is_rvalue_reference(void * const &)); + static_assert(!__is_rvalue_reference(int)); + static_assert(!__is_rvalue_reference(const int)); + static_assert(!__is_rvalue_reference(void *)); } #endif void is_fundamental() { - int t01[T(__is_fundamental(float))]; - int t02[T(__is_fundamental(double))]; - int t03[T(__is_fundamental(long double))]; - int t11[T(__is_fundamental(bool))]; - int t12[T(__is_fundamental(char))]; - int t13[T(__is_fundamental(signed char))]; - int t14[T(__is_fundamental(unsigned char))]; - //int t15[T(__is_fundamental(char16_t))]; - //int t16[T(__is_fundamental(char32_t))]; - int t17[T(__is_fundamental(wchar_t))]; - int t18[T(__is_fundamental(short))]; - int t19[T(__is_fundamental(unsigned short))]; - int t20[T(__is_fundamental(int))]; - int t21[T(__is_fundamental(unsigned int))]; - int t22[T(__is_fundamental(long))]; - int t23[T(__is_fundamental(unsigned long))]; - int t24[T(__is_fundamental(void))]; - int t25[T(__is_fundamental(cvoid))]; - int t26[T(__is_fundamental(decltype(nullptr)))]; - - int t30[F(__is_fundamental(Union))]; - int t31[F(__is_fundamental(UnionAr))]; - int t32[F(__is_fundamental(Derives))]; - int t33[F(__is_fundamental(ClassType))]; - int t34[F(__is_fundamental(Enum))]; - int t35[F(__is_fundamental(IntArNB))]; + static_assert(__is_fundamental(float)); + static_assert(__is_fundamental(double)); + static_assert(__is_fundamental(long double)); + static_assert(__is_fundamental(bool)); + static_assert(__is_fundamental(char)); + static_assert(__is_fundamental(signed char)); + static_assert(__is_fundamental(unsigned char)); + //static_assert(__is_fundamental(char16_t)); + //static_assert(__is_fundamental(char32_t)); + static_assert(__is_fundamental(wchar_t)); + static_assert(__is_fundamental(short)); + static_assert(__is_fundamental(unsigned short)); + static_assert(__is_fundamental(int)); + static_assert(__is_fundamental(unsigned int)); + static_assert(__is_fundamental(long)); + static_assert(__is_fundamental(unsigned long)); + static_assert(__is_fundamental(void)); + static_assert(__is_fundamental(cvoid)); + static_assert(__is_fundamental(decltype(nullptr))); + + static_assert(!__is_fundamental(Union)); + static_assert(!__is_fundamental(UnionAr)); + static_assert(!__is_fundamental(Derives)); + static_assert(!__is_fundamental(ClassType)); + static_assert(!__is_fundamental(Enum)); + static_assert(!__is_fundamental(IntArNB)); } void is_object() { - int t01[T(__is_object(int))]; - int t02[T(__is_object(int *))]; - int t03[T(__is_object(void *))]; - int t04[T(__is_object(Union))]; - int t05[T(__is_object(UnionAr))]; - int t06[T(__is_object(ClassType))]; - int t07[T(__is_object(Enum))]; - - int t10[F(__is_object(type_wrapper::type))]; - int t11[F(__is_object(int&))]; - int t12[F(__is_object(void))]; + static_assert(__is_object(int)); + static_assert(__is_object(int *)); + static_assert(__is_object(void *)); + static_assert(__is_object(Union)); + static_assert(__is_object(UnionAr)); + static_assert(__is_object(ClassType)); + static_assert(__is_object(Enum)); + + static_assert(!__is_object(type_wrapper::type)); + static_assert(!__is_object(int&)); + static_assert(!__is_object(void)); } void is_scalar() { - int t01[T(__is_scalar(float))]; - int t02[T(__is_scalar(double))]; - int t03[T(__is_scalar(long double))]; - int t04[T(__is_scalar(bool))]; - int t05[T(__is_scalar(char))]; - int t06[T(__is_scalar(signed char))]; - int t07[T(__is_scalar(unsigned char))]; - int t08[T(__is_scalar(wchar_t))]; - int t09[T(__is_scalar(short))]; - int t10[T(__is_scalar(unsigned short))]; - int t11[T(__is_scalar(int))]; - int t12[T(__is_scalar(unsigned int))]; - int t13[T(__is_scalar(long))]; - int t14[T(__is_scalar(unsigned long))]; - int t15[T(__is_scalar(Enum))]; - int t16[T(__is_scalar(void*))]; - int t17[T(__is_scalar(cvoid*))]; - - int t20[F(__is_scalar(void))]; - int t21[F(__is_scalar(cvoid))]; - int t22[F(__is_scalar(Union))]; - int t23[F(__is_scalar(UnionAr))]; - int t24[F(__is_scalar(Derives))]; - int t25[F(__is_scalar(ClassType))]; - int t26[F(__is_scalar(IntArNB))]; + static_assert(__is_scalar(float)); + static_assert(__is_scalar(double)); + static_assert(__is_scalar(long double)); + static_assert(__is_scalar(bool)); + static_assert(__is_scalar(char)); + static_assert(__is_scalar(signed char)); + static_assert(__is_scalar(unsigned char)); + static_assert(__is_scalar(wchar_t)); + static_assert(__is_scalar(short)); + static_assert(__is_scalar(unsigned short)); + static_assert(__is_scalar(int)); + static_assert(__is_scalar(unsigned int)); + static_assert(__is_scalar(long)); + static_assert(__is_scalar(unsigned long)); + static_assert(__is_scalar(Enum)); + static_assert(__is_scalar(void*)); + static_assert(__is_scalar(cvoid*)); + + static_assert(!__is_scalar(void)); + static_assert(!__is_scalar(cvoid)); + static_assert(!__is_scalar(Union)); + static_assert(!__is_scalar(UnionAr)); + static_assert(!__is_scalar(Derives)); + static_assert(!__is_scalar(ClassType)); + static_assert(!__is_scalar(IntArNB)); } struct StructWithMembers { @@ -972,314 +970,314 @@ struct StructWithMembers { void is_compound() { - int t01[T(__is_compound(void*))]; - int t02[T(__is_compound(cvoid*))]; - int t03[T(__is_compound(void (*)()))]; - int t04[T(__is_compound(int StructWithMembers::*))]; - int t05[T(__is_compound(void (StructWithMembers::*)()))]; - int t06[T(__is_compound(int&))]; - int t07[T(__is_compound(Union))]; - int t08[T(__is_compound(UnionAr))]; - int t09[T(__is_compound(Derives))]; - int t10[T(__is_compound(ClassType))]; - int t11[T(__is_compound(IntArNB))]; - int t12[T(__is_compound(Enum))]; - - int t20[F(__is_compound(float))]; - int t21[F(__is_compound(double))]; - int t22[F(__is_compound(long double))]; - int t23[F(__is_compound(bool))]; - int t24[F(__is_compound(char))]; - int t25[F(__is_compound(signed char))]; - int t26[F(__is_compound(unsigned char))]; - int t27[F(__is_compound(wchar_t))]; - int t28[F(__is_compound(short))]; - int t29[F(__is_compound(unsigned short))]; - int t30[F(__is_compound(int))]; - int t31[F(__is_compound(unsigned int))]; - int t32[F(__is_compound(long))]; - int t33[F(__is_compound(unsigned long))]; - int t34[F(__is_compound(void))]; - int t35[F(__is_compound(cvoid))]; + static_assert(__is_compound(void*)); + static_assert(__is_compound(cvoid*)); + static_assert(__is_compound(void (*)())); + static_assert(__is_compound(int StructWithMembers::*)); + static_assert(__is_compound(void (StructWithMembers::*)())); + static_assert(__is_compound(int&)); + static_assert(__is_compound(Union)); + static_assert(__is_compound(UnionAr)); + static_assert(__is_compound(Derives)); + static_assert(__is_compound(ClassType)); + static_assert(__is_compound(IntArNB)); + static_assert(__is_compound(Enum)); + + static_assert(!__is_compound(float)); + static_assert(!__is_compound(double)); + static_assert(!__is_compound(long double)); + static_assert(!__is_compound(bool)); + static_assert(!__is_compound(char)); + static_assert(!__is_compound(signed char)); + static_assert(!__is_compound(unsigned char)); + static_assert(!__is_compound(wchar_t)); + static_assert(!__is_compound(short)); + static_assert(!__is_compound(unsigned short)); + static_assert(!__is_compound(int)); + static_assert(!__is_compound(unsigned int)); + static_assert(!__is_compound(long)); + static_assert(!__is_compound(unsigned long)); + static_assert(!__is_compound(void)); + static_assert(!__is_compound(cvoid)); } void is_pointer() { StructWithMembers x; - int t01[T(__is_pointer(void*))]; - int t02[T(__is_pointer(cvoid*))]; - int t03[T(__is_pointer(cvoid*))]; - int t04[T(__is_pointer(char*))]; - int t05[T(__is_pointer(int*))]; - int t06[T(__is_pointer(int**))]; - int t07[T(__is_pointer(ClassType*))]; - int t08[T(__is_pointer(Derives*))]; - int t09[T(__is_pointer(Enum*))]; - int t10[T(__is_pointer(IntArNB*))]; - int t11[T(__is_pointer(Union*))]; - int t12[T(__is_pointer(UnionAr*))]; - int t13[T(__is_pointer(StructWithMembers*))]; - int t14[T(__is_pointer(void (*)()))]; - - int t20[F(__is_pointer(void))]; - int t21[F(__is_pointer(cvoid))]; - int t22[F(__is_pointer(cvoid))]; - int t23[F(__is_pointer(char))]; - int t24[F(__is_pointer(int))]; - int t25[F(__is_pointer(int))]; - int t26[F(__is_pointer(ClassType))]; - int t27[F(__is_pointer(Derives))]; - int t28[F(__is_pointer(Enum))]; - int t29[F(__is_pointer(IntArNB))]; - int t30[F(__is_pointer(Union))]; - int t31[F(__is_pointer(UnionAr))]; - int t32[F(__is_pointer(StructWithMembers))]; - int t33[F(__is_pointer(int StructWithMembers::*))]; - int t34[F(__is_pointer(void (StructWithMembers::*) ()))]; + static_assert(__is_pointer(void*)); + static_assert(__is_pointer(cvoid*)); + static_assert(__is_pointer(cvoid*)); + static_assert(__is_pointer(char*)); + static_assert(__is_pointer(int*)); + static_assert(__is_pointer(int**)); + static_assert(__is_pointer(ClassType*)); + static_assert(__is_pointer(Derives*)); + static_assert(__is_pointer(Enum*)); + static_assert(__is_pointer(IntArNB*)); + static_assert(__is_pointer(Union*)); + static_assert(__is_pointer(UnionAr*)); + static_assert(__is_pointer(StructWithMembers*)); + static_assert(__is_pointer(void (*)())); + + static_assert(!__is_pointer(void)); + static_assert(!__is_pointer(cvoid)); + static_assert(!__is_pointer(cvoid)); + static_assert(!__is_pointer(char)); + static_assert(!__is_pointer(int)); + static_assert(!__is_pointer(int)); + static_assert(!__is_pointer(ClassType)); + static_assert(!__is_pointer(Derives)); + static_assert(!__is_pointer(Enum)); + static_assert(!__is_pointer(IntArNB)); + static_assert(!__is_pointer(Union)); + static_assert(!__is_pointer(UnionAr)); + static_assert(!__is_pointer(StructWithMembers)); + static_assert(!__is_pointer(int StructWithMembers::*)); + static_assert(!__is_pointer(void (StructWithMembers::*) ())); } void is_null_pointer() { StructWithMembers x; - static_assert(__is_nullptr(decltype(nullptr)), ""); - static_assert(!__is_nullptr(void *), ""); - static_assert(!__is_nullptr(cvoid *), ""); - static_assert(!__is_nullptr(cvoid *), ""); - static_assert(!__is_nullptr(char *), ""); - static_assert(!__is_nullptr(int *), ""); - static_assert(!__is_nullptr(int **), ""); - static_assert(!__is_nullptr(ClassType *), ""); - static_assert(!__is_nullptr(Derives *), ""); - static_assert(!__is_nullptr(Enum *), ""); - static_assert(!__is_nullptr(IntArNB *), ""); - static_assert(!__is_nullptr(Union *), ""); - static_assert(!__is_nullptr(UnionAr *), ""); - static_assert(!__is_nullptr(StructWithMembers *), ""); - static_assert(!__is_nullptr(void (*)()), ""); - - static_assert(!__is_nullptr(void), ""); - static_assert(!__is_nullptr(cvoid), ""); - static_assert(!__is_nullptr(cvoid), ""); - static_assert(!__is_nullptr(char), ""); - static_assert(!__is_nullptr(int), ""); - static_assert(!__is_nullptr(int), ""); - static_assert(!__is_nullptr(ClassType), ""); - static_assert(!__is_nullptr(Derives), ""); - static_assert(!__is_nullptr(Enum), ""); - static_assert(!__is_nullptr(IntArNB), ""); - static_assert(!__is_nullptr(Union), ""); - static_assert(!__is_nullptr(UnionAr), ""); - static_assert(!__is_nullptr(StructWithMembers), ""); - static_assert(!__is_nullptr(int StructWithMembers::*), ""); - static_assert(!__is_nullptr(void(StructWithMembers::*)()), ""); + static_assert(__is_nullptr(decltype(nullptr))); + static_assert(!__is_nullptr(void *)); + static_assert(!__is_nullptr(cvoid *)); + static_assert(!__is_nullptr(cvoid *)); + static_assert(!__is_nullptr(char *)); + static_assert(!__is_nullptr(int *)); + static_assert(!__is_nullptr(int **)); + static_assert(!__is_nullptr(ClassType *)); + static_assert(!__is_nullptr(Derives *)); + static_assert(!__is_nullptr(Enum *)); + static_assert(!__is_nullptr(IntArNB *)); + static_assert(!__is_nullptr(Union *)); + static_assert(!__is_nullptr(UnionAr *)); + static_assert(!__is_nullptr(StructWithMembers *)); + static_assert(!__is_nullptr(void (*)())); + + static_assert(!__is_nullptr(void)); + static_assert(!__is_nullptr(cvoid)); + static_assert(!__is_nullptr(cvoid)); + static_assert(!__is_nullptr(char)); + static_assert(!__is_nullptr(int)); + static_assert(!__is_nullptr(int)); + static_assert(!__is_nullptr(ClassType)); + static_assert(!__is_nullptr(Derives)); + static_assert(!__is_nullptr(Enum)); + static_assert(!__is_nullptr(IntArNB)); + static_assert(!__is_nullptr(Union)); + static_assert(!__is_nullptr(UnionAr)); + static_assert(!__is_nullptr(StructWithMembers)); + static_assert(!__is_nullptr(int StructWithMembers::*)); + static_assert(!__is_nullptr(void(StructWithMembers::*)())); } void is_member_object_pointer() { StructWithMembers x; - int t01[T(__is_member_object_pointer(int StructWithMembers::*))]; - - int t10[F(__is_member_object_pointer(void (StructWithMembers::*) ()))]; - int t11[F(__is_member_object_pointer(void*))]; - int t12[F(__is_member_object_pointer(cvoid*))]; - int t13[F(__is_member_object_pointer(cvoid*))]; - int t14[F(__is_member_object_pointer(char*))]; - int t15[F(__is_member_object_pointer(int*))]; - int t16[F(__is_member_object_pointer(int**))]; - int t17[F(__is_member_object_pointer(ClassType*))]; - int t18[F(__is_member_object_pointer(Derives*))]; - int t19[F(__is_member_object_pointer(Enum*))]; - int t20[F(__is_member_object_pointer(IntArNB*))]; - int t21[F(__is_member_object_pointer(Union*))]; - int t22[F(__is_member_object_pointer(UnionAr*))]; - int t23[F(__is_member_object_pointer(StructWithMembers*))]; - int t24[F(__is_member_object_pointer(void))]; - int t25[F(__is_member_object_pointer(cvoid))]; - int t26[F(__is_member_object_pointer(cvoid))]; - int t27[F(__is_member_object_pointer(char))]; - int t28[F(__is_member_object_pointer(int))]; - int t29[F(__is_member_object_pointer(int))]; - int t30[F(__is_member_object_pointer(ClassType))]; - int t31[F(__is_member_object_pointer(Derives))]; - int t32[F(__is_member_object_pointer(Enum))]; - int t33[F(__is_member_object_pointer(IntArNB))]; - int t34[F(__is_member_object_pointer(Union))]; - int t35[F(__is_member_object_pointer(UnionAr))]; - int t36[F(__is_member_object_pointer(StructWithMembers))]; - int t37[F(__is_member_object_pointer(void (*)()))]; + static_assert(__is_member_object_pointer(int StructWithMembers::*)); + + static_assert(!__is_member_object_pointer(void (StructWithMembers::*) ())); + static_assert(!__is_member_object_pointer(void*)); + static_assert(!__is_member_object_pointer(cvoid*)); + static_assert(!__is_member_object_pointer(cvoid*)); + static_assert(!__is_member_object_pointer(char*)); + static_assert(!__is_member_object_pointer(int*)); + static_assert(!__is_member_object_pointer(int**)); + static_assert(!__is_member_object_pointer(ClassType*)); + static_assert(!__is_member_object_pointer(Derives*)); + static_assert(!__is_member_object_pointer(Enum*)); + static_assert(!__is_member_object_pointer(IntArNB*)); + static_assert(!__is_member_object_pointer(Union*)); + static_assert(!__is_member_object_pointer(UnionAr*)); + static_assert(!__is_member_object_pointer(StructWithMembers*)); + static_assert(!__is_member_object_pointer(void)); + static_assert(!__is_member_object_pointer(cvoid)); + static_assert(!__is_member_object_pointer(cvoid)); + static_assert(!__is_member_object_pointer(char)); + static_assert(!__is_member_object_pointer(int)); + static_assert(!__is_member_object_pointer(int)); + static_assert(!__is_member_object_pointer(ClassType)); + static_assert(!__is_member_object_pointer(Derives)); + static_assert(!__is_member_object_pointer(Enum)); + static_assert(!__is_member_object_pointer(IntArNB)); + static_assert(!__is_member_object_pointer(Union)); + static_assert(!__is_member_object_pointer(UnionAr)); + static_assert(!__is_member_object_pointer(StructWithMembers)); + static_assert(!__is_member_object_pointer(void (*)())); } void is_member_function_pointer() { StructWithMembers x; - int t01[T(__is_member_function_pointer(void (StructWithMembers::*) ()))]; - - int t10[F(__is_member_function_pointer(int StructWithMembers::*))]; - int t11[F(__is_member_function_pointer(void*))]; - int t12[F(__is_member_function_pointer(cvoid*))]; - int t13[F(__is_member_function_pointer(cvoid*))]; - int t14[F(__is_member_function_pointer(char*))]; - int t15[F(__is_member_function_pointer(int*))]; - int t16[F(__is_member_function_pointer(int**))]; - int t17[F(__is_member_function_pointer(ClassType*))]; - int t18[F(__is_member_function_pointer(Derives*))]; - int t19[F(__is_member_function_pointer(Enum*))]; - int t20[F(__is_member_function_pointer(IntArNB*))]; - int t21[F(__is_member_function_pointer(Union*))]; - int t22[F(__is_member_function_pointer(UnionAr*))]; - int t23[F(__is_member_function_pointer(StructWithMembers*))]; - int t24[F(__is_member_function_pointer(void))]; - int t25[F(__is_member_function_pointer(cvoid))]; - int t26[F(__is_member_function_pointer(cvoid))]; - int t27[F(__is_member_function_pointer(char))]; - int t28[F(__is_member_function_pointer(int))]; - int t29[F(__is_member_function_pointer(int))]; - int t30[F(__is_member_function_pointer(ClassType))]; - int t31[F(__is_member_function_pointer(Derives))]; - int t32[F(__is_member_function_pointer(Enum))]; - int t33[F(__is_member_function_pointer(IntArNB))]; - int t34[F(__is_member_function_pointer(Union))]; - int t35[F(__is_member_function_pointer(UnionAr))]; - int t36[F(__is_member_function_pointer(StructWithMembers))]; - int t37[F(__is_member_function_pointer(void (*)()))]; + static_assert(__is_member_function_pointer(void (StructWithMembers::*) ())); + + static_assert(!__is_member_function_pointer(int StructWithMembers::*)); + static_assert(!__is_member_function_pointer(void*)); + static_assert(!__is_member_function_pointer(cvoid*)); + static_assert(!__is_member_function_pointer(cvoid*)); + static_assert(!__is_member_function_pointer(char*)); + static_assert(!__is_member_function_pointer(int*)); + static_assert(!__is_member_function_pointer(int**)); + static_assert(!__is_member_function_pointer(ClassType*)); + static_assert(!__is_member_function_pointer(Derives*)); + static_assert(!__is_member_function_pointer(Enum*)); + static_assert(!__is_member_function_pointer(IntArNB*)); + static_assert(!__is_member_function_pointer(Union*)); + static_assert(!__is_member_function_pointer(UnionAr*)); + static_assert(!__is_member_function_pointer(StructWithMembers*)); + static_assert(!__is_member_function_pointer(void)); + static_assert(!__is_member_function_pointer(cvoid)); + static_assert(!__is_member_function_pointer(cvoid)); + static_assert(!__is_member_function_pointer(char)); + static_assert(!__is_member_function_pointer(int)); + static_assert(!__is_member_function_pointer(int)); + static_assert(!__is_member_function_pointer(ClassType)); + static_assert(!__is_member_function_pointer(Derives)); + static_assert(!__is_member_function_pointer(Enum)); + static_assert(!__is_member_function_pointer(IntArNB)); + static_assert(!__is_member_function_pointer(Union)); + static_assert(!__is_member_function_pointer(UnionAr)); + static_assert(!__is_member_function_pointer(StructWithMembers)); + static_assert(!__is_member_function_pointer(void (*)())); } void is_member_pointer() { StructWithMembers x; - int t01[T(__is_member_pointer(int StructWithMembers::*))]; - int t02[T(__is_member_pointer(void (StructWithMembers::*) ()))]; - - int t10[F(__is_member_pointer(void*))]; - int t11[F(__is_member_pointer(cvoid*))]; - int t12[F(__is_member_pointer(cvoid*))]; - int t13[F(__is_member_pointer(char*))]; - int t14[F(__is_member_pointer(int*))]; - int t15[F(__is_member_pointer(int**))]; - int t16[F(__is_member_pointer(ClassType*))]; - int t17[F(__is_member_pointer(Derives*))]; - int t18[F(__is_member_pointer(Enum*))]; - int t19[F(__is_member_pointer(IntArNB*))]; - int t20[F(__is_member_pointer(Union*))]; - int t21[F(__is_member_pointer(UnionAr*))]; - int t22[F(__is_member_pointer(StructWithMembers*))]; - int t23[F(__is_member_pointer(void))]; - int t24[F(__is_member_pointer(cvoid))]; - int t25[F(__is_member_pointer(cvoid))]; - int t26[F(__is_member_pointer(char))]; - int t27[F(__is_member_pointer(int))]; - int t28[F(__is_member_pointer(int))]; - int t29[F(__is_member_pointer(ClassType))]; - int t30[F(__is_member_pointer(Derives))]; - int t31[F(__is_member_pointer(Enum))]; - int t32[F(__is_member_pointer(IntArNB))]; - int t33[F(__is_member_pointer(Union))]; - int t34[F(__is_member_pointer(UnionAr))]; - int t35[F(__is_member_pointer(StructWithMembers))]; - int t36[F(__is_member_pointer(void (*)()))]; + static_assert(__is_member_pointer(int StructWithMembers::*)); + static_assert(__is_member_pointer(void (StructWithMembers::*) ())); + + static_assert(!__is_member_pointer(void*)); + static_assert(!__is_member_pointer(cvoid*)); + static_assert(!__is_member_pointer(cvoid*)); + static_assert(!__is_member_pointer(char*)); + static_assert(!__is_member_pointer(int*)); + static_assert(!__is_member_pointer(int**)); + static_assert(!__is_member_pointer(ClassType*)); + static_assert(!__is_member_pointer(Derives*)); + static_assert(!__is_member_pointer(Enum*)); + static_assert(!__is_member_pointer(IntArNB*)); + static_assert(!__is_member_pointer(Union*)); + static_assert(!__is_member_pointer(UnionAr*)); + static_assert(!__is_member_pointer(StructWithMembers*)); + static_assert(!__is_member_pointer(void)); + static_assert(!__is_member_pointer(cvoid)); + static_assert(!__is_member_pointer(cvoid)); + static_assert(!__is_member_pointer(char)); + static_assert(!__is_member_pointer(int)); + static_assert(!__is_member_pointer(int)); + static_assert(!__is_member_pointer(ClassType)); + static_assert(!__is_member_pointer(Derives)); + static_assert(!__is_member_pointer(Enum)); + static_assert(!__is_member_pointer(IntArNB)); + static_assert(!__is_member_pointer(Union)); + static_assert(!__is_member_pointer(UnionAr)); + static_assert(!__is_member_pointer(StructWithMembers)); + static_assert(!__is_member_pointer(void (*)())); } void is_const() { - int t01[T(__is_const(cvoid))]; - int t02[T(__is_const(const char))]; - int t03[T(__is_const(const int))]; - int t04[T(__is_const(const long))]; - int t05[T(__is_const(const short))]; - int t06[T(__is_const(const signed char))]; - int t07[T(__is_const(const wchar_t))]; - int t08[T(__is_const(const bool))]; - int t09[T(__is_const(const float))]; - int t10[T(__is_const(const double))]; - int t11[T(__is_const(const long double))]; - int t12[T(__is_const(const unsigned char))]; - int t13[T(__is_const(const unsigned int))]; - int t14[T(__is_const(const unsigned long long))]; - int t15[T(__is_const(const unsigned long))]; - int t16[T(__is_const(const unsigned short))]; - int t17[T(__is_const(const void))]; - int t18[T(__is_const(const ClassType))]; - int t19[T(__is_const(const Derives))]; - int t20[T(__is_const(const Enum))]; - int t21[T(__is_const(const IntArNB))]; - int t22[T(__is_const(const Union))]; - int t23[T(__is_const(const UnionAr))]; - - int t30[F(__is_const(char))]; - int t31[F(__is_const(int))]; - int t32[F(__is_const(long))]; - int t33[F(__is_const(short))]; - int t34[F(__is_const(signed char))]; - int t35[F(__is_const(wchar_t))]; - int t36[F(__is_const(bool))]; - int t37[F(__is_const(float))]; - int t38[F(__is_const(double))]; - int t39[F(__is_const(long double))]; - int t40[F(__is_const(unsigned char))]; - int t41[F(__is_const(unsigned int))]; - int t42[F(__is_const(unsigned long long))]; - int t43[F(__is_const(unsigned long))]; - int t44[F(__is_const(unsigned short))]; - int t45[F(__is_const(void))]; - int t46[F(__is_const(ClassType))]; - int t47[F(__is_const(Derives))]; - int t48[F(__is_const(Enum))]; - int t49[F(__is_const(IntArNB))]; - int t50[F(__is_const(Union))]; - int t51[F(__is_const(UnionAr))]; + static_assert(__is_const(cvoid)); + static_assert(__is_const(const char)); + static_assert(__is_const(const int)); + static_assert(__is_const(const long)); + static_assert(__is_const(const short)); + static_assert(__is_const(const signed char)); + static_assert(__is_const(const wchar_t)); + static_assert(__is_const(const bool)); + static_assert(__is_const(const float)); + static_assert(__is_const(const double)); + static_assert(__is_const(const long double)); + static_assert(__is_const(const unsigned char)); + static_assert(__is_const(const unsigned int)); + static_assert(__is_const(const unsigned long long)); + static_assert(__is_const(const unsigned long)); + static_assert(__is_const(const unsigned short)); + static_assert(__is_const(const void)); + static_assert(__is_const(const ClassType)); + static_assert(__is_const(const Derives)); + static_assert(__is_const(const Enum)); + static_assert(__is_const(const IntArNB)); + static_assert(__is_const(const Union)); + static_assert(__is_const(const UnionAr)); + + static_assert(!__is_const(char)); + static_assert(!__is_const(int)); + static_assert(!__is_const(long)); + static_assert(!__is_const(short)); + static_assert(!__is_const(signed char)); + static_assert(!__is_const(wchar_t)); + static_assert(!__is_const(bool)); + static_assert(!__is_const(float)); + static_assert(!__is_const(double)); + static_assert(!__is_const(long double)); + static_assert(!__is_const(unsigned char)); + static_assert(!__is_const(unsigned int)); + static_assert(!__is_const(unsigned long long)); + static_assert(!__is_const(unsigned long)); + static_assert(!__is_const(unsigned short)); + static_assert(!__is_const(void)); + static_assert(!__is_const(ClassType)); + static_assert(!__is_const(Derives)); + static_assert(!__is_const(Enum)); + static_assert(!__is_const(IntArNB)); + static_assert(!__is_const(Union)); + static_assert(!__is_const(UnionAr)); } void is_volatile() { - int t02[T(__is_volatile(volatile char))]; - int t03[T(__is_volatile(volatile int))]; - int t04[T(__is_volatile(volatile long))]; - int t05[T(__is_volatile(volatile short))]; - int t06[T(__is_volatile(volatile signed char))]; - int t07[T(__is_volatile(volatile wchar_t))]; - int t08[T(__is_volatile(volatile bool))]; - int t09[T(__is_volatile(volatile float))]; - int t10[T(__is_volatile(volatile double))]; - int t11[T(__is_volatile(volatile long double))]; - int t12[T(__is_volatile(volatile unsigned char))]; - int t13[T(__is_volatile(volatile unsigned int))]; - int t14[T(__is_volatile(volatile unsigned long long))]; - int t15[T(__is_volatile(volatile unsigned long))]; - int t16[T(__is_volatile(volatile unsigned short))]; - int t17[T(__is_volatile(volatile void))]; - int t18[T(__is_volatile(volatile ClassType))]; - int t19[T(__is_volatile(volatile Derives))]; - int t20[T(__is_volatile(volatile Enum))]; - int t21[T(__is_volatile(volatile IntArNB))]; - int t22[T(__is_volatile(volatile Union))]; - int t23[T(__is_volatile(volatile UnionAr))]; - - int t30[F(__is_volatile(char))]; - int t31[F(__is_volatile(int))]; - int t32[F(__is_volatile(long))]; - int t33[F(__is_volatile(short))]; - int t34[F(__is_volatile(signed char))]; - int t35[F(__is_volatile(wchar_t))]; - int t36[F(__is_volatile(bool))]; - int t37[F(__is_volatile(float))]; - int t38[F(__is_volatile(double))]; - int t39[F(__is_volatile(long double))]; - int t40[F(__is_volatile(unsigned char))]; - int t41[F(__is_volatile(unsigned int))]; - int t42[F(__is_volatile(unsigned long long))]; - int t43[F(__is_volatile(unsigned long))]; - int t44[F(__is_volatile(unsigned short))]; - int t45[F(__is_volatile(void))]; - int t46[F(__is_volatile(ClassType))]; - int t47[F(__is_volatile(Derives))]; - int t48[F(__is_volatile(Enum))]; - int t49[F(__is_volatile(IntArNB))]; - int t50[F(__is_volatile(Union))]; - int t51[F(__is_volatile(UnionAr))]; + static_assert(__is_volatile(volatile char)); + static_assert(__is_volatile(volatile int)); + static_assert(__is_volatile(volatile long)); + static_assert(__is_volatile(volatile short)); + static_assert(__is_volatile(volatile signed char)); + static_assert(__is_volatile(volatile wchar_t)); + static_assert(__is_volatile(volatile bool)); + static_assert(__is_volatile(volatile float)); + static_assert(__is_volatile(volatile double)); + static_assert(__is_volatile(volatile long double)); + static_assert(__is_volatile(volatile unsigned char)); + static_assert(__is_volatile(volatile unsigned int)); + static_assert(__is_volatile(volatile unsigned long long)); + static_assert(__is_volatile(volatile unsigned long)); + static_assert(__is_volatile(volatile unsigned short)); + static_assert(__is_volatile(volatile void)); + static_assert(__is_volatile(volatile ClassType)); + static_assert(__is_volatile(volatile Derives)); + static_assert(__is_volatile(volatile Enum)); + static_assert(__is_volatile(volatile IntArNB)); + static_assert(__is_volatile(volatile Union)); + static_assert(__is_volatile(volatile UnionAr)); + + static_assert(!__is_volatile(char)); + static_assert(!__is_volatile(int)); + static_assert(!__is_volatile(long)); + static_assert(!__is_volatile(short)); + static_assert(!__is_volatile(signed char)); + static_assert(!__is_volatile(wchar_t)); + static_assert(!__is_volatile(bool)); + static_assert(!__is_volatile(float)); + static_assert(!__is_volatile(double)); + static_assert(!__is_volatile(long double)); + static_assert(!__is_volatile(unsigned char)); + static_assert(!__is_volatile(unsigned int)); + static_assert(!__is_volatile(unsigned long long)); + static_assert(!__is_volatile(unsigned long)); + static_assert(!__is_volatile(unsigned short)); + static_assert(!__is_volatile(void)); + static_assert(!__is_volatile(ClassType)); + static_assert(!__is_volatile(Derives)); + static_assert(!__is_volatile(Enum)); + static_assert(!__is_volatile(IntArNB)); + static_assert(!__is_volatile(Union)); + static_assert(!__is_volatile(UnionAr)); } struct TrivialStruct { @@ -1352,87 +1350,87 @@ ExtDefaulted::~ExtDefaulted() = default; void is_trivial2() { - int t01[T(__is_trivial(char))]; - int t02[T(__is_trivial(int))]; - int t03[T(__is_trivial(long))]; - int t04[T(__is_trivial(short))]; - int t05[T(__is_trivial(signed char))]; - int t06[T(__is_trivial(wchar_t))]; - int t07[T(__is_trivial(bool))]; - int t08[T(__is_trivial(float))]; - int t09[T(__is_trivial(double))]; - int t10[T(__is_trivial(long double))]; - int t11[T(__is_trivial(unsigned char))]; - int t12[T(__is_trivial(unsigned int))]; - int t13[T(__is_trivial(unsigned long long))]; - int t14[T(__is_trivial(unsigned long))]; - int t15[T(__is_trivial(unsigned short))]; - int t16[T(__is_trivial(ClassType))]; - int t17[T(__is_trivial(Derives))]; - int t18[T(__is_trivial(Enum))]; - int t19[T(__is_trivial(IntAr))]; - int t20[T(__is_trivial(Union))]; - int t21[T(__is_trivial(UnionAr))]; - int t22[T(__is_trivial(TrivialStruct))]; - int t23[T(__is_trivial(AllDefaulted))]; - int t24[T(__is_trivial(AllDeleted))]; - - int t30[F(__is_trivial(void))]; - int t31[F(__is_trivial(NonTrivialStruct))]; - int t32[F(__is_trivial(SuperNonTrivialStruct))]; - int t33[F(__is_trivial(NonTCStruct))]; - int t34[F(__is_trivial(ExtDefaulted))]; - - int t40[T(__is_trivial(ACompleteType))]; - int t41[F(__is_trivial(AnIncompleteType))]; // expected-error {{incomplete type}} - int t42[F(__is_trivial(AnIncompleteType[]))]; // expected-error {{incomplete type}} - int t43[F(__is_trivial(AnIncompleteType[1]))]; // expected-error {{incomplete type}} - int t44[F(__is_trivial(void))]; - int t45[F(__is_trivial(const volatile void))]; + static_assert(__is_trivial(char)); + static_assert(__is_trivial(int)); + static_assert(__is_trivial(long)); + static_assert(__is_trivial(short)); + static_assert(__is_trivial(signed char)); + static_assert(__is_trivial(wchar_t)); + static_assert(__is_trivial(bool)); + static_assert(__is_trivial(float)); + static_assert(__is_trivial(double)); + static_assert(__is_trivial(long double)); + static_assert(__is_trivial(unsigned char)); + static_assert(__is_trivial(unsigned int)); + static_assert(__is_trivial(unsigned long long)); + static_assert(__is_trivial(unsigned long)); + static_assert(__is_trivial(unsigned short)); + static_assert(__is_trivial(ClassType)); + static_assert(__is_trivial(Derives)); + static_assert(__is_trivial(Enum)); + static_assert(__is_trivial(IntAr)); + static_assert(__is_trivial(Union)); + static_assert(__is_trivial(UnionAr)); + static_assert(__is_trivial(TrivialStruct)); + static_assert(__is_trivial(AllDefaulted)); + static_assert(__is_trivial(AllDeleted)); + + static_assert(!__is_trivial(void)); + static_assert(!__is_trivial(NonTrivialStruct)); + static_assert(!__is_trivial(SuperNonTrivialStruct)); + static_assert(!__is_trivial(NonTCStruct)); + static_assert(!__is_trivial(ExtDefaulted)); + + static_assert(__is_trivial(ACompleteType)); + static_assert(!__is_trivial(AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_trivial(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__is_trivial(AnIncompleteType[1])); // expected-error {{incomplete type}} + static_assert(!__is_trivial(void)); + static_assert(!__is_trivial(const volatile void)); } void is_trivially_copyable2() { - int t01[T(__is_trivially_copyable(char))]; - int t02[T(__is_trivially_copyable(int))]; - int t03[T(__is_trivially_copyable(long))]; - int t04[T(__is_trivially_copyable(short))]; - int t05[T(__is_trivially_copyable(signed char))]; - int t06[T(__is_trivially_copyable(wchar_t))]; - int t07[T(__is_trivially_copyable(bool))]; - int t08[T(__is_trivially_copyable(float))]; - int t09[T(__is_trivially_copyable(double))]; - int t10[T(__is_trivially_copyable(long double))]; - int t11[T(__is_trivially_copyable(unsigned char))]; - int t12[T(__is_trivially_copyable(unsigned int))]; - int t13[T(__is_trivially_copyable(unsigned long long))]; - int t14[T(__is_trivially_copyable(unsigned long))]; - int t15[T(__is_trivially_copyable(unsigned short))]; - int t16[T(__is_trivially_copyable(ClassType))]; - int t17[T(__is_trivially_copyable(Derives))]; - int t18[T(__is_trivially_copyable(Enum))]; - int t19[T(__is_trivially_copyable(IntAr))]; - int t20[T(__is_trivially_copyable(Union))]; - int t21[T(__is_trivially_copyable(UnionAr))]; - int t22[T(__is_trivially_copyable(TrivialStruct))]; - int t23[T(__is_trivially_copyable(NonTrivialStruct))]; - int t24[T(__is_trivially_copyable(AllDefaulted))]; - int t25[T(__is_trivially_copyable(AllDeleted))]; - - int t30[F(__is_trivially_copyable(void))]; - int t31[F(__is_trivially_copyable(SuperNonTrivialStruct))]; - int t32[F(__is_trivially_copyable(NonTCStruct))]; - int t33[F(__is_trivially_copyable(ExtDefaulted))]; - - int t34[T(__is_trivially_copyable(const int))]; - int t35[T(__is_trivially_copyable(volatile int))]; - - int t40[T(__is_trivially_copyable(ACompleteType))]; - int t41[F(__is_trivially_copyable(AnIncompleteType))]; // expected-error {{incomplete type}} - int t42[F(__is_trivially_copyable(AnIncompleteType[]))]; // expected-error {{incomplete type}} - int t43[F(__is_trivially_copyable(AnIncompleteType[1]))]; // expected-error {{incomplete type}} - int t44[F(__is_trivially_copyable(void))]; - int t45[F(__is_trivially_copyable(const volatile void))]; + static_assert(__is_trivially_copyable(char)); + static_assert(__is_trivially_copyable(int)); + static_assert(__is_trivially_copyable(long)); + static_assert(__is_trivially_copyable(short)); + static_assert(__is_trivially_copyable(signed char)); + static_assert(__is_trivially_copyable(wchar_t)); + static_assert(__is_trivially_copyable(bool)); + static_assert(__is_trivially_copyable(float)); + static_assert(__is_trivially_copyable(double)); + static_assert(__is_trivially_copyable(long double)); + static_assert(__is_trivially_copyable(unsigned char)); + static_assert(__is_trivially_copyable(unsigned int)); + static_assert(__is_trivially_copyable(unsigned long long)); + static_assert(__is_trivially_copyable(unsigned long)); + static_assert(__is_trivially_copyable(unsigned short)); + static_assert(__is_trivially_copyable(ClassType)); + static_assert(__is_trivially_copyable(Derives)); + static_assert(__is_trivially_copyable(Enum)); + static_assert(__is_trivially_copyable(IntAr)); + static_assert(__is_trivially_copyable(Union)); + static_assert(__is_trivially_copyable(UnionAr)); + static_assert(__is_trivially_copyable(TrivialStruct)); + static_assert(__is_trivially_copyable(NonTrivialStruct)); + static_assert(__is_trivially_copyable(AllDefaulted)); + static_assert(__is_trivially_copyable(AllDeleted)); + + static_assert(!__is_trivially_copyable(void)); + static_assert(!__is_trivially_copyable(SuperNonTrivialStruct)); + static_assert(!__is_trivially_copyable(NonTCStruct)); + static_assert(!__is_trivially_copyable(ExtDefaulted)); + + static_assert(__is_trivially_copyable(const int)); + static_assert(__is_trivially_copyable(volatile int)); + + static_assert(__is_trivially_copyable(ACompleteType)); + static_assert(!__is_trivially_copyable(AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_trivially_copyable(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__is_trivially_copyable(AnIncompleteType[1])); // expected-error {{incomplete type}} + static_assert(!__is_trivially_copyable(void)); + static_assert(!__is_trivially_copyable(const volatile void)); } struct CStruct { @@ -1477,32 +1475,32 @@ void is_standard_layout() typedef ConstInt ConstIntAr[4]; typedef CppStructStandard CppStructStandardAr[4]; - int t01[T(__is_standard_layout(int))]; - int t02[T(__is_standard_layout(ConstInt))]; - int t03[T(__is_standard_layout(ConstIntAr))]; - int t04[T(__is_standard_layout(CStruct))]; - int t05[T(__is_standard_layout(CppStructStandard))]; - int t06[T(__is_standard_layout(CppStructStandardAr))]; - int t07[T(__is_standard_layout(Vector))]; - int t08[T(__is_standard_layout(VectorExt))]; + static_assert(__is_standard_layout(int)); + static_assert(__is_standard_layout(ConstInt)); + static_assert(__is_standard_layout(ConstIntAr)); + static_assert(__is_standard_layout(CStruct)); + static_assert(__is_standard_layout(CppStructStandard)); + static_assert(__is_standard_layout(CppStructStandardAr)); + static_assert(__is_standard_layout(Vector)); + static_assert(__is_standard_layout(VectorExt)); typedef CppStructNonStandardByBase CppStructNonStandardByBaseAr[4]; - int t10[F(__is_standard_layout(CppStructNonStandardByVirt))]; - int t11[F(__is_standard_layout(CppStructNonStandardByMemb))]; - int t12[F(__is_standard_layout(CppStructNonStandardByProt))]; - int t13[F(__is_standard_layout(CppStructNonStandardByVirtBase))]; - int t14[F(__is_standard_layout(CppStructNonStandardByBase))]; - int t15[F(__is_standard_layout(CppStructNonStandardByBaseAr))]; - int t16[F(__is_standard_layout(CppStructNonStandardBySameBase))]; - int t17[F(__is_standard_layout(CppStructNonStandardBy2ndVirtBase))]; - - int t40[T(__is_standard_layout(ACompleteType))]; - int t41[F(__is_standard_layout(AnIncompleteType))]; // expected-error {{incomplete type}} - int t42[F(__is_standard_layout(AnIncompleteType[]))]; // expected-error {{incomplete type}} - int t43[F(__is_standard_layout(AnIncompleteType[1]))]; // expected-error {{incomplete type}} - int t44[F(__is_standard_layout(void))]; - int t45[F(__is_standard_layout(const volatile void))]; + static_assert(!__is_standard_layout(CppStructNonStandardByVirt)); + static_assert(!__is_standard_layout(CppStructNonStandardByMemb)); + static_assert(!__is_standard_layout(CppStructNonStandardByProt)); + static_assert(!__is_standard_layout(CppStructNonStandardByVirtBase)); + static_assert(!__is_standard_layout(CppStructNonStandardByBase)); + static_assert(!__is_standard_layout(CppStructNonStandardByBaseAr)); + static_assert(!__is_standard_layout(CppStructNonStandardBySameBase)); + static_assert(!__is_standard_layout(CppStructNonStandardBy2ndVirtBase)); + + static_assert(__is_standard_layout(ACompleteType)); + static_assert(!__is_standard_layout(AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_standard_layout(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__is_standard_layout(AnIncompleteType[1])); // expected-error {{incomplete type}} + static_assert(!__is_standard_layout(void)); + static_assert(!__is_standard_layout(const volatile void)); struct HasAnonEmptyBitfield { int : 0; }; struct HasAnonBitfield { int : 4; }; @@ -1510,11 +1508,11 @@ void is_standard_layout() struct DerivesFromBitfieldWithBitfield : HasAnonBitfield { int : 5; }; struct DerivesFromBitfieldTwice : DerivesFromBitfield, HasAnonEmptyBitfield {}; - int t50[T(__is_standard_layout(HasAnonEmptyBitfield))]; - int t51[T(__is_standard_layout(HasAnonBitfield))]; - int t52[T(__is_standard_layout(DerivesFromBitfield))]; - int t53[F(__is_standard_layout(DerivesFromBitfieldWithBitfield))]; - int t54[F(__is_standard_layout(DerivesFromBitfieldTwice))]; + static_assert(__is_standard_layout(HasAnonEmptyBitfield)); + static_assert(__is_standard_layout(HasAnonBitfield)); + static_assert(__is_standard_layout(DerivesFromBitfield)); + static_assert(!__is_standard_layout(DerivesFromBitfieldWithBitfield)); + static_assert(!__is_standard_layout(DerivesFromBitfieldTwice)); struct Empty {}; struct HasEmptyBase : Empty {}; @@ -1528,16 +1526,16 @@ void is_standard_layout() struct HasEmptyIndirectBaseAsSecondMember : HasEmptyBase { int n; Empty e; }; struct HasEmptyIndirectBaseAfterBitfield : HasEmptyBase { int : 4; Empty e; }; - int t60[T(__is_standard_layout(Empty))]; - int t61[T(__is_standard_layout(HasEmptyBase))]; - int t62[F(__is_standard_layout(HasRepeatedEmptyBase))]; - int t63[F(__is_standard_layout(HasEmptyBaseAsMember))]; - int t64[F(__is_standard_layout(HasEmptyBaseAsSubobjectOfMember1))]; - int t65[T(__is_standard_layout(HasEmptyBaseAsSubobjectOfMember2))]; // FIXME: standard bug? - int t66[F(__is_standard_layout(HasEmptyBaseAsSubobjectOfMember3))]; - int t67[F(__is_standard_layout(HasEmptyIndirectBaseAsMember))]; - int t68[T(__is_standard_layout(HasEmptyIndirectBaseAsSecondMember))]; - int t69[F(__is_standard_layout(HasEmptyIndirectBaseAfterBitfield))]; // FIXME: standard bug? + static_assert(__is_standard_layout(Empty)); + static_assert(__is_standard_layout(HasEmptyBase)); + static_assert(!__is_standard_layout(HasRepeatedEmptyBase)); + static_assert(!__is_standard_layout(HasEmptyBaseAsMember)); + static_assert(!__is_standard_layout(HasEmptyBaseAsSubobjectOfMember1)); + static_assert(__is_standard_layout(HasEmptyBaseAsSubobjectOfMember2)); // FIXME: standard bug? + static_assert(!__is_standard_layout(HasEmptyBaseAsSubobjectOfMember3)); + static_assert(!__is_standard_layout(HasEmptyIndirectBaseAsMember)); + static_assert(__is_standard_layout(HasEmptyIndirectBaseAsSecondMember)); + static_assert(!__is_standard_layout(HasEmptyIndirectBaseAfterBitfield)); // FIXME: standard bug? struct StructWithEmptyFields { int n; @@ -1554,8 +1552,8 @@ void is_standard_layout() UnionWithEmptyFields u; }; - int t70[T(__is_standard_layout(HasEmptyIndirectBaseAsSecondStructMember))]; - int t71[F(__is_standard_layout(HasEmptyIndirectBaseAsSecondUnionMember))]; + static_assert(__is_standard_layout(HasEmptyIndirectBaseAsSecondStructMember)); + static_assert(!__is_standard_layout(HasEmptyIndirectBaseAsSecondUnionMember)); } struct CStruct2 { @@ -1813,62 +1811,62 @@ void is_layout_compatible(int n) void is_signed() { - //int t01[T(__is_signed(char))]; - int t02[T(__is_signed(int))]; - int t03[T(__is_signed(long))]; - int t04[T(__is_signed(short))]; - int t05[T(__is_signed(signed char))]; - int t06[T(__is_signed(wchar_t))]; - int t07[T(__is_signed(float))]; - int t08[T(__is_signed(double))]; - int t09[T(__is_signed(long double))]; - - int t13[F(__is_signed(bool))]; - int t14[F(__is_signed(cvoid))]; - int t15[F(__is_signed(unsigned char))]; - int t16[F(__is_signed(unsigned int))]; - int t17[F(__is_signed(unsigned long long))]; - int t18[F(__is_signed(unsigned long))]; - int t19[F(__is_signed(unsigned short))]; - int t20[F(__is_signed(void))]; - int t21[F(__is_signed(ClassType))]; - int t22[F(__is_signed(Derives))]; - int t23[F(__is_signed(Enum))]; - int t24[F(__is_signed(SignedEnum))]; - int t25[F(__is_signed(IntArNB))]; - int t26[F(__is_signed(Union))]; - int t27[F(__is_signed(UnionAr))]; - int t28[F(__is_signed(UnsignedEnum))]; + //static_assert(__is_signed(char)); + static_assert(__is_signed(int)); + static_assert(__is_signed(long)); + static_assert(__is_signed(short)); + static_assert(__is_signed(signed char)); + static_assert(__is_signed(wchar_t)); + static_assert(__is_signed(float)); + static_assert(__is_signed(double)); + static_assert(__is_signed(long double)); + + static_assert(!__is_signed(bool)); + static_assert(!__is_signed(cvoid)); + static_assert(!__is_signed(unsigned char)); + static_assert(!__is_signed(unsigned int)); + static_assert(!__is_signed(unsigned long long)); + static_assert(!__is_signed(unsigned long)); + static_assert(!__is_signed(unsigned short)); + static_assert(!__is_signed(void)); + static_assert(!__is_signed(ClassType)); + static_assert(!__is_signed(Derives)); + static_assert(!__is_signed(Enum)); + static_assert(!__is_signed(SignedEnum)); + static_assert(!__is_signed(IntArNB)); + static_assert(!__is_signed(Union)); + static_assert(!__is_signed(UnionAr)); + static_assert(!__is_signed(UnsignedEnum)); } void is_unsigned() { - int t01[T(__is_unsigned(bool))]; - int t02[T(__is_unsigned(unsigned char))]; - int t03[T(__is_unsigned(unsigned short))]; - int t04[T(__is_unsigned(unsigned int))]; - int t05[T(__is_unsigned(unsigned long))]; - int t06[T(__is_unsigned(unsigned long long))]; - - int t10[F(__is_unsigned(void))]; - int t11[F(__is_unsigned(cvoid))]; - int t12[F(__is_unsigned(float))]; - int t13[F(__is_unsigned(double))]; - int t14[F(__is_unsigned(long double))]; - int t16[F(__is_unsigned(char))]; - int t17[F(__is_unsigned(signed char))]; - int t18[F(__is_unsigned(wchar_t))]; - int t19[F(__is_unsigned(short))]; - int t20[F(__is_unsigned(int))]; - int t21[F(__is_unsigned(long))]; - int t22[F(__is_unsigned(Union))]; - int t23[F(__is_unsigned(UnionAr))]; - int t24[F(__is_unsigned(Derives))]; - int t25[F(__is_unsigned(ClassType))]; - int t26[F(__is_unsigned(IntArNB))]; - int t27[F(__is_unsigned(Enum))]; - int t28[F(__is_unsigned(UnsignedEnum))]; - int t29[F(__is_unsigned(SignedEnum))]; + static_assert(__is_unsigned(bool)); + static_assert(__is_unsigned(unsigned char)); + static_assert(__is_unsigned(unsigned short)); + static_assert(__is_unsigned(unsigned int)); + static_assert(__is_unsigned(unsigned long)); + static_assert(__is_unsigned(unsigned long long)); + + static_assert(!__is_unsigned(void)); + static_assert(!__is_unsigned(cvoid)); + static_assert(!__is_unsigned(float)); + static_assert(!__is_unsigned(double)); + static_assert(!__is_unsigned(long double)); + static_assert(!__is_unsigned(char)); + static_assert(!__is_unsigned(signed char)); + static_assert(!__is_unsigned(wchar_t)); + static_assert(!__is_unsigned(short)); + static_assert(!__is_unsigned(int)); + static_assert(!__is_unsigned(long)); + static_assert(!__is_unsigned(Union)); + static_assert(!__is_unsigned(UnionAr)); + static_assert(!__is_unsigned(Derives)); + static_assert(!__is_unsigned(ClassType)); + static_assert(!__is_unsigned(IntArNB)); + static_assert(!__is_unsigned(Enum)); + static_assert(!__is_unsigned(UnsignedEnum)); + static_assert(!__is_unsigned(SignedEnum)); } typedef Int& IntRef; @@ -1891,35 +1889,35 @@ struct HasTemplateCons { }; void has_trivial_default_constructor() { - { int arr[T(__has_trivial_constructor(Int))]; } - { int arr[T(__has_trivial_constructor(IntAr))]; } - { int arr[T(__has_trivial_constructor(Union))]; } - { int arr[T(__has_trivial_constructor(UnionAr))]; } - { int arr[T(__has_trivial_constructor(POD))]; } - { int arr[T(__has_trivial_constructor(Derives))]; } - { int arr[T(__has_trivial_constructor(DerivesAr))]; } - { int arr[T(__has_trivial_constructor(ConstIntAr))]; } - { int arr[T(__has_trivial_constructor(ConstIntArAr))]; } - { int arr[T(__has_trivial_constructor(HasDest))]; } - { int arr[T(__has_trivial_constructor(HasPriv))]; } - { int arr[T(__has_trivial_constructor(HasCopyAssign))]; } - { int arr[T(__has_trivial_constructor(HasMoveAssign))]; } - { int arr[T(__has_trivial_constructor(const Int))]; } - { int arr[T(__has_trivial_constructor(AllDefaulted))]; } - { int arr[T(__has_trivial_constructor(AllDeleted))]; } - { int arr[T(__has_trivial_constructor(ACompleteType[]))]; } - - { int arr[F(__has_trivial_constructor(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[F(__has_trivial_constructor(HasCons))]; } - { int arr[F(__has_trivial_constructor(HasRef))]; } - { int arr[F(__has_trivial_constructor(HasCopy))]; } - { int arr[F(__has_trivial_constructor(IntRef))]; } - { int arr[F(__has_trivial_constructor(VirtAr))]; } - { int arr[F(__has_trivial_constructor(void))]; } - { int arr[F(__has_trivial_constructor(cvoid))]; } - { int arr[F(__has_trivial_constructor(HasTemplateCons))]; } - { int arr[F(__has_trivial_constructor(AllPrivate))]; } - { int arr[F(__has_trivial_constructor(ExtDefaulted))]; } + static_assert(__has_trivial_constructor(Int)); + static_assert(__has_trivial_constructor(IntAr)); + static_assert(__has_trivial_constructor(Union)); + static_assert(__has_trivial_constructor(UnionAr)); + static_assert(__has_trivial_constructor(POD)); + static_assert(__has_trivial_constructor(Derives)); + static_assert(__has_trivial_constructor(DerivesAr)); + static_assert(__has_trivial_constructor(ConstIntAr)); + static_assert(__has_trivial_constructor(ConstIntArAr)); + static_assert(__has_trivial_constructor(HasDest)); + static_assert(__has_trivial_constructor(HasPriv)); + static_assert(__has_trivial_constructor(HasCopyAssign)); + static_assert(__has_trivial_constructor(HasMoveAssign)); + static_assert(__has_trivial_constructor(const Int)); + static_assert(__has_trivial_constructor(AllDefaulted)); + static_assert(__has_trivial_constructor(AllDeleted)); + static_assert(__has_trivial_constructor(ACompleteType[])); + + static_assert(!__has_trivial_constructor(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__has_trivial_constructor(HasCons)); + static_assert(!__has_trivial_constructor(HasRef)); + static_assert(!__has_trivial_constructor(HasCopy)); + static_assert(!__has_trivial_constructor(IntRef)); + static_assert(!__has_trivial_constructor(VirtAr)); + static_assert(!__has_trivial_constructor(void)); + static_assert(!__has_trivial_constructor(cvoid)); + static_assert(!__has_trivial_constructor(HasTemplateCons)); + static_assert(!__has_trivial_constructor(AllPrivate)); + static_assert(!__has_trivial_constructor(ExtDefaulted)); } void has_trivial_move_constructor() { @@ -1935,127 +1933,127 @@ void has_trivial_move_constructor() { // type (or array thereof), the constructor selected // to copy/move that member is trivial; // otherwise the copy/move constructor is non-trivial. - { int arr[T(__has_trivial_move_constructor(POD))]; } - { int arr[T(__has_trivial_move_constructor(Union))]; } - { int arr[T(__has_trivial_move_constructor(HasCons))]; } - { int arr[T(__has_trivial_move_constructor(HasStaticMemberMoveCtor))]; } - { int arr[T(__has_trivial_move_constructor(AllDeleted))]; } - { int arr[T(__has_trivial_move_constructor(ACompleteType[]))]; } - - { int arr[F(__has_trivial_move_constructor(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[F(__has_trivial_move_constructor(HasVirt))]; } - { int arr[F(__has_trivial_move_constructor(DerivesVirt))]; } - { int arr[F(__has_trivial_move_constructor(HasMoveCtor))]; } - { int arr[F(__has_trivial_move_constructor(DerivesHasMoveCtor))]; } - { int arr[F(__has_trivial_move_constructor(HasMemberMoveCtor))]; } + static_assert(__has_trivial_move_constructor(POD)); + static_assert(__has_trivial_move_constructor(Union)); + static_assert(__has_trivial_move_constructor(HasCons)); + static_assert(__has_trivial_move_constructor(HasStaticMemberMoveCtor)); + static_assert(__has_trivial_move_constructor(AllDeleted)); + static_assert(__has_trivial_move_constructor(ACompleteType[])); + + static_assert(!__has_trivial_move_constructor(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__has_trivial_move_constructor(HasVirt)); + static_assert(!__has_trivial_move_constructor(DerivesVirt)); + static_assert(!__has_trivial_move_constructor(HasMoveCtor)); + static_assert(!__has_trivial_move_constructor(DerivesHasMoveCtor)); + static_assert(!__has_trivial_move_constructor(HasMemberMoveCtor)); } void has_trivial_copy_constructor() { - { int arr[T(__has_trivial_copy(Int))]; } - { int arr[T(__has_trivial_copy(IntAr))]; } - { int arr[T(__has_trivial_copy(Union))]; } - { int arr[T(__has_trivial_copy(UnionAr))]; } - { int arr[T(__has_trivial_copy(POD))]; } - { int arr[T(__has_trivial_copy(Derives))]; } - { int arr[T(__has_trivial_copy(ConstIntAr))]; } - { int arr[T(__has_trivial_copy(ConstIntArAr))]; } - { int arr[T(__has_trivial_copy(HasDest))]; } - { int arr[T(__has_trivial_copy(HasPriv))]; } - { int arr[T(__has_trivial_copy(HasCons))]; } - { int arr[T(__has_trivial_copy(HasRef))]; } - { int arr[T(__has_trivial_copy(HasMove))]; } - { int arr[T(__has_trivial_copy(IntRef))]; } - { int arr[T(__has_trivial_copy(HasCopyAssign))]; } - { int arr[T(__has_trivial_copy(HasMoveAssign))]; } - { int arr[T(__has_trivial_copy(const Int))]; } - { int arr[T(__has_trivial_copy(AllDefaulted))]; } - { int arr[T(__has_trivial_copy(AllDeleted))]; } - { int arr[T(__has_trivial_copy(DerivesAr))]; } - { int arr[T(__has_trivial_copy(DerivesHasRef))]; } - { int arr[T(__has_trivial_copy(ACompleteType[]))]; } - - { int arr[F(__has_trivial_copy(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[F(__has_trivial_copy(HasCopy))]; } - { int arr[F(__has_trivial_copy(HasTemplateCons))]; } - { int arr[F(__has_trivial_copy(VirtAr))]; } - { int arr[F(__has_trivial_copy(void))]; } - { int arr[F(__has_trivial_copy(cvoid))]; } - { int arr[F(__has_trivial_copy(AllPrivate))]; } - { int arr[F(__has_trivial_copy(ExtDefaulted))]; } + static_assert(__has_trivial_copy(Int)); + static_assert(__has_trivial_copy(IntAr)); + static_assert(__has_trivial_copy(Union)); + static_assert(__has_trivial_copy(UnionAr)); + static_assert(__has_trivial_copy(POD)); + static_assert(__has_trivial_copy(Derives)); + static_assert(__has_trivial_copy(ConstIntAr)); + static_assert(__has_trivial_copy(ConstIntArAr)); + static_assert(__has_trivial_copy(HasDest)); + static_assert(__has_trivial_copy(HasPriv)); + static_assert(__has_trivial_copy(HasCons)); + static_assert(__has_trivial_copy(HasRef)); + static_assert(__has_trivial_copy(HasMove)); + static_assert(__has_trivial_copy(IntRef)); + static_assert(__has_trivial_copy(HasCopyAssign)); + static_assert(__has_trivial_copy(HasMoveAssign)); + static_assert(__has_trivial_copy(const Int)); + static_assert(__has_trivial_copy(AllDefaulted)); + static_assert(__has_trivial_copy(AllDeleted)); + static_assert(__has_trivial_copy(DerivesAr)); + static_assert(__has_trivial_copy(DerivesHasRef)); + static_assert(__has_trivial_copy(ACompleteType[])); + + static_assert(!__has_trivial_copy(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__has_trivial_copy(HasCopy)); + static_assert(!__has_trivial_copy(HasTemplateCons)); + static_assert(!__has_trivial_copy(VirtAr)); + static_assert(!__has_trivial_copy(void)); + static_assert(!__has_trivial_copy(cvoid)); + static_assert(!__has_trivial_copy(AllPrivate)); + static_assert(!__has_trivial_copy(ExtDefaulted)); } void has_trivial_copy_assignment() { - { int arr[T(__has_trivial_assign(Int))]; } - { int arr[T(__has_trivial_assign(IntAr))]; } - { int arr[T(__has_trivial_assign(Union))]; } - { int arr[T(__has_trivial_assign(UnionAr))]; } - { int arr[T(__has_trivial_assign(POD))]; } - { int arr[T(__has_trivial_assign(Derives))]; } - { int arr[T(__has_trivial_assign(HasDest))]; } - { int arr[T(__has_trivial_assign(HasPriv))]; } - { int arr[T(__has_trivial_assign(HasCons))]; } - { int arr[T(__has_trivial_assign(HasRef))]; } - { int arr[T(__has_trivial_assign(HasCopy))]; } - { int arr[T(__has_trivial_assign(HasMove))]; } - { int arr[T(__has_trivial_assign(HasMoveAssign))]; } - { int arr[T(__has_trivial_assign(AllDefaulted))]; } - { int arr[T(__has_trivial_assign(AllDeleted))]; } - { int arr[T(__has_trivial_assign(DerivesAr))]; } - { int arr[T(__has_trivial_assign(DerivesHasRef))]; } - { int arr[T(__has_trivial_assign(ACompleteType[]))]; } - - { int arr[F(__has_trivial_assign(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[F(__has_trivial_assign(IntRef))]; } - { int arr[F(__has_trivial_assign(HasCopyAssign))]; } - { int arr[F(__has_trivial_assign(const Int))]; } - { int arr[F(__has_trivial_assign(ConstIntAr))]; } - { int arr[F(__has_trivial_assign(ConstIntArAr))]; } - { int arr[F(__has_trivial_assign(VirtAr))]; } - { int arr[F(__has_trivial_assign(void))]; } - { int arr[F(__has_trivial_assign(cvoid))]; } - { int arr[F(__has_trivial_assign(AllPrivate))]; } - { int arr[F(__has_trivial_assign(ExtDefaulted))]; } + static_assert(__has_trivial_assign(Int)); + static_assert(__has_trivial_assign(IntAr)); + static_assert(__has_trivial_assign(Union)); + static_assert(__has_trivial_assign(UnionAr)); + static_assert(__has_trivial_assign(POD)); + static_assert(__has_trivial_assign(Derives)); + static_assert(__has_trivial_assign(HasDest)); + static_assert(__has_trivial_assign(HasPriv)); + static_assert(__has_trivial_assign(HasCons)); + static_assert(__has_trivial_assign(HasRef)); + static_assert(__has_trivial_assign(HasCopy)); + static_assert(__has_trivial_assign(HasMove)); + static_assert(__has_trivial_assign(HasMoveAssign)); + static_assert(__has_trivial_assign(AllDefaulted)); + static_assert(__has_trivial_assign(AllDeleted)); + static_assert(__has_trivial_assign(DerivesAr)); + static_assert(__has_trivial_assign(DerivesHasRef)); + static_assert(__has_trivial_assign(ACompleteType[])); + + static_assert(!__has_trivial_assign(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__has_trivial_assign(IntRef)); + static_assert(!__has_trivial_assign(HasCopyAssign)); + static_assert(!__has_trivial_assign(const Int)); + static_assert(!__has_trivial_assign(ConstIntAr)); + static_assert(!__has_trivial_assign(ConstIntArAr)); + static_assert(!__has_trivial_assign(VirtAr)); + static_assert(!__has_trivial_assign(void)); + static_assert(!__has_trivial_assign(cvoid)); + static_assert(!__has_trivial_assign(AllPrivate)); + static_assert(!__has_trivial_assign(ExtDefaulted)); } void has_trivial_destructor() { - { int arr[T(__has_trivial_destructor(Int))]; } - { int arr[T(__has_trivial_destructor(IntAr))]; } - { int arr[T(__has_trivial_destructor(Union))]; } - { int arr[T(__has_trivial_destructor(UnionAr))]; } - { int arr[T(__has_trivial_destructor(POD))]; } - { int arr[T(__has_trivial_destructor(Derives))]; } - { int arr[T(__has_trivial_destructor(ConstIntAr))]; } - { int arr[T(__has_trivial_destructor(ConstIntArAr))]; } - { int arr[T(__has_trivial_destructor(HasPriv))]; } - { int arr[T(__has_trivial_destructor(HasCons))]; } - { int arr[T(__has_trivial_destructor(HasRef))]; } - { int arr[T(__has_trivial_destructor(HasCopy))]; } - { int arr[T(__has_trivial_destructor(HasMove))]; } - { int arr[T(__has_trivial_destructor(IntRef))]; } - { int arr[T(__has_trivial_destructor(HasCopyAssign))]; } - { int arr[T(__has_trivial_destructor(HasMoveAssign))]; } - { int arr[T(__has_trivial_destructor(const Int))]; } - { int arr[T(__has_trivial_destructor(DerivesAr))]; } - { int arr[T(__has_trivial_destructor(VirtAr))]; } - { int arr[T(__has_trivial_destructor(AllDefaulted))]; } - { int arr[T(__has_trivial_destructor(AllDeleted))]; } - { int arr[T(__has_trivial_destructor(DerivesHasRef))]; } - { int arr[T(__has_trivial_destructor(ACompleteType[]))]; } - - { int arr[F(__has_trivial_destructor(HasDest))]; } - { int arr[F(__has_trivial_destructor(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[F(__has_trivial_destructor(void))]; } - { int arr[F(__has_trivial_destructor(cvoid))]; } - { int arr[F(__has_trivial_destructor(AllPrivate))]; } - { int arr[F(__has_trivial_destructor(ExtDefaulted))]; } + static_assert(__has_trivial_destructor(Int)); + static_assert(__has_trivial_destructor(IntAr)); + static_assert(__has_trivial_destructor(Union)); + static_assert(__has_trivial_destructor(UnionAr)); + static_assert(__has_trivial_destructor(POD)); + static_assert(__has_trivial_destructor(Derives)); + static_assert(__has_trivial_destructor(ConstIntAr)); + static_assert(__has_trivial_destructor(ConstIntArAr)); + static_assert(__has_trivial_destructor(HasPriv)); + static_assert(__has_trivial_destructor(HasCons)); + static_assert(__has_trivial_destructor(HasRef)); + static_assert(__has_trivial_destructor(HasCopy)); + static_assert(__has_trivial_destructor(HasMove)); + static_assert(__has_trivial_destructor(IntRef)); + static_assert(__has_trivial_destructor(HasCopyAssign)); + static_assert(__has_trivial_destructor(HasMoveAssign)); + static_assert(__has_trivial_destructor(const Int)); + static_assert(__has_trivial_destructor(DerivesAr)); + static_assert(__has_trivial_destructor(VirtAr)); + static_assert(__has_trivial_destructor(AllDefaulted)); + static_assert(__has_trivial_destructor(AllDeleted)); + static_assert(__has_trivial_destructor(DerivesHasRef)); + static_assert(__has_trivial_destructor(ACompleteType[])); + + static_assert(!__has_trivial_destructor(HasDest)); + static_assert(!__has_trivial_destructor(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__has_trivial_destructor(void)); + static_assert(!__has_trivial_destructor(cvoid)); + static_assert(!__has_trivial_destructor(AllPrivate)); + static_assert(!__has_trivial_destructor(ExtDefaulted)); } struct A { ~A() {} }; template struct B : A { }; void f() { - { int arr[F(__has_trivial_destructor(A))]; } - { int arr[F(__has_trivial_destructor(B))]; } + static_assert(!__has_trivial_destructor(A)); + static_assert(!__has_trivial_destructor(B)); } class PR11110 { @@ -2076,69 +2074,69 @@ public: }; void has_nothrow_assign() { - { int arr[T(__has_nothrow_assign(Int))]; } - { int arr[T(__has_nothrow_assign(IntAr))]; } - { int arr[T(__has_nothrow_assign(Union))]; } - { int arr[T(__has_nothrow_assign(UnionAr))]; } - { int arr[T(__has_nothrow_assign(POD))]; } - { int arr[T(__has_nothrow_assign(Derives))]; } - { int arr[T(__has_nothrow_assign(HasDest))]; } - { int arr[T(__has_nothrow_assign(HasPriv))]; } - { int arr[T(__has_nothrow_assign(HasCons))]; } - { int arr[T(__has_nothrow_assign(HasRef))]; } - { int arr[T(__has_nothrow_assign(HasCopy))]; } - { int arr[T(__has_nothrow_assign(HasMove))]; } - { int arr[T(__has_nothrow_assign(HasMoveAssign))]; } - { int arr[T(__has_nothrow_assign(HasNoThrowCopyAssign))]; } - { int arr[T(__has_nothrow_assign(HasMultipleNoThrowCopyAssign))]; } - { int arr[T(__has_nothrow_assign(HasVirtDest))]; } - { int arr[T(__has_nothrow_assign(AllPrivate))]; } - { int arr[T(__has_nothrow_assign(UsingAssign))]; } - { int arr[T(__has_nothrow_assign(DerivesAr))]; } - { int arr[T(__has_nothrow_assign(ACompleteType[]))]; } - - { int arr[F(__has_nothrow_assign(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[F(__has_nothrow_assign(IntRef))]; } - { int arr[F(__has_nothrow_assign(HasCopyAssign))]; } - { int arr[F(__has_nothrow_assign(HasMultipleCopyAssign))]; } - { int arr[F(__has_nothrow_assign(const Int))]; } - { int arr[F(__has_nothrow_assign(ConstIntAr))]; } - { int arr[F(__has_nothrow_assign(ConstIntArAr))]; } - { int arr[F(__has_nothrow_assign(VirtAr))]; } - { int arr[F(__has_nothrow_assign(void))]; } - { int arr[F(__has_nothrow_assign(cvoid))]; } - { int arr[F(__has_nothrow_assign(PR11110))]; } + static_assert(__has_nothrow_assign(Int)); + static_assert(__has_nothrow_assign(IntAr)); + static_assert(__has_nothrow_assign(Union)); + static_assert(__has_nothrow_assign(UnionAr)); + static_assert(__has_nothrow_assign(POD)); + static_assert(__has_nothrow_assign(Derives)); + static_assert(__has_nothrow_assign(HasDest)); + static_assert(__has_nothrow_assign(HasPriv)); + static_assert(__has_nothrow_assign(HasCons)); + static_assert(__has_nothrow_assign(HasRef)); + static_assert(__has_nothrow_assign(HasCopy)); + static_assert(__has_nothrow_assign(HasMove)); + static_assert(__has_nothrow_assign(HasMoveAssign)); + static_assert(__has_nothrow_assign(HasNoThrowCopyAssign)); + static_assert(__has_nothrow_assign(HasMultipleNoThrowCopyAssign)); + static_assert(__has_nothrow_assign(HasVirtDest)); + static_assert(__has_nothrow_assign(AllPrivate)); + static_assert(__has_nothrow_assign(UsingAssign)); + static_assert(__has_nothrow_assign(DerivesAr)); + static_assert(__has_nothrow_assign(ACompleteType[])); + + static_assert(!__has_nothrow_assign(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__has_nothrow_assign(IntRef)); + static_assert(!__has_nothrow_assign(HasCopyAssign)); + static_assert(!__has_nothrow_assign(HasMultipleCopyAssign)); + static_assert(!__has_nothrow_assign(const Int)); + static_assert(!__has_nothrow_assign(ConstIntAr)); + static_assert(!__has_nothrow_assign(ConstIntArAr)); + static_assert(!__has_nothrow_assign(VirtAr)); + static_assert(!__has_nothrow_assign(void)); + static_assert(!__has_nothrow_assign(cvoid)); + static_assert(!__has_nothrow_assign(PR11110)); } void has_nothrow_move_assign() { - { int arr[T(__has_nothrow_move_assign(Int))]; } - { int arr[T(__has_nothrow_move_assign(Enum))]; } - { int arr[T(__has_nothrow_move_assign(Int*))]; } - { int arr[T(__has_nothrow_move_assign(Enum POD::*))]; } - { int arr[T(__has_nothrow_move_assign(POD))]; } - { int arr[T(__has_nothrow_move_assign(HasPriv))]; } - { int arr[T(__has_nothrow_move_assign(HasNoThrowMoveAssign))]; } - { int arr[T(__has_nothrow_move_assign(HasNoExceptNoThrowMoveAssign))]; } - { int arr[T(__has_nothrow_move_assign(HasMemberNoThrowMoveAssign))]; } - { int arr[T(__has_nothrow_move_assign(HasMemberNoExceptNoThrowMoveAssign))]; } - { int arr[T(__has_nothrow_move_assign(AllDeleted))]; } - { int arr[T(__has_nothrow_move_assign(ACompleteType[]))]; } - - { int arr[F(__has_nothrow_move_assign(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[F(__has_nothrow_move_assign(HasThrowMoveAssign))]; } - { int arr[F(__has_nothrow_move_assign(HasNoExceptFalseMoveAssign))]; } - { int arr[F(__has_nothrow_move_assign(HasMemberThrowMoveAssign))]; } - { int arr[F(__has_nothrow_move_assign(HasMemberNoExceptFalseMoveAssign))]; } - { int arr[F(__has_nothrow_move_assign(NoDefaultMoveAssignDueToUDCopyCtor))]; } - { int arr[F(__has_nothrow_move_assign(NoDefaultMoveAssignDueToUDCopyAssign))]; } - { int arr[F(__has_nothrow_move_assign(NoDefaultMoveAssignDueToDtor))]; } - - - { int arr[T(__is_nothrow_assignable(HasNoThrowMoveAssign, HasNoThrowMoveAssign))]; } - { int arr[F(__is_nothrow_assignable(HasThrowMoveAssign, HasThrowMoveAssign))]; } - - { int arr[T(__is_assignable(HasNoThrowMoveAssign, HasNoThrowMoveAssign))]; } - { int arr[T(__is_assignable(HasThrowMoveAssign, HasThrowMoveAssign))]; } + static_assert(__has_nothrow_move_assign(Int)); + static_assert(__has_nothrow_move_assign(Enum)); + static_assert(__has_nothrow_move_assign(Int*)); + static_assert(__has_nothrow_move_assign(Enum POD::*)); + static_assert(__has_nothrow_move_assign(POD)); + static_assert(__has_nothrow_move_assign(HasPriv)); + static_assert(__has_nothrow_move_assign(HasNoThrowMoveAssign)); + static_assert(__has_nothrow_move_assign(HasNoExceptNoThrowMoveAssign)); + static_assert(__has_nothrow_move_assign(HasMemberNoThrowMoveAssign)); + static_assert(__has_nothrow_move_assign(HasMemberNoExceptNoThrowMoveAssign)); + static_assert(__has_nothrow_move_assign(AllDeleted)); + static_assert(__has_nothrow_move_assign(ACompleteType[])); + + static_assert(!__has_nothrow_move_assign(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__has_nothrow_move_assign(HasThrowMoveAssign)); + static_assert(!__has_nothrow_move_assign(HasNoExceptFalseMoveAssign)); + static_assert(!__has_nothrow_move_assign(HasMemberThrowMoveAssign)); + static_assert(!__has_nothrow_move_assign(HasMemberNoExceptFalseMoveAssign)); + static_assert(!__has_nothrow_move_assign(NoDefaultMoveAssignDueToUDCopyCtor)); + static_assert(!__has_nothrow_move_assign(NoDefaultMoveAssignDueToUDCopyAssign)); + static_assert(!__has_nothrow_move_assign(NoDefaultMoveAssignDueToDtor)); + + + static_assert(__is_nothrow_assignable(HasNoThrowMoveAssign, HasNoThrowMoveAssign)); + static_assert(!__is_nothrow_assignable(HasThrowMoveAssign, HasThrowMoveAssign)); + + static_assert(__is_assignable(HasNoThrowMoveAssign, HasNoThrowMoveAssign)); + static_assert(__is_assignable(HasThrowMoveAssign, HasThrowMoveAssign)); } void has_trivial_move_assign() { @@ -2153,120 +2151,120 @@ void has_trivial_move_assign() { // - for each non-static data member of X that is of class type // (or array thereof), the assignment operator // selected to copy/move that member is trivial; - { int arr[T(__has_trivial_move_assign(Int))]; } - { int arr[T(__has_trivial_move_assign(HasStaticMemberMoveAssign))]; } - { int arr[T(__has_trivial_move_assign(AllDeleted))]; } - { int arr[T(__has_trivial_move_assign(ACompleteType[]))]; } - - { int arr[F(__has_trivial_move_assign(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[F(__has_trivial_move_assign(HasVirt))]; } - { int arr[F(__has_trivial_move_assign(DerivesVirt))]; } - { int arr[F(__has_trivial_move_assign(HasMoveAssign))]; } - { int arr[F(__has_trivial_move_assign(DerivesHasMoveAssign))]; } - { int arr[F(__has_trivial_move_assign(HasMemberMoveAssign))]; } - { int arr[F(__has_nothrow_move_assign(NoDefaultMoveAssignDueToUDCopyCtor))]; } - { int arr[F(__has_nothrow_move_assign(NoDefaultMoveAssignDueToUDCopyAssign))]; } + static_assert(__has_trivial_move_assign(Int)); + static_assert(__has_trivial_move_assign(HasStaticMemberMoveAssign)); + static_assert(__has_trivial_move_assign(AllDeleted)); + static_assert(__has_trivial_move_assign(ACompleteType[])); + + static_assert(!__has_trivial_move_assign(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__has_trivial_move_assign(HasVirt)); + static_assert(!__has_trivial_move_assign(DerivesVirt)); + static_assert(!__has_trivial_move_assign(HasMoveAssign)); + static_assert(!__has_trivial_move_assign(DerivesHasMoveAssign)); + static_assert(!__has_trivial_move_assign(HasMemberMoveAssign)); + static_assert(!__has_nothrow_move_assign(NoDefaultMoveAssignDueToUDCopyCtor)); + static_assert(!__has_nothrow_move_assign(NoDefaultMoveAssignDueToUDCopyAssign)); } void has_nothrow_copy() { - { int arr[T(__has_nothrow_copy(Int))]; } - { int arr[T(__has_nothrow_copy(IntAr))]; } - { int arr[T(__has_nothrow_copy(Union))]; } - { int arr[T(__has_nothrow_copy(UnionAr))]; } - { int arr[T(__has_nothrow_copy(POD))]; } - { int arr[T(__has_nothrow_copy(const Int))]; } - { int arr[T(__has_nothrow_copy(ConstIntAr))]; } - { int arr[T(__has_nothrow_copy(ConstIntArAr))]; } - { int arr[T(__has_nothrow_copy(Derives))]; } - { int arr[T(__has_nothrow_copy(IntRef))]; } - { int arr[T(__has_nothrow_copy(HasDest))]; } - { int arr[T(__has_nothrow_copy(HasPriv))]; } - { int arr[T(__has_nothrow_copy(HasCons))]; } - { int arr[T(__has_nothrow_copy(HasRef))]; } - { int arr[T(__has_nothrow_copy(HasMove))]; } - { int arr[T(__has_nothrow_copy(HasCopyAssign))]; } - { int arr[T(__has_nothrow_copy(HasMoveAssign))]; } - { int arr[T(__has_nothrow_copy(HasNoThrowCopy))]; } - { int arr[T(__has_nothrow_copy(HasMultipleNoThrowCopy))]; } - { int arr[T(__has_nothrow_copy(HasVirtDest))]; } - { int arr[T(__has_nothrow_copy(HasTemplateCons))]; } - { int arr[T(__has_nothrow_copy(AllPrivate))]; } - { int arr[T(__has_nothrow_copy(DerivesAr))]; } - { int arr[T(__has_nothrow_copy(ACompleteType[]))]; } - - { int arr[F(__has_nothrow_copy(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[F(__has_nothrow_copy(HasCopy))]; } - { int arr[F(__has_nothrow_copy(HasMultipleCopy))]; } - { int arr[F(__has_nothrow_copy(VirtAr))]; } - { int arr[F(__has_nothrow_copy(void))]; } - { int arr[F(__has_nothrow_copy(cvoid))]; } + static_assert(__has_nothrow_copy(Int)); + static_assert(__has_nothrow_copy(IntAr)); + static_assert(__has_nothrow_copy(Union)); + static_assert(__has_nothrow_copy(UnionAr)); + static_assert(__has_nothrow_copy(POD)); + static_assert(__has_nothrow_copy(const Int)); + static_assert(__has_nothrow_copy(ConstIntAr)); + static_assert(__has_nothrow_copy(ConstIntArAr)); + static_assert(__has_nothrow_copy(Derives)); + static_assert(__has_nothrow_copy(IntRef)); + static_assert(__has_nothrow_copy(HasDest)); + static_assert(__has_nothrow_copy(HasPriv)); + static_assert(__has_nothrow_copy(HasCons)); + static_assert(__has_nothrow_copy(HasRef)); + static_assert(__has_nothrow_copy(HasMove)); + static_assert(__has_nothrow_copy(HasCopyAssign)); + static_assert(__has_nothrow_copy(HasMoveAssign)); + static_assert(__has_nothrow_copy(HasNoThrowCopy)); + static_assert(__has_nothrow_copy(HasMultipleNoThrowCopy)); + static_assert(__has_nothrow_copy(HasVirtDest)); + static_assert(__has_nothrow_copy(HasTemplateCons)); + static_assert(__has_nothrow_copy(AllPrivate)); + static_assert(__has_nothrow_copy(DerivesAr)); + static_assert(__has_nothrow_copy(ACompleteType[])); + + static_assert(!__has_nothrow_copy(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__has_nothrow_copy(HasCopy)); + static_assert(!__has_nothrow_copy(HasMultipleCopy)); + static_assert(!__has_nothrow_copy(VirtAr)); + static_assert(!__has_nothrow_copy(void)); + static_assert(!__has_nothrow_copy(cvoid)); } void has_nothrow_constructor() { - { int arr[T(__has_nothrow_constructor(Int))]; } - { int arr[T(__has_nothrow_constructor(IntAr))]; } - { int arr[T(__has_nothrow_constructor(Union))]; } - { int arr[T(__has_nothrow_constructor(UnionAr))]; } - { int arr[T(__has_nothrow_constructor(POD))]; } - { int arr[T(__has_nothrow_constructor(Derives))]; } - { int arr[T(__has_nothrow_constructor(DerivesAr))]; } - { int arr[T(__has_nothrow_constructor(ConstIntAr))]; } - { int arr[T(__has_nothrow_constructor(ConstIntArAr))]; } - { int arr[T(__has_nothrow_constructor(HasDest))]; } - { int arr[T(__has_nothrow_constructor(HasPriv))]; } - { int arr[T(__has_nothrow_constructor(HasCopyAssign))]; } - { int arr[T(__has_nothrow_constructor(const Int))]; } - { int arr[T(__has_nothrow_constructor(HasNoThrowConstructor))]; } - { int arr[T(__has_nothrow_constructor(HasVirtDest))]; } - // { int arr[T(__has_nothrow_constructor(VirtAr))]; } // not implemented - { int arr[T(__has_nothrow_constructor(AllPrivate))]; } - { int arr[T(__has_nothrow_constructor(ACompleteType[]))]; } - - { int arr[F(__has_nothrow_constructor(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[F(__has_nothrow_constructor(HasCons))]; } - { int arr[F(__has_nothrow_constructor(HasRef))]; } - { int arr[F(__has_nothrow_constructor(HasCopy))]; } - { int arr[F(__has_nothrow_constructor(HasMove))]; } - { int arr[F(__has_nothrow_constructor(HasNoThrowConstructorWithArgs))]; } - { int arr[F(__has_nothrow_constructor(IntRef))]; } - { int arr[F(__has_nothrow_constructor(void))]; } - { int arr[F(__has_nothrow_constructor(cvoid))]; } - { int arr[F(__has_nothrow_constructor(HasTemplateCons))]; } - - { int arr[F(__has_nothrow_constructor(HasMultipleDefaultConstructor1))]; } - { int arr[F(__has_nothrow_constructor(HasMultipleDefaultConstructor2))]; } + static_assert(__has_nothrow_constructor(Int)); + static_assert(__has_nothrow_constructor(IntAr)); + static_assert(__has_nothrow_constructor(Union)); + static_assert(__has_nothrow_constructor(UnionAr)); + static_assert(__has_nothrow_constructor(POD)); + static_assert(__has_nothrow_constructor(Derives)); + static_assert(__has_nothrow_constructor(DerivesAr)); + static_assert(__has_nothrow_constructor(ConstIntAr)); + static_assert(__has_nothrow_constructor(ConstIntArAr)); + static_assert(__has_nothrow_constructor(HasDest)); + static_assert(__has_nothrow_constructor(HasPriv)); + static_assert(__has_nothrow_constructor(HasCopyAssign)); + static_assert(__has_nothrow_constructor(const Int)); + static_assert(__has_nothrow_constructor(HasNoThrowConstructor)); + static_assert(__has_nothrow_constructor(HasVirtDest)); + // static_assert(__has_nothrow_constructor(VirtAr)); // not implemented + static_assert(__has_nothrow_constructor(AllPrivate)); + static_assert(__has_nothrow_constructor(ACompleteType[])); + + static_assert(!__has_nothrow_constructor(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__has_nothrow_constructor(HasCons)); + static_assert(!__has_nothrow_constructor(HasRef)); + static_assert(!__has_nothrow_constructor(HasCopy)); + static_assert(!__has_nothrow_constructor(HasMove)); + static_assert(!__has_nothrow_constructor(HasNoThrowConstructorWithArgs)); + static_assert(!__has_nothrow_constructor(IntRef)); + static_assert(!__has_nothrow_constructor(void)); + static_assert(!__has_nothrow_constructor(cvoid)); + static_assert(!__has_nothrow_constructor(HasTemplateCons)); + + static_assert(!__has_nothrow_constructor(HasMultipleDefaultConstructor1)); + static_assert(!__has_nothrow_constructor(HasMultipleDefaultConstructor2)); } void has_virtual_destructor() { - { int arr[F(__has_virtual_destructor(Int))]; } - { int arr[F(__has_virtual_destructor(IntAr))]; } - { int arr[F(__has_virtual_destructor(Union))]; } - { int arr[F(__has_virtual_destructor(UnionAr))]; } - { int arr[F(__has_virtual_destructor(POD))]; } - { int arr[F(__has_virtual_destructor(Derives))]; } - { int arr[F(__has_virtual_destructor(DerivesAr))]; } - { int arr[F(__has_virtual_destructor(const Int))]; } - { int arr[F(__has_virtual_destructor(ConstIntAr))]; } - { int arr[F(__has_virtual_destructor(ConstIntArAr))]; } - { int arr[F(__has_virtual_destructor(HasDest))]; } - { int arr[F(__has_virtual_destructor(HasPriv))]; } - { int arr[F(__has_virtual_destructor(HasCons))]; } - { int arr[F(__has_virtual_destructor(HasRef))]; } - { int arr[F(__has_virtual_destructor(HasCopy))]; } - { int arr[F(__has_virtual_destructor(HasMove))]; } - { int arr[F(__has_virtual_destructor(HasCopyAssign))]; } - { int arr[F(__has_virtual_destructor(HasMoveAssign))]; } - { int arr[F(__has_virtual_destructor(IntRef))]; } - { int arr[F(__has_virtual_destructor(VirtAr))]; } - { int arr[F(__has_virtual_destructor(ACompleteType[]))]; } - - { int arr[F(__has_virtual_destructor(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[T(__has_virtual_destructor(HasVirtDest))]; } - { int arr[T(__has_virtual_destructor(DerivedVirtDest))]; } - { int arr[F(__has_virtual_destructor(VirtDestAr))]; } - { int arr[F(__has_virtual_destructor(void))]; } - { int arr[F(__has_virtual_destructor(cvoid))]; } - { int arr[F(__has_virtual_destructor(AllPrivate))]; } + static_assert(!__has_virtual_destructor(Int)); + static_assert(!__has_virtual_destructor(IntAr)); + static_assert(!__has_virtual_destructor(Union)); + static_assert(!__has_virtual_destructor(UnionAr)); + static_assert(!__has_virtual_destructor(POD)); + static_assert(!__has_virtual_destructor(Derives)); + static_assert(!__has_virtual_destructor(DerivesAr)); + static_assert(!__has_virtual_destructor(const Int)); + static_assert(!__has_virtual_destructor(ConstIntAr)); + static_assert(!__has_virtual_destructor(ConstIntArAr)); + static_assert(!__has_virtual_destructor(HasDest)); + static_assert(!__has_virtual_destructor(HasPriv)); + static_assert(!__has_virtual_destructor(HasCons)); + static_assert(!__has_virtual_destructor(HasRef)); + static_assert(!__has_virtual_destructor(HasCopy)); + static_assert(!__has_virtual_destructor(HasMove)); + static_assert(!__has_virtual_destructor(HasCopyAssign)); + static_assert(!__has_virtual_destructor(HasMoveAssign)); + static_assert(!__has_virtual_destructor(IntRef)); + static_assert(!__has_virtual_destructor(VirtAr)); + static_assert(!__has_virtual_destructor(ACompleteType[])); + + static_assert(!__has_virtual_destructor(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(__has_virtual_destructor(HasVirtDest)); + static_assert(__has_virtual_destructor(DerivedVirtDest)); + static_assert(!__has_virtual_destructor(VirtDestAr)); + static_assert(!__has_virtual_destructor(void)); + static_assert(!__has_virtual_destructor(cvoid)); + static_assert(!__has_virtual_destructor(AllPrivate)); } @@ -2282,66 +2280,57 @@ template struct CrazyDerived : T { }; class class_forward; // expected-note 2 {{forward declaration of 'class_forward'}} -template -void isBaseOfT() { - int t[T(__is_base_of(Base, Derived))]; -}; -template -void isBaseOfF() { - int t[F(__is_base_of(Base, Derived))]; -}; - template class DerivedTemp : Base {}; template class NonderivedTemp {}; template class UndefinedTemp; // expected-note {{declared here}} void is_base_of() { - { int arr[T(__is_base_of(Base, Derived))]; } - { int arr[T(__is_base_of(const Base, Derived))]; } - { int arr[F(__is_base_of(Derived, Base))]; } - { int arr[F(__is_base_of(Derived, int))]; } - { int arr[T(__is_base_of(Base, Base))]; } - { int arr[T(__is_base_of(Base, Derived3))]; } - { int arr[T(__is_base_of(Derived, Derived3))]; } - { int arr[T(__is_base_of(Derived2b, Derived3))]; } - { int arr[T(__is_base_of(Derived2a, Derived3))]; } - { int arr[T(__is_base_of(BaseA, DerivedB))]; } - { int arr[F(__is_base_of(DerivedB, BaseA))]; } - { int arr[T(__is_base_of(Base, CrazyDerived))]; } - { int arr[F(__is_base_of(Union, Union))]; } - { int arr[T(__is_base_of(Empty, Empty))]; } - { int arr[T(__is_base_of(class_forward, class_forward))]; } - { int arr[F(__is_base_of(Empty, class_forward))]; } // expected-error {{incomplete type 'class_forward' used in type trait expression}} - { int arr[F(__is_base_of(Base&, Derived&))]; } - int t18[F(__is_base_of(Base[10], Derived[10]))]; - { int arr[F(__is_base_of(int, int))]; } - { int arr[F(__is_base_of(long, int))]; } - { int arr[T(__is_base_of(Base, DerivedTemp))]; } - { int arr[F(__is_base_of(Base, NonderivedTemp))]; } - { int arr[F(__is_base_of(Base, UndefinedTemp))]; } // expected-error {{implicit instantiation of undefined template 'UndefinedTemp'}} - - { int arr[F(__is_base_of(IncompleteUnion, IncompleteUnion))]; } - { int arr[F(__is_base_of(Union, IncompleteUnion))]; } - { int arr[F(__is_base_of(IncompleteUnion, Union))]; } - { int arr[F(__is_base_of(IncompleteStruct, IncompleteUnion))]; } - { int arr[F(__is_base_of(IncompleteUnion, IncompleteStruct))]; } - { int arr[F(__is_base_of(Empty, IncompleteUnion))]; } - { int arr[F(__is_base_of(IncompleteUnion, Empty))]; } - { int arr[F(__is_base_of(int, IncompleteUnion))]; } - { int arr[F(__is_base_of(IncompleteUnion, int))]; } - { int arr[F(__is_base_of(Empty, Union))]; } - { int arr[F(__is_base_of(Union, Empty))]; } - { int arr[F(__is_base_of(int, Empty))]; } - { int arr[F(__is_base_of(Union, int))]; } - - isBaseOfT(); - isBaseOfF(); - - isBaseOfT >(); - isBaseOfF, Base>(); - - isBaseOfT, DerivedB >(); - isBaseOfF, BaseA >(); + static_assert(__is_base_of(Base, Derived)); + static_assert(__is_base_of(const Base, Derived)); + static_assert(!__is_base_of(Derived, Base)); + static_assert(!__is_base_of(Derived, int)); + static_assert(__is_base_of(Base, Base)); + static_assert(__is_base_of(Base, Derived3)); + static_assert(__is_base_of(Derived, Derived3)); + static_assert(__is_base_of(Derived2b, Derived3)); + static_assert(__is_base_of(Derived2a, Derived3)); + static_assert(__is_base_of(BaseA, DerivedB)); + static_assert(!__is_base_of(DerivedB, BaseA)); + static_assert(__is_base_of(Base, CrazyDerived)); + static_assert(!__is_base_of(Union, Union)); + static_assert(__is_base_of(Empty, Empty)); + static_assert(__is_base_of(class_forward, class_forward)); + static_assert(!__is_base_of(Empty, class_forward)); // expected-error {{incomplete type 'class_forward' used in type trait expression}} + static_assert(!__is_base_of(Base&, Derived&)); + static_assert(!__is_base_of(Base[10], Derived[10])); + static_assert(!__is_base_of(int, int)); + static_assert(!__is_base_of(long, int)); + static_assert(__is_base_of(Base, DerivedTemp)); + static_assert(!__is_base_of(Base, NonderivedTemp)); + static_assert(!__is_base_of(Base, UndefinedTemp)); // expected-error {{implicit instantiation of undefined template 'UndefinedTemp'}} + + static_assert(!__is_base_of(IncompleteUnion, IncompleteUnion)); + static_assert(!__is_base_of(Union, IncompleteUnion)); + static_assert(!__is_base_of(IncompleteUnion, Union)); + static_assert(!__is_base_of(IncompleteStruct, IncompleteUnion)); + static_assert(!__is_base_of(IncompleteUnion, IncompleteStruct)); + static_assert(!__is_base_of(Empty, IncompleteUnion)); + static_assert(!__is_base_of(IncompleteUnion, Empty)); + static_assert(!__is_base_of(int, IncompleteUnion)); + static_assert(!__is_base_of(IncompleteUnion, int)); + static_assert(!__is_base_of(Empty, Union)); + static_assert(!__is_base_of(Union, Empty)); + static_assert(!__is_base_of(int, Empty)); + static_assert(!__is_base_of(Union, int)); + + static_assert(__is_base_of(Base, Derived)); + static_assert(!__is_base_of(Derived, Base)); + + static_assert(__is_base_of(Base, CrazyDerived)); + static_assert(!__is_base_of(CrazyDerived, Base)); + + static_assert(__is_base_of(BaseA, DerivedB)); + static_assert(!__is_base_of(DerivedB, BaseA)); } template @@ -2354,17 +2343,17 @@ typedef class Base BaseTypedef; void is_same() { - int t01[T(__is_same(Base, Base))]; - int t02[T(__is_same(Base, BaseTypedef))]; - int t03[T(__is_same(TemplateClass, TemplateAlias))]; + static_assert(__is_same(Base, Base)); + static_assert(__is_same(Base, BaseTypedef)); + static_assert(__is_same(TemplateClass, TemplateAlias)); - int t10[F(__is_same(Base, const Base))]; - int t11[F(__is_same(Base, Base&))]; - int t12[F(__is_same(Base, Derived))]; + static_assert(!__is_same(Base, const Base)); + static_assert(!__is_same(Base, Base&)); + static_assert(!__is_same(Base, Derived)); // __is_same_as is a GCC compatibility synonym for __is_same. - int t20[T(__is_same_as(int, int))]; - int t21[F(__is_same_as(int, float))]; + static_assert(__is_same_as(int, int)); + static_assert(!__is_same_as(int, float)); } struct IntWrapper @@ -2392,26 +2381,26 @@ struct FloatWrapper void is_convertible() { - int t01[T(__is_convertible(IntWrapper, IntWrapper))]; - int t02[T(__is_convertible(IntWrapper, const IntWrapper))]; - int t03[T(__is_convertible(IntWrapper, int))]; - int t04[T(__is_convertible(int, IntWrapper))]; - int t05[T(__is_convertible(IntWrapper, FloatWrapper))]; - int t06[T(__is_convertible(FloatWrapper, IntWrapper))]; - int t07[T(__is_convertible(FloatWrapper, float))]; - int t08[T(__is_convertible(float, FloatWrapper))]; + static_assert(__is_convertible(IntWrapper, IntWrapper)); + static_assert(__is_convertible(IntWrapper, const IntWrapper)); + static_assert(__is_convertible(IntWrapper, int)); + static_assert(__is_convertible(int, IntWrapper)); + static_assert(__is_convertible(IntWrapper, FloatWrapper)); + static_assert(__is_convertible(FloatWrapper, IntWrapper)); + static_assert(__is_convertible(FloatWrapper, float)); + static_assert(__is_convertible(float, FloatWrapper)); } void is_nothrow_convertible() { - int t01[T(__is_nothrow_convertible(IntWrapper, IntWrapper))]; - int t02[T(__is_nothrow_convertible(IntWrapper, const IntWrapper))]; - int t03[T(__is_nothrow_convertible(IntWrapper, int))]; - int t04[F(__is_nothrow_convertible(int, IntWrapper))]; - int t05[F(__is_nothrow_convertible(IntWrapper, FloatWrapper))]; - int t06[F(__is_nothrow_convertible(FloatWrapper, IntWrapper))]; - int t07[F(__is_nothrow_convertible(FloatWrapper, float))]; - int t08[T(__is_nothrow_convertible(float, FloatWrapper))]; + static_assert(__is_nothrow_convertible(IntWrapper, IntWrapper)); + static_assert(__is_nothrow_convertible(IntWrapper, const IntWrapper)); + static_assert(__is_nothrow_convertible(IntWrapper, int)); + static_assert(!__is_nothrow_convertible(int, IntWrapper)); + static_assert(!__is_nothrow_convertible(IntWrapper, FloatWrapper)); + static_assert(!__is_nothrow_convertible(FloatWrapper, IntWrapper)); + static_assert(!__is_nothrow_convertible(FloatWrapper, float)); + static_assert(__is_nothrow_convertible(float, FloatWrapper)); } struct FromInt { FromInt(int); }; @@ -2432,30 +2421,30 @@ struct X0 { struct Abstract { virtual void f() = 0; }; void is_convertible_to() { - { int arr[T(__is_convertible_to(Int, Int))]; } - { int arr[F(__is_convertible_to(Int, IntAr))]; } - { int arr[F(__is_convertible_to(IntAr, IntAr))]; } - { int arr[T(__is_convertible_to(void, void))]; } - { int arr[T(__is_convertible_to(cvoid, void))]; } - { int arr[T(__is_convertible_to(void, cvoid))]; } - { int arr[T(__is_convertible_to(cvoid, cvoid))]; } - { int arr[T(__is_convertible_to(int, FromInt))]; } - { int arr[T(__is_convertible_to(long, FromInt))]; } - { int arr[T(__is_convertible_to(double, FromInt))]; } - { int arr[T(__is_convertible_to(const int, FromInt))]; } - { int arr[T(__is_convertible_to(const int&, FromInt))]; } - { int arr[T(__is_convertible_to(ToInt, int))]; } - { int arr[T(__is_convertible_to(ToInt, const int&))]; } - { int arr[T(__is_convertible_to(ToInt, long))]; } - { int arr[F(__is_convertible_to(ToInt, int&))]; } - { int arr[F(__is_convertible_to(ToInt, FromInt))]; } - { int arr[T(__is_convertible_to(IntAr&, IntAr&))]; } - { int arr[T(__is_convertible_to(IntAr&, const IntAr&))]; } - { int arr[F(__is_convertible_to(const IntAr&, IntAr&))]; } - { int arr[F(__is_convertible_to(Function, Function))]; } - { int arr[F(__is_convertible_to(PrivateCopy, PrivateCopy))]; } - { int arr[T(__is_convertible_to(X0, X0))]; } - { int arr[F(__is_convertible_to(Abstract, Abstract))]; } + static_assert(__is_convertible_to(Int, Int)); + static_assert(!__is_convertible_to(Int, IntAr)); + static_assert(!__is_convertible_to(IntAr, IntAr)); + static_assert(__is_convertible_to(void, void)); + static_assert(__is_convertible_to(cvoid, void)); + static_assert(__is_convertible_to(void, cvoid)); + static_assert(__is_convertible_to(cvoid, cvoid)); + static_assert(__is_convertible_to(int, FromInt)); + static_assert(__is_convertible_to(long, FromInt)); + static_assert(__is_convertible_to(double, FromInt)); + static_assert(__is_convertible_to(const int, FromInt)); + static_assert(__is_convertible_to(const int&, FromInt)); + static_assert(__is_convertible_to(ToInt, int)); + static_assert(__is_convertible_to(ToInt, const int&)); + static_assert(__is_convertible_to(ToInt, long)); + static_assert(!__is_convertible_to(ToInt, int&)); + static_assert(!__is_convertible_to(ToInt, FromInt)); + static_assert(__is_convertible_to(IntAr&, IntAr&)); + static_assert(__is_convertible_to(IntAr&, const IntAr&)); + static_assert(!__is_convertible_to(const IntAr&, IntAr&)); + static_assert(!__is_convertible_to(Function, Function)); + static_assert(!__is_convertible_to(PrivateCopy, PrivateCopy)); + static_assert(__is_convertible_to(X0, X0)); + static_assert(!__is_convertible_to(Abstract, Abstract)); } namespace is_convertible_to_instantiate { @@ -2466,269 +2455,269 @@ namespace is_convertible_to_instantiate { void is_trivial() { - { int arr[T(__is_trivial(int))]; } - { int arr[T(__is_trivial(Enum))]; } - { int arr[T(__is_trivial(POD))]; } - { int arr[T(__is_trivial(Int))]; } - { int arr[T(__is_trivial(IntAr))]; } - { int arr[T(__is_trivial(IntArNB))]; } - { int arr[T(__is_trivial(Statics))]; } - { int arr[T(__is_trivial(Empty))]; } - { int arr[T(__is_trivial(EmptyUnion))]; } - { int arr[T(__is_trivial(Union))]; } - { int arr[T(__is_trivial(Derives))]; } - { int arr[T(__is_trivial(DerivesAr))]; } - { int arr[T(__is_trivial(DerivesArNB))]; } - { int arr[T(__is_trivial(DerivesEmpty))]; } - { int arr[T(__is_trivial(HasFunc))]; } - { int arr[T(__is_trivial(HasOp))]; } - { int arr[T(__is_trivial(HasConv))]; } - { int arr[T(__is_trivial(HasAssign))]; } - { int arr[T(__is_trivial(HasAnonymousUnion))]; } - { int arr[T(__is_trivial(HasPriv))]; } - { int arr[T(__is_trivial(HasProt))]; } - { int arr[T(__is_trivial(DerivesHasPriv))]; } - { int arr[T(__is_trivial(DerivesHasProt))]; } - { int arr[T(__is_trivial(Vector))]; } - { int arr[T(__is_trivial(VectorExt))]; } - - { int arr[F(__is_trivial(HasCons))]; } - { int arr[F(__is_trivial(HasCopyAssign))]; } - { int arr[F(__is_trivial(HasMoveAssign))]; } - { int arr[F(__is_trivial(HasDest))]; } - { int arr[F(__is_trivial(HasRef))]; } - { int arr[F(__is_trivial(HasNonPOD))]; } - { int arr[F(__is_trivial(HasVirt))]; } - { int arr[F(__is_trivial(DerivesHasCons))]; } - { int arr[F(__is_trivial(DerivesHasCopyAssign))]; } - { int arr[F(__is_trivial(DerivesHasMoveAssign))]; } - { int arr[F(__is_trivial(DerivesHasDest))]; } - { int arr[F(__is_trivial(DerivesHasRef))]; } - { int arr[F(__is_trivial(DerivesHasVirt))]; } - { int arr[F(__is_trivial(void))]; } - { int arr[F(__is_trivial(cvoid))]; } + static_assert(__is_trivial(int)); + static_assert(__is_trivial(Enum)); + static_assert(__is_trivial(POD)); + static_assert(__is_trivial(Int)); + static_assert(__is_trivial(IntAr)); + static_assert(__is_trivial(IntArNB)); + static_assert(__is_trivial(Statics)); + static_assert(__is_trivial(Empty)); + static_assert(__is_trivial(EmptyUnion)); + static_assert(__is_trivial(Union)); + static_assert(__is_trivial(Derives)); + static_assert(__is_trivial(DerivesAr)); + static_assert(__is_trivial(DerivesArNB)); + static_assert(__is_trivial(DerivesEmpty)); + static_assert(__is_trivial(HasFunc)); + static_assert(__is_trivial(HasOp)); + static_assert(__is_trivial(HasConv)); + static_assert(__is_trivial(HasAssign)); + static_assert(__is_trivial(HasAnonymousUnion)); + static_assert(__is_trivial(HasPriv)); + static_assert(__is_trivial(HasProt)); + static_assert(__is_trivial(DerivesHasPriv)); + static_assert(__is_trivial(DerivesHasProt)); + static_assert(__is_trivial(Vector)); + static_assert(__is_trivial(VectorExt)); + + static_assert(!__is_trivial(HasCons)); + static_assert(!__is_trivial(HasCopyAssign)); + static_assert(!__is_trivial(HasMoveAssign)); + static_assert(!__is_trivial(HasDest)); + static_assert(!__is_trivial(HasRef)); + static_assert(!__is_trivial(HasNonPOD)); + static_assert(!__is_trivial(HasVirt)); + static_assert(!__is_trivial(DerivesHasCons)); + static_assert(!__is_trivial(DerivesHasCopyAssign)); + static_assert(!__is_trivial(DerivesHasMoveAssign)); + static_assert(!__is_trivial(DerivesHasDest)); + static_assert(!__is_trivial(DerivesHasRef)); + static_assert(!__is_trivial(DerivesHasVirt)); + static_assert(!__is_trivial(void)); + static_assert(!__is_trivial(cvoid)); } template struct TriviallyConstructibleTemplate {}; void trivial_checks() { - { int arr[T(__is_trivially_copyable(int))]; } - { int arr[T(__is_trivially_copyable(Enum))]; } - { int arr[T(__is_trivially_copyable(POD))]; } - { int arr[T(__is_trivially_copyable(Int))]; } - { int arr[T(__is_trivially_copyable(IntAr))]; } - { int arr[T(__is_trivially_copyable(IntArNB))]; } - { int arr[T(__is_trivially_copyable(Statics))]; } - { int arr[T(__is_trivially_copyable(Empty))]; } - { int arr[T(__is_trivially_copyable(EmptyUnion))]; } - { int arr[T(__is_trivially_copyable(Union))]; } - { int arr[T(__is_trivially_copyable(Derives))]; } - { int arr[T(__is_trivially_copyable(DerivesAr))]; } - { int arr[T(__is_trivially_copyable(DerivesArNB))]; } - { int arr[T(__is_trivially_copyable(DerivesEmpty))]; } - { int arr[T(__is_trivially_copyable(HasFunc))]; } - { int arr[T(__is_trivially_copyable(HasOp))]; } - { int arr[T(__is_trivially_copyable(HasConv))]; } - { int arr[T(__is_trivially_copyable(HasAssign))]; } - { int arr[T(__is_trivially_copyable(HasAnonymousUnion))]; } - { int arr[T(__is_trivially_copyable(HasPriv))]; } - { int arr[T(__is_trivially_copyable(HasProt))]; } - { int arr[T(__is_trivially_copyable(DerivesHasPriv))]; } - { int arr[T(__is_trivially_copyable(DerivesHasProt))]; } - { int arr[T(__is_trivially_copyable(Vector))]; } - { int arr[T(__is_trivially_copyable(VectorExt))]; } - { int arr[T(__is_trivially_copyable(HasCons))]; } - { int arr[T(__is_trivially_copyable(HasRef))]; } - { int arr[T(__is_trivially_copyable(HasNonPOD))]; } - { int arr[T(__is_trivially_copyable(DerivesHasCons))]; } - { int arr[T(__is_trivially_copyable(DerivesHasRef))]; } - { int arr[T(__is_trivially_copyable(NonTrivialDefault))]; } - { int arr[T(__is_trivially_copyable(NonTrivialDefault[]))]; } - { int arr[T(__is_trivially_copyable(NonTrivialDefault[3]))]; } - - { int arr[F(__is_trivially_copyable(HasCopyAssign))]; } - { int arr[F(__is_trivially_copyable(HasMoveAssign))]; } - { int arr[F(__is_trivially_copyable(HasDest))]; } - { int arr[F(__is_trivially_copyable(HasVirt))]; } - { int arr[F(__is_trivially_copyable(DerivesHasCopyAssign))]; } - { int arr[F(__is_trivially_copyable(DerivesHasMoveAssign))]; } - { int arr[F(__is_trivially_copyable(DerivesHasDest))]; } - { int arr[F(__is_trivially_copyable(DerivesHasVirt))]; } - { int arr[F(__is_trivially_copyable(void))]; } - { int arr[F(__is_trivially_copyable(cvoid))]; } - - { int arr[T((__is_trivially_constructible(int)))]; } - { int arr[T((__is_trivially_constructible(int, int)))]; } - { int arr[T((__is_trivially_constructible(int, float)))]; } - { int arr[T((__is_trivially_constructible(int, int&)))]; } - { int arr[T((__is_trivially_constructible(int, const int&)))]; } - { int arr[T((__is_trivially_constructible(int, int)))]; } - { int arr[T((__is_trivially_constructible(HasCopyAssign, HasCopyAssign)))]; } - { int arr[T((__is_trivially_constructible(HasCopyAssign, const HasCopyAssign&)))]; } - { int arr[T((__is_trivially_constructible(HasCopyAssign, HasCopyAssign&&)))]; } - { int arr[T((__is_trivially_constructible(HasCopyAssign)))]; } - { int arr[T((__is_trivially_constructible(NonTrivialDefault, - const NonTrivialDefault&)))]; } - { int arr[T((__is_trivially_constructible(NonTrivialDefault, - NonTrivialDefault&&)))]; } - { int arr[T((__is_trivially_constructible(AllDefaulted)))]; } - { int arr[T((__is_trivially_constructible(AllDefaulted, - const AllDefaulted &)))]; } - { int arr[T((__is_trivially_constructible(AllDefaulted, - AllDefaulted &&)))]; } - - { int arr[F((__is_trivially_constructible(int, int*)))]; } - { int arr[F((__is_trivially_constructible(NonTrivialDefault)))]; } - { int arr[F((__is_trivially_constructible(ThreeArgCtor, int*, char*, int&)))]; } - { int arr[F((__is_trivially_constructible(AllDeleted)))]; } - { int arr[F((__is_trivially_constructible(AllDeleted, - const AllDeleted &)))]; } - { int arr[F((__is_trivially_constructible(AllDeleted, - AllDeleted &&)))]; } - { int arr[F((__is_trivially_constructible(ExtDefaulted)))]; } - { int arr[F((__is_trivially_constructible(ExtDefaulted, - const ExtDefaulted &)))]; } - { int arr[F((__is_trivially_constructible(ExtDefaulted, - ExtDefaulted &&)))]; } - - { int arr[T((__is_trivially_constructible(TriviallyConstructibleTemplate)))]; } - { int arr[F((__is_trivially_constructible(class_forward)))]; } // expected-error {{incomplete type 'class_forward' used in type trait expression}} - { int arr[F((__is_trivially_constructible(class_forward[])))]; } - { int arr[F((__is_trivially_constructible(void)))]; } - - { int arr[T((__is_trivially_assignable(int&, int)))]; } - { int arr[T((__is_trivially_assignable(int&, int&)))]; } - { int arr[T((__is_trivially_assignable(int&, int&&)))]; } - { int arr[T((__is_trivially_assignable(int&, const int&)))]; } - { int arr[T((__is_trivially_assignable(POD&, POD)))]; } - { int arr[T((__is_trivially_assignable(POD&, POD&)))]; } - { int arr[T((__is_trivially_assignable(POD&, POD&&)))]; } - { int arr[T((__is_trivially_assignable(POD&, const POD&)))]; } - { int arr[T((__is_trivially_assignable(int*&, int*)))]; } - { int arr[T((__is_trivially_assignable(AllDefaulted, - const AllDefaulted &)))]; } - { int arr[T((__is_trivially_assignable(AllDefaulted, - AllDefaulted &&)))]; } - - { int arr[F((__is_trivially_assignable(int*&, float*)))]; } - { int arr[F((__is_trivially_assignable(HasCopyAssign&, HasCopyAssign)))]; } - { int arr[F((__is_trivially_assignable(HasCopyAssign&, HasCopyAssign&)))]; } - { int arr[F((__is_trivially_assignable(HasCopyAssign&, const HasCopyAssign&)))]; } - { int arr[F((__is_trivially_assignable(HasCopyAssign&, HasCopyAssign&&)))]; } - { int arr[F((__is_trivially_assignable(TrivialMoveButNotCopy&, - TrivialMoveButNotCopy&)))]; } - { int arr[F((__is_trivially_assignable(TrivialMoveButNotCopy&, - const TrivialMoveButNotCopy&)))]; } - { int arr[F((__is_trivially_assignable(AllDeleted, - const AllDeleted &)))]; } - { int arr[F((__is_trivially_assignable(AllDeleted, - AllDeleted &&)))]; } - { int arr[F((__is_trivially_assignable(ExtDefaulted, - const ExtDefaulted &)))]; } - { int arr[F((__is_trivially_assignable(ExtDefaulted, - ExtDefaulted &&)))]; } - - { int arr[T((__is_trivially_assignable(HasDefaultTrivialCopyAssign&, - HasDefaultTrivialCopyAssign&)))]; } - { int arr[T((__is_trivially_assignable(HasDefaultTrivialCopyAssign&, - const HasDefaultTrivialCopyAssign&)))]; } - { int arr[T((__is_trivially_assignable(TrivialMoveButNotCopy&, - TrivialMoveButNotCopy)))]; } - { int arr[T((__is_trivially_assignable(TrivialMoveButNotCopy&, - TrivialMoveButNotCopy&&)))]; } - { int arr[T((__is_trivially_assignable(int&, int)))]; } - { int arr[T((__is_trivially_assignable(int&, int&)))]; } - { int arr[T((__is_trivially_assignable(int&, int&&)))]; } - { int arr[T((__is_trivially_assignable(int&, const int&)))]; } - { int arr[T((__is_trivially_assignable(POD&, POD)))]; } - { int arr[T((__is_trivially_assignable(POD&, POD&)))]; } - { int arr[T((__is_trivially_assignable(POD&, POD&&)))]; } - { int arr[T((__is_trivially_assignable(POD&, const POD&)))]; } - { int arr[T((__is_trivially_assignable(int*&, int*)))]; } - { int arr[T((__is_trivially_assignable(AllDefaulted, - const AllDefaulted &)))]; } - { int arr[T((__is_trivially_assignable(AllDefaulted, - AllDefaulted &&)))]; } - - { int arr[F((__is_assignable(int *&, float *)))]; } - { int arr[T((__is_assignable(HasCopyAssign &, HasCopyAssign)))]; } - { int arr[T((__is_assignable(HasCopyAssign &, HasCopyAssign &)))]; } - { int arr[T((__is_assignable(HasCopyAssign &, const HasCopyAssign &)))]; } - { int arr[T((__is_assignable(HasCopyAssign &, HasCopyAssign &&)))]; } - { int arr[T((__is_assignable(TrivialMoveButNotCopy &, - TrivialMoveButNotCopy &)))]; } - { int arr[T((__is_assignable(TrivialMoveButNotCopy &, - const TrivialMoveButNotCopy &)))]; } - { int arr[F((__is_assignable(AllDeleted, - const AllDeleted &)))]; } - { int arr[F((__is_assignable(AllDeleted, - AllDeleted &&)))]; } - { int arr[T((__is_assignable(ExtDefaulted, - const ExtDefaulted &)))]; } - { int arr[T((__is_assignable(ExtDefaulted, - ExtDefaulted &&)))]; } - - { int arr[T((__is_assignable(HasDefaultTrivialCopyAssign &, - HasDefaultTrivialCopyAssign &)))]; } - { int arr[T((__is_assignable(HasDefaultTrivialCopyAssign &, - const HasDefaultTrivialCopyAssign &)))]; } - { int arr[T((__is_assignable(TrivialMoveButNotCopy &, - TrivialMoveButNotCopy)))]; } - { int arr[T((__is_assignable(TrivialMoveButNotCopy &, - TrivialMoveButNotCopy &&)))]; } - - { int arr[T(__is_assignable(ACompleteType, ACompleteType))]; } - { int arr[F(__is_assignable(AnIncompleteType, AnIncompleteType))]; } // expected-error {{incomplete type}} - { int arr[F(__is_assignable(AnIncompleteType[], AnIncompleteType[]))]; } - { int arr[F(__is_assignable(AnIncompleteType[1], AnIncompleteType[1]))]; } // expected-error {{incomplete type}} - { int arr[F(__is_assignable(void, void))]; } - { int arr[F(__is_assignable(const volatile void, const volatile void))]; } + static_assert(__is_trivially_copyable(int)); + static_assert(__is_trivially_copyable(Enum)); + static_assert(__is_trivially_copyable(POD)); + static_assert(__is_trivially_copyable(Int)); + static_assert(__is_trivially_copyable(IntAr)); + static_assert(__is_trivially_copyable(IntArNB)); + static_assert(__is_trivially_copyable(Statics)); + static_assert(__is_trivially_copyable(Empty)); + static_assert(__is_trivially_copyable(EmptyUnion)); + static_assert(__is_trivially_copyable(Union)); + static_assert(__is_trivially_copyable(Derives)); + static_assert(__is_trivially_copyable(DerivesAr)); + static_assert(__is_trivially_copyable(DerivesArNB)); + static_assert(__is_trivially_copyable(DerivesEmpty)); + static_assert(__is_trivially_copyable(HasFunc)); + static_assert(__is_trivially_copyable(HasOp)); + static_assert(__is_trivially_copyable(HasConv)); + static_assert(__is_trivially_copyable(HasAssign)); + static_assert(__is_trivially_copyable(HasAnonymousUnion)); + static_assert(__is_trivially_copyable(HasPriv)); + static_assert(__is_trivially_copyable(HasProt)); + static_assert(__is_trivially_copyable(DerivesHasPriv)); + static_assert(__is_trivially_copyable(DerivesHasProt)); + static_assert(__is_trivially_copyable(Vector)); + static_assert(__is_trivially_copyable(VectorExt)); + static_assert(__is_trivially_copyable(HasCons)); + static_assert(__is_trivially_copyable(HasRef)); + static_assert(__is_trivially_copyable(HasNonPOD)); + static_assert(__is_trivially_copyable(DerivesHasCons)); + static_assert(__is_trivially_copyable(DerivesHasRef)); + static_assert(__is_trivially_copyable(NonTrivialDefault)); + static_assert(__is_trivially_copyable(NonTrivialDefault[])); + static_assert(__is_trivially_copyable(NonTrivialDefault[3])); + + static_assert(!__is_trivially_copyable(HasCopyAssign)); + static_assert(!__is_trivially_copyable(HasMoveAssign)); + static_assert(!__is_trivially_copyable(HasDest)); + static_assert(!__is_trivially_copyable(HasVirt)); + static_assert(!__is_trivially_copyable(DerivesHasCopyAssign)); + static_assert(!__is_trivially_copyable(DerivesHasMoveAssign)); + static_assert(!__is_trivially_copyable(DerivesHasDest)); + static_assert(!__is_trivially_copyable(DerivesHasVirt)); + static_assert(!__is_trivially_copyable(void)); + static_assert(!__is_trivially_copyable(cvoid)); + + static_assert((__is_trivially_constructible(int))); + static_assert((__is_trivially_constructible(int, int))); + static_assert((__is_trivially_constructible(int, float))); + static_assert((__is_trivially_constructible(int, int&))); + static_assert((__is_trivially_constructible(int, const int&))); + static_assert((__is_trivially_constructible(int, int))); + static_assert((__is_trivially_constructible(HasCopyAssign, HasCopyAssign))); + static_assert((__is_trivially_constructible(HasCopyAssign, const HasCopyAssign&))); + static_assert((__is_trivially_constructible(HasCopyAssign, HasCopyAssign&&))); + static_assert((__is_trivially_constructible(HasCopyAssign))); + static_assert((__is_trivially_constructible(NonTrivialDefault, + const NonTrivialDefault&))); + static_assert((__is_trivially_constructible(NonTrivialDefault, + NonTrivialDefault&&))); + static_assert((__is_trivially_constructible(AllDefaulted))); + static_assert((__is_trivially_constructible(AllDefaulted, + const AllDefaulted &))); + static_assert((__is_trivially_constructible(AllDefaulted, + AllDefaulted &&))); + + static_assert(!(__is_trivially_constructible(int, int*))); + static_assert(!(__is_trivially_constructible(NonTrivialDefault))); + static_assert(!(__is_trivially_constructible(ThreeArgCtor, int*, char*, int&))); + static_assert(!(__is_trivially_constructible(AllDeleted))); + static_assert(!(__is_trivially_constructible(AllDeleted, + const AllDeleted &))); + static_assert(!(__is_trivially_constructible(AllDeleted, + AllDeleted &&))); + static_assert(!(__is_trivially_constructible(ExtDefaulted))); + static_assert(!(__is_trivially_constructible(ExtDefaulted, + const ExtDefaulted &))); + static_assert(!(__is_trivially_constructible(ExtDefaulted, + ExtDefaulted &&))); + + static_assert((__is_trivially_constructible(TriviallyConstructibleTemplate))); + static_assert(!(__is_trivially_constructible(class_forward))); // expected-error {{incomplete type 'class_forward' used in type trait expression}} + static_assert(!(__is_trivially_constructible(class_forward[]))); + static_assert(!(__is_trivially_constructible(void))); + + static_assert((__is_trivially_assignable(int&, int))); + static_assert((__is_trivially_assignable(int&, int&))); + static_assert((__is_trivially_assignable(int&, int&&))); + static_assert((__is_trivially_assignable(int&, const int&))); + static_assert((__is_trivially_assignable(POD&, POD))); + static_assert((__is_trivially_assignable(POD&, POD&))); + static_assert((__is_trivially_assignable(POD&, POD&&))); + static_assert((__is_trivially_assignable(POD&, const POD&))); + static_assert((__is_trivially_assignable(int*&, int*))); + static_assert((__is_trivially_assignable(AllDefaulted, + const AllDefaulted &))); + static_assert((__is_trivially_assignable(AllDefaulted, + AllDefaulted &&))); + + static_assert(!(__is_trivially_assignable(int*&, float*))); + static_assert(!(__is_trivially_assignable(HasCopyAssign&, HasCopyAssign))); + static_assert(!(__is_trivially_assignable(HasCopyAssign&, HasCopyAssign&))); + static_assert(!(__is_trivially_assignable(HasCopyAssign&, const HasCopyAssign&))); + static_assert(!(__is_trivially_assignable(HasCopyAssign&, HasCopyAssign&&))); + static_assert(!(__is_trivially_assignable(TrivialMoveButNotCopy&, + TrivialMoveButNotCopy&))); + static_assert(!(__is_trivially_assignable(TrivialMoveButNotCopy&, + const TrivialMoveButNotCopy&))); + static_assert(!(__is_trivially_assignable(AllDeleted, + const AllDeleted &))); + static_assert(!(__is_trivially_assignable(AllDeleted, + AllDeleted &&))); + static_assert(!(__is_trivially_assignable(ExtDefaulted, + const ExtDefaulted &))); + static_assert(!(__is_trivially_assignable(ExtDefaulted, + ExtDefaulted &&))); + + static_assert((__is_trivially_assignable(HasDefaultTrivialCopyAssign&, + HasDefaultTrivialCopyAssign&))); + static_assert((__is_trivially_assignable(HasDefaultTrivialCopyAssign&, + const HasDefaultTrivialCopyAssign&))); + static_assert((__is_trivially_assignable(TrivialMoveButNotCopy&, + TrivialMoveButNotCopy))); + static_assert((__is_trivially_assignable(TrivialMoveButNotCopy&, + TrivialMoveButNotCopy&&))); + static_assert((__is_trivially_assignable(int&, int))); + static_assert((__is_trivially_assignable(int&, int&))); + static_assert((__is_trivially_assignable(int&, int&&))); + static_assert((__is_trivially_assignable(int&, const int&))); + static_assert((__is_trivially_assignable(POD&, POD))); + static_assert((__is_trivially_assignable(POD&, POD&))); + static_assert((__is_trivially_assignable(POD&, POD&&))); + static_assert((__is_trivially_assignable(POD&, const POD&))); + static_assert((__is_trivially_assignable(int*&, int*))); + static_assert((__is_trivially_assignable(AllDefaulted, + const AllDefaulted &))); + static_assert((__is_trivially_assignable(AllDefaulted, + AllDefaulted &&))); + + static_assert(!(__is_assignable(int *&, float *))); + static_assert((__is_assignable(HasCopyAssign &, HasCopyAssign))); + static_assert((__is_assignable(HasCopyAssign &, HasCopyAssign &))); + static_assert((__is_assignable(HasCopyAssign &, const HasCopyAssign &))); + static_assert((__is_assignable(HasCopyAssign &, HasCopyAssign &&))); + static_assert((__is_assignable(TrivialMoveButNotCopy &, + TrivialMoveButNotCopy &))); + static_assert((__is_assignable(TrivialMoveButNotCopy &, + const TrivialMoveButNotCopy &))); + static_assert(!(__is_assignable(AllDeleted, + const AllDeleted &))); + static_assert(!(__is_assignable(AllDeleted, + AllDeleted &&))); + static_assert((__is_assignable(ExtDefaulted, + const ExtDefaulted &))); + static_assert((__is_assignable(ExtDefaulted, + ExtDefaulted &&))); + + static_assert((__is_assignable(HasDefaultTrivialCopyAssign &, + HasDefaultTrivialCopyAssign &))); + static_assert((__is_assignable(HasDefaultTrivialCopyAssign &, + const HasDefaultTrivialCopyAssign &))); + static_assert((__is_assignable(TrivialMoveButNotCopy &, + TrivialMoveButNotCopy))); + static_assert((__is_assignable(TrivialMoveButNotCopy &, + TrivialMoveButNotCopy &&))); + + static_assert(__is_assignable(ACompleteType, ACompleteType)); + static_assert(!__is_assignable(AnIncompleteType, AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_assignable(AnIncompleteType[], AnIncompleteType[])); + static_assert(!__is_assignable(AnIncompleteType[1], AnIncompleteType[1])); // expected-error {{incomplete type}} + static_assert(!__is_assignable(void, void)); + static_assert(!__is_assignable(const volatile void, const volatile void)); } void constructible_checks() { - { int arr[T(__is_constructible(HasNoThrowConstructorWithArgs))]; } - { int arr[F(__is_nothrow_constructible(HasNoThrowConstructorWithArgs))]; } // MSVC doesn't look into default args and gets this wrong. + static_assert(__is_constructible(HasNoThrowConstructorWithArgs)); + static_assert(!__is_nothrow_constructible(HasNoThrowConstructorWithArgs)); // MSVC doesn't look into default args and gets this wrong. - { int arr[T(__is_constructible(HasNoThrowConstructorWithArgs, HasCons))]; } - { int arr[T(__is_nothrow_constructible(HasNoThrowConstructorWithArgs, HasCons))]; } + static_assert(__is_constructible(HasNoThrowConstructorWithArgs, HasCons)); + static_assert(__is_nothrow_constructible(HasNoThrowConstructorWithArgs, HasCons)); - { int arr[T(__is_constructible(NonTrivialDefault))]; } - { int arr[F(__is_nothrow_constructible(NonTrivialDefault))]; } + static_assert(__is_constructible(NonTrivialDefault)); + static_assert(!__is_nothrow_constructible(NonTrivialDefault)); - { int arr[T(__is_constructible(int))]; } - { int arr[T(__is_nothrow_constructible(int))]; } + static_assert(__is_constructible(int)); + static_assert(__is_nothrow_constructible(int)); - { int arr[F(__is_constructible(NonPOD))]; } - { int arr[F(__is_nothrow_constructible(NonPOD))]; } + static_assert(!__is_constructible(NonPOD)); + static_assert(!__is_nothrow_constructible(NonPOD)); - { int arr[T(__is_constructible(NonPOD, int))]; } - { int arr[F(__is_nothrow_constructible(NonPOD, int))]; } + static_assert(__is_constructible(NonPOD, int)); + static_assert(!__is_nothrow_constructible(NonPOD, int)); // PR19178 - { int arr[F(__is_constructible(Abstract))]; } - { int arr[F(__is_nothrow_constructible(Abstract))]; } + static_assert(!__is_constructible(Abstract)); + static_assert(!__is_nothrow_constructible(Abstract)); // PR20228 - { int arr[T(__is_constructible(VariadicCtor, - int, int, int, int, int, int, int, int, int))]; } + static_assert(__is_constructible(VariadicCtor, + int, int, int, int, int, int, int, int, int)); // PR25513 - { int arr[F(__is_constructible(int(int)))]; } - { int arr[T(__is_constructible(int const &, long))]; } - - { int arr[T(__is_constructible(ACompleteType))]; } - { int arr[T(__is_nothrow_constructible(ACompleteType))]; } - { int arr[F(__is_constructible(AnIncompleteType))]; } // expected-error {{incomplete type}} - { int arr[F(__is_nothrow_constructible(AnIncompleteType))]; } // expected-error {{incomplete type}} - { int arr[F(__is_constructible(AnIncompleteType[]))]; } - { int arr[F(__is_nothrow_constructible(AnIncompleteType[]))]; } - { int arr[F(__is_constructible(AnIncompleteType[1]))]; } // expected-error {{incomplete type}} - { int arr[F(__is_nothrow_constructible(AnIncompleteType[1]))]; } // expected-error {{incomplete type}} - { int arr[F(__is_constructible(void))]; } - { int arr[F(__is_nothrow_constructible(void))]; } - { int arr[F(__is_constructible(const volatile void))]; } - { int arr[F(__is_nothrow_constructible(const volatile void))]; } + static_assert(!__is_constructible(int(int))); + static_assert(__is_constructible(int const &, long)); + + static_assert(__is_constructible(ACompleteType)); + static_assert(__is_nothrow_constructible(ACompleteType)); + static_assert(!__is_constructible(AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_nothrow_constructible(AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_constructible(AnIncompleteType[])); + static_assert(!__is_nothrow_constructible(AnIncompleteType[])); + static_assert(!__is_constructible(AnIncompleteType[1])); // expected-error {{incomplete type}} + static_assert(!__is_nothrow_constructible(AnIncompleteType[1])); // expected-error {{incomplete type}} + static_assert(!__is_constructible(void)); + static_assert(!__is_nothrow_constructible(void)); + static_assert(!__is_constructible(const volatile void)); + static_assert(!__is_nothrow_constructible(const volatile void)); } // Instantiation of __is_trivially_constructible @@ -2738,32 +2727,32 @@ struct is_trivially_constructible { }; void is_trivially_constructible_test() { - { int arr[T((is_trivially_constructible::value))]; } - { int arr[T((is_trivially_constructible::value))]; } - { int arr[T((is_trivially_constructible::value))]; } - { int arr[T((is_trivially_constructible::value))]; } - { int arr[T((is_trivially_constructible::value))]; } - { int arr[T((is_trivially_constructible::value))]; } - { int arr[T((is_trivially_constructible::value))]; } - { int arr[T((is_trivially_constructible::value))]; } - { int arr[T((is_trivially_constructible::value))]; } - { int arr[T((is_trivially_constructible::value))]; } - { int arr[T((is_trivially_constructible::value))]; } - { int arr[T((is_trivially_constructible::value))]; } - - { int arr[F((is_trivially_constructible::value))]; } - { int arr[F((is_trivially_constructible::value))]; } - { int arr[F((is_trivially_constructible::value))]; } - { int arr[F((is_trivially_constructible::value))]; } // PR19178 - - { int arr[T(__is_trivially_constructible(ACompleteType))]; } - { int arr[F(__is_trivially_constructible(AnIncompleteType))]; } // expected-error {{incomplete type}} - { int arr[F(__is_trivially_constructible(AnIncompleteType[]))]; } - { int arr[F(__is_trivially_constructible(AnIncompleteType[1]))]; } // expected-error {{incomplete type}} - { int arr[F(__is_trivially_constructible(void))]; } - { int arr[F(__is_trivially_constructible(const volatile void))]; } + static_assert((is_trivially_constructible::value)); + static_assert((is_trivially_constructible::value)); + static_assert((is_trivially_constructible::value)); + static_assert((is_trivially_constructible::value)); + static_assert((is_trivially_constructible::value)); + static_assert((is_trivially_constructible::value)); + static_assert((is_trivially_constructible::value)); + static_assert((is_trivially_constructible::value)); + static_assert((is_trivially_constructible::value)); + static_assert((is_trivially_constructible::value)); + static_assert((is_trivially_constructible::value)); + static_assert((is_trivially_constructible::value)); + + static_assert(!(is_trivially_constructible::value)); + static_assert(!(is_trivially_constructible::value)); + static_assert(!(is_trivially_constructible::value)); + static_assert(!(is_trivially_constructible::value)); // PR19178 + + static_assert(__is_trivially_constructible(ACompleteType)); + static_assert(!__is_trivially_constructible(AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_trivially_constructible(AnIncompleteType[])); + static_assert(!__is_trivially_constructible(AnIncompleteType[1])); // expected-error {{incomplete type}} + static_assert(!__is_trivially_constructible(void)); + static_assert(!__is_trivially_constructible(const volatile void)); } template @@ -2773,365 +2762,359 @@ struct ConvertsToRef { }; void reference_binds_to_temporary_checks() { - { int arr[F((__reference_binds_to_temporary(int &, int &)))]; } - { int arr[F((__reference_binds_to_temporary(int &, int &&)))]; } + static_assert(!(__reference_binds_to_temporary(int &, int &))); + static_assert(!(__reference_binds_to_temporary(int &, int &&))); - { int arr[F((__reference_binds_to_temporary(int const &, int &)))]; } - { int arr[F((__reference_binds_to_temporary(int const &, int const &)))]; } - { int arr[F((__reference_binds_to_temporary(int const &, int &&)))]; } + static_assert(!(__reference_binds_to_temporary(int const &, int &))); + static_assert(!(__reference_binds_to_temporary(int const &, int const &))); + static_assert(!(__reference_binds_to_temporary(int const &, int &&))); - { int arr[F((__reference_binds_to_temporary(int &, long &)))]; } // doesn't construct - { int arr[T((__reference_binds_to_temporary(int const &, long &)))]; } - { int arr[T((__reference_binds_to_temporary(int const &, long &&)))]; } - { int arr[T((__reference_binds_to_temporary(int &&, long &)))]; } + static_assert(!(__reference_binds_to_temporary(int &, long &))); // doesn't construct + static_assert((__reference_binds_to_temporary(int const &, long &))); + static_assert((__reference_binds_to_temporary(int const &, long &&))); + static_assert((__reference_binds_to_temporary(int &&, long &))); using LRef = ConvertsToRef; using RRef = ConvertsToRef; using CLRef = ConvertsToRef; using LongRef = ConvertsToRef; - { int arr[T((__is_constructible(int &, LRef)))]; } - { int arr[F((__reference_binds_to_temporary(int &, LRef)))]; } + static_assert((__is_constructible(int &, LRef))); + static_assert(!(__reference_binds_to_temporary(int &, LRef))); - { int arr[T((__is_constructible(int &&, RRef)))]; } - { int arr[F((__reference_binds_to_temporary(int &&, RRef)))]; } + static_assert((__is_constructible(int &&, RRef))); + static_assert(!(__reference_binds_to_temporary(int &&, RRef))); - { int arr[T((__is_constructible(int const &, CLRef)))]; } - { int arr[F((__reference_binds_to_temporary(int &&, CLRef)))]; } + static_assert((__is_constructible(int const &, CLRef))); + static_assert(!(__reference_binds_to_temporary(int &&, CLRef))); - { int arr[T((__is_constructible(int const &, LongRef)))]; } - { int arr[T((__reference_binds_to_temporary(int const &, LongRef)))]; } + static_assert((__is_constructible(int const &, LongRef))); + static_assert((__reference_binds_to_temporary(int const &, LongRef))); // Test that it doesn't accept non-reference types as input. - { int arr[F((__reference_binds_to_temporary(int, long)))]; } + static_assert(!(__reference_binds_to_temporary(int, long))); - { int arr[T((__reference_binds_to_temporary(const int &, long)))]; } + static_assert((__reference_binds_to_temporary(const int &, long))); } void reference_constructs_from_temporary_checks() { - static_assert(!__reference_constructs_from_temporary(int &, int &), ""); - static_assert(!__reference_constructs_from_temporary(int &, int &&), ""); + static_assert(!__reference_constructs_from_temporary(int &, int &)); + static_assert(!__reference_constructs_from_temporary(int &, int &&)); - static_assert(!__reference_constructs_from_temporary(int const &, int &), ""); - static_assert(!__reference_constructs_from_temporary(int const &, int const &), ""); - static_assert(!__reference_constructs_from_temporary(int const &, int &&), ""); + static_assert(!__reference_constructs_from_temporary(int const &, int &)); + static_assert(!__reference_constructs_from_temporary(int const &, int const &)); + static_assert(!__reference_constructs_from_temporary(int const &, int &&)); - static_assert(!__reference_constructs_from_temporary(int &, long &), ""); // doesn't construct + static_assert(!__reference_constructs_from_temporary(int &, long &)); // doesn't construct - static_assert(__reference_constructs_from_temporary(int const &, long &), ""); - static_assert(__reference_constructs_from_temporary(int const &, long &&), ""); - static_assert(__reference_constructs_from_temporary(int &&, long &), ""); + static_assert(__reference_constructs_from_temporary(int const &, long &)); + static_assert(__reference_constructs_from_temporary(int const &, long &&)); + static_assert(__reference_constructs_from_temporary(int &&, long &)); using LRef = ConvertsToRef; using RRef = ConvertsToRef; using CLRef = ConvertsToRef; using LongRef = ConvertsToRef; - static_assert(__is_constructible(int &, LRef), ""); - static_assert(!__reference_constructs_from_temporary(int &, LRef), ""); + static_assert(__is_constructible(int &, LRef)); + static_assert(!__reference_constructs_from_temporary(int &, LRef)); - static_assert(__is_constructible(int &&, RRef), ""); - static_assert(!__reference_constructs_from_temporary(int &&, RRef), ""); + static_assert(__is_constructible(int &&, RRef)); + static_assert(!__reference_constructs_from_temporary(int &&, RRef)); - static_assert(__is_constructible(int const &, CLRef), ""); - static_assert(!__reference_constructs_from_temporary(int &&, CLRef), ""); + static_assert(__is_constructible(int const &, CLRef)); + static_assert(!__reference_constructs_from_temporary(int &&, CLRef)); - static_assert(__is_constructible(int const &, LongRef), ""); - static_assert(__reference_constructs_from_temporary(int const &, LongRef), ""); + static_assert(__is_constructible(int const &, LongRef)); + static_assert(__reference_constructs_from_temporary(int const &, LongRef)); // Test that it doesn't accept non-reference types as input. - static_assert(!__reference_constructs_from_temporary(int, long), ""); + static_assert(!__reference_constructs_from_temporary(int, long)); - static_assert(__reference_constructs_from_temporary(const int &, long), ""); + static_assert(__reference_constructs_from_temporary(const int &, long)); // Additional checks - static_assert(__reference_constructs_from_temporary(POD const&, Derives), ""); - static_assert(__reference_constructs_from_temporary(int&&, int), ""); - static_assert(__reference_constructs_from_temporary(const int&, int), ""); - static_assert(!__reference_constructs_from_temporary(int&&, int&&), ""); - static_assert(!__reference_constructs_from_temporary(const int&, int&&), ""); - static_assert(__reference_constructs_from_temporary(int&&, long&&), ""); - static_assert(__reference_constructs_from_temporary(int&&, long), ""); + static_assert(__reference_constructs_from_temporary(POD const&, Derives)); + static_assert(__reference_constructs_from_temporary(int&&, int)); + static_assert(__reference_constructs_from_temporary(const int&, int)); + static_assert(!__reference_constructs_from_temporary(int&&, int&&)); + static_assert(!__reference_constructs_from_temporary(const int&, int&&)); + static_assert(__reference_constructs_from_temporary(int&&, long&&)); + static_assert(__reference_constructs_from_temporary(int&&, long)); } void array_rank() { - int t01[T(__array_rank(IntAr) == 1)]; - int t02[T(__array_rank(ConstIntArAr) == 2)]; + static_assert(__array_rank(IntAr) == 1); + static_assert(__array_rank(ConstIntArAr) == 2); } void array_extent() { - int t01[T(__array_extent(IntAr, 0) == 10)]; - int t02[T(__array_extent(ConstIntArAr, 0) == 4)]; - int t03[T(__array_extent(ConstIntArAr, 1) == 10)]; + static_assert(__array_extent(IntAr, 0) == 10); + static_assert(__array_extent(ConstIntArAr, 0) == 4); + static_assert(__array_extent(ConstIntArAr, 1) == 10); } void is_destructible_test() { - { int arr[T(__is_destructible(int))]; } - { int arr[T(__is_destructible(int[2]))]; } - { int arr[F(__is_destructible(int[]))]; } - { int arr[F(__is_destructible(void))]; } - { int arr[T(__is_destructible(int &))]; } - { int arr[T(__is_destructible(HasDest))]; } - { int arr[F(__is_destructible(AllPrivate))]; } - { int arr[T(__is_destructible(SuperNonTrivialStruct))]; } - { int arr[T(__is_destructible(AllDefaulted))]; } - { int arr[F(__is_destructible(AllDeleted))]; } - { int arr[T(__is_destructible(ThrowingDtor))]; } - { int arr[T(__is_destructible(NoThrowDtor))]; } - - { int arr[T(__is_destructible(ACompleteType))]; } - { int arr[F(__is_destructible(AnIncompleteType))]; } // expected-error {{incomplete type}} - { int arr[F(__is_destructible(AnIncompleteType[]))]; } - { int arr[F(__is_destructible(AnIncompleteType[1]))]; } // expected-error {{incomplete type}} - { int arr[F(__is_destructible(void))]; } - { int arr[F(__is_destructible(const volatile void))]; } + static_assert(__is_destructible(int)); + static_assert(__is_destructible(int[2])); + static_assert(!__is_destructible(int[])); + static_assert(!__is_destructible(void)); + static_assert(__is_destructible(int &)); + static_assert(__is_destructible(HasDest)); + static_assert(!__is_destructible(AllPrivate)); + static_assert(__is_destructible(SuperNonTrivialStruct)); + static_assert(__is_destructible(AllDefaulted)); + static_assert(!__is_destructible(AllDeleted)); + static_assert(__is_destructible(ThrowingDtor)); + static_assert(__is_destructible(NoThrowDtor)); + + static_assert(__is_destructible(ACompleteType)); + static_assert(!__is_destructible(AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_destructible(AnIncompleteType[])); + static_assert(!__is_destructible(AnIncompleteType[1])); // expected-error {{incomplete type}} + static_assert(!__is_destructible(void)); + static_assert(!__is_destructible(const volatile void)); } void is_nothrow_destructible_test() { - { int arr[T(__is_nothrow_destructible(int))]; } - { int arr[T(__is_nothrow_destructible(int[2]))]; } - { int arr[F(__is_nothrow_destructible(int[]))]; } - { int arr[F(__is_nothrow_destructible(void))]; } - { int arr[T(__is_nothrow_destructible(int &))]; } - { int arr[T(__is_nothrow_destructible(HasDest))]; } - { int arr[F(__is_nothrow_destructible(AllPrivate))]; } - { int arr[T(__is_nothrow_destructible(SuperNonTrivialStruct))]; } - { int arr[T(__is_nothrow_destructible(AllDefaulted))]; } - { int arr[F(__is_nothrow_destructible(AllDeleted))]; } - { int arr[F(__is_nothrow_destructible(ThrowingDtor))]; } - { int arr[T(__is_nothrow_destructible(NoExceptDtor))]; } - { int arr[T(__is_nothrow_destructible(NoThrowDtor))]; } - - { int arr[T(__is_nothrow_destructible(ACompleteType))]; } - { int arr[F(__is_nothrow_destructible(AnIncompleteType))]; } // expected-error {{incomplete type}} - { int arr[F(__is_nothrow_destructible(AnIncompleteType[]))]; } - { int arr[F(__is_nothrow_destructible(AnIncompleteType[1]))]; } // expected-error {{incomplete type}} - { int arr[F(__is_nothrow_destructible(void))]; } - { int arr[F(__is_nothrow_destructible(const volatile void))]; } + static_assert(__is_nothrow_destructible(int)); + static_assert(__is_nothrow_destructible(int[2])); + static_assert(!__is_nothrow_destructible(int[])); + static_assert(!__is_nothrow_destructible(void)); + static_assert(__is_nothrow_destructible(int &)); + static_assert(__is_nothrow_destructible(HasDest)); + static_assert(!__is_nothrow_destructible(AllPrivate)); + static_assert(__is_nothrow_destructible(SuperNonTrivialStruct)); + static_assert(__is_nothrow_destructible(AllDefaulted)); + static_assert(!__is_nothrow_destructible(AllDeleted)); + static_assert(!__is_nothrow_destructible(ThrowingDtor)); + static_assert(__is_nothrow_destructible(NoExceptDtor)); + static_assert(__is_nothrow_destructible(NoThrowDtor)); + + static_assert(__is_nothrow_destructible(ACompleteType)); + static_assert(!__is_nothrow_destructible(AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_nothrow_destructible(AnIncompleteType[])); + static_assert(!__is_nothrow_destructible(AnIncompleteType[1])); // expected-error {{incomplete type}} + static_assert(!__is_nothrow_destructible(void)); + static_assert(!__is_nothrow_destructible(const volatile void)); } void is_trivially_destructible_test() { - { int arr[T(__is_trivially_destructible(int))]; } - { int arr[T(__is_trivially_destructible(int[2]))]; } - { int arr[F(__is_trivially_destructible(int[]))]; } - { int arr[F(__is_trivially_destructible(void))]; } - { int arr[T(__is_trivially_destructible(int &))]; } - { int arr[F(__is_trivially_destructible(HasDest))]; } - { int arr[F(__is_trivially_destructible(AllPrivate))]; } - { int arr[F(__is_trivially_destructible(SuperNonTrivialStruct))]; } - { int arr[T(__is_trivially_destructible(AllDefaulted))]; } - { int arr[F(__is_trivially_destructible(AllDeleted))]; } - { int arr[F(__is_trivially_destructible(ThrowingDtor))]; } - { int arr[F(__is_trivially_destructible(NoThrowDtor))]; } - - { int arr[T(__is_trivially_destructible(ACompleteType))]; } - { int arr[F(__is_trivially_destructible(AnIncompleteType))]; } // expected-error {{incomplete type}} - { int arr[F(__is_trivially_destructible(AnIncompleteType[]))]; } - { int arr[F(__is_trivially_destructible(AnIncompleteType[1]))]; } // expected-error {{incomplete type}} - { int arr[F(__is_trivially_destructible(void))]; } - { int arr[F(__is_trivially_destructible(const volatile void))]; } + static_assert(__is_trivially_destructible(int)); + static_assert(__is_trivially_destructible(int[2])); + static_assert(!__is_trivially_destructible(int[])); + static_assert(!__is_trivially_destructible(void)); + static_assert(__is_trivially_destructible(int &)); + static_assert(!__is_trivially_destructible(HasDest)); + static_assert(!__is_trivially_destructible(AllPrivate)); + static_assert(!__is_trivially_destructible(SuperNonTrivialStruct)); + static_assert(__is_trivially_destructible(AllDefaulted)); + static_assert(!__is_trivially_destructible(AllDeleted)); + static_assert(!__is_trivially_destructible(ThrowingDtor)); + static_assert(!__is_trivially_destructible(NoThrowDtor)); + + static_assert(__is_trivially_destructible(ACompleteType)); + static_assert(!__is_trivially_destructible(AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_trivially_destructible(AnIncompleteType[])); + static_assert(!__is_trivially_destructible(AnIncompleteType[1])); // expected-error {{incomplete type}} + static_assert(!__is_trivially_destructible(void)); + static_assert(!__is_trivially_destructible(const volatile void)); } -// Instantiation of __has_unique_object_representations -template -struct has_unique_object_representations { - static const bool value = __has_unique_object_representations(T); -}; - -static_assert(!has_unique_object_representations::value, "void is never unique"); -static_assert(!has_unique_object_representations::value, "void is never unique"); -static_assert(!has_unique_object_representations::value, "void is never unique"); -static_assert(!has_unique_object_representations::value, "void is never unique"); +static_assert(!__has_unique_object_representations(void), "void is never unique"); +static_assert(!__has_unique_object_representations(const void), "void is never unique"); +static_assert(!__has_unique_object_representations(volatile void), "void is never unique"); +static_assert(!__has_unique_object_representations(const volatile void), "void is never unique"); -static_assert(has_unique_object_representations::value, "integrals are"); -static_assert(has_unique_object_representations::value, "integrals are"); -static_assert(has_unique_object_representations::value, "integrals are"); -static_assert(has_unique_object_representations::value, "integrals are"); +static_assert(__has_unique_object_representations(int), "integrals are"); +static_assert(__has_unique_object_representations(const int), "integrals are"); +static_assert(__has_unique_object_representations(volatile int), "integrals are"); +static_assert(__has_unique_object_representations(const volatile int), "integrals are"); -static_assert(has_unique_object_representations::value, "as are pointers"); -static_assert(has_unique_object_representations::value, "as are pointers"); -static_assert(has_unique_object_representations::value, "are pointers"); -static_assert(has_unique_object_representations::value, "as are pointers"); +static_assert(__has_unique_object_representations(void *), "as are pointers"); +static_assert(__has_unique_object_representations(const void *), "as are pointers"); +static_assert(__has_unique_object_representations(volatile void *), "are pointers"); +static_assert(__has_unique_object_representations(const volatile void *), "as are pointers"); -static_assert(has_unique_object_representations::value, "as are pointers"); -static_assert(has_unique_object_representations::value, "as are pointers"); -static_assert(has_unique_object_representations::value, "as are pointers"); -static_assert(has_unique_object_representations::value, "as are pointers"); +static_assert(__has_unique_object_representations(int *), "as are pointers"); +static_assert(__has_unique_object_representations(const int *), "as are pointers"); +static_assert(__has_unique_object_representations(volatile int *), "as are pointers"); +static_assert(__has_unique_object_representations(const volatile int *), "as are pointers"); class C {}; using FP = int (*)(int); using PMF = int (C::*)(int); using PMD = int C::*; -static_assert(has_unique_object_representations::value, "even function pointers"); -static_assert(has_unique_object_representations::value, "even function pointers"); -static_assert(has_unique_object_representations::value, "even function pointers"); -static_assert(has_unique_object_representations::value, "even function pointers"); - -static_assert(has_unique_object_representations::value, "and pointer to members"); -static_assert(has_unique_object_representations::value, "and pointer to members"); -static_assert(has_unique_object_representations::value, "and pointer to members"); -static_assert(has_unique_object_representations::value, "and pointer to members"); - -static_assert(has_unique_object_representations::value, "and pointer to members"); -static_assert(has_unique_object_representations::value, "and pointer to members"); -static_assert(has_unique_object_representations::value, "and pointer to members"); -static_assert(has_unique_object_representations::value, "and pointer to members"); - -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); - -static_assert(!has_unique_object_representations::value, "but not void!"); -static_assert(!has_unique_object_representations::value, "or nullptr_t"); -static_assert(!has_unique_object_representations::value, "definitely not Floating Point"); -static_assert(!has_unique_object_representations::value, "definitely not Floating Point"); -static_assert(!has_unique_object_representations::value, "definitely not Floating Point"); +static_assert(__has_unique_object_representations(FP), "even function pointers"); +static_assert(__has_unique_object_representations(const FP), "even function pointers"); +static_assert(__has_unique_object_representations(volatile FP), "even function pointers"); +static_assert(__has_unique_object_representations(const volatile FP), "even function pointers"); + +static_assert(__has_unique_object_representations(PMF), "and pointer to members"); +static_assert(__has_unique_object_representations(const PMF), "and pointer to members"); +static_assert(__has_unique_object_representations(volatile PMF), "and pointer to members"); +static_assert(__has_unique_object_representations(const volatile PMF), "and pointer to members"); + +static_assert(__has_unique_object_representations(PMD), "and pointer to members"); +static_assert(__has_unique_object_representations(const PMD), "and pointer to members"); +static_assert(__has_unique_object_representations(volatile PMD), "and pointer to members"); +static_assert(__has_unique_object_representations(const volatile PMD), "and pointer to members"); + +static_assert(__has_unique_object_representations(bool), "yes, all integral types"); +static_assert(__has_unique_object_representations(char), "yes, all integral types"); +static_assert(__has_unique_object_representations(signed char), "yes, all integral types"); +static_assert(__has_unique_object_representations(unsigned char), "yes, all integral types"); +static_assert(__has_unique_object_representations(short), "yes, all integral types"); +static_assert(__has_unique_object_representations(unsigned short), "yes, all integral types"); +static_assert(__has_unique_object_representations(int), "yes, all integral types"); +static_assert(__has_unique_object_representations(unsigned int), "yes, all integral types"); +static_assert(__has_unique_object_representations(long), "yes, all integral types"); +static_assert(__has_unique_object_representations(unsigned long), "yes, all integral types"); +static_assert(__has_unique_object_representations(long long), "yes, all integral types"); +static_assert(__has_unique_object_representations(unsigned long long), "yes, all integral types"); +static_assert(__has_unique_object_representations(wchar_t), "yes, all integral types"); +static_assert(__has_unique_object_representations(char16_t), "yes, all integral types"); +static_assert(__has_unique_object_representations(char32_t), "yes, all integral types"); + +static_assert(!__has_unique_object_representations(void), "but not void!"); +static_assert(!__has_unique_object_representations(decltype(nullptr)), "or nullptr_t"); +static_assert(!__has_unique_object_representations(float), "definitely not Floating Point"); +static_assert(!__has_unique_object_representations(double), "definitely not Floating Point"); +static_assert(!__has_unique_object_representations(long double), "definitely not Floating Point"); struct NoPadding { int a; int b; }; -static_assert(has_unique_object_representations::value, "types without padding are"); +static_assert(__has_unique_object_representations(NoPadding), "types without padding are"); struct InheritsFromNoPadding : NoPadding { int c; int d; }; -static_assert(has_unique_object_representations::value, "types without padding are"); +static_assert(__has_unique_object_representations(InheritsFromNoPadding), "types without padding are"); struct VirtuallyInheritsFromNoPadding : virtual NoPadding { int c; int d; }; -static_assert(!has_unique_object_representations::value, "No virtual inheritance"); +static_assert(!__has_unique_object_representations(VirtuallyInheritsFromNoPadding), "No virtual inheritance"); struct Padding { char a; int b; }; -//static_assert(!has_unique_object_representations::value, "but not with padding"); +//static_assert(!__has_unique_object_representations(Padding), "but not with padding"); struct InheritsFromPadding : Padding { int c; int d; }; -static_assert(!has_unique_object_representations::value, "or its subclasses"); +static_assert(!__has_unique_object_representations(InheritsFromPadding), "or its subclasses"); struct TailPadding { int a; char b; }; -static_assert(!has_unique_object_representations::value, "even at the end"); +static_assert(!__has_unique_object_representations(TailPadding), "even at the end"); struct TinyStruct { char a; }; -static_assert(has_unique_object_representations::value, "Should be no padding"); +static_assert(__has_unique_object_representations(TinyStruct), "Should be no padding"); struct InheritsFromTinyStruct : TinyStruct { int b; }; -static_assert(!has_unique_object_representations::value, "Inherit causes padding"); +static_assert(!__has_unique_object_representations(InheritsFromTinyStruct), "Inherit causes padding"); union NoPaddingUnion { int a; unsigned int b; }; -static_assert(has_unique_object_representations::value, "unions follow the same rules as structs"); +static_assert(__has_unique_object_representations(NoPaddingUnion), "unions follow the same rules as structs"); union PaddingUnion { int a; long long b; }; -static_assert(!has_unique_object_representations::value, "unions follow the same rules as structs"); +static_assert(!__has_unique_object_representations(PaddingUnion), "unions follow the same rules as structs"); struct NotTriviallyCopyable { int x; NotTriviallyCopyable(const NotTriviallyCopyable &) {} }; -static_assert(!has_unique_object_representations::value, "must be trivially copyable"); +static_assert(!__has_unique_object_representations(NotTriviallyCopyable), "must be trivially copyable"); struct HasNonUniqueMember { float x; }; -static_assert(!has_unique_object_representations::value, "all members must be unique"); +static_assert(!__has_unique_object_representations(HasNonUniqueMember), "all members must be unique"); enum ExampleEnum { xExample, yExample }; enum LLEnum : long long { xLongExample, yLongExample }; -static_assert(has_unique_object_representations::value, "Enums are integrals, so unique!"); -static_assert(has_unique_object_representations::value, "Enums are integrals, so unique!"); +static_assert(__has_unique_object_representations(ExampleEnum), "Enums are integrals, so unique!"); +static_assert(__has_unique_object_representations(LLEnum), "Enums are integrals, so unique!"); enum class ExampleEnumClass { xExample, yExample }; enum class LLEnumClass : long long { xLongExample, yLongExample }; -static_assert(has_unique_object_representations::value, "Enums are integrals, so unique!"); -static_assert(has_unique_object_representations::value, "Enums are integrals, so unique!"); +static_assert(__has_unique_object_representations(ExampleEnumClass), "Enums are integrals, so unique!"); +static_assert(__has_unique_object_representations(LLEnumClass), "Enums are integrals, so unique!"); // because references aren't trivially copyable. -static_assert(!has_unique_object_representations::value, "No references!"); -static_assert(!has_unique_object_representations::value, "No references!"); -static_assert(!has_unique_object_representations::value, "No references!"); -static_assert(!has_unique_object_representations::value, "No references!"); -static_assert(!has_unique_object_representations::value, "No empty types!"); -static_assert(!has_unique_object_representations::value, "No empty types!"); +static_assert(!__has_unique_object_representations(int &), "No references!"); +static_assert(!__has_unique_object_representations(const int &), "No references!"); +static_assert(!__has_unique_object_representations(volatile int &), "No references!"); +static_assert(!__has_unique_object_representations(const volatile int &), "No references!"); +static_assert(!__has_unique_object_representations(Empty), "No empty types!"); +static_assert(!__has_unique_object_representations(EmptyUnion), "No empty types!"); class Compressed : Empty { int x; }; -static_assert(has_unique_object_representations::value, "But inheriting from one is ok"); +static_assert(__has_unique_object_representations(Compressed), "But inheriting from one is ok"); class EmptyInheritor : Compressed {}; -static_assert(has_unique_object_representations::value, "As long as the base has items, empty is ok"); +static_assert(__has_unique_object_representations(EmptyInheritor), "As long as the base has items, empty is ok"); class Dynamic { virtual void A(); int i; }; -static_assert(!has_unique_object_representations::value, "Dynamic types are not valid"); +static_assert(!__has_unique_object_representations(Dynamic), "Dynamic types are not valid"); class InheritsDynamic : Dynamic { int j; }; -static_assert(!has_unique_object_representations::value, "Dynamic types are not valid"); +static_assert(!__has_unique_object_representations(InheritsDynamic), "Dynamic types are not valid"); -static_assert(has_unique_object_representations::value, "Arrays are fine, as long as their value type is"); -static_assert(has_unique_object_representations::value, "Arrays are fine, as long as their value type is"); -static_assert(has_unique_object_representations::value, "Arrays are fine, as long as their value type is"); -static_assert(!has_unique_object_representations::value, "So no array of doubles!"); -static_assert(!has_unique_object_representations::value, "So no array of doubles!"); -static_assert(!has_unique_object_representations::value, "So no array of doubles!"); +static_assert(__has_unique_object_representations(int[42]), "Arrays are fine, as long as their value type is"); +static_assert(__has_unique_object_representations(int[]), "Arrays are fine, as long as their value type is"); +static_assert(__has_unique_object_representations(int[][42]), "Arrays are fine, as long as their value type is"); +static_assert(!__has_unique_object_representations(double[42]), "So no array of doubles!"); +static_assert(!__has_unique_object_representations(double[]), "So no array of doubles!"); +static_assert(!__has_unique_object_representations(double[][42]), "So no array of doubles!"); struct __attribute__((aligned(16))) WeirdAlignment { int i; @@ -3139,9 +3122,9 @@ struct __attribute__((aligned(16))) WeirdAlignment { union __attribute__((aligned(16))) WeirdAlignmentUnion { int i; }; -static_assert(!has_unique_object_representations::value, "Alignment causes padding"); -static_assert(!has_unique_object_representations::value, "Alignment causes padding"); -static_assert(!has_unique_object_representations::value, "Also no arrays that have padding"); +static_assert(!__has_unique_object_representations(WeirdAlignment), "Alignment causes padding"); +static_assert(!__has_unique_object_representations(WeirdAlignmentUnion), "Alignment causes padding"); +static_assert(!__has_unique_object_representations(WeirdAlignment[42]), "Also no arrays that have padding"); struct __attribute__((packed)) PackedNoPadding1 { short i; @@ -3151,41 +3134,41 @@ struct __attribute__((packed)) PackedNoPadding2 { int j; short i; }; -static_assert(has_unique_object_representations::value, "Packed structs have no padding"); -static_assert(has_unique_object_representations::value, "Packed structs have no padding"); - -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); - -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); +static_assert(__has_unique_object_representations(PackedNoPadding1), "Packed structs have no padding"); +static_assert(__has_unique_object_representations(PackedNoPadding2), "Packed structs have no padding"); + +static_assert(!__has_unique_object_representations(int(int)), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int) const), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int) volatile), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int) const volatile), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int) &), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int) const &), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int) volatile &), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int) const volatile &), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int) &&), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int) const &&), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int) volatile &&), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int) const volatile &&), "Functions are not unique"); + +static_assert(!__has_unique_object_representations(int(int, ...)), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int, ...) const), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int, ...) volatile), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int, ...) const volatile), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int, ...) &), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int, ...) const &), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int, ...) volatile &), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int, ...) const volatile &), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int, ...) &&), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int, ...) const &&), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int, ...) volatile &&), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int, ...) const volatile &&), "Functions are not unique"); void foo(){ static auto lambda = []() {}; - static_assert(!has_unique_object_representations::value, "Lambdas follow struct rules"); + static_assert(!__has_unique_object_representations(decltype(lambda)), "Lambdas follow struct rules"); int i; static auto lambda2 = [i]() {}; - static_assert(has_unique_object_representations::value, "Lambdas follow struct rules"); + static_assert(__has_unique_object_representations(decltype(lambda2)), "Lambdas follow struct rules"); } struct PaddedBitfield { @@ -3225,26 +3208,26 @@ struct UnnamedEmptyBitfieldSplit { short also_named; }; -static_assert(!has_unique_object_representations::value, "Bitfield padding"); -static_assert(has_unique_object_representations::value, "Bitfield padding"); -static_assert(!has_unique_object_representations::value, "Bitfield padding"); -static_assert(!has_unique_object_representations::value, "Bitfield padding"); -static_assert(!has_unique_object_representations::value, "Bitfield padding"); -static_assert(has_unique_object_representations::value, "Bitfield padding"); +static_assert(!__has_unique_object_representations(PaddedBitfield), "Bitfield padding"); +static_assert(__has_unique_object_representations(UnPaddedBitfield), "Bitfield padding"); +static_assert(!__has_unique_object_representations(AlignedPaddedBitfield), "Bitfield padding"); +static_assert(!__has_unique_object_representations(UnnamedBitfield), "Bitfield padding"); +static_assert(!__has_unique_object_representations(UnnamedBitfieldPacked), "Bitfield padding"); +static_assert(__has_unique_object_representations(UnnamedEmptyBitfield), "Bitfield padding"); static_assert(sizeof(UnnamedEmptyBitfieldSplit) != (sizeof(short) * 2), "Wrong size"); -static_assert(!has_unique_object_representations::value, "Bitfield padding"); +static_assert(!__has_unique_object_representations(UnnamedEmptyBitfieldSplit), "Bitfield padding"); struct BoolBitfield { bool b : 8; }; -static_assert(has_unique_object_representations::value, "Bitfield bool"); +static_assert(__has_unique_object_representations(BoolBitfield), "Bitfield bool"); struct BoolBitfield2 { bool b : 16; }; -static_assert(!has_unique_object_representations::value, "Bitfield bool"); +static_assert(!__has_unique_object_representations(BoolBitfield2), "Bitfield bool"); struct GreaterSizeBitfield { //expected-warning@+1 {{width of bit-field 'n'}} @@ -3252,13 +3235,13 @@ struct GreaterSizeBitfield { }; static_assert(sizeof(GreaterSizeBitfield) == 128, "Bitfield Size"); -static_assert(!has_unique_object_representations::value, "Bitfield padding"); +static_assert(!__has_unique_object_representations(GreaterSizeBitfield), "Bitfield padding"); struct StructWithRef { int &I; }; -static_assert(has_unique_object_representations::value, "References are still unique"); +static_assert(__has_unique_object_representations(StructWithRef), "References are still unique"); struct NotUniqueBecauseTailPadding { int &r; @@ -3268,12 +3251,12 @@ struct CanBeUniqueIfNoPadding : NotUniqueBecauseTailPadding { char b[7]; }; -static_assert(!has_unique_object_representations::value, +static_assert(!__has_unique_object_representations(NotUniqueBecauseTailPadding), "non trivial"); // Can be unique on Itanium, since the is child class' data is 'folded' into the // parent's tail padding. static_assert(sizeof(CanBeUniqueIfNoPadding) != 16 || - has_unique_object_representations::value, + __has_unique_object_representations(CanBeUniqueIfNoPadding), "inherit from std layout"); namespace ErrorType { @@ -3285,10 +3268,10 @@ namespace ErrorType { bool b = __has_unique_object_representations(T); }; -static_assert(!has_unique_object_representations<_BitInt(7)>::value, "BitInt:"); -static_assert(has_unique_object_representations<_BitInt(8)>::value, "BitInt:"); -static_assert(!has_unique_object_representations<_BitInt(127)>::value, "BitInt:"); -static_assert(has_unique_object_representations<_BitInt(128)>::value, "BitInt:"); +static_assert(!__has_unique_object_representations(_BitInt(7)), "BitInt:"); +static_assert(__has_unique_object_representations(_BitInt(8)), "BitInt:"); +static_assert(!__has_unique_object_representations(_BitInt(127)), "BitInt:"); +static_assert(__has_unique_object_representations(_BitInt(128)), "BitInt:"); namespace PR46209 { @@ -3303,8 +3286,8 @@ namespace PR46209 { Foo foo; }; - static_assert(!__is_trivially_assignable(Foo &, const Foo &), ""); - static_assert(!__is_trivially_assignable(Bar &, const Bar &), ""); + static_assert(!__is_trivially_assignable(Foo &, const Foo &)); + static_assert(!__is_trivially_assignable(Bar &, const Bar &)); // Foo2 has both a trivial assignment operator and a non-trivial one. struct Foo2 { @@ -3317,8 +3300,8 @@ namespace PR46209 { Foo2 foo; }; - static_assert(__is_trivially_assignable(Foo2 &, const Foo2 &), ""); - static_assert(__is_trivially_assignable(Bar2 &, const Bar2 &), ""); + static_assert(__is_trivially_assignable(Foo2 &, const Foo2 &)); + static_assert(__is_trivially_assignable(Bar2 &, const Bar2 &)); } namespace ConstClass { @@ -3328,7 +3311,7 @@ namespace ConstClass { struct B { const A a; }; - static_assert(!__is_trivially_assignable(B&, const B&), ""); + static_assert(!__is_trivially_assignable(B&, const B&)); } namespace type_trait_expr_numargs_overflow { @@ -3354,21 +3337,21 @@ void test() { (void) __is_constructible(int, T32768(int)); } namespace is_trivially_relocatable { -static_assert(!__is_trivially_relocatable(void), ""); -static_assert(__is_trivially_relocatable(int), ""); -static_assert(__is_trivially_relocatable(int[]), ""); +static_assert(!__is_trivially_relocatable(void)); +static_assert(__is_trivially_relocatable(int)); +static_assert(__is_trivially_relocatable(int[])); enum Enum {}; -static_assert(__is_trivially_relocatable(Enum), ""); -static_assert(__is_trivially_relocatable(Enum[]), ""); +static_assert(__is_trivially_relocatable(Enum)); +static_assert(__is_trivially_relocatable(Enum[])); union Union {int x;}; -static_assert(__is_trivially_relocatable(Union), ""); -static_assert(__is_trivially_relocatable(Union[]), ""); +static_assert(__is_trivially_relocatable(Union)); +static_assert(__is_trivially_relocatable(Union[])); struct Trivial {}; -static_assert(__is_trivially_relocatable(Trivial), ""); -static_assert(__is_trivially_relocatable(Trivial[]), ""); +static_assert(__is_trivially_relocatable(Trivial)); +static_assert(__is_trivially_relocatable(Trivial[])); struct Incomplete; // expected-note {{forward declaration of 'is_trivially_relocatable::Incomplete'}} bool unused = __is_trivially_relocatable(Incomplete); // expected-error {{incomplete type}} @@ -3376,67 +3359,67 @@ bool unused = __is_trivially_relocatable(Incomplete); // expected-error {{incomp struct NontrivialDtor { ~NontrivialDtor() {} }; -static_assert(!__is_trivially_relocatable(NontrivialDtor), ""); -static_assert(!__is_trivially_relocatable(NontrivialDtor[]), ""); +static_assert(!__is_trivially_relocatable(NontrivialDtor)); +static_assert(!__is_trivially_relocatable(NontrivialDtor[])); struct NontrivialCopyCtor { NontrivialCopyCtor(const NontrivialCopyCtor&) {} }; -static_assert(!__is_trivially_relocatable(NontrivialCopyCtor), ""); -static_assert(!__is_trivially_relocatable(NontrivialCopyCtor[]), ""); +static_assert(!__is_trivially_relocatable(NontrivialCopyCtor)); +static_assert(!__is_trivially_relocatable(NontrivialCopyCtor[])); struct NontrivialMoveCtor { NontrivialMoveCtor(NontrivialMoveCtor&&) {} }; -static_assert(!__is_trivially_relocatable(NontrivialMoveCtor), ""); -static_assert(!__is_trivially_relocatable(NontrivialMoveCtor[]), ""); +static_assert(!__is_trivially_relocatable(NontrivialMoveCtor)); +static_assert(!__is_trivially_relocatable(NontrivialMoveCtor[])); struct [[clang::trivial_abi]] TrivialAbiNontrivialDtor { ~TrivialAbiNontrivialDtor() {} }; -static_assert(__is_trivially_relocatable(TrivialAbiNontrivialDtor), ""); -static_assert(__is_trivially_relocatable(TrivialAbiNontrivialDtor[]), ""); +static_assert(__is_trivially_relocatable(TrivialAbiNontrivialDtor)); +static_assert(__is_trivially_relocatable(TrivialAbiNontrivialDtor[])); struct [[clang::trivial_abi]] TrivialAbiNontrivialCopyCtor { TrivialAbiNontrivialCopyCtor(const TrivialAbiNontrivialCopyCtor&) {} }; -static_assert(__is_trivially_relocatable(TrivialAbiNontrivialCopyCtor), ""); -static_assert(__is_trivially_relocatable(TrivialAbiNontrivialCopyCtor[]), ""); +static_assert(__is_trivially_relocatable(TrivialAbiNontrivialCopyCtor)); +static_assert(__is_trivially_relocatable(TrivialAbiNontrivialCopyCtor[])); // A more complete set of tests for the behavior of trivial_abi can be found in // clang/test/SemaCXX/attr-trivial-abi.cpp struct [[clang::trivial_abi]] TrivialAbiNontrivialMoveCtor { TrivialAbiNontrivialMoveCtor(TrivialAbiNontrivialMoveCtor&&) {} }; -static_assert(__is_trivially_relocatable(TrivialAbiNontrivialMoveCtor), ""); -static_assert(__is_trivially_relocatable(TrivialAbiNontrivialMoveCtor[]), ""); +static_assert(__is_trivially_relocatable(TrivialAbiNontrivialMoveCtor)); +static_assert(__is_trivially_relocatable(TrivialAbiNontrivialMoveCtor[])); } // namespace is_trivially_relocatable namespace is_trivially_equality_comparable { struct ForwardDeclared; // expected-note {{forward declaration of 'is_trivially_equality_comparable::ForwardDeclared'}} -static_assert(!__is_trivially_equality_comparable(ForwardDeclared), ""); // expected-error {{incomplete type 'ForwardDeclared' used in type trait expression}} +static_assert(!__is_trivially_equality_comparable(ForwardDeclared)); // expected-error {{incomplete type 'ForwardDeclared' used in type trait expression}} -static_assert(!__is_trivially_equality_comparable(void), ""); -static_assert(__is_trivially_equality_comparable(int), ""); -static_assert(!__is_trivially_equality_comparable(int[]), ""); -static_assert(!__is_trivially_equality_comparable(int[3]), ""); -static_assert(!__is_trivially_equality_comparable(float), ""); -static_assert(!__is_trivially_equality_comparable(double), ""); -static_assert(!__is_trivially_equality_comparable(long double), ""); +static_assert(!__is_trivially_equality_comparable(void)); +static_assert(__is_trivially_equality_comparable(int)); +static_assert(!__is_trivially_equality_comparable(int[])); +static_assert(!__is_trivially_equality_comparable(int[3])); +static_assert(!__is_trivially_equality_comparable(float)); +static_assert(!__is_trivially_equality_comparable(double)); +static_assert(!__is_trivially_equality_comparable(long double)); struct NonTriviallyEqualityComparableNoComparator { int i; int j; }; -static_assert(!__is_trivially_equality_comparable(NonTriviallyEqualityComparableNoComparator), ""); +static_assert(!__is_trivially_equality_comparable(NonTriviallyEqualityComparableNoComparator)); struct NonTriviallyEqualityComparableNonDefaultedComparator { int i; int j; bool operator==(const NonTriviallyEqualityComparableNonDefaultedComparator&); }; -static_assert(!__is_trivially_equality_comparable(NonTriviallyEqualityComparableNonDefaultedComparator), ""); +static_assert(!__is_trivially_equality_comparable(NonTriviallyEqualityComparableNonDefaultedComparator)); #if __cplusplus >= 202002L @@ -3490,7 +3473,7 @@ struct NotTriviallyEqualityComparableHasPadding { bool operator==(const NotTriviallyEqualityComparableHasPadding&) const = default; }; -static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasPadding), ""); +static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasPadding)); struct NotTriviallyEqualityComparableHasFloat { float i; @@ -3498,7 +3481,7 @@ struct NotTriviallyEqualityComparableHasFloat { bool operator==(const NotTriviallyEqualityComparableHasFloat&) const = default; }; -static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasFloat), ""); +static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasFloat)); struct NotTriviallyEqualityComparableHasTailPadding { int i; @@ -3506,14 +3489,14 @@ struct NotTriviallyEqualityComparableHasTailPadding { bool operator==(const NotTriviallyEqualityComparableHasTailPadding&) const = default; }; -static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasTailPadding), ""); +static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasTailPadding)); struct NotTriviallyEqualityComparableBase : NotTriviallyEqualityComparableHasTailPadding { char j; bool operator==(const NotTriviallyEqualityComparableBase&) const = default; }; -static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableBase), ""); +static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableBase)); class TriviallyEqualityComparablePaddedOutBase { int i; @@ -3522,14 +3505,14 @@ class TriviallyEqualityComparablePaddedOutBase { public: bool operator==(const TriviallyEqualityComparablePaddedOutBase&) const = default; }; -static_assert(!__is_trivially_equality_comparable(TriviallyEqualityComparablePaddedOutBase), ""); +static_assert(!__is_trivially_equality_comparable(TriviallyEqualityComparablePaddedOutBase)); struct TriviallyEqualityComparablePaddedOut : TriviallyEqualityComparablePaddedOutBase { char j[3]; bool operator==(const TriviallyEqualityComparablePaddedOut&) const = default; }; -static_assert(__is_trivially_equality_comparable(TriviallyEqualityComparablePaddedOut), ""); +static_assert(__is_trivially_equality_comparable(TriviallyEqualityComparablePaddedOut)); struct TriviallyEqualityComparable1 { char i; @@ -3652,7 +3635,7 @@ struct TriviallyEqualityComparable { friend bool operator==(const TriviallyEqualityComparable&, const TriviallyEqualityComparable&) = default; }; -static_assert(__is_trivially_equality_comparable(TriviallyEqualityComparable), ""); +static_assert(__is_trivially_equality_comparable(TriviallyEqualityComparable)); struct TriviallyEqualityComparableNonTriviallyCopyable { TriviallyEqualityComparableNonTriviallyCopyable(const TriviallyEqualityComparableNonTriviallyCopyable&); @@ -3668,7 +3651,7 @@ struct NotTriviallyEqualityComparableHasPadding { friend bool operator==(const NotTriviallyEqualityComparableHasPadding&, const NotTriviallyEqualityComparableHasPadding&) = default; }; -static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasPadding), ""); +static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasPadding)); struct NotTriviallyEqualityComparableHasFloat { float i; @@ -3676,7 +3659,7 @@ struct NotTriviallyEqualityComparableHasFloat { friend bool operator==(const NotTriviallyEqualityComparableHasFloat&, const NotTriviallyEqualityComparableHasFloat&) = default; }; -static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasFloat), ""); +static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasFloat)); struct NotTriviallyEqualityComparableHasTailPadding { int i; @@ -3684,14 +3667,14 @@ struct NotTriviallyEqualityComparableHasTailPadding { friend bool operator==(const NotTriviallyEqualityComparableHasTailPadding&, const NotTriviallyEqualityComparableHasTailPadding&) = default; }; -static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasTailPadding), ""); +static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasTailPadding)); struct NotTriviallyEqualityComparableBase : NotTriviallyEqualityComparableHasTailPadding { char j; friend bool operator==(const NotTriviallyEqualityComparableBase&, const NotTriviallyEqualityComparableBase&) = default; }; -static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableBase), ""); +static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableBase)); class TriviallyEqualityComparablePaddedOutBase { int i; @@ -3700,14 +3683,14 @@ class TriviallyEqualityComparablePaddedOutBase { public: friend bool operator==(const TriviallyEqualityComparablePaddedOutBase&, const TriviallyEqualityComparablePaddedOutBase&) = default; }; -static_assert(!__is_trivially_equality_comparable(TriviallyEqualityComparablePaddedOutBase), ""); +static_assert(!__is_trivially_equality_comparable(TriviallyEqualityComparablePaddedOutBase)); struct TriviallyEqualityComparablePaddedOut : TriviallyEqualityComparablePaddedOutBase { char j[3]; friend bool operator==(const TriviallyEqualityComparablePaddedOut&, const TriviallyEqualityComparablePaddedOut&) = default; }; -static_assert(__is_trivially_equality_comparable(TriviallyEqualityComparablePaddedOut), ""); +static_assert(__is_trivially_equality_comparable(TriviallyEqualityComparablePaddedOut)); struct TriviallyEqualityComparable1 { char i; @@ -3824,10 +3807,10 @@ union D { int x; }; -static_assert(__can_pass_in_regs(A), ""); -static_assert(__can_pass_in_regs(A), ""); -static_assert(!__can_pass_in_regs(B), ""); -static_assert(__can_pass_in_regs(D), ""); +static_assert(__can_pass_in_regs(A)); +static_assert(__can_pass_in_regs(A)); +static_assert(!__can_pass_in_regs(B)); +static_assert(__can_pass_in_regs(D)); void test_errors() { (void)__can_pass_in_regs(const A); // expected-error {{not an unqualified class type}} @@ -3844,379 +3827,379 @@ struct S {}; template using remove_const_t = __remove_const(T); void check_remove_const() { - static_assert(__is_same(remove_const_t, void), ""); - static_assert(__is_same(remove_const_t, void), ""); - static_assert(__is_same(remove_const_t, int), ""); - static_assert(__is_same(remove_const_t, int), ""); - static_assert(__is_same(remove_const_t, volatile int), ""); - static_assert(__is_same(remove_const_t, volatile int), ""); - static_assert(__is_same(remove_const_t, int *), ""); - static_assert(__is_same(remove_const_t, int *), ""); - static_assert(__is_same(remove_const_t, int const *), ""); - static_assert(__is_same(remove_const_t, int const *__restrict), ""); - static_assert(__is_same(remove_const_t, int &), ""); - static_assert(__is_same(remove_const_t, int const &), ""); - static_assert(__is_same(remove_const_t, int &&), ""); - static_assert(__is_same(remove_const_t, int const &&), ""); - static_assert(__is_same(remove_const_t, int()), ""); - static_assert(__is_same(remove_const_t, int (*)()), ""); - static_assert(__is_same(remove_const_t, int (&)()), ""); - - static_assert(__is_same(remove_const_t, S), ""); - static_assert(__is_same(remove_const_t, S), ""); - static_assert(__is_same(remove_const_t, volatile S), ""); - static_assert(__is_same(remove_const_t, S *__restrict), ""); - static_assert(__is_same(remove_const_t, volatile S), ""); - static_assert(__is_same(remove_const_t, S *volatile __restrict), ""); - static_assert(__is_same(remove_const_t, int S::*), ""); - static_assert(__is_same(remove_const_t, int(S::*)()), ""); + static_assert(__is_same(remove_const_t, void)); + static_assert(__is_same(remove_const_t, void)); + static_assert(__is_same(remove_const_t, int)); + static_assert(__is_same(remove_const_t, int)); + static_assert(__is_same(remove_const_t, volatile int)); + static_assert(__is_same(remove_const_t, volatile int)); + static_assert(__is_same(remove_const_t, int *)); + static_assert(__is_same(remove_const_t, int *)); + static_assert(__is_same(remove_const_t, int const *)); + static_assert(__is_same(remove_const_t, int const *__restrict)); + static_assert(__is_same(remove_const_t, int &)); + static_assert(__is_same(remove_const_t, int const &)); + static_assert(__is_same(remove_const_t, int &&)); + static_assert(__is_same(remove_const_t, int const &&)); + static_assert(__is_same(remove_const_t, int())); + static_assert(__is_same(remove_const_t, int (*)())); + static_assert(__is_same(remove_const_t, int (&)())); + + static_assert(__is_same(remove_const_t, S)); + static_assert(__is_same(remove_const_t, S)); + static_assert(__is_same(remove_const_t, volatile S)); + static_assert(__is_same(remove_const_t, S *__restrict)); + static_assert(__is_same(remove_const_t, volatile S)); + static_assert(__is_same(remove_const_t, S *volatile __restrict)); + static_assert(__is_same(remove_const_t, int S::*)); + static_assert(__is_same(remove_const_t, int(S::*)())); } template using remove_restrict_t = __remove_restrict(T); void check_remove_restrict() { - static_assert(__is_same(remove_restrict_t, void), ""); - static_assert(__is_same(remove_restrict_t, int), ""); - static_assert(__is_same(remove_restrict_t, const int), ""); - static_assert(__is_same(remove_restrict_t, volatile int), ""); - static_assert(__is_same(remove_restrict_t, int *), ""); - static_assert(__is_same(remove_restrict_t, int *const volatile), ""); - static_assert(__is_same(remove_restrict_t, int *), ""); - static_assert(__is_same(remove_restrict_t, int *), ""); - static_assert(__is_same(remove_restrict_t, int &), ""); - static_assert(__is_same(remove_restrict_t, int &), ""); - static_assert(__is_same(remove_restrict_t, int &&), ""); - static_assert(__is_same(remove_restrict_t, int &&), ""); - static_assert(__is_same(remove_restrict_t, int()), ""); - static_assert(__is_same(remove_restrict_t, int (*const volatile)()), ""); - static_assert(__is_same(remove_restrict_t, int (&)()), ""); - - static_assert(__is_same(remove_restrict_t, S), ""); - static_assert(__is_same(remove_restrict_t, const S), ""); - static_assert(__is_same(remove_restrict_t, volatile S), ""); - static_assert(__is_same(remove_restrict_t, S *), ""); - static_assert(__is_same(remove_restrict_t, S *const volatile), ""); - static_assert(__is_same(remove_restrict_t, int S::*), ""); - static_assert(__is_same(remove_restrict_t, int(S::*const volatile)()), ""); + static_assert(__is_same(remove_restrict_t, void)); + static_assert(__is_same(remove_restrict_t, int)); + static_assert(__is_same(remove_restrict_t, const int)); + static_assert(__is_same(remove_restrict_t, volatile int)); + static_assert(__is_same(remove_restrict_t, int *)); + static_assert(__is_same(remove_restrict_t, int *const volatile)); + static_assert(__is_same(remove_restrict_t, int *)); + static_assert(__is_same(remove_restrict_t, int *)); + static_assert(__is_same(remove_restrict_t, int &)); + static_assert(__is_same(remove_restrict_t, int &)); + static_assert(__is_same(remove_restrict_t, int &&)); + static_assert(__is_same(remove_restrict_t, int &&)); + static_assert(__is_same(remove_restrict_t, int())); + static_assert(__is_same(remove_restrict_t, int (*const volatile)())); + static_assert(__is_same(remove_restrict_t, int (&)())); + + static_assert(__is_same(remove_restrict_t, S)); + static_assert(__is_same(remove_restrict_t, const S)); + static_assert(__is_same(remove_restrict_t, volatile S)); + static_assert(__is_same(remove_restrict_t, S *)); + static_assert(__is_same(remove_restrict_t, S *const volatile)); + static_assert(__is_same(remove_restrict_t, int S::*)); + static_assert(__is_same(remove_restrict_t, int(S::*const volatile)())); } template using remove_volatile_t = __remove_volatile(T); void check_remove_volatile() { - static_assert(__is_same(remove_volatile_t, void), ""); - static_assert(__is_same(remove_volatile_t, void), ""); - static_assert(__is_same(remove_volatile_t, int), ""); - static_assert(__is_same(remove_volatile_t, const int), ""); - static_assert(__is_same(remove_volatile_t, int), ""); - static_assert(__is_same(remove_volatile_t, int *__restrict), ""); - static_assert(__is_same(remove_volatile_t, const int), ""); - static_assert(__is_same(remove_volatile_t, int *const __restrict), ""); - static_assert(__is_same(remove_volatile_t, int *), ""); - static_assert(__is_same(remove_volatile_t, int *), ""); - static_assert(__is_same(remove_volatile_t, int volatile *), ""); - static_assert(__is_same(remove_volatile_t, int &), ""); - static_assert(__is_same(remove_volatile_t, int volatile &), ""); - static_assert(__is_same(remove_volatile_t, int &&), ""); - static_assert(__is_same(remove_volatile_t, int volatile &&), ""); - static_assert(__is_same(remove_volatile_t, int()), ""); - static_assert(__is_same(remove_volatile_t, int (*)()), ""); - static_assert(__is_same(remove_volatile_t, int (&)()), ""); - - static_assert(__is_same(remove_volatile_t, S), ""); - static_assert(__is_same(remove_volatile_t, const S), ""); - static_assert(__is_same(remove_volatile_t, S), ""); - static_assert(__is_same(remove_volatile_t, const S), ""); - static_assert(__is_same(remove_volatile_t, int S::*), ""); - static_assert(__is_same(remove_volatile_t, int(S::*)()), ""); + static_assert(__is_same(remove_volatile_t, void)); + static_assert(__is_same(remove_volatile_t, void)); + static_assert(__is_same(remove_volatile_t, int)); + static_assert(__is_same(remove_volatile_t, const int)); + static_assert(__is_same(remove_volatile_t, int)); + static_assert(__is_same(remove_volatile_t, int *__restrict)); + static_assert(__is_same(remove_volatile_t, const int)); + static_assert(__is_same(remove_volatile_t, int *const __restrict)); + static_assert(__is_same(remove_volatile_t, int *)); + static_assert(__is_same(remove_volatile_t, int *)); + static_assert(__is_same(remove_volatile_t, int volatile *)); + static_assert(__is_same(remove_volatile_t, int &)); + static_assert(__is_same(remove_volatile_t, int volatile &)); + static_assert(__is_same(remove_volatile_t, int &&)); + static_assert(__is_same(remove_volatile_t, int volatile &&)); + static_assert(__is_same(remove_volatile_t, int())); + static_assert(__is_same(remove_volatile_t, int (*)())); + static_assert(__is_same(remove_volatile_t, int (&)())); + + static_assert(__is_same(remove_volatile_t, S)); + static_assert(__is_same(remove_volatile_t, const S)); + static_assert(__is_same(remove_volatile_t, S)); + static_assert(__is_same(remove_volatile_t, const S)); + static_assert(__is_same(remove_volatile_t, int S::*)); + static_assert(__is_same(remove_volatile_t, int(S::*)())); } template using remove_cv_t = __remove_cv(T); void check_remove_cv() { - static_assert(__is_same(remove_cv_t, void), ""); - static_assert(__is_same(remove_cv_t, void), ""); - static_assert(__is_same(remove_cv_t, int), ""); - static_assert(__is_same(remove_cv_t, int), ""); - static_assert(__is_same(remove_cv_t, int), ""); - static_assert(__is_same(remove_cv_t, int), ""); - static_assert(__is_same(remove_cv_t, int *), ""); - static_assert(__is_same(remove_cv_t, int *), ""); - static_assert(__is_same(remove_cv_t, int const *), ""); - static_assert(__is_same(remove_cv_t, int const *__restrict), ""); - static_assert(__is_same(remove_cv_t, int const *_Nonnull), ""); - static_assert(__is_same(remove_cv_t, int &), ""); - static_assert(__is_same(remove_cv_t, int const volatile &), ""); - static_assert(__is_same(remove_cv_t, int &&), ""); - static_assert(__is_same(remove_cv_t, int const volatile &&), ""); - static_assert(__is_same(remove_cv_t, int()), ""); - static_assert(__is_same(remove_cv_t, int (*)()), ""); - static_assert(__is_same(remove_cv_t, int (&)()), ""); - - static_assert(__is_same(remove_cv_t, S), ""); - static_assert(__is_same(remove_cv_t, S), ""); - static_assert(__is_same(remove_cv_t, S), ""); - static_assert(__is_same(remove_cv_t, S), ""); - static_assert(__is_same(remove_cv_t, int S::*), ""); - static_assert(__is_same(remove_cv_t, int(S::*)()), ""); + static_assert(__is_same(remove_cv_t, void)); + static_assert(__is_same(remove_cv_t, void)); + static_assert(__is_same(remove_cv_t, int)); + static_assert(__is_same(remove_cv_t, int)); + static_assert(__is_same(remove_cv_t, int)); + static_assert(__is_same(remove_cv_t, int)); + static_assert(__is_same(remove_cv_t, int *)); + static_assert(__is_same(remove_cv_t, int *)); + static_assert(__is_same(remove_cv_t, int const *)); + static_assert(__is_same(remove_cv_t, int const *__restrict)); + static_assert(__is_same(remove_cv_t, int const *_Nonnull)); + static_assert(__is_same(remove_cv_t, int &)); + static_assert(__is_same(remove_cv_t, int const volatile &)); + static_assert(__is_same(remove_cv_t, int &&)); + static_assert(__is_same(remove_cv_t, int const volatile &&)); + static_assert(__is_same(remove_cv_t, int())); + static_assert(__is_same(remove_cv_t, int (*)())); + static_assert(__is_same(remove_cv_t, int (&)())); + + static_assert(__is_same(remove_cv_t, S)); + static_assert(__is_same(remove_cv_t, S)); + static_assert(__is_same(remove_cv_t, S)); + static_assert(__is_same(remove_cv_t, S)); + static_assert(__is_same(remove_cv_t, int S::*)); + static_assert(__is_same(remove_cv_t, int(S::*)())); } template using add_pointer_t = __add_pointer(T); void add_pointer() { - static_assert(__is_same(add_pointer_t, void *), ""); - static_assert(__is_same(add_pointer_t, const void *), ""); - static_assert(__is_same(add_pointer_t, volatile void *), ""); - static_assert(__is_same(add_pointer_t, const volatile void *), ""); - static_assert(__is_same(add_pointer_t, int *), ""); - static_assert(__is_same(add_pointer_t, const int *), ""); - static_assert(__is_same(add_pointer_t, volatile int *), ""); - static_assert(__is_same(add_pointer_t, const volatile int *), ""); - static_assert(__is_same(add_pointer_t, int **), ""); - static_assert(__is_same(add_pointer_t, int *), ""); - static_assert(__is_same(add_pointer_t, int *), ""); - static_assert(__is_same(add_pointer_t, int (*)()), ""); - static_assert(__is_same(add_pointer_t, int (**)()), ""); - static_assert(__is_same(add_pointer_t, int (*)()), ""); - - static_assert(__is_same(add_pointer_t, S *), ""); - static_assert(__is_same(add_pointer_t, const S *), ""); - static_assert(__is_same(add_pointer_t, volatile S *), ""); - static_assert(__is_same(add_pointer_t, const volatile S *), ""); - static_assert(__is_same(add_pointer_t, int S::**), ""); - static_assert(__is_same(add_pointer_t, int(S::**)()), ""); - - static_assert(__is_same(add_pointer_t, int __attribute__((address_space(1))) *), ""); - static_assert(__is_same(add_pointer_t, S __attribute__((address_space(2))) *), ""); + static_assert(__is_same(add_pointer_t, void *)); + static_assert(__is_same(add_pointer_t, const void *)); + static_assert(__is_same(add_pointer_t, volatile void *)); + static_assert(__is_same(add_pointer_t, const volatile void *)); + static_assert(__is_same(add_pointer_t, int *)); + static_assert(__is_same(add_pointer_t, const int *)); + static_assert(__is_same(add_pointer_t, volatile int *)); + static_assert(__is_same(add_pointer_t, const volatile int *)); + static_assert(__is_same(add_pointer_t, int **)); + static_assert(__is_same(add_pointer_t, int *)); + static_assert(__is_same(add_pointer_t, int *)); + static_assert(__is_same(add_pointer_t, int (*)())); + static_assert(__is_same(add_pointer_t, int (**)())); + static_assert(__is_same(add_pointer_t, int (*)())); + + static_assert(__is_same(add_pointer_t, S *)); + static_assert(__is_same(add_pointer_t, const S *)); + static_assert(__is_same(add_pointer_t, volatile S *)); + static_assert(__is_same(add_pointer_t, const volatile S *)); + static_assert(__is_same(add_pointer_t, int S::**)); + static_assert(__is_same(add_pointer_t, int(S::**)())); + + static_assert(__is_same(add_pointer_t, int __attribute__((address_space(1))) *)); + static_assert(__is_same(add_pointer_t, S __attribute__((address_space(2))) *)); } template using remove_pointer_t = __remove_pointer(T); void remove_pointer() { - static_assert(__is_same(remove_pointer_t, void), ""); - static_assert(__is_same(remove_pointer_t, const void), ""); - static_assert(__is_same(remove_pointer_t, volatile void), ""); - static_assert(__is_same(remove_pointer_t, const volatile void), ""); - static_assert(__is_same(remove_pointer_t, int), ""); - static_assert(__is_same(remove_pointer_t, const int), ""); - static_assert(__is_same(remove_pointer_t, volatile int), ""); - static_assert(__is_same(remove_pointer_t, const volatile int), ""); - static_assert(__is_same(remove_pointer_t, int), ""); - static_assert(__is_same(remove_pointer_t, const int), ""); - static_assert(__is_same(remove_pointer_t, volatile int), ""); - static_assert(__is_same(remove_pointer_t, const volatile int), ""); - static_assert(__is_same(remove_pointer_t, int), ""); - static_assert(__is_same(remove_pointer_t, int), ""); - static_assert(__is_same(remove_pointer_t, int), ""); - static_assert(__is_same(remove_pointer_t, int &), ""); - static_assert(__is_same(remove_pointer_t, int &&), ""); - static_assert(__is_same(remove_pointer_t, int()), ""); - static_assert(__is_same(remove_pointer_t, int()), ""); - static_assert(__is_same(remove_pointer_t, int (&)()), ""); - - static_assert(__is_same(remove_pointer_t, S), ""); - static_assert(__is_same(remove_pointer_t, const S), ""); - static_assert(__is_same(remove_pointer_t, volatile S), ""); - static_assert(__is_same(remove_pointer_t, const volatile S), ""); - static_assert(__is_same(remove_pointer_t, int S::*), ""); - static_assert(__is_same(remove_pointer_t, int(S::*)()), ""); - - static_assert(__is_same(remove_pointer_t, int __attribute__((address_space(1)))), ""); - static_assert(__is_same(remove_pointer_t, S __attribute__((address_space(2)))), ""); - - static_assert(__is_same(remove_pointer_t, int (^)(char)), ""); + static_assert(__is_same(remove_pointer_t, void)); + static_assert(__is_same(remove_pointer_t, const void)); + static_assert(__is_same(remove_pointer_t, volatile void)); + static_assert(__is_same(remove_pointer_t, const volatile void)); + static_assert(__is_same(remove_pointer_t, int)); + static_assert(__is_same(remove_pointer_t, const int)); + static_assert(__is_same(remove_pointer_t, volatile int)); + static_assert(__is_same(remove_pointer_t, const volatile int)); + static_assert(__is_same(remove_pointer_t, int)); + static_assert(__is_same(remove_pointer_t, const int)); + static_assert(__is_same(remove_pointer_t, volatile int)); + static_assert(__is_same(remove_pointer_t, const volatile int)); + static_assert(__is_same(remove_pointer_t, int)); + static_assert(__is_same(remove_pointer_t, int)); + static_assert(__is_same(remove_pointer_t, int)); + static_assert(__is_same(remove_pointer_t, int &)); + static_assert(__is_same(remove_pointer_t, int &&)); + static_assert(__is_same(remove_pointer_t, int())); + static_assert(__is_same(remove_pointer_t, int())); + static_assert(__is_same(remove_pointer_t, int (&)())); + + static_assert(__is_same(remove_pointer_t, S)); + static_assert(__is_same(remove_pointer_t, const S)); + static_assert(__is_same(remove_pointer_t, volatile S)); + static_assert(__is_same(remove_pointer_t, const volatile S)); + static_assert(__is_same(remove_pointer_t, int S::*)); + static_assert(__is_same(remove_pointer_t, int(S::*)())); + + static_assert(__is_same(remove_pointer_t, int __attribute__((address_space(1))))); + static_assert(__is_same(remove_pointer_t, S __attribute__((address_space(2))))); + + static_assert(__is_same(remove_pointer_t, int (^)(char))); } template using add_lvalue_reference_t = __add_lvalue_reference(T); void add_lvalue_reference() { - static_assert(__is_same(add_lvalue_reference_t, void), ""); - static_assert(__is_same(add_lvalue_reference_t, const void), ""); - static_assert(__is_same(add_lvalue_reference_t, volatile void), ""); - static_assert(__is_same(add_lvalue_reference_t, const volatile void), ""); - static_assert(__is_same(add_lvalue_reference_t, int &), ""); - static_assert(__is_same(add_lvalue_reference_t, const int &), ""); - static_assert(__is_same(add_lvalue_reference_t, volatile int &), ""); - static_assert(__is_same(add_lvalue_reference_t, const volatile int &), ""); - static_assert(__is_same(add_lvalue_reference_t, int *&), ""); - static_assert(__is_same(add_lvalue_reference_t, int &), ""); - static_assert(__is_same(add_lvalue_reference_t, int &), ""); // reference collapsing - static_assert(__is_same(add_lvalue_reference_t, int (&)()), ""); - static_assert(__is_same(add_lvalue_reference_t, int (*&)()), ""); - static_assert(__is_same(add_lvalue_reference_t, int (&)()), ""); - - static_assert(__is_same(add_lvalue_reference_t, S &), ""); - static_assert(__is_same(add_lvalue_reference_t, const S &), ""); - static_assert(__is_same(add_lvalue_reference_t, volatile S &), ""); - static_assert(__is_same(add_lvalue_reference_t, const volatile S &), ""); - static_assert(__is_same(add_lvalue_reference_t, int S::*&), ""); - static_assert(__is_same(add_lvalue_reference_t, int(S::*&)()), ""); + static_assert(__is_same(add_lvalue_reference_t, void)); + static_assert(__is_same(add_lvalue_reference_t, const void)); + static_assert(__is_same(add_lvalue_reference_t, volatile void)); + static_assert(__is_same(add_lvalue_reference_t, const volatile void)); + static_assert(__is_same(add_lvalue_reference_t, int &)); + static_assert(__is_same(add_lvalue_reference_t, const int &)); + static_assert(__is_same(add_lvalue_reference_t, volatile int &)); + static_assert(__is_same(add_lvalue_reference_t, const volatile int &)); + static_assert(__is_same(add_lvalue_reference_t, int *&)); + static_assert(__is_same(add_lvalue_reference_t, int &)); + static_assert(__is_same(add_lvalue_reference_t, int &)); // reference collapsing + static_assert(__is_same(add_lvalue_reference_t, int (&)())); + static_assert(__is_same(add_lvalue_reference_t, int (*&)())); + static_assert(__is_same(add_lvalue_reference_t, int (&)())); + + static_assert(__is_same(add_lvalue_reference_t, S &)); + static_assert(__is_same(add_lvalue_reference_t, const S &)); + static_assert(__is_same(add_lvalue_reference_t, volatile S &)); + static_assert(__is_same(add_lvalue_reference_t, const volatile S &)); + static_assert(__is_same(add_lvalue_reference_t, int S::*&)); + static_assert(__is_same(add_lvalue_reference_t, int(S::*&)())); } template using add_rvalue_reference_t = __add_rvalue_reference(T); void add_rvalue_reference() { - static_assert(__is_same(add_rvalue_reference_t, void), ""); - static_assert(__is_same(add_rvalue_reference_t, const void), ""); - static_assert(__is_same(add_rvalue_reference_t, volatile void), ""); - static_assert(__is_same(add_rvalue_reference_t, const volatile void), ""); - static_assert(__is_same(add_rvalue_reference_t, int &&), ""); - static_assert(__is_same(add_rvalue_reference_t, const int &&), ""); - static_assert(__is_same(add_rvalue_reference_t, volatile int &&), ""); - static_assert(__is_same(add_rvalue_reference_t, const volatile int &&), ""); - static_assert(__is_same(add_rvalue_reference_t, int *&&), ""); - static_assert(__is_same(add_rvalue_reference_t, int &), ""); // reference collapsing - static_assert(__is_same(add_rvalue_reference_t, int &&), ""); - static_assert(__is_same(add_rvalue_reference_t, int(&&)()), ""); - static_assert(__is_same(add_rvalue_reference_t, int (*&&)()), ""); - static_assert(__is_same(add_rvalue_reference_t, int (&)()), ""); // reference collapsing - - static_assert(__is_same(add_rvalue_reference_t, S &&), ""); - static_assert(__is_same(add_rvalue_reference_t, const S &&), ""); - static_assert(__is_same(add_rvalue_reference_t, volatile S &&), ""); - static_assert(__is_same(add_rvalue_reference_t, const volatile S &&), ""); - static_assert(__is_same(add_rvalue_reference_t, int S::*&&), ""); - static_assert(__is_same(add_rvalue_reference_t, int(S::* &&)()), ""); + static_assert(__is_same(add_rvalue_reference_t, void)); + static_assert(__is_same(add_rvalue_reference_t, const void)); + static_assert(__is_same(add_rvalue_reference_t, volatile void)); + static_assert(__is_same(add_rvalue_reference_t, const volatile void)); + static_assert(__is_same(add_rvalue_reference_t, int &&)); + static_assert(__is_same(add_rvalue_reference_t, const int &&)); + static_assert(__is_same(add_rvalue_reference_t, volatile int &&)); + static_assert(__is_same(add_rvalue_reference_t, const volatile int &&)); + static_assert(__is_same(add_rvalue_reference_t, int *&&)); + static_assert(__is_same(add_rvalue_reference_t, int &)); // reference collapsing + static_assert(__is_same(add_rvalue_reference_t, int &&)); + static_assert(__is_same(add_rvalue_reference_t, int(&&)())); + static_assert(__is_same(add_rvalue_reference_t, int (*&&)())); + static_assert(__is_same(add_rvalue_reference_t, int (&)())); // reference collapsing + + static_assert(__is_same(add_rvalue_reference_t, S &&)); + static_assert(__is_same(add_rvalue_reference_t, const S &&)); + static_assert(__is_same(add_rvalue_reference_t, volatile S &&)); + static_assert(__is_same(add_rvalue_reference_t, const volatile S &&)); + static_assert(__is_same(add_rvalue_reference_t, int S::*&&)); + static_assert(__is_same(add_rvalue_reference_t, int(S::* &&)())); } template using remove_reference_t = __remove_reference_t(T); void check_remove_reference() { - static_assert(__is_same(remove_reference_t, void), ""); - static_assert(__is_same(remove_reference_t, const volatile void), ""); - static_assert(__is_same(remove_reference_t, int), ""); - static_assert(__is_same(remove_reference_t, const int), ""); - static_assert(__is_same(remove_reference_t, volatile int), ""); - static_assert(__is_same(remove_reference_t, const volatile int), ""); - static_assert(__is_same(remove_reference_t, int *), ""); - static_assert(__is_same(remove_reference_t, int *const volatile), ""); - static_assert(__is_same(remove_reference_t, int const *const volatile), ""); - static_assert(__is_same(remove_reference_t, int), ""); - static_assert(__is_same(remove_reference_t, int const volatile), ""); - static_assert(__is_same(remove_reference_t, int), ""); - static_assert(__is_same(remove_reference_t, int const volatile), ""); - static_assert(__is_same(remove_reference_t, int()), ""); - static_assert(__is_same(remove_reference_t, int (*const volatile)()), ""); - static_assert(__is_same(remove_reference_t, int()), ""); - - static_assert(__is_same(remove_reference_t, S), ""); - static_assert(__is_same(remove_reference_t, S), ""); - static_assert(__is_same(remove_reference_t, S), ""); - static_assert(__is_same(remove_reference_t, const S), ""); - static_assert(__is_same(remove_reference_t, const S), ""); - static_assert(__is_same(remove_reference_t, const S), ""); - static_assert(__is_same(remove_reference_t, volatile S), ""); - static_assert(__is_same(remove_reference_t, volatile S), ""); - static_assert(__is_same(remove_reference_t, volatile S), ""); - static_assert(__is_same(remove_reference_t, const volatile S), ""); - static_assert(__is_same(remove_reference_t, const volatile S), ""); - static_assert(__is_same(remove_reference_t, const volatile S), ""); - static_assert(__is_same(remove_reference_t, int S::*const volatile), ""); - static_assert(__is_same(remove_reference_t, int(S::*const volatile)()), ""); - static_assert(__is_same(remove_reference_t, int(S::*const volatile)() &), ""); + static_assert(__is_same(remove_reference_t, void)); + static_assert(__is_same(remove_reference_t, const volatile void)); + static_assert(__is_same(remove_reference_t, int)); + static_assert(__is_same(remove_reference_t, const int)); + static_assert(__is_same(remove_reference_t, volatile int)); + static_assert(__is_same(remove_reference_t, const volatile int)); + static_assert(__is_same(remove_reference_t, int *)); + static_assert(__is_same(remove_reference_t, int *const volatile)); + static_assert(__is_same(remove_reference_t, int const *const volatile)); + static_assert(__is_same(remove_reference_t, int)); + static_assert(__is_same(remove_reference_t, int const volatile)); + static_assert(__is_same(remove_reference_t, int)); + static_assert(__is_same(remove_reference_t, int const volatile)); + static_assert(__is_same(remove_reference_t, int())); + static_assert(__is_same(remove_reference_t, int (*const volatile)())); + static_assert(__is_same(remove_reference_t, int())); + + static_assert(__is_same(remove_reference_t, S)); + static_assert(__is_same(remove_reference_t, S)); + static_assert(__is_same(remove_reference_t, S)); + static_assert(__is_same(remove_reference_t, const S)); + static_assert(__is_same(remove_reference_t, const S)); + static_assert(__is_same(remove_reference_t, const S)); + static_assert(__is_same(remove_reference_t, volatile S)); + static_assert(__is_same(remove_reference_t, volatile S)); + static_assert(__is_same(remove_reference_t, volatile S)); + static_assert(__is_same(remove_reference_t, const volatile S)); + static_assert(__is_same(remove_reference_t, const volatile S)); + static_assert(__is_same(remove_reference_t, const volatile S)); + static_assert(__is_same(remove_reference_t, int S::*const volatile)); + static_assert(__is_same(remove_reference_t, int(S::*const volatile)())); + static_assert(__is_same(remove_reference_t, int(S::*const volatile)() &)); } template using remove_cvref_t = __remove_cvref(T); void check_remove_cvref() { - static_assert(__is_same(remove_cvref_t, void), ""); - static_assert(__is_same(remove_cvref_t, void), ""); - static_assert(__is_same(remove_cvref_t, int), ""); - static_assert(__is_same(remove_cvref_t, int), ""); - static_assert(__is_same(remove_cvref_t, int), ""); - static_assert(__is_same(remove_cvref_t, int), ""); - static_assert(__is_same(remove_cvref_t, int *), ""); - static_assert(__is_same(remove_cvref_t, int *), ""); - static_assert(__is_same(remove_cvref_t, int const *), ""); - static_assert(__is_same(remove_cvref_t, int const *__restrict), ""); - static_assert(__is_same(remove_cvref_t, int const *_Nonnull), ""); - static_assert(__is_same(remove_cvref_t, int), ""); - static_assert(__is_same(remove_cvref_t, int), ""); - static_assert(__is_same(remove_cvref_t, int), ""); - static_assert(__is_same(remove_cvref_t, int), ""); - static_assert(__is_same(remove_cvref_t, int()), ""); - static_assert(__is_same(remove_cvref_t, int (*)()), ""); - static_assert(__is_same(remove_cvref_t, int()), ""); - - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, int S::*), ""); - static_assert(__is_same(remove_cvref_t, int(S::*)()), ""); - static_assert(__is_same(remove_cvref_t, int(S::*)() &), ""); - static_assert(__is_same(remove_cvref_t, int(S::*)() &&), ""); + static_assert(__is_same(remove_cvref_t, void)); + static_assert(__is_same(remove_cvref_t, void)); + static_assert(__is_same(remove_cvref_t, int)); + static_assert(__is_same(remove_cvref_t, int)); + static_assert(__is_same(remove_cvref_t, int)); + static_assert(__is_same(remove_cvref_t, int)); + static_assert(__is_same(remove_cvref_t, int *)); + static_assert(__is_same(remove_cvref_t, int *)); + static_assert(__is_same(remove_cvref_t, int const *)); + static_assert(__is_same(remove_cvref_t, int const *__restrict)); + static_assert(__is_same(remove_cvref_t, int const *_Nonnull)); + static_assert(__is_same(remove_cvref_t, int)); + static_assert(__is_same(remove_cvref_t, int)); + static_assert(__is_same(remove_cvref_t, int)); + static_assert(__is_same(remove_cvref_t, int)); + static_assert(__is_same(remove_cvref_t, int())); + static_assert(__is_same(remove_cvref_t, int (*)())); + static_assert(__is_same(remove_cvref_t, int())); + + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, int S::*)); + static_assert(__is_same(remove_cvref_t, int(S::*)())); + static_assert(__is_same(remove_cvref_t, int(S::*)() &)); + static_assert(__is_same(remove_cvref_t, int(S::*)() &&)); } template using decay_t = __decay(T); void check_decay() { - static_assert(__is_same(decay_t, void), ""); - static_assert(__is_same(decay_t, void), ""); - static_assert(__is_same(decay_t, int), ""); - static_assert(__is_same(decay_t, int), ""); - static_assert(__is_same(decay_t, int), ""); - static_assert(__is_same(decay_t, int), ""); - static_assert(__is_same(decay_t, int *), ""); - static_assert(__is_same(decay_t, int *), ""); - static_assert(__is_same(decay_t, int *), ""); - static_assert(__is_same(decay_t, int const *), ""); - static_assert(__is_same(decay_t, int const *), ""); - static_assert(__is_same(decay_t, int), ""); - static_assert(__is_same(decay_t, int), ""); - static_assert(__is_same(decay_t, int), ""); - static_assert(__is_same(decay_t, int), ""); - static_assert(__is_same(decay_t, int (*)()), ""); - static_assert(__is_same(decay_t, int (*)()), ""); - static_assert(__is_same(decay_t, int (*)()), ""); - static_assert(__is_same(decay_t, int (*)()), ""); - static_assert(__is_same(decay_t, int (*)()), ""); - static_assert(__is_same(decay_t, int (*)()), ""); - static_assert(__is_same(decay_t, int *), ""); - static_assert(__is_same(decay_t, int *), ""); - - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, int S::*), ""); - static_assert(__is_same(decay_t, int(S::*)()), ""); - static_assert(__is_same(decay_t, int S::*), ""); - static_assert(__is_same(decay_t, int(S::*)()), ""); - static_assert(__is_same(decay_t, int S::*), ""); + static_assert(__is_same(decay_t, void)); + static_assert(__is_same(decay_t, void)); + static_assert(__is_same(decay_t, int)); + static_assert(__is_same(decay_t, int)); + static_assert(__is_same(decay_t, int)); + static_assert(__is_same(decay_t, int)); + static_assert(__is_same(decay_t, int *)); + static_assert(__is_same(decay_t, int *)); + static_assert(__is_same(decay_t, int *)); + static_assert(__is_same(decay_t, int const *)); + static_assert(__is_same(decay_t, int const *)); + static_assert(__is_same(decay_t, int)); + static_assert(__is_same(decay_t, int)); + static_assert(__is_same(decay_t, int)); + static_assert(__is_same(decay_t, int)); + static_assert(__is_same(decay_t, int (*)())); + static_assert(__is_same(decay_t, int (*)())); + static_assert(__is_same(decay_t, int (*)())); + static_assert(__is_same(decay_t, int (*)())); + static_assert(__is_same(decay_t, int (*)())); + static_assert(__is_same(decay_t, int (*)())); + static_assert(__is_same(decay_t, int *)); + static_assert(__is_same(decay_t, int *)); + + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, int S::*)); + static_assert(__is_same(decay_t, int(S::*)())); + static_assert(__is_same(decay_t, int S::*)); + static_assert(__is_same(decay_t, int(S::*)())); + static_assert(__is_same(decay_t, int S::*)); } template struct CheckAbominableFunction {}; template struct CheckAbominableFunction { static void checks() { - static_assert(__is_same(add_lvalue_reference_t, M), ""); - static_assert(__is_same(add_pointer_t, M), ""); - static_assert(__is_same(add_rvalue_reference_t, M), ""); - static_assert(__is_same(decay_t, M), ""); - static_assert(__is_same(remove_const_t, M), ""); - static_assert(__is_same(remove_volatile_t, M), ""); - static_assert(__is_same(remove_cv_t, M), ""); - static_assert(__is_same(remove_cvref_t, M), ""); - static_assert(__is_same(remove_pointer_t, M), ""); - static_assert(__is_same(remove_reference_t, M), ""); - - static_assert(!__is_referenceable(M), ""); + static_assert(__is_same(add_lvalue_reference_t, M)); + static_assert(__is_same(add_pointer_t, M)); + static_assert(__is_same(add_rvalue_reference_t, M)); + static_assert(__is_same(decay_t, M)); + static_assert(__is_same(remove_const_t, M)); + static_assert(__is_same(remove_volatile_t, M)); + static_assert(__is_same(remove_cv_t, M)); + static_assert(__is_same(remove_cvref_t, M)); + static_assert(__is_same(remove_pointer_t, M)); + static_assert(__is_same(remove_reference_t, M)); + + static_assert(!__is_referenceable(M)); } }; @@ -4237,10 +4220,10 @@ void check_abominable_function() { template using make_signed_t = __make_signed(T); template void check_make_signed() { - static_assert(__is_same(make_signed_t, Expected), ""); - static_assert(__is_same(make_signed_t, const Expected), ""); - static_assert(__is_same(make_signed_t, volatile Expected), ""); - static_assert(__is_same(make_signed_t, const volatile Expected), ""); + static_assert(__is_same(make_signed_t, Expected)); + static_assert(__is_same(make_signed_t, const Expected)); + static_assert(__is_same(make_signed_t, volatile Expected)); + static_assert(__is_same(make_signed_t, const volatile Expected)); } #if defined(__ILP32__) || defined(__LLP64__) @@ -4350,10 +4333,10 @@ using make_unsigned_t = __make_unsigned(T); template void check_make_unsigned() { - static_assert(__is_same(make_unsigned_t, Expected), ""); - static_assert(__is_same(make_unsigned_t, const Expected), ""); - static_assert(__is_same(make_unsigned_t, volatile Expected), ""); - static_assert(__is_same(make_unsigned_t, const volatile Expected), ""); + static_assert(__is_same(make_unsigned_t, Expected)); + static_assert(__is_same(make_unsigned_t, const Expected)); + static_assert(__is_same(make_unsigned_t, volatile Expected)); + static_assert(__is_same(make_unsigned_t, const volatile Expected)); } void make_unsigned() { @@ -4436,74 +4419,74 @@ void make_unsigned() { template using remove_extent_t = __remove_extent(T); void remove_extent() { - static_assert(__is_same(remove_extent_t, void), ""); - static_assert(__is_same(remove_extent_t, int), ""); - static_assert(__is_same(remove_extent_t, int), ""); - static_assert(__is_same(remove_extent_t, int), ""); - static_assert(__is_same(remove_extent_t, int[2]), ""); - static_assert(__is_same(remove_extent_t, int[2]), ""); - static_assert(__is_same(remove_extent_t, const int), ""); - static_assert(__is_same(remove_extent_t, const int), ""); - static_assert(__is_same(remove_extent_t, const int[2]), ""); - static_assert(__is_same(remove_extent_t, const int[2]), ""); - static_assert(__is_same(remove_extent_t, volatile int), ""); - static_assert(__is_same(remove_extent_t, volatile int), ""); - static_assert(__is_same(remove_extent_t, volatile int[2]), ""); - static_assert(__is_same(remove_extent_t, volatile int[2]), ""); - static_assert(__is_same(remove_extent_t, const volatile int), ""); - static_assert(__is_same(remove_extent_t, const volatile int), ""); - static_assert(__is_same(remove_extent_t, const volatile int[2]), ""); - static_assert(__is_same(remove_extent_t, const volatile int[2]), ""); - static_assert(__is_same(remove_extent_t, int *), ""); - static_assert(__is_same(remove_extent_t, int &), ""); - static_assert(__is_same(remove_extent_t, int &&), ""); - static_assert(__is_same(remove_extent_t, int()), ""); - static_assert(__is_same(remove_extent_t, int (*)()), ""); - static_assert(__is_same(remove_extent_t, int (&)()), ""); - - static_assert(__is_same(remove_extent_t, S), ""); - static_assert(__is_same(remove_extent_t, int S::*), ""); - static_assert(__is_same(remove_extent_t, int(S::*)()), ""); + static_assert(__is_same(remove_extent_t, void)); + static_assert(__is_same(remove_extent_t, int)); + static_assert(__is_same(remove_extent_t, int)); + static_assert(__is_same(remove_extent_t, int)); + static_assert(__is_same(remove_extent_t, int[2])); + static_assert(__is_same(remove_extent_t, int[2])); + static_assert(__is_same(remove_extent_t, const int)); + static_assert(__is_same(remove_extent_t, const int)); + static_assert(__is_same(remove_extent_t, const int[2])); + static_assert(__is_same(remove_extent_t, const int[2])); + static_assert(__is_same(remove_extent_t, volatile int)); + static_assert(__is_same(remove_extent_t, volatile int)); + static_assert(__is_same(remove_extent_t, volatile int[2])); + static_assert(__is_same(remove_extent_t, volatile int[2])); + static_assert(__is_same(remove_extent_t, const volatile int)); + static_assert(__is_same(remove_extent_t, const volatile int)); + static_assert(__is_same(remove_extent_t, const volatile int[2])); + static_assert(__is_same(remove_extent_t, const volatile int[2])); + static_assert(__is_same(remove_extent_t, int *)); + static_assert(__is_same(remove_extent_t, int &)); + static_assert(__is_same(remove_extent_t, int &&)); + static_assert(__is_same(remove_extent_t, int())); + static_assert(__is_same(remove_extent_t, int (*)())); + static_assert(__is_same(remove_extent_t, int (&)())); + + static_assert(__is_same(remove_extent_t, S)); + static_assert(__is_same(remove_extent_t, int S::*)); + static_assert(__is_same(remove_extent_t, int(S::*)())); using SomeArray = int[1][2]; - static_assert(__is_same(remove_extent_t, const int[2]), ""); + static_assert(__is_same(remove_extent_t, const int[2])); } template using remove_all_extents_t = __remove_all_extents(T); void remove_all_extents() { - static_assert(__is_same(remove_all_extents_t, void), ""); - static_assert(__is_same(remove_all_extents_t, int), ""); - static_assert(__is_same(remove_all_extents_t, const int), ""); - static_assert(__is_same(remove_all_extents_t, volatile int), ""); - static_assert(__is_same(remove_all_extents_t, const volatile int), ""); - static_assert(__is_same(remove_all_extents_t, int), ""); - static_assert(__is_same(remove_all_extents_t, int), ""); - static_assert(__is_same(remove_all_extents_t, int), ""); - static_assert(__is_same(remove_all_extents_t, int), ""); - static_assert(__is_same(remove_all_extents_t, const int), ""); - static_assert(__is_same(remove_all_extents_t, const int), ""); - static_assert(__is_same(remove_all_extents_t, const int), ""); - static_assert(__is_same(remove_all_extents_t, const int), ""); - static_assert(__is_same(remove_all_extents_t, volatile int), ""); - static_assert(__is_same(remove_all_extents_t, volatile int), ""); - static_assert(__is_same(remove_all_extents_t, volatile int), ""); - static_assert(__is_same(remove_all_extents_t, volatile int), ""); - static_assert(__is_same(remove_all_extents_t, const volatile int), ""); - static_assert(__is_same(remove_all_extents_t, const volatile int), ""); - static_assert(__is_same(remove_all_extents_t, const volatile int), ""); - static_assert(__is_same(remove_all_extents_t, const volatile int), ""); - static_assert(__is_same(remove_all_extents_t, int *), ""); - static_assert(__is_same(remove_all_extents_t, int &), ""); - static_assert(__is_same(remove_all_extents_t, int &&), ""); - static_assert(__is_same(remove_all_extents_t, int()), ""); - static_assert(__is_same(remove_all_extents_t, int (*)()), ""); - static_assert(__is_same(remove_all_extents_t, int (&)()), ""); - - static_assert(__is_same(remove_all_extents_t, S), ""); - static_assert(__is_same(remove_all_extents_t, int S::*), ""); - static_assert(__is_same(remove_all_extents_t, int(S::*)()), ""); + static_assert(__is_same(remove_all_extents_t, void)); + static_assert(__is_same(remove_all_extents_t, int)); + static_assert(__is_same(remove_all_extents_t, const int)); + static_assert(__is_same(remove_all_extents_t, volatile int)); + static_assert(__is_same(remove_all_extents_t, const volatile int)); + static_assert(__is_same(remove_all_extents_t, int)); + static_assert(__is_same(remove_all_extents_t, int)); + static_assert(__is_same(remove_all_extents_t, int)); + static_assert(__is_same(remove_all_extents_t, int)); + static_assert(__is_same(remove_all_extents_t, const int)); + static_assert(__is_same(remove_all_extents_t, const int)); + static_assert(__is_same(remove_all_extents_t, const int)); + static_assert(__is_same(remove_all_extents_t, const int)); + static_assert(__is_same(remove_all_extents_t, volatile int)); + static_assert(__is_same(remove_all_extents_t, volatile int)); + static_assert(__is_same(remove_all_extents_t, volatile int)); + static_assert(__is_same(remove_all_extents_t, volatile int)); + static_assert(__is_same(remove_all_extents_t, const volatile int)); + static_assert(__is_same(remove_all_extents_t, const volatile int)); + static_assert(__is_same(remove_all_extents_t, const volatile int)); + static_assert(__is_same(remove_all_extents_t, const volatile int)); + static_assert(__is_same(remove_all_extents_t, int *)); + static_assert(__is_same(remove_all_extents_t, int &)); + static_assert(__is_same(remove_all_extents_t, int &&)); + static_assert(__is_same(remove_all_extents_t, int())); + static_assert(__is_same(remove_all_extents_t, int (*)())); + static_assert(__is_same(remove_all_extents_t, int (&)())); + + static_assert(__is_same(remove_all_extents_t, S)); + static_assert(__is_same(remove_all_extents_t, int S::*)); + static_assert(__is_same(remove_all_extents_t, int(S::*)())); using SomeArray = int[1][2]; - static_assert(__is_same(remove_all_extents_t, const int), ""); + static_assert(__is_same(remove_all_extents_t, const int)); } -- GitLab From 235332150d52d11b340a10be1bb88432d2cf4179 Mon Sep 17 00:00:00 2001 From: David Spickett Date: Fri, 8 Mar 2024 11:27:54 +0000 Subject: [PATCH 592/929] [lldb][Docs] Add Curses version note to build page This explains a thing that hit me on FreeBSD because the base system has an ncursesw at one version and I installed from pkg another version that was simply ncurses (no wide char support). For whatever reason, when we pass -lcurses to the linker it ends up picking bits of both installs. This led to lldb crashing immediately if you tried to use the `gui` command. In a way that gave little information but I stumbled onto https://github.com/vifm/vifm/issues/325 which is very similar. ``` ec2-user@freebsd:~/build-llvm $ ldd ./bin/lldb | grep curses libncursesw.so.9 => /lib/libncursesw.so.9 (0x6a515206e000) libncurses.so.6 => /usr/local/lib/libncurses.so.6 (0x6a5158e86000) ``` We should only see one version, and it and libpanel etc should all have "w" or not have "w". This was not the case for my build. What I can see from the CMake side seemed fine, it found the pkg installed ncurses in /usr/local. Something else must decide that -lcurses should pull in the other one. Regardless, I don't know how to fix that but the solution for most people is just not to add another ncurses if they already have one. So I've added a note saying so, and how to check what your lldb is using. --- lldb/docs/resources/build.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lldb/docs/resources/build.rst b/lldb/docs/resources/build.rst index fe8e293db642..5f4d35ced623 100644 --- a/lldb/docs/resources/build.rst +++ b/lldb/docs/resources/build.rst @@ -85,6 +85,13 @@ commands below. versions of python unusable. This primarily affects Windows, as debug builds of LLDB must use debug python as well. +.. note:: + Installing multiple versions of Curses, particularly when only one is built with + wide character support, can cause lldb to be linked with an incorrect set of + libraries. If your system already has Curses, we recommend you use that version. + If you do install another one, use a tool like ``ldd`` to ensure only one version + of Curses is being used in the final ``lldb`` executable. + Windows ******* -- GitLab From 4d478bcb4fc4fcf6532b725a0d070cb0c0cf9430 Mon Sep 17 00:00:00 2001 From: Dmitry Chernenkov Date: Fri, 8 Mar 2024 11:53:15 +0000 Subject: [PATCH 593/929] [bazel] Add support_uint dep in libc unittest --- utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel index 44692947af7c..37eb21c77e49 100644 --- a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel @@ -126,5 +126,6 @@ libc_support_library( deps = [ "//libc:__support_cpp_string", "//libc:__support_cpp_type_traits", + "//libc:__support_uint", ], ) -- GitLab From bd6eb54886ad12fb523e924e7291abfa2b010e3c Mon Sep 17 00:00:00 2001 From: Paul Walker Date: Fri, 8 Mar 2024 12:09:05 +0000 Subject: [PATCH 594/929] [LLVM][CodeGen] Teach SelectionDAG how to expand FREM to a vector math call. (#83859) This removes, at least when a vector library is available, a failure case for scalable vectors. Doing so means we can confidently cost vector FREM instructions without making an assumption that later passes will transform the IR before it gets to the code generator. NOTE: Whilst only FREM has been implemented the same mechanism can be used for the other libm related ISD nodes. --- .../SelectionDAG/LegalizeVectorOps.cpp | 128 ++++++++++++++++++ llvm/lib/CodeGen/TargetPassConfig.cpp | 6 +- .../CodeGen/AArch64/fp-veclib-expansion.ll | 116 ++++++++++++++++ 3 files changed, 249 insertions(+), 1 deletion(-) create mode 100644 llvm/test/CodeGen/AArch64/fp-veclib-expansion.ll diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp index 6074498d9144..57a3f6a65e00 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp @@ -28,6 +28,8 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/VectorUtils.h" #include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/SelectionDAGNodes.h" @@ -147,6 +149,14 @@ class VectorLegalizer { void ExpandStrictFPOp(SDNode *Node, SmallVectorImpl &Results); void ExpandREM(SDNode *Node, SmallVectorImpl &Results); + bool tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall LC, + SmallVectorImpl &Results); + bool tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall Call_F32, + RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, + RTLIB::Libcall Call_F128, + RTLIB::Libcall Call_PPCF128, + SmallVectorImpl &Results); + void UnrollStrictFPOp(SDNode *Node, SmallVectorImpl &Results); /// Implements vector promotion. @@ -1139,6 +1149,13 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl &Results) { case ISD::VP_MERGE: Results.push_back(ExpandVP_MERGE(Node)); return; + case ISD::FREM: + if (tryExpandVecMathCall(Node, RTLIB::REM_F32, RTLIB::REM_F64, + RTLIB::REM_F80, RTLIB::REM_F128, + RTLIB::REM_PPCF128, Results)) + return; + + break; } SDValue Unrolled = DAG.UnrollVectorOp(Node); @@ -1842,6 +1859,117 @@ void VectorLegalizer::ExpandREM(SDNode *Node, Results.push_back(Result); } +// Try to expand libm nodes into vector math routine calls. Callers provide the +// LibFunc equivalent of the passed in Node, which is used to lookup mappings +// within TargetLibraryInfo. The only mappings considered are those where the +// result and all operands are the same vector type. While predicated nodes are +// not supported, we will emit calls to masked routines by passing in an all +// true mask. +bool VectorLegalizer::tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall LC, + SmallVectorImpl &Results) { + // Chain must be propagated but currently strict fp operations are down + // converted to their none strict counterpart. + assert(!Node->isStrictFPOpcode() && "Unexpected strict fp operation!"); + + const char *LCName = TLI.getLibcallName(LC); + if (!LCName) + return false; + LLVM_DEBUG(dbgs() << "Looking for vector variant of " << LCName << "\n"); + + EVT VT = Node->getValueType(0); + ElementCount VL = VT.getVectorElementCount(); + + // Lookup a vector function equivalent to the specified libcall. Prefer + // unmasked variants but we will generate a mask if need be. + const TargetLibraryInfo &TLibInfo = DAG.getLibInfo(); + const VecDesc *VD = TLibInfo.getVectorMappingInfo(LCName, VL, false); + if (!VD) + VD = TLibInfo.getVectorMappingInfo(LCName, VL, /*Masked=*/true); + if (!VD) + return false; + + LLVMContext *Ctx = DAG.getContext(); + Type *Ty = VT.getTypeForEVT(*Ctx); + Type *ScalarTy = Ty->getScalarType(); + + // Construct a scalar function type based on Node's operands. + SmallVector ArgTys; + for (unsigned i = 0; i < Node->getNumOperands(); ++i) { + assert(Node->getOperand(i).getValueType() == VT && + "Expected matching vector types!"); + ArgTys.push_back(ScalarTy); + } + FunctionType *ScalarFTy = FunctionType::get(ScalarTy, ArgTys, false); + + // Generate call information for the vector function. + const std::string MangledName = VD->getVectorFunctionABIVariantString(); + auto OptVFInfo = VFABI::tryDemangleForVFABI(MangledName, ScalarFTy); + if (!OptVFInfo) + return false; + + LLVM_DEBUG(dbgs() << "Found vector variant " << VD->getVectorFnName() + << "\n"); + + // Sanity check just in case OptVFInfo has unexpected parameters. + if (OptVFInfo->Shape.Parameters.size() != + Node->getNumOperands() + VD->isMasked()) + return false; + + // Collect vector call operands. + + SDLoc DL(Node); + TargetLowering::ArgListTy Args; + TargetLowering::ArgListEntry Entry; + Entry.IsSExt = false; + Entry.IsZExt = false; + + unsigned OpNum = 0; + for (auto &VFParam : OptVFInfo->Shape.Parameters) { + if (VFParam.ParamKind == VFParamKind::GlobalPredicate) { + EVT MaskVT = TLI.getSetCCResultType(DAG.getDataLayout(), *Ctx, VT); + Entry.Node = DAG.getBoolConstant(true, DL, MaskVT, VT); + Entry.Ty = MaskVT.getTypeForEVT(*Ctx); + Args.push_back(Entry); + continue; + } + + // Only vector operands are supported. + if (VFParam.ParamKind != VFParamKind::Vector) + return false; + + Entry.Node = Node->getOperand(OpNum++); + Entry.Ty = Ty; + Args.push_back(Entry); + } + + // Emit a call to the vector function. + SDValue Callee = DAG.getExternalSymbol(VD->getVectorFnName().data(), + TLI.getPointerTy(DAG.getDataLayout())); + TargetLowering::CallLoweringInfo CLI(DAG); + CLI.setDebugLoc(DL) + .setChain(DAG.getEntryNode()) + .setLibCallee(CallingConv::C, Ty, Callee, std::move(Args)); + + std::pair CallResult = TLI.LowerCallTo(CLI); + Results.push_back(CallResult.first); + return true; +} + +/// Try to expand the node to a vector libcall based on the result type. +bool VectorLegalizer::tryExpandVecMathCall( + SDNode *Node, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, + RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, + RTLIB::Libcall Call_PPCF128, SmallVectorImpl &Results) { + RTLIB::Libcall LC = RTLIB::getFPLibCall( + Node->getValueType(0).getVectorElementType(), Call_F32, Call_F64, + Call_F80, Call_F128, Call_PPCF128); + + if (LC == RTLIB::UNKNOWN_LIBCALL) + return false; + + return tryExpandVecMathCall(Node, LC, Results); +} + void VectorLegalizer::UnrollStrictFPOp(SDNode *Node, SmallVectorImpl &Results) { EVT VT = Node->getValueType(0); diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp index cf068ece8d4c..8832b51333d9 100644 --- a/llvm/lib/CodeGen/TargetPassConfig.cpp +++ b/llvm/lib/CodeGen/TargetPassConfig.cpp @@ -205,6 +205,10 @@ static cl::opt MISchedPostRA( static cl::opt EarlyLiveIntervals("early-live-intervals", cl::Hidden, cl::desc("Run live interval analysis earlier in the pipeline")); +static cl::opt DisableReplaceWithVecLib( + "disable-replace-with-vec-lib", cl::Hidden, + cl::desc("Disable replace with vector math call pass")); + /// Option names for limiting the codegen pipeline. /// Those are used in error reporting and we didn't want /// to duplicate their names all over the place. @@ -856,7 +860,7 @@ void TargetPassConfig::addIRPasses() { if (getOptLevel() != CodeGenOptLevel::None && !DisableConstantHoisting) addPass(createConstantHoistingPass()); - if (getOptLevel() != CodeGenOptLevel::None) + if (getOptLevel() != CodeGenOptLevel::None && !DisableReplaceWithVecLib) addPass(createReplaceWithVeclibLegacyPass()); if (getOptLevel() != CodeGenOptLevel::None && !DisablePartialLibcallInlining) diff --git a/llvm/test/CodeGen/AArch64/fp-veclib-expansion.ll b/llvm/test/CodeGen/AArch64/fp-veclib-expansion.ll new file mode 100644 index 000000000000..67c056c780cc --- /dev/null +++ b/llvm/test/CodeGen/AArch64/fp-veclib-expansion.ll @@ -0,0 +1,116 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc --disable-replace-with-vec-lib --vector-library=ArmPL < %s -o - | FileCheck --check-prefix=ARMPL %s +; RUN: llc --disable-replace-with-vec-lib --vector-library=sleefgnuabi < %s -o - | FileCheck --check-prefix=SLEEF %s + +target triple = "aarch64-unknown-linux-gnu" + +define <2 x double> @frem_v2f64(<2 x double> %unused, <2 x double> %a, <2 x double> %b) #0 { +; ARMPL-LABEL: frem_v2f64: +; ARMPL: // %bb.0: +; ARMPL-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; ARMPL-NEXT: .cfi_def_cfa_offset 16 +; ARMPL-NEXT: .cfi_offset w30, -16 +; ARMPL-NEXT: mov v0.16b, v1.16b +; ARMPL-NEXT: mov v1.16b, v2.16b +; ARMPL-NEXT: bl armpl_vfmodq_f64 +; ARMPL-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; ARMPL-NEXT: ret +; +; SLEEF-LABEL: frem_v2f64: +; SLEEF: // %bb.0: +; SLEEF-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; SLEEF-NEXT: .cfi_def_cfa_offset 16 +; SLEEF-NEXT: .cfi_offset w30, -16 +; SLEEF-NEXT: mov v0.16b, v1.16b +; SLEEF-NEXT: mov v1.16b, v2.16b +; SLEEF-NEXT: bl _ZGVnN2vv_fmod +; SLEEF-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; SLEEF-NEXT: ret + %res = frem <2 x double> %a, %b + ret <2 x double> %res +} + +define <4 x float> @frem_strict_v4f32(<4 x float> %unused, <4 x float> %a, <4 x float> %b) #1 { +; ARMPL-LABEL: frem_strict_v4f32: +; ARMPL: // %bb.0: +; ARMPL-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; ARMPL-NEXT: .cfi_def_cfa_offset 16 +; ARMPL-NEXT: .cfi_offset w30, -16 +; ARMPL-NEXT: mov v0.16b, v1.16b +; ARMPL-NEXT: mov v1.16b, v2.16b +; ARMPL-NEXT: bl armpl_vfmodq_f32 +; ARMPL-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; ARMPL-NEXT: ret +; +; SLEEF-LABEL: frem_strict_v4f32: +; SLEEF: // %bb.0: +; SLEEF-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; SLEEF-NEXT: .cfi_def_cfa_offset 16 +; SLEEF-NEXT: .cfi_offset w30, -16 +; SLEEF-NEXT: mov v0.16b, v1.16b +; SLEEF-NEXT: mov v1.16b, v2.16b +; SLEEF-NEXT: bl _ZGVnN4vv_fmodf +; SLEEF-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; SLEEF-NEXT: ret + %res = frem <4 x float> %a, %b + ret <4 x float> %res +} + +define @frem_nxv4f32( %unused, %a, %b) #0 { +; ARMPL-LABEL: frem_nxv4f32: +; ARMPL: // %bb.0: +; ARMPL-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; ARMPL-NEXT: .cfi_def_cfa_offset 16 +; ARMPL-NEXT: .cfi_offset w30, -16 +; ARMPL-NEXT: ptrue p0.s +; ARMPL-NEXT: mov z0.d, z1.d +; ARMPL-NEXT: mov z1.d, z2.d +; ARMPL-NEXT: bl armpl_svfmod_f32_x +; ARMPL-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; ARMPL-NEXT: ret +; +; SLEEF-LABEL: frem_nxv4f32: +; SLEEF: // %bb.0: +; SLEEF-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; SLEEF-NEXT: .cfi_def_cfa_offset 16 +; SLEEF-NEXT: .cfi_offset w30, -16 +; SLEEF-NEXT: ptrue p0.s +; SLEEF-NEXT: mov z0.d, z1.d +; SLEEF-NEXT: mov z1.d, z2.d +; SLEEF-NEXT: bl _ZGVsMxvv_fmodf +; SLEEF-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; SLEEF-NEXT: ret + %res = frem %a, %b + ret %res +} + +define @frem_strict_nxv2f64( %unused, %a, %b) #1 { +; ARMPL-LABEL: frem_strict_nxv2f64: +; ARMPL: // %bb.0: +; ARMPL-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; ARMPL-NEXT: .cfi_def_cfa_offset 16 +; ARMPL-NEXT: .cfi_offset w30, -16 +; ARMPL-NEXT: ptrue p0.d +; ARMPL-NEXT: mov z0.d, z1.d +; ARMPL-NEXT: mov z1.d, z2.d +; ARMPL-NEXT: bl armpl_svfmod_f64_x +; ARMPL-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; ARMPL-NEXT: ret +; +; SLEEF-LABEL: frem_strict_nxv2f64: +; SLEEF: // %bb.0: +; SLEEF-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; SLEEF-NEXT: .cfi_def_cfa_offset 16 +; SLEEF-NEXT: .cfi_offset w30, -16 +; SLEEF-NEXT: ptrue p0.d +; SLEEF-NEXT: mov z0.d, z1.d +; SLEEF-NEXT: mov z1.d, z2.d +; SLEEF-NEXT: bl _ZGVsMxvv_fmod +; SLEEF-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; SLEEF-NEXT: ret + %res = frem %a, %b + ret %res +} + +attributes #0 = { "target-features"="+sve" } +attributes #1 = { "target-features"="+sve" strictfp } -- GitLab From 5d98d8822c69bc9d32c25190beaf1eaccf9ddd37 Mon Sep 17 00:00:00 2001 From: Younan Zhang Date: Fri, 8 Mar 2024 20:26:31 +0800 Subject: [PATCH 595/929] [Sema] Avoid unnecessary copy on MultiLevelTemplateArgumentList. NFC (#84459) We don't modify the MLTAL parameter in `SetupConstraintScope`, and it is better if we don't copy the 120-byte object each time we call the function. --- clang/include/clang/Sema/Sema.h | 10 ++++++---- clang/lib/Sema/SemaConcept.cpp | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index 592c7871a4a5..2cac7e6ca08f 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -11607,12 +11607,14 @@ private: LocalInstantiationScope &Scope, const MultiLevelTemplateArgumentList &TemplateArgs); - /// used by SetupConstraintCheckingTemplateArgumentsAndScope to recursively(in + /// Used by SetupConstraintCheckingTemplateArgumentsAndScope to recursively(in /// the case of lambdas) set up the LocalInstantiationScope of the current /// function. - bool SetupConstraintScope( - FunctionDecl *FD, std::optional> TemplateArgs, - MultiLevelTemplateArgumentList MLTAL, LocalInstantiationScope &Scope); + bool + SetupConstraintScope(FunctionDecl *FD, + std::optional> TemplateArgs, + const MultiLevelTemplateArgumentList &MLTAL, + LocalInstantiationScope &Scope); /// Used during constraint checking, sets up the constraint template argument /// lists, and calls SetupConstraintScope to set up the diff --git a/clang/lib/Sema/SemaConcept.cpp b/clang/lib/Sema/SemaConcept.cpp index 2878e4d31ee8..a8e387e35fb4 100644 --- a/clang/lib/Sema/SemaConcept.cpp +++ b/clang/lib/Sema/SemaConcept.cpp @@ -586,7 +586,8 @@ bool Sema::addInstantiatedCapturesToScope( bool Sema::SetupConstraintScope( FunctionDecl *FD, std::optional> TemplateArgs, - MultiLevelTemplateArgumentList MLTAL, LocalInstantiationScope &Scope) { + const MultiLevelTemplateArgumentList &MLTAL, + LocalInstantiationScope &Scope) { if (FD->isTemplateInstantiation() && FD->getPrimaryTemplate()) { FunctionTemplateDecl *PrimaryTemplate = FD->getPrimaryTemplate(); InstantiatingTemplate Inst( -- GitLab From 430de48a61369ea41efe1cbb98654a86568ae95f Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Fri, 8 Mar 2024 12:26:30 +0000 Subject: [PATCH 596/929] [GISel] Simplify getConstantVRegValWithLookThrough. NFC. --- llvm/lib/CodeGen/GlobalISel/Utils.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp index 23ad68b331c9..a9fa73b60a09 100644 --- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp +++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp @@ -355,18 +355,17 @@ std::optional getConstantVRegValWithLookThrough( if (!MaybeVal) return std::nullopt; APInt &Val = *MaybeVal; - while (!SeenOpcodes.empty()) { - std::pair OpcodeAndSize = SeenOpcodes.pop_back_val(); - switch (OpcodeAndSize.first) { + for (auto [Opcode, Size] : reverse(SeenOpcodes)) { + switch (Opcode) { case TargetOpcode::G_TRUNC: - Val = Val.trunc(OpcodeAndSize.second); + Val = Val.trunc(Size); break; case TargetOpcode::G_ANYEXT: case TargetOpcode::G_SEXT: - Val = Val.sext(OpcodeAndSize.second); + Val = Val.sext(Size); break; case TargetOpcode::G_ZEXT: - Val = Val.zext(OpcodeAndSize.second); + Val = Val.zext(Size); break; } } -- GitLab From d74287226ac8c144a2700ba589d34a8bebd85d0d Mon Sep 17 00:00:00 2001 From: Kiran Chandramohan Date: Fri, 8 Mar 2024 12:43:39 +0000 Subject: [PATCH 597/929] [Flang][AArch64] Add support for complex16 params/returns (#84217) Fixes #84088 --- flang/lib/Optimizer/CodeGen/Target.cpp | 6 ++++-- flang/test/Fir/target-complex16.f90 | 29 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 flang/test/Fir/target-complex16.f90 diff --git a/flang/lib/Optimizer/CodeGen/Target.cpp b/flang/lib/Optimizer/CodeGen/Target.cpp index 7c77bdd79008..cea7a1f97f41 100644 --- a/flang/lib/Optimizer/CodeGen/Target.cpp +++ b/flang/lib/Optimizer/CodeGen/Target.cpp @@ -737,7 +737,8 @@ struct TargetAArch64 : public GenericTarget { CodeGenSpecifics::Marshalling marshal; const auto *sem = &floatToSemantics(kindMap, eleTy); if (sem == &llvm::APFloat::IEEEsingle() || - sem == &llvm::APFloat::IEEEdouble()) { + sem == &llvm::APFloat::IEEEdouble() || + sem == &llvm::APFloat::IEEEquad()) { // [2 x t] array of 2 eleTy marshal.emplace_back(fir::SequenceType::get({2}, eleTy), AT{}); } else { @@ -751,7 +752,8 @@ struct TargetAArch64 : public GenericTarget { CodeGenSpecifics::Marshalling marshal; const auto *sem = &floatToSemantics(kindMap, eleTy); if (sem == &llvm::APFloat::IEEEsingle() || - sem == &llvm::APFloat::IEEEdouble()) { + sem == &llvm::APFloat::IEEEdouble() || + sem == &llvm::APFloat::IEEEquad()) { // Use a type that will be translated into LLVM as: // { t, t } struct of 2 eleTy marshal.emplace_back(mlir::TupleType::get(eleTy.getContext(), diff --git a/flang/test/Fir/target-complex16.f90 b/flang/test/Fir/target-complex16.f90 new file mode 100644 index 000000000000..9245b205f68a --- /dev/null +++ b/flang/test/Fir/target-complex16.f90 @@ -0,0 +1,29 @@ +// RUN: tco --target=aarch64-unknown-linux-gnu %s | FileCheck %s --check-prefix=AARCH64 + +// AARCH64-LABEL: define { fp128, fp128 } @gen16() +func.func @gen16() -> !fir.complex<16> { + // AARCH64: %[[VAL1:.*]] = alloca { fp128, fp128 }, i64 1, align 16 + %1 = fir.undefined !fir.complex<16> + %2 = arith.constant 1.0 : f128 + %3 = arith.constant -4.0 : f128 + %c0 = arith.constant 0 : i32 + // AARCH64: store { fp128, fp128 } { fp128 0xL0000000000000000C001000000000000, fp128 0xL00000000000000003FFF000000000000 }, ptr %[[VAL1]], align 16 + %4 = fir.insert_value %1, %3, [0 : index] : (!fir.complex<16>, f128) -> !fir.complex<16> + %c1 = arith.constant 1 : i32 + %5 = fir.insert_value %4, %2, [1 : index] : (!fir.complex<16>, f128) -> !fir.complex<16> + // AARCH64: %[[VAL2:.*]] = load { fp128, fp128 }, ptr %[[VAL1]], align 16 + // AARCH64: ret { fp128, fp128 } %[[VAL2]] + return %5 : !fir.complex<16> +} + +// AARCH64: declare void @sink16([2 x fp128]) +func.func private @sink16(!fir.complex<16>) -> () + +// AARCH64-LABEL: define void @call16() +func.func @call16() { + // AARCH64: = call { fp128, fp128 } @gen16() + %1 = fir.call @gen16() : () -> !fir.complex<16> + // AARCH64: call void @sink16([2 x fp128] % + fir.call @sink16(%1) : (!fir.complex<16>) -> () + return +} -- GitLab From 1ca8092e87aea58f1c3752d03c20bdfc4259e409 Mon Sep 17 00:00:00 2001 From: Shengchen Kan Date: Fri, 8 Mar 2024 20:54:33 +0800 Subject: [PATCH 598/929] [X86][MC] Support encoding/decoding for APX CCMP/CTEST (#83863) APX assembly syntax recommendations: https://cdrdv2.intel.com/v1/dl/getContent/817241 NOTE: The change in llvm/tools/llvm-exegesis/lib/X86/Target.cpp is for test LLVM :: tools/llvm-exegesis/X86/latency/latency-SETCCr-cond-codes-sweep.s For `SETcc`, llvm-exegesis would randomly choose 1 other instruction to test with `SETcc`, after selecting the instruction, llvm-exegesis would check if the operand is initialized and valid, if not `randomizeTargetMCOperand` would choose a value for invalid operand, it misses support for condition code operand, which cause the flaky failure after `CCMP` supported. llvm-exegesis can choose `CCMP` without specifying ccmp feature b/c it use `MCSubtarget` and only16/32/64 bit is considered. llvm-exegesis doesn't choose other instructions b/c requirement in `hasAliasingRegistersThrough`: the instruction should use GPR (defined by `SETcc`) and define `EFLAGS` (used by `SETcc`). --- .../Support/X86DisassemblerDecoderCommon.h | 1 + .../lib/Target/X86/AsmParser/X86AsmParser.cpp | 67 + .../X86/Disassembler/X86Disassembler.cpp | 43 +- .../X86/Disassembler/X86DisassemblerDecoder.h | 8 +- .../lib/Target/X86/MCTargetDesc/X86BaseInfo.h | 5 +- .../X86/MCTargetDesc/X86InstPrinterCommon.cpp | 29 +- .../X86/MCTargetDesc/X86InstPrinterCommon.h | 1 + .../X86/MCTargetDesc/X86MCCodeEmitter.cpp | 51 +- llvm/lib/Target/X86/X86InstrAsmAlias.td | 128 ++ .../Target/X86/X86InstrConditionalCompare.td | 110 ++ llvm/lib/Target/X86/X86InstrFormats.td | 2 + llvm/lib/Target/X86/X86InstrInfo.td | 1 + llvm/lib/Target/X86/X86InstrOperands.td | 6 + llvm/test/MC/Disassembler/X86/apx/ccmp.txt | 1598 +++++++++++++++++ .../Disassembler/X86/apx/reverse-encoding.txt | 18 + llvm/test/MC/X86/apx/ccmp-att-error.s | 29 + llvm/test/MC/X86/apx/ccmp-att.s | 1213 +++++++++++++ llvm/test/MC/X86/apx/ccmp-intel-error.s | 29 + llvm/test/MC/X86/apx/ccmp-intel.s | 1210 +++++++++++++ llvm/test/TableGen/x86-fold-tables.inc | 23 + llvm/tools/llvm-exegesis/lib/X86/Target.cpp | 4 + llvm/utils/TableGen/X86RecognizableInstr.cpp | 11 + llvm/utils/TableGen/X86RecognizableInstr.h | 2 + 23 files changed, 4574 insertions(+), 15 deletions(-) create mode 100644 llvm/lib/Target/X86/X86InstrConditionalCompare.td create mode 100644 llvm/test/MC/Disassembler/X86/apx/ccmp.txt create mode 100644 llvm/test/MC/X86/apx/ccmp-att-error.s create mode 100644 llvm/test/MC/X86/apx/ccmp-att.s create mode 100644 llvm/test/MC/X86/apx/ccmp-intel-error.s create mode 100644 llvm/test/MC/X86/apx/ccmp-intel.s diff --git a/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h b/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h index 0dc974ea9efd..5daae45df2f8 100644 --- a/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h +++ b/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h @@ -434,6 +434,7 @@ enum ModRMDecisionType { MODRMTYPES MODRM_max }; ENUM_ENTRY(ENCODING_Rv, \ "Register code of operand size added to the opcode byte") \ ENUM_ENTRY(ENCODING_CC, "Condition code encoded in opcode") \ + ENUM_ENTRY(ENCODING_CF, "Condition flags encoded in EVEX.VVVV") \ ENUM_ENTRY(ENCODING_DUP, \ "Duplicate of another operand; ID is encoded in type") \ ENUM_ENTRY(ENCODING_SI, "Source index; encoded in OpSize/Adsize prefix") \ diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp index 051f6caa8c04..e3701d69934c 100644 --- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp +++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp @@ -1122,6 +1122,7 @@ private: unsigned IdentifyMasmOperator(StringRef Name); bool ParseMasmOperator(unsigned OpKind, int64_t &Val); bool ParseRoundingModeOp(SMLoc Start, OperandVector &Operands); + bool parseCFlagsOp(OperandVector &Operands); bool ParseIntelNamedOperator(StringRef Name, IntelExprStateMachine &SM, bool &ParseError, SMLoc &End); bool ParseMasmNamedOperator(StringRef Name, IntelExprStateMachine &SM, @@ -2306,6 +2307,67 @@ bool X86AsmParser::ParseRoundingModeOp(SMLoc Start, OperandVector &Operands) { return Error(Tok.getLoc(), "unknown token in expression"); } +/// Parse condtional flags for CCMP/CTEST, e.g {dfv=of,sf,zf,cf} right after +/// mnemonic. +bool X86AsmParser::parseCFlagsOp(OperandVector &Operands) { + MCAsmParser &Parser = getParser(); + AsmToken Tok = Parser.getTok(); + const SMLoc Start = Tok.getLoc(); + if (!Tok.is(AsmToken::LCurly)) + return Error(Tok.getLoc(), "Expected { at this point"); + Parser.Lex(); // Eat "{" + Tok = Parser.getTok(); + if (Tok.getIdentifier() != "dfv") + return Error(Tok.getLoc(), "Expected dfv at this point"); + Parser.Lex(); // Eat "dfv" + Tok = Parser.getTok(); + if (!Tok.is(AsmToken::Equal)) + return Error(Tok.getLoc(), "Expected = at this point"); + Parser.Lex(); // Eat "=" + + Tok = Parser.getTok(); + SMLoc End; + if (Tok.is(AsmToken::RCurly)) { + End = Tok.getEndLoc(); + Operands.push_back(X86Operand::CreateImm( + MCConstantExpr::create(0, Parser.getContext()), Start, End)); + Parser.Lex(); // Eat "}" + return false; + } + unsigned CFlags = 0; + for (unsigned I = 0; I < 4; ++I) { + Tok = Parser.getTok(); + unsigned CFlag = StringSwitch(Tok.getIdentifier()) + .Case("of", 0x8) + .Case("sf", 0x4) + .Case("zf", 0x2) + .Case("cf", 0x1) + .Default(~0U); + if (CFlag == ~0U) + return Error(Tok.getLoc(), "Invalid conditional flags"); + + if (CFlags & CFlag) + return Error(Tok.getLoc(), "Duplicated conditional flag"); + CFlags |= CFlag; + + Parser.Lex(); // Eat one conditional flag + Tok = Parser.getTok(); + if (Tok.is(AsmToken::RCurly)) { + End = Tok.getEndLoc(); + Operands.push_back(X86Operand::CreateImm( + MCConstantExpr::create(CFlags, Parser.getContext()), Start, End)); + Parser.Lex(); // Eat "}" + return false; + } else if (I == 3) { + return Error(Tok.getLoc(), "Expected } at this point"); + } else if (Tok.isNot(AsmToken::Comma)) { + return Error(Tok.getLoc(), "Expected } or , at this point"); + } + Parser.Lex(); // Eat "," + } + llvm_unreachable("Unexpected control flow"); +} + /// Parse the '.' operator. bool X86AsmParser::ParseIntelDotOperator(IntelExprStateMachine &SM, SMLoc &End) { @@ -3461,6 +3523,11 @@ bool X86AsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name, Operands.push_back(X86Operand::CreateImm(ImmOp, NameLoc, NameLoc)); } + // Parse condtional flags after mnemonic. + if ((Name.starts_with("ccmp") || Name.starts_with("ctest")) && + parseCFlagsOp(Operands)) + return true; + // This does the actual operand parsing. Don't parse any more if we have a // prefix juxtaposed with an operation like "lock incl 4(%rax)", because we // just want to parse the "lock" as the first instruction and the "incl" as diff --git a/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp b/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp index dbc2cef39d86..0ff440bdbe0d 100644 --- a/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp +++ b/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp @@ -1141,6 +1141,25 @@ static int getInstructionIDWithAttrMask(uint16_t *instructionID, return 0; } +static bool isCCMPOrCTEST(InternalInstruction *insn) { + if (insn->opcodeType != MAP4) + return false; + if (insn->opcode == 0x83 && regFromModRM(insn->modRM) == 7) + return true; + switch (insn->opcode & 0xfe) { + default: + return false; + case 0x38: + case 0x3a: + case 0x84: + return true; + case 0x80: + return regFromModRM(insn->modRM) == 7; + case 0xf6: + return regFromModRM(insn->modRM) == 0; + } +} + static bool isNF(InternalInstruction *insn) { if (!nfFromEVEX4of4(insn->vectorExtensionPrefix[3])) return false; @@ -1197,9 +1216,12 @@ static int getInstructionID(struct InternalInstruction *insn, attrMask |= ATTR_EVEXKZ; if (bFromEVEX4of4(insn->vectorExtensionPrefix[3])) attrMask |= ATTR_EVEXB; - if (isNF(insn)) // NF bit is the MSB of aaa. + if (isNF(insn) && !readModRM(insn) && + !isCCMPOrCTEST(insn)) // NF bit is the MSB of aaa. attrMask |= ATTR_EVEXNF; - else if (aaaFromEVEX4of4(insn->vectorExtensionPrefix[3])) + // aaa is not used a opmask in MAP4 + else if (aaaFromEVEX4of4(insn->vectorExtensionPrefix[3]) && + (insn->opcodeType != MAP4)) attrMask |= ATTR_EVEXK; if (lFromEVEX4of4(insn->vectorExtensionPrefix[3])) attrMask |= ATTR_VEXL; @@ -1732,8 +1754,15 @@ static int readOperands(struct InternalInstruction *insn) { if (readOpcodeRegister(insn, 0)) return -1; break; + case ENCODING_CF: + insn->immediates[1] = oszcFromEVEX3of4(insn->vectorExtensionPrefix[2]); + needVVVV = false; // oszc shares the same bits with VVVV + break; case ENCODING_CC: - insn->immediates[1] = insn->opcode & 0xf; + if (isCCMPOrCTEST(insn)) + insn->immediates[2] = scFromEVEX4of4(insn->vectorExtensionPrefix[3]); + else + insn->immediates[1] = insn->opcode & 0xf; break; case ENCODING_FP: break; @@ -2371,9 +2400,15 @@ static bool translateOperand(MCInst &mcInst, const OperandSpecifier &operand, case ENCODING_Rv: translateRegister(mcInst, insn.opcodeRegister); return false; - case ENCODING_CC: + case ENCODING_CF: mcInst.addOperand(MCOperand::createImm(insn.immediates[1])); return false; + case ENCODING_CC: + if (isCCMPOrCTEST(&insn)) + mcInst.addOperand(MCOperand::createImm(insn.immediates[2])); + else + mcInst.addOperand(MCOperand::createImm(insn.immediates[1])); + return false; case ENCODING_FP: translateFPRegister(mcInst, insn.modRM & 7); return false; diff --git a/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h b/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h index 9cae0f02926f..7cd01c151ed2 100644 --- a/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h +++ b/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h @@ -33,6 +33,8 @@ namespace X86Disassembler { #define twoBitsFromOffset6(val) (((val) >> 6) & 0x3) #define threeBitsFromOffset0(val) ((val) & 0x7) #define threeBitsFromOffset3(val) (((val) >> 3) & 0x7) +#define fourBitsFromOffset0(val) ((val) & 0xf) +#define fourBitsFromOffset3(val) (((val) >> 3) & 0xf) #define fiveBitsFromOffset0(val) ((val) & 0x1f) #define invertedBitFromOffset2(val) (((~(val)) >> 2) & 0x1) #define invertedBitFromOffset3(val) (((~(val)) >> 3) & 0x1) @@ -97,6 +99,7 @@ namespace X86Disassembler { #define vvvvFromEVEX3of4(evex) invertedFourBitsFromOffset3(evex) #define x2FromEVEX3of4(evex) invertedBitFromOffset2(evex) #define ppFromEVEX3of4(evex) twoBitsFromOffset0(evex) +#define oszcFromEVEX3of4(evex) fourBitsFromOffset3(evex) #define zFromEVEX4of4(evex) bitFromOffset7(evex) #define l2FromEVEX4of4(evex) bitFromOffset6(evex) #define lFromEVEX4of4(evex) bitFromOffset5(evex) @@ -104,6 +107,7 @@ namespace X86Disassembler { #define v2FromEVEX4of4(evex) invertedBitFromOffset3(evex) #define aaaFromEVEX4of4(evex) threeBitsFromOffset0(evex) #define nfFromEVEX4of4(evex) bitFromOffset2(evex) +#define scFromEVEX4of4(evex) fourBitsFromOffset0(evex) // These enums represent Intel registers for use by the decoder. #define REGS_8BIT \ @@ -755,10 +759,10 @@ struct InternalInstruction { // The displacement, used for memory operands int32_t displacement; - // Immediates. There can be two in some cases + // Immediates. There can be three in some cases uint8_t numImmediatesConsumed; uint8_t numImmediatesTranslated; - uint64_t immediates[2]; + uint64_t immediates[3]; // A register or immediate operand encoded into the opcode Reg opcodeRegister; diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h b/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h index 28a067d525e0..65b8ebc0b9b9 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h +++ b/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h @@ -875,7 +875,10 @@ enum : uint64_t { ExplicitOpPrefixMask = 3ULL << ExplicitOpPrefixShift, /// EVEX_NF - Set if this instruction has EVEX.NF field set. EVEX_NFShift = ExplicitOpPrefixShift + 2, - EVEX_NF = 1ULL << EVEX_NFShift + EVEX_NF = 1ULL << EVEX_NFShift, + // TwoConditionalOps - Set if this instruction has two conditional operands + TwoConditionalOps_Shift = EVEX_NFShift + 1, + TwoConditionalOps = 1ULL << TwoConditionalOps_Shift }; /// \returns true if the instruction with given opcode is a prefix. diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp index 7422a9897346..fd46e4e1df82 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp @@ -29,7 +29,9 @@ using namespace llvm; void X86InstPrinterCommon::printCondCode(const MCInst *MI, unsigned Op, raw_ostream &O) { int64_t Imm = MI->getOperand(Op).getImm(); - bool IsCMPCCXADD = X86::isCMPCCXADD(MI->getOpcode()); + unsigned Opc = MI->getOpcode(); + bool IsCMPCCXADD = X86::isCMPCCXADD(Opc); + bool IsCCMPOrCTEST = X86::isCCMPCC(Opc) || X86::isCTESTCC(Opc); // clang-format off switch (Imm) { @@ -44,8 +46,8 @@ void X86InstPrinterCommon::printCondCode(const MCInst *MI, unsigned Op, case 7: O << (IsCMPCCXADD ? "nbe" : "a"); break; case 8: O << "s"; break; case 9: O << "ns"; break; - case 0xa: O << "p"; break; - case 0xb: O << "np"; break; + case 0xa: O << (IsCCMPOrCTEST ? "t" : "p"); break; + case 0xb: O << (IsCCMPOrCTEST ? "f" : "np"); break; case 0xc: O << "l"; break; case 0xd: O << (IsCMPCCXADD ? "nl" : "ge"); break; case 0xe: O << "le"; break; @@ -54,6 +56,27 @@ void X86InstPrinterCommon::printCondCode(const MCInst *MI, unsigned Op, // clang-format on } +void X86InstPrinterCommon::printCondFlags(const MCInst *MI, unsigned Op, + raw_ostream &O) { + // +----+----+----+----+ + // | OF | SF | ZF | CF | + // +----+----+----+----+ + int64_t Imm = MI->getOperand(Op).getImm(); + assert(Imm >= 0 && Imm < 16 && "Invalid condition flags"); + O << "{dfv="; + std::string Flags; + if (Imm & 0x8) + Flags += "of,"; + if (Imm & 0x4) + Flags += "sf,"; + if (Imm & 0x2) + Flags += "zf,"; + if (Imm & 0x1) + Flags += "cf,"; + StringRef SimplifiedFlags = StringRef(Flags).rtrim(","); + O << SimplifiedFlags << "}"; +} + void X86InstPrinterCommon::printSSEAVXCC(const MCInst *MI, unsigned Op, raw_ostream &O) { int64_t Imm = MI->getOperand(Op).getImm(); diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.h b/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.h index 0cb5bf014b20..221102e17c65 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.h +++ b/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.h @@ -24,6 +24,7 @@ public: virtual void printOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O) = 0; void printCondCode(const MCInst *MI, unsigned Op, raw_ostream &OS); + void printCondFlags(const MCInst *MI, unsigned Op, raw_ostream &OS); void printSSEAVXCC(const MCInst *MI, unsigned Op, raw_ostream &OS); void printVPCOMMnemonic(const MCInst *MI, raw_ostream &OS); void printVPCMPMnemonic(const MCInst *MI, raw_ostream &OS); diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp index 1fa676eeb79b..dcd0551b7b76 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp @@ -200,8 +200,12 @@ public: void setB(const MCInst &MI, unsigned OpNum) { B = getRegEncoding(MI, OpNum) >> 3 & 1; } - void set4V(const MCInst &MI, unsigned OpNum) { - set4V(getRegEncoding(MI, OpNum)); + void set4V(const MCInst &MI, unsigned OpNum, bool IsImm = false) { + // OF, SF, ZF and CF reuse VEX_4V bits but are not reversed + if (IsImm) + set4V(~(MI.getOperand(OpNum).getImm())); + else + set4V(getRegEncoding(MI, OpNum)); } void setL(bool V) { VEX_L = V; } void setPP(unsigned V) { VEX_PP = V; } @@ -252,6 +256,11 @@ public: EVEX_aaa = getRegEncoding(MI, OpNum); } void setNF(bool V) { EVEX_aaa |= V << 2; } + void setSC(const MCInst &MI, unsigned OpNum) { + unsigned Encoding = MI.getOperand(OpNum).getImm(); + EVEX_V2 = ~(Encoding >> 3) & 0x1; + EVEX_aaa = Encoding & 0x7; + } X86OpcodePrefixHelper(const MCRegisterInfo &MRI) : W(0), R(0), X(0), B(0), M(0), R2(0), X2(0), B2(0), VEX_4V(0), VEX_L(0), @@ -1045,6 +1054,7 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, uint8_t EVEX_rc = 0; unsigned CurOp = X86II::getOperandBias(Desc); + bool HasTwoConditionalOps = TSFlags & X86II::TwoConditionalOps; switch (TSFlags & X86II::FormMask) { default: @@ -1086,6 +1096,10 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, Prefix.set4VV2(MI, CurOp++); Prefix.setRR2(MI, CurOp++); + if (HasTwoConditionalOps) { + Prefix.set4V(MI, CurOp++, /*IsImm=*/true); + Prefix.setSC(MI, CurOp++); + } break; } case X86II::MRMSrcMemFSIB: @@ -1115,7 +1129,11 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg); Prefix.setXX2(MI, MemOperand + X86::AddrIndexReg); Prefix.setV2(MI, MemOperand + X86::AddrIndexReg, HasVEX_4V); - + CurOp += X86::AddrNumOperands; + if (HasTwoConditionalOps) { + Prefix.set4V(MI, CurOp++, /*IsImm=*/true); + Prefix.setSC(MI, CurOp++); + } break; } case X86II::MRMSrcMem4VOp3: { @@ -1155,7 +1173,11 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg); Prefix.setXX2(MI, MemOperand + X86::AddrIndexReg); Prefix.setV2(MI, MemOperand + X86::AddrIndexReg, HasVEX_4V); - + CurOp += X86::AddrNumOperands + 1; // Skip first imm. + if (HasTwoConditionalOps) { + Prefix.set4V(MI, CurOp++, /*IsImm=*/true); + Prefix.setSC(MI, CurOp++); + } break; } case X86II::MRMSrcReg: { @@ -1183,6 +1205,11 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, Prefix.setX(MI, CurOp, 4); ++CurOp; + if (HasTwoConditionalOps) { + Prefix.set4V(MI, CurOp++, /*IsImm=*/true); + Prefix.setSC(MI, CurOp++); + } + if (TSFlags & X86II::EVEX_B) { if (HasEVEX_RC) { unsigned NumOps = Desc.getNumOperands(); @@ -1236,6 +1263,10 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, Prefix.set4VV2(MI, CurOp++); Prefix.setRR2(MI, CurOp++); + if (HasTwoConditionalOps) { + Prefix.set4V(MI, CurOp++, /*IsImm=*/true); + Prefix.setSC(MI, CurOp++); + } if (TSFlags & X86II::EVEX_B) EncodeRC = true; break; @@ -1266,6 +1297,10 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, Prefix.setBB2(MI, CurOp); Prefix.setX(MI, CurOp, 4); ++CurOp; + if (HasTwoConditionalOps) { + Prefix.set4V(MI, ++CurOp, /*IsImm=*/true); + Prefix.setSC(MI, ++CurOp); + } break; } } @@ -1525,6 +1560,7 @@ void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, unsigned OpcodeOffset = 0; bool IsND = X86II::hasNewDataDest(TSFlags); + bool HasTwoConditionalOps = TSFlags & X86II::TwoConditionalOps; uint64_t Form = TSFlags & X86II::FormMask; switch (Form) { @@ -1914,11 +1950,16 @@ void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, // If there is a remaining operand, it must be a trailing immediate. Emit it // according to the right size for the instruction. Some instructions // (SSE4a extrq and insertq) have two trailing immediates. - while (CurOp != NumOps && NumOps - CurOp <= 2) { + + // Skip two trainling conditional operands encoded in EVEX prefix + unsigned RemaningOps = NumOps - CurOp - 2 * HasTwoConditionalOps; + while (RemaningOps) { emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags), StartByte, CB, Fixups); + --RemaningOps; } + CurOp += 2 * HasTwoConditionalOps; } if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow) diff --git a/llvm/lib/Target/X86/X86InstrAsmAlias.td b/llvm/lib/Target/X86/X86InstrAsmAlias.td index 2590be8651d5..fab058dc390b 100644 --- a/llvm/lib/Target/X86/X86InstrAsmAlias.td +++ b/llvm/lib/Target/X86/X86InstrAsmAlias.td @@ -62,6 +62,134 @@ multiclass CMPCCXADD_Aliases { (CMPCCXADDmr64_EVEX GR64:$dst, i64mem:$dstsrc2, GR64:$src3, CC), 0>; } +// CCMP Instructions Alias +multiclass CCMP_Aliases { +let Predicates = [In64BitMode] in { +def : InstAlias<"ccmp"#Cond#"{b} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP8rr GR8:$src1, GR8:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{w} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP16rr GR16:$src1, GR16:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{l} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP32rr GR32:$src1, GR32:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{q} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP64rr GR64:$src1, GR64:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{b} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP8rm GR8:$src1, i8mem:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{w} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP16rm GR16:$src1, i16mem:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{l} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP32rm GR32:$src1, i32mem:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{q} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP64rm GR64:$src1, i64mem:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{b} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP8mr i8mem:$src1, GR8:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{w} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP16mr i16mem:$src1, GR16:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{l} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP32mr i32mem:$src1, GR32:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{q} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP64mr i64mem:$src1, GR64:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{b} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP8ri GR8:$src1, i8imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{w} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP16ri GR16:$src1, i16imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{l} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP32ri GR32:$src1, i32imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{q} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP64ri32 GR64:$src1, i64i32imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{w} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP16ri8 GR16:$src1, i16i8imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{l} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP32ri8 GR32:$src1, i32i8imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{q} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP64ri8 GR64:$src1, i64i8imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{b} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP8mi i8mem:$src1, i8imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{w} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP16mi i16mem:$src1, i16imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{l} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP32mi i32mem:$src1, i32imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{q} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP64mi32 i64mem:$src1, i64i32imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{w} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP16mi8 i16mem:$src1, i16i8imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{l} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP32mi8 i32mem:$src1, i32i8imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{q} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP64mi8 i64mem:$src1, i64i8imm:$src2, cflags:$dcf, CC), 0>; +} +} +defm : CCMP_Aliases<"o" , 0>; +defm : CCMP_Aliases<"no", 1>; +defm : CCMP_Aliases<"b" , 2>; +defm : CCMP_Aliases<"ae", 3>; +defm : CCMP_Aliases<"e" , 4>; +defm : CCMP_Aliases<"ne", 5>; +defm : CCMP_Aliases<"be", 6>; +defm : CCMP_Aliases<"a" , 7>; +defm : CCMP_Aliases<"s" , 8>; +defm : CCMP_Aliases<"ns", 9>; +defm : CCMP_Aliases<"t" , 10>; +defm : CCMP_Aliases<"f", 11>; +defm : CCMP_Aliases<"l" , 12>; +defm : CCMP_Aliases<"ge", 13>; +defm : CCMP_Aliases<"le", 14>; +defm : CCMP_Aliases<"g" , 15>; + +// CTEST Instructions Alias +multiclass CTEST_Aliases { +let Predicates = [In64BitMode] in { +def : InstAlias<"ctest"#Cond#"{b} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST8rr GR8:$src1, GR8:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{w} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST16rr GR16:$src1, GR16:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{l} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST32rr GR32:$src1, GR32:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{q} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST64rr GR64:$src1, GR64:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{b} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST8mr i8mem:$src1, GR8:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{w} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST16mr i16mem:$src1, GR16:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{l} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST32mr i32mem:$src1, GR32:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{q} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST64mr i64mem:$src1, GR64:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{b} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST8ri GR8:$src1, i8imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{w} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST16ri GR16:$src1, i16imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{l} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST32ri GR32:$src1, i32imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{q} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST64ri32 GR64:$src1, i64i32imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{b} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST8mi i8mem:$src1, i8imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{w} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST16mi i16mem:$src1, i16imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{l} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST32mi i32mem:$src1, i32imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{q} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST64mi32 i64mem:$src1, i64i32imm:$src2, cflags:$dcf, CC), 0>; +} +} +defm : CTEST_Aliases<"o" , 0>; +defm : CTEST_Aliases<"no", 1>; +defm : CTEST_Aliases<"b" , 2>; +defm : CTEST_Aliases<"ae", 3>; +defm : CTEST_Aliases<"e" , 4>; +defm : CTEST_Aliases<"ne", 5>; +defm : CTEST_Aliases<"be", 6>; +defm : CTEST_Aliases<"a" , 7>; +defm : CTEST_Aliases<"s" , 8>; +defm : CTEST_Aliases<"ns", 9>; +defm : CTEST_Aliases<"t" , 10>; +defm : CTEST_Aliases<"f", 11>; +defm : CTEST_Aliases<"l" , 12>; +defm : CTEST_Aliases<"ge", 13>; +defm : CTEST_Aliases<"le", 14>; +defm : CTEST_Aliases<"g" , 15>; + //===----------------------------------------------------------------------===// // Assembler Mnemonic Aliases //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/X86/X86InstrConditionalCompare.td b/llvm/lib/Target/X86/X86InstrConditionalCompare.td new file mode 100644 index 000000000000..e5c1143eba87 --- /dev/null +++ b/llvm/lib/Target/X86/X86InstrConditionalCompare.td @@ -0,0 +1,110 @@ +//===-- X86InstrConditionalCompare.td - Conditional Compare --*- tablegen -*-==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file describes the X86 conditional compare instructions. +// +//===----------------------------------------------------------------------===// + +class BinCondOp o, Format f, X86TypeInfo t, DAGOperand op1, DAGOperand op2, string m> + : ITy, T_MAP4, EVEX, Requires<[In64BitMode]> { + let isCodeGenOnly = 1; + let ForceDisassemble = 1; + let Uses = [EFLAGS]; + let Defs = [EFLAGS]; + let hasTwoConditionalOps = 1; + let ImmT = !if(!eq(op2, i16i8imm), Imm8, + !if(!eq(op2, i32i8imm), Imm8, + !if(!eq(op2, i64i8imm), Imm8, + !if(!eq(op2, i8imm), Imm8, + !if(!eq(op2, i16imm), Imm16, + !if(!eq(op2, i32imm), Imm32, + !if(!eq(op2, i64i32imm), Imm32S, NoImm))))))); +} + +class Ccmp o, Format f, X86TypeInfo t, DAGOperand op1, DAGOperand op2>: + BinCondOp; + +class Ctest o, Format f, X86TypeInfo t, DAGOperand op1, DAGOperand op2>: + BinCondOp; + +//===----------------------------------------------------------------------===// +// CCMP Instructions +// +let SchedRW = [WriteALU] in { + def CCMP8rr : Ccmp<0x38, MRMDestReg, Xi8, GR8, GR8>; + def CCMP16rr: Ccmp<0x39, MRMDestReg, Xi16, GR16, GR16>, PD; + def CCMP32rr: Ccmp<0x39, MRMDestReg, Xi32, GR32, GR32>; + def CCMP64rr: Ccmp<0x39, MRMDestReg, Xi64, GR64, GR64>; + def CCMP8rr_REV : Ccmp<0x3a, MRMSrcReg, Xi8, GR8, GR8>; + def CCMP16rr_REV: Ccmp<0x3b, MRMSrcReg, Xi16, GR16, GR16>, PD; + def CCMP32rr_REV: Ccmp<0x3b, MRMSrcReg, Xi32, GR32, GR32>; + def CCMP64rr_REV: Ccmp<0x3b, MRMSrcReg, Xi64, GR64, GR64>; + def CCMP16ri8: Ccmp<0x83, MRM7r, Xi16, GR16, i16i8imm>, PD; + def CCMP32ri8: Ccmp<0x83, MRM7r, Xi32, GR32, i32i8imm>; + def CCMP64ri8: Ccmp<0x83, MRM7r, Xi64, GR64, i64i8imm>; + + def CCMP8ri : Ccmp<0x80, MRM7r, Xi8, GR8, i8imm>; + def CCMP16ri: Ccmp<0x81, MRM7r, Xi16, GR16, i16imm>, PD; + def CCMP32ri: Ccmp<0x81, MRM7r, Xi32, GR32, i32imm>; + def CCMP64ri32: Ccmp<0x81, MRM7r, Xi64, GR64, i64i32imm>; +} + +let mayLoad = 1 in { + let SchedRW = [WriteALU.Folded] in { + def CCMP16mi8: Ccmp<0x83, MRM7m, Xi16, i16mem, i16i8imm>, PD; + def CCMP32mi8: Ccmp<0x83, MRM7m, Xi32, i32mem, i32i8imm>; + def CCMP64mi8: Ccmp<0x83, MRM7m, Xi64, i64mem, i64i8imm>; + def CCMP8mi : Ccmp<0x80, MRM7m, Xi8, i8mem, i8imm>; + def CCMP16mi: Ccmp<0x81, MRM7m, Xi16, i16mem, i16imm>, PD; + def CCMP32mi: Ccmp<0x81, MRM7m, Xi32, i32mem, i32imm>; + def CCMP64mi32: Ccmp<0x81, MRM7m, Xi64, i64mem, i64i32imm>; + } + let SchedRW = [WriteALU.Folded, WriteALU.ReadAfterFold] in { + def CCMP8rm : Ccmp<0x3a, MRMSrcMem, Xi8, GR8, i8mem>; + def CCMP16rm: Ccmp<0x3b, MRMSrcMem, Xi16, GR16, i16mem>, PD; + def CCMP32rm: Ccmp<0x3b, MRMSrcMem, Xi32, GR32, i32mem>; + def CCMP64rm: Ccmp<0x3b, MRMSrcMem, Xi64, GR64, i64mem>; + + def CCMP8mr : Ccmp<0x38, MRMDestMem, Xi8, i8mem, GR8>; + def CCMP16mr: Ccmp<0x39, MRMDestMem, Xi16, i16mem, GR16>, PD; + def CCMP32mr: Ccmp<0x39, MRMDestMem, Xi32, i32mem, GR32>; + def CCMP64mr: Ccmp<0x39, MRMDestMem, Xi64, i64mem, GR64>; + } +} + +//===----------------------------------------------------------------------===// +// CTEST Instructions +// +let SchedRW = [WriteALU] in { + let isCommutable = 1 in { + def CTEST8rr : Ctest<0x84, MRMDestReg, Xi8, GR8, GR8>; + def CTEST16rr: Ctest<0x85, MRMDestReg, Xi16, GR16, GR16>, PD; + def CTEST32rr: Ctest<0x85, MRMDestReg, Xi32, GR32, GR32>; + def CTEST64rr: Ctest<0x85, MRMDestReg, Xi64, GR64, GR64>; + } + def CTEST8ri : Ctest<0xF6, MRM0r, Xi8, GR8, i8imm>; + def CTEST16ri: Ctest<0xF7, MRM0r, Xi16, GR16, i16imm>, PD; + def CTEST32ri: Ctest<0xF7, MRM0r, Xi32, GR32, i32imm>; + def CTEST64ri32: Ctest<0xF7, MRM0r, Xi64, GR64, i64i32imm>; +} + +let mayLoad = 1 in { + let SchedRW = [WriteALU.Folded] in { + def CTEST8mi : Ctest<0xF6, MRM0m, Xi8, i8mem, i8imm>; + def CTEST16mi: Ctest<0xF7, MRM0m, Xi16, i16mem, i16imm>, PD; + def CTEST32mi: Ctest<0xF7, MRM0m, Xi32, i32mem, i32imm>; + def CTEST64mi32: Ctest<0xF7, MRM0m, Xi64, i64mem, i64i32imm>; + } + let SchedRW = [WriteALU.Folded, WriteALU.ReadAfterFold] in { + def CTEST8mr : Ctest<0x84, MRMDestMem, Xi8, i8mem, GR8>; + def CTEST16mr: Ctest<0x85, MRMDestMem, Xi16, i16mem, GR16>, PD; + def CTEST32mr: Ctest<0x85, MRMDestMem, Xi32, i32mem, GR32>; + def CTEST64mr: Ctest<0x85, MRMDestMem, Xi64, i64mem, GR64>; + } +} diff --git a/llvm/lib/Target/X86/X86InstrFormats.td b/llvm/lib/Target/X86/X86InstrFormats.td index 8798b13a1761..13014293267b 100644 --- a/llvm/lib/Target/X86/X86InstrFormats.td +++ b/llvm/lib/Target/X86/X86InstrFormats.td @@ -255,6 +255,7 @@ class X86Inst opcod, Format f, ImmType i, dag outs, dag ins, bit hasEVEX_L2 = 0; // Does this inst set the EVEX_L2 field? bit hasEVEX_B = 0; // Does this inst set the EVEX_B field? bit hasEVEX_NF = 0; // Does this inst set the EVEX_NF field? + bit hasTwoConditionalOps = 0; // Does this inst have two conditional operands? bits<3> CD8_Form = 0; // Compressed disp8 form - vector-width. // Declare it int rather than bits<4> so that all bits are defined when // assigning to bits<7>. @@ -305,4 +306,5 @@ class X86Inst opcod, Format f, ImmType i, dag outs, dag ins, let TSFlags{49} = hasNoTrackPrefix; let TSFlags{51-50} = explicitOpPrefixBits; let TSFlags{52} = hasEVEX_NF; + let TSFlags{53} = hasTwoConditionalOps; } diff --git a/llvm/lib/Target/X86/X86InstrInfo.td b/llvm/lib/Target/X86/X86InstrInfo.td index ee54796323b8..479278433610 100644 --- a/llvm/lib/Target/X86/X86InstrInfo.td +++ b/llvm/lib/Target/X86/X86InstrInfo.td @@ -45,6 +45,7 @@ include "X86InstrMisc.td" include "X86InstrTBM.td" include "X86InstrArithmetic.td" include "X86InstrCMovSetCC.td" +include "X86InstrConditionalCompare.td" include "X86InstrExtension.td" include "X86InstrControl.td" include "X86InstrShiftRotate.td" diff --git a/llvm/lib/Target/X86/X86InstrOperands.td b/llvm/lib/Target/X86/X86InstrOperands.td index 761458f9cffc..f8f5cd83166e 100644 --- a/llvm/lib/Target/X86/X86InstrOperands.td +++ b/llvm/lib/Target/X86/X86InstrOperands.td @@ -424,6 +424,12 @@ def u4imm : Operand { let OperandType = "OPERAND_IMMEDIATE"; } +def cflags : Operand { + let PrintMethod = "printCondFlags"; + let ParserMatchClass = ImmUnsignedi4AsmOperand; + let OperandType = "OPERAND_IMMEDIATE"; +} + // Unsigned 8-bit immediate used by SSE/AVX instructions. def u8imm : Operand { let PrintMethod = "printU8Imm"; diff --git a/llvm/test/MC/Disassembler/X86/apx/ccmp.txt b/llvm/test/MC/Disassembler/X86/apx/ccmp.txt new file mode 100644 index 000000000000..d2656d62972d --- /dev/null +++ b/llvm/test/MC/Disassembler/X86/apx/ccmp.txt @@ -0,0 +1,1598 @@ +# RUN: llvm-mc -triple x86_64 -disassemble %s | FileCheck %s --check-prefix=ATT +# RUN: llvm-mc -triple x86_64 -disassemble -output-asm-variant=1 %s | FileCheck %s --check-prefix=INTEL + +## Condition flags + +# ATT: ccmpoq {dfv=} %rax, %rbx +# INTEL: ccmpo {dfv=} rbx, rax +0x62,0xf4,0x84,0x00,0x39,0xc3 + +# ATT: ccmpoq {dfv=of} %rax, %rbx +# INTEL: ccmpo {dfv=of} rbx, rax +0x62,0xf4,0xc4,0x00,0x39,0xc3 + +# ATT: ccmpoq {dfv=sf} %rax, %rbx +# INTEL: ccmpo {dfv=sf} rbx, rax +0x62,0xf4,0xa4,0x00,0x39,0xc3 + +# ATT: ccmpoq {dfv=zf} %rax, %rbx +# INTEL: ccmpo {dfv=zf} rbx, rax +0x62,0xf4,0x94,0x00,0x39,0xc3 + +# ATT: ccmpoq {dfv=cf} %rax, %rbx +# INTEL: ccmpo {dfv=cf} rbx, rax +0x62,0xf4,0x8c,0x00,0x39,0xc3 + +# ATT: ccmpoq {dfv=of,sf} %rax, %rbx +# INTEL: ccmpo {dfv=of,sf} rbx, rax +0x62,0xf4,0xe4,0x00,0x39,0xc3 + +# ATT: ccmpoq {dfv=of,sf,zf} %rax, %rbx +# INTEL: ccmpo {dfv=of,sf,zf} rbx, rax +0x62,0xf4,0xf4,0x00,0x39,0xc3 + +# ATT: ccmpoq {dfv=of,sf,zf,cf} %rax, %rbx +# INTEL: ccmpo {dfv=of,sf,zf,cf} rbx, rax +0x62,0xf4,0xfc,0x00,0x39,0xc3 + +## Condition code + +# ATT: ccmpbb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpb {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x02,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpbw {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpb {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x02,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpbw {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpb {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x02,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpbl {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpb {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x02,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpbl {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpb {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x02,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpbq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpb {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x02,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpbq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpb {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x02,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpbb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpb {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x02,0x38,0x5c,0x80,0x7b + +# ATT: ccmpbw {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpb {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x02,0x39,0x54,0x80,0x7b + +# ATT: ccmpbl {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpb {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x02,0x39,0x4c,0x80,0x7b + +# ATT: ccmpbq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpb {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x02,0x39,0x4c,0x80,0x7b + +# ATT: ccmpbb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpb {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x02,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpbw {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpb {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x02,0x3b,0x54,0x80,0x7b + +# ATT: ccmpbl {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpb {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x02,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpbq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpb {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x02,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpbb {dfv=of} $123, %bl +# INTEL: ccmpb {dfv=of} bl, 123 +0x62,0xf4,0x44,0x02,0x80,0xfb,0x7b + +# ATT: ccmpbw {dfv=of} $123, %dx +# INTEL: ccmpb {dfv=of} dx, 123 +0x62,0xf4,0x45,0x02,0x83,0xfa,0x7b + +# ATT: ccmpbl {dfv=of} $123, %ecx +# INTEL: ccmpb {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x02,0x83,0xf9,0x7b + +# ATT: ccmpbq {dfv=of} $123, %r9 +# INTEL: ccmpb {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x02,0x83,0xf9,0x7b + +# ATT: ccmpbw {dfv=of} $1234, %dx +# INTEL: ccmpb {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x02,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpbl {dfv=of} $123456, %ecx +# INTEL: ccmpb {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x02,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpbq {dfv=of} $123456, %r9 +# INTEL: ccmpb {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x02,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpbb {dfv=of} %bl, %dl +# INTEL: ccmpb {dfv=of} dl, bl +0x62,0xf4,0x44,0x02,0x3a,0xd3 + +# ATT: ccmpbw {dfv=of} %dx, %ax +# INTEL: ccmpb {dfv=of} ax, dx +0x62,0xf4,0x45,0x02,0x3b,0xc2 + +# ATT: ccmpbl {dfv=of} %ecx, %edx +# INTEL: ccmpb {dfv=of} edx, ecx +0x62,0xf4,0x44,0x02,0x3b,0xd1 + +# ATT: ccmpbq {dfv=of} %r9, %r15 +# INTEL: ccmpb {dfv=of} r15, r9 +0x62,0x54,0xc4,0x02,0x3b,0xf9 + +# ATT: ccmpfb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpf {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0b,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpfw {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpf {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x0b,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpfw {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpf {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x0b,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpfl {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpf {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0b,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpfl {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpf {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x0b,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpfq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpf {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x0b,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpfq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpf {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x0b,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpfb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpf {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x0b,0x38,0x5c,0x80,0x7b + +# ATT: ccmpfw {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpf {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x0b,0x39,0x54,0x80,0x7b + +# ATT: ccmpfl {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpf {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x0b,0x39,0x4c,0x80,0x7b + +# ATT: ccmpfq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpf {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x0b,0x39,0x4c,0x80,0x7b + +# ATT: ccmpfb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpf {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0b,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpfw {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpf {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x0b,0x3b,0x54,0x80,0x7b + +# ATT: ccmpfl {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpf {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0b,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpfq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpf {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x0b,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpfb {dfv=of} $123, %bl +# INTEL: ccmpf {dfv=of} bl, 123 +0x62,0xf4,0x44,0x0b,0x80,0xfb,0x7b + +# ATT: ccmpfw {dfv=of} $123, %dx +# INTEL: ccmpf {dfv=of} dx, 123 +0x62,0xf4,0x45,0x0b,0x83,0xfa,0x7b + +# ATT: ccmpfl {dfv=of} $123, %ecx +# INTEL: ccmpf {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x0b,0x83,0xf9,0x7b + +# ATT: ccmpfq {dfv=of} $123, %r9 +# INTEL: ccmpf {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x0b,0x83,0xf9,0x7b + +# ATT: ccmpfw {dfv=of} $1234, %dx +# INTEL: ccmpf {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x0b,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpfl {dfv=of} $123456, %ecx +# INTEL: ccmpf {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x0b,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpfq {dfv=of} $123456, %r9 +# INTEL: ccmpf {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x0b,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpfb {dfv=of} %bl, %dl +# INTEL: ccmpf {dfv=of} dl, bl +0x62,0xf4,0x44,0x0b,0x3a,0xd3 + +# ATT: ccmpfw {dfv=of} %dx, %ax +# INTEL: ccmpf {dfv=of} ax, dx +0x62,0xf4,0x45,0x0b,0x3b,0xc2 + +# ATT: ccmpfl {dfv=of} %ecx, %edx +# INTEL: ccmpf {dfv=of} edx, ecx +0x62,0xf4,0x44,0x0b,0x3b,0xd1 + +# ATT: ccmpfq {dfv=of} %r9, %r15 +# INTEL: ccmpf {dfv=of} r15, r9 +0x62,0x54,0xc4,0x0b,0x3b,0xf9 + +# ATT: ccmplb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpl {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0c,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmplw {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpl {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x0c,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmplw {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpl {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x0c,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpll {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpl {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0c,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpll {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpl {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x0c,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmplq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpl {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x0c,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmplq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpl {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x0c,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmplb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpl {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x0c,0x38,0x5c,0x80,0x7b + +# ATT: ccmplw {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpl {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x0c,0x39,0x54,0x80,0x7b + +# ATT: ccmpll {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpl {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x0c,0x39,0x4c,0x80,0x7b + +# ATT: ccmplq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpl {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x0c,0x39,0x4c,0x80,0x7b + +# ATT: ccmplb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpl {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0c,0x3a,0x5c,0x80,0x7b + +# ATT: ccmplw {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpl {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x0c,0x3b,0x54,0x80,0x7b + +# ATT: ccmpll {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpl {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0c,0x3b,0x4c,0x80,0x7b + +# ATT: ccmplq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpl {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x0c,0x3b,0x4c,0x80,0x7b + +# ATT: ccmplb {dfv=of} $123, %bl +# INTEL: ccmpl {dfv=of} bl, 123 +0x62,0xf4,0x44,0x0c,0x80,0xfb,0x7b + +# ATT: ccmplw {dfv=of} $123, %dx +# INTEL: ccmpl {dfv=of} dx, 123 +0x62,0xf4,0x45,0x0c,0x83,0xfa,0x7b + +# ATT: ccmpll {dfv=of} $123, %ecx +# INTEL: ccmpl {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x0c,0x83,0xf9,0x7b + +# ATT: ccmplq {dfv=of} $123, %r9 +# INTEL: ccmpl {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x0c,0x83,0xf9,0x7b + +# ATT: ccmplw {dfv=of} $1234, %dx +# INTEL: ccmpl {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x0c,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpll {dfv=of} $123456, %ecx +# INTEL: ccmpl {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x0c,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmplq {dfv=of} $123456, %r9 +# INTEL: ccmpl {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x0c,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmplb {dfv=of} %bl, %dl +# INTEL: ccmpl {dfv=of} dl, bl +0x62,0xf4,0x44,0x0c,0x3a,0xd3 + +# ATT: ccmplw {dfv=of} %dx, %ax +# INTEL: ccmpl {dfv=of} ax, dx +0x62,0xf4,0x45,0x0c,0x3b,0xc2 + +# ATT: ccmpll {dfv=of} %ecx, %edx +# INTEL: ccmpl {dfv=of} edx, ecx +0x62,0xf4,0x44,0x0c,0x3b,0xd1 + +# ATT: ccmplq {dfv=of} %r9, %r15 +# INTEL: ccmpl {dfv=of} r15, r9 +0x62,0x54,0xc4,0x0c,0x3b,0xf9 + +# ATT: ccmpleb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmple {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0e,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmplew {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmple {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x0e,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmplew {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmple {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x0e,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmplel {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmple {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0e,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmplel {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmple {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x0e,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpleq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmple {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x0e,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpleq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmple {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x0e,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpleb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmple {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x0e,0x38,0x5c,0x80,0x7b + +# ATT: ccmplew {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmple {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x0e,0x39,0x54,0x80,0x7b + +# ATT: ccmplel {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmple {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x0e,0x39,0x4c,0x80,0x7b + +# ATT: ccmpleq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmple {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x0e,0x39,0x4c,0x80,0x7b + +# ATT: ccmpleb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmple {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0e,0x3a,0x5c,0x80,0x7b + +# ATT: ccmplew {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmple {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x0e,0x3b,0x54,0x80,0x7b + +# ATT: ccmplel {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmple {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0e,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpleq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmple {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x0e,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpleb {dfv=of} $123, %bl +# INTEL: ccmple {dfv=of} bl, 123 +0x62,0xf4,0x44,0x0e,0x80,0xfb,0x7b + +# ATT: ccmplew {dfv=of} $123, %dx +# INTEL: ccmple {dfv=of} dx, 123 +0x62,0xf4,0x45,0x0e,0x83,0xfa,0x7b + +# ATT: ccmplel {dfv=of} $123, %ecx +# INTEL: ccmple {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x0e,0x83,0xf9,0x7b + +# ATT: ccmpleq {dfv=of} $123, %r9 +# INTEL: ccmple {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x0e,0x83,0xf9,0x7b + +# ATT: ccmplew {dfv=of} $1234, %dx +# INTEL: ccmple {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x0e,0x81,0xfa,0xd2,0x04 + +# ATT: ccmplel {dfv=of} $123456, %ecx +# INTEL: ccmple {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x0e,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpleq {dfv=of} $123456, %r9 +# INTEL: ccmple {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x0e,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpleb {dfv=of} %bl, %dl +# INTEL: ccmple {dfv=of} dl, bl +0x62,0xf4,0x44,0x0e,0x3a,0xd3 + +# ATT: ccmplew {dfv=of} %dx, %ax +# INTEL: ccmple {dfv=of} ax, dx +0x62,0xf4,0x45,0x0e,0x3b,0xc2 + +# ATT: ccmplel {dfv=of} %ecx, %edx +# INTEL: ccmple {dfv=of} edx, ecx +0x62,0xf4,0x44,0x0e,0x3b,0xd1 + +# ATT: ccmpleq {dfv=of} %r9, %r15 +# INTEL: ccmple {dfv=of} r15, r9 +0x62,0x54,0xc4,0x0e,0x3b,0xf9 + +# ATT: ccmpaeb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpae {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x03,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpaew {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpae {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x03,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpaew {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpae {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x03,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpael {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpae {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x03,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpael {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpae {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x03,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpaeq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpae {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x03,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpaeq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpae {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x03,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpaeb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpae {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x03,0x38,0x5c,0x80,0x7b + +# ATT: ccmpaew {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpae {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x03,0x39,0x54,0x80,0x7b + +# ATT: ccmpael {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpae {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x03,0x39,0x4c,0x80,0x7b + +# ATT: ccmpaeq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpae {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x03,0x39,0x4c,0x80,0x7b + +# ATT: ccmpaeb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpae {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x03,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpaew {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpae {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x03,0x3b,0x54,0x80,0x7b + +# ATT: ccmpael {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpae {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x03,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpaeq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpae {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x03,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpaeb {dfv=of} $123, %bl +# INTEL: ccmpae {dfv=of} bl, 123 +0x62,0xf4,0x44,0x03,0x80,0xfb,0x7b + +# ATT: ccmpaew {dfv=of} $123, %dx +# INTEL: ccmpae {dfv=of} dx, 123 +0x62,0xf4,0x45,0x03,0x83,0xfa,0x7b + +# ATT: ccmpael {dfv=of} $123, %ecx +# INTEL: ccmpae {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x03,0x83,0xf9,0x7b + +# ATT: ccmpaeq {dfv=of} $123, %r9 +# INTEL: ccmpae {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x03,0x83,0xf9,0x7b + +# ATT: ccmpaew {dfv=of} $1234, %dx +# INTEL: ccmpae {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x03,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpael {dfv=of} $123456, %ecx +# INTEL: ccmpae {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x03,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpaeq {dfv=of} $123456, %r9 +# INTEL: ccmpae {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x03,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpaeb {dfv=of} %bl, %dl +# INTEL: ccmpae {dfv=of} dl, bl +0x62,0xf4,0x44,0x03,0x3a,0xd3 + +# ATT: ccmpaew {dfv=of} %dx, %ax +# INTEL: ccmpae {dfv=of} ax, dx +0x62,0xf4,0x45,0x03,0x3b,0xc2 + +# ATT: ccmpael {dfv=of} %ecx, %edx +# INTEL: ccmpae {dfv=of} edx, ecx +0x62,0xf4,0x44,0x03,0x3b,0xd1 + +# ATT: ccmpaeq {dfv=of} %r9, %r15 +# INTEL: ccmpae {dfv=of} r15, r9 +0x62,0x54,0xc4,0x03,0x3b,0xf9 + +# ATT: ccmpab {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpa {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x07,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpaw {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpa {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x07,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpaw {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpa {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x07,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpal {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpa {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x07,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpal {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpa {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x07,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpaq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpa {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x07,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpaq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpa {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x07,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpab {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpa {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x07,0x38,0x5c,0x80,0x7b + +# ATT: ccmpaw {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpa {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x07,0x39,0x54,0x80,0x7b + +# ATT: ccmpal {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpa {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x07,0x39,0x4c,0x80,0x7b + +# ATT: ccmpaq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpa {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x07,0x39,0x4c,0x80,0x7b + +# ATT: ccmpab {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpa {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x07,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpaw {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpa {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x07,0x3b,0x54,0x80,0x7b + +# ATT: ccmpal {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpa {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x07,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpaq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpa {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x07,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpab {dfv=of} $123, %bl +# INTEL: ccmpa {dfv=of} bl, 123 +0x62,0xf4,0x44,0x07,0x80,0xfb,0x7b + +# ATT: ccmpaw {dfv=of} $123, %dx +# INTEL: ccmpa {dfv=of} dx, 123 +0x62,0xf4,0x45,0x07,0x83,0xfa,0x7b + +# ATT: ccmpal {dfv=of} $123, %ecx +# INTEL: ccmpa {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x07,0x83,0xf9,0x7b + +# ATT: ccmpaq {dfv=of} $123, %r9 +# INTEL: ccmpa {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x07,0x83,0xf9,0x7b + +# ATT: ccmpaw {dfv=of} $1234, %dx +# INTEL: ccmpa {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x07,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpal {dfv=of} $123456, %ecx +# INTEL: ccmpa {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x07,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpaq {dfv=of} $123456, %r9 +# INTEL: ccmpa {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x07,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpab {dfv=of} %bl, %dl +# INTEL: ccmpa {dfv=of} dl, bl +0x62,0xf4,0x44,0x07,0x3a,0xd3 + +# ATT: ccmpaw {dfv=of} %dx, %ax +# INTEL: ccmpa {dfv=of} ax, dx +0x62,0xf4,0x45,0x07,0x3b,0xc2 + +# ATT: ccmpal {dfv=of} %ecx, %edx +# INTEL: ccmpa {dfv=of} edx, ecx +0x62,0xf4,0x44,0x07,0x3b,0xd1 + +# ATT: ccmpaq {dfv=of} %r9, %r15 +# INTEL: ccmpa {dfv=of} r15, r9 +0x62,0x54,0xc4,0x07,0x3b,0xf9 + +# ATT: ccmpgeb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpge {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0d,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpgew {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpge {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x0d,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpgew {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpge {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x0d,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpgel {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpge {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0d,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpgel {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpge {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x0d,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpgeq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpge {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x0d,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpgeq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpge {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x0d,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpgeb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpge {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x0d,0x38,0x5c,0x80,0x7b + +# ATT: ccmpgew {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpge {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x0d,0x39,0x54,0x80,0x7b + +# ATT: ccmpgel {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpge {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x0d,0x39,0x4c,0x80,0x7b + +# ATT: ccmpgeq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpge {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x0d,0x39,0x4c,0x80,0x7b + +# ATT: ccmpgeb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpge {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0d,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpgew {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpge {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x0d,0x3b,0x54,0x80,0x7b + +# ATT: ccmpgel {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpge {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0d,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpgeq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpge {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x0d,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpgeb {dfv=of} $123, %bl +# INTEL: ccmpge {dfv=of} bl, 123 +0x62,0xf4,0x44,0x0d,0x80,0xfb,0x7b + +# ATT: ccmpgew {dfv=of} $123, %dx +# INTEL: ccmpge {dfv=of} dx, 123 +0x62,0xf4,0x45,0x0d,0x83,0xfa,0x7b + +# ATT: ccmpgel {dfv=of} $123, %ecx +# INTEL: ccmpge {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x0d,0x83,0xf9,0x7b + +# ATT: ccmpgeq {dfv=of} $123, %r9 +# INTEL: ccmpge {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x0d,0x83,0xf9,0x7b + +# ATT: ccmpgew {dfv=of} $1234, %dx +# INTEL: ccmpge {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x0d,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpgel {dfv=of} $123456, %ecx +# INTEL: ccmpge {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x0d,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpgeq {dfv=of} $123456, %r9 +# INTEL: ccmpge {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x0d,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpgeb {dfv=of} %bl, %dl +# INTEL: ccmpge {dfv=of} dl, bl +0x62,0xf4,0x44,0x0d,0x3a,0xd3 + +# ATT: ccmpgew {dfv=of} %dx, %ax +# INTEL: ccmpge {dfv=of} ax, dx +0x62,0xf4,0x45,0x0d,0x3b,0xc2 + +# ATT: ccmpgel {dfv=of} %ecx, %edx +# INTEL: ccmpge {dfv=of} edx, ecx +0x62,0xf4,0x44,0x0d,0x3b,0xd1 + +# ATT: ccmpgeq {dfv=of} %r9, %r15 +# INTEL: ccmpge {dfv=of} r15, r9 +0x62,0x54,0xc4,0x0d,0x3b,0xf9 + +# ATT: ccmpgb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpg {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0f,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpgw {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpg {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x0f,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpgw {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpg {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x0f,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpgl {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpg {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0f,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpgl {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpg {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x0f,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpgq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpg {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x0f,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpgq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpg {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x0f,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpgb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpg {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x0f,0x38,0x5c,0x80,0x7b + +# ATT: ccmpgw {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpg {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x0f,0x39,0x54,0x80,0x7b + +# ATT: ccmpgl {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpg {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x0f,0x39,0x4c,0x80,0x7b + +# ATT: ccmpgq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpg {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x0f,0x39,0x4c,0x80,0x7b + +# ATT: ccmpgb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpg {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0f,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpgw {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpg {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x0f,0x3b,0x54,0x80,0x7b + +# ATT: ccmpgl {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpg {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0f,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpgq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpg {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x0f,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpgb {dfv=of} $123, %bl +# INTEL: ccmpg {dfv=of} bl, 123 +0x62,0xf4,0x44,0x0f,0x80,0xfb,0x7b + +# ATT: ccmpgw {dfv=of} $123, %dx +# INTEL: ccmpg {dfv=of} dx, 123 +0x62,0xf4,0x45,0x0f,0x83,0xfa,0x7b + +# ATT: ccmpgl {dfv=of} $123, %ecx +# INTEL: ccmpg {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x0f,0x83,0xf9,0x7b + +# ATT: ccmpgq {dfv=of} $123, %r9 +# INTEL: ccmpg {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x0f,0x83,0xf9,0x7b + +# ATT: ccmpgw {dfv=of} $1234, %dx +# INTEL: ccmpg {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x0f,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpgl {dfv=of} $123456, %ecx +# INTEL: ccmpg {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x0f,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpgq {dfv=of} $123456, %r9 +# INTEL: ccmpg {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x0f,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpgb {dfv=of} %bl, %dl +# INTEL: ccmpg {dfv=of} dl, bl +0x62,0xf4,0x44,0x0f,0x3a,0xd3 + +# ATT: ccmpgw {dfv=of} %dx, %ax +# INTEL: ccmpg {dfv=of} ax, dx +0x62,0xf4,0x45,0x0f,0x3b,0xc2 + +# ATT: ccmpgl {dfv=of} %ecx, %edx +# INTEL: ccmpg {dfv=of} edx, ecx +0x62,0xf4,0x44,0x0f,0x3b,0xd1 + +# ATT: ccmpgq {dfv=of} %r9, %r15 +# INTEL: ccmpg {dfv=of} r15, r9 +0x62,0x54,0xc4,0x0f,0x3b,0xf9 + +# ATT: ccmpnob {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpno {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x01,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpnow {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpno {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x01,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpnow {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpno {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x01,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpnol {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpno {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x01,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpnol {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpno {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x01,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpnoq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpno {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x01,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpnoq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpno {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x01,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpnob {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpno {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x01,0x38,0x5c,0x80,0x7b + +# ATT: ccmpnow {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpno {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x01,0x39,0x54,0x80,0x7b + +# ATT: ccmpnol {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpno {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x01,0x39,0x4c,0x80,0x7b + +# ATT: ccmpnoq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpno {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x01,0x39,0x4c,0x80,0x7b + +# ATT: ccmpnob {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpno {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x01,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpnow {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpno {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x01,0x3b,0x54,0x80,0x7b + +# ATT: ccmpnol {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpno {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x01,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpnoq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpno {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x01,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpnob {dfv=of} $123, %bl +# INTEL: ccmpno {dfv=of} bl, 123 +0x62,0xf4,0x44,0x01,0x80,0xfb,0x7b + +# ATT: ccmpnow {dfv=of} $123, %dx +# INTEL: ccmpno {dfv=of} dx, 123 +0x62,0xf4,0x45,0x01,0x83,0xfa,0x7b + +# ATT: ccmpnol {dfv=of} $123, %ecx +# INTEL: ccmpno {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x01,0x83,0xf9,0x7b + +# ATT: ccmpnoq {dfv=of} $123, %r9 +# INTEL: ccmpno {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x01,0x83,0xf9,0x7b + +# ATT: ccmpnow {dfv=of} $1234, %dx +# INTEL: ccmpno {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x01,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpnol {dfv=of} $123456, %ecx +# INTEL: ccmpno {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x01,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpnoq {dfv=of} $123456, %r9 +# INTEL: ccmpno {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x01,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpnob {dfv=of} %bl, %dl +# INTEL: ccmpno {dfv=of} dl, bl +0x62,0xf4,0x44,0x01,0x3a,0xd3 + +# ATT: ccmpnow {dfv=of} %dx, %ax +# INTEL: ccmpno {dfv=of} ax, dx +0x62,0xf4,0x45,0x01,0x3b,0xc2 + +# ATT: ccmpnol {dfv=of} %ecx, %edx +# INTEL: ccmpno {dfv=of} edx, ecx +0x62,0xf4,0x44,0x01,0x3b,0xd1 + +# ATT: ccmpnoq {dfv=of} %r9, %r15 +# INTEL: ccmpno {dfv=of} r15, r9 +0x62,0x54,0xc4,0x01,0x3b,0xf9 + +# ATT: ccmpnsb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpns {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x09,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpnsw {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpns {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x09,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpnsw {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpns {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x09,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpnsl {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpns {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x09,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpnsl {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpns {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x09,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpnsq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpns {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x09,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpnsq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpns {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x09,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpnsb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpns {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x09,0x38,0x5c,0x80,0x7b + +# ATT: ccmpnsw {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpns {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x09,0x39,0x54,0x80,0x7b + +# ATT: ccmpnsl {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpns {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x09,0x39,0x4c,0x80,0x7b + +# ATT: ccmpnsq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpns {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x09,0x39,0x4c,0x80,0x7b + +# ATT: ccmpnsb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpns {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x09,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpnsw {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpns {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x09,0x3b,0x54,0x80,0x7b + +# ATT: ccmpnsl {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpns {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x09,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpnsq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpns {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x09,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpnsb {dfv=of} $123, %bl +# INTEL: ccmpns {dfv=of} bl, 123 +0x62,0xf4,0x44,0x09,0x80,0xfb,0x7b + +# ATT: ccmpnsw {dfv=of} $123, %dx +# INTEL: ccmpns {dfv=of} dx, 123 +0x62,0xf4,0x45,0x09,0x83,0xfa,0x7b + +# ATT: ccmpnsl {dfv=of} $123, %ecx +# INTEL: ccmpns {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x09,0x83,0xf9,0x7b + +# ATT: ccmpnsq {dfv=of} $123, %r9 +# INTEL: ccmpns {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x09,0x83,0xf9,0x7b + +# ATT: ccmpnsw {dfv=of} $1234, %dx +# INTEL: ccmpns {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x09,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpnsl {dfv=of} $123456, %ecx +# INTEL: ccmpns {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x09,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpnsq {dfv=of} $123456, %r9 +# INTEL: ccmpns {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x09,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpnsb {dfv=of} %bl, %dl +# INTEL: ccmpns {dfv=of} dl, bl +0x62,0xf4,0x44,0x09,0x3a,0xd3 + +# ATT: ccmpnsw {dfv=of} %dx, %ax +# INTEL: ccmpns {dfv=of} ax, dx +0x62,0xf4,0x45,0x09,0x3b,0xc2 + +# ATT: ccmpnsl {dfv=of} %ecx, %edx +# INTEL: ccmpns {dfv=of} edx, ecx +0x62,0xf4,0x44,0x09,0x3b,0xd1 + +# ATT: ccmpnsq {dfv=of} %r9, %r15 +# INTEL: ccmpns {dfv=of} r15, r9 +0x62,0x54,0xc4,0x09,0x3b,0xf9 + +# ATT: ccmpneb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpne {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x05,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpnew {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpne {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x05,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpnew {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpne {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x05,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpnel {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpne {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x05,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpnel {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpne {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x05,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpneq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpne {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x05,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpneq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpne {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x05,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpneb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpne {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x05,0x38,0x5c,0x80,0x7b + +# ATT: ccmpnew {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpne {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x05,0x39,0x54,0x80,0x7b + +# ATT: ccmpnel {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpne {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x05,0x39,0x4c,0x80,0x7b + +# ATT: ccmpneq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpne {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x05,0x39,0x4c,0x80,0x7b + +# ATT: ccmpneb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpne {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x05,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpnew {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpne {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x05,0x3b,0x54,0x80,0x7b + +# ATT: ccmpnel {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpne {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x05,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpneq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpne {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x05,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpneb {dfv=of} $123, %bl +# INTEL: ccmpne {dfv=of} bl, 123 +0x62,0xf4,0x44,0x05,0x80,0xfb,0x7b + +# ATT: ccmpnew {dfv=of} $123, %dx +# INTEL: ccmpne {dfv=of} dx, 123 +0x62,0xf4,0x45,0x05,0x83,0xfa,0x7b + +# ATT: ccmpnel {dfv=of} $123, %ecx +# INTEL: ccmpne {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x05,0x83,0xf9,0x7b + +# ATT: ccmpneq {dfv=of} $123, %r9 +# INTEL: ccmpne {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x05,0x83,0xf9,0x7b + +# ATT: ccmpnew {dfv=of} $1234, %dx +# INTEL: ccmpne {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x05,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpnel {dfv=of} $123456, %ecx +# INTEL: ccmpne {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x05,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpneq {dfv=of} $123456, %r9 +# INTEL: ccmpne {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x05,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpneb {dfv=of} %bl, %dl +# INTEL: ccmpne {dfv=of} dl, bl +0x62,0xf4,0x44,0x05,0x3a,0xd3 + +# ATT: ccmpnew {dfv=of} %dx, %ax +# INTEL: ccmpne {dfv=of} ax, dx +0x62,0xf4,0x45,0x05,0x3b,0xc2 + +# ATT: ccmpnel {dfv=of} %ecx, %edx +# INTEL: ccmpne {dfv=of} edx, ecx +0x62,0xf4,0x44,0x05,0x3b,0xd1 + +# ATT: ccmpneq {dfv=of} %r9, %r15 +# INTEL: ccmpne {dfv=of} r15, r9 +0x62,0x54,0xc4,0x05,0x3b,0xf9 + +# ATT: ccmpob {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpo {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x00,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpow {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpo {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x00,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpow {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpo {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x00,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpol {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpo {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x00,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpol {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpo {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x00,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpoq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpo {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x00,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpoq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpo {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x00,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpob {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpo {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x00,0x38,0x5c,0x80,0x7b + +# ATT: ccmpow {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpo {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x00,0x39,0x54,0x80,0x7b + +# ATT: ccmpol {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpo {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x00,0x39,0x4c,0x80,0x7b + +# ATT: ccmpoq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpo {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x00,0x39,0x4c,0x80,0x7b + +# ATT: ccmpob {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpo {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x00,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpow {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpo {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x00,0x3b,0x54,0x80,0x7b + +# ATT: ccmpol {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpo {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x00,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpoq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpo {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x00,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpob {dfv=of} $123, %bl +# INTEL: ccmpo {dfv=of} bl, 123 +0x62,0xf4,0x44,0x00,0x80,0xfb,0x7b + +# ATT: ccmpow {dfv=of} $123, %dx +# INTEL: ccmpo {dfv=of} dx, 123 +0x62,0xf4,0x45,0x00,0x83,0xfa,0x7b + +# ATT: ccmpol {dfv=of} $123, %ecx +# INTEL: ccmpo {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x00,0x83,0xf9,0x7b + +# ATT: ccmpoq {dfv=of} $123, %r9 +# INTEL: ccmpo {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x00,0x83,0xf9,0x7b + +# ATT: ccmpow {dfv=of} $1234, %dx +# INTEL: ccmpo {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x00,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpol {dfv=of} $123456, %ecx +# INTEL: ccmpo {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x00,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpoq {dfv=of} $123456, %r9 +# INTEL: ccmpo {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x00,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpob {dfv=of} %bl, %dl +# INTEL: ccmpo {dfv=of} dl, bl +0x62,0xf4,0x44,0x00,0x3a,0xd3 + +# ATT: ccmpow {dfv=of} %dx, %ax +# INTEL: ccmpo {dfv=of} ax, dx +0x62,0xf4,0x45,0x00,0x3b,0xc2 + +# ATT: ccmpol {dfv=of} %ecx, %edx +# INTEL: ccmpo {dfv=of} edx, ecx +0x62,0xf4,0x44,0x00,0x3b,0xd1 + +# ATT: ccmpoq {dfv=of} %r9, %r15 +# INTEL: ccmpo {dfv=of} r15, r9 +0x62,0x54,0xc4,0x00,0x3b,0xf9 + +# ATT: ccmpsb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmps {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x08,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpsw {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmps {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x08,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpsw {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmps {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x08,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpsl {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmps {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x08,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpsl {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmps {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x08,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpsq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmps {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x08,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpsq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmps {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x08,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpsb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmps {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x08,0x38,0x5c,0x80,0x7b + +# ATT: ccmpsw {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmps {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x08,0x39,0x54,0x80,0x7b + +# ATT: ccmpsl {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmps {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x08,0x39,0x4c,0x80,0x7b + +# ATT: ccmpsq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmps {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x08,0x39,0x4c,0x80,0x7b + +# ATT: ccmpsb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmps {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x08,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpsw {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmps {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x08,0x3b,0x54,0x80,0x7b + +# ATT: ccmpsl {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmps {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x08,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpsq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmps {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x08,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpsb {dfv=of} $123, %bl +# INTEL: ccmps {dfv=of} bl, 123 +0x62,0xf4,0x44,0x08,0x80,0xfb,0x7b + +# ATT: ccmpsw {dfv=of} $123, %dx +# INTEL: ccmps {dfv=of} dx, 123 +0x62,0xf4,0x45,0x08,0x83,0xfa,0x7b + +# ATT: ccmpsl {dfv=of} $123, %ecx +# INTEL: ccmps {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x08,0x83,0xf9,0x7b + +# ATT: ccmpsq {dfv=of} $123, %r9 +# INTEL: ccmps {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x08,0x83,0xf9,0x7b + +# ATT: ccmpsw {dfv=of} $1234, %dx +# INTEL: ccmps {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x08,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpsl {dfv=of} $123456, %ecx +# INTEL: ccmps {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x08,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpsq {dfv=of} $123456, %r9 +# INTEL: ccmps {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x08,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpsb {dfv=of} %bl, %dl +# INTEL: ccmps {dfv=of} dl, bl +0x62,0xf4,0x44,0x08,0x3a,0xd3 + +# ATT: ccmpsw {dfv=of} %dx, %ax +# INTEL: ccmps {dfv=of} ax, dx +0x62,0xf4,0x45,0x08,0x3b,0xc2 + +# ATT: ccmpsl {dfv=of} %ecx, %edx +# INTEL: ccmps {dfv=of} edx, ecx +0x62,0xf4,0x44,0x08,0x3b,0xd1 + +# ATT: ccmpsq {dfv=of} %r9, %r15 +# INTEL: ccmps {dfv=of} r15, r9 +0x62,0x54,0xc4,0x08,0x3b,0xf9 + +# ATT: ccmptb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpt {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0a,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmptw {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpt {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x0a,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmptw {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpt {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x0a,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmptl {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpt {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0a,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmptl {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpt {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x0a,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmptq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpt {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x0a,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmptq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpt {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x0a,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmptb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpt {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x0a,0x38,0x5c,0x80,0x7b + +# ATT: ccmptw {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpt {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x0a,0x39,0x54,0x80,0x7b + +# ATT: ccmptl {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpt {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x0a,0x39,0x4c,0x80,0x7b + +# ATT: ccmptq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpt {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x0a,0x39,0x4c,0x80,0x7b + +# ATT: ccmptb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpt {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0a,0x3a,0x5c,0x80,0x7b + +# ATT: ccmptw {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpt {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x0a,0x3b,0x54,0x80,0x7b + +# ATT: ccmptl {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpt {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0a,0x3b,0x4c,0x80,0x7b + +# ATT: ccmptq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpt {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x0a,0x3b,0x4c,0x80,0x7b + +# ATT: ccmptb {dfv=of} $123, %bl +# INTEL: ccmpt {dfv=of} bl, 123 +0x62,0xf4,0x44,0x0a,0x80,0xfb,0x7b + +# ATT: ccmptw {dfv=of} $123, %dx +# INTEL: ccmpt {dfv=of} dx, 123 +0x62,0xf4,0x45,0x0a,0x83,0xfa,0x7b + +# ATT: ccmptl {dfv=of} $123, %ecx +# INTEL: ccmpt {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x0a,0x83,0xf9,0x7b + +# ATT: ccmptq {dfv=of} $123, %r9 +# INTEL: ccmpt {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x0a,0x83,0xf9,0x7b + +# ATT: ccmptw {dfv=of} $1234, %dx +# INTEL: ccmpt {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x0a,0x81,0xfa,0xd2,0x04 + +# ATT: ccmptl {dfv=of} $123456, %ecx +# INTEL: ccmpt {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x0a,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmptq {dfv=of} $123456, %r9 +# INTEL: ccmpt {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x0a,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmptb {dfv=of} %bl, %dl +# INTEL: ccmpt {dfv=of} dl, bl +0x62,0xf4,0x44,0x0a,0x3a,0xd3 + +# ATT: ccmptw {dfv=of} %dx, %ax +# INTEL: ccmpt {dfv=of} ax, dx +0x62,0xf4,0x45,0x0a,0x3b,0xc2 + +# ATT: ccmptl {dfv=of} %ecx, %edx +# INTEL: ccmpt {dfv=of} edx, ecx +0x62,0xf4,0x44,0x0a,0x3b,0xd1 + +# ATT: ccmptq {dfv=of} %r9, %r15 +# INTEL: ccmpt {dfv=of} r15, r9 +0x62,0x54,0xc4,0x0a,0x3b,0xf9 + +# ATT: ccmpeb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpe {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x04,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpew {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpe {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x04,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpew {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpe {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x04,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpel {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpe {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x04,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpel {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpe {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x04,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpeq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpe {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x04,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpeq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpe {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x04,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpeb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpe {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x04,0x38,0x5c,0x80,0x7b + +# ATT: ccmpew {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpe {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x04,0x39,0x54,0x80,0x7b + +# ATT: ccmpel {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpe {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x04,0x39,0x4c,0x80,0x7b + +# ATT: ccmpeq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpe {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x04,0x39,0x4c,0x80,0x7b + +# ATT: ccmpeb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpe {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x04,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpew {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpe {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x04,0x3b,0x54,0x80,0x7b + +# ATT: ccmpel {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpe {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x04,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpeq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpe {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x04,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpeb {dfv=of} $123, %bl +# INTEL: ccmpe {dfv=of} bl, 123 +0x62,0xf4,0x44,0x04,0x80,0xfb,0x7b + +# ATT: ccmpew {dfv=of} $123, %dx +# INTEL: ccmpe {dfv=of} dx, 123 +0x62,0xf4,0x45,0x04,0x83,0xfa,0x7b + +# ATT: ccmpel {dfv=of} $123, %ecx +# INTEL: ccmpe {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x04,0x83,0xf9,0x7b + +# ATT: ccmpeq {dfv=of} $123, %r9 +# INTEL: ccmpe {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x04,0x83,0xf9,0x7b + +# ATT: ccmpew {dfv=of} $1234, %dx +# INTEL: ccmpe {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x04,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpel {dfv=of} $123456, %ecx +# INTEL: ccmpe {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x04,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpeq {dfv=of} $123456, %r9 +# INTEL: ccmpe {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x04,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpeb {dfv=of} %bl, %dl +# INTEL: ccmpe {dfv=of} dl, bl +0x62,0xf4,0x44,0x04,0x3a,0xd3 + +# ATT: ccmpew {dfv=of} %dx, %ax +# INTEL: ccmpe {dfv=of} ax, dx +0x62,0xf4,0x45,0x04,0x3b,0xc2 + +# ATT: ccmpel {dfv=of} %ecx, %edx +# INTEL: ccmpe {dfv=of} edx, ecx +0x62,0xf4,0x44,0x04,0x3b,0xd1 + +# ATT: ccmpeq {dfv=of} %r9, %r15 +# INTEL: ccmpe {dfv=of} r15, r9 +0x62,0x54,0xc4,0x04,0x3b,0xf9 diff --git a/llvm/test/MC/Disassembler/X86/apx/reverse-encoding.txt b/llvm/test/MC/Disassembler/X86/apx/reverse-encoding.txt index 94499299477d..9e812e370146 100644 --- a/llvm/test/MC/Disassembler/X86/apx/reverse-encoding.txt +++ b/llvm/test/MC/Disassembler/X86/apx/reverse-encoding.txt @@ -412,3 +412,21 @@ # ATT: movbeq %r16, %r17 # INTEL: movbe r17, r16 0x62,0xec,0xfc,0x08,0x60,0xc8 + +## ccmp + +# ATT: ccmpob {dfv=} %r16b, %r17b +# INTEL: ccmpo {dfv=} r17b, r16b +0x62,0xec,0x04,0x00,0x3a,0xc8 + +# ATT: ccmpow {dfv=} %r16w, %r17w +# INTEL: ccmpo {dfv=} r17w, r16w +0x62,0xec,0x05,0x00,0x3b,0xc8 + +# ATT: ccmpol {dfv=} %r16d, %r17d +# INTEL: ccmpo {dfv=} r17d, r16d +0x62,0xec,0x04,0x00,0x3b,0xc8 + +# ATT: ccmpoq {dfv=} %r16, %r17 +# INTEL: ccmpo {dfv=} r17, r16 +0x62,0xec,0x84,0x00,0x3b,0xc8 diff --git a/llvm/test/MC/X86/apx/ccmp-att-error.s b/llvm/test/MC/X86/apx/ccmp-att-error.s new file mode 100644 index 000000000000..5a1237195a22 --- /dev/null +++ b/llvm/test/MC/X86/apx/ccmp-att-error.s @@ -0,0 +1,29 @@ +# RUN: not llvm-mc -triple x86_64 -show-encoding %s 2>&1 | FileCheck --strict-whitespace %s + +# CHECK: [[#@LINE+2]]:8: error: Expected { at this point +# CHECK: ccmpeq $1 %rax, %rbx +ccmpeq $1 %rax, %rbx + +# CHECK: [[#@LINE+2]]:9: error: Expected dfv at this point +# CHECK: ccmpeq {sf} %rax, %rbx +ccmpeq {sf} %rax, %rbx + +# CHECK: [[#@LINE+2]]:12: error: Expected = at this point +# CHECK: ccmpeq {dfv:sf} %rax, %rbx +ccmpeq {dfv:sf} %rax, %rbx + +# CHECK: [[#@LINE+2]]:18: error: Expected } or , at this point +# CHECK: ccmpeq {dfv=sf,cf%rax, %rbx +ccmpeq {dfv=sf,cf%rax, %rbx + +# CHECK: [[#@LINE+2]]:13: error: Invalid conditional flags +# CHECK: ccmpeq {dfv=pf} %rax, %rbx +ccmpeq {dfv=pf} %rax, %rbx + +# CHECK: [[#@LINE+2]]:19: error: Duplicated conditional flag +# CHECK: ccmpeq {dfv=of,zf,of} %rax, %rbx +ccmpeq {dfv=of,zf,of} %rax, %rbx + +# CHECK: [[#@LINE+2]]:24: error: Expected } at this point +# CHECK: ccmpeq {dfv=of,sf,zf,cf,of} %rax, %rbx +ccmpeq {dfv=of,sf,zf,cf,of} %rax, %rbx diff --git a/llvm/test/MC/X86/apx/ccmp-att.s b/llvm/test/MC/X86/apx/ccmp-att.s new file mode 100644 index 000000000000..0b78562f32e1 --- /dev/null +++ b/llvm/test/MC/X86/apx/ccmp-att.s @@ -0,0 +1,1213 @@ +# RUN: llvm-mc -triple x86_64 -show-encoding %s | FileCheck %s +# RUN: not llvm-mc -triple i386 -show-encoding %s 2>&1 | FileCheck %s --check-prefix=ERROR + +# ERROR-COUNT-401: error: +# ERROR-NOT: error: +## Condition flags + +# CHECK: ccmpoq {dfv=} %rax, %rbx +# CHECK: encoding: [0x62,0xf4,0x84,0x00,0x39,0xc3] + ccmpoq {dfv=} %rax, %rbx +# CHECK: ccmpoq {dfv=of} %rax, %rbx +# CHECK: encoding: [0x62,0xf4,0xc4,0x00,0x39,0xc3] + ccmpoq {dfv=of} %rax, %rbx +# CHECK: ccmpoq {dfv=sf} %rax, %rbx +# CHECK: encoding: [0x62,0xf4,0xa4,0x00,0x39,0xc3] + ccmpoq {dfv=sf} %rax, %rbx +# CHECK: ccmpoq {dfv=zf} %rax, %rbx +# CHECK: encoding: [0x62,0xf4,0x94,0x00,0x39,0xc3] + ccmpoq {dfv=zf} %rax, %rbx +# CHECK: ccmpoq {dfv=cf} %rax, %rbx +# CHECK: encoding: [0x62,0xf4,0x8c,0x00,0x39,0xc3] + ccmpoq {dfv=cf} %rax, %rbx +# CHECK: ccmpoq {dfv=of,sf} %rax, %rbx +# CHECK: encoding: [0x62,0xf4,0xe4,0x00,0x39,0xc3] + ccmpoq {dfv=of,sf} %rax, %rbx +# CHECK: ccmpoq {dfv=of,sf} %rax, %rbx +# CHECK: encoding: [0x62,0xf4,0xe4,0x00,0x39,0xc3] + ccmpoq {dfv=sf,of} %rax, %rbx +# CHECK: ccmpoq {dfv=of,sf,zf} %rax, %rbx +# CHECK: encoding: [0x62,0xf4,0xf4,0x00,0x39,0xc3] + ccmpoq {dfv=of,sf,zf} %rax, %rbx +# CHECK: ccmpoq {dfv=of,sf,zf} %rax, %rbx +# CHECK: encoding: [0x62,0xf4,0xf4,0x00,0x39,0xc3] + ccmpoq {dfv=zf,of,sf} %rax, %rbx +# CHECK: ccmpoq {dfv=of,sf,zf,cf} %rax, %rbx +# CHECK: encoding: [0x62,0xf4,0xfc,0x00,0x39,0xc3] + ccmpoq {dfv=of,sf,zf,cf} %rax, %rbx +# CHECK: ccmpoq {dfv=of,sf,zf,cf} %rax, %rbx +# CHECK: encoding: [0x62,0xf4,0xfc,0x00,0x39,0xc3] + ccmpoq {dfv=cf,zf,sf,of} %rax, %rbx + +## Condition code + +# CHECK: ccmpbb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x80,0x7c,0x80,0x7b,0x7b] + ccmpbb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpbw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x02,0x83,0x7c,0x80,0x7b,0x7b] + ccmpbw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpbw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x02,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpbw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpbl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x83,0x7c,0x80,0x7b,0x7b] + ccmpbl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpbl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpbl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpbq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x02,0x83,0x7c,0x80,0x7b,0x7b] + ccmpbq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpbq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x02,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpbq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpbb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x38,0x5c,0x80,0x7b] + ccmpbb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpbw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x02,0x39,0x54,0x80,0x7b] + ccmpbw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpbl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x39,0x4c,0x80,0x7b] + ccmpbl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpbq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x02,0x39,0x4c,0x80,0x7b] + ccmpbq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpbb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x3a,0x5c,0x80,0x7b] + ccmpbb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpbw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x02,0x3b,0x54,0x80,0x7b] + ccmpbw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpbl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x3b,0x4c,0x80,0x7b] + ccmpbl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpbq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x02,0x3b,0x4c,0x80,0x7b] + ccmpbq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpbb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x02,0x80,0xfb,0x7b] + ccmpbb {dfv=of} $123, %bl +# CHECK: ccmpbw {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x02,0x83,0xfa,0x7b] + ccmpbw {dfv=of} $123, %dx +# CHECK: ccmpbl {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x02,0x83,0xf9,0x7b] + ccmpbl {dfv=of} $123, %ecx +# CHECK: ccmpbq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x02,0x83,0xf9,0x7b] + ccmpbq {dfv=of} $123, %r9 +# CHECK: ccmpbw {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x02,0x81,0xfa,0xd2,0x04] + ccmpbw {dfv=of} $1234, %dx +# CHECK: ccmpbl {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x02,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpbl {dfv=of} $123456, %ecx +# CHECK: ccmpbq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x02,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpbq {dfv=of} $123456, %r9 +# CHECK: ccmpbb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x02,0x38,0xda] + ccmpbb {dfv=of} %bl, %dl +# CHECK: ccmpbw {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x02,0x39,0xd0] + ccmpbw {dfv=of} %dx, %ax +# CHECK: ccmpbl {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x02,0x39,0xca] + ccmpbl {dfv=of} %ecx, %edx +# CHECK: ccmpbq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x02,0x39,0xcf] + ccmpbq {dfv=of} %r9, %r15 +# CHECK: ccmpfb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x80,0x7c,0x80,0x7b,0x7b] + ccmpfb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpfw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0b,0x83,0x7c,0x80,0x7b,0x7b] + ccmpfw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpfw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0b,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpfw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpfl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x83,0x7c,0x80,0x7b,0x7b] + ccmpfl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpfl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpfl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpfq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0b,0x83,0x7c,0x80,0x7b,0x7b] + ccmpfq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpfq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0b,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpfq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpfb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x38,0x5c,0x80,0x7b] + ccmpfb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpfw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0b,0x39,0x54,0x80,0x7b] + ccmpfw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpfl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x39,0x4c,0x80,0x7b] + ccmpfl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpfq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x0b,0x39,0x4c,0x80,0x7b] + ccmpfq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpfb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x3a,0x5c,0x80,0x7b] + ccmpfb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpfw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0b,0x3b,0x54,0x80,0x7b] + ccmpfw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpfl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x3b,0x4c,0x80,0x7b] + ccmpfl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpfq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0b,0x3b,0x4c,0x80,0x7b] + ccmpfq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpfb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0b,0x80,0xfb,0x7b] + ccmpfb {dfv=of} $123, %bl +# CHECK: ccmpfw {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0b,0x83,0xfa,0x7b] + ccmpfw {dfv=of} $123, %dx +# CHECK: ccmpfl {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0b,0x83,0xf9,0x7b] + ccmpfl {dfv=of} $123, %ecx +# CHECK: ccmpfq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0b,0x83,0xf9,0x7b] + ccmpfq {dfv=of} $123, %r9 +# CHECK: ccmpfw {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0b,0x81,0xfa,0xd2,0x04] + ccmpfw {dfv=of} $1234, %dx +# CHECK: ccmpfl {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0b,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpfl {dfv=of} $123456, %ecx +# CHECK: ccmpfq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0b,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpfq {dfv=of} $123456, %r9 +# CHECK: ccmpfb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x0b,0x38,0xda] + ccmpfb {dfv=of} %bl, %dl +# CHECK: ccmpfw {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x0b,0x39,0xd0] + ccmpfw {dfv=of} %dx, %ax +# CHECK: ccmpfl {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x0b,0x39,0xca] + ccmpfl {dfv=of} %ecx, %edx +# CHECK: ccmpfq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x0b,0x39,0xcf] + ccmpfq {dfv=of} %r9, %r15 +# CHECK: ccmplb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x80,0x7c,0x80,0x7b,0x7b] + ccmplb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmplw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0c,0x83,0x7c,0x80,0x7b,0x7b] + ccmplw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmplw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0c,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmplw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpll {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x83,0x7c,0x80,0x7b,0x7b] + ccmpll {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpll {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpll {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmplq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0c,0x83,0x7c,0x80,0x7b,0x7b] + ccmplq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmplq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0c,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmplq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmplb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x38,0x5c,0x80,0x7b] + ccmplb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmplw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0c,0x39,0x54,0x80,0x7b] + ccmplw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpll {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x39,0x4c,0x80,0x7b] + ccmpll {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmplq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x0c,0x39,0x4c,0x80,0x7b] + ccmplq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmplb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x3a,0x5c,0x80,0x7b] + ccmplb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmplw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0c,0x3b,0x54,0x80,0x7b] + ccmplw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpll {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x3b,0x4c,0x80,0x7b] + ccmpll {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmplq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0c,0x3b,0x4c,0x80,0x7b] + ccmplq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmplb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0c,0x80,0xfb,0x7b] + ccmplb {dfv=of} $123, %bl +# CHECK: ccmplw {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0c,0x83,0xfa,0x7b] + ccmplw {dfv=of} $123, %dx +# CHECK: ccmpll {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0c,0x83,0xf9,0x7b] + ccmpll {dfv=of} $123, %ecx +# CHECK: ccmplq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0c,0x83,0xf9,0x7b] + ccmplq {dfv=of} $123, %r9 +# CHECK: ccmplw {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0c,0x81,0xfa,0xd2,0x04] + ccmplw {dfv=of} $1234, %dx +# CHECK: ccmpll {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0c,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpll {dfv=of} $123456, %ecx +# CHECK: ccmplq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0c,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmplq {dfv=of} $123456, %r9 +# CHECK: ccmplb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x0c,0x38,0xda] + ccmplb {dfv=of} %bl, %dl +# CHECK: ccmplw {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x0c,0x39,0xd0] + ccmplw {dfv=of} %dx, %ax +# CHECK: ccmpll {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x0c,0x39,0xca] + ccmpll {dfv=of} %ecx, %edx +# CHECK: ccmplq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x0c,0x39,0xcf] + ccmplq {dfv=of} %r9, %r15 +# CHECK: ccmpleb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x80,0x7c,0x80,0x7b,0x7b] + ccmpleb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmplew {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0e,0x83,0x7c,0x80,0x7b,0x7b] + ccmplew {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmplew {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0e,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmplew {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmplel {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x83,0x7c,0x80,0x7b,0x7b] + ccmplel {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmplel {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmplel {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpleq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0e,0x83,0x7c,0x80,0x7b,0x7b] + ccmpleq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpleq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0e,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpleq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpleb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x38,0x5c,0x80,0x7b] + ccmpleb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmplew {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0e,0x39,0x54,0x80,0x7b] + ccmplew {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmplel {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x39,0x4c,0x80,0x7b] + ccmplel {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpleq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x0e,0x39,0x4c,0x80,0x7b] + ccmpleq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpleb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x3a,0x5c,0x80,0x7b] + ccmpleb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmplew {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0e,0x3b,0x54,0x80,0x7b] + ccmplew {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmplel {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x3b,0x4c,0x80,0x7b] + ccmplel {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpleq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0e,0x3b,0x4c,0x80,0x7b] + ccmpleq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpleb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0e,0x80,0xfb,0x7b] + ccmpleb {dfv=of} $123, %bl +# CHECK: ccmplew {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0e,0x83,0xfa,0x7b] + ccmplew {dfv=of} $123, %dx +# CHECK: ccmplel {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0e,0x83,0xf9,0x7b] + ccmplel {dfv=of} $123, %ecx +# CHECK: ccmpleq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0e,0x83,0xf9,0x7b] + ccmpleq {dfv=of} $123, %r9 +# CHECK: ccmplew {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0e,0x81,0xfa,0xd2,0x04] + ccmplew {dfv=of} $1234, %dx +# CHECK: ccmplel {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0e,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmplel {dfv=of} $123456, %ecx +# CHECK: ccmpleq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0e,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpleq {dfv=of} $123456, %r9 +# CHECK: ccmpleb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x0e,0x38,0xda] + ccmpleb {dfv=of} %bl, %dl +# CHECK: ccmplew {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x0e,0x39,0xd0] + ccmplew {dfv=of} %dx, %ax +# CHECK: ccmplel {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x0e,0x39,0xca] + ccmplel {dfv=of} %ecx, %edx +# CHECK: ccmpleq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x0e,0x39,0xcf] + ccmpleq {dfv=of} %r9, %r15 +# CHECK: ccmpaeb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x80,0x7c,0x80,0x7b,0x7b] + ccmpaeb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpaew {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x03,0x83,0x7c,0x80,0x7b,0x7b] + ccmpaew {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpaew {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x03,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpaew {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpael {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x83,0x7c,0x80,0x7b,0x7b] + ccmpael {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpael {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpael {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpaeq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x03,0x83,0x7c,0x80,0x7b,0x7b] + ccmpaeq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpaeq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x03,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpaeq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpaeb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x38,0x5c,0x80,0x7b] + ccmpaeb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpaew {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x03,0x39,0x54,0x80,0x7b] + ccmpaew {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpael {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x39,0x4c,0x80,0x7b] + ccmpael {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpaeq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x03,0x39,0x4c,0x80,0x7b] + ccmpaeq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpaeb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x3a,0x5c,0x80,0x7b] + ccmpaeb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpaew {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x03,0x3b,0x54,0x80,0x7b] + ccmpaew {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpael {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x3b,0x4c,0x80,0x7b] + ccmpael {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpaeq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x03,0x3b,0x4c,0x80,0x7b] + ccmpaeq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpaeb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x03,0x80,0xfb,0x7b] + ccmpaeb {dfv=of} $123, %bl +# CHECK: ccmpaew {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x03,0x83,0xfa,0x7b] + ccmpaew {dfv=of} $123, %dx +# CHECK: ccmpael {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x03,0x83,0xf9,0x7b] + ccmpael {dfv=of} $123, %ecx +# CHECK: ccmpaeq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x03,0x83,0xf9,0x7b] + ccmpaeq {dfv=of} $123, %r9 +# CHECK: ccmpaew {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x03,0x81,0xfa,0xd2,0x04] + ccmpaew {dfv=of} $1234, %dx +# CHECK: ccmpael {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x03,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpael {dfv=of} $123456, %ecx +# CHECK: ccmpaeq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x03,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpaeq {dfv=of} $123456, %r9 +# CHECK: ccmpaeb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x03,0x38,0xda] + ccmpaeb {dfv=of} %bl, %dl +# CHECK: ccmpaew {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x03,0x39,0xd0] + ccmpaew {dfv=of} %dx, %ax +# CHECK: ccmpael {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x03,0x39,0xca] + ccmpael {dfv=of} %ecx, %edx +# CHECK: ccmpaeq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x03,0x39,0xcf] + ccmpaeq {dfv=of} %r9, %r15 +# CHECK: ccmpab {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x80,0x7c,0x80,0x7b,0x7b] + ccmpab {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpaw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x07,0x83,0x7c,0x80,0x7b,0x7b] + ccmpaw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpaw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x07,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpaw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpal {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x83,0x7c,0x80,0x7b,0x7b] + ccmpal {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpal {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpal {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpaq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x07,0x83,0x7c,0x80,0x7b,0x7b] + ccmpaq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpaq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x07,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpaq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpab {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x38,0x5c,0x80,0x7b] + ccmpab {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpaw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x07,0x39,0x54,0x80,0x7b] + ccmpaw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpal {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x39,0x4c,0x80,0x7b] + ccmpal {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpaq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x07,0x39,0x4c,0x80,0x7b] + ccmpaq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpab {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x3a,0x5c,0x80,0x7b] + ccmpab {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpaw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x07,0x3b,0x54,0x80,0x7b] + ccmpaw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpal {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x3b,0x4c,0x80,0x7b] + ccmpal {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpaq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x07,0x3b,0x4c,0x80,0x7b] + ccmpaq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpab {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x07,0x80,0xfb,0x7b] + ccmpab {dfv=of} $123, %bl +# CHECK: ccmpaw {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x07,0x83,0xfa,0x7b] + ccmpaw {dfv=of} $123, %dx +# CHECK: ccmpal {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x07,0x83,0xf9,0x7b] + ccmpal {dfv=of} $123, %ecx +# CHECK: ccmpaq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x07,0x83,0xf9,0x7b] + ccmpaq {dfv=of} $123, %r9 +# CHECK: ccmpaw {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x07,0x81,0xfa,0xd2,0x04] + ccmpaw {dfv=of} $1234, %dx +# CHECK: ccmpal {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x07,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpal {dfv=of} $123456, %ecx +# CHECK: ccmpaq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x07,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpaq {dfv=of} $123456, %r9 +# CHECK: ccmpab {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x07,0x38,0xda] + ccmpab {dfv=of} %bl, %dl +# CHECK: ccmpaw {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x07,0x39,0xd0] + ccmpaw {dfv=of} %dx, %ax +# CHECK: ccmpal {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x07,0x39,0xca] + ccmpal {dfv=of} %ecx, %edx +# CHECK: ccmpaq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x07,0x39,0xcf] + ccmpaq {dfv=of} %r9, %r15 +# CHECK: ccmpgeb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x80,0x7c,0x80,0x7b,0x7b] + ccmpgeb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpgew {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0d,0x83,0x7c,0x80,0x7b,0x7b] + ccmpgew {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpgew {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0d,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpgew {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpgel {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x83,0x7c,0x80,0x7b,0x7b] + ccmpgel {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpgel {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpgel {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpgeq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0d,0x83,0x7c,0x80,0x7b,0x7b] + ccmpgeq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpgeq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0d,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpgeq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpgeb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x38,0x5c,0x80,0x7b] + ccmpgeb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpgew {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0d,0x39,0x54,0x80,0x7b] + ccmpgew {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpgel {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x39,0x4c,0x80,0x7b] + ccmpgel {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpgeq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x0d,0x39,0x4c,0x80,0x7b] + ccmpgeq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpgeb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x3a,0x5c,0x80,0x7b] + ccmpgeb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpgew {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0d,0x3b,0x54,0x80,0x7b] + ccmpgew {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpgel {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x3b,0x4c,0x80,0x7b] + ccmpgel {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpgeq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0d,0x3b,0x4c,0x80,0x7b] + ccmpgeq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpgeb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0d,0x80,0xfb,0x7b] + ccmpgeb {dfv=of} $123, %bl +# CHECK: ccmpgew {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0d,0x83,0xfa,0x7b] + ccmpgew {dfv=of} $123, %dx +# CHECK: ccmpgel {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0d,0x83,0xf9,0x7b] + ccmpgel {dfv=of} $123, %ecx +# CHECK: ccmpgeq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0d,0x83,0xf9,0x7b] + ccmpgeq {dfv=of} $123, %r9 +# CHECK: ccmpgew {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0d,0x81,0xfa,0xd2,0x04] + ccmpgew {dfv=of} $1234, %dx +# CHECK: ccmpgel {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0d,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpgel {dfv=of} $123456, %ecx +# CHECK: ccmpgeq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0d,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpgeq {dfv=of} $123456, %r9 +# CHECK: ccmpgeb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x0d,0x38,0xda] + ccmpgeb {dfv=of} %bl, %dl +# CHECK: ccmpgew {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x0d,0x39,0xd0] + ccmpgew {dfv=of} %dx, %ax +# CHECK: ccmpgel {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x0d,0x39,0xca] + ccmpgel {dfv=of} %ecx, %edx +# CHECK: ccmpgeq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x0d,0x39,0xcf] + ccmpgeq {dfv=of} %r9, %r15 +# CHECK: ccmpgb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x80,0x7c,0x80,0x7b,0x7b] + ccmpgb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpgw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0f,0x83,0x7c,0x80,0x7b,0x7b] + ccmpgw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpgw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0f,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpgw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpgl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x83,0x7c,0x80,0x7b,0x7b] + ccmpgl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpgl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpgl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpgq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0f,0x83,0x7c,0x80,0x7b,0x7b] + ccmpgq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpgq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0f,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpgq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpgb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x38,0x5c,0x80,0x7b] + ccmpgb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpgw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0f,0x39,0x54,0x80,0x7b] + ccmpgw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpgl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x39,0x4c,0x80,0x7b] + ccmpgl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpgq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x0f,0x39,0x4c,0x80,0x7b] + ccmpgq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpgb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x3a,0x5c,0x80,0x7b] + ccmpgb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpgw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0f,0x3b,0x54,0x80,0x7b] + ccmpgw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpgl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x3b,0x4c,0x80,0x7b] + ccmpgl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpgq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0f,0x3b,0x4c,0x80,0x7b] + ccmpgq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpgb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0f,0x80,0xfb,0x7b] + ccmpgb {dfv=of} $123, %bl +# CHECK: ccmpgw {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0f,0x83,0xfa,0x7b] + ccmpgw {dfv=of} $123, %dx +# CHECK: ccmpgl {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0f,0x83,0xf9,0x7b] + ccmpgl {dfv=of} $123, %ecx +# CHECK: ccmpgq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0f,0x83,0xf9,0x7b] + ccmpgq {dfv=of} $123, %r9 +# CHECK: ccmpgw {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0f,0x81,0xfa,0xd2,0x04] + ccmpgw {dfv=of} $1234, %dx +# CHECK: ccmpgl {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0f,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpgl {dfv=of} $123456, %ecx +# CHECK: ccmpgq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0f,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpgq {dfv=of} $123456, %r9 +# CHECK: ccmpgb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x0f,0x38,0xda] + ccmpgb {dfv=of} %bl, %dl +# CHECK: ccmpgw {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x0f,0x39,0xd0] + ccmpgw {dfv=of} %dx, %ax +# CHECK: ccmpgl {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x0f,0x39,0xca] + ccmpgl {dfv=of} %ecx, %edx +# CHECK: ccmpgq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x0f,0x39,0xcf] + ccmpgq {dfv=of} %r9, %r15 +# CHECK: ccmpnob {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x80,0x7c,0x80,0x7b,0x7b] + ccmpnob {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpnow {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x01,0x83,0x7c,0x80,0x7b,0x7b] + ccmpnow {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpnow {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x01,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpnow {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpnol {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x83,0x7c,0x80,0x7b,0x7b] + ccmpnol {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpnol {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpnol {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpnoq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x01,0x83,0x7c,0x80,0x7b,0x7b] + ccmpnoq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpnoq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x01,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpnoq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpnob {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x38,0x5c,0x80,0x7b] + ccmpnob {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpnow {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x01,0x39,0x54,0x80,0x7b] + ccmpnow {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpnol {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x39,0x4c,0x80,0x7b] + ccmpnol {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpnoq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x01,0x39,0x4c,0x80,0x7b] + ccmpnoq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpnob {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x3a,0x5c,0x80,0x7b] + ccmpnob {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpnow {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x01,0x3b,0x54,0x80,0x7b] + ccmpnow {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpnol {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x3b,0x4c,0x80,0x7b] + ccmpnol {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpnoq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x01,0x3b,0x4c,0x80,0x7b] + ccmpnoq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpnob {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x01,0x80,0xfb,0x7b] + ccmpnob {dfv=of} $123, %bl +# CHECK: ccmpnow {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x01,0x83,0xfa,0x7b] + ccmpnow {dfv=of} $123, %dx +# CHECK: ccmpnol {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x01,0x83,0xf9,0x7b] + ccmpnol {dfv=of} $123, %ecx +# CHECK: ccmpnoq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x01,0x83,0xf9,0x7b] + ccmpnoq {dfv=of} $123, %r9 +# CHECK: ccmpnow {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x01,0x81,0xfa,0xd2,0x04] + ccmpnow {dfv=of} $1234, %dx +# CHECK: ccmpnol {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x01,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpnol {dfv=of} $123456, %ecx +# CHECK: ccmpnoq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x01,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpnoq {dfv=of} $123456, %r9 +# CHECK: ccmpnob {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x01,0x38,0xda] + ccmpnob {dfv=of} %bl, %dl +# CHECK: ccmpnow {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x01,0x39,0xd0] + ccmpnow {dfv=of} %dx, %ax +# CHECK: ccmpnol {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x01,0x39,0xca] + ccmpnol {dfv=of} %ecx, %edx +# CHECK: ccmpnoq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x01,0x39,0xcf] + ccmpnoq {dfv=of} %r9, %r15 +# CHECK: ccmpnsb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x80,0x7c,0x80,0x7b,0x7b] + ccmpnsb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpnsw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x09,0x83,0x7c,0x80,0x7b,0x7b] + ccmpnsw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpnsw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x09,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpnsw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpnsl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x83,0x7c,0x80,0x7b,0x7b] + ccmpnsl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpnsl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpnsl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpnsq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x09,0x83,0x7c,0x80,0x7b,0x7b] + ccmpnsq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpnsq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x09,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpnsq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpnsb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x38,0x5c,0x80,0x7b] + ccmpnsb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpnsw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x09,0x39,0x54,0x80,0x7b] + ccmpnsw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpnsl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x39,0x4c,0x80,0x7b] + ccmpnsl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpnsq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x09,0x39,0x4c,0x80,0x7b] + ccmpnsq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpnsb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x3a,0x5c,0x80,0x7b] + ccmpnsb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpnsw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x09,0x3b,0x54,0x80,0x7b] + ccmpnsw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpnsl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x3b,0x4c,0x80,0x7b] + ccmpnsl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpnsq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x09,0x3b,0x4c,0x80,0x7b] + ccmpnsq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpnsb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x09,0x80,0xfb,0x7b] + ccmpnsb {dfv=of} $123, %bl +# CHECK: ccmpnsw {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x09,0x83,0xfa,0x7b] + ccmpnsw {dfv=of} $123, %dx +# CHECK: ccmpnsl {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x09,0x83,0xf9,0x7b] + ccmpnsl {dfv=of} $123, %ecx +# CHECK: ccmpnsq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x09,0x83,0xf9,0x7b] + ccmpnsq {dfv=of} $123, %r9 +# CHECK: ccmpnsw {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x09,0x81,0xfa,0xd2,0x04] + ccmpnsw {dfv=of} $1234, %dx +# CHECK: ccmpnsl {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x09,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpnsl {dfv=of} $123456, %ecx +# CHECK: ccmpnsq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x09,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpnsq {dfv=of} $123456, %r9 +# CHECK: ccmpnsb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x09,0x38,0xda] + ccmpnsb {dfv=of} %bl, %dl +# CHECK: ccmpnsw {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x09,0x39,0xd0] + ccmpnsw {dfv=of} %dx, %ax +# CHECK: ccmpnsl {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x09,0x39,0xca] + ccmpnsl {dfv=of} %ecx, %edx +# CHECK: ccmpnsq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x09,0x39,0xcf] + ccmpnsq {dfv=of} %r9, %r15 +# CHECK: ccmpneb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x80,0x7c,0x80,0x7b,0x7b] + ccmpneb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpnew {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x05,0x83,0x7c,0x80,0x7b,0x7b] + ccmpnew {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpnew {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x05,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpnew {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpnel {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x83,0x7c,0x80,0x7b,0x7b] + ccmpnel {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpnel {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpnel {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpneq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x05,0x83,0x7c,0x80,0x7b,0x7b] + ccmpneq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpneq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x05,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpneq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpneb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x38,0x5c,0x80,0x7b] + ccmpneb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpnew {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x05,0x39,0x54,0x80,0x7b] + ccmpnew {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpnel {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x39,0x4c,0x80,0x7b] + ccmpnel {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpneq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x05,0x39,0x4c,0x80,0x7b] + ccmpneq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpneb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x3a,0x5c,0x80,0x7b] + ccmpneb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpnew {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x05,0x3b,0x54,0x80,0x7b] + ccmpnew {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpnel {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x3b,0x4c,0x80,0x7b] + ccmpnel {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpneq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x05,0x3b,0x4c,0x80,0x7b] + ccmpneq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpneb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x05,0x80,0xfb,0x7b] + ccmpneb {dfv=of} $123, %bl +# CHECK: ccmpnew {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x05,0x83,0xfa,0x7b] + ccmpnew {dfv=of} $123, %dx +# CHECK: ccmpnel {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x05,0x83,0xf9,0x7b] + ccmpnel {dfv=of} $123, %ecx +# CHECK: ccmpneq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x05,0x83,0xf9,0x7b] + ccmpneq {dfv=of} $123, %r9 +# CHECK: ccmpnew {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x05,0x81,0xfa,0xd2,0x04] + ccmpnew {dfv=of} $1234, %dx +# CHECK: ccmpnel {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x05,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpnel {dfv=of} $123456, %ecx +# CHECK: ccmpneq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x05,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpneq {dfv=of} $123456, %r9 +# CHECK: ccmpneb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x05,0x38,0xda] + ccmpneb {dfv=of} %bl, %dl +# CHECK: ccmpnew {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x05,0x39,0xd0] + ccmpnew {dfv=of} %dx, %ax +# CHECK: ccmpnel {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x05,0x39,0xca] + ccmpnel {dfv=of} %ecx, %edx +# CHECK: ccmpneq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x05,0x39,0xcf] + ccmpneq {dfv=of} %r9, %r15 +# CHECK: ccmpob {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x80,0x7c,0x80,0x7b,0x7b] + ccmpob {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpow {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x00,0x83,0x7c,0x80,0x7b,0x7b] + ccmpow {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpow {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x00,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpow {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpol {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x83,0x7c,0x80,0x7b,0x7b] + ccmpol {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpol {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpol {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpoq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x00,0x83,0x7c,0x80,0x7b,0x7b] + ccmpoq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpoq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x00,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpoq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpob {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x38,0x5c,0x80,0x7b] + ccmpob {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpow {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x00,0x39,0x54,0x80,0x7b] + ccmpow {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpol {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x39,0x4c,0x80,0x7b] + ccmpol {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpoq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x00,0x39,0x4c,0x80,0x7b] + ccmpoq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpob {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x3a,0x5c,0x80,0x7b] + ccmpob {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpow {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x00,0x3b,0x54,0x80,0x7b] + ccmpow {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpol {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x3b,0x4c,0x80,0x7b] + ccmpol {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpoq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x00,0x3b,0x4c,0x80,0x7b] + ccmpoq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpob {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x00,0x80,0xfb,0x7b] + ccmpob {dfv=of} $123, %bl +# CHECK: ccmpow {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x00,0x83,0xfa,0x7b] + ccmpow {dfv=of} $123, %dx +# CHECK: ccmpol {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x00,0x83,0xf9,0x7b] + ccmpol {dfv=of} $123, %ecx +# CHECK: ccmpoq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x00,0x83,0xf9,0x7b] + ccmpoq {dfv=of} $123, %r9 +# CHECK: ccmpow {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x00,0x81,0xfa,0xd2,0x04] + ccmpow {dfv=of} $1234, %dx +# CHECK: ccmpol {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x00,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpol {dfv=of} $123456, %ecx +# CHECK: ccmpoq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x00,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpoq {dfv=of} $123456, %r9 +# CHECK: ccmpob {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x00,0x38,0xda] + ccmpob {dfv=of} %bl, %dl +# CHECK: ccmpow {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x00,0x39,0xd0] + ccmpow {dfv=of} %dx, %ax +# CHECK: ccmpol {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x00,0x39,0xca] + ccmpol {dfv=of} %ecx, %edx +# CHECK: ccmpoq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x00,0x39,0xcf] + ccmpoq {dfv=of} %r9, %r15 +# CHECK: ccmpsb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x80,0x7c,0x80,0x7b,0x7b] + ccmpsb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpsw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x08,0x83,0x7c,0x80,0x7b,0x7b] + ccmpsw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpsw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x08,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpsw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpsl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x83,0x7c,0x80,0x7b,0x7b] + ccmpsl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpsl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpsl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpsq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x08,0x83,0x7c,0x80,0x7b,0x7b] + ccmpsq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpsq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x08,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpsq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpsb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x38,0x5c,0x80,0x7b] + ccmpsb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpsw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x08,0x39,0x54,0x80,0x7b] + ccmpsw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpsl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x39,0x4c,0x80,0x7b] + ccmpsl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpsq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x08,0x39,0x4c,0x80,0x7b] + ccmpsq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpsb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x3a,0x5c,0x80,0x7b] + ccmpsb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpsw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x08,0x3b,0x54,0x80,0x7b] + ccmpsw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpsl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x3b,0x4c,0x80,0x7b] + ccmpsl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpsq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x08,0x3b,0x4c,0x80,0x7b] + ccmpsq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpsb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x08,0x80,0xfb,0x7b] + ccmpsb {dfv=of} $123, %bl +# CHECK: ccmpsw {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x08,0x83,0xfa,0x7b] + ccmpsw {dfv=of} $123, %dx +# CHECK: ccmpsl {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x08,0x83,0xf9,0x7b] + ccmpsl {dfv=of} $123, %ecx +# CHECK: ccmpsq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x08,0x83,0xf9,0x7b] + ccmpsq {dfv=of} $123, %r9 +# CHECK: ccmpsw {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x08,0x81,0xfa,0xd2,0x04] + ccmpsw {dfv=of} $1234, %dx +# CHECK: ccmpsl {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x08,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpsl {dfv=of} $123456, %ecx +# CHECK: ccmpsq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x08,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpsq {dfv=of} $123456, %r9 +# CHECK: ccmpsb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x08,0x38,0xda] + ccmpsb {dfv=of} %bl, %dl +# CHECK: ccmpsw {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x08,0x39,0xd0] + ccmpsw {dfv=of} %dx, %ax +# CHECK: ccmpsl {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x08,0x39,0xca] + ccmpsl {dfv=of} %ecx, %edx +# CHECK: ccmpsq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x08,0x39,0xcf] + ccmpsq {dfv=of} %r9, %r15 +# CHECK: ccmptb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x80,0x7c,0x80,0x7b,0x7b] + ccmptb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmptw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0a,0x83,0x7c,0x80,0x7b,0x7b] + ccmptw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmptw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0a,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmptw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmptl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x83,0x7c,0x80,0x7b,0x7b] + ccmptl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmptl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmptl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmptq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0a,0x83,0x7c,0x80,0x7b,0x7b] + ccmptq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmptq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0a,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmptq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmptb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x38,0x5c,0x80,0x7b] + ccmptb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmptw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0a,0x39,0x54,0x80,0x7b] + ccmptw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmptl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x39,0x4c,0x80,0x7b] + ccmptl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmptq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x0a,0x39,0x4c,0x80,0x7b] + ccmptq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmptb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x3a,0x5c,0x80,0x7b] + ccmptb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmptw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0a,0x3b,0x54,0x80,0x7b] + ccmptw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmptl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x3b,0x4c,0x80,0x7b] + ccmptl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmptq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0a,0x3b,0x4c,0x80,0x7b] + ccmptq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmptb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0a,0x80,0xfb,0x7b] + ccmptb {dfv=of} $123, %bl +# CHECK: ccmptw {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0a,0x83,0xfa,0x7b] + ccmptw {dfv=of} $123, %dx +# CHECK: ccmptl {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0a,0x83,0xf9,0x7b] + ccmptl {dfv=of} $123, %ecx +# CHECK: ccmptq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0a,0x83,0xf9,0x7b] + ccmptq {dfv=of} $123, %r9 +# CHECK: ccmptw {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0a,0x81,0xfa,0xd2,0x04] + ccmptw {dfv=of} $1234, %dx +# CHECK: ccmptl {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0a,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmptl {dfv=of} $123456, %ecx +# CHECK: ccmptq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0a,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmptq {dfv=of} $123456, %r9 +# CHECK: ccmptb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x0a,0x38,0xda] + ccmptb {dfv=of} %bl, %dl +# CHECK: ccmptw {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x0a,0x39,0xd0] + ccmptw {dfv=of} %dx, %ax +# CHECK: ccmptl {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x0a,0x39,0xca] + ccmptl {dfv=of} %ecx, %edx +# CHECK: ccmptq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x0a,0x39,0xcf] + ccmptq {dfv=of} %r9, %r15 +# CHECK: ccmpeb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x80,0x7c,0x80,0x7b,0x7b] + ccmpeb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpew {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x04,0x83,0x7c,0x80,0x7b,0x7b] + ccmpew {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpew {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x04,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpew {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpel {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x83,0x7c,0x80,0x7b,0x7b] + ccmpel {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpel {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpel {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpeq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x04,0x83,0x7c,0x80,0x7b,0x7b] + ccmpeq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpeq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x04,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpeq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpeb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x38,0x5c,0x80,0x7b] + ccmpeb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpew {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x04,0x39,0x54,0x80,0x7b] + ccmpew {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpel {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x39,0x4c,0x80,0x7b] + ccmpel {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpeq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x04,0x39,0x4c,0x80,0x7b] + ccmpeq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpeb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x3a,0x5c,0x80,0x7b] + ccmpeb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpew {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x04,0x3b,0x54,0x80,0x7b] + ccmpew {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpel {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x3b,0x4c,0x80,0x7b] + ccmpel {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpeq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x04,0x3b,0x4c,0x80,0x7b] + ccmpeq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpeb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x04,0x80,0xfb,0x7b] + ccmpeb {dfv=of} $123, %bl +# CHECK: ccmpew {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x04,0x83,0xfa,0x7b] + ccmpew {dfv=of} $123, %dx +# CHECK: ccmpel {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x04,0x83,0xf9,0x7b] + ccmpel {dfv=of} $123, %ecx +# CHECK: ccmpeq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x04,0x83,0xf9,0x7b] + ccmpeq {dfv=of} $123, %r9 +# CHECK: ccmpew {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x04,0x81,0xfa,0xd2,0x04] + ccmpew {dfv=of} $1234, %dx +# CHECK: ccmpel {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x04,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpel {dfv=of} $123456, %ecx +# CHECK: ccmpeq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x04,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpeq {dfv=of} $123456, %r9 +# CHECK: ccmpeb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x04,0x38,0xda] + ccmpeb {dfv=of} %bl, %dl +# CHECK: ccmpew {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x04,0x39,0xd0] + ccmpew {dfv=of} %dx, %ax +# CHECK: ccmpel {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x04,0x39,0xca] + ccmpel {dfv=of} %ecx, %edx +# CHECK: ccmpeq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x04,0x39,0xcf] + ccmpeq {dfv=of} %r9, %r15 diff --git a/llvm/test/MC/X86/apx/ccmp-intel-error.s b/llvm/test/MC/X86/apx/ccmp-intel-error.s new file mode 100644 index 000000000000..e37cc53548ae --- /dev/null +++ b/llvm/test/MC/X86/apx/ccmp-intel-error.s @@ -0,0 +1,29 @@ +# RUN: not llvm-mc -triple x86_64 -show-encoding -x86-asm-syntax=intel -output-asm-variant=1 %s 2>&1 | FileCheck --strict-whitespace %s + +# CHECK: [[#@LINE+2]]:7: error: Expected { at this point +# CHECK: ccmpe 1 rbx, rax +ccmpe 1 rbx, rax + +# CHECK: [[#@LINE+2]]:8: error: Expected dfv at this point +# CHECK: ccmpe {sf} rbx, rax +ccmpe {sf} rbx, rax + +# CHECK: [[#@LINE+2]]:11: error: Expected = at this point +# CHECK: ccmpe {dfv:sf} rbx, rax +ccmpe {dfv:sf} rbx, rax + +# CHECK: [[#@LINE+2]]:18: error: Expected } or , at this point +# CHECK: ccmpe {dfv=sf,cf rbx, rax +ccmpe {dfv=sf,cf rbx, rax + +# CHECK: [[#@LINE+2]]:12: error: Invalid conditional flags +# CHECK: ccmpe {dfv=pf} rbx, rax +ccmpe {dfv=pf} rbx, rax + +# CHECK: [[#@LINE+2]]:19: error: Duplicated conditional flag +# CHECK: ccmpeq {dfv=of,zf,of} rbx, rax +ccmpeq {dfv=of,zf,of} rbx, rax + +# CHECK: [[#@LINE+2]]:24: error: Expected } at this point +# CHECK: ccmpeq {dfv=of,sf,zf,cf,of} rbx, rax +ccmpeq {dfv=of,sf,zf,cf,of} rbx, rax diff --git a/llvm/test/MC/X86/apx/ccmp-intel.s b/llvm/test/MC/X86/apx/ccmp-intel.s new file mode 100644 index 000000000000..095f7fb71b7c --- /dev/null +++ b/llvm/test/MC/X86/apx/ccmp-intel.s @@ -0,0 +1,1210 @@ +# RUN: llvm-mc -triple x86_64 -show-encoding -x86-asm-syntax=intel -output-asm-variant=1 %s | FileCheck %s + +## Condition flags + +# CHECK: ccmpo {dfv=} rbx, rax +# CHECK: encoding: [0x62,0xf4,0x84,0x00,0x39,0xc3] + ccmpo {dfv=} rbx, rax +# CHECK: ccmpo {dfv=of} rbx, rax +# CHECK: encoding: [0x62,0xf4,0xc4,0x00,0x39,0xc3] + ccmpo {dfv=of} rbx, rax +# CHECK: ccmpo {dfv=sf} rbx, rax +# CHECK: encoding: [0x62,0xf4,0xa4,0x00,0x39,0xc3] + ccmpo {dfv=sf} rbx, rax +# CHECK: ccmpo {dfv=zf} rbx, rax +# CHECK: encoding: [0x62,0xf4,0x94,0x00,0x39,0xc3] + ccmpo {dfv=zf} rbx, rax +# CHECK: ccmpo {dfv=cf} rbx, rax +# CHECK: encoding: [0x62,0xf4,0x8c,0x00,0x39,0xc3] + ccmpo {dfv=cf} rbx, rax +# CHECK: ccmpo {dfv=of,sf} rbx, rax +# CHECK: encoding: [0x62,0xf4,0xe4,0x00,0x39,0xc3] + ccmpo {dfv=of,sf} rbx, rax +# CHECK: ccmpo {dfv=of,sf} rbx, rax +# CHECK: encoding: [0x62,0xf4,0xe4,0x00,0x39,0xc3] + ccmpo {dfv=sf,of} rbx, rax +# CHECK: ccmpo {dfv=of,sf,zf} rbx, rax +# CHECK: encoding: [0x62,0xf4,0xf4,0x00,0x39,0xc3] + ccmpo {dfv=of,sf,zf} rbx, rax +# CHECK: ccmpo {dfv=of,sf,zf} rbx, rax +# CHECK: encoding: [0x62,0xf4,0xf4,0x00,0x39,0xc3] + ccmpo {dfv=zf,of,sf} rbx, rax +# CHECK: ccmpo {dfv=of,sf,zf,cf} rbx, rax +# CHECK: encoding: [0x62,0xf4,0xfc,0x00,0x39,0xc3] + ccmpo {dfv=of,sf,zf,cf} rbx, rax +# CHECK: ccmpo {dfv=of,sf,zf,cf} rbx, rax +# CHECK: encoding: [0x62,0xf4,0xfc,0x00,0x39,0xc3] + ccmpo {dfv=cf,zf,sf,of} rbx, rax + +## Condition code + +# CHECK: ccmpb {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x80,0x7c,0x80,0x7b,0x7b] + ccmpb {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpb {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x02,0x83,0x7c,0x80,0x7b,0x7b] + ccmpb {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpb {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x02,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpb {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpb {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x83,0x7c,0x80,0x7b,0x7b] + ccmpb {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpb {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpb {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpb {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x02,0x83,0x7c,0x80,0x7b,0x7b] + ccmpb {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpb {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x02,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpb {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpb {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x38,0x5c,0x80,0x7b] + ccmpb {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpb {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x02,0x39,0x54,0x80,0x7b] + ccmpb {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpb {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x39,0x4c,0x80,0x7b] + ccmpb {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpb {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x02,0x39,0x4c,0x80,0x7b] + ccmpb {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpb {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x3a,0x5c,0x80,0x7b] + ccmpb {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpb {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x02,0x3b,0x54,0x80,0x7b] + ccmpb {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpb {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x3b,0x4c,0x80,0x7b] + ccmpb {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpb {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x02,0x3b,0x4c,0x80,0x7b] + ccmpb {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpb {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x02,0x80,0xfb,0x7b] + ccmpb {dfv=of} bl, 123 +# CHECK: ccmpb {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x02,0x83,0xfa,0x7b] + ccmpb {dfv=of} dx, 123 +# CHECK: ccmpb {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x02,0x83,0xf9,0x7b] + ccmpb {dfv=of} ecx, 123 +# CHECK: ccmpb {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x02,0x83,0xf9,0x7b] + ccmpb {dfv=of} r9, 123 +# CHECK: ccmpb {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x02,0x81,0xfa,0xd2,0x04] + ccmpb {dfv=of} dx, 1234 +# CHECK: ccmpb {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x02,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpb {dfv=of} ecx, 123456 +# CHECK: ccmpb {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x02,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpb {dfv=of} r9, 123456 +# CHECK: ccmpb {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x02,0x38,0xda] + ccmpb {dfv=of} dl, bl +# CHECK: ccmpb {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x02,0x39,0xd0] + ccmpb {dfv=of} ax, dx +# CHECK: ccmpb {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x02,0x39,0xca] + ccmpb {dfv=of} edx, ecx +# CHECK: ccmpb {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x02,0x39,0xcf] + ccmpb {dfv=of} r15, r9 +# CHECK: ccmpf {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x80,0x7c,0x80,0x7b,0x7b] + ccmpf {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpf {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x0b,0x83,0x7c,0x80,0x7b,0x7b] + ccmpf {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpf {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x0b,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpf {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpf {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x83,0x7c,0x80,0x7b,0x7b] + ccmpf {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpf {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpf {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpf {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0b,0x83,0x7c,0x80,0x7b,0x7b] + ccmpf {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpf {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0b,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpf {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpf {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x38,0x5c,0x80,0x7b] + ccmpf {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpf {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0b,0x39,0x54,0x80,0x7b] + ccmpf {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpf {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x39,0x4c,0x80,0x7b] + ccmpf {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpf {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0b,0x39,0x4c,0x80,0x7b] + ccmpf {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpf {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x3a,0x5c,0x80,0x7b] + ccmpf {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpf {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x0b,0x3b,0x54,0x80,0x7b] + ccmpf {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpf {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x3b,0x4c,0x80,0x7b] + ccmpf {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpf {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x0b,0x3b,0x4c,0x80,0x7b] + ccmpf {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpf {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0b,0x80,0xfb,0x7b] + ccmpf {dfv=of} bl, 123 +# CHECK: ccmpf {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x0b,0x83,0xfa,0x7b] + ccmpf {dfv=of} dx, 123 +# CHECK: ccmpf {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0b,0x83,0xf9,0x7b] + ccmpf {dfv=of} ecx, 123 +# CHECK: ccmpf {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0b,0x83,0xf9,0x7b] + ccmpf {dfv=of} r9, 123 +# CHECK: ccmpf {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x0b,0x81,0xfa,0xd2,0x04] + ccmpf {dfv=of} dx, 1234 +# CHECK: ccmpf {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x0b,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpf {dfv=of} ecx, 123456 +# CHECK: ccmpf {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0b,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpf {dfv=of} r9, 123456 +# CHECK: ccmpf {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0b,0x38,0xda] + ccmpf {dfv=of} dl, bl +# CHECK: ccmpf {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0b,0x39,0xd0] + ccmpf {dfv=of} ax, dx +# CHECK: ccmpf {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0b,0x39,0xca] + ccmpf {dfv=of} edx, ecx +# CHECK: ccmpf {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0b,0x39,0xcf] + ccmpf {dfv=of} r15, r9 +# CHECK: ccmpl {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x80,0x7c,0x80,0x7b,0x7b] + ccmpl {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpl {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x0c,0x83,0x7c,0x80,0x7b,0x7b] + ccmpl {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpl {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x0c,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpl {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpl {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x83,0x7c,0x80,0x7b,0x7b] + ccmpl {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpl {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpl {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpl {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0c,0x83,0x7c,0x80,0x7b,0x7b] + ccmpl {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpl {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0c,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpl {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpl {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x38,0x5c,0x80,0x7b] + ccmpl {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpl {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0c,0x39,0x54,0x80,0x7b] + ccmpl {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpl {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x39,0x4c,0x80,0x7b] + ccmpl {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpl {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0c,0x39,0x4c,0x80,0x7b] + ccmpl {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpl {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x3a,0x5c,0x80,0x7b] + ccmpl {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpl {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x0c,0x3b,0x54,0x80,0x7b] + ccmpl {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpl {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x3b,0x4c,0x80,0x7b] + ccmpl {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpl {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x0c,0x3b,0x4c,0x80,0x7b] + ccmpl {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpl {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0c,0x80,0xfb,0x7b] + ccmpl {dfv=of} bl, 123 +# CHECK: ccmpl {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x0c,0x83,0xfa,0x7b] + ccmpl {dfv=of} dx, 123 +# CHECK: ccmpl {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0c,0x83,0xf9,0x7b] + ccmpl {dfv=of} ecx, 123 +# CHECK: ccmpl {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0c,0x83,0xf9,0x7b] + ccmpl {dfv=of} r9, 123 +# CHECK: ccmpl {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x0c,0x81,0xfa,0xd2,0x04] + ccmpl {dfv=of} dx, 1234 +# CHECK: ccmpl {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x0c,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpl {dfv=of} ecx, 123456 +# CHECK: ccmpl {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0c,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpl {dfv=of} r9, 123456 +# CHECK: ccmpl {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0c,0x38,0xda] + ccmpl {dfv=of} dl, bl +# CHECK: ccmpl {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0c,0x39,0xd0] + ccmpl {dfv=of} ax, dx +# CHECK: ccmpl {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0c,0x39,0xca] + ccmpl {dfv=of} edx, ecx +# CHECK: ccmpl {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0c,0x39,0xcf] + ccmpl {dfv=of} r15, r9 +# CHECK: ccmple {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x80,0x7c,0x80,0x7b,0x7b] + ccmple {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmple {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x0e,0x83,0x7c,0x80,0x7b,0x7b] + ccmple {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmple {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x0e,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmple {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmple {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x83,0x7c,0x80,0x7b,0x7b] + ccmple {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmple {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmple {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmple {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0e,0x83,0x7c,0x80,0x7b,0x7b] + ccmple {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmple {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0e,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmple {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmple {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x38,0x5c,0x80,0x7b] + ccmple {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmple {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0e,0x39,0x54,0x80,0x7b] + ccmple {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmple {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x39,0x4c,0x80,0x7b] + ccmple {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmple {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0e,0x39,0x4c,0x80,0x7b] + ccmple {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmple {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x3a,0x5c,0x80,0x7b] + ccmple {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmple {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x0e,0x3b,0x54,0x80,0x7b] + ccmple {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmple {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x3b,0x4c,0x80,0x7b] + ccmple {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmple {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x0e,0x3b,0x4c,0x80,0x7b] + ccmple {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmple {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0e,0x80,0xfb,0x7b] + ccmple {dfv=of} bl, 123 +# CHECK: ccmple {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x0e,0x83,0xfa,0x7b] + ccmple {dfv=of} dx, 123 +# CHECK: ccmple {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0e,0x83,0xf9,0x7b] + ccmple {dfv=of} ecx, 123 +# CHECK: ccmple {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0e,0x83,0xf9,0x7b] + ccmple {dfv=of} r9, 123 +# CHECK: ccmple {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x0e,0x81,0xfa,0xd2,0x04] + ccmple {dfv=of} dx, 1234 +# CHECK: ccmple {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x0e,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmple {dfv=of} ecx, 123456 +# CHECK: ccmple {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0e,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmple {dfv=of} r9, 123456 +# CHECK: ccmple {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0e,0x38,0xda] + ccmple {dfv=of} dl, bl +# CHECK: ccmple {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0e,0x39,0xd0] + ccmple {dfv=of} ax, dx +# CHECK: ccmple {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0e,0x39,0xca] + ccmple {dfv=of} edx, ecx +# CHECK: ccmple {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0e,0x39,0xcf] + ccmple {dfv=of} r15, r9 +# CHECK: ccmpae {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x80,0x7c,0x80,0x7b,0x7b] + ccmpae {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpae {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x03,0x83,0x7c,0x80,0x7b,0x7b] + ccmpae {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpae {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x03,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpae {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpae {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x83,0x7c,0x80,0x7b,0x7b] + ccmpae {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpae {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpae {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpae {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x03,0x83,0x7c,0x80,0x7b,0x7b] + ccmpae {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpae {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x03,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpae {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpae {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x38,0x5c,0x80,0x7b] + ccmpae {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpae {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x03,0x39,0x54,0x80,0x7b] + ccmpae {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpae {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x39,0x4c,0x80,0x7b] + ccmpae {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpae {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x03,0x39,0x4c,0x80,0x7b] + ccmpae {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpae {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x3a,0x5c,0x80,0x7b] + ccmpae {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpae {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x03,0x3b,0x54,0x80,0x7b] + ccmpae {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpae {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x3b,0x4c,0x80,0x7b] + ccmpae {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpae {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x03,0x3b,0x4c,0x80,0x7b] + ccmpae {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpae {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x03,0x80,0xfb,0x7b] + ccmpae {dfv=of} bl, 123 +# CHECK: ccmpae {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x03,0x83,0xfa,0x7b] + ccmpae {dfv=of} dx, 123 +# CHECK: ccmpae {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x03,0x83,0xf9,0x7b] + ccmpae {dfv=of} ecx, 123 +# CHECK: ccmpae {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x03,0x83,0xf9,0x7b] + ccmpae {dfv=of} r9, 123 +# CHECK: ccmpae {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x03,0x81,0xfa,0xd2,0x04] + ccmpae {dfv=of} dx, 1234 +# CHECK: ccmpae {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x03,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpae {dfv=of} ecx, 123456 +# CHECK: ccmpae {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x03,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpae {dfv=of} r9, 123456 +# CHECK: ccmpae {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x03,0x38,0xda] + ccmpae {dfv=of} dl, bl +# CHECK: ccmpae {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x03,0x39,0xd0] + ccmpae {dfv=of} ax, dx +# CHECK: ccmpae {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x03,0x39,0xca] + ccmpae {dfv=of} edx, ecx +# CHECK: ccmpae {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x03,0x39,0xcf] + ccmpae {dfv=of} r15, r9 +# CHECK: ccmpa {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x80,0x7c,0x80,0x7b,0x7b] + ccmpa {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpa {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x07,0x83,0x7c,0x80,0x7b,0x7b] + ccmpa {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpa {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x07,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpa {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpa {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x83,0x7c,0x80,0x7b,0x7b] + ccmpa {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpa {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpa {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpa {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x07,0x83,0x7c,0x80,0x7b,0x7b] + ccmpa {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpa {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x07,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpa {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpa {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x38,0x5c,0x80,0x7b] + ccmpa {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpa {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x07,0x39,0x54,0x80,0x7b] + ccmpa {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpa {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x39,0x4c,0x80,0x7b] + ccmpa {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpa {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x07,0x39,0x4c,0x80,0x7b] + ccmpa {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpa {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x3a,0x5c,0x80,0x7b] + ccmpa {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpa {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x07,0x3b,0x54,0x80,0x7b] + ccmpa {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpa {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x3b,0x4c,0x80,0x7b] + ccmpa {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpa {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x07,0x3b,0x4c,0x80,0x7b] + ccmpa {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpa {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x07,0x80,0xfb,0x7b] + ccmpa {dfv=of} bl, 123 +# CHECK: ccmpa {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x07,0x83,0xfa,0x7b] + ccmpa {dfv=of} dx, 123 +# CHECK: ccmpa {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x07,0x83,0xf9,0x7b] + ccmpa {dfv=of} ecx, 123 +# CHECK: ccmpa {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x07,0x83,0xf9,0x7b] + ccmpa {dfv=of} r9, 123 +# CHECK: ccmpa {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x07,0x81,0xfa,0xd2,0x04] + ccmpa {dfv=of} dx, 1234 +# CHECK: ccmpa {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x07,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpa {dfv=of} ecx, 123456 +# CHECK: ccmpa {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x07,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpa {dfv=of} r9, 123456 +# CHECK: ccmpa {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x07,0x38,0xda] + ccmpa {dfv=of} dl, bl +# CHECK: ccmpa {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x07,0x39,0xd0] + ccmpa {dfv=of} ax, dx +# CHECK: ccmpa {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x07,0x39,0xca] + ccmpa {dfv=of} edx, ecx +# CHECK: ccmpa {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x07,0x39,0xcf] + ccmpa {dfv=of} r15, r9 +# CHECK: ccmpge {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x80,0x7c,0x80,0x7b,0x7b] + ccmpge {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpge {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x0d,0x83,0x7c,0x80,0x7b,0x7b] + ccmpge {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpge {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x0d,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpge {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpge {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x83,0x7c,0x80,0x7b,0x7b] + ccmpge {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpge {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpge {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpge {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0d,0x83,0x7c,0x80,0x7b,0x7b] + ccmpge {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpge {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0d,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpge {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpge {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x38,0x5c,0x80,0x7b] + ccmpge {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpge {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0d,0x39,0x54,0x80,0x7b] + ccmpge {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpge {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x39,0x4c,0x80,0x7b] + ccmpge {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpge {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0d,0x39,0x4c,0x80,0x7b] + ccmpge {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpge {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x3a,0x5c,0x80,0x7b] + ccmpge {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpge {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x0d,0x3b,0x54,0x80,0x7b] + ccmpge {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpge {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x3b,0x4c,0x80,0x7b] + ccmpge {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpge {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x0d,0x3b,0x4c,0x80,0x7b] + ccmpge {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpge {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0d,0x80,0xfb,0x7b] + ccmpge {dfv=of} bl, 123 +# CHECK: ccmpge {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x0d,0x83,0xfa,0x7b] + ccmpge {dfv=of} dx, 123 +# CHECK: ccmpge {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0d,0x83,0xf9,0x7b] + ccmpge {dfv=of} ecx, 123 +# CHECK: ccmpge {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0d,0x83,0xf9,0x7b] + ccmpge {dfv=of} r9, 123 +# CHECK: ccmpge {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x0d,0x81,0xfa,0xd2,0x04] + ccmpge {dfv=of} dx, 1234 +# CHECK: ccmpge {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x0d,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpge {dfv=of} ecx, 123456 +# CHECK: ccmpge {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0d,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpge {dfv=of} r9, 123456 +# CHECK: ccmpge {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0d,0x38,0xda] + ccmpge {dfv=of} dl, bl +# CHECK: ccmpge {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0d,0x39,0xd0] + ccmpge {dfv=of} ax, dx +# CHECK: ccmpge {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0d,0x39,0xca] + ccmpge {dfv=of} edx, ecx +# CHECK: ccmpge {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0d,0x39,0xcf] + ccmpge {dfv=of} r15, r9 +# CHECK: ccmpg {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x80,0x7c,0x80,0x7b,0x7b] + ccmpg {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpg {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x0f,0x83,0x7c,0x80,0x7b,0x7b] + ccmpg {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpg {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x0f,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpg {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpg {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x83,0x7c,0x80,0x7b,0x7b] + ccmpg {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpg {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpg {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpg {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0f,0x83,0x7c,0x80,0x7b,0x7b] + ccmpg {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpg {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0f,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpg {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpg {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x38,0x5c,0x80,0x7b] + ccmpg {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpg {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0f,0x39,0x54,0x80,0x7b] + ccmpg {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpg {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x39,0x4c,0x80,0x7b] + ccmpg {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpg {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0f,0x39,0x4c,0x80,0x7b] + ccmpg {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpg {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x3a,0x5c,0x80,0x7b] + ccmpg {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpg {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x0f,0x3b,0x54,0x80,0x7b] + ccmpg {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpg {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x3b,0x4c,0x80,0x7b] + ccmpg {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpg {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x0f,0x3b,0x4c,0x80,0x7b] + ccmpg {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpg {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0f,0x80,0xfb,0x7b] + ccmpg {dfv=of} bl, 123 +# CHECK: ccmpg {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x0f,0x83,0xfa,0x7b] + ccmpg {dfv=of} dx, 123 +# CHECK: ccmpg {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0f,0x83,0xf9,0x7b] + ccmpg {dfv=of} ecx, 123 +# CHECK: ccmpg {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0f,0x83,0xf9,0x7b] + ccmpg {dfv=of} r9, 123 +# CHECK: ccmpg {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x0f,0x81,0xfa,0xd2,0x04] + ccmpg {dfv=of} dx, 1234 +# CHECK: ccmpg {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x0f,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpg {dfv=of} ecx, 123456 +# CHECK: ccmpg {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0f,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpg {dfv=of} r9, 123456 +# CHECK: ccmpg {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0f,0x38,0xda] + ccmpg {dfv=of} dl, bl +# CHECK: ccmpg {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0f,0x39,0xd0] + ccmpg {dfv=of} ax, dx +# CHECK: ccmpg {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0f,0x39,0xca] + ccmpg {dfv=of} edx, ecx +# CHECK: ccmpg {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0f,0x39,0xcf] + ccmpg {dfv=of} r15, r9 +# CHECK: ccmpno {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x80,0x7c,0x80,0x7b,0x7b] + ccmpno {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpno {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x01,0x83,0x7c,0x80,0x7b,0x7b] + ccmpno {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpno {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x01,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpno {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpno {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x83,0x7c,0x80,0x7b,0x7b] + ccmpno {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpno {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpno {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpno {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x01,0x83,0x7c,0x80,0x7b,0x7b] + ccmpno {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpno {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x01,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpno {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpno {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x38,0x5c,0x80,0x7b] + ccmpno {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpno {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x01,0x39,0x54,0x80,0x7b] + ccmpno {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpno {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x39,0x4c,0x80,0x7b] + ccmpno {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpno {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x01,0x39,0x4c,0x80,0x7b] + ccmpno {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpno {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x3a,0x5c,0x80,0x7b] + ccmpno {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpno {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x01,0x3b,0x54,0x80,0x7b] + ccmpno {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpno {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x3b,0x4c,0x80,0x7b] + ccmpno {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpno {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x01,0x3b,0x4c,0x80,0x7b] + ccmpno {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpno {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x01,0x80,0xfb,0x7b] + ccmpno {dfv=of} bl, 123 +# CHECK: ccmpno {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x01,0x83,0xfa,0x7b] + ccmpno {dfv=of} dx, 123 +# CHECK: ccmpno {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x01,0x83,0xf9,0x7b] + ccmpno {dfv=of} ecx, 123 +# CHECK: ccmpno {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x01,0x83,0xf9,0x7b] + ccmpno {dfv=of} r9, 123 +# CHECK: ccmpno {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x01,0x81,0xfa,0xd2,0x04] + ccmpno {dfv=of} dx, 1234 +# CHECK: ccmpno {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x01,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpno {dfv=of} ecx, 123456 +# CHECK: ccmpno {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x01,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpno {dfv=of} r9, 123456 +# CHECK: ccmpno {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x01,0x38,0xda] + ccmpno {dfv=of} dl, bl +# CHECK: ccmpno {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x01,0x39,0xd0] + ccmpno {dfv=of} ax, dx +# CHECK: ccmpno {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x01,0x39,0xca] + ccmpno {dfv=of} edx, ecx +# CHECK: ccmpno {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x01,0x39,0xcf] + ccmpno {dfv=of} r15, r9 +# CHECK: ccmpns {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x80,0x7c,0x80,0x7b,0x7b] + ccmpns {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpns {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x09,0x83,0x7c,0x80,0x7b,0x7b] + ccmpns {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpns {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x09,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpns {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpns {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x83,0x7c,0x80,0x7b,0x7b] + ccmpns {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpns {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpns {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpns {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x09,0x83,0x7c,0x80,0x7b,0x7b] + ccmpns {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpns {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x09,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpns {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpns {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x38,0x5c,0x80,0x7b] + ccmpns {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpns {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x09,0x39,0x54,0x80,0x7b] + ccmpns {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpns {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x39,0x4c,0x80,0x7b] + ccmpns {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpns {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x09,0x39,0x4c,0x80,0x7b] + ccmpns {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpns {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x3a,0x5c,0x80,0x7b] + ccmpns {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpns {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x09,0x3b,0x54,0x80,0x7b] + ccmpns {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpns {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x3b,0x4c,0x80,0x7b] + ccmpns {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpns {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x09,0x3b,0x4c,0x80,0x7b] + ccmpns {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpns {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x09,0x80,0xfb,0x7b] + ccmpns {dfv=of} bl, 123 +# CHECK: ccmpns {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x09,0x83,0xfa,0x7b] + ccmpns {dfv=of} dx, 123 +# CHECK: ccmpns {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x09,0x83,0xf9,0x7b] + ccmpns {dfv=of} ecx, 123 +# CHECK: ccmpns {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x09,0x83,0xf9,0x7b] + ccmpns {dfv=of} r9, 123 +# CHECK: ccmpns {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x09,0x81,0xfa,0xd2,0x04] + ccmpns {dfv=of} dx, 1234 +# CHECK: ccmpns {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x09,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpns {dfv=of} ecx, 123456 +# CHECK: ccmpns {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x09,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpns {dfv=of} r9, 123456 +# CHECK: ccmpns {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x09,0x38,0xda] + ccmpns {dfv=of} dl, bl +# CHECK: ccmpns {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x09,0x39,0xd0] + ccmpns {dfv=of} ax, dx +# CHECK: ccmpns {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x09,0x39,0xca] + ccmpns {dfv=of} edx, ecx +# CHECK: ccmpns {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x09,0x39,0xcf] + ccmpns {dfv=of} r15, r9 +# CHECK: ccmpne {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x80,0x7c,0x80,0x7b,0x7b] + ccmpne {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpne {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x05,0x83,0x7c,0x80,0x7b,0x7b] + ccmpne {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpne {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x05,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpne {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpne {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x83,0x7c,0x80,0x7b,0x7b] + ccmpne {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpne {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpne {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpne {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x05,0x83,0x7c,0x80,0x7b,0x7b] + ccmpne {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpne {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x05,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpne {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpne {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x38,0x5c,0x80,0x7b] + ccmpne {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpne {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x05,0x39,0x54,0x80,0x7b] + ccmpne {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpne {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x39,0x4c,0x80,0x7b] + ccmpne {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpne {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x05,0x39,0x4c,0x80,0x7b] + ccmpne {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpne {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x3a,0x5c,0x80,0x7b] + ccmpne {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpne {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x05,0x3b,0x54,0x80,0x7b] + ccmpne {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpne {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x3b,0x4c,0x80,0x7b] + ccmpne {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpne {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x05,0x3b,0x4c,0x80,0x7b] + ccmpne {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpne {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x05,0x80,0xfb,0x7b] + ccmpne {dfv=of} bl, 123 +# CHECK: ccmpne {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x05,0x83,0xfa,0x7b] + ccmpne {dfv=of} dx, 123 +# CHECK: ccmpne {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x05,0x83,0xf9,0x7b] + ccmpne {dfv=of} ecx, 123 +# CHECK: ccmpne {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x05,0x83,0xf9,0x7b] + ccmpne {dfv=of} r9, 123 +# CHECK: ccmpne {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x05,0x81,0xfa,0xd2,0x04] + ccmpne {dfv=of} dx, 1234 +# CHECK: ccmpne {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x05,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpne {dfv=of} ecx, 123456 +# CHECK: ccmpne {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x05,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpne {dfv=of} r9, 123456 +# CHECK: ccmpne {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x05,0x38,0xda] + ccmpne {dfv=of} dl, bl +# CHECK: ccmpne {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x05,0x39,0xd0] + ccmpne {dfv=of} ax, dx +# CHECK: ccmpne {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x05,0x39,0xca] + ccmpne {dfv=of} edx, ecx +# CHECK: ccmpne {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x05,0x39,0xcf] + ccmpne {dfv=of} r15, r9 +# CHECK: ccmpo {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x80,0x7c,0x80,0x7b,0x7b] + ccmpo {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpo {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x00,0x83,0x7c,0x80,0x7b,0x7b] + ccmpo {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpo {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x00,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpo {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpo {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x83,0x7c,0x80,0x7b,0x7b] + ccmpo {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpo {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpo {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpo {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x00,0x83,0x7c,0x80,0x7b,0x7b] + ccmpo {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpo {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x00,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpo {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpo {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x38,0x5c,0x80,0x7b] + ccmpo {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpo {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x00,0x39,0x54,0x80,0x7b] + ccmpo {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpo {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x39,0x4c,0x80,0x7b] + ccmpo {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpo {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x00,0x39,0x4c,0x80,0x7b] + ccmpo {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpo {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x3a,0x5c,0x80,0x7b] + ccmpo {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpo {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x00,0x3b,0x54,0x80,0x7b] + ccmpo {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpo {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x3b,0x4c,0x80,0x7b] + ccmpo {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpo {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x00,0x3b,0x4c,0x80,0x7b] + ccmpo {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpo {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x00,0x80,0xfb,0x7b] + ccmpo {dfv=of} bl, 123 +# CHECK: ccmpo {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x00,0x83,0xfa,0x7b] + ccmpo {dfv=of} dx, 123 +# CHECK: ccmpo {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x00,0x83,0xf9,0x7b] + ccmpo {dfv=of} ecx, 123 +# CHECK: ccmpo {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x00,0x83,0xf9,0x7b] + ccmpo {dfv=of} r9, 123 +# CHECK: ccmpo {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x00,0x81,0xfa,0xd2,0x04] + ccmpo {dfv=of} dx, 1234 +# CHECK: ccmpo {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x00,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpo {dfv=of} ecx, 123456 +# CHECK: ccmpo {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x00,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpo {dfv=of} r9, 123456 +# CHECK: ccmpo {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x00,0x38,0xda] + ccmpo {dfv=of} dl, bl +# CHECK: ccmpo {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x00,0x39,0xd0] + ccmpo {dfv=of} ax, dx +# CHECK: ccmpo {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x00,0x39,0xca] + ccmpo {dfv=of} edx, ecx +# CHECK: ccmpo {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x00,0x39,0xcf] + ccmpo {dfv=of} r15, r9 +# CHECK: ccmps {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x80,0x7c,0x80,0x7b,0x7b] + ccmps {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmps {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x08,0x83,0x7c,0x80,0x7b,0x7b] + ccmps {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmps {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x08,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmps {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmps {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x83,0x7c,0x80,0x7b,0x7b] + ccmps {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmps {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmps {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmps {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x08,0x83,0x7c,0x80,0x7b,0x7b] + ccmps {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmps {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x08,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmps {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmps {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x38,0x5c,0x80,0x7b] + ccmps {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmps {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x08,0x39,0x54,0x80,0x7b] + ccmps {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmps {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x39,0x4c,0x80,0x7b] + ccmps {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmps {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x08,0x39,0x4c,0x80,0x7b] + ccmps {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmps {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x3a,0x5c,0x80,0x7b] + ccmps {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmps {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x08,0x3b,0x54,0x80,0x7b] + ccmps {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmps {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x3b,0x4c,0x80,0x7b] + ccmps {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmps {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x08,0x3b,0x4c,0x80,0x7b] + ccmps {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmps {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x08,0x80,0xfb,0x7b] + ccmps {dfv=of} bl, 123 +# CHECK: ccmps {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x08,0x83,0xfa,0x7b] + ccmps {dfv=of} dx, 123 +# CHECK: ccmps {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x08,0x83,0xf9,0x7b] + ccmps {dfv=of} ecx, 123 +# CHECK: ccmps {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x08,0x83,0xf9,0x7b] + ccmps {dfv=of} r9, 123 +# CHECK: ccmps {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x08,0x81,0xfa,0xd2,0x04] + ccmps {dfv=of} dx, 1234 +# CHECK: ccmps {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x08,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmps {dfv=of} ecx, 123456 +# CHECK: ccmps {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x08,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmps {dfv=of} r9, 123456 +# CHECK: ccmps {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x08,0x38,0xda] + ccmps {dfv=of} dl, bl +# CHECK: ccmps {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x08,0x39,0xd0] + ccmps {dfv=of} ax, dx +# CHECK: ccmps {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x08,0x39,0xca] + ccmps {dfv=of} edx, ecx +# CHECK: ccmps {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x08,0x39,0xcf] + ccmps {dfv=of} r15, r9 +# CHECK: ccmpt {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x80,0x7c,0x80,0x7b,0x7b] + ccmpt {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpt {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x0a,0x83,0x7c,0x80,0x7b,0x7b] + ccmpt {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpt {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x0a,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpt {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpt {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x83,0x7c,0x80,0x7b,0x7b] + ccmpt {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpt {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpt {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpt {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0a,0x83,0x7c,0x80,0x7b,0x7b] + ccmpt {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpt {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0a,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpt {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpt {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x38,0x5c,0x80,0x7b] + ccmpt {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpt {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0a,0x39,0x54,0x80,0x7b] + ccmpt {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpt {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x39,0x4c,0x80,0x7b] + ccmpt {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpt {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0a,0x39,0x4c,0x80,0x7b] + ccmpt {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpt {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x3a,0x5c,0x80,0x7b] + ccmpt {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpt {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x0a,0x3b,0x54,0x80,0x7b] + ccmpt {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpt {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x3b,0x4c,0x80,0x7b] + ccmpt {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpt {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x0a,0x3b,0x4c,0x80,0x7b] + ccmpt {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpt {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0a,0x80,0xfb,0x7b] + ccmpt {dfv=of} bl, 123 +# CHECK: ccmpt {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x0a,0x83,0xfa,0x7b] + ccmpt {dfv=of} dx, 123 +# CHECK: ccmpt {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0a,0x83,0xf9,0x7b] + ccmpt {dfv=of} ecx, 123 +# CHECK: ccmpt {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0a,0x83,0xf9,0x7b] + ccmpt {dfv=of} r9, 123 +# CHECK: ccmpt {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x0a,0x81,0xfa,0xd2,0x04] + ccmpt {dfv=of} dx, 1234 +# CHECK: ccmpt {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x0a,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpt {dfv=of} ecx, 123456 +# CHECK: ccmpt {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0a,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpt {dfv=of} r9, 123456 +# CHECK: ccmpt {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0a,0x38,0xda] + ccmpt {dfv=of} dl, bl +# CHECK: ccmpt {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0a,0x39,0xd0] + ccmpt {dfv=of} ax, dx +# CHECK: ccmpt {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0a,0x39,0xca] + ccmpt {dfv=of} edx, ecx +# CHECK: ccmpt {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0a,0x39,0xcf] + ccmpt {dfv=of} r15, r9 +# CHECK: ccmpe {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x80,0x7c,0x80,0x7b,0x7b] + ccmpe {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpe {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x04,0x83,0x7c,0x80,0x7b,0x7b] + ccmpe {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpe {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x04,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpe {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpe {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x83,0x7c,0x80,0x7b,0x7b] + ccmpe {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpe {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpe {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpe {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x04,0x83,0x7c,0x80,0x7b,0x7b] + ccmpe {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpe {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x04,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpe {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpe {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x38,0x5c,0x80,0x7b] + ccmpe {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpe {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x04,0x39,0x54,0x80,0x7b] + ccmpe {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpe {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x39,0x4c,0x80,0x7b] + ccmpe {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpe {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x04,0x39,0x4c,0x80,0x7b] + ccmpe {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpe {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x3a,0x5c,0x80,0x7b] + ccmpe {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpe {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x04,0x3b,0x54,0x80,0x7b] + ccmpe {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpe {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x3b,0x4c,0x80,0x7b] + ccmpe {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpe {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x04,0x3b,0x4c,0x80,0x7b] + ccmpe {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpe {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x04,0x80,0xfb,0x7b] + ccmpe {dfv=of} bl, 123 +# CHECK: ccmpe {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x04,0x83,0xfa,0x7b] + ccmpe {dfv=of} dx, 123 +# CHECK: ccmpe {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x04,0x83,0xf9,0x7b] + ccmpe {dfv=of} ecx, 123 +# CHECK: ccmpe {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x04,0x83,0xf9,0x7b] + ccmpe {dfv=of} r9, 123 +# CHECK: ccmpe {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x04,0x81,0xfa,0xd2,0x04] + ccmpe {dfv=of} dx, 1234 +# CHECK: ccmpe {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x04,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpe {dfv=of} ecx, 123456 +# CHECK: ccmpe {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x04,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpe {dfv=of} r9, 123456 +# CHECK: ccmpe {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x04,0x38,0xda] + ccmpe {dfv=of} dl, bl +# CHECK: ccmpe {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x04,0x39,0xd0] + ccmpe {dfv=of} ax, dx +# CHECK: ccmpe {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x04,0x39,0xca] + ccmpe {dfv=of} edx, ecx +# CHECK: ccmpe {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x04,0x39,0xcf] + ccmpe {dfv=of} r15, r9 diff --git a/llvm/test/TableGen/x86-fold-tables.inc b/llvm/test/TableGen/x86-fold-tables.inc index abeef29c80ed..c35f22ff36de 100644 --- a/llvm/test/TableGen/x86-fold-tables.inc +++ b/llvm/test/TableGen/x86-fold-tables.inc @@ -363,6 +363,17 @@ static const X86FoldTableEntry Table0[] = { {X86::CALL32r_NT, X86::CALL32m_NT, TB_FOLDED_LOAD}, {X86::CALL64r, X86::CALL64m, TB_FOLDED_LOAD}, {X86::CALL64r_NT, X86::CALL64m_NT, TB_FOLDED_LOAD}, + {X86::CCMP16ri, X86::CCMP16mi, TB_FOLDED_LOAD}, + {X86::CCMP16ri8, X86::CCMP16mi8, TB_FOLDED_LOAD}, + {X86::CCMP16rr, X86::CCMP16mr, TB_FOLDED_LOAD}, + {X86::CCMP32ri, X86::CCMP32mi, TB_FOLDED_LOAD}, + {X86::CCMP32ri8, X86::CCMP32mi8, TB_FOLDED_LOAD}, + {X86::CCMP32rr, X86::CCMP32mr, TB_FOLDED_LOAD}, + {X86::CCMP64ri32, X86::CCMP64mi32, TB_FOLDED_LOAD}, + {X86::CCMP64ri8, X86::CCMP64mi8, TB_FOLDED_LOAD}, + {X86::CCMP64rr, X86::CCMP64mr, TB_FOLDED_LOAD}, + {X86::CCMP8ri, X86::CCMP8mi, TB_FOLDED_LOAD}, + {X86::CCMP8rr, X86::CCMP8mr, TB_FOLDED_LOAD}, {X86::CMP16ri, X86::CMP16mi, TB_FOLDED_LOAD}, {X86::CMP16ri8, X86::CMP16mi8, TB_FOLDED_LOAD}, {X86::CMP16rr, X86::CMP16mr, TB_FOLDED_LOAD}, @@ -375,6 +386,14 @@ static const X86FoldTableEntry Table0[] = { {X86::CMP8ri, X86::CMP8mi, TB_FOLDED_LOAD}, {X86::CMP8ri8, X86::CMP8mi8, TB_FOLDED_LOAD}, {X86::CMP8rr, X86::CMP8mr, TB_FOLDED_LOAD}, + {X86::CTEST16ri, X86::CTEST16mi, TB_FOLDED_LOAD}, + {X86::CTEST16rr, X86::CTEST16mr, TB_FOLDED_LOAD}, + {X86::CTEST32ri, X86::CTEST32mi, TB_FOLDED_LOAD}, + {X86::CTEST32rr, X86::CTEST32mr, TB_FOLDED_LOAD}, + {X86::CTEST64ri32, X86::CTEST64mi32, TB_FOLDED_LOAD}, + {X86::CTEST64rr, X86::CTEST64mr, TB_FOLDED_LOAD}, + {X86::CTEST8ri, X86::CTEST8mi, TB_FOLDED_LOAD}, + {X86::CTEST8rr, X86::CTEST8mr, TB_FOLDED_LOAD}, {X86::DIV16r, X86::DIV16m, TB_FOLDED_LOAD}, {X86::DIV16r_NF, X86::DIV16m_NF, TB_FOLDED_LOAD}, {X86::DIV32r, X86::DIV32m, TB_FOLDED_LOAD}, @@ -675,6 +694,10 @@ static const X86FoldTableEntry Table1[] = { {X86::BZHI64rr, X86::BZHI64rm, 0}, {X86::BZHI64rr_EVEX, X86::BZHI64rm_EVEX, 0}, {X86::BZHI64rr_NF, X86::BZHI64rm_NF, 0}, + {X86::CCMP16rr, X86::CCMP16rm, 0}, + {X86::CCMP32rr, X86::CCMP32rm, 0}, + {X86::CCMP64rr, X86::CCMP64rm, 0}, + {X86::CCMP8rr, X86::CCMP8rm, 0}, {X86::CMP16rr, X86::CMP16rm, 0}, {X86::CMP32rr, X86::CMP32rm, 0}, {X86::CMP64rr, X86::CMP64rm, 0}, diff --git a/llvm/tools/llvm-exegesis/lib/X86/Target.cpp b/llvm/tools/llvm-exegesis/lib/X86/Target.cpp index a41a995f5560..31bec2545926 100644 --- a/llvm/tools/llvm-exegesis/lib/X86/Target.cpp +++ b/llvm/tools/llvm-exegesis/lib/X86/Target.cpp @@ -884,6 +884,10 @@ Error ExegesisX86Target::randomizeTargetMCOperand( const BitVector &ForbiddenRegs) const { const Operand &Op = Instr.getPrimaryOperand(Var); switch (Op.getExplicitOperandInfo().OperandType) { + case X86::OperandType::OPERAND_COND_CODE: + AssignedValue = + MCOperand::createImm(randomIndex(X86::CondCode::LAST_VALID_COND)); + return Error::success(); case X86::OperandType::OPERAND_ROUNDING_CONTROL: AssignedValue = MCOperand::createImm(randomIndex(X86::STATIC_ROUNDING::TO_ZERO)); diff --git a/llvm/utils/TableGen/X86RecognizableInstr.cpp b/llvm/utils/TableGen/X86RecognizableInstr.cpp index 5aa2e51c78ba..39006a40daeb 100644 --- a/llvm/utils/TableGen/X86RecognizableInstr.cpp +++ b/llvm/utils/TableGen/X86RecognizableInstr.cpp @@ -127,6 +127,7 @@ RecognizableInstrBase::RecognizableInstrBase(const CodeGenInstruction &insn) { HasEVEX_KZ = Rec->getValueAsBit("hasEVEX_Z"); HasEVEX_B = Rec->getValueAsBit("hasEVEX_B"); HasEVEX_NF = Rec->getValueAsBit("hasEVEX_NF"); + HasTwoConditionalOps = Rec->getValueAsBit("hasTwoConditionalOps"); IsCodeGenOnly = Rec->getValueAsBit("isCodeGenOnly"); IsAsmParserOnly = Rec->getValueAsBit("isAsmParserOnly"); ForceDisassemble = Rec->getValueAsBit("ForceDisassemble"); @@ -494,6 +495,8 @@ void RecognizableInstr::emitInstructionSpecifier() { ++additionalOperands; if (HasEVEX_K) ++additionalOperands; + if (HasTwoConditionalOps) + additionalOperands += 2; #endif bool IsND = OpMap == X86Local::T_MAP4 && HasEVEX_B && HasVEX_4V; @@ -561,6 +564,7 @@ void RecognizableInstr::emitInstructionSpecifier() { HANDLE_OPERAND(roRegister) HANDLE_OPTIONAL(immediate) + HANDLE_OPTIONAL(immediate) break; case X86Local::MRMDestMem4VOp3CC: // Operand 1 is a register operand in the Reg/Opcode field. @@ -599,6 +603,7 @@ void RecognizableInstr::emitInstructionSpecifier() { HANDLE_OPERAND(roRegister) HANDLE_OPTIONAL(immediate) + HANDLE_OPTIONAL(immediate) break; case X86Local::MRMSrcReg: // Operand 1 is a register operand in the Reg/Opcode field. @@ -735,6 +740,7 @@ void RecognizableInstr::emitInstructionSpecifier() { HANDLE_OPTIONAL(rmRegister) HANDLE_OPTIONAL(relocation) HANDLE_OPTIONAL(immediate) + HANDLE_OPTIONAL(immediate) break; case X86Local::MRMXmCC: assert(numPhysicalOperands == 2 && @@ -763,6 +769,8 @@ void RecognizableInstr::emitInstructionSpecifier() { HANDLE_OPERAND(writemaskRegister) HANDLE_OPERAND(memory) HANDLE_OPTIONAL(relocation) + HANDLE_OPTIONAL(immediate) + HANDLE_OPTIONAL(immediate) break; case X86Local::RawFrmImm8: // operand 1 is a 16-bit immediate @@ -1032,6 +1040,7 @@ OperandType RecognizableInstr::typeFromString(const std::string &s, TYPE("i16imm_brtarget", TYPE_REL) TYPE("i32imm_brtarget", TYPE_REL) TYPE("ccode", TYPE_IMM) + TYPE("cflags", TYPE_IMM) TYPE("AVX512RC", TYPE_IMM) TYPE("brtarget32", TYPE_REL) TYPE("brtarget16", TYPE_REL) @@ -1127,6 +1136,8 @@ RecognizableInstr::immediateEncodingFromString(const std::string &s, ENCODING("i64i32imm", ENCODING_ID) ENCODING("i64i8imm", ENCODING_IB) ENCODING("i8imm", ENCODING_IB) + ENCODING("ccode", ENCODING_CC) + ENCODING("cflags", ENCODING_CF) ENCODING("u4imm", ENCODING_IB) ENCODING("u8imm", ENCODING_IB) ENCODING("i16u8imm", ENCODING_IB) diff --git a/llvm/utils/TableGen/X86RecognizableInstr.h b/llvm/utils/TableGen/X86RecognizableInstr.h index 549fc5b4d543..ad319b3f28b7 100644 --- a/llvm/utils/TableGen/X86RecognizableInstr.h +++ b/llvm/utils/TableGen/X86RecognizableInstr.h @@ -214,6 +214,8 @@ struct RecognizableInstrBase { bool HasEVEX_B; /// The hasEVEX_NF field from the record bool HasEVEX_NF; + /// The hasTwoConditionalOps field from the record + bool HasTwoConditionalOps; /// Indicates that the instruction uses the L and L' fields for RC. bool EncodeRC; /// The isCodeGenOnly field from the record -- GitLab From 4d1d1271b8612c72e0020c9d9f42d2ef70f717e7 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 8 Mar 2024 06:54:52 -0600 Subject: [PATCH 599/929] [HIP] Make the HIP default architecture use the enum value (#84400) Summary: This default enum is used in other places, we should keep it consistent. --- clang/include/clang/Basic/Cuda.h | 2 +- clang/lib/Driver/Driver.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/clang/include/clang/Basic/Cuda.h b/clang/include/clang/Basic/Cuda.h index 916cb4b7ef34..3e77a74c7c00 100644 --- a/clang/include/clang/Basic/Cuda.h +++ b/clang/include/clang/Basic/Cuda.h @@ -123,7 +123,7 @@ enum class CudaArch { LAST, CudaDefault = CudaArch::SM_52, - HIPDefault = CudaArch::GFX803, + HIPDefault = CudaArch::GFX906, }; static inline bool IsNVIDIAGpuArch(CudaArch A) { diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp index 96e6ad77f5e5..fce43430a913 100644 --- a/clang/lib/Driver/Driver.cpp +++ b/clang/lib/Driver/Driver.cpp @@ -3373,7 +3373,7 @@ class OffloadingActionBuilder final { const Driver::InputList &Inputs) : CudaActionBuilderBase(C, Args, Inputs, Action::OFK_HIP) { - DefaultCudaArch = CudaArch::GFX906; + DefaultCudaArch = CudaArch::HIPDefault; if (Args.hasArg(options::OPT_fhip_emit_relocatable, options::OPT_fno_hip_emit_relocatable)) { -- GitLab From 7457e2c1535acd548d2619dfb34eb93d27d15908 Mon Sep 17 00:00:00 2001 From: Balazs Benics Date: Fri, 8 Mar 2024 13:57:27 +0100 Subject: [PATCH 600/929] [clang][ASTMatcher] Add matchers for isExplicitObjectMemberFunction() (#84446) Note that this patch will be necessary to fix `forEachArgumentWithParam()` and `forEachArgumentWithParamType()` matchers for deducing "this"; which is my true motivation. There the bug is that with explicit obj params, one should not adjust the number of arguments in presence of `CXXMethodDecls`, and this causes a mismatch there mapping the argument to the wrong param. But, I'll come back there once we have this matcher. --- clang/docs/LibASTMatchersReference.html | 15 +++++++++++++++ clang/docs/ReleaseNotes.rst | 1 + clang/include/clang/ASTMatchers/ASTMatchers.h | 19 +++++++++++++++++++ clang/include/clang/Testing/CommandLineArgs.h | 1 + clang/include/clang/Testing/TestClangConfig.h | 16 +++++++++++----- clang/lib/ASTMatchers/Dynamic/Registry.cpp | 1 + clang/lib/Testing/CommandLineArgs.cpp | 7 +++++++ .../ASTMatchers/ASTMatchersNarrowingTest.cpp | 14 ++++++++++++++ .../ASTMatchers/ASTMatchersNodeTest.cpp | 2 +- clang/unittests/ASTMatchers/ASTMatchersTest.h | 14 ++++++++++---- 10 files changed, 80 insertions(+), 10 deletions(-) diff --git a/clang/docs/LibASTMatchersReference.html b/clang/docs/LibASTMatchersReference.html index 8a06084955aa..bb1b68f6671b 100644 --- a/clang/docs/LibASTMatchersReference.html +++ b/clang/docs/LibASTMatchersReference.html @@ -3546,6 +3546,21 @@ cxxMethodDecl(isConst()) matches A::foo() but not A::bar() +Matcher<CXXMethodDecl>isExplicitObjectMemberFunction +
Matches if the given method declaration declares a member function with an explicit object parameter.
+
+Given
+struct A {
+  int operator-(this A, int);
+  void fun(this A &&self);
+  static int operator()(int);
+  int operator+(int);
+};
+
+cxxMethodDecl(isExplicitObjectMemberFunction()) matches the first two methods but not the last two.
+
+ + Matcher<CXXMethodDecl>isCopyAssignmentOperator 
Matches if the given method declaration declares a copy assignment
 operator.
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 0ee2801766a9..73901a14631b 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -433,6 +433,7 @@ AST Matchers
 ------------
 
 - ``isInStdNamespace`` now supports Decl declared with ``extern "C++"``.
+- Add ``isExplicitObjectMemberFunction``.
 
 clang-format
 ------------
diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h
index ced89ff127ab..96dbcdc344e1 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchers.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -6366,6 +6366,25 @@ AST_MATCHER(CXXMethodDecl, isConst) {
   return Node.isConst();
 }
 
+/// Matches if the given method declaration declares a member function with an
+/// explicit object parameter.
+///
+/// Given
+/// \code
+/// struct A {
+///  int operator-(this A, int);
+///  void fun(this A &&self);
+///  static int operator()(int);
+///  int operator+(int);
+/// };
+/// \endcode
+///
+/// cxxMethodDecl(isExplicitObjectMemberFunction()) matches the first two
+/// methods but not the last two.
+AST_MATCHER(CXXMethodDecl, isExplicitObjectMemberFunction) {
+  return Node.isExplicitObjectMemberFunction();
+}
+
 /// Matches if the given method declaration declares a copy assignment
 /// operator.
 ///
diff --git a/clang/include/clang/Testing/CommandLineArgs.h b/clang/include/clang/Testing/CommandLineArgs.h
index 4dd28718dfa6..e71907e8bbd0 100644
--- a/clang/include/clang/Testing/CommandLineArgs.h
+++ b/clang/include/clang/Testing/CommandLineArgs.h
@@ -28,6 +28,7 @@ enum TestLanguage {
   Lang_CXX14,
   Lang_CXX17,
   Lang_CXX20,
+  Lang_CXX23,
   Lang_OpenCL,
   Lang_OBJC,
   Lang_OBJCXX
diff --git a/clang/include/clang/Testing/TestClangConfig.h b/clang/include/clang/Testing/TestClangConfig.h
index 92d5cc3cff99..1b4efca80e9d 100644
--- a/clang/include/clang/Testing/TestClangConfig.h
+++ b/clang/include/clang/Testing/TestClangConfig.h
@@ -34,24 +34,30 @@ struct TestClangConfig {
   bool isCXX() const {
     return Language == Lang_CXX03 || Language == Lang_CXX11 ||
            Language == Lang_CXX14 || Language == Lang_CXX17 ||
-           Language == Lang_CXX20;
+           Language == Lang_CXX20 || Language == Lang_CXX23;
   }
 
   bool isCXX11OrLater() const {
     return Language == Lang_CXX11 || Language == Lang_CXX14 ||
-           Language == Lang_CXX17 || Language == Lang_CXX20;
+           Language == Lang_CXX17 || Language == Lang_CXX20 ||
+           Language == Lang_CXX23;
   }
 
   bool isCXX14OrLater() const {
     return Language == Lang_CXX14 || Language == Lang_CXX17 ||
-           Language == Lang_CXX20;
+           Language == Lang_CXX20 || Language == Lang_CXX23;
   }
 
   bool isCXX17OrLater() const {
-    return Language == Lang_CXX17 || Language == Lang_CXX20;
+    return Language == Lang_CXX17 || Language == Lang_CXX20 ||
+           Language == Lang_CXX23;
   }
 
-  bool isCXX20OrLater() const { return Language == Lang_CXX20; }
+  bool isCXX20OrLater() const {
+    return Language == Lang_CXX20 || Language == Lang_CXX23;
+  }
+
+  bool isCXX23OrLater() const { return Language == Lang_CXX23; }
 
   bool supportsCXXDynamicExceptionSpecification() const {
     return Language == Lang_CXX03 || Language == Lang_CXX11 ||
diff --git a/clang/lib/ASTMatchers/Dynamic/Registry.cpp b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
index 15dad022df5f..2c75e6beb743 100644
--- a/clang/lib/ASTMatchers/Dynamic/Registry.cpp
+++ b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
@@ -432,6 +432,7 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(isExpansionInMainFile);
   REGISTER_MATCHER(isExpansionInSystemHeader);
   REGISTER_MATCHER(isExplicit);
+  REGISTER_MATCHER(isExplicitObjectMemberFunction);
   REGISTER_MATCHER(isExplicitTemplateSpecialization);
   REGISTER_MATCHER(isExpr);
   REGISTER_MATCHER(isExternC);
diff --git a/clang/lib/Testing/CommandLineArgs.cpp b/clang/lib/Testing/CommandLineArgs.cpp
index 0da087c33e3f..3abc689b93e8 100644
--- a/clang/lib/Testing/CommandLineArgs.cpp
+++ b/clang/lib/Testing/CommandLineArgs.cpp
@@ -37,6 +37,9 @@ std::vector getCommandLineArgsForTesting(TestLanguage Lang) {
   case Lang_CXX20:
     Args = {"-std=c++20", "-frtti"};
     break;
+  case Lang_CXX23:
+    Args = {"-std=c++23", "-frtti"};
+    break;
   case Lang_OBJC:
     Args = {"-x", "objective-c", "-frtti", "-fobjc-nonfragile-abi"};
     break;
@@ -73,6 +76,9 @@ std::vector getCC1ArgsForTesting(TestLanguage Lang) {
   case Lang_CXX20:
     Args = {"-std=c++20"};
     break;
+  case Lang_CXX23:
+    Args = {"-std=c++23"};
+    break;
   case Lang_OBJC:
     Args = {"-xobjective-c"};
     break;
@@ -96,6 +102,7 @@ StringRef getFilenameForTesting(TestLanguage Lang) {
   case Lang_CXX14:
   case Lang_CXX17:
   case Lang_CXX20:
+  case Lang_CXX23:
     return "input.cc";
 
   case Lang_OpenCL:
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
index b75da7bc1ed0..87774b00956a 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
@@ -2107,6 +2107,20 @@ TEST_P(ASTMatchersTest, IsPure) {
   EXPECT_TRUE(notMatches("class X { int f(); };", cxxMethodDecl(isPure())));
 }
 
+TEST_P(ASTMatchersTest, IsExplicitObjectMemberFunction) {
+  if (!GetParam().isCXX23OrLater()) {
+    return;
+  }
+
+  auto ExpObjParamFn = cxxMethodDecl(isExplicitObjectMemberFunction());
+  EXPECT_TRUE(
+      notMatches("struct A { static int operator()(int); };", ExpObjParamFn));
+  EXPECT_TRUE(notMatches("struct A { int operator+(int); };", ExpObjParamFn));
+  EXPECT_TRUE(
+      matches("struct A { int operator-(this A, int); };", ExpObjParamFn));
+  EXPECT_TRUE(matches("struct A { void fun(this A &&self); };", ExpObjParamFn));
+}
+
 TEST_P(ASTMatchersTest, IsCopyAssignmentOperator) {
   if (!GetParam().isCXX()) {
     return;
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
index ae30c03126d7..0edc65162fbe 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
@@ -2754,7 +2754,7 @@ TEST(MatchFinderAPI, MatchesDynamic) {
 static std::vector allTestClangConfigs() {
   std::vector all_configs;
   for (TestLanguage lang : {Lang_C89, Lang_C99, Lang_CXX03, Lang_CXX11,
-                            Lang_CXX14, Lang_CXX17, Lang_CXX20}) {
+                            Lang_CXX14, Lang_CXX17, Lang_CXX20, Lang_CXX23}) {
     TestClangConfig config;
     config.Language = lang;
 
diff --git a/clang/unittests/ASTMatchers/ASTMatchersTest.h b/clang/unittests/ASTMatchers/ASTMatchersTest.h
index 79c618605483..1ed1b5958a8b 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersTest.h
+++ b/clang/unittests/ASTMatchers/ASTMatchersTest.h
@@ -62,22 +62,28 @@ private:
 
 inline ArrayRef langCxx11OrLater() {
   static const TestLanguage Result[] = {Lang_CXX11, Lang_CXX14, Lang_CXX17,
-                                        Lang_CXX20};
+                                        Lang_CXX20, Lang_CXX23};
   return ArrayRef(Result);
 }
 
 inline ArrayRef langCxx14OrLater() {
-  static const TestLanguage Result[] = {Lang_CXX14, Lang_CXX17, Lang_CXX20};
+  static const TestLanguage Result[] = {Lang_CXX14, Lang_CXX17, Lang_CXX20,
+                                        Lang_CXX23};
   return ArrayRef(Result);
 }
 
 inline ArrayRef langCxx17OrLater() {
-  static const TestLanguage Result[] = {Lang_CXX17, Lang_CXX20};
+  static const TestLanguage Result[] = {Lang_CXX17, Lang_CXX20, Lang_CXX23};
   return ArrayRef(Result);
 }
 
 inline ArrayRef langCxx20OrLater() {
-  static const TestLanguage Result[] = {Lang_CXX20};
+  static const TestLanguage Result[] = {Lang_CXX20, Lang_CXX23};
+  return ArrayRef(Result);
+}
+
+inline ArrayRef langCxx23OrLater() {
+  static const TestLanguage Result[] = {Lang_CXX23};
   return ArrayRef(Result);
 }
 
-- 
GitLab


From 9baa4140f2b633a8d075f859690139ef2d79b78b Mon Sep 17 00:00:00 2001
From: Guillaume Chatelet 
Date: Fri, 8 Mar 2024 14:16:07 +0100
Subject: [PATCH 601/929] [libc][NFC] Simplify `BigInt::mul` (#84468)

---
 libc/src/__support/UInt.h | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/libc/src/__support/UInt.h b/libc/src/__support/UInt.h
index cf0c5a669ae8..fadccf545006 100644
--- a/libc/src/__support/UInt.h
+++ b/libc/src/__support/UInt.h
@@ -262,15 +262,13 @@ struct BigInt {
   // Returns the carry value produced by the multiplication operation.
   LIBC_INLINE constexpr WordType mul(WordType x) {
     BigInt<2 * WORD_SIZE, Signed, WordType> partial_sum(0);
-    WordType carry = 0;
     for (size_t i = 0; i < WORD_COUNT; ++i) {
       NumberPair prod = full_mul(val[i], x);
       BigInt<2 * WORD_SIZE, Signed, WordType> tmp({prod.lo, prod.hi});
-      carry += partial_sum.add(tmp);
+      const WordType carry = partial_sum.add(tmp);
       val[i] = partial_sum.val[0];
       partial_sum.val[0] = partial_sum.val[1];
       partial_sum.val[1] = carry;
-      carry = 0;
     }
     return partial_sum.val[1];
   }
-- 
GitLab


From aa6e7a61c56823afb4db65e42af1e6ce26de6186 Mon Sep 17 00:00:00 2001
From: Guillaume Chatelet 
Date: Fri, 8 Mar 2024 14:16:42 +0100
Subject: [PATCH 602/929] [libc][NFC] Remove integer_utils.h (#84466)

Its sole user is `BigInt` so moving `full_mul` inside UInt.h.
---
 libc/src/__support/CMakeLists.txt             | 13 ----
 libc/src/__support/UInt.h                     | 60 ++++++++++++++--
 libc/src/__support/integer_utils.h            | 69 -------------------
 .../llvm-project-overlay/libc/BUILD.bazel     | 12 ----
 4 files changed, 54 insertions(+), 100 deletions(-)
 delete mode 100644 libc/src/__support/integer_utils.h

diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt
index 17c04aa57e6f..66e4d808333f 100644
--- a/libc/src/__support/CMakeLists.txt
+++ b/libc/src/__support/CMakeLists.txt
@@ -194,24 +194,11 @@ add_header_library(
     libc.src.__support.CPP.type_traits
 )
 
-add_header_library(
-  integer_utils
-  HDRS
-    integer_utils.h
-  DEPENDS
-    .math_extras
-    .number_pair
-    libc.src.__support.common
-    libc.src.__support.CPP.bit
-    libc.src.__support.CPP.type_traits
-)
-
 add_header_library(
   uint
   HDRS
     UInt.h
   DEPENDS
-    .integer_utils
     .math_extras
     .number_pair
     libc.src.__support.CPP.array
diff --git a/libc/src/__support/UInt.h b/libc/src/__support/UInt.h
index fadccf545006..94c8d86fd764 100644
--- a/libc/src/__support/UInt.h
+++ b/libc/src/__support/UInt.h
@@ -14,7 +14,6 @@
 #include "src/__support/CPP/limits.h"
 #include "src/__support/CPP/optional.h"
 #include "src/__support/CPP/type_traits.h"
-#include "src/__support/integer_utils.h"
 #include "src/__support/macros/attributes.h"   // LIBC_INLINE
 #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
 #include "src/__support/math_extras.h"         // SumCarry, DiffBorrow
@@ -36,6 +35,52 @@ template <> struct half_width<__uint128_t> : cpp::type_identity {};
 #endif // __SIZEOF_INT128__
 
 template  using half_width_t = typename half_width::type;
+
+template  constexpr NumberPair full_mul(T a, T b) {
+  NumberPair pa = split(a);
+  NumberPair pb = split(b);
+  NumberPair prod;
+
+  prod.lo = pa.lo * pb.lo;                    // exact
+  prod.hi = pa.hi * pb.hi;                    // exact
+  NumberPair lo_hi = split(pa.lo * pb.hi); // exact
+  NumberPair hi_lo = split(pa.hi * pb.lo); // exact
+
+  constexpr size_t HALF_BIT_WIDTH = sizeof(T) * CHAR_BIT / 2;
+
+  auto r1 = add_with_carry(prod.lo, lo_hi.lo << HALF_BIT_WIDTH, T(0));
+  prod.lo = r1.sum;
+  prod.hi = add_with_carry(prod.hi, lo_hi.hi, r1.carry).sum;
+
+  auto r2 = add_with_carry(prod.lo, hi_lo.lo << HALF_BIT_WIDTH, T(0));
+  prod.lo = r2.sum;
+  prod.hi = add_with_carry(prod.hi, hi_lo.hi, r2.carry).sum;
+
+  return prod;
+}
+
+template <>
+LIBC_INLINE constexpr NumberPair full_mul(uint32_t a,
+                                                              uint32_t b) {
+  uint64_t prod = uint64_t(a) * uint64_t(b);
+  NumberPair result;
+  result.lo = uint32_t(prod);
+  result.hi = uint32_t(prod >> 32);
+  return result;
+}
+
+#ifdef __SIZEOF_INT128__
+template <>
+LIBC_INLINE constexpr NumberPair full_mul(uint64_t a,
+                                                              uint64_t b) {
+  __uint128_t prod = __uint128_t(a) * __uint128_t(b);
+  NumberPair result;
+  result.lo = uint64_t(prod);
+  result.hi = uint64_t(prod >> 64);
+  return result;
+}
+#endif // __SIZEOF_INT128__
+
 } // namespace internal
 
 template 
@@ -263,7 +308,7 @@ struct BigInt {
   LIBC_INLINE constexpr WordType mul(WordType x) {
     BigInt<2 * WORD_SIZE, Signed, WordType> partial_sum(0);
     for (size_t i = 0; i < WORD_COUNT; ++i) {
-      NumberPair prod = full_mul(val[i], x);
+      NumberPair prod = internal::full_mul(val[i], x);
       BigInt<2 * WORD_SIZE, Signed, WordType> tmp({prod.lo, prod.hi});
       const WordType carry = partial_sum.add(tmp);
       val[i] = partial_sum.val[0];
@@ -296,7 +341,8 @@ struct BigInt {
         WordType carry = 0;
         for (size_t i = 0; i < WORD_COUNT; ++i) {
           for (size_t j = 0; j <= i; j++) {
-            NumberPair prod = full_mul(val[j], other.val[i - j]);
+            NumberPair prod =
+                internal::full_mul(val[j], other.val[i - j]);
             BigInt<2 * WORD_SIZE, Signed, WordType> tmp({prod.lo, prod.hi});
             carry += partial_sum.add(tmp);
           }
@@ -324,7 +370,8 @@ struct BigInt {
           i < OTHER_WORDCOUNT ? 0 : i - OTHER_WORDCOUNT + 1;
       const size_t upper_idx = i < WORD_COUNT ? i : WORD_COUNT - 1;
       for (size_t j = lower_idx; j <= upper_idx; ++j) {
-        NumberPair prod = full_mul(val[j], other.val[i - j]);
+        NumberPair prod =
+            internal::full_mul(val[j], other.val[i - j]);
         BigInt<2 * WORD_SIZE, Signed, WordType> tmp({prod.lo, prod.hi});
         carry += partial_sum.add(tmp);
       }
@@ -366,7 +413,7 @@ struct BigInt {
     // product.
     for (size_t i = 0; i < WORD_COUNT; ++i) {
       NumberPair prod =
-          full_mul(val[i], other.val[WORD_COUNT - 1 - i]);
+          internal::full_mul(val[i], other.val[WORD_COUNT - 1 - i]);
       BigInt<2 * WORD_SIZE, Signed, WordType> tmp({prod.lo, prod.hi});
       carry += partial_sum.add(tmp);
     }
@@ -375,7 +422,8 @@ struct BigInt {
       partial_sum.val[1] = carry;
       carry = 0;
       for (size_t j = i - WORD_COUNT + 1; j < WORD_COUNT; ++j) {
-        NumberPair prod = full_mul(val[j], other.val[i - j]);
+        NumberPair prod =
+            internal::full_mul(val[j], other.val[i - j]);
         BigInt<2 * WORD_SIZE, Signed, WordType> tmp({prod.lo, prod.hi});
         carry += partial_sum.add(tmp);
       }
diff --git a/libc/src/__support/integer_utils.h b/libc/src/__support/integer_utils.h
deleted file mode 100644
index 15e04bda8082..000000000000
--- a/libc/src/__support/integer_utils.h
+++ /dev/null
@@ -1,69 +0,0 @@
-//===-- Utilities for integers. ---------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_INTEGER_UTILS_H
-#define LLVM_LIBC_SRC___SUPPORT_INTEGER_UTILS_H
-
-#include "src/__support/CPP/type_traits.h"
-#include "src/__support/common.h"
-
-#include "math_extras.h"
-#include "number_pair.h"
-
-#include 
-
-namespace LIBC_NAMESPACE {
-
-template  constexpr NumberPair full_mul(T a, T b) {
-  NumberPair pa = split(a);
-  NumberPair pb = split(b);
-  NumberPair prod;
-
-  prod.lo = pa.lo * pb.lo;                    // exact
-  prod.hi = pa.hi * pb.hi;                    // exact
-  NumberPair lo_hi = split(pa.lo * pb.hi); // exact
-  NumberPair hi_lo = split(pa.hi * pb.lo); // exact
-
-  constexpr size_t HALF_BIT_WIDTH = sizeof(T) * CHAR_BIT / 2;
-
-  auto r1 = add_with_carry(prod.lo, lo_hi.lo << HALF_BIT_WIDTH, T(0));
-  prod.lo = r1.sum;
-  prod.hi = add_with_carry(prod.hi, lo_hi.hi, r1.carry).sum;
-
-  auto r2 = add_with_carry(prod.lo, hi_lo.lo << HALF_BIT_WIDTH, T(0));
-  prod.lo = r2.sum;
-  prod.hi = add_with_carry(prod.hi, hi_lo.hi, r2.carry).sum;
-
-  return prod;
-}
-
-template <>
-LIBC_INLINE constexpr NumberPair full_mul(uint32_t a,
-                                                              uint32_t b) {
-  uint64_t prod = uint64_t(a) * uint64_t(b);
-  NumberPair result;
-  result.lo = uint32_t(prod);
-  result.hi = uint32_t(prod >> 32);
-  return result;
-}
-
-#ifdef __SIZEOF_INT128__
-template <>
-LIBC_INLINE constexpr NumberPair full_mul(uint64_t a,
-                                                              uint64_t b) {
-  __uint128_t prod = __uint128_t(a) * __uint128_t(b);
-  NumberPair result;
-  result.lo = uint64_t(prod);
-  result.hi = uint64_t(prod >> 64);
-  return result;
-}
-#endif // __SIZEOF_INT128__
-
-} // namespace LIBC_NAMESPACE
-
-#endif // LLVM_LIBC_SRC___SUPPORT_INTEGER_UTILS_H
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 5c6cf761ebe7..127059c529ec 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -438,17 +438,6 @@ libc_support_library(
     ],
 )
 
-libc_support_library(
-    name = "__support_integer_utils",
-    hdrs = ["src/__support/integer_utils.h"],
-    deps = [
-        ":__support_common",
-        ":__support_cpp_type_traits",
-        ":__support_math_extras",
-        ":__support_number_pair",
-    ],
-)
-
 libc_support_library(
     name = "__support_uint",
     hdrs = ["src/__support/UInt.h"],
@@ -458,7 +447,6 @@ libc_support_library(
         ":__support_cpp_limits",
         ":__support_cpp_optional",
         ":__support_cpp_type_traits",
-        ":__support_integer_utils",
         ":__support_macros_attributes",
         ":__support_macros_optimization",
         ":__support_math_extras",
-- 
GitLab


From 912ea6e335ca6174b3433fd960ed8b592794361b Mon Sep 17 00:00:00 2001
From: Congcong Cai 
Date: Fri, 8 Mar 2024 21:18:58 +0800
Subject: [PATCH 603/929] [clang-tidy]avoid bugprone-unused-return-value false
 positive for function with the same prefix as the default argument (#84333)

---
 .../bugprone/UnusedReturnValueCheck.cpp       | 184 +++++++++---------
 clang-tools-extra/docs/ReleaseNotes.rst       |   4 +-
 .../checks/bugprone/unused-return-value.rst   |  33 ++--
 .../checkers/bugprone/unused-return-value.cpp |  14 ++
 4 files changed, 126 insertions(+), 109 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp
index b4bf85c912c3..1252b2f23805 100644
--- a/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp
@@ -34,102 +34,102 @@ UnusedReturnValueCheck::UnusedReturnValueCheck(llvm::StringRef Name,
                                                ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
       CheckedFunctions(utils::options::parseStringList(
-          Options.get("CheckedFunctions", "::std::async;"
-                                          "::std::launder;"
-                                          "::std::remove;"
-                                          "::std::remove_if;"
-                                          "::std::unique;"
-                                          "::std::unique_ptr::release;"
-                                          "::std::basic_string::empty;"
-                                          "::std::vector::empty;"
-                                          "::std::back_inserter;"
-                                          "::std::distance;"
-                                          "::std::find;"
-                                          "::std::find_if;"
-                                          "::std::inserter;"
-                                          "::std::lower_bound;"
-                                          "::std::make_pair;"
-                                          "::std::map::count;"
-                                          "::std::map::find;"
-                                          "::std::map::lower_bound;"
-                                          "::std::multimap::equal_range;"
-                                          "::std::multimap::upper_bound;"
-                                          "::std::set::count;"
-                                          "::std::set::find;"
-                                          "::std::setfill;"
-                                          "::std::setprecision;"
-                                          "::std::setw;"
-                                          "::std::upper_bound;"
-                                          "::std::vector::at;"
+          Options.get("CheckedFunctions", "::std::async$;"
+                                          "::std::launder$;"
+                                          "::std::remove$;"
+                                          "::std::remove_if$;"
+                                          "::std::unique$;"
+                                          "::std::unique_ptr::release$;"
+                                          "::std::basic_string::empty$;"
+                                          "::std::vector::empty$;"
+                                          "::std::back_inserter$;"
+                                          "::std::distance$;"
+                                          "::std::find$;"
+                                          "::std::find_if$;"
+                                          "::std::inserter$;"
+                                          "::std::lower_bound$;"
+                                          "::std::make_pair$;"
+                                          "::std::map::count$;"
+                                          "::std::map::find$;"
+                                          "::std::map::lower_bound$;"
+                                          "::std::multimap::equal_range$;"
+                                          "::std::multimap::upper_bound$;"
+                                          "::std::set::count$;"
+                                          "::std::set::find$;"
+                                          "::std::setfill$;"
+                                          "::std::setprecision$;"
+                                          "::std::setw$;"
+                                          "::std::upper_bound$;"
+                                          "::std::vector::at$;"
                                           // C standard library
-                                          "::bsearch;"
-                                          "::ferror;"
-                                          "::feof;"
-                                          "::isalnum;"
-                                          "::isalpha;"
-                                          "::isblank;"
-                                          "::iscntrl;"
-                                          "::isdigit;"
-                                          "::isgraph;"
-                                          "::islower;"
-                                          "::isprint;"
-                                          "::ispunct;"
-                                          "::isspace;"
-                                          "::isupper;"
-                                          "::iswalnum;"
-                                          "::iswprint;"
-                                          "::iswspace;"
-                                          "::isxdigit;"
-                                          "::memchr;"
-                                          "::memcmp;"
-                                          "::strcmp;"
-                                          "::strcoll;"
-                                          "::strncmp;"
-                                          "::strpbrk;"
-                                          "::strrchr;"
-                                          "::strspn;"
-                                          "::strstr;"
-                                          "::wcscmp;"
+                                          "::bsearch$;"
+                                          "::ferror$;"
+                                          "::feof$;"
+                                          "::isalnum$;"
+                                          "::isalpha$;"
+                                          "::isblank$;"
+                                          "::iscntrl$;"
+                                          "::isdigit$;"
+                                          "::isgraph$;"
+                                          "::islower$;"
+                                          "::isprint$;"
+                                          "::ispunct$;"
+                                          "::isspace$;"
+                                          "::isupper$;"
+                                          "::iswalnum$;"
+                                          "::iswprint$;"
+                                          "::iswspace$;"
+                                          "::isxdigit$;"
+                                          "::memchr$;"
+                                          "::memcmp$;"
+                                          "::strcmp$;"
+                                          "::strcoll$;"
+                                          "::strncmp$;"
+                                          "::strpbrk$;"
+                                          "::strrchr$;"
+                                          "::strspn$;"
+                                          "::strstr$;"
+                                          "::wcscmp$;"
                                           // POSIX
-                                          "::access;"
-                                          "::bind;"
-                                          "::connect;"
-                                          "::difftime;"
-                                          "::dlsym;"
-                                          "::fnmatch;"
-                                          "::getaddrinfo;"
-                                          "::getopt;"
-                                          "::htonl;"
-                                          "::htons;"
-                                          "::iconv_open;"
-                                          "::inet_addr;"
-                                          "::isascii;"
-                                          "::isatty;"
-                                          "::mmap;"
-                                          "::newlocale;"
-                                          "::openat;"
-                                          "::pathconf;"
-                                          "::pthread_equal;"
-                                          "::pthread_getspecific;"
-                                          "::pthread_mutex_trylock;"
-                                          "::readdir;"
-                                          "::readlink;"
-                                          "::recvmsg;"
-                                          "::regexec;"
-                                          "::scandir;"
-                                          "::semget;"
-                                          "::setjmp;"
-                                          "::shm_open;"
-                                          "::shmget;"
-                                          "::sigismember;"
-                                          "::strcasecmp;"
-                                          "::strsignal;"
+                                          "::access$;"
+                                          "::bind$;"
+                                          "::connect$;"
+                                          "::difftime$;"
+                                          "::dlsym$;"
+                                          "::fnmatch$;"
+                                          "::getaddrinfo$;"
+                                          "::getopt$;"
+                                          "::htonl$;"
+                                          "::htons$;"
+                                          "::iconv_open$;"
+                                          "::inet_addr$;"
+                                          "::isascii$;"
+                                          "::isatty$;"
+                                          "::mmap$;"
+                                          "::newlocale$;"
+                                          "::openat$;"
+                                          "::pathconf$;"
+                                          "::pthread_equal$;"
+                                          "::pthread_getspecific$;"
+                                          "::pthread_mutex_trylock$;"
+                                          "::readdir$;"
+                                          "::readlink$;"
+                                          "::recvmsg$;"
+                                          "::regexec$;"
+                                          "::scandir$;"
+                                          "::semget$;"
+                                          "::setjmp$;"
+                                          "::shm_open$;"
+                                          "::shmget$;"
+                                          "::sigismember$;"
+                                          "::strcasecmp$;"
+                                          "::strsignal$;"
                                           "::ttyname"))),
       CheckedReturnTypes(utils::options::parseStringList(
-          Options.get("CheckedReturnTypes", "::std::error_code;"
-                                            "::std::error_condition;"
-                                            "::std::errc;"
-                                            "::std::expected;"
+          Options.get("CheckedReturnTypes", "::std::error_code$;"
+                                            "::std::error_condition$;"
+                                            "::std::errc$;"
+                                            "::std::expected$;"
                                             "::boost::system::error_code"))),
       AllowCastToVoid(Options.get("AllowCastToVoid", false)) {}
 
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index d98c4ff9a750..b5f025ce467a 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -152,7 +152,9 @@ Changes in existing checks
 
 - Improved :doc:`bugprone-unused-return-value
   ` check by updating the
-  parameter `CheckedFunctions` to support regexp.
+  parameter `CheckedFunctions` to support regexp and avoiding false postive for
+  function with the same prefix as the default argument, e.g. ``std::unique_ptr``
+  and ``std::unique``.
 
 - Improved :doc:`bugprone-use-after-move
   ` check to also handle
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone/unused-return-value.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone/unused-return-value.rst
index 8d5eddbe215c..9c01ef50b538 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/bugprone/unused-return-value.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone/unused-return-value.rst
@@ -14,22 +14,23 @@ Options
    This parameter supports regexp. The function is checked if the name
    and scope matches, with any arguments.
    By default the following functions are checked:
-   ``std::async, std::launder, std::remove, std::remove_if, std::unique,
-   std::unique_ptr::release, std::basic_string::empty, std::vector::empty,
-   std::back_inserter, std::distance, std::find, std::find_if, std::inserter,
-   std::lower_bound, std::make_pair, std::map::count, std::map::find,
-   std::map::lower_bound, std::multimap::equal_range,
-   std::multimap::upper_bound, std::set::count, std::set::find, std::setfill,
-   std::setprecision, std::setw, std::upper_bound, std::vector::at,
-   bsearch, ferror, feof, isalnum, isalpha, isblank, iscntrl, isdigit, isgraph,
-   islower, isprint, ispunct, isspace, isupper, iswalnum, iswprint, iswspace,
-   isxdigit, memchr, memcmp, strcmp, strcoll, strncmp, strpbrk, strrchr,
-   strspn, strstr, wcscmp, access, bind, connect, difftime, dlsym, fnmatch,
-   getaddrinfo, getopt, htonl, htons, iconv_open, inet_addr, isascii, isatty,
-   mmap, newlocale, openat, pathconf, pthread_equal, pthread_getspecific,
-   pthread_mutex_trylock, readdir, readlink, recvmsg, regexec, scandir,
-   semget, setjmp, shm_open, shmget, sigismember, strcasecmp, strsignal,
-   ttyname``
+   ``::std::async$, ::std::launder$, ::std::remove$, ::std::remove_if$, ::std::unique$,
+   ::std::unique_ptr::release$, ::std::basic_string::empty$, ::std::vector::empty$,
+   ::std::back_inserter$, ::std::distance$, ::std::find$, ::std::find_if$, ::std::inserter$,
+   ::std::lower_bound$, ::std::make_pair$, ::std::map::count$, ::std::map::find$,
+   ::std::map::lower_bound$, ::std::multimap::equal_range$, ::std::multimap::upper_bound$,
+   ::std::set::count$, ::std::set::find$, ::std::setfill$, ::std::setprecision$,
+   ::std::setw$, ::std::upper_bound$, ::std::vector::at$, ::bsearch$, ::ferror$,
+   ::feof$, ::isalnum$, ::isalpha$, ::isblank$, ::iscntrl$, ::isdigit$, ::isgraph$,
+   ::islower$, ::isprint$, ::ispunct$, ::isspace$, ::isupper$, ::iswalnum$, ::iswprint$,
+   ::iswspace$, ::isxdigit$, ::memchr$, ::memcmp$, ::strcmp$, ::strcoll$, ::strncmp$,
+   ::strpbrk$, ::strrchr$, ::strspn$, ::strstr$, ::wcscmp$, ::access$, ::bind$,
+   ::connect$, ::difftime$, ::dlsym$, ::fnmatch$, ::getaddrinfo$, ::getopt$,
+   ::htonl$, ::htons$, ::iconv_open$, ::inet_addr$, isascii$, isatty$, ::mmap$,
+   ::newlocale$, ::openat$, ::pathconf$, ::pthread_equal$, ::pthread_getspecific$,
+   ::pthread_mutex_trylock$, ::readdir$, ::readlink$, ::recvmsg$, ::regexec$, ::scandir$,
+   ::semget$, ::setjmp$, ::shm_open$, ::shmget$, ::sigismember$, ::strcasecmp$, ::strsignal$,
+   ::ttyname$``
 
    - ``std::async()``. Not using the return value makes the call synchronous.
    - ``std::launder()``. Not using the return value usually means that the
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/unused-return-value.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/unused-return-value.cpp
index 5c6ce1e4bf1f..e784c9b85172 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/unused-return-value.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/unused-return-value.cpp
@@ -30,6 +30,11 @@ struct default_delete;
 
 template >
 struct unique_ptr {
+  unique_ptr();
+  unique_ptr(unique_ptr const&);
+  unique_ptr(unique_ptr &&);
+  unique_ptr& operator=(unique_ptr const&);
+  unique_ptr& operator=(unique_ptr &&);
   T *release() noexcept;
 };
 
@@ -254,3 +259,12 @@ void noWarning() {
   ({ std::async(increment, 42); });
   auto StmtExprRetval = ({ std::async(increment, 42); });
 }
+
+namespace gh84314 {
+
+extern std::unique_ptr alloc();
+void f1(std::unique_ptr &foo) {
+    foo = alloc();
+}
+
+} // namespace gh84314
\ No newline at end of file
-- 
GitLab


From 10b01563da1d7c98d5fff6aa8b4a78be56a8099d Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek 
Date: Fri, 8 Mar 2024 07:20:20 -0600
Subject: [PATCH 604/929] [flang][HLFIR] Fix use-after-free when rewriting
 users in `canonicalize` (#84371)

Rewriting an op can invalidate the operator range being iterated on.
Store the users in a separate list, and iterate over the list instead.

This was detected by address sanitizer.
---
 flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
index 8bc92a991a69..3568fe202caf 100644
--- a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
+++ b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
+
 #include "flang/Optimizer/Dialect/FIROpsSupport.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Optimizer/Dialect/Support/FIRContext.h"
@@ -1152,7 +1153,9 @@ hlfir::MatmulOp::canonicalize(MatmulOp matmulOp,
 
       // but we do need to get rid of the hlfir.destroy for the hlfir.transpose
       // result (which is entirely removed)
-      for (mlir::Operation *user : transposeOp->getResult(0).getUsers())
+      llvm::SmallVector users(
+          transposeOp->getResult(0).getUsers());
+      for (mlir::Operation *user : users)
         if (auto destroyOp = mlir::dyn_cast_or_null(user))
           rewriter.eraseOp(destroyOp);
       rewriter.eraseOp(transposeOp);
@@ -1864,7 +1867,8 @@ hlfir::ForallIndexOp::canonicalize(hlfir::ForallIndexOp indexOp,
       return mlir::failure();
 
   auto insertPt = rewriter.saveInsertionPoint();
-  for (mlir::Operation *user : indexOp->getResult(0).getUsers())
+  llvm::SmallVector users(indexOp->getResult(0).getUsers());
+  for (mlir::Operation *user : users)
     if (auto loadOp = mlir::dyn_cast(user)) {
       rewriter.setInsertionPoint(loadOp);
       rewriter.replaceOpWithNewOp(
-- 
GitLab


From 65524fcb5d4e5f4f5450c38d3e075cce192586bb Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek 
Date: Fri, 8 Mar 2024 07:20:55 -0600
Subject: [PATCH 605/929] [flang][CodeGen] Replace correct op in
 BoxedProcedurePass (#84375)

---
 flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp b/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
index 846a78931acb..4cf39716a737 100644
--- a/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
+++ b/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
@@ -342,7 +342,7 @@ public:
             auto toTy = typeConverter.convertType(ty);
             auto toOnTy = typeConverter.convertType(onTy);
             rewriter.replaceOpWithNewOp(
-                mem, toTy, index.getFieldId(), toOnTy, index.getTypeparams());
+                index, toTy, index.getFieldId(), toOnTy, index.getTypeparams());
           }
         } else if (op->getDialect() == firDialect) {
           rewriter.startOpModification(op);
-- 
GitLab


From aa26faf05ecd7ee77487772fe0406f2de1df0dac Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek 
Date: Fri, 8 Mar 2024 07:21:21 -0600
Subject: [PATCH 606/929] [flang][unittests] Fix buffer overrun in
 FrontendActionTest (#84381)

When` SmallVector` is used as a backing storage, it can't be
assumed to end with a \x0. When creating a `StringRef` from it, pass the
length explicitly.

This was detected by address sanitizer.
---
 flang/unittests/Frontend/FrontendActionTest.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/flang/unittests/Frontend/FrontendActionTest.cpp b/flang/unittests/Frontend/FrontendActionTest.cpp
index 6ec15832d96d..123f428cc8b4 100644
--- a/flang/unittests/Frontend/FrontendActionTest.cpp
+++ b/flang/unittests/Frontend/FrontendActionTest.cpp
@@ -198,7 +198,7 @@ TEST_F(FrontendActionTest, EmitLLVM) {
   EXPECT_TRUE(success);
   EXPECT_TRUE(!outputFileBuffer.empty());
 
-  EXPECT_TRUE(llvm::StringRef(outputFileBuffer.data())
+  EXPECT_TRUE(llvm::StringRef(outputFileBuffer.begin(), outputFileBuffer.size())
                   .contains("define void @_QQmain()"));
 }
 
@@ -227,6 +227,7 @@ TEST_F(FrontendActionTest, EmitAsm) {
   EXPECT_TRUE(success);
   EXPECT_TRUE(!outputFileBuffer.empty());
 
-  EXPECT_TRUE(llvm::StringRef(outputFileBuffer.data()).contains("_QQmain"));
+  EXPECT_TRUE(llvm::StringRef(outputFileBuffer.begin(), outputFileBuffer.size())
+                  .contains("_QQmain"));
 }
 } // namespace
-- 
GitLab


From 141ebdd242f35bfebc576c51d8b60182ae4eac2d Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek 
Date: Fri, 8 Mar 2024 07:21:42 -0600
Subject: [PATCH 607/929] =?UTF-8?q?[Frontend][OpenMP]=20introduce=20OMP.h?=
 =?UTF-8?q?=20header=20file,=20use=20it=20instead=20of=20OMP=E2=80=A6=20(#?=
 =?UTF-8?q?84188)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

….h.inc

The consumers of OpenMP-related definitions include a TableGen-generated
file OMP.h.inc. Having a separate OMP.h allows putting additional
declarations in there that are not auto-generated.

This patch is NFC.
---
 flang/lib/Semantics/rewrite-directives.cpp     |  2 +-
 llvm/include/llvm/Frontend/OpenMP/OMP.h        | 18 ++++++++++++++++++
 .../llvm/Frontend/OpenMP/OMPConstants.h        |  3 +--
 llvm/lib/Frontend/OpenMP/OMP.cpp               |  2 +-
 4 files changed, 21 insertions(+), 4 deletions(-)
 create mode 100644 llvm/include/llvm/Frontend/OpenMP/OMP.h

diff --git a/flang/lib/Semantics/rewrite-directives.cpp b/flang/lib/Semantics/rewrite-directives.cpp
index bab91d253082..2c3c87f2546a 100644
--- a/flang/lib/Semantics/rewrite-directives.cpp
+++ b/flang/lib/Semantics/rewrite-directives.cpp
@@ -11,7 +11,7 @@
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/semantics.h"
 #include "flang/Semantics/symbol.h"
-#include "llvm/Frontend/OpenMP/OMP.h.inc"
+#include "llvm/Frontend/OpenMP/OMP.h"
 #include 
 
 namespace Fortran::semantics {
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.h b/llvm/include/llvm/Frontend/OpenMP/OMP.h
new file mode 100644
index 000000000000..a85cd9d344c6
--- /dev/null
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.h
@@ -0,0 +1,18 @@
+//===-- OMP.h - Core OpenMP definitions and declarations ---------- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the core set of OpenMP definitions and declarations.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FRONTEND_OPENMP_OMP_H
+#define LLVM_FRONTEND_OPENMP_OMP_H
+
+#include "llvm/Frontend/OpenMP/OMP.h.inc"
+
+#endif // LLVM_FRONTEND_OPENMP_OMP_H
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h
index f8812e7955b8..1bfaf3718552 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h
@@ -15,9 +15,8 @@
 #define LLVM_FRONTEND_OPENMP_OMPCONSTANTS_H
 
 #include "llvm/ADT/BitmaskEnum.h"
-
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Frontend/OpenMP/OMP.h.inc"
+#include "llvm/Frontend/OpenMP/OMP.h"
 
 namespace llvm {
 namespace omp {
diff --git a/llvm/lib/Frontend/OpenMP/OMP.cpp b/llvm/lib/Frontend/OpenMP/OMP.cpp
index eac5d4168c5c..4f2f95392648 100644
--- a/llvm/lib/Frontend/OpenMP/OMP.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMP.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Frontend/OpenMP/OMP.h.inc"
+#include "llvm/Frontend/OpenMP/OMP.h"
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
-- 
GitLab


From 7415524b45392651969374c067041daa82dc89e7 Mon Sep 17 00:00:00 2001
From: Haojian Wu 
Date: Fri, 8 Mar 2024 14:24:03 +0100
Subject: [PATCH 608/929] [clang] Implement CTAD for type alias template.
 (#77890)

Fixes #54051

This patch implements the C++20 feature -- CTAD for alias templates (P1814R0, specified in https://eel.is/c++draft/over.match.class.deduct#3). It is an initial patch:
- it cover major pieces, thus it works for most cases;
- the big missing piece is to implement the associated constraints (over.match.class.deduct#3.3) for the synthesized deduction guides, see the FIXME in code and tests;
- Some enhancements on the TreeTransform&TemplateInstantiator to allow performing instantiation on `BuildingDeductionGuides` mode;
---
 clang/docs/ReleaseNotes.rst                   |   4 +
 .../clang/Basic/DiagnosticSemaKinds.td        |  10 +-
 clang/include/clang/Sema/Sema.h               |  17 +-
 clang/lib/Sema/SemaInit.cpp                   |  37 +-
 clang/lib/Sema/SemaTemplate.cpp               | 498 +++++++++++++++---
 clang/lib/Sema/SemaTemplateDeduction.cpp      |   9 +
 clang/lib/Sema/SemaTemplateInstantiate.cpp    |  69 ++-
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  |  29 +-
 clang/lib/Sema/TreeTransform.h                |   8 +
 clang/test/SemaCXX/cxx17-compat.cpp           |  11 +
 ...xx1z-class-template-argument-deduction.cpp |   6 +-
 clang/test/SemaCXX/cxx20-ctad-type-alias.cpp  | 232 ++++++++
 clang/www/cxx_status.html                     |   8 +-
 13 files changed, 827 insertions(+), 111 deletions(-)
 create mode 100644 clang/test/SemaCXX/cxx20-ctad-type-alias.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 73901a14631b..24b452bd487f 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -91,6 +91,10 @@ C++20 Feature Support
   current module units.
   Fixes `#84002 `_.
 
+- Initial support for class template argument deduction (CTAD) for type alias
+  templates (`P1814R0 `_).
+  (#GH54051).
+
 C++23 Feature Support
 ^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index a80c8b75e863..6da49facd27e 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -2515,10 +2515,11 @@ def err_deduced_class_template_compound_type : Error<
   "cannot %select{form pointer to|form reference to|form array of|"
   "form function returning|use parentheses when declaring variable with}0 "
   "deduced class template specialization type">;
-def err_deduced_non_class_template_specialization_type : Error<
+def err_deduced_non_class_or_alias_template_specialization_type : Error<
   "%select{|function template|variable template|alias template|"
   "template template parameter|concept|template}0 %1 requires template "
-  "arguments; argument deduction only allowed for class templates">;
+  "arguments; argument deduction only allowed for class templates or alias "
+  "templates">;
 def err_deduced_class_template_ctor_ambiguous : Error<
   "ambiguous deduction for template arguments of %0">;
 def err_deduced_class_template_ctor_no_viable : Error<
@@ -8201,6 +8202,11 @@ let CategoryName = "Lambda Issue" in {
   def warn_cxx17_compat_lambda_def_ctor_assign : Warning<
     "%select{default construction|assignment}0 of lambda is incompatible with "
     "C++ standards before C++20">, InGroup, DefaultIgnore;
+
+  // C++20 class template argument deduction for alias templates.
+  def warn_cxx17_compat_ctad_for_alias_templates : Warning<
+  "class template argument deduction for alias templates is incompatible with "
+  "C++ standards before C++20">, InGroup, DefaultIgnore;
 }
 
 def err_return_in_captured_stmt : Error<
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 2cac7e6ca08f..5b4d67494f2a 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -9827,6 +9827,12 @@ public:
                           ArrayRef TemplateArgs,
                           sema::TemplateDeductionInfo &Info);
 
+  TemplateDeductionResult DeduceTemplateArguments(
+      TemplateParameterList *TemplateParams, ArrayRef Ps,
+      ArrayRef As, sema::TemplateDeductionInfo &Info,
+      SmallVectorImpl &Deduced,
+      bool NumberOfArgumentsMustMatch);
+
   TemplateDeductionResult SubstituteExplicitTemplateArguments(
       FunctionTemplateDecl *FunctionTemplate,
       TemplateArgumentListInfo &ExplicitTemplateArgs,
@@ -10378,6 +10384,9 @@ public:
     InstantiatingTemplate &operator=(const InstantiatingTemplate &) = delete;
   };
 
+  bool SubstTemplateArgument(const TemplateArgumentLoc &Input,
+                             const MultiLevelTemplateArgumentList &TemplateArgs,
+                             TemplateArgumentLoc &Output);
   bool
   SubstTemplateArguments(ArrayRef Args,
                          const MultiLevelTemplateArgumentList &TemplateArgs,
@@ -10862,9 +10871,11 @@ public:
                                   ParmVarDecl *Param);
   void InstantiateExceptionSpec(SourceLocation PointOfInstantiation,
                                 FunctionDecl *Function);
-  FunctionDecl *InstantiateFunctionDeclaration(FunctionTemplateDecl *FTD,
-                                               const TemplateArgumentList *Args,
-                                               SourceLocation Loc);
+  FunctionDecl *InstantiateFunctionDeclaration(
+      FunctionTemplateDecl *FTD, const TemplateArgumentList *Args,
+      SourceLocation Loc,
+      CodeSynthesisContext::SynthesisKind CSC =
+          CodeSynthesisContext::ExplicitTemplateArgumentSubstitution);
   void InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
                                      FunctionDecl *Function,
                                      bool Recursive = false,
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 011deed7a9a9..aa470adb30b4 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -10720,13 +10720,40 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
   if (TemplateName.isDependent())
     return SubstAutoTypeDependent(TSInfo->getType());
 
-  // We can only perform deduction for class templates.
+  // We can only perform deduction for class templates or alias templates.
   auto *Template =
       dyn_cast_or_null(TemplateName.getAsTemplateDecl());
+  TemplateDecl *LookupTemplateDecl = Template;
+  if (!Template) {
+    if (auto *AliasTemplate = dyn_cast_or_null(
+            TemplateName.getAsTemplateDecl())) {
+      Diag(Kind.getLocation(),
+           diag::warn_cxx17_compat_ctad_for_alias_templates);
+      LookupTemplateDecl = AliasTemplate;
+      auto UnderlyingType = AliasTemplate->getTemplatedDecl()
+                                ->getUnderlyingType()
+                                .getCanonicalType();
+      // C++ [over.match.class.deduct#3]: ..., the defining-type-id of A must be
+      // of the form
+      //   [typename] [nested-name-specifier] [template] simple-template-id
+      if (const auto *TST =
+              UnderlyingType->getAs()) {
+        Template = dyn_cast_or_null(
+            TST->getTemplateName().getAsTemplateDecl());
+      } else if (const auto *RT = UnderlyingType->getAs()) {
+        // Cases where template arguments in the RHS of the alias are not
+        // dependent. e.g.
+        //   using AliasFoo = Foo;
+        if (const auto *CTSD = llvm::dyn_cast(
+                RT->getAsCXXRecordDecl()))
+          Template = CTSD->getSpecializedTemplate();
+      }
+    }
+  }
   if (!Template) {
     Diag(Kind.getLocation(),
-         diag::err_deduced_non_class_template_specialization_type)
-      << (int)getTemplateNameKindForDiagnostics(TemplateName) << TemplateName;
+         diag::err_deduced_non_class_or_alias_template_specialization_type)
+        << (int)getTemplateNameKindForDiagnostics(TemplateName) << TemplateName;
     if (auto *TD = TemplateName.getAsTemplateDecl())
       NoteTemplateLocation(*TD);
     return QualType();
@@ -10753,10 +10780,10 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
   //     template-name, a function template [...]
   //  - For each deduction-guide, a function or function template [...]
   DeclarationNameInfo NameInfo(
-      Context.DeclarationNames.getCXXDeductionGuideName(Template),
+      Context.DeclarationNames.getCXXDeductionGuideName(LookupTemplateDecl),
       TSInfo->getTypeLoc().getEndLoc());
   LookupResult Guides(*this, NameInfo, LookupOrdinaryName);
-  LookupQualifiedName(Guides, Template->getDeclContext());
+  LookupQualifiedName(Guides, LookupTemplateDecl->getDeclContext());
 
   // FIXME: Do not diagnose inaccessible deduction guides. The standard isn't
   // clear on this, but they're not found by name so access does not apply.
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 83eee83aa6ca..d62095558d0f 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -2262,6 +2262,92 @@ public:
   }
 };
 
+// Build a deduction guide with the specified parameter types.
+FunctionTemplateDecl *buildDeductionGuide(
+    Sema &SemaRef, TemplateDecl *OriginalTemplate,
+    TemplateParameterList *TemplateParams, CXXConstructorDecl *Ctor,
+    ExplicitSpecifier ES, TypeSourceInfo *TInfo, SourceLocation LocStart,
+    SourceLocation Loc, SourceLocation LocEnd, bool IsImplicit,
+    llvm::ArrayRef MaterializedTypedefs = {}) {
+  DeclContext *DC = OriginalTemplate->getDeclContext();
+  auto DeductionGuideName =
+      SemaRef.Context.DeclarationNames.getCXXDeductionGuideName(
+          OriginalTemplate);
+
+  DeclarationNameInfo Name(DeductionGuideName, Loc);
+  ArrayRef Params =
+      TInfo->getTypeLoc().castAs().getParams();
+
+  // Build the implicit deduction guide template.
+  auto *Guide =
+      CXXDeductionGuideDecl::Create(SemaRef.Context, DC, LocStart, ES, Name,
+                                    TInfo->getType(), TInfo, LocEnd, Ctor);
+  Guide->setImplicit(IsImplicit);
+  Guide->setParams(Params);
+
+  for (auto *Param : Params)
+    Param->setDeclContext(Guide);
+  for (auto *TD : MaterializedTypedefs)
+    TD->setDeclContext(Guide);
+
+  auto *GuideTemplate = FunctionTemplateDecl::Create(
+      SemaRef.Context, DC, Loc, DeductionGuideName, TemplateParams, Guide);
+  GuideTemplate->setImplicit(IsImplicit);
+  Guide->setDescribedFunctionTemplate(GuideTemplate);
+
+  if (isa(DC)) {
+    Guide->setAccess(AS_public);
+    GuideTemplate->setAccess(AS_public);
+  }
+
+  DC->addDecl(GuideTemplate);
+  return GuideTemplate;
+}
+
+// Transform a given template type parameter `TTP`.
+TemplateTypeParmDecl *
+transformTemplateTypeParam(Sema &SemaRef, DeclContext *DC,
+                           TemplateTypeParmDecl *TTP,
+                           MultiLevelTemplateArgumentList &Args,
+                           unsigned NewDepth, unsigned NewIndex) {
+  // TemplateTypeParmDecl's index cannot be changed after creation, so
+  // substitute it directly.
+  auto *NewTTP = TemplateTypeParmDecl::Create(
+      SemaRef.Context, DC, TTP->getBeginLoc(), TTP->getLocation(), NewDepth,
+      NewIndex, TTP->getIdentifier(), TTP->wasDeclaredWithTypename(),
+      TTP->isParameterPack(), TTP->hasTypeConstraint(),
+      TTP->isExpandedParameterPack()
+          ? std::optional(TTP->getNumExpansionParameters())
+          : std::nullopt);
+  if (const auto *TC = TTP->getTypeConstraint())
+    SemaRef.SubstTypeConstraint(NewTTP, TC, Args,
+                                /*EvaluateConstraint=*/true);
+  if (TTP->hasDefaultArgument()) {
+    TypeSourceInfo *InstantiatedDefaultArg =
+        SemaRef.SubstType(TTP->getDefaultArgumentInfo(), Args,
+                          TTP->getDefaultArgumentLoc(), TTP->getDeclName());
+    if (InstantiatedDefaultArg)
+      NewTTP->setDefaultArgument(InstantiatedDefaultArg);
+  }
+  SemaRef.CurrentInstantiationScope->InstantiatedLocal(TTP, NewTTP);
+  return NewTTP;
+}
+// Similar to above, but for non-type template or template template parameters.
+template 
+NonTypeTemplateOrTemplateTemplateParmDecl *
+transformTemplateParam(Sema &SemaRef, DeclContext *DC,
+                       NonTypeTemplateOrTemplateTemplateParmDecl *OldParam,
+                       MultiLevelTemplateArgumentList &Args, unsigned NewIndex,
+                       unsigned NewDepth) {
+  // Ask the template instantiator to do the heavy lifting for us, then adjust
+  // the index of the parameter once it's done.
+  auto *NewParam = cast(
+      SemaRef.SubstDecl(OldParam, DC, Args));
+  NewParam->setPosition(NewIndex);
+  NewParam->setDepth(NewDepth);
+  return NewParam;
+}
+
 /// Transform to convert portions of a constructor declaration into the
 /// corresponding deduction guide, per C++1z [over.match.class.deduct]p1.
 struct ConvertConstructorToDeductionGuideTransform {
@@ -2338,7 +2424,6 @@ struct ConvertConstructorToDeductionGuideTransform {
         NamedDecl *NewParam = transformTemplateParameter(Param, Args);
         if (!NewParam)
           return nullptr;
-
         // Constraints require that we substitute depth-1 arguments
         // to match depths when substituted for evaluation later
         Depth1Args.push_back(SemaRef.Context.getCanonicalTemplateArgument(
@@ -2411,9 +2496,10 @@ struct ConvertConstructorToDeductionGuideTransform {
       return nullptr;
     TypeSourceInfo *NewTInfo = TLB.getTypeSourceInfo(SemaRef.Context, NewType);
 
-    return buildDeductionGuide(TemplateParams, CD, CD->getExplicitSpecifier(),
-                               NewTInfo, CD->getBeginLoc(), CD->getLocation(),
-                               CD->getEndLoc(), MaterializedTypedefs);
+    return buildDeductionGuide(
+        SemaRef, Template, TemplateParams, CD, CD->getExplicitSpecifier(),
+        NewTInfo, CD->getBeginLoc(), CD->getLocation(), CD->getEndLoc(),
+        /*IsImplicit=*/true, MaterializedTypedefs);
   }
 
   /// Build a deduction guide with the specified parameter types.
@@ -2448,8 +2534,9 @@ struct ConvertConstructorToDeductionGuideTransform {
       Params.push_back(NewParam);
     }
 
-    return buildDeductionGuide(GetTemplateParameterList(Template), nullptr,
-                               ExplicitSpecifier(), TSI, Loc, Loc, Loc);
+    return buildDeductionGuide(
+        SemaRef, Template, GetTemplateParameterList(Template), nullptr,
+        ExplicitSpecifier(), TSI, Loc, Loc, Loc, /*IsImplicit=*/true);
   }
 
 private:
@@ -2458,50 +2545,18 @@ private:
   /// renumbering as we go.
   NamedDecl *transformTemplateParameter(NamedDecl *TemplateParam,
                                         MultiLevelTemplateArgumentList &Args) {
-    if (auto *TTP = dyn_cast(TemplateParam)) {
-      // TemplateTypeParmDecl's index cannot be changed after creation, so
-      // substitute it directly.
-      auto *NewTTP = TemplateTypeParmDecl::Create(
-          SemaRef.Context, DC, TTP->getBeginLoc(), TTP->getLocation(),
-          TTP->getDepth() - 1, Depth1IndexAdjustment + TTP->getIndex(),
-          TTP->getIdentifier(), TTP->wasDeclaredWithTypename(),
-          TTP->isParameterPack(), TTP->hasTypeConstraint(),
-          TTP->isExpandedParameterPack()
-              ? std::optional(TTP->getNumExpansionParameters())
-              : std::nullopt);
-      if (const auto *TC = TTP->getTypeConstraint())
-        SemaRef.SubstTypeConstraint(NewTTP, TC, Args,
-                                    /*EvaluateConstraint*/ true);
-      if (TTP->hasDefaultArgument()) {
-        TypeSourceInfo *InstantiatedDefaultArg =
-            SemaRef.SubstType(TTP->getDefaultArgumentInfo(), Args,
-                              TTP->getDefaultArgumentLoc(), TTP->getDeclName());
-        if (InstantiatedDefaultArg)
-          NewTTP->setDefaultArgument(InstantiatedDefaultArg);
-      }
-      SemaRef.CurrentInstantiationScope->InstantiatedLocal(TemplateParam,
-                                                           NewTTP);
-      return NewTTP;
-    }
-
+    if (auto *TTP = dyn_cast(TemplateParam))
+      return transformTemplateTypeParam(
+          SemaRef, DC, TTP, Args, TTP->getDepth() - 1,
+          Depth1IndexAdjustment + TTP->getIndex());
     if (auto *TTP = dyn_cast(TemplateParam))
-      return transformTemplateParameterImpl(TTP, Args);
-
-    return transformTemplateParameterImpl(
-        cast(TemplateParam), Args);
-  }
-  template
-  TemplateParmDecl *
-  transformTemplateParameterImpl(TemplateParmDecl *OldParam,
-                                 MultiLevelTemplateArgumentList &Args) {
-    // Ask the template instantiator to do the heavy lifting for us, then adjust
-    // the index of the parameter once it's done.
-    auto *NewParam =
-        cast(SemaRef.SubstDecl(OldParam, DC, Args));
-    assert(NewParam->getDepth() == OldParam->getDepth() - 1 &&
-           "unexpected template param depth");
-    NewParam->setPosition(NewParam->getPosition() + Depth1IndexAdjustment);
-    return NewParam;
+      return transformTemplateParam(SemaRef, DC, TTP, Args,
+                                    Depth1IndexAdjustment + TTP->getIndex(),
+                                    TTP->getDepth() - 1);
+    auto *NTTP = cast(TemplateParam);
+    return transformTemplateParam(SemaRef, DC, NTTP, Args,
+                                  Depth1IndexAdjustment + NTTP->getIndex(),
+                                  NTTP->getDepth() - 1);
   }
 
   QualType transformFunctionProtoType(
@@ -2596,7 +2651,7 @@ private:
       // placeholder to indicate there is a default argument.
       QualType ParamTy = NewDI->getType();
       NewDefArg = new (SemaRef.Context)
-          OpaqueValueExpr(OldParam->getDefaultArgRange().getBegin(),
+          OpaqueValueExpr(OldParam->getDefaultArg()->getBeginLoc(),
                           ParamTy.getNonLValueExprType(SemaRef.Context),
                           ParamTy->isLValueReferenceType()   ? VK_LValue
                           : ParamTy->isRValueReferenceType() ? VK_XValue
@@ -2616,44 +2671,310 @@ private:
     SemaRef.CurrentInstantiationScope->InstantiatedLocal(OldParam, NewParam);
     return NewParam;
   }
+};
+
+// Find all template parameters that appear in the given DeducedArgs.
+// Return the indices of the template parameters in the TemplateParams.
+SmallVector TemplateParamsReferencedInTemplateArgumentList(
+    ArrayRef TemplateParams,
+    ArrayRef DeducedArgs) {
+  struct TemplateParamsReferencedFinder
+      : public RecursiveASTVisitor {
+    llvm::DenseSet TemplateParams;
+    llvm::DenseSet ReferencedTemplateParams;
+
+    TemplateParamsReferencedFinder(ArrayRef TemplateParams)
+        : TemplateParams(TemplateParams.begin(), TemplateParams.end()) {}
+
+    bool VisitTemplateTypeParmType(TemplateTypeParmType *TTP) {
+      TTP->getIndex();
+      MarkAppeared(TTP->getDecl());
+      return true;
+    }
+    bool VisitDeclRefExpr(DeclRefExpr *DRE) {
+      MarkAppeared(DRE->getFoundDecl());
+      return true;
+    }
+
+    void MarkAppeared(NamedDecl *ND) {
+      if (TemplateParams.contains(ND))
+        ReferencedTemplateParams.insert(ND);
+    }
+  };
+  TemplateParamsReferencedFinder Finder(TemplateParams);
+  Finder.TraverseTemplateArguments(DeducedArgs);
 
-  FunctionTemplateDecl *buildDeductionGuide(
-      TemplateParameterList *TemplateParams, CXXConstructorDecl *Ctor,
-      ExplicitSpecifier ES, TypeSourceInfo *TInfo, SourceLocation LocStart,
-      SourceLocation Loc, SourceLocation LocEnd,
-      llvm::ArrayRef MaterializedTypedefs = {}) {
-    DeclarationNameInfo Name(DeductionGuideName, Loc);
-    ArrayRef Params =
-        TInfo->getTypeLoc().castAs().getParams();
+  SmallVector Results;
+  for (unsigned Index = 0; Index < TemplateParams.size(); ++Index) {
+    if (Finder.ReferencedTemplateParams.contains(TemplateParams[Index]))
+      Results.push_back(Index);
+  }
+  return Results;
+}
 
-    // Build the implicit deduction guide template.
-    auto *Guide =
-        CXXDeductionGuideDecl::Create(SemaRef.Context, DC, LocStart, ES, Name,
-                                      TInfo->getType(), TInfo, LocEnd, Ctor);
-    Guide->setImplicit();
-    Guide->setParams(Params);
+bool hasDeclaredDeductionGuides(DeclarationName Name, DeclContext *DC) {
+  // Check whether we've already declared deduction guides for this template.
+  // FIXME: Consider storing a flag on the template to indicate this.
+  assert(Name.getNameKind() ==
+             DeclarationName::NameKind::CXXDeductionGuideName &&
+         "name must be a deduction guide name");
+  auto Existing = DC->lookup(Name);
+  for (auto *D : Existing)
+    if (D->isImplicit())
+      return true;
+  return false;
+}
 
-    for (auto *Param : Params)
-      Param->setDeclContext(Guide);
-    for (auto *TD : MaterializedTypedefs)
-      TD->setDeclContext(Guide);
+// Build deduction guides for a type alias template.
+void DeclareImplicitDeductionGuidesForTypeAlias(
+    Sema &SemaRef, TypeAliasTemplateDecl *AliasTemplate, SourceLocation Loc) {
+  auto &Context = SemaRef.Context;
+  // FIXME: if there is an explicit deduction guide after the first use of the
+  // type alias usage, we will not cover this explicit deduction guide. fix this
+  // case.
+  if (hasDeclaredDeductionGuides(
+          Context.DeclarationNames.getCXXDeductionGuideName(AliasTemplate),
+          AliasTemplate->getDeclContext()))
+    return;
+  // Unwrap the sugared ElaboratedType.
+  auto RhsType = AliasTemplate->getTemplatedDecl()
+                     ->getUnderlyingType()
+                     .getSingleStepDesugaredType(Context);
+  TemplateDecl *Template = nullptr;
+  llvm::ArrayRef AliasRhsTemplateArgs;
+  if (const auto *TST = RhsType->getAs()) {
+    // Cases where the RHS of the alias is dependent. e.g.
+    //   template
+    //   using AliasFoo1 = Foo; // a class/type alias template specialization
+    Template = TST->getTemplateName().getAsTemplateDecl();
+    AliasRhsTemplateArgs = TST->template_arguments();
+  } else if (const auto *RT = RhsType->getAs()) {
+    // Cases where template arguments in the RHS of the alias are not
+    // dependent. e.g.
+    //   using AliasFoo = Foo;
+    if (const auto *CTSD = llvm::dyn_cast(
+            RT->getAsCXXRecordDecl())) {
+      Template = CTSD->getSpecializedTemplate();
+      AliasRhsTemplateArgs = CTSD->getTemplateArgs().asArray();
+    }
+  } else {
+    assert(false && "unhandled RHS type of the alias");
+  }
+  if (!Template)
+    return;
+  DeclarationNameInfo NameInfo(
+      Context.DeclarationNames.getCXXDeductionGuideName(Template), Loc);
+  LookupResult Guides(SemaRef, NameInfo, clang::Sema::LookupOrdinaryName);
+  SemaRef.LookupQualifiedName(Guides, Template->getDeclContext());
+  Guides.suppressDiagnostics();
+
+  for (auto *G : Guides) {
+    FunctionTemplateDecl *F = dyn_cast(G);
+    if (!F)
+      continue;
+    auto RType = F->getTemplatedDecl()->getReturnType();
+    // The (trailing) return type of the deduction guide.
+    const TemplateSpecializationType *FReturnType =
+        RType->getAs();
+    if (const auto *InjectedCNT = RType->getAs())
+      // implicitly-generated deduction guide.
+      FReturnType = InjectedCNT->getInjectedTST();
+    else if (const auto *ET = RType->getAs())
+      // explicit deduction guide.
+      FReturnType = ET->getNamedType()->getAs();
+    assert(FReturnType && "expected to see a return type");
+    // Deduce template arguments of the deduction guide f from the RHS of
+    // the alias.
+    //
+    // C++ [over.match.class.deduct]p3: ...For each function or function
+    // template f in the guides of the template named by the
+    // simple-template-id of the defining-type-id, the template arguments
+    // of the return type of f are deduced from the defining-type-id of A
+    // according to the process in [temp.deduct.type] with the exception
+    // that deduction does not fail if not all template arguments are
+    // deduced.
+    //
+    //
+    //  template
+    //  f(X, Y) -> f;
+    //
+    //  template
+    //  using alias = f;
+    //
+    // The RHS of alias is f, we deduced the template arguments of
+    // the return type of the deduction guide from it: Y->int, X->U
+    sema::TemplateDeductionInfo TDeduceInfo(Loc);
+    // Must initialize n elements, this is required by DeduceTemplateArguments.
+    SmallVector DeduceResults(
+        F->getTemplateParameters()->size());
+
+    // FIXME: DeduceTemplateArguments stops immediately at the first
+    // non-deducible template argument. However, this doesn't seem to casue
+    // issues for practice cases, we probably need to extend it to continue
+    // performing deduction for rest of arguments to align with the C++
+    // standard.
+    SemaRef.DeduceTemplateArguments(
+        F->getTemplateParameters(), FReturnType->template_arguments(),
+        AliasRhsTemplateArgs, TDeduceInfo, DeduceResults,
+        /*NumberOfArgumentsMustMatch=*/false);
+
+    SmallVector DeducedArgs;
+    SmallVector NonDeducedTemplateParamsInFIndex;
+    // !!NOTE: DeduceResults respects the sequence of template parameters of
+    // the deduction guide f.
+    for (unsigned Index = 0; Index < DeduceResults.size(); ++Index) {
+      if (const auto &D = DeduceResults[Index]; !D.isNull()) // Deduced
+        DeducedArgs.push_back(D);
+      else
+        NonDeducedTemplateParamsInFIndex.push_back(Index);
+    }
+    auto DeducedAliasTemplateParams =
+        TemplateParamsReferencedInTemplateArgumentList(
+            AliasTemplate->getTemplateParameters()->asArray(), DeducedArgs);
+    // All template arguments null by default.
+    SmallVector TemplateArgsForBuildingFPrime(
+        F->getTemplateParameters()->size());
+
+    Sema::InstantiatingTemplate BuildingDeductionGuides(
+        SemaRef, AliasTemplate->getLocation(), F,
+        Sema::InstantiatingTemplate::BuildingDeductionGuidesTag{});
+    if (BuildingDeductionGuides.isInvalid())
+      return;
+    LocalInstantiationScope Scope(SemaRef);
 
-    auto *GuideTemplate = FunctionTemplateDecl::Create(
-        SemaRef.Context, DC, Loc, DeductionGuideName, TemplateParams, Guide);
-    GuideTemplate->setImplicit();
-    Guide->setDescribedFunctionTemplate(GuideTemplate);
+    // Create a template parameter list for the synthesized deduction guide f'.
+    //
+    // C++ [over.match.class.deduct]p3.2:
+    //   If f is a function template, f' is a function template whose template
+    //   parameter list consists of all the template parameters of A
+    //   (including their default template arguments) that appear in the above
+    //   deductions or (recursively) in their default template arguments
+    SmallVector FPrimeTemplateParams;
+    // Store template arguments that refer to the newly-created template
+    // parameters, used for building `TemplateArgsForBuildingFPrime`.
+    SmallVector TransformedDeducedAliasArgs(
+        AliasTemplate->getTemplateParameters()->size());
+    auto TransformTemplateParameter =
+        [&SemaRef](DeclContext *DC, NamedDecl *TemplateParam,
+                   MultiLevelTemplateArgumentList &Args,
+                   unsigned NewIndex) -> NamedDecl * {
+      if (auto *TTP = dyn_cast(TemplateParam))
+        return transformTemplateTypeParam(SemaRef, DC, TTP, Args,
+                                          TTP->getDepth(), NewIndex);
+      if (auto *TTP = dyn_cast(TemplateParam))
+        return transformTemplateParam(SemaRef, DC, TTP, Args, NewIndex,
+                                      TTP->getDepth());
+      if (auto *NTTP = dyn_cast(TemplateParam))
+        return transformTemplateParam(SemaRef, DC, NTTP, Args, NewIndex,
+                                      NTTP->getDepth());
+      return nullptr;
+    };
 
-    if (isa(DC)) {
-      Guide->setAccess(AS_public);
-      GuideTemplate->setAccess(AS_public);
+    for (unsigned AliasTemplateParamIdx : DeducedAliasTemplateParams) {
+      auto *TP = AliasTemplate->getTemplateParameters()->getParam(
+          AliasTemplateParamIdx);
+      // Rebuild any internal references to earlier parameters and reindex as
+      // we go.
+      MultiLevelTemplateArgumentList Args;
+      Args.setKind(TemplateSubstitutionKind::Rewrite);
+      Args.addOuterTemplateArguments(TransformedDeducedAliasArgs);
+      NamedDecl *NewParam =
+          TransformTemplateParameter(AliasTemplate->getDeclContext(), TP, Args,
+                                     /*NewIndex*/ FPrimeTemplateParams.size());
+      FPrimeTemplateParams.push_back(NewParam);
+
+      auto NewTemplateArgument = Context.getCanonicalTemplateArgument(
+          Context.getInjectedTemplateArg(NewParam));
+      TransformedDeducedAliasArgs[AliasTemplateParamIdx] = NewTemplateArgument;
+    }
+    //   ...followed by the template parameters of f that were not deduced
+    //   (including their default template arguments)
+    for (unsigned FTemplateParamIdx : NonDeducedTemplateParamsInFIndex) {
+      auto *TP = F->getTemplateParameters()->getParam(FTemplateParamIdx);
+      MultiLevelTemplateArgumentList Args;
+      Args.setKind(TemplateSubstitutionKind::Rewrite);
+      // We take a shortcut here, it is ok to reuse the
+      // TemplateArgsForBuildingFPrime.
+      Args.addOuterTemplateArguments(TemplateArgsForBuildingFPrime);
+      NamedDecl *NewParam = TransformTemplateParameter(
+          F->getDeclContext(), TP, Args, FPrimeTemplateParams.size());
+      FPrimeTemplateParams.push_back(NewParam);
+
+      assert(TemplateArgsForBuildingFPrime[FTemplateParamIdx].isNull() &&
+             "The argument must be null before setting");
+      TemplateArgsForBuildingFPrime[FTemplateParamIdx] =
+          Context.getCanonicalTemplateArgument(
+              Context.getInjectedTemplateArg(NewParam));
+    }
+    // FIXME: implement the associated constraint per C++
+    // [over.match.class.deduct]p3.3:
+    //    The associated constraints ([temp.constr.decl]) are the
+    //    conjunction of the associated constraints of g and a
+    //    constraint that is satisfied if and only if the arguments
+    //    of A are deducible (see below) from the return type.
+    auto *FPrimeTemplateParamList = TemplateParameterList::Create(
+        Context, AliasTemplate->getTemplateParameters()->getTemplateLoc(),
+        AliasTemplate->getTemplateParameters()->getLAngleLoc(),
+        FPrimeTemplateParams,
+        AliasTemplate->getTemplateParameters()->getRAngleLoc(),
+        /*RequiresClause=*/nullptr);
+
+    // To form a deduction guide f' from f, we leverage clang's instantiation
+    // mechanism, we construct a template argument list where the template
+    // arguments refer to the newly-created template parameters of f', and
+    // then apply instantiation on this template argument list to instantiate
+    // f, this ensures all template parameter occurrences are updated
+    // correctly.
+    //
+    // The template argument list is formed from the `DeducedArgs`, two parts:
+    //  1) appeared template parameters of alias: transfrom the deduced
+    //  template argument;
+    //  2) non-deduced template parameters of f: rebuild a
+    //  template argument;
+    //
+    // 2) has been built already (when rebuilding the new template
+    // parameters), we now perform 1).
+    MultiLevelTemplateArgumentList Args;
+    Args.setKind(TemplateSubstitutionKind::Rewrite);
+    Args.addOuterTemplateArguments(TransformedDeducedAliasArgs);
+    for (unsigned Index = 0; Index < DeduceResults.size(); ++Index) {
+      const auto &D = DeduceResults[Index];
+      if (D.isNull()) {
+        // 2): Non-deduced template parameter has been built already.
+        assert(!TemplateArgsForBuildingFPrime[Index].isNull() &&
+               "template arguments for non-deduced template parameters should "
+               "be been set!");
+        continue;
+      }
+      TemplateArgumentLoc Input = SemaRef.getTrivialTemplateArgumentLoc(
+          D, QualType(), SourceLocation{});
+      TemplateArgumentLoc Output;
+      if (!SemaRef.SubstTemplateArgument(Input, Args, Output)) {
+        assert(TemplateArgsForBuildingFPrime[Index].isNull() &&
+               "InstantiatedArgs must be null before setting");
+        TemplateArgsForBuildingFPrime[Index] = (Output.getArgument());
+      }
     }
 
-    DC->addDecl(GuideTemplate);
-    return GuideTemplate;
+    auto *TemplateArgListForBuildingFPrime = TemplateArgumentList::CreateCopy(
+        Context, TemplateArgsForBuildingFPrime);
+    // Form the f' by substituting the template arguments into f.
+    if (auto *FPrime = SemaRef.InstantiateFunctionDeclaration(
+            F, TemplateArgListForBuildingFPrime, AliasTemplate->getLocation(),
+            Sema::CodeSynthesisContext::BuildingDeductionGuides)) {
+      auto *GG = dyn_cast(FPrime);
+      buildDeductionGuide(SemaRef, AliasTemplate, FPrimeTemplateParamList,
+                          GG->getCorrespondingConstructor(),
+                          GG->getExplicitSpecifier(), GG->getTypeSourceInfo(),
+                          AliasTemplate->getBeginLoc(),
+                          AliasTemplate->getLocation(),
+                          AliasTemplate->getEndLoc(), F->isImplicit());
+    }
   }
-};
 }
 
+} // namespace
+
 FunctionTemplateDecl *Sema::DeclareImplicitDeductionGuideFromInitList(
     TemplateDecl *Template, MutableArrayRef ParamTypes,
     SourceLocation Loc) {
@@ -2697,6 +3018,10 @@ FunctionTemplateDecl *Sema::DeclareImplicitDeductionGuideFromInitList(
 
 void Sema::DeclareImplicitDeductionGuides(TemplateDecl *Template,
                                           SourceLocation Loc) {
+  if (auto *AliasTemplate = llvm::dyn_cast(Template)) {
+    DeclareImplicitDeductionGuidesForTypeAlias(*this, AliasTemplate, Loc);
+    return;
+  }
   if (CXXRecordDecl *DefRecord =
           cast(Template->getTemplatedDecl())->getDefinition()) {
     if (TemplateDecl *DescribedTemplate = DefRecord->getDescribedClassTemplate())
@@ -2712,12 +3037,8 @@ void Sema::DeclareImplicitDeductionGuides(TemplateDecl *Template,
   if (!isCompleteType(Loc, Transform.DeducedType))
     return;
 
-  // Check whether we've already declared deduction guides for this template.
-  // FIXME: Consider storing a flag on the template to indicate this.
-  auto Existing = DC->lookup(Transform.DeductionGuideName);
-  for (auto *D : Existing)
-    if (D->isImplicit())
-      return;
+  if (hasDeclaredDeductionGuides(Transform.DeductionGuideName, DC))
+    return;
 
   // In case we were expanding a pack when we attempted to declare deduction
   // guides, turn off pack expansion for everything we're about to do.
@@ -5368,6 +5689,15 @@ bool Sema::CheckTemplateTypeArgument(
     [[fallthrough]];
   }
   default: {
+    // We allow instantiateing a template with template argument packs when
+    // building deduction guides.
+    if (Arg.getKind() == TemplateArgument::Pack &&
+        CodeSynthesisContexts.back().Kind ==
+            Sema::CodeSynthesisContext::BuildingDeductionGuides) {
+      SugaredConverted.push_back(Arg);
+      CanonicalConverted.push_back(Arg);
+      return false;
+    }
     // We have a template type parameter but the template argument
     // is not a type.
     SourceRange SR = AL.getSourceRange();
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 65f7fa15b20d..97f8445bf819 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -2531,6 +2531,15 @@ DeduceTemplateArguments(Sema &S, TemplateParameterList *TemplateParams,
   return TemplateDeductionResult::Success;
 }
 
+TemplateDeductionResult Sema::DeduceTemplateArguments(
+    TemplateParameterList *TemplateParams, ArrayRef Ps,
+    ArrayRef As, sema::TemplateDeductionInfo &Info,
+    SmallVectorImpl &Deduced,
+    bool NumberOfArgumentsMustMatch) {
+  return ::DeduceTemplateArguments(*this, TemplateParams, Ps, As, Info, Deduced,
+                                   NumberOfArgumentsMustMatch);
+}
+
 /// Determine whether two template arguments are the same.
 static bool isSameTemplateArg(ASTContext &Context,
                               TemplateArgument X,
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index 371378485626..d9994d7fd37a 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -21,6 +21,7 @@
 #include "clang/AST/ExprConcepts.h"
 #include "clang/AST/PrettyDeclStackTrace.h"
 #include "clang/AST/Type.h"
+#include "clang/AST/TypeLoc.h"
 #include "clang/AST/TypeVisitor.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/Stack.h"
@@ -547,9 +548,9 @@ Sema::InstantiatingTemplate::InstantiatingTemplate(
     : InstantiatingTemplate(SemaRef, Kind, PointOfInstantiation,
                             InstantiationRange, FunctionTemplate, nullptr,
                             TemplateArgs, &DeductionInfo) {
-  assert(
-    Kind == CodeSynthesisContext::ExplicitTemplateArgumentSubstitution ||
-    Kind == CodeSynthesisContext::DeducedTemplateArgumentSubstitution);
+  assert(Kind == CodeSynthesisContext::ExplicitTemplateArgumentSubstitution ||
+         Kind == CodeSynthesisContext::DeducedTemplateArgumentSubstitution ||
+         Kind == CodeSynthesisContext::BuildingDeductionGuides);
 }
 
 Sema::InstantiatingTemplate::InstantiatingTemplate(
@@ -1446,6 +1447,59 @@ namespace {
       return inherited::TransformFunctionProtoType(TLB, TL);
     }
 
+    QualType TransformInjectedClassNameType(TypeLocBuilder &TLB,
+                                            InjectedClassNameTypeLoc TL) {
+      auto Type = inherited::TransformInjectedClassNameType(TLB, TL);
+      // Special case for transforming a deduction guide, we return a
+      // transformed TemplateSpecializationType.
+      if (Type.isNull() &&
+          SemaRef.CodeSynthesisContexts.back().Kind ==
+              Sema::CodeSynthesisContext::BuildingDeductionGuides) {
+        // Return a TemplateSpecializationType for transforming a deduction
+        // guide.
+        if (auto *ICT = TL.getType()->getAs()) {
+          auto Type =
+              inherited::TransformType(ICT->getInjectedSpecializationType());
+          TLB.pushTrivial(SemaRef.Context, Type, TL.getNameLoc());
+          return Type;
+        }
+      }
+      return Type;
+    }
+    // Override the default version to handle a rewrite-template-arg-pack case
+    // for building a deduction guide.
+    bool TransformTemplateArgument(const TemplateArgumentLoc &Input,
+                                   TemplateArgumentLoc &Output,
+                                   bool Uneval = false) {
+      const TemplateArgument &Arg = Input.getArgument();
+      std::vector TArgs;
+      switch (Arg.getKind()) {
+      case TemplateArgument::Pack:
+        // Literally rewrite the template argument pack, instead of unpacking
+        // it.
+        assert(
+            SemaRef.CodeSynthesisContexts.back().Kind ==
+                Sema::CodeSynthesisContext::BuildingDeductionGuides &&
+            "Transforming a template argument pack is only allowed in building "
+            "deduction guide");
+        for (auto &pack : Arg.getPackAsArray()) {
+          TemplateArgumentLoc Input = SemaRef.getTrivialTemplateArgumentLoc(
+              pack, QualType(), SourceLocation{});
+          TemplateArgumentLoc Output;
+          if (SemaRef.SubstTemplateArgument(Input, TemplateArgs, Output))
+            return true; // fails
+          TArgs.push_back(Output.getArgument());
+        }
+        Output = SemaRef.getTrivialTemplateArgumentLoc(
+            TemplateArgument(llvm::ArrayRef(TArgs).copy(SemaRef.Context)),
+            QualType(), SourceLocation{});
+        return false;
+      default:
+        break;
+      }
+      return inherited::TransformTemplateArgument(Input, Output, Uneval);
+    }
+
     template
     QualType TransformFunctionProtoType(TypeLocBuilder &TLB,
                                         FunctionProtoTypeLoc TL,
@@ -4138,6 +4192,15 @@ Sema::SubstStmt(Stmt *S, const MultiLevelTemplateArgumentList &TemplateArgs) {
   return Instantiator.TransformStmt(S);
 }
 
+bool Sema::SubstTemplateArgument(
+    const TemplateArgumentLoc &Input,
+    const MultiLevelTemplateArgumentList &TemplateArgs,
+    TemplateArgumentLoc &Output) {
+  TemplateInstantiator Instantiator(*this, TemplateArgs, SourceLocation(),
+                                    DeclarationName());
+  return Instantiator.TransformTemplateArgument(Input, Output);
+}
+
 bool Sema::SubstTemplateArguments(
     ArrayRef Args,
     const MultiLevelTemplateArgumentList &TemplateArgs,
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 9c696e072ba4..20c2c93ac9c7 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -2219,7 +2219,9 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(
       FunctionTemplate->setInstantiatedFromMemberTemplate(
                                            D->getDescribedFunctionTemplate());
     }
-  } else if (FunctionTemplate) {
+  } else if (FunctionTemplate &&
+             SemaRef.CodeSynthesisContexts.back().Kind !=
+                 Sema::CodeSynthesisContext::BuildingDeductionGuides) {
     // Record this function template specialization.
     ArrayRef Innermost = TemplateArgs.getInnermost();
     Function->setFunctionTemplateSpecialization(FunctionTemplate,
@@ -4853,16 +4855,13 @@ bool TemplateDeclInstantiator::SubstDefaultedFunction(FunctionDecl *New,
 ///
 /// Usually this should not be used, and template argument deduction should be
 /// used in its place.
-FunctionDecl *
-Sema::InstantiateFunctionDeclaration(FunctionTemplateDecl *FTD,
-                                     const TemplateArgumentList *Args,
-                                     SourceLocation Loc) {
+FunctionDecl *Sema::InstantiateFunctionDeclaration(
+    FunctionTemplateDecl *FTD, const TemplateArgumentList *Args,
+    SourceLocation Loc, CodeSynthesisContext::SynthesisKind CSC) {
   FunctionDecl *FD = FTD->getTemplatedDecl();
 
   sema::TemplateDeductionInfo Info(Loc);
-  InstantiatingTemplate Inst(
-      *this, Loc, FTD, Args->asArray(),
-      CodeSynthesisContext::ExplicitTemplateArgumentSubstitution, Info);
+  InstantiatingTemplate Inst(*this, Loc, FTD, Args->asArray(), CSC, Info);
   if (Inst.isInvalid())
     return nullptr;
 
@@ -6286,8 +6285,18 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
           QualType T = CheckTemplateIdType(TemplateName(TD), Loc, Args);
           if (T.isNull())
             return nullptr;
-          auto *SubstRecord = T->getAsCXXRecordDecl();
-          assert(SubstRecord && "class template id not a class type?");
+          CXXRecordDecl *SubstRecord = T->getAsCXXRecordDecl();
+
+          if (!SubstRecord) {
+            // T can be a dependent TemplateSpecializationType when performing a
+            // substitution for building a deduction guide.
+            assert(CodeSynthesisContexts.back().Kind ==
+                   CodeSynthesisContext::BuildingDeductionGuides);
+            // Return a nullptr as a sentinel value, we handle it properly in
+            // the TemplateInstantiator::TransformInjectedClassNameType
+            // override, which we transform it to a TemplateSpecializationType.
+            return nullptr;
+          }
           // Check that this template-id names the primary template and not a
           // partial or explicit specialization. (In the latter cases, it's
           // meaningless to attempt to find an instantiation of D within the
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 409aee73d960..2d22692f3ab7 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -4785,6 +4785,14 @@ bool TreeTransform::TransformTemplateArguments(
     TemplateArgumentLoc In = *First;
 
     if (In.getArgument().getKind() == TemplateArgument::Pack) {
+      // When building the deduction guides, we rewrite the argument packs
+      // instead of unpacking.
+      if (getSema().CodeSynthesisContexts.back().Kind ==
+          Sema::CodeSynthesisContext::BuildingDeductionGuides) {
+        if (getDerived().TransformTemplateArgument(In, Out, Uneval))
+          return true;
+        continue;
+      }
       // Unpack argument packs, which we translate them into separate
       // arguments.
       // FIXME: We could do much better if we could guarantee that the
diff --git a/clang/test/SemaCXX/cxx17-compat.cpp b/clang/test/SemaCXX/cxx17-compat.cpp
index d53d80f0d42c..54ea3384022d 100644
--- a/clang/test/SemaCXX/cxx17-compat.cpp
+++ b/clang/test/SemaCXX/cxx17-compat.cpp
@@ -131,3 +131,14 @@ namespace NTTP {
   // expected-warning@-4 {{non-type template parameter of type 'A' is incompatible with C++ standards before C++20}}
 #endif
 }
+
+namespace CTADForAliasTemplate {
+template struct A { A(T); };
+template using B = A;
+B b = {1};
+#if __cplusplus <= 201703L
+  // FIXME: diagnose as well
+#else
+  // expected-warning@-4 {{class template argument deduction for alias templates is incompatible with C++ standards before C++20}}
+#endif
+}
diff --git a/clang/test/SemaCXX/cxx1z-class-template-argument-deduction.cpp b/clang/test/SemaCXX/cxx1z-class-template-argument-deduction.cpp
index 33ed4295c2e4..2f067ea53a50 100644
--- a/clang/test/SemaCXX/cxx1z-class-template-argument-deduction.cpp
+++ b/clang/test/SemaCXX/cxx1z-class-template-argument-deduction.cpp
@@ -101,13 +101,13 @@ namespace dependent {
   struct B {
     template struct X { X(T); };
     X(int) -> X;
-    template using Y = X; // expected-note {{template}}
+    template using Y = X;
   };
   template void f() {
     typename T::X tx = 0;
-    typename T::Y ty = 0; // expected-error {{alias template 'Y' requires template arguments; argument deduction only allowed for class templates}}
+    typename T::Y ty = 0;
   }
-  template void f(); // expected-note {{in instantiation of}}
+  template void f();
 
   template struct C { C(T); };
   template C(T) -> C;
diff --git a/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
new file mode 100644
index 000000000000..794496ed4184
--- /dev/null
+++ b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
@@ -0,0 +1,232 @@
+// RUN: %clang_cc1 -fsyntax-only -Wno-c++11-narrowing -Wno-literal-conversion -std=c++20 -verify %s
+
+namespace test1 {
+template 
+struct Foo { T t; };
+template 
+using Bar = Foo;
+
+Bar s = {1};
+}  // namespace test1
+
+namespace test2 {
+template 
+struct XYpair {
+  X x;
+  Y y;
+};
+// A tricky explicit deduction guide that swapping X and Y.
+template 
+XYpair(X, Y) -> XYpair;
+template 
+using AliasXYpair = XYpair;
+
+AliasXYpair xy = {1.1, 2};  // XYpair
+static_assert(__is_same(decltype(xy.x), int));
+static_assert(__is_same(decltype(xy.y), double));
+}  // namespace test2
+
+namespace test3 {
+template 
+struct container {
+  // test with default arguments.
+  container(T a, T b = T());
+};
+
+template 
+using vector = container;
+vector v(0, 0);
+}  // namespace test3
+
+namespace test4 {
+// Explicit deduction guide.
+template 
+struct X {
+  T t;
+  X(T);
+};
+
+template 
+X(T) -> X;
+
+template 
+using AX = X;
+
+AX s = {1};
+static_assert(__is_same(decltype(s.t), double)); // explicit one is picked.
+}  // namespace test4
+
+namespace test5 {
+template 
+struct Foo {};
+// Template parameter pack
+template 
+using AF = Foo<1>;
+auto a = AF{};
+}  // namespace test5
+
+namespace test6 {
+// non-type template argument.
+template 
+struct Foo {
+  Foo(T);
+};
+template 
+using AF = Foo;
+
+AF b{0}; 
+}  // namespace test6
+
+namespace test7 {
+template 
+struct Foo {
+  Foo(T);
+};
+// using alias chain.
+template 
+using AF1 = Foo;
+template 
+using AF2 = AF1;  
+AF2 b = 1;  
+}  // namespace test7
+
+namespace test8 {
+template 
+struct Foo {
+  Foo(T const (&)[N]);
+};
+
+template 
+using Bar = Foo;
+
+Bar s = {{1}};
+}  // namespace test8
+
+namespace test9 {
+template 
+struct Foo {
+  Foo(T const (&)[N]);
+};
+
+template 
+using Bar = Foo;
+
+// FIXME: we should reject this case? GCC rejects it, MSVC accepts it.
+Bar s = {{1}};
+}  // namespace test9
+
+namespace test10 {
+template 
+struct Foo {
+  template 
+  Foo(U);
+};
+
+template 
+Foo(U) -> Foo;
+
+template 
+using A = Foo;
+A a(2);  // Foo
+}  // namespace test10
+
+namespace test11 {
+struct A {};
+template struct Foo { T c; };
+template using AFoo = Foo;
+
+AFoo s = {1};
+} // namespace test11
+
+namespace test12 {
+// no crash on null access attribute
+template
+struct Foo {
+  template
+  struct Bar { 
+    Bar(K);
+  };
+
+  template
+  using ABar = Bar;
+  void test() { ABar k = 2; }
+};
+
+void func(Foo s) {
+  s.test();
+}
+} // namespace test12
+
+namespace test13 {
+template 
+struct Foo {
+  Foo(Ts...);
+};
+
+template 
+using AFoo = Foo;
+
+auto b = AFoo{};
+} // namespace test13
+
+namespace test14 {
+template
+concept IsInt = __is_same(decltype(T()), int);
+
+template
+struct Foo {
+  Foo(T const (&)[N]);
+};
+
+template 
+using Bar = Foo; // expected-note {{constraints not satisfied for class template 'Foo'}}
+// expected-note@-1 {{candidate template ignored: could not match}}
+double abc[3];
+Bar s2 = {abc}; // expected-error {{no viable constructor or deduction guide for deduction }}
+} // namespace test14
+
+namespace test15 {
+template  struct Foo { Foo(T); };
+
+template using AFoo = Foo;
+template concept False = false;
+template using BFoo = AFoo;
+int i = 0;
+AFoo a1(&i); // OK, deduce Foo
+
+// FIXME: we should reject this case as the W is not deduced from the deduced
+// type Foo.
+BFoo b2(&i); 
+} // namespace test15
+
+namespace test16 {
+struct X { X(int); X(const X&); };
+template
+struct Foo {
+  T t;
+  Foo(T t) : t(t) {}
+};
+template
+using AFoo = Foo;
+int i = 0;
+AFoo s{i};
+static_assert(__is_same(decltype(s.t), int));
+
+// explicit deduction guide.
+Foo(int) -> Foo;
+AFoo s2{i};
+// FIXME: the type should be X because of the above explicit deduction guide.
+static_assert(__is_same(decltype(s2.t), int));
+} // namespace test16
+
+namespace test17 {
+template 
+struct Foo { T t; };
+
+// CTAD for alias templates only works for the RHS of the alias of form of
+//  [typename] [nested-name-specifier] [template] simple-template-id
+template 
+using AFoo = Foo*; // expected-note {{template is declared here}}
+
+AFoo s = {1}; // expected-error {{alias template 'AFoo' requires template arguments; argument deduction only allowed for}}
+} // namespace test17
diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html
index fa00e7685610..a3090adb5d47 100755
--- a/clang/www/cxx_status.html
+++ b/clang/www/cxx_status.html
@@ -836,7 +836,13 @@ C++23, informally referred to as C++26.

     
       Class template argument deduction for alias templates
       P1814R0
-      No
+      
+        

+          Clang 19 (Partial)
+          The associated constraints (over.match.class.deduct#3.3) for the
+          synthesized deduction guides are not yet implemented.
+        

+      
     
     
       Permit conversions to arrays of unknown bound
-- 
GitLab


From d2dff43170858459e03f9c8c764bf472ae7a59fa Mon Sep 17 00:00:00 2001
From: paperchalice 
Date: Fri, 8 Mar 2024 21:26:10 +0800
Subject: [PATCH 609/929] [CodeGen] Skip declaration in
 ModuleToMachineFunctionPassAdaptor (#84417)

Like ModuleToFunctionPassAdaptor.

---------

Co-authored-by: Matt Arsenault 
---
 llvm/lib/CodeGen/MachinePassManager.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/CodeGen/MachinePassManager.cpp b/llvm/lib/CodeGen/MachinePassManager.cpp
index 9a750b5bed43..2763193b2c30 100644
--- a/llvm/lib/CodeGen/MachinePassManager.cpp
+++ b/llvm/lib/CodeGen/MachinePassManager.cpp
@@ -79,7 +79,7 @@ ModuleToMachineFunctionPassAdaptor::run(Module &M, ModuleAnalysisManager &AM) {
   for (Function &F : M) {
     // Do not codegen any 'available_externally' functions at all, they have
     // definitions outside the translation unit.
-    if (F.hasAvailableExternallyLinkage())
+    if (F.isDeclaration() || F.hasAvailableExternallyLinkage())
       continue;
 
     MachineFunction &MF = MMI.getOrCreateMachineFunction(F);
-- 
GitLab


From 7c21495fee927359d642793d2eeb24696a301262 Mon Sep 17 00:00:00 2001
From: AtariDreams <83477269+AtariDreams@users.noreply.github.com>
Date: Fri, 8 Mar 2024 08:35:00 -0500
Subject: [PATCH 610/929] Reapply "Convert many LivePhysRegs uses to
 LiveRegUnits" (#84338)

This only converts the instances where all that is needed is to change
the variable type name.

Basically, anything that involves a function that LiveRegUnits does not
directly have was skipped to play it safe.

Reverts
https://github.com/llvm/llvm-project/commit/7a0e222a17058a311b69153d0b6f1b4459414778
---
 llvm/lib/CodeGen/ReachingDefAnalysis.cpp      | 24 +++++++++----------
 .../Target/AMDGPU/SIOptimizeExecMasking.cpp   |  8 +++----
 llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp |  6 ++---
 llvm/lib/Target/Hexagon/HexagonGenMux.cpp     | 12 +++-------
 .../lib/Target/SystemZ/SystemZElimCompare.cpp |  6 ++---
 llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp  |  6 ++---
 .../lib/Target/SystemZ/SystemZShortenInst.cpp |  8 +++----
 llvm/lib/Target/X86/X86FloatingPoint.cpp      |  8 +++----
 8 files changed, 36 insertions(+), 42 deletions(-)

diff --git a/llvm/lib/CodeGen/ReachingDefAnalysis.cpp b/llvm/lib/CodeGen/ReachingDefAnalysis.cpp
index 61a668907be7..07fa92889d88 100644
--- a/llvm/lib/CodeGen/ReachingDefAnalysis.cpp
+++ b/llvm/lib/CodeGen/ReachingDefAnalysis.cpp
@@ -6,10 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/ADT/SmallSet.h"
-#include "llvm/ADT/SetOperations.h"
-#include "llvm/CodeGen/LivePhysRegs.h"
 #include "llvm/CodeGen/ReachingDefAnalysis.h"
+#include "llvm/ADT/SetOperations.h"
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/CodeGen/LiveRegUnits.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/Support/Debug.h"
@@ -421,9 +421,9 @@ void ReachingDefAnalysis::getLiveOuts(MachineBasicBlock *MBB,
     return;
 
   VisitedBBs.insert(MBB);
-  LivePhysRegs LiveRegs(*TRI);
+  LiveRegUnits LiveRegs(*TRI);
   LiveRegs.addLiveOuts(*MBB);
-  if (LiveRegs.available(MBB->getParent()->getRegInfo(), PhysReg))
+  if (LiveRegs.available(PhysReg))
     return;
 
   if (auto *Def = getLocalLiveOutMIDef(MBB, PhysReg))
@@ -469,11 +469,11 @@ MachineInstr *ReachingDefAnalysis::getMIOperand(MachineInstr *MI,
 bool ReachingDefAnalysis::isRegUsedAfter(MachineInstr *MI,
                                          MCRegister PhysReg) const {
   MachineBasicBlock *MBB = MI->getParent();
-  LivePhysRegs LiveRegs(*TRI);
+  LiveRegUnits LiveRegs(*TRI);
   LiveRegs.addLiveOuts(*MBB);
 
   // Yes if the register is live out of the basic block.
-  if (!LiveRegs.available(MBB->getParent()->getRegInfo(), PhysReg))
+  if (!LiveRegs.available(PhysReg))
     return true;
 
   // Walk backwards through the block to see if the register is live at some
@@ -481,7 +481,7 @@ bool ReachingDefAnalysis::isRegUsedAfter(MachineInstr *MI,
   for (MachineInstr &Last :
        instructionsWithoutDebug(MBB->instr_rbegin(), MBB->instr_rend())) {
     LiveRegs.stepBackward(Last);
-    if (!LiveRegs.available(MBB->getParent()->getRegInfo(), PhysReg))
+    if (!LiveRegs.available(PhysReg))
       return InstIds.lookup(&Last) > InstIds.lookup(MI);
   }
   return false;
@@ -504,9 +504,9 @@ bool ReachingDefAnalysis::isRegDefinedAfter(MachineInstr *MI,
 bool ReachingDefAnalysis::isReachingDefLiveOut(MachineInstr *MI,
                                                MCRegister PhysReg) const {
   MachineBasicBlock *MBB = MI->getParent();
-  LivePhysRegs LiveRegs(*TRI);
+  LiveRegUnits LiveRegs(*TRI);
   LiveRegs.addLiveOuts(*MBB);
-  if (LiveRegs.available(MBB->getParent()->getRegInfo(), PhysReg))
+  if (LiveRegs.available(PhysReg))
     return false;
 
   auto Last = MBB->getLastNonDebugInstr();
@@ -525,9 +525,9 @@ bool ReachingDefAnalysis::isReachingDefLiveOut(MachineInstr *MI,
 MachineInstr *
 ReachingDefAnalysis::getLocalLiveOutMIDef(MachineBasicBlock *MBB,
                                           MCRegister PhysReg) const {
-  LivePhysRegs LiveRegs(*TRI);
+  LiveRegUnits LiveRegs(*TRI);
   LiveRegs.addLiveOuts(*MBB);
-  if (LiveRegs.available(MBB->getParent()->getRegInfo(), PhysReg))
+  if (LiveRegs.available(PhysReg))
     return nullptr;
 
   auto Last = MBB->getLastNonDebugInstr();
diff --git a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
index e3f54d01eb22..3c60459e54e8 100644
--- a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
+++ b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
@@ -11,7 +11,7 @@
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
 #include "SIRegisterInfo.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/CodeGen/LivePhysRegs.h"
+#include "llvm/CodeGen/LiveRegUnits.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
@@ -313,7 +313,7 @@ MachineBasicBlock::reverse_iterator SIOptimizeExecMasking::findExecCopy(
   return E;
 }
 
-// XXX - Seems LivePhysRegs doesn't work correctly since it will incorrectly
+// XXX - Seems LiveRegUnits doesn't work correctly since it will incorrectly
 // report the register as unavailable because a super-register with a lane mask
 // is unavailable.
 static bool isLiveOut(const MachineBasicBlock &MBB, unsigned Reg) {
@@ -383,7 +383,7 @@ bool SIOptimizeExecMasking::isRegisterInUseBetween(MachineInstr &Stop,
                                                    MCRegister Reg,
                                                    bool UseLiveOuts,
                                                    bool IgnoreStart) const {
-  LivePhysRegs LR(*TRI);
+  LiveRegUnits LR(*TRI);
   if (UseLiveOuts)
     LR.addLiveOuts(*Stop.getParent());
 
@@ -396,7 +396,7 @@ bool SIOptimizeExecMasking::isRegisterInUseBetween(MachineInstr &Stop,
     LR.stepBackward(*A);
   }
 
-  return !LR.available(*MRI, Reg);
+  return !LR.available(Reg) || MRI->isReserved(Reg);
 }
 
 // Determine if a register Reg is not re-defined and still in use
diff --git a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
index 6121055eb021..469340784284 100644
--- a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
@@ -31,7 +31,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/Analysis/AliasAnalysis.h"
-#include "llvm/CodeGen/LivePhysRegs.h"
+#include "llvm/CodeGen/LiveRegUnits.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
@@ -109,7 +109,7 @@ namespace {
     const ARMSubtarget *STI;
     const TargetLowering *TL;
     ARMFunctionInfo *AFI;
-    LivePhysRegs LiveRegs;
+    LiveRegUnits LiveRegs;
     RegisterClassInfo RegClassInfo;
     MachineBasicBlock::const_iterator LiveRegPos;
     bool LiveRegsValid;
@@ -589,7 +589,7 @@ unsigned ARMLoadStoreOpt::findFreeReg(const TargetRegisterClass &RegClass) {
   }
 
   for (unsigned Reg : RegClassInfo.getOrder(&RegClass))
-    if (LiveRegs.available(MF->getRegInfo(), Reg))
+    if (LiveRegs.available(Reg) && !MF->getRegInfo().isReserved(Reg))
       return Reg;
   return 0;
 }
diff --git a/llvm/lib/Target/Hexagon/HexagonGenMux.cpp b/llvm/lib/Target/Hexagon/HexagonGenMux.cpp
index 509144ceb6a0..74ab58e02966 100644
--- a/llvm/lib/Target/Hexagon/HexagonGenMux.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonGenMux.cpp
@@ -26,7 +26,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/CodeGen/LivePhysRegs.h"
+#include "llvm/CodeGen/LiveRegUnits.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
@@ -346,14 +346,8 @@ bool HexagonGenMux::genMuxInBlock(MachineBasicBlock &B) {
 
   // Fix up kill flags.
 
-  LivePhysRegs LPR(*HRI);
+  LiveRegUnits LPR(*HRI);
   LPR.addLiveOuts(B);
-  auto IsLive = [&LPR, this](unsigned Reg) -> bool {
-    for (MCPhysReg S : HRI->subregs_inclusive(Reg))
-      if (LPR.contains(S))
-        return true;
-    return false;
-  };
   for (MachineInstr &I : llvm::reverse(B)) {
     if (I.isDebugInstr())
       continue;
@@ -365,7 +359,7 @@ bool HexagonGenMux::genMuxInBlock(MachineBasicBlock &B) {
       if (!Op.isReg() || !Op.isUse())
         continue;
       assert(Op.getSubReg() == 0 && "Should have physical registers only");
-      bool Live = IsLive(Op.getReg());
+      bool Live = !LPR.available(Op.getReg());
       Op.setIsKill(!Live);
     }
     LPR.stepBackward(I);
diff --git a/llvm/lib/Target/SystemZ/SystemZElimCompare.cpp b/llvm/lib/Target/SystemZ/SystemZElimCompare.cpp
index 7423ed429ffb..e58f50e471fc 100644
--- a/llvm/lib/Target/SystemZ/SystemZElimCompare.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZElimCompare.cpp
@@ -18,7 +18,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/CodeGen/LivePhysRegs.h"
+#include "llvm/CodeGen/LiveRegUnits.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
@@ -690,9 +690,9 @@ bool SystemZElimCompare::processBlock(MachineBasicBlock &MBB) {
   // Walk backwards through the block looking for comparisons, recording
   // all CC users as we go.  The subroutines can delete Compare and
   // instructions before it.
-  LivePhysRegs LiveRegs(*TRI);
+  LiveRegUnits LiveRegs(*TRI);
   LiveRegs.addLiveOuts(MBB);
-  bool CompleteCCUsers = !LiveRegs.contains(SystemZ::CC);
+  bool CompleteCCUsers = LiveRegs.available(SystemZ::CC);
   SmallVector CCUsers;
   MachineBasicBlock::iterator MBBI = MBB.end();
   while (MBBI != MBB.begin()) {
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
index 046a12208467..53e9bf9a9d1b 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
@@ -18,7 +18,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/CodeGen/LiveInterval.h"
 #include "llvm/CodeGen/LiveIntervals.h"
-#include "llvm/CodeGen/LivePhysRegs.h"
+#include "llvm/CodeGen/LiveRegUnits.h"
 #include "llvm/CodeGen/LiveVariables.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
@@ -1874,9 +1874,9 @@ prepareCompareSwapOperands(MachineBasicBlock::iterator const MBBI) const {
     }
   }
   if (CCLive) {
-    LivePhysRegs LiveRegs(*MBB->getParent()->getSubtarget().getRegisterInfo());
+    LiveRegUnits LiveRegs(*MBB->getParent()->getSubtarget().getRegisterInfo());
     LiveRegs.addLiveOuts(*MBB);
-    if (LiveRegs.contains(SystemZ::CC))
+    if (!LiveRegs.available(SystemZ::CC))
       return false;
   }
 
diff --git a/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp b/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp
index 30b22fa1ce92..c0adfdbf120b 100644
--- a/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp
@@ -13,7 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "SystemZTargetMachine.h"
-#include "llvm/CodeGen/LivePhysRegs.h"
+#include "llvm/CodeGen/LiveRegUnits.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
@@ -46,7 +46,7 @@ private:
 
   const SystemZInstrInfo *TII;
   const TargetRegisterInfo *TRI;
-  LivePhysRegs LiveRegs;
+  LiveRegUnits LiveRegs;
 };
 
 char SystemZShortenInst::ID = 0;
@@ -88,7 +88,7 @@ bool SystemZShortenInst::shortenIIF(MachineInstr &MI, unsigned LLIxL,
   unsigned GR64BitReg =
       TRI->getMatchingSuperReg(Reg, thisSubRegIdx, &SystemZ::GR64BitRegClass);
   Register OtherReg = TRI->getSubReg(GR64BitReg, otherSubRegIdx);
-  if (LiveRegs.contains(OtherReg))
+  if (!LiveRegs.available(OtherReg))
     return false;
 
   uint64_t Imm = MI.getOperand(1).getImm();
@@ -143,7 +143,7 @@ bool SystemZShortenInst::shortenOn001(MachineInstr &MI, unsigned Opcode) {
 // Calls shortenOn001 if CCLive is false. CC def operand is added in
 // case of success.
 bool SystemZShortenInst::shortenOn001AddCC(MachineInstr &MI, unsigned Opcode) {
-  if (!LiveRegs.contains(SystemZ::CC) && shortenOn001(MI, Opcode)) {
+  if (LiveRegs.available(SystemZ::CC) && shortenOn001(MI, Opcode)) {
     MachineInstrBuilder(*MI.getParent()->getParent(), &MI)
       .addReg(SystemZ::CC, RegState::ImplicitDefine | RegState::Dead);
     return true;
diff --git a/llvm/lib/Target/X86/X86FloatingPoint.cpp b/llvm/lib/Target/X86/X86FloatingPoint.cpp
index ca4d03913d09..260879ffaa4f 100644
--- a/llvm/lib/Target/X86/X86FloatingPoint.cpp
+++ b/llvm/lib/Target/X86/X86FloatingPoint.cpp
@@ -30,7 +30,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/CodeGen/EdgeBundles.h"
-#include "llvm/CodeGen/LivePhysRegs.h"
+#include "llvm/CodeGen/LiveRegUnits.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
@@ -1751,7 +1751,7 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &Inst) {
 void FPS::setKillFlags(MachineBasicBlock &MBB) const {
   const TargetRegisterInfo &TRI =
       *MBB.getParent()->getSubtarget().getRegisterInfo();
-  LivePhysRegs LPR(TRI);
+  LiveRegUnits LPR(TRI);
 
   LPR.addLiveOuts(MBB);
 
@@ -1773,14 +1773,14 @@ void FPS::setKillFlags(MachineBasicBlock &MBB) const {
 
       if (MO.isDef()) {
         Defs.set(Reg);
-        if (!LPR.contains(MO.getReg()))
+        if (LPR.available(MO.getReg()))
           MO.setIsDead();
       } else
         Uses.push_back(&MO);
     }
 
     for (auto *MO : Uses)
-      if (Defs.test(getFPReg(*MO)) || !LPR.contains(MO->getReg()))
+      if (Defs.test(getFPReg(*MO)) || LPR.available(MO->getReg()))
         MO->setIsKill();
 
     LPR.stepBackward(MI);
-- 
GitLab


From f07157eb2e8212068cb9469d77f9bc3779ef2ea4 Mon Sep 17 00:00:00 2001
From: Balazs Benics 
Date: Fri, 8 Mar 2024 14:52:19 +0100
Subject: [PATCH 612/929] Fixup 7457e2c1535acd54 by adding a switch case

This fixups 7457e2c1535acd548d2619dfb34eb93d27d15908 as I forgot one
switch to update, which then broke this build bot:
https://lab.llvm.org/buildbot/#/builders/57/builds/33272
---
 clang/unittests/AST/MatchVerifier.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/clang/unittests/AST/MatchVerifier.h b/clang/unittests/AST/MatchVerifier.h
index 8bcf05642cb5..da1e351da4a0 100644
--- a/clang/unittests/AST/MatchVerifier.h
+++ b/clang/unittests/AST/MatchVerifier.h
@@ -116,6 +116,10 @@ MatchVerifier::match(const std::string &Code,
     Args.push_back("-std=c++20");
     FileName = "input.cc";
     break;
+  case Lang_CXX23:
+    Args.push_back("-std=c++23");
+    FileName = "input.cc";
+    break;
   case Lang_OpenCL:
     Args.push_back("-cl-no-stdinc");
     FileName = "input.cl";
-- 
GitLab


From ba2236d3000645d3127f972aa7ac1844c47e299c Mon Sep 17 00:00:00 2001
From: Will Hawkins 
Date: Fri, 8 Mar 2024 09:25:51 -0500
Subject: [PATCH 613/929] [libcxx][NFC] Consolidate testing concept CanBePiped
 (#80154)

Almost every test needed a CanBePiped concept and each implemented it
separately, but identically. Consolidate all implementations into
test_range.h.
---
 .../test/std/ranges/range.adaptors/range.all/all.pass.cpp | 8 ++------
 .../ranges/range.adaptors/range.chunk.by/adaptor.pass.cpp | 6 +-----
 .../range.adaptors/range.common.view/adaptor.pass.cpp     | 6 +-----
 .../range.adaptors/range.drop.while/adaptor.pass.cpp      | 8 ++------
 .../std/ranges/range.adaptors/range.drop/adaptor.pass.cpp | 7 ++-----
 .../ranges/range.adaptors/range.elements/adaptor.pass.cpp | 8 ++------
 .../ranges/range.adaptors/range.filter/adaptor.pass.cpp   | 6 +-----
 .../std/ranges/range.adaptors/range.join/adaptor.pass.cpp | 6 +-----
 .../range.adaptors/range.lazy.split/adaptor.pass.cpp      | 6 +-----
 .../ranges/range.adaptors/range.reverse/adaptor.pass.cpp  | 6 +-----
 .../ranges/range.adaptors/range.split/adaptor.pass.cpp    | 6 +-----
 .../range.adaptors/range.take.while/adaptor.pass.cpp      | 7 +------
 .../std/ranges/range.adaptors/range.take/adaptor.pass.cpp | 7 ++-----
 .../range.adaptors/range.transform/adaptor.pass.cpp       | 6 +-----
 libcxx/test/support/test_range.h                          | 5 +++++
 15 files changed, 24 insertions(+), 74 deletions(-)

diff --git a/libcxx/test/std/ranges/range.adaptors/range.all/all.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.all/all.pass.cpp
index d90a25d93e34..d31373fcad78 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.all/all.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.all/all.pass.cpp
@@ -17,8 +17,9 @@
 #include 
 #include 
 
-#include "test_macros.h"
 #include "test_iterators.h"
+#include "test_macros.h"
+#include "test_range.h"
 
 int globalBuff[8];
 
@@ -82,11 +83,6 @@ struct RandomAccessRange {
 template<>
 inline constexpr bool std::ranges::enable_borrowed_range = true;
 
-template 
-concept CanBePiped = requires (View&& view, T&& t) {
-  { std::forward(view) | std::forward(t) };
-};
-
 constexpr bool test() {
   {
     ASSERT_SAME_TYPE(decltype(std::views::all(View())), View);
diff --git a/libcxx/test/std/ranges/range.adaptors/range.chunk.by/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.chunk.by/adaptor.pass.cpp
index 423c481127e2..697085a1d5c5 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.chunk.by/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.chunk.by/adaptor.pass.cpp
@@ -22,11 +22,7 @@
 #include 
 
 #include "test_iterators.h"
-
-template 
-concept CanBePiped = requires(View&& view, T&& t) {
-  { std::forward(view) | std::forward(t) };
-};
+#include "test_range.h"
 
 struct Pred {
   constexpr bool operator()(int x, int y) const { return x != -y; }
diff --git a/libcxx/test/std/ranges/range.adaptors/range.common.view/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.common.view/adaptor.pass.cpp
index 143f7df1d372..b0e72e9e7914 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.common.view/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.common.view/adaptor.pass.cpp
@@ -18,13 +18,9 @@
 #include 
 
 #include "test_iterators.h"
+#include "test_range.h"
 #include "types.h"
 
-template 
-concept CanBePiped = requires (View&& view, T&& t) {
-  { std::forward(view) | std::forward(t) };
-};
-
 constexpr bool test() {
   int buf[] = {1, 2, 3};
 
diff --git a/libcxx/test/std/ranges/range.adaptors/range.drop.while/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.drop.while/adaptor.pass.cpp
index 409b400f7f87..c41d4172f199 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.drop.while/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.drop.while/adaptor.pass.cpp
@@ -16,6 +16,8 @@
 #include 
 #include 
 
+#include "test_range.h"
+
 struct Pred {
   constexpr bool operator()(int i) const { return i < 3; }
 };
@@ -51,12 +53,6 @@ static_assert(std::is_invocable_v);
 static_assert(std::is_invocable_v);
 
-template 
-concept CanBePiped =
-    requires(View&& view, T&& t) {
-      { std::forward(view) | std::forward(t) };
-    };
-
 static_assert(!CanBePiped);
 static_assert(CanBePiped);
 static_assert(!CanBePiped);
diff --git a/libcxx/test/std/ranges/range.adaptors/range.drop/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.drop/adaptor.pass.cpp
index 457b137c8045..d07fe9ac0ea7 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.drop/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.drop/adaptor.pass.cpp
@@ -18,12 +18,9 @@
 #include 
 #include 
 #include 
-#include "test_iterators.h"
 
-template 
-concept CanBePiped = requires (View&& view, T&& t) {
-  { std::forward(view) | std::forward(t) };
-};
+#include "test_iterators.h"
+#include "test_range.h"
 
 struct SizedView : std::ranges::view_base {
   int* begin_ = nullptr;
diff --git a/libcxx/test/std/ranges/range.adaptors/range.elements/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.elements/adaptor.pass.cpp
index d68d6e57e2ed..35589c96e4b7 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.elements/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.elements/adaptor.pass.cpp
@@ -19,6 +19,8 @@
 #include 
 #include 
 
+#include "test_range.h"
+
 template 
 struct View : std::ranges::view_base {
   T* begin() const;
@@ -41,12 +43,6 @@ static_assert(!std::is_invocable_v>);
 static_assert(std::is_invocable_v>>);
 static_assert(!std::is_invocable_v>>);
 
-template 
-concept CanBePiped =
-    requires(View&& view, T&& t) {
-      { std::forward(view) | std::forward(t) };
-    };
-
 static_assert(!CanBePiped, decltype((std::views::elements<0>))>);
 static_assert(CanBePiped>, decltype((std::views::elements<0>))>);
 static_assert(CanBePiped>, decltype((std::views::elements<0>))>);
diff --git a/libcxx/test/std/ranges/range.adaptors/range.filter/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.filter/adaptor.pass.cpp
index 1adf1b4630b8..a44e96e1d4bd 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.filter/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.filter/adaptor.pass.cpp
@@ -19,11 +19,7 @@
 #include 
 
 #include "test_iterators.h"
-
-template 
-concept CanBePiped = requires (View&& view, T&& t) {
-  { std::forward(view) | std::forward(t) };
-};
+#include "test_range.h"
 
 struct NonCopyablePredicate {
   NonCopyablePredicate(NonCopyablePredicate const&) = delete;
diff --git a/libcxx/test/std/ranges/range.adaptors/range.join/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.join/adaptor.pass.cpp
index 9beb3d282a27..68fdfb5b8475 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.join/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.join/adaptor.pass.cpp
@@ -15,6 +15,7 @@
 #include 
 #include 
 
+#include "test_range.h"
 #include "types.h"
 
 struct MoveOnlyOuter : SimpleForwardCommonOuter {
@@ -32,11 +33,6 @@ struct Foo {
   constexpr Foo(int ii) : i(ii) {}
 };
 
-template 
-concept CanBePiped = requires(View&& view, T&& t) {
-  { std::forward(view) | std::forward(t) };
-};
-
 constexpr bool test() {
   int buffer1[3] = {1, 2, 3};
   int buffer2[2] = {4, 5};
diff --git a/libcxx/test/std/ranges/range.adaptors/range.lazy.split/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.lazy.split/adaptor.pass.cpp
index 6bfa0ab487ba..7bb7e95b02da 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.lazy.split/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.lazy.split/adaptor.pass.cpp
@@ -19,13 +19,9 @@
 #include 
 
 #include "test_iterators.h"
+#include "test_range.h"
 #include "types.h"
 
-template 
-concept CanBePiped = requires (View&& view, T&& t) {
-  { std::forward(view) | std::forward(t) };
-};
-
 struct SomeView : std::ranges::view_base {
   const std::string_view* v_;
   constexpr SomeView(const std::string_view& v) : v_(&v) {}
diff --git a/libcxx/test/std/ranges/range.adaptors/range.reverse/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.reverse/adaptor.pass.cpp
index 5e0a0a3ce3fb..26c9ac08d338 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.reverse/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.reverse/adaptor.pass.cpp
@@ -17,13 +17,9 @@
 #include 
 #include 
 
+#include "test_range.h"
 #include "types.h"
 
-template 
-concept CanBePiped = requires (View&& view, T&& t) {
-  { std::forward(view) | std::forward(t) };
-};
-
 constexpr bool test() {
   int buf[] = {1, 2, 3};
 
diff --git a/libcxx/test/std/ranges/range.adaptors/range.split/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.split/adaptor.pass.cpp
index 85d13ac5c29d..34c0f15ce346 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.split/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.split/adaptor.pass.cpp
@@ -19,11 +19,7 @@
 #include 
 
 #include "test_iterators.h"
-
-template 
-concept CanBePiped = requires (View&& view, T&& t) {
-  { std::forward(view) | std::forward(t) };
-};
+#include "test_range.h"
 
 struct SomeView : std::ranges::view_base {
   const std::string_view* v_;
diff --git a/libcxx/test/std/ranges/range.adaptors/range.take.while/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.take.while/adaptor.pass.cpp
index 8796f9df63ce..c224d198d16b 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.take.while/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.take.while/adaptor.pass.cpp
@@ -16,6 +16,7 @@
 #include 
 #include 
 
+#include "test_range.h"
 #include "types.h"
 
 struct Pred {
@@ -42,12 +43,6 @@ static_assert(std::is_invocable_v);
 static_assert(std::is_invocable_v);
 
-template 
-concept CanBePiped =
-    requires(View&& view, T&& t) {
-      { std::forward(view) | std::forward(t) };
-    };
-
 static_assert(!CanBePiped);
 static_assert(CanBePiped);
 static_assert(!CanBePiped);
diff --git a/libcxx/test/std/ranges/range.adaptors/range.take/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.take/adaptor.pass.cpp
index bb5b5f5ff490..b30c4fbe3fd9 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.take/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.take/adaptor.pass.cpp
@@ -17,12 +17,9 @@
 #include 
 #include 
 #include 
-#include "test_iterators.h"
 
-template 
-concept CanBePiped = requires (View&& view, T&& t) {
-  { std::forward(view) | std::forward(t) };
-};
+#include "test_iterators.h"
+#include "test_range.h"
 
 struct SizedView : std::ranges::view_base {
   int* begin_ = nullptr;
diff --git a/libcxx/test/std/ranges/range.adaptors/range.transform/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.transform/adaptor.pass.cpp
index 59dd85bf28bc..91d6f2eab18b 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.transform/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.transform/adaptor.pass.cpp
@@ -18,13 +18,9 @@
 #include 
 
 #include "test_macros.h"
+#include "test_range.h"
 #include "types.h"
 
-template 
-concept CanBePiped = requires (View&& view, T&& t) {
-  { std::forward(view) | std::forward(t) };
-};
-
 struct NonCopyableFunction {
   NonCopyableFunction(NonCopyableFunction const&) = delete;
   template 
diff --git a/libcxx/test/support/test_range.h b/libcxx/test/support/test_range.h
index 6061f710a263..c5eeb25bb59e 100644
--- a/libcxx/test/support/test_range.h
+++ b/libcxx/test/support/test_range.h
@@ -89,4 +89,9 @@ concept simple_view =
     std::same_as, std::ranges::iterator_t> &&
     std::same_as, std::ranges::sentinel_t>;
 
+template 
+concept CanBePiped = requires(View&& view, T&& t) {
+  { std::forward(view) | std::forward(t) };
+};
+
 #endif // LIBCXX_TEST_SUPPORT_TEST_RANGE_H
-- 
GitLab


From 01e5d4609bed5db52890cb66a91c65e9bb6a122f Mon Sep 17 00:00:00 2001
From: Stephen Tozer 
Date: Fri, 8 Mar 2024 15:03:15 +0000
Subject: [PATCH 614/929] [RemoveDIs] Add documentation for IR debug records
 (#81156)

This patch adds minimal documentation for the IR representation of the
RemoveDIs model of debug info. This patch assumes
that the default for all cases is still debug intrinsic functions, and
so does not update all existing text to refer to debug records, but only
adds a section in the LangRef and SourceLevelDebugging documents to
explain the new format.
---
 llvm/docs/LangRef.rst              | 31 +++++++++++++-
 llvm/docs/SourceLevelDebugging.rst | 66 ++++++++++++++++++++++++++++++
 2 files changed, 95 insertions(+), 2 deletions(-)

diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index e0f301aa06dc..a7b77d6f776a 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -908,7 +908,8 @@ Syntax::
 
 A function definition contains a list of basic blocks, forming the CFG (Control
 Flow Graph) for the function. Each basic block may optionally start with a label
-(giving the basic block a symbol table entry), contains a list of instructions,
+(giving the basic block a symbol table entry), contains a list of instructions
+and :ref:`debug records `,
 and ends with a :ref:`terminator ` instruction (such as a branch or
 function return). If an explicit label name is not provided, a block is assigned
 an implicit numbered label, using the next value from the same counter as used
@@ -8541,7 +8542,10 @@ The LLVM instruction set consists of several different classifications
 of instructions: :ref:`terminator instructions `, :ref:`binary
 instructions `, :ref:`bitwise binary
 instructions `, :ref:`memory instructions `, and
-:ref:`other instructions `.
+:ref:`other instructions `. There are also :ref:`debug records
+`, which are not instructions themselves but are printed
+interleaved with instructions to describe changes in the state of the program's
+debug information at each position in the program's execution.
 
 .. _terminators:
 
@@ -12695,6 +12699,29 @@ Example:
 
       %tok = cleanuppad within %cs []
 
+.. _debugrecords:
+
+Debug Records
+-----------------------
+
+Debug records appear interleaved with instructions, but are not instructions;
+they are used only to define debug information, and have no effect on generated
+code. They are distinguished from instructions by the use of a leading `#` and
+an extra level of indentation. As an example:
+
+.. code-block:: llvm
+
+  %inst1 = op1 %a, %b
+    #dbg_value(%inst1, !10, !DIExpression(), !11)
+  %inst2 = op2 %inst1, %c
+
+These debug records are an optional replacement for
+:ref:`debug intrinsics`. Debug records will be output if the
+``--write-experimental-debuginfo`` flag is passed to LLVM; it is an error for both
+records and intrinsics to appear in the same module. More information about
+debug records can be found in the `LLVM Source Level Debugging
+`_ document.
+
 .. _intrinsics:
 
 Intrinsic Functions
diff --git a/llvm/docs/SourceLevelDebugging.rst b/llvm/docs/SourceLevelDebugging.rst
index e1df7c355ee0..7f7e595eb14d 100644
--- a/llvm/docs/SourceLevelDebugging.rst
+++ b/llvm/docs/SourceLevelDebugging.rst
@@ -167,6 +167,17 @@ conventions used by the C and C++ front-ends.
 Debug information descriptors are `specialized metadata nodes
 `_, first-class subclasses of ``Metadata``.
 
+There are two models for defining the values of source variables at different
+states of the program and tracking these values through optimization and code
+generation: :ref:`intrinsic function calls `, the
+current default, and :ref:`debug records `, which are a new
+non-instruction-based model
+(for an explanation of how this works and why it is desirable, see the
+`RemoveDIs `_ document). Each module must use one or
+the other; they may never be mixed within an IR module. To enable writing debug
+records instead of intrinsic calls, use the flag
+``--write-experimental-debuginfo``.
+
 .. _format_common_intrinsics:
 
 Debugger intrinsic functions
@@ -268,6 +279,61 @@ The formal LLVM-IR signature is:
 
 See :doc:`AssignmentTracking` for more info.
 
+.. _debug_records:
+
+Debug Records
+----------------------------
+
+LLVM also has an alternative to intrinsic functions, debug records, which
+function similarly but are not instructions. The basic syntax for debug records
+is:
+
+.. code-block:: llvm
+
+    #dbg_([, ]* )
+  ; Using the intrinsic model, the above is equivalent to:
+  call void llvm.dbg.([metadata , ]*), !dbg 
+
+A debug intrinsic function can be converted to a debug record with the
+following steps:
+
+1. Add an extra level of indentation.
+2. Replace everything prior to the intrinsic kind (declare/value/assign) with
+   ``#dbg_``.
+3. Remove the leading ``metadata`` from the intrinsic's arguments.
+4. Transfer the ``!dbg`` attachment to be an argument, dropping the leading
+   ``!dbg``.
+
+For each kind of intrinsic function, there is an equivalent debug record.
+
+``#dbg_declare``
+^^^^^^^^^^^^^^^^
+
+.. code-block:: llvm
+
+    #dbg_declare([Value|MDNode], DILocalVariable, DIExpression, DILocation)
+
+Equivalent to the ``llvm.dbg.declare`` intrinsic.
+
+``#dbg_value``
+^^^^^^^^^^^^^^
+
+.. code-block:: llvm
+
+    #dbg_value([Value|DIArgList|MDNode], DILocalVariable, DIExpression, DILocation)
+
+Equivalent to the ``llvm.dbg.value`` intrinsic.
+
+``#dbg_assign``
+^^^^^^^^^^^^^^^
+
+.. code-block:: llvm
+
+    #dbg_assign([Value|DIArgList|MDNode], DILocalVariable, DIExpression,
+                DIAssignID, [Value|MDNode], DIExpression, DILocation)
+
+Equivalent to the ``llvm.dbg.assign`` intrinsic.
+
 Object lifetimes and scoping
 ============================
 
-- 
GitLab


From 23c397c7c9407deea3398e787f2901f86207dc11 Mon Sep 17 00:00:00 2001
From: Guillaume Chatelet 
Date: Fri, 8 Mar 2024 16:06:56 +0100
Subject: [PATCH 615/929] [libc] Provide `LIBC_TYPES_HAS_INT128` (#84149)

Umbrella bug #83182
---
 libc/src/__support/CMakeLists.txt             |  2 ++
 libc/src/__support/CPP/CMakeLists.txt         |  1 +
 libc/src/__support/CPP/limits.h               |  5 ++--
 .../__support/CPP/type_traits/is_integral.h   |  3 ++-
 .../__support/CPP/type_traits/make_signed.h   |  3 ++-
 .../__support/CPP/type_traits/make_unsigned.h |  3 ++-
 libc/src/__support/UInt.h                     | 23 ++++++++++---------
 libc/src/__support/UInt128.h                  |  5 ++--
 libc/src/__support/macros/properties/types.h  |  7 ++++++
 libc/test/UnitTest/CMakeLists.txt             |  1 +
 libc/test/UnitTest/LibcTest.cpp               |  5 ++--
 libc/test/UnitTest/TestLogger.cpp             |  5 ++--
 libc/test/src/__support/CMakeLists.txt        |  6 +++--
 libc/test/src/__support/CPP/CMakeLists.txt    |  2 ++
 libc/test/src/__support/CPP/bit_test.cpp      |  9 ++++----
 libc/test/src/__support/CPP/limits_test.cpp   |  5 ++--
 .../src/__support/integer_literals_test.cpp   |  5 ++--
 libc/test/src/__support/uint_test.cpp         |  9 ++++----
 .../llvm-project-overlay/libc/BUILD.bazel     |  3 +++
 .../libc/test/UnitTest/BUILD.bazel            |  2 ++
 .../libc/test/src/__support/BUILD.bazel       |  2 ++
 .../libc/test/src/__support/CPP/BUILD.bazel   |  2 ++
 22 files changed, 72 insertions(+), 36 deletions(-)

diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt
index 66e4d808333f..2e5a026bf423 100644
--- a/libc/src/__support/CMakeLists.txt
+++ b/libc/src/__support/CMakeLists.txt
@@ -205,6 +205,7 @@ add_header_library(
     libc.src.__support.CPP.bit
     libc.src.__support.CPP.type_traits
     libc.src.__support.macros.optimization
+    libc.src.__support.macros.properties.types
 )
 
 add_header_library(
@@ -213,6 +214,7 @@ add_header_library(
     UInt128.h
   DEPENDS
     .uint
+    libc.src.__support.macros.properties.types
 )
 
 add_header_library(
diff --git a/libc/src/__support/CPP/CMakeLists.txt b/libc/src/__support/CPP/CMakeLists.txt
index 6c35bc709081..6216505eae23 100644
--- a/libc/src/__support/CPP/CMakeLists.txt
+++ b/libc/src/__support/CPP/CMakeLists.txt
@@ -49,6 +49,7 @@ add_header_library(
   DEPENDS
     .type_traits
     libc.include.llvm-libc-macros.limits_macros
+    libc.src.__support.macros.properties.types
 )
 
 add_header_library(
diff --git a/libc/src/__support/CPP/limits.h b/libc/src/__support/CPP/limits.h
index 1ffde5f9556f..5b9b3e755c72 100644
--- a/libc/src/__support/CPP/limits.h
+++ b/libc/src/__support/CPP/limits.h
@@ -12,7 +12,8 @@
 #include "include/llvm-libc-macros/limits-macros.h" // CHAR_BIT
 #include "src/__support/CPP/type_traits/is_integral.h"
 #include "src/__support/CPP/type_traits/is_signed.h"
-#include "src/__support/macros/attributes.h" // LIBC_INLINE
+#include "src/__support/macros/attributes.h"       // LIBC_INLINE
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
 
 namespace LIBC_NAMESPACE {
 namespace cpp {
@@ -76,7 +77,7 @@ template <>
 struct numeric_limits
     : public internal::integer_impl {};
 
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
 // On platform where UInt128 resolves to __uint128_t, this specialization
 // provides the limits of UInt128.
 template <>
diff --git a/libc/src/__support/CPP/type_traits/is_integral.h b/libc/src/__support/CPP/type_traits/is_integral.h
index 2808be594b20..68e16ff84183 100644
--- a/libc/src/__support/CPP/type_traits/is_integral.h
+++ b/libc/src/__support/CPP/type_traits/is_integral.h
@@ -11,6 +11,7 @@
 #include "src/__support/CPP/type_traits/is_same.h"
 #include "src/__support/CPP/type_traits/remove_cv.h"
 #include "src/__support/macros/attributes.h"
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
 
 namespace LIBC_NAMESPACE::cpp {
 
@@ -25,7 +26,7 @@ private:
 public:
   LIBC_INLINE_VAR static constexpr bool value = __is_unqualified_any_of<
       T,
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
       __int128_t, __uint128_t,
 #endif
       char, signed char, unsigned char, short, unsigned short, int,
diff --git a/libc/src/__support/CPP/type_traits/make_signed.h b/libc/src/__support/CPP/type_traits/make_signed.h
index 21302850bfd4..4652d8b6bfa5 100644
--- a/libc/src/__support/CPP/type_traits/make_signed.h
+++ b/libc/src/__support/CPP/type_traits/make_signed.h
@@ -9,6 +9,7 @@
 #define LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_MAKE_SIGNED_H
 
 #include "src/__support/CPP/type_traits/type_identity.h"
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
 
 namespace LIBC_NAMESPACE::cpp {
 
@@ -26,7 +27,7 @@ template <> struct make_signed : type_identity {};
 template <> struct make_signed : type_identity {};
 template <>
 struct make_signed : type_identity {};
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
 template <> struct make_signed<__int128_t> : type_identity<__int128_t> {};
 template <> struct make_signed<__uint128_t> : type_identity<__int128_t> {};
 #endif
diff --git a/libc/src/__support/CPP/type_traits/make_unsigned.h b/libc/src/__support/CPP/type_traits/make_unsigned.h
index 20948014a665..1e814ae002a7 100644
--- a/libc/src/__support/CPP/type_traits/make_unsigned.h
+++ b/libc/src/__support/CPP/type_traits/make_unsigned.h
@@ -9,6 +9,7 @@
 #define LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_MAKE_UNSIGNED_H
 
 #include "src/__support/CPP/type_traits/type_identity.h"
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
 
 namespace LIBC_NAMESPACE::cpp {
 
@@ -31,7 +32,7 @@ template <>
 struct make_unsigned : type_identity {};
 template <>
 struct make_unsigned : type_identity {};
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
 template <> struct make_unsigned<__int128_t> : type_identity<__uint128_t> {};
 template <> struct make_unsigned<__uint128_t> : type_identity<__uint128_t> {};
 #endif
diff --git a/libc/src/__support/UInt.h b/libc/src/__support/UInt.h
index 94c8d86fd764..c49c8314cd49 100644
--- a/libc/src/__support/UInt.h
+++ b/libc/src/__support/UInt.h
@@ -14,9 +14,10 @@
 #include "src/__support/CPP/limits.h"
 #include "src/__support/CPP/optional.h"
 #include "src/__support/CPP/type_traits.h"
-#include "src/__support/macros/attributes.h"   // LIBC_INLINE
-#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
-#include "src/__support/math_extras.h"         // SumCarry, DiffBorrow
+#include "src/__support/macros/attributes.h"       // LIBC_INLINE
+#include "src/__support/macros/optimization.h"     // LIBC_UNLIKELY
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
+#include "src/__support/math_extras.h"             // SumCarry, DiffBorrow
 #include "src/__support/number_pair.h"
 
 #include  // For size_t
@@ -30,9 +31,9 @@ template  struct half_width;
 template <> struct half_width : cpp::type_identity {};
 template <> struct half_width : cpp::type_identity {};
 template <> struct half_width : cpp::type_identity {};
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
 template <> struct half_width<__uint128_t> : cpp::type_identity {};
-#endif // __SIZEOF_INT128__
+#endif // LIBC_TYPES_HAS_INT128
 
 template  using half_width_t = typename half_width::type;
 
@@ -69,7 +70,7 @@ LIBC_INLINE constexpr NumberPair full_mul(uint32_t a,
   return result;
 }
 
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
 template <>
 LIBC_INLINE constexpr NumberPair full_mul(uint64_t a,
                                                               uint64_t b) {
@@ -79,7 +80,7 @@ LIBC_INLINE constexpr NumberPair full_mul(uint64_t a,
   result.hi = uint64_t(prod >> 64);
   return result;
 }
-#endif // __SIZEOF_INT128__
+#endif // LIBC_TYPES_HAS_INT128
 
 } // namespace internal
 
@@ -682,7 +683,7 @@ struct BigInt {
       val[1] = uint32_t(tmp >> 32);
       return;
     }
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
     if constexpr ((Bits == 128) && (WORD_SIZE == 64)) {
       // Use builtin 128 bits if available;
       if (s >= 128) {
@@ -696,7 +697,7 @@ struct BigInt {
       val[1] = uint64_t(tmp >> 64);
       return;
     }
-#endif // __SIZEOF_INT128__
+#endif // LIBC_TYPES_HAS_INT128
     if (LIBC_UNLIKELY(s == 0))
       return;
 
@@ -753,7 +754,7 @@ struct BigInt {
       val[1] = uint32_t(tmp >> 32);
       return;
     }
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
     if constexpr ((Bits == 128) && (WORD_SIZE == 64)) {
       // Use builtin 128 bits if available;
       if (s >= 128) {
@@ -771,7 +772,7 @@ struct BigInt {
       val[1] = uint64_t(tmp >> 64);
       return;
     }
-#endif // __SIZEOF_INT128__
+#endif // LIBC_TYPES_HAS_INT128
 
     if (LIBC_UNLIKELY(s == 0))
       return;
diff --git a/libc/src/__support/UInt128.h b/libc/src/__support/UInt128.h
index 06696b7a6158..b6ef9ca18eb0 100644
--- a/libc/src/__support/UInt128.h
+++ b/libc/src/__support/UInt128.h
@@ -10,13 +10,14 @@
 #define LLVM_LIBC_SRC___SUPPORT_UINT128_H
 
 #include "UInt.h"
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
 
-#if defined(__SIZEOF_INT128__)
+#ifdef LIBC_TYPES_HAS_INT128
 using UInt128 = __uint128_t;
 using Int128 = __int128_t;
 #else
 using UInt128 = LIBC_NAMESPACE::UInt<128>;
 using Int128 = LIBC_NAMESPACE::Int<128>;
-#endif
+#endif // LIBC_TYPES_HAS_INT128
 
 #endif // LLVM_LIBC_SRC___SUPPORT_UINT128_H
diff --git a/libc/src/__support/macros/properties/types.h b/libc/src/__support/macros/properties/types.h
index 8760f78875c4..42345e4743ce 100644
--- a/libc/src/__support/macros/properties/types.h
+++ b/libc/src/__support/macros/properties/types.h
@@ -17,6 +17,8 @@
 #include "src/__support/macros/properties/cpu_features.h"
 #include "src/__support/macros/properties/os.h"
 
+#include  // __SIZEOF_INT128__
+
 // 'long double' properties.
 #if (LDBL_MANT_DIG == 53)
 #define LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64
@@ -26,6 +28,11 @@
 #define LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128
 #endif
 
+// int128 / uint128 support
+#if defined(__SIZEOF_INT128__)
+#define LIBC_TYPES_HAS_INT128
+#endif // defined(__SIZEOF_INT128__)
+
 // -- float16 support ---------------------------------------------------------
 // TODO: move this logic to "llvm-libc-types/float16.h"
 #if defined(LIBC_TARGET_ARCH_IS_X86_64) && defined(LIBC_TARGET_CPU_HAS_SSE2)
diff --git a/libc/test/UnitTest/CMakeLists.txt b/libc/test/UnitTest/CMakeLists.txt
index 8a35f1204eb5..f7a6f4a91fab 100644
--- a/libc/test/UnitTest/CMakeLists.txt
+++ b/libc/test/UnitTest/CMakeLists.txt
@@ -73,6 +73,7 @@ add_unittest_framework_library(
     libc.src.__support.CPP.string_view
     libc.src.__support.CPP.type_traits
     libc.src.__support.fixed_point.fx_rep
+    libc.src.__support.macros.properties.types
     libc.src.__support.OSUtil.osutil
     libc.src.__support.uint
     libc.src.__support.uint128
diff --git a/libc/test/UnitTest/LibcTest.cpp b/libc/test/UnitTest/LibcTest.cpp
index 7dc85d0d4e2d..03cd25191ecd 100644
--- a/libc/test/UnitTest/LibcTest.cpp
+++ b/libc/test/UnitTest/LibcTest.cpp
@@ -13,6 +13,7 @@
 #include "src/__support/CPP/string_view.h"
 #include "src/__support/UInt128.h"
 #include "src/__support/fixed_point/fx_rep.h"
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
 #include "test/UnitTest/TestLogger.h"
 
 #if __STDC_HOSTED__
@@ -215,11 +216,11 @@ TEST_SPECIALIZATION(bool);
 
 // We cannot just use a single UInt128 specialization as that resolves to only
 // one type, UInt<128> or __uint128_t. We want both overloads as we want to
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
 // When builtin __uint128_t type is available, include its specialization
 // also.
 TEST_SPECIALIZATION(__uint128_t);
-#endif
+#endif // LIBC_TYPES_HAS_INT128
 
 TEST_SPECIALIZATION(LIBC_NAMESPACE::Int<128>);
 
diff --git a/libc/test/UnitTest/TestLogger.cpp b/libc/test/UnitTest/TestLogger.cpp
index 32f0ec52041f..4756188b46cb 100644
--- a/libc/test/UnitTest/TestLogger.cpp
+++ b/libc/test/UnitTest/TestLogger.cpp
@@ -4,6 +4,7 @@
 #include "src/__support/OSUtil/io.h" // write_to_stderr
 #include "src/__support/UInt.h"      // is_big_int
 #include "src/__support/UInt128.h"
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
 
 #include 
 
@@ -72,9 +73,9 @@ template TestLogger &TestLogger::operator<< (unsigned long);
 template TestLogger &
     TestLogger::operator<< (unsigned long long);
 
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
 template TestLogger &TestLogger::operator<< <__uint128_t>(__uint128_t);
-#endif
+#endif // LIBC_TYPES_HAS_INT128
 template TestLogger &TestLogger::operator<< >(UInt<128>);
 template TestLogger &TestLogger::operator<< >(UInt<192>);
 template TestLogger &TestLogger::operator<< >(UInt<256>);
diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt
index adbacb9728cc..91dd0dc4decf 100644
--- a/libc/test/src/__support/CMakeLists.txt
+++ b/libc/test/src/__support/CMakeLists.txt
@@ -106,8 +106,9 @@ if(NOT LIBC_TARGET_ARCHITECTURE_IS_NVPTX)
     SRCS
       uint_test.cpp
     DEPENDS
-      libc.src.__support.uint
       libc.src.__support.CPP.optional
+      libc.src.__support.macros.properties.types
+      libc.src.__support.uint
   )
 endif()
 
@@ -118,8 +119,9 @@ add_libc_test(
   SRCS
     integer_literals_test.cpp
   DEPENDS
-    libc.src.__support.integer_literals
     libc.src.__support.CPP.optional
+    libc.src.__support.integer_literals
+    libc.src.__support.macros.properties.types
 )
 
 add_libc_test(
diff --git a/libc/test/src/__support/CPP/CMakeLists.txt b/libc/test/src/__support/CPP/CMakeLists.txt
index d7f332f5b0fb..f94429e03b3c 100644
--- a/libc/test/src/__support/CPP/CMakeLists.txt
+++ b/libc/test/src/__support/CPP/CMakeLists.txt
@@ -8,6 +8,7 @@ add_libc_test(
     bit_test.cpp
   DEPENDS
     libc.src.__support.CPP.bit
+    libc.src.__support.macros.properties.types
     libc.src.__support.uint
 )
 
@@ -49,6 +50,7 @@ add_libc_test(
     limits_test.cpp
   DEPENDS
     libc.src.__support.CPP.limits
+    libc.src.__support.macros.properties.types
     libc.src.__support.uint
 )
 
diff --git a/libc/test/src/__support/CPP/bit_test.cpp b/libc/test/src/__support/CPP/bit_test.cpp
index da7bb3389b2f..3deb1f41dcf8 100644
--- a/libc/test/src/__support/CPP/bit_test.cpp
+++ b/libc/test/src/__support/CPP/bit_test.cpp
@@ -8,6 +8,7 @@
 
 #include "src/__support/CPP/bit.h"
 #include "src/__support/UInt.h"
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
 #include "test/UnitTest/Test.h"
 
 #include 
@@ -15,16 +16,16 @@
 namespace LIBC_NAMESPACE::cpp {
 
 using UnsignedTypesNoBigInt = testing::TypeList<
-#if defined(__SIZEOF_INT128__)
+#if defined(LIBC_TYPES_HAS_INT128)
     __uint128_t,
-#endif
+#endif // LIBC_TYPES_HAS_INT128
     unsigned char, unsigned short, unsigned int, unsigned long,
     unsigned long long>;
 
 using UnsignedTypes = testing::TypeList<
-#if defined(__SIZEOF_INT128__)
+#if defined(LIBC_TYPES_HAS_INT128)
     __uint128_t,
-#endif
+#endif // LIBC_TYPES_HAS_INT128
     unsigned char, unsigned short, unsigned int, unsigned long,
     unsigned long long, UInt<128>>;
 
diff --git a/libc/test/src/__support/CPP/limits_test.cpp b/libc/test/src/__support/CPP/limits_test.cpp
index d83e307fb196..efcd6839d073 100644
--- a/libc/test/src/__support/CPP/limits_test.cpp
+++ b/libc/test/src/__support/CPP/limits_test.cpp
@@ -8,6 +8,7 @@
 
 #include "src/__support/CPP/limits.h"
 #include "src/__support/UInt.h"
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
 #include "test/UnitTest/Test.h"
 
 namespace LIBC_NAMESPACE {
@@ -36,9 +37,9 @@ TEST(LlvmLibcLimitsTest, UInt128Limits) {
   auto umax64 = LIBC_NAMESPACE::UInt<128>(cpp::numeric_limits::max());
   EXPECT_GT(umax128, umax64);
   ASSERT_EQ(~LIBC_NAMESPACE::UInt<128>(0), umax128);
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
   ASSERT_EQ(~__uint128_t(0), cpp::numeric_limits<__uint128_t>::max());
-#endif
+#endif // LIBC_TYPES_HAS_INT128
 }
 
 } // namespace LIBC_NAMESPACE
diff --git a/libc/test/src/__support/integer_literals_test.cpp b/libc/test/src/__support/integer_literals_test.cpp
index b8b399d9cf40..5298cf30156e 100644
--- a/libc/test/src/__support/integer_literals_test.cpp
+++ b/libc/test/src/__support/integer_literals_test.cpp
@@ -8,6 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/integer_literals.h"
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
 #include "test/UnitTest/Test.h"
 
 using LIBC_NAMESPACE::operator""_u8;
@@ -66,7 +67,7 @@ TEST(LlvmLibcIntegerLiteralTest, u64) {
 }
 
 TEST(LlvmLibcIntegerLiteralTest, u128) {
-#if defined(__SIZEOF_INT128__)
+#ifdef LIBC_TYPES_HAS_INT128
   const __uint128_t ZERO = 0;
   const __uint128_t U8_MAX = UINT8_MAX;
   const __uint128_t U16_MAX = UINT16_MAX;
@@ -80,7 +81,7 @@ TEST(LlvmLibcIntegerLiteralTest, u128) {
   const UInt128 U32_MAX = UINT32_MAX;
   const UInt128 U64_MAX = UINT64_MAX;
   const UInt128 U128_MAX = (U64_MAX << 64) | U64_MAX;
-#endif
+#endif // LIBC_TYPES_HAS_INT128
   EXPECT_EQ(ZERO, 0_u128);
   EXPECT_EQ(U8_MAX, 255_u128);
   EXPECT_EQ(U8_MAX, 0xFF_u128);
diff --git a/libc/test/src/__support/uint_test.cpp b/libc/test/src/__support/uint_test.cpp
index eb1db9729cf4..851656e9fcbb 100644
--- a/libc/test/src/__support/uint_test.cpp
+++ b/libc/test/src/__support/uint_test.cpp
@@ -8,6 +8,7 @@
 
 #include "src/__support/CPP/optional.h"
 #include "src/__support/UInt.h"
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
 
 #include "test/UnitTest/Test.h"
 #include  // HUGE_VALF, HUGE_VALF
@@ -41,7 +42,7 @@ TEST(LlvmLibcUIntClassTest, BitCastToFromDouble) {
   }
 }
 
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
 TEST(LlvmLibcUIntClassTest, BitCastToFromNativeUint128) {
   static_assert(cpp::is_trivially_copyable::value);
   static_assert(sizeof(LL_UInt128) == sizeof(__uint128_t));
@@ -52,7 +53,7 @@ TEST(LlvmLibcUIntClassTest, BitCastToFromNativeUint128) {
     EXPECT_TRUE(value == forth);
   }
 }
-#endif
+#endif // LIBC_TYPES_HAS_INT128
 
 #ifdef LIBC_TYPES_HAS_FLOAT128
 TEST(LlvmLibcUIntClassTest, BitCastToFromNativeFloat128) {
@@ -652,7 +653,7 @@ TEST(LlvmLibcUIntClassTest, BasicArithmeticInt128Tests) {
   ASSERT_EQ(c * b, b);
 }
 
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
 
 TEST(LlvmLibcUIntClassTest, ConstructorFromUInt128Tests) {
   __uint128_t a = (__uint128_t(123) << 64) + 1;
@@ -707,7 +708,7 @@ TEST(LlvmLibcUIntClassTest, WordTypeUInt128Tests) {
   EXPECT_TRUE(f == r);
 }
 
-#endif // __SIZEOF_INT128__
+#endif // LIBC_TYPES_HAS_INT128
 
 TEST(LlvmLibcUIntClassTest, OtherWordTypeTests) {
   using LL_UInt96 = BigInt<96, false, uint32_t>;
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 127059c529ec..5f6c43cd6af7 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -211,6 +211,7 @@ libc_support_library(
     deps = [
         "__support_cpp_type_traits",
         "__support_macros_attributes",
+        ":__support_macros_properties_types",
         ":llvm_libc_macros_limits_macros",
     ],
 )
@@ -449,6 +450,7 @@ libc_support_library(
         ":__support_cpp_type_traits",
         ":__support_macros_attributes",
         ":__support_macros_optimization",
+        ":__support_macros_properties_types",
         ":__support_math_extras",
         ":__support_number_pair",
     ],
@@ -458,6 +460,7 @@ libc_support_library(
     name = "__support_uint128",
     hdrs = ["src/__support/UInt128.h"],
     deps = [
+        ":__support_macros_properties_types",
         ":__support_uint",
     ],
 )
diff --git a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
index 37eb21c77e49..4a94916e1205 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
@@ -17,6 +17,7 @@ libc_support_library(
     deps = [
         "//libc:__support_cpp_string",
         "//libc:__support_cpp_string_view",
+        "//libc:__support_macros_properties_types",
         "//libc:__support_osutil_io",
         "//libc:__support_uint",
         "//libc:__support_uint128",
@@ -52,6 +53,7 @@ libc_support_library(
         "//libc:__support_fputil_fpbits_str",
         "//libc:__support_fputil_rounding_mode",
         "//libc:__support_macros_properties_architectures",
+        "//libc:__support_macros_properties_types",
         "//libc:__support_stringutil",
         "//libc:__support_uint128",
         "//libc:errno",
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel
index 19d4c7869799..6837b9880d5a 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel
@@ -86,6 +86,7 @@ libc_test(
     srcs = ["uint_test.cpp"],
     deps = [
         "//libc:__support_cpp_optional",
+        "//libc:__support_macros_properties_types",
         "//libc:__support_uint",
     ],
 )
@@ -111,5 +112,6 @@ libc_test(
     srcs = ["integer_literals_test.cpp"],
     deps = [
         "//libc:__support_integer_literals",
+        "//libc:__support_macros_properties_types",
     ],
 )
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/__support/CPP/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/__support/CPP/BUILD.bazel
index 07d9ac1d5008..dad1c7708e44 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/__support/CPP/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/__support/CPP/BUILD.bazel
@@ -27,6 +27,7 @@ libc_test(
     srcs = ["bit_test.cpp"],
     deps = [
         "//libc:__support_cpp_bit",
+        "//libc:__support_macros_properties_types",
         "//libc:__support_uint",
     ],
 )
@@ -48,6 +49,7 @@ libc_test(
     srcs = ["limits_test.cpp"],
     deps = [
         "//libc:__support_cpp_limits",
+        "//libc:__support_macros_properties_types",
         "//libc:__support_uint",
     ],
 )
-- 
GitLab


From cb6f657a70f7a8d6ecd4fcc2101550a7400f94a7 Mon Sep 17 00:00:00 2001
From: Joseph Huber 
Date: Fri, 8 Mar 2024 09:06:12 -0600
Subject: [PATCH 616/929] [CUDA][HIP][NFC] Precommit new driver changes to
 tests

Summary:
A lot of these tests implicitly rely on `--no-offload-new-driver` a
patch in the future will change the default so this patch makes it
explicit in locations where it matters.
---
 clang/test/Driver/cuda-bad-arch.cu            |   4 +-
 clang/test/Driver/cuda-external-tools.cu      |  14 +--
 clang/test/Driver/cuda-phases.cu              |  24 ++--
 clang/test/Driver/hip-binding.hip             |  10 +-
 clang/test/Driver/hip-cuid-hash.hip           |   8 +-
 clang/test/Driver/hip-cuid.hip                |   5 +
 clang/test/Driver/hip-dependent-options.hip   |   4 +-
 clang/test/Driver/hip-device-compile.hip      |  22 ++--
 clang/test/Driver/hip-link-bc-to-bc.hip       |   4 +-
 clang/test/Driver/hip-link-bundle-archive.hip |  22 ++--
 clang/test/Driver/hip-link-save-temps.hip     |   8 +-
 clang/test/Driver/hip-link-shared-library.hip |   3 +-
 clang/test/Driver/hip-link-static-library.hip |   6 +-
 .../test/Driver/hip-offload-compress-zlib.hip |   6 +-
 .../test/Driver/hip-offload-compress-zstd.hip |   6 +-
 clang/test/Driver/hip-output-file-name.hip    |  26 ++---
 clang/test/Driver/hip-partial-link.hip        |  14 +--
 clang/test/Driver/hip-phases.hip              | 106 +++++++++---------
 clang/test/Driver/hip-rdc-device-only.hip     |  14 +--
 clang/test/Driver/hip-target-id.hip           |  10 +-
 clang/test/Driver/hip-toolchain-features.hip  |  24 ++--
 .../Driver/hip-toolchain-rdc-separate.hip     |   8 +-
 .../Driver/hip-toolchain-rdc-static-lib.hip   |   2 +-
 clang/test/Driver/hip-toolchain-rdc.hip       |   4 +-
 clang/test/Driver/hip-unbundle-preproc.hipi   |   6 +-
 clang/test/Driver/hipspv-toolchain-rdc.hip    |   2 +-
 clang/test/Driver/hipspv-toolchain.hip        |   2 +-
 clang/test/Driver/lto.cu                      |  22 ++--
 clang/test/Driver/thinlto.cu                  |   4 +-
 29 files changed, 197 insertions(+), 193 deletions(-)

diff --git a/clang/test/Driver/cuda-bad-arch.cu b/clang/test/Driver/cuda-bad-arch.cu
index 877b20bc9351..35a56a8bef0f 100644
--- a/clang/test/Driver/cuda-bad-arch.cu
+++ b/clang/test/Driver/cuda-bad-arch.cu
@@ -30,9 +30,9 @@
 // RUN: | FileCheck -check-prefix OK %s
 
 // We don't allow using NVPTX/AMDGCN for host compilation.
-// RUN: not %clang -### --cuda-host-only --target=nvptx-nvidia-cuda -nogpulib -nogpuinc -c %s 2>&1 \
+// RUN: not %clang -### --no-offload-new-driver --cuda-host-only --target=nvptx-nvidia-cuda -nogpulib -nogpuinc -c %s 2>&1 \
 // RUN: | FileCheck -check-prefix HOST_NVPTX %s
-// RUN: not %clang -### --cuda-host-only --target=amdgcn-amd-amdhsa -nogpulib -nogpuinc -c %s 2>&1 \
+// RUN: not %clang -### --no-offload-new-driver --cuda-host-only --target=amdgcn-amd-amdhsa -nogpulib -nogpuinc -c %s 2>&1 \
 // RUN: | FileCheck -check-prefix HOST_AMDGCN %s
 
 // OK-NOT: error: Unsupported CUDA gpu architecture
diff --git a/clang/test/Driver/cuda-external-tools.cu b/clang/test/Driver/cuda-external-tools.cu
index 1aa87cc09982..946e144fce38 100644
--- a/clang/test/Driver/cuda-external-tools.cu
+++ b/clang/test/Driver/cuda-external-tools.cu
@@ -25,7 +25,7 @@
 // RUN: | FileCheck -check-prefixes=CHECK,ARCH64,SM35,OPT3 %s
 // Generating relocatable device code
 // RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -c %s 2>&1 \
-// RUN:   --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
+// RUN:   --no-offload-new-driver --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
 // RUN: | FileCheck -check-prefixes=CHECK,ARCH64,SM35,RDC %s
 
 // With debugging enabled, ptxas should be run with with no ptxas optimizations.
@@ -59,7 +59,7 @@
 // RUN: | FileCheck -check-prefixes=CHECK,ARCH64,SM35 %s
 // Separate compilation targeting sm_35.
 // RUN: %clang -### --target=x86_64-linux-gnu --cuda-gpu-arch=sm_35 -fgpu-rdc -c %s 2>&1 \
-// RUN:   --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
+// RUN:   --no-offload-new-driver --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
 // RUN: | FileCheck -check-prefixes=CHECK,ARCH64,SM35,RDC %s
 
 // 32-bit compile.
@@ -68,7 +68,7 @@
 // RUN: | FileCheck -check-prefixes=CHECK,ARCH32,SM35 %s
 // 32-bit compile when generating relocatable device code.
 // RUN: %clang -### --target=i386-linux-gnu -fgpu-rdc -c %s 2>&1 \
-// RUN:   --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
+// RUN:   --no-offload-new-driver --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
 // RUN: | FileCheck -check-prefixes=CHECK,ARCH32,SM35,RDC %s
 
 // Compile with -fintegrated-as.  This should still cause us to invoke ptxas.
@@ -77,7 +77,7 @@
 // RUN: | FileCheck -check-prefixes=CHECK,ARCH64,SM35,OPT0 %s
 // Check that we still pass -c when generating relocatable device code.
 // RUN: %clang -### --target=x86_64-linux-gnu -fintegrated-as -fgpu-rdc -c %s 2>&1 \
-// RUN:   --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
+// RUN:   --no-offload-new-driver --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
 // RUN: | FileCheck -check-prefixes=CHECK,ARCH64,SM35,RDC %s
 
 // Check -Xcuda-ptxas and -Xcuda-fatbinary
@@ -99,13 +99,13 @@
 
 // Check relocatable device code generation on MacOS.
 // RUN: %clang -### --target=x86_64-apple-macosx -O0 -fgpu-rdc -c %s 2>&1 \
-// RUN:   --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
+// RUN:   --no-offload-new-driver --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
 // RUN: | FileCheck -check-prefixes=CHECK,ARCH64,SM35,RDC %s
 // RUN: %clang -### --target=x86_64-apple-macosx --cuda-gpu-arch=sm_35 -fgpu-rdc -c %s 2>&1 \
-// RUN:   --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
+// RUN:   --no-offload-new-driver --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
 // RUN: | FileCheck -check-prefixes=CHECK,ARCH64,SM35,RDC %s
 // RUN: %clang -### --target=i386-apple-macosx -fgpu-rdc -c %s 2>&1 \
-// RUN:   --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
+// RUN:   --no-offload-new-driver --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
 // RUN: | FileCheck -check-prefixes=CHECK,ARCH32,SM35,RDC %s
 
 // Check that CLANG forwards the -v flag to PTXAS.
diff --git a/clang/test/Driver/cuda-phases.cu b/clang/test/Driver/cuda-phases.cu
index a1c3c9b51b1e..85b1a550524d 100644
--- a/clang/test/Driver/cuda-phases.cu
+++ b/clang/test/Driver/cuda-phases.cu
@@ -11,7 +11,7 @@
 //
 // Test CUDA NVPTX phases.
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 %s 2>&1 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 %s 2>&1 \
 // RUN: | FileCheck -check-prefixes=BIN %s
 //
 // BIN-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (host-[[T]])
@@ -34,7 +34,7 @@
 // Test single gpu architecture up to the assemble phase.
 //
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 %s -S 2>&1 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 %s -S 2>&1 \
 // RUN: | FileCheck -check-prefixes=ASM %s
 // ASM-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (device-[[T]], [[ARCH:sm_30]])
 // ASM-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH]])
@@ -50,7 +50,7 @@
 // Test two gpu architectures with complete compilation.
 //
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s 2>&1 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s 2>&1 \
 // RUN: | FileCheck -check-prefixes=BIN2 %s
 // BIN2-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (host-[[T]])
 // BIN2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]])
@@ -79,7 +79,7 @@
 // Test two gpu architecturess up to the assemble phase.
 //
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s -S 2>&1 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s -S 2>&1 \
 // RUN: | FileCheck -check-prefixes=ASM2 %s
 // ASM2-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (device-[[T]], [[ARCH1:sm_30]])
 // ASM2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH1]])
@@ -101,7 +101,7 @@
 // compilation mode.
 //
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 %s --cuda-host-only 2>&1 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 %s --cuda-host-only 2>&1 \
 // RUN: | FileCheck -check-prefixes=HBIN %s
 // HBIN-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (host-[[T]])
 // HBIN-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]])
@@ -115,7 +115,7 @@
 // compilation mode.
 //
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 %s --cuda-host-only -S 2>&1 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 %s --cuda-host-only -S 2>&1 \
 // RUN: | FileCheck -check-prefixes=HASM %s
 // HASM-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (host-[[T]])
 // HASM-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]])
@@ -128,7 +128,7 @@
 // compilation mode.
 //
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s --cuda-host-only 2>&1 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s --cuda-host-only 2>&1 \
 // RUN: | FileCheck -check-prefixes=HBIN2 %s
 // HBIN2-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (host-[[T]])
 // HBIN2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]])
@@ -143,7 +143,7 @@
 // compilation mode.
 //
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s --cuda-host-only -S \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s --cuda-host-only -S \
 // RUN: 2>&1 | FileCheck -check-prefixes=HASM2 %s
 // HASM2-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (host-[[T]])
 // HASM2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]])
@@ -156,7 +156,7 @@
 // compilation mode.
 //
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 %s --cuda-device-only 2>&1 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 %s --cuda-device-only 2>&1 \
 // RUN: | FileCheck -check-prefixes=DBIN %s
 // DBIN-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (device-[[T]], [[ARCH:sm_30]])
 // DBIN-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH]])
@@ -170,7 +170,7 @@
 // compilation mode.
 //
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 %s --cuda-device-only -S 2>&1 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 %s --cuda-device-only -S 2>&1 \
 // RUN: | FileCheck -check-prefixes=DASM %s
 // DASM-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (device-[[T]], [[ARCH:sm_30]])
 // DASM-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH]])
@@ -184,7 +184,7 @@
 // compilation mode.
 //
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s --cuda-device-only 2>&1 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s --cuda-device-only 2>&1 \
 // RUN: | FileCheck -check-prefixes=DBIN2 %s
 // DBIN2-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (device-[[T]], [[ARCH:sm_30]])
 // DBIN2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH]])
@@ -204,7 +204,7 @@
 // compilation mode.
 //
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s --cuda-device-only -S \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s --cuda-device-only -S \
 // RUN: 2>&1 | FileCheck -check-prefixes=DASM2 %s
 // DASM2-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (device-[[T]], [[ARCH:sm_30]])
 // DASM2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH]])
diff --git a/clang/test/Driver/hip-binding.hip b/clang/test/Driver/hip-binding.hip
index c48397168a60..79ec2039edb7 100644
--- a/clang/test/Driver/hip-binding.hip
+++ b/clang/test/Driver/hip-binding.hip
@@ -3,10 +3,10 @@
 
 // RUN: %clang -ccc-print-bindings --target=x86_64-linux-gnu \
 // RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
-// RUN:   -c 2>&1 | FileCheck -check-prefix=NRDCS %s
+// RUN:   --no-offload-new-driver -c 2>&1 | FileCheck -check-prefix=NRDCS %s
 // RUN: %clang -ccc-print-bindings --target=x86_64-linux-gnu --offload-new-driver \
 // RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
-// RUN:   -c 2>&1 | FileCheck -check-prefix=NRDCS %s
+// RUN:   --no-offload-new-driver -c 2>&1 | FileCheck -check-prefix=NRDCS %s
 // NRDCS: # "amdgcn-amd-amdhsa" - "clang", inputs: ["[[IN:.*hip-binding.hip]]"], output: "[[OBJ1:.*o]]"
 // NRDCS: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[OBJ1]]"], output: "[[IMG1:.*]]"
 // NRDCS: # "amdgcn-amd-amdhsa" - "clang", inputs: ["[[IN:.*hip-binding.hip]]"], output: "[[OBJ2:.*o]]"
@@ -16,7 +16,7 @@
 
 // RUN: %clang -ccc-print-bindings --target=x86_64-linux-gnu \
 // RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
-// RUN:   -c -fgpu-rdc 2>&1 | FileCheck -check-prefix=RDCS %s
+// RUN:   --no-offload-new-driver -c -fgpu-rdc 2>&1 | FileCheck -check-prefix=RDCS %s
 // RDCS: # "amdgcn-amd-amdhsa" - "clang", inputs: ["[[IN:.*hip-binding.hip]]"], output: "[[BC1:.*bc]]"
 // RDCS: # "amdgcn-amd-amdhsa" - "clang", inputs: ["[[IN:.*hip-binding.hip]]"], output: "[[BC2:.*bc]]"
 // RDCS: # "x86_64-unknown-linux-gnu" - "clang", inputs: ["[[IN]]"], output: "[[HOSTOBJ:.*o]]"
@@ -32,7 +32,7 @@
 
 // RUN: touch %t.o
 // RUN: %clang --hip-link -ccc-print-bindings --target=x86_64-linux-gnu \
-// RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 -fgpu-rdc %t.o\
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 -fgpu-rdc %t.o\
 // RUN: 2>&1 | FileCheck %s
 
 // CHECK: # "x86_64-unknown-linux-gnu" - "offload bundler", inputs: ["[[IN:.*o]]"], outputs: ["[[HOSTOBJ:.*o]]", "{{.*o}}", "{{.*o}}"]
@@ -46,7 +46,7 @@
 // CHECK: # "x86_64-unknown-linux-gnu" - "GNU::Linker", inputs: ["[[HOSTOBJ]]", "[[FATBINOBJ]]"], output: "a.out"
 
 // RUN: %clang --hip-link -ccc-print-bindings --target=x86_64-linux-gnu \
-// RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t.o\
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t.o\
 // RUN: 2>&1 | FileCheck -check-prefix=NORDC %s
 
 // NORDC-NOT: offload bundler
diff --git a/clang/test/Driver/hip-cuid-hash.hip b/clang/test/Driver/hip-cuid-hash.hip
index 1b4d26c471c1..ef2a32a69c8f 100644
--- a/clang/test/Driver/hip-cuid-hash.hip
+++ b/clang/test/Driver/hip-cuid-hash.hip
@@ -4,11 +4,11 @@
 // Check CUID generated by hash.
 // The same CUID is generated for the same file with the same options.
 
-// RUN: %clang -### -x hip --target=x86_64-unknown-linux-gnu \
+// RUN: %clang -### -x hip --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN:   --offload-arch=gfx906 -c -nogpuinc -nogpulib -fuse-cuid=hash \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu >%t.out 2>&1
 
-// RUN: %clang -### -x hip --target=x86_64-unknown-linux-gnu \
+// RUN: %clang -### -x hip --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN:   --offload-arch=gfx906 -c -nogpuinc -nogpulib -fuse-cuid=hash \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu >>%t.out 2>&1
 
@@ -17,11 +17,11 @@
 // Check CUID generated by hash.
 // Different CUID's are generated for the same file with different options.
 
-// RUN: %clang -### -x hip --target=x86_64-unknown-linux-gnu -DX=1 \
+// RUN: %clang -### -x hip --target=x86_64-unknown-linux-gnu -DX=1 --no-offload-new-driver \
 // RUN:   --offload-arch=gfx906 -c -nogpuinc -nogpulib -fuse-cuid=hash \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu >%t.out 2>&1
 
-// RUN: %clang -### -x hip --target=x86_64-unknown-linux-gnu -DX=2 \
+// RUN: %clang -### -x hip --target=x86_64-unknown-linux-gnu -DX=2 --no-offload-new-driver \
 // RUN:   --offload-arch=gfx906 -c -nogpuinc -nogpulib -fuse-cuid=hash \
 // RUN:   %S/Inputs/../Inputs/hip_multiple_inputs/a.cu >>%t.out 2>&1
 
diff --git a/clang/test/Driver/hip-cuid.hip b/clang/test/Driver/hip-cuid.hip
index 421810b824fd..ce3d2de3501e 100644
--- a/clang/test/Driver/hip-cuid.hip
+++ b/clang/test/Driver/hip-cuid.hip
@@ -5,6 +5,7 @@
 
 // RUN: not %clang -### -x hip \
 // RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver \
 // RUN:   --offload-arch=gfx900 \
 // RUN:   --offload-arch=gfx906 \
 // RUN:   -c -nogpuinc -nogpulib -fuse-cuid=invalid \
@@ -16,6 +17,7 @@
 
 // RUN: %clang -### -x hip \
 // RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver \
 // RUN:   --offload-arch=gfx900 \
 // RUN:   --offload-arch=gfx906 \
 // RUN:   -c -nogpuinc -nogpulib -fuse-cuid=random \
@@ -27,6 +29,7 @@
 
 // RUN: %clang -### -x hip \
 // RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver \
 // RUN:   --offload-arch=gfx900 \
 // RUN:   --offload-arch=gfx906 \
 // RUN:   -c -nogpuinc -nogpulib -cuid=xyz_123 \
@@ -38,6 +41,7 @@
 
 // RUN: %clang -### -x hip \
 // RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver \
 // RUN:   --offload-arch=gfx900 \
 // RUN:   --offload-arch=gfx906 \
 // RUN:   -c -nogpuinc -nogpulib -fuse-cuid=random -cuid=xyz_123 \
@@ -49,6 +53,7 @@
 
 // RUN: %clang -### -x hip \
 // RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver \
 // RUN:   --offload-arch=gfx900 \
 // RUN:   --offload-arch=gfx906 \
 // RUN:   -c -nogpuinc -nogpulib -fuse-cuid=hash \
diff --git a/clang/test/Driver/hip-dependent-options.hip b/clang/test/Driver/hip-dependent-options.hip
index 405b092d0f83..b0dc7f289e81 100644
--- a/clang/test/Driver/hip-dependent-options.hip
+++ b/clang/test/Driver/hip-dependent-options.hip
@@ -1,4 +1,4 @@
-// RUN: not %clang -### --target=x86_64-linux-gnu \
+// RUN: not %clang -### --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -x hip --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -c -fhip-emit-relocatable -nogpuinc -nogpulib --cuda-device-only -fgpu-rdc \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
@@ -7,7 +7,7 @@
 
 // RELOCRDC: error: option '-fhip-emit-relocatable' cannot be specified with '-fgpu-rdc'
 
-// RUN: not %clang -### --target=x86_64-linux-gnu \
+// RUN: not %clang -### --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -x hip --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -c -fhip-emit-relocatable -nogpuinc -nogpulib \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
diff --git a/clang/test/Driver/hip-device-compile.hip b/clang/test/Driver/hip-device-compile.hip
index 3c3e38785624..74be9c6cf2ee 100644
--- a/clang/test/Driver/hip-device-compile.hip
+++ b/clang/test/Driver/hip-device-compile.hip
@@ -8,7 +8,7 @@
 // Output unbundled bitcode.
 // RUN: %clang -c -emit-llvm --cuda-device-only -### --target=x86_64-linux-gnu \
 // RUN:   --rocm-path=%S/Inputs/rocm -o a.bc -x hip --cuda-gpu-arch=gfx900 --no-gpu-bundle-output \
-// RUN:   --hip-device-lib=lib1.bc \
+// RUN:   --no-offload-new-driver --hip-device-lib=lib1.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN: 2>&1 | FileCheck -check-prefixes=CHECK,BC,NBUN %s
@@ -16,7 +16,7 @@
 // Output bundled bitcode.
 // RUN: %clang -c -emit-llvm --cuda-device-only -### --target=x86_64-linux-gnu \
 // RUN:   --rocm-path=%S/Inputs/rocm -o a.bc -x hip --cuda-gpu-arch=gfx900 --no-gpu-bundle-output \
-// RUN:   --hip-device-lib=lib1.bc \
+// RUN:   --no-offload-new-driver --hip-device-lib=lib1.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu --gpu-bundle-output \
 // RUN: 2>&1 | FileCheck -check-prefixes=CHECK,BCBUN %s
@@ -24,7 +24,7 @@
 // Output unbundled LLVM IR.
 // RUN: %clang -c -S -emit-llvm --cuda-device-only -### --target=x86_64-linux-gnu \
 // RUN:   --rocm-path=%S/Inputs/rocm -o a.ll -x hip --cuda-gpu-arch=gfx900 --no-gpu-bundle-output \
-// RUN:   --hip-device-lib=lib1.bc \
+// RUN:   --no-offload-new-driver --hip-device-lib=lib1.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN: 2>&1 | FileCheck -check-prefixes=CHECK,LL,NBUN %s
@@ -32,7 +32,7 @@
 // Output bundled LLVM IR.
 // RUN: %clang -c -S -emit-llvm --cuda-device-only -### --target=x86_64-linux-gnu \
 // RUN:   --rocm-path=%S/Inputs/rocm -o a.ll -x hip --cuda-gpu-arch=gfx900 --no-gpu-bundle-output \
-// RUN:   --hip-device-lib=lib1.bc \
+// RUN:   --no-offload-new-driver --hip-device-lib=lib1.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu --gpu-bundle-output \
 // RUN: 2>&1 | FileCheck -check-prefixes=CHECK,LLBUN %s
@@ -40,7 +40,7 @@
 // Output unbundled assembly.
 // RUN: %clang -c -S --cuda-device-only -### --target=x86_64-linux-gnu \
 // RUN:   --rocm-path=%S/Inputs/rocm -o a.s -x hip --cuda-gpu-arch=gfx900 --no-gpu-bundle-output \
-// RUN:   --hip-device-lib=lib1.bc \
+// RUN:   --no-offload-new-driver --hip-device-lib=lib1.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN: 2>&1 | FileCheck -check-prefixes=CHECK,ASM,NBUN %s
@@ -48,7 +48,7 @@
 // Output relocatable.
 // RUN: %clang -c --cuda-device-only -### --target=x86_64-linux-gnu \
 // RUN:   --rocm-path=%S/Inputs/rocm -o a.o -x hip --cuda-gpu-arch=gfx900 -fhip-emit-relocatable \
-// RUN:   --hip-device-lib=lib1.bc \
+// RUN:   --no-offload-new-driver --hip-device-lib=lib1.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN: 2>&1 | FileCheck -check-prefixes=CHECK,NBUN,RELOC %s
@@ -56,7 +56,7 @@
 // Output bundled assembly.
 // RUN: %clang -c -S --cuda-device-only -### --target=x86_64-linux-gnu \
 // RUN:   --rocm-path=%S/Inputs/rocm -o a.s -x hip --cuda-gpu-arch=gfx900 --no-gpu-bundle-output \
-// RUN:   --hip-device-lib=lib1.bc \
+// RUN:   --no-offload-new-driver --hip-device-lib=lib1.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu --gpu-bundle-output \
 // RUN: 2>&1 | FileCheck -check-prefixes=CHECK,ASMBUN %s
@@ -96,7 +96,7 @@
 // Output bundled code objects.
 // RUN: %clang -c --cuda-device-only -### --target=x86_64-linux-gnu \
 // RUN:   --rocm-path=%S/Inputs/rocm -o a.o -x hip --cuda-gpu-arch=gfx900 \
-// RUN:   --hip-device-lib=lib1.bc \
+// RUN:   --no-offload-new-driver --hip-device-lib=lib1.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN: 2>&1 | FileCheck -check-prefixes=OBJ,OBJ-BUN %s
@@ -104,7 +104,7 @@
 // Output unbundled code objects.
 // RUN: %clang -c --cuda-device-only -### --target=x86_64-linux-gnu \
 // RUN:   --rocm-path=%S/Inputs/rocm -o a.o -x hip --cuda-gpu-arch=gfx900 \
-// RUN:   --hip-device-lib=lib1.bc \
+// RUN:   --no-offload-new-driver --hip-device-lib=lib1.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu --no-gpu-bundle-output \
 // RUN: 2>&1 | FileCheck -check-prefixes=OBJ,OBJ-UBUN %s
@@ -112,7 +112,7 @@
 // Output bundled code objects.
 // RUN: %clang --cuda-device-only -### --target=x86_64-linux-gnu \
 // RUN:   --rocm-path=%S/Inputs/rocm -o a.o -x hip --cuda-gpu-arch=gfx900 \
-// RUN:   --hip-device-lib=lib1.bc \
+// RUN:   --no-offload-new-driver --hip-device-lib=lib1.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN: 2>&1 | FileCheck -check-prefixes=OBJ,OBJ-BUN %s
@@ -120,7 +120,7 @@
 // Output unbundled code objects.
 // RUN: %clang --cuda-device-only -### --target=x86_64-linux-gnu \
 // RUN:   --rocm-path=%S/Inputs/rocm -o a.o -x hip --cuda-gpu-arch=gfx900 \
-// RUN:   --hip-device-lib=lib1.bc \
+// RUN:   --no-offload-new-driver --hip-device-lib=lib1.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu --no-gpu-bundle-output \
 // RUN: 2>&1 | FileCheck -check-prefixes=OBJ,OBJ-UBUN %s
diff --git a/clang/test/Driver/hip-link-bc-to-bc.hip b/clang/test/Driver/hip-link-bc-to-bc.hip
index 52eab97bcebb..7b232f2656a9 100644
--- a/clang/test/Driver/hip-link-bc-to-bc.hip
+++ b/clang/test/Driver/hip-link-bc-to-bc.hip
@@ -7,7 +7,7 @@
 
 // RUN: %clang -### --target=x86_64-unknown-linux-gnu --offload-arch=gfx906 --hip-link \
 // RUN:   -nogpulib -nogpuinc -emit-llvm -fgpu-rdc --cuda-device-only \
-// RUN:   %t/bundle1.bc %t/bundle2.bc \
+// RUN:   --no-offload-new-driver %t/bundle1.bc %t/bundle2.bc \
 // RUN:   2>&1 | FileCheck -check-prefix=BITCODE %s
 
 // BITCODE: "{{.*}}clang-offload-bundler" "-type=bc" "-targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa-gfx906" "-input={{.*}}bundle1.bc" "-output=[[B1HOST:.*\.bc]]" "-output=[[B1DEV1:.*\.bc]]" "-unbundle" "-allow-missing-bundles"
@@ -24,7 +24,7 @@
 
 // RUN: %clang -### --target=x86_64-unknown-linux-gnu --offload-arch=gfx906 --hip-link \
 // RUN:   -nogpulib -nogpuinc -emit-llvm -fgpu-rdc --cuda-device-only \
-// RUN:   %t/bundle.bc -L%t -lhipbundle \
+// RUN:   --no-offload-new-driver %t/bundle.bc -L%t -lhipbundle \
 // RUN:   2>&1 | FileCheck -check-prefix=ARCHIVE %s
 
 // ARCHIVE: "{{.*}}clang-offload-bundler" "-type=bc" "-targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa-gfx906" "-input={{.*}}bundle.bc" "-output=[[HOST:.*\.bc]]" "-output=[[DEV1:.*\.bc]]" "-unbundle" "-allow-missing-bundles"
diff --git a/clang/test/Driver/hip-link-bundle-archive.hip b/clang/test/Driver/hip-link-bundle-archive.hip
index dd1d779fe19a..cfbf71372260 100644
--- a/clang/test/Driver/hip-link-bundle-archive.hip
+++ b/clang/test/Driver/hip-link-bundle-archive.hip
@@ -9,23 +9,23 @@
 // RUN: touch %t/dummy.bc
 // RUN: llvm-ar cr %t/libhipBundled.a %t/dummy.bc
 // RUN: %clang -### --offload-arch=gfx906 --offload-arch=gfx1030 \
-// RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver --target=x86_64-unknown-linux-gnu \
 // RUN:   -nogpuinc -nogpulib %s -fgpu-rdc -L%t -lhipBundled \
 // RUN:   2>&1 | FileCheck -check-prefixes=GNU,GNU1,GNU-L %s
 
 // RUN: %clang -### --offload-arch=gfx906 --offload-arch=gfx1030 -nogpuinc \
-// RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver --target=x86_64-unknown-linux-gnu \
 // RUN:   -nogpuinc -nogpulib %s -fgpu-rdc -L%t -l:libhipBundled.a \
 // RUN:   2>&1 | FileCheck -check-prefixes=GNU,GNU1,GNU-LA %s
 
 // RUN: %clang -### --offload-arch=gfx906 --offload-arch=gfx1030 \
-// RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver --target=x86_64-unknown-linux-gnu \
 // RUN:   -nogpuinc -nogpulib %s -fgpu-rdc %t/libhipBundled.a \
 // RUN:   2>&1 | FileCheck -check-prefixes=GNU,GNU1,GNU-A %s
 
 // RUN: llvm-ar cr %t/libhipBundled.a.5.2 %t/dummy.bc
 // RUN: %clang -### --offload-arch=gfx906 --offload-arch=gfx1030 \
-// RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver --target=x86_64-unknown-linux-gnu \
 // RUN:   -nogpuinc -nogpulib %s -fgpu-rdc %t/libhipBundled.a.5.2 \
 // RUN:   2>&1 | FileCheck -check-prefixes=GNU,GNU2,GNU-A %s
 
@@ -33,22 +33,22 @@
 
 // RUN: touch %t/libNonArchive.a
 // RUN: %clang -### --offload-arch=gfx906 --offload-arch=gfx1030 \
-// RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver --target=x86_64-unknown-linux-gnu \
 // RUN:   -nogpuinc -nogpulib %s -fgpu-rdc -L%t -lNonArchive \
 // RUN:   2>&1 | FileCheck -check-prefixes=NONARCHIVE %s
 // RUN: %clang -### --offload-arch=gfx906 --offload-arch=gfx1030 \
-// RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver --target=x86_64-unknown-linux-gnu \
 // RUN:   -nogpuinc -nogpulib %s -fgpu-rdc -L%t -l:libNonArchive.a \
 // RUN:   2>&1 | FileCheck -check-prefixes=NONARCHIVE %s
 // RUN: not %clang -### --offload-arch=gfx906 --offload-arch=gfx1030 \
-// RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver --target=x86_64-unknown-linux-gnu \
 // RUN:   -nogpuinc -nogpulib %s -fgpu-rdc -L%t libNonArchive.a \
 // RUN:   2>&1 | FileCheck -check-prefixes=NONARCHIVE %s
 
 // Check if a file does not exist, it is not unbundled.
 
 // RUN: not %clang -### --offload-arch=gfx906 --offload-arch=gfx1030 \
-// RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver --target=x86_64-unknown-linux-gnu \
 // RUN:   -nogpuinc -nogpulib %s -fgpu-rdc %t/NoneExist.a \
 // RUN:   2>&1 | FileCheck -check-prefixes=NONE %s
 
@@ -56,17 +56,17 @@
 
 // RUN: llvm-ar cr %t/hipBundled2.lib %t/dummy.bc
 // RUN: %clang -### --offload-arch=gfx906 --offload-arch=gfx1030 \
-// RUN:   --target=x86_64-pc-windows-msvc -fuse-ld= \
+// RUN:   --no-offload-new-driver --target=x86_64-pc-windows-msvc -fuse-ld= \
 // RUN:   -nogpuinc -nogpulib %s -fgpu-rdc -L%t -lhipBundled2 \
 // RUN:   2>&1 | FileCheck -check-prefix=MSVC %s
 
 // RUN: %clang -### --offload-arch=gfx906 --offload-arch=gfx1030 \
-// RUN:   --target=x86_64-pc-windows-msvc -fuse-ld= \
+// RUN:   --no-offload-new-driver --target=x86_64-pc-windows-msvc -fuse-ld= \
 // RUN:   -nogpuinc -nogpulib %s -fgpu-rdc -L%t -l:hipBundled2.lib \
 // RUN:   2>&1 | FileCheck -check-prefix=MSVC %s
 
 // RUN: %clang -### --offload-arch=gfx906 --offload-arch=gfx1030 \
-// RUN:   --target=x86_64-pc-windows-msvc -fuse-ld= \
+// RUN:   --no-offload-new-driver --target=x86_64-pc-windows-msvc -fuse-ld= \
 // RUN:   -nogpuinc -nogpulib %s -fgpu-rdc %t/hipBundled2.lib \
 // RUN:   2>&1 | FileCheck -check-prefix=MSVC %s
 
diff --git a/clang/test/Driver/hip-link-save-temps.hip b/clang/test/Driver/hip-link-save-temps.hip
index b8a5dcefb8cf..e54be63c578b 100644
--- a/clang/test/Driver/hip-link-save-temps.hip
+++ b/clang/test/Driver/hip-link-save-temps.hip
@@ -7,14 +7,14 @@
 // RUN: touch %t/obj2.o
 // RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -save-temps \
 // RUN:   --hip-link -o executable -fgpu-rdc --cuda-gpu-arch=gfx900 \
-// RUN:   --offload-arch=gfx906  %t/obj1.o %t/obj2.o 2>&1 | \
+// RUN:   --no-offload-new-driver --offload-arch=gfx906  %t/obj1.o %t/obj2.o 2>&1 | \
 // RUN:   FileCheck -check-prefixes=CHECK,OUT %s
 
 // -fgpu-rdc link without output
 // RUN: touch %t/obj1.o
 // RUN: touch %t/obj2.o
 // RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -save-temps \
-// RUN:   --hip-link -fgpu-rdc --cuda-gpu-arch=gfx900 \
+// RUN:   --no-offload-new-driver --hip-link -fgpu-rdc --cuda-gpu-arch=gfx900 \
 // RUN:   --offload-arch=gfx906  %t/obj1.o %t/obj2.o 2>&1 | \
 // RUN:   FileCheck -check-prefixes=CHECK,NOUT %s
 
@@ -23,7 +23,7 @@
 // RUN: touch %t/obj2.o
 // RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -save-temps \
 // RUN:   --hip-link -o libTest.a -fgpu-rdc --cuda-gpu-arch=gfx900 \
-// RUN:   --emit-static-lib \
+// RUN:   --no-offload-new-driver --emit-static-lib \
 // RUN:   --offload-arch=gfx906  %t/obj1.o %t/obj2.o 2>&1 | \
 // RUN:   FileCheck -check-prefixes=CHECK,SLO %s
 
@@ -32,7 +32,7 @@
 // RUN: touch %t/obj2.o
 // RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -save-temps \
 // RUN:   --hip-link -fgpu-rdc --cuda-gpu-arch=gfx900 \
-// RUN:   --emit-static-lib \
+// RUN:   --no-offload-new-driver --emit-static-lib \
 // RUN:   --offload-arch=gfx906  %t/obj1.o %t/obj2.o 2>&1 | \
 // RUN:   FileCheck -check-prefixes=CHECK,SLNO %s
 
diff --git a/clang/test/Driver/hip-link-shared-library.hip b/clang/test/Driver/hip-link-shared-library.hip
index fc1f9f53b954..73643682dda8 100644
--- a/clang/test/Driver/hip-link-shared-library.hip
+++ b/clang/test/Driver/hip-link-shared-library.hip
@@ -1,7 +1,7 @@
 // RUN: touch %t.o
 // RUN: %clang --hip-link -ccc-print-bindings --target=x86_64-linux-gnu \
 // RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t.o %S/Inputs/in.so \
-// RUN:   -fgpu-rdc 2>&1 | FileCheck %s
+// RUN:   --no-offload-new-driver -fgpu-rdc 2>&1 | FileCheck %s
 
 // CHECK: # "x86_64-unknown-linux-gnu" - "offload bundler", inputs: ["[[IN:.*o]]"], outputs: ["[[HOSTOBJ:.*o]]", "{{.*o}}", "{{.*o}}"]
 // CHECK: # "amdgcn-amd-amdhsa" - "offload bundler", inputs: ["[[IN]]"], outputs: ["{{.*o}}", "[[DOBJ1:.*o]]", "[[DOBJ2:.*o]]"]
@@ -12,4 +12,3 @@
 // CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[IMG1]]", "[[IMG2]]"], output: "[[FATBINOBJ:.*o]]"
 // CHECK-NOT: offload bundler
 // CHECK: # "x86_64-unknown-linux-gnu" - "GNU::Linker", inputs: ["[[HOSTOBJ]]", "{{.*}}/Inputs/in.so", "[[FATBINOBJ]]"], output: "a.out"
-
diff --git a/clang/test/Driver/hip-link-static-library.hip b/clang/test/Driver/hip-link-static-library.hip
index 63675ffd6262..3159b5f3984f 100644
--- a/clang/test/Driver/hip-link-static-library.hip
+++ b/clang/test/Driver/hip-link-static-library.hip
@@ -3,7 +3,7 @@
 
 // RUN: touch %t.o
 // RUN: %clang --hip-link -ccc-print-bindings --target=x86_64-linux-gnu \
-// RUN:   --emit-static-lib \
+// RUN:   --no-offload-new-driver --emit-static-lib \
 // RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 -fgpu-rdc %t.o\
 // RUN: 2>&1 | FileCheck %s
 
@@ -18,7 +18,7 @@
 // CHECK: # "x86_64-unknown-linux-gnu" - "GNU::StaticLibTool", inputs: ["[[HOSTOBJ]]", "[[FATBINOBJ]]"], output: "a.out"
 
 // RUN: %clang --hip-link -ccc-print-bindings --target=x86_64-linux-gnu \
-// RUN:   --emit-static-lib \
+// RUN:   --no-offload-new-driver --emit-static-lib \
 // RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t.o\
 // RUN: 2>&1 | FileCheck -check-prefix=NORDC %s
 
@@ -26,7 +26,7 @@
 // NORDC: # "x86_64-unknown-linux-gnu" - "GNU::StaticLibTool", inputs: ["{{.*o}}"], output: "a.out"
 
 // RUN: %clang --hip-link -### --target=x86_64-linux-gnu \
-// RUN:   --emit-static-lib -lgcc \
+// RUN:   --no-offload-new-driver --emit-static-lib -lgcc \
 // RUN:   -Wl,--enable-new-dtags -Wl,--rpath=/opt \
 // RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 -fgpu-rdc %t.o\
 // RUN: 2>&1 | FileCheck -check-prefix=NOFLAG %s
diff --git a/clang/test/Driver/hip-offload-compress-zlib.hip b/clang/test/Driver/hip-offload-compress-zlib.hip
index 7557fdde8786..7a269c566bb9 100644
--- a/clang/test/Driver/hip-offload-compress-zlib.hip
+++ b/clang/test/Driver/hip-offload-compress-zlib.hip
@@ -7,7 +7,7 @@
 // RUN: rm -rf %t.bc
 // RUN: %clang -c -v --target=x86_64-linux-gnu \
 // RUN:   -x hip --offload-arch=gfx1100 --offload-arch=gfx1101 \
-// RUN:   -fgpu-rdc -nogpuinc -nogpulib \
+// RUN:   --no-offload-new-driver -fgpu-rdc -nogpuinc -nogpulib \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN:   --offload-compress --offload-device-only --gpu-bundle-output \
 // RUN:   -o %t.bc \
@@ -22,7 +22,7 @@
 
 // RUN: %clang --hip-link -### -v --target=x86_64-linux-gnu \
 // RUN:   --offload-arch=gfx1100 --offload-arch=gfx1101 \
-// RUN:   -fgpu-rdc -nogpulib \
+// RUN:   --no-offload-new-driver -fgpu-rdc -nogpulib \
 // RUN:   %t.bc --offload-device-only \
 // RUN: 2>&1 | FileCheck -check-prefix=UNBUNDLE %s
 
@@ -35,7 +35,7 @@
 
 // RUN: %clang -c -### -v --target=x86_64-linux-gnu \
 // RUN:   -x hip --offload-arch=gfx1100 --offload-arch=gfx1101 \
-// RUN:   -nogpuinc -nogpulib \
+// RUN:   --no-offload-new-driver -nogpuinc -nogpulib \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN:   --offload-compress \
 // RUN: 2>&1 | FileCheck -check-prefix=CO %s
diff --git a/clang/test/Driver/hip-offload-compress-zstd.hip b/clang/test/Driver/hip-offload-compress-zstd.hip
index 3680ae47974a..fa7fb3b6d5b5 100644
--- a/clang/test/Driver/hip-offload-compress-zstd.hip
+++ b/clang/test/Driver/hip-offload-compress-zstd.hip
@@ -7,7 +7,7 @@
 // RUN: rm -rf %t.bc
 // RUN: %clang -c -v --target=x86_64-linux-gnu \
 // RUN:   -x hip --offload-arch=gfx1100 --offload-arch=gfx1101 \
-// RUN:   -fgpu-rdc -nogpuinc -nogpulib \
+// RUN:   --no-offload-new-driver -fgpu-rdc -nogpuinc -nogpulib \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN:   --offload-compress --offload-device-only --gpu-bundle-output \
 // RUN:   -o %t.bc \
@@ -22,7 +22,7 @@
 
 // RUN: %clang --hip-link -### -v --target=x86_64-linux-gnu \
 // RUN:   --offload-arch=gfx1100 --offload-arch=gfx1101 \
-// RUN:   -fgpu-rdc -nogpulib \
+// RUN:   --no-offload-new-driver -fgpu-rdc -nogpulib \
 // RUN:   %t.bc --offload-device-only \
 // RUN: 2>&1 | FileCheck -check-prefix=UNBUNDLE %s
 
@@ -35,7 +35,7 @@
 
 // RUN: %clang -c -### -v --target=x86_64-linux-gnu \
 // RUN:   -x hip --offload-arch=gfx1100 --offload-arch=gfx1101 \
-// RUN:   -nogpuinc -nogpulib \
+// RUN:   --no-offload-new-driver -nogpuinc -nogpulib \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN:   --offload-compress \
 // RUN: 2>&1 | FileCheck -check-prefix=CO %s
diff --git a/clang/test/Driver/hip-output-file-name.hip b/clang/test/Driver/hip-output-file-name.hip
index 746678b81e25..aca64346e0c4 100644
--- a/clang/test/Driver/hip-output-file-name.hip
+++ b/clang/test/Driver/hip-output-file-name.hip
@@ -2,7 +2,7 @@
 // REQUIRES: amdgpu-registered-target
 
 // Output bundled code objects for combined compilation.
-// RUN: %clang -### -c --target=x86_64-linux-gnu -fgpu-rdc \
+// RUN: %clang -### -c --target=x86_64-linux-gnu -fgpu-rdc --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 --offload-arch=gfx900 %s \
 // RUN: 2>&1 | FileCheck %s
 
@@ -13,42 +13,42 @@
 // is used to bundle the final output.
 
 // Output bundled PPE for one GPU for mixed compliation.
-// RUN: %clang -### -E --target=x86_64-linux-gnu \
+// RUN: %clang -### -E --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 %s \
 // RUN: 2>&1 | FileCheck -check-prefixes=DASH %s
 
 // Output unbundled PPE for one GPU for device only compilation.
 // RUN: %clang -### -E --offload-device-only --target=x86_64-linux-gnu \
-// RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 %s \
+// RUN:   --no-offload-new-driver -nogpulib -nogpuinc --offload-arch=gfx803 %s \
 // RUN: 2>&1 | FileCheck -check-prefixes=CLANG-DASH %s
 
 // Output bundled PPE for two GPUs for mixed compilation.
-// RUN: %clang -### -E --target=x86_64-linux-gnu \
+// RUN: %clang -### -E --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 --offload-arch=gfx900 %s \
 // RUN: 2>&1 | FileCheck -check-prefixes=DASH %s
 
 // Output bundled PPE for two GPUs for mixed compilation with -save-temps.
-// RUN: %clang -### -E -save-temps --target=x86_64-linux-gnu \
+// RUN: %clang -### -E -save-temps --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 --offload-arch=gfx900 %s \
 // RUN: 2>&1 | FileCheck -check-prefixes=DASH %s
 
 // Output unbundled PPE for two GPUs for device only compilation.
-// RUN: %clang -### -E --offload-device-only --target=x86_64-linux-gnu \
+// RUN: %clang -### -E --offload-device-only --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 --offload-arch=gfx900 %s \
 // RUN: 2>&1 | FileCheck -check-prefixes=CLANG-DASH %s
 
 // Output bundled PPE for two GPUs for device only compilation with --gpu-bundle-output.
-// RUN: %clang -### -E --offload-device-only --target=x86_64-linux-gnu \
+// RUN: %clang -### -E --offload-device-only --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 --offload-arch=gfx900 %s --gpu-bundle-output \
 // RUN: 2>&1 | FileCheck -check-prefixes=DASH %s
 
 // Output unbundled PPE for two GPUs for device only compilation with --no-gpu-bundle-output.
-// RUN: %clang -### -E --offload-device-only --target=x86_64-linux-gnu \
+// RUN: %clang -### -E --offload-device-only --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 --offload-arch=gfx900 %s --no-gpu-bundle-output \
 // RUN: 2>&1 | FileCheck -check-prefixes=CLANG-DASH %s
 
 // Output unbundled PPE for host only compilation.
-// RUN: %clang -### -E --offload-host-only --target=x86_64-linux-gnu \
+// RUN: %clang -### -E --offload-host-only --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 --offload-arch=gfx900 %s \
 // RUN: 2>&1 | FileCheck -check-prefixes=CLANG-DASH %s
 
@@ -60,22 +60,22 @@
 // Check -E with -o.
 
 // Output bundled PPE for two GPUs for mixed compilation.
-// RUN: %clang -### -E -o test.cui --target=x86_64-linux-gnu \
+// RUN: %clang -### -E -o test.cui --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 --offload-arch=gfx900 %s \
 // RUN: 2>&1 | FileCheck -check-prefixes=OUT %s
 
 // Output bundled PPE for two GPUs for mixed compilation.
-// RUN: %clang -### -E -o test.cui -save-temps --target=x86_64-linux-gnu \
+// RUN: %clang -### -E -o test.cui -save-temps --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 --offload-arch=gfx900 %s \
 // RUN: 2>&1 | FileCheck -check-prefixes=OUT %s
 
 // Output bundled PPE for two GPUs for device only compilation with --gpu-bundle-output.
-// RUN: %clang -### -E -o test.cui --offload-device-only --target=x86_64-linux-gnu \
+// RUN: %clang -### -E -o test.cui --offload-device-only --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 --offload-arch=gfx900 --gpu-bundle-output %s \
 // RUN: 2>&1 | FileCheck -check-prefixes=OUT %s
 
 // Output unbundled PPE for two GPUs for device only compilation.
-// RUN: %clang -### -E -o test.cui --offload-host-only --target=x86_64-linux-gnu \
+// RUN: %clang -### -E -o test.cui --offload-host-only --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 --offload-arch=gfx900 %s \
 // RUN: 2>&1 | FileCheck -check-prefixes=CLANG-OUT %s
 
diff --git a/clang/test/Driver/hip-partial-link.hip b/clang/test/Driver/hip-partial-link.hip
index a1d31f9a6519..faa185972abc 100644
--- a/clang/test/Driver/hip-partial-link.hip
+++ b/clang/test/Driver/hip-partial-link.hip
@@ -1,14 +1,14 @@
 // REQUIRES: x86-registered-target, amdgpu-registered-target, lld, system-linux
 
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN:   --offload-arch=gfx906 -c -nostdinc -nogpuinc -nohipwrapperinc \
 // RUN:   -nogpulib -fgpu-rdc -I%S/Inputs %s -o %t.1.o
 
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -DLIB \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -DLIB --no-offload-new-driver \
 // RUN:   --offload-arch=gfx906 -c -nostdinc -nogpuinc -nohipwrapperinc \
 // RUN:   -nogpulib -fgpu-rdc -I%S/Inputs %s -o %t.2.o
 
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -DMAIN \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -DMAIN --no-offload-new-driver \
 // RUN:   --offload-arch=gfx906 -c -nostdinc -nogpuinc -nohipwrapperinc \
 // RUN:   -nogpulib -fgpu-rdc -I%S/Inputs %s -o %t.main.o
 
@@ -24,7 +24,7 @@
 
 // Link %t.1.o and %t.2.o by -r and then link with %t.main.o
 
-// RUN: %clang -v --target=x86_64-unknown-linux-gnu \
+// RUN: %clang -v --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN:   --hip-link -fgpu-rdc --offload-arch=gfx906 \
 // RUN:   -r -fuse-ld=lld -nostdlib %t.1.o %t.2.o -o %t.lib.o \
 // RUN:   2>&1 | FileCheck -check-prefix=LD-R %s
@@ -46,7 +46,7 @@
 // OBJ:  D __hip_gpubin_handle_[[ID1]]
 // OBJ:  D __hip_gpubin_handle_[[ID2]]
 
-// RUN: %clang -v --target=x86_64-unknown-linux-gnu \
+// RUN: %clang -v --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN:   --hip-link -no-hip-rt -fgpu-rdc --offload-arch=gfx906 \
 // RUN:   -fuse-ld=lld -nostdlib -r %t.main.o %t.lib.o -o %t.final.o \
 // RUN:   2>&1 | FileCheck -check-prefix=LINK-O %s
@@ -54,7 +54,7 @@
 
 // Generate a static lib with %t.1.o and %t.2.o then link with %t.main.o
 
-// RUN: %clang -v --target=x86_64-unknown-linux-gnu \
+// RUN: %clang -v --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN:   --hip-link -fgpu-rdc --offload-arch=gfx906 \
 // RUN:   --emit-static-lib -fuse-ld=lld -nostdlib %t.1.o %t.2.o -o %t.a \
 // RUN:   2>&1 | FileCheck -check-prefix=STATIC %s
@@ -68,7 +68,7 @@
 // STATIC: "{{.*}}/llvm-mc" -triple x86_64-unknown-linux-gnu
 // STATIC: "{{.*}}/llvm-ar"
 
-// RUN: %clang -v --target=x86_64-unknown-linux-gnu \
+// RUN: %clang -v --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN:   --hip-link -no-hip-rt -fgpu-rdc --offload-arch=gfx906 \
 // RUN:   -fuse-ld=lld -nostdlib -r %t.main.o %t.a -o %t.final.o \
 // RUN:   2>&1 | FileCheck -check-prefix=LINK-A %s
diff --git a/clang/test/Driver/hip-phases.hip b/clang/test/Driver/hip-phases.hip
index e976583820cc..ca63d4304d39 100644
--- a/clang/test/Driver/hip-phases.hip
+++ b/clang/test/Driver/hip-phases.hip
@@ -10,14 +10,14 @@
 // Test single gpu architecture with complete compilation.
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 %s 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 %s 2>&1 \
 // RUN: | FileCheck -check-prefixes=BIN,NRD,OLD %s
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
 // RUN: --offload-new-driver --cuda-gpu-arch=gfx803 %s 2>&1 \
 // RUN: | FileCheck -check-prefixes=BIN,NRD,NEW %s
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 -fgpu-rdc %s 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 -fgpu-rdc %s 2>&1 \
 // RUN: | FileCheck -check-prefixes=BIN,RDC %s
 //
 // BIN-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]])
@@ -49,7 +49,7 @@
 // Test single gpu architecture up to the assemble phase.
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 %s -S 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 %s -S 2>&1 \
 // RUN: | FileCheck -check-prefixes=ASM %s
 // ASM-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (device-[[T]], [[ARCH:gfx803]])
 // ASM-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH]])
@@ -64,11 +64,11 @@
 // Test two gpu architectures with complete compilation with -fno-gpu-rdc.
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s 2>&1 \
 // RUN: | FileCheck -check-prefixes=NRD2,NCL2 %s
 
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s -c 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s -c 2>&1 \
 // RUN: | FileCheck -check-prefixes=NRD2 %s
 
 // NRD2-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]])
@@ -100,11 +100,11 @@
 // Test two gpu architectures with complete compilation with -fgpu-rdc.
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s -fgpu-rdc 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s -fgpu-rdc 2>&1 \
 // RUN: | FileCheck -check-prefixes=RDC2,RCL2 %s
 
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s -fgpu-rdc -c 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s -fgpu-rdc -c 2>&1 \
 // RUN: | FileCheck -check-prefixes=RDC2,RC2 %s
 
 // RCL2-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]])
@@ -144,7 +144,7 @@
 // Test two gpu architecturess up to the assemble phase.
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s -S 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s -S 2>&1 \
 // RUN: | FileCheck -check-prefixes=ASM2 %s
 // ASM2-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (device-[[T]], [[ARCH1:gfx803]])
 // ASM2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH1]])
@@ -162,7 +162,7 @@
 // compilation mode.
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 %s --cuda-host-only 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 %s --cuda-host-only 2>&1 \
 // RUN: | FileCheck -check-prefixes=HBIN %s
 // HBIN-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]])
 // HBIN-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]])
@@ -176,7 +176,7 @@
 // compilation mode.
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 %s --cuda-host-only -S 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 %s --cuda-host-only -S 2>&1 \
 // RUN: | FileCheck -check-prefixes=HASM %s
 // HASM-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]])
 // HASM-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]])
@@ -189,7 +189,7 @@
 // compilation mode.
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s --cuda-host-only 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s --cuda-host-only 2>&1 \
 // RUN: | FileCheck -check-prefixes=HBIN2 %s
 // HBIN2-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]])
 // HBIN2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]])
@@ -204,7 +204,7 @@
 // compilation mode.
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s --cuda-host-only -S \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s --cuda-host-only -S \
 // RUN: 2>&1 | FileCheck -check-prefixes=HASM2 %s
 // HASM2-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]])
 // HASM2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]])
@@ -217,14 +217,14 @@
 // compilation mode.
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 %s --cuda-device-only 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 %s --cuda-device-only 2>&1 \
 // RUN: | FileCheck -check-prefixes=DBIN %s
 //
 // Test single gpu architecture with complete compilation in device-only
 // compilation mode with an unused host linker flag.
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 %s --cuda-device-only -Wl,--disable-new-dtags 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 %s --cuda-device-only -Wl,--disable-new-dtags 2>&1 \
 // RUN: | FileCheck -check-prefixes=DBIN %s
 
 // DBIN-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (device-[[T]], [[ARCH:gfx803]])
@@ -242,7 +242,7 @@
 // Test single gpu architecture up to the assemble phase in device-only
 // compilation mode.
 //
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases --no-offload-new-driver \
 // RUN: --cuda-gpu-arch=gfx803 %s --cuda-device-only -S --no-gpu-bundle-output 2>&1 \
 // RUN: | FileCheck -check-prefixes=DASM %s
 // DASM-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (device-[[T]], [[ARCH:gfx803]])
@@ -257,11 +257,11 @@
 // Test single gpu architecture with compile to relocatable in device-only
 // compilation mode.
 //
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases --no-offload-new-driver \
 // RUN: --cuda-gpu-arch=gfx803 %s --cuda-device-only -fhip-emit-relocatable 2>&1 \
 // RUN: | FileCheck -check-prefixes=RELOC %s
 //
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases --no-offload-new-driver \
 // RUN: --cuda-gpu-arch=gfx803 %s --cuda-device-only -fhip-emit-relocatable -Wl,--disable-new-dtags \
 // RUN: 2>&1 | FileCheck -check-prefixes=RELOC %s
 //
@@ -278,11 +278,11 @@
 // Test two gpu architectures with compile to relocatable in device-only
 // compilation mode.
 //
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases --no-offload-new-driver \
 // RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s --cuda-device-only -fhip-emit-relocatable 2>&1 \
 // RUN: | FileCheck -check-prefixes=RELOC2 %s
 //
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases --no-offload-new-driver \
 // RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s --cuda-device-only -fhip-emit-relocatable \
 // RUN: -Wl,--disable-new-dtags 2>&1 | FileCheck -check-prefixes=RELOC2 %s
 //
@@ -306,14 +306,14 @@
 // Test two gpu architectures with complete compilation in device-only
 // compilation mode.
 //
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases --no-offload-new-driver \
 // RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s --cuda-device-only \
 // RUN: 2>&1 | FileCheck -check-prefixes=DBIN2 %s
 //
 // Test two gpu architectures with complete compilation in device-only
 // compilation mode with an unused host linker flag.
 //
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases --no-offload-new-driver \
 // RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s --cuda-device-only \
 // RUN: -Wl,--disable-new-dtags 2>&1 | FileCheck -check-prefixes=DBIN2 %s
 
@@ -339,19 +339,19 @@
 // Test two gpu architectures up to the assemble phase in device-only
 // compilation mode.
 //
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
 // RUN: --cuda-device-only -S -o %t.s 2>&1 \
 // RUN: | FileCheck -check-prefixes=DASM2,DASM2-NOBUNDLE %s
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
 // RUN: --cuda-device-only -S -o %t.s --no-gpu-bundle-output 2>&1 \
 // RUN: | FileCheck -check-prefixes=DASM2,DASM2-NOBUNDLE %s
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
 // RUN: --cuda-device-only -S 2>&1 \
 // RUN: | FileCheck -check-prefixes=DASM2,DASM2-NOBUNDLE %s
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
 // RUN: --cuda-device-only -S --gpu-bundle-output 2>&1 \
 // RUN: | FileCheck -check-prefixes=DASM2,DASM2-BUNDLE %s
@@ -376,29 +376,29 @@
 // RUN: touch %t/obj1.o %t/obj2.o
 
 // RUN: %clang --target=x86_64-unknown-linux-gnu -ccc-print-phases --hip-link \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o 2>&1 \
 // RUN: | FileCheck -check-prefixes=L2,NL2 %s
 //
 // RUN: %clang --target=x86_64-unknown-linux-gnu -ccc-print-phases --hip-link \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o \
 // RUN: -fgpu-rdc 2>&1 | FileCheck -check-prefixes=L2,RL2,RL2-EM %s
 //
 // RUN: %clang --target=x86_64-unknown-linux-gnu -ccc-print-phases --hip-link \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o \
 // RUN: -fgpu-rdc --cuda-device-only 2>&1 | FileCheck -check-prefixes=L2,RL2,RL2-DEV %s
 
 // RUN: %clang --target=x86_64-unknown-linux-gnu -ccc-print-phases --hip-link \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o \
 // RUN: -fgpu-rdc --cuda-device-only -Wl,--disable-new-dtags 2>&1 \
 // RUN: | FileCheck -check-prefixes=L2,RL2,RL2-DEV %s
 
 // RUN: %clang --target=x86_64-unknown-linux-gnu -ccc-print-phases --hip-link \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o \
 // RUN: -fgpu-rdc --cuda-device-only --no-gpu-bundle-output 2>&1 \
 // RUN: | FileCheck -check-prefixes=L2,RL2,RL2-NB %s
 
 // RUN: %clang --target=x86_64-unknown-linux-gnu -ccc-print-phases --hip-link \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o \
 // RUN: -fgpu-rdc --cuda-device-only --no-gpu-bundle-output -Wl,--disable-new-dtags 2>&1 \
 // RUN: | FileCheck -check-prefixes=L2,RL2,RL2-NB %s
 
@@ -428,12 +428,12 @@
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 %s \
-// RUN: --cuda-device-only -E 2>&1 \
+// RUN: --no-offload-new-driver --cuda-device-only -E 2>&1 \
 // RUN: | FileCheck -check-prefixes=PPE,PPEN %s
 
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 %s \
-// RUN: --cuda-device-only -E --no-gpu-bundle-output 2>&1 \
+// RUN: --no-offload-new-driver --cuda-device-only -E --no-gpu-bundle-output 2>&1 \
 // RUN: | FileCheck -check-prefixes=PPE,PPEN %s
 
 // Test one gpu architectures up to the preprocessor expansion output phase in device-only
@@ -441,7 +441,7 @@
 
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 %s \
-// RUN: --cuda-device-only -E --gpu-bundle-output 2>&1 \
+// RUN: --no-offload-new-driver --cuda-device-only -E --gpu-bundle-output 2>&1 \
 // RUN: | FileCheck -check-prefixes=PPE,PPEB %s
 
 // Test two gpu architectures up to the preprocessor expansion output phase in device-only
@@ -449,12 +449,12 @@
 
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
-// RUN: --cuda-device-only -E 2>&1 \
+// RUN: --no-offload-new-driver --cuda-device-only -E 2>&1 \
 // RUN: | FileCheck -check-prefixes=PPE2,PPE2N %s
 
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
-// RUN: --cuda-device-only -E --no-gpu-bundle-output 2>&1 \
+// RUN: --no-offload-new-driver --cuda-device-only -E --no-gpu-bundle-output 2>&1 \
 // RUN: | FileCheck -check-prefixes=PPE2,PPE2N %s
 
 // Test two gpu architectures up to the preprocessor expansion output phase in device-only
@@ -462,7 +462,7 @@
 
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
-// RUN: --cuda-device-only -E --gpu-bundle-output 2>&1 \
+// RUN: --no-offload-new-driver --cuda-device-only -E --gpu-bundle-output 2>&1 \
 // RUN: | FileCheck -check-prefixes=PPE2,PPE2B %s
 
 // Test one gpu architectures up to the LLVM IR output phase in device-only
@@ -470,7 +470,7 @@
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 %s \
-// RUN: --cuda-device-only -c -emit-llvm 2>&1 \
+// RUN: --no-offload-new-driver --cuda-device-only -c -emit-llvm 2>&1 \
 // RUN: | FileCheck -check-prefixes=LLVM %s
 
 // Test two gpu architectures up to the LLVM IR output phase in device-only
@@ -478,7 +478,7 @@
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
-// RUN: --cuda-device-only -c -emit-llvm -o %t.bc --gpu-bundle-output 2>&1 \
+// RUN: --no-offload-new-driver --cuda-device-only -c -emit-llvm -o %t.bc --gpu-bundle-output 2>&1 \
 // RUN: | FileCheck -check-prefixes=LLVM2 %s
 
 // Test two gpu architectures up to the LLVM IR output phase in device-only
@@ -486,7 +486,7 @@
 //
 // RUN: %clang -x hip-cpp-output --target=x86_64-unknown-linux-gnu \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
-// RUN: --cuda-device-only -c -emit-llvm -o %t.bc --gpu-bundle-output 2>&1 \
+// RUN: --no-offload-new-driver --cuda-device-only -c -emit-llvm -o %t.bc --gpu-bundle-output 2>&1 \
 // RUN: | FileCheck -check-prefixes=PPELLVM2 %s
 
 // PPE-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (device-[[T]], [[ARCH:gfx803]])
@@ -541,50 +541,50 @@
 // C++ program should have no offload kind.
 
 // Test compile empty.hip and empty.cpp.
-// RUN: %clang --target=x86_64-unknown-linux-gnu \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: -c %S/Inputs/empty.hip %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED %s
-// RUN: %clang --target=x86_64-unknown-linux-gnu \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: -c %S/Inputs/empty.hip %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED-NEG %s
 
 // Test compile and link empty.hip and empty.cpp.
-// RUN: %clang --target=x86_64-unknown-linux-gnu \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: %S/Inputs/empty.hip %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED %s
-// RUN: %clang --target=x86_64-unknown-linux-gnu \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: %S/Inputs/empty.hip %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED-NEG %s
 
 // Test compile and link empty.hip and empty.cpp with --hip-link -fgpu-rdc.
-// RUN: %clang --target=x86_64-unknown-linux-gnu --hip-link -fgpu-rdc \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --hip-link -fgpu-rdc --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: %S/Inputs/empty.hip %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED %s
-// RUN: %clang --target=x86_64-unknown-linux-gnu --hip-link -fgpu-rdc \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --hip-link -fgpu-rdc --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: %S/Inputs/empty.hip %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED-NEG %s
 
 // Test compile and link -x hip empty.hip and -x c++ empty.cpp.
-// RUN: %clang --target=x86_64-unknown-linux-gnu \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: -x hip %S/Inputs/empty.hip -x c++ %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED %s
-// RUN: %clang --target=x86_64-unknown-linux-gnu \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: -x hip %S/Inputs/empty.hip -x c++ %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED-NEG %s
 
 // Test compile and link -x hip empty.hip and empty.cpp.
-// RUN: %clang --target=x86_64-unknown-linux-gnu \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: -x hip %S/Inputs/empty.hip %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED2 %s
-// RUN: %clang --target=x86_64-unknown-linux-gnu \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: -x hip %S/Inputs/empty.hip %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED2-NEG %s
 
 // Test compile and link empty.hip and -x hip empty.cpp.
-// RUN: %clang --target=x86_64-unknown-linux-gnu \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: %S/Inputs/empty.hip -x hip %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED2 %s
-// RUN: %clang --target=x86_64-unknown-linux-gnu \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: -x hip %S/Inputs/empty.hip %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED2-NEG %s
 
@@ -609,7 +609,7 @@
 // RUN: touch %t/bitcodeA.bc
 // RUN: touch %t/bitcodeB.bc
 // RUN: %clang -ccc-print-phases --hip-link -emit-llvm --cuda-device-only \
-// RUN: --offload-arch=gfx906 %t/bitcodeA.bc %t/bitcodeB.bc 2>&1 \
+// RUN: --no-offload-new-driver --offload-arch=gfx906 %t/bitcodeA.bc %t/bitcodeB.bc 2>&1 \
 // RUN: | FileCheck -check-prefixes=CHECK %s
 
 // CHECK: [[A0:[0-9]+]]: input, "{{.*}}bitcodeA.bc", ir
diff --git a/clang/test/Driver/hip-rdc-device-only.hip b/clang/test/Driver/hip-rdc-device-only.hip
index d972927ff7a3..d79cc1febf3d 100644
--- a/clang/test/Driver/hip-rdc-device-only.hip
+++ b/clang/test/Driver/hip-rdc-device-only.hip
@@ -1,7 +1,7 @@
 // REQUIRES: x86-registered-target
 // REQUIRES: amdgpu-registered-target
 
-// RUN: %clang -### --target=x86_64-linux-gnu \
+// RUN: %clang -### --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -x hip --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -c -nogpuinc -nogpulib --cuda-device-only -fgpu-rdc \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
@@ -11,7 +11,7 @@
 // With `-emit-llvm`, the output should be the same as the aforementioned line
 // as `-fgpu-rdc` in HIP implies `-emit-llvm`.
 
-// RUN: %clang -### --target=x86_64-linux-gnu \
+// RUN: %clang -### --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -x hip --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -c -emit-llvm -nogpuinc -nogpulib --cuda-device-only -fgpu-rdc \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
@@ -21,14 +21,14 @@
 // With `-fno-hip-emit-relocatable`, the output should be the same as the aforementioned line
 // as `-fgpu-rdc` in HIP implies `-fno-hip-emit-relocatable`.
 
-// RUN: %clang -### --target=x86_64-linux-gnu \
+// RUN: %clang -### --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -x hip --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -c -fno-hip-emit-relocatable -nogpuinc -nogpulib --cuda-device-only -fgpu-rdc \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN:   %S/Inputs/hip_multiple_inputs/b.hip --gpu-bundle-output \
 // RUN: 2>&1 | FileCheck -check-prefixes=COMMON,EMITBC %s
 
-// RUN: %clang -### --target=x86_64-linux-gnu \
+// RUN: %clang -### --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -x hip --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -S -nogpuinc -nogpulib --cuda-device-only -fgpu-rdc \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
@@ -38,7 +38,7 @@
 // With `-emit-llvm`, the output should be the same as the aforementioned line
 // as `-fgpu-rdc` in HIP implies `-emit-llvm`.
 
-// RUN: %clang -### --target=x86_64-linux-gnu \
+// RUN: %clang -### --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -x hip --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -S -emit-llvm -nogpuinc -nogpulib --cuda-device-only -fgpu-rdc \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
@@ -49,7 +49,7 @@
 // output, there should 3 steps (preprocessor, compile, and backend) per source
 // and per target, totally 12 steps.
 
-// RUN: %clang -### -save-temps --target=x86_64-linux-gnu \
+// RUN: %clang -### -save-temps --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -x hip --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -S -nogpuinc -nogpulib --cuda-device-only -fgpu-rdc \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
@@ -58,7 +58,7 @@
 
 // Check output one file without bundling cause error.
 
-// RUN: not %clang -### --target=x86_64-linux-gnu \
+// RUN: not %clang -### --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -x hip --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -S -nogpuinc -nogpulib --cuda-device-only -fgpu-rdc \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu -o %t.s --no-gpu-bundle-output \
diff --git a/clang/test/Driver/hip-target-id.hip b/clang/test/Driver/hip-target-id.hip
index 703d6b50be2e..e7ba456565c0 100644
--- a/clang/test/Driver/hip-target-id.hip
+++ b/clang/test/Driver/hip-target-id.hip
@@ -5,7 +5,7 @@
 // RUN:   -x hip \
 // RUN:   --offload-arch=gfx908:xnack+:sramecc+ \
 // RUN:   --offload-arch=gfx908:xnack+:sramecc- \
-// RUN:   --rocm-path=%S/Inputs/rocm \
+// RUN:   --no-offload-new-driver --rocm-path=%S/Inputs/rocm \
 // RUN:   %s 2>&1 | FileCheck %s
 
 // RUN: %clang -### --target=x86_64-linux-gnu \
@@ -13,7 +13,7 @@
 // RUN:   --offload-arch=gfx908:xnack+:sramecc+ \
 // RUN:   --offload-arch=gfx908:xnack+:sramecc- \
 // RUN:   --rocm-path=%S/Inputs/rocm \
-// RUN:   -save-temps \
+// RUN:   --no-offload-new-driver -save-temps \
 // RUN:   %s 2>&1 | FileCheck --check-prefixes=CHECK,TMP %s
 
 // RUN: %clang -### --target=x86_64-linux-gnu \
@@ -21,7 +21,7 @@
 // RUN:   --offload-arch=gfx908:xnack+:sramecc+ \
 // RUN:   --offload-arch=gfx908:xnack+:sramecc- \
 // RUN:   --rocm-path=%S/Inputs/rocm \
-// RUN:   -fgpu-rdc \
+// RUN:   --no-offload-new-driver -fgpu-rdc \
 // RUN:   %s 2>&1 | FileCheck --check-prefixes=CHECK %s
 
 // CHECK: [[CLANG:"[^"]*clang[^"]*"]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
@@ -55,7 +55,7 @@
 // RUN:   --offload-arch=fiji \
 // RUN:   --offload-arch=gfx803 \
 // RUN:   --offload-arch=fiji \
-// RUN:   --rocm-path=%S/Inputs/rocm \
+// RUN:   --no-offload-new-driver --rocm-path=%S/Inputs/rocm \
 // RUN:   %s 2>&1 | FileCheck -check-prefix=FIJI %s
 // FIJI: "-targets=host-x86_64-unknown-linux,hipv4-amdgcn-amd-amdhsa--gfx803"
 
@@ -66,6 +66,6 @@
 // RUN:   --offload-arch=gfx908:sramecc+ \
 // RUN:   --offload-arch=gfx908:sramecc- \
 // RUN:   --offload-arch=gfx906 \
-// RUN:   --rocm-path=%S/Inputs/rocm \
+// RUN:   --no-offload-new-driver --rocm-path=%S/Inputs/rocm \
 // RUN:   %s 2>&1 | FileCheck -check-prefix=MULTI %s
 // MULTI: "-targets=host-x86_64-unknown-linux,hipv4-amdgcn-amd-amdhsa--gfx900:xnack+,hipv4-amdgcn-amd-amdhsa--gfx900:xnack-,hipv4-amdgcn-amd-amdhsa--gfx906,hipv4-amdgcn-amd-amdhsa--gfx908:sramecc+,hipv4-amdgcn-amd-amdhsa--gfx908:sramecc-"
diff --git a/clang/test/Driver/hip-toolchain-features.hip b/clang/test/Driver/hip-toolchain-features.hip
index 2e11ce38403e..551d8ef42e02 100644
--- a/clang/test/Driver/hip-toolchain-features.hip
+++ b/clang/test/Driver/hip-toolchain-features.hip
@@ -1,10 +1,10 @@
 // REQUIRES: x86-registered-target
 // REQUIRES: amdgpu-registered-target
 
-// RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
+// RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib --no-offload-new-driver \
 // RUN:   -nogpuinc --offload-arch=gfx906:xnack+ --offload-arch=gfx900:xnack+ %s \
 // RUN:   2>&1 | FileCheck %s -check-prefix=XNACK
-// RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
+// RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib --no-offload-new-driver \
 // RUN:   -nogpuinc --offload-arch=gfx906:xnack- --offload-arch=gfx900:xnack- %s \
 // RUN:   2>&1 | FileCheck %s -check-prefix=NOXNACK
 
@@ -14,10 +14,10 @@
 // NOXNACK: {{.*}}lld{{.*}} "-plugin-opt=-mattr=-xnack"
 
 // RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   -nogpuinc --offload-arch=gfx908:sramecc+ %s \
+// RUN:   -nogpuinc --offload-arch=gfx908:sramecc+ --no-offload-new-driver %s \
 // RUN:   2>&1 | FileCheck %s -check-prefix=SRAM
 // RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   -nogpuinc --offload-arch=gfx908:sramecc- %s \
+// RUN:   -nogpuinc --offload-arch=gfx908:sramecc- --no-offload-new-driver %s \
 // RUN:   2>&1 | FileCheck %s -check-prefix=NOSRAM
 
 // SRAM: {{.*}}clang{{.*}}"-target-feature" "+sramecc"
@@ -26,10 +26,10 @@
 // NOTSRAM: {{.*}}lld{{.*}} "-plugin-opt=-mattr=-sramecc"
 
 // RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   -nogpuinc --offload-arch=gfx1010 %s \
+// RUN:   -nogpuinc --offload-arch=gfx1010 --no-offload-new-driver %s \
 // RUN:   -mcumode  2>&1 | FileCheck %s -check-prefix=CUMODE
 // RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   -nogpuinc --offload-arch=gfx1010 %s \
+// RUN:   -nogpuinc --offload-arch=gfx1010 --no-offload-new-driver %s \
 // RUN:   -mno-cumode  2>&1 | FileCheck %s -check-prefix=NOTCUMODE
 
 // CUMODE: {{.*}}clang{{.*}}"-target-feature" "+cumode"
@@ -38,20 +38,20 @@
 // NOTCUMODE: {{.*}}lld{{.*}} "-plugin-opt=-mattr=-cumode"
 
 // RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   -nogpuinc --offload-arch=gfx908:xnack+:sramecc+ %s \
+// RUN:   -nogpuinc --offload-arch=gfx908:xnack+:sramecc+ --no-offload-new-driver %s \
 // RUN:   2>&1 | FileCheck %s -check-prefix=ALL3
 // RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   -nogpuinc --offload-arch=gfx908:xnack-:sramecc- %s \
+// RUN:   -nogpuinc --offload-arch=gfx908:xnack-:sramecc- --no-offload-new-driver %s \
 // RUN:   2>&1 | FileCheck %s -check-prefix=NOALL3
 
 // ALL3: {{.*}}clang{{.*}}"-target-feature" "+sramecc" "-target-feature" "+xnack"
 // NOALL3: {{.*}}clang{{.*}}"-target-feature" "-sramecc" "-target-feature" "-xnack"
 
 // RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   -nogpuinc --offload-arch=gfx1010 %s \
+// RUN:   -nogpuinc --offload-arch=gfx1010 --no-offload-new-driver %s \
 // RUN:   -mtgsplit  2>&1 | FileCheck %s -check-prefix=TGSPLIT
 // RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   -nogpuinc --offload-arch=gfx1010 %s \
+// RUN:   -nogpuinc --offload-arch=gfx1010 --no-offload-new-driver %s \
 // RUN:   -mno-tgsplit  2>&1 | FileCheck %s -check-prefix=NOTTGSPLIT
 
 // TGSPLIT: {{.*}}clang{{.*}}"-target-feature" "+tgsplit"
@@ -60,7 +60,7 @@
 // NOTTGSPLIT: {{.*}}lld{{.*}} "-plugin-opt=-mattr=-tgsplit"
 
 // RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   -nogpuinc --offload-arch=gfx1010 %s \
+// RUN:   -nogpuinc --offload-arch=gfx1010 --no-offload-new-driver %s \
 // RUN:   -mcumode -mcumode -mno-cumode -mwavefrontsize64 -mcumode \
 // RUN:   -mwavefrontsize64 -mno-wavefrontsize64 2>&1 \
 // RUN:   | FileCheck %s -check-prefix=DUP
@@ -71,7 +71,7 @@
 // DUP: {{.*}}lld{{.*}} "-plugin-opt=-mattr=+cumode"
 
 // RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   -nogpuinc --offload-arch=gfx1010 %s \
+// RUN:   -nogpuinc --offload-arch=gfx1010 --no-offload-new-driver %s \
 // RUN:   -mno-wavefrontsize64 -mwavefrontsize64 2>&1 \
 // RUN:   | FileCheck %s -check-prefix=WAVE64
 // WAVE64: {{.*}}clang{{.*}} "-target-feature" "+wavefrontsize64"
diff --git a/clang/test/Driver/hip-toolchain-rdc-separate.hip b/clang/test/Driver/hip-toolchain-rdc-separate.hip
index e52184fdeacf..6efca87dc0db 100644
--- a/clang/test/Driver/hip-toolchain-rdc-separate.hip
+++ b/clang/test/Driver/hip-toolchain-rdc-separate.hip
@@ -7,7 +7,7 @@
 // RUN:   --hip-device-lib=lib1.bc --hip-device-lib=lib2.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib2 \
-// RUN:   -fuse-ld=lld -B%S/Inputs/lld -fgpu-rdc -nogpuinc \
+// RUN:   --no-offload-new-driver -fuse-ld=lld -B%S/Inputs/lld -fgpu-rdc -nogpuinc \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
 // RUN: 2>&1 | FileCheck %s
@@ -84,19 +84,19 @@
 
 // RUN: touch %t/a.o %t/b.o
 // RUN: %clang --hip-link -### --target=x86_64-linux-gnu \
-// RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -fuse-ld=lld -B%S/Inputs/lld -fgpu-rdc -nogpuinc \
 // RUN:   %t/a.o %t/b.o \
 // RUN: 2>&1 | FileCheck -check-prefixes=LINK,LINK-HOST-UNBUNDLE,LLD-TMP,LINK-BUNDLE,LINK-EMBED %s
 
 // RUN: %clang --hip-link -### --target=x86_64-linux-gnu \
-// RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -fuse-ld=lld -B%S/Inputs/lld -fgpu-rdc -nogpuinc \
 // RUN:   %t/a.o %t/b.o --cuda-device-only \
 // RUN: 2>&1 | FileCheck -check-prefixes=LINK,LLD-TMP,LINK-BUNDLE,LINK-NOEMBED %s
 
 // RUN: %clang --hip-link -### --target=x86_64-linux-gnu \
-// RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -fuse-ld=lld -B%S/Inputs/lld -fgpu-rdc -nogpuinc \
 // RUN:   %t/a.o %t/b.o --cuda-device-only --no-gpu-bundle-output \
 // RUN: 2>&1 | FileCheck -check-prefixes=LINK,LLD-FIN,LINK-NOBUNDLE,LINK-NOEMBED %s
diff --git a/clang/test/Driver/hip-toolchain-rdc-static-lib.hip b/clang/test/Driver/hip-toolchain-rdc-static-lib.hip
index fd12d48a2b82..2cfb24852386 100644
--- a/clang/test/Driver/hip-toolchain-rdc-static-lib.hip
+++ b/clang/test/Driver/hip-toolchain-rdc-static-lib.hip
@@ -3,7 +3,7 @@
 
 // RUN: %clang -### --target=x86_64-linux-gnu \
 // RUN:   -x hip --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
-// RUN:   --emit-static-lib -nogpulib \
+// RUN:   --no-offload-new-driver --emit-static-lib -nogpulib \
 // RUN:   -fuse-ld=lld -B%S/Inputs/lld -fgpu-rdc -nogpuinc \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
diff --git a/clang/test/Driver/hip-toolchain-rdc.hip b/clang/test/Driver/hip-toolchain-rdc.hip
index d19d8ccd6cb2..49acc40ec6f9 100644
--- a/clang/test/Driver/hip-toolchain-rdc.hip
+++ b/clang/test/Driver/hip-toolchain-rdc.hip
@@ -7,7 +7,7 @@
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib2 \
 // RUN:   -fuse-ld=lld -B%S/Inputs/lld -fgpu-rdc -nogpuinc \
-// RUN:   -fhip-dump-offload-linker-script \
+// RUN:   --no-offload-new-driver -fhip-dump-offload-linker-script \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
 // RUN: 2>&1 | FileCheck -check-prefixes=CHECK,LNX %s
@@ -18,7 +18,7 @@
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib2 \
 // RUN:   -fuse-ld=lld -B%S/Inputs/lld -fgpu-rdc -nogpuinc \
-// RUN:   -fhip-dump-offload-linker-script \
+// RUN:   --no-offload-new-driver -fhip-dump-offload-linker-script \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
 // RUN: 2>&1 | FileCheck -check-prefixes=CHECK,MSVC %s
diff --git a/clang/test/Driver/hip-unbundle-preproc.hipi b/clang/test/Driver/hip-unbundle-preproc.hipi
index fecdfdbe7d14..6d92d2381973 100644
--- a/clang/test/Driver/hip-unbundle-preproc.hipi
+++ b/clang/test/Driver/hip-unbundle-preproc.hipi
@@ -1,11 +1,11 @@
 // REQUIRES: amdgpu-registered-target
 
 // RUN: %clang -### --target=x86_64-unknown-linux-gnu \
-// RUN:   --offload-arch=gfx803 -nogpulib \
+// RUN:   --no-offload-new-driver --offload-arch=gfx803 -nogpulib \
 // RUN:   -x hip-cpp-output %s 2>&1 | FileCheck %s
 
 // RUN: %clang -### --target=x86_64-unknown-linux-gnu \
-// RUN:   --offload-arch=gfx803 -nogpulib \
+// RUN:   --no-offload-new-driver --offload-arch=gfx803 -nogpulib \
 // RUN:   %s 2>&1 | FileCheck %s
 
 // CHECK: {{".*clang-offload-bundler.*"}} {{.*}}"-output=[[HOST_PP:.*hipi]]" "-output=[[DEV_PP:.*hipi]]" "-unbundle"
@@ -16,7 +16,7 @@
 // CHECK: {{".*ld.*"}} {{.*}}"[[HOST_O]]"
 
 // RUN: %clang -### --target=x86_64-unknown-linux-gnu \
-// RUN:   --offload-arch=gfx803 -nogpulib -fgpu-rdc \
+// RUN:   --no-offload-new-driver --offload-arch=gfx803 -nogpulib -fgpu-rdc \
 // RUN:   %s 2>&1 | FileCheck -check-prefix=RDC %s
 
 // RDC: {{".*clang-offload-bundler.*"}} {{.*}}"-output=[[HOST_PP:.*hipi]]" "-output=[[DEV_PP:.*hipi]]" "-unbundle"
diff --git a/clang/test/Driver/hipspv-toolchain-rdc.hip b/clang/test/Driver/hipspv-toolchain-rdc.hip
index 2bfcec977d39..d4e612cc5437 100644
--- a/clang/test/Driver/hipspv-toolchain-rdc.hip
+++ b/clang/test/Driver/hipspv-toolchain-rdc.hip
@@ -2,7 +2,7 @@
 // UNSUPPORTED: system-windows
 
 // RUN: %clang -### -x hip -target x86_64-linux-gnu --offload=spirv64 \
-// RUN:   -fgpu-rdc --hip-path=%S/Inputs/hipspv -nohipwrapperinc \
+// RUN:   --no-offload-new-driver -fgpu-rdc --hip-path=%S/Inputs/hipspv -nohipwrapperinc \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
 // RUN: 2>&1 | FileCheck %s
diff --git a/clang/test/Driver/hipspv-toolchain.hip b/clang/test/Driver/hipspv-toolchain.hip
index bfae41049ba4..4602cd3fc8d6 100644
--- a/clang/test/Driver/hipspv-toolchain.hip
+++ b/clang/test/Driver/hipspv-toolchain.hip
@@ -2,7 +2,7 @@
 // UNSUPPORTED: system-windows
 
 // RUN: %clang -### -target x86_64-linux-gnu --offload=spirv64 \
-// RUN:   --hip-path=%S/Inputs/hipspv -nohipwrapperinc %s \
+// RUN:   --no-offload-new-driver --hip-path=%S/Inputs/hipspv -nohipwrapperinc %s \
 // RUN: 2>&1 | FileCheck %s
 
 // CHECK: [[CLANG:".*clang.*"]] "-cc1" "-triple" "spirv64"
diff --git a/clang/test/Driver/lto.cu b/clang/test/Driver/lto.cu
index dadf35d8e6c9..fb8cc5cba393 100644
--- a/clang/test/Driver/lto.cu
+++ b/clang/test/Driver/lto.cu
@@ -2,14 +2,14 @@
 // REQUIRES: nvptx-registered-target
 
 // -flto causes a switch to llvm-bc object files.
-// RUN: %clangxx --target=x86_64-unknown-linux-gnu -nocudainc -nocudalib -ccc-print-phases -c %s -flto 2> %t
+// RUN: %clangxx --target=x86_64-unknown-linux-gnu --no-offload-new-driver -nocudainc -nocudalib -ccc-print-phases -c %s -flto 2> %t
 // RUN: FileCheck -check-prefix=CHECK-COMPILE-ACTIONS < %t %s
 //
 // CHECK-COMPILE-ACTIONS: 2: compiler, {1}, ir, (host-cuda)
 // CHECK-COMPILE-ACTIONS-NOT: lto-bc
 // CHECK-COMPILE-ACTIONS: 12: backend, {11}, lto-bc, (host-cuda)
 
-// RUN: %clangxx --target=x86_64-unknown-linux-gnu -nocudainc -nocudalib -ccc-print-phases %s -flto 2> %t
+// RUN: %clangxx --target=x86_64-unknown-linux-gnu --no-offload-new-driver -nocudainc -nocudalib -ccc-print-phases %s -flto 2> %t
 // RUN: FileCheck -check-prefix=CHECK-COMPILELINK-ACTIONS < %t %s
 //
 // CHECK-COMPILELINK-ACTIONS: 0: input, "{{.*}}lto.cu", cuda, (host-cuda)
@@ -29,7 +29,7 @@
 
 // llvm-bc and llvm-ll outputs need to match regular suffixes
 // (unfortunately).
-// RUN: %clangxx %s --target=x86_64-unknown-linux-gnu -nocudainc -nocudalib -flto -save-temps --cuda-path=%S/Inputs/CUDA_80/usr/local/cuda -### 2> %t
+// RUN: %clangxx %s --target=x86_64-unknown-linux-gnu --no-offload-new-driver -nocudainc -nocudalib -flto -save-temps --cuda-path=%S/Inputs/CUDA_80/usr/local/cuda -### 2> %t
 // RUN: FileCheck -check-prefix=CHECK-COMPILELINK-SUFFIXES < %t %s
 //
 // CHECK-COMPILELINK-SUFFIXES: "-o" "[[CPP:.*lto-host.*\.cui]]" "-x" "cuda" "{{.*}}lto.cu"
@@ -37,36 +37,36 @@
 // CHECK-COMPILELINK-SUFFIXES: "-o" "[[OBJ:.*lto-host.*\.o]]" {{.*}}[[BC]]"
 // CHECK-COMPILELINK-SUFFIXES: "{{.*}}a.{{(out|exe)}}" {{.*}}[[OBJ]]"
 
-// RUN: %clangxx --target=x86_64-unknown-linux-gnu %s -nocudainc -nocudalib -flto -S -### 2> %t
+// RUN: %clangxx --target=x86_64-unknown-linux-gnu %s --no-offload-new-driver -nocudainc -nocudalib -flto -S -### 2> %t
 // RUN: FileCheck -check-prefix=CHECK-COMPILE-SUFFIXES < %t %s
 //
 // CHECK-COMPILE-SUFFIXES: "-o" "{{.*}}lto.s" "-x" "cuda" "{{.*}}lto.cu"
 
-// RUN: not %clangxx --target=x86_64-unknown-linux-gnu -nocudainc -nocudalib %s -emit-llvm 2>&1 \
+// RUN: not %clangxx --target=x86_64-unknown-linux-gnu --no-offload-new-driver -nocudainc -nocudalib %s -emit-llvm 2>&1 \
 // RUN:    | FileCheck --check-prefix=LLVM-LINK %s
 // LLVM-LINK: -emit-llvm cannot be used when linking
 
 /// With ld.bfd or gold, link against LLVMgold.
 // RUN: %clangxx -nocudainc -nocudalib --target=x86_64-unknown-linux-gnu --offload-arch=sm_52 --sysroot=%S/Inputs/basic_cross_linux_tree %s \
-// RUN:   -fuse-ld=bfd -flto=thin -### 2>&1 | FileCheck --check-prefix=LLVMGOLD %s
+// RUN:   --no-offload-new-driver -fuse-ld=bfd -flto=thin -### 2>&1 | FileCheck --check-prefix=LLVMGOLD %s
 // RUN: %clangxx -nocudainc -nocudalib --target=x86_64-unknown-linux-gnu --offload-arch=sm_52 --sysroot=%S/Inputs/basic_cross_linux_tree %s \
-// RUN:   -fuse-ld=gold -flto=full -### 2>&1 | FileCheck --check-prefix=LLVMGOLD %s
+// RUN:   --no-offload-new-driver -fuse-ld=gold -flto=full -### 2>&1 | FileCheck --check-prefix=LLVMGOLD %s
 //
 // LLVMGOLD: "-plugin" "{{.*}}{{[/\\]}}LLVMgold.{{dll|dylib|so}}"
 
 /// lld does not need LLVMgold.
 // RUN: %clangxx -nocudainc -nocudalib --target=x86_64-unknown-linux-gnu --offload-arch=sm_52 --sysroot=%S/Inputs/basic_cross_linux_tree %s \
-// RUN:   -fuse-ld=lld -flto=full -### 2>&1 | FileCheck --check-prefix=NO-LLVMGOLD %s
+// RUN:   --no-offload-new-driver -fuse-ld=lld -flto=full -### 2>&1 | FileCheck --check-prefix=NO-LLVMGOLD %s
 // RUN: %clangxx -nocudainc -nocudalib --target=x86_64-unknown-linux-gnu --offload-arch=sm_52 --sysroot=%S/Inputs/basic_cross_linux_tree %s \
-// RUN:   -fuse-ld=gold -flto=full -fno-lto -### 2>&1 | FileCheck --check-prefix=NO-LLVMGOLD %s
+// RUN:   --no-offload-new-driver -fuse-ld=gold -flto=full -fno-lto -### 2>&1 | FileCheck --check-prefix=NO-LLVMGOLD %s
 //
 // NO-LLVMGOLD-NOT: "-plugin" "{{.*}}{{[/\\]}}LLVMgold.{{dll|dylib|so}}"
 
 // -flto passes along an explicit debugger tuning argument.
-// RUN: %clangxx -nocudainc -nocudalib \
+// RUN: %clangxx -nocudainc -nocudalib --no-offload-new-driver \
 // RUN:          --target=x86_64-unknown-linux -### %s -flto -glldb --offload-arch=sm_52 --cuda-path=%S/Inputs/CUDA_80/usr/local/cuda 2> %t
 // RUN: FileCheck -check-prefix=CHECK-TUNING-LLDB < %t %s
-// RUN: %clangxx -nocudainc -nocudalib \
+// RUN: %clangxx -nocudainc -nocudalib --no-offload-new-driver \
 // RUN:          --target=x86_64-unknown-linux -### %s -flto -g --offload-arch=sm_52 --cuda-path=%S/Inputs/CUDA_80/usr/local/cuda 2> %t
 // RUN: FileCheck -check-prefix=CHECK-NO-TUNING < %t %s
 //
diff --git a/clang/test/Driver/thinlto.cu b/clang/test/Driver/thinlto.cu
index d4f265baf370..7c51a5194e0b 100644
--- a/clang/test/Driver/thinlto.cu
+++ b/clang/test/Driver/thinlto.cu
@@ -2,14 +2,14 @@
 // REQUIRES: nvptx-registered-target
 
 // -flto=thin causes a switch to llvm-bc object files.
-// RUN: %clangxx -ccc-print-phases -nocudainc -nocudalib -c %s -flto=thin 2> %t
+// RUN: %clangxx -ccc-print-phases --no-offload-new-driver -nocudainc -nocudalib -c %s -flto=thin 2> %t
 // RUN: FileCheck -check-prefix=CHECK-COMPILE-ACTIONS < %t %s
 //
 // CHECK-COMPILE-ACTIONS: 2: compiler, {1}, ir, (host-cuda)
 // CHECK-COMPILE-ACTIONS-NOT: lto-bc
 // CHECK-COMPILE-ACTIONS: 12: backend, {11}, lto-bc, (host-cuda)
 
-// RUN: %clangxx -ccc-print-phases -nocudainc -nocudalib %s -flto=thin 2> %t
+// RUN: %clangxx -ccc-print-phases --no-offload-new-driver -nocudainc -nocudalib %s -flto=thin 2> %t
 // RUN: FileCheck -check-prefix=CHECK-COMPILELINK-ACTIONS < %t %s
 //
 // CHECK-COMPILELINK-ACTIONS: 0: input, "{{.*}}thinlto.cu", cuda, (host-cuda)
-- 
GitLab


From ce683273aa3dbdd7bce2168c29b1a7652ce73608 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim 
Date: Fri, 8 Mar 2024 13:50:06 +0000
Subject: [PATCH 617/929] [X86] PBLENDW instructions can run on Port1 or Port5

When we copied the SkylakeServer model we missed this diff

Confirmed with uops.info and Agner
---
 llvm/lib/Target/X86/X86SchedIceLake.td                 |  6 +++---
 .../tools/llvm-mca/X86/IceLakeServer/resources-avx1.s  | 10 +++++-----
 .../tools/llvm-mca/X86/IceLakeServer/resources-avx2.s  | 10 +++++-----
 .../tools/llvm-mca/X86/IceLakeServer/resources-sse41.s | 10 +++++-----
 4 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/llvm/lib/Target/X86/X86SchedIceLake.td b/llvm/lib/Target/X86/X86SchedIceLake.td
index 3144327670c7..c9ae9901ed5b 100644
--- a/llvm/lib/Target/X86/X86SchedIceLake.td
+++ b/llvm/lib/Target/X86/X86SchedIceLake.td
@@ -396,9 +396,9 @@ defm : ICXWriteResPair; // Vector v
 defm : ICXWriteResPair;
 defm : ICXWriteResPair;
 defm : ICXWriteResPair;
-defm : ICXWriteResPair; // Vector blends.
-defm : ICXWriteResPair;
-defm : ICXWriteResPair;
+defm : ICXWriteResPair; // Vector blends.
+defm : ICXWriteResPair;
+defm : ICXWriteResPair;
 defm : ICXWriteResPair; // Vector variable blends.
 defm : ICXWriteResPair;
 defm : ICXWriteResPair;
diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s
index ad7bc8d27e21..e467c4e48ebd 100644
--- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s
+++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s
@@ -1401,8 +1401,8 @@ vzeroupper
 # CHECK-NEXT:  2      7     0.50    *                   vpavgw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  2      2     0.67                        vpblendvb	%xmm3, %xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  3      8     0.67    *                   vpblendvb	%xmm3, (%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      1     1.00                        vpblendw	$11, %xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  2      7     1.00    *                   vpblendw	$11, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpblendw	$11, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      7     0.50    *                   vpblendw	$11, (%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      6     1.00                        vpclmulqdq	$11, %xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  2      12    1.00    *                   vpclmulqdq	$11, (%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpcmpeqb	%xmm0, %xmm1, %xmm2
@@ -1738,7 +1738,7 @@ vzeroupper
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT:  -     126.00 322.92 231.92 160.50 160.50 19.00  297.92 6.25   19.00  19.00  19.00
+# CHECK-NEXT:  -     126.00 322.92 232.92 160.50 160.50 19.00  296.92 6.25   19.00  19.00  19.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -2113,8 +2113,8 @@ vzeroupper
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpavgw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.67   0.67    -      -      -     0.67    -      -      -      -     vpblendvb	%xmm3, %xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.67   0.67   0.50   0.50    -     0.67    -      -      -      -     vpblendvb	%xmm3, (%rax), %xmm1, %xmm2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     vpblendw	$11, %xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     vpblendw	$11, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -      -      -     vpblendw	$11, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -      -      -     vpblendw	$11, (%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     vpclmulqdq	$11, %xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     vpclmulqdq	$11, (%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     vpcmpeqb	%xmm0, %xmm1, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx2.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx2.s
index e65504f79ff1..97f0d052f455 100644
--- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx2.s
+++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx2.s
@@ -524,8 +524,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      8     0.50    *                   vpblendd	$11, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      2     0.67                        vpblendvb	%ymm3, %ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  3      8     0.67    *                   vpblendvb	%ymm3, (%rax), %ymm1, %ymm2
-# CHECK-NEXT:  1      1     1.00                        vpblendw	$11, %ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  2      8     1.00    *                   vpblendw	$11, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpblendw	$11, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      8     0.50    *                   vpblendw	$11, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      3     1.00                        vpbroadcastb	%xmm0, %xmm0
 # CHECK-NEXT:  2      7     1.00    *                   vpbroadcastb	(%rax), %xmm0
 # CHECK-NEXT:  1      3     1.00                        vpbroadcastb	%xmm0, %ymm0
@@ -778,7 +778,7 @@ vpxor           (%rax), %ymm1, %ymm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT:  -      -     110.33 102.33 98.00  98.00  2.50   151.33  -     2.50   2.50   2.50
+# CHECK-NEXT:  -      -     110.33 103.33 98.00  98.00  2.50   150.33  -     2.50   2.50   2.50
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -846,8 +846,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -      -      -     vpblendd	$11, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -     0.67   0.67    -      -      -     0.67    -      -      -      -     vpblendvb	%ymm3, %ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  -      -     0.67   0.67   0.50   0.50    -     0.67    -      -      -      -     vpblendvb	%ymm3, (%rax), %ymm1, %ymm2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     vpblendw	$11, %ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     vpblendw	$11, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -      -      -     vpblendw	$11, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -      -      -     vpblendw	$11, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     vpbroadcastb	%xmm0, %xmm0
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     vpbroadcastb	(%rax), %xmm0
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     vpbroadcastb	%xmm0, %ymm0
diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse41.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse41.s
index 4b4c6b90541a..554d7aad54ba 100644
--- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse41.s
+++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse41.s
@@ -178,8 +178,8 @@ roundss     $1, (%rax), %xmm2
 # CHECK-NEXT:  2      10    1.00    *                   packusdw	(%rax), %xmm2
 # CHECK-NEXT:  2      2     0.67                        pblendvb	%xmm0, %xmm0, %xmm2
 # CHECK-NEXT:  3      8     0.67    *                   pblendvb	%xmm0, (%rax), %xmm2
-# CHECK-NEXT:  1      1     1.00                        pblendw	$11, %xmm0, %xmm2
-# CHECK-NEXT:  2      7     1.00    *                   pblendw	$11, (%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pblendw	$11, %xmm0, %xmm2
+# CHECK-NEXT:  2      7     0.50    *                   pblendw	$11, (%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pcmpeqq	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   pcmpeqq	(%rax), %xmm2
 # CHECK-NEXT:  2      3     1.00                        pextrb	$1, %xmm0, %ecx
@@ -268,7 +268,7 @@ roundss     $1, (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT:  -      -     36.67  40.67  22.00  22.00  2.50   54.67   -     2.50   2.50   2.50
+# CHECK-NEXT:  -      -     36.67  41.67  22.00  22.00  2.50   53.67   -     2.50   2.50   2.50
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -295,8 +295,8 @@ roundss     $1, (%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     packusdw	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.67   0.67    -      -      -     0.67    -      -      -      -     pblendvb	%xmm0, %xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.67   0.67   0.50   0.50    -     0.67    -      -      -      -     pblendvb	%xmm0, (%rax), %xmm2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     pblendw	$11, %xmm0, %xmm2
-# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     pblendw	$11, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -      -      -     pblendw	$11, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -      -      -     pblendw	$11, (%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     pcmpeqq	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     pcmpeqq	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -      -      -     pextrb	$1, %xmm0, %ecx
-- 
GitLab


From c4a89f1538bbcac19628570b6b489cd9ce4265e2 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek 
Date: Fri, 8 Mar 2024 09:15:59 -0600
Subject: [PATCH 618/929] [flang][OpenMP] Fix use-after-free in
 OMPFunctionFiltering (#84373)

When walking over functions (in pre-order), if the function being
visited needs to be erased, skip visiting its regions.

This was detected by address sanitizer.
---
 flang/lib/Optimizer/Transforms/OMPFunctionFiltering.cpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/flang/lib/Optimizer/Transforms/OMPFunctionFiltering.cpp b/flang/lib/Optimizer/Transforms/OMPFunctionFiltering.cpp
index 466bf53e8dbd..959099d039a5 100644
--- a/flang/lib/Optimizer/Transforms/OMPFunctionFiltering.cpp
+++ b/flang/lib/Optimizer/Transforms/OMPFunctionFiltering.cpp
@@ -79,12 +79,15 @@ public:
           // Remove the callOp
           callOp->erase();
         }
-        if (!hasTargetRegion)
+        if (!hasTargetRegion) {
           funcOp.erase();
-        else if (declareTargetOp)
+          return WalkResult::skip();
+        }
+        if (declareTargetOp)
           declareTargetOp.setDeclareTarget(declareType,
                                            omp::DeclareTargetCaptureClause::to);
       }
+      return WalkResult::advance();
     });
   }
 };
-- 
GitLab


From e0d49066c1acfa4ae0f0e8ea49b0f0b6bb3f1a25 Mon Sep 17 00:00:00 2001
From: Andreas Jonson 
Date: Fri, 8 Mar 2024 16:20:04 +0100
Subject: [PATCH 619/929] [IR] Add new Range attribute using new ConstantRange
 Attribute type (#83171)

implementation as discussed in
https://discourse.llvm.org/t/rfc-metadata-attachments-for-function-arguments/76420
---
 llvm/docs/LangRef.rst                         | 16 ++++
 llvm/include/llvm/ADT/FoldingSet.h            |  7 ++
 llvm/include/llvm/AsmParser/LLParser.h        |  1 +
 llvm/include/llvm/Bitcode/LLVMBitCodes.h      |  1 +
 llvm/include/llvm/IR/Attributes.h             | 23 ++++++
 llvm/include/llvm/IR/Attributes.td            |  6 ++
 llvm/lib/AsmParser/LLParser.cpp               | 43 ++++++++++
 llvm/lib/Bitcode/Reader/BitcodeReader.cpp     | 41 ++++++++++
 llvm/lib/Bitcode/Writer/BitcodeWriter.cpp     | 61 +++++++++-----
 llvm/lib/IR/AttributeImpl.h                   | 28 ++++++-
 llvm/lib/IR/Attributes.cpp                    | 80 ++++++++++++++++++-
 llvm/lib/IR/Verifier.cpp                      |  5 ++
 llvm/lib/Transforms/Utils/CodeExtractor.cpp   |  1 +
 .../range-attribute-invalid-range.ll          |  6 ++
 .../Assembler/range-attribute-invalid-type.ll |  6 ++
 llvm/test/Bitcode/attributes.ll               | 10 +++
 llvm/test/Verifier/range-attr.ll              | 19 +++++
 llvm/utils/TableGen/Attributes.cpp            |  9 ++-
 18 files changed, 337 insertions(+), 26 deletions(-)
 create mode 100644 llvm/test/Assembler/range-attribute-invalid-range.ll
 create mode 100644 llvm/test/Assembler/range-attribute-invalid-type.ll
 create mode 100644 llvm/test/Verifier/range-attr.ll

diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index a7b77d6f776a..b70220dec926 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -1635,6 +1635,22 @@ Currently, only the following parameter attributes are defined:
 
     This attribute cannot be applied to return values.
 
+``range( , )``
+    This attribute expresses the possible range of the parameter or return value.
+    If the value is not in the specified range, it is converted to poison.
+    The arguments passed to ``range`` have the following properties:
+
+    -  The type must match the scalar type of the parameter or return value.
+    -  The pair ``a,b`` represents the range ``[a,b)``.
+    -  Both ``a`` and ``b`` are constants.
+    -  The range is allowed to wrap.
+    -  The range should not represent the full or empty set. That is, ``a!=b``.
+    
+    This attribute may only be applied to parameters or return values with integer 
+    or vector of integer types.
+    
+    For vector-typed parameters, the range is applied element-wise.
+
 .. _gc:
 
 Garbage Collector Strategy Names
diff --git a/llvm/include/llvm/ADT/FoldingSet.h b/llvm/include/llvm/ADT/FoldingSet.h
index f82eabd5044b..ddc3e52255d6 100644
--- a/llvm/include/llvm/ADT/FoldingSet.h
+++ b/llvm/include/llvm/ADT/FoldingSet.h
@@ -16,6 +16,7 @@
 #ifndef LLVM_ADT_FOLDINGSET_H
 #define LLVM_ADT_FOLDINGSET_H
 
+#include "llvm/ADT/APInt.h"
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/SmallVector.h"
@@ -354,6 +355,12 @@ public:
     AddInteger(unsigned(I));
     AddInteger(unsigned(I >> 32));
   }
+  void AddInteger(const APInt &Int) {
+    const auto *Parts = Int.getRawData();
+    for (int i = 0, N = Int.getNumWords(); i < N; ++i) {
+      AddInteger(Parts[i]);
+    }
+  }
 
   void AddBoolean(bool B) { AddInteger(B ? 1U : 0U); }
   void AddString(StringRef String);
diff --git a/llvm/include/llvm/AsmParser/LLParser.h b/llvm/include/llvm/AsmParser/LLParser.h
index e5e1ade8b38b..e85728aa3c0d 100644
--- a/llvm/include/llvm/AsmParser/LLParser.h
+++ b/llvm/include/llvm/AsmParser/LLParser.h
@@ -369,6 +369,7 @@ namespace llvm {
     bool parseFnAttributeValuePairs(AttrBuilder &B,
                                     std::vector &FwdRefAttrGrps,
                                     bool inAttrGrp, LocTy &BuiltinLoc);
+    bool parseRangeAttr(AttrBuilder &B);
     bool parseRequiredTypeAttr(AttrBuilder &B, lltok::Kind AttrToken,
                                Attribute::AttrKind AttrKind);
 
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index c6f0ddf29a6d..c0a52d64a101 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -724,6 +724,7 @@ enum AttributeKindCodes {
   ATTR_KIND_WRITABLE = 89,
   ATTR_KIND_CORO_ONLY_DESTROY_WHEN_COMPLETE = 90,
   ATTR_KIND_DEAD_ON_UNWIND = 91,
+  ATTR_KIND_RANGE = 92,
 };
 
 enum ComdatSelectionKindCodes {
diff --git a/llvm/include/llvm/IR/Attributes.h b/llvm/include/llvm/IR/Attributes.h
index a4ebe5d732f5..0c2a02514ba0 100644
--- a/llvm/include/llvm/IR/Attributes.h
+++ b/llvm/include/llvm/IR/Attributes.h
@@ -37,6 +37,7 @@ class AttributeMask;
 class AttributeImpl;
 class AttributeListImpl;
 class AttributeSetNode;
+class ConstantRange;
 class FoldingSetNodeID;
 class Function;
 class LLVMContext;
@@ -103,6 +104,9 @@ public:
   static bool isTypeAttrKind(AttrKind Kind) {
     return Kind >= FirstTypeAttr && Kind <= LastTypeAttr;
   }
+  static bool isConstantRangeAttrKind(AttrKind Kind) {
+    return Kind >= FirstConstantRangeAttr && Kind <= LastConstantRangeAttr;
+  }
 
   static bool canUseAsFnAttr(AttrKind Kind);
   static bool canUseAsParamAttr(AttrKind Kind);
@@ -125,6 +129,8 @@ public:
   static Attribute get(LLVMContext &Context, StringRef Kind,
                        StringRef Val = StringRef());
   static Attribute get(LLVMContext &Context, AttrKind Kind, Type *Ty);
+  static Attribute get(LLVMContext &Context, AttrKind Kind,
+                       const ConstantRange &CR);
 
   /// Return a uniquified Attribute object that has the specific
   /// alignment set.
@@ -180,6 +186,9 @@ public:
   /// Return true if the attribute is a type attribute.
   bool isTypeAttribute() const;
 
+  /// Return true if the attribute is a ConstantRange attribute.
+  bool isConstantRangeAttribute() const;
+
   /// Return true if the attribute is any kind of attribute.
   bool isValid() const { return pImpl; }
 
@@ -213,6 +222,10 @@ public:
   /// a type attribute.
   Type *getValueAsType() const;
 
+  /// Return the attribute's value as a ConstantRange. This requires the
+  /// attribute to be a ConstantRange attribute.
+  ConstantRange getValueAsConstantRange() const;
+
   /// Returns the alignment field of an attribute as a byte alignment
   /// value.
   MaybeAlign getAlignment() const;
@@ -251,6 +264,9 @@ public:
   /// Return the FPClassTest for nofpclass
   FPClassTest getNoFPClass() const;
 
+  /// Returns the value of the range attribute.
+  ConstantRange getRange() const;
+
   /// The Attribute is converted to a string of equivalent mnemonic. This
   /// is, presumably, for writing out the mnemonics for the assembly writer.
   std::string getAsString(bool InAttrGrp = false) const;
@@ -1189,6 +1205,13 @@ public:
   // Add nofpclass attribute
   AttrBuilder &addNoFPClassAttr(FPClassTest NoFPClassMask);
 
+  /// Add a ConstantRange attribute with the given range.
+  AttrBuilder &addConstantRangeAttr(Attribute::AttrKind Kind,
+                                    const ConstantRange &CR);
+
+  /// Add range attribute.
+  AttrBuilder &addRangeAttr(const ConstantRange &CR);
+
   ArrayRef attrs() const { return Attrs; }
 
   bool operator==(const AttrBuilder &B) const;
diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td
index 08afecf32015..cef8b17769f0 100644
--- a/llvm/include/llvm/IR/Attributes.td
+++ b/llvm/include/llvm/IR/Attributes.td
@@ -44,6 +44,9 @@ class StrBoolAttr : Attr;
 /// Arbitrary string attribute.
 class ComplexStrAttr P> : Attr;
 
+/// ConstantRange attribute.
+class ConstantRangeAttr P> : Attr;
+
 /// Target-independent enum attributes.
 
 /// Alignment of parameter (5 bits) stored as log2 of alignment with +1 bias.
@@ -218,6 +221,9 @@ def OptimizeNone : EnumAttr<"optnone", [FnAttr]>;
 /// Similar to byval but without a copy.
 def Preallocated : TypeAttr<"preallocated", [FnAttr, ParamAttr]>;
 
+/// Parameter or return value is within the specified range.
+def Range : ConstantRangeAttr<"range", [ParamAttr, RetAttr]>;
+
 /// Function does not access memory.
 def ReadNone : EnumAttr<"readnone", [ParamAttr]>;
 
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index e140c9419520..78bcd94e23fa 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -1596,6 +1596,8 @@ bool LLParser::parseEnumAttribute(Attribute::AttrKind Attr, AttrBuilder &B,
 
     return true;
   }
+  case Attribute::Range:
+    return parseRangeAttr(B);
   default:
     B.addAttribute(Attr);
     Lex.Lex();
@@ -3008,6 +3010,47 @@ bool LLParser::parseRequiredTypeAttr(AttrBuilder &B, lltok::Kind AttrToken,
   return false;
 }
 
+/// parseRangeAttr
+///   ::= range( ,)
+bool LLParser::parseRangeAttr(AttrBuilder &B) {
+  Lex.Lex();
+
+  APInt Lower;
+  APInt Upper;
+  Type *Ty = nullptr;
+  LocTy TyLoc;
+
+  auto ParseAPSInt = [&](unsigned BitWidth, APInt &Val) {
+    if (Lex.getKind() != lltok::APSInt)
+      return tokError("expected integer");
+    if (Lex.getAPSIntVal().getBitWidth() > BitWidth)
+      return tokError(
+          "integer is too large for the bit width of specified type");
+    Val = Lex.getAPSIntVal().extend(BitWidth);
+    Lex.Lex();
+    return false;
+  };
+
+  if (parseToken(lltok::lparen, "expected '('") || parseType(Ty, TyLoc))
+    return true;
+  if (!Ty->isIntegerTy())
+    return error(TyLoc, "the range must have integer type!");
+
+  unsigned BitWidth = Ty->getPrimitiveSizeInBits();
+
+  if (ParseAPSInt(BitWidth, Lower) ||
+      parseToken(lltok::comma, "expected ','") || ParseAPSInt(BitWidth, Upper))
+    return true;
+  if (Lower == Upper)
+    return tokError("the range should not represent the full or empty set!");
+
+  if (parseToken(lltok::rparen, "expected ')'"))
+    return true;
+
+  B.addRangeAttr(ConstantRange(Lower, Upper));
+  return false;
+}
+
 /// parseOptionalOperandBundles
 ///    ::= /*empty*/
 ///    ::= '[' OperandBundle [, OperandBundle ]* ']'
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 832907a3f53f..9c63116114f3 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -815,6 +815,30 @@ private:
     return getFnValueByID(ValNo, Ty, TyID, ConstExprInsertBB);
   }
 
+  Expected readConstantRange(ArrayRef Record,
+                                            unsigned &OpNum) {
+    if (Record.size() - OpNum < 3)
+      return error("Too few records for range");
+    unsigned BitWidth = Record[OpNum++];
+    if (BitWidth > 64) {
+      unsigned LowerActiveWords = Record[OpNum];
+      unsigned UpperActiveWords = Record[OpNum++] >> 32;
+      if (Record.size() - OpNum < LowerActiveWords + UpperActiveWords)
+        return error("Too few records for range");
+      APInt Lower =
+          readWideAPInt(ArrayRef(&Record[OpNum], LowerActiveWords), BitWidth);
+      OpNum += LowerActiveWords;
+      APInt Upper =
+          readWideAPInt(ArrayRef(&Record[OpNum], UpperActiveWords), BitWidth);
+      OpNum += UpperActiveWords;
+      return ConstantRange(Lower, Upper);
+    } else {
+      int64_t Start = BitcodeReader::decodeSignRotatedValue(Record[OpNum++]);
+      int64_t End = BitcodeReader::decodeSignRotatedValue(Record[OpNum++]);
+      return ConstantRange(APInt(BitWidth, Start), APInt(BitWidth, End));
+    }
+  }
+
   /// Upgrades old-style typeless byval/sret/inalloca attributes by adding the
   /// corresponding argument's pointee type. Also upgrades intrinsics that now
   /// require an elementtype attribute.
@@ -2103,6 +2127,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
     return Attribute::CoroDestroyOnlyWhenComplete;
   case bitc::ATTR_KIND_DEAD_ON_UNWIND:
     return Attribute::DeadOnUnwind;
+  case bitc::ATTR_KIND_RANGE:
+    return Attribute::Range;
   }
 }
 
@@ -2272,6 +2298,21 @@ Error BitcodeReader::parseAttributeGroupBlock() {
             return error("Not a type attribute");
 
           B.addTypeAttr(Kind, HasType ? getTypeByID(Record[++i]) : nullptr);
+        } else if (Record[i] == 7) {
+          Attribute::AttrKind Kind;
+
+          i++;
+          if (Error Err = parseAttrKind(Record[i++], &Kind))
+            return Err;
+          if (!Attribute::isConstantRangeAttrKind(Kind))
+            return error("Not a ConstantRange attribute");
+
+          Expected MaybeCR = readConstantRange(Record, i);
+          if (!MaybeCR)
+            return MaybeCR.takeError();
+          i--;
+
+          B.addConstantRangeAttr(Kind, MaybeCR.get());
         } else {
           return error("Invalid attribute group entry");
         }
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 656f2a6ce870..597f49332fad 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -844,6 +844,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
     return bitc::ATTR_KIND_CORO_ONLY_DESTROY_WHEN_COMPLETE;
   case Attribute::DeadOnUnwind:
     return bitc::ATTR_KIND_DEAD_ON_UNWIND;
+  case Attribute::Range:
+    return bitc::ATTR_KIND_RANGE;
   case Attribute::EndAttrKinds:
     llvm_unreachable("Can not encode end-attribute kinds marker.");
   case Attribute::None:
@@ -856,6 +858,39 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
   llvm_unreachable("Trying to encode unknown attribute");
 }
 
+static void emitSignedInt64(SmallVectorImpl &Vals, uint64_t V) {
+  if ((int64_t)V >= 0)
+    Vals.push_back(V << 1);
+  else
+    Vals.push_back((-V << 1) | 1);
+}
+
+static void emitWideAPInt(SmallVectorImpl &Vals, const APInt &A) {
+  // We have an arbitrary precision integer value to write whose
+  // bit width is > 64. However, in canonical unsigned integer
+  // format it is likely that the high bits are going to be zero.
+  // So, we only write the number of active words.
+  unsigned NumWords = A.getActiveWords();
+  const uint64_t *RawData = A.getRawData();
+  for (unsigned i = 0; i < NumWords; i++)
+    emitSignedInt64(Vals, RawData[i]);
+}
+
+static void emitConstantRange(SmallVectorImpl &Record,
+                              const ConstantRange &CR) {
+  unsigned BitWidth = CR.getBitWidth();
+  Record.push_back(BitWidth);
+  if (BitWidth > 64) {
+    Record.push_back(CR.getLower().getActiveWords() |
+                     (uint64_t(CR.getUpper().getActiveWords()) << 32));
+    emitWideAPInt(Record, CR.getLower());
+    emitWideAPInt(Record, CR.getUpper());
+  } else {
+    emitSignedInt64(Record, CR.getLower().getSExtValue());
+    emitSignedInt64(Record, CR.getUpper().getSExtValue());
+  }
+}
+
 void ModuleBitcodeWriter::writeAttributeGroupTable() {
   const std::vector &AttrGrps =
       VE.getAttributeGroups();
@@ -889,13 +924,17 @@ void ModuleBitcodeWriter::writeAttributeGroupTable() {
           Record.append(Val.begin(), Val.end());
           Record.push_back(0);
         }
-      } else {
-        assert(Attr.isTypeAttribute());
+      } else if (Attr.isTypeAttribute()) {
         Type *Ty = Attr.getValueAsType();
         Record.push_back(Ty ? 6 : 5);
         Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum()));
         if (Ty)
           Record.push_back(VE.getTypeID(Attr.getValueAsType()));
+      } else {
+        assert(Attr.isConstantRangeAttribute());
+        Record.push_back(7);
+        Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum()));
+        emitConstantRange(Record, Attr.getValueAsConstantRange());
       }
     }
 
@@ -1716,24 +1755,6 @@ void ModuleBitcodeWriter::writeDIGenericSubrange(
   Record.clear();
 }
 
-static void emitSignedInt64(SmallVectorImpl &Vals, uint64_t V) {
-  if ((int64_t)V >= 0)
-    Vals.push_back(V << 1);
-  else
-    Vals.push_back((-V << 1) | 1);
-}
-
-static void emitWideAPInt(SmallVectorImpl &Vals, const APInt &A) {
-  // We have an arbitrary precision integer value to write whose
-  // bit width is > 64. However, in canonical unsigned integer
-  // format it is likely that the high bits are going to be zero.
-  // So, we only write the number of active words.
-  unsigned NumWords = A.getActiveWords();
-  const uint64_t *RawData = A.getRawData();
-  for (unsigned i = 0; i < NumWords; i++)
-    emitSignedInt64(Vals, RawData[i]);
-}
-
 void ModuleBitcodeWriter::writeDIEnumerator(const DIEnumerator *N,
                                             SmallVectorImpl &Record,
                                             unsigned Abbrev) {
diff --git a/llvm/lib/IR/AttributeImpl.h b/llvm/lib/IR/AttributeImpl.h
index 78496786b0ae..9a6427bbc3d5 100644
--- a/llvm/lib/IR/AttributeImpl.h
+++ b/llvm/lib/IR/AttributeImpl.h
@@ -20,6 +20,7 @@
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/Attributes.h"
+#include "llvm/IR/ConstantRange.h"
 #include "llvm/Support/TrailingObjects.h"
 #include 
 #include 
@@ -46,6 +47,7 @@ protected:
     IntAttrEntry,
     StringAttrEntry,
     TypeAttrEntry,
+    ConstantRangeAttrEntry,
   };
 
   AttributeImpl(AttrEntryKind KindID) : KindID(KindID) {}
@@ -59,6 +61,9 @@ public:
   bool isIntAttribute() const { return KindID == IntAttrEntry; }
   bool isStringAttribute() const { return KindID == StringAttrEntry; }
   bool isTypeAttribute() const { return KindID == TypeAttrEntry; }
+  bool isConstantRangeAttribute() const {
+    return KindID == ConstantRangeAttrEntry;
+  }
 
   bool hasAttribute(Attribute::AttrKind A) const;
   bool hasAttribute(StringRef Kind) const;
@@ -72,6 +77,8 @@ public:
 
   Type *getValueAsType() const;
 
+  ConstantRange getValueAsConstantRange() const;
+
   /// Used when sorting the attributes.
   bool operator<(const AttributeImpl &AI) const;
 
@@ -82,8 +89,10 @@ public:
       Profile(ID, getKindAsEnum(), getValueAsInt());
     else if (isStringAttribute())
       Profile(ID, getKindAsString(), getValueAsString());
-    else
+    else if (isTypeAttribute())
       Profile(ID, getKindAsEnum(), getValueAsType());
+    else
+      Profile(ID, getKindAsEnum(), getValueAsConstantRange());
   }
 
   static void Profile(FoldingSetNodeID &ID, Attribute::AttrKind Kind) {
@@ -108,6 +117,13 @@ public:
     ID.AddInteger(Kind);
     ID.AddPointer(Ty);
   }
+
+  static void Profile(FoldingSetNodeID &ID, Attribute::AttrKind Kind,
+                      const ConstantRange &CR) {
+    ID.AddInteger(Kind);
+    ID.AddInteger(CR.getLower());
+    ID.AddInteger(CR.getUpper());
+  }
 };
 
 static_assert(std::is_trivially_destructible::value,
@@ -196,6 +212,16 @@ public:
   Type *getTypeValue() const { return Ty; }
 };
 
+class ConstantRangeAttributeImpl : public EnumAttributeImpl {
+  ConstantRange CR;
+
+public:
+  ConstantRangeAttributeImpl(Attribute::AttrKind Kind, const ConstantRange &CR)
+      : EnumAttributeImpl(ConstantRangeAttrEntry, Kind), CR(CR) {}
+
+  ConstantRange getConstantRangeValue() const { return CR; }
+};
+
 class AttributeBitSet {
   /// Bitset with a bit for each available attribute Attribute::AttrKind.
   uint8_t AvailableAttrs[12] = {};
diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp
index 00acbbe7989d..fec34d0a4a81 100644
--- a/llvm/lib/IR/Attributes.cpp
+++ b/llvm/lib/IR/Attributes.cpp
@@ -24,6 +24,7 @@
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/AttributeMask.h"
+#include "llvm/IR/ConstantRange.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Type.h"
@@ -165,6 +166,30 @@ Attribute Attribute::get(LLVMContext &Context, Attribute::AttrKind Kind,
   return Attribute(PA);
 }
 
+Attribute Attribute::get(LLVMContext &Context, Attribute::AttrKind Kind,
+                         const ConstantRange &CR) {
+  assert(Attribute::isConstantRangeAttrKind(Kind) &&
+         "Not a ConstantRange attribute");
+  LLVMContextImpl *pImpl = Context.pImpl;
+  FoldingSetNodeID ID;
+  ID.AddInteger(Kind);
+  ID.AddInteger(CR.getLower());
+  ID.AddInteger(CR.getUpper());
+
+  void *InsertPoint;
+  AttributeImpl *PA = pImpl->AttrsSet.FindNodeOrInsertPos(ID, InsertPoint);
+
+  if (!PA) {
+    // If we didn't find any existing attributes of the same shape then create a
+    // new one and insert it.
+    PA = new (pImpl->Alloc) ConstantRangeAttributeImpl(Kind, CR);
+    pImpl->AttrsSet.InsertNode(PA, InsertPoint);
+  }
+
+  // Return the Attribute that we found or created.
+  return Attribute(PA);
+}
+
 Attribute Attribute::getWithAlignment(LLVMContext &Context, Align A) {
   assert(A <= llvm::Value::MaximumAlignment && "Alignment too large.");
   return get(Context, Alignment, A.value());
@@ -287,9 +312,14 @@ bool Attribute::isTypeAttribute() const {
   return pImpl && pImpl->isTypeAttribute();
 }
 
+bool Attribute::isConstantRangeAttribute() const {
+  return pImpl && pImpl->isConstantRangeAttribute();
+}
+
 Attribute::AttrKind Attribute::getKindAsEnum() const {
   if (!pImpl) return None;
-  assert((isEnumAttribute() || isIntAttribute() || isTypeAttribute()) &&
+  assert((isEnumAttribute() || isIntAttribute() || isTypeAttribute() ||
+          isConstantRangeAttribute()) &&
          "Invalid attribute type to get the kind as an enum!");
   return pImpl->getKindAsEnum();
 }
@@ -329,6 +359,11 @@ Type *Attribute::getValueAsType() const {
   return pImpl->getValueAsType();
 }
 
+ConstantRange Attribute::getValueAsConstantRange() const {
+  assert(isConstantRangeAttribute() &&
+         "Invalid attribute type to get the value as a ConstantRange!");
+  return pImpl->getValueAsConstantRange();
+}
 
 bool Attribute::hasAttribute(AttrKind Kind) const {
   return (pImpl && pImpl->hasAttribute(Kind)) || (!pImpl && Kind == None);
@@ -408,6 +443,12 @@ FPClassTest Attribute::getNoFPClass() const {
   return static_cast(pImpl->getValueAsInt());
 }
 
+ConstantRange Attribute::getRange() const {
+  assert(hasAttribute(Attribute::Range) &&
+         "Trying to get range args from non-range attribute");
+  return pImpl->getValueAsConstantRange();
+}
+
 static const char *getModRefStr(ModRefInfo MR) {
   switch (MR) {
   case ModRefInfo::NoModRef:
@@ -562,6 +603,18 @@ std::string Attribute::getAsString(bool InAttrGrp) const {
     return Result;
   }
 
+  if (hasAttribute(Attribute::Range)) {
+    std::string Result;
+    raw_string_ostream OS(Result);
+    ConstantRange CR = getValueAsConstantRange();
+    OS << "range(";
+    OS << "i" << CR.getBitWidth() << " ";
+    OS << CR.getLower() << ", " << CR.getUpper();
+    OS << ")";
+    OS.flush();
+    return Result;
+  }
+
   // Convert target-dependent attributes to strings of the form:
   //
   //   "kind"
@@ -651,7 +704,8 @@ bool AttributeImpl::hasAttribute(StringRef Kind) const {
 }
 
 Attribute::AttrKind AttributeImpl::getKindAsEnum() const {
-  assert(isEnumAttribute() || isIntAttribute() || isTypeAttribute());
+  assert(isEnumAttribute() || isIntAttribute() || isTypeAttribute() ||
+         isConstantRangeAttribute());
   return static_cast(this)->getEnumKind();
 }
 
@@ -680,6 +734,12 @@ Type *AttributeImpl::getValueAsType() const {
   return static_cast(this)->getTypeValue();
 }
 
+ConstantRange AttributeImpl::getValueAsConstantRange() const {
+  assert(isConstantRangeAttribute());
+  return static_cast(this)
+      ->getConstantRangeValue();
+}
+
 bool AttributeImpl::operator<(const AttributeImpl &AI) const {
   if (this == &AI)
     return false;
@@ -693,6 +753,7 @@ bool AttributeImpl::operator<(const AttributeImpl &AI) const {
       return getKindAsEnum() < AI.getKindAsEnum();
     assert(!AI.isEnumAttribute() && "Non-unique attribute");
     assert(!AI.isTypeAttribute() && "Comparison of types would be unstable");
+    assert(!AI.isConstantRangeAttribute() && "Unclear how to compare ranges");
     // TODO: Is this actually needed?
     assert(AI.isIntAttribute() && "Only possibility left");
     return getValueAsInt() < AI.getValueAsInt();
@@ -1881,6 +1942,15 @@ AttrBuilder &AttrBuilder::addInAllocaAttr(Type *Ty) {
   return addTypeAttr(Attribute::InAlloca, Ty);
 }
 
+AttrBuilder &AttrBuilder::addConstantRangeAttr(Attribute::AttrKind Kind,
+                                               const ConstantRange &CR) {
+  return addAttribute(Attribute::get(Ctx, Kind, CR));
+}
+
+AttrBuilder &AttrBuilder::addRangeAttr(const ConstantRange &CR) {
+  return addConstantRangeAttr(Attribute::Range, CR);
+}
+
 AttrBuilder &AttrBuilder::merge(const AttrBuilder &B) {
   // TODO: Could make this O(n) as we're merging two sorted lists.
   for (const auto &I : B.attrs())
@@ -1952,6 +2022,12 @@ AttributeMask AttributeFuncs::typeIncompatible(Type *Ty,
       Incompatible.addAttribute(Attribute::SExt).addAttribute(Attribute::ZExt);
   }
 
+  if (!Ty->isIntOrIntVectorTy()) {
+    // Attributes that only apply to integers or vector of integers.
+    if (ASK & ASK_SAFE_TO_DROP)
+      Incompatible.addAttribute(Attribute::Range);
+  }
+
   if (!Ty->isPointerTy()) {
     // Attributes that only apply to pointers.
     if (ASK & ASK_SAFE_TO_DROP)
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index fd5f7d57c258..3cf5e81efb3b 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -2039,6 +2039,11 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty,
     Check((Val & ~static_cast(fcAllFlags)) == 0,
           "Invalid value for 'nofpclass' test mask", V);
   }
+  if (Attrs.hasAttribute(Attribute::Range)) {
+    auto CR = Attrs.getAttribute(Attribute::Range).getValueAsConstantRange();
+    Check(Ty->isIntOrIntVectorTy(CR.getBitWidth()),
+          "Range bit width must match type bit width!", V);
+  }
 }
 
 void Verifier::checkUnsignedBaseTenFuncAttr(AttributeList Attrs, StringRef Attr,
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index 3071ec0c9113..ab2d25c3f17c 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -999,6 +999,7 @@ Function *CodeExtractor::constructFunction(const ValueSet &inputs,
       case Attribute::WriteOnly:
       case Attribute::Writable:
       case Attribute::DeadOnUnwind:
+      case Attribute::Range:
       //  These are not really attributes.
       case Attribute::None:
       case Attribute::EndAttrKinds:
diff --git a/llvm/test/Assembler/range-attribute-invalid-range.ll b/llvm/test/Assembler/range-attribute-invalid-range.ll
new file mode 100644
index 000000000000..cf6d3f080183
--- /dev/null
+++ b/llvm/test/Assembler/range-attribute-invalid-range.ll
@@ -0,0 +1,6 @@
+; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s
+
+; CHECK: the range should not represent the full or empty set!
+define void @range_empty(i8 range(i8 0, 0) %a) {
+  ret void
+}
diff --git a/llvm/test/Assembler/range-attribute-invalid-type.ll b/llvm/test/Assembler/range-attribute-invalid-type.ll
new file mode 100644
index 000000000000..cc09149a94dc
--- /dev/null
+++ b/llvm/test/Assembler/range-attribute-invalid-type.ll
@@ -0,0 +1,6 @@
+; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s
+
+; CHECK: the range must have integer type!
+define void @range_vector_type(i8 range(<4 x i32> 0, 0) %a) {
+  ret void
+}
diff --git a/llvm/test/Bitcode/attributes.ll b/llvm/test/Bitcode/attributes.ll
index 6921f11a352d..26163b4d38c8 100644
--- a/llvm/test/Bitcode/attributes.ll
+++ b/llvm/test/Bitcode/attributes.ll
@@ -526,6 +526,16 @@ define void @f91(ptr dead_on_unwind %p) {
   ret void
 }
 
+; CHECK: define range(i32 -1, 42) i32 @range_attribute(<4 x i32> range(i32 -1, 42) %a)
+define range(i32 -1, 42) i32 @range_attribute(<4 x i32> range(i32 -1, 42) %a) {
+  ret i32 0
+}
+
+; CHECK: define void @wide_range_attribute(i128 range(i128 618970019642690137449562111, 618970019642690137449562114) %a)
+define void @wide_range_attribute(i128 range(i128 618970019642690137449562111, 618970019642690137449562114) %a) {
+  ret void
+}
+
 ; CHECK: attributes #0 = { noreturn }
 ; CHECK: attributes #1 = { nounwind }
 ; CHECK: attributes #2 = { memory(none) }
diff --git a/llvm/test/Verifier/range-attr.ll b/llvm/test/Verifier/range-attr.ll
new file mode 100644
index 000000000000..f985ab696eac
--- /dev/null
+++ b/llvm/test/Verifier/range-attr.ll
@@ -0,0 +1,19 @@
+; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+
+; CHECK: Range bit width must match type bit width!
+; CHECK-NEXT: ptr @bit_widths_do_not_match
+define void @bit_widths_do_not_match(i32 range(i8 1, 0) %a) {
+  ret void
+}
+
+; CHECK: Range bit width must match type bit width!
+; CHECK-NEXT: ptr @bit_widths_do_not_match_vector
+define void @bit_widths_do_not_match_vector(<4 x i32> range(i8 1, 0) %a) {
+  ret void
+}
+
+; CHECK: Attribute 'range(i8 1, 0)' applied to incompatible type!
+; CHECK-NEXT: ptr @not-integer-type
+define void @not-integer-type(ptr range(i8 1, 0) %a) {
+  ret void
+}
diff --git a/llvm/utils/TableGen/Attributes.cpp b/llvm/utils/TableGen/Attributes.cpp
index db3c4decccb4..d9fc7834416c 100644
--- a/llvm/utils/TableGen/Attributes.cpp
+++ b/llvm/utils/TableGen/Attributes.cpp
@@ -53,7 +53,8 @@ void Attributes::emitTargetIndependentNames(raw_ostream &OS) {
   };
 
   // Emit attribute enums in the same order llvm::Attribute::operator< expects.
-  Emit({"EnumAttr", "TypeAttr", "IntAttr"}, "ATTRIBUTE_ENUM");
+  Emit({"EnumAttr", "TypeAttr", "IntAttr", "ConstantRangeAttr"},
+       "ATTRIBUTE_ENUM");
   Emit({"StrBoolAttr"}, "ATTRIBUTE_STRBOOL");
   Emit({"ComplexStrAttr"}, "ATTRIBUTE_COMPLEXSTR");
 
@@ -63,7 +64,8 @@ void Attributes::emitTargetIndependentNames(raw_ostream &OS) {
   OS << "#ifdef GET_ATTR_ENUM\n";
   OS << "#undef GET_ATTR_ENUM\n";
   unsigned Value = 1; // Leave zero for AttrKind::None.
-  for (StringRef KindName : {"EnumAttr", "TypeAttr", "IntAttr"}) {
+  for (StringRef KindName :
+       {"EnumAttr", "TypeAttr", "IntAttr", "ConstantRangeAttr"}) {
     OS << "First" << KindName << " = " << Value << ",\n";
     for (auto *A : Records.getAllDerivedDefinitions(KindName)) {
       OS << A->getName() << " = " << Value << ",\n";
@@ -117,7 +119,8 @@ void Attributes::emitAttributeProperties(raw_ostream &OS) {
   OS << "#ifdef GET_ATTR_PROP_TABLE\n";
   OS << "#undef GET_ATTR_PROP_TABLE\n";
   OS << "static const uint8_t AttrPropTable[] = {\n";
-  for (StringRef KindName : {"EnumAttr", "TypeAttr", "IntAttr"}) {
+  for (StringRef KindName :
+       {"EnumAttr", "TypeAttr", "IntAttr", "ConstantRangeAttr"}) {
     for (auto *A : Records.getAllDerivedDefinitions(KindName)) {
       OS << "0";
       for (Init *P : *A->getValueAsListInit("Properties"))
-- 
GitLab


From 2b4d8188b263019477fa996b74b3da8a87a0e04b Mon Sep 17 00:00:00 2001
From: Lukacma 
Date: Fri, 8 Mar 2024 15:35:48 +0000
Subject: [PATCH 620/929] [Clang][LLVM][SVE2.1]  Created intrinsics for DUPQ
 instr. (#83260)

This patch adds clang and llvm support for following intrinsic and maps
it to DUPQ instruction:
```
   // Variants are also available for:
   // _s8, _u16, _s16, _u32, _s32, _u64, _s64
   // _bf16, _f16, _f32, _f64
   svuint8_t svdup_laneq[_u8](svuint8_t zn, uint64_t imm_idx);
```
---
 clang/include/clang/Basic/arm_sve.td          |   9 +
 .../acle_sve2p1_dupq.c                        | 213 ++++++++++++++++++
 .../acle_sve2p1_imm.cpp                       |  29 +++
 llvm/include/llvm/IR/IntrinsicsAArch64.td     |   7 +
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td |   2 +-
 llvm/lib/Target/AArch64/SVEInstrFormats.td    |  20 +-
 .../CodeGen/AArch64/sve2p1-intrinsics-dupq.ll |  83 +++++++
 7 files changed, 357 insertions(+), 6 deletions(-)
 create mode 100644 clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_dupq.c
 create mode 100644 llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dupq.ll

diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td
index 6da30e08e752..6cc249837d3f 100644
--- a/clang/include/clang/Basic/arm_sve.td
+++ b/clang/include/clang/Basic/arm_sve.td
@@ -2215,6 +2215,15 @@ let TargetGuard = "sve2p1" in {
   def SVTBXQ : SInst<"svtbxq[_{d}]", "dddu", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_tbxq">;
   // EXTQ
   def EXTQ : SInst<"svextq[_{d}]", "dddk", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_extq", [], [ImmCheck<2, ImmCheck0_15>]>;
+  // DUPQ
+  def SVDUP_LANEQ_B  : SInst<"svdup_laneq[_{d}]", "ddi",  "cUc", MergeNone, "aarch64_sve_dup_laneq", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_15>]>;
+  def SVDUP_LANEQ_H  : SInst<"svdup_laneq[_{d}]", "ddi",  "sUsh", MergeNone, "aarch64_sve_dup_laneq", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_7>]>;
+  def SVDUP_LANEQ_S  : SInst<"svdup_laneq[_{d}]", "ddi",  "iUif", MergeNone, "aarch64_sve_dup_laneq", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_3>]>;
+  def SVDUP_LANEQ_D  : SInst<"svdup_laneq[_{d}]", "ddi",  "lUld", MergeNone, "aarch64_sve_dup_laneq", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_1>]>;
+
+  let TargetGuard = "bf16" in {
+    def SVDUP_LANEQ_BF16  : SInst<"svdup_laneq[_{d}]", "ddi",  "b", MergeNone, "aarch64_sve_dup_laneq", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_7>]>;
+  }
   // PMOV
   // Move to Pred
   multiclass PMOV_TO_PRED flags=[], ImmCheckType immCh > {
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_dupq.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_dupq.c
new file mode 100644
index 000000000000..587a67aa6b7c
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_dupq.c
@@ -0,0 +1,213 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -target-feature +bf16\
+// RUN:   -S -Werror -emit-llvm -disable-O0-optnone -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -target-feature +bf16\
+// RUN:   -S -Werror -emit-llvm -disable-O0-optnone -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -target-feature +bf16\
+// RUN:   -S -Werror -emit-llvm -disable-O0-optnone -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -target-feature +bf16\
+// RUN:   -S -Werror -emit-llvm -disable-O0-optnone -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -target-feature +bf16 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+
+
+#include 
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED) A1
+#else
+#define SVE_ACLE_FUNC(A1, A2) A1##A2
+#endif
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_s8
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv16i8( [[ZN]], i32 0)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z19test_svdup_laneq_s8u10__SVInt8_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0:[0-9]+]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv16i8( [[ZN]], i32 0)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svint8_t test_svdup_laneq_s8(svint8_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _s8)(zn, 0);
+}
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_u8
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv16i8( [[ZN]], i32 15)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z19test_svdup_laneq_u8u11__SVUint8_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv16i8( [[ZN]], i32 15)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svuint8_t test_svdup_laneq_u8(svuint8_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _u8)(zn, 15);
+}
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_s16
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv8i16( [[ZN]], i32 1)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z20test_svdup_laneq_s16u11__SVInt16_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv8i16( [[ZN]], i32 1)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svint16_t test_svdup_laneq_s16(svint16_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _s16)(zn, 1);
+}
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_u16
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv8i16( [[ZN]], i32 7)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z20test_svdup_laneq_u16u12__SVUint16_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv8i16( [[ZN]], i32 7)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svuint16_t test_svdup_laneq_u16(svuint16_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _u16)(zn, 7);
+}
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_s32
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv4i32( [[ZN]], i32 2)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z20test_svdup_laneq_s32u11__SVInt32_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv4i32( [[ZN]], i32 2)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svint32_t test_svdup_laneq_s32(svint32_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _s32)(zn, 2);
+}
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_u32
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv4i32( [[ZN]], i32 3)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z20test_svdup_laneq_u32u12__SVUint32_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv4i32( [[ZN]], i32 3)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svuint32_t test_svdup_laneq_u32(svuint32_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _u32)(zn, 3);
+}
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_s64
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv2i64( [[ZN]], i32 0)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z20test_svdup_laneq_s64u11__SVInt64_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv2i64( [[ZN]], i32 0)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svint64_t test_svdup_laneq_s64(svint64_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _s64)(zn, 0);
+}
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_u64
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv2i64( [[ZN]], i32 1)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z20test_svdup_laneq_u64u12__SVUint64_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv2i64( [[ZN]], i32 1)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svuint64_t test_svdup_laneq_u64(svuint64_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _u64)(zn, 1);
+}
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_f16
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv8f16( [[ZN]], i32 4)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z20test_svdup_laneq_f16u13__SVFloat16_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv8f16( [[ZN]], i32 4)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svfloat16_t test_svdup_laneq_f16(svfloat16_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _f16)(zn, 4);
+}
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_f32
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv4f32( [[ZN]], i32 1)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z20test_svdup_laneq_f32u13__SVFloat32_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv4f32( [[ZN]], i32 1)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svfloat32_t test_svdup_laneq_f32(svfloat32_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _f32)(zn, 1);
+}
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_f64
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv2f64( [[ZN]], i32 1)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z20test_svdup_laneq_f64u13__SVFloat64_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv2f64( [[ZN]], i32 1)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svfloat64_t test_svdup_laneq_f64(svfloat64_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _f64)(zn, 1);
+}
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_bf16
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv8bf16( [[ZN]], i32 3)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z21test_svdup_laneq_bf16u14__SVBfloat16_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv8bf16( [[ZN]], i32 3)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svbfloat16_t test_svdup_laneq_bf16(svbfloat16_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _bf16)(zn, 3);
+}
diff --git a/clang/test/Sema/aarch64-sve2p1-intrinsics/acle_sve2p1_imm.cpp b/clang/test/Sema/aarch64-sve2p1-intrinsics/acle_sve2p1_imm.cpp
index d857608d4415..a6ec5150f0aa 100644
--- a/clang/test/Sema/aarch64-sve2p1-intrinsics/acle_sve2p1_imm.cpp
+++ b/clang/test/Sema/aarch64-sve2p1-intrinsics/acle_sve2p1_imm.cpp
@@ -188,3 +188,32 @@ void test_svget_svset_b(uint64_t idx, svboolx2_t tuple2, svboolx4_t tuple4, svbo
   svget2_b(tuple2, idx); // expected-error {{argument to 'svget2_b' must be a constant integer}}
   svget4_b(tuple4, idx); // expected-error {{argument to 'svget4_b' must be a constant integer}}
 }
+
+__attribute__((target("+sve2p1")))
+void test_svdup_laneq(){  
+  svuint8_t zn_u8;
+  svuint16_t zn_u16;
+  svuint32_t zn_u32;
+  svuint64_t zn_u64;
+  svint8_t zn_s8;
+  svint16_t zn_s16;
+  svint32_t zn_s32;
+  svint64_t zn_s64;
+  svfloat16_t zn_f16;
+  svfloat32_t zn_f32;
+  svfloat64_t zn_f64;
+  svbfloat16_t zn_bf16;
+
+  svdup_laneq_u8(zn_u8,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 15]}}
+  svdup_laneq_u16(zn_u16,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 7]}}
+  svdup_laneq_u32(zn_u32,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+  svdup_laneq_u64(zn_u64,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 1]}}
+  svdup_laneq_s8(zn_s8,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 15]}}
+  svdup_laneq_s16(zn_s16,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 7]}}
+  svdup_laneq_s32(zn_s32,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+  svdup_laneq_s64(zn_s64,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 1]}}
+  svdup_laneq_f16(zn_f16,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 7]}}
+  svdup_laneq_f32(zn_f32,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+  svdup_laneq_f64(zn_f64,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 1]}}
+  svdup_laneq_bf16(zn_bf16,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 7]}}
+}
\ No newline at end of file
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 5a9a7c4b43a1..bcaa37de74b6 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -1360,6 +1360,12 @@ let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
                  LLVMSubdivide2VectorType<0>,
                  llvm_i32_ty],
                 [IntrNoMem, ImmArg>]>;
+  
+  class SVE2_1VectorArgIndexed_Intrinsic
+    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+                [LLVMMatchType<0>,
+                 llvm_i32_ty],
+                [IntrNoMem, ImmArg>]>;
 
   class AdvSIMD_SVE_CDOT_LANE_Intrinsic
     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
@@ -1913,6 +1919,7 @@ def int_aarch64_sve_clastb    : AdvSIMD_Pred2VectorArg_Intrinsic;
 def int_aarch64_sve_clastb_n  : AdvSIMD_SVE_ReduceWithInit_Intrinsic;
 def int_aarch64_sve_compact   : AdvSIMD_Pred1VectorArg_Intrinsic;
 def int_aarch64_sve_dupq_lane : AdvSIMD_SVE_DUPQ_Intrinsic;
+def int_aarch64_sve_dup_laneq : SVE2_1VectorArgIndexed_Intrinsic;
 def int_aarch64_sve_ext       : AdvSIMD_2VectorArgIndexed_Intrinsic;
 def int_aarch64_sve_sel       : AdvSIMD_Pred2VectorArg_Intrinsic;
 def int_aarch64_sve_lasta     : AdvSIMD_SVE_Reduce_Intrinsic;
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 7c98f934a131..e0a010af4155 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4101,7 +4101,7 @@ defm FMINNMQV : sve2p1_fp_reduction_q<0b101, "fminnmqv", int_aarch64_sve_fminnmq
 defm FMAXQV   : sve2p1_fp_reduction_q<0b110, "fmaxqv", int_aarch64_sve_fmaxqv>;
 defm FMINQV   : sve2p1_fp_reduction_q<0b111, "fminqv", int_aarch64_sve_fminqv>;
 
-defm DUPQ_ZZI : sve2p1_dupq<"dupq">;
+defm DUPQ_ZZI : sve2p1_dupq<"dupq", int_aarch64_sve_dup_laneq>;
 defm EXTQ_ZZI : sve2p1_extq<"extq", int_aarch64_sve_extq>;
 
 defm PMOV_PZI : sve2p1_vector_to_pred<"pmov", int_aarch64_sve_pmov_to_pred_lane, int_aarch64_sve_pmov_to_pred_lane_zero>;
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 8baf3a6d3d81..c19e02bb03d1 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -9893,23 +9893,33 @@ class sve2p1_dupq ind_tsz, string mnemonic, ZPRRegOp zprty, Operand ityp
   let hasSideEffects = 0;
 }
 
-multiclass sve2p1_dupq {
-  def _B : sve2p1_dupq<{?, ?, ?, ?, 1}, mnemonic, ZPR8, VectorIndexB32b> {
+multiclass sve2p1_dupq {
+  def _B : sve2p1_dupq<{?, ?, ?, ?, 1}, mnemonic, ZPR8, VectorIndexB32b_timm> {
     bits<4> index;
     let Inst{20-17} = index;
   }
-  def _H : sve2p1_dupq<{?, ?, ?, 1, 0}, mnemonic, ZPR16, VectorIndexH32b> {
+  def _H : sve2p1_dupq<{?, ?, ?, 1, 0}, mnemonic, ZPR16, VectorIndexH32b_timm> {
     bits<3> index;
     let Inst{20-18} = index;
   }
-  def _S : sve2p1_dupq<{?, ?, 1, 0, 0}, mnemonic, ZPR32, VectorIndexS32b> {
+  def _S : sve2p1_dupq<{?, ?, 1, 0, 0}, mnemonic, ZPR32, VectorIndexS32b_timm> {
     bits<2> index;
     let Inst{20-19} = index;
   }
-  def _D : sve2p1_dupq<{?, 1, 0, 0, 0}, mnemonic, ZPR64, VectorIndexD32b> {
+  def _D : sve2p1_dupq<{?, 1, 0, 0, 0}, mnemonic, ZPR64, VectorIndexD32b_timm> {
     bits<1> index;
     let Inst{20} = index;
   }
+
+  def : SVE_2_Op_Imm_Pat(NAME # _B)>;
+  def : SVE_2_Op_Imm_Pat(NAME # _H)>;
+  def : SVE_2_Op_Imm_Pat(NAME # _S)>;
+  def : SVE_2_Op_Imm_Pat(NAME # _D)>;
+
+  def : SVE_2_Op_Imm_Pat(NAME # _H)>;
+  def : SVE_2_Op_Imm_Pat(NAME # _S)>;
+  def : SVE_2_Op_Imm_Pat(NAME # _D)>;
+  def : SVE_2_Op_Imm_Pat(NAME # _H)>;
 }
 
 
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dupq.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dupq.ll
new file mode 100644
index 000000000000..f1a423b02ac2
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dupq.ll
@@ -0,0 +1,83 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve2p1,+bf16 < %s | FileCheck %s
+
+define  @test_dupq_i8 ( %zn) {
+; CHECK-LABEL: test_dupq_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dupq z0.b, z0.b[15]
+; CHECK-NEXT:    ret
+  %res = call  @llvm.aarch64.sve.dup.laneq.nxv16i8( %zn, i32 15)
+  ret  %res
+}
+
+define  @test_dupq_i16 ( %zn) {
+; CHECK-LABEL: test_dupq_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dupq z0.h, z0.h[7]
+; CHECK-NEXT:    ret
+  %res = call  @llvm.aarch64.sve.dup.laneq.nxv8i16( %zn, i32 7)
+  ret  %res
+}
+
+define  @test_dupq__i32 ( %zn) {
+; CHECK-LABEL: test_dupq__i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dupq z0.s, z0.s[3]
+; CHECK-NEXT:    ret
+  %res = call  @llvm.aarch64.sve.dup.laneq.nxv4i32( %zn, i32 3)
+  ret  %res
+}
+
+define  @test_dupq_i64 ( %zn) {
+; CHECK-LABEL: test_dupq_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dupq z0.d, z0.d[1]
+; CHECK-NEXT:    ret
+  %res = call  @llvm.aarch64.sve.dup.laneq.nxv2i64( %zn, i32 1)
+  ret  %res
+}
+
+define  @test_dupq_f16( %zn) {
+; CHECK-LABEL: test_dupq_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dupq z0.h, z0.h[4]
+; CHECK-NEXT:    ret
+  %res = call  @llvm.aarch64.sve.dup.laneq.nxv8f16( %zn, i32 4)
+  ret  %res
+}
+
+define  @test_dupq_f32( %zn) {
+; CHECK-LABEL: test_dupq_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dupq z0.s, z0.s[2]
+; CHECK-NEXT:    ret
+  %res = call  @llvm.aarch64.sve.dup.laneq.nxv4f32( %zn, i32 2)
+  ret  %res
+}
+
+define  @test_dupq_f64( %zn) {
+; CHECK-LABEL: test_dupq_f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dupq z0.d, z0.d[0]
+; CHECK-NEXT:    ret
+  %res = call  @llvm.aarch64.sve.dup.laneq.nxv2f64( %zn, i32 0)
+  ret  %res
+}
+
+define  @test_dupq_bf16( %zn) {
+; CHECK-LABEL: test_dupq_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dupq z0.h, z0.h[1]
+; CHECK-NEXT:    ret
+  %res = call  @llvm.aarch64.sve.dup.laneq.nxv8bf16( %zn, i32 1)
+  ret  %res
+}
+
+declare  @llvm.aarch64.sve.dup.laneq.nxv16i8(, i32)
+declare  @llvm.aarch64.sve.dup.laneq.nxv8i16(, i32)
+declare  @llvm.aarch64.sve.dup.laneq.nxv4i32(, i32)
+declare  @llvm.aarch64.sve.dup.laneq.nxv2i64(, i32)
+declare  @llvm.aarch64.sve.dup.laneq.nxv8f16(, i32)
+declare  @llvm.aarch64.sve.dup.laneq.nxv4f32(, i32)
+declare  @llvm.aarch64.sve.dup.laneq.nxv2f64(, i32)
+declare  @llvm.aarch64.sve.dup.laneq.nxv8bf16(, i32)
-- 
GitLab


From 29afd64615c24af9d89f3e1e8d24e9827a34224c Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan 
Date: Fri, 8 Mar 2024 10:36:22 -0500
Subject: [PATCH 621/929] [libc][c23] make inttypes.h compat with c23 (#84493)

---
 .../llvm-libc-macros/inttypes-macros.h        | 137 +++++++++++++++++-
 libc/spec/stdc.td                             |   4 +-
 2 files changed, 137 insertions(+), 4 deletions(-)

diff --git a/libc/include/llvm-libc-macros/inttypes-macros.h b/libc/include/llvm-libc-macros/inttypes-macros.h
index 8e7d4f558a63..9b554670271e 100644
--- a/libc/include/llvm-libc-macros/inttypes-macros.h
+++ b/libc/include/llvm-libc-macros/inttypes-macros.h
@@ -9,11 +9,9 @@
 #define LLVM_LIBC_MACROS_INTTYPES_MACROS_H
 
 // fprintf/scanf format macros.
-// POSIX.1-2008, Technical Corrigendum 1, XBD/TC1-2008/0050 [211] is applied.
+#define __STDC_VERSION_INTTYPES_H__ 202311L
 
 // clang provides these macros, so we don't need to define them.
-// TODO: ISO C23 will provide binary notations.
-
 #ifndef __clang__
 #if __UINTPTR_MAX__ == __UINT64_MAX__
 #define __PRI64 "l"
@@ -117,6 +115,94 @@
 #define __UINTPTR_FMTX__ __PRIPTR "X"
 #endif
 
+// only recent clang provides these macros, so we need to check if they are
+// available.
+#ifndef __UINT8_FMTb__
+#define __UINT8_FMTb__ "hhb"
+#endif
+#ifndef __UINT16_FMTb__
+#define __UINT16_FMTb__ "hb"
+#endif
+#ifndef __UINT32_FMTb__
+#define __UINT32_FMTb__ "b"
+#endif
+#ifndef __UINT64_FMTb__
+#define __UINT64_FMTb__ __PRI64 "b"
+#endif
+#ifndef __UINT_LEAST8_FMTb__
+#define __UINT_LEAST8_FMTb__ "hhb"
+#endif
+#ifndef __UINT_LEAST16_FMTb__
+#define __UINT_LEAST16_FMTb__ "hb"
+#endif
+#ifndef __UINT_LEAST32_FMTb__
+#define __UINT_LEAST32_FMTb__ "b"
+#endif
+#ifndef __UINT_LEAST64_FMTb__
+#define __UINT_LEAST64_FMTb__ __PRI64 "b"
+#endif
+#ifndef __UINT_FAST8_FMTb__
+#define __UINT_FAST8_FMTb__ "hhb"
+#endif
+#ifndef __UINT_FAST16_FMTb__
+#define __UINT_FAST16_FMTb__ "hb"
+#endif
+#ifndef __UINT_FAST32_FMTb__
+#define __UINT_FAST32_FMTb__ "b"
+#endif
+#ifndef __UINT_FAST64_FMTb__
+#define __UINT_FAST64_FMTb__ __PRI64 "b"
+#endif
+#ifndef __UINTMAX_FMTb__
+#define __UINTMAX_FMTb__ __PRI64 "b"
+#endif
+#ifndef __UINTPTR_FMTb__
+#define __UINTPTR_FMTb__ __PRIPTR "b"
+#endif
+
+#ifndef __UINT8_FMTB__
+#define __UINT8_FMTB__ "hhB"
+#endif
+#ifndef __UINT16_FMTB__
+#define __UINT16_FMTB__ "hB"
+#endif
+#ifndef __UINT32_FMTB__
+#define __UINT32_FMTB__ "B"
+#endif
+#ifndef __UINT64_FMTB__
+#define __UINT64_FMTB__ __PRI64 "B"
+#endif
+#ifndef __UINT_LEAST8_FMTB__
+#define __UINT_LEAST8_FMTB__ "hhB"
+#endif
+#ifndef __UINT_LEAST16_FMTB__
+#define __UINT_LEAST16_FMTB__ "hB"
+#endif
+#ifndef __UINT_LEAST32_FMTB__
+#define __UINT_LEAST32_FMTB__ "B"
+#endif
+#ifndef __UINT_LEAST64_FMTB__
+#define __UINT_LEAST64_FMTB__ __PRI64 "B"
+#endif
+#ifndef __UINT_FAST8_FMTB__
+#define __UINT_FAST8_FMTB__ "hhB"
+#endif
+#ifndef __UINT_FAST16_FMTB__
+#define __UINT_FAST16_FMTB__ "hB"
+#endif
+#ifndef __UINT_FAST32_FMTB__
+#define __UINT_FAST32_FMTB__ "B"
+#endif
+#ifndef __UINT_FAST64_FMTB__
+#define __UINT_FAST64_FMTB__ __PRI64 "B"
+#endif
+#ifndef __UINTMAX_FMTB__
+#define __UINTMAX_FMTB__ __PRI64 "B"
+#endif
+#ifndef __UINTPTR_FMTB__
+#define __UINTPTR_FMTB__ __PRIPTR "B"
+#endif
+
 // The fprintf() macros for signed integers.
 #define PRId8 __INT8_FMTd__
 #define PRId16 __INT16_FMTd__
@@ -209,6 +295,36 @@
 #define PRIXMAX __UINTMAX_FMTX__
 #define PRIXPTR __UINTPTR_FMTX__
 
+#define PRIb8 __UINT8_FMTb__
+#define PRIb16 __UINT16_FMTb__
+#define PRIb32 __UINT32_FMTb__
+#define PRIb64 __UINT64_FMTb__
+#define PRIbLEAST8 __UINT_LEAST8_FMTb__
+#define PRIbLEAST16 __UINT_LEAST16_FMTb__
+#define PRIbLEAST32 __UINT_LEAST32_FMTb__
+#define PRIbLEAST64 __UINT_LEAST64_FMTb__
+#define PRIbFAST8 __UINT_FAST8_FMTb__
+#define PRIbFAST16 __UINT_FAST16_FMTb__
+#define PRIbFAST32 __UINT_FAST32_FMTb__
+#define PRIbFAST64 __UINT_FAST64_FMTb__
+#define PRIbMAX __UINTMAX_FMTb__
+#define PRIbPTR __UINTPTR_FMTb__
+
+#define PRIB8 __UINT8_FMTB__
+#define PRIB16 __UINT16_FMTB__
+#define PRIB32 __UINT32_FMTB__
+#define PRIB64 __UINT64_FMTB__
+#define PRIBLEAST8 __UINT_LEAST8_FMTB__
+#define PRIBLEAST16 __UINT_LEAST16_FMTB__
+#define PRIBLEAST32 __UINT_LEAST32_FMTB__
+#define PRIBLEAST64 __UINT_LEAST64_FMTB__
+#define PRIBFAST8 __UINT_FAST8_FMTB__
+#define PRIBFAST16 __UINT_FAST16_FMTB__
+#define PRIBFAST32 __UINT_FAST32_FMTB__
+#define PRIBFAST64 __UINT_FAST64_FMTB__
+#define PRIBMAX __UINTMAX_FMTB__
+#define PRIBPTR __UINTPTR_FMTB__
+
 // The fscanf() macros for signed integers.
 #define SCNd8 __INT8_FMTd__
 #define SCNd16 __INT16_FMTd__
@@ -286,4 +402,19 @@
 #define SCNxMAX __UINTMAX_FMTx__
 #define SCNxPTR __UINTPTR_FMTx__
 
+#define SCNb8 __UINT8_FMTb__
+#define SCNb16 __UINT16_FMTb__
+#define SCNb32 __UINT32_FMTb__
+#define SCNb64 __UINT64_FMTb__
+#define SCNbLEAST8 __UINT_LEAST8_FMTb__
+#define SCNbLEAST16 __UINT_LEAST16_FMTb__
+#define SCNbLEAST32 __UINT_LEAST32_FMTb__
+#define SCNbLEAST64 __UINT_LEAST64_FMTb__
+#define SCNbFAST8 __UINT_FAST8_FMTb__
+#define SCNbFAST16 __UINT_FAST16_FMTb__
+#define SCNbFAST32 __UINT_FAST32_FMTb__
+#define SCNbFAST64 __UINT_FAST64_FMTb__
+#define SCNbMAX __UINTMAX_FMTb__
+#define SCNbPTR __UINTPTR_FMTb__
+
 #endif // LLVM_LIBC_MACROS_INTTYPES_MACROS_H
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
index e09cce0efd9b..d84eb3570e5c 100644
--- a/libc/spec/stdc.td
+++ b/libc/spec/stdc.td
@@ -948,7 +948,9 @@ def StdC : StandardSpec<"stdc"> {
 
   HeaderSpec IntTypes = HeaderSpec<
       "inttypes.h",
-      [], // Macros
+      [
+        Macro<"__STDC_VERSION_INTTYPES_H__">,
+      ], // Macros
       [
         IMaxDivTType,
       ], // Types
-- 
GitLab


From b46f980454d5ceafc8dab37dbdb1011e333ae6de Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" 
Date: Fri, 8 Mar 2024 10:50:38 -0500
Subject: [PATCH 622/929] [HIP] fix host-used external kernel (#83870)

In -fgpu-rdc mode, when an external kernel is used by a host function
with weak_odr linkage (e.g. explicitly instantiated template function),
the kernel should not be marked as host-used external kernel, since the
host function may be dropped by the linker. Mark the external kernel as
host-used external kernel will force a reference to the external kernel,
which the user may not define in other TU.

Fixes: https://github.com/llvm/llvm-project/issues/83771
---
 clang/lib/Sema/SemaCUDA.cpp                |  5 ++++-
 clang/lib/Sema/SemaExpr.cpp                |  5 ++++-
 clang/test/CodeGenCUDA/host-used-extern.cu | 13 +++++++++++++
 3 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/clang/lib/Sema/SemaCUDA.cpp b/clang/lib/Sema/SemaCUDA.cpp
index 6a66ecf6f94c..4d4f4b6a2d4d 100644
--- a/clang/lib/Sema/SemaCUDA.cpp
+++ b/clang/lib/Sema/SemaCUDA.cpp
@@ -895,7 +895,10 @@ bool Sema::CheckCUDACall(SourceLocation Loc, FunctionDecl *Callee) {
   if (DiagKind == SemaDiagnosticBuilder::K_Nop) {
     // For -fgpu-rdc, keep track of external kernels used by host functions.
     if (LangOpts.CUDAIsDevice && LangOpts.GPURelocatableDeviceCode &&
-        Callee->hasAttr() && !Callee->isDefined())
+        Callee->hasAttr() && !Callee->isDefined() &&
+        (!Caller || (!Caller->getDescribedFunctionTemplate() &&
+                     getASTContext().GetGVALinkageForFunction(Caller) ==
+                         GVA_StrongExternal)))
       getASTContext().CUDAExternalDeviceDeclODRUsedByHost.insert(Callee);
     return true;
   }
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 47bb263f56aa..93f82e68ab64 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -19218,7 +19218,10 @@ MarkVarDeclODRUsed(ValueDecl *V, SourceLocation Loc, Sema &SemaRef,
       // externalize the static device side variable ODR-used by host code.
       if (!Var->hasExternalStorage())
         SemaRef.getASTContext().CUDADeviceVarODRUsedByHost.insert(Var);
-      else if (SemaRef.LangOpts.GPURelocatableDeviceCode)
+      else if (SemaRef.LangOpts.GPURelocatableDeviceCode &&
+               (!FD || (!FD->getDescribedFunctionTemplate() &&
+                        SemaRef.getASTContext().GetGVALinkageForFunction(FD) ==
+                            GVA_StrongExternal)))
         SemaRef.getASTContext().CUDAExternalDeviceDeclODRUsedByHost.insert(Var);
     }
   }
diff --git a/clang/test/CodeGenCUDA/host-used-extern.cu b/clang/test/CodeGenCUDA/host-used-extern.cu
index e8f8e12aad47..1ae644ae981a 100644
--- a/clang/test/CodeGenCUDA/host-used-extern.cu
+++ b/clang/test/CodeGenCUDA/host-used-extern.cu
@@ -24,6 +24,7 @@
 
 // NEG-NOT: @__clang_gpu_used_external = {{.*}} @_Z7kernel2v
 // NEG-NOT: @__clang_gpu_used_external = {{.*}} @_Z7kernel3v
+// NEG-NOT: @__clang_gpu_used_external = {{.*}} @_Z7kernel5v
 // NEG-NOT: @__clang_gpu_used_external = {{.*}} @var2
 // NEG-NOT: @__clang_gpu_used_external = {{.*}} @var3
 // NEG-NOT: @__clang_gpu_used_external = {{.*}} @ext_shvar
@@ -44,6 +45,10 @@ __global__ void kernel3();
 // kernel4 is marked as used even though it is not called.
 __global__ void kernel4();
 
+// kernel5 is not marked as used since it is called by host function
+// with weak_odr linkage, which may be dropped by linker.
+__global__ void kernel5();
+
 extern __device__ int var1;
 
 __device__ int var2;
@@ -67,3 +72,11 @@ __global__ void test_lambda_using_extern_shared() {
   };
   lambda();
 }
+
+template
+void template_caller() {
+  kernel5<<<1, 1>>>();
+  var1 = 1;
+}
+
+template void template_caller();
-- 
GitLab


From 5620542e4d19f794a0d6bd1bd1861bcedd7a788a Mon Sep 17 00:00:00 2001
From: Antonio Frighetto 
Date: Fri, 8 Mar 2024 16:32:03 +0100
Subject: [PATCH 623/929] [IR] Add `getNoWrapKind` method for
 `OverflowingBinaryOperator` (NFC)

---
 llvm/include/llvm/IR/Operator.h     | 12 ++++++++++++
 llvm/lib/Analysis/LazyValueInfo.cpp |  7 +------
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/IR/Operator.h b/llvm/include/llvm/IR/Operator.h
index 7168128648d8..e52649833901 100644
--- a/llvm/include/llvm/IR/Operator.h
+++ b/llvm/include/llvm/IR/Operator.h
@@ -109,6 +109,18 @@ public:
     return (SubclassOptionalData & NoSignedWrap) != 0;
   }
 
+  /// Returns the no-wrap kind of the operation.
+  unsigned getNoWrapKind() const {
+    unsigned NoWrapKind = 0;
+    if (hasNoUnsignedWrap())
+      NoWrapKind |= NoUnsignedWrap;
+
+    if (hasNoSignedWrap())
+      NoWrapKind |= NoSignedWrap;
+
+    return NoWrapKind;
+  }
+
   static bool classof(const Instruction *I) {
     return I->getOpcode() == Instruction::Add ||
            I->getOpcode() == Instruction::Sub ||
diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index b948eb6ebd12..9ae31d165235 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -997,12 +997,7 @@ LazyValueInfoImpl::solveBlockValueBinaryOp(BinaryOperator *BO, BasicBlock *BB) {
   assert(BO->getOperand(0)->getType()->isSized() &&
          "all operands to binary operators are sized");
   if (auto *OBO = dyn_cast(BO)) {
-    unsigned NoWrapKind = 0;
-    if (OBO->hasNoUnsignedWrap())
-      NoWrapKind |= OverflowingBinaryOperator::NoUnsignedWrap;
-    if (OBO->hasNoSignedWrap())
-      NoWrapKind |= OverflowingBinaryOperator::NoSignedWrap;
-
+    unsigned NoWrapKind = OBO->getNoWrapKind();
     return solveBlockValueBinaryOpImpl(
         BO, BB,
         [BO, NoWrapKind](const ConstantRange &CR1, const ConstantRange &CR2) {
-- 
GitLab


From 9d5fdad7e5002dbe087bbf0a78ffe9423b6aab56 Mon Sep 17 00:00:00 2001
From: Louis Dionne 
Date: Fri, 8 Mar 2024 11:03:09 -0500
Subject: [PATCH 624/929] [libc++] Disable module_std and module_std_compat
 tests

Those have been crippling the CI for over a week now. This is the only
solution I see until we gain a better understanding of why they fail.
Otherwise all other patches are blocked on these spurious failures in
the stage1 of the CI.
---
 libcxx/test/libcxx/module_std.gen.py        | 3 +++
 libcxx/test/libcxx/module_std_compat.gen.py | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/libcxx/test/libcxx/module_std.gen.py b/libcxx/test/libcxx/module_std.gen.py
index fc23985caf30..932af742640c 100644
--- a/libcxx/test/libcxx/module_std.gen.py
+++ b/libcxx/test/libcxx/module_std.gen.py
@@ -18,6 +18,9 @@
 
 # RUN: %{python} %s %{libcxx-dir}/utils
 
+# TODO: Re-enable this test once we understand why it keeps timing out.
+# UNSUPPORTED: clang
+
 import sys
 
 sys.path.append(sys.argv[1])
diff --git a/libcxx/test/libcxx/module_std_compat.gen.py b/libcxx/test/libcxx/module_std_compat.gen.py
index 000aa2998612..d29f490ab135 100644
--- a/libcxx/test/libcxx/module_std_compat.gen.py
+++ b/libcxx/test/libcxx/module_std_compat.gen.py
@@ -18,6 +18,9 @@
 
 # RUN: %{python} %s %{libcxx-dir}/utils
 
+# TODO: Re-enable this test once we understand why it keeps timing out.
+# UNSUPPORTED: clang
+
 import sys
 
 sys.path.append(sys.argv[1])
-- 
GitLab


From 0456a32a2a622a73e01b01c65c21973ac5a689fc Mon Sep 17 00:00:00 2001
From: Jay Foad 
Date: Fri, 8 Mar 2024 16:08:09 +0000
Subject: [PATCH 625/929] [AMDGPU] Simplify renamed BUF instruction
 definitions. NFC. (#84503)

Use optional arguments instead of separate (multi)classes for renamed
instructions.
---
 llvm/lib/Target/AMDGPU/BUFInstructions.td | 239 +++++++++++-----------
 1 file changed, 114 insertions(+), 125 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index 9c6934865bfa..dedf381afca8 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -2466,51 +2466,50 @@ class Mnem_gfx11  :
 class Mnem_gfx12  :
   MnemonicAlias, Requires<[isGFX12Plus]>;
 
-multiclass MUBUF_Real_AllAddr_gfx11_Renamed_Impl2 op, string real_name> {
+multiclass MUBUF_Real_AllAddr_gfx11_Impl2 op, string real_name> {
   defm _BOTHEN : MUBUF_Real_gfx11;
   defm _IDXEN  : MUBUF_Real_gfx11;
   defm _OFFEN  : MUBUF_Real_gfx11;
   defm _OFFSET : MUBUF_Real_gfx11;
 }
 
-multiclass MUBUF_Real_AllAddr_gfx12_Renamed_Impl2 op, string real_name> {
+multiclass MUBUF_Real_AllAddr_gfx12_Impl2 op, string real_name> {
   defm _VBUFFER_BOTHEN : VBUFFER_MUBUF_Real_gfx12;
   defm _VBUFFER_IDXEN  : VBUFFER_MUBUF_Real_gfx12;
   defm _VBUFFER_OFFEN  : VBUFFER_MUBUF_Real_gfx12;
   defm _VBUFFER_OFFSET : VBUFFER_MUBUF_Real_gfx12;
 }
 
-multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl2 op, string real_name> :
-  MUBUF_Real_AllAddr_gfx11_Renamed_Impl2,
-  MUBUF_Real_AllAddr_gfx12_Renamed_Impl2;
+multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Impl2 op, string real_name> :
+  MUBUF_Real_AllAddr_gfx11_Impl2,
+  MUBUF_Real_AllAddr_gfx12_Impl2;
 
-multiclass MUBUF_Real_AllAddr_gfx11_Renamed_Impl op, string real_name,
+multiclass MUBUF_Real_AllAddr_gfx11_Impl op, string real_name,
                                                  bit hasTFE = 1> {
-  defm NAME : MUBUF_Real_AllAddr_gfx11_Renamed_Impl2;
+  defm NAME : MUBUF_Real_AllAddr_gfx11_Impl2;
   if hasTFE then
-    defm _TFE : MUBUF_Real_AllAddr_gfx11_Renamed_Impl2;
+    defm _TFE : MUBUF_Real_AllAddr_gfx11_Impl2;
 }
 
-multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl op, string real_name,
+multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Impl op, string real_name,
                                                  bit hasTFE = 1> {
-  defm NAME : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl2;
+  defm NAME : MUBUF_Real_AllAddr_gfx11_gfx12_Impl2;
   if hasTFE then
-    defm _TFE : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl2;
+    defm _TFE : MUBUF_Real_AllAddr_gfx11_gfx12_Impl2;
 }
 
 // Non-renamed, non-atomic gfx11/gfx12 mubuf instructions.
 multiclass MUBUF_Real_AllAddr_gfx11 op, bit hasTFE = 1> :
-  MUBUF_Real_AllAddr_gfx11_Renamed_Impl.Mnemonic, hasTFE>;
+  MUBUF_Real_AllAddr_gfx11_Impl.Mnemonic, hasTFE>;
 
-multiclass MUBUF_Real_AllAddr_gfx11_gfx12 op, bit hasTFE = 1> :
-  MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl.Mnemonic, hasTFE>;
-
-multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Renamed op, string real_name> :
-  MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl {
-  def : Mnem_gfx11_gfx12.Mnemonic, real_name>;
+multiclass MUBUF_Real_AllAddr_gfx11_gfx12 op, string real_name = !tolower(NAME)> :
+  MUBUF_Real_AllAddr_gfx11_gfx12_Impl {
+  defvar ps = get_BUF_ps;
+  if !ne(ps.Mnemonic, real_name) then
+    def : Mnem_gfx11_gfx12;
 }
 
-multiclass MUBUF_Real_Atomic_gfx11_Renamed_impl op, bit is_return,
+multiclass MUBUF_Real_Atomic_gfx11_impl op, bit is_return,
                                                 string real_name> {
   defvar Rtn = !if(is_return, "_RTN", "");
   defm _BOTHEN#Rtn : MUBUF_Real_gfx11;
@@ -2519,7 +2518,7 @@ multiclass MUBUF_Real_Atomic_gfx11_Renamed_impl op, bit is_return,
   defm _OFFSET#Rtn : MUBUF_Real_gfx11;
 }
 
-multiclass MUBUF_Real_Atomic_gfx12_Renamed_impl op, bit is_return,
+multiclass MUBUF_Real_Atomic_gfx12_impl op, bit is_return,
                                                 string real_name> {
   defvar Rtn = !if(is_return, "_RTN", "");
   defm _VBUFFER_BOTHEN#Rtn : VBUFFER_MUBUF_Real_gfx12;
@@ -2528,124 +2527,117 @@ multiclass MUBUF_Real_Atomic_gfx12_Renamed_impl op, bit is_return,
   defm _VBUFFER_OFFSET#Rtn : VBUFFER_MUBUF_Real_gfx12;
 }
 
-multiclass MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl op, bit is_return,
+multiclass MUBUF_Real_Atomic_gfx11_gfx12_impl op, bit is_return,
                                                 string real_name> :
-  MUBUF_Real_Atomic_gfx11_Renamed_impl,
-  MUBUF_Real_Atomic_gfx12_Renamed_impl;
-
-// Non-renamed gfx11/gfx12 mubuf atomic.
-multiclass MUBUF_Real_Atomic_gfx11_gfx12 op> :
-  MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl.Mnemonic>,
-  MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl.Mnemonic>;
+  MUBUF_Real_Atomic_gfx11_impl,
+  MUBUF_Real_Atomic_gfx12_impl;
 
 multiclass MUBUF_Real_Atomic_gfx12 op> :
-  MUBUF_Real_Atomic_gfx12_Renamed_impl.Mnemonic>,
-  MUBUF_Real_Atomic_gfx12_Renamed_impl.Mnemonic>;
-
-multiclass MUBUF_Real_Atomic_gfx11_Renamed op, string real_name> :
-  MUBUF_Real_Atomic_gfx11_Renamed_impl,
-  MUBUF_Real_Atomic_gfx11_Renamed_impl {
-  def : Mnem_gfx11_gfx12.Mnemonic, real_name>;
-}
+  MUBUF_Real_Atomic_gfx12_impl.Mnemonic>,
+  MUBUF_Real_Atomic_gfx12_impl.Mnemonic>;
 
-multiclass MUBUF_Real_Atomic_gfx11_gfx12_Renamed op, string real_name> :
-  MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl,
-  MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl  {
+multiclass MUBUF_Real_Atomic_gfx11 op, string real_name> :
+  MUBUF_Real_Atomic_gfx11_impl,
+  MUBUF_Real_Atomic_gfx11_impl {
   def : Mnem_gfx11_gfx12.Mnemonic, real_name>;
 }
 
-multiclass MUBUF_Real_Atomic_gfx11_gfx12_Renamed_gfx12_Renamed op, string gfx12_name, string gfx11_name> :
-  MUBUF_Real_Atomic_gfx11_Renamed_impl,
-  MUBUF_Real_Atomic_gfx11_Renamed_impl,
-  MUBUF_Real_Atomic_gfx12_Renamed_impl,
-  MUBUF_Real_Atomic_gfx12_Renamed_impl {
-  def : Mnem_gfx11.Mnemonic, gfx11_name>;
-  def : Mnem_gfx12.Mnemonic, gfx12_name>;
-  def : Mnem_gfx12;
+multiclass MUBUF_Real_Atomic_gfx11_gfx12 op, string gfx12_name = !tolower(NAME), string gfx11_name = gfx12_name> :
+  MUBUF_Real_Atomic_gfx11_impl,
+  MUBUF_Real_Atomic_gfx11_impl,
+  MUBUF_Real_Atomic_gfx12_impl,
+  MUBUF_Real_Atomic_gfx12_impl {
+  defvar ps = get_BUF_ps;
+  if !ne(ps.Mnemonic, gfx11_name) then
+    def : Mnem_gfx11;
+  if !ne(ps.Mnemonic, gfx12_name) then
+    def : Mnem_gfx12;
+  if !ne(gfx11_name, gfx12_name) then
+    def : Mnem_gfx12;
 }
 
 defm BUFFER_GL0_INV               : MUBUF_Real_gfx11<0x02B>;
 defm BUFFER_GL1_INV               : MUBUF_Real_gfx11<0x02C>;
 
-defm BUFFER_LOAD_DWORD            : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x014, "buffer_load_b32">;
-defm BUFFER_LOAD_DWORDX2          : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x015, "buffer_load_b64">;
-defm BUFFER_LOAD_DWORDX3          : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x016, "buffer_load_b96">;
-defm BUFFER_LOAD_DWORDX4          : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x017, "buffer_load_b128">;
-defm BUFFER_LOAD_SHORT_D16        : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x020, "buffer_load_d16_b16">;
-defm BUFFER_LOAD_FORMAT_D16_X     : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x008, "buffer_load_d16_format_x">;
-defm BUFFER_LOAD_FORMAT_D16_XY    : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x009, "buffer_load_d16_format_xy">;
-defm BUFFER_LOAD_FORMAT_D16_XYZ   : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00a, "buffer_load_d16_format_xyz">;
-defm BUFFER_LOAD_FORMAT_D16_XYZW  : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00b, "buffer_load_d16_format_xyzw">;
-defm BUFFER_LOAD_SHORT_D16_HI     : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x023, "buffer_load_d16_hi_b16">;
-defm BUFFER_LOAD_FORMAT_D16_HI_X  : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x026, "buffer_load_d16_hi_format_x">;
-defm BUFFER_LOAD_SBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x022, "buffer_load_d16_hi_i8">;
-defm BUFFER_LOAD_UBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x021, "buffer_load_d16_hi_u8">;
-defm BUFFER_LOAD_SBYTE_D16        : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01f, "buffer_load_d16_i8">;
-defm BUFFER_LOAD_UBYTE_D16        : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01e, "buffer_load_d16_u8">;
+defm BUFFER_LOAD_DWORD            : MUBUF_Real_AllAddr_gfx11_gfx12<0x014, "buffer_load_b32">;
+defm BUFFER_LOAD_DWORDX2          : MUBUF_Real_AllAddr_gfx11_gfx12<0x015, "buffer_load_b64">;
+defm BUFFER_LOAD_DWORDX3          : MUBUF_Real_AllAddr_gfx11_gfx12<0x016, "buffer_load_b96">;
+defm BUFFER_LOAD_DWORDX4          : MUBUF_Real_AllAddr_gfx11_gfx12<0x017, "buffer_load_b128">;
+defm BUFFER_LOAD_SHORT_D16        : MUBUF_Real_AllAddr_gfx11_gfx12<0x020, "buffer_load_d16_b16">;
+defm BUFFER_LOAD_FORMAT_D16_X     : MUBUF_Real_AllAddr_gfx11_gfx12<0x008, "buffer_load_d16_format_x">;
+defm BUFFER_LOAD_FORMAT_D16_XY    : MUBUF_Real_AllAddr_gfx11_gfx12<0x009, "buffer_load_d16_format_xy">;
+defm BUFFER_LOAD_FORMAT_D16_XYZ   : MUBUF_Real_AllAddr_gfx11_gfx12<0x00a, "buffer_load_d16_format_xyz">;
+defm BUFFER_LOAD_FORMAT_D16_XYZW  : MUBUF_Real_AllAddr_gfx11_gfx12<0x00b, "buffer_load_d16_format_xyzw">;
+defm BUFFER_LOAD_SHORT_D16_HI     : MUBUF_Real_AllAddr_gfx11_gfx12<0x023, "buffer_load_d16_hi_b16">;
+defm BUFFER_LOAD_FORMAT_D16_HI_X  : MUBUF_Real_AllAddr_gfx11_gfx12<0x026, "buffer_load_d16_hi_format_x">;
+defm BUFFER_LOAD_SBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx11_gfx12<0x022, "buffer_load_d16_hi_i8">;
+defm BUFFER_LOAD_UBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx11_gfx12<0x021, "buffer_load_d16_hi_u8">;
+defm BUFFER_LOAD_SBYTE_D16        : MUBUF_Real_AllAddr_gfx11_gfx12<0x01f, "buffer_load_d16_i8">;
+defm BUFFER_LOAD_UBYTE_D16        : MUBUF_Real_AllAddr_gfx11_gfx12<0x01e, "buffer_load_d16_u8">;
 defm BUFFER_LOAD_FORMAT_X         : MUBUF_Real_AllAddr_gfx11_gfx12<0x000>;
 defm BUFFER_LOAD_FORMAT_XY        : MUBUF_Real_AllAddr_gfx11_gfx12<0x001>;
 defm BUFFER_LOAD_FORMAT_XYZ       : MUBUF_Real_AllAddr_gfx11_gfx12<0x002>;
 defm BUFFER_LOAD_FORMAT_XYZW      : MUBUF_Real_AllAddr_gfx11_gfx12<0x003>;
-defm BUFFER_LOAD_SBYTE            : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x011, "buffer_load_i8">;
-defm BUFFER_LOAD_SSHORT           : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x013, "buffer_load_i16">;
-defm BUFFER_LOAD_UBYTE            : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x010, "buffer_load_u8">;
-defm BUFFER_LOAD_USHORT           : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x012, "buffer_load_u16">;
+defm BUFFER_LOAD_SBYTE            : MUBUF_Real_AllAddr_gfx11_gfx12<0x011, "buffer_load_i8">;
+defm BUFFER_LOAD_SSHORT           : MUBUF_Real_AllAddr_gfx11_gfx12<0x013, "buffer_load_i16">;
+defm BUFFER_LOAD_UBYTE            : MUBUF_Real_AllAddr_gfx11_gfx12<0x010, "buffer_load_u8">;
+defm BUFFER_LOAD_USHORT           : MUBUF_Real_AllAddr_gfx11_gfx12<0x012, "buffer_load_u16">;
 defm BUFFER_LOAD_LDS_B32          : MUBUF_Real_AllAddr_gfx11<0x031, 0>;
 defm BUFFER_LOAD_LDS_FORMAT_X     : MUBUF_Real_AllAddr_gfx11<0x032, 0>;
 defm BUFFER_LOAD_LDS_I8           : MUBUF_Real_AllAddr_gfx11<0x02e, 0>;
 defm BUFFER_LOAD_LDS_I16          : MUBUF_Real_AllAddr_gfx11<0x030, 0>;
 defm BUFFER_LOAD_LDS_U8           : MUBUF_Real_AllAddr_gfx11<0x02d, 0>;
 defm BUFFER_LOAD_LDS_U16          : MUBUF_Real_AllAddr_gfx11<0x02f, 0>;
-defm BUFFER_STORE_BYTE            : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x018, "buffer_store_b8">;
-defm BUFFER_STORE_SHORT           : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x019, "buffer_store_b16">;
-defm BUFFER_STORE_DWORD           : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01A, "buffer_store_b32">;
-defm BUFFER_STORE_DWORDX2         : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01B, "buffer_store_b64">;
-defm BUFFER_STORE_DWORDX3         : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01C, "buffer_store_b96">;
-defm BUFFER_STORE_DWORDX4         : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01D, "buffer_store_b128">;
-defm BUFFER_STORE_FORMAT_D16_X    : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00C, "buffer_store_d16_format_x">;
-defm BUFFER_STORE_FORMAT_D16_XY   : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00D, "buffer_store_d16_format_xy">;
-defm BUFFER_STORE_FORMAT_D16_XYZ  : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00E, "buffer_store_d16_format_xyz">;
-defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00F, "buffer_store_d16_format_xyzw">;
-defm BUFFER_STORE_BYTE_D16_HI     : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x024, "buffer_store_d16_hi_b8">;
-defm BUFFER_STORE_SHORT_D16_HI    : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x025, "buffer_store_d16_hi_b16">;
-defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x027, "buffer_store_d16_hi_format_x">;
+defm BUFFER_STORE_BYTE            : MUBUF_Real_AllAddr_gfx11_gfx12<0x018, "buffer_store_b8">;
+defm BUFFER_STORE_SHORT           : MUBUF_Real_AllAddr_gfx11_gfx12<0x019, "buffer_store_b16">;
+defm BUFFER_STORE_DWORD           : MUBUF_Real_AllAddr_gfx11_gfx12<0x01A, "buffer_store_b32">;
+defm BUFFER_STORE_DWORDX2         : MUBUF_Real_AllAddr_gfx11_gfx12<0x01B, "buffer_store_b64">;
+defm BUFFER_STORE_DWORDX3         : MUBUF_Real_AllAddr_gfx11_gfx12<0x01C, "buffer_store_b96">;
+defm BUFFER_STORE_DWORDX4         : MUBUF_Real_AllAddr_gfx11_gfx12<0x01D, "buffer_store_b128">;
+defm BUFFER_STORE_FORMAT_D16_X    : MUBUF_Real_AllAddr_gfx11_gfx12<0x00C, "buffer_store_d16_format_x">;
+defm BUFFER_STORE_FORMAT_D16_XY   : MUBUF_Real_AllAddr_gfx11_gfx12<0x00D, "buffer_store_d16_format_xy">;
+defm BUFFER_STORE_FORMAT_D16_XYZ  : MUBUF_Real_AllAddr_gfx11_gfx12<0x00E, "buffer_store_d16_format_xyz">;
+defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12<0x00F, "buffer_store_d16_format_xyzw">;
+defm BUFFER_STORE_BYTE_D16_HI     : MUBUF_Real_AllAddr_gfx11_gfx12<0x024, "buffer_store_d16_hi_b8">;
+defm BUFFER_STORE_SHORT_D16_HI    : MUBUF_Real_AllAddr_gfx11_gfx12<0x025, "buffer_store_d16_hi_b16">;
+defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x027, "buffer_store_d16_hi_format_x">;
 defm BUFFER_STORE_FORMAT_X        : MUBUF_Real_AllAddr_gfx11_gfx12<0x004>;
 defm BUFFER_STORE_FORMAT_XY       : MUBUF_Real_AllAddr_gfx11_gfx12<0x005>;
 defm BUFFER_STORE_FORMAT_XYZ      : MUBUF_Real_AllAddr_gfx11_gfx12<0x006>;
 defm BUFFER_STORE_FORMAT_XYZW     : MUBUF_Real_AllAddr_gfx11_gfx12<0x007>;
 defm BUFFER_ATOMIC_ADD_F32        : MUBUF_Real_Atomic_gfx11_gfx12<0x056>;
-defm BUFFER_ATOMIC_ADD            : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x035, "buffer_atomic_add_u32">;
-defm BUFFER_ATOMIC_ADD_X2         : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x043, "buffer_atomic_add_u64">;
-defm BUFFER_ATOMIC_AND            : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03C, "buffer_atomic_and_b32">;
-defm BUFFER_ATOMIC_AND_X2         : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x049, "buffer_atomic_and_b64">;
-defm BUFFER_ATOMIC_CMPSWAP        : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x034, "buffer_atomic_cmpswap_b32">;
-defm BUFFER_ATOMIC_CMPSWAP_X2     : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x042, "buffer_atomic_cmpswap_b64">;
-defm BUFFER_ATOMIC_FCMPSWAP       : MUBUF_Real_Atomic_gfx11_Renamed<0x050, "buffer_atomic_cmpswap_f32">;
+defm BUFFER_ATOMIC_ADD            : MUBUF_Real_Atomic_gfx11_gfx12<0x035, "buffer_atomic_add_u32">;
+defm BUFFER_ATOMIC_ADD_X2         : MUBUF_Real_Atomic_gfx11_gfx12<0x043, "buffer_atomic_add_u64">;
+defm BUFFER_ATOMIC_AND            : MUBUF_Real_Atomic_gfx11_gfx12<0x03C, "buffer_atomic_and_b32">;
+defm BUFFER_ATOMIC_AND_X2         : MUBUF_Real_Atomic_gfx11_gfx12<0x049, "buffer_atomic_and_b64">;
+defm BUFFER_ATOMIC_CMPSWAP        : MUBUF_Real_Atomic_gfx11_gfx12<0x034, "buffer_atomic_cmpswap_b32">;
+defm BUFFER_ATOMIC_CMPSWAP_X2     : MUBUF_Real_Atomic_gfx11_gfx12<0x042, "buffer_atomic_cmpswap_b64">;
+defm BUFFER_ATOMIC_FCMPSWAP       : MUBUF_Real_Atomic_gfx11<0x050, "buffer_atomic_cmpswap_f32">;
 defm BUFFER_ATOMIC_COND_SUB_U32   : MUBUF_Real_Atomic_gfx12<0x050>;
-defm BUFFER_ATOMIC_CSUB           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed_gfx12_Renamed<0x037, "buffer_atomic_sub_clamp_u32", "buffer_atomic_csub_u32">;
+defm BUFFER_ATOMIC_CSUB           : MUBUF_Real_Atomic_gfx11_gfx12<0x037, "buffer_atomic_sub_clamp_u32", "buffer_atomic_csub_u32">;
 def : Mnem_gfx11_gfx12<"buffer_atomic_csub", "buffer_atomic_csub_u32">;
-defm BUFFER_ATOMIC_DEC            : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x040, "buffer_atomic_dec_u32">;
-defm BUFFER_ATOMIC_DEC_X2         : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x04D, "buffer_atomic_dec_u64">;
-defm BUFFER_ATOMIC_INC            : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03F, "buffer_atomic_inc_u32">;
-defm BUFFER_ATOMIC_INC_X2         : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x04C, "buffer_atomic_inc_u64">;
-defm BUFFER_ATOMIC_FMAX           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed_gfx12_Renamed<0x052, "buffer_atomic_max_num_f32", "buffer_atomic_max_f32">;
-defm BUFFER_ATOMIC_SMAX           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03A, "buffer_atomic_max_i32">;
-defm BUFFER_ATOMIC_SMAX_X2        : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x047, "buffer_atomic_max_i64">;
-defm BUFFER_ATOMIC_UMAX           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03B, "buffer_atomic_max_u32">;
-defm BUFFER_ATOMIC_UMAX_X2        : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x048, "buffer_atomic_max_u64">;
-defm BUFFER_ATOMIC_FMIN           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed_gfx12_Renamed<0x051, "buffer_atomic_min_num_f32", "buffer_atomic_min_f32">;
-defm BUFFER_ATOMIC_SMIN           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x038, "buffer_atomic_min_i32">;
-defm BUFFER_ATOMIC_SMIN_X2        : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x045, "buffer_atomic_min_i64">;
-defm BUFFER_ATOMIC_UMIN           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x039, "buffer_atomic_min_u32">;
-defm BUFFER_ATOMIC_UMIN_X2        : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x046, "buffer_atomic_min_u64">;
-defm BUFFER_ATOMIC_OR             : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03D, "buffer_atomic_or_b32">;
-defm BUFFER_ATOMIC_OR_X2          : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x04A, "buffer_atomic_or_b64">;
-defm BUFFER_ATOMIC_SUB            : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x036, "buffer_atomic_sub_u32">;
-defm BUFFER_ATOMIC_SUB_X2         : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x044, "buffer_atomic_sub_u64">;
-defm BUFFER_ATOMIC_SWAP           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x033, "buffer_atomic_swap_b32">;
-defm BUFFER_ATOMIC_SWAP_X2        : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x041, "buffer_atomic_swap_b64">;
-defm BUFFER_ATOMIC_XOR            : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03E, "buffer_atomic_xor_b32">;
-defm BUFFER_ATOMIC_XOR_X2         : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x04B, "buffer_atomic_xor_b64">;
+defm BUFFER_ATOMIC_DEC            : MUBUF_Real_Atomic_gfx11_gfx12<0x040, "buffer_atomic_dec_u32">;
+defm BUFFER_ATOMIC_DEC_X2         : MUBUF_Real_Atomic_gfx11_gfx12<0x04D, "buffer_atomic_dec_u64">;
+defm BUFFER_ATOMIC_INC            : MUBUF_Real_Atomic_gfx11_gfx12<0x03F, "buffer_atomic_inc_u32">;
+defm BUFFER_ATOMIC_INC_X2         : MUBUF_Real_Atomic_gfx11_gfx12<0x04C, "buffer_atomic_inc_u64">;
+defm BUFFER_ATOMIC_FMAX           : MUBUF_Real_Atomic_gfx11_gfx12<0x052, "buffer_atomic_max_num_f32", "buffer_atomic_max_f32">;
+defm BUFFER_ATOMIC_SMAX           : MUBUF_Real_Atomic_gfx11_gfx12<0x03A, "buffer_atomic_max_i32">;
+defm BUFFER_ATOMIC_SMAX_X2        : MUBUF_Real_Atomic_gfx11_gfx12<0x047, "buffer_atomic_max_i64">;
+defm BUFFER_ATOMIC_UMAX           : MUBUF_Real_Atomic_gfx11_gfx12<0x03B, "buffer_atomic_max_u32">;
+defm BUFFER_ATOMIC_UMAX_X2        : MUBUF_Real_Atomic_gfx11_gfx12<0x048, "buffer_atomic_max_u64">;
+defm BUFFER_ATOMIC_FMIN           : MUBUF_Real_Atomic_gfx11_gfx12<0x051, "buffer_atomic_min_num_f32", "buffer_atomic_min_f32">;
+defm BUFFER_ATOMIC_SMIN           : MUBUF_Real_Atomic_gfx11_gfx12<0x038, "buffer_atomic_min_i32">;
+defm BUFFER_ATOMIC_SMIN_X2        : MUBUF_Real_Atomic_gfx11_gfx12<0x045, "buffer_atomic_min_i64">;
+defm BUFFER_ATOMIC_UMIN           : MUBUF_Real_Atomic_gfx11_gfx12<0x039, "buffer_atomic_min_u32">;
+defm BUFFER_ATOMIC_UMIN_X2        : MUBUF_Real_Atomic_gfx11_gfx12<0x046, "buffer_atomic_min_u64">;
+defm BUFFER_ATOMIC_OR             : MUBUF_Real_Atomic_gfx11_gfx12<0x03D, "buffer_atomic_or_b32">;
+defm BUFFER_ATOMIC_OR_X2          : MUBUF_Real_Atomic_gfx11_gfx12<0x04A, "buffer_atomic_or_b64">;
+defm BUFFER_ATOMIC_SUB            : MUBUF_Real_Atomic_gfx11_gfx12<0x036, "buffer_atomic_sub_u32">;
+defm BUFFER_ATOMIC_SUB_X2         : MUBUF_Real_Atomic_gfx11_gfx12<0x044, "buffer_atomic_sub_u64">;
+defm BUFFER_ATOMIC_SWAP           : MUBUF_Real_Atomic_gfx11_gfx12<0x033, "buffer_atomic_swap_b32">;
+defm BUFFER_ATOMIC_SWAP_X2        : MUBUF_Real_Atomic_gfx11_gfx12<0x041, "buffer_atomic_swap_b64">;
+defm BUFFER_ATOMIC_XOR            : MUBUF_Real_Atomic_gfx11_gfx12<0x03E, "buffer_atomic_xor_b32">;
+defm BUFFER_ATOMIC_XOR_X2         : MUBUF_Real_Atomic_gfx11_gfx12<0x04B, "buffer_atomic_xor_b64">;
 defm BUFFER_ATOMIC_PK_ADD_F16     : MUBUF_Real_Atomic_gfx12<0x059>;
 defm BUFFER_ATOMIC_PK_ADD_BF16    : MUBUF_Real_Atomic_gfx12<0x05a>;
 
@@ -2883,7 +2875,7 @@ class Base_MTBUF_Real_gfx6_gfx7_gfx10 op, MTBUF_Pseudo ps, int ef> :
 // MTBUF - GFX11.
 //===----------------------------------------------------------------------===//
 
-multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl op, string real_name> {
+multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Impl op, string real_name> {
   defm _BOTHEN : MTBUF_Real_gfx11;
   defm _IDXEN  : MTBUF_Real_gfx11;
   defm _OFFEN  : MTBUF_Real_gfx11;
@@ -2895,28 +2887,25 @@ multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl op, string real_n
   defm _VBUFFER_OFFSET : VBUFFER_MTBUF_Real_gfx12;
 }
 
-multiclass MTBUF_Real_AllAddr_gfx11_gfx12 op>
- : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl.Mnemonic>;
-
-
-multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Renamed op, string real_name>
-  : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl {
+multiclass MTBUF_Real_AllAddr_gfx11_gfx12 op, string real_name = !tolower(NAME)>
+  : MTBUF_Real_AllAddr_gfx11_gfx12_Impl {
   defvar ps = get_BUF_ps;
-  def : Mnem_gfx11_gfx12;
+  if !ne(ps.Mnemonic, real_name) then
+    def : Mnem_gfx11_gfx12;
 }
 
-defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x008, "tbuffer_load_d16_format_x">;
-defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x009, "tbuffer_load_d16_format_xy">;
-defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00a, "tbuffer_load_d16_format_xyz">;
-defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00b, "tbuffer_load_d16_format_xyzw">;
+defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Real_AllAddr_gfx11_gfx12<0x008, "tbuffer_load_d16_format_x">;
+defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Real_AllAddr_gfx11_gfx12<0x009, "tbuffer_load_d16_format_xy">;
+defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Real_AllAddr_gfx11_gfx12<0x00a, "tbuffer_load_d16_format_xyz">;
+defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Real_AllAddr_gfx11_gfx12<0x00b, "tbuffer_load_d16_format_xyzw">;
 defm TBUFFER_LOAD_FORMAT_X         : MTBUF_Real_AllAddr_gfx11_gfx12<0x000>;
 defm TBUFFER_LOAD_FORMAT_XY        : MTBUF_Real_AllAddr_gfx11_gfx12<0x001>;
 defm TBUFFER_LOAD_FORMAT_XYZ       : MTBUF_Real_AllAddr_gfx11_gfx12<0x002>;
 defm TBUFFER_LOAD_FORMAT_XYZW      : MTBUF_Real_AllAddr_gfx11_gfx12<0x003>;
-defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00c, "tbuffer_store_d16_format_x">;
-defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00d, "tbuffer_store_d16_format_xy">;
-defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00e, "tbuffer_store_d16_format_xyz">;
-defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00f, "tbuffer_store_d16_format_xyzw">;
+defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Real_AllAddr_gfx11_gfx12<0x00c, "tbuffer_store_d16_format_x">;
+defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Real_AllAddr_gfx11_gfx12<0x00d, "tbuffer_store_d16_format_xy">;
+defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Real_AllAddr_gfx11_gfx12<0x00e, "tbuffer_store_d16_format_xyz">;
+defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12<0x00f, "tbuffer_store_d16_format_xyzw">;
 defm TBUFFER_STORE_FORMAT_X        : MTBUF_Real_AllAddr_gfx11_gfx12<0x004>;
 defm TBUFFER_STORE_FORMAT_XY       : MTBUF_Real_AllAddr_gfx11_gfx12<0x005>;
 defm TBUFFER_STORE_FORMAT_XYZ      : MTBUF_Real_AllAddr_gfx11_gfx12<0x006>;
-- 
GitLab


From 35d3b33ba5c9b90443ac985f2521b78f84b611fe Mon Sep 17 00:00:00 2001
From: Andreas Fertig 
Date: Fri, 8 Mar 2024 17:10:20 +0100
Subject: [PATCH 626/929] [C++20][Coroutines] Lambda-coroutine with operator
 new in promise_type (#84193)

Fix #84064

According to http://eel.is/c++draft/dcl.fct.def.coroutine#9 the first
parameter for overload resolution of `operator new` is `size_t` followed
by the arguments of the coroutine function.
http://eel.is/c++draft/dcl.fct.def.coroutine#4 states that the first
argument is the lvalue of `*this` if the coroutine is a member function.

Before this patch, Clang handled class types correctly but ignored
lambdas. This patch adds support for lambda coroutines with a
`promise_type` that implements a custom `operator new`.

The patch does consider C++23 `static operator()`, which already worked
as there is no `this` parameter.
---
 clang/include/clang/Sema/Sema.h  | 12 ++++-
 clang/lib/Sema/SemaCoroutine.cpp | 18 +++++++-
 clang/lib/Sema/SemaExprCXX.cpp   | 16 +++++--
 clang/test/SemaCXX/gh84064-1.cpp | 79 ++++++++++++++++++++++++++++++++
 clang/test/SemaCXX/gh84064-2.cpp | 53 +++++++++++++++++++++
 5 files changed, 169 insertions(+), 9 deletions(-)
 create mode 100644 clang/test/SemaCXX/gh84064-1.cpp
 create mode 100644 clang/test/SemaCXX/gh84064-2.cpp

diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 5b4d67494f2a..cfc1c3b34947 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -6752,10 +6752,18 @@ public:
                             SourceLocation RParenLoc);
 
   //// ActOnCXXThis -  Parse 'this' pointer.
-  ExprResult ActOnCXXThis(SourceLocation loc);
+  ///
+  /// \param ThisRefersToClosureObject Whether to skip the 'this' check for a
+  /// lambda because 'this' refers to the closure object.
+  ExprResult ActOnCXXThis(SourceLocation loc,
+                          bool ThisRefersToClosureObject = false);
 
   /// Build a CXXThisExpr and mark it referenced in the current context.
-  Expr *BuildCXXThisExpr(SourceLocation Loc, QualType Type, bool IsImplicit);
+  ///
+  /// \param ThisRefersToClosureObject Whether to skip the 'this' check for a
+  /// lambda because 'this' refers to the closure object.
+  Expr *BuildCXXThisExpr(SourceLocation Loc, QualType Type, bool IsImplicit,
+                         bool ThisRefersToClosureObject = false);
   void MarkThisReferenced(CXXThisExpr *This);
 
   /// Try to retrieve the type of the 'this' pointer.
diff --git a/clang/lib/Sema/SemaCoroutine.cpp b/clang/lib/Sema/SemaCoroutine.cpp
index a969b9383563..301a5ff72a3b 100644
--- a/clang/lib/Sema/SemaCoroutine.cpp
+++ b/clang/lib/Sema/SemaCoroutine.cpp
@@ -25,6 +25,7 @@
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/Overload.h"
 #include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/Sema.h"
 #include "clang/Sema/SemaInternal.h"
 #include "llvm/ADT/SmallSet.h"
 
@@ -1378,8 +1379,21 @@ bool CoroutineStmtBuilder::makeReturnOnAllocFailure() {
 static bool collectPlacementArgs(Sema &S, FunctionDecl &FD, SourceLocation Loc,
                                  SmallVectorImpl &PlacementArgs) {
   if (auto *MD = dyn_cast(&FD)) {
-    if (MD->isImplicitObjectMemberFunction() && !isLambdaCallOperator(MD)) {
-      ExprResult ThisExpr = S.ActOnCXXThis(Loc);
+    if (MD->isImplicitObjectMemberFunction()) {
+      ExprResult ThisExpr{};
+
+      if (isLambdaCallOperator(MD) && !MD->isStatic()) {
+        Qualifiers ThisQuals = MD->getMethodQualifiers();
+        CXXRecordDecl *Record = MD->getParent();
+
+        Sema::CXXThisScopeRAII ThisScope(S, Record, ThisQuals,
+                                         Record != nullptr);
+
+        ThisExpr = S.ActOnCXXThis(Loc, /*ThisRefersToClosureObject=*/true);
+      } else {
+        ThisExpr = S.ActOnCXXThis(Loc);
+      }
+
       if (ThisExpr.isInvalid())
         return false;
       ThisExpr = S.CreateBuiltinUnaryOp(Loc, UO_Deref, ThisExpr.get());
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index c34a40fa7c81..88e3d9ced044 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -1414,7 +1414,8 @@ bool Sema::CheckCXXThisCapture(SourceLocation Loc, const bool Explicit,
   return false;
 }
 
-ExprResult Sema::ActOnCXXThis(SourceLocation Loc) {
+ExprResult Sema::ActOnCXXThis(SourceLocation Loc,
+                              bool ThisRefersToClosureObject) {
   /// C++ 9.3.2: In the body of a non-static member function, the keyword this
   /// is a non-lvalue expression whose value is the address of the object for
   /// which the function is called.
@@ -1434,13 +1435,18 @@ ExprResult Sema::ActOnCXXThis(SourceLocation Loc) {
     return Diag(Loc, diag::err_invalid_this_use) << 0;
   }
 
-  return BuildCXXThisExpr(Loc, ThisTy, /*IsImplicit=*/false);
+  return BuildCXXThisExpr(Loc, ThisTy, /*IsImplicit=*/false,
+                          ThisRefersToClosureObject);
 }
 
-Expr *Sema::BuildCXXThisExpr(SourceLocation Loc, QualType Type,
-                             bool IsImplicit) {
+Expr *Sema::BuildCXXThisExpr(SourceLocation Loc, QualType Type, bool IsImplicit,
+                             bool ThisRefersToClosureObject) {
   auto *This = CXXThisExpr::Create(Context, Loc, Type, IsImplicit);
-  MarkThisReferenced(This);
+
+  if (!ThisRefersToClosureObject) {
+    MarkThisReferenced(This);
+  }
+
   return This;
 }
 
diff --git a/clang/test/SemaCXX/gh84064-1.cpp b/clang/test/SemaCXX/gh84064-1.cpp
new file mode 100644
index 000000000000..d9c2738a002b
--- /dev/null
+++ b/clang/test/SemaCXX/gh84064-1.cpp
@@ -0,0 +1,79 @@
+// RUN: %clang_cc1 -fsyntax-only -verify -I%S/Inputs -std=c++20 %s
+
+// expected-no-diagnostics
+
+#include "std-coroutine.h"
+
+using size_t = decltype(sizeof(0));
+
+struct Generator {
+  struct promise_type {
+    int _val{};
+
+    Generator get_return_object() noexcept
+    {
+      return {};
+    }
+
+    std::suspend_never initial_suspend() noexcept
+    {
+      return {};
+    }
+
+    std::suspend_always final_suspend() noexcept
+    {
+      return {};
+    }
+
+    void return_void() noexcept {}
+    void unhandled_exception() noexcept {}
+
+    template
+    static void*
+    operator new(size_t size,
+                 This&,
+                 TheRest&&...) noexcept
+    {
+        return nullptr;
+    }
+
+    static void operator delete(void*, size_t)
+    {
+    }
+  };
+};
+
+struct CapturingThisTest
+{
+    int x{};
+
+    void AsPointer()
+    {
+      auto lamb = [=,this]() -> Generator {
+        int y = x;
+        co_return;
+      };
+
+      static_assert(sizeof(decltype(lamb)) == sizeof(void*));
+    }
+
+    void AsStarThis()
+    {
+      auto lamb = [*this]() -> Generator {
+        int y = x;
+        co_return;
+      };
+
+      static_assert(sizeof(decltype(lamb)) == sizeof(int));
+    }
+};
+
+int main()
+{
+  auto lamb = []() -> Generator {
+    co_return;
+  };
+
+  static_assert(sizeof(decltype(lamb)) == 1);
+}
+
diff --git a/clang/test/SemaCXX/gh84064-2.cpp b/clang/test/SemaCXX/gh84064-2.cpp
new file mode 100644
index 000000000000..457de43eab6d
--- /dev/null
+++ b/clang/test/SemaCXX/gh84064-2.cpp
@@ -0,0 +1,53 @@
+// RUN: %clang_cc1 -fsyntax-only -verify -I%S/Inputs -std=c++23 %s
+
+// expected-no-diagnostics
+
+#include "std-coroutine.h"
+
+using size_t = decltype(sizeof(0));
+
+struct GeneratorStatic {
+  struct promise_type {
+    int _val{};
+
+    GeneratorStatic get_return_object() noexcept
+    {
+      return {};
+    }
+
+    std::suspend_never initial_suspend() noexcept
+    {
+      return {};
+    }
+
+    std::suspend_always final_suspend() noexcept
+    {
+      return {};
+    }
+
+    void return_void() noexcept {}
+    void unhandled_exception() noexcept {}
+
+    template
+    static void*
+    operator new(size_t  size,
+                 TheRest&&...) noexcept
+    {
+        return nullptr;
+    }
+
+    static void operator delete(void*, size_t)
+    {
+    }
+  };
+};
+
+
+int main()
+{
+  auto lambCpp23 = []() static -> GeneratorStatic {
+    co_return;
+  };
+
+  static_assert(sizeof(decltype(lambCpp23)) == 1);
+}
-- 
GitLab


From 81601391369c267216199db3f7fcb9864ccf6fec Mon Sep 17 00:00:00 2001
From: Jonathan Thackray 
Date: Fri, 8 Mar 2024 16:11:36 +0000
Subject: [PATCH 627/929] Add support for Arm Cortex A78AE CPU (#84485)

Add support for Arm Cortex A78AE CPU

Technical Reference Manual for Arm Cortex A78AE:
   https://developer.arm.com/documentation/101779/0003

Fixes #84450
---
 clang/docs/ReleaseNotes.rst                   |  2 ++
 clang/test/Driver/aarch64-mcpu.c              |  2 ++
 clang/test/Driver/arm-cortex-cpus-2.c         |  7 +++++++
 clang/test/Misc/target-invalid-cpu-note.c     |  6 +++---
 .../llvm/TargetParser/AArch64TargetParser.h   |  5 +++++
 .../llvm/TargetParser/ARMTargetParser.def     |  2 ++
 llvm/lib/Target/AArch64/AArch64.td            | 18 ++++++++++++++++++
 llvm/lib/Target/AArch64/AArch64Subtarget.cpp  |  1 +
 llvm/lib/Target/AArch64/AArch64Subtarget.h    |  1 +
 llvm/lib/Target/ARM/ARM.td                    | 10 ++++++++++
 llvm/lib/Target/ARM/ARMSubtarget.cpp          |  1 +
 llvm/lib/Target/ARM/ARMSubtarget.h            |  1 +
 llvm/lib/TargetParser/Host.cpp                |  1 +
 .../TargetParser/TargetParserTest.cpp         | 19 +++++++++++++++++--
 14 files changed, 71 insertions(+), 5 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 24b452bd487f..690fc7ed271a 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -388,6 +388,8 @@ Arm and AArch64 Support
   instructions (rdm). The identifier is available on the command line as
   a feature modifier for -march and -mcpu as well as via target attributes
   like ``target_version`` or ``target_clones``.
+- Support has been added for the following processors (-mcpu identifiers in parenthesis):
+    * Arm Cortex-A78AE (cortex-a78ae).
 
 Android Support
 ^^^^^^^^^^^^^^^
diff --git a/clang/test/Driver/aarch64-mcpu.c b/clang/test/Driver/aarch64-mcpu.c
index 3e07f3597f34..cacfc691058d 100644
--- a/clang/test/Driver/aarch64-mcpu.c
+++ b/clang/test/Driver/aarch64-mcpu.c
@@ -50,6 +50,8 @@
 // CORTEXA78: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "cortex-a78"
 // RUN: %clang --target=aarch64 -mcpu=cortex-a78c  -### -c %s 2>&1 | FileCheck -check-prefix=CORTEX-A78C %s
 // CORTEX-A78C: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "cortex-a78c"
+// RUN: %clang --target=aarch64 -mcpu=cortex-a78ae  -### -c %s 2>&1 | FileCheck -check-prefix=CORTEX-A78AE %s
+// CORTEX-A78AE: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "cortex-a78ae"
 // RUN: %clang --target=aarch64 -mcpu=cortex-a715  -### -c %s 2>&1 | FileCheck -check-prefix=CORTEX-A715 %s
 // CORTEX-A715: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "cortex-a715"
 // RUN: %clang --target=aarch64 -mcpu=cortex-a720  -### -c %s 2>&1 | FileCheck -check-prefix=CORTEX-A720 %s
diff --git a/clang/test/Driver/arm-cortex-cpus-2.c b/clang/test/Driver/arm-cortex-cpus-2.c
index c322303d2278..5ce3758655b5 100644
--- a/clang/test/Driver/arm-cortex-cpus-2.c
+++ b/clang/test/Driver/arm-cortex-cpus-2.c
@@ -537,6 +537,13 @@
 // CHECK-CORTEX-A78C-MFPU: "-target-feature" "+sha2"
 // CHECK-CORTEX-A78C-MFPU: "-target-feature" "+aes"
 
+// RUN: %clang -target armv8a-arm-none-eabi -mcpu=cortex-a78ae -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CORTEX-A78AE %s
+// RUN: %clang -target armv8a-arm-none-eabi -mcpu=cortex-a78ae -mfpu=crypto-neon-fp-armv8 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CORTEX-A78AE-MFPU %s
+// CHECK-CORTEX-A78AE: "-cc1"{{.*}} "-triple" "armv8.2a-{{.*}} "-target-cpu" "cortex-a78ae"
+// CHECK-CORTEX-A78AE-MFPU: "-cc1"{{.*}} "-target-feature" "+fp-armv8"
+// CHECK-CORTEX-A78AE-MFPU: "-target-feature" "+sha2"
+// CHECK-CORTEX-A78AE-MFPU: "-target-feature" "+aes"
+
 // RUN: %clang -target armv8a-arm-none-eabi -mcpu=cortex-a710 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CORTEX-A710 %s
 // RUN: %clang -target armv8a-arm-none-eabi -mcpu=cortex-a710 -mfpu=crypto-neon-fp-armv8 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CORTEX-A710-MFPU %s
 // CHECK-CORTEX-A710: "-cc1"{{.*}} "-triple" "armv9a-{{.*}} "-target-cpu" "cortex-a710"
diff --git a/clang/test/Misc/target-invalid-cpu-note.c b/clang/test/Misc/target-invalid-cpu-note.c
index ef2c1c0cfd3d..b65a8fb057ee 100644
--- a/clang/test/Misc/target-invalid-cpu-note.c
+++ b/clang/test/Misc/target-invalid-cpu-note.c
@@ -1,15 +1,15 @@
 // Use CHECK-NEXT instead of multiple CHECK-SAME to ensure we will fail if there is anything extra in the output.
 // RUN: not %clang_cc1 -triple armv5--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix ARM
 // ARM: error: unknown target CPU 'not-a-cpu'
-// ARM-NEXT: note: valid target CPU values are: arm8, arm810, strongarm, strongarm110, strongarm1100, strongarm1110, arm7tdmi, arm7tdmi-s, arm710t, arm720t, arm9, arm9tdmi, arm920, arm920t, arm922t, arm940t, ep9312, arm10tdmi, arm1020t, arm9e, arm946e-s, arm966e-s, arm968e-s, arm10e, arm1020e, arm1022e, arm926ej-s, arm1136j-s, arm1136jf-s, mpcore, mpcorenovfp, arm1176jz-s, arm1176jzf-s, arm1156t2-s, arm1156t2f-s, cortex-m0, cortex-m0plus, cortex-m1, sc000, cortex-a5, cortex-a7, cortex-a8, cortex-a9, cortex-a12, cortex-a15, cortex-a17, krait, cortex-r4, cortex-r4f, cortex-r5, cortex-r7, cortex-r8, cortex-r52, sc300, cortex-m3, cortex-m4, cortex-m7, cortex-m23, cortex-m33, cortex-m35p, cortex-m55, cortex-m85, cortex-m52, cortex-a32, cortex-a35, cortex-a53, cortex-a55, cortex-a57, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78c, cortex-a710, cortex-x1, cortex-x1c, neoverse-n1, neoverse-n2, neoverse-v1, cyclone, exynos-m3, exynos-m4, exynos-m5, kryo, iwmmxt, xscale, swift{{$}}
+// ARM-NEXT: note: valid target CPU values are: arm8, arm810, strongarm, strongarm110, strongarm1100, strongarm1110, arm7tdmi, arm7tdmi-s, arm710t, arm720t, arm9, arm9tdmi, arm920, arm920t, arm922t, arm940t, ep9312, arm10tdmi, arm1020t, arm9e, arm946e-s, arm966e-s, arm968e-s, arm10e, arm1020e, arm1022e, arm926ej-s, arm1136j-s, arm1136jf-s, mpcore, mpcorenovfp, arm1176jz-s, arm1176jzf-s, arm1156t2-s, arm1156t2f-s, cortex-m0, cortex-m0plus, cortex-m1, sc000, cortex-a5, cortex-a7, cortex-a8, cortex-a9, cortex-a12, cortex-a15, cortex-a17, krait, cortex-r4, cortex-r4f, cortex-r5, cortex-r7, cortex-r8, cortex-r52, sc300, cortex-m3, cortex-m4, cortex-m7, cortex-m23, cortex-m33, cortex-m35p, cortex-m55, cortex-m85, cortex-m52, cortex-a32, cortex-a35, cortex-a53, cortex-a55, cortex-a57, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78ae, cortex-a78c, cortex-a710, cortex-x1, cortex-x1c, neoverse-n1, neoverse-n2, neoverse-v1, cyclone, exynos-m3, exynos-m4, exynos-m5, kryo, iwmmxt, xscale, swift{{$}}
 
 // RUN: not %clang_cc1 -triple arm64--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix AARCH64
 // AARCH64: error: unknown target CPU 'not-a-cpu'
-// AARCH64-NEXT: note: valid target CPU values are: cortex-a34, cortex-a35, cortex-a53, cortex-a55, cortex-a510, cortex-a520, cortex-a57, cortex-a65, cortex-a65ae, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78c, cortex-a710, cortex-a715, cortex-a720, cortex-r82, cortex-x1, cortex-x1c, cortex-x2, cortex-x3, cortex-x4, neoverse-e1, neoverse-n1, neoverse-n2, neoverse-512tvb, neoverse-v1, neoverse-v2, cyclone, apple-a7, apple-a8, apple-a9, apple-a10, apple-a11, apple-a12, apple-a13, apple-a14, apple-a15, apple-a16, apple-a17, apple-m1, apple-m2, apple-m3, apple-s4, apple-s5, exynos-m3, exynos-m4, exynos-m5, falkor, saphira, kryo, thunderx2t99, thunderx3t110, thunderx, thunderxt88, thunderxt81, thunderxt83, tsv110, a64fx, carmel, ampere1, ampere1a, ampere1b, cobalt-100, grace{{$}}
+// AARCH64-NEXT: note: valid target CPU values are: cortex-a34, cortex-a35, cortex-a53, cortex-a55, cortex-a510, cortex-a520, cortex-a57, cortex-a65, cortex-a65ae, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78ae, cortex-a78c, cortex-a710, cortex-a715, cortex-a720, cortex-r82, cortex-x1, cortex-x1c, cortex-x2, cortex-x3, cortex-x4, neoverse-e1, neoverse-n1, neoverse-n2, neoverse-512tvb, neoverse-v1, neoverse-v2, cyclone, apple-a7, apple-a8, apple-a9, apple-a10, apple-a11, apple-a12, apple-a13, apple-a14, apple-a15, apple-a16, apple-a17, apple-m1, apple-m2, apple-m3, apple-s4, apple-s5, exynos-m3, exynos-m4, exynos-m5, falkor, saphira, kryo, thunderx2t99, thunderx3t110, thunderx, thunderxt88, thunderxt81, thunderxt83, tsv110, a64fx, carmel, ampere1, ampere1a, ampere1b, cobalt-100, grace{{$}}
 
 // RUN: not %clang_cc1 -triple arm64--- -tune-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix TUNE_AARCH64
 // TUNE_AARCH64: error: unknown target CPU 'not-a-cpu'
-// TUNE_AARCH64-NEXT: note: valid target CPU values are: cortex-a34, cortex-a35, cortex-a53, cortex-a55, cortex-a510, cortex-a520, cortex-a57, cortex-a65, cortex-a65ae, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78c, cortex-a710, cortex-a715, cortex-a720, cortex-r82, cortex-x1, cortex-x1c, cortex-x2, cortex-x3, cortex-x4, neoverse-e1, neoverse-n1, neoverse-n2, neoverse-512tvb, neoverse-v1, neoverse-v2, cyclone, apple-a7, apple-a8, apple-a9, apple-a10, apple-a11, apple-a12, apple-a13, apple-a14, apple-a15, apple-a16, apple-a17, apple-m1, apple-m2, apple-m3, apple-s4, apple-s5, exynos-m3, exynos-m4, exynos-m5, falkor, saphira, kryo, thunderx2t99, thunderx3t110, thunderx, thunderxt88, thunderxt81, thunderxt83, tsv110, a64fx, carmel, ampere1, ampere1a, ampere1b, cobalt-100, grace{{$}}
+// TUNE_AARCH64-NEXT: note: valid target CPU values are: cortex-a34, cortex-a35, cortex-a53, cortex-a55, cortex-a510, cortex-a520, cortex-a57, cortex-a65, cortex-a65ae, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78ae, cortex-a78c, cortex-a710, cortex-a715, cortex-a720, cortex-r82, cortex-x1, cortex-x1c, cortex-x2, cortex-x3, cortex-x4, neoverse-e1, neoverse-n1, neoverse-n2, neoverse-512tvb, neoverse-v1, neoverse-v2, cyclone, apple-a7, apple-a8, apple-a9, apple-a10, apple-a11, apple-a12, apple-a13, apple-a14, apple-a15, apple-a16, apple-a17, apple-m1, apple-m2, apple-m3, apple-s4, apple-s5, exynos-m3, exynos-m4, exynos-m5, falkor, saphira, kryo, thunderx2t99, thunderx3t110, thunderx, thunderxt88, thunderxt81, thunderxt83, tsv110, a64fx, carmel, ampere1, ampere1a, ampere1b, cobalt-100, grace{{$}}
 
 // RUN: not %clang_cc1 -triple i386--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix X86
 // X86: error: unknown target CPU 'not-a-cpu'
diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
index e74e8fa32a62..f4bb94f98bcd 100644
--- a/llvm/include/llvm/TargetParser/AArch64TargetParser.h
+++ b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
@@ -592,6 +592,11 @@ inline constexpr CpuInfo CpuInfos[] = {
                                AArch64::AEK_FP16, AArch64::AEK_DOTPROD,
                                AArch64::AEK_RCPC, AArch64::AEK_SSBS,
                                AArch64::AEK_PROFILE})},
+    {"cortex-a78ae", ARMV8_2A,
+     AArch64::ExtensionBitset({AArch64::AEK_AES, AArch64::AEK_SHA2,
+                               AArch64::AEK_FP16, AArch64::AEK_DOTPROD,
+                               AArch64::AEK_RCPC, AArch64::AEK_SSBS,
+                               AArch64::AEK_PROFILE})},
     {"cortex-a78c", ARMV8_2A,
      AArch64::ExtensionBitset(
          {AArch64::AEK_AES, AArch64::AEK_SHA2, AArch64::AEK_FP16,
diff --git a/llvm/include/llvm/TargetParser/ARMTargetParser.def b/llvm/include/llvm/TargetParser/ARMTargetParser.def
index fa6cfdfeda28..b821d224d7a8 100644
--- a/llvm/include/llvm/TargetParser/ARMTargetParser.def
+++ b/llvm/include/llvm/TargetParser/ARMTargetParser.def
@@ -363,6 +363,8 @@ ARM_CPU_NAME("cortex-a77", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
              (ARM::AEK_FP16 | ARM::AEK_DOTPROD))
 ARM_CPU_NAME("cortex-a78", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
              (ARM::AEK_FP16 | ARM::AEK_DOTPROD))
+ARM_CPU_NAME("cortex-a78ae", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
+             (ARM::AEK_RAS | ARM::AEK_DOTPROD))
 ARM_CPU_NAME("cortex-a78c", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
              ARM::AEK_FP16 | ARM::AEK_DOTPROD)
 ARM_CPU_NAME("cortex-a710", ARMV9A, FK_NEON_FP_ARMV8, false,
diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td
index b837066554f3..402c7292d7f8 100644
--- a/llvm/lib/Target/AArch64/AArch64.td
+++ b/llvm/lib/Target/AArch64/AArch64.td
@@ -944,6 +944,18 @@ def TuneA78 : SubtargetFeature<"a78", "ARMProcFamily", "CortexA78",
                                FeatureEnableSelectOptimize,
                                FeaturePredictableSelectIsExpensive]>;
 
+def TuneA78AE : SubtargetFeature<"a78ae", "ARMProcFamily",
+                                 "CortexA78AE",
+                                 "Cortex-A78AE ARM processors", [
+                                 FeatureCmpBccFusion,
+                                 FeatureFuseAES,
+                                 FeatureFuseAdrpAdd,
+                                 FeatureAddrLSLFast,
+                                 FeatureALULSLFast,
+                                 FeaturePostRAScheduler,
+                                 FeatureEnableSelectOptimize,
+                                 FeaturePredictableSelectIsExpensive]>;
+
 def TuneA78C : SubtargetFeature<"a78c", "ARMProcFamily",
                                 "CortexA78C",
                                 "Cortex-A78C ARM processors", [
@@ -1425,6 +1437,10 @@ def ProcessorFeatures {
                                  FeatureNEON, FeatureFullFP16, FeatureDotProd,
                                  FeatureRCPC, FeaturePerfMon, FeatureSPE,
                                  FeatureSSBS];
+  list A78AE = [HasV8_2aOps, FeatureCrypto, FeatureFPARMv8,
+                                  FeatureNEON, FeatureFullFP16, FeatureDotProd,
+                                  FeatureRCPC, FeaturePerfMon, FeatureSPE,
+                                  FeatureSSBS];
   list A78C = [HasV8_2aOps, FeatureCrypto, FeatureFPARMv8,
                                  FeatureNEON, FeatureFullFP16, FeatureDotProd,
                                  FeatureFlagM, FeaturePAuth,
@@ -1602,6 +1618,8 @@ def : ProcessorModel<"cortex-a77", CortexA57Model, ProcessorFeatures.A77,
                      [TuneA77]>;
 def : ProcessorModel<"cortex-a78", CortexA57Model, ProcessorFeatures.A78,
                      [TuneA78]>;
+def : ProcessorModel<"cortex-a78ae", CortexA57Model, ProcessorFeatures.A78AE,
+                     [TuneA78AE]>;
 def : ProcessorModel<"cortex-a78c", CortexA57Model, ProcessorFeatures.A78C,
                      [TuneA78C]>;
 def : ProcessorModel<"cortex-a710", NeoverseN2Model, ProcessorFeatures.A710,
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
index 2fa87f6afc7a..23b1deb3697f 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
@@ -140,6 +140,7 @@ void AArch64Subtarget::initializeProperties(bool HasMinSize) {
   case CortexA76:
   case CortexA77:
   case CortexA78:
+  case CortexA78AE:
   case CortexA78C:
   case CortexR82:
   case CortexX1:
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
index 4dae3ab28d3e..95bef7a76bca 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -66,6 +66,7 @@ public:
     CortexA76,
     CortexA77,
     CortexA78,
+    CortexA78AE,
     CortexA78C,
     CortexA710,
     CortexA715,
diff --git a/llvm/lib/Target/ARM/ARM.td b/llvm/lib/Target/ARM/ARM.td
index b62e1a032631..4510c7cf4f42 100644
--- a/llvm/lib/Target/ARM/ARM.td
+++ b/llvm/lib/Target/ARM/ARM.td
@@ -809,6 +809,8 @@ def ProcA77     : SubtargetFeature<"a77", "ARMProcFamily", "CortexA77",
                                    "Cortex-A77 ARM processors", []>;
 def ProcA78     : SubtargetFeature<"cortex-a78", "ARMProcFamily", "CortexA78",
                                    "Cortex-A78 ARM processors", []>;
+def ProcA78AE   : SubtargetFeature<"cortex-a78ae", "ARMProcFamily", "CortexA78AE",
+                                   "Cortex-A78AE ARM processors", []>;
 def ProcA78C    : SubtargetFeature<"a78c", "ARMProcFamily", "CortexA78C",
                                    "Cortex-A78C ARM processors", []>;
 def ProcA710    : SubtargetFeature<"cortex-a710", "ARMProcFamily",
@@ -1632,6 +1634,14 @@ def : ProcNoItin<"cortex-a78",                          [ARMv82a, ProcA78,
                                                          FeatureFullFP16,
                                                          FeatureDotProd]>;
 
+def : ProcNoItin<"cortex-a78ae",                        [ARMv82a, ProcA78AE,
+                                                         FeatureHWDivThumb,
+                                                         FeatureHWDivARM,
+                                                         FeatureCrypto,
+                                                         FeatureCRC,
+                                                         FeatureFullFP16,
+                                                         FeatureDotProd]>;
+
 def : ProcNoItin<"cortex-a78c",                         [ARMv82a, ProcA78C,
                                                          FeatureHWDivThumb,
                                                          FeatureHWDivARM,
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.cpp b/llvm/lib/Target/ARM/ARMSubtarget.cpp
index 691715dc2963..717e61518c6e 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.cpp
+++ b/llvm/lib/Target/ARM/ARMSubtarget.cpp
@@ -289,6 +289,7 @@ void ARMSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
   case CortexA76:
   case CortexA77:
   case CortexA78:
+  case CortexA78AE:
   case CortexA78C:
   case CortexA710:
   case CortexR4:
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.h b/llvm/lib/Target/ARM/ARMSubtarget.h
index 044b1c4c54e0..497ae160fde2 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.h
+++ b/llvm/lib/Target/ARM/ARMSubtarget.h
@@ -66,6 +66,7 @@ protected:
     CortexA76,
     CortexA77,
     CortexA78,
+    CortexA78AE,
     CortexA78C,
     CortexA710,
     CortexA8,
diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp
index a4cc757a9214..ae58ddf03f86 100644
--- a/llvm/lib/TargetParser/Host.cpp
+++ b/llvm/lib/TargetParser/Host.cpp
@@ -217,6 +217,7 @@ StringRef sys::detail::getHostCPUNameForARM(StringRef ProcCpuinfoContent) {
         .Case("0xd0b", "cortex-a76")
         .Case("0xd0d", "cortex-a77")
         .Case("0xd41", "cortex-a78")
+        .Case("0xd42", "cortex-a78ae")
         .Case("0xd47", "cortex-a710")
         .Case("0xd4d", "cortex-a715")
         .Case("0xd81", "cortex-a720")
diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp
index 7e57c4fc378a..3773f59a3c5a 100644
--- a/llvm/unittests/TargetParser/TargetParserTest.cpp
+++ b/llvm/unittests/TargetParser/TargetParserTest.cpp
@@ -454,6 +454,12 @@ INSTANTIATE_TEST_SUITE_P(
                 ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB |
                 ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS,
             "8.2-A"),
+        ARMCPUTestParams(
+            "cortex-a78ae", "armv8.2-a", "crypto-neon-fp-armv8",
+            ARM::AEK_RAS | ARM::AEK_DOTPROD | ARM::AEK_SEC | ARM::AEK_MP |
+                ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB |
+                ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS,
+            "8.2-A"),
         ARMCPUTestParams(
             "cortex-x1", "armv8.2-a", "crypto-neon-fp-armv8",
             ARM::AEK_RAS | ARM::AEK_FP16 | ARM::AEK_DOTPROD | ARM::AEK_SEC |
@@ -542,7 +548,7 @@ INSTANTIATE_TEST_SUITE_P(
                                    "7-S")),
     ARMCPUTestParams::PrintToStringParamName);
 
-static constexpr unsigned NumARMCPUArchs = 90;
+static constexpr unsigned NumARMCPUArchs = 91;
 
 TEST(TargetParserTest, testARMCPUArchList) {
   SmallVector List;
@@ -1209,6 +1215,15 @@ INSTANTIATE_TEST_SUITE_P(
                  AArch64::AEK_DOTPROD, AArch64::AEK_RCPC, AArch64::AEK_SSBS,
                  AArch64::AEK_PROFILE}),
             "8.2-A"),
+        ARMCPUTestParams(
+            "cortex-a78ae", "armv8.2-a", "crypto-neon-fp-armv8",
+            AArch64::ExtensionBitset(
+                {AArch64::AEK_CRC, AArch64::AEK_AES, AArch64::AEK_SHA2,
+                 AArch64::AEK_FP, AArch64::AEK_RDM, AArch64::AEK_SIMD,
+                 AArch64::AEK_RAS, AArch64::AEK_LSE, AArch64::AEK_FP16,
+                 AArch64::AEK_DOTPROD, AArch64::AEK_RCPC, AArch64::AEK_SSBS,
+                 AArch64::AEK_PROFILE}),
+            "8.2-A"),
         ARMCPUTestParams(
             "cortex-a78c", "armv8.2-a", "crypto-neon-fp-armv8",
             AArch64::ExtensionBitset(
@@ -1702,7 +1717,7 @@ INSTANTIATE_TEST_SUITE_P(
     ARMCPUTestParams::PrintToStringParamName);
 
 // Note: number of CPUs includes aliases.
-static constexpr unsigned NumAArch64CPUArchs = 69;
+static constexpr unsigned NumAArch64CPUArchs = 70;
 
 TEST(TargetParserTest, testAArch64CPUArchList) {
   SmallVector List;
-- 
GitLab


From abfac563f5b5a123e4bf773c3a09777e6fc4f50c Mon Sep 17 00:00:00 2001
From: Boian Petkantchin 
Date: Fri, 8 Mar 2024 08:14:36 -0800
Subject: [PATCH 628/929] [mlir][mesh] Make sharding propagation and
 spmdization work on FuncOpInterface (#84415)

Make them more general instead of only supporting `func::FuncOp`.
---
 .../mlir/Dialect/Mesh/Transforms/Passes.td     |  4 ++--
 .../Mesh/Transforms/ShardingPropagation.cpp    |  5 +++--
 .../Dialect/Mesh/Transforms/Spmdization.cpp    | 18 ++++++++++--------
 mlir/test/Dialect/Linalg/mesh-spmdization.mlir |  3 +--
 .../Dialect/Mesh/sharding-propagation.mlir     |  2 +-
 mlir/test/Dialect/Mesh/spmdization.mlir        |  4 +++-
 6 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Mesh/Transforms/Passes.td b/mlir/include/mlir/Dialect/Mesh/Transforms/Passes.td
index 7fb6631574b4..06ebf151e7d6 100644
--- a/mlir/include/mlir/Dialect/Mesh/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/Mesh/Transforms/Passes.td
@@ -16,7 +16,7 @@ include "mlir/Pass/PassBase.td"
 // ShardingPropagation
 //===----------------------------------------------------------------------===//
 
-def ShardingPropagation : Pass<"sharding-propagation", "mlir::func::FuncOp"> {
+def ShardingPropagation : InterfacePass<"sharding-propagation", "mlir::FunctionOpInterface"> {
   let summary = "sharding propagation";
   let description = [{
     Propagates sharding information throughout the graph. After this pass, each
@@ -29,7 +29,7 @@ def ShardingPropagation : Pass<"sharding-propagation", "mlir::func::FuncOp"> {
   ];
 }
 
-def Spmdization : Pass<"mesh-spmdization", "mlir::func::FuncOp"> {
+def Spmdization : InterfacePass<"mesh-spmdization", "mlir::FunctionOpInterface"> {
   let summary = "Partition a function into SPMD form.";
   let description = [{
     This pass fits in right after a pass that annotates the function with
diff --git a/mlir/lib/Dialect/Mesh/Transforms/ShardingPropagation.cpp b/mlir/lib/Dialect/Mesh/Transforms/ShardingPropagation.cpp
index 9f2647b21cbf..29320f1e339f 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/ShardingPropagation.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/ShardingPropagation.cpp
@@ -12,6 +12,7 @@
 #include "mlir/Dialect/Mesh/IR/MeshDialect.h"
 #include "mlir/Dialect/Mesh/IR/MeshOps.h"
 #include "mlir/Dialect/Mesh/Interfaces/ShardingInterface.h"
+#include "mlir/Interfaces/FunctionInterfaces.h"
 #include "mlir/Pass/Pass.h"
 #include "llvm/Support/Debug.h"
 #include 
@@ -172,9 +173,9 @@ static LogicalResult visitOp(Operation *op, OpBuilder &builder) {
 struct ShardingPropagation
     : public mesh::impl::ShardingPropagationBase {
   void runOnOperation() override {
-    func::FuncOp funcOp = getOperation();
+    FunctionOpInterface funcOp = getOperation();
     MLIRContext *ctx = funcOp.getContext();
-    Region ®ion = funcOp.getBody();
+    Region ®ion = funcOp.getFunctionBody();
     OpBuilder builder(ctx);
     if (!region.hasOneBlock()) {
       funcOp.emitOpError() << "only one block is supported!";
diff --git a/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp b/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
index c4d8b0b15e46..e4868435135e 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
@@ -24,6 +24,8 @@
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/IR/Value.h"
+#include "mlir/Interfaces/ControlFlowInterfaces.h"
+#include "mlir/Interfaces/FunctionInterfaces.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Support/LLVM.h"
 #include "mlir/Support/LogicalResult.h"
@@ -694,7 +696,7 @@ static LogicalResult spmdizeBlock(Block &block, IRMapping &spmdizationMap,
 }
 
 static LogicalResult
-spmdizeFuncOp(func::FuncOp op, IRMapping &spmdizationMap,
+spmdizeFuncOp(FunctionOpInterface op, IRMapping &spmdizationMap,
               SymbolTableCollection &symbolTableCollection) {
   OpBuilder builder(op.getFunctionBody());
 
@@ -717,21 +719,21 @@ spmdizeFuncOp(func::FuncOp op, IRMapping &spmdizationMap,
 
   // Find a return op and change the function results signature to its operands
   // signature.
-  func::ReturnOp returnOp;
-  for (Block &block : op.getBody()) {
+  Operation *returnOp = nullptr;
+  for (Block &block : op.getFunctionBody()) {
     if (block.empty()) {
       continue;
     }
 
-    returnOp = llvm::cast(block.back());
-    if (returnOp) {
+    if (block.back().hasTrait()) {
+      returnOp = &block.back();
       break;
     }
   }
   assert(returnOp);
-  op.setFunctionType(FunctionType::get(op->getContext(),
-                                       op.getBody().front().getArgumentTypes(),
-                                       returnOp->getOperandTypes()));
+  op.setType(FunctionType::get(op->getContext(),
+                               op.getFunctionBody().front().getArgumentTypes(),
+                               returnOp->getOperandTypes()));
 
   return success();
 }
diff --git a/mlir/test/Dialect/Linalg/mesh-spmdization.mlir b/mlir/test/Dialect/Linalg/mesh-spmdization.mlir
index 6d21def8de27..bd56c801283b 100644
--- a/mlir/test/Dialect/Linalg/mesh-spmdization.mlir
+++ b/mlir/test/Dialect/Linalg/mesh-spmdization.mlir
@@ -1,6 +1,5 @@
 // RUN: mlir-opt \
-// RUN:  --mesh-spmdization \
-// RUN:  --test-constant-fold \
+// RUN:  --pass-pipeline="builtin.module(func.func(mesh-spmdization,test-constant-fold))" \
 // RUN:  --split-input-file \
 // RUN:  %s | FileCheck %s
 
diff --git a/mlir/test/Dialect/Mesh/sharding-propagation.mlir b/mlir/test/Dialect/Mesh/sharding-propagation.mlir
index 94f8d94073c5..270787ab5188 100644
--- a/mlir/test/Dialect/Mesh/sharding-propagation.mlir
+++ b/mlir/test/Dialect/Mesh/sharding-propagation.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -sharding-propagation %s | FileCheck %s
+// RUN: mlir-opt --pass-pipeline="builtin.module(func.func(sharding-propagation))" %s | FileCheck %s
 
 mesh.mesh @mesh_1d(shape = ?)
 mesh.mesh @mesh_2d(shape = 2x4)
diff --git a/mlir/test/Dialect/Mesh/spmdization.mlir b/mlir/test/Dialect/Mesh/spmdization.mlir
index 572d3eb55eaa..2df247aba351 100644
--- a/mlir/test/Dialect/Mesh/spmdization.mlir
+++ b/mlir/test/Dialect/Mesh/spmdization.mlir
@@ -1,4 +1,6 @@
-// RUN: mlir-opt --mesh-spmdization --test-constant-fold %s | FileCheck %s
+// RUN: mlir-opt \
+// RUN:   --pass-pipeline="builtin.module(func.func(mesh-spmdization,test-constant-fold))" \
+// RUN:   %s | FileCheck %s
 
 mesh.mesh @mesh_1d(shape = 2)
 
-- 
GitLab


From c54e0524eeffcf2c5428cd2bc0449a1989e82cd8 Mon Sep 17 00:00:00 2001
From: erichkeane 
Date: Wed, 6 Mar 2024 12:28:58 -0800
Subject: [PATCH 629/929] [OpenACC] Add Compute Context Serialization test, fix
 a bug

While working on a followup patch, I discovered we didn't
serialize/deserialize the Compute Context properly, so this patch fixes
it, then adds a PCH mode to the ast test.
---
 clang/lib/Serialization/ASTReaderStmt.cpp        | 2 +-
 clang/lib/Serialization/ASTWriterStmt.cpp        | 2 +-
 clang/test/SemaOpenACC/compute-construct-ast.cpp | 9 +++++++++
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 3da44ffccc38..674ed47581df 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -2805,7 +2805,7 @@ void ASTStmtReader::VisitOpenACCAssociatedStmtConstruct(
 
 void ASTStmtReader::VisitOpenACCComputeConstruct(OpenACCComputeConstruct *S) {
   VisitStmt(S);
-  VisitOpenACCConstructStmt(S);
+  VisitOpenACCAssociatedStmtConstruct(S);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index 484621ae8130..7ce48fede383 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -2855,7 +2855,7 @@ void ASTStmtWriter::VisitOpenACCAssociatedStmtConstruct(
 
 void ASTStmtWriter::VisitOpenACCComputeConstruct(OpenACCComputeConstruct *S) {
   VisitStmt(S);
-  VisitOpenACCConstructStmt(S);
+  VisitOpenACCAssociatedStmtConstruct(S);
   Code = serialization::STMT_OPENACC_COMPUTE_CONSTRUCT;
 }
 
diff --git a/clang/test/SemaOpenACC/compute-construct-ast.cpp b/clang/test/SemaOpenACC/compute-construct-ast.cpp
index 0d1d0b266573..55c080838a18 100644
--- a/clang/test/SemaOpenACC/compute-construct-ast.cpp
+++ b/clang/test/SemaOpenACC/compute-construct-ast.cpp
@@ -1,5 +1,12 @@
 // RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
 
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -include-pch %t -ast-dump-all | FileCheck %s
+
+#ifndef PCH_HELPER
+#define PCH_HELPER
+
 void NormalFunc() {
   // FIXME: Add a test once we have clauses for this.
   // CHECK-LABEL: NormalFunc
@@ -104,3 +111,5 @@ struct S {
 void use() {
   TemplFunc();
 }
+#endif
+
-- 
GitLab


From 37f3661f4e8d180dc702ab42fa015bc3c21ceb51 Mon Sep 17 00:00:00 2001
From: Nick Desaulniers 
Date: Fri, 8 Mar 2024 08:50:41 -0800
Subject: [PATCH 630/929] [libc][docs] add slides link for Tue's 2023 talk
 (#84516)

---
 libc/docs/talks.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libc/docs/talks.rst b/libc/docs/talks.rst
index 6daae9f1e7b6..174de0f56e7d 100644
--- a/libc/docs/talks.rst
+++ b/libc/docs/talks.rst
@@ -6,6 +6,7 @@ Talks
 ----
 * Math functions in LLVM libc or yet another correctly rounded libm - Tue Ly
 
+  * `slides `__
   * `video `__
 * The LLVM C Library for GPUs - Joseph Huber
 
-- 
GitLab


From 599ab6d63e5b12fd9c421140653f0510f0d29eab Mon Sep 17 00:00:00 2001
From: Louis Dionne 
Date: Fri, 8 Mar 2024 12:09:35 -0500
Subject: [PATCH 631/929] Revert "[libc++] Disable module_std and
 module_std_compat tests"

It looks like that broke the CI even more. Reverting until I can
investigate.
---
 libcxx/test/libcxx/module_std.gen.py        | 3 ---
 libcxx/test/libcxx/module_std_compat.gen.py | 3 ---
 2 files changed, 6 deletions(-)

diff --git a/libcxx/test/libcxx/module_std.gen.py b/libcxx/test/libcxx/module_std.gen.py
index 932af742640c..fc23985caf30 100644
--- a/libcxx/test/libcxx/module_std.gen.py
+++ b/libcxx/test/libcxx/module_std.gen.py
@@ -18,9 +18,6 @@
 
 # RUN: %{python} %s %{libcxx-dir}/utils
 
-# TODO: Re-enable this test once we understand why it keeps timing out.
-# UNSUPPORTED: clang
-
 import sys
 
 sys.path.append(sys.argv[1])
diff --git a/libcxx/test/libcxx/module_std_compat.gen.py b/libcxx/test/libcxx/module_std_compat.gen.py
index d29f490ab135..000aa2998612 100644
--- a/libcxx/test/libcxx/module_std_compat.gen.py
+++ b/libcxx/test/libcxx/module_std_compat.gen.py
@@ -18,9 +18,6 @@
 
 # RUN: %{python} %s %{libcxx-dir}/utils
 
-# TODO: Re-enable this test once we understand why it keeps timing out.
-# UNSUPPORTED: clang
-
 import sys
 
 sys.path.append(sys.argv[1])
-- 
GitLab


From 60d7bf3dbe6c32ed67ed65cb15727cddd9531979 Mon Sep 17 00:00:00 2001
From: lntue <35648136+lntue@users.noreply.github.com>
Date: Fri, 8 Mar 2024 12:15:02 -0500
Subject: [PATCH 632/929] [libc][math][c23] Add (l|ll)rintf128 and
 (l|ll)roundf128 math functions. (#84504)

---
 libc/config/linux/aarch64/entrypoints.txt     |   5 +
 libc/config/linux/riscv/entrypoints.txt       |   5 +
 libc/config/linux/x86_64/entrypoints.txt      |   5 +
 libc/docs/math/index.rst                      |  10 ++
 libc/spec/gnu_ext.td                          |   2 -
 libc/spec/stdc.td                             |   8 ++
 libc/src/math/CMakeLists.txt                  |   5 +
 libc/src/math/generic/CMakeLists.txt          | 115 ++++++++++++++----
 libc/src/math/generic/llrintf128.cpp          |  21 ++++
 libc/src/math/generic/llroundf128.cpp         |  19 +++
 libc/src/math/generic/lrintf128.cpp           |  20 +++
 libc/src/math/generic/lroundf128.cpp          |  19 +++
 libc/src/math/generic/rintf128.cpp            |  19 +++
 libc/src/math/llrintf128.h                    |  20 +++
 libc/src/math/llroundf128.h                   |  20 +++
 libc/src/math/lrintf128.h                     |  20 +++
 libc/src/math/lroundf128.h                    |  20 +++
 libc/src/math/rintf128.h                      |  20 +++
 libc/test/src/math/smoke/CMakeLists.txt       |  81 ++++++++++++
 libc/test/src/math/smoke/llrintf128_test.cpp  |  14 +++
 libc/test/src/math/smoke/llroundf128_test.cpp |  13 ++
 libc/test/src/math/smoke/lrintf128_test.cpp   |  14 +++
 libc/test/src/math/smoke/lroundf128_test.cpp  |  13 ++
 libc/test/src/math/smoke/rintf128_test.cpp    |  13 ++
 24 files changed, 474 insertions(+), 27 deletions(-)
 create mode 100644 libc/src/math/generic/llrintf128.cpp
 create mode 100644 libc/src/math/generic/llroundf128.cpp
 create mode 100644 libc/src/math/generic/lrintf128.cpp
 create mode 100644 libc/src/math/generic/lroundf128.cpp
 create mode 100644 libc/src/math/generic/rintf128.cpp
 create mode 100644 libc/src/math/llrintf128.h
 create mode 100644 libc/src/math/llroundf128.h
 create mode 100644 libc/src/math/lrintf128.h
 create mode 100644 libc/src/math/lroundf128.h
 create mode 100644 libc/src/math/rintf128.h
 create mode 100644 libc/test/src/math/smoke/llrintf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/llroundf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/lrintf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/lroundf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/rintf128_test.cpp

diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index c32773f67cda..fa15ddd17aef 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -430,6 +430,11 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.ldexpf128
     libc.src.math.llogbf128
     libc.src.math.logbf128
+    libc.src.math.llrintf128
+    libc.src.math.llroundf128
+    libc.src.math.lrintf128
+    libc.src.math.lroundf128
+    libc.src.math.rintf128
     libc.src.math.roundf128
     libc.src.math.sqrtf128
     libc.src.math.truncf128
diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt
index bf518083b51f..924cf2f1d68b 100644
--- a/libc/config/linux/riscv/entrypoints.txt
+++ b/libc/config/linux/riscv/entrypoints.txt
@@ -438,6 +438,11 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.ldexpf128
     libc.src.math.llogbf128
     libc.src.math.logbf128
+    libc.src.math.llrintf128
+    libc.src.math.llroundf128
+    libc.src.math.lrintf128
+    libc.src.math.lroundf128
+    libc.src.math.rintf128
     libc.src.math.roundf128
     libc.src.math.sqrtf128
     libc.src.math.truncf128
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 0b77a9e170aa..0880c372b373 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -468,6 +468,11 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.ldexpf128
     libc.src.math.llogbf128
     libc.src.math.logbf128
+    libc.src.math.llrintf128
+    libc.src.math.llroundf128
+    libc.src.math.lrintf128
+    libc.src.math.lroundf128
+    libc.src.math.rintf128
     libc.src.math.roundf128
     libc.src.math.sqrtf128
     libc.src.math.truncf128
diff --git a/libc/docs/math/index.rst b/libc/docs/math/index.rst
index 80d12718edcc..81d95d9b6cfa 100644
--- a/libc/docs/math/index.rst
+++ b/libc/docs/math/index.rst
@@ -209,12 +209,16 @@ Basic Operations
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | llrintl      | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| llrintf128   | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | llround      | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | llroundf     | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | llroundl     | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| llroundf128  | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | logb         | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | logbf        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
@@ -229,12 +233,16 @@ Basic Operations
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | lrintl       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| lrintf128    | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | lround       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | lroundf      | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | lroundl      | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| lroundf128   | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | modf         | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | modff        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
@@ -283,6 +291,8 @@ Basic Operations
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | rintl        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| rintf128     | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | round        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | roundf       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
diff --git a/libc/spec/gnu_ext.td b/libc/spec/gnu_ext.td
index add07d75050d..161bb4e4a0d9 100644
--- a/libc/spec/gnu_ext.td
+++ b/libc/spec/gnu_ext.td
@@ -33,8 +33,6 @@ def GnuExtensions : StandardSpec<"GNUExtensions"> {
             RetValSpec,
             [ArgSpec, ArgSpec, ArgSpec]
         >,
-        FunctionSpec<"exp10", RetValSpec, [ArgSpec]>,
-        FunctionSpec<"exp10f", RetValSpec, [ArgSpec]>,
       ]
   >;
 
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
index d84eb3570e5c..cc845a93a333 100644
--- a/libc/spec/stdc.td
+++ b/libc/spec/stdc.td
@@ -470,6 +470,9 @@ def StdC : StandardSpec<"stdc"> {
           FunctionSpec<"expm1", RetValSpec, [ArgSpec]>,
           FunctionSpec<"expm1f", RetValSpec, [ArgSpec]>,
 
+          FunctionSpec<"exp10", RetValSpec, [ArgSpec]>,
+          FunctionSpec<"exp10f", RetValSpec, [ArgSpec]>,
+
           FunctionSpec<"remainderf", RetValSpec, [ArgSpec, ArgSpec]>,
           FunctionSpec<"remainder", RetValSpec, [ArgSpec, ArgSpec]>,
           FunctionSpec<"remainderl", RetValSpec, [ArgSpec, ArgSpec]>,
@@ -486,22 +489,27 @@ def StdC : StandardSpec<"stdc"> {
           FunctionSpec<"lround", RetValSpec, [ArgSpec]>,
           FunctionSpec<"lroundf", RetValSpec, [ArgSpec]>,
           FunctionSpec<"lroundl", RetValSpec, [ArgSpec]>,
+          GuardedFunctionSpec<"lroundf128", RetValSpec, [ArgSpec], "LIBC_TYPES_HAS_FLOAT128">,
 
           FunctionSpec<"llround", RetValSpec, [ArgSpec]>,
           FunctionSpec<"llroundf", RetValSpec, [ArgSpec]>,
           FunctionSpec<"llroundl", RetValSpec, [ArgSpec]>,
+          GuardedFunctionSpec<"llroundf128", RetValSpec, [ArgSpec], "LIBC_TYPES_HAS_FLOAT128">,
 
           FunctionSpec<"rint", RetValSpec, [ArgSpec]>,
           FunctionSpec<"rintf", RetValSpec, [ArgSpec]>,
           FunctionSpec<"rintl", RetValSpec, [ArgSpec]>,
+          GuardedFunctionSpec<"rintf128", RetValSpec, [ArgSpec], "LIBC_TYPES_HAS_FLOAT128">,
 
           FunctionSpec<"lrint", RetValSpec, [ArgSpec]>,
           FunctionSpec<"lrintf", RetValSpec, [ArgSpec]>,
           FunctionSpec<"lrintl", RetValSpec, [ArgSpec]>,
+          GuardedFunctionSpec<"lrintf128", RetValSpec, [ArgSpec], "LIBC_TYPES_HAS_FLOAT128">,
 
           FunctionSpec<"llrint", RetValSpec, [ArgSpec]>,
           FunctionSpec<"llrintf", RetValSpec, [ArgSpec]>,
           FunctionSpec<"llrintl", RetValSpec, [ArgSpec]>,
+          GuardedFunctionSpec<"llrintf128", RetValSpec, [ArgSpec], "LIBC_TYPES_HAS_FLOAT128">,
 
           FunctionSpec<"sqrt", RetValSpec, [ArgSpec]>,
           FunctionSpec<"sqrtf", RetValSpec, [ArgSpec]>,
diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt
index 882befd9f7e7..035eefd82d36 100644
--- a/libc/src/math/CMakeLists.txt
+++ b/libc/src/math/CMakeLists.txt
@@ -163,18 +163,22 @@ add_math_entrypoint_object(logbf128)
 add_math_entrypoint_object(llrint)
 add_math_entrypoint_object(llrintf)
 add_math_entrypoint_object(llrintl)
+add_math_entrypoint_object(llrintf128)
 
 add_math_entrypoint_object(llround)
 add_math_entrypoint_object(llroundf)
 add_math_entrypoint_object(llroundl)
+add_math_entrypoint_object(llroundf128)
 
 add_math_entrypoint_object(lrint)
 add_math_entrypoint_object(lrintf)
 add_math_entrypoint_object(lrintl)
+add_math_entrypoint_object(lrintf128)
 
 add_math_entrypoint_object(lround)
 add_math_entrypoint_object(lroundf)
 add_math_entrypoint_object(lroundl)
+add_math_entrypoint_object(lroundf128)
 
 add_math_entrypoint_object(modf)
 add_math_entrypoint_object(modff)
@@ -210,6 +214,7 @@ add_math_entrypoint_object(remquol)
 add_math_entrypoint_object(rint)
 add_math_entrypoint_object(rintf)
 add_math_entrypoint_object(rintl)
+add_math_entrypoint_object(rintf128)
 
 add_math_entrypoint_object(round)
 add_math_entrypoint_object(roundf)
diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index 82d2a5e66af7..a7b7065980b1 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -375,10 +375,10 @@ add_entrypoint_object(
     lround.cpp
   HDRS
     ../lround.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    -O2
 )
 
 add_entrypoint_object(
@@ -387,10 +387,10 @@ add_entrypoint_object(
     lroundf.cpp
   HDRS
     ../lroundf.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    -O2
 )
 
 add_entrypoint_object(
@@ -399,10 +399,23 @@ add_entrypoint_object(
     lroundl.cpp
   HDRS
     ../lroundl.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
+)
+
+add_entrypoint_object(
+  lroundf128
+  SRCS
+    lroundf128.cpp
+  HDRS
+    ../lroundf128.h
   COMPILE_OPTIONS
-    -O2
+    -O3
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.nearest_integer_operations
 )
 
 add_entrypoint_object(
@@ -411,10 +424,10 @@ add_entrypoint_object(
     llround.cpp
   HDRS
     ../llround.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    -O2
 )
 
 add_entrypoint_object(
@@ -423,10 +436,10 @@ add_entrypoint_object(
     llroundf.cpp
   HDRS
     ../llroundf.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    -O2
 )
 
 add_entrypoint_object(
@@ -435,10 +448,23 @@ add_entrypoint_object(
     llroundl.cpp
   HDRS
     ../llroundl.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
+)
+
+add_entrypoint_object(
+  llroundf128
+  SRCS
+    llroundf128.cpp
+  HDRS
+    ../llroundf128.h
   COMPILE_OPTIONS
-    -O2
+    -O3
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.nearest_integer_operations
 )
 
 add_entrypoint_object(
@@ -447,10 +473,10 @@ add_entrypoint_object(
     rint.cpp
   HDRS
     ../rint.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    -O2
 )
 
 add_entrypoint_object(
@@ -459,10 +485,10 @@ add_entrypoint_object(
     rintf.cpp
   HDRS
     ../rintf.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    -O2
 )
 
 add_entrypoint_object(
@@ -471,10 +497,23 @@ add_entrypoint_object(
     rintl.cpp
   HDRS
     ../rintl.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
+)
+
+add_entrypoint_object(
+  rintf128
+  SRCS
+    rintf128.cpp
+  HDRS
+    ../rintf128.h
   COMPILE_OPTIONS
-    -O2
+    -O3
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.nearest_integer_operations
 )
 
 add_entrypoint_object(
@@ -483,10 +522,10 @@ add_entrypoint_object(
     lrint.cpp
   HDRS
     ../lrint.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    -O2
 )
 
 add_entrypoint_object(
@@ -495,10 +534,10 @@ add_entrypoint_object(
     lrintf.cpp
   HDRS
     ../lrintf.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    -O2
 )
 
 add_entrypoint_object(
@@ -507,10 +546,23 @@ add_entrypoint_object(
     lrintl.cpp
   HDRS
     ../lrintl.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
+)
+
+add_entrypoint_object(
+  lrintf128
+  SRCS
+    lrintf128.cpp
+  HDRS
+    ../lrintf128.h
   COMPILE_OPTIONS
-    -O2
+    -O3
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.nearest_integer_operations
 )
 
 add_entrypoint_object(
@@ -519,10 +571,10 @@ add_entrypoint_object(
     llrint.cpp
   HDRS
     ../llrint.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    -O2
 )
 
 add_entrypoint_object(
@@ -531,10 +583,10 @@ add_entrypoint_object(
     llrintf.cpp
   HDRS
     ../llrintf.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    -O2
 )
 
 add_entrypoint_object(
@@ -543,10 +595,23 @@ add_entrypoint_object(
     llrintl.cpp
   HDRS
     ../llrintl.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
+)
+
+add_entrypoint_object(
+  llrintf128
+  SRCS
+    llrintf128.cpp
+  HDRS
+    ../llrintf128.h
   COMPILE_OPTIONS
-    -O2
+    -O3
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.nearest_integer_operations
 )
 
 add_entrypoint_object(
diff --git a/libc/src/math/generic/llrintf128.cpp b/libc/src/math/generic/llrintf128.cpp
new file mode 100644
index 000000000000..e5a4c50a26e8
--- /dev/null
+++ b/libc/src/math/generic/llrintf128.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of llrintf128 function -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/llrintf128.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(long long, llrintf128, (float128 x)) {
+  return fputil::round_to_signed_integer_using_current_rounding_mode(
+      x);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/llroundf128.cpp b/libc/src/math/generic/llroundf128.cpp
new file mode 100644
index 000000000000..25791631dd7e
--- /dev/null
+++ b/libc/src/math/generic/llroundf128.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of llroundf128 function ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/llroundf128.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(long long, llroundf128, (float128 x)) {
+  return fputil::round_to_signed_integer(x);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/lrintf128.cpp b/libc/src/math/generic/lrintf128.cpp
new file mode 100644
index 000000000000..8e06062fc580
--- /dev/null
+++ b/libc/src/math/generic/lrintf128.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of lrintf128 function ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/lrintf128.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(long, lrintf128, (float128 x)) {
+  return fputil::round_to_signed_integer_using_current_rounding_mode(x);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/lroundf128.cpp b/libc/src/math/generic/lroundf128.cpp
new file mode 100644
index 000000000000..f93c47503825
--- /dev/null
+++ b/libc/src/math/generic/lroundf128.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of lroundf128 function -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/lroundf128.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(long, lroundf128, (float128 x)) {
+  return fputil::round_to_signed_integer(x);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/rintf128.cpp b/libc/src/math/generic/rintf128.cpp
new file mode 100644
index 000000000000..ba9912d6f853
--- /dev/null
+++ b/libc/src/math/generic/rintf128.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of rintf128 function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/rintf128.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float128, rintf128, (float128 x)) {
+  return fputil::round_using_current_rounding_mode(x);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/llrintf128.h b/libc/src/math/llrintf128.h
new file mode 100644
index 000000000000..ac9c249342cc
--- /dev/null
+++ b/libc/src/math/llrintf128.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for llrintf128 --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LLRINTF128_H
+#define LLVM_LIBC_SRC_MATH_LLRINTF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+long long llrintf128(float128 x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_LLRINTF128_H
diff --git a/libc/src/math/llroundf128.h b/libc/src/math/llroundf128.h
new file mode 100644
index 000000000000..3245dfafc4d5
--- /dev/null
+++ b/libc/src/math/llroundf128.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for llroundf128 -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LLROUNDF128_H
+#define LLVM_LIBC_SRC_MATH_LLROUNDF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+long long llroundf128(float128 x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_LLROUNDF128_H
diff --git a/libc/src/math/lrintf128.h b/libc/src/math/lrintf128.h
new file mode 100644
index 000000000000..8f3f5ceabd3c
--- /dev/null
+++ b/libc/src/math/lrintf128.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for lrintf128 ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LRINTF128_H
+#define LLVM_LIBC_SRC_MATH_LRINTF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+long lrintf128(float128 x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_LRINTF128_H
diff --git a/libc/src/math/lroundf128.h b/libc/src/math/lroundf128.h
new file mode 100644
index 000000000000..663b3732655b
--- /dev/null
+++ b/libc/src/math/lroundf128.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for lroundf128 --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LROUNDF128_H
+#define LLVM_LIBC_SRC_MATH_LROUNDF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+long lroundf128(float128 x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_LROUNDF128_H
diff --git a/libc/src/math/rintf128.h b/libc/src/math/rintf128.h
new file mode 100644
index 000000000000..2d9248974f02
--- /dev/null
+++ b/libc/src/math/rintf128.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for rintf128 ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_RINTF128_H
+#define LLVM_LIBC_SRC_MATH_RINTF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float128 rintf128(float128 x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_RINTF128_H
diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt
index be1810944495..63faaa9d4e4c 100644
--- a/libc/test/src/math/smoke/CMakeLists.txt
+++ b/libc/test/src/math/smoke/CMakeLists.txt
@@ -387,6 +387,24 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  lroundf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    lroundf128_test.cpp
+  HDRS
+    RoundToIntegerTest.h
+  DEPENDS
+    libc.include.math
+    libc.src.errno.errno
+    libc.src.fenv.feclearexcept
+    libc.src.fenv.feraiseexcept
+    libc.src.fenv.fetestexcept
+    libc.src.math.lroundf128
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   llround_test
   SUITE
@@ -441,6 +459,24 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  llroundf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    llroundf128_test.cpp
+  HDRS
+    RoundToIntegerTest.h
+  DEPENDS
+    libc.include.math
+    libc.src.errno.errno
+    libc.src.fenv.feclearexcept
+    libc.src.fenv.feraiseexcept
+    libc.src.fenv.fetestexcept
+    libc.src.math.llroundf128
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   rint_test
   SUITE
@@ -486,6 +522,21 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  rintf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    rintf128_test.cpp
+  HDRS
+    RIntTest.h
+  DEPENDS
+    libc.include.math
+    libc.src.math.rintf128
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   lrint_test
   SUITE
@@ -531,6 +582,21 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  lrintf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    lrintf128_test.cpp
+  HDRS
+    RoundToIntegerTest.h
+  DEPENDS
+    libc.include.math
+    libc.src.math.lrintf128
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   llrint_test
   SUITE
@@ -576,6 +642,21 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  llrintf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    llrintf128_test.cpp
+  HDRS
+    RoundToIntegerTest.h
+  DEPENDS
+    libc.include.math
+    libc.src.math.llrintf128
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   expf_test
   SUITE
diff --git a/libc/test/src/math/smoke/llrintf128_test.cpp b/libc/test/src/math/smoke/llrintf128_test.cpp
new file mode 100644
index 000000000000..8847b2918e63
--- /dev/null
+++ b/libc/test/src/math/smoke/llrintf128_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for llrintf128 ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundToIntegerTest.h"
+
+#include "src/math/llrintf128.h"
+
+LIST_ROUND_TO_INTEGER_TESTS_WITH_MODES(float128, long long,
+                                       LIBC_NAMESPACE::llrintf128)
diff --git a/libc/test/src/math/smoke/llroundf128_test.cpp b/libc/test/src/math/smoke/llroundf128_test.cpp
new file mode 100644
index 000000000000..b00055e759f8
--- /dev/null
+++ b/libc/test/src/math/smoke/llroundf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for llroundf128 -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundToIntegerTest.h"
+
+#include "src/math/llroundf128.h"
+
+LIST_ROUND_TO_INTEGER_TESTS(float128, long long, LIBC_NAMESPACE::llroundf128)
diff --git a/libc/test/src/math/smoke/lrintf128_test.cpp b/libc/test/src/math/smoke/lrintf128_test.cpp
new file mode 100644
index 000000000000..a559ccf0eda7
--- /dev/null
+++ b/libc/test/src/math/smoke/lrintf128_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for lrintf128 -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundToIntegerTest.h"
+
+#include "src/math/lrintf128.h"
+
+LIST_ROUND_TO_INTEGER_TESTS_WITH_MODES(float128, long,
+                                       LIBC_NAMESPACE::lrintf128)
diff --git a/libc/test/src/math/smoke/lroundf128_test.cpp b/libc/test/src/math/smoke/lroundf128_test.cpp
new file mode 100644
index 000000000000..ed87c9af0489
--- /dev/null
+++ b/libc/test/src/math/smoke/lroundf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for lroundf128 ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundToIntegerTest.h"
+
+#include "src/math/lroundf128.h"
+
+LIST_ROUND_TO_INTEGER_TESTS(float128, long, LIBC_NAMESPACE::lroundf128)
diff --git a/libc/test/src/math/smoke/rintf128_test.cpp b/libc/test/src/math/smoke/rintf128_test.cpp
new file mode 100644
index 000000000000..11078d795bac
--- /dev/null
+++ b/libc/test/src/math/smoke/rintf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for rintf128 --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RIntTest.h"
+
+#include "src/math/rintf128.h"
+
+LIST_RINT_TESTS(float128, LIBC_NAMESPACE::rintf128)
-- 
GitLab


From 73b2d672c1c8318cd16a02812c39ae3997b9dbcd Mon Sep 17 00:00:00 2001
From: David Spickett 
Date: Fri, 8 Mar 2024 16:54:01 +0000
Subject: [PATCH 633/929] [lldb][FreeBSD] Fix crash when execve fails with
 asserts enabled

535da10842c7309e9eeaf9828cf6bb034fecaf16 introduced a check, when execve fails,
to see if we are allowed to trace programs at all.

Unfortunately because we like to call Status vars "error" and "status"
in various combinations, one got misnamed. This lead to lldb-server trying
to make an error value out of a success value when you did the following:

```
$ ./bin/lldb-server gdbserver 127.0.0.1:1234 -- is_not_a_file
Assertion failed: (Err && "Cannot create Expected from Error success value."), function Expected...
```

This happened because the execve fails, but the check whether we can
trace says yes we can trace, but then we use the Status from the check
to create the return value. That Status is in fact a success value not
the failed Status we got from the execve attempt.

With the name corrected you now get:
```
$ ./bin/lldb-server gdbserver 127.0.0.1:1234 -- is_not_a_file
error: failed to launch 'is_not_a_file': execve failed: No such file or directory
```

Which is what we expect to see. This also fixes the test `TestGDBRemoteLaunch.py`
when asserts are enabled.
---
 lldb/source/Plugins/Process/FreeBSD/NativeProcessFreeBSD.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lldb/source/Plugins/Process/FreeBSD/NativeProcessFreeBSD.cpp b/lldb/source/Plugins/Process/FreeBSD/NativeProcessFreeBSD.cpp
index 9c620e4807e3..064bddd37052 100644
--- a/lldb/source/Plugins/Process/FreeBSD/NativeProcessFreeBSD.cpp
+++ b/lldb/source/Plugins/Process/FreeBSD/NativeProcessFreeBSD.cpp
@@ -76,9 +76,9 @@ NativeProcessFreeBSD::Manager::Launch(ProcessLaunchInfo &launch_info,
                     .GetProcessId();
   LLDB_LOG(log, "pid = {0:x}", pid);
   if (status.Fail()) {
-    auto error = CanTrace();
     LLDB_LOG(log, "failed to launch process: {0}", status);
-    if (status.Fail())
+    auto error = CanTrace();
+    if (error.Fail())
       return error.ToError();
     return status.ToError();
   }
-- 
GitLab


From 0bbada93a559b604797fe57978f3eca5e41edaeb Mon Sep 17 00:00:00 2001
From: Eric Fiselier 
Date: Fri, 8 Mar 2024 12:17:07 -0500
Subject: [PATCH 634/929] Actually disable the module generation tests.

LIT was never really meant to generate tests during discovery,
and we probably shouldn't be doing this.

This hack is even worse than the initial attempt because it buries the
"UNSUPPORTED" at the bottom of the test.
---
 libcxx/test/libcxx/module_std.gen.py        | 5 +++++
 libcxx/test/libcxx/module_std_compat.gen.py | 5 +++++
 2 files changed, 10 insertions(+)

diff --git a/libcxx/test/libcxx/module_std.gen.py b/libcxx/test/libcxx/module_std.gen.py
index fc23985caf30..5acaa837d37e 100644
--- a/libcxx/test/libcxx/module_std.gen.py
+++ b/libcxx/test/libcxx/module_std.gen.py
@@ -16,7 +16,11 @@
 # to be one monolitic test. Since the test doesn't take very long it's
 # not a huge issue.
 
+# WARNING: Disabled at the bottom. Fix this test and remove the UNSUPPORTED line
+# TODO: Re-enable this test once we understand why it keeps timing out.
+
 # RUN: %{python} %s %{libcxx-dir}/utils
+# END.
 
 import sys
 
@@ -35,4 +39,5 @@ generator = module_test_generator(
 
 
 print("//--- module_std.sh.cpp")
+print('// UNSUPPORTED: clang')
 generator.write_test("std")
diff --git a/libcxx/test/libcxx/module_std_compat.gen.py b/libcxx/test/libcxx/module_std_compat.gen.py
index 000aa2998612..a502276a1ccf 100644
--- a/libcxx/test/libcxx/module_std_compat.gen.py
+++ b/libcxx/test/libcxx/module_std_compat.gen.py
@@ -16,7 +16,11 @@
 # to be one monolitic test. Since the test doesn't take very long it's
 # not a huge issue.
 
+# WARNING: Disabled at the bottom. Fix this test and remove the UNSUPPORTED line
+# TODO: Re-enable this test once we understand why it keeps timing out.
+
 # RUN: %{python} %s %{libcxx-dir}/utils
+# END.
 
 import sys
 
@@ -36,6 +40,7 @@ generator = module_test_generator(
 
 
 print("//--- module_std_compat.sh.cpp")
+print("// UNSUPPORTED: clang")
 generator.write_test(
     "std.compat",
     module_c_headers,
-- 
GitLab


From dbca8aa1472b9140a68918b2615519dc7dad593f Mon Sep 17 00:00:00 2001
From: Derek Schuff 
Date: Fri, 8 Mar 2024 09:20:44 -0800
Subject: [PATCH 635/929] [WebAssembly] Use SetVector instead of SmallPtrSet in
 FixBrTableDefaults (#84418)

This pass inserts all the MBBs into a set and then iterates over them.
But when
the number of elements gets large enough, SmallPtrSet expands into a
hash table
which wouldn't have a deterministic iteration order since the elements
are
pointers. This results in nondeterministic jump table layouts.
Use SetVector instead for a deterministic iteration order.
---
 .../Target/WebAssembly/WebAssemblyFixBrTableDefaults.cpp  | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyFixBrTableDefaults.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyFixBrTableDefaults.cpp
index 495f19a7ccde..4252fc1f55fc 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyFixBrTableDefaults.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyFixBrTableDefaults.cpp
@@ -159,19 +159,21 @@ bool WebAssemblyFixBrTableDefaults::runOnMachineFunction(MachineFunction &MF) {
                     << MF.getName() << '\n');
 
   bool Changed = false;
-  SmallPtrSet MBBSet;
+  SetVector,
+            DenseSet, 16>
+      MBBSet;
   for (auto &MBB : MF)
     MBBSet.insert(&MBB);
 
   while (!MBBSet.empty()) {
     MachineBasicBlock *MBB = *MBBSet.begin();
-    MBBSet.erase(MBB);
+    MBBSet.remove(MBB);
     for (auto &MI : *MBB) {
       if (WebAssembly::isBrTable(MI.getOpcode())) {
         fixBrTableIndex(MI, MBB, MF);
         auto *Fixed = fixBrTableDefault(MI, MBB, MF);
         if (Fixed != nullptr) {
-          MBBSet.erase(Fixed);
+          MBBSet.remove(Fixed);
           Changed = true;
         }
         break;
-- 
GitLab


From ea12c1fa15093e24818785b2ca6e06588372a3bf Mon Sep 17 00:00:00 2001
From: sylvain-audi <62035306+sylvain-audi@users.noreply.github.com>
Date: Fri, 8 Mar 2024 12:29:15 -0500
Subject: [PATCH 636/929] [Asan] Add "funclet" OpBundle to generated runtime
 calls if required by EH personality (#82533)

Previously, runtime calls introduced by ASan instrumentation into EH
pads were missing the funclet token expected by WinEHPrepare.
WinEHPrepare would then identify the containing BB as invalid and
discard it, causing invalid code generation that most likely crashes.

Also fixed localescape test, switching its EH personality to match code
without funclets.

This PR is based on the Phabricator patch
https://reviews.llvm.org/D143108

Fixes https://github.com/llvm/llvm-project/issues/64990
---
 .../asan/TestCases/Windows/issue64990.cpp     |  18 +
 .../Instrumentation/AddressSanitizer.cpp      |  62 ++-
 .../AddressSanitizer/asan-funclet.ll          | 527 +++++++++++-------
 .../AddressSanitizer/localescape.ll           |   4 +-
 4 files changed, 402 insertions(+), 209 deletions(-)
 create mode 100644 compiler-rt/test/asan/TestCases/Windows/issue64990.cpp

diff --git a/compiler-rt/test/asan/TestCases/Windows/issue64990.cpp b/compiler-rt/test/asan/TestCases/Windows/issue64990.cpp
new file mode 100644
index 000000000000..a5a46b5a81dd
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/Windows/issue64990.cpp
@@ -0,0 +1,18 @@
+// Repro for the issue #64990: Asan with Windows EH generates __asan_xxx runtime calls without required funclet tokens
+// RUN: %clang_cl_asan %Od %s -EHsc %Fe%t
+// RUN: not %run %t 2>&1 | FileCheck %s
+
+char buff1[6] = "hello";
+char buff2[6] = "hello";
+
+int main(int argc, char **argv) {
+  try {
+    throw 1;
+  } catch (...) {
+    // Make asan generate call to __asan_memcpy inside the EH pad.
+    __builtin_memcpy(buff1, buff2 + 3, 6);
+  }
+  return 0;
+}
+
+// CHECK: SUMMARY: AddressSanitizer: global-buffer-overflow {{.*}} in __asan_memcpy
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index db75eec21a37..26fedbfd65dd 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -43,6 +43,7 @@
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/EHPersonalities.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalAlias.h"
 #include "llvm/IR/GlobalValue.h"
@@ -643,21 +644,70 @@ static uint64_t GetCtorAndDtorPriority(Triple &TargetTriple) {
 
 namespace {
 /// Helper RAII class to post-process inserted asan runtime calls during a
-/// pass on a single Function. This is a no-op implementation, for a first NFC
-/// commit. Coming up: detect and add "funclet" opBundle to function calls that
-/// need them.
+/// pass on a single Function. Upon end of scope, detects and applies the
+/// required funclet OpBundle.
 class RuntimeCallInserter {
   Function *OwnerFn = nullptr;
+  bool TrackInsertedCalls = false;
+  SmallVector InsertedCalls;
 
 public:
-  RuntimeCallInserter(Function &Fn) : OwnerFn(&Fn) {}
+  RuntimeCallInserter(Function &Fn) : OwnerFn(&Fn) {
+    if (Fn.hasPersonalityFn()) {
+      auto Personality = classifyEHPersonality(Fn.getPersonalityFn());
+      if (isScopedEHPersonality(Personality))
+        TrackInsertedCalls = true;
+    }
+  }
+
+  ~RuntimeCallInserter() {
+    if (InsertedCalls.empty())
+      return;
+    assert(TrackInsertedCalls && "Calls were wrongly tracked");
+
+    DenseMap BlockColors = colorEHFunclets(*OwnerFn);
+    for (CallInst *CI : InsertedCalls) {
+      BasicBlock *BB = CI->getParent();
+      assert(BB && "Instruction doesn't belong to a BasicBlock");
+      assert(BB->getParent() == OwnerFn &&
+             "Instruction doesn't belong to the expected Function!");
+
+      ColorVector &Colors = BlockColors[BB];
+      // funclet opbundles are only valid in monochromatic BBs.
+      // Note that unreachable BBs are seen as colorless by colorEHFunclets()
+      // and will be DCE'ed later.
+      if (Colors.empty())
+        continue;
+      if (Colors.size() != 1) {
+        OwnerFn->getContext().emitError(
+            "Instruction's BasicBlock is not monochromatic");
+        continue;
+      }
+
+      BasicBlock *Color = Colors.front();
+      Instruction *EHPad = Color->getFirstNonPHI();
+
+      if (EHPad && EHPad->isEHPad()) {
+        // Replace CI with a clone with an added funclet OperandBundle
+        OperandBundleDef OB("funclet", EHPad);
+        auto *NewCall =
+            CallBase::addOperandBundle(CI, LLVMContext::OB_funclet, OB, CI);
+        NewCall->copyMetadata(*CI);
+        CI->replaceAllUsesWith(NewCall);
+        CI->eraseFromParent();
+      }
+    }
+  }
 
   CallInst *createRuntimeCall(IRBuilder<> &IRB, FunctionCallee Callee,
                               ArrayRef Args = {},
                               const Twine &Name = "") {
     assert(IRB.GetInsertBlock()->getParent() == OwnerFn);
-    (void)OwnerFn;
-    return IRB.CreateCall(Callee, Args, Name, nullptr);
+
+    CallInst *Inst = IRB.CreateCall(Callee, Args, Name, nullptr);
+    if (TrackInsertedCalls)
+      InsertedCalls.push_back(Inst);
+    return Inst;
   }
 };
 
diff --git a/llvm/test/Instrumentation/AddressSanitizer/asan-funclet.ll b/llvm/test/Instrumentation/AddressSanitizer/asan-funclet.ll
index f0a5c67365ab..6d40e2b7eaf9 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/asan-funclet.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/asan-funclet.ll
@@ -1,8 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
 
-; Warning! The output of this test is currently invalid.
-; It serves as a base for the bugfix patch to highlight the modified generated code.
-
 ; Test appropriate tagging of funclet for function calls generated by asan.
 ; RUN: opt -S -passes=asan,win-eh-prepare -asan-use-stack-safety=0 -asan-max-inline-poisoning-size=0 \
 ; RUN:   -asan-detect-invalid-pointer-cmp -asan-detect-invalid-pointer-sub -asan-use-after-scope < %s | FileCheck %s --check-prefixes=CHECK,CHECK-INLINE
@@ -53,137 +50,221 @@ define void @FuncletPersonality(ptr %ptrParam) sanitize_address personality ptr
 ; CHECK-INLINE-NEXT:    store i64 [[TMP12]], ptr [[ASAN_LOCAL_STACK_BASE]], align 8
 ; CHECK-INLINE-NEXT:    [[TMP13:%.*]] = add i64 [[TMP12]], 32
 ; CHECK-INLINE-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[TMP13]] to ptr
-; CHECK-INLINE-NEXT:    [[TMP15:%.*]] = add i64 [[TMP12]], 8528
+; CHECK-INLINE-NEXT:    [[TMP15:%.*]] = add i64 [[TMP12]], 8480
 ; CHECK-INLINE-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
-; CHECK-INLINE-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[TMP12]] to ptr
-; CHECK-INLINE-NEXT:    store i64 1102416563, ptr [[TMP17]], align 8
-; CHECK-INLINE-NEXT:    [[TMP18:%.*]] = add i64 [[TMP12]], 8
-; CHECK-INLINE-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
-; CHECK-INLINE-NEXT:    store i64 ptrtoint (ptr @___asan_gen_ to i64), ptr [[TMP19]], align 8
-; CHECK-INLINE-NEXT:    [[TMP20:%.*]] = add i64 [[TMP12]], 16
-; CHECK-INLINE-NEXT:    [[TMP21:%.*]] = inttoptr i64 [[TMP20]] to ptr
-; CHECK-INLINE-NEXT:    store i64 ptrtoint (ptr @FuncletPersonality to i64), ptr [[TMP21]], align 8
-; CHECK-INLINE-NEXT:    [[TMP22:%.*]] = lshr i64 [[TMP12]], 3
-; CHECK-INLINE-NEXT:    [[TMP23:%.*]] = add i64 [[TMP22]], [[TMP1]]
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f1(i64 [[TMP23]], i64 4)
-; CHECK-INLINE-NEXT:    [[TMP24:%.*]] = add i64 [[TMP23]], 1028
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP24]], i64 32)
-; CHECK-INLINE-NEXT:    [[TMP25:%.*]] = add i64 [[TMP23]], 1060
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP25]], i64 1)
-; CHECK-INLINE-NEXT:    [[TMP26:%.*]] = add i64 [[TMP23]], 1061
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP26]], i64 1)
-; CHECK-INLINE-NEXT:    [[TMP27:%.*]] = add i64 [[TMP23]], 1062
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP27]], i64 1)
-; CHECK-INLINE-NEXT:    [[TMP28:%.*]] = add i64 [[TMP23]], 1063
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP28]], i64 1)
-; CHECK-INLINE-NEXT:    [[TMP29:%.*]] = add i64 [[TMP23]], 1064
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP29]], i64 1)
-; CHECK-INLINE-NEXT:    [[TMP30:%.*]] = add i64 [[TMP23]], 1065
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP30]], i64 1)
-; CHECK-INLINE-NEXT:    [[TMP31:%.*]] = add i64 [[TMP23]], 1066
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP31]], i64 1)
-; CHECK-INLINE-NEXT:    [[TMP32:%.*]] = add i64 [[TMP23]], 1067
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f3(i64 [[TMP32]], i64 1)
-; CHECK-INLINE-NEXT:    [[TMP33:%.*]] = add i64 [[TMP23]], 1066
+; CHECK-INLINE-NEXT:    [[TMP17:%.*]] = add i64 [[TMP12]], 8496
+; CHECK-INLINE-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP19:%.*]] = add i64 [[TMP12]], 8512
+; CHECK-INLINE-NEXT:    [[TMP20:%.*]] = inttoptr i64 [[TMP19]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP21:%.*]] = add i64 [[TMP12]], 8528
+; CHECK-INLINE-NEXT:    [[TMP22:%.*]] = inttoptr i64 [[TMP21]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-INLINE-NEXT:    store i64 1102416563, ptr [[TMP23]], align 8
+; CHECK-INLINE-NEXT:    [[TMP24:%.*]] = add i64 [[TMP12]], 8
+; CHECK-INLINE-NEXT:    [[TMP25:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; CHECK-INLINE-NEXT:    store i64 ptrtoint (ptr @___asan_gen_ to i64), ptr [[TMP25]], align 8
+; CHECK-INLINE-NEXT:    [[TMP26:%.*]] = add i64 [[TMP12]], 16
+; CHECK-INLINE-NEXT:    [[TMP27:%.*]] = inttoptr i64 [[TMP26]] to ptr
+; CHECK-INLINE-NEXT:    store i64 ptrtoint (ptr @FuncletPersonality to i64), ptr [[TMP27]], align 8
+; CHECK-INLINE-NEXT:    [[TMP28:%.*]] = lshr i64 [[TMP12]], 3
+; CHECK-INLINE-NEXT:    [[TMP29:%.*]] = add i64 [[TMP28]], [[TMP1]]
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f1(i64 [[TMP29]], i64 4)
+; CHECK-INLINE-NEXT:    [[TMP30:%.*]] = add i64 [[TMP29]], 1028
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP30]], i64 32)
+; CHECK-INLINE-NEXT:    [[TMP31:%.*]] = add i64 [[TMP29]], 1060
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP31]], i64 1)
+; CHECK-INLINE-NEXT:    [[TMP32:%.*]] = add i64 [[TMP29]], 1061
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP32]], i64 1)
+; CHECK-INLINE-NEXT:    [[TMP33:%.*]] = add i64 [[TMP29]], 1062
 ; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP33]], i64 1)
-; CHECK-INLINE-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[TMP16]])
-; CHECK-INLINE-NEXT:    [[TMP34:%.*]] = lshr i64 [[TMP15]], 3
-; CHECK-INLINE-NEXT:    [[TMP35:%.*]] = add i64 [[TMP34]], [[TMP1]]
-; CHECK-INLINE-NEXT:    [[TMP36:%.*]] = inttoptr i64 [[TMP35]] to ptr
-; CHECK-INLINE-NEXT:    [[TMP37:%.*]] = load i8, ptr [[TMP36]], align 1
-; CHECK-INLINE-NEXT:    [[TMP38:%.*]] = icmp ne i8 [[TMP37]], 0
-; CHECK-INLINE-NEXT:    br i1 [[TMP38]], label [[TMP39:%.*]], label [[TMP44:%.*]], !prof [[PROF0:![0-9]+]]
-; CHECK-INLINE:       39:
-; CHECK-INLINE-NEXT:    [[TMP40:%.*]] = and i64 [[TMP15]], 7
-; CHECK-INLINE-NEXT:    [[TMP41:%.*]] = trunc i64 [[TMP40]] to i8
-; CHECK-INLINE-NEXT:    [[TMP42:%.*]] = icmp sge i8 [[TMP41]], [[TMP37]]
-; CHECK-INLINE-NEXT:    br i1 [[TMP42]], label [[TMP43:%.*]], label [[TMP44]]
-; CHECK-INLINE:       43:
-; CHECK-INLINE-NEXT:    call void @__asan_report_store1(i64 [[TMP15]]) #[[ATTR8:[0-9]+]]
+; CHECK-INLINE-NEXT:    [[TMP34:%.*]] = add i64 [[TMP29]], 1063
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP34]], i64 1)
+; CHECK-INLINE-NEXT:    [[TMP35:%.*]] = add i64 [[TMP29]], 1064
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP35]], i64 1)
+; CHECK-INLINE-NEXT:    [[TMP36:%.*]] = add i64 [[TMP29]], 1065
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP36]], i64 1)
+; CHECK-INLINE-NEXT:    [[TMP37:%.*]] = add i64 [[TMP29]], 1066
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP37]], i64 1)
+; CHECK-INLINE-NEXT:    [[TMP38:%.*]] = add i64 [[TMP29]], 1067
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f3(i64 [[TMP38]], i64 1)
+; CHECK-INLINE-NEXT:    [[TMP39:%.*]] = add i64 [[TMP29]], 1066
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP39]], i64 1)
+; CHECK-INLINE-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[TMP22]])
+; CHECK-INLINE-NEXT:    [[TMP40:%.*]] = lshr i64 [[TMP21]], 3
+; CHECK-INLINE-NEXT:    [[TMP41:%.*]] = add i64 [[TMP40]], [[TMP1]]
+; CHECK-INLINE-NEXT:    [[TMP42:%.*]] = inttoptr i64 [[TMP41]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP43:%.*]] = load i8, ptr [[TMP42]], align 1
+; CHECK-INLINE-NEXT:    [[TMP44:%.*]] = icmp ne i8 [[TMP43]], 0
+; CHECK-INLINE-NEXT:    br i1 [[TMP44]], label [[TMP45:%.*]], label [[TMP50:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK-INLINE:       45:
+; CHECK-INLINE-NEXT:    [[TMP46:%.*]] = and i64 [[TMP21]], 7
+; CHECK-INLINE-NEXT:    [[TMP47:%.*]] = trunc i64 [[TMP46]] to i8
+; CHECK-INLINE-NEXT:    [[TMP48:%.*]] = icmp sge i8 [[TMP47]], [[TMP43]]
+; CHECK-INLINE-NEXT:    br i1 [[TMP48]], label [[TMP49:%.*]], label [[TMP50]]
+; CHECK-INLINE:       49:
+; CHECK-INLINE-NEXT:    call void @__asan_report_store1(i64 [[TMP21]]) #[[ATTR8:[0-9]+]]
 ; CHECK-INLINE-NEXT:    unreachable
-; CHECK-INLINE:       44:
-; CHECK-INLINE-NEXT:    store volatile i8 0, ptr [[TMP16]], align 1
-; CHECK-INLINE-NEXT:    [[TMP45:%.*]] = add i64 [[TMP23]], 1066
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP45]], i64 1)
-; CHECK-INLINE-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP16]])
-; CHECK-INLINE-NEXT:    [[TMP46:%.*]] = alloca i8, i64 96, align 32
-; CHECK-INLINE-NEXT:    [[TMP47:%.*]] = ptrtoint ptr [[TMP46]] to i64
-; CHECK-INLINE-NEXT:    [[TMP48:%.*]] = add i64 [[TMP47]], 32
-; CHECK-INLINE-NEXT:    call void @__asan_alloca_poison(i64 [[TMP48]], i64 4)
-; CHECK-INLINE-NEXT:    [[TMP49:%.*]] = ptrtoint ptr [[TMP46]] to i64
-; CHECK-INLINE-NEXT:    store i64 [[TMP49]], ptr [[TMP0]], align 8
-; CHECK-INLINE-NEXT:    [[TMP50:%.*]] = inttoptr i64 [[TMP48]] to ptr
-; CHECK-INLINE-NEXT:    [[TMP51:%.*]] = alloca i8, i64 96, align 32
-; CHECK-INLINE-NEXT:    [[TMP52:%.*]] = ptrtoint ptr [[TMP51]] to i64
-; CHECK-INLINE-NEXT:    [[TMP53:%.*]] = add i64 [[TMP52]], 32
-; CHECK-INLINE-NEXT:    call void @__asan_alloca_poison(i64 [[TMP53]], i64 8)
-; CHECK-INLINE-NEXT:    [[TMP54:%.*]] = ptrtoint ptr [[TMP51]] to i64
-; CHECK-INLINE-NEXT:    store i64 [[TMP54]], ptr [[TMP0]], align 8
-; CHECK-INLINE-NEXT:    [[TMP55:%.*]] = inttoptr i64 [[TMP53]] to ptr
-; CHECK-INLINE-NEXT:    [[TMP56:%.*]] = lshr i64 [[TMP53]], 3
-; CHECK-INLINE-NEXT:    [[TMP57:%.*]] = add i64 [[TMP56]], [[TMP1]]
-; CHECK-INLINE-NEXT:    [[TMP58:%.*]] = inttoptr i64 [[TMP57]] to ptr
-; CHECK-INLINE-NEXT:    [[TMP59:%.*]] = load i8, ptr [[TMP58]], align 1
-; CHECK-INLINE-NEXT:    [[TMP60:%.*]] = icmp ne i8 [[TMP59]], 0
-; CHECK-INLINE-NEXT:    br i1 [[TMP60]], label [[TMP61:%.*]], label [[TMP62:%.*]]
-; CHECK-INLINE:       61:
-; CHECK-INLINE-NEXT:    call void @__asan_report_store8(i64 [[TMP53]]) #[[ATTR8]]
+; CHECK-INLINE:       50:
+; CHECK-INLINE-NEXT:    store volatile i8 0, ptr [[TMP22]], align 1
+; CHECK-INLINE-NEXT:    [[TMP51:%.*]] = add i64 [[TMP29]], 1066
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP51]], i64 1)
+; CHECK-INLINE-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP22]])
+; CHECK-INLINE-NEXT:    [[TMP52:%.*]] = alloca i8, i64 96, align 32
+; CHECK-INLINE-NEXT:    [[TMP53:%.*]] = ptrtoint ptr [[TMP52]] to i64
+; CHECK-INLINE-NEXT:    [[TMP54:%.*]] = add i64 [[TMP53]], 32
+; CHECK-INLINE-NEXT:    call void @__asan_alloca_poison(i64 [[TMP54]], i64 4)
+; CHECK-INLINE-NEXT:    [[TMP55:%.*]] = ptrtoint ptr [[TMP52]] to i64
+; CHECK-INLINE-NEXT:    store i64 [[TMP55]], ptr [[TMP0]], align 8
+; CHECK-INLINE-NEXT:    [[TMP56:%.*]] = inttoptr i64 [[TMP54]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP57:%.*]] = alloca i8, i64 96, align 32
+; CHECK-INLINE-NEXT:    [[TMP58:%.*]] = ptrtoint ptr [[TMP57]] to i64
+; CHECK-INLINE-NEXT:    [[TMP59:%.*]] = add i64 [[TMP58]], 32
+; CHECK-INLINE-NEXT:    call void @__asan_alloca_poison(i64 [[TMP59]], i64 8)
+; CHECK-INLINE-NEXT:    [[TMP60:%.*]] = ptrtoint ptr [[TMP57]] to i64
+; CHECK-INLINE-NEXT:    store i64 [[TMP60]], ptr [[TMP0]], align 8
+; CHECK-INLINE-NEXT:    [[TMP61:%.*]] = inttoptr i64 [[TMP59]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP62:%.*]] = lshr i64 [[TMP59]], 3
+; CHECK-INLINE-NEXT:    [[TMP63:%.*]] = add i64 [[TMP62]], [[TMP1]]
+; CHECK-INLINE-NEXT:    [[TMP64:%.*]] = inttoptr i64 [[TMP63]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP65:%.*]] = load i8, ptr [[TMP64]], align 1
+; CHECK-INLINE-NEXT:    [[TMP66:%.*]] = icmp ne i8 [[TMP65]], 0
+; CHECK-INLINE-NEXT:    br i1 [[TMP66]], label [[TMP67:%.*]], label [[TMP68:%.*]]
+; CHECK-INLINE:       67:
+; CHECK-INLINE-NEXT:    call void @__asan_report_store8(i64 [[TMP59]]) #[[ATTR8]]
 ; CHECK-INLINE-NEXT:    unreachable
-; CHECK-INLINE:       62:
-; CHECK-INLINE-NEXT:    store volatile i64 0, ptr [[TMP55]], align 8
-; CHECK-INLINE-NEXT:    [[TMPCOPYI64:%.*]] = load i64, ptr [[TMP55]], align 8
-; CHECK-INLINE-NEXT:    [[TMP63:%.*]] = and i64 [[TMPCOPYI64]], 31
-; CHECK-INLINE-NEXT:    [[TMP64:%.*]] = sub i64 32, [[TMP63]]
-; CHECK-INLINE-NEXT:    [[TMP65:%.*]] = icmp ne i64 [[TMP64]], 32
-; CHECK-INLINE-NEXT:    [[TMP66:%.*]] = select i1 [[TMP65]], i64 [[TMP64]], i64 0
-; CHECK-INLINE-NEXT:    [[TMP67:%.*]] = add i64 64, [[TMP66]]
-; CHECK-INLINE-NEXT:    [[TMP68:%.*]] = add i64 [[TMPCOPYI64]], [[TMP67]]
-; CHECK-INLINE-NEXT:    [[TMP69:%.*]] = alloca i8, i64 [[TMP68]], align 32
-; CHECK-INLINE-NEXT:    [[TMP70:%.*]] = ptrtoint ptr [[TMP69]] to i64
-; CHECK-INLINE-NEXT:    [[TMP71:%.*]] = add i64 [[TMP70]], 32
-; CHECK-INLINE-NEXT:    call void @__asan_alloca_poison(i64 [[TMP71]], i64 [[TMPCOPYI64]])
-; CHECK-INLINE-NEXT:    [[TMP72:%.*]] = ptrtoint ptr [[TMP69]] to i64
-; CHECK-INLINE-NEXT:    store i64 [[TMP72]], ptr [[TMP0]], align 8
-; CHECK-INLINE-NEXT:    [[TMP73:%.*]] = inttoptr i64 [[TMP71]] to ptr
-; CHECK-INLINE-NEXT:    [[TMP74:%.*]] = lshr i64 [[TMP71]], 3
-; CHECK-INLINE-NEXT:    [[TMP75:%.*]] = add i64 [[TMP74]], [[TMP1]]
-; CHECK-INLINE-NEXT:    [[TMP76:%.*]] = inttoptr i64 [[TMP75]] to ptr
-; CHECK-INLINE-NEXT:    [[TMP77:%.*]] = load i8, ptr [[TMP76]], align 1
-; CHECK-INLINE-NEXT:    [[TMP78:%.*]] = icmp ne i8 [[TMP77]], 0
-; CHECK-INLINE-NEXT:    br i1 [[TMP78]], label [[TMP79:%.*]], label [[TMP84:%.*]], !prof [[PROF0]]
-; CHECK-INLINE:       79:
-; CHECK-INLINE-NEXT:    [[TMP80:%.*]] = and i64 [[TMP71]], 7
-; CHECK-INLINE-NEXT:    [[TMP81:%.*]] = trunc i64 [[TMP80]] to i8
-; CHECK-INLINE-NEXT:    [[TMP82:%.*]] = icmp sge i8 [[TMP81]], [[TMP77]]
-; CHECK-INLINE-NEXT:    br i1 [[TMP82]], label [[TMP83:%.*]], label [[TMP84]]
-; CHECK-INLINE:       83:
-; CHECK-INLINE-NEXT:    call void @__asan_report_store1(i64 [[TMP71]]) #[[ATTR8]]
+; CHECK-INLINE:       68:
+; CHECK-INLINE-NEXT:    store volatile i64 0, ptr [[TMP61]], align 8
+; CHECK-INLINE-NEXT:    [[TMPCOPYI64:%.*]] = load i64, ptr [[TMP61]], align 8
+; CHECK-INLINE-NEXT:    [[TMP69:%.*]] = and i64 [[TMPCOPYI64]], 31
+; CHECK-INLINE-NEXT:    [[TMP70:%.*]] = sub i64 32, [[TMP69]]
+; CHECK-INLINE-NEXT:    [[TMP71:%.*]] = icmp ne i64 [[TMP70]], 32
+; CHECK-INLINE-NEXT:    [[TMP72:%.*]] = select i1 [[TMP71]], i64 [[TMP70]], i64 0
+; CHECK-INLINE-NEXT:    [[TMP73:%.*]] = add i64 64, [[TMP72]]
+; CHECK-INLINE-NEXT:    [[TMP74:%.*]] = add i64 [[TMPCOPYI64]], [[TMP73]]
+; CHECK-INLINE-NEXT:    [[TMP75:%.*]] = alloca i8, i64 [[TMP74]], align 32
+; CHECK-INLINE-NEXT:    [[TMP76:%.*]] = ptrtoint ptr [[TMP75]] to i64
+; CHECK-INLINE-NEXT:    [[TMP77:%.*]] = add i64 [[TMP76]], 32
+; CHECK-INLINE-NEXT:    call void @__asan_alloca_poison(i64 [[TMP77]], i64 [[TMPCOPYI64]])
+; CHECK-INLINE-NEXT:    [[TMP78:%.*]] = ptrtoint ptr [[TMP75]] to i64
+; CHECK-INLINE-NEXT:    store i64 [[TMP78]], ptr [[TMP0]], align 8
+; CHECK-INLINE-NEXT:    [[TMP79:%.*]] = inttoptr i64 [[TMP77]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP80:%.*]] = lshr i64 [[TMP77]], 3
+; CHECK-INLINE-NEXT:    [[TMP81:%.*]] = add i64 [[TMP80]], [[TMP1]]
+; CHECK-INLINE-NEXT:    [[TMP82:%.*]] = inttoptr i64 [[TMP81]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP83:%.*]] = load i8, ptr [[TMP82]], align 1
+; CHECK-INLINE-NEXT:    [[TMP84:%.*]] = icmp ne i8 [[TMP83]], 0
+; CHECK-INLINE-NEXT:    br i1 [[TMP84]], label [[TMP85:%.*]], label [[TMP90:%.*]], !prof [[PROF0]]
+; CHECK-INLINE:       85:
+; CHECK-INLINE-NEXT:    [[TMP86:%.*]] = and i64 [[TMP77]], 7
+; CHECK-INLINE-NEXT:    [[TMP87:%.*]] = trunc i64 [[TMP86]] to i8
+; CHECK-INLINE-NEXT:    [[TMP88:%.*]] = icmp sge i8 [[TMP87]], [[TMP83]]
+; CHECK-INLINE-NEXT:    br i1 [[TMP88]], label [[TMP89:%.*]], label [[TMP90]]
+; CHECK-INLINE:       89:
+; CHECK-INLINE-NEXT:    call void @__asan_report_store1(i64 [[TMP77]]) #[[ATTR8]]
 ; CHECK-INLINE-NEXT:    unreachable
-; CHECK-INLINE:       84:
-; CHECK-INLINE-NEXT:    store volatile i8 0, ptr [[TMP73]], align 1
+; CHECK-INLINE:       90:
+; CHECK-INLINE-NEXT:    store volatile i8 0, ptr [[TMP79]], align 1
 ; CHECK-INLINE-NEXT:    invoke void @MayThrowFunc()
 ; CHECK-INLINE-NEXT:            to label [[INVOKE_CONT:%.*]] unwind label [[EHCLEANUP:%.*]]
 ; CHECK-INLINE:       invoke.cont:
 ; CHECK-INLINE-NEXT:    call void @DeInit(ptr [[TMP14]])
-; CHECK-INLINE-NEXT:    [[TMP85:%.*]] = ptrtoint ptr [[TMP0]] to i64
-; CHECK-INLINE-NEXT:    [[TMP86:%.*]] = load i64, ptr [[TMP0]], align 8
-; CHECK-INLINE-NEXT:    call void @__asan_allocas_unpoison(i64 [[TMP86]], i64 [[TMP85]])
-; CHECK-INLINE-NEXT:    store i64 1172321806, ptr [[TMP17]], align 8
-; CHECK-INLINE-NEXT:    [[TMP87:%.*]] = icmp ne i64 [[TMP7]], 0
-; CHECK-INLINE-NEXT:    br i1 [[TMP87]], label [[TMP88:%.*]], label [[TMP89:%.*]]
-; CHECK-INLINE:       88:
+; CHECK-INLINE-NEXT:    [[TMP91:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; CHECK-INLINE-NEXT:    [[TMP92:%.*]] = load i64, ptr [[TMP0]], align 8
+; CHECK-INLINE-NEXT:    call void @__asan_allocas_unpoison(i64 [[TMP92]], i64 [[TMP91]])
+; CHECK-INLINE-NEXT:    store i64 1172321806, ptr [[TMP23]], align 8
+; CHECK-INLINE-NEXT:    [[TMP93:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-INLINE-NEXT:    br i1 [[TMP93]], label [[TMP94:%.*]], label [[TMP95:%.*]]
+; CHECK-INLINE:       94:
 ; CHECK-INLINE-NEXT:    call void @__asan_stack_free_8(i64 [[TMP7]], i64 8544)
-; CHECK-INLINE-NEXT:    br label [[TMP91:%.*]]
-; CHECK-INLINE:       89:
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP23]], i64 4)
-; CHECK-INLINE-NEXT:    [[TMP90:%.*]] = add i64 [[TMP23]], 1028
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP90]], i64 40)
-; CHECK-INLINE-NEXT:    br label [[TMP91]]
-; CHECK-INLINE:       91:
+; CHECK-INLINE-NEXT:    br label [[TMP97:%.*]]
+; CHECK-INLINE:       95:
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP29]], i64 4)
+; CHECK-INLINE-NEXT:    [[TMP96:%.*]] = add i64 [[TMP29]], 1028
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP96]], i64 40)
+; CHECK-INLINE-NEXT:    br label [[TMP97]]
+; CHECK-INLINE:       97:
 ; CHECK-INLINE-NEXT:    ret void
 ; CHECK-INLINE:       ehcleanup:
-; CHECK-INLINE-NEXT:    [[TMP92:%.*]] = cleanuppad within none []
+; CHECK-INLINE-NEXT:    [[TMP98:%.*]] = cleanuppad within none []
+; CHECK-INLINE-NEXT:    call void @__asan_unpoison_stack_memory(i64 [[TMP54]], i64 4) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[TMP56]])
+; CHECK-INLINE-NEXT:    [[TMP99:%.*]] = lshr i64 [[TMP54]], 3
+; CHECK-INLINE-NEXT:    [[TMP100:%.*]] = add i64 [[TMP99]], [[TMP1]]
+; CHECK-INLINE-NEXT:    [[TMP101:%.*]] = inttoptr i64 [[TMP100]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP102:%.*]] = load i8, ptr [[TMP101]], align 1
+; CHECK-INLINE-NEXT:    [[TMP103:%.*]] = icmp ne i8 [[TMP102]], 0
+; CHECK-INLINE-NEXT:    br i1 [[TMP103]], label [[TMP104:%.*]], label [[TMP109:%.*]], !prof [[PROF0]]
+; CHECK-INLINE:       104:
+; CHECK-INLINE-NEXT:    [[TMP105:%.*]] = and i64 [[TMP54]], 7
+; CHECK-INLINE-NEXT:    [[TMP106:%.*]] = trunc i64 [[TMP105]] to i8
+; CHECK-INLINE-NEXT:    [[TMP107:%.*]] = icmp sge i8 [[TMP106]], [[TMP102]]
+; CHECK-INLINE-NEXT:    br i1 [[TMP107]], label [[TMP108:%.*]], label [[TMP109]]
+; CHECK-INLINE:       108:
+; CHECK-INLINE-NEXT:    call void @__asan_report_store1(i64 [[TMP54]]) #[[ATTR8]] [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    unreachable
+; CHECK-INLINE:       109:
+; CHECK-INLINE-NEXT:    store volatile i8 0, ptr [[TMP56]], align 1
+; CHECK-INLINE-NEXT:    call void @__asan_poison_stack_memory(i64 [[TMP54]], i64 4) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP56]])
+; CHECK-INLINE-NEXT:    call void @DeInit(ptr [[TMP14]]) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    [[TMP110:%.*]] = call ptr @__asan_memset(ptr [[TMP16]], i32 0, i64 4) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    [[TMP111:%.*]] = call ptr @__asan_memcpy(ptr [[TMP18]], ptr [[TMP16]], i64 4) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    [[TMP112:%.*]] = call ptr @__asan_memmove(ptr [[TMP20]], ptr [[TMP16]], i64 4) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    call void @__sanitizer_ptr_cmp(i64 [[TMP15]], i64 [[TMP17]]) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    call void @__sanitizer_ptr_sub(i64 [[TMP15]], i64 [[TMP17]]) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    [[TMP113:%.*]] = ptrtoint ptr [[PTRPARAM]] to i64
+; CHECK-INLINE-NEXT:    [[TMP114:%.*]] = add i64 [[TMP113]], 7
+; CHECK-INLINE-NEXT:    [[TMP115:%.*]] = inttoptr i64 [[TMP114]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP116:%.*]] = ptrtoint ptr [[PTRPARAM]] to i64
+; CHECK-INLINE-NEXT:    [[TMP117:%.*]] = lshr i64 [[TMP116]], 3
+; CHECK-INLINE-NEXT:    [[TMP118:%.*]] = add i64 [[TMP117]], [[TMP1]]
+; CHECK-INLINE-NEXT:    [[TMP119:%.*]] = inttoptr i64 [[TMP118]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP120:%.*]] = load i8, ptr [[TMP119]], align 1
+; CHECK-INLINE-NEXT:    [[TMP121:%.*]] = icmp ne i8 [[TMP120]], 0
+; CHECK-INLINE-NEXT:    br i1 [[TMP121]], label [[TMP122:%.*]], label [[TMP127:%.*]], !prof [[PROF0]]
+; CHECK-INLINE:       122:
+; CHECK-INLINE-NEXT:    [[TMP123:%.*]] = and i64 [[TMP116]], 7
+; CHECK-INLINE-NEXT:    [[TMP124:%.*]] = trunc i64 [[TMP123]] to i8
+; CHECK-INLINE-NEXT:    [[TMP125:%.*]] = icmp sge i8 [[TMP124]], [[TMP120]]
+; CHECK-INLINE-NEXT:    br i1 [[TMP125]], label [[TMP126:%.*]], label [[TMP127]]
+; CHECK-INLINE:       126:
+; CHECK-INLINE-NEXT:    call void @__asan_report_store_n(i64 [[TMP116]], i64 8) #[[ATTR8]] [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    unreachable
+; CHECK-INLINE:       127:
+; CHECK-INLINE-NEXT:    [[TMP128:%.*]] = lshr i64 [[TMP114]], 3
+; CHECK-INLINE-NEXT:    [[TMP129:%.*]] = add i64 [[TMP128]], [[TMP1]]
+; CHECK-INLINE-NEXT:    [[TMP130:%.*]] = inttoptr i64 [[TMP129]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP131:%.*]] = load i8, ptr [[TMP130]], align 1
+; CHECK-INLINE-NEXT:    [[TMP132:%.*]] = icmp ne i8 [[TMP131]], 0
+; CHECK-INLINE-NEXT:    br i1 [[TMP132]], label [[TMP133:%.*]], label [[EHEXIT:%.*]], !prof [[PROF0]]
+; CHECK-INLINE:       133:
+; CHECK-INLINE-NEXT:    [[TMP134:%.*]] = and i64 [[TMP114]], 7
+; CHECK-INLINE-NEXT:    [[TMP135:%.*]] = trunc i64 [[TMP134]] to i8
+; CHECK-INLINE-NEXT:    [[TMP136:%.*]] = icmp sge i8 [[TMP135]], [[TMP131]]
+; CHECK-INLINE-NEXT:    br i1 [[TMP136]], label [[TMP137:%.*]], label [[EHEXIT]]
+; CHECK-INLINE:       137:
+; CHECK-INLINE-NEXT:    call void @__asan_report_store_n(i64 [[TMP114]], i64 8) #[[ATTR8]] [ "funclet"(token [[TMP98]]) ]
 ; CHECK-INLINE-NEXT:    unreachable
+; CHECK-INLINE:       ehexit:
+; CHECK-INLINE-NEXT:    store i64 0, ptr [[PTRPARAM]], align 1
+; CHECK-INLINE-NEXT:    [[TMP138:%.*]] = call i64 @llvm.get.dynamic.area.offset.i64()
+; CHECK-INLINE-NEXT:    [[TMP139:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; CHECK-INLINE-NEXT:    [[TMP140:%.*]] = add i64 [[TMP139]], [[TMP138]]
+; CHECK-INLINE-NEXT:    [[TMP141:%.*]] = load i64, ptr [[TMP0]], align 8
+; CHECK-INLINE-NEXT:    call void @__asan_allocas_unpoison(i64 [[TMP141]], i64 [[TMP140]]) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    store i64 1172321806, ptr [[TMP23]], align 8
+; CHECK-INLINE-NEXT:    [[TMP142:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-INLINE-NEXT:    br i1 [[TMP142]], label [[TMP143:%.*]], label [[TMP144:%.*]]
+; CHECK-INLINE:       143:
+; CHECK-INLINE-NEXT:    call void @__asan_stack_free_8(i64 [[TMP7]], i64 8544) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    br label [[TMP146:%.*]]
+; CHECK-INLINE:       144:
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP29]], i64 4) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    [[TMP145:%.*]] = add i64 [[TMP29]], 1028
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP145]], i64 40) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    br label [[TMP146]]
+; CHECK-INLINE:       146:
+; CHECK-INLINE-NEXT:    cleanupret from [[TMP98]] unwind to caller
 ;
 ; CHECK-OUTLINE-LABEL: define void @FuncletPersonality(
 ; CHECK-OUTLINE-SAME: ptr [[PTRPARAM:%.*]]) #[[ATTR4:[0-9]+]] personality ptr @__CxxFrameHandler3 {
@@ -211,96 +292,140 @@ define void @FuncletPersonality(ptr %ptrParam) sanitize_address personality ptr
 ; CHECK-OUTLINE-NEXT:    store i64 [[TMP12]], ptr [[ASAN_LOCAL_STACK_BASE]], align 8
 ; CHECK-OUTLINE-NEXT:    [[TMP13:%.*]] = add i64 [[TMP12]], 32
 ; CHECK-OUTLINE-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[TMP13]] to ptr
-; CHECK-OUTLINE-NEXT:    [[TMP15:%.*]] = add i64 [[TMP12]], 8528
+; CHECK-OUTLINE-NEXT:    [[TMP15:%.*]] = add i64 [[TMP12]], 8480
 ; CHECK-OUTLINE-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
-; CHECK-OUTLINE-NEXT:    [[TMP17:%.*]] = add i64 [[TMP12]], 8560
+; CHECK-OUTLINE-NEXT:    [[TMP17:%.*]] = add i64 [[TMP12]], 8496
 ; CHECK-OUTLINE-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
-; CHECK-OUTLINE-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP12]] to ptr
-; CHECK-OUTLINE-NEXT:    store i64 1102416563, ptr [[TMP19]], align 8
-; CHECK-OUTLINE-NEXT:    [[TMP20:%.*]] = add i64 [[TMP12]], 8
-; CHECK-OUTLINE-NEXT:    [[TMP21:%.*]] = inttoptr i64 [[TMP20]] to ptr
-; CHECK-OUTLINE-NEXT:    store i64 ptrtoint (ptr @___asan_gen_ to i64), ptr [[TMP21]], align 8
-; CHECK-OUTLINE-NEXT:    [[TMP22:%.*]] = add i64 [[TMP12]], 16
-; CHECK-OUTLINE-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
-; CHECK-OUTLINE-NEXT:    store i64 ptrtoint (ptr @FuncletPersonality to i64), ptr [[TMP23]], align 8
-; CHECK-OUTLINE-NEXT:    [[TMP24:%.*]] = lshr i64 [[TMP12]], 3
-; CHECK-OUTLINE-NEXT:    [[TMP25:%.*]] = add i64 [[TMP24]], [[TMP1]]
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f1(i64 [[TMP25]], i64 4)
-; CHECK-OUTLINE-NEXT:    [[TMP26:%.*]] = add i64 [[TMP25]], 1028
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP26]], i64 32)
-; CHECK-OUTLINE-NEXT:    [[TMP27:%.*]] = add i64 [[TMP25]], 1060
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP27]], i64 1)
-; CHECK-OUTLINE-NEXT:    [[TMP28:%.*]] = add i64 [[TMP25]], 1061
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP28]], i64 1)
-; CHECK-OUTLINE-NEXT:    [[TMP29:%.*]] = add i64 [[TMP25]], 1062
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP29]], i64 1)
-; CHECK-OUTLINE-NEXT:    [[TMP30:%.*]] = add i64 [[TMP25]], 1063
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP30]], i64 1)
-; CHECK-OUTLINE-NEXT:    [[TMP31:%.*]] = add i64 [[TMP25]], 1064
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP31]], i64 1)
-; CHECK-OUTLINE-NEXT:    [[TMP32:%.*]] = add i64 [[TMP25]], 1065
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP32]], i64 1)
-; CHECK-OUTLINE-NEXT:    [[TMP33:%.*]] = add i64 [[TMP25]], 1066
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP33]], i64 1)
-; CHECK-OUTLINE-NEXT:    [[TMP34:%.*]] = add i64 [[TMP25]], 1067
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP34]], i64 1)
-; CHECK-OUTLINE-NEXT:    [[TMP35:%.*]] = add i64 [[TMP25]], 1068
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP35]], i64 1)
-; CHECK-OUTLINE-NEXT:    [[TMP36:%.*]] = add i64 [[TMP25]], 1069
+; CHECK-OUTLINE-NEXT:    [[TMP19:%.*]] = add i64 [[TMP12]], 8512
+; CHECK-OUTLINE-NEXT:    [[TMP20:%.*]] = inttoptr i64 [[TMP19]] to ptr
+; CHECK-OUTLINE-NEXT:    [[TMP21:%.*]] = add i64 [[TMP12]], 8528
+; CHECK-OUTLINE-NEXT:    [[TMP22:%.*]] = inttoptr i64 [[TMP21]] to ptr
+; CHECK-OUTLINE-NEXT:    [[TMP23:%.*]] = add i64 [[TMP12]], 8544
+; CHECK-OUTLINE-NEXT:    [[TMP24:%.*]] = inttoptr i64 [[TMP23]] to ptr
+; CHECK-OUTLINE-NEXT:    [[TMP25:%.*]] = add i64 [[TMP12]], 8560
+; CHECK-OUTLINE-NEXT:    [[TMP26:%.*]] = inttoptr i64 [[TMP25]] to ptr
+; CHECK-OUTLINE-NEXT:    [[TMP27:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-OUTLINE-NEXT:    store i64 1102416563, ptr [[TMP27]], align 8
+; CHECK-OUTLINE-NEXT:    [[TMP28:%.*]] = add i64 [[TMP12]], 8
+; CHECK-OUTLINE-NEXT:    [[TMP29:%.*]] = inttoptr i64 [[TMP28]] to ptr
+; CHECK-OUTLINE-NEXT:    store i64 ptrtoint (ptr @___asan_gen_ to i64), ptr [[TMP29]], align 8
+; CHECK-OUTLINE-NEXT:    [[TMP30:%.*]] = add i64 [[TMP12]], 16
+; CHECK-OUTLINE-NEXT:    [[TMP31:%.*]] = inttoptr i64 [[TMP30]] to ptr
+; CHECK-OUTLINE-NEXT:    store i64 ptrtoint (ptr @FuncletPersonality to i64), ptr [[TMP31]], align 8
+; CHECK-OUTLINE-NEXT:    [[TMP32:%.*]] = lshr i64 [[TMP12]], 3
+; CHECK-OUTLINE-NEXT:    [[TMP33:%.*]] = add i64 [[TMP32]], [[TMP1]]
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f1(i64 [[TMP33]], i64 4)
+; CHECK-OUTLINE-NEXT:    [[TMP34:%.*]] = add i64 [[TMP33]], 1028
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP34]], i64 32)
+; CHECK-OUTLINE-NEXT:    [[TMP35:%.*]] = add i64 [[TMP33]], 1060
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP35]], i64 1)
+; CHECK-OUTLINE-NEXT:    [[TMP36:%.*]] = add i64 [[TMP33]], 1061
 ; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP36]], i64 1)
-; CHECK-OUTLINE-NEXT:    [[TMP37:%.*]] = add i64 [[TMP25]], 1071
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f3(i64 [[TMP37]], i64 5)
-; CHECK-OUTLINE-NEXT:    [[TMP38:%.*]] = add i64 [[TMP25]], 1066
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP38]], i64 1)
-; CHECK-OUTLINE-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[TMP16]])
-; CHECK-OUTLINE-NEXT:    call void @__asan_store1(i64 [[TMP15]])
-; CHECK-OUTLINE-NEXT:    store volatile i8 0, ptr [[TMP16]], align 1
-; CHECK-OUTLINE-NEXT:    [[TMP39:%.*]] = add i64 [[TMP25]], 1066
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP39]], i64 1)
-; CHECK-OUTLINE-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP16]])
-; CHECK-OUTLINE-NEXT:    call void @__asan_store8(i64 [[TMP17]])
-; CHECK-OUTLINE-NEXT:    store volatile i64 0, ptr [[TMP18]], align 8
-; CHECK-OUTLINE-NEXT:    [[TMPCOPYI64:%.*]] = load i64, ptr [[TMP18]], align 8
-; CHECK-OUTLINE-NEXT:    [[TMP40:%.*]] = and i64 [[TMPCOPYI64]], 31
-; CHECK-OUTLINE-NEXT:    [[TMP41:%.*]] = sub i64 32, [[TMP40]]
-; CHECK-OUTLINE-NEXT:    [[TMP42:%.*]] = icmp ne i64 [[TMP41]], 32
-; CHECK-OUTLINE-NEXT:    [[TMP43:%.*]] = select i1 [[TMP42]], i64 [[TMP41]], i64 0
-; CHECK-OUTLINE-NEXT:    [[TMP44:%.*]] = add i64 64, [[TMP43]]
-; CHECK-OUTLINE-NEXT:    [[TMP45:%.*]] = add i64 [[TMPCOPYI64]], [[TMP44]]
-; CHECK-OUTLINE-NEXT:    [[TMP46:%.*]] = alloca i8, i64 [[TMP45]], align 32
-; CHECK-OUTLINE-NEXT:    [[TMP47:%.*]] = ptrtoint ptr [[TMP46]] to i64
-; CHECK-OUTLINE-NEXT:    [[TMP48:%.*]] = add i64 [[TMP47]], 32
-; CHECK-OUTLINE-NEXT:    call void @__asan_alloca_poison(i64 [[TMP48]], i64 [[TMPCOPYI64]])
-; CHECK-OUTLINE-NEXT:    [[TMP49:%.*]] = ptrtoint ptr [[TMP46]] to i64
-; CHECK-OUTLINE-NEXT:    store i64 [[TMP49]], ptr [[TMP0]], align 8
-; CHECK-OUTLINE-NEXT:    [[TMP50:%.*]] = inttoptr i64 [[TMP48]] to ptr
-; CHECK-OUTLINE-NEXT:    call void @__asan_store1(i64 [[TMP48]])
-; CHECK-OUTLINE-NEXT:    store volatile i8 0, ptr [[TMP50]], align 1
+; CHECK-OUTLINE-NEXT:    [[TMP37:%.*]] = add i64 [[TMP33]], 1062
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP37]], i64 1)
+; CHECK-OUTLINE-NEXT:    [[TMP38:%.*]] = add i64 [[TMP33]], 1063
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP38]], i64 1)
+; CHECK-OUTLINE-NEXT:    [[TMP39:%.*]] = add i64 [[TMP33]], 1064
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP39]], i64 1)
+; CHECK-OUTLINE-NEXT:    [[TMP40:%.*]] = add i64 [[TMP33]], 1065
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP40]], i64 1)
+; CHECK-OUTLINE-NEXT:    [[TMP41:%.*]] = add i64 [[TMP33]], 1066
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP41]], i64 1)
+; CHECK-OUTLINE-NEXT:    [[TMP42:%.*]] = add i64 [[TMP33]], 1067
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP42]], i64 1)
+; CHECK-OUTLINE-NEXT:    [[TMP43:%.*]] = add i64 [[TMP33]], 1068
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP43]], i64 1)
+; CHECK-OUTLINE-NEXT:    [[TMP44:%.*]] = add i64 [[TMP33]], 1069
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP44]], i64 1)
+; CHECK-OUTLINE-NEXT:    [[TMP45:%.*]] = add i64 [[TMP33]], 1071
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f3(i64 [[TMP45]], i64 5)
+; CHECK-OUTLINE-NEXT:    [[TMP46:%.*]] = add i64 [[TMP33]], 1066
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP46]], i64 1)
+; CHECK-OUTLINE-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[TMP22]])
+; CHECK-OUTLINE-NEXT:    call void @__asan_store1(i64 [[TMP21]])
+; CHECK-OUTLINE-NEXT:    store volatile i8 0, ptr [[TMP22]], align 1
+; CHECK-OUTLINE-NEXT:    [[TMP47:%.*]] = add i64 [[TMP33]], 1066
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP47]], i64 1)
+; CHECK-OUTLINE-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP22]])
+; CHECK-OUTLINE-NEXT:    call void @__asan_store8(i64 [[TMP25]])
+; CHECK-OUTLINE-NEXT:    store volatile i64 0, ptr [[TMP26]], align 8
+; CHECK-OUTLINE-NEXT:    [[TMPCOPYI64:%.*]] = load i64, ptr [[TMP26]], align 8
+; CHECK-OUTLINE-NEXT:    [[TMP48:%.*]] = and i64 [[TMPCOPYI64]], 31
+; CHECK-OUTLINE-NEXT:    [[TMP49:%.*]] = sub i64 32, [[TMP48]]
+; CHECK-OUTLINE-NEXT:    [[TMP50:%.*]] = icmp ne i64 [[TMP49]], 32
+; CHECK-OUTLINE-NEXT:    [[TMP51:%.*]] = select i1 [[TMP50]], i64 [[TMP49]], i64 0
+; CHECK-OUTLINE-NEXT:    [[TMP52:%.*]] = add i64 64, [[TMP51]]
+; CHECK-OUTLINE-NEXT:    [[TMP53:%.*]] = add i64 [[TMPCOPYI64]], [[TMP52]]
+; CHECK-OUTLINE-NEXT:    [[TMP54:%.*]] = alloca i8, i64 [[TMP53]], align 32
+; CHECK-OUTLINE-NEXT:    [[TMP55:%.*]] = ptrtoint ptr [[TMP54]] to i64
+; CHECK-OUTLINE-NEXT:    [[TMP56:%.*]] = add i64 [[TMP55]], 32
+; CHECK-OUTLINE-NEXT:    call void @__asan_alloca_poison(i64 [[TMP56]], i64 [[TMPCOPYI64]])
+; CHECK-OUTLINE-NEXT:    [[TMP57:%.*]] = ptrtoint ptr [[TMP54]] to i64
+; CHECK-OUTLINE-NEXT:    store i64 [[TMP57]], ptr [[TMP0]], align 8
+; CHECK-OUTLINE-NEXT:    [[TMP58:%.*]] = inttoptr i64 [[TMP56]] to ptr
+; CHECK-OUTLINE-NEXT:    call void @__asan_store1(i64 [[TMP56]])
+; CHECK-OUTLINE-NEXT:    store volatile i8 0, ptr [[TMP58]], align 1
 ; CHECK-OUTLINE-NEXT:    invoke void @MayThrowFunc()
 ; CHECK-OUTLINE-NEXT:            to label [[INVOKE_CONT:%.*]] unwind label [[EHCLEANUP:%.*]]
 ; CHECK-OUTLINE:       invoke.cont:
 ; CHECK-OUTLINE-NEXT:    call void @DeInit(ptr [[TMP14]])
-; CHECK-OUTLINE-NEXT:    [[TMP51:%.*]] = ptrtoint ptr [[TMP0]] to i64
-; CHECK-OUTLINE-NEXT:    [[TMP52:%.*]] = load i64, ptr [[TMP0]], align 8
-; CHECK-OUTLINE-NEXT:    call void @__asan_allocas_unpoison(i64 [[TMP52]], i64 [[TMP51]])
-; CHECK-OUTLINE-NEXT:    store i64 1172321806, ptr [[TMP19]], align 8
-; CHECK-OUTLINE-NEXT:    [[TMP53:%.*]] = icmp ne i64 [[TMP7]], 0
-; CHECK-OUTLINE-NEXT:    br i1 [[TMP53]], label [[TMP54:%.*]], label [[TMP55:%.*]]
-; CHECK-OUTLINE:       54:
+; CHECK-OUTLINE-NEXT:    [[TMP59:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; CHECK-OUTLINE-NEXT:    [[TMP60:%.*]] = load i64, ptr [[TMP0]], align 8
+; CHECK-OUTLINE-NEXT:    call void @__asan_allocas_unpoison(i64 [[TMP60]], i64 [[TMP59]])
+; CHECK-OUTLINE-NEXT:    store i64 1172321806, ptr [[TMP27]], align 8
+; CHECK-OUTLINE-NEXT:    [[TMP61:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-OUTLINE-NEXT:    br i1 [[TMP61]], label [[TMP62:%.*]], label [[TMP63:%.*]]
+; CHECK-OUTLINE:       62:
 ; CHECK-OUTLINE-NEXT:    call void @__asan_stack_free_8(i64 [[TMP7]], i64 8608)
-; CHECK-OUTLINE-NEXT:    br label [[TMP58:%.*]]
-; CHECK-OUTLINE:       55:
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP25]], i64 4)
-; CHECK-OUTLINE-NEXT:    [[TMP56:%.*]] = add i64 [[TMP25]], 1028
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP56]], i64 42)
-; CHECK-OUTLINE-NEXT:    [[TMP57:%.*]] = add i64 [[TMP25]], 1071
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP57]], i64 5)
-; CHECK-OUTLINE-NEXT:    br label [[TMP58]]
-; CHECK-OUTLINE:       58:
+; CHECK-OUTLINE-NEXT:    br label [[TMP66:%.*]]
+; CHECK-OUTLINE:       63:
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP33]], i64 4)
+; CHECK-OUTLINE-NEXT:    [[TMP64:%.*]] = add i64 [[TMP33]], 1028
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP64]], i64 42)
+; CHECK-OUTLINE-NEXT:    [[TMP65:%.*]] = add i64 [[TMP33]], 1071
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP65]], i64 5)
+; CHECK-OUTLINE-NEXT:    br label [[TMP66]]
+; CHECK-OUTLINE:       66:
 ; CHECK-OUTLINE-NEXT:    ret void
 ; CHECK-OUTLINE:       ehcleanup:
-; CHECK-OUTLINE-NEXT:    [[TMP59:%.*]] = cleanuppad within none []
-; CHECK-OUTLINE-NEXT:    unreachable
+; CHECK-OUTLINE-NEXT:    [[TMP67:%.*]] = cleanuppad within none []
+; CHECK-OUTLINE-NEXT:    [[TMP68:%.*]] = add i64 [[TMP33]], 1068
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP68]], i64 1) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[TMP24]])
+; CHECK-OUTLINE-NEXT:    call void @__asan_store1(i64 [[TMP23]]) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    store volatile i8 0, ptr [[TMP24]], align 1
+; CHECK-OUTLINE-NEXT:    [[TMP69:%.*]] = add i64 [[TMP33]], 1068
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP69]], i64 1) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP24]])
+; CHECK-OUTLINE-NEXT:    call void @DeInit(ptr [[TMP14]]) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    [[TMP70:%.*]] = call ptr @__asan_memset(ptr [[TMP16]], i32 0, i64 4) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    [[TMP71:%.*]] = call ptr @__asan_memcpy(ptr [[TMP18]], ptr [[TMP16]], i64 4) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    [[TMP72:%.*]] = call ptr @__asan_memmove(ptr [[TMP20]], ptr [[TMP16]], i64 4) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    call void @__sanitizer_ptr_cmp(i64 [[TMP15]], i64 [[TMP17]]) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    call void @__sanitizer_ptr_sub(i64 [[TMP15]], i64 [[TMP17]]) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    [[TMP73:%.*]] = ptrtoint ptr [[PTRPARAM]] to i64
+; CHECK-OUTLINE-NEXT:    call void @__asan_storeN(i64 [[TMP73]], i64 8) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    store i64 0, ptr [[PTRPARAM]], align 1
+; CHECK-OUTLINE-NEXT:    [[TMP74:%.*]] = call i64 @llvm.get.dynamic.area.offset.i64()
+; CHECK-OUTLINE-NEXT:    [[TMP75:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; CHECK-OUTLINE-NEXT:    [[TMP76:%.*]] = add i64 [[TMP75]], [[TMP74]]
+; CHECK-OUTLINE-NEXT:    [[TMP77:%.*]] = load i64, ptr [[TMP0]], align 8
+; CHECK-OUTLINE-NEXT:    call void @__asan_allocas_unpoison(i64 [[TMP77]], i64 [[TMP76]]) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    store i64 1172321806, ptr [[TMP27]], align 8
+; CHECK-OUTLINE-NEXT:    [[TMP78:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-OUTLINE-NEXT:    br i1 [[TMP78]], label [[TMP79:%.*]], label [[TMP80:%.*]]
+; CHECK-OUTLINE:       79:
+; CHECK-OUTLINE-NEXT:    call void @__asan_stack_free_8(i64 [[TMP7]], i64 8608) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    br label [[TMP83:%.*]]
+; CHECK-OUTLINE:       80:
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP33]], i64 4) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    [[TMP81:%.*]] = add i64 [[TMP33]], 1028
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP81]], i64 42) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    [[TMP82:%.*]] = add i64 [[TMP33]], 1071
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP82]], i64 5) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    br label [[TMP83]]
+; CHECK-OUTLINE:       83:
+; CHECK-OUTLINE-NEXT:    cleanupret from [[TMP67]] unwind to caller
 ;
 
 
diff --git a/llvm/test/Instrumentation/AddressSanitizer/localescape.ll b/llvm/test/Instrumentation/AddressSanitizer/localescape.ll
index 4bce2203b3ee..9806dec0fd94 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/localescape.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/localescape.ll
@@ -14,10 +14,10 @@ declare ptr @llvm.eh.recoverfp(ptr, ptr)
 declare ptr @llvm.localrecover(ptr, ptr, i32)
 declare void @llvm.localescape(...) #1
 
-declare i32 @_except_handler3(...)
+declare i32 @__gcc_personality_v0(...)
 declare void @may_throw(ptr %r)
 
-define i32 @main() sanitize_address personality ptr @_except_handler3 {
+define i32 @main() sanitize_address personality ptr @__gcc_personality_v0 {
 entry:
   %r = alloca i32, align 4
   %__exception_code = alloca i32, align 4
-- 
GitLab


From 9fb552812c65a13cd19eedea1cc2ef739ad652e9 Mon Sep 17 00:00:00 2001
From: Piotr Zegar 
Date: Fri, 8 Mar 2024 18:34:26 +0100
Subject: [PATCH 637/929] [clang-tidy] Fix Hungarian Prefix in
 readability-identifier-naming (#84236)

Fix handling of Hungarian Prefix when configured to LowerCase.
Warnings no longer will be emited for names that already match
this style.

Fixes: #80268
---
 .../readability/IdentifierNamingCheck.cpp     |  10 +-
 clang-tools-extra/docs/ReleaseNotes.rst       |   3 +-
 .../checks/readability/identifier-naming.rst  |  28 +-
 .../hungarian-notation3/.clang-tidy           |  60 ++
 ...g-hungarian-notation-lower-case-prefix.cpp | 678 ++++++++++++++++++
 5 files changed, 766 insertions(+), 13 deletions(-)
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/readability/Inputs/identifier-naming/hungarian-notation3/.clang-tidy
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-hungarian-notation-lower-case-prefix.cpp

diff --git a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
index 335c3de25b86..dc30531ebda0 100644
--- a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
@@ -888,8 +888,14 @@ bool IdentifierNamingCheck::matchesStyle(
     return false;
   if (IdentifierNamingCheck::HungarianPrefixType::HPT_Off != Style.HPType) {
     std::string HNPrefix = HungarianNotation.getPrefix(Decl, HNOption);
-    if (!Name.consume_front(HNPrefix))
-      return false;
+    if (!HNPrefix.empty()) {
+      if (!Name.consume_front(HNPrefix))
+        return false;
+      if (Style.HPType ==
+              IdentifierNamingCheck::HungarianPrefixType::HPT_LowerCase &&
+          !Name.consume_front("_"))
+        return false;
+    }
   }
 
   // Ensure the name doesn't have any extra underscores beyond those specified
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index b5f025ce467a..a005adf76b8b 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -235,7 +235,8 @@ Changes in existing checks
 
 - Improved :doc:`readability-identifier-naming
   ` check in `GetConfigPerFile`
-  mode by resolving symbolic links to header files.
+  mode by resolving symbolic links to header files. Fixed handling of Hungarian
+  Prefix when configured to `LowerCase`.
 
 Removed checks
 ^^^^^^^^^^^^^^
diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability/identifier-naming.rst b/clang-tools-extra/docs/clang-tidy/checks/readability/identifier-naming.rst
index 2affb55cfa9a..8a54687a2570 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/readability/identifier-naming.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/readability/identifier-naming.rst
@@ -11,14 +11,14 @@ another if a mismatch is detected
 
 Casing types include:
 
- - ``lower_case``,
- - ``UPPER_CASE``,
- - ``camelBack``,
- - ``CamelCase``,
- - ``camel_Snake_Back``,
- - ``Camel_Snake_Case``,
- - ``aNy_CasE``,
- - ``Leading_upper_snake_case``.
+ - ``lower_case``
+ - ``UPPER_CASE``
+ - ``camelBack``
+ - ``CamelCase``
+ - ``camel_Snake_Back``
+ - ``Camel_Snake_Case``
+ - ``aNy_CasE``
+ - ``Leading_upper_snake_case``
 
 It also supports a fixed prefix and suffix that will be prepended or appended
 to the identifiers, regardless of the casing.
@@ -32,8 +32,16 @@ but not where they are overridden, as it can't be fixed locally there.
 This also applies for pseudo-override patterns like CRTP.
 
 ``Leading_upper_snake_case`` is a naming convention where the first word is capitalized
-followed by lower case word(s) seperated by underscore(s) '_'. Examples include:
-Cap_snake_case, Cobra_case, Foo_bar_baz, and Master_copy_8gb.
+followed by lower case word(s) separated by underscore(s) '_'. Examples include:
+`Cap_snake_case`, `Cobra_case`, `Foo_bar_baz`, and `Master_copy_8gb`.
+
+Hungarian notation can be customized using different *HungarianPrefix* settings.
+The options and their corresponding values are:
+
+ - ``Off`` - the default setting
+ - ``On`` - example: ``int iVariable``
+ - ``LowerCase`` - example: ``int i_Variable``
+ - ``CamelCase`` - example: ``int IVariable``
 
 Options
 -------
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/Inputs/identifier-naming/hungarian-notation3/.clang-tidy b/clang-tools-extra/test/clang-tidy/checkers/readability/Inputs/identifier-naming/hungarian-notation3/.clang-tidy
new file mode 100644
index 000000000000..45b4cd46c249
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/Inputs/identifier-naming/hungarian-notation3/.clang-tidy
@@ -0,0 +1,60 @@
+Checks: readability-identifier-naming
+CheckOptions:
+  readability-identifier-naming.AbstractClassCase: CamelCase
+  readability-identifier-naming.StructCase: CamelCase
+  readability-identifier-naming.UnionCase: camelBack
+  readability-identifier-naming.ClassCase: CamelCase
+  readability-identifier-naming.ClassConstantCase: CamelCase
+  readability-identifier-naming.ClassMemberCase: CamelCase
+  readability-identifier-naming.ConstantCase: CamelCase
+  readability-identifier-naming.ConstantMemberCase: CamelCase
+  readability-identifier-naming.ConstantParameterCase: CamelCase
+  readability-identifier-naming.ConstantPointerParameterCase: CamelCase
+  readability-identifier-naming.ConstexprVariableCase: CamelCase
+  readability-identifier-naming.EnumConstantCase: CamelCase
+  readability-identifier-naming.GlobalConstantCase: CamelCase
+  readability-identifier-naming.GlobalConstantPointerCase: CamelCase
+  readability-identifier-naming.GlobalPointerCase: CamelCase
+  readability-identifier-naming.GlobalVariableCase: CamelCase
+  readability-identifier-naming.LocalConstantCase: CamelCase
+  readability-identifier-naming.LocalConstantPointerCase: CamelCase
+  readability-identifier-naming.LocalPointerCase: CamelCase
+  readability-identifier-naming.LocalVariableCase: CamelCase
+  readability-identifier-naming.MemberCase: CamelCase
+  readability-identifier-naming.ParameterCase: CamelCase
+  readability-identifier-naming.PointerParameterCase: CamelCase
+  readability-identifier-naming.PrivateMemberCase: CamelCase
+  readability-identifier-naming.ProtectedMemberCase: CamelCase
+  readability-identifier-naming.PublicMemberCase: CamelCase
+  readability-identifier-naming.ScopedEnumConstantCase: CamelCase
+  readability-identifier-naming.StaticConstantCase: CamelCase
+  readability-identifier-naming.StaticVariableCase: CamelCase
+  readability-identifier-naming.VariableCase: CamelCase
+  readability-identifier-naming.AbstractClassHungarianPrefix: LowerCase
+  readability-identifier-naming.ClassHungarianPrefix: LowerCase
+  readability-identifier-naming.ClassConstantHungarianPrefix: LowerCase
+  readability-identifier-naming.ClassMemberHungarianPrefix: LowerCase
+  readability-identifier-naming.ConstantHungarianPrefix: LowerCase
+  readability-identifier-naming.ConstantMemberHungarianPrefix: LowerCase
+  readability-identifier-naming.ConstantParameterHungarianPrefix: LowerCase
+  readability-identifier-naming.ConstantPointerParameterHungarianPrefix: LowerCase
+  readability-identifier-naming.ConstexprVariableHungarianPrefix: LowerCase
+  readability-identifier-naming.EnumConstantHungarianPrefix: LowerCase
+  readability-identifier-naming.GlobalConstantHungarianPrefix: LowerCase
+  readability-identifier-naming.GlobalConstantPointerHungarianPrefix: LowerCase
+  readability-identifier-naming.GlobalPointerHungarianPrefix: LowerCase
+  readability-identifier-naming.GlobalVariableHungarianPrefix: LowerCase
+  readability-identifier-naming.LocalConstantHungarianPrefix: LowerCase
+  readability-identifier-naming.LocalConstantPointerHungarianPrefix: LowerCase
+  readability-identifier-naming.LocalPointerHungarianPrefix: LowerCase
+  readability-identifier-naming.LocalVariableHungarianPrefix: LowerCase
+  readability-identifier-naming.MemberHungarianPrefix: LowerCase
+  readability-identifier-naming.ParameterHungarianPrefix: LowerCase
+  readability-identifier-naming.PointerParameterHungarianPrefix: LowerCase
+  readability-identifier-naming.PrivateMemberHungarianPrefix: LowerCase
+  readability-identifier-naming.ProtectedMemberHungarianPrefix: LowerCase
+  readability-identifier-naming.PublicMemberHungarianPrefix: LowerCase
+  readability-identifier-naming.ScopedEnumConstantHungarianPrefix: LowerCase
+  readability-identifier-naming.StaticConstantHungarianPrefix: LowerCase
+  readability-identifier-naming.StaticVariableHungarianPrefix: LowerCase
+  readability-identifier-naming.VariableHungarianPrefix: LowerCase
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-hungarian-notation-lower-case-prefix.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-hungarian-notation-lower-case-prefix.cpp
new file mode 100644
index 000000000000..65aee7e9f6ce
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-hungarian-notation-lower-case-prefix.cpp
@@ -0,0 +1,678 @@
+// RUN: %check_clang_tidy %s readability-identifier-naming %t -- \
+// RUN:   --config-file=%S/Inputs/identifier-naming/hungarian-notation3/.clang-tidy -- -I %S
+
+#include "identifier-naming-standard-types.h"
+
+// clang-format off
+//===----------------------------------------------------------------------===//
+// Cases to CheckOptions
+//===----------------------------------------------------------------------===//
+class C_MyClass1 {
+public:
+  static int ClassMemberCase;
+  // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: invalid case style for class member 'ClassMemberCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  static int i_ClassMemberCase;
+
+  char const ConstantMemberCase = 0;
+  // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: invalid case style for constant member 'ConstantMemberCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  char const c_ConstantMemberCase = 0;
+
+  void MyFunc1(const int ConstantParameterCase);
+  // CHECK-MESSAGES: :[[@LINE-1]]:26: warning: invalid case style for constant parameter 'ConstantParameterCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  void MyFunc1(const int i_ConstantParameterCase);
+
+  void MyFunc2(const int* ConstantPointerParameterCase);
+  // CHECK-MESSAGES: :[[@LINE-1]]:27: warning: invalid case style for pointer parameter 'ConstantPointerParameterCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  void MyFunc2(const int* pi_ConstantPointerParameterCase);
+
+  static constexpr int ConstexprVariableCase = 123;
+  // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: invalid case style for constexpr variable 'ConstexprVariableCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  static constexpr int i_ConstexprVariableCase = 123;
+};
+
+const int GlobalConstantCase = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:11: warning: invalid case style for global constant 'GlobalConstantCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}const int i_GlobalConstantCase = 0;
+
+const int* GlobalConstantPointerCase = nullptr;
+// CHECK-MESSAGES: :[[@LINE-1]]:12: warning: invalid case style for global pointer 'GlobalConstantPointerCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}const int* pi_GlobalConstantPointerCase = nullptr;
+
+int* GlobalPointerCase = nullptr;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global pointer 'GlobalPointerCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int* pi_GlobalPointerCase = nullptr;
+
+int GlobalVariableCase = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'GlobalVariableCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_GlobalVariableCase = 0;
+
+void Func1(){
+  int const LocalConstantCase = 3;
+  // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: invalid case style for local constant 'LocalConstantCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  int const i_LocalConstantCase = 3;
+
+  unsigned const ConstantCase = 1;
+  // CHECK-MESSAGES: :[[@LINE-1]]:18: warning: invalid case style for local constant 'ConstantCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  unsigned const u_ConstantCase = 1;
+
+  int* const LocalConstantPointerCase = nullptr;
+  // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: invalid case style for local constant pointer 'LocalConstantPointerCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  int* const pi_LocalConstantPointerCase = nullptr;
+
+  int *LocalPointerCase = nullptr;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for local pointer 'LocalPointerCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  int *pi_LocalPointerCase = nullptr;
+
+  int LocalVariableCase = 0;
+  // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for local variable 'LocalVariableCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  int i_LocalVariableCase = 0;
+}
+
+class C_MyClass2 {
+  char MemberCase;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for private member 'MemberCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  char c_MemberCase;
+
+  void Func1(int ParameterCase);
+  // CHECK-MESSAGES: :[[@LINE-1]]:18: warning: invalid case style for parameter 'ParameterCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  void Func1(int i_ParameterCase);
+
+  void Func2(const int ParameterCase);
+  // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: invalid case style for constant parameter 'ParameterCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  void Func2(const int i_ParameterCase);
+
+  void Func3(const int *PointerParameterCase);
+  // CHECK-MESSAGES: :[[@LINE-1]]:25: warning: invalid case style for pointer parameter 'PointerParameterCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  void Func3(const int *pi_PointerParameterCase);
+};
+
+class C_MyClass3 {
+private:
+  char PrivateMemberCase;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for private member 'PrivateMemberCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  char c_PrivateMemberCase;
+
+protected:
+  char ProtectedMemberCase;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for protected member 'ProtectedMemberCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  char c_ProtectedMemberCase;
+
+public:
+  char PublicMemberCase;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for public member 'PublicMemberCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  char c_PublicMemberCase;
+};
+
+static const int StaticConstantCase = 3;
+// CHECK-MESSAGES: :[[@LINE-1]]:18: warning: invalid case style for global constant 'StaticConstantCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}static const int i_StaticConstantCase = 3;
+
+static int StaticVariableCase = 3;
+// CHECK-MESSAGES: :[[@LINE-1]]:12: warning: invalid case style for global variable 'StaticVariableCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}static int i_StaticVariableCase = 3;
+
+struct MyStruct { int StructCase; };
+// CHECK-MESSAGES: :[[@LINE-1]]:23: warning: invalid case style for public member 'StructCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}struct MyStruct { int i_StructCase; };
+
+struct shouldBeCamelCaseStruct { int i_Field; };
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for struct 'shouldBeCamelCaseStruct' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}struct ShouldBeCamelCaseStruct { int i_Field; };
+
+union MyUnion { int UnionCase; long l_UnionCase; };
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for union 'MyUnion' [readability-identifier-naming]
+// CHECK-MESSAGES: :[[@LINE-2]]:21: warning: invalid case style for public member 'UnionCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}union myUnion { int i_UnionCase; long l_UnionCase; };
+
+//===----------------------------------------------------------------------===//
+// C string
+//===----------------------------------------------------------------------===//
+const char *NamePtr = "Name";
+// CHECK-MESSAGES: :[[@LINE-1]]:13: warning: invalid case style for global pointer 'NamePtr' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}const char *sz_NamePtr = "Name";
+
+const char NameArray[] = "Name";
+// CHECK-MESSAGES: :[[@LINE-1]]:12: warning: invalid case style for global constant 'NameArray' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}const char sz_NameArray[] = "Name";
+
+const char *NamePtrArray[] = {"AA", "BB"};
+// CHECK-MESSAGES: :[[@LINE-1]]:13: warning: invalid case style for global variable 'NamePtrArray' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}const char *psz_NamePtrArray[] = {"AA", "BB"};
+
+const wchar_t *WideNamePtr = L"Name";
+// CHECK-MESSAGES: :[[@LINE-1]]:16: warning: invalid case style for global pointer 'WideNamePtr' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}const wchar_t *wsz_WideNamePtr = L"Name";
+
+const wchar_t WideNameArray[] = L"Name";
+// CHECK-MESSAGES: :[[@LINE-1]]:15: warning: invalid case style for global constant 'WideNameArray' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}const wchar_t wsz_WideNameArray[] = L"Name";
+
+const wchar_t *WideNamePtrArray[] = {L"AA", L"BB"};
+// CHECK-MESSAGES: :[[@LINE-1]]:16: warning: invalid case style for global variable 'WideNamePtrArray' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}const wchar_t *pwsz_WideNamePtrArray[] = {L"AA", L"BB"};
+
+class C_MyClass4 {
+private:
+  char *Name = "Text";
+  // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for private member 'Name' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  char *sz_Name = "Text";
+
+  const char *ConstName = "Text";
+  // CHECK-MESSAGES: :[[@LINE-1]]:15: warning: invalid case style for private member 'ConstName' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  const char *sz_ConstName = "Text";
+
+public:
+  const char* DuplicateString(const char* Input, size_t n_RequiredSize);
+  // CHECK-MESSAGES: :[[@LINE-1]]:43: warning: invalid case style for pointer parameter 'Input' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  const char* DuplicateString(const char* sz_Input, size_t n_RequiredSize);
+
+  size_t UpdateText(const char* Buffer, size_t n_BufferSize);
+  // CHECK-MESSAGES: :[[@LINE-1]]:33: warning: invalid case style for pointer parameter 'Buffer' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  size_t UpdateText(const char* sz_Buffer, size_t n_BufferSize);
+};
+
+
+//===----------------------------------------------------------------------===//
+// Microsoft Windows data types
+//===----------------------------------------------------------------------===//
+DWORD MsDword = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsDword' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}DWORD dw_MsDword = 0;
+
+BYTE MsByte = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsByte' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}BYTE by_MsByte = 0;
+
+WORD MsWord = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsWord' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}WORD w_MsWord = 0;
+
+BOOL MsBool = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsBool' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}BOOL b_MsBool = 0;
+
+BOOLEAN MsBoolean = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'MsBoolean' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}BOOLEAN b_MsBoolean = 0;
+
+CHAR MsValueChar = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsValueChar' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}CHAR c_MsValueChar = 0;
+
+UCHAR MsValueUchar = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueUchar' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}UCHAR uc_MsValueUchar = 0;
+
+SHORT MsValueShort = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueShort' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}SHORT s_MsValueShort = 0;
+
+USHORT MsValueUshort = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'MsValueUshort' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}USHORT us_MsValueUshort = 0;
+
+WORD MsValueWord = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsValueWord' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}WORD w_MsValueWord = 0;
+
+DWORD MsValueDword = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueDword' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}DWORD dw_MsValueDword = 0;
+
+DWORD32 MsValueDword32 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'MsValueDword32' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}DWORD32 dw32_MsValueDword32 = 0;
+
+DWORD64 MsValueDword64 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'MsValueDword64' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}DWORD64 dw64_MsValueDword64 = 0;
+
+LONG MsValueLong = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsValueLong' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}LONG l_MsValueLong = 0;
+
+ULONG MsValueUlong = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueUlong' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}ULONG ul_MsValueUlong = 0;
+
+ULONG32 MsValueUlong32 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'MsValueUlong32' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}ULONG32 ul32_MsValueUlong32 = 0;
+
+ULONG64 MsValueUlong64 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'MsValueUlong64' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}ULONG64 ul64_MsValueUlong64 = 0;
+
+ULONGLONG MsValueUlongLong = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:11: warning: invalid case style for global variable 'MsValueUlongLong' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}ULONGLONG ull_MsValueUlongLong = 0;
+
+HANDLE MsValueHandle = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global pointer 'MsValueHandle' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}HANDLE h_MsValueHandle = 0;
+
+INT MsValueInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'MsValueInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}INT i_MsValueInt = 0;
+
+INT8 MsValueInt8 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsValueInt8' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}INT8 i8_MsValueInt8 = 0;
+
+INT16 MsValueInt16 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueInt16' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}INT16 i16_MsValueInt16 = 0;
+
+INT32 MsValueInt32 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueInt32' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}INT32 i32_MsValueInt32 = 0;
+
+INT64 MsValueINt64 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueINt64' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}INT64 i64_MsValueINt64 = 0;
+
+UINT MsValueUint = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsValueUint' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}UINT ui_MsValueUint = 0;
+
+UINT8 MsValueUint8 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueUint8' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}UINT8 u8_MsValueUint8 = 0;
+
+UINT16 MsValueUint16 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'MsValueUint16' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}UINT16 u16_MsValueUint16 = 0;
+
+UINT32 MsValueUint32 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'MsValueUint32' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}UINT32 u32_MsValueUint32 = 0;
+
+UINT64 MsValueUint64 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'MsValueUint64' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}UINT64 u64_MsValueUint64 = 0;
+
+PVOID MsValuePvoid = NULL;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global pointer 'MsValuePvoid' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}PVOID p_MsValuePvoid = NULL;
+
+
+//===----------------------------------------------------------------------===//
+// Array
+//===----------------------------------------------------------------------===//
+unsigned GlobalUnsignedArray[] = {1, 2, 3};
+// CHECK-MESSAGES: :[[@LINE-1]]:10: warning: invalid case style for global variable 'GlobalUnsignedArray' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned a_GlobalUnsignedArray[] = {1, 2, 3};
+
+int GlobalIntArray[] = {1, 2, 3};
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'GlobalIntArray' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int a_GlobalIntArray[] = {1, 2, 3};
+
+int DataInt[1] = {0};
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'DataInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int a_DataInt[1] = {0};
+
+int DataArray[2] = {0};
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'DataArray' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int a_DataArray[2] = {0};
+
+
+//===----------------------------------------------------------------------===//
+// Pointer
+//===----------------------------------------------------------------------===//
+int *DataIntPtr[1] = {0};
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'DataIntPtr' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int *pa_DataIntPtr[1] = {0};
+
+void *BufferPtr1;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global pointer 'BufferPtr1' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}void *p_BufferPtr1;
+
+void **BufferPtr2;
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global pointer 'BufferPtr2' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}void **pp_BufferPtr2;
+
+void **pBufferPtr3;
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global pointer 'pBufferPtr3' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}void **pp_BufferPtr3;
+
+int *pBufferPtr4;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global pointer 'pBufferPtr4' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int *pi_BufferPtr4;
+
+typedef void (*FUNC_PTR_HELLO)();
+FUNC_PTR_HELLO Hello = NULL;
+// CHECK-MESSAGES: :[[@LINE-1]]:16: warning: invalid case style for global pointer 'Hello' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}FUNC_PTR_HELLO fn_Hello = NULL;
+
+void *ValueVoidPtr = NULL;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global pointer 'ValueVoidPtr' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}void *p_ValueVoidPtr = NULL;
+
+ptrdiff_t PtrDiff = NULL;
+// CHECK-MESSAGES: :[[@LINE-1]]:11: warning: invalid case style for global variable 'PtrDiff' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}ptrdiff_t p_PtrDiff = NULL;
+
+int8_t *ValueI8Ptr;
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global pointer 'ValueI8Ptr' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int8_t *pi8_ValueI8Ptr;
+
+uint8_t *ValueU8Ptr;
+// CHECK-MESSAGES: :[[@LINE-1]]:10: warning: invalid case style for global pointer 'ValueU8Ptr' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}uint8_t *pu8_ValueU8Ptr;
+
+unsigned char *ValueUcPtr;
+// CHECK-MESSAGES: :[[@LINE-1]]:16: warning: invalid case style for global pointer 'ValueUcPtr' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned char *puc_ValueUcPtr;
+
+unsigned char **ValueUcPtr2;
+// CHECK-MESSAGES: :[[@LINE-1]]:17: warning: invalid case style for global pointer 'ValueUcPtr2' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned char **ppuc_ValueUcPtr2;
+
+void MyFunc2(void* Val){}
+// CHECK-MESSAGES: :[[@LINE-1]]:20: warning: invalid case style for pointer parameter 'Val' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}void MyFunc2(void* p_Val){}
+
+
+//===----------------------------------------------------------------------===//
+// Reference
+//===----------------------------------------------------------------------===//
+int i_ValueIndex = 1;
+int &RefValueIndex = i_ValueIndex;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'RefValueIndex' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int &i_RefValueIndex = i_ValueIndex;
+
+const int &ConstRefValue = i_ValueIndex;
+// CHECK-MESSAGES: :[[@LINE-1]]:12: warning: invalid case style for global variable 'ConstRefValue' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}const int &i_ConstRefValue = i_ValueIndex;
+
+long long ll_ValueLongLong = 2;
+long long &RefValueLongLong = ll_ValueLongLong;
+// CHECK-MESSAGES: :[[@LINE-1]]:12: warning: invalid case style for global variable 'RefValueLongLong' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}long long &ll_RefValueLongLong = ll_ValueLongLong;
+
+
+//===----------------------------------------------------------------------===//
+// Various types
+//===----------------------------------------------------------------------===//
+int8_t ValueI8;
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'ValueI8' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int8_t i8_ValueI8;
+
+int16_t ValueI16 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'ValueI16' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int16_t i16_ValueI16 = 0;
+
+int32_t ValueI32 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'ValueI32' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int32_t i32_ValueI32 = 0;
+
+int64_t ValueI64 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'ValueI64' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int64_t i64_ValueI64 = 0;
+
+uint8_t ValueU8 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'ValueU8' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}uint8_t u8_ValueU8 = 0;
+
+uint16_t ValueU16 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:10: warning: invalid case style for global variable 'ValueU16' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}uint16_t u16_ValueU16 = 0;
+
+uint32_t ValueU32 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:10: warning: invalid case style for global variable 'ValueU32' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}uint32_t u32_ValueU32 = 0;
+
+uint64_t ValueU64 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:10: warning: invalid case style for global variable 'ValueU64' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}uint64_t u64_ValueU64 = 0;
+
+float ValueFloat = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'ValueFloat' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}float f_ValueFloat = 0;
+
+double ValueDouble = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'ValueDouble' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}double d_ValueDouble = 0;
+
+char ValueChar = 'c';
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'ValueChar' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}char c_ValueChar = 'c';
+
+bool ValueBool = true;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'ValueBool' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}bool b_ValueBool = true;
+
+int ValueInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'ValueInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_ValueInt = 0;
+
+size_t ValueSize = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'ValueSize' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}size_t n_ValueSize = 0;
+
+wchar_t ValueWchar = 'w';
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'ValueWchar' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}wchar_t wc_ValueWchar = 'w';
+
+short ValueShort = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'ValueShort' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}short s_ValueShort = 0;
+
+unsigned ValueUnsigned = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:10: warning: invalid case style for global variable 'ValueUnsigned' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned u_ValueUnsigned = 0;
+
+signed ValueSigned = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'ValueSigned' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}signed s_ValueSigned = 0;
+
+long ValueLong = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'ValueLong' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}long l_ValueLong = 0;
+
+long long ValueLongLong = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:11: warning: invalid case style for global variable 'ValueLongLong' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}long long ll_ValueLongLong = 0;
+
+long long int ValueLongLongInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:15: warning: invalid case style for global variable 'ValueLongLongInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}long long int lli_ValueLongLongInt = 0;
+
+long double ValueLongDouble = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:13: warning: invalid case style for global variable 'ValueLongDouble' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}long double ld_ValueLongDouble = 0;
+
+signed int ValueSignedInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:12: warning: invalid case style for global variable 'ValueSignedInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}signed int si_ValueSignedInt = 0;
+
+signed short ValueSignedShort = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:14: warning: invalid case style for global variable 'ValueSignedShort' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}signed short ss_ValueSignedShort = 0;
+
+signed short int ValueSignedShortInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:18: warning: invalid case style for global variable 'ValueSignedShortInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}signed short int ssi_ValueSignedShortInt = 0;
+
+signed long long ValueSignedLongLong = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:18: warning: invalid case style for global variable 'ValueSignedLongLong' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}signed long long sll_ValueSignedLongLong = 0;
+
+signed long int ValueSignedLongInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:17: warning: invalid case style for global variable 'ValueSignedLongInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}signed long int sli_ValueSignedLongInt = 0;
+
+signed long ValueSignedLong = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:13: warning: invalid case style for global variable 'ValueSignedLong' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}signed long sl_ValueSignedLong = 0;
+
+unsigned long long int ValueUnsignedLongLongInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:24: warning: invalid case style for global variable 'ValueUnsignedLongLongInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned long long int ulli_ValueUnsignedLongLongInt = 0;
+
+unsigned long long ValueUnsignedLongLong = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:20: warning: invalid case style for global variable 'ValueUnsignedLongLong' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned long long ull_ValueUnsignedLongLong = 0;
+
+unsigned long int ValueUnsignedLongInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:19: warning: invalid case style for global variable 'ValueUnsignedLongInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned long int uli_ValueUnsignedLongInt = 0;
+
+unsigned long ValueUnsignedLong = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:15: warning: invalid case style for global variable 'ValueUnsignedLong' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned long ul_ValueUnsignedLong = 0;
+
+unsigned short int ValueUnsignedShortInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:20: warning: invalid case style for global variable 'ValueUnsignedShortInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned short int usi_ValueUnsignedShortInt = 0;
+
+unsigned short ValueUnsignedShort = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:16: warning: invalid case style for global variable 'ValueUnsignedShort' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned short us_ValueUnsignedShort = 0;
+
+unsigned int ValueUnsignedInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:14: warning: invalid case style for global variable 'ValueUnsignedInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned int ui_ValueUnsignedInt = 0;
+
+unsigned char ValueUnsignedChar = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:15: warning: invalid case style for global variable 'ValueUnsignedChar' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned char uc_ValueUnsignedChar = 0;
+
+long int ValueLongInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:10: warning: invalid case style for global variable 'ValueLongInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}long int li_ValueLongInt = 0;
+
+
+//===----------------------------------------------------------------------===//
+// Specifier, Qualifier, Other keywords
+//===----------------------------------------------------------------------===//
+volatile int VolatileInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:14: warning: invalid case style for global variable 'VolatileInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}volatile int i_VolatileInt = 0;
+
+thread_local int ThreadLocalValueInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:18: warning: invalid case style for global variable 'ThreadLocalValueInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}thread_local int i_ThreadLocalValueInt = 0;
+
+extern int ExternValueInt;
+// CHECK-MESSAGES: :[[@LINE-1]]:12: warning: invalid case style for global variable 'ExternValueInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}extern int i_ExternValueInt;
+
+struct DataBuffer {
+    mutable size_t Size;
+};
+// CHECK-MESSAGES: :[[@LINE-2]]:20: warning: invalid case style for public member 'Size' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}    mutable size_t n_Size;
+
+static constexpr int const &ConstExprInt = 42;
+// CHECK-MESSAGES: :[[@LINE-1]]:29: warning: invalid case style for constexpr variable 'ConstExprInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}static constexpr int const &i_ConstExprInt = 42;
+
+
+//===----------------------------------------------------------------------===//
+// Redefined types
+//===----------------------------------------------------------------------===//
+typedef int INDEX;
+INDEX iIndex = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'iIndex' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}INDEX Index = 0;
+
+
+//===----------------------------------------------------------------------===//
+// Class
+//===----------------------------------------------------------------------===//
+class ClassCase { int Func(); };
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for class 'ClassCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}class C_ClassCase { int Func(); };
+
+class AbstractClassCase { virtual int Func() = 0; };
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for abstract class 'AbstractClassCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}class I_AbstractClassCase { virtual int Func() = 0; };
+
+class AbstractClassCase1 { virtual int Func1() = 0; int Func2(); };
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for abstract class 'AbstractClassCase1' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}class I_AbstractClassCase1 { virtual int Func1() = 0; int Func2(); };
+
+class ClassConstantCase { public: static const int i_ConstantCase; };
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for class 'ClassConstantCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}class C_ClassConstantCase { public: static const int i_ConstantCase; };
+
+//===----------------------------------------------------------------------===//
+// Other Cases
+//===----------------------------------------------------------------------===//
+int lower_case = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'lower_case' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_LowerCase = 0;
+
+int lower_case1 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'lower_case1' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_LowerCase1 = 0;
+
+int lower_case_2 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'lower_case_2' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_LowerCase2 = 0;
+
+int UPPER_CASE = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'UPPER_CASE' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_UpperCase = 0;
+
+int UPPER_CASE_1 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'UPPER_CASE_1' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_UpperCase1 = 0;
+
+int camelBack = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'camelBack' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_CamelBack = 0;
+
+int camelBack_1 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'camelBack_1' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_CamelBack1 = 0;
+
+int camelBack2 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'camelBack2' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_CamelBack2 = 0;
+
+int CamelCase = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'CamelCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_CamelCase = 0;
+
+int CamelCase_1 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'CamelCase_1' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_CamelCase1 = 0;
+
+int CamelCase2 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'CamelCase2' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_CamelCase2 = 0;
+
+int camel_Snake_Back = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'camel_Snake_Back' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_CamelSnakeBack = 0;
+
+int camel_Snake_Back_1 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'camel_Snake_Back_1' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_CamelSnakeBack1 = 0;
+
+int Camel_Snake_Case = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'Camel_Snake_Case' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_CamelSnakeCase = 0;
+
+int Camel_Snake_Case_1 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'Camel_Snake_Case_1' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_CamelSnakeCase1 = 0;
+
+//===----------------------------------------------------------------------===//
+// Enum
+//===----------------------------------------------------------------------===//
+enum REV_TYPE { RevValid };
+// CHECK-MESSAGES: :[[@LINE-1]]:17: warning: invalid case style for enum constant 'RevValid' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}enum REV_TYPE { rt_RevValid };
+
+enum EnumConstantCase { OneByte, TwoByte };
+// CHECK-MESSAGES: :[[@LINE-1]]:25: warning: invalid case style for enum constant 'OneByte' [readability-identifier-naming]
+// CHECK-MESSAGES: :[[@LINE-2]]:34: warning: invalid case style for enum constant 'TwoByte' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}enum EnumConstantCase { ecc_OneByte, ecc_TwoByte };
+
+enum class ScopedEnumConstantCase { Case1 };
+// CHECK-MESSAGES: :[[@LINE-1]]:37: warning: invalid case style for scoped enum constant 'Case1' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}enum class ScopedEnumConstantCase { secc_Case1 };
+// clang-format on
-- 
GitLab


From 81798d516067a58c8e4ba51072dae7351d445064 Mon Sep 17 00:00:00 2001
From: Fangrui Song 
Date: Fri, 8 Mar 2024 09:42:59 -0800
Subject: [PATCH 638/929] [ELF] Sort sections in order. NFC

---
 lld/ELF/DWARF.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lld/ELF/DWARF.cpp b/lld/ELF/DWARF.cpp
index a56454c53a6d..ac28aa8c7ca6 100644
--- a/lld/ELF/DWARF.cpp
+++ b/lld/ELF/DWARF.cpp
@@ -38,11 +38,11 @@ template  LLDDwarfObj::LLDDwarfObj(ObjFile *obj) {
                 .Case(".debug_addr", &addrSection)
                 .Case(".debug_gnu_pubnames", &gnuPubnamesSection)
                 .Case(".debug_gnu_pubtypes", &gnuPubtypesSection)
+                .Case(".debug_line", &lineSection)
                 .Case(".debug_loclists", &loclistsSection)
                 .Case(".debug_ranges", &rangesSection)
                 .Case(".debug_rnglists", &rnglistsSection)
                 .Case(".debug_str_offsets", &strOffsetsSection)
-                .Case(".debug_line", &lineSection)
                 .Default(nullptr)) {
       m->Data = toStringRef(sec->contentMaybeDecompress());
       m->sec = sec;
-- 
GitLab


From 2e0ddfc1638597e0d08fb028fdd6754387732107 Mon Sep 17 00:00:00 2001
From: Chris B 
Date: Fri, 8 Mar 2024 11:44:31 -0600
Subject: [PATCH 639/929] [DirectX][docs] Expand DXILArchitecture documentation
 (#84077)

This just seeks to give a high level overview of the current state of
the DirectX backend. It doesn't go in too deep, but should be a starting
point for further documentation.

---------

Co-authored-by: S. Bharadwaj Yadavalli 
---
 llvm/docs/DirectX/DXILArchitecture.rst | 46 ++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/llvm/docs/DirectX/DXILArchitecture.rst b/llvm/docs/DirectX/DXILArchitecture.rst
index f47db5413631..d6712bea4f77 100644
--- a/llvm/docs/DirectX/DXILArchitecture.rst
+++ b/llvm/docs/DirectX/DXILArchitecture.rst
@@ -89,6 +89,52 @@ leverage LLVM's current bitcode libraries to do a lot of the work, but
 it's possible that at some point in the future it will need to be
 completely separate as modern LLVM bitcode evolves.
 
+DirectX Backend Flow
+--------------------
+
+The code generation flow for DXIL is broken into a series of passes. The passes
+are grouped into two flows:
+
+#. Generating DXIL IR.
+#. Generating DXIL Binary.
+
+The passes to generate DXIL IR follow the flow:
+
+  DXILOpLowering -> DXILPrepare -> DXILTranslateMetadata
+
+Each of these passes has a defined responsibility:
+
+#. DXILOpLowering translates LLVM intrinsic calls to dx.op calls.
+#. DXILPrepare transforms the DXIL IR to be compatible with LLVM 3.7, and
+   inserts bitcasts to allow typed pointers to be inserted.
+#. DXILTranslateMetadata emits the DXIL Metadata structures.
+
+The passes to encode DXIL to binary in the DX Container follow the flow:
+
+  DXILEmbedder -> DXContainerGlobals -> AsmPrinter
+
+Each of these passes have the following defined responsibilities:
+
+#. DXILEmbedder runs the DXIL bitcode writer to generate a bitcode stream and
+   embeds the binary data inside a global in the original module.
+#. DXContainerGlobals generates binary data globals for the other DX Container
+   parts based on computed analysis passes.
+#. AsmPrinter is the standard LLVM infrastructure for emitting object files.
+
+When emitting DXIL into a DX Container file the MC layer is used in a similar
+way to how the Clang ``-fembed-bitcode`` option operates. The DX Container
+object writer knows how to construct the headers and structural fields of the
+container, and reads global variables from the module to fill in the remaining
+part data.
+
+DirectX Container
+-----------------
+
+The DirectX container format is treated in LLVM as an object file format.
+Reading is implemented between the BinaryFormat and Object libraries, and
+writing is implemented in the MC layer. Additional testing and inspection
+support are implemented in the ObjectYAML library and tools.
+
 Testing
 =======
 
-- 
GitLab


From 5a95378659506b0ce94ceb79a43477e73c9756f4 Mon Sep 17 00:00:00 2001
From: erichkeane 
Date: Fri, 8 Mar 2024 10:15:41 -0800
Subject: [PATCH 640/929] [NFC] Remove unnecessary 'Builtins.def' file.

This was replaced with tablegen anyway, so it just contained a stale
comment.  This patch removes it.
---
 clang/include/clang/Basic/Builtins.def | 102 -------------------------
 1 file changed, 102 deletions(-)
 delete mode 100644 clang/include/clang/Basic/Builtins.def

diff --git a/clang/include/clang/Basic/Builtins.def b/clang/include/clang/Basic/Builtins.def
deleted file mode 100644
index f356f881d5ef..000000000000
--- a/clang/include/clang/Basic/Builtins.def
+++ /dev/null
@@ -1,102 +0,0 @@
-//===--- Builtins.def - Builtin function info database ----------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// This is only documentation for the database layout. This will be removed once
-// all builtin databases are converted to tablegen files
-
-// The second value provided to the macro specifies the type of the function
-// (result value, then each argument) as follows:
-//  v -> void
-//  b -> boolean
-//  c -> char
-//  s -> short
-//  i -> int
-//  h -> half (__fp16, OpenCL)
-//  x -> half (_Float16)
-//  y -> half (__bf16)
-//  f -> float
-//  d -> double
-//  z -> size_t
-//  w -> wchar_t
-//  F -> constant CFString
-//  G -> id
-//  H -> SEL
-//  M -> struct objc_super
-//  a -> __builtin_va_list
-//  A -> "reference" to __builtin_va_list
-//  V -> Vector, followed by the number of elements and the base type.
-//  q -> Scalable vector, followed by the number of elements and the base type.
-//  Q -> target builtin type, followed by a character to distinguish the builtin type
-//    Qa -> AArch64 svcount_t builtin type.
-//  E -> ext_vector, followed by the number of elements and the base type.
-//  X -> _Complex, followed by the base type.
-//  Y -> ptrdiff_t
-//  P -> FILE
-//  J -> jmp_buf
-//  SJ -> sigjmp_buf
-//  K -> ucontext_t
-//  p -> pid_t
-//  . -> "...".  This may only occur at the end of the function list.
-//
-// Types may be prefixed with the following modifiers:
-//  L   -> long (e.g. Li for 'long int', Ld for 'long double')
-//  LL  -> long long (e.g. LLi for 'long long int', LLd for __float128)
-//  LLL -> __int128_t (e.g. LLLi)
-//  Z   -> int32_t (require a native 32-bit integer type on the target)
-//  W   -> int64_t (require a native 64-bit integer type on the target)
-//  N   -> 'int' size if target is LP64, 'L' otherwise.
-//  O   -> long for OpenCL targets, long long otherwise.
-//  S   -> signed
-//  U   -> unsigned
-//  I   -> Required to constant fold to an integer constant expression.
-//
-// Types may be postfixed with the following modifiers:
-// * -> pointer (optionally followed by an address space number, if no address
-//               space is specified than any address space will be accepted)
-// & -> reference (optionally followed by an address space number)
-// C -> const
-// D -> volatile
-// R -> restrict
-
-// The third value provided to the macro specifies information about attributes
-// of the function.  These must be kept in sync with the predicates in the
-// Builtin::Context class.  Currently we have:
-//  n -> nothrow
-//  r -> noreturn
-//  U -> pure
-//  c -> const
-//  t -> signature is meaningless, use custom typechecking
-//  T -> type is not important to semantic analysis and codegen; recognize as
-//       builtin even if type doesn't match signature, and don't warn if we
-//       can't be sure the type is right
-//  F -> this is a libc/libm function with a '__builtin_' prefix added.
-//  f -> this is a libc/libm function without a '__builtin_' prefix, or with
-//       'z', a C++ standard library function in namespace std::. This builtin
-//       is disableable by '-fno-builtin-foo' / '-fno-builtin-std-foo'.
-//  h -> this function requires a specific header or an explicit declaration.
-//  i -> this is a runtime library implemented function without the
-//       '__builtin_' prefix. It will be implemented in compiler-rt or libgcc.
-//  p:N: -> this is a printf-like function whose Nth argument is the format
-//          string.
-//  P:N: -> similar to the p:N: attribute, but the function is like vprintf
-//          in that it accepts its arguments as a va_list rather than
-//          through an ellipsis
-//  s:N: -> this is a scanf-like function whose Nth argument is the format
-//          string.
-//  S:N: -> similar to the s:N: attribute, but the function is like vscanf
-//          in that it accepts its arguments as a va_list rather than
-//          through an ellipsis
-//  e -> const, but only when -fno-math-errno and FP exceptions are ignored
-//  g -> const when FP exceptions are ignored
-//  j -> returns_twice (like setjmp)
-//  u -> arguments are not evaluated for their side-effects
-//  V:N: -> requires vectors of at least N bits to be legal
-//  C -> callback behavior: argument N is called with argument
-//                      M_0, ..., M_k as payload
-//  z -> this is a function in (possibly-versioned) namespace std
-//  E -> this function can be constant evaluated by Clang frontend
-- 
GitLab


From a456885efc95eb141d051ade517432c531d427f7 Mon Sep 17 00:00:00 2001
From: Craig Topper 
Date: Fri, 8 Mar 2024 10:17:54 -0800
Subject: [PATCH 641/929] [SelectionDAG] Allow FREEZE to be hoisted before
 integer SETCC. (#84241)

Teach canCreateUndefOrPoison that ISD::SETCC with integer operands can
never create undef/poison. FP SETCC is more complicated and will be
handled in a future patch.

Teach isGuaranteedNotToBeUndefOrPoison that ISD::CONDCODE is not
poison/undef. Its a special constant only used by setcc/select_cc like
nodes. This is needed since the hoisting will only hoist if exactly one
operand might be poison. setcc has 3 operand including the condition
code.

Recovers some regression from #84232.
---
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |   6 +
 llvm/test/CodeGen/RISCV/alu64.ll              |   3 +-
 llvm/test/CodeGen/RISCV/double-convert.ll     |  40 +++---
 llvm/test/CodeGen/RISCV/forced-atomics.ll     |   7 +-
 llvm/test/CodeGen/RISCV/fpclamptosat.ll       | 114 +++++++-----------
 .../CodeGen/RISCV/rvv/fpclamptosat_vec.ll     |  96 +++++++--------
 .../CodeGen/RISCV/signed-truncation-check.ll  |   9 +-
 7 files changed, 128 insertions(+), 147 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index f7ace79e8c51..50f53bbb04b6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -4998,6 +4998,7 @@ bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
     return true;
 
   switch (Opcode) {
+  case ISD::CONDCODE:
   case ISD::VALUETYPE:
   case ISD::FrameIndex:
   case ISD::TargetFrameIndex:
@@ -5090,6 +5091,11 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
   case ISD::BUILD_PAIR:
     return false;
 
+  case ISD::SETCC:
+    // Integer setcc cannot create undef or poison.
+    // FIXME: Support FP.
+    return !Op.getOperand(0).getValueType().isInteger();
+
   // Matches hasPoisonGeneratingFlags().
   case ISD::ZERO_EXTEND:
     return ConsiderFlags && Op->getFlags().hasNonNeg();
diff --git a/llvm/test/CodeGen/RISCV/alu64.ll b/llvm/test/CodeGen/RISCV/alu64.ll
index e16f6abcca24..d2ee80e6aa95 100644
--- a/llvm/test/CodeGen/RISCV/alu64.ll
+++ b/llvm/test/CodeGen/RISCV/alu64.ll
@@ -57,9 +57,8 @@ define i64 @sltiu(i64 %a) nounwind {
 ;
 ; RV32I-LABEL: sltiu:
 ; RV32I:       # %bb.0:
+; RV32I-NEXT:    seqz a1, a1
 ; RV32I-NEXT:    sltiu a0, a0, 3
-; RV32I-NEXT:    snez a1, a1
-; RV32I-NEXT:    addi a1, a1, -1
 ; RV32I-NEXT:    and a0, a1, a0
 ; RV32I-NEXT:    li a1, 0
 ; RV32I-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/double-convert.ll b/llvm/test/CodeGen/RISCV/double-convert.ll
index f2e37f55521b..93cc32e76af4 100644
--- a/llvm/test/CodeGen/RISCV/double-convert.ll
+++ b/llvm/test/CodeGen/RISCV/double-convert.ll
@@ -868,32 +868,33 @@ define i64 @fcvt_l_d_sat(double %a) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s4, 8(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s5, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 0(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lui a3, 278016
-; RV32I-NEXT:    addi a3, a3, -1
-; RV32I-NEXT:    li a2, -1
-; RV32I-NEXT:    call __gtdf2
-; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    lui a3, 802304
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s0
 ; RV32I-NEXT:    li a2, 0
 ; RV32I-NEXT:    call __gedf2
-; RV32I-NEXT:    mv s3, a0
+; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s0
 ; RV32I-NEXT:    call __fixdfdi
-; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    mv s5, a1
-; RV32I-NEXT:    lui a0, 524288
-; RV32I-NEXT:    bgez s3, .LBB12_2
+; RV32I-NEXT:    mv s3, a0
+; RV32I-NEXT:    mv s4, a1
+; RV32I-NEXT:    lui s6, 524288
+; RV32I-NEXT:    bgez s2, .LBB12_2
 ; RV32I-NEXT:  # %bb.1: # %start
-; RV32I-NEXT:    lui s5, 524288
+; RV32I-NEXT:    lui s4, 524288
 ; RV32I-NEXT:  .LBB12_2: # %start
-; RV32I-NEXT:    blez s2, .LBB12_4
+; RV32I-NEXT:    lui a3, 278016
+; RV32I-NEXT:    addi a3, a3, -1
+; RV32I-NEXT:    li a2, -1
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s0
+; RV32I-NEXT:    call __gtdf2
+; RV32I-NEXT:    mv s5, a0
+; RV32I-NEXT:    blez a0, .LBB12_4
 ; RV32I-NEXT:  # %bb.3: # %start
-; RV32I-NEXT:    addi s5, a0, -1
+; RV32I-NEXT:    addi s4, s6, -1
 ; RV32I-NEXT:  .LBB12_4: # %start
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s0
@@ -902,11 +903,11 @@ define i64 @fcvt_l_d_sat(double %a) nounwind {
 ; RV32I-NEXT:    call __unorddf2
 ; RV32I-NEXT:    snez a0, a0
 ; RV32I-NEXT:    addi a0, a0, -1
-; RV32I-NEXT:    and a1, a0, s5
-; RV32I-NEXT:    slti a2, s3, 0
+; RV32I-NEXT:    and a1, a0, s4
+; RV32I-NEXT:    slti a2, s2, 0
 ; RV32I-NEXT:    addi a2, a2, -1
-; RV32I-NEXT:    and a2, a2, s4
-; RV32I-NEXT:    sgtz a3, s2
+; RV32I-NEXT:    and a2, a2, s3
+; RV32I-NEXT:    sgtz a3, s5
 ; RV32I-NEXT:    neg a3, a3
 ; RV32I-NEXT:    or a2, a3, a2
 ; RV32I-NEXT:    and a0, a0, a2
@@ -917,6 +918,7 @@ define i64 @fcvt_l_d_sat(double %a) nounwind {
 ; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s4, 8(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s5, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s6, 0(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/forced-atomics.ll b/llvm/test/CodeGen/RISCV/forced-atomics.ll
index 659e0748dd53..c303690aadff 100644
--- a/llvm/test/CodeGen/RISCV/forced-atomics.ll
+++ b/llvm/test/CodeGen/RISCV/forced-atomics.ll
@@ -3567,8 +3567,8 @@ define i64 @rmw64_umax_seq_cst(ptr %p) nounwind {
 ; RV32-NEXT:    # in Loop: Header=BB51_2 Depth=1
 ; RV32-NEXT:    neg a3, a0
 ; RV32-NEXT:    and a3, a3, a1
-; RV32-NEXT:    sw a4, 0(sp)
 ; RV32-NEXT:    sw a1, 4(sp)
+; RV32-NEXT:    sw a4, 0(sp)
 ; RV32-NEXT:    mv a1, sp
 ; RV32-NEXT:    li a4, 5
 ; RV32-NEXT:    li a5, 5
@@ -3659,8 +3659,8 @@ define i64 @rmw64_umin_seq_cst(ptr %p) nounwind {
 ; RV32-NEXT:    # in Loop: Header=BB52_2 Depth=1
 ; RV32-NEXT:    neg a3, a0
 ; RV32-NEXT:    and a3, a3, a1
-; RV32-NEXT:    sw a4, 0(sp)
 ; RV32-NEXT:    sw a1, 4(sp)
+; RV32-NEXT:    sw a4, 0(sp)
 ; RV32-NEXT:    mv a1, sp
 ; RV32-NEXT:    li a4, 5
 ; RV32-NEXT:    li a5, 5
@@ -3672,8 +3672,7 @@ define i64 @rmw64_umin_seq_cst(ptr %p) nounwind {
 ; RV32-NEXT:  .LBB52_2: # %atomicrmw.start
 ; RV32-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32-NEXT:    sltiu a0, a4, 2
-; RV32-NEXT:    snez a2, a1
-; RV32-NEXT:    addi a2, a2, -1
+; RV32-NEXT:    seqz a2, a1
 ; RV32-NEXT:    and a0, a2, a0
 ; RV32-NEXT:    mv a2, a4
 ; RV32-NEXT:    bnez a0, .LBB52_1
diff --git a/llvm/test/CodeGen/RISCV/fpclamptosat.ll b/llvm/test/CodeGen/RISCV/fpclamptosat.ll
index 630d16e7c888..06ab813faf02 100644
--- a/llvm/test/CodeGen/RISCV/fpclamptosat.ll
+++ b/llvm/test/CodeGen/RISCV/fpclamptosat.ll
@@ -114,9 +114,8 @@ define i32 @utest_f64i32(double %x) {
 ; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    .cfi_offset ra, -4
 ; RV32IF-NEXT:    call __fixunsdfdi
+; RV32IF-NEXT:    seqz a1, a1
 ; RV32IF-NEXT:    sltiu a2, a0, -1
-; RV32IF-NEXT:    snez a1, a1
-; RV32IF-NEXT:    addi a1, a1, -1
 ; RV32IF-NEXT:    and a1, a1, a2
 ; RV32IF-NEXT:    addi a1, a1, -1
 ; RV32IF-NEXT:    or a0, a1, a0
@@ -430,9 +429,8 @@ define i32 @utesth_f16i32(half %x) {
 ; RV32-NEXT:    .cfi_offset ra, -4
 ; RV32-NEXT:    call __extendhfsf2
 ; RV32-NEXT:    call __fixunssfdi
+; RV32-NEXT:    seqz a1, a1
 ; RV32-NEXT:    sltiu a2, a0, -1
-; RV32-NEXT:    snez a1, a1
-; RV32-NEXT:    addi a1, a1, -1
 ; RV32-NEXT:    and a1, a1, a2
 ; RV32-NEXT:    addi a1, a1, -1
 ; RV32-NEXT:    or a0, a1, a0
@@ -1229,10 +1227,8 @@ define i64 @utest_f64i64(double %x) {
 ; RV32IF-NEXT:    lw a1, 20(sp)
 ; RV32IF-NEXT:    lw a2, 12(sp)
 ; RV32IF-NEXT:    lw a3, 8(sp)
-; RV32IF-NEXT:    seqz a4, a0
-; RV32IF-NEXT:    snez a5, a1
-; RV32IF-NEXT:    addi a5, a5, -1
-; RV32IF-NEXT:    and a4, a5, a4
+; RV32IF-NEXT:    or a4, a1, a0
+; RV32IF-NEXT:    seqz a4, a4
 ; RV32IF-NEXT:    xori a0, a0, 1
 ; RV32IF-NEXT:    or a0, a0, a1
 ; RV32IF-NEXT:    seqz a0, a0
@@ -1271,10 +1267,8 @@ define i64 @utest_f64i64(double %x) {
 ; RV32IFD-NEXT:    lw a1, 20(sp)
 ; RV32IFD-NEXT:    lw a2, 12(sp)
 ; RV32IFD-NEXT:    lw a3, 8(sp)
-; RV32IFD-NEXT:    seqz a4, a0
-; RV32IFD-NEXT:    snez a5, a1
-; RV32IFD-NEXT:    addi a5, a5, -1
-; RV32IFD-NEXT:    and a4, a5, a4
+; RV32IFD-NEXT:    or a4, a1, a0
+; RV32IFD-NEXT:    seqz a4, a4
 ; RV32IFD-NEXT:    xori a0, a0, 1
 ; RV32IFD-NEXT:    or a0, a0, a1
 ; RV32IFD-NEXT:    seqz a0, a0
@@ -1324,8 +1318,8 @@ define i64 @ustest_f64i64(double %x) {
 ; RV32IF-NEXT:  # %bb.4: # %entry
 ; RV32IF-NEXT:    li a0, 1
 ; RV32IF-NEXT:  .LBB20_5: # %entry
-; RV32IF-NEXT:    lw a4, 8(sp)
-; RV32IF-NEXT:    lw a3, 12(sp)
+; RV32IF-NEXT:    lw a3, 8(sp)
+; RV32IF-NEXT:    lw a4, 12(sp)
 ; RV32IF-NEXT:    and a5, a2, a1
 ; RV32IF-NEXT:    beqz a5, .LBB20_7
 ; RV32IF-NEXT:  # %bb.6: # %entry
@@ -1334,18 +1328,17 @@ define i64 @ustest_f64i64(double %x) {
 ; RV32IF-NEXT:  .LBB20_7:
 ; RV32IF-NEXT:    snez a1, a0
 ; RV32IF-NEXT:  .LBB20_8: # %entry
-; RV32IF-NEXT:    and a3, a2, a3
+; RV32IF-NEXT:    and a4, a2, a4
 ; RV32IF-NEXT:    or a0, a0, a5
-; RV32IF-NEXT:    and a2, a2, a4
+; RV32IF-NEXT:    and a2, a2, a3
 ; RV32IF-NEXT:    bnez a0, .LBB20_10
 ; RV32IF-NEXT:  # %bb.9:
-; RV32IF-NEXT:    snez a0, a3
-; RV32IF-NEXT:    snez a1, a2
-; RV32IF-NEXT:    or a1, a1, a0
+; RV32IF-NEXT:    or a0, a2, a4
+; RV32IF-NEXT:    snez a1, a0
 ; RV32IF-NEXT:  .LBB20_10: # %entry
 ; RV32IF-NEXT:    neg a1, a1
 ; RV32IF-NEXT:    and a0, a1, a2
-; RV32IF-NEXT:    and a1, a1, a3
+; RV32IF-NEXT:    and a1, a1, a4
 ; RV32IF-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 32
 ; RV32IF-NEXT:    ret
@@ -1404,8 +1397,8 @@ define i64 @ustest_f64i64(double %x) {
 ; RV32IFD-NEXT:  # %bb.4: # %entry
 ; RV32IFD-NEXT:    li a0, 1
 ; RV32IFD-NEXT:  .LBB20_5: # %entry
-; RV32IFD-NEXT:    lw a4, 8(sp)
-; RV32IFD-NEXT:    lw a3, 12(sp)
+; RV32IFD-NEXT:    lw a3, 8(sp)
+; RV32IFD-NEXT:    lw a4, 12(sp)
 ; RV32IFD-NEXT:    and a5, a2, a1
 ; RV32IFD-NEXT:    beqz a5, .LBB20_7
 ; RV32IFD-NEXT:  # %bb.6: # %entry
@@ -1414,18 +1407,17 @@ define i64 @ustest_f64i64(double %x) {
 ; RV32IFD-NEXT:  .LBB20_7:
 ; RV32IFD-NEXT:    snez a1, a0
 ; RV32IFD-NEXT:  .LBB20_8: # %entry
-; RV32IFD-NEXT:    and a3, a2, a3
+; RV32IFD-NEXT:    and a4, a2, a4
 ; RV32IFD-NEXT:    or a0, a0, a5
-; RV32IFD-NEXT:    and a2, a2, a4
+; RV32IFD-NEXT:    and a2, a2, a3
 ; RV32IFD-NEXT:    bnez a0, .LBB20_10
 ; RV32IFD-NEXT:  # %bb.9:
-; RV32IFD-NEXT:    snez a0, a3
-; RV32IFD-NEXT:    snez a1, a2
-; RV32IFD-NEXT:    or a1, a1, a0
+; RV32IFD-NEXT:    or a0, a2, a4
+; RV32IFD-NEXT:    snez a1, a0
 ; RV32IFD-NEXT:  .LBB20_10: # %entry
 ; RV32IFD-NEXT:    neg a1, a1
 ; RV32IFD-NEXT:    and a0, a1, a2
-; RV32IFD-NEXT:    and a1, a1, a3
+; RV32IFD-NEXT:    and a1, a1, a4
 ; RV32IFD-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 32
 ; RV32IFD-NEXT:    ret
@@ -1531,10 +1523,8 @@ define i64 @utest_f32i64(float %x) {
 ; RV32-NEXT:    lw a1, 20(sp)
 ; RV32-NEXT:    lw a2, 12(sp)
 ; RV32-NEXT:    lw a3, 8(sp)
-; RV32-NEXT:    seqz a4, a0
-; RV32-NEXT:    snez a5, a1
-; RV32-NEXT:    addi a5, a5, -1
-; RV32-NEXT:    and a4, a5, a4
+; RV32-NEXT:    or a4, a1, a0
+; RV32-NEXT:    seqz a4, a4
 ; RV32-NEXT:    xori a0, a0, 1
 ; RV32-NEXT:    or a0, a0, a1
 ; RV32-NEXT:    seqz a0, a0
@@ -1596,8 +1586,8 @@ define i64 @ustest_f32i64(float %x) {
 ; RV32-NEXT:  # %bb.4: # %entry
 ; RV32-NEXT:    li a0, 1
 ; RV32-NEXT:  .LBB23_5: # %entry
-; RV32-NEXT:    lw a4, 8(sp)
-; RV32-NEXT:    lw a3, 12(sp)
+; RV32-NEXT:    lw a3, 8(sp)
+; RV32-NEXT:    lw a4, 12(sp)
 ; RV32-NEXT:    and a5, a2, a1
 ; RV32-NEXT:    beqz a5, .LBB23_7
 ; RV32-NEXT:  # %bb.6: # %entry
@@ -1606,18 +1596,17 @@ define i64 @ustest_f32i64(float %x) {
 ; RV32-NEXT:  .LBB23_7:
 ; RV32-NEXT:    snez a1, a0
 ; RV32-NEXT:  .LBB23_8: # %entry
-; RV32-NEXT:    and a3, a2, a3
+; RV32-NEXT:    and a4, a2, a4
 ; RV32-NEXT:    or a0, a0, a5
-; RV32-NEXT:    and a2, a2, a4
+; RV32-NEXT:    and a2, a2, a3
 ; RV32-NEXT:    bnez a0, .LBB23_10
 ; RV32-NEXT:  # %bb.9:
-; RV32-NEXT:    snez a0, a3
-; RV32-NEXT:    snez a1, a2
-; RV32-NEXT:    or a1, a1, a0
+; RV32-NEXT:    or a0, a2, a4
+; RV32-NEXT:    snez a1, a0
 ; RV32-NEXT:  .LBB23_10: # %entry
 ; RV32-NEXT:    neg a1, a1
 ; RV32-NEXT:    and a0, a1, a2
-; RV32-NEXT:    and a1, a1, a3
+; RV32-NEXT:    and a1, a1, a4
 ; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    addi sp, sp, 32
 ; RV32-NEXT:    ret
@@ -1783,10 +1772,8 @@ define i64 @utesth_f16i64(half %x) {
 ; RV32-NEXT:    lw a1, 20(sp)
 ; RV32-NEXT:    lw a2, 12(sp)
 ; RV32-NEXT:    lw a3, 8(sp)
-; RV32-NEXT:    seqz a4, a0
-; RV32-NEXT:    snez a5, a1
-; RV32-NEXT:    addi a5, a5, -1
-; RV32-NEXT:    and a4, a5, a4
+; RV32-NEXT:    or a4, a1, a0
+; RV32-NEXT:    seqz a4, a4
 ; RV32-NEXT:    xori a0, a0, 1
 ; RV32-NEXT:    or a0, a0, a1
 ; RV32-NEXT:    seqz a0, a0
@@ -1850,8 +1837,8 @@ define i64 @ustest_f16i64(half %x) {
 ; RV32-NEXT:  # %bb.4: # %entry
 ; RV32-NEXT:    li a0, 1
 ; RV32-NEXT:  .LBB26_5: # %entry
-; RV32-NEXT:    lw a4, 8(sp)
-; RV32-NEXT:    lw a3, 12(sp)
+; RV32-NEXT:    lw a3, 8(sp)
+; RV32-NEXT:    lw a4, 12(sp)
 ; RV32-NEXT:    and a5, a2, a1
 ; RV32-NEXT:    beqz a5, .LBB26_7
 ; RV32-NEXT:  # %bb.6: # %entry
@@ -1860,18 +1847,17 @@ define i64 @ustest_f16i64(half %x) {
 ; RV32-NEXT:  .LBB26_7:
 ; RV32-NEXT:    snez a1, a0
 ; RV32-NEXT:  .LBB26_8: # %entry
-; RV32-NEXT:    and a3, a2, a3
+; RV32-NEXT:    and a4, a2, a4
 ; RV32-NEXT:    or a0, a0, a5
-; RV32-NEXT:    and a2, a2, a4
+; RV32-NEXT:    and a2, a2, a3
 ; RV32-NEXT:    bnez a0, .LBB26_10
 ; RV32-NEXT:  # %bb.9:
-; RV32-NEXT:    snez a0, a3
-; RV32-NEXT:    snez a1, a2
-; RV32-NEXT:    or a1, a1, a0
+; RV32-NEXT:    or a0, a2, a4
+; RV32-NEXT:    snez a1, a0
 ; RV32-NEXT:  .LBB26_10: # %entry
 ; RV32-NEXT:    neg a1, a1
 ; RV32-NEXT:    and a0, a1, a2
-; RV32-NEXT:    and a1, a1, a3
+; RV32-NEXT:    and a1, a1, a4
 ; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    addi sp, sp, 32
 ; RV32-NEXT:    ret
@@ -3087,10 +3073,8 @@ define i64 @utest_f64i64_mm(double %x) {
 ; RV32IF-NEXT:    lw a1, 20(sp)
 ; RV32IF-NEXT:    lw a2, 12(sp)
 ; RV32IF-NEXT:    lw a3, 8(sp)
-; RV32IF-NEXT:    seqz a4, a0
-; RV32IF-NEXT:    snez a5, a1
-; RV32IF-NEXT:    addi a5, a5, -1
-; RV32IF-NEXT:    and a4, a5, a4
+; RV32IF-NEXT:    or a4, a1, a0
+; RV32IF-NEXT:    seqz a4, a4
 ; RV32IF-NEXT:    xori a0, a0, 1
 ; RV32IF-NEXT:    or a0, a0, a1
 ; RV32IF-NEXT:    seqz a0, a0
@@ -3129,10 +3113,8 @@ define i64 @utest_f64i64_mm(double %x) {
 ; RV32IFD-NEXT:    lw a1, 20(sp)
 ; RV32IFD-NEXT:    lw a2, 12(sp)
 ; RV32IFD-NEXT:    lw a3, 8(sp)
-; RV32IFD-NEXT:    seqz a4, a0
-; RV32IFD-NEXT:    snez a5, a1
-; RV32IFD-NEXT:    addi a5, a5, -1
-; RV32IFD-NEXT:    and a4, a5, a4
+; RV32IFD-NEXT:    or a4, a1, a0
+; RV32IFD-NEXT:    seqz a4, a4
 ; RV32IFD-NEXT:    xori a0, a0, 1
 ; RV32IFD-NEXT:    or a0, a0, a1
 ; RV32IFD-NEXT:    seqz a0, a0
@@ -3345,10 +3327,8 @@ define i64 @utest_f32i64_mm(float %x) {
 ; RV32-NEXT:    lw a1, 20(sp)
 ; RV32-NEXT:    lw a2, 12(sp)
 ; RV32-NEXT:    lw a3, 8(sp)
-; RV32-NEXT:    seqz a4, a0
-; RV32-NEXT:    snez a5, a1
-; RV32-NEXT:    addi a5, a5, -1
-; RV32-NEXT:    and a4, a5, a4
+; RV32-NEXT:    or a4, a1, a0
+; RV32-NEXT:    seqz a4, a4
 ; RV32-NEXT:    xori a0, a0, 1
 ; RV32-NEXT:    or a0, a0, a1
 ; RV32-NEXT:    seqz a0, a0
@@ -3570,10 +3550,8 @@ define i64 @utesth_f16i64_mm(half %x) {
 ; RV32-NEXT:    lw a1, 20(sp)
 ; RV32-NEXT:    lw a2, 12(sp)
 ; RV32-NEXT:    lw a3, 8(sp)
-; RV32-NEXT:    seqz a4, a0
-; RV32-NEXT:    snez a5, a1
-; RV32-NEXT:    addi a5, a5, -1
-; RV32-NEXT:    and a4, a5, a4
+; RV32-NEXT:    or a4, a1, a0
+; RV32-NEXT:    seqz a4, a4
 ; RV32-NEXT:    xori a0, a0, 1
 ; RV32-NEXT:    or a0, a0, a1
 ; RV32-NEXT:    seqz a0, a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
index 3ada24bd9846..b3bda5973eb8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
@@ -5811,15 +5811,15 @@ define <2 x i64> @ustest_f64i64_mm(<2 x double> %x) {
 ; CHECK-NOV-NEXT:    mv s1, a1
 ; CHECK-NOV-NEXT:    fmv.d fa0, fs0
 ; CHECK-NOV-NEXT:    call __fixdfti
-; CHECK-NOV-NEXT:    mv a2, a1
+; CHECK-NOV-NEXT:    mv a2, s1
+; CHECK-NOV-NEXT:    mv a3, a1
 ; CHECK-NOV-NEXT:    blez a1, .LBB47_2
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
-; CHECK-NOV-NEXT:    li a2, 1
+; CHECK-NOV-NEXT:    li a3, 1
 ; CHECK-NOV-NEXT:  .LBB47_2: # %entry
-; CHECK-NOV-NEXT:    mv a3, s1
-; CHECK-NOV-NEXT:    blez s1, .LBB47_4
+; CHECK-NOV-NEXT:    blez a2, .LBB47_4
 ; CHECK-NOV-NEXT:  # %bb.3: # %entry
-; CHECK-NOV-NEXT:    li a3, 1
+; CHECK-NOV-NEXT:    li a2, 1
 ; CHECK-NOV-NEXT:  .LBB47_4: # %entry
 ; CHECK-NOV-NEXT:    slti a1, a1, 1
 ; CHECK-NOV-NEXT:    neg a1, a1
@@ -5827,11 +5827,11 @@ define <2 x i64> @ustest_f64i64_mm(<2 x double> %x) {
 ; CHECK-NOV-NEXT:    slti a0, s1, 1
 ; CHECK-NOV-NEXT:    neg a0, a0
 ; CHECK-NOV-NEXT:    and a0, a0, s0
-; CHECK-NOV-NEXT:    slti a3, a3, 0
-; CHECK-NOV-NEXT:    addi a3, a3, -1
-; CHECK-NOV-NEXT:    and a0, a3, a0
 ; CHECK-NOV-NEXT:    slti a2, a2, 0
 ; CHECK-NOV-NEXT:    addi a2, a2, -1
+; CHECK-NOV-NEXT:    and a0, a2, a0
+; CHECK-NOV-NEXT:    slti a2, a3, 0
+; CHECK-NOV-NEXT:    addi a2, a2, -1
 ; CHECK-NOV-NEXT:    and a1, a2, a1
 ; CHECK-NOV-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
@@ -5867,15 +5867,15 @@ define <2 x i64> @ustest_f64i64_mm(<2 x double> %x) {
 ; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
 ; CHECK-V-NEXT:    vfmv.f.s fa0, v8
 ; CHECK-V-NEXT:    call __fixdfti
-; CHECK-V-NEXT:    mv a2, a1
+; CHECK-V-NEXT:    mv a2, s1
+; CHECK-V-NEXT:    mv a3, a1
 ; CHECK-V-NEXT:    blez a1, .LBB47_2
 ; CHECK-V-NEXT:  # %bb.1: # %entry
-; CHECK-V-NEXT:    li a2, 1
+; CHECK-V-NEXT:    li a3, 1
 ; CHECK-V-NEXT:  .LBB47_2: # %entry
-; CHECK-V-NEXT:    mv a3, s1
-; CHECK-V-NEXT:    blez s1, .LBB47_4
+; CHECK-V-NEXT:    blez a2, .LBB47_4
 ; CHECK-V-NEXT:  # %bb.3: # %entry
-; CHECK-V-NEXT:    li a3, 1
+; CHECK-V-NEXT:    li a2, 1
 ; CHECK-V-NEXT:  .LBB47_4: # %entry
 ; CHECK-V-NEXT:    slti a1, a1, 1
 ; CHECK-V-NEXT:    neg a1, a1
@@ -5883,11 +5883,11 @@ define <2 x i64> @ustest_f64i64_mm(<2 x double> %x) {
 ; CHECK-V-NEXT:    slti a1, s1, 1
 ; CHECK-V-NEXT:    neg a1, a1
 ; CHECK-V-NEXT:    and a1, a1, s0
-; CHECK-V-NEXT:    slti a3, a3, 0
-; CHECK-V-NEXT:    addi a3, a3, -1
-; CHECK-V-NEXT:    and a1, a3, a1
 ; CHECK-V-NEXT:    slti a2, a2, 0
 ; CHECK-V-NEXT:    addi a2, a2, -1
+; CHECK-V-NEXT:    and a1, a2, a1
+; CHECK-V-NEXT:    slti a2, a3, 0
+; CHECK-V-NEXT:    addi a2, a2, -1
 ; CHECK-V-NEXT:    and a0, a2, a0
 ; CHECK-V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-V-NEXT:    vmv.s.x v8, a0
@@ -6197,15 +6197,15 @@ define <2 x i64> @ustest_f32i64_mm(<2 x float> %x) {
 ; CHECK-NOV-NEXT:    mv s1, a1
 ; CHECK-NOV-NEXT:    fmv.s fa0, fs0
 ; CHECK-NOV-NEXT:    call __fixsfti
-; CHECK-NOV-NEXT:    mv a2, a1
+; CHECK-NOV-NEXT:    mv a2, s1
+; CHECK-NOV-NEXT:    mv a3, a1
 ; CHECK-NOV-NEXT:    blez a1, .LBB50_2
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
-; CHECK-NOV-NEXT:    li a2, 1
+; CHECK-NOV-NEXT:    li a3, 1
 ; CHECK-NOV-NEXT:  .LBB50_2: # %entry
-; CHECK-NOV-NEXT:    mv a3, s1
-; CHECK-NOV-NEXT:    blez s1, .LBB50_4
+; CHECK-NOV-NEXT:    blez a2, .LBB50_4
 ; CHECK-NOV-NEXT:  # %bb.3: # %entry
-; CHECK-NOV-NEXT:    li a3, 1
+; CHECK-NOV-NEXT:    li a2, 1
 ; CHECK-NOV-NEXT:  .LBB50_4: # %entry
 ; CHECK-NOV-NEXT:    slti a1, a1, 1
 ; CHECK-NOV-NEXT:    neg a1, a1
@@ -6213,11 +6213,11 @@ define <2 x i64> @ustest_f32i64_mm(<2 x float> %x) {
 ; CHECK-NOV-NEXT:    slti a0, s1, 1
 ; CHECK-NOV-NEXT:    neg a0, a0
 ; CHECK-NOV-NEXT:    and a0, a0, s0
-; CHECK-NOV-NEXT:    slti a3, a3, 0
-; CHECK-NOV-NEXT:    addi a3, a3, -1
-; CHECK-NOV-NEXT:    and a0, a3, a0
 ; CHECK-NOV-NEXT:    slti a2, a2, 0
 ; CHECK-NOV-NEXT:    addi a2, a2, -1
+; CHECK-NOV-NEXT:    and a0, a2, a0
+; CHECK-NOV-NEXT:    slti a2, a3, 0
+; CHECK-NOV-NEXT:    addi a2, a2, -1
 ; CHECK-NOV-NEXT:    and a1, a2, a1
 ; CHECK-NOV-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
@@ -6253,15 +6253,15 @@ define <2 x i64> @ustest_f32i64_mm(<2 x float> %x) {
 ; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
 ; CHECK-V-NEXT:    vfmv.f.s fa0, v8
 ; CHECK-V-NEXT:    call __fixsfti
-; CHECK-V-NEXT:    mv a2, a1
+; CHECK-V-NEXT:    mv a2, s1
+; CHECK-V-NEXT:    mv a3, a1
 ; CHECK-V-NEXT:    blez a1, .LBB50_2
 ; CHECK-V-NEXT:  # %bb.1: # %entry
-; CHECK-V-NEXT:    li a2, 1
+; CHECK-V-NEXT:    li a3, 1
 ; CHECK-V-NEXT:  .LBB50_2: # %entry
-; CHECK-V-NEXT:    mv a3, s1
-; CHECK-V-NEXT:    blez s1, .LBB50_4
+; CHECK-V-NEXT:    blez a2, .LBB50_4
 ; CHECK-V-NEXT:  # %bb.3: # %entry
-; CHECK-V-NEXT:    li a3, 1
+; CHECK-V-NEXT:    li a2, 1
 ; CHECK-V-NEXT:  .LBB50_4: # %entry
 ; CHECK-V-NEXT:    slti a1, a1, 1
 ; CHECK-V-NEXT:    neg a1, a1
@@ -6269,11 +6269,11 @@ define <2 x i64> @ustest_f32i64_mm(<2 x float> %x) {
 ; CHECK-V-NEXT:    slti a1, s1, 1
 ; CHECK-V-NEXT:    neg a1, a1
 ; CHECK-V-NEXT:    and a1, a1, s0
-; CHECK-V-NEXT:    slti a3, a3, 0
-; CHECK-V-NEXT:    addi a3, a3, -1
-; CHECK-V-NEXT:    and a1, a3, a1
 ; CHECK-V-NEXT:    slti a2, a2, 0
 ; CHECK-V-NEXT:    addi a2, a2, -1
+; CHECK-V-NEXT:    and a1, a2, a1
+; CHECK-V-NEXT:    slti a2, a3, 0
+; CHECK-V-NEXT:    addi a2, a2, -1
 ; CHECK-V-NEXT:    and a0, a2, a0
 ; CHECK-V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-V-NEXT:    vmv.s.x v8, a0
@@ -6575,15 +6575,15 @@ define <2 x i64> @ustest_f16i64_mm(<2 x half> %x) {
 ; CHECK-NOV-NEXT:    fmv.w.x fa0, s2
 ; CHECK-NOV-NEXT:    call __extendhfsf2
 ; CHECK-NOV-NEXT:    call __fixsfti
-; CHECK-NOV-NEXT:    mv a2, a1
+; CHECK-NOV-NEXT:    mv a2, s1
+; CHECK-NOV-NEXT:    mv a3, a1
 ; CHECK-NOV-NEXT:    blez a1, .LBB53_2
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
-; CHECK-NOV-NEXT:    li a2, 1
+; CHECK-NOV-NEXT:    li a3, 1
 ; CHECK-NOV-NEXT:  .LBB53_2: # %entry
-; CHECK-NOV-NEXT:    mv a3, s1
-; CHECK-NOV-NEXT:    blez s1, .LBB53_4
+; CHECK-NOV-NEXT:    blez a2, .LBB53_4
 ; CHECK-NOV-NEXT:  # %bb.3: # %entry
-; CHECK-NOV-NEXT:    li a3, 1
+; CHECK-NOV-NEXT:    li a2, 1
 ; CHECK-NOV-NEXT:  .LBB53_4: # %entry
 ; CHECK-NOV-NEXT:    slti a1, a1, 1
 ; CHECK-NOV-NEXT:    neg a1, a1
@@ -6591,11 +6591,11 @@ define <2 x i64> @ustest_f16i64_mm(<2 x half> %x) {
 ; CHECK-NOV-NEXT:    slti a0, s1, 1
 ; CHECK-NOV-NEXT:    neg a0, a0
 ; CHECK-NOV-NEXT:    and a0, a0, s0
-; CHECK-NOV-NEXT:    slti a3, a3, 0
-; CHECK-NOV-NEXT:    addi a3, a3, -1
-; CHECK-NOV-NEXT:    and a0, a3, a0
 ; CHECK-NOV-NEXT:    slti a2, a2, 0
 ; CHECK-NOV-NEXT:    addi a2, a2, -1
+; CHECK-NOV-NEXT:    and a0, a2, a0
+; CHECK-NOV-NEXT:    slti a2, a3, 0
+; CHECK-NOV-NEXT:    addi a2, a2, -1
 ; CHECK-NOV-NEXT:    and a1, a2, a1
 ; CHECK-NOV-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
@@ -6625,15 +6625,15 @@ define <2 x i64> @ustest_f16i64_mm(<2 x half> %x) {
 ; CHECK-V-NEXT:    fmv.w.x fa0, s2
 ; CHECK-V-NEXT:    call __extendhfsf2
 ; CHECK-V-NEXT:    call __fixsfti
-; CHECK-V-NEXT:    mv a2, a1
+; CHECK-V-NEXT:    mv a2, s1
+; CHECK-V-NEXT:    mv a3, a1
 ; CHECK-V-NEXT:    blez a1, .LBB53_2
 ; CHECK-V-NEXT:  # %bb.1: # %entry
-; CHECK-V-NEXT:    li a2, 1
+; CHECK-V-NEXT:    li a3, 1
 ; CHECK-V-NEXT:  .LBB53_2: # %entry
-; CHECK-V-NEXT:    mv a3, s1
-; CHECK-V-NEXT:    blez s1, .LBB53_4
+; CHECK-V-NEXT:    blez a2, .LBB53_4
 ; CHECK-V-NEXT:  # %bb.3: # %entry
-; CHECK-V-NEXT:    li a3, 1
+; CHECK-V-NEXT:    li a2, 1
 ; CHECK-V-NEXT:  .LBB53_4: # %entry
 ; CHECK-V-NEXT:    slti a1, a1, 1
 ; CHECK-V-NEXT:    neg a1, a1
@@ -6641,11 +6641,11 @@ define <2 x i64> @ustest_f16i64_mm(<2 x half> %x) {
 ; CHECK-V-NEXT:    slti a1, s1, 1
 ; CHECK-V-NEXT:    neg a1, a1
 ; CHECK-V-NEXT:    and a1, a1, s0
-; CHECK-V-NEXT:    slti a3, a3, 0
-; CHECK-V-NEXT:    addi a3, a3, -1
-; CHECK-V-NEXT:    and a1, a3, a1
 ; CHECK-V-NEXT:    slti a2, a2, 0
 ; CHECK-V-NEXT:    addi a2, a2, -1
+; CHECK-V-NEXT:    and a1, a2, a1
+; CHECK-V-NEXT:    slti a2, a3, 0
+; CHECK-V-NEXT:    addi a2, a2, -1
 ; CHECK-V-NEXT:    and a0, a2, a0
 ; CHECK-V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-V-NEXT:    vmv.s.x v9, a0
diff --git a/llvm/test/CodeGen/RISCV/signed-truncation-check.ll b/llvm/test/CodeGen/RISCV/signed-truncation-check.ll
index 069b2febc334..de36bcdb9106 100644
--- a/llvm/test/CodeGen/RISCV/signed-truncation-check.ll
+++ b/llvm/test/CodeGen/RISCV/signed-truncation-check.ll
@@ -422,8 +422,7 @@ define i1 @add_ugecmp_i64_i16(i64 %x) nounwind {
 ; RV32I-NEXT:    lui a1, 1048560
 ; RV32I-NEXT:    addi a1, a1, -1
 ; RV32I-NEXT:    sltu a1, a1, a2
-; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    addi a0, a0, -1
+; RV32I-NEXT:    seqz a0, a0
 ; RV32I-NEXT:    and a0, a0, a1
 ; RV32I-NEXT:    ret
 ;
@@ -463,8 +462,7 @@ define i1 @add_ugecmp_i64_i8(i64 %x) nounwind {
 ; RV32I-NEXT:    addi a2, a0, -128
 ; RV32I-NEXT:    sltu a0, a2, a0
 ; RV32I-NEXT:    add a0, a1, a0
-; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    addi a0, a0, -1
+; RV32I-NEXT:    seqz a0, a0
 ; RV32I-NEXT:    sltiu a1, a2, -256
 ; RV32I-NEXT:    xori a1, a1, 1
 ; RV32I-NEXT:    and a0, a0, a1
@@ -693,8 +691,7 @@ define i1 @add_ultcmp_i64_i8(i64 %x) nounwind {
 ; RV32I-NEXT:    addi a2, a0, 128
 ; RV32I-NEXT:    sltu a0, a2, a0
 ; RV32I-NEXT:    add a0, a1, a0
-; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    addi a0, a0, -1
+; RV32I-NEXT:    seqz a0, a0
 ; RV32I-NEXT:    sltiu a1, a2, 256
 ; RV32I-NEXT:    and a0, a0, a1
 ; RV32I-NEXT:    ret
-- 
GitLab


From 839a8fecb4c5dfe1b4484d5fc942a9490867c47a Mon Sep 17 00:00:00 2001
From: Changpeng Fang 
Date: Fri, 8 Mar 2024 10:30:01 -0800
Subject: [PATCH 642/929] AMDGPU: Copy SubtargetPredicate from pseudo to real
 for dpp16 and dpp8 (#84517)

We usually expect to copy SubtargetPredicate (and OtherPredicates) from
pseudo to real. However, in dpp16 and dpp8, there are assignments like
SubtargetPredicate = HasDPP/HasDPP16/HasDpp8. These assignments override
predicates copied from pseudo, and thus the predicates used to define
pseudo get lost.

Losing predicates is a subtle issue usually not easy to be found. It may
result in instructions being generated on GPUs that do not support the
features to generate them.
https://github.com/llvm/llvm-project/pull/84354 addressed one of such
issues, and inspired this work.

Fortunately, we found that the assignment of SubtargetPredicate usually
comes together with assignment of AssemblerPredicate, and with the same
value. For example:
  let AssemblerPredicate = HasDPP16;
  let SubtargetPredicate = HasDPP16;
One of them is redundant and can be removed.

In this work, we remove the redundant assignment of SubtargetPredicate,
and then copy it from pseudo for VOP*_DPP and VOP*_DPP8. With this
change, we can safely use SubtargetPredicate to define pseudo
instructions.
---
 llvm/lib/Target/AMDGPU/VOP1Instructions.td  | 3 ++-
 llvm/lib/Target/AMDGPU/VOP2Instructions.td  | 3 ++-
 llvm/lib/Target/AMDGPU/VOP3PInstructions.td | 3 ++-
 llvm/lib/Target/AMDGPU/VOPInstructions.td   | 6 ++----
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index dbb1977183d1..2341e0d9d32b 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -730,6 +730,7 @@ class VOP1_DPP op, VOP1_DPP_Pseudo ps, VOPProfile p = ps.Pfl, bit isDPP1
   let SchedRW = ps.SchedRW;
   let Uses = ps.Uses;
   let TRANS = ps.TRANS;
+  let SubtargetPredicate = ps.SubtargetPredicate;
   let OtherPredicates = ps.OtherPredicates;
 
   bits<8> vdst;
@@ -743,7 +744,6 @@ class VOP1_DPP16 op, VOP1_DPP_Pseudo ps, int subtarget, VOPProfile p = p
     VOP1_DPP,
     SIMCInstr  {
   let AssemblerPredicate = HasDPP16;
-  let SubtargetPredicate = HasDPP16;
 }
 
 class VOP1_DPP16_Gen op, VOP1_DPP_Pseudo ps, GFXGen Gen, VOPProfile p = ps.Pfl> :
@@ -758,6 +758,7 @@ class VOP1_DPP8 op, VOP1_Pseudo ps, VOPProfile p = ps.Pfl> :
   let Defs = ps.Defs;
   let SchedRW = ps.SchedRW;
   let Uses = ps.Uses;
+  let SubtargetPredicate = ps.SubtargetPredicate;
   let OtherPredicates = ps.OtherPredicates;
 
   bits<8> vdst;
diff --git a/llvm/lib/Target/AMDGPU/VOP2Instructions.td b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
index 53578682e002..8a92aa8228f1 100644
--- a/llvm/lib/Target/AMDGPU/VOP2Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
@@ -1259,7 +1259,7 @@ class Base_VOP2_DPP16 op, VOP2_DPP_Pseudo ps,
                  string opName = ps.OpName, VOPProfile p = ps.Pfl> :
     VOP2_DPP {
   let AssemblerPredicate = HasDPP16;
-  let SubtargetPredicate = HasDPP16;
+  let SubtargetPredicate = ps.SubtargetPredicate;
   let OtherPredicates = ps.OtherPredicates;
 }
 
@@ -1294,6 +1294,7 @@ class VOP2_DPP8 op, VOP2_Pseudo ps,
   let Inst{30-25} = op;
   let Inst{31}    = 0x0;
 
+  let SubtargetPredicate = ps.SubtargetPredicate;
   let OtherPredicates = ps.OtherPredicates;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index e1131bbb78d3..4ca2835eea58 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -1353,7 +1353,7 @@ class VOP3P_DPP16 op, VOP_DPP_Pseudo ps, int subtarget,
   let SchedRW = ps.SchedRW;
   let Uses = ps.Uses;
   let AssemblerPredicate = HasDPP16;
-  let SubtargetPredicate = HasDPP16;
+  let SubtargetPredicate = ps.SubtargetPredicate;
   let OtherPredicates = ps.OtherPredicates;
   let IsPacked = ps.IsPacked;
 }
@@ -1364,6 +1364,7 @@ class VOP3P_DPP8_Base op, VOP_Pseudo ps, string opName = ps.OpName>
   let Defs = ps.Defs;
   let SchedRW = ps.SchedRW;
   let Uses = ps.Uses;
+  let SubtargetPredicate = ps.SubtargetPredicate;
   let OtherPredicates = ps.OtherPredicates;
   let IsPacked = ps.IsPacked;
 }
diff --git a/llvm/lib/Target/AMDGPU/VOPInstructions.td b/llvm/lib/Target/AMDGPU/VOPInstructions.td
index fa8d46608f5d..a6272e946c51 100644
--- a/llvm/lib/Target/AMDGPU/VOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOPInstructions.td
@@ -832,7 +832,6 @@ class VOP_DPP_Pseudo  pattern=[],
   string AsmOperands = asmOps;
 
   let AsmMatchConverter = !if(P.HasModifiers, "cvtDPP", "");
-  let SubtargetPredicate = !if(P.HasExt64BitDPP, HasDPALU_DPP, HasDPP);
   let AssemblerPredicate = !if(P.HasExt64BitDPP, HasDPALU_DPP, HasDPP);
   let AsmVariantName = !if(P.HasExtDPP, AMDGPUAsmVariants.DPP,
                                         AMDGPUAsmVariants.Disable);
@@ -903,7 +902,6 @@ class VOP_DPP_Base  op, VOP_DPP_Pseudo ps, string opName = ps.OpName>
   let SchedRW = ps.SchedRW;
   let Uses = ps.Uses;
   let AssemblerPredicate = HasDPP16;
-  let SubtargetPredicate = HasDPP16;
+  let SubtargetPredicate = ps.SubtargetPredicate;
   let OtherPredicates = ps.OtherPredicates;
 }
 
@@ -1366,6 +1363,7 @@ class Base_VOP3_DPP8 op, VOP_Pseudo ps, string opName = ps.OpName>
   let SchedRW = ps.SchedRW;
   let Uses = ps.Uses;
 
+  let SubtargetPredicate = ps.SubtargetPredicate;
   let OtherPredicates = ps.OtherPredicates;
 }
 
-- 
GitLab


From 99118c809367d518ffe4de60c16da953744b68b9 Mon Sep 17 00:00:00 2001
From: Adrian Prantl 
Date: Fri, 8 Mar 2024 10:39:34 -0800
Subject: [PATCH 643/929] Change GetNumChildren()/CalculateNumChildren()
 methods return llvm::Expected (#84219)

Change GetNumChildren()/CalculateNumChildren() methods return
llvm::Expected

This is an NFC change that does not yet add any error handling or change
any code to return any errors.

This is the second big change in the patch series started with
https://github.com/llvm/llvm-project/pull/83501

A follow-up PR will wire up error handling.
---
 lldb/include/lldb/Core/ValueObject.h          | 11 +++-
 lldb/include/lldb/Core/ValueObjectCast.h      |  2 +-
 lldb/include/lldb/Core/ValueObjectChild.h     |  2 +-
 .../lldb/Core/ValueObjectConstResult.h        |  2 +-
 .../lldb/Core/ValueObjectDynamicValue.h       |  2 +-
 lldb/include/lldb/Core/ValueObjectMemory.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectRegister.h  |  4 +-
 .../lldb/Core/ValueObjectSyntheticFilter.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectVTable.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectVariable.h  |  2 +-
 .../lldb/DataFormatters/TypeSynthetic.h       | 20 ++++--
 .../lldb/DataFormatters/VectorIterator.h      |  2 +-
 lldb/include/lldb/Symbol/CompilerType.h       |  5 +-
 lldb/include/lldb/Symbol/Type.h               |  2 +-
 lldb/include/lldb/Symbol/TypeSystem.h         |  7 +-
 .../lldb/Target/StackFrameRecognizer.h        |  3 +-
 lldb/include/lldb/Utility/Log.h               | 14 ++++
 lldb/source/API/SBValue.cpp                   |  2 +-
 lldb/source/Core/FormatEntity.cpp             |  2 +-
 lldb/source/Core/IOHandlerCursesGUI.cpp       |  2 +-
 lldb/source/Core/ValueObject.cpp              | 31 ++++++---
 lldb/source/Core/ValueObjectCast.cpp          |  6 +-
 lldb/source/Core/ValueObjectChild.cpp         |  6 +-
 lldb/source/Core/ValueObjectConstResult.cpp   |  7 +-
 lldb/source/Core/ValueObjectDynamicValue.cpp  |  7 +-
 lldb/source/Core/ValueObjectMemory.cpp        | 10 ++-
 lldb/source/Core/ValueObjectRegister.cpp      | 12 ++--
 .../Core/ValueObjectSyntheticFilter.cpp       | 24 ++++---
 lldb/source/Core/ValueObjectVTable.cpp        |  6 +-
 lldb/source/Core/ValueObjectVariable.cpp      |  7 +-
 lldb/source/DataFormatters/FormatManager.cpp  | 11 +++-
 lldb/source/DataFormatters/TypeSynthetic.cpp  | 17 ++++-
 .../DataFormatters/ValueObjectPrinter.cpp     |  2 +-
 lldb/source/DataFormatters/VectorType.cpp     | 16 +++--
 .../Clang/ClangExpressionSourceCode.cpp       |  2 +-
 .../TSan/InstrumentationRuntimeTSan.cpp       |  4 +-
 .../Language/CPlusPlus/BlockPointer.cpp       |  4 +-
 .../Plugins/Language/CPlusPlus/Coroutines.cpp |  8 ++-
 .../Plugins/Language/CPlusPlus/Coroutines.h   |  2 +-
 .../Language/CPlusPlus/GenericBitset.cpp      |  4 +-
 .../Language/CPlusPlus/GenericOptional.cpp    |  6 +-
 .../Plugins/Language/CPlusPlus/LibCxx.cpp     | 18 +++---
 .../Plugins/Language/CPlusPlus/LibCxx.h       |  8 +--
 .../Language/CPlusPlus/LibCxxAtomic.cpp       |  6 +-
 .../CPlusPlus/LibCxxInitializerList.cpp       |  6 +-
 .../Plugins/Language/CPlusPlus/LibCxxList.cpp | 12 ++--
 .../Plugins/Language/CPlusPlus/LibCxxMap.cpp  | 14 ++--
 .../Language/CPlusPlus/LibCxxQueue.cpp        |  2 +-
 .../CPlusPlus/LibCxxRangesRefView.cpp         |  2 +-
 .../Plugins/Language/CPlusPlus/LibCxxSpan.cpp |  6 +-
 .../Language/CPlusPlus/LibCxxTuple.cpp        |  4 +-
 .../Language/CPlusPlus/LibCxxUnorderedMap.cpp |  8 +--
 .../Language/CPlusPlus/LibCxxValarray.cpp     |  6 +-
 .../Language/CPlusPlus/LibCxxVariant.cpp      |  2 +-
 .../Language/CPlusPlus/LibCxxVector.cpp       | 14 ++--
 .../Plugins/Language/CPlusPlus/LibStdcpp.cpp  | 17 +++--
 .../Language/CPlusPlus/LibStdcppTuple.cpp     |  7 +-
 .../CPlusPlus/LibStdcppUniquePointer.cpp      |  5 +-
 lldb/source/Plugins/Language/ObjC/Cocoa.cpp   |  2 +-
 lldb/source/Plugins/Language/ObjC/NSArray.cpp | 30 ++++-----
 .../Plugins/Language/ObjC/NSDictionary.cpp    | 64 +++++++++----------
 lldb/source/Plugins/Language/ObjC/NSError.cpp |  2 +-
 .../Plugins/Language/ObjC/NSException.cpp     |  4 +-
 .../Plugins/Language/ObjC/NSIndexPath.cpp     |  6 +-
 lldb/source/Plugins/Language/ObjC/NSSet.cpp   | 32 +++++-----
 .../AppleObjCRuntime/AppleObjCRuntime.cpp     |  3 +-
 .../TypeSystem/Clang/TypeSystemClang.cpp      | 40 ++++++++----
 .../TypeSystem/Clang/TypeSystemClang.h        |  7 +-
 lldb/source/Symbol/CompilerType.cpp           |  5 +-
 lldb/source/Symbol/Type.cpp                   |  2 +-
 lldb/source/Symbol/Variable.cpp               |  6 +-
 lldb/source/Target/StackFrame.cpp             | 19 +++---
 72 files changed, 394 insertions(+), 247 deletions(-)

diff --git a/lldb/include/lldb/Core/ValueObject.h b/lldb/include/lldb/Core/ValueObject.h
index b4d2c8098edc..e7e35e2b2bff 100644
--- a/lldb/include/lldb/Core/ValueObject.h
+++ b/lldb/include/lldb/Core/ValueObject.h
@@ -476,7 +476,13 @@ public:
 
   virtual size_t GetIndexOfChildWithName(llvm::StringRef name);
 
-  uint32_t GetNumChildren(uint32_t max = UINT32_MAX);
+  llvm::Expected GetNumChildren(uint32_t max = UINT32_MAX);
+  /// Like \c GetNumChildren but returns 0 on error.  You probably
+  /// shouldn't be using this function. It exists primarily to ease the
+  /// transition to more pervasive error handling while not all APIs
+  /// have been updated.
+  uint32_t GetNumChildrenIgnoringErrors(uint32_t max = UINT32_MAX);
+  bool HasChildren() { return GetNumChildrenIgnoringErrors() > 0; }
 
   const Value &GetValue() const { return m_value; }
 
@@ -958,7 +964,8 @@ protected:
                                           int32_t synthetic_index);
 
   /// Should only be called by ValueObject::GetNumChildren().
-  virtual uint32_t CalculateNumChildren(uint32_t max = UINT32_MAX) = 0;
+  virtual llvm::Expected
+  CalculateNumChildren(uint32_t max = UINT32_MAX) = 0;
 
   void SetNumChildren(uint32_t num_children);
 
diff --git a/lldb/include/lldb/Core/ValueObjectCast.h b/lldb/include/lldb/Core/ValueObjectCast.h
index 51c647680d52..ba25e166f326 100644
--- a/lldb/include/lldb/Core/ValueObjectCast.h
+++ b/lldb/include/lldb/Core/ValueObjectCast.h
@@ -33,7 +33,7 @@ public:
 
   std::optional GetByteSize() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectChild.h b/lldb/include/lldb/Core/ValueObjectChild.h
index 47a13be08bb8..1f88e607cb57 100644
--- a/lldb/include/lldb/Core/ValueObjectChild.h
+++ b/lldb/include/lldb/Core/ValueObjectChild.h
@@ -39,7 +39,7 @@ public:
 
   lldb::ValueType GetValueType() const override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectConstResult.h b/lldb/include/lldb/Core/ValueObjectConstResult.h
index 9f1246cf2a78..37dc0867f26c 100644
--- a/lldb/include/lldb/Core/ValueObjectConstResult.h
+++ b/lldb/include/lldb/Core/ValueObjectConstResult.h
@@ -67,7 +67,7 @@ public:
 
   lldb::ValueType GetValueType() const override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectDynamicValue.h b/lldb/include/lldb/Core/ValueObjectDynamicValue.h
index 21a9b409fd5b..82c20eee0cd4 100644
--- a/lldb/include/lldb/Core/ValueObjectDynamicValue.h
+++ b/lldb/include/lldb/Core/ValueObjectDynamicValue.h
@@ -43,7 +43,7 @@ public:
 
   ConstString GetDisplayTypeName() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectMemory.h b/lldb/include/lldb/Core/ValueObjectMemory.h
index a74b325546b0..a8fb0353d601 100644
--- a/lldb/include/lldb/Core/ValueObjectMemory.h
+++ b/lldb/include/lldb/Core/ValueObjectMemory.h
@@ -47,7 +47,7 @@ public:
 
   ConstString GetDisplayTypeName() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectRegister.h b/lldb/include/lldb/Core/ValueObjectRegister.h
index 6c470c1a6865..fec8566ba33d 100644
--- a/lldb/include/lldb/Core/ValueObjectRegister.h
+++ b/lldb/include/lldb/Core/ValueObjectRegister.h
@@ -47,7 +47,7 @@ public:
 
   ConstString GetQualifiedTypeName() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member,
                                   int32_t synthetic_index) override;
@@ -95,7 +95,7 @@ public:
 
   ConstString GetTypeName() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   bool SetValueFromCString(const char *value_str, Status &error) override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h b/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
index 1e54babc94f3..ca6d6c728005 100644
--- a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
+++ b/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
@@ -47,7 +47,7 @@ public:
 
   bool MightHaveChildren() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectVTable.h b/lldb/include/lldb/Core/ValueObjectVTable.h
index e7e14fc83d78..4662f395a4dd 100644
--- a/lldb/include/lldb/Core/ValueObjectVTable.h
+++ b/lldb/include/lldb/Core/ValueObjectVTable.h
@@ -64,7 +64,7 @@ public:
 
   std::optional GetByteSize() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member,
                                   int32_t synthetic_index) override;
diff --git a/lldb/include/lldb/Core/ValueObjectVariable.h b/lldb/include/lldb/Core/ValueObjectVariable.h
index da270300df0b..db3847f14a0b 100644
--- a/lldb/include/lldb/Core/ValueObjectVariable.h
+++ b/lldb/include/lldb/Core/ValueObjectVariable.h
@@ -46,7 +46,7 @@ public:
 
   ConstString GetDisplayTypeName() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/DataFormatters/TypeSynthetic.h b/lldb/include/lldb/DataFormatters/TypeSynthetic.h
index 38f3ce0fa5f0..ede7442a02bf 100644
--- a/lldb/include/lldb/DataFormatters/TypeSynthetic.h
+++ b/lldb/include/lldb/DataFormatters/TypeSynthetic.h
@@ -38,13 +38,17 @@ public:
 
   virtual ~SyntheticChildrenFrontEnd() = default;
 
-  virtual uint32_t CalculateNumChildren() = 0;
+  virtual llvm::Expected CalculateNumChildren() = 0;
 
-  virtual uint32_t CalculateNumChildren(uint32_t max) {
+  virtual llvm::Expected CalculateNumChildren(uint32_t max) {
     auto count = CalculateNumChildren();
-    return count <= max ? count : max;
+    if (!count)
+      return count;
+    return *count <= max ? *count : max;
   }
 
+  uint32_t CalculateNumChildrenIgnoringErrors(uint32_t max = UINT32_MAX);
+
   virtual lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) = 0;
 
   virtual size_t GetIndexOfChildWithName(ConstString name) = 0;
@@ -109,7 +113,7 @@ public:
 
   ~SyntheticValueProviderFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override { return 0; }
+  llvm::Expected CalculateNumChildren() override { return 0; }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override { return nullptr; }
 
@@ -322,7 +326,9 @@ public:
 
     ~FrontEnd() override = default;
 
-    uint32_t CalculateNumChildren() override { return filter->GetCount(); }
+    llvm::Expected CalculateNumChildren() override {
+      return filter->GetCount();
+    }
 
     lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
       if (idx >= filter->GetCount())
@@ -426,9 +432,9 @@ public:
 
     bool IsValid();
 
-    uint32_t CalculateNumChildren() override;
+    llvm::Expected CalculateNumChildren() override;
 
-    uint32_t CalculateNumChildren(uint32_t max) override;
+    llvm::Expected CalculateNumChildren(uint32_t max) override;
 
     lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/include/lldb/DataFormatters/VectorIterator.h b/lldb/include/lldb/DataFormatters/VectorIterator.h
index 7711b9de95db..70bcf50ca1b1 100644
--- a/lldb/include/lldb/DataFormatters/VectorIterator.h
+++ b/lldb/include/lldb/DataFormatters/VectorIterator.h
@@ -24,7 +24,7 @@ public:
   VectorIteratorSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp,
                                   llvm::ArrayRef item_names);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/include/lldb/Symbol/CompilerType.h b/lldb/include/lldb/Symbol/CompilerType.h
index 414c44275aaa..c1dce4ccbf79 100644
--- a/lldb/include/lldb/Symbol/CompilerType.h
+++ b/lldb/include/lldb/Symbol/CompilerType.h
@@ -386,8 +386,9 @@ public:
 
   std::optional GetTypeBitAlign(ExecutionContextScope *exe_scope) const;
 
-  uint32_t GetNumChildren(bool omit_empty_base_classes,
-                          const ExecutionContext *exe_ctx) const;
+  llvm::Expected
+  GetNumChildren(bool omit_empty_base_classes,
+                 const ExecutionContext *exe_ctx) const;
 
   lldb::BasicType GetBasicTypeEnumeration() const;
 
diff --git a/lldb/include/lldb/Symbol/Type.h b/lldb/include/lldb/Symbol/Type.h
index acd1a769f13c..b5eac5fa732d 100644
--- a/lldb/include/lldb/Symbol/Type.h
+++ b/lldb/include/lldb/Symbol/Type.h
@@ -440,7 +440,7 @@ public:
 
   std::optional GetByteSize(ExecutionContextScope *exe_scope);
 
-  uint32_t GetNumChildren(bool omit_empty_base_classes);
+  llvm::Expected GetNumChildren(bool omit_empty_base_classes);
 
   bool IsAggregateType();
 
diff --git a/lldb/include/lldb/Symbol/TypeSystem.h b/lldb/include/lldb/Symbol/TypeSystem.h
index 63829131556e..f647fcbf1636 100644
--- a/lldb/include/lldb/Symbol/TypeSystem.h
+++ b/lldb/include/lldb/Symbol/TypeSystem.h
@@ -300,9 +300,10 @@ public:
 
   virtual lldb::Format GetFormat(lldb::opaque_compiler_type_t type) = 0;
 
-  virtual uint32_t GetNumChildren(lldb::opaque_compiler_type_t type,
-                                  bool omit_empty_base_classes,
-                                  const ExecutionContext *exe_ctx) = 0;
+  virtual llvm::Expected
+  GetNumChildren(lldb::opaque_compiler_type_t type,
+                 bool omit_empty_base_classes,
+                 const ExecutionContext *exe_ctx) = 0;
 
   virtual CompilerType GetBuiltinTypeByName(ConstString name);
 
diff --git a/lldb/include/lldb/Target/StackFrameRecognizer.h b/lldb/include/lldb/Target/StackFrameRecognizer.h
index e111f4a4dc70..5e8e12b2a4e9 100644
--- a/lldb/include/lldb/Target/StackFrameRecognizer.h
+++ b/lldb/include/lldb/Target/StackFrameRecognizer.h
@@ -164,7 +164,8 @@ class ValueObjectRecognizerSynthesizedValue : public ValueObject {
     m_value = m_parent->GetValue();
     return true;
   }
-  uint32_t CalculateNumChildren(uint32_t max = UINT32_MAX) override {
+  llvm::Expected
+  CalculateNumChildren(uint32_t max = UINT32_MAX) override {
     return m_parent->GetNumChildren(max);
   }
   CompilerType GetCompilerTypeImpl() override {
diff --git a/lldb/include/lldb/Utility/Log.h b/lldb/include/lldb/Utility/Log.h
index 1fe28d61b9da..01876ad732d4 100644
--- a/lldb/include/lldb/Utility/Log.h
+++ b/lldb/include/lldb/Utility/Log.h
@@ -373,4 +373,18 @@ template  Log *GetLog(Cat mask) {
       ::llvm::consumeError(::std::move(error_private));                        \
   } while (0)
 
+// Write message to the verbose log, if error is set. In the log
+// message refer to the error with {0}. Error is cleared regardless of
+// whether logging is enabled.
+#define LLDB_LOG_ERRORV(log, error, ...)                                       \
+  do {                                                                         \
+    ::lldb_private::Log *log_private = (log);                                  \
+    ::llvm::Error error_private = (error);                                     \
+    if (log_private && log_private->GetVerbose() && error_private) {           \
+      log_private->FormatError(::std::move(error_private), __FILE__, __func__, \
+                               __VA_ARGS__);                                   \
+    } else                                                                     \
+      ::llvm::consumeError(::std::move(error_private));                        \
+  } while (0)
+
 #endif // LLDB_UTILITY_LOG_H
diff --git a/lldb/source/API/SBValue.cpp b/lldb/source/API/SBValue.cpp
index 89d26a1fbe28..94a8f3ea319e 100644
--- a/lldb/source/API/SBValue.cpp
+++ b/lldb/source/API/SBValue.cpp
@@ -947,7 +947,7 @@ uint32_t SBValue::GetNumChildren(uint32_t max) {
   ValueLocker locker;
   lldb::ValueObjectSP value_sp(GetSP(locker));
   if (value_sp)
-    num_children = value_sp->GetNumChildren(max);
+    num_children = value_sp->GetNumChildrenIgnoringErrors(max);
 
   return num_children;
 }
diff --git a/lldb/source/Core/FormatEntity.cpp b/lldb/source/Core/FormatEntity.cpp
index fa5eadc6ff4e..cf82676bedda 100644
--- a/lldb/source/Core/FormatEntity.cpp
+++ b/lldb/source/Core/FormatEntity.cpp
@@ -926,7 +926,7 @@ static bool DumpValue(Stream &s, const SymbolContext *sc,
     s.PutChar('[');
 
     if (index_higher < 0)
-      index_higher = valobj->GetNumChildren() - 1;
+      index_higher = valobj->GetNumChildrenIgnoringErrors() - 1;
 
     uint32_t max_num_children =
         target->GetTargetSP()->GetMaximumNumberOfChildrenToDisplay();
diff --git a/lldb/source/Core/IOHandlerCursesGUI.cpp b/lldb/source/Core/IOHandlerCursesGUI.cpp
index 620e68a28510..f86dce247135 100644
--- a/lldb/source/Core/IOHandlerCursesGUI.cpp
+++ b/lldb/source/Core/IOHandlerCursesGUI.cpp
@@ -4519,7 +4519,7 @@ struct Row {
       calculated_children = true;
       ValueObjectSP valobj = value.GetSP();
       if (valobj) {
-        const size_t num_children = valobj->GetNumChildren();
+        const uint32_t num_children = valobj->GetNumChildrenIgnoringErrors();
         for (size_t i = 0; i < num_children; ++i) {
           children.push_back(Row(valobj->GetChildAtIndex(i), this));
         }
diff --git a/lldb/source/Core/ValueObject.cpp b/lldb/source/Core/ValueObject.cpp
index 0ed7f03be25c..d813044d02ff 100644
--- a/lldb/source/Core/ValueObject.cpp
+++ b/lldb/source/Core/ValueObject.cpp
@@ -377,7 +377,7 @@ ValueObjectSP ValueObject::GetChildAtIndex(uint32_t idx, bool can_create) {
   // We may need to update our value if we are dynamic
   if (IsPossibleDynamicType())
     UpdateValueIfNeeded(false);
-  if (idx < GetNumChildren()) {
+  if (idx < GetNumChildrenIgnoringErrors()) {
     // Check if we have already made the child value object?
     if (can_create && !m_children.HasChildAtIndex(idx)) {
       // No we haven't created the child at this index, so lets have our
@@ -440,7 +440,7 @@ ValueObjectSP ValueObject::GetChildMemberWithName(llvm::StringRef name,
   return child_sp;
 }
 
-uint32_t ValueObject::GetNumChildren(uint32_t max) {
+llvm::Expected ValueObject::GetNumChildren(uint32_t max) {
   UpdateValueIfNeeded();
 
   if (max < UINT32_MAX) {
@@ -452,11 +452,24 @@ uint32_t ValueObject::GetNumChildren(uint32_t max) {
   }
 
   if (!m_flags.m_children_count_valid) {
-    SetNumChildren(CalculateNumChildren());
+    auto num_children_or_err = CalculateNumChildren();
+    if (num_children_or_err)
+      SetNumChildren(*num_children_or_err);
+    else
+      return num_children_or_err;
   }
   return m_children.GetChildrenCount();
 }
 
+uint32_t ValueObject::GetNumChildrenIgnoringErrors(uint32_t max) {
+  auto value_or_err = GetNumChildren(max);
+  if (value_or_err)
+    return *value_or_err;
+  LLDB_LOG_ERRORV(GetLog(LLDBLog::DataFormatters), value_or_err.takeError(),
+                  "{0}");
+  return 0;
+}
+
 bool ValueObject::MightHaveChildren() {
   bool has_children = false;
   const uint32_t type_info = GetTypeInfo();
@@ -464,7 +477,7 @@ bool ValueObject::MightHaveChildren() {
     if (type_info & (eTypeHasChildren | eTypeIsPointer | eTypeIsReference))
       has_children = true;
   } else {
-    has_children = GetNumChildren() > 0;
+    has_children = GetNumChildrenIgnoringErrors() > 0;
   }
   return has_children;
 }
@@ -1176,7 +1189,7 @@ bool ValueObject::DumpPrintableRepresentation(
       if (flags.Test(eTypeIsArray)) {
         if ((custom_format == eFormatBytes) ||
             (custom_format == eFormatBytesWithASCII)) {
-          const size_t count = GetNumChildren();
+          const size_t count = GetNumChildrenIgnoringErrors();
 
           s << '[';
           for (size_t low = 0; low < count; low++) {
@@ -1215,7 +1228,7 @@ bool ValueObject::DumpPrintableRepresentation(
                                                      // format should be printed
                                                      // directly
         {
-          const size_t count = GetNumChildren();
+          const size_t count = GetNumChildrenIgnoringErrors();
 
           Format format = FormatManager::GetSingleItemFormat(custom_format);
 
@@ -1294,7 +1307,7 @@ bool ValueObject::DumpPrintableRepresentation(
       break;
 
     case eValueObjectRepresentationStyleChildrenCount:
-      strm.Printf("%" PRIu64 "", (uint64_t)GetNumChildren());
+      strm.Printf("%" PRIu64 "", (uint64_t)GetNumChildrenIgnoringErrors());
       str = strm.GetString();
       break;
 
@@ -2320,7 +2333,9 @@ ValueObjectSP ValueObject::GetValueForExpressionPath_Impl(
             child_valobj_sp = root->GetSyntheticArrayMember(index, true);
           if (!child_valobj_sp)
             if (root->HasSyntheticValue() &&
-                root->GetSyntheticValue()->GetNumChildren() > index)
+                llvm::expectedToStdOptional(
+                    root->GetSyntheticValue()->GetNumChildren())
+                        .value_or(0) > index)
               child_valobj_sp =
                   root->GetSyntheticValue()->GetChildAtIndex(index);
           if (child_valobj_sp) {
diff --git a/lldb/source/Core/ValueObjectCast.cpp b/lldb/source/Core/ValueObjectCast.cpp
index a5c555f86b13..c8e316415141 100644
--- a/lldb/source/Core/ValueObjectCast.cpp
+++ b/lldb/source/Core/ValueObjectCast.cpp
@@ -41,11 +41,13 @@ ValueObjectCast::~ValueObjectCast() = default;
 
 CompilerType ValueObjectCast::GetCompilerTypeImpl() { return m_cast_type; }
 
-uint32_t ValueObjectCast::CalculateNumChildren(uint32_t max) {
+llvm::Expected ValueObjectCast::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(
       true, &exe_ctx);
-  return children_count <= max ? children_count : max;
+  if (!children_count)
+    return children_count;
+  return *children_count <= max ? *children_count : max;
 }
 
 std::optional ValueObjectCast::GetByteSize() {
diff --git a/lldb/source/Core/ValueObjectChild.cpp b/lldb/source/Core/ValueObjectChild.cpp
index 2e55dd7726bd..c6a97dd1a5cd 100644
--- a/lldb/source/Core/ValueObjectChild.cpp
+++ b/lldb/source/Core/ValueObjectChild.cpp
@@ -49,10 +49,12 @@ lldb::ValueType ValueObjectChild::GetValueType() const {
   return m_parent->GetValueType();
 }
 
-uint32_t ValueObjectChild::CalculateNumChildren(uint32_t max) {
+llvm::Expected ValueObjectChild::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-  return children_count <= max ? children_count : max;
+  if (!children_count)
+    return children_count;
+  return *children_count <= max ? *children_count : max;
 }
 
 static void AdjustForBitfieldness(ConstString &name,
diff --git a/lldb/source/Core/ValueObjectConstResult.cpp b/lldb/source/Core/ValueObjectConstResult.cpp
index 5c7aa4452b70..8ac2c1cac2f6 100644
--- a/lldb/source/Core/ValueObjectConstResult.cpp
+++ b/lldb/source/Core/ValueObjectConstResult.cpp
@@ -216,10 +216,13 @@ std::optional ValueObjectConstResult::GetByteSize() {
 
 void ValueObjectConstResult::SetByteSize(size_t size) { m_byte_size = size; }
 
-uint32_t ValueObjectConstResult::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectConstResult::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-  return children_count <= max ? children_count : max;
+  if (!children_count)
+    return children_count;
+  return *children_count <= max ? *children_count : max;
 }
 
 ConstString ValueObjectConstResult::GetTypeName() {
diff --git a/lldb/source/Core/ValueObjectDynamicValue.cpp b/lldb/source/Core/ValueObjectDynamicValue.cpp
index 4e64760371ae..4695febdf8ca 100644
--- a/lldb/source/Core/ValueObjectDynamicValue.cpp
+++ b/lldb/source/Core/ValueObjectDynamicValue.cpp
@@ -85,12 +85,15 @@ ConstString ValueObjectDynamicValue::GetDisplayTypeName() {
   return m_parent->GetDisplayTypeName();
 }
 
-uint32_t ValueObjectDynamicValue::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectDynamicValue::CalculateNumChildren(uint32_t max) {
   const bool success = UpdateValueIfNeeded(false);
   if (success && m_dynamic_type_info.HasType()) {
     ExecutionContext exe_ctx(GetExecutionContextRef());
     auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-    return children_count <= max ? children_count : max;
+    if (!children_count)
+      return children_count;
+    return *children_count <= max ? *children_count : max;
   } else
     return m_parent->GetNumChildren(max);
 }
diff --git a/lldb/source/Core/ValueObjectMemory.cpp b/lldb/source/Core/ValueObjectMemory.cpp
index 7f68236c7884..f555ab82f441 100644
--- a/lldb/source/Core/ValueObjectMemory.cpp
+++ b/lldb/source/Core/ValueObjectMemory.cpp
@@ -126,17 +126,21 @@ ConstString ValueObjectMemory::GetDisplayTypeName() {
   return m_compiler_type.GetDisplayTypeName();
 }
 
-uint32_t ValueObjectMemory::CalculateNumChildren(uint32_t max) {
+llvm::Expected ValueObjectMemory::CalculateNumChildren(uint32_t max) {
   if (m_type_sp) {
     auto child_count = m_type_sp->GetNumChildren(true);
-    return child_count <= max ? child_count : max;
+    if (!child_count)
+      return child_count;
+    return *child_count <= max ? *child_count : max;
   }
 
   ExecutionContext exe_ctx(GetExecutionContextRef());
   const bool omit_empty_base_classes = true;
   auto child_count =
       m_compiler_type.GetNumChildren(omit_empty_base_classes, &exe_ctx);
-  return child_count <= max ? child_count : max;
+  if (!child_count)
+    return child_count;
+  return *child_count <= max ? *child_count : max;
 }
 
 std::optional ValueObjectMemory::GetByteSize() {
diff --git a/lldb/source/Core/ValueObjectRegister.cpp b/lldb/source/Core/ValueObjectRegister.cpp
index d4c144cc7edb..9d9da8bb119d 100644
--- a/lldb/source/Core/ValueObjectRegister.cpp
+++ b/lldb/source/Core/ValueObjectRegister.cpp
@@ -74,7 +74,8 @@ ConstString ValueObjectRegisterSet::GetQualifiedTypeName() {
   return ConstString();
 }
 
-uint32_t ValueObjectRegisterSet::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectRegisterSet::CalculateNumChildren(uint32_t max) {
   const RegisterSet *reg_set = m_reg_ctx_sp->GetRegisterSet(m_reg_set_idx);
   if (reg_set) {
     auto reg_count = reg_set->num_registers;
@@ -118,7 +119,7 @@ ValueObject *ValueObjectRegisterSet::CreateChildAtIndex(
     size_t idx, bool synthetic_array_member, int32_t synthetic_index) {
   ValueObject *valobj = nullptr;
   if (m_reg_ctx_sp && m_reg_set) {
-    const size_t num_children = GetNumChildren();
+    uint32_t num_children = GetNumChildrenIgnoringErrors();
     if (idx < num_children)
       valobj = new ValueObjectRegister(
           *this, m_reg_ctx_sp,
@@ -220,10 +221,13 @@ ConstString ValueObjectRegister::GetTypeName() {
   return m_type_name;
 }
 
-uint32_t ValueObjectRegister::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectRegister::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-  return children_count <= max ? children_count : max;
+  if (!children_count)
+    return children_count;
+  return *children_count <= max ? *children_count : max;
 }
 
 std::optional ValueObjectRegister::GetByteSize() {
diff --git a/lldb/source/Core/ValueObjectSyntheticFilter.cpp b/lldb/source/Core/ValueObjectSyntheticFilter.cpp
index 7f8a9a34cb35..adac1b400705 100644
--- a/lldb/source/Core/ValueObjectSyntheticFilter.cpp
+++ b/lldb/source/Core/ValueObjectSyntheticFilter.cpp
@@ -31,7 +31,9 @@ public:
   DummySyntheticFrontEnd(ValueObject &backend)
       : SyntheticChildrenFrontEnd(backend) {}
 
-  uint32_t CalculateNumChildren() override { return m_backend.GetNumChildren(); }
+  llvm::Expected CalculateNumChildren() override {
+    return m_backend.GetNumChildren();
+  }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     return m_backend.GetChildAtIndex(idx);
@@ -84,7 +86,8 @@ ConstString ValueObjectSynthetic::GetDisplayTypeName() {
   return m_parent->GetDisplayTypeName();
 }
 
-uint32_t ValueObjectSynthetic::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectSynthetic::CalculateNumChildren(uint32_t max) {
   Log *log = GetLog(LLDBLog::DataFormatters);
 
   UpdateValueIfNeeded();
@@ -92,18 +95,23 @@ uint32_t ValueObjectSynthetic::CalculateNumChildren(uint32_t max) {
     return m_synthetic_children_count <= max ? m_synthetic_children_count : max;
 
   if (max < UINT32_MAX) {
-    size_t num_children = m_synth_filter_up->CalculateNumChildren(max);
+    auto num_children = m_synth_filter_up->CalculateNumChildren(max);
     LLDB_LOGF(log,
               "[ValueObjectSynthetic::CalculateNumChildren] for VO of name "
-              "%s and type %s, the filter returned %zu child values",
-              GetName().AsCString(), GetTypeName().AsCString(), num_children);
+              "%s and type %s, the filter returned %u child values",
+              GetName().AsCString(), GetTypeName().AsCString(),
+              num_children ? *num_children : 0);
     return num_children;
   } else {
-    size_t num_children = (m_synthetic_children_count =
-                               m_synth_filter_up->CalculateNumChildren(max));
+    auto num_children_or_err = m_synth_filter_up->CalculateNumChildren(max);
+    if (!num_children_or_err) {
+      m_synthetic_children_count = 0;
+      return num_children_or_err;
+    }
+    auto num_children = (m_synthetic_children_count = *num_children_or_err);
     LLDB_LOGF(log,
               "[ValueObjectSynthetic::CalculateNumChildren] for VO of name "
-              "%s and type %s, the filter returned %zu child values",
+              "%s and type %s, the filter returned %u child values",
               GetName().AsCString(), GetTypeName().AsCString(), num_children);
     return num_children;
   }
diff --git a/lldb/source/Core/ValueObjectVTable.cpp b/lldb/source/Core/ValueObjectVTable.cpp
index 4d1cbb8d2f6f..9bfe8dc9700d 100644
--- a/lldb/source/Core/ValueObjectVTable.cpp
+++ b/lldb/source/Core/ValueObjectVTable.cpp
@@ -33,7 +33,9 @@ public:
 
   std::optional GetByteSize() override { return m_addr_size; };
 
-  uint32_t CalculateNumChildren(uint32_t max) override { return 0; };
+  llvm::Expected CalculateNumChildren(uint32_t max) override {
+    return 0;
+  };
 
   ValueType GetValueType() const override { return eValueTypeVTableEntry; };
 
@@ -159,7 +161,7 @@ std::optional ValueObjectVTable::GetByteSize() {
   return std::nullopt;
 }
 
-uint32_t ValueObjectVTable::CalculateNumChildren(uint32_t max) {
+llvm::Expected ValueObjectVTable::CalculateNumChildren(uint32_t max) {
   if (UpdateValueIfNeeded(false))
     return m_num_vtable_entries <= max ? m_num_vtable_entries : max;
   return 0;
diff --git a/lldb/source/Core/ValueObjectVariable.cpp b/lldb/source/Core/ValueObjectVariable.cpp
index dc62bb6358dc..fb29c22c0ab5 100644
--- a/lldb/source/Core/ValueObjectVariable.cpp
+++ b/lldb/source/Core/ValueObjectVariable.cpp
@@ -94,7 +94,8 @@ ConstString ValueObjectVariable::GetQualifiedTypeName() {
   return ConstString();
 }
 
-uint32_t ValueObjectVariable::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectVariable::CalculateNumChildren(uint32_t max) {
   CompilerType type(GetCompilerType());
 
   if (!type.IsValid())
@@ -103,7 +104,9 @@ uint32_t ValueObjectVariable::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   const bool omit_empty_base_classes = true;
   auto child_count = type.GetNumChildren(omit_empty_base_classes, &exe_ctx);
-  return child_count <= max ? child_count : max;
+  if (!child_count)
+    return child_count;
+  return *child_count <= max ? *child_count : max;
 }
 
 std::optional ValueObjectVariable::GetByteSize() {
diff --git a/lldb/source/DataFormatters/FormatManager.cpp b/lldb/source/DataFormatters/FormatManager.cpp
index 092fa3c8ce49..d7ba5b4b70c9 100644
--- a/lldb/source/DataFormatters/FormatManager.cpp
+++ b/lldb/source/DataFormatters/FormatManager.cpp
@@ -451,8 +451,13 @@ bool FormatManager::ShouldPrintAsOneLiner(ValueObject &valobj) {
   if (valobj.GetSummaryFormat().get() != nullptr)
     return valobj.GetSummaryFormat()->IsOneLiner();
 
+  auto num_children = valobj.GetNumChildren();
+  if (!num_children) {
+    llvm::consumeError(num_children.takeError());
+    return true;
+  }
   // no children, no party
-  if (valobj.GetNumChildren() == 0)
+  if (*num_children == 0)
     return false;
 
   // ask the type if it has any opinion about this eLazyBoolCalculate == no
@@ -471,7 +476,7 @@ bool FormatManager::ShouldPrintAsOneLiner(ValueObject &valobj) {
 
   size_t total_children_name_len = 0;
 
-  for (size_t idx = 0; idx < valobj.GetNumChildren(); idx++) {
+  for (size_t idx = 0; idx < *num_children; idx++) {
     bool is_synth_val = false;
     ValueObjectSP child_sp(valobj.GetChildAtIndex(idx));
     // something is wrong here - bail out
@@ -523,7 +528,7 @@ bool FormatManager::ShouldPrintAsOneLiner(ValueObject &valobj) {
     }
 
     // if this child has children..
-    if (child_sp->GetNumChildren()) {
+    if (child_sp->HasChildren()) {
       // ...and no summary...
       // (if it had a summary and the summary wanted children, we would have
       // bailed out anyway
diff --git a/lldb/source/DataFormatters/TypeSynthetic.cpp b/lldb/source/DataFormatters/TypeSynthetic.cpp
index 0ae38c4d31f2..e3f613606aa8 100644
--- a/lldb/source/DataFormatters/TypeSynthetic.cpp
+++ b/lldb/source/DataFormatters/TypeSynthetic.cpp
@@ -115,6 +115,17 @@ std::string CXXSyntheticChildren::GetDescription() {
   return std::string(sstr.GetString());
 }
 
+uint32_t
+SyntheticChildrenFrontEnd::CalculateNumChildrenIgnoringErrors(uint32_t max) {
+  auto value_or_err = CalculateNumChildren(max);
+  if (value_or_err)
+    return *value_or_err;
+  Log *log = GetLog(LLDBLog::DataFormatters);
+  if (log && log->GetVerbose())
+    LLDB_LOG_ERROR(log, value_or_err.takeError(), "{0}");
+  return 0;
+}
+
 lldb::ValueObjectSP SyntheticChildrenFrontEnd::CreateValueObjectFromExpression(
     llvm::StringRef name, llvm::StringRef expression,
     const ExecutionContext &exe_ctx) {
@@ -178,13 +189,15 @@ bool ScriptedSyntheticChildren::FrontEnd::IsValid() {
   return (m_wrapper_sp && m_wrapper_sp->IsValid() && m_interpreter);
 }
 
-uint32_t ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren() {
+llvm::Expected
+ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren() {
   if (!m_wrapper_sp || m_interpreter == nullptr)
     return 0;
   return m_interpreter->CalculateNumChildren(m_wrapper_sp, UINT32_MAX);
 }
 
-uint32_t ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren(uint32_t max) {
   if (!m_wrapper_sp || m_interpreter == nullptr)
     return 0;
   return m_interpreter->CalculateNumChildren(m_wrapper_sp, max);
diff --git a/lldb/source/DataFormatters/ValueObjectPrinter.cpp b/lldb/source/DataFormatters/ValueObjectPrinter.cpp
index 46e50a8d421a..b853199e878c 100644
--- a/lldb/source/DataFormatters/ValueObjectPrinter.cpp
+++ b/lldb/source/DataFormatters/ValueObjectPrinter.cpp
@@ -627,7 +627,7 @@ uint32_t ValueObjectPrinter::GetMaxNumChildrenToPrint(bool &print_dotdotdot) {
   if (m_options.m_pointer_as_array)
     return m_options.m_pointer_as_array.m_element_count;
 
-  size_t num_children = synth_valobj.GetNumChildren();
+  uint32_t num_children = synth_valobj.GetNumChildrenIgnoringErrors();
   print_dotdotdot = false;
   if (num_children) {
     const size_t max_num_children = GetMostSpecializedValue()
diff --git a/lldb/source/DataFormatters/VectorType.cpp b/lldb/source/DataFormatters/VectorType.cpp
index a0626a8cba77..19de204c2435 100644
--- a/lldb/source/DataFormatters/VectorType.cpp
+++ b/lldb/source/DataFormatters/VectorType.cpp
@@ -9,6 +9,7 @@
 #include "lldb/DataFormatters/VectorType.h"
 
 #include "lldb/Core/ValueObject.h"
+#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Symbol/TypeSystem.h"
@@ -224,10 +225,16 @@ public:
 
   ~VectorTypeSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override { return m_num_children; }
+  llvm::Expected CalculateNumChildren() override {
+    return m_num_children;
+  }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
-    if (idx >= CalculateNumChildren())
+    auto num_children_or_err = CalculateNumChildren();
+    if (!num_children_or_err)
+      return ValueObjectConstResult::Create(
+          nullptr, Status(num_children_or_err.takeError()));
+    if (idx >= *num_children_or_err)
       return {};
     std::optional size = m_child_type.GetByteSize(nullptr);
     if (!size)
@@ -266,7 +273,7 @@ public:
   size_t GetIndexOfChildWithName(ConstString name) override {
     const char *item_name = name.GetCString();
     uint32_t idx = ExtractIndexFromString(item_name);
-    if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+    if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
       return UINT32_MAX;
     return idx;
   }
@@ -293,7 +300,8 @@ bool lldb_private::formatters::VectorTypeSummaryProvider(
   s.PutChar('(');
   bool first = true;
 
-  size_t idx = 0, len = synthetic_children->CalculateNumChildren();
+  size_t idx = 0,
+         len = synthetic_children->CalculateNumChildrenIgnoringErrors();
 
   for (; idx < len; idx++) {
     auto child_sp = synthetic_children->GetChildAtIndex(idx);
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp
index b48bbbecc0cd..3d43ed3f99ff 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp
@@ -229,7 +229,7 @@ void AddLambdaCaptureDecls(StreamString &stream, StackFrame *frame,
   assert(frame);
 
   if (auto thisValSP = ClangExpressionUtil::GetLambdaValueObject(frame)) {
-    uint32_t numChildren = thisValSP->GetNumChildren();
+    uint32_t numChildren = thisValSP->GetNumChildrenIgnoringErrors();
     for (uint32_t i = 0; i < numChildren; ++i) {
       auto childVal = thisValSP->GetChildAtIndex(i);
       ConstString childName(childVal ? childVal->GetName() : ConstString(""));
diff --git a/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp b/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
index 72293c5331f4..b2781aa5e7db 100644
--- a/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
+++ b/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
@@ -27,6 +27,8 @@
 #include "lldb/Target/StopInfo.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Target/Thread.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegularExpression.h"
 #include "lldb/Utility/Stream.h"
 
@@ -211,7 +213,7 @@ CreateStackTrace(ValueObjectSP o,
   auto trace_sp = std::make_shared();
   ValueObjectSP trace_value_object =
       o->GetValueForExpressionPath(trace_item_name.c_str());
-  size_t count = trace_value_object->GetNumChildren();
+  size_t count = trace_value_object->GetNumChildrenIgnoringErrors();
   for (size_t j = 0; j < count; j++) {
     addr_t trace_addr =
         trace_value_object->GetChildAtIndex(j)->GetValueAsUnsigned(0);
diff --git a/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp b/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
index ef0f67d1e9f9..9a6e135e0083 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
@@ -74,7 +74,7 @@ public:
 
   ~BlockPointerSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override {
+  llvm::Expected CalculateNumChildren() override {
     const bool omit_empty_base_classes = false;
     return m_block_struct_type.GetNumChildren(omit_empty_base_classes, nullptr);
   }
@@ -84,7 +84,7 @@ public:
       return lldb::ValueObjectSP();
     }
 
-    if (idx >= CalculateNumChildren()) {
+    if (idx >= CalculateNumChildrenIgnoringErrors()) {
       return lldb::ValueObjectSP();
     }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp
index 3827f9c21eff..5e63d1d7b214 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp
@@ -11,6 +11,8 @@
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
 #include "lldb/Symbol/Function.h"
 #include "lldb/Symbol/VariableList.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/Log.h"
 
 using namespace lldb;
 using namespace lldb_private;
@@ -22,7 +24,7 @@ static lldb::addr_t GetCoroFramePtrFromHandle(ValueObjectSP valobj_sp) {
 
   // We expect a single pointer in the `coroutine_handle` class.
   // We don't care about its name.
-  if (valobj_sp->GetNumChildren() != 1)
+  if (valobj_sp->GetNumChildrenIgnoringErrors() != 1)
     return LLDB_INVALID_ADDRESS;
   ValueObjectSP ptr_sp(valobj_sp->GetChildAtIndex(0));
   if (!ptr_sp)
@@ -104,8 +106,8 @@ lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd::
 lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd::
     ~StdlibCoroutineHandleSyntheticFrontEnd() = default;
 
-uint32_t lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    StdlibCoroutineHandleSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_resume_ptr_sp || !m_destroy_ptr_sp)
     return 0;
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
index 5c6a80b57ff4..1d4bc65e2637 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
@@ -34,7 +34,7 @@ public:
 
   ~StdlibCoroutineHandleSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp b/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp
index 6a9da1d17c76..33955dccb6cc 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp
@@ -34,7 +34,9 @@ public:
 
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
-  uint32_t CalculateNumChildren() override { return m_elements.size(); }
+  llvm::Expected CalculateNumChildren() override {
+    return m_elements.size();
+  }
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
 private:
diff --git a/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp b/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp
index c06afb53eb8a..23756de7f1e6 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp
@@ -19,7 +19,7 @@ using namespace lldb_private;
 bool lldb_private::formatters::GenericOptionalSummaryProvider(
     ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
   stream.Printf(" Has Value=%s ",
-                valobj.GetNumChildren() == 0 ? "false" : "true");
+                valobj.GetNumChildrenIgnoringErrors() == 0 ? "false" : "true");
 
   return true;
 }
@@ -41,7 +41,9 @@ public:
   }
 
   bool MightHaveChildren() override { return true; }
-  uint32_t CalculateNumChildren() override { return m_has_value ? 1U : 0U; }
+  llvm::Expected CalculateNumChildren() override {
+    return m_has_value ? 1U : 0U;
+  }
 
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
   lldb::ChildCacheState Update() override;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
index bba887fec3ac..10a1fe039189 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
@@ -63,7 +63,7 @@ lldb::ValueObjectSP
 lldb_private::formatters::GetSecondValueOfLibCXXCompressedPair(
     ValueObject &pair) {
   ValueObjectSP value;
-  if (pair.GetNumChildren() > 1) {
+  if (pair.GetNumChildrenIgnoringErrors() > 1) {
     ValueObjectSP second_child = pair.GetChildAtIndex(1);
     if (second_child) {
       value = second_child->GetChildMemberWithName("__value_");
@@ -351,8 +351,8 @@ lldb_private::formatters::LibCxxMapIteratorSyntheticFrontEnd::Update() {
   return lldb::ChildCacheState::eRefetch;
 }
 
-uint32_t lldb_private::formatters::LibCxxMapIteratorSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibCxxMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
   return 2;
 }
 
@@ -509,8 +509,8 @@ lldb::ChildCacheState lldb_private::formatters::
   return lldb::ChildCacheState::eRefetch;
 }
 
-uint32_t lldb_private::formatters::LibCxxUnorderedMapIteratorSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibCxxUnorderedMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
   return 2;
 }
 
@@ -566,8 +566,8 @@ lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::
     Update();
 }
 
-uint32_t lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxSharedPtrSyntheticFrontEnd::CalculateNumChildren() {
   return (m_cntrl ? 1 : 0);
 }
 
@@ -661,8 +661,8 @@ lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEndCreator(
                     : nullptr);
 }
 
-uint32_t lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxUniquePtrSyntheticFrontEnd::CalculateNumChildren() {
   if (m_value_ptr_sp)
     return m_deleter_sp ? 2 : 1;
   return 0;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
index ad2f58508ab7..a59f21841ec8 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
@@ -87,7 +87,7 @@ class LibCxxMapIteratorSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibCxxMapIteratorSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -135,7 +135,7 @@ public:
 
   ~LibCxxUnorderedMapIteratorSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -166,7 +166,7 @@ class LibcxxSharedPtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibcxxSharedPtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -186,7 +186,7 @@ class LibcxxUniquePtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibcxxUniquePtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp
index 8e4c36103a74..7f30dc186291 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp
@@ -90,7 +90,7 @@ public:
 
   ~LibcxxStdAtomicSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -124,8 +124,8 @@ bool lldb_private::formatters::LibcxxStdAtomicSyntheticFrontEnd::
   return true;
 }
 
-uint32_t lldb_private::formatters::LibcxxStdAtomicSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdAtomicSyntheticFrontEnd::CalculateNumChildren() {
   return m_real_child ? 1 : 0;
 }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp
index 00012dfc056e..bd9c72497664 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp
@@ -26,7 +26,7 @@ public:
 
   ~LibcxxInitializerListSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -59,8 +59,8 @@ lldb_private::formatters::LibcxxInitializerListSyntheticFrontEnd::
   // delete m_start;
 }
 
-uint32_t lldb_private::formatters::LibcxxInitializerListSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxInitializerListSyntheticFrontEnd::CalculateNumChildren() {
   m_num_elements = 0;
   ValueObjectSP size_sp(m_backend.GetChildMemberWithName("__size_"));
   if (size_sp)
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
index 17f6b737d9f6..d7cfeb30557c 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
@@ -136,7 +136,7 @@ class ForwardListFrontEnd : public AbstractListFrontEnd {
 public:
   ForwardListFrontEnd(ValueObject &valobj);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
   lldb::ChildCacheState Update() override;
 };
@@ -147,7 +147,7 @@ public:
 
   ~ListFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -240,7 +240,7 @@ ForwardListFrontEnd::ForwardListFrontEnd(ValueObject &valobj)
   Update();
 }
 
-uint32_t ForwardListFrontEnd::CalculateNumChildren() {
+llvm::Expected ForwardListFrontEnd::CalculateNumChildren() {
   if (m_count != UINT32_MAX)
     return m_count;
 
@@ -254,7 +254,7 @@ uint32_t ForwardListFrontEnd::CalculateNumChildren() {
 }
 
 ValueObjectSP ForwardListFrontEnd::GetChildAtIndex(uint32_t idx) {
-  if (idx >= CalculateNumChildren())
+  if (idx >= CalculateNumChildrenIgnoringErrors())
     return nullptr;
 
   if (!m_head)
@@ -308,7 +308,7 @@ ListFrontEnd::ListFrontEnd(lldb::ValueObjectSP valobj_sp)
     Update();
 }
 
-uint32_t ListFrontEnd::CalculateNumChildren() {
+llvm::Expected ListFrontEnd::CalculateNumChildren() {
   if (m_count != UINT32_MAX)
     return m_count;
   if (!m_head || !m_tail || m_node_address == 0)
@@ -347,7 +347,7 @@ lldb::ValueObjectSP ListFrontEnd::GetChildAtIndex(uint32_t idx) {
   static ConstString g_value("__value_");
   static ConstString g_next("__next_");
 
-  if (idx >= CalculateNumChildren())
+  if (idx >= CalculateNumChildrenIgnoringErrors())
     return lldb::ValueObjectSP();
 
   if (!m_head || !m_tail || m_node_address == 0)
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp
index 6d24eb03779c..ec5b320e2218 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp
@@ -177,7 +177,7 @@ public:
 
   ~LibcxxStdMapSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -209,8 +209,8 @@ lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::
     Update();
 }
 
-uint32_t lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdMapSyntheticFrontEnd::CalculateNumChildren() {
   if (m_count != UINT32_MAX)
     return m_count;
 
@@ -311,13 +311,13 @@ lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
   static ConstString g_cc_("__cc_"), g_cc("__cc");
   static ConstString g_nc("__nc");
-
-  if (idx >= CalculateNumChildren())
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
+  if (idx >= num_children)
     return lldb::ValueObjectSP();
   if (m_tree == nullptr || m_root_node == nullptr)
     return lldb::ValueObjectSP();
 
-  MapIterator iterator(m_root_node, CalculateNumChildren());
+  MapIterator iterator(m_root_node, num_children);
 
   const bool need_to_skip = (idx > 0);
   size_t actual_advancde = idx;
@@ -382,7 +382,7 @@ lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::GetChildAtIndex(
   name.Printf("[%" PRIu64 "]", (uint64_t)idx);
   auto potential_child_sp = iterated_sp->Clone(ConstString(name.GetString()));
   if (potential_child_sp) {
-    switch (potential_child_sp->GetNumChildren()) {
+    switch (potential_child_sp->GetNumChildrenIgnoringErrors()) {
     case 1: {
       auto child0_sp = potential_child_sp->GetChildAtIndex(0);
       if (child0_sp &&
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp
index fbadee89b7b7..5b459a17fe29 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp
@@ -28,7 +28,7 @@ public:
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
 
-  uint32_t CalculateNumChildren() override {
+  llvm::Expected CalculateNumChildren() override {
     return m_container_sp ? m_container_sp->GetNumChildren() : 0;
   }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp
index 74f54f767356..01a7b8f142ec 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp
@@ -27,7 +27,7 @@ public:
 
   ~LibcxxStdRangesRefViewSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override {
+  llvm::Expected CalculateNumChildren() override {
     // __range_ will be the sole child of this type
     return 1;
   }
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp
index af2b51d2b540..9895f336bfd0 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp
@@ -27,7 +27,7 @@ public:
 
   ~LibcxxStdSpanSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -73,8 +73,8 @@ lldb_private::formatters::LibcxxStdSpanSyntheticFrontEnd::
     Update();
 }
 
-uint32_t lldb_private::formatters::LibcxxStdSpanSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdSpanSyntheticFrontEnd::CalculateNumChildren() {
   return m_num_elements;
 }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp
index 62bb7d619267..3e3259ab428d 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp
@@ -26,7 +26,9 @@ public:
 
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
-  uint32_t CalculateNumChildren() override { return m_elements.size(); }
+  llvm::Expected CalculateNumChildren() override {
+    return m_elements.size();
+  }
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
 private:
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
index b3c364294335..af29fdb6d001 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
@@ -33,7 +33,7 @@ public:
 
   ~LibcxxStdUnorderedMapSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -62,8 +62,8 @@ lldb_private::formatters::LibcxxStdUnorderedMapSyntheticFrontEnd::
     Update();
 }
 
-uint32_t lldb_private::formatters::LibcxxStdUnorderedMapSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdUnorderedMapSyntheticFrontEnd::CalculateNumChildren() {
   return m_num_elements;
 }
 
@@ -94,7 +94,7 @@ static bool isUnorderedMap(ConstString type_name) {
 
 lldb::ValueObjectSP lldb_private::formatters::
     LibcxxStdUnorderedMapSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
-  if (idx >= CalculateNumChildren())
+  if (idx >= CalculateNumChildrenIgnoringErrors())
     return lldb::ValueObjectSP();
   if (m_tree == nullptr)
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp
index 463c7b8d7ce3..99f94406e99a 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp
@@ -24,7 +24,7 @@ public:
 
   ~LibcxxStdValarraySyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -63,8 +63,8 @@ lldb_private::formatters::LibcxxStdValarraySyntheticFrontEnd::
   // delete m_finish;
 }
 
-uint32_t lldb_private::formatters::LibcxxStdValarraySyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdValarraySyntheticFrontEnd::CalculateNumChildren() {
   if (!m_start || !m_finish)
     return 0;
   uint64_t start_val = m_start->GetValueAsUnsigned(0);
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp
index 1f62062f09be..62794318e077 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp
@@ -205,7 +205,7 @@ public:
 
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
-  uint32_t CalculateNumChildren() override { return m_size; }
+  llvm::Expected CalculateNumChildren() override { return m_size; }
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
 private:
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
index fcf727ad2ea0..461fed35164b 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
@@ -25,7 +25,7 @@ public:
 
   ~LibcxxStdVectorSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -46,7 +46,7 @@ class LibcxxVectorBoolSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibcxxVectorBoolSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -82,8 +82,8 @@ lldb_private::formatters::LibcxxStdVectorSyntheticFrontEnd::
   // delete m_finish;
 }
 
-uint32_t lldb_private::formatters::LibcxxStdVectorSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdVectorSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_start || !m_finish)
     return 0;
   uint64_t start_val = m_start->GetValueAsUnsigned(0);
@@ -165,8 +165,8 @@ lldb_private::formatters::LibcxxVectorBoolSyntheticFrontEnd::
   }
 }
 
-uint32_t lldb_private::formatters::LibcxxVectorBoolSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxVectorBoolSyntheticFrontEnd::CalculateNumChildren() {
   return m_count;
 }
 
@@ -259,7 +259,7 @@ size_t lldb_private::formatters::LibcxxVectorBoolSyntheticFrontEnd::
     return UINT32_MAX;
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
index 5abb3d50674b..86bb575af5ca 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
@@ -43,7 +43,7 @@ class LibstdcppMapIteratorSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibstdcppMapIteratorSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -64,7 +64,7 @@ class LibStdcppSharedPtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibStdcppSharedPtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -132,7 +132,8 @@ lldb::ChildCacheState LibstdcppMapIteratorSyntheticFrontEnd::Update() {
   return lldb::ChildCacheState::eReuse;
 }
 
-uint32_t LibstdcppMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected
+LibstdcppMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
   return 2;
 }
 
@@ -219,7 +220,10 @@ lldb::ChildCacheState VectorIteratorSyntheticFrontEnd::Update() {
   return lldb::ChildCacheState::eRefetch;
 }
 
-uint32_t VectorIteratorSyntheticFrontEnd::CalculateNumChildren() { return 1; }
+llvm::Expected
+VectorIteratorSyntheticFrontEnd::CalculateNumChildren() {
+  return 1;
+}
 
 lldb::ValueObjectSP
 VectorIteratorSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
@@ -371,7 +375,10 @@ LibStdcppSharedPtrSyntheticFrontEnd::LibStdcppSharedPtrSyntheticFrontEnd(
     Update();
 }
 
-uint32_t LibStdcppSharedPtrSyntheticFrontEnd::CalculateNumChildren() { return 1; }
+llvm::Expected
+LibStdcppSharedPtrSyntheticFrontEnd::CalculateNumChildren() {
+  return 1;
+}
 
 lldb::ValueObjectSP
 LibStdcppSharedPtrSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp
index 64d2ec9d943a..05199ba35b9a 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp
@@ -26,7 +26,7 @@ class LibStdcppTupleSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibStdcppTupleSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -65,7 +65,7 @@ lldb::ChildCacheState LibStdcppTupleSyntheticFrontEnd::Update() {
     ValueObjectSP current_child = next_child_sp;
     next_child_sp = nullptr;
 
-    size_t child_count = current_child->GetNumChildren();
+    size_t child_count = current_child->GetNumChildrenIgnoringErrors();
     for (size_t i = 0; i < child_count; ++i) {
       ValueObjectSP child_sp = current_child->GetChildAtIndex(i);
       llvm::StringRef name_str = child_sp->GetName().GetStringRef();
@@ -95,7 +95,8 @@ LibStdcppTupleSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
   return lldb::ValueObjectSP();
 }
 
-uint32_t LibStdcppTupleSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected
+LibStdcppTupleSyntheticFrontEnd::CalculateNumChildren() {
   return m_members.size();
 }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
index 3a48fe412e07..92f540d9ca52 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
@@ -26,7 +26,7 @@ class LibStdcppUniquePtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibStdcppUniquePtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -135,7 +135,8 @@ LibStdcppUniquePtrSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
   return lldb::ValueObjectSP();
 }
 
-uint32_t LibStdcppUniquePtrSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected
+LibStdcppUniquePtrSyntheticFrontEnd::CalculateNumChildren() {
   if (m_del_obj)
     return 2;
   return 1;
diff --git a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
index cb740f8e71e1..96166657ceeb 100644
--- a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
+++ b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
@@ -1038,7 +1038,7 @@ public:
 
   ~ObjCClassSyntheticChildrenFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override { return 0; }
+  llvm::Expected CalculateNumChildren() override { return 0; }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/Language/ObjC/NSArray.cpp b/lldb/source/Plugins/Language/ObjC/NSArray.cpp
index 7f060b2613d6..67d0cd08f51a 100644
--- a/lldb/source/Plugins/Language/ObjC/NSArray.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSArray.cpp
@@ -50,7 +50,7 @@ public:
 
   ~NSArrayMSyntheticFrontEndBase() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -214,7 +214,7 @@ public:
 
   ~GenericNSArrayISyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -302,7 +302,7 @@ public:
 
   ~NSArray0SyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -319,7 +319,7 @@ public:
 
   ~NSArray1SyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -477,15 +477,15 @@ lldb_private::formatters::
     : NSArrayMSyntheticFrontEndBase(valobj_sp), m_data_32(nullptr),
       m_data_64(nullptr) {}
 
-uint32_t lldb_private::formatters::NSArrayMSyntheticFrontEndBase::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    NSArrayMSyntheticFrontEndBase::CalculateNumChildren() {
   return GetUsedCount();
 }
 
 lldb::ValueObjectSP
 lldb_private::formatters::NSArrayMSyntheticFrontEndBase::GetChildAtIndex(
     uint32_t idx) {
-  if (idx >= CalculateNumChildren())
+  if (idx >= CalculateNumChildrenIgnoringErrors())
     return lldb::ValueObjectSP();
   lldb::addr_t object_at_idx = GetDataAddress();
   size_t pyhs_idx = idx;
@@ -542,7 +542,7 @@ lldb_private::formatters::NSArrayMSyntheticFrontEndBase::GetIndexOfChildWithName
     ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
@@ -628,15 +628,15 @@ lldb_private::formatters::GenericNSArrayISyntheticFrontEnd::
   GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
 template 
-uint32_t
-lldb_private::formatters::GenericNSArrayISyntheticFrontEnd::
-  CalculateNumChildren() {
+llvm::Expected
+lldb_private::formatters::GenericNSArrayISyntheticFrontEnd<
+    D32, D64, Inline>::CalculateNumChildren() {
   return m_data_32 ? m_data_32->used : m_data_64->used;
 }
 
@@ -685,7 +685,7 @@ template 
 lldb::ValueObjectSP
 lldb_private::formatters::GenericNSArrayISyntheticFrontEnd::
   GetChildAtIndex(uint32_t idx) {
-  if (idx >= CalculateNumChildren())
+  if (idx >= CalculateNumChildrenIgnoringErrors())
     return lldb::ValueObjectSP();
   lldb::addr_t object_at_idx;
   if (Inline) {
@@ -719,7 +719,7 @@ lldb_private::formatters::NSArray0SyntheticFrontEnd::GetIndexOfChildWithName(
   return UINT32_MAX;
 }
 
-uint32_t
+llvm::Expected
 lldb_private::formatters::NSArray0SyntheticFrontEnd::CalculateNumChildren() {
   return 0;
 }
@@ -754,7 +754,7 @@ lldb_private::formatters::NSArray1SyntheticFrontEnd::GetIndexOfChildWithName(
   return UINT32_MAX;
 }
 
-uint32_t
+llvm::Expected
 lldb_private::formatters::NSArray1SyntheticFrontEnd::CalculateNumChildren() {
   return 1;
 }
diff --git a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
index da94eda1529c..ec6fd756394a 100644
--- a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
@@ -103,7 +103,7 @@ public:
 
   ~NSDictionaryISyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -144,7 +144,7 @@ class NSConstantDictionarySyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   NSConstantDictionarySyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -176,7 +176,7 @@ class NSCFDictionarySyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   NSCFDictionarySyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -209,7 +209,7 @@ public:
 
   ~NSDictionary1SyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -230,7 +230,7 @@ public:
 
   ~GenericNSDictionaryMSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -262,9 +262,9 @@ namespace Foundation1100 {
     NSDictionaryMSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
     
     ~NSDictionaryMSyntheticFrontEnd() override;
-    
-    uint32_t CalculateNumChildren() override;
-    
+
+    llvm::Expected CalculateNumChildren() override;
+
     lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
     lldb::ChildCacheState Update() override;
@@ -601,13 +601,13 @@ size_t lldb_private::formatters::NSDictionaryISyntheticFrontEnd::
     GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t lldb_private::formatters::NSDictionaryISyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    NSDictionaryISyntheticFrontEnd::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
   return (m_data_32 ? m_data_32->_used : m_data_64->_used);
@@ -656,7 +656,7 @@ bool lldb_private::formatters::NSDictionaryISyntheticFrontEnd::
 lldb::ValueObjectSP
 lldb_private::formatters::NSDictionaryISyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -739,13 +739,13 @@ size_t lldb_private::formatters::NSCFDictionarySyntheticFrontEnd::
     GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   const uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t lldb_private::formatters::NSCFDictionarySyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    NSCFDictionarySyntheticFrontEnd::CalculateNumChildren() {
   if (!m_hashtable.IsValid())
     return 0;
   return m_hashtable.GetCount();
@@ -781,7 +781,7 @@ lldb_private::formatters::NSCFDictionarySyntheticFrontEnd::GetChildAtIndex(
   lldb::addr_t m_keys_ptr = m_hashtable.GetKeyPointer();
   lldb::addr_t m_values_ptr = m_hashtable.GetValuePointer();
 
-  const uint32_t num_children = CalculateNumChildren();
+  const uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -875,13 +875,13 @@ size_t lldb_private::formatters::NSConstantDictionarySyntheticFrontEnd::
     GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t lldb_private::formatters::NSConstantDictionarySyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    NSConstantDictionarySyntheticFrontEnd::CalculateNumChildren() {
   return m_size;
 }
 
@@ -921,7 +921,7 @@ bool lldb_private::formatters::NSConstantDictionarySyntheticFrontEnd::
 
 lldb::ValueObjectSP lldb_private::formatters::
     NSConstantDictionarySyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -994,8 +994,8 @@ size_t lldb_private::formatters::NSDictionary1SyntheticFrontEnd::
   return name == g_zero ? 0 : UINT32_MAX;
 }
 
-uint32_t lldb_private::formatters::NSDictionary1SyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    NSDictionary1SyntheticFrontEnd::CalculateNumChildren() {
   return 1;
 }
 
@@ -1081,17 +1081,18 @@ size_t lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd<
     D32, D64>::GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
 template 
-uint32_t
-lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected
+lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd<
+    D32, D64>::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
-  return (m_data_32 ? m_data_32->_used : m_data_64->_used);
+  return (m_data_32 ? (uint32_t)m_data_32->_used : (uint32_t)m_data_64->_used);
 }
 
 template 
@@ -1153,7 +1154,7 @@ lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd<
     m_values_ptr = m_data_64->_buffer + (m_ptr_size * size);
   }
 
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -1245,14 +1246,13 @@ lldb_private::formatters::Foundation1100::
   NSDictionaryMSyntheticFrontEnd::GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t
-lldb_private::formatters::Foundation1100::
-  NSDictionaryMSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::Foundation1100::
+    NSDictionaryMSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
   return (m_data_32 ? m_data_32->_used : m_data_64->_used);
@@ -1306,7 +1306,7 @@ lldb_private::formatters::Foundation1100::
   lldb::addr_t m_values_ptr =
       (m_data_32 ? m_data_32->_objs_addr : m_data_64->_objs_addr);
 
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/Language/ObjC/NSError.cpp b/lldb/source/Plugins/Language/ObjC/NSError.cpp
index b034e799b716..5ef7edc7e80c 100644
--- a/lldb/source/Plugins/Language/ObjC/NSError.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSError.cpp
@@ -116,7 +116,7 @@ public:
   // no need to delete m_child_ptr - it's kept alive by the cluster manager on
   // our behalf
 
-  uint32_t CalculateNumChildren() override {
+  llvm::Expected CalculateNumChildren() override {
     if (m_child_ptr)
       return 1;
     if (m_child_sp)
diff --git a/lldb/source/Plugins/Language/ObjC/NSException.cpp b/lldb/source/Plugins/Language/ObjC/NSException.cpp
index 09d3a1b42b74..e7ce26ea4c6f 100644
--- a/lldb/source/Plugins/Language/ObjC/NSException.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSException.cpp
@@ -123,9 +123,7 @@ public:
 
   ~NSExceptionSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override {
-    return 4;
-  }
+  llvm::Expected CalculateNumChildren() override { return 4; }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     switch (idx) {
diff --git a/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp b/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
index 10bb907c58ed..a434cee09d38 100644
--- a/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
@@ -40,7 +40,9 @@ public:
 
   ~NSIndexPathSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override { return m_impl.GetNumIndexes(); }
+  llvm::Expected CalculateNumChildren() override {
+    return m_impl.GetNumIndexes();
+  }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     return m_impl.GetIndexAtIndex(idx, m_uint_star_type);
@@ -127,7 +129,7 @@ public:
   size_t GetIndexOfChildWithName(ConstString name) override {
     const char *item_name = name.GetCString();
     uint32_t idx = ExtractIndexFromString(item_name);
-    if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+    if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
       return UINT32_MAX;
     return idx;
   }
diff --git a/lldb/source/Plugins/Language/ObjC/NSSet.cpp b/lldb/source/Plugins/Language/ObjC/NSSet.cpp
index c965a2a13400..7d0a6a507211 100644
--- a/lldb/source/Plugins/Language/ObjC/NSSet.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSSet.cpp
@@ -46,7 +46,7 @@ public:
 
   ~NSSetISyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -84,7 +84,7 @@ class NSCFSetSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   NSCFSetSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -117,7 +117,7 @@ public:
 
   ~GenericNSSetMSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -233,7 +233,7 @@ public:
 
   ~NSSetCodeRunningSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -414,12 +414,12 @@ lldb_private::formatters::NSSetISyntheticFrontEnd::GetIndexOfChildWithName(
     ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t
+llvm::Expected
 lldb_private::formatters::NSSetISyntheticFrontEnd::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
@@ -468,7 +468,7 @@ bool lldb_private::formatters::NSSetISyntheticFrontEnd::MightHaveChildren() {
 lldb::ValueObjectSP
 lldb_private::formatters::NSSetISyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -551,12 +551,12 @@ lldb_private::formatters::NSCFSetSyntheticFrontEnd::GetIndexOfChildWithName(
     ConstString name) {
   const char *item_name = name.GetCString();
   const uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t
+llvm::Expected
 lldb_private::formatters::NSCFSetSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_hashtable.IsValid())
     return 0;
@@ -591,7 +591,7 @@ lldb_private::formatters::NSCFSetSyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
   lldb::addr_t m_values_ptr = m_hashtable.GetValuePointer();
 
-  const uint32_t num_children = CalculateNumChildren();
+  const uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -691,18 +691,18 @@ lldb_private::formatters::
     ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
 template 
-uint32_t
-lldb_private::formatters::
-  GenericNSSetMSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected
+lldb_private::formatters::GenericNSSetMSyntheticFrontEnd<
+    D32, D64>::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
-  return (m_data_32 ? m_data_32->_used : m_data_64->_used);
+  return (m_data_32 ? (uint32_t)m_data_32->_used : (uint32_t)m_data_64->_used);
 }
 
 template 
@@ -753,7 +753,7 @@ lldb_private::formatters::
   lldb::addr_t m_objs_addr =
       (m_data_32 ? m_data_32->_objs_addr : m_data_64->_objs_addr);
 
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
index f08f9f0f815d..9434376f7d9e 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
@@ -539,7 +539,8 @@ ThreadSP AppleObjCRuntime::GetBacktraceThreadFromException(
     return object;
   };
 
-  for (size_t idx = 0; idx < reserved_dict->GetNumChildren(); idx++) {
+  for (size_t idx = 0; idx < reserved_dict->GetNumChildrenIgnoringErrors();
+       idx++) {
     ValueObjectSP dict_entry = reserved_dict->GetChildAtIndex(idx);
 
     DataExtractor data;
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index 51ab13108feb..c02b08cb4782 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -5263,9 +5263,10 @@ GetDynamicArrayInfo(TypeSystemClang &ast, SymbolFile *sym_file,
   return std::nullopt;
 }
 
-uint32_t TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
-                                         bool omit_empty_base_classes,
-                                         const ExecutionContext *exe_ctx) {
+llvm::Expected
+TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
+                                bool omit_empty_base_classes,
+                                const ExecutionContext *exe_ctx) {
   if (!type)
     return 0;
 
@@ -5361,9 +5362,13 @@ uint32_t TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
     CompilerType pointee_clang_type(GetPointeeType(type));
 
     uint32_t num_pointee_children = 0;
-    if (pointee_clang_type.IsAggregateType())
-      num_pointee_children =
+    if (pointee_clang_type.IsAggregateType()) {
+      auto num_children_or_err =
           pointee_clang_type.GetNumChildren(omit_empty_base_classes, exe_ctx);
+      if (!num_children_or_err)
+        return num_children_or_err;
+      num_pointee_children = *num_children_or_err;
+    }
     // If this type points to a simple type, then it has 1 child
     if (num_pointee_children == 0)
       num_children = 1;
@@ -5397,9 +5402,13 @@ uint32_t TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
     clang::QualType pointee_type(pointer_type->getPointeeType());
     CompilerType pointee_clang_type(GetType(pointee_type));
     uint32_t num_pointee_children = 0;
-    if (pointee_clang_type.IsAggregateType())
-      num_pointee_children =
+    if (pointee_clang_type.IsAggregateType()) {
+      auto num_children_or_err =
           pointee_clang_type.GetNumChildren(omit_empty_base_classes, exe_ctx);
+      if (!num_children_or_err)
+        return num_children_or_err;
+      num_pointee_children = *num_children_or_err;
+    }
     if (num_pointee_children == 0) {
       // We have a pointer to a pointee type that claims it has no children. We
       // will want to look at
@@ -6108,8 +6117,15 @@ CompilerType TypeSystemClang::GetChildCompilerTypeAtIndex(
   child_is_base_class = false;
   language_flags = 0;
 
-  const bool idx_is_valid =
-      idx < GetNumChildren(type, omit_empty_base_classes, exe_ctx);
+  auto num_children_or_err =
+      GetNumChildren(type, omit_empty_base_classes, exe_ctx);
+  if (!num_children_or_err) {
+    LLDB_LOG_ERRORV(GetLog(LLDBLog::Types), num_children_or_err.takeError(),
+                    "{0}");
+    return {};
+  }
+
+  const bool idx_is_valid = idx < *num_children_or_err;
   int32_t bit_offset;
   switch (parent_type_class) {
   case clang::Type::Builtin:
@@ -6265,8 +6281,10 @@ CompilerType TypeSystemClang::GetChildCompilerTypeAtIndex(
               CompilerType base_class_clang_type =
                   GetType(getASTContext().getObjCInterfaceType(
                       superclass_interface_decl));
-              if (base_class_clang_type.GetNumChildren(omit_empty_base_classes,
-                                                       exe_ctx) > 0) {
+              if (llvm::expectedToStdOptional(
+                      base_class_clang_type.GetNumChildren(
+                          omit_empty_base_classes, exe_ctx))
+                      .value_or(0) > 0) {
                 if (idx == 0) {
                   clang::QualType ivar_qual_type(
                       getASTContext().getObjCInterfaceType(
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
index a73164895baa..05c303baa416 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
@@ -833,9 +833,10 @@ public:
   GetTypeBitAlign(lldb::opaque_compiler_type_t type,
                   ExecutionContextScope *exe_scope) override;
 
-  uint32_t GetNumChildren(lldb::opaque_compiler_type_t type,
-                          bool omit_empty_base_classes,
-                          const ExecutionContext *exe_ctx) override;
+  llvm::Expected
+  GetNumChildren(lldb::opaque_compiler_type_t type,
+                 bool omit_empty_base_classes,
+                 const ExecutionContext *exe_ctx) override;
 
   CompilerType GetBuiltinTypeByName(ConstString name) override;
 
diff --git a/lldb/source/Symbol/CompilerType.cpp b/lldb/source/Symbol/CompilerType.cpp
index 76b79daa6ac1..85dd2d841a5a 100644
--- a/lldb/source/Symbol/CompilerType.cpp
+++ b/lldb/source/Symbol/CompilerType.cpp
@@ -770,8 +770,9 @@ lldb::Format CompilerType::GetFormat() const {
   return lldb::eFormatDefault;
 }
 
-uint32_t CompilerType::GetNumChildren(bool omit_empty_base_classes,
-                                      const ExecutionContext *exe_ctx) const {
+llvm::Expected
+CompilerType::GetNumChildren(bool omit_empty_base_classes,
+                             const ExecutionContext *exe_ctx) const {
   if (IsValid())
     if (auto type_system_sp = GetTypeSystem())
       return type_system_sp->GetNumChildren(m_type, omit_empty_base_classes,
diff --git a/lldb/source/Symbol/Type.cpp b/lldb/source/Symbol/Type.cpp
index 6069d066eaf6..44a24d7178f5 100644
--- a/lldb/source/Symbol/Type.cpp
+++ b/lldb/source/Symbol/Type.cpp
@@ -488,7 +488,7 @@ std::optional Type::GetByteSize(ExecutionContextScope *exe_scope) {
   return {};
 }
 
-uint32_t Type::GetNumChildren(bool omit_empty_base_classes) {
+llvm::Expected Type::GetNumChildren(bool omit_empty_base_classes) {
   return GetForwardCompilerType().GetNumChildren(omit_empty_base_classes, nullptr);
 }
 
diff --git a/lldb/source/Symbol/Variable.cpp b/lldb/source/Symbol/Variable.cpp
index a33c3433d9e2..90edede0f065 100644
--- a/lldb/source/Symbol/Variable.cpp
+++ b/lldb/source/Symbol/Variable.cpp
@@ -27,6 +27,8 @@
 #include "lldb/Target/StackFrame.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Target/Thread.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegularExpression.h"
 #include "lldb/Utility/Stream.h"
 
@@ -568,7 +570,9 @@ static void PrivateAutoComplete(
       case eTypeClassObjCObjectPointer:
       case eTypeClassPointer: {
         bool omit_empty_base_classes = true;
-        if (compiler_type.GetNumChildren(omit_empty_base_classes, nullptr) > 0)
+        if (llvm::expectedToStdOptional(
+                compiler_type.GetNumChildren(omit_empty_base_classes, nullptr))
+                .value_or(0))
           request.AddCompletion((prefix_path + "->").str());
         else {
           request.AddCompletion(prefix_path.str());
diff --git a/lldb/source/Target/StackFrame.cpp b/lldb/source/Target/StackFrame.cpp
index 50cf01e63cd4..c29a71d92572 100644
--- a/lldb/source/Target/StackFrame.cpp
+++ b/lldb/source/Target/StackFrame.cpp
@@ -857,10 +857,11 @@ ValueObjectSP StackFrame::GetValueForVariableExpressionPath(
                   "\"(%s) %s\" is not an array type",
                   valobj_sp->GetTypeName().AsCString(""),
                   var_expr_path_strm.GetData());
-            } else if (
-                static_cast(child_index) >=
-                synthetic
-                    ->GetNumChildren() /* synthetic does not have that many values */) {
+            } else if (static_cast(child_index) >=
+                       synthetic
+                           ->GetNumChildrenIgnoringErrors() /* synthetic does
+                                                                not have that
+                                                                many values */) {
               valobj_sp->GetExpressionPath(var_expr_path_strm);
               error.SetErrorStringWithFormat(
                   "array index %ld is not valid for \"(%s) %s\"", child_index,
@@ -929,10 +930,9 @@ ValueObjectSP StackFrame::GetValueForVariableExpressionPath(
                 "\"(%s) %s\" is not an array type",
                 valobj_sp->GetTypeName().AsCString(""),
                 var_expr_path_strm.GetData());
-          } else if (
-              static_cast(child_index) >=
-              synthetic
-                  ->GetNumChildren() /* synthetic does not have that many values */) {
+          } else if (static_cast(child_index) >=
+                     synthetic->GetNumChildrenIgnoringErrors() /* synthetic
+                                     does not have that many values */) {
             valobj_sp->GetExpressionPath(var_expr_path_strm);
             error.SetErrorStringWithFormat(
                 "array index %ld is not valid for \"(%s) %s\"", child_index,
@@ -1397,7 +1397,8 @@ ValueObjectSP GetValueForOffset(StackFrame &frame, ValueObjectSP &parent,
     return parent;
   }
 
-  for (int ci = 0, ce = parent->GetNumChildren(); ci != ce; ++ci) {
+  for (int ci = 0, ce = parent->GetNumChildrenIgnoringErrors(); ci != ce;
+       ++ci) {
     ValueObjectSP child_sp = parent->GetChildAtIndex(ci);
 
     if (!child_sp) {
-- 
GitLab


From addda68f08191ecb91f22f48570f3defbd364b1d Mon Sep 17 00:00:00 2001
From: Changpeng Fang 
Date: Fri, 8 Mar 2024 10:53:28 -0800
Subject: [PATCH 644/929] AMDGPU: Rename HasVinterInsts to HasVINTERPEncoding,
 NFC (#84535)

---
 llvm/lib/Target/AMDGPU/AMDGPU.td              | 2 +-
 llvm/lib/Target/AMDGPU/GCNSubtarget.h         | 2 +-
 llvm/lib/Target/AMDGPU/VINTERPInstructions.td | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 5d905f5cadc0..7183148e1310 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -1884,7 +1884,7 @@ def NotLDSRequiresM0Init : Predicate<"!Subtarget->ldsRequiresM0Init()">;
 def HasExportInsts : Predicate<"Subtarget->hasExportInsts()">,
   AssemblerPredicate<(all_of (not FeatureGFX90AInsts))>;
 
-def HasInterpInsts : Predicate<"Subtarget->hasInterpInsts()">,
+def HasVINTERPEncoding : Predicate<"Subtarget->hasVINTERPEncoding()">,
   AssemblerPredicate<(all_of FeatureGFX11Insts)>;
 
 def HasDSAddTid : Predicate<"Subtarget->getGeneration() >= AMDGPUSubtarget::GFX9">,
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index afe3af07dc37..ca51da659c33 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -652,7 +652,7 @@ public:
     return !hasGFX940Insts();
   }
 
-  bool hasInterpInsts() const {
+  bool hasVINTERPEncoding() const {
     return GFX11Insts;
   }
 
diff --git a/llvm/lib/Target/AMDGPU/VINTERPInstructions.td b/llvm/lib/Target/AMDGPU/VINTERPInstructions.td
index 0303d1e23a0a..77063e2b70f6 100644
--- a/llvm/lib/Target/AMDGPU/VINTERPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VINTERPInstructions.td
@@ -105,7 +105,7 @@ class VOP3_VINTERP_F16  ArgVT> : VOPProfile {
 // VINTERP Pseudo Instructions
 //===----------------------------------------------------------------------===//
 
-let SubtargetPredicate = HasInterpInsts in {
+let SubtargetPredicate = HasVINTERPEncoding in {
 
 let Uses = [M0, EXEC, MODE] in {
 def V_INTERP_P10_F32_inreg : VINTERP_Pseudo <"v_interp_p10_f32", VOP3_VINTERP_F32>;
@@ -123,7 +123,7 @@ def V_INTERP_P2_RTZ_F16_F32_inreg :
   VINTERP_Pseudo <"v_interp_p2_rtz_f16_f32", VOP3_VINTERP_F16<[f16, f32, f32, f32]>>;
 } // Uses = [M0, EXEC]
 
-} // SubtargetPredicate = HasInterpInsts.
+} // SubtargetPredicate = HasVINTERPEncoding.
 
 class VInterpF32Pat  : GCNPat <
    (f32 (op
-- 
GitLab


From 2bd369b48dbf0bc3128becb7ef8f8a1b82514b87 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <5361294+alexey-bataev@users.noreply.github.com>
Date: Fri, 8 Mar 2024 13:57:02 -0500
Subject: [PATCH 645/929] [SLP]Improve minbitwidth analysis.

This improves overall analysis for minbitwidth in SLP. It allows to
analyze the trees with store/insertelement root nodes. Also, instead of
using single minbitwidth, detected from the very first analysis stage,
it tries to detect the best one for each trunc/ext subtree in the graph
and use it for the subtree.
Results in better code and less vector register pressure.

Metric: size..text

Program                                                                                                                                                size..text
                                                                                                                                                       results     results0    diff
                                                                      test-suite :: SingleSource/Benchmarks/Adobe-C++/simple_types_loop_invariant.test    92549.00    92609.00  0.1%
                                                                                  test-suite :: External/SPEC/CINT2017speed/625.x264_s/625.x264_s.test   663381.00   663493.00  0.0%
                                                                                   test-suite :: External/SPEC/CINT2017rate/525.x264_r/525.x264_r.test   663381.00   663493.00  0.0%
                                                                                               test-suite :: MultiSource/Benchmarks/Bullet/bullet.test   307182.00   307214.00  0.0%
                                                                             test-suite :: External/SPEC/CFP2017speed/638.imagick_s/638.imagick_s.test  1394420.00  1394484.00  0.0%
                                                                              test-suite :: External/SPEC/CFP2017rate/538.imagick_r/538.imagick_r.test  1394420.00  1394484.00  0.0%
                                                                                test-suite :: External/SPEC/CFP2017rate/510.parest_r/510.parest_r.test  2040257.00  2040273.00  0.0%

                                                                              test-suite :: External/SPEC/CFP2017rate/526.blender_r/526.blender_r.test 12396098.00 12395858.00 -0.0%
                                                                                         test-suite :: External/SPEC/CINT2006/445.gobmk/445.gobmk.test   909944.00   909768.00 -0.0%

SingleSource/Benchmarks/Adobe-C++/simple_types_loop_invariant - 4 scalar
instructions remain scalar (good).
Spec2017/x264 - the whole function idct4x4dc is vectorized using <16
x i16> instead of <16 x i32>, also zext/trunc are removed. In other
places last vector zext/sext removed and replaced by
extractelement + scalar zext/sext pair.
MultiSource/Benchmarks/Bullet/bullet - reduce or <4 x i32> replaced by
reduce or <4 x i8>
Spec2017/imagick - Removed extra zext from 2 packs of the operations.
Spec2017/parest - Removed extra zext, replaced by extractelement+scalar
zext
Spec2017/blender - the whole bunch of vector zext/sext replaced by
extractelement+scalar zext/sext, some extra code vectorized in smaller
types.
Spec2006/gobmk - fixed cost estimation, some small code remains scalar.

Original Pull Request: https://github.com/llvm/llvm-project/pull/84334

The patch has the same functionality (no test changes, no changes in
benchmarks) as the original patch, just has some compile time
improvements + fixes for xxhash unittest, discovered earlier in the
previous version of the patch.

Reviewers:

Pull Request: https://github.com/llvm/llvm-project/pull/84536
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 676 +++++++++++++-----
 .../SLPVectorizer/AArch64/ext-trunc.ll        |   9 +-
 .../SLPVectorizer/AArch64/getelementptr2.ll   |   4 +-
 .../SLPVectorizer/AArch64/reduce-add-i64.ll   |  20 +-
 .../SLPVectorizer/RISCV/reductions.ll         |   7 +-
 .../Transforms/SLPVectorizer/X86/PR35777.ll   |   9 +-
 .../X86/int-bitcast-minbitwidth.ll            |   2 +-
 ...minbitwidth-multiuse-with-insertelement.ll |  17 +-
 .../X86/minbitwidth-transformed-operand.ll    |  21 +-
 .../SLPVectorizer/X86/minimum-sizes.ll        |  43 +-
 .../SLPVectorizer/X86/phi-undef-input.ll      |  24 +-
 .../Transforms/SLPVectorizer/X86/resched.ll   |  32 +-
 .../X86/reused-reductions-with-minbitwidth.ll |  10 +-
 .../X86/store-insertelement-minbitwidth.ll    |  22 +-
 .../SLPVectorizer/alt-cmp-vectorize.ll        |   4 +-
 15 files changed, 594 insertions(+), 306 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 36dc9094538a..a5c34bfbf9b4 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1085,6 +1085,9 @@ public:
       BS->clear();
     }
     MinBWs.clear();
+    ReductionBitWidth = 0;
+    CastMaxMinBWSizes.reset();
+    TruncNodes.clear();
     InstrElementSize.clear();
     UserIgnoreList = nullptr;
     PostponedGathers.clear();
@@ -2287,6 +2290,7 @@ public:
   void clearReductionData() {
     AnalyzedReductionsRoots.clear();
     AnalyzedReductionVals.clear();
+    AnalyzedMinBWVals.clear();
   }
   /// Checks if the given value is gathered in one of the nodes.
   bool isAnyGathered(const SmallDenseSet &Vals) const {
@@ -2307,9 +2311,11 @@ private:
   /// constant and to be demoted. Required to correctly identify constant nodes
   /// to be demoted.
   bool collectValuesToDemote(
-      Value *V, SmallVectorImpl &ToDemote,
+      Value *V, bool IsProfitableToDemoteRoot, unsigned &BitWidth,
+      SmallVectorImpl &ToDemote,
       DenseMap> &DemotedConsts,
-      SmallVectorImpl &Roots, DenseSet &Visited) const;
+      DenseSet &Visited, unsigned &MaxDepthLevel,
+      bool &IsProfitableToDemote) const;
 
   /// Check if the operands on the edges \p Edges of the \p UserTE allows
   /// reordering (i.e. the operands can be reordered because they have only one
@@ -2375,6 +2381,10 @@ private:
   /// \ returns the graph entry for the \p Idx operand of the \p E entry.
   const TreeEntry *getOperandEntry(const TreeEntry *E, unsigned Idx) const;
 
+  /// \returns Cast context for the given graph node.
+  TargetTransformInfo::CastContextHint
+  getCastContextHint(const TreeEntry &TE) const;
+
   /// \returns the cost of the vectorizable entry.
   InstructionCost getEntryCost(const TreeEntry *E,
                                ArrayRef VectorizedVals,
@@ -2925,11 +2935,18 @@ private:
       }
       assert(!BundleMember && "Bundle and VL out of sync");
     } else {
-      MustGather.insert(VL.begin(), VL.end());
       // Build a map for gathered scalars to the nodes where they are used.
+      bool AllConstsOrCasts = true;
       for (Value *V : VL)
-        if (!isConstant(V))
+        if (!isConstant(V)) {
+          auto *I = dyn_cast(V);
+          AllConstsOrCasts &= I && I->getType()->isIntegerTy();
           ValueToGatherNodes.try_emplace(V).first->getSecond().insert(Last);
+        }
+      if (AllConstsOrCasts)
+        CastMaxMinBWSizes =
+            std::make_pair(std::numeric_limits::max(), 1);
+      MustGather.insert(VL.begin(), VL.end());
     }
 
     if (UserTreeIdx.UserTE)
@@ -3054,6 +3071,10 @@ private:
   /// Set of hashes for the list of reduction values already being analyzed.
   DenseSet AnalyzedReductionVals;
 
+  /// Values, already been analyzed for mininmal bitwidth and found to be
+  /// non-profitable.
+  DenseSet AnalyzedMinBWVals;
+
   /// A list of values that need to extracted out of the tree.
   /// This list holds pairs of (Internal Scalar : External User). External User
   /// can be nullptr, it means that this Internal Scalar will be used later,
@@ -3629,6 +3650,18 @@ private:
   /// value must be signed-extended, rather than zero-extended, back to its
   /// original width.
   DenseMap> MinBWs;
+
+  /// Final size of the reduced vector, if the current graph represents the
+  /// input for the reduction and it was possible to narrow the size of the
+  /// reduction.
+  unsigned ReductionBitWidth = 0;
+
+  /// If the tree contains any zext/sext/trunc nodes, contains max-min pair of
+  /// type sizes, used in the tree.
+  std::optional> CastMaxMinBWSizes;
+
+  /// Indices of the vectorized trunc nodes.
+  DenseSet TruncNodes;
 };
 
 } // end namespace slpvectorizer
@@ -6539,8 +6572,29 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth,
     case Instruction::Trunc:
     case Instruction::FPTrunc:
     case Instruction::BitCast: {
+      auto [PrevMaxBW, PrevMinBW] = CastMaxMinBWSizes.value_or(
+          std::make_pair(std::numeric_limits::min(),
+                         std::numeric_limits::max()));
+      if (ShuffleOrOp == Instruction::ZExt ||
+          ShuffleOrOp == Instruction::SExt) {
+        CastMaxMinBWSizes = std::make_pair(
+            std::max(DL->getTypeSizeInBits(VL0->getType()),
+                               PrevMaxBW),
+            std::min(
+                DL->getTypeSizeInBits(VL0->getOperand(0)->getType()),
+                PrevMinBW));
+      } else if (ShuffleOrOp == Instruction::Trunc) {
+        CastMaxMinBWSizes = std::make_pair(
+            std::max(
+                DL->getTypeSizeInBits(VL0->getOperand(0)->getType()),
+                PrevMaxBW),
+            std::min(DL->getTypeSizeInBits(VL0->getType()),
+                               PrevMinBW));
+        TruncNodes.insert(VectorizableTree.size());
+      }
       TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
                                    ReuseShuffleIndicies);
+
       LLVM_DEBUG(dbgs() << "SLP: added a vector of casts.\n");
 
       TE->setOperandsInOrder();
@@ -8362,6 +8416,22 @@ const BoUpSLP::TreeEntry *BoUpSLP::getOperandEntry(const TreeEntry *E,
   return It->get();
 }
 
+TTI::CastContextHint BoUpSLP::getCastContextHint(const TreeEntry &TE) const {
+  if (TE.State == TreeEntry::ScatterVectorize ||
+      TE.State == TreeEntry::StridedVectorize)
+    return TTI::CastContextHint::GatherScatter;
+  if (TE.State == TreeEntry::Vectorize && TE.getOpcode() == Instruction::Load &&
+      !TE.isAltShuffle()) {
+    if (TE.ReorderIndices.empty())
+      return TTI::CastContextHint::Normal;
+    SmallVector Mask;
+    inversePermutation(TE.ReorderIndices, Mask);
+    if (ShuffleVectorInst::isReverseMask(Mask, Mask.size()))
+      return TTI::CastContextHint::Reversed;
+  }
+  return TTI::CastContextHint::None;
+}
+
 InstructionCost
 BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals,
                       SmallPtrSetImpl &CheckedExtracts) {
@@ -8384,6 +8454,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals,
   // If we have computed a smaller type for the expression, update VecTy so
   // that the costs will be accurate.
   auto It = MinBWs.find(E);
+  Type *OrigScalarTy = ScalarTy;
   if (It != MinBWs.end()) {
     ScalarTy = IntegerType::get(F->getContext(), It->second.first);
     VecTy = FixedVectorType::get(ScalarTy, VL.size());
@@ -8441,24 +8512,11 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals,
     UsedScalars.set(I);
   }
   auto GetCastContextHint = [&](Value *V) {
-    if (const TreeEntry *OpTE = getTreeEntry(V)) {
-      if (OpTE->State == TreeEntry::ScatterVectorize ||
-          OpTE->State == TreeEntry::StridedVectorize)
-        return TTI::CastContextHint::GatherScatter;
-      if (OpTE->State == TreeEntry::Vectorize &&
-          OpTE->getOpcode() == Instruction::Load && !OpTE->isAltShuffle()) {
-        if (OpTE->ReorderIndices.empty())
-          return TTI::CastContextHint::Normal;
-        SmallVector Mask;
-        inversePermutation(OpTE->ReorderIndices, Mask);
-        if (ShuffleVectorInst::isReverseMask(Mask, Mask.size()))
-          return TTI::CastContextHint::Reversed;
-      }
-    } else {
-      InstructionsState SrcState = getSameOpcode(E->getOperand(0), *TLI);
-      if (SrcState.getOpcode() == Instruction::Load && !SrcState.isAltShuffle())
-        return TTI::CastContextHint::GatherScatter;
-    }
+    if (const TreeEntry *OpTE = getTreeEntry(V))
+      return getCastContextHint(*OpTE);
+    InstructionsState SrcState = getSameOpcode(E->getOperand(0), *TLI);
+    if (SrcState.getOpcode() == Instruction::Load && !SrcState.isAltShuffle())
+      return TTI::CastContextHint::GatherScatter;
     return TTI::CastContextHint::None;
   };
   auto GetCostDiff =
@@ -8507,8 +8565,6 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals,
               TTI::CastContextHint CCH = GetCastContextHint(VL0);
               VecCost += TTI->getCastInstrCost(VecOpcode, UserVecTy, VecTy, CCH,
                                                CostKind);
-              ScalarCost += Sz * TTI->getCastInstrCost(VecOpcode, UserScalarTy,
-                                                       ScalarTy, CCH, CostKind);
             }
           }
         }
@@ -8525,7 +8581,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals,
     InstructionCost ScalarCost = 0;
     InstructionCost VecCost = 0;
     std::tie(ScalarCost, VecCost) = getGEPCosts(
-        *TTI, Ptrs, BasePtr, E->getOpcode(), CostKind, ScalarTy, VecTy);
+        *TTI, Ptrs, BasePtr, E->getOpcode(), CostKind, OrigScalarTy, VecTy);
     LLVM_DEBUG(dumpTreeCosts(E, 0, VecCost, ScalarCost,
                              "Calculated GEPs cost for Tree"));
 
@@ -8572,7 +8628,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals,
           NumElts = ATy->getNumElements();
         else
           NumElts = AggregateTy->getStructNumElements();
-        SrcVecTy = FixedVectorType::get(ScalarTy, NumElts);
+        SrcVecTy = FixedVectorType::get(OrigScalarTy, NumElts);
       }
       if (I->hasOneUse()) {
         Instruction *Ext = I->user_back();
@@ -8740,13 +8796,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals,
       }
     }
     auto GetScalarCost = [&](unsigned Idx) -> InstructionCost {
-      // Do not count cost here if minimum bitwidth is in effect and it is just
-      // a bitcast (here it is just a noop).
-      if (VecOpcode != Opcode && VecOpcode == Instruction::BitCast)
-        return TTI::TCC_Free;
-      auto *VI = VL0->getOpcode() == Opcode
-                     ? cast(UniqueValues[Idx])
-                     : nullptr;
+      auto *VI = cast(UniqueValues[Idx]);
       return TTI->getCastInstrCost(Opcode, VL0->getType(),
                                    VL0->getOperand(0)->getType(),
                                    TTI::getCastContextHint(VI), CostKind, VI);
@@ -8789,7 +8839,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals,
                                        ? CmpInst::BAD_FCMP_PREDICATE
                                        : CmpInst::BAD_ICMP_PREDICATE;
 
-      return TTI->getCmpSelInstrCost(E->getOpcode(), ScalarTy,
+      return TTI->getCmpSelInstrCost(E->getOpcode(), OrigScalarTy,
                                      Builder.getInt1Ty(), CurrentPred, CostKind,
                                      VI);
     };
@@ -8844,7 +8894,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals,
       TTI::OperandValueInfo Op2Info =
           TTI::getOperandInfo(VI->getOperand(OpIdx));
       SmallVector Operands(VI->operand_values());
-      return TTI->getArithmeticInstrCost(ShuffleOrOp, ScalarTy, CostKind,
+      return TTI->getArithmeticInstrCost(ShuffleOrOp, OrigScalarTy, CostKind,
                                          Op1Info, Op2Info, Operands, VI);
     };
     auto GetVectorCost = [=](InstructionCost CommonCost) {
@@ -8863,9 +8913,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals,
   case Instruction::Load: {
     auto GetScalarCost = [&](unsigned Idx) {
       auto *VI = cast(UniqueValues[Idx]);
-      return TTI->getMemoryOpCost(Instruction::Load, ScalarTy, VI->getAlign(),
-                                  VI->getPointerAddressSpace(), CostKind,
-                                  TTI::OperandValueInfo(), VI);
+      return TTI->getMemoryOpCost(Instruction::Load, OrigScalarTy,
+                                  VI->getAlign(), VI->getPointerAddressSpace(),
+                                  CostKind, TTI::OperandValueInfo(), VI);
     };
     auto *LI0 = cast(VL0);
     auto GetVectorCost = [&](InstructionCost CommonCost) {
@@ -8908,9 +8958,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals,
     auto GetScalarCost = [=](unsigned Idx) {
       auto *VI = cast(VL[Idx]);
       TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(VI->getValueOperand());
-      return TTI->getMemoryOpCost(Instruction::Store, ScalarTy, VI->getAlign(),
-                                  VI->getPointerAddressSpace(), CostKind,
-                                  OpInfo, VI);
+      return TTI->getMemoryOpCost(Instruction::Store, OrigScalarTy,
+                                  VI->getAlign(), VI->getPointerAddressSpace(),
+                                  CostKind, OpInfo, VI);
     };
     auto *BaseSI =
         cast(IsReorder ? VL[E->ReorderIndices.front()] : VL0);
@@ -9772,6 +9822,44 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef VectorizedVals) {
     Cost -= InsertCost;
   }
 
+  // Add the cost for reduced value resize (if required).
+  if (ReductionBitWidth != 0) {
+    assert(UserIgnoreList && "Expected reduction tree.");
+    const TreeEntry &E = *VectorizableTree.front().get();
+    auto It = MinBWs.find(&E);
+    if (It != MinBWs.end() && It->second.first != ReductionBitWidth) {
+      unsigned SrcSize = It->second.first;
+      unsigned DstSize = ReductionBitWidth;
+      unsigned Opcode = Instruction::Trunc;
+      if (SrcSize < DstSize)
+        Opcode = It->second.second ? Instruction::SExt : Instruction::ZExt;
+      auto *SrcVecTy =
+          FixedVectorType::get(Builder.getIntNTy(SrcSize), E.getVectorFactor());
+      auto *DstVecTy =
+          FixedVectorType::get(Builder.getIntNTy(DstSize), E.getVectorFactor());
+      TTI::CastContextHint CCH = getCastContextHint(E);
+      InstructionCost CastCost;
+      switch (E.getOpcode()) {
+      case Instruction::SExt:
+      case Instruction::ZExt:
+      case Instruction::Trunc: {
+        const TreeEntry *OpTE = getOperandEntry(&E, 0);
+        CCH = getCastContextHint(*OpTE);
+        break;
+      }
+      default:
+        break;
+      }
+      CastCost += TTI->getCastInstrCost(Opcode, DstVecTy, SrcVecTy, CCH,
+                                        TTI::TCK_RecipThroughput);
+      Cost += CastCost;
+      LLVM_DEBUG(dbgs() << "SLP: Adding cost " << CastCost
+                        << " for final resize for reduction from " << SrcVecTy
+                        << " to " << DstVecTy << "\n";
+                 dbgs() << "SLP: Current total cost = " << Cost << "\n");
+    }
+  }
+
 #ifndef NDEBUG
   SmallString<256> Str;
   {
@@ -10042,7 +10130,7 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
             continue;
           VTE = *It->getSecond().begin();
           // Iterate through all vectorized nodes.
-          auto *MIt = find_if(It->getSecond(), [](const TreeEntry *MTE) {
+          auto *MIt = find_if(It->getSecond(), [&](const TreeEntry *MTE) {
             return MTE->State == TreeEntry::Vectorize;
           });
           if (MIt == It->getSecond().end())
@@ -10053,11 +10141,6 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
       Instruction &LastBundleInst = getLastInstructionInBundle(VTE);
       if (&LastBundleInst == TEInsertPt || !CheckOrdering(&LastBundleInst))
         continue;
-      auto It = MinBWs.find(VTE);
-      // If vectorize node is demoted - do not match.
-      if (It != MinBWs.end() &&
-          It->second.first != DL->getTypeSizeInBits(V->getType()))
-        continue;
       VToTEs.insert(VTE);
     }
     if (VToTEs.empty())
@@ -10105,6 +10188,57 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
     return std::nullopt;
   }
 
+  // Filter out entries with larger bitwidth of elements.
+  Type *ScalarTy = VL.front()->getType();
+  unsigned BitWidth = 0;
+  if (ScalarTy->isIntegerTy()) {
+    // Check if the used TEs supposed to be resized and choose the best
+    // candidates.
+    BitWidth = DL->getTypeStoreSize(ScalarTy);
+    if (TEUseEI.UserTE->getOpcode() != Instruction::Select ||
+        TEUseEI.EdgeIdx != 0) {
+      auto UserIt = MinBWs.find(TEUseEI.UserTE);
+      if (UserIt != MinBWs.end())
+        BitWidth = UserIt->second.second;
+    }
+    // Check if the used TEs supposed to be resized and choose the best
+    // candidates.
+    unsigned NodesBitWidth = 0;
+    auto CheckBitwidth = [&](const TreeEntry &TE) {
+      unsigned TEBitWidth = BitWidth;
+      auto UserIt = MinBWs.find(TEUseEI.UserTE);
+      if (UserIt != MinBWs.end())
+        TEBitWidth = UserIt->second.second;
+      if (BitWidth <= TEBitWidth) {
+        if (NodesBitWidth == 0)
+          NodesBitWidth = TEBitWidth;
+        return NodesBitWidth == TEBitWidth;
+      }
+      return false;
+    };
+    for (auto [Idx, Set] : enumerate(UsedTEs)) {
+      DenseSet ForRemoval;
+      for (const TreeEntry *TE : Set) {
+        if (!CheckBitwidth(*TE))
+          ForRemoval.insert(TE);
+      }
+      // All elements must be removed - remove the whole container.
+      if (ForRemoval.size() == Set.size()) {
+        Set.clear();
+        continue;
+      }
+      for (const TreeEntry *TE : ForRemoval)
+        Set.erase(TE);
+    }
+    for (auto *It = UsedTEs.begin(); It != UsedTEs.end();) {
+      if (It->empty()) {
+        UsedTEs.erase(It);
+        continue;
+      }
+      std::advance(It, 1);
+    }
+  }
+
   unsigned VF = 0;
   if (UsedTEs.size() == 1) {
     // Keep the order to avoid non-determinism.
@@ -12929,7 +13063,21 @@ Value *BoUpSLP::vectorizeTree(
   Builder.ClearInsertionPoint();
   InstrElementSize.clear();
 
-  return VectorizableTree[0]->VectorizedValue;
+  const TreeEntry &RootTE = *VectorizableTree.front().get();
+  Value *Vec = RootTE.VectorizedValue;
+  if (auto It = MinBWs.find(&RootTE); ReductionBitWidth != 0 &&
+                                      It != MinBWs.end() &&
+                                      ReductionBitWidth != It->second.first) {
+    IRBuilder<>::InsertPointGuard Guard(Builder);
+    Builder.SetInsertPoint(ReductionRoot->getParent(),
+                           ReductionRoot->getIterator());
+    Vec = Builder.CreateIntCast(
+        Vec,
+        VectorType::get(Builder.getIntNTy(ReductionBitWidth),
+                        cast(Vec->getType())->getElementCount()),
+        It->second.second);
+  }
+  return Vec;
 }
 
 void BoUpSLP::optimizeGatherSequence() {
@@ -13749,23 +13897,48 @@ unsigned BoUpSLP::getVectorElementSize(Value *V) {
 // smaller type with a truncation. We collect the values that will be demoted
 // in ToDemote and additional roots that require investigating in Roots.
 bool BoUpSLP::collectValuesToDemote(
-    Value *V, SmallVectorImpl &ToDemote,
+    Value *V, bool IsProfitableToDemoteRoot, unsigned &BitWidth,
+    SmallVectorImpl &ToDemote,
     DenseMap> &DemotedConsts,
-    SmallVectorImpl &Roots, DenseSet &Visited) const {
+    DenseSet &Visited, unsigned &MaxDepthLevel,
+    bool &IsProfitableToDemote) const {
   // We can always demote constants.
-  if (isa(V))
+  if (isa(V)) {
+    MaxDepthLevel = 1;
     return true;
+  }
+
+  if (DL->getTypeSizeInBits(V->getType()) == BitWidth) {
+    MaxDepthLevel = 1;
+    return true;
+  }
 
   // If the value is not a vectorized instruction in the expression and not used
   // by the insertelement instruction and not used in multiple vector nodes, it
   // cannot be demoted.
+  // TODO: improve handling of gathered values and others.
   auto *I = dyn_cast(V);
-  if (!I || !getTreeEntry(I) || MultiNodeScalars.contains(I) ||
-      !Visited.insert(I).second || all_of(I->users(), [&](User *U) {
+  const TreeEntry *ITE = I ? getTreeEntry(I) : nullptr;
+  if (!ITE || !Visited.insert(I).second || MultiNodeScalars.contains(I) ||
+      all_of(I->users(), [&](User *U) {
         return isa(U) && !getTreeEntry(U);
       }))
     return false;
 
+  auto IsPotentiallyTruncated = [&](Value *V, unsigned &BitWidth) -> bool {
+    if (MultiNodeScalars.contains(V))
+      return false;
+    uint32_t OrigBitWidth = DL->getTypeSizeInBits(V->getType());
+    APInt Mask = APInt::getBitsSetFrom(OrigBitWidth, BitWidth);
+    if (MaskedValueIsZero(V, Mask, SimplifyQuery(*DL)))
+      return true;
+    auto NumSignBits = ComputeNumSignBits(V, *DL, 0, AC, nullptr, DT);
+    unsigned BitWidth1 = OrigBitWidth - NumSignBits;
+    if (!isKnownNonNegative(V, SimplifyQuery(*DL)))
+      ++BitWidth1;
+    BitWidth = std::max(BitWidth, BitWidth1);
+    return BitWidth > 0 && OrigBitWidth >= (BitWidth * 2);
+  };
   unsigned Start = 0;
   unsigned End = I->getNumOperands();
   switch (I->getOpcode()) {
@@ -13773,12 +13946,14 @@ bool BoUpSLP::collectValuesToDemote(
   // We can always demote truncations and extensions. Since truncations can
   // seed additional demotion, we save the truncated value.
   case Instruction::Trunc:
-    Roots.push_back(I->getOperand(0));
+    MaxDepthLevel = 1;
+    if (IsProfitableToDemoteRoot)
+      IsProfitableToDemote = true;
     break;
   case Instruction::ZExt:
   case Instruction::SExt:
-    if (isa(I->getOperand(0)))
-      return false;
+    MaxDepthLevel = 1;
+    IsProfitableToDemote = true;
     break;
 
   // We can demote certain binary operations if we can demote both of their
@@ -13788,23 +13963,36 @@ bool BoUpSLP::collectValuesToDemote(
   case Instruction::Mul:
   case Instruction::And:
   case Instruction::Or:
-  case Instruction::Xor:
-    if (!collectValuesToDemote(I->getOperand(0), ToDemote, DemotedConsts, Roots,
-                               Visited) ||
-        !collectValuesToDemote(I->getOperand(1), ToDemote, DemotedConsts, Roots,
-                               Visited))
+  case Instruction::Xor: {
+    unsigned Level1, Level2;
+    if ((ITE->UserTreeIndices.size() > 1 &&
+         !IsPotentiallyTruncated(I, BitWidth)) ||
+        !collectValuesToDemote(I->getOperand(0), IsProfitableToDemoteRoot,
+                               BitWidth, ToDemote, DemotedConsts, Visited,
+                               Level1, IsProfitableToDemote) ||
+        !collectValuesToDemote(I->getOperand(1), IsProfitableToDemoteRoot,
+                               BitWidth, ToDemote, DemotedConsts, Visited,
+                               Level2, IsProfitableToDemote))
       return false;
+    MaxDepthLevel = std::max(Level1, Level2);
     break;
+  }
 
   // We can demote selects if we can demote their true and false values.
   case Instruction::Select: {
     Start = 1;
+    unsigned Level1, Level2;
     SelectInst *SI = cast(I);
-    if (!collectValuesToDemote(SI->getTrueValue(), ToDemote, DemotedConsts,
-                               Roots, Visited) ||
-        !collectValuesToDemote(SI->getFalseValue(), ToDemote, DemotedConsts,
-                               Roots, Visited))
+    if ((ITE->UserTreeIndices.size() > 1 &&
+         !IsPotentiallyTruncated(I, BitWidth)) ||
+        !collectValuesToDemote(SI->getTrueValue(), IsProfitableToDemoteRoot,
+                               BitWidth, ToDemote, DemotedConsts, Visited,
+                               Level1, IsProfitableToDemote) ||
+        !collectValuesToDemote(SI->getFalseValue(), IsProfitableToDemoteRoot,
+                               BitWidth, ToDemote, DemotedConsts, Visited,
+                               Level2, IsProfitableToDemote))
       return false;
+    MaxDepthLevel = std::max(Level1, Level2);
     break;
   }
 
@@ -13812,172 +14000,270 @@ bool BoUpSLP::collectValuesToDemote(
   // we don't need to worry about cycles since we ensure single use above.
   case Instruction::PHI: {
     PHINode *PN = cast(I);
-    for (Value *IncValue : PN->incoming_values())
-      if (!collectValuesToDemote(IncValue, ToDemote, DemotedConsts, Roots,
-                                 Visited))
+    MaxDepthLevel = 0;
+    if (ITE->UserTreeIndices.size() > 1 && !IsPotentiallyTruncated(I, BitWidth))
+      return false;
+    for (Value *IncValue : PN->incoming_values()) {
+      unsigned Level;
+      if (!collectValuesToDemote(IncValue, IsProfitableToDemoteRoot, BitWidth,
+                                 ToDemote, DemotedConsts, Visited, Level,
+                                 IsProfitableToDemote))
         return false;
+      MaxDepthLevel = std::max(MaxDepthLevel, Level);
+    }
     break;
   }
 
   // Otherwise, conservatively give up.
   default:
-    return false;
+    MaxDepthLevel = 1;
+    return IsProfitableToDemote && IsPotentiallyTruncated(I, BitWidth);
   }
 
+  ++MaxDepthLevel;
   // Gather demoted constant operands.
   for (unsigned Idx : seq(Start, End))
     if (isa(I->getOperand(Idx)))
       DemotedConsts.try_emplace(I).first->getSecond().push_back(Idx);
   // Record the value that we can demote.
   ToDemote.push_back(V);
-  return true;
+  return IsProfitableToDemote;
 }
 
 void BoUpSLP::computeMinimumValueSizes() {
   // We only attempt to truncate integer expressions.
-  auto &TreeRoot = VectorizableTree[0]->Scalars;
-  auto *TreeRootIT = dyn_cast(TreeRoot[0]->getType());
-  if (!TreeRootIT || VectorizableTree.front()->State == TreeEntry::NeedToGather)
+  bool IsStoreOrInsertElt =
+      VectorizableTree.front()->getOpcode() == Instruction::Store ||
+      VectorizableTree.front()->getOpcode() == Instruction::InsertElement;
+  if ((IsStoreOrInsertElt || UserIgnoreList) && TruncNodes.size() <= 1 &&
+      (!CastMaxMinBWSizes || CastMaxMinBWSizes->second == 0 ||
+       CastMaxMinBWSizes->first / CastMaxMinBWSizes->second <= 2))
     return;
 
+  unsigned NodeIdx = 0;
+  if (IsStoreOrInsertElt &&
+      VectorizableTree.front()->State != TreeEntry::NeedToGather)
+    NodeIdx = 1;
+
   // Ensure the roots of the vectorizable tree don't form a cycle.
-  if (!VectorizableTree.front()->UserTreeIndices.empty())
+  if (VectorizableTree[NodeIdx]->State == TreeEntry::NeedToGather ||
+      (NodeIdx == 0 && !VectorizableTree[NodeIdx]->UserTreeIndices.empty()) ||
+      (NodeIdx != 0 && any_of(VectorizableTree[NodeIdx]->UserTreeIndices,
+                              [NodeIdx](const EdgeInfo &EI) {
+                                return EI.UserTE->Idx >
+                                       static_cast(NodeIdx);
+                              })))
     return;
 
-  // Conservatively determine if we can actually truncate the roots of the
-  // expression. Collect the values that can be demoted in ToDemote and
-  // additional roots that require investigating in Roots.
-  SmallVector ToDemote;
-  DenseMap> DemotedConsts;
-  SmallVector Roots;
-  for (auto *Root : TreeRoot) {
-    DenseSet Visited;
-    if (!collectValuesToDemote(Root, ToDemote, DemotedConsts, Roots, Visited))
-      return;
-  }
-
-  // The maximum bit width required to represent all the values that can be
-  // demoted without loss of precision. It would be safe to truncate the roots
-  // of the expression to this width.
-  auto MaxBitWidth = 1u;
-
-  // We first check if all the bits of the roots are demanded. If they're not,
-  // we can truncate the roots to this narrower type.
-  for (auto *Root : TreeRoot) {
-    auto Mask = DB->getDemandedBits(cast(Root));
-    MaxBitWidth = std::max(Mask.getBitWidth() - Mask.countl_zero(),
-                                     MaxBitWidth);
-  }
-
-  // True if the roots can be zero-extended back to their original type, rather
-  // than sign-extended. We know that if the leading bits are not demanded, we
-  // can safely zero-extend. So we initialize IsKnownPositive to True.
-  bool IsKnownPositive = true;
-
-  // If all the bits of the roots are demanded, we can try a little harder to
-  // compute a narrower type. This can happen, for example, if the roots are
-  // getelementptr indices. InstCombine promotes these indices to the pointer
-  // width. Thus, all their bits are technically demanded even though the
-  // address computation might be vectorized in a smaller type.
-  //
-  // We start by looking at each entry that can be demoted. We compute the
-  // maximum bit width required to store the scalar by using ValueTracking to
-  // compute the number of high-order bits we can truncate.
-  if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType()) &&
-      all_of(TreeRoot, [](Value *V) {
-        return all_of(V->users(),
-                      [](User *U) { return isa(U); });
-      })) {
-    MaxBitWidth = 8u;
+  // The first value node for store/insertelement is sext/zext/trunc? Skip it,
+  // resize to the final type.
+  bool IsProfitableToDemoteRoot = !IsStoreOrInsertElt;
+  if (NodeIdx != 0 &&
+      VectorizableTree[NodeIdx]->State == TreeEntry::Vectorize &&
+      (VectorizableTree[NodeIdx]->getOpcode() == Instruction::ZExt ||
+       VectorizableTree[NodeIdx]->getOpcode() == Instruction::SExt ||
+       VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc)) {
+    assert(IsStoreOrInsertElt && "Expected store/insertelement seeded graph.");
+    ++NodeIdx;
+    IsProfitableToDemoteRoot = true;
+  }
+
+  // Analyzed in reduction already and not profitable - exit.
+  if (AnalyzedMinBWVals.contains(VectorizableTree[NodeIdx]->Scalars.front()))
+    return;
 
+  SmallVector ToDemote;
+  DenseMap> DemotedConsts;
+  auto ComputeMaxBitWidth = [&](ArrayRef TreeRoot, unsigned VF,
+                                bool IsTopRoot, bool IsProfitableToDemoteRoot,
+                                unsigned Opcode, unsigned Limit) {
+    ToDemote.clear();
+    auto *TreeRootIT = dyn_cast(TreeRoot[0]->getType());
+    if (!TreeRootIT || !Opcode)
+      return 0u;
+
+    if (AnalyzedMinBWVals.contains(TreeRoot.front()))
+      return 0u;
+
+    unsigned NumParts = TTI->getNumberOfParts(
+        FixedVectorType::get(TreeRoot.front()->getType(), VF));
+
+    // The maximum bit width required to represent all the values that can be
+    // demoted without loss of precision. It would be safe to truncate the roots
+    // of the expression to this width.
+    unsigned MaxBitWidth = 1u;
+
+    // True if the roots can be zero-extended back to their original type,
+    // rather than sign-extended. We know that if the leading bits are not
+    // demanded, we can safely zero-extend. So we initialize IsKnownPositive to
+    // True.
     // Determine if the sign bit of all the roots is known to be zero. If not,
     // IsKnownPositive is set to False.
-    IsKnownPositive = llvm::all_of(TreeRoot, [&](Value *R) {
+    bool IsKnownPositive = all_of(TreeRoot, [&](Value *R) {
       KnownBits Known = computeKnownBits(R, *DL);
       return Known.isNonNegative();
     });
 
-    // Determine the maximum number of bits required to store the scalar
-    // values.
-    for (auto *Scalar : ToDemote) {
-      auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, nullptr, DT);
-      auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType());
-      MaxBitWidth = std::max(NumTypeBits - NumSignBits, MaxBitWidth);
-    }
-
-    // If we can't prove that the sign bit is zero, we must add one to the
-    // maximum bit width to account for the unknown sign bit. This preserves
-    // the existing sign bit so we can safely sign-extend the root back to the
-    // original type. Otherwise, if we know the sign bit is zero, we will
-    // zero-extend the root instead.
-    //
-    // FIXME: This is somewhat suboptimal, as there will be cases where adding
-    //        one to the maximum bit width will yield a larger-than-necessary
-    //        type. In general, we need to add an extra bit only if we can't
-    //        prove that the upper bit of the original type is equal to the
-    //        upper bit of the proposed smaller type. If these two bits are the
-    //        same (either zero or one) we know that sign-extending from the
-    //        smaller type will result in the same value. Here, since we can't
-    //        yet prove this, we are just making the proposed smaller type
-    //        larger to ensure correctness.
-    if (!IsKnownPositive)
-      ++MaxBitWidth;
-  }
-
-  // Round MaxBitWidth up to the next power-of-two.
-  MaxBitWidth = llvm::bit_ceil(MaxBitWidth);
-
-  // If the maximum bit width we compute is less than the with of the roots'
-  // type, we can proceed with the narrowing. Otherwise, do nothing.
-  if (MaxBitWidth >= TreeRootIT->getBitWidth())
-    return;
+    // We first check if all the bits of the roots are demanded. If they're not,
+    // we can truncate the roots to this narrower type.
+    for (auto *Root : TreeRoot) {
+      unsigned NumSignBits = ComputeNumSignBits(Root, *DL, 0, AC, nullptr, DT);
+      TypeSize NumTypeBits = DL->getTypeSizeInBits(Root->getType());
+      unsigned BitWidth1 = NumTypeBits - NumSignBits;
+      // If we can't prove that the sign bit is zero, we must add one to the
+      // maximum bit width to account for the unknown sign bit. This preserves
+      // the existing sign bit so we can safely sign-extend the root back to the
+      // original type. Otherwise, if we know the sign bit is zero, we will
+      // zero-extend the root instead.
+      //
+      // FIXME: This is somewhat suboptimal, as there will be cases where adding
+      //        one to the maximum bit width will yield a larger-than-necessary
+      //        type. In general, we need to add an extra bit only if we can't
+      //        prove that the upper bit of the original type is equal to the
+      //        upper bit of the proposed smaller type. If these two bits are
+      //        the same (either zero or one) we know that sign-extending from
+      //        the smaller type will result in the same value. Here, since we
+      //        can't yet prove this, we are just making the proposed smaller
+      //        type larger to ensure correctness.
+      if (!IsKnownPositive)
+        ++BitWidth1;
+
+      APInt Mask = DB->getDemandedBits(cast(Root));
+      unsigned BitWidth2 = Mask.getBitWidth() - Mask.countl_zero();
+      MaxBitWidth =
+          std::max(std::min(BitWidth1, BitWidth2), MaxBitWidth);
+    }
+
+    if (MaxBitWidth < 8 && MaxBitWidth > 1)
+      MaxBitWidth = 8;
+
+    // If the original type is large, but reduced type does not improve the reg
+    // use - ignore it.
+    if (NumParts > 1 &&
+        NumParts ==
+            TTI->getNumberOfParts(FixedVectorType::get(
+                IntegerType::get(F->getContext(), bit_ceil(MaxBitWidth)), VF)))
+      return 0u;
+
+    bool IsProfitableToDemote = Opcode == Instruction::Trunc ||
+                                Opcode == Instruction::SExt ||
+                                Opcode == Instruction::ZExt || NumParts > 1;
+    // Conservatively determine if we can actually truncate the roots of the
+    // expression. Collect the values that can be demoted in ToDemote and
+    // additional roots that require investigating in Roots.
+    for (auto *Root : TreeRoot) {
+      DenseSet Visited;
+      unsigned MaxDepthLevel = 0;
+      bool NeedToDemote = IsProfitableToDemote;
+
+      if (!collectValuesToDemote(Root, IsProfitableToDemoteRoot, MaxBitWidth,
+                                 ToDemote, DemotedConsts, Visited,
+                                 MaxDepthLevel, NeedToDemote) ||
+          (MaxDepthLevel <= Limit &&
+           !(((Opcode == Instruction::SExt || Opcode == Instruction::ZExt) &&
+              (!IsTopRoot || !(IsStoreOrInsertElt || UserIgnoreList) ||
+               DL->getTypeSizeInBits(Root->getType()) /
+                       DL->getTypeSizeInBits(
+                           cast(Root)->getOperand(0)->getType()) >
+                   2)) ||
+             (Opcode == Instruction::Trunc &&
+              (!IsTopRoot || !(IsStoreOrInsertElt || UserIgnoreList) ||
+               DL->getTypeSizeInBits(
+                   cast(Root)->getOperand(0)->getType()) /
+                       DL->getTypeSizeInBits(Root->getType()) >
+                   2)))))
+        return 0u;
+    }
+    // Round MaxBitWidth up to the next power-of-two.
+    MaxBitWidth = bit_ceil(MaxBitWidth);
+
+    return MaxBitWidth;
+  };
 
   // If we can truncate the root, we must collect additional values that might
   // be demoted as a result. That is, those seeded by truncations we will
   // modify.
-  while (!Roots.empty()) {
-    DenseSet Visited;
-    collectValuesToDemote(Roots.pop_back_val(), ToDemote, DemotedConsts, Roots,
-                          Visited);
-  }
-
-  // Check that all users are marked for demotion.
-  DenseSet Demoted(ToDemote.begin(), ToDemote.end());
-  DenseSet Visited;
-  for (Value *V: ToDemote) {
-    const TreeEntry *TE = getTreeEntry(V);
-    assert(TE && "Expected vectorized scalar.");
-    if (!Visited.insert(TE).second)
-      continue;
-    if (!all_of(TE->UserTreeIndices, [&](const EdgeInfo &EI) {
-          return all_of(EI.UserTE->Scalars,
-                        [&](Value *V) { return Demoted.contains(V); });
-        }))
-      return;
-  }
-  // Finally, map the values we can demote to the maximum bit with we computed.
-  for (auto *Scalar : ToDemote) {
-    auto *TE = getTreeEntry(Scalar);
-    assert(TE && "Expected vectorized scalar.");
-    if (MinBWs.contains(TE))
+  // Add reduction ops sizes, if any.
+  if (UserIgnoreList &&
+      isa(VectorizableTree.front()->Scalars.front()->getType())) {
+    for (Value *V : *UserIgnoreList) {
+      auto NumSignBits = ComputeNumSignBits(V, *DL, 0, AC, nullptr, DT);
+      auto NumTypeBits = DL->getTypeSizeInBits(V->getType());
+      unsigned BitWidth1 = NumTypeBits - NumSignBits;
+      if (!isKnownNonNegative(V, SimplifyQuery(*DL)))
+        ++BitWidth1;
+      auto Mask = DB->getDemandedBits(cast(V));
+      unsigned BitWidth2 = Mask.getBitWidth() - Mask.countl_zero();
+      ReductionBitWidth =
+          std::max(std::min(BitWidth1, BitWidth2), ReductionBitWidth);
+    }
+    if (ReductionBitWidth < 8 && ReductionBitWidth > 1)
+      ReductionBitWidth = 8;
+
+    ReductionBitWidth = bit_ceil(ReductionBitWidth);
+  }
+  bool IsTopRoot = NodeIdx == 0;
+  while (NodeIdx < VectorizableTree.size() &&
+         VectorizableTree[NodeIdx]->State == TreeEntry::Vectorize &&
+         VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc)
+    ++NodeIdx;
+  while (NodeIdx < VectorizableTree.size()) {
+    ArrayRef TreeRoot = VectorizableTree[NodeIdx]->Scalars;
+    unsigned Limit = 2;
+    unsigned Opcode = VectorizableTree[NodeIdx]->getOpcode();
+    if (IsTopRoot &&
+        ReductionBitWidth ==
+            DL->getTypeSizeInBits(
+                VectorizableTree.front()->Scalars.front()->getType()))
+      Limit = 3;
+    unsigned MaxBitWidth = ComputeMaxBitWidth(
+        TreeRoot, VectorizableTree[NodeIdx]->getVectorFactor(), IsTopRoot,
+        IsProfitableToDemoteRoot, Opcode, Limit);
+    IsTopRoot = false;
+    IsProfitableToDemoteRoot = true;
+
+    if (TruncNodes.empty()) {
+      NodeIdx = VectorizableTree.size();
+    } else {
+      NodeIdx = *TruncNodes.begin() + 1;
+      TruncNodes.erase(TruncNodes.begin());
+    }
+
+    // If the maximum bit width we compute is less than the with of the roots'
+    // type, we can proceed with the narrowing. Otherwise, do nothing.
+    if (MaxBitWidth == 0 ||
+        MaxBitWidth >=
+            cast(TreeRoot.front()->getType())->getBitWidth()) {
+      if (UserIgnoreList)
+        AnalyzedMinBWVals.insert(TreeRoot.begin(), TreeRoot.end());
       continue;
-    bool IsSigned = any_of(TE->Scalars, [&](Value *R) {
-      KnownBits Known = computeKnownBits(R, *DL);
-      return !Known.isNonNegative();
-    });
-    MinBWs.try_emplace(TE, MaxBitWidth, IsSigned);
-    const auto *I = cast(Scalar);
-    auto DCIt = DemotedConsts.find(I);
-    if (DCIt != DemotedConsts.end()) {
-      for (unsigned Idx : DCIt->getSecond()) {
-        // Check that all instructions operands are demoted.
-        if (all_of(TE->Scalars, [&](Value *V) {
-              auto SIt = DemotedConsts.find(cast(V));
-              return SIt != DemotedConsts.end() &&
-                     is_contained(SIt->getSecond(), Idx);
-            })) {
+    }
+
+    // Finally, map the values we can demote to the maximum bit with we
+    // computed.
+    for (Value *Scalar : ToDemote) {
+      TreeEntry *TE = getTreeEntry(Scalar);
+      assert(TE && "Expected vectorized scalar.");
+      if (MinBWs.contains(TE))
+        continue;
+      bool IsSigned = any_of(TE->Scalars, [&](Value *R) {
+        return !isKnownNonNegative(R, SimplifyQuery(*DL));
+      });
+      MinBWs.try_emplace(TE, MaxBitWidth, IsSigned);
+      const auto *I = cast(Scalar);
+      auto DCIt = DemotedConsts.find(I);
+      if (DCIt != DemotedConsts.end()) {
+        for (unsigned Idx : DCIt->getSecond()) {
+          // Check that all instructions operands are demoted.
           const TreeEntry *CTE = getOperandEntry(TE, Idx);
-          MinBWs.try_emplace(CTE, MaxBitWidth, IsSigned);
+          if (all_of(TE->Scalars,
+                     [&](Value *V) {
+                       auto SIt = DemotedConsts.find(cast(V));
+                       return SIt != DemotedConsts.end() &&
+                              is_contained(SIt->getSecond(), Idx);
+                     }) ||
+              all_of(CTE->Scalars, Constant::classof))
+            MinBWs.try_emplace(CTE, MaxBitWidth, IsSigned);
         }
       }
     }
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll
index cef791633655..5e3fd156666f 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll
@@ -17,12 +17,13 @@ define void @test1(<4 x i16> %a, <4 x i16> %b, ptr %p) {
 ; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i64 [[S0]]
 ; CHECK-NEXT:    [[LOAD0:%.*]] = load i64, ptr [[GEP0]], align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i32> [[SUB0]], <4 x i32> poison, <2 x i32> 
-; CHECK-NEXT:    [[TMP1:%.*]] = sext <2 x i32> [[TMP0]] to <2 x i64>
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i32> [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[TMP1]] to i64
 ; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[LOAD1:%.*]] = load i64, ptr [[GEP1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
-; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = sext i32 [[TMP3]] to i64
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP4]]
 ; CHECK-NEXT:    [[LOAD2:%.*]] = load i64, ptr [[GEP2]], align 4
 ; CHECK-NEXT:    [[E3:%.*]] = extractelement <4 x i32> [[SUB0]], i32 3
 ; CHECK-NEXT:    [[S3:%.*]] = sext i32 [[E3]] to i64
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll
index 47485e514ec2..1cce52060c47 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ;test_i16_extend NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer,dce,instcombine -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer,dce,instcombine -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s
 ; RUN: cat %t | FileCheck -check-prefix=YAML %s
-; RUN: opt -S -mtriple=aarch64--linux-gnu -passes='slp-vectorizer,dce,instcombine' -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: opt -S -mtriple=aarch64--linux-gnu -passes='slp-vectorizer,dce,instcombine' -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s
 ; RUN: cat %t | FileCheck -check-prefix=YAML %s
 
 
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll
index d67fdc1cd6aa..a7a7f642ced5 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll
@@ -28,21 +28,11 @@ entry:
 define i64 @red_zext_ld_4xi64(ptr %ptr) {
 ; CHECK-LABEL: @red_zext_ld_4xi64(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LD0:%.*]] = load i8, ptr [[PTR:%.*]], align 1
-; CHECK-NEXT:    [[ZEXT:%.*]] = zext i8 [[LD0]] to i64
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 1
-; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[GEP]], align 1
-; CHECK-NEXT:    [[ZEXT_1:%.*]] = zext i8 [[LD1]] to i64
-; CHECK-NEXT:    [[ADD_1:%.*]] = add nuw nsw i64 [[ZEXT]], [[ZEXT_1]]
-; CHECK-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 2
-; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[GEP_1]], align 1
-; CHECK-NEXT:    [[ZEXT_2:%.*]] = zext i8 [[LD2]] to i64
-; CHECK-NEXT:    [[ADD_2:%.*]] = add nuw nsw i64 [[ADD_1]], [[ZEXT_2]]
-; CHECK-NEXT:    [[GEP_2:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 3
-; CHECK-NEXT:    [[LD3:%.*]] = load i8, ptr [[GEP_2]], align 1
-; CHECK-NEXT:    [[ZEXT_3:%.*]] = zext i8 [[LD3]] to i64
-; CHECK-NEXT:    [[ADD_3:%.*]] = add nuw nsw i64 [[ADD_2]], [[ZEXT_3]]
-; CHECK-NEXT:    ret i64 [[ADD_3]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr [[PTR:%.*]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i16>
+; CHECK-NEXT:    [[TMP2:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[TMP2]] to i64
+; CHECK-NEXT:    ret i64 [[TMP3]]
 ;
 entry:
   %ld0 = load i8, ptr %ptr
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll
index 000e7a56df37..500f10659f04 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll
@@ -802,9 +802,10 @@ define i64 @red_zext_ld_4xi64(ptr %ptr) {
 ; CHECK-LABEL: @red_zext_ld_4xi64(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr [[PTR:%.*]], align 1
-; CHECK-NEXT:    [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i64>
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP1]])
-; CHECK-NEXT:    ret i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i16>
+; CHECK-NEXT:    [[TMP2:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[TMP2]] to i64
+; CHECK-NEXT:    ret i64 [[TMP3]]
 ;
 entry:
   %ld0 = load i8, ptr %ptr
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll
index 4565d4928ba4..05511f843a68 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll
@@ -15,11 +15,12 @@ define { i64, i64 } @patatino(double %arg) {
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x double>, ptr getelementptr inbounds ([6 x double], ptr @global, i64 0, i64 4), align 16
 ; CHECK-NEXT:    [[TMP7:%.*]] = fadd <2 x double> [[TMP6]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = fptosi <2 x double> [[TMP7]] to <2 x i32>
-; CHECK-NEXT:    [[TMP9:%.*]] = sext <2 x i32> [[TMP8]] to <2 x i64>
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i64> [[TMP9]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i32> [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = sext i32 [[TMP9]] to i64
 ; CHECK-NEXT:    [[T16:%.*]] = insertvalue { i64, i64 } undef, i64 [[TMP10]], 0
-; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i64> [[TMP9]], i32 1
-; CHECK-NEXT:    [[T17:%.*]] = insertvalue { i64, i64 } [[T16]], i64 [[TMP11]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i32> [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = sext i32 [[TMP11]] to i64
+; CHECK-NEXT:    [[T17:%.*]] = insertvalue { i64, i64 } [[T16]], i64 [[TMP12]], 1
 ; CHECK-NEXT:    ret { i64, i64 } [[T17]]
 ;
 bb:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll
index a0af8e36b36c..5ee801607653 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-3 < %s | FileCheck %s
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-6 < %s | FileCheck %s
 
 define void @t(i64 %v) {
 ; CHECK-LABEL: define void @t(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll
index 6e512fcbb739..6051638562b5 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll
@@ -6,18 +6,17 @@ define void @test(i8 %0) {
 ; CHECK-SAME: i8 [[TMP0:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> , i8 [[TMP0]], i32 1
-; CHECK-NEXT:    [[TMP2:%.*]] = sext <2 x i8> [[TMP1]] to <2 x i16>
-; CHECK-NEXT:    [[TMP3:%.*]] = sext <2 x i16> [[TMP2]] to <2 x i32>
-; CHECK-NEXT:    [[TMP4:%.*]] = mul <2 x i16> [[TMP2]], zeroinitializer
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i16> [[TMP4]], i32 0
-; CHECK-NEXT:    [[TMP6:%.*]] = zext i16 [[TMP5]] to i32
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i16> [[TMP4]], i32 1
-; CHECK-NEXT:    [[TMP8:%.*]] = zext i16 [[TMP7]] to i32
-; CHECK-NEXT:    [[ADD:%.*]] = or i32 [[TMP6]], [[TMP8]]
+; CHECK-NEXT:    [[TMP2:%.*]] = sext <2 x i8> [[TMP1]] to <2 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = mul <2 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i8> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = zext i8 [[TMP4]] to i32
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i8> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = zext i8 [[TMP6]] to i32
+; CHECK-NEXT:    [[ADD:%.*]] = or i32 [[TMP5]], [[TMP7]]
 ; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[ADD]], 1
 ; CHECK-NEXT:    [[CONV9:%.*]] = trunc i32 [[SHR]] to i8
 ; CHECK-NEXT:    store i8 [[CONV9]], ptr null, align 1
-; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <8 x i32> 
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> 
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll
index 2c834616becc..4acd63078b82 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll
@@ -6,15 +6,20 @@ define void @test(i64 %d.promoted.i) {
 ; CHECK-SAME: i64 [[D_PROMOTED_I:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[AND_1_I:%.*]] = and i64 0, [[D_PROMOTED_I]]
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x i64> , i64 [[AND_1_I]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[TMP0]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = mul <8 x i1> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    [[AND_1_I_1:%.*]] = and i64 0, 0
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <16 x i64> , i64 [[AND_1_I_1]], i32 1
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <16 x i64> [[TMP0]], i64 [[AND_1_I]], i32 9
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc <16 x i64> [[TMP1]] to <16 x i1>
-; CHECK-NEXT:    [[TMP3:%.*]] = mul <16 x i1> [[TMP2]], zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP3]])
-; CHECK-NEXT:    [[TMP5:%.*]] = zext i1 [[TMP4]] to i32
-; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[TMP5]], 0
-; CHECK-NEXT:    store i32 [[TMP6]], ptr null, align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <8 x i64> , i64 [[AND_1_I_1]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc <8 x i64> [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = mul <8 x i1> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = zext i1 [[TMP6]] to i32
+; CHECK-NEXT:    [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP2]])
+; CHECK-NEXT:    [[TMP9:%.*]] = zext i1 [[TMP8]] to i32
+; CHECK-NEXT:    [[OP_RDX:%.*]] = or i32 [[TMP7]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = and i32 [[OP_RDX]], 0
+; CHECK-NEXT:    store i32 [[TMP10]], ptr null, align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
index 651631de2c35..a316415dcc6b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
@@ -17,12 +17,15 @@ target triple = "x86_64-unknown-linux-gnu"
 define i8 @PR31243_zext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) {
 ; SSE-LABEL: @PR31243_zext(
 ; SSE-NEXT:  entry:
-; SSE-NEXT:    [[TMP0:%.*]] = or i8 [[V0:%.*]], 1
-; SSE-NEXT:    [[TMP1:%.*]] = or i8 [[V1:%.*]], 1
-; SSE-NEXT:    [[TMP2:%.*]] = zext i8 [[TMP0]] to i64
-; SSE-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP2]]
-; SSE-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP1]] to i64
-; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP3]]
+; SSE-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0
+; SSE-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1
+; SSE-NEXT:    [[TMP2:%.*]] = or <2 x i8> [[TMP1]], 
+; SSE-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0
+; SSE-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP3]] to i64
+; SSE-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]]
+; SSE-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1
+; SSE-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i64
+; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]]
 ; SSE-NEXT:    [[T6:%.*]] = load i8, ptr [[T4]], align 1
 ; SSE-NEXT:    [[T7:%.*]] = load i8, ptr [[T5]], align 1
 ; SSE-NEXT:    [[T8:%.*]] = add i8 [[T6]], [[T7]]
@@ -73,12 +76,15 @@ entry:
 define i8 @PR31243_sext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) {
 ; SSE-LABEL: @PR31243_sext(
 ; SSE-NEXT:  entry:
-; SSE-NEXT:    [[TMP0:%.*]] = or i8 [[V0:%.*]], 1
-; SSE-NEXT:    [[TMP1:%.*]] = or i8 [[V1:%.*]], 1
-; SSE-NEXT:    [[TMP2:%.*]] = sext i8 [[TMP0]] to i64
-; SSE-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP2]]
-; SSE-NEXT:    [[TMP3:%.*]] = sext i8 [[TMP1]] to i64
-; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP3]]
+; SSE-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0
+; SSE-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1
+; SSE-NEXT:    [[TMP2:%.*]] = or <2 x i8> [[TMP1]], 
+; SSE-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0
+; SSE-NEXT:    [[TMP4:%.*]] = sext i8 [[TMP3]] to i64
+; SSE-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]]
+; SSE-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1
+; SSE-NEXT:    [[TMP6:%.*]] = sext i8 [[TMP5]] to i64
+; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]]
 ; SSE-NEXT:    [[T6:%.*]] = load i8, ptr [[T4]], align 1
 ; SSE-NEXT:    [[T7:%.*]] = load i8, ptr [[T5]], align 1
 ; SSE-NEXT:    [[T8:%.*]] = add i8 [[T6]], [[T7]]
@@ -89,13 +95,12 @@ define i8 @PR31243_sext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) {
 ; AVX-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0
 ; AVX-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1
 ; AVX-NEXT:    [[TMP2:%.*]] = or <2 x i8> [[TMP1]], 
-; AVX-NEXT:    [[TMP3:%.*]] = sext <2 x i8> [[TMP2]] to <2 x i16>
-; AVX-NEXT:    [[TMP4:%.*]] = extractelement <2 x i16> [[TMP3]], i64 0
-; AVX-NEXT:    [[TMP5:%.*]] = sext i16 [[TMP4]] to i64
-; AVX-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP5]]
-; AVX-NEXT:    [[TMP6:%.*]] = extractelement <2 x i16> [[TMP3]], i64 1
-; AVX-NEXT:    [[TMP7:%.*]] = sext i16 [[TMP6]] to i64
-; AVX-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP7]]
+; AVX-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0
+; AVX-NEXT:    [[TMP4:%.*]] = sext i8 [[TMP3]] to i64
+; AVX-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]]
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1
+; AVX-NEXT:    [[TMP6:%.*]] = sext i8 [[TMP5]] to i64
+; AVX-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]]
 ; AVX-NEXT:    [[T6:%.*]] = load i8, ptr [[T4]], align 1
 ; AVX-NEXT:    [[T7:%.*]] = load i8, ptr [[T5]], align 1
 ; AVX-NEXT:    [[T8:%.*]] = add i8 [[T6]], [[T7]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll
index 88f75c37846e..3cc32c1fc7b2 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll
@@ -15,8 +15,8 @@ define i32 @phi3UndefInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %arg3) {
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32>
-; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]])
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
 ; CHECK-NEXT:    ret i32 [[TMP6]]
 ;
 entry:
@@ -52,8 +52,8 @@ define i32 @phi2UndefInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %arg3) {
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32>
-; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]])
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
 ; CHECK-NEXT:    ret i32 [[TMP6]]
 ;
 entry:
@@ -89,8 +89,8 @@ define i32 @phi1UndefInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %arg3) {
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32>
-; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]])
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
 ; CHECK-NEXT:    ret i32 [[TMP6]]
 ;
 entry:
@@ -127,8 +127,8 @@ define i32 @phi1Undef1PoisonInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %ar
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32>
-; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]])
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
 ; CHECK-NEXT:    ret i32 [[TMP6]]
 ;
 entry:
@@ -165,8 +165,8 @@ define i32 @phi1Undef2PoisonInputs(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %a
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32>
-; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]])
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
 ; CHECK-NEXT:    ret i32 [[TMP6]]
 ;
 entry:
@@ -202,8 +202,8 @@ define i32 @phi1Undef1PoisonGapInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32>
-; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]])
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
 ; CHECK-NEXT:    ret i32 [[TMP6]]
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/resched.ll b/llvm/test/Transforms/SLPVectorizer/X86/resched.ll
index 78c6d9516a3d..b7237cbb02bb 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/resched.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/resched.ll
@@ -11,26 +11,26 @@ define fastcc void @_ZN12_GLOBAL__N_127PolynomialMultiplyRecognize9recognizeEv()
 ; CHECK:       if.then22.i:
 ; CHECK-NEXT:    [[SUB_I:%.*]] = add nsw i32 undef, -1
 ; CHECK-NEXT:    [[CONV31_I:%.*]] = and i32 undef, [[SUB_I]]
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[CONV31_I]], i32 0
-; CHECK-NEXT:    [[SHUFFLE1:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = lshr <4 x i32> [[SHUFFLE1]], 
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[CONV31_I]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <4 x i32> [[TMP1]], 
 ; CHECK-NEXT:    [[SHR_4_I_I:%.*]] = lshr i32 [[CONV31_I]], 5
 ; CHECK-NEXT:    [[SHR_5_I_I:%.*]] = lshr i32 [[CONV31_I]], 6
 ; CHECK-NEXT:    [[SHR_6_I_I:%.*]] = lshr i32 [[CONV31_I]], 7
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <8 x i32> poison, i32 [[CONV31_I]], i32 0
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = lshr <8 x i32> [[SHUFFLE]], 
-; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <16 x i32> poison, i32 [[SUB_I]], i32 0
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <16 x i32> 
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> [[TMP6]], <16 x i32> 
-; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <16 x i32> [[TMP7]], i32 [[SHR_4_I_I]], i32 5
-; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <16 x i32> [[TMP8]], i32 [[SHR_5_I_I]], i32 6
-; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <16 x i32> [[TMP9]], i32 [[SHR_6_I_I]], i32 7
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <16 x i32> 
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <16 x i32> [[TMP10]], <16 x i32> [[TMP11]], <16 x i32> 
-; CHECK-NEXT:    [[TMP13:%.*]] = trunc <16 x i32> [[TMP12]] to <16 x i8>
-; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i8> [[TMP13]], 
-; CHECK-NEXT:    store <16 x i8> [[TMP14]], ptr undef, align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr <8 x i32> [[TMP4]], 
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <16 x i32> poison, i32 [[SUB_I]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <16 x i32> 
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i32> [[TMP6]], <16 x i32> [[TMP7]], <16 x i32> 
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <16 x i32> [[TMP8]], i32 [[SHR_4_I_I]], i32 5
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <16 x i32> [[TMP9]], i32 [[SHR_5_I_I]], i32 6
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <16 x i32> [[TMP10]], i32 [[SHR_6_I_I]], i32 7
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> poison, <16 x i32> 
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <16 x i32> [[TMP11]], <16 x i32> [[TMP12]], <16 x i32> 
+; CHECK-NEXT:    [[TMP14:%.*]] = trunc <16 x i32> [[TMP13]] to <16 x i8>
+; CHECK-NEXT:    [[TMP15:%.*]] = and <16 x i8> [[TMP14]], 
+; CHECK-NEXT:    store <16 x i8> [[TMP15]], ptr undef, align 1
 ; CHECK-NEXT:    unreachable
 ; CHECK:       if.end50.i:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll
index 5d22b5a4873b..1d1fcec2a7ae 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll
@@ -7,12 +7,10 @@ define i1 @test(i1 %cmp5.not.31) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i1> , i1 [[CMP5_NOT_31]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> zeroinitializer, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc <4 x i32> [[TMP1]] to <4 x i1>
-; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32>
-; CHECK-NEXT:    [[TMP4:%.*]] = mul <4 x i32> [[TMP3]], 
-; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])
-; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[TMP5]], 0
-; CHECK-NEXT:    [[CMP_NOT_I_I:%.*]] = icmp eq i32 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = mul <4 x i32> [[TMP1]], 
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP3]], 0
+; CHECK-NEXT:    [[CMP_NOT_I_I:%.*]] = icmp eq i32 [[TMP4]], 0
 ; CHECK-NEXT:    ret i1 [[CMP_NOT_I_I]]
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll
index c1dd90d0e9a7..2f6868d8dfd6 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll
@@ -8,17 +8,18 @@
 ; YAML-NEXT:  Function:        stores
 ; YAML-NEXT:  Args:
 ; YAML-NEXT:    - String:          'Stores SLP vectorized with cost '
-; YAML-NEXT:    - Cost:            '-3'
+; YAML-NEXT:    - Cost:            '-7'
 ; YAML-NEXT:    - String:          ' and with tree size '
 ; YAML-NEXT:    - TreeSize:        '6'
 define void @stores(ptr noalias %in, ptr noalias %inn, ptr noalias %out) {
 ; CHECK-LABEL: @stores(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i8>, ptr [[IN:%.*]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i8>, ptr [[INN:%.*]], align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i64>
-; CHECK-NEXT:    [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i64>
-; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i64> [[TMP3]], [[TMP4]]
-; CHECK-NEXT:    store <4 x i64> [[TMP5]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i16> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = zext <4 x i16> [[TMP5]] to <4 x i64>
+; CHECK-NEXT:    store <4 x i64> [[TMP6]], ptr [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %load.1 = load i8, ptr %in, align 1
@@ -63,17 +64,18 @@ define void @stores(ptr noalias %in, ptr noalias %inn, ptr noalias %out) {
 ; YAML-NEXT:  Function:        insertelems
 ; YAML-NEXT:  Args:
 ; YAML-NEXT:    - String:          'SLP vectorized with cost '
-; YAML-NEXT:    - Cost:            '-5'
+; YAML-NEXT:    - Cost:            '-9'
 ; YAML-NEXT:    - String:          ' and with tree size '
 ; YAML-NEXT:    - TreeSize:        '6'
 define <4 x i64> @insertelems(ptr noalias %in, ptr noalias %inn) {
 ; CHECK-LABEL: @insertelems(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i8>, ptr [[IN:%.*]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i8>, ptr [[INN:%.*]], align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i64>
-; CHECK-NEXT:    [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i64>
-; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i64> [[TMP3]], [[TMP4]]
-; CHECK-NEXT:    ret <4 x i64> [[TMP5]]
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i16> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = zext <4 x i16> [[TMP5]] to <4 x i64>
+; CHECK-NEXT:    ret <4 x i64> [[TMP6]]
 ;
   %load.1 = load i8, ptr %in, align 1
   %gep.1 = getelementptr inbounds i8, ptr %in, i64 1
diff --git a/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll b/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll
index 061fbdb45a13..ff6f0bdd3db8 100644
--- a/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll
+++ b/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll
@@ -10,8 +10,8 @@ define i32 @alt_cmp(i16 %call46) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult <4 x i16> [[TMP0]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt <4 x i16> [[TMP0]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i1> [[TMP2]], <4 x i1> [[TMP3]], <4 x i32> 
-; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i1> [[TMP4]] to <4 x i16>
-; CHECK-NEXT:    [[TMP6:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP5]])
+; CHECK-NEXT:    [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i1 [[TMP5]] to i16
 ; CHECK-NEXT:    [[OP_RDX:%.*]] = or i16 [[TMP6]], 0
 ; CHECK-NEXT:    [[EXT:%.*]] = zext i16 [[OP_RDX]] to i32
 ; CHECK-NEXT:    ret i32 [[EXT]]
-- 
GitLab


From 9405d5af65853ac548cce2656497195010db1d86 Mon Sep 17 00:00:00 2001
From: Andrew Browne 
Date: Fri, 8 Mar 2024 11:04:04 -0800
Subject: [PATCH 646/929] [DFSan] Add missing documentation for
 -dfsan-reaches-function-callbacks. (#84218)

---
 clang/docs/DataFlowSanitizer.rst | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/clang/docs/DataFlowSanitizer.rst b/clang/docs/DataFlowSanitizer.rst
index a18b8ed1948f..5ff50b85dcdc 100644
--- a/clang/docs/DataFlowSanitizer.rst
+++ b/clang/docs/DataFlowSanitizer.rst
@@ -233,6 +233,28 @@ labels of just ``v1`` and ``v2``.
   or, and can be accessed using
   ``dfsan_label dfsan_get_labels_in_signal_conditional();``.
 
+* ``-dfsan-reaches-function-callbacks`` -- An experimental feature that inserts
+  callbacks for data entering a function.
+
+  In addition to this compilation flag, a callback handler must be registered
+  using ``dfsan_set_reaches_function_callback(my_callback);``, where my_callback is
+  a function with a signature matching
+  ``void my_callback(dfsan_label label, dfsan_origin origin, const char *file, unsigned int line, const char *function);``
+  This signature is the same when origin tracking is disabled - in this case
+  the dfsan_origin passed in it will always be 0.
+
+  The callback will be called when a tained value reach stack/registers
+  in the context of a function. Tainted values can reach a function:
+  * via the arguments of the function
+  * via the return value of a call that occurs in the function
+  * via the loaded value of a load that occurs in the function
+
+  The callback will be skipped for conditional expressions inside signal
+  handlers, as this is prone to deadlock. Tainted values reaching functions
+  inside signal handlers will instead be aggregated via bitwise or, and can
+  be accessed using
+  ``dfsan_label dfsan_get_labels_in_signal_reaches_function()``.
+
 * ``-dfsan-track-origins`` -- Controls how to track origins. When its value is
   0, the runtime does not track origins. When its value is 1, the runtime tracks
   origins at memory store operations. When its value is 2, the runtime tracks
-- 
GitLab


From 51207756b0692f325cf75560185cf0336239b3e0 Mon Sep 17 00:00:00 2001
From: Tom Stellard 
Date: Fri, 8 Mar 2024 11:13:34 -0800
Subject: [PATCH 647/929] workflows: Fixes for building the release binaries
 (#83694)

Since aa02002491333c42060373bc84f1ff5d2c76b4ce we weren't installing the
correct dependencies, and since 2836d8edbfbcd461b25101ed58f93c862d65903a
we must pass a custom token to github-upload-release.py for verifying
permissions.
---
 .github/workflows/release-binaries.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
index e487b0f1d4b5..1dba91746dae 100644
--- a/.github/workflows/release-binaries.yml
+++ b/.github/workflows/release-binaries.yml
@@ -48,11 +48,16 @@ jobs:
     - name: Checkout LLVM
       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
+    - name: Install Dependencies
+      run: |
+        pip install -r ./llvm/utils/git/requirements.txt
+
     - name: Check Permissions
       env:
         GITHUB_TOKEN: ${{ github.token }}
+        USER_TOKEN: ${{ secrets.RELEASE_TASKS_USER_TOKEN }}
       run: |
-        ./llvm/utils/release/./github-upload-release.py --token "$GITHUB_TOKEN" --user ${{ github.actor }} check-permissions
+        ./llvm/utils/release/./github-upload-release.py --token "$GITHUB_TOKEN" --user ${{ github.actor }} --user-token "$USER_TOKEN" check-permissions
 
     - name: Collect Variables
       id: vars
-- 
GitLab


From e460da14ec0d85af657a018605decabdd5221ded Mon Sep 17 00:00:00 2001
From: Jay Foad 
Date: Fri, 8 Mar 2024 19:21:27 +0000
Subject: [PATCH 648/929] [AMDGPU] Use get_BUF_ps to default real_name of BUF
 instructions. NFC. (#84524)

---
 llvm/lib/Target/AMDGPU/BUFInstructions.td | 31 +++++++++++++----------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index dedf381afca8..d1cbe2840a02 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -2484,15 +2484,15 @@ multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Impl2 op, string real_name> :
   MUBUF_Real_AllAddr_gfx11_Impl2,
   MUBUF_Real_AllAddr_gfx12_Impl2;
 
-multiclass MUBUF_Real_AllAddr_gfx11_Impl op, string real_name,
-                                                 bit hasTFE = 1> {
+multiclass MUBUF_Real_AllAddr_gfx11_Impl op, bit hasTFE,
+                                 string real_name = get_BUF_ps.Mnemonic> {
   defm NAME : MUBUF_Real_AllAddr_gfx11_Impl2;
   if hasTFE then
     defm _TFE : MUBUF_Real_AllAddr_gfx11_Impl2;
 }
 
-multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Impl op, string real_name,
-                                                 bit hasTFE = 1> {
+multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Impl op, bit hasTFE,
+                                               string real_name> {
   defm NAME : MUBUF_Real_AllAddr_gfx11_gfx12_Impl2;
   if hasTFE then
     defm _TFE : MUBUF_Real_AllAddr_gfx11_gfx12_Impl2;
@@ -2500,10 +2500,11 @@ multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Impl op, string real_name,
 
 // Non-renamed, non-atomic gfx11/gfx12 mubuf instructions.
 multiclass MUBUF_Real_AllAddr_gfx11 op, bit hasTFE = 1> :
-  MUBUF_Real_AllAddr_gfx11_Impl.Mnemonic, hasTFE>;
+  MUBUF_Real_AllAddr_gfx11_Impl;
 
-multiclass MUBUF_Real_AllAddr_gfx11_gfx12 op, string real_name = !tolower(NAME)> :
-  MUBUF_Real_AllAddr_gfx11_gfx12_Impl {
+multiclass MUBUF_Real_AllAddr_gfx11_gfx12 op,
+                                 string real_name = get_BUF_ps.Mnemonic> :
+  MUBUF_Real_AllAddr_gfx11_gfx12_Impl {
   defvar ps = get_BUF_ps;
   if !ne(ps.Mnemonic, real_name) then
     def : Mnem_gfx11_gfx12;
@@ -2519,7 +2520,7 @@ multiclass MUBUF_Real_Atomic_gfx11_impl op, bit is_return,
 }
 
 multiclass MUBUF_Real_Atomic_gfx12_impl op, bit is_return,
-                                                string real_name> {
+                                 string real_name = get_BUF_ps.Mnemonic> {
   defvar Rtn = !if(is_return, "_RTN", "");
   defm _VBUFFER_BOTHEN#Rtn : VBUFFER_MUBUF_Real_gfx12;
   defm _VBUFFER_IDXEN#Rtn  : VBUFFER_MUBUF_Real_gfx12;
@@ -2533,16 +2534,19 @@ multiclass MUBUF_Real_Atomic_gfx11_gfx12_impl op, bit is_return,
   MUBUF_Real_Atomic_gfx12_impl;
 
 multiclass MUBUF_Real_Atomic_gfx12 op> :
-  MUBUF_Real_Atomic_gfx12_impl.Mnemonic>,
-  MUBUF_Real_Atomic_gfx12_impl.Mnemonic>;
+  MUBUF_Real_Atomic_gfx12_impl,
+  MUBUF_Real_Atomic_gfx12_impl;
 
 multiclass MUBUF_Real_Atomic_gfx11 op, string real_name> :
   MUBUF_Real_Atomic_gfx11_impl,
   MUBUF_Real_Atomic_gfx11_impl {
-  def : Mnem_gfx11_gfx12.Mnemonic, real_name>;
+  defvar ps = get_BUF_ps;
+  def : Mnem_gfx11_gfx12;
 }
 
-multiclass MUBUF_Real_Atomic_gfx11_gfx12 op, string gfx12_name = !tolower(NAME), string gfx11_name = gfx12_name> :
+multiclass MUBUF_Real_Atomic_gfx11_gfx12 op,
+                                  string gfx12_name = get_BUF_ps.Mnemonic,
+                                  string gfx11_name = gfx12_name> :
   MUBUF_Real_Atomic_gfx11_impl,
   MUBUF_Real_Atomic_gfx11_impl,
   MUBUF_Real_Atomic_gfx12_impl,
@@ -2887,7 +2891,8 @@ multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Impl op, string real_name> {
   defm _VBUFFER_OFFSET : VBUFFER_MTBUF_Real_gfx12;
 }
 
-multiclass MTBUF_Real_AllAddr_gfx11_gfx12 op, string real_name = !tolower(NAME)>
+multiclass MTBUF_Real_AllAddr_gfx11_gfx12 op,
+                                   string real_name = get_BUF_ps.Mnemonic>
   : MTBUF_Real_AllAddr_gfx11_gfx12_Impl {
   defvar ps = get_BUF_ps;
   if !ne(ps.Mnemonic, real_name) then
-- 
GitLab


From 074fe3bac634ae9054e2d35be2e1bcf8a19bc256 Mon Sep 17 00:00:00 2001
From: Jay Foad 
Date: Fri, 8 Mar 2024 19:22:08 +0000
Subject: [PATCH 649/929] [AMDGPU] Simplify and refactor VBUFFER_Real class
 definitions. NFC. (#84521)

Abstracting out a new base class VBUFFER_Real_gfx12 just highlights that
the only difference between the MUBUF and MTBUF forms is in the handling
of the "format" field.
---
 llvm/lib/Target/AMDGPU/BUFInstructions.td | 31 ++++++++++-------------
 1 file changed, 14 insertions(+), 17 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index d1cbe2840a02..a1bbe170ee29 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -2365,7 +2365,7 @@ multiclass MUBUF_Real_gfx7 op> {
 // Base ENC_VBUFFER for GFX12.
 //===----------------------------------------------------------------------===//
 
-class VBUFFER_Real  :
+class VBUFFER_Real  op, BUF_Pseudo ps, string real_name> :
   InstSI , Enc96 {
 
   let isPseudo = 0;
@@ -2389,6 +2389,8 @@ class VBUFFER_Real  :
   let IsAtomicNoRet      = ps.IsAtomicNoRet;
   let VALU               = ps.VALU;
   let LGKM_CNT           = ps.LGKM_CNT;
+  let MUBUF              = ps.MUBUF;
+  let MTBUF              = ps.MTBUF;
 
   bits<24> offset;
   bits<8>  vaddr;
@@ -2405,6 +2407,7 @@ class VBUFFER_Real  :
   let Inst{47-41} = !if(ps.has_srsrc, srsrc, ?);
   let Inst{49-48} = 0b00;
   let Inst{6-0}   = !if(ps.has_soffset, soffset, ?);
+  let Inst{21-14} = op;
   let Inst{22}    = ps.tfe;
   let Inst{62}    = ps.offen;
   let Inst{63}    = ps.idxen;
@@ -2416,33 +2419,27 @@ class VBUFFER_Real  :
   let Inst{31-26} = 0b110001;
 }
 
+class VBUFFER_Real_gfx12 op, BUF_Pseudo ps, string real_name> :
+    VBUFFER_Real,
+    SIMCInstr {
+  let AssemblerPredicate = isGFX12Only;
+  let DecoderNamespace = "GFX12";
+}
+
 multiclass VBUFFER_MUBUF_Real_gfx12 op, string real_name> {
   defvar ps = !cast(NAME);
-  def _gfx12 : VBUFFER_Real,
-               SIMCInstr {
-    let MUBUF = 1;
-
+  def _gfx12 : VBUFFER_Real_gfx12 {
     // Set the last bit of format to 1 to avoid round-trip issues, as some tools
     // print BUF_FMT_INVALID for format 0.
     let Inst{55} = 0b1;
-    let Inst{21-14} = op;
-    let AssemblerPredicate = isGFX12Only;
-    let DecoderNamespace = "GFX12";
   }
 }
 
 multiclass VBUFFER_MTBUF_Real_gfx12 op, string real_name> {
   defvar ps = !cast(NAME);
-  def _gfx12 : VBUFFER_Real, SIMCInstr {
-    let MTBUF = 1;
-
-    bits<7>  format;
-
-    let Inst{17-14} = op;
-    let Inst{21-18} = 0b1000;
+  def _gfx12 : VBUFFER_Real_gfx12<{0b1000, op}, ps, real_name> {
+    bits<7> format;
     let Inst{61-55} = format;
-    let AssemblerPredicate = isGFX12Only;
-    let DecoderNamespace = "GFX12";
   }
 }
 
-- 
GitLab


From 70da9155efb3aeb2c6b37678168cf29f91e80734 Mon Sep 17 00:00:00 2001
From: Mads Marquart 
Date: Fri, 8 Mar 2024 20:34:25 +0100
Subject: [PATCH 650/929] Update host CPU detection for Apple M2 and Apple M3
 (#82100)

`CPUFAMILY_ARM_BLIZZARD_AVALANCHE` and `CPUFAMILY_ARM_EVEREST_SAWTOOTH`
are taken from `` in `Kernel.framework`.
---
 llvm/lib/TargetParser/Host.cpp | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp
index ae58ddf03f86..ee4fd0425ca5 100644
--- a/llvm/lib/TargetParser/Host.cpp
+++ b/llvm/lib/TargetParser/Host.cpp
@@ -1466,6 +1466,8 @@ StringRef sys::getHostCPUName() {
 #define CPUFAMILY_ARM_VORTEX_TEMPEST 0x07d34b9f
 #define CPUFAMILY_ARM_LIGHTNING_THUNDER 0x462504d2
 #define CPUFAMILY_ARM_FIRESTORM_ICESTORM 0x1b588bb3
+#define CPUFAMILY_ARM_BLIZZARD_AVALANCHE 0xda33d83d
+#define CPUFAMILY_ARM_EVEREST_SAWTOOTH 0x8765edea
 
 StringRef sys::getHostCPUName() {
   uint32_t Family;
@@ -1491,9 +1493,13 @@ StringRef sys::getHostCPUName() {
     return "apple-a13";
   case CPUFAMILY_ARM_FIRESTORM_ICESTORM:
     return "apple-m1";
+  case CPUFAMILY_ARM_BLIZZARD_AVALANCHE:
+    return "apple-m2";
+  case CPUFAMILY_ARM_EVEREST_SAWTOOTH:
+    return "apple-m3";
   default:
     // Default to the newest CPU we know about.
-    return "apple-m1";
+    return "apple-m3";
   }
 }
 #elif defined(_AIX)
-- 
GitLab


From 087666811452e1b6490d59572337aaa907f93e7c Mon Sep 17 00:00:00 2001
From: Nikolas Klauser 
Date: Fri, 8 Mar 2024 20:43:10 +0100
Subject: [PATCH 651/929] [libc++][NFC] Move __format/format_fwd.h to
 __fwd/format.h (#84336)

---
 libcxx/include/CMakeLists.txt                            | 2 +-
 libcxx/include/__format/concepts.h                       | 2 +-
 libcxx/include/__format/format_arg.h                     | 2 +-
 libcxx/include/__format/format_args.h                    | 2 +-
 libcxx/include/__format/format_context.h                 | 2 +-
 libcxx/include/__format/formatter.h                      | 2 +-
 libcxx/include/{__format/format_fwd.h => __fwd/format.h} | 6 +++---
 libcxx/include/format                                    | 2 +-
 libcxx/include/libcxx.imp                                | 2 +-
 libcxx/include/module.modulemap                          | 2 +-
 10 files changed, 12 insertions(+), 12 deletions(-)
 rename libcxx/include/{__format/format_fwd.h => __fwd/format.h} (89%)

diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 3ea3360186dc..e37c4ac4fddd 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -380,7 +380,6 @@ set(files
   __format/format_context.h
   __format/format_error.h
   __format/format_functions.h
-  __format/format_fwd.h
   __format/format_parse_context.h
   __format/format_string.h
   __format/format_to_n_result.h
@@ -430,6 +429,7 @@ set(files
   __fwd/array.h
   __fwd/bit_reference.h
   __fwd/complex.h
+  __fwd/format.h
   __fwd/fstream.h
   __fwd/functional.h
   __fwd/ios.h
diff --git a/libcxx/include/__format/concepts.h b/libcxx/include/__format/concepts.h
index 299c5f40ee35..d7b5a9d16df7 100644
--- a/libcxx/include/__format/concepts.h
+++ b/libcxx/include/__format/concepts.h
@@ -13,8 +13,8 @@
 #include <__concepts/same_as.h>
 #include <__concepts/semiregular.h>
 #include <__config>
-#include <__format/format_fwd.h>
 #include <__format/format_parse_context.h>
+#include <__fwd/format.h>
 #include <__type_traits/is_specialization.h>
 #include <__type_traits/remove_const.h>
 #include <__utility/pair.h>
diff --git a/libcxx/include/__format/format_arg.h b/libcxx/include/__format/format_arg.h
index b786ac3b3620..4924e5fb3253 100644
--- a/libcxx/include/__format/format_arg.h
+++ b/libcxx/include/__format/format_arg.h
@@ -14,9 +14,9 @@
 #include <__concepts/arithmetic.h>
 #include <__config>
 #include <__format/concepts.h>
-#include <__format/format_fwd.h>
 #include <__format/format_parse_context.h>
 #include <__functional/invoke.h>
+#include <__fwd/format.h>
 #include <__memory/addressof.h>
 #include <__type_traits/conditional.h>
 #include <__utility/forward.h>
diff --git a/libcxx/include/__format/format_args.h b/libcxx/include/__format/format_args.h
index 9e0afecc0ae9..79fe51f96c6a 100644
--- a/libcxx/include/__format/format_args.h
+++ b/libcxx/include/__format/format_args.h
@@ -14,7 +14,7 @@
 #include <__config>
 #include <__format/format_arg.h>
 #include <__format/format_arg_store.h>
-#include <__format/format_fwd.h>
+#include <__fwd/format.h>
 #include 
 #include 
 
diff --git a/libcxx/include/__format/format_context.h b/libcxx/include/__format/format_context.h
index 68dcdb49d3aa..d131e942aca6 100644
--- a/libcxx/include/__format/format_context.h
+++ b/libcxx/include/__format/format_context.h
@@ -18,7 +18,7 @@
 #include <__format/format_arg_store.h>
 #include <__format/format_args.h>
 #include <__format/format_error.h>
-#include <__format/format_fwd.h>
+#include <__fwd/format.h>
 #include <__iterator/back_insert_iterator.h>
 #include <__iterator/concepts.h>
 #include <__memory/addressof.h>
diff --git a/libcxx/include/__format/formatter.h b/libcxx/include/__format/formatter.h
index 079befc5bd9c..47e35789b817 100644
--- a/libcxx/include/__format/formatter.h
+++ b/libcxx/include/__format/formatter.h
@@ -12,7 +12,7 @@
 
 #include <__availability>
 #include <__config>
-#include <__format/format_fwd.h>
+#include <__fwd/format.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__format/format_fwd.h b/libcxx/include/__fwd/format.h
similarity index 89%
rename from libcxx/include/__format/format_fwd.h
rename to libcxx/include/__fwd/format.h
index 120b2fc8d47d..6f5c71243711 100644
--- a/libcxx/include/__format/format_fwd.h
+++ b/libcxx/include/__fwd/format.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_FORMAT_FWD_H
-#define _LIBCPP___FORMAT_FORMAT_FWD_H
+#ifndef _LIBCPP___FWD_FORMAT_H
+#define _LIBCPP___FWD_FORMAT_H
 
 #include <__availability>
 #include <__config>
@@ -36,4 +36,4 @@ struct _LIBCPP_TEMPLATE_VIS formatter;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_FORMAT_FWD_H
+#endif // _LIBCPP___FWD_FORMAT_H
diff --git a/libcxx/include/format b/libcxx/include/format
index b2fe0053b974..c0485c5a1035 100644
--- a/libcxx/include/format
+++ b/libcxx/include/format
@@ -199,7 +199,6 @@ namespace std {
 #include <__format/format_context.h>
 #include <__format/format_error.h>
 #include <__format/format_functions.h>
-#include <__format/format_fwd.h>
 #include <__format/format_parse_context.h>
 #include <__format/format_string.h>
 #include <__format/format_to_n_result.h>
@@ -215,6 +214,7 @@ namespace std {
 #include <__format/range_default_formatter.h>
 #include <__format/range_formatter.h>
 #include <__format/unicode.h>
+#include <__fwd/format.h>
 #include 
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/libcxx.imp b/libcxx/include/libcxx.imp
index cdbb0a63fc0e..e02dc8da6ba1 100644
--- a/libcxx/include/libcxx.imp
+++ b/libcxx/include/libcxx.imp
@@ -374,7 +374,6 @@
   { include: [ "<__format/format_context.h>", "private", "", "public" ] },
   { include: [ "<__format/format_error.h>", "private", "", "public" ] },
   { include: [ "<__format/format_functions.h>", "private", "", "public" ] },
-  { include: [ "<__format/format_fwd.h>", "private", "", "public" ] },
   { include: [ "<__format/format_parse_context.h>", "private", "", "public" ] },
   { include: [ "<__format/format_string.h>", "private", "", "public" ] },
   { include: [ "<__format/format_to_n_result.h>", "private", "", "public" ] },
@@ -425,6 +424,7 @@
   { include: [ "<__fwd/bit_reference.h>", "private", "", "public" ] },
   { include: [ "<__fwd/bit_reference.h>", "private", "", "public" ] },
   { include: [ "<__fwd/complex.h>", "private", "", "public" ] },
+  { include: [ "<__fwd/format.h>", "private", "", "public" ] },
   { include: [ "<__fwd/fstream.h>", "private", "", "public" ] },
   { include: [ "<__fwd/functional.h>", "private", "", "public" ] },
   { include: [ "<__fwd/ios.h>", "private", "", "public" ] },
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index b247f97c1804..98890e890cdb 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -1315,7 +1315,7 @@ module std_private_format_format_functions                [system] {
   header "__format/format_functions.h"
   export std_string
 }
-module std_private_format_format_fwd                      [system] { header "__format/format_fwd.h" }
+module std_private_format_fwd                             [system] { header "__fwd/format.h" }
 module std_private_format_format_parse_context            [system] { header "__format/format_parse_context.h" }
 module std_private_format_format_string                   [system] { header "__format/format_string.h" }
 module std_private_format_format_to_n_result              [system] {
-- 
GitLab


From 52b3e89ff4517521e40f5e416a40e574321eaee9 Mon Sep 17 00:00:00 2001
From: Fangrui Song 
Date: Fri, 8 Mar 2024 11:45:43 -0800
Subject: [PATCH 652/929] [ELF] Simplify GdbIndexSection. NFC

---
 lld/ELF/SyntheticSections.cpp | 27 ++++++++++-----------------
 1 file changed, 10 insertions(+), 17 deletions(-)

diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index b6bdc350bc0d..66b3e835cabc 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -2805,21 +2805,16 @@ createSymbols(
     cuIdx += chunks[i].compilationUnits.size();
   }
 
-  // The number of symbols we will handle in this function is of the order
-  // of millions for very large executables, so we use multi-threading to
-  // speed it up.
+  // Collect the compilation unitss for each unique name. Speed it up using
+  // multi-threading as the number of symbols can be in the order of millions.
+  // Shard GdbSymbols by hash's high bits.
   constexpr size_t numShards = 32;
   const size_t concurrency =
       llvm::bit_floor(std::min(config->threadCount, numShards));
-
-  // A sharded map to uniquify symbols by name.
+  const size_t shift = 32 - llvm::countr_zero(numShards);
   auto map =
       std::make_unique[]>(numShards);
-  size_t shift = 32 - llvm::countr_zero(numShards);
-
-  // Instantiate GdbSymbols while uniqufying them by name.
   auto symbols = std::make_unique[]>(numShards);
-
   parallelFor(0, concurrency, [&](size_t threadId) {
     uint32_t i = 0;
     for (ArrayRef entries : nameAttrs) {
@@ -2829,14 +2824,12 @@ createSymbols(
           continue;
 
         uint32_t v = ent.cuIndexAndAttrs + cuIdxs[i];
-        size_t &idx = map[shardId][ent.name];
-        if (idx) {
-          symbols[shardId][idx - 1].cuVector.push_back(v);
-          continue;
-        }
-
-        idx = symbols[shardId].size() + 1;
-        symbols[shardId].push_back({ent.name, {v}, 0, 0});
+        auto [it, inserted] =
+            map[shardId].try_emplace(ent.name, symbols[shardId].size());
+        if (inserted)
+          symbols[shardId].push_back({ent.name, {v}, 0, 0});
+        else
+          symbols[shardId][it->second].cuVector.push_back(v);
       }
       ++i;
     }
-- 
GitLab


From e963d0740e64fc70b4018f39325469d204f6217a Mon Sep 17 00:00:00 2001
From: Shilei Tian 
Date: Fri, 8 Mar 2024 14:49:52 -0500
Subject: [PATCH 653/929] [AMDGPU] Replace `isInlinableLiteral16` with specific
 version (#84402)

The current implementation of `isInlinableLiteral16` assumes, a 16-bit
inlinable
literal is either an `i16` or a `fp16`. This is not always true because
of
`bf16`. However, we can't tell `fp16` and `bf16` apart by just looking
at the
value. This patch splits `isInlinableLiteral16` into three versions,
`i16`,
`fp16`, `bf16` respectively, and call the corresponding version.
---
 llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp |  52 ++---
 .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp      |  97 +++++++--
 .../AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp |  29 +--
 .../AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h   |   4 +-
 .../MCTargetDesc/AMDGPUMCCodeEmitter.cpp      |  11 +-
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |  28 ++-
 llvm/lib/Target/AMDGPU/SIInstrInfo.cpp        |  25 ++-
 llvm/lib/Target/AMDGPU/SIInstrInfo.h          |   4 +-
 .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp    |   8 +-
 llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h |   8 +-
 llvm/lib/Target/AMDGPU/VOP3PInstructions.td   |   1 +
 llvm/test/CodeGen/AMDGPU/immv216.ll           |  42 ++--
 .../test/CodeGen/AMDGPU/inline-constraints.ll |  12 --
 .../CodeGen/AMDGPU/shrink-add-sub-constant.ll | 144 ++++++++-----
 .../test/CodeGen/AMDGPU/wmma-gfx12-w32-imm.ll |   6 +-
 .../test/CodeGen/AMDGPU/wmma-gfx12-w64-imm.ll |   5 +-
 llvm/test/MC/AMDGPU/gfx10_asm_vop1.s          |  16 +-
 llvm/test/MC/AMDGPU/gfx10_asm_vop3.s          | 192 +++++++++---------
 llvm/test/MC/AMDGPU/gfx10_asm_vopc.s          |  96 ++++-----
 llvm/test/MC/AMDGPU/gfx10_asm_vopc_e64.s      | 192 +++++++++---------
 llvm/test/MC/AMDGPU/gfx10_asm_vopcx.s         | 144 ++++++-------
 llvm/test/MC/AMDGPU/gfx11_asm_vop1.s          |  10 +-
 llvm/test/MC/AMDGPU/gfx11_asm_vop3.s          | 108 +++++-----
 .../test/MC/AMDGPU/gfx11_asm_vop3_from_vop1.s |  10 +-
 .../test/MC/AMDGPU/gfx11_asm_vop3_from_vopc.s |  96 ++++-----
 .../MC/AMDGPU/gfx11_asm_vop3_from_vopcx.s     |  48 ++---
 llvm/test/MC/AMDGPU/gfx11_asm_vopc.s          |  48 ++---
 llvm/test/MC/AMDGPU/gfx11_asm_vopcx.s         |  24 +--
 llvm/test/MC/AMDGPU/gfx12_asm_vop1.s          |  10 +-
 llvm/test/MC/AMDGPU/gfx12_asm_vop3.s          | 108 +++++-----
 .../test/MC/AMDGPU/gfx12_asm_vop3_from_vop1.s |  10 +-
 llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s         |  96 ++++-----
 llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s        |  48 ++---
 llvm/test/MC/AMDGPU/gfx12_asm_vopc.s          |  48 ++---
 llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s         |  24 +--
 llvm/test/MC/AMDGPU/gfx8_asm_vop1.s           |   8 +-
 llvm/test/MC/AMDGPU/gfx8_asm_vop2.s           |  44 ++--
 llvm/test/MC/AMDGPU/gfx8_asm_vopc.s           | 128 ++++++------
 llvm/test/MC/AMDGPU/gfx9-asm-err.s            |  20 +-
 llvm/test/MC/AMDGPU/gfx9_asm_vop1.s           |   8 +-
 llvm/test/MC/AMDGPU/gfx9_asm_vop2.s           |  46 +++--
 llvm/test/MC/AMDGPU/gfx9_asm_vopc.s           | 128 ++++++------
 llvm/test/MC/AMDGPU/gfx9_err_pos.s            |  10 -
 llvm/test/MC/AMDGPU/vop3-gfx10.s              |   4 +-
 llvm/test/MC/AMDGPU/vop3-gfx9.s               |  16 +-
 llvm/test/MC/AMDGPU/vop3.s                    |   4 +-
 llvm/test/MC/AMDGPU/vop_sdwa.s                |   4 +-
 47 files changed, 1180 insertions(+), 1044 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index d1af74e6e0d3..c99e49001466 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -3327,35 +3327,41 @@ bool AMDGPUDAGToDAGISel::SelectWMMAVISrc(SDValue In, SDValue &Src) const {
 
   // 16 bit splat
   SDValue SplatSrc32 = stripBitcast(In);
-  if (auto *SplatSrc32BV = dyn_cast(SplatSrc32)) {
+  if (auto *SplatSrc32BV = dyn_cast(SplatSrc32))
     if (SDValue Splat32 = SplatSrc32BV->getSplatValue()) {
       SDValue SplatSrc16 = stripBitcast(Splat32);
-      if (auto *SplatSrc16BV = dyn_cast(SplatSrc16)) {
+      if (auto *SplatSrc16BV = dyn_cast(SplatSrc16))
         if (SDValue Splat = SplatSrc16BV->getSplatValue()) {
-
-          // f16
-          if (isInlineImmediate(Splat.getNode())) {
-            const ConstantFPSDNode *C = dyn_cast(Splat);
-            int64_t Imm = C->getValueAPF().bitcastToAPInt().getSExtValue();
-            Src = CurDAG->getTargetConstant(Imm, SDLoc(In), MVT::i16);
-            return true;
-          }
-
-          // bf16
-          if (const ConstantSDNode *C = dyn_cast(Splat)) {
-            const SIInstrInfo *TII = Subtarget->getInstrInfo();
-            APInt BF16Value = C->getAPIntValue();
-            APInt F32Value = BF16Value.zext(32).shl(16);
-            if (TII->isInlineConstant(F32Value)) {
-              int64_t Imm = F32Value.getSExtValue();
-              Src = CurDAG->getTargetConstant(Imm, SDLoc(In), MVT::i32);
-              return true;
-            }
+          const SIInstrInfo *TII = Subtarget->getInstrInfo();
+          std::optional RawValue;
+          if (const ConstantFPSDNode *C = dyn_cast(Splat))
+            RawValue = C->getValueAPF().bitcastToAPInt();
+          else if (const ConstantSDNode *C = dyn_cast(Splat))
+            RawValue = C->getAPIntValue();
+
+          if (RawValue.has_value()) {
+            EVT VT = In.getValueType().getScalarType();
+            if (VT.getSimpleVT() == MVT::f16 || VT.getSimpleVT() == MVT::bf16) {
+              APFloat FloatVal(VT.getSimpleVT() == MVT::f16
+                                   ? APFloatBase::IEEEhalf()
+                                   : APFloatBase::BFloat(),
+                               RawValue.value());
+              if (TII->isInlineConstant(FloatVal)) {
+                Src = CurDAG->getTargetConstant(RawValue.value(), SDLoc(In),
+                                                MVT::i16);
+                return true;
+              }
+            } else if (VT.getSimpleVT() == MVT::i16) {
+              if (TII->isInlineConstant(RawValue.value())) {
+                Src = CurDAG->getTargetConstant(RawValue.value(), SDLoc(In),
+                                                MVT::i16);
+                return true;
+              }
+            } else
+              llvm_unreachable("unknown 16-bit type");
           }
         }
-      }
     }
-  }
 
   return false;
 }
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index d5efd4415562..027dd0f2c224 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -1926,6 +1926,11 @@ static const fltSemantics *getFltSemantics(MVT VT) {
 
 static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
   switch (OperandType) {
+  // When floating-point immediate is used as operand of type i16, the 32-bit
+   // representation of the constant truncated to the 16 LSBs should be used.
+  case AMDGPU::OPERAND_REG_IMM_INT16:
+  case AMDGPU::OPERAND_REG_INLINE_C_INT16:
+  case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
   case AMDGPU::OPERAND_REG_IMM_INT32:
   case AMDGPU::OPERAND_REG_IMM_FP32:
   case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED:
@@ -1949,13 +1954,10 @@ static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
   case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
     return &APFloat::IEEEdouble();
-  case AMDGPU::OPERAND_REG_IMM_INT16:
   case AMDGPU::OPERAND_REG_IMM_FP16:
   case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED:
-  case AMDGPU::OPERAND_REG_INLINE_C_INT16:
   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
-  case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
   case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
   case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16:
   case AMDGPU::OPERAND_REG_IMM_V2FP16:
@@ -2001,13 +2003,15 @@ static bool isSafeTruncation(int64_t Val, unsigned Size) {
 }
 
 static bool isInlineableLiteralOp16(int64_t Val, MVT VT, bool HasInv2Pi) {
-  if (VT.getScalarType() == MVT::i16) {
-    // FP immediate values are broken.
-    return isInlinableIntLiteral(Val);
-  }
+  if (VT.getScalarType() == MVT::i16)
+    return isInlinableLiteral32(Val, HasInv2Pi);
+
+  if (VT.getScalarType() == MVT::f16)
+    return AMDGPU::isInlinableLiteralFP16(Val, HasInv2Pi);
 
-  // f16/v2f16 operands work correctly for all values.
-  return AMDGPU::isInlinableLiteral16(Val, HasInv2Pi);
+  assert(VT.getScalarType() == MVT::bf16);
+
+  return AMDGPU::isInlinableLiteralBF16(Val, HasInv2Pi);
 }
 
 bool AMDGPUOperand::isInlinableImm(MVT type) const {
@@ -2041,9 +2045,30 @@ bool AMDGPUOperand::isInlinableImm(MVT type) const {
       return false;
 
     if (type.getScalarSizeInBits() == 16) {
-      return isInlineableLiteralOp16(
-        static_cast(FPLiteral.bitcastToAPInt().getZExtValue()),
-        type, AsmParser->hasInv2PiInlineImm());
+      bool Lost = false;
+      switch (type.getScalarType().SimpleTy) {
+      default:
+        llvm_unreachable("unknown 16-bit type");
+      case MVT::bf16:
+        FPLiteral.convert(APFloatBase::BFloat(), APFloat::rmNearestTiesToEven,
+                          &Lost);
+        break;
+      case MVT::f16:
+        FPLiteral.convert(APFloatBase::IEEEhalf(), APFloat::rmNearestTiesToEven,
+                          &Lost);
+        break;
+      case MVT::i16:
+        FPLiteral.convert(APFloatBase::IEEEsingle(),
+                          APFloat::rmNearestTiesToEven, &Lost);
+        break;
+      }
+      // We need to use 32-bit representation here because when a floating-point
+      // inline constant is used as an i16 operand, its 32-bit representation
+      // representation will be used. We will need the 32-bit value to check if
+      // it is FP inline constant.
+      uint32_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
+      return isInlineableLiteralOp16(ImmVal, type,
+                                     AsmParser->hasInv2PiInlineImm());
     }
 
     // Check if single precision literal is inlinable
@@ -2375,15 +2400,26 @@ void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyMo
     return;
 
   case AMDGPU::OPERAND_REG_IMM_INT16:
-  case AMDGPU::OPERAND_REG_IMM_FP16:
-  case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED:
   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
-  case AMDGPU::OPERAND_REG_INLINE_C_FP16:
   case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
+    if (isSafeTruncation(Val, 16) &&
+        AMDGPU::isInlinableIntLiteral(static_cast(Val))) {
+      Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
+      setImmKindConst();
+      return;
+    }
+
+    Inst.addOperand(MCOperand::createImm(Val & 0xffff));
+    setImmKindLiteral();
+    return;
+
+  case AMDGPU::OPERAND_REG_INLINE_C_FP16:
+  case AMDGPU::OPERAND_REG_IMM_FP16:
+  case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED:
   case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
     if (isSafeTruncation(Val, 16) &&
-        AMDGPU::isInlinableLiteral16(static_cast(Val),
-                                     AsmParser->hasInv2PiInlineImm())) {
+        AMDGPU::isInlinableLiteralFP16(static_cast(Val),
+                                       AsmParser->hasInv2PiInlineImm())) {
       Inst.addOperand(MCOperand::createImm(Val));
       setImmKindConst();
       return;
@@ -2410,12 +2446,17 @@ void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyMo
     return;
 
   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
+  case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16: {
+    assert(isSafeTruncation(Val, 16));
+    assert(AMDGPU::isInlinableIntLiteral(static_cast(Val)));
+    Inst.addOperand(MCOperand::createImm(Val));
+    return;
+  }
   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
-  case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16:
   case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: {
     assert(isSafeTruncation(Val, 16));
-    assert(AMDGPU::isInlinableLiteral16(static_cast(Val),
-                                        AsmParser->hasInv2PiInlineImm()));
+    assert(AMDGPU::isInlinableLiteralFP16(static_cast(Val),
+                                          AsmParser->hasInv2PiInlineImm()));
 
     Inst.addOperand(MCOperand::createImm(Val));
     return;
@@ -3542,7 +3583,7 @@ bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
     if (OperandType == AMDGPU::OPERAND_REG_IMM_INT16 ||
         OperandType == AMDGPU::OPERAND_REG_INLINE_C_INT16 ||
         OperandType == AMDGPU::OPERAND_REG_INLINE_AC_INT16)
-      return AMDGPU::isInlinableIntLiteral(Val);
+      return AMDGPU::isInlinableLiteralI16(Val, hasInv2PiInlineImm());
 
     if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
         OperandType == AMDGPU::OPERAND_REG_INLINE_AC_V2INT16 ||
@@ -3559,7 +3600,19 @@ bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
         OperandType == AMDGPU::OPERAND_REG_IMM_V2BF16)
       return AMDGPU::isInlinableLiteralV2BF16(Val);
 
-    return AMDGPU::isInlinableLiteral16(Val, hasInv2PiInlineImm());
+    if (OperandType == AMDGPU::OPERAND_REG_IMM_FP16 ||
+        OperandType == AMDGPU::OPERAND_REG_INLINE_C_FP16 ||
+        OperandType == AMDGPU::OPERAND_REG_INLINE_AC_FP16 ||
+        OperandType == AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED)
+      return AMDGPU::isInlinableLiteralFP16(Val, hasInv2PiInlineImm());
+
+    if (OperandType == AMDGPU::OPERAND_REG_IMM_BF16 ||
+        OperandType == AMDGPU::OPERAND_REG_INLINE_C_BF16 ||
+        OperandType == AMDGPU::OPERAND_REG_INLINE_AC_BF16 ||
+        OperandType == AMDGPU::OPERAND_REG_IMM_BF16_DEFERRED)
+      return AMDGPU::isInlinableLiteralBF16(Val, hasInv2PiInlineImm());
+
+    llvm_unreachable("invalid operand type");
   }
   default:
     llvm_unreachable("invalid operand size");
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
index a32be1e50a60..b6a95906bc45 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
@@ -451,19 +451,20 @@ void AMDGPUInstPrinter::printVINTRPDst(const MCInst *MI, unsigned OpNo,
 void AMDGPUInstPrinter::printImmediateInt16(uint32_t Imm,
                                             const MCSubtargetInfo &STI,
                                             raw_ostream &O) {
-  int16_t SImm = static_cast(Imm);
+  int32_t SImm = static_cast(Imm);
   if (isInlinableIntLiteral(SImm)) {
     O << SImm;
-  } else {
-    uint64_t Imm16 = static_cast(Imm);
-    O << formatHex(Imm16);
+    return;
   }
+
+  if (printImmediateFloat32(Imm, STI, O))
+    return;
+
+  O << formatHex(static_cast(Imm & 0xffff));
 }
 
-// This must accept a 32-bit immediate value to correctly handle packed 16-bit
-// operations.
-static bool printImmediateFloat16(uint32_t Imm, const MCSubtargetInfo &STI,
-                                  raw_ostream &O) {
+static bool printImmediateFP16(uint32_t Imm, const MCSubtargetInfo &STI,
+                               raw_ostream &O) {
   if (Imm == 0x3C00)
     O << "1.0";
   else if (Imm == 0xBC00)
@@ -529,9 +530,9 @@ void AMDGPUInstPrinter::printImmediateBF16(uint32_t Imm,
   O << formatHex(static_cast(Imm));
 }
 
-void AMDGPUInstPrinter::printImmediate16(uint32_t Imm,
-                                         const MCSubtargetInfo &STI,
-                                         raw_ostream &O) {
+void AMDGPUInstPrinter::printImmediateF16(uint32_t Imm,
+                                          const MCSubtargetInfo &STI,
+                                          raw_ostream &O) {
   int16_t SImm = static_cast(Imm);
   if (isInlinableIntLiteral(SImm)) {
     O << SImm;
@@ -539,7 +540,7 @@ void AMDGPUInstPrinter::printImmediate16(uint32_t Imm,
   }
 
   uint16_t HImm = static_cast(Imm);
-  if (printImmediateFloat16(HImm, STI, O))
+  if (printImmediateFP16(HImm, STI, O))
     return;
 
   uint64_t Imm16 = static_cast(Imm);
@@ -566,7 +567,7 @@ void AMDGPUInstPrinter::printImmediateV216(uint32_t Imm, uint8_t OpType,
   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
   case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16:
     if (isUInt<16>(Imm) &&
-        printImmediateFloat16(static_cast(Imm), STI, O))
+        printImmediateFP16(static_cast(Imm), STI, O))
       return;
     break;
   case AMDGPU::OPERAND_REG_IMM_V2BF16:
@@ -845,7 +846,7 @@ void AMDGPUInstPrinter::printRegularOperand(const MCInst *MI, unsigned OpNo,
     case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
     case AMDGPU::OPERAND_REG_IMM_FP16:
     case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED:
-      printImmediate16(Op.getImm(), STI, O);
+      printImmediateF16(Op.getImm(), STI, O);
       break;
     case AMDGPU::OPERAND_REG_INLINE_C_BF16:
     case AMDGPU::OPERAND_REG_INLINE_AC_BF16:
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
index 15ecbf2e5e59..c801eaf1111e 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
@@ -86,10 +86,10 @@ private:
                       raw_ostream &O);
   void printImmediateInt16(uint32_t Imm, const MCSubtargetInfo &STI,
                            raw_ostream &O);
-  void printImmediate16(uint32_t Imm, const MCSubtargetInfo &STI,
-                        raw_ostream &O);
   void printImmediateBF16(uint32_t Imm, const MCSubtargetInfo &STI,
                           raw_ostream &O);
+  void printImmediateF16(uint32_t Imm, const MCSubtargetInfo &STI,
+                         raw_ostream &O);
   void printImmediateV216(uint32_t Imm, uint8_t OpType,
                           const MCSubtargetInfo &STI, raw_ostream &O);
   bool printImmediateFloat32(uint32_t Imm, const MCSubtargetInfo &STI,
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
index e51bb40132f9..fb93f45e3e87 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
@@ -116,11 +116,6 @@ static uint32_t getIntInlineImmEncoding(IntTy Imm) {
   return 0;
 }
 
-static uint32_t getLit16IntEncoding(uint16_t Val, const MCSubtargetInfo &STI) {
-  uint16_t IntImm = getIntInlineImmEncoding(static_cast(Val));
-  return IntImm == 0 ? 255 : IntImm;
-}
-
 static uint32_t getLit16Encoding(uint16_t Val, const MCSubtargetInfo &STI) {
   uint16_t IntImm = getIntInlineImmEncoding(static_cast(Val));
   if (IntImm != 0)
@@ -214,6 +209,10 @@ static uint32_t getLit32Encoding(uint32_t Val, const MCSubtargetInfo &STI) {
   return 255;
 }
 
+static uint32_t getLit16IntEncoding(uint32_t Val, const MCSubtargetInfo &STI) {
+  return getLit32Encoding(Val, STI);
+}
+
 static uint32_t getLit64Encoding(uint64_t Val, const MCSubtargetInfo &STI) {
   uint32_t IntImm = getIntInlineImmEncoding(static_cast(Val));
   if (IntImm != 0)
@@ -296,7 +295,7 @@ AMDGPUMCCodeEmitter::getLitEncoding(const MCOperand &MO,
   case AMDGPU::OPERAND_REG_IMM_INT16:
   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
   case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
-    return getLit16IntEncoding(static_cast(Imm), STI);
+    return getLit16IntEncoding(static_cast(Imm), STI);
 
   case AMDGPU::OPERAND_REG_IMM_FP16:
   case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED:
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index e105db313548..1889ab007288 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -15495,16 +15495,32 @@ bool SITargetLowering::checkAsmConstraintVal(SDValue Op, StringRef Constraint,
   llvm_unreachable("Invalid asm constraint");
 }
 
-bool SITargetLowering::checkAsmConstraintValA(SDValue Op,
-                                              uint64_t Val,
+bool SITargetLowering::checkAsmConstraintValA(SDValue Op, uint64_t Val,
                                               unsigned MaxSize) const {
   unsigned Size = std::min(Op.getScalarValueSizeInBits(), MaxSize);
   bool HasInv2Pi = Subtarget->hasInv2PiInlineImm();
-  if ((Size == 16 && AMDGPU::isInlinableLiteral16(Val, HasInv2Pi)) ||
-      (Size == 32 && AMDGPU::isInlinableLiteral32(Val, HasInv2Pi)) ||
-      (Size == 64 && AMDGPU::isInlinableLiteral64(Val, HasInv2Pi))) {
-    return true;
+  if (Size == 16) {
+    MVT VT = Op.getSimpleValueType();
+    switch (VT.SimpleTy) {
+    default:
+      return false;
+    case MVT::i16:
+      return AMDGPU::isInlinableLiteralI16(Val, HasInv2Pi);
+    case MVT::f16:
+      return AMDGPU::isInlinableLiteralFP16(Val, HasInv2Pi);
+    case MVT::bf16:
+      return AMDGPU::isInlinableLiteralBF16(Val, HasInv2Pi);
+    case MVT::v2i16:
+      return AMDGPU::getInlineEncodingV2I16(Val).has_value();
+    case MVT::v2f16:
+      return AMDGPU::getInlineEncodingV2F16(Val).has_value();
+    case MVT::v2bf16:
+      return AMDGPU::getInlineEncodingV2BF16(Val).has_value();
+    }
   }
+  if ((Size == 32 && AMDGPU::isInlinableLiteral32(Val, HasInv2Pi)) ||
+      (Size == 64 && AMDGPU::isInlinableLiteral64(Val, HasInv2Pi)))
+    return true;
   return false;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index ca2c2d87009f..e8022c8b0afa 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -4121,13 +4121,32 @@ bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
                                         ST.hasInv2PiInlineImm());
   case 16:
     return ST.has16BitInsts() &&
-           AMDGPU::isInlinableLiteral16(Imm.getSExtValue(),
-                                        ST.hasInv2PiInlineImm());
+           AMDGPU::isInlinableLiteralI16(Imm.getSExtValue(),
+                                         ST.hasInv2PiInlineImm());
   default:
     llvm_unreachable("invalid bitwidth");
   }
 }
 
+bool SIInstrInfo::isInlineConstant(const APFloat &Imm) const {
+  APInt IntImm = Imm.bitcastToAPInt();
+  int64_t IntImmVal = IntImm.getSExtValue();
+  bool HasInv2Pi = ST.hasInv2PiInlineImm();
+  switch (APFloat::SemanticsToEnum(Imm.getSemantics())) {
+  default:
+    llvm_unreachable("invalid fltSemantics");
+  case APFloatBase::S_IEEEsingle:
+  case APFloatBase::S_IEEEdouble:
+    return isInlineConstant(IntImm);
+  case APFloatBase::S_BFloat:
+    return ST.has16BitInsts() &&
+           AMDGPU::isInlinableLiteralBF16(IntImmVal, HasInv2Pi);
+  case APFloatBase::S_IEEEhalf:
+    return ST.has16BitInsts() &&
+           AMDGPU::isInlinableLiteralFP16(IntImmVal, HasInv2Pi);
+  }
+}
+
 bool SIInstrInfo::isInlineConstant(const MachineOperand &MO,
                                    uint8_t OperandType) const {
   assert(!MO.isReg() && "isInlineConstant called on register operand!");
@@ -4200,7 +4219,7 @@ bool SIInstrInfo::isInlineConstant(const MachineOperand &MO,
       // constants in these cases
       int16_t Trunc = static_cast(Imm);
       return ST.has16BitInsts() &&
-             AMDGPU::isInlinableLiteral16(Trunc, ST.hasInv2PiInlineImm());
+             AMDGPU::isInlinableLiteralFP16(Trunc, ST.hasInv2PiInlineImm());
     }
 
     return false;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 4200e0c8a29e..a62bf779fe2e 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -984,9 +984,7 @@ public:
 
   bool isInlineConstant(const APInt &Imm) const;
 
-  bool isInlineConstant(const APFloat &Imm) const {
-    return isInlineConstant(Imm.bitcastToAPInt());
-  }
+  bool isInlineConstant(const APFloat &Imm) const;
 
   // Returns true if this non-register operand definitely does not need to be
   // encoded as a 32-bit literal. Note that this function handles all kinds of
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index 62903a244dc8..edb0e50da289 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -2657,13 +2657,15 @@ bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi) {
          Val == 0x3E22;   // 1.0 / (2.0 * pi)
 }
 
-bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi) {
+bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi) {
+  return isInlinableLiteral32(Literal, HasInv2Pi);
+}
+
+bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi) {
   if (!HasInv2Pi)
     return false;
-
   if (isInlinableIntLiteral(Literal))
     return true;
-
   uint16_t Val = static_cast(Literal);
   return Val == 0x3C00 || // 1.0
          Val == 0xBC00 || // -1.0
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index bb307cb67c9b..d7ea2a3eff4b 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -1404,7 +1404,13 @@ LLVM_READNONE
 bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi);
 
 LLVM_READNONE
-bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi);
+bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi);
+
+LLVM_READNONE
+bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi);
+
+LLVM_READNONE
+bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi);
 
 LLVM_READNONE
 std::optional getInlineEncodingV2I16(uint32_t Literal);
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index 4ca2835eea58..a7d63fdb2e04 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -1007,6 +1007,7 @@ class VOP3PWMMA_Profile ArgTy, bit _IsSWMMAC, int _IndexType,
                                !cast("VISrc_"#ArgTy[3].Size#
                                                       !cond(IsC_F32: "_f32",
                                                             IsC_F16: "_f16",
+                                                            IsC_BF16: "_bf16",
                                                             1: "_b32")));
 
   // For f16 and bf16 matrices A and B, each element can be modified by
diff --git a/llvm/test/CodeGen/AMDGPU/immv216.ll b/llvm/test/CodeGen/AMDGPU/immv216.ll
index b66ca71a3274..ae51c3edf1c7 100644
--- a/llvm/test/CodeGen/AMDGPU/immv216.ll
+++ b/llvm/test/CodeGen/AMDGPU/immv216.ll
@@ -577,40 +577,40 @@ define amdgpu_kernel void @add_inline_imm_64_v2f16(ptr addrspace(1) %out, <2 x h
 }
 
 ; GCN-LABEL: {{^}}mul_inline_imm_0.5_v2i16:
-; GFX9: s_mov_b32 [[K:s[0-9]+]], 0x38003800
-; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]]
+; GFX9: s_movk_i32 [[K:s[0-9]+]], 0x3800
+; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] op_sel_hi:[1,0]
 
-; GFX10: v_pk_mul_lo_u16 v0, 0x38003800, v0 ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0x38,0x00,0x38]
+; GFX10: v_pk_mul_lo_u16 v0, 0x3800, v0 op_sel_hi:[0,1] ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0x38,0x00,0x00]
 define <2 x i16> @mul_inline_imm_0.5_v2i16(<2 x i16> %x) {
   %y = mul <2 x i16> %x, bitcast (<2 x half>  to <2 x i16>)
   ret <2 x i16> %y
 }
 
 ; GCN-LABEL: {{^}}mul_inline_imm_neg_0.5_v2i16:
-; GFX9: s_mov_b32 [[K:s[0-9]+]], 0xb800b800
-; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]]
+; GFX9: s_movk_i32 [[K:s[0-9]+]], 0xb800
+; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] op_sel_hi:[1,0]
 
-; GFX10: v_pk_mul_lo_u16 v0, 0xb800b800, v0 ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0xb8,0x00,0xb8]
+; GFX10: v_pk_mul_lo_u16 v0, 0xffffb800, v0 op_sel_hi:[0,1] ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0xb8,0xff,0xff]
 define <2 x i16> @mul_inline_imm_neg_0.5_v2i16(<2 x i16> %x) {
   %y = mul <2 x i16> %x, bitcast (<2 x half>  to <2 x i16>)
   ret <2 x i16> %y
 }
 
 ; GCN-LABEL: {{^}}mul_inline_imm_1.0_v2i16:
-; GFX9: s_mov_b32 [[K:s[0-9]+]], 0x3c003c00
-; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]]
+; GFX9: s_movk_i32 [[K:s[0-9]+]], 0x3c00
+; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] op_sel_hi:[1,0]
 
-; GFX10: v_pk_mul_lo_u16 v0, 0x3c003c00, v0 ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0x3c,0x00,0x3c]
+; GFX10: v_pk_mul_lo_u16 v0, 0x3c00, v0 op_sel_hi:[0,1] ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0x3c,0x00,0x00]
 define <2 x i16> @mul_inline_imm_1.0_v2i16(<2 x i16> %x) {
   %y = mul <2 x i16> %x, bitcast (<2 x half>  to <2 x i16>)
   ret <2 x i16> %y
 }
 
 ; GCN-LABEL: {{^}}mul_inline_imm_neg_1.0_v2i16:
-; GFX9: s_mov_b32 [[K:s[0-9]+]], 0xbc00bc00
-; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]]
+; GFX9: s_movk_i32 [[K:s[0-9]+]], 0xbc00
+; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] op_sel_hi:[1,0]
 
-; GFX10: v_pk_mul_lo_u16 v0, 0xbc00bc00, v0 ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0xbc,0x00,0xbc]
+; GFX10: v_pk_mul_lo_u16 v0, 0xffffbc00, v0 op_sel_hi:[0,1] ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0xbc,0xff,0xff]
 define <2 x i16> @mul_inline_imm_neg_1.0_v2i16(<2 x i16> %x) {
   %y = mul <2 x i16> %x, bitcast (<2 x half>  to <2 x i16>)
   ret <2 x i16> %y
@@ -635,10 +635,10 @@ define <2 x i16> @shl_inline_imm_neg_2.0_v2i16(<2 x i16> %x) {
 }
 
 ; GCN-LABEL: {{^}}mul_inline_imm_4.0_v2i16:
-; GFX9: s_mov_b32 [[K:s[0-9]+]], 0x44004400
-; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]]
+; GFX9: s_movk_i32 [[K:s[0-9]+]], 0x4400
+; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] op_sel_hi:[1,0]
 
-; GFX10: v_pk_mul_lo_u16 v0, 0x44004400, v0 ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0x44,0x00,0x44]
+; GFX10: v_pk_mul_lo_u16 v0, 0x4400, v0 op_sel_hi:[0,1] ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0x44,0x00,0x00]
 define <2 x i16> @mul_inline_imm_4.0_v2i16(<2 x i16> %x) {
   %y = mul <2 x i16> %x, bitcast (<2 x half>  to <2 x i16>)
   ret <2 x i16> %y
@@ -646,20 +646,20 @@ define <2 x i16> @mul_inline_imm_4.0_v2i16(<2 x i16> %x) {
 }
 
 ; GCN-LABEL: {{^}}mul_inline_imm_neg_4.0_v2i16:
-; GFX9: s_mov_b32 [[K:s[0-9]+]], 0xc400c400
-; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]]
+; GFX9: s_movk_i32 [[K:s[0-9]+]], 0xc400
+; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] op_sel_hi:[1,0]
 
-; GFX10: v_pk_mul_lo_u16 v0, 0xc400c400, v0 ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0xc4,0x00,0xc4]
+; GFX10: v_pk_mul_lo_u16 v0, 0xffffc400, v0 op_sel_hi:[0,1] ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0xc4,0xff,0xff]
 define <2 x i16> @mul_inline_imm_neg_4.0_v2i16(<2 x i16> %x) {
   %y = mul <2 x i16> %x, bitcast (<2 x half>  to <2 x i16>)
   ret <2 x i16> %y
 }
 
 ; GCN-LABEL: {{^}}mul_inline_imm_inv2pi_v2i16:
-; GFX9: s_mov_b32 [[K:s[0-9]+]], 0x31183118
-; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]]
+; GFX9: s_movk_i32 [[K:s[0-9]+]], 0x3118
+; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] op_sel_hi:[1,0]
 
-; GFX10: v_pk_mul_lo_u16 v0, 0x31183118, v0 ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x18,0x31,0x18,0x31]
+; GFX10: v_pk_mul_lo_u16 v0, 0x3118, v0 op_sel_hi:[0,1] ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x18,0x31,0x00,0x00]
 define <2 x i16> @mul_inline_imm_inv2pi_v2i16(<2 x i16> %x) {
   %y = mul <2 x i16> %x, bitcast (<2 x half>  to <2 x i16>)
   ret <2 x i16> %y
diff --git a/llvm/test/CodeGen/AMDGPU/inline-constraints.ll b/llvm/test/CodeGen/AMDGPU/inline-constraints.ll
index 9ef246fe2e10..7bd6b037386b 100644
--- a/llvm/test/CodeGen/AMDGPU/inline-constraints.ll
+++ b/llvm/test/CodeGen/AMDGPU/inline-constraints.ll
@@ -97,7 +97,6 @@ define i32 @inline_A_constraint_H1() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'A'
 ; VI-LABEL: {{^}}inline_A_constraint_H2:
-; VI: v_mov_b32 {{v[0-9]+}}, 0x3c00
 define i32 @inline_A_constraint_H2() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,A"(i16 bitcast (half 1.0 to i16))
   ret i32 %v0
@@ -105,7 +104,6 @@ define i32 @inline_A_constraint_H2() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'A'
 ; VI-LABEL: {{^}}inline_A_constraint_H3:
-; VI: v_mov_b32 {{v[0-9]+}}, 0xbc00
 define i32 @inline_A_constraint_H3() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,A"(i16 bitcast (half -1.0 to i16))
   ret i32 %v0
@@ -113,7 +111,6 @@ define i32 @inline_A_constraint_H3() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'A'
 ; VI-LABEL: {{^}}inline_A_constraint_H4:
-; VI: v_mov_b32 {{v[0-9]+}}, 0x3118
 define i32 @inline_A_constraint_H4() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,A"(half 0xH3118)
   ret i32 %v0
@@ -121,7 +118,6 @@ define i32 @inline_A_constraint_H4() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'A'
 ; VI-LABEL: {{^}}inline_A_constraint_H5:
-; VI: v_mov_b32 {{v[0-9]+}}, 0x3118
 define i32 @inline_A_constraint_H5() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,A"(i16 bitcast (half 0xH3118 to i16))
   ret i32 %v0
@@ -129,7 +125,6 @@ define i32 @inline_A_constraint_H5() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'A'
 ; VI-LABEL: {{^}}inline_A_constraint_H6:
-; VI: v_mov_b32 {{v[0-9]+}}, 0xb800
 define i32 @inline_A_constraint_H6() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,A"(half -0.5)
   ret i32 %v0
@@ -293,7 +288,6 @@ define i32 @inline_A_constraint_V0() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'A'
 ; VI-LABEL: {{^}}inline_A_constraint_V1:
-; VI: v_mov_b32 {{v[0-9]+}}, 0xb800
 define i32 @inline_A_constraint_V1() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,A"(<2 x half> )
   ret i32 %v0
@@ -970,7 +964,6 @@ define i32 @inline_DA_constraint_H1() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'DA'
 ; VI-LABEL: {{^}}inline_DA_constraint_H2:
-; VI: v_mov_b32 {{v[0-9]+}}, 0x3c00
 define i32 @inline_DA_constraint_H2() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i16 bitcast (half 1.0 to i16))
   ret i32 %v0
@@ -978,7 +971,6 @@ define i32 @inline_DA_constraint_H2() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'DA'
 ; VI-LABEL: {{^}}inline_DA_constraint_H3:
-; VI: v_mov_b32 {{v[0-9]+}}, 0xbc00
 define i32 @inline_DA_constraint_H3() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i16 bitcast (half -1.0 to i16))
   ret i32 %v0
@@ -986,7 +978,6 @@ define i32 @inline_DA_constraint_H3() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'DA'
 ; VI-LABEL: {{^}}inline_DA_constraint_H4:
-; VI: v_mov_b32 {{v[0-9]+}}, 0x3118
 define i32 @inline_DA_constraint_H4() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(half 0xH3118)
   ret i32 %v0
@@ -994,7 +985,6 @@ define i32 @inline_DA_constraint_H4() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'DA'
 ; VI-LABEL: {{^}}inline_DA_constraint_H5:
-; VI: v_mov_b32 {{v[0-9]+}}, 0x3118
 define i32 @inline_DA_constraint_H5() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i16 bitcast (half 0xH3118 to i16))
   ret i32 %v0
@@ -1002,7 +992,6 @@ define i32 @inline_DA_constraint_H5() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'DA'
 ; VI-LABEL: {{^}}inline_DA_constraint_H6:
-; VI: v_mov_b32 {{v[0-9]+}}, 0xb800
 define i32 @inline_DA_constraint_H6() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(half -0.5)
   ret i32 %v0
@@ -1164,7 +1153,6 @@ define i32 @inline_DA_constraint_V0() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'DA'
 ; VI-LABEL: {{^}}inline_DA_constraint_V1:
-; VI: v_mov_b32 {{v[0-9]+}}, 0xb800
 define i32 @inline_DA_constraint_V1() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(<2 x half> )
   ret i32 %v0
diff --git a/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll b/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
index 5de9b0b92c9a..1a55bf608ebf 100644
--- a/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
+++ b/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
@@ -3400,9 +3400,9 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_fpone(ptr addrspace(1) %out, p
 ; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX9-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-SDAG-NEXT:    global_load_dword v1, v0, s[2:3]
-; GFX9-SDAG-NEXT:    s_mov_b32 s2, 0xc400c400
+; GFX9-SDAG-NEXT:    s_movk_i32 s2, 0xc400
 ; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT:    v_pk_add_u16 v1, v1, s2
+; GFX9-SDAG-NEXT:    v_pk_add_u16 v1, v1, s2 op_sel_hi:[1,0]
 ; GFX9-SDAG-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX9-SDAG-NEXT:    s_endpgm
 ;
@@ -3418,29 +3418,53 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_fpone(ptr addrspace(1) %out, p
 ; GFX9-GISEL-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX9-GISEL-NEXT:    s_endpgm
 ;
-; GFX10-LABEL: v_test_v2i16_x_add_neg_fpone:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
-; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_pk_add_u16 v1, 0xc400c400, v1
-; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX10-NEXT:    s_endpgm
+; GFX10-SDAG-LABEL: v_test_v2i16_x_add_neg_fpone:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT:    global_load_dword v1, v0, s[2:3]
+; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-SDAG-NEXT:    v_pk_add_u16 v1, 0xffffc400, v1 op_sel_hi:[0,1]
+; GFX10-SDAG-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-SDAG-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: v_test_v2i16_x_add_neg_fpone:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
-; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    global_load_b32 v1, v0, s[2:3]
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_pk_add_u16 v1, 0xc400c400, v1
-; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
-; GFX11-NEXT:    s_nop 0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX11-NEXT:    s_endpgm
+; GFX10-GISEL-LABEL: v_test_v2i16_x_add_neg_fpone:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT:    global_load_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-GISEL-NEXT:    v_pk_add_u16 v1, 0xc400c400, v1
+; GFX10-GISEL-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-GISEL-NEXT:    s_endpgm
+;
+; GFX11-SDAG-LABEL: v_test_v2i16_x_add_neg_fpone:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT:    v_pk_add_u16 v1, 0xffffc400, v1 op_sel_hi:[0,1]
+; GFX11-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-SDAG-NEXT:    s_nop 0
+; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: v_test_v2i16_x_add_neg_fpone:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT:    v_pk_add_u16 v1, 0xc400c400, v1
+; GFX11-GISEL-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-GISEL-NEXT:    s_nop 0
+; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-GISEL-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds <2 x i16>, ptr addrspace(1) %in, i64 %tid.ext
@@ -3541,9 +3565,9 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_negfpone(ptr addrspace(1) %out
 ; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX9-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-SDAG-NEXT:    global_load_dword v1, v0, s[2:3]
-; GFX9-SDAG-NEXT:    s_mov_b32 s2, 0x44004400
+; GFX9-SDAG-NEXT:    s_movk_i32 s2, 0x4400
 ; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT:    v_pk_add_u16 v1, v1, s2
+; GFX9-SDAG-NEXT:    v_pk_add_u16 v1, v1, s2 op_sel_hi:[1,0]
 ; GFX9-SDAG-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX9-SDAG-NEXT:    s_endpgm
 ;
@@ -3559,29 +3583,53 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_negfpone(ptr addrspace(1) %out
 ; GFX9-GISEL-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX9-GISEL-NEXT:    s_endpgm
 ;
-; GFX10-LABEL: v_test_v2i16_x_add_neg_negfpone:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
-; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_pk_add_u16 v1, 0x44004400, v1
-; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX10-NEXT:    s_endpgm
+; GFX10-SDAG-LABEL: v_test_v2i16_x_add_neg_negfpone:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT:    global_load_dword v1, v0, s[2:3]
+; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-SDAG-NEXT:    v_pk_add_u16 v1, 0x4400, v1 op_sel_hi:[0,1]
+; GFX10-SDAG-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-SDAG-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: v_test_v2i16_x_add_neg_negfpone:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
-; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    global_load_b32 v1, v0, s[2:3]
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_pk_add_u16 v1, 0x44004400, v1
-; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
-; GFX11-NEXT:    s_nop 0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX11-NEXT:    s_endpgm
+; GFX10-GISEL-LABEL: v_test_v2i16_x_add_neg_negfpone:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT:    global_load_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-GISEL-NEXT:    v_pk_add_u16 v1, 0x44004400, v1
+; GFX10-GISEL-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-GISEL-NEXT:    s_endpgm
+;
+; GFX11-SDAG-LABEL: v_test_v2i16_x_add_neg_negfpone:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT:    v_pk_add_u16 v1, 0x4400, v1 op_sel_hi:[0,1]
+; GFX11-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-SDAG-NEXT:    s_nop 0
+; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: v_test_v2i16_x_add_neg_negfpone:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT:    v_pk_add_u16 v1, 0x44004400, v1
+; GFX11-GISEL-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-GISEL-NEXT:    s_nop 0
+; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-GISEL-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds <2 x i16>, ptr addrspace(1) %in, i64 %tid.ext
diff --git a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-imm.ll b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-imm.ll
index c4adc8c32128..a3973970d50d 100644
--- a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-imm.ll
+++ b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-imm.ll
@@ -112,7 +112,11 @@ bb:
 define amdgpu_ps void @test_wmma_bf16_16x16x16_bf16_imm(<8 x i16> %A, <8 x i16> %B, ptr addrspace(1) %out) {
 ; GFX12-LABEL: test_wmma_bf16_16x16x16_bf16_imm:
 ; GFX12:       ; %bb.0: ; %bb
-; GFX12-NEXT:    v_wmma_bf16_16x16x16_bf16 v[10:13], v[0:3], v[4:7], 1.0
+; GFX12-NEXT:    v_mov_b32_e32 v10, 0x3f803f80
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_dual_mov_b32 v11, v10 :: v_dual_mov_b32 v12, v10
+; GFX12-NEXT:    v_mov_b32_e32 v13, v10
+; GFX12-NEXT:    v_wmma_bf16_16x16x16_bf16 v[10:13], v[0:3], v[4:7], v[10:13]
 ; GFX12-NEXT:    global_store_b128 v[8:9], v[10:13], off
 ; GFX12-NEXT:    s_nop 0
 ; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
diff --git a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-imm.ll b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-imm.ll
index 462fc01e8e79..829d77018ff9 100644
--- a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-imm.ll
+++ b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-imm.ll
@@ -101,7 +101,10 @@ bb:
 define amdgpu_ps void @test_wmma_bf16_16x16x16_bf16_imm(<4 x i16> %A, <4 x i16> %B, ptr addrspace(1) %out) {
 ; GFX12-LABEL: test_wmma_bf16_16x16x16_bf16_imm:
 ; GFX12:       ; %bb.0: ; %bb
-; GFX12-NEXT:    v_wmma_bf16_16x16x16_bf16 v[6:7], v[0:1], v[2:3], 1.0
+; GFX12-NEXT:    v_mov_b32_e32 v6, 0x3f803f80
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_mov_b32_e32 v7, v6
+; GFX12-NEXT:    v_wmma_bf16_16x16x16_bf16 v[6:7], v[0:1], v[2:3], v[6:7]
 ; GFX12-NEXT:    global_store_b64 v[4:5], v[6:7], off
 ; GFX12-NEXT:    s_nop 0
 ; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
diff --git a/llvm/test/MC/AMDGPU/gfx10_asm_vop1.s b/llvm/test/MC/AMDGPU/gfx10_asm_vop1.s
index 1cfafebe2c3c..5a1673ef1cce 100644
--- a/llvm/test/MC/AMDGPU/gfx10_asm_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx10_asm_vop1.s
@@ -11189,10 +11189,10 @@ v_cvt_f16_u16_e32 v5, -1
 // GFX10: encoding: [0xc1,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16_e32 v5, 0.5
-// GFX10: encoding: [0xff,0xa0,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16_e32 v5, -4.0
-// GFX10: encoding: [0xff,0xa0,0x0a,0x7e,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16_e32 v5, 0xfe0b
 // GFX10: encoding: [0xff,0xa0,0x0a,0x7e,0x0b,0xfe,0x00,0x00]
@@ -11237,10 +11237,10 @@ v_cvt_f16_u16_e64 v5, -1
 // GFX10: encoding: [0x05,0x00,0xd0,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_f16_u16_e64 v5, 0.5
-// GFX10: encoding: [0x05,0x00,0xd0,0xd5,0xff,0x00,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0xd0,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_f16_u16_e64 v5, -4.0
-// GFX10: encoding: [0x05,0x00,0xd0,0xd5,0xff,0x00,0x00,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0xd0,0xd5,0xf7,0x00,0x00,0x00]
 
 v_cvt_f16_u16_e64 v5, v1 clamp
 // GFX10: encoding: [0x05,0x80,0xd0,0xd5,0x01,0x01,0x00,0x00]
@@ -11435,10 +11435,10 @@ v_cvt_f16_i16_e32 v5, -1
 // GFX10: encoding: [0xc1,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16_e32 v5, 0.5
-// GFX10: encoding: [0xff,0xa2,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16_e32 v5, -4.0
-// GFX10: encoding: [0xff,0xa2,0x0a,0x7e,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16_e32 v5, 0xfe0b
 // GFX10: encoding: [0xff,0xa2,0x0a,0x7e,0x0b,0xfe,0x00,0x00]
@@ -11483,10 +11483,10 @@ v_cvt_f16_i16_e64 v5, -1
 // GFX10: encoding: [0x05,0x00,0xd1,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_f16_i16_e64 v5, 0.5
-// GFX10: encoding: [0x05,0x00,0xd1,0xd5,0xff,0x00,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0xd1,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_f16_i16_e64 v5, -4.0
-// GFX10: encoding: [0x05,0x00,0xd1,0xd5,0xff,0x00,0x00,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0xd1,0xd5,0xf7,0x00,0x00,0x00]
 
 v_cvt_f16_i16_e64 v5, v1 clamp
 // GFX10: encoding: [0x05,0x80,0xd1,0xd5,0x01,0x01,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx10_asm_vop3.s b/llvm/test/MC/AMDGPU/gfx10_asm_vop3.s
index 09d833d0ce4b..b4256f1a628d 100644
--- a/llvm/test/MC/AMDGPU/gfx10_asm_vop3.s
+++ b/llvm/test/MC/AMDGPU/gfx10_asm_vop3.s
@@ -8741,10 +8741,10 @@ v_add_nc_u16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x03,0xd7,0xc1,0x04,0x02,0x00]
 
 v_add_nc_u16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x03,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x03,0xd7,0xf0,0x04,0x02,0x00]
 
 v_add_nc_u16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x03,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x03,0xd7,0xf7,0x04,0x02,0x00]
 
 v_add_nc_u16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x03,0xd7,0x01,0xff,0x03,0x00]
@@ -8777,13 +8777,13 @@ v_add_nc_u16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x03,0xd7,0x01,0x83,0x01,0x00]
 
 v_add_nc_u16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x03,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x03,0xd7,0x01,0xe1,0x01,0x00]
 
 v_add_nc_u16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x03,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x03,0xd7,0x01,0xef,0x01,0x00]
 
 v_add_nc_u16 v5, v1, -4.0 clamp
-// GFX10: encoding: [0x05,0x80,0x03,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x80,0x03,0xd7,0x01,0xef,0x01,0x00]
 
 v_add_nc_u16 v5, v1, v2 op_sel:[1,1,1]
 // GFX10: [0x05,0x58,0x03,0xd7,0x01,0x05,0x02,0x00]
@@ -8837,10 +8837,10 @@ v_sub_nc_u16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x04,0xd7,0xc1,0x04,0x02,0x00]
 
 v_sub_nc_u16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x04,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x04,0xd7,0xf0,0x04,0x02,0x00]
 
 v_sub_nc_u16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x04,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x04,0xd7,0xf7,0x04,0x02,0x00]
 
 v_sub_nc_u16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x04,0xd7,0x01,0xff,0x03,0x00]
@@ -8873,13 +8873,13 @@ v_sub_nc_u16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x04,0xd7,0x01,0x83,0x01,0x00]
 
 v_sub_nc_u16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x04,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x04,0xd7,0x01,0xe1,0x01,0x00]
 
 v_sub_nc_u16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x04,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x04,0xd7,0x01,0xef,0x01,0x00]
 
 v_sub_nc_u16 v5, v1, -4.0 clamp
-// GFX10: encoding: [0x05,0x80,0x04,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x80,0x04,0xd7,0x01,0xef,0x01,0x00]
 
 v_sub_nc_u16 v5, v1, v2 op_sel:[1,1,1]
 // GFX10: [0x05,0x58,0x04,0xd7,0x01,0x05,0x02,0x00]
@@ -8933,10 +8933,10 @@ v_mul_lo_u16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x05,0xd7,0xc1,0x04,0x02,0x00]
 
 v_mul_lo_u16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x05,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x05,0xd7,0xf0,0x04,0x02,0x00]
 
 v_mul_lo_u16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x05,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x05,0xd7,0xf7,0x04,0x02,0x00]
 
 v_mul_lo_u16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x05,0xd7,0x01,0xff,0x03,0x00]
@@ -8969,10 +8969,10 @@ v_mul_lo_u16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x05,0xd7,0x01,0x83,0x01,0x00]
 
 v_mul_lo_u16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x05,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x05,0xd7,0x01,0xe1,0x01,0x00]
 
 v_mul_lo_u16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x05,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x05,0xd7,0x01,0xef,0x01,0x00]
 
 v_lshrrev_b16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x07,0xd7,0x01,0x05,0x02,0x00]
@@ -9011,10 +9011,10 @@ v_lshrrev_b16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x07,0xd7,0xc1,0x04,0x02,0x00]
 
 v_lshrrev_b16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x07,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x07,0xd7,0xf0,0x04,0x02,0x00]
 
 v_lshrrev_b16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x07,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x07,0xd7,0xf7,0x04,0x02,0x00]
 
 v_lshrrev_b16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x07,0xd7,0x01,0xff,0x03,0x00]
@@ -9047,10 +9047,10 @@ v_lshrrev_b16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x07,0xd7,0x01,0x83,0x01,0x00]
 
 v_lshrrev_b16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x07,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x07,0xd7,0x01,0xe1,0x01,0x00]
 
 v_lshrrev_b16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x07,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x07,0xd7,0x01,0xef,0x01,0x00]
 
 v_ashrrev_i16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x08,0xd7,0x01,0x05,0x02,0x00]
@@ -9089,10 +9089,10 @@ v_ashrrev_i16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x08,0xd7,0xc1,0x04,0x02,0x00]
 
 v_ashrrev_i16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x08,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x08,0xd7,0xf0,0x04,0x02,0x00]
 
 v_ashrrev_i16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x08,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x08,0xd7,0xf7,0x04,0x02,0x00]
 
 v_ashrrev_i16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x08,0xd7,0x01,0xff,0x03,0x00]
@@ -9125,10 +9125,10 @@ v_ashrrev_i16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x08,0xd7,0x01,0x83,0x01,0x00]
 
 v_ashrrev_i16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x08,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x08,0xd7,0x01,0xe1,0x01,0x00]
 
 v_ashrrev_i16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x08,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x08,0xd7,0x01,0xef,0x01,0x00]
 
 v_max_u16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x09,0xd7,0x01,0x05,0x02,0x00]
@@ -9167,10 +9167,10 @@ v_max_u16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x09,0xd7,0xc1,0x04,0x02,0x00]
 
 v_max_u16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x09,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x09,0xd7,0xf0,0x04,0x02,0x00]
 
 v_max_u16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x09,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x09,0xd7,0xf7,0x04,0x02,0x00]
 
 v_max_u16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x09,0xd7,0x01,0xff,0x03,0x00]
@@ -9203,10 +9203,10 @@ v_max_u16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x09,0xd7,0x01,0x83,0x01,0x00]
 
 v_max_u16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x09,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x09,0xd7,0x01,0xe1,0x01,0x00]
 
 v_max_u16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x09,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x09,0xd7,0x01,0xef,0x01,0x00]
 
 v_max_i16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x0a,0xd7,0x01,0x05,0x02,0x00]
@@ -9245,10 +9245,10 @@ v_max_i16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x0a,0xd7,0xc1,0x04,0x02,0x00]
 
 v_max_i16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x0a,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0a,0xd7,0xf0,0x04,0x02,0x00]
 
 v_max_i16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x0a,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0a,0xd7,0xf7,0x04,0x02,0x00]
 
 v_max_i16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x0a,0xd7,0x01,0xff,0x03,0x00]
@@ -9281,10 +9281,10 @@ v_max_i16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x0a,0xd7,0x01,0x83,0x01,0x00]
 
 v_max_i16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x0a,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0a,0xd7,0x01,0xe1,0x01,0x00]
 
 v_max_i16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x0a,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0a,0xd7,0x01,0xef,0x01,0x00]
 
 v_min_u16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x0b,0xd7,0x01,0x05,0x02,0x00]
@@ -9323,10 +9323,10 @@ v_min_u16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x0b,0xd7,0xc1,0x04,0x02,0x00]
 
 v_min_u16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x0b,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0b,0xd7,0xf0,0x04,0x02,0x00]
 
 v_min_u16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x0b,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0b,0xd7,0xf7,0x04,0x02,0x00]
 
 v_min_u16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x0b,0xd7,0x01,0xff,0x03,0x00]
@@ -9359,10 +9359,10 @@ v_min_u16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x0b,0xd7,0x01,0x83,0x01,0x00]
 
 v_min_u16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x0b,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0b,0xd7,0x01,0xe1,0x01,0x00]
 
 v_min_u16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x0b,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0b,0xd7,0x01,0xef,0x01,0x00]
 
 v_min_i16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x0c,0xd7,0x01,0x05,0x02,0x00]
@@ -9401,10 +9401,10 @@ v_min_i16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x0c,0xd7,0xc1,0x04,0x02,0x00]
 
 v_min_i16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x0c,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0c,0xd7,0xf0,0x04,0x02,0x00]
 
 v_min_i16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x0c,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0c,0xd7,0xf7,0x04,0x02,0x00]
 
 v_min_i16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x0c,0xd7,0x01,0xff,0x03,0x00]
@@ -9437,10 +9437,10 @@ v_min_i16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x0c,0xd7,0x01,0x83,0x01,0x00]
 
 v_min_i16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x0c,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0c,0xd7,0x01,0xe1,0x01,0x00]
 
 v_min_i16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x0c,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0c,0xd7,0x01,0xef,0x01,0x00]
 
 v_add_nc_i16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x0d,0xd7,0x01,0x05,0x02,0x00]
@@ -9479,10 +9479,10 @@ v_add_nc_i16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x0d,0xd7,0xc1,0x04,0x02,0x00]
 
 v_add_nc_i16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x0d,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0d,0xd7,0xf0,0x04,0x02,0x00]
 
 v_add_nc_i16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x0d,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0d,0xd7,0xf7,0x04,0x02,0x00]
 
 v_add_nc_i16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x0d,0xd7,0x01,0xff,0x03,0x00]
@@ -9515,10 +9515,10 @@ v_add_nc_i16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x0d,0xd7,0x01,0x83,0x01,0x00]
 
 v_add_nc_i16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x0d,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0d,0xd7,0x01,0xe1,0x01,0x00]
 
 v_add_nc_i16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x0d,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0d,0xd7,0x01,0xef,0x01,0x00]
 
 v_sub_nc_i16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x0e,0xd7,0x01,0x05,0x02,0x00]
@@ -9557,10 +9557,10 @@ v_sub_nc_i16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x0e,0xd7,0xc1,0x04,0x02,0x00]
 
 v_sub_nc_i16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x0e,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0e,0xd7,0xf0,0x04,0x02,0x00]
 
 v_sub_nc_i16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x0e,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0e,0xd7,0xf7,0x04,0x02,0x00]
 
 v_sub_nc_i16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x0e,0xd7,0x01,0xff,0x03,0x00]
@@ -9593,10 +9593,10 @@ v_sub_nc_i16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x0e,0xd7,0x01,0x83,0x01,0x00]
 
 v_sub_nc_i16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x0e,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0e,0xd7,0x01,0xe1,0x01,0x00]
 
 v_sub_nc_i16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x0e,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0e,0xd7,0x01,0xef,0x01,0x00]
 
 v_pack_b32_f16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x11,0xd7,0x01,0x05,0x02,0x00]
@@ -9968,10 +9968,10 @@ v_lshlrev_b16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x14,0xd7,0xc1,0x04,0x02,0x00]
 
 v_lshlrev_b16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x14,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x14,0xd7,0xf0,0x04,0x02,0x00]
 
 v_lshlrev_b16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x14,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x14,0xd7,0xf7,0x04,0x02,0x00]
 
 v_lshlrev_b16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x14,0xd7,0x01,0xff,0x03,0x00]
@@ -10004,10 +10004,10 @@ v_lshlrev_b16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x14,0xd7,0x01,0x83,0x01,0x00]
 
 v_lshlrev_b16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x14,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x14,0xd7,0x01,0xe1,0x01,0x00]
 
 v_lshlrev_b16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x14,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x14,0xd7,0x01,0xef,0x01,0x00]
 
 v_mad_u16 v5, 0, v2, v3
 // GFX10: encoding: [0x05,0x00,0x40,0xd7,0x80,0x04,0x0e,0x04]
@@ -10016,7 +10016,7 @@ v_mad_u16 v5, v1, -1, v3
 // GFX10: encoding: [0x05,0x00,0x40,0xd7,0x01,0x83,0x0d,0x04]
 
 v_mad_u16 v5, v1, v2, -4.0
-// GFX10: encoding: [0x05,0x00,0x40,0xd7,0x01,0x05,0xfe,0x03,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x40,0xd7,0x01,0x05,0xde,0x03]
 
 v_mad_u16 v5, v1, v2, v3 clamp
 // GFX10: encoding: [0x05,0x80,0x40,0xd7,0x01,0x05,0x0e,0x04]
@@ -10755,10 +10755,10 @@ v_min3_i16 v5, -1, v2, v3
 // GFX10: encoding: [0x05,0x00,0x52,0xd7,0xc1,0x04,0x0e,0x04]
 
 v_min3_i16 v5, 0.5, v2, v3
-// GFX10: encoding: [0x05,0x00,0x52,0xd7,0xff,0x04,0x0e,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x52,0xd7,0xf0,0x04,0x0e,0x04]
 
 v_min3_i16 v5, -4.0, v2, v3
-// GFX10: encoding: [0x05,0x00,0x52,0xd7,0xff,0x04,0x0e,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x52,0xd7,0xf7,0x04,0x0e,0x04]
 
 v_min3_i16 v5, v1, v255, v3
 // GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0xff,0x0f,0x04]
@@ -10791,10 +10791,10 @@ v_min3_i16 v5, v1, -1, v3
 // GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0x83,0x0d,0x04]
 
 v_min3_i16 v5, v1, 0.5, v3
-// GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0xff,0x0d,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0xe1,0x0d,0x04]
 
 v_min3_i16 v5, v1, -4.0, v3
-// GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0xff,0x0d,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0xef,0x0d,0x04]
 
 v_min3_i16 v5, v1, v2, v255
 // GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0x05,0xfe,0x07]
@@ -10827,10 +10827,10 @@ v_min3_i16 v5, v1, v2, -1
 // GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0x05,0x06,0x03]
 
 v_min3_i16 v5, v1, v2, 0.5
-// GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0x05,0xfe,0x03,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0x05,0xc2,0x03]
 
 v_min3_i16 v5, v1, v2, -4.0
-// GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0x05,0xfe,0x03,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0x05,0xde,0x03]
 
 v_min3_i16 v5, v1, v2, v3 op_sel:[0,0,0,0]
 // GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0x05,0x0e,0x04]
@@ -10890,10 +10890,10 @@ v_min3_u16 v5, -1, v2, v3
 // GFX10: encoding: [0x05,0x00,0x53,0xd7,0xc1,0x04,0x0e,0x04]
 
 v_min3_u16 v5, 0.5, v2, v3
-// GFX10: encoding: [0x05,0x00,0x53,0xd7,0xff,0x04,0x0e,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x53,0xd7,0xf0,0x04,0x0e,0x04]
 
 v_min3_u16 v5, -4.0, v2, v3
-// GFX10: encoding: [0x05,0x00,0x53,0xd7,0xff,0x04,0x0e,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x53,0xd7,0xf7,0x04,0x0e,0x04]
 
 v_min3_u16 v5, v1, v255, v3
 // GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0xff,0x0f,0x04]
@@ -10926,10 +10926,10 @@ v_min3_u16 v5, v1, -1, v3
 // GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0x83,0x0d,0x04]
 
 v_min3_u16 v5, v1, 0.5, v3
-// GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0xff,0x0d,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0xe1,0x0d,0x04]
 
 v_min3_u16 v5, v1, -4.0, v3
-// GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0xff,0x0d,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0xef,0x0d,0x04]
 
 v_min3_u16 v5, v1, v2, v255
 // GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0x05,0xfe,0x07]
@@ -10962,10 +10962,10 @@ v_min3_u16 v5, v1, v2, -1
 // GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0x05,0x06,0x03]
 
 v_min3_u16 v5, v1, v2, 0.5
-// GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0x05,0xfe,0x03,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0x05,0xc2,0x03]
 
 v_min3_u16 v5, v1, v2, -4.0
-// GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0x05,0xfe,0x03,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0x05,0xde,0x03]
 
 v_min3_u16 v5, v1, v2, v3 op_sel:[0,0,0,0]
 // GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0x05,0x0e,0x04]
@@ -11184,10 +11184,10 @@ v_max3_i16 v5, -1, v2, v3
 // GFX10: encoding: [0x05,0x00,0x55,0xd7,0xc1,0x04,0x0e,0x04]
 
 v_max3_i16 v5, 0.5, v2, v3
-// GFX10: encoding: [0x05,0x00,0x55,0xd7,0xff,0x04,0x0e,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x55,0xd7,0xf0,0x04,0x0e,0x04]
 
 v_max3_i16 v5, -4.0, v2, v3
-// GFX10: encoding: [0x05,0x00,0x55,0xd7,0xff,0x04,0x0e,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x55,0xd7,0xf7,0x04,0x0e,0x04]
 
 v_max3_i16 v5, v1, v255, v3
 // GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0xff,0x0f,0x04]
@@ -11220,10 +11220,10 @@ v_max3_i16 v5, v1, -1, v3
 // GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0x83,0x0d,0x04]
 
 v_max3_i16 v5, v1, 0.5, v3
-// GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0xff,0x0d,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0xe1,0x0d,0x04]
 
 v_max3_i16 v5, v1, -4.0, v3
-// GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0xff,0x0d,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0xef,0x0d,0x04]
 
 v_max3_i16 v5, v1, v2, v255
 // GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0x05,0xfe,0x07]
@@ -11256,10 +11256,10 @@ v_max3_i16 v5, v1, v2, -1
 // GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0x05,0x06,0x03]
 
 v_max3_i16 v5, v1, v2, 0.5
-// GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0x05,0xfe,0x03,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0x05,0xc2,0x03]
 
 v_max3_i16 v5, v1, v2, -4.0
-// GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0x05,0xfe,0x03,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0x05,0xde,0x03]
 
 v_max3_i16 v5, v1, v2, v3 op_sel:[0,0,0,0]
 // GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0x05,0x0e,0x04]
@@ -11319,10 +11319,10 @@ v_max3_u16 v5, -1, v2, v3
 // GFX10: encoding: [0x05,0x00,0x56,0xd7,0xc1,0x04,0x0e,0x04]
 
 v_max3_u16 v5, 0.5, v2, v3
-// GFX10: encoding: [0x05,0x00,0x56,0xd7,0xff,0x04,0x0e,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x56,0xd7,0xf0,0x04,0x0e,0x04]
 
 v_max3_u16 v5, -4.0, v2, v3
-// GFX10: encoding: [0x05,0x00,0x56,0xd7,0xff,0x04,0x0e,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x56,0xd7,0xf7,0x04,0x0e,0x04]
 
 v_max3_u16 v5, v1, v255, v3
 // GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0xff,0x0f,0x04]
@@ -11355,10 +11355,10 @@ v_max3_u16 v5, v1, -1, v3
 // GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0x83,0x0d,0x04]
 
 v_max3_u16 v5, v1, 0.5, v3
-// GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0xff,0x0d,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0xe1,0x0d,0x04]
 
 v_max3_u16 v5, v1, -4.0, v3
-// GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0xff,0x0d,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0xef,0x0d,0x04]
 
 v_max3_u16 v5, v1, v2, v255
 // GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0x05,0xfe,0x07]
@@ -11391,10 +11391,10 @@ v_max3_u16 v5, v1, v2, -1
 // GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0x05,0x06,0x03]
 
 v_max3_u16 v5, v1, v2, 0.5
-// GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0x05,0xfe,0x03,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0x05,0xc2,0x03]
 
 v_max3_u16 v5, v1, v2, -4.0
-// GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0x05,0xfe,0x03,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0x05,0xde,0x03]
 
 v_max3_u16 v5, v1, v2, v3 op_sel:[0,0,0,0]
 // GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0x05,0x0e,0x04]
@@ -11613,10 +11613,10 @@ v_med3_i16 v5, -1, v2, v3
 // GFX10: encoding: [0x05,0x00,0x58,0xd7,0xc1,0x04,0x0e,0x04]
 
 v_med3_i16 v5, 0.5, v2, v3
-// GFX10: encoding: [0x05,0x00,0x58,0xd7,0xff,0x04,0x0e,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x58,0xd7,0xf0,0x04,0x0e,0x04]
 
 v_med3_i16 v5, -4.0, v2, v3
-// GFX10: encoding: [0x05,0x00,0x58,0xd7,0xff,0x04,0x0e,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x58,0xd7,0xf7,0x04,0x0e,0x04]
 
 v_med3_i16 v5, v1, v255, v3
 // GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0xff,0x0f,0x04]
@@ -11649,10 +11649,10 @@ v_med3_i16 v5, v1, -1, v3
 // GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0x83,0x0d,0x04]
 
 v_med3_i16 v5, v1, 0.5, v3
-// GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0xff,0x0d,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0xe1,0x0d,0x04]
 
 v_med3_i16 v5, v1, -4.0, v3
-// GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0xff,0x0d,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0xef,0x0d,0x04]
 
 v_med3_i16 v5, v1, v2, v255
 // GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0x05,0xfe,0x07]
@@ -11685,10 +11685,10 @@ v_med3_i16 v5, v1, v2, -1
 // GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0x05,0x06,0x03]
 
 v_med3_i16 v5, v1, v2, 0.5
-// GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0x05,0xfe,0x03,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0x05,0xc2,0x03]
 
 v_med3_i16 v5, v1, v2, -4.0
-// GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0x05,0xfe,0x03,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0x05,0xde,0x03]
 
 v_med3_i16 v5, v1, v2, v3 op_sel:[0,0,0,0]
 // GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0x05,0x0e,0x04]
@@ -11748,10 +11748,10 @@ v_med3_u16 v5, -1, v2, v3
 // GFX10: encoding: [0x05,0x00,0x59,0xd7,0xc1,0x04,0x0e,0x04]
 
 v_med3_u16 v5, 0.5, v2, v3
-// GFX10: encoding: [0x05,0x00,0x59,0xd7,0xff,0x04,0x0e,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x59,0xd7,0xf0,0x04,0x0e,0x04]
 
 v_med3_u16 v5, -4.0, v2, v3
-// GFX10: encoding: [0x05,0x00,0x59,0xd7,0xff,0x04,0x0e,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x59,0xd7,0xf7,0x04,0x0e,0x04]
 
 v_med3_u16 v5, v1, v255, v3
 // GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0xff,0x0f,0x04]
@@ -11784,10 +11784,10 @@ v_med3_u16 v5, v1, -1, v3
 // GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0x83,0x0d,0x04]
 
 v_med3_u16 v5, v1, 0.5, v3
-// GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0xff,0x0d,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0xe1,0x0d,0x04]
 
 v_med3_u16 v5, v1, -4.0, v3
-// GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0xff,0x0d,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0xef,0x0d,0x04]
 
 v_med3_u16 v5, v1, v2, v255
 // GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0x05,0xfe,0x07]
@@ -11820,10 +11820,10 @@ v_med3_u16 v5, v1, v2, -1
 // GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0x05,0x06,0x03]
 
 v_med3_u16 v5, v1, v2, 0.5
-// GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0x05,0xfe,0x03,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0x05,0xc2,0x03]
 
 v_med3_u16 v5, v1, v2, -4.0
-// GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0x05,0xfe,0x03,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0x05,0xde,0x03]
 
 v_med3_u16 v5, v1, v2, v3 op_sel:[0,0,0,0]
 // GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0x05,0x0e,0x04]
@@ -11868,7 +11868,7 @@ v_mad_i16 v5, v1, -1, v3
 // GFX10: encoding: [0x05,0x00,0x5e,0xd7,0x01,0x83,0x0d,0x04]
 
 v_mad_i16 v5, v1, v2, -4.0
-// GFX10: encoding: [0x05,0x00,0x5e,0xd7,0x01,0x05,0xfe,0x03,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x5e,0xd7,0x01,0x05,0xde,0x03]
 
 v_mad_i16 v5, v1, v2, v3 clamp
 // GFX10: encoding: [0x05,0x80,0x5e,0xd7,0x01,0x05,0x0e,0x04]
@@ -12399,10 +12399,10 @@ v_mad_u32_u16 v5, -1, v2, v3
 // GFX10: encoding: [0x05,0x00,0x73,0xd7,0xc1,0x04,0x0e,0x04]
 
 v_mad_u32_u16 v5, 0.5, v2, v3
-// GFX10: encoding: [0x05,0x00,0x73,0xd7,0xff,0x04,0x0e,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x73,0xd7,0xf0,0x04,0x0e,0x04]
 
 v_mad_u32_u16 v5, -4.0, v2, v3
-// GFX10: encoding: [0x05,0x00,0x73,0xd7,0xff,0x04,0x0e,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x73,0xd7,0xf7,0x04,0x0e,0x04]
 
 v_mad_u32_u16 v5, v1, v255, v3
 // GFX10: encoding: [0x05,0x00,0x73,0xd7,0x01,0xff,0x0f,0x04]
@@ -12435,10 +12435,10 @@ v_mad_u32_u16 v5, v1, -1, v3
 // GFX10: encoding: [0x05,0x00,0x73,0xd7,0x01,0x83,0x0d,0x04]
 
 v_mad_u32_u16 v5, v1, 0.5, v3
-// GFX10: encoding: [0x05,0x00,0x73,0xd7,0x01,0xff,0x0d,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x73,0xd7,0x01,0xe1,0x0d,0x04]
 
 v_mad_u32_u16 v5, v1, -4.0, v3
-// GFX10: encoding: [0x05,0x00,0x73,0xd7,0x01,0xff,0x0d,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x73,0xd7,0x01,0xef,0x0d,0x04]
 
 v_mad_u32_u16 v5, v1, v2, v255
 // GFX10: encoding: [0x05,0x00,0x73,0xd7,0x01,0x05,0xfe,0x07]
@@ -12534,10 +12534,10 @@ v_mad_i32_i16 v5, -1, v2, v3
 // GFX10: encoding: [0x05,0x00,0x75,0xd7,0xc1,0x04,0x0e,0x04]
 
 v_mad_i32_i16 v5, 0.5, v2, v3
-// GFX10: encoding: [0x05,0x00,0x75,0xd7,0xff,0x04,0x0e,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x75,0xd7,0xf0,0x04,0x0e,0x04]
 
 v_mad_i32_i16 v5, -4.0, v2, v3
-// GFX10: encoding: [0x05,0x00,0x75,0xd7,0xff,0x04,0x0e,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x75,0xd7,0xf7,0x04,0x0e,0x04]
 
 v_mad_i32_i16 v5, v1, v255, v3
 // GFX10: encoding: [0x05,0x00,0x75,0xd7,0x01,0xff,0x0f,0x04]
@@ -12570,10 +12570,10 @@ v_mad_i32_i16 v5, v1, -1, v3
 // GFX10: encoding: [0x05,0x00,0x75,0xd7,0x01,0x83,0x0d,0x04]
 
 v_mad_i32_i16 v5, v1, 0.5, v3
-// GFX10: encoding: [0x05,0x00,0x75,0xd7,0x01,0xff,0x0d,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x75,0xd7,0x01,0xe1,0x0d,0x04]
 
 v_mad_i32_i16 v5, v1, -4.0, v3
-// GFX10: encoding: [0x05,0x00,0x75,0xd7,0x01,0xff,0x0d,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x75,0xd7,0x01,0xef,0x0d,0x04]
 
 v_mad_i32_i16 v5, v1, v2, v255
 // GFX10: encoding: [0x05,0x00,0x75,0xd7,0x01,0x05,0xfe,0x07]
diff --git a/llvm/test/MC/AMDGPU/gfx10_asm_vopc.s b/llvm/test/MC/AMDGPU/gfx10_asm_vopc.s
index 521394829069..40618d1ea523 100644
--- a/llvm/test/MC/AMDGPU/gfx10_asm_vopc.s
+++ b/llvm/test/MC/AMDGPU/gfx10_asm_vopc.s
@@ -5044,11 +5044,11 @@ v_cmp_lt_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x12,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x12,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x12,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x12,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc, 0xfe0b, v2
@@ -5108,11 +5108,11 @@ v_cmp_eq_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x14,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x14,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x14,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x14,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc, 0xfe0b, v2
@@ -5172,11 +5172,11 @@ v_cmp_le_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x16,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x16,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x16,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x16,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc, 0xfe0b, v2
@@ -5236,11 +5236,11 @@ v_cmp_gt_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x18,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x18,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x18,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x18,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc, 0xfe0b, v2
@@ -5300,11 +5300,11 @@ v_cmp_ne_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x1a,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x1a,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x1a,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x1a,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc, 0xfe0b, v2
@@ -5364,11 +5364,11 @@ v_cmp_ge_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x1c,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x1c,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x1c,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x1c,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc, 0xfe0b, v2
@@ -5428,11 +5428,11 @@ v_cmp_lt_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x12,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x12,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x12,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x12,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc_lo, 0xfe0b, v2
@@ -5492,11 +5492,11 @@ v_cmp_eq_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x14,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x14,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x14,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x14,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc_lo, 0xfe0b, v2
@@ -5556,11 +5556,11 @@ v_cmp_le_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x16,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x16,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x16,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x16,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc_lo, 0xfe0b, v2
@@ -5620,11 +5620,11 @@ v_cmp_gt_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x18,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x18,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x18,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x18,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc_lo, 0xfe0b, v2
@@ -5684,11 +5684,11 @@ v_cmp_ne_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x1a,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x1a,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x1a,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x1a,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc_lo, 0xfe0b, v2
@@ -5748,11 +5748,11 @@ v_cmp_ge_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x1c,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x1c,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x1c,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x1c,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc_lo, 0xfe0b, v2
@@ -7780,11 +7780,11 @@ v_cmp_lt_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x52,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x52,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x52,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x52,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc, 0xfe0b, v2
@@ -7844,11 +7844,11 @@ v_cmp_eq_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x54,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x54,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x54,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x54,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc, 0xfe0b, v2
@@ -7908,11 +7908,11 @@ v_cmp_le_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x56,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x56,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x56,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x56,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc, 0xfe0b, v2
@@ -7972,11 +7972,11 @@ v_cmp_gt_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x58,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x58,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x58,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x58,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc, 0xfe0b, v2
@@ -8036,11 +8036,11 @@ v_cmp_ne_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x5a,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x5a,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x5a,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x5a,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc, 0xfe0b, v2
@@ -8100,11 +8100,11 @@ v_cmp_ge_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x5c,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x5c,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x5c,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x5c,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc, 0xfe0b, v2
@@ -9700,11 +9700,11 @@ v_cmp_lt_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x52,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x52,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x52,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x52,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc_lo, 0xfe0b, v2
@@ -9764,11 +9764,11 @@ v_cmp_eq_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x54,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x54,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x54,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x54,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc_lo, 0xfe0b, v2
@@ -9828,11 +9828,11 @@ v_cmp_le_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x56,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x56,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x56,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x56,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc_lo, 0xfe0b, v2
@@ -9892,11 +9892,11 @@ v_cmp_gt_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x58,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x58,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x58,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x58,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc_lo, 0xfe0b, v2
@@ -9956,11 +9956,11 @@ v_cmp_ne_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x5a,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x5a,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x5a,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x5a,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc_lo, 0xfe0b, v2
@@ -10020,11 +10020,11 @@ v_cmp_ge_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x5c,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x5c,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x5c,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x5c,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc_lo, 0xfe0b, v2
diff --git a/llvm/test/MC/AMDGPU/gfx10_asm_vopc_e64.s b/llvm/test/MC/AMDGPU/gfx10_asm_vopc_e64.s
index fc8fe95a9cf2..669e79cdab10 100644
--- a/llvm/test/MC/AMDGPU/gfx10_asm_vopc_e64.s
+++ b/llvm/test/MC/AMDGPU/gfx10_asm_vopc_e64.s
@@ -9768,11 +9768,11 @@ v_cmp_lt_i16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0x89,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x89,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0x89,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x89,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s[10:11], v1, v255
@@ -9816,11 +9816,11 @@ v_cmp_lt_i16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0x89,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x89,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0x89,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x89,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s[10:11], v1, v2
@@ -9880,11 +9880,11 @@ v_cmp_eq_i16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0x8a,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8a,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0x8a,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8a,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s[10:11], v1, v255
@@ -9928,11 +9928,11 @@ v_cmp_eq_i16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0x8a,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8a,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0x8a,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8a,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s[10:11], v1, v2
@@ -9992,11 +9992,11 @@ v_cmp_le_i16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0x8b,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8b,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0x8b,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8b,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s[10:11], v1, v255
@@ -10040,11 +10040,11 @@ v_cmp_le_i16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0x8b,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8b,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0x8b,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8b,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s[10:11], v1, v2
@@ -10104,11 +10104,11 @@ v_cmp_gt_i16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0x8c,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8c,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0x8c,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8c,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s[10:11], v1, v255
@@ -10152,11 +10152,11 @@ v_cmp_gt_i16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0x8c,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8c,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0x8c,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8c,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s[10:11], v1, v2
@@ -10216,11 +10216,11 @@ v_cmp_ne_i16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0x8d,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8d,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0x8d,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8d,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s[10:11], v1, v255
@@ -10264,11 +10264,11 @@ v_cmp_ne_i16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0x8d,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8d,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0x8d,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8d,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s[10:11], v1, v2
@@ -10328,11 +10328,11 @@ v_cmp_ge_i16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0x8e,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8e,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0x8e,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8e,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s[10:11], v1, v255
@@ -10376,11 +10376,11 @@ v_cmp_ge_i16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0x8e,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8e,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0x8e,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8e,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s10, v1, v2
@@ -10440,11 +10440,11 @@ v_cmp_lt_i16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0x89,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x89,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0x89,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x89,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s10, v1, v255
@@ -10488,11 +10488,11 @@ v_cmp_lt_i16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0x89,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x89,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0x89,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x89,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s10, v1, v2
@@ -10552,11 +10552,11 @@ v_cmp_eq_i16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0x8a,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8a,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0x8a,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8a,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s10, v1, v255
@@ -10600,11 +10600,11 @@ v_cmp_eq_i16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0x8a,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8a,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0x8a,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8a,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s10, v1, v2
@@ -10664,11 +10664,11 @@ v_cmp_le_i16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0x8b,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8b,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0x8b,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8b,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s10, v1, v255
@@ -10712,11 +10712,11 @@ v_cmp_le_i16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0x8b,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8b,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0x8b,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8b,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s10, v1, v2
@@ -10776,11 +10776,11 @@ v_cmp_gt_i16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0x8c,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8c,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0x8c,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8c,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s10, v1, v255
@@ -10824,11 +10824,11 @@ v_cmp_gt_i16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0x8c,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8c,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0x8c,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8c,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s10, v1, v2
@@ -10888,11 +10888,11 @@ v_cmp_ne_i16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0x8d,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8d,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0x8d,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8d,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s10, v1, v255
@@ -10936,11 +10936,11 @@ v_cmp_ne_i16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0x8d,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8d,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0x8d,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8d,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s10, v1, v2
@@ -11000,11 +11000,11 @@ v_cmp_ge_i16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0x8e,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8e,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0x8e,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8e,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s10, v1, v255
@@ -11048,11 +11048,11 @@ v_cmp_ge_i16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0x8e,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8e,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0x8e,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8e,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_class_f16_e64 s[10:11], v1, v2
@@ -14516,11 +14516,11 @@ v_cmp_lt_u16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0xa9,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xa9,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0xa9,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xa9,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s[10:11], v1, v255
@@ -14564,11 +14564,11 @@ v_cmp_lt_u16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0xa9,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xa9,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0xa9,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xa9,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s[10:11], v1, v2
@@ -14628,11 +14628,11 @@ v_cmp_eq_u16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0xaa,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xaa,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0xaa,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xaa,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s[10:11], v1, v255
@@ -14676,11 +14676,11 @@ v_cmp_eq_u16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0xaa,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xaa,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0xaa,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xaa,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s[10:11], v1, v2
@@ -14740,11 +14740,11 @@ v_cmp_le_u16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0xab,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xab,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0xab,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xab,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s[10:11], v1, v255
@@ -14788,11 +14788,11 @@ v_cmp_le_u16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0xab,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xab,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0xab,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xab,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s[10:11], v1, v2
@@ -14852,11 +14852,11 @@ v_cmp_gt_u16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0xac,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xac,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0xac,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xac,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s[10:11], v1, v255
@@ -14900,11 +14900,11 @@ v_cmp_gt_u16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0xac,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xac,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0xac,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xac,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s[10:11], v1, v2
@@ -14964,11 +14964,11 @@ v_cmp_ne_u16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0xad,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xad,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0xad,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xad,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s[10:11], v1, v255
@@ -15012,11 +15012,11 @@ v_cmp_ne_u16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0xad,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xad,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0xad,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xad,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s[10:11], v1, v2
@@ -15076,11 +15076,11 @@ v_cmp_ge_u16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0xae,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xae,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0xae,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xae,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s[10:11], v1, v255
@@ -15124,11 +15124,11 @@ v_cmp_ge_u16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0xae,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xae,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0xae,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xae,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_f_u32_e64 s[10:11], v1, v2
@@ -18132,11 +18132,11 @@ v_cmp_lt_u16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0xa9,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xa9,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0xa9,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xa9,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s10, v1, v255
@@ -18180,11 +18180,11 @@ v_cmp_lt_u16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0xa9,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xa9,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0xa9,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xa9,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s10, v1, v2
@@ -18244,11 +18244,11 @@ v_cmp_eq_u16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0xaa,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xaa,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0xaa,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xaa,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s10, v1, v255
@@ -18292,11 +18292,11 @@ v_cmp_eq_u16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0xaa,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xaa,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0xaa,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xaa,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s10, v1, v2
@@ -18356,11 +18356,11 @@ v_cmp_le_u16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0xab,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xab,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0xab,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xab,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s10, v1, v255
@@ -18404,11 +18404,11 @@ v_cmp_le_u16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0xab,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xab,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0xab,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xab,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s10, v1, v2
@@ -18468,11 +18468,11 @@ v_cmp_gt_u16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0xac,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xac,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0xac,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xac,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s10, v1, v255
@@ -18516,11 +18516,11 @@ v_cmp_gt_u16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0xac,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xac,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0xac,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xac,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s10, v1, v2
@@ -18580,11 +18580,11 @@ v_cmp_ne_u16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0xad,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xad,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0xad,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xad,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s10, v1, v255
@@ -18628,11 +18628,11 @@ v_cmp_ne_u16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0xad,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xad,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0xad,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xad,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s10, v1, v2
@@ -18692,11 +18692,11 @@ v_cmp_ge_u16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0xae,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xae,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0xae,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xae,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s10, v1, v255
@@ -18740,11 +18740,11 @@ v_cmp_ge_u16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0xae,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xae,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0xae,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xae,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_f_u32_e64 s10, v1, v2
diff --git a/llvm/test/MC/AMDGPU/gfx10_asm_vopcx.s b/llvm/test/MC/AMDGPU/gfx10_asm_vopcx.s
index f441feb88db4..eed284e56bbe 100644
--- a/llvm/test/MC/AMDGPU/gfx10_asm_vopcx.s
+++ b/llvm/test/MC/AMDGPU/gfx10_asm_vopcx.s
@@ -6330,10 +6330,10 @@ v_cmpx_lt_i16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x32,0x7d]
 
 v_cmpx_lt_i16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x32,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x32,0x7d]
 
 v_cmpx_lt_i16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x32,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x32,0x7d]
 
 v_cmpx_lt_i16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x32,0x7d,0x0b,0xfe,0x00,0x00]
@@ -6378,10 +6378,10 @@ v_cmpx_lt_i16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0x99,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_lt_i16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0x99,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x99,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_lt_i16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0x99,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x99,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_lt_i16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0x99,0xd4,0x01,0xff,0x03,0x00]
@@ -6414,10 +6414,10 @@ v_cmpx_lt_i16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0x99,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_lt_i16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0x99,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x99,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_lt_i16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0x99,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x99,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_eq_i16 v1, v2
 // GFX10: encoding: [0x01,0x05,0x34,0x7d]
@@ -6453,10 +6453,10 @@ v_cmpx_eq_i16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x34,0x7d]
 
 v_cmpx_eq_i16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x34,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x34,0x7d]
 
 v_cmpx_eq_i16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x34,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x34,0x7d]
 
 v_cmpx_eq_i16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x34,0x7d,0x0b,0xfe,0x00,0x00]
@@ -6501,10 +6501,10 @@ v_cmpx_eq_i16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0x9a,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_eq_i16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0x9a,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9a,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_eq_i16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0x9a,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9a,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_eq_i16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0x9a,0xd4,0x01,0xff,0x03,0x00]
@@ -6537,10 +6537,10 @@ v_cmpx_eq_i16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0x9a,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_eq_i16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0x9a,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9a,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_eq_i16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0x9a,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9a,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_le_i16 v1, v2
 // GFX10: encoding: [0x01,0x05,0x36,0x7d]
@@ -6576,10 +6576,10 @@ v_cmpx_le_i16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x36,0x7d]
 
 v_cmpx_le_i16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x36,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x36,0x7d]
 
 v_cmpx_le_i16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x36,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x36,0x7d]
 
 v_cmpx_le_i16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x36,0x7d,0x0b,0xfe,0x00,0x00]
@@ -6624,10 +6624,10 @@ v_cmpx_le_i16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0x9b,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_le_i16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0x9b,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9b,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_le_i16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0x9b,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9b,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_le_i16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0x9b,0xd4,0x01,0xff,0x03,0x00]
@@ -6660,10 +6660,10 @@ v_cmpx_le_i16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0x9b,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_le_i16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0x9b,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9b,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_le_i16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0x9b,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9b,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_gt_i16 v1, v2
 // GFX10: encoding: [0x01,0x05,0x38,0x7d]
@@ -6699,10 +6699,10 @@ v_cmpx_gt_i16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x38,0x7d]
 
 v_cmpx_gt_i16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x38,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x38,0x7d]
 
 v_cmpx_gt_i16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x38,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x38,0x7d]
 
 v_cmpx_gt_i16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x38,0x7d,0x0b,0xfe,0x00,0x00]
@@ -6747,10 +6747,10 @@ v_cmpx_gt_i16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0x9c,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_gt_i16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0x9c,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9c,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_gt_i16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0x9c,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9c,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_gt_i16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0x9c,0xd4,0x01,0xff,0x03,0x00]
@@ -6783,10 +6783,10 @@ v_cmpx_gt_i16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0x9c,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_gt_i16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0x9c,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9c,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_gt_i16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0x9c,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9c,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_ne_i16 v1, v2
 // GFX10: encoding: [0x01,0x05,0x3a,0x7d]
@@ -6822,10 +6822,10 @@ v_cmpx_ne_i16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x3a,0x7d]
 
 v_cmpx_ne_i16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x3a,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x3a,0x7d]
 
 v_cmpx_ne_i16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x3a,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x3a,0x7d]
 
 v_cmpx_ne_i16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x3a,0x7d,0x0b,0xfe,0x00,0x00]
@@ -6870,10 +6870,10 @@ v_cmpx_ne_i16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0x9d,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_ne_i16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0x9d,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9d,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_ne_i16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0x9d,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9d,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_ne_i16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0x9d,0xd4,0x01,0xff,0x03,0x00]
@@ -6906,10 +6906,10 @@ v_cmpx_ne_i16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0x9d,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_ne_i16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0x9d,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9d,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_ne_i16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0x9d,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9d,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_ge_i16 v1, v2
 // GFX10: encoding: [0x01,0x05,0x3c,0x7d]
@@ -6945,10 +6945,10 @@ v_cmpx_ge_i16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x3c,0x7d]
 
 v_cmpx_ge_i16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x3c,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x3c,0x7d]
 
 v_cmpx_ge_i16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x3c,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x3c,0x7d]
 
 v_cmpx_ge_i16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x3c,0x7d,0x0b,0xfe,0x00,0x00]
@@ -6993,10 +6993,10 @@ v_cmpx_ge_i16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0x9e,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_ge_i16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0x9e,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9e,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_ge_i16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0x9e,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9e,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_ge_i16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0x9e,0xd4,0x01,0xff,0x03,0x00]
@@ -7029,10 +7029,10 @@ v_cmpx_ge_i16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0x9e,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_ge_i16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0x9e,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9e,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_ge_i16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0x9e,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9e,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_class_f16 v1, v2
 // GFX10: encoding: [0x01,0x05,0x3e,0x7d]
@@ -8094,10 +8094,10 @@ v_cmpx_lt_u16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x72,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x72,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x72,0x7d,0x0b,0xfe,0x00,0x00]
@@ -8142,10 +8142,10 @@ v_cmpx_lt_u16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0xb9,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_lt_u16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0xb9,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xb9,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_lt_u16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0xb9,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xb9,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_lt_u16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0xb9,0xd4,0x01,0xff,0x03,0x00]
@@ -8178,10 +8178,10 @@ v_cmpx_lt_u16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0xb9,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_lt_u16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0xb9,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xb9,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_lt_u16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0xb9,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xb9,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_eq_u16 v1, v2
 // GFX10: encoding: [0x01,0x05,0x74,0x7d]
@@ -8217,10 +8217,10 @@ v_cmpx_eq_u16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x74,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x74,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x74,0x7d,0x0b,0xfe,0x00,0x00]
@@ -8265,10 +8265,10 @@ v_cmpx_eq_u16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0xba,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_eq_u16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0xba,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xba,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_eq_u16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0xba,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xba,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_eq_u16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0xba,0xd4,0x01,0xff,0x03,0x00]
@@ -8301,10 +8301,10 @@ v_cmpx_eq_u16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0xba,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_eq_u16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0xba,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xba,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_eq_u16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0xba,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xba,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_le_u16 v1, v2
 // GFX10: encoding: [0x01,0x05,0x76,0x7d]
@@ -8340,10 +8340,10 @@ v_cmpx_le_u16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x76,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x76,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x76,0x7d,0x0b,0xfe,0x00,0x00]
@@ -8388,10 +8388,10 @@ v_cmpx_le_u16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0xbb,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_le_u16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0xbb,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbb,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_le_u16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0xbb,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbb,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_le_u16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0xbb,0xd4,0x01,0xff,0x03,0x00]
@@ -8424,10 +8424,10 @@ v_cmpx_le_u16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0xbb,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_le_u16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0xbb,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbb,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_le_u16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0xbb,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbb,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_gt_u16 v1, v2
 // GFX10: encoding: [0x01,0x05,0x78,0x7d]
@@ -8463,10 +8463,10 @@ v_cmpx_gt_u16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x78,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x78,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x78,0x7d,0x0b,0xfe,0x00,0x00]
@@ -8511,10 +8511,10 @@ v_cmpx_gt_u16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0xbc,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_gt_u16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0xbc,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbc,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_gt_u16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0xbc,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbc,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_gt_u16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0xbc,0xd4,0x01,0xff,0x03,0x00]
@@ -8547,10 +8547,10 @@ v_cmpx_gt_u16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0xbc,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_gt_u16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0xbc,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbc,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_gt_u16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0xbc,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbc,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_ne_u16 v1, v2
 // GFX10: encoding: [0x01,0x05,0x7a,0x7d]
@@ -8586,10 +8586,10 @@ v_cmpx_ne_u16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x7a,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x7a,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x7a,0x7d,0x0b,0xfe,0x00,0x00]
@@ -8634,10 +8634,10 @@ v_cmpx_ne_u16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0xbd,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_ne_u16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0xbd,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbd,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_ne_u16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0xbd,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbd,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_ne_u16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0xbd,0xd4,0x01,0xff,0x03,0x00]
@@ -8670,10 +8670,10 @@ v_cmpx_ne_u16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0xbd,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_ne_u16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0xbd,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbd,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_ne_u16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0xbd,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbd,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_ge_u16 v1, v2
 // GFX10: encoding: [0x01,0x05,0x7c,0x7d]
@@ -8709,10 +8709,10 @@ v_cmpx_ge_u16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x7c,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x7c,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x7c,0x7d,0x0b,0xfe,0x00,0x00]
@@ -8757,10 +8757,10 @@ v_cmpx_ge_u16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0xbe,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_ge_u16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0xbe,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbe,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_ge_u16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0xbe,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbe,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_ge_u16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0xbe,0xd4,0x01,0xff,0x03,0x00]
@@ -8793,10 +8793,10 @@ v_cmpx_ge_u16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0xbe,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_ge_u16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0xbe,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbe,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_ge_u16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0xbe,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbe,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_f_u32 v1, v2
 // GFX10: encoding: [0x01,0x05,0xa0,0x7d]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop1.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop1.s
index 5990f7cb4570..993bfedb1b64 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop1.s
@@ -485,7 +485,7 @@ v_cvt_f16_i16 v5, -1
 // GFX11: encoding: [0xc1,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, 0.5
-// GFX11: encoding: [0xff,0xa2,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, src_scc
 // GFX11: encoding: [0xfd,0xa2,0x0a,0x7e]
@@ -530,7 +530,7 @@ v_cvt_f16_u16 v5, -1
 // GFX11: encoding: [0xc1,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, 0.5
-// GFX11: encoding: [0xff,0xa0,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, src_scc
 // GFX11: encoding: [0xfd,0xa0,0x0a,0x7e]
@@ -1277,7 +1277,7 @@ v_cvt_i32_i16 v5, -1
 // GFX11: encoding: [0xc1,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, 0.5
-// GFX11: encoding: [0xff,0xd4,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, src_scc
 // GFX11: encoding: [0xfd,0xd4,0x0a,0x7e]
@@ -1673,7 +1673,7 @@ v_cvt_u32_u16 v5, -1
 // GFX11: encoding: [0xc1,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, 0.5
-// GFX11: encoding: [0xff,0xd6,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, src_scc
 // GFX11: encoding: [0xfd,0xd6,0x0a,0x7e]
@@ -2666,7 +2666,7 @@ v_not_b16 v5, -1
 // GFX11: encoding: [0xc1,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, 0.5
-// GFX11: encoding: [0xff,0xd2,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, src_scc
 // GFX11: encoding: [0xfd,0xd2,0x0a,0x7e]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3.s
index d288c02a22c9..c0ae7ecbdbdd 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3.s
@@ -266,7 +266,7 @@ v_add_nc_i16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x0d,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_add_nc_i16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x0d,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x0d,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_add_nc_i16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x0d,0xd7,0x7e,0x82,0x01,0x00]
@@ -281,7 +281,7 @@ v_add_nc_i16 v5, -1, exec_hi op_sel:[0,0,0]
 // GFX11: encoding: [0x05,0x00,0x0d,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_add_nc_i16 v5, 0.5, m0 op_sel:[1,0,0]
-// GFX11: encoding: [0x05,0x08,0x0d,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x08,0x0d,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_add_nc_i16 v5, src_scc, vcc_lo op_sel:[0,1,0]
 // GFX11: encoding: [0x05,0x10,0x0d,0xd7,0xfd,0xd4,0x00,0x00]
@@ -356,7 +356,7 @@ v_add_nc_u16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x03,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_add_nc_u16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x03,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x03,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_add_nc_u16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x03,0xd7,0x7e,0x82,0x01,0x00]
@@ -371,7 +371,7 @@ v_add_nc_u16 v5, -1, exec_hi op_sel:[0,0,0]
 // GFX11: encoding: [0x05,0x00,0x03,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_add_nc_u16 v5, 0.5, m0 op_sel:[1,0,0]
-// GFX11: encoding: [0x05,0x08,0x03,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x08,0x03,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_add_nc_u16 v5, src_scc, vcc_lo op_sel:[0,1,0]
 // GFX11: encoding: [0x05,0x10,0x03,0xd7,0xfd,0xd4,0x00,0x00]
@@ -491,7 +491,7 @@ v_and_b16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x62,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_and_b16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x62,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x62,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_and_b16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x62,0xd7,0x7e,0x82,0x01,0x00]
@@ -506,7 +506,7 @@ v_and_b16 v5, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x62,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_and_b16 v5, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x62,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x62,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_and_b16 v5, src_scc, vcc_lo
 // GFX11: encoding: [0x05,0x00,0x62,0xd7,0xfd,0xd4,0x00,0x00]
@@ -581,7 +581,7 @@ v_ashrrev_i16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x3a,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_ashrrev_i16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x3a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x3a,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_ashrrev_i16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x3a,0xd7,0x7e,0x82,0x01,0x00]
@@ -596,7 +596,7 @@ v_ashrrev_i16 v5, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x3a,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_ashrrev_i16 v5, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x3a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x3a,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_ashrrev_i16 v5, src_scc, vcc_lo
 // GFX11: encoding: [0x05,0x00,0x3a,0xd7,0xfd,0xd4,0x00,0x00]
@@ -861,7 +861,7 @@ v_cndmask_b16 v5, v1, src_scc, s3
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, v255, 0.5, s3
-// W32: encoding: [0x05,0x00,0x5d,0xd6,0xff,0xff,0x0d,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x5d,0xd6,0xff,0xe1,0x0d,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, s105, s105, s3
@@ -897,7 +897,7 @@ v_cndmask_b16 v5, -1, -|vcc_lo|, vcc_lo
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, 0.5, -1, vcc_hi
-// W32: encoding: [0x05,0x00,0x5d,0xd6,0xff,0x82,0xad,0x01,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x5d,0xd6,0xf0,0x82,0xad,0x01]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, -|src_scc|, null, ttmp15
@@ -909,7 +909,7 @@ v_cndmask_b16 v5, v1, src_scc, s[6:7]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, v255, 0.5, s[6:7]
-// W64: encoding: [0x05,0x00,0x5d,0xd6,0xff,0xff,0x19,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x05,0x00,0x5d,0xd6,0xff,0xe1,0x19,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, s105, s105, s[6:7]
@@ -945,7 +945,7 @@ v_cndmask_b16 v5, -1, -|vcc_lo|, s[104:105]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, 0.5, -1, vcc
-// W64: encoding: [0x05,0x00,0x5d,0xd6,0xff,0x82,0xa9,0x01,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x05,0x00,0x5d,0xd6,0xf0,0x82,0xa9,0x01]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, -|src_scc|, null, ttmp[14:15]
@@ -2639,7 +2639,7 @@ v_lshlrev_b16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x38,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_lshlrev_b16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x38,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x38,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_lshlrev_b16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x38,0xd7,0x7e,0x82,0x01,0x00]
@@ -2654,7 +2654,7 @@ v_lshlrev_b16 v5, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x38,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_lshlrev_b16 v5, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x38,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x38,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_lshlrev_b16 v5, src_scc, vcc_lo
 // GFX11: encoding: [0x05,0x00,0x38,0xd7,0xfd,0xd4,0x00,0x00]
@@ -2711,7 +2711,7 @@ v_lshrrev_b16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x39,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_lshrrev_b16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x39,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x39,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_lshrrev_b16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x39,0xd7,0x7e,0x82,0x01,0x00]
@@ -2726,7 +2726,7 @@ v_lshrrev_b16 v5, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x39,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_lshrrev_b16 v5, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x39,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x39,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_lshrrev_b16 v5, src_scc, vcc_lo
 // GFX11: encoding: [0x05,0x00,0x39,0xd7,0xfd,0xd4,0x00,0x00]
@@ -2783,7 +2783,7 @@ v_mad_i16 v5, ttmp15, src_scc, ttmp15
 // GFX11: encoding: [0x05,0x00,0x53,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_mad_i16 v5, m0, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x53,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x53,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_mad_i16 v5, exec_lo, -1, vcc_hi
 // GFX11: encoding: [0x05,0x00,0x53,0xd6,0x7e,0x82,0xad,0x01]
@@ -2798,7 +2798,7 @@ v_mad_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX11: encoding: [0x05,0x08,0x53,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_mad_i16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX11: encoding: [0x05,0x10,0x53,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x10,0x53,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_mad_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX11: encoding: [0x05,0x20,0x53,0xd6,0xfd,0xd4,0x04,0x03]
@@ -2828,7 +2828,7 @@ v_mad_i32_i16 v5, ttmp15, src_scc, ttmp15
 // GFX11: encoding: [0x05,0x00,0x5a,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_mad_i32_i16 v5, m0, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x5a,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x5a,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_mad_i32_i16 v5, exec_lo, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x5a,0xd6,0x7e,0x82,0xfd,0x01]
@@ -2843,7 +2843,7 @@ v_mad_i32_i16 v5, -1, exec_hi, 0xaf123456
 // GFX11: encoding: [0x05,0x00,0x5a,0xd6,0xc1,0xfe,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
 v_mad_i32_i16 v5, 0.5, m0, -1 op_sel:[0,0,0,0]
-// GFX11: encoding: [0x05,0x00,0x5a,0xd6,0xff,0xfa,0x04,0x03,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x5a,0xd6,0xf0,0xfa,0x04,0x03]
 
 v_mad_i32_i16 v5, src_scc, vcc_lo, src_scc op_sel:[1,0,0,0]
 // GFX11: encoding: [0x05,0x08,0x5a,0xd6,0xfd,0xd4,0xf4,0x03]
@@ -2993,7 +2993,7 @@ v_mad_u16 v5, ttmp15, src_scc, ttmp15
 // GFX11: encoding: [0x05,0x00,0x41,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_mad_u16 v5, m0, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x41,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x41,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_mad_u16 v5, exec_lo, -1, vcc_hi
 // GFX11: encoding: [0x05,0x00,0x41,0xd6,0x7e,0x82,0xad,0x01]
@@ -3008,7 +3008,7 @@ v_mad_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX11: encoding: [0x05,0x08,0x41,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_mad_u16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX11: encoding: [0x05,0x10,0x41,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x10,0x41,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_mad_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX11: encoding: [0x05,0x20,0x41,0xd6,0xfd,0xd4,0x04,0x03]
@@ -3038,7 +3038,7 @@ v_mad_u32_u16 v5, ttmp15, src_scc, ttmp15
 // GFX11: encoding: [0x05,0x00,0x59,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_mad_u32_u16 v5, m0, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x59,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x59,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_mad_u32_u16 v5, exec_lo, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x59,0xd6,0x7e,0x82,0xfd,0x01]
@@ -3053,7 +3053,7 @@ v_mad_u32_u16 v5, -1, exec_hi, 0xaf123456
 // GFX11: encoding: [0x05,0x00,0x59,0xd6,0xc1,0xfe,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
 v_mad_u32_u16 v5, 0.5, m0, -1 op_sel:[0,0,0,0]
-// GFX11: encoding: [0x05,0x00,0x59,0xd6,0xff,0xfa,0x04,0x03,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x59,0xd6,0xf0,0xfa,0x04,0x03]
 
 v_mad_u32_u16 v5, src_scc, vcc_lo, src_scc op_sel:[1,0,0,0]
 // GFX11: encoding: [0x05,0x08,0x59,0xd6,0xfd,0xd4,0xf4,0x03]
@@ -3296,7 +3296,7 @@ v_max3_i16 v5, ttmp15, src_scc, ttmp15
 // GFX11: encoding: [0x05,0x00,0x4d,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_max3_i16 v5, m0, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x4d,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x4d,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_max3_i16 v5, exec_lo, -1, vcc_hi
 // GFX11: encoding: [0x05,0x00,0x4d,0xd6,0x7e,0x82,0xad,0x01]
@@ -3311,7 +3311,7 @@ v_max3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX11: encoding: [0x05,0x08,0x4d,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_max3_i16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX11: encoding: [0x05,0x10,0x4d,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x10,0x4d,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_max3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX11: encoding: [0x05,0x20,0x4d,0xd6,0xfd,0xd4,0x04,0x03]
@@ -3386,7 +3386,7 @@ v_max3_u16 v5, ttmp15, src_scc, ttmp15
 // GFX11: encoding: [0x05,0x00,0x4e,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_max3_u16 v5, m0, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x4e,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x4e,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_max3_u16 v5, exec_lo, -1, vcc_hi
 // GFX11: encoding: [0x05,0x00,0x4e,0xd6,0x7e,0x82,0xad,0x01]
@@ -3401,7 +3401,7 @@ v_max3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX11: encoding: [0x05,0x08,0x4e,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_max3_u16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX11: encoding: [0x05,0x10,0x4e,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x10,0x4e,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_max3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX11: encoding: [0x05,0x20,0x4e,0xd6,0xfd,0xd4,0x04,0x03]
@@ -3512,7 +3512,7 @@ v_max_i16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x0a,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_max_i16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x0a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x0a,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_max_i16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x0a,0xd7,0x7e,0x82,0x01,0x00]
@@ -3527,7 +3527,7 @@ v_max_i16 v5, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x0a,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_max_i16 v5, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x0a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x0a,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_max_i16 v5, src_scc, vcc_lo
 // GFX11: encoding: [0x05,0x00,0x0a,0xd7,0xfd,0xd4,0x00,0x00]
@@ -3557,7 +3557,7 @@ v_max_u16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x09,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_max_u16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x09,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x09,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_max_u16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x09,0xd7,0x7e,0x82,0x01,0x00]
@@ -3572,7 +3572,7 @@ v_max_u16 v5, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x09,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_max_u16 v5, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x09,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x09,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_max_u16 v5, src_scc, vcc_lo
 // GFX11: encoding: [0x05,0x00,0x09,0xd7,0xfd,0xd4,0x00,0x00]
@@ -3965,7 +3965,7 @@ v_med3_i16 v5, ttmp15, src_scc, ttmp15
 // GFX11: encoding: [0x05,0x00,0x50,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_med3_i16 v5, m0, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x50,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x50,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_med3_i16 v5, exec_lo, -1, vcc_hi
 // GFX11: encoding: [0x05,0x00,0x50,0xd6,0x7e,0x82,0xad,0x01]
@@ -3980,7 +3980,7 @@ v_med3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX11: encoding: [0x05,0x08,0x50,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_med3_i16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX11: encoding: [0x05,0x10,0x50,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x10,0x50,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_med3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX11: encoding: [0x05,0x20,0x50,0xd6,0xfd,0xd4,0x04,0x03]
@@ -4055,7 +4055,7 @@ v_med3_u16 v5, ttmp15, src_scc, ttmp15
 // GFX11: encoding: [0x05,0x00,0x51,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_med3_u16 v5, m0, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x51,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x51,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_med3_u16 v5, exec_lo, -1, vcc_hi
 // GFX11: encoding: [0x05,0x00,0x51,0xd6,0x7e,0x82,0xad,0x01]
@@ -4070,7 +4070,7 @@ v_med3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX11: encoding: [0x05,0x08,0x51,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_med3_u16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX11: encoding: [0x05,0x10,0x51,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x10,0x51,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_med3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX11: encoding: [0x05,0x20,0x51,0xd6,0xfd,0xd4,0x04,0x03]
@@ -4238,7 +4238,7 @@ v_min3_i16 v5, ttmp15, src_scc, ttmp15
 // GFX11: encoding: [0x05,0x00,0x4a,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_min3_i16 v5, m0, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x4a,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x4a,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_min3_i16 v5, exec_lo, -1, vcc_hi
 // GFX11: encoding: [0x05,0x00,0x4a,0xd6,0x7e,0x82,0xad,0x01]
@@ -4253,7 +4253,7 @@ v_min3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX11: encoding: [0x05,0x08,0x4a,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_min3_i16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX11: encoding: [0x05,0x10,0x4a,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x10,0x4a,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_min3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX11: encoding: [0x05,0x20,0x4a,0xd6,0xfd,0xd4,0x04,0x03]
@@ -4328,7 +4328,7 @@ v_min3_u16 v5, ttmp15, src_scc, ttmp15
 // GFX11: encoding: [0x05,0x00,0x4b,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_min3_u16 v5, m0, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x4b,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x4b,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_min3_u16 v5, exec_lo, -1, vcc_hi
 // GFX11: encoding: [0x05,0x00,0x4b,0xd6,0x7e,0x82,0xad,0x01]
@@ -4343,7 +4343,7 @@ v_min3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX11: encoding: [0x05,0x08,0x4b,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_min3_u16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX11: encoding: [0x05,0x10,0x4b,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x10,0x4b,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_min3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX11: encoding: [0x05,0x20,0x4b,0xd6,0xfd,0xd4,0x04,0x03]
@@ -4454,7 +4454,7 @@ v_min_i16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x0c,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_min_i16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x0c,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x0c,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_min_i16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x0c,0xd7,0x7e,0x82,0x01,0x00]
@@ -4469,7 +4469,7 @@ v_min_i16 v5, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x0c,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_min_i16 v5, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x0c,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x0c,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_min_i16 v5, src_scc, vcc_lo
 // GFX11: encoding: [0x05,0x00,0x0c,0xd7,0xfd,0xd4,0x00,0x00]
@@ -4499,7 +4499,7 @@ v_min_u16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x0b,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_min_u16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x0b,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x0b,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_min_u16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x0b,0xd7,0x7e,0x82,0x01,0x00]
@@ -4514,7 +4514,7 @@ v_min_u16 v5, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x0b,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_min_u16 v5, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x0b,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x0b,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_min_u16 v5, src_scc, vcc_lo
 // GFX11: encoding: [0x05,0x00,0x0b,0xd7,0xfd,0xd4,0x00,0x00]
@@ -4985,7 +4985,7 @@ v_mul_lo_u16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x05,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_mul_lo_u16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x05,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x05,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_mul_lo_u16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x05,0xd7,0x7e,0x82,0x01,0x00]
@@ -5000,7 +5000,7 @@ v_mul_lo_u16 v5, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x05,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_mul_lo_u16 v5, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x05,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x05,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_mul_lo_u16 v5, src_scc, vcc_lo
 // GFX11: encoding: [0x05,0x00,0x05,0xd7,0xfd,0xd4,0x00,0x00]
@@ -5165,7 +5165,7 @@ v_or_b16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x63,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_or_b16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x63,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x63,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_or_b16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x63,0xd7,0x7e,0x82,0x01,0x00]
@@ -5180,7 +5180,7 @@ v_or_b16 v5, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x63,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_or_b16 v5, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x63,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x63,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_or_b16 v5, src_scc, vcc_lo
 // GFX11: encoding: [0x05,0x00,0x63,0xd7,0xfd,0xd4,0x00,0x00]
@@ -5751,7 +5751,7 @@ v_sub_nc_i16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x0e,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_sub_nc_i16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x0e,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x0e,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_sub_nc_i16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x0e,0xd7,0x7e,0x82,0x01,0x00]
@@ -5766,7 +5766,7 @@ v_sub_nc_i16 v5, -1, exec_hi op_sel:[0,0,0]
 // GFX11: encoding: [0x05,0x00,0x0e,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_sub_nc_i16 v5, 0.5, m0 op_sel:[1,0,0]
-// GFX11: encoding: [0x05,0x08,0x0e,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x08,0x0e,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_sub_nc_i16 v5, src_scc, vcc_lo op_sel:[0,1,0]
 // GFX11: encoding: [0x05,0x10,0x0e,0xd7,0xfd,0xd4,0x00,0x00]
@@ -5841,7 +5841,7 @@ v_sub_nc_u16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x04,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_sub_nc_u16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x04,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x04,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_sub_nc_u16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x04,0xd7,0x7e,0x82,0x01,0x00]
@@ -5856,7 +5856,7 @@ v_sub_nc_u16 v5, -1, exec_hi op_sel:[0,0,0]
 // GFX11: encoding: [0x05,0x00,0x04,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_sub_nc_u16 v5, 0.5, m0 op_sel:[1,0,0]
-// GFX11: encoding: [0x05,0x08,0x04,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x08,0x04,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_sub_nc_u16 v5, src_scc, vcc_lo op_sel:[0,1,0]
 // GFX11: encoding: [0x05,0x10,0x04,0xd7,0xfd,0xd4,0x00,0x00]
@@ -6175,7 +6175,7 @@ v_xor_b16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x64,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_xor_b16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x64,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x64,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_xor_b16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x64,0xd7,0x7e,0x82,0x01,0x00]
@@ -6190,7 +6190,7 @@ v_xor_b16 v5, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x64,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_xor_b16 v5, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x64,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x64,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_xor_b16 v5, src_scc, vcc_lo
 // GFX11: encoding: [0x05,0x00,0x64,0xd7,0xfd,0xd4,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vop1.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vop1.s
index fb4e9108fe1d..71e0f8250415 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vop1.s
@@ -478,7 +478,7 @@ v_cvt_f16_i16_e64 v5, -1
 // GFX11: encoding: [0x05,0x00,0xd1,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_f16_i16_e64 v5, 0.5 mul:2
-// GFX11: encoding: [0x05,0x00,0xd1,0xd5,0xff,0x00,0x00,0x08,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0xd1,0xd5,0xf0,0x00,0x00,0x08]
 
 v_cvt_f16_i16_e64 v5, src_scc mul:4
 // GFX11: encoding: [0x05,0x00,0xd1,0xd5,0xfd,0x00,0x00,0x10]
@@ -523,7 +523,7 @@ v_cvt_f16_u16_e64 v5, -1
 // GFX11: encoding: [0x05,0x00,0xd0,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_f16_u16_e64 v5, 0.5 mul:2
-// GFX11: encoding: [0x05,0x00,0xd0,0xd5,0xff,0x00,0x00,0x08,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0xd0,0xd5,0xf0,0x00,0x00,0x08]
 
 v_cvt_f16_u16_e64 v5, src_scc mul:4
 // GFX11: encoding: [0x05,0x00,0xd0,0xd5,0xfd,0x00,0x00,0x10]
@@ -1270,7 +1270,7 @@ v_cvt_i32_i16_e64 v5, -1
 // GFX11: encoding: [0x05,0x00,0xea,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_i32_i16_e64 v5, 0.5
-// GFX11: encoding: [0x05,0x00,0xea,0xd5,0xff,0x00,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0xea,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_i32_i16_e64 v5, src_scc
 // GFX11: encoding: [0x05,0x00,0xea,0xd5,0xfd,0x00,0x00,0x00]
@@ -1666,7 +1666,7 @@ v_cvt_u32_u16_e64 v5, -1
 // GFX11: encoding: [0x05,0x00,0xeb,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_u32_u16_e64 v5, 0.5
-// GFX11: encoding: [0x05,0x00,0xeb,0xd5,0xff,0x00,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0xeb,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_u32_u16_e64 v5, src_scc
 // GFX11: encoding: [0x05,0x00,0xeb,0xd5,0xfd,0x00,0x00,0x00]
@@ -2641,7 +2641,7 @@ v_not_b16_e64 v5, -1
 // GFX11: encoding: [0x05,0x00,0xe9,0xd5,0xc1,0x00,0x00,0x00]
 
 v_not_b16_e64 v5, 0.5
-// GFX11: encoding: [0x05,0x00,0xe9,0xd5,0xff,0x00,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0xe9,0xd5,0xf0,0x00,0x00,0x00]
 
 v_not_b16_e64 v5, src_scc
 // GFX11: encoding: [0x05,0x00,0xe9,0xd5,0xfd,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopc.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopc.s
index 3a6a61891d29..72894715408a 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopc.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopc.s
@@ -702,7 +702,7 @@ v_cmp_eq_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x32,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x32,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s5, exec_lo, -1
@@ -722,7 +722,7 @@ v_cmp_eq_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x32,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x32,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 ttmp15, src_scc, vcc_lo
@@ -758,7 +758,7 @@ v_cmp_eq_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x32,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s[10:11], exec_lo, -1
@@ -778,7 +778,7 @@ v_cmp_eq_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x32,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x32,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -1023,7 +1023,7 @@ v_cmp_eq_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x3a,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s5, exec_lo, -1
@@ -1043,7 +1043,7 @@ v_cmp_eq_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x3a,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x3a,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 ttmp15, src_scc, vcc_lo
@@ -1079,7 +1079,7 @@ v_cmp_eq_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s[10:11], exec_lo, -1
@@ -1099,7 +1099,7 @@ v_cmp_eq_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x3a,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x3a,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -2398,7 +2398,7 @@ v_cmp_ge_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x36,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x36,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s5, exec_lo, -1
@@ -2418,7 +2418,7 @@ v_cmp_ge_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x36,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x36,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 ttmp15, src_scc, vcc_lo
@@ -2454,7 +2454,7 @@ v_cmp_ge_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x36,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s[10:11], exec_lo, -1
@@ -2474,7 +2474,7 @@ v_cmp_ge_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x36,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x36,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -2719,7 +2719,7 @@ v_cmp_ge_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x3e,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s5, exec_lo, -1
@@ -2739,7 +2739,7 @@ v_cmp_ge_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x3e,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x3e,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 ttmp15, src_scc, vcc_lo
@@ -2775,7 +2775,7 @@ v_cmp_ge_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s[10:11], exec_lo, -1
@@ -2795,7 +2795,7 @@ v_cmp_ge_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x3e,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x3e,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -3361,7 +3361,7 @@ v_cmp_gt_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x34,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x34,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s5, exec_lo, -1
@@ -3381,7 +3381,7 @@ v_cmp_gt_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x34,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x34,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 ttmp15, src_scc, vcc_lo
@@ -3417,7 +3417,7 @@ v_cmp_gt_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x34,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s[10:11], exec_lo, -1
@@ -3437,7 +3437,7 @@ v_cmp_gt_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x34,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x34,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -3682,7 +3682,7 @@ v_cmp_gt_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x3c,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s5, exec_lo, -1
@@ -3702,7 +3702,7 @@ v_cmp_gt_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x3c,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x3c,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 ttmp15, src_scc, vcc_lo
@@ -3738,7 +3738,7 @@ v_cmp_gt_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s[10:11], exec_lo, -1
@@ -3758,7 +3758,7 @@ v_cmp_gt_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x3c,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x3c,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -4324,7 +4324,7 @@ v_cmp_le_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x33,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x33,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s5, exec_lo, -1
@@ -4344,7 +4344,7 @@ v_cmp_le_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x33,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x33,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 ttmp15, src_scc, vcc_lo
@@ -4380,7 +4380,7 @@ v_cmp_le_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x33,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s[10:11], exec_lo, -1
@@ -4400,7 +4400,7 @@ v_cmp_le_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x33,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x33,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -4645,7 +4645,7 @@ v_cmp_le_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x3b,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s5, exec_lo, -1
@@ -4665,7 +4665,7 @@ v_cmp_le_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x3b,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x3b,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 ttmp15, src_scc, vcc_lo
@@ -4701,7 +4701,7 @@ v_cmp_le_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s[10:11], exec_lo, -1
@@ -4721,7 +4721,7 @@ v_cmp_le_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x3b,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x3b,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -5608,7 +5608,7 @@ v_cmp_lt_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x31,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x31,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s5, exec_lo, -1
@@ -5628,7 +5628,7 @@ v_cmp_lt_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x31,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x31,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 ttmp15, src_scc, vcc_lo
@@ -5664,7 +5664,7 @@ v_cmp_lt_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x31,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s[10:11], exec_lo, -1
@@ -5684,7 +5684,7 @@ v_cmp_lt_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x31,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x31,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -5929,7 +5929,7 @@ v_cmp_lt_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x39,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x39,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s5, exec_lo, -1
@@ -5949,7 +5949,7 @@ v_cmp_lt_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x39,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x39,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 ttmp15, src_scc, vcc_lo
@@ -5985,7 +5985,7 @@ v_cmp_lt_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x39,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s[10:11], exec_lo, -1
@@ -6005,7 +6005,7 @@ v_cmp_lt_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x39,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x39,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -6250,7 +6250,7 @@ v_cmp_ne_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x35,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x35,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s5, exec_lo, -1
@@ -6270,7 +6270,7 @@ v_cmp_ne_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x35,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x35,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 ttmp15, src_scc, vcc_lo
@@ -6306,7 +6306,7 @@ v_cmp_ne_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x35,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s[10:11], exec_lo, -1
@@ -6326,7 +6326,7 @@ v_cmp_ne_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x35,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x35,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -6571,7 +6571,7 @@ v_cmp_ne_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x3d,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s5, exec_lo, -1
@@ -6591,7 +6591,7 @@ v_cmp_ne_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x3d,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x3d,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 ttmp15, src_scc, vcc_lo
@@ -6627,7 +6627,7 @@ v_cmp_ne_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s[10:11], exec_lo, -1
@@ -6647,7 +6647,7 @@ v_cmp_ne_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x3d,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x3d,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 ttmp[14:15], src_scc, vcc_lo
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopcx.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopcx.s
index f50a47777f64..79409a568475 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopcx.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopcx.s
@@ -287,7 +287,7 @@ v_cmpx_eq_i16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xb2,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_eq_i16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xb2,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb2,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_eq_i16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xb2,0xd4,0x7e,0x82,0x01,0x00]
@@ -302,7 +302,7 @@ v_cmpx_eq_i16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xb2,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_eq_i16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xb2,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb2,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_eq_i16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xb2,0xd4,0xfd,0xd4,0x00,0x00]
@@ -413,7 +413,7 @@ v_cmpx_eq_u16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xba,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_eq_u16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xba,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xba,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_eq_u16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xba,0xd4,0x7e,0x82,0x01,0x00]
@@ -428,7 +428,7 @@ v_cmpx_eq_u16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xba,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_eq_u16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xba,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xba,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_eq_u16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xba,0xd4,0xfd,0xd4,0x00,0x00]
@@ -953,7 +953,7 @@ v_cmpx_ge_i16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xb6,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_ge_i16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xb6,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb6,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_ge_i16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xb6,0xd4,0x7e,0x82,0x01,0x00]
@@ -968,7 +968,7 @@ v_cmpx_ge_i16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xb6,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_ge_i16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xb6,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb6,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_ge_i16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xb6,0xd4,0xfd,0xd4,0x00,0x00]
@@ -1079,7 +1079,7 @@ v_cmpx_ge_u16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xbe,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_ge_u16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xbe,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xbe,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_ge_u16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xbe,0xd4,0x7e,0x82,0x01,0x00]
@@ -1094,7 +1094,7 @@ v_cmpx_ge_u16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xbe,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_ge_u16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xbe,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xbe,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_ge_u16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xbe,0xd4,0xfd,0xd4,0x00,0x00]
@@ -1331,7 +1331,7 @@ v_cmpx_gt_i16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xb4,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_gt_i16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xb4,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb4,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_gt_i16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xb4,0xd4,0x7e,0x82,0x01,0x00]
@@ -1346,7 +1346,7 @@ v_cmpx_gt_i16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xb4,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_gt_i16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xb4,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb4,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_gt_i16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xb4,0xd4,0xfd,0xd4,0x00,0x00]
@@ -1457,7 +1457,7 @@ v_cmpx_gt_u16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xbc,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_gt_u16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xbc,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xbc,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_gt_u16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xbc,0xd4,0x7e,0x82,0x01,0x00]
@@ -1472,7 +1472,7 @@ v_cmpx_gt_u16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xbc,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_gt_u16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xbc,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xbc,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_gt_u16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xbc,0xd4,0xfd,0xd4,0x00,0x00]
@@ -1709,7 +1709,7 @@ v_cmpx_le_i16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xb3,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_le_i16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xb3,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb3,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_le_i16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xb3,0xd4,0x7e,0x82,0x01,0x00]
@@ -1724,7 +1724,7 @@ v_cmpx_le_i16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xb3,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_le_i16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xb3,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb3,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_le_i16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xb3,0xd4,0xfd,0xd4,0x00,0x00]
@@ -1835,7 +1835,7 @@ v_cmpx_le_u16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xbb,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_le_u16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xbb,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xbb,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_le_u16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xbb,0xd4,0x7e,0x82,0x01,0x00]
@@ -1850,7 +1850,7 @@ v_cmpx_le_u16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xbb,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_le_u16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xbb,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xbb,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_le_u16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xbb,0xd4,0xfd,0xd4,0x00,0x00]
@@ -2213,7 +2213,7 @@ v_cmpx_lt_i16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xb1,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_lt_i16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xb1,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb1,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_lt_i16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xb1,0xd4,0x7e,0x82,0x01,0x00]
@@ -2228,7 +2228,7 @@ v_cmpx_lt_i16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xb1,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_lt_i16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xb1,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb1,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_lt_i16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xb1,0xd4,0xfd,0xd4,0x00,0x00]
@@ -2339,7 +2339,7 @@ v_cmpx_lt_u16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xb9,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_lt_u16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xb9,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb9,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_lt_u16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xb9,0xd4,0x7e,0x82,0x01,0x00]
@@ -2354,7 +2354,7 @@ v_cmpx_lt_u16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xb9,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_lt_u16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xb9,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb9,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_lt_u16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xb9,0xd4,0xfd,0xd4,0x00,0x00]
@@ -2465,7 +2465,7 @@ v_cmpx_ne_i16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xb5,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_ne_i16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xb5,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb5,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_ne_i16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xb5,0xd4,0x7e,0x82,0x01,0x00]
@@ -2480,7 +2480,7 @@ v_cmpx_ne_i16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xb5,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_ne_i16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xb5,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb5,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_ne_i16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xb5,0xd4,0xfd,0xd4,0x00,0x00]
@@ -2591,7 +2591,7 @@ v_cmpx_ne_u16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xbd,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_ne_u16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xbd,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xbd,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_ne_u16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xbd,0xd4,0x7e,0x82,0x01,0x00]
@@ -2606,7 +2606,7 @@ v_cmpx_ne_u16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xbd,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_ne_u16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xbd,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xbd,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_ne_u16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xbd,0xd4,0xfd,0xd4,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopc.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopc.s
index c9272f24e25a..f2025e4bd6d6 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vopc.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopc.s
@@ -724,7 +724,7 @@ v_cmp_eq_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x64,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x64,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc_lo, src_scc, v2
@@ -784,7 +784,7 @@ v_cmp_eq_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x64,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x64,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc, src_scc, v2
@@ -1060,7 +1060,7 @@ v_cmp_eq_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x74,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x74,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc_lo, src_scc, v2
@@ -1120,7 +1120,7 @@ v_cmp_eq_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x74,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x74,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc, src_scc, v2
@@ -2500,7 +2500,7 @@ v_cmp_ge_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x6c,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x6c,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc_lo, src_scc, v2
@@ -2560,7 +2560,7 @@ v_cmp_ge_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x6c,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x6c,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc, src_scc, v2
@@ -2836,7 +2836,7 @@ v_cmp_ge_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x7c,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x7c,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc_lo, src_scc, v2
@@ -2896,7 +2896,7 @@ v_cmp_ge_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x7c,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x7c,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc, src_scc, v2
@@ -3508,7 +3508,7 @@ v_cmp_gt_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x68,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x68,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc_lo, src_scc, v2
@@ -3568,7 +3568,7 @@ v_cmp_gt_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x68,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x68,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc, src_scc, v2
@@ -3844,7 +3844,7 @@ v_cmp_gt_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x78,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x78,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc_lo, src_scc, v2
@@ -3904,7 +3904,7 @@ v_cmp_gt_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x78,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x78,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc, src_scc, v2
@@ -4516,7 +4516,7 @@ v_cmp_le_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x66,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x66,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc_lo, src_scc, v2
@@ -4576,7 +4576,7 @@ v_cmp_le_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x66,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x66,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc, src_scc, v2
@@ -4852,7 +4852,7 @@ v_cmp_le_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x76,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x76,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc_lo, src_scc, v2
@@ -4912,7 +4912,7 @@ v_cmp_le_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x76,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x76,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc, src_scc, v2
@@ -5860,7 +5860,7 @@ v_cmp_lt_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x62,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x62,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc_lo, src_scc, v2
@@ -5920,7 +5920,7 @@ v_cmp_lt_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x62,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x62,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc, src_scc, v2
@@ -6196,7 +6196,7 @@ v_cmp_lt_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x72,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x72,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc_lo, src_scc, v2
@@ -6256,7 +6256,7 @@ v_cmp_lt_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x72,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x72,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc, src_scc, v2
@@ -6532,7 +6532,7 @@ v_cmp_ne_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x6a,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x6a,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc_lo, src_scc, v2
@@ -6592,7 +6592,7 @@ v_cmp_ne_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x6a,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x6a,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc, src_scc, v2
@@ -6868,7 +6868,7 @@ v_cmp_ne_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x7a,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x7a,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc_lo, src_scc, v2
@@ -6928,7 +6928,7 @@ v_cmp_ne_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x7a,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x7a,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc, src_scc, v2
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx.s
index 2512f1a4b94b..d52034fedc35 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx.s
@@ -290,7 +290,7 @@ v_cmpx_eq_i16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x64,0x7d]
 
 v_cmpx_eq_i16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x64,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x64,0x7d]
 
 v_cmpx_eq_i16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x64,0x7d]
@@ -416,7 +416,7 @@ v_cmpx_eq_u16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x74,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x74,0x7d]
@@ -956,7 +956,7 @@ v_cmpx_ge_i16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x6c,0x7d]
 
 v_cmpx_ge_i16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x6c,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x6c,0x7d]
 
 v_cmpx_ge_i16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x6c,0x7d]
@@ -1082,7 +1082,7 @@ v_cmpx_ge_u16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x7c,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x7c,0x7d]
@@ -1334,7 +1334,7 @@ v_cmpx_gt_i16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x68,0x7d]
 
 v_cmpx_gt_i16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x68,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x68,0x7d]
 
 v_cmpx_gt_i16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x68,0x7d]
@@ -1460,7 +1460,7 @@ v_cmpx_gt_u16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x78,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x78,0x7d]
@@ -1712,7 +1712,7 @@ v_cmpx_le_i16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x66,0x7d]
 
 v_cmpx_le_i16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x66,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x66,0x7d]
 
 v_cmpx_le_i16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x66,0x7d]
@@ -1838,7 +1838,7 @@ v_cmpx_le_u16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x76,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x76,0x7d]
@@ -2216,7 +2216,7 @@ v_cmpx_lt_i16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x62,0x7d]
 
 v_cmpx_lt_i16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x62,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x62,0x7d]
 
 v_cmpx_lt_i16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x62,0x7d]
@@ -2342,7 +2342,7 @@ v_cmpx_lt_u16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x72,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x72,0x7d]
@@ -2468,7 +2468,7 @@ v_cmpx_ne_i16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x6a,0x7d]
 
 v_cmpx_ne_i16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x6a,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x6a,0x7d]
 
 v_cmpx_ne_i16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x6a,0x7d]
@@ -2594,7 +2594,7 @@ v_cmpx_ne_u16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x7a,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x7a,0x7d]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s
index a21ef11bfb87..5279588f0506 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s
@@ -524,7 +524,7 @@ v_cvt_f16_i16 v5, -1
 // GFX12: encoding: [0xc1,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, 0.5
-// GFX12: encoding: [0xff,0xa2,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, src_scc
 // GFX12: encoding: [0xfd,0xa2,0x0a,0x7e]
@@ -569,7 +569,7 @@ v_cvt_f16_u16 v5, -1
 // GFX12: encoding: [0xc1,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, 0.5
-// GFX12: encoding: [0xff,0xa0,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, src_scc
 // GFX12: encoding: [0xfd,0xa0,0x0a,0x7e]
@@ -1316,7 +1316,7 @@ v_cvt_i32_i16 v5, -1
 // GFX12: encoding: [0xc1,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, 0.5
-// GFX12: encoding: [0xff,0xd4,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, src_scc
 // GFX12: encoding: [0xfd,0xd4,0x0a,0x7e]
@@ -1712,7 +1712,7 @@ v_cvt_u32_u16 v5, -1
 // GFX12: encoding: [0xc1,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, 0.5
-// GFX12: encoding: [0xff,0xd6,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, src_scc
 // GFX12: encoding: [0xfd,0xd6,0x0a,0x7e]
@@ -2687,7 +2687,7 @@ v_not_b16 v5, -1
 // GFX12: encoding: [0xc1,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, 0.5
-// GFX12: encoding: [0xff,0xd2,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, src_scc
 // GFX12: encoding: [0xfd,0xd2,0x0a,0x7e]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s
index 9d0488fd2f1c..be9edc3e019e 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s
@@ -230,7 +230,7 @@ v_add_nc_i16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x0d,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_add_nc_i16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x0d,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x0d,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_add_nc_i16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x0d,0xd7,0x7e,0x82,0x01,0x00]
@@ -245,7 +245,7 @@ v_add_nc_i16 v5, -1, exec_hi op_sel:[0,0,0]
 // GFX12: encoding: [0x05,0x00,0x0d,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_add_nc_i16 v5, 0.5, m0 op_sel:[1,0,0]
-// GFX12: encoding: [0x05,0x08,0x0d,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x08,0x0d,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_add_nc_i16 v5, src_scc, vcc_lo op_sel:[0,1,0]
 // GFX12: encoding: [0x05,0x10,0x0d,0xd7,0xfd,0xd4,0x00,0x00]
@@ -320,7 +320,7 @@ v_add_nc_u16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x03,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_add_nc_u16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x03,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x03,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_add_nc_u16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x03,0xd7,0x7e,0x82,0x01,0x00]
@@ -335,7 +335,7 @@ v_add_nc_u16 v5, -1, exec_hi op_sel:[0,0,0]
 // GFX12: encoding: [0x05,0x00,0x03,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_add_nc_u16 v5, 0.5, m0 op_sel:[1,0,0]
-// GFX12: encoding: [0x05,0x08,0x03,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x08,0x03,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_add_nc_u16 v5, src_scc, vcc_lo op_sel:[0,1,0]
 // GFX12: encoding: [0x05,0x10,0x03,0xd7,0xfd,0xd4,0x00,0x00]
@@ -455,7 +455,7 @@ v_and_b16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x62,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_and_b16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x62,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x62,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_and_b16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x62,0xd7,0x7e,0x82,0x01,0x00]
@@ -470,7 +470,7 @@ v_and_b16 v5, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x62,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_and_b16 v5, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x62,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x62,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_and_b16 v5, src_scc, vcc_lo
 // GFX12: encoding: [0x05,0x00,0x62,0xd7,0xfd,0xd4,0x00,0x00]
@@ -545,7 +545,7 @@ v_ashrrev_i16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x3a,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_ashrrev_i16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x3a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x3a,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_ashrrev_i16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x3a,0xd7,0x7e,0x82,0x01,0x00]
@@ -560,7 +560,7 @@ v_ashrrev_i16 v5, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x3a,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_ashrrev_i16 v5, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x3a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x3a,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_ashrrev_i16 v5, src_scc, vcc_lo
 // GFX12: encoding: [0x05,0x00,0x3a,0xd7,0xfd,0xd4,0x00,0x00]
@@ -825,7 +825,7 @@ v_cndmask_b16 v5, v1, src_scc, s3
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, v255, 0.5, s3
-// W32: encoding: [0x05,0x00,0x5d,0xd6,0xff,0xff,0x0d,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x5d,0xd6,0xff,0xe1,0x0d,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, s105, s105, s3
@@ -861,7 +861,7 @@ v_cndmask_b16 v5, -1, -|vcc_lo|, vcc_lo
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, 0.5, -1, vcc_hi
-// W32: encoding: [0x05,0x00,0x5d,0xd6,0xff,0x82,0xad,0x01,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x5d,0xd6,0xf0,0x82,0xad,0x01]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, -|src_scc|, null, ttmp15
@@ -873,7 +873,7 @@ v_cndmask_b16 v5, v1, src_scc, s[6:7]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, v255, 0.5, s[6:7]
-// W64: encoding: [0x05,0x00,0x5d,0xd6,0xff,0xff,0x19,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x05,0x00,0x5d,0xd6,0xff,0xe1,0x19,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, s105, s105, s[6:7]
@@ -909,7 +909,7 @@ v_cndmask_b16 v5, -1, -|vcc_lo|, s[104:105]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, 0.5, -1, vcc
-// W64: encoding: [0x05,0x00,0x5d,0xd6,0xff,0x82,0xa9,0x01,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x05,0x00,0x5d,0xd6,0xf0,0x82,0xa9,0x01]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, -|src_scc|, null, ttmp[14:15]
@@ -2621,7 +2621,7 @@ v_lshlrev_b16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x38,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_lshlrev_b16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x38,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x38,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_lshlrev_b16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x38,0xd7,0x7e,0x82,0x01,0x00]
@@ -2636,7 +2636,7 @@ v_lshlrev_b16 v5, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x38,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_lshlrev_b16 v5, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x38,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x38,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_lshlrev_b16 v5, src_scc, vcc_lo
 // GFX12: encoding: [0x05,0x00,0x38,0xd7,0xfd,0xd4,0x00,0x00]
@@ -2666,7 +2666,7 @@ v_lshrrev_b16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x39,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_lshrrev_b16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x39,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x39,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_lshrrev_b16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x39,0xd7,0x7e,0x82,0x01,0x00]
@@ -2681,7 +2681,7 @@ v_lshrrev_b16 v5, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x39,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_lshrrev_b16 v5, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x39,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x39,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_lshrrev_b16 v5, src_scc, vcc_lo
 // GFX12: encoding: [0x05,0x00,0x39,0xd7,0xfd,0xd4,0x00,0x00]
@@ -2738,7 +2738,7 @@ v_mad_i16 v5, ttmp15, src_scc, ttmp15
 // GFX12: encoding: [0x05,0x00,0x53,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_mad_i16 v5, m0, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x53,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x53,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_mad_i16 v5, exec_lo, -1, vcc_hi
 // GFX12: encoding: [0x05,0x00,0x53,0xd6,0x7e,0x82,0xad,0x01]
@@ -2753,7 +2753,7 @@ v_mad_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX12: encoding: [0x05,0x08,0x53,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_mad_i16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX12: encoding: [0x05,0x10,0x53,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x10,0x53,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_mad_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX12: encoding: [0x05,0x20,0x53,0xd6,0xfd,0xd4,0x04,0x03]
@@ -2783,7 +2783,7 @@ v_mad_i32_i16 v5, ttmp15, src_scc, ttmp15
 // GFX12: encoding: [0x05,0x00,0x5a,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_mad_i32_i16 v5, m0, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x5a,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x5a,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_mad_i32_i16 v5, exec_lo, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x5a,0xd6,0x7e,0x82,0xfd,0x01]
@@ -2798,7 +2798,7 @@ v_mad_i32_i16 v5, -1, exec_hi, 0xaf123456
 // GFX12: encoding: [0x05,0x00,0x5a,0xd6,0xc1,0xfe,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
 v_mad_i32_i16 v5, 0.5, m0, -1 op_sel:[0,0,0,0]
-// GFX12: encoding: [0x05,0x00,0x5a,0xd6,0xff,0xfa,0x04,0x03,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x5a,0xd6,0xf0,0xfa,0x04,0x03]
 
 v_mad_i32_i16 v5, src_scc, vcc_lo, src_scc op_sel:[1,0,0,0]
 // GFX12: encoding: [0x05,0x08,0x5a,0xd6,0xfd,0xd4,0xf4,0x03]
@@ -2948,7 +2948,7 @@ v_mad_u16 v5, ttmp15, src_scc, ttmp15
 // GFX12: encoding: [0x05,0x00,0x41,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_mad_u16 v5, m0, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x41,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x41,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_mad_u16 v5, exec_lo, -1, vcc_hi
 // GFX12: encoding: [0x05,0x00,0x41,0xd6,0x7e,0x82,0xad,0x01]
@@ -2963,7 +2963,7 @@ v_mad_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX12: encoding: [0x05,0x08,0x41,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_mad_u16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX12: encoding: [0x05,0x10,0x41,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x10,0x41,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_mad_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX12: encoding: [0x05,0x20,0x41,0xd6,0xfd,0xd4,0x04,0x03]
@@ -2993,7 +2993,7 @@ v_mad_u32_u16 v5, ttmp15, src_scc, ttmp15
 // GFX12: encoding: [0x05,0x00,0x59,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_mad_u32_u16 v5, m0, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x59,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x59,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_mad_u32_u16 v5, exec_lo, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x59,0xd6,0x7e,0x82,0xfd,0x01]
@@ -3008,7 +3008,7 @@ v_mad_u32_u16 v5, -1, exec_hi, 0xaf123456
 // GFX12: encoding: [0x05,0x00,0x59,0xd6,0xc1,0xfe,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
 v_mad_u32_u16 v5, 0.5, m0, -1 op_sel:[0,0,0,0]
-// GFX12: encoding: [0x05,0x00,0x59,0xd6,0xff,0xfa,0x04,0x03,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x59,0xd6,0xf0,0xfa,0x04,0x03]
 
 v_mad_u32_u16 v5, src_scc, vcc_lo, src_scc op_sel:[1,0,0,0]
 // GFX12: encoding: [0x05,0x08,0x59,0xd6,0xfd,0xd4,0xf4,0x03]
@@ -3248,7 +3248,7 @@ v_max3_i16 v5, ttmp15, src_scc, ttmp15
 // GFX12: encoding: [0x05,0x00,0x4d,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_max3_i16 v5, m0, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x4d,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x4d,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_max3_i16 v5, exec_lo, -1, vcc_hi
 // GFX12: encoding: [0x05,0x00,0x4d,0xd6,0x7e,0x82,0xad,0x01]
@@ -3263,7 +3263,7 @@ v_max3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX12: encoding: [0x05,0x08,0x4d,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_max3_i16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX12: encoding: [0x05,0x10,0x4d,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x10,0x4d,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_max3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX12: encoding: [0x05,0x20,0x4d,0xd6,0xfd,0xd4,0x04,0x03]
@@ -3338,7 +3338,7 @@ v_max3_u16 v5, ttmp15, src_scc, ttmp15
 // GFX12: encoding: [0x05,0x00,0x4e,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_max3_u16 v5, m0, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x4e,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x4e,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_max3_u16 v5, exec_lo, -1, vcc_hi
 // GFX12: encoding: [0x05,0x00,0x4e,0xd6,0x7e,0x82,0xad,0x01]
@@ -3353,7 +3353,7 @@ v_max3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX12: encoding: [0x05,0x08,0x4e,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_max3_u16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX12: encoding: [0x05,0x10,0x4e,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x10,0x4e,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_max3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX12: encoding: [0x05,0x20,0x4e,0xd6,0xfd,0xd4,0x04,0x03]
@@ -3428,7 +3428,7 @@ v_max_i16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x0a,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_max_i16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x0a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x0a,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_max_i16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x0a,0xd7,0x7e,0x82,0x01,0x00]
@@ -3443,7 +3443,7 @@ v_max_i16 v5, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x0a,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_max_i16 v5, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x0a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x0a,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_max_i16 v5, src_scc, vcc_lo
 // GFX12: encoding: [0x05,0x00,0x0a,0xd7,0xfd,0xd4,0x00,0x00]
@@ -3473,7 +3473,7 @@ v_max_u16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x09,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_max_u16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x09,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x09,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_max_u16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x09,0xd7,0x7e,0x82,0x01,0x00]
@@ -3488,7 +3488,7 @@ v_max_u16 v5, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x09,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_max_u16 v5, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x09,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x09,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_max_u16 v5, src_scc, vcc_lo
 // GFX12: encoding: [0x05,0x00,0x09,0xd7,0xfd,0xd4,0x00,0x00]
@@ -3878,7 +3878,7 @@ v_med3_i16 v5, ttmp15, src_scc, ttmp15
 // GFX12: encoding: [0x05,0x00,0x50,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_med3_i16 v5, m0, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x50,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x50,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_med3_i16 v5, exec_lo, -1, vcc_hi
 // GFX12: encoding: [0x05,0x00,0x50,0xd6,0x7e,0x82,0xad,0x01]
@@ -3893,7 +3893,7 @@ v_med3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX12: encoding: [0x05,0x08,0x50,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_med3_i16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX12: encoding: [0x05,0x10,0x50,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x10,0x50,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_med3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX12: encoding: [0x05,0x20,0x50,0xd6,0xfd,0xd4,0x04,0x03]
@@ -3968,7 +3968,7 @@ v_med3_u16 v5, ttmp15, src_scc, ttmp15
 // GFX12: encoding: [0x05,0x00,0x51,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_med3_u16 v5, m0, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x51,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x51,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_med3_u16 v5, exec_lo, -1, vcc_hi
 // GFX12: encoding: [0x05,0x00,0x51,0xd6,0x7e,0x82,0xad,0x01]
@@ -3983,7 +3983,7 @@ v_med3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX12: encoding: [0x05,0x08,0x51,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_med3_u16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX12: encoding: [0x05,0x10,0x51,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x10,0x51,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_med3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX12: encoding: [0x05,0x20,0x51,0xd6,0xfd,0xd4,0x04,0x03]
@@ -4148,7 +4148,7 @@ v_min3_i16 v5, ttmp15, src_scc, ttmp15
 // GFX12: encoding: [0x05,0x00,0x4a,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_min3_i16 v5, m0, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x4a,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x4a,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_min3_i16 v5, exec_lo, -1, vcc_hi
 // GFX12: encoding: [0x05,0x00,0x4a,0xd6,0x7e,0x82,0xad,0x01]
@@ -4163,7 +4163,7 @@ v_min3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX12: encoding: [0x05,0x08,0x4a,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_min3_i16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX12: encoding: [0x05,0x10,0x4a,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x10,0x4a,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_min3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX12: encoding: [0x05,0x20,0x4a,0xd6,0xfd,0xd4,0x04,0x03]
@@ -4238,7 +4238,7 @@ v_min3_u16 v5, ttmp15, src_scc, ttmp15
 // GFX12: encoding: [0x05,0x00,0x4b,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_min3_u16 v5, m0, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x4b,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x4b,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_min3_u16 v5, exec_lo, -1, vcc_hi
 // GFX12: encoding: [0x05,0x00,0x4b,0xd6,0x7e,0x82,0xad,0x01]
@@ -4253,7 +4253,7 @@ v_min3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX12: encoding: [0x05,0x08,0x4b,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_min3_u16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX12: encoding: [0x05,0x10,0x4b,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x10,0x4b,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_min3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX12: encoding: [0x05,0x20,0x4b,0xd6,0xfd,0xd4,0x04,0x03]
@@ -4328,7 +4328,7 @@ v_min_i16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x0c,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_min_i16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x0c,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x0c,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_min_i16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x0c,0xd7,0x7e,0x82,0x01,0x00]
@@ -4343,7 +4343,7 @@ v_min_i16 v5, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x0c,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_min_i16 v5, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x0c,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x0c,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_min_i16 v5, src_scc, vcc_lo
 // GFX12: encoding: [0x05,0x00,0x0c,0xd7,0xfd,0xd4,0x00,0x00]
@@ -4373,7 +4373,7 @@ v_min_u16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x0b,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_min_u16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x0b,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x0b,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_min_u16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x0b,0xd7,0x7e,0x82,0x01,0x00]
@@ -4388,7 +4388,7 @@ v_min_u16 v5, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x0b,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_min_u16 v5, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x0b,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x0b,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_min_u16 v5, src_scc, vcc_lo
 // GFX12: encoding: [0x05,0x00,0x0b,0xd7,0xfd,0xd4,0x00,0x00]
@@ -4823,7 +4823,7 @@ v_mul_lo_u16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x05,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_mul_lo_u16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x05,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x05,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_mul_lo_u16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x05,0xd7,0x7e,0x82,0x01,0x00]
@@ -4838,7 +4838,7 @@ v_mul_lo_u16 v5, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x05,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_mul_lo_u16 v5, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x05,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x05,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_mul_lo_u16 v5, src_scc, vcc_lo
 // GFX12: encoding: [0x05,0x00,0x05,0xd7,0xfd,0xd4,0x00,0x00]
@@ -5003,7 +5003,7 @@ v_or_b16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x63,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_or_b16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x63,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x63,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_or_b16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x63,0xd7,0x7e,0x82,0x01,0x00]
@@ -5018,7 +5018,7 @@ v_or_b16 v5, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x63,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_or_b16 v5, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x63,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x63,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_or_b16 v5, src_scc, vcc_lo
 // GFX12: encoding: [0x05,0x00,0x63,0xd7,0xfd,0xd4,0x00,0x00]
@@ -5622,7 +5622,7 @@ v_sub_nc_i16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x0e,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_sub_nc_i16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x0e,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x0e,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_sub_nc_i16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x0e,0xd7,0x7e,0x82,0x01,0x00]
@@ -5637,7 +5637,7 @@ v_sub_nc_i16 v5, -1, exec_hi op_sel:[0,0,0]
 // GFX12: encoding: [0x05,0x00,0x0e,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_sub_nc_i16 v5, 0.5, m0 op_sel:[1,0,0]
-// GFX12: encoding: [0x05,0x08,0x0e,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x08,0x0e,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_sub_nc_i16 v5, src_scc, vcc_lo op_sel:[0,1,0]
 // GFX12: encoding: [0x05,0x10,0x0e,0xd7,0xfd,0xd4,0x00,0x00]
@@ -5712,7 +5712,7 @@ v_sub_nc_u16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x04,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_sub_nc_u16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x04,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x04,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_sub_nc_u16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x04,0xd7,0x7e,0x82,0x01,0x00]
@@ -5727,7 +5727,7 @@ v_sub_nc_u16 v5, -1, exec_hi op_sel:[0,0,0]
 // GFX12: encoding: [0x05,0x00,0x04,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_sub_nc_u16 v5, 0.5, m0 op_sel:[1,0,0]
-// GFX12: encoding: [0x05,0x08,0x04,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x08,0x04,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_sub_nc_u16 v5, src_scc, vcc_lo op_sel:[0,1,0]
 // GFX12: encoding: [0x05,0x10,0x04,0xd7,0xfd,0xd4,0x00,0x00]
@@ -6046,7 +6046,7 @@ v_xor_b16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x64,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_xor_b16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x64,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x64,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_xor_b16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x64,0xd7,0x7e,0x82,0x01,0x00]
@@ -6061,7 +6061,7 @@ v_xor_b16 v5, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x64,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_xor_b16 v5, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x64,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x64,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_xor_b16 v5, src_scc, vcc_lo
 // GFX12: encoding: [0x05,0x00,0x64,0xd7,0xfd,0xd4,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1.s
index 9c6a3b8e68cd..662905809ad9 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1.s
@@ -616,7 +616,7 @@ v_cvt_f16_i16_e64 v5, -1
 // GFX12: encoding: [0x05,0x00,0xd1,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_f16_i16_e64 v5, 0.5 mul:2
-// GFX12: encoding: [0x05,0x00,0xd1,0xd5,0xff,0x00,0x00,0x08,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0xd1,0xd5,0xf0,0x00,0x00,0x08]
 
 v_cvt_f16_i16_e64 v5, src_scc mul:4
 // GFX12: encoding: [0x05,0x00,0xd1,0xd5,0xfd,0x00,0x00,0x10]
@@ -661,7 +661,7 @@ v_cvt_f16_u16_e64 v5, -1
 // GFX12: encoding: [0x05,0x00,0xd0,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_f16_u16_e64 v5, 0.5 mul:2
-// GFX12: encoding: [0x05,0x00,0xd0,0xd5,0xff,0x00,0x00,0x08,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0xd0,0xd5,0xf0,0x00,0x00,0x08]
 
 v_cvt_f16_u16_e64 v5, src_scc mul:4
 // GFX12: encoding: [0x05,0x00,0xd0,0xd5,0xfd,0x00,0x00,0x10]
@@ -1408,7 +1408,7 @@ v_cvt_i32_i16_e64 v5, -1
 // GFX12: encoding: [0x05,0x00,0xea,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_i32_i16_e64 v5, 0.5
-// GFX12: encoding: [0x05,0x00,0xea,0xd5,0xff,0x00,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0xea,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_i32_i16_e64 v5, src_scc
 // GFX12: encoding: [0x05,0x00,0xea,0xd5,0xfd,0x00,0x00,0x00]
@@ -1804,7 +1804,7 @@ v_cvt_u32_u16_e64 v5, -1
 // GFX12: encoding: [0x05,0x00,0xeb,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_u32_u16_e64 v5, 0.5
-// GFX12: encoding: [0x05,0x00,0xeb,0xd5,0xff,0x00,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0xeb,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_u32_u16_e64 v5, src_scc
 // GFX12: encoding: [0x05,0x00,0xeb,0xd5,0xfd,0x00,0x00,0x00]
@@ -2779,7 +2779,7 @@ v_not_b16_e64 v5, -1
 // GFX12: encoding: [0x05,0x00,0xe9,0xd5,0xc1,0x00,0x00,0x00]
 
 v_not_b16_e64 v5, 0.5
-// GFX12: encoding: [0x05,0x00,0xe9,0xd5,0xff,0x00,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0xe9,0xd5,0xf0,0x00,0x00,0x00]
 
 v_not_b16_e64 v5, src_scc
 // GFX12: encoding: [0x05,0x00,0xe9,0xd5,0xfd,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s
index 8bc776498c06..e03bb88e0771 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s
@@ -698,7 +698,7 @@ v_cmp_eq_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x32,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x32,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s5, exec_lo, -1
@@ -718,7 +718,7 @@ v_cmp_eq_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x32,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x32,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 ttmp15, src_scc, vcc_lo
@@ -754,7 +754,7 @@ v_cmp_eq_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x32,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s[10:11], exec_lo, -1
@@ -774,7 +774,7 @@ v_cmp_eq_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x32,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x32,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -1019,7 +1019,7 @@ v_cmp_eq_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x3a,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s5, exec_lo, -1
@@ -1039,7 +1039,7 @@ v_cmp_eq_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x3a,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x3a,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 ttmp15, src_scc, vcc_lo
@@ -1075,7 +1075,7 @@ v_cmp_eq_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s[10:11], exec_lo, -1
@@ -1095,7 +1095,7 @@ v_cmp_eq_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x3a,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x3a,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -1661,7 +1661,7 @@ v_cmp_ge_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x36,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x36,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s5, exec_lo, -1
@@ -1681,7 +1681,7 @@ v_cmp_ge_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x36,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x36,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 ttmp15, src_scc, vcc_lo
@@ -1717,7 +1717,7 @@ v_cmp_ge_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x36,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s[10:11], exec_lo, -1
@@ -1737,7 +1737,7 @@ v_cmp_ge_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x36,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x36,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -1982,7 +1982,7 @@ v_cmp_ge_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x3e,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s5, exec_lo, -1
@@ -2002,7 +2002,7 @@ v_cmp_ge_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x3e,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x3e,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 ttmp15, src_scc, vcc_lo
@@ -2038,7 +2038,7 @@ v_cmp_ge_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s[10:11], exec_lo, -1
@@ -2058,7 +2058,7 @@ v_cmp_ge_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x3e,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x3e,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -2624,7 +2624,7 @@ v_cmp_gt_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x34,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x34,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s5, exec_lo, -1
@@ -2644,7 +2644,7 @@ v_cmp_gt_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x34,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x34,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 ttmp15, src_scc, vcc_lo
@@ -2680,7 +2680,7 @@ v_cmp_gt_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x34,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s[10:11], exec_lo, -1
@@ -2700,7 +2700,7 @@ v_cmp_gt_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x34,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x34,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -2945,7 +2945,7 @@ v_cmp_gt_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x3c,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s5, exec_lo, -1
@@ -2965,7 +2965,7 @@ v_cmp_gt_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x3c,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x3c,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 ttmp15, src_scc, vcc_lo
@@ -3001,7 +3001,7 @@ v_cmp_gt_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s[10:11], exec_lo, -1
@@ -3021,7 +3021,7 @@ v_cmp_gt_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x3c,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x3c,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -3587,7 +3587,7 @@ v_cmp_le_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x33,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x33,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s5, exec_lo, -1
@@ -3607,7 +3607,7 @@ v_cmp_le_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x33,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x33,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 ttmp15, src_scc, vcc_lo
@@ -3643,7 +3643,7 @@ v_cmp_le_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x33,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s[10:11], exec_lo, -1
@@ -3663,7 +3663,7 @@ v_cmp_le_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x33,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x33,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -3908,7 +3908,7 @@ v_cmp_le_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x3b,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s5, exec_lo, -1
@@ -3928,7 +3928,7 @@ v_cmp_le_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x3b,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x3b,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 ttmp15, src_scc, vcc_lo
@@ -3964,7 +3964,7 @@ v_cmp_le_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s[10:11], exec_lo, -1
@@ -3984,7 +3984,7 @@ v_cmp_le_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x3b,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x3b,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -4871,7 +4871,7 @@ v_cmp_lt_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x31,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x31,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s5, exec_lo, -1
@@ -4891,7 +4891,7 @@ v_cmp_lt_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x31,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x31,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 ttmp15, src_scc, vcc_lo
@@ -4927,7 +4927,7 @@ v_cmp_lt_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x31,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s[10:11], exec_lo, -1
@@ -4947,7 +4947,7 @@ v_cmp_lt_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x31,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x31,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -5192,7 +5192,7 @@ v_cmp_lt_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x39,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x39,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s5, exec_lo, -1
@@ -5212,7 +5212,7 @@ v_cmp_lt_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x39,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x39,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 ttmp15, src_scc, vcc_lo
@@ -5248,7 +5248,7 @@ v_cmp_lt_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x39,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s[10:11], exec_lo, -1
@@ -5268,7 +5268,7 @@ v_cmp_lt_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x39,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x39,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -5513,7 +5513,7 @@ v_cmp_ne_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x35,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x35,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s5, exec_lo, -1
@@ -5533,7 +5533,7 @@ v_cmp_ne_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x35,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x35,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 ttmp15, src_scc, vcc_lo
@@ -5569,7 +5569,7 @@ v_cmp_ne_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x35,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s[10:11], exec_lo, -1
@@ -5589,7 +5589,7 @@ v_cmp_ne_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x35,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x35,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -5834,7 +5834,7 @@ v_cmp_ne_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x3d,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s5, exec_lo, -1
@@ -5854,7 +5854,7 @@ v_cmp_ne_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x3d,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x3d,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 ttmp15, src_scc, vcc_lo
@@ -5890,7 +5890,7 @@ v_cmp_ne_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s[10:11], exec_lo, -1
@@ -5910,7 +5910,7 @@ v_cmp_ne_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x3d,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x3d,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 ttmp[14:15], src_scc, vcc_lo
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s
index 46ecb9f4d2ca..d0fe55ae0a7e 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s
@@ -284,7 +284,7 @@ v_cmpx_eq_i16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xb2,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_eq_i16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xb2,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb2,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_eq_i16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xb2,0xd4,0x7e,0x82,0x01,0x00]
@@ -299,7 +299,7 @@ v_cmpx_eq_i16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xb2,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_eq_i16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xb2,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb2,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_eq_i16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xb2,0xd4,0xfd,0xd4,0x00,0x00]
@@ -410,7 +410,7 @@ v_cmpx_eq_u16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xba,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_eq_u16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xba,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xba,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_eq_u16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xba,0xd4,0x7e,0x82,0x01,0x00]
@@ -425,7 +425,7 @@ v_cmpx_eq_u16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xba,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_eq_u16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xba,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xba,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_eq_u16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xba,0xd4,0xfd,0xd4,0x00,0x00]
@@ -662,7 +662,7 @@ v_cmpx_ge_i16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xb6,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_ge_i16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xb6,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb6,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_ge_i16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xb6,0xd4,0x7e,0x82,0x01,0x00]
@@ -677,7 +677,7 @@ v_cmpx_ge_i16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xb6,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_ge_i16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xb6,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb6,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_ge_i16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xb6,0xd4,0xfd,0xd4,0x00,0x00]
@@ -788,7 +788,7 @@ v_cmpx_ge_u16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xbe,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_ge_u16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xbe,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xbe,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_ge_u16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xbe,0xd4,0x7e,0x82,0x01,0x00]
@@ -803,7 +803,7 @@ v_cmpx_ge_u16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xbe,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_ge_u16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xbe,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xbe,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_ge_u16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xbe,0xd4,0xfd,0xd4,0x00,0x00]
@@ -1040,7 +1040,7 @@ v_cmpx_gt_i16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xb4,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_gt_i16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xb4,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb4,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_gt_i16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xb4,0xd4,0x7e,0x82,0x01,0x00]
@@ -1055,7 +1055,7 @@ v_cmpx_gt_i16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xb4,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_gt_i16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xb4,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb4,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_gt_i16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xb4,0xd4,0xfd,0xd4,0x00,0x00]
@@ -1166,7 +1166,7 @@ v_cmpx_gt_u16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xbc,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_gt_u16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xbc,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xbc,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_gt_u16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xbc,0xd4,0x7e,0x82,0x01,0x00]
@@ -1181,7 +1181,7 @@ v_cmpx_gt_u16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xbc,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_gt_u16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xbc,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xbc,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_gt_u16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xbc,0xd4,0xfd,0xd4,0x00,0x00]
@@ -1418,7 +1418,7 @@ v_cmpx_le_i16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xb3,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_le_i16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xb3,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb3,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_le_i16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xb3,0xd4,0x7e,0x82,0x01,0x00]
@@ -1433,7 +1433,7 @@ v_cmpx_le_i16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xb3,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_le_i16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xb3,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb3,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_le_i16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xb3,0xd4,0xfd,0xd4,0x00,0x00]
@@ -1544,7 +1544,7 @@ v_cmpx_le_u16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xbb,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_le_u16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xbb,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xbb,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_le_u16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xbb,0xd4,0x7e,0x82,0x01,0x00]
@@ -1559,7 +1559,7 @@ v_cmpx_le_u16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xbb,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_le_u16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xbb,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xbb,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_le_u16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xbb,0xd4,0xfd,0xd4,0x00,0x00]
@@ -1922,7 +1922,7 @@ v_cmpx_lt_i16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xb1,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_lt_i16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xb1,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb1,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_lt_i16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xb1,0xd4,0x7e,0x82,0x01,0x00]
@@ -1937,7 +1937,7 @@ v_cmpx_lt_i16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xb1,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_lt_i16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xb1,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb1,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_lt_i16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xb1,0xd4,0xfd,0xd4,0x00,0x00]
@@ -2048,7 +2048,7 @@ v_cmpx_lt_u16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xb9,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_lt_u16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xb9,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb9,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_lt_u16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xb9,0xd4,0x7e,0x82,0x01,0x00]
@@ -2063,7 +2063,7 @@ v_cmpx_lt_u16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xb9,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_lt_u16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xb9,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb9,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_lt_u16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xb9,0xd4,0xfd,0xd4,0x00,0x00]
@@ -2174,7 +2174,7 @@ v_cmpx_ne_i16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xb5,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_ne_i16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xb5,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb5,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_ne_i16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xb5,0xd4,0x7e,0x82,0x01,0x00]
@@ -2189,7 +2189,7 @@ v_cmpx_ne_i16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xb5,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_ne_i16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xb5,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb5,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_ne_i16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xb5,0xd4,0xfd,0xd4,0x00,0x00]
@@ -2300,7 +2300,7 @@ v_cmpx_ne_u16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xbd,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_ne_u16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xbd,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xbd,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_ne_u16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xbd,0xd4,0x7e,0x82,0x01,0x00]
@@ -2315,7 +2315,7 @@ v_cmpx_ne_u16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xbd,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_ne_u16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xbd,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xbd,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_ne_u16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xbd,0xd4,0xfd,0xd4,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopc.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopc.s
index e0d48972575e..3ae0c0f314d9 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vopc.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopc.s
@@ -724,7 +724,7 @@ v_cmp_eq_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x64,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x64,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc_lo, src_scc, v2
@@ -784,7 +784,7 @@ v_cmp_eq_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x64,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x64,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc, src_scc, v2
@@ -1060,7 +1060,7 @@ v_cmp_eq_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x74,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x74,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc_lo, src_scc, v2
@@ -1120,7 +1120,7 @@ v_cmp_eq_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x74,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x74,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc, src_scc, v2
@@ -1732,7 +1732,7 @@ v_cmp_ge_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x6c,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x6c,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc_lo, src_scc, v2
@@ -1792,7 +1792,7 @@ v_cmp_ge_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x6c,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x6c,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc, src_scc, v2
@@ -2068,7 +2068,7 @@ v_cmp_ge_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x7c,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x7c,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc_lo, src_scc, v2
@@ -2128,7 +2128,7 @@ v_cmp_ge_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x7c,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x7c,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc, src_scc, v2
@@ -2740,7 +2740,7 @@ v_cmp_gt_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x68,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x68,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc_lo, src_scc, v2
@@ -2800,7 +2800,7 @@ v_cmp_gt_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x68,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x68,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc, src_scc, v2
@@ -3076,7 +3076,7 @@ v_cmp_gt_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x78,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x78,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc_lo, src_scc, v2
@@ -3136,7 +3136,7 @@ v_cmp_gt_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x78,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x78,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc, src_scc, v2
@@ -3748,7 +3748,7 @@ v_cmp_le_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x66,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x66,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc_lo, src_scc, v2
@@ -3808,7 +3808,7 @@ v_cmp_le_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x66,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x66,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc, src_scc, v2
@@ -4084,7 +4084,7 @@ v_cmp_le_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x76,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x76,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc_lo, src_scc, v2
@@ -4144,7 +4144,7 @@ v_cmp_le_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x76,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x76,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc, src_scc, v2
@@ -5092,7 +5092,7 @@ v_cmp_lt_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x62,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x62,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc_lo, src_scc, v2
@@ -5152,7 +5152,7 @@ v_cmp_lt_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x62,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x62,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc, src_scc, v2
@@ -5428,7 +5428,7 @@ v_cmp_lt_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x72,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x72,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc_lo, src_scc, v2
@@ -5488,7 +5488,7 @@ v_cmp_lt_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x72,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x72,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc, src_scc, v2
@@ -5764,7 +5764,7 @@ v_cmp_ne_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x6a,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x6a,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc_lo, src_scc, v2
@@ -5824,7 +5824,7 @@ v_cmp_ne_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x6a,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x6a,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc, src_scc, v2
@@ -6100,7 +6100,7 @@ v_cmp_ne_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x7a,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x7a,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc_lo, src_scc, v2
@@ -6160,7 +6160,7 @@ v_cmp_ne_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x7a,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x7a,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc, src_scc, v2
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s
index ffbf50cce76e..68c41abaf05f 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s
@@ -290,7 +290,7 @@ v_cmpx_eq_i16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x64,0x7d]
 
 v_cmpx_eq_i16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x64,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x64,0x7d]
 
 v_cmpx_eq_i16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x64,0x7d]
@@ -416,7 +416,7 @@ v_cmpx_eq_u16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x74,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x74,0x7d]
@@ -668,7 +668,7 @@ v_cmpx_ge_i16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x6c,0x7d]
 
 v_cmpx_ge_i16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x6c,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x6c,0x7d]
 
 v_cmpx_ge_i16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x6c,0x7d]
@@ -794,7 +794,7 @@ v_cmpx_ge_u16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x7c,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x7c,0x7d]
@@ -1046,7 +1046,7 @@ v_cmpx_gt_i16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x68,0x7d]
 
 v_cmpx_gt_i16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x68,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x68,0x7d]
 
 v_cmpx_gt_i16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x68,0x7d]
@@ -1172,7 +1172,7 @@ v_cmpx_gt_u16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x78,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x78,0x7d]
@@ -1424,7 +1424,7 @@ v_cmpx_le_i16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x66,0x7d]
 
 v_cmpx_le_i16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x66,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x66,0x7d]
 
 v_cmpx_le_i16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x66,0x7d]
@@ -1550,7 +1550,7 @@ v_cmpx_le_u16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x76,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x76,0x7d]
@@ -1928,7 +1928,7 @@ v_cmpx_lt_i16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x62,0x7d]
 
 v_cmpx_lt_i16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x62,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x62,0x7d]
 
 v_cmpx_lt_i16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x62,0x7d]
@@ -2054,7 +2054,7 @@ v_cmpx_lt_u16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x72,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x72,0x7d]
@@ -2180,7 +2180,7 @@ v_cmpx_ne_i16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x6a,0x7d]
 
 v_cmpx_ne_i16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x6a,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x6a,0x7d]
 
 v_cmpx_ne_i16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x6a,0x7d]
@@ -2306,7 +2306,7 @@ v_cmpx_ne_u16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x7a,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x7a,0x7d]
diff --git a/llvm/test/MC/AMDGPU/gfx8_asm_vop1.s b/llvm/test/MC/AMDGPU/gfx8_asm_vop1.s
index 32ebbd9e1028..52a7762f7fde 100644
--- a/llvm/test/MC/AMDGPU/gfx8_asm_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx8_asm_vop1.s
@@ -3958,10 +3958,10 @@ v_cvt_f16_u16 v5, -1
 // CHECK: [0xc1,0x72,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, 0.5
-// CHECK: [0xff,0x72,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x72,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, -4.0
-// CHECK: [0xff,0x72,0x0a,0x7e,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x72,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, src_vccz
 // CHECK: [0xfb,0x72,0x0a,0x7e]
@@ -4039,10 +4039,10 @@ v_cvt_f16_i16 v5, -1
 // CHECK: [0xc1,0x74,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, 0.5
-// CHECK: [0xff,0x74,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x74,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, -4.0
-// CHECK: [0xff,0x74,0x0a,0x7e,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x74,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, src_vccz
 // CHECK: [0xfb,0x74,0x0a,0x7e]
diff --git a/llvm/test/MC/AMDGPU/gfx8_asm_vop2.s b/llvm/test/MC/AMDGPU/gfx8_asm_vop2.s
index 1c355af95028..5fc7f406a716 100644
--- a/llvm/test/MC/AMDGPU/gfx8_asm_vop2.s
+++ b/llvm/test/MC/AMDGPU/gfx8_asm_vop2.s
@@ -2785,10 +2785,10 @@ v_add_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x4c]
 
 v_add_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x4c,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x4c]
 
 v_add_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x4c,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x4c]
 
 v_add_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x4c]
@@ -2869,10 +2869,10 @@ v_sub_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x4e]
 
 v_sub_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x4e,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x4e]
 
 v_sub_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x4e,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x4e]
 
 v_sub_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x4e]
@@ -2953,10 +2953,10 @@ v_subrev_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x50]
 
 v_subrev_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x50,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x50]
 
 v_subrev_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x50,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x50]
 
 v_subrev_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x50]
@@ -3034,10 +3034,10 @@ v_mul_lo_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x52]
 
 v_mul_lo_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x52,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x52]
 
 v_mul_lo_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x52,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x52]
 
 v_mul_lo_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x52]
@@ -3118,10 +3118,10 @@ v_lshlrev_b16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x54]
 
 v_lshlrev_b16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x54,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x54]
 
 v_lshlrev_b16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x54,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x54]
 
 v_lshlrev_b16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x54]
@@ -3199,10 +3199,10 @@ v_lshrrev_b16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x56]
 
 v_lshrrev_b16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x56,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x56]
 
 v_lshrrev_b16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x56,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x56]
 
 v_lshrrev_b16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x56]
@@ -3280,10 +3280,10 @@ v_ashrrev_i16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x58]
 
 v_ashrrev_i16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x58,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x58]
 
 v_ashrrev_i16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x58,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x58]
 
 v_ashrrev_i16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x58]
@@ -3529,10 +3529,10 @@ v_max_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x5e]
 
 v_max_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x5e,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x5e]
 
 v_max_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x5e,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x5e]
 
 v_max_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x5e]
@@ -3613,10 +3613,10 @@ v_max_i16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x60]
 
 v_max_i16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x60,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x60]
 
 v_max_i16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x60,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x60]
 
 v_max_i16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x60]
@@ -3697,10 +3697,10 @@ v_min_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x62]
 
 v_min_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x62,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x62]
 
 v_min_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x62,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x62]
 
 v_min_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x62]
@@ -3781,10 +3781,10 @@ v_min_i16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x64]
 
 v_min_i16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x64,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x64]
 
 v_min_i16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x64,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x64]
 
 v_min_i16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x64]
diff --git a/llvm/test/MC/AMDGPU/gfx8_asm_vopc.s b/llvm/test/MC/AMDGPU/gfx8_asm_vopc.s
index adbe3f6a2683..8a431727604b 100644
--- a/llvm/test/MC/AMDGPU/gfx8_asm_vopc.s
+++ b/llvm/test/MC/AMDGPU/gfx8_asm_vopc.s
@@ -7705,10 +7705,10 @@ v_cmp_f_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x40,0x7d]
 
 v_cmp_f_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x40,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x40,0x7d]
 
 v_cmp_f_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x40,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x40,0x7d]
 
 v_cmp_f_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x40,0x7d]
@@ -7786,10 +7786,10 @@ v_cmp_lt_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x42,0x7d]
 
 v_cmp_lt_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x42,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x42,0x7d]
 
 v_cmp_lt_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x42,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x42,0x7d]
 
 v_cmp_lt_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x42,0x7d]
@@ -7867,10 +7867,10 @@ v_cmp_eq_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x44,0x7d]
 
 v_cmp_eq_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x44,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x44,0x7d]
 
 v_cmp_eq_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x44,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x44,0x7d]
 
 v_cmp_eq_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x44,0x7d]
@@ -7948,10 +7948,10 @@ v_cmp_le_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x46,0x7d]
 
 v_cmp_le_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x46,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x46,0x7d]
 
 v_cmp_le_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x46,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x46,0x7d]
 
 v_cmp_le_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x46,0x7d]
@@ -8029,10 +8029,10 @@ v_cmp_gt_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x48,0x7d]
 
 v_cmp_gt_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x48,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x48,0x7d]
 
 v_cmp_gt_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x48,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x48,0x7d]
 
 v_cmp_gt_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x48,0x7d]
@@ -8110,10 +8110,10 @@ v_cmp_ne_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x4a,0x7d]
 
 v_cmp_ne_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x4a,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x4a,0x7d]
 
 v_cmp_ne_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x4a,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x4a,0x7d]
 
 v_cmp_ne_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x4a,0x7d]
@@ -8191,10 +8191,10 @@ v_cmp_ge_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x4c,0x7d]
 
 v_cmp_ge_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x4c,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x4c,0x7d]
 
 v_cmp_ge_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x4c,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x4c,0x7d]
 
 v_cmp_ge_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x4c,0x7d]
@@ -8272,10 +8272,10 @@ v_cmp_t_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x4e,0x7d]
 
 v_cmp_t_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x4e,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x4e,0x7d]
 
 v_cmp_t_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x4e,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x4e,0x7d]
 
 v_cmp_t_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x4e,0x7d]
@@ -8353,10 +8353,10 @@ v_cmp_f_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x50,0x7d]
 
 v_cmp_f_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x50,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x50,0x7d]
 
 v_cmp_f_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x50,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x50,0x7d]
 
 v_cmp_f_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x50,0x7d]
@@ -8434,10 +8434,10 @@ v_cmp_lt_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x52,0x7d]
 
 v_cmp_lt_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x52,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x52,0x7d]
 
 v_cmp_lt_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x52,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x52,0x7d]
 
 v_cmp_lt_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x52,0x7d]
@@ -8515,10 +8515,10 @@ v_cmp_eq_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x54,0x7d]
 
 v_cmp_eq_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x54,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x54,0x7d]
 
 v_cmp_eq_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x54,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x54,0x7d]
 
 v_cmp_eq_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x54,0x7d]
@@ -8596,10 +8596,10 @@ v_cmp_le_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x56,0x7d]
 
 v_cmp_le_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x56,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x56,0x7d]
 
 v_cmp_le_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x56,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x56,0x7d]
 
 v_cmp_le_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x56,0x7d]
@@ -8677,10 +8677,10 @@ v_cmp_gt_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x58,0x7d]
 
 v_cmp_gt_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x58,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x58,0x7d]
 
 v_cmp_gt_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x58,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x58,0x7d]
 
 v_cmp_gt_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x58,0x7d]
@@ -8758,10 +8758,10 @@ v_cmp_ne_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x5a,0x7d]
 
 v_cmp_ne_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x5a,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x5a,0x7d]
 
 v_cmp_ne_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x5a,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x5a,0x7d]
 
 v_cmp_ne_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x5a,0x7d]
@@ -8839,10 +8839,10 @@ v_cmp_ge_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x5c,0x7d]
 
 v_cmp_ge_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x5c,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x5c,0x7d]
 
 v_cmp_ge_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x5c,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x5c,0x7d]
 
 v_cmp_ge_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x5c,0x7d]
@@ -8920,10 +8920,10 @@ v_cmp_t_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x5e,0x7d]
 
 v_cmp_t_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x5e,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x5e,0x7d]
 
 v_cmp_t_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x5e,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x5e,0x7d]
 
 v_cmp_t_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x5e,0x7d]
@@ -9001,10 +9001,10 @@ v_cmpx_f_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x60,0x7d]
 
 v_cmpx_f_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x60,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x60,0x7d]
 
 v_cmpx_f_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x60,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x60,0x7d]
 
 v_cmpx_f_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x60,0x7d]
@@ -9082,10 +9082,10 @@ v_cmpx_lt_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x62,0x7d]
 
 v_cmpx_lt_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x62,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x62,0x7d]
 
 v_cmpx_lt_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x62,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x62,0x7d]
 
 v_cmpx_lt_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x62,0x7d]
@@ -9163,10 +9163,10 @@ v_cmpx_eq_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x64,0x7d]
 
 v_cmpx_eq_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x64,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x64,0x7d]
 
 v_cmpx_eq_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x64,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x64,0x7d]
 
 v_cmpx_eq_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x64,0x7d]
@@ -9244,10 +9244,10 @@ v_cmpx_le_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x66,0x7d]
 
 v_cmpx_le_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x66,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x66,0x7d]
 
 v_cmpx_le_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x66,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x66,0x7d]
 
 v_cmpx_le_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x66,0x7d]
@@ -9325,10 +9325,10 @@ v_cmpx_gt_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x68,0x7d]
 
 v_cmpx_gt_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x68,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x68,0x7d]
 
 v_cmpx_gt_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x68,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x68,0x7d]
 
 v_cmpx_gt_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x68,0x7d]
@@ -9406,10 +9406,10 @@ v_cmpx_ne_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x6a,0x7d]
 
 v_cmpx_ne_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x6a,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x6a,0x7d]
 
 v_cmpx_ne_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x6a,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x6a,0x7d]
 
 v_cmpx_ne_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x6a,0x7d]
@@ -9487,10 +9487,10 @@ v_cmpx_ge_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x6c,0x7d]
 
 v_cmpx_ge_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x6c,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x6c,0x7d]
 
 v_cmpx_ge_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x6c,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x6c,0x7d]
 
 v_cmpx_ge_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x6c,0x7d]
@@ -9568,10 +9568,10 @@ v_cmpx_t_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x6e,0x7d]
 
 v_cmpx_t_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x6e,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x6e,0x7d]
 
 v_cmpx_t_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x6e,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x6e,0x7d]
 
 v_cmpx_t_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x6e,0x7d]
@@ -9649,10 +9649,10 @@ v_cmpx_f_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x70,0x7d]
 
 v_cmpx_f_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x70,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x70,0x7d]
 
 v_cmpx_f_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x70,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x70,0x7d]
 
 v_cmpx_f_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x70,0x7d]
@@ -9730,10 +9730,10 @@ v_cmpx_lt_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x72,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x72,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x72,0x7d]
@@ -9811,10 +9811,10 @@ v_cmpx_eq_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x74,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x74,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x74,0x7d]
@@ -9892,10 +9892,10 @@ v_cmpx_le_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x76,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x76,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x76,0x7d]
@@ -9973,10 +9973,10 @@ v_cmpx_gt_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x78,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x78,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x78,0x7d]
@@ -10054,10 +10054,10 @@ v_cmpx_ne_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x7a,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x7a,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x7a,0x7d]
@@ -10135,10 +10135,10 @@ v_cmpx_ge_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x7c,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x7c,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x7c,0x7d]
@@ -10216,10 +10216,10 @@ v_cmpx_t_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x7e,0x7d]
 
 v_cmpx_t_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x7e,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x7e,0x7d]
 
 v_cmpx_t_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x7e,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x7e,0x7d]
 
 v_cmpx_t_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x7e,0x7d]
diff --git a/llvm/test/MC/AMDGPU/gfx9-asm-err.s b/llvm/test/MC/AMDGPU/gfx9-asm-err.s
index 451fab99e12a..93138a829185 100644
--- a/llvm/test/MC/AMDGPU/gfx9-asm-err.s
+++ b/llvm/test/MC/AMDGPU/gfx9-asm-err.s
@@ -1,33 +1,33 @@
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx900 %s 2>&1 | FileCheck -check-prefix=GFX9ERR --implicit-check-not=error: %s
 
-v_cvt_f16_u16_e64 v5, 0.5
+v_cvt_f16_u16_e64 v5, 0.6
 // GFX9ERR: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
 
-v_cvt_f16_u16_e64 v5, -4.0
+v_cvt_f16_u16_e64 v5, -4.1
 // GFX9ERR: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
 
-v_add_u16_e64 v5, v1, 0.5
+v_add_u16_e64 v5, v1, 0.4
 // GFX9ERR: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
 
-v_add_u16_e64 v5, v1, -4.0
+v_add_u16_e64 v5, v1, -4.2
 // GFX9ERR: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
 
-v_cvt_f16_i16_e64 v5, 0.5
+v_cvt_f16_i16_e64 v5, 0.3
 // GFX9ERR: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
 
-v_cvt_f16_i16_e64 v5, -4.0
+v_cvt_f16_i16_e64 v5, -4.5
 // GFX9ERR: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
 
-v_add_u16_e64 v5, 0.5, v2
+v_add_u16_e64 v5, 0.1, v2
 // GFX9ERR: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
 
-v_add_u16_e64 v5, -4.0, v2
+v_add_u16_e64 v5, -3.8, v2
 // GFX9ERR: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
 
-v_subrev_u16_e64 v5, v1, 0.5
+v_subrev_u16_e64 v5, v1, 0.4
 // GFX9ERR: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
 
-v_subrev_u16_e64 v5, v1, -4.0
+v_subrev_u16_e64 v5, v1, -4.2
 // GFX9ERR: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
 
 v_cvt_u32_f64 v5, v[0:1] quad_perm:[0,2,1,1] row_mask:0xf bank_mask:0xf
diff --git a/llvm/test/MC/AMDGPU/gfx9_asm_vop1.s b/llvm/test/MC/AMDGPU/gfx9_asm_vop1.s
index 038b4d2185a4..5227fcd9d22a 100644
--- a/llvm/test/MC/AMDGPU/gfx9_asm_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx9_asm_vop1.s
@@ -3436,10 +3436,10 @@ v_cvt_f16_u16 v5, -1
 // CHECK: [0xc1,0x72,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, 0.5
-// CHECK: [0xff,0x72,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x72,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, -4.0
-// CHECK: [0xff,0x72,0x0a,0x7e,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x72,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, src_vccz
 // CHECK: [0xfb,0x72,0x0a,0x7e]
@@ -3505,10 +3505,10 @@ v_cvt_f16_i16 v5, -1
 // CHECK: [0xc1,0x74,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, 0.5
-// CHECK: [0xff,0x74,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x74,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, -4.0
-// CHECK: [0xff,0x74,0x0a,0x7e,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x74,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, src_vccz
 // CHECK: [0xfb,0x74,0x0a,0x7e]
diff --git a/llvm/test/MC/AMDGPU/gfx9_asm_vop2.s b/llvm/test/MC/AMDGPU/gfx9_asm_vop2.s
index 7b53da641b76..fe7fc66d95a2 100644
--- a/llvm/test/MC/AMDGPU/gfx9_asm_vop2.s
+++ b/llvm/test/MC/AMDGPU/gfx9_asm_vop2.s
@@ -1,5 +1,7 @@
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx900 -show-encoding %s | FileCheck %s
 
+.text
+
 v_cndmask_b32 v5, v1, v2, vcc
 // CHECK: [0x01,0x05,0x0a,0x00]
 
@@ -2419,10 +2421,10 @@ v_add_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x4c]
 
 v_add_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x4c,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x4c]
 
 v_add_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x4c,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x4c]
 
 v_add_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x4c]
@@ -2491,10 +2493,10 @@ v_sub_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x4e]
 
 v_sub_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x4e,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x4e]
 
 v_sub_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x4e,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x4e]
 
 v_sub_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x4e]
@@ -2563,10 +2565,10 @@ v_subrev_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x50]
 
 v_subrev_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x50,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x50]
 
 v_subrev_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x50,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x50]
 
 v_subrev_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x50]
@@ -2632,10 +2634,10 @@ v_mul_lo_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x52]
 
 v_mul_lo_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x52,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x52]
 
 v_mul_lo_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x52,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x52]
 
 v_mul_lo_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x52]
@@ -2704,10 +2706,10 @@ v_lshlrev_b16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x54]
 
 v_lshlrev_b16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x54,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x54]
 
 v_lshlrev_b16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x54,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x54]
 
 v_lshlrev_b16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x54]
@@ -2773,10 +2775,10 @@ v_lshrrev_b16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x56]
 
 v_lshrrev_b16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x56,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x56]
 
 v_lshrrev_b16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x56,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x56]
 
 v_lshrrev_b16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x56]
@@ -2842,10 +2844,10 @@ v_ashrrev_i16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x58]
 
 v_ashrrev_i16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x58,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x58]
 
 v_ashrrev_i16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x58,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x58]
 
 v_ashrrev_i16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x58]
@@ -3055,10 +3057,10 @@ v_max_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x5e]
 
 v_max_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x5e,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x5e]
 
 v_max_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x5e,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x5e]
 
 v_max_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x5e]
@@ -3127,10 +3129,10 @@ v_max_i16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x60]
 
 v_max_i16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x60,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x60]
 
 v_max_i16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x60,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x60]
 
 v_max_i16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x60]
@@ -3199,10 +3201,10 @@ v_min_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x62]
 
 v_min_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x62,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x62]
 
 v_min_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x62,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x62]
 
 v_min_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x62]
@@ -3271,10 +3273,10 @@ v_min_i16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x64]
 
 v_min_i16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x64,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x64]
 
 v_min_i16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x64,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x64]
 
 v_min_i16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x64]
diff --git a/llvm/test/MC/AMDGPU/gfx9_asm_vopc.s b/llvm/test/MC/AMDGPU/gfx9_asm_vopc.s
index 345afa2149dc..bcb717096607 100644
--- a/llvm/test/MC/AMDGPU/gfx9_asm_vopc.s
+++ b/llvm/test/MC/AMDGPU/gfx9_asm_vopc.s
@@ -6673,10 +6673,10 @@ v_cmp_f_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x40,0x7d]
 
 v_cmp_f_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x40,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x40,0x7d]
 
 v_cmp_f_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x40,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x40,0x7d]
 
 v_cmp_f_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x40,0x7d]
@@ -6742,10 +6742,10 @@ v_cmp_lt_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x42,0x7d]
 
 v_cmp_lt_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x42,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x42,0x7d]
 
 v_cmp_lt_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x42,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x42,0x7d]
 
 v_cmp_lt_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x42,0x7d]
@@ -6811,10 +6811,10 @@ v_cmp_eq_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x44,0x7d]
 
 v_cmp_eq_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x44,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x44,0x7d]
 
 v_cmp_eq_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x44,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x44,0x7d]
 
 v_cmp_eq_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x44,0x7d]
@@ -6880,10 +6880,10 @@ v_cmp_le_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x46,0x7d]
 
 v_cmp_le_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x46,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x46,0x7d]
 
 v_cmp_le_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x46,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x46,0x7d]
 
 v_cmp_le_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x46,0x7d]
@@ -6949,10 +6949,10 @@ v_cmp_gt_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x48,0x7d]
 
 v_cmp_gt_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x48,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x48,0x7d]
 
 v_cmp_gt_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x48,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x48,0x7d]
 
 v_cmp_gt_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x48,0x7d]
@@ -7018,10 +7018,10 @@ v_cmp_ne_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x4a,0x7d]
 
 v_cmp_ne_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x4a,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x4a,0x7d]
 
 v_cmp_ne_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x4a,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x4a,0x7d]
 
 v_cmp_ne_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x4a,0x7d]
@@ -7087,10 +7087,10 @@ v_cmp_ge_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x4c,0x7d]
 
 v_cmp_ge_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x4c,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x4c,0x7d]
 
 v_cmp_ge_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x4c,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x4c,0x7d]
 
 v_cmp_ge_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x4c,0x7d]
@@ -7156,10 +7156,10 @@ v_cmp_t_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x4e,0x7d]
 
 v_cmp_t_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x4e,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x4e,0x7d]
 
 v_cmp_t_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x4e,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x4e,0x7d]
 
 v_cmp_t_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x4e,0x7d]
@@ -7225,10 +7225,10 @@ v_cmp_f_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x50,0x7d]
 
 v_cmp_f_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x50,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x50,0x7d]
 
 v_cmp_f_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x50,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x50,0x7d]
 
 v_cmp_f_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x50,0x7d]
@@ -7294,10 +7294,10 @@ v_cmp_lt_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x52,0x7d]
 
 v_cmp_lt_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x52,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x52,0x7d]
 
 v_cmp_lt_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x52,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x52,0x7d]
 
 v_cmp_lt_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x52,0x7d]
@@ -7363,10 +7363,10 @@ v_cmp_eq_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x54,0x7d]
 
 v_cmp_eq_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x54,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x54,0x7d]
 
 v_cmp_eq_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x54,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x54,0x7d]
 
 v_cmp_eq_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x54,0x7d]
@@ -7432,10 +7432,10 @@ v_cmp_le_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x56,0x7d]
 
 v_cmp_le_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x56,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x56,0x7d]
 
 v_cmp_le_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x56,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x56,0x7d]
 
 v_cmp_le_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x56,0x7d]
@@ -7501,10 +7501,10 @@ v_cmp_gt_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x58,0x7d]
 
 v_cmp_gt_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x58,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x58,0x7d]
 
 v_cmp_gt_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x58,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x58,0x7d]
 
 v_cmp_gt_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x58,0x7d]
@@ -7570,10 +7570,10 @@ v_cmp_ne_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x5a,0x7d]
 
 v_cmp_ne_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x5a,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x5a,0x7d]
 
 v_cmp_ne_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x5a,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x5a,0x7d]
 
 v_cmp_ne_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x5a,0x7d]
@@ -7639,10 +7639,10 @@ v_cmp_ge_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x5c,0x7d]
 
 v_cmp_ge_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x5c,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x5c,0x7d]
 
 v_cmp_ge_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x5c,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x5c,0x7d]
 
 v_cmp_ge_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x5c,0x7d]
@@ -7708,10 +7708,10 @@ v_cmp_t_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x5e,0x7d]
 
 v_cmp_t_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x5e,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x5e,0x7d]
 
 v_cmp_t_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x5e,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x5e,0x7d]
 
 v_cmp_t_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x5e,0x7d]
@@ -7777,10 +7777,10 @@ v_cmpx_f_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x60,0x7d]
 
 v_cmpx_f_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x60,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x60,0x7d]
 
 v_cmpx_f_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x60,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x60,0x7d]
 
 v_cmpx_f_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x60,0x7d]
@@ -7846,10 +7846,10 @@ v_cmpx_lt_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x62,0x7d]
 
 v_cmpx_lt_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x62,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x62,0x7d]
 
 v_cmpx_lt_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x62,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x62,0x7d]
 
 v_cmpx_lt_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x62,0x7d]
@@ -7915,10 +7915,10 @@ v_cmpx_eq_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x64,0x7d]
 
 v_cmpx_eq_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x64,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x64,0x7d]
 
 v_cmpx_eq_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x64,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x64,0x7d]
 
 v_cmpx_eq_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x64,0x7d]
@@ -7984,10 +7984,10 @@ v_cmpx_le_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x66,0x7d]
 
 v_cmpx_le_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x66,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x66,0x7d]
 
 v_cmpx_le_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x66,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x66,0x7d]
 
 v_cmpx_le_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x66,0x7d]
@@ -8053,10 +8053,10 @@ v_cmpx_gt_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x68,0x7d]
 
 v_cmpx_gt_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x68,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x68,0x7d]
 
 v_cmpx_gt_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x68,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x68,0x7d]
 
 v_cmpx_gt_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x68,0x7d]
@@ -8122,10 +8122,10 @@ v_cmpx_ne_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x6a,0x7d]
 
 v_cmpx_ne_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x6a,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x6a,0x7d]
 
 v_cmpx_ne_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x6a,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x6a,0x7d]
 
 v_cmpx_ne_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x6a,0x7d]
@@ -8191,10 +8191,10 @@ v_cmpx_ge_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x6c,0x7d]
 
 v_cmpx_ge_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x6c,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x6c,0x7d]
 
 v_cmpx_ge_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x6c,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x6c,0x7d]
 
 v_cmpx_ge_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x6c,0x7d]
@@ -8260,10 +8260,10 @@ v_cmpx_t_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x6e,0x7d]
 
 v_cmpx_t_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x6e,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x6e,0x7d]
 
 v_cmpx_t_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x6e,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x6e,0x7d]
 
 v_cmpx_t_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x6e,0x7d]
@@ -8329,10 +8329,10 @@ v_cmpx_f_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x70,0x7d]
 
 v_cmpx_f_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x70,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x70,0x7d]
 
 v_cmpx_f_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x70,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x70,0x7d]
 
 v_cmpx_f_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x70,0x7d]
@@ -8398,10 +8398,10 @@ v_cmpx_lt_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x72,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x72,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x72,0x7d]
@@ -8467,10 +8467,10 @@ v_cmpx_eq_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x74,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x74,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x74,0x7d]
@@ -8536,10 +8536,10 @@ v_cmpx_le_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x76,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x76,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x76,0x7d]
@@ -8605,10 +8605,10 @@ v_cmpx_gt_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x78,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x78,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x78,0x7d]
@@ -8674,10 +8674,10 @@ v_cmpx_ne_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x7a,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x7a,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x7a,0x7d]
@@ -8743,10 +8743,10 @@ v_cmpx_ge_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x7c,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x7c,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x7c,0x7d]
@@ -8812,10 +8812,10 @@ v_cmpx_t_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x7e,0x7d]
 
 v_cmpx_t_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x7e,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x7e,0x7d]
 
 v_cmpx_t_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x7e,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x7e,0x7d]
 
 v_cmpx_t_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x7e,0x7d]
diff --git a/llvm/test/MC/AMDGPU/gfx9_err_pos.s b/llvm/test/MC/AMDGPU/gfx9_err_pos.s
index 350a699fe0d7..578a063ebb70 100644
--- a/llvm/test/MC/AMDGPU/gfx9_err_pos.s
+++ b/llvm/test/MC/AMDGPU/gfx9_err_pos.s
@@ -168,16 +168,6 @@ v_bfe_u32 v0, v2, undef, v3
 // CHECK-NEXT:{{^}}v_bfe_u32 v0, v2, undef, v3
 // CHECK-NEXT:{{^}}                  ^
 
-v_add_i16 v5, v1, 0.5
-// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
-// CHECK-NEXT:{{^}}v_add_i16 v5, v1, 0.5
-// CHECK-NEXT:{{^}}                  ^
-
-v_add_i16 v5, 0.5, v2
-// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
-// CHECK-NEXT:{{^}}v_add_i16 v5, 0.5, v2
-// CHECK-NEXT:{{^}}              ^
-
 //==============================================================================
 // too few operands for instruction
 
diff --git a/llvm/test/MC/AMDGPU/vop3-gfx10.s b/llvm/test/MC/AMDGPU/vop3-gfx10.s
index 50810e82647b..2bd78ea80435 100644
--- a/llvm/test/MC/AMDGPU/vop3-gfx10.s
+++ b/llvm/test/MC/AMDGPU/vop3-gfx10.s
@@ -1,13 +1,13 @@
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1010 -show-encoding %s | FileCheck -check-prefix=GFX10 %s
 
 v_mad_i16 v5, v1, 4.0, v3
-// GFX10: v_mad_i16 v5, v1, 0x4400, v3    ; encoding: [0x05,0x00,0x5e,0xd7,0x01,0xff,0x0d,0x04,0x00,0x44,0x00,0x00]
+// GFX10: v_mad_i16 v5, v1, 4.0, v3 ; encoding: [0x05,0x00,0x5e,0xd7,0x01,0xed,0x0d,0x04]
 
 v_mad_i16 v5, v1, 0x4400, v3
 // GFX10: v_mad_i16 v5, v1, 0x4400, v3    ; encoding: [0x05,0x00,0x5e,0xd7,0x01,0xff,0x0d,0x04,0x00,0x44,0x00,0x00]
 
 v_mad_i16 v5, v1, -4.0, v3
-// GFX10: v_mad_i16 v5, v1, 0xc400, v3    ; encoding: [0x05,0x00,0x5e,0xd7,0x01,0xff,0x0d,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: v_mad_i16 v5, v1, -4.0, v3 ; encoding: [0x05,0x00,0x5e,0xd7,0x01,0xef,0x0d,0x04]
 
 v_mad_i16 v5, v1, 0xc400, v3
 // GFX10: v_mad_i16 v5, v1, 0xc400, v3    ; encoding: [0x05,0x00,0x5e,0xd7,0x01,0xff,0x0d,0x04,0x00,0xc4,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/vop3-gfx9.s b/llvm/test/MC/AMDGPU/vop3-gfx9.s
index b61f690d2023..a61b0c87e199 100644
--- a/llvm/test/MC/AMDGPU/vop3-gfx9.s
+++ b/llvm/test/MC/AMDGPU/vop3-gfx9.s
@@ -426,9 +426,9 @@ v_mad_i16 v5, v1, -1, v3
 // VI: v_mad_i16 v5, v1, -1, v3 ; encoding: [0x05,0x00,0xec,0xd1,0x01,0x83,0x0d,0x04]
 
 v_mad_i16 v5, v1, v2, -4.0
-// NOGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
+// GFX9: v_mad_i16 v5, v1, v2, -4.0 ; encoding: [0x05,0x00,0x05,0xd2,0x01,0x05,0xde,0x03]
 // NOSICI: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
-// NOVI: :[[@LINE-3]]:{{[0-9]+}}: error: literal operands are not supported
+// VI: v_mad_i16 v5, v1, v2, -4.0 ; encoding: [0x05,0x00,0xec,0xd1,0x01,0x05,0xde,0x03]
 
 v_mad_i16 v5, v1, v2, v3 clamp
 // GFX9: v_mad_i16 v5, v1, v2, v3 clamp ; encoding: [0x05,0x80,0x05,0xd2,0x01,0x05,0x0e,0x04]
@@ -478,11 +478,11 @@ v_mad_legacy_i16 v5, v1, -1, v3
 // NOGCN: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
 v_mad_legacy_i16 v5, v1, v2, -4.0
-// NOGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
+// GFX9: v_mad_legacy_i16 v5, v1, v2, -4.0 ; encoding: [0x05,0x00,0xec,0xd1,0x01,0x05,0xde,0x03]
 // NOGCN: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
 v_mad_legacy_i16 v5, v1, v2, -4.0 clamp
-// NOGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
+// GFX9: v_mad_legacy_i16 v5, v1, v2, -4.0 clamp ; encoding: [0x05,0x80,0xec,0xd1,0x01,0x05,0xde,0x03]
 // NOGCN: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
 v_mad_legacy_u16_e64 v5, 0, v2, v3
@@ -494,11 +494,11 @@ v_mad_legacy_u16 v5, v1, -1, v3
 // NOGCN: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
 v_mad_legacy_u16 v5, v1, v2, -4.0
-// NOGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
+// GFX9: v_mad_legacy_u16 v5, v1, v2, -4.0 ; encoding: [0x05,0x00,0xeb,0xd1,0x01,0x05,0xde,0x03]
 // NOGCN: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
 v_mad_legacy_u16 v5, v1, v2, -4.0 clamp
-// NOGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
+// GFX9: v_mad_legacy_u16 v5, v1, v2, -4.0 clamp ; encoding: [0x05,0x80,0xeb,0xd1,0x01,0x05,0xde,0x03]
 // NOGCN: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
 v_mad_u16_e64 v5, 0, v2, v3
@@ -512,9 +512,9 @@ v_mad_u16 v5, v1, -1, v3
 // VI: v_mad_u16 v5, v1, -1, v3 ; encoding: [0x05,0x00,0xeb,0xd1,0x01,0x83,0x0d,0x04]
 
 v_mad_u16 v5, v1, v2, -4.0
-// NOGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
+// GFX9: v_mad_u16 v5, v1, v2, -4.0 ; encoding: [0x05,0x00,0x04,0xd2,0x01,0x05,0xde,0x03]
 // NOSICI: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
-// NOVI: :[[@LINE-3]]:{{[0-9]+}}: error: literal operands are not supported
+// VI: v_mad_u16 v5, v1, v2, -4.0 ; encoding: [0x05,0x00,0xeb,0xd1,0x01,0x05,0xde,0x03]
 
 v_mad_u16 v5, v1, v2, v3 clamp
 // GFX9: v_mad_u16 v5, v1, v2, v3 clamp ; encoding: [0x05,0x80,0x04,0xd2,0x01,0x05,0x0e,0x04]
diff --git a/llvm/test/MC/AMDGPU/vop3.s b/llvm/test/MC/AMDGPU/vop3.s
index 27fcf5e1ba33..0d2544002a9f 100644
--- a/llvm/test/MC/AMDGPU/vop3.s
+++ b/llvm/test/MC/AMDGPU/vop3.s
@@ -599,7 +599,7 @@ v_mad_i16_e64 v5, -1, v2, v3
 // NOSICI: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
 v_mad_i16 v5, v1, -4.0, v3
-// NOVI: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
+// VI: v_mad_i16 v5, v1, -4.0, v3 ; encoding: [0x05,0x00,0xec,0xd1,0x01,0xef,0x0d,0x04]
 // NOSICI: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
 v_mad_i16 v5, v1, v2, 0
@@ -615,7 +615,7 @@ v_mad_u16 v5, v1, 0, v3
 // NOSICI: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
 v_mad_u16 v5, v1, v2, -4.0
-// NOVI: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
+// VI: v_mad_u16 v5, v1, v2, -4.0 ; encoding: [0x05,0x00,0xeb,0xd1,0x01,0x05,0xde,0x03]
 // NOSICI: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
 ///===---------------------------------------------------------------------===//
diff --git a/llvm/test/MC/AMDGPU/vop_sdwa.s b/llvm/test/MC/AMDGPU/vop_sdwa.s
index 6c8482aaddd9..974bb49174bd 100644
--- a/llvm/test/MC/AMDGPU/vop_sdwa.s
+++ b/llvm/test/MC/AMDGPU/vop_sdwa.s
@@ -963,12 +963,12 @@ v_exp_f16_sdwa v5, -|0.5|
 
 // NOSICI: :[[@LINE+3]]:{{[0-9]+}}: error: instruction not supported on this GPU
 // NOVI: :[[@LINE+2]]:{{[0-9]+}}: error: invalid operand for instruction
-// NOGFX9: :[[@LINE+1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX9: v_max_i16_sdwa v5, -4.0, v2 dst_sel:DWORD dst_unused:UNUSED_PRESERVE src0_sel:DWORD src1_sel:DWORD ; encoding: [0xf9,0x04,0x0a,0x60,0xf7,0x16,0x86,0x06]
 v_max_i16_sdwa v5, -4.0, v2 dst_sel:DWORD dst_unused:UNUSED_PRESERVE src0_sel:DWORD src1_sel:DWORD
 
 // NOSICI: :[[@LINE+3]]:{{[0-9]+}}: error: instruction not supported on this GPU
 // NOVI: :[[@LINE+2]]:{{[0-9]+}}: error: invalid operand for instruction
-// NOGFX9: :[[@LINE+1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX9: v_max_i16_sdwa v5, sext(-4.0), v2 dst_sel:DWORD dst_unused:UNUSED_PRESERVE src0_sel:DWORD src1_sel:DWORD ; encoding: [0xf9,0x04,0x0a,0x60,0xf7,0x16,0x8e,0x06]
 v_max_i16_sdwa v5, sext(-4.0), v2 dst_sel:DWORD dst_unused:UNUSED_PRESERVE src0_sel:DWORD src1_sel:DWORD
 
 // NOSICI: :[[@LINE+3]]:{{[0-9]+}}: error: instruction not supported on this GPU
-- 
GitLab


From 5f935e91810eb28854611faf13bb7d07a8dbf470 Mon Sep 17 00:00:00 2001
From: David Majnemer 
Date: Thu, 7 Mar 2024 19:35:38 +0000
Subject: [PATCH 654/929] [AArch64] Optimize fp64 <-> fp16 SIMD conversions

Legalization would result in needless scalarization. Add some
DAGCombines to fix this up.
---
 .../Target/AArch64/AArch64ISelLowering.cpp    | 95 ++++++++++++++++++-
 .../lib/Target/AArch64/AArch64InstrFormats.td |  4 +-
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   | 11 ++-
 llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll     | 56 +++++++++++
 .../CodeGen/AArch64/fp16-v8-instructions.ll   | 50 +++-------
 llvm/test/CodeGen/AArch64/fpext.ll            | 64 +++++++------
 llvm/test/CodeGen/AArch64/fptrunc.ll          | 20 +---
 ...e-streaming-mode-fixed-length-fcopysign.ll | 60 +++++-------
 llvm/test/CodeGen/AArch64/vector-fcopysign.ll | 62 ++++--------
 9 files changed, 252 insertions(+), 170 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 2290223a06f8..89b697b2d515 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -4507,13 +4507,16 @@ SDValue AArch64TargetLowering::LowerINT_TO_FP(SDValue Op,
   };
 
   if (Op.getValueType() == MVT::bf16) {
+    unsigned MaxWidth = IsSigned
+                            ? DAG.ComputeMaxSignificantBits(SrcVal)
+                            : DAG.computeKnownBits(SrcVal).countMaxActiveBits();
     // bf16 conversions are promoted to f32 when converting from i16.
-    if (DAG.ComputeMaxSignificantBits(SrcVal) <= 24) {
+    if (MaxWidth <= 24) {
       return IntToFpViaPromotion(MVT::f32);
     }
 
     // bf16 conversions are promoted to f64 when converting from i32.
-    if (DAG.ComputeMaxSignificantBits(SrcVal) <= 53) {
+    if (MaxWidth <= 53) {
       return IntToFpViaPromotion(MVT::f64);
     }
 
@@ -19376,6 +19379,94 @@ static SDValue performBuildVectorCombine(SDNode *N,
   SDLoc DL(N);
   EVT VT = N->getValueType(0);
 
+  if (VT == MVT::v4f16 || VT == MVT::v4bf16) {
+    SDValue Elt0 = N->getOperand(0), Elt1 = N->getOperand(1),
+            Elt2 = N->getOperand(2), Elt3 = N->getOperand(3);
+    if (Elt0->getOpcode() == ISD::FP_ROUND &&
+        Elt1->getOpcode() == ISD::FP_ROUND &&
+        isa(Elt0->getOperand(1)) &&
+        isa(Elt1->getOperand(1)) &&
+        Elt0->getConstantOperandVal(1) == Elt1->getConstantOperandVal(1) &&
+        Elt0->getOperand(0)->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+        Elt1->getOperand(0)->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+        // Constant index.
+        isa(Elt0->getOperand(0)->getOperand(1)) &&
+        isa(Elt1->getOperand(0)->getOperand(1)) &&
+        Elt0->getOperand(0)->getOperand(0) ==
+            Elt1->getOperand(0)->getOperand(0) &&
+        Elt0->getOperand(0)->getConstantOperandVal(1) == 0 &&
+        Elt1->getOperand(0)->getConstantOperandVal(1) == 1) {
+      SDValue LowLanesSrcVec = Elt0->getOperand(0)->getOperand(0);
+      if (LowLanesSrcVec.getValueType() == MVT::v2f64) {
+        SDValue HighLanes;
+        if (Elt2->getOpcode() == ISD::UNDEF &&
+            Elt3->getOpcode() == ISD::UNDEF) {
+          HighLanes = DAG.getUNDEF(MVT::v2f32);
+        } else if (Elt2->getOpcode() == ISD::FP_ROUND &&
+                   Elt3->getOpcode() == ISD::FP_ROUND &&
+                   isa(Elt2->getOperand(1)) &&
+                   isa(Elt3->getOperand(1)) &&
+                   Elt2->getConstantOperandVal(1) ==
+                       Elt3->getConstantOperandVal(1) &&
+                   Elt2->getOperand(0)->getOpcode() ==
+                       ISD::EXTRACT_VECTOR_ELT &&
+                   Elt3->getOperand(0)->getOpcode() ==
+                       ISD::EXTRACT_VECTOR_ELT &&
+                   // Constant index.
+                   isa(Elt2->getOperand(0)->getOperand(1)) &&
+                   isa(Elt3->getOperand(0)->getOperand(1)) &&
+                   Elt2->getOperand(0)->getOperand(0) ==
+                       Elt3->getOperand(0)->getOperand(0) &&
+                   Elt2->getOperand(0)->getConstantOperandVal(1) == 0 &&
+                   Elt3->getOperand(0)->getConstantOperandVal(1) == 1) {
+          SDValue HighLanesSrcVec = Elt2->getOperand(0)->getOperand(0);
+          HighLanes =
+              DAG.getNode(AArch64ISD::FCVTXN, DL, MVT::v2f32, HighLanesSrcVec);
+        }
+        if (HighLanes) {
+          SDValue DoubleToSingleSticky =
+              DAG.getNode(AArch64ISD::FCVTXN, DL, MVT::v2f32, LowLanesSrcVec);
+          SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32,
+                                       DoubleToSingleSticky, HighLanes);
+          return DAG.getNode(ISD::FP_ROUND, DL, VT, Concat,
+                             Elt0->getOperand(1));
+        }
+      }
+    }
+  }
+
+  if (VT == MVT::v2f64) {
+    SDValue Elt0 = N->getOperand(0), Elt1 = N->getOperand(1);
+    if (Elt0->getOpcode() == ISD::FP_EXTEND &&
+        Elt1->getOpcode() == ISD::FP_EXTEND &&
+        Elt0->getOperand(0)->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+        Elt1->getOperand(0)->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+        Elt0->getOperand(0)->getOperand(0) ==
+            Elt1->getOperand(0)->getOperand(0) &&
+        // Constant index.
+        isa(Elt0->getOperand(0)->getOperand(1)) &&
+        isa(Elt1->getOperand(0)->getOperand(1)) &&
+        Elt0->getOperand(0)->getConstantOperandVal(1) + 1 ==
+            Elt1->getOperand(0)->getConstantOperandVal(1) &&
+        // EXTRACT_SUBVECTOR requires that Idx be a constant multiple of
+        // ResultType's known minimum vector length.
+        Elt0->getOperand(0)->getConstantOperandVal(1) %
+                VT.getVectorMinNumElements() ==
+            0) {
+      SDValue SrcVec = Elt0->getOperand(0)->getOperand(0);
+      if (SrcVec.getValueType() == MVT::v4f16 ||
+          SrcVec.getValueType() == MVT::v4bf16) {
+        SDValue HalfToSingle =
+            DAG.getNode(ISD::FP_EXTEND, DL, MVT::v4f32, SrcVec);
+        SDValue SubvectorIdx = Elt0->getOperand(0)->getOperand(1);
+        SDValue Extract = DAG.getNode(
+            ISD::EXTRACT_SUBVECTOR, DL, VT.changeVectorElementType(MVT::f32),
+            HalfToSingle, SubvectorIdx);
+        return DAG.getNode(ISD::FP_EXTEND, DL, VT, Extract);
+      }
+    }
+  }
+
   // A build vector of two extracted elements is equivalent to an
   // extract subvector where the inner vector is any-extended to the
   // extract_vector_elt VT.
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index 091db559a337..8360bef8e2f8 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -6832,7 +6832,7 @@ multiclass SIMDFPNarrowTwoVector opc, string asm> {
 }
 
 multiclass SIMDFPInexactCvtTwoVector opc, string asm,
-                                     Intrinsic OpNode> {
+                                     SDPatternOperator OpNode> {
   def v2f32 : BaseSIMDFPCvtTwoVector<0, U, {S,1}, opc, V64, V128,
                                      asm, ".2s", ".2d",
                           [(set (v2f32 V64:$Rd), (OpNode (v2f64 V128:$Rn)))]>;
@@ -7547,7 +7547,7 @@ class BaseSIMDCmpTwoScalar size, bits<2> size2, bits<5> opcode,
 let mayRaiseFPException = 1, Uses = [FPCR] in
 class SIMDInexactCvtTwoScalar opcode, string asm>
   : I<(outs FPR32:$Rd), (ins FPR64:$Rn), asm, "\t$Rd, $Rn", "",
-     [(set (f32 FPR32:$Rd), (AArch64fcvtxn (f64 FPR64:$Rn)))]>,
+     [(set (f32 FPR32:$Rd), (AArch64fcvtxnsdr (f64 FPR64:$Rn)))]>,
     Sched<[WriteVd]> {
   bits<5> Rd;
   bits<5> Rn;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index f8c6d9019ef6..3c67f616c1b9 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -757,9 +757,12 @@ def AArch64fcmlez: SDNode<"AArch64ISD::FCMLEz", SDT_AArch64fcmpz>;
 def AArch64fcmltz: SDNode<"AArch64ISD::FCMLTz", SDT_AArch64fcmpz>;
 
 def AArch64fcvtxn_n: SDNode<"AArch64ISD::FCVTXN", SDTFPRoundOp>;
-def AArch64fcvtxn: PatFrags<(ops node:$Rn),
-                            [(f32 (int_aarch64_sisd_fcvtxn (f64 node:$Rn))),
-                             (f32 (AArch64fcvtxn_n (f64 node:$Rn)))]>;
+def AArch64fcvtxnsdr: PatFrags<(ops node:$Rn),
+                               [(f32 (int_aarch64_sisd_fcvtxn (f64 node:$Rn))),
+                                (f32 (AArch64fcvtxn_n (f64 node:$Rn)))]>;
+def AArch64fcvtxnv: PatFrags<(ops node:$Rn),
+                                 [(int_aarch64_neon_fcvtxn node:$Rn),
+                                  (AArch64fcvtxn_n node:$Rn)]>;
 
 def AArch64bici: SDNode<"AArch64ISD::BICi", SDT_AArch64vecimm>;
 def AArch64orri: SDNode<"AArch64ISD::ORRi", SDT_AArch64vecimm>;
@@ -5042,7 +5045,7 @@ def : Pat<(concat_vectors V64:$Rd, (v4f16 (any_fpround (v4f32 V128:$Rn)))),
 defm FCVTPS : SIMDTwoVectorFPToInt<0,1,0b11010, "fcvtps",int_aarch64_neon_fcvtps>;
 defm FCVTPU : SIMDTwoVectorFPToInt<1,1,0b11010, "fcvtpu",int_aarch64_neon_fcvtpu>;
 defm FCVTXN : SIMDFPInexactCvtTwoVector<1, 0, 0b10110, "fcvtxn",
-                                        int_aarch64_neon_fcvtxn>;
+                                        AArch64fcvtxnv>;
 defm FCVTZS : SIMDTwoVectorFPToInt<0, 1, 0b11011, "fcvtzs", any_fp_to_sint>;
 defm FCVTZU : SIMDTwoVectorFPToInt<1, 1, 0b11011, "fcvtzu", any_fp_to_uint>;
 
diff --git a/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll b/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll
index aa6b7cb495f1..cafee32ada68 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll
@@ -199,6 +199,60 @@ define <2 x float> @test_vcvt_f32_f64(<2 x double> %v) nounwind readnone ssp {
   ret <2 x float> %vcvt1.i
 }
 
+; FALLBACK-NOT: remark{{.*}}G_FPEXT{{.*}}(in function: test_vcvt_bf16_f64)
+; FALLBACK-NOT: remark{{.*}}fpext{{.*}}(in function: test_vcvt_bf16_f64)
+define <2 x bfloat> @test_vcvt_bf16_f64(<2 x double> %v) nounwind readnone ssp {
+; GENERIC-LABEL: test_vcvt_bf16_f64:
+; GENERIC:       // %bb.0:
+; GENERIC-NEXT:    fcvtxn v0.2s, v0.2d
+; GENERIC-NEXT:    movi.4s v1, #127, msl #8
+; GENERIC-NEXT:    movi.4s v2, #1
+; GENERIC-NEXT:    ushr.4s v3, v0, #16
+; GENERIC-NEXT:    add.4s v1, v0, v1
+; GENERIC-NEXT:    and.16b v2, v3, v2
+; GENERIC-NEXT:    add.4s v1, v2, v1
+; GENERIC-NEXT:    fcmeq.4s v2, v0, v0
+; GENERIC-NEXT:    orr.4s v0, #64, lsl #16
+; GENERIC-NEXT:    bit.16b v0, v1, v2
+; GENERIC-NEXT:    shrn.4h v0, v0, #16
+; GENERIC-NEXT:    ret
+;
+; FAST-LABEL: test_vcvt_bf16_f64:
+; FAST:       // %bb.0:
+; FAST-NEXT:    fcvtxn v1.2s, v0.2d
+; FAST-NEXT:    // implicit-def: $q0
+; FAST-NEXT:    fmov d0, d1
+; FAST-NEXT:    ushr.4s v1, v0, #16
+; FAST-NEXT:    movi.4s v2, #1
+; FAST-NEXT:    and.16b v1, v1, v2
+; FAST-NEXT:    add.4s v1, v1, v0
+; FAST-NEXT:    movi.4s v2, #127, msl #8
+; FAST-NEXT:    add.4s v1, v1, v2
+; FAST-NEXT:    mov.16b v2, v0
+; FAST-NEXT:    orr.4s v2, #64, lsl #16
+; FAST-NEXT:    fcmeq.4s v0, v0, v0
+; FAST-NEXT:    bsl.16b v0, v1, v2
+; FAST-NEXT:    shrn.4h v0, v0, #16
+; FAST-NEXT:    ret
+;
+; GISEL-LABEL: test_vcvt_bf16_f64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcvtxn v0.2s, v0.2d
+; GISEL-NEXT:    movi.4s v1, #127, msl #8
+; GISEL-NEXT:    movi.4s v2, #1
+; GISEL-NEXT:    ushr.4s v3, v0, #16
+; GISEL-NEXT:    add.4s v1, v0, v1
+; GISEL-NEXT:    and.16b v2, v3, v2
+; GISEL-NEXT:    add.4s v1, v2, v1
+; GISEL-NEXT:    fcmeq.4s v2, v0, v0
+; GISEL-NEXT:    orr.4s v0, #64, lsl #16
+; GISEL-NEXT:    bit.16b v0, v1, v2
+; GISEL-NEXT:    shrn.4h v0, v0, #16
+; GISEL-NEXT:    ret
+  %vcvt1.i = fptrunc <2 x double> %v to <2 x bfloat>
+  ret <2 x bfloat> %vcvt1.i
+}
+
 define half @test_vcvt_f16_f32(<1 x float> %x) {
 ; GENERIC-LABEL: test_vcvt_f16_f32:
 ; GENERIC:       // %bb.0:
@@ -350,3 +404,5 @@ define float @from_half(i16 %in) {
 
 declare float @llvm.convert.from.fp16.f32(i16) #1
 declare i16 @llvm.convert.to.fp16.f32(float) #1
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; FALLBACK: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll b/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
index 7ff61d9bcb0c..ded343b990ac 100644
--- a/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
@@ -312,25 +312,12 @@ define <8 x half> @s_to_h(<8 x float> %a) {
 define <8 x half> @d_to_h(<8 x double> %a) {
 ; CHECK-LABEL: d_to_h:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov d5, v0.d[1]
-; CHECK-NEXT:    fcvt h0, d0
-; CHECK-NEXT:    fcvt h4, d1
-; CHECK-NEXT:    mov d1, v1.d[1]
-; CHECK-NEXT:    fcvt h5, d5
-; CHECK-NEXT:    fcvt h1, d1
-; CHECK-NEXT:    mov v0.h[1], v5.h[0]
-; CHECK-NEXT:    mov v0.h[2], v4.h[0]
-; CHECK-NEXT:    mov v0.h[3], v1.h[0]
-; CHECK-NEXT:    fcvt h1, d2
-; CHECK-NEXT:    mov d2, v2.d[1]
-; CHECK-NEXT:    mov v0.h[4], v1.h[0]
-; CHECK-NEXT:    fcvt h1, d2
-; CHECK-NEXT:    mov d2, v3.d[1]
-; CHECK-NEXT:    mov v0.h[5], v1.h[0]
-; CHECK-NEXT:    fcvt h1, d3
-; CHECK-NEXT:    mov v0.h[6], v1.h[0]
-; CHECK-NEXT:    fcvt h1, d2
-; CHECK-NEXT:    mov v0.h[7], v1.h[0]
+; CHECK-NEXT:    fcvtxn v0.2s, v0.2d
+; CHECK-NEXT:    fcvtxn v2.2s, v2.2d
+; CHECK-NEXT:    fcvtxn2 v0.4s, v1.2d
+; CHECK-NEXT:    fcvtxn2 v2.4s, v3.2d
+; CHECK-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NEXT:    fcvtn2 v0.8h, v2.4s
 ; CHECK-NEXT:    ret
   %1 = fptrunc <8 x double> %a to <8 x half>
   ret <8 x half> %1
@@ -349,25 +336,12 @@ define <8 x float> @h_to_s(<8 x half> %a) {
 define <8 x double> @h_to_d(<8 x half> %a) {
 ; CHECK-LABEL: h_to_d:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    mov h1, v0.h[1]
-; CHECK-NEXT:    mov h3, v0.h[3]
-; CHECK-NEXT:    mov h4, v0.h[2]
-; CHECK-NEXT:    fcvt d0, h0
-; CHECK-NEXT:    mov h5, v2.h[1]
-; CHECK-NEXT:    mov h6, v2.h[3]
-; CHECK-NEXT:    mov h7, v2.h[2]
-; CHECK-NEXT:    fcvt d16, h1
-; CHECK-NEXT:    fcvt d17, h3
-; CHECK-NEXT:    fcvt d1, h4
-; CHECK-NEXT:    fcvt d2, h2
-; CHECK-NEXT:    fcvt d4, h5
-; CHECK-NEXT:    fcvt d5, h6
-; CHECK-NEXT:    fcvt d3, h7
-; CHECK-NEXT:    mov v0.d[1], v16.d[0]
-; CHECK-NEXT:    mov v1.d[1], v17.d[0]
-; CHECK-NEXT:    mov v2.d[1], v4.d[0]
-; CHECK-NEXT:    mov v3.d[1], v5.d[0]
+; CHECK-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-NEXT:    fcvtl v0.2d, v1.2s
+; CHECK-NEXT:    fcvtl2 v3.2d, v2.4s
+; CHECK-NEXT:    fcvtl2 v1.2d, v1.4s
+; CHECK-NEXT:    fcvtl v2.2d, v2.2s
 ; CHECK-NEXT:    ret
   %1 = fpext <8 x half> %a to <8 x double>
   ret <8 x double> %1
diff --git a/llvm/test/CodeGen/AArch64/fpext.ll b/llvm/test/CodeGen/AArch64/fpext.ll
index eca3389bcd88..86f7322f7c4e 100644
--- a/llvm/test/CodeGen/AArch64/fpext.ll
+++ b/llvm/test/CodeGen/AArch64/fpext.ll
@@ -85,29 +85,46 @@ entry:
 }
 
 define <2 x double> @fpext_v2f16_v2f64(<2 x half> %a) {
-; CHECK-LABEL: fpext_v2f16_v2f64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    mov h1, v0.h[1]
-; CHECK-NEXT:    fcvt d0, h0
-; CHECK-NEXT:    fcvt d1, h1
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fpext_v2f16_v2f64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fpext_v2f16_v2f64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    fcvt d0, h0
+; CHECK-GI-NEXT:    fcvt d1, h1
+; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-GI-NEXT:    ret
 entry:
   %c = fpext <2 x half> %a to <2 x double>
   ret <2 x double> %c
 }
 
 define <3 x double> @fpext_v3f16_v3f64(<3 x half> %a) {
-; CHECK-LABEL: fpext_v3f16_v3f64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    mov h1, v0.h[1]
-; CHECK-NEXT:    mov h2, v0.h[2]
-; CHECK-NEXT:    fcvt d0, h0
-; CHECK-NEXT:    fcvt d1, h1
-; CHECK-NEXT:    fcvt d2, h2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fpext_v3f16_v3f64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NEXT:    fcvtl v0.2d, v1.2s
+; CHECK-SD-NEXT:    fcvtl2 v2.2d, v1.4s
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fpext_v3f16_v3f64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    fcvt d0, h0
+; CHECK-GI-NEXT:    fcvt d1, h1
+; CHECK-GI-NEXT:    fcvt d2, h2
+; CHECK-GI-NEXT:    ret
 entry:
   %c = fpext <3 x half> %a to <3 x double>
   ret <3 x double> %c
@@ -116,16 +133,9 @@ entry:
 define <4 x double> @fpext_v4f16_v4f64(<4 x half> %a) {
 ; CHECK-SD-LABEL: fpext_v4f16_v4f64:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NEXT:    mov h2, v0.h[3]
-; CHECK-SD-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NEXT:    fcvt d0, h0
-; CHECK-SD-NEXT:    fcvt d4, h1
-; CHECK-SD-NEXT:    fcvt d2, h2
-; CHECK-SD-NEXT:    fcvt d1, h3
-; CHECK-SD-NEXT:    mov v0.d[1], v4.d[0]
-; CHECK-SD-NEXT:    mov v1.d[1], v2.d[0]
+; CHECK-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NEXT:    fcvtl2 v1.2d, v0.4s
+; CHECK-SD-NEXT:    fcvtl v0.2d, v0.2s
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: fpext_v4f16_v4f64:
diff --git a/llvm/test/CodeGen/AArch64/fptrunc.ll b/llvm/test/CodeGen/AArch64/fptrunc.ll
index 9425988af834..3efc98ab5fd5 100644
--- a/llvm/test/CodeGen/AArch64/fptrunc.ll
+++ b/llvm/test/CodeGen/AArch64/fptrunc.ll
@@ -84,11 +84,8 @@ entry:
 define <2 x half> @fptrunc_v2f64_v2f16(<2 x double> %a) {
 ; CHECK-SD-LABEL: fptrunc_v2f64_v2f16:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov d1, v0.d[1]
-; CHECK-SD-NEXT:    fcvt h0, d0
-; CHECK-SD-NEXT:    fcvt h1, d1
-; CHECK-SD-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    fcvtxn v0.2s, v0.2d
+; CHECK-SD-NEXT:    fcvtn v0.4h, v0.4s
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: fptrunc_v2f64_v2f16:
@@ -135,16 +132,9 @@ entry:
 define <4 x half> @fptrunc_v4f64_v4f16(<4 x double> %a) {
 ; CHECK-SD-LABEL: fptrunc_v4f64_v4f16:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov d2, v0.d[1]
-; CHECK-SD-NEXT:    fcvt h0, d0
-; CHECK-SD-NEXT:    fcvt h2, d2
-; CHECK-SD-NEXT:    mov v0.h[1], v2.h[0]
-; CHECK-SD-NEXT:    fcvt h2, d1
-; CHECK-SD-NEXT:    mov d1, v1.d[1]
-; CHECK-SD-NEXT:    mov v0.h[2], v2.h[0]
-; CHECK-SD-NEXT:    fcvt h1, d1
-; CHECK-SD-NEXT:    mov v0.h[3], v1.h[0]
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    fcvtxn v0.2s, v0.2d
+; CHECK-SD-NEXT:    fcvtxn2 v0.4s, v1.2d
+; CHECK-SD-NEXT:    fcvtn v0.4h, v0.4s
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: fptrunc_v4f64_v4f16:
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll
index da9b79a56a95..2ace0bca274a 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll
@@ -427,49 +427,35 @@ define void @test_copysign_v4f16_v4f32(ptr %ap, ptr %bp) {
 define void @test_copysign_v4f16_v4f64(ptr %ap, ptr %bp) {
 ; SVE-LABEL: test_copysign_v4f16_v4f64:
 ; SVE:       // %bb.0:
-; SVE-NEXT:    sub sp, sp, #16
-; SVE-NEXT:    .cfi_def_cfa_offset 16
-; SVE-NEXT:    ldp q1, q0, [x1]
-; SVE-NEXT:    ldr d4, [x0]
-; SVE-NEXT:    and z4.h, z4.h, #0x7fff
-; SVE-NEXT:    mov z2.d, z0.d[1]
-; SVE-NEXT:    mov z3.d, z1.d[1]
-; SVE-NEXT:    fcvt h0, d0
-; SVE-NEXT:    fcvt h1, d1
-; SVE-NEXT:    fcvt h2, d2
-; SVE-NEXT:    fcvt h3, d3
-; SVE-NEXT:    str h0, [sp, #12]
-; SVE-NEXT:    str h1, [sp, #8]
-; SVE-NEXT:    str h2, [sp, #14]
-; SVE-NEXT:    str h3, [sp, #10]
-; SVE-NEXT:    ldr d0, [sp, #8]
+; SVE-NEXT:    ldp q0, q1, [x1]
+; SVE-NEXT:    ptrue p0.s, vl2
+; SVE-NEXT:    ptrue p1.s
+; SVE-NEXT:    fcvtxn v1.2s, v1.2d
+; SVE-NEXT:    fcvtxn v0.2s, v0.2d
+; SVE-NEXT:    splice z0.s, p0, z0.s, z1.s
+; SVE-NEXT:    ldr d1, [x0]
+; SVE-NEXT:    and z1.h, z1.h, #0x7fff
+; SVE-NEXT:    fcvt z0.h, p1/m, z0.s
+; SVE-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; SVE-NEXT:    and z0.h, z0.h, #0x8000
-; SVE-NEXT:    orr z0.d, z4.d, z0.d
+; SVE-NEXT:    orr z0.d, z1.d, z0.d
 ; SVE-NEXT:    str d0, [x0]
-; SVE-NEXT:    add sp, sp, #16
 ; SVE-NEXT:    ret
 ;
 ; SVE2-LABEL: test_copysign_v4f16_v4f64:
 ; SVE2:       // %bb.0:
-; SVE2-NEXT:    sub sp, sp, #16
-; SVE2-NEXT:    .cfi_def_cfa_offset 16
-; SVE2-NEXT:    ldp q2, q1, [x1]
-; SVE2-NEXT:    mov z0.h, #32767 // =0x7fff
-; SVE2-NEXT:    ldr d5, [x0]
-; SVE2-NEXT:    mov z3.d, z1.d[1]
-; SVE2-NEXT:    mov z4.d, z2.d[1]
-; SVE2-NEXT:    fcvt h1, d1
-; SVE2-NEXT:    fcvt h2, d2
-; SVE2-NEXT:    fcvt h3, d3
-; SVE2-NEXT:    fcvt h4, d4
-; SVE2-NEXT:    str h1, [sp, #12]
-; SVE2-NEXT:    str h2, [sp, #8]
-; SVE2-NEXT:    str h3, [sp, #14]
-; SVE2-NEXT:    str h4, [sp, #10]
-; SVE2-NEXT:    ldr d1, [sp, #8]
-; SVE2-NEXT:    bsl z5.d, z5.d, z1.d, z0.d
-; SVE2-NEXT:    str d5, [x0]
-; SVE2-NEXT:    add sp, sp, #16
+; SVE2-NEXT:    ldp q0, q1, [x1]
+; SVE2-NEXT:    ptrue p0.s, vl2
+; SVE2-NEXT:    ptrue p1.s
+; SVE2-NEXT:    ldr d2, [x0]
+; SVE2-NEXT:    fcvtxn v1.2s, v1.2d
+; SVE2-NEXT:    fcvtxn v0.2s, v0.2d
+; SVE2-NEXT:    splice z0.s, p0, z0.s, z1.s
+; SVE2-NEXT:    mov z1.h, #32767 // =0x7fff
+; SVE2-NEXT:    fcvt z0.h, p1/m, z0.s
+; SVE2-NEXT:    uzp1 z0.h, z0.h, z0.h
+; SVE2-NEXT:    bsl z2.d, z2.d, z0.d, z1.d
+; SVE2-NEXT:    str d2, [x0]
 ; SVE2-NEXT:    ret
   %a = load <4 x half>, ptr %ap
   %b = load <4 x double>, ptr %bp
diff --git a/llvm/test/CodeGen/AArch64/vector-fcopysign.ll b/llvm/test/CodeGen/AArch64/vector-fcopysign.ll
index c33759331bbc..de26676b5c73 100644
--- a/llvm/test/CodeGen/AArch64/vector-fcopysign.ll
+++ b/llvm/test/CodeGen/AArch64/vector-fcopysign.ll
@@ -209,16 +209,10 @@ define <4 x half> @test_copysign_v4f16_v4f32(<4 x half> %a, <4 x float> %b) #0 {
 define <4 x half> @test_copysign_v4f16_v4f64(<4 x half> %a, <4 x double> %b) #0 {
 ; CHECK-LABEL: test_copysign_v4f16_v4f64:
 ; CHECK:       ; %bb.0:
-; CHECK-NEXT:    mov d3, v1[1]
-; CHECK-NEXT:    fcvt h1, d1
-; CHECK-NEXT:    fcvt h3, d3
-; CHECK-NEXT:    mov.h v1[1], v3[0]
-; CHECK-NEXT:    fcvt h3, d2
-; CHECK-NEXT:    mov d2, v2[1]
-; CHECK-NEXT:    mov.h v1[2], v3[0]
-; CHECK-NEXT:    fcvt h2, d2
-; CHECK-NEXT:    mov.h v1[3], v2[0]
+; CHECK-NEXT:    fcvtxn v1.2s, v1.2d
+; CHECK-NEXT:    fcvtxn2 v1.4s, v2.2d
 ; CHECK-NEXT:    mvni.4h v2, #128, lsl #8
+; CHECK-NEXT:    fcvtn v1.4h, v1.4s
 ; CHECK-NEXT:    bif.8b v0, v1, v2
 ; CHECK-NEXT:    ret
   %tmp0 = fptrunc <4 x double> %b to <4 x half>
@@ -291,42 +285,20 @@ define <4 x bfloat> @test_copysign_v4bf16_v4f32(<4 x bfloat> %a, <4 x float> %b)
 define <4 x bfloat> @test_copysign_v4bf16_v4f64(<4 x bfloat> %a, <4 x double> %b) #0 {
 ; CHECK-LABEL: test_copysign_v4bf16_v4f64:
 ; CHECK:       ; %bb.0:
-; CHECK-NEXT:    mov d3, v1[1]
-; CHECK-NEXT:    fcvtxn s1, d1
-; CHECK-NEXT:    mov w8, #32767 ; =0x7fff
-; CHECK-NEXT:    fcvtxn s3, d3
-; CHECK-NEXT:    fmov w10, s1
-; CHECK-NEXT:    ubfx w12, w10, #16, #1
-; CHECK-NEXT:    add w10, w10, w8
-; CHECK-NEXT:    fmov w9, s3
-; CHECK-NEXT:    fcvtxn s3, d2
-; CHECK-NEXT:    mov d2, v2[1]
-; CHECK-NEXT:    add w10, w12, w10
-; CHECK-NEXT:    lsr w10, w10, #16
-; CHECK-NEXT:    ubfx w11, w9, #16, #1
-; CHECK-NEXT:    add w9, w9, w8
-; CHECK-NEXT:    fcvtxn s1, d2
-; CHECK-NEXT:    add w9, w11, w9
-; CHECK-NEXT:    fmov w11, s3
-; CHECK-NEXT:    fmov s3, w10
-; CHECK-NEXT:    lsr w9, w9, #16
-; CHECK-NEXT:    ubfx w12, w11, #16, #1
-; CHECK-NEXT:    fmov s2, w9
-; CHECK-NEXT:    add w9, w11, w8
-; CHECK-NEXT:    fmov w10, s1
-; CHECK-NEXT:    add w9, w12, w9
-; CHECK-NEXT:    lsr w9, w9, #16
-; CHECK-NEXT:    mov.h v3[1], v2[0]
-; CHECK-NEXT:    ubfx w11, w10, #16, #1
-; CHECK-NEXT:    add w8, w10, w8
-; CHECK-NEXT:    fmov s1, w9
-; CHECK-NEXT:    add w8, w11, w8
-; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    mov.h v3[2], v1[0]
-; CHECK-NEXT:    fmov s1, w8
-; CHECK-NEXT:    mov.h v3[3], v1[0]
-; CHECK-NEXT:    mvni.4h v1, #128, lsl #8
-; CHECK-NEXT:    bif.8b v0, v3, v1
+; CHECK-NEXT:    fcvtxn v1.2s, v1.2d
+; CHECK-NEXT:    movi.4s v3, #1
+; CHECK-NEXT:    fcvtxn2 v1.4s, v2.2d
+; CHECK-NEXT:    movi.4s v2, #127, msl #8
+; CHECK-NEXT:    ushr.4s v4, v1, #16
+; CHECK-NEXT:    add.4s v2, v1, v2
+; CHECK-NEXT:    and.16b v3, v4, v3
+; CHECK-NEXT:    add.4s v2, v3, v2
+; CHECK-NEXT:    fcmeq.4s v3, v1, v1
+; CHECK-NEXT:    orr.4s v1, #64, lsl #16
+; CHECK-NEXT:    bit.16b v1, v2, v3
+; CHECK-NEXT:    mvni.4h v2, #128, lsl #8
+; CHECK-NEXT:    shrn.4h v1, v1, #16
+; CHECK-NEXT:    bif.8b v0, v1, v2
 ; CHECK-NEXT:    ret
   %tmp0 = fptrunc <4 x double> %b to <4 x bfloat>
   %r = call <4 x bfloat> @llvm.copysign.v4bf16(<4 x bfloat> %a, <4 x bfloat> %tmp0)
-- 
GitLab


From edc1c3d24e6f8ed548340ce0369138fb40427a24 Mon Sep 17 00:00:00 2001
From: David Majnemer 
Date: Fri, 8 Mar 2024 18:22:40 +0000
Subject: [PATCH 655/929] [AArch64] Make more vector f16 operations legal

v8f16 is a legal type but promoting to v16f16 would result in an illegal
type.

Let's legalize these by a combination of splitting+promoting resulting
in a pair of v4f16.

Also, we were being overly cautious with different v4f16 nodes. Mark
more of them safe to promote to v4f32.
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |  144 +-
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   |   91 ++
 llvm/test/CodeGen/AArch64/faddp-half.ll       |  218 +--
 llvm/test/CodeGen/AArch64/faddsub.ll          |  496 +-----
 llvm/test/CodeGen/AArch64/fcvt.ll             | 1421 +++--------------
 llvm/test/CodeGen/AArch64/fcvt_combine.ll     |  140 +-
 llvm/test/CodeGen/AArch64/fdiv.ll             |  247 +--
 llvm/test/CodeGen/AArch64/fmla.ll             |  879 ++--------
 llvm/test/CodeGen/AArch64/fmul.ll             |  248 +--
 .../CodeGen/AArch64/fp16-v8-instructions.ll   |  248 +--
 .../test/CodeGen/AArch64/sve-fp-reciprocal.ll |    5 -
 llvm/test/CodeGen/AArch64/vecreduce-fadd.ll   |  196 +--
 llvm/test/CodeGen/AArch64/vecreduce-fmul.ll   |  196 +--
 13 files changed, 864 insertions(+), 3665 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 89b697b2d515..054311d39e7b 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -701,43 +701,45 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   }
 
   auto LegalizeNarrowFP = [this](MVT ScalarVT) {
-    for (auto Op : {ISD::SETCC,
-                    ISD::SELECT_CC,
-                    ISD::BR_CC,
-                    ISD::FADD,
-                    ISD::FSUB,
-                    ISD::FMUL,
-                    ISD::FDIV,
-                    ISD::FMA,
-                    ISD::FCEIL,
-                    ISD::FSQRT,
-                    ISD::FFLOOR,
-                    ISD::FNEARBYINT,
-                    ISD::FRINT,
-                    ISD::FROUND,
-                    ISD::FROUNDEVEN,
-                    ISD::FTRUNC,
-                    ISD::FMINNUM,
-                    ISD::FMAXNUM,
-                    ISD::FMINIMUM,
-                    ISD::FMAXIMUM,
-                    ISD::STRICT_FADD,
-                    ISD::STRICT_FSUB,
-                    ISD::STRICT_FMUL,
-                    ISD::STRICT_FDIV,
-                    ISD::STRICT_FMA,
-                    ISD::STRICT_FCEIL,
-                    ISD::STRICT_FFLOOR,
-                    ISD::STRICT_FSQRT,
-                    ISD::STRICT_FRINT,
-                    ISD::STRICT_FNEARBYINT,
-                    ISD::STRICT_FROUND,
-                    ISD::STRICT_FTRUNC,
-                    ISD::STRICT_FROUNDEVEN,
-                    ISD::STRICT_FMINNUM,
-                    ISD::STRICT_FMAXNUM,
-                    ISD::STRICT_FMINIMUM,
-                    ISD::STRICT_FMAXIMUM})
+    for (auto Op : {
+             ISD::SETCC,
+             ISD::SELECT_CC,
+             ISD::BR_CC,
+             ISD::FADD,
+             ISD::FSUB,
+             ISD::FMUL,
+             ISD::FDIV,
+             ISD::FMA,
+             ISD::FCEIL,
+             ISD::FSQRT,
+             ISD::FFLOOR,
+             ISD::FNEARBYINT,
+             ISD::FRINT,
+             ISD::FROUND,
+             ISD::FROUNDEVEN,
+             ISD::FTRUNC,
+             ISD::FMINNUM,
+             ISD::FMAXNUM,
+             ISD::FMINIMUM,
+             ISD::FMAXIMUM,
+             ISD::STRICT_FADD,
+             ISD::STRICT_FSUB,
+             ISD::STRICT_FMUL,
+             ISD::STRICT_FDIV,
+             ISD::STRICT_FMA,
+             ISD::STRICT_FCEIL,
+             ISD::STRICT_FFLOOR,
+             ISD::STRICT_FSQRT,
+             ISD::STRICT_FRINT,
+             ISD::STRICT_FNEARBYINT,
+             ISD::STRICT_FROUND,
+             ISD::STRICT_FTRUNC,
+             ISD::STRICT_FROUNDEVEN,
+             ISD::STRICT_FMINNUM,
+             ISD::STRICT_FMAXNUM,
+             ISD::STRICT_FMINIMUM,
+             ISD::STRICT_FMAXIMUM,
+         })
       setOperationAction(Op, ScalarVT, Promote);
 
     for (auto Op : {ISD::FNEG, ISD::FABS})
@@ -752,45 +754,45 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
 
     // promote v4f16 to v4f32 when that is known to be safe.
     auto V4Narrow = MVT::getVectorVT(ScalarVT, 4);
-    setOperationPromotedToType(ISD::FADD, V4Narrow, MVT::v4f32);
-    setOperationPromotedToType(ISD::FSUB, V4Narrow, MVT::v4f32);
-    setOperationPromotedToType(ISD::FMUL, V4Narrow, MVT::v4f32);
-    setOperationPromotedToType(ISD::FDIV, V4Narrow, MVT::v4f32);
-
-    setOperationAction(ISD::FABS, V4Narrow, Legal);
-    setOperationAction(ISD::FNEG, V4Narrow, Legal);
-    setOperationAction(ISD::FROUND,      V4Narrow, Expand);
-    setOperationAction(ISD::FROUNDEVEN,  V4Narrow, Expand);
+    setOperationPromotedToType(ISD::FADD,       V4Narrow, MVT::v4f32);
+    setOperationPromotedToType(ISD::FSUB,       V4Narrow, MVT::v4f32);
+    setOperationPromotedToType(ISD::FMUL,       V4Narrow, MVT::v4f32);
+    setOperationPromotedToType(ISD::FDIV,       V4Narrow, MVT::v4f32);
+    setOperationPromotedToType(ISD::FCEIL,      V4Narrow, MVT::v4f32);
+    setOperationPromotedToType(ISD::FFLOOR,     V4Narrow, MVT::v4f32);
+    setOperationPromotedToType(ISD::FROUND,     V4Narrow, MVT::v4f32);
+    setOperationPromotedToType(ISD::FTRUNC,     V4Narrow, MVT::v4f32);
+    setOperationPromotedToType(ISD::FROUNDEVEN, V4Narrow, MVT::v4f32);
+    setOperationPromotedToType(ISD::FRINT,      V4Narrow, MVT::v4f32);
+    setOperationPromotedToType(ISD::FNEARBYINT, V4Narrow, MVT::v4f32);
+
+    setOperationAction(ISD::FABS,        V4Narrow, Legal);
+    setOperationAction(ISD::FNEG, 	 V4Narrow, Legal);
     setOperationAction(ISD::FMA,         V4Narrow, Expand);
     setOperationAction(ISD::SETCC,       V4Narrow, Custom);
     setOperationAction(ISD::BR_CC,       V4Narrow, Expand);
     setOperationAction(ISD::SELECT,      V4Narrow, Expand);
     setOperationAction(ISD::SELECT_CC,   V4Narrow, Expand);
-    setOperationAction(ISD::FTRUNC,      V4Narrow, Expand);
-    setOperationAction(ISD::FCOPYSIGN, V4Narrow, Custom);
-    setOperationAction(ISD::FFLOOR,      V4Narrow, Expand);
-    setOperationAction(ISD::FCEIL,       V4Narrow, Expand);
-    setOperationAction(ISD::FRINT,       V4Narrow, Expand);
-    setOperationAction(ISD::FNEARBYINT,  V4Narrow, Expand);
+    setOperationAction(ISD::FCOPYSIGN,   V4Narrow, Custom);
     setOperationAction(ISD::FSQRT,       V4Narrow, Expand);
 
     auto V8Narrow = MVT::getVectorVT(ScalarVT, 8);
-    setOperationAction(ISD::FABS, V8Narrow, Legal);
-    setOperationAction(ISD::FADD,        V8Narrow, Expand);
-    setOperationAction(ISD::FCEIL,       V8Narrow, Expand);
-    setOperationAction(ISD::FCOPYSIGN, V8Narrow, Custom);
-    setOperationAction(ISD::FDIV,        V8Narrow, Expand);
-    setOperationAction(ISD::FFLOOR,      V8Narrow, Expand);
+    setOperationAction(ISD::FABS,        V8Narrow, Legal);
+    setOperationAction(ISD::FADD,        V8Narrow, Legal);
+    setOperationAction(ISD::FCEIL,       V8Narrow, Legal);
+    setOperationAction(ISD::FCOPYSIGN,   V8Narrow, Custom);
+    setOperationAction(ISD::FDIV,        V8Narrow, Legal);
+    setOperationAction(ISD::FFLOOR,      V8Narrow, Legal);
     setOperationAction(ISD::FMA,         V8Narrow, Expand);
-    setOperationAction(ISD::FMUL,        V8Narrow, Expand);
-    setOperationAction(ISD::FNEARBYINT,  V8Narrow, Expand);
-    setOperationAction(ISD::FNEG, V8Narrow, Legal);
-    setOperationAction(ISD::FROUND,      V8Narrow, Expand);
-    setOperationAction(ISD::FROUNDEVEN,  V8Narrow, Expand);
-    setOperationAction(ISD::FRINT,       V8Narrow, Expand);
+    setOperationAction(ISD::FMUL,        V8Narrow, Legal);
+    setOperationAction(ISD::FNEARBYINT,  V8Narrow, Legal);
+    setOperationAction(ISD::FNEG, 	 V8Narrow, Legal);
+    setOperationAction(ISD::FROUND,      V8Narrow, Legal);
+    setOperationAction(ISD::FROUNDEVEN,  V8Narrow, Legal);
+    setOperationAction(ISD::FRINT,       V8Narrow, Legal);
     setOperationAction(ISD::FSQRT,       V8Narrow, Expand);
-    setOperationAction(ISD::FSUB,        V8Narrow, Expand);
-    setOperationAction(ISD::FTRUNC,      V8Narrow, Expand);
+    setOperationAction(ISD::FSUB,        V8Narrow, Legal);
+    setOperationAction(ISD::FTRUNC,      V8Narrow, Legal);
     setOperationAction(ISD::SETCC,       V8Narrow, Expand);
     setOperationAction(ISD::BR_CC,       V8Narrow, Expand);
     setOperationAction(ISD::SELECT,      V8Narrow, Expand);
@@ -10593,13 +10595,19 @@ static SDValue getEstimate(const AArch64Subtarget *ST, unsigned Opcode,
         VT == MVT::v4f32)) ||
       (ST->hasSVE() &&
        (VT == MVT::nxv8f16 || VT == MVT::nxv4f32 || VT == MVT::nxv2f64))) {
-    if (ExtraSteps == TargetLoweringBase::ReciprocalEstimate::Unspecified)
+    if (ExtraSteps == TargetLoweringBase::ReciprocalEstimate::Unspecified) {
       // For the reciprocal estimates, convergence is quadratic, so the number
       // of digits is doubled after each iteration.  In ARMv8, the accuracy of
       // the initial estimate is 2^-8.  Thus the number of extra steps to refine
       // the result for float (23 mantissa bits) is 2 and for double (52
       // mantissa bits) is 3.
-      ExtraSteps = VT.getScalarType() == MVT::f64 ? 3 : 2;
+      constexpr unsigned AccurateBits = 8;
+      unsigned DesiredBits =
+          APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(VT));
+      ExtraSteps = DesiredBits <= AccurateBits
+                       ? 0
+                       : Log2_64_Ceil(DesiredBits) - Log2_64_Ceil(AccurateBits);
+    }
 
     return DAG.getNode(Opcode, SDLoc(Operand), VT, Operand);
   }
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 3c67f616c1b9..6254e68326f7 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -128,6 +128,7 @@ def HasRDM           : Predicate<"Subtarget->hasRDM()">,
                                  AssemblerPredicateWithAll<(all_of FeatureRDM), "rdm">;
 def HasFullFP16      : Predicate<"Subtarget->hasFullFP16()">,
                                  AssemblerPredicateWithAll<(all_of FeatureFullFP16), "fullfp16">;
+def HasNoFullFP16    : Predicate<"!Subtarget->hasFullFP16()">;
 def HasFP16FML       : Predicate<"Subtarget->hasFP16FML()">,
                                  AssemblerPredicateWithAll<(all_of FeatureFP16FML), "fp16fml">;
 def HasSPE           : Predicate<"Subtarget->hasSPE()">,
@@ -254,6 +255,7 @@ def HasTRBE          : Predicate<"Subtarget->hasTRBE()">,
                        AssemblerPredicateWithAll<(all_of FeatureTRBE), "trbe">;
 def HasBF16          : Predicate<"Subtarget->hasBF16()">,
                        AssemblerPredicateWithAll<(all_of FeatureBF16), "bf16">;
+def HasNoBF16        : Predicate<"!Subtarget->hasBF16()">;
 def HasMatMulInt8    : Predicate<"Subtarget->hasMatMulInt8()">,
                        AssemblerPredicateWithAll<(all_of FeatureMatMulInt8), "i8mm">;
 def HasMatMulFP32    : Predicate<"Subtarget->hasMatMulFP32()">,
@@ -764,6 +766,8 @@ def AArch64fcvtxnv: PatFrags<(ops node:$Rn),
                                  [(int_aarch64_neon_fcvtxn node:$Rn),
                                   (AArch64fcvtxn_n node:$Rn)]>;
 
+//def Aarch64softf32tobf16v8: SDNode<"AArch64ISD::", SDTFPRoundOp>;
+
 def AArch64bici: SDNode<"AArch64ISD::BICi", SDT_AArch64vecimm>;
 def AArch64orri: SDNode<"AArch64ISD::ORRi", SDT_AArch64vecimm>;
 
@@ -9739,6 +9743,93 @@ let Predicates = [HasCPA] in {
   def MSUBPT : MulAccumCPA<1, "msubpt">;
 }
 
+def round_v4fp32_to_v4bf16 :
+  OutPatFrag<(ops node:$Rn),
+             // NaN? Round : Quiet(NaN)
+             (BSPv16i8 (FCMEQv4f32 $Rn, $Rn),
+                       (ADDv4i32
+                         (ADDv4i32 $Rn,
+                           // Extract the LSB of the fp32 *truncated* to bf16.
+                           (ANDv16i8 (USHRv4i32_shift V128:$Rn, (i32 16)),
+                                     (MOVIv4i32 (i32 1), (i32 0)))),
+                         // Bias which will help us break ties correctly.
+                         (MOVIv4s_msl (i32 127), (i32 264))),
+                       // Set the quiet bit in the NaN.
+                       (ORRv4i32 $Rn, (i32 64), (i32 16)))>;
+
+multiclass PromoteUnaryv8f16Tov4f32 {
+  let Predicates = [HasNoFullFP16] in
+  def : Pat<(InOp (v8f16 V128:$Rn)),
+            (v8f16 (FCVTNv8i16
+              (INSERT_SUBREG (IMPLICIT_DEF),
+                             (v4f16 (FCVTNv4i16
+                               (v4f32 (OutInst
+                                 (v4f32 (FCVTLv4i16 (v4i16 (EXTRACT_SUBREG V128:$Rn, dsub)))))))),
+               dsub),
+              (v4f32 (OutInst (v4f32 (FCVTLv8i16 V128:$Rn))))))>;
+
+  let Predicates = [HasBF16] in
+  def : Pat<(InOp (v8bf16 V128:$Rn)),
+            (v8bf16 (BFCVTN2
+              (v8bf16 (BFCVTN
+                (v4f32 (OutInst
+                  (v4f32 (SHLLv4i16 (v4i16 (EXTRACT_SUBREG V128:$Rn, dsub)))))))),
+              (v4f32 (OutInst (v4f32 (SHLLv8i16 V128:$Rn))))))>;
+
+  let Predicates = [HasNoBF16] in
+  def : Pat<(InOp (v8bf16 V128:$Rn)),
+            (UZP2v8i16
+              (round_v4fp32_to_v4bf16 (v4f32 (OutInst
+                  (v4f32 (SHLLv4i16 (v4i16 (EXTRACT_SUBREG V128:$Rn, dsub))))))),
+              (round_v4fp32_to_v4bf16 (v4f32 (OutInst
+                  (v4f32 (SHLLv8i16 V128:$Rn))))))>;
+}
+defm : PromoteUnaryv8f16Tov4f32;
+defm : PromoteUnaryv8f16Tov4f32;
+defm : PromoteUnaryv8f16Tov4f32;
+defm : PromoteUnaryv8f16Tov4f32;
+defm : PromoteUnaryv8f16Tov4f32;
+defm : PromoteUnaryv8f16Tov4f32;
+defm : PromoteUnaryv8f16Tov4f32;
+
+multiclass PromoteBinaryv8f16Tov4f32 {
+  let Predicates = [HasNoFullFP16] in
+  def : Pat<(InOp (v8f16 V128:$Rn), (v8f16 V128:$Rm)),
+            (v8f16 (FCVTNv8i16
+              (INSERT_SUBREG (IMPLICIT_DEF),
+                             (v4f16 (FCVTNv4i16
+                               (v4f32 (OutInst
+                                 (v4f32 (FCVTLv4i16 (v4i16 (EXTRACT_SUBREG V128:$Rn, dsub)))),
+                                 (v4f32 (FCVTLv4i16 (v4i16 (EXTRACT_SUBREG V128:$Rm, dsub)))))))),
+               dsub),
+              (v4f32 (OutInst (v4f32 (FCVTLv8i16 V128:$Rn)),
+                              (v4f32 (FCVTLv8i16 V128:$Rm))))))>;
+
+  let Predicates = [HasBF16] in
+  def : Pat<(InOp (v8bf16 V128:$Rn), (v8bf16 V128:$Rm)),
+            (v8bf16 (BFCVTN2
+              (v8bf16 (BFCVTN
+                (v4f32 (OutInst
+                  (v4f32 (SHLLv4i16 (v4i16 (EXTRACT_SUBREG V128:$Rn, dsub)))),
+                  (v4f32 (SHLLv4i16 (v4i16 (EXTRACT_SUBREG V128:$Rm, dsub)))))))),
+              (v4f32 (OutInst (v4f32 (SHLLv8i16 V128:$Rn)),
+                              (v4f32 (SHLLv8i16 V128:$Rm))))))>;
+
+  let Predicates = [HasNoBF16] in
+  def : Pat<(InOp (v8bf16 V128:$Rn), (v8bf16 V128:$Rm)),
+            (UZP2v8i16
+              (round_v4fp32_to_v4bf16 (v4f32 (OutInst
+                  (v4f32 (SHLLv4i16 (v4i16 (EXTRACT_SUBREG V128:$Rn, dsub)))),
+                  (v4f32 (SHLLv4i16 (v4i16 (EXTRACT_SUBREG V128:$Rm, dsub))))))),
+              (round_v4fp32_to_v4bf16 (v4f32 (OutInst
+                  (v4f32 (SHLLv8i16 V128:$Rn)),
+                  (v4f32 (SHLLv8i16 V128:$Rm))))))>;
+}
+defm : PromoteBinaryv8f16Tov4f32;
+defm : PromoteBinaryv8f16Tov4f32;
+defm : PromoteBinaryv8f16Tov4f32;
+defm : PromoteBinaryv8f16Tov4f32;
+
 include "AArch64InstrAtomics.td"
 include "AArch64SVEInstrInfo.td"
 include "AArch64SMEInstrInfo.td"
diff --git a/llvm/test/CodeGen/AArch64/faddp-half.ll b/llvm/test/CodeGen/AArch64/faddp-half.ll
index 6068a4742eea..447476e76ec6 100644
--- a/llvm/test/CodeGen/AArch64/faddp-half.ll
+++ b/llvm/test/CodeGen/AArch64/faddp-half.ll
@@ -107,10 +107,15 @@ define half @faddp_8xhalf(<8 x half> %a) {
 ; CHECKNOFP16-LABEL: faddp_8xhalf:
 ; CHECKNOFP16:       // %bb.0: // %entry
 ; CHECKNOFP16-NEXT:    dup v1.8h, v0.h[1]
-; CHECKNOFP16-NEXT:    fcvt s0, h0
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fadd s0, s0, s1
-; CHECKNOFP16-NEXT:    fcvt h0, s0
+; CHECKNOFP16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECKNOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECKNOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECKNOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECKNOFP16-NEXT:    fadd v2.4s, v2.4s, v3.4s
+; CHECKNOFP16-NEXT:    fadd v1.4s, v0.4s, v1.4s
+; CHECKNOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECKNOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
+; CHECKNOFP16-NEXT:    // kill: def $h0 killed $h0 killed $q0
 ; CHECKNOFP16-NEXT:    ret
 entry:
   %shift = shufflevector <8 x half> %a, <8 x half> undef, <8 x i32> 
@@ -128,10 +133,15 @@ define half @faddp_8xhalf_commute(<8 x half> %a) {
 ; CHECKNOFP16-LABEL: faddp_8xhalf_commute:
 ; CHECKNOFP16:       // %bb.0: // %entry
 ; CHECKNOFP16-NEXT:    dup v1.8h, v0.h[1]
-; CHECKNOFP16-NEXT:    fcvt s0, h0
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fadd s0, s1, s0
-; CHECKNOFP16-NEXT:    fcvt h0, s0
+; CHECKNOFP16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECKNOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECKNOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECKNOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECKNOFP16-NEXT:    fadd v2.4s, v3.4s, v2.4s
+; CHECKNOFP16-NEXT:    fadd v1.4s, v1.4s, v0.4s
+; CHECKNOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECKNOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
+; CHECKNOFP16-NEXT:    // kill: def $h0 killed $h0 killed $q0
 ; CHECKNOFP16-NEXT:    ret
 entry:
   %shift = shufflevector <8 x half> %a, <8 x half> undef, <8 x i32> 
@@ -149,61 +159,15 @@ define <8 x half> @addp_v8f16(<8 x half> %a) {
 ;
 ; CHECKNOFP16-LABEL: addp_v8f16:
 ; CHECKNOFP16:       // %bb.0: // %entry
-; CHECKNOFP16-NEXT:    rev32 v2.8h, v0.8h
-; CHECKNOFP16-NEXT:    mov h1, v0.h[1]
-; CHECKNOFP16-NEXT:    fcvt s4, h0
-; CHECKNOFP16-NEXT:    mov h5, v0.h[2]
-; CHECKNOFP16-NEXT:    mov h16, v0.h[3]
-; CHECKNOFP16-NEXT:    mov h3, v2.h[1]
-; CHECKNOFP16-NEXT:    fcvt s6, h2
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    mov h7, v2.h[2]
-; CHECKNOFP16-NEXT:    fcvt s5, h5
-; CHECKNOFP16-NEXT:    fcvt s16, h16
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fadd s4, s6, s4
-; CHECKNOFP16-NEXT:    mov h6, v2.h[3]
-; CHECKNOFP16-NEXT:    fcvt s7, h7
-; CHECKNOFP16-NEXT:    fadd s3, s3, s1
-; CHECKNOFP16-NEXT:    fcvt s6, h6
-; CHECKNOFP16-NEXT:    fcvt h1, s4
-; CHECKNOFP16-NEXT:    fadd s4, s7, s5
-; CHECKNOFP16-NEXT:    mov h5, v0.h[4]
-; CHECKNOFP16-NEXT:    mov h7, v2.h[4]
-; CHECKNOFP16-NEXT:    fcvt h3, s3
-; CHECKNOFP16-NEXT:    fadd s6, s6, s16
-; CHECKNOFP16-NEXT:    mov h16, v2.h[5]
-; CHECKNOFP16-NEXT:    fcvt h4, s4
-; CHECKNOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECKNOFP16-NEXT:    fcvt s3, h5
-; CHECKNOFP16-NEXT:    fcvt s5, h7
-; CHECKNOFP16-NEXT:    mov h7, v0.h[5]
-; CHECKNOFP16-NEXT:    fcvt h6, s6
-; CHECKNOFP16-NEXT:    fcvt s16, h16
-; CHECKNOFP16-NEXT:    mov v1.h[2], v4.h[0]
-; CHECKNOFP16-NEXT:    mov h4, v0.h[6]
-; CHECKNOFP16-NEXT:    fadd s3, s5, s3
-; CHECKNOFP16-NEXT:    mov h5, v2.h[6]
-; CHECKNOFP16-NEXT:    fcvt s7, h7
-; CHECKNOFP16-NEXT:    mov h0, v0.h[7]
-; CHECKNOFP16-NEXT:    mov h2, v2.h[7]
-; CHECKNOFP16-NEXT:    mov v1.h[3], v6.h[0]
-; CHECKNOFP16-NEXT:    fcvt s4, h4
-; CHECKNOFP16-NEXT:    fcvt h3, s3
-; CHECKNOFP16-NEXT:    fcvt s5, h5
-; CHECKNOFP16-NEXT:    fadd s6, s16, s7
-; CHECKNOFP16-NEXT:    fcvt s0, h0
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    mov v1.h[4], v3.h[0]
-; CHECKNOFP16-NEXT:    fadd s4, s5, s4
-; CHECKNOFP16-NEXT:    fcvt h3, s6
-; CHECKNOFP16-NEXT:    fadd s0, s2, s0
-; CHECKNOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECKNOFP16-NEXT:    fcvt h3, s4
-; CHECKNOFP16-NEXT:    fcvt h0, s0
-; CHECKNOFP16-NEXT:    mov v1.h[6], v3.h[0]
-; CHECKNOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECKNOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECKNOFP16-NEXT:    rev32 v1.8h, v0.8h
+; CHECKNOFP16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECKNOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECKNOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECKNOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECKNOFP16-NEXT:    fadd v2.4s, v3.4s, v2.4s
+; CHECKNOFP16-NEXT:    fadd v1.4s, v1.4s, v0.4s
+; CHECKNOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECKNOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECKNOFP16-NEXT:    ret
 entry:
   %s = shufflevector <8 x half> %a, <8 x half> poison, <8 x i32> 
@@ -221,116 +185,24 @@ define <16 x half> @addp_v16f16(<16 x half> %a) {
 ;
 ; CHECKNOFP16-LABEL: addp_v16f16:
 ; CHECKNOFP16:       // %bb.0: // %entry
-; CHECKNOFP16-NEXT:    rev32 v5.8h, v0.8h
-; CHECKNOFP16-NEXT:    rev32 v4.8h, v1.8h
-; CHECKNOFP16-NEXT:    mov h3, v0.h[1]
-; CHECKNOFP16-NEXT:    mov h6, v1.h[1]
-; CHECKNOFP16-NEXT:    fcvt s16, h0
-; CHECKNOFP16-NEXT:    mov h17, v0.h[2]
-; CHECKNOFP16-NEXT:    fcvt s20, h1
-; CHECKNOFP16-NEXT:    mov h21, v1.h[2]
-; CHECKNOFP16-NEXT:    mov h2, v5.h[1]
-; CHECKNOFP16-NEXT:    mov h7, v4.h[1]
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fcvt s18, h5
-; CHECKNOFP16-NEXT:    mov h19, v5.h[2]
-; CHECKNOFP16-NEXT:    fcvt s6, h6
-; CHECKNOFP16-NEXT:    fcvt s22, h4
-; CHECKNOFP16-NEXT:    mov h23, v4.h[2]
-; CHECKNOFP16-NEXT:    fcvt s17, h17
-; CHECKNOFP16-NEXT:    mov h24, v5.h[3]
-; CHECKNOFP16-NEXT:    fcvt s21, h21
-; CHECKNOFP16-NEXT:    mov h25, v4.h[6]
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fcvt s7, h7
-; CHECKNOFP16-NEXT:    fadd s16, s18, s16
-; CHECKNOFP16-NEXT:    fcvt s18, h19
-; CHECKNOFP16-NEXT:    mov h19, v0.h[3]
-; CHECKNOFP16-NEXT:    fadd s20, s22, s20
-; CHECKNOFP16-NEXT:    fcvt s22, h23
-; CHECKNOFP16-NEXT:    mov h23, v4.h[3]
-; CHECKNOFP16-NEXT:    fadd s3, s2, s3
-; CHECKNOFP16-NEXT:    fadd s6, s7, s6
-; CHECKNOFP16-NEXT:    mov h7, v1.h[3]
-; CHECKNOFP16-NEXT:    fcvt h2, s16
-; CHECKNOFP16-NEXT:    fadd s16, s18, s17
-; CHECKNOFP16-NEXT:    fcvt s18, h19
-; CHECKNOFP16-NEXT:    fcvt s19, h24
-; CHECKNOFP16-NEXT:    mov h24, v5.h[6]
-; CHECKNOFP16-NEXT:    fcvt h17, s3
-; CHECKNOFP16-NEXT:    fcvt h3, s20
-; CHECKNOFP16-NEXT:    fadd s20, s22, s21
-; CHECKNOFP16-NEXT:    fcvt h6, s6
-; CHECKNOFP16-NEXT:    fcvt s7, h7
-; CHECKNOFP16-NEXT:    fcvt s22, h23
-; CHECKNOFP16-NEXT:    mov h21, v0.h[4]
-; CHECKNOFP16-NEXT:    mov h23, v5.h[4]
-; CHECKNOFP16-NEXT:    fcvt h16, s16
-; CHECKNOFP16-NEXT:    fadd s18, s19, s18
-; CHECKNOFP16-NEXT:    mov h19, v4.h[4]
-; CHECKNOFP16-NEXT:    mov v2.h[1], v17.h[0]
-; CHECKNOFP16-NEXT:    mov h17, v1.h[4]
-; CHECKNOFP16-NEXT:    mov v3.h[1], v6.h[0]
-; CHECKNOFP16-NEXT:    fcvt h6, s20
-; CHECKNOFP16-NEXT:    fadd s7, s22, s7
-; CHECKNOFP16-NEXT:    fcvt s20, h21
-; CHECKNOFP16-NEXT:    mov h21, v0.h[5]
-; CHECKNOFP16-NEXT:    mov h22, v5.h[5]
-; CHECKNOFP16-NEXT:    fcvt h18, s18
-; CHECKNOFP16-NEXT:    fcvt s19, h19
-; CHECKNOFP16-NEXT:    mov h5, v5.h[7]
-; CHECKNOFP16-NEXT:    mov v2.h[2], v16.h[0]
-; CHECKNOFP16-NEXT:    fcvt s16, h23
-; CHECKNOFP16-NEXT:    fcvt s17, h17
-; CHECKNOFP16-NEXT:    mov v3.h[2], v6.h[0]
-; CHECKNOFP16-NEXT:    fcvt h6, s7
-; CHECKNOFP16-NEXT:    mov h7, v1.h[5]
-; CHECKNOFP16-NEXT:    mov h23, v4.h[5]
-; CHECKNOFP16-NEXT:    mov h4, v4.h[7]
-; CHECKNOFP16-NEXT:    fcvt s5, h5
-; CHECKNOFP16-NEXT:    fadd s16, s16, s20
-; CHECKNOFP16-NEXT:    mov h20, v0.h[6]
-; CHECKNOFP16-NEXT:    fadd s17, s19, s17
-; CHECKNOFP16-NEXT:    mov h19, v1.h[6]
-; CHECKNOFP16-NEXT:    mov v2.h[3], v18.h[0]
-; CHECKNOFP16-NEXT:    fcvt s18, h21
-; CHECKNOFP16-NEXT:    fcvt s21, h22
-; CHECKNOFP16-NEXT:    mov v3.h[3], v6.h[0]
-; CHECKNOFP16-NEXT:    fcvt s6, h7
-; CHECKNOFP16-NEXT:    fcvt s7, h23
-; CHECKNOFP16-NEXT:    fcvt s22, h24
-; CHECKNOFP16-NEXT:    fcvt s23, h25
-; CHECKNOFP16-NEXT:    fcvt h16, s16
-; CHECKNOFP16-NEXT:    fcvt s20, h20
-; CHECKNOFP16-NEXT:    fcvt h17, s17
-; CHECKNOFP16-NEXT:    fcvt s19, h19
-; CHECKNOFP16-NEXT:    mov h0, v0.h[7]
-; CHECKNOFP16-NEXT:    mov h1, v1.h[7]
-; CHECKNOFP16-NEXT:    fadd s18, s21, s18
-; CHECKNOFP16-NEXT:    fcvt s4, h4
-; CHECKNOFP16-NEXT:    fadd s6, s7, s6
-; CHECKNOFP16-NEXT:    mov v2.h[4], v16.h[0]
-; CHECKNOFP16-NEXT:    fadd s7, s22, s20
-; CHECKNOFP16-NEXT:    mov v3.h[4], v17.h[0]
-; CHECKNOFP16-NEXT:    fadd s16, s23, s19
-; CHECKNOFP16-NEXT:    fcvt s0, h0
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fcvt h17, s18
-; CHECKNOFP16-NEXT:    fcvt h6, s6
-; CHECKNOFP16-NEXT:    fadd s0, s5, s0
-; CHECKNOFP16-NEXT:    fcvt h5, s7
-; CHECKNOFP16-NEXT:    fadd s1, s4, s1
-; CHECKNOFP16-NEXT:    mov v2.h[5], v17.h[0]
-; CHECKNOFP16-NEXT:    mov v3.h[5], v6.h[0]
-; CHECKNOFP16-NEXT:    fcvt h6, s16
-; CHECKNOFP16-NEXT:    fcvt h0, s0
-; CHECKNOFP16-NEXT:    fcvt h1, s1
-; CHECKNOFP16-NEXT:    mov v2.h[6], v5.h[0]
-; CHECKNOFP16-NEXT:    mov v3.h[6], v6.h[0]
-; CHECKNOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECKNOFP16-NEXT:    mov v3.h[7], v1.h[0]
-; CHECKNOFP16-NEXT:    mov v0.16b, v2.16b
-; CHECKNOFP16-NEXT:    mov v1.16b, v3.16b
+; CHECKNOFP16-NEXT:    rev32 v2.8h, v0.8h
+; CHECKNOFP16-NEXT:    rev32 v3.8h, v1.8h
+; CHECKNOFP16-NEXT:    fcvtl v4.4s, v0.4h
+; CHECKNOFP16-NEXT:    fcvtl v6.4s, v1.4h
+; CHECKNOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECKNOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECKNOFP16-NEXT:    fcvtl v5.4s, v2.4h
+; CHECKNOFP16-NEXT:    fcvtl v7.4s, v3.4h
+; CHECKNOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECKNOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
+; CHECKNOFP16-NEXT:    fadd v4.4s, v5.4s, v4.4s
+; CHECKNOFP16-NEXT:    fadd v5.4s, v7.4s, v6.4s
+; CHECKNOFP16-NEXT:    fadd v2.4s, v2.4s, v0.4s
+; CHECKNOFP16-NEXT:    fadd v3.4s, v3.4s, v1.4s
+; CHECKNOFP16-NEXT:    fcvtn v0.4h, v4.4s
+; CHECKNOFP16-NEXT:    fcvtn v1.4h, v5.4s
+; CHECKNOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECKNOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECKNOFP16-NEXT:    ret
 entry:
   %s = shufflevector <16 x half> %a, <16 x half> poison, <16 x i32> 
diff --git a/llvm/test/CodeGen/AArch64/faddsub.ll b/llvm/test/CodeGen/AArch64/faddsub.ll
index 31389f5a77d6..f8970dc9e8d5 100644
--- a/llvm/test/CodeGen/AArch64/faddsub.ll
+++ b/llvm/test/CodeGen/AArch64/faddsub.ll
@@ -169,60 +169,14 @@ entry:
 define <7 x half> @fadd_v7f16(<7 x half> %a, <7 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fadd_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fadd s5, s5, s16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h7
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s6, s3
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fadd s6, s16, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fadd v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v1.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fadd_v7f16:
@@ -309,60 +263,14 @@ entry:
 define <8 x half> @fadd_v8f16(<8 x half> %a, <8 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fadd_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fadd s5, s5, s16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h7
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s6, s3
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fadd s6, s16, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fadd v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v1.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fadd_v8f16:
@@ -394,114 +302,22 @@ entry:
 define <16 x half> @fadd_v16f16(<16 x half> %a, <16 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fadd_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h6, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-SD-NOFP16-NEXT:    mov h16, v3.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h18, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h1
-; CHECK-SD-NOFP16-NEXT:    mov h22, v3.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h24, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h19
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fadd s20, s21, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h22
-; CHECK-SD-NOFP16-NEXT:    mov h22, v3.h[3]
-; CHECK-SD-NOFP16-NEXT:    fadd s6, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h7, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h25, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fadd s5, s16, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h23
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fadd s17, s18, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt h19, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s20
-; CHECK-SD-NOFP16-NEXT:    fadd s16, s16, s21
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h23
-; CHECK-SD-NOFP16-NEXT:    fcvt h17, s17
-; CHECK-SD-NOFP16-NEXT:    mov h21, v2.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[1], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h22
-; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fadd s7, s18, s7
-; CHECK-SD-NOFP16-NEXT:    mov h18, v3.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[1], v19.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    fadd s6, s20, s6
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[2], v17.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h23
-; CHECK-SD-NOFP16-NEXT:    mov h21, v2.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[2], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v3.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    fadd s17, s19, s17
-; CHECK-SD-NOFP16-NEXT:    mov h19, v2.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[3], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    fadd s18, s20, s18
-; CHECK-SD-NOFP16-NEXT:    mov h20, v3.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h24
-; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[3], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h23
-; CHECK-SD-NOFP16-NEXT:    fcvt h17, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s23, h25
-; CHECK-SD-NOFP16-NEXT:    fcvt h18, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    mov h3, v3.h[7]
-; CHECK-SD-NOFP16-NEXT:    fadd s7, s21, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fadd s6, s16, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[4], v17.h[0]
-; CHECK-SD-NOFP16-NEXT:    fadd s16, s22, s19
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[4], v18.h[0]
-; CHECK-SD-NOFP16-NEXT:    fadd s17, s23, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s16
-; CHECK-SD-NOFP16-NEXT:    fadd s1, s1, s3
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[5], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[5], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[6], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v4.16b
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v5.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v4.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v5.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v6.4s, v3.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v7.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fadd v4.4s, v5.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v5.4s, v7.4s, v6.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v2.4s, v0.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v3.4s, v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fadd_v16f16:
@@ -705,60 +521,14 @@ entry:
 define <7 x half> @fsub_v7f16(<7 x half> %a, <7 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fsub_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fsub s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fsub s3, s3, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    fsub s4, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fsub s5, s5, s16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h7
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fsub s3, s6, s3
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fsub s6, s16, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fsub s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    fsub s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fsub v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fsub v1.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fsub_v7f16:
@@ -845,60 +615,14 @@ entry:
 define <8 x half> @fsub_v8f16(<8 x half> %a, <8 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fsub_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fsub s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fsub s3, s3, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    fsub s4, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fsub s5, s5, s16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h7
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fsub s3, s6, s3
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fsub s6, s16, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fsub s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    fsub s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fsub v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fsub v1.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fsub_v8f16:
@@ -930,114 +654,22 @@ entry:
 define <16 x half> @fsub_v16f16(<16 x half> %a, <16 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fsub_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h6, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-SD-NOFP16-NEXT:    mov h16, v3.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h18, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h1
-; CHECK-SD-NOFP16-NEXT:    mov h22, v3.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h24, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fsub s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h19
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fsub s20, s21, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h22
-; CHECK-SD-NOFP16-NEXT:    mov h22, v3.h[3]
-; CHECK-SD-NOFP16-NEXT:    fsub s6, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h7, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h25, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fsub s5, s16, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h23
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fsub s17, s18, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt h19, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s20
-; CHECK-SD-NOFP16-NEXT:    fsub s16, s16, s21
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h23
-; CHECK-SD-NOFP16-NEXT:    fcvt h17, s17
-; CHECK-SD-NOFP16-NEXT:    mov h21, v2.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[1], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h22
-; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fsub s7, s18, s7
-; CHECK-SD-NOFP16-NEXT:    mov h18, v3.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[1], v19.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    fsub s6, s20, s6
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[2], v17.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h23
-; CHECK-SD-NOFP16-NEXT:    mov h21, v2.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[2], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v3.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    fsub s17, s19, s17
-; CHECK-SD-NOFP16-NEXT:    mov h19, v2.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[3], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    fsub s18, s20, s18
-; CHECK-SD-NOFP16-NEXT:    mov h20, v3.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h24
-; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[3], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h23
-; CHECK-SD-NOFP16-NEXT:    fcvt h17, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s23, h25
-; CHECK-SD-NOFP16-NEXT:    fcvt h18, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    mov h3, v3.h[7]
-; CHECK-SD-NOFP16-NEXT:    fsub s7, s21, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fsub s6, s16, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[4], v17.h[0]
-; CHECK-SD-NOFP16-NEXT:    fsub s16, s22, s19
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[4], v18.h[0]
-; CHECK-SD-NOFP16-NEXT:    fsub s17, s23, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    fsub s0, s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s16
-; CHECK-SD-NOFP16-NEXT:    fsub s1, s1, s3
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[5], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[5], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[6], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v4.16b
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v5.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v4.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v5.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v6.4s, v3.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v7.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fsub v4.4s, v5.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fsub v5.4s, v7.4s, v6.4s
+; CHECK-SD-NOFP16-NEXT:    fsub v2.4s, v0.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fsub v3.4s, v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fsub_v16f16:
diff --git a/llvm/test/CodeGen/AArch64/fcvt.ll b/llvm/test/CodeGen/AArch64/fcvt.ll
index 584174d6d064..3b8a22a052b8 100644
--- a/llvm/test/CodeGen/AArch64/fcvt.ll
+++ b/llvm/test/CodeGen/AArch64/fcvt.ll
@@ -148,45 +148,12 @@ entry:
 define <7 x half> @ceil_v7f16(<7 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: ceil_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintp s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frintp s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintp s2, s3
-; CHECK-SD-NOFP16-NEXT:    frintp s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintp s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frintp s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintp s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frintp s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frintp v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frintp v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: ceil_v7f16:
@@ -234,26 +201,9 @@ entry:
 define <4 x half> @ceil_v4f16(<4 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: ceil_v4f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintp s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h4
-; CHECK-SD-NOFP16-NEXT:    frintp s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    frintp s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintp s2, s3
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[3], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    frintp v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: ceil_v4f16:
@@ -280,45 +230,12 @@ entry:
 define <8 x half> @ceil_v8f16(<8 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: ceil_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintp s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frintp s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintp s2, s3
-; CHECK-SD-NOFP16-NEXT:    frintp s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintp s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frintp s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintp s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frintp s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frintp v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frintp v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: ceil_v8f16:
@@ -348,84 +265,18 @@ entry:
 define <16 x half> @ceil_v16f16(<16 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: ceil_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h0
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h1
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    frintp s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    frintp s6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    frintp s18, s2
-; CHECK-SD-NOFP16-NEXT:    frintp s19, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    frintp s4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    frintp s6, s7
-; CHECK-SD-NOFP16-NEXT:    frintp s16, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintp s5, s17
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintp s18, s18
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintp s4, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    mov h19, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    frintp s7, s7
-; CHECK-SD-NOFP16-NEXT:    frintp s16, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintp s4, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s16
-; CHECK-SD-NOFP16-NEXT:    frintp s7, s17
-; CHECK-SD-NOFP16-NEXT:    frintp s0, s0
-; CHECK-SD-NOFP16-NEXT:    frintp s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    frintp v2.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frintp v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    frintp v3.4s, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frintp v2.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: ceil_v16f16:
@@ -604,45 +455,12 @@ entry:
 define <7 x half> @floor_v7f16(<7 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: floor_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintm s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frintm s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintm s2, s3
-; CHECK-SD-NOFP16-NEXT:    frintm s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintm s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frintm s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintm s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frintm s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frintm v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frintm v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: floor_v7f16:
@@ -690,26 +508,9 @@ entry:
 define <4 x half> @floor_v4f16(<4 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: floor_v4f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintm s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h4
-; CHECK-SD-NOFP16-NEXT:    frintm s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    frintm s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintm s2, s3
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[3], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    frintm v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: floor_v4f16:
@@ -736,45 +537,12 @@ entry:
 define <8 x half> @floor_v8f16(<8 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: floor_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintm s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frintm s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintm s2, s3
-; CHECK-SD-NOFP16-NEXT:    frintm s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintm s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frintm s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintm s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frintm s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frintm v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frintm v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: floor_v8f16:
@@ -804,84 +572,18 @@ entry:
 define <16 x half> @floor_v16f16(<16 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: floor_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h0
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h1
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    frintm s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    frintm s6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    frintm s18, s2
-; CHECK-SD-NOFP16-NEXT:    frintm s19, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    frintm s4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    frintm s6, s7
-; CHECK-SD-NOFP16-NEXT:    frintm s16, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintm s5, s17
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintm s18, s18
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintm s4, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    mov h19, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    frintm s7, s7
-; CHECK-SD-NOFP16-NEXT:    frintm s16, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintm s4, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s16
-; CHECK-SD-NOFP16-NEXT:    frintm s7, s17
-; CHECK-SD-NOFP16-NEXT:    frintm s0, s0
-; CHECK-SD-NOFP16-NEXT:    frintm s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    frintm v2.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frintm v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    frintm v3.4s, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frintm v2.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: floor_v16f16:
@@ -1060,45 +762,12 @@ entry:
 define <7 x half> @nearbyint_v7f16(<7 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: nearbyint_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frinti s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frinti s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frinti s2, s3
-; CHECK-SD-NOFP16-NEXT:    frinti s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frinti s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frinti s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frinti s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frinti s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frinti v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frinti v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: nearbyint_v7f16:
@@ -1146,26 +815,9 @@ entry:
 define <4 x half> @nearbyint_v4f16(<4 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: nearbyint_v4f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frinti s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h4
-; CHECK-SD-NOFP16-NEXT:    frinti s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    frinti s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frinti s2, s3
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[3], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    frinti v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: nearbyint_v4f16:
@@ -1192,45 +844,12 @@ entry:
 define <8 x half> @nearbyint_v8f16(<8 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: nearbyint_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frinti s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frinti s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frinti s2, s3
-; CHECK-SD-NOFP16-NEXT:    frinti s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frinti s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frinti s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frinti s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frinti s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frinti v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frinti v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: nearbyint_v8f16:
@@ -1260,84 +879,18 @@ entry:
 define <16 x half> @nearbyint_v16f16(<16 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: nearbyint_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h0
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h1
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    frinti s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    frinti s6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    frinti s18, s2
-; CHECK-SD-NOFP16-NEXT:    frinti s19, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    frinti s4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    frinti s6, s7
-; CHECK-SD-NOFP16-NEXT:    frinti s16, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    frinti s5, s17
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    frinti s18, s18
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frinti s4, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    mov h19, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    frinti s7, s7
-; CHECK-SD-NOFP16-NEXT:    frinti s16, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frinti s4, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s16
-; CHECK-SD-NOFP16-NEXT:    frinti s7, s17
-; CHECK-SD-NOFP16-NEXT:    frinti s0, s0
-; CHECK-SD-NOFP16-NEXT:    frinti s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    frinti v2.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frinti v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    frinti v3.4s, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frinti v2.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: nearbyint_v16f16:
@@ -1516,45 +1069,12 @@ entry:
 define <7 x half> @roundeven_v7f16(<7 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: roundeven_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintn s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frintn s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintn s2, s3
-; CHECK-SD-NOFP16-NEXT:    frintn s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintn s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frintn s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintn s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frintn s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frintn v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frintn v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: roundeven_v7f16:
@@ -1602,26 +1122,9 @@ entry:
 define <4 x half> @roundeven_v4f16(<4 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: roundeven_v4f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintn s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h4
-; CHECK-SD-NOFP16-NEXT:    frintn s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    frintn s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintn s2, s3
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[3], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    frintn v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: roundeven_v4f16:
@@ -1648,45 +1151,12 @@ entry:
 define <8 x half> @roundeven_v8f16(<8 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: roundeven_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintn s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frintn s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintn s2, s3
-; CHECK-SD-NOFP16-NEXT:    frintn s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintn s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frintn s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintn s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frintn s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frintn v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frintn v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: roundeven_v8f16:
@@ -1716,84 +1186,18 @@ entry:
 define <16 x half> @roundeven_v16f16(<16 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: roundeven_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h0
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h1
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    frintn s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    frintn s6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    frintn s18, s2
-; CHECK-SD-NOFP16-NEXT:    frintn s19, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    frintn s4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    frintn s6, s7
-; CHECK-SD-NOFP16-NEXT:    frintn s16, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintn s5, s17
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintn s18, s18
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintn s4, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    mov h19, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    frintn s7, s7
-; CHECK-SD-NOFP16-NEXT:    frintn s16, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintn s4, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s16
-; CHECK-SD-NOFP16-NEXT:    frintn s7, s17
-; CHECK-SD-NOFP16-NEXT:    frintn s0, s0
-; CHECK-SD-NOFP16-NEXT:    frintn s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    frintn v2.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frintn v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    frintn v3.4s, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frintn v2.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: roundeven_v16f16:
@@ -1972,45 +1376,12 @@ entry:
 define <7 x half> @rint_v7f16(<7 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: rint_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintx s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frintx s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintx s2, s3
-; CHECK-SD-NOFP16-NEXT:    frintx s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintx s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frintx s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintx s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frintx s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frintx v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frintx v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: rint_v7f16:
@@ -2058,26 +1429,9 @@ entry:
 define <4 x half> @rint_v4f16(<4 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: rint_v4f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintx s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h4
-; CHECK-SD-NOFP16-NEXT:    frintx s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    frintx s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintx s2, s3
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[3], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    frintx v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: rint_v4f16:
@@ -2104,45 +1458,12 @@ entry:
 define <8 x half> @rint_v8f16(<8 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: rint_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintx s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frintx s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintx s2, s3
-; CHECK-SD-NOFP16-NEXT:    frintx s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintx s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frintx s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintx s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frintx s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frintx v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frintx v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: rint_v8f16:
@@ -2172,84 +1493,18 @@ entry:
 define <16 x half> @rint_v16f16(<16 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: rint_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h0
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h1
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    frintx s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    frintx s6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    frintx s18, s2
-; CHECK-SD-NOFP16-NEXT:    frintx s19, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    frintx s4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    frintx s6, s7
-; CHECK-SD-NOFP16-NEXT:    frintx s16, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintx s5, s17
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintx s18, s18
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintx s4, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    mov h19, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    frintx s7, s7
-; CHECK-SD-NOFP16-NEXT:    frintx s16, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintx s4, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s16
-; CHECK-SD-NOFP16-NEXT:    frintx s7, s17
-; CHECK-SD-NOFP16-NEXT:    frintx s0, s0
-; CHECK-SD-NOFP16-NEXT:    frintx s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    frintx v2.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frintx v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    frintx v3.4s, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frintx v2.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: rint_v16f16:
@@ -2428,45 +1683,12 @@ entry:
 define <7 x half> @round_v7f16(<7 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: round_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frinta s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frinta s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frinta s2, s3
-; CHECK-SD-NOFP16-NEXT:    frinta s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frinta s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frinta s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frinta s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frinta s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frinta v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frinta v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: round_v7f16:
@@ -2514,26 +1736,9 @@ entry:
 define <4 x half> @round_v4f16(<4 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: round_v4f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frinta s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h4
-; CHECK-SD-NOFP16-NEXT:    frinta s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    frinta s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frinta s2, s3
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[3], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    frinta v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: round_v4f16:
@@ -2560,45 +1765,12 @@ entry:
 define <8 x half> @round_v8f16(<8 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: round_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frinta s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frinta s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frinta s2, s3
-; CHECK-SD-NOFP16-NEXT:    frinta s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frinta s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frinta s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frinta s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frinta s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frinta v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frinta v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: round_v8f16:
@@ -2628,84 +1800,18 @@ entry:
 define <16 x half> @round_v16f16(<16 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: round_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h0
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h1
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    frinta s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    frinta s6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    frinta s18, s2
-; CHECK-SD-NOFP16-NEXT:    frinta s19, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    frinta s4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    frinta s6, s7
-; CHECK-SD-NOFP16-NEXT:    frinta s16, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    frinta s5, s17
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    frinta s18, s18
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frinta s4, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    mov h19, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    frinta s7, s7
-; CHECK-SD-NOFP16-NEXT:    frinta s16, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frinta s4, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s16
-; CHECK-SD-NOFP16-NEXT:    frinta s7, s17
-; CHECK-SD-NOFP16-NEXT:    frinta s0, s0
-; CHECK-SD-NOFP16-NEXT:    frinta s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    frinta v2.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frinta v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    frinta v3.4s, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frinta v2.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: round_v16f16:
@@ -2884,45 +1990,12 @@ entry:
 define <7 x half> @trunc_v7f16(<7 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: trunc_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintz s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frintz s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintz s2, s3
-; CHECK-SD-NOFP16-NEXT:    frintz s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintz s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frintz s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintz s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frintz s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frintz v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frintz v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: trunc_v7f16:
@@ -2970,26 +2043,9 @@ entry:
 define <4 x half> @trunc_v4f16(<4 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: trunc_v4f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintz s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h4
-; CHECK-SD-NOFP16-NEXT:    frintz s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    frintz s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintz s2, s3
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[3], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    frintz v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: trunc_v4f16:
@@ -3016,45 +2072,12 @@ entry:
 define <8 x half> @trunc_v8f16(<8 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: trunc_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintz s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frintz s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintz s2, s3
-; CHECK-SD-NOFP16-NEXT:    frintz s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintz s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frintz s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintz s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frintz s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frintz v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frintz v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: trunc_v8f16:
@@ -3084,84 +2107,18 @@ entry:
 define <16 x half> @trunc_v16f16(<16 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: trunc_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h0
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h1
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    frintz s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    frintz s6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    frintz s18, s2
-; CHECK-SD-NOFP16-NEXT:    frintz s19, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    frintz s4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    frintz s6, s7
-; CHECK-SD-NOFP16-NEXT:    frintz s16, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintz s5, s17
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintz s18, s18
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintz s4, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    mov h19, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    frintz s7, s7
-; CHECK-SD-NOFP16-NEXT:    frintz s16, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintz s4, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s16
-; CHECK-SD-NOFP16-NEXT:    frintz s7, s17
-; CHECK-SD-NOFP16-NEXT:    frintz s0, s0
-; CHECK-SD-NOFP16-NEXT:    frintz s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    frintz v2.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frintz v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    frintz v3.4s, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frintz v2.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: trunc_v16f16:
diff --git a/llvm/test/CodeGen/AArch64/fcvt_combine.ll b/llvm/test/CodeGen/AArch64/fcvt_combine.ll
index 37133cf0aa1d..29170aab9656 100644
--- a/llvm/test/CodeGen/AArch64/fcvt_combine.ll
+++ b/llvm/test/CodeGen/AArch64/fcvt_combine.ll
@@ -180,48 +180,19 @@ define <3 x i32> @test_illegal_fp_to_int(<3 x float> %in) {
 define <8 x i16> @test_v8f16(<8 x half> %in) {
 ; CHECK-NO16-LABEL: test_v8f16:
 ; CHECK-NO16:       // %bb.0:
-; CHECK-NO16-NEXT:    mov h2, v0.h[1]
-; CHECK-NO16-NEXT:    mov h3, v0.h[4]
-; CHECK-NO16-NEXT:    mov h4, v0.h[5]
-; CHECK-NO16-NEXT:    mov h5, v0.h[2]
-; CHECK-NO16-NEXT:    fcvt s6, h0
-; CHECK-NO16-NEXT:    mov h7, v0.h[6]
-; CHECK-NO16-NEXT:    fmov s1, #4.00000000
-; CHECK-NO16-NEXT:    mov h16, v0.h[3]
-; CHECK-NO16-NEXT:    mov h0, v0.h[7]
-; CHECK-NO16-NEXT:    fcvt s2, h2
-; CHECK-NO16-NEXT:    fcvt s3, h3
-; CHECK-NO16-NEXT:    fcvt s4, h4
-; CHECK-NO16-NEXT:    fmul s6, s6, s1
-; CHECK-NO16-NEXT:    fcvt s5, h5
-; CHECK-NO16-NEXT:    fcvt s7, h7
-; CHECK-NO16-NEXT:    fcvt s16, h16
-; CHECK-NO16-NEXT:    fcvt s0, h0
-; CHECK-NO16-NEXT:    fmul s2, s2, s1
-; CHECK-NO16-NEXT:    fmul s3, s3, s1
-; CHECK-NO16-NEXT:    fmul s4, s4, s1
-; CHECK-NO16-NEXT:    fmul s5, s5, s1
-; CHECK-NO16-NEXT:    fcvt h6, s6
-; CHECK-NO16-NEXT:    fmul s7, s7, s1
-; CHECK-NO16-NEXT:    fmul s16, s16, s1
-; CHECK-NO16-NEXT:    fmul s0, s0, s1
-; CHECK-NO16-NEXT:    fcvt h2, s2
-; CHECK-NO16-NEXT:    fcvt h3, s3
-; CHECK-NO16-NEXT:    fcvt h4, s4
-; CHECK-NO16-NEXT:    fcvt h5, s5
-; CHECK-NO16-NEXT:    fcvt h1, s7
-; CHECK-NO16-NEXT:    fcvt h0, s0
-; CHECK-NO16-NEXT:    mov v6.h[1], v2.h[0]
-; CHECK-NO16-NEXT:    fcvt h2, s16
-; CHECK-NO16-NEXT:    mov v3.h[1], v4.h[0]
-; CHECK-NO16-NEXT:    mov v6.h[2], v5.h[0]
-; CHECK-NO16-NEXT:    mov v3.h[2], v1.h[0]
-; CHECK-NO16-NEXT:    mov v6.h[3], v2.h[0]
-; CHECK-NO16-NEXT:    mov v3.h[3], v0.h[0]
-; CHECK-NO16-NEXT:    fcvtl v1.4s, v6.4h
-; CHECK-NO16-NEXT:    fcvtl v0.4s, v3.4h
-; CHECK-NO16-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NO16-NEXT:    movi v1.8h, #68, lsl #8
+; CHECK-NO16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-NO16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NO16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-NO16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-NO16-NEXT:    fmul v2.4s, v2.4s, v3.4s
+; CHECK-NO16-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-NO16-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-NO16-NEXT:    fcvtn2 v1.8h, v0.4s
+; CHECK-NO16-NEXT:    fcvtl2 v0.4s, v1.8h
+; CHECK-NO16-NEXT:    fcvtl v1.4s, v1.4h
 ; CHECK-NO16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NO16-NEXT:    fcvtzs v1.4s, v1.4s
 ; CHECK-NO16-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
 ; CHECK-NO16-NEXT:    ret
 ;
@@ -496,96 +467,67 @@ define <3 x i32> @test_illegal_fp_to_int_sat_sat(<3 x float> %in) {
 define <8 x i16> @test_v8f16_sat(<8 x half> %in) {
 ; CHECK-NO16-LABEL: test_v8f16_sat:
 ; CHECK-NO16:       // %bb.0:
-; CHECK-NO16-NEXT:    mov h2, v0.h[4]
-; CHECK-NO16-NEXT:    mov h3, v0.h[5]
+; CHECK-NO16-NEXT:    movi v1.8h, #68, lsl #8
+; CHECK-NO16-NEXT:    fcvtl v2.4s, v0.4h
 ; CHECK-NO16-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-NO16-NEXT:    mov h4, v0.h[6]
-; CHECK-NO16-NEXT:    fmov s1, #4.00000000
+; CHECK-NO16-NEXT:    fcvtl2 v0.4s, v0.8h
 ; CHECK-NO16-NEXT:    mov w11, #-32768 // =0xffff8000
-; CHECK-NO16-NEXT:    mov h5, v0.h[7]
-; CHECK-NO16-NEXT:    mov h6, v0.h[1]
-; CHECK-NO16-NEXT:    mov h7, v0.h[2]
-; CHECK-NO16-NEXT:    fcvt s16, h0
-; CHECK-NO16-NEXT:    mov h0, v0.h[3]
-; CHECK-NO16-NEXT:    fcvt s2, h2
-; CHECK-NO16-NEXT:    fcvt s3, h3
-; CHECK-NO16-NEXT:    fcvt s4, h4
-; CHECK-NO16-NEXT:    fcvt s5, h5
-; CHECK-NO16-NEXT:    fcvt s6, h6
-; CHECK-NO16-NEXT:    fmul s2, s2, s1
-; CHECK-NO16-NEXT:    fmul s3, s3, s1
-; CHECK-NO16-NEXT:    fmul s4, s4, s1
-; CHECK-NO16-NEXT:    fmul s5, s5, s1
-; CHECK-NO16-NEXT:    fmul s6, s6, s1
-; CHECK-NO16-NEXT:    fcvt h2, s2
-; CHECK-NO16-NEXT:    fcvt h3, s3
-; CHECK-NO16-NEXT:    fcvt h4, s4
-; CHECK-NO16-NEXT:    fcvt h5, s5
-; CHECK-NO16-NEXT:    fcvt h6, s6
-; CHECK-NO16-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-NO16-NEXT:    fcvt s3, h7
-; CHECK-NO16-NEXT:    fmul s7, s16, s1
-; CHECK-NO16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-NO16-NEXT:    fcvt s4, h0
-; CHECK-NO16-NEXT:    fmul s3, s3, s1
-; CHECK-NO16-NEXT:    fcvt h0, s7
-; CHECK-NO16-NEXT:    mov v2.h[3], v5.h[0]
-; CHECK-NO16-NEXT:    fmul s1, s4, s1
-; CHECK-NO16-NEXT:    fcvt h3, s3
-; CHECK-NO16-NEXT:    mov v0.h[1], v6.h[0]
-; CHECK-NO16-NEXT:    fcvtl v2.4s, v2.4h
-; CHECK-NO16-NEXT:    fcvt h1, s1
-; CHECK-NO16-NEXT:    mov v0.h[2], v3.h[0]
-; CHECK-NO16-NEXT:    mov s4, v2.s[1]
-; CHECK-NO16-NEXT:    fcvtzs w10, s2
-; CHECK-NO16-NEXT:    mov v0.h[3], v1.h[0]
-; CHECK-NO16-NEXT:    mov s1, v2.s[2]
-; CHECK-NO16-NEXT:    mov s2, v2.s[3]
-; CHECK-NO16-NEXT:    fcvtzs w9, s4
-; CHECK-NO16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-NO16-NEXT:    fcvtzs w12, s1
-; CHECK-NO16-NEXT:    fcvtzs w13, s2
+; CHECK-NO16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-NO16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-NO16-NEXT:    fmul v2.4s, v2.4s, v3.4s
+; CHECK-NO16-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-NO16-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-NO16-NEXT:    fcvtn2 v1.8h, v0.4s
+; CHECK-NO16-NEXT:    fcvtl2 v0.4s, v1.8h
+; CHECK-NO16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-NO16-NEXT:    mov s2, v0.s[1]
+; CHECK-NO16-NEXT:    fcvtzs w10, s0
+; CHECK-NO16-NEXT:    fcvtzs w15, s1
+; CHECK-NO16-NEXT:    fcvtzs w9, s2
+; CHECK-NO16-NEXT:    mov s2, v0.s[2]
+; CHECK-NO16-NEXT:    mov s0, v0.s[3]
 ; CHECK-NO16-NEXT:    cmp w9, w8
+; CHECK-NO16-NEXT:    fcvtzs w12, s2
+; CHECK-NO16-NEXT:    mov s2, v1.s[1]
 ; CHECK-NO16-NEXT:    csel w9, w9, w8, lt
-; CHECK-NO16-NEXT:    mov s1, v0.s[1]
-; CHECK-NO16-NEXT:    fcvtzs w15, s0
+; CHECK-NO16-NEXT:    fcvtzs w13, s0
+; CHECK-NO16-NEXT:    mov s0, v1.s[2]
 ; CHECK-NO16-NEXT:    cmn w9, #8, lsl #12 // =32768
 ; CHECK-NO16-NEXT:    csel w9, w9, w11, gt
 ; CHECK-NO16-NEXT:    cmp w10, w8
 ; CHECK-NO16-NEXT:    csel w10, w10, w8, lt
+; CHECK-NO16-NEXT:    fcvtzs w14, s2
 ; CHECK-NO16-NEXT:    cmn w10, #8, lsl #12 // =32768
-; CHECK-NO16-NEXT:    fcvtzs w14, s1
-; CHECK-NO16-NEXT:    mov s1, v0.s[2]
+; CHECK-NO16-NEXT:    fcvtzs w16, s0
+; CHECK-NO16-NEXT:    mov s0, v1.s[3]
 ; CHECK-NO16-NEXT:    csel w10, w10, w11, gt
 ; CHECK-NO16-NEXT:    cmp w12, w8
-; CHECK-NO16-NEXT:    mov s0, v0.s[3]
 ; CHECK-NO16-NEXT:    csel w12, w12, w8, lt
+; CHECK-NO16-NEXT:    fmov s1, w10
 ; CHECK-NO16-NEXT:    cmn w12, #8, lsl #12 // =32768
 ; CHECK-NO16-NEXT:    csel w12, w12, w11, gt
 ; CHECK-NO16-NEXT:    cmp w13, w8
-; CHECK-NO16-NEXT:    fcvtzs w16, s1
 ; CHECK-NO16-NEXT:    csel w13, w13, w8, lt
-; CHECK-NO16-NEXT:    fmov s1, w10
+; CHECK-NO16-NEXT:    mov v1.s[1], w9
+; CHECK-NO16-NEXT:    fcvtzs w9, s0
 ; CHECK-NO16-NEXT:    cmn w13, #8, lsl #12 // =32768
 ; CHECK-NO16-NEXT:    csel w13, w13, w11, gt
 ; CHECK-NO16-NEXT:    cmp w14, w8
 ; CHECK-NO16-NEXT:    csel w14, w14, w8, lt
-; CHECK-NO16-NEXT:    mov v1.s[1], w9
-; CHECK-NO16-NEXT:    fcvtzs w9, s0
 ; CHECK-NO16-NEXT:    cmn w14, #8, lsl #12 // =32768
+; CHECK-NO16-NEXT:    mov v1.s[2], w12
 ; CHECK-NO16-NEXT:    csel w14, w14, w11, gt
 ; CHECK-NO16-NEXT:    cmp w15, w8
 ; CHECK-NO16-NEXT:    csel w15, w15, w8, lt
 ; CHECK-NO16-NEXT:    cmn w15, #8, lsl #12 // =32768
-; CHECK-NO16-NEXT:    mov v1.s[2], w12
 ; CHECK-NO16-NEXT:    csel w10, w15, w11, gt
 ; CHECK-NO16-NEXT:    cmp w16, w8
+; CHECK-NO16-NEXT:    mov v1.s[3], w13
 ; CHECK-NO16-NEXT:    fmov s2, w10
 ; CHECK-NO16-NEXT:    csel w10, w16, w8, lt
 ; CHECK-NO16-NEXT:    cmn w10, #8, lsl #12 // =32768
 ; CHECK-NO16-NEXT:    csel w10, w10, w11, gt
 ; CHECK-NO16-NEXT:    cmp w9, w8
-; CHECK-NO16-NEXT:    mov v1.s[3], w13
 ; CHECK-NO16-NEXT:    mov v2.s[1], w14
 ; CHECK-NO16-NEXT:    csel w8, w9, w8, lt
 ; CHECK-NO16-NEXT:    cmn w8, #8, lsl #12 // =32768
diff --git a/llvm/test/CodeGen/AArch64/fdiv.ll b/llvm/test/CodeGen/AArch64/fdiv.ll
index fa87c4fa2d16..e73124fbb595 100644
--- a/llvm/test/CodeGen/AArch64/fdiv.ll
+++ b/llvm/test/CodeGen/AArch64/fdiv.ll
@@ -169,60 +169,14 @@ entry:
 define <7 x half> @fdiv_v7f16(<7 x half> %a, <7 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fdiv_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h0
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fdiv s2, s3, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h1
-; CHECK-SD-NOFP16-NEXT:    fdiv s3, s4, s3
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt h18, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fdiv s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v18.h[0]
-; CHECK-SD-NOFP16-NEXT:    fdiv s5, s6, s5
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fdiv s6, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fdiv s7, s16, s7
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fdiv s3, s17, s16
-; CHECK-SD-NOFP16-NEXT:    fdiv s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s6
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s7
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s3
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fdiv v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fdiv v1.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fdiv_v7f16:
@@ -309,60 +263,14 @@ entry:
 define <8 x half> @fdiv_v8f16(<8 x half> %a, <8 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fdiv_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h0
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fdiv s2, s3, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h1
-; CHECK-SD-NOFP16-NEXT:    fdiv s3, s4, s3
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt h18, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fdiv s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v18.h[0]
-; CHECK-SD-NOFP16-NEXT:    fdiv s5, s6, s5
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fdiv s6, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fdiv s7, s16, s7
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fdiv s3, s17, s16
-; CHECK-SD-NOFP16-NEXT:    fdiv s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s6
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s7
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s3
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fdiv v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fdiv v1.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fdiv_v8f16:
@@ -394,113 +302,22 @@ entry:
 define <16 x half> @fdiv_v16f16(<16 x half> %a, <16 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fdiv_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h4, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h0
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h18, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h1
-; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h22, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h22
-; CHECK-SD-NOFP16-NEXT:    mov h24, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fdiv s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s23, h23
-; CHECK-SD-NOFP16-NEXT:    mov h25, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h24
-; CHECK-SD-NOFP16-NEXT:    fcvt s25, h25
-; CHECK-SD-NOFP16-NEXT:    fdiv s5, s6, s5
-; CHECK-SD-NOFP16-NEXT:    mov h6, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fdiv s7, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h6, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fdiv s6, s16, s6
-; CHECK-SD-NOFP16-NEXT:    mov h16, v2.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fdiv s16, s17, s16
-; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fdiv s17, s18, s17
-; CHECK-SD-NOFP16-NEXT:    mov h18, v2.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fdiv s18, s19, s18
-; CHECK-SD-NOFP16-NEXT:    fdiv s19, s0, s2
-; CHECK-SD-NOFP16-NEXT:    mov h0, v3.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fdiv s2, s2, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h3
-; CHECK-SD-NOFP16-NEXT:    fdiv s20, s20, s0
-; CHECK-SD-NOFP16-NEXT:    mov h0, v3.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fdiv s21, s21, s0
-; CHECK-SD-NOFP16-NEXT:    mov h0, v3.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fdiv s22, s22, s0
-; CHECK-SD-NOFP16-NEXT:    mov h0, v3.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fdiv s23, s23, s0
-; CHECK-SD-NOFP16-NEXT:    mov h0, v3.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fdiv s24, s24, s0
-; CHECK-SD-NOFP16-NEXT:    mov h0, v3.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v3.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s26, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s7
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s21
-; CHECK-SD-NOFP16-NEXT:    fdiv s20, s25, s26
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s6
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s22
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s23
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fdiv s1, s1, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s24
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[4], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s17
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s20
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s18
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[6], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[7], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v2.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fdiv v4.4s, v5.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v5.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fdiv v0.4s, v0.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v3.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
+; CHECK-SD-NOFP16-NEXT:    fdiv v2.4s, v5.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fdiv v3.4s, v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fdiv_v16f16:
diff --git a/llvm/test/CodeGen/AArch64/fmla.ll b/llvm/test/CodeGen/AArch64/fmla.ll
index 339ade5fc707..336c9705f399 100644
--- a/llvm/test/CodeGen/AArch64/fmla.ll
+++ b/llvm/test/CodeGen/AArch64/fmla.ll
@@ -840,99 +840,22 @@ entry:
 define <7 x half> @fmuladd_v7f16(<7 x half> %a, <7 x half> %b, <7 x half> %c) {
 ; CHECK-SD-NOFP16-LABEL: fmuladd_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h0
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h18, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h2
-; CHECK-SD-NOFP16-NEXT:    mov h20, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fmul s5, s6, s5
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s4, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h17
-; CHECK-SD-NOFP16-NEXT:    fmul s7, s16, s7
-; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fmul s5, s6, s5
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s4, s19
-; CHECK-SD-NOFP16-NEXT:    mov h19, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fmul s16, s18, s16
-; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fadd s17, s3, s17
-; CHECK-SD-NOFP16-NEXT:    fmul s6, s6, s21
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s7, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s17
-; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fadd s5, s5, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fmul s18, s20, s18
-; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    mov h1, v2.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v2.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fadd s16, s16, s17
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fadd s6, s6, s7
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s6
-; CHECK-SD-NOFP16-NEXT:    fadd s1, s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v4.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fmul v3.4s, v4.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v0.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v1.4s, v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fmuladd_v7f16:
@@ -1045,99 +968,22 @@ entry:
 define <8 x half> @fmuladd_v8f16(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
 ; CHECK-SD-NOFP16-LABEL: fmuladd_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h0
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h18, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h2
-; CHECK-SD-NOFP16-NEXT:    mov h20, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fmul s5, s6, s5
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s4, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h17
-; CHECK-SD-NOFP16-NEXT:    fmul s7, s16, s7
-; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fmul s5, s6, s5
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s4, s19
-; CHECK-SD-NOFP16-NEXT:    mov h19, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fmul s16, s18, s16
-; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fadd s17, s3, s17
-; CHECK-SD-NOFP16-NEXT:    fmul s6, s6, s21
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s7, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s17
-; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fadd s5, s5, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fmul s18, s20, s18
-; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    mov h1, v2.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v2.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fadd s16, s16, s17
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fadd s6, s6, s7
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s6
-; CHECK-SD-NOFP16-NEXT:    fadd s1, s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v4.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fmul v3.4s, v4.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v0.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v1.4s, v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fmuladd_v8f16:
@@ -1179,192 +1025,38 @@ entry:
 define <16 x half> @fmuladd_v16f16(<16 x half> %a, <16 x half> %b, <16 x half> %c) {
 ; CHECK-SD-NOFP16-LABEL: fmuladd_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h6, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h0
-; CHECK-SD-NOFP16-NEXT:    mov h18, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h21, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h24, v3.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h25, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h26, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s27, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h29, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fmul s16, s17, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    mov h17, v4.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h24
-; CHECK-SD-NOFP16-NEXT:    fcvt s25, h25
-; CHECK-SD-NOFP16-NEXT:    mov h30, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fmul s6, s7, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h4
-; CHECK-SD-NOFP16-NEXT:    fmul s23, s19, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h22, s16
-; CHECK-SD-NOFP16-NEXT:    mov h18, v2.h[4]
-; CHECK-SD-NOFP16-NEXT:    fmul s20, s21, s20
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v4.h[2]
-; CHECK-SD-NOFP16-NEXT:    fmul s24, s25, s24
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    mov h25, v3.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt h22, s23
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt h20, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt s23, h6
-; CHECK-SD-NOFP16-NEXT:    mov h6, v4.h[3]
-; CHECK-SD-NOFP16-NEXT:    fadd s7, s21, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s18, s19, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h20
-; CHECK-SD-NOFP16-NEXT:    fadd s17, s23, s17
-; CHECK-SD-NOFP16-NEXT:    mov h23, v3.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s28, h6
-; CHECK-SD-NOFP16-NEXT:    fmul s22, s27, s22
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s7
-; CHECK-SD-NOFP16-NEXT:    fadd s7, s21, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h26
-; CHECK-SD-NOFP16-NEXT:    mov h26, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt h18, s18
-; CHECK-SD-NOFP16-NEXT:    mov h27, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s17
-; CHECK-SD-NOFP16-NEXT:    mov h17, v4.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h23
-; CHECK-SD-NOFP16-NEXT:    fadd s19, s19, s28
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    mov h23, v2.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fmul s20, s21, s20
-; CHECK-SD-NOFP16-NEXT:    mov v6.h[1], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v5.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt h21, s22
-; CHECK-SD-NOFP16-NEXT:    fcvt h22, s24
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h25
-; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
-; CHECK-SD-NOFP16-NEXT:    mov h26, v5.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt h19, s19
-; CHECK-SD-NOFP16-NEXT:    mov v6.h[2], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v3.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s28, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt h20, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h22
-; CHECK-SD-NOFP16-NEXT:    fadd s16, s18, s17
-; CHECK-SD-NOFP16-NEXT:    fmul s18, s25, s24
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h5
-; CHECK-SD-NOFP16-NEXT:    mov h24, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
-; CHECK-SD-NOFP16-NEXT:    fcvt s26, h27
-; CHECK-SD-NOFP16-NEXT:    mov v6.h[3], v19.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h23
-; CHECK-SD-NOFP16-NEXT:    mov h23, v2.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s27, h29
-; CHECK-SD-NOFP16-NEXT:    fadd s22, s22, s28
-; CHECK-SD-NOFP16-NEXT:    fadd s17, s21, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt h18, s18
-; CHECK-SD-NOFP16-NEXT:    mov h21, v5.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h28, v3.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h29, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fmul s19, s26, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s23, h23
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h24
-; CHECK-SD-NOFP16-NEXT:    fadd s20, s20, s25
-; CHECK-SD-NOFP16-NEXT:    fmul s25, s27, s7
-; CHECK-SD-NOFP16-NEXT:    mov h27, v3.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt h22, s22
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h28
-; CHECK-SD-NOFP16-NEXT:    fcvt s28, h29
-; CHECK-SD-NOFP16-NEXT:    fmul s23, s24, s23
-; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h27
-; CHECK-SD-NOFP16-NEXT:    fcvt s26, h30
-; CHECK-SD-NOFP16-NEXT:    fcvt h20, s20
-; CHECK-SD-NOFP16-NEXT:    mov v7.h[1], v22.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h22, s25
-; CHECK-SD-NOFP16-NEXT:    mov h25, v5.h[4]
-; CHECK-SD-NOFP16-NEXT:    fadd s17, s17, s18
-; CHECK-SD-NOFP16-NEXT:    fmul s18, s28, s21
-; CHECK-SD-NOFP16-NEXT:    mov h3, v3.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt h19, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fmul s21, s26, s24
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov h24, v4.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt s25, h25
-; CHECK-SD-NOFP16-NEXT:    mov v7.h[2], v20.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h17, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt h18, s18
-; CHECK-SD-NOFP16-NEXT:    mov h20, v5.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt h23, s23
-; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
-; CHECK-SD-NOFP16-NEXT:    mov h2, v4.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt h21, s21
-; CHECK-SD-NOFP16-NEXT:    fadd s22, s22, s25
-; CHECK-SD-NOFP16-NEXT:    mov h25, v5.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    mov v7.h[3], v17.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h24
-; CHECK-SD-NOFP16-NEXT:    fmul s1, s1, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h23
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt h22, s22
-; CHECK-SD-NOFP16-NEXT:    fcvt s23, h25
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fadd s17, s17, s19
-; CHECK-SD-NOFP16-NEXT:    mov h4, v4.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v5.h[7]
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v6.h[4], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s20, s2
-; CHECK-SD-NOFP16-NEXT:    mov v7.h[4], v22.h[0]
-; CHECK-SD-NOFP16-NEXT:    fadd s16, s21, s23
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt h17, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v6.h[5], v17.h[0]
-; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s4
-; CHECK-SD-NOFP16-NEXT:    mov v7.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s16
-; CHECK-SD-NOFP16-NEXT:    fadd s1, s1, s5
-; CHECK-SD-NOFP16-NEXT:    mov v6.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v7.h[6], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v6.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v7.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v6.16b
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v7.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v6.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v7.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v16.4s, v3.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v17.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fmul v6.4s, v7.4s, v6.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v7.4s, v17.4s, v16.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v1.4s, v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v2.4h, v6.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v6.4s, v5.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtn v3.4h, v7.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v2.8h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v4.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v4.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v3.8h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v7.4s, v3.4h
+; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v1.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v3.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v5.8h
+; CHECK-SD-NOFP16-NEXT:    fadd v5.4s, v7.4s, v6.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v2.4s, v2.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v3.4s, v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fmuladd_v16f16:
@@ -1650,99 +1342,22 @@ entry:
 define <7 x half> @fmul_v7f16(<7 x half> %a, <7 x half> %b, <7 x half> %c) {
 ; CHECK-SD-NOFP16-LABEL: fmul_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h0
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h18, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h2
-; CHECK-SD-NOFP16-NEXT:    mov h22, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fmul s5, s6, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h18
-; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s4, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h17
-; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fmul s4, s6, s4
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fmul s7, s16, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h19
-; CHECK-SD-NOFP16-NEXT:    mov h19, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    fadd s5, s5, s21
-; CHECK-SD-NOFP16-NEXT:    mov h21, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fmul s17, s18, s17
-; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fmul s6, s20, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h20, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s5
-; CHECK-SD-NOFP16-NEXT:    mov h5, v2.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s4, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h21
-; CHECK-SD-NOFP16-NEXT:    fadd s7, s7, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v2.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fmul s17, s18, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s5, s16, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v2.h[6]
-; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s17
-; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s6, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s6, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v4.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fmul v3.4s, v4.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v0.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v1.4s, v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fmul_v7f16:
@@ -1857,99 +1472,22 @@ entry:
 define <8 x half> @fmul_v8f16(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
 ; CHECK-SD-NOFP16-LABEL: fmul_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h0
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h18, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h2
-; CHECK-SD-NOFP16-NEXT:    mov h22, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fmul s5, s6, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h18
-; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s4, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h17
-; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fmul s4, s6, s4
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fmul s7, s16, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h19
-; CHECK-SD-NOFP16-NEXT:    mov h19, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    fadd s5, s5, s21
-; CHECK-SD-NOFP16-NEXT:    mov h21, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fmul s17, s18, s17
-; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fmul s6, s20, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h20, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s5
-; CHECK-SD-NOFP16-NEXT:    mov h5, v2.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s4, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h21
-; CHECK-SD-NOFP16-NEXT:    fadd s7, s7, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v2.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fmul s17, s18, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s5, s16, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v2.h[6]
-; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s17
-; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s6, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s6, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v4.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fmul v3.4s, v4.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v0.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v1.4s, v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fmul_v8f16:
@@ -1992,199 +1530,38 @@ entry:
 define <16 x half> @fmul_v16f16(<16 x half> %a, <16 x half> %b, <16 x half> %c) {
 ; CHECK-SD-NOFP16-LABEL: fmul_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    stp d11, d10, [sp, #-32]! // 16-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    stp d9, d8, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset b8, -8
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset b9, -16
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset b10, -24
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset b11, -32
-; CHECK-SD-NOFP16-NEXT:    mov h6, v3.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v3.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h22, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h23, v3.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h24, v3.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h26, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v3.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h27, v3.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h28, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s25, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h23
-; CHECK-SD-NOFP16-NEXT:    fcvt s23, h24
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h26
-; CHECK-SD-NOFP16-NEXT:    mov h26, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v3.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s29, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fmul s6, s25, s6
-; CHECK-SD-NOFP16-NEXT:    mov h25, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s27, h27
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s22, s21
-; CHECK-SD-NOFP16-NEXT:    fcvt s28, h28
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fmul s23, s24, s23
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h25
-; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
-; CHECK-SD-NOFP16-NEXT:    fmul s26, s1, s29
-; CHECK-SD-NOFP16-NEXT:    fmul s27, s28, s27
-; CHECK-SD-NOFP16-NEXT:    mov h28, v2.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fmul s1, s18, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt h17, s23
-; CHECK-SD-NOFP16-NEXT:    mov h29, v4.h[1]
-; CHECK-SD-NOFP16-NEXT:    fmul s21, s22, s21
-; CHECK-SD-NOFP16-NEXT:    fmul s16, s16, s19
-; CHECK-SD-NOFP16-NEXT:    mov h8, v2.h[6]
-; CHECK-SD-NOFP16-NEXT:    fmul s23, s25, s24
-; CHECK-SD-NOFP16-NEXT:    mov h24, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h25, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fmul s7, s7, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt h18, s26
-; CHECK-SD-NOFP16-NEXT:    fcvt h19, s27
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h28
-; CHECK-SD-NOFP16-NEXT:    mov h26, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h27, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt h21, s21
-; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h11, v2.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h28, s23
-; CHECK-SD-NOFP16-NEXT:    fcvt s30, h24
-; CHECK-SD-NOFP16-NEXT:    fcvt s31, h25
-; CHECK-SD-NOFP16-NEXT:    mov h24, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s29, h29
-; CHECK-SD-NOFP16-NEXT:    mov h9, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
-; CHECK-SD-NOFP16-NEXT:    fcvt s26, h27
-; CHECK-SD-NOFP16-NEXT:    mov h10, v2.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    mov h23, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h11
-; CHECK-SD-NOFP16-NEXT:    fmul s27, s31, s30
-; CHECK-SD-NOFP16-NEXT:    fcvt s28, h28
-; CHECK-SD-NOFP16-NEXT:    fcvt s30, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s31, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h8
-; CHECK-SD-NOFP16-NEXT:    mov h8, v5.h[1]
-; CHECK-SD-NOFP16-NEXT:    fmul s25, s26, s25
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h24
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s20, s22
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h9
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h10
-; CHECK-SD-NOFP16-NEXT:    fadd s26, s28, s29
-; CHECK-SD-NOFP16-NEXT:    fcvt s23, h23
-; CHECK-SD-NOFP16-NEXT:    fcvt h27, s27
-; CHECK-SD-NOFP16-NEXT:    fadd s28, s30, s31
-; CHECK-SD-NOFP16-NEXT:    mov h29, v4.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h30, v5.h[2]
-; CHECK-SD-NOFP16-NEXT:    fmul s24, s24, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt s31, h8
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fmul s21, s22, s21
-; CHECK-SD-NOFP16-NEXT:    fcvt s8, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h25, s25
-; CHECK-SD-NOFP16-NEXT:    fmul s20, s23, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt h26, s26
-; CHECK-SD-NOFP16-NEXT:    fcvt s27, h27
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s28
-; CHECK-SD-NOFP16-NEXT:    mov h28, v4.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s29, h29
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s30, h30
-; CHECK-SD-NOFP16-NEXT:    fadd s19, s19, s31
-; CHECK-SD-NOFP16-NEXT:    fadd s18, s18, s8
-; CHECK-SD-NOFP16-NEXT:    ldp d9, d8, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    fcvt h22, s1
-; CHECK-SD-NOFP16-NEXT:    mov h23, v5.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s25, h25
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v26.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s26, h28
-; CHECK-SD-NOFP16-NEXT:    fadd s27, s27, s29
-; CHECK-SD-NOFP16-NEXT:    fcvt h24, s24
-; CHECK-SD-NOFP16-NEXT:    fadd s17, s17, s30
-; CHECK-SD-NOFP16-NEXT:    mov h28, v4.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h19, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h23
-; CHECK-SD-NOFP16-NEXT:    fcvt h20, s20
-; CHECK-SD-NOFP16-NEXT:    fadd s23, s25, s26
-; CHECK-SD-NOFP16-NEXT:    mov h25, v5.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h26, s27
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h24
-; CHECK-SD-NOFP16-NEXT:    fcvt s27, h28
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v19.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v4.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h17, s17
-; CHECK-SD-NOFP16-NEXT:    fadd s18, s18, s22
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt h21, s21
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h25
-; CHECK-SD-NOFP16-NEXT:    mov h25, v5.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], v26.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h23, s23
-; CHECK-SD-NOFP16-NEXT:    fadd s24, s24, s27
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    mov h26, v4.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v17.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h17, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fadd s16, s16, s22
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h25
-; CHECK-SD-NOFP16-NEXT:    mov h22, v5.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[3], v23.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h23, s24
-; CHECK-SD-NOFP16-NEXT:    fadd s19, s20, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h26
-; CHECK-SD-NOFP16-NEXT:    mov h4, v4.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v17.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    fadd s7, s7, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h22
-; CHECK-SD-NOFP16-NEXT:    mov h5, v5.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[4], v23.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h18, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fadd s19, s20, s21
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[5], v18.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s19
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s6, s5
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[6], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[7], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    ldp d11, d10, [sp], #32 // 16-byte Folded Reload
+; CHECK-SD-NOFP16-NEXT:    fcvtl v6.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v7.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v16.4s, v3.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v17.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fmul v6.4s, v7.4s, v6.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v7.4s, v17.4s, v16.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v1.4s, v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v2.4h, v6.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v6.4s, v5.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtn v3.4h, v7.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v2.8h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v4.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v4.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v3.8h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v7.4s, v3.4h
+; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v1.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v3.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v5.8h
+; CHECK-SD-NOFP16-NEXT:    fadd v5.4s, v7.4s, v6.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v2.4s, v2.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v3.4s, v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fmul_v16f16:
diff --git a/llvm/test/CodeGen/AArch64/fmul.ll b/llvm/test/CodeGen/AArch64/fmul.ll
index fe84fe1f317a..1f49601a1827 100644
--- a/llvm/test/CodeGen/AArch64/fmul.ll
+++ b/llvm/test/CodeGen/AArch64/fmul.ll
@@ -169,60 +169,14 @@ entry:
 define <7 x half> @fmul_v7f16(<7 x half> %a, <7 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fmul_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s3, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    fmul s4, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fmul s5, s5, s16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h7
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s6, s3
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fmul s6, s16, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fmul s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fmul v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v1.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fmul_v7f16:
@@ -309,60 +263,14 @@ entry:
 define <8 x half> @fmul_v8f16(<8 x half> %a, <8 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fmul_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s3, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    fmul s4, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fmul s5, s5, s16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h7
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s6, s3
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fmul s6, s16, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fmul s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fmul v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v1.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fmul_v8f16:
@@ -394,114 +302,22 @@ entry:
 define <16 x half> @fmul_v16f16(<16 x half> %a, <16 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fmul_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h4, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v3.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h0
-; CHECK-SD-NOFP16-NEXT:    mov h18, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h1
-; CHECK-SD-NOFP16-NEXT:    mov h22, v3.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fmul s16, s17, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h24, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h19
-; CHECK-SD-NOFP16-NEXT:    mov h19, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    fmul s20, s21, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h23
-; CHECK-SD-NOFP16-NEXT:    fmul s5, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h25, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fmul s6, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h7, v3.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h24
-; CHECK-SD-NOFP16-NEXT:    fmul s17, s18, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s16
-; CHECK-SD-NOFP16-NEXT:    fmul s18, s22, s21
-; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h23
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h20, v2.h[4]
-; CHECK-SD-NOFP16-NEXT:    fmul s19, s24, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h17, s17
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h24, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[1], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v3.h[4]
-; CHECK-SD-NOFP16-NEXT:    fmul s7, s21, s7
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[1], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt h19, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h22
-; CHECK-SD-NOFP16-NEXT:    mov h21, v2.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[2], v17.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h23
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[2], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s7
-; CHECK-SD-NOFP16-NEXT:    mov h7, v3.h[5]
-; CHECK-SD-NOFP16-NEXT:    fmul s18, s20, s18
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v2.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fmul s16, s17, s16
-; CHECK-SD-NOFP16-NEXT:    mov h17, v3.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[3], v19.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h22
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[3], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h23
-; CHECK-SD-NOFP16-NEXT:    fcvt h18, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h24
-; CHECK-SD-NOFP16-NEXT:    fcvt s23, h25
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    mov h3, v3.h[7]
-; CHECK-SD-NOFP16-NEXT:    fmul s19, s21, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fmul s6, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[4], v18.h[0]
-; CHECK-SD-NOFP16-NEXT:    fmul s7, s22, s20
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[4], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    fmul s16, s23, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h17, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s7
-; CHECK-SD-NOFP16-NEXT:    fmul s1, s1, s3
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[5], v17.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[5], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[6], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v4.16b
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v5.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v4.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v5.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v6.4s, v3.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v7.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fmul v4.4s, v5.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v5.4s, v7.4s, v6.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v2.4s, v0.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v3.4s, v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fmul_v16f16:
diff --git a/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll b/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
index ded343b990ac..d4130e7a848b 100644
--- a/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
@@ -5,60 +5,14 @@
 define <8 x half> @add_h(<8 x half> %a, <8 x half> %b) {
 ; CHECK-CVT-LABEL: add_h:
 ; CHECK-CVT:       // %bb.0: // %entry
-; CHECK-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-CVT-NEXT:    mov h3, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s4, h1
-; CHECK-CVT-NEXT:    fcvt s5, h0
-; CHECK-CVT-NEXT:    mov h6, v1.h[2]
-; CHECK-CVT-NEXT:    mov h7, v0.h[2]
-; CHECK-CVT-NEXT:    mov h16, v1.h[3]
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fadd s4, s5, s4
-; CHECK-CVT-NEXT:    mov h5, v0.h[3]
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    fcvt s7, h7
-; CHECK-CVT-NEXT:    fcvt s16, h16
-; CHECK-CVT-NEXT:    fadd s3, s3, s2
-; CHECK-CVT-NEXT:    fcvt s5, h5
-; CHECK-CVT-NEXT:    fcvt h2, s4
-; CHECK-CVT-NEXT:    fadd s4, s7, s6
-; CHECK-CVT-NEXT:    mov h6, v1.h[4]
-; CHECK-CVT-NEXT:    mov h7, v0.h[4]
-; CHECK-CVT-NEXT:    fcvt h3, s3
-; CHECK-CVT-NEXT:    fadd s5, s5, s16
-; CHECK-CVT-NEXT:    mov h16, v0.h[5]
-; CHECK-CVT-NEXT:    fcvt h4, s4
-; CHECK-CVT-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-CVT-NEXT:    fcvt s3, h6
-; CHECK-CVT-NEXT:    fcvt s6, h7
-; CHECK-CVT-NEXT:    mov h7, v1.h[5]
-; CHECK-CVT-NEXT:    fcvt h5, s5
-; CHECK-CVT-NEXT:    fcvt s16, h16
-; CHECK-CVT-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-CVT-NEXT:    mov h4, v1.h[6]
-; CHECK-CVT-NEXT:    fadd s3, s6, s3
-; CHECK-CVT-NEXT:    mov h6, v0.h[6]
-; CHECK-CVT-NEXT:    fcvt s7, h7
-; CHECK-CVT-NEXT:    mov h1, v1.h[7]
-; CHECK-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-CVT-NEXT:    mov v2.h[3], v5.h[0]
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    fcvt h3, s3
-; CHECK-CVT-NEXT:    fcvt s5, h6
-; CHECK-CVT-NEXT:    fadd s6, s16, s7
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    mov v2.h[4], v3.h[0]
-; CHECK-CVT-NEXT:    fadd s4, s5, s4
-; CHECK-CVT-NEXT:    fcvt h3, s6
-; CHECK-CVT-NEXT:    fadd s0, s0, s1
-; CHECK-CVT-NEXT:    mov v2.h[5], v3.h[0]
-; CHECK-CVT-NEXT:    fcvt h3, s4
-; CHECK-CVT-NEXT:    fcvt h0, s0
-; CHECK-CVT-NEXT:    mov v2.h[6], v3.h[0]
-; CHECK-CVT-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-CVT-NEXT:    mov v0.16b, v2.16b
+; CHECK-CVT-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-CVT-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-CVT-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-CVT-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-CVT-NEXT:    fadd v2.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    fadd v1.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-CVT-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: add_h:
@@ -74,60 +28,14 @@ entry:
 define <8 x half> @sub_h(<8 x half> %a, <8 x half> %b) {
 ; CHECK-CVT-LABEL: sub_h:
 ; CHECK-CVT:       // %bb.0: // %entry
-; CHECK-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-CVT-NEXT:    mov h3, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s4, h1
-; CHECK-CVT-NEXT:    fcvt s5, h0
-; CHECK-CVT-NEXT:    mov h6, v1.h[2]
-; CHECK-CVT-NEXT:    mov h7, v0.h[2]
-; CHECK-CVT-NEXT:    mov h16, v1.h[3]
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fsub s4, s5, s4
-; CHECK-CVT-NEXT:    mov h5, v0.h[3]
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    fcvt s7, h7
-; CHECK-CVT-NEXT:    fcvt s16, h16
-; CHECK-CVT-NEXT:    fsub s3, s3, s2
-; CHECK-CVT-NEXT:    fcvt s5, h5
-; CHECK-CVT-NEXT:    fcvt h2, s4
-; CHECK-CVT-NEXT:    fsub s4, s7, s6
-; CHECK-CVT-NEXT:    mov h6, v1.h[4]
-; CHECK-CVT-NEXT:    mov h7, v0.h[4]
-; CHECK-CVT-NEXT:    fcvt h3, s3
-; CHECK-CVT-NEXT:    fsub s5, s5, s16
-; CHECK-CVT-NEXT:    mov h16, v0.h[5]
-; CHECK-CVT-NEXT:    fcvt h4, s4
-; CHECK-CVT-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-CVT-NEXT:    fcvt s3, h6
-; CHECK-CVT-NEXT:    fcvt s6, h7
-; CHECK-CVT-NEXT:    mov h7, v1.h[5]
-; CHECK-CVT-NEXT:    fcvt h5, s5
-; CHECK-CVT-NEXT:    fcvt s16, h16
-; CHECK-CVT-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-CVT-NEXT:    mov h4, v1.h[6]
-; CHECK-CVT-NEXT:    fsub s3, s6, s3
-; CHECK-CVT-NEXT:    mov h6, v0.h[6]
-; CHECK-CVT-NEXT:    fcvt s7, h7
-; CHECK-CVT-NEXT:    mov h1, v1.h[7]
-; CHECK-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-CVT-NEXT:    mov v2.h[3], v5.h[0]
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    fcvt h3, s3
-; CHECK-CVT-NEXT:    fcvt s5, h6
-; CHECK-CVT-NEXT:    fsub s6, s16, s7
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    mov v2.h[4], v3.h[0]
-; CHECK-CVT-NEXT:    fsub s4, s5, s4
-; CHECK-CVT-NEXT:    fcvt h3, s6
-; CHECK-CVT-NEXT:    fsub s0, s0, s1
-; CHECK-CVT-NEXT:    mov v2.h[5], v3.h[0]
-; CHECK-CVT-NEXT:    fcvt h3, s4
-; CHECK-CVT-NEXT:    fcvt h0, s0
-; CHECK-CVT-NEXT:    mov v2.h[6], v3.h[0]
-; CHECK-CVT-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-CVT-NEXT:    mov v0.16b, v2.16b
+; CHECK-CVT-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-CVT-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-CVT-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-CVT-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-CVT-NEXT:    fsub v2.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    fsub v1.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-CVT-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: sub_h:
@@ -143,60 +51,14 @@ entry:
 define <8 x half> @mul_h(<8 x half> %a, <8 x half> %b) {
 ; CHECK-CVT-LABEL: mul_h:
 ; CHECK-CVT:       // %bb.0: // %entry
-; CHECK-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-CVT-NEXT:    mov h3, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s4, h1
-; CHECK-CVT-NEXT:    fcvt s5, h0
-; CHECK-CVT-NEXT:    mov h6, v1.h[2]
-; CHECK-CVT-NEXT:    mov h7, v0.h[2]
-; CHECK-CVT-NEXT:    mov h16, v1.h[3]
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fmul s4, s5, s4
-; CHECK-CVT-NEXT:    mov h5, v0.h[3]
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    fcvt s7, h7
-; CHECK-CVT-NEXT:    fcvt s16, h16
-; CHECK-CVT-NEXT:    fmul s3, s3, s2
-; CHECK-CVT-NEXT:    fcvt s5, h5
-; CHECK-CVT-NEXT:    fcvt h2, s4
-; CHECK-CVT-NEXT:    fmul s4, s7, s6
-; CHECK-CVT-NEXT:    mov h6, v1.h[4]
-; CHECK-CVT-NEXT:    mov h7, v0.h[4]
-; CHECK-CVT-NEXT:    fcvt h3, s3
-; CHECK-CVT-NEXT:    fmul s5, s5, s16
-; CHECK-CVT-NEXT:    mov h16, v0.h[5]
-; CHECK-CVT-NEXT:    fcvt h4, s4
-; CHECK-CVT-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-CVT-NEXT:    fcvt s3, h6
-; CHECK-CVT-NEXT:    fcvt s6, h7
-; CHECK-CVT-NEXT:    mov h7, v1.h[5]
-; CHECK-CVT-NEXT:    fcvt h5, s5
-; CHECK-CVT-NEXT:    fcvt s16, h16
-; CHECK-CVT-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-CVT-NEXT:    mov h4, v1.h[6]
-; CHECK-CVT-NEXT:    fmul s3, s6, s3
-; CHECK-CVT-NEXT:    mov h6, v0.h[6]
-; CHECK-CVT-NEXT:    fcvt s7, h7
-; CHECK-CVT-NEXT:    mov h1, v1.h[7]
-; CHECK-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-CVT-NEXT:    mov v2.h[3], v5.h[0]
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    fcvt h3, s3
-; CHECK-CVT-NEXT:    fcvt s5, h6
-; CHECK-CVT-NEXT:    fmul s6, s16, s7
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    mov v2.h[4], v3.h[0]
-; CHECK-CVT-NEXT:    fmul s4, s5, s4
-; CHECK-CVT-NEXT:    fcvt h3, s6
-; CHECK-CVT-NEXT:    fmul s0, s0, s1
-; CHECK-CVT-NEXT:    mov v2.h[5], v3.h[0]
-; CHECK-CVT-NEXT:    fcvt h3, s4
-; CHECK-CVT-NEXT:    fcvt h0, s0
-; CHECK-CVT-NEXT:    mov v2.h[6], v3.h[0]
-; CHECK-CVT-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-CVT-NEXT:    mov v0.16b, v2.16b
+; CHECK-CVT-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-CVT-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-CVT-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-CVT-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-CVT-NEXT:    fmul v2.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    fmul v1.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-CVT-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: mul_h:
@@ -212,60 +74,14 @@ entry:
 define <8 x half> @div_h(<8 x half> %a, <8 x half> %b) {
 ; CHECK-CVT-LABEL: div_h:
 ; CHECK-CVT:       // %bb.0: // %entry
-; CHECK-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-CVT-NEXT:    mov h3, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s4, h0
-; CHECK-CVT-NEXT:    mov h5, v0.h[2]
-; CHECK-CVT-NEXT:    mov h6, v0.h[3]
-; CHECK-CVT-NEXT:    mov h7, v0.h[4]
-; CHECK-CVT-NEXT:    mov h16, v0.h[5]
-; CHECK-CVT-NEXT:    mov h17, v0.h[6]
-; CHECK-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcvt s5, h5
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    fcvt s7, h7
-; CHECK-CVT-NEXT:    fcvt s16, h16
-; CHECK-CVT-NEXT:    fcvt s17, h17
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    fdiv s2, s3, s2
-; CHECK-CVT-NEXT:    fcvt s3, h1
-; CHECK-CVT-NEXT:    fdiv s3, s4, s3
-; CHECK-CVT-NEXT:    mov h4, v1.h[2]
-; CHECK-CVT-NEXT:    fcvt h18, s2
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    fdiv s4, s5, s4
-; CHECK-CVT-NEXT:    mov h5, v1.h[3]
-; CHECK-CVT-NEXT:    fcvt h2, s3
-; CHECK-CVT-NEXT:    fcvt s5, h5
-; CHECK-CVT-NEXT:    mov v2.h[1], v18.h[0]
-; CHECK-CVT-NEXT:    fdiv s5, s6, s5
-; CHECK-CVT-NEXT:    mov h6, v1.h[4]
-; CHECK-CVT-NEXT:    fcvt h4, s4
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-CVT-NEXT:    fdiv s6, s7, s6
-; CHECK-CVT-NEXT:    mov h7, v1.h[5]
-; CHECK-CVT-NEXT:    fcvt h4, s5
-; CHECK-CVT-NEXT:    fcvt s7, h7
-; CHECK-CVT-NEXT:    mov v2.h[3], v4.h[0]
-; CHECK-CVT-NEXT:    fdiv s7, s16, s7
-; CHECK-CVT-NEXT:    mov h16, v1.h[6]
-; CHECK-CVT-NEXT:    mov h1, v1.h[7]
-; CHECK-CVT-NEXT:    fcvt s16, h16
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fdiv s3, s17, s16
-; CHECK-CVT-NEXT:    fdiv s0, s0, s1
-; CHECK-CVT-NEXT:    fcvt h1, s6
-; CHECK-CVT-NEXT:    mov v2.h[4], v1.h[0]
-; CHECK-CVT-NEXT:    fcvt h1, s7
-; CHECK-CVT-NEXT:    mov v2.h[5], v1.h[0]
-; CHECK-CVT-NEXT:    fcvt h1, s3
-; CHECK-CVT-NEXT:    mov v2.h[6], v1.h[0]
-; CHECK-CVT-NEXT:    fcvt h0, s0
-; CHECK-CVT-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-CVT-NEXT:    mov v0.16b, v2.16b
+; CHECK-CVT-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-CVT-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-CVT-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-CVT-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-CVT-NEXT:    fdiv v2.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    fdiv v1.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-CVT-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: div_h:
diff --git a/llvm/test/CodeGen/AArch64/sve-fp-reciprocal.ll b/llvm/test/CodeGen/AArch64/sve-fp-reciprocal.ll
index 1b2e0be6111c..aefc8de43143 100644
--- a/llvm/test/CodeGen/AArch64/sve-fp-reciprocal.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fp-reciprocal.ll
@@ -17,8 +17,6 @@ define  @fdiv_recip_8f16( %a,  @fsqrt_recip_8f16( %a) #0 {
 ; CHECK-NEXT:    fcmne p0.h, p0/z, z0.h, #0.0
 ; CHECK-NEXT:    frsqrts z2.h, z0.h, z2.h
 ; CHECK-NEXT:    fmul z1.h, z1.h, z2.h
-; CHECK-NEXT:    fmul z2.h, z1.h, z1.h
-; CHECK-NEXT:    frsqrts z2.h, z0.h, z2.h
-; CHECK-NEXT:    fmul z1.h, z1.h, z2.h
 ; CHECK-NEXT:    fmul z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    ret
   %fsqrt = call fast  @llvm.sqrt.nxv8f16( %a)
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll b/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll
index 99c6808724b5..03db1d0d433d 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll
@@ -147,79 +147,48 @@ define double @add_D(<2 x double> %bin.rdx)  {
 define half @add_2H(<16 x half> %bin.rdx)  {
 ; CHECK-SD-NOFP16-LABEL: add_2H:
 ; CHECK-SD-NOFP16:       // %bb.0:
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s3, s2
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s5, s3
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s4, s2
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s3
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fadd v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    mov h0, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s2, s0
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s5, s3
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
 ; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s3
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s5, s4
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[5]
 ; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
 ; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
 ; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fadd s0, s1, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s1
 ; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
@@ -574,78 +543,47 @@ exit:
 define half @fadd_reduct_reassoc_v8f16(<8 x half> %a, <8 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fadd_reduct_reassoc_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0:
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s4, s2
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s5, s3
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s5
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s5
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fadd v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    mov h0, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s2, s0
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s5
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s5
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
 ; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s0, s2, s0
-; CHECK-SD-NOFP16-NEXT:    fadd s1, s3, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
 ; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
 ; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
 ; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
 ; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s1
 ; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fmul.ll b/llvm/test/CodeGen/AArch64/vecreduce-fmul.ll
index e85384e46222..da75a8059521 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fmul.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fmul.ll
@@ -182,79 +182,48 @@ define double @mul_D(<2 x double> %bin.rdx)  {
 define half @mul_2H(<16 x half> %bin.rdx)  {
 ; CHECK-SD-NOFP16-LABEL: mul_2H:
 ; CHECK-SD-NOFP16:       // %bb.0:
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s3, s2
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s5, s3
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s4, s2
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fmul s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s2, s3
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fmul v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    mov h0, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s2, s0
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s5, s3
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fmul s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s2, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
 ; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s2, s3
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s5, s4
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[5]
 ; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
 ; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s2, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
 ; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fmul s0, s1, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s1
 ; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
@@ -361,78 +330,47 @@ define float @mul_S_init_42(<4 x float> %bin.rdx)  {
 define half @fmul_reduct_reassoc_v8f16(<8 x half> %a, <8 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fmul_reduct_reassoc_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0:
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s4, s2
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s5, s3
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s3, s5
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s3, s5
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fmul v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    mov h0, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s2, s0
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s3, s5
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s3, s5
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
 ; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s0, s2, s0
-; CHECK-SD-NOFP16-NEXT:    fmul s1, s3, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
 ; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
 ; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
 ; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
 ; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s1
 ; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    ret
-- 
GitLab


From 0d7dba6300b452c751cde4eeaedcbfeb86f6185c Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot 
Date: Fri, 8 Mar 2024 20:02:42 +0000
Subject: [PATCH 656/929] [gn build] Port 087666811452

---
 llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 72aa8b4e8e38..cb52d99e5b2a 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -450,7 +450,6 @@ if (current_toolchain == default_toolchain) {
       "__format/format_context.h",
       "__format/format_error.h",
       "__format/format_functions.h",
-      "__format/format_fwd.h",
       "__format/format_parse_context.h",
       "__format/format_string.h",
       "__format/format_to_n_result.h",
@@ -500,6 +499,7 @@ if (current_toolchain == default_toolchain) {
       "__fwd/array.h",
       "__fwd/bit_reference.h",
       "__fwd/complex.h",
+      "__fwd/format.h",
       "__fwd/fstream.h",
       "__fwd/functional.h",
       "__fwd/ios.h",
-- 
GitLab


From 0861755e597e671424ee369f36dff5591b7178ee Mon Sep 17 00:00:00 2001
From: Florian Mayer 
Date: Fri, 8 Mar 2024 12:12:35 -0800
Subject: [PATCH 657/929] Revert "[IR] Add new Range attribute using new
 ConstantRange Attribute type" (#84549)

Reverts llvm/llvm-project#83171

broke sanitizer buildbot
https://lab.llvm.org/buildbot/#/builders/168/builds/19110/steps/10/logs/stdio
---
 llvm/docs/LangRef.rst                         | 16 ----
 llvm/include/llvm/ADT/FoldingSet.h            |  7 --
 llvm/include/llvm/AsmParser/LLParser.h        |  1 -
 llvm/include/llvm/Bitcode/LLVMBitCodes.h      |  1 -
 llvm/include/llvm/IR/Attributes.h             | 23 ------
 llvm/include/llvm/IR/Attributes.td            |  6 --
 llvm/lib/AsmParser/LLParser.cpp               | 43 ----------
 llvm/lib/Bitcode/Reader/BitcodeReader.cpp     | 41 ----------
 llvm/lib/Bitcode/Writer/BitcodeWriter.cpp     | 61 +++++---------
 llvm/lib/IR/AttributeImpl.h                   | 28 +------
 llvm/lib/IR/Attributes.cpp                    | 80 +------------------
 llvm/lib/IR/Verifier.cpp                      |  5 --
 llvm/lib/Transforms/Utils/CodeExtractor.cpp   |  1 -
 .../range-attribute-invalid-range.ll          |  6 --
 .../Assembler/range-attribute-invalid-type.ll |  6 --
 llvm/test/Bitcode/attributes.ll               | 10 ---
 llvm/test/Verifier/range-attr.ll              | 19 -----
 llvm/utils/TableGen/Attributes.cpp            |  9 +--
 18 files changed, 26 insertions(+), 337 deletions(-)
 delete mode 100644 llvm/test/Assembler/range-attribute-invalid-range.ll
 delete mode 100644 llvm/test/Assembler/range-attribute-invalid-type.ll
 delete mode 100644 llvm/test/Verifier/range-attr.ll

diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index b70220dec926..a7b77d6f776a 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -1635,22 +1635,6 @@ Currently, only the following parameter attributes are defined:
 
     This attribute cannot be applied to return values.
 
-``range( , )``
-    This attribute expresses the possible range of the parameter or return value.
-    If the value is not in the specified range, it is converted to poison.
-    The arguments passed to ``range`` have the following properties:
-
-    -  The type must match the scalar type of the parameter or return value.
-    -  The pair ``a,b`` represents the range ``[a,b)``.
-    -  Both ``a`` and ``b`` are constants.
-    -  The range is allowed to wrap.
-    -  The range should not represent the full or empty set. That is, ``a!=b``.
-    
-    This attribute may only be applied to parameters or return values with integer 
-    or vector of integer types.
-    
-    For vector-typed parameters, the range is applied element-wise.
-
 .. _gc:
 
 Garbage Collector Strategy Names
diff --git a/llvm/include/llvm/ADT/FoldingSet.h b/llvm/include/llvm/ADT/FoldingSet.h
index ddc3e52255d6..f82eabd5044b 100644
--- a/llvm/include/llvm/ADT/FoldingSet.h
+++ b/llvm/include/llvm/ADT/FoldingSet.h
@@ -16,7 +16,6 @@
 #ifndef LLVM_ADT_FOLDINGSET_H
 #define LLVM_ADT_FOLDINGSET_H
 
-#include "llvm/ADT/APInt.h"
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/SmallVector.h"
@@ -355,12 +354,6 @@ public:
     AddInteger(unsigned(I));
     AddInteger(unsigned(I >> 32));
   }
-  void AddInteger(const APInt &Int) {
-    const auto *Parts = Int.getRawData();
-    for (int i = 0, N = Int.getNumWords(); i < N; ++i) {
-      AddInteger(Parts[i]);
-    }
-  }
 
   void AddBoolean(bool B) { AddInteger(B ? 1U : 0U); }
   void AddString(StringRef String);
diff --git a/llvm/include/llvm/AsmParser/LLParser.h b/llvm/include/llvm/AsmParser/LLParser.h
index e85728aa3c0d..e5e1ade8b38b 100644
--- a/llvm/include/llvm/AsmParser/LLParser.h
+++ b/llvm/include/llvm/AsmParser/LLParser.h
@@ -369,7 +369,6 @@ namespace llvm {
     bool parseFnAttributeValuePairs(AttrBuilder &B,
                                     std::vector &FwdRefAttrGrps,
                                     bool inAttrGrp, LocTy &BuiltinLoc);
-    bool parseRangeAttr(AttrBuilder &B);
     bool parseRequiredTypeAttr(AttrBuilder &B, lltok::Kind AttrToken,
                                Attribute::AttrKind AttrKind);
 
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index c0a52d64a101..c6f0ddf29a6d 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -724,7 +724,6 @@ enum AttributeKindCodes {
   ATTR_KIND_WRITABLE = 89,
   ATTR_KIND_CORO_ONLY_DESTROY_WHEN_COMPLETE = 90,
   ATTR_KIND_DEAD_ON_UNWIND = 91,
-  ATTR_KIND_RANGE = 92,
 };
 
 enum ComdatSelectionKindCodes {
diff --git a/llvm/include/llvm/IR/Attributes.h b/llvm/include/llvm/IR/Attributes.h
index 0c2a02514ba0..a4ebe5d732f5 100644
--- a/llvm/include/llvm/IR/Attributes.h
+++ b/llvm/include/llvm/IR/Attributes.h
@@ -37,7 +37,6 @@ class AttributeMask;
 class AttributeImpl;
 class AttributeListImpl;
 class AttributeSetNode;
-class ConstantRange;
 class FoldingSetNodeID;
 class Function;
 class LLVMContext;
@@ -104,9 +103,6 @@ public:
   static bool isTypeAttrKind(AttrKind Kind) {
     return Kind >= FirstTypeAttr && Kind <= LastTypeAttr;
   }
-  static bool isConstantRangeAttrKind(AttrKind Kind) {
-    return Kind >= FirstConstantRangeAttr && Kind <= LastConstantRangeAttr;
-  }
 
   static bool canUseAsFnAttr(AttrKind Kind);
   static bool canUseAsParamAttr(AttrKind Kind);
@@ -129,8 +125,6 @@ public:
   static Attribute get(LLVMContext &Context, StringRef Kind,
                        StringRef Val = StringRef());
   static Attribute get(LLVMContext &Context, AttrKind Kind, Type *Ty);
-  static Attribute get(LLVMContext &Context, AttrKind Kind,
-                       const ConstantRange &CR);
 
   /// Return a uniquified Attribute object that has the specific
   /// alignment set.
@@ -186,9 +180,6 @@ public:
   /// Return true if the attribute is a type attribute.
   bool isTypeAttribute() const;
 
-  /// Return true if the attribute is a ConstantRange attribute.
-  bool isConstantRangeAttribute() const;
-
   /// Return true if the attribute is any kind of attribute.
   bool isValid() const { return pImpl; }
 
@@ -222,10 +213,6 @@ public:
   /// a type attribute.
   Type *getValueAsType() const;
 
-  /// Return the attribute's value as a ConstantRange. This requires the
-  /// attribute to be a ConstantRange attribute.
-  ConstantRange getValueAsConstantRange() const;
-
   /// Returns the alignment field of an attribute as a byte alignment
   /// value.
   MaybeAlign getAlignment() const;
@@ -264,9 +251,6 @@ public:
   /// Return the FPClassTest for nofpclass
   FPClassTest getNoFPClass() const;
 
-  /// Returns the value of the range attribute.
-  ConstantRange getRange() const;
-
   /// The Attribute is converted to a string of equivalent mnemonic. This
   /// is, presumably, for writing out the mnemonics for the assembly writer.
   std::string getAsString(bool InAttrGrp = false) const;
@@ -1205,13 +1189,6 @@ public:
   // Add nofpclass attribute
   AttrBuilder &addNoFPClassAttr(FPClassTest NoFPClassMask);
 
-  /// Add a ConstantRange attribute with the given range.
-  AttrBuilder &addConstantRangeAttr(Attribute::AttrKind Kind,
-                                    const ConstantRange &CR);
-
-  /// Add range attribute.
-  AttrBuilder &addRangeAttr(const ConstantRange &CR);
-
   ArrayRef attrs() const { return Attrs; }
 
   bool operator==(const AttrBuilder &B) const;
diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td
index cef8b17769f0..08afecf32015 100644
--- a/llvm/include/llvm/IR/Attributes.td
+++ b/llvm/include/llvm/IR/Attributes.td
@@ -44,9 +44,6 @@ class StrBoolAttr : Attr;
 /// Arbitrary string attribute.
 class ComplexStrAttr P> : Attr;
 
-/// ConstantRange attribute.
-class ConstantRangeAttr P> : Attr;
-
 /// Target-independent enum attributes.
 
 /// Alignment of parameter (5 bits) stored as log2 of alignment with +1 bias.
@@ -221,9 +218,6 @@ def OptimizeNone : EnumAttr<"optnone", [FnAttr]>;
 /// Similar to byval but without a copy.
 def Preallocated : TypeAttr<"preallocated", [FnAttr, ParamAttr]>;
 
-/// Parameter or return value is within the specified range.
-def Range : ConstantRangeAttr<"range", [ParamAttr, RetAttr]>;
-
 /// Function does not access memory.
 def ReadNone : EnumAttr<"readnone", [ParamAttr]>;
 
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 78bcd94e23fa..e140c9419520 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -1596,8 +1596,6 @@ bool LLParser::parseEnumAttribute(Attribute::AttrKind Attr, AttrBuilder &B,
 
     return true;
   }
-  case Attribute::Range:
-    return parseRangeAttr(B);
   default:
     B.addAttribute(Attr);
     Lex.Lex();
@@ -3010,47 +3008,6 @@ bool LLParser::parseRequiredTypeAttr(AttrBuilder &B, lltok::Kind AttrToken,
   return false;
 }
 
-/// parseRangeAttr
-///   ::= range( ,)
-bool LLParser::parseRangeAttr(AttrBuilder &B) {
-  Lex.Lex();
-
-  APInt Lower;
-  APInt Upper;
-  Type *Ty = nullptr;
-  LocTy TyLoc;
-
-  auto ParseAPSInt = [&](unsigned BitWidth, APInt &Val) {
-    if (Lex.getKind() != lltok::APSInt)
-      return tokError("expected integer");
-    if (Lex.getAPSIntVal().getBitWidth() > BitWidth)
-      return tokError(
-          "integer is too large for the bit width of specified type");
-    Val = Lex.getAPSIntVal().extend(BitWidth);
-    Lex.Lex();
-    return false;
-  };
-
-  if (parseToken(lltok::lparen, "expected '('") || parseType(Ty, TyLoc))
-    return true;
-  if (!Ty->isIntegerTy())
-    return error(TyLoc, "the range must have integer type!");
-
-  unsigned BitWidth = Ty->getPrimitiveSizeInBits();
-
-  if (ParseAPSInt(BitWidth, Lower) ||
-      parseToken(lltok::comma, "expected ','") || ParseAPSInt(BitWidth, Upper))
-    return true;
-  if (Lower == Upper)
-    return tokError("the range should not represent the full or empty set!");
-
-  if (parseToken(lltok::rparen, "expected ')'"))
-    return true;
-
-  B.addRangeAttr(ConstantRange(Lower, Upper));
-  return false;
-}
-
 /// parseOptionalOperandBundles
 ///    ::= /*empty*/
 ///    ::= '[' OperandBundle [, OperandBundle ]* ']'
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 9c63116114f3..832907a3f53f 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -815,30 +815,6 @@ private:
     return getFnValueByID(ValNo, Ty, TyID, ConstExprInsertBB);
   }
 
-  Expected readConstantRange(ArrayRef Record,
-                                            unsigned &OpNum) {
-    if (Record.size() - OpNum < 3)
-      return error("Too few records for range");
-    unsigned BitWidth = Record[OpNum++];
-    if (BitWidth > 64) {
-      unsigned LowerActiveWords = Record[OpNum];
-      unsigned UpperActiveWords = Record[OpNum++] >> 32;
-      if (Record.size() - OpNum < LowerActiveWords + UpperActiveWords)
-        return error("Too few records for range");
-      APInt Lower =
-          readWideAPInt(ArrayRef(&Record[OpNum], LowerActiveWords), BitWidth);
-      OpNum += LowerActiveWords;
-      APInt Upper =
-          readWideAPInt(ArrayRef(&Record[OpNum], UpperActiveWords), BitWidth);
-      OpNum += UpperActiveWords;
-      return ConstantRange(Lower, Upper);
-    } else {
-      int64_t Start = BitcodeReader::decodeSignRotatedValue(Record[OpNum++]);
-      int64_t End = BitcodeReader::decodeSignRotatedValue(Record[OpNum++]);
-      return ConstantRange(APInt(BitWidth, Start), APInt(BitWidth, End));
-    }
-  }
-
   /// Upgrades old-style typeless byval/sret/inalloca attributes by adding the
   /// corresponding argument's pointee type. Also upgrades intrinsics that now
   /// require an elementtype attribute.
@@ -2127,8 +2103,6 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
     return Attribute::CoroDestroyOnlyWhenComplete;
   case bitc::ATTR_KIND_DEAD_ON_UNWIND:
     return Attribute::DeadOnUnwind;
-  case bitc::ATTR_KIND_RANGE:
-    return Attribute::Range;
   }
 }
 
@@ -2298,21 +2272,6 @@ Error BitcodeReader::parseAttributeGroupBlock() {
             return error("Not a type attribute");
 
           B.addTypeAttr(Kind, HasType ? getTypeByID(Record[++i]) : nullptr);
-        } else if (Record[i] == 7) {
-          Attribute::AttrKind Kind;
-
-          i++;
-          if (Error Err = parseAttrKind(Record[i++], &Kind))
-            return Err;
-          if (!Attribute::isConstantRangeAttrKind(Kind))
-            return error("Not a ConstantRange attribute");
-
-          Expected MaybeCR = readConstantRange(Record, i);
-          if (!MaybeCR)
-            return MaybeCR.takeError();
-          i--;
-
-          B.addConstantRangeAttr(Kind, MaybeCR.get());
         } else {
           return error("Invalid attribute group entry");
         }
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 597f49332fad..656f2a6ce870 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -844,8 +844,6 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
     return bitc::ATTR_KIND_CORO_ONLY_DESTROY_WHEN_COMPLETE;
   case Attribute::DeadOnUnwind:
     return bitc::ATTR_KIND_DEAD_ON_UNWIND;
-  case Attribute::Range:
-    return bitc::ATTR_KIND_RANGE;
   case Attribute::EndAttrKinds:
     llvm_unreachable("Can not encode end-attribute kinds marker.");
   case Attribute::None:
@@ -858,39 +856,6 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
   llvm_unreachable("Trying to encode unknown attribute");
 }
 
-static void emitSignedInt64(SmallVectorImpl &Vals, uint64_t V) {
-  if ((int64_t)V >= 0)
-    Vals.push_back(V << 1);
-  else
-    Vals.push_back((-V << 1) | 1);
-}
-
-static void emitWideAPInt(SmallVectorImpl &Vals, const APInt &A) {
-  // We have an arbitrary precision integer value to write whose
-  // bit width is > 64. However, in canonical unsigned integer
-  // format it is likely that the high bits are going to be zero.
-  // So, we only write the number of active words.
-  unsigned NumWords = A.getActiveWords();
-  const uint64_t *RawData = A.getRawData();
-  for (unsigned i = 0; i < NumWords; i++)
-    emitSignedInt64(Vals, RawData[i]);
-}
-
-static void emitConstantRange(SmallVectorImpl &Record,
-                              const ConstantRange &CR) {
-  unsigned BitWidth = CR.getBitWidth();
-  Record.push_back(BitWidth);
-  if (BitWidth > 64) {
-    Record.push_back(CR.getLower().getActiveWords() |
-                     (uint64_t(CR.getUpper().getActiveWords()) << 32));
-    emitWideAPInt(Record, CR.getLower());
-    emitWideAPInt(Record, CR.getUpper());
-  } else {
-    emitSignedInt64(Record, CR.getLower().getSExtValue());
-    emitSignedInt64(Record, CR.getUpper().getSExtValue());
-  }
-}
-
 void ModuleBitcodeWriter::writeAttributeGroupTable() {
   const std::vector &AttrGrps =
       VE.getAttributeGroups();
@@ -924,17 +889,13 @@ void ModuleBitcodeWriter::writeAttributeGroupTable() {
           Record.append(Val.begin(), Val.end());
           Record.push_back(0);
         }
-      } else if (Attr.isTypeAttribute()) {
+      } else {
+        assert(Attr.isTypeAttribute());
         Type *Ty = Attr.getValueAsType();
         Record.push_back(Ty ? 6 : 5);
         Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum()));
         if (Ty)
           Record.push_back(VE.getTypeID(Attr.getValueAsType()));
-      } else {
-        assert(Attr.isConstantRangeAttribute());
-        Record.push_back(7);
-        Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum()));
-        emitConstantRange(Record, Attr.getValueAsConstantRange());
       }
     }
 
@@ -1755,6 +1716,24 @@ void ModuleBitcodeWriter::writeDIGenericSubrange(
   Record.clear();
 }
 
+static void emitSignedInt64(SmallVectorImpl &Vals, uint64_t V) {
+  if ((int64_t)V >= 0)
+    Vals.push_back(V << 1);
+  else
+    Vals.push_back((-V << 1) | 1);
+}
+
+static void emitWideAPInt(SmallVectorImpl &Vals, const APInt &A) {
+  // We have an arbitrary precision integer value to write whose
+  // bit width is > 64. However, in canonical unsigned integer
+  // format it is likely that the high bits are going to be zero.
+  // So, we only write the number of active words.
+  unsigned NumWords = A.getActiveWords();
+  const uint64_t *RawData = A.getRawData();
+  for (unsigned i = 0; i < NumWords; i++)
+    emitSignedInt64(Vals, RawData[i]);
+}
+
 void ModuleBitcodeWriter::writeDIEnumerator(const DIEnumerator *N,
                                             SmallVectorImpl &Record,
                                             unsigned Abbrev) {
diff --git a/llvm/lib/IR/AttributeImpl.h b/llvm/lib/IR/AttributeImpl.h
index 9a6427bbc3d5..78496786b0ae 100644
--- a/llvm/lib/IR/AttributeImpl.h
+++ b/llvm/lib/IR/AttributeImpl.h
@@ -20,7 +20,6 @@
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/Attributes.h"
-#include "llvm/IR/ConstantRange.h"
 #include "llvm/Support/TrailingObjects.h"
 #include 
 #include 
@@ -47,7 +46,6 @@ protected:
     IntAttrEntry,
     StringAttrEntry,
     TypeAttrEntry,
-    ConstantRangeAttrEntry,
   };
 
   AttributeImpl(AttrEntryKind KindID) : KindID(KindID) {}
@@ -61,9 +59,6 @@ public:
   bool isIntAttribute() const { return KindID == IntAttrEntry; }
   bool isStringAttribute() const { return KindID == StringAttrEntry; }
   bool isTypeAttribute() const { return KindID == TypeAttrEntry; }
-  bool isConstantRangeAttribute() const {
-    return KindID == ConstantRangeAttrEntry;
-  }
 
   bool hasAttribute(Attribute::AttrKind A) const;
   bool hasAttribute(StringRef Kind) const;
@@ -77,8 +72,6 @@ public:
 
   Type *getValueAsType() const;
 
-  ConstantRange getValueAsConstantRange() const;
-
   /// Used when sorting the attributes.
   bool operator<(const AttributeImpl &AI) const;
 
@@ -89,10 +82,8 @@ public:
       Profile(ID, getKindAsEnum(), getValueAsInt());
     else if (isStringAttribute())
       Profile(ID, getKindAsString(), getValueAsString());
-    else if (isTypeAttribute())
-      Profile(ID, getKindAsEnum(), getValueAsType());
     else
-      Profile(ID, getKindAsEnum(), getValueAsConstantRange());
+      Profile(ID, getKindAsEnum(), getValueAsType());
   }
 
   static void Profile(FoldingSetNodeID &ID, Attribute::AttrKind Kind) {
@@ -117,13 +108,6 @@ public:
     ID.AddInteger(Kind);
     ID.AddPointer(Ty);
   }
-
-  static void Profile(FoldingSetNodeID &ID, Attribute::AttrKind Kind,
-                      const ConstantRange &CR) {
-    ID.AddInteger(Kind);
-    ID.AddInteger(CR.getLower());
-    ID.AddInteger(CR.getUpper());
-  }
 };
 
 static_assert(std::is_trivially_destructible::value,
@@ -212,16 +196,6 @@ public:
   Type *getTypeValue() const { return Ty; }
 };
 
-class ConstantRangeAttributeImpl : public EnumAttributeImpl {
-  ConstantRange CR;
-
-public:
-  ConstantRangeAttributeImpl(Attribute::AttrKind Kind, const ConstantRange &CR)
-      : EnumAttributeImpl(ConstantRangeAttrEntry, Kind), CR(CR) {}
-
-  ConstantRange getConstantRangeValue() const { return CR; }
-};
-
 class AttributeBitSet {
   /// Bitset with a bit for each available attribute Attribute::AttrKind.
   uint8_t AvailableAttrs[12] = {};
diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp
index fec34d0a4a81..00acbbe7989d 100644
--- a/llvm/lib/IR/Attributes.cpp
+++ b/llvm/lib/IR/Attributes.cpp
@@ -24,7 +24,6 @@
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/AttributeMask.h"
-#include "llvm/IR/ConstantRange.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Type.h"
@@ -166,30 +165,6 @@ Attribute Attribute::get(LLVMContext &Context, Attribute::AttrKind Kind,
   return Attribute(PA);
 }
 
-Attribute Attribute::get(LLVMContext &Context, Attribute::AttrKind Kind,
-                         const ConstantRange &CR) {
-  assert(Attribute::isConstantRangeAttrKind(Kind) &&
-         "Not a ConstantRange attribute");
-  LLVMContextImpl *pImpl = Context.pImpl;
-  FoldingSetNodeID ID;
-  ID.AddInteger(Kind);
-  ID.AddInteger(CR.getLower());
-  ID.AddInteger(CR.getUpper());
-
-  void *InsertPoint;
-  AttributeImpl *PA = pImpl->AttrsSet.FindNodeOrInsertPos(ID, InsertPoint);
-
-  if (!PA) {
-    // If we didn't find any existing attributes of the same shape then create a
-    // new one and insert it.
-    PA = new (pImpl->Alloc) ConstantRangeAttributeImpl(Kind, CR);
-    pImpl->AttrsSet.InsertNode(PA, InsertPoint);
-  }
-
-  // Return the Attribute that we found or created.
-  return Attribute(PA);
-}
-
 Attribute Attribute::getWithAlignment(LLVMContext &Context, Align A) {
   assert(A <= llvm::Value::MaximumAlignment && "Alignment too large.");
   return get(Context, Alignment, A.value());
@@ -312,14 +287,9 @@ bool Attribute::isTypeAttribute() const {
   return pImpl && pImpl->isTypeAttribute();
 }
 
-bool Attribute::isConstantRangeAttribute() const {
-  return pImpl && pImpl->isConstantRangeAttribute();
-}
-
 Attribute::AttrKind Attribute::getKindAsEnum() const {
   if (!pImpl) return None;
-  assert((isEnumAttribute() || isIntAttribute() || isTypeAttribute() ||
-          isConstantRangeAttribute()) &&
+  assert((isEnumAttribute() || isIntAttribute() || isTypeAttribute()) &&
          "Invalid attribute type to get the kind as an enum!");
   return pImpl->getKindAsEnum();
 }
@@ -359,11 +329,6 @@ Type *Attribute::getValueAsType() const {
   return pImpl->getValueAsType();
 }
 
-ConstantRange Attribute::getValueAsConstantRange() const {
-  assert(isConstantRangeAttribute() &&
-         "Invalid attribute type to get the value as a ConstantRange!");
-  return pImpl->getValueAsConstantRange();
-}
 
 bool Attribute::hasAttribute(AttrKind Kind) const {
   return (pImpl && pImpl->hasAttribute(Kind)) || (!pImpl && Kind == None);
@@ -443,12 +408,6 @@ FPClassTest Attribute::getNoFPClass() const {
   return static_cast(pImpl->getValueAsInt());
 }
 
-ConstantRange Attribute::getRange() const {
-  assert(hasAttribute(Attribute::Range) &&
-         "Trying to get range args from non-range attribute");
-  return pImpl->getValueAsConstantRange();
-}
-
 static const char *getModRefStr(ModRefInfo MR) {
   switch (MR) {
   case ModRefInfo::NoModRef:
@@ -603,18 +562,6 @@ std::string Attribute::getAsString(bool InAttrGrp) const {
     return Result;
   }
 
-  if (hasAttribute(Attribute::Range)) {
-    std::string Result;
-    raw_string_ostream OS(Result);
-    ConstantRange CR = getValueAsConstantRange();
-    OS << "range(";
-    OS << "i" << CR.getBitWidth() << " ";
-    OS << CR.getLower() << ", " << CR.getUpper();
-    OS << ")";
-    OS.flush();
-    return Result;
-  }
-
   // Convert target-dependent attributes to strings of the form:
   //
   //   "kind"
@@ -704,8 +651,7 @@ bool AttributeImpl::hasAttribute(StringRef Kind) const {
 }
 
 Attribute::AttrKind AttributeImpl::getKindAsEnum() const {
-  assert(isEnumAttribute() || isIntAttribute() || isTypeAttribute() ||
-         isConstantRangeAttribute());
+  assert(isEnumAttribute() || isIntAttribute() || isTypeAttribute());
   return static_cast(this)->getEnumKind();
 }
 
@@ -734,12 +680,6 @@ Type *AttributeImpl::getValueAsType() const {
   return static_cast(this)->getTypeValue();
 }
 
-ConstantRange AttributeImpl::getValueAsConstantRange() const {
-  assert(isConstantRangeAttribute());
-  return static_cast(this)
-      ->getConstantRangeValue();
-}
-
 bool AttributeImpl::operator<(const AttributeImpl &AI) const {
   if (this == &AI)
     return false;
@@ -753,7 +693,6 @@ bool AttributeImpl::operator<(const AttributeImpl &AI) const {
       return getKindAsEnum() < AI.getKindAsEnum();
     assert(!AI.isEnumAttribute() && "Non-unique attribute");
     assert(!AI.isTypeAttribute() && "Comparison of types would be unstable");
-    assert(!AI.isConstantRangeAttribute() && "Unclear how to compare ranges");
     // TODO: Is this actually needed?
     assert(AI.isIntAttribute() && "Only possibility left");
     return getValueAsInt() < AI.getValueAsInt();
@@ -1942,15 +1881,6 @@ AttrBuilder &AttrBuilder::addInAllocaAttr(Type *Ty) {
   return addTypeAttr(Attribute::InAlloca, Ty);
 }
 
-AttrBuilder &AttrBuilder::addConstantRangeAttr(Attribute::AttrKind Kind,
-                                               const ConstantRange &CR) {
-  return addAttribute(Attribute::get(Ctx, Kind, CR));
-}
-
-AttrBuilder &AttrBuilder::addRangeAttr(const ConstantRange &CR) {
-  return addConstantRangeAttr(Attribute::Range, CR);
-}
-
 AttrBuilder &AttrBuilder::merge(const AttrBuilder &B) {
   // TODO: Could make this O(n) as we're merging two sorted lists.
   for (const auto &I : B.attrs())
@@ -2022,12 +1952,6 @@ AttributeMask AttributeFuncs::typeIncompatible(Type *Ty,
       Incompatible.addAttribute(Attribute::SExt).addAttribute(Attribute::ZExt);
   }
 
-  if (!Ty->isIntOrIntVectorTy()) {
-    // Attributes that only apply to integers or vector of integers.
-    if (ASK & ASK_SAFE_TO_DROP)
-      Incompatible.addAttribute(Attribute::Range);
-  }
-
   if (!Ty->isPointerTy()) {
     // Attributes that only apply to pointers.
     if (ASK & ASK_SAFE_TO_DROP)
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 3cf5e81efb3b..fd5f7d57c258 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -2039,11 +2039,6 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty,
     Check((Val & ~static_cast(fcAllFlags)) == 0,
           "Invalid value for 'nofpclass' test mask", V);
   }
-  if (Attrs.hasAttribute(Attribute::Range)) {
-    auto CR = Attrs.getAttribute(Attribute::Range).getValueAsConstantRange();
-    Check(Ty->isIntOrIntVectorTy(CR.getBitWidth()),
-          "Range bit width must match type bit width!", V);
-  }
 }
 
 void Verifier::checkUnsignedBaseTenFuncAttr(AttributeList Attrs, StringRef Attr,
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index ab2d25c3f17c..3071ec0c9113 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -999,7 +999,6 @@ Function *CodeExtractor::constructFunction(const ValueSet &inputs,
       case Attribute::WriteOnly:
       case Attribute::Writable:
       case Attribute::DeadOnUnwind:
-      case Attribute::Range:
       //  These are not really attributes.
       case Attribute::None:
       case Attribute::EndAttrKinds:
diff --git a/llvm/test/Assembler/range-attribute-invalid-range.ll b/llvm/test/Assembler/range-attribute-invalid-range.ll
deleted file mode 100644
index cf6d3f080183..000000000000
--- a/llvm/test/Assembler/range-attribute-invalid-range.ll
+++ /dev/null
@@ -1,6 +0,0 @@
-; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s
-
-; CHECK: the range should not represent the full or empty set!
-define void @range_empty(i8 range(i8 0, 0) %a) {
-  ret void
-}
diff --git a/llvm/test/Assembler/range-attribute-invalid-type.ll b/llvm/test/Assembler/range-attribute-invalid-type.ll
deleted file mode 100644
index cc09149a94dc..000000000000
--- a/llvm/test/Assembler/range-attribute-invalid-type.ll
+++ /dev/null
@@ -1,6 +0,0 @@
-; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s
-
-; CHECK: the range must have integer type!
-define void @range_vector_type(i8 range(<4 x i32> 0, 0) %a) {
-  ret void
-}
diff --git a/llvm/test/Bitcode/attributes.ll b/llvm/test/Bitcode/attributes.ll
index 26163b4d38c8..6921f11a352d 100644
--- a/llvm/test/Bitcode/attributes.ll
+++ b/llvm/test/Bitcode/attributes.ll
@@ -526,16 +526,6 @@ define void @f91(ptr dead_on_unwind %p) {
   ret void
 }
 
-; CHECK: define range(i32 -1, 42) i32 @range_attribute(<4 x i32> range(i32 -1, 42) %a)
-define range(i32 -1, 42) i32 @range_attribute(<4 x i32> range(i32 -1, 42) %a) {
-  ret i32 0
-}
-
-; CHECK: define void @wide_range_attribute(i128 range(i128 618970019642690137449562111, 618970019642690137449562114) %a)
-define void @wide_range_attribute(i128 range(i128 618970019642690137449562111, 618970019642690137449562114) %a) {
-  ret void
-}
-
 ; CHECK: attributes #0 = { noreturn }
 ; CHECK: attributes #1 = { nounwind }
 ; CHECK: attributes #2 = { memory(none) }
diff --git a/llvm/test/Verifier/range-attr.ll b/llvm/test/Verifier/range-attr.ll
deleted file mode 100644
index f985ab696eac..000000000000
--- a/llvm/test/Verifier/range-attr.ll
+++ /dev/null
@@ -1,19 +0,0 @@
-; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
-
-; CHECK: Range bit width must match type bit width!
-; CHECK-NEXT: ptr @bit_widths_do_not_match
-define void @bit_widths_do_not_match(i32 range(i8 1, 0) %a) {
-  ret void
-}
-
-; CHECK: Range bit width must match type bit width!
-; CHECK-NEXT: ptr @bit_widths_do_not_match_vector
-define void @bit_widths_do_not_match_vector(<4 x i32> range(i8 1, 0) %a) {
-  ret void
-}
-
-; CHECK: Attribute 'range(i8 1, 0)' applied to incompatible type!
-; CHECK-NEXT: ptr @not-integer-type
-define void @not-integer-type(ptr range(i8 1, 0) %a) {
-  ret void
-}
diff --git a/llvm/utils/TableGen/Attributes.cpp b/llvm/utils/TableGen/Attributes.cpp
index d9fc7834416c..db3c4decccb4 100644
--- a/llvm/utils/TableGen/Attributes.cpp
+++ b/llvm/utils/TableGen/Attributes.cpp
@@ -53,8 +53,7 @@ void Attributes::emitTargetIndependentNames(raw_ostream &OS) {
   };
 
   // Emit attribute enums in the same order llvm::Attribute::operator< expects.
-  Emit({"EnumAttr", "TypeAttr", "IntAttr", "ConstantRangeAttr"},
-       "ATTRIBUTE_ENUM");
+  Emit({"EnumAttr", "TypeAttr", "IntAttr"}, "ATTRIBUTE_ENUM");
   Emit({"StrBoolAttr"}, "ATTRIBUTE_STRBOOL");
   Emit({"ComplexStrAttr"}, "ATTRIBUTE_COMPLEXSTR");
 
@@ -64,8 +63,7 @@ void Attributes::emitTargetIndependentNames(raw_ostream &OS) {
   OS << "#ifdef GET_ATTR_ENUM\n";
   OS << "#undef GET_ATTR_ENUM\n";
   unsigned Value = 1; // Leave zero for AttrKind::None.
-  for (StringRef KindName :
-       {"EnumAttr", "TypeAttr", "IntAttr", "ConstantRangeAttr"}) {
+  for (StringRef KindName : {"EnumAttr", "TypeAttr", "IntAttr"}) {
     OS << "First" << KindName << " = " << Value << ",\n";
     for (auto *A : Records.getAllDerivedDefinitions(KindName)) {
       OS << A->getName() << " = " << Value << ",\n";
@@ -119,8 +117,7 @@ void Attributes::emitAttributeProperties(raw_ostream &OS) {
   OS << "#ifdef GET_ATTR_PROP_TABLE\n";
   OS << "#undef GET_ATTR_PROP_TABLE\n";
   OS << "static const uint8_t AttrPropTable[] = {\n";
-  for (StringRef KindName :
-       {"EnumAttr", "TypeAttr", "IntAttr", "ConstantRangeAttr"}) {
+  for (StringRef KindName : {"EnumAttr", "TypeAttr", "IntAttr"}) {
     for (auto *A : Records.getAllDerivedDefinitions(KindName)) {
       OS << "0";
       for (Init *P : *A->getValueAsListInit("Properties"))
-- 
GitLab


From 300a39bdad4593fdc2618eb28f6e838df735619a Mon Sep 17 00:00:00 2001
From: Florian Mayer 
Date: Fri, 8 Mar 2024 12:14:22 -0800
Subject: [PATCH 658/929] Revert "Change
 GetNumChildren()/CalculateNumChildren() methods return llvm::Expected
 (#84219)"

This reverts commit 99118c809367d518ffe4de60c16da953744b68b9.
---
 lldb/include/lldb/Core/ValueObject.h          | 11 +---
 lldb/include/lldb/Core/ValueObjectCast.h      |  2 +-
 lldb/include/lldb/Core/ValueObjectChild.h     |  2 +-
 .../lldb/Core/ValueObjectConstResult.h        |  2 +-
 .../lldb/Core/ValueObjectDynamicValue.h       |  2 +-
 lldb/include/lldb/Core/ValueObjectMemory.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectRegister.h  |  4 +-
 .../lldb/Core/ValueObjectSyntheticFilter.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectVTable.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectVariable.h  |  2 +-
 .../lldb/DataFormatters/TypeSynthetic.h       | 20 ++----
 .../lldb/DataFormatters/VectorIterator.h      |  2 +-
 lldb/include/lldb/Symbol/CompilerType.h       |  5 +-
 lldb/include/lldb/Symbol/Type.h               |  2 +-
 lldb/include/lldb/Symbol/TypeSystem.h         |  7 +-
 .../lldb/Target/StackFrameRecognizer.h        |  3 +-
 lldb/include/lldb/Utility/Log.h               | 14 ----
 lldb/source/API/SBValue.cpp                   |  2 +-
 lldb/source/Core/FormatEntity.cpp             |  2 +-
 lldb/source/Core/IOHandlerCursesGUI.cpp       |  2 +-
 lldb/source/Core/ValueObject.cpp              | 31 +++------
 lldb/source/Core/ValueObjectCast.cpp          |  6 +-
 lldb/source/Core/ValueObjectChild.cpp         |  6 +-
 lldb/source/Core/ValueObjectConstResult.cpp   |  7 +-
 lldb/source/Core/ValueObjectDynamicValue.cpp  |  7 +-
 lldb/source/Core/ValueObjectMemory.cpp        | 10 +--
 lldb/source/Core/ValueObjectRegister.cpp      | 12 ++--
 .../Core/ValueObjectSyntheticFilter.cpp       | 24 +++----
 lldb/source/Core/ValueObjectVTable.cpp        |  6 +-
 lldb/source/Core/ValueObjectVariable.cpp      |  7 +-
 lldb/source/DataFormatters/FormatManager.cpp  | 11 +---
 lldb/source/DataFormatters/TypeSynthetic.cpp  | 17 +----
 .../DataFormatters/ValueObjectPrinter.cpp     |  2 +-
 lldb/source/DataFormatters/VectorType.cpp     | 16 ++---
 .../Clang/ClangExpressionSourceCode.cpp       |  2 +-
 .../TSan/InstrumentationRuntimeTSan.cpp       |  4 +-
 .../Language/CPlusPlus/BlockPointer.cpp       |  4 +-
 .../Plugins/Language/CPlusPlus/Coroutines.cpp |  8 +--
 .../Plugins/Language/CPlusPlus/Coroutines.h   |  2 +-
 .../Language/CPlusPlus/GenericBitset.cpp      |  4 +-
 .../Language/CPlusPlus/GenericOptional.cpp    |  6 +-
 .../Plugins/Language/CPlusPlus/LibCxx.cpp     | 18 +++---
 .../Plugins/Language/CPlusPlus/LibCxx.h       |  8 +--
 .../Language/CPlusPlus/LibCxxAtomic.cpp       |  6 +-
 .../CPlusPlus/LibCxxInitializerList.cpp       |  6 +-
 .../Plugins/Language/CPlusPlus/LibCxxList.cpp | 12 ++--
 .../Plugins/Language/CPlusPlus/LibCxxMap.cpp  | 14 ++--
 .../Language/CPlusPlus/LibCxxQueue.cpp        |  2 +-
 .../CPlusPlus/LibCxxRangesRefView.cpp         |  2 +-
 .../Plugins/Language/CPlusPlus/LibCxxSpan.cpp |  6 +-
 .../Language/CPlusPlus/LibCxxTuple.cpp        |  4 +-
 .../Language/CPlusPlus/LibCxxUnorderedMap.cpp |  8 +--
 .../Language/CPlusPlus/LibCxxValarray.cpp     |  6 +-
 .../Language/CPlusPlus/LibCxxVariant.cpp      |  2 +-
 .../Language/CPlusPlus/LibCxxVector.cpp       | 14 ++--
 .../Plugins/Language/CPlusPlus/LibStdcpp.cpp  | 17 ++---
 .../Language/CPlusPlus/LibStdcppTuple.cpp     |  7 +-
 .../CPlusPlus/LibStdcppUniquePointer.cpp      |  5 +-
 lldb/source/Plugins/Language/ObjC/Cocoa.cpp   |  2 +-
 lldb/source/Plugins/Language/ObjC/NSArray.cpp | 30 ++++-----
 .../Plugins/Language/ObjC/NSDictionary.cpp    | 64 +++++++++----------
 lldb/source/Plugins/Language/ObjC/NSError.cpp |  2 +-
 .../Plugins/Language/ObjC/NSException.cpp     |  4 +-
 .../Plugins/Language/ObjC/NSIndexPath.cpp     |  6 +-
 lldb/source/Plugins/Language/ObjC/NSSet.cpp   | 32 +++++-----
 .../AppleObjCRuntime/AppleObjCRuntime.cpp     |  3 +-
 .../TypeSystem/Clang/TypeSystemClang.cpp      | 40 ++++--------
 .../TypeSystem/Clang/TypeSystemClang.h        |  7 +-
 lldb/source/Symbol/CompilerType.cpp           |  5 +-
 lldb/source/Symbol/Type.cpp                   |  2 +-
 lldb/source/Symbol/Variable.cpp               |  6 +-
 lldb/source/Target/StackFrame.cpp             | 19 +++---
 72 files changed, 247 insertions(+), 394 deletions(-)

diff --git a/lldb/include/lldb/Core/ValueObject.h b/lldb/include/lldb/Core/ValueObject.h
index e7e35e2b2bff..b4d2c8098edc 100644
--- a/lldb/include/lldb/Core/ValueObject.h
+++ b/lldb/include/lldb/Core/ValueObject.h
@@ -476,13 +476,7 @@ public:
 
   virtual size_t GetIndexOfChildWithName(llvm::StringRef name);
 
-  llvm::Expected GetNumChildren(uint32_t max = UINT32_MAX);
-  /// Like \c GetNumChildren but returns 0 on error.  You probably
-  /// shouldn't be using this function. It exists primarily to ease the
-  /// transition to more pervasive error handling while not all APIs
-  /// have been updated.
-  uint32_t GetNumChildrenIgnoringErrors(uint32_t max = UINT32_MAX);
-  bool HasChildren() { return GetNumChildrenIgnoringErrors() > 0; }
+  uint32_t GetNumChildren(uint32_t max = UINT32_MAX);
 
   const Value &GetValue() const { return m_value; }
 
@@ -964,8 +958,7 @@ protected:
                                           int32_t synthetic_index);
 
   /// Should only be called by ValueObject::GetNumChildren().
-  virtual llvm::Expected
-  CalculateNumChildren(uint32_t max = UINT32_MAX) = 0;
+  virtual uint32_t CalculateNumChildren(uint32_t max = UINT32_MAX) = 0;
 
   void SetNumChildren(uint32_t num_children);
 
diff --git a/lldb/include/lldb/Core/ValueObjectCast.h b/lldb/include/lldb/Core/ValueObjectCast.h
index ba25e166f326..51c647680d52 100644
--- a/lldb/include/lldb/Core/ValueObjectCast.h
+++ b/lldb/include/lldb/Core/ValueObjectCast.h
@@ -33,7 +33,7 @@ public:
 
   std::optional GetByteSize() override;
 
-  llvm::Expected CalculateNumChildren(uint32_t max) override;
+  uint32_t CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectChild.h b/lldb/include/lldb/Core/ValueObjectChild.h
index 1f88e607cb57..47a13be08bb8 100644
--- a/lldb/include/lldb/Core/ValueObjectChild.h
+++ b/lldb/include/lldb/Core/ValueObjectChild.h
@@ -39,7 +39,7 @@ public:
 
   lldb::ValueType GetValueType() const override;
 
-  llvm::Expected CalculateNumChildren(uint32_t max) override;
+  uint32_t CalculateNumChildren(uint32_t max) override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectConstResult.h b/lldb/include/lldb/Core/ValueObjectConstResult.h
index 37dc0867f26c..9f1246cf2a78 100644
--- a/lldb/include/lldb/Core/ValueObjectConstResult.h
+++ b/lldb/include/lldb/Core/ValueObjectConstResult.h
@@ -67,7 +67,7 @@ public:
 
   lldb::ValueType GetValueType() const override;
 
-  llvm::Expected CalculateNumChildren(uint32_t max) override;
+  uint32_t CalculateNumChildren(uint32_t max) override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectDynamicValue.h b/lldb/include/lldb/Core/ValueObjectDynamicValue.h
index 82c20eee0cd4..21a9b409fd5b 100644
--- a/lldb/include/lldb/Core/ValueObjectDynamicValue.h
+++ b/lldb/include/lldb/Core/ValueObjectDynamicValue.h
@@ -43,7 +43,7 @@ public:
 
   ConstString GetDisplayTypeName() override;
 
-  llvm::Expected CalculateNumChildren(uint32_t max) override;
+  uint32_t CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectMemory.h b/lldb/include/lldb/Core/ValueObjectMemory.h
index a8fb0353d601..a74b325546b0 100644
--- a/lldb/include/lldb/Core/ValueObjectMemory.h
+++ b/lldb/include/lldb/Core/ValueObjectMemory.h
@@ -47,7 +47,7 @@ public:
 
   ConstString GetDisplayTypeName() override;
 
-  llvm::Expected CalculateNumChildren(uint32_t max) override;
+  uint32_t CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectRegister.h b/lldb/include/lldb/Core/ValueObjectRegister.h
index fec8566ba33d..6c470c1a6865 100644
--- a/lldb/include/lldb/Core/ValueObjectRegister.h
+++ b/lldb/include/lldb/Core/ValueObjectRegister.h
@@ -47,7 +47,7 @@ public:
 
   ConstString GetQualifiedTypeName() override;
 
-  llvm::Expected CalculateNumChildren(uint32_t max) override;
+  uint32_t CalculateNumChildren(uint32_t max) override;
 
   ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member,
                                   int32_t synthetic_index) override;
@@ -95,7 +95,7 @@ public:
 
   ConstString GetTypeName() override;
 
-  llvm::Expected CalculateNumChildren(uint32_t max) override;
+  uint32_t CalculateNumChildren(uint32_t max) override;
 
   bool SetValueFromCString(const char *value_str, Status &error) override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h b/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
index ca6d6c728005..1e54babc94f3 100644
--- a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
+++ b/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
@@ -47,7 +47,7 @@ public:
 
   bool MightHaveChildren() override;
 
-  llvm::Expected CalculateNumChildren(uint32_t max) override;
+  uint32_t CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectVTable.h b/lldb/include/lldb/Core/ValueObjectVTable.h
index 4662f395a4dd..e7e14fc83d78 100644
--- a/lldb/include/lldb/Core/ValueObjectVTable.h
+++ b/lldb/include/lldb/Core/ValueObjectVTable.h
@@ -64,7 +64,7 @@ public:
 
   std::optional GetByteSize() override;
 
-  llvm::Expected CalculateNumChildren(uint32_t max) override;
+  uint32_t CalculateNumChildren(uint32_t max) override;
 
   ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member,
                                   int32_t synthetic_index) override;
diff --git a/lldb/include/lldb/Core/ValueObjectVariable.h b/lldb/include/lldb/Core/ValueObjectVariable.h
index db3847f14a0b..da270300df0b 100644
--- a/lldb/include/lldb/Core/ValueObjectVariable.h
+++ b/lldb/include/lldb/Core/ValueObjectVariable.h
@@ -46,7 +46,7 @@ public:
 
   ConstString GetDisplayTypeName() override;
 
-  llvm::Expected CalculateNumChildren(uint32_t max) override;
+  uint32_t CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/DataFormatters/TypeSynthetic.h b/lldb/include/lldb/DataFormatters/TypeSynthetic.h
index ede7442a02bf..38f3ce0fa5f0 100644
--- a/lldb/include/lldb/DataFormatters/TypeSynthetic.h
+++ b/lldb/include/lldb/DataFormatters/TypeSynthetic.h
@@ -38,17 +38,13 @@ public:
 
   virtual ~SyntheticChildrenFrontEnd() = default;
 
-  virtual llvm::Expected CalculateNumChildren() = 0;
+  virtual uint32_t CalculateNumChildren() = 0;
 
-  virtual llvm::Expected CalculateNumChildren(uint32_t max) {
+  virtual uint32_t CalculateNumChildren(uint32_t max) {
     auto count = CalculateNumChildren();
-    if (!count)
-      return count;
-    return *count <= max ? *count : max;
+    return count <= max ? count : max;
   }
 
-  uint32_t CalculateNumChildrenIgnoringErrors(uint32_t max = UINT32_MAX);
-
   virtual lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) = 0;
 
   virtual size_t GetIndexOfChildWithName(ConstString name) = 0;
@@ -113,7 +109,7 @@ public:
 
   ~SyntheticValueProviderFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override { return 0; }
+  uint32_t CalculateNumChildren() override { return 0; }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override { return nullptr; }
 
@@ -326,9 +322,7 @@ public:
 
     ~FrontEnd() override = default;
 
-    llvm::Expected CalculateNumChildren() override {
-      return filter->GetCount();
-    }
+    uint32_t CalculateNumChildren() override { return filter->GetCount(); }
 
     lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
       if (idx >= filter->GetCount())
@@ -432,9 +426,9 @@ public:
 
     bool IsValid();
 
-    llvm::Expected CalculateNumChildren() override;
+    uint32_t CalculateNumChildren() override;
 
-    llvm::Expected CalculateNumChildren(uint32_t max) override;
+    uint32_t CalculateNumChildren(uint32_t max) override;
 
     lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/include/lldb/DataFormatters/VectorIterator.h b/lldb/include/lldb/DataFormatters/VectorIterator.h
index 70bcf50ca1b1..7711b9de95db 100644
--- a/lldb/include/lldb/DataFormatters/VectorIterator.h
+++ b/lldb/include/lldb/DataFormatters/VectorIterator.h
@@ -24,7 +24,7 @@ public:
   VectorIteratorSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp,
                                   llvm::ArrayRef item_names);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/include/lldb/Symbol/CompilerType.h b/lldb/include/lldb/Symbol/CompilerType.h
index c1dce4ccbf79..414c44275aaa 100644
--- a/lldb/include/lldb/Symbol/CompilerType.h
+++ b/lldb/include/lldb/Symbol/CompilerType.h
@@ -386,9 +386,8 @@ public:
 
   std::optional GetTypeBitAlign(ExecutionContextScope *exe_scope) const;
 
-  llvm::Expected
-  GetNumChildren(bool omit_empty_base_classes,
-                 const ExecutionContext *exe_ctx) const;
+  uint32_t GetNumChildren(bool omit_empty_base_classes,
+                          const ExecutionContext *exe_ctx) const;
 
   lldb::BasicType GetBasicTypeEnumeration() const;
 
diff --git a/lldb/include/lldb/Symbol/Type.h b/lldb/include/lldb/Symbol/Type.h
index b5eac5fa732d..acd1a769f13c 100644
--- a/lldb/include/lldb/Symbol/Type.h
+++ b/lldb/include/lldb/Symbol/Type.h
@@ -440,7 +440,7 @@ public:
 
   std::optional GetByteSize(ExecutionContextScope *exe_scope);
 
-  llvm::Expected GetNumChildren(bool omit_empty_base_classes);
+  uint32_t GetNumChildren(bool omit_empty_base_classes);
 
   bool IsAggregateType();
 
diff --git a/lldb/include/lldb/Symbol/TypeSystem.h b/lldb/include/lldb/Symbol/TypeSystem.h
index f647fcbf1636..63829131556e 100644
--- a/lldb/include/lldb/Symbol/TypeSystem.h
+++ b/lldb/include/lldb/Symbol/TypeSystem.h
@@ -300,10 +300,9 @@ public:
 
   virtual lldb::Format GetFormat(lldb::opaque_compiler_type_t type) = 0;
 
-  virtual llvm::Expected
-  GetNumChildren(lldb::opaque_compiler_type_t type,
-                 bool omit_empty_base_classes,
-                 const ExecutionContext *exe_ctx) = 0;
+  virtual uint32_t GetNumChildren(lldb::opaque_compiler_type_t type,
+                                  bool omit_empty_base_classes,
+                                  const ExecutionContext *exe_ctx) = 0;
 
   virtual CompilerType GetBuiltinTypeByName(ConstString name);
 
diff --git a/lldb/include/lldb/Target/StackFrameRecognizer.h b/lldb/include/lldb/Target/StackFrameRecognizer.h
index 5e8e12b2a4e9..e111f4a4dc70 100644
--- a/lldb/include/lldb/Target/StackFrameRecognizer.h
+++ b/lldb/include/lldb/Target/StackFrameRecognizer.h
@@ -164,8 +164,7 @@ class ValueObjectRecognizerSynthesizedValue : public ValueObject {
     m_value = m_parent->GetValue();
     return true;
   }
-  llvm::Expected
-  CalculateNumChildren(uint32_t max = UINT32_MAX) override {
+  uint32_t CalculateNumChildren(uint32_t max = UINT32_MAX) override {
     return m_parent->GetNumChildren(max);
   }
   CompilerType GetCompilerTypeImpl() override {
diff --git a/lldb/include/lldb/Utility/Log.h b/lldb/include/lldb/Utility/Log.h
index 01876ad732d4..1fe28d61b9da 100644
--- a/lldb/include/lldb/Utility/Log.h
+++ b/lldb/include/lldb/Utility/Log.h
@@ -373,18 +373,4 @@ template  Log *GetLog(Cat mask) {
       ::llvm::consumeError(::std::move(error_private));                        \
   } while (0)
 
-// Write message to the verbose log, if error is set. In the log
-// message refer to the error with {0}. Error is cleared regardless of
-// whether logging is enabled.
-#define LLDB_LOG_ERRORV(log, error, ...)                                       \
-  do {                                                                         \
-    ::lldb_private::Log *log_private = (log);                                  \
-    ::llvm::Error error_private = (error);                                     \
-    if (log_private && log_private->GetVerbose() && error_private) {           \
-      log_private->FormatError(::std::move(error_private), __FILE__, __func__, \
-                               __VA_ARGS__);                                   \
-    } else                                                                     \
-      ::llvm::consumeError(::std::move(error_private));                        \
-  } while (0)
-
 #endif // LLDB_UTILITY_LOG_H
diff --git a/lldb/source/API/SBValue.cpp b/lldb/source/API/SBValue.cpp
index 94a8f3ea319e..89d26a1fbe28 100644
--- a/lldb/source/API/SBValue.cpp
+++ b/lldb/source/API/SBValue.cpp
@@ -947,7 +947,7 @@ uint32_t SBValue::GetNumChildren(uint32_t max) {
   ValueLocker locker;
   lldb::ValueObjectSP value_sp(GetSP(locker));
   if (value_sp)
-    num_children = value_sp->GetNumChildrenIgnoringErrors(max);
+    num_children = value_sp->GetNumChildren(max);
 
   return num_children;
 }
diff --git a/lldb/source/Core/FormatEntity.cpp b/lldb/source/Core/FormatEntity.cpp
index cf82676bedda..fa5eadc6ff4e 100644
--- a/lldb/source/Core/FormatEntity.cpp
+++ b/lldb/source/Core/FormatEntity.cpp
@@ -926,7 +926,7 @@ static bool DumpValue(Stream &s, const SymbolContext *sc,
     s.PutChar('[');
 
     if (index_higher < 0)
-      index_higher = valobj->GetNumChildrenIgnoringErrors() - 1;
+      index_higher = valobj->GetNumChildren() - 1;
 
     uint32_t max_num_children =
         target->GetTargetSP()->GetMaximumNumberOfChildrenToDisplay();
diff --git a/lldb/source/Core/IOHandlerCursesGUI.cpp b/lldb/source/Core/IOHandlerCursesGUI.cpp
index f86dce247135..620e68a28510 100644
--- a/lldb/source/Core/IOHandlerCursesGUI.cpp
+++ b/lldb/source/Core/IOHandlerCursesGUI.cpp
@@ -4519,7 +4519,7 @@ struct Row {
       calculated_children = true;
       ValueObjectSP valobj = value.GetSP();
       if (valobj) {
-        const uint32_t num_children = valobj->GetNumChildrenIgnoringErrors();
+        const size_t num_children = valobj->GetNumChildren();
         for (size_t i = 0; i < num_children; ++i) {
           children.push_back(Row(valobj->GetChildAtIndex(i), this));
         }
diff --git a/lldb/source/Core/ValueObject.cpp b/lldb/source/Core/ValueObject.cpp
index d813044d02ff..0ed7f03be25c 100644
--- a/lldb/source/Core/ValueObject.cpp
+++ b/lldb/source/Core/ValueObject.cpp
@@ -377,7 +377,7 @@ ValueObjectSP ValueObject::GetChildAtIndex(uint32_t idx, bool can_create) {
   // We may need to update our value if we are dynamic
   if (IsPossibleDynamicType())
     UpdateValueIfNeeded(false);
-  if (idx < GetNumChildrenIgnoringErrors()) {
+  if (idx < GetNumChildren()) {
     // Check if we have already made the child value object?
     if (can_create && !m_children.HasChildAtIndex(idx)) {
       // No we haven't created the child at this index, so lets have our
@@ -440,7 +440,7 @@ ValueObjectSP ValueObject::GetChildMemberWithName(llvm::StringRef name,
   return child_sp;
 }
 
-llvm::Expected ValueObject::GetNumChildren(uint32_t max) {
+uint32_t ValueObject::GetNumChildren(uint32_t max) {
   UpdateValueIfNeeded();
 
   if (max < UINT32_MAX) {
@@ -452,24 +452,11 @@ llvm::Expected ValueObject::GetNumChildren(uint32_t max) {
   }
 
   if (!m_flags.m_children_count_valid) {
-    auto num_children_or_err = CalculateNumChildren();
-    if (num_children_or_err)
-      SetNumChildren(*num_children_or_err);
-    else
-      return num_children_or_err;
+    SetNumChildren(CalculateNumChildren());
   }
   return m_children.GetChildrenCount();
 }
 
-uint32_t ValueObject::GetNumChildrenIgnoringErrors(uint32_t max) {
-  auto value_or_err = GetNumChildren(max);
-  if (value_or_err)
-    return *value_or_err;
-  LLDB_LOG_ERRORV(GetLog(LLDBLog::DataFormatters), value_or_err.takeError(),
-                  "{0}");
-  return 0;
-}
-
 bool ValueObject::MightHaveChildren() {
   bool has_children = false;
   const uint32_t type_info = GetTypeInfo();
@@ -477,7 +464,7 @@ bool ValueObject::MightHaveChildren() {
     if (type_info & (eTypeHasChildren | eTypeIsPointer | eTypeIsReference))
       has_children = true;
   } else {
-    has_children = GetNumChildrenIgnoringErrors() > 0;
+    has_children = GetNumChildren() > 0;
   }
   return has_children;
 }
@@ -1189,7 +1176,7 @@ bool ValueObject::DumpPrintableRepresentation(
       if (flags.Test(eTypeIsArray)) {
         if ((custom_format == eFormatBytes) ||
             (custom_format == eFormatBytesWithASCII)) {
-          const size_t count = GetNumChildrenIgnoringErrors();
+          const size_t count = GetNumChildren();
 
           s << '[';
           for (size_t low = 0; low < count; low++) {
@@ -1228,7 +1215,7 @@ bool ValueObject::DumpPrintableRepresentation(
                                                      // format should be printed
                                                      // directly
         {
-          const size_t count = GetNumChildrenIgnoringErrors();
+          const size_t count = GetNumChildren();
 
           Format format = FormatManager::GetSingleItemFormat(custom_format);
 
@@ -1307,7 +1294,7 @@ bool ValueObject::DumpPrintableRepresentation(
       break;
 
     case eValueObjectRepresentationStyleChildrenCount:
-      strm.Printf("%" PRIu64 "", (uint64_t)GetNumChildrenIgnoringErrors());
+      strm.Printf("%" PRIu64 "", (uint64_t)GetNumChildren());
       str = strm.GetString();
       break;
 
@@ -2333,9 +2320,7 @@ ValueObjectSP ValueObject::GetValueForExpressionPath_Impl(
             child_valobj_sp = root->GetSyntheticArrayMember(index, true);
           if (!child_valobj_sp)
             if (root->HasSyntheticValue() &&
-                llvm::expectedToStdOptional(
-                    root->GetSyntheticValue()->GetNumChildren())
-                        .value_or(0) > index)
+                root->GetSyntheticValue()->GetNumChildren() > index)
               child_valobj_sp =
                   root->GetSyntheticValue()->GetChildAtIndex(index);
           if (child_valobj_sp) {
diff --git a/lldb/source/Core/ValueObjectCast.cpp b/lldb/source/Core/ValueObjectCast.cpp
index c8e316415141..a5c555f86b13 100644
--- a/lldb/source/Core/ValueObjectCast.cpp
+++ b/lldb/source/Core/ValueObjectCast.cpp
@@ -41,13 +41,11 @@ ValueObjectCast::~ValueObjectCast() = default;
 
 CompilerType ValueObjectCast::GetCompilerTypeImpl() { return m_cast_type; }
 
-llvm::Expected ValueObjectCast::CalculateNumChildren(uint32_t max) {
+uint32_t ValueObjectCast::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(
       true, &exe_ctx);
-  if (!children_count)
-    return children_count;
-  return *children_count <= max ? *children_count : max;
+  return children_count <= max ? children_count : max;
 }
 
 std::optional ValueObjectCast::GetByteSize() {
diff --git a/lldb/source/Core/ValueObjectChild.cpp b/lldb/source/Core/ValueObjectChild.cpp
index c6a97dd1a5cd..2e55dd7726bd 100644
--- a/lldb/source/Core/ValueObjectChild.cpp
+++ b/lldb/source/Core/ValueObjectChild.cpp
@@ -49,12 +49,10 @@ lldb::ValueType ValueObjectChild::GetValueType() const {
   return m_parent->GetValueType();
 }
 
-llvm::Expected ValueObjectChild::CalculateNumChildren(uint32_t max) {
+uint32_t ValueObjectChild::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-  if (!children_count)
-    return children_count;
-  return *children_count <= max ? *children_count : max;
+  return children_count <= max ? children_count : max;
 }
 
 static void AdjustForBitfieldness(ConstString &name,
diff --git a/lldb/source/Core/ValueObjectConstResult.cpp b/lldb/source/Core/ValueObjectConstResult.cpp
index 8ac2c1cac2f6..5c7aa4452b70 100644
--- a/lldb/source/Core/ValueObjectConstResult.cpp
+++ b/lldb/source/Core/ValueObjectConstResult.cpp
@@ -216,13 +216,10 @@ std::optional ValueObjectConstResult::GetByteSize() {
 
 void ValueObjectConstResult::SetByteSize(size_t size) { m_byte_size = size; }
 
-llvm::Expected
-ValueObjectConstResult::CalculateNumChildren(uint32_t max) {
+uint32_t ValueObjectConstResult::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-  if (!children_count)
-    return children_count;
-  return *children_count <= max ? *children_count : max;
+  return children_count <= max ? children_count : max;
 }
 
 ConstString ValueObjectConstResult::GetTypeName() {
diff --git a/lldb/source/Core/ValueObjectDynamicValue.cpp b/lldb/source/Core/ValueObjectDynamicValue.cpp
index 4695febdf8ca..4e64760371ae 100644
--- a/lldb/source/Core/ValueObjectDynamicValue.cpp
+++ b/lldb/source/Core/ValueObjectDynamicValue.cpp
@@ -85,15 +85,12 @@ ConstString ValueObjectDynamicValue::GetDisplayTypeName() {
   return m_parent->GetDisplayTypeName();
 }
 
-llvm::Expected
-ValueObjectDynamicValue::CalculateNumChildren(uint32_t max) {
+uint32_t ValueObjectDynamicValue::CalculateNumChildren(uint32_t max) {
   const bool success = UpdateValueIfNeeded(false);
   if (success && m_dynamic_type_info.HasType()) {
     ExecutionContext exe_ctx(GetExecutionContextRef());
     auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-    if (!children_count)
-      return children_count;
-    return *children_count <= max ? *children_count : max;
+    return children_count <= max ? children_count : max;
   } else
     return m_parent->GetNumChildren(max);
 }
diff --git a/lldb/source/Core/ValueObjectMemory.cpp b/lldb/source/Core/ValueObjectMemory.cpp
index f555ab82f441..7f68236c7884 100644
--- a/lldb/source/Core/ValueObjectMemory.cpp
+++ b/lldb/source/Core/ValueObjectMemory.cpp
@@ -126,21 +126,17 @@ ConstString ValueObjectMemory::GetDisplayTypeName() {
   return m_compiler_type.GetDisplayTypeName();
 }
 
-llvm::Expected ValueObjectMemory::CalculateNumChildren(uint32_t max) {
+uint32_t ValueObjectMemory::CalculateNumChildren(uint32_t max) {
   if (m_type_sp) {
     auto child_count = m_type_sp->GetNumChildren(true);
-    if (!child_count)
-      return child_count;
-    return *child_count <= max ? *child_count : max;
+    return child_count <= max ? child_count : max;
   }
 
   ExecutionContext exe_ctx(GetExecutionContextRef());
   const bool omit_empty_base_classes = true;
   auto child_count =
       m_compiler_type.GetNumChildren(omit_empty_base_classes, &exe_ctx);
-  if (!child_count)
-    return child_count;
-  return *child_count <= max ? *child_count : max;
+  return child_count <= max ? child_count : max;
 }
 
 std::optional ValueObjectMemory::GetByteSize() {
diff --git a/lldb/source/Core/ValueObjectRegister.cpp b/lldb/source/Core/ValueObjectRegister.cpp
index 9d9da8bb119d..d4c144cc7edb 100644
--- a/lldb/source/Core/ValueObjectRegister.cpp
+++ b/lldb/source/Core/ValueObjectRegister.cpp
@@ -74,8 +74,7 @@ ConstString ValueObjectRegisterSet::GetQualifiedTypeName() {
   return ConstString();
 }
 
-llvm::Expected
-ValueObjectRegisterSet::CalculateNumChildren(uint32_t max) {
+uint32_t ValueObjectRegisterSet::CalculateNumChildren(uint32_t max) {
   const RegisterSet *reg_set = m_reg_ctx_sp->GetRegisterSet(m_reg_set_idx);
   if (reg_set) {
     auto reg_count = reg_set->num_registers;
@@ -119,7 +118,7 @@ ValueObject *ValueObjectRegisterSet::CreateChildAtIndex(
     size_t idx, bool synthetic_array_member, int32_t synthetic_index) {
   ValueObject *valobj = nullptr;
   if (m_reg_ctx_sp && m_reg_set) {
-    uint32_t num_children = GetNumChildrenIgnoringErrors();
+    const size_t num_children = GetNumChildren();
     if (idx < num_children)
       valobj = new ValueObjectRegister(
           *this, m_reg_ctx_sp,
@@ -221,13 +220,10 @@ ConstString ValueObjectRegister::GetTypeName() {
   return m_type_name;
 }
 
-llvm::Expected
-ValueObjectRegister::CalculateNumChildren(uint32_t max) {
+uint32_t ValueObjectRegister::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-  if (!children_count)
-    return children_count;
-  return *children_count <= max ? *children_count : max;
+  return children_count <= max ? children_count : max;
 }
 
 std::optional ValueObjectRegister::GetByteSize() {
diff --git a/lldb/source/Core/ValueObjectSyntheticFilter.cpp b/lldb/source/Core/ValueObjectSyntheticFilter.cpp
index adac1b400705..7f8a9a34cb35 100644
--- a/lldb/source/Core/ValueObjectSyntheticFilter.cpp
+++ b/lldb/source/Core/ValueObjectSyntheticFilter.cpp
@@ -31,9 +31,7 @@ public:
   DummySyntheticFrontEnd(ValueObject &backend)
       : SyntheticChildrenFrontEnd(backend) {}
 
-  llvm::Expected CalculateNumChildren() override {
-    return m_backend.GetNumChildren();
-  }
+  uint32_t CalculateNumChildren() override { return m_backend.GetNumChildren(); }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     return m_backend.GetChildAtIndex(idx);
@@ -86,8 +84,7 @@ ConstString ValueObjectSynthetic::GetDisplayTypeName() {
   return m_parent->GetDisplayTypeName();
 }
 
-llvm::Expected
-ValueObjectSynthetic::CalculateNumChildren(uint32_t max) {
+uint32_t ValueObjectSynthetic::CalculateNumChildren(uint32_t max) {
   Log *log = GetLog(LLDBLog::DataFormatters);
 
   UpdateValueIfNeeded();
@@ -95,23 +92,18 @@ ValueObjectSynthetic::CalculateNumChildren(uint32_t max) {
     return m_synthetic_children_count <= max ? m_synthetic_children_count : max;
 
   if (max < UINT32_MAX) {
-    auto num_children = m_synth_filter_up->CalculateNumChildren(max);
+    size_t num_children = m_synth_filter_up->CalculateNumChildren(max);
     LLDB_LOGF(log,
               "[ValueObjectSynthetic::CalculateNumChildren] for VO of name "
-              "%s and type %s, the filter returned %u child values",
-              GetName().AsCString(), GetTypeName().AsCString(),
-              num_children ? *num_children : 0);
+              "%s and type %s, the filter returned %zu child values",
+              GetName().AsCString(), GetTypeName().AsCString(), num_children);
     return num_children;
   } else {
-    auto num_children_or_err = m_synth_filter_up->CalculateNumChildren(max);
-    if (!num_children_or_err) {
-      m_synthetic_children_count = 0;
-      return num_children_or_err;
-    }
-    auto num_children = (m_synthetic_children_count = *num_children_or_err);
+    size_t num_children = (m_synthetic_children_count =
+                               m_synth_filter_up->CalculateNumChildren(max));
     LLDB_LOGF(log,
               "[ValueObjectSynthetic::CalculateNumChildren] for VO of name "
-              "%s and type %s, the filter returned %u child values",
+              "%s and type %s, the filter returned %zu child values",
               GetName().AsCString(), GetTypeName().AsCString(), num_children);
     return num_children;
   }
diff --git a/lldb/source/Core/ValueObjectVTable.cpp b/lldb/source/Core/ValueObjectVTable.cpp
index 9bfe8dc9700d..4d1cbb8d2f6f 100644
--- a/lldb/source/Core/ValueObjectVTable.cpp
+++ b/lldb/source/Core/ValueObjectVTable.cpp
@@ -33,9 +33,7 @@ public:
 
   std::optional GetByteSize() override { return m_addr_size; };
 
-  llvm::Expected CalculateNumChildren(uint32_t max) override {
-    return 0;
-  };
+  uint32_t CalculateNumChildren(uint32_t max) override { return 0; };
 
   ValueType GetValueType() const override { return eValueTypeVTableEntry; };
 
@@ -161,7 +159,7 @@ std::optional ValueObjectVTable::GetByteSize() {
   return std::nullopt;
 }
 
-llvm::Expected ValueObjectVTable::CalculateNumChildren(uint32_t max) {
+uint32_t ValueObjectVTable::CalculateNumChildren(uint32_t max) {
   if (UpdateValueIfNeeded(false))
     return m_num_vtable_entries <= max ? m_num_vtable_entries : max;
   return 0;
diff --git a/lldb/source/Core/ValueObjectVariable.cpp b/lldb/source/Core/ValueObjectVariable.cpp
index fb29c22c0ab5..dc62bb6358dc 100644
--- a/lldb/source/Core/ValueObjectVariable.cpp
+++ b/lldb/source/Core/ValueObjectVariable.cpp
@@ -94,8 +94,7 @@ ConstString ValueObjectVariable::GetQualifiedTypeName() {
   return ConstString();
 }
 
-llvm::Expected
-ValueObjectVariable::CalculateNumChildren(uint32_t max) {
+uint32_t ValueObjectVariable::CalculateNumChildren(uint32_t max) {
   CompilerType type(GetCompilerType());
 
   if (!type.IsValid())
@@ -104,9 +103,7 @@ ValueObjectVariable::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   const bool omit_empty_base_classes = true;
   auto child_count = type.GetNumChildren(omit_empty_base_classes, &exe_ctx);
-  if (!child_count)
-    return child_count;
-  return *child_count <= max ? *child_count : max;
+  return child_count <= max ? child_count : max;
 }
 
 std::optional ValueObjectVariable::GetByteSize() {
diff --git a/lldb/source/DataFormatters/FormatManager.cpp b/lldb/source/DataFormatters/FormatManager.cpp
index d7ba5b4b70c9..092fa3c8ce49 100644
--- a/lldb/source/DataFormatters/FormatManager.cpp
+++ b/lldb/source/DataFormatters/FormatManager.cpp
@@ -451,13 +451,8 @@ bool FormatManager::ShouldPrintAsOneLiner(ValueObject &valobj) {
   if (valobj.GetSummaryFormat().get() != nullptr)
     return valobj.GetSummaryFormat()->IsOneLiner();
 
-  auto num_children = valobj.GetNumChildren();
-  if (!num_children) {
-    llvm::consumeError(num_children.takeError());
-    return true;
-  }
   // no children, no party
-  if (*num_children == 0)
+  if (valobj.GetNumChildren() == 0)
     return false;
 
   // ask the type if it has any opinion about this eLazyBoolCalculate == no
@@ -476,7 +471,7 @@ bool FormatManager::ShouldPrintAsOneLiner(ValueObject &valobj) {
 
   size_t total_children_name_len = 0;
 
-  for (size_t idx = 0; idx < *num_children; idx++) {
+  for (size_t idx = 0; idx < valobj.GetNumChildren(); idx++) {
     bool is_synth_val = false;
     ValueObjectSP child_sp(valobj.GetChildAtIndex(idx));
     // something is wrong here - bail out
@@ -528,7 +523,7 @@ bool FormatManager::ShouldPrintAsOneLiner(ValueObject &valobj) {
     }
 
     // if this child has children..
-    if (child_sp->HasChildren()) {
+    if (child_sp->GetNumChildren()) {
       // ...and no summary...
       // (if it had a summary and the summary wanted children, we would have
       // bailed out anyway
diff --git a/lldb/source/DataFormatters/TypeSynthetic.cpp b/lldb/source/DataFormatters/TypeSynthetic.cpp
index e3f613606aa8..0ae38c4d31f2 100644
--- a/lldb/source/DataFormatters/TypeSynthetic.cpp
+++ b/lldb/source/DataFormatters/TypeSynthetic.cpp
@@ -115,17 +115,6 @@ std::string CXXSyntheticChildren::GetDescription() {
   return std::string(sstr.GetString());
 }
 
-uint32_t
-SyntheticChildrenFrontEnd::CalculateNumChildrenIgnoringErrors(uint32_t max) {
-  auto value_or_err = CalculateNumChildren(max);
-  if (value_or_err)
-    return *value_or_err;
-  Log *log = GetLog(LLDBLog::DataFormatters);
-  if (log && log->GetVerbose())
-    LLDB_LOG_ERROR(log, value_or_err.takeError(), "{0}");
-  return 0;
-}
-
 lldb::ValueObjectSP SyntheticChildrenFrontEnd::CreateValueObjectFromExpression(
     llvm::StringRef name, llvm::StringRef expression,
     const ExecutionContext &exe_ctx) {
@@ -189,15 +178,13 @@ bool ScriptedSyntheticChildren::FrontEnd::IsValid() {
   return (m_wrapper_sp && m_wrapper_sp->IsValid() && m_interpreter);
 }
 
-llvm::Expected
-ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren() {
+uint32_t ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren() {
   if (!m_wrapper_sp || m_interpreter == nullptr)
     return 0;
   return m_interpreter->CalculateNumChildren(m_wrapper_sp, UINT32_MAX);
 }
 
-llvm::Expected
-ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren(uint32_t max) {
+uint32_t ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren(uint32_t max) {
   if (!m_wrapper_sp || m_interpreter == nullptr)
     return 0;
   return m_interpreter->CalculateNumChildren(m_wrapper_sp, max);
diff --git a/lldb/source/DataFormatters/ValueObjectPrinter.cpp b/lldb/source/DataFormatters/ValueObjectPrinter.cpp
index b853199e878c..46e50a8d421a 100644
--- a/lldb/source/DataFormatters/ValueObjectPrinter.cpp
+++ b/lldb/source/DataFormatters/ValueObjectPrinter.cpp
@@ -627,7 +627,7 @@ uint32_t ValueObjectPrinter::GetMaxNumChildrenToPrint(bool &print_dotdotdot) {
   if (m_options.m_pointer_as_array)
     return m_options.m_pointer_as_array.m_element_count;
 
-  uint32_t num_children = synth_valobj.GetNumChildrenIgnoringErrors();
+  size_t num_children = synth_valobj.GetNumChildren();
   print_dotdotdot = false;
   if (num_children) {
     const size_t max_num_children = GetMostSpecializedValue()
diff --git a/lldb/source/DataFormatters/VectorType.cpp b/lldb/source/DataFormatters/VectorType.cpp
index 19de204c2435..a0626a8cba77 100644
--- a/lldb/source/DataFormatters/VectorType.cpp
+++ b/lldb/source/DataFormatters/VectorType.cpp
@@ -9,7 +9,6 @@
 #include "lldb/DataFormatters/VectorType.h"
 
 #include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Symbol/TypeSystem.h"
@@ -225,16 +224,10 @@ public:
 
   ~VectorTypeSyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override {
-    return m_num_children;
-  }
+  uint32_t CalculateNumChildren() override { return m_num_children; }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
-    auto num_children_or_err = CalculateNumChildren();
-    if (!num_children_or_err)
-      return ValueObjectConstResult::Create(
-          nullptr, Status(num_children_or_err.takeError()));
-    if (idx >= *num_children_or_err)
+    if (idx >= CalculateNumChildren())
       return {};
     std::optional size = m_child_type.GetByteSize(nullptr);
     if (!size)
@@ -273,7 +266,7 @@ public:
   size_t GetIndexOfChildWithName(ConstString name) override {
     const char *item_name = name.GetCString();
     uint32_t idx = ExtractIndexFromString(item_name);
-    if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+    if (idx < UINT32_MAX && idx >= CalculateNumChildren())
       return UINT32_MAX;
     return idx;
   }
@@ -300,8 +293,7 @@ bool lldb_private::formatters::VectorTypeSummaryProvider(
   s.PutChar('(');
   bool first = true;
 
-  size_t idx = 0,
-         len = synthetic_children->CalculateNumChildrenIgnoringErrors();
+  size_t idx = 0, len = synthetic_children->CalculateNumChildren();
 
   for (; idx < len; idx++) {
     auto child_sp = synthetic_children->GetChildAtIndex(idx);
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp
index 3d43ed3f99ff..b48bbbecc0cd 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp
@@ -229,7 +229,7 @@ void AddLambdaCaptureDecls(StreamString &stream, StackFrame *frame,
   assert(frame);
 
   if (auto thisValSP = ClangExpressionUtil::GetLambdaValueObject(frame)) {
-    uint32_t numChildren = thisValSP->GetNumChildrenIgnoringErrors();
+    uint32_t numChildren = thisValSP->GetNumChildren();
     for (uint32_t i = 0; i < numChildren; ++i) {
       auto childVal = thisValSP->GetChildAtIndex(i);
       ConstString childName(childVal ? childVal->GetName() : ConstString(""));
diff --git a/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp b/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
index b2781aa5e7db..72293c5331f4 100644
--- a/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
+++ b/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
@@ -27,8 +27,6 @@
 #include "lldb/Target/StopInfo.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Target/Thread.h"
-#include "lldb/Utility/LLDBLog.h"
-#include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegularExpression.h"
 #include "lldb/Utility/Stream.h"
 
@@ -213,7 +211,7 @@ CreateStackTrace(ValueObjectSP o,
   auto trace_sp = std::make_shared();
   ValueObjectSP trace_value_object =
       o->GetValueForExpressionPath(trace_item_name.c_str());
-  size_t count = trace_value_object->GetNumChildrenIgnoringErrors();
+  size_t count = trace_value_object->GetNumChildren();
   for (size_t j = 0; j < count; j++) {
     addr_t trace_addr =
         trace_value_object->GetChildAtIndex(j)->GetValueAsUnsigned(0);
diff --git a/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp b/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
index 9a6e135e0083..ef0f67d1e9f9 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
@@ -74,7 +74,7 @@ public:
 
   ~BlockPointerSyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override {
+  uint32_t CalculateNumChildren() override {
     const bool omit_empty_base_classes = false;
     return m_block_struct_type.GetNumChildren(omit_empty_base_classes, nullptr);
   }
@@ -84,7 +84,7 @@ public:
       return lldb::ValueObjectSP();
     }
 
-    if (idx >= CalculateNumChildrenIgnoringErrors()) {
+    if (idx >= CalculateNumChildren()) {
       return lldb::ValueObjectSP();
     }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp
index 5e63d1d7b214..3827f9c21eff 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp
@@ -11,8 +11,6 @@
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
 #include "lldb/Symbol/Function.h"
 #include "lldb/Symbol/VariableList.h"
-#include "lldb/Utility/LLDBLog.h"
-#include "lldb/Utility/Log.h"
 
 using namespace lldb;
 using namespace lldb_private;
@@ -24,7 +22,7 @@ static lldb::addr_t GetCoroFramePtrFromHandle(ValueObjectSP valobj_sp) {
 
   // We expect a single pointer in the `coroutine_handle` class.
   // We don't care about its name.
-  if (valobj_sp->GetNumChildrenIgnoringErrors() != 1)
+  if (valobj_sp->GetNumChildren() != 1)
     return LLDB_INVALID_ADDRESS;
   ValueObjectSP ptr_sp(valobj_sp->GetChildAtIndex(0));
   if (!ptr_sp)
@@ -106,8 +104,8 @@ lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd::
 lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd::
     ~StdlibCoroutineHandleSyntheticFrontEnd() = default;
 
-llvm::Expected lldb_private::formatters::
-    StdlibCoroutineHandleSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd::
+    CalculateNumChildren() {
   if (!m_resume_ptr_sp || !m_destroy_ptr_sp)
     return 0;
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
index 1d4bc65e2637..5c6a80b57ff4 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
@@ -34,7 +34,7 @@ public:
 
   ~StdlibCoroutineHandleSyntheticFrontEnd() override;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp b/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp
index 33955dccb6cc..6a9da1d17c76 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp
@@ -34,9 +34,7 @@ public:
 
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
-  llvm::Expected CalculateNumChildren() override {
-    return m_elements.size();
-  }
+  uint32_t CalculateNumChildren() override { return m_elements.size(); }
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
 private:
diff --git a/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp b/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp
index 23756de7f1e6..c06afb53eb8a 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp
@@ -19,7 +19,7 @@ using namespace lldb_private;
 bool lldb_private::formatters::GenericOptionalSummaryProvider(
     ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
   stream.Printf(" Has Value=%s ",
-                valobj.GetNumChildrenIgnoringErrors() == 0 ? "false" : "true");
+                valobj.GetNumChildren() == 0 ? "false" : "true");
 
   return true;
 }
@@ -41,9 +41,7 @@ public:
   }
 
   bool MightHaveChildren() override { return true; }
-  llvm::Expected CalculateNumChildren() override {
-    return m_has_value ? 1U : 0U;
-  }
+  uint32_t CalculateNumChildren() override { return m_has_value ? 1U : 0U; }
 
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
   lldb::ChildCacheState Update() override;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
index 10a1fe039189..bba887fec3ac 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
@@ -63,7 +63,7 @@ lldb::ValueObjectSP
 lldb_private::formatters::GetSecondValueOfLibCXXCompressedPair(
     ValueObject &pair) {
   ValueObjectSP value;
-  if (pair.GetNumChildrenIgnoringErrors() > 1) {
+  if (pair.GetNumChildren() > 1) {
     ValueObjectSP second_child = pair.GetChildAtIndex(1);
     if (second_child) {
       value = second_child->GetChildMemberWithName("__value_");
@@ -351,8 +351,8 @@ lldb_private::formatters::LibCxxMapIteratorSyntheticFrontEnd::Update() {
   return lldb::ChildCacheState::eRefetch;
 }
 
-llvm::Expected lldb_private::formatters::
-    LibCxxMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibCxxMapIteratorSyntheticFrontEnd::
+    CalculateNumChildren() {
   return 2;
 }
 
@@ -509,8 +509,8 @@ lldb::ChildCacheState lldb_private::formatters::
   return lldb::ChildCacheState::eRefetch;
 }
 
-llvm::Expected lldb_private::formatters::
-    LibCxxUnorderedMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibCxxUnorderedMapIteratorSyntheticFrontEnd::
+    CalculateNumChildren() {
   return 2;
 }
 
@@ -566,8 +566,8 @@ lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::
     Update();
 }
 
-llvm::Expected lldb_private::formatters::
-    LibcxxSharedPtrSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::
+    CalculateNumChildren() {
   return (m_cntrl ? 1 : 0);
 }
 
@@ -661,8 +661,8 @@ lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEndCreator(
                     : nullptr);
 }
 
-llvm::Expected lldb_private::formatters::
-    LibcxxUniquePtrSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEnd::
+    CalculateNumChildren() {
   if (m_value_ptr_sp)
     return m_deleter_sp ? 2 : 1;
   return 0;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
index a59f21841ec8..ad2f58508ab7 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
@@ -87,7 +87,7 @@ class LibCxxMapIteratorSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibCxxMapIteratorSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -135,7 +135,7 @@ public:
 
   ~LibCxxUnorderedMapIteratorSyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -166,7 +166,7 @@ class LibcxxSharedPtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibcxxSharedPtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -186,7 +186,7 @@ class LibcxxUniquePtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibcxxUniquePtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp
index 7f30dc186291..8e4c36103a74 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp
@@ -90,7 +90,7 @@ public:
 
   ~LibcxxStdAtomicSyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -124,8 +124,8 @@ bool lldb_private::formatters::LibcxxStdAtomicSyntheticFrontEnd::
   return true;
 }
 
-llvm::Expected lldb_private::formatters::
-    LibcxxStdAtomicSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibcxxStdAtomicSyntheticFrontEnd::
+    CalculateNumChildren() {
   return m_real_child ? 1 : 0;
 }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp
index bd9c72497664..00012dfc056e 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp
@@ -26,7 +26,7 @@ public:
 
   ~LibcxxInitializerListSyntheticFrontEnd() override;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -59,8 +59,8 @@ lldb_private::formatters::LibcxxInitializerListSyntheticFrontEnd::
   // delete m_start;
 }
 
-llvm::Expected lldb_private::formatters::
-    LibcxxInitializerListSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibcxxInitializerListSyntheticFrontEnd::
+    CalculateNumChildren() {
   m_num_elements = 0;
   ValueObjectSP size_sp(m_backend.GetChildMemberWithName("__size_"));
   if (size_sp)
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
index d7cfeb30557c..17f6b737d9f6 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
@@ -136,7 +136,7 @@ class ForwardListFrontEnd : public AbstractListFrontEnd {
 public:
   ForwardListFrontEnd(ValueObject &valobj);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
   lldb::ChildCacheState Update() override;
 };
@@ -147,7 +147,7 @@ public:
 
   ~ListFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -240,7 +240,7 @@ ForwardListFrontEnd::ForwardListFrontEnd(ValueObject &valobj)
   Update();
 }
 
-llvm::Expected ForwardListFrontEnd::CalculateNumChildren() {
+uint32_t ForwardListFrontEnd::CalculateNumChildren() {
   if (m_count != UINT32_MAX)
     return m_count;
 
@@ -254,7 +254,7 @@ llvm::Expected ForwardListFrontEnd::CalculateNumChildren() {
 }
 
 ValueObjectSP ForwardListFrontEnd::GetChildAtIndex(uint32_t idx) {
-  if (idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx >= CalculateNumChildren())
     return nullptr;
 
   if (!m_head)
@@ -308,7 +308,7 @@ ListFrontEnd::ListFrontEnd(lldb::ValueObjectSP valobj_sp)
     Update();
 }
 
-llvm::Expected ListFrontEnd::CalculateNumChildren() {
+uint32_t ListFrontEnd::CalculateNumChildren() {
   if (m_count != UINT32_MAX)
     return m_count;
   if (!m_head || !m_tail || m_node_address == 0)
@@ -347,7 +347,7 @@ lldb::ValueObjectSP ListFrontEnd::GetChildAtIndex(uint32_t idx) {
   static ConstString g_value("__value_");
   static ConstString g_next("__next_");
 
-  if (idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx >= CalculateNumChildren())
     return lldb::ValueObjectSP();
 
   if (!m_head || !m_tail || m_node_address == 0)
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp
index ec5b320e2218..6d24eb03779c 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp
@@ -177,7 +177,7 @@ public:
 
   ~LibcxxStdMapSyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -209,8 +209,8 @@ lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::
     Update();
 }
 
-llvm::Expected lldb_private::formatters::
-    LibcxxStdMapSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::
+    CalculateNumChildren() {
   if (m_count != UINT32_MAX)
     return m_count;
 
@@ -311,13 +311,13 @@ lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
   static ConstString g_cc_("__cc_"), g_cc("__cc");
   static ConstString g_nc("__nc");
-  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
-  if (idx >= num_children)
+
+  if (idx >= CalculateNumChildren())
     return lldb::ValueObjectSP();
   if (m_tree == nullptr || m_root_node == nullptr)
     return lldb::ValueObjectSP();
 
-  MapIterator iterator(m_root_node, num_children);
+  MapIterator iterator(m_root_node, CalculateNumChildren());
 
   const bool need_to_skip = (idx > 0);
   size_t actual_advancde = idx;
@@ -382,7 +382,7 @@ lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::GetChildAtIndex(
   name.Printf("[%" PRIu64 "]", (uint64_t)idx);
   auto potential_child_sp = iterated_sp->Clone(ConstString(name.GetString()));
   if (potential_child_sp) {
-    switch (potential_child_sp->GetNumChildrenIgnoringErrors()) {
+    switch (potential_child_sp->GetNumChildren()) {
     case 1: {
       auto child0_sp = potential_child_sp->GetChildAtIndex(0);
       if (child0_sp &&
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp
index 5b459a17fe29..fbadee89b7b7 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp
@@ -28,7 +28,7 @@ public:
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
 
-  llvm::Expected CalculateNumChildren() override {
+  uint32_t CalculateNumChildren() override {
     return m_container_sp ? m_container_sp->GetNumChildren() : 0;
   }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp
index 01a7b8f142ec..74f54f767356 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp
@@ -27,7 +27,7 @@ public:
 
   ~LibcxxStdRangesRefViewSyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override {
+  uint32_t CalculateNumChildren() override {
     // __range_ will be the sole child of this type
     return 1;
   }
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp
index 9895f336bfd0..af2b51d2b540 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp
@@ -27,7 +27,7 @@ public:
 
   ~LibcxxStdSpanSyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -73,8 +73,8 @@ lldb_private::formatters::LibcxxStdSpanSyntheticFrontEnd::
     Update();
 }
 
-llvm::Expected lldb_private::formatters::
-    LibcxxStdSpanSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibcxxStdSpanSyntheticFrontEnd::
+    CalculateNumChildren() {
   return m_num_elements;
 }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp
index 3e3259ab428d..62bb7d619267 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp
@@ -26,9 +26,7 @@ public:
 
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
-  llvm::Expected CalculateNumChildren() override {
-    return m_elements.size();
-  }
+  uint32_t CalculateNumChildren() override { return m_elements.size(); }
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
 private:
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
index af29fdb6d001..b3c364294335 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
@@ -33,7 +33,7 @@ public:
 
   ~LibcxxStdUnorderedMapSyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -62,8 +62,8 @@ lldb_private::formatters::LibcxxStdUnorderedMapSyntheticFrontEnd::
     Update();
 }
 
-llvm::Expected lldb_private::formatters::
-    LibcxxStdUnorderedMapSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibcxxStdUnorderedMapSyntheticFrontEnd::
+    CalculateNumChildren() {
   return m_num_elements;
 }
 
@@ -94,7 +94,7 @@ static bool isUnorderedMap(ConstString type_name) {
 
 lldb::ValueObjectSP lldb_private::formatters::
     LibcxxStdUnorderedMapSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
-  if (idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx >= CalculateNumChildren())
     return lldb::ValueObjectSP();
   if (m_tree == nullptr)
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp
index 99f94406e99a..463c7b8d7ce3 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp
@@ -24,7 +24,7 @@ public:
 
   ~LibcxxStdValarraySyntheticFrontEnd() override;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -63,8 +63,8 @@ lldb_private::formatters::LibcxxStdValarraySyntheticFrontEnd::
   // delete m_finish;
 }
 
-llvm::Expected lldb_private::formatters::
-    LibcxxStdValarraySyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibcxxStdValarraySyntheticFrontEnd::
+    CalculateNumChildren() {
   if (!m_start || !m_finish)
     return 0;
   uint64_t start_val = m_start->GetValueAsUnsigned(0);
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp
index 62794318e077..1f62062f09be 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp
@@ -205,7 +205,7 @@ public:
 
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
-  llvm::Expected CalculateNumChildren() override { return m_size; }
+  uint32_t CalculateNumChildren() override { return m_size; }
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
 private:
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
index 461fed35164b..fcf727ad2ea0 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
@@ -25,7 +25,7 @@ public:
 
   ~LibcxxStdVectorSyntheticFrontEnd() override;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -46,7 +46,7 @@ class LibcxxVectorBoolSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibcxxVectorBoolSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -82,8 +82,8 @@ lldb_private::formatters::LibcxxStdVectorSyntheticFrontEnd::
   // delete m_finish;
 }
 
-llvm::Expected lldb_private::formatters::
-    LibcxxStdVectorSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibcxxStdVectorSyntheticFrontEnd::
+    CalculateNumChildren() {
   if (!m_start || !m_finish)
     return 0;
   uint64_t start_val = m_start->GetValueAsUnsigned(0);
@@ -165,8 +165,8 @@ lldb_private::formatters::LibcxxVectorBoolSyntheticFrontEnd::
   }
 }
 
-llvm::Expected lldb_private::formatters::
-    LibcxxVectorBoolSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibcxxVectorBoolSyntheticFrontEnd::
+    CalculateNumChildren() {
   return m_count;
 }
 
@@ -259,7 +259,7 @@ size_t lldb_private::formatters::LibcxxVectorBoolSyntheticFrontEnd::
     return UINT32_MAX;
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
     return UINT32_MAX;
   return idx;
 }
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
index 86bb575af5ca..5abb3d50674b 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
@@ -43,7 +43,7 @@ class LibstdcppMapIteratorSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibstdcppMapIteratorSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -64,7 +64,7 @@ class LibStdcppSharedPtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibStdcppSharedPtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -132,8 +132,7 @@ lldb::ChildCacheState LibstdcppMapIteratorSyntheticFrontEnd::Update() {
   return lldb::ChildCacheState::eReuse;
 }
 
-llvm::Expected
-LibstdcppMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t LibstdcppMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
   return 2;
 }
 
@@ -220,10 +219,7 @@ lldb::ChildCacheState VectorIteratorSyntheticFrontEnd::Update() {
   return lldb::ChildCacheState::eRefetch;
 }
 
-llvm::Expected
-VectorIteratorSyntheticFrontEnd::CalculateNumChildren() {
-  return 1;
-}
+uint32_t VectorIteratorSyntheticFrontEnd::CalculateNumChildren() { return 1; }
 
 lldb::ValueObjectSP
 VectorIteratorSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
@@ -375,10 +371,7 @@ LibStdcppSharedPtrSyntheticFrontEnd::LibStdcppSharedPtrSyntheticFrontEnd(
     Update();
 }
 
-llvm::Expected
-LibStdcppSharedPtrSyntheticFrontEnd::CalculateNumChildren() {
-  return 1;
-}
+uint32_t LibStdcppSharedPtrSyntheticFrontEnd::CalculateNumChildren() { return 1; }
 
 lldb::ValueObjectSP
 LibStdcppSharedPtrSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp
index 05199ba35b9a..64d2ec9d943a 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp
@@ -26,7 +26,7 @@ class LibStdcppTupleSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibStdcppTupleSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -65,7 +65,7 @@ lldb::ChildCacheState LibStdcppTupleSyntheticFrontEnd::Update() {
     ValueObjectSP current_child = next_child_sp;
     next_child_sp = nullptr;
 
-    size_t child_count = current_child->GetNumChildrenIgnoringErrors();
+    size_t child_count = current_child->GetNumChildren();
     for (size_t i = 0; i < child_count; ++i) {
       ValueObjectSP child_sp = current_child->GetChildAtIndex(i);
       llvm::StringRef name_str = child_sp->GetName().GetStringRef();
@@ -95,8 +95,7 @@ LibStdcppTupleSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
   return lldb::ValueObjectSP();
 }
 
-llvm::Expected
-LibStdcppTupleSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t LibStdcppTupleSyntheticFrontEnd::CalculateNumChildren() {
   return m_members.size();
 }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
index 92f540d9ca52..3a48fe412e07 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
@@ -26,7 +26,7 @@ class LibStdcppUniquePtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibStdcppUniquePtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -135,8 +135,7 @@ LibStdcppUniquePtrSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
   return lldb::ValueObjectSP();
 }
 
-llvm::Expected
-LibStdcppUniquePtrSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t LibStdcppUniquePtrSyntheticFrontEnd::CalculateNumChildren() {
   if (m_del_obj)
     return 2;
   return 1;
diff --git a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
index 96166657ceeb..cb740f8e71e1 100644
--- a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
+++ b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
@@ -1038,7 +1038,7 @@ public:
 
   ~ObjCClassSyntheticChildrenFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override { return 0; }
+  uint32_t CalculateNumChildren() override { return 0; }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/Language/ObjC/NSArray.cpp b/lldb/source/Plugins/Language/ObjC/NSArray.cpp
index 67d0cd08f51a..7f060b2613d6 100644
--- a/lldb/source/Plugins/Language/ObjC/NSArray.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSArray.cpp
@@ -50,7 +50,7 @@ public:
 
   ~NSArrayMSyntheticFrontEndBase() override = default;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -214,7 +214,7 @@ public:
 
   ~GenericNSArrayISyntheticFrontEnd() override;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -302,7 +302,7 @@ public:
 
   ~NSArray0SyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -319,7 +319,7 @@ public:
 
   ~NSArray1SyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -477,15 +477,15 @@ lldb_private::formatters::
     : NSArrayMSyntheticFrontEndBase(valobj_sp), m_data_32(nullptr),
       m_data_64(nullptr) {}
 
-llvm::Expected lldb_private::formatters::
-    NSArrayMSyntheticFrontEndBase::CalculateNumChildren() {
+uint32_t lldb_private::formatters::NSArrayMSyntheticFrontEndBase::
+    CalculateNumChildren() {
   return GetUsedCount();
 }
 
 lldb::ValueObjectSP
 lldb_private::formatters::NSArrayMSyntheticFrontEndBase::GetChildAtIndex(
     uint32_t idx) {
-  if (idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx >= CalculateNumChildren())
     return lldb::ValueObjectSP();
   lldb::addr_t object_at_idx = GetDataAddress();
   size_t pyhs_idx = idx;
@@ -542,7 +542,7 @@ lldb_private::formatters::NSArrayMSyntheticFrontEndBase::GetIndexOfChildWithName
     ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
     return UINT32_MAX;
   return idx;
 }
@@ -628,15 +628,15 @@ lldb_private::formatters::GenericNSArrayISyntheticFrontEnd::
   GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
     return UINT32_MAX;
   return idx;
 }
 
 template 
-llvm::Expected
-lldb_private::formatters::GenericNSArrayISyntheticFrontEnd<
-    D32, D64, Inline>::CalculateNumChildren() {
+uint32_t
+lldb_private::formatters::GenericNSArrayISyntheticFrontEnd::
+  CalculateNumChildren() {
   return m_data_32 ? m_data_32->used : m_data_64->used;
 }
 
@@ -685,7 +685,7 @@ template 
 lldb::ValueObjectSP
 lldb_private::formatters::GenericNSArrayISyntheticFrontEnd::
   GetChildAtIndex(uint32_t idx) {
-  if (idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx >= CalculateNumChildren())
     return lldb::ValueObjectSP();
   lldb::addr_t object_at_idx;
   if (Inline) {
@@ -719,7 +719,7 @@ lldb_private::formatters::NSArray0SyntheticFrontEnd::GetIndexOfChildWithName(
   return UINT32_MAX;
 }
 
-llvm::Expected
+uint32_t
 lldb_private::formatters::NSArray0SyntheticFrontEnd::CalculateNumChildren() {
   return 0;
 }
@@ -754,7 +754,7 @@ lldb_private::formatters::NSArray1SyntheticFrontEnd::GetIndexOfChildWithName(
   return UINT32_MAX;
 }
 
-llvm::Expected
+uint32_t
 lldb_private::formatters::NSArray1SyntheticFrontEnd::CalculateNumChildren() {
   return 1;
 }
diff --git a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
index ec6fd756394a..da94eda1529c 100644
--- a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
@@ -103,7 +103,7 @@ public:
 
   ~NSDictionaryISyntheticFrontEnd() override;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -144,7 +144,7 @@ class NSConstantDictionarySyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   NSConstantDictionarySyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -176,7 +176,7 @@ class NSCFDictionarySyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   NSCFDictionarySyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -209,7 +209,7 @@ public:
 
   ~NSDictionary1SyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -230,7 +230,7 @@ public:
 
   ~GenericNSDictionaryMSyntheticFrontEnd() override;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -262,9 +262,9 @@ namespace Foundation1100 {
     NSDictionaryMSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
     
     ~NSDictionaryMSyntheticFrontEnd() override;
-
-    llvm::Expected CalculateNumChildren() override;
-
+    
+    uint32_t CalculateNumChildren() override;
+    
     lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
     lldb::ChildCacheState Update() override;
@@ -601,13 +601,13 @@ size_t lldb_private::formatters::NSDictionaryISyntheticFrontEnd::
     GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
     return UINT32_MAX;
   return idx;
 }
 
-llvm::Expected lldb_private::formatters::
-    NSDictionaryISyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::NSDictionaryISyntheticFrontEnd::
+    CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
   return (m_data_32 ? m_data_32->_used : m_data_64->_used);
@@ -656,7 +656,7 @@ bool lldb_private::formatters::NSDictionaryISyntheticFrontEnd::
 lldb::ValueObjectSP
 lldb_private::formatters::NSDictionaryISyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
-  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
+  uint32_t num_children = CalculateNumChildren();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -739,13 +739,13 @@ size_t lldb_private::formatters::NSCFDictionarySyntheticFrontEnd::
     GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   const uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
     return UINT32_MAX;
   return idx;
 }
 
-llvm::Expected lldb_private::formatters::
-    NSCFDictionarySyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::NSCFDictionarySyntheticFrontEnd::
+    CalculateNumChildren() {
   if (!m_hashtable.IsValid())
     return 0;
   return m_hashtable.GetCount();
@@ -781,7 +781,7 @@ lldb_private::formatters::NSCFDictionarySyntheticFrontEnd::GetChildAtIndex(
   lldb::addr_t m_keys_ptr = m_hashtable.GetKeyPointer();
   lldb::addr_t m_values_ptr = m_hashtable.GetValuePointer();
 
-  const uint32_t num_children = CalculateNumChildrenIgnoringErrors();
+  const uint32_t num_children = CalculateNumChildren();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -875,13 +875,13 @@ size_t lldb_private::formatters::NSConstantDictionarySyntheticFrontEnd::
     GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
     return UINT32_MAX;
   return idx;
 }
 
-llvm::Expected lldb_private::formatters::
-    NSConstantDictionarySyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::NSConstantDictionarySyntheticFrontEnd::
+    CalculateNumChildren() {
   return m_size;
 }
 
@@ -921,7 +921,7 @@ bool lldb_private::formatters::NSConstantDictionarySyntheticFrontEnd::
 
 lldb::ValueObjectSP lldb_private::formatters::
     NSConstantDictionarySyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
-  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
+  uint32_t num_children = CalculateNumChildren();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -994,8 +994,8 @@ size_t lldb_private::formatters::NSDictionary1SyntheticFrontEnd::
   return name == g_zero ? 0 : UINT32_MAX;
 }
 
-llvm::Expected lldb_private::formatters::
-    NSDictionary1SyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::NSDictionary1SyntheticFrontEnd::
+    CalculateNumChildren() {
   return 1;
 }
 
@@ -1081,18 +1081,17 @@ size_t lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd<
     D32, D64>::GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
     return UINT32_MAX;
   return idx;
 }
 
 template 
-llvm::Expected
-lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd<
-    D32, D64>::CalculateNumChildren() {
+uint32_t
+lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
-  return (m_data_32 ? (uint32_t)m_data_32->_used : (uint32_t)m_data_64->_used);
+  return (m_data_32 ? m_data_32->_used : m_data_64->_used);
 }
 
 template 
@@ -1154,7 +1153,7 @@ lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd<
     m_values_ptr = m_data_64->_buffer + (m_ptr_size * size);
   }
 
-  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
+  uint32_t num_children = CalculateNumChildren();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -1246,13 +1245,14 @@ lldb_private::formatters::Foundation1100::
   NSDictionaryMSyntheticFrontEnd::GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
     return UINT32_MAX;
   return idx;
 }
 
-llvm::Expected lldb_private::formatters::Foundation1100::
-    NSDictionaryMSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t
+lldb_private::formatters::Foundation1100::
+  NSDictionaryMSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
   return (m_data_32 ? m_data_32->_used : m_data_64->_used);
@@ -1306,7 +1306,7 @@ lldb_private::formatters::Foundation1100::
   lldb::addr_t m_values_ptr =
       (m_data_32 ? m_data_32->_objs_addr : m_data_64->_objs_addr);
 
-  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
+  uint32_t num_children = CalculateNumChildren();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/Language/ObjC/NSError.cpp b/lldb/source/Plugins/Language/ObjC/NSError.cpp
index 5ef7edc7e80c..b034e799b716 100644
--- a/lldb/source/Plugins/Language/ObjC/NSError.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSError.cpp
@@ -116,7 +116,7 @@ public:
   // no need to delete m_child_ptr - it's kept alive by the cluster manager on
   // our behalf
 
-  llvm::Expected CalculateNumChildren() override {
+  uint32_t CalculateNumChildren() override {
     if (m_child_ptr)
       return 1;
     if (m_child_sp)
diff --git a/lldb/source/Plugins/Language/ObjC/NSException.cpp b/lldb/source/Plugins/Language/ObjC/NSException.cpp
index e7ce26ea4c6f..09d3a1b42b74 100644
--- a/lldb/source/Plugins/Language/ObjC/NSException.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSException.cpp
@@ -123,7 +123,9 @@ public:
 
   ~NSExceptionSyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override { return 4; }
+  uint32_t CalculateNumChildren() override {
+    return 4;
+  }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     switch (idx) {
diff --git a/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp b/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
index a434cee09d38..10bb907c58ed 100644
--- a/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
@@ -40,9 +40,7 @@ public:
 
   ~NSIndexPathSyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override {
-    return m_impl.GetNumIndexes();
-  }
+  uint32_t CalculateNumChildren() override { return m_impl.GetNumIndexes(); }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     return m_impl.GetIndexAtIndex(idx, m_uint_star_type);
@@ -129,7 +127,7 @@ public:
   size_t GetIndexOfChildWithName(ConstString name) override {
     const char *item_name = name.GetCString();
     uint32_t idx = ExtractIndexFromString(item_name);
-    if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+    if (idx < UINT32_MAX && idx >= CalculateNumChildren())
       return UINT32_MAX;
     return idx;
   }
diff --git a/lldb/source/Plugins/Language/ObjC/NSSet.cpp b/lldb/source/Plugins/Language/ObjC/NSSet.cpp
index 7d0a6a507211..c965a2a13400 100644
--- a/lldb/source/Plugins/Language/ObjC/NSSet.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSSet.cpp
@@ -46,7 +46,7 @@ public:
 
   ~NSSetISyntheticFrontEnd() override;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -84,7 +84,7 @@ class NSCFSetSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   NSCFSetSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -117,7 +117,7 @@ public:
 
   ~GenericNSSetMSyntheticFrontEnd() override;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -233,7 +233,7 @@ public:
 
   ~NSSetCodeRunningSyntheticFrontEnd() override;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -414,12 +414,12 @@ lldb_private::formatters::NSSetISyntheticFrontEnd::GetIndexOfChildWithName(
     ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
     return UINT32_MAX;
   return idx;
 }
 
-llvm::Expected
+uint32_t
 lldb_private::formatters::NSSetISyntheticFrontEnd::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
@@ -468,7 +468,7 @@ bool lldb_private::formatters::NSSetISyntheticFrontEnd::MightHaveChildren() {
 lldb::ValueObjectSP
 lldb_private::formatters::NSSetISyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
-  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
+  uint32_t num_children = CalculateNumChildren();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -551,12 +551,12 @@ lldb_private::formatters::NSCFSetSyntheticFrontEnd::GetIndexOfChildWithName(
     ConstString name) {
   const char *item_name = name.GetCString();
   const uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
     return UINT32_MAX;
   return idx;
 }
 
-llvm::Expected
+uint32_t
 lldb_private::formatters::NSCFSetSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_hashtable.IsValid())
     return 0;
@@ -591,7 +591,7 @@ lldb_private::formatters::NSCFSetSyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
   lldb::addr_t m_values_ptr = m_hashtable.GetValuePointer();
 
-  const uint32_t num_children = CalculateNumChildrenIgnoringErrors();
+  const uint32_t num_children = CalculateNumChildren();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -691,18 +691,18 @@ lldb_private::formatters::
     ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
     return UINT32_MAX;
   return idx;
 }
 
 template 
-llvm::Expected
-lldb_private::formatters::GenericNSSetMSyntheticFrontEnd<
-    D32, D64>::CalculateNumChildren() {
+uint32_t
+lldb_private::formatters::
+  GenericNSSetMSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
-  return (m_data_32 ? (uint32_t)m_data_32->_used : (uint32_t)m_data_64->_used);
+  return (m_data_32 ? m_data_32->_used : m_data_64->_used);
 }
 
 template 
@@ -753,7 +753,7 @@ lldb_private::formatters::
   lldb::addr_t m_objs_addr =
       (m_data_32 ? m_data_32->_objs_addr : m_data_64->_objs_addr);
 
-  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
+  uint32_t num_children = CalculateNumChildren();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
index 9434376f7d9e..f08f9f0f815d 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
@@ -539,8 +539,7 @@ ThreadSP AppleObjCRuntime::GetBacktraceThreadFromException(
     return object;
   };
 
-  for (size_t idx = 0; idx < reserved_dict->GetNumChildrenIgnoringErrors();
-       idx++) {
+  for (size_t idx = 0; idx < reserved_dict->GetNumChildren(); idx++) {
     ValueObjectSP dict_entry = reserved_dict->GetChildAtIndex(idx);
 
     DataExtractor data;
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index c02b08cb4782..51ab13108feb 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -5263,10 +5263,9 @@ GetDynamicArrayInfo(TypeSystemClang &ast, SymbolFile *sym_file,
   return std::nullopt;
 }
 
-llvm::Expected
-TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
-                                bool omit_empty_base_classes,
-                                const ExecutionContext *exe_ctx) {
+uint32_t TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
+                                         bool omit_empty_base_classes,
+                                         const ExecutionContext *exe_ctx) {
   if (!type)
     return 0;
 
@@ -5362,13 +5361,9 @@ TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
     CompilerType pointee_clang_type(GetPointeeType(type));
 
     uint32_t num_pointee_children = 0;
-    if (pointee_clang_type.IsAggregateType()) {
-      auto num_children_or_err =
+    if (pointee_clang_type.IsAggregateType())
+      num_pointee_children =
           pointee_clang_type.GetNumChildren(omit_empty_base_classes, exe_ctx);
-      if (!num_children_or_err)
-        return num_children_or_err;
-      num_pointee_children = *num_children_or_err;
-    }
     // If this type points to a simple type, then it has 1 child
     if (num_pointee_children == 0)
       num_children = 1;
@@ -5402,13 +5397,9 @@ TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
     clang::QualType pointee_type(pointer_type->getPointeeType());
     CompilerType pointee_clang_type(GetType(pointee_type));
     uint32_t num_pointee_children = 0;
-    if (pointee_clang_type.IsAggregateType()) {
-      auto num_children_or_err =
+    if (pointee_clang_type.IsAggregateType())
+      num_pointee_children =
           pointee_clang_type.GetNumChildren(omit_empty_base_classes, exe_ctx);
-      if (!num_children_or_err)
-        return num_children_or_err;
-      num_pointee_children = *num_children_or_err;
-    }
     if (num_pointee_children == 0) {
       // We have a pointer to a pointee type that claims it has no children. We
       // will want to look at
@@ -6117,15 +6108,8 @@ CompilerType TypeSystemClang::GetChildCompilerTypeAtIndex(
   child_is_base_class = false;
   language_flags = 0;
 
-  auto num_children_or_err =
-      GetNumChildren(type, omit_empty_base_classes, exe_ctx);
-  if (!num_children_or_err) {
-    LLDB_LOG_ERRORV(GetLog(LLDBLog::Types), num_children_or_err.takeError(),
-                    "{0}");
-    return {};
-  }
-
-  const bool idx_is_valid = idx < *num_children_or_err;
+  const bool idx_is_valid =
+      idx < GetNumChildren(type, omit_empty_base_classes, exe_ctx);
   int32_t bit_offset;
   switch (parent_type_class) {
   case clang::Type::Builtin:
@@ -6281,10 +6265,8 @@ CompilerType TypeSystemClang::GetChildCompilerTypeAtIndex(
               CompilerType base_class_clang_type =
                   GetType(getASTContext().getObjCInterfaceType(
                       superclass_interface_decl));
-              if (llvm::expectedToStdOptional(
-                      base_class_clang_type.GetNumChildren(
-                          omit_empty_base_classes, exe_ctx))
-                      .value_or(0) > 0) {
+              if (base_class_clang_type.GetNumChildren(omit_empty_base_classes,
+                                                       exe_ctx) > 0) {
                 if (idx == 0) {
                   clang::QualType ivar_qual_type(
                       getASTContext().getObjCInterfaceType(
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
index 05c303baa416..a73164895baa 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
@@ -833,10 +833,9 @@ public:
   GetTypeBitAlign(lldb::opaque_compiler_type_t type,
                   ExecutionContextScope *exe_scope) override;
 
-  llvm::Expected
-  GetNumChildren(lldb::opaque_compiler_type_t type,
-                 bool omit_empty_base_classes,
-                 const ExecutionContext *exe_ctx) override;
+  uint32_t GetNumChildren(lldb::opaque_compiler_type_t type,
+                          bool omit_empty_base_classes,
+                          const ExecutionContext *exe_ctx) override;
 
   CompilerType GetBuiltinTypeByName(ConstString name) override;
 
diff --git a/lldb/source/Symbol/CompilerType.cpp b/lldb/source/Symbol/CompilerType.cpp
index 85dd2d841a5a..76b79daa6ac1 100644
--- a/lldb/source/Symbol/CompilerType.cpp
+++ b/lldb/source/Symbol/CompilerType.cpp
@@ -770,9 +770,8 @@ lldb::Format CompilerType::GetFormat() const {
   return lldb::eFormatDefault;
 }
 
-llvm::Expected
-CompilerType::GetNumChildren(bool omit_empty_base_classes,
-                             const ExecutionContext *exe_ctx) const {
+uint32_t CompilerType::GetNumChildren(bool omit_empty_base_classes,
+                                      const ExecutionContext *exe_ctx) const {
   if (IsValid())
     if (auto type_system_sp = GetTypeSystem())
       return type_system_sp->GetNumChildren(m_type, omit_empty_base_classes,
diff --git a/lldb/source/Symbol/Type.cpp b/lldb/source/Symbol/Type.cpp
index 44a24d7178f5..6069d066eaf6 100644
--- a/lldb/source/Symbol/Type.cpp
+++ b/lldb/source/Symbol/Type.cpp
@@ -488,7 +488,7 @@ std::optional Type::GetByteSize(ExecutionContextScope *exe_scope) {
   return {};
 }
 
-llvm::Expected Type::GetNumChildren(bool omit_empty_base_classes) {
+uint32_t Type::GetNumChildren(bool omit_empty_base_classes) {
   return GetForwardCompilerType().GetNumChildren(omit_empty_base_classes, nullptr);
 }
 
diff --git a/lldb/source/Symbol/Variable.cpp b/lldb/source/Symbol/Variable.cpp
index 90edede0f065..a33c3433d9e2 100644
--- a/lldb/source/Symbol/Variable.cpp
+++ b/lldb/source/Symbol/Variable.cpp
@@ -27,8 +27,6 @@
 #include "lldb/Target/StackFrame.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Target/Thread.h"
-#include "lldb/Utility/LLDBLog.h"
-#include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegularExpression.h"
 #include "lldb/Utility/Stream.h"
 
@@ -570,9 +568,7 @@ static void PrivateAutoComplete(
       case eTypeClassObjCObjectPointer:
       case eTypeClassPointer: {
         bool omit_empty_base_classes = true;
-        if (llvm::expectedToStdOptional(
-                compiler_type.GetNumChildren(omit_empty_base_classes, nullptr))
-                .value_or(0))
+        if (compiler_type.GetNumChildren(omit_empty_base_classes, nullptr) > 0)
           request.AddCompletion((prefix_path + "->").str());
         else {
           request.AddCompletion(prefix_path.str());
diff --git a/lldb/source/Target/StackFrame.cpp b/lldb/source/Target/StackFrame.cpp
index c29a71d92572..50cf01e63cd4 100644
--- a/lldb/source/Target/StackFrame.cpp
+++ b/lldb/source/Target/StackFrame.cpp
@@ -857,11 +857,10 @@ ValueObjectSP StackFrame::GetValueForVariableExpressionPath(
                   "\"(%s) %s\" is not an array type",
                   valobj_sp->GetTypeName().AsCString(""),
                   var_expr_path_strm.GetData());
-            } else if (static_cast(child_index) >=
-                       synthetic
-                           ->GetNumChildrenIgnoringErrors() /* synthetic does
-                                                                not have that
-                                                                many values */) {
+            } else if (
+                static_cast(child_index) >=
+                synthetic
+                    ->GetNumChildren() /* synthetic does not have that many values */) {
               valobj_sp->GetExpressionPath(var_expr_path_strm);
               error.SetErrorStringWithFormat(
                   "array index %ld is not valid for \"(%s) %s\"", child_index,
@@ -930,9 +929,10 @@ ValueObjectSP StackFrame::GetValueForVariableExpressionPath(
                 "\"(%s) %s\" is not an array type",
                 valobj_sp->GetTypeName().AsCString(""),
                 var_expr_path_strm.GetData());
-          } else if (static_cast(child_index) >=
-                     synthetic->GetNumChildrenIgnoringErrors() /* synthetic
-                                     does not have that many values */) {
+          } else if (
+              static_cast(child_index) >=
+              synthetic
+                  ->GetNumChildren() /* synthetic does not have that many values */) {
             valobj_sp->GetExpressionPath(var_expr_path_strm);
             error.SetErrorStringWithFormat(
                 "array index %ld is not valid for \"(%s) %s\"", child_index,
@@ -1397,8 +1397,7 @@ ValueObjectSP GetValueForOffset(StackFrame &frame, ValueObjectSP &parent,
     return parent;
   }
 
-  for (int ci = 0, ce = parent->GetNumChildrenIgnoringErrors(); ci != ce;
-       ++ci) {
+  for (int ci = 0, ce = parent->GetNumChildren(); ci != ce; ++ci) {
     ValueObjectSP child_sp = parent->GetChildAtIndex(ci);
 
     if (!child_sp) {
-- 
GitLab


From 08ddd2ce4048b0adbddc3ab9b770ccf43f81d65d Mon Sep 17 00:00:00 2001
From: Teresa Johnson 
Date: Fri, 8 Mar 2024 12:27:46 -0800
Subject: [PATCH 659/929] [PGO] Add support for writing previous indexed format
 (#84505)

Enable temporary support to ease use of new llvm-profdata with slightly
older indexed profiles after 16e74fd48988ac95551d0f64e1b36f78a82a89a2,
which bumped the indexed format for type profiling.
---
 .../llvm/ProfileData/InstrProfWriter.h        |  10 +-
 llvm/lib/ProfileData/InstrProfWriter.cpp      | 113 +++++++++++-------
 .../tools/llvm-profdata/profile-version.test  |   9 ++
 llvm/tools/llvm-profdata/llvm-profdata.cpp    |  13 +-
 4 files changed, 102 insertions(+), 43 deletions(-)
 create mode 100644 llvm/test/tools/llvm-profdata/profile-version.test

diff --git a/llvm/include/llvm/ProfileData/InstrProfWriter.h b/llvm/include/llvm/ProfileData/InstrProfWriter.h
index 7a806fd7fcf3..f70574d1f756 100644
--- a/llvm/include/llvm/ProfileData/InstrProfWriter.h
+++ b/llvm/include/llvm/ProfileData/InstrProfWriter.h
@@ -68,10 +68,18 @@ private:
   // Use raw pointer here for the incomplete type object.
   InstrProfRecordWriterTrait *InfoObj;
 
+  // Temporary support for writing the previous version of the format, to enable
+  // some forward compatibility. Currently this suppresses the writing of the
+  // new vtable names section and header fields.
+  // TODO: Consider enabling this with future version changes as well, to ease
+  // deployment of newer versions of llvm-profdata.
+  bool WritePrevVersion = false;
+
 public:
   InstrProfWriter(bool Sparse = false,
                   uint64_t TemporalProfTraceReservoirSize = 0,
-                  uint64_t MaxTemporalProfTraceLength = 0);
+                  uint64_t MaxTemporalProfTraceLength = 0,
+                  bool WritePrevVersion = false);
   ~InstrProfWriter();
 
   StringMap &getProfileData() { return FunctionData; }
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index 3e0a0e0d7011..e757c735719b 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -181,10 +181,12 @@ public:
 
 InstrProfWriter::InstrProfWriter(bool Sparse,
                                  uint64_t TemporalProfTraceReservoirSize,
-                                 uint64_t MaxTemporalProfTraceLength)
+                                 uint64_t MaxTemporalProfTraceLength,
+                                 bool WritePrevVersion)
     : Sparse(Sparse), MaxTemporalProfTraceLength(MaxTemporalProfTraceLength),
       TemporalProfTraceReservoirSize(TemporalProfTraceReservoirSize),
-      InfoObj(new InstrProfRecordWriterTrait()) {}
+      InfoObj(new InstrProfRecordWriterTrait()),
+      WritePrevVersion(WritePrevVersion) {}
 
 InstrProfWriter::~InstrProfWriter() { delete InfoObj; }
 
@@ -432,7 +434,13 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
   // Write the header.
   IndexedInstrProf::Header Header;
   Header.Magic = IndexedInstrProf::Magic;
-  Header.Version = IndexedInstrProf::ProfVersion::CurrentVersion;
+  Header.Version = WritePrevVersion
+                       ? IndexedInstrProf::ProfVersion::Version11
+                       : IndexedInstrProf::ProfVersion::CurrentVersion;
+  // The WritePrevVersion handling will either need to be removed or updated
+  // if the version is advanced beyond 12.
+  assert(IndexedInstrProf::ProfVersion::CurrentVersion ==
+         IndexedInstrProf::ProfVersion::Version12);
   if (static_cast(ProfileKind & InstrProfKind::IRInstrumentation))
     Header.Version |= VARIANT_MASK_IR_PROF;
   if (static_cast(ProfileKind & InstrProfKind::ContextSensitive))
@@ -484,7 +492,8 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
   OS.write(0);
 
   uint64_t VTableNamesOffset = OS.tell();
-  OS.write(0);
+  if (!WritePrevVersion)
+    OS.write(0);
 
   // Reserve space to write profile summary data.
   uint32_t NumEntries = ProfileSummaryBuilder::DefaultCutoffs.size();
@@ -608,27 +617,29 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
 
   uint64_t VTableNamesSectionStart = OS.tell();
 
-  // Use a dummy (and uncompressed) string as compressed vtable names and get
-  // the necessary profile format change in place for version 12.
-  // TODO: Store the list of vtable names in InstrProfWriter and use the
-  // real compressed name.
-  std::string CompressedVTableNames = "VTableNames";
+  if (!WritePrevVersion) {
+    // Use a dummy (and uncompressed) string as compressed vtable names and get
+    // the necessary profile format change in place for version 12.
+    // TODO: Store the list of vtable names in InstrProfWriter and use the
+    // real compressed name.
+    std::string CompressedVTableNames = "VTableNames";
 
-  uint64_t CompressedStringLen = CompressedVTableNames.length();
+    uint64_t CompressedStringLen = CompressedVTableNames.length();
 
-  // Record the length of compressed string.
-  OS.write(CompressedStringLen);
+    // Record the length of compressed string.
+    OS.write(CompressedStringLen);
 
-  // Write the chars in compressed strings.
-  for (auto &c : CompressedVTableNames)
-    OS.writeByte(static_cast(c));
+    // Write the chars in compressed strings.
+    for (auto &c : CompressedVTableNames)
+      OS.writeByte(static_cast(c));
 
-  // Pad up to a multiple of 8.
-  // InstrProfReader would read bytes according to 'CompressedStringLen'.
-  uint64_t PaddedLength = alignTo(CompressedStringLen, 8);
+    // Pad up to a multiple of 8.
+    // InstrProfReader would read bytes according to 'CompressedStringLen'.
+    uint64_t PaddedLength = alignTo(CompressedStringLen, 8);
 
-  for (uint64_t K = CompressedStringLen; K < PaddedLength; K++) {
-    OS.writeByte(0);
+    for (uint64_t K = CompressedStringLen; K < PaddedLength; K++) {
+      OS.writeByte(0);
+    }
   }
 
   uint64_t TemporalProfTracesSectionStart = 0;
@@ -662,26 +673,48 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
   }
   InfoObj->CSSummaryBuilder = nullptr;
 
-  // Now do the final patch:
-  PatchItem PatchItems[] = {
-      // Patch the Header.HashOffset field.
-      {HashTableStartFieldOffset, &HashTableStart, 1},
-      // Patch the Header.MemProfOffset (=0 for profiles without MemProf
-      // data).
-      {MemProfSectionOffset, &MemProfSectionStart, 1},
-      // Patch the Header.BinaryIdSectionOffset.
-      {BinaryIdSectionOffset, &BinaryIdSectionStart, 1},
-      // Patch the Header.TemporalProfTracesOffset (=0 for profiles without
-      // traces).
-      {TemporalProfTracesOffset, &TemporalProfTracesSectionStart, 1},
-      {VTableNamesOffset, &VTableNamesSectionStart, 1},
-      // Patch the summary data.
-      {SummaryOffset, reinterpret_cast(TheSummary.get()),
-       (int)(SummarySize / sizeof(uint64_t))},
-      {CSSummaryOffset, reinterpret_cast(TheCSSummary.get()),
-       (int)CSSummarySize}};
-
-  OS.patch(PatchItems, std::size(PatchItems));
+  if (!WritePrevVersion) {
+    // Now do the final patch:
+    PatchItem PatchItems[] = {
+        // Patch the Header.HashOffset field.
+        {HashTableStartFieldOffset, &HashTableStart, 1},
+        // Patch the Header.MemProfOffset (=0 for profiles without MemProf
+        // data).
+        {MemProfSectionOffset, &MemProfSectionStart, 1},
+        // Patch the Header.BinaryIdSectionOffset.
+        {BinaryIdSectionOffset, &BinaryIdSectionStart, 1},
+        // Patch the Header.TemporalProfTracesOffset (=0 for profiles without
+        // traces).
+        {TemporalProfTracesOffset, &TemporalProfTracesSectionStart, 1},
+        {VTableNamesOffset, &VTableNamesSectionStart, 1},
+        // Patch the summary data.
+        {SummaryOffset, reinterpret_cast(TheSummary.get()),
+         (int)(SummarySize / sizeof(uint64_t))},
+        {CSSummaryOffset, reinterpret_cast(TheCSSummary.get()),
+         (int)CSSummarySize}};
+
+    OS.patch(PatchItems, std::size(PatchItems));
+  } else {
+    // Now do the final patch:
+    PatchItem PatchItems[] = {
+        // Patch the Header.HashOffset field.
+        {HashTableStartFieldOffset, &HashTableStart, 1},
+        // Patch the Header.MemProfOffset (=0 for profiles without MemProf
+        // data).
+        {MemProfSectionOffset, &MemProfSectionStart, 1},
+        // Patch the Header.BinaryIdSectionOffset.
+        {BinaryIdSectionOffset, &BinaryIdSectionStart, 1},
+        // Patch the Header.TemporalProfTracesOffset (=0 for profiles without
+        // traces).
+        {TemporalProfTracesOffset, &TemporalProfTracesSectionStart, 1},
+        // Patch the summary data.
+        {SummaryOffset, reinterpret_cast(TheSummary.get()),
+         (int)(SummarySize / sizeof(uint64_t))},
+        {CSSummaryOffset, reinterpret_cast(TheCSSummary.get()),
+         (int)CSSummarySize}};
+
+    OS.patch(PatchItems, std::size(PatchItems));
+  }
 
   for (const auto &I : FunctionData)
     for (const auto &F : I.getValue())
diff --git a/llvm/test/tools/llvm-profdata/profile-version.test b/llvm/test/tools/llvm-profdata/profile-version.test
new file mode 100644
index 000000000000..cb68a648d5e5
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/profile-version.test
@@ -0,0 +1,9 @@
+Test the profile version.
+
+RUN: llvm-profdata merge -o %t.profdata %p/Inputs/basic.proftext
+RUN: llvm-profdata show --profile-version %t.profdata | FileCheck %s
+CHECK: Profile version: 12
+
+RUN: llvm-profdata merge -o %t.prev.profdata %p/Inputs/basic.proftext --write-prev-version
+RUN: llvm-profdata show --profile-version %t.prev.profdata | FileCheck %s --check-prefix=PREV
+PREV: Profile version: 11
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 8400b0769944..3a7bd061d3d2 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -291,6 +291,15 @@ cl::opt DropProfileSymbolList(
     cl::desc("Drop the profile symbol list when merging AutoFDO profiles "
              "(only meaningful for -sample)"));
 
+// Temporary support for writing the previous version of the format, to enable
+// some forward compatibility.
+// TODO: Consider enabling this with future version changes as well, to ease
+// deployment of newer versions of llvm-profdata.
+cl::opt DoWritePrevVersion(
+    "write-prev-version", cl::init(false), cl::Hidden,
+    cl::desc("Write the previous version of indexed format, to enable "
+             "some forward compatibility."));
+
 // Options specific to overlap subcommand.
 cl::opt BaseFilename(cl::Positional, cl::Required,
                                   cl::desc(""),
@@ -579,8 +588,8 @@ struct WriterContext {
   WriterContext(bool IsSparse, std::mutex &ErrLock,
                 SmallSet &WriterErrorCodes,
                 uint64_t ReservoirSize = 0, uint64_t MaxTraceLength = 0)
-      : Writer(IsSparse, ReservoirSize, MaxTraceLength), ErrLock(ErrLock),
-        WriterErrorCodes(WriterErrorCodes) {}
+      : Writer(IsSparse, ReservoirSize, MaxTraceLength, DoWritePrevVersion),
+        ErrLock(ErrLock), WriterErrorCodes(WriterErrorCodes) {}
 };
 
 /// Computer the overlap b/w profile BaseFilename and TestFileName,
-- 
GitLab


From 05280b582aa73ce0f41bb47aea15818658f32929 Mon Sep 17 00:00:00 2001
From: David CARLIER 
Date: Fri, 8 Mar 2024 20:34:43 +0000
Subject: [PATCH 660/929] [OpenMP] Implements __kmp_is_address_mapped for
 Solaris/Illumos. (#82930)

Also fixing OpenMP build itself for this platform.
---
 openmp/runtime/cmake/LibompHandleFlags.cmake |  5 +-
 openmp/runtime/src/z_Linux_util.cpp          | 72 ++++++++++++++++++--
 2 files changed, 72 insertions(+), 5 deletions(-)

diff --git a/openmp/runtime/cmake/LibompHandleFlags.cmake b/openmp/runtime/cmake/LibompHandleFlags.cmake
index eed98036dd51..1aba1fbea829 100644
--- a/openmp/runtime/cmake/LibompHandleFlags.cmake
+++ b/openmp/runtime/cmake/LibompHandleFlags.cmake
@@ -147,8 +147,11 @@ function(libomp_get_libflags libflags)
     if (${CMAKE_SYSTEM_NAME} STREQUAL "DragonFly")
       libomp_append(libflags_local "-lkvm")
     endif()
-  elseif(${CMAKE_SYSTEM_NAME} MATCHES "Linux|NetBSD|Solaris")
+  elseif(${CMAKE_SYSTEM_NAME} MATCHES "Linux|NetBSD|SunOS")
     libomp_append(libflags_local -lm)
+    if (${CMAKE_SYSTEM_NAME} STREQUAL "SunOS")
+      libomp_append(libflags_local "-lproc")
+    endif()
   endif()
   set(libflags_local ${libflags_local} ${LIBOMP_LIBFLAGS})
   libomp_setup_flags(libflags_local)
diff --git a/openmp/runtime/src/z_Linux_util.cpp b/openmp/runtime/src/z_Linux_util.cpp
index 3636266677d9..a8e5a9e6bbb0 100644
--- a/openmp/runtime/src/z_Linux_util.cpp
+++ b/openmp/runtime/src/z_Linux_util.cpp
@@ -66,6 +66,9 @@
 #include 
 #include 
 #elif KMP_OS_SOLARIS
+#include 
+#include 
+#include 
 #include 
 #endif
 
@@ -418,7 +421,7 @@ void __kmp_terminate_thread(int gtid) {
   KMP_YIELD(TRUE);
 } //
 
-/* Set thread stack info according to values returned by pthread_getattr_np().
+/* Set thread stack info.
    If values are unreasonable, assume call failed and use incremental stack
    refinement method instead. Returns TRUE if the stack parameters could be
    determined exactly, FALSE if incremental refinement is necessary. */
@@ -426,7 +429,6 @@ static kmp_int32 __kmp_set_stack_info(int gtid, kmp_info_t *th) {
   int stack_data;
 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD ||     \
     KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX
-  pthread_attr_t attr;
   int status;
   size_t size = 0;
   void *addr = 0;
@@ -436,6 +438,19 @@ static kmp_int32 __kmp_set_stack_info(int gtid, kmp_info_t *th) {
      pthread_attr_getstack may cause thread gtid aliasing */
   if (!KMP_UBER_GTID(gtid)) {
 
+#if KMP_OS_SOLARIS
+    stack_t s;
+    if ((status = thr_stksegment(&s)) < 0) {
+      KMP_CHECK_SYSFAIL("thr_stksegment", status);
+    }
+
+    addr = s.ss_sp;
+    size = s.ss_size;
+    KA_TRACE(60, ("__kmp_set_stack_info: T#%d thr_stksegment returned size:"
+                  " %lu, low addr: %p\n",
+                  gtid, size, addr));
+#else
+    pthread_attr_t attr;
     /* Fetch the real thread attributes */
     status = pthread_attr_init(&attr);
     KMP_CHECK_SYSFAIL("pthread_attr_init", status);
@@ -454,6 +469,7 @@ static kmp_int32 __kmp_set_stack_info(int gtid, kmp_info_t *th) {
               gtid, size, addr));
     status = pthread_attr_destroy(&attr);
     KMP_CHECK_SYSFAIL("pthread_attr_destroy", status);
+#endif
   }
 
   if (size != 0 && addr != 0) { // was stack parameter determination successful?
@@ -2175,6 +2191,54 @@ int __kmp_is_address_mapped(void *addr) {
   }
 
   kvm_close(fd);
+#elif KMP_OS_SOLARIS
+  prmap_t *cur, *map;
+  void *buf;
+  uintptr_t uaddr;
+  ssize_t rd;
+  int err;
+  int file;
+
+  pid_t pid = getpid();
+  struct ps_prochandle *fd = Pgrab(pid, PGRAB_RDONLY, &err);
+  ;
+
+  if (!fd) {
+    return 0;
+  }
+
+  char *name = __kmp_str_format("/proc/%d/map", pid);
+  size_t sz = (1 << 20);
+  file = open(name, O_RDONLY);
+  if (file == -1) {
+    KMP_INTERNAL_FREE(name);
+    return 0;
+  }
+
+  buf = kmpc_malloc(sz);
+
+  while (sz > 0 && (rd = pread(file, buf, sz, 0)) == sz) {
+    void *newbuf;
+    sz <<= 1;
+    newbuf = kmpc_realloc(buf, sz);
+    buf = newbuf;
+  }
+
+  map = reinterpret_cast(buf);
+  uaddr = reinterpret_cast(addr);
+
+  for (cur = map; rd > 0; cur++, rd = -sizeof(*map)) {
+    if ((uaddr >= cur->pr_vaddr) && (uaddr < cur->pr_vaddr)) {
+      if ((cur->pr_mflags & MA_READ) != 0 && (cur->pr_mflags & MA_WRITE) != 0) {
+        found = 1;
+        break;
+      }
+    }
+  }
+
+  kmpc_free(map);
+  close(file);
+  KMP_INTERNAL_FREE(name);
 #elif KMP_OS_DARWIN
 
   /* On OS X*, /proc pseudo filesystem is not available. Try to read memory
@@ -2253,9 +2317,9 @@ int __kmp_is_address_mapped(void *addr) {
   }
 #elif KMP_OS_WASI
   found = (int)addr < (__builtin_wasm_memory_size(0) * PAGESIZE);
-#elif KMP_OS_SOLARIS || KMP_OS_AIX
+#elif KMP_OS_AIX
 
-  // FIXME(Solaris, AIX): Implement this
+  // FIXME(AIX): Implement this
   found = 1;
 
 #else
-- 
GitLab


From 755b439694432d4f68e20e979b479cbc30602bb1 Mon Sep 17 00:00:00 2001
From: yingopq <115543042+yingopq@users.noreply.github.com>
Date: Sat, 9 Mar 2024 04:41:31 +0800
Subject: [PATCH 661/929] [Mips] Fix missing sign extension in expansion of
 sub-word atomic max (#77072)

Add sign extension "SEB/SEH" before compare.

Fix #61881
---
 llvm/lib/Target/Mips/MipsExpandPseudo.cpp |  51 +-
 llvm/test/CodeGen/Mips/atomic-min-max.ll  | 655 ++++++++++++++++++----
 2 files changed, 577 insertions(+), 129 deletions(-)

diff --git a/llvm/lib/Target/Mips/MipsExpandPseudo.cpp b/llvm/lib/Target/Mips/MipsExpandPseudo.cpp
index 2c2554b5b4bc..bded59439a73 100644
--- a/llvm/lib/Target/Mips/MipsExpandPseudo.cpp
+++ b/llvm/lib/Target/Mips/MipsExpandPseudo.cpp
@@ -388,18 +388,32 @@ bool MipsExpandPseudo::expandAtomicBinOpSubword(
     Opcode = Mips::XOR;
     break;
   case Mips::ATOMIC_LOAD_UMIN_I8_POSTRA:
+    IsUnsigned = true;
+    IsMin = true;
+    break;
   case Mips::ATOMIC_LOAD_UMIN_I16_POSTRA:
     IsUnsigned = true;
-    [[fallthrough]];
+    IsMin = true;
+    break;
   case Mips::ATOMIC_LOAD_MIN_I8_POSTRA:
+    SEOp = Mips::SEB;
+    IsMin = true;
+    break;
   case Mips::ATOMIC_LOAD_MIN_I16_POSTRA:
     IsMin = true;
     break;
   case Mips::ATOMIC_LOAD_UMAX_I8_POSTRA:
+    IsUnsigned = true;
+    IsMax = true;
+    break;
   case Mips::ATOMIC_LOAD_UMAX_I16_POSTRA:
     IsUnsigned = true;
-    [[fallthrough]];
+    IsMax = true;
+    break;
   case Mips::ATOMIC_LOAD_MAX_I8_POSTRA:
+    SEOp = Mips::SEB;
+    IsMax = true;
+    break;
   case Mips::ATOMIC_LOAD_MAX_I16_POSTRA:
     IsMax = true;
     break;
@@ -461,14 +475,33 @@ bool MipsExpandPseudo::expandAtomicBinOpSubword(
 
     // For little endian we need to clear uninterested bits.
     if (STI->isLittle()) {
-      // and OldVal, OldVal, Mask
-      // and Incr, Incr, Mask
-      BuildMI(loopMBB, DL, TII->get(Mips::AND), OldVal)
-          .addReg(OldVal)
-          .addReg(Mask);
-      BuildMI(loopMBB, DL, TII->get(Mips::AND), Incr).addReg(Incr).addReg(Mask);
+      if (!IsUnsigned) {
+        BuildMI(loopMBB, DL, TII->get(Mips::SRAV), OldVal)
+            .addReg(OldVal)
+            .addReg(ShiftAmnt);
+        BuildMI(loopMBB, DL, TII->get(Mips::SRAV), Incr)
+            .addReg(Incr)
+            .addReg(ShiftAmnt);
+        if (STI->hasMips32r2()) {
+          BuildMI(loopMBB, DL, TII->get(SEOp), OldVal).addReg(OldVal);
+          BuildMI(loopMBB, DL, TII->get(SEOp), Incr).addReg(Incr);
+        } else {
+          const unsigned ShiftImm = SEOp == Mips::SEH ? 16 : 24;
+          BuildMI(loopMBB, DL, TII->get(Mips::SLL), OldVal)
+              .addReg(OldVal, RegState::Kill)
+              .addImm(ShiftImm);
+          BuildMI(loopMBB, DL, TII->get(Mips::SRA), OldVal)
+              .addReg(OldVal, RegState::Kill)
+              .addImm(ShiftImm);
+          BuildMI(loopMBB, DL, TII->get(Mips::SLL), Incr)
+              .addReg(Incr, RegState::Kill)
+              .addImm(ShiftImm);
+          BuildMI(loopMBB, DL, TII->get(Mips::SRA), Incr)
+              .addReg(Incr, RegState::Kill)
+              .addImm(ShiftImm);
+        }
+      }
     }
-
     // unsigned: sltu Scratch4, oldVal, Incr
     // signed:   slt Scratch4, oldVal, Incr
     BuildMI(loopMBB, DL, TII->get(SLTScratch4), Scratch4)
diff --git a/llvm/test/CodeGen/Mips/atomic-min-max.ll b/llvm/test/CodeGen/Mips/atomic-min-max.ll
index f953c885ea73..bc3643f3947a 100644
--- a/llvm/test/CodeGen/Mips/atomic-min-max.ll
+++ b/llvm/test/CodeGen/Mips/atomic-min-max.ll
@@ -3,6 +3,7 @@
 ; RUN: llc -march=mips -O0 -mcpu=mips32r6 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPSR6
 ; RUN: llc -march=mips -O0 -mcpu=mips32r2 -mattr=+micromips -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MM
 ; RUN: llc -march=mips -O0 -mcpu=mips32r6 -mattr=+micromips -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MMR6
+; RUN: llc -march=mipsel -O0 -mcpu=mips32 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS32
 ; RUN: llc -march=mipsel -O0 -mcpu=mips32r2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPSEL
 ; RUN: llc -march=mipsel -O0 -mcpu=mips32r6 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPSELR6
 ; RUN: llc -march=mipsel -O0 -mcpu=mips32r2 -mattr=+micromips -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MMEL
@@ -77,6 +78,23 @@ define i32 @test_max_32(ptr nocapture %ptr, i32 signext %val) {
 ; MMR6-NEXT:    sync
 ; MMR6-NEXT:    jrc $ra
 ;
+; MIPS32-LABEL: test_max_32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:  $BB0_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($4)
+; MIPS32-NEXT:    slt $3, $2, $5
+; MIPS32-NEXT:    move $1, $2
+; MIPS32-NEXT:    movn $1, $5, $3
+; MIPS32-NEXT:    sc $1, 0($4)
+; MIPS32-NEXT:    beqz $1, $BB0_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
 ; MIPSEL-LABEL: test_max_32:
 ; MIPSEL:       # %bb.0: # %entry
 ; MIPSEL-NEXT:    sync
@@ -280,6 +298,23 @@ define i32 @test_min_32(ptr nocapture %ptr, i32 signext %val) {
 ; MMR6-NEXT:    sync
 ; MMR6-NEXT:    jrc $ra
 ;
+; MIPS32-LABEL: test_min_32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:  $BB1_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($4)
+; MIPS32-NEXT:    slt $3, $2, $5
+; MIPS32-NEXT:    move $1, $2
+; MIPS32-NEXT:    movz $1, $5, $3
+; MIPS32-NEXT:    sc $1, 0($4)
+; MIPS32-NEXT:    beqz $1, $BB1_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
 ; MIPSEL-LABEL: test_min_32:
 ; MIPSEL:       # %bb.0: # %entry
 ; MIPSEL-NEXT:    sync
@@ -483,6 +518,23 @@ define i32 @test_umax_32(ptr nocapture %ptr, i32 signext %val) {
 ; MMR6-NEXT:    sync
 ; MMR6-NEXT:    jrc $ra
 ;
+; MIPS32-LABEL: test_umax_32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:  $BB2_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($4)
+; MIPS32-NEXT:    sltu $3, $2, $5
+; MIPS32-NEXT:    move $1, $2
+; MIPS32-NEXT:    movn $1, $5, $3
+; MIPS32-NEXT:    sc $1, 0($4)
+; MIPS32-NEXT:    beqz $1, $BB2_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
 ; MIPSEL-LABEL: test_umax_32:
 ; MIPSEL:       # %bb.0: # %entry
 ; MIPSEL-NEXT:    sync
@@ -686,6 +738,23 @@ define i32 @test_umin_32(ptr nocapture %ptr, i32 signext %val) {
 ; MMR6-NEXT:    sync
 ; MMR6-NEXT:    jrc $ra
 ;
+; MIPS32-LABEL: test_umin_32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:  $BB3_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($4)
+; MIPS32-NEXT:    sltu $3, $2, $5
+; MIPS32-NEXT:    move $1, $2
+; MIPS32-NEXT:    movz $1, $5, $3
+; MIPS32-NEXT:    sc $1, 0($4)
+; MIPS32-NEXT:    beqz $1, $BB3_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
 ; MIPSEL-LABEL: test_umin_32:
 ; MIPSEL:       # %bb.0: # %entry
 ; MIPSEL-NEXT:    sync
@@ -980,6 +1049,53 @@ define i16 @test_max_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMR6-NEXT:    sync
 ; MMR6-NEXT:    addiu $sp, $sp, 8
 ; MMR6-NEXT:    jrc $ra
+;
+; MIPS32-LABEL: test_max_16:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -8
+; MIPS32-NEXT:    .cfi_def_cfa_offset 8
+; MIPS32-NEXT:    # kill: def $at killed $a1
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $1, $zero, -4
+; MIPS32-NEXT:    and $6, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 3
+; MIPS32-NEXT:    sll $10, $1, 3
+; MIPS32-NEXT:    ori $1, $zero, 65535
+; MIPS32-NEXT:    sllv $8, $1, $10
+; MIPS32-NEXT:    nor $9, $zero, $8
+; MIPS32-NEXT:    sllv $7, $5, $10
+; MIPS32-NEXT:  $BB4_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($6)
+; MIPS32-NEXT:    srav $2, $2, $10
+; MIPS32-NEXT:    srav $7, $7, $10
+; MIPS32-NEXT:    sll $2, $2, 16
+; MIPS32-NEXT:    sra $2, $2, 16
+; MIPS32-NEXT:    sll $7, $7, 16
+; MIPS32-NEXT:    sra $7, $7, 16
+; MIPS32-NEXT:    slt $5, $2, $7
+; MIPS32-NEXT:    move $3, $2
+; MIPS32-NEXT:    movn $3, $7, $5
+; MIPS32-NEXT:    and $3, $3, $8
+; MIPS32-NEXT:    and $4, $2, $9
+; MIPS32-NEXT:    or $4, $4, $3
+; MIPS32-NEXT:    sc $4, 0($6)
+; MIPS32-NEXT:    beqz $4, $BB4_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    and $1, $2, $8
+; MIPS32-NEXT:    srlv $1, $1, $10
+; MIPS32-NEXT:    sll $1, $1, 16
+; MIPS32-NEXT:    sra $1, $1, 16
+; MIPS32-NEXT:  # %bb.3: # %entry
+; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:  # %bb.4: # %entry
+; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $sp, $sp, 8
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+
 ;
 ; MIPSEL-LABEL: test_max_16:
 ; MIPSEL:       # %bb.0: # %entry
@@ -998,8 +1114,10 @@ define i16 @test_max_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPSEL-NEXT:  $BB4_1: # %entry
 ; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSEL-NEXT:    ll $2, 0($6)
-; MIPSEL-NEXT:    and $2, $2, $8
-; MIPSEL-NEXT:    and $7, $7, $8
+; MIPSEL-NEXT:    srav $2, $2, $10
+; MIPSEL-NEXT:    srav $7, $7, $10
+; MIPSEL-NEXT:    seh $2, $2
+; MIPSEL-NEXT:    seh $7, $7
 ; MIPSEL-NEXT:    slt $5, $2, $7
 ; MIPSEL-NEXT:    move $3, $2
 ; MIPSEL-NEXT:    movn $3, $7, $5
@@ -1039,8 +1157,10 @@ define i16 @test_max_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPSELR6-NEXT:  $BB4_1: # %entry
 ; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSELR6-NEXT:    ll $2, 0($6)
-; MIPSELR6-NEXT:    and $2, $2, $8
-; MIPSELR6-NEXT:    and $7, $7, $8
+; MIPSELR6-NEXT:    srav $2, $2, $10
+; MIPSELR6-NEXT:    srav $7, $7, $10
+; MIPSELR6-NEXT:    seh $2, $2
+; MIPSELR6-NEXT:    seh $7, $7
 ; MIPSELR6-NEXT:    slt $5, $2, $7
 ; MIPSELR6-NEXT:    seleqz $3, $2, $5
 ; MIPSELR6-NEXT:    selnez $5, $7, $5
@@ -1079,8 +1199,10 @@ define i16 @test_max_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMEL-NEXT:  $BB4_1: # %entry
 ; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMEL-NEXT:    ll $2, 0($6)
-; MMEL-NEXT:    and $2, $2, $8
-; MMEL-NEXT:    and $7, $7, $8
+; MMEL-NEXT:    srav $2, $2, $10
+; MMEL-NEXT:    srav $7, $7, $10
+; MMEL-NEXT:    seh $2, $2
+; MMEL-NEXT:    seh $7, $7
 ; MMEL-NEXT:    slt $5, $2, $7
 ; MMEL-NEXT:    or $3, $2, $zero
 ; MMEL-NEXT:    movn $3, $7, $5
@@ -1118,8 +1240,10 @@ define i16 @test_max_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMELR6-NEXT:  $BB4_1: # %entry
 ; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMELR6-NEXT:    ll $2, 0($6)
-; MMELR6-NEXT:    and $2, $2, $8
-; MMELR6-NEXT:    and $7, $7, $8
+; MMELR6-NEXT:    srav $2, $2, $10
+; MMELR6-NEXT:    srav $7, $7, $10
+; MMELR6-NEXT:    seh $2, $2
+; MMELR6-NEXT:    seh $7, $7
 ; MMELR6-NEXT:    slt $5, $2, $7
 ; MMELR6-NEXT:    seleqz $3, $2, $5
 ; MMELR6-NEXT:    selnez $5, $7, $5
@@ -1237,8 +1361,10 @@ define i16 @test_max_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPS64EL-NEXT:  .LBB4_1: # %entry
 ; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64EL-NEXT:    ll $2, 0($6)
-; MIPS64EL-NEXT:    and $2, $2, $8
-; MIPS64EL-NEXT:    and $7, $7, $8
+; MIPS64EL-NEXT:    srav $2, $2, $10
+; MIPS64EL-NEXT:    srav $7, $7, $10
+; MIPS64EL-NEXT:    seh $2, $2
+; MIPS64EL-NEXT:    seh $7, $7
 ; MIPS64EL-NEXT:    slt $5, $2, $7
 ; MIPS64EL-NEXT:    move $3, $2
 ; MIPS64EL-NEXT:    movn $3, $7, $5
@@ -1278,8 +1404,10 @@ define i16 @test_max_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPS64ELR6-NEXT:  .LBB4_1: # %entry
 ; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64ELR6-NEXT:    ll $2, 0($6)
-; MIPS64ELR6-NEXT:    and $2, $2, $8
-; MIPS64ELR6-NEXT:    and $7, $7, $8
+; MIPS64ELR6-NEXT:    srav $2, $2, $10
+; MIPS64ELR6-NEXT:    srav $7, $7, $10
+; MIPS64ELR6-NEXT:    seh $2, $2
+; MIPS64ELR6-NEXT:    seh $7, $7
 ; MIPS64ELR6-NEXT:    slt $5, $2, $7
 ; MIPS64ELR6-NEXT:    seleqz $3, $2, $5
 ; MIPS64ELR6-NEXT:    selnez $5, $7, $5
@@ -1462,6 +1590,52 @@ define i16 @test_min_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMR6-NEXT:    addiu $sp, $sp, 8
 ; MMR6-NEXT:    jrc $ra
 ;
+; MIPS32-LABEL: test_min_16:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -8
+; MIPS32-NEXT:    .cfi_def_cfa_offset 8
+; MIPS32-NEXT:    # kill: def $at killed $a1
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $1, $zero, -4
+; MIPS32-NEXT:    and $6, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 3
+; MIPS32-NEXT:    sll $10, $1, 3
+; MIPS32-NEXT:    ori $1, $zero, 65535
+; MIPS32-NEXT:    sllv $8, $1, $10
+; MIPS32-NEXT:    nor $9, $zero, $8
+; MIPS32-NEXT:    sllv $7, $5, $10
+; MIPS32-NEXT:  $BB5_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($6)
+; MIPS32-NEXT:    srav $2, $2, $10
+; MIPS32-NEXT:    srav $7, $7, $10
+; MIPS32-NEXT:    sll $2, $2, 16
+; MIPS32-NEXT:    sra $2, $2, 16
+; MIPS32-NEXT:    sll $7, $7, 16
+; MIPS32-NEXT:    sra $7, $7, 16
+; MIPS32-NEXT:    slt $5, $2, $7
+; MIPS32-NEXT:    move $3, $2
+; MIPS32-NEXT:    movz $3, $7, $5
+; MIPS32-NEXT:    and $3, $3, $8
+; MIPS32-NEXT:    and $4, $2, $9
+; MIPS32-NEXT:    or $4, $4, $3
+; MIPS32-NEXT:    sc $4, 0($6)
+; MIPS32-NEXT:    beqz $4, $BB5_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    and $1, $2, $8
+; MIPS32-NEXT:    srlv $1, $1, $10
+; MIPS32-NEXT:    sll $1, $1, 16
+; MIPS32-NEXT:    sra $1, $1, 16
+; MIPS32-NEXT:  # %bb.3: # %entry
+; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:  # %bb.4: # %entry
+; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $sp, $sp, 8
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
 ; MIPSEL-LABEL: test_min_16:
 ; MIPSEL:       # %bb.0: # %entry
 ; MIPSEL-NEXT:    addiu $sp, $sp, -8
@@ -1479,8 +1653,10 @@ define i16 @test_min_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPSEL-NEXT:  $BB5_1: # %entry
 ; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSEL-NEXT:    ll $2, 0($6)
-; MIPSEL-NEXT:    and $2, $2, $8
-; MIPSEL-NEXT:    and $7, $7, $8
+; MIPSEL-NEXT:    srav $2, $2, $10
+; MIPSEL-NEXT:    srav $7, $7, $10
+; MIPSEL-NEXT:    seh $2, $2
+; MIPSEL-NEXT:    seh $7, $7
 ; MIPSEL-NEXT:    slt $5, $2, $7
 ; MIPSEL-NEXT:    move $3, $2
 ; MIPSEL-NEXT:    movz $3, $7, $5
@@ -1520,8 +1696,10 @@ define i16 @test_min_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPSELR6-NEXT:  $BB5_1: # %entry
 ; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSELR6-NEXT:    ll $2, 0($6)
-; MIPSELR6-NEXT:    and $2, $2, $8
-; MIPSELR6-NEXT:    and $7, $7, $8
+; MIPSELR6-NEXT:    srav $2, $2, $10
+; MIPSELR6-NEXT:    srav $7, $7, $10
+; MIPSELR6-NEXT:    seh $2, $2
+; MIPSELR6-NEXT:    seh $7, $7
 ; MIPSELR6-NEXT:    slt $5, $2, $7
 ; MIPSELR6-NEXT:    selnez $3, $2, $5
 ; MIPSELR6-NEXT:    seleqz $5, $7, $5
@@ -1560,8 +1738,10 @@ define i16 @test_min_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMEL-NEXT:  $BB5_1: # %entry
 ; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMEL-NEXT:    ll $2, 0($6)
-; MMEL-NEXT:    and $2, $2, $8
-; MMEL-NEXT:    and $7, $7, $8
+; MMEL-NEXT:    srav $2, $2, $10
+; MMEL-NEXT:    srav $7, $7, $10
+; MMEL-NEXT:    seh $2, $2
+; MMEL-NEXT:    seh $7, $7
 ; MMEL-NEXT:    slt $5, $2, $7
 ; MMEL-NEXT:    or $3, $2, $zero
 ; MMEL-NEXT:    movz $3, $7, $5
@@ -1599,8 +1779,10 @@ define i16 @test_min_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMELR6-NEXT:  $BB5_1: # %entry
 ; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMELR6-NEXT:    ll $2, 0($6)
-; MMELR6-NEXT:    and $2, $2, $8
-; MMELR6-NEXT:    and $7, $7, $8
+; MMELR6-NEXT:    srav $2, $2, $10
+; MMELR6-NEXT:    srav $7, $7, $10
+; MMELR6-NEXT:    seh $2, $2
+; MMELR6-NEXT:    seh $7, $7
 ; MMELR6-NEXT:    slt $5, $2, $7
 ; MMELR6-NEXT:    selnez $3, $2, $5
 ; MMELR6-NEXT:    seleqz $5, $7, $5
@@ -1718,8 +1900,10 @@ define i16 @test_min_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPS64EL-NEXT:  .LBB5_1: # %entry
 ; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64EL-NEXT:    ll $2, 0($6)
-; MIPS64EL-NEXT:    and $2, $2, $8
-; MIPS64EL-NEXT:    and $7, $7, $8
+; MIPS64EL-NEXT:    srav $2, $2, $10
+; MIPS64EL-NEXT:    srav $7, $7, $10
+; MIPS64EL-NEXT:    seh $2, $2
+; MIPS64EL-NEXT:    seh $7, $7
 ; MIPS64EL-NEXT:    slt $5, $2, $7
 ; MIPS64EL-NEXT:    move $3, $2
 ; MIPS64EL-NEXT:    movz $3, $7, $5
@@ -1759,8 +1943,10 @@ define i16 @test_min_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPS64ELR6-NEXT:  .LBB5_1: # %entry
 ; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64ELR6-NEXT:    ll $2, 0($6)
-; MIPS64ELR6-NEXT:    and $2, $2, $8
-; MIPS64ELR6-NEXT:    and $7, $7, $8
+; MIPS64ELR6-NEXT:    srav $2, $2, $10
+; MIPS64ELR6-NEXT:    srav $7, $7, $10
+; MIPS64ELR6-NEXT:    seh $2, $2
+; MIPS64ELR6-NEXT:    seh $7, $7
 ; MIPS64ELR6-NEXT:    slt $5, $2, $7
 ; MIPS64ELR6-NEXT:    selnez $3, $2, $5
 ; MIPS64ELR6-NEXT:    seleqz $5, $7, $5
@@ -1943,6 +2129,46 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMR6-NEXT:    addiu $sp, $sp, 8
 ; MMR6-NEXT:    jrc $ra
 ;
+; MIPS32-LABEL: test_umax_16:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -8
+; MIPS32-NEXT:    .cfi_def_cfa_offset 8
+; MIPS32-NEXT:    # kill: def $at killed $a1
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $1, $zero, -4
+; MIPS32-NEXT:    and $6, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 3
+; MIPS32-NEXT:    sll $10, $1, 3
+; MIPS32-NEXT:    ori $1, $zero, 65535
+; MIPS32-NEXT:    sllv $8, $1, $10
+; MIPS32-NEXT:    nor $9, $zero, $8
+; MIPS32-NEXT:    sllv $7, $5, $10
+; MIPS32-NEXT:  $BB6_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($6)
+; MIPS32-NEXT:    sltu $5, $2, $7
+; MIPS32-NEXT:    move $3, $2
+; MIPS32-NEXT:    movn $3, $7, $5
+; MIPS32-NEXT:    and $3, $3, $8
+; MIPS32-NEXT:    and $4, $2, $9
+; MIPS32-NEXT:    or $4, $4, $3
+; MIPS32-NEXT:    sc $4, 0($6)
+; MIPS32-NEXT:    beqz $4, $BB6_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    and $1, $2, $8
+; MIPS32-NEXT:    srlv $1, $1, $10
+; MIPS32-NEXT:    sll $1, $1, 16
+; MIPS32-NEXT:    sra $1, $1, 16
+; MIPS32-NEXT:  # %bb.3: # %entry
+; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:  # %bb.4: # %entry
+; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $sp, $sp, 8
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
 ; MIPSEL-LABEL: test_umax_16:
 ; MIPSEL:       # %bb.0: # %entry
 ; MIPSEL-NEXT:    addiu $sp, $sp, -8
@@ -1960,8 +2186,6 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPSEL-NEXT:  $BB6_1: # %entry
 ; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSEL-NEXT:    ll $2, 0($6)
-; MIPSEL-NEXT:    and $2, $2, $8
-; MIPSEL-NEXT:    and $7, $7, $8
 ; MIPSEL-NEXT:    sltu $5, $2, $7
 ; MIPSEL-NEXT:    move $3, $2
 ; MIPSEL-NEXT:    movn $3, $7, $5
@@ -2001,8 +2225,6 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPSELR6-NEXT:  $BB6_1: # %entry
 ; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSELR6-NEXT:    ll $2, 0($6)
-; MIPSELR6-NEXT:    and $2, $2, $8
-; MIPSELR6-NEXT:    and $7, $7, $8
 ; MIPSELR6-NEXT:    sltu $5, $2, $7
 ; MIPSELR6-NEXT:    seleqz $3, $2, $5
 ; MIPSELR6-NEXT:    selnez $5, $7, $5
@@ -2041,8 +2263,6 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMEL-NEXT:  $BB6_1: # %entry
 ; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMEL-NEXT:    ll $2, 0($6)
-; MMEL-NEXT:    and $2, $2, $8
-; MMEL-NEXT:    and $7, $7, $8
 ; MMEL-NEXT:    sltu $5, $2, $7
 ; MMEL-NEXT:    or $3, $2, $zero
 ; MMEL-NEXT:    movn $3, $7, $5
@@ -2080,8 +2300,6 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMELR6-NEXT:  $BB6_1: # %entry
 ; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMELR6-NEXT:    ll $2, 0($6)
-; MMELR6-NEXT:    and $2, $2, $8
-; MMELR6-NEXT:    and $7, $7, $8
 ; MMELR6-NEXT:    sltu $5, $2, $7
 ; MMELR6-NEXT:    seleqz $3, $2, $5
 ; MMELR6-NEXT:    selnez $5, $7, $5
@@ -2199,8 +2417,6 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPS64EL-NEXT:  .LBB6_1: # %entry
 ; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64EL-NEXT:    ll $2, 0($6)
-; MIPS64EL-NEXT:    and $2, $2, $8
-; MIPS64EL-NEXT:    and $7, $7, $8
 ; MIPS64EL-NEXT:    sltu $5, $2, $7
 ; MIPS64EL-NEXT:    move $3, $2
 ; MIPS64EL-NEXT:    movn $3, $7, $5
@@ -2240,8 +2456,6 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPS64ELR6-NEXT:  .LBB6_1: # %entry
 ; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64ELR6-NEXT:    ll $2, 0($6)
-; MIPS64ELR6-NEXT:    and $2, $2, $8
-; MIPS64ELR6-NEXT:    and $7, $7, $8
 ; MIPS64ELR6-NEXT:    sltu $5, $2, $7
 ; MIPS64ELR6-NEXT:    seleqz $3, $2, $5
 ; MIPS64ELR6-NEXT:    selnez $5, $7, $5
@@ -2424,6 +2638,47 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMR6-NEXT:    addiu $sp, $sp, 8
 ; MMR6-NEXT:    jrc $ra
 ;
+; MIPS32-LABEL: test_umin_16:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -8
+; MIPS32-NEXT:    .cfi_def_cfa_offset 8
+; MIPS32-NEXT:    # kill: def $at killed $a1
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $1, $zero, -4
+; MIPS32-NEXT:    and $6, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 3
+; MIPS32-NEXT:    sll $10, $1, 3
+; MIPS32-NEXT:    ori $1, $zero, 65535
+; MIPS32-NEXT:    sllv $8, $1, $10
+; MIPS32-NEXT:    nor $9, $zero, $8
+; MIPS32-NEXT:    sllv $7, $5, $10
+; MIPS32-NEXT:  $BB7_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($6)
+; MIPS32-NEXT:    sltu $5, $2, $7
+; MIPS32-NEXT:    move $3, $2
+; MIPS32-NEXT:    movz $3, $7, $5
+; MIPS32-NEXT:    and $3, $3, $8
+; MIPS32-NEXT:    and $4, $2, $9
+; MIPS32-NEXT:    or $4, $4, $3
+; MIPS32-NEXT:    sc $4, 0($6)
+; MIPS32-NEXT:    beqz $4, $BB7_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    and $1, $2, $8
+; MIPS32-NEXT:    srlv $1, $1, $10
+; MIPS32-NEXT:    sll $1, $1, 16
+; MIPS32-NEXT:    sra $1, $1, 16
+; MIPS32-NEXT:  # %bb.3: # %entry
+; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:  # %bb.4: # %entry
+; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $sp, $sp, 8
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
+;
 ; MIPSEL-LABEL: test_umin_16:
 ; MIPSEL:       # %bb.0: # %entry
 ; MIPSEL-NEXT:    addiu $sp, $sp, -8
@@ -2441,8 +2696,6 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPSEL-NEXT:  $BB7_1: # %entry
 ; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSEL-NEXT:    ll $2, 0($6)
-; MIPSEL-NEXT:    and $2, $2, $8
-; MIPSEL-NEXT:    and $7, $7, $8
 ; MIPSEL-NEXT:    sltu $5, $2, $7
 ; MIPSEL-NEXT:    move $3, $2
 ; MIPSEL-NEXT:    movz $3, $7, $5
@@ -2482,8 +2735,6 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPSELR6-NEXT:  $BB7_1: # %entry
 ; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSELR6-NEXT:    ll $2, 0($6)
-; MIPSELR6-NEXT:    and $2, $2, $8
-; MIPSELR6-NEXT:    and $7, $7, $8
 ; MIPSELR6-NEXT:    sltu $5, $2, $7
 ; MIPSELR6-NEXT:    selnez $3, $2, $5
 ; MIPSELR6-NEXT:    seleqz $5, $7, $5
@@ -2522,8 +2773,6 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMEL-NEXT:  $BB7_1: # %entry
 ; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMEL-NEXT:    ll $2, 0($6)
-; MMEL-NEXT:    and $2, $2, $8
-; MMEL-NEXT:    and $7, $7, $8
 ; MMEL-NEXT:    sltu $5, $2, $7
 ; MMEL-NEXT:    or $3, $2, $zero
 ; MMEL-NEXT:    movz $3, $7, $5
@@ -2561,8 +2810,6 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMELR6-NEXT:  $BB7_1: # %entry
 ; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMELR6-NEXT:    ll $2, 0($6)
-; MMELR6-NEXT:    and $2, $2, $8
-; MMELR6-NEXT:    and $7, $7, $8
 ; MMELR6-NEXT:    sltu $5, $2, $7
 ; MMELR6-NEXT:    selnez $3, $2, $5
 ; MMELR6-NEXT:    seleqz $5, $7, $5
@@ -2680,8 +2927,6 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPS64EL-NEXT:  .LBB7_1: # %entry
 ; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64EL-NEXT:    ll $2, 0($6)
-; MIPS64EL-NEXT:    and $2, $2, $8
-; MIPS64EL-NEXT:    and $7, $7, $8
 ; MIPS64EL-NEXT:    sltu $5, $2, $7
 ; MIPS64EL-NEXT:    move $3, $2
 ; MIPS64EL-NEXT:    movz $3, $7, $5
@@ -2721,8 +2966,6 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPS64ELR6-NEXT:  .LBB7_1: # %entry
 ; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64ELR6-NEXT:    ll $2, 0($6)
-; MIPS64ELR6-NEXT:    and $2, $2, $8
-; MIPS64ELR6-NEXT:    and $7, $7, $8
 ; MIPS64ELR6-NEXT:    sltu $5, $2, $7
 ; MIPS64ELR6-NEXT:    selnez $3, $2, $5
 ; MIPS64ELR6-NEXT:    seleqz $5, $7, $5
@@ -2780,7 +3023,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS-NEXT:  # %bb.2: # %entry
 ; MIPS-NEXT:    and $1, $2, $8
 ; MIPS-NEXT:    srlv $1, $1, $10
-; MIPS-NEXT:    seh $1, $1
+; MIPS-NEXT:    seb $1, $1
 ; MIPS-NEXT:  # %bb.3: # %entry
 ; MIPS-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MIPS-NEXT:  # %bb.4: # %entry
@@ -2820,7 +3063,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSR6-NEXT:  # %bb.2: # %entry
 ; MIPSR6-NEXT:    and $1, $2, $8
 ; MIPSR6-NEXT:    srlv $1, $1, $10
-; MIPSR6-NEXT:    seh $1, $1
+; MIPSR6-NEXT:    seb $1, $1
 ; MIPSR6-NEXT:  # %bb.3: # %entry
 ; MIPSR6-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MIPSR6-NEXT:  # %bb.4: # %entry
@@ -2858,7 +3101,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MM-NEXT:  # %bb.2: # %entry
 ; MM-NEXT:    and $1, $2, $8
 ; MM-NEXT:    srlv $1, $1, $10
-; MM-NEXT:    seh $1, $1
+; MM-NEXT:    seb $1, $1
 ; MM-NEXT:  # %bb.3: # %entry
 ; MM-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MM-NEXT:  # %bb.4: # %entry
@@ -2897,7 +3140,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMR6-NEXT:  # %bb.2: # %entry
 ; MMR6-NEXT:    and $1, $2, $8
 ; MMR6-NEXT:    srlv $1, $1, $10
-; MMR6-NEXT:    seh $1, $1
+; MMR6-NEXT:    seb $1, $1
 ; MMR6-NEXT:  # %bb.3: # %entry
 ; MMR6-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MMR6-NEXT:  # %bb.4: # %entry
@@ -2906,6 +3149,52 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMR6-NEXT:    addiu $sp, $sp, 8
 ; MMR6-NEXT:    jrc $ra
 ;
+; MIPS32-LABEL: test_max_8:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -8
+; MIPS32-NEXT:    .cfi_def_cfa_offset 8
+; MIPS32-NEXT:    # kill: def $at killed $a1
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $1, $zero, -4
+; MIPS32-NEXT:    and $6, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 3
+; MIPS32-NEXT:    sll $10, $1, 3
+; MIPS32-NEXT:    ori $1, $zero, 255
+; MIPS32-NEXT:    sllv $8, $1, $10
+; MIPS32-NEXT:    nor $9, $zero, $8
+; MIPS32-NEXT:    sllv $7, $5, $10
+; MIPS32-NEXT:  $BB8_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($6)
+; MIPS32-NEXT:    srav $2, $2, $10
+; MIPS32-NEXT:    srav $7, $7, $10
+; MIPS32-NEXT:    sll $2, $2, 24
+; MIPS32-NEXT:    sra $2, $2, 24
+; MIPS32-NEXT:    sll $7, $7, 24
+; MIPS32-NEXT:    sra $7, $7, 24
+; MIPS32-NEXT:    slt $5, $2, $7
+; MIPS32-NEXT:    move $3, $2
+; MIPS32-NEXT:    movn $3, $7, $5
+; MIPS32-NEXT:    and $3, $3, $8
+; MIPS32-NEXT:    and $4, $2, $9
+; MIPS32-NEXT:    or $4, $4, $3
+; MIPS32-NEXT:    sc $4, 0($6)
+; MIPS32-NEXT:    beqz $4, $BB8_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    and $1, $2, $8
+; MIPS32-NEXT:    srlv $1, $1, $10
+; MIPS32-NEXT:    sll $1, $1, 24
+; MIPS32-NEXT:    sra $1, $1, 24
+; MIPS32-NEXT:  # %bb.3: # %entry
+; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:  # %bb.4: # %entry
+; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $sp, $sp, 8
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
 ; MIPSEL-LABEL: test_max_8:
 ; MIPSEL:       # %bb.0: # %entry
 ; MIPSEL-NEXT:    addiu $sp, $sp, -8
@@ -2923,8 +3212,10 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSEL-NEXT:  $BB8_1: # %entry
 ; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSEL-NEXT:    ll $2, 0($6)
-; MIPSEL-NEXT:    and $2, $2, $8
-; MIPSEL-NEXT:    and $7, $7, $8
+; MIPSEL-NEXT:    srav $2, $2, $10
+; MIPSEL-NEXT:    srav $7, $7, $10
+; MIPSEL-NEXT:    seb $2, $2
+; MIPSEL-NEXT:    seb $7, $7
 ; MIPSEL-NEXT:    slt $5, $2, $7
 ; MIPSEL-NEXT:    move $3, $2
 ; MIPSEL-NEXT:    movn $3, $7, $5
@@ -2937,7 +3228,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSEL-NEXT:  # %bb.2: # %entry
 ; MIPSEL-NEXT:    and $1, $2, $8
 ; MIPSEL-NEXT:    srlv $1, $1, $10
-; MIPSEL-NEXT:    seh $1, $1
+; MIPSEL-NEXT:    seb $1, $1
 ; MIPSEL-NEXT:  # %bb.3: # %entry
 ; MIPSEL-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MIPSEL-NEXT:  # %bb.4: # %entry
@@ -2964,8 +3255,10 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSELR6-NEXT:  $BB8_1: # %entry
 ; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSELR6-NEXT:    ll $2, 0($6)
-; MIPSELR6-NEXT:    and $2, $2, $8
-; MIPSELR6-NEXT:    and $7, $7, $8
+; MIPSELR6-NEXT:    srav $2, $2, $10
+; MIPSELR6-NEXT:    srav $7, $7, $10
+; MIPSELR6-NEXT:    seb $2, $2
+; MIPSELR6-NEXT:    seb $7, $7
 ; MIPSELR6-NEXT:    slt $5, $2, $7
 ; MIPSELR6-NEXT:    seleqz $3, $2, $5
 ; MIPSELR6-NEXT:    selnez $5, $7, $5
@@ -2978,7 +3271,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSELR6-NEXT:  # %bb.2: # %entry
 ; MIPSELR6-NEXT:    and $1, $2, $8
 ; MIPSELR6-NEXT:    srlv $1, $1, $10
-; MIPSELR6-NEXT:    seh $1, $1
+; MIPSELR6-NEXT:    seb $1, $1
 ; MIPSELR6-NEXT:  # %bb.3: # %entry
 ; MIPSELR6-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MIPSELR6-NEXT:  # %bb.4: # %entry
@@ -3004,8 +3297,10 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMEL-NEXT:  $BB8_1: # %entry
 ; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMEL-NEXT:    ll $2, 0($6)
-; MMEL-NEXT:    and $2, $2, $8
-; MMEL-NEXT:    and $7, $7, $8
+; MMEL-NEXT:    srav $2, $2, $10
+; MMEL-NEXT:    srav $7, $7, $10
+; MMEL-NEXT:    seb $2, $2
+; MMEL-NEXT:    seb $7, $7
 ; MMEL-NEXT:    slt $5, $2, $7
 ; MMEL-NEXT:    or $3, $2, $zero
 ; MMEL-NEXT:    movn $3, $7, $5
@@ -3017,7 +3312,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMEL-NEXT:  # %bb.2: # %entry
 ; MMEL-NEXT:    and $1, $2, $8
 ; MMEL-NEXT:    srlv $1, $1, $10
-; MMEL-NEXT:    seh $1, $1
+; MMEL-NEXT:    seb $1, $1
 ; MMEL-NEXT:  # %bb.3: # %entry
 ; MMEL-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MMEL-NEXT:  # %bb.4: # %entry
@@ -3043,8 +3338,10 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMELR6-NEXT:  $BB8_1: # %entry
 ; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMELR6-NEXT:    ll $2, 0($6)
-; MMELR6-NEXT:    and $2, $2, $8
-; MMELR6-NEXT:    and $7, $7, $8
+; MMELR6-NEXT:    srav $2, $2, $10
+; MMELR6-NEXT:    srav $7, $7, $10
+; MMELR6-NEXT:    seb $2, $2
+; MMELR6-NEXT:    seb $7, $7
 ; MMELR6-NEXT:    slt $5, $2, $7
 ; MMELR6-NEXT:    seleqz $3, $2, $5
 ; MMELR6-NEXT:    selnez $5, $7, $5
@@ -3057,7 +3354,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMELR6-NEXT:  # %bb.2: # %entry
 ; MMELR6-NEXT:    and $1, $2, $8
 ; MMELR6-NEXT:    srlv $1, $1, $10
-; MMELR6-NEXT:    seh $1, $1
+; MMELR6-NEXT:    seb $1, $1
 ; MMELR6-NEXT:  # %bb.3: # %entry
 ; MMELR6-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MMELR6-NEXT:  # %bb.4: # %entry
@@ -3096,7 +3393,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64-NEXT:  # %bb.2: # %entry
 ; MIPS64-NEXT:    and $1, $2, $8
 ; MIPS64-NEXT:    srlv $1, $1, $10
-; MIPS64-NEXT:    seh $1, $1
+; MIPS64-NEXT:    seb $1, $1
 ; MIPS64-NEXT:  # %bb.3: # %entry
 ; MIPS64-NEXT:    sw $1, 12($sp) # 4-byte Folded Spill
 ; MIPS64-NEXT:  # %bb.4: # %entry
@@ -3136,7 +3433,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64R6-NEXT:  # %bb.2: # %entry
 ; MIPS64R6-NEXT:    and $1, $2, $8
 ; MIPS64R6-NEXT:    srlv $1, $1, $10
-; MIPS64R6-NEXT:    seh $1, $1
+; MIPS64R6-NEXT:    seb $1, $1
 ; MIPS64R6-NEXT:  # %bb.3: # %entry
 ; MIPS64R6-NEXT:    sw $1, 12($sp) # 4-byte Folded Spill
 ; MIPS64R6-NEXT:  # %bb.4: # %entry
@@ -3162,8 +3459,10 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64EL-NEXT:  .LBB8_1: # %entry
 ; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64EL-NEXT:    ll $2, 0($6)
-; MIPS64EL-NEXT:    and $2, $2, $8
-; MIPS64EL-NEXT:    and $7, $7, $8
+; MIPS64EL-NEXT:    srav $2, $2, $10
+; MIPS64EL-NEXT:    srav $7, $7, $10
+; MIPS64EL-NEXT:    seb $2, $2
+; MIPS64EL-NEXT:    seb $7, $7
 ; MIPS64EL-NEXT:    slt $5, $2, $7
 ; MIPS64EL-NEXT:    move $3, $2
 ; MIPS64EL-NEXT:    movn $3, $7, $5
@@ -3176,7 +3475,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64EL-NEXT:  # %bb.2: # %entry
 ; MIPS64EL-NEXT:    and $1, $2, $8
 ; MIPS64EL-NEXT:    srlv $1, $1, $10
-; MIPS64EL-NEXT:    seh $1, $1
+; MIPS64EL-NEXT:    seb $1, $1
 ; MIPS64EL-NEXT:  # %bb.3: # %entry
 ; MIPS64EL-NEXT:    sw $1, 12($sp) # 4-byte Folded Spill
 ; MIPS64EL-NEXT:  # %bb.4: # %entry
@@ -3203,8 +3502,10 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64ELR6-NEXT:  .LBB8_1: # %entry
 ; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64ELR6-NEXT:    ll $2, 0($6)
-; MIPS64ELR6-NEXT:    and $2, $2, $8
-; MIPS64ELR6-NEXT:    and $7, $7, $8
+; MIPS64ELR6-NEXT:    srav $2, $2, $10
+; MIPS64ELR6-NEXT:    srav $7, $7, $10
+; MIPS64ELR6-NEXT:    seb $2, $2
+; MIPS64ELR6-NEXT:    seb $7, $7
 ; MIPS64ELR6-NEXT:    slt $5, $2, $7
 ; MIPS64ELR6-NEXT:    seleqz $3, $2, $5
 ; MIPS64ELR6-NEXT:    selnez $5, $7, $5
@@ -3217,7 +3518,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64ELR6-NEXT:  # %bb.2: # %entry
 ; MIPS64ELR6-NEXT:    and $1, $2, $8
 ; MIPS64ELR6-NEXT:    srlv $1, $1, $10
-; MIPS64ELR6-NEXT:    seh $1, $1
+; MIPS64ELR6-NEXT:    seb $1, $1
 ; MIPS64ELR6-NEXT:  # %bb.3: # %entry
 ; MIPS64ELR6-NEXT:    sw $1, 12($sp) # 4-byte Folded Spill
 ; MIPS64ELR6-NEXT:  # %bb.4: # %entry
@@ -3261,7 +3562,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS-NEXT:  # %bb.2: # %entry
 ; MIPS-NEXT:    and $1, $2, $8
 ; MIPS-NEXT:    srlv $1, $1, $10
-; MIPS-NEXT:    seh $1, $1
+; MIPS-NEXT:    seb $1, $1
 ; MIPS-NEXT:  # %bb.3: # %entry
 ; MIPS-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MIPS-NEXT:  # %bb.4: # %entry
@@ -3301,7 +3602,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSR6-NEXT:  # %bb.2: # %entry
 ; MIPSR6-NEXT:    and $1, $2, $8
 ; MIPSR6-NEXT:    srlv $1, $1, $10
-; MIPSR6-NEXT:    seh $1, $1
+; MIPSR6-NEXT:    seb $1, $1
 ; MIPSR6-NEXT:  # %bb.3: # %entry
 ; MIPSR6-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MIPSR6-NEXT:  # %bb.4: # %entry
@@ -3339,7 +3640,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MM-NEXT:  # %bb.2: # %entry
 ; MM-NEXT:    and $1, $2, $8
 ; MM-NEXT:    srlv $1, $1, $10
-; MM-NEXT:    seh $1, $1
+; MM-NEXT:    seb $1, $1
 ; MM-NEXT:  # %bb.3: # %entry
 ; MM-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MM-NEXT:  # %bb.4: # %entry
@@ -3378,7 +3679,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMR6-NEXT:  # %bb.2: # %entry
 ; MMR6-NEXT:    and $1, $2, $8
 ; MMR6-NEXT:    srlv $1, $1, $10
-; MMR6-NEXT:    seh $1, $1
+; MMR6-NEXT:    seb $1, $1
 ; MMR6-NEXT:  # %bb.3: # %entry
 ; MMR6-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MMR6-NEXT:  # %bb.4: # %entry
@@ -3387,6 +3688,52 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMR6-NEXT:    addiu $sp, $sp, 8
 ; MMR6-NEXT:    jrc $ra
 ;
+; MIPS32-LABEL: test_min_8:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -8
+; MIPS32-NEXT:    .cfi_def_cfa_offset 8
+; MIPS32-NEXT:    # kill: def $at killed $a1
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $1, $zero, -4
+; MIPS32-NEXT:    and $6, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 3
+; MIPS32-NEXT:    sll $10, $1, 3
+; MIPS32-NEXT:    ori $1, $zero, 255
+; MIPS32-NEXT:    sllv $8, $1, $10
+; MIPS32-NEXT:    nor $9, $zero, $8
+; MIPS32-NEXT:    sllv $7, $5, $10
+; MIPS32-NEXT:  $BB9_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($6)
+; MIPS32-NEXT:    srav $2, $2, $10
+; MIPS32-NEXT:    srav $7, $7, $10
+; MIPS32-NEXT:    sll $2, $2, 24
+; MIPS32-NEXT:    sra $2, $2, 24
+; MIPS32-NEXT:    sll $7, $7, 24
+; MIPS32-NEXT:    sra $7, $7, 24
+; MIPS32-NEXT:    slt $5, $2, $7
+; MIPS32-NEXT:    move $3, $2
+; MIPS32-NEXT:    movz $3, $7, $5
+; MIPS32-NEXT:    and $3, $3, $8
+; MIPS32-NEXT:    and $4, $2, $9
+; MIPS32-NEXT:    or $4, $4, $3
+; MIPS32-NEXT:    sc $4, 0($6)
+; MIPS32-NEXT:    beqz $4, $BB9_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    and $1, $2, $8
+; MIPS32-NEXT:    srlv $1, $1, $10
+; MIPS32-NEXT:    sll $1, $1, 24
+; MIPS32-NEXT:    sra $1, $1, 24
+; MIPS32-NEXT:  # %bb.3: # %entry
+; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:  # %bb.4: # %entry
+; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $sp, $sp, 8
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
 ; MIPSEL-LABEL: test_min_8:
 ; MIPSEL:       # %bb.0: # %entry
 ; MIPSEL-NEXT:    addiu $sp, $sp, -8
@@ -3404,8 +3751,10 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSEL-NEXT:  $BB9_1: # %entry
 ; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSEL-NEXT:    ll $2, 0($6)
-; MIPSEL-NEXT:    and $2, $2, $8
-; MIPSEL-NEXT:    and $7, $7, $8
+; MIPSEL-NEXT:    srav $2, $2, $10
+; MIPSEL-NEXT:    srav $7, $7, $10
+; MIPSEL-NEXT:    seb $2, $2
+; MIPSEL-NEXT:    seb $7, $7
 ; MIPSEL-NEXT:    slt $5, $2, $7
 ; MIPSEL-NEXT:    move $3, $2
 ; MIPSEL-NEXT:    movz $3, $7, $5
@@ -3418,7 +3767,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSEL-NEXT:  # %bb.2: # %entry
 ; MIPSEL-NEXT:    and $1, $2, $8
 ; MIPSEL-NEXT:    srlv $1, $1, $10
-; MIPSEL-NEXT:    seh $1, $1
+; MIPSEL-NEXT:    seb $1, $1
 ; MIPSEL-NEXT:  # %bb.3: # %entry
 ; MIPSEL-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MIPSEL-NEXT:  # %bb.4: # %entry
@@ -3445,8 +3794,10 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSELR6-NEXT:  $BB9_1: # %entry
 ; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSELR6-NEXT:    ll $2, 0($6)
-; MIPSELR6-NEXT:    and $2, $2, $8
-; MIPSELR6-NEXT:    and $7, $7, $8
+; MIPSELR6-NEXT:    srav $2, $2, $10
+; MIPSELR6-NEXT:    srav $7, $7, $10
+; MIPSELR6-NEXT:    seb $2, $2
+; MIPSELR6-NEXT:    seb $7, $7
 ; MIPSELR6-NEXT:    slt $5, $2, $7
 ; MIPSELR6-NEXT:    selnez $3, $2, $5
 ; MIPSELR6-NEXT:    seleqz $5, $7, $5
@@ -3459,7 +3810,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSELR6-NEXT:  # %bb.2: # %entry
 ; MIPSELR6-NEXT:    and $1, $2, $8
 ; MIPSELR6-NEXT:    srlv $1, $1, $10
-; MIPSELR6-NEXT:    seh $1, $1
+; MIPSELR6-NEXT:    seb $1, $1
 ; MIPSELR6-NEXT:  # %bb.3: # %entry
 ; MIPSELR6-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MIPSELR6-NEXT:  # %bb.4: # %entry
@@ -3485,8 +3836,10 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMEL-NEXT:  $BB9_1: # %entry
 ; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMEL-NEXT:    ll $2, 0($6)
-; MMEL-NEXT:    and $2, $2, $8
-; MMEL-NEXT:    and $7, $7, $8
+; MMEL-NEXT:    srav $2, $2, $10
+; MMEL-NEXT:    srav $7, $7, $10
+; MMEL-NEXT:    seb $2, $2
+; MMEL-NEXT:    seb $7, $7
 ; MMEL-NEXT:    slt $5, $2, $7
 ; MMEL-NEXT:    or $3, $2, $zero
 ; MMEL-NEXT:    movz $3, $7, $5
@@ -3498,7 +3851,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMEL-NEXT:  # %bb.2: # %entry
 ; MMEL-NEXT:    and $1, $2, $8
 ; MMEL-NEXT:    srlv $1, $1, $10
-; MMEL-NEXT:    seh $1, $1
+; MMEL-NEXT:    seb $1, $1
 ; MMEL-NEXT:  # %bb.3: # %entry
 ; MMEL-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MMEL-NEXT:  # %bb.4: # %entry
@@ -3524,8 +3877,10 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMELR6-NEXT:  $BB9_1: # %entry
 ; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMELR6-NEXT:    ll $2, 0($6)
-; MMELR6-NEXT:    and $2, $2, $8
-; MMELR6-NEXT:    and $7, $7, $8
+; MMELR6-NEXT:    srav $2, $2, $10
+; MMELR6-NEXT:    srav $7, $7, $10
+; MMELR6-NEXT:    seb $2, $2
+; MMELR6-NEXT:    seb $7, $7
 ; MMELR6-NEXT:    slt $5, $2, $7
 ; MMELR6-NEXT:    selnez $3, $2, $5
 ; MMELR6-NEXT:    seleqz $5, $7, $5
@@ -3538,7 +3893,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMELR6-NEXT:  # %bb.2: # %entry
 ; MMELR6-NEXT:    and $1, $2, $8
 ; MMELR6-NEXT:    srlv $1, $1, $10
-; MMELR6-NEXT:    seh $1, $1
+; MMELR6-NEXT:    seb $1, $1
 ; MMELR6-NEXT:  # %bb.3: # %entry
 ; MMELR6-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MMELR6-NEXT:  # %bb.4: # %entry
@@ -3577,7 +3932,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64-NEXT:  # %bb.2: # %entry
 ; MIPS64-NEXT:    and $1, $2, $8
 ; MIPS64-NEXT:    srlv $1, $1, $10
-; MIPS64-NEXT:    seh $1, $1
+; MIPS64-NEXT:    seb $1, $1
 ; MIPS64-NEXT:  # %bb.3: # %entry
 ; MIPS64-NEXT:    sw $1, 12($sp) # 4-byte Folded Spill
 ; MIPS64-NEXT:  # %bb.4: # %entry
@@ -3617,7 +3972,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64R6-NEXT:  # %bb.2: # %entry
 ; MIPS64R6-NEXT:    and $1, $2, $8
 ; MIPS64R6-NEXT:    srlv $1, $1, $10
-; MIPS64R6-NEXT:    seh $1, $1
+; MIPS64R6-NEXT:    seb $1, $1
 ; MIPS64R6-NEXT:  # %bb.3: # %entry
 ; MIPS64R6-NEXT:    sw $1, 12($sp) # 4-byte Folded Spill
 ; MIPS64R6-NEXT:  # %bb.4: # %entry
@@ -3643,8 +3998,10 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64EL-NEXT:  .LBB9_1: # %entry
 ; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64EL-NEXT:    ll $2, 0($6)
-; MIPS64EL-NEXT:    and $2, $2, $8
-; MIPS64EL-NEXT:    and $7, $7, $8
+; MIPS64EL-NEXT:    srav $2, $2, $10
+; MIPS64EL-NEXT:    srav $7, $7, $10
+; MIPS64EL-NEXT:    seb $2, $2
+; MIPS64EL-NEXT:    seb $7, $7
 ; MIPS64EL-NEXT:    slt $5, $2, $7
 ; MIPS64EL-NEXT:    move $3, $2
 ; MIPS64EL-NEXT:    movz $3, $7, $5
@@ -3657,7 +4014,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64EL-NEXT:  # %bb.2: # %entry
 ; MIPS64EL-NEXT:    and $1, $2, $8
 ; MIPS64EL-NEXT:    srlv $1, $1, $10
-; MIPS64EL-NEXT:    seh $1, $1
+; MIPS64EL-NEXT:    seb $1, $1
 ; MIPS64EL-NEXT:  # %bb.3: # %entry
 ; MIPS64EL-NEXT:    sw $1, 12($sp) # 4-byte Folded Spill
 ; MIPS64EL-NEXT:  # %bb.4: # %entry
@@ -3684,8 +4041,10 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64ELR6-NEXT:  .LBB9_1: # %entry
 ; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64ELR6-NEXT:    ll $2, 0($6)
-; MIPS64ELR6-NEXT:    and $2, $2, $8
-; MIPS64ELR6-NEXT:    and $7, $7, $8
+; MIPS64ELR6-NEXT:    srav $2, $2, $10
+; MIPS64ELR6-NEXT:    srav $7, $7, $10
+; MIPS64ELR6-NEXT:    seb $2, $2
+; MIPS64ELR6-NEXT:    seb $7, $7
 ; MIPS64ELR6-NEXT:    slt $5, $2, $7
 ; MIPS64ELR6-NEXT:    selnez $3, $2, $5
 ; MIPS64ELR6-NEXT:    seleqz $5, $7, $5
@@ -3698,7 +4057,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64ELR6-NEXT:  # %bb.2: # %entry
 ; MIPS64ELR6-NEXT:    and $1, $2, $8
 ; MIPS64ELR6-NEXT:    srlv $1, $1, $10
-; MIPS64ELR6-NEXT:    seh $1, $1
+; MIPS64ELR6-NEXT:    seb $1, $1
 ; MIPS64ELR6-NEXT:  # %bb.3: # %entry
 ; MIPS64ELR6-NEXT:    sw $1, 12($sp) # 4-byte Folded Spill
 ; MIPS64ELR6-NEXT:  # %bb.4: # %entry
@@ -3868,6 +4227,46 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMR6-NEXT:    addiu $sp, $sp, 8
 ; MMR6-NEXT:    jrc $ra
 ;
+; MIPS32-LABEL: test_umax_8:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -8
+; MIPS32-NEXT:    .cfi_def_cfa_offset 8
+; MIPS32-NEXT:    # kill: def $at killed $a1
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $1, $zero, -4
+; MIPS32-NEXT:    and $6, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 3
+; MIPS32-NEXT:    sll $10, $1, 3
+; MIPS32-NEXT:    ori $1, $zero, 255
+; MIPS32-NEXT:    sllv $8, $1, $10
+; MIPS32-NEXT:    nor $9, $zero, $8
+; MIPS32-NEXT:    sllv $7, $5, $10
+; MIPS32-NEXT:  $BB10_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($6)
+; MIPS32-NEXT:    sltu $5, $2, $7
+; MIPS32-NEXT:    move $3, $2
+; MIPS32-NEXT:    movn $3, $7, $5
+; MIPS32-NEXT:    and $3, $3, $8
+; MIPS32-NEXT:    and $4, $2, $9
+; MIPS32-NEXT:    or $4, $4, $3
+; MIPS32-NEXT:    sc $4, 0($6)
+; MIPS32-NEXT:    beqz $4, $BB10_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    and $1, $2, $8
+; MIPS32-NEXT:    srlv $1, $1, $10
+; MIPS32-NEXT:    sll $1, $1, 16
+; MIPS32-NEXT:    sra $1, $1, 16
+; MIPS32-NEXT:  # %bb.3: # %entry
+; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:  # %bb.4: # %entry
+; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $sp, $sp, 8
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
 ; MIPSEL-LABEL: test_umax_8:
 ; MIPSEL:       # %bb.0: # %entry
 ; MIPSEL-NEXT:    addiu $sp, $sp, -8
@@ -3885,8 +4284,6 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSEL-NEXT:  $BB10_1: # %entry
 ; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSEL-NEXT:    ll $2, 0($6)
-; MIPSEL-NEXT:    and $2, $2, $8
-; MIPSEL-NEXT:    and $7, $7, $8
 ; MIPSEL-NEXT:    sltu $5, $2, $7
 ; MIPSEL-NEXT:    move $3, $2
 ; MIPSEL-NEXT:    movn $3, $7, $5
@@ -3926,8 +4323,6 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSELR6-NEXT:  $BB10_1: # %entry
 ; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSELR6-NEXT:    ll $2, 0($6)
-; MIPSELR6-NEXT:    and $2, $2, $8
-; MIPSELR6-NEXT:    and $7, $7, $8
 ; MIPSELR6-NEXT:    sltu $5, $2, $7
 ; MIPSELR6-NEXT:    seleqz $3, $2, $5
 ; MIPSELR6-NEXT:    selnez $5, $7, $5
@@ -3966,8 +4361,6 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMEL-NEXT:  $BB10_1: # %entry
 ; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMEL-NEXT:    ll $2, 0($6)
-; MMEL-NEXT:    and $2, $2, $8
-; MMEL-NEXT:    and $7, $7, $8
 ; MMEL-NEXT:    sltu $5, $2, $7
 ; MMEL-NEXT:    or $3, $2, $zero
 ; MMEL-NEXT:    movn $3, $7, $5
@@ -4005,8 +4398,6 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMELR6-NEXT:  $BB10_1: # %entry
 ; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMELR6-NEXT:    ll $2, 0($6)
-; MMELR6-NEXT:    and $2, $2, $8
-; MMELR6-NEXT:    and $7, $7, $8
 ; MMELR6-NEXT:    sltu $5, $2, $7
 ; MMELR6-NEXT:    seleqz $3, $2, $5
 ; MMELR6-NEXT:    selnez $5, $7, $5
@@ -4124,8 +4515,6 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64EL-NEXT:  .LBB10_1: # %entry
 ; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64EL-NEXT:    ll $2, 0($6)
-; MIPS64EL-NEXT:    and $2, $2, $8
-; MIPS64EL-NEXT:    and $7, $7, $8
 ; MIPS64EL-NEXT:    sltu $5, $2, $7
 ; MIPS64EL-NEXT:    move $3, $2
 ; MIPS64EL-NEXT:    movn $3, $7, $5
@@ -4165,8 +4554,6 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64ELR6-NEXT:  .LBB10_1: # %entry
 ; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64ELR6-NEXT:    ll $2, 0($6)
-; MIPS64ELR6-NEXT:    and $2, $2, $8
-; MIPS64ELR6-NEXT:    and $7, $7, $8
 ; MIPS64ELR6-NEXT:    sltu $5, $2, $7
 ; MIPS64ELR6-NEXT:    seleqz $3, $2, $5
 ; MIPS64ELR6-NEXT:    selnez $5, $7, $5
@@ -4349,6 +4736,46 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMR6-NEXT:    addiu $sp, $sp, 8
 ; MMR6-NEXT:    jrc $ra
 ;
+; MIPS32-LABEL: test_umin_8:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -8
+; MIPS32-NEXT:    .cfi_def_cfa_offset 8
+; MIPS32-NEXT:    # kill: def $at killed $a1
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $1, $zero, -4
+; MIPS32-NEXT:    and $6, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 3
+; MIPS32-NEXT:    sll $10, $1, 3
+; MIPS32-NEXT:    ori $1, $zero, 255
+; MIPS32-NEXT:    sllv $8, $1, $10
+; MIPS32-NEXT:    nor $9, $zero, $8
+; MIPS32-NEXT:    sllv $7, $5, $10
+; MIPS32-NEXT:  $BB11_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($6)
+; MIPS32-NEXT:    sltu $5, $2, $7
+; MIPS32-NEXT:    move $3, $2
+; MIPS32-NEXT:    movz $3, $7, $5
+; MIPS32-NEXT:    and $3, $3, $8
+; MIPS32-NEXT:    and $4, $2, $9
+; MIPS32-NEXT:    or $4, $4, $3
+; MIPS32-NEXT:    sc $4, 0($6)
+; MIPS32-NEXT:    beqz $4, $BB11_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    and $1, $2, $8
+; MIPS32-NEXT:    srlv $1, $1, $10
+; MIPS32-NEXT:    sll $1, $1, 16
+; MIPS32-NEXT:    sra $1, $1, 16
+; MIPS32-NEXT:  # %bb.3: # %entry
+; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:  # %bb.4: # %entry
+; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $sp, $sp, 8
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
 ; MIPSEL-LABEL: test_umin_8:
 ; MIPSEL:       # %bb.0: # %entry
 ; MIPSEL-NEXT:    addiu $sp, $sp, -8
@@ -4366,8 +4793,6 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSEL-NEXT:  $BB11_1: # %entry
 ; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSEL-NEXT:    ll $2, 0($6)
-; MIPSEL-NEXT:    and $2, $2, $8
-; MIPSEL-NEXT:    and $7, $7, $8
 ; MIPSEL-NEXT:    sltu $5, $2, $7
 ; MIPSEL-NEXT:    move $3, $2
 ; MIPSEL-NEXT:    movz $3, $7, $5
@@ -4407,8 +4832,6 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSELR6-NEXT:  $BB11_1: # %entry
 ; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSELR6-NEXT:    ll $2, 0($6)
-; MIPSELR6-NEXT:    and $2, $2, $8
-; MIPSELR6-NEXT:    and $7, $7, $8
 ; MIPSELR6-NEXT:    sltu $5, $2, $7
 ; MIPSELR6-NEXT:    selnez $3, $2, $5
 ; MIPSELR6-NEXT:    seleqz $5, $7, $5
@@ -4447,8 +4870,6 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMEL-NEXT:  $BB11_1: # %entry
 ; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMEL-NEXT:    ll $2, 0($6)
-; MMEL-NEXT:    and $2, $2, $8
-; MMEL-NEXT:    and $7, $7, $8
 ; MMEL-NEXT:    sltu $5, $2, $7
 ; MMEL-NEXT:    or $3, $2, $zero
 ; MMEL-NEXT:    movz $3, $7, $5
@@ -4486,8 +4907,6 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMELR6-NEXT:  $BB11_1: # %entry
 ; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMELR6-NEXT:    ll $2, 0($6)
-; MMELR6-NEXT:    and $2, $2, $8
-; MMELR6-NEXT:    and $7, $7, $8
 ; MMELR6-NEXT:    sltu $5, $2, $7
 ; MMELR6-NEXT:    selnez $3, $2, $5
 ; MMELR6-NEXT:    seleqz $5, $7, $5
@@ -4605,8 +5024,6 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64EL-NEXT:  .LBB11_1: # %entry
 ; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64EL-NEXT:    ll $2, 0($6)
-; MIPS64EL-NEXT:    and $2, $2, $8
-; MIPS64EL-NEXT:    and $7, $7, $8
 ; MIPS64EL-NEXT:    sltu $5, $2, $7
 ; MIPS64EL-NEXT:    move $3, $2
 ; MIPS64EL-NEXT:    movz $3, $7, $5
@@ -4646,8 +5063,6 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64ELR6-NEXT:  .LBB11_1: # %entry
 ; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64ELR6-NEXT:    ll $2, 0($6)
-; MIPS64ELR6-NEXT:    and $2, $2, $8
-; MIPS64ELR6-NEXT:    and $7, $7, $8
 ; MIPS64ELR6-NEXT:    sltu $5, $2, $7
 ; MIPS64ELR6-NEXT:    selnez $3, $2, $5
 ; MIPS64ELR6-NEXT:    seleqz $5, $7, $5
-- 
GitLab


From 83789ffbd857bb78b69d6e54560310ff2d347f04 Mon Sep 17 00:00:00 2001
From: Florian Hahn 
Date: Fri, 8 Mar 2024 20:43:35 +0000
Subject: [PATCH 662/929] [TBAA] Add bail-out to skip tbaa generation to
 getTBAAStructInfo. (#84386)

Without this bail out, we may generate fields with null nodes as tags
are generated by using getTypeInfo which has the same bail out.

PR: https://github.com/llvm/llvm-project/pull/84386
---
 clang/lib/CodeGen/CodeGenTBAA.cpp                             | 3 +++
 clang/test/CodeGen/tbaa-struct-relaxed-aliasing-with-tsan.cpp | 2 --
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/clang/lib/CodeGen/CodeGenTBAA.cpp b/clang/lib/CodeGen/CodeGenTBAA.cpp
index 8a0816121939..1f07205a5af2 100644
--- a/clang/lib/CodeGen/CodeGenTBAA.cpp
+++ b/clang/lib/CodeGen/CodeGenTBAA.cpp
@@ -343,6 +343,9 @@ CodeGenTBAA::CollectFields(uint64_t BaseOffset,
 
 llvm::MDNode *
 CodeGenTBAA::getTBAAStructInfo(QualType QTy) {
+  if (CodeGenOpts.OptimizationLevel == 0 || CodeGenOpts.RelaxedAliasing)
+    return nullptr;
+
   const Type *Ty = Context.getCanonicalType(QTy).getTypePtr();
 
   if (llvm::MDNode *N = StructMetadataCache[Ty])
diff --git a/clang/test/CodeGen/tbaa-struct-relaxed-aliasing-with-tsan.cpp b/clang/test/CodeGen/tbaa-struct-relaxed-aliasing-with-tsan.cpp
index 931ff2476cd1..ce613b9d6b23 100644
--- a/clang/test/CodeGen/tbaa-struct-relaxed-aliasing-with-tsan.cpp
+++ b/clang/test/CodeGen/tbaa-struct-relaxed-aliasing-with-tsan.cpp
@@ -5,9 +5,7 @@
 // RUN:     FileCheck %s
 //
 // Check that we do not create tbaa for instructions generated for copies.
-// FIXME: !tbaa.struct is generated with null node as tag.
 
-// CHECK: !tbaa.struct
 // CHECK-NOT: !tbaa
 
 struct A {
-- 
GitLab


From 8d85cd3a59f65315bbadaf11c4b3f107238456db Mon Sep 17 00:00:00 2001
From: Louis Dionne 
Date: Fri, 8 Mar 2024 15:47:43 -0500
Subject: [PATCH 663/929] [libc++] Don't build against libcxxrt by default on
 FreeBSD (#84484)

The libc++ CMake build aims to provide a general-purpose configuration
that matches the canonical LLVM setup (libc++ / libc++abi / libunwind),
not the configuration used for compatibility on any given system. For
these "compatibility" configurations, we use CMake caches like
Apple.cmake and FreeBSD.cmake.

Defaulting to libcxxrt on FreeBSD makes it look as though we're trying
to build the system compatible configuration on FreeBSD, which isn't
really correct and causes confusion more than anything else.

Fixes #84476
---
 libcxx/CMakeLists.txt | 2 --
 1 file changed, 2 deletions(-)

diff --git a/libcxx/CMakeLists.txt b/libcxx/CMakeLists.txt
index d392e95077ac..e565c47c7668 100644
--- a/libcxx/CMakeLists.txt
+++ b/libcxx/CMakeLists.txt
@@ -229,8 +229,6 @@ option(LIBCXX_USE_COMPILER_RT "Use compiler-rt instead of libgcc" OFF)
 # ABI Library options ---------------------------------------------------------
 if (MSVC)
   set(LIBCXX_DEFAULT_ABI_LIBRARY "vcruntime")
-elseif (${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD")
-  set(LIBCXX_DEFAULT_ABI_LIBRARY "libcxxrt")
 else()
   set(LIBCXX_DEFAULT_ABI_LIBRARY "libcxxabi")
 endif()
-- 
GitLab


From 29762e372294e5b591684f57979f33c1c48bbfba Mon Sep 17 00:00:00 2001
From: Joseph Huber 
Date: Fri, 8 Mar 2024 15:02:19 -0600
Subject: [PATCH 664/929]  [libc][NFCI] Remove lane size template argument on
 RPC server  (#84557)

Summary:
We previously changed the data layout for the RPC buffer to make it lane
size agnostic. I put off changing the size for the server case to make
the patch smaller. This patch simply reorganizes code by making the lane
size an argument to the port rather than a templtae size. Heavily
simplifies a lot of code, no more `std::variant`.
---
 libc/src/__support/RPC/rpc.h         |  23 +-
 libc/utils/gpu/server/rpc_server.cpp | 484 ++++++++++++---------------
 2 files changed, 234 insertions(+), 273 deletions(-)

diff --git a/libc/src/__support/RPC/rpc.h b/libc/src/__support/RPC/rpc.h
index 5ed39ae0d7f7..5dcae518bb6f 100644
--- a/libc/src/__support/RPC/rpc.h
+++ b/libc/src/__support/RPC/rpc.h
@@ -310,7 +310,7 @@ private:
   LIBC_INLINE Port &operator=(Port &&) = default;
 
   friend struct Client;
-  template  friend struct Server;
+  friend struct Server;
   friend class cpp::optional>;
 
 public:
@@ -369,7 +369,7 @@ static_assert(cpp::is_trivially_copyable::value &&
               "The client is not trivially copyable from the server");
 
 /// The RPC server used to respond to the client.
-template  struct Server {
+struct Server {
   LIBC_INLINE Server() = default;
   LIBC_INLINE Server(const Server &) = delete;
   LIBC_INLINE Server &operator=(const Server &) = delete;
@@ -379,10 +379,12 @@ template  struct Server {
       : process(port_count, buffer) {}
 
   using Port = rpc::Port;
-  LIBC_INLINE cpp::optional try_open(uint32_t start = 0);
-  LIBC_INLINE Port open();
+  LIBC_INLINE cpp::optional try_open(uint32_t lane_size,
+                                           uint32_t start = 0);
+  LIBC_INLINE Port open(uint32_t lane_size);
 
-  LIBC_INLINE static uint64_t allocation_size(uint32_t port_count) {
+  LIBC_INLINE static uint64_t allocation_size(uint32_t lane_size,
+                                              uint32_t port_count) {
     return Process::allocation_size(port_count, lane_size);
   }
 
@@ -556,10 +558,8 @@ template 
 
 /// Attempts to open a port to use as the server. The server can only open a
 /// port if it has a pending receive operation
-template 
-[[clang::convergent]] LIBC_INLINE
-    cpp::optional::Port>
-    Server::try_open(uint32_t start) {
+[[clang::convergent]] LIBC_INLINE cpp::optional
+Server::try_open(uint32_t lane_size, uint32_t start) {
   // Perform a naive linear scan for a port that has a pending request.
   for (uint32_t index = start; index < process.port_count; ++index) {
     uint64_t lane_mask = gpu::get_lane_mask();
@@ -588,10 +588,9 @@ template 
   return cpp::nullopt;
 }
 
-template 
-LIBC_INLINE typename Server::Port Server::open() {
+LIBC_INLINE Server::Port Server::open(uint32_t lane_size) {
   for (;;) {
-    if (cpp::optional p = try_open())
+    if (cpp::optional p = try_open(lane_size))
       return cpp::move(p.value());
     sleep_briefly();
   }
diff --git a/libc/utils/gpu/server/rpc_server.cpp b/libc/utils/gpu/server/rpc_server.cpp
index 707807a5cbaf..2c8186c13ffa 100644
--- a/libc/utils/gpu/server/rpc_server.cpp
+++ b/libc/utils/gpu/server/rpc_server.cpp
@@ -27,228 +27,217 @@ static_assert(sizeof(rpc_buffer_t) == sizeof(rpc::Buffer),
 static_assert(RPC_MAXIMUM_PORT_COUNT == rpc::MAX_PORT_COUNT,
               "Incorrect maximum port count");
 
-// The client needs to support different lane sizes for the SIMT model. Because
-// of this we need to select between the possible sizes that the client can use.
-struct Server {
-  template 
-  Server(std::unique_ptr> &&server)
-      : server(std::move(server)) {}
-
-  rpc_status_t handle_server(
-      const std::unordered_map &callbacks,
-      const std::unordered_map &callback_data,
-      uint32_t &index) {
-    rpc_status_t ret = RPC_STATUS_SUCCESS;
-    std::visit(
-        [&](auto &server) {
-          ret = handle_server(*server, callbacks, callback_data, index);
-        },
-        server);
-    return ret;
-  }
-
-private:
-  template 
-  rpc_status_t handle_server(
-      rpc::Server &server,
-      const std::unordered_map &callbacks,
-      const std::unordered_map &callback_data,
-      uint32_t &index) {
-    auto port = server.try_open(index);
-    if (!port)
-      return RPC_STATUS_SUCCESS;
-
-    switch (port->get_opcode()) {
-    case RPC_WRITE_TO_STREAM:
-    case RPC_WRITE_TO_STDERR:
-    case RPC_WRITE_TO_STDOUT:
-    case RPC_WRITE_TO_STDOUT_NEWLINE: {
-      uint64_t sizes[lane_size] = {0};
-      void *strs[lane_size] = {nullptr};
-      FILE *files[lane_size] = {nullptr};
-      if (port->get_opcode() == RPC_WRITE_TO_STREAM) {
-        port->recv([&](rpc::Buffer *buffer, uint32_t id) {
-          files[id] = reinterpret_cast(buffer->data[0]);
-        });
-      } else if (port->get_opcode() == RPC_WRITE_TO_STDERR) {
-        std::fill(files, files + lane_size, stderr);
-      } else {
-        std::fill(files, files + lane_size, stdout);
-      }
-
-      port->recv_n(strs, sizes, [&](uint64_t size) { return new char[size]; });
-      port->send([&](rpc::Buffer *buffer, uint32_t id) {
-        flockfile(files[id]);
-        buffer->data[0] = fwrite_unlocked(strs[id], 1, sizes[id], files[id]);
-        if (port->get_opcode() == RPC_WRITE_TO_STDOUT_NEWLINE &&
-            buffer->data[0] == sizes[id])
-          buffer->data[0] += fwrite_unlocked("\n", 1, 1, files[id]);
-        funlockfile(files[id]);
-        delete[] reinterpret_cast(strs[id]);
-      });
-      break;
-    }
-    case RPC_READ_FROM_STREAM: {
-      uint64_t sizes[lane_size] = {0};
-      void *data[lane_size] = {nullptr};
-      port->recv([&](rpc::Buffer *buffer, uint32_t id) {
-        data[id] = new char[buffer->data[0]];
-        sizes[id] = fread(data[id], 1, buffer->data[0],
-                          file::to_stream(buffer->data[1]));
-      });
-      port->send_n(data, sizes);
-      port->send([&](rpc::Buffer *buffer, uint32_t id) {
-        delete[] reinterpret_cast(data[id]);
-        std::memcpy(buffer->data, &sizes[id], sizeof(uint64_t));
-      });
-      break;
-    }
-    case RPC_READ_FGETS: {
-      uint64_t sizes[lane_size] = {0};
-      void *data[lane_size] = {nullptr};
-      port->recv([&](rpc::Buffer *buffer, uint32_t id) {
-        data[id] = new char[buffer->data[0]];
-        const char *str =
-            fgets(reinterpret_cast(data[id]), buffer->data[0],
-                  file::to_stream(buffer->data[1]));
-        sizes[id] = !str ? 0 : std::strlen(str) + 1;
-      });
-      port->send_n(data, sizes);
-      for (uint32_t id = 0; id < lane_size; ++id)
-        if (data[id])
-          delete[] reinterpret_cast(data[id]);
-      break;
-    }
-    case RPC_OPEN_FILE: {
-      uint64_t sizes[lane_size] = {0};
-      void *paths[lane_size] = {nullptr};
-      port->recv_n(paths, sizes, [&](uint64_t size) { return new char[size]; });
-      port->recv_and_send([&](rpc::Buffer *buffer, uint32_t id) {
-        FILE *file = fopen(reinterpret_cast(paths[id]),
-                           reinterpret_cast(buffer->data));
-        buffer->data[0] = reinterpret_cast(file);
-      });
-      break;
-    }
-    case RPC_CLOSE_FILE: {
-      port->recv_and_send([&](rpc::Buffer *buffer, uint32_t id) {
-        FILE *file = reinterpret_cast(buffer->data[0]);
-        buffer->data[0] = fclose(file);
-      });
-      break;
-    }
-    case RPC_EXIT: {
-      // Send a response to the client to signal that we are ready to exit.
-      port->recv_and_send([](rpc::Buffer *) {});
-      port->recv([](rpc::Buffer *buffer) {
-        int status = 0;
-        std::memcpy(&status, buffer->data, sizeof(int));
-        exit(status);
-      });
-      break;
-    }
-    case RPC_ABORT: {
-      // Send a response to the client to signal that we are ready to abort.
-      port->recv_and_send([](rpc::Buffer *) {});
-      port->recv([](rpc::Buffer *) {});
-      abort();
-      break;
-    }
-    case RPC_HOST_CALL: {
-      uint64_t sizes[lane_size] = {0};
-      void *args[lane_size] = {nullptr};
-      port->recv_n(args, sizes, [&](uint64_t size) { return new char[size]; });
+template 
+rpc_status_t handle_server_impl(
+    rpc::Server &server,
+    const std::unordered_map &callbacks,
+    const std::unordered_map &callback_data,
+    uint32_t &index) {
+  auto port = server.try_open(lane_size, index);
+  if (!port)
+    return RPC_STATUS_SUCCESS;
+
+  switch (port->get_opcode()) {
+  case RPC_WRITE_TO_STREAM:
+  case RPC_WRITE_TO_STDERR:
+  case RPC_WRITE_TO_STDOUT:
+  case RPC_WRITE_TO_STDOUT_NEWLINE: {
+    uint64_t sizes[lane_size] = {0};
+    void *strs[lane_size] = {nullptr};
+    FILE *files[lane_size] = {nullptr};
+    if (port->get_opcode() == RPC_WRITE_TO_STREAM) {
       port->recv([&](rpc::Buffer *buffer, uint32_t id) {
-        reinterpret_cast(buffer->data[0])(args[id]);
-      });
-      port->send([&](rpc::Buffer *, uint32_t id) {
-        delete[] reinterpret_cast(args[id]);
-      });
-      break;
-    }
-    case RPC_FEOF: {
-      port->recv_and_send([](rpc::Buffer *buffer) {
-        buffer->data[0] = feof(file::to_stream(buffer->data[0]));
-      });
-      break;
-    }
-    case RPC_FERROR: {
-      port->recv_and_send([](rpc::Buffer *buffer) {
-        buffer->data[0] = ferror(file::to_stream(buffer->data[0]));
-      });
-      break;
-    }
-    case RPC_CLEARERR: {
-      port->recv_and_send([](rpc::Buffer *buffer) {
-        clearerr(file::to_stream(buffer->data[0]));
-      });
-      break;
-    }
-    case RPC_FSEEK: {
-      port->recv_and_send([](rpc::Buffer *buffer) {
-        buffer->data[0] = fseek(file::to_stream(buffer->data[0]),
-                                static_cast(buffer->data[1]),
-                                static_cast(buffer->data[2]));
-      });
-      break;
-    }
-    case RPC_FTELL: {
-      port->recv_and_send([](rpc::Buffer *buffer) {
-        buffer->data[0] = ftell(file::to_stream(buffer->data[0]));
-      });
-      break;
-    }
-    case RPC_FFLUSH: {
-      port->recv_and_send([](rpc::Buffer *buffer) {
-        buffer->data[0] = fflush(file::to_stream(buffer->data[0]));
+        files[id] = reinterpret_cast(buffer->data[0]);
       });
-      break;
-    }
-    case RPC_UNGETC: {
-      port->recv_and_send([](rpc::Buffer *buffer) {
-        buffer->data[0] = ungetc(static_cast(buffer->data[0]),
-                                 file::to_stream(buffer->data[1]));
-      });
-      break;
-    }
-    case RPC_NOOP: {
-      port->recv([](rpc::Buffer *) {});
-      break;
-    }
-    default: {
-      auto handler =
-          callbacks.find(static_cast(port->get_opcode()));
-
-      // We error out on an unhandled opcode.
-      if (handler == callbacks.end())
-        return RPC_STATUS_UNHANDLED_OPCODE;
-
-      // Invoke the registered callback with a reference to the port.
-      void *data =
-          callback_data.at(static_cast(port->get_opcode()));
-      rpc_port_t port_ref{reinterpret_cast(&*port), lane_size};
-      (handler->second)(port_ref, data);
-    }
+    } else if (port->get_opcode() == RPC_WRITE_TO_STDERR) {
+      std::fill(files, files + lane_size, stderr);
+    } else {
+      std::fill(files, files + lane_size, stdout);
     }
 
-    // Increment the index so we start the scan after this port.
-    index = port->get_index() + 1;
-    port->close();
-    return RPC_STATUS_CONTINUE;
+    port->recv_n(strs, sizes, [&](uint64_t size) { return new char[size]; });
+    port->send([&](rpc::Buffer *buffer, uint32_t id) {
+      flockfile(files[id]);
+      buffer->data[0] = fwrite_unlocked(strs[id], 1, sizes[id], files[id]);
+      if (port->get_opcode() == RPC_WRITE_TO_STDOUT_NEWLINE &&
+          buffer->data[0] == sizes[id])
+        buffer->data[0] += fwrite_unlocked("\n", 1, 1, files[id]);
+      funlockfile(files[id]);
+      delete[] reinterpret_cast(strs[id]);
+    });
+    break;
+  }
+  case RPC_READ_FROM_STREAM: {
+    uint64_t sizes[lane_size] = {0};
+    void *data[lane_size] = {nullptr};
+    port->recv([&](rpc::Buffer *buffer, uint32_t id) {
+      data[id] = new char[buffer->data[0]];
+      sizes[id] =
+          fread(data[id], 1, buffer->data[0], file::to_stream(buffer->data[1]));
+    });
+    port->send_n(data, sizes);
+    port->send([&](rpc::Buffer *buffer, uint32_t id) {
+      delete[] reinterpret_cast(data[id]);
+      std::memcpy(buffer->data, &sizes[id], sizeof(uint64_t));
+    });
+    break;
+  }
+  case RPC_READ_FGETS: {
+    uint64_t sizes[lane_size] = {0};
+    void *data[lane_size] = {nullptr};
+    port->recv([&](rpc::Buffer *buffer, uint32_t id) {
+      data[id] = new char[buffer->data[0]];
+      const char *str =
+          fgets(reinterpret_cast(data[id]), buffer->data[0],
+                file::to_stream(buffer->data[1]));
+      sizes[id] = !str ? 0 : std::strlen(str) + 1;
+    });
+    port->send_n(data, sizes);
+    for (uint32_t id = 0; id < lane_size; ++id)
+      if (data[id])
+        delete[] reinterpret_cast(data[id]);
+    break;
+  }
+  case RPC_OPEN_FILE: {
+    uint64_t sizes[lane_size] = {0};
+    void *paths[lane_size] = {nullptr};
+    port->recv_n(paths, sizes, [&](uint64_t size) { return new char[size]; });
+    port->recv_and_send([&](rpc::Buffer *buffer, uint32_t id) {
+      FILE *file = fopen(reinterpret_cast(paths[id]),
+                         reinterpret_cast(buffer->data));
+      buffer->data[0] = reinterpret_cast(file);
+    });
+    break;
+  }
+  case RPC_CLOSE_FILE: {
+    port->recv_and_send([&](rpc::Buffer *buffer, uint32_t id) {
+      FILE *file = reinterpret_cast(buffer->data[0]);
+      buffer->data[0] = fclose(file);
+    });
+    break;
+  }
+  case RPC_EXIT: {
+    // Send a response to the client to signal that we are ready to exit.
+    port->recv_and_send([](rpc::Buffer *) {});
+    port->recv([](rpc::Buffer *buffer) {
+      int status = 0;
+      std::memcpy(&status, buffer->data, sizeof(int));
+      exit(status);
+    });
+    break;
+  }
+  case RPC_ABORT: {
+    // Send a response to the client to signal that we are ready to abort.
+    port->recv_and_send([](rpc::Buffer *) {});
+    port->recv([](rpc::Buffer *) {});
+    abort();
+    break;
+  }
+  case RPC_HOST_CALL: {
+    uint64_t sizes[lane_size] = {0};
+    void *args[lane_size] = {nullptr};
+    port->recv_n(args, sizes, [&](uint64_t size) { return new char[size]; });
+    port->recv([&](rpc::Buffer *buffer, uint32_t id) {
+      reinterpret_cast(buffer->data[0])(args[id]);
+    });
+    port->send([&](rpc::Buffer *, uint32_t id) {
+      delete[] reinterpret_cast(args[id]);
+    });
+    break;
+  }
+  case RPC_FEOF: {
+    port->recv_and_send([](rpc::Buffer *buffer) {
+      buffer->data[0] = feof(file::to_stream(buffer->data[0]));
+    });
+    break;
+  }
+  case RPC_FERROR: {
+    port->recv_and_send([](rpc::Buffer *buffer) {
+      buffer->data[0] = ferror(file::to_stream(buffer->data[0]));
+    });
+    break;
+  }
+  case RPC_CLEARERR: {
+    port->recv_and_send([](rpc::Buffer *buffer) {
+      clearerr(file::to_stream(buffer->data[0]));
+    });
+    break;
+  }
+  case RPC_FSEEK: {
+    port->recv_and_send([](rpc::Buffer *buffer) {
+      buffer->data[0] = fseek(file::to_stream(buffer->data[0]),
+                              static_cast(buffer->data[1]),
+                              static_cast(buffer->data[2]));
+    });
+    break;
+  }
+  case RPC_FTELL: {
+    port->recv_and_send([](rpc::Buffer *buffer) {
+      buffer->data[0] = ftell(file::to_stream(buffer->data[0]));
+    });
+    break;
+  }
+  case RPC_FFLUSH: {
+    port->recv_and_send([](rpc::Buffer *buffer) {
+      buffer->data[0] = fflush(file::to_stream(buffer->data[0]));
+    });
+    break;
+  }
+  case RPC_UNGETC: {
+    port->recv_and_send([](rpc::Buffer *buffer) {
+      buffer->data[0] = ungetc(static_cast(buffer->data[0]),
+                               file::to_stream(buffer->data[1]));
+    });
+    break;
+  }
+  case RPC_NOOP: {
+    port->recv([](rpc::Buffer *) {});
+    break;
+  }
+  default: {
+    auto handler =
+        callbacks.find(static_cast(port->get_opcode()));
+
+    // We error out on an unhandled opcode.
+    if (handler == callbacks.end())
+      return RPC_STATUS_UNHANDLED_OPCODE;
+
+    // Invoke the registered callback with a reference to the port.
+    void *data =
+        callback_data.at(static_cast(port->get_opcode()));
+    rpc_port_t port_ref{reinterpret_cast(&*port), lane_size};
+    (handler->second)(port_ref, data);
+  }
   }
 
-  std::variant>,
-               std::unique_ptr>,
-               std::unique_ptr>>
-      server;
-};
+  // Increment the index so we start the scan after this port.
+  index = port->get_index() + 1;
+  port->close();
+
+  return RPC_STATUS_CONTINUE;
+}
 
 struct Device {
-  template 
-  Device(uint32_t num_ports, void *buffer, std::unique_ptr &&server)
-      : buffer(buffer), server(std::move(server)), client(num_ports, buffer) {}
+  Device(uint32_t lane_size, uint32_t num_ports, void *buffer)
+      : lane_size(lane_size), buffer(buffer), server(num_ports, buffer),
+        client(num_ports, buffer) {}
+
+  rpc_status_t handle_server(uint32_t &index) {
+    switch (lane_size) {
+    case 1:
+      return handle_server_impl<1>(server, callbacks, callback_data, index);
+    case 32:
+      return handle_server_impl<32>(server, callbacks, callback_data, index);
+    case 64:
+      return handle_server_impl<64>(server, callbacks, callback_data, index);
+    default:
+      return RPC_STATUS_INVALID_LANE_SIZE;
+    }
+  }
+
+  uint32_t lane_size;
   void *buffer;
-  Server server;
+  rpc::Server server;
   rpc::Client client;
   std::unordered_map callbacks;
   std::unordered_map callback_data;
@@ -287,24 +276,6 @@ rpc_status_t rpc_shutdown(void) {
   return RPC_STATUS_SUCCESS;
 }
 
-template 
-rpc_status_t server_init_impl(uint32_t device_id, uint64_t num_ports,
-                              rpc_alloc_ty alloc, void *data) {
-  uint64_t size = rpc::Server::allocation_size(num_ports);
-  void *buffer = alloc(size, data);
-
-  if (!buffer)
-    return RPC_STATUS_ERROR;
-
-  state->devices[device_id] = std::make_unique(
-      num_ports, buffer,
-      std::make_unique>(num_ports, buffer));
-  if (!state->devices[device_id])
-    return RPC_STATUS_ERROR;
-
-  return RPC_STATUS_SUCCESS;
-}
-
 rpc_status_t rpc_server_init(uint32_t device_id, uint64_t num_ports,
                              uint32_t lane_size, rpc_alloc_ty alloc,
                              void *data) {
@@ -312,28 +283,20 @@ rpc_status_t rpc_server_init(uint32_t device_id, uint64_t num_ports,
     return RPC_STATUS_NOT_INITIALIZED;
   if (device_id >= state->num_devices)
     return RPC_STATUS_OUT_OF_RANGE;
+  if (lane_size != 1 && lane_size != 32 && lane_size != 64)
+    return RPC_STATUS_INVALID_LANE_SIZE;
 
   if (!state->devices[device_id]) {
-    switch (lane_size) {
-    case 1:
-      if (rpc_status_t err =
-              server_init_impl<1>(device_id, num_ports, alloc, data))
-        return err;
-      break;
-    case 32: {
-      if (rpc_status_t err =
-              server_init_impl<32>(device_id, num_ports, alloc, data))
-        return err;
-      break;
-    }
-    case 64:
-      if (rpc_status_t err =
-              server_init_impl<64>(device_id, num_ports, alloc, data))
-        return err;
-      break;
-    default:
-      return RPC_STATUS_INVALID_LANE_SIZE;
-    }
+    uint64_t size = rpc::Server::allocation_size(lane_size, num_ports);
+    void *buffer = alloc(size, data);
+
+    if (!buffer)
+      return RPC_STATUS_ERROR;
+
+    state->devices[device_id] =
+        std::make_unique(lane_size, num_ports, buffer);
+    if (!state->devices[device_id])
+      return RPC_STATUS_ERROR;
   }
 
   return RPC_STATUS_SUCCESS;
@@ -365,9 +328,8 @@ rpc_status_t rpc_handle_server(uint32_t device_id) {
 
   uint32_t index = 0;
   for (;;) {
-    auto &device = *state->devices[device_id];
-    rpc_status_t status = device.server.handle_server(
-        device.callbacks, device.callback_data, index);
+    Device &device = *state->devices[device_id];
+    rpc_status_t status = device.handle_server(index);
     if (status != RPC_STATUS_CONTINUE)
       return status;
   }
@@ -396,26 +358,26 @@ const void *rpc_get_client_buffer(uint32_t device_id) {
 
 uint64_t rpc_get_client_size() { return sizeof(rpc::Client); }
 
-using ServerPort = std::variant::Port *>;
+using ServerPort = std::variant;
 
 ServerPort get_port(rpc_port_t ref) {
-  return reinterpret_cast::Port *>(ref.handle);
+  return reinterpret_cast(ref.handle);
 }
 
 void rpc_send(rpc_port_t ref, rpc_port_callback_ty callback, void *data) {
-  auto port = reinterpret_cast::Port *>(ref.handle);
+  auto port = reinterpret_cast(ref.handle);
   port->send([=](rpc::Buffer *buffer) {
     callback(reinterpret_cast(buffer), data);
   });
 }
 
 void rpc_send_n(rpc_port_t ref, const void *const *src, uint64_t *size) {
-  auto port = reinterpret_cast::Port *>(ref.handle);
+  auto port = reinterpret_cast(ref.handle);
   port->send_n(src, size);
 }
 
 void rpc_recv(rpc_port_t ref, rpc_port_callback_ty callback, void *data) {
-  auto port = reinterpret_cast::Port *>(ref.handle);
+  auto port = reinterpret_cast(ref.handle);
   port->recv([=](rpc::Buffer *buffer) {
     callback(reinterpret_cast(buffer), data);
   });
@@ -423,14 +385,14 @@ void rpc_recv(rpc_port_t ref, rpc_port_callback_ty callback, void *data) {
 
 void rpc_recv_n(rpc_port_t ref, void **dst, uint64_t *size, rpc_alloc_ty alloc,
                 void *data) {
-  auto port = reinterpret_cast::Port *>(ref.handle);
+  auto port = reinterpret_cast(ref.handle);
   auto alloc_fn = [=](uint64_t size) { return alloc(size, data); };
   port->recv_n(dst, size, alloc_fn);
 }
 
 void rpc_recv_and_send(rpc_port_t ref, rpc_port_callback_ty callback,
                        void *data) {
-  auto port = reinterpret_cast::Port *>(ref.handle);
+  auto port = reinterpret_cast(ref.handle);
   port->recv_and_send([=](rpc::Buffer *buffer) {
     callback(reinterpret_cast(buffer), data);
   });
-- 
GitLab


From 03c6c73b022415378f2efbc13398bdb2c22d3180 Mon Sep 17 00:00:00 2001
From: Florian Mayer 
Date: Fri, 8 Mar 2024 13:28:18 -0800
Subject: [PATCH 665/929] [NFC] [hwasan] remove unused method

---
 llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index 6bae679e11be..422406e46bdb 100644
--- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -316,7 +316,6 @@ private:
     Value *PtrTag = nullptr;
     Value *MemTag = nullptr;
   };
-  void setSSI(const StackSafetyGlobalInfo *S) { SSI = S; }
 
   void initializeModule();
   void createHwasanCtorComdat();
-- 
GitLab


From 3ceebcb48135612b8d90500f65693398f7e526cd Mon Sep 17 00:00:00 2001
From: Peiming Liu 
Date: Fri, 8 Mar 2024 13:30:34 -0800
Subject: [PATCH 666/929] =?UTF-8?q?[mlir]=20tentative=20fix=20for=20"'GTES?=
 =?UTF-8?q?T=5FNO=5FLLVM=5FSUPPORT'=20is=20not=20defined"=20war=E2=80=A6?=
 =?UTF-8?q?=20(#84539)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…ning when unittest is enabled
---
 mlir/unittests/CMakeLists.txt | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/mlir/unittests/CMakeLists.txt b/mlir/unittests/CMakeLists.txt
index d0e222091c9f..6fad249a0b2f 100644
--- a/mlir/unittests/CMakeLists.txt
+++ b/mlir/unittests/CMakeLists.txt
@@ -1,5 +1,8 @@
 set_target_properties(MLIRUnitTests PROPERTIES FOLDER "MLIR Tests")
 
+# To silence warning caused by Wundef.
+add_definitions(-DGTEST_NO_LLVM_SUPPORT=0)
+
 function(add_mlir_unittest test_dirname)
   add_unittest(MLIRUnitTests ${test_dirname} ${ARGN})
 endfunction()
-- 
GitLab


From 1cf428a05a62711200d37a1b12722ca16c33a6ea Mon Sep 17 00:00:00 2001
From: Florian Mayer 
Date: Fri, 8 Mar 2024 13:46:08 -0800
Subject: [PATCH 667/929] Skip MemtagBasicDeathTest#Unsupported when running
 with HWASan (#84243)

Tested in AOSP.
---
 compiler-rt/lib/scudo/standalone/tests/memtag_test.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/compiler-rt/lib/scudo/standalone/tests/memtag_test.cpp b/compiler-rt/lib/scudo/standalone/tests/memtag_test.cpp
index fd277f962a9a..37a18858e67c 100644
--- a/compiler-rt/lib/scudo/standalone/tests/memtag_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/memtag_test.cpp
@@ -12,12 +12,17 @@
 #include "platform.h"
 #include "tests/scudo_unit_test.h"
 
+extern "C" void __hwasan_init() __attribute__((weak));
+
 #if SCUDO_LINUX
 namespace scudo {
 
 TEST(MemtagBasicDeathTest, Unsupported) {
   if (archSupportsMemoryTagging())
     GTEST_SKIP();
+  // Skip when running with HWASan.
+  if (&__hwasan_init != 0)
+    GTEST_SKIP();
 
   EXPECT_DEATH(archMemoryTagGranuleSize(), "not supported");
   EXPECT_DEATH(untagPointer((uptr)0), "not supported");
-- 
GitLab


From e93489c434da5fd93ed54c3c41b0c06f718bb8ff Mon Sep 17 00:00:00 2001
From: Jakub Kuderski 
Date: Fri, 8 Mar 2024 17:43:56 -0500
Subject: [PATCH 668/929] [mlir] Add missing build deps for Mesh transforms
 (#84581)

---
 mlir/lib/Dialect/Mesh/Transforms/CMakeLists.txt   | 2 ++
 utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/mlir/lib/Dialect/Mesh/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Mesh/Transforms/CMakeLists.txt
index 28af82044007..212ea6d6948b 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Mesh/Transforms/CMakeLists.txt
@@ -17,7 +17,9 @@ add_mlir_dialect_library(MLIRMeshTransforms
   MLIRArithDialect
   MLIRArithUtils
   MLIRControlFlowDialect
+  MLIRControlFlowInterfaces
   MLIRFuncDialect
+  MLIRFunctionInterfaces
   MLIRIR
   MLIRMeshDialect
   MLIRPass
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index cadcbcc5a092..3be8f34ae0f4 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -3410,8 +3410,10 @@ cc_library(
         ":ArithDialect",
         ":ArithUtils",
         ":ControlFlowDialect",
+        ":ControlFlowInterfaces",
         ":DialectUtils",
         ":FuncDialect",
+        ":FunctionInterfaces",
         ":IR",
         ":MeshDialect",
         ":MeshShardingInterface",
-- 
GitLab


From cb6ff746e0c7b9218b6f5c11db44162cacd623a4 Mon Sep 17 00:00:00 2001
From: Kojo Acquah 
Date: Fri, 8 Mar 2024 22:50:13 +0000
Subject: [PATCH 669/929] [mlir][ArmNeon] Implements LowerVectorToArmNeon
 Pattern for SMMLA (#81895)

This patch adds a the `LowerVectorToArmNeonPattern` patterns to the
ArmNeon.

This pattern inspects `vector.contract` ops that can be 1-1 mapped to an
`arm.neon.smmla` intrinsic. The contract ops must be separated into
tiles who's inputs must fit that of a single smmla op (`2x8xi32` inputs
and `2x2xi32` output). The `vector.contract` inputs must be sign
extended from narrow types (<=i8) to be converted. If all conditions are
met, an smmla op is inserted with additional `vector.shape_casts` to
handle linearizing the input and output dimension.
---
 .../include/mlir/Dialect/ArmNeon/Transforms.h |  21 +++
 .../Conversion/VectorToLLVM/CMakeLists.txt    |   1 +
 mlir/lib/Dialect/ArmNeon/CMakeLists.txt       |  15 +-
 mlir/lib/Dialect/ArmNeon/IR/CMakeLists.txt    |  13 ++
 .../Dialect/ArmNeon/Transforms/CMakeLists.txt |  14 ++
 .../LowerContractionToSMMLAPattern.cpp        | 146 ++++++++++++++++++
 .../Dialect/ArmNeon/lower-to-arm-neon.mlir    |  42 +++++
 mlir/test/lib/Dialect/ArmNeon/CMakeLists.txt  |  13 ++
 .../Dialect/ArmNeon/TestLowerToArmNeon.cpp    |  61 ++++++++
 mlir/test/lib/Dialect/CMakeLists.txt          |   1 +
 mlir/tools/mlir-opt/CMakeLists.txt            |   1 +
 mlir/tools/mlir-opt/mlir-opt.cpp              |   2 +
 .../llvm-project-overlay/mlir/BUILD.bazel     |  21 +++
 13 files changed, 338 insertions(+), 13 deletions(-)
 create mode 100644 mlir/include/mlir/Dialect/ArmNeon/Transforms.h
 create mode 100644 mlir/lib/Dialect/ArmNeon/IR/CMakeLists.txt
 create mode 100644 mlir/lib/Dialect/ArmNeon/Transforms/CMakeLists.txt
 create mode 100644 mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToSMMLAPattern.cpp
 create mode 100644 mlir/test/Dialect/ArmNeon/lower-to-arm-neon.mlir
 create mode 100644 mlir/test/lib/Dialect/ArmNeon/CMakeLists.txt
 create mode 100644 mlir/test/lib/Dialect/ArmNeon/TestLowerToArmNeon.cpp

diff --git a/mlir/include/mlir/Dialect/ArmNeon/Transforms.h b/mlir/include/mlir/Dialect/ArmNeon/Transforms.h
new file mode 100644
index 000000000000..49cad22defec
--- /dev/null
+++ b/mlir/include/mlir/Dialect/ArmNeon/Transforms.h
@@ -0,0 +1,21 @@
+//===- Transforms.h - ArmNeon Transformation Entrypoints --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_ARMNEON_TRANSFORMS_H
+#define MLIR_DIALECT_ARMNEON_TRANSFORMS_H
+
+namespace mlir {
+
+namespace arm_neon {
+void populateLowerContractionToSMMLAPatternPatterns(
+    RewritePatternSet &patterns);
+} // namespace arm_neon
+
+} // namespace mlir
+
+#endif // MLIR_DIALECT_ARMNEON_TRANSFORMS_H
diff --git a/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt b/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt
index 5fbb50f62395..a0fce139f274 100644
--- a/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt
+++ b/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt
@@ -34,6 +34,7 @@ add_mlir_conversion_library(MLIRVectorToLLVMPass
   MLIRVectorToLLVM
 
   MLIRArmNeonDialect
+  MLIRArmNeonTransforms
   MLIRArmSMEDialect
   MLIRArmSMETransforms
   MLIRArmSVEDialect
diff --git a/mlir/lib/Dialect/ArmNeon/CMakeLists.txt b/mlir/lib/Dialect/ArmNeon/CMakeLists.txt
index 060b6df1b334..9f57627c321f 100644
--- a/mlir/lib/Dialect/ArmNeon/CMakeLists.txt
+++ b/mlir/lib/Dialect/ArmNeon/CMakeLists.txt
@@ -1,13 +1,2 @@
-add_mlir_dialect_library(MLIRArmNeonDialect
-  IR/ArmNeonDialect.cpp
-
-  ADDITIONAL_HEADER_DIRS
-  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/ArmNeon
-
-  DEPENDS
-  MLIRArmNeonIncGen
-
-  LINK_LIBS PUBLIC
-  MLIRIR
-  MLIRSideEffectInterfaces
-  )
+add_subdirectory(IR)
+add_subdirectory(Transforms)
diff --git a/mlir/lib/Dialect/ArmNeon/IR/CMakeLists.txt b/mlir/lib/Dialect/ArmNeon/IR/CMakeLists.txt
new file mode 100644
index 000000000000..b04919a3a318
--- /dev/null
+++ b/mlir/lib/Dialect/ArmNeon/IR/CMakeLists.txt
@@ -0,0 +1,13 @@
+add_mlir_dialect_library(MLIRArmNeonDialect
+  ArmNeonDialect.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/ArmNeon
+
+  DEPENDS
+  MLIRArmNeonIncGen
+
+  LINK_LIBS PUBLIC
+  MLIRIR
+  MLIRSideEffectInterfaces
+  )
diff --git a/mlir/lib/Dialect/ArmNeon/Transforms/CMakeLists.txt b/mlir/lib/Dialect/ArmNeon/Transforms/CMakeLists.txt
new file mode 100644
index 000000000000..84fb1b0116d2
--- /dev/null
+++ b/mlir/lib/Dialect/ArmNeon/Transforms/CMakeLists.txt
@@ -0,0 +1,14 @@
+add_mlir_dialect_library(MLIRArmNeonTransforms
+  LowerContractionToSMMLAPattern.cpp
+
+  DEPENDS
+  MLIRArmNeonIncGen
+
+  LINK_LIBS PUBLIC
+  MLIRArmNeonDialect
+  MLIRFuncDialect
+  MLIRVectorDialect
+  MLIRIR
+  MLIRLLVMCommonConversion
+  MLIRLLVMDialect
+  )
diff --git a/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToSMMLAPattern.cpp b/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToSMMLAPattern.cpp
new file mode 100644
index 000000000000..47c84708f3c3
--- /dev/null
+++ b/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToSMMLAPattern.cpp
@@ -0,0 +1,146 @@
+//===- LowerContractionToSMMLAPattern.cpp - Contract to SMMLA ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements lowering patterns from vector.contract to
+// arm_neon.intr.smmla
+//
+//===---
+
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
+#include "mlir/Dialect/ArmNeon/Transforms.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Support/LogicalResult.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+
+#define DEBUG_TYPE "lower-contract-to-arm-neon"
+
+using namespace mlir;
+using namespace mlir::arm_neon;
+
+namespace {
+
+/// Return the shaped type with new element type.
+static Type matchContainerType(Type element, Type container) {
+  if (auto shapedTy = dyn_cast(container)) {
+    return shapedTy.clone(element);
+  }
+  return element;
+}
+
+/// Lowering from a single vector::contractOp directly to the arm neon smmla
+/// intrinsic. The shapes of the contract and intrinsic must match.
+class LowerContractionToSMMLAPattern
+    : public OpRewritePattern {
+public:
+  using OpRewritePattern::OpRewritePattern;
+  LogicalResult matchAndRewrite(vector::ContractionOp op,
+                                PatternRewriter &rewriter) const override {
+    Location loc = op.getLoc();
+    Value lhs = op.getLhs();
+    Value rhs = op.getRhs();
+    Value res = op.getAcc();
+
+    // Check index maps that represent M N K in contract.
+    auto indexingMaps = op.getIndexingMapsArray();
+    if (llvm::any_of(indexingMaps, [](mlir::AffineMap affineMap) {
+          return affineMap.isPermutation() || affineMap.getNumDims() != 3 ||
+                 affineMap.getNumResults() != 2;
+        })) {
+      return failure();
+    }
+
+    // Check iterator types for contract.
+    auto iteratorTypes = op.getIteratorTypesArray();
+    if (iteratorTypes.size() != 3 ||
+        iteratorTypes[0] != vector::IteratorType::parallel ||
+        iteratorTypes[1] != vector::IteratorType::parallel ||
+        iteratorTypes[2] != vector::IteratorType::reduction) {
+      return failure();
+    }
+
+    // Check the tile size by mapping the dimensions of the contract.
+    mlir::VectorType lhsType = op.getLhsType();
+    mlir::VectorType rhsType = op.getRhsType();
+    auto dimM = lhsType.getDimSize(0);
+    auto dimN = rhsType.getDimSize(0);
+    auto dimK = lhsType.getDimSize(1);
+    if (rhsType.getDimSize(1) != dimK || dimM != 2 || dimN != 2 || dimK != 8) {
+      return failure();
+    }
+
+    // Check two extsi inputs Rhs Lhs for contract.
+    arith::ExtSIOp origLhsExtOp =
+        dyn_cast_or_null(lhs.getDefiningOp());
+    arith::ExtSIOp origRhsExtOp =
+        dyn_cast_or_null(rhs.getDefiningOp());
+    if (!origLhsExtOp || !origRhsExtOp) {
+      return failure();
+    }
+
+    // Match any iX to i32 for X<8 then turn into an i8 output. Feed into
+    // following neon instruction. Check inputs for extsi are <=i8
+    Value extsiLhs;
+    Value extsiRhs;
+    if (auto lhsExtInType =
+            origLhsExtOp.getIn().getType().dyn_cast()) {
+      if (lhsExtInType.getElementTypeBitWidth() <= 8) {
+        Type targetLhsExtTy =
+            matchContainerType(rewriter.getI8Type(), lhsExtInType);
+        extsiLhs = rewriter.createOrFold(loc, targetLhsExtTy,
+                                                         origLhsExtOp.getIn());
+      }
+    }
+    if (auto rhsExtInType =
+            origRhsExtOp.getIn().getType().dyn_cast()) {
+      if (rhsExtInType.getElementTypeBitWidth() <= 8) {
+        Type targetRhsExtTy =
+            matchContainerType(rewriter.getI8Type(), rhsExtInType);
+        extsiRhs = rewriter.createOrFold(loc, targetRhsExtTy,
+                                                         origRhsExtOp.getIn());
+      }
+    }
+
+    if (!extsiLhs || !extsiRhs) {
+      return failure();
+    }
+
+    // Collapse to 1D vectors required by smmla intrinsic
+    auto collapsedInputType = VectorType::get(
+        {16}, extsiLhs.getType().cast().getElementType());
+    auto collapsedOutputType =
+        VectorType::get({4}, res.getType().cast().getElementType());
+    auto collapsedLhs = rewriter.createOrFold(
+        extsiLhs.getLoc(), collapsedInputType, extsiLhs);
+    auto collapsedRhs = rewriter.createOrFold(
+        extsiRhs.getLoc(), collapsedInputType, extsiRhs);
+    auto collapsedRes = rewriter.createOrFold(
+        res.getLoc(), collapsedOutputType, res);
+
+    // Replace the contract with a neon op
+    auto smmlaOp = rewriter.createOrFold(
+        op.getLoc(), collapsedRes.getType(), collapsedRes, collapsedLhs,
+        collapsedRhs);
+
+    // Reshape output back to 2D
+    rewriter.replaceOpWithNewOp(op, op.getResultType(),
+                                                     smmlaOp);
+    return success();
+  }
+};
+
+} // namespace
+
+void mlir::arm_neon::populateLowerContractionToSMMLAPatternPatterns(
+    RewritePatternSet &patterns) {
+  MLIRContext *context = patterns.getContext();
+  patterns.add(context, /*benefit=*/1);
+}
diff --git a/mlir/test/Dialect/ArmNeon/lower-to-arm-neon.mlir b/mlir/test/Dialect/ArmNeon/lower-to-arm-neon.mlir
new file mode 100644
index 000000000000..cba7b00ba77a
--- /dev/null
+++ b/mlir/test/Dialect/ArmNeon/lower-to-arm-neon.mlir
@@ -0,0 +1,42 @@
+// RUN: mlir-opt -test-lower-to-arm-neon -verify-diagnostics -split-input-file %s | FileCheck %s
+
+// CHECK-LABEL: test_lower_vector_arm_neon_mixed_types
+// CHECK-SAME:    %[[A0:.*]]: vector<2x8xi8>, %[[A1:.*]]: vector<2x8xi4>, %[[A2:.*]]: vector<2x2xi32>
+// CHECK-DAG: %[[D0:.*]] = arith.extsi %[[A1]] : vector<2x8xi4> to vector<2x8xi8>
+// CHECK-DAG: %[[D1:.*]] = vector.shape_cast %[[A0]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-DAG: %[[D2:.*]] = vector.shape_cast %[[D0]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-DAG: %[[D3:.*]] = vector.shape_cast %[[A2]] : vector<2x2xi32> to vector<4xi32>
+// CHECK-DAG: %[[D4:.*]] = arm_neon.intr.smmla %[[D3]], %[[D1]], %[[D2]] : vector<16xi8> to vector<4xi32>
+// CHECK-DAG: %[[D5:.*]] = vector.shape_cast %[[D4]] : vector<4xi32> to vector<2x2xi32>
+func.func @test_lower_vector_arm_neon_mixed_types(%lhs: vector<2x8xi8>, %rhs: vector<2x8xi4>, %acc : vector<2x2xi32>) -> vector<2x2xi32> {
+  %lhs_extsi = arith.extsi %lhs : vector<2x8xi8> to vector<2x8xi32>
+  %rhs_extsi = arith.extsi %rhs : vector<2x8xi4> to vector<2x8xi32>
+  %res = vector.contract {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>, affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1)>], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind} %lhs_extsi, %rhs_extsi, %acc : vector<2x8xi32>, vector<2x8xi32> into vector<2x2xi32>
+  return %res : vector<2x2xi32>
+}
+
+// -----
+
+// CHECK-LABEL: test_lower_vector_arm_neon_same_types
+// CHECK-SAME:    %[[A0:.*]]: vector<2x8xi8>, %[[A1:.*]]: vector<2x8xi8>, %[[A2:.*]]: vector<2x2xi32>
+// CHECK-DAG: %[[D0:.*]] = vector.shape_cast %[[A0]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-DAG: %[[D1:.*]] = vector.shape_cast %[[A1]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-DAG: %[[D2:.*]] = vector.shape_cast %[[A2]] : vector<2x2xi32> to vector<4xi32>
+// CHECK-DAG: %[[D3:.*]] = arm_neon.intr.smmla %[[D2]], %[[D0]], %[[D1]] : vector<16xi8> to vector<4xi32>
+// CHECK-DAG: %[[D4:.*]] = vector.shape_cast %[[D3]] : vector<4xi32> to vector<2x2xi32>
+func.func @test_lower_vector_arm_neon_same_types(%lhs: vector<2x8xi8>, %rhs: vector<2x8xi8>, %acc : vector<2x2xi32>) -> vector<2x2xi32> {
+  %lhs_extsi = arith.extsi %lhs : vector<2x8xi8> to vector<2x8xi32>
+  %rhs_extsi = arith.extsi %rhs : vector<2x8xi8> to vector<2x8xi32>
+  %res = vector.contract {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>, affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1)>], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind} %lhs_extsi, %rhs_extsi, %acc : vector<2x8xi32>, vector<2x8xi32> into vector<2x2xi32>
+  return %res : vector<2x2xi32>
+}
+
+// -----
+
+// CHECK-LABEL: test_lower_vector_arm_neon_without_extsi
+// CHECK-SAME:    %[[A0:.*]]: vector<2x8xi32>, %[[A1:.*]]: vector<2x8xi32>, %[[A2:.*]]: vector<2x2xi32>
+// CHECK-DAG: %[[D0:.*]] = vector.contract
+func.func @test_lower_vector_arm_neon_without_extsi(%lhs: vector<2x8xi32>, %rhs: vector<2x8xi32>, %acc : vector<2x2xi32>) -> vector<2x2xi32> {
+  %res = vector.contract {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>, affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1)>], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind} %lhs, %rhs, %acc : vector<2x8xi32>, vector<2x8xi32> into vector<2x2xi32>
+  return %res : vector<2x2xi32>
+}
diff --git a/mlir/test/lib/Dialect/ArmNeon/CMakeLists.txt b/mlir/test/lib/Dialect/ArmNeon/CMakeLists.txt
new file mode 100644
index 000000000000..21548ca57701
--- /dev/null
+++ b/mlir/test/lib/Dialect/ArmNeon/CMakeLists.txt
@@ -0,0 +1,13 @@
+# Exclude tests from libMLIR.so
+add_mlir_library(MLIRArmNeonTestPasses
+  TestLowerToArmNeon.cpp
+
+  EXCLUDE_FROM_LIBMLIR
+
+  LINK_LIBS PUBLIC
+  MLIRArmNeonDialect
+  MLIRArmNeonTransforms
+  MLIRIR
+  MLIRPass
+  MLIRTransforms
+  )
diff --git a/mlir/test/lib/Dialect/ArmNeon/TestLowerToArmNeon.cpp b/mlir/test/lib/Dialect/ArmNeon/TestLowerToArmNeon.cpp
new file mode 100644
index 000000000000..2e5a4a988882
--- /dev/null
+++ b/mlir/test/lib/Dialect/ArmNeon/TestLowerToArmNeon.cpp
@@ -0,0 +1,61 @@
+//===- TestLowerToArmNeon.cpp - Test lowering to ArmNeon as a sink pass -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a pass for testing the lowering to ArmNeon as a
+// generally usable sink pass.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
+#include "mlir/Dialect/ArmNeon/Transforms.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Pass/PassManager.h"
+#include "mlir/Support/LogicalResult.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+
+#define PASS_NAME "test-lower-to-arm-neon"
+
+using namespace mlir;
+using namespace mlir::arm_neon;
+
+namespace {
+struct TestLowerToArmNeon
+    : public PassWrapper> {
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestLowerToArmNeon)
+
+  StringRef getArgument() const final { return PASS_NAME; }
+  StringRef getDescription() const final { return "Tests lower to arm Neon."; }
+  TestLowerToArmNeon() = default;
+  TestLowerToArmNeon(const TestLowerToArmNeon &pass) = default;
+
+  void getDependentDialects(DialectRegistry ®istry) const override {
+    registry.insert();
+  }
+
+  void runOnOperation() override;
+};
+
+} // namespace
+
+void TestLowerToArmNeon::runOnOperation() {
+  MLIRContext *context = &getContext();
+  RewritePatternSet patterns(context);
+  populateLowerContractionToSMMLAPatternPatterns(patterns);
+  if (failed(applyPatternsAndFoldGreedily(getOperation(), std::move(patterns))))
+    return signalPassFailure();
+}
+
+namespace mlir {
+namespace test {
+
+void registerTestLowerToArmNeon() { PassRegistration(); }
+
+} // namespace test
+} // namespace mlir
diff --git a/mlir/test/lib/Dialect/CMakeLists.txt b/mlir/test/lib/Dialect/CMakeLists.txt
index e20cd4473a35..29fb4441a24f 100644
--- a/mlir/test/lib/Dialect/CMakeLists.txt
+++ b/mlir/test/lib/Dialect/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_subdirectory(Affine)
 add_subdirectory(Arith)
+add_subdirectory(ArmNeon)
 add_subdirectory(ArmSME)
 add_subdirectory(Bufferization)
 add_subdirectory(ControlFlow)
diff --git a/mlir/tools/mlir-opt/CMakeLists.txt b/mlir/tools/mlir-opt/CMakeLists.txt
index 4389840af10e..e8091bca3326 100644
--- a/mlir/tools/mlir-opt/CMakeLists.txt
+++ b/mlir/tools/mlir-opt/CMakeLists.txt
@@ -17,6 +17,7 @@ if(MLIR_INCLUDE_TESTS)
     MLIRTestFuncToLLVM
     MLIRAffineTransformsTestPasses
     MLIRArithTestPasses
+    MLIRArmNeonTestPasses
     MLIRArmSMETestPasses
     MLIRBufferizationTestPasses
     MLIRControlFlowTestPasses
diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp
index e4d05631c35f..82b3881792bf 100644
--- a/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -111,6 +111,7 @@ void registerTestLoopFusion();
 void registerTestCFGLoopInfoPass();
 void registerTestLoopMappingPass();
 void registerTestLoopUnrollingPass();
+void registerTestLowerToArmNeon();
 void registerTestLowerToArmSME();
 void registerTestLowerToLLVM();
 void registerTestMakeIsolatedFromAbovePass();
@@ -237,6 +238,7 @@ void registerTestPasses() {
   mlir::test::registerTestCFGLoopInfoPass();
   mlir::test::registerTestLoopMappingPass();
   mlir::test::registerTestLoopUnrollingPass();
+  mlir::test::registerTestLowerToArmNeon();
   mlir::test::registerTestLowerToArmSME();
   mlir::test::registerTestLowerToLLVM();
   mlir::test::registerTestMakeIsolatedFromAbovePass();
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 3be8f34ae0f4..8da2b51ffc99 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -1931,6 +1931,27 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "ArmNeonTransforms",
+    srcs = ["lib/Dialect/ArmNeon/Transforms/LowerVectorToArmNeon.cpp"],
+    hdrs = ["include/mlir/Dialect/ArmNeon/Transforms.h"],
+    includes = ["include"],
+    deps = [
+        ":ArithDialect",
+        ":ArmNeonIncGen",
+        ":ArmNeonDialect",
+        ":FuncDialect",
+        ":IR",
+        ":LLVMDialect",
+        ":SideEffectInterfaces",
+        ":Support",
+        ":VectorDialect",
+        ":Transforms",
+        "//llvm:Core",
+        "//llvm:Support",
+    ],
+)
+
 gentbl_cc_library(
     name = "ArmNeonConversionIncGen",
     tbl_outs = [
-- 
GitLab


From c22828991e7ca7b99048761c078252e94403ba6e Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh 
Date: Fri, 8 Mar 2024 15:13:20 -0800
Subject: [PATCH 670/929] =?UTF-8?q?Revert=20"[builtins]=20Disable=20COMPIL?=
 =?UTF-8?q?ER=5FRT=5FCRT=5FUSE=5FEH=5FFRAME=5FREGISTRY=20by=20d=E2=80=A6?=
 =?UTF-8?q?=20(#84580)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…efault (#83201)"

This reverts commit 062cfada643c1aa48a1bb81894e2920d390fe8cf. See issue
#84574.
---
 compiler-rt/lib/builtins/CMakeLists.txt    |  2 +-
 compiler-rt/test/builtins/Unit/ctor_dtor.c | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt
index 3f2c0f043639..f9611574a562 100644
--- a/compiler-rt/lib/builtins/CMakeLists.txt
+++ b/compiler-rt/lib/builtins/CMakeLists.txt
@@ -917,7 +917,7 @@ cmake_dependent_option(COMPILER_RT_BUILD_CRT "Build crtbegin.o/crtend.o" ON "COM
 if (COMPILER_RT_BUILD_CRT)
   add_compiler_rt_component(crt)
 
-  option(COMPILER_RT_CRT_USE_EH_FRAME_REGISTRY "Use eh_frame in crtbegin.o/crtend.o" OFF)
+  option(COMPILER_RT_CRT_USE_EH_FRAME_REGISTRY "Use eh_frame in crtbegin.o/crtend.o" ON)
 
   include(CheckSectionExists)
   check_section_exists(".init_array" COMPILER_RT_HAS_INITFINI_ARRAY
diff --git a/compiler-rt/test/builtins/Unit/ctor_dtor.c b/compiler-rt/test/builtins/Unit/ctor_dtor.c
index 3d5f895a0a1c..47560722a9f7 100644
--- a/compiler-rt/test/builtins/Unit/ctor_dtor.c
+++ b/compiler-rt/test/builtins/Unit/ctor_dtor.c
@@ -9,13 +9,23 @@
 
 // Ensure the various startup functions are called in the proper order.
 
+// CHECK: __register_frame_info()
 /// ctor() is here if ld.so/libc supports DT_INIT/DT_FINI
 // CHECK:      main()
 /// dtor() is here if ld.so/libc supports DT_INIT/DT_FINI
+// CHECK:      __deregister_frame_info()
 
 struct object;
 static int counter;
 
+void __register_frame_info(const void *fi, struct object *obj) {
+  printf("__register_frame_info()\n");
+}
+
+void __deregister_frame_info(const void *fi) {
+  printf("__deregister_frame_info()\n");
+}
+
 void __attribute__((constructor)) ctor() {
   printf("ctor()\n");
   ++counter;
-- 
GitLab


From 624ea68cbc3ce422b3ee110c0c0af839eec2e278 Mon Sep 17 00:00:00 2001
From: Adrian Prantl 
Date: Fri, 8 Mar 2024 10:39:34 -0800
Subject: [PATCH 671/929] Change GetNumChildren()/CalculateNumChildren()
 methods return llvm::Expected (#84219)

Change GetNumChildren()/CalculateNumChildren() methods return
llvm::Expected

This is an NFC change that does not yet add any error handling or change
any code to return any errors.

This is the second big change in the patch series started with
https://github.com/llvm/llvm-project/pull/83501

A follow-up PR will wire up error handling.
---
 lldb/include/lldb/Core/ValueObject.h          | 11 +++-
 lldb/include/lldb/Core/ValueObjectCast.h      |  2 +-
 lldb/include/lldb/Core/ValueObjectChild.h     |  2 +-
 .../lldb/Core/ValueObjectConstResult.h        |  2 +-
 .../lldb/Core/ValueObjectDynamicValue.h       |  2 +-
 lldb/include/lldb/Core/ValueObjectMemory.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectRegister.h  |  4 +-
 .../lldb/Core/ValueObjectSyntheticFilter.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectVTable.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectVariable.h  |  2 +-
 .../lldb/DataFormatters/TypeSynthetic.h       | 20 ++++--
 .../lldb/DataFormatters/VectorIterator.h      |  2 +-
 lldb/include/lldb/Symbol/CompilerType.h       |  5 +-
 lldb/include/lldb/Symbol/Type.h               |  2 +-
 lldb/include/lldb/Symbol/TypeSystem.h         |  7 +-
 .../lldb/Target/StackFrameRecognizer.h        |  3 +-
 lldb/include/lldb/Utility/Log.h               | 14 ++++
 lldb/source/API/SBValue.cpp                   |  2 +-
 lldb/source/Core/FormatEntity.cpp             |  2 +-
 lldb/source/Core/IOHandlerCursesGUI.cpp       |  2 +-
 lldb/source/Core/ValueObject.cpp              | 31 ++++++---
 lldb/source/Core/ValueObjectCast.cpp          |  6 +-
 lldb/source/Core/ValueObjectChild.cpp         |  6 +-
 lldb/source/Core/ValueObjectConstResult.cpp   |  7 +-
 lldb/source/Core/ValueObjectDynamicValue.cpp  |  7 +-
 lldb/source/Core/ValueObjectMemory.cpp        | 10 ++-
 lldb/source/Core/ValueObjectRegister.cpp      | 12 ++--
 .../Core/ValueObjectSyntheticFilter.cpp       | 24 ++++---
 lldb/source/Core/ValueObjectVTable.cpp        |  6 +-
 lldb/source/Core/ValueObjectVariable.cpp      |  7 +-
 lldb/source/DataFormatters/FormatManager.cpp  | 11 +++-
 lldb/source/DataFormatters/TypeSynthetic.cpp  | 16 ++++-
 .../DataFormatters/ValueObjectPrinter.cpp     |  2 +-
 lldb/source/DataFormatters/VectorType.cpp     | 16 +++--
 .../Plugins/ABI/PowerPC/ABISysV_ppc64.cpp     |  7 +-
 .../Clang/ClangExpressionSourceCode.cpp       |  2 +-
 .../TSan/InstrumentationRuntimeTSan.cpp       |  4 +-
 .../Language/CPlusPlus/BlockPointer.cpp       |  4 +-
 .../Plugins/Language/CPlusPlus/Coroutines.cpp |  8 ++-
 .../Plugins/Language/CPlusPlus/Coroutines.h   |  2 +-
 .../Language/CPlusPlus/GenericBitset.cpp      |  4 +-
 .../Language/CPlusPlus/GenericOptional.cpp    |  6 +-
 .../Plugins/Language/CPlusPlus/LibCxx.cpp     | 18 +++---
 .../Plugins/Language/CPlusPlus/LibCxx.h       |  8 +--
 .../Language/CPlusPlus/LibCxxAtomic.cpp       |  6 +-
 .../CPlusPlus/LibCxxInitializerList.cpp       |  6 +-
 .../Plugins/Language/CPlusPlus/LibCxxList.cpp | 12 ++--
 .../Plugins/Language/CPlusPlus/LibCxxMap.cpp  | 14 ++--
 .../Language/CPlusPlus/LibCxxQueue.cpp        |  2 +-
 .../CPlusPlus/LibCxxRangesRefView.cpp         |  2 +-
 .../Plugins/Language/CPlusPlus/LibCxxSpan.cpp |  6 +-
 .../Language/CPlusPlus/LibCxxTuple.cpp        |  4 +-
 .../Language/CPlusPlus/LibCxxUnorderedMap.cpp |  8 +--
 .../Language/CPlusPlus/LibCxxValarray.cpp     |  6 +-
 .../Language/CPlusPlus/LibCxxVariant.cpp      |  2 +-
 .../Language/CPlusPlus/LibCxxVector.cpp       | 14 ++--
 .../Plugins/Language/CPlusPlus/LibStdcpp.cpp  | 17 +++--
 .../Language/CPlusPlus/LibStdcppTuple.cpp     |  7 +-
 .../CPlusPlus/LibStdcppUniquePointer.cpp      |  5 +-
 lldb/source/Plugins/Language/ObjC/Cocoa.cpp   |  2 +-
 lldb/source/Plugins/Language/ObjC/NSArray.cpp | 30 ++++-----
 .../Plugins/Language/ObjC/NSDictionary.cpp    | 64 +++++++++----------
 lldb/source/Plugins/Language/ObjC/NSError.cpp |  2 +-
 .../Plugins/Language/ObjC/NSException.cpp     |  4 +-
 .../Plugins/Language/ObjC/NSIndexPath.cpp     |  6 +-
 lldb/source/Plugins/Language/ObjC/NSSet.cpp   | 32 +++++-----
 .../AppleObjCRuntime/AppleObjCRuntime.cpp     |  3 +-
 .../TypeSystem/Clang/TypeSystemClang.cpp      | 40 ++++++++----
 .../TypeSystem/Clang/TypeSystemClang.h        |  7 +-
 lldb/source/Symbol/CompilerType.cpp           |  5 +-
 lldb/source/Symbol/Type.cpp                   |  2 +-
 lldb/source/Symbol/Variable.cpp               |  6 +-
 lldb/source/Target/StackFrame.cpp             | 19 +++---
 73 files changed, 399 insertions(+), 248 deletions(-)

diff --git a/lldb/include/lldb/Core/ValueObject.h b/lldb/include/lldb/Core/ValueObject.h
index b4d2c8098edc..e7e35e2b2bff 100644
--- a/lldb/include/lldb/Core/ValueObject.h
+++ b/lldb/include/lldb/Core/ValueObject.h
@@ -476,7 +476,13 @@ public:
 
   virtual size_t GetIndexOfChildWithName(llvm::StringRef name);
 
-  uint32_t GetNumChildren(uint32_t max = UINT32_MAX);
+  llvm::Expected GetNumChildren(uint32_t max = UINT32_MAX);
+  /// Like \c GetNumChildren but returns 0 on error.  You probably
+  /// shouldn't be using this function. It exists primarily to ease the
+  /// transition to more pervasive error handling while not all APIs
+  /// have been updated.
+  uint32_t GetNumChildrenIgnoringErrors(uint32_t max = UINT32_MAX);
+  bool HasChildren() { return GetNumChildrenIgnoringErrors() > 0; }
 
   const Value &GetValue() const { return m_value; }
 
@@ -958,7 +964,8 @@ protected:
                                           int32_t synthetic_index);
 
   /// Should only be called by ValueObject::GetNumChildren().
-  virtual uint32_t CalculateNumChildren(uint32_t max = UINT32_MAX) = 0;
+  virtual llvm::Expected
+  CalculateNumChildren(uint32_t max = UINT32_MAX) = 0;
 
   void SetNumChildren(uint32_t num_children);
 
diff --git a/lldb/include/lldb/Core/ValueObjectCast.h b/lldb/include/lldb/Core/ValueObjectCast.h
index 51c647680d52..ba25e166f326 100644
--- a/lldb/include/lldb/Core/ValueObjectCast.h
+++ b/lldb/include/lldb/Core/ValueObjectCast.h
@@ -33,7 +33,7 @@ public:
 
   std::optional GetByteSize() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectChild.h b/lldb/include/lldb/Core/ValueObjectChild.h
index 47a13be08bb8..1f88e607cb57 100644
--- a/lldb/include/lldb/Core/ValueObjectChild.h
+++ b/lldb/include/lldb/Core/ValueObjectChild.h
@@ -39,7 +39,7 @@ public:
 
   lldb::ValueType GetValueType() const override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectConstResult.h b/lldb/include/lldb/Core/ValueObjectConstResult.h
index 9f1246cf2a78..37dc0867f26c 100644
--- a/lldb/include/lldb/Core/ValueObjectConstResult.h
+++ b/lldb/include/lldb/Core/ValueObjectConstResult.h
@@ -67,7 +67,7 @@ public:
 
   lldb::ValueType GetValueType() const override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectDynamicValue.h b/lldb/include/lldb/Core/ValueObjectDynamicValue.h
index 21a9b409fd5b..82c20eee0cd4 100644
--- a/lldb/include/lldb/Core/ValueObjectDynamicValue.h
+++ b/lldb/include/lldb/Core/ValueObjectDynamicValue.h
@@ -43,7 +43,7 @@ public:
 
   ConstString GetDisplayTypeName() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectMemory.h b/lldb/include/lldb/Core/ValueObjectMemory.h
index a74b325546b0..a8fb0353d601 100644
--- a/lldb/include/lldb/Core/ValueObjectMemory.h
+++ b/lldb/include/lldb/Core/ValueObjectMemory.h
@@ -47,7 +47,7 @@ public:
 
   ConstString GetDisplayTypeName() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectRegister.h b/lldb/include/lldb/Core/ValueObjectRegister.h
index 6c470c1a6865..fec8566ba33d 100644
--- a/lldb/include/lldb/Core/ValueObjectRegister.h
+++ b/lldb/include/lldb/Core/ValueObjectRegister.h
@@ -47,7 +47,7 @@ public:
 
   ConstString GetQualifiedTypeName() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member,
                                   int32_t synthetic_index) override;
@@ -95,7 +95,7 @@ public:
 
   ConstString GetTypeName() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   bool SetValueFromCString(const char *value_str, Status &error) override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h b/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
index 1e54babc94f3..ca6d6c728005 100644
--- a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
+++ b/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
@@ -47,7 +47,7 @@ public:
 
   bool MightHaveChildren() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectVTable.h b/lldb/include/lldb/Core/ValueObjectVTable.h
index e7e14fc83d78..4662f395a4dd 100644
--- a/lldb/include/lldb/Core/ValueObjectVTable.h
+++ b/lldb/include/lldb/Core/ValueObjectVTable.h
@@ -64,7 +64,7 @@ public:
 
   std::optional GetByteSize() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member,
                                   int32_t synthetic_index) override;
diff --git a/lldb/include/lldb/Core/ValueObjectVariable.h b/lldb/include/lldb/Core/ValueObjectVariable.h
index da270300df0b..db3847f14a0b 100644
--- a/lldb/include/lldb/Core/ValueObjectVariable.h
+++ b/lldb/include/lldb/Core/ValueObjectVariable.h
@@ -46,7 +46,7 @@ public:
 
   ConstString GetDisplayTypeName() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/DataFormatters/TypeSynthetic.h b/lldb/include/lldb/DataFormatters/TypeSynthetic.h
index 38f3ce0fa5f0..ede7442a02bf 100644
--- a/lldb/include/lldb/DataFormatters/TypeSynthetic.h
+++ b/lldb/include/lldb/DataFormatters/TypeSynthetic.h
@@ -38,13 +38,17 @@ public:
 
   virtual ~SyntheticChildrenFrontEnd() = default;
 
-  virtual uint32_t CalculateNumChildren() = 0;
+  virtual llvm::Expected CalculateNumChildren() = 0;
 
-  virtual uint32_t CalculateNumChildren(uint32_t max) {
+  virtual llvm::Expected CalculateNumChildren(uint32_t max) {
     auto count = CalculateNumChildren();
-    return count <= max ? count : max;
+    if (!count)
+      return count;
+    return *count <= max ? *count : max;
   }
 
+  uint32_t CalculateNumChildrenIgnoringErrors(uint32_t max = UINT32_MAX);
+
   virtual lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) = 0;
 
   virtual size_t GetIndexOfChildWithName(ConstString name) = 0;
@@ -109,7 +113,7 @@ public:
 
   ~SyntheticValueProviderFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override { return 0; }
+  llvm::Expected CalculateNumChildren() override { return 0; }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override { return nullptr; }
 
@@ -322,7 +326,9 @@ public:
 
     ~FrontEnd() override = default;
 
-    uint32_t CalculateNumChildren() override { return filter->GetCount(); }
+    llvm::Expected CalculateNumChildren() override {
+      return filter->GetCount();
+    }
 
     lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
       if (idx >= filter->GetCount())
@@ -426,9 +432,9 @@ public:
 
     bool IsValid();
 
-    uint32_t CalculateNumChildren() override;
+    llvm::Expected CalculateNumChildren() override;
 
-    uint32_t CalculateNumChildren(uint32_t max) override;
+    llvm::Expected CalculateNumChildren(uint32_t max) override;
 
     lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/include/lldb/DataFormatters/VectorIterator.h b/lldb/include/lldb/DataFormatters/VectorIterator.h
index 7711b9de95db..70bcf50ca1b1 100644
--- a/lldb/include/lldb/DataFormatters/VectorIterator.h
+++ b/lldb/include/lldb/DataFormatters/VectorIterator.h
@@ -24,7 +24,7 @@ public:
   VectorIteratorSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp,
                                   llvm::ArrayRef item_names);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/include/lldb/Symbol/CompilerType.h b/lldb/include/lldb/Symbol/CompilerType.h
index 414c44275aaa..c1dce4ccbf79 100644
--- a/lldb/include/lldb/Symbol/CompilerType.h
+++ b/lldb/include/lldb/Symbol/CompilerType.h
@@ -386,8 +386,9 @@ public:
 
   std::optional GetTypeBitAlign(ExecutionContextScope *exe_scope) const;
 
-  uint32_t GetNumChildren(bool omit_empty_base_classes,
-                          const ExecutionContext *exe_ctx) const;
+  llvm::Expected
+  GetNumChildren(bool omit_empty_base_classes,
+                 const ExecutionContext *exe_ctx) const;
 
   lldb::BasicType GetBasicTypeEnumeration() const;
 
diff --git a/lldb/include/lldb/Symbol/Type.h b/lldb/include/lldb/Symbol/Type.h
index acd1a769f13c..b5eac5fa732d 100644
--- a/lldb/include/lldb/Symbol/Type.h
+++ b/lldb/include/lldb/Symbol/Type.h
@@ -440,7 +440,7 @@ public:
 
   std::optional GetByteSize(ExecutionContextScope *exe_scope);
 
-  uint32_t GetNumChildren(bool omit_empty_base_classes);
+  llvm::Expected GetNumChildren(bool omit_empty_base_classes);
 
   bool IsAggregateType();
 
diff --git a/lldb/include/lldb/Symbol/TypeSystem.h b/lldb/include/lldb/Symbol/TypeSystem.h
index 63829131556e..f647fcbf1636 100644
--- a/lldb/include/lldb/Symbol/TypeSystem.h
+++ b/lldb/include/lldb/Symbol/TypeSystem.h
@@ -300,9 +300,10 @@ public:
 
   virtual lldb::Format GetFormat(lldb::opaque_compiler_type_t type) = 0;
 
-  virtual uint32_t GetNumChildren(lldb::opaque_compiler_type_t type,
-                                  bool omit_empty_base_classes,
-                                  const ExecutionContext *exe_ctx) = 0;
+  virtual llvm::Expected
+  GetNumChildren(lldb::opaque_compiler_type_t type,
+                 bool omit_empty_base_classes,
+                 const ExecutionContext *exe_ctx) = 0;
 
   virtual CompilerType GetBuiltinTypeByName(ConstString name);
 
diff --git a/lldb/include/lldb/Target/StackFrameRecognizer.h b/lldb/include/lldb/Target/StackFrameRecognizer.h
index e111f4a4dc70..5e8e12b2a4e9 100644
--- a/lldb/include/lldb/Target/StackFrameRecognizer.h
+++ b/lldb/include/lldb/Target/StackFrameRecognizer.h
@@ -164,7 +164,8 @@ class ValueObjectRecognizerSynthesizedValue : public ValueObject {
     m_value = m_parent->GetValue();
     return true;
   }
-  uint32_t CalculateNumChildren(uint32_t max = UINT32_MAX) override {
+  llvm::Expected
+  CalculateNumChildren(uint32_t max = UINT32_MAX) override {
     return m_parent->GetNumChildren(max);
   }
   CompilerType GetCompilerTypeImpl() override {
diff --git a/lldb/include/lldb/Utility/Log.h b/lldb/include/lldb/Utility/Log.h
index 1fe28d61b9da..01876ad732d4 100644
--- a/lldb/include/lldb/Utility/Log.h
+++ b/lldb/include/lldb/Utility/Log.h
@@ -373,4 +373,18 @@ template  Log *GetLog(Cat mask) {
       ::llvm::consumeError(::std::move(error_private));                        \
   } while (0)
 
+// Write message to the verbose log, if error is set. In the log
+// message refer to the error with {0}. Error is cleared regardless of
+// whether logging is enabled.
+#define LLDB_LOG_ERRORV(log, error, ...)                                       \
+  do {                                                                         \
+    ::lldb_private::Log *log_private = (log);                                  \
+    ::llvm::Error error_private = (error);                                     \
+    if (log_private && log_private->GetVerbose() && error_private) {           \
+      log_private->FormatError(::std::move(error_private), __FILE__, __func__, \
+                               __VA_ARGS__);                                   \
+    } else                                                                     \
+      ::llvm::consumeError(::std::move(error_private));                        \
+  } while (0)
+
 #endif // LLDB_UTILITY_LOG_H
diff --git a/lldb/source/API/SBValue.cpp b/lldb/source/API/SBValue.cpp
index 89d26a1fbe28..94a8f3ea319e 100644
--- a/lldb/source/API/SBValue.cpp
+++ b/lldb/source/API/SBValue.cpp
@@ -947,7 +947,7 @@ uint32_t SBValue::GetNumChildren(uint32_t max) {
   ValueLocker locker;
   lldb::ValueObjectSP value_sp(GetSP(locker));
   if (value_sp)
-    num_children = value_sp->GetNumChildren(max);
+    num_children = value_sp->GetNumChildrenIgnoringErrors(max);
 
   return num_children;
 }
diff --git a/lldb/source/Core/FormatEntity.cpp b/lldb/source/Core/FormatEntity.cpp
index fa5eadc6ff4e..cf82676bedda 100644
--- a/lldb/source/Core/FormatEntity.cpp
+++ b/lldb/source/Core/FormatEntity.cpp
@@ -926,7 +926,7 @@ static bool DumpValue(Stream &s, const SymbolContext *sc,
     s.PutChar('[');
 
     if (index_higher < 0)
-      index_higher = valobj->GetNumChildren() - 1;
+      index_higher = valobj->GetNumChildrenIgnoringErrors() - 1;
 
     uint32_t max_num_children =
         target->GetTargetSP()->GetMaximumNumberOfChildrenToDisplay();
diff --git a/lldb/source/Core/IOHandlerCursesGUI.cpp b/lldb/source/Core/IOHandlerCursesGUI.cpp
index 620e68a28510..f86dce247135 100644
--- a/lldb/source/Core/IOHandlerCursesGUI.cpp
+++ b/lldb/source/Core/IOHandlerCursesGUI.cpp
@@ -4519,7 +4519,7 @@ struct Row {
       calculated_children = true;
       ValueObjectSP valobj = value.GetSP();
       if (valobj) {
-        const size_t num_children = valobj->GetNumChildren();
+        const uint32_t num_children = valobj->GetNumChildrenIgnoringErrors();
         for (size_t i = 0; i < num_children; ++i) {
           children.push_back(Row(valobj->GetChildAtIndex(i), this));
         }
diff --git a/lldb/source/Core/ValueObject.cpp b/lldb/source/Core/ValueObject.cpp
index 0ed7f03be25c..d813044d02ff 100644
--- a/lldb/source/Core/ValueObject.cpp
+++ b/lldb/source/Core/ValueObject.cpp
@@ -377,7 +377,7 @@ ValueObjectSP ValueObject::GetChildAtIndex(uint32_t idx, bool can_create) {
   // We may need to update our value if we are dynamic
   if (IsPossibleDynamicType())
     UpdateValueIfNeeded(false);
-  if (idx < GetNumChildren()) {
+  if (idx < GetNumChildrenIgnoringErrors()) {
     // Check if we have already made the child value object?
     if (can_create && !m_children.HasChildAtIndex(idx)) {
       // No we haven't created the child at this index, so lets have our
@@ -440,7 +440,7 @@ ValueObjectSP ValueObject::GetChildMemberWithName(llvm::StringRef name,
   return child_sp;
 }
 
-uint32_t ValueObject::GetNumChildren(uint32_t max) {
+llvm::Expected ValueObject::GetNumChildren(uint32_t max) {
   UpdateValueIfNeeded();
 
   if (max < UINT32_MAX) {
@@ -452,11 +452,24 @@ uint32_t ValueObject::GetNumChildren(uint32_t max) {
   }
 
   if (!m_flags.m_children_count_valid) {
-    SetNumChildren(CalculateNumChildren());
+    auto num_children_or_err = CalculateNumChildren();
+    if (num_children_or_err)
+      SetNumChildren(*num_children_or_err);
+    else
+      return num_children_or_err;
   }
   return m_children.GetChildrenCount();
 }
 
+uint32_t ValueObject::GetNumChildrenIgnoringErrors(uint32_t max) {
+  auto value_or_err = GetNumChildren(max);
+  if (value_or_err)
+    return *value_or_err;
+  LLDB_LOG_ERRORV(GetLog(LLDBLog::DataFormatters), value_or_err.takeError(),
+                  "{0}");
+  return 0;
+}
+
 bool ValueObject::MightHaveChildren() {
   bool has_children = false;
   const uint32_t type_info = GetTypeInfo();
@@ -464,7 +477,7 @@ bool ValueObject::MightHaveChildren() {
     if (type_info & (eTypeHasChildren | eTypeIsPointer | eTypeIsReference))
       has_children = true;
   } else {
-    has_children = GetNumChildren() > 0;
+    has_children = GetNumChildrenIgnoringErrors() > 0;
   }
   return has_children;
 }
@@ -1176,7 +1189,7 @@ bool ValueObject::DumpPrintableRepresentation(
       if (flags.Test(eTypeIsArray)) {
         if ((custom_format == eFormatBytes) ||
             (custom_format == eFormatBytesWithASCII)) {
-          const size_t count = GetNumChildren();
+          const size_t count = GetNumChildrenIgnoringErrors();
 
           s << '[';
           for (size_t low = 0; low < count; low++) {
@@ -1215,7 +1228,7 @@ bool ValueObject::DumpPrintableRepresentation(
                                                      // format should be printed
                                                      // directly
         {
-          const size_t count = GetNumChildren();
+          const size_t count = GetNumChildrenIgnoringErrors();
 
           Format format = FormatManager::GetSingleItemFormat(custom_format);
 
@@ -1294,7 +1307,7 @@ bool ValueObject::DumpPrintableRepresentation(
       break;
 
     case eValueObjectRepresentationStyleChildrenCount:
-      strm.Printf("%" PRIu64 "", (uint64_t)GetNumChildren());
+      strm.Printf("%" PRIu64 "", (uint64_t)GetNumChildrenIgnoringErrors());
       str = strm.GetString();
       break;
 
@@ -2320,7 +2333,9 @@ ValueObjectSP ValueObject::GetValueForExpressionPath_Impl(
             child_valobj_sp = root->GetSyntheticArrayMember(index, true);
           if (!child_valobj_sp)
             if (root->HasSyntheticValue() &&
-                root->GetSyntheticValue()->GetNumChildren() > index)
+                llvm::expectedToStdOptional(
+                    root->GetSyntheticValue()->GetNumChildren())
+                        .value_or(0) > index)
               child_valobj_sp =
                   root->GetSyntheticValue()->GetChildAtIndex(index);
           if (child_valobj_sp) {
diff --git a/lldb/source/Core/ValueObjectCast.cpp b/lldb/source/Core/ValueObjectCast.cpp
index a5c555f86b13..c8e316415141 100644
--- a/lldb/source/Core/ValueObjectCast.cpp
+++ b/lldb/source/Core/ValueObjectCast.cpp
@@ -41,11 +41,13 @@ ValueObjectCast::~ValueObjectCast() = default;
 
 CompilerType ValueObjectCast::GetCompilerTypeImpl() { return m_cast_type; }
 
-uint32_t ValueObjectCast::CalculateNumChildren(uint32_t max) {
+llvm::Expected ValueObjectCast::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(
       true, &exe_ctx);
-  return children_count <= max ? children_count : max;
+  if (!children_count)
+    return children_count;
+  return *children_count <= max ? *children_count : max;
 }
 
 std::optional ValueObjectCast::GetByteSize() {
diff --git a/lldb/source/Core/ValueObjectChild.cpp b/lldb/source/Core/ValueObjectChild.cpp
index 2e55dd7726bd..c6a97dd1a5cd 100644
--- a/lldb/source/Core/ValueObjectChild.cpp
+++ b/lldb/source/Core/ValueObjectChild.cpp
@@ -49,10 +49,12 @@ lldb::ValueType ValueObjectChild::GetValueType() const {
   return m_parent->GetValueType();
 }
 
-uint32_t ValueObjectChild::CalculateNumChildren(uint32_t max) {
+llvm::Expected ValueObjectChild::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-  return children_count <= max ? children_count : max;
+  if (!children_count)
+    return children_count;
+  return *children_count <= max ? *children_count : max;
 }
 
 static void AdjustForBitfieldness(ConstString &name,
diff --git a/lldb/source/Core/ValueObjectConstResult.cpp b/lldb/source/Core/ValueObjectConstResult.cpp
index 5c7aa4452b70..8ac2c1cac2f6 100644
--- a/lldb/source/Core/ValueObjectConstResult.cpp
+++ b/lldb/source/Core/ValueObjectConstResult.cpp
@@ -216,10 +216,13 @@ std::optional ValueObjectConstResult::GetByteSize() {
 
 void ValueObjectConstResult::SetByteSize(size_t size) { m_byte_size = size; }
 
-uint32_t ValueObjectConstResult::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectConstResult::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-  return children_count <= max ? children_count : max;
+  if (!children_count)
+    return children_count;
+  return *children_count <= max ? *children_count : max;
 }
 
 ConstString ValueObjectConstResult::GetTypeName() {
diff --git a/lldb/source/Core/ValueObjectDynamicValue.cpp b/lldb/source/Core/ValueObjectDynamicValue.cpp
index 4e64760371ae..4695febdf8ca 100644
--- a/lldb/source/Core/ValueObjectDynamicValue.cpp
+++ b/lldb/source/Core/ValueObjectDynamicValue.cpp
@@ -85,12 +85,15 @@ ConstString ValueObjectDynamicValue::GetDisplayTypeName() {
   return m_parent->GetDisplayTypeName();
 }
 
-uint32_t ValueObjectDynamicValue::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectDynamicValue::CalculateNumChildren(uint32_t max) {
   const bool success = UpdateValueIfNeeded(false);
   if (success && m_dynamic_type_info.HasType()) {
     ExecutionContext exe_ctx(GetExecutionContextRef());
     auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-    return children_count <= max ? children_count : max;
+    if (!children_count)
+      return children_count;
+    return *children_count <= max ? *children_count : max;
   } else
     return m_parent->GetNumChildren(max);
 }
diff --git a/lldb/source/Core/ValueObjectMemory.cpp b/lldb/source/Core/ValueObjectMemory.cpp
index 7f68236c7884..f555ab82f441 100644
--- a/lldb/source/Core/ValueObjectMemory.cpp
+++ b/lldb/source/Core/ValueObjectMemory.cpp
@@ -126,17 +126,21 @@ ConstString ValueObjectMemory::GetDisplayTypeName() {
   return m_compiler_type.GetDisplayTypeName();
 }
 
-uint32_t ValueObjectMemory::CalculateNumChildren(uint32_t max) {
+llvm::Expected ValueObjectMemory::CalculateNumChildren(uint32_t max) {
   if (m_type_sp) {
     auto child_count = m_type_sp->GetNumChildren(true);
-    return child_count <= max ? child_count : max;
+    if (!child_count)
+      return child_count;
+    return *child_count <= max ? *child_count : max;
   }
 
   ExecutionContext exe_ctx(GetExecutionContextRef());
   const bool omit_empty_base_classes = true;
   auto child_count =
       m_compiler_type.GetNumChildren(omit_empty_base_classes, &exe_ctx);
-  return child_count <= max ? child_count : max;
+  if (!child_count)
+    return child_count;
+  return *child_count <= max ? *child_count : max;
 }
 
 std::optional ValueObjectMemory::GetByteSize() {
diff --git a/lldb/source/Core/ValueObjectRegister.cpp b/lldb/source/Core/ValueObjectRegister.cpp
index d4c144cc7edb..9d9da8bb119d 100644
--- a/lldb/source/Core/ValueObjectRegister.cpp
+++ b/lldb/source/Core/ValueObjectRegister.cpp
@@ -74,7 +74,8 @@ ConstString ValueObjectRegisterSet::GetQualifiedTypeName() {
   return ConstString();
 }
 
-uint32_t ValueObjectRegisterSet::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectRegisterSet::CalculateNumChildren(uint32_t max) {
   const RegisterSet *reg_set = m_reg_ctx_sp->GetRegisterSet(m_reg_set_idx);
   if (reg_set) {
     auto reg_count = reg_set->num_registers;
@@ -118,7 +119,7 @@ ValueObject *ValueObjectRegisterSet::CreateChildAtIndex(
     size_t idx, bool synthetic_array_member, int32_t synthetic_index) {
   ValueObject *valobj = nullptr;
   if (m_reg_ctx_sp && m_reg_set) {
-    const size_t num_children = GetNumChildren();
+    uint32_t num_children = GetNumChildrenIgnoringErrors();
     if (idx < num_children)
       valobj = new ValueObjectRegister(
           *this, m_reg_ctx_sp,
@@ -220,10 +221,13 @@ ConstString ValueObjectRegister::GetTypeName() {
   return m_type_name;
 }
 
-uint32_t ValueObjectRegister::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectRegister::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-  return children_count <= max ? children_count : max;
+  if (!children_count)
+    return children_count;
+  return *children_count <= max ? *children_count : max;
 }
 
 std::optional ValueObjectRegister::GetByteSize() {
diff --git a/lldb/source/Core/ValueObjectSyntheticFilter.cpp b/lldb/source/Core/ValueObjectSyntheticFilter.cpp
index 7f8a9a34cb35..adac1b400705 100644
--- a/lldb/source/Core/ValueObjectSyntheticFilter.cpp
+++ b/lldb/source/Core/ValueObjectSyntheticFilter.cpp
@@ -31,7 +31,9 @@ public:
   DummySyntheticFrontEnd(ValueObject &backend)
       : SyntheticChildrenFrontEnd(backend) {}
 
-  uint32_t CalculateNumChildren() override { return m_backend.GetNumChildren(); }
+  llvm::Expected CalculateNumChildren() override {
+    return m_backend.GetNumChildren();
+  }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     return m_backend.GetChildAtIndex(idx);
@@ -84,7 +86,8 @@ ConstString ValueObjectSynthetic::GetDisplayTypeName() {
   return m_parent->GetDisplayTypeName();
 }
 
-uint32_t ValueObjectSynthetic::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectSynthetic::CalculateNumChildren(uint32_t max) {
   Log *log = GetLog(LLDBLog::DataFormatters);
 
   UpdateValueIfNeeded();
@@ -92,18 +95,23 @@ uint32_t ValueObjectSynthetic::CalculateNumChildren(uint32_t max) {
     return m_synthetic_children_count <= max ? m_synthetic_children_count : max;
 
   if (max < UINT32_MAX) {
-    size_t num_children = m_synth_filter_up->CalculateNumChildren(max);
+    auto num_children = m_synth_filter_up->CalculateNumChildren(max);
     LLDB_LOGF(log,
               "[ValueObjectSynthetic::CalculateNumChildren] for VO of name "
-              "%s and type %s, the filter returned %zu child values",
-              GetName().AsCString(), GetTypeName().AsCString(), num_children);
+              "%s and type %s, the filter returned %u child values",
+              GetName().AsCString(), GetTypeName().AsCString(),
+              num_children ? *num_children : 0);
     return num_children;
   } else {
-    size_t num_children = (m_synthetic_children_count =
-                               m_synth_filter_up->CalculateNumChildren(max));
+    auto num_children_or_err = m_synth_filter_up->CalculateNumChildren(max);
+    if (!num_children_or_err) {
+      m_synthetic_children_count = 0;
+      return num_children_or_err;
+    }
+    auto num_children = (m_synthetic_children_count = *num_children_or_err);
     LLDB_LOGF(log,
               "[ValueObjectSynthetic::CalculateNumChildren] for VO of name "
-              "%s and type %s, the filter returned %zu child values",
+              "%s and type %s, the filter returned %u child values",
               GetName().AsCString(), GetTypeName().AsCString(), num_children);
     return num_children;
   }
diff --git a/lldb/source/Core/ValueObjectVTable.cpp b/lldb/source/Core/ValueObjectVTable.cpp
index 4d1cbb8d2f6f..9bfe8dc9700d 100644
--- a/lldb/source/Core/ValueObjectVTable.cpp
+++ b/lldb/source/Core/ValueObjectVTable.cpp
@@ -33,7 +33,9 @@ public:
 
   std::optional GetByteSize() override { return m_addr_size; };
 
-  uint32_t CalculateNumChildren(uint32_t max) override { return 0; };
+  llvm::Expected CalculateNumChildren(uint32_t max) override {
+    return 0;
+  };
 
   ValueType GetValueType() const override { return eValueTypeVTableEntry; };
 
@@ -159,7 +161,7 @@ std::optional ValueObjectVTable::GetByteSize() {
   return std::nullopt;
 }
 
-uint32_t ValueObjectVTable::CalculateNumChildren(uint32_t max) {
+llvm::Expected ValueObjectVTable::CalculateNumChildren(uint32_t max) {
   if (UpdateValueIfNeeded(false))
     return m_num_vtable_entries <= max ? m_num_vtable_entries : max;
   return 0;
diff --git a/lldb/source/Core/ValueObjectVariable.cpp b/lldb/source/Core/ValueObjectVariable.cpp
index dc62bb6358dc..fb29c22c0ab5 100644
--- a/lldb/source/Core/ValueObjectVariable.cpp
+++ b/lldb/source/Core/ValueObjectVariable.cpp
@@ -94,7 +94,8 @@ ConstString ValueObjectVariable::GetQualifiedTypeName() {
   return ConstString();
 }
 
-uint32_t ValueObjectVariable::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectVariable::CalculateNumChildren(uint32_t max) {
   CompilerType type(GetCompilerType());
 
   if (!type.IsValid())
@@ -103,7 +104,9 @@ uint32_t ValueObjectVariable::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   const bool omit_empty_base_classes = true;
   auto child_count = type.GetNumChildren(omit_empty_base_classes, &exe_ctx);
-  return child_count <= max ? child_count : max;
+  if (!child_count)
+    return child_count;
+  return *child_count <= max ? *child_count : max;
 }
 
 std::optional ValueObjectVariable::GetByteSize() {
diff --git a/lldb/source/DataFormatters/FormatManager.cpp b/lldb/source/DataFormatters/FormatManager.cpp
index 092fa3c8ce49..d7ba5b4b70c9 100644
--- a/lldb/source/DataFormatters/FormatManager.cpp
+++ b/lldb/source/DataFormatters/FormatManager.cpp
@@ -451,8 +451,13 @@ bool FormatManager::ShouldPrintAsOneLiner(ValueObject &valobj) {
   if (valobj.GetSummaryFormat().get() != nullptr)
     return valobj.GetSummaryFormat()->IsOneLiner();
 
+  auto num_children = valobj.GetNumChildren();
+  if (!num_children) {
+    llvm::consumeError(num_children.takeError());
+    return true;
+  }
   // no children, no party
-  if (valobj.GetNumChildren() == 0)
+  if (*num_children == 0)
     return false;
 
   // ask the type if it has any opinion about this eLazyBoolCalculate == no
@@ -471,7 +476,7 @@ bool FormatManager::ShouldPrintAsOneLiner(ValueObject &valobj) {
 
   size_t total_children_name_len = 0;
 
-  for (size_t idx = 0; idx < valobj.GetNumChildren(); idx++) {
+  for (size_t idx = 0; idx < *num_children; idx++) {
     bool is_synth_val = false;
     ValueObjectSP child_sp(valobj.GetChildAtIndex(idx));
     // something is wrong here - bail out
@@ -523,7 +528,7 @@ bool FormatManager::ShouldPrintAsOneLiner(ValueObject &valobj) {
     }
 
     // if this child has children..
-    if (child_sp->GetNumChildren()) {
+    if (child_sp->HasChildren()) {
       // ...and no summary...
       // (if it had a summary and the summary wanted children, we would have
       // bailed out anyway
diff --git a/lldb/source/DataFormatters/TypeSynthetic.cpp b/lldb/source/DataFormatters/TypeSynthetic.cpp
index 0ae38c4d31f2..7aa0670190b2 100644
--- a/lldb/source/DataFormatters/TypeSynthetic.cpp
+++ b/lldb/source/DataFormatters/TypeSynthetic.cpp
@@ -115,6 +115,16 @@ std::string CXXSyntheticChildren::GetDescription() {
   return std::string(sstr.GetString());
 }
 
+uint32_t
+SyntheticChildrenFrontEnd::CalculateNumChildrenIgnoringErrors(uint32_t max) {
+  auto value_or_err = CalculateNumChildren(max);
+  if (value_or_err)
+    return *value_or_err;
+  LLDB_LOG_ERRORV(GetLog(LLDBLog::DataFormatters), value_or_err.takeError(),
+                  "{0}");
+  return 0;
+}
+
 lldb::ValueObjectSP SyntheticChildrenFrontEnd::CreateValueObjectFromExpression(
     llvm::StringRef name, llvm::StringRef expression,
     const ExecutionContext &exe_ctx) {
@@ -178,13 +188,15 @@ bool ScriptedSyntheticChildren::FrontEnd::IsValid() {
   return (m_wrapper_sp && m_wrapper_sp->IsValid() && m_interpreter);
 }
 
-uint32_t ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren() {
+llvm::Expected
+ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren() {
   if (!m_wrapper_sp || m_interpreter == nullptr)
     return 0;
   return m_interpreter->CalculateNumChildren(m_wrapper_sp, UINT32_MAX);
 }
 
-uint32_t ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren(uint32_t max) {
   if (!m_wrapper_sp || m_interpreter == nullptr)
     return 0;
   return m_interpreter->CalculateNumChildren(m_wrapper_sp, max);
diff --git a/lldb/source/DataFormatters/ValueObjectPrinter.cpp b/lldb/source/DataFormatters/ValueObjectPrinter.cpp
index 46e50a8d421a..b853199e878c 100644
--- a/lldb/source/DataFormatters/ValueObjectPrinter.cpp
+++ b/lldb/source/DataFormatters/ValueObjectPrinter.cpp
@@ -627,7 +627,7 @@ uint32_t ValueObjectPrinter::GetMaxNumChildrenToPrint(bool &print_dotdotdot) {
   if (m_options.m_pointer_as_array)
     return m_options.m_pointer_as_array.m_element_count;
 
-  size_t num_children = synth_valobj.GetNumChildren();
+  uint32_t num_children = synth_valobj.GetNumChildrenIgnoringErrors();
   print_dotdotdot = false;
   if (num_children) {
     const size_t max_num_children = GetMostSpecializedValue()
diff --git a/lldb/source/DataFormatters/VectorType.cpp b/lldb/source/DataFormatters/VectorType.cpp
index a0626a8cba77..19de204c2435 100644
--- a/lldb/source/DataFormatters/VectorType.cpp
+++ b/lldb/source/DataFormatters/VectorType.cpp
@@ -9,6 +9,7 @@
 #include "lldb/DataFormatters/VectorType.h"
 
 #include "lldb/Core/ValueObject.h"
+#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Symbol/TypeSystem.h"
@@ -224,10 +225,16 @@ public:
 
   ~VectorTypeSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override { return m_num_children; }
+  llvm::Expected CalculateNumChildren() override {
+    return m_num_children;
+  }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
-    if (idx >= CalculateNumChildren())
+    auto num_children_or_err = CalculateNumChildren();
+    if (!num_children_or_err)
+      return ValueObjectConstResult::Create(
+          nullptr, Status(num_children_or_err.takeError()));
+    if (idx >= *num_children_or_err)
       return {};
     std::optional size = m_child_type.GetByteSize(nullptr);
     if (!size)
@@ -266,7 +273,7 @@ public:
   size_t GetIndexOfChildWithName(ConstString name) override {
     const char *item_name = name.GetCString();
     uint32_t idx = ExtractIndexFromString(item_name);
-    if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+    if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
       return UINT32_MAX;
     return idx;
   }
@@ -293,7 +300,8 @@ bool lldb_private::formatters::VectorTypeSummaryProvider(
   s.PutChar('(');
   bool first = true;
 
-  size_t idx = 0, len = synthetic_children->CalculateNumChildren();
+  size_t idx = 0,
+         len = synthetic_children->CalculateNumChildrenIgnoringErrors();
 
   for (; idx < len; idx++) {
     auto child_sp = synthetic_children->GetChildAtIndex(idx);
diff --git a/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc64.cpp b/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc64.cpp
index 656f07437095..173b5613d1b8 100644
--- a/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc64.cpp
+++ b/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc64.cpp
@@ -768,7 +768,12 @@ private:
 
     // get number of children
     const bool omit_empty_base_classes = true;
-    uint32_t n = m_type.GetNumChildren(omit_empty_base_classes, nullptr);
+    auto n_or_err = m_type.GetNumChildren(omit_empty_base_classes, nullptr);
+    if (!n_or_err) {
+      LLDB_LOG_ERROR(m_log, n_or_err.takeError(), LOG_PREFIX "{0}");
+      return {};
+    }
+    uint32_t n = *n_or_err;
     if (!n) {
       LLDB_LOG(m_log, LOG_PREFIX "No children found in struct");
       return {};
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp
index b48bbbecc0cd..3d43ed3f99ff 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp
@@ -229,7 +229,7 @@ void AddLambdaCaptureDecls(StreamString &stream, StackFrame *frame,
   assert(frame);
 
   if (auto thisValSP = ClangExpressionUtil::GetLambdaValueObject(frame)) {
-    uint32_t numChildren = thisValSP->GetNumChildren();
+    uint32_t numChildren = thisValSP->GetNumChildrenIgnoringErrors();
     for (uint32_t i = 0; i < numChildren; ++i) {
       auto childVal = thisValSP->GetChildAtIndex(i);
       ConstString childName(childVal ? childVal->GetName() : ConstString(""));
diff --git a/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp b/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
index 72293c5331f4..b2781aa5e7db 100644
--- a/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
+++ b/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
@@ -27,6 +27,8 @@
 #include "lldb/Target/StopInfo.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Target/Thread.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegularExpression.h"
 #include "lldb/Utility/Stream.h"
 
@@ -211,7 +213,7 @@ CreateStackTrace(ValueObjectSP o,
   auto trace_sp = std::make_shared();
   ValueObjectSP trace_value_object =
       o->GetValueForExpressionPath(trace_item_name.c_str());
-  size_t count = trace_value_object->GetNumChildren();
+  size_t count = trace_value_object->GetNumChildrenIgnoringErrors();
   for (size_t j = 0; j < count; j++) {
     addr_t trace_addr =
         trace_value_object->GetChildAtIndex(j)->GetValueAsUnsigned(0);
diff --git a/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp b/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
index ef0f67d1e9f9..9a6e135e0083 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
@@ -74,7 +74,7 @@ public:
 
   ~BlockPointerSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override {
+  llvm::Expected CalculateNumChildren() override {
     const bool omit_empty_base_classes = false;
     return m_block_struct_type.GetNumChildren(omit_empty_base_classes, nullptr);
   }
@@ -84,7 +84,7 @@ public:
       return lldb::ValueObjectSP();
     }
 
-    if (idx >= CalculateNumChildren()) {
+    if (idx >= CalculateNumChildrenIgnoringErrors()) {
       return lldb::ValueObjectSP();
     }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp
index 3827f9c21eff..5e63d1d7b214 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp
@@ -11,6 +11,8 @@
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
 #include "lldb/Symbol/Function.h"
 #include "lldb/Symbol/VariableList.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/Log.h"
 
 using namespace lldb;
 using namespace lldb_private;
@@ -22,7 +24,7 @@ static lldb::addr_t GetCoroFramePtrFromHandle(ValueObjectSP valobj_sp) {
 
   // We expect a single pointer in the `coroutine_handle` class.
   // We don't care about its name.
-  if (valobj_sp->GetNumChildren() != 1)
+  if (valobj_sp->GetNumChildrenIgnoringErrors() != 1)
     return LLDB_INVALID_ADDRESS;
   ValueObjectSP ptr_sp(valobj_sp->GetChildAtIndex(0));
   if (!ptr_sp)
@@ -104,8 +106,8 @@ lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd::
 lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd::
     ~StdlibCoroutineHandleSyntheticFrontEnd() = default;
 
-uint32_t lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    StdlibCoroutineHandleSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_resume_ptr_sp || !m_destroy_ptr_sp)
     return 0;
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
index 5c6a80b57ff4..1d4bc65e2637 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
@@ -34,7 +34,7 @@ public:
 
   ~StdlibCoroutineHandleSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp b/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp
index 6a9da1d17c76..33955dccb6cc 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp
@@ -34,7 +34,9 @@ public:
 
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
-  uint32_t CalculateNumChildren() override { return m_elements.size(); }
+  llvm::Expected CalculateNumChildren() override {
+    return m_elements.size();
+  }
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
 private:
diff --git a/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp b/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp
index c06afb53eb8a..23756de7f1e6 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp
@@ -19,7 +19,7 @@ using namespace lldb_private;
 bool lldb_private::formatters::GenericOptionalSummaryProvider(
     ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
   stream.Printf(" Has Value=%s ",
-                valobj.GetNumChildren() == 0 ? "false" : "true");
+                valobj.GetNumChildrenIgnoringErrors() == 0 ? "false" : "true");
 
   return true;
 }
@@ -41,7 +41,9 @@ public:
   }
 
   bool MightHaveChildren() override { return true; }
-  uint32_t CalculateNumChildren() override { return m_has_value ? 1U : 0U; }
+  llvm::Expected CalculateNumChildren() override {
+    return m_has_value ? 1U : 0U;
+  }
 
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
   lldb::ChildCacheState Update() override;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
index bba887fec3ac..10a1fe039189 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
@@ -63,7 +63,7 @@ lldb::ValueObjectSP
 lldb_private::formatters::GetSecondValueOfLibCXXCompressedPair(
     ValueObject &pair) {
   ValueObjectSP value;
-  if (pair.GetNumChildren() > 1) {
+  if (pair.GetNumChildrenIgnoringErrors() > 1) {
     ValueObjectSP second_child = pair.GetChildAtIndex(1);
     if (second_child) {
       value = second_child->GetChildMemberWithName("__value_");
@@ -351,8 +351,8 @@ lldb_private::formatters::LibCxxMapIteratorSyntheticFrontEnd::Update() {
   return lldb::ChildCacheState::eRefetch;
 }
 
-uint32_t lldb_private::formatters::LibCxxMapIteratorSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibCxxMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
   return 2;
 }
 
@@ -509,8 +509,8 @@ lldb::ChildCacheState lldb_private::formatters::
   return lldb::ChildCacheState::eRefetch;
 }
 
-uint32_t lldb_private::formatters::LibCxxUnorderedMapIteratorSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibCxxUnorderedMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
   return 2;
 }
 
@@ -566,8 +566,8 @@ lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::
     Update();
 }
 
-uint32_t lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxSharedPtrSyntheticFrontEnd::CalculateNumChildren() {
   return (m_cntrl ? 1 : 0);
 }
 
@@ -661,8 +661,8 @@ lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEndCreator(
                     : nullptr);
 }
 
-uint32_t lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxUniquePtrSyntheticFrontEnd::CalculateNumChildren() {
   if (m_value_ptr_sp)
     return m_deleter_sp ? 2 : 1;
   return 0;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
index ad2f58508ab7..a59f21841ec8 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
@@ -87,7 +87,7 @@ class LibCxxMapIteratorSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibCxxMapIteratorSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -135,7 +135,7 @@ public:
 
   ~LibCxxUnorderedMapIteratorSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -166,7 +166,7 @@ class LibcxxSharedPtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibcxxSharedPtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -186,7 +186,7 @@ class LibcxxUniquePtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibcxxUniquePtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp
index 8e4c36103a74..7f30dc186291 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp
@@ -90,7 +90,7 @@ public:
 
   ~LibcxxStdAtomicSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -124,8 +124,8 @@ bool lldb_private::formatters::LibcxxStdAtomicSyntheticFrontEnd::
   return true;
 }
 
-uint32_t lldb_private::formatters::LibcxxStdAtomicSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdAtomicSyntheticFrontEnd::CalculateNumChildren() {
   return m_real_child ? 1 : 0;
 }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp
index 00012dfc056e..bd9c72497664 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp
@@ -26,7 +26,7 @@ public:
 
   ~LibcxxInitializerListSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -59,8 +59,8 @@ lldb_private::formatters::LibcxxInitializerListSyntheticFrontEnd::
   // delete m_start;
 }
 
-uint32_t lldb_private::formatters::LibcxxInitializerListSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxInitializerListSyntheticFrontEnd::CalculateNumChildren() {
   m_num_elements = 0;
   ValueObjectSP size_sp(m_backend.GetChildMemberWithName("__size_"));
   if (size_sp)
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
index 17f6b737d9f6..d7cfeb30557c 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
@@ -136,7 +136,7 @@ class ForwardListFrontEnd : public AbstractListFrontEnd {
 public:
   ForwardListFrontEnd(ValueObject &valobj);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
   lldb::ChildCacheState Update() override;
 };
@@ -147,7 +147,7 @@ public:
 
   ~ListFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -240,7 +240,7 @@ ForwardListFrontEnd::ForwardListFrontEnd(ValueObject &valobj)
   Update();
 }
 
-uint32_t ForwardListFrontEnd::CalculateNumChildren() {
+llvm::Expected ForwardListFrontEnd::CalculateNumChildren() {
   if (m_count != UINT32_MAX)
     return m_count;
 
@@ -254,7 +254,7 @@ uint32_t ForwardListFrontEnd::CalculateNumChildren() {
 }
 
 ValueObjectSP ForwardListFrontEnd::GetChildAtIndex(uint32_t idx) {
-  if (idx >= CalculateNumChildren())
+  if (idx >= CalculateNumChildrenIgnoringErrors())
     return nullptr;
 
   if (!m_head)
@@ -308,7 +308,7 @@ ListFrontEnd::ListFrontEnd(lldb::ValueObjectSP valobj_sp)
     Update();
 }
 
-uint32_t ListFrontEnd::CalculateNumChildren() {
+llvm::Expected ListFrontEnd::CalculateNumChildren() {
   if (m_count != UINT32_MAX)
     return m_count;
   if (!m_head || !m_tail || m_node_address == 0)
@@ -347,7 +347,7 @@ lldb::ValueObjectSP ListFrontEnd::GetChildAtIndex(uint32_t idx) {
   static ConstString g_value("__value_");
   static ConstString g_next("__next_");
 
-  if (idx >= CalculateNumChildren())
+  if (idx >= CalculateNumChildrenIgnoringErrors())
     return lldb::ValueObjectSP();
 
   if (!m_head || !m_tail || m_node_address == 0)
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp
index 6d24eb03779c..ec5b320e2218 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp
@@ -177,7 +177,7 @@ public:
 
   ~LibcxxStdMapSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -209,8 +209,8 @@ lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::
     Update();
 }
 
-uint32_t lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdMapSyntheticFrontEnd::CalculateNumChildren() {
   if (m_count != UINT32_MAX)
     return m_count;
 
@@ -311,13 +311,13 @@ lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
   static ConstString g_cc_("__cc_"), g_cc("__cc");
   static ConstString g_nc("__nc");
-
-  if (idx >= CalculateNumChildren())
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
+  if (idx >= num_children)
     return lldb::ValueObjectSP();
   if (m_tree == nullptr || m_root_node == nullptr)
     return lldb::ValueObjectSP();
 
-  MapIterator iterator(m_root_node, CalculateNumChildren());
+  MapIterator iterator(m_root_node, num_children);
 
   const bool need_to_skip = (idx > 0);
   size_t actual_advancde = idx;
@@ -382,7 +382,7 @@ lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::GetChildAtIndex(
   name.Printf("[%" PRIu64 "]", (uint64_t)idx);
   auto potential_child_sp = iterated_sp->Clone(ConstString(name.GetString()));
   if (potential_child_sp) {
-    switch (potential_child_sp->GetNumChildren()) {
+    switch (potential_child_sp->GetNumChildrenIgnoringErrors()) {
     case 1: {
       auto child0_sp = potential_child_sp->GetChildAtIndex(0);
       if (child0_sp &&
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp
index fbadee89b7b7..5b459a17fe29 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp
@@ -28,7 +28,7 @@ public:
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
 
-  uint32_t CalculateNumChildren() override {
+  llvm::Expected CalculateNumChildren() override {
     return m_container_sp ? m_container_sp->GetNumChildren() : 0;
   }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp
index 74f54f767356..01a7b8f142ec 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp
@@ -27,7 +27,7 @@ public:
 
   ~LibcxxStdRangesRefViewSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override {
+  llvm::Expected CalculateNumChildren() override {
     // __range_ will be the sole child of this type
     return 1;
   }
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp
index af2b51d2b540..9895f336bfd0 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp
@@ -27,7 +27,7 @@ public:
 
   ~LibcxxStdSpanSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -73,8 +73,8 @@ lldb_private::formatters::LibcxxStdSpanSyntheticFrontEnd::
     Update();
 }
 
-uint32_t lldb_private::formatters::LibcxxStdSpanSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdSpanSyntheticFrontEnd::CalculateNumChildren() {
   return m_num_elements;
 }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp
index 62bb7d619267..3e3259ab428d 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp
@@ -26,7 +26,9 @@ public:
 
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
-  uint32_t CalculateNumChildren() override { return m_elements.size(); }
+  llvm::Expected CalculateNumChildren() override {
+    return m_elements.size();
+  }
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
 private:
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
index b3c364294335..af29fdb6d001 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
@@ -33,7 +33,7 @@ public:
 
   ~LibcxxStdUnorderedMapSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -62,8 +62,8 @@ lldb_private::formatters::LibcxxStdUnorderedMapSyntheticFrontEnd::
     Update();
 }
 
-uint32_t lldb_private::formatters::LibcxxStdUnorderedMapSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdUnorderedMapSyntheticFrontEnd::CalculateNumChildren() {
   return m_num_elements;
 }
 
@@ -94,7 +94,7 @@ static bool isUnorderedMap(ConstString type_name) {
 
 lldb::ValueObjectSP lldb_private::formatters::
     LibcxxStdUnorderedMapSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
-  if (idx >= CalculateNumChildren())
+  if (idx >= CalculateNumChildrenIgnoringErrors())
     return lldb::ValueObjectSP();
   if (m_tree == nullptr)
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp
index 463c7b8d7ce3..99f94406e99a 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp
@@ -24,7 +24,7 @@ public:
 
   ~LibcxxStdValarraySyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -63,8 +63,8 @@ lldb_private::formatters::LibcxxStdValarraySyntheticFrontEnd::
   // delete m_finish;
 }
 
-uint32_t lldb_private::formatters::LibcxxStdValarraySyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdValarraySyntheticFrontEnd::CalculateNumChildren() {
   if (!m_start || !m_finish)
     return 0;
   uint64_t start_val = m_start->GetValueAsUnsigned(0);
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp
index 1f62062f09be..62794318e077 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp
@@ -205,7 +205,7 @@ public:
 
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
-  uint32_t CalculateNumChildren() override { return m_size; }
+  llvm::Expected CalculateNumChildren() override { return m_size; }
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
 private:
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
index fcf727ad2ea0..461fed35164b 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
@@ -25,7 +25,7 @@ public:
 
   ~LibcxxStdVectorSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -46,7 +46,7 @@ class LibcxxVectorBoolSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibcxxVectorBoolSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -82,8 +82,8 @@ lldb_private::formatters::LibcxxStdVectorSyntheticFrontEnd::
   // delete m_finish;
 }
 
-uint32_t lldb_private::formatters::LibcxxStdVectorSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdVectorSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_start || !m_finish)
     return 0;
   uint64_t start_val = m_start->GetValueAsUnsigned(0);
@@ -165,8 +165,8 @@ lldb_private::formatters::LibcxxVectorBoolSyntheticFrontEnd::
   }
 }
 
-uint32_t lldb_private::formatters::LibcxxVectorBoolSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxVectorBoolSyntheticFrontEnd::CalculateNumChildren() {
   return m_count;
 }
 
@@ -259,7 +259,7 @@ size_t lldb_private::formatters::LibcxxVectorBoolSyntheticFrontEnd::
     return UINT32_MAX;
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
index 5abb3d50674b..86bb575af5ca 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
@@ -43,7 +43,7 @@ class LibstdcppMapIteratorSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibstdcppMapIteratorSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -64,7 +64,7 @@ class LibStdcppSharedPtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibStdcppSharedPtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -132,7 +132,8 @@ lldb::ChildCacheState LibstdcppMapIteratorSyntheticFrontEnd::Update() {
   return lldb::ChildCacheState::eReuse;
 }
 
-uint32_t LibstdcppMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected
+LibstdcppMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
   return 2;
 }
 
@@ -219,7 +220,10 @@ lldb::ChildCacheState VectorIteratorSyntheticFrontEnd::Update() {
   return lldb::ChildCacheState::eRefetch;
 }
 
-uint32_t VectorIteratorSyntheticFrontEnd::CalculateNumChildren() { return 1; }
+llvm::Expected
+VectorIteratorSyntheticFrontEnd::CalculateNumChildren() {
+  return 1;
+}
 
 lldb::ValueObjectSP
 VectorIteratorSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
@@ -371,7 +375,10 @@ LibStdcppSharedPtrSyntheticFrontEnd::LibStdcppSharedPtrSyntheticFrontEnd(
     Update();
 }
 
-uint32_t LibStdcppSharedPtrSyntheticFrontEnd::CalculateNumChildren() { return 1; }
+llvm::Expected
+LibStdcppSharedPtrSyntheticFrontEnd::CalculateNumChildren() {
+  return 1;
+}
 
 lldb::ValueObjectSP
 LibStdcppSharedPtrSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp
index 64d2ec9d943a..05199ba35b9a 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp
@@ -26,7 +26,7 @@ class LibStdcppTupleSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibStdcppTupleSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -65,7 +65,7 @@ lldb::ChildCacheState LibStdcppTupleSyntheticFrontEnd::Update() {
     ValueObjectSP current_child = next_child_sp;
     next_child_sp = nullptr;
 
-    size_t child_count = current_child->GetNumChildren();
+    size_t child_count = current_child->GetNumChildrenIgnoringErrors();
     for (size_t i = 0; i < child_count; ++i) {
       ValueObjectSP child_sp = current_child->GetChildAtIndex(i);
       llvm::StringRef name_str = child_sp->GetName().GetStringRef();
@@ -95,7 +95,8 @@ LibStdcppTupleSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
   return lldb::ValueObjectSP();
 }
 
-uint32_t LibStdcppTupleSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected
+LibStdcppTupleSyntheticFrontEnd::CalculateNumChildren() {
   return m_members.size();
 }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
index 3a48fe412e07..92f540d9ca52 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
@@ -26,7 +26,7 @@ class LibStdcppUniquePtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibStdcppUniquePtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -135,7 +135,8 @@ LibStdcppUniquePtrSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
   return lldb::ValueObjectSP();
 }
 
-uint32_t LibStdcppUniquePtrSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected
+LibStdcppUniquePtrSyntheticFrontEnd::CalculateNumChildren() {
   if (m_del_obj)
     return 2;
   return 1;
diff --git a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
index cb740f8e71e1..96166657ceeb 100644
--- a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
+++ b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
@@ -1038,7 +1038,7 @@ public:
 
   ~ObjCClassSyntheticChildrenFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override { return 0; }
+  llvm::Expected CalculateNumChildren() override { return 0; }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/Language/ObjC/NSArray.cpp b/lldb/source/Plugins/Language/ObjC/NSArray.cpp
index 7f060b2613d6..67d0cd08f51a 100644
--- a/lldb/source/Plugins/Language/ObjC/NSArray.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSArray.cpp
@@ -50,7 +50,7 @@ public:
 
   ~NSArrayMSyntheticFrontEndBase() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -214,7 +214,7 @@ public:
 
   ~GenericNSArrayISyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -302,7 +302,7 @@ public:
 
   ~NSArray0SyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -319,7 +319,7 @@ public:
 
   ~NSArray1SyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -477,15 +477,15 @@ lldb_private::formatters::
     : NSArrayMSyntheticFrontEndBase(valobj_sp), m_data_32(nullptr),
       m_data_64(nullptr) {}
 
-uint32_t lldb_private::formatters::NSArrayMSyntheticFrontEndBase::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    NSArrayMSyntheticFrontEndBase::CalculateNumChildren() {
   return GetUsedCount();
 }
 
 lldb::ValueObjectSP
 lldb_private::formatters::NSArrayMSyntheticFrontEndBase::GetChildAtIndex(
     uint32_t idx) {
-  if (idx >= CalculateNumChildren())
+  if (idx >= CalculateNumChildrenIgnoringErrors())
     return lldb::ValueObjectSP();
   lldb::addr_t object_at_idx = GetDataAddress();
   size_t pyhs_idx = idx;
@@ -542,7 +542,7 @@ lldb_private::formatters::NSArrayMSyntheticFrontEndBase::GetIndexOfChildWithName
     ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
@@ -628,15 +628,15 @@ lldb_private::formatters::GenericNSArrayISyntheticFrontEnd::
   GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
 template 
-uint32_t
-lldb_private::formatters::GenericNSArrayISyntheticFrontEnd::
-  CalculateNumChildren() {
+llvm::Expected
+lldb_private::formatters::GenericNSArrayISyntheticFrontEnd<
+    D32, D64, Inline>::CalculateNumChildren() {
   return m_data_32 ? m_data_32->used : m_data_64->used;
 }
 
@@ -685,7 +685,7 @@ template 
 lldb::ValueObjectSP
 lldb_private::formatters::GenericNSArrayISyntheticFrontEnd::
   GetChildAtIndex(uint32_t idx) {
-  if (idx >= CalculateNumChildren())
+  if (idx >= CalculateNumChildrenIgnoringErrors())
     return lldb::ValueObjectSP();
   lldb::addr_t object_at_idx;
   if (Inline) {
@@ -719,7 +719,7 @@ lldb_private::formatters::NSArray0SyntheticFrontEnd::GetIndexOfChildWithName(
   return UINT32_MAX;
 }
 
-uint32_t
+llvm::Expected
 lldb_private::formatters::NSArray0SyntheticFrontEnd::CalculateNumChildren() {
   return 0;
 }
@@ -754,7 +754,7 @@ lldb_private::formatters::NSArray1SyntheticFrontEnd::GetIndexOfChildWithName(
   return UINT32_MAX;
 }
 
-uint32_t
+llvm::Expected
 lldb_private::formatters::NSArray1SyntheticFrontEnd::CalculateNumChildren() {
   return 1;
 }
diff --git a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
index da94eda1529c..ec6fd756394a 100644
--- a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
@@ -103,7 +103,7 @@ public:
 
   ~NSDictionaryISyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -144,7 +144,7 @@ class NSConstantDictionarySyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   NSConstantDictionarySyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -176,7 +176,7 @@ class NSCFDictionarySyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   NSCFDictionarySyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -209,7 +209,7 @@ public:
 
   ~NSDictionary1SyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -230,7 +230,7 @@ public:
 
   ~GenericNSDictionaryMSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -262,9 +262,9 @@ namespace Foundation1100 {
     NSDictionaryMSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
     
     ~NSDictionaryMSyntheticFrontEnd() override;
-    
-    uint32_t CalculateNumChildren() override;
-    
+
+    llvm::Expected CalculateNumChildren() override;
+
     lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
     lldb::ChildCacheState Update() override;
@@ -601,13 +601,13 @@ size_t lldb_private::formatters::NSDictionaryISyntheticFrontEnd::
     GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t lldb_private::formatters::NSDictionaryISyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    NSDictionaryISyntheticFrontEnd::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
   return (m_data_32 ? m_data_32->_used : m_data_64->_used);
@@ -656,7 +656,7 @@ bool lldb_private::formatters::NSDictionaryISyntheticFrontEnd::
 lldb::ValueObjectSP
 lldb_private::formatters::NSDictionaryISyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -739,13 +739,13 @@ size_t lldb_private::formatters::NSCFDictionarySyntheticFrontEnd::
     GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   const uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t lldb_private::formatters::NSCFDictionarySyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    NSCFDictionarySyntheticFrontEnd::CalculateNumChildren() {
   if (!m_hashtable.IsValid())
     return 0;
   return m_hashtable.GetCount();
@@ -781,7 +781,7 @@ lldb_private::formatters::NSCFDictionarySyntheticFrontEnd::GetChildAtIndex(
   lldb::addr_t m_keys_ptr = m_hashtable.GetKeyPointer();
   lldb::addr_t m_values_ptr = m_hashtable.GetValuePointer();
 
-  const uint32_t num_children = CalculateNumChildren();
+  const uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -875,13 +875,13 @@ size_t lldb_private::formatters::NSConstantDictionarySyntheticFrontEnd::
     GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t lldb_private::formatters::NSConstantDictionarySyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    NSConstantDictionarySyntheticFrontEnd::CalculateNumChildren() {
   return m_size;
 }
 
@@ -921,7 +921,7 @@ bool lldb_private::formatters::NSConstantDictionarySyntheticFrontEnd::
 
 lldb::ValueObjectSP lldb_private::formatters::
     NSConstantDictionarySyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -994,8 +994,8 @@ size_t lldb_private::formatters::NSDictionary1SyntheticFrontEnd::
   return name == g_zero ? 0 : UINT32_MAX;
 }
 
-uint32_t lldb_private::formatters::NSDictionary1SyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    NSDictionary1SyntheticFrontEnd::CalculateNumChildren() {
   return 1;
 }
 
@@ -1081,17 +1081,18 @@ size_t lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd<
     D32, D64>::GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
 template 
-uint32_t
-lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected
+lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd<
+    D32, D64>::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
-  return (m_data_32 ? m_data_32->_used : m_data_64->_used);
+  return (m_data_32 ? (uint32_t)m_data_32->_used : (uint32_t)m_data_64->_used);
 }
 
 template 
@@ -1153,7 +1154,7 @@ lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd<
     m_values_ptr = m_data_64->_buffer + (m_ptr_size * size);
   }
 
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -1245,14 +1246,13 @@ lldb_private::formatters::Foundation1100::
   NSDictionaryMSyntheticFrontEnd::GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t
-lldb_private::formatters::Foundation1100::
-  NSDictionaryMSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::Foundation1100::
+    NSDictionaryMSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
   return (m_data_32 ? m_data_32->_used : m_data_64->_used);
@@ -1306,7 +1306,7 @@ lldb_private::formatters::Foundation1100::
   lldb::addr_t m_values_ptr =
       (m_data_32 ? m_data_32->_objs_addr : m_data_64->_objs_addr);
 
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/Language/ObjC/NSError.cpp b/lldb/source/Plugins/Language/ObjC/NSError.cpp
index b034e799b716..5ef7edc7e80c 100644
--- a/lldb/source/Plugins/Language/ObjC/NSError.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSError.cpp
@@ -116,7 +116,7 @@ public:
   // no need to delete m_child_ptr - it's kept alive by the cluster manager on
   // our behalf
 
-  uint32_t CalculateNumChildren() override {
+  llvm::Expected CalculateNumChildren() override {
     if (m_child_ptr)
       return 1;
     if (m_child_sp)
diff --git a/lldb/source/Plugins/Language/ObjC/NSException.cpp b/lldb/source/Plugins/Language/ObjC/NSException.cpp
index 09d3a1b42b74..e7ce26ea4c6f 100644
--- a/lldb/source/Plugins/Language/ObjC/NSException.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSException.cpp
@@ -123,9 +123,7 @@ public:
 
   ~NSExceptionSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override {
-    return 4;
-  }
+  llvm::Expected CalculateNumChildren() override { return 4; }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     switch (idx) {
diff --git a/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp b/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
index 10bb907c58ed..a434cee09d38 100644
--- a/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
@@ -40,7 +40,9 @@ public:
 
   ~NSIndexPathSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override { return m_impl.GetNumIndexes(); }
+  llvm::Expected CalculateNumChildren() override {
+    return m_impl.GetNumIndexes();
+  }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     return m_impl.GetIndexAtIndex(idx, m_uint_star_type);
@@ -127,7 +129,7 @@ public:
   size_t GetIndexOfChildWithName(ConstString name) override {
     const char *item_name = name.GetCString();
     uint32_t idx = ExtractIndexFromString(item_name);
-    if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+    if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
       return UINT32_MAX;
     return idx;
   }
diff --git a/lldb/source/Plugins/Language/ObjC/NSSet.cpp b/lldb/source/Plugins/Language/ObjC/NSSet.cpp
index c965a2a13400..7d0a6a507211 100644
--- a/lldb/source/Plugins/Language/ObjC/NSSet.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSSet.cpp
@@ -46,7 +46,7 @@ public:
 
   ~NSSetISyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -84,7 +84,7 @@ class NSCFSetSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   NSCFSetSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -117,7 +117,7 @@ public:
 
   ~GenericNSSetMSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -233,7 +233,7 @@ public:
 
   ~NSSetCodeRunningSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -414,12 +414,12 @@ lldb_private::formatters::NSSetISyntheticFrontEnd::GetIndexOfChildWithName(
     ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t
+llvm::Expected
 lldb_private::formatters::NSSetISyntheticFrontEnd::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
@@ -468,7 +468,7 @@ bool lldb_private::formatters::NSSetISyntheticFrontEnd::MightHaveChildren() {
 lldb::ValueObjectSP
 lldb_private::formatters::NSSetISyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -551,12 +551,12 @@ lldb_private::formatters::NSCFSetSyntheticFrontEnd::GetIndexOfChildWithName(
     ConstString name) {
   const char *item_name = name.GetCString();
   const uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t
+llvm::Expected
 lldb_private::formatters::NSCFSetSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_hashtable.IsValid())
     return 0;
@@ -591,7 +591,7 @@ lldb_private::formatters::NSCFSetSyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
   lldb::addr_t m_values_ptr = m_hashtable.GetValuePointer();
 
-  const uint32_t num_children = CalculateNumChildren();
+  const uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -691,18 +691,18 @@ lldb_private::formatters::
     ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
 template 
-uint32_t
-lldb_private::formatters::
-  GenericNSSetMSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected
+lldb_private::formatters::GenericNSSetMSyntheticFrontEnd<
+    D32, D64>::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
-  return (m_data_32 ? m_data_32->_used : m_data_64->_used);
+  return (m_data_32 ? (uint32_t)m_data_32->_used : (uint32_t)m_data_64->_used);
 }
 
 template 
@@ -753,7 +753,7 @@ lldb_private::formatters::
   lldb::addr_t m_objs_addr =
       (m_data_32 ? m_data_32->_objs_addr : m_data_64->_objs_addr);
 
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
index f08f9f0f815d..9434376f7d9e 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
@@ -539,7 +539,8 @@ ThreadSP AppleObjCRuntime::GetBacktraceThreadFromException(
     return object;
   };
 
-  for (size_t idx = 0; idx < reserved_dict->GetNumChildren(); idx++) {
+  for (size_t idx = 0; idx < reserved_dict->GetNumChildrenIgnoringErrors();
+       idx++) {
     ValueObjectSP dict_entry = reserved_dict->GetChildAtIndex(idx);
 
     DataExtractor data;
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index 51ab13108feb..c02b08cb4782 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -5263,9 +5263,10 @@ GetDynamicArrayInfo(TypeSystemClang &ast, SymbolFile *sym_file,
   return std::nullopt;
 }
 
-uint32_t TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
-                                         bool omit_empty_base_classes,
-                                         const ExecutionContext *exe_ctx) {
+llvm::Expected
+TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
+                                bool omit_empty_base_classes,
+                                const ExecutionContext *exe_ctx) {
   if (!type)
     return 0;
 
@@ -5361,9 +5362,13 @@ uint32_t TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
     CompilerType pointee_clang_type(GetPointeeType(type));
 
     uint32_t num_pointee_children = 0;
-    if (pointee_clang_type.IsAggregateType())
-      num_pointee_children =
+    if (pointee_clang_type.IsAggregateType()) {
+      auto num_children_or_err =
           pointee_clang_type.GetNumChildren(omit_empty_base_classes, exe_ctx);
+      if (!num_children_or_err)
+        return num_children_or_err;
+      num_pointee_children = *num_children_or_err;
+    }
     // If this type points to a simple type, then it has 1 child
     if (num_pointee_children == 0)
       num_children = 1;
@@ -5397,9 +5402,13 @@ uint32_t TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
     clang::QualType pointee_type(pointer_type->getPointeeType());
     CompilerType pointee_clang_type(GetType(pointee_type));
     uint32_t num_pointee_children = 0;
-    if (pointee_clang_type.IsAggregateType())
-      num_pointee_children =
+    if (pointee_clang_type.IsAggregateType()) {
+      auto num_children_or_err =
           pointee_clang_type.GetNumChildren(omit_empty_base_classes, exe_ctx);
+      if (!num_children_or_err)
+        return num_children_or_err;
+      num_pointee_children = *num_children_or_err;
+    }
     if (num_pointee_children == 0) {
       // We have a pointer to a pointee type that claims it has no children. We
       // will want to look at
@@ -6108,8 +6117,15 @@ CompilerType TypeSystemClang::GetChildCompilerTypeAtIndex(
   child_is_base_class = false;
   language_flags = 0;
 
-  const bool idx_is_valid =
-      idx < GetNumChildren(type, omit_empty_base_classes, exe_ctx);
+  auto num_children_or_err =
+      GetNumChildren(type, omit_empty_base_classes, exe_ctx);
+  if (!num_children_or_err) {
+    LLDB_LOG_ERRORV(GetLog(LLDBLog::Types), num_children_or_err.takeError(),
+                    "{0}");
+    return {};
+  }
+
+  const bool idx_is_valid = idx < *num_children_or_err;
   int32_t bit_offset;
   switch (parent_type_class) {
   case clang::Type::Builtin:
@@ -6265,8 +6281,10 @@ CompilerType TypeSystemClang::GetChildCompilerTypeAtIndex(
               CompilerType base_class_clang_type =
                   GetType(getASTContext().getObjCInterfaceType(
                       superclass_interface_decl));
-              if (base_class_clang_type.GetNumChildren(omit_empty_base_classes,
-                                                       exe_ctx) > 0) {
+              if (llvm::expectedToStdOptional(
+                      base_class_clang_type.GetNumChildren(
+                          omit_empty_base_classes, exe_ctx))
+                      .value_or(0) > 0) {
                 if (idx == 0) {
                   clang::QualType ivar_qual_type(
                       getASTContext().getObjCInterfaceType(
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
index a73164895baa..05c303baa416 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
@@ -833,9 +833,10 @@ public:
   GetTypeBitAlign(lldb::opaque_compiler_type_t type,
                   ExecutionContextScope *exe_scope) override;
 
-  uint32_t GetNumChildren(lldb::opaque_compiler_type_t type,
-                          bool omit_empty_base_classes,
-                          const ExecutionContext *exe_ctx) override;
+  llvm::Expected
+  GetNumChildren(lldb::opaque_compiler_type_t type,
+                 bool omit_empty_base_classes,
+                 const ExecutionContext *exe_ctx) override;
 
   CompilerType GetBuiltinTypeByName(ConstString name) override;
 
diff --git a/lldb/source/Symbol/CompilerType.cpp b/lldb/source/Symbol/CompilerType.cpp
index 76b79daa6ac1..85dd2d841a5a 100644
--- a/lldb/source/Symbol/CompilerType.cpp
+++ b/lldb/source/Symbol/CompilerType.cpp
@@ -770,8 +770,9 @@ lldb::Format CompilerType::GetFormat() const {
   return lldb::eFormatDefault;
 }
 
-uint32_t CompilerType::GetNumChildren(bool omit_empty_base_classes,
-                                      const ExecutionContext *exe_ctx) const {
+llvm::Expected
+CompilerType::GetNumChildren(bool omit_empty_base_classes,
+                             const ExecutionContext *exe_ctx) const {
   if (IsValid())
     if (auto type_system_sp = GetTypeSystem())
       return type_system_sp->GetNumChildren(m_type, omit_empty_base_classes,
diff --git a/lldb/source/Symbol/Type.cpp b/lldb/source/Symbol/Type.cpp
index 6069d066eaf6..44a24d7178f5 100644
--- a/lldb/source/Symbol/Type.cpp
+++ b/lldb/source/Symbol/Type.cpp
@@ -488,7 +488,7 @@ std::optional Type::GetByteSize(ExecutionContextScope *exe_scope) {
   return {};
 }
 
-uint32_t Type::GetNumChildren(bool omit_empty_base_classes) {
+llvm::Expected Type::GetNumChildren(bool omit_empty_base_classes) {
   return GetForwardCompilerType().GetNumChildren(omit_empty_base_classes, nullptr);
 }
 
diff --git a/lldb/source/Symbol/Variable.cpp b/lldb/source/Symbol/Variable.cpp
index a33c3433d9e2..90edede0f065 100644
--- a/lldb/source/Symbol/Variable.cpp
+++ b/lldb/source/Symbol/Variable.cpp
@@ -27,6 +27,8 @@
 #include "lldb/Target/StackFrame.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Target/Thread.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegularExpression.h"
 #include "lldb/Utility/Stream.h"
 
@@ -568,7 +570,9 @@ static void PrivateAutoComplete(
       case eTypeClassObjCObjectPointer:
       case eTypeClassPointer: {
         bool omit_empty_base_classes = true;
-        if (compiler_type.GetNumChildren(omit_empty_base_classes, nullptr) > 0)
+        if (llvm::expectedToStdOptional(
+                compiler_type.GetNumChildren(omit_empty_base_classes, nullptr))
+                .value_or(0))
           request.AddCompletion((prefix_path + "->").str());
         else {
           request.AddCompletion(prefix_path.str());
diff --git a/lldb/source/Target/StackFrame.cpp b/lldb/source/Target/StackFrame.cpp
index 50cf01e63cd4..c29a71d92572 100644
--- a/lldb/source/Target/StackFrame.cpp
+++ b/lldb/source/Target/StackFrame.cpp
@@ -857,10 +857,11 @@ ValueObjectSP StackFrame::GetValueForVariableExpressionPath(
                   "\"(%s) %s\" is not an array type",
                   valobj_sp->GetTypeName().AsCString(""),
                   var_expr_path_strm.GetData());
-            } else if (
-                static_cast(child_index) >=
-                synthetic
-                    ->GetNumChildren() /* synthetic does not have that many values */) {
+            } else if (static_cast(child_index) >=
+                       synthetic
+                           ->GetNumChildrenIgnoringErrors() /* synthetic does
+                                                                not have that
+                                                                many values */) {
               valobj_sp->GetExpressionPath(var_expr_path_strm);
               error.SetErrorStringWithFormat(
                   "array index %ld is not valid for \"(%s) %s\"", child_index,
@@ -929,10 +930,9 @@ ValueObjectSP StackFrame::GetValueForVariableExpressionPath(
                 "\"(%s) %s\" is not an array type",
                 valobj_sp->GetTypeName().AsCString(""),
                 var_expr_path_strm.GetData());
-          } else if (
-              static_cast(child_index) >=
-              synthetic
-                  ->GetNumChildren() /* synthetic does not have that many values */) {
+          } else if (static_cast(child_index) >=
+                     synthetic->GetNumChildrenIgnoringErrors() /* synthetic
+                                     does not have that many values */) {
             valobj_sp->GetExpressionPath(var_expr_path_strm);
             error.SetErrorStringWithFormat(
                 "array index %ld is not valid for \"(%s) %s\"", child_index,
@@ -1397,7 +1397,8 @@ ValueObjectSP GetValueForOffset(StackFrame &frame, ValueObjectSP &parent,
     return parent;
   }
 
-  for (int ci = 0, ce = parent->GetNumChildren(); ci != ce; ++ci) {
+  for (int ci = 0, ce = parent->GetNumChildrenIgnoringErrors(); ci != ce;
+       ++ci) {
     ValueObjectSP child_sp = parent->GetChildAtIndex(ci);
 
     if (!child_sp) {
-- 
GitLab


From 4d323e404d43526ad8263769f00aace9db2e57c5 Mon Sep 17 00:00:00 2001
From: Nikolas Klauser 
Date: Sat, 9 Mar 2024 01:09:28 +0100
Subject: [PATCH 672/929] [libc++] Allow the use of extensions in the
 implementation (#79532)

We've talked about allowing extensions on
[discourse](https://discourse.llvm.org/t/rfc-use-language-extensions-from-future-standards-in-libc/71898/5)
and in a libc++ monthly meeting and agreed to test it out in the LLVM 18
release. We've done that with the `tuple` constructor overload set
(using conditional `explicit`). Since we haven't heard about any
breakages, it seems safe to do. This patch enables the use of extension
from later C++ standards inside the versioned `std` namespaces. This
should be good enough, since almost all of our code is inside that
namespace. This approach also avoids the use of extensions inside the
test `std` suite. That part of the code base should stay clean, since
it's a test suite that is also used by other vendors to test their
implementations.
---
 libcxx/include/__config | 61 +++++++++++++++++++++++++++--------------
 libcxx/include/tuple    |  6 ----
 2 files changed, 41 insertions(+), 26 deletions(-)

diff --git a/libcxx/include/__config b/libcxx/include/__config
index 3a438e85a7b8..11e13e0c2498 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -406,6 +406,10 @@ _LIBCPP_HARDENING_MODE_DEBUG
 #    define __has_include(...) 0
 #  endif
 
+#  ifndef __has_warning
+#    define __has_warning(...) 0
+#  endif
+
 #  if !defined(_LIBCPP_COMPILER_CLANG_BASED) && __cplusplus < 201103L
 #    error "libc++ only supports C++03 with Clang-based compilers. Please enable C++11"
 #  endif
@@ -723,6 +727,23 @@ typedef __char32_t char32_t;
 #    define _LIBCPP_EXCLUDE_FROM_EXPLICIT_INSTANTIATION _LIBCPP_ALWAYS_INLINE
 #  endif
 
+#  ifdef _LIBCPP_COMPILER_CLANG_BASED
+#    define _LIBCPP_DIAGNOSTIC_PUSH _Pragma("clang diagnostic push")
+#    define _LIBCPP_DIAGNOSTIC_POP _Pragma("clang diagnostic pop")
+#    define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED(str) _Pragma(_LIBCPP_TOSTRING(clang diagnostic ignored str))
+#    define _LIBCPP_GCC_DIAGNOSTIC_IGNORED(str)
+#  elif defined(_LIBCPP_COMPILER_GCC)
+#    define _LIBCPP_DIAGNOSTIC_PUSH _Pragma("GCC diagnostic push")
+#    define _LIBCPP_DIAGNOSTIC_POP _Pragma("GCC diagnostic pop")
+#    define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED(str)
+#    define _LIBCPP_GCC_DIAGNOSTIC_IGNORED(str) _Pragma(_LIBCPP_TOSTRING(GCC diagnostic ignored str))
+#  else
+#    define _LIBCPP_DIAGNOSTIC_PUSH
+#    define _LIBCPP_DIAGNOSTIC_POP
+#    define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED(str)
+#    define _LIBCPP_GCC_DIAGNOSTIC_IGNORED(str)
+#  endif
+
 #  if _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_FAST
 #    define _LIBCPP_HARDENING_SIG f
 #  elif _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_EXTENSIVE
@@ -810,16 +831,33 @@ typedef __char32_t char32_t;
 #    define _LIBCPP_HIDE_FROM_ABI_AFTER_V1 _LIBCPP_HIDE_FROM_ABI
 #  endif
 
+// TODO: Remove this workaround once we drop support for Clang 16
+#if __has_warning("-Wc++23-extensions")
+#  define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++23-extensions")
+#else
+#  define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++2b-extensions")
+#endif
+
 // Inline namespaces are available in Clang/GCC/MSVC regardless of C++ dialect.
 // clang-format off
-#  define _LIBCPP_BEGIN_NAMESPACE_STD namespace _LIBCPP_TYPE_VISIBILITY_DEFAULT std {                                  \
+#  define _LIBCPP_BEGIN_NAMESPACE_STD _LIBCPP_DIAGNOSTIC_PUSH                                                          \
+                                      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++11-extensions")                           \
+                                      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++14-extensions")                           \
+                                      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++17-extensions")                           \
+                                      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++20-extensions")                           \
+                                      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION                                 \
+                                      _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++14-extensions")                             \
+                                      _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++17-extensions")                             \
+                                      _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++20-extensions")                             \
+                                      _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++23-extensions")                             \
+                                      namespace _LIBCPP_TYPE_VISIBILITY_DEFAULT std {                                  \
                                inline namespace _LIBCPP_ABI_NAMESPACE {
-#  define _LIBCPP_END_NAMESPACE_STD }}
+#  define _LIBCPP_END_NAMESPACE_STD }} _LIBCPP_DIAGNOSTIC_POP
 
 #  define _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM _LIBCPP_BEGIN_NAMESPACE_STD                                               \
                                              inline namespace __fs { namespace filesystem {
 
-#  define _LIBCPP_END_NAMESPACE_FILESYSTEM _LIBCPP_END_NAMESPACE_STD }}
+#  define _LIBCPP_END_NAMESPACE_FILESYSTEM }} _LIBCPP_END_NAMESPACE_STD
 // clang-format on
 
 #  if __has_attribute(__enable_if__)
@@ -1256,23 +1294,6 @@ __sanitizer_verify_double_ended_contiguous_container(const void*, const void*, c
 // the ABI inconsistent.
 #  endif
 
-#  ifdef _LIBCPP_COMPILER_CLANG_BASED
-#    define _LIBCPP_DIAGNOSTIC_PUSH _Pragma("clang diagnostic push")
-#    define _LIBCPP_DIAGNOSTIC_POP _Pragma("clang diagnostic pop")
-#    define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED(str) _Pragma(_LIBCPP_TOSTRING(clang diagnostic ignored str))
-#    define _LIBCPP_GCC_DIAGNOSTIC_IGNORED(str)
-#  elif defined(_LIBCPP_COMPILER_GCC)
-#    define _LIBCPP_DIAGNOSTIC_PUSH _Pragma("GCC diagnostic push")
-#    define _LIBCPP_DIAGNOSTIC_POP _Pragma("GCC diagnostic pop")
-#    define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED(str)
-#    define _LIBCPP_GCC_DIAGNOSTIC_IGNORED(str) _Pragma(_LIBCPP_TOSTRING(GCC diagnostic ignored str))
-#  else
-#    define _LIBCPP_DIAGNOSTIC_PUSH
-#    define _LIBCPP_DIAGNOSTIC_POP
-#    define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED(str)
-#    define _LIBCPP_GCC_DIAGNOSTIC_IGNORED(str)
-#  endif
-
 // c8rtomb() and mbrtoc8() were added in C++20 and C23. Support for these
 // functions is gradually being added to existing C libraries. The conditions
 // below check for known C library versions and conditions under which these
diff --git a/libcxx/include/tuple b/libcxx/include/tuple
index 0101d64aea4a..8808db6739fb 100644
--- a/libcxx/include/tuple
+++ b/libcxx/include/tuple
@@ -548,10 +548,6 @@ class _LIBCPP_TEMPLATE_VIS tuple {
 public:
   // [tuple.cnstr]
 
-  _LIBCPP_DIAGNOSTIC_PUSH
-  _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++20-extensions")
-  _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++20-extensions")
-
   // tuple() constructors (including allocator_arg_t variants)
   template